From 9df57442f7cbdac38609c17c39f5fc142bdc5be6 Mon Sep 17 00:00:00 2001 From: Gian Date: Thu, 5 Nov 2026 12:37:50 +0800 Subject: [PATCH 01/17] Consolidate tree flush and snapshot changes --- .agents/skills/coding/SKILL.md | 69 - .agents/skills/console-ui-e2e/SKILL.md | 69 - .../console-ui-e2e/references/diagnosis.md | 26 - .agents/skills/debug-test/SKILL.md | 70 - .agents/skills/doc-backlog/SKILL.md | 42 - .agents/skills/doc-design/SKILL.md | 35 - .agents/skills/doc-working-design/SKILL.md | 39 - .agents/skills/doc-working-plan/SKILL.md | 33 - .agents/skills/doc/SKILL.md | 34 - .agents/skills/implement-requirement/SKILL.md | 39 - .agents/skills/review/SKILL.md | 41 - .clang-tidy | 5 +- AGENTS.md | 20 +- CONTRIBUTING.md | 6 +- Cargo.lock | 145 +- Cargo.toml | 11 +- README.md | 1 - app/crowdb-access-server/Cargo.toml | 4 +- .../conf/crowdb_access_server_config.toml | 7 +- .../crowdb_access_server_config.toml.ports.in | 69 + app/crowdb-access-server/src/credentials.rs | 6 +- .../src/credentials/ensure.rs | 2 +- app/crowdb-access-server/src/dataset.rs | 8 +- .../src/iceberg/runtime.rs | 2 +- app/crowdb-access-server/src/main.rs | 6 +- app/crowdb-access-server/tests/README.md | 4 +- .../tests/common/access_probe_budget.rs | 2 +- .../tests/common/credentials.rs | 10 +- .../tests/credential_authority_test.rs | 2 +- app/crowdb-chunk-kv-server/Cargo.toml | 5 +- .../conf/crowdb_chunk_kv_server_config.toml | 292 ++ app/crowdb-chunk-kv-server/src/config.rs | 51 +- .../src/control_store.rs | 8 +- .../src/host_lifecycle.rs | 96 + app/crowdb-chunk-kv-server/src/main.rs | 110 +- app/crowdb-chunk-kv-server/src/management.rs | 2 +- app/crowdb-chunk-kv-server/src/metrics.rs | 11 +- .../src/metrics/resources.rs | 66 + app/crowdb-chunk-kv-server/src/rpc.rs | 69 +- app/crowdb-chunk-kv-server/src/rpc/tasks.rs | 55 + app/crowdb-chunk-kv-server/src/server.rs | 11 +- .../src/server/drain.rs | 22 +- .../src/server/legacy_retirement.rs | 63 + .../src/server/resources.rs | 21 + .../src/serving/lease.rs | 2 +- .../src/serving/lease/context.rs | 2 +- .../src/serving/process.rs | 6 +- .../src/serving/worker.rs | 4 +- app/crowdb-chunk-kv-server/src/storage.rs | 271 +- .../src/storage/bootstrap.rs | 69 +- .../src/storage/legacy_retirement.rs | 250 + .../storage/legacy_retirement/allocation.rs | 194 + .../src/storage/legacy_retirement/proof.rs | 235 + .../src/storage/legacy_retirement/record.rs | 131 + .../src/storage/root_catalog.rs | 210 +- .../storage/root_catalog/legacy_migration.rs | 89 + .../storage/root_catalog/legacy_tracking.rs | 134 + .../src/storage/root_catalog/prepared.rs | 78 + .../src/storage/root_catalog/protection.rs | 186 + .../root_catalog/protection/candidate.rs | 158 + .../storage/root_catalog/protection/commit.rs | 91 + .../storage/root_catalog/protection/guards.rs | 260 + .../root_catalog/protection/operation.rs | 69 + .../root_catalog/protection/pin_owner.rs | 64 + .../root_catalog/protection/process.rs | 169 + .../storage/root_catalog/protection/prune.rs | 110 + .../storage/root_catalog/protection/record.rs | 156 + .../root_catalog/protection/release.rs | 139 + .../storage/root_catalog/protection/scan.rs | 71 + .../storage/root_catalog/protection/writer.rs | 218 + .../src/storage/root_catalog/retention.rs | 201 + .../src/storage/root_catalog/staging.rs | 194 + .../src/storage/root_catalog/transaction.rs | 51 + .../src/storage/root_catalog_test.rs | 60 + .../src/storage/split_recovery.rs | 21 +- .../src/storage/tree_owner.rs | 177 + .../tests/backing_observation_test.rs | 107 + .../tests/bootstrap_recovery_test.rs | 166 +- .../tests/common/legacy_retirement.rs | 265 + .../tests/common/metadata.rs | 38 +- .../tests/common/process.rs | 3 +- .../tests/common/process_kv.rs | 10 +- .../tests/common/root_process.rs | 167 + .../tests/config_test.rs | 35 +- .../tests/legacy_retirement/allocation.rs | 36 + .../tests/legacy_retirement/readers.rs | 101 + .../tests/legacy_retirement_test.rs | 288 ++ .../tests/perf_log_test.rs | 7 +- .../tests/process_session_test.rs | 4 +- .../root_catalog_legacy_migration_test.rs | 102 + .../root_catalog_prepared_staging_test.rs | 156 + .../tests/root_catalog_process_test.rs | 555 +++ .../tests/root_catalog_reference_test.rs | 460 ++ .../tests/root_catalog_staging_test.rs | 207 + app/crowdb-chunk-kv-server/tests/rpc_test.rs | 19 + .../tests/split_recovery_test.rs | 40 + .../tests/stream_metadata_reclaim_test.rs | 4 +- .../tests/transfer_storage_test.rs | 4 +- .../tests/tree_config_test.rs | 173 + app/crowdb-chunkdb/Cargo.toml | 9 +- app/crowdb-chunkdb/src/allocator/pool.rs | 2 +- app/crowdb-chunkdb/src/chunkdb_config.rs | 4 +- app/crowdb-chunkdb/src/conversion/io.rs | 2 +- app/crowdb-chunkdb/src/lifecycle/handler.rs | 107 +- .../src/lifecycle/handler/allocation.rs | 10 +- .../src/lifecycle/handler/deletion.rs | 169 + app/crowdb-chunkdb/src/main.rs | 12 +- app/crowdb-chunkdb/src/range_guard.rs | 14 +- app/crowdb-chunkdb/src/readiness.rs | 4 +- app/crowdb-chunkdb/src/runtime.rs | 4 +- app/crowdb-chunkdb/src/runtime/executor.rs | 2 +- .../service/chunkdb_rpc_service/mutations.rs | 4 +- app/crowdb-chunkdb/src/storage.rs | 16 +- .../src/storage/range_retirement.rs | 4 +- app/crowdb-chunkdb/src/storage/reservation.rs | 8 +- app/crowdb-chunkdb/src/task/store.rs | 6 +- app/crowdb-chunkdb/src/topology.rs | 8 +- .../src/topology/availability.rs | 4 +- app/crowdb-chunkdb/src/topology/notify.rs | 2 +- app/crowdb-chunkdb/src/topology/refresh.rs | 2 +- .../tests/allocation_fence_test.rs | 111 + .../tests/allocation_retry_test.rs | 2 +- .../tests/allocation_rounding_test.rs | 55 + .../tests/allocator_owner_change_test.rs | 2 +- .../tests/allocator_pool_test.rs | 4 +- .../tests/chunk_tree_runtime_test.rs | 430 ++ .../tests/client_owner_refresh_test.rs | 2 +- app/crowdb-chunkdb/tests/common/cluster.rs | 14 +- .../tests/common/diskio_stub.rs | 55 +- .../tests/common/dynamic_runtime.rs | 4 +- .../tests/common/readiness_failures.rs | 6 +- .../tests/common/readiness_registry.rs | 6 +- .../tests/common/repair_targets.rs | 2 +- .../tests/common/runtime_context.rs | 4 +- .../tests/common/slot_groups.rs | 12 +- .../tests/common/topology_availability.rs | 2 +- .../tests/dynamic_retry_test.rs | 2 +- app/crowdb-chunkdb/tests/e2e_test.rs | 4 +- .../tests/http_readiness_test.rs | 2 +- .../tests/journal_handoff_test.rs | 39 +- .../tests/mixed_geometry_test.rs | 2 +- .../tests/readiness_probe_test.rs | 8 +- .../tests/repair_target_recovery_test.rs | 2 +- .../tests/runtime_scope_test.rs | 2 +- app/crowdb-chunkdb/tests/slot_storage_test.rs | 2 +- .../tests/submission_epoch_test.rs | 2 +- .../tests/topology_availability_test.rs | 8 +- app/crowdb-cli/Cargo.toml | 8 +- app/crowdb-cli/src/commands.rs | 10 +- app/crowdb-cli/src/commands/bench.rs | 5 +- app/crowdb-cli/src/commands/bench/chunk.rs | 4 +- .../src/commands/bench/{kv => }/client.rs | 12 +- app/crowdb-cli/src/commands/bench/disk/db.rs | 4 +- app/crowdb-cli/src/commands/bench/kv.rs | 14 - .../src/commands/bench/kv/prepare.rs | 115 - app/crowdb-cli/src/commands/bench/kv/read.rs | 164 - app/crowdb-cli/src/commands/bench/kv/scan.rs | 183 - app/crowdb-cli/src/commands/bench/kv/write.rs | 460 -- app/crowdb-cli/src/commands/bench/loader.rs | 140 +- app/crowdb-cli/src/commands/bench/metrics.rs | 15 +- app/crowdb-cli/src/commands/bench/paxos_kv.rs | 13 + .../src/commands/bench/paxos_kv/args.rs | 127 + .../src/commands/bench/paxos_kv/client.rs | 99 + .../src/commands/bench/paxos_kv/collect.rs | 17 + .../src/commands/bench/paxos_kv/counters.rs | 105 + .../src/commands/bench/paxos_kv/dataset.rs | 204 + .../src/commands/bench/paxos_kv/deployment.rs | 156 + .../bench/paxos_kv/deployment/memory.rs | 12 + .../bench/paxos_kv/deployment/native.rs | 106 + .../src/commands/bench/paxos_kv/run.rs | 272 + .../commands/bench/paxos_kv/verification.rs | 18 + app/crowdb-cli/src/commands/bench/result.rs | 330 -- app/crowdb-cli/src/commands/bench/verb.rs | 186 +- app/crowdb-cli/src/commands/bench/workload.rs | 252 + app/crowdb-cli/src/commands/cluster/local.rs | 2 +- .../src/commands/cluster/local/host.rs | 2 +- .../src/commands/cluster/local/host/args.rs | 2 +- .../src/commands/cluster/local/launch.rs | 29 +- .../src/commands/cluster/local/state.rs | 30 +- app/crowdb-cli/src/commands/cluster/target.rs | 28 +- .../src/commands/{kv.rs => paxos_kv.rs} | 4 +- .../src/commands/{kv => paxos_kv}/data.rs | 18 +- .../src/commands/{kv => paxos_kv}/logical.rs | 0 app/crowdb-cli/src/commands/port_alloc.rs | 16 +- app/crowdb-cli/src/main.rs | 32 +- app/crowdb-cli/tests/cluster_cli_test.rs | 4 +- .../tests/cluster_lifecycle_test.rs | 100 +- app/crowdb-cli/tests/cluster_target_test.rs | 3 +- .../tests/cluster_workloads_test.rs | 4 +- .../tests/common/target_commands.rs | 9 +- app/crowdb-cli/tests/direct_cli_test.rs | 2 +- app/crowdb-cli/tests/kv_bench_test.rs | 237 + app/crowdb-diskdb/Cargo.toml | 4 +- .../conf/crowdb_diskdb_config.toml | 8 +- .../conf/crowdb_diskdb_config.toml.ports.in | 71 + app/crowdb-diskdb/src/ddb_config.rs | 20 +- app/crowdb-diskdb/src/ddb_kv_client.rs | 81 +- .../src/ddb_kv_client/disk_writes.rs | 4 +- .../src/ddb_kv_client/ownership.rs | 4 +- .../src/ddb_kv_client/retained.rs | 4 +- .../src/ddb_kv_client/zone_init.rs | 9 +- .../src/liveness/keepalive/observation.rs | 2 +- .../src/liveness/keepalive/online.rs | 2 +- .../src/liveness/keepalive/scheduler.rs | 2 +- .../src/liveness/keepalive/session.rs | 2 +- app/crowdb-diskdb/src/liveness/notify.rs | 2 +- app/crowdb-diskdb/src/main.rs | 32 +- app/crowdb-diskdb/src/model/alloc.rs | 16 +- app/crowdb-diskdb/src/rebalance/relocation.rs | 2 +- app/crowdb-diskdb/src/recovery.rs | 6 +- .../src/recovery/disk_recovery.rs | 2 +- .../src/recovery/journal_replay.rs | 4 +- app/crowdb-diskdb/src/scanner/integrity.rs | 8 +- app/crowdb-diskdb/tests/bg_task_test.rs | 4 +- .../tests/common/journal_recovery.rs | 8 +- app/crowdb-diskdb/tests/common/metadata.rs | 12 +- app/crowdb-diskdb/tests/common/online.rs | 2 +- app/crowdb-diskdb/tests/config_reload_test.rs | 12 +- app/crowdb-diskdb/tests/ddb_config_test.rs | 12 +- .../tests/disk_initialization_test.rs | 5 +- .../tests/journal_gap_recovery_test.rs | 10 +- .../tests/keepalive_sync_test.rs | 5 +- app/crowdb-diskdb/tests/metadata_flow_test.rs | 4 +- .../tests/ownership_fence_test.rs | 18 +- app/crowdb-diskdb/tests/query_handler_test.rs | 4 +- .../tests/recovery_failure_test.rs | 6 +- app/crowdb-diskdb/tests/recovery_test.rs | 10 +- .../tests/relocation_journal_test.rs | 2 +- app/crowdb-diskdb/tests/scanner_test.rs | 2 +- app/crowdb-diskdb/tests/startup_sync_test.rs | 4 +- app/crowdb-diskio/CMakeLists.txt | 24 +- app/crowdb-diskio/src/dio_config.cpp | 2 +- app/crowdb-diskio/src/dio_config.h | 6 +- app/crowdb-diskio/src/dio_main.cpp | 2 +- .../src/group0/group0_exclusion.cpp | 2 +- app/crowdb-diskio/src/group0/group0_sync.cpp | 2 +- app/crowdb-diskio/src/group0/group0_sync.h | 4 +- app/crowdb-diskio/src/rpc/dio_read.cpp | 55 +- .../src/rpc/dio_request_deadline.h | 36 + app/crowdb-diskio/src/rpc/dio_server.cpp | 16 +- app/crowdb-diskio/src/rpc/dio_server.h | 4 +- .../src/rpc/dio_server_write.cpp | 32 +- app/crowdb-diskio/tests/dio_lifetime_test.cpp | 53 +- .../tests/ordered_drain_test.cpp | 31 + .../conf/crowdb_kv_server_config.toml | 8 - .../Cargo.toml | 26 +- .../build.rs | 0 .../conf/crowdb_paxos_kv_server_config.toml | 199 + .../examples/openapi-export.rs | 2 +- .../src/background.rs | 0 .../src/background/discovery.rs | 0 .../src/background/domain_monitor.rs | 16 +- .../src/background/domain_monitor/chunk_kv.rs | 2 +- .../domain_monitor/chunk_kv/balance.rs | 2 +- .../chunk_kv/balance/materialization.rs | 0 .../chunk_kv/balance/observation.rs | 0 .../chunk_kv/balance/selection.rs | 0 .../chunk_kv/balance/selection/candidates.rs | 0 .../chunk_kv/balance/selection/diagnostics.rs | 0 .../domain_monitor/chunk_kv/balance/split.rs | 0 .../domain_monitor/chunk_kv/catalog.rs | 2 +- .../chunk_kv/catalog/endpoints.rs | 0 .../domain_monitor/chunk_kv/grants.rs | 4 +- .../domain_monitor/chunk_kv/liveness.rs | 0 .../src/background/domain_monitor/chunkdb.rs | 0 .../domain_monitor/chunkdb_dynamic.rs | 2 +- .../domain_monitor/chunkdb_dynamic/owners.rs | 0 .../chunkdb_dynamic/snapshot.rs | 0 .../src/background/domain_monitor/diskdb.rs | 0 .../domain_monitor/diskdb_placement.rs | 2 +- .../src/background/identity.rs | 2 +- .../src/background/keepalive.rs | 12 +- .../src/cli.rs | 20 +- .../src/engine_collector.rs | 28 +- .../src/engine_collector/resources.rs | 55 + .../src/group0_control_plane.rs | 6 +- .../src/lib.rs | 5 +- .../src/main.rs | 212 +- .../src/mgmt.rs | 11 +- .../src/mgmt/discovery.rs | 0 .../src/mgmt/group_creation.rs | 2 +- .../src/mgmt/group_ops.rs | 53 +- .../src/mgmt/operation_registry.rs | 39 +- .../src/mgmt/replica_ops.rs | 27 +- .../src/mgmt/runtime_shutdown.rs | 28 + .../src/mgmt/store_ops.rs | 29 +- .../src/mgmt/system_bootstrap.rs | 2 +- .../src/mgmt/system_bootstrap/cleanup.rs | 2 +- .../src/mgmt/system_init.rs | 11 +- .../src/mgmt/system_join.rs | 0 .../src/mgmt/system_store.rs | 4 +- .../src/mgmt/topology.rs | 8 +- .../src/recovery.rs | 0 .../src/recovery/group_rebuild.rs | 8 +- .../src/recovery/member_routes.rs | 14 +- .../src/recovery/reconcile.rs | 18 +- .../src/recovery/restore.rs | 72 +- .../src/recovery/startup.rs | 95 +- app/crowdb-paxos-kv-server/src/shutdown.rs | 47 + .../src/store_registry.rs | 34 +- .../tests/chunk_dynamic_monitor_test.rs | 6 +- .../tests/chunk_kv_balance_planning_test.rs | 6 +- .../committed_transfer.rs | 0 .../recovering_owner.rs | 1 + .../tests/chunk_slot_monitor_test.rs | 8 +- .../tests/cli_parse_test.rs | 44 +- .../tests/common/balance_distribution.rs | 2 +- .../tests/common/balance_planning.rs | 2 +- .../tests/common/catalog_publication.rs | 2 +- .../tests/common/dynamic_slots.rs | 6 +- .../tests/common/endpoint_publication.rs | 2 +- .../tests/common/grant_isolation.rs | 2 +- .../tests/common/grant_process.rs | 2 +- .../tests/common/group_creation.rs | 18 +- .../tests/common/process_lease.rs | 3 +- .../tests/common/runtime_shutdown.rs | 155 + .../tests/common/split_materialization.rs | 0 .../common/split_materialization/transfer.rs | 0 .../tests/common/system_bootstrap.rs | 18 +- .../tests/diskdb_placement_test.rs | 16 +- .../tests/domain_monitor_test.rs | 24 +- .../tests/engine_resource_metrics_test.rs | 64 + .../tests/group0_control_plane_test.rs | 10 +- .../tests/group0_discovery_test.rs | 2 +- .../tests/group_creation_test.rs | 4 +- .../tests/keepalive_recovery_test.rs | 20 +- .../tests/member_routes_test.rs | 4 +- .../tests/monitor_supervisor_test.rs | 2 +- .../tests/reconcile_test.rs | 18 +- .../tests/restore_test.rs | 8 +- .../tests/runtime_shutdown_test.rs | 104 + .../tests/startup_test.rs | 153 +- .../tests/startup_test/reclamation.rs | 6 +- .../tests/tree_config_test.rs | 103 + .../tests/tree_runtime_host_test.rs | 96 + app/crowdb-web/Cargo.toml | 4 +- app/crowdb-web/src/chunk.rs | 2 +- app/crowdb-web/src/chunk/slots.rs | 2 +- app/crowdb-web/src/chunk_kv.rs | 4 +- app/crowdb-web/src/chunk_kv/balance.rs | 6 +- .../src/chunk_kv/runtime/discovery.rs | 2 +- app/crowdb-web/src/corr_id.rs | 6 +- app/crowdb-web/src/diskdb/lifecycle.rs | 65 +- app/crowdb-web/src/diskdb/proxy.rs | 6 +- app/crowdb-web/src/diskdb/remote.rs | 8 +- app/crowdb-web/src/error.rs | 2 +- app/crowdb-web/src/kv.rs | 2 +- app/crowdb-web/src/lib.rs | 2 +- app/crowdb-web/src/lifecycle.rs | 8 +- app/crowdb-web/src/lifecycle/kv_process.rs | 12 +- app/crowdb-web/src/main.rs | 28 +- app/crowdb-web/src/managed.rs | 10 +- app/crowdb-web/src/mgmt.rs | 18 +- app/crowdb-web/src/mgmt/group_ops.rs | 2 +- app/crowdb-web/src/node/admission.rs | 5 +- app/crowdb-web/src/node/admission/recovery.rs | 2 +- app/crowdb-web/src/node/authority.rs | 2 +- .../src/node/authority/projection.rs | 8 +- app/crowdb-web/src/node/auto_join/runner.rs | 2 +- app/crowdb-web/src/node/cluster_status.rs | 4 +- app/crowdb-web/src/node/storage/authority.rs | 2 +- app/crowdb-web/src/node/storage/lifecycle.rs | 2 +- app/crowdb-web/src/node/target.rs | 2 +- app/crowdb-web/src/physical/disk.rs | 2 +- app/crowdb-web/src/physical/node.rs | 2 +- app/crowdb-web/src/services/defaults.rs | 78 +- app/crowdb-web/src/services/dependencies.rs | 4 +- .../src/services/deployment/chunk_slots.rs | 2 +- .../src/services/deployment/credentials.rs | 2 +- .../src/services/deployment/geometry.rs | 2 +- .../src/services/deployment/launch.rs | 7 +- .../src/services/deployment/planned.rs | 3 - app/crowdb-web/src/services/observation.rs | 2 +- app/crowdb-web/src/services/plans.rs | 12 +- app/crowdb-web/src/standalone/readiness.rs | 2 +- app/crowdb-web/src/standalone/recovery.rs | 2 +- app/crowdb-web/src/state.rs | 75 +- app/crowdb-web/tests/bound_client_test.rs | 4 +- app/crowdb-web/tests/chunk_kv_catalog_test.rs | 10 +- app/crowdb-web/tests/cluster_target_test.rs | 2 +- .../tests/common/catalog_balance.rs | 4 +- app/crowdb-web/tests/common/cluster_target.rs | 6 +- app/crowdb-web/tests/common/memory_kv.rs | 4 +- .../tests/common/storage_lifecycle.rs | 4 +- .../tests/common/storage_readiness.rs | 2 +- .../tests/corr_id_propagation_test.rs | 2 +- .../tests/deployment_cancellation_test.rs | 17 +- app/crowdb-web/tests/diskdb_binding_test.rs | 2 +- .../diskdb_deployment_configuration_test.rs | 13 +- app/crowdb-web/tests/diskdb_routes_test.rs | 4 +- app/crowdb-web/tests/lifecycle_routes_test.rs | 4 +- app/crowdb-web/tests/managed_mode_test.rs | 4 +- .../tests/node_storage_authority_test.rs | 4 +- .../tests/remote_kv_projection_test.rs | 8 +- .../tests/service_lifecycle_test.rs | 10 +- app/crowdb-web/tests/service_plans_test.rs | 4 +- .../tests/storage_readiness_test.rs | 4 +- app/crowdb-web/ui/src/App.tsx | 13 +- app/crowdb-web/ui/src/api.ts | 7 +- .../src/components/dialogs/AddNodeDialog.tsx | 5 +- .../components/dialogs/DeployDiskdbDialog.tsx | 8 +- .../components/dialogs/DeployServerDialog.tsx | 6 +- .../components/dialogs/createFlows.test.tsx | 94 +- .../ui/src/components/dialogs/defaults.ts | 37 +- app/crowdb-web/ui/src/embed.ts | 2 +- app/crowdb-web/ui/src/ports.ts | 21 + .../ui/src/services/DeployServiceDialog.tsx | 11 +- .../ui/src/services/ServicePlanCard.tsx | 2 +- .../ui/src/services/useNodeServicePlans.ts | 2 +- .../ui/src/shell/NodeSetupOptions.tsx | 3 +- app/crowdb-web/ui/src/shell/nodeSetup.ts | 6 +- app/crowdb-web/ui/src/types/index.ts | 6 +- app/crowdb-web/ui/vite.config.ts | 19 +- container/crowdb-monitor/Cargo.toml | 4 +- .../crowdb-monitor/src/bootstrap/chunk.rs | 6 +- .../crowdb-monitor/src/bootstrap/hardware.rs | 8 +- .../crowdb-monitor/src/bootstrap/iceberg.rs | 6 +- .../src/bootstrap/iceberg/shared_console.rs | 6 +- .../crowdb-monitor/src/bootstrap/logical.rs | 8 +- .../src/bootstrap/storage_probe.rs | 6 +- container/crowdb-monitor/src/main.rs | 2 +- container/crowdb-monitor/src/node/runtime.rs | 12 +- .../src/node/runtime/admission.rs | 6 +- .../src/node/runtime/file_admission.rs | 4 +- .../src/node/runtime/file_preparation.rs | 4 +- .../crowdb-monitor/src/node/runtime/kv.rs | 7 +- .../src/node/runtime/recovery/confirmed.rs | 4 +- .../src/node/runtime/recovery/publication.rs | 8 +- .../src/node/runtime/retained_storage.rs | 4 +- .../src/node/runtime/services.rs | 5 +- .../src/node/runtime/services/dependencies.rs | 10 +- .../src/node/runtime/services/execution.rs | 6 +- .../src/node/runtime/setup/execution.rs | 4 +- .../node/runtime/setup/execution/metadata.rs | 4 +- .../node/runtime/setup/execution/services.rs | 4 +- .../node/runtime/setup/execution/storage.rs | 10 +- .../node/runtime/setup/execution/supplied.rs | 6 +- .../src/node/runtime/storage_health.rs | 4 +- .../src/node/runtime/supplied_storage.rs | 4 +- .../runtime/supplied_storage/execution.rs | 4 +- .../crowdb-monitor/src/node/startup/policy.rs | 1 - .../tests/access_bootstrap_test.rs | 9 +- .../crowdb-monitor/tests/blank_medium_test.rs | 24 +- .../crowdb-monitor/tests/common/node_setup.rs | 12 +- .../tests/common/node_setup_file.rs | 4 +- .../tests/common/service_dependency.rs | 6 +- .../tests/console_catalog_shared_test.rs | 2 +- .../tests/file_admission_test.rs | 4 +- .../crowdb-monitor/tests/node_policy_test.rs | 10 +- .../crowdb-monitor/tests/node_profile_test.rs | 10 +- .../tests/node_run_inputs_test.rs | 12 +- .../tests/node_service_dependencies_test.rs | 8 +- .../tests/node_setup_supplied_test.rs | 6 +- .../crowdb-monitor/tests/node_setup_test.rs | 12 +- .../crowdb-monitor/tests/profile_test.rs | 2 +- .../tests/recovery_owners_test.rs | 4 +- .../tests/single_node_profile_test.rs | 30 +- container/oci-image/Dockerfile | 3 +- container/oci-image/Dockerfile.ports.in | 50 + container/oci-image/README.md | 51 +- container/oci-image/collect-libs.sh | 4 +- container/oci-image/compose-three.yaml | 43 +- .../oci-image/compose-three.yaml.ports.in | 58 + container/oci-image/compose.yaml | 15 +- container/oci-image/compose.yaml.ports.in | 24 + container/oci-image/docker_local_launch.py | 27 +- container/oci-image/docker_local_state.py | 7 +- container/oci-image/host_inputs.py | 4 +- container/oci-image/node-profile.toml | 25 +- .../oci-image/node-profile.toml.ports.in | 149 + container/oci-image/paxos-kv-cluster.py | 212 + container/oci-image/ports.py | 22 + container/oci-image/run-node.py | 21 +- container/oci-image/runtime.py | 6 +- container/oci-image/stage.sh | 7 +- .../oci-image/tests/docker_local_test.py | 56 +- container/oci-image/tests/host_inputs_test.py | 4 +- container/oci-image/tests/ports_test.py | 49 + container/oci-image/tests/staging_test.py | 10 +- .../oci-image/tests/task_entrypoints_test.py | 4 +- container/single-node-container/README.md | 11 +- .../single-node-container/dev-container.sh | 9 +- .../dev-container.sh.ports.in | 153 + container/single-node-container/entrypoint.sh | 18 +- .../entrypoint.sh.ports.in | 175 + container/single-node-container/profile.toml | 49 +- .../profile.toml.ports.in | 218 + .../templates/access.toml | 9 +- .../templates/access.toml.ports.in | 92 + .../templates/chunk-kv.toml | 11 +- .../templates/chunk-kv.toml.ports.in | 32 + .../templates/chunkdb.toml | 7 +- .../templates/chunkdb.toml.ports.in | 38 + .../templates/crowdb-web.toml | 5 +- .../templates/crowdb-web.toml.ports.in | 10 + .../templates/diskdb.toml | 8 +- .../templates/diskdb.toml.ports.in | 64 + .../templates/diskio.toml | 3 +- .../templates/diskio.toml.ports.in | 43 + .../single-node-container/templates/kv.toml | 1 + .../templates/kv.toml.ports.in | 6 + .../tests/image-smoke.sh | 17 +- .../R102-diskdb-dynamic-binding-migration.md | 412 -- doc/backlog/R103-chunkdb-range-migration.md | 287 -- doc/backlog/R139-group0-service-config.md | 147 - doc/backlog/R144-chunk-kv-partition-merge.md | 261 - doc/backlog/R147-tree-chunk-gc.md | 139 - doc/backlog/R148-chunk-stream-scale-out.md | 154 - .../R168-s3-shared-object-reclamation.md | 73 - doc/backlog/R169-s3-shared-chunk-tree-gc.md | 77 - .../R185-access-iceberg-cache-invalidation.md | 129 - .../R186-access-iceberg-orc-validation.md | 68 - .../R193-chunkdb-node-failure-budget.md | 47 - doc/backlog/R196-refine-benchmark-tests.md | 282 -- .../R207-chunkdb-repo-metadata-chunk-kv.md | 318 -- .../R233-sdk-local-cache-checkpoint.md | 91 - doc/backlog/R4-bounded-mempool.md | 27 - .../R60-tree-scan-sibling-leaf-readahead.md | 137 - doc/backlog/R80-diskdb-rebalance.md | 127 - doc/backlog/R82-kv-watch-notify-coalescing.md | 191 - .../R83-chunkdb-complete-recovery-flow.md | 287 -- ...hunkdb-post-disk-move-placement-scanner.md | 246 - doc/backlog/R92-chunkdb-in-chunk-gc.md | 17 - doc/backlog/R95-chunkdb-chunk-range-delete.md | 201 - .../R96-chunkdb-console-cli-integration.md | 12 - doc/backlog/backlog.md | 95 - .../design-crowdb-chunkdb-range-binding.md | 20 +- doc/design/chunkdb/design-crowdb-chunkdb.md | 16 +- doc/design/chunkds/design-crowdb-chunk-kv.md | 141 +- .../chunkds/design-crowdb-chunk-stream.md | 2 +- doc/design/config/design-crowdb-config.md | 4 +- .../console/design-crowdb-console-ui.md | 2 +- doc/design/console/design-crowdb-console.md | 28 +- doc/design/deploy/design-crowdb-deploy.md | 4 +- .../design-crowdb-diskdb-space-metrics.md | 14 +- .../design-crowdb-diskdb-zone-management.md | 17 +- doc/design/diskdb/design-crowdb-diskdb.md | 54 +- .../diskdb/diskdb-allocate-flow-analysis.md | 2 +- ...p0.md => design-crowdb-paxos-kv-group0.md} | 70 +- ...design-crowdb-paxos-kv-leader-election.md} | 18 +- ...> design-crowdb-paxos-kv-observability.md} | 28 +- ...design-crowdb-paxos-kv-reconfiguration.md} | 18 +- ...d => design-crowdb-paxos-kv-rpc-client.md} | 38 +- ...v-rpc.md => design-crowdb-paxos-kv-rpc.md} | 45 +- ...er.md => design-crowdb-paxos-kv-server.md} | 32 +- ...slot.md => design-crowdb-paxos-kv-slot.md} | 38 +- ...> design-crowdb-paxos-kv-state-machine.md} | 37 +- ...test.md => design-crowdb-paxos-kv-test.md} | 52 +- ...v-wal.md => design-crowdb-paxos-kv-wal.md} | 20 +- ...=> design-crowdb-paxos-kv-watch-notify.md} | 43 +- ...crowdb-kv.md => design-crowdb-paxos-kv.md} | 52 +- .../protocol/design-crowdb-protocol-key.md | 16 +- .../protocol/design-crowdb-protocol-types.md | 37 +- doc/design/protocol/design-crowdb-protocol.md | 6 +- doc/design/rpc/design-crowdb-rpc.md | 9 +- .../tree/design-crowdb-tree-chunk-storage.md | 595 +-- .../design-crowdb-tree-engine-flush-flow.md | 304 +- ...design-crowdb-tree-engine-snapshot-flow.md | 280 +- doc/design/tree/design-crowdb-tree-engine.md | 1150 ++--- doc/design/tree/design-crowdb-tree-storage.md | 1020 +--- doc/design/tree/design-crowdb-tree.md | 385 +- doc/dev/crash_debugging.md | 6 +- doc/dev/env_setup.md | 6 +- doc/doc_index.md | 140 +- doc/working/plan-rdma-multi-node.md | 117 - doc/working/test.md | 220 - doc/working/todo_code.md | 52 - e2e-test/Cargo.toml | 4 +- e2e-test/README.md | 17 +- e2e-test/admission_profile.py | 2 +- .../flows/11-cluster-diskdb-lifecycle.spec.ts | 6 + e2e-test/clients/ecosystem/README.md | 2 +- e2e-test/clients/ecosystem/acceptance.py | 2 +- e2e-test/console_access_fixture.py | 4 +- e2e-test/console_bootstrap_fixture.py | 4 +- e2e-test/console_cli_fixture.py | 39 +- e2e-test/console_cli_operations.py | 28 +- e2e-test/console_cli_recovery.py | 4 +- e2e-test/console_cli_target.py | 24 +- e2e-test/console_discovery_browser_fixture.py | 10 +- e2e-test/console_fixture.py | 2 +- e2e-test/console_lifetime_fixture.py | 8 +- e2e-test/console_native_fixture.py | 16 +- e2e-test/console_recovery_fixture.py | 4 +- e2e-test/console_registry_fixture.py | 6 +- e2e-test/console_shell_fixture.py | 2 +- e2e-test/console_standalone_fixture.py | 4 +- e2e-test/console_standalone_observation.py | 2 +- e2e-test/console_typed_fixture.py | 4 +- e2e-test/console_wal_fixture.py | 4 +- e2e-test/console_web_fixture.py | 6 +- e2e-test/default_ports.py | 22 + e2e-test/deployment_anonymous_fixture.py | 6 +- e2e-test/deployment_browser_fixture.py | 4 +- e2e-test/deployment_listener_fixture.py | 4 +- e2e-test/diskdb_fixture.py | 1 - e2e-test/image_identity.py | 7 + e2e-test/kv_node.py | 2 +- e2e-test/managed_fixture.py | 21 +- e2e-test/oci_host_node.py | 6 +- e2e-test/oci_local_host_node.py | 2 +- e2e-test/oci_reference_recovery.py | 4 +- e2e-test/oci_reference_runtime.py | 6 +- e2e-test/profiles/bridge_node.py | 2 +- e2e-test/profiles/containerd-cluster.py | 2 +- e2e-test/profiles/node-recovery.py | 5 +- e2e-test/release.py | 4 +- e2e-test/s3_node_fixture.py | 1 - e2e-test/scenarios.py | 2 + .../access_iceberg/file_http/capacity.rs | 4 +- .../access_iceberg/file_http/streaming.rs | 8 +- .../access_iceberg/gc/notifications/large.rs | 4 +- .../access_iceberg/gc/notifications/proof.rs | 4 +- e2e-test/tests/access_iceberg/native.rs | 4 +- e2e-test/tests/access_s3.rs | 4 +- e2e-test/tests/access_s3/node_failure.rs | 8 +- .../tests/access_s3/node_failure/recovery.rs | 4 +- e2e-test/tests/access_s3/policy.rs | 8 +- e2e-test/tests/admission_profile_test.py | 2 +- e2e-test/tests/chunk_client/large_repair.rs | 4 +- e2e-test/tests/chunk_client/reader.rs | 4 +- e2e-test/tests/chunk_client/small_flow.rs | 4 +- e2e-test/tests/chunk_client/stack.rs | 2 +- e2e-test/tests/chunk_kv/recovery.rs | 4 +- e2e-test/tests/chunkdb/allocation.rs | 2 +- e2e-test/tests/chunkdb/conversion.rs | 2 +- e2e-test/tests/chunkdb/handoff.rs | 6 +- e2e-test/tests/chunkdb/hardware.rs | 2 +- e2e-test/tests/chunkdb/io.rs | 2 +- e2e-test/tests/chunkdb/io/readiness.rs | 2 +- e2e-test/tests/chunkdb/lifecycle.rs | 2 +- e2e-test/tests/chunkdb/placement.rs | 2 +- e2e-test/tests/chunkdb/recovery.rs | 2 +- e2e-test/tests/chunkdb/repair/topology.rs | 2 +- e2e-test/tests/chunkdb/slots.rs | 14 +- e2e-test/tests/chunkdb/tasks.rs | 2 +- e2e-test/tests/chunkdb/voting.rs | 4 +- e2e-test/tests/common.rs | 8 +- e2e-test/tests/common/hardware.rs | 6 +- .../tests/common/native_s3/signed_copy.rs | 4 +- e2e-test/tests/common/test_client.rs | 20 +- .../tests/console/bootstrap_publication.rs | 8 +- e2e-test/tests/console/deployment.rs | 4 +- e2e-test/tests/console/discovery.rs | 4 +- e2e-test/tests/console/fixture.rs | 4 +- e2e-test/tests/console/group_publication.rs | 2 +- e2e-test/tests/console/native_balance.rs | 10 +- e2e-test/tests/console/native_binding.rs | 2 +- e2e-test/tests/console/native_capacity.rs | 2 +- e2e-test/tests/console/native_chunks.rs | 2 +- e2e-test/tests/console/native_context.rs | 13 +- .../tests/console/native_disk_lifecycle.rs | 5 +- e2e-test/tests/console/native_disk_usage.rs | 2 +- e2e-test/tests/console/native_load.rs | 2 +- e2e-test/tests/console/native_s3.rs | 2 +- e2e-test/tests/console/native_storage.rs | 11 +- .../tests/console/native_storage/media.rs | 2 +- e2e-test/tests/console/server_data.rs | 12 +- e2e-test/tests/console/web_access.rs | 5 +- e2e-test/tests/console/web_authority.rs | 9 +- e2e-test/tests/console/web_catalog.rs | 7 +- .../tests/console/web_cluster_recovery.rs | 9 +- .../tests/console/web_diskio_discovery.rs | 7 +- e2e-test/tests/console/web_isolation.rs | 3 +- e2e-test/tests/console/web_kv_lifecycle.rs | 5 +- e2e-test/tests/console/web_legacy.rs | 3 +- e2e-test/tests/console/web_lifecycle.rs | 22 +- e2e-test/tests/console/web_lifetime.rs | 3 +- e2e-test/tests/console/web_recovery.rs | 3 +- e2e-test/tests/console/web_registry.rs | 3 +- .../tests/console/web_sealed_bootstrap.rs | 3 +- e2e-test/tests/console/web_slots.rs | 7 +- e2e-test/tests/console/web_standalone.rs | 13 +- e2e-test/tests/console/web_voters.rs | 3 +- e2e-test/tests/console/web_wal_restart.rs | 5 +- e2e-test/tests/console/web_wal_restart/wal.rs | 6 +- e2e-test/tests/console_lifetime_test.py | 2 +- e2e-test/tests/deployment.rs | 19 +- e2e-test/tests/deployment/balance.rs | 7 +- e2e-test/tests/deployment/clients.rs | 19 +- .../tests/deployment/hardware_bootstrap.rs | 6 +- e2e-test/tests/deployment/kv_bootstrap.rs | 10 +- e2e-test/tests/deployment/listeners.rs | 12 +- e2e-test/tests/deployment/storage.rs | 15 +- e2e-test/tests/deployment/supplied_media.rs | 6 +- e2e-test/tests/diskdb.rs | 6 +- e2e-test/tests/diskdb/hardware.rs | 2 +- e2e-test/tests/diskdb/recovery.rs | 2 +- e2e-test/tests/diskio.rs | 8 +- e2e-test/tests/diskio/hardware.rs | 2 +- e2e-test/tests/domain_test.rs | 2 +- e2e-test/tests/kv.rs | 1 + e2e-test/tests/kv/authority.rs | 12 +- e2e-test/tests/kv/client_data.rs | 8 +- e2e-test/tests/kv/client_scan.rs | 18 +- e2e-test/tests/kv/cluster.rs | 10 +- e2e-test/tests/kv/discovery.rs | 4 +- e2e-test/tests/kv/domain_monitor.rs | 2 +- e2e-test/tests/kv/failover.rs | 8 +- e2e-test/tests/kv/handoff_fixture.rs | 4 +- e2e-test/tests/kv/maintenance.rs | 6 +- e2e-test/tests/kv/membership_cas.rs | 4 +- e2e-test/tests/kv/reconfiguration.rs | 12 +- e2e-test/tests/kv/routing.rs | 4 +- e2e-test/tests/kv/slot_handoff.rs | 2 +- e2e-test/tests/kv/slot_map.rs | 30 +- e2e-test/tests/kv/slot_map_recovery.rs | 79 + e2e-test/tests/kv/slot_map_validation.rs | 2 +- e2e-test/tests/kv/slot_publication.rs | 18 +- e2e-test/tests/kv/snapshot.rs | 16 +- e2e-test/tests/managed_supervision_test.py | 2 +- e2e-test/tests/native_containerd_test.py | 13 +- e2e-test/tests/oci_reference_test.py | 9 +- e2e-test/tests/oci_workloads_test.py | 8 +- e2e-test/tests/paxos_kv_diagnostics_test.py | 126 + e2e-test/tests/regression_fixture_test.py | 4 +- e2e-test/tests/selection_test.py | 5 +- lib/crowdb-access-iceberg/Cargo.toml | 2 +- lib/crowdb-access-iceberg/src/storage.rs | 4 +- lib/crowdb-access-s3/Cargo.toml | 2 +- lib/crowdb-access-s3/src/storage.rs | 6 +- .../common/multipart_completion_request.rs | 4 +- .../tests/common/multipart_expiry_request.rs | 4 +- .../tests/multipart_repository/independent.rs | 4 +- lib/crowdb-chunk-client/Cargo.toml | 2 +- lib/crowdb-chunk-client/src/client.rs | 4 +- .../src/client/connection.rs | 10 +- .../src/client/readiness.rs | 2 +- .../src/disk_io/routing.rs | 2 +- .../tests/common/readiness_schedule.rs | 2 +- lib/crowdb-chunk-kv-client/Cargo.toml | 2 +- .../src/bin/crowdb-chunk-kv-cli.rs | 6 +- .../src/catalog/store.rs | 6 +- lib/crowdb-chunk-kv/Cargo.toml | 1 + lib/crowdb-chunk-kv/src/lib.rs | 8 +- lib/crowdb-chunk-kv/src/memory.rs | 116 +- .../src/memory/prepared_split.rs | 291 ++ lib/crowdb-chunk-kv/src/partition.rs | 114 +- lib/crowdb-chunk-kv/src/partition/config.rs | 57 + lib/crowdb-chunk-kv/src/partition/ingress.rs | 26 +- .../src/partition/maintenance.rs | 5 +- lib/crowdb-chunk-kv/src/partition/native.rs | 6 +- lib/crowdb-chunk-kv/src/partition/replay.rs | 2 + lib/crowdb-chunk-kv/src/partition/split.rs | 52 +- .../src/partition/split/inheritance.rs | 105 + .../src/partition/split/journal_proof.rs | 24 + .../src/partition/split/preparation.rs | 5 +- .../src/partition/split/recovery.rs | 32 +- .../src/partition/split/stable_session.rs | 296 +- lib/crowdb-chunk-kv/src/partition/transfer.rs | 2 + lib/crowdb-chunk-kv/src/partition/tree.rs | 177 +- .../src/partition/tree/snapshot.rs | 200 + .../src/partition/tree/split_overlay.rs | 66 + .../src/partition/worker_completion.rs | 39 + .../src/partition/worker_sender.rs | 7 + .../tests/bootstrap_recovery_test.rs | 55 +- .../tests/common/root_catalog.rs | 47 +- .../tests/common/split_publish_tree.rs | 349 ++ .../tests/history_maintenance_test.rs | 41 + .../tests/immutable_runtime_split_test.rs | 83 + .../tests/native_checkpoint_test.rs | 57 +- .../tests/native_runtime_test.rs | 64 +- .../tests/native_split_handoff_test.rs | 446 +- .../tests/partition_empty_cursor_test.rs | 75 + lib/crowdb-chunk-kv/tests/partition_test.rs | 70 +- .../tests/prepared_split_overlay_test.rs | 105 + .../tests/split_cutover_journal_test.rs | 306 ++ .../tests/split_handoff_recovery_test.rs | 97 +- .../tests/split_ingress_test.rs | 7 +- lib/crowdb-chunk-stream/Cargo.toml | 4 +- .../benches/production_bounds.rs | 4 +- lib/crowdb-chunk-stream/src/kv.rs | 26 +- lib/crowdb-chunk-stream/src/kv/owner.rs | 25 +- .../src/kv/record_backend.rs | 8 +- .../src/memory/record_backend.rs | 12 +- lib/crowdb-chunk-stream/src/production.rs | 8 +- lib/crowdb-chunk-stream/src/records.rs | 8 +- .../src/records/backend.rs | 2 +- .../src/records/candidate.rs | 2 +- .../src/records/collection.rs | 2 +- lib/crowdb-chunk-stream/src/records/pin.rs | 2 +- .../src/records/publication.rs | 2 +- .../tests/common/metadata_server.rs | 14 +- .../tests/common/runtime_kv.rs | 10 +- .../tests/ordered_kv_record_test.rs | 8 +- .../tests/production_chunk_test.rs | 4 +- lib/crowdb-chunkdb-client/Cargo.toml | 2 +- lib/crowdb-chunkdb-client/src/client.rs | 2 +- lib/crowdb-chunkdb-client/src/lib.rs | 4 +- .../src/rpc_transport.rs | 1 + lib/crowdb-common/cpp/CMakeLists.txt | 16 +- .../cpp/include/crowdb-common/diskio_uring.h | 38 + .../metrics/inline_callback_gauge.h | 73 + .../include/crowdb-common/metrics/metrics.h | 24 +- .../crowdb-common/metrics/system_metrics.h | 2 +- lib/crowdb-common/cpp/src/diskio_owned.cpp | 125 + lib/crowdb-common/cpp/src/diskio_uring.cpp | 58 +- lib/crowdb-common/cpp/src/metrics/metrics.cpp | 33 +- .../cpp/src/metrics/metrics_inline.cpp | 42 + .../cpp/tests/diskio_owned_test.cpp | 168 + lib/crowdb-common/rust/src/lib.rs | 4 +- lib/crowdb-common/rust/src/logging.rs | 8 +- lib/crowdb-common/rust/src/metrics.rs | 2 + lib/crowdb-common/rust/src/metrics/runner.rs | 46 +- lib/crowdb-common/rust/src/metrics/sampler.rs | 14 + .../rust/tests/metrics_sampler_test.rs | 57 + lib/crowdb-console-shared/Cargo.toml | 10 +- lib/crowdb-console-shared/src/clients.rs | 8 +- .../src/clients/console.rs | 16 +- lib/crowdb-console-shared/src/clients/http.rs | 6 +- lib/crowdb-console-shared/src/cluster.rs | 12 +- .../src/cluster_deployer.rs | 11 +- lib/crowdb-console-shared/src/config.rs | 10 +- lib/crowdb-console-shared/src/corr_id.rs | 10 +- .../src/deployment/bootstrap.rs | 2 +- .../src/deployment/node_update.rs | 10 +- .../src/deployment/preparation.rs | 2 +- .../src/deployment/recovery/confirmed.rs | 9 +- .../src/deployment/registry.rs | 19 +- .../src/deployment/services.rs | 10 +- .../src/deployment/services/planned_start.rs | 4 +- .../src/deployment/services/readiness.rs | 8 +- .../src/deployment/target.rs | 4 +- lib/crowdb-console-shared/src/diskdb.rs | 2 - lib/crowdb-console-shared/src/error.rs | 2 +- lib/crowdb-console-shared/src/lib.rs | 2 +- lib/crowdb-console-shared/src/lifecycle.rs | 101 +- lib/crowdb-console-shared/src/mgmt.rs | 4 +- lib/crowdb-console-shared/src/monitor.rs | 2 +- lib/crowdb-console-shared/src/ops/chunk.rs | 2 +- lib/crowdb-console-shared/src/ops/cluster.rs | 24 +- .../src/ops/cluster/bootstrap.rs | 2 +- .../src/ops/cluster/bootstrap/publication.rs | 2 +- .../src/ops/cluster/chunk_slots.rs | 2 +- .../src/ops/cluster/readiness/collect.rs | 2 +- .../ops/cluster/readiness/collect/storage.rs | 12 +- .../readiness/collect/storage/checks.rs | 4 +- .../readiness/collect/storage/supplied.rs | 6 +- .../src/ops/cluster/storage_readiness.rs | 4 +- lib/crowdb-console-shared/src/ops/context.rs | 36 +- .../src/ops/hardware/authority.rs | 2 +- .../src/ops/hardware/authority_storage.rs | 2 +- .../src/ops/hardware/relocation.rs | 2 +- lib/crowdb-console-shared/src/ops/kv_data.rs | 4 +- .../src/ops/kv_logical.rs | 10 +- .../src/ops/kv_logical/default_group.rs | 2 +- .../src/ops/kv_logical/publication.rs | 2 +- .../src/ops/kv_logical/replica_join.rs | 6 +- .../src/ops/kv_server.rs | 13 +- lib/crowdb-console-shared/src/ssh.rs | 10 +- .../src/ssh/known_hosts.rs | 8 +- .../tests/admission_control_failure_test.rs | 2 +- .../bootstrap_hardware_membership_test.rs | 4 +- .../tests/bootstrap_intent_test.rs | 2 +- .../tests/cluster_storage_observation_test.rs | 4 +- ...uster_supplied_storage_observation_test.rs | 2 +- .../tests/common/default_metadata.rs | 16 +- .../tests/common/default_preparation.rs | 8 +- .../tests/common/regression/inspection.rs | 2 +- .../tests/config_web_test.rs | 2 +- .../tests/default_metadata_test.rs | 6 +- .../tests/default_preparation_test.rs | 2 +- .../tests/group_publication_replay_test.rs | 8 +- .../tests/launch_ssh_test.rs | 2 +- .../tests/member_recovery_test.rs | 2 +- .../tests/ops_context_test.rs | 4 +- .../tests/ops_kv_server_test.rs | 2 +- .../tests/planned_service_start_test.rs | 10 +- .../tests/topology_aggregate_test.rs | 2 +- lib/crowdb-diskdb-client/Cargo.toml | 4 +- lib/crowdb-diskdb-client/src/client.rs | 2 +- lib/crowdb-diskdb-client/src/lib.rs | 2 +- .../tests/routing_snapshot_test.rs | 4 +- lib/crowdb-diskio-client/Cargo.toml | 4 +- lib/crowdb-diskio-client/src/client.rs | 6 +- lib/crowdb-diskio-client/src/semantic.rs | 2 +- .../src/semantic/connection.rs | 4 +- lib/crowdb-diskio-client/src/topology.rs | 2 +- .../tests/scoped_discovery_test.rs | 10 +- .../Cargo.toml | 20 +- .../include/crowdb-paxos-kv-client}/c_api.h | 2 +- .../src/binding.rs | 0 .../src/binding/chunk_slots.rs | 14 +- .../src/binding/chunk_slots/handoff.rs | 0 .../src/binding/chunk_slots/initialization.rs | 154 + .../src/binding/chunk_slots/publication.rs | 0 .../binding/chunk_slots/publication/epochs.rs | 0 .../src/binding/chunkdb_strategy.rs | 6 +- .../src/binding/framework.rs | 10 +- .../src/binding/range.rs | 8 +- .../src/client.rs | 2 +- .../src/client/admin.rs | 8 +- .../src/client/core.rs | 30 +- .../src/client/core/conditional.rs | 8 +- .../src/client/core/operations.rs | 14 +- .../src/client/core/owned.rs | 6 +- .../src/client/core/scans.rs | 14 +- .../src/client/retry.rs | 8 +- .../src/client/topology.rs | 2 +- .../src/client/topology/address.rs | 0 .../src/client/topology/recovery.rs | 0 .../src/config.rs | 0 .../src/error.rs | 6 +- .../src/ffi.rs | 10 +- .../src/ffi/disk_exclusion.rs | 0 .../src/ffi/diskio_scope.rs | 0 .../src/hardware.rs | 0 .../src/hardware/hierarchy.rs | 30 +- .../src/hardware/hierarchy/disk_exclusion.rs | 0 .../src/hardware/hierarchy/disk_identity.rs | 0 .../src/hardware/hierarchy/disk_status.rs | 0 .../src/hardware/hierarchy/group_identity.rs | 0 .../src/hardware/hierarchy/legacy_cascade.rs | 0 .../src/hardware/hierarchy/legacy_media.rs | 0 .../src/hardware/hierarchy/mutation.rs | 6 +- .../src/hardware/hierarchy/owner_session.rs | 0 .../src/hardware/hierarchy/ownership.rs | 0 .../hardware/hierarchy/storage_activation.rs | 0 .../hardware/hierarchy/storage_admission.rs | 0 .../hierarchy/storage_compatibility.rs | 0 .../src/hardware/hierarchy/storage_guard.rs | 0 .../src/hardware/hierarchy/storage_health.rs | 0 .../hardware/hierarchy/storage_observation.rs | 2 +- .../src/hardware/hierarchy/storage_online.rs | 0 .../hierarchy/storage_online/completion.rs | 0 .../src/hardware/hierarchy/storage_page.rs | 0 .../src/hardware/hierarchy/storage_rejoin.rs | 0 .../src/hardware/hierarchy/storage_removal.rs | 0 .../hierarchy/storage_removal/conditions.rs | 0 .../hardware/hierarchy/storage_resource.rs | 0 .../hierarchy/storage_resource/authority.rs | 4 +- .../hierarchy/storage_resource/legacy.rs | 0 .../src/hardware/space_usage.rs | 0 .../src/hardware/sysmd.rs | 32 +- .../src/lib.rs | 16 +- .../src/membership.rs | 6 +- .../src/metrics.rs | 2 +- .../src/service.rs | 0 .../src/service/discovery.rs | 20 +- .../src/service/domain_monitor.rs | 6 +- .../src/service/registry.rs | 22 +- .../src/service/registry/diskio.rs | 0 .../src/service/registry/session.rs | 0 .../src/service/stream_lease.rs | 12 +- .../src/service/stream_lease/authority.rs | 7 + .../src/service/stream_lease/exclusion.rs | 52 + .../src/service/stream_lease/observation.rs | 2 +- .../src/service/watch_notify.rs | 20 +- .../src/transport.rs | 4 +- .../src/transport/cluster.rs | 56 +- .../src/transport/comparison.rs | 0 .../src/transport/rpc_transport.rs | 22 +- .../transport/rpc_transport/connections.rs | 4 +- .../src/transport/rpc_transport/owned.rs | 6 +- .../transport/rpc_transport/point_reads.rs | 4 +- .../tests/chunk_slot_initialization_test.rs | 107 + .../tests/chunk_slot_owner_fence_test.rs | 4 +- .../tests/chunk_slot_publication_test.rs | 6 +- .../tests/chunkdb_partition_test.rs | 6 +- .../tests/common/chunk_slot_publication.rs | 10 +- .../tests/common/conditional_servers.rs | 16 +- .../tests/common/owner_fence_server.rs | 12 +- .../tests/common/point_read_peer.rs | 0 .../tests/common/publication_peer.rs | 0 .../tests/common/slot_map.rs | 59 + .../tests/common/storage.rs | 8 +- .../tests/common/stream_lease.rs | 10 +- .../tests/common/stream_lease_clock.rs | 1 + .../tests/common/topology_recovery.rs | 10 +- .../tests/comparison_wire_test.rs | 36 +- .../tests/conditional_observation_test.rs | 4 +- .../tests/conditional_retry_test.rs | 20 +- .../tests/disk_identity_test.rs | 0 .../tests/disk_resource_test.rs | 2 +- .../tests/disk_status_test.rs | 10 +- .../tests/diskdb_session_test.rs | 2 +- .../tests/e2e_follower_read_test.rs | 46 +- .../tests/e2e_single_node_test.rs | 24 +- .../tests/ffi_exclusion_test.rs | 2 +- .../tests/group_identity_test.rs | 0 .../tests/journal_scan_test.rs | 6 +- .../tests/metrics_lock_test.rs | 2 +- .../tests/multi_seed_discovery_test.rs | 10 +- .../tests/point_read_isolation_test.rs | 22 +- .../tests/storage_admission_test.rs | 2 +- .../tests/storage_compatibility_test.rs | 2 +- .../tests/storage_exclusion_test.rs | 4 +- .../tests/storage_health_test.rs | 0 .../tests/storage_rejoin_test.rs | 2 +- .../tests/storage_removal_test.rs | 2 +- .../tests/stream_process_authority_test.rs | 2 +- .../tests/stream_process_exclusion_test.rs | 141 + .../tests/stream_process_lease_test.rs | 2 +- .../tests/stream_process_observation_test.rs | 2 +- .../tests/sysmd_client_test.rs | 10 +- .../tests/topology_publication_test.rs | 8 +- .../tests/topology_retry_discovery_test.rs | 2 +- lib/{crowdb-kv => crowdb-paxos-kv}/Cargo.toml | 32 +- .../benches/slot_list.rs | 6 +- .../benches/wal.rs | 12 +- .../benches/wal_bench_history.md | 0 lib/{crowdb-kv => crowdb-paxos-kv}/build.rs | 0 .../src/cluster.rs | 7 +- .../src/cluster/group.rs | 38 +- .../src/cluster/group_accept.rs | 0 .../src/cluster/group_cas.rs | 4 +- .../src/cluster/group_coalesce.rs | 68 +- .../src/cluster/group_config.rs | 0 .../src/cluster/group_election.rs | 0 .../src/cluster/group_election/recovery.rs | 0 .../src/cluster/group_election_candidate.rs | 0 .../src/cluster/group_election_leader.rs | 0 .../src/cluster/group_fetchgap.rs | 0 .../src/cluster/group_inflight.rs | 60 +- .../src/cluster/group_maintenance.rs | 127 +- .../src/cluster/group_membership.rs | 135 +- .../src/cluster/group_operations.rs | 9 +- .../src/cluster/group_owner_fence.rs | 2 +- .../src/cluster/group_prepare.rs | 0 .../src/cluster/group_propose.rs | 53 +- .../src/cluster/group_retention.rs | 0 .../src/cluster/group_routes.rs | 0 .../src/cluster/group_shutdown.rs | 59 + .../src/cluster/group_tests.rs | 0 .../src/cluster/kv_store.rs | 0 .../src/cluster/local_replica.rs | 33 +- .../src/cluster/local_replica_accept.rs | 8 + .../src/cluster/local_replica_apply.rs | 130 +- .../src/cluster/local_replica_election.rs | 0 .../src/cluster/local_replica_replay.rs | 29 +- .../src/cluster/local_replica_shutdown.rs | 76 + .../src/cluster/node_config.rs | 0 .../src/cluster/paxos_kv_server.rs} | 22 +- .../src/cluster/px_kv_store.rs | 46 +- .../src/cluster/px_kv_store_handler.rs | 0 .../src/cluster/px_kv_store_owner.rs | 0 .../src/cluster/remote_replica.rs | 0 .../src/cluster/replica.rs | 2 +- .../src/cluster/status.rs | 4 +- .../src/cluster/store_shutdown.rs | 66 + .../src/cluster/watch_registry.rs | 59 +- .../src/common.rs | 0 .../src/common/config.rs | 96 +- .../src/common/logging.rs | 18 +- .../src/common/report.rs | 2 +- .../src/common/time.rs | 2 +- lib/{crowdb-kv => crowdb-paxos-kv}/src/io.rs | 0 .../src/io/async_file.rs | 0 lib/{crowdb-kv => crowdb-paxos-kv}/src/kv.rs | 7 + lib/crowdb-paxos-kv/src/kv/admission.rs | 26 + .../src/kv/crowdb_tree_engine.rs | 267 +- .../src/kv/crowdb_tree_engine/admission.rs | 75 + .../src/kv/crowdb_tree_engine/maintenance.rs | 93 + .../src/kv/crowdb_tree_engine/metrics.rs | 122 + .../src/kv/kv_engine.rs | 66 +- .../src/kv/kv_future.rs | 10 +- lib/crowdb-paxos-kv/src/kv/maintenance.rs | 105 + .../src/kv/op.rs | 47 + lib/{crowdb-kv => crowdb-paxos-kv}/src/lib.rs | 2 +- .../src/metrics.rs | 2 +- .../src/paxos.rs | 0 .../src/paxos/acceptor.rs | 0 .../src/paxos/error.rs | 0 .../src/paxos/learner.rs | 98 +- .../src/paxos/learner/accepted_apply.rs | 43 + .../src/paxos/roles.rs | 0 .../src/paxos/slot_list.rs | 0 .../src/paxos/slot_node.rs | 0 lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc.rs | 6 +- .../src/rpc/paxos_kv_rpc_service.rs} | 26 +- .../rpc/paxos_kv_rpc_service}/comparison.rs | 0 .../rpc/paxos_kv_rpc_service}/owner_fence.rs | 0 .../response_correlation.rs | 0 .../src/rpc/paxos_kv_rpc_service}/watch.rs | 6 +- .../src/rpc/px_rpc_service.rs | 2 +- .../src/rpc/px_rpc_service/accept.rs | 0 .../src/rpc/px_rpc_service/election.rs | 0 .../src/rpc/px_rpc_service/heartbeat.rs | 0 .../src/rpc/px_rpc_service/notifications.rs | 0 .../src/rpc/px_rpc_service/prepare.rs | 0 .../src/rpc/px_rpc_service/responses.rs | 24 +- .../src/rpc/px_rpc_service/snapshots.rs | 0 .../src/rpc/px_rpc_transport.rs | 27 +- .../src/rpc/px_rpc_transport/connection.rs | 0 .../src/rpc/px_rpc_transport/heartbeat.rs | 0 .../src/rpc/snapshot_registry.rs | 2 +- lib/{crowdb-kv => crowdb-paxos-kv}/src/wal.rs | 0 .../src/wal/block_backend.rs | 0 .../src/wal/file_backend.rs | 0 .../src/wal/gc.rs | 0 .../src/wal/index.rs | 0 .../src/wal/io_backend.rs | 0 .../src/wal/pipeline.rs | 0 .../src/wal/pipeline_backend.rs | 0 .../src/wal/pipeline_writer.rs | 0 .../src/wal/record.rs | 0 .../src/wal/replay.rs | 0 .../src/wal/replay/records.rs | 0 .../src/wal/segment.rs | 0 .../src/wal/segment/read_buffer.rs | 0 .../src/wal/uring_backend.rs | 0 .../src/wal/wal_engine.rs | 0 .../src/wal/wal_file.rs | 0 .../tests/common.rs | 2 +- .../tests/common/cluster.rs | 16 +- .../tests/common/logging.rs | 4 +- .../tests/common/net_lock.rs | 2 +- .../tests/common/payload_owner.rs | 0 .../tests/common/simdisk.rs | 0 .../tests/common/test_client.rs | 30 +- .../tests/common/test_util.rs | 14 +- .../tests/common/timer.rs | 0 .../tests/election_test.rs | 0 .../tests/election_test/apply_loop_test.rs | 67 +- .../tests/election_test/frontier_test.rs | 6 +- .../tests/election_test/heartbeat_test.rs | 6 +- .../tests/election_test/lease_test.rs | 4 +- .../tests/election_test/metrics_test.rs | 8 +- .../tests/election_test/role_test.rs | 4 +- .../tests/election_test/step_down_test.rs | 4 +- .../tests/election_test/term_fence_test.rs | 4 +- .../tests/election_test/vote_test.rs | 8 +- .../tests/group_test.rs | 3 + .../a3_crash_restart_no_data_loss_test.rs | 12 +- .../tests/group_test/cas_test.rs | 4 +- .../tests/group_test/coalesce_test.rs | 117 +- .../tests/group_test/election_test.rs | 36 +- .../group_test/fetch_gap_durability_test.rs | 12 +- .../group_test/forward_correlation_test.rs | 14 +- .../group_test/full_restart_delete_test.rs | 24 +- .../group_test/g1_step_down_survival_test.rs | 4 +- .../g2_crash_restart_no_data_loss_test.rs | 22 +- .../tests/group_test/g3_leader_change_test.rs | 14 +- .../group_test/g4_learner_stream_test.rs | 2 +- .../tests/group_test/g5_recovery_test.rs | 4 +- .../tests/group_test/g6_reconfig_test.rs | 12 +- .../g7_reconfig_remove_leader_test.rs | 8 +- .../tests/group_test/group_config_test.rs | 2 +- .../tests/group_test/group_propose_test.rs | 69 +- .../tests/group_test/group_test.rs | 6 +- .../tests/group_test/kv_correctness_test.rs | 2 +- .../tests/group_test/kv_edge_case_test.rs | 2 +- .../tests/group_test/kv_forward_test.rs | 47 +- .../tests/group_test/kv_slot_retry_test.rs | 13 +- .../tests/group_test/kv_test.rs | 2 +- .../group_test/learner_heartbeat_test.rs | 6 +- .../tests/group_test/maintenance_test.rs | 28 +- .../group_test/membership_epoch_fence_test.rs | 4 +- .../tests/group_test/owner_fence_test.rs | 24 +- .../tests/group_test/paxos_error_test.rs | 17 +- .../tests/group_test/preemption_retry_test.rs | 13 +- .../tests/group_test/proposer_test.rs | 83 +- .../tests/group_test/quorum_drain_test.rs | 13 +- .../tests/group_test/r65_replication_test.rs | 4 +- .../tests/group_test/recovery_backlog_test.rs | 2 +- .../tests/group_test/remote_error_test.rs | 8 +- .../tests/group_test/retention_test.rs | 14 +- .../tests/group_test/safe_slot_test.rs | 6 +- .../tests/group_test/snapshot_join_test.rs | 47 +- .../tests/group_test/snapshot_slot_test.rs | 8 +- .../group_test/t1_early_ack_crash_test.rs | 24 +- .../tests/kv_test.rs | 3 + .../tests/kv_test/conformance_test.rs | 178 +- .../tests/kv_test/crowdb_tree_engine_test.rs | 274 +- .../tests/kv_test/kv_future_test.rs | 2 +- .../tests/kv_test/mem_kv_impl_test.rs | 7 +- .../tests/kv_test/mem_kv_test.rs | 79 +- .../tests/kv_test/op_codec_test.rs | 40 +- .../tests/kv_test/tree_work_metrics_test.rs | 49 + .../tests/metrics_test.rs | 16 +- .../tests/node_config_test.rs | 4 +- .../tests/paxos_test.rs | 12 + .../tests/paxos_test/acceptor_test.rs | 6 +- .../tests/paxos_test/cas_rejection_test.rs | 292 ++ .../tests/paxos_test/error_test.rs | 4 +- .../tests/paxos_test/learner_async_test.rs | 8 +- .../tests/paxos_test/learner_dedup_test.rs | 35 +- .../tests/paxos_test/learner_test.rs | 2 +- .../paxos_test/native_apply_window_test.rs | 165 + .../tests/paxos_test/roles_test.rs | 2 +- .../tests/paxos_test/runtime_gap_test.rs | 192 + .../paxos_test/runtime_shutdown_owner_test.rs | 32 + .../paxos_test/runtime_shutdown_rpc_test.rs | 165 + .../tests/paxos_test/runtime_shutdown_test.rs | 212 + .../tests/replay_compaction_memory_test.rs | 8 +- .../tests/replay_memory_test.rs | 8 +- .../tests/replica_test.rs | 0 .../tests/replica_test/concurrent_test.rs | 8 +- .../tests/replica_test/dedup_test.rs | 15 +- .../replica_test/election_metrics_test.rs | 2 +- .../tests/replica_test/election_test.rs | 12 +- .../tests/replica_test/lease_read_test.rs | 2 +- .../tests/replica_test/op_correctness_test.rs | 4 +- .../tests/replica_test/persistence_test.rs | 15 +- .../tests/replica_test/prepare_accept_test.rs | 6 +- .../replica_test/replay_ordering_test.rs | 34 +- .../tests/replica_test/snapshot_test.rs | 12 +- .../tests/rpc_migration_test.rs | 33 +- .../tests/rpc_notification_test.rs | 66 + .../tests/slot_test.rs | 0 .../tests/slot_test/concurrent_test.rs | 2 +- .../tests/slot_test/reclaim_test.rs | 2 +- .../tests/slot_test/slot_list_test.rs | 6 +- .../tests/store_test.rs | 6 +- .../tests/store_test/apply_fence_test.rs | 6 +- .../tests/store_test/bounded_scan_test.rs | 12 +- .../tests/store_test/group_operations_test.rs | 10 +- .../tests/store_test/health_test.rs | 10 +- .../tests/store_test/journal_recovery_test.rs | 30 +- .../tests/store_test/kv_correctness_test.rs | 8 +- .../tests/store_test/multi_group_test.rs | 10 +- .../store_test/multi_node_multi_group_test.rs | 10 +- .../tests/store_test/node_test.rs | 16 +- .../tests/store_test/persistence_test.rs | 16 +- .../tests/store_test/read_metrics_test.rs | 8 +- .../tests/store_test/readindex_batch_test.rs | 8 +- .../tests/store_test/shutdown_test.rs | 8 +- .../store_test/shutdown_under_load_test.rs | 8 +- .../tests/store_test/snapshot_api_test.rs | 52 +- .../tests/store_test/status_test.rs | 20 +- .../tests/store_test/wal_isolation_test.rs | 12 +- .../tests/tree_admission_test.rs | 256 + .../tests/tree_idle_writeback_test.rs | 142 + .../tests/tree_maintenance_test.rs | 324 ++ .../tests/tree_native_admission_test.rs | 153 + .../tests/tree_snapshot_owner_test.rs | 94 + .../tests/wal_test.rs | 0 .../tests/wal_test/block_backend_test.rs | 6 +- .../tests/wal_test/block_restore_test.rs | 10 +- .../tests/wal_test/block_runtime_test.rs | 6 +- .../tests/wal_test/file_backend_test.rs | 2 +- .../tests/wal_test/file_restore_test.rs | 10 +- .../tests/wal_test/gc_test.rs | 12 +- .../tests/wal_test/index_test.rs | 2 +- .../tests/wal_test/io_backend_test.rs | 2 +- .../tests/wal_test/pipeline_backend_test.rs | 4 +- .../tests/wal_test/pipeline_writer_test.rs | 8 +- .../tests/wal_test/record_test.rs | 4 +- .../tests/wal_test/replay_compaction_test.rs | 10 +- .../tests/wal_test/replay_test.rs | 103 +- .../tests/wal_test/segment_test.rs | 10 +- .../tests/wal_test/wal_engine_test.rs | 16 +- .../tests/watch_registry_test.rs | 41 + .../cpp/include/crowdb-protocol/flatbuffer.h | 27 + .../cpp/include/crowdb-protocol/ports.h | 28 + lib/crowdb-protocol/src/chunk_slot/map.rs | 2 +- .../src/chunk_stream/ownership.rs | 3 + lib/crowdb-protocol/src/fbs/chunkdb.fbs | 2 + lib/crowdb-protocol/src/fbs/diskio.fbs | 5 + lib/crowdb-protocol/src/fbs/kv_client.fbs | 2 +- lib/crowdb-protocol/src/key.rs | 4 +- lib/crowdb-protocol/src/key/chunkdb.rs | 2 +- lib/crowdb-protocol/src/key/encoding.rs | 4 +- lib/crowdb-protocol/src/key/kv_cluster.rs | 2 +- lib/crowdb-protocol/src/lib.rs | 6 +- lib/crowdb-protocol/src/mgmt.rs | 12 +- .../src/mgmt/node/deployment.rs | 37 +- lib/crowdb-protocol/src/mgmt/node/setup.rs | 2 - .../src/mgmt/node/setup/validation.rs | 10 - lib/crowdb-protocol/src/port/alloc.rs | 36 +- lib/crowdb-protocol/src/port/ports.rs | 165 +- lib/crowdb-protocol/src/sysdata.rs | 4 +- lib/crowdb-protocol/src/types/kv_client.rs | 2 +- lib/crowdb-protocol/src/types/kv_consensus.rs | 2 +- lib/crowdb-protocol/tests/fb_schema_test.rs | 2 + .../tests/node_deployment_test.rs | 14 +- lib/crowdb-protocol/tests/node_setup_test.rs | 10 +- lib/crowdb-protocol/tests/ports_test.rs | 126 +- .../tests/runtime_namespace_test.rs | 4 +- .../tests/stream_process_lease_test.rs | 10 + lib/crowdb-rpc/CMakeLists.txt | 17 +- lib/crowdb-rpc/cmake/folly-project.cmake | 6 + .../cmake/folly-thread-sanitizer.cmake | 37 + lib/crowdb-rpc/ffi/build.rs | 6 + lib/crowdb-rpc/ffi/src/buffer.rs | 27 +- lib/crowdb-rpc/ffi/src/buffer/owned.rs | 123 + lib/crowdb-rpc/ffi/src/lib.rs | 2 +- lib/crowdb-rpc/ffi/src/sys.rs | 3 + lib/crowdb-rpc/ffi/src/sys/owner.rs | 20 + .../ffi/tests/accounted_owner_test.rs | 101 + lib/crowdb-rpc/include/crowdb-rpc/buffer.h | 11 +- lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 2 + .../include/crowdb-rpc/c_api_internal.h | 6 + .../include/crowdb-rpc/connection.h | 5 +- .../include/crowdb-rpc/control_allocator.h | 58 + .../crowdb-rpc/control_allocator_c_api.h | 39 + .../crowdb-rpc/transport/socket_transport.h | 2 +- lib/crowdb-rpc/src/buffer.cpp | 29 +- lib/crowdb-rpc/src/buffer_arena.cpp | 26 +- lib/crowdb-rpc/src/buffer_arena.h | 9 +- lib/crowdb-rpc/src/c_api.cpp | 45 +- lib/crowdb-rpc/src/c_api/buffer_lease.cpp | 4 +- lib/crowdb-rpc/src/c_api/buffer_owner.cpp | 62 + lib/crowdb-rpc/src/c_api/frame.cpp | 4 +- lib/crowdb-rpc/src/c_api/producer.cpp | 4 +- lib/crowdb-rpc/src/connection.cpp | 12 + lib/crowdb-rpc/src/server/message.cpp | 7 +- lib/crowdb-rpc/src/server/server.cpp | 15 +- .../src/transport/rdma/rdma_connection.cpp | 83 +- .../src/transport/rdma/rdma_control_codec.cpp | 34 +- .../src/transport/socket_transport.cpp | 6 +- lib/crowdb-rpc/tests/buffer_test.cpp | 33 + lib/crowdb-rpc/tests/c_api_load_test.cpp | 4 +- .../tests/client_disconnect_test.cpp | 29 + lib/crowdb-rpc/tests/client_pool_test.cpp | 23 +- .../tests/common/test_rdma_control.cpp | 6 +- .../tests/control_allocator_test.cpp | 100 + lib/crowdb-rpc/tests/load_test.cpp | 3 +- lib/crowdb-rpc/tests/loopback_test.cpp | 3 +- lib/crowdb-rpc/tests/rdma_completion_test.cpp | 9 +- .../tests/rdma_control_codec_test.cpp | 83 +- .../tests/rdma_direct_hardware_test.cpp | 52 +- .../tests/rdma_listen_hardware_test.cpp | 38 + lib/crowdb-rpc/tests/rdma_test.cpp | 35 +- lib/crowdb-rpc/tests/rdma_timed_load_test.cpp | 8 +- lib/crowdb-rpc/tests/rdma_worker_test.cpp | 2 + lib/crowdb-rpc/tests/write_progress_test.cpp | 41 + lib/crowdb-sdk-native/Cargo.toml | 2 +- .../src/runtime/attachment.rs | 4 +- lib/crowdb-sdk-native/src/runtime/storage.rs | 8 +- lib/crowdb-sdk-native/src/topology.rs | 2 +- .../tests/common/dataset_transport.rs | 4 +- .../tests/common/read_storage.rs | 4 +- .../tests/common/routed_write_batch.rs | 4 +- .../tests/common/rpc_read.rs | 1 + lib/crowdb-sdk-native/tests/common/storage.rs | 4 +- .../tests/common/write_storage.rs | 4 +- lib/crowdb-tree/CMakeLists.txt | 25 +- lib/crowdb-tree/bench/README.md | 42 + .../bench/parallel_flush_profile.cpp | 358 ++ lib/crowdb-tree/cmake/prepare_stdexec.py | 46 + lib/crowdb-tree/cmake/stdexec-pool-base.json | 4 + lib/crowdb-tree/cmake/stdexec-pool.patch | 258 + lib/crowdb-tree/ffi/Cargo.toml | 7 +- lib/crowdb-tree/ffi/build.rs | 34 +- lib/crowdb-tree/ffi/src/admission.rs | 174 + lib/crowdb-tree/ffi/src/async_tree.rs | 163 +- .../ffi/src/async_tree/chunk_collection.rs | 67 + .../ffi/src/async_tree/compaction.rs | 84 + .../ffi/src/async_tree/maintenance.rs | 77 + .../ffi/src/async_tree/owned_get.rs | 41 + lib/crowdb-tree/ffi/src/async_tree/scan.rs | 38 + lib/crowdb-tree/ffi/src/backend_release.rs | 107 + lib/crowdb-tree/ffi/src/batch.rs | 114 +- lib/crowdb-tree/ffi/src/batch/allocation.rs | 162 + lib/crowdb-tree/ffi/src/chunk.rs | 116 +- lib/crowdb-tree/ffi/src/chunk/legacy.rs | 140 + lib/crowdb-tree/ffi/src/chunk/pin_service.rs | 140 + lib/crowdb-tree/ffi/src/chunk/protection.rs | 143 + .../ffi/src/chunk/protection_callbacks.rs | 147 + .../chunk/protection_callbacks/mutation.rs | 93 + lib/crowdb-tree/ffi/src/chunk/runtime.rs | 75 + lib/crowdb-tree/ffi/src/chunk/stage.rs | 138 + lib/crowdb-tree/ffi/src/chunk/stage/codec.rs | 139 + .../ffi/src/chunk/stage_callbacks.rs | 190 + lib/crowdb-tree/ffi/src/chunk/test_util.rs | 129 + lib/crowdb-tree/ffi/src/config.rs | 13 + lib/crowdb-tree/ffi/src/config/engine.rs | 100 + lib/crowdb-tree/ffi/src/config/runtime.rs | 162 + lib/crowdb-tree/ffi/src/immutable_snapshot.rs | 119 + lib/crowdb-tree/ffi/src/lib.rs | 47 +- lib/crowdb-tree/ffi/src/maintenance.rs | 96 + lib/crowdb-tree/ffi/src/owner_budget.rs | 56 + lib/crowdb-tree/ffi/src/reactor.rs | 134 +- lib/crowdb-tree/ffi/src/runtime.rs | 178 + lib/crowdb-tree/ffi/src/runtime/stats.rs | 63 + lib/crowdb-tree/ffi/src/runtime/usage.rs | 118 + .../ffi/src/runtime/usage/window.rs | 212 + lib/crowdb-tree/ffi/src/scan.rs | 155 +- lib/crowdb-tree/ffi/src/scan/options.rs | 30 + lib/crowdb-tree/ffi/src/scan/owner.rs | 120 + lib/crowdb-tree/ffi/src/snapshot.rs | 9 + lib/crowdb-tree/ffi/src/snapshot/owner.rs | 62 + lib/crowdb-tree/ffi/src/stats.rs | 30 +- lib/crowdb-tree/ffi/src/stats/backing.rs | 40 + lib/crowdb-tree/ffi/src/stats/state.rs | 163 + lib/crowdb-tree/ffi/src/stats/work.rs | 261 + .../ffi/src/stats/work/counters.rs | 106 + .../ffi/src/stats/work/histograms.rs | 115 + lib/crowdb-tree/ffi/src/stats/work/waits.rs | 77 + lib/crowdb-tree/ffi/src/sys.rs | 202 +- lib/crowdb-tree/ffi/src/sys/admission.rs | 31 + .../ffi/src/sys/backend_release.rs | 31 + lib/crowdb-tree/ffi/src/sys/backing_stats.rs | 18 + .../ffi/src/sys/chunk_migration.rs | 12 + .../ffi/src/sys/chunk_pin_service.rs | 40 + .../ffi/src/sys/chunk_protection.rs | 83 + lib/crowdb-tree/ffi/src/sys/chunk_runtime.rs | 49 + lib/crowdb-tree/ffi/src/sys/chunk_stage.rs | 63 + lib/crowdb-tree/ffi/src/sys/compaction.rs | 33 + .../ffi/src/sys/immutable_snapshot.rs | 34 + lib/crowdb-tree/ffi/src/sys/maintenance.rs | 57 + .../ffi/src/sys/owned_allocation.rs | 40 + lib/crowdb-tree/ffi/src/sys/owned_budget.rs | 21 + lib/crowdb-tree/ffi/src/sys/runtime.rs | 128 + lib/crowdb-tree/ffi/src/sys/scan_owner.rs | 57 + lib/crowdb-tree/ffi/src/sys/snapshot_owner.rs | 24 + lib/crowdb-tree/ffi/src/sys/split_overlay.rs | 51 + lib/crowdb-tree/ffi/src/sys/state_stats.rs | 89 + lib/crowdb-tree/ffi/src/sys/wait_metrics.rs | 32 + .../ffi/src/sys/work_histograms.rs | 47 + lib/crowdb-tree/ffi/src/sys/work_metrics.rs | 100 + lib/crowdb-tree/ffi/src/tree.rs | 119 +- lib/crowdb-tree/ffi/src/tree/split_overlay.rs | 117 + lib/crowdb-tree/ffi/src/tuning.rs | 98 + lib/crowdb-tree/ffi/src/write_handle.rs | 25 +- .../ffi/tests/backend_release_test.rs | 38 + .../ffi/tests/backing_stats_test.rs | 42 + .../ffi/tests/chunk_pin_service_test.rs | 28 + lib/crowdb-tree/ffi/tests/chunk_stage_test.rs | 101 + .../ffi/tests/common/external_owner.rs | 31 + .../ffi/tests/dispatch_metrics_test.rs | 40 + .../ffi/tests/external_owner_test.rs | 130 + lib/crowdb-tree/ffi/tests/ffi_test.rs | 652 +-- .../ffi/tests/immutable_snapshot_test.rs | 107 + .../ffi/tests/legacy_migration_test.rs | 56 + .../ffi/tests/owned_allocation_test.rs | 166 + .../ffi/tests/page_inspection_test.rs | 1 + .../ffi/tests/resource_usage_test.rs | 271 + .../ffi/tests/runtime_admission_test.rs | 133 + .../ffi/tests/runtime_async_scan_test.rs | 97 + .../ffi/tests/runtime_bytes_owner_test.rs | 80 + .../ffi/tests/runtime_compaction_test.rs | 77 + .../ffi/tests/runtime_maintenance_test.rs | 78 + .../ffi/tests/runtime_replay_test.rs | 140 + .../ffi/tests/runtime_scan_owner_test.rs | 68 + .../ffi/tests/runtime_snapshot_owner_test.rs | 46 + .../tests/runtime_source_retention_test.rs | 32 + .../ffi/tests/runtime_stats_test.rs | 52 + lib/crowdb-tree/ffi/tests/runtime_test.rs | 127 + .../ffi/tests/runtime_wait_metrics_test.rs | 19 + .../ffi/tests/runtime_work_histograms_test.rs | 69 + .../ffi/tests/runtime_work_metrics_test.rs | 77 + lib/crowdb-tree/ffi/tests/split_hint_test.rs | 12 +- .../include/crowdb-tree/apply_c_api.h | 31 + .../crowdb-tree/backend/async_page_store.h | 27 +- .../backend/backend_release_service.h | 93 + .../crowdb-tree/backend/block_page_store.h | 166 +- .../include/crowdb-tree/backend/page_store.h | 50 +- .../crowdb-tree/backend_release_c_api.h | 32 + .../include/crowdb-tree/backing_stats_c_api.h | 30 + .../include/crowdb-tree/btree/diagnostics.h | 117 +- .../include/crowdb-tree/btree/range_rebuild.h | 10 +- .../include/crowdb-tree/btree/read_result.h | 121 +- .../include/crowdb-tree/btree/scan_packed.h | 76 +- .../include/crowdb-tree/btree/tree.h | 795 +-- .../include/crowdb-tree/btree/work_cause.h | 21 + lib/crowdb-tree/include/crowdb-tree/buffer.h | 36 +- lib/crowdb-tree/include/crowdb-tree/c_api.h | 177 +- .../crowdb-tree/chunk_migration_c_api.h | 18 + .../crowdb-tree/chunk_pin_service_c_api.h | 32 + .../crowdb-tree/chunk_protection_c_api.h | 63 + .../include/crowdb-tree/chunk_runtime_c_api.h | 52 + .../include/crowdb-tree/chunk_stage_c_api.h | 41 + lib/crowdb-tree/include/crowdb-tree/config.h | 114 +- lib/crowdb-tree/include/crowdb-tree/epoch.h | 102 +- .../include/crowdb-tree/external_allocation.h | 67 + .../crowdb-tree/immutable_snapshot_c_api.h | 26 + .../include/crowdb-tree/maintenance_c_api.h | 88 + .../include/crowdb-tree/maptable/compressor.h | 9 + .../include/crowdb-tree/maptable/frame_page.h | 98 +- .../crowdb-tree/maptable/mapping_geometry.h | 14 + .../crowdb-tree/maptable/mapping_persist.h | 27 +- .../crowdb-tree/maptable/mapping_slot.h | 27 +- .../include/crowdb-tree/maptable/page.h | 136 +- .../include/crowdb-tree/memory_budget.h | 260 + .../include/crowdb-tree/memtable/allocation.h | 6 +- .../include/crowdb-tree/memtable/generation.h | 102 - .../include/crowdb-tree/memtable/memtable.h | 120 +- .../include/crowdb-tree/memtable/skip_list.h | 211 +- .../include/crowdb-tree/memtable/source.h | 126 - .../crowdb-tree/memtable/source_stats.h | 66 + .../crowdb-tree/owned_allocation_c_api.h | 49 + .../include/crowdb-tree/owned_budget_c_api.h | 36 + .../include/crowdb-tree/resource_usage.h | 27 + .../include/crowdb-tree/runtime_c_api.h | 117 + .../crowdb-tree/snapshot/native_frame.h | 66 +- .../include/crowdb-tree/snapshot/prepared.h | 121 - .../include/crowdb-tree/snapshot/snapshot.h | 101 +- .../crowdb-tree/snapshot/snapshot_io.h | 94 +- .../include/crowdb-tree/split_overlay_c_api.h | 57 + lib/crowdb-tree/include/crowdb-tree/status.h | 5 + .../crowdb-tree/tree_execution_options.h | 30 + .../include/crowdb-tree/tree_runtime.h | 130 + .../include/crowdb-tree/tree_stats_c_api.h | 92 + .../crowdb-tree/tree_wait_metrics_c_api.h | 48 + .../crowdb-tree/tree_work_histograms_c_api.h | 49 + .../crowdb-tree/tree_work_metrics_c_api.h | 110 + .../include/crowdb-tree/tuning_c_api.h | 39 + .../src/backend/backend_release_c_api.cpp | 102 + .../src/backend/backend_release_service.cpp | 153 + lib/crowdb-tree/src/backend/backing_io.cpp | 54 + lib/crowdb-tree/src/backend/backing_io.h | 16 + .../src/backend/checkpoint_backend.h | 70 + .../chunk/chunk_backend_checkpoint.cpp | 86 + .../src/backend/chunk/chunk_backend_state.cpp | 96 + .../src/backend/chunk/chunk_backend_state.h | 62 + .../src/backend/chunk/chunk_backing.cpp | 189 + .../src/backend/chunk/chunk_backing.h | 70 + .../src/backend/chunk/chunk_catalog_scan.cpp | 108 + .../chunk/chunk_collection_operation.cpp | 112 + .../chunk/chunk_collection_operation.h | 33 + .../src/backend/chunk/chunk_collector.cpp | 179 + .../src/backend/chunk/chunk_collector.h | 45 + .../src/backend/chunk/chunk_fixed_pack.cpp | 193 + .../src/backend/chunk/chunk_fixed_pack.h | 72 + .../backend/chunk/chunk_fixed_pack_codec.cpp | 79 + .../src/backend/chunk/chunk_io_context.h | 23 + .../backend/chunk/chunk_legacy_runtime.cpp | 13 + .../backend/chunk/chunk_migration_c_api.cpp | 14 + .../src/backend/chunk/chunk_pack_pipeline.cpp | 2 +- .../src/backend/chunk/chunk_page_store.cpp | 40 +- .../src/backend/chunk/chunk_page_store.h | 146 +- .../backend/chunk/chunk_physical_registry.cpp | 180 + .../backend/chunk/chunk_physical_registry.h | 76 + .../chunk/chunk_physical_registry_api.cpp | 92 + .../chunk/chunk_physical_registry_internal.h | 65 + .../src/backend/chunk/chunk_pin_release.cpp | 220 + .../src/backend/chunk/chunk_pin_release.h | 96 + .../chunk/chunk_pin_release_internal.h | 20 + .../backend/chunk/chunk_pin_service_c_api.cpp | 129 + .../src/backend/chunk/chunk_protection.h | 79 + .../backend/chunk/chunk_protection_c_api.cpp | 39 + .../chunk/chunk_protection_callback.cpp | 213 + .../backend/chunk/chunk_protection_callback.h | 72 + .../chunk/chunk_protection_mutation.cpp | 108 + .../chunk/chunk_protocol_operation.cpp | 181 + .../backend/chunk/chunk_protocol_operation.h | 35 + .../src/backend/chunk/chunk_runtime_c_api.cpp | 52 + .../src/backend/chunk/chunk_runtime_ids.cpp | 52 + .../src/backend/chunk/chunk_runtime_ids.h | 26 + .../chunk/chunk_runtime_page_store.cpp | 110 + .../backend/chunk/chunk_runtime_page_store.h | 103 + .../src/backend/chunk/chunk_stage.cpp | 79 + .../src/backend/chunk/chunk_stage.h | 60 + .../src/backend/chunk/chunk_stage_c_api.cpp | 151 + .../backend/chunk/chunk_stage_callback.cpp | 174 + .../src/backend/chunk/chunk_stage_codec.cpp | 152 + .../src/backend/chunk/chunk_stage_gc.cpp | 123 + .../src/backend/chunk/chunk_stage_gc.h | 26 + .../src/backend/chunk/chunk_transport.h | 83 +- .../chunk/chunk_transport_registry.cpp | 167 + .../src/backend/chunk/legacy_chunk_layout.cpp | 136 + .../src/backend/chunk/legacy_chunk_layout.h | 37 + .../chunk/legacy_chunk_migration_receipt.cpp | 59 + .../backend/chunk/legacy_chunk_read_store.cpp | 199 + .../backend/chunk/legacy_chunk_read_store.h | 74 + .../backend/chunk/legacy_chunk_recovery.cpp | 147 + .../src/backend/chunk/legacy_chunk_recovery.h | 13 + .../src/backend/chunk/rpc_chunk_allocate.cpp | 74 + .../chunk/rpc_chunk_allocation_identity.cpp | 65 + .../src/backend/chunk/rpc_chunk_delete.cpp | 47 + .../src/backend/chunk/rpc_chunk_drain.cpp | 102 + .../src/backend/chunk/rpc_chunk_growth.cpp | 46 - .../backend/chunk/rpc_chunk_read_fence.cpp | 57 + .../src/backend/chunk/rpc_chunk_result.h | 8 +- .../src/backend/chunk/rpc_chunk_transport.cpp | 45 +- .../src/backend/chunk/rpc_chunk_transport.h | 21 + .../chunk/rpc_chunk_transport_internal.h | 20 +- .../backend/local/backend_io_allocation.cpp | 35 + .../src/backend/local/backend_io_allocation.h | 10 + .../src/backend/local/backend_metadata.cpp | 70 + .../src/backend/local/backend_metadata.h | 34 + .../src/backend/local/block_account.cpp | 192 + .../src/backend/local/block_account.h | 57 + .../src/backend/local/block_async_io.cpp | 233 + .../src/backend/local/block_async_io.h | 21 + .../backend/local/block_async_page_store.cpp | 379 +- .../src/backend/local/block_directory.cpp | 293 ++ .../src/backend/local/block_directory.h | 101 + .../backend/local/block_directory_release.cpp | 65 + .../src/backend/local/block_file_close.cpp | 67 + .../src/backend/local/block_medium.cpp | 232 + .../backend/local/block_operation_budget.cpp | 48 + .../src/backend/local/block_page_store.cpp | 801 +-- .../local/block_page_store_directory.cpp | 345 ++ .../src/backend/local/block_page_store_io.cpp | 323 ++ .../src/backend/local/extent_ledger.cpp | 224 + .../src/backend/local/extent_ledger.h | 97 + .../backend/local/extent_ledger_allocate.cpp | 135 + .../src/backend/local/extent_ledger_apply.cpp | 74 + .../backend/local/extent_ledger_exclusion.cpp | 78 + .../backend/local/extent_ledger_internal.h | 114 + .../src/backend/local/extent_writeback.cpp | 33 + .../src/backend/local/extent_writeback.h | 14 + .../src/backend/local/local_backend_state.cpp | 36 + .../src/backend/local/local_backend_state.h | 52 + .../src/backend/local/local_block_gc.cpp | 106 + .../src/backend/local/local_block_gc.h | 38 + .../src/backend/local/local_sparse_gc.cpp | 59 + .../src/backend/local/local_sparse_gc.h | 23 + .../src/backend/local/memory_backing_test.h | 14 + .../src/backend/local/page_store.cpp | 142 +- .../backend/local/scheduled_page_store.cpp | 321 ++ .../src/backend/local/scheduled_page_store.h | 51 + .../src/backend/pack_allocation_bound.cpp | 46 + lib/crowdb-tree/src/backend/pack_backing.cpp | 71 + .../src/backend/pack_physical_bound.cpp | 20 + lib/crowdb-tree/src/btree/admitted_batch.cpp | 229 + lib/crowdb-tree/src/btree/admitted_batch.h | 75 + .../src/btree/admitted_batch_plan.cpp | 111 + .../src/btree/allocation_account.cpp | 188 + .../src/btree/allocation_account.h | 97 + .../src/btree/allocation_account_control.cpp | 125 + .../src/btree/allocation_account_credit.cpp | 72 + .../src/btree/allocation_account_grant.cpp | 129 + .../src/btree/allocation_account_grant.h | 46 + lib/crowdb-tree/src/btree/allocation_bound.h | 71 + .../src/btree/allocation_observation.cpp | 57 + .../src/btree/allocation_reservation.cpp | 105 + .../src/btree/allocation_reservation.h | 155 + .../src/btree/apply_dispatcher.cpp | 125 + lib/crowdb-tree/src/btree/apply_dispatcher.h | 61 + lib/crowdb-tree/src/btree/apply_frontier.cpp | 115 + lib/crowdb-tree/src/btree/apply_frontier.h | 49 + lib/crowdb-tree/src/btree/apply_progress.cpp | 167 + lib/crowdb-tree/src/btree/apply_progress.h | 63 + lib/crowdb-tree/src/btree/apply_waiters.cpp | 29 + lib/crowdb-tree/src/btree/apply_waiters.h | 32 + .../src/btree/backing_identity.cpp | 20 + lib/crowdb-tree/src/btree/backing_identity.h | 10 + lib/crowdb-tree/src/btree/backing_metrics.cpp | 41 + lib/crowdb-tree/src/btree/backing_metrics.h | 32 + lib/crowdb-tree/src/btree/backing_ref.h | 96 + .../src/btree/backing_stats_sampler.cpp | 17 + .../src/btree/backing_stats_sampler.h | 20 + .../src/btree/budget_memory_resource.cpp | 62 + .../src/btree/budget_memory_resource.h | 52 + .../src/btree/checkpoint_metrics.cpp | 237 + lib/crowdb-tree/src/btree/crowdb-tree.cpp | 4361 +---------------- lib/crowdb-tree/src/btree/entry_key_arena.h | 40 + lib/crowdb-tree/src/btree/flush_batch.cpp | 238 + lib/crowdb-tree/src/btree/flush_batch.h | 107 + .../src/btree/flush_batch_budget.cpp | 111 + .../src/btree/flush_batch_dispatch.cpp | 32 + .../src/btree/flush_batch_handoff.cpp | 324 ++ .../src/btree/flush_batch_internal.h | 177 + .../src/btree/flush_batch_route.cpp | 176 + lib/crowdb-tree/src/btree/flush_batch_run.cpp | 87 + .../src/btree/flush_controller.cpp | 291 ++ .../src/btree/flush_controller_covered.cpp | 63 + .../src/btree/flush_controller_internal.h | 50 + .../src/btree/flush_controller_pause.cpp | 28 + .../src/btree/flush_controller_plan.cpp | 79 + .../src/btree/flush_controller_test.h | 12 + .../src/btree/flush_dispatch_metrics.h | 85 + lib/crowdb-tree/src/btree/flush_operation.cpp | 384 ++ lib/crowdb-tree/src/btree/flush_operation.h | 128 + .../src/btree/flush_operation_admission.cpp | 69 + .../src/btree/flush_operation_internal.h | 117 + .../src/btree/flush_operation_stages.cpp | 230 + .../src/btree/flush_operation_wait.cpp | 62 + .../src/btree/flush_range_join.cpp | 180 + .../src/btree/flush_report_sender.h | 82 + .../src/btree/flush_reservation.cpp | 163 + lib/crowdb-tree/src/btree/flush_reservation.h | 81 + .../src/btree/flush_reservation_internal.h | 16 + .../src/btree/flush_reserve_bound.cpp | 148 + lib/crowdb-tree/src/btree/flush_runtime.cpp | 185 + lib/crowdb-tree/src/btree/flush_runtime.h | 165 + .../src/btree/flush_runtime_internal.h | 193 + .../src/btree/flush_runtime_progress.cpp | 159 + .../src/btree/generation_replace.cpp | 217 +- lib/crowdb-tree/src/btree/ingest.cpp | 230 +- lib/crowdb-tree/src/btree/inline_function.h | 135 + lib/crowdb-tree/src/btree/inspect.cpp | 75 +- .../src/btree/local_compaction_internal.h | 71 + .../src/btree/local_compaction_operation.cpp | 181 + .../src/btree/local_compaction_operation.h | 46 + .../src/btree/local_compaction_steps.cpp | 133 + lib/crowdb-tree/src/btree/logical_page.cpp | 349 ++ lib/crowdb-tree/src/btree/logical_page.h | 374 ++ .../src/btree/logical_page_cursor.cpp | 188 + .../src/btree/logical_page_cursor_window.cpp | 85 + .../src/btree/maintenance_cold.cpp | 212 + .../src/btree/maintenance_cold_siblings.cpp | 55 + .../src/btree/maintenance_inner_merge.cpp | 97 + .../src/btree/maintenance_leaf_merge.cpp | 59 + .../src/btree/maintenance_operation.cpp | 393 ++ .../src/btree/maintenance_operation.h | 68 + .../btree/maintenance_operation_internal.h | 152 + .../src/btree/maintenance_operation_step.cpp | 368 ++ .../src/btree/maintenance_publication.cpp | 256 + .../src/btree/maintenance_publication.h | 75 + .../btree/maintenance_publication_internal.h | 83 + .../btree/maintenance_publication_state.cpp | 199 + .../src/btree/maintenance_reserve.cpp | 172 + .../src/btree/maintenance_routing.cpp | 226 + .../src/btree/maintenance_split_layout.cpp | 126 + .../src/btree/maintenance_split_layout.h | 67 + .../src/btree/maintenance_split_step.cpp | 207 + lib/crowdb-tree/src/btree/memtable_flush.cpp | 268 +- .../src/btree/memtable_sources.cpp | 103 +- lib/crowdb-tree/src/btree/memtable_split.cpp | 208 +- .../src/btree/merge_pair_metrics.cpp | 310 ++ .../src/btree/merge_pair_metrics.h | 126 + lib/crowdb-tree/src/btree/merge_sources.h | 12 +- lib/crowdb-tree/src/btree/native_frames.h | 255 - lib/crowdb-tree/src/btree/native_import.h | 44 + .../src/btree/native_import_dependencies.cpp | 49 + .../src/btree/native_import_frames.cpp | 183 + lib/crowdb-tree/src/btree/native_iterator.cpp | 403 +- .../src/btree/native_iterator_owned.cpp | 21 + lib/crowdb-tree/src/btree/overflow_backing.h | 41 + lib/crowdb-tree/src/btree/owned_key.cpp | 98 + lib/crowdb-tree/src/btree/owned_key.h | 121 + lib/crowdb-tree/src/btree/page_cache.cpp | 298 ++ lib/crowdb-tree/src/btree/page_cache.h | 108 + .../src/btree/page_cache_frame.cpp | 69 + lib/crowdb-tree/src/btree/page_cache_frame.h | 43 + .../src/btree/page_cache_identity.h | 48 + .../src/btree/page_cache_internal.h | 124 + lib/crowdb-tree/src/btree/page_cache_load.cpp | 342 ++ .../src/btree/page_cache_queue.cpp | 103 + .../src/btree/page_cache_residency.cpp | 195 + .../src/btree/page_cache_residency.h | 56 + .../src/btree/page_cache_residency_apply.cpp | 243 + .../src/btree/page_cache_residency_internal.h | 157 + .../btree/page_cache_residency_prepare.cpp | 83 + lib/crowdb-tree/src/btree/page_cache_trim.cpp | 63 + lib/crowdb-tree/src/btree/page_encode.cpp | 178 + lib/crowdb-tree/src/btree/page_encode.h | 156 + .../src/btree/page_encode_frame.cpp | 103 + .../src/btree/page_encode_internal.h | 53 + .../src/btree/page_encode_leaf.cpp | 139 + .../src/btree/page_encode_plan.cpp | 70 + lib/crowdb-tree/src/btree/page_key_range.h | 23 + lib/crowdb-tree/src/btree/page_layout.cpp | 115 + .../src/btree/page_layout_stats.cpp | 24 + lib/crowdb-tree/src/btree/page_load.cpp | 197 + lib/crowdb-tree/src/btree/page_load.h | 102 + lib/crowdb-tree/src/btree/page_load_frame.cpp | 35 + lib/crowdb-tree/src/btree/page_load_inner.cpp | 224 + .../src/btree/page_load_internal.h | 119 + lib/crowdb-tree/src/btree/page_load_leaf.cpp | 170 + lib/crowdb-tree/src/btree/page_load_leaf.h | 16 + .../src/btree/page_load_lifetime.h | 186 + .../src/btree/page_load_overflow.cpp | 273 ++ .../src/btree/page_maintenance.cpp | 199 + lib/crowdb-tree/src/btree/page_maintenance.h | 135 + .../src/btree/page_maintenance_structure.cpp | 193 + .../src/btree/page_split_operation.cpp | 105 + .../src/btree/page_split_operation.h | 46 + lib/crowdb-tree/src/btree/page_writeback.cpp | 219 + lib/crowdb-tree/src/btree/page_writeback.h | 78 + .../src/btree/page_writeback_encode.cpp | 163 + .../src/btree/page_writeback_internal.h | 61 + .../src/btree/pending_page_times.cpp | 98 + .../src/btree/pending_page_times.h | 42 + lib/crowdb-tree/src/btree/portable_import.cpp | 195 + lib/crowdb-tree/src/btree/portable_import.h | 60 + .../src/btree/portable_import_frames.cpp | 107 + .../src/btree/portable_import_plan.cpp | 77 + .../src/btree/portable_import_plan.h | 35 + .../src/btree/prepared_split_arm.cpp | 118 + .../src/btree/prepared_split_install.cpp | 146 + .../src/btree/prepared_split_internal.h | 33 + .../src/btree/prepared_split_overlay.h | 49 + .../src/btree/publication_admission.cpp | 49 + .../src/btree/publication_admission.h | 11 + .../src/btree/publication_bootstrap.cpp | 121 + .../src/btree/publication_checkpoint.cpp | 41 + .../src/btree/publication_checkpoint.h | 12 + .../publication_checkpoint_completion.cpp | 27 + .../src/btree/publication_commit.cpp | 479 ++ .../src/btree/publication_complete_view.cpp | 39 + .../src/btree/publication_generation.cpp | 105 + .../btree/publication_generation_internal.h | 46 + .../src/btree/publication_lane.cpp | 52 + lib/crowdb-tree/src/btree/publication_lane.h | 31 + .../src/btree/publication_maintenance.cpp | 41 + .../src/btree/publication_reconcile.cpp | 22 + .../src/btree/publication_snapshot.cpp | 157 + .../src/btree/publication_snapshot.h | 16 + .../src/btree/published_inspection.cpp | 292 ++ .../src/btree/published_inspection.h | 11 + .../src/btree/published_mapping.cpp | 262 + .../src/btree/published_mapping_visit.cpp | 69 + lib/crowdb-tree/src/btree/published_read.cpp | 194 + lib/crowdb-tree/src/btree/published_read.h | 109 + .../src/btree/published_read_budget.cpp | 42 + .../src/btree/published_read_internal.h | 65 + .../src/btree/published_read_pump.cpp | 193 + .../src/btree/published_read_route.cpp | 154 + lib/crowdb-tree/src/btree/published_route.h | 93 + lib/crowdb-tree/src/btree/published_scan.cpp | 275 ++ .../src/btree/published_scan_internal.h | 51 + .../src/btree/published_scan_sources.cpp | 149 + lib/crowdb-tree/src/btree/published_state.cpp | 259 + lib/crowdb-tree/src/btree/published_state.h | 635 +++ lib/crowdb-tree/src/btree/range_rebuild.cpp | 437 +- lib/crowdb-tree/src/btree/read_value_bound.h | 22 + .../src/btree/recovered_frame_identity.h | 22 + lib/crowdb-tree/src/btree/replay_controls.cpp | 117 + lib/crowdb-tree/src/btree/replay_controls.h | 51 + lib/crowdb-tree/src/btree/resource_budget.cpp | 824 ++++ lib/crowdb-tree/src/btree/resource_budget.h | 266 + .../src/btree/resource_observation.cpp | 43 + .../src/btree/resource_observation.h | 29 + lib/crowdb-tree/src/btree/retired_shared.h | 77 + .../src/btree/retirement_queue.cpp | 211 + lib/crowdb-tree/src/btree/retirement_queue.h | 54 + lib/crowdb-tree/src/btree/retirement_traits.h | 24 + .../src/btree/runtime_range_native.cpp | 129 + .../src/btree/runtime_range_portable.cpp | 115 + .../src/btree/runtime_range_rebuild.cpp | 80 + .../src/btree/runtime_range_rebuild.h | 32 + lib/crowdb-tree/src/btree/scan_output.h | 24 + .../src/btree/snapshot_changes.cpp | 181 + lib/crowdb-tree/src/btree/snapshot_changes.h | 103 + .../src/btree/source_admission.cpp | 687 +++ lib/crowdb-tree/src/btree/source_admission.h | 276 ++ .../src/btree/source_admission_internal.h | 55 + lib/crowdb-tree/src/btree/source_capture.cpp | 137 + lib/crowdb-tree/src/btree/source_pause.cpp | 52 + lib/crowdb-tree/src/btree/source_pause.h | 55 + lib/crowdb-tree/src/btree/source_rotation.cpp | 318 ++ lib/crowdb-tree/src/btree/source_rotation.h | 173 + .../src/btree/source_rotation_prepare.cpp | 76 + .../src/btree/source_rotation_wait.cpp | 186 + lib/crowdb-tree/src/btree/source_tail.cpp | 146 + lib/crowdb-tree/src/btree/source_tail.h | 112 + .../src/btree/source_tail_domain.cpp | 124 + .../src/btree/source_write_budget.cpp | 200 + .../src/btree/source_write_budget.h | 137 + .../src/btree/source_write_pause.cpp | 82 + .../src/btree/source_write_prepaid.cpp | 28 + lib/crowdb-tree/src/btree/source_writer.cpp | 105 + .../src/btree/source_writer_prepared.cpp | 59 + lib/crowdb-tree/src/btree/split_hint.cpp | 159 +- .../src/btree/tree_apply_operation.cpp | 126 + .../src/btree/tree_apply_operation.h | 47 + .../src/btree/tree_apply_operation_internal.h | 155 + .../src/btree/tree_apply_operation_steps.cpp | 187 + .../src/btree/tree_apply_operation_wait.cpp | 109 + .../src/btree/tree_apply_pressure.cpp | 87 + .../src/btree/tree_apply_replay.cpp | 167 + .../src/btree/tree_checkpoint_diagnostics.cpp | 21 + lib/crowdb-tree/src/btree/tree_core.cpp | 369 ++ lib/crowdb-tree/src/btree/tree_core.h | 200 + lib/crowdb-tree/src/btree/tree_core_access.h | 35 + .../src/btree/tree_core_admitted.cpp | 58 + lib/crowdb-tree/src/btree/tree_core_apply.cpp | 136 + lib/crowdb-tree/src/btree/tree_core_batch.cpp | 292 ++ lib/crowdb-tree/src/btree/tree_core_batch.h | 37 + .../src/btree/tree_core_block_gc.cpp | 84 + .../src/btree/tree_core_block_gc_async.cpp | 239 + .../src/btree/tree_core_checkpoint.cpp | 112 + .../src/btree/tree_core_checkpoint_async.cpp | 115 + .../src/btree/tree_core_checkpoint_budget.cpp | 212 + .../src/btree/tree_core_checkpoint_internal.h | 107 + .../btree/tree_core_checkpoint_structure.cpp | 84 + .../btree/tree_core_checkpoint_work_async.cpp | 135 + .../src/btree/tree_core_chunk_gc.cpp | 66 + .../src/btree/tree_core_chunk_migration.cpp | 121 + .../src/btree/tree_core_chunk_open.cpp | 310 ++ .../src/btree/tree_core_compact.cpp | 72 + lib/crowdb-tree/src/btree/tree_core_evict.cpp | 252 + .../src/btree/tree_core_facade.cpp | 59 + lib/crowdb-tree/src/btree/tree_core_flush.cpp | 180 + .../src/btree/tree_core_generation.cpp | 33 + .../src/btree/tree_core_geometry.cpp | 41 + .../src/btree/tree_core_geometry.h | 11 + lib/crowdb-tree/src/btree/tree_core_get.cpp | 152 + .../src/btree/tree_core_histograms.cpp | 41 + .../src/btree/tree_core_import.cpp | 184 + .../src/btree/tree_core_internal.h | 91 + .../src/btree/tree_core_lifecycle.cpp | 238 + .../src/btree/tree_core_metrics.cpp | 163 + lib/crowdb-tree/src/btree/tree_core_open.cpp | 238 + .../src/btree/tree_core_pressure.cpp | 80 + .../src/btree/tree_core_read_budget.cpp | 114 + .../src/btree/tree_core_read_internal.h | 35 + lib/crowdb-tree/src/btree/tree_core_scan.cpp | 150 + .../src/btree/tree_core_scan_budget.cpp | 159 + .../src/btree/tree_core_scan_internal.h | 56 + .../src/btree/tree_core_scan_sync.cpp | 109 + .../src/btree/tree_core_snapshot.cpp | 47 + .../src/btree/tree_core_snapshot_ids.cpp | 48 + .../src/btree/tree_core_source_stats.cpp | 28 + .../src/btree/tree_core_split_baseline.cpp | 90 + .../src/btree/tree_core_split_capture.cpp | 198 + .../src/btree/tree_core_split_handoff.cpp | 191 + .../src/btree/tree_core_split_internal.h | 64 + .../src/btree/tree_core_split_overlay.cpp | 188 + .../src/btree/tree_core_split_session.cpp | 54 + .../src/btree/tree_core_split_wait.h | 83 + .../src/btree/tree_core_state_stats.cpp | 52 + .../src/btree/tree_diagnostics.cpp | 113 + .../src/btree/tree_generation_lease.cpp | 151 + .../src/btree/tree_generation_lease.h | 36 + .../btree/tree_generation_lease_internal.h | 29 + .../src/btree/tree_generation_prepare.cpp | 152 + lib/crowdb-tree/src/btree/tree_metrics.cpp | 331 ++ lib/crowdb-tree/src/btree/tree_metrics.h | 358 ++ lib/crowdb-tree/src/btree/tree_options.cpp | 18 + lib/crowdb-tree/src/btree/tree_options.h | 9 + lib/crowdb-tree/src/btree/tree_reads.cpp | 54 + lib/crowdb-tree/src/btree/tree_scans.cpp | 95 + lib/crowdb-tree/src/btree/tree_snapshots.cpp | 53 + .../src/btree/tree_state_reconcile.cpp | 108 + .../src/btree/tree_state_reconcile.h | 57 + .../src/btree/tree_state_reconcile_ledger.cpp | 75 + .../src/btree/tree_wait_metrics.cpp | 121 + lib/crowdb-tree/src/btree/tree_wait_metrics.h | 76 + lib/crowdb-tree/src/btree/tree_work_age.cpp | 76 + .../src/btree/tree_work_counters.cpp | 53 + .../src/btree/tree_work_counters.h | 163 + .../src/btree/tree_work_histograms.cpp | 131 + .../src/btree/tree_work_histograms.h | 33 + .../src/btree/tree_work_registry.cpp | 174 + .../src/btree/tree_work_registry.h | 15 + .../src/btree/writeback_metrics.cpp | 59 + lib/crowdb-tree/src/btree/writeback_metrics.h | 31 + .../src/btree/writeback_rebase.cpp | 141 + lib/crowdb-tree/src/btree/writeback_rebase.h | 21 + lib/crowdb-tree/src/buffer_external.cpp | 43 + lib/crowdb-tree/src/c_api.cpp | 480 +- lib/crowdb-tree/src/c_api_apply.cpp | 224 + .../src/c_api_chunk_collection.cpp | 49 + .../src/c_api_chunk_collection_start.cpp | 15 + lib/crowdb-tree/src/c_api_compaction.cpp | 75 + lib/crowdb-tree/src/c_api_future_internal.h | 78 + lib/crowdb-tree/src/c_api_get.cpp | 58 + .../src/c_api_immutable_snapshot.cpp | 113 + .../src/c_api_immutable_snapshot_internal.h | 18 + .../src/c_api_immutable_snapshot_scan.cpp | 54 + lib/crowdb-tree/src/c_api_internal.h | 36 +- .../src/c_api_maintenance_internal.h | 113 + .../src/c_api_maintenance_result.cpp | 95 + .../src/c_api_maintenance_start.cpp | 31 + lib/crowdb-tree/src/c_api_maintenance_start.h | 52 + .../src/c_api_owned_allocation.cpp | 117 + lib/crowdb-tree/src/c_api_owned_allocation.h | 20 + lib/crowdb-tree/src/c_api_owned_batch.cpp | 90 + lib/crowdb-tree/src/c_api_owned_budget.cpp | 92 + lib/crowdb-tree/src/c_api_owned_budget.h | 14 + lib/crowdb-tree/src/c_api_scan_owner.cpp | 199 + lib/crowdb-tree/src/c_api_snapshot_export.cpp | 198 + lib/crowdb-tree/src/c_api_split_overlay.cpp | 110 + lib/crowdb-tree/src/epoch.cpp | 178 +- lib/crowdb-tree/src/epoch_backend_close.cpp | 32 + lib/crowdb-tree/src/epoch_init.cpp | 58 + lib/crowdb-tree/src/epoch_internal.h | 53 + lib/crowdb-tree/src/epoch_retirement.cpp | 168 + lib/crowdb-tree/src/external_owner_bridge.cpp | 41 + lib/crowdb-tree/src/external_owner_bridge.h | 46 + lib/crowdb-tree/src/host_completion.h | 55 + .../src/maptable/compressor_bounded.cpp | 79 + lib/crowdb-tree/src/maptable/frame_page.cpp | 50 +- lib/crowdb-tree/src/memtable/admission.cpp | 68 +- lib/crowdb-tree/src/memtable/cell_version.cpp | 14 + lib/crowdb-tree/src/memtable/memtable.cpp | 68 +- .../src/memtable/prepared_cell.cpp | 34 + .../src/memtable/prepared_write.cpp | 208 + lib/crowdb-tree/src/memtable/skip_list.cpp | 71 +- .../src/memtable/source_membership.cpp | 51 + .../src/memtable/source_memory.cpp | 67 + lib/crowdb-tree/src/memtable/source_stats.cpp | 99 + lib/crowdb-tree/src/memtable/versions.cpp | 93 +- .../runtime/allocation_memory_resource.cpp | 36 + .../src/runtime/allocation_memory_resource.h | 30 + lib/crowdb-tree/src/runtime/binding_metrics.h | 44 + .../src/runtime/bootstrap_budget.cpp | 152 + .../src/runtime/bootstrap_budget.h | 42 + .../src/runtime/default_runtime.cpp | 78 + lib/crowdb-tree/src/runtime/default_runtime.h | 14 + lib/crowdb-tree/src/runtime/pool_storage.cpp | 110 + lib/crowdb-tree/src/runtime/pool_storage.h | 36 + .../src/runtime/tree_operation_lifetime.cpp | 151 + .../src/runtime/tree_operation_lifetime.h | 71 + lib/crowdb-tree/src/runtime/tree_runtime.cpp | 215 + .../src/runtime/tree_runtime_binding.cpp | 96 + .../src/runtime/tree_runtime_binding.h | 96 + .../src/runtime/tree_runtime_c_api.cpp | 325 ++ .../src/runtime/tree_runtime_c_api.h | 18 + .../src/runtime/tree_runtime_internal.h | 37 + .../src/runtime/tree_tuning_c_api.cpp | 90 + .../src/runtime/tree_tuning_c_api.h | 11 + .../src/snapshot/captured_frames.cpp | 244 + .../src/snapshot/captured_frames.h | 120 + .../src/snapshot/captured_frames_internal.h | 82 + .../src/snapshot/captured_frames_load.cpp | 89 + .../src/snapshot/captured_frames_reuse.cpp | 65 + .../src/snapshot/captured_frames_step.cpp | 270 + .../src/snapshot/captured_plan.cpp | 175 + lib/crowdb-tree/src/snapshot/captured_plan.h | 68 + .../src/snapshot/captured_rebase.cpp | 165 + .../src/snapshot/captured_rebase.h | 55 + .../src/snapshot/captured_rebase_batch.cpp | 99 + .../src/snapshot/captured_rebase_internal.h | 54 + .../src/snapshot/captured_rebase_step.cpp | 70 + .../src/snapshot/captured_rebase_summary.cpp | 65 + .../src/snapshot/captured_write.cpp | 209 + lib/crowdb-tree/src/snapshot/captured_write.h | 71 + .../src/snapshot/captured_write_internal.h | 146 + .../src/snapshot/captured_write_page.cpp | 117 + .../src/snapshot/captured_write_pump.cpp | 259 + .../src/snapshot/captured_write_relocate.cpp | 133 + .../src/snapshot/captured_write_resident.cpp | 101 + .../snapshot/checkpoint_allocation_bound.cpp | 12 + .../src/snapshot/checkpoint_operation.cpp | 262 + .../src/snapshot/checkpoint_operation.h | 111 + .../snapshot/checkpoint_operation_backend.cpp | 41 + .../snapshot/checkpoint_operation_internal.h | 165 + .../snapshot/checkpoint_operation_plan.cpp | 194 + .../snapshot/checkpoint_operation_steps.cpp | 405 ++ .../checkpoint_operation_structure.cpp | 70 + .../src/snapshot/chunk_index_codec.cpp | 174 + .../src/snapshot/chunk_root_codec.cpp | 135 + .../src/snapshot/chunk_root_codec.h | 80 + .../src/snapshot/chunk_root_codec_internal.h | 89 + .../src/snapshot/chunk_root_commit.cpp | 172 + .../src/snapshot/chunk_root_commit.h | 123 + .../chunk_root_commit_dependencies.cpp | 43 + .../src/snapshot/chunk_root_commit_finish.cpp | 109 + .../src/snapshot/chunk_root_commit_index.cpp | 133 + .../src/snapshot/chunk_root_recovery.cpp | 144 + .../src/snapshot/chunk_root_recovery.h | 110 + .../chunk_root_recovery_dependencies.cpp | 61 + .../snapshot/chunk_root_recovery_frames.cpp | 172 + .../src/snapshot/commit_anchor.cpp | 24 +- lib/crowdb-tree/src/snapshot/commit_anchor.h | 8 +- .../src/snapshot/graph_summary.cpp | 57 + lib/crowdb-tree/src/snapshot/graph_summary.h | 13 + .../src/snapshot/local_checkpoint.cpp | 173 + .../src/snapshot/local_checkpoint.h | 133 + .../src/snapshot/local_checkpoint_internal.h | 105 + .../src/snapshot/local_checkpoint_io.cpp | 75 + .../snapshot/local_checkpoint_metadata.cpp | 126 + .../src/snapshot/local_checkpoint_pump.cpp | 136 + .../snapshot/local_checkpoint_registry.cpp | 81 + .../src/snapshot/local_recovery.cpp | 178 + lib/crowdb-tree/src/snapshot/local_recovery.h | 57 + .../src/snapshot/local_recovery_frames.cpp | 216 + .../src/snapshot/local_recovery_internal.h | 157 + .../src/snapshot/local_recovery_io.cpp | 88 + .../src/snapshot/local_recovery_metadata.cpp | 194 + .../src/snapshot/local_recovery_pump.cpp | 130 + .../src/snapshot/metadata_codec.cpp | 127 + lib/crowdb-tree/src/snapshot/metadata_codec.h | 46 + .../src/snapshot/metadata_decode.cpp | 184 + .../src/snapshot/metadata_decode.h | 52 + .../src/snapshot/native_frame_read.cpp | 270 + .../src/snapshot/native_frame_read.h | 42 + .../src/snapshot/native_layout.cpp | 155 + lib/crowdb-tree/src/snapshot/native_layout.h | 66 + lib/crowdb-tree/src/snapshot/native_plan.cpp | 213 + lib/crowdb-tree/src/snapshot/native_plan.h | 34 + lib/crowdb-tree/src/snapshot/operation_wait.h | 48 + lib/crowdb-tree/src/snapshot/persist.cpp | 1318 +---- .../src/snapshot/primary_backing.cpp | 93 + .../src/snapshot/primary_backing.h | 48 + .../src/snapshot/published_byte_export.cpp | 100 + .../src/snapshot/published_byte_export.h | 40 + .../snapshot/published_byte_export_internal.h | 52 + .../snapshot/published_byte_export_next.cpp | 228 + .../snapshot/published_byte_export_plan.cpp | 90 + .../src/snapshot/published_native.cpp | 186 + .../src/snapshot/published_native.h | 32 + .../src/snapshot/published_native_collect.cpp | 115 + .../src/snapshot/published_native_collect.h | 14 + .../src/snapshot/published_native_internal.h | 38 + .../src/snapshot/published_native_next.cpp | 177 + .../src/snapshot/published_native_rewind.cpp | 65 + .../src/snapshot/published_snapshot.cpp | 297 ++ .../src/snapshot/published_snapshot.h | 91 + .../src/snapshot/published_snapshot_budget.h | 25 + .../snapshot/published_snapshot_entries.cpp | 201 + .../src/snapshot/published_snapshot_entries.h | 30 + .../published_snapshot_materialize.cpp | 162 + .../snapshot/published_snapshot_materialize.h | 29 + .../src/snapshot/published_snapshot_read.cpp | 105 + .../src/snapshot/published_snapshot_scan.cpp | 224 + .../src/snapshot/published_snapshot_view.cpp | 111 + .../src/snapshot/published_snapshot_view.h | 51 + .../src/snapshot/readable_page_catalog.h | 67 + .../src/snapshot/recovered_state.cpp | 186 + .../src/snapshot/recovered_state.h | 61 + .../src/snapshot/recovered_state_graph.cpp | 218 + .../src/snapshot/recovered_state_internal.h | 96 + .../src/snapshot/recovered_state_leaf.cpp | 115 + .../src/snapshot/recovered_state_load.cpp | 159 + lib/crowdb-tree/src/snapshot/retry.cpp | 66 - .../src/snapshot/runtime_snapshot_import.cpp | 111 + .../src/snapshot/runtime_snapshot_import.h | 72 + .../runtime_snapshot_import_fields.cpp | 90 + .../runtime_snapshot_import_parse.cpp | 146 + .../src/snapshot/snapshot_baseline.h | 25 + lib/crowdb-tree/src/snapshot/snapshot_crc.h | 48 + .../snapshot/snapshot_export_materialized.cpp | 191 + .../src/snapshot/snapshot_export_owned.cpp | 44 + lib/crowdb-tree/src/snapshot/snapshot_io.cpp | 571 +-- lib/crowdb-tree/src/snapshot/snapshot_wire.h | 56 + .../src/snapshot/stored_overflow.cpp | 45 + .../src/snapshot/stored_overflow.h | 12 + lib/crowdb-tree/src/snapshot/write.cpp | 365 -- .../src/snapshot/writeback_selection.cpp | 165 + .../src/snapshot/writeback_selection.h | 15 + .../src/tree_backing_stats_c_api.cpp | 28 + .../src/tree_backing_stats_c_api_internal.h | 10 + lib/crowdb-tree/src/tree_progress_c_api.cpp | 33 + lib/crowdb-tree/src/tree_stats_c_api.cpp | 75 + .../src/tree_wait_metrics_c_api.cpp | 50 + .../src/tree_wait_metrics_c_api_internal.h | 10 + .../src/tree_work_histograms_c_api.cpp | 62 + .../src/tree_work_histograms_c_api_internal.h | 10 + .../src/tree_work_metrics_c_api.cpp | 316 ++ lib/crowdb-tree/tests/.clang-tidy | 12 +- .../tests/allocation_account_test_support.h | 93 + .../buffer_pool_test_support.cpp} | 56 +- .../buffer_pool_test_support.h} | 82 +- .../tests/captured_write_test_support.h | 71 + .../tests/checkpoint_structure_test_support.h | 198 + .../tests/common/test_chunk_backing.h | 144 + .../tests/common/test_chunk_collector.h | 165 + .../tests/common/test_chunk_physical.h | 98 + .../tests/common/test_chunk_root.h | 103 + .../tests/common/test_chunk_root_catalog.h | 288 ++ .../tests/common/test_chunk_route_service.h | 31 +- .../tests/common/test_chunk_runtime_c_api.cpp | 120 + .../tests/common/test_chunk_stage_backend.h | 136 + .../tests/common/test_legacy_chunk.h | 177 + .../tests/common/test_root_transport.h | 118 + .../tests/common/test_runtime_catalog.h | 101 + .../tests/common/test_stage_catalog.h | 112 + .../tests/common/test_tree_core_chunk.h | 71 + .../tests/compaction_fault_store.h | 80 + .../tests/data/legacy-chunk-v7.bin | Bin 0 -> 115472 bytes .../tests/data/legacy-chunk-v7.json | 24 + .../tests/extent_ledger_test_support.h | 55 + .../tests/flush_preparation_test_support.h | 179 + .../tests/inner_load_test_support.h | 179 + .../tests/integration/alignment_test.cpp | 2 +- .../tests/integration/async_get_test.cpp | 24 +- .../tests/integration/async_scan_test.cpp | 29 +- .../integration/backing_stats_c_api_test.cpp | 82 + .../integration/borrowed_read_budget_test.cpp | 102 + .../tests/integration/budgeted_flush_test.cpp | 97 + .../tests/integration/c_api_test.cpp | 4 + .../captured_frame_window_test.cpp | 79 + .../integration/captured_frames_test.cpp | 648 +++ .../captured_write_backend_test.cpp | 90 + ...aptured_write_dependency_lifetime_test.cpp | 184 + .../captured_write_resident_test.cpp | 79 + .../tests/integration/captured_write_test.cpp | 312 ++ .../checkpoint_handoff_report_test.cpp | 206 + .../integration/checkpoint_operation_test.cpp | 342 ++ .../checkpoint_retention_reuse_test.cpp | 78 + .../checkpoint_split_progress_test.cpp | 73 + .../checkpoint_structure_public_test.cpp | 195 + .../integration/checkpoint_structure_test.cpp | 173 + .../integration/chunk_backing_pin_test.cpp | 62 + .../tests/integration/chunk_backing_test.cpp | 142 + .../chunk_backing_visibility_test.cpp | 60 + .../chunk_collection_c_api_test.cpp | 123 + .../chunk_collection_operation_test.cpp | 168 + .../chunk_legacy_runtime_open_test.cpp | 31 + .../integration/chunk_page_store_test.cpp | 160 +- .../integration/chunk_physical_core_test.cpp | 35 + .../integration/chunk_root_commit_test.cpp | 140 + .../integration/chunk_runtime_c_api_test.cpp | 97 + .../chunk_snapshot_export_test.cpp | 87 + .../integration/chunk_snapshot_retry_test.cpp | 137 +- .../tests/integration/compression_test.cpp | 27 +- .../tests/integration/crash_recovery_test.cpp | 98 +- .../tests/integration/double_buffer_test.cpp | 69 +- .../tests/integration/eviction_test.cpp | 53 +- .../integration/extent_writeback_test.cpp | 195 + .../integration/external_ownership_test.cpp | 64 +- .../facade_checkpoint_diagnostics_test.cpp | 103 + .../integration/flush_capture_report_test.cpp | 94 + .../integration/flush_continuation_test.cpp | 188 + .../tests/integration/flush_dispatch_test.cpp | 157 + .../integration/flush_operation_test.cpp | 394 ++ .../integration/flush_range_scaling_test.cpp | 109 + .../integration/flush_reservation_test.cpp | 638 +++ .../tests/integration/flush_routing_test.cpp | 190 + .../tests/integration/flush_window_test.cpp | 199 + .../host_writeback_window_test.cpp | 209 + .../immutable_chunk_snapshot_test.cpp | 49 + .../incremental_checkpoint_test.cpp | 165 +- .../large_value_checkpoint_test.cpp | 93 + .../legacy_chunk_migration_test.cpp | 279 ++ .../tests/integration/local_block_gc_test.cpp | 166 + .../local_checkpoint_backend_test.cpp | 91 + .../local_checkpoint_directory_test.cpp | 100 + .../local_checkpoint_failure_test.cpp | 132 + .../integration/local_checkpoint_test.cpp | 150 + .../local_compaction_async_test.cpp | 152 + .../local_recovery_backend_test.cpp | 130 + .../local_recovery_failure_test.cpp | 149 + .../local_recovery_frames_test.cpp | 271 + .../local_recovery_lifetime_test.cpp | 33 + .../tests/integration/local_recovery_test.cpp | 299 ++ .../integration/maintenance_cold_test.cpp | 132 + .../maintenance_operation_test.cpp | 641 +++ .../integration/memory_backing_gc_test.cpp | 147 + .../integration/memtable_failure_test.cpp | 148 +- .../integration/memtable_handoff_test.cpp | 155 +- .../integration/native_import_export_test.cpp | 70 + .../page_writeback_backend_test.cpp | 175 + .../tests/integration/page_writeback_test.cpp | 219 + .../tests/integration/parallel_flush_test.cpp | 404 ++ .../tests/integration/parity_test.cpp | 41 +- .../tests/integration/persist_test.cpp | 315 +- .../prepared_split_overlay_test.cpp | 305 ++ .../primary_layout_admission_test.cpp | 97 + .../published_byte_export_test.cpp | 218 + .../published_native_collect_test.cpp | 89 + .../integration/published_native_test.cpp | 241 + .../integration/published_read_delta_test.cpp | 72 + .../published_snapshot_context_test.cpp | 153 + .../published_snapshot_domain_test.cpp | 108 + .../published_snapshot_entries_range_test.cpp | 183 + .../published_snapshot_scan_test.cpp | 186 + .../published_snapshot_view_test.cpp | 192 + lib/crowdb-tree/tests/integration/r6_test.cpp | 102 +- .../integration/range_rebuild_chunk_test.cpp | 202 + .../tests/integration/range_rebuild_test.cpp | 306 +- .../range_runtime_binding_test.cpp | 76 + .../recovered_frame_identity_test.cpp | 86 + .../recovered_state_backend_test.cpp | 199 + .../integration/recovered_state_test.cpp | 253 + .../integration/rpc_chunk_collection_test.cpp | 208 + .../integration/rpc_chunk_fixed_io_test.cpp | 133 + .../integration/rpc_chunk_staging_test.cpp | 63 + .../runtime_block_compaction_test.cpp | 96 + .../integration/runtime_eviction_test.cpp | 55 + .../integration/runtime_inspection_test.cpp | 95 + .../integration/runtime_migration_test.cpp | 185 + .../runtime_range_rebuild_test.cpp | 167 + .../integration/runtime_read_window_test.cpp | 68 + .../runtime_snapshot_import_test.cpp | 265 + .../integration/runtime_split_hint_test.cpp | 51 + .../integration/runtime_summary_test.cpp | 74 + .../snapshot_change_effect_test.cpp | 76 + .../integration/snapshot_export_test.cpp | 16 +- .../integration/snapshot_memory_test.cpp | 39 +- .../source_generation_rotation_test.cpp | 54 + .../integration/source_host_wait_test.cpp | 420 ++ .../source_rotation_report_test.cpp | 261 + .../integration/source_rotation_wait_test.cpp | 155 + .../sparse_block_relocation_test.cpp | 128 + .../tests/integration/split_merge_test.cpp | 121 +- .../integration/staged_writeback_test.cpp | 208 + .../tests/integration/stress_test.cpp | 219 +- .../integration/tree_apply_grant_test.cpp | 108 + .../integration/tree_core_admitted_test.cpp | 297 ++ .../integration/tree_core_auto_write_test.cpp | 182 + .../tree_core_block_account_test.cpp | 145 + .../integration/tree_core_checkpoint_test.cpp | 193 + .../integration/tree_core_chunk_read_test.cpp | 148 + .../integration/tree_core_chunk_test.cpp | 215 + .../tree_core_chunk_writeback_test.cpp | 144 + .../tree_core_empty_open_budget_test.cpp | 53 + .../tree_core_flush_read_budget_test.cpp | 33 + .../tree_core_generation_checkpoint_test.cpp | 93 + .../integration/tree_core_generation_test.cpp | 193 + .../tree_core_native_checkpoint_test.cpp | 118 + .../tree_core_native_import_test.cpp | 139 + .../tree_core_primary_reuse_test.cpp | 216 + .../integration/tree_core_public_test.cpp | 201 + .../integration/tree_core_replay_test.cpp | 321 ++ .../integration/tree_core_retirement_test.cpp | 70 + .../tree_core_source_stats_test.cpp | 134 + .../integration/tree_core_split_test.cpp | 465 ++ .../tree_core_sync_backend_test.cpp | 76 + .../tree_generation_lease_test.cpp | 103 + .../integration/tree_host_progress_test.cpp | 184 + .../tree_point_read_bound_test.cpp | 89 + .../tree_retirement_observation_test.cpp | 64 + .../tree_scan_total_window_test.cpp | 115 + .../integration/tree_source_controls_test.cpp | 190 + .../integration/tree_source_window_test.cpp | 177 + .../integration/tree_state_reconcile_test.cpp | 111 + .../integration/tree_work_counters_test.cpp | 117 + .../tests/integration/write_path_test.cpp | 24 +- .../tests/load_budget_test_support.h | 54 + .../tests/local_checkpoint_test_support.h | 175 + .../tests/local_recovery_test_support.h | 128 + .../mapping_segment_test_support.h} | 33 +- .../mapping_table_test_support.cpp} | 77 +- .../mapping_table_test_support.h} | 83 +- lib/crowdb-tree/tests/memtable_access.h | 93 +- .../tests/native_import_test_support.h | 54 + .../tests/overflow_load_test_support.h | 101 + .../tests/page_cache_residency_test_support.h | 74 + .../page_descent_test_support.h} | 15 +- .../tests/page_load_delta_test_support.h | 123 + .../tests/page_writeback_test_support.h | 133 + .../tests/published_read_test_support.h | 172 + .../tests/recovered_state_test_support.h | 120 + lib/crowdb-tree/tests/runtime_tree_access.h | 33 + .../tests/snapshot_publication_test_support.h | 50 + lib/crowdb-tree/tests/stdexec_overlay_test.py | 48 + .../tests/unit/admitted_batch_test.cpp | 227 + .../unit/allocation_account_adoption_test.cpp | 72 + .../unit/allocation_account_grant_test.cpp | 119 + .../allocation_account_interleaving_test.cpp | 99 + .../tests/unit/allocation_account_test.cpp | 104 + .../unit/allocation_account_transfer_test.cpp | 127 + .../unit/allocation_observation_test.cpp | 120 + .../unit/allocation_reservation_test.cpp | 474 ++ .../tests/unit/apply_dispatcher_test.cpp | 157 + .../tests/unit/apply_frontier_test.cpp | 134 + .../tests/unit/apply_progress_test.cpp | 168 + .../tests/unit/backend_factory_test.cpp | 73 + .../tests/unit/backend_release_c_api_test.cpp | 87 + .../unit/backend_release_service_test.cpp | 187 + .../tests/unit/block_account_grant_test.cpp | 214 + .../tests/unit/block_account_test.cpp | 129 + .../tests/unit/block_async_account_test.cpp | 103 + .../tests/unit/block_directory_test.cpp | 114 + .../tests/unit/block_release_service_test.cpp | 110 + .../tests/unit/block_sync_generation_test.cpp | 134 + .../unit/budget_memory_resource_test.cpp | 149 + .../tests/unit/buffer_pool_test.cpp | 87 +- .../tests/unit/c_api_admission_test.cpp | 96 + .../unit/c_api_maintenance_report_test.cpp | 199 + .../tests/unit/c_api_owned_budget_test.cpp | 62 + .../tests/unit/c_api_read_budget_test.cpp | 89 + .../tests/unit/c_api_scan_async_v2_test.cpp | 104 + .../tests/unit/c_api_scan_owner_test.cpp | 84 + .../c_api_snapshot_backend_owner_test.cpp | 106 + .../tests/unit/c_api_snapshot_owner_test.cpp | 72 + .../tests/unit/captured_rebase_reuse_test.cpp | 159 + .../tests/unit/chunk_collector_test.cpp | 114 + .../tests/unit/chunk_fixed_pack_test.cpp | 140 + .../tests/unit/chunk_physical_gc_test.cpp | 193 + .../unit/chunk_physical_registry_test.cpp | 195 + .../unit/chunk_protection_callback_test.cpp | 352 ++ .../tests/unit/chunk_registry_scope_test.cpp | 79 + .../tests/unit/chunk_root_codec_test.cpp | 171 + .../tests/unit/chunk_stage_gc_test.cpp | 109 + .../tests/unit/chunk_stage_test.cpp | 83 + .../tests/unit/consolidation_test.cpp | 102 +- .../tests/unit/default_runtime_test.cpp | 70 + .../tests/unit/epoch_budget_test.cpp | 119 + .../tests/unit/extent_exclusion_test.cpp | 82 + .../tests/unit/extent_gc_lease_test.cpp | 82 + .../tests/unit/extent_ledger_test.cpp | 269 + .../tests/unit/external_allocation_test.cpp | 176 + .../tests/unit/external_owner_bridge_test.cpp | 207 + .../tests/unit/flush_report_sender_test.cpp | 382 ++ .../unit/flush_runtime_progress_test.cpp | 273 ++ .../unit/flush_runtime_scheduler_test.cpp | 70 + .../tests/unit/flush_runtime_test.cpp | 348 ++ .../tests/unit/flush_runtime_usage_test.cpp | 61 + .../tests/unit/frame_page_test.cpp | 19 + .../tests/unit/frame_summary_test.cpp | 80 + lib/crowdb-tree/tests/unit/gc_test.cpp | 94 +- .../tests/unit/graph_summary_test.cpp | 50 + .../tests/unit/host_completion_test.cpp | 75 + .../unit/immutable_snapshot_c_api_test.cpp | 118 + .../tests/unit/inner_page_test.cpp | 49 +- .../unit/logical_page_cursor_window_test.cpp | 92 + .../tests/unit/logical_page_test.cpp | 401 ++ .../tests/unit/maintenance_budget_test.cpp | 223 + .../unit/maintenance_inner_merge_test.cpp | 262 + .../unit/maintenance_publication_test.cpp | 383 ++ .../unit/maintenance_split_step_test.cpp | 216 + .../tests/unit/maintenance_window_test.cpp | 149 + .../tests/unit/mapping_segment_test.cpp | 12 +- .../tests/unit/mapping_table_test.cpp | 130 +- .../unit/memory_backing_failure_test.cpp | 36 + .../tests/unit/memory_backing_test.cpp | 81 + .../tests/unit/memtable_budget_test.cpp | 272 + .../tests/unit/merge_pair_metrics_test.cpp | 76 + .../unit/metrics_registry_allocation_test.cpp | 164 + lib/crowdb-tree/tests/unit/metrics_test.cpp | 11 +- .../tests/unit/native_import_test.cpp | 173 + .../tests/unit/native_layout_test.cpp | 130 + .../tests/unit/native_readable_test.cpp | 162 + .../unit/owned_allocation_c_api_test.cpp | 245 + lib/crowdb-tree/tests/unit/owned_key_test.cpp | 70 + .../unit/page_cache_residency_budget_test.cpp | 225 + .../tests/unit/page_cache_residency_test.cpp | 288 ++ .../tests/unit/page_cache_test.cpp | 572 +++ .../tests/unit/page_cache_trim_test.cpp | 117 + .../tests/unit/page_cache_wake_test.cpp | 74 + .../tests/unit/page_compression_test.cpp | 66 +- .../tests/unit/page_encode_test.cpp | 352 ++ .../tests/unit/page_load_delta_test.cpp | 174 + .../tests/unit/page_load_inner_proof_test.cpp | 70 + .../tests/unit/page_load_inner_test.cpp | 151 + .../tests/unit/page_load_lifetime_test.cpp | 389 ++ .../tests/unit/page_load_overflow_test.cpp | 255 + lib/crowdb-tree/tests/unit/page_load_test.cpp | 325 ++ .../tests/unit/page_maintenance_test.cpp | 356 ++ .../tests/unit/portable_import_bound_test.cpp | 83 + .../tests/unit/portable_import_test.cpp | 79 + .../tests/unit/prepared_split_c_api_test.cpp | 153 + .../tests/unit/primary_backing_test.cpp | 173 + .../tests/unit/publication_admission_test.cpp | 243 + .../tests/unit/publication_bootstrap_test.cpp | 181 + .../publication_checkpoint_capacity_test.cpp | 105 + .../publication_checkpoint_failure_test.cpp | 235 + .../unit/publication_checkpoint_test.cpp | 294 ++ .../tests/unit/publication_gc_floor_test.cpp | 102 + .../unit/publication_generation_test.cpp | 97 + .../tests/unit/publication_handoff_test.cpp | 169 + .../tests/unit/publication_lane_test.cpp | 82 + .../unit/publication_merge_pair_test.cpp | 398 ++ .../tests/unit/publication_metrics_test.cpp | 172 + .../tests/unit/publication_reconcile_test.cpp | 178 + .../tests/unit/publication_safety_test.cpp | 232 + .../tests/unit/publication_test.cpp | 262 + .../tests/unit/published_inner_read_test.cpp | 194 + .../tests/unit/published_read_budget_test.cpp | 167 + .../tests/unit/published_read_test.cpp | 171 + .../unit/published_route_pruning_test.cpp | 60 + .../unit/published_scan_options_test.cpp | 119 + .../tests/unit/published_scan_test.cpp | 217 + .../tests/unit/read_value_bound_test.cpp | 88 + .../unit/resource_account_stats_test.cpp | 47 + .../unit/resource_budget_progress_test.cpp | 91 + .../tests/unit/resource_budget_test.cpp | 303 ++ .../tests/unit/resource_registration_test.cpp | 137 + .../tests/unit/resource_usage_test.cpp | 85 + .../tests/unit/retirement_queue_test.cpp | 449 ++ .../runtime_bootstrap_accounting_test.cpp | 33 + .../unit/runtime_bootstrap_budget_test.cpp | 63 + .../tests/unit/runtime_pool_storage_test.cpp | 146 + .../tests/unit/runtime_registry_test.cpp | 73 + .../tests/unit/runtime_snapshot_test.cpp | 104 + .../tests/unit/scan_output_test.cpp | 47 + .../tests/unit/scheduled_page_store_test.cpp | 293 ++ .../tests/unit/snapshot_changes_test.cpp | 180 + .../tests/unit/snapshot_crc_test.cpp | 27 + .../unit/snapshot_metadata_codec_test.cpp | 112 + .../unit/snapshot_metadata_decode_test.cpp | 273 ++ .../tests/unit/snapshot_publication_test.cpp | 180 + .../tests/unit/source_admission_test.cpp | 289 ++ .../tests/unit/source_ledger_test.cpp | 204 + .../tests/unit/source_memory_test.cpp | 127 + .../tests/unit/source_pause_test.cpp | 143 + .../tests/unit/source_prepared_write_test.cpp | 151 + .../tests/unit/source_rotation_test.cpp | 87 + .../tests/unit/source_tail_test.cpp | 82 + .../unit/source_write_partition_test.cpp | 93 + .../unit/source_write_reservation_test.cpp | 139 + .../tests/unit/source_writer_budget_test.cpp | 287 ++ .../tests/unit/tree_core_geometry_test.cpp | 60 + .../tests/unit/tree_core_read_test.cpp | 282 ++ .../unit/tree_operation_lifetime_test.cpp | 118 + .../tests/unit/tree_physical_stats_test.cpp | 60 + .../tests/unit/tree_progress_metrics_test.cpp | 172 + .../tests/unit/tree_runtime_c_api_test.cpp | 139 + .../tests/unit/tree_runtime_test.cpp | 195 + .../tests/unit/tree_state_metrics_test.cpp | 352 ++ .../tests/unit/tree_stats_c_api_test.cpp | 60 + .../tests/unit/tree_tuning_c_api_test.cpp | 102 + .../unit/tree_wait_metrics_c_api_test.cpp | 46 + .../tests/unit/tree_wait_metrics_test.cpp | 261 + .../tests/unit/tree_work_age_c_api_test.cpp | 62 + .../tests/unit/tree_work_age_test.cpp | 141 + .../tests/unit/tree_work_counters_test.cpp | 259 + .../unit/tree_work_histograms_c_api_test.cpp | 150 + .../tests/unit/tree_work_histograms_test.cpp | 222 + .../unit/tree_work_metrics_c_api_test.cpp | 134 + lib/crowdb-tree/tests/unit/version_test.cpp | 7 +- .../tests/unit/writeback_metrics_test.cpp | 37 + .../tests/unit/writeback_rebase_test.cpp | 236 + .../tests/unit/zero_copy_apply_test.cpp | 8 +- pixi.toml | 4 +- third-party/bytes/.github/workflows/ci.yml | 211 + third-party/bytes/.gitignore | 2 + third-party/bytes/CHANGELOG.md | 449 ++ third-party/bytes/CROWDB.md | 29 + third-party/bytes/Cargo.lock | 444 ++ third-party/bytes/Cargo.toml | 141 + third-party/bytes/Cargo.toml.orig | 45 + third-party/bytes/LICENSE | 25 + third-party/bytes/README.md | 73 + third-party/bytes/SECURITY.md | 9 + third-party/bytes/benches/buf.rs | 186 + third-party/bytes/benches/bytes.rs | 120 + third-party/bytes/benches/bytes_mut.rs | 266 + third-party/bytes/ci/miri.sh | 13 + third-party/bytes/ci/panic-abort.sh | 4 + third-party/bytes/ci/test-stable.sh | 25 + third-party/bytes/ci/tsan.sh | 13 + third-party/bytes/clippy.toml | 1 + third-party/bytes/crowdb-upstream.json | 55 + third-party/bytes/src/buf/buf_impl.rs | 2962 +++++++++++ third-party/bytes/src/buf/buf_mut.rs | 1671 +++++++ third-party/bytes/src/buf/chain.rs | 240 + third-party/bytes/src/buf/iter.rs | 127 + third-party/bytes/src/buf/limit.rs | 75 + third-party/bytes/src/buf/mod.rs | 39 + third-party/bytes/src/buf/reader.rs | 81 + third-party/bytes/src/buf/take.rs | 187 + third-party/bytes/src/buf/uninit_slice.rs | 257 + third-party/bytes/src/buf/vec_deque.rs | 40 + third-party/bytes/src/buf/writer.rs | 88 + third-party/bytes/src/bytes.rs | 1677 +++++++ .../bytes/src/bytes/accounted_owner.rs | 171 + third-party/bytes/src/bytes/owner_lease.rs | 139 + third-party/bytes/src/bytes_mut.rs | 1942 ++++++++ third-party/bytes/src/fmt/debug.rs | 40 + third-party/bytes/src/fmt/hex.rs | 27 + third-party/bytes/src/fmt/mod.rs | 15 + third-party/bytes/src/lib.rs | 182 + third-party/bytes/src/loom.rs | 33 + third-party/bytes/src/serde.rs | 89 + .../bytes/tests/test_accounted_owner.rs | 214 + third-party/bytes/tests/test_buf.rs | 439 ++ third-party/bytes/tests/test_buf_mut.rs | 284 ++ third-party/bytes/tests/test_bytes.rs | 1722 +++++++ .../bytes/tests/test_bytes_odd_alloc.rs | 147 + .../bytes/tests/test_bytes_vec_alloc.rs | 146 + third-party/bytes/tests/test_chain.rs | 177 + third-party/bytes/tests/test_debug.rs | 35 + third-party/bytes/tests/test_iter.rs | 21 + third-party/bytes/tests/test_limit.rs | 86 + third-party/bytes/tests/test_reader.rs | 38 + third-party/bytes/tests/test_serde.rs | 20 + third-party/bytes/tests/test_take.rs | 84 + tools/README.md | 5 +- tools/benchmark/bench-chunk-kv-regression.sh | 2 +- .../bench-chunk-stream-regression.sh | 4 +- tools/benchmark/bench-chunkdb-regression.sh | 8 +- .../bench-chunkio-read-regression.sh | 2 +- .../bench-chunkio-small-write-regression.sh | 2 +- .../bench-chunkio-write-regression.sh | 6 +- tools/benchmark/bench-diskdb-regression.sh | 10 +- tools/benchmark/bench-kv-common.sh | 79 + tools/benchmark/bench-kv-read-regression.sh | 205 +- tools/benchmark/bench-kv-scan-regression.sh | 264 +- tools/benchmark/bench-kv-write-regression.sh | 406 +- tools/benchmark/bench-kv-write-sentinel.sh | 59 +- tools/benchmark/bench-leak-inspect.sh | 121 +- tools/benchmark/bench-regression-common.sh | 4 - tools/benchmark/fixture-smoke.py | 28 +- tools/benchmark/fixture.py | 9 +- .../benchmark/test-bench-kv-write-sentinel.sh | 17 +- tools/ci-checks/check-ci-test-tasks.py | 4 +- tools/ci-checks/check-production-dashmap.py | 10 +- tools/cpp-checks/sanitize-paxos-image.py | 74 + tools/cpp-checks/sanitize-paxos-kv.sh | 84 + tools/cpp-checks/sanitize-regression.sh | 152 +- tools/cpp-checks/tree-link-isolation.sh | 22 +- .../tree-link-isolation/public_header.cpp | 1 + tools/load-tpch.py | 4 +- tools/pixi-tasks/build-release.sh | 4 +- tools/pixi-tasks/test.py | 6 +- tools/pixi-tasks/tree-asan.sh | 11 +- tools/pixi-tasks/tree-tsan.sh | 18 +- tools/pixi-tasks/tree-ubsan.sh | 13 +- tools/ports/README.md | 26 + tools/ports/default_ports.py | 22 + tools/ports/generate.py | 69 + tools/profiling/container-server-pids.py | 79 + tools/profiling/profile-write.sh | 264 +- tools/profiling/setup-perf.sh | 2 +- tools/rdma/run-chunk-kv.py | 5 +- 2557 files changed, 176467 insertions(+), 30742 deletions(-) delete mode 100644 .agents/skills/coding/SKILL.md delete mode 100644 .agents/skills/console-ui-e2e/SKILL.md delete mode 100644 .agents/skills/console-ui-e2e/references/diagnosis.md delete mode 100644 .agents/skills/debug-test/SKILL.md delete mode 100644 .agents/skills/doc-backlog/SKILL.md delete mode 100644 .agents/skills/doc-design/SKILL.md delete mode 100644 .agents/skills/doc-working-design/SKILL.md delete mode 100644 .agents/skills/doc-working-plan/SKILL.md delete mode 100644 .agents/skills/doc/SKILL.md delete mode 100644 .agents/skills/implement-requirement/SKILL.md delete mode 100644 .agents/skills/review/SKILL.md create mode 100644 app/crowdb-access-server/conf/crowdb_access_server_config.toml.ports.in create mode 100644 app/crowdb-chunk-kv-server/conf/crowdb_chunk_kv_server_config.toml create mode 100644 app/crowdb-chunk-kv-server/src/host_lifecycle.rs create mode 100644 app/crowdb-chunk-kv-server/src/metrics/resources.rs create mode 100644 app/crowdb-chunk-kv-server/src/rpc/tasks.rs create mode 100644 app/crowdb-chunk-kv-server/src/server/legacy_retirement.rs create mode 100644 app/crowdb-chunk-kv-server/src/server/resources.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/legacy_retirement.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/legacy_retirement/allocation.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/legacy_retirement/proof.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/legacy_retirement/record.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/legacy_migration.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/legacy_tracking.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/prepared.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/candidate.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/commit.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/guards.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/operation.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/pin_owner.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/process.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/prune.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/record.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/release.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/scan.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/writer.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/retention.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/staging.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog/transaction.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/root_catalog_test.rs create mode 100644 app/crowdb-chunk-kv-server/src/storage/tree_owner.rs create mode 100644 app/crowdb-chunk-kv-server/tests/backing_observation_test.rs create mode 100644 app/crowdb-chunk-kv-server/tests/common/legacy_retirement.rs create mode 100644 app/crowdb-chunk-kv-server/tests/common/root_process.rs create mode 100644 app/crowdb-chunk-kv-server/tests/legacy_retirement/allocation.rs create mode 100644 app/crowdb-chunk-kv-server/tests/legacy_retirement/readers.rs create mode 100644 app/crowdb-chunk-kv-server/tests/legacy_retirement_test.rs create mode 100644 app/crowdb-chunk-kv-server/tests/root_catalog_legacy_migration_test.rs create mode 100644 app/crowdb-chunk-kv-server/tests/root_catalog_prepared_staging_test.rs create mode 100644 app/crowdb-chunk-kv-server/tests/root_catalog_process_test.rs create mode 100644 app/crowdb-chunk-kv-server/tests/root_catalog_reference_test.rs create mode 100644 app/crowdb-chunk-kv-server/tests/root_catalog_staging_test.rs create mode 100644 app/crowdb-chunk-kv-server/tests/tree_config_test.rs create mode 100644 app/crowdb-chunkdb/src/lifecycle/handler/deletion.rs create mode 100644 app/crowdb-chunkdb/tests/allocation_fence_test.rs create mode 100644 app/crowdb-chunkdb/tests/allocation_rounding_test.rs create mode 100644 app/crowdb-chunkdb/tests/chunk_tree_runtime_test.rs rename app/crowdb-cli/src/commands/bench/{kv => }/client.rs (83%) delete mode 100644 app/crowdb-cli/src/commands/bench/kv.rs delete mode 100644 app/crowdb-cli/src/commands/bench/kv/prepare.rs delete mode 100644 app/crowdb-cli/src/commands/bench/kv/read.rs delete mode 100644 app/crowdb-cli/src/commands/bench/kv/scan.rs delete mode 100644 app/crowdb-cli/src/commands/bench/kv/write.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/args.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/client.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/collect.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/counters.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/dataset.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/deployment.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/deployment/memory.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/deployment/native.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/run.rs create mode 100644 app/crowdb-cli/src/commands/bench/paxos_kv/verification.rs delete mode 100644 app/crowdb-cli/src/commands/bench/result.rs create mode 100644 app/crowdb-cli/src/commands/bench/workload.rs rename app/crowdb-cli/src/commands/{kv.rs => paxos_kv.rs} (72%) rename app/crowdb-cli/src/commands/{kv => paxos_kv}/data.rs (94%) rename app/crowdb-cli/src/commands/{kv => paxos_kv}/logical.rs (100%) create mode 100644 app/crowdb-cli/tests/kv_bench_test.rs create mode 100644 app/crowdb-diskdb/conf/crowdb_diskdb_config.toml.ports.in create mode 100644 app/crowdb-diskio/src/rpc/dio_request_deadline.h create mode 100644 app/crowdb-diskio/tests/ordered_drain_test.cpp delete mode 100644 app/crowdb-kv-server/conf/crowdb_kv_server_config.toml rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/Cargo.toml (79%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/build.rs (100%) create mode 100644 app/crowdb-paxos-kv-server/conf/crowdb_paxos_kv_server_config.toml rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/examples/openapi-export.rs (91%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/discovery.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor.rs (96%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv.rs (99%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/balance.rs (99%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/balance/materialization.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/balance/observation.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/balance/selection.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/balance/selection/candidates.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/balance/selection/diagnostics.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/balance/split.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/catalog.rs (99%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/catalog/endpoints.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/grants.rs (97%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunk_kv/liveness.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunkdb.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunkdb_dynamic.rs (98%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunkdb_dynamic/owners.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/chunkdb_dynamic/snapshot.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/diskdb.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/domain_monitor/diskdb_placement.rs (99%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/identity.rs (97%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/background/keepalive.rs (96%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/cli.rs (95%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/engine_collector.rs (91%) create mode 100644 app/crowdb-paxos-kv-server/src/engine_collector/resources.rs rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/group0_control_plane.rs (97%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/lib.rs (71%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/main.rs (76%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt.rs (96%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/discovery.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/group_creation.rs (99%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/group_ops.rs (95%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/operation_registry.rs (80%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/replica_ops.rs (96%) create mode 100644 app/crowdb-paxos-kv-server/src/mgmt/runtime_shutdown.rs rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/store_ops.rs (90%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/system_bootstrap.rs (99%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/system_bootstrap/cleanup.rs (98%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/system_init.rs (94%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/system_join.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/system_store.rs (94%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/mgmt/topology.rs (94%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/recovery.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/recovery/group_rebuild.rs (94%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/recovery/member_routes.rs (94%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/recovery/reconcile.rs (94%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/recovery/restore.rs (78%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/recovery/startup.rs (72%) create mode 100644 app/crowdb-paxos-kv-server/src/shutdown.rs rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/src/store_registry.rs (86%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/chunk_dynamic_monitor_test.rs (95%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/chunk_kv_balance_planning_test.rs (97%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/chunk_kv_balance_planning_test/committed_transfer.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/chunk_kv_balance_planning_test/recovering_owner.rs (99%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/chunk_slot_monitor_test.rs (95%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/cli_parse_test.rs (85%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/balance_distribution.rs (98%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/balance_planning.rs (98%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/catalog_publication.rs (99%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/dynamic_slots.rs (96%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/endpoint_publication.rs (98%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/grant_isolation.rs (98%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/grant_process.rs (97%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/group_creation.rs (89%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/process_lease.rs (94%) create mode 100644 app/crowdb-paxos-kv-server/tests/common/runtime_shutdown.rs rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/split_materialization.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/split_materialization/transfer.rs (100%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/common/system_bootstrap.rs (87%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/diskdb_placement_test.rs (92%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/domain_monitor_test.rs (97%) create mode 100644 app/crowdb-paxos-kv-server/tests/engine_resource_metrics_test.rs rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/group0_control_plane_test.rs (85%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/group0_discovery_test.rs (92%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/group_creation_test.rs (98%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/keepalive_recovery_test.rs (87%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/member_routes_test.rs (95%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/monitor_supervisor_test.rs (94%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/reconcile_test.rs (89%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/restore_test.rs (89%) create mode 100644 app/crowdb-paxos-kv-server/tests/runtime_shutdown_test.rs rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/startup_test.rs (64%) rename app/{crowdb-kv-server => crowdb-paxos-kv-server}/tests/startup_test/reclamation.rs (96%) create mode 100644 app/crowdb-paxos-kv-server/tests/tree_config_test.rs create mode 100644 app/crowdb-paxos-kv-server/tests/tree_runtime_host_test.rs create mode 100644 app/crowdb-web/ui/src/ports.ts create mode 100644 container/oci-image/Dockerfile.ports.in create mode 100644 container/oci-image/compose-three.yaml.ports.in create mode 100644 container/oci-image/compose.yaml.ports.in create mode 100644 container/oci-image/node-profile.toml.ports.in create mode 100644 container/oci-image/paxos-kv-cluster.py create mode 100644 container/oci-image/ports.py create mode 100644 container/oci-image/tests/ports_test.py create mode 100644 container/single-node-container/dev-container.sh.ports.in create mode 100644 container/single-node-container/entrypoint.sh.ports.in create mode 100644 container/single-node-container/profile.toml.ports.in create mode 100644 container/single-node-container/templates/access.toml.ports.in create mode 100644 container/single-node-container/templates/chunk-kv.toml.ports.in create mode 100644 container/single-node-container/templates/chunkdb.toml.ports.in create mode 100644 container/single-node-container/templates/crowdb-web.toml.ports.in create mode 100644 container/single-node-container/templates/diskdb.toml.ports.in create mode 100644 container/single-node-container/templates/diskio.toml.ports.in create mode 100644 container/single-node-container/templates/kv.toml.ports.in delete mode 100644 doc/backlog/R102-diskdb-dynamic-binding-migration.md delete mode 100644 doc/backlog/R103-chunkdb-range-migration.md delete mode 100644 doc/backlog/R139-group0-service-config.md delete mode 100644 doc/backlog/R144-chunk-kv-partition-merge.md delete mode 100644 doc/backlog/R147-tree-chunk-gc.md delete mode 100644 doc/backlog/R148-chunk-stream-scale-out.md delete mode 100644 doc/backlog/R168-s3-shared-object-reclamation.md delete mode 100644 doc/backlog/R169-s3-shared-chunk-tree-gc.md delete mode 100644 doc/backlog/R185-access-iceberg-cache-invalidation.md delete mode 100644 doc/backlog/R186-access-iceberg-orc-validation.md delete mode 100644 doc/backlog/R193-chunkdb-node-failure-budget.md delete mode 100644 doc/backlog/R196-refine-benchmark-tests.md delete mode 100644 doc/backlog/R207-chunkdb-repo-metadata-chunk-kv.md delete mode 100644 doc/backlog/R233-sdk-local-cache-checkpoint.md delete mode 100644 doc/backlog/R4-bounded-mempool.md delete mode 100644 doc/backlog/R60-tree-scan-sibling-leaf-readahead.md delete mode 100644 doc/backlog/R80-diskdb-rebalance.md delete mode 100644 doc/backlog/R82-kv-watch-notify-coalescing.md delete mode 100644 doc/backlog/R83-chunkdb-complete-recovery-flow.md delete mode 100644 doc/backlog/R84-chunkdb-post-disk-move-placement-scanner.md delete mode 100644 doc/backlog/R92-chunkdb-in-chunk-gc.md delete mode 100644 doc/backlog/R95-chunkdb-chunk-range-delete.md delete mode 100644 doc/backlog/R96-chunkdb-console-cli-integration.md delete mode 100644 doc/backlog/backlog.md rename doc/design/kv/{design-crowdb-kv-group0.md => design-crowdb-paxos-kv-group0.md} (90%) rename doc/design/kv/{design-crowdb-kv-leader-election.md => design-crowdb-paxos-kv-leader-election.md} (96%) rename doc/design/kv/{design-crowdb-kv-observability.md => design-crowdb-paxos-kv-observability.md} (96%) rename doc/design/kv/{design-crowdb-kv-reconfiguration.md => design-crowdb-paxos-kv-reconfiguration.md} (95%) rename doc/design/kv/{design-crowdb-kv-rpc-client.md => design-crowdb-paxos-kv-rpc-client.md} (89%) rename doc/design/kv/{design-crowdb-kv-rpc.md => design-crowdb-paxos-kv-rpc.md} (93%) rename doc/design/kv/{design-crowdb-kv-server.md => design-crowdb-paxos-kv-server.md} (89%) rename doc/design/kv/{design-crowdb-kv-slot.md => design-crowdb-paxos-kv-slot.md} (94%) rename doc/design/kv/{design-crowdb-kv-state-machine.md => design-crowdb-paxos-kv-state-machine.md} (92%) rename doc/design/kv/{design-crowdb-kv-test.md => design-crowdb-paxos-kv-test.md} (80%) rename doc/design/kv/{design-crowdb-kv-wal.md => design-crowdb-paxos-kv-wal.md} (96%) rename doc/design/kv/{design-crowdb-kv-watch-notify.md => design-crowdb-paxos-kv-watch-notify.md} (96%) rename doc/design/kv/{design-crowdb-kv.md => design-crowdb-paxos-kv.md} (92%) delete mode 100644 doc/working/plan-rdma-multi-node.md delete mode 100644 doc/working/test.md delete mode 100644 doc/working/todo_code.md create mode 100644 e2e-test/default_ports.py create mode 100644 e2e-test/tests/kv/slot_map_recovery.rs create mode 100644 e2e-test/tests/paxos_kv_diagnostics_test.py create mode 100644 lib/crowdb-chunk-kv/src/memory/prepared_split.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/config.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/split/inheritance.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/split/journal_proof.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/tree/snapshot.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/tree/split_overlay.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/worker_completion.rs create mode 100644 lib/crowdb-chunk-kv/tests/common/split_publish_tree.rs create mode 100644 lib/crowdb-chunk-kv/tests/immutable_runtime_split_test.rs create mode 100644 lib/crowdb-chunk-kv/tests/partition_empty_cursor_test.rs create mode 100644 lib/crowdb-chunk-kv/tests/prepared_split_overlay_test.rs create mode 100644 lib/crowdb-chunk-kv/tests/split_cutover_journal_test.rs create mode 100644 lib/crowdb-common/cpp/include/crowdb-common/metrics/inline_callback_gauge.h create mode 100644 lib/crowdb-common/cpp/src/diskio_owned.cpp create mode 100644 lib/crowdb-common/cpp/src/metrics/metrics_inline.cpp create mode 100644 lib/crowdb-common/cpp/tests/diskio_owned_test.cpp create mode 100644 lib/crowdb-common/rust/src/metrics/sampler.rs create mode 100644 lib/crowdb-common/rust/tests/metrics_sampler_test.rs rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/Cargo.toml (71%) rename lib/{crowdb-kv-client/include/crowdb-kv-client => crowdb-paxos-kv-client/include/crowdb-paxos-kv-client}/c_api.h (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/binding.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/binding/chunk_slots.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/binding/chunk_slots/handoff.rs (100%) create mode 100644 lib/crowdb-paxos-kv-client/src/binding/chunk_slots/initialization.rs rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/binding/chunk_slots/publication.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/binding/chunk_slots/publication/epochs.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/binding/chunkdb_strategy.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/binding/framework.rs (97%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/binding/range.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client.rs (74%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/admin.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/core.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/core/conditional.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/core/operations.rs (97%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/core/owned.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/core/scans.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/retry.rs (97%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/topology.rs (99%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/topology/address.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/client/topology/recovery.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/config.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/error.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/ffi.rs (97%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/ffi/disk_exclusion.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/ffi/diskio_scope.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/disk_exclusion.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/disk_identity.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/disk_status.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/group_identity.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/legacy_cascade.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/legacy_media.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/mutation.rs (92%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/owner_session.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/ownership.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_activation.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_admission.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_compatibility.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_guard.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_health.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_observation.rs (99%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_online.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_online/completion.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_page.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_rejoin.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_removal.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_removal/conditions.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_resource.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_resource/authority.rs (93%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/hierarchy/storage_resource/legacy.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/space_usage.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/hardware/sysmd.rs (94%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/lib.rs (81%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/membership.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/metrics.rs (99%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/discovery.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/domain_monitor.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/registry.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/registry/diskio.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/registry/session.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/stream_lease.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/stream_lease/authority.rs (97%) create mode 100644 lib/crowdb-paxos-kv-client/src/service/stream_lease/exclusion.rs rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/stream_lease/observation.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/service/watch_notify.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/transport.rs (65%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/transport/cluster.rs (92%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/transport/comparison.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/transport/rpc_transport.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/transport/rpc_transport/connections.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/transport/rpc_transport/owned.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/src/transport/rpc_transport/point_reads.rs (96%) create mode 100644 lib/crowdb-paxos-kv-client/tests/chunk_slot_initialization_test.rs rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/chunk_slot_owner_fence_test.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/chunk_slot_publication_test.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/chunkdb_partition_test.rs (83%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/chunk_slot_publication.rs (83%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/conditional_servers.rs (88%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/owner_fence_server.rs (70%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/point_read_peer.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/publication_peer.rs (100%) create mode 100644 lib/crowdb-paxos-kv-client/tests/common/slot_map.rs rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/storage.rs (90%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/stream_lease.rs (70%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/stream_lease_clock.rs (96%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/common/topology_recovery.rs (90%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/comparison_wire_test.rs (75%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/conditional_observation_test.rs (97%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/conditional_retry_test.rs (88%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/disk_identity_test.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/disk_resource_test.rs (99%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/disk_status_test.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/diskdb_session_test.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/e2e_follower_read_test.rs (95%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/e2e_single_node_test.rs (90%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/ffi_exclusion_test.rs (93%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/group_identity_test.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/journal_scan_test.rs (88%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/metrics_lock_test.rs (94%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/multi_seed_discovery_test.rs (90%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/point_read_isolation_test.rs (86%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/storage_admission_test.rs (99%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/storage_compatibility_test.rs (99%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/storage_exclusion_test.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/storage_health_test.rs (100%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/storage_rejoin_test.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/storage_removal_test.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/stream_process_authority_test.rs (98%) create mode 100644 lib/crowdb-paxos-kv-client/tests/stream_process_exclusion_test.rs rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/stream_process_lease_test.rs (99%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/stream_process_observation_test.rs (98%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/sysmd_client_test.rs (76%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/topology_publication_test.rs (93%) rename lib/{crowdb-kv-client => crowdb-paxos-kv-client}/tests/topology_retry_discovery_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/Cargo.toml (81%) rename lib/{crowdb-kv => crowdb-paxos-kv}/benches/slot_list.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/benches/wal.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/benches/wal_bench_history.md (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/build.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster.rs (92%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_accept.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_cas.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_coalesce.rs (82%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_config.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_election.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_election/recovery.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_election_candidate.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_election_leader.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_fetchgap.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_inflight.rs (86%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_maintenance.rs (82%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_membership.rs (91%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_operations.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_owner_fence.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_prepare.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_propose.rs (91%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_retention.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_routes.rs (100%) create mode 100644 lib/crowdb-paxos-kv/src/cluster/group_shutdown.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/group_tests.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/kv_store.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/local_replica.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/local_replica_accept.rs (92%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/local_replica_apply.rs (75%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/local_replica_election.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/local_replica_replay.rs (85%) create mode 100644 lib/crowdb-paxos-kv/src/cluster/local_replica_shutdown.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/node_config.rs (100%) rename lib/{crowdb-kv/src/cluster/kv_server.rs => crowdb-paxos-kv/src/cluster/paxos_kv_server.rs} (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/px_kv_store.rs (94%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/px_kv_store_handler.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/px_kv_store_owner.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/remote_replica.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/replica.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/status.rs (97%) create mode 100644 lib/crowdb-paxos-kv/src/cluster/store_shutdown.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/src/cluster/watch_registry.rs (91%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/common.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/common/config.rs (90%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/common/logging.rs (72%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/common/report.rs (80%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/common/time.rs (81%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/io.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/io/async_file.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/kv.rs (84%) create mode 100644 lib/crowdb-paxos-kv/src/kv/admission.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/src/kv/crowdb_tree_engine.rs (56%) create mode 100644 lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/admission.rs create mode 100644 lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/maintenance.rs create mode 100644 lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/metrics.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/src/kv/kv_engine.rs (85%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/kv/kv_future.rs (82%) create mode 100644 lib/crowdb-paxos-kv/src/kv/maintenance.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/src/kv/op.rs (59%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/lib.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/metrics.rs (92%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/paxos.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/paxos/acceptor.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/paxos/error.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/paxos/learner.rs (93%) create mode 100644 lib/crowdb-paxos-kv/src/paxos/learner/accepted_apply.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/src/paxos/roles.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/paxos/slot_list.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/paxos/slot_node.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc.rs (93%) rename lib/{crowdb-kv/src/rpc/kv_rpc_service.rs => crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service.rs} (98%) rename lib/{crowdb-kv/src/rpc/kv_rpc_service => crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service}/comparison.rs (100%) rename lib/{crowdb-kv/src/rpc/kv_rpc_service => crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service}/owner_fence.rs (100%) rename lib/{crowdb-kv/src/rpc/kv_rpc_service => crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service}/response_correlation.rs (100%) rename lib/{crowdb-kv/src/rpc/kv_rpc_service => crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service}/watch.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_service.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_service/accept.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_service/election.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_service/heartbeat.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_service/notifications.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_service/prepare.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_service/responses.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_service/snapshots.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_transport.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_transport/connection.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/px_rpc_transport/heartbeat.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/rpc/snapshot_registry.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/block_backend.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/file_backend.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/gc.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/index.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/io_backend.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/pipeline.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/pipeline_backend.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/pipeline_writer.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/record.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/replay.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/replay/records.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/segment.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/segment/read_buffer.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/uring_backend.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/wal_engine.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/src/wal/wal_file.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common.rs (82%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common/cluster.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common/logging.rs (86%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common/net_lock.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common/payload_owner.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common/simdisk.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common/test_client.rs (89%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common/test_util.rs (82%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/common/timer.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/apply_loop_test.rs (73%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/frontier_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/heartbeat_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/lease_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/metrics_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/role_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/step_down_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/term_fence_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/election_test/vote_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/a3_crash_restart_no_data_loss_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/cas_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/coalesce_test.rs (72%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/election_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/fetch_gap_durability_test.rs (92%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/forward_correlation_test.rs (86%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/full_restart_delete_test.rs (94%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/g1_step_down_survival_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/g2_crash_restart_no_data_loss_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/g3_leader_change_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/g4_learner_stream_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/g5_recovery_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/g6_reconfig_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/g7_reconfig_remove_leader_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/group_config_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/group_propose_test.rs (83%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/group_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/kv_correctness_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/kv_edge_case_test.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/kv_forward_test.rs (85%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/kv_slot_retry_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/kv_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/learner_heartbeat_test.rs (92%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/maintenance_test.rs (94%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/membership_epoch_fence_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/owner_fence_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/paxos_error_test.rs (91%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/preemption_retry_test.rs (77%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/proposer_test.rs (73%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/quorum_drain_test.rs (82%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/r65_replication_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/recovery_backlog_test.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/remote_error_test.rs (89%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/retention_test.rs (94%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/safe_slot_test.rs (91%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/snapshot_join_test.rs (73%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/snapshot_slot_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/group_test/t1_early_ack_crash_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/kv_test.rs (91%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/kv_test/conformance_test.rs (75%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/kv_test/crowdb_tree_engine_test.rs (72%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/kv_test/kv_future_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/kv_test/mem_kv_impl_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/kv_test/mem_kv_test.rs (72%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/kv_test/op_codec_test.rs (77%) create mode 100644 lib/crowdb-paxos-kv/tests/kv_test/tree_work_metrics_test.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/metrics_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/node_config_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/paxos_test.rs (75%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/paxos_test/acceptor_test.rs (98%) create mode 100644 lib/crowdb-paxos-kv/tests/paxos_test/cas_rejection_test.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/paxos_test/error_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/paxos_test/learner_async_test.rs (90%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/paxos_test/learner_dedup_test.rs (89%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/paxos_test/learner_test.rs (97%) create mode 100644 lib/crowdb-paxos-kv/tests/paxos_test/native_apply_window_test.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/paxos_test/roles_test.rs (97%) create mode 100644 lib/crowdb-paxos-kv/tests/paxos_test/runtime_gap_test.rs create mode 100644 lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_owner_test.rs create mode 100644 lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_rpc_test.rs create mode 100644 lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_test.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replay_compaction_memory_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replay_memory_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/concurrent_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/dedup_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/election_metrics_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/election_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/lease_read_test.rs (94%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/op_correctness_test.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/persistence_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/prepare_accept_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/replay_ordering_test.rs (90%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/replica_test/snapshot_test.rs (94%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/rpc_migration_test.rs (90%) create mode 100644 lib/crowdb-paxos-kv/tests/rpc_notification_test.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/slot_test.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/slot_test/concurrent_test.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/slot_test/reclaim_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/slot_test/slot_list_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test.rs (86%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/apply_fence_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/bounded_scan_test.rs (91%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/group_operations_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/health_test.rs (85%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/journal_recovery_test.rs (70%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/kv_correctness_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/multi_group_test.rs (94%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/multi_node_multi_group_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/node_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/persistence_test.rs (90%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/read_metrics_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/readindex_batch_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/shutdown_test.rs (88%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/shutdown_under_load_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/snapshot_api_test.rs (84%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/status_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/store_test/wal_isolation_test.rs (92%) create mode 100644 lib/crowdb-paxos-kv/tests/tree_admission_test.rs create mode 100644 lib/crowdb-paxos-kv/tests/tree_idle_writeback_test.rs create mode 100644 lib/crowdb-paxos-kv/tests/tree_maintenance_test.rs create mode 100644 lib/crowdb-paxos-kv/tests/tree_native_admission_test.rs create mode 100644 lib/crowdb-paxos-kv/tests/tree_snapshot_owner_test.rs rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test.rs (100%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/block_backend_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/block_restore_test.rs (93%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/block_runtime_test.rs (95%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/file_backend_test.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/file_restore_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/gc_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/index_test.rs (98%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/io_backend_test.rs (99%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/pipeline_backend_test.rs (94%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/pipeline_writer_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/record_test.rs (97%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/replay_compaction_test.rs (92%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/replay_test.rs (84%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/segment_test.rs (96%) rename lib/{crowdb-kv => crowdb-paxos-kv}/tests/wal_test/wal_engine_test.rs (98%) create mode 100644 lib/crowdb-paxos-kv/tests/watch_registry_test.rs create mode 100644 lib/crowdb-protocol/cpp/include/crowdb-protocol/flatbuffer.h create mode 100644 lib/crowdb-protocol/cpp/include/crowdb-protocol/ports.h create mode 100644 lib/crowdb-rpc/cmake/folly-project.cmake create mode 100644 lib/crowdb-rpc/cmake/folly-thread-sanitizer.cmake create mode 100644 lib/crowdb-rpc/ffi/src/buffer/owned.rs create mode 100644 lib/crowdb-rpc/ffi/src/sys/owner.rs create mode 100644 lib/crowdb-rpc/ffi/tests/accounted_owner_test.rs create mode 100644 lib/crowdb-rpc/include/crowdb-rpc/control_allocator.h create mode 100644 lib/crowdb-rpc/include/crowdb-rpc/control_allocator_c_api.h create mode 100644 lib/crowdb-rpc/src/c_api/buffer_owner.cpp create mode 100644 lib/crowdb-rpc/tests/control_allocator_test.cpp create mode 100644 lib/crowdb-rpc/tests/rdma_listen_hardware_test.cpp create mode 100644 lib/crowdb-tree/bench/parallel_flush_profile.cpp create mode 100644 lib/crowdb-tree/cmake/prepare_stdexec.py create mode 100644 lib/crowdb-tree/cmake/stdexec-pool-base.json create mode 100644 lib/crowdb-tree/cmake/stdexec-pool.patch create mode 100644 lib/crowdb-tree/ffi/src/admission.rs create mode 100644 lib/crowdb-tree/ffi/src/async_tree/chunk_collection.rs create mode 100644 lib/crowdb-tree/ffi/src/async_tree/compaction.rs create mode 100644 lib/crowdb-tree/ffi/src/async_tree/maintenance.rs create mode 100644 lib/crowdb-tree/ffi/src/async_tree/owned_get.rs create mode 100644 lib/crowdb-tree/ffi/src/async_tree/scan.rs create mode 100644 lib/crowdb-tree/ffi/src/backend_release.rs create mode 100644 lib/crowdb-tree/ffi/src/batch/allocation.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/legacy.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/pin_service.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/protection.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/protection_callbacks.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/protection_callbacks/mutation.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/runtime.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/stage.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/stage/codec.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/stage_callbacks.rs create mode 100644 lib/crowdb-tree/ffi/src/chunk/test_util.rs create mode 100644 lib/crowdb-tree/ffi/src/config/engine.rs create mode 100644 lib/crowdb-tree/ffi/src/config/runtime.rs create mode 100644 lib/crowdb-tree/ffi/src/immutable_snapshot.rs create mode 100644 lib/crowdb-tree/ffi/src/maintenance.rs create mode 100644 lib/crowdb-tree/ffi/src/owner_budget.rs create mode 100644 lib/crowdb-tree/ffi/src/runtime.rs create mode 100644 lib/crowdb-tree/ffi/src/runtime/stats.rs create mode 100644 lib/crowdb-tree/ffi/src/runtime/usage.rs create mode 100644 lib/crowdb-tree/ffi/src/runtime/usage/window.rs create mode 100644 lib/crowdb-tree/ffi/src/scan/options.rs create mode 100644 lib/crowdb-tree/ffi/src/scan/owner.rs create mode 100644 lib/crowdb-tree/ffi/src/snapshot/owner.rs create mode 100644 lib/crowdb-tree/ffi/src/stats/backing.rs create mode 100644 lib/crowdb-tree/ffi/src/stats/state.rs create mode 100644 lib/crowdb-tree/ffi/src/stats/work.rs create mode 100644 lib/crowdb-tree/ffi/src/stats/work/counters.rs create mode 100644 lib/crowdb-tree/ffi/src/stats/work/histograms.rs create mode 100644 lib/crowdb-tree/ffi/src/stats/work/waits.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/admission.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/backend_release.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/backing_stats.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/chunk_migration.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/chunk_pin_service.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/chunk_protection.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/chunk_runtime.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/chunk_stage.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/compaction.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/immutable_snapshot.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/maintenance.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/owned_allocation.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/owned_budget.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/runtime.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/scan_owner.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/snapshot_owner.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/split_overlay.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/state_stats.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/wait_metrics.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/work_histograms.rs create mode 100644 lib/crowdb-tree/ffi/src/sys/work_metrics.rs create mode 100644 lib/crowdb-tree/ffi/src/tree/split_overlay.rs create mode 100644 lib/crowdb-tree/ffi/src/tuning.rs create mode 100644 lib/crowdb-tree/ffi/tests/backend_release_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/backing_stats_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/chunk_pin_service_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/chunk_stage_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/common/external_owner.rs create mode 100644 lib/crowdb-tree/ffi/tests/dispatch_metrics_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/external_owner_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/immutable_snapshot_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/legacy_migration_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/owned_allocation_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/resource_usage_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_admission_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_async_scan_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_bytes_owner_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_compaction_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_maintenance_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_replay_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_scan_owner_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_snapshot_owner_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_source_retention_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_stats_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_wait_metrics_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_work_histograms_test.rs create mode 100644 lib/crowdb-tree/ffi/tests/runtime_work_metrics_test.rs create mode 100644 lib/crowdb-tree/include/crowdb-tree/apply_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/backend/backend_release_service.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/backend_release_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/backing_stats_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/btree/work_cause.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/chunk_migration_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/chunk_pin_service_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/chunk_protection_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/chunk_runtime_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/chunk_stage_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/external_allocation.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/immutable_snapshot_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/maintenance_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/maptable/mapping_geometry.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/memory_budget.h delete mode 100644 lib/crowdb-tree/include/crowdb-tree/memtable/generation.h delete mode 100644 lib/crowdb-tree/include/crowdb-tree/memtable/source.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/memtable/source_stats.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/owned_allocation_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/owned_budget_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/resource_usage.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/runtime_c_api.h delete mode 100644 lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/split_overlay_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/tree_execution_options.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/tree_runtime.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/tree_stats_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/tree_wait_metrics_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/tree_work_histograms_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/tree_work_metrics_c_api.h create mode 100644 lib/crowdb-tree/include/crowdb-tree/tuning_c_api.h create mode 100644 lib/crowdb-tree/src/backend/backend_release_c_api.cpp create mode 100644 lib/crowdb-tree/src/backend/backend_release_service.cpp create mode 100644 lib/crowdb-tree/src/backend/backing_io.cpp create mode 100644 lib/crowdb-tree/src/backend/backing_io.h create mode 100644 lib/crowdb-tree/src/backend/checkpoint_backend.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_backend_checkpoint.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_backend_state.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_backend_state.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_backing.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_backing.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_catalog_scan.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_collector.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_collector.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack_codec.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_io_context.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_legacy_runtime.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_migration_c_api.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_physical_registry.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_physical_registry.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_physical_registry_api.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_physical_registry_internal.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_pin_release.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_pin_release.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_pin_release_internal.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_pin_service_c_api.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_protection.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_protection_c_api.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_protection_callback.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_protection_callback.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_protection_mutation.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_runtime_c_api.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_runtime_ids.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_runtime_ids.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_runtime_page_store.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_runtime_page_store.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_stage.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_stage.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_stage_c_api.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_stage_callback.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_stage_codec.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_stage_gc.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_stage_gc.h create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_transport_registry.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/legacy_chunk_layout.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/legacy_chunk_layout.h create mode 100644 lib/crowdb-tree/src/backend/chunk/legacy_chunk_migration_receipt.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/legacy_chunk_read_store.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/legacy_chunk_read_store.h create mode 100644 lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.h create mode 100644 lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocate.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocation_identity.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/rpc_chunk_delete.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/rpc_chunk_drain.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/rpc_chunk_read_fence.cpp create mode 100644 lib/crowdb-tree/src/backend/local/backend_io_allocation.cpp create mode 100644 lib/crowdb-tree/src/backend/local/backend_io_allocation.h create mode 100644 lib/crowdb-tree/src/backend/local/backend_metadata.cpp create mode 100644 lib/crowdb-tree/src/backend/local/backend_metadata.h create mode 100644 lib/crowdb-tree/src/backend/local/block_account.cpp create mode 100644 lib/crowdb-tree/src/backend/local/block_account.h create mode 100644 lib/crowdb-tree/src/backend/local/block_async_io.cpp create mode 100644 lib/crowdb-tree/src/backend/local/block_async_io.h create mode 100644 lib/crowdb-tree/src/backend/local/block_directory.cpp create mode 100644 lib/crowdb-tree/src/backend/local/block_directory.h create mode 100644 lib/crowdb-tree/src/backend/local/block_directory_release.cpp create mode 100644 lib/crowdb-tree/src/backend/local/block_file_close.cpp create mode 100644 lib/crowdb-tree/src/backend/local/block_medium.cpp create mode 100644 lib/crowdb-tree/src/backend/local/block_operation_budget.cpp create mode 100644 lib/crowdb-tree/src/backend/local/block_page_store_directory.cpp create mode 100644 lib/crowdb-tree/src/backend/local/block_page_store_io.cpp create mode 100644 lib/crowdb-tree/src/backend/local/extent_ledger.cpp create mode 100644 lib/crowdb-tree/src/backend/local/extent_ledger.h create mode 100644 lib/crowdb-tree/src/backend/local/extent_ledger_allocate.cpp create mode 100644 lib/crowdb-tree/src/backend/local/extent_ledger_apply.cpp create mode 100644 lib/crowdb-tree/src/backend/local/extent_ledger_exclusion.cpp create mode 100644 lib/crowdb-tree/src/backend/local/extent_ledger_internal.h create mode 100644 lib/crowdb-tree/src/backend/local/extent_writeback.cpp create mode 100644 lib/crowdb-tree/src/backend/local/extent_writeback.h create mode 100644 lib/crowdb-tree/src/backend/local/local_backend_state.cpp create mode 100644 lib/crowdb-tree/src/backend/local/local_backend_state.h create mode 100644 lib/crowdb-tree/src/backend/local/local_block_gc.cpp create mode 100644 lib/crowdb-tree/src/backend/local/local_block_gc.h create mode 100644 lib/crowdb-tree/src/backend/local/local_sparse_gc.cpp create mode 100644 lib/crowdb-tree/src/backend/local/local_sparse_gc.h create mode 100644 lib/crowdb-tree/src/backend/local/memory_backing_test.h create mode 100644 lib/crowdb-tree/src/backend/local/scheduled_page_store.cpp create mode 100644 lib/crowdb-tree/src/backend/local/scheduled_page_store.h create mode 100644 lib/crowdb-tree/src/backend/pack_allocation_bound.cpp create mode 100644 lib/crowdb-tree/src/backend/pack_backing.cpp create mode 100644 lib/crowdb-tree/src/backend/pack_physical_bound.cpp create mode 100644 lib/crowdb-tree/src/btree/admitted_batch.cpp create mode 100644 lib/crowdb-tree/src/btree/admitted_batch.h create mode 100644 lib/crowdb-tree/src/btree/admitted_batch_plan.cpp create mode 100644 lib/crowdb-tree/src/btree/allocation_account.cpp create mode 100644 lib/crowdb-tree/src/btree/allocation_account.h create mode 100644 lib/crowdb-tree/src/btree/allocation_account_control.cpp create mode 100644 lib/crowdb-tree/src/btree/allocation_account_credit.cpp create mode 100644 lib/crowdb-tree/src/btree/allocation_account_grant.cpp create mode 100644 lib/crowdb-tree/src/btree/allocation_account_grant.h create mode 100644 lib/crowdb-tree/src/btree/allocation_bound.h create mode 100644 lib/crowdb-tree/src/btree/allocation_observation.cpp create mode 100644 lib/crowdb-tree/src/btree/allocation_reservation.cpp create mode 100644 lib/crowdb-tree/src/btree/allocation_reservation.h create mode 100644 lib/crowdb-tree/src/btree/apply_dispatcher.cpp create mode 100644 lib/crowdb-tree/src/btree/apply_dispatcher.h create mode 100644 lib/crowdb-tree/src/btree/apply_frontier.cpp create mode 100644 lib/crowdb-tree/src/btree/apply_frontier.h create mode 100644 lib/crowdb-tree/src/btree/apply_progress.cpp create mode 100644 lib/crowdb-tree/src/btree/apply_progress.h create mode 100644 lib/crowdb-tree/src/btree/apply_waiters.cpp create mode 100644 lib/crowdb-tree/src/btree/apply_waiters.h create mode 100644 lib/crowdb-tree/src/btree/backing_identity.cpp create mode 100644 lib/crowdb-tree/src/btree/backing_identity.h create mode 100644 lib/crowdb-tree/src/btree/backing_metrics.cpp create mode 100644 lib/crowdb-tree/src/btree/backing_metrics.h create mode 100644 lib/crowdb-tree/src/btree/backing_ref.h create mode 100644 lib/crowdb-tree/src/btree/backing_stats_sampler.cpp create mode 100644 lib/crowdb-tree/src/btree/backing_stats_sampler.h create mode 100644 lib/crowdb-tree/src/btree/budget_memory_resource.cpp create mode 100644 lib/crowdb-tree/src/btree/budget_memory_resource.h create mode 100644 lib/crowdb-tree/src/btree/checkpoint_metrics.cpp create mode 100644 lib/crowdb-tree/src/btree/entry_key_arena.h create mode 100644 lib/crowdb-tree/src/btree/flush_batch.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_batch.h create mode 100644 lib/crowdb-tree/src/btree/flush_batch_budget.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_batch_dispatch.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_batch_handoff.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_batch_internal.h create mode 100644 lib/crowdb-tree/src/btree/flush_batch_route.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_batch_run.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_controller.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_controller_covered.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_controller_internal.h create mode 100644 lib/crowdb-tree/src/btree/flush_controller_pause.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_controller_plan.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_controller_test.h create mode 100644 lib/crowdb-tree/src/btree/flush_dispatch_metrics.h create mode 100644 lib/crowdb-tree/src/btree/flush_operation.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_operation.h create mode 100644 lib/crowdb-tree/src/btree/flush_operation_admission.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_operation_internal.h create mode 100644 lib/crowdb-tree/src/btree/flush_operation_stages.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_operation_wait.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_range_join.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_report_sender.h create mode 100644 lib/crowdb-tree/src/btree/flush_reservation.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_reservation.h create mode 100644 lib/crowdb-tree/src/btree/flush_reservation_internal.h create mode 100644 lib/crowdb-tree/src/btree/flush_reserve_bound.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_runtime.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_runtime.h create mode 100644 lib/crowdb-tree/src/btree/flush_runtime_internal.h create mode 100644 lib/crowdb-tree/src/btree/flush_runtime_progress.cpp create mode 100644 lib/crowdb-tree/src/btree/inline_function.h create mode 100644 lib/crowdb-tree/src/btree/local_compaction_internal.h create mode 100644 lib/crowdb-tree/src/btree/local_compaction_operation.cpp create mode 100644 lib/crowdb-tree/src/btree/local_compaction_operation.h create mode 100644 lib/crowdb-tree/src/btree/local_compaction_steps.cpp create mode 100644 lib/crowdb-tree/src/btree/logical_page.cpp create mode 100644 lib/crowdb-tree/src/btree/logical_page.h create mode 100644 lib/crowdb-tree/src/btree/logical_page_cursor.cpp create mode 100644 lib/crowdb-tree/src/btree/logical_page_cursor_window.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_cold.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_cold_siblings.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_inner_merge.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_leaf_merge.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_operation.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_operation.h create mode 100644 lib/crowdb-tree/src/btree/maintenance_operation_internal.h create mode 100644 lib/crowdb-tree/src/btree/maintenance_operation_step.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_publication.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_publication.h create mode 100644 lib/crowdb-tree/src/btree/maintenance_publication_internal.h create mode 100644 lib/crowdb-tree/src/btree/maintenance_publication_state.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_reserve.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_routing.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_split_layout.cpp create mode 100644 lib/crowdb-tree/src/btree/maintenance_split_layout.h create mode 100644 lib/crowdb-tree/src/btree/maintenance_split_step.cpp create mode 100644 lib/crowdb-tree/src/btree/merge_pair_metrics.cpp create mode 100644 lib/crowdb-tree/src/btree/merge_pair_metrics.h delete mode 100644 lib/crowdb-tree/src/btree/native_frames.h create mode 100644 lib/crowdb-tree/src/btree/native_import.h create mode 100644 lib/crowdb-tree/src/btree/native_import_dependencies.cpp create mode 100644 lib/crowdb-tree/src/btree/native_import_frames.cpp create mode 100644 lib/crowdb-tree/src/btree/native_iterator_owned.cpp create mode 100644 lib/crowdb-tree/src/btree/overflow_backing.h create mode 100644 lib/crowdb-tree/src/btree/owned_key.cpp create mode 100644 lib/crowdb-tree/src/btree/owned_key.h create mode 100644 lib/crowdb-tree/src/btree/page_cache.cpp create mode 100644 lib/crowdb-tree/src/btree/page_cache.h create mode 100644 lib/crowdb-tree/src/btree/page_cache_frame.cpp create mode 100644 lib/crowdb-tree/src/btree/page_cache_frame.h create mode 100644 lib/crowdb-tree/src/btree/page_cache_identity.h create mode 100644 lib/crowdb-tree/src/btree/page_cache_internal.h create mode 100644 lib/crowdb-tree/src/btree/page_cache_load.cpp create mode 100644 lib/crowdb-tree/src/btree/page_cache_queue.cpp create mode 100644 lib/crowdb-tree/src/btree/page_cache_residency.cpp create mode 100644 lib/crowdb-tree/src/btree/page_cache_residency.h create mode 100644 lib/crowdb-tree/src/btree/page_cache_residency_apply.cpp create mode 100644 lib/crowdb-tree/src/btree/page_cache_residency_internal.h create mode 100644 lib/crowdb-tree/src/btree/page_cache_residency_prepare.cpp create mode 100644 lib/crowdb-tree/src/btree/page_cache_trim.cpp create mode 100644 lib/crowdb-tree/src/btree/page_encode.cpp create mode 100644 lib/crowdb-tree/src/btree/page_encode.h create mode 100644 lib/crowdb-tree/src/btree/page_encode_frame.cpp create mode 100644 lib/crowdb-tree/src/btree/page_encode_internal.h create mode 100644 lib/crowdb-tree/src/btree/page_encode_leaf.cpp create mode 100644 lib/crowdb-tree/src/btree/page_encode_plan.cpp create mode 100644 lib/crowdb-tree/src/btree/page_key_range.h create mode 100644 lib/crowdb-tree/src/btree/page_layout.cpp create mode 100644 lib/crowdb-tree/src/btree/page_layout_stats.cpp create mode 100644 lib/crowdb-tree/src/btree/page_load.cpp create mode 100644 lib/crowdb-tree/src/btree/page_load.h create mode 100644 lib/crowdb-tree/src/btree/page_load_frame.cpp create mode 100644 lib/crowdb-tree/src/btree/page_load_inner.cpp create mode 100644 lib/crowdb-tree/src/btree/page_load_internal.h create mode 100644 lib/crowdb-tree/src/btree/page_load_leaf.cpp create mode 100644 lib/crowdb-tree/src/btree/page_load_leaf.h create mode 100644 lib/crowdb-tree/src/btree/page_load_lifetime.h create mode 100644 lib/crowdb-tree/src/btree/page_load_overflow.cpp create mode 100644 lib/crowdb-tree/src/btree/page_maintenance.cpp create mode 100644 lib/crowdb-tree/src/btree/page_maintenance.h create mode 100644 lib/crowdb-tree/src/btree/page_maintenance_structure.cpp create mode 100644 lib/crowdb-tree/src/btree/page_split_operation.cpp create mode 100644 lib/crowdb-tree/src/btree/page_split_operation.h create mode 100644 lib/crowdb-tree/src/btree/page_writeback.cpp create mode 100644 lib/crowdb-tree/src/btree/page_writeback.h create mode 100644 lib/crowdb-tree/src/btree/page_writeback_encode.cpp create mode 100644 lib/crowdb-tree/src/btree/page_writeback_internal.h create mode 100644 lib/crowdb-tree/src/btree/pending_page_times.cpp create mode 100644 lib/crowdb-tree/src/btree/pending_page_times.h create mode 100644 lib/crowdb-tree/src/btree/portable_import.cpp create mode 100644 lib/crowdb-tree/src/btree/portable_import.h create mode 100644 lib/crowdb-tree/src/btree/portable_import_frames.cpp create mode 100644 lib/crowdb-tree/src/btree/portable_import_plan.cpp create mode 100644 lib/crowdb-tree/src/btree/portable_import_plan.h create mode 100644 lib/crowdb-tree/src/btree/prepared_split_arm.cpp create mode 100644 lib/crowdb-tree/src/btree/prepared_split_install.cpp create mode 100644 lib/crowdb-tree/src/btree/prepared_split_internal.h create mode 100644 lib/crowdb-tree/src/btree/prepared_split_overlay.h create mode 100644 lib/crowdb-tree/src/btree/publication_admission.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_admission.h create mode 100644 lib/crowdb-tree/src/btree/publication_bootstrap.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_checkpoint.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_checkpoint.h create mode 100644 lib/crowdb-tree/src/btree/publication_checkpoint_completion.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_commit.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_complete_view.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_generation.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_generation_internal.h create mode 100644 lib/crowdb-tree/src/btree/publication_lane.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_lane.h create mode 100644 lib/crowdb-tree/src/btree/publication_maintenance.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_reconcile.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_snapshot.cpp create mode 100644 lib/crowdb-tree/src/btree/publication_snapshot.h create mode 100644 lib/crowdb-tree/src/btree/published_inspection.cpp create mode 100644 lib/crowdb-tree/src/btree/published_inspection.h create mode 100644 lib/crowdb-tree/src/btree/published_mapping.cpp create mode 100644 lib/crowdb-tree/src/btree/published_mapping_visit.cpp create mode 100644 lib/crowdb-tree/src/btree/published_read.cpp create mode 100644 lib/crowdb-tree/src/btree/published_read.h create mode 100644 lib/crowdb-tree/src/btree/published_read_budget.cpp create mode 100644 lib/crowdb-tree/src/btree/published_read_internal.h create mode 100644 lib/crowdb-tree/src/btree/published_read_pump.cpp create mode 100644 lib/crowdb-tree/src/btree/published_read_route.cpp create mode 100644 lib/crowdb-tree/src/btree/published_route.h create mode 100644 lib/crowdb-tree/src/btree/published_scan.cpp create mode 100644 lib/crowdb-tree/src/btree/published_scan_internal.h create mode 100644 lib/crowdb-tree/src/btree/published_scan_sources.cpp create mode 100644 lib/crowdb-tree/src/btree/published_state.cpp create mode 100644 lib/crowdb-tree/src/btree/published_state.h create mode 100644 lib/crowdb-tree/src/btree/read_value_bound.h create mode 100644 lib/crowdb-tree/src/btree/recovered_frame_identity.h create mode 100644 lib/crowdb-tree/src/btree/replay_controls.cpp create mode 100644 lib/crowdb-tree/src/btree/replay_controls.h create mode 100644 lib/crowdb-tree/src/btree/resource_budget.cpp create mode 100644 lib/crowdb-tree/src/btree/resource_budget.h create mode 100644 lib/crowdb-tree/src/btree/resource_observation.cpp create mode 100644 lib/crowdb-tree/src/btree/resource_observation.h create mode 100644 lib/crowdb-tree/src/btree/retired_shared.h create mode 100644 lib/crowdb-tree/src/btree/retirement_queue.cpp create mode 100644 lib/crowdb-tree/src/btree/retirement_queue.h create mode 100644 lib/crowdb-tree/src/btree/retirement_traits.h create mode 100644 lib/crowdb-tree/src/btree/runtime_range_native.cpp create mode 100644 lib/crowdb-tree/src/btree/runtime_range_portable.cpp create mode 100644 lib/crowdb-tree/src/btree/runtime_range_rebuild.cpp create mode 100644 lib/crowdb-tree/src/btree/runtime_range_rebuild.h create mode 100644 lib/crowdb-tree/src/btree/scan_output.h create mode 100644 lib/crowdb-tree/src/btree/snapshot_changes.cpp create mode 100644 lib/crowdb-tree/src/btree/snapshot_changes.h create mode 100644 lib/crowdb-tree/src/btree/source_admission.cpp create mode 100644 lib/crowdb-tree/src/btree/source_admission.h create mode 100644 lib/crowdb-tree/src/btree/source_admission_internal.h create mode 100644 lib/crowdb-tree/src/btree/source_capture.cpp create mode 100644 lib/crowdb-tree/src/btree/source_pause.cpp create mode 100644 lib/crowdb-tree/src/btree/source_pause.h create mode 100644 lib/crowdb-tree/src/btree/source_rotation.cpp create mode 100644 lib/crowdb-tree/src/btree/source_rotation.h create mode 100644 lib/crowdb-tree/src/btree/source_rotation_prepare.cpp create mode 100644 lib/crowdb-tree/src/btree/source_rotation_wait.cpp create mode 100644 lib/crowdb-tree/src/btree/source_tail.cpp create mode 100644 lib/crowdb-tree/src/btree/source_tail.h create mode 100644 lib/crowdb-tree/src/btree/source_tail_domain.cpp create mode 100644 lib/crowdb-tree/src/btree/source_write_budget.cpp create mode 100644 lib/crowdb-tree/src/btree/source_write_budget.h create mode 100644 lib/crowdb-tree/src/btree/source_write_pause.cpp create mode 100644 lib/crowdb-tree/src/btree/source_write_prepaid.cpp create mode 100644 lib/crowdb-tree/src/btree/source_writer.cpp create mode 100644 lib/crowdb-tree/src/btree/source_writer_prepared.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_apply_operation.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_apply_operation.h create mode 100644 lib/crowdb-tree/src/btree/tree_apply_operation_internal.h create mode 100644 lib/crowdb-tree/src/btree/tree_apply_operation_steps.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_apply_operation_wait.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_apply_pressure.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_apply_replay.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_checkpoint_diagnostics.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_access.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_admitted.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_apply.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_batch.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_batch.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_block_gc.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_block_gc_async.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_checkpoint.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_checkpoint_async.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_checkpoint_budget.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_checkpoint_internal.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_checkpoint_structure.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_checkpoint_work_async.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_chunk_gc.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_chunk_migration.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_chunk_open.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_compact.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_evict.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_facade.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_flush.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_generation.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_geometry.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_geometry.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_get.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_histograms.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_import.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_internal.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_lifecycle.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_metrics.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_open.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_pressure.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_read_budget.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_read_internal.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_scan.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_scan_budget.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_scan_internal.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_scan_sync.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_snapshot.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_snapshot_ids.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_source_stats.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_split_baseline.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_split_capture.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_split_handoff.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_split_internal.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_split_overlay.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_split_session.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_core_split_wait.h create mode 100644 lib/crowdb-tree/src/btree/tree_core_state_stats.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_diagnostics.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_generation_lease.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_generation_lease.h create mode 100644 lib/crowdb-tree/src/btree/tree_generation_lease_internal.h create mode 100644 lib/crowdb-tree/src/btree/tree_generation_prepare.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_metrics.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_metrics.h create mode 100644 lib/crowdb-tree/src/btree/tree_options.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_options.h create mode 100644 lib/crowdb-tree/src/btree/tree_reads.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_scans.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_snapshots.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_state_reconcile.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_state_reconcile.h create mode 100644 lib/crowdb-tree/src/btree/tree_state_reconcile_ledger.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_wait_metrics.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_wait_metrics.h create mode 100644 lib/crowdb-tree/src/btree/tree_work_age.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_work_counters.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_work_counters.h create mode 100644 lib/crowdb-tree/src/btree/tree_work_histograms.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_work_histograms.h create mode 100644 lib/crowdb-tree/src/btree/tree_work_registry.cpp create mode 100644 lib/crowdb-tree/src/btree/tree_work_registry.h create mode 100644 lib/crowdb-tree/src/btree/writeback_metrics.cpp create mode 100644 lib/crowdb-tree/src/btree/writeback_metrics.h create mode 100644 lib/crowdb-tree/src/btree/writeback_rebase.cpp create mode 100644 lib/crowdb-tree/src/btree/writeback_rebase.h create mode 100644 lib/crowdb-tree/src/buffer_external.cpp create mode 100644 lib/crowdb-tree/src/c_api_apply.cpp create mode 100644 lib/crowdb-tree/src/c_api_chunk_collection.cpp create mode 100644 lib/crowdb-tree/src/c_api_chunk_collection_start.cpp create mode 100644 lib/crowdb-tree/src/c_api_compaction.cpp create mode 100644 lib/crowdb-tree/src/c_api_future_internal.h create mode 100644 lib/crowdb-tree/src/c_api_get.cpp create mode 100644 lib/crowdb-tree/src/c_api_immutable_snapshot.cpp create mode 100644 lib/crowdb-tree/src/c_api_immutable_snapshot_internal.h create mode 100644 lib/crowdb-tree/src/c_api_immutable_snapshot_scan.cpp create mode 100644 lib/crowdb-tree/src/c_api_maintenance_internal.h create mode 100644 lib/crowdb-tree/src/c_api_maintenance_result.cpp create mode 100644 lib/crowdb-tree/src/c_api_maintenance_start.cpp create mode 100644 lib/crowdb-tree/src/c_api_maintenance_start.h create mode 100644 lib/crowdb-tree/src/c_api_owned_allocation.cpp create mode 100644 lib/crowdb-tree/src/c_api_owned_allocation.h create mode 100644 lib/crowdb-tree/src/c_api_owned_batch.cpp create mode 100644 lib/crowdb-tree/src/c_api_owned_budget.cpp create mode 100644 lib/crowdb-tree/src/c_api_owned_budget.h create mode 100644 lib/crowdb-tree/src/c_api_scan_owner.cpp create mode 100644 lib/crowdb-tree/src/c_api_snapshot_export.cpp create mode 100644 lib/crowdb-tree/src/c_api_split_overlay.cpp create mode 100644 lib/crowdb-tree/src/epoch_backend_close.cpp create mode 100644 lib/crowdb-tree/src/epoch_init.cpp create mode 100644 lib/crowdb-tree/src/epoch_internal.h create mode 100644 lib/crowdb-tree/src/epoch_retirement.cpp create mode 100644 lib/crowdb-tree/src/external_owner_bridge.cpp create mode 100644 lib/crowdb-tree/src/external_owner_bridge.h create mode 100644 lib/crowdb-tree/src/host_completion.h create mode 100644 lib/crowdb-tree/src/maptable/compressor_bounded.cpp create mode 100644 lib/crowdb-tree/src/memtable/cell_version.cpp create mode 100644 lib/crowdb-tree/src/memtable/prepared_cell.cpp create mode 100644 lib/crowdb-tree/src/memtable/prepared_write.cpp create mode 100644 lib/crowdb-tree/src/memtable/source_membership.cpp create mode 100644 lib/crowdb-tree/src/memtable/source_memory.cpp create mode 100644 lib/crowdb-tree/src/memtable/source_stats.cpp create mode 100644 lib/crowdb-tree/src/runtime/allocation_memory_resource.cpp create mode 100644 lib/crowdb-tree/src/runtime/allocation_memory_resource.h create mode 100644 lib/crowdb-tree/src/runtime/binding_metrics.h create mode 100644 lib/crowdb-tree/src/runtime/bootstrap_budget.cpp create mode 100644 lib/crowdb-tree/src/runtime/bootstrap_budget.h create mode 100644 lib/crowdb-tree/src/runtime/default_runtime.cpp create mode 100644 lib/crowdb-tree/src/runtime/default_runtime.h create mode 100644 lib/crowdb-tree/src/runtime/pool_storage.cpp create mode 100644 lib/crowdb-tree/src/runtime/pool_storage.h create mode 100644 lib/crowdb-tree/src/runtime/tree_operation_lifetime.cpp create mode 100644 lib/crowdb-tree/src/runtime/tree_operation_lifetime.h create mode 100644 lib/crowdb-tree/src/runtime/tree_runtime.cpp create mode 100644 lib/crowdb-tree/src/runtime/tree_runtime_binding.cpp create mode 100644 lib/crowdb-tree/src/runtime/tree_runtime_binding.h create mode 100644 lib/crowdb-tree/src/runtime/tree_runtime_c_api.cpp create mode 100644 lib/crowdb-tree/src/runtime/tree_runtime_c_api.h create mode 100644 lib/crowdb-tree/src/runtime/tree_runtime_internal.h create mode 100644 lib/crowdb-tree/src/runtime/tree_tuning_c_api.cpp create mode 100644 lib/crowdb-tree/src/runtime/tree_tuning_c_api.h create mode 100644 lib/crowdb-tree/src/snapshot/captured_frames.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_frames.h create mode 100644 lib/crowdb-tree/src/snapshot/captured_frames_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/captured_frames_load.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_frames_reuse.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_frames_step.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_plan.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_plan.h create mode 100644 lib/crowdb-tree/src/snapshot/captured_rebase.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_rebase.h create mode 100644 lib/crowdb-tree/src/snapshot/captured_rebase_batch.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_rebase_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/captured_rebase_step.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_rebase_summary.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_write.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_write.h create mode 100644 lib/crowdb-tree/src/snapshot/captured_write_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/captured_write_page.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_write_pump.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_write_relocate.cpp create mode 100644 lib/crowdb-tree/src/snapshot/captured_write_resident.cpp create mode 100644 lib/crowdb-tree/src/snapshot/checkpoint_allocation_bound.cpp create mode 100644 lib/crowdb-tree/src/snapshot/checkpoint_operation.cpp create mode 100644 lib/crowdb-tree/src/snapshot/checkpoint_operation.h create mode 100644 lib/crowdb-tree/src/snapshot/checkpoint_operation_backend.cpp create mode 100644 lib/crowdb-tree/src/snapshot/checkpoint_operation_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/checkpoint_operation_plan.cpp create mode 100644 lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp create mode 100644 lib/crowdb-tree/src/snapshot/checkpoint_operation_structure.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_index_codec.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_codec.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_codec.h create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_codec_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_commit.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_commit.h create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_commit_dependencies.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_commit_finish.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_commit_index.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_recovery.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_recovery.h create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_recovery_dependencies.cpp create mode 100644 lib/crowdb-tree/src/snapshot/chunk_root_recovery_frames.cpp create mode 100644 lib/crowdb-tree/src/snapshot/graph_summary.cpp create mode 100644 lib/crowdb-tree/src/snapshot/graph_summary.h create mode 100644 lib/crowdb-tree/src/snapshot/local_checkpoint.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_checkpoint.h create mode 100644 lib/crowdb-tree/src/snapshot/local_checkpoint_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/local_checkpoint_io.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_checkpoint_pump.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_checkpoint_registry.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_recovery.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_recovery.h create mode 100644 lib/crowdb-tree/src/snapshot/local_recovery_frames.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_recovery_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/local_recovery_io.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp create mode 100644 lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp create mode 100644 lib/crowdb-tree/src/snapshot/metadata_codec.cpp create mode 100644 lib/crowdb-tree/src/snapshot/metadata_codec.h create mode 100644 lib/crowdb-tree/src/snapshot/metadata_decode.cpp create mode 100644 lib/crowdb-tree/src/snapshot/metadata_decode.h create mode 100644 lib/crowdb-tree/src/snapshot/native_frame_read.cpp create mode 100644 lib/crowdb-tree/src/snapshot/native_frame_read.h create mode 100644 lib/crowdb-tree/src/snapshot/native_layout.cpp create mode 100644 lib/crowdb-tree/src/snapshot/native_layout.h create mode 100644 lib/crowdb-tree/src/snapshot/native_plan.cpp create mode 100644 lib/crowdb-tree/src/snapshot/native_plan.h create mode 100644 lib/crowdb-tree/src/snapshot/operation_wait.h create mode 100644 lib/crowdb-tree/src/snapshot/primary_backing.cpp create mode 100644 lib/crowdb-tree/src/snapshot/primary_backing.h create mode 100644 lib/crowdb-tree/src/snapshot/published_byte_export.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_byte_export.h create mode 100644 lib/crowdb-tree/src/snapshot/published_byte_export_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/published_byte_export_next.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_byte_export_plan.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_native.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_native.h create mode 100644 lib/crowdb-tree/src/snapshot/published_native_collect.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_native_collect.h create mode 100644 lib/crowdb-tree/src/snapshot/published_native_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/published_native_next.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_native_rewind.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot.h create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_budget.h create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_entries.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_entries.h create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_materialize.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_materialize.h create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_read.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_scan.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_view.cpp create mode 100644 lib/crowdb-tree/src/snapshot/published_snapshot_view.h create mode 100644 lib/crowdb-tree/src/snapshot/readable_page_catalog.h create mode 100644 lib/crowdb-tree/src/snapshot/recovered_state.cpp create mode 100644 lib/crowdb-tree/src/snapshot/recovered_state.h create mode 100644 lib/crowdb-tree/src/snapshot/recovered_state_graph.cpp create mode 100644 lib/crowdb-tree/src/snapshot/recovered_state_internal.h create mode 100644 lib/crowdb-tree/src/snapshot/recovered_state_leaf.cpp create mode 100644 lib/crowdb-tree/src/snapshot/recovered_state_load.cpp delete mode 100644 lib/crowdb-tree/src/snapshot/retry.cpp create mode 100644 lib/crowdb-tree/src/snapshot/runtime_snapshot_import.cpp create mode 100644 lib/crowdb-tree/src/snapshot/runtime_snapshot_import.h create mode 100644 lib/crowdb-tree/src/snapshot/runtime_snapshot_import_fields.cpp create mode 100644 lib/crowdb-tree/src/snapshot/runtime_snapshot_import_parse.cpp create mode 100644 lib/crowdb-tree/src/snapshot/snapshot_baseline.h create mode 100644 lib/crowdb-tree/src/snapshot/snapshot_crc.h create mode 100644 lib/crowdb-tree/src/snapshot/snapshot_export_materialized.cpp create mode 100644 lib/crowdb-tree/src/snapshot/snapshot_export_owned.cpp create mode 100644 lib/crowdb-tree/src/snapshot/snapshot_wire.h create mode 100644 lib/crowdb-tree/src/snapshot/stored_overflow.cpp create mode 100644 lib/crowdb-tree/src/snapshot/stored_overflow.h delete mode 100644 lib/crowdb-tree/src/snapshot/write.cpp create mode 100644 lib/crowdb-tree/src/snapshot/writeback_selection.cpp create mode 100644 lib/crowdb-tree/src/snapshot/writeback_selection.h create mode 100644 lib/crowdb-tree/src/tree_backing_stats_c_api.cpp create mode 100644 lib/crowdb-tree/src/tree_backing_stats_c_api_internal.h create mode 100644 lib/crowdb-tree/src/tree_progress_c_api.cpp create mode 100644 lib/crowdb-tree/src/tree_stats_c_api.cpp create mode 100644 lib/crowdb-tree/src/tree_wait_metrics_c_api.cpp create mode 100644 lib/crowdb-tree/src/tree_wait_metrics_c_api_internal.h create mode 100644 lib/crowdb-tree/src/tree_work_histograms_c_api.cpp create mode 100644 lib/crowdb-tree/src/tree_work_histograms_c_api_internal.h create mode 100644 lib/crowdb-tree/src/tree_work_metrics_c_api.cpp create mode 100644 lib/crowdb-tree/tests/allocation_account_test_support.h rename lib/crowdb-tree/{src/maptable/buffer_pool.cpp => tests/buffer_pool_test_support.cpp} (86%) rename lib/crowdb-tree/{include/crowdb-tree/maptable/buffer_pool.h => tests/buffer_pool_test_support.h} (69%) create mode 100644 lib/crowdb-tree/tests/captured_write_test_support.h create mode 100644 lib/crowdb-tree/tests/checkpoint_structure_test_support.h create mode 100644 lib/crowdb-tree/tests/common/test_chunk_backing.h create mode 100644 lib/crowdb-tree/tests/common/test_chunk_collector.h create mode 100644 lib/crowdb-tree/tests/common/test_chunk_physical.h create mode 100644 lib/crowdb-tree/tests/common/test_chunk_root.h create mode 100644 lib/crowdb-tree/tests/common/test_chunk_root_catalog.h create mode 100644 lib/crowdb-tree/tests/common/test_chunk_runtime_c_api.cpp create mode 100644 lib/crowdb-tree/tests/common/test_chunk_stage_backend.h create mode 100644 lib/crowdb-tree/tests/common/test_legacy_chunk.h create mode 100644 lib/crowdb-tree/tests/common/test_root_transport.h create mode 100644 lib/crowdb-tree/tests/common/test_runtime_catalog.h create mode 100644 lib/crowdb-tree/tests/common/test_stage_catalog.h create mode 100644 lib/crowdb-tree/tests/common/test_tree_core_chunk.h create mode 100644 lib/crowdb-tree/tests/compaction_fault_store.h create mode 100644 lib/crowdb-tree/tests/data/legacy-chunk-v7.bin create mode 100644 lib/crowdb-tree/tests/data/legacy-chunk-v7.json create mode 100644 lib/crowdb-tree/tests/extent_ledger_test_support.h create mode 100644 lib/crowdb-tree/tests/flush_preparation_test_support.h create mode 100644 lib/crowdb-tree/tests/inner_load_test_support.h create mode 100644 lib/crowdb-tree/tests/integration/backing_stats_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/borrowed_read_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/budgeted_flush_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/captured_frame_window_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/captured_frames_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/captured_write_backend_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/captured_write_dependency_lifetime_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/captured_write_resident_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/captured_write_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/checkpoint_handoff_report_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/checkpoint_operation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/checkpoint_retention_reuse_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/checkpoint_split_progress_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/checkpoint_structure_public_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/checkpoint_structure_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_backing_pin_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_backing_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_backing_visibility_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_collection_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_collection_operation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_legacy_runtime_open_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_physical_core_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_root_commit_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_runtime_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_snapshot_export_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/extent_writeback_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/facade_checkpoint_diagnostics_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/flush_capture_report_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/flush_continuation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/flush_dispatch_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/flush_operation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/flush_range_scaling_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/flush_reservation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/flush_routing_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/flush_window_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/host_writeback_window_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/immutable_chunk_snapshot_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/large_value_checkpoint_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/legacy_chunk_migration_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_block_gc_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_checkpoint_backend_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_checkpoint_failure_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_compaction_async_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_recovery_backend_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_recovery_failure_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_recovery_frames_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_recovery_lifetime_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/local_recovery_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/maintenance_cold_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/maintenance_operation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/memory_backing_gc_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/native_import_export_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/page_writeback_backend_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/page_writeback_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/parallel_flush_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/prepared_split_overlay_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/primary_layout_admission_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_byte_export_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_native_collect_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_native_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_read_delta_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_snapshot_context_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_snapshot_domain_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_snapshot_entries_range_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_snapshot_scan_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/published_snapshot_view_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/range_rebuild_chunk_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/range_runtime_binding_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/recovered_frame_identity_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/recovered_state_backend_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/recovered_state_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/rpc_chunk_collection_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/rpc_chunk_fixed_io_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/rpc_chunk_staging_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_block_compaction_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_eviction_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_inspection_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_migration_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_range_rebuild_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_read_window_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_snapshot_import_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_split_hint_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/runtime_summary_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/snapshot_change_effect_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/source_generation_rotation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/source_host_wait_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/source_rotation_report_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/source_rotation_wait_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/sparse_block_relocation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/staged_writeback_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_apply_grant_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_admitted_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_auto_write_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_block_account_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_checkpoint_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_chunk_read_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_chunk_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_chunk_writeback_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_empty_open_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_flush_read_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_generation_checkpoint_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_generation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_native_checkpoint_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_native_import_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_primary_reuse_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_public_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_replay_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_retirement_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_source_stats_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_split_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_core_sync_backend_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_generation_lease_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_host_progress_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_point_read_bound_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_retirement_observation_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_scan_total_window_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_source_controls_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_source_window_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_state_reconcile_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/tree_work_counters_test.cpp create mode 100644 lib/crowdb-tree/tests/load_budget_test_support.h create mode 100644 lib/crowdb-tree/tests/local_checkpoint_test_support.h create mode 100644 lib/crowdb-tree/tests/local_recovery_test_support.h rename lib/crowdb-tree/{include/crowdb-tree/maptable/mapping_segment.h => tests/mapping_segment_test_support.h} (54%) rename lib/crowdb-tree/{src/maptable/mapping_table.cpp => tests/mapping_table_test_support.cpp} (70%) rename lib/crowdb-tree/{include/crowdb-tree/maptable/mapping_table.h => tests/mapping_table_test_support.h} (53%) create mode 100644 lib/crowdb-tree/tests/native_import_test_support.h create mode 100644 lib/crowdb-tree/tests/overflow_load_test_support.h create mode 100644 lib/crowdb-tree/tests/page_cache_residency_test_support.h rename lib/crowdb-tree/{include/crowdb-tree/btree/descent.h => tests/page_descent_test_support.h} (73%) create mode 100644 lib/crowdb-tree/tests/page_load_delta_test_support.h create mode 100644 lib/crowdb-tree/tests/page_writeback_test_support.h create mode 100644 lib/crowdb-tree/tests/published_read_test_support.h create mode 100644 lib/crowdb-tree/tests/recovered_state_test_support.h create mode 100644 lib/crowdb-tree/tests/runtime_tree_access.h create mode 100644 lib/crowdb-tree/tests/snapshot_publication_test_support.h create mode 100644 lib/crowdb-tree/tests/stdexec_overlay_test.py create mode 100644 lib/crowdb-tree/tests/unit/admitted_batch_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/allocation_account_adoption_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/allocation_account_grant_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/allocation_account_interleaving_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/allocation_account_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/allocation_account_transfer_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/allocation_observation_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/allocation_reservation_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/apply_dispatcher_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/apply_frontier_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/apply_progress_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/backend_factory_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/backend_release_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/backend_release_service_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/block_account_grant_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/block_account_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/block_async_account_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/block_directory_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/block_release_service_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/block_sync_generation_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/budget_memory_resource_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/c_api_admission_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/c_api_maintenance_report_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/c_api_owned_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/c_api_read_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/c_api_scan_async_v2_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/c_api_scan_owner_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/c_api_snapshot_backend_owner_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/c_api_snapshot_owner_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/captured_rebase_reuse_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_collector_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_fixed_pack_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_physical_gc_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_physical_registry_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_protection_callback_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_registry_scope_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_root_codec_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_stage_gc_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/chunk_stage_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/default_runtime_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/epoch_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/extent_exclusion_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/extent_gc_lease_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/extent_ledger_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/external_allocation_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/external_owner_bridge_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/flush_report_sender_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/flush_runtime_progress_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/flush_runtime_scheduler_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/flush_runtime_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/flush_runtime_usage_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/frame_summary_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/graph_summary_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/host_completion_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/immutable_snapshot_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/logical_page_cursor_window_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/logical_page_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/maintenance_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/maintenance_inner_merge_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/maintenance_publication_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/maintenance_split_step_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/maintenance_window_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/memory_backing_failure_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/memory_backing_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/memtable_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/merge_pair_metrics_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/metrics_registry_allocation_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/native_import_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/native_layout_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/native_readable_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/owned_allocation_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/owned_key_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_cache_residency_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_cache_residency_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_cache_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_cache_trim_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_cache_wake_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_encode_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_load_delta_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_load_inner_proof_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_load_inner_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_load_lifetime_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_load_overflow_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_load_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/page_maintenance_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/portable_import_bound_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/portable_import_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/prepared_split_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/primary_backing_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_admission_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_bootstrap_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_checkpoint_capacity_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_checkpoint_failure_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_checkpoint_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_gc_floor_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_generation_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_handoff_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_lane_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_merge_pair_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_metrics_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_reconcile_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_safety_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/publication_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/published_inner_read_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/published_read_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/published_read_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/published_route_pruning_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/published_scan_options_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/published_scan_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/read_value_bound_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/resource_account_stats_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/resource_budget_progress_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/resource_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/resource_registration_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/resource_usage_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/retirement_queue_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/runtime_bootstrap_accounting_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/runtime_bootstrap_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/runtime_pool_storage_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/runtime_registry_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/runtime_snapshot_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/scan_output_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/scheduled_page_store_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/snapshot_changes_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/snapshot_crc_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/snapshot_metadata_codec_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/snapshot_metadata_decode_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/snapshot_publication_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_admission_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_ledger_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_memory_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_pause_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_prepared_write_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_rotation_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_tail_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_write_partition_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_write_reservation_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/source_writer_budget_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_core_geometry_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_core_read_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_operation_lifetime_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_physical_stats_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_progress_metrics_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_runtime_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_runtime_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_state_metrics_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_stats_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_tuning_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_wait_metrics_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_wait_metrics_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_work_age_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_work_age_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_work_counters_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_work_histograms_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_work_histograms_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/tree_work_metrics_c_api_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/writeback_metrics_test.cpp create mode 100644 lib/crowdb-tree/tests/unit/writeback_rebase_test.cpp create mode 100644 third-party/bytes/.github/workflows/ci.yml create mode 100644 third-party/bytes/.gitignore create mode 100644 third-party/bytes/CHANGELOG.md create mode 100644 third-party/bytes/CROWDB.md create mode 100644 third-party/bytes/Cargo.lock create mode 100644 third-party/bytes/Cargo.toml create mode 100644 third-party/bytes/Cargo.toml.orig create mode 100644 third-party/bytes/LICENSE create mode 100644 third-party/bytes/README.md create mode 100644 third-party/bytes/SECURITY.md create mode 100644 third-party/bytes/benches/buf.rs create mode 100644 third-party/bytes/benches/bytes.rs create mode 100644 third-party/bytes/benches/bytes_mut.rs create mode 100644 third-party/bytes/ci/miri.sh create mode 100644 third-party/bytes/ci/panic-abort.sh create mode 100644 third-party/bytes/ci/test-stable.sh create mode 100644 third-party/bytes/ci/tsan.sh create mode 100644 third-party/bytes/clippy.toml create mode 100644 third-party/bytes/crowdb-upstream.json create mode 100644 third-party/bytes/src/buf/buf_impl.rs create mode 100644 third-party/bytes/src/buf/buf_mut.rs create mode 100644 third-party/bytes/src/buf/chain.rs create mode 100644 third-party/bytes/src/buf/iter.rs create mode 100644 third-party/bytes/src/buf/limit.rs create mode 100644 third-party/bytes/src/buf/mod.rs create mode 100644 third-party/bytes/src/buf/reader.rs create mode 100644 third-party/bytes/src/buf/take.rs create mode 100644 third-party/bytes/src/buf/uninit_slice.rs create mode 100644 third-party/bytes/src/buf/vec_deque.rs create mode 100644 third-party/bytes/src/buf/writer.rs create mode 100644 third-party/bytes/src/bytes.rs create mode 100644 third-party/bytes/src/bytes/accounted_owner.rs create mode 100644 third-party/bytes/src/bytes/owner_lease.rs create mode 100644 third-party/bytes/src/bytes_mut.rs create mode 100644 third-party/bytes/src/fmt/debug.rs create mode 100644 third-party/bytes/src/fmt/hex.rs create mode 100644 third-party/bytes/src/fmt/mod.rs create mode 100644 third-party/bytes/src/lib.rs create mode 100644 third-party/bytes/src/loom.rs create mode 100644 third-party/bytes/src/serde.rs create mode 100644 third-party/bytes/tests/test_accounted_owner.rs create mode 100644 third-party/bytes/tests/test_buf.rs create mode 100644 third-party/bytes/tests/test_buf_mut.rs create mode 100644 third-party/bytes/tests/test_bytes.rs create mode 100644 third-party/bytes/tests/test_bytes_odd_alloc.rs create mode 100644 third-party/bytes/tests/test_bytes_vec_alloc.rs create mode 100644 third-party/bytes/tests/test_chain.rs create mode 100644 third-party/bytes/tests/test_debug.rs create mode 100644 third-party/bytes/tests/test_iter.rs create mode 100644 third-party/bytes/tests/test_limit.rs create mode 100644 third-party/bytes/tests/test_reader.rs create mode 100644 third-party/bytes/tests/test_serde.rs create mode 100644 third-party/bytes/tests/test_take.rs create mode 100644 tools/benchmark/bench-kv-common.sh create mode 100644 tools/cpp-checks/sanitize-paxos-image.py create mode 100644 tools/cpp-checks/sanitize-paxos-kv.sh create mode 100644 tools/ports/README.md create mode 100644 tools/ports/default_ports.py create mode 100644 tools/ports/generate.py create mode 100644 tools/profiling/container-server-pids.py diff --git a/.agents/skills/coding/SKILL.md b/.agents/skills/coding/SKILL.md deleted file mode 100644 index 30ddb1dc1..000000000 --- a/.agents/skills/coding/SKILL.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -name: coding -description: Apply CROWDB conventions while changing production or test code; not for read-only questions or reviews. ---- - - - - -# Coding - -Inspect the touched module and its callers. Read `doc/doc_index.md` and the one -matched design section only when the change alters documented behavior, -crosses module boundaries, or leaves architectural intent unclear. - -## Logging - -- `critical!`: broken invariant; include `next step:`. -- `error!`: recoverable failure and its handling. -- `warn!`: actionable anomaly or transient failure. -- `info!`: lifecycle and topology changes only; never per request. -- `debug!`: requests, hot-path decisions, and routine detail. -- Do not leave production `trace!` calls. - -Use structured Paxos fields when available: `store_id`, `group_id`, -`replica_l_id`, `replica_r_id`, `slot`, `ballot`. Instrument public Paxos -object methods so fields propagate. Defaults remain file=`debug`, console=`info`. - -## Tests and status - -- Put integration tests in `/tests/*_test.rs`; helpers belong in - `tests/common/` and use `Test*` names. Do not add inline test modules, - `tests/testkit/`, or fixtures under `src/`. -- Keep Paxos tests under `lib/crowdb-kv/tests/paxos_test/`, entered by - `lib/crowdb-kv/tests/paxos_test.rs`. -- Use `CROWDB_TEST_LOG=1` and `tests/common/logging.rs` for test tracing. -- Expose useful new state through `StatusLevel` or the relevant `*Status` - type in `lib/crowdb-kv/src/cluster/status.rs`; shared wire types live in - `lib/crowdb-protocol/src/mgmt.rs`. -- Track new TODO/FIXME items in `doc/todo_code.md`; remove both together. - -## Layout - -- Keep roots for entry points, facades, ABI boundaries, configuration, and - genuinely shared primitives. Put implementation under its owning product - domain; avoid catch-all `handlers`, `types`, and `utils` folders. -- Use the non-`mod.rs` layout: `foo.rs` + `foo/`; `foo.rs` contains module docs, - declarations, and deliberate re-exports. Keep internal children private - unless callers need them. -- Mirror public C++ subsystem ownership under `include//` and `src/`; - keep private headers with their implementation. Avoid forwarding headers - unless compatibility requires them. -- Use established domain names consistently. Keep one nameable responsibility - per module and separate domain invariants from runtime wiring. -- Improve nearby layout only when cohesive with the requested change. -- Keep code files near 300 lines; split before adding to one over 1000. Keep - functions near 40 lines, at most 80 for orchestration; split over 150. -- Use the narrowest visibility. Test hooks require `test-util` and `_for_tests`. - -## C++ clang-tidy - -- Run `pixi run tree-lint` before committing; it must exit 0. -- No per-line `NOLINT`. Disable checks in `.clang-tidy` with a comment. -- Auto-fix only with `--extra-arg=-w` (blocks compiler fix-its leaking - in). Build and run `pixi run test-cpp` after each check group. -- Review fix-its that change behavior or perf: `static`/`const` method - conversions, `std::move` removals, value-param → `const &`. -- Suppressed checks and reasons are listed in `.clang-tidy`. - -For visible console UI or Playwright work, also apply `/console-ui-e2e`. diff --git a/.agents/skills/console-ui-e2e/SKILL.md b/.agents/skills/console-ui-e2e/SKILL.md deleted file mode 100644 index 37a1fc056..000000000 --- a/.agents/skills/console-ui-e2e/SKILL.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -name: console-ui-e2e -description: Change visible CROWDB console UI or work on its Playwright E2E tests; not for backend-only changes or general questions. ---- - - - - -# Console UI E2E - -Apply to `app/crowdb-web/ui/src/**` and `e2e-test/clients/console-ui/**`, including -browser-test failures and runtime regressions. Run the affected spec for every -visible change; add a regression assertion for every UI bug fix. - -## Structure and cost - -- Specs: `e2e/flows/NN--.spec.ts`. Prefixes order the - single-worker suite: `0x` shell, `1x` physical, `2x` KV cluster, `3x` KV - data, `4x` inspector, `5x` capacity/DiskDB, `9x` cross-function. -- Extend the existing page-function spec; add one only for a new function. -- Register each mutating flow under the root runner with an owned release - fixture. Use inspected node IPs and fixed ports; host browser clients never - launch server binaries. Fresh fixtures provide empty-backend isolation. -- Preserve original distinct assertions when coalescing flow owners. Important - deterministic UI logic remains in component tests. -- Record `// Baseline: Xs (date)`; investigate runtime above 2x. -- Measure the current exact test before changing it; saved timing reports may - describe an earlier worktree. Compare Playwright's per-test time, not the - command wall time that includes UI build and web-server startup. -- Instrument setup, mutation response, lifecycle poll, DOM refresh, and - teardown separately with `stepTimer`. Optimize the first slow boundary. -- Before removing repeated setup from a cross-function flow, locate the - dedicated specs that retain each behavior and assertion. - -## Assertions - -- Trace handler -> JSON -> hook -> props -> DOM before asserting unfamiliar - state. Probe raw API data when its shape is uncertain. -- Prefer role, label, test-id, and scoped locators. Add `data-testid` for - ambiguity; never resolve it with page-level `.first()` or `.last()`. -- Assertions time out at 3 seconds; leader election at 10 seconds. - `expect.poll` uses `intervals: [100]`. -- Poll lifecycle state; never sleep. -- Do not assert on toast alerts. If one intercepts a click, click via `evaluate`. -- Never swallow errors, weaken assertions, add retries, or inflate waits. - Use `waitForResponse` only when the response is an assertion target, not as - a substitute for observing lifecycle or DOM state. - -## Diagnose - -For a failing, flaky, or slow E2E test, read -[references/diagnosis.md](references/diagnosis.md). Ordinary UI implementation -does not need that reference. - -## Run - -Follow [test and CI progression](../debug-test/SKILL.md#test-and-ci-progression) -when sequencing suites or CI jobs. Fix a failure with its affected spec/group, -then resume the remaining selection; use this skill for browser diagnosis. - -```sh -pixi run clean --runtime-only -pixi run test-container-e2e --scope console.operations --case --concurrency 1 --node-cpus 0.25 --cpus 8 --memory-mib 16384 -``` - -Use the root runner for the full Console container domain. `pixi run test-console-ui` -runs UI component tests and types without servers or Docker. Browser tests use -an installed system browser; do not install a private browser. If blocked, -keep the regression assertion and report why. diff --git a/.agents/skills/console-ui-e2e/references/diagnosis.md b/.agents/skills/console-ui-e2e/references/diagnosis.md deleted file mode 100644 index d2d7a1c55..000000000 --- a/.agents/skills/console-ui-e2e/references/diagnosis.md +++ /dev/null @@ -1,26 +0,0 @@ - - - -# Console E2E diagnosis - -- Reproduce the exact test, then its spec. If isolated execution passes, run - the original ordered selection and inspect shared state, IDs, ports, and - leaked processes; compare serial execution before changing code. -- Locate the first divergence with the trace, `error-context.md`, console - errors, raw APIs, and `stepTimer`. Separate backend mutation from DOM refresh. -- Scope strict locators to the named region. Never hide ambiguity with - page-level `.first()` or `.last()`. -- Reuse one `APIRequestContext` per poll phase. Poll resulting API state for - asynchronous services; use an explicit test-mode cadence only when the real - production interval dominates runtime. -- Verify lifecycle through both API state and the OS process list. - -Useful log signals: - -- `new standalone instance created`: transport was not shared. -- `no mgmt seeds configured`: invalid empty seed set. -- `no KV servers deployed`: cluster is not initialized. -- `no rpc endpoint resolved` or `no group-0 endpoint found`: no group-0 route. -- `group-0 query failed`: RPC failed and config fallback was used. -- `topology refresh failed` or `topology discovery failed`: verify whether the - seed is a store listen address or node RPC address. diff --git a/.agents/skills/debug-test/SKILL.md b/.agents/skills/debug-test/SKILL.md deleted file mode 100644 index 76d865870..000000000 --- a/.agents/skills/debug-test/SKILL.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -name: debug-test -description: Run CROWDB test and CI jobs incrementally, and diagnose non-browser failures from first divergence to root cause. ---- - - - - -# Debug Test - -Do not increase timeouts, suppress errors, weaken assertions, add caller-side -retries, or otherwise make only the symptom pass. - -## Test and CI progression - -For expensive suites or multiple jobs, preserve an ordered list of commands, -completed results, logs, relevant source/image inputs, and the next unrun item. -Use existing runner records where available. - -- Run until an error, then fix it before advancing the affected sequence. Do - not first run every job against a known defect. Prioritize bugs that make - subsequent tests slow, hang, or fail through the same dependency. -- For container E2E, use `--fail-fast` to stop queued cases while letting - already owned fixtures finish and clean up. Read failure records throughout - the log, not only its tail or completed-case count. Do not interrupt both a - launcher and its child; forwarded duplicate signals can break cleanup. -- Reproduce and repair the failure with its test or smallest meaningful test - group. Run that group's regression and relevant quality gates after the fix. -- Resume the remaining tests/jobs from the saved position. Retain earlier - passing results; repeat them only when the fix changes their behavior or - inputs. Preserve original ordering and coverage when selecting a remainder. -- Independent work already in flight may finish when it is unaffected. Stop - dispatching work that depends on the defect or would repeat its slowdown. -- Complete shared build prerequisites before parallel test jobs. Never run - concurrent builders against the same output directory; use independent - output directories when parallel builds are necessary. Perform shared - environment cleanup before dispatch; do not repeat `clean --runtime-only` while other - local jobs still own running fixtures. Host browser fixtures that require - stable Docker networks also share that constraint across local CI jobs; - keep their browser phases separate from other jobs' bridge creation/removal. -- Before long suites, inspect actual fixture slots and declared CPU/memory - costs. An outer parallel job count does not prove cases run concurrently. - Distinguish build, fixture startup, client execution, and cleanup time. -- Reuse valid builds and images. Rebuild when their relevant inputs change; - avoid unrelated feature/profile changes and repeated rebuilds while debugging. -- After the requested jobs have each been repaired and locally verified, run - the requested or required overall verification once as a closing gate. If - that gate finds another defect, repair its group and resume the affected - remainder rather than restarting valid completed groups from the beginning. -- Report actual coverage, failures, and unrun/blocked work. A focused pass does - not by itself establish that a whole suite or CI job passed. - -## Diagnose a failure - -1. Check relevant environment variables, stale processes, logs, persisted - state, and build artifacts. -2. Reproduce the exact test. If it passes alone, compare the affected suite - under its default concurrency and serial execution; inspect shared IDs, - ports, temp directories, and leaked processes. -3. List setup, action, and assertion steps. Find the first divergence using - logs, APIs, persisted data, and timing. -4. Classify it as code/design, timing/order, or environment. -5. For an unexplained process exit, analyze its crash report before changing code. -6. Add focused temporary instrumentation only when evidence is insufficient; - remove it afterward. -7. Fix the earliest upstream cause. Rerun the test, affected suite, and quality - gate. Add a regression test for a code bug. - -For browser or console UI tests, use `/console-ui-e2e` instead. During active -requirement work, its retry and blocking rules remain authoritative. diff --git a/.agents/skills/doc-backlog/SKILL.md b/.agents/skills/doc-backlog/SKILL.md deleted file mode 100644 index f8d0ebd89..000000000 --- a/.agents/skills/doc-backlog/SKILL.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: doc-backlog -description: Write a CROWDB requirement document as a testable high-level design and contract. ---- - - - - -# Backlog Requirement - -Write `doc/backlog/R**--.md` as one requirement's high-level -design and testable contract. It is removed by `/implement-requirement`. - -Keep the contract concrete: describe CrowDB data, state transitions, APIs, and -verification. Do not invent concepts for completeness. Cite primary sources -when borrowing terminology or behavior from another system, and distinguish -that reference from CrowDB's own semantics. - -Use this exact order: - -1. `### R**: — ` after the license. -2. Optional `Status` when the item is explicitly deferred or blocked; state the - reason and what would unblock it. -3. `Problem`: current behavior, impact, root cause, root-design link, and - concrete operator/system scenarios. -4. `Solution`: architecture, invariants, component responsibilities, numbered - work items naming real modules, and edge-case outcomes. If unsettled, list - the human decisions and alternatives instead of inventing one. -5. `Dependencies`: incoming/outgoing requirements, named artifacts, and - fallback for unlanded work. -6. `Acceptance`: one test case per solution and edge-case claim. -7. `Open Questions`, only for human decisions; include alternatives and trade-offs. - -Each acceptance case states setup -> action -> assertion, names the invariant, -and ends with `Unit test`, `Integration test`, or `E2E test`. End with exact -pixi test, fmt, and clippy commands. - -The requirement is the implementation's high-level design. Keep signatures, -file lists, sequencing, and other execution detail in the working plan; do not -create a separate working design. Requirement numbers are allowed here. Use an -ASCII flow only for three or more components or a primary/fallback path. Do not -use tables. diff --git a/.agents/skills/doc-design/SKILL.md b/.agents/skills/doc-design/SKILL.md deleted file mode 100644 index 28d4eae27..000000000 --- a/.agents/skills/doc-design/SKILL.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -name: doc-design -description: Write or refine permanent CROWDB architecture when that document itself is in scope; not a mandatory requirement cleanup step. ---- - -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# Formal Design - -Root docs define an area's architecture, rationale, non-goals, and sub-design -map. Sub-designs hold one topic's detail and link upstream with `Depends on:` -and `Satisfies:`. - -Use a license, `# CROWDB - Design: <Title>`, short scope introduction, a table -of contents, and numbered `## N. <Title>` sections. Include only applicable -content: problem/why, concepts and named invariants, design, neighbor -interactions, tunables, correctness, risks, and open questions. Give invariants -stable IDs such as `I1`. - -Describe current state only. Omit requirement numbers, change history, -before/after prose, file paths, and line numbers. Refer to searchable symbols. -Keep architecture in root docs, detail in sub-designs, and operations in the -relevant design or component README; link rather than repeat. - -Keep the design concrete and implementation-oriented. Define only concepts -needed by CrowDB's data structures, state transitions, interfaces, and -correctness rules; do not introduce product concepts merely to make the prose -sound complete. When reusing terminology or behavior from another system, -cite its primary documentation and state which part is only a reference, not a -CrowDB semantic. - -When explicitly promoting a working draft, remove temporary scaffolding and -requirement references, rewrite as current state, update `doc/doc_index.md`, -and delete the draft. diff --git a/.agents/skills/doc-working-design/SKILL.md b/.agents/skills/doc-working-design/SKILL.md deleted file mode 100644 index 671c43b5e..000000000 --- a/.agents/skills/doc-working-design/SKILL.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: doc-working-design -description: Write a temporary design exploration only when the user explicitly requests a separate draft; not for normal requirement implementation. ---- - -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# Working Design - -Write `doc/working/design-<topic>.md` only for an explicitly requested design -exploration. Normal `/implement-requirement` work uses the requirement as its -high-level design and records implementation detail in the working plan. - -Keep the exploration concrete and implementation-oriented. Introduce only -concepts needed to explain CrowDB data, state, interfaces, or verification. -When borrowing terminology or behavior, cite primary documentation and mark it -as external reference rather than CrowDB semantics. - -Link the backlog and root design; do not repeat their problem, dependencies, -architecture, or rationale. - -Include: - -- License, `# <Title> (R**)`, and a short introduction with upstream links and - landed dependencies. -- Numbered solution sections by component: why a new mechanism is needed, - signatures, behavior, failures, and fallback. -- `Scope`: every changed path and intended change. -- `Complexity`: High/Medium/Low and the actual implementation difficulty. -- `Test Design`: concrete unit and E2E cases for every acceptance item and - edge case; each states setup -> action -> assertion and invariant. -- `Module Structure`: annotated file tree. -- `Config Extensions` and `Server Wiring` when applicable. -- `Open Questions` only for unresolved human decisions, with trade-offs. - -Use real paths and symbols. Fence signatures and diagrams; use ordered prose -for behavior. Requirement references are allowed only while the draft is -temporary and are removed when it is folded into formal design. diff --git a/.agents/skills/doc-working-plan/SKILL.md b/.agents/skills/doc-working-plan/SKILL.md deleted file mode 100644 index 499d72282..000000000 --- a/.agents/skills/doc-working-plan/SKILL.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: doc-working-plan -description: Write and maintain a requirement execution plan. ---- - -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# Working Plan - -Write `doc/working/plan-<topic>.md` from the requirement's solution and -acceptance criteria. It holds implementation detail, is kept live, and is -deleted after completion. - -Keep tasks tied to concrete CrowDB data structures, state transitions, APIs, -and tests. Do not create new product concepts just to organize prose. Cite -primary sources for borrowed terminology or behavior and keep those references -separate from CrowDB's own contract. - -- Start with the license, `# <Title> Plan`, upstream links, and one-line goal. -- Group dependency-ordered tasks by phase or component. -- Use `- [ ] **Name**: action. Files: paths.`; `[~]` marks the only active - task and `[x]` a verified task. -- Split tasks until each is completable and maps to identifiable diff hunks. -- Record relevant symbols, signatures, ordering, edge cases, and migration - steps needed to execute the high-level design without duplicating it. -- Include a consolidated file list and tests grouped by unit, integration, E2E. -- Keep status truthful as work proceeds. -- Add `## Blocked` only under `/implement-requirement` conditions, recording - the decision/failure, alternatives or retries, and why work cannot continue. - -For a persistent plan, state the exception in its header and remove completed -tasks instead of deleting the file. diff --git a/.agents/skills/doc/SKILL.md b/.agents/skills/doc/SKILL.md deleted file mode 100644 index 1f2f0ace0..000000000 --- a/.agents/skills/doc/SKILL.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: doc -description: Route an unclear CROWDB documentation task or maintain general docs; do not combine with a known doc-type skill. ---- - -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# Documentation - -Start at `doc/doc_index.md`; open only the row and section matching the task. - -## Locations - -- `doc/design/<area>/design-crowdb-<area>.md`: architecture and rationale. -- `doc/design/<area>/design-crowdb-<area>-<topic>.md`: permanent topic detail. -- `doc/design/kv/kv-*-flow-analysis.md`: permanent KV path analysis. -- `doc/design/`: permanent architecture and operational behavior. -- `doc/backlog/`: requirement index and analysis. -- `doc/working/`: implementation plans and explicitly requested design drafts. - -## Rules - -- Keep one source of truth. Link by section anchor instead of copying text. -- Fix conflicting upstream design before downstream docs or code. -- Index permanent document changes, not backlog or working files. -- Permanent design describes current state; backlog requirements hold - high-level proposed design, and working plans hold execution detail. -- Split independent topics; delete working files when their work completes. -- Prefer concrete, tight prose and raw-readable bullets. Remove filler, - repetition, rhetorical openings, adjective lists, and excessive em dashes. - -When the target is a design, backlog requirement, design draft, or working -plan, use only its matching `/doc-*` guide instead of this router. diff --git a/.agents/skills/implement-requirement/SKILL.md b/.agents/skills/implement-requirement/SKILL.md deleted file mode 100644 index c2dda3418..000000000 --- a/.agents/skills/implement-requirement/SKILL.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: implement-requirement -description: Implement one CROWDB backlog requirement from its high-level design through focused verification and cleanup. ---- - -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# Implement Requirement - -Use for an item in `doc/backlog/backlog.md`. Its detail is the high-level -design and testable contract; do not create a separate design draft. - -1. Read the matched requirement and relevant code. Read the backlog index only - for selection, dependency ordering, or status; read design sections only to - resolve behavior or module interaction. -2. Update a missing high-level decision with `/doc-backlog`; block only on a - real human choice. -3. Create one `doc/working/plan-<topic>.md` with `/doc-working-plan`; keep all - file, symbol, sequencing, and test detail there. -4. Implement in plan order, keep it current, and commit coherent tasks. -5. Run affected acceptance tests and fmt/lint gates separately. -6. Update permanent design only for an in-scope architecture change. -7. Delete the completed requirement, backlog entry, and plan in a final cleanup - commit. - -Find tests with `pixi task list`; prefix server-spawning tests with -`pixi run clean --runtime-only &&`. Before push run fmt and `pixi run rs-lint`. - -Proceed autonomously except when a design has multiple valid choices with no -clear winner, or a test still fails after five root-cause-driven attempts. -Then record the alternatives or failed command, attempts, diagnosis, and exact -failure under plan `## Blocked`; commit that state and ask the user. - -`Open Questions` contains only decisions requiring user input. Keep unfinished -implementation work in the plan instead of presenting it as an open issue. - -This overrides ordinary commit cadence. Every commit passes its applicable -gate. diff --git a/.agents/skills/review/SKILL.md b/.agents/skills/review/SKILL.md deleted file mode 100644 index 8e908fd86..000000000 --- a/.agents/skills/review/SKILL.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -name: review -description: Review a CROWDB diff when code review or pre-push review is requested; not an automatic coding handoff step. ---- - -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# Review - -Review the diff, callers, and tests. Report concrete findings before style. -Apply repository code, test-placement, and module-ownership rules without -loading another skill solely for review. - -Treat `propose`, `accept`, `learn`, `kv_get`, `kv_put`, `kv_delete`, -and `kv_batch_write` as hot paths: - -- Borrow instead of copying; justify non-trivial clones. -- Pre-size collections or use stack storage. -- Do not add locks. `std::sync::Mutex` is limited to lifecycle work; - `tokio::sync::Mutex` requires state held across `.await` and user approval. -- Avoid nested `Arc`; retain it only when data outlives its parent. Never move - a borrow into a task that may outlive it. - -For crowdb-rpc flatbuffers, read fields from the frame. Do not create owned -intermediates or use allocating accessors on hot paths. Shared wrappers belong -in `crowdb-protocol`; copy payload only beyond frame lifetime. See -`doc/design/rpc/design-crowdb-rpc.md` section 6. - -Check correctness invariants, errors, shutdown, async lifetimes, status -exposure, dead code, dependencies, visibility, `Debug`, module cohesion, -integration-test placement, and performance regressions. Avoid non-test panic. -Prefer enum dispatch unless implementations are open across crates. Use -`get_or_try_init` for fallible initialization. - -For module cohesion, flag implementation left at roots, catch-all folders, -inconsistent Rust/C++ ownership, or unclear names. Recommend a concrete owner, -but keep relocation findings scoped to the changed area. - -`async_fn_in_trait` is the only documented clippy exception. Add any new -exception here before suppressing it. diff --git a/.clang-tidy b/.clang-tidy index cda17cdd9..e33df3a26 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,8 +1,10 @@ --- # Suppressed checks — fix-its are unsafe or the warning source is -# third-party macro noise. See coding SKILL.md § "C++ clang-tidy". +# third-party macro noise. # These checks can change ownership, ABI, overload resolution, or hot-path # behavior through their fix-its; keep them visible only when reviewed locally. +# Public member const-qualification changes C++ ABI and requires deliberate +# interface review rather than a lint-driven conversion. # Fixed async completion tokens transfer heap ownership through opaque context # pointers. NewDeleteLeaks does not model the backend invoking their deleters; # completion ownership is covered by ASan and async lifetime tests instead. @@ -24,6 +26,7 @@ Checks: >- -performance-unnecessary-value-param, -performance-move-const-arg, -readability-convert-member-functions-to-static, + -readability-make-member-function-const, -modernize-use-ranges, -modernize-avoid-c-arrays, -readability-non-const-parameter, diff --git a/AGENTS.md b/AGENTS.md index 79396c51e..1bf9045e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,13 +28,12 @@ and transport exposed through FFI. - Preserve existing work. Never hard-reset, revert, or restore files from an older commit without explicit approval. Use stash or a temporary branch. - Ordinary interactive work is committed only when asked, as one coherent - commit. Requirement work follows `/implement-requirement`. + commit. - Commit messages are single-line subjects without bodies, trailers, doc references, or requirement numbers. Code comments also omit those references. - Before committing, run the relevant gate through pixi: Rust fmt and clippy, changed C++ format/tree-lint, and affected tests. Fix ordinary failures up to three times; report confirmed pre-existing failures. - Requirement work uses `/implement-requirement` retry and blocking rules. - Playwright uses an installed system browser. Do not install one locally. - Use 60-second shell timeouts by default. Start hang-prone commands in the background and poll. Show complete output; do not hide errors with filters. @@ -51,17 +50,8 @@ and transport exposed through FFI. unilaterally change the spec or introduce UI behavior that is absent from the agreed spec. Each visible UI change needs focused test or E2E coverage. -## Skill dispatch +## Documentation -- Repository skills live only under `.agents/skills/`; do not create legacy - aliases or compatibility symlinks such as `.devin`. -- Code changes: `/coding`; add `/console-ui-e2e` only for visible UI or - Playwright work. -- Test execution and CI progression: `/debug-test`; use `/console-ui-e2e` for - browser/UI diagnosis. -- Docs: one matched `/doc-*` guide; `/doc` only for general or unclear targets. -- Requirements: `/implement-requirement`; open the detail directly and consult - the backlog index only for selection, ordering, or status. -- Pre-push or explicitly requested code review: `/review`. -- Design questions: one section selected through `doc/doc_index.md`. -- Operations/user behavior: the relevant `doc/design/` or component README. +- Use `doc/doc_index.md` to locate the relevant permanent design. +- Keep architecture and operational contracts under `doc/design/` or the + owning component README. Documentation must be usable from this repository. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index de5c99a6a..0eda9d6a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ Playwright uses an installed system browser; do not install a repository-local browser. See [pixi.toml](pixi.toml) for focused component tasks and [tools/README.md](tools/README.md) for their scripts. Native Iceberg and official SDK suites use the `iceberg-e2e` environment; see -[the test inventory](doc/working/test.md) for CI coverage and timing. +[the repository tools guide](tools/README.md) for CI coverage and timing collection. Container acceptance is a separate Linux amd64 gate requiring Docker: @@ -78,8 +78,8 @@ pixi run test-single-node-container Start documentation work at `doc/doc_index.md`. Permanent architecture and operational behavior belong under `doc/design/` or the relevant component -README, future contracts in `doc/backlog/`, and temporary execution plans in -`doc/working/`. +README. Keep documentation independently readable with links to files in this +repository or public sources. ## Code and tests diff --git a/Cargo.lock b/Cargo.lock index 13bfcda59..9a52ee31a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,8 +306,6 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cast" @@ -648,7 +646,7 @@ dependencies = [ "crowdb-chunk-client", "crowdb-chunk-kv-client", "crowdb-common", - "crowdb-kv-client", + "crowdb-paxos-kv-client", "crowdb-protocol", "data-encoding", "flatbuffers", @@ -694,7 +692,7 @@ dependencies = [ "crowdb-chunk-client", "crowdb-chunk-kv-client", "crowdb-common", - "crowdb-kv-client", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "flatbuffers", @@ -740,8 +738,8 @@ dependencies = [ "crowdb-console-shared", "crowdb-diskdb-client", "crowdb-diskio-client", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -782,7 +780,7 @@ dependencies = [ "crowdb-common", "crowdb-diskdb-client", "crowdb-diskio-client", - "crowdb-kv-client", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -822,7 +820,7 @@ dependencies = [ "async-trait", "bytes", "clap", - "crowdb-kv-client", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "futures", @@ -848,8 +846,8 @@ dependencies = [ "crowdb-chunk-kv-server", "crowdb-chunk-stream", "crowdb-common", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -876,8 +874,8 @@ dependencies = [ "crowdb-chunk-client", "crowdb-chunk-stream", "crowdb-common", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-test-harness", "futures", @@ -900,6 +898,9 @@ dependencies = [ "clap", "crossbeam-skiplist", "crowdb-chunk-client", + "crowdb-chunk-kv", + "crowdb-chunk-kv-server", + "crowdb-chunk-stream", "crowdb-chunkdb", "crowdb-chunkdb-client", "crowdb-common", @@ -907,11 +908,12 @@ dependencies = [ "crowdb-diskdb", "crowdb-diskdb-client", "crowdb-diskio-client", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", + "crowdb-tree-ffi", "dashmap", "flatbuffers", "futures", @@ -934,7 +936,7 @@ version = "0.3.0" dependencies = [ "arc-swap", "bytes", - "crowdb-kv-client", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "flatbuffers", @@ -956,7 +958,7 @@ dependencies = [ "crowdb-common", "crowdb-console-shared", "crowdb-diskdb-client", - "crowdb-kv-client", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -1006,9 +1008,9 @@ dependencies = [ "clap", "crowdb-chunkdb-client", "crowdb-console-shared", - "crowdb-kv", - "crowdb-kv-client", - "crowdb-kv-server", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", + "crowdb-paxos-kv-server", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -1047,8 +1049,8 @@ dependencies = [ "crowdb-diskdb", "crowdb-diskdb-client", "crowdb-diskio-client", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -1073,7 +1075,7 @@ dependencies = [ "arc-swap", "bytes", "crowdb-diskdb-client", - "crowdb-kv-client", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -1093,19 +1095,52 @@ dependencies = [ "bytes", "crowdb-common", "crowdb-diskio-client", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", + "crowdb-protocol", + "crowdb-rpc-ffi", + "flatbuffers", + "futures", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "crowdb-monitor" +version = "0.3.0" +dependencies = [ + "axum", + "clap", + "crowdb-chunk-kv-client", + "crowdb-console-shared", + "crowdb-diskio-client", + "crowdb-monitor", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", + "crowdb-test-harness", "flatbuffers", "futures", + "if-addrs 0.13.4", + "mdns-sd", + "rand 0.8.6", + "reqwest", + "rustix", + "serde", + "serde_json", + "sha2", + "ssh-key", "thiserror 2.0.18", "tokio", + "toml", "tracing", + "uuid", ] [[package]] -name = "crowdb-kv" +name = "crowdb-paxos-kv" version = "0.3.0" dependencies = [ "arc-swap", @@ -1113,8 +1148,8 @@ dependencies = [ "criterion", "crossbeam-skiplist", "crowdb-common", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -1138,7 +1173,7 @@ dependencies = [ ] [[package]] -name = "crowdb-kv-client" +name = "crowdb-paxos-kv-client" version = "0.3.0" dependencies = [ "arc-swap", @@ -1146,8 +1181,8 @@ dependencies = [ "bincode", "bytes", "crowdb-common", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "dashmap", @@ -1163,7 +1198,7 @@ dependencies = [ ] [[package]] -name = "crowdb-kv-server" +name = "crowdb-paxos-kv-server" version = "0.3.0" dependencies = [ "arc-swap", @@ -1171,8 +1206,8 @@ dependencies = [ "bytes", "clap", "crowdb-common", - "crowdb-kv", - "crowdb-kv-client", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -1188,39 +1223,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "crowdb-monitor" -version = "0.3.0" -dependencies = [ - "axum", - "clap", - "crowdb-chunk-kv-client", - "crowdb-console-shared", - "crowdb-diskio-client", - "crowdb-kv", - "crowdb-kv-client", - "crowdb-monitor", - "crowdb-protocol", - "crowdb-rpc-ffi", - "crowdb-test-harness", - "flatbuffers", - "futures", - "if-addrs 0.13.4", - "mdns-sd", - "rand 0.8.6", - "reqwest", - "rustix", - "serde", - "serde_json", - "sha2", - "ssh-key", - "thiserror 2.0.18", - "tokio", - "toml", - "tracing", - "uuid", -] - [[package]] name = "crowdb-protocol" version = "0.3.0" @@ -1291,7 +1293,7 @@ dependencies = [ "crowdb-chunk-client", "crowdb-chunk-kv-client", "crowdb-diskio-client", - "crowdb-kv-client", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-sdk-native", @@ -1336,6 +1338,7 @@ dependencies = [ "crowdb-rpc-ffi", "crowdb-test-harness", "crowdb-tree-ffi", + "serde", "tokio", ] @@ -1355,9 +1358,9 @@ dependencies = [ "crowdb-common", "crowdb-console-shared", "crowdb-diskdb-client", - "crowdb-kv", - "crowdb-kv-client", "crowdb-monitor", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -1572,9 +1575,9 @@ dependencies = [ "crowdb-console-shared", "crowdb-diskdb-client", "crowdb-diskio-client", - "crowdb-kv", - "crowdb-kv-client", "crowdb-monitor", + "crowdb-paxos-kv", + "crowdb-paxos-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-sdk-native", diff --git a/Cargo.toml b/Cargo.toml index 737475838..ec5caf1cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,10 +2,10 @@ resolver = "2" members = [ "lib/crowdb-common/rust", - "lib/crowdb-kv", - "lib/crowdb-kv-client", + "lib/crowdb-paxos-kv", + "lib/crowdb-paxos-kv-client", "lib/crowdb-protocol", - "app/crowdb-kv-server", + "app/crowdb-paxos-kv-server", "app/crowdb-diskdb", "lib/crowdb-diskdb-client", "app/crowdb-chunkdb", @@ -33,9 +33,9 @@ members = [ "lib/crowdb-sdk-native", "lib/crowdb-sdk-native-python", ] -exclude = ["third-party/hyper", "sdk"] +exclude = ["third-party/hyper", "third-party/bytes", "sdk"] # : crowdb-tree/ffi moved from `exclude` into `members` now that -# crowdb-kv depends on it (CrowdbTreeEngine) -- building crowdb-kv already triggers +# crowdb-paxos-kv depends on it (CrowdbTreeEngine) -- building crowdb-paxos-kv already triggers # the C++ build step via crowdb-tree-ffi's build.rs regardless of workspace # membership, so the original "keep the C build out of the main workspace" # rationale for excluding it no longer holds; membership instead unifies @@ -66,6 +66,7 @@ pin-project-lite = "0.2" hyper = { git = "https://github.com/buzzcrow/hyper.git", branch = "feature-crowdb", version = "1.11.1", default-features = false } [patch.crates-io] +bytes = { path = "third-party/bytes" } hyper = { git = "https://github.com/buzzcrow/hyper.git", branch = "feature-crowdb" } [workspace.lints.rust] diff --git a/README.md b/README.md index 0170ff4c6..c1730bc13 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,6 @@ documents. Start with: - [Contributing](CONTRIBUTING.md) for the development environment and workflow. - [Documentation index](doc/doc_index.md) for subsystem designs. -- [Backlog](doc/backlog/backlog.md) for current delivery scope. - [Single-node container](container/single-node-container/README.md) for image development and packaging. diff --git a/app/crowdb-access-server/Cargo.toml b/app/crowdb-access-server/Cargo.toml index b36a54e3c..c8a3c9124 100644 --- a/app/crowdb-access-server/Cargo.toml +++ b/app/crowdb-access-server/Cargo.toml @@ -30,7 +30,7 @@ crowdb-access-s3 = { path = "../../lib/crowdb-access-s3" } crowdb-chunk-client = { path = "../../lib/crowdb-chunk-client" } crowdb-chunk-kv-client = { path = "../../lib/crowdb-chunk-kv-client" } crowdb-common = { path = "../../lib/crowdb-common/rust" } -crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../../lib/crowdb-paxos-kv-client" } crowdb-protocol = { path = "../../lib/crowdb-protocol" } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi" } futures = "0.3" @@ -56,7 +56,7 @@ sync_wrapper = { version = "1", features = ["futures"] } [dev-dependencies] crowdb-access-dataset = { path = "../../lib/crowdb-access-dataset", features = ["test-util"] } -crowdb-kv = { path = "../../lib/crowdb-kv" } +crowdb-paxos-kv = { path = "../../lib/crowdb-paxos-kv" } bytes = "1" tempfile = "3" bincode = "1" diff --git a/app/crowdb-access-server/conf/crowdb_access_server_config.toml b/app/crowdb-access-server/conf/crowdb_access_server_config.toml index 9a89cdd34..dcdfd81ad 100644 --- a/app/crowdb-access-server/conf/crowdb_access_server_config.toml +++ b/app/crowdb-access-server/conf/crowdb_access_server_config.toml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. # Canonical configuration for the S3 and Iceberg access processes. # Credentials and bearer tokens are supplied through environment variables. @@ -6,7 +7,7 @@ mode = "production" max_node_failures = 1 [common] -management_seeds = ["http://127.0.0.1:10000"] +management_seeds = ["http://127.0.0.1:10135"] diskio_connections_per_endpoint = 2 diskio_rpc_workers = 2 @@ -31,7 +32,7 @@ chunk_capacity_bytes = 268435456 small_strip_prefetch_count = 32 [s3] -listen = "127.0.0.1:9091" +listen = "127.0.0.1:7135" tenant = "preview" region = "us-east-1" trusted_network = false @@ -46,7 +47,7 @@ ec_code = 4 max_chunk_size = 1073741824 [iceberg] -listen = "127.0.0.1:9092" +listen = "127.0.0.1:8135" native_budget_bytes = 268435456 max_chunk_size = 1073741824 diff --git a/app/crowdb-access-server/conf/crowdb_access_server_config.toml.ports.in b/app/crowdb-access-server/conf/crowdb_access_server_config.toml.ports.in new file mode 100644 index 000000000..b6d7b48e9 --- /dev/null +++ b/app/crowdb-access-server/conf/crowdb_access_server_config.toml.ports.in @@ -0,0 +1,69 @@ +# Canonical configuration for the S3 and Iceberg access processes. +# Credentials and bearer tokens are supplied through environment variables. + +[deployment] +mode = "production" +max_node_failures = 1 + +[common] +management_seeds = ["http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@"] +diskio_connections_per_endpoint = 2 +diskio_rpc_workers = 2 + +[read] +stream_window_bytes = 1048576 +stream_slots = 3 +global_stream_bytes = 268435456 +recovery_memory_bytes = 268435456 + +[small_write] +threshold_ratio = 0.9 +disk_block_bytes = 1048576 +conversion_enabled = true +ec_data = 8 +ec_code = 4 +memory_budget_bytes = 1342177280 +queue_capacity = 1024 +min_pipelines = 1 +max_pipelines = 32 +max_batch_bytes = 1048576 +chunk_capacity_bytes = 268435456 +small_strip_prefetch_count = 32 + +[s3] +listen = "127.0.0.1:@ACCESS_SERVER_HTTP_BASE@" +tenant = "preview" +region = "us-east-1" +trusted_network = false +small_object_limit = 8388608 +list_scan_items = 1001 +list_scan_bytes = 4194304 +continuation_ttl_seconds = 900 +native_budget_bytes = 268435456 +cleanup_backlog_limit = 10000 +ec_data = 8 +ec_code = 4 +max_chunk_size = 1073741824 + +[iceberg] +listen = "127.0.0.1:@ACCESS_SERVER_ICEBERG_HTTP_BASE@" +native_budget_bytes = 268435456 +max_chunk_size = 1073741824 + +[iceberg.gc] +enabled = false +interval_ms = 1800000 +step_bytes = 8388608 +step_ms = 1000 +page_items = 64 +page_bytes = 49152 +concurrency = 1 +kv_bytes = 67108864 +kv_requests = 128 +chunk_bytes = 8388608 +chunk_requests = 128 +retry_base_ms = 1000 +retry_max_ms = 60000 +corruption_attempts = 3 +minimum_retention_ms = 604800000 +catalogs = [] diff --git a/app/crowdb-access-server/src/credentials.rs b/app/crowdb-access-server/src/credentials.rs index 4e42c014a..14e41e875 100644 --- a/app/crowdb-access-server/src/credentials.rs +++ b/app/crowdb-access-server/src/credentials.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use crowdb_access_s3::auth::{ Credential, CredentialCipher, DurableCredentialRecord, IssuedUserToken, SecretError, }; -use crowdb_kv_client::{CrowdbKvClient, Error as KvError, ReadMode}; +use crowdb_paxos_kv_client::{Error as KvError, PaxosKvClient, ReadMode}; mod ensure; @@ -16,7 +16,7 @@ const CREDENTIAL_PREFIX: &[u8] = b"crowdb/s3/credential/"; const CREATE_ATTEMPTS: usize = 8; pub struct CredentialAuthority { - control: Arc<CrowdbKvClient>, + control: Arc<PaxosKvClient>, cipher: Arc<CredentialCipher>, } @@ -40,7 +40,7 @@ pub enum CredentialAuthorityError { impl CredentialAuthority { #[must_use] - pub fn new(control: Arc<CrowdbKvClient>, cipher: Arc<CredentialCipher>) -> Self { + pub fn new(control: Arc<PaxosKvClient>, cipher: Arc<CredentialCipher>) -> Self { Self { control, cipher } } diff --git a/app/crowdb-access-server/src/credentials/ensure.rs b/app/crowdb-access-server/src/credentials/ensure.rs index 6a0693e7d..40763279e 100644 --- a/app/crowdb-access-server/src/credentials/ensure.rs +++ b/app/crowdb-access-server/src/credentials/ensure.rs @@ -5,7 +5,7 @@ use super::{credential_key, CredentialAuthority, CredentialAuthorityError}; use crowdb_access_s3::auth::{DurableCredentialRecord, IssuedUserToken}; -use crowdb_kv_client::{Error as KvError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{Error as KvError, GetOutcome, ReadMode}; use sha2::{Digest, Sha256}; impl CredentialAuthority { diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index 1e11e221a..c70f40cf4 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -18,7 +18,7 @@ use crowdb_access_dataset::{ use crowdb_chunk_kv_client::{ ChunkKvClient, ChunkKvRpcTransport, ClientConfig as ChunkKvConfig, Group0ChunkKvRangeCatalogSource, }; -use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig as KvConfig, PaxosKvClient}; use http_body_util::{BodyExt, Full}; use hyper::body::{Bytes, Incoming}; use hyper::server::conn::http1; @@ -151,7 +151,7 @@ pub async fn run_with_shutdown( .listen .clone() .or_else(|| std::env::var("CROWDB_DATASET_LISTEN").ok()) - .unwrap_or_else(|| "127.0.0.1:9093".into()); + .unwrap_or_else(|| format!("127.0.0.1:{}", crowdb_protocol::ACCESS_SERVER_DATASET_HTTP_BASE)); let _: std::net::SocketAddr = address.parse()?; let seeds = if access.common.management_seeds.is_empty() { std::env::var("CROWDB_MANAGEMENT_SEEDS")? @@ -166,7 +166,7 @@ pub async fn run_with_shutdown( if seeds.is_empty() { return Err("one or more management seeds are required".into()); } - let control = Arc::new(CrowdbKvClient::new(KvConfig::new(seeds.clone()))); + let control = Arc::new(PaxosKvClient::new(KvConfig::new(seeds.clone()))); let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&control))); let chunk_config = ChunkKvConfig::default(); let transport = Arc::new(ChunkKvRpcTransport::with_config( @@ -240,7 +240,7 @@ pub fn start_dataset_cleanup_for_tests( async fn connect_payloads( seeds: Vec<String>, - control: Arc<CrowdbKvClient>, + control: Arc<PaxosKvClient>, capacity: usize, mut small_write: crowdb_chunk_client::SmallWritePolicy, ) -> Result<crowdb_chunk_client::ChunkIoClient, BoxError> { diff --git a/app/crowdb-access-server/src/iceberg/runtime.rs b/app/crowdb-access-server/src/iceberg/runtime.rs index 65acdceaa..7410a76cf 100644 --- a/app/crowdb-access-server/src/iceberg/runtime.rs +++ b/app/crowdb-access-server/src/iceberg/runtime.rs @@ -66,7 +66,7 @@ impl IcebergRuntimeConfig { .listen .clone() .or_else(|| std::env::var("CROWDB_ICEBERG_LISTEN").ok()) - .unwrap_or_else(|| "127.0.0.1:9092".into()); + .unwrap_or_else(|| format!("127.0.0.1:{}", crowdb_protocol::ACCESS_SERVER_ICEBERG_HTTP_BASE)); let _: std::net::SocketAddr = listen.parse()?; Ok(Self { listen, diff --git a/app/crowdb-access-server/src/main.rs b/app/crowdb-access-server/src/main.rs index 82504ab04..39667c7af 100644 --- a/app/crowdb-access-server/src/main.rs +++ b/app/crowdb-access-server/src/main.rs @@ -32,7 +32,7 @@ use crowdb_access_server::storage::S3StorageClients; #[cfg(feature = "s3")] use crowdb_chunk_client::LargeWritePolicy; #[cfg(feature = "s3")] -use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig as KvConfig, PaxosKvClient}; #[cfg(feature = "s3")] use tokio::net::TcpListener; #[cfg(feature = "s3")] @@ -271,7 +271,7 @@ async fn bind_health_listener(access_config: &AccessConfig) -> std::io::Result<T .listen .clone() .or_else(|| std::env::var("CROWDB_ACCESS_HEALTH_LISTEN").ok()) - .unwrap_or_else(|| "127.0.0.1:9094".into()); + .unwrap_or_else(|| format!("127.0.0.1:{}", crowdb_protocol::ACCESS_SERVER_HEALTH_HTTP_BASE)); TcpListener::bind(address).await } @@ -435,7 +435,7 @@ async fn issue_user(args: &[String], access: &AccessConfig) -> Result<(), Box<dy } else { access.common.management_seeds.clone() }; - let control = Arc::new(CrowdbKvClient::new(KvConfig::new(seeds))); + let control = Arc::new(PaxosKvClient::new(KvConfig::new(seeds))); let authority = CredentialAuthority::new(control, cipher); let token = match command.as_str() { "ensure-user" => authority.ensure_user(user.as_bytes()).await?, diff --git a/app/crowdb-access-server/tests/README.md b/app/crowdb-access-server/tests/README.md index c240a481f..b4ef82773 100644 --- a/app/crowdb-access-server/tests/README.md +++ b/app/crowdb-access-server/tests/README.md @@ -13,9 +13,7 @@ Client recipes: [container development](../../../container/single-node-container only actual pinned client operations against the container and a volume restart. - [S3 language SDKs](../../../e2e-test/clients/S3-SDK.md) and the Rust Iceberg SDK retain independent manual workflows. They are not default test dependencies. -- Selected ORC and optional caches remain independent backlog ownership under - [ORC validation](../../../doc/backlog/R186-access-iceberg-orc-validation.md) - and [cache optimization](../../../doc/backlog/R185-access-iceberg-cache-invalidation.md). +- Selected ORC validation and optional cache optimization remain deferred. The functional fixtures select Parquet and establish no cache speed guarantee. - Physical GC remains an independent opt-in runtime contract. Logical drop, clear, expiry and CAS loss do not authorize physical deletion by TTL alone. diff --git a/app/crowdb-access-server/tests/common/access_probe_budget.rs b/app/crowdb-access-server/tests/common/access_probe_budget.rs index 5bacf305b..1c232acfd 100644 --- a/app/crowdb-access-server/tests/common/access_probe_budget.rs +++ b/app/crowdb-access-server/tests/common/access_probe_budget.rs @@ -13,7 +13,7 @@ use crowdb_access_server::s3::{ S3OperationsFuture, }; use crowdb_chunk_client::{metrics::ReadinessMetrics, ChunkIoClient, DiskWriter, Result}; -use crowdb_kv_client::DiskRecord; +use crowdb_paxos_kv_client::DiskRecord; use crowdb_protocol::{ common::{DiskId, HwStatus}, diskdb::rpc::{DiskValue, Segment}, diff --git a/app/crowdb-access-server/tests/common/credentials.rs b/app/crowdb-access-server/tests/common/credentials.rs index 74b0fbe78..2655bdc00 100644 --- a/app/crowdb-access-server/tests/common/credentials.rs +++ b/app/crowdb-access-server/tests/common/credentials.rs @@ -3,15 +3,15 @@ use crowdb_access_s3::auth::{CredentialCipher, MasterKey}; use crowdb_access_server::credentials::CredentialAuthority; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use std::sync::Arc; pub struct TestCredentials { store: Arc<PxKvStore>, - pub control: Arc<CrowdbKvClient>, + pub control: Arc<PaxosKvClient>, cipher: Arc<CredentialCipher>, } @@ -23,7 +23,7 @@ impl TestCredentials { PxLocalReplica::new(1, PxLocalReplicaRole::Leader), )); store.start().await.unwrap(); - let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + let client = PaxosKvClient::new(ClientConfig::new(vec![])); client.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); let key = MasterKey::from_hex(&"11".repeat(32)).unwrap(); Self { diff --git a/app/crowdb-access-server/tests/credential_authority_test.rs b/app/crowdb-access-server/tests/credential_authority_test.rs index 192a15aaa..fd4e1e275 100644 --- a/app/crowdb-access-server/tests/credential_authority_test.rs +++ b/app/crowdb-access-server/tests/credential_authority_test.rs @@ -5,7 +5,7 @@ mod credentials; use credentials::TestCredentials; use crowdb_access_server::credentials::CredentialAuthorityError; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; const USER: &[u8] = b"console-root"; const PREPARED: &[u8] = b"crowdb/s3/user-bootstrap/"; diff --git a/app/crowdb-chunk-kv-server/Cargo.toml b/app/crowdb-chunk-kv-server/Cargo.toml index 5d23d95c9..f442813af 100644 --- a/app/crowdb-chunk-kv-server/Cargo.toml +++ b/app/crowdb-chunk-kv-server/Cargo.toml @@ -22,7 +22,7 @@ crowdb-chunk-kv = { path = "../../lib/crowdb-chunk-kv" } crowdb-chunk-client = { path = "../../lib/crowdb-chunk-client" } crowdb-chunk-stream = { path = "../../lib/crowdb-chunk-stream" } crowdb-common = { workspace = true } -crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../../lib/crowdb-paxos-kv-client" } crowdb-protocol = { path = "../../lib/crowdb-protocol" } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi" } crowdb-tree-ffi = { path = "../../lib/crowdb-tree/ffi", features = ["chunk-rpc"] } @@ -35,10 +35,11 @@ toml = "0.8" tracing = { workspace = true } [dev-dependencies] +crowdb-tree-ffi = { path = "../../lib/crowdb-tree/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } crowdb-chunk-kv-client = { path = "../../lib/crowdb-chunk-kv-client" } sha2 = "0.10" -crowdb-kv = { path = "../../lib/crowdb-kv", features = ["test-util"] } +crowdb-paxos-kv = { path = "../../lib/crowdb-paxos-kv", features = ["test-util"] } crowdb-chunk-kv-server = { path = ".", features = ["test-util"] } crowdb-chunk-kv = { path = "../../lib/crowdb-chunk-kv", features = ["test-util"] } crowdb-chunk-stream = { path = "../../lib/crowdb-chunk-stream", features = ["test-util"] } diff --git a/app/crowdb-chunk-kv-server/conf/crowdb_chunk_kv_server_config.toml b/app/crowdb-chunk-kv-server/conf/crowdb_chunk_kv_server_config.toml new file mode 100644 index 000000000..5ce798419 --- /dev/null +++ b/app/crowdb-chunk-kv-server/conf/crowdb_chunk_kv_server_config.toml @@ -0,0 +1,292 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# +# Chunk KV Server +# crowdb-chunk-kv-server --config <this-file> +# Read at startup. Sizes are integer bytes; 1 MiB = 1,048,576 bytes. +# Find maximum sustainable flush throughput with ample resources first, then +# reduce resources using measurements. Queues retain work; they do not add throughput. + +# Unique process identity. Set node_id when registering through node management. +instance_id = 1 +# node_id = 1 +# Bind addresses may be unspecified; advertised addresses must be routable. +rpc_listen_addr = "0.0.0.0:21135" +rpc_advertise_addr = "127.0.0.1:21135" +http_listen_addr = "0.0.0.0:20135" +# Omit to leave the optional management endpoint out of instance observations. +# http_advertise_addr = "127.0.0.1:20135" +# Paxos KV Group-0 management endpoints used to discover storage and assignments. +group0_mgmt_seeds = ["http://127.0.0.1:10135"] +# RPC I/O workers are separate from Tree CPU/backend workers. +rpc_workers = 2 +# Partitions share one process-wide Tree runtime. Per-partition caches/budgets +# still contribute to total memory use; this cap is not a process RSS limit. +max_hosted_partitions = 256 +# Fallback catalog/grant reconciliation interval; watch events trigger earlier refreshes. +catalog_refresh_interval_ms = 5_000 +# Remaining WAL records allowed before split preparation enters its final catch-up. +max_split_catchup_lag_records = 1_024 +# Deadline for draining admitted work and checkpointing during shutdown. +shutdown_drain_timeout_ms = 30_000 + +[storage] +# Paxos KV store containing the range catalog and durable root metadata. +metadata_store_id = 1 +# Stream ownership lease in milliseconds. +stream_writer_lease_ms = 30_000 +# Replica count for both WAL streams and Tree chunks (1..5). +stream_mirror_copies = 2 +# Tree and WAL allocation capacities, each in 1..256 MiB. +tree_chunk_capacity_bytes = 268_435_456 +stream_chunk_capacity_bytes = 268_435_456 +# 0 selects the route default; otherwise use a whole number of MiB in 1..256 MiB. +tree_strip_capacity_bytes = 0 +# Host-wide live pin and pending final-release slots for all Tree writers/readers. +tree_pin_release_slots = 1_048_576 +# DiskIO connections per endpoint and I/O workers, independent of the Tree pools. +diskio_connections_per_endpoint = 1 +diskio_rpc_workers = 2 + +[partition] +# Per-partition ingress bounds, including admitted requests still being processed. +# Larger queues absorb bursts and retain memory without increasing service rate. +queue_requests = 1_024 +queue_bytes = 67_108_864 +# Maximum journal batch count and target bytes. One admitted request may exceed +# batch_bytes and is then handled alone; apply remains ordered within each partition. +batch_requests = 64 +batch_bytes = 4_194_304 +# API key/value limits; native Tree key and frame limits also apply. +max_key_bytes = 65_536 +max_value_bytes = 16_777_216 +# Recent deduplication results kept for retry handling; older requests can expire. +retained_results = 65_536 +# Bound metadata reclamation work per maintenance pass. +metadata_reclaim_records_per_pass = 128 + +[monitor] +# Cluster-wide monitor identity and protocol versions. All owners in this domain +# must register a compatible descriptor; coordinate policy changes across owners. +domain = "chunk-kv" +service_registry_name = "chunk-kv" +driver_version = 1 +capability_version = 1 +# Heartbeat <= suspect <= dead. A dead owner must be fenced before lease expiry. +heartbeat_interval_ms = 2_000 +suspect_after_ms = 6_000 +dead_after_ms = 10_000 +lease_duration_ms = 12_000 +# Lease > clock skew + self-fence margin, and dead + margin <= lease. +max_clock_skew_ms = 1_000 +self_fence_margin_ms = 1_000 +# AutomaticSharedStorage permits automatic recovery; OperatorOnly requires intervention. +failure_policy = "AutomaticSharedStorage" +balance_policy = "count-first-v1" +# chunk_kv_range_balance is derived from [balance] on load; do not duplicate it here. + +[balance] +# Disable automatic owner moves; local partition splitting remains available. +enabled = true +# Reserved: must remain 0 until reliable per-range live-byte measurements are available. +byte_weight_percent = 0 +# Owner-count imbalance tolerated before proposing a move (0..100). +imbalance_tolerance_percent = 20 +# Desired minimum partition count per live owner and local split size threshold. +target_partitions_per_owner = 4 +target_partition_bytes = 1_073_741_824 +# Required weighted improvement for moves (0..100) and minimum time between moves. +minimum_weighted_improvement_percent = 25 +cooldown_ms = 60_000 +# 0 disables the request-rate ceiling for candidate target owners. +max_owner_request_rate = 0 + +# Optional first-partition bootstrap. Omit for ordinary catalog-driven startup. +# Use persistent, nonzero identities for an intentional empty-cluster bootstrap. +# [bootstrap_partition] +# partition_id = { high = 1, low = 1 } +# tree_id = 1 +# stream_name = { high = 1, low = 2 } +# owner_epoch = 1 +# metadata_group_id = 1 + +# Optional explicit RPC RDMA configuration. Omit to retain transport environment +# defaults. This complete example selects the 256 KiB control/data class. +# Uncomment the table and its fields together, then set device/source for this host. +# [rpc_rdma] +# enabled = true +# allow_tcp_fallback = true +# listen_enabled = true +# port = 1 +# device = "" +# source_address = "" +# gid = "" +# CQ/QP depths and posted receives; target <= rq_depth and SQ + target <= cq_depth. +# cq_count = 1 +# cq_depth = 8_192 +# sq_depth = 1_024 +# rq_depth = 1_024 +# posted_receive_target = 1_024 +# wire_layout = "control_data" +# wire_slot_capacity = 263_168 +# Send/receive batching and outstanding sends. A zero send window uses SQ depth; +# zero send_window_bytes disables the additional byte ceiling. +# send_batch_size = 16 +# receive_batch_size = 1 +# send_window_size = 0 +# send_window_bytes = 0 +# Completion polling limits and the power-of-two pending send queue capacity. +# poll_batch = 256 +# verbs_poll_batch = 32 +# poll_empty_budget = 256 +# send_queue_capacity = 4_096 +# Optional small-message aggregation (bounded by the transport frame capacity). +# aggregation_enabled = false +# aggregation_max_bytes = 65_536 +# aggregation_max_frames = 256 +# Registered send/receive arenas; size them together with the SQ/RQ windows. +# A zero registration budget derives the budget from the transport geometry. +# send_arena_bytes = 268_435_456 +# receive_arena_bytes = 269_484_032 +# registered_memory_budget_bytes = 0 +# Large control-message assembly bounds, timeout, and aggregate byte budgets. +# max_control_message_bytes = 33_554_432 +# control_assembly_timeout_ms = 30_000 +# control_send_budget_bytes = 67_108_864 +# control_receive_budget_bytes = 67_108_864 + +# Keep Tree field additions, removals, renames and explanations in sync with: +# app/crowdb-paxos-kv-server/conf/crowdb_paxos_kv_server_config.toml +# Each server owns its values independently; no shared include or override is loaded. + +# Process-wide CPU/backend pools. Check CPU quotas and utilization when scaling. +# Resource diagnostics sample every 100 ms and flush at the server metrics-log +# interval. Each host/Tree resource reports its limit, current usage, average +# held sampled usage, and sampled maximum for that log window. The maximum is +# not an exact transient peak; sample count, age, gaps and errors show coverage. +# Averages retain fractional counts: worker avg=0.750 means 75% of one worker. +# Reserved capacity and allocated storage are reported separately. Budget +# denial counters distinguish Tree limits from shared host limits. Queue-slot +# usage includes prepaid progress registrations even when no job is runnable. +# Tree cache rows cover decoded bytes, page slots and request slots; they +# overlap the storage charged to budget partitions and must not be added to it. +[tree_runtime.host] +workers = 4 +# Workers for synchronous PageStore calls; async concurrency also depends on the backend. +backend_workers = 2 +# Larger queues retain more work without increasing service rate. Recheck +# lifecycle/Routing reservations when increasing queue capacity. +cpu_queue_limit = 256 +backend_queue_limit = 256 +# Maximum pending waiters per resource partition. +max_waiters_per_class = 256 +# Retained queue/control reservation charged to host Routing, not an extra partition. +lifecycle_bytes = 4_194_304 + +# Shared limits for all Trees in this process. Resource classes cannot borrow +# from each other. These limits do not bound total process RSS. +[tree_runtime.host.limits] +# Source capture, run/range construction and complete-batch handoff. +handoff_bytes = 268_435_456 +# Routing, publication, cache/load and controls, including lifecycle reservations. +routing_bytes = 536_870_912 +# Consolidation, split/merge and checkpoint/writeback work. +reclaim_bytes = 134_217_728 +# Ordinary writes, payload owners and retained COW allocations. +writer_bytes = 536_870_912 +# Independent progress capacity for chosen/replay/gap apply work. +gap_repair_bytes = 67_108_864 +# In-flight I/O tickets. More tickets alone do not parallelize a serial write pipeline. +io_operations = 128 + +# Per-Tree limits. All Trees still compete for the shared host resources above. +[tree_runtime.tree] +# The producer window must fit this CPU queue. Local queues must fit host queues. +# This also bounds ordinary apply/backend operation owners; replay has two spare slots. +cpu_queue_limit = 64 +backend_queue_limit = 64 +# Fixed queue/core/cache descriptors, charged to this Tree's Routing partition. +# Increase this together with cpu_queue_limit and tuning.cache_max_pages; a +# larger Routing limit alone does not enlarge this retained control reserve. +lifecycle_bytes = 1_048_576 + +[tree_runtime.tree.limits] +# Same categories as the host limits. Check both levels when changing a partition. +handoff_bytes = 67_108_864 +routing_bytes = 268_435_456 +# 64 MiB is an adjustable default, not a fixed checkpoint memory target. +reclaim_bytes = 67_108_864 +writer_bytes = 134_217_728 +gap_repair_bytes = 16_777_216 +io_operations = 16 + +[tree] +# Durable frame size; the current public format supports at most 64 KiB. +# A logical leaf may exceed one frame. +# Must be at least 512 bytes and satisfy backend IU alignment. Logical working +# set size is independent of durable frame size. +frame_bytes = 65_536 +# Decoded-cache byte watermark; tuning.cache_max_pages also limits page count. +buffer_pool_bytes = 67_108_864 +# 0 = frame_bytes/4. Values larger than this spill to overflow; exactly 16 KiB +# remains inline with the default frame size. +max_inline_value = 0 +# LZ4 trades CPU for storage bandwidth/space. Keep compression fixed in comparisons. +compression_lz4 = false +# Local block allocation/reclamation unit; ignored by file/mem-block and injected Chunk storage. +block_size = 67_108_864 +# 0 = backend default: block 4 KiB, mem-block 1 byte, Chunk 64 KiB. A nonzero value +# must be a power of two that divides frame_bytes. +iu_size = 0 + +# Engine work granularity. Select these values from measurements. +[tree.tuning] +# Soft per-source rotation thresholds. Crossing either requests rotation; +# 0 requests rotation after every nonempty write. +# The source_max_* hard admission capacities below also apply. +memtable_flush_bytes = 16_777_216 +memtable_flush_entries = 100_000 +# Includes the writable source; the current read/handoff window requires 2..64. +max_memtable_count = 10 +# Per-source admitted version/key capacities, including writes not yet applied. +# Raising soft thresholds alone may not enlarge batches if these limits bind first. +source_max_entries = 4_096 +source_max_key_bytes = 1_048_576 +# Additional accepted/replay allowance. Size it for accepted apply bursts. +# Only the next contiguous slot may use gap source capacity; higher holes use ordinary capacity or wait. +source_max_gap_entries = 64 +source_max_gap_key_bytes = 131_072 +# 0 = min(host workers, Tree io_operations). Explicit values must fit both limits. +flush_parallelism = 0 +# Per-handoff-run entry/key/work-byte limits. Value owners are retained independently. +run_max_entries = 128 +# Effective key capacity is max(run_max_key_bytes, max_key_size). +run_max_key_bytes = 262_144 +run_max_bytes = 1_048_576 +# Key-arena allocation granularity; must not exceed run_max_key_bytes. +run_key_block_bytes = 1_024 +# Initial run-install window; later windows double. Balances early visible +# progress against prefix-array copying cost. +install_run_window = 64 +# Visits per step/pass; the step size must not exceed the pass limit. +maintenance_visits_per_step = 64 +maintenance_max_visits = 4_096 +# Candidates/steps per pass. Larger limits increase work and reservations; +# they do not repair starvation caused by an incorrect scan cursor. +maintenance_max_candidates = 256 +maintenance_max_steps = 1_024 +# Local scratch per maintenance operation; independent of the total Reclaim budget. +maintenance_metadata_bytes = 1_048_576 +# Decoded-cache page limit. Check this together with buffer_pool_bytes. +cache_max_pages = 256 +# Split/merge thresholds for the physical primary record region, not logical leaf limits. +# Split is capped by frame/format capacity. Merge=0 disables merging; merged +# output must also fit the hysteresis target of about 75% of the split threshold. +leaf_split_bytes = 65_502 +leaf_merge_bytes = 16_384 +# Inner separator limit; encoded output must also fit the frame byte capacity. +inner_max_keys = 256 +# 0 = max(1, inner_max_keys/4). +inner_merge_keys = 0 +# Maximum key size; 0 = frame_bytes/2. Oversized keys are rejected before admission. +max_key_size = 0 diff --git a/app/crowdb-chunk-kv-server/src/config.rs b/app/crowdb-chunk-kv-server/src/config.rs index ad44e2e38..b1839f4e7 100644 --- a/app/crowdb-chunk-kv-server/src/config.rs +++ b/app/crowdb-chunk-kv-server/src/config.rs @@ -8,7 +8,7 @@ use crowdb_protocol::chunk_kv::{ ChunkKvRangeBalancePolicy, DomainFailurePolicy, DomainMonitorDescriptor, Id128, }; use crowdb_protocol::chunk_stream::StreamName; -use crowdb_protocol::{CHUNK_KV_HTTP_BASE, CHUNK_KV_RPC_BASE, KV_SERVER_MGMT_BASE}; +use crowdb_protocol::{CHUNK_KV_HTTP_BASE, CHUNK_KV_RPC_BASE, PAXOS_KV_SERVER_MGMT_BASE}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -41,6 +41,9 @@ pub struct ChunkKvServerConfig { pub rpc_workers: u32, pub rpc_rdma: Option<crowdb_rpc_ffi::RdmaConfig>, pub storage: StorageConfig, + pub partition: crowdb_chunk_kv::PartitionConfig, + pub tree_runtime: crowdb_tree_ffi::TreeRuntimeConfig, + pub tree: crowdb_tree_ffi::TreeEngineConfig, pub bootstrap_partition: Option<BootstrapPartitionConfig>, pub monitor: DomainMonitorDescriptor, pub balance: BalanceConfig, @@ -55,7 +58,7 @@ impl Default for ChunkKvServerConfig { rpc_advertise_addr: format!("127.0.0.1:{CHUNK_KV_RPC_BASE}"), http_listen_addr: format!("0.0.0.0:{CHUNK_KV_HTTP_BASE}"), http_advertise_addr: None, - group0_mgmt_seeds: vec![format!("http://127.0.0.1:{KV_SERVER_MGMT_BASE}")], + group0_mgmt_seeds: vec![format!("http://127.0.0.1:{PAXOS_KV_SERVER_MGMT_BASE}")], max_hosted_partitions: 256, catalog_refresh_interval_ms: 5_000, max_split_catchup_lag_records: 1_024, @@ -63,6 +66,9 @@ impl Default for ChunkKvServerConfig { rpc_workers: 2, rpc_rdma: None, storage: StorageConfig::default(), + partition: crowdb_chunk_kv::PartitionConfig::default(), + tree_runtime: crowdb_tree_ffi::TreeRuntimeConfig::default(), + tree: crowdb_tree_ffi::TreeEngineConfig::default(), bootstrap_partition: None, monitor: default_monitor(), balance: BalanceConfig::default(), @@ -71,6 +77,17 @@ impl Default for ChunkKvServerConfig { } impl ChunkKvServerConfig { + /// Build native engine and per-partition limits; the process runtime is attached by storage. + #[must_use] + pub fn native_tree_options(&self) -> crowdb_tree_ffi::Config { + let mut options = crowdb_tree_ffi::Config { + runtime_binding: self.tree_runtime.binding_options(), + ..Default::default() + }; + self.tree.apply_to(&mut options); + options + } + /// Loads a TOML configuration file and applies field defaults. /// /// # Errors @@ -131,7 +148,11 @@ impl ChunkKvServerConfig { "partition capacity and lifecycle intervals must be nonzero".into(), )); } + self.validate_tree_config()?; self.storage.validate()?; + self.partition + .validate() + .map_err(|error| ConfigError::Invalid(error.to_string()))?; if self .rpc_rdma .as_ref() @@ -164,6 +185,25 @@ impl ChunkKvServerConfig { } Ok(()) } + + fn validate_tree_config(&self) -> Result<(), ConfigError> { + self.tree_runtime.validate().map_err(ConfigError::Invalid)?; + self.tree + .validate(&self.tree_runtime) + .map_err(ConfigError::Invalid)?; + let iu = if self.tree.iu_size == 0 { + 64 * 1024 + } else { + self.tree.iu_size + }; + if self.tree.frame_bytes % iu != 0 { + return Err(ConfigError::Invalid( + "tree.frame_bytes must align to the Chunk backend IU (64 KiB when tree.iu_size is zero)" + .to_owned(), + )); + } + Ok(()) + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -199,6 +239,7 @@ pub struct StorageConfig { pub stream_mirror_copies: u32, pub tree_chunk_capacity_bytes: u64, pub tree_strip_capacity_bytes: u32, + pub tree_pin_release_slots: usize, pub stream_chunk_capacity_bytes: u64, pub diskio_connections_per_endpoint: usize, pub diskio_rpc_workers: u32, @@ -212,6 +253,7 @@ impl Default for StorageConfig { stream_mirror_copies: 2, tree_chunk_capacity_bytes: 256 * 1024 * 1024, tree_strip_capacity_bytes: 0, + tree_pin_release_slots: 1_048_576, stream_chunk_capacity_bytes: 256 * 1024 * 1024, diskio_connections_per_endpoint: 1, diskio_rpc_workers: 2, @@ -221,6 +263,11 @@ impl Default for StorageConfig { impl StorageConfig { fn validate(&self) -> Result<(), ConfigError> { + if self.tree_pin_release_slots == 0 || self.tree_pin_release_slots > usize::MAX / 2 { + return Err(ConfigError::Invalid( + "storage.tree_pin_release_slots must fit a nonzero native queue capacity".into(), + )); + } if self.stream_writer_lease_ms == 0 || self.stream_mirror_copies == 0 || self.stream_mirror_copies > 5 diff --git a/app/crowdb-chunk-kv-server/src/control_store.rs b/app/crowdb-chunk-kv-server/src/control_store.rs index 0bb88108d..c06c2f337 100644 --- a/app/crowdb-chunk-kv-server/src/control_store.rs +++ b/app/crowdb-chunk-kv-server/src/control_store.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_trait::async_trait; -use crowdb_kv_client::{CrowdbKvClient, Error as KvClientError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{Error as KvClientError, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, DomainMonitorDescriptor, EnsureDomainMonitorOutcome, ServingGrant, SplitTransition, TransferTransition, @@ -49,7 +49,7 @@ pub trait Group0Kv: Send + Sync { } #[async_trait] -impl Group0Kv for CrowdbKvClient { +impl Group0Kv for PaxosKvClient { async fn get(&self, key: &[u8]) -> Result<Option<VersionedValue>, Group0KvError> { match self .get(GROUP0_STORE, GROUP0_GROUP, key, ReadMode::Linearizable, None) @@ -125,7 +125,7 @@ impl Group0Kv for CrowdbKvClient { async fn put_cas(&self, key: &[u8], value: &[u8], expected_revision: u64) -> Result<(), Group0KvError> { const MAX_BUSY_ATTEMPTS: usize = 20; for attempt in 0..MAX_BUSY_ATTEMPTS { - match CrowdbKvClient::put_cas(self, GROUP0_STORE, GROUP0_GROUP, key, value, expected_revision) + match PaxosKvClient::put_cas(self, GROUP0_STORE, GROUP0_GROUP, key, value, expected_revision) .await { Ok(_) => return Ok(()), @@ -158,7 +158,7 @@ impl Group0ControlStore { } #[must_use] - pub fn from_client(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_client(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } diff --git a/app/crowdb-chunk-kv-server/src/host_lifecycle.rs b/app/crowdb-chunk-kv-server/src/host_lifecycle.rs new file mode 100644 index 000000000..b204e1e8d --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/host_lifecycle.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! One explicit host exit path also covers partially initialized services. + +use crowdb_chunk_kv_server::serving::process::ProcessLeaseSession; +use crowdb_chunk_kv_server::{ChunkKvRpcService, ChunkKvService, ChunkKvStorage}; +use crowdb_rpc_ffi::RpcServer; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use tokio::sync::Notify; +use tokio::task::JoinHandle; +use tracing::{error, warn}; + +pub(super) struct HostLifecycle { + pub storage: Arc<ChunkKvStorage>, + pub process: Option<Arc<ProcessLeaseSession>>, + pub service: Option<Arc<ChunkKvService>>, + pub rpc: Option<Arc<ChunkKvRpcService>>, + pub server: Option<Arc<RpcServer>>, + pub tasks: [Option<JoinHandle<()>>; 4], + pub stop: Arc<HostStop>, +} + +impl HostLifecycle { + pub fn new(storage: Arc<ChunkKvStorage>) -> Self { + Self { + storage, + process: None, + service: None, + rpc: None, + server: None, + tasks: std::array::from_fn(|_| None), + stop: Arc::default(), + } + } + + pub async fn close(&mut self) { + if let Some(service) = &self.service { + service.begin_drain(); + } + if let Some(server) = &self.server { + server.stop(); + } + self.stop.close(); + for task in &mut self.tasks { + if let Some(task) = task.take() { + let _ = task.await; + } + } + if let Some(service) = &self.service { + if let Err(error) = service.drain_writers().await { + warn!(%error, "WAL writer drain incomplete; restart must retain lease exclusion"); + } + } + if let Some(rpc) = &self.rpc { + rpc.close_and_drain().await; + } + if let Some(service) = &self.service { + if let Err(error) = service.release_hosted_trees().await { + error!(%error, "native Tree close failed during host shutdown"); + } + } + self.storage.drain_pin_releases().await; + if let Some(process) = &self.process { + if let Err(error) = process.withdraw_observation().await { + warn!(%error, "failed to unregister chunk KV instance"); + } + } + } +} + +#[derive(Default)] +pub(super) struct HostStop { + closed: AtomicBool, + wake: Notify, +} +impl HostStop { + fn close(&self) { + self.closed.store(true, Ordering::Release); + self.wake.notify_waiters(); + } + pub async fn wait(&self) { + loop { + let wake = self.wake.notified(); + tokio::pin!(wake); + wake.as_mut().enable(); + if self.closed.load(Ordering::Acquire) { + return; + } + wake.await; + } + } +} diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index 8320ab374..3211df7dd 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -14,7 +14,7 @@ use crowdb_chunk_kv_server::{ ChunkKvStorage, DomainMonitorRegistry, Group0ControlStore, ManagementState, StorageRuntimeError, TransitionExecutor, TransitionProcessor, }; -use crowdb_kv_client::{WatchNotifyClient, WatchSubscription}; +use crowdb_paxos_kv_client::{WatchNotifyClient, WatchSubscription}; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, EnsureDomainMonitorOutcome, EnsureDomainMonitorRequest, KeyRange, @@ -25,6 +25,8 @@ use crowdb_protocol::key::{ }; use tracing::{error, info, warn}; +mod host_lifecycle; + #[derive(Debug, Parser)] #[command( name = "crowdb-chunk-kv-server", @@ -146,7 +148,7 @@ async fn main() -> std::process::ExitCode { "crowdb-chunk-kv-server starting" ); - let mut metrics_runner = crowdb_chunk_kv_server::metrics::start_stream_perf_log( + let mut metrics_runner = crowdb_chunk_kv_server::metrics::create_perf_log( &log_dir, args.metrics_interval, args.log_max_file_mb, @@ -161,6 +163,33 @@ async fn main() -> std::process::ExitCode { return std::process::ExitCode::FAILURE; } }; + let mut host = host_lifecycle::HostLifecycle::new(storage); + let result = serve_host( + &mut host, + &config, + http_addr, + rpc_addr, + rpc_advertise_addr, + metrics_runner.as_mut(), + ) + .await; + host.close().await; + if let Some(runner) = &mut metrics_runner { + runner.stop().await; + } + result +} + +#[allow(clippy::too_many_lines)] +async fn serve_host( + host: &mut host_lifecycle::HostLifecycle, + config: &ChunkKvServerConfig, + http_addr: SocketAddr, + rpc_addr: SocketAddr, + rpc_advertise_addr: SocketAddr, + metrics_runner: Option<&mut crowdb_common::metrics::MetricsRunner>, +) -> std::process::ExitCode { + let storage = host.storage.clone(); let control_store = Arc::new(Group0ControlStore::from_client(Arc::clone(storage.kv()))); let monitor_registry = DomainMonitorRegistry::new( control_store.clone(), @@ -198,12 +227,9 @@ async fn main() -> std::process::ExitCode { return std::process::ExitCode::FAILURE; } }; - if let Err(error) = storage - .streams() - .registry() - .bind_owner_process(process_lease.authority()) - { - error!(%error, "failed to bind WAL stream process authority"); + host.process = Some(process_lease.clone()); + if let Err(error) = storage.bind_owner_process(process_lease.authority()) { + error!(%error, "failed to bind tree and WAL process authority"); return std::process::ExitCode::FAILURE; } let service = match ChunkKvService::new(process_lease.authority(), config.max_hosted_partitions) { @@ -214,6 +240,14 @@ async fn main() -> std::process::ExitCode { } }; + host.service = Some(service.clone()); + if let Some(runner) = metrics_runner { + runner.set_sampler( + crowdb_chunk_kv_server::metrics::TreeResources::new(storage.tree_runtime().clone(), &service) + .with_storage(&storage), + ); + runner.start(); + } let catalog = Arc::new(ChunkKvRangeCatalogPublisher::new(control_store.clone())); let recovery_storage = &storage; let recovery_service = &service; @@ -308,11 +342,13 @@ async fn main() -> std::process::ExitCode { } }; let refresh_interval = std::time::Duration::from_millis(config.catalog_refresh_interval_ms); + let stop = host.stop.clone(); let refresh_task = tokio::spawn(async move { let mut interval = tokio::time::interval(refresh_interval); interval.tick().await; loop { let grant_only = tokio::select! { + () = stop.wait() => break, _ = interval.tick() => false, alive = receive_transition_notify(&mut catalog_watch) => { if !alive { @@ -375,6 +411,7 @@ async fn main() -> std::process::ExitCode { } } }); + host.tasks[0] = Some(refresh_task); let transition_executor = match TransitionExecutor::new( config.instance_id, Arc::clone(&service), @@ -384,7 +421,6 @@ async fn main() -> std::process::ExitCode { Ok(executor) => Arc::new(executor), Err(error) => { error!(%error, "failed to initialize transition worker"); - refresh_task.abort(); return std::process::ExitCode::FAILURE; } }; @@ -409,10 +445,12 @@ async fn main() -> std::process::ExitCode { } }; let transition_interval = std::time::Duration::from_millis(config.catalog_refresh_interval_ms); + let stop = host.stop.clone(); let transition_task = tokio::spawn(async move { let mut interval = tokio::time::interval(transition_interval); loop { tokio::select! { + () = stop.wait() => break, _ = interval.tick() => {} alive = receive_transition_notify(&mut transfer_watch) => { if !alive { @@ -431,6 +469,7 @@ async fn main() -> std::process::ExitCode { } }); + host.tasks[1] = Some(transition_task); let capacity_bytes = u64::try_from(config.max_hosted_partitions) .unwrap_or(u64::MAX) .saturating_mul(config.balance.target_partition_bytes); @@ -445,13 +484,13 @@ async fn main() -> std::process::ExitCode { .await { error!(%error, "failed to register chunk KV instance"); - transition_task.abort(); - refresh_task.abort(); return std::process::ExitCode::FAILURE; } - install_latest_grant(&control_store, &service, &config).await; + install_latest_grant(&control_store, &service, config).await; let materialization_service = Arc::clone(&service); + let materialization_storage = Arc::clone(&storage); let materialization_interval = std::time::Duration::from_millis(config.catalog_refresh_interval_ms); + let stop = host.stop.clone(); let materialization_task = tokio::spawn(async move { let mut interval = tokio::time::interval(materialization_interval); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); @@ -460,6 +499,7 @@ async fn main() -> std::process::ExitCode { loop { tokio::select! { biased; + () = stop.wait() => break, _ = interval.tick() => { if let Err(error) = materialization_service.materialize_split_overlays().await { warn!(%error, "chunk KV split-overlay materialization pass failed"); @@ -474,13 +514,22 @@ async fn main() -> std::process::ExitCode { false } }; + match materialization_service + .resume_legacy_retirement(&materialization_storage) + .await + { + Ok(backlog) => pending |= backlog, + Err(error) => warn!(%error, "legacy Chunk retirement deferred; retaining durable claims"), + } } }); + host.tasks[2] = Some(materialization_task); let heartbeat_service = Arc::clone(&service); let heartbeat_lease = Arc::clone(&process_lease); let heartbeat_store = Arc::clone(&control_store); let heartbeat_config = config.clone(); let heartbeat_endpoint = rpc_advertise_addr.to_string(); + let stop = host.stop.clone(); let heartbeat_task = tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_millis( heartbeat_config.monitor.heartbeat_interval_ms, @@ -489,7 +538,10 @@ async fn main() -> std::process::ExitCode { let mut previous_requests = heartbeat_service.metrics().snapshot().requests; let mut previous_ms = heartbeat_service.monotonic_ms(); loop { - interval.tick().await; + tokio::select! { + () = stop.wait() => break, + _ = interval.tick() => {}, + } let now_ms = heartbeat_service.monotonic_ms(); let requests = heartbeat_service.metrics().snapshot().requests; let elapsed_ms = now_ms.saturating_sub(previous_ms).max(1); @@ -521,6 +573,7 @@ async fn main() -> std::process::ExitCode { } }); + host.tasks[3] = Some(heartbeat_task); let rpc_server = match ChunkKvRpcService::create_server(config.rpc_workers, config.rpc_rdma.as_ref()) { Ok(server) => server, Err(error) => { @@ -528,6 +581,7 @@ async fn main() -> std::process::ExitCode { return std::process::ExitCode::FAILURE; } }; + host.server = Some(rpc_server.clone()); if let Err(error) = rpc_server.listen(&rpc_addr.ip().to_string(), i32::from(rpc_addr.port())) { error!(%rpc_addr, %error, "data RPC bind failed"); return std::process::ExitCode::FAILURE; @@ -536,6 +590,7 @@ async fn main() -> std::process::ExitCode { Arc::clone(&service), tokio::runtime::Handle::current(), )); + host.rpc = Some(rpc_service.clone()); rpc_service.register_handlers(&rpc_server); rpc_server.start(); info!(%rpc_addr, "data RPC server listening"); @@ -574,35 +629,6 @@ async fn main() -> std::process::ExitCode { if let Err(error) = &serving { error!(%error, "HTTP management server failed"); } - heartbeat_task.abort(); - materialization_task.abort(); - transition_task.abort(); - refresh_task.abort(); - // These workers only observe the service and have already been fenced by - // begin_drain. Drop their handles after cancellation: a worker may be in - // a synchronous storage observation, and waiting for that observation here - // would hold the management restart request open until the supervisor kills - // the process. The service drain below is the durable shutdown barrier. - drop(( - heartbeat_task, - materialization_task, - transition_task, - refresh_task, - )); - info!("chunk KV background workers stopped"); - match service.drain_writers().await { - Ok(()) => info!("chunk KV WAL writer drain complete"), - Err(error) => { - warn!(%error, "WAL writer drain incomplete; restart must retain lease exclusion"); - } - } - if let Err(error) = process_lease.withdraw_observation().await { - warn!(%error, "failed to unregister chunk KV instance"); - } - info!("chunk KV observation withdrawal complete"); - if let Some(runner) = &mut metrics_runner { - runner.stop().await; - } if serving.is_ok() { std::process::ExitCode::SUCCESS } else { diff --git a/app/crowdb-chunk-kv-server/src/management.rs b/app/crowdb-chunk-kv-server/src/management.rs index 977285fae..33a2e30ca 100644 --- a/app/crowdb-chunk-kv-server/src/management.rs +++ b/app/crowdb-chunk-kv-server/src/management.rs @@ -28,7 +28,7 @@ impl ManagementState { #[derive(Debug, Serialize)] struct HealthResponse { instance_id: u64, - process_lease: crowdb_kv_client::StreamProcessLeaseStatus, + process_lease: crowdb_paxos_kv_client::StreamProcessLeaseStatus, lifecycle: &'static str, catalog_generation: u64, hosted_partitions: usize, diff --git a/app/crowdb-chunk-kv-server/src/metrics.rs b/app/crowdb-chunk-kv-server/src/metrics.rs index 0d92e219e..a9feb4f76 100644 --- a/app/crowdb-chunk-kv-server/src/metrics.rs +++ b/app/crowdb-chunk-kv-server/src/metrics.rs @@ -6,6 +6,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; use crowdb_protocol::chunk_kv::{ChunkKvResponse, ChunkKvRpcErrorCode}; use serde::Serialize; +mod resources; +pub use resources::TreeResources; + #[derive(Debug, Default)] pub struct ServerMetrics { requests: AtomicU64, @@ -151,7 +154,8 @@ fn increment(counter: &AtomicU64) { counter.fetch_add(1, Ordering::Relaxed); } -/// Starts periodic stream phase output alongside the service logs. +/// Creates periodic stream phase output alongside the service logs. +/// Install the Tree sampler before calling `start()`. /// Zero interval disables output. Call `stop().await` to flush the last window. /// /// # Errors @@ -159,7 +163,7 @@ fn increment(counter: &AtomicU64) { /// /// # Panics /// Panics if the metrics registry is poisoned or a histogram name conflicts. -pub fn start_stream_perf_log( +pub fn create_perf_log( log_dir: &str, interval_secs: u64, max_file_mb: usize, @@ -171,10 +175,9 @@ pub fn start_stream_perf_log( let file = crowdb_common::logging::open_metrics_log(log_dir, "crowdb-chunk-kv-server", max_file_mb, max_files) .map_err(std::io::Error::other)?; - let mut runner = crowdb_common::metrics::MetricsRunner::new(file, interval_secs); + let runner = crowdb_common::metrics::MetricsRunner::new(file, interval_secs); crowdb_chunk_stream::register_stream_perf_metrics( &mut runner.registry().lock().expect("metrics registry poisoned"), ); - runner.start(); Ok(Some(runner)) } diff --git a/app/crowdb-chunk-kv-server/src/metrics/resources.rs b/app/crowdb-chunk-kv-server/src/metrics/resources.rs new file mode 100644 index 000000000..7ee540290 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/metrics/resources.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::{Arc, Weak}; +use std::time::Instant; + +use crowdb_common::metrics::MetricsSampler; +use crowdb_tree_ffi::runtime::{ResourceUsageLog, ResourceUsageSnapshot}; + +use crate::{ChunkKvService, ChunkKvStorage}; + +pub struct TreeResources { + runtime: Arc<crowdb_tree_ffi::TreeRuntime>, + service: Weak<ChunkKvService>, + storage: Weak<ChunkKvStorage>, + windows: ResourceUsageLog, +} + +impl TreeResources { + #[must_use] + pub fn new(runtime: Arc<crowdb_tree_ffi::TreeRuntime>, service: &Arc<ChunkKvService>) -> Self { + Self { + runtime, + service: Arc::downgrade(service), + storage: Weak::new(), + windows: ResourceUsageLog::default(), + } + } + + #[must_use] + pub fn with_storage(mut self, storage: &Arc<ChunkKvStorage>) -> Self { + self.storage = Arc::downgrade(storage); + self + } + + fn host_usage(&self) -> Result<ResourceUsageSnapshot, crowdb_tree_ffi::CtError> { + let mut snapshot = self.runtime.resource_usage()?; + if let Some(storage) = self.storage.upgrade() { + snapshot.resources[20] = storage.pin_release_resource_usage()?; + snapshot.resource_mask |= 1 << 20; + } + Ok(snapshot) + } +} + +impl MetricsSampler for TreeResources { + fn sample(&mut self) { + self.windows.begin_sample(); + match self.host_usage() { + Ok(snapshot) => self.windows.observe("host.tree", snapshot, Instant::now()), + Err(_) => self.windows.sample_failed("host.tree"), + } + if let Some(service) = self.service.upgrade() { + service.sample_tree_resources(|name, result| match result { + Ok(Some(snapshot)) => self.windows.observe(name, snapshot, Instant::now()), + Ok(None) => {} + Err(_) => self.windows.sample_failed(name), + }); + } + self.windows.end_sample(Instant::now()); + } + + fn flush(&mut self, writer: &mut dyn std::io::Write, timestamp: &str) -> std::io::Result<()> { + self.windows.flush(writer, timestamp, Instant::now()) + } +} diff --git a/app/crowdb-chunk-kv-server/src/rpc.rs b/app/crowdb-chunk-kv-server/src/rpc.rs index da3f7ef17..b777bd707 100644 --- a/app/crowdb-chunk-kv-server/src/rpc.rs +++ b/app/crowdb-chunk-kv-server/src/rpc.rs @@ -26,6 +26,7 @@ use tokio::runtime::Handle; use crate::ChunkKvService; mod response; +mod tasks; mod transport; use response::{submit_group_response, submit_response}; @@ -33,12 +34,42 @@ use response::{submit_group_response, submit_response}; pub struct ChunkKvRpcService { service: Arc<ChunkKvService>, runtime: Handle, + tasks: Arc<tasks::RpcTasks>, } impl ChunkKvRpcService { #[must_use] pub fn new(service: Arc<ChunkKvService>, runtime: Handle) -> Self { - Self { service, runtime } + Self { + service, + runtime, + tasks: Arc::new(tasks::RpcTasks::default()), + } + } + + /// Stop asynchronous handler admission and wait until their request, + /// partition and response owners have all been dropped. + pub async fn close_and_drain(&self) { + self.tasks.close_and_drain().await; + } + + fn spawn_owned<F>(&self, permit: tasks::RpcTask, future: F) + where + F: std::future::Future<Output = ()> + Send + 'static, + { + self.runtime.spawn(async move { + { + let mut future = std::pin::pin!(future); + future.as_mut().await; + } + drop(permit); + }); + } + + #[cfg(feature = "test-util")] + #[must_use] + pub fn hold_request_for_tests(&self) -> Option<impl Send + 'static> { + self.tasks.enter() } pub fn register_handlers(self: &Arc<Self>, server: &Arc<RpcServer>) { @@ -75,9 +106,13 @@ impl ChunkKvRpcService { } fn handle_point(&self, request: ServerRequest, server: &Arc<RpcServer>) { + let Some(permit) = self.tasks.enter() else { + request.close_connection(); + return; + }; let service = Arc::clone(&self.service); let server = Arc::clone(server); - self.runtime.spawn(async move { + self.spawn_owned(permit, async move { let rpc_request_id = request.request_id; let response = match decode_point_request(request.control()) { Ok(envelope) if envelope.rpc_request_id == rpc_request_id => { @@ -93,9 +128,13 @@ impl ChunkKvRpcService { } fn handle_seek(&self, request: ServerRequest, server: &Arc<RpcServer>) { + let Some(permit) = self.tasks.enter() else { + request.close_connection(); + return; + }; let service = Arc::clone(&self.service); let server = Arc::clone(server); - self.runtime.spawn(async move { + self.spawn_owned(permit, async move { let rpc_request_id = request.request_id; let response = match decode_seek_request(request.control()) { Ok(envelope) if envelope.rpc_request_id == rpc_request_id => { @@ -111,9 +150,13 @@ impl ChunkKvRpcService { } fn handle_scan(&self, request: ServerRequest, server: &Arc<RpcServer>) { + let Some(permit) = self.tasks.enter() else { + request.close_connection(); + return; + }; let service = Arc::clone(&self.service); let server = Arc::clone(server); - self.runtime.spawn(async move { + self.spawn_owned(permit, async move { let rpc_request_id = request.request_id; let response = match decode_scan_request(request.control()) { Ok(envelope) if envelope.rpc_request_id == rpc_request_id => { @@ -129,9 +172,13 @@ impl ChunkKvRpcService { } fn handle_bounded_scan(&self, request: ServerRequest, server: &Arc<RpcServer>) { + let Some(permit) = self.tasks.enter() else { + request.close_connection(); + return; + }; let service = Arc::clone(&self.service); let server = Arc::clone(server); - self.runtime.spawn(async move { + self.spawn_owned(permit, async move { let id = request.request_id; let response = match decode_bounded_scan_request(request.control()) { Ok((envelope, budget)) if envelope.rpc_request_id == id => { @@ -147,9 +194,13 @@ impl ChunkKvRpcService { } fn handle_multi_get(&self, request: ServerRequest, server: &Arc<RpcServer>) { + let Some(permit) = self.tasks.enter() else { + request.close_connection(); + return; + }; let service = Arc::clone(&self.service); let server = Arc::clone(server); - self.runtime.spawn(async move { + self.spawn_owned(permit, async move { let id = request.request_id; let create_nano = request.rpc_create_nano; let response = match decode_multi_get_request(request.control()) { @@ -177,9 +228,13 @@ impl ChunkKvRpcService { } fn handle_batch_mutation(&self, request: ServerRequest, server: &Arc<RpcServer>) { + let Some(permit) = self.tasks.enter() else { + request.close_connection(); + return; + }; let service = Arc::clone(&self.service); let server = Arc::clone(server); - self.runtime.spawn(async move { + self.spawn_owned(permit, async move { let id = request.request_id; let create_nano = request.rpc_create_nano; let response = match decode_batch_mutation_request(request.control()) { diff --git a/app/crowdb-chunk-kv-server/src/rpc/tasks.rs b/app/crowdb-chunk-kv-server/src/rpc/tasks.rs new file mode 100644 index 000000000..b2eef3202 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/rpc/tasks.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +//! One atomic gate covers handler admission through the last owned request tail. +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::sync::Notify; +const CLOSED: usize = 1 << (usize::BITS - 1); +#[derive(Default)] +pub(super) struct RpcTasks { + state: AtomicUsize, + changed: Notify, +} +pub(super) struct RpcTask { + owner: Arc<RpcTasks>, +} +impl RpcTasks { + pub fn enter(self: &Arc<Self>) -> Option<RpcTask> { + let mut observed = self.state.load(Ordering::Acquire); + loop { + if observed & CLOSED != 0 || observed == CLOSED - 1 { + return None; + } + match self.state.compare_exchange_weak( + observed, + observed + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Some(RpcTask { owner: self.clone() }), + Err(actual) => observed = actual, + } + } + } + pub async fn close_and_drain(&self) { + self.state.fetch_or(CLOSED, Ordering::AcqRel); + loop { + let change = self.changed.notified(); + tokio::pin!(change); + change.as_mut().enable(); + if self.state.load(Ordering::Acquire) == CLOSED { + return; + } + change.await; + } + } +} +impl Drop for RpcTask { + fn drop(&mut self) { + if self.owner.state.fetch_sub(1, Ordering::AcqRel) == CLOSED + 1 { + self.owner.changed.notify_waiters(); + } + } +} diff --git a/app/crowdb-chunk-kv-server/src/server.rs b/app/crowdb-chunk-kv-server/src/server.rs index 60480a628..454e63dff 100644 --- a/app/crowdb-chunk-kv-server/src/server.rs +++ b/app/crowdb-chunk-kv-server/src/server.rs @@ -11,7 +11,7 @@ use crowdb_chunk_kv::{ ChunkKvError, CompareCondition, JournalPosition, MutationOperation, MutationResult, Partition, PreparedSplitWriterArtifact, RequestId, SplitArtifact, ValueRevision, }; -use crowdb_kv_client::StreamProcessAuthority; +use crowdb_paxos_kv_client::StreamProcessAuthority; use crowdb_protocol::chunk_kv::{ BatchMutationRequest, BatchMutationResponse, BatchMutationResult, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPartitionState, ChunkKvResponse, @@ -34,6 +34,7 @@ mod authority; mod drain; mod group; use assignments::{local_partition_for_entry, partition_matches_entry}; +mod legacy_retirement; mod load_sampling; mod maintenance; mod observation; @@ -42,6 +43,7 @@ mod readiness; mod recovery; pub use readiness::RangeReadiness; mod reconcile; +mod resources; mod routing; mod scan; pub(crate) use observation::PageQuery; @@ -378,9 +380,14 @@ impl ChunkKvService { #[must_use] pub fn registry_observation(&self, capacity_bytes: u64, request_rate: u64) -> ChunkKvExtra { let partitions = self.observed_partition_snapshot(); + let mut scopes = std::collections::BTreeSet::new(); let durable_bytes = partitions .values() - .filter_map(|partition| partition.estimated_bytes().ok()) + .filter_map(|partition| match partition.backing_storage_stats() { + Ok(Some(stats)) => scopes.insert(stats.scope_id).then_some(stats.allocated_bytes), + Ok(None) => partition.estimated_bytes().ok(), + Err(_) => None, + }) .fold(0_u64, u64::saturating_add); let mut hosted: Vec<_> = partitions .values() diff --git a/app/crowdb-chunk-kv-server/src/server/drain.rs b/app/crowdb-chunk-kv-server/src/server/drain.rs index ea1b7b427..d87b4ac38 100644 --- a/app/crowdb-chunk-kv-server/src/server/drain.rs +++ b/app/crowdb-chunk-kv-server/src/server/drain.rs @@ -22,8 +22,14 @@ impl ChunkKvService { self.begin_drain(); self.load_sampling.drain().await; let partitions = self.partitions.load_full(); + let prepared = self.prepared_transfer_targets.load_full(); + let splits = self.local_split_sessions.load_full(); let mut first_error = None; - for partition in partitions.values() { + for partition in partitions + .values() + .chain(prepared.values()) + .chain(splits.values().map(|entry| &entry.dispatcher)) + { if let Err(error) = partition .suspend_for_shutdown(partition.snapshot().ownership_epoch) .await @@ -33,4 +39,18 @@ impl ChunkKvService { } first_error.map_or(Ok(()), Err) } + + /// Release hosted Tree owners after background tasks and RPC handlers end. + /// Independent snapshots keep their exact backend owners until their own drop. + /// # Errors + /// Reports a failure of the host blocking task that closes native Trees. + pub async fn release_hosted_trees(&self) -> Result<(), ChunkKvError> { + self.begin_drain(); + let partitions = self.partitions.swap(std::sync::Arc::default()); + let prepared = self.prepared_transfer_targets.swap(std::sync::Arc::default()); + let splits = self.local_split_sessions.swap(std::sync::Arc::default()); + tokio::task::spawn_blocking(move || drop((partitions, prepared, splits))) + .await + .map_err(|error| ChunkKvError::MaintenanceDegraded(error.to_string())) + } } diff --git a/app/crowdb-chunk-kv-server/src/server/legacy_retirement.rs b/app/crowdb-chunk-kv-server/src/server/legacy_retirement.rs new file mode 100644 index 000000000..81184c062 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/legacy_retirement.rs @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Resume explicitly started namespace retirement through the existing host pump. +use super::{local_partition_for_entry, ChunkKvRangeCatalogPartitionState, ChunkKvService}; +use crate::storage::ChunkKvStorage; +use crowdb_chunk_kv::ChunkKvError; +use crowdb_chunk_stream::StreamRegistry; +use std::collections::HashSet; + +impl ChunkKvService { + /// Resume only durable, explicitly started legacy retirements for live local + /// metadata namespaces. Missing proof waits for the next scheduled pass. + /// # Errors + /// Returns metadata, ownership, or physical cleanup failures without dropping claims. + pub async fn resume_legacy_retirement(&self, storage: &ChunkKvStorage) -> Result<bool, ChunkKvError> { + let catalog = self.catalog.load_full(); + let partitions = self.partitions.load_full(); + let mut groups = HashSet::new(); + let mut pending = false; + for entry in &catalog.entries { + if entry.owner.instance_id != self.instance_id + || entry.state != ChunkKvRangeCatalogPartitionState::Serving + || entry.transition_id.is_some() + || self + .authority + .authorize( + catalog.generation, + entry.partition_id, + entry.owner_epoch, + self.monotonic_ms(), + ) + .is_err() + || partitions + .get(&entry.partition_id) + .and_then(|partition| local_partition_for_entry(partition, entry)) + .is_none() + { + continue; + } + let record = storage + .streams() + .registry() + .load(entry.artifact.stream_name) + .await + .map_err(|error| ChunkKvError::MaintenanceDegraded(error.to_string()))? + .ok_or_else(|| { + ChunkKvError::MaintenanceDegraded("legacy namespace stream is absent".into()) + })?; + if !groups.insert(record.metadata_group_id) { + continue; + } + if let Some(progress) = storage + .resume_legacy_storage(record.metadata_group_id, 64) + .await + .map_err(|error| ChunkKvError::MaintenanceDegraded(error.to_string()))? + { + pending |= !progress.complete && progress.pending.is_none(); + } + } + Ok(pending) + } +} diff --git a/app/crowdb-chunk-kv-server/src/server/resources.rs b/app/crowdb-chunk-kv-server/src/server/resources.rs new file mode 100644 index 000000000..ec6ee4475 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/resources.rs @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::ChunkKvService; + +impl ChunkKvService { + pub(crate) fn sample_tree_resources( + &self, + mut observe: impl FnMut( + &str, + crowdb_chunk_kv::Result<Option<crowdb_tree_ffi::runtime::ResourceUsageSnapshot>>, + ), + ) { + // Include prepared transfer targets, whose Trees already consume host + // resources before activation into the serving catalog. + for partition in self.observed_partition_snapshot().values() { + let name = format!("chunk.tree.{}", partition.tree_id()); + observe(&name, partition.tree_resource_usage()); + } + } +} diff --git a/app/crowdb-chunk-kv-server/src/serving/lease.rs b/app/crowdb-chunk-kv-server/src/serving/lease.rs index a626aba49..5224c6707 100644 --- a/app/crowdb-chunk-kv-server/src/serving/lease.rs +++ b/app/crowdb-chunk-kv-server/src/serving/lease.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use std::time::Instant; use arc_swap::ArcSwap; -use crowdb_kv_client::StreamProcessAuthority; +use crowdb_paxos_kv_client::StreamProcessAuthority; use crowdb_protocol::chunk_kv::{DomainMonitorDescriptor, Id128, InstanceHealth, ServingGrant}; use thiserror::Error; diff --git a/app/crowdb-chunk-kv-server/src/serving/lease/context.rs b/app/crowdb-chunk-kv-server/src/serving/lease/context.rs index 8f5f57cad..df2ab5dcd 100644 --- a/app/crowdb-chunk-kv-server/src/serving/lease/context.rs +++ b/app/crowdb-chunk-kv-server/src/serving/lease/context.rs @@ -4,7 +4,7 @@ //! Immutable admission context and shared grant-snapshot validation. use super::{AuthorityError, AuthoritySnapshot, AuthorityState, ServingAuthority}; -use crowdb_kv_client::StreamProcessContext; +use crowdb_paxos_kv_client::StreamProcessContext; use crowdb_protocol::chunk_kv::{Id128, ServingAssignment}; use std::time::Instant; diff --git a/app/crowdb-chunk-kv-server/src/serving/process.rs b/app/crowdb-chunk-kv-server/src/serving/process.rs index 4c6d3406d..136bd99f5 100644 --- a/app/crowdb-chunk-kv-server/src/serving/process.rs +++ b/app/crowdb-chunk-kv-server/src/serving/process.rs @@ -8,8 +8,8 @@ use std::{ time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use crowdb_kv_client::{ - CrowdbKvClient, StreamProcessAuthority, StreamProcessContext, StreamProcessLeaseClient, +use crowdb_paxos_kv_client::{ + PaxosKvClient, StreamProcessAuthority, StreamProcessContext, StreamProcessLeaseClient, StreamProcessLeaseError, StreamProcessLeaseTiming, }; use crowdb_protocol::{ @@ -54,7 +54,7 @@ impl ProcessLeaseSession { /// # Errors /// Returns invalid policy, a changed predecessor or an unavailable lease. pub async fn start( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, instance_id: u64, policy: &DomainMonitorDescriptor, ) -> Result<Self, StreamProcessLeaseError> { diff --git a/app/crowdb-chunk-kv-server/src/serving/worker.rs b/app/crowdb-chunk-kv-server/src/serving/worker.rs index 7d181f7e6..1c45e4dd3 100644 --- a/app/crowdb-chunk-kv-server/src/serving/worker.rs +++ b/app/crowdb-chunk-kv-server/src/serving/worker.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use async_trait::async_trait; -use crowdb_chunk_kv::{Partition, PartitionConfig, SplitArtifact}; +use crowdb_chunk_kv::{Partition, SplitArtifact}; use crowdb_protocol::chunk_kv::{ AuthorityReleaseProof, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, PartitionArtifact, SplitPhase, SplitReadinessProof, SplitTransition, TailOverlayArtifact, TargetReadinessProof, @@ -475,7 +475,7 @@ impl TransitionStorage for ChunkKvStorage { let artifact = prepared.artifact.clone(); let child = prepared .child - .open_warmed(PartitionConfig::default()) + .open_warmed(self.partition_config().clone()) .map_err(|error| plan_error(&error.to_string()))?; let retained_parent = parent .split_ingress() diff --git a/app/crowdb-chunk-kv-server/src/storage.rs b/app/crowdb-chunk-kv-server/src/storage.rs index ea7606605..b6815f253 100644 --- a/app/crowdb-chunk-kv-server/src/storage.rs +++ b/app/crowdb-chunk-kv-server/src/storage.rs @@ -7,12 +7,12 @@ use std::sync::Arc; use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, ChunkReadPolicy, SmallWritePolicy}; use crowdb_chunk_kv::{ - Partition, PartitionConfig, PartitionId, PartitionJournal, PartitionRange, PartitionTree, PreparedSplit, + Partition, PartitionConfig, PartitionId, PartitionJournal, PartitionRange, PreparedSplit, PreparedSplitWriterArtifact, SplitArtifact, SplitChild, SplitPlan, SplitWriterTarget, StreamPartitionJournal, TransitionId, }; use crowdb_chunk_stream::{ChunkStream, ProductionStreamRuntime, StreamConfig, StreamName, StreamRegistry}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, PartitionArtifact, SplitChildAssignment, SplitTransition, TailOverlayArtifact, TransferTransition, @@ -26,8 +26,14 @@ use thiserror::Error; use crate::{BootstrapPartitionConfig, ChunkKvServerConfig}; mod bootstrap; +pub mod legacy_retirement; mod root_catalog; -use bootstrap::{initialize_bootstrap_root, recover_bootstrap_partition}; +#[cfg(feature = "test-util")] +mod root_catalog_test; +mod tree_owner; +use bootstrap::{initialize_bootstrap_root, open_bootstrap_partition, recover_bootstrap_partition}; +#[cfg(feature = "test-util")] +pub use root_catalog_test::{open_root_catalog_with_process_for_tests, TestOwnedRootCatalog}; mod split_handoff; mod split_recovery; mod streams; @@ -47,19 +53,42 @@ pub enum StorageRuntimeError { /// Process-wide production clients shared by every hosted partition stream. pub struct ChunkKvStorage { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, chunk_io: ChunkIoClient, streams: Arc<ProductionStreamRuntime>, tree_transport: Arc<ChunkTransport>, + tree_runtime: Arc<crowdb_tree_ffi::TreeRuntime>, + tree_options: crowdb_tree_ffi::Config, + partition_config: PartitionConfig, + pin_releases: crowdb_tree_ffi::chunk::pin_service::ChunkPinReleaseService, tree_mirror_copies: u32, tree_chunk_capacity_bytes: u64, metadata_store_id: u64, + owner_process: arc_swap::ArcSwapOption<crowdb_paxos_kv_client::StreamProcessAuthority>, } -struct TreeStorageGeometry { +struct TreeStorageOptions { mirror_copies: u32, chunk_capacity_bytes: u64, strip_capacity_bytes: u32, + pin_release_slots: usize, + runtime: crowdb_tree_ffi::TreeRuntimeOptions, + native: crowdb_tree_ffi::Config, + partition: PartitionConfig, +} + +impl Default for TreeStorageOptions { + fn default() -> Self { + Self { + mirror_copies: 2, + chunk_capacity_bytes: 256 * 1024 * 1024, + strip_capacity_bytes: 0, + pin_release_slots: 1_048_576, + runtime: crowdb_tree_ffi::TreeRuntimeOptions::default(), + native: crowdb_tree_ffi::Config::default(), + partition: PartitionConfig::default(), + } + } } impl ChunkKvStorage { @@ -70,7 +99,7 @@ impl ChunkKvStorage { /// /// Returns an endpoint discovery or storage configuration error. pub async fn connect(config: &ChunkKvServerConfig) -> Result<Self, StorageRuntimeError> { - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( config.group0_mgmt_seeds.clone(), ))); let chunk_io = ChunkIoClient::connect_with_kv( @@ -84,15 +113,19 @@ impl ChunkKvStorage { ) .await .map_err(|error| StorageRuntimeError::ChunkIo(error.to_string()))?; - Self::from_parts_with_geometry( + Self::from_parts_with_options( kv, chunk_io, config.storage.metadata_store_id, config.storage.stream_writer_lease_ms, - TreeStorageGeometry { + TreeStorageOptions { mirror_copies: config.storage.stream_mirror_copies, chunk_capacity_bytes: config.storage.tree_chunk_capacity_bytes, strip_capacity_bytes: config.storage.tree_strip_capacity_bytes, + pin_release_slots: config.storage.tree_pin_release_slots, + runtime: config.tree_runtime.runtime_options(), + native: config.native_tree_options(), + partition: config.partition.clone(), }, StreamConfig { chunk_capacity_bytes: config.storage.stream_chunk_capacity_bytes, @@ -106,12 +139,12 @@ impl ChunkKvStorage { .await } - async fn from_parts_with_geometry( - kv: Arc<CrowdbKvClient>, + async fn from_parts_with_options( + kv: Arc<PaxosKvClient>, chunk_io: ChunkIoClient, metadata_store_id: u64, writer_lease_ms: u64, - tree: TreeStorageGeometry, + tree: TreeStorageOptions, stream_config: StreamConfig, read_policy: ChunkReadPolicy, ) -> Result<Self, StorageRuntimeError> { @@ -135,21 +168,22 @@ impl ChunkKvStorage { /// /// Returns an error for an invalid stream lease or read policy. pub async fn from_parts( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, chunk_io: ChunkIoClient, metadata_store_id: u64, writer_lease_ms: u64, stream_mirror_copies: u32, ) -> Result<Self, StorageRuntimeError> { - Self::from_parts_with_geometry( + Self::from_parts_with_options( kv, chunk_io, metadata_store_id, writer_lease_ms, - TreeStorageGeometry { + TreeStorageOptions { mirror_copies: stream_mirror_copies, chunk_capacity_bytes: 256 * 1024 * 1024, strip_capacity_bytes: 0, + ..TreeStorageOptions::default() }, StreamConfig::default(), ChunkReadPolicy::default(), @@ -158,12 +192,12 @@ impl ChunkKvStorage { } async fn assemble( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, chunk_io: ChunkIoClient, streams: Arc<ProductionStreamRuntime>, metadata_store_id: u64, writer_lease_ms: u64, - tree: TreeStorageGeometry, + tree: TreeStorageOptions, ) -> Result<Self, StorageRuntimeError> { let (chunkdb, disks) = chunk_io .native_storage_routes(writer_lease_ms) @@ -189,19 +223,53 @@ impl ChunkKvStorage { }) .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?, ); + let tree_runtime = Arc::new( + crowdb_tree_ffi::TreeRuntime::new(tree.runtime) + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?, + ); + let pin_releases = crowdb_tree_ffi::chunk::pin_service::ChunkPinReleaseService::new( + &tree_runtime, + tree.pin_release_slots, + ) + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?; Ok(Self { kv, chunk_io, streams, tree_transport, + tree_runtime, + tree_options: tree.native, + partition_config: tree.partition, + pin_releases, tree_mirror_copies: tree.mirror_copies, tree_chunk_capacity_bytes: tree.chunk_capacity_bytes, metadata_store_id, + owner_process: arc_swap::ArcSwapOption::empty(), }) } #[must_use] - pub fn kv(&self) -> &Arc<CrowdbKvClient> { + pub fn tree_runtime(&self) -> &Arc<crowdb_tree_ffi::TreeRuntime> { + &self.tree_runtime + } + + pub(crate) fn pin_release_resource_usage( + &self, + ) -> Result<crowdb_tree_ffi::runtime::ResourceUsage, crowdb_tree_ffi::CtError> { + self.pin_releases.resource_usage() + } + + pub(crate) fn partition_config(&self) -> &PartitionConfig { + &self.partition_config + } + + /// Await real final-pin metadata work before the host transport/runtime stop. + pub async fn drain_pin_releases(&self) { + self.pin_releases.shutdown().await; + } + + #[must_use] + pub fn kv(&self) -> &Arc<PaxosKvClient> { &self.kv } @@ -229,7 +297,34 @@ impl ChunkKvStorage { ) -> Result<Arc<PageStore>, StorageRuntimeError> { options.mirror_copies = self.tree_mirror_copies; options.max_chunk_bytes = self.tree_chunk_capacity_bytes; - PageStore::open_chunk(options, catalog, Some(&self.tree_transport)) + if self.owner_process.load().is_none() { + return Err(StorageRuntimeError::Tree( + "tree process authority is not bound".into(), + )); + } + self.pin_releases + .bind(&catalog) + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?; + let opened = PageStore::open_chunk_runtime( + &self.tree_runtime, + crowdb_tree_ffi::ChunkRuntimeOptions { + tree_id: options.tree_id, + owner_epoch: options.owner_epoch, + open_generation: options.open_generation, + iu_size: if options.iu_size != 0 { + options.iu_size + } else if self.tree_options.iu_size != 0 { + self.tree_options.iu_size + } else { + 64 * 1024 + }, + mirror_copies: options.mirror_copies, + max_chunk_bytes: options.max_chunk_bytes, + }, + catalog, + &self.tree_transport, + ); + opened .map(Arc::new) .map_err(|error| StorageRuntimeError::Tree(error.to_string())) } @@ -245,21 +340,9 @@ impl ChunkKvStorage { options: ChunkPageStoreOptions, metadata_group_id: u64, ) -> Result<Arc<PageStore>, StorageRuntimeError> { - let catalog = KvRootCatalogStore::claim( - Arc::clone(&self.kv), - self.metadata_store_id, - metadata_group_id, - options.tree_id, - options.owner_epoch, - ) - .await?; - self.open_tree_page_store( - options, - Arc::new( - ChunkRootCatalog::open_callback(Arc::new(catalog)) - .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?, - ), - ) + self.open_durable_tree(options, metadata_group_id) + .await + .map(|opened| opened.0) } /// Initializes the stream selected by an Active catalog record. @@ -331,36 +414,7 @@ impl ChunkKvStorage { entry.owner_epoch )) })?; - let open_generation = entry - .artifact - .tail_overlay - .as_ref() - .map_or(0, |overlay| overlay.base_root_manifest_generation); - let options = ChunkPageStoreOptions { - tree_id: entry.artifact.tree_id, - owner_epoch: entry.owner_epoch, - open_generation, - pack_bytes: 0, - iu_size: 0, - max_concurrent_packs: 0, - materialization_bytes_per_pass: 0, - mirror_copies: 0, - max_chunk_bytes: 0, - }; - let catalog = KvRootCatalogStore::for_assignment( - self.kv.clone(), - self.metadata_store_id, - record.metadata_group_id, - entry, - ) - .await?; - let page_store = self.open_tree_page_store( - options, - Arc::new( - ChunkRootCatalog::open_callback(Arc::new(catalog)) - .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?, - ), - )?; + let (page_store, catalog) = self.open_assignment_tree(entry, record.metadata_group_id).await?; if let Some(overlay) = &entry.artifact.tail_overlay { let parent_stream = self .streams @@ -373,18 +427,20 @@ impl ChunkKvStorage { )) })?; let artifact = prepared_overlay_artifact(entry, overlay); - return Partition::recover_native_prepared_overlay( + let partition = Partition::recover_native_prepared_overlay( artifact, - PartitionConfig::default(), - crowdb_tree_ffi::Config::default(), - page_store, + self.partition_config.clone(), + self.tree_config(), + page_store.clone(), stream, parent_stream, ) .await - .map_err(|error| overlay_recovery_error(entry, overlay, &error)); + .map_err(|error| overlay_recovery_error(entry, overlay, &error))?; + Self::complete_tree_recovery(&partition, &page_store, &catalog).await?; + return Ok(partition); } - Partition::recover_native_latest_prepared_assignment( + let partition = Partition::recover_native_latest_prepared_assignment( PartitionId { high: entry.partition_id.high, low: entry.partition_id.low, @@ -395,13 +451,15 @@ impl ChunkKvStorage { }, entry.owner_epoch, entry.artifact.tree_id, - PartitionConfig::default(), - crowdb_tree_ffi::Config::default(), - page_store, + self.partition_config.clone(), + self.tree_config(), + page_store.clone(), stream, ) .await - .map_err(|error| assignment_recovery_error(entry, &error)) + .map_err(|error| assignment_recovery_error(entry, &error))?; + Self::complete_tree_recovery(&partition, &page_store, &catalog).await?; + Ok(partition) } /// Advances one already-open transfer target from its preparation cursor @@ -507,8 +565,8 @@ impl ChunkKvStorage { ) .await .map_err(|error| StorageRuntimeError::Stream(error.to_string()))?; - let page_store = self - .open_durable_tree_page_store( + let (page_store, catalog) = self + .open_durable_tree( ChunkPageStoreOptions { tree_id: config.tree_id, owner_epoch: config.owner_epoch, @@ -523,35 +581,28 @@ impl ChunkKvStorage { config.metadata_group_id, ) .await?; - let partition_id = PartitionId { - high: config.partition_id.high, - low: config.partition_id.low, - }; - let range = PartitionRange { - start: Some(Vec::new()), - end: None, - }; if !fresh { - return recover_bootstrap_partition(config, page_store, stream).await; - } - let owner_epoch = config.owner_epoch; - let tree_id = config.tree_id; - let partition = tokio::task::spawn_blocking(move || { - Partition::open_native( - partition_id, - range, - owner_epoch, - PartitionConfig::default(), - tree_id, - crowdb_tree_ffi::Config::default(), - page_store, + let partition = recover_bootstrap_partition( + config, + self.partition_config.clone(), + self.tree_config(), + page_store.clone(), stream, ) - }) - .await - .map_err(|error| StorageRuntimeError::Partition(format!("bootstrap tree open: {error}")))? - .map_err(|error| StorageRuntimeError::Partition(format!("bootstrap tree open: {error}")))?; + .await?; + Self::complete_tree_recovery(&partition, &page_store, &catalog).await?; + return Ok(partition); + } + let partition = open_bootstrap_partition( + config, + self.partition_config.clone(), + self.tree_config(), + page_store.clone(), + stream, + ) + .await?; initialize_bootstrap_root(&partition, config).await?; + Self::complete_tree_recovery(&partition, &page_store, &catalog).await?; Ok(partition) } @@ -605,16 +656,14 @@ impl ChunkKvStorage { "nonempty child WAL has no committed handoff proof", )); } - let tree_id = child.tree_id; - let tree_config = child.tree_config.clone(); let transition_id = split_plan(transition).transition_id; - tokio::task::spawn_blocking(move || { - crowdb_chunk_kv::CrowdbPartitionTree::open(tree_id, &tree_config)? - .unpin_generation(transition_id) - }) - .await - .map_err(|error| storage_plan_error(&error.to_string()))? - .map_err(|error| storage_plan_error(&error.to_string()))?; + child + .tree_config + .page_store + .as_ref() + .ok_or_else(|| storage_plan_error("split child page store is absent"))? + .unpin_chunk_generation(child.tree_id, transition_id.high, transition_id.low) + .map_err(|error| storage_plan_error(&error.to_string()))?; child }; let handoff = Arc::new(split_handoff::SplitHandoffCommit { @@ -735,7 +784,7 @@ impl ChunkKvStorage { tree_id: artifact.tree_id, tree_config: crowdb_tree_ffi::Config { page_store: Some(page_store), - ..crowdb_tree_ffi::Config::default() + ..self.tree_config() }, journal: Arc::new( StreamPartitionJournal::new(stream, artifact.stream_name) @@ -886,7 +935,7 @@ fn assignment_recovery_error( /// Returns a root identity, KV availability or stale authority error. #[cfg(feature = "test-util")] pub async fn open_root_catalog_for_tests( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, store_id: u64, group_id: u64, entry: &ChunkKvRangeCatalogEntry, diff --git a/app/crowdb-chunk-kv-server/src/storage/bootstrap.rs b/app/crowdb-chunk-kv-server/src/storage/bootstrap.rs index e7f818b78..b7c299616 100644 --- a/app/crowdb-chunk-kv-server/src/storage/bootstrap.rs +++ b/app/crowdb-chunk-kv-server/src/storage/bootstrap.rs @@ -7,16 +7,32 @@ use super::StorageRuntimeError; #[cfg(feature = "test-util")] impl super::ChunkKvStorage { + /// Open a private bootstrap with the ordinary production engine settings. + /// + /// # Errors + /// Returns invalid native configuration or worker initialization errors. + pub async fn open_bootstrap_partition_for_tests( + config: &BootstrapPartitionConfig, + partition_config: PartitionConfig, + tree_config: crowdb_tree_ffi::Config, + page_store: Arc<PageStore>, + stream: ChunkStream, + ) -> Result<Partition, StorageRuntimeError> { + open_bootstrap_partition(config, partition_config, tree_config, page_store, stream).await + } + /// Resume a private bootstrap before its first catalog publication. /// /// # Errors /// Returns durable identity, WAL replay, and checkpoint errors. pub async fn recover_bootstrap_partition_for_tests( config: &BootstrapPartitionConfig, + partition_config: PartitionConfig, + tree_config: crowdb_tree_ffi::Config, page_store: Arc<PageStore>, stream: ChunkStream, ) -> Result<Partition, StorageRuntimeError> { - recover_bootstrap_partition(config, page_store, stream).await + recover_bootstrap_partition(config, partition_config, tree_config, page_store, stream).await } } use crate::BootstrapPartitionConfig; @@ -27,16 +43,46 @@ use crowdb_chunk_stream::ChunkStream; use crowdb_tree_ffi::{CtError, PageStore}; use std::sync::Arc; -pub(super) async fn recover_bootstrap_partition( +pub(super) async fn open_bootstrap_partition( config: &BootstrapPartitionConfig, + partition_config: PartitionConfig, + tree_config: crowdb_tree_ffi::Config, page_store: Arc<PageStore>, stream: ChunkStream, ) -> Result<Partition, StorageRuntimeError> { - let pending = match page_store.chunk_manifest_generation() { - Ok(generation) => generation == 0, - Err(CtError::NotFound) => true, - Err(error) => return Err(StorageRuntimeError::Tree(error.to_string())), + let partition_id = PartitionId { + high: config.partition_id.high, + low: config.partition_id.low, }; + let owner_epoch = config.owner_epoch; + let tree_id = config.tree_id; + tokio::task::spawn_blocking(move || { + Partition::open_native( + partition_id, + PartitionRange { + start: Some(Vec::new()), + end: None, + }, + owner_epoch, + partition_config, + tree_id, + tree_config, + page_store, + stream, + ) + }) + .await + .map_err(|error| StorageRuntimeError::Partition(error.to_string()))? + .map_err(|error| StorageRuntimeError::Partition(format!("bootstrap tree open: {error}"))) +} + +pub(super) async fn recover_bootstrap_partition( + config: &BootstrapPartitionConfig, + partition_config: PartitionConfig, + tree_config: crowdb_tree_ffi::Config, + page_store: Arc<PageStore>, + stream: ChunkStream, +) -> Result<Partition, StorageRuntimeError> { let partition = Partition::recover_native_latest_prepared_assignment( PartitionId { high: config.partition_id.high, @@ -48,13 +94,18 @@ pub(super) async fn recover_bootstrap_partition( }, config.owner_epoch, config.tree_id, - PartitionConfig::default(), - crowdb_tree_ffi::Config::default(), - page_store, + partition_config, + tree_config, + page_store.clone(), stream, ) .await .map_err(|error| StorageRuntimeError::Partition(error.to_string()))?; + let pending = match page_store.chunk_manifest_generation() { + Ok(generation) => generation == 0, + Err(CtError::NotFound) => true, + Err(error) => return Err(StorageRuntimeError::Tree(error.to_string())), + }; if pending { partition .activate_recovered(config.owner_epoch) diff --git a/app/crowdb-chunk-kv-server/src/storage/legacy_retirement.rs b/app/crowdb-chunk-kv-server/src/storage/legacy_retirement.rs new file mode 100644 index 000000000..bad23cc8c --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/legacy_retirement.rs @@ -0,0 +1,250 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Explicit stopped-legacy-format retirement. Unknown historical owners retain their receipts. +use super::ChunkKvStorage; +use crowdb_chunk_client::{ChunkAllocator, DiskWriter}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode, StreamProcessAuthority}; +use crowdb_tree_ffi::CtError; +use serde::{Deserialize, Serialize}; +use std::{sync::Arc, time::Instant}; +mod allocation; +mod proof; +pub(crate) mod record; +use record::{Namespace, Phase, NAMESPACE_KEY, ROOT_PREFIX}; + +/// A durable reason why a known legacy receipt is still physically retained. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum LegacyRetirementPending { + LegacyRoot { + tree: u64, + }, + MissingOwnerProof { + tree: u64, + generation: u64, + }, + LiveProcess { + tree: u64, + generation: u64, + instance: u64, + epoch: u64, + }, + GenerationPin { + tree: u64, + generation: u64, + }, + MissingAllocation { + high: u64, + low: u64, + }, + AllocationChanged { + high: u64, + low: u64, + }, +} +/// Work in this invocation; `complete` covers persisted, known manifest receipts only. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LegacyRetirementProgress { + pub examined: u32, + pub reclaimed_chunks: u32, + pub reclaimed_bytes: u64, + pub complete: bool, + pub pending: Option<LegacyRetirementPending>, +} + +impl ChunkKvStorage { + /// Permanently stop legacy-format publication in this metadata namespace and + /// retire known receipts only after all registered legacy readers are excluded. + /// No writer-quiesced flag substitutes for authoritative process exclusion. + /// Historical allocations lacking a retained manifest are outside this census. + /// + /// # Errors + /// Returns fencing, metadata, or physical-IO errors. The durable cursor and + /// allocation claim survive unknown replies; the next call resumes them. + pub async fn collect_legacy_storage( + &self, + metadata_group_id: u64, + limit: u32, + ) -> Result<LegacyRetirementProgress, CtError> { + self.legacy_collector(metadata_group_id)?.step(limit).await + } + + pub(crate) async fn resume_legacy_storage( + &self, + metadata_group_id: u64, + limit: u32, + ) -> Result<Option<LegacyRetirementProgress>, CtError> { + self.legacy_collector(metadata_group_id)?.resume(limit).await + } + + fn legacy_collector(&self, metadata_group_id: u64) -> Result<Collector, CtError> { + let process = self.owner_process.load_full().ok_or(CtError::NotSupported)?; + let (allocator, disks) = self.chunk_io.storage_parts(); + Ok(Collector { + kv: self.kv.clone(), + store: self.metadata_store_id, + group: metadata_group_id, + process, + allocator, + disks, + }) + } +} + +pub(super) struct Collector { + pub kv: Arc<PaxosKvClient>, + pub store: u64, + pub group: u64, + pub process: Arc<StreamProcessAuthority>, + pub allocator: Arc<dyn ChunkAllocator>, + pub disks: Arc<dyn DiskWriter>, +} +impl Collector { + async fn resume(&self, limit: u32) -> Result<Option<LegacyRetirementProgress>, CtError> { + if self.get(NAMESPACE_KEY).await?.is_none() { + return Ok(None); + } + self.step(limit).await.map(Some) + } + + async fn get(&self, key: &[u8]) -> Result<Option<(bytes::Bytes, u64)>, CtError> { + match self + .kv + .get(self.store, self.group, key, ReadMode::Linearizable, None) + .await + .map_err(|_| CtError::Unavailable)? + { + GetOutcome::NotFound => Ok(None), + GetOutcome::Found { value, revision } => Ok(Some((value, revision))), + } + } + async fn put(&self, key: &[u8], bytes: &[u8], revision: u64) -> Result<(), CtError> { + self.kv + .put_cas(self.store, self.group, key, bytes, revision) + .await + .map(|_| ()) + .map_err(|_| CtError::Unavailable) + } + pub(super) async fn step(&self, limit: u32) -> Result<LegacyRetirementProgress, CtError> { + if self.group == 0 || limit == 0 || limit > 64 { + return Err(CtError::InvalidArgument); + } + self.process + .capture(Instant::now()) + .map_err(|_| CtError::Unavailable)?; + let (mut state, revision) = if let Some((bytes, revision)) = self.get(NAMESPACE_KEY).await? { + (record::decode::<Namespace>(&bytes)?, revision) + } else { + self.put(NAMESPACE_KEY, &record::encode(&Namespace::default())?, 0) + .await?; + let (bytes, revision) = self.get(NAMESPACE_KEY).await?.ok_or(CtError::Unavailable)?; + (record::decode::<Namespace>(&bytes)?, revision) + }; + if state.version != 1 { + return Err(CtError::Corruption); + } + state.pending = None; + let mut progress = LegacyRetirementProgress::default(); + while progress.examined < limit && state.phase != Phase::Complete { + let page = self + .kv + .scan( + self.store, + self.group, + ROOT_PREFIX, + &state.after, + &[], + 1, + ReadMode::Linearizable, + None, + true, + None, + ) + .await + .map_err(|_| CtError::Unavailable)?; + if page.timed_out { + return Err(CtError::Unavailable); + } + let Some((key, _)) = page.items.into_iter().next() else { + state.phase = if state.phase == Phase::Proving { + Phase::Retiring + } else { + Phase::Complete + }; + state.after.clear(); + state.pack_index = 0; + continue; + }; + progress.examined += 1; + let finished = if state.phase == Phase::Proving { + self.prove_key(&key, &mut state).await? + } else { + self.retire_key(&key, &mut state, &mut progress).await? + }; + if state.pending.is_some() { + break; + } + if finished { + state.after = key.to_vec(); + state.pack_index = 0; + state.proof_after.clear(); + state.pins_done = false; + state.owners_done = false; + } + } + self.process + .capture(Instant::now()) + .map_err(|_| CtError::Unavailable)?; + self.put(NAMESPACE_KEY, &record::encode(&state)?, revision) + .await?; + progress.pending = state.pending; + progress.complete = state.phase == Phase::Complete; + Ok(progress) + } +} + +/// Run the production collector against protocol fixture seams. +/// # Errors +/// Returns the same proof, metadata and physical IO failures as the host entry. +#[cfg(feature = "test-util")] +pub async fn collect_legacy_storage_for_tests( + kv: Arc<PaxosKvClient>, + process: Arc<StreamProcessAuthority>, + allocator: Arc<dyn ChunkAllocator>, + disks: Arc<dyn DiskWriter>, + limit: u32, +) -> Result<LegacyRetirementProgress, CtError> { + Collector { + kv, + store: 0, + group: 1, + process, + allocator, + disks, + } + .step(limit) + .await +} + +/// Run the host's resume-only path without constructing native RPC routes. +/// # Errors +/// Returns the production resume result. +#[cfg(feature = "test-util")] +pub async fn resume_legacy_storage_for_tests( + kv: Arc<PaxosKvClient>, + process: Arc<StreamProcessAuthority>, + allocator: Arc<dyn ChunkAllocator>, + disks: Arc<dyn DiskWriter>, + limit: u32, +) -> Result<Option<LegacyRetirementProgress>, CtError> { + Collector { + kv, + store: 0, + group: 1, + process, + allocator, + disks, + } + .resume(limit) + .await +} diff --git a/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/allocation.rs b/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/allocation.rs new file mode 100644 index 000000000..6fb0f1621 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/allocation.rs @@ -0,0 +1,194 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{record, Collector, LegacyRetirementPending, LegacyRetirementProgress}; +use crowdb_protocol::{ + chunkdb::rpc::{Chunk, ChunkState, DeleteChunkRequest, QueryChunkRequest, Strip, StripType}, + common::ChunkId, +}; +use crowdb_tree_ffi::{chunk::legacy::LegacyManifest, CtError}; +use std::time::{Duration, Instant}; + +impl Collector { + pub(super) async fn retire_key( + &self, + key: &[u8], + state: &mut record::Namespace, + progress: &mut LegacyRetirementProgress, + ) -> Result<bool, CtError> { + let (tree, kind, generation) = record::parse_key(key)?; + if kind != b"legacy-migration" { + return Ok(true); + } + let (bytes, _) = self.get(key).await?.ok_or(CtError::Corruption)?; + let manifest = LegacyManifest::decode(&bytes)?; + if manifest.tree_id() != tree || manifest.generation() != generation { + return Err(CtError::Corruption); + } + let Some(pack) = manifest.packs().nth(state.pack_index) else { + return Ok(true); + }; + let id = ChunkId { + high: pack.chunk_high, + low: pack.chunk_low, + }; + let end = pack + .offset + .checked_add(u64::from(pack.length)) + .ok_or(CtError::Corruption)?; + self.retire_allocation(id, end, state, progress).await?; + if state.pending.is_none() { + state.pack_index += 1; + } + let finished = state.pack_index == manifest.packs().len(); + Ok(finished) + } + async fn query(&self, id: ChunkId) -> Result<Option<Chunk>, CtError> { + match self + .allocator + .query_chunk(QueryChunkRequest { chunk_id: Some(id) }) + .await + { + Ok(response) => response.chunk.map(Some).ok_or(CtError::Corruption), + Err(crowdb_chunk_client::IoError::ChunkNotFound(_)) => Ok(None), + Err(_) => Err(CtError::Unavailable), + } + } + async fn retire_allocation( + &self, + id: ChunkId, + end: u64, + state: &mut record::Namespace, + progress: &mut LegacyRetirementProgress, + ) -> Result<(), CtError> { + let key = record::allocation_key(id.high, id.low); + let (mut receipt, revision) = if let Some((bytes, revision)) = self.get(&key).await? { + (record::decode::<record::AllocationReceipt>(&bytes)?, revision) + } else { + let Some(chunk) = self.query(id).await? else { + state.pending = Some(LegacyRetirementPending::MissingAllocation { + high: id.high, + low: id.low, + }); + return Ok(()); + }; + if !matches_allocation(&chunk, id, end) { + state.pending = Some(LegacyRetirementPending::AllocationChanged { + high: id.high, + low: id.low, + }); + return Ok(()); + } + let receipt = record::AllocationReceipt { + version: 1, + physical_bytes: physical_bytes(&chunk)?, + allocation: chunk, + complete: false, + }; + self.put(&key, &record::encode(&receipt)?, 0).await?; + let (bytes, revision) = self.get(&key).await?.ok_or(CtError::Unavailable)?; + (record::decode::<record::AllocationReceipt>(&bytes)?, revision) + }; + if receipt.version != 1 || receipt.allocation.id != Some(id) { + return Err(CtError::Corruption); + } + if receipt.complete { + return Ok(()); + } + let Some(current) = self.query(id).await? else { + state.pending = Some(LegacyRetirementPending::MissingAllocation { + high: id.high, + low: id.low, + }); + return Ok(()); + }; + if current.id != Some(id) + || current.create_ts_ms != receipt.allocation.create_ts_ms + || current.chunk_type != receipt.allocation.chunk_type + { + state.pending = Some(LegacyRetirementPending::AllocationChanged { + high: id.high, + low: id.low, + }); + return Ok(()); + } + if !physically_deleted(¤t) { + if current.strips != receipt.allocation.strips || !current.cleanup_intents.is_empty() { + state.pending = Some(LegacyRetirementPending::AllocationChanged { + high: id.high, + low: id.low, + }); + return Ok(()); + } + self.drain(&receipt.allocation).await?; + let deleted = self + .allocator + .delete_chunk(DeleteChunkRequest { chunk_id: Some(id) }) + .await + .map_err(|_| CtError::Unavailable)? + .chunk + .ok_or(CtError::Corruption)?; + if deleted.id != Some(id) + || deleted.create_ts_ms != receipt.allocation.create_ts_ms + || !physically_deleted(&deleted) + { + return Err(CtError::Unavailable); + } + } + receipt.complete = true; + self.put(&key, &record::encode(&receipt)?, revision).await?; + progress.reclaimed_chunks += 1; + progress.reclaimed_bytes = progress + .reclaimed_bytes + .checked_add(receipt.physical_bytes) + .ok_or(CtError::Corruption)?; + Ok(()) + } + async fn drain(&self, chunk: &Chunk) -> Result<(), CtError> { + let deadline = Instant::now() + Duration::from_secs(10); + for strip in &chunk.strips { + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(CtError::Corruption); + }; + for segment in &mirror.segments { + self.disks + .drain_segment(segment, u64::from(strip.unit_kb) * 1024, deadline) + .await + .map_err(|_| CtError::Unavailable)?; + } + } + Ok(()) + } +} +fn physically_deleted(chunk: &Chunk) -> bool { + chunk.state == ChunkState::Deleted as i32 + && chunk.strips.is_empty() + && chunk.capacity == 0 + && chunk.cleanup_intents.is_empty() +} +fn matches_allocation(chunk: &Chunk, id: ChunkId, end: u64) -> bool { + chunk.id == Some(id) + && matches!(chunk.chunk_type, 2 | 3) + && u64::try_from(chunk.chunk_type).ok() == Some(id.high >> 56) + && chunk.create_ts_ms != 0 + && chunk.cleanup_intents.is_empty() + && (physically_deleted(chunk) || u64::from(chunk.capacity) >= end) +} +fn physical_bytes(chunk: &Chunk) -> Result<u64, CtError> { + let mut bytes = 0_u64; + for strip in &chunk.strips { + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(CtError::Corruption); + }; + if strip.strip_type != StripType::Mirror as i32 || strip.unit_kb == 0 || mirror.segments.is_empty() { + return Err(CtError::Corruption); + } + for segment in &mirror.segments { + let segment_bytes = u64::from(segment.unit_count) + .checked_mul(u64::from(strip.unit_kb)) + .and_then(|units| units.checked_mul(1024)) + .ok_or(CtError::Corruption)?; + bytes = bytes.checked_add(segment_bytes).ok_or(CtError::Corruption)?; + } + } + Ok(bytes) +} diff --git a/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/proof.rs b/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/proof.rs new file mode 100644 index 000000000..1d915a6f1 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/proof.rs @@ -0,0 +1,235 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{record, Collector, LegacyRetirementPending}; +use bytes::Bytes; +use crowdb_paxos_kv_client::{BatchOp, ReadMode, StreamProcessLeaseClient}; +use crowdb_protocol::chunk_stream::StreamProcessIdentity; +use crowdb_tree_ffi::{chunk::legacy::LegacyManifest, CtError}; +use std::time::{SystemTime, UNIX_EPOCH}; + +impl Collector { + pub(super) async fn prove_key(&self, key: &[u8], state: &mut record::Namespace) -> Result<bool, CtError> { + let (tree, kind, generation) = record::parse_key(key)?; + if kind == b"authority" { + self.freeze_tree(tree, state).await?; + } else if kind == b"manifest" || kind == b"legacy-migration" { + if let Some((bytes, _)) = self.get(key).await? { + if record::is_legacy(&bytes) { + let manifest = LegacyManifest::decode(&bytes)?; + if manifest.tree_id() != tree || manifest.generation() != generation { + return Err(CtError::Corruption); + } + self.save_receipt(tree, generation, bytes.clone()).await?; + if !state.owners_done && !self.prove_pack_owners(manifest, state).await? { + return Ok(false); + } + return self.prove_generation(tree, generation, state).await; + } + } + } + Ok(state.pending.is_none()) + } + + async fn prove_pack_owners( + &self, + manifest: LegacyManifest<'_>, + state: &mut record::Namespace, + ) -> Result<bool, CtError> { + let own_key = record::key(manifest.tree_id(), b"legacy-inventory", 0); + if self + .get(&own_key) + .await? + .map_or(true, |(value, _)| value.as_ref() != b"1") + { + // Still retain the original directory when old publication provenance is absent. + state.pending = Some(LegacyRetirementPending::MissingOwnerProof { + tree: manifest.tree_id(), + generation: manifest.generation(), + }); + return Ok(false); + } + for pack in manifest.packs().skip(state.pack_index).take(64) { + let key = record::key(pack.owner_tree_id, b"legacy-inventory", 0); + if self + .get(&key) + .await? + .map_or(true, |(value, _)| value.as_ref() != b"1") + { + state.pending = Some(LegacyRetirementPending::MissingOwnerProof { + tree: pack.owner_tree_id, + generation: manifest.generation(), + }); + return Ok(false); + } + state.pack_index += 1; + } + state.owners_done = state.pack_index == manifest.packs().len(); + Ok(state.owners_done) + } + + async fn freeze_tree(&self, tree: u64, state: &mut record::Namespace) -> Result<(), CtError> { + let authority_key = record::key(tree, b"authority", 0); + let (authority, revision) = self.get(&authority_key).await?.ok_or(CtError::Corruption)?; + if authority.len() != 16 { + return Err(CtError::Corruption); + } + if let Some((current, _)) = self.get(&record::key(tree, b"current", 0)).await? { + if record::is_legacy(¤t) { + LegacyManifest::decode(¤t)?; + state.pending = Some(LegacyRetirementPending::LegacyRoot { tree }); + return Ok(()); + } + } else if authority[8..] != 0_u64.to_be_bytes() { + return Err(CtError::Corruption); + } + let key = record::key(tree, b"legacy-frozen", 0); + if self.get(&key).await?.is_some() { + return Ok(()); + } + let ops = [ + BatchOp::Put { + key: Bytes::from(authority_key.clone()), + value: authority, + }, + BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from_static(b"1"), + }, + ]; + self.kv + .batch_write_cas(self.store, self.group, &ops, &authority_key, revision) + .await + .map(|_| ()) + .map_err(|_| CtError::Unavailable) + } + + async fn save_receipt(&self, tree: u64, generation: u64, manifest: Bytes) -> Result<(), CtError> { + let key = record::key(tree, b"legacy-migration", generation); + if let Some((old, _)) = self.get(&key).await? { + return if old == manifest { + Ok(()) + } else { + Err(CtError::Corruption) + }; + } + self.put(&key, &manifest, 0).await + } + + async fn prove_generation( + &self, + tree: u64, + generation: u64, + state: &mut record::Namespace, + ) -> Result<bool, CtError> { + if self + .get(&record::key(tree, b"legacy-tracked", generation)) + .await? + .map_or(true, |(bytes, _)| bytes.as_ref() != b"1") + || self + .get(&record::key(tree, b"legacy-untracked", generation)) + .await? + .is_some() + { + state.pending = Some(LegacyRetirementPending::MissingOwnerProof { tree, generation }); + return Ok(false); + } + let Some((current, _)) = self.get(&record::key(tree, b"current", 0)).await? else { + return Err(CtError::Corruption); + }; + record::typed_root(¤t, tree, generation)?; + if !state.pins_done && !self.check_pins(tree, generation, state).await? { + return Ok(false); + } + let mut prefix = record::key(tree, b"legacy-reader", generation); + prefix.push(b'/'); + let owners = self + .kv + .scan( + self.store, + self.group, + &prefix, + &state.proof_after, + &[], + 64, + ReadMode::Linearizable, + None, + false, + None, + ) + .await + .map_err(|_| CtError::Unavailable)?; + if owners.timed_out { + return Err(CtError::Unavailable); + } + if owners.items.is_empty() && state.proof_after.is_empty() { + state.pending = Some(LegacyRetirementPending::MissingOwnerProof { tree, generation }); + return Ok(false); + } + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| CtError::Unavailable)?; + let now = u64::try_from(now.as_millis()).map_err(|_| CtError::Unavailable)?; + let leases = StreamProcessLeaseClient::new(self.kv.clone()); + for (key, bytes) in owners.items { + let owner: StreamProcessIdentity = record::decode(&bytes)?; + if leases + .confirm_exclusion(owner, now) + .await + .map_err(|_| CtError::Unavailable)? + .is_none() + { + state.pending = Some(LegacyRetirementPending::LiveProcess { + tree, + generation, + instance: owner.instance_id, + epoch: owner.process_epoch, + }); + return Ok(false); + } + state.proof_after = key.to_vec(); + } + Ok(!owners.truncated) + } + async fn check_pins( + &self, + tree: u64, + generation: u64, + state: &mut record::Namespace, + ) -> Result<bool, CtError> { + let mut prefix = record::key(tree, b"pin", 0); + prefix.truncate(prefix.len() - 8); + let pins = self + .kv + .scan( + self.store, + self.group, + &prefix, + &state.proof_after, + &[], + 64, + ReadMode::Linearizable, + None, + false, + None, + ) + .await + .map_err(|_| CtError::Unavailable)?; + if pins.timed_out { + return Err(CtError::Unavailable); + } + for (key, value) in pins.items { + let pinned = u64::from_be_bytes(value.as_ref().try_into().map_err(|_| CtError::Corruption)?); + if pinned == generation { + state.pending = Some(LegacyRetirementPending::GenerationPin { tree, generation }); + return Ok(false); + } + state.proof_after = key.to_vec(); + } + if pins.truncated { + return Ok(false); + } + state.proof_after.clear(); + state.pins_done = true; + Ok(true) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/record.rs b/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/record.rs new file mode 100644 index 000000000..8481f07fc --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/legacy_retirement/record.rs @@ -0,0 +1,131 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::LegacyRetirementPending; +use crowdb_protocol::chunkdb::rpc::Chunk; +use crowdb_tree_ffi::CtError; +use serde::{Deserialize, Serialize}; + +pub(crate) const NAMESPACE_KEY: &[u8] = b"\0crowdb/chunk-kv/legacy-retirement/v1/state"; +pub(super) const ROOT_PREFIX: &[u8] = b"\0crowdb/chunk-kv/root/v1/"; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct Namespace { + pub version: u32, + pub phase: Phase, + pub after: Vec<u8>, + pub pack_index: usize, + pub proof_after: Vec<u8>, + pub pins_done: bool, + pub owners_done: bool, + pub pending: Option<LegacyRetirementPending>, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub(super) enum Phase { + Proving, + Retiring, + Complete, +} +impl Default for Namespace { + fn default() -> Self { + Self { + version: 1, + phase: Phase::Proving, + after: Vec::new(), + pack_index: 0, + proof_after: Vec::new(), + pins_done: false, + owners_done: false, + pending: None, + } + } +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct AllocationReceipt { + pub version: u32, + pub allocation: Chunk, + pub physical_bytes: u64, + pub complete: bool, +} + +pub(crate) fn is_legacy(bytes: &[u8]) -> bool { + bytes.get(..4) == Some(&7_u32.to_le_bytes()) +} +pub(super) fn key(tree: u64, kind: &[u8], generation: u64) -> Vec<u8> { + let mut key = ROOT_PREFIX.to_vec(); + key.extend_from_slice(&tree.to_be_bytes()); + key.push(b'/'); + key.extend_from_slice(kind); + key.push(b'/'); + key.extend_from_slice(&generation.to_be_bytes()); + key +} +pub(super) fn parse_key(bytes: &[u8]) -> Result<(u64, &[u8], u64), CtError> { + let tail = bytes.strip_prefix(ROOT_PREFIX).ok_or(CtError::Corruption)?; + if tail.len() < 18 || tail[8] != b'/' { + return Err(CtError::Corruption); + } + let end = tail[9..] + .iter() + .position(|byte| *byte == b'/') + .ok_or(CtError::Corruption)? + + 9; + if tail.len() < end + 9 { + return Err(CtError::Corruption); + } + Ok(( + u64::from_be_bytes(tail[..8].try_into().map_err(|_| CtError::Corruption)?), + &tail[9..end], + u64::from_be_bytes( + tail[end + 1..end + 9] + .try_into() + .map_err(|_| CtError::Corruption)?, + ), + )) +} +pub(super) fn allocation_key(high: u64, low: u64) -> Vec<u8> { + let mut key = b"\0crowdb/chunk-kv/legacy-retirement/v1/allocation/".to_vec(); + key.extend_from_slice(&high.to_be_bytes()); + key.extend_from_slice(&low.to_be_bytes()); + key +} +pub(super) fn encode<T: Serialize>(value: &T) -> Result<Vec<u8>, CtError> { + serde_json::to_vec(value).map_err(|_| CtError::Internal) +} +pub(super) fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, CtError> { + serde_json::from_slice(bytes).map_err(|_| CtError::Corruption) +} + +// This checks the immutable format envelope. The production recovery decoder +// additionally validates every directory entry before opening a typed root. +pub(super) fn typed_root(bytes: &[u8], tree: u64, generation: u64) -> Result<(), CtError> { + const HEADER: usize = 128; + const ENTRY: usize = 96; + if bytes.len() < HEADER + 4 { + return Err(CtError::Corruption); + } + let word = |index: usize| { + u64::from_le_bytes( + bytes[index * 8..index * 8 + 8] + .try_into() + .expect("header checked"), + ) + }; + let crc = u32::from_le_bytes( + bytes[bytes.len() - 4..] + .try_into() + .map_err(|_| CtError::Corruption)?, + ); + let entries = usize::try_from(word(13)).map_err(|_| CtError::Corruption)?; + if word(0) != 0x3354_4F4F_524B_4843 + || word(1) != 1 + || word(2) != tree + || word(3) <= generation + || entries != (bytes.len() - HEADER - 4) / ENTRY + || (bytes.len() - HEADER - 4) % ENTRY != 0 + || crowdb_tree_ffi::crc32c(&bytes[..bytes.len() - 4]) != crc + { + return Err(CtError::Corruption); + } + Ok(()) +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs index be867266b..c00820d0b 100644 --- a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs @@ -7,22 +7,35 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use arc_swap::ArcSwap; +use arc_swap::{ArcSwap, ArcSwapOption}; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome, PaxosKvClient, ReadMode}; +use crowdb_tree_ffi::chunk::stage::{StageId, StageRecord}; use crowdb_tree_ffi::{RootCatalogObject, RootCatalogStore}; +mod legacy_migration; +mod legacy_tracking; +mod prepared; +mod protection; +mod retention; +mod staging; +mod transaction; + use super::StorageRuntimeError; use crate::{ChunkKvRangeCatalogPublisher, Group0ControlStore}; use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState}; pub(super) struct KvRootCatalogStore { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, runtime: tokio::runtime::Handle, store_id: u64, group_id: u64, tree_id: u64, owner_epoch: u64, + process: Option<Arc<crowdb_paxos_kv_client::StreamProcessAuthority>>, + writer_incarnation: u64, + writer_quiesced: AtomicBool, + recovered: ArcSwapOption<protection::RecoveryReceipt>, publication_authorized: AtomicBool, prepared_assignment: Option<ChunkKvRangeCatalogEntry>, published_objects: ArcSwap<HashMap<Vec<u8>, Vec<u8>>>, @@ -30,7 +43,7 @@ pub(super) struct KvRootCatalogStore { impl KvRootCatalogStore { pub(super) async fn claim( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, store_id: u64, group_id: u64, tree_id: u64, @@ -74,7 +87,9 @@ impl KvRootCatalogStore { .await { Ok(_) => break, - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} + Err( + crowdb_paxos_kv_client::Error::CasFailed { .. } | crowdb_paxos_kv_client::Error::CasBusy, + ) => {} Err(error) => return Err(StorageRuntimeError::Tree(error.to_string())), } } @@ -85,6 +100,10 @@ impl KvRootCatalogStore { group_id, tree_id, owner_epoch, + process: None, + writer_incarnation: 0, + writer_quiesced: AtomicBool::new(false), + recovered: ArcSwapOption::empty(), publication_authorized: AtomicBool::new(true), prepared_assignment: None, published_objects: ArcSwap::from_pointee(HashMap::new()), @@ -92,7 +111,7 @@ impl KvRootCatalogStore { } pub(super) async fn for_assignment( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, store_id: u64, group_id: u64, entry: &ChunkKvRangeCatalogEntry, @@ -112,7 +131,7 @@ impl KvRootCatalogStore { } pub(super) fn prepare( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, store_id: u64, group_id: u64, entry: &ChunkKvRangeCatalogEntry, @@ -127,6 +146,10 @@ impl KvRootCatalogStore { group_id, tree_id: entry.artifact.tree_id, owner_epoch: entry.owner_epoch, + process: None, + writer_incarnation: 0, + writer_quiesced: AtomicBool::new(false), + recovered: ArcSwapOption::empty(), publication_authorized: AtomicBool::new(false), prepared_assignment: Some(entry.clone()), published_objects: ArcSwap::from_pointee(HashMap::new()), @@ -136,7 +159,9 @@ impl KvRootCatalogStore { async fn authorize_publication(&self) -> Result<(), crowdb_tree_ffi::CtError> { self.claim_published(false) .await - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable) + .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)?; + self.register_writer().await?; + self.persist_completed_recovery().await } pub(super) async fn claim_published(&self, allow_catching_up: bool) -> Result<(), StorageRuntimeError> { @@ -194,7 +219,7 @@ impl KvRootCatalogStore { } } - async fn get(&self, key: &[u8]) -> Result<Option<(Bytes, u64)>, crowdb_kv_client::Error> { + async fn get(&self, key: &[u8]) -> Result<Option<(Bytes, u64)>, crowdb_paxos_kv_client::Error> { self.kv .get(self.store_id, self.group_id, key, ReadMode::Linearizable, None) .await @@ -214,17 +239,62 @@ impl KvRootCatalogStore { } impl RootCatalogStore for KvRootCatalogStore { + fn prepare_legacy_migration( + &self, + tree: u64, + generation: u64, + manifest: &[u8], + ) -> Result<(), crowdb_tree_ffi::CtError> { + self.retain_legacy_migration(tree, generation, manifest) + } + fn load_legacy_migration( + &self, + tree: u64, + generation: u64, + ) -> Result<Option<Vec<u8>>, crowdb_tree_ffi::CtError> { + self.load_migration_receipt(tree, generation) + } + fn stage_protection(&self) -> Option<&dyn crowdb_tree_ffi::chunk::protection::StageProtectionCatalog> { + Some(self) + } + fn supports_staging(&self) -> bool { + true + } + fn load_stage(&self, tree_id: u64, id: StageId) -> Result<Option<StageRecord>, crowdb_tree_ffi::CtError> { + self.stage_load(tree_id, id) + } + fn compare_stage( + &self, + tree_id: u64, + owner_epoch: u64, + expected: Option<&StageRecord>, + next: Option<&StageRecord>, + ) -> Result<(), crowdb_tree_ffi::CtError> { + self.stage_compare(tree_id, owner_epoch, expected, next) + } + fn list_stages( + &self, + tree_id: u64, + after: Option<StageId>, + limit: u32, + ) -> Result<Vec<StageRecord>, crowdb_tree_ffi::CtError> { + self.stage_list(tree_id, after, limit) + } fn load( &self, tree_id: u64, object: RootCatalogObject, ) -> Result<Option<Vec<u8>>, crowdb_tree_ffi::CtError> { - if tree_id != self.tree_id { + if tree_id != self.tree_id && !matches!(object, RootCatalogObject::ReferenceSegment(_)) { return Err(crowdb_tree_ffi::CtError::InvalidArgument); } let key = object_key(tree_id, object); - if let Some(value) = self.published_objects.load().get(&key) { - return Ok(Some(value.clone())); + // Exact generations can be reclaimed by another catalog instance. + // A cached descriptor must not resurrect an absent durable root. + if matches!(object, RootCatalogObject::ReferenceSegment(_)) { + if let Some(value) = self.published_objects.load().get(&key) { + return Ok(Some(value.clone())); + } } let value = self .wait(self.get(&key)) @@ -242,6 +312,11 @@ impl RootCatalogStore for KvRootCatalogStore { } } } + if let Some((bytes, _)) = &value { + if !matches!(object, RootCatalogObject::ReferenceSegment(_)) { + self.wait(self.track_legacy_reader(bytes))?; + } + } Ok(value.map(|(bytes, _)| bytes.to_vec())) } @@ -282,10 +357,11 @@ impl RootCatalogStore for KvRootCatalogStore { else { return Err(crowdb_tree_ffi::CtError::Unavailable); }; + self.wait(self.check_registered_writer())?; if decode_authority(&authority) != Some((owner_epoch, expected_generation)) { return Err(crowdb_tree_ffi::CtError::Unavailable); } - let ops = [ + let mut ops = vec![ BatchOp::Put { key: Bytes::from(authority_key.clone()), value: Bytes::copy_from_slice(&encode_authority(owner_epoch, generation)), @@ -299,6 +375,7 @@ impl RootCatalogStore for KvRootCatalogStore { value: Bytes::copy_from_slice(manifest), }, ]; + ops.extend(self.wait(self.legacy_publication_ops(generation, manifest))?); self.wait( self.kv .batch_write_cas(self.store_id, self.group_id, &ops, &authority_key, revision), @@ -337,7 +414,9 @@ impl RootCatalogStore for KvRootCatalogStore { revision, )) { Ok(_) => return Ok(current), - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} + Err( + crowdb_paxos_kv_client::Error::CasFailed { .. } | crowdb_paxos_kv_client::Error::CasBusy, + ) => {} Err(_) => return Err(crowdb_tree_ffi::CtError::Unavailable), } } @@ -358,67 +437,8 @@ impl RootCatalogStore for KvRootCatalogStore { } fn reclaim_before(&self, tree_id: u64, generation: u64) -> u64 { - if tree_id != self.tree_id || generation <= 2 { - return 0; - } - let pin_prefix = catalog_pin_prefix(tree_id); - let pinned_generation = match self.wait(self.kv.scan( - self.store_id, - self.group_id, - &pin_prefix, - &[], - &[], - 0, - ReadMode::Linearizable, - None, - false, - None, - )) { - Ok(outcome) => { - let mut oldest = None; - for (_, value) in outcome.items { - let Some(pin) = decode_u64(&value) else { - return 0; - }; - oldest = Some(oldest.map_or(pin, |current: u64| current.min(pin))); - } - oldest - } - Err(_) => return 0, - }; - let generation = pinned_generation.map_or(generation, |pin| generation.min(pin)); - if generation <= 2 { - return 0; - } - let floor_key = catalog_key(tree_id, b"reclaim-floor", 0); - let (floor, revision) = match self.wait(self.get(&floor_key)) { - Ok(Some((value, revision))) => match decode_u64(&value) { - Some(floor) => (floor, revision), - None => return 0, - }, - Ok(None) => (1, 0), - Err(_) => return 0, - }; - let end = generation.saturating_sub(1).min(floor.saturating_add(128)); - if floor >= end { - return 0; - } - let mut ops = (floor..end) - .map(|old_generation| BatchOp::Delete { - key: Bytes::from(catalog_key(tree_id, b"manifest", old_generation)), - }) - .collect::<Vec<_>>(); - ops.push(BatchOp::Put { - key: Bytes::from(floor_key.clone()), - value: Bytes::copy_from_slice(&end.to_be_bytes()), - }); - self.wait( - self.kv - .batch_write_cas(self.store_id, self.group_id, &ops, &floor_key, revision), - ) - .map_or(0, |_| end - floor) + self.reclaim_generations(tree_id, generation) } - fn pin_generation( &self, tree_id: u64, @@ -426,53 +446,15 @@ impl RootCatalogStore for KvRootCatalogStore { transition_low: u64, generation: u64, ) -> Result<(), crowdb_tree_ffi::CtError> { - if tree_id != self.tree_id || (transition_high == 0 && transition_low == 0) || generation == 0 { - return Err(crowdb_tree_ffi::CtError::InvalidArgument); - } - if self - .wait(self.get(&catalog_key(tree_id, b"manifest", generation))) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? - .is_none() - { - return Err(crowdb_tree_ffi::CtError::NotFound); - } - let key = catalog_pin_key(tree_id, transition_high, transition_low); - for _ in 0..3 { - if let Some((value, _)) = self - .wait(self.get(&key)) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable)? - { - return if decode_u64(&value) == Some(generation) { - Ok(()) - } else { - Err(crowdb_tree_ffi::CtError::InvalidArgument) - }; - } - match self.wait( - self.kv - .put_cas(self.store_id, self.group_id, &key, &generation.to_be_bytes(), 0), - ) { - Ok(_) => return Ok(()), - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => {} - Err(_) => return Err(crowdb_tree_ffi::CtError::Unavailable), - } - } - Err(crowdb_tree_ffi::CtError::Unavailable) + self.retain_generation(tree_id, transition_high, transition_low, generation) } - fn unpin_generation( &self, tree_id: u64, transition_high: u64, transition_low: u64, ) -> Result<(), crowdb_tree_ffi::CtError> { - if tree_id != self.tree_id || (transition_high == 0 && transition_low == 0) { - return Err(crowdb_tree_ffi::CtError::InvalidArgument); - } - let key = catalog_pin_key(tree_id, transition_high, transition_low); - self.wait(self.kv.delete(self.store_id, self.group_id, &key, None)) - .map(|_| ()) - .map_err(|_| crowdb_tree_ffi::CtError::Unavailable) + self.release_generation(tree_id, transition_high, transition_low) } } diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/legacy_migration.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/legacy_migration.rs new file mode 100644 index 000000000..7ce29d486 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/legacy_migration.rs @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Immutable pre-CAS migration receipts also retain the legacy pack directory. +use super::{catalog_key, decode_authority, KvRootCatalogStore}; +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_tree_ffi::{chunk::legacy::LegacyManifest, CtError}; + +impl KvRootCatalogStore { + pub(super) fn retain_legacy_migration( + &self, + tree: u64, + generation: u64, + manifest: &[u8], + ) -> Result<(), CtError> { + let original = LegacyManifest::decode(manifest)?; + if tree != self.tree_id + || original.tree_id() != tree + || original.generation() != generation + || original.owner_epoch() > self.owner_epoch + { + return Err(CtError::InvalidArgument); + } + self.wait(self.authority_mutation(|| async { + let authority_key = catalog_key(tree, b"authority", 0); + let (authority, revision) = self + .get(&authority_key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)?; + self.check_registered_writer().await?; + let (epoch, current_generation) = decode_authority(&authority).ok_or(CtError::Corruption)?; + if epoch != self.owner_epoch || current_generation < generation { + return Err(CtError::Unavailable.into()); + } + let key = catalog_key(tree, b"legacy-migration", generation); + if let Some((existing, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? { + return if existing.as_ref() == manifest { + Ok(()) + } else { + Err(CtError::Corruption.into()) + }; + } + let current = self + .get(&catalog_key(tree, b"current", 0)) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)? + .0; + if current_generation != generation || current.as_ref() != manifest { + return Err(CtError::Unavailable.into()); + } + let operations = [ + BatchOp::Put { + key: Bytes::from(authority_key.clone()), + value: authority, + }, + BatchOp::Put { + key: Bytes::from(key), + value: Bytes::copy_from_slice(manifest), + }, + ]; + self.kv + .batch_write_cas( + self.store_id, + self.group_id, + &operations, + &authority_key, + revision, + ) + .await + .map(|_| ()) + .map_err(|error| super::transaction::MutationError::from_cas(&error)) + })) + } + pub(super) fn load_migration_receipt( + &self, + tree: u64, + generation: u64, + ) -> Result<Option<Vec<u8>>, CtError> { + if tree != self.tree_id || generation == 0 { + return Err(CtError::InvalidArgument); + } + self.wait(self.get(&catalog_key(tree, b"legacy-migration", generation))) + .map(|value| value.map(|(bytes, _)| bytes.to_vec())) + .map_err(|_| CtError::Unavailable) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/legacy_tracking.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/legacy_tracking.rs new file mode 100644 index 000000000..4c47f1b10 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/legacy_tracking.rs @@ -0,0 +1,134 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Generation provenance is committed with publication; late readers share its authority. +use super::{catalog_key, KvRootCatalogStore}; +use crate::storage::legacy_retirement::record::{is_legacy, NAMESPACE_KEY}; +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_tree_ffi::{chunk::legacy::LegacyManifest, CtError}; + +impl KvRootCatalogStore { + fn legacy_owner_op(&self, generation: u64) -> Result<BatchOp, CtError> { + let Some(owner) = self.process_identity()? else { + return Ok(BatchOp::Put { + key: Bytes::from(catalog_key(self.tree_id, b"legacy-untracked", generation)), + value: Bytes::from_static(b"1"), + }); + }; + self.check_process_owner()?; + let mut key = catalog_key(self.tree_id, b"legacy-reader", generation); + key.push(b'/'); + key.extend_from_slice(&owner.instance_id.to_be_bytes()); + key.extend_from_slice(&owner.process_epoch.to_be_bytes()); + key.extend_from_slice(&owner.boot_nonce); + Ok(BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from(serde_json::to_vec(&owner).map_err(|_| CtError::Internal)?), + }) + } + + pub(super) async fn legacy_publication_ops( + &self, + generation: u64, + bytes: &[u8], + ) -> Result<Vec<BatchOp>, CtError> { + if !is_legacy(bytes) { + return Ok(Vec::new()); + } + let manifest = LegacyManifest::decode(bytes)?; + if manifest.tree_id() != self.tree_id || manifest.generation() != generation { + return Err(CtError::InvalidArgument); + } + if self + .get(NAMESPACE_KEY) + .await + .map_err(|_| CtError::Unavailable)? + .is_some() + { + return Err(CtError::NotSupported); + } + let mut ops = vec![ + BatchOp::Put { + key: Bytes::from(catalog_key(self.tree_id, b"legacy-tracked", generation)), + value: Bytes::from_static(b"1"), + }, + self.legacy_owner_op(generation)?, + ]; + // Only the first publication can prove this lineage has no untracked history. + if generation == 1 && self.process_identity()?.is_some() { + ops.push(BatchOp::Put { + key: Bytes::from(catalog_key(self.tree_id, b"legacy-inventory", 0)), + value: Bytes::from_static(b"1"), + }); + } + Ok(ops) + } + + pub(super) async fn track_legacy_reader(&self, bytes: &[u8]) -> Result<(), CtError> { + if !is_legacy(bytes) { + return Ok(()); + } + let generation = LegacyManifest::decode(bytes)?.generation(); + self.authority_mutation(|| async { + let authority = self.protection_authority(false).await?; + if self + .get(&catalog_key(self.tree_id, b"legacy-frozen", 0)) + .await + .map_err(|_| CtError::Unavailable)? + .is_some() + { + return Err(CtError::Unavailable.into()); + } + let op = self.legacy_owner_op(generation)?; + self.protection_write(authority, vec![op]).await + }) + .await + } + + pub(super) async fn check_legacy_pin_admission(&self, generation: u64) -> Result<(), CtError> { + let Some((manifest, _)) = self + .get(&catalog_key(self.tree_id, b"manifest", generation)) + .await + .map_err(|_| CtError::Unavailable)? + else { + return Ok(()); + }; + if is_legacy(&manifest) + && self + .get(&catalog_key(self.tree_id, b"legacy-frozen", 0)) + .await + .map_err(|_| CtError::Unavailable)? + .is_some() + { + return Err(CtError::Unavailable); + } + Ok(()) + } + + pub(super) async fn legacy_history_ops(&self, generation: u64) -> Result<Vec<BatchOp>, CtError> { + let Some((manifest, _)) = self + .get(&catalog_key(self.tree_id, b"manifest", generation)) + .await + .map_err(|_| CtError::Unavailable)? + else { + return Ok(Vec::new()); + }; + if !is_legacy(&manifest) { + return Ok(Vec::new()); + } + LegacyManifest::decode(&manifest)?; + let key = catalog_key(self.tree_id, b"legacy-migration", generation); + if let Some((old, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? { + return if old == manifest { + Ok(Vec::new()) + } else { + Err(CtError::Corruption) + }; + } + Ok(vec![BatchOp::Put { + key: Bytes::from(key), + value: manifest, + }]) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/prepared.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/prepared.rs new file mode 100644 index 000000000..7da9c8124 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/prepared.rs @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Preparation can stage private packs while the published source still writes. + +use super::{same_recovery_base, KvRootCatalogStore}; +use crate::Group0ControlStore; +use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogEntry, TransferPhase}; +use crowdb_tree_ffi::CtError; +use std::sync::atomic::Ordering; + +impl KvRootCatalogStore { + pub(super) async fn check_prepared_staging(&self, authority_epoch: u64) -> Result<(), CtError> { + self.check_process_owner()?; + let prepared = self.prepared_assignment.as_ref().ok_or(CtError::Unavailable)?; + let identity = self + .process + .as_ref() + .ok_or(CtError::Unavailable)? + .status(std::time::Instant::now()) + .lease + .identity; + let id = prepared.transition_id.ok_or(CtError::Unavailable)?; + let control = Group0ControlStore::from_client(self.kv.clone()); + let (transition, _) = control + .load_transfer_transition(id) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)?; + let allowed = matches!( + transition.phase, + TransferPhase::TargetPreparing + | TransferPhase::TargetPrepared + | TransferPhase::AwaitingFence + | TransferPhase::TargetCatchingUp + | TransferPhase::CatchupPublished + | TransferPhase::TargetReady + ); + let current = ChunkKvRangeCatalogEntry { + partition_id: transition.partition_id, + range: transition.range, + owner: transition.target, + owner_epoch: transition.target_epoch, + state: prepared.state, + artifact: transition.target_artifact, + transition_id: Some(transition.transition_id), + }; + if !allowed + || identity.instance_id != current.owner.instance_id + || (authority_epoch != transition.source_epoch && authority_epoch != self.owner_epoch) + || current.partition_id != prepared.partition_id + || current.range != prepared.range + || current.owner != prepared.owner + || current.owner_epoch != self.owner_epoch + || current.artifact.tree_id != self.tree_id + || current.artifact.stream_name != prepared.artifact.stream_name + || !same_recovery_base(¤t, prepared) + { + return Err(CtError::Unavailable); + } + Ok(()) + } + + pub(super) async fn authorize_staging(&self) -> Result<(), CtError> { + if self.publication_authorized.load(Ordering::Acquire) { + return self.authorize_publication().await; + } + // Only the existing exact published-owner proof may advance root epoch. + // Otherwise a current durable preparation intent permits private packs. + if self.claim_published(false).await.is_ok() { + self.register_writer().await?; + return self.persist_completed_recovery().await; + } + let authority = self.protection_authority(false).await?; + self.check_prepared_staging(authority.epoch).await?; + self.register_writer().await + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection.rs new file mode 100644 index 000000000..4fa0071bd --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection.rs @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Reference intents, live pins and GC use one root authority revision. + +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_tree_ffi::chunk::protection::{ + validate_operation, CandidateStatus, ManifestCandidate, StageProtectionCatalog, +}; +use crowdb_tree_ffi::chunk::stage::StageId; +use crowdb_tree_ffi::CtError; + +use super::{catalog_key, decode_authority, KvRootCatalogStore}; +mod candidate; +mod commit; +mod guards; +mod operation; +mod pin_owner; +mod process; +mod prune; +mod record; +mod release; +mod scan; +mod writer; +pub(super) use process::RecoveryReceipt; +use record::{operation_key, CandidateRecord}; + +// Native metadata proposals reserve progress for at most 64 distinct keys. +pub(super) const MAX_METADATA_MUTATIONS: usize = 64; + +pub(super) struct Authority { + key: Vec<u8>, + value: Bytes, + revision: u64, + pub(super) epoch: u64, + generation: u64, +} + +impl KvRootCatalogStore { + pub(super) async fn protection_authority(&self, require_owner: bool) -> Result<Authority, CtError> { + let key = catalog_key(self.tree_id, b"authority", 0); + let (value, revision) = self + .get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)?; + let (epoch, generation) = decode_authority(&value).ok_or(CtError::Corruption)?; + if require_owner { + self.check_registered_writer().await?; + } + if require_owner && epoch != self.owner_epoch { + return Err(CtError::Unavailable); + } + Ok(Authority { + key, + value, + revision, + epoch, + generation, + }) + } + + pub(super) async fn protection_write( + &self, + authority: Authority, + mut ops: Vec<BatchOp>, + ) -> super::transaction::MutationResult<()> { + ops.push(BatchOp::Put { + key: Bytes::from(authority.key.clone()), + value: authority.value, + }); + self.kv + .batch_write_cas( + self.store_id, + self.group_id, + &ops, + &authority.key, + authority.revision, + ) + .await + .map(|_| ()) + .map_err(|error| super::transaction::MutationError::from_cas(&error)) + } + + fn protection_identity(&self, tree_id: u64, id: StageId, owner: bool) -> Result<(), CtError> { + validate_operation(id)?; + if tree_id != self.tree_id { + return Err(CtError::InvalidArgument); + } + if owner && id.owner_epoch != self.owner_epoch { + return Err(CtError::Unavailable); + } + Ok(()) + } + + async fn candidate_record(&self, id: StageId) -> Result<Option<CandidateRecord>, CtError> { + let key = operation_key(self.tree_id, b"candidate", id); + self.get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .map(|(bytes, _)| CandidateRecord::decode(id, &bytes)) + .transpose() + } +} + +impl StageProtectionCatalog for KvRootCatalogStore { + fn recover_manifest_candidate(&self, tree: u64, id: StageId) -> Result<(), CtError> { + self.protection_identity(tree, id, false)?; + self.wait(self.recover_candidate(id)) + } + fn writer_quiesced(&self, tree: u64, epoch: u64) -> Result<(), CtError> { + if tree != self.tree_id || epoch != self.owner_epoch { + return Err(CtError::InvalidArgument); + } + self.wait(self.publish_writer_quiescence()) + } + fn check_read_owner(&self) -> Result<(), CtError> { + self.check_process_owner() + } + fn prune_stage_pins( + &self, + tree: u64, + stage: StageId, + after: Option<StageId>, + limit: u32, + ) -> Result<crowdb_tree_ffi::chunk::protection::PinPruneProgress, CtError> { + self.protection_identity(tree, stage, false)?; + self.wait(self.prune_pins(stage, after, limit)) + } + + fn load_manifest_candidate(&self, tree: u64, id: StageId) -> Result<Option<CandidateStatus>, CtError> { + self.protection_identity(tree, id, false)?; + self.wait(self.candidate_record(id)) + .map(|record| record.map(|record| record.status)) + } + fn list_manifest_candidates( + &self, + tree: u64, + after: Option<StageId>, + limit: u32, + ) -> Result<Vec<CandidateStatus>, CtError> { + if tree != self.tree_id { + return Err(CtError::InvalidArgument); + } + self.wait(self.scan_candidates(after, limit)) + } + fn begin_manifest_candidate(&self, tree: u64, candidate: &ManifestCandidate) -> Result<(), CtError> { + candidate.validate()?; + self.protection_identity(tree, candidate.id, true)?; + self.wait(self.authorize_publication())?; + self.wait(self.begin_candidate(candidate)) + } + fn prepare_manifest_references( + &self, + tree: u64, + id: StageId, + window: u64, + refs: &[StageId], + ) -> Result<(), CtError> { + self.protection_identity(tree, id, true)?; + self.wait(self.prepare_candidate(id, window, refs)) + } + fn commit_manifest_candidate(&self, tree: u64, id: StageId) -> Result<(), CtError> { + self.protection_identity(tree, id, true)?; + self.wait(self.commit_candidate(id)) + } + fn abort_manifest_candidate(&self, tree: u64, id: StageId) -> Result<(), CtError> { + self.protection_identity(tree, id, true)?; + self.wait(self.abort_candidate(id)) + } + fn release_manifest_references(&self, tree: u64, id: StageId, window: u64) -> Result<(), CtError> { + self.protection_identity(tree, id, false)?; + self.wait(self.release_candidate(id, window)) + } + fn pin_stage(&self, tree: u64, stage: StageId, pin: StageId) -> Result<(), CtError> { + self.protection_identity(tree, stage, false)?; + self.protection_identity(tree, pin, true)?; + self.wait(self.change_stage_pin(stage, pin, true)) + } + fn unpin_stage(&self, tree: u64, stage: StageId, pin: StageId) -> Result<(), CtError> { + self.protection_identity(tree, stage, false)?; + self.protection_identity(tree, pin, true)?; + self.wait(self.change_stage_pin(stage, pin, false)) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/candidate.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/candidate.rs new file mode 100644 index 000000000..8be2c423e --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/candidate.rs @@ -0,0 +1,158 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::record::{encode_references, operation_key, window_key, CandidateRecord}; +use super::KvRootCatalogStore; +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_tree_ffi::chunk::protection::{ + reference_checksum, validate_operation, CandidateState, CandidateStatus, ManifestCandidate, + MAX_REFERENCE_WINDOW, +}; +use crowdb_tree_ffi::chunk::stage::StageId; +use crowdb_tree_ffi::{crc32c, CtError}; + +impl KvRootCatalogStore { + pub(super) async fn begin_candidate(&self, candidate: &ManifestCandidate) -> Result<(), CtError> { + self.authority_mutation(|| async { + let authority = self.protection_authority(true).await?; + let manifest_key = operation_key(self.tree_id, b"candidate-manifest", candidate.id); + if let Some(current) = self.candidate_record(candidate.id).await? { + let manifest = self.get(&manifest_key).await.map_err(|_| CtError::Unavailable)?; + if current.status.generation == candidate.expected_generation + 1 + && current.status.reference_count == candidate.reference_count + && current.expected_checksum == candidate.reference_checksum + && current.manifest_bytes == candidate.manifest.len() as u64 + && current.manifest_checksum == crc32c(&candidate.manifest) + && (manifest.as_ref().map(|(bytes, _)| bytes.as_ref()) + == Some(candidate.manifest.as_slice()) + || (manifest.is_none() + && matches!( + current.status.state, + CandidateState::Aborted | CandidateState::Reclaimed + ))) + { + return Ok(()); + } + return Err(CtError::InvalidArgument.into()); + } + if authority.generation != candidate.expected_generation { + return Err(CtError::Unavailable.into()); + } + let record = CandidateRecord { + status: CandidateStatus { + id: candidate.id, + state: if candidate.reference_count == 0 { + CandidateState::Ready + } else { + CandidateState::Preparing + }, + generation: candidate.expected_generation + 1, + reference_count: candidate.reference_count, + prepared_references: 0, + prepared_windows: 0, + released_windows: 0, + }, + expected_checksum: candidate.reference_checksum, + prepared_checksum: 0, + manifest_checksum: crc32c(&candidate.manifest), + manifest_bytes: candidate.manifest.len() as u64, + last_stage: StageId::default(), + }; + let mut ops = vec![ + BatchOp::Put { + key: Bytes::from(operation_key(self.tree_id, b"candidate", candidate.id)), + value: Bytes::from(record.encode()), + }, + BatchOp::Put { + key: Bytes::from(manifest_key), + value: Bytes::copy_from_slice(&candidate.manifest), + }, + ]; + if let Some(owner) = self.writer_owner()? { + ops.push(BatchOp::Put { + key: Bytes::from(operation_key(self.tree_id, b"candidate-owner", candidate.id)), + value: Bytes::from(owner.encode()), + }); + } + self.protection_write(authority, ops).await + }) + .await + } + + pub(super) async fn prepare_candidate( + &self, + id: StageId, + window: u64, + refs: &[StageId], + ) -> Result<(), CtError> { + self.authority_mutation(|| async { + if refs.is_empty() + || refs.len() > MAX_REFERENCE_WINDOW + || refs.windows(2).any(|pair| pair[0] >= pair[1]) + { + return Err(CtError::InvalidArgument.into()); + } + for reference in refs { + validate_operation(*reference)?; + } + let authority = self.protection_authority(true).await?; + let key = window_key(self.tree_id, id, window); + let encoded = encode_references(refs); + if let Some((current, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? { + return if current.as_ref() == encoded { + Ok(()) + } else { + Err(CtError::InvalidArgument.into()) + }; + } + let mut record = self.candidate_record(id).await?.ok_or(CtError::NotFound)?; + if record.status.state != CandidateState::Preparing + || window != record.status.prepared_windows + || refs[0] <= record.last_stage + || authority.generation.checked_add(1) != Some(record.status.generation) + { + return Err(CtError::Unavailable.into()); + } + let total = record + .status + .prepared_references + .checked_add(refs.len() as u64) + .ok_or(CtError::ResourceExhausted)?; + if total > record.status.reference_count { + return Err(CtError::InvalidArgument.into()); + } + let checksum = reference_checksum(record.prepared_checksum, refs); + if total == record.status.reference_count && checksum != record.expected_checksum { + return Err(CtError::Corruption.into()); + } + let mut ops = Vec::with_capacity(refs.len() + 3); + for reference in refs { + self.readable_stage(*reference).await?; + let mut guard = self.load_stage_guard(*reference).await?; + guard.references = guard + .references + .checked_add(1) + .ok_or(CtError::ResourceExhausted)?; + ops.push(self.stage_guard_put(*reference, guard)); + } + record.status.prepared_references = total; + record.status.prepared_windows += 1; + record.prepared_checksum = checksum; + record.last_stage = *refs.last().ok_or(CtError::InvalidArgument)?; + if total == record.status.reference_count { + record.status.state = CandidateState::Ready; + } + ops.push(BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from(encoded), + }); + ops.push(BatchOp::Put { + key: Bytes::from(operation_key(self.tree_id, b"candidate", id)), + value: Bytes::from(record.encode()), + }); + self.protection_write(authority, ops).await + }) + .await + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/commit.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/commit.rs new file mode 100644 index 000000000..d5cbd2c37 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/commit.rs @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::record::{encode_words, operation_key}; +use super::{catalog_key, KvRootCatalogStore}; +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_tree_ffi::chunk::protection::CandidateState; +use crowdb_tree_ffi::chunk::stage::StageId; +use crowdb_tree_ffi::{crc32c, CtError}; + +impl KvRootCatalogStore { + pub(super) async fn commit_candidate(&self, id: StageId) -> Result<(), CtError> { + self.authority_mutation(|| async { + let mut authority = self.protection_authority(true).await?; + let mut record = self.candidate_record(id).await?.ok_or(CtError::NotFound)?; + if record.status.state == CandidateState::Committed { + return Ok(()); + } + if record.status.state != CandidateState::Ready + || authority.generation.checked_add(1) != Some(record.status.generation) + { + return Err(CtError::Unavailable.into()); + } + let (manifest, _) = self + .get(&operation_key(self.tree_id, b"candidate-manifest", id)) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Corruption)?; + if manifest.len() as u64 != record.manifest_bytes || crc32c(&manifest) != record.manifest_checksum + { + return Err(CtError::Corruption.into()); + } + record.status.state = CandidateState::Committed; + authority.value = Bytes::copy_from_slice(&super::super::encode_authority( + authority.epoch, + record.status.generation, + )); + let ops = vec![ + BatchOp::Put { + key: Bytes::from(catalog_key(self.tree_id, b"current", 0)), + value: manifest.clone(), + }, + BatchOp::Put { + key: Bytes::from(catalog_key(self.tree_id, b"manifest", record.status.generation)), + value: manifest, + }, + BatchOp::Put { + key: Bytes::from(catalog_key( + self.tree_id, + b"manifest-candidate", + record.status.generation, + )), + value: Bytes::from(encode_words(&[ + id.owner_epoch, + id.operation_high, + id.operation_low, + ])), + }, + BatchOp::Put { + key: Bytes::from(operation_key(self.tree_id, b"candidate", id)), + value: Bytes::from(record.encode()), + }, + ]; + // Intents already own every reference. This CAS changes only the root + // and candidate state, so publication never creates an unprotected gap. + self.protection_write(authority, ops).await + }) + .await + } + + pub(super) async fn abort_candidate(&self, id: StageId) -> Result<(), CtError> { + self.authority_mutation(|| async { + let authority = self.protection_authority(true).await?; + let mut record = self.candidate_record(id).await?.ok_or(CtError::NotFound)?; + if record.status.state == CandidateState::Aborted { + return Ok(()); + } + if !matches!( + record.status.state, + CandidateState::Preparing | CandidateState::Ready + ) { + return Err(CtError::Unavailable.into()); + } + record.status.state = CandidateState::Aborted; + self.protection_write(authority, self.terminal_candidate_ops(&record)) + .await + }) + .await + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/guards.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/guards.rs new file mode 100644 index 000000000..e9106d609 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/guards.rs @@ -0,0 +1,260 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::record::{decode_words, encode_words, operation_key}; +use super::writer::WriterOwner; +use super::KvRootCatalogStore; +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_tree_ffi::chunk::stage::{StageId, StageRecord, StageState}; +use crowdb_tree_ffi::CtError; + +#[derive(Clone, Copy, Default)] +pub(super) struct StageGuard { + pub references: u64, + pub pins: u64, + pub creator: Option<WriterOwner>, +} + +impl KvRootCatalogStore { + pub(super) async fn load_stage_guard(&self, id: StageId) -> Result<StageGuard, CtError> { + let key = operation_key(self.tree_id, b"stage-guard", id); + let (value, _) = self + .get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)?; + if value.len() == 28 { + let words = decode_words::<3>(&value)?; + if words[0] != 1 { + return Err(CtError::Corruption); + } + return Ok(StageGuard { + references: words[1], + pins: words[2], + creator: None, + }); + } + let words = decode_words::<12>(&value)?; + if words[0] != 2 || words[3] > 1 { + return Err(CtError::Corruption); + } + let creator = if words[3] == 0 { + if words[4..] != [0; 8] { + return Err(CtError::Corruption); + } + None + } else { + Some(WriterOwner::decode(&encode_words(&words[4..]))?) + }; + Ok(StageGuard { + references: words[1], + pins: words[2], + creator, + }) + } + pub(super) fn stage_guard_put(&self, id: StageId, guard: StageGuard) -> BatchOp { + let mut words = [0; 12]; + words[..3].copy_from_slice(&[2, guard.references, guard.pins]); + if let Some(owner) = guard.creator { + words[3] = 1; + words[4..].copy_from_slice(&decode_words::<8>(&owner.encode()).expect("valid writer encoding")); + } + BatchOp::Put { + key: Bytes::from(operation_key(self.tree_id, b"stage-guard", id)), + value: Bytes::from(encode_words(&words)), + } + } + pub(super) async fn readable_stage(&self, id: StageId) -> Result<StageRecord, CtError> { + let key = super::super::staging::stage_key(self.tree_id, id); + let (value, _) = self + .get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::NotFound)?; + let record = StageRecord::decode(&value)?; + if record.id != id || record.tree_id != self.tree_id { + return Err(CtError::Corruption); + } + if !matches!(record.state, StageState::Reloadable | StageState::Referenced) { + return Err(CtError::Unavailable); + } + Ok(record) + } + + pub(in super::super) async fn stage_guard_transition( + &self, + record: &StageRecord, + creating: bool, + next: Option<&StageRecord>, + ) -> Result<Vec<BatchOp>, CtError> { + let mut key = super::catalog_key(self.tree_id, b"chunk-stage", record.chunk_high); + key.extend_from_slice(&record.chunk_low.to_be_bytes()); + let current = self.get(&key).await.map_err(|_| CtError::Unavailable)?; + let identity = encode_words(&[ + record.id.owner_epoch, + record.id.operation_high, + record.id.operation_low, + ]); + if creating { + // The identity tombstone remains after stage cleanup. A fixed ChunkId + // cannot be allocated a second time under a different operation. + if current.is_some() { + return Err(CtError::Unavailable); + } + let creator = self.writer_owner()?; + let mut ops = vec![ + BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from(identity), + }, + self.stage_guard_put( + record.id, + StageGuard { + creator, + pins: u64::from(creator.is_some()), + ..StageGuard::default() + }, + ), + ]; + if let Some(owner) = creator { + // Protect the output before Reloadable publication. The native + // operation transfers this exact pin to its returned backing. + let mut pin_key = operation_key(self.tree_id, b"stage-pin", record.id); + for part in [ + record.id.owner_epoch, + record.id.operation_high, + record.id.operation_low, + ] { + pin_key.extend_from_slice(&part.to_be_bytes()); + } + // A release that won before an unknown create reply leaves a + // tombstone under this same authority revision. Never resurrect it. + if self + .get(&pin_key) + .await + .map_err(|_| CtError::Unavailable)? + .is_some() + { + return Err(CtError::Unavailable); + } + let pin = super::pin_owner::PinOwner { + live: true, + process: Some(owner.process), + }; + ops.push(BatchOp::Put { + key: Bytes::from(pin_key), + value: Bytes::from(pin.encode()), + }); + } + return Ok(ops); + } + if current.as_ref().map(|(bytes, _)| bytes.as_ref()) != Some(identity.as_slice()) { + return Err(CtError::Unavailable); + } + if next.map_or(true, |next| { + matches!( + next.state, + StageState::Retired | StageState::Deleting | StageState::Reclaimed + ) + }) { + let guard = self.load_stage_guard(record.id).await?; + if let Some(creator) = guard.creator { + // Sealed packs have no outstanding disk writes. Unsealed packs + // remain protected until their exact writer has stopped. + if matches!(record.state, StageState::Reserved | StageState::Writing) { + self.require_completed_recovery().await?; + if !self.operation_is_excluded(record.id, creator).await? { + return Err(CtError::Unavailable); + } + } + } else if record.id.owner_epoch != self.owner_epoch + && matches!(record.state, StageState::Reserved | StageState::Writing) + { + // Old unsealed records have no durable creator identity from + // which a completed-write exclusion can be proved. + return Err(CtError::Unavailable); + } + if guard.references != 0 || guard.pins != 0 { + return Err(CtError::Unavailable); + } + } + Ok(Vec::new()) + } + + pub(super) async fn change_stage_pin( + &self, + stage: StageId, + pin: StageId, + acquire: bool, + ) -> Result<(), CtError> { + self.authority_mutation(|| async { + use super::pin_owner::PinOwner; + let owner = PinOwner { + live: acquire, + process: self.process_identity()?, + }; + if acquire && owner.process.is_some() { + self.check_process_owner()?; + } + // Reader ownership is process-scoped. A retained old reader need not + // be the current writer, but its pin still competes atomically with GC. + let authority = self + .protection_authority(acquire && owner.process.is_none()) + .await?; + let mut key = operation_key(self.tree_id, b"stage-pin", stage); + for value in [pin.owner_epoch, pin.operation_high, pin.operation_low] { + key.extend_from_slice(&value.to_be_bytes()); + } + let current = self.get(&key).await.map_err(|_| CtError::Unavailable)?; + let current = current.map(|(bytes, _)| PinOwner::decode(&bytes)).transpose()?; + if current == Some(owner) { + return Ok(()); + } + if current.is_some_and(|current| current.process != owner.process) { + return Err(CtError::Unavailable.into()); + } + if acquire { + if current.is_some() { + return Err(CtError::Unavailable.into()); + } + self.readable_stage(stage).await?; + } + let stage_key = super::super::staging::stage_key(self.tree_id, stage); + let stage_exists = self + .get(&stage_key) + .await + .map_err(|_| CtError::Unavailable)? + .is_some(); + let mut ops = Vec::with_capacity(3); + if stage_exists { + let mut guard = self.load_stage_guard(stage).await?; + if acquire { + guard.pins = guard.pins.checked_add(1).ok_or(CtError::ResourceExhausted)?; + } else if current.is_some_and(|current| current.live) { + guard.pins = guard.pins.checked_sub(1).ok_or(CtError::Corruption)?; + } + if !acquire && pin == stage { + if let Some(creator) = guard.creator { + ops.push(self.operation_quiescence(stage, creator).await?); + } + } + ops.push(self.stage_guard_put(stage, guard)); + } else if acquire || current.is_some_and(|value| value.live) { + return Err(CtError::Corruption.into()); + } + // Missing stages are not evidence that an unknown create can no + // longer commit. The inactive pin also fences any later exact retry. + ops.push(BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from(owner.encode()), + }); + self.protection_write(authority, ops).await?; + if acquire && owner.process.is_some() { + self.check_process_owner()?; + } + Ok(()) + }) + .await + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/operation.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/operation.rs new file mode 100644 index 000000000..a11961317 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/operation.rs @@ -0,0 +1,69 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Initial-pin release is the terminal boundary of one accepted native write. +use super::record::{decode_words, encode_words, operation_key}; +use super::writer::WriterOwner; +use super::KvRootCatalogStore; +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_protocol::timing::{MAX_WRITE_CLOCK_SKEW_MS, MAX_WRITE_REQUEST_AGE_MS}; +use crowdb_tree_ffi::chunk::stage::StageId; +use crowdb_tree_ffi::CtError; + +impl KvRootCatalogStore { + // The caller atomically includes this record with releasing the initial + // pin, after the native operation and every accepted local I/O tail ended. + // Independent reader pins never imply that a writer is quiescent. + pub(super) async fn operation_quiescence( + &self, + stage: StageId, + owner: WriterOwner, + ) -> Result<BatchOp, CtError> { + if self.writer_owner()? != Some(owner) { + return Err(CtError::Unavailable); + } + let key = operation_key(self.tree_id, b"stage-quiesced", stage); + let value = if let Some((bytes, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? { + decode_receipt(&bytes, owner)?; + bytes + } else { + let mut words = [0; 10]; + words[..2].copy_from_slice(&[1, super::writer::wall_time_ms()?]); + words[2..].copy_from_slice(&decode_words::<8>(&owner.encode())?); + Bytes::from(encode_words(&words)) + }; + Ok(BatchOp::Put { + key: Bytes::from(key), + value, + }) + } + + pub(super) async fn operation_is_excluded( + &self, + stage: StageId, + owner: WriterOwner, + ) -> Result<bool, CtError> { + let key = operation_key(self.tree_id, b"stage-quiesced", stage); + let Some((bytes, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? else { + return self.writer_is_excluded(stage.owner_epoch, owner).await; + }; + let released_at = decode_receipt(&bytes, owner)?; + let deadline = released_at + .checked_add(MAX_WRITE_CLOCK_SKEW_MS) + .and_then(|value| value.checked_add(MAX_WRITE_REQUEST_AGE_MS)) + .ok_or(CtError::Corruption)?; + if super::writer::wall_time_ms()? >= deadline { + return Ok(true); + } + self.writer_is_excluded(stage.owner_epoch, owner).await + } +} + +fn decode_receipt(bytes: &[u8], owner: WriterOwner) -> Result<u64, CtError> { + let words = decode_words::<10>(bytes)?; + if words[0] != 1 || words[1] == 0 || WriterOwner::decode(&encode_words(&words[2..]))? != owner { + return Err(CtError::Corruption); + } + Ok(words[1]) +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/pin_owner.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/pin_owner.rs new file mode 100644 index 000000000..fb92b933e --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/pin_owner.rs @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::record::{decode_words, encode_words}; +use crowdb_protocol::chunk_stream::{StreamProcessIdentity, StreamPurpose}; +use crowdb_tree_ffi::CtError; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct PinOwner { + pub live: bool, + pub process: Option<StreamProcessIdentity>, +} +impl PinOwner { + pub fn encode(self) -> Vec<u8> { + let mut words = [1, u64::from(self.live), 0, 0, 0, 0, 0, 0]; + if let Some(identity) = self.process { + words[2] = 1; + words[3] = identity.instance_id; + words[4] = identity.process_epoch; + words[5] = u64::from_le_bytes(identity.boot_nonce[..8].try_into().expect("nonce bounds")); + words[6] = u64::from_le_bytes(identity.boot_nonce[8..].try_into().expect("nonce bounds")); + } + encode_words(&words) + } + pub fn decode(bytes: &[u8]) -> Result<Self, CtError> { + if bytes.len() == 12 { + let [live] = decode_words::<1>(bytes)?; + if live > 1 { + return Err(CtError::Corruption); + } + return Ok(Self { + live: live != 0, + process: None, + }); + } + let words = decode_words::<8>(bytes)?; + if words[0] != 1 || words[1] > 1 || words[2] > 1 || words[7] != 0 { + return Err(CtError::Corruption); + } + let process = if words[2] == 0 { + if words[3..7] != [0, 0, 0, 0] { + return Err(CtError::Corruption); + } + None + } else { + if words[3] == 0 || words[4] == 0 || words[5] == 0 && words[6] == 0 { + return Err(CtError::Corruption); + } + let mut boot_nonce = [0; 16]; + boot_nonce[..8].copy_from_slice(&words[5].to_le_bytes()); + boot_nonce[8..].copy_from_slice(&words[6].to_le_bytes()); + Some(StreamProcessIdentity { + purpose: StreamPurpose::Wal, + instance_id: words[3], + process_epoch: words[4], + boot_nonce, + }) + }; + Ok(Self { + live: words[1] != 0, + process, + }) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/process.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/process.rs new file mode 100644 index 000000000..7995fc6eb --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/process.rs @@ -0,0 +1,169 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::record::{decode_words, encode_words}; +use super::{catalog_key, KvRootCatalogStore}; +use bytes::Bytes; +use crowdb_paxos_kv_client::{BatchOp, StreamProcessAuthority, StreamProcessLeaseClient}; +use crowdb_protocol::chunk_stream::{StreamProcessIdentity, StreamPurpose}; +use crowdb_tree_ffi::CtError; +use std::sync::Arc; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +pub(in super::super) struct RecoveryReceipt { + root_generation: u64, + stream_high: u64, + stream_low: u64, + applied_seq: u64, + replay_end: u64, +} + +impl KvRootCatalogStore { + pub(in super::super::super) fn with_process( + mut self, + process: Option<Arc<StreamProcessAuthority>>, + ) -> Result<Self, CtError> { + self.writer_incarnation = Self::next_writer_incarnation()?; + self.process = process; + Ok(self) + } + + pub(in super::super) fn process_identity(&self) -> Result<Option<StreamProcessIdentity>, CtError> { + self.process + .as_ref() + .map(|process| { + let identity = process.status(Instant::now()).lease.identity; + if identity.purpose != StreamPurpose::Wal { + return Err(CtError::InvalidArgument); + } + Ok(identity) + }) + .transpose() + } + + pub(in super::super) fn check_process_owner(&self) -> Result<(), CtError> { + let process = self.process.as_ref().ok_or(CtError::NotSupported)?; + process + .capture(Instant::now()) + .map(|_| ()) + .map_err(|_| CtError::Unavailable) + } + + /// Called only after the real root and complete WAL replay have succeeded. + /// Prepared transfer roots retain the receipt locally until publication + /// authority has actually moved to this owner. + pub(in super::super::super) async fn complete_recovery( + &self, + partition: &crowdb_chunk_kv::Partition, + root_generation: u64, + ) -> Result<(), CtError> { + if self.process.is_none() { + return Ok(()); + } + self.check_process_owner()?; + let snapshot = partition.snapshot(); + if partition.tree_id() != self.tree_id + || snapshot.ownership_epoch != self.owner_epoch + || snapshot.applied_seq != snapshot.journal_durable_seq + { + return Err(CtError::Unavailable); + } + self.recovered.store(Some(Arc::new(RecoveryReceipt { + root_generation, + stream_high: snapshot.stream_name.high, + stream_low: snapshot.stream_name.low, + applied_seq: snapshot.applied_seq, + replay_end: snapshot.journal_durable_offset, + }))); + let authority = self.protection_authority(false).await?; + if authority.epoch < self.owner_epoch { + return Ok(()); + } + if authority.epoch != self.owner_epoch { + return Err(CtError::Unavailable); + } + self.register_writer().await?; + self.persist_completed_recovery().await + } + + pub(in super::super) async fn persist_completed_recovery(&self) -> Result<(), CtError> { + self.authority_mutation(|| async { + let Some(receipt) = self.recovered.load_full() else { + return Ok(()); + }; + self.check_process_owner()?; + let Some(identity) = self.process_identity()? else { + return Err(CtError::NotSupported.into()); + }; + let authority = self.protection_authority(true).await?; + if authority.generation < receipt.root_generation { + return Err(CtError::Corruption.into()); + } + let key = catalog_key(self.tree_id, b"completed-recovery", self.owner_epoch); + let words = [ + 1, + identity.instance_id, + identity.process_epoch, + u64::from_le_bytes(identity.boot_nonce[..8].try_into().expect("nonce bounds")), + u64::from_le_bytes(identity.boot_nonce[8..].try_into().expect("nonce bounds")), + self.writer_incarnation, + receipt.root_generation, + receipt.stream_high, + receipt.stream_low, + receipt.applied_seq, + receipt.replay_end, + ]; + let value = encode_words(&words); + if let Some((old, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? { + if old.as_ref() == value { + return Ok(()); + } + // Another boot must prove its own recovery, even at the same Tree epoch. + } + self.protection_write( + authority, + vec![BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from(value), + }], + ) + .await?; + self.check_process_owner().map_err(Into::into) + }) + .await + } + + pub(in super::super) async fn require_completed_recovery(&self) -> Result<(), CtError> { + self.check_process_owner()?; + let identity = self.process_identity()?.ok_or(CtError::NotSupported)?; + let key = catalog_key(self.tree_id, b"completed-recovery", self.owner_epoch); + let (value, _) = self + .get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)?; + let words = decode_words::<11>(&value)?; + if words[0] != 1 + || words[1] != identity.instance_id + || words[2] != identity.process_epoch + || words[3].to_le_bytes() != identity.boot_nonce[..8] + || words[4].to_le_bytes() != identity.boot_nonce[8..] + || words[5] != self.writer_incarnation + { + return Err(CtError::Unavailable); + } + Ok(()) + } + + pub(super) async fn process_is_excluded(&self, identity: StreamProcessIdentity) -> Result<bool, CtError> { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| CtError::Unavailable)?; + let now_ms = now.as_millis().try_into().map_err(|_| CtError::Unavailable)?; + StreamProcessLeaseClient::new(self.kv.clone()) + .confirm_exclusion(identity, now_ms) + .await + .map(|proof| proof.is_some()) + .map_err(|_| CtError::Unavailable) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/prune.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/prune.rs new file mode 100644 index 000000000..c34e9e5bf --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/prune.rs @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::pin_owner::PinOwner; +use super::record::operation_key; +use super::KvRootCatalogStore; +use bytes::Bytes; +use crowdb_paxos_kv_client::{BatchOp, ReadMode}; +use crowdb_tree_ffi::chunk::protection::PinPruneProgress; +use crowdb_tree_ffi::chunk::stage::StageId; +use crowdb_tree_ffi::CtError; + +impl KvRootCatalogStore { + pub(super) async fn prune_pins( + &self, + stage: StageId, + after: Option<StageId>, + limit: u32, + ) -> Result<PinPruneProgress, CtError> { + self.authority_mutation(|| async { + if !(1..=64).contains(&limit) { + return Err(CtError::InvalidArgument.into()); + } + // Every pruning transaction also updates the stage guard and authority. + let limit = limit.min( + u32::try_from(super::MAX_METADATA_MUTATIONS - 2).map_err(|_| CtError::ResourceExhausted)?, + ); + self.require_completed_recovery().await?; + let authority = self.protection_authority(true).await?; + let prefix = operation_key(self.tree_id, b"stage-pin", stage); + let start = after + .map(|id| { + let mut key = prefix.clone(); + for value in [id.owner_epoch, id.operation_high, id.operation_low] { + key.extend_from_slice(&value.to_be_bytes()); + } + key + }) + .unwrap_or_default(); + let result = self + .kv + .scan( + self.store_id, + self.group_id, + &prefix, + &start, + &[], + limit, + ReadMode::Linearizable, + None, + false, + None, + ) + .await + .map_err(|_| CtError::Unavailable)?; + if result.timed_out { + return Err(CtError::Unavailable.into()); + } + let mut progress = PinPruneProgress { + more: result.truncated, + ..PinPruneProgress::default() + }; + let mut ops = Vec::with_capacity(result.items.len() + 2); + for (key, value) in result.items { + if key.len() != prefix.len() + 24 { + return Err(CtError::Corruption.into()); + } + let bytes = &key[prefix.len()..]; + let at = |offset| { + u64::from_be_bytes(bytes[offset..offset + 8].try_into().expect("pin key bounds")) + }; + let id = StageId { + owner_epoch: at(0), + operation_high: at(8), + operation_low: at(16), + }; + if after.is_some_and(|after| id <= after) { + return Err(CtError::Corruption.into()); + } + let mut owner = PinOwner::decode(&value)?; + progress.examined += 1; + progress.after = Some(id); + let Some(process) = owner.process else { + continue; + }; + if !owner.live || !self.process_is_excluded(process).await? { + continue; + } + owner.live = false; + progress.released += 1; + ops.push(BatchOp::Put { + key, + value: Bytes::from(owner.encode()), + }); + } + if progress.released != 0 { + let mut guard = self.load_stage_guard(stage).await?; + guard.pins = guard + .pins + .checked_sub(progress.released.into()) + .ok_or(CtError::Corruption)?; + ops.push(self.stage_guard_put(stage, guard)); + self.protection_write(authority, ops).await?; + } + self.check_process_owner()?; + Ok(progress) + }) + .await + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/record.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/record.rs new file mode 100644 index 000000000..564a33609 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/record.rs @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_tree_ffi::chunk::protection::{CandidateState, CandidateStatus, MAX_CANDIDATE_REFERENCES}; +use crowdb_tree_ffi::chunk::stage::StageId; +use crowdb_tree_ffi::{crc32c, CtError}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct CandidateRecord { + pub status: CandidateStatus, + pub expected_checksum: u32, + pub prepared_checksum: u32, + pub manifest_checksum: u32, + pub manifest_bytes: u64, + pub last_stage: StageId, +} + +impl CandidateRecord { + pub fn encode(&self) -> Vec<u8> { + encode_words(&[ + 1, + self.status.state as u64, + self.status.generation, + self.status.reference_count, + self.status.prepared_references, + self.status.prepared_windows, + self.status.released_windows, + self.expected_checksum.into(), + self.prepared_checksum.into(), + self.manifest_checksum.into(), + self.manifest_bytes, + self.last_stage.owner_epoch, + self.last_stage.operation_high, + self.last_stage.operation_low, + ]) + } + pub fn decode(id: StageId, bytes: &[u8]) -> Result<Self, CtError> { + let words = decode_words::<14>(bytes)?; + let state = match words[1] { + 1 => CandidateState::Preparing, + 2 => CandidateState::Ready, + 3 => CandidateState::Committed, + 4 => CandidateState::Aborted, + 5 => CandidateState::Reclaimed, + _ => return Err(CtError::Corruption), + }; + if words[0] != 1 + || words[2] == 0 + || words[3] > MAX_CANDIDATE_REFERENCES + || words[4] > words[3] + || words[5] > words[4] + || words[6] > words[5] + { + return Err(CtError::Corruption); + } + Ok(Self { + status: CandidateStatus { + id, + state, + generation: words[2], + reference_count: words[3], + prepared_references: words[4], + prepared_windows: words[5], + released_windows: words[6], + }, + expected_checksum: u32::try_from(words[7]).map_err(|_| CtError::Corruption)?, + prepared_checksum: u32::try_from(words[8]).map_err(|_| CtError::Corruption)?, + manifest_checksum: u32::try_from(words[9]).map_err(|_| CtError::Corruption)?, + manifest_bytes: words[10], + last_stage: StageId { + owner_epoch: words[11], + operation_high: words[12], + operation_low: words[13], + }, + }) + } +} + +pub(super) fn encode_words(words: &[u64]) -> Vec<u8> { + let mut bytes = Vec::with_capacity(8 * words.len() + 4); + for value in words { + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes.extend_from_slice(&crc32c(&bytes).to_le_bytes()); + bytes +} + +pub(super) fn decode_words<const N: usize>(bytes: &[u8]) -> Result<[u64; N], CtError> { + if bytes.len() != N * 8 + 4 { + return Err(CtError::Corruption); + } + let checksum = u32::from_le_bytes(bytes[N * 8..].try_into().map_err(|_| CtError::Corruption)?); + if crc32c(&bytes[..N * 8]) != checksum { + return Err(CtError::Corruption); + } + Ok(std::array::from_fn(|index| { + u64::from_le_bytes( + bytes[index * 8..index * 8 + 8] + .try_into() + .expect("fixed record bounds"), + ) + })) +} + +pub(super) fn operation_key(tree: u64, kind: &[u8], id: StageId) -> Vec<u8> { + let mut key = super::catalog_key(tree, kind, id.owner_epoch); + key.extend_from_slice(&id.operation_high.to_be_bytes()); + key.extend_from_slice(&id.operation_low.to_be_bytes()); + key +} + +pub(super) fn window_key(tree: u64, id: StageId, window: u64) -> Vec<u8> { + let mut key = operation_key(tree, b"candidate-window", id); + key.extend_from_slice(&window.to_be_bytes()); + key +} + +pub(super) fn encode_references(refs: &[StageId]) -> Vec<u8> { + let words = refs + .iter() + .flat_map(|id| [id.owner_epoch, id.operation_high, id.operation_low]) + .collect::<Vec<_>>(); + encode_words(&words) +} + +pub(super) fn decode_references(bytes: &[u8]) -> Result<Vec<StageId>, CtError> { + if bytes.len() < 28 || (bytes.len() - 4) % 24 != 0 || bytes.len() > 64 * 24 + 4 { + return Err(CtError::Corruption); + } + let payload = &bytes[..bytes.len() - 4]; + if crc32c(payload) + != u32::from_le_bytes( + bytes[bytes.len() - 4..] + .try_into() + .map_err(|_| CtError::Corruption)?, + ) + { + return Err(CtError::Corruption); + } + let refs = payload + .chunks_exact(24) + .map(|chunk| { + let at = + |offset| u64::from_le_bytes(chunk[offset..offset + 8].try_into().expect("reference bounds")); + StageId { + owner_epoch: at(0), + operation_high: at(8), + operation_low: at(16), + } + }) + .collect::<Vec<_>>(); + if refs.windows(2).any(|pair| pair[0] >= pair[1]) { + return Err(CtError::Corruption); + } + Ok(refs) +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/release.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/release.rs new file mode 100644 index 000000000..58ac6ba58 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/release.rs @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::record::{decode_references, decode_words, operation_key, window_key, CandidateRecord}; +use super::{catalog_key, KvRootCatalogStore}; +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_tree_ffi::chunk::protection::CandidateState; +use crowdb_tree_ffi::chunk::stage::StageId; +use crowdb_tree_ffi::CtError; + +impl KvRootCatalogStore { + pub(super) fn terminal_candidate_ops(&self, record: &CandidateRecord) -> Vec<BatchOp> { + let mut ops = vec![BatchOp::Put { + key: Bytes::from(operation_key(self.tree_id, b"candidate", record.status.id)), + value: Bytes::from(record.encode()), + }]; + if record.status.released_windows == record.status.prepared_windows { + ops.push(BatchOp::Delete { + key: Bytes::from(operation_key( + self.tree_id, + b"candidate-manifest", + record.status.id, + )), + }); + } + ops + } + + pub(super) async fn recover_candidate(&self, id: StageId) -> Result<(), CtError> { + self.authority_mutation(|| async { + self.require_completed_recovery().await?; + let authority = self.protection_authority(true).await?; + let mut record = self.candidate_record(id).await?.ok_or(CtError::NotFound)?; + if matches!( + record.status.state, + CandidateState::Aborted | CandidateState::Reclaimed + ) { + return Ok(()); + } + if record.status.state == CandidateState::Committed || id.owner_epoch > self.owner_epoch { + return Err(CtError::Unavailable.into()); + } + if id.owner_epoch == self.owner_epoch { + let key = operation_key(self.tree_id, b"candidate-owner", id); + let (bytes, _) = self + .get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)?; + let creator = super::writer::WriterOwner::decode(&bytes)?; + if self.writer_owner()? == Some(creator) + || !self.writer_is_excluded(id.owner_epoch, creator).await? + { + return Err(CtError::Unavailable.into()); + } + } + record.status.state = CandidateState::Aborted; + self.protection_write(authority, self.terminal_candidate_ops(&record)) + .await + }) + .await + } + + pub(super) async fn release_candidate(&self, id: StageId, window: u64) -> Result<(), CtError> { + self.authority_mutation(|| async { + let authority = self.protection_authority(true).await?; + let mut record = self.candidate_record(id).await?.ok_or(CtError::NotFound)?; + if !matches!( + record.status.state, + CandidateState::Aborted | CandidateState::Reclaimed + ) { + return Err(CtError::Unavailable.into()); + } + if window < record.status.released_windows { + return Ok(()); + } + if window != record.status.released_windows || window >= record.status.prepared_windows { + return Err(CtError::InvalidArgument.into()); + } + let key = window_key(self.tree_id, id, window); + let (bytes, _) = self + .get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Corruption)?; + let refs = decode_references(&bytes)?; + let mut ops = Vec::with_capacity(refs.len() + 4); + for reference in refs { + let mut guard = self.load_stage_guard(reference).await?; + guard.references = guard.references.checked_sub(1).ok_or(CtError::Corruption)?; + ops.push(self.stage_guard_put(reference, guard)); + } + record.status.released_windows += 1; + if record.status.released_windows == record.status.prepared_windows { + ops.push(BatchOp::Delete { + key: Bytes::from(operation_key(self.tree_id, b"candidate-manifest", id)), + }); + } + ops.push(BatchOp::Delete { + key: Bytes::from(key), + }); + ops.push(BatchOp::Put { + key: Bytes::from(operation_key(self.tree_id, b"candidate", id)), + value: Bytes::from(record.encode()), + }); + self.protection_write(authority, ops).await + }) + .await + } + + /// Called while the caller still owns the root authority revision used to + /// reclaim the matching immutable manifest and its generation-pin proof. + pub(in super::super) async fn reclaim_candidate_ops( + &self, + generation: u64, + ) -> Result<Vec<BatchOp>, CtError> { + let key = catalog_key(self.tree_id, b"manifest-candidate", generation); + let Some((value, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? else { + return Ok(Vec::new()); + }; + let words = decode_words::<3>(&value)?; + let id = StageId { + owner_epoch: words[0], + operation_high: words[1], + operation_low: words[2], + }; + let mut record = self.candidate_record(id).await?.ok_or(CtError::Corruption)?; + if record.status.state != CandidateState::Committed || record.status.generation != generation { + return Err(CtError::Corruption); + } + record.status.state = CandidateState::Reclaimed; + let mut ops = self.terminal_candidate_ops(&record); + ops.push(BatchOp::Delete { + key: Bytes::from(key), + }); + Ok(ops) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/scan.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/scan.rs new file mode 100644 index 000000000..15884dc18 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/scan.rs @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::record::{operation_key, CandidateRecord}; +use super::{catalog_key, KvRootCatalogStore}; +use crowdb_paxos_kv_client::ReadMode; +use crowdb_tree_ffi::chunk::protection::CandidateStatus; +use crowdb_tree_ffi::chunk::stage::StageId; +use crowdb_tree_ffi::CtError; + +impl KvRootCatalogStore { + pub(super) async fn scan_candidates( + &self, + after: Option<StageId>, + limit: u32, + ) -> Result<Vec<CandidateStatus>, CtError> { + if !(1..=64).contains(&limit) { + return Err(CtError::InvalidArgument); + } + let mut prefix = catalog_key(self.tree_id, b"candidate", 0); + prefix.truncate(prefix.len() - 8); + let start = after + .map(|id| operation_key(self.tree_id, b"candidate", id)) + .unwrap_or_default(); + let result = self + .kv + .scan( + self.store_id, + self.group_id, + &prefix, + &start, + &[], + limit, + ReadMode::Linearizable, + None, + false, + None, + ) + .await + .map_err(|_| CtError::Unavailable)?; + if result.timed_out { + return Err(CtError::Unavailable); + } + result + .items + .into_iter() + .map(|(key, value)| { + if key.len() != prefix.len() + 24 { + return Err(CtError::Corruption); + } + let bytes = &key[prefix.len()..]; + let at = |offset| { + u64::from_be_bytes( + bytes[offset..offset + 8] + .try_into() + .expect("candidate key bounds"), + ) + }; + let id = StageId { + owner_epoch: at(0), + operation_high: at(8), + operation_low: at(16), + }; + if after.is_some_and(|after| id <= after) { + return Err(CtError::Corruption); + } + Ok(CandidateRecord::decode(id, &value)?.status) + }) + .collect() + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/writer.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/writer.rs new file mode 100644 index 000000000..27f90155d --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/protection/writer.rs @@ -0,0 +1,218 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::pin_owner::PinOwner; +use super::record::{decode_words, encode_words}; +use super::{catalog_key, KvRootCatalogStore}; +use bytes::Bytes; +use crowdb_paxos_kv_client::BatchOp; +use crowdb_protocol::chunk_stream::StreamProcessIdentity; +use crowdb_protocol::timing::{MAX_WRITE_CLOCK_SKEW_MS, MAX_WRITE_REQUEST_AGE_MS}; +use crowdb_tree_ffi::CtError; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +static NEXT_WRITER: AtomicU64 = AtomicU64::new(1); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct WriterOwner { + pub process: StreamProcessIdentity, + pub incarnation: u64, +} + +impl WriterOwner { + pub(super) fn encode(self) -> Vec<u8> { + let owner = PinOwner { + live: true, + process: Some(self.process), + }; + let mut words = decode_words::<8>(&owner.encode()).expect("valid process encoding"); + words[7] = self.incarnation; + encode_words(&words) + } + pub(super) fn decode(bytes: &[u8]) -> Result<Self, CtError> { + let mut words = decode_words::<8>(bytes)?; + let incarnation = words[7]; + words[7] = 0; + let owner = PinOwner::decode(&encode_words(&words))?; + if !owner.live || incarnation == 0 { + return Err(CtError::Corruption); + } + Ok(Self { + process: owner.process.ok_or(CtError::Corruption)?, + incarnation, + }) + } +} + +impl KvRootCatalogStore { + pub(in super::super) fn next_writer_incarnation() -> Result<u64, CtError> { + NEXT_WRITER + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) + .map_err(|_| CtError::ResourceExhausted) + } + + pub(super) fn writer_owner(&self) -> Result<Option<WriterOwner>, CtError> { + Ok(self.process_identity()?.map(|process| WriterOwner { + process, + incarnation: self.writer_incarnation, + })) + } + + fn quiescence_key(&self, epoch: u64, owner: WriterOwner) -> Vec<u8> { + let mut key = catalog_key(self.tree_id, b"writer-quiesced", epoch); + key.extend_from_slice(&owner.encode()); + key + } + + async fn writer_was_closed(&self, epoch: u64, owner: WriterOwner) -> Result<bool, CtError> { + let Some((value, _)) = self + .get(&self.quiescence_key(epoch, owner)) + .await + .map_err(|_| CtError::Unavailable)? + else { + return Ok(false); + }; + let words = decode_words::<2>(&value)?; + if words[0] != 1 || words[1] == 0 { + return Err(CtError::Corruption); + } + Ok(true) + } + + pub(super) async fn writer_is_quiesced(&self, epoch: u64, owner: WriterOwner) -> Result<bool, CtError> { + match self + .get(&self.quiescence_key(epoch, owner)) + .await + .map_err(|_| CtError::Unavailable)? + { + None => Ok(false), + Some((value, _)) => { + let words = decode_words::<2>(&value)?; + if words[0] != 1 { + return Err(CtError::Corruption); + } + let deadline = words[1] + .checked_add(MAX_WRITE_CLOCK_SKEW_MS) + .and_then(|value| value.checked_add(MAX_WRITE_REQUEST_AGE_MS)) + .ok_or(CtError::Corruption)?; + Ok(wall_time_ms()? >= deadline) + } + } + } + + pub(super) async fn writer_is_excluded(&self, epoch: u64, owner: WriterOwner) -> Result<bool, CtError> { + if self.writer_is_quiesced(epoch, owner).await? { + return Ok(true); + } + self.process_is_excluded(owner.process).await + } + + // Reading the registration after authority is essential: replacement also + // advances that authority revision, fencing every already-prepared mutation. + pub(in super::super) async fn check_registered_writer(&self) -> Result<(), CtError> { + let Some(owner) = self.writer_owner()? else { + return Ok(()); + }; + self.check_process_owner()?; + if self.writer_quiesced.load(Ordering::Acquire) { + return Err(CtError::Unavailable); + } + let key = catalog_key(self.tree_id, b"writer-owner", self.owner_epoch); + let value = self + .get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)? + .0; + if WriterOwner::decode(&value)? != owner + || self + .get(&self.quiescence_key(self.owner_epoch, owner)) + .await + .map_err(|_| CtError::Unavailable)? + .is_some() + { + return Err(CtError::Unavailable); + } + Ok(()) + } + + pub(in super::super::super) async fn register_writer(&self) -> Result<(), CtError> { + self.authority_mutation(|| async { + let Some(owner) = self.writer_owner()? else { + return Ok(()); + }; + self.check_process_owner()?; + if self.writer_quiesced.load(Ordering::Acquire) { + return Err(CtError::Unavailable.into()); + } + let authority = self.protection_authority(false).await?; + if authority.epoch != self.owner_epoch { + if authority.epoch > self.owner_epoch { + return Err(CtError::Unavailable.into()); + } + self.check_prepared_staging(authority.epoch).await?; + } + let key = catalog_key(self.tree_id, b"writer-owner", self.owner_epoch); + if let Some((bytes, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? { + let previous = WriterOwner::decode(&bytes)?; + if previous == owner { + return self.check_registered_writer().await.map_err(Into::into); + } + // Writer replacement is fenced by this authority CAS immediately. + // Reclaiming an old unsealed pack still waits for the separate + // server request-age exclusion in writer_is_excluded. + if !self.writer_was_closed(self.owner_epoch, previous).await? + && !self.process_is_excluded(previous.process).await? + { + return Err(CtError::Unavailable.into()); + } + } + self.protection_write( + authority, + vec![BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from(owner.encode()), + }], + ) + .await?; + self.check_process_owner().map_err(Into::into) + }) + .await + } + + // The caller must have closed write admission and drained accepted native + // backend work first. Reader/export pins are deliberately untouched. + pub(super) async fn publish_writer_quiescence(&self) -> Result<(), CtError> { + self.authority_mutation(|| async { + let owner = self.writer_owner()?.ok_or(CtError::NotSupported)?; + self.writer_quiesced.store(true, Ordering::Release); + let authority = self.protection_authority(false).await?; + let key = self.quiescence_key(self.owner_epoch, owner); + if let Some((value, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? { + let words = decode_words::<2>(&value)?; + return if words[0] == 1 { + Ok(()) + } else { + Err(CtError::Corruption.into()) + }; + } + self.protection_write( + authority, + vec![BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from(encode_words(&[1, wall_time_ms()?])), + }], + ) + .await + }) + .await + } +} + +pub(super) fn wall_time_ms() -> Result<u64, CtError> { + let time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| CtError::Unavailable)?; + time.as_millis().try_into().map_err(|_| CtError::Unavailable) +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/retention.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/retention.rs new file mode 100644 index 000000000..65c6d4562 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/retention.rs @@ -0,0 +1,201 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Pins and reclamation mutate the same authority revision as root publication. + +use super::{ + catalog_key, catalog_pin_key, catalog_pin_prefix, decode_authority, decode_u64, KvRootCatalogStore, +}; +use bytes::Bytes; +use crowdb_paxos_kv_client::{BatchOp, ReadMode}; +use crowdb_tree_ffi::CtError; + +impl KvRootCatalogStore { + async fn retention_authority(&self) -> Result<(Vec<u8>, Bytes, u64), CtError> { + let key = catalog_key(self.tree_id, b"authority", 0); + let (value, revision) = self + .get(&key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)?; + decode_authority(&value).ok_or(CtError::Corruption)?; + Ok((key, value, revision)) + } + + pub(super) fn retain_generation( + &self, + tree_id: u64, + high: u64, + low: u64, + generation: u64, + ) -> Result<(), CtError> { + if tree_id != self.tree_id || (high == 0 && low == 0) || generation == 0 { + return Err(CtError::InvalidArgument); + } + self.wait(self.authority_mutation(|| async { + let (authority_key, authority, revision) = self.retention_authority().await?; + if self.process.is_some() { + self.check_process_owner()?; + } + if self + .get(&catalog_key(tree_id, b"manifest", generation)) + .await + .map_err(|_| CtError::Unavailable)? + .is_none() + { + return Err(CtError::NotFound.into()); + } + self.check_legacy_pin_admission(generation).await?; + let key = catalog_pin_key(tree_id, high, low); + if let Some((value, _)) = self.get(&key).await.map_err(|_| CtError::Unavailable)? { + return if decode_u64(&value) == Some(generation) { + Ok(()) + } else { + Err(CtError::InvalidArgument.into()) + }; + } + let ops = [ + BatchOp::Put { + key: Bytes::from(authority_key.clone()), + value: authority, + }, + BatchOp::Put { + key: Bytes::from(key), + value: Bytes::copy_from_slice(&generation.to_be_bytes()), + }, + ]; + self.kv + .batch_write_cas(self.store_id, self.group_id, &ops, &authority_key, revision) + .await + .map(|_| ()) + .map_err(|error| super::transaction::MutationError::from_cas(&error)) + })) + } + + pub(super) fn release_generation(&self, tree_id: u64, high: u64, low: u64) -> Result<(), CtError> { + if tree_id != self.tree_id || (high == 0 && low == 0) { + return Err(CtError::InvalidArgument); + } + self.wait(self.authority_mutation(|| async { + let (authority_key, authority, revision) = self.retention_authority().await?; + let key = catalog_pin_key(tree_id, high, low); + if self.get(&key).await.map_err(|_| CtError::Unavailable)?.is_none() { + return Ok(()); + } + let ops = [ + BatchOp::Put { + key: Bytes::from(authority_key.clone()), + value: authority, + }, + BatchOp::Delete { + key: Bytes::from(key), + }, + ]; + self.kv + .batch_write_cas(self.store_id, self.group_id, &ops, &authority_key, revision) + .await + .map(|_| ()) + .map_err(|error| super::transaction::MutationError::from_cas(&error)) + })) + } + + pub(super) fn reclaim_generations(&self, tree_id: u64, generation: u64) -> u64 { + if tree_id != self.tree_id || generation <= 2 { + return 0; + } + self.wait(self.reclaim_generations_inner(generation)).unwrap_or(0) + } + + async fn reclaim_generations_inner(&self, generation: u64) -> Result<u64, CtError> { + self.authority_mutation(|| async { + let (authority_key, authority, revision) = self.retention_authority().await?; + self.check_registered_writer().await?; + let (epoch, current_generation) = decode_authority(&authority).ok_or(CtError::Corruption)?; + if epoch != self.owner_epoch { + return Err(CtError::Unavailable.into()); + } + let prefix = catalog_pin_prefix(self.tree_id); + // A saturated pin window conservatively retains history for this pass. + let pins = self + .kv + .scan( + self.store_id, + self.group_id, + &prefix, + &[], + &[], + 1024, + ReadMode::Linearizable, + None, + false, + None, + ) + .await + .map_err(|_| CtError::Unavailable)?; + if pins.truncated || pins.timed_out { + return Err(CtError::ResourceExhausted.into()); + } + let mut generation = generation.min(current_generation); + for (_, value) in pins.items { + generation = generation.min(decode_u64(&value).ok_or(CtError::Corruption)?); + } + if generation <= 2 { + return Ok(0); + } + let floor_key = catalog_key(self.tree_id, b"reclaim-floor", 0); + let floor = match self.get(&floor_key).await.map_err(|_| CtError::Unavailable)? { + Some((value, _)) => decode_u64(&value).ok_or(CtError::Corruption)?, + None => 1, + }; + let limit = generation.saturating_sub(1).min(floor.saturating_add(128)); + if floor >= limit { + return Ok(0); + } + let mut end = floor; + let mut ops = Vec::with_capacity(super::protection::MAX_METADATA_MUTATIONS); + for old in floor..limit { + let preserved = self.legacy_history_ops(old).await?; + let candidate = self.reclaim_candidate_ops(old).await?; + // Reserve the floor and authority keys; never split one + // generation's manifest, candidate and legacy-history mutation. + if ops.len() + 1 + preserved.len() + candidate.len() + 2 + > super::protection::MAX_METADATA_MUTATIONS + { + break; + } + ops.push(BatchOp::Delete { + key: Bytes::from(catalog_key(self.tree_id, b"manifest", old)), + }); + let copied_legacy = !preserved.is_empty(); + ops.extend(preserved); + ops.extend(candidate); + end = old + 1; + // Retain at most one newly copied legacy image in this pass. + if copied_legacy { + break; + } + } + ops.push(BatchOp::Put { + key: Bytes::from(floor_key), + value: Bytes::copy_from_slice(&end.to_be_bytes()), + }); + ops.push(BatchOp::Put { + key: Bytes::from(authority_key.clone()), + value: authority, + }); + self.kv + .batch_write_cas(self.store_id, self.group_id, &ops, &authority_key, revision) + .await + .map_err(|error| super::transaction::MutationError::from_cas(&error))?; + self.published_objects.rcu(|current| { + let mut next = (**current).clone(); + for old in floor..end { + next.remove(&catalog_key(self.tree_id, b"manifest", old)); + } + next + }); + Ok(end - floor) + }) + .await + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/staging.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/staging.rs new file mode 100644 index 000000000..3d7ea2dbe --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/staging.rs @@ -0,0 +1,194 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Conditional staging mutations share the root authority revision. + +use bytes::Bytes; +use crowdb_paxos_kv_client::{BatchOp, ReadMode}; +use crowdb_tree_ffi::chunk::stage::{validate_stage_transition, StageId, StageRecord, MAX_STAGE_SCAN}; +use crowdb_tree_ffi::CtError; + +use super::{catalog_key, decode_authority, KvRootCatalogStore}; + +impl KvRootCatalogStore { + pub(super) fn stage_load(&self, tree_id: u64, id: StageId) -> Result<Option<StageRecord>, CtError> { + if tree_id != self.tree_id || id.owner_epoch == 0 { + return Err(CtError::InvalidArgument); + } + let value = self + .wait(self.get(&stage_key(tree_id, id))) + .map_err(|_| CtError::Unavailable)?; + value + .map(|(bytes, _)| { + let record = StageRecord::decode(&bytes)?; + if record.tree_id != tree_id || record.id != id { + return Err(CtError::Corruption); + } + Ok(record) + }) + .transpose() + } + + pub(super) fn stage_compare( + &self, + tree_id: u64, + owner_epoch: u64, + expected: Option<&StageRecord>, + next: Option<&StageRecord>, + ) -> Result<(), CtError> { + validate_stage_transition(expected, next)?; + let record = next.or(expected).ok_or(CtError::InvalidArgument)?; + if tree_id != self.tree_id || record.tree_id != tree_id { + return Err(CtError::InvalidArgument); + } + if owner_epoch != self.owner_epoch || record.id.owner_epoch > owner_epoch { + return Err(CtError::Unavailable); + } + if record.id.owner_epoch != owner_epoch { + use crowdb_tree_ffi::chunk::stage::StageState; + if expected.is_none() + || next.is_some_and(|value| { + !matches!( + value.state, + StageState::Retired | StageState::Deleting | StageState::Reclaimed + ) + }) + { + return Err(CtError::Unavailable); + } + self.wait(self.require_completed_recovery())?; + } + if record.id.owner_epoch == owner_epoch { + self.wait(self.authorize_staging())?; + } else { + self.wait(self.authorize_publication())?; + } + self.wait(self.stage_compare_authorized(owner_epoch, record.id, expected, next)) + } + + async fn stage_compare_authorized( + &self, + owner_epoch: u64, + id: StageId, + expected: Option<&StageRecord>, + next: Option<&StageRecord>, + ) -> Result<(), CtError> { + self.authority_mutation(|| async { + let authority_key = catalog_key(self.tree_id, b"authority", 0); + let (authority, revision) = self + .get(&authority_key) + .await + .map_err(|_| CtError::Unavailable)? + .ok_or(CtError::Unavailable)?; + let (epoch, _) = decode_authority(&authority).ok_or(CtError::Corruption)?; + self.check_registered_writer().await?; + if epoch != owner_epoch { + if epoch > owner_epoch || id.owner_epoch != owner_epoch { + return Err(CtError::Unavailable.into()); + } + self.check_prepared_staging(epoch).await?; + } + let key = stage_key(self.tree_id, id); + let current = self.get(&key).await.map_err(|_| CtError::Unavailable)?; + let expected_record = expected; + let next_record = next; + let expected = expected.map(StageRecord::encode).transpose()?; + let next = next.map(StageRecord::encode).transpose()?; + let current = current.as_ref().map(|(bytes, _)| bytes.as_ref()); + if current == next.as_deref() { + return Ok(()); + } + if current != expected.as_deref() { + return Err(CtError::Unavailable.into()); + } + let update = match next { + Some(bytes) => BatchOp::Put { + key: Bytes::from(key), + value: Bytes::from(bytes), + }, + None => BatchOp::Delete { + key: Bytes::from(key), + }, + }; + // Writing the unchanged authority value advances its KV revision. Root + // publication, stage transitions, pinning, and reclamation serialize here. + let mut ops = vec![ + BatchOp::Put { + key: Bytes::from(authority_key.clone()), + value: authority, + }, + update, + ]; + let record = expected_record.or(next_record).ok_or(CtError::InvalidArgument)?; + ops.extend( + self.stage_guard_transition(record, expected_record.is_none(), next_record) + .await?, + ); + self.kv + .batch_write_cas(self.store_id, self.group_id, &ops, &authority_key, revision) + .await + .map(|_| ()) + .map_err(|error| super::transaction::MutationError::from_cas(&error)) + }) + .await + } + + pub(super) fn stage_list( + &self, + tree_id: u64, + after: Option<StageId>, + limit: u32, + ) -> Result<Vec<StageRecord>, CtError> { + if tree_id != self.tree_id || !(1..=MAX_STAGE_SCAN).contains(&limit) { + return Err(CtError::InvalidArgument); + } + let prefix = stage_prefix(tree_id); + let start = after.map(|id| stage_key(tree_id, id)).unwrap_or_default(); + let result = self + .wait(self.kv.scan( + self.store_id, + self.group_id, + &prefix, + &start, + &[], + limit, + ReadMode::Linearizable, + None, + false, + None, + )) + .map_err(|_| CtError::Unavailable)?; + if result.timed_out { + return Err(CtError::Unavailable); + } + result + .items + .into_iter() + .map(|(key, value)| { + let record = StageRecord::decode(&value)?; + if record.tree_id != tree_id + || key.as_ref() != stage_key(tree_id, record.id).as_slice() + || after.is_some_and(|id| record.id <= id) + { + return Err(CtError::Corruption); + } + Ok(record) + }) + .collect() + } +} + +fn stage_prefix(tree_id: u64) -> Vec<u8> { + let mut key = b"\0crowdb/chunk-kv/root/v1/".to_vec(); + key.extend_from_slice(&tree_id.to_be_bytes()); + key.extend_from_slice(b"/stage/v1/"); + key +} + +pub(super) fn stage_key(tree_id: u64, id: StageId) -> Vec<u8> { + let mut key = stage_prefix(tree_id); + for value in [id.owner_epoch, id.operation_high, id.operation_low] { + key.extend_from_slice(&value.to_be_bytes()); + } + key +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog/transaction.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog/transaction.rs new file mode 100644 index 000000000..f02819bf1 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog/transaction.rs @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Rebuild authority mutations after a definite revision conflict. + +use super::KvRootCatalogStore; +use crowdb_paxos_kv_client::Error; +use crowdb_tree_ffi::CtError; +use std::future::Future; + +pub(super) enum MutationError { + Conflict, + Status(CtError), +} +pub(super) type MutationResult<T> = Result<T, MutationError>; + +impl From<CtError> for MutationError { + fn from(status: CtError) -> Self { + Self::Status(status) + } +} +impl MutationError { + pub(super) fn from_cas(error: &Error) -> Self { + match error { + Error::CasFailed { .. } | Error::CasBusy => Self::Conflict, + // An unknown reply requires the caller's durable operation lookup. + // Do not treat transport or ownership failures as revision conflicts. + _ => Self::Status(CtError::Unavailable), + } + } +} +impl KvRootCatalogStore { + pub(super) async fn authority_mutation<T, F, Fut>(&self, mut prepare: F) -> Result<T, CtError> + where + F: FnMut() -> Fut, + Fut: Future<Output = MutationResult<T>>, + { + // Pins, root publication and GC deliberately share one revision. A + // definite conflict discards all derived counters and proofs; each + // attempt rereads authority, revalidates ownership and rebuilds ops. + // A finite bound preserves availability under sustained contention. + for _ in 0..16 { + match prepare().await { + Ok(value) => return Ok(value), + Err(MutationError::Status(status)) => return Err(status), + Err(MutationError::Conflict) => tokio::task::yield_now().await, + } + } + Err(CtError::Unavailable) + } +} diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog_test.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog_test.rs new file mode 100644 index 000000000..3b55ff65e --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog_test.rs @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Feature-gated access to the production process-bound root authority. +use super::{KvRootCatalogStore, StorageRuntimeError}; +use crowdb_chunk_kv::Partition; +use crowdb_paxos_kv_client::{PaxosKvClient, StreamProcessAuthority}; +use crowdb_protocol::chunk_kv::ChunkKvRangeCatalogEntry; +use crowdb_tree_ffi::{CtError, RootCatalogStore}; +use std::sync::Arc; + +pub struct TestOwnedRootCatalog { + catalog: Arc<KvRootCatalogStore>, +} +impl TestOwnedRootCatalog { + #[must_use] + pub fn store(&self) -> &dyn RootCatalogStore { + self.catalog.as_ref() + } + /// # Errors + /// Requires matching, actually recovered root and replay frontiers. + pub async fn complete_recovery( + &self, + partition: &Partition, + root_generation: u64, + ) -> Result<(), CtError> { + self.catalog.complete_recovery(partition, root_generation).await + } + + /// # Errors + /// Returns native manifest inspection or recovery receipt errors. + pub async fn complete_tree_recovery_for_tests( + &self, + partition: &Partition, + store: &crowdb_tree_ffi::PageStore, + ) -> Result<(), StorageRuntimeError> { + super::ChunkKvStorage::complete_tree_recovery(partition, store, &self.catalog).await + } +} +/// # Errors +/// Returns identity, process registration or KV availability errors. +pub async fn open_root_catalog_with_process_for_tests( + kv: Arc<PaxosKvClient>, + store_id: u64, + group_id: u64, + entry: &ChunkKvRangeCatalogEntry, + process: Arc<StreamProcessAuthority>, +) -> Result<TestOwnedRootCatalog, StorageRuntimeError> { + let catalog = KvRootCatalogStore::for_assignment(kv, store_id, group_id, entry) + .await? + .with_process(Some(process)) + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?; + catalog + .register_writer() + .await + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?; + Ok(TestOwnedRootCatalog { + catalog: Arc::new(catalog), + }) +} diff --git a/app/crowdb-chunk-kv-server/src/storage/split_recovery.rs b/app/crowdb-chunk-kv-server/src/storage/split_recovery.rs index 74213f740..c57030e08 100644 --- a/app/crowdb-chunk-kv-server/src/storage/split_recovery.rs +++ b/app/crowdb-chunk-kv-server/src/storage/split_recovery.rs @@ -5,7 +5,7 @@ use super::{prepared_overlay_artifact, split_plan, storage_plan_error, ChunkKvStorage}; use crate::MonitorError; -use crowdb_chunk_kv::{Partition, PartitionConfig, PreparedSplit}; +use crowdb_chunk_kv::{Partition, PreparedSplit}; use crowdb_chunk_stream::StreamRegistry; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, SplitTransition, @@ -42,13 +42,19 @@ impl ChunkKvStorage { ) .await .map_err(|error| storage_plan_error(&error.to_string()))?; + // A live retry may still have a synchronized but unpublished parent + // suffix. Publish it before opening the independent recovery reader. + parent + .publish_split_journal_prefix(proof.cutover_offset) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; let source_stream = self .streams .open_read_only_current(proof.source_stream_name, self.metadata_store_id) .await .map_err(|error| storage_plan_error(&error.to_string()))?; - let page_store = self - .open_durable_tree_page_store( + let (page_store, catalog) = self + .open_durable_tree( ChunkPageStoreOptions { tree_id: transition.child.artifact.tree_id, owner_epoch: transition.child.owner_epoch, @@ -75,14 +81,17 @@ impl ChunkKvStorage { }; let (child, artifact) = Partition::recover_native_split_handoff( prepared_overlay_artifact(&entry, proof), - PartitionConfig::default(), - crowdb_tree_ffi::Config::default(), - page_store, + self.partition_config.clone(), + self.tree_config(), + page_store.clone(), child_stream, source_stream, ) .await .map_err(|error| storage_plan_error(&error.to_string()))?; + Self::complete_tree_recovery(&child, &page_store, &catalog) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; parent .resume_split_child_session(split_plan(transition), child, artifact) .await diff --git a/app/crowdb-chunk-kv-server/src/storage/tree_owner.rs b/app/crowdb-chunk-kv-server/src/storage/tree_owner.rs new file mode 100644 index 000000000..4fefc979f --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/tree_owner.rs @@ -0,0 +1,177 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! One fixed process lease covers tree writers and independent read windows. +use super::{ChunkKvStorage, KvRootCatalogStore, StorageRuntimeError}; +use crowdb_chunk_kv::Partition; +use crowdb_paxos_kv_client::StreamProcessAuthority; +use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, CtError, PageStore}; +use std::sync::Arc; + +impl ChunkKvStorage { + pub(super) fn tree_config(&self) -> crowdb_tree_ffi::Config { + crowdb_tree_ffi::Config { + runtime: Some(self.tree_runtime.clone()), + ..self.tree_options.clone() + } + } + + /// Bind the same fixed process authority used by serving grants and WAL. + /// Must run before opening any hosted tree. + /// # Errors + /// Rejects a conflicting process binding or invalid WAL authority. + pub fn bind_owner_process( + &self, + process: Arc<StreamProcessAuthority>, + ) -> Result<(), StorageRuntimeError> { + self.streams + .registry() + .bind_owner_process(process.clone()) + .map_err(|error| StorageRuntimeError::Stream(error.to_string()))?; + let expected = process.clone(); + let previous = self + .owner_process + .compare_and_swap(&None::<Arc<StreamProcessAuthority>>, Some(process)); + if previous.as_ref().is_some_and(|old| !Arc::ptr_eq(old, &expected)) { + return Err(StorageRuntimeError::Tree( + "tree process authority is already bound".into(), + )); + } + Ok(()) + } + + pub(super) fn bind_catalog( + &self, + catalog: KvRootCatalogStore, + ) -> Result<Arc<KvRootCatalogStore>, StorageRuntimeError> { + let process = Some( + self.owner_process + .load_full() + .ok_or_else(|| StorageRuntimeError::Tree("tree process authority is not bound".into()))?, + ); + catalog + .with_process(process) + .map(Arc::new) + .map_err(|error| StorageRuntimeError::Tree(error.to_string())) + } + + pub(super) async fn open_durable_tree( + &self, + options: ChunkPageStoreOptions, + metadata_group_id: u64, + ) -> Result<(Arc<PageStore>, Arc<KvRootCatalogStore>), StorageRuntimeError> { + let catalog = KvRootCatalogStore::claim( + self.kv.clone(), + self.metadata_store_id, + metadata_group_id, + options.tree_id, + options.owner_epoch, + ) + .await?; + let catalog = self.bind_catalog(catalog)?; + catalog + .register_writer() + .await + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?; + let store = self.open_tree_page_store( + options, + Arc::new( + ChunkRootCatalog::open_callback(catalog.clone()) + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?, + ), + )?; + Ok((store, catalog)) + } + + pub(super) async fn open_assignment_tree( + &self, + entry: &crowdb_protocol::chunk_kv::ChunkKvRangeCatalogEntry, + metadata_group_id: u64, + ) -> Result<(Arc<PageStore>, Arc<KvRootCatalogStore>), StorageRuntimeError> { + let open_generation = entry + .artifact + .tail_overlay + .as_ref() + .map_or(0, |overlay| overlay.base_root_manifest_generation); + let options = ChunkPageStoreOptions { + tree_id: entry.artifact.tree_id, + owner_epoch: entry.owner_epoch, + open_generation, + pack_bytes: 0, + iu_size: 0, + max_concurrent_packs: 0, + materialization_bytes_per_pass: 0, + mirror_copies: 0, + max_chunk_bytes: 0, + }; + let catalog = KvRootCatalogStore::for_assignment( + self.kv.clone(), + self.metadata_store_id, + metadata_group_id, + entry, + ) + .await?; + let catalog = self.bind_catalog(catalog)?; + catalog + .register_writer() + .await + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?; + let page_store = self.open_tree_page_store( + options, + Arc::new( + ChunkRootCatalog::open_callback(catalog.clone()) + .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?, + ), + )?; + Ok((page_store, catalog)) + } + + pub(super) async fn complete_tree_recovery( + partition: &Partition, + store: &PageStore, + catalog: &KvRootCatalogStore, + ) -> Result<(), StorageRuntimeError> { + // Each caller has already awaited root and WAL recovery. A prepared + // split retains an overlay artifact but has no live transfer catch-up + // request to complete its separate initialization future. + let generation = match store.chunk_manifest_generation() { + Ok(value) => value, + Err(CtError::NotFound) => 0, + Err(error) => return Err(StorageRuntimeError::Tree(error.to_string())), + }; + catalog + .complete_recovery(partition, generation) + .await + .map_err(|error| StorageRuntimeError::Tree(error.to_string())) + } +} + +#[cfg(feature = "test-util")] +impl ChunkKvStorage { + /// Assemble the production runtime backend with already connected test services. + /// # Errors + /// Returns native route, shared host budget, or storage initialization errors. + pub async fn from_parts_parallel_for_tests( + kv: Arc<crowdb_paxos_kv_client::PaxosKvClient>, + chunk_io: crowdb_chunk_client::ChunkIoClient, + metadata_store_id: u64, + writer_lease_ms: u64, + mirrors: u32, + ) -> Result<Self, StorageRuntimeError> { + Self::from_parts_with_options( + kv, + chunk_io, + metadata_store_id, + writer_lease_ms, + super::TreeStorageOptions { + mirror_copies: mirrors, + chunk_capacity_bytes: 256 * 1024 * 1024, + strip_capacity_bytes: 0, + ..super::TreeStorageOptions::default() + }, + crowdb_chunk_stream::StreamConfig::default(), + crowdb_chunk_client::ChunkReadPolicy::default(), + ) + .await + } +} diff --git a/app/crowdb-chunk-kv-server/tests/backing_observation_test.rs b/app/crowdb-chunk-kv-server/tests/backing_observation_test.rs new file mode 100644 index 000000000..fc494ca7f --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/backing_observation_test.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#[path = "common/process.rs"] +mod test_process; + +use std::sync::Arc; + +use crowdb_chunk_kv::{ + CrowdbPartitionTree, MutationOperation, Partition, PartitionConfig, PartitionId, PartitionRange, + PartitionTree, StreamPartitionJournal, +}; +use crowdb_chunk_kv_server::ChunkKvService; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, +}; +use crowdb_tree_ffi::{chunk::test_util::TestChunkBackend, Config}; + +async fn partition(id: u64, tree: Arc<dyn PartitionTree>) -> Partition { + let store = Arc::new(MemoryStreamStore::new(4096)); + let name = StreamName { high: 1, low: id }; + let stream = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Wal, + stream_name: name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + Partition::open( + PartitionId { high: 1, low: id }, + PartitionRange::default(), + 1, + PartitionConfig::default(), + tree, + Arc::new(StreamPartitionJournal::new(stream, name).unwrap()), + ) + .unwrap() +} + +#[tokio::test] +async fn registry_counts_one_physical_ledger_once_across_partition_views() { + let backend = TestChunkBackend::new(82, 1).unwrap(); + let tree = Arc::new( + CrowdbPartitionTree::open( + 82, + &Config { + page_store: Some(backend.page_store(0).unwrap()), + ..backend.config() + }, + ) + .unwrap(), + ); + tree.apply( + 1, + &MutationOperation::Put { + key: b"key".to_vec(), + value: vec![1; 8192], + }, + ) + .await + .unwrap(); + tree.checkpoint(0).await.unwrap(); + let expected = tree.backing_stats().unwrap().unwrap(); + assert!(expected.allocated_bytes >= 8192); + assert_eq!(tree.estimated_bytes().unwrap(), expected.allocated_bytes); + assert!(tree.chunk_stats().is_err()); + let first = partition(1, tree.clone()).await; + let second = partition(2, tree.clone()).await; + assert_eq!( + first.backing_storage_stats().unwrap(), + second.backing_storage_stats().unwrap() + ); + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 8).unwrap(); + service.install_partition(&first).unwrap(); + service.install_partition(&second).unwrap(); + assert_eq!( + service.registry_observation(1 << 20, 0).durable_bytes, + expected.allocated_bytes + ); + let observed = service + .registry_observation_with_load_samples(1 << 20, 0, 0) + .await + .unwrap(); + assert_eq!(observed.partition_loads.len(), 2); + assert!(observed + .partition_loads + .iter() + .all(|load| load.durable_bytes == expected.allocated_bytes)); + first.suspend_for_shutdown(1).await.unwrap(); + second.suspend_for_shutdown(1).await.unwrap(); + drop(service); + first.close().await.unwrap(); + second.close().await.unwrap(); +} diff --git a/app/crowdb-chunk-kv-server/tests/bootstrap_recovery_test.rs b/app/crowdb-chunk-kv-server/tests/bootstrap_recovery_test.rs index 348a6bfed..dd541fa16 100644 --- a/app/crowdb-chunk-kv-server/tests/bootstrap_recovery_test.rs +++ b/app/crowdb-chunk-kv-server/tests/bootstrap_recovery_test.rs @@ -1,20 +1,44 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_chunk_kv::{ - MutationOperation, Partition, PartitionConfig, PartitionId, PartitionRange, RequestId, -}; -use crowdb_chunk_kv_server::{BootstrapPartitionConfig, ChunkKvStorage}; +use crowdb_chunk_kv::{ChunkKvError, MutationOperation, Partition, RequestId}; +use crowdb_chunk_kv_server::{BootstrapPartitionConfig, ChunkKvServerConfig, ChunkKvStorage}; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, StreamRecordState, }; use crowdb_protocol::chunk_kv::Id128; -use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, PageStore}; +use crowdb_tree_ffi::chunk::test_util::TestChunkBackend; use std::sync::Arc; +#[path = "common/process.rs"] +mod test_process; + +fn assert_resource_log(partition: &Partition, runtime: &Arc<crowdb_tree_ffi::TreeRuntime>) { + use crowdb_common::metrics::MetricsSampler; + let service = Arc::new( + crowdb_chunk_kv_server::ChunkKvService::new(test_process::TestProcess::authority(7), 8).unwrap(), + ); + service.install_partition(partition).unwrap(); + let mut sampler = crowdb_chunk_kv_server::metrics::TreeResources::new(runtime.clone(), &service); + sampler.sample(); + let mut bytes = Vec::new(); + sampler.flush(&mut bytes, "test").unwrap(); + let log = String::from_utf8(bytes).unwrap(); + assert!(log.contains("host.tree.resource.cpu_workers"), "{log}"); + assert!(log.contains("chunk.tree.45.resource.writer"), "{log}"); + assert!(log.contains("allocated_sample_max="), "{log}"); +} + #[tokio::test] async fn bootstrap_resume_finishes_its_first_checkpoint_and_does_not_reinitialize_it() { + let mut server_config = ChunkKvServerConfig::load( + &std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("conf/crowdb_chunk_kv_server_config.toml"), + ) + .unwrap(); + server_config.partition.max_key_bytes = 3; + server_config.partition.batch_requests = 2; + server_config.validate().unwrap(); let config = BootstrapPartitionConfig { partition_id: Id128 { high: 1, low: 11 }, tree_id: 45, @@ -24,52 +48,24 @@ async fn bootstrap_resume_finishes_its_first_checkpoint_and_does_not_reinitializ }; let name = StreamName { high: 2, low: 11 }; let store = Arc::new(MemoryStreamStore::new(4096)); - let stream = ChunkStream::create( - StreamRecord { - purpose: StreamPurpose::Wal, - stream_name: name, - metadata_group_id: 7, - record_version: 1, - state: StreamRecordState::Active, - consumer_kind: Some("chunk-kv-partition".into()), - consumer_id: None, - created_at_ms: StreamRecord::creation_time_ms(), - }, - 4, - StreamConfig::default(), - store.clone(), - store.clone(), - store.clone(), - ) - .await - .unwrap(); - let catalog = Arc::new(ChunkRootCatalog::open_memory(4).unwrap()); - let options = ChunkPageStoreOptions { - tree_id: 45, - owner_epoch: 4, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - max_chunk_bytes: 0, - mirror_copies: 0, - materialization_bytes_per_pass: 4096, + let stream = create_stream(name, &store).await; + let mut backend = TestChunkBackend::new(45, 4).unwrap(); + let pages = backend.page_store(0).unwrap(); + let tree_config = crowdb_tree_ffi::Config { + runtime: Some(backend.runtime.clone()), + ..server_config.native_tree_options() }; - let pages = Arc::new(PageStore::open_chunk(options, catalog.clone(), None).unwrap()); - let partition = Partition::open_native( - PartitionId { high: 1, low: 11 }, - PartitionRange { - start: Some(Vec::new()), - end: None, - }, - 4, - PartitionConfig::default(), - 45, - crowdb_tree_ffi::Config::default(), + let partition = ChunkKvStorage::open_bootstrap_partition_for_tests( + &config, + server_config.partition.clone(), + tree_config.clone(), pages, stream, ) + .await .unwrap(); + assert_resource_log(&partition, &backend.runtime); + assert_partition_limit(&partition).await; partition .mutate( 4, @@ -86,25 +82,43 @@ async fn bootstrap_resume_finishes_its_first_checkpoint_and_does_not_reinitializ .await .unwrap(); partition.suspend_for_transfer(4).await.unwrap(); - drop(partition); + partition.close().await.unwrap(); let stream = reopen(name, &store).await; - let pages = Arc::new(PageStore::open_chunk(options, catalog.clone(), None).unwrap()); - let recovered = ChunkKvStorage::recover_bootstrap_partition_for_tests(&config, pages.clone(), stream) - .await - .unwrap(); + backend.next_writer().unwrap(); + let pages = backend.page_store(0).unwrap(); + let recovered = ChunkKvStorage::recover_bootstrap_partition_for_tests( + &config, + server_config.partition.clone(), + tree_config.clone(), + pages.clone(), + stream, + ) + .await + .unwrap(); assert!(recovered.get(4, b"", None).await.unwrap().is_none()); + assert_partition_limit(&recovered).await; let generation = pages.chunk_manifest_generation().unwrap(); assert!(generation > 0); recovered.suspend_for_transfer(4).await.unwrap(); - drop(recovered); + recovered.close().await.unwrap(); let stream = reopen(name, &store).await; - let pages = Arc::new(PageStore::open_chunk(options, catalog, None).unwrap()); - let recovered = ChunkKvStorage::recover_bootstrap_partition_for_tests(&config, pages.clone(), stream) - .await - .unwrap(); + backend.next_writer().unwrap(); + let pages = backend.page_store(0).unwrap(); + let recovered = ChunkKvStorage::recover_bootstrap_partition_for_tests( + &config, + server_config.partition.clone(), + tree_config, + pages.clone(), + stream, + ) + .await + .unwrap(); assert_eq!(pages.chunk_manifest_generation().unwrap(), generation); recovered.activate_recovered(4).unwrap(); + assert_partition_limit(&recovered).await; assert!(recovered.get(4, b"", None).await.unwrap().is_none()); + recovered.suspend_for_shutdown(4).await.unwrap(); + recovered.close().await.unwrap(); } async fn reopen(name: StreamName, store: &Arc<MemoryStreamStore>) -> ChunkStream { @@ -119,3 +133,43 @@ async fn reopen(name: StreamName, store: &Arc<MemoryStreamStore>) -> ChunkStream .await .unwrap() } + +async fn assert_partition_limit(partition: &Partition) { + let result = partition + .mutate( + 4, + RequestId { + client_high: 1, + client_low: 12, + client_sequence: 1, + }, + MutationOperation::Put { + key: b"four".to_vec(), + value: b"value".to_vec(), + }, + ) + .await; + assert!(matches!(result, Err(ChunkKvError::InvalidRequest(_)))); +} + +async fn create_stream(name: StreamName, store: &Arc<MemoryStreamStore>) -> ChunkStream { + ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Wal, + stream_name: name, + metadata_group_id: 7, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 4, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap() +} diff --git a/app/crowdb-chunk-kv-server/tests/common/legacy_retirement.rs b/app/crowdb-chunk-kv-server/tests/common/legacy_retirement.rs new file mode 100644 index 000000000..a5f3c0ce2 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/legacy_retirement.rs @@ -0,0 +1,265 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{ChunkAllocator, DiskWriter, IoError, Result}; +use crowdb_protocol::{ + chunkdb::rpc::*, + common::{ChunkId, DiskId}, + diskdb::rpc::Segment, +}; +use crowdb_tree_ffi::chunk::legacy::LegacyManifest; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Mutex, + }, + time::Instant, +}; + +pub fn manifest() -> &'static [u8] { + let fixture = include_bytes!("../../../../lib/crowdb-tree/tests/data/legacy-chunk-v7.bin"); + let length = usize::try_from(u64::from_le_bytes(fixture[8..16].try_into().unwrap())).unwrap(); + &fixture[16..16 + length] +} +pub fn typed_root() -> Vec<u8> { + let mut bytes = vec![0; 132]; + for (index, value) in [ + (0, 0x3354_4F4F_524B_4843_u64), + (1, 1), + (2, 42), + (3, 2), + (4, 7), + (5, 1), + (6, 1), + (9, u64::MAX), + ] { + bytes[index * 8..index * 8 + 8].copy_from_slice(&value.to_le_bytes()); + } + let crc = crowdb_tree_ffi::crc32c(&bytes[..128]); + bytes[128..].copy_from_slice(&crc.to_le_bytes()); + bytes +} +#[derive(Default)] +pub struct TestPhysical { + chunks: Mutex<HashMap<(u64, u64), Chunk>>, + pub deletes: AtomicUsize, + pub drains: AtomicUsize, + pub lose_reply: AtomicBool, +} +impl TestPhysical { + pub fn new() -> Self { + let mut chunks = HashMap::new(); + for pack in LegacyManifest::decode(manifest()).unwrap().packs() { + let capacity = + u32::try_from((pack.offset + u64::from(pack.length)).div_ceil(1024) * 1024).unwrap(); + let id = ChunkId { + high: pack.chunk_high, + low: pack.chunk_low, + }; + let chunk = chunks.entry((id.high, id.low)).or_insert_with(|| Chunk { + id: Some(id), + create_ts_ms: 10, + modify_ts: 1, + chunk_type: i32::try_from(id.high >> 56).unwrap(), + state: ChunkState::Sealed as i32, + ..Chunk::default() + }); + chunk.capacity = chunk.capacity.max(capacity); + } + for chunk in chunks.values_mut() { + chunk.strips.push(ChunkStrip { + unit_kb: 1, + capacity: chunk.capacity, + strip_type: StripType::Mirror as i32, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: vec![Segment { + disk_id: Some(DiskId { high: 1, low: 2 }), + zone_index: 0, + unit_offset: 0, + unit_count: chunk.capacity / 1024, + ..Segment::default() + }], + })), + ..ChunkStrip::default() + }); + } + Self { + chunks: Mutex::new(chunks), + ..Self::default() + } + } + pub fn set_physical_layout(&self, unit_kb: u32, unit_count: u32) { + for chunk in self.chunks.lock().unwrap().values_mut() { + for strip in &mut chunk.strips { + strip.unit_kb = unit_kb; + let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip else { + panic!("fixture requires mirror strips"); + }; + for segment in &mut mirror.segments { + segment.unit_count = unit_count; + } + } + } + } + pub fn count(&self) -> usize { + self.chunks.lock().unwrap().len() + } + pub fn physical_bytes(&self) -> u64 { + self.chunks + .lock() + .unwrap() + .values() + .map(|chunk| u64::from(chunk.capacity)) + .sum() + } + pub fn all_deleted(&self) -> bool { + self.chunks.lock().unwrap().values().all(|chunk| { + chunk.state == ChunkState::Deleted as i32 && chunk.capacity == 0 && chunk.strips.is_empty() + }) + } +} +#[async_trait] +impl ChunkAllocator for TestPhysical { + async fn allocate_chunk(&self, _: AllocateChunkRequest) -> Result<AllocateChunkResponse> { + Err(IoError::Unsupported("fixture allocation".into())) + } + async fn append_chunk(&self, _: AppendChunkRequest) -> Result<AppendChunkResponse> { + Err(IoError::Unsupported("fixture append".into())) + } + async fn seal_chunk(&self, _: SealChunkRequest) -> Result<SealChunkResponse> { + Err(IoError::Unsupported("fixture seal".into())) + } + async fn query_chunk(&self, request: QueryChunkRequest) -> Result<QueryChunkResponse> { + let id = request.chunk_id.unwrap(); + let chunk = self + .chunks + .lock() + .unwrap() + .get(&(id.high, id.low)) + .cloned() + .ok_or_else(|| IoError::ChunkNotFound("fixture absent".into()))?; + Ok(QueryChunkResponse { + chunk: Some(chunk), + ..QueryChunkResponse::default() + }) + } + async fn delete_chunk(&self, request: DeleteChunkRequest) -> Result<DeleteChunkResponse> { + let id = request.chunk_id.unwrap(); + let mut chunks = self.chunks.lock().unwrap(); + let chunk = chunks.get_mut(&(id.high, id.low)).unwrap(); + assert!(self.drains.load(Ordering::Relaxed) > self.deletes.load(Ordering::Relaxed)); + self.deletes.fetch_add(1, Ordering::Relaxed); + chunk.state = ChunkState::Deleted as i32; + chunk.capacity = 0; + chunk.strips.clear(); + if self.lose_reply.swap(false, Ordering::Relaxed) { + return Err(IoError::WriteFailed("delete committed but reply lost".into())); + } + Ok(DeleteChunkResponse { + chunk: Some(chunk.clone()), + }) + } +} +#[async_trait] +impl DiskWriter for TestPhysical { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + Err(IoError::Unsupported("fixture write".into())) + } + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + Err(IoError::Unsupported("fixture write".into())) + } + async fn drain_segment(&self, _: &Segment, _: u64, deadline: Instant) -> Result<()> { + assert!(deadline > Instant::now()); + self.drains.fetch_add(1, Ordering::Relaxed); + Ok(()) + } +} + +use crowdb_paxos_kv_client::{ + PaxosKvClient, StreamProcessAuthority, StreamProcessLeaseClient, StreamProcessLeaseTiming, +}; +use crowdb_protocol::{ + chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, + PartitionArtifact, + }, + chunk_stream::{StreamName, StreamPurpose}, + key::{StreamProcessLeaseKey, TextKey}, +}; +use std::{ + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; +pub fn entry(epoch: u64) -> ChunkKvRangeCatalogEntry { + ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 1 }, + range: KeyRange { + start: vec![], + end: None, + }, + owner: OwnerDescriptor { + instance_id: epoch, + rpc_endpoint: "127.0.0.1:1".into(), + }, + owner_epoch: epoch, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 42, + stream_name: StreamName { high: 1, low: 2 }, + tail_overlay: None, + }, + transition_id: None, + } +} +pub fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + .try_into() + .unwrap() +} +pub async fn process(kv: &Arc<PaxosKvClient>, instance: u64) -> Arc<StreamProcessAuthority> { + let now = now(); + let lease = StreamProcessLeaseClient::new(kv.clone()) + .claim(StreamPurpose::Wal, instance, [1; 16], None, now, 300_000) + .await + .unwrap(); + Arc::new( + StreamProcessAuthority::new(lease, StreamProcessLeaseTiming::default(), now, Instant::now()).unwrap(), + ) +} +pub async fn expire(kv: &Arc<PaxosKvClient>, process: &StreamProcessAuthority) { + process.fence(); + let mut lease = process.status(Instant::now()).lease; + lease.expires_at_ms = now() - 60_000; + let key = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id: lease.identity.instance_id, + } + .to_path(); + kv.put(0, 0, key.as_bytes(), &serde_json::to_vec(&lease).unwrap(), None) + .await + .unwrap(); +} + +pub fn foreign_manifest(tree: u64) -> Vec<u8> { + let mut bytes = manifest().to_vec(); + bytes[4..12].copy_from_slice(&tree.to_le_bytes()); + let segments = usize::try_from(u64::from_le_bytes(bytes[72..80].try_into().unwrap())).unwrap(); + let packs = 80 + segments * 33 + 8; + let crc = crowdb_tree_ffi::crc32c_update(crowdb_tree_ffi::crc32c(&bytes[..68]), &bytes[packs..]); + let crc = crowdb_tree_ffi::crc32c_update(crc, &bytes[80..packs - 8]); + bytes[68..72].copy_from_slice(&crc.to_le_bytes()); + LegacyManifest::decode(&bytes).unwrap(); + bytes +} +pub fn foreign_typed_root(tree: u64) -> Vec<u8> { + let mut bytes = typed_root(); + bytes[16..24].copy_from_slice(&tree.to_le_bytes()); + let crc = crowdb_tree_ffi::crc32c(&bytes[..128]); + bytes[128..].copy_from_slice(&crc.to_le_bytes()); + bytes +} diff --git a/app/crowdb-chunk-kv-server/tests/common/metadata.rs b/app/crowdb-chunk-kv-server/tests/common/metadata.rs index f6887f81a..a69d066a6 100644 --- a/app/crowdb-chunk-kv-server/tests/common/metadata.rs +++ b/app/crowdb-chunk-kv-server/tests/common/metadata.rs @@ -5,28 +5,46 @@ use std::sync::Arc; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::IoBackend; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; +use crowdb_tree_ffi::{TreeRuntime, TreeRuntimeOptions}; pub struct TestMetadata { store: Arc<PxKvStore>, - pub client: Arc<CrowdbKvClient>, + pub client: Arc<PaxosKvClient>, } impl TestMetadata { pub async fn start() -> Self { let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + // Parallel fixtures are independent hosts. Their two real Tree engines + // share one bounded runtime, without competing with unrelated fixtures. + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let replay = replay_group(&Arc::new(IoBackend::File), &[], 1).await.unwrap(); for id in [0, 1] { - store.add_group(PxGroup::new( - id, - PxLocalReplica::new(1, PxLocalReplicaRole::Leader), - )); + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(runtime.clone()), + ..Default::default() + }) + .unwrap(); + let replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(engine), + ) + .await + .unwrap(); + store.add_group(PxGroup::new(id, replica)); } store.start().await.expect("start pinned memory metadata"); let endpoint = store.listen_addr().unwrap().to_string(); - let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + let client = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![]))); for id in [0, 1] { client.seed_leader(0, id, endpoint.clone()); } diff --git a/app/crowdb-chunk-kv-server/tests/common/process.rs b/app/crowdb-chunk-kv-server/tests/common/process.rs index c0c9885b8..0f63a621a 100644 --- a/app/crowdb-chunk-kv-server/tests/common/process.rs +++ b/app/crowdb-chunk-kv-server/tests/common/process.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{StreamProcessAuthority, StreamProcessLeaseTiming}; +use crowdb_paxos_kv_client::{StreamProcessAuthority, StreamProcessLeaseTiming}; use crowdb_protocol::chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}; use std::{sync::Arc, time::Instant}; @@ -22,6 +22,7 @@ impl TestProcess { prior_exclusion_deadline_ms: 0, grant_sequence: 0, grant_expires_at_ms: 0, + revoked: false, }, StreamProcessLeaseTiming::default(), 1_000, diff --git a/app/crowdb-chunk-kv-server/tests/common/process_kv.rs b/app/crowdb-chunk-kv-server/tests/common/process_kv.rs index 99e893b27..1954ef309 100644 --- a/app/crowdb-chunk-kv-server/tests/common/process_kv.rs +++ b/app/crowdb-chunk-kv-server/tests/common/process_kv.rs @@ -1,16 +1,16 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunk_kv::{DomainFailurePolicy, DomainMonitorDescriptor}; use std::sync::Arc; pub struct TestProcessKv { pub store: Arc<PxKvStore>, - pub kv: Arc<CrowdbKvClient>, + pub kv: Arc<PaxosKvClient>, } impl TestProcessKv { @@ -21,7 +21,7 @@ impl TestProcessKv { PxLocalReplica::new(1, PxLocalReplicaRole::Leader), )); store.start().await.unwrap(); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![]))); kv.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); Self { store, kv } } diff --git a/app/crowdb-chunk-kv-server/tests/common/root_process.rs b/app/crowdb-chunk-kv-server/tests/common/root_process.rs new file mode 100644 index 000000000..75537ffdd --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/root_process.rs @@ -0,0 +1,167 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, Checkpoint, Partition, PartitionConfig, PartitionId, PartitionRange, + StreamPartitionJournal, +}; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamRecord, StreamRecordState, +}; +use crowdb_paxos_kv_client::{ + PaxosKvClient, StreamProcessAuthority, StreamProcessLeaseClient, StreamProcessLeaseTiming, +}; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, + PartitionArtifact, +}; +use crowdb_protocol::chunk_stream::{StreamName, StreamPurpose}; +use crowdb_protocol::key::{StreamProcessLeaseKey, TextKey}; +use crowdb_tree_ffi::chunk::stage::{StageId, StageRecord, StageState}; +use std::{ + sync::Arc, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +pub fn entry(epoch: u64) -> ChunkKvRangeCatalogEntry { + ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 1 }, + range: KeyRange { + start: vec![], + end: None, + }, + owner: OwnerDescriptor { + instance_id: epoch, + rpc_endpoint: "127.0.0.1:1".into(), + }, + owner_epoch: epoch, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 42, + stream_name: StreamName { high: 1, low: 2 }, + tail_overlay: None, + }, + transition_id: None, + } +} +pub fn stage() -> StageRecord { + StageRecord { + tree_id: 42, + id: StageId { + owner_epoch: 1, + operation_high: 3, + operation_low: 4, + }, + revision: 1, + chunk_high: 2_u64 << 56, + chunk_low: 10, + capacity: 4096, + purpose: 2, + state: StageState::Reserved, + manifest_generation: 0, + packs: vec![], + } +} +pub fn next(old: &StageRecord, state: StageState) -> StageRecord { + let mut next = old.clone(); + next.state = state; + next.revision += 1; + next +} +pub fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + .try_into() + .unwrap() +} +pub async fn process(kv: &Arc<PaxosKvClient>, instance: u64) -> Arc<StreamProcessAuthority> { + let now = now(); + let lease = StreamProcessLeaseClient::new(kv.clone()) + .claim(StreamPurpose::Wal, instance, [1; 16], None, now, 300_000) + .await + .unwrap(); + Arc::new( + StreamProcessAuthority::new(lease, StreamProcessLeaseTiming::default(), now, Instant::now()).unwrap(), + ) +} +// Deterministically install an already elapsed authoritative crash exclusion boundary. +// The old local process is fenced before the metadata fixture changes the clock record. +pub async fn expire(kv: &Arc<PaxosKvClient>, process: &StreamProcessAuthority) { + process.fence(); + let mut lease = process.status(Instant::now()).lease; + lease.expires_at_ms = now() - 60_000; + let key = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id: lease.identity.instance_id, + } + .to_path(); + kv.put(0, 0, key.as_bytes(), &serde_json::to_vec(&lease).unwrap(), None) + .await + .unwrap(); +} +pub async fn recovered(epoch: u64) -> Partition { + let store = Arc::new(MemoryStreamStore::new(4096)); + let name = entry(epoch).artifact.stream_name; + let stream = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Wal, + stream_name: name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: now(), + }, + epoch, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + Partition::recover_prepared_assignment( + PartitionId { high: 1, low: 1 }, + PartitionRange::default(), + epoch, + Checkpoint { + tree_id: 42, + tree_manifest: 0, + root_manifest_generation: 0, + applied_seq: 0, + stream_name: name, + stream_generation: 1, + replay_offset: 0, + }, + PartitionConfig::default(), + Arc::new(MemoryPartitionTree::with_tree_id(42)), + Arc::new(StreamPartitionJournal::new(stream, name).unwrap()), + ) + .await + .unwrap() +} + +// Age the already committed operation receipt without waiting for wall time. +// The native final-owner release must have created this exact proof first. +pub async fn age_operation_release(kv: &PaxosKvClient, stage: StageId) { + use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; + let mut key = b"\0crowdb/chunk-kv/root/v1/".to_vec(); + key.extend_from_slice(&42_u64.to_be_bytes()); + key.extend_from_slice(b"/stage-quiesced/"); + for word in [stage.owner_epoch, stage.operation_high, stage.operation_low] { + key.extend_from_slice(&word.to_be_bytes()); + } + let GetOutcome::Found { value, revision } = + kv.get(0, 1, &key, ReadMode::Linearizable, None).await.unwrap() + else { + panic!("native operation release has no durable receipt"); + }; + let mut bytes = value.to_vec(); + assert_eq!(bytes.len(), 84); + bytes[8..16].copy_from_slice(&(now() - 60_000).to_le_bytes()); + let checksum = crowdb_tree_ffi::crc32c(&bytes[..80]); + bytes[80..].copy_from_slice(&checksum.to_le_bytes()); + kv.put_cas(0, 1, &key, &bytes, revision).await.unwrap(); +} diff --git a/app/crowdb-chunk-kv-server/tests/config_test.rs b/app/crowdb-chunk-kv-server/tests/config_test.rs index b39d5906f..072e2209d 100644 --- a/app/crowdb-chunk-kv-server/tests/config_test.rs +++ b/app/crowdb-chunk-kv-server/tests/config_test.rs @@ -57,11 +57,15 @@ fn defaults_close_the_documented_timing_contract() { assert_eq!(config.storage.stream_writer_lease_ms, 30_000); assert_eq!(config.storage.tree_chunk_capacity_bytes, 256 * 1024 * 1024); assert_eq!(config.storage.tree_strip_capacity_bytes, 0); + assert_eq!(config.storage.tree_pin_release_slots, 1_048_576); assert_eq!(config.storage.stream_chunk_capacity_bytes, 256 * 1024 * 1024); assert_eq!(config.storage.diskio_connections_per_endpoint, 1); assert_eq!(config.storage.diskio_rpc_workers, 2); assert_eq!(config.rpc_workers, 2); - assert_eq!(config.rpc_advertise_addr, "127.0.0.1:15200"); + assert_eq!( + config.rpc_advertise_addr, + format!("127.0.0.1:{}", crowdb_protocol::CHUNK_KV_RPC_BASE) + ); config.monitor.self_fence_margin_ms = 3_000; assert!(matches!(config.validate(), Err(ConfigError::Invalid(_)))); @@ -179,3 +183,32 @@ fn data_weight_is_reserved_until_tree_range_metrics_are_available() { config.balance.byte_weight_percent = 80; assert!(config.validate().is_err()); } + +#[test] +fn storage_defaults_round_trip_without_a_tree_mode_switch() { + let config = ChunkKvServerConfig { + instance_id: 1, + ..Default::default() + }; + config.validate().unwrap(); + let encoded = toml::to_string(&config).unwrap(); + let restored: ChunkKvServerConfig = toml::from_str(&encoded).unwrap(); + assert_eq!(config, restored); +} + +#[test] +fn pin_release_capacity_rejects_zero_and_the_native_closed_bit() { + let mut config = ChunkKvServerConfig { + instance_id: 1, + ..Default::default() + }; + for capacity in [0, usize::MAX] { + config.storage.tree_pin_release_slots = capacity; + let error = config.validate().unwrap_err(); + assert!(error.to_string().contains("tree_pin_release_slots"), "{error}"); + } + config.storage.tree_pin_release_slots = 64; + config.validate().unwrap(); + let restored: ChunkKvServerConfig = toml::from_str(&toml::to_string(&config).unwrap()).unwrap(); + assert_eq!(restored.storage.tree_pin_release_slots, 64); +} diff --git a/app/crowdb-chunk-kv-server/tests/legacy_retirement/allocation.rs b/app/crowdb-chunk-kv-server/tests/legacy_retirement/allocation.rs new file mode 100644 index 000000000..364d5fca7 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/legacy_retirement/allocation.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::*; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn overflowing_physical_layout_is_corruption_before_drain_or_delete() { + let metadata = metadata::TestMetadata::start().await; + let owner = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(7), owner.clone()) + .await + .unwrap(); + old.store().publish(42, 0, 7, 1, manifest()).unwrap(); + old.store().publish(42, 1, 7, 2, &typed_root()).unwrap(); + expire(&metadata.client, &owner).await; + let current = process(&metadata.client, 20).await; + let physical = Arc::new(TestPhysical::new()); + physical.set_physical_layout(u32::MAX, u32::MAX); + assert_eq!( + collect( + metadata.client.clone(), + current, + physical.clone(), + physical.clone(), + 64, + ) + .await, + Err(CtError::Corruption) + ); + assert_eq!(physical.drains.load(Ordering::Relaxed), 0); + assert_eq!(physical.deletes.load(Ordering::Relaxed), 0); + assert!(!physical.all_deleted()); + assert_eq!( + old.store().load_legacy_migration(42, 1).unwrap().unwrap(), + manifest() + ); +} diff --git a/app/crowdb-chunk-kv-server/tests/legacy_retirement/readers.rs b/app/crowdb-chunk-kv-server/tests/legacy_retirement/readers.rs new file mode 100644 index 000000000..2104c0859 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/legacy_retirement/readers.rs @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::*; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_later_registered_reader_survives_publication_writer_exclusion() { + let metadata = metadata::TestMetadata::start().await; + let owner = process(&metadata.client, 10).await; + let original = open(metadata.client.clone(), 0, 1, &entry(7), owner.clone()) + .await + .unwrap(); + original.store().publish(42, 0, 7, 1, manifest()).unwrap(); + original + .store() + .stage_protection() + .unwrap() + .writer_quiesced(42, 7) + .unwrap(); + let reader_process = process(&metadata.client, 30).await; + let reader = open(metadata.client.clone(), 0, 1, &entry(7), reader_process.clone()) + .await + .unwrap(); + assert_eq!( + reader + .store() + .load(42, crowdb_tree_ffi::RootCatalogObject::Manifest(1)) + .unwrap() + .unwrap(), + manifest() + ); + reader.store().publish(42, 1, 7, 2, &typed_root()).unwrap(); + reader + .store() + .stage_protection() + .unwrap() + .writer_quiesced(42, 7) + .unwrap(); + expire(&metadata.client, &owner).await; + let current = process(&metadata.client, 20).await; + let physical = Arc::new(TestPhysical::new()); + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert!(matches!( + progress.pending, + Some(Pending::LiveProcess { instance: 30, .. }) + )); + assert_eq!(physical.deletes.load(Ordering::Relaxed), 0); + assert_eq!( + reader + .store() + .load(42, crowdb_tree_ffi::RootCatalogObject::Manifest(1)), + Err(CtError::Unavailable) + ); + expire(&metadata.client, &reader_process).await; + for step in 0..16 { + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert!(progress.pending.is_none(), "{progress:?}"); + if progress.complete { + break; + } + assert!(step < 15); + } + assert!(physical.all_deleted()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn scheduled_resume_does_not_start_a_namespace_upgrade() { + let metadata = metadata::TestMetadata::start().await; + let current = process(&metadata.client, 20).await; + let physical = Arc::new(TestPhysical::new()); + let progress = crowdb_chunk_kv_server::storage::legacy_retirement::resume_legacy_storage_for_tests( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert!(progress.is_none()); + let old = open(metadata.client.clone(), 0, 1, &entry(7), current.clone()) + .await + .unwrap(); + old.store().publish(42, 0, 7, 1, manifest()).unwrap(); + assert_eq!(physical.deletes.load(Ordering::Relaxed), 0); +} diff --git a/app/crowdb-chunk-kv-server/tests/legacy_retirement_test.rs b/app/crowdb-chunk-kv-server/tests/legacy_retirement_test.rs new file mode 100644 index 000000000..c6fdfea96 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/legacy_retirement_test.rs @@ -0,0 +1,288 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_chunk_kv_server::storage::{ + legacy_retirement::{collect_legacy_storage_for_tests as collect, LegacyRetirementPending as Pending}, + open_root_catalog_with_process_for_tests as open, +}; +use crowdb_tree_ffi::CtError; +use std::sync::{atomic::Ordering, Arc}; +#[path = "common/legacy_retirement.rs"] +mod fixture; +#[path = "common/metadata.rs"] +mod metadata; +use fixture::{entry, expire, process}; +use fixture::{manifest, typed_root, TestPhysical}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn quiesced_writer_is_not_a_reader_exclusion_and_deleted_bytes_are_confirmed() { + let metadata = metadata::TestMetadata::start().await; + let old_process = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(7), old_process.clone()) + .await + .unwrap(); + old.store().publish(42, 0, 7, 1, manifest()).unwrap(); + old.store().prepare_legacy_migration(42, 1, manifest()).unwrap(); + old.store().publish(42, 1, 7, 2, &typed_root()).unwrap(); + old.store() + .stage_protection() + .unwrap() + .writer_quiesced(42, 7) + .unwrap(); + let current = process(&metadata.client, 20).await; + let physical = Arc::new(TestPhysical::new()); + let bytes = physical.physical_bytes(); + let count = physical.count(); + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert!(matches!( + progress.pending, + Some(Pending::LiveProcess { + tree: 42, + generation: 1, + .. + }) + )); + assert_eq!(physical.deletes.load(Ordering::Relaxed), 0); + expire(&metadata.client, &old_process).await; + let mut reclaimed = 0; + for step in 0..128 { + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 4, + ) + .await + .unwrap(); + assert!(progress.pending.is_none(), "{progress:?}"); + reclaimed += progress.reclaimed_bytes; + if progress.complete { + break; + } + assert!(step < 127, "bounded invocations did not finish"); + } + assert!(physical.all_deleted()); + assert_eq!(physical.deletes.load(Ordering::Relaxed), count); + assert_eq!(physical.drains.load(Ordering::Relaxed), count); + assert_eq!(reclaimed, bytes); + assert_eq!( + old.store().load_legacy_migration(42, 1).unwrap().unwrap(), + manifest() + ); + let again = collect( + metadata.client.clone(), + current, + physical.clone(), + physical.clone(), + 4, + ) + .await + .unwrap(); + assert!(again.complete); + assert_eq!(again.reclaimed_bytes, 0); + assert_eq!(physical.deletes.load(Ordering::Relaxed), count); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn legacy_generation_pin_must_be_released_and_new_pins_are_fenced() { + let metadata = metadata::TestMetadata::start().await; + let owner = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(7), owner.clone()) + .await + .unwrap(); + old.store().publish(42, 0, 7, 1, manifest()).unwrap(); + old.store().pin_generation(42, 9, 3, 1).unwrap(); + old.store().publish(42, 1, 7, 2, &typed_root()).unwrap(); + let current = process(&metadata.client, 20).await; + let physical = Arc::new(TestPhysical::new()); + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert_eq!( + progress.pending, + Some(Pending::GenerationPin { + tree: 42, + generation: 1 + }) + ); + assert_eq!(old.store().pin_generation(42, 9, 4, 1), Err(CtError::Unavailable)); + old.store().unpin_generation(42, 9, 3).unwrap(); + expire(&metadata.client, &owner).await; + for step in 0..16 { + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert!(progress.pending.is_none(), "{progress:?}"); + if progress.complete { + break; + } + assert!(step < 15); + } + assert!(physical.all_deleted()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unknown_delete_reply_resumes_the_exact_claim_without_a_second_delete() { + let metadata = metadata::TestMetadata::start().await; + let owner = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(7), owner.clone()) + .await + .unwrap(); + old.store().publish(42, 0, 7, 1, manifest()).unwrap(); + old.store().publish(42, 1, 7, 2, &typed_root()).unwrap(); + expire(&metadata.client, &owner).await; + let current = process(&metadata.client, 20).await; + let physical = Arc::new(TestPhysical::new()); + physical.lose_reply.store(true, Ordering::Relaxed); + assert_eq!( + collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64 + ) + .await, + Err(CtError::Unavailable) + ); + assert_eq!(physical.deletes.load(Ordering::Relaxed), 1); + for step in 0..16 { + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert!(progress.pending.is_none(), "{progress:?}"); + if progress.complete { + break; + } + assert!(step < 15); + } + assert!(physical.all_deleted()); + assert_eq!(physical.deletes.load(Ordering::Relaxed), physical.count()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn historical_manifest_without_publication_provenance_stays_durably_pending() { + let metadata = metadata::TestMetadata::start().await; + let old = crowdb_chunk_kv_server::storage::open_root_catalog_for_tests( + metadata.client.clone(), + 0, + 1, + &entry(7), + ) + .await + .unwrap(); + old.publish(42, 0, 7, 1, manifest()).unwrap(); + old.publish(42, 1, 7, 2, &typed_root()).unwrap(); + let current = process(&metadata.client, 20).await; + let physical = Arc::new(TestPhysical::new()); + for _ in 0..2 { + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert_eq!( + progress.pending, + Some(Pending::MissingOwnerProof { + tree: 42, + generation: 1 + }) + ); + assert!(!progress.complete); + } + assert_eq!(physical.deletes.load(Ordering::Relaxed), 0); + assert_eq!(old.load_legacy_migration(42, 1).unwrap().unwrap(), manifest()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn foreign_legacy_root_blocks_the_namespace_and_shared_chunks_delete_once() { + let metadata = metadata::TestMetadata::start().await; + let owner = process(&metadata.client, 10).await; + let parent = open(metadata.client.clone(), 0, 1, &entry(7), owner.clone()) + .await + .unwrap(); + parent.store().publish(42, 0, 7, 1, manifest()).unwrap(); + let mut child_entry = entry(7); + child_entry.artifact.tree_id = 43; + let child = open(metadata.client.clone(), 0, 1, &child_entry, owner.clone()) + .await + .unwrap(); + let foreign = fixture::foreign_manifest(43); + child.store().publish(43, 0, 7, 1, &foreign).unwrap(); + parent.store().publish(42, 1, 7, 2, &typed_root()).unwrap(); + expire(&metadata.client, &owner).await; + let current = process(&metadata.client, 20).await; + let physical = Arc::new(TestPhysical::new()); + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert_eq!(progress.pending, Some(Pending::LegacyRoot { tree: 43 })); + assert_eq!(physical.deletes.load(Ordering::Relaxed), 0); + let replacement = open(metadata.client.clone(), 0, 1, &child_entry, current.clone()) + .await + .unwrap(); + replacement + .store() + .publish(43, 1, 7, 2, &fixture::foreign_typed_root(43)) + .unwrap(); + for step in 0..16 { + let progress = collect( + metadata.client.clone(), + current.clone(), + physical.clone(), + physical.clone(), + 64, + ) + .await + .unwrap(); + assert!(progress.pending.is_none(), "{progress:?}"); + if progress.complete { + break; + } + assert!(step < 15); + } + assert!(physical.all_deleted()); + assert_eq!(physical.deletes.load(Ordering::Relaxed), physical.count()); +} + +#[path = "legacy_retirement/readers.rs"] +mod readers; + +#[path = "legacy_retirement/allocation.rs"] +mod allocation; diff --git a/app/crowdb-chunk-kv-server/tests/perf_log_test.rs b/app/crowdb-chunk-kv-server/tests/perf_log_test.rs index da464ab31..2ec54eb03 100644 --- a/app/crowdb-chunk-kv-server/tests/perf_log_test.rs +++ b/app/crowdb-chunk-kv-server/tests/perf_log_test.rs @@ -1,16 +1,17 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_chunk_kv_server::metrics::start_stream_perf_log; +use crowdb_chunk_kv_server::metrics::create_perf_log; use crowdb_test_harness::test_dirs::tempdir_in_test_data; #[tokio::test] async fn service_perf_log_flushes_registered_stream_histograms_on_shutdown() { let dir = tempdir_in_test_data("stream-perf-log"); let path = dir.path().to_str().unwrap(); - assert!(start_stream_perf_log(path, 0, 10, 2).unwrap().is_none()); + assert!(create_perf_log(path, 0, 10, 2).unwrap().is_none()); assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); - let mut runner = start_stream_perf_log(path, 1, 10, 2).unwrap().unwrap(); + let mut runner = create_perf_log(path, 1, 10, 2).unwrap().unwrap(); + runner.start(); assert!(runner .registry() .lock() diff --git a/app/crowdb-chunk-kv-server/tests/process_session_test.rs b/app/crowdb-chunk-kv-server/tests/process_session_test.rs index f1b363ec1..a68d5fda8 100644 --- a/app/crowdb-chunk-kv-server/tests/process_session_test.rs +++ b/app/crowdb-chunk-kv-server/tests/process_session_test.rs @@ -5,7 +5,7 @@ mod fixture; use crowdb_chunk_kv_server::serving::process::ProcessLeaseSession; -use crowdb_kv_client::{GetOutcome, ReadMode, StreamProcessLeaseClient, StreamProcessLeaseError}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode, StreamProcessLeaseClient, StreamProcessLeaseError}; use crowdb_protocol::{ chunk_stream::StreamPurpose, common::{ChunkKvExtra, InstanceValue}, @@ -280,7 +280,7 @@ async fn queued_heartbeat_cannot_extend_the_original_fence_deadline() { assert!(authority.status(Instant::now()).fenced); } -async fn wait_for_pending_renewal(group: &crowdb_kv::cluster::group::PxGroup) { +async fn wait_for_pending_renewal(group: &crowdb_paxos_kv::cluster::group::PxGroup) { let lease_key = StreamProcessLeaseKey { purpose: StreamPurpose::Wal, instance_id: 7, diff --git a/app/crowdb-chunk-kv-server/tests/root_catalog_legacy_migration_test.rs b/app/crowdb-chunk-kv-server/tests/root_catalog_legacy_migration_test.rs new file mode 100644 index 000000000..eed566ff4 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/root_catalog_legacy_migration_test.rs @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_chunk_kv_server::storage::open_root_catalog_for_tests; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, + PartitionArtifact, +}; +use crowdb_protocol::chunk_stream::StreamName; +use crowdb_tree_ffi::{CtError, RootCatalogObject}; +#[path = "common/metadata.rs"] +mod metadata; + +fn entry() -> ChunkKvRangeCatalogEntry { + ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 1 }, + range: KeyRange { + start: vec![], + end: None, + }, + owner: OwnerDescriptor { + instance_id: 7, + rpc_endpoint: "127.0.0.1:1".into(), + }, + owner_epoch: 7, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 42, + stream_name: StreamName { high: 1, low: 2 }, + tail_overlay: None, + }, + transition_id: None, + } +} +fn manifest() -> &'static [u8] { + let fixture = include_bytes!("../../../lib/crowdb-tree/tests/data/legacy-chunk-v7.bin"); + let length = usize::try_from(u64::from_le_bytes(fixture[8..16].try_into().unwrap())).unwrap(); + &fixture[16..16 + length] +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn receipt_is_idempotent_and_survives_old_manifest_retention() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry()) + .await + .unwrap(); + store.publish(42, 0, 7, 1, manifest()).unwrap(); + store.prepare_legacy_migration(42, 1, manifest()).unwrap(); + store.prepare_legacy_migration(42, 1, manifest()).unwrap(); + for generation in 2..=4 { + store + .publish(42, generation - 1, 7, generation, b"new root") + .unwrap(); + } + assert_eq!(store.reclaim_before(42, 4), 2); + assert!(store.load(42, RootCatalogObject::Manifest(1)).unwrap().is_none()); + assert_eq!(store.load_legacy_migration(42, 1).unwrap().unwrap(), manifest()); + store.prepare_legacy_migration(42, 1, manifest()).unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn changed_generation_or_corrupt_bytes_cannot_prepare_a_receipt() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry()) + .await + .unwrap(); + store.publish(42, 0, 7, 1, manifest()).unwrap(); + assert_eq!( + store.prepare_legacy_migration(42, 2, manifest()), + Err(CtError::InvalidArgument) + ); + let mut damaged = manifest().to_vec(); + let last = damaged.len() - 1; + damaged[last] ^= 1; + assert_eq!( + store.prepare_legacy_migration(42, 1, &damaged), + Err(CtError::Corruption) + ); + store.publish(42, 1, 7, 2, b"replacement root").unwrap(); + assert_eq!( + store.prepare_legacy_migration(42, 1, manifest()), + Err(CtError::Unavailable) + ); + assert!(store.load_legacy_migration(42, 1).unwrap().is_none()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_generation_pin_and_receipt_preserve_both_authority_updates() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry()) + .await + .unwrap(); + store.publish(42, 0, 7, 1, manifest()).unwrap(); + let writer = store.clone(); + let receipt = tokio::task::spawn_blocking(move || writer.prepare_legacy_migration(42, 1, manifest())); + let reader = store.clone(); + let pin = tokio::task::spawn_blocking(move || reader.pin_generation(42, 9, 3, 1)); + receipt.await.unwrap().unwrap(); + pin.await.unwrap().unwrap(); + assert_eq!(store.load_legacy_migration(42, 1).unwrap().unwrap(), manifest()); + assert_eq!(store.reclaim_before(42, 4), 0); + store.unpin_generation(42, 9, 3).unwrap(); +} diff --git a/app/crowdb-chunk-kv-server/tests/root_catalog_prepared_staging_test.rs b/app/crowdb-chunk-kv-server/tests/root_catalog_prepared_staging_test.rs new file mode 100644 index 000000000..bff685e62 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/root_catalog_prepared_staging_test.rs @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv_server::{storage::open_root_catalog_with_process_for_tests, Group0ControlStore}; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, Id128, TailOverlayArtifact, TransferPhase, + TransferReadinessLimits, TransferTransition, +}; +use crowdb_tree_ffi::{ + chunk::stage::{StagePack, StageState}, + CtError, +}; + +#[path = "common/metadata.rs"] +mod metadata; +#[allow(dead_code)] +#[path = "common/root_process.rs"] +mod root_process; + +fn preparation() -> (TransferTransition, ChunkKvRangeCatalogEntry) { + let source = root_process::entry(1); + let mut target = root_process::entry(2); + target.state = ChunkKvRangeCatalogPartitionState::Prepared; + target.transition_id = Some(Id128 { high: 3, low: 4 }); + target.artifact.stream_name.low = 3; + target.artifact.tail_overlay = Some(TailOverlayArtifact { + source_partition_id: source.partition_id, + source_epoch: 1, + source_stream_name: source.artifact.stream_name, + source_stream_generation: 1, + replay_offset: 0, + cutover_offset: 0, + base_root_manifest_generation: 1, + base_tree_manifest: 1, + base_applied_seq: 0, + cutover_seq: 0, + target_stream_start_seq: 1, + }); + let transition = TransferTransition { + transition_id: target.transition_id.unwrap(), + partition_id: target.partition_id, + range: target.range.clone(), + source: source.owner, + source_epoch: source.owner_epoch, + target: target.owner.clone(), + target_epoch: target.owner_epoch, + artifact: source.artifact, + target_artifact: target.artifact.clone(), + readiness_limits: TransferReadinessLimits { + max_tail_records: 100, + max_tail_bytes: 1_000_000, + max_estimated_catchup_ms: 1000, + prepare_deadline_ms: 1000, + forwarding_grace_ms: 1000, + }, + planned_at_ms: 0, + old_grant_expires_at_ms: 100, + phase: TransferPhase::TargetPreparing, + release_proof: None, + readiness_proof: None, + catchup_proof: None, + failure: None, + }; + (transition, target) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn durable_preparation_stages_without_fencing_source_and_abort_stops_next_window() { + let metadata = metadata::TestMetadata::start().await; + let source_process = root_process::process(&metadata.client, 1).await; + let source = open_root_catalog_with_process_for_tests( + metadata.client.clone(), + 0, + 1, + &root_process::entry(1), + source_process, + ) + .await + .unwrap(); + source.store().publish(42, 0, 1, 1, b"source-one").unwrap(); + let (mut transition, entry) = preparation(); + let control = Group0ControlStore::from_client(metadata.client.clone()); + let revision = control.persist_transfer_transition(&transition, 0).await.unwrap(); + let target_process = root_process::process(&metadata.client, 2).await; + let target = + open_root_catalog_with_process_for_tests(metadata.client.clone(), 0, 1, &entry, target_process) + .await + .unwrap(); + let mut stage = root_process::stage(); + stage.id.owner_epoch = 2; + target.store().compare_stage(42, 2, None, Some(&stage)).unwrap(); + source.store().publish(42, 1, 1, 2, b"source-two").unwrap(); + assert_eq!( + target.store().publish(42, 2, 2, 3, b"premature-target"), + Err(CtError::Unavailable) + ); + let mut writing = root_process::next(&stage, StageState::Writing); + writing.packs.push(StagePack { + offset: 0, + length: 4096, + checksum: 1, + }); + target + .store() + .compare_stage(42, 2, Some(&stage), Some(&writing)) + .unwrap(); + transition.phase = TransferPhase::Aborted; + transition.failure = Some("preparation cancelled".into()); + control + .persist_transfer_transition(&transition, revision) + .await + .unwrap(); + let reloadable = root_process::next(&writing, StageState::Reloadable); + assert_eq!( + target + .store() + .compare_stage(42, 2, Some(&writing), Some(&reloadable)), + Err(CtError::Unavailable) + ); + source.store().publish(42, 2, 1, 3, b"source-three").unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn preparation_requires_exact_recovery_base_and_fixed_target_process() { + let metadata = metadata::TestMetadata::start().await; + let source_process = root_process::process(&metadata.client, 1).await; + let source = open_root_catalog_with_process_for_tests( + metadata.client.clone(), + 0, + 1, + &root_process::entry(1), + source_process, + ) + .await + .unwrap(); + let (transition, entry) = preparation(); + Group0ControlStore::from_client(metadata.client.clone()) + .persist_transfer_transition(&transition, 0) + .await + .unwrap(); + let target_process = root_process::process(&metadata.client, 2).await; + let mut wrong = entry.clone(); + wrong.artifact.tail_overlay.as_mut().unwrap().base_tree_manifest += 1; + assert!( + open_root_catalog_with_process_for_tests(metadata.client.clone(), 0, 1, &wrong, target_process,) + .await + .is_err() + ); + let foreign_process = root_process::process(&metadata.client, 3).await; + assert!( + open_root_catalog_with_process_for_tests(metadata.client.clone(), 0, 1, &entry, foreign_process,) + .await + .is_err() + ); + source.store().publish(42, 0, 1, 1, b"still-source").unwrap(); +} diff --git a/app/crowdb-chunk-kv-server/tests/root_catalog_process_test.rs b/app/crowdb-chunk-kv-server/tests/root_catalog_process_test.rs new file mode 100644 index 000000000..8709415b1 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/root_catalog_process_test.rs @@ -0,0 +1,555 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_chunk_kv_server::storage::open_root_catalog_with_process_for_tests as open; +use crowdb_tree_ffi::chunk::protection::{reference_checksum, CandidateState, ManifestCandidate}; +use crowdb_tree_ffi::chunk::stage::{StageId, StagePack, StageState}; +use crowdb_tree_ffi::{CtError, RootCatalogObject}; +#[path = "common/root_process.rs"] +mod fixture; +#[path = "common/metadata.rs"] +mod metadata; +use fixture::{entry, expire, next, process, recovered, stage}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn saturated_pin_pruning_commits_bounded_progress() { + let metadata = metadata::TestMetadata::start().await; + let old_process = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(1), old_process.clone()) + .await + .unwrap(); + let reserved = stage(); + old.store().compare_stage(42, 1, None, Some(&reserved)).unwrap(); + let mut writing = next(&reserved, StageState::Writing); + writing.packs.push(StagePack { + offset: 0, + length: 100, + checksum: 7, + }); + old.store() + .compare_stage(42, 1, Some(&reserved), Some(&writing)) + .unwrap(); + let ready = next(&writing, StageState::Reloadable); + old.store() + .compare_stage(42, 1, Some(&writing), Some(&ready)) + .unwrap(); + let protection = old.store().stage_protection().unwrap(); + for low in 100..164 { + protection + .pin_stage( + 42, + ready.id, + StageId { + operation_low: low, + ..ready.id + }, + ) + .unwrap(); + } + let replacement = open( + metadata.client.clone(), + 0, + 1, + &entry(2), + process(&metadata.client, 20).await, + ) + .await + .unwrap(); + replacement + .complete_recovery(&recovered(2).await, 0) + .await + .unwrap(); + expire(&metadata.client, &old_process).await; + let protection = replacement.store().stage_protection().unwrap(); + let first = protection.prune_stage_pins(42, ready.id, None, 64).unwrap(); + assert_eq!(first.released, 62); + assert!(first.more); + let second = protection + .prune_stage_pins(42, ready.id, first.after, 64) + .unwrap(); + assert_eq!(second.released, 3); + assert!(!second.more); + replacement + .store() + .compare_stage(42, 2, Some(&ready), Some(&next(&ready, StageState::Retired))) + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn new_epoch_and_completed_recovery_do_not_erase_a_live_old_reader() { + let metadata = metadata::TestMetadata::start().await; + let old_process = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(1), old_process.clone()) + .await + .unwrap(); + let reserved = stage(); + old.store().compare_stage(42, 1, None, Some(&reserved)).unwrap(); + let mut writing = next(&reserved, StageState::Writing); + writing.packs.push(StagePack { + offset: 0, + length: 100, + checksum: 7, + }); + old.store() + .compare_stage(42, 1, Some(&reserved), Some(&writing)) + .unwrap(); + let ready = next(&writing, StageState::Reloadable); + old.store() + .compare_stage(42, 1, Some(&writing), Some(&ready)) + .unwrap(); + let pin = StageId { + operation_low: 99, + ..ready.id + }; + let protection = old.store().stage_protection().unwrap(); + protection.pin_stage(42, ready.id, pin).unwrap(); + protection.unpin_stage(42, ready.id, ready.id).unwrap(); + protection.writer_quiesced(42, 1).unwrap(); + assert_eq!(protection.check_read_owner(), Ok(())); + let replacement = open( + metadata.client.clone(), + 0, + 1, + &entry(2), + process(&metadata.client, 20).await, + ) + .await + .unwrap(); + replacement + .complete_recovery(&recovered(2).await, 0) + .await + .unwrap(); + let reader = replacement.store().stage_protection().unwrap(); + let progress = reader.prune_stage_pins(42, ready.id, None, 64).unwrap(); + assert_eq!(progress.released, 0); + let retired = next(&ready, StageState::Retired); + assert_eq!( + replacement + .store() + .compare_stage(42, 2, Some(&ready), Some(&retired)), + Err(CtError::Unavailable) + ); + protection.unpin_stage(42, ready.id, pin).unwrap(); + replacement + .store() + .compare_stage(42, 2, Some(&ready), Some(&retired)) + .unwrap(); + assert_eq!( + protection.pin_stage( + 42, + ready.id, + StageId { + operation_low: 100, + ..pin + } + ), + Err(CtError::Unavailable) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn actual_recovery_and_authoritative_process_exclusion_release_crash_pins() { + let metadata = metadata::TestMetadata::start().await; + let old_process = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(1), old_process.clone()) + .await + .unwrap(); + let reserved = stage(); + old.store().compare_stage(42, 1, None, Some(&reserved)).unwrap(); + let replacement = open( + metadata.client.clone(), + 0, + 1, + &entry(2), + process(&metadata.client, 20).await, + ) + .await + .unwrap(); + let protection = replacement.store().stage_protection().unwrap(); + assert_eq!( + protection.prune_stage_pins(42, reserved.id, None, 64), + Err(CtError::Unavailable) + ); + replacement + .complete_recovery(&recovered(2).await, 0) + .await + .unwrap(); + assert_eq!( + protection + .prune_stage_pins(42, reserved.id, None, 64) + .unwrap() + .released, + 0 + ); + expire(&metadata.client, &old_process).await; + assert_eq!( + old.store().stage_protection().unwrap().check_read_owner(), + Err(CtError::Unavailable) + ); + let progress = protection.prune_stage_pins(42, reserved.id, None, 64).unwrap(); + assert_eq!(progress.examined, 1); + assert_eq!(progress.released, 1); + assert_eq!( + protection + .prune_stage_pins(42, reserved.id, None, 64) + .unwrap() + .released, + 0 + ); + replacement + .store() + .compare_stage( + 42, + 2, + Some(&reserved), + Some(&next(&reserved, StageState::Retired)), + ) + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn same_epoch_reopen_cannot_steal_a_live_writer_or_reuse_its_recovery_receipt() { + let metadata = metadata::TestMetadata::start().await; + let old_process = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(1), old_process.clone()) + .await + .unwrap(); + old.complete_recovery(&recovered(1).await, 0).await.unwrap(); + for generation in 1..=3 { + old.store() + .publish(42, generation - 1, 1, generation, b"root") + .unwrap(); + } + assert!( + open(metadata.client.clone(), 0, 1, &entry(1), old_process.clone()) + .await + .is_err() + ); + let reserved = stage(); + old.store().compare_stage(42, 1, None, Some(&reserved)).unwrap(); + expire(&metadata.client, &old_process).await; + let replacement = open( + metadata.client.clone(), + 0, + 1, + &entry(1), + process(&metadata.client, 20).await, + ) + .await + .unwrap(); + assert_eq!( + replacement + .store() + .stage_protection() + .unwrap() + .prune_stage_pins(42, reserved.id, None, 64), + Err(CtError::Unavailable) + ); + assert_eq!(old.store().reclaim_before(42, 3), 0); + assert_eq!(old.store().pin_generation(42, 9, 9, 1), Err(CtError::Unavailable)); + assert_eq!( + replacement + .store() + .load(42, RootCatalogObject::Manifest(1)) + .unwrap(), + Some(b"root".to_vec()) + ); + replacement + .complete_recovery(&recovered(1).await, 3) + .await + .unwrap(); + assert_eq!(replacement.store().reclaim_before(42, 3), 1); + assert_eq!( + replacement + .store() + .stage_protection() + .unwrap() + .prune_stage_pins(42, reserved.id, None, 64) + .unwrap() + .released, + 1 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn recovered_new_owner_aborts_old_candidate_before_releasing_its_references() { + let metadata = metadata::TestMetadata::start().await; + let old_process = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(1), old_process.clone()) + .await + .unwrap(); + let reserved = stage(); + old.store().compare_stage(42, 1, None, Some(&reserved)).unwrap(); + let mut writing = next(&reserved, StageState::Writing); + writing.packs.push(StagePack { + offset: 0, + length: 100, + checksum: 7, + }); + old.store() + .compare_stage(42, 1, Some(&reserved), Some(&writing)) + .unwrap(); + let ready = next(&writing, StageState::Reloadable); + old.store() + .compare_stage(42, 1, Some(&writing), Some(&ready)) + .unwrap(); + let id = StageId { + operation_low: 66, + ..ready.id + }; + let candidate = ManifestCandidate { + id, + expected_generation: 0, + reference_count: 1, + reference_checksum: reference_checksum(0, &[ready.id]), + manifest: b"pending-root".to_vec(), + }; + let protection = old.store().stage_protection().unwrap(); + protection.begin_manifest_candidate(42, &candidate).unwrap(); + protection + .prepare_manifest_references(42, id, 0, &[ready.id]) + .unwrap(); + let replacement = open( + metadata.client.clone(), + 0, + 1, + &entry(2), + process(&metadata.client, 20).await, + ) + .await + .unwrap(); + let next_protection = replacement.store().stage_protection().unwrap(); + assert_eq!( + next_protection.recover_manifest_candidate(42, id), + Err(CtError::Unavailable) + ); + replacement + .complete_recovery(&recovered(2).await, 0) + .await + .unwrap(); + next_protection.recover_manifest_candidate(42, id).unwrap(); + assert_eq!( + next_protection + .load_manifest_candidate(42, id) + .unwrap() + .unwrap() + .state, + CandidateState::Aborted + ); + assert_eq!( + protection.commit_manifest_candidate(42, id), + Err(CtError::Unavailable) + ); + next_protection.release_manifest_references(42, id, 0).unwrap(); + next_protection.release_manifest_references(42, id, 0).unwrap(); + let retired = next(&ready, StageState::Retired); + assert_eq!( + replacement + .store() + .compare_stage(42, 2, Some(&ready), Some(&retired)), + Err(CtError::Unavailable) + ); + expire(&metadata.client, &old_process).await; + assert_eq!( + next_protection + .prune_stage_pins(42, ready.id, None, 64) + .unwrap() + .released, + 1 + ); + replacement + .store() + .compare_stage(42, 2, Some(&ready), Some(&retired)) + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unsealed_stage_requires_creator_exclusion_even_after_the_last_pin() { + let metadata = metadata::TestMetadata::start().await; + let old_process = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(1), old_process.clone()) + .await + .unwrap(); + old.complete_recovery(&recovered(1).await, 0).await.unwrap(); + let reserved = stage(); + old.store().compare_stage(42, 1, None, Some(&reserved)).unwrap(); + old.store() + .stage_protection() + .unwrap() + .unpin_stage(42, reserved.id, reserved.id) + .unwrap(); + let retired = next(&reserved, StageState::Retired); + assert_eq!( + old.store().compare_stage(42, 1, Some(&reserved), Some(&retired)), + Err(CtError::Unavailable) + ); + let replacement = open( + metadata.client.clone(), + 0, + 1, + &entry(2), + process(&metadata.client, 20).await, + ) + .await + .unwrap(); + replacement + .complete_recovery(&recovered(2).await, 0) + .await + .unwrap(); + assert_eq!( + replacement + .store() + .compare_stage(42, 2, Some(&reserved), Some(&retired)), + Err(CtError::Unavailable) + ); + expire(&metadata.client, &old_process).await; + replacement + .store() + .compare_stage(42, 2, Some(&reserved), Some(&retired)) + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_before_unknown_stage_creation_fences_its_late_retry() { + let metadata = metadata::TestMetadata::start().await; + let owner = open( + metadata.client.clone(), + 0, + 1, + &entry(1), + process(&metadata.client, 10).await, + ) + .await + .unwrap(); + let reserved = stage(); + let protection = owner.store().stage_protection().unwrap(); + protection.unpin_stage(42, reserved.id, reserved.id).unwrap(); + protection.unpin_stage(42, reserved.id, reserved.id).unwrap(); + assert_eq!( + owner.store().compare_stage(42, 1, None, Some(&reserved)), + Err(CtError::Unavailable) + ); + assert_eq!(owner.store().load_stage(42, reserved.id).unwrap(), None); + let mut created_first = reserved.clone(); + created_first.id.operation_low += 1; + created_first.chunk_low += 1; + owner + .store() + .compare_stage(42, 1, None, Some(&created_first)) + .unwrap(); + protection + .unpin_stage(42, created_first.id, created_first.id) + .unwrap(); + protection + .unpin_stage(42, created_first.id, created_first.id) + .unwrap(); + assert_eq!( + protection.prune_stage_pins(42, created_first.id, None, 64), + Err(CtError::Unavailable) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn terminal_unknown_operation_can_retire_while_its_process_and_tree_keep_serving() { + let metadata = metadata::TestMetadata::start().await; + let process = process(&metadata.client, 10).await; + let owner = open(metadata.client.clone(), 0, 1, &entry(1), process.clone()) + .await + .unwrap(); + owner.complete_recovery(&recovered(1).await, 0).await.unwrap(); + let reserved = stage(); + owner.store().compare_stage(42, 1, None, Some(&reserved)).unwrap(); + let protection = owner.store().stage_protection().unwrap(); + protection.unpin_stage(42, reserved.id, reserved.id).unwrap(); + let retired = next(&reserved, StageState::Retired); + assert_eq!( + owner + .store() + .compare_stage(42, 1, Some(&reserved), Some(&retired)), + Err(CtError::Unavailable) + ); + fixture::age_operation_release(&metadata.client, reserved.id).await; + protection.unpin_stage(42, reserved.id, reserved.id).unwrap(); + assert_eq!(protection.check_read_owner(), Ok(())); + owner + .store() + .compare_stage(42, 1, Some(&reserved), Some(&retired)) + .unwrap(); + let mut ongoing = reserved; + ongoing.id.operation_low += 1; + ongoing.chunk_low += 1; + owner.store().compare_stage(42, 1, None, Some(&ongoing)).unwrap(); + assert_eq!(protection.check_read_owner(), Ok(())); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn drained_writer_reopens_immediately_without_expiring_its_old_readers() { + let metadata = metadata::TestMetadata::start().await; + let process = process(&metadata.client, 10).await; + let old = open(metadata.client.clone(), 0, 1, &entry(1), process.clone()) + .await + .unwrap(); + let old_reader = old.store().stage_protection().unwrap(); + old_reader.writer_quiesced(42, 1).unwrap(); + let replacement = open(metadata.client.clone(), 0, 1, &entry(1), process) + .await + .unwrap(); + assert_eq!(old_reader.check_read_owner(), Ok(())); + assert_eq!( + old.store().compare_stage(42, 1, None, Some(&stage())), + Err(CtError::Unavailable) + ); + replacement + .store() + .compare_stage(42, 1, None, Some(&stage())) + .unwrap(); + // An idempotent late release still refers only to the old incarnation. + old_reader.writer_quiesced(42, 1).unwrap(); + let mut another = stage(); + another.id.operation_low += 1; + another.chunk_low += 1; + replacement + .store() + .compare_stage(42, 1, None, Some(&another)) + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sealed_legacy_stage_can_retire_after_recovery_without_an_invented_process_owner() { + let metadata = metadata::TestMetadata::start().await; + let old = crowdb_chunk_kv_server::storage::open_root_catalog_for_tests( + metadata.client.clone(), + 0, + 1, + &entry(1), + ) + .await + .unwrap(); + let reserved = stage(); + old.compare_stage(42, 1, None, Some(&reserved)).unwrap(); + let mut writing = next(&reserved, StageState::Writing); + writing.packs.push(StagePack { + offset: 0, + length: 100, + checksum: 7, + }); + old.compare_stage(42, 1, Some(&reserved), Some(&writing)).unwrap(); + let ready = next(&writing, StageState::Reloadable); + old.compare_stage(42, 1, Some(&writing), Some(&ready)).unwrap(); + let replacement = open( + metadata.client.clone(), + 0, + 1, + &entry(2), + process(&metadata.client, 20).await, + ) + .await + .unwrap(); + replacement + .complete_recovery(&recovered(2).await, 0) + .await + .unwrap(); + replacement + .store() + .compare_stage(42, 2, Some(&ready), Some(&next(&ready, StageState::Retired))) + .unwrap(); +} diff --git a/app/crowdb-chunk-kv-server/tests/root_catalog_reference_test.rs b/app/crowdb-chunk-kv-server/tests/root_catalog_reference_test.rs new file mode 100644 index 000000000..3c280f1d5 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/root_catalog_reference_test.rs @@ -0,0 +1,460 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv_server::storage::open_root_catalog_for_tests; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, + PartitionArtifact, +}; +use crowdb_protocol::chunk_stream::StreamName; +use crowdb_tree_ffi::chunk::protection::{ + reference_checksum, CandidateState, ManifestCandidate, MAX_REFERENCE_WINDOW, +}; +use crowdb_tree_ffi::chunk::stage::{StageId, StagePack, StageRecord, StageState}; +use crowdb_tree_ffi::{CtError, RootCatalogObject, RootCatalogStore}; +#[path = "common/metadata.rs"] +mod metadata; + +fn entry(epoch: u64) -> ChunkKvRangeCatalogEntry { + ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 1 }, + range: KeyRange { + start: vec![], + end: None, + }, + owner: OwnerDescriptor { + instance_id: epoch, + rpc_endpoint: "127.0.0.1:1".into(), + }, + owner_epoch: epoch, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 42, + stream_name: StreamName { high: 1, low: 2 }, + tail_overlay: None, + }, + transition_id: None, + } +} +fn id(low: u64) -> StageId { + StageId { + owner_epoch: 1, + operation_high: 100, + operation_low: low, + } +} +fn next(old: &StageRecord, state: StageState) -> StageRecord { + let mut value = old.clone(); + value.state = state; + value.revision += 1; + value +} +fn ready(store: &dyn RootCatalogStore, low: u64) -> StageRecord { + let reserved = StageRecord { + tree_id: 42, + id: id(low), + revision: 1, + chunk_high: 2_u64 << 56, + chunk_low: low, + capacity: 4096, + purpose: 2, + state: StageState::Reserved, + manifest_generation: 0, + packs: vec![], + }; + store.compare_stage(42, 1, None, Some(&reserved)).unwrap(); + let mut writing = next(&reserved, StageState::Writing); + writing.packs.push(StagePack { + offset: 0, + length: 4000, + checksum: 7, + }); + store + .compare_stage(42, 1, Some(&reserved), Some(&writing)) + .unwrap(); + let ready = next(&writing, StageState::Reloadable); + store.compare_stage(42, 1, Some(&writing), Some(&ready)).unwrap(); + ready +} +fn candidate(low: u64, generation: u64, refs: &[StageId]) -> ManifestCandidate { + ManifestCandidate { + id: id(low), + expected_generation: generation - 1, + reference_count: refs.len() as u64, + reference_checksum: reference_checksum(0, refs), + manifest: format!("fixed manifest {low}").into_bytes(), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn full_reference_windows_prepare_and_release_with_native_metadata_admission() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let stages: Vec<_> = (1..=MAX_REFERENCE_WINDOW as u64 + 1) + .map(|low| ready(store.as_ref(), low)) + .collect(); + let refs: Vec<_> = stages.iter().map(|stage| stage.id).collect(); + let protection = store.stage_protection().unwrap(); + // Exercise both a full final window and a full non-final window. + for (index, refs) in [&refs[..MAX_REFERENCE_WINDOW], refs.as_slice()] + .into_iter() + .enumerate() + { + let plan = candidate(100 + index as u64, 1, refs); + protection.begin_manifest_candidate(42, &plan).unwrap(); + for (window, refs) in refs.chunks(MAX_REFERENCE_WINDOW).enumerate() { + protection + .prepare_manifest_references(42, plan.id, window as u64, refs) + .unwrap(); + } + protection.abort_manifest_candidate(42, plan.id).unwrap(); + for window in 0..refs.len().div_ceil(MAX_REFERENCE_WINDOW) { + protection + .release_manifest_references(42, plan.id, window as u64) + .unwrap(); + } + } + for stage in stages { + let retired = next(&stage, StageState::Retired); + store.compare_stage(42, 1, Some(&stage), Some(&retired)).unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn generation_reclamation_advances_within_complete_transaction_limit() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let protection = store.stage_protection().unwrap(); + for generation in 1..=70 { + let plan = candidate(100 + generation, generation, &[]); + protection.begin_manifest_candidate(42, &plan).unwrap(); + protection.commit_manifest_candidate(42, plan.id).unwrap(); + } + let mut reclaimed = 0; + while reclaimed < 68 { + let count = store.reclaim_before(42, 70); + assert!(count > 0, "reclamation stopped at generation {reclaimed}"); + reclaimed += count; + } + assert_eq!(reclaimed, 68); + assert_eq!(store.reclaim_before(42, 70), 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn prepared_references_protect_before_commit_and_survive_unknown_commit_reply() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let first = ready(store.as_ref(), 1); + let second = ready(store.as_ref(), 2); + let refs = [first.id, second.id]; + let plan = candidate(10, 1, &refs); + let protection = store.stage_protection().unwrap(); + protection.begin_manifest_candidate(42, &plan).unwrap(); + protection + .prepare_manifest_references(42, plan.id, 0, &refs[..1]) + .unwrap(); + let retired = next(&first, StageState::Retired); + assert_eq!( + store.compare_stage(42, 1, Some(&first), Some(&retired)), + Err(CtError::Unavailable) + ); + assert_eq!( + protection.commit_manifest_candidate(42, plan.id), + Err(CtError::Unavailable) + ); + protection + .prepare_manifest_references(42, plan.id, 1, &refs[1..]) + .unwrap(); + protection + .prepare_manifest_references(42, plan.id, 1, &refs[1..]) + .unwrap(); + // Discard the result, query exactly this operation, then repeat it. + protection.commit_manifest_candidate(42, plan.id).unwrap(); + assert_eq!( + protection + .load_manifest_candidate(42, plan.id) + .unwrap() + .unwrap() + .state, + CandidateState::Committed + ); + protection.commit_manifest_candidate(42, plan.id).unwrap(); + assert_eq!( + protection.abort_manifest_candidate(42, plan.id), + Err(CtError::Unavailable) + ); + assert_eq!( + store.load(42, RootCatalogObject::CurrentManifest).unwrap(), + Some(plan.manifest) + ); + assert_eq!(protection.list_manifest_candidates(42, None, 1).unwrap().len(), 1); + assert!(protection + .list_manifest_candidates(42, Some(plan.id), 1) + .unwrap() + .is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn generation_pin_and_live_pin_both_prevent_pack_deletion() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let first = ready(store.as_ref(), 1); + let protection = store.stage_protection().unwrap(); + let plan = candidate(10, 1, &[first.id]); + protection.begin_manifest_candidate(42, &plan).unwrap(); + protection + .prepare_manifest_references(42, plan.id, 0, &[first.id]) + .unwrap(); + protection.commit_manifest_candidate(42, plan.id).unwrap(); + store.pin_generation(42, 1, 99, 1).unwrap(); + protection.pin_stage(42, first.id, id(200)).unwrap(); + for generation in 2..=4 { + store + .publish(42, generation - 1, 1, generation, b"other root") + .unwrap(); + } + assert_eq!(store.reclaim_before(42, 4), 0); + assert_eq!( + protection.release_manifest_references(42, plan.id, 0), + Err(CtError::Unavailable) + ); + store.unpin_generation(42, 1, 99).unwrap(); + assert_eq!(store.reclaim_before(42, 4), 2); + assert_eq!( + protection + .load_manifest_candidate(42, plan.id) + .unwrap() + .unwrap() + .state, + CandidateState::Reclaimed + ); + protection.release_manifest_references(42, plan.id, 0).unwrap(); + protection.release_manifest_references(42, plan.id, 0).unwrap(); + let retired = next(&first, StageState::Retired); + assert_eq!( + store.compare_stage(42, 1, Some(&first), Some(&retired)), + Err(CtError::Unavailable) + ); + protection.unpin_stage(42, first.id, id(200)).unwrap(); + store.compare_stage(42, 1, Some(&first), Some(&retired)).unwrap(); + let deleting = next(&retired, StageState::Deleting); + store + .compare_stage(42, 1, Some(&retired), Some(&deleting)) + .unwrap(); + assert_eq!( + protection.pin_stage(42, first.id, id(201)), + Err(CtError::Unavailable) + ); + let new_plan = candidate(11, 5, &[first.id]); + protection.begin_manifest_candidate(42, &new_plan).unwrap(); + assert_eq!( + protection.prepare_manifest_references(42, new_plan.id, 0, &[first.id]), + Err(CtError::Unavailable) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn abort_releases_only_prepared_windows_and_fixed_identity_cannot_be_reallocated() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let first = ready(store.as_ref(), 1); + let second = ready(store.as_ref(), 2); + let protection = store.stage_protection().unwrap(); + let plan = candidate(10, 1, &[first.id, second.id]); + protection.begin_manifest_candidate(42, &plan).unwrap(); + protection + .prepare_manifest_references(42, plan.id, 0, &[first.id]) + .unwrap(); + protection.abort_manifest_candidate(42, plan.id).unwrap(); + assert_eq!( + protection.prepare_manifest_references(42, plan.id, 1, &[second.id]), + Err(CtError::Unavailable) + ); + protection.release_manifest_references(42, plan.id, 0).unwrap(); + let mut last = first; + for state in [StageState::Retired, StageState::Deleting, StageState::Reclaimed] { + let value = next(&last, state); + store.compare_stage(42, 1, Some(&last), Some(&value)).unwrap(); + last = value; + } + store.compare_stage(42, 1, Some(&last), None).unwrap(); + let duplicate = StageRecord { + id: id(999), + revision: 1, + state: StageState::Reserved, + packs: vec![], + ..last + }; + assert_eq!( + store.compare_stage(42, 1, None, Some(&duplicate)), + Err(CtError::Unavailable) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn checksum_failure_does_not_publish_protection_and_released_pin_never_resurrects() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let first = ready(store.as_ref(), 1); + let protection = store.stage_protection().unwrap(); + let mut plan = candidate(10, 1, &[first.id]); + plan.reference_checksum ^= 1; + protection.begin_manifest_candidate(42, &plan).unwrap(); + assert_eq!( + protection.prepare_manifest_references(42, plan.id, 0, &[first.id]), + Err(CtError::Corruption) + ); + protection.unpin_stage(42, first.id, id(100)).unwrap(); + assert_eq!( + protection.pin_stage(42, first.id, id(100)), + Err(CtError::Unavailable) + ); + let retired = next(&first, StageState::Retired); + store.compare_stage(42, 1, Some(&first), Some(&retired)).unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_pin_invalidates_an_observed_deletion_proof_at_the_authority_revision() { + use bytes::Bytes; + use crowdb_paxos_kv_client::{BatchOp, GetOutcome, ReadMode}; + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let first = ready(store.as_ref(), 1); + let mut prefix = b"\0crowdb/chunk-kv/root/v1/".to_vec(); + prefix.extend_from_slice(&42_u64.to_be_bytes()); + let mut authority = prefix.clone(); + authority.extend_from_slice(b"/authority/"); + authority.extend_from_slice(&0_u64.to_be_bytes()); + let GetOutcome::Found { value, revision } = metadata + .client + .get(0, 1, &authority, ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!("authority must exist"); + }; + store + .stage_protection() + .unwrap() + .pin_stage(42, first.id, id(200)) + .unwrap(); + let mut stage_key = prefix; + stage_key.extend_from_slice(b"/stage/v1/"); + for part in [ + first.id.owner_epoch, + first.id.operation_high, + first.id.operation_low, + ] { + stage_key.extend_from_slice(&part.to_be_bytes()); + } + let ops = [ + BatchOp::Put { + key: Bytes::from(authority.clone()), + value, + }, + BatchOp::Put { + key: Bytes::from(stage_key), + value: Bytes::from(next(&first, StageState::Retired).encode().unwrap()), + }, + ]; + assert!(matches!( + metadata + .client + .batch_write_cas(0, 1, &ops, &authority, revision) + .await, + Err(crowdb_paxos_kv_client::Error::CasFailed { .. } | crowdb_paxos_kv_client::Error::CasBusy) + )); + assert_eq!(store.load_stage(42, first.id).unwrap(), Some(first)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn candidate_decode_rejects_oversized_checksum_words_even_with_valid_record_crc() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let candidate_id = id(50); + let mut key = b"\0crowdb/chunk-kv/root/v1/".to_vec(); + key.extend_from_slice(&42_u64.to_be_bytes()); + key.extend_from_slice(b"/candidate/"); + for value in [ + candidate_id.owner_epoch, + candidate_id.operation_high, + candidate_id.operation_low, + ] { + key.extend_from_slice(&value.to_be_bytes()); + } + for checksum_index in 7..10 { + let mut words = [0_u64; 14]; + words[..3].copy_from_slice(&[1, 2, 1]); + words[checksum_index] = u64::from(u32::MAX) + 1; + let mut bytes = words.into_iter().flat_map(u64::to_le_bytes).collect::<Vec<_>>(); + bytes.extend_from_slice(&crowdb_tree_ffi::crc32c(&bytes).to_le_bytes()); + metadata.client.put(0, 1, &key, &bytes, None).await.unwrap(); + assert_eq!( + store + .stage_protection() + .unwrap() + .load_manifest_candidate(42, candidate_id), + Err(CtError::Corruption), + "oversized checksum word {checksum_index} must not truncate" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_pin_mutations_rebuild_guard_counts_before_gc() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let stage = ready(store.as_ref(), 1); + let retired = next(&stage, StageState::Retired); + for turn in 0..4 { + let pins = [id(100 + 2 * turn), id(101 + 2 * turn)]; + let start = std::sync::Arc::new(std::sync::Barrier::new(2)); + let jobs = pins.map(|pin| { + let catalog = store.clone(); + let start = start.clone(); + let stage_id = stage.id; + tokio::task::spawn_blocking(move || { + start.wait(); + catalog.stage_protection().unwrap().pin_stage(42, stage_id, pin) + }) + }); + for job in jobs { + job.await.unwrap().unwrap(); + } + store + .stage_protection() + .unwrap() + .unpin_stage(42, stage.id, pins[0]) + .unwrap(); + assert_eq!( + store.compare_stage(42, 1, Some(&stage), Some(&retired)), + Err(CtError::Unavailable), + "one remaining live pin must retain its exact guard count" + ); + store + .stage_protection() + .unwrap() + .unpin_stage(42, stage.id, pins[1]) + .unwrap(); + } + store.compare_stage(42, 1, Some(&stage), Some(&retired)).unwrap(); + assert_eq!(store.load_stage(42, stage.id).unwrap(), Some(retired)); +} diff --git a/app/crowdb-chunk-kv-server/tests/root_catalog_staging_test.rs b/app/crowdb-chunk-kv-server/tests/root_catalog_staging_test.rs new file mode 100644 index 000000000..bf59bbcb7 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/root_catalog_staging_test.rs @@ -0,0 +1,207 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_chunk_kv_server::storage::open_root_catalog_for_tests; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome, ReadMode}; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, + PartitionArtifact, +}; +use crowdb_protocol::chunk_stream::StreamName; +use crowdb_tree_ffi::chunk::stage::{StageId, StagePack, StageRecord, StageState}; +use crowdb_tree_ffi::{ChunkRootCatalog, CtError, RootCatalogObject}; +use std::sync::Arc; +#[path = "common/metadata.rs"] +mod metadata; + +fn entry(epoch: u64) -> ChunkKvRangeCatalogEntry { + ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 1 }, + range: KeyRange { + start: vec![], + end: None, + }, + owner: OwnerDescriptor { + instance_id: epoch, + rpc_endpoint: "127.0.0.1:1".into(), + }, + owner_epoch: epoch, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 42, + stream_name: StreamName { high: 1, low: 2 }, + tail_overlay: None, + }, + transition_id: None, + } +} +fn record(low: u64) -> StageRecord { + StageRecord { + tree_id: 42, + id: StageId { + owner_epoch: 1, + operation_high: 100, + operation_low: low, + }, + revision: 1, + chunk_high: 2_u64 << 56, + chunk_low: low, + capacity: 4096, + purpose: 2, + state: StageState::Reserved, + manifest_generation: 0, + packs: vec![], + } +} +fn next(old: &StageRecord, state: StageState) -> StageRecord { + let mut out = old.clone(); + out.revision += 1; + out.state = state; + out +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn callback_stage_roundtrip_survives_reopen_and_retries_the_same_operation() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let catalog = ChunkRootCatalog::open_callback(Arc::clone(&store)).unwrap(); + assert!(catalog.supports_staging()); + let first = record(1); + catalog.compare_stage(42, 1, None, Some(&first)).unwrap(); + // Discarding the first result models a successful commit whose reply was lost. + catalog.compare_stage(42, 1, None, Some(&first)).unwrap(); + assert_eq!(catalog.load_stage(42, first.id).unwrap(), Some(first.clone())); + let mut writing = next(&first, StageState::Writing); + writing.packs.push(StagePack { + offset: 0, + length: 4096, + checksum: 7, + }); + catalog + .compare_stage(42, 1, Some(&first), Some(&writing)) + .unwrap(); + catalog + .compare_stage(42, 1, Some(&first), Some(&writing)) + .unwrap(); + let second = record(2); + catalog.compare_stage(42, 1, None, Some(&second)).unwrap(); + assert_eq!(catalog.list_stages(42, None, 1).unwrap(), vec![writing.clone()]); + assert_eq!(catalog.list_stages(42, Some(first.id), 1).unwrap(), vec![second]); + drop(catalog); + drop(store); + let reopened = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + assert_eq!(reopened.load_stage(42, first.id).unwrap(), Some(writing)); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn epoch_fencing_rejects_stale_transitions_and_keeps_unresolved_allocations() { + let metadata = metadata::TestMetadata::start().await; + let old = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let first = record(1); + old.compare_stage(42, 1, None, Some(&first)).unwrap(); + let writing = next(&first, StageState::Writing); + let current = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(2)) + .await + .unwrap(); + assert_eq!( + old.compare_stage(42, 1, Some(&first), Some(&writing)), + Err(CtError::Unavailable) + ); + assert_eq!( + current.compare_stage(42, 2, Some(&first), Some(&writing)), + Err(CtError::Unavailable) + ); + assert_eq!(current.load_stage(42, first.id).unwrap(), Some(first)); + assert_eq!(current.list_stages(42, None, 64).unwrap().len(), 1); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stale_cas_and_cleanup_cannot_skip_reclamation_completion() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + let catalog = ChunkRootCatalog::open_callback(store).unwrap(); + let first = record(1); + catalog.compare_stage(42, 1, None, Some(&first)).unwrap(); + let writing = next(&first, StageState::Writing); + catalog + .compare_stage(42, 1, Some(&first), Some(&writing)) + .unwrap(); + let stale_retired = next(&first, StageState::Retired); + assert_eq!( + catalog.compare_stage(42, 1, Some(&first), Some(&stale_retired)), + Err(CtError::Unavailable) + ); + assert_eq!( + catalog.compare_stage(42, 1, Some(&writing), None), + Err(CtError::InvalidArgument) + ); + let mut last = writing; + for state in [StageState::Retired, StageState::Deleting, StageState::Reclaimed] { + let value = next(&last, state); + catalog.compare_stage(42, 1, Some(&last), Some(&value)).unwrap(); + last = value; + } + catalog.compare_stage(42, 1, Some(&last), None).unwrap(); + catalog.compare_stage(42, 1, Some(&last), None).unwrap(); + assert_eq!(catalog.load_stage(42, last.id).unwrap(), None); + assert!(catalog.list_stages(42, None, 1).unwrap().is_empty()); +} +fn key(kind: &[u8], id: u64) -> Vec<u8> { + let mut key = b"\0crowdb/chunk-kv/root/v1/".to_vec(); + key.extend_from_slice(&42_u64.to_be_bytes()); + key.push(b'/'); + key.extend_from_slice(kind); + key.push(b'/'); + key.extend_from_slice(&id.to_be_bytes()); + key +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pin_invalidates_a_reclamation_proof_at_the_shared_authority_key() { + let metadata = metadata::TestMetadata::start().await; + let store = open_root_catalog_for_tests(metadata.client.clone(), 0, 1, &entry(1)) + .await + .unwrap(); + for generation in 1..=4 { + store + .publish(42, generation - 1, 1, generation, b"manifest") + .unwrap(); + } + let authority = key(b"authority", 0); + let GetOutcome::Found { value, revision } = metadata + .client + .get(0, 1, &authority, ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!("authority must exist"); + }; + store.pin_generation(42, 1, 1, 1).unwrap(); + let ops = [ + BatchOp::Put { + key: Bytes::from(authority.clone()), + value, + }, + BatchOp::Delete { + key: Bytes::from(key(b"manifest", 1)), + }, + ]; + assert!(matches!( + metadata + .client + .batch_write_cas(0, 1, &ops, &authority, revision) + .await, + Err(crowdb_paxos_kv_client::Error::CasFailed { .. } | crowdb_paxos_kv_client::Error::CasBusy) + )); + assert_eq!(store.reclaim_before(42, 4), 0); + store.unpin_generation(42, 1, 1).unwrap(); + assert_eq!(store.reclaim_before(42, u64::MAX), 2); + assert_eq!(store.load(42, RootCatalogObject::Manifest(1)).unwrap(), None); + assert_eq!(store.pin_generation(42, 2, 2, 1), Err(CtError::NotFound)); + assert!(store.load(42, RootCatalogObject::Manifest(3)).unwrap().is_some()); +} diff --git a/app/crowdb-chunk-kv-server/tests/rpc_test.rs b/app/crowdb-chunk-kv-server/tests/rpc_test.rs index 249e8d3aa..472036199 100644 --- a/app/crowdb-chunk-kv-server/tests/rpc_test.rs +++ b/app/crowdb-chunk-kv-server/tests/rpc_test.rs @@ -97,3 +97,22 @@ async fn point_request_crosses_real_rpc_boundary_with_typed_response() { server.stop(); } + +#[tokio::test] +async fn shutdown_drains_owned_rpc_tail_and_permanently_closes_admission() { + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(7), 8).unwrap()); + let rpc = Arc::new(ChunkKvRpcService::new(service, tokio::runtime::Handle::current())); + let held = rpc.hold_request_for_tests().unwrap(); + let shutdown = rpc.close_and_drain(); + tokio::pin!(shutdown); + std::future::poll_fn(|context| { + assert!(std::future::Future::poll(shutdown.as_mut(), context).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + assert!(rpc.hold_request_for_tests().is_none()); + drop(held); + shutdown.await; + assert!(rpc.hold_request_for_tests().is_none()); + rpc.close_and_drain().await; +} diff --git a/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs b/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs index c2734a60d..8ac8e543f 100644 --- a/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs +++ b/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs @@ -1,6 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +#[path = "common/metadata.rs"] +mod metadata; #[path = "common/process.rs"] mod test_process; @@ -19,6 +21,44 @@ use crowdb_protocol::chunk_kv::{ }; use std::sync::Arc; +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn recovered_split_records_tree_completion_without_a_live_transfer() { + use crowdb_chunk_kv_server::storage::open_root_catalog_with_process_for_tests; + use crowdb_tree_ffi::chunk::test_util::TestChunkBackend; + + let metadata = metadata::TestMetadata::start().await; + let entry = entry(&transition(), false); + let partition = recovered(&entry).await; + assert!(transfer_initialization_pending(&partition).await); + let catalog = open_root_catalog_with_process_for_tests( + metadata.client.clone(), + 0, + 1, + &entry, + test_process::TestProcess::authority(1), + ) + .await + .unwrap(); + let backend = TestChunkBackend::new(entry.artifact.tree_id, entry.owner_epoch).unwrap(); + let pages = backend.page_store(0).unwrap(); + tokio::time::timeout( + std::time::Duration::from_secs(1), + catalog.complete_tree_recovery_for_tests(&partition, &pages), + ) + .await + .expect("completed split replay must not wait for a transfer command") + .unwrap(); + assert!(transfer_initialization_pending(&partition).await); + partition.close().await.unwrap(); +} + +async fn transfer_initialization_pending(partition: &Partition) -> bool { + use std::future::Future; + let mut completion = std::pin::pin!(partition.await_transfer_initialization()); + std::future::poll_fn(|context| std::task::Poll::Ready(completion.as_mut().poll(context).is_pending())) + .await +} + #[tokio::test] async fn committed_split_proves_both_recovered_writers_and_rejects_mismatches() { let transition = transition(); diff --git a/app/crowdb-chunk-kv-server/tests/stream_metadata_reclaim_test.rs b/app/crowdb-chunk-kv-server/tests/stream_metadata_reclaim_test.rs index 1d1fdbac3..ff6bb0e11 100644 --- a/app/crowdb-chunk-kv-server/tests/stream_metadata_reclaim_test.rs +++ b/app/crowdb-chunk-kv-server/tests/stream_metadata_reclaim_test.rs @@ -8,14 +8,14 @@ use crowdb_chunk_stream::{ memory::MemoryStreamStore, ChunkStream, KvStreamMetadataStore, StreamConfig, StreamMetadataStore, StreamName, StreamPurpose, StreamRecord, StreamRecordState, }; -use crowdb_kv_client::{CrowdbKvClient, ReadMode}; +use crowdb_paxos_kv_client::{PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_stream::decode_stream_chunk_record; use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey}; #[path = "common/metadata.rs"] mod metadata; -async fn pages(kv: &CrowdbKvClient, name: StreamName) -> Vec<StreamChunkRecordKey> { +async fn pages(kv: &PaxosKvClient, name: StreamName) -> Vec<StreamChunkRecordKey> { kv.scan( 0, 1, diff --git a/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs b/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs index ddadaca8d..a7ff5b1c3 100644 --- a/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs +++ b/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use crowdb_chunk_kv_server::storage::open_root_catalog_for_tests; use crowdb_chunk_kv_server::{ChunkKvRangeCatalogPublisher, Group0ControlStore}; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, PartitionArtifact, @@ -88,7 +88,7 @@ async fn preparing_target_does_not_fence_source_or_mutate_shared_root() { recovered.publish(42, 2, 1, 3, b"source-checkpoint-3").unwrap(); } -async fn publish_assignment(kv: Arc<CrowdbKvClient>, entry: ChunkKvRangeCatalogEntry, generation: u64) { +async fn publish_assignment(kv: Arc<PaxosKvClient>, entry: ChunkKvRangeCatalogEntry, generation: u64) { let mut page = ChunkKvRangeCatalogPage { generation, page_index: 0, diff --git a/app/crowdb-chunk-kv-server/tests/tree_config_test.rs b/app/crowdb-chunk-kv-server/tests/tree_config_test.rs new file mode 100644 index 000000000..84f62f483 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/tree_config_test.rs @@ -0,0 +1,173 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use crowdb_chunk_kv_server::ChunkKvServerConfig; +use crowdb_tree_ffi::{Crowdbtree, CtError, TreeRuntime}; + +const CANONICAL: &str = include_str!("../conf/crowdb_chunk_kv_server_config.toml"); +const PAXOS: &str = include_str!("../../crowdb-paxos-kv-server/conf/crowdb_paxos_kv_server_config.toml"); + +fn load(text: &str) -> Result<ChunkKvServerConfig, String> { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("chunk-tree-config"); + let path = root.path().join("server.toml"); + std::fs::write(&path, text).unwrap(); + ChunkKvServerConfig::load(&path).map_err(|error| error.to_string()) +} + +#[test] +fn canonical_config_loads_all_service_defaults() { + let config = load(CANONICAL).unwrap(); + assert_eq!( + config, + ChunkKvServerConfig { + instance_id: 1, + tree_runtime: config.tree_runtime.clone(), + ..Default::default() + } + ); + let expected = ChunkKvServerConfig::default().tree_runtime.runtime_options(); + let host = config.tree_runtime.runtime_options(); + assert_eq!(host.workers, expected.workers); + assert_eq!(host.backend_workers, expected.backend_workers); + assert_eq!(host.limits, expected.limits); +} + +fn field_paths(value: &toml::Value, prefix: &str, paths: &mut BTreeSet<String>) { + if let Some(table) = value.as_table() { + for (key, child) in table { + field_paths(child, &format!("{prefix}.{key}"), paths); + } + } else { + paths.insert(prefix.to_owned()); + } +} + +#[test] +fn independent_server_files_keep_the_same_tree_fields() { + let chunk: toml::Value = toml::from_str(CANONICAL).unwrap(); + let paxos: toml::Value = toml::from_str(PAXOS).unwrap(); + for section in ["tree", "tree_runtime"] { + let mut chunk_fields = BTreeSet::new(); + let mut paxos_fields = BTreeSet::new(); + field_paths(&chunk[section], section, &mut chunk_fields); + field_paths(&paxos[section], section, &mut paxos_fields); + assert_eq!(chunk_fields, paxos_fields, "{section}"); + } +} + +#[test] +fn configured_native_engine_and_shared_runtime_are_used() { + let mut config = load(&CANONICAL.replace("max_key_size = 0", "max_key_size = 4")).unwrap(); + config.tree_runtime.host.workers = Some(2); + config.tree_runtime.host.backend_workers = Some(1); + config.tree_runtime.tree.cpu_queue_limit = Some(32); + config.tree.tuning.flush_parallelism = 2; + config.validate().unwrap(); + let runtime = Arc::new(TreeRuntime::new(config.tree_runtime.runtime_options()).unwrap()); + let mut options = config.native_tree_options(); + assert_eq!(options.runtime_binding.cpu_queue_limit, 32); + options.runtime = Some(runtime.clone()); + let first = Crowdbtree::open(&options).unwrap(); + let second = Crowdbtree::open(&options).unwrap(); + assert_eq!(runtime.snapshot().unwrap().bindings, 2); + for tree in [&first, &second] { + assert_eq!(tree.put(b"abcde", b"value"), Err(CtError::InvalidArgument)); + tree.put(b"abcd", b"value").unwrap(); + assert_eq!(tree.get(b"abcd").unwrap().unwrap().1, b"value"); + } +} + +#[test] +fn configured_chunk_tree_survives_range_rebuild() { + use crowdb_tree_ffi::chunk::test_util::TestChunkBackend; + + let config = load(&CANONICAL.replace("max_key_size = 0", "max_key_size = 4")).unwrap(); + let source_backend = TestChunkBackend::new(84, 1).unwrap(); + let child_backend = TestChunkBackend::new(85, 1).unwrap(); + let source_options = crowdb_tree_ffi::Config { + runtime: Some(source_backend.runtime.clone()), + page_store: Some(source_backend.page_store(0).unwrap()), + ..config.native_tree_options() + }; + let source = Crowdbtree::open(&source_options).unwrap(); + source.put(b"abcd", b"value").unwrap(); + let destination = crowdb_tree_ffi::Config { + runtime: Some(child_backend.runtime.clone()), + page_store: Some(child_backend.page_store(0).unwrap()), + ..config.native_tree_options() + }; + let (child, stats) = source.rebuild_range(&destination).unwrap(); + assert_eq!(stats.entries_emitted, 1); + assert_eq!(child.get(b"abcd").unwrap().unwrap().1, b"value"); + assert_eq!(child.put(b"abcde", b"value"), Err(CtError::InvalidArgument)); +} + +#[test] +fn chunk_backend_alignment_is_validated_before_open() { + let mut config = load(CANONICAL).unwrap(); + config.tree.frame_bytes = 16 * 1024; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("Chunk backend IU")); + config.tree.iu_size = 4096; + config.validate().unwrap(); + assert_eq!(config.native_tree_options().iu_size, 4096); +} + +#[test] +fn invalid_runtime_engine_and_partition_bounds_fail_on_load() { + for (from, to, field) in [ + ( + "flush_parallelism = 0", + "flush_parallelism = 5", + "flush_parallelism", + ), + ("cache_max_pages = 256", "cache_max_pages = 0", "cache_max_pages"), + ("queue_requests = 1_024", "queue_requests = 0", "partition bounds"), + ("batch_bytes = 4_194_304", "batch_bytes = 0", "partition bounds"), + ("backend_workers = 2", "backend_workers = 0", "backend workers"), + ( + "run_max_entries = 128", + "run_entries_typo = 128", + "run_entries_typo", + ), + ("queue_requests = 1_024", "queue_typo = 1_024", "queue_typo"), + ] { + assert!( + load(&CANONICAL.replace(from, to)).unwrap_err().contains(field), + "{field}" + ); + } +} + +#[test] +fn partial_partition_table_keeps_native_defaults() { + let config = load("instance_id = 1\n[partition]\nbatch_requests = 16\n").unwrap(); + assert_eq!(config.partition.batch_requests, 16); + assert_eq!(config.partition.queue_requests, 1024); + assert_eq!(config.partition.max_value_bytes, 16 * 1024 * 1024); +} + +#[test] +fn commented_optional_examples_are_loadable() { + let start = CANONICAL.find("# [bootstrap_partition]").unwrap(); + let end = CANONICAL.find("# Keep Tree field additions").unwrap(); + let examples = CANONICAL[start..end] + .lines() + .filter_map(|line| line.strip_prefix("# ")) + .filter(|line| line.starts_with('[') || line.contains(" = ")) + .collect::<Vec<_>>() + .join("\n"); + let config = load(&format!("{CANONICAL}\n{examples}")).unwrap(); + assert_eq!(config.bootstrap_partition.unwrap().tree_id, 1); + let mut rdma = config.rpc_rdma.unwrap(); + assert_eq!(rdma.control_data_capacity(), Some(262_144)); + // Exercise explicit server configuration without requiring an RDMA device. + rdma.enabled = false; + crowdb_chunk_kv_server::ChunkKvRpcService::create_server(1, Some(&rdma)).unwrap(); +} diff --git a/app/crowdb-chunkdb/Cargo.toml b/app/crowdb-chunkdb/Cargo.toml index a7b7bfffd..69fca90a3 100644 --- a/app/crowdb-chunkdb/Cargo.toml +++ b/app/crowdb-chunkdb/Cargo.toml @@ -26,7 +26,7 @@ workspace = true [dependencies] crowdb-protocol = { path = "../../lib/crowdb-protocol" } -crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../../lib/crowdb-paxos-kv-client" } crowdb-diskdb-client = { path = "../../lib/crowdb-diskdb-client" } crowdb-diskio-client = { path = "../../lib/crowdb-diskio-client" } crowdb-chunk-client = { path = "../../lib/crowdb-chunk-client" } @@ -55,7 +55,7 @@ futures = "0.3" [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } -crowdb-kv = { path = "../../lib/crowdb-kv", features = ["test-util"] } +crowdb-paxos-kv = { path = "../../lib/crowdb-paxos-kv", features = ["test-util"] } crowdb-chunkdb-client = { path = "../../lib/crowdb-chunkdb-client" } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } @@ -64,3 +64,8 @@ reqwest = { version = "0.12", features = ["json"] } serde_json = "1" crowdb-diskdb = { path = "../crowdb-diskdb" } crowdb-chunkdb = { path = ".", features = ["test-util"] } + +crowdb-chunk-kv-server = { path = "../crowdb-chunk-kv-server", features = ["test-util"] } +crowdb-chunk-kv = { path = "../../lib/crowdb-chunk-kv", features = ["test-util"] } +crowdb-chunk-stream = { path = "../../lib/crowdb-chunk-stream", features = ["test-util"] } +crowdb-tree-ffi = { path = "../../lib/crowdb-tree/ffi", features = ["test-util", "chunk-rpc"] } diff --git a/app/crowdb-chunkdb/src/allocator/pool.rs b/app/crowdb-chunkdb/src/allocator/pool.rs index 4a8e522f3..6608f3a8f 100644 --- a/app/crowdb-chunkdb/src/allocator/pool.rs +++ b/app/crowdb-chunkdb/src/allocator/pool.rs @@ -15,7 +15,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use arc_swap::ArcSwap; use crowdb_diskdb_client::{DiskdbClientError, DiskdbRpcTransport}; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::{ AllocateBlocksRequest, AllocateResponse, ExecuteRelocationRequest, ExecuteRelocationResponse, diff --git a/app/crowdb-chunkdb/src/chunkdb_config.rs b/app/crowdb-chunkdb/src/chunkdb_config.rs index e00cf64fa..609b3663d 100644 --- a/app/crowdb-chunkdb/src/chunkdb_config.rs +++ b/app/crowdb-chunkdb/src/chunkdb_config.rs @@ -6,7 +6,7 @@ use std::net::SocketAddr; use crowdb_common::config::BaseConfig; -use crowdb_protocol::{CHUNKDB_HTTP_BASE, CHUNKDB_RPC_BASE, KV_SERVER_MGMT_BASE}; +use crowdb_protocol::{CHUNKDB_HTTP_BASE, CHUNKDB_RPC_BASE, PAXOS_KV_SERVER_MGMT_BASE}; use serde::{Deserialize, Serialize}; use crate::selector::FailureDomainPriority; @@ -459,7 +459,7 @@ impl Default for ServerConfig { http_listen_addr: format!("0.0.0.0:{CHUNKDB_HTTP_BASE}"), rpc_listen_addr: default_rpc_listen_addr(), instance_id: None, - kv_server_mgmt_seeds: vec![format!("http://127.0.0.1:{KV_SERVER_MGMT_BASE}")], + kv_server_mgmt_seeds: vec![format!("http://127.0.0.1:{PAXOS_KV_SERVER_MGMT_BASE}")], kv_pool_size: default_client_pool_size(), kv_rpc_workers: default_client_rpc_workers(), diskdb_pool_size: default_client_pool_size(), diff --git a/app/crowdb-chunkdb/src/conversion/io.rs b/app/crowdb-chunkdb/src/conversion/io.rs index 90f356e6a..27c90a090 100644 --- a/app/crowdb-chunkdb/src/conversion/io.rs +++ b/app/crowdb-chunkdb/src/conversion/io.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use arc_swap::ArcSwapOption; use bytes::Bytes; use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, Durability, SegmentTarget}; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::diskdb::rpc::Segment; use crate::chunkdb_config::ConversionIoConfig; diff --git a/app/crowdb-chunkdb/src/lifecycle/handler.rs b/app/crowdb-chunkdb/src/lifecycle/handler.rs index 057bb9955..57bb861e4 100644 --- a/app/crowdb-chunkdb/src/lifecycle/handler.rs +++ b/app/crowdb-chunkdb/src/lifecycle/handler.rs @@ -39,6 +39,7 @@ use crate::task::TaskStore; use crate::topology::TopologyCache; mod allocation; +mod deletion; use super::replacement::validate_replacement_geometry; use super::state::{ChunkState, StateTransitionError}; @@ -603,112 +604,6 @@ impl LifecycleHandler { }); } - /// Delete a chunk — marks deleted and frees segments. - /// Repeated deletion returns the existing tombstone so retries are idempotent. - pub async fn delete_chunk(&self, chunk_id: &ChunkId) -> Result<Chunk, LifecycleError> { - self.check_range(chunk_id)?; - - let mut guard = if let Some(locks) = &self.locks { - Some( - locks - .acquire(chunk_id, &self.store, &LockPolicy::default(), CacheHint::Cache) - .await?, - ) - } else { - None - }; - - let mut chunk = match &guard { - Some(g) => g - .chunk() - .unwrap_or_else(|| unreachable!("acquire guarantees chunk on Ok")) - .clone(), - None => self.store.get_chunk(chunk_id).await?, - }; - let current_state = ChunkState::from_proto(chunk.state); - - // A Deleted record that still owns strips is a durable cleanup intent. - if current_state == ChunkState::Deleted { - if chunk.strips.is_empty() { - return Ok(chunk); - } - let segments: Vec<_> = chunk.strips.iter().flat_map(extract_segments).collect(); - self.allocator - .pool() - .free_blocks(segments) - .await - .map_err(LifecycleError::Cleanup)?; - chunk.strips.clear(); - chunk.capacity = 0; - chunk.modify_ts = chunk.modify_ts.saturating_add(1); - self.store.put_chunk(&chunk).await?; - if let Some(ref mut g) = guard { - g.refresh(chunk.clone()); - } - return Ok(chunk); - } - - current_state.check_can_delete()?; - - let reservation_groups = self.store.list_reservation_groups(chunk_id).await?; - if reservation_groups - .iter() - .any(|group| group.states.contains(&(StripReservationState::Consumed as i32))) - { - return Err(LifecycleError::StateConflict); - } - let mut reserved_strips = Vec::new(); - let mut reserved_parity = Vec::new(); - let mut reserved_parity_groups = Vec::new(); - for mut group in reservation_groups { - reservation::validate_group_shape(&group)?; - reserved_parity.extend_from_slice(&group.parity_segments); - for (index, state) in group.states.iter_mut().enumerate() { - if *state == StripReservationState::Reserved as i32 { - *state = StripReservationState::Cancelled as i32; - reserved_strips.push(group.strips[index].clone()); - } else if *state == StripReservationState::Cancelled as i32 { - reserved_strips.push(group.strips[index].clone()); - } - } - self.store.put_reservation_group(&group).await?; - reserved_parity_groups.push(group); - } - self.rollback_reserved_resources(&reserved_strips, reserved_parity, &reserved_parity_groups) - .await?; - for group in self.store.list_reservation_groups(chunk_id).await? { - if let Some(group_id) = group.group_id { - self.store.delete_reservation_group(chunk_id, &group_id).await?; - } - } - - // Publish Deleted before making any referenced block reusable. - let all_segments: Vec<_> = chunk.strips.iter().flat_map(extract_segments).collect(); - chunk.state = ProtoChunkState::Deleted as i32; - chunk.modify_ts = chunk.modify_ts.saturating_add(1); - self.store.put_chunk(&chunk).await?; - - if let Some(ref mut g) = guard { - g.refresh(chunk.clone()); - } - if !all_segments.is_empty() { - self.allocator - .pool() - .free_blocks(all_segments) - .await - .map_err(LifecycleError::Cleanup)?; - } - chunk.strips.clear(); - chunk.capacity = 0; - chunk.modify_ts = chunk.modify_ts.saturating_add(1); - self.store.put_chunk(&chunk).await?; - if let Some(ref mut g) = guard { - g.refresh(chunk.clone()); - } - info!(chunk_id = ?chunk_id, "chunk deleted"); - Ok(chunk) - } - /// Atomically replace a capacity-compatible strip range under a revision /// fence. Newly introduced segments are committed before publication; /// removed segments stay allocated until the reader-layout grace expires. diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/allocation.rs b/app/crowdb-chunkdb/src/lifecycle/handler/allocation.rs index 66e312f5e..a2c4a3305 100644 --- a/app/crowdb-chunkdb/src/lifecycle/handler/allocation.rs +++ b/app/crowdb-chunkdb/src/lifecycle/handler/allocation.rs @@ -205,13 +205,11 @@ impl LifecycleHandler { let constraints = self.allocation_constraints(strip_type, &snap); // Convert write_granularity (KB) to unit_count using the unit - // size from the topology snapshot. Fall back to treating KB as - // units if unit_size_bytes is unavailable (0). + // size from the topology snapshot, rounding up to cover the complete + // requested strip. Fall back to treating KB as units if the unit size + // is unavailable (0). let unit_size_kb = snap.unit_size_bytes() / 1024; - let unit_count = write_granularity_kb - .checked_div(unit_size_kb) - .unwrap_or(write_granularity_kb) - .max(1); + let unit_count = write_granularity_kb.div_ceil(unit_size_kb.max(1)).max(1); let mut strips = self .allocator diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/deletion.rs b/app/crowdb-chunkdb/src/lifecycle/handler/deletion.rs new file mode 100644 index 000000000..d1b9e09c4 --- /dev/null +++ b/app/crowdb-chunkdb/src/lifecycle/handler/deletion.rs @@ -0,0 +1,169 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Physical deletion and fixed allocation identity fencing. +use super::{ + extract_segments, info, reservation, unix_time_ms, CacheHint, Chunk, ChunkGuard, ChunkId, ChunkState, + LifecycleError, LifecycleHandler, LockPolicy, ProtoChunkState, StoreError, StripReservationState, +}; + +impl LifecycleHandler { + /// Delete a chunk — marks deleted and frees segments. + /// Repeated deletion returns the existing tombstone so retries are idempotent. + pub async fn delete_chunk(&self, chunk_id: &ChunkId) -> Result<Chunk, LifecycleError> { + self.delete_chunk_fenced(chunk_id, false).await + } + + /// Fence a fixed identity even when its allocation request has not arrived. + pub async fn delete_chunk_fenced( + &self, + chunk_id: &ChunkId, + fence_missing: bool, + ) -> Result<Chunk, LifecycleError> { + self.check_range(chunk_id)?; + if fence_missing && !matches!(chunk_id.high >> 56, 2 | 3) { + return Err(LifecycleError::InvalidRequest( + "allocation fencing requires a tree chunk identity".into(), + )); + } + let mut guard = if let Some(locks) = &self.locks { + Some(if fence_missing { + locks + .acquire_for_create(chunk_id, &LockPolicy::default(), CacheHint::Cache) + .await? + } else { + locks + .acquire(chunk_id, &self.store, &LockPolicy::default(), CacheHint::Cache) + .await? + }) + } else { + None + }; + + let chunk = if fence_missing { + match self.store.get_chunk(chunk_id).await { + Ok(chunk) => chunk, + Err(StoreError::ChunkNotFound) => { + let chunk = Chunk { + id: Some(*chunk_id), + modify_ts: 1, + state: ProtoChunkState::Deleted as i32, + create_ts_ms: unix_time_ms(), + chunk_type: if chunk_id.high >> 56 == 2 { 2 } else { 3 }, + ..Default::default() + }; + self.store.put_chunk(&chunk).await?; + if let Some(ref mut guard) = guard { + guard.refresh(chunk.clone()); + } + return Ok(chunk); + } + Err(error) => return Err(error.into()), + } + } else { + match &guard { + Some(guard) => guard + .chunk() + .unwrap_or_else(|| unreachable!("acquire guarantees a chunk")) + .clone(), + None => self.store.get_chunk(chunk_id).await?, + } + }; + self.finish_deletion(chunk_id, chunk, guard).await + } + + async fn finish_deletion( + &self, + chunk_id: &ChunkId, + mut chunk: Chunk, + mut guard: Option<ChunkGuard>, + ) -> Result<Chunk, LifecycleError> { + let current_state = ChunkState::from_proto(chunk.state); + + // A Deleted record that still owns strips is a durable cleanup intent. + if current_state == ChunkState::Deleted { + if chunk.strips.is_empty() { + return Ok(chunk); + } + let segments: Vec<_> = chunk.strips.iter().flat_map(extract_segments).collect(); + self.allocator + .pool() + .free_blocks(segments) + .await + .map_err(LifecycleError::Cleanup)?; + chunk.strips.clear(); + chunk.capacity = 0; + chunk.modify_ts = chunk.modify_ts.saturating_add(1); + self.store.put_chunk(&chunk).await?; + if let Some(ref mut g) = guard { + g.refresh(chunk.clone()); + } + return Ok(chunk); + } + + current_state.check_can_delete()?; + + self.retire_reservation_groups(chunk_id).await?; + + // Publish Deleted before making any referenced block reusable. + let all_segments: Vec<_> = chunk.strips.iter().flat_map(extract_segments).collect(); + chunk.state = ProtoChunkState::Deleted as i32; + chunk.modify_ts = chunk.modify_ts.saturating_add(1); + self.store.put_chunk(&chunk).await?; + + if let Some(ref mut g) = guard { + g.refresh(chunk.clone()); + } + if !all_segments.is_empty() { + self.allocator + .pool() + .free_blocks(all_segments) + .await + .map_err(LifecycleError::Cleanup)?; + } + chunk.strips.clear(); + chunk.capacity = 0; + chunk.modify_ts = chunk.modify_ts.saturating_add(1); + self.store.put_chunk(&chunk).await?; + if let Some(ref mut g) = guard { + g.refresh(chunk.clone()); + } + info!(chunk_id = ?chunk_id, "chunk deleted"); + Ok(chunk) + } + async fn retire_reservation_groups(&self, chunk_id: &ChunkId) -> Result<(), LifecycleError> { + let reservation_groups = self.store.list_reservation_groups(chunk_id).await?; + if reservation_groups + .iter() + .any(|group| group.states.contains(&(StripReservationState::Consumed as i32))) + { + return Err(LifecycleError::StateConflict); + } + let mut reserved_strips = Vec::new(); + let mut reserved_parity = Vec::new(); + let mut reserved_parity_groups = Vec::new(); + for mut group in reservation_groups { + reservation::validate_group_shape(&group)?; + reserved_parity.extend_from_slice(&group.parity_segments); + for (index, state) in group.states.iter_mut().enumerate() { + if *state == StripReservationState::Reserved as i32 { + *state = StripReservationState::Cancelled as i32; + reserved_strips.push(group.strips[index].clone()); + } else if *state == StripReservationState::Cancelled as i32 { + reserved_strips.push(group.strips[index].clone()); + } + } + self.store.put_reservation_group(&group).await?; + reserved_parity_groups.push(group); + } + self.rollback_reserved_resources(&reserved_strips, reserved_parity, &reserved_parity_groups) + .await?; + for group in self.store.list_reservation_groups(chunk_id).await? { + if let Some(group_id) = group.group_id { + self.store.delete_reservation_group(chunk_id, &group_id).await?; + } + } + + Ok(()) + } +} diff --git a/app/crowdb-chunkdb/src/main.rs b/app/crowdb-chunkdb/src/main.rs index 8a8741091..4742b4413 100644 --- a/app/crowdb-chunkdb/src/main.rs +++ b/app/crowdb-chunkdb/src/main.rs @@ -23,8 +23,8 @@ use crowdb_chunkdb::topology::{ build_snapshot, notify::NotifyHandler, refresh::run_refresh_loop, TopologyCache, }; use crowdb_common::metrics::{MetricsRegistry, MetricsRunner}; -use crowdb_kv_client::{ - ClientConfig, CrowdbKvClient, DomainMonitorClient, HardwareClient, KVClusterMetaClient, +use crowdb_paxos_kv_client::{ + ClientConfig, DomainMonitorClient, HardwareClient, PaxosKvClient, PaxosKvClusterMetaClient, ServiceRegistryClient, WatchNotifyClient, }; use tracing::{error, info, warn}; @@ -169,7 +169,7 @@ async fn main() { let mut kv_config = ClientConfig::new(config.server.kv_server_mgmt_seeds.clone()); kv_config.pool_size_per_endpoint = config.server.kv_pool_size; kv_config.rpc_workers = config.server.kv_rpc_workers; - let kv = Arc::new(CrowdbKvClient::new_with_isolated_point_reads(kv_config)); + let kv = Arc::new(PaxosKvClient::new_with_isolated_point_reads(kv_config)); let hw = HardwareClient::from_shared(Arc::clone(&kv)); let refresh_hw = HardwareClient::from_shared(Arc::clone(&kv)); let watch = WatchNotifyClient::from_shared(Arc::clone(&kv)); @@ -256,7 +256,7 @@ async fn main() { // Binding cache + chunk store. let bindings = BindingCache::new(); - let slot_maps = crowdb_kv_client::ChunkSlotMapClient::new(Arc::clone(&kv)); + let slot_maps = crowdb_paxos_kv_client::ChunkSlotMapClient::new(Arc::clone(&kv)); if let Some(bootstrap) = &config.slot_bootstrap { if let Err(error) = slot_maps.initialize_layout(bootstrap).await { error!(%error, "chunk slot initialization failed; refusing startup"); @@ -695,14 +695,14 @@ fn load_config(args: &Cli) -> ChunkdbConfig { config } -async fn validate_voting_topology(kv: &Arc<CrowdbKvClient>, mode: DeploymentMode) -> Result<(), String> { +async fn validate_voting_topology(kv: &Arc<PaxosKvClient>, mode: DeploymentMode) -> Result<(), String> { use std::collections::{BTreeMap, BTreeSet}; if mode == DeploymentMode::TestUnsafePlacement { return Ok(()); } - let metadata = KVClusterMetaClient::from_shared(Arc::clone(kv)); + let metadata = PaxosKvClusterMetaClient::from_shared(Arc::clone(kv)); let groups = metadata .list_all_groups() .await diff --git a/app/crowdb-chunkdb/src/range_guard.rs b/app/crowdb-chunkdb/src/range_guard.rs index dbe28929f..c23b21089 100644 --- a/app/crowdb-chunkdb/src/range_guard.rs +++ b/app/crowdb-chunkdb/src/range_guard.rs @@ -4,7 +4,7 @@ //! Immutable service-slot admission snapshots, independent of storage destinations. use arc_swap::ArcSwapOption; -use crowdb_kv_client::{ChunkSlotMapClient, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, PaxosKvClient}; use crowdb_protocol::chunk_slot::{ ChunkSlot, ChunkSlotAuthority, ChunkSlotBitmap, ChunkSlotMap, CHUNK_SLOT_COUNT, }; @@ -98,7 +98,7 @@ impl RangeGuard { &self, map: &ChunkSlotMap<ChunkSlotAuthority>, instance_id: u64, - ) -> crowdb_kv_client::Result<()> { + ) -> crowdb_paxos_kv_client::Result<()> { let slots = ChunkSlot::all() .filter(|slot| map.owner(*slot).instance_id() == instance_id) .collect(); @@ -178,7 +178,7 @@ impl RangeGuard { /// # Errors /// Rejects unsupported changes to initialized authority. New instances /// start with no slots until an explicit ownership migration assigns work. - pub fn install(&self, map: &ChunkSlotMap<u64>, instance_id: u64) -> crowdb_kv_client::Result<()> { + pub fn install(&self, map: &ChunkSlotMap<u64>, instance_id: u64) -> crowdb_paxos_kv_client::Result<()> { let binding = map .bindings() .iter() @@ -214,9 +214,9 @@ impl RangeGuard { /// Returns read, validation or unsupported ownership-change errors. pub async fn load_from_group0( &self, - kv: &Arc<CrowdbKvClient>, + kv: &Arc<PaxosKvClient>, instance_id: u64, - ) -> crowdb_kv_client::Result<()> { + ) -> crowdb_paxos_kv_client::Result<()> { let maps = ChunkSlotMapClient::new(Arc::clone(kv)); if let Some(snapshot) = maps.read_dynamic_service_snapshot().await? { self.install_dynamic(snapshot.authority(), instance_id) @@ -293,8 +293,8 @@ pub fn chunk_bucket(chunk_id: &ChunkId) -> u16 { ChunkSlot::for_chunk(chunk_id).value() } -fn invalid(reason: &str) -> crowdb_kv_client::Error { - crowdb_kv_client::Error::SysdataDecode { +fn invalid(reason: &str) -> crowdb_paxos_kv_client::Error { + crowdb_paxos_kv_client::Error::SysdataDecode { key: "/chunkdb/slot_service/".into(), reason: reason.into(), } diff --git a/app/crowdb-chunkdb/src/readiness.rs b/app/crowdb-chunkdb/src/readiness.rs index fbe4320e3..82da80c07 100644 --- a/app/crowdb-chunkdb/src/readiness.rs +++ b/app/crowdb-chunkdb/src/readiness.rs @@ -6,7 +6,7 @@ use crate::range_guard::RangeGuard; use arc_swap::ArcSwapOption; use axum::{http::StatusCode, routing::get, Router}; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use std::{future::Future, sync::Arc, time::Duration}; use tokio::{ sync::watch, @@ -71,7 +71,7 @@ impl RangeReadiness { /// Own one bounded refresh at a time; callers never initiate extra reads. pub fn start( self: Arc<Self>, - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, instance_id: u64, stop: watch::Receiver<bool>, ) -> JoinHandle<()> { diff --git a/app/crowdb-chunkdb/src/runtime.rs b/app/crowdb-chunkdb/src/runtime.rs index 85322d6b5..ded31ec9f 100644 --- a/app/crowdb-chunkdb/src/runtime.rs +++ b/app/crowdb-chunkdb/src/runtime.rs @@ -20,7 +20,7 @@ use crate::service::ChunkdbRpcService; use crate::storage::ChunkStore; use crate::task::{TaskManager, TaskStore}; use crate::topology::TopologyCache; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_protocol::chunk_domain::ChunkDomain; mod background; @@ -30,7 +30,7 @@ mod recovery; #[derive(Clone)] pub struct RuntimeContext { pub config: ChunkdbConfig, - pub kv: Arc<CrowdbKvClient>, + pub kv: Arc<PaxosKvClient>, pub bindings: BindingCache, pub range_guard: Arc<RangeGuard>, pub allocator: Arc<ChunkAllocator>, diff --git a/app/crowdb-chunkdb/src/runtime/executor.rs b/app/crowdb-chunkdb/src/runtime/executor.rs index 823ce087f..78ae3b6e7 100644 --- a/app/crowdb-chunkdb/src/runtime/executor.rs +++ b/app/crowdb-chunkdb/src/runtime/executor.rs @@ -14,7 +14,7 @@ use crate::repair::{RepairCoordinator, RepairStripTaskHandler}; use crate::task::{ RelocateSegmentTaskHandler, TaskExecutor, TaskHandler, TaskManager, TaskScanner, TaskStore, }; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use std::sync::Arc; use std::time::Duration; use tracing::warn; diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs index f3fee7ee9..fd14605a4 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs @@ -977,7 +977,9 @@ impl ChunkdbRpcService { return; }; - let result = handler.delete_chunk(&chunk_id).await; + let result = handler + .delete_chunk_fenced(&chunk_id, fb_req.fence_missing()) + .await; if result.is_ok() { request.mark_success(); } diff --git a/app/crowdb-chunkdb/src/storage.rs b/app/crowdb-chunkdb/src/storage.rs index 80f1f17a7..8641c54c8 100644 --- a/app/crowdb-chunkdb/src/storage.rs +++ b/app/crowdb-chunkdb/src/storage.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use tracing::warn; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_task::ChunkTaskValue; use crowdb_protocol::chunkdb::rpc::Chunk; use crowdb_protocol::common::ChunkId; @@ -57,14 +57,14 @@ pub type Result<T> = std::result::Result<T, StoreError>; /// Chunk metadata store — persists `Chunk` records to CROWDB KV. pub struct ChunkStore { - pub(super) kv: Arc<CrowdbKvClient>, + pub(super) kv: Arc<PaxosKvClient>, pub(super) bindings: BindingCache, scope: Option<(Arc<RangeGuard>, ChunkDomain)>, } impl ChunkStore { #[must_use] - pub fn new(kv: Arc<CrowdbKvClient>, bindings: BindingCache) -> Self { + pub fn new(kv: Arc<PaxosKvClient>, bindings: BindingCache) -> Self { Self { kv, bindings, @@ -116,12 +116,12 @@ impl ChunkStore { .await { Ok(_) => Ok(()), - Err(crowdb_kv_client::Error::OutcomeUnknown) => match self.get_chunk(id).await { + Err(crowdb_paxos_kv_client::Error::OutcomeUnknown) => match self.get_chunk(id).await { Ok(stored) if stored == *chunk => Ok(()), Ok(_) | Err(StoreError::ChunkNotFound) => Err(StoreError::Conflict), Err(error) => Err(StoreError::OutcomeUnknown(error.to_string())), }, - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => { + Err(crowdb_paxos_kv_client::Error::CasFailed { .. } | crowdb_paxos_kv_client::Error::CasBusy) => { Err(StoreError::ChunkAlreadyExists) } Err(error) => Err(StoreError::Kv(error.to_string())), @@ -181,9 +181,9 @@ impl ChunkStore { { Ok(_) => Ok(()), Err( - crowdb_kv_client::Error::CasFailed { .. } - | crowdb_kv_client::Error::CasBusy - | crowdb_kv_client::Error::OutcomeUnknown, + crowdb_paxos_kv_client::Error::CasFailed { .. } + | crowdb_paxos_kv_client::Error::CasBusy + | crowdb_paxos_kv_client::Error::OutcomeUnknown, ) => { match self .kv diff --git a/app/crowdb-chunkdb/src/storage/range_retirement.rs b/app/crowdb-chunkdb/src/storage/range_retirement.rs index 510dba10a..01650ae59 100644 --- a/app/crowdb-chunkdb/src/storage/range_retirement.rs +++ b/app/crowdb-chunkdb/src/storage/range_retirement.rs @@ -8,7 +8,7 @@ pub use record::{ChunkRangeRetirement, RetiredByteRange}; use super::{chunk_key, decode_chunk, encode_chunk, ChunkStore, Result, StoreError}; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome, ReadMode}; use crowdb_protocol::chunk_domain::ChunkDomain; use crowdb_protocol::chunkdb::rpc::{ChunkState, DeleteChunkRangeRequest, DeleteChunkRangeResponse}; use crowdb_protocol::common::ChunkId; @@ -106,7 +106,7 @@ impl ChunkStore { .await { Ok(_) => Ok(DeleteChunkRangeResponse { reclaimed: false }), - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => { + Err(crowdb_paxos_kv_client::Error::CasFailed { .. } | crowdb_paxos_kv_client::Error::CasBusy) => { Err(StoreError::Conflict) } Err(error) => Err(StoreError::Kv(error.to_string())), diff --git a/app/crowdb-chunkdb/src/storage/reservation.rs b/app/crowdb-chunkdb/src/storage/reservation.rs index 9fe718177..64ebe0d47 100644 --- a/app/crowdb-chunkdb/src/storage/reservation.rs +++ b/app/crowdb-chunkdb/src/storage/reservation.rs @@ -4,7 +4,7 @@ //! Durable strip-reservation values co-located with their owning chunk. use bytes::Bytes; -use crowdb_kv_client::{BatchOp, GetOutcome, ReadMode, ScanOutcome}; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome, ReadMode, ScanOutcome}; use crowdb_protocol::chunkdb::rpc::{Chunk, StripReservationGroup}; use crowdb_protocol::common::ChunkId; @@ -217,9 +217,9 @@ impl ChunkStore { { Ok(_) => Ok(()), Err( - crowdb_kv_client::Error::CasFailed { .. } - | crowdb_kv_client::Error::CasBusy - | crowdb_kv_client::Error::OutcomeUnknown, + crowdb_paxos_kv_client::Error::CasFailed { .. } + | crowdb_paxos_kv_client::Error::CasBusy + | crowdb_paxos_kv_client::Error::OutcomeUnknown, ) => match self .kv .get(store_id, group_id, &key, ReadMode::Linearizable, None) diff --git a/app/crowdb-chunkdb/src/task/store.rs b/app/crowdb-chunkdb/src/task/store.rs index 58f8791b9..5b9e6e351 100644 --- a/app/crowdb-chunkdb/src/task/store.rs +++ b/app/crowdb-chunkdb/src/task/store.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use crate::range_guard::RangeGuard; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, CrowdbKvClient, Error as KvError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, Error as KvError, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_domain::ChunkDomain; use crowdb_protocol::chunk_task::{ChunkTaskState, ChunkTaskValue, TASK_KIND_FINALIZE_CHUNK}; use crowdb_protocol::common::ChunkId; @@ -44,7 +44,7 @@ pub enum TaskStoreError { /// Persistent task storage. It is lock-free and relies on the caller's /// existing resource lifecycle guard for same-process transition ordering. pub struct TaskStore { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, bindings: BindingCache, scope: Option<(Arc<RangeGuard>, ChunkDomain)>, } @@ -68,7 +68,7 @@ impl TaskStore { Ok(()) } #[must_use] - pub fn new(kv: Arc<CrowdbKvClient>, bindings: BindingCache) -> Self { + pub fn new(kv: Arc<PaxosKvClient>, bindings: BindingCache) -> Self { Self { kv, bindings, diff --git a/app/crowdb-chunkdb/src/topology.rs b/app/crowdb-chunkdb/src/topology.rs index bb11af798..5a5e12909 100644 --- a/app/crowdb-chunkdb/src/topology.rs +++ b/app/crowdb-chunkdb/src/topology.rs @@ -334,7 +334,7 @@ impl TopologySnapshot { } } - fn set_allocation_geometry(&mut self, disks: &[crowdb_kv_client::DiskRecord]) { + fn set_allocation_geometry(&mut self, disks: &[crowdb_paxos_kv_client::DiskRecord]) { match crowdb_protocol::chunk_allocation_geometry::uniform_allocation_unit( disks .iter() @@ -472,7 +472,7 @@ impl Default for TopologyCache { /// Fetches racks, nodes, and disk-groups from group-0 and assembles /// them into a snapshot. If the fetch fails or returns empty, returns /// `None` (caller should keep the previous snapshot). -pub async fn build_snapshot(hw: &crowdb_kv_client::HardwareClient) -> Option<TopologySnapshot> { +pub async fn build_snapshot(hw: &crowdb_paxos_kv_client::HardwareClient) -> Option<TopologySnapshot> { let racks = match hw.list_racks().await { Ok(r) => r, Err(e) => { @@ -505,7 +505,7 @@ pub async fn build_snapshot(hw: &crowdb_kv_client::HardwareClient) -> Option<Top } }; - let mut usages = match crowdb_kv_client::SpaceUsageClient::from_shared(hw.clone()) + let mut usages = match crowdb_paxos_kv_client::SpaceUsageClient::from_shared(hw.clone()) .list_disk_group_usages() .await { @@ -515,7 +515,7 @@ pub async fn build_snapshot(hw: &crowdb_kv_client::HardwareClient) -> Option<Top Vec::new() } }; - let service = crowdb_kv_client::ServiceRegistryClient::from_shared(hw.shared_kv()); + let service = crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(hw.shared_kv()); let diskio = availability::read(&service).await?; if let Ok(instances) = service.read_all_diskdb_instances().await { for (_, instance) in instances { diff --git a/app/crowdb-chunkdb/src/topology/availability.rs b/app/crowdb-chunkdb/src/topology/availability.rs index 83418b31f..1d11b119d 100644 --- a/app/crowdb-chunkdb/src/topology/availability.rs +++ b/app/crowdb-chunkdb/src/topology/availability.rs @@ -10,7 +10,7 @@ use std::{ }; pub(super) async fn read( - service: &crowdb_kv_client::ServiceRegistryClient, + service: &crowdb_paxos_kv_client::ServiceRegistryClient, ) -> Option<Vec<(u64, InstanceValue)>> { match service.read_all_instance_observations("diskio").await { Ok(instances) => Some(instances), @@ -53,7 +53,7 @@ pub(super) fn deadlines( let deadline = if extra.owned_dg_ids.contains(&group.dg_id) { instance .last_heartbeat_ms - .saturating_add(crowdb_kv_client::SERVICE_INSTANCE_TTL_MS) + .saturating_add(crowdb_paxos_kv_client::SERVICE_INSTANCE_TTL_MS) } else { 0 }; diff --git a/app/crowdb-chunkdb/src/topology/notify.rs b/app/crowdb-chunkdb/src/topology/notify.rs index 0fed5f5fb..23f4b6a25 100644 --- a/app/crowdb-chunkdb/src/topology/notify.rs +++ b/app/crowdb-chunkdb/src/topology/notify.rs @@ -12,7 +12,7 @@ use tokio::sync::mpsc; use tracing::{info, warn}; use super::{build_snapshot, TopologyCache, CHUNKDB_WATCH_PREFIXES, G0_GROUP, G0_STORE}; -use crowdb_kv_client::{HardwareClient, WatchNotifyClient}; +use crowdb_paxos_kv_client::{HardwareClient, WatchNotifyClient}; /// chunkdb-side watch/notify handler for topology updates. pub struct NotifyHandler { diff --git a/app/crowdb-chunkdb/src/topology/refresh.rs b/app/crowdb-chunkdb/src/topology/refresh.rs index 67f230525..97b1b4d0e 100644 --- a/app/crowdb-chunkdb/src/topology/refresh.rs +++ b/app/crowdb-chunkdb/src/topology/refresh.rs @@ -11,7 +11,7 @@ use std::time::Duration; use tracing::{info, warn}; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use super::{build_snapshot, TopologyCache}; diff --git a/app/crowdb-chunkdb/tests/allocation_fence_test.rs b/app/crowdb-chunkdb/tests/allocation_fence_test.rs new file mode 100644 index 000000000..4d9931399 --- /dev/null +++ b/app/crowdb-chunkdb/tests/allocation_fence_test.rs @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +//! A durable tombstone fences a fixed identity before a delayed allocation. +#[allow(dead_code)] +mod common; + +use common::cluster::{seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster}; +use crowdb_chunkdb::lifecycle::LifecycleError; +use crowdb_protocol::chunkdb::rpc::{ChunkState, ChunkType, StripType}; +use crowdb_protocol::common::ChunkId; +use std::sync::Arc; + +#[tokio::test] +async fn delete_before_allocate_fences_the_identity_across_handler_restart() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let id = ChunkId { + high: 2 << 56, + low: 1007, + }; + let (send, receive) = tokio::sync::oneshot::channel(); + let handler = Arc::clone(&harness.handler); + let delayed = tokio::spawn(async move { + receive.await.unwrap(); + handler + .allocate_chunk( + Some(id), + 1, + 1, + StripType::Mirror, + 0, + 0, + 3, + ChunkType::BtreePage, + 17, + 30_000, + ) + .await + }); + let deleted = harness.handler.delete_chunk_fenced(&id, true).await.unwrap(); + assert_eq!(deleted.state, ChunkState::Deleted as i32); + assert!(deleted.strips.is_empty()); + assert_eq!(deleted.capacity, 0); + send.send(()).unwrap(); + assert!(matches!( + delayed.await.unwrap(), + Err(LifecycleError::ChunkAlreadyExists) + )); + let restarted = ChunkdbHarness::start(&cluster).await; + assert_eq!(restarted.store.get_chunk(&id).await.unwrap(), deleted); + assert_eq!( + restarted.handler.delete_chunk_fenced(&id, true).await.unwrap(), + deleted + ); + assert!(restarted + .handler + .allocate_chunk( + Some(id), + 1, + 1, + StripType::Mirror, + 0, + 0, + 3, + ChunkType::BtreePage, + 18, + 30_000 + ) + .await + .is_err()); +} + +#[tokio::test] +async fn allocated_tree_chunk_delete_releases_strips_before_success() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let id = ChunkId { + high: 3 << 56, + low: 1011, + }; + let active = harness + .handler + .allocate_chunk( + Some(id), + 1, + 1, + StripType::Mirror, + 0, + 0, + 3, + ChunkType::PageIndex, + 17, + 30_000, + ) + .await + .unwrap(); + assert!(!active.strips.is_empty()); + let deleted = harness.handler.delete_chunk_fenced(&id, true).await.unwrap(); + assert_eq!(deleted.state, ChunkState::Deleted as i32); + assert!(deleted.strips.is_empty()); + assert_eq!(deleted.capacity, 0); + assert_eq!(harness.store.get_chunk(&id).await.unwrap(), deleted); + assert_eq!( + harness.handler.delete_chunk_fenced(&id, true).await.unwrap(), + deleted + ); +} diff --git a/app/crowdb-chunkdb/tests/allocation_retry_test.rs b/app/crowdb-chunkdb/tests/allocation_retry_test.rs index 9eb726321..05134ff80 100644 --- a/app/crowdb-chunkdb/tests/allocation_retry_test.rs +++ b/app/crowdb-chunkdb/tests/allocation_retry_test.rs @@ -6,7 +6,7 @@ mod slot_groups; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError, ChunkdbRpcTransport}; -use crowdb_kv_client::{ChunkSlotMapClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, ServiceRegistryClient}; use crowdb_protocol::{ chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, chunkdb::rpc::{AllocateChunkRequest, ChunkType}, diff --git a/app/crowdb-chunkdb/tests/allocation_rounding_test.rs b/app/crowdb-chunkdb/tests/allocation_rounding_test.rs new file mode 100644 index 000000000..e28d8e31e --- /dev/null +++ b/app/crowdb-chunkdb/tests/allocation_rounding_test.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +mod common; + +use common::cluster::{seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster, UNIT_SIZE_BYTES}; +use crowdb_protocol::chunkdb::rpc::{ChunkType, Strip, StripType}; + +#[tokio::test] +async fn requested_strip_capacity_rounds_up_to_complete_disk_units() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let unit_kb = UNIT_SIZE_BYTES / 1024; + for requested in [unit_kb - 1, unit_kb, unit_kb + 1] { + let chunk = harness + .handler + .allocate_chunk( + None, + requested, + 2, + StripType::Mirror, + 0, + 0, + 3, + ChunkType::BtreePage, + 17, + 30_000, + ) + .await + .unwrap(); + let expected_units = requested.div_ceil(unit_kb); + assert_eq!(chunk.capacity, 2 * expected_units * unit_kb); + assert_eq!(chunk.strips.len(), 2); + for (index, strip) in chunk.strips.iter().enumerate() { + assert!( + strip.capacity >= requested, + "requested {requested} KiB, allocated {} KiB", + strip.capacity + ); + assert_eq!(strip.capacity, expected_units * unit_kb); + assert_eq!(strip.chunk_offset, u32::try_from(index).unwrap() * strip.capacity); + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + panic!("expected mirror layout") + }; + assert_eq!(mirror.segments.len(), 3); + assert!(mirror + .segments + .iter() + .all(|segment| segment.unit_count == expected_units)); + } + assert_eq!(harness.store.get_chunk(&chunk.id.unwrap()).await.unwrap(), chunk); + } +} diff --git a/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs b/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs index e29eeab16..b4252ce47 100644 --- a/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs +++ b/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs @@ -7,7 +7,7 @@ mod slot_groups; use crowdb_chunkdb::allocator::DiskdbClientPool; use crowdb_diskdb_client::DiskdbClientError; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::{ common::ChunkId, diskdb_fb::{FBAllocateResponse, FBAllocateResponseArgs, FBDiskdbRetCode}, diff --git a/app/crowdb-chunkdb/tests/allocator_pool_test.rs b/app/crowdb-chunkdb/tests/allocator_pool_test.rs index e61858e0d..a686c49c2 100644 --- a/app/crowdb-chunkdb/tests/allocator_pool_test.rs +++ b/app/crowdb-chunkdb/tests/allocator_pool_test.rs @@ -4,10 +4,10 @@ use std::sync::Arc; use crowdb_chunkdb::allocator::DiskdbClientPool; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; fn pool() -> DiskdbClientPool { - let service = ServiceRegistryClient::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))); + let service = ServiceRegistryClient::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))); DiskdbClientPool::new(service) } diff --git a/app/crowdb-chunkdb/tests/chunk_tree_runtime_test.rs b/app/crowdb-chunkdb/tests/chunk_tree_runtime_test.rs new file mode 100644 index 000000000..61c4a41ce --- /dev/null +++ b/app/crowdb-chunkdb/tests/chunk_tree_runtime_test.rs @@ -0,0 +1,430 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Real KV, `ChunkDB` and `DiskDB` handlers with the existing volatile `DiskIO` wire +//! fixture. This proves the host factory/catalog/Tree path, not disk durability. + +mod common; + +use common::cluster::{seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster}; +use common::diskio_stub::TestDiskio; +use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, SmallWritePolicy}; +use crowdb_chunk_kv::{ + MutationOperation, Partition, PartitionConfig, PartitionId, PartitionRange, RequestId, +}; +use crowdb_chunk_kv_server::{BootstrapPartitionConfig, ChunkKvStorage}; +use crowdb_chunk_stream::{StreamName, StreamPurpose, StreamRecord, StreamRecordState, StreamRegistry}; +use crowdb_chunkdb::{metrics::ChunkdbMetrics, service::ChunkdbRpcService}; +use crowdb_common::metrics::{MetricsRegistry, MetricsSampler}; +use crowdb_paxos_kv_client::{StreamProcessAuthority, StreamProcessLeaseClient, StreamProcessLeaseTiming}; +use crowdb_protocol::chunk_kv::Id128; +use crowdb_protocol::chunkdb::rpc::{ChunkState, ChunkType}; +use crowdb_tree_ffi::{ChunkPageStoreOptions, Config}; +use std::{sync::Arc, time::Instant}; + +async fn storage( + cluster: &KvCluster, + io: &ChunkIoClient, + process: &Arc<StreamProcessAuthority>, +) -> ChunkKvStorage { + let storage = ChunkKvStorage::from_parts_parallel_for_tests( + cluster.make_crowdb_client(), + io.clone(), + 0, + 300_000, + 2, + ) + .await + .unwrap(); + storage.bind_owner_process(process.clone()).unwrap(); + storage +} + +struct TestRuntimeStack { + cluster: KvCluster, + _diskdb: DiskdbServer, + _disks: Vec<TestDiskio>, + harness: ChunkdbHarness, + rpc: Arc<crowdb_rpc_ffi::RpcServer>, + io: ChunkIoClient, + process: Arc<StreamProcessAuthority>, +} +impl TestRuntimeStack { + async fn start() -> Self { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let diskdb = DiskdbServer::start(&cluster).await; + let mut disks = Vec::new(); + for index in 0..4_u64 { + disks.push( + TestDiskio::start( + &cluster.make_service_registry_client(), + 2000 + index, + 100 + index, + 10 + index, + 1000 + index, + ) + .await, + ); + } + let harness = ChunkdbHarness::start(&cluster).await; + let rpc = Arc::new(crowdb_rpc_ffi::RpcServer::new(None)); + rpc.listen("127.0.0.1", 0).unwrap(); + let mut metrics = MetricsRegistry::new(); + let service = Arc::new(ChunkdbRpcService::new( + harness.handler.clone(), + Arc::new(ChunkdbMetrics::register(&mut metrics)), + tokio::runtime::Handle::current(), + )); + service.register_handlers(&rpc); + rpc.start(); + cluster + .make_service_registry_client() + .heartbeat_chunkdb(100, &format!("http://127.0.0.1:{}", rpc.port())) + .await + .unwrap(); + let slots = crowdb_paxos_kv_client::ChunkSlotMapClient::new(cluster.make_crowdb_client()); + slots + .initialize_layout(&crowdb_protocol::chunk_slot::ChunkSlotBootstrap { + service_instances: vec![100], + storage_groups: vec![crowdb_protocol::chunk_slot::ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }) + .await + .unwrap(); + slots.initialize_service_epochs().await.unwrap(); + let io = ChunkIoClient::connect_with_kv( + ChunkIoClientConfig { + management_seeds: vec![], + diskio_connections_per_endpoint: 1, + diskio_rpc_workers: 1, + small_write: SmallWritePolicy::new(ChunkType::Wal), + }, + cluster.make_crowdb_client(), + ) + .await + .unwrap(); + let now = StreamRecord::creation_time_ms(); + let lease = StreamProcessLeaseClient::new(cluster.make_crowdb_client()) + .claim(StreamPurpose::Wal, 71, [7; 16], None, now, 300_000) + .await + .unwrap(); + let process = Arc::new( + StreamProcessAuthority::new(lease, StreamProcessLeaseTiming::default(), now, Instant::now()) + .unwrap(), + ); + Self { + cluster, + _diskdb: diskdb, + _disks: disks, + harness, + rpc, + io, + process, + } + } +} +impl Drop for TestRuntimeStack { + fn drop(&mut self) { + self.rpc.stop(); + } +} + +async fn checkpoint_values( + host: &ChunkKvStorage, + bootstrap: &BootstrapPartitionConfig, +) -> ( + Partition, + Arc<crowdb_tree_ffi::PageStore>, + crowdb_chunk_kv::Checkpoint, +) { + host.streams() + .registry() + .create(StreamRecord { + purpose: StreamPurpose::Wal, + stream_name: bootstrap.stream_name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: Some(format!( + "{:016x}{:016x}", + bootstrap.partition_id.high, bootstrap.partition_id.low + )), + created_at_ms: StreamRecord::creation_time_ms(), + }) + .await + .unwrap(); + let stream = host + .create_registered_stream(bootstrap.stream_name, 1) + .await + .unwrap(); + let pages = host + .open_durable_tree_page_store( + ChunkPageStoreOptions { + tree_id: 42, + owner_epoch: 1, + open_generation: 0, + pack_bytes: 0, + iu_size: 0, + max_concurrent_packs: 0, + materialization_bytes_per_pass: 0, + max_chunk_bytes: 0, + mirror_copies: 0, + }, + 1, + ) + .await + .unwrap(); + let owned_pages = pages.clone(); + let config = Config { + runtime: Some(host.tree_runtime().clone()), + ..Default::default() + }; + let partition = tokio::task::spawn_blocking(move || { + Partition::open_native( + PartitionId { high: 17, low: 18 }, + PartitionRange { + start: Some(vec![]), + end: None, + }, + 1, + PartitionConfig { + retained_results: 1, + ..Default::default() + }, + 42, + config, + owned_pages, + stream, + ) + }) + .await + .unwrap() + .unwrap(); + let mut checkpoint = None; + for sequence in 1..=4_u64 { + partition + .mutate( + 1, + RequestId { + client_high: 99, + client_low: 1, + client_sequence: sequence, + }, + MutationOperation::Put { + key: b"key".to_vec(), + value: vec![u8::try_from(sequence).unwrap(); 20_000], + }, + ) + .await + .unwrap(); + checkpoint = Some(partition.checkpoint(1).await.unwrap()); + } + let checkpoint = checkpoint.unwrap(); + (partition, pages, checkpoint) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn runtime_host_recovers_root_wal_pair_and_collects_real_allocations() { + let stack = TestRuntimeStack::start().await; + let host = Arc::new(storage(&stack.cluster, &stack.io, &stack.process).await); + let bootstrap = BootstrapPartitionConfig { + partition_id: Id128 { high: 17, low: 18 }, + tree_id: 42, + stream_name: StreamName { high: 19, low: 20 }, + owner_epoch: 1, + metadata_group_id: 1, + }; + let (partition, pages, checkpoint) = checkpoint_values(&host, &bootstrap).await; + let service = Arc::new(crowdb_chunk_kv_server::ChunkKvService::new(stack.process.clone(), 8).unwrap()); + service.install_partition(&partition).unwrap(); + let mut sampler = + crowdb_chunk_kv_server::metrics::TreeResources::new(host.tree_runtime().clone(), &service) + .with_storage(&host); + sampler.sample(); + let mut resource_log = Vec::new(); + sampler.flush(&mut resource_log, "test").unwrap(); + let resource_log = String::from_utf8(resource_log).unwrap(); + let pins = resource_log + .lines() + .find(|line| line.contains("host.tree.resource.chunk_pin_release_slots ")) + .unwrap(); + assert!(pins.contains("limit=1048576"), "{pins}"); + let registered: u64 = pins + .split_whitespace() + .find_map(|field| field.strip_prefix("used=")) + .unwrap() + .parse() + .unwrap(); + assert!(registered > 0, "{pins}"); + assert!(pins.contains("allocated_sample_max="), "{pins}"); + drop((sampler, service)); + assert!(checkpoint.replay_offset > 0); + assert_eq!(pages.wal_replay_offset().unwrap(), checkpoint.replay_offset); + assert!(pages.reclaim_chunk_generations_before(42, checkpoint.root_manifest_generation) > 0); + let before = stack.harness.store.list_chunks(None, 100).await.unwrap(); + assert!(before + .iter() + .any(|chunk| chunk.chunk_type == ChunkType::BtreePage as i32)); + partition.suspend_for_shutdown(1).await.unwrap(); + tokio::task::spawn_blocking(move || drop((partition, pages))) + .await + .unwrap(); + host.drain_pin_releases().await; + drop(host); + + let reopened_host = storage(&stack.cluster, &stack.io, &stack.process).await; + let recovered = reopened_host.bootstrap_partition(&bootstrap).await.unwrap(); + recovered.activate_recovered(1).unwrap(); + assert_eq!( + recovered.get(1, b"key", None).await.unwrap().unwrap().value, + vec![4; 20_000] + ); + let stable = recovered.checkpoint(1).await.unwrap(); + assert_eq!(stable.replay_offset, checkpoint.replay_offset); + let frontier = recovered.snapshot(); + for _ in 0..64 { + if !recovered.maintain_serving_metadata(1).await.unwrap() { + break; + } + } + let after = stack.harness.store.list_chunks(None, 100).await.unwrap(); + assert!(after + .iter() + .any(|chunk| chunk.chunk_type == ChunkType::BtreePage as i32 + && chunk.state == ChunkState::Deleted as i32 + && chunk.strips.is_empty())); + assert_eq!(recovered.snapshot().applied_seq, frontier.applied_seq); + assert_eq!( + recovered.snapshot().journal_durable_offset, + frontier.journal_durable_offset + ); + recovered.suspend_for_shutdown(1).await.unwrap(); + tokio::task::spawn_blocking(move || drop(recovered)) + .await + .unwrap(); + reopened_host.drain_pin_releases().await; +} + +async fn runtime_tree_config(host: &ChunkKvStorage, tree_id: u64) -> Config { + let pages = host + .open_durable_tree_page_store( + ChunkPageStoreOptions { + tree_id, + owner_epoch: 1, + open_generation: 0, + pack_bytes: 0, + iu_size: 0, + max_concurrent_packs: 0, + materialization_bytes_per_pass: 0, + max_chunk_bytes: 0, + mirror_copies: 0, + }, + 1, + ) + .await + .unwrap(); + Config { + page_store: Some(pages), + runtime: Some(host.tree_runtime().clone()), + ..Config::default() + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn chunk_partition_snapshot_rebuilds_after_writer_close_without_reopening_source() { + use crowdb_chunk_kv::{ChunkKvError, CrowdbPartitionTree, PartitionTree}; + let stack = TestRuntimeStack::start().await; + let host = storage(&stack.cluster, &stack.io, &stack.process).await; + let config = runtime_tree_config(&host, 42).await; + let source = tokio::task::spawn_blocking(move || CrowdbPartitionTree::open(42, &config)) + .await + .unwrap() + .unwrap(); + source + .apply( + 1, + &MutationOperation::Put { + key: b"a".to_vec(), + value: b"left".to_vec(), + }, + ) + .await + .unwrap(); + source + .apply( + 2, + &MutationOperation::Put { + key: b"z".to_vec(), + value: vec![b'v'; 9000], + }, + ) + .await + .unwrap(); + let (_, coverage, snapshot) = source.checkpoint_snapshot(0).await.unwrap(); + source + .apply( + 3, + &MutationOperation::Put { + key: b"z".to_vec(), + value: b"new".to_vec(), + }, + ) + .await + .unwrap(); + tokio::task::spawn_blocking(move || drop(source)).await.unwrap(); + assert_eq!(snapshot.get(b"z").await.unwrap().unwrap().value, vec![b'v'; 9000]); + assert!(matches!( + snapshot.advance_noop(4).await, + Err(ChunkKvError::InvalidRequest(_)) + )); + let config = runtime_tree_config(&host, 43).await; + let (child, stats) = snapshot + .rebuild_range( + 43, + &PartitionRange { + start: Some(b"m".to_vec()), + end: None, + }, + config, + ) + .await + .unwrap(); + assert_eq!(stats.entries_emitted, 1); + assert_eq!(child.last_applied_seq(), coverage); + assert_eq!(child.get(b"z").await.unwrap().unwrap().value, vec![b'v'; 9000]); + tokio::task::spawn_blocking(move || drop((child, snapshot))) + .await + .unwrap(); + host.drain_pin_releases().await; +} + +#[test] +fn diskio_fixture_only_acknowledges_valid_fixed_read_ages() { + use common::diskio_stub::checked_read_age; + use crowdb_protocol::diskio_fb::FBDiskIoRetCode; + use crowdb_protocol::timing::{MAX_WRITE_CLOCK_SKEW_MS, MAX_WRITE_REQUEST_AGE_MS}; + let now = 1_000_000; + assert_eq!(checked_read_age(now, true, 0, now), Ok(true)); + assert_eq!(checked_read_age(now, false, 16, now), Ok(true)); + assert_eq!(checked_read_age(0, false, 16, now), Ok(false)); + for (created, probe, size) in [ + (0, true, 0), + (now, true, 1), + ( + now - MAX_WRITE_REQUEST_AGE_MS - MAX_WRITE_CLOCK_SKEW_MS, + false, + 16, + ), + (now + MAX_WRITE_CLOCK_SKEW_MS + 1, false, 16), + (u64::MAX, false, 16), + ] { + assert_eq!( + checked_read_age(created, probe, size, now), + Err(FBDiskIoRetCode::OldRequest) + ); + } +} diff --git a/app/crowdb-chunkdb/tests/client_owner_refresh_test.rs b/app/crowdb-chunkdb/tests/client_owner_refresh_test.rs index a7fa3f8d5..bd55000fc 100644 --- a/app/crowdb-chunkdb/tests/client_owner_refresh_test.rs +++ b/app/crowdb-chunkdb/tests/client_owner_refresh_test.rs @@ -6,7 +6,7 @@ mod slot_groups; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv_client::{ChunkSlotMapClient, RangeBindingClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunk_slot::{ChunkSlot, ChunkSlotBootstrap, ChunkStorageGroup}; use crowdb_protocol::chunkdb::rpc::AllocateChunkRequest; use crowdb_protocol::fb::FBMsgType; diff --git a/app/crowdb-chunkdb/tests/common/cluster.rs b/app/crowdb-chunkdb/tests/common/cluster.rs index 3d9aa5fa1..a6207e851 100644 --- a/app/crowdb-chunkdb/tests/common/cluster.rs +++ b/app/crowdb-chunkdb/tests/common/cluster.rs @@ -33,22 +33,22 @@ use crowdb_diskdb::recovery::ZoneLoader; use crowdb_diskdb::scanner::ScanState; use crowdb_diskdb::service::DiskdbRpcService; use crowdb_diskdb_client::DiskdbRpcTransport; -use crowdb_kv::cluster::KvServer; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv::cluster::PaxosKvServer; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{RelocateSegmentHandoffRequest, RelocationHandoffDisposition}; use crowdb_protocol::common::{DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{DiskGroupValue, DiskType, DiskValue, Segment}; pub struct KvCluster { - memory_store: Arc<crowdb_kv::cluster::PxKvStore>, + memory_store: Arc<crowdb_paxos_kv::cluster::PxKvStore>, endpoint: String, } impl KvCluster { /// Pinned memory dependency for component handler tests. pub async fn start_pinned() -> Self { - use crowdb_kv::cluster::group::PxGroup; - use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; + use crowdb_paxos_kv::cluster::group::PxGroup; + use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); for id in [0, 1] { store.add_group(PxGroup::new( @@ -64,8 +64,8 @@ impl KvCluster { } } - pub fn make_crowdb_client(&self) -> Arc<CrowdbKvClient> { - let kv = CrowdbKvClient::new(ClientConfig::new(vec![])); + pub fn make_crowdb_client(&self) -> Arc<PaxosKvClient> { + let kv = PaxosKvClient::new(ClientConfig::new(vec![])); kv.seed_leader(0, 0, self.endpoint.clone()); kv.seed_leader(0, 1, self.endpoint.clone()); Arc::new(kv) diff --git a/app/crowdb-chunkdb/tests/common/diskio_stub.rs b/app/crowdb-chunkdb/tests/common/diskio_stub.rs index 0f1d5ac4e..8d5a9fce8 100644 --- a/app/crowdb-chunkdb/tests/common/diskio_stub.rs +++ b/app/crowdb-chunkdb/tests/common/diskio_stub.rs @@ -4,10 +4,11 @@ //! Volatile protocol stub for pinned placement/task proofs. Packaged Diskio //! discovery, bytes and durability are verified by the container acceptance. -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::diskio_fb::{ - FBDiskFsyncRequest, FBDiskFsyncResponse, FBDiskFsyncResponseArgs, FBDiskReadRequest, FBDiskReadResponse, - FBDiskReadResponseArgs, FBDiskWriteRequest, FBDiskWriteResponse, FBDiskWriteResponseArgs, + FBDiskFsyncRequest, FBDiskFsyncResponse, FBDiskFsyncResponseArgs, FBDiskIoRetCode, FBDiskReadRequest, + FBDiskReadResponse, FBDiskReadResponseArgs, FBDiskWriteRequest, FBDiskWriteResponse, + FBDiskWriteResponseArgs, }; use crowdb_protocol::fb::{ ConnectionPingRequest, ConnectionPingResponse, ConnectionPingResponseArgs, FBMsgType, @@ -155,11 +156,34 @@ fn response( } else if kind == FBMsgType::EDiskReadRequest.0 as u16 { control.read_calls.fetch_add(1, Ordering::Relaxed); let request = flatbuffers::root::<FBDiskReadRequest>(ctrl).unwrap(); - payload = read_blocks(blocks, &request); + let now_ms = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + let age = checked_read_age( + request.read_create_time_ms(), + request.fence_probe(), + request.size(), + now_ms, + ); + let (ret_code, request_age_checked) = match age { + Ok(checked) => { + if !request.fence_probe() { + payload = read_blocks(blocks, &request); + } + (FBDiskIoRetCode::Success, checked) + } + Err(error) => (error, false), + }; let reply = FBDiskReadResponse::create( &mut builder, &FBDiskReadResponseArgs { id: request.id(), + ret_code, + request_age_checked, ..Default::default() }, ); @@ -208,3 +232,26 @@ fn read_blocks(blocks: &Blocks, request: &FBDiskReadRequest<'_>) -> Vec<u8> { } bytes } + +// Mirror the production DiskIO admission age ceiling for synchronous fixture reads. +pub fn checked_read_age( + created_ms: u64, + probe: bool, + size: u32, + now_ms: u64, +) -> Result<bool, FBDiskIoRetCode> { + use crowdb_protocol::timing::{MAX_WRITE_CLOCK_SKEW_MS, MAX_WRITE_REQUEST_AGE_MS}; + if probe && (created_ms == 0 || size != 0) { + return Err(FBDiskIoRetCode::OldRequest); + } + if created_ms == 0 { + return Ok(false); + } + let deadline = created_ms.checked_add(MAX_WRITE_REQUEST_AGE_MS + MAX_WRITE_CLOCK_SKEW_MS); + if created_ms.saturating_sub(now_ms) > MAX_WRITE_CLOCK_SKEW_MS + || deadline.map_or(true, |deadline| deadline <= now_ms) + { + return Err(FBDiskIoRetCode::OldRequest); + } + Ok(true) +} diff --git a/app/crowdb-chunkdb/tests/common/dynamic_runtime.rs b/app/crowdb-chunkdb/tests/common/dynamic_runtime.rs index 6c856f06e..aa3269558 100644 --- a/app/crowdb-chunkdb/tests/common/dynamic_runtime.rs +++ b/app/crowdb-chunkdb/tests/common/dynamic_runtime.rs @@ -6,7 +6,7 @@ use crowdb_chunkdb::{ range_guard::RangeGuard, routing::{BindingCache, BindingTable}, }; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_protocol::{ chunk_domain::ChunkDomain, chunk_slot::{ChunkSlotAuthority, ChunkSlotBootstrap, ChunkSlotMap, ChunkStorageGroup}, @@ -22,7 +22,7 @@ pub struct TestDynamicRuntime { workers: Vec<tokio::task::JoinHandle<()>>, } impl TestDynamicRuntime { - pub async fn start(kv: Arc<CrowdbKvClient>, map: &ChunkSlotMap<ChunkSlotAuthority>, owner: u64) -> Self { + pub async fn start(kv: Arc<PaxosKvClient>, map: &ChunkSlotMap<ChunkSlotAuthority>, owner: u64) -> Self { let routes = BindingCache::new(); routes .replace(BindingTable::new( diff --git a/app/crowdb-chunkdb/tests/common/readiness_failures.rs b/app/crowdb-chunkdb/tests/common/readiness_failures.rs index c50dc1fd9..95513f5e7 100644 --- a/app/crowdb-chunkdb/tests/common/readiness_failures.rs +++ b/app/crowdb-chunkdb/tests/common/readiness_failures.rs @@ -3,15 +3,15 @@ use crowdb_chunkdb::range_guard::RangeGuard; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError}; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_protocol::chunk_id::{ChunkIdParts, CHUNK_TYPE_S3}; use crowdb_rpc_ffi::RpcServer; use std::time::Duration; pub async fn assert_storage_and_owner_failures( client: &ChunkdbClient, - data: &CrowdbKvClient, - control: &CrowdbKvClient, + data: &PaxosKvClient, + control: &PaxosKvClient, guard: &RangeGuard, server: &RpcServer, ) { diff --git a/app/crowdb-chunkdb/tests/common/readiness_registry.rs b/app/crowdb-chunkdb/tests/common/readiness_registry.rs index adf4896fc..676baad56 100644 --- a/app/crowdb-chunkdb/tests/common/readiness_registry.rs +++ b/app/crowdb-chunkdb/tests/common/readiness_registry.rs @@ -2,14 +2,14 @@ // Licensed under the Apache License, Version 2.0. use crowdb_chunkdb_client::ChunkdbClient; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_rpc_ffi::RpcServer; use std::time::Duration; pub async fn assert_warm_registry_independence( client: &ChunkdbClient, - control: &CrowdbKvClient, - data: &CrowdbKvClient, + control: &PaxosKvClient, + data: &PaxosKvClient, server: &RpcServer, endpoint: String, ) { diff --git a/app/crowdb-chunkdb/tests/common/repair_targets.rs b/app/crowdb-chunkdb/tests/common/repair_targets.rs index cf9e15e4b..e6d81d203 100644 --- a/app/crowdb-chunkdb/tests/common/repair_targets.rs +++ b/app/crowdb-chunkdb/tests/common/repair_targets.rs @@ -14,7 +14,7 @@ use crowdb_chunkdb::routing::{default_binding_table, BindingCache}; use crowdb_chunkdb::task::{TaskExecutor, TaskManager, TaskStore}; use crowdb_chunkdb::topology::build_snapshot; use crowdb_common::metrics::MetricsRegistry; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::chunk_task::{ChunkTaskValue, RepairTargetCheckpoint, RepairTargetPhase}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkType, Strip, StripType}; use crowdb_protocol::common::{ChunkId, InstanceValue}; diff --git a/app/crowdb-chunkdb/tests/common/runtime_context.rs b/app/crowdb-chunkdb/tests/common/runtime_context.rs index b3f9fa351..2570f7681 100644 --- a/app/crowdb-chunkdb/tests/common/runtime_context.rs +++ b/app/crowdb-chunkdb/tests/common/runtime_context.rs @@ -10,12 +10,12 @@ use crowdb_chunkdb::routing::BindingCache; use crowdb_chunkdb::runtime::RuntimeContext; use crowdb_chunkdb::topology::TopologyCache; use crowdb_common::metrics::MetricsRegistry; -use crowdb_kv_client::{CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{PaxosKvClient, ServiceRegistryClient}; use std::sync::Arc; use std::time::Duration; pub fn runtime_context( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, bindings: BindingCache, guard: Arc<RangeGuard>, ) -> (RuntimeContext, tokio::sync::watch::Sender<bool>) { diff --git a/app/crowdb-chunkdb/tests/common/slot_groups.rs b/app/crowdb-chunkdb/tests/common/slot_groups.rs index 1954464b4..69c28c50b 100644 --- a/app/crowdb-chunkdb/tests/common/slot_groups.rs +++ b/app/crowdb-chunkdb/tests/common/slot_groups.rs @@ -1,10 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunk_task::{ ChunkTaskState, ChunkTaskValue, CHUNK_TASK_SCHEMA_VERSION, TASK_KIND_FINALIZE_CHUNK, }; @@ -13,7 +13,7 @@ use std::sync::Arc; pub struct TestGroups { pub server: Arc<PxKvStore>, - pub kv: Arc<CrowdbKvClient>, + pub kv: Arc<PaxosKvClient>, } impl TestGroups { @@ -26,7 +26,7 @@ impl TestGroups { )); } server.start().await.unwrap(); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))); for group in 0..=3 { kv.seed_leader(0, group, server.listen_addr().unwrap().to_string()); } diff --git a/app/crowdb-chunkdb/tests/common/topology_availability.rs b/app/crowdb-chunkdb/tests/common/topology_availability.rs index 6efba4a20..23a1f6cf0 100644 --- a/app/crowdb-chunkdb/tests/common/topology_availability.rs +++ b/app/crowdb-chunkdb/tests/common/topology_availability.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::{common::DiskioHardwareScope, sysdata::DiskGroupEntry}; pub async fn heartbeat(registry: &ServiceRegistryClient, group: &DiskGroupEntry) { diff --git a/app/crowdb-chunkdb/tests/dynamic_retry_test.rs b/app/crowdb-chunkdb/tests/dynamic_retry_test.rs index 05ad8c568..313d4b0af 100644 --- a/app/crowdb-chunkdb/tests/dynamic_retry_test.rs +++ b/app/crowdb-chunkdb/tests/dynamic_retry_test.rs @@ -13,7 +13,7 @@ use crowdb_chunkdb::{ storage::ChunkStore, }; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv_client::{ChunkSlotMapClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, ServiceRegistryClient}; use crowdb_protocol::chunk_slot::{ChunkSlot, ChunkSlotBootstrap, ChunkStorageGroup}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, DeleteChunkRequest}; use crowdb_protocol::common::ChunkId; diff --git a/app/crowdb-chunkdb/tests/e2e_test.rs b/app/crowdb-chunkdb/tests/e2e_test.rs index cc06b8e39..ed287249f 100644 --- a/app/crowdb-chunkdb/tests/e2e_test.rs +++ b/app/crowdb-chunkdb/tests/e2e_test.rs @@ -20,7 +20,7 @@ use crowdb_chunkdb::service::ChunkdbRpcService; use crowdb_chunkdb::storage::ChunkStore; use crowdb_chunkdb::topology::TopologyCache; use crowdb_common::metrics::MetricsRegistry; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::common::{ChunkId, HwStatus}; use crowdb_protocol::diskdb::rpc::DiskGroupValue; use crowdb_protocol::sysdata::DiskGroupEntry; @@ -49,7 +49,7 @@ fn build_test_topology() -> TopologyCache { /// Build a fully wired LifecycleHandler with a dummy KV client /// (calls will fail, but construction verifies all wiring). fn build_handler() -> LifecycleHandler { - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![ "http://127.0.0.1:1".into() ]))); let svc = ServiceRegistryClient::from_shared(Arc::clone(&kv)); diff --git a/app/crowdb-chunkdb/tests/http_readiness_test.rs b/app/crowdb-chunkdb/tests/http_readiness_test.rs index ebaa33153..972e8e1bc 100644 --- a/app/crowdb-chunkdb/tests/http_readiness_test.rs +++ b/app/crowdb-chunkdb/tests/http_readiness_test.rs @@ -8,7 +8,7 @@ mod http_readiness; mod slot_groups; use crowdb_chunkdb::{range_guard::RangeGuard, readiness::RangeReadiness}; -use crowdb_kv_client::ChunkSlotMapClient; +use crowdb_paxos_kv_client::ChunkSlotMapClient; use http_readiness::TestReadiness; use std::{ sync::{atomic::Ordering, Arc}, diff --git a/app/crowdb-chunkdb/tests/journal_handoff_test.rs b/app/crowdb-chunkdb/tests/journal_handoff_test.rs index 7618a3a11..75d6a1929 100644 --- a/app/crowdb-chunkdb/tests/journal_handoff_test.rs +++ b/app/crowdb-chunkdb/tests/journal_handoff_test.rs @@ -6,11 +6,11 @@ use std::time::Duration; use bytes::Bytes; use crowdb_chunkdb::storage::{decode_chunk_for_tests, encode_chunk_for_tests}; -use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; -use crowdb_kv::paxos::learner::PxLearner; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; +use crowdb_paxos_kv::paxos::learner::PxLearner; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkType}; use crowdb_protocol::common::ChunkId; @@ -97,13 +97,30 @@ async fn journal_cas_retains_future_revision_through_flush_and_reopen() { handle.flush().unwrap(); assert_eq!(handle.snapshot().unwrap(), 851); - learn(&group, 853, encode_puts(&[(&chunk_key(), &metadata(60, 24040))])).await; + let pending_group = Arc::clone(&group); + let pending = tokio::spawn(async move { + learn( + &pending_group, + 853, + encode_puts(&[(&chunk_key(), &metadata(60, 24040))]), + ) + .await; + }); + tokio::time::timeout(Duration::from_secs(5), async { + while group.local_replica().last_chosen_slot() < 853 { + tokio::task::yield_now().await; + } + }) + .await + .expect("slot 853 must be chosen while its contiguous apply waits for 852"); let engine = group.local_replica().learner.engine(); - // Keep slot 852 absent: every maintenance pass must retain the future value. + // The chosen task owns the future value until the gap is repaired. Flush + // must preserve that work while reads still observe the contiguous prefix. for _ in 0..3 { handle.flush().unwrap(); assert_eq!(handle.snapshot().unwrap(), 851); - assert_metadata(engine, 853, 60, 24040).await; + assert_metadata(engine, 843, 58, 23039).await; + assert!(!pending.is_finished(), "future apply must wait for the gap"); } group.set_next_slot(854); let journal: Vec<u8> = (0..24643).map(|i| u8::try_from(i % 251).unwrap()).collect(); @@ -116,10 +133,14 @@ async fn journal_cas_retains_future_revision_through_flush_and_reopen() { tokio::task::yield_now().await; handle.flush().unwrap(); assert_eq!(handle.snapshot().unwrap(), 851); - assert_metadata(engine, 853, 60, 24040).await; + assert_metadata(engine, 843, 58, 23039).await; assert!(!cas.is_finished(), "the apply fence must not cross the gap"); } learn(&group, 852, Vec::new()).await; + tokio::time::timeout(Duration::from_secs(5), pending) + .await + .expect("gap repair must release the chosen apply") + .unwrap(); let result = tokio::time::timeout(Duration::from_secs(5), cas) .await .unwrap() diff --git a/app/crowdb-chunkdb/tests/mixed_geometry_test.rs b/app/crowdb-chunkdb/tests/mixed_geometry_test.rs index a6e2c2f04..693a2478a 100644 --- a/app/crowdb-chunkdb/tests/mixed_geometry_test.rs +++ b/app/crowdb-chunkdb/tests/mixed_geometry_test.rs @@ -6,7 +6,7 @@ mod common; use crowdb_chunkdb::allocator::{ChunkAllocator, DiskdbClientPool, StripAllocType}; use crowdb_chunkdb::selector::PlacementConstraints; use crowdb_chunkdb::topology::build_snapshot; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use std::sync::Arc; #[tokio::test] diff --git a/app/crowdb-chunkdb/tests/readiness_probe_test.rs b/app/crowdb-chunkdb/tests/readiness_probe_test.rs index cc34224e1..a08abb9d6 100644 --- a/app/crowdb-chunkdb/tests/readiness_probe_test.rs +++ b/app/crowdb-chunkdb/tests/readiness_probe_test.rs @@ -9,8 +9,8 @@ use crowdb_chunkdb::{ storage::ChunkStore, }; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv_client::{ChunkSlotMapClient, ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, ClientConfig, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::{ chunk_domain::ChunkDomain, chunk_id::CHUNK_TYPE_S3, @@ -66,7 +66,7 @@ async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() .unwrap() .unwrap(); } - let control = Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))); + let control = Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))); control.seed_leader(0, 0, test.server.listen_addr().unwrap().to_string()); let registry = ServiceRegistryClient::from_shared(Arc::clone(&control)); registry @@ -75,7 +75,7 @@ async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() .unwrap(); let client = ChunkdbClient::new(registry, Arc::new(ChunkdbRpcTransport::new())); Box::pin(client.observe_readiness()).await.unwrap(); - let reference = crowdb_kv_client::RangeBindingClient::from_shared(Arc::clone(&control)); + let reference = crowdb_paxos_kv_client::RangeBindingClient::from_shared(Arc::clone(&control)); let _ = control.drain_window(); reference.refresh().await.unwrap(); let binding_scans = control.drain_window().scan.len(); diff --git a/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs b/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs index f559c1926..e4ffddbe0 100644 --- a/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs +++ b/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs @@ -7,7 +7,7 @@ mod repair_targets; use crowdb_chunkdb::repair::decode_payload; use crowdb_chunkdb::task::SegmentOwnerResolver; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::chunk_task::{ChunkTaskState, RepairTargetPhase}; use crowdb_protocol::chunkdb::rpc::{SegmentOwnerDisposition, Strip}; use crowdb_protocol::key::{BinaryKey, BusyBlockKey}; diff --git a/app/crowdb-chunkdb/tests/runtime_scope_test.rs b/app/crowdb-chunkdb/tests/runtime_scope_test.rs index 3a780b6e6..7683d752d 100644 --- a/app/crowdb-chunkdb/tests/runtime_scope_test.rs +++ b/app/crowdb-chunkdb/tests/runtime_scope_test.rs @@ -110,7 +110,7 @@ async fn production_runtime_dispatches_all_purposes_and_lists_only_owned_chunks( #[tokio::test] async fn native_snapshots_route_by_service_slot_and_refresh_endpoints_without_storage_map() { use crowdb_chunkdb_client::ChunkdbClient; - use crowdb_kv_client::{ChunkSlotMapClient, ServiceRegistryClient}; + use crowdb_paxos_kv_client::{ChunkSlotMapClient, ServiceRegistryClient}; use crowdb_protocol::chunk_slot::ChunkSlot; use crowdb_protocol::common::ChunkId; diff --git a/app/crowdb-chunkdb/tests/slot_storage_test.rs b/app/crowdb-chunkdb/tests/slot_storage_test.rs index d2c9e9205..bd06fd0f7 100644 --- a/app/crowdb-chunkdb/tests/slot_storage_test.rs +++ b/app/crowdb-chunkdb/tests/slot_storage_test.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use crowdb_chunkdb::routing::{BindingCache, BindingTable}; use crowdb_chunkdb::storage::ChunkStore; use crowdb_chunkdb::task::TaskStore; -use crowdb_kv_client::{ChunkSlotMapClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, GetOutcome, ReadMode}; use crowdb_protocol::chunk_slot::{ChunkSlot, ChunkSlotBootstrap, ChunkStorageGroup}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkType}; use crowdb_protocol::common::ChunkId; diff --git a/app/crowdb-chunkdb/tests/submission_epoch_test.rs b/app/crowdb-chunkdb/tests/submission_epoch_test.rs index 683b91932..f5f9f828d 100644 --- a/app/crowdb-chunkdb/tests/submission_epoch_test.rs +++ b/app/crowdb-chunkdb/tests/submission_epoch_test.rs @@ -158,7 +158,7 @@ async fn stale_execution_cannot_write_metadata_reservations_or_tasks_and_fresh_e #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn epoch_publication_does_not_drain_an_accepted_data_write_and_the_final_check_race_is_allowed() { - use crowdb_kv_client::{ChunkSlotMapClient, GetOutcome, ReadMode}; + use crowdb_paxos_kv_client::{ChunkSlotMapClient, GetOutcome, ReadMode}; use crowdb_protocol::chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}; use std::{sync::Arc, time::Duration}; let test = slot_groups::TestGroups::start().await; diff --git a/app/crowdb-chunkdb/tests/topology_availability_test.rs b/app/crowdb-chunkdb/tests/topology_availability_test.rs index 8bf884342..1d585ea2e 100644 --- a/app/crowdb-chunkdb/tests/topology_availability_test.rs +++ b/app/crowdb-chunkdb/tests/topology_availability_test.rs @@ -9,7 +9,7 @@ use crowdb_chunkdb::{ selector::{MirrorPlacement, PlacementConstraints}, topology::build_snapshot, }; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::{ common::{HwStatus, InstanceValue}, key::InstanceKey, @@ -35,7 +35,7 @@ async fn expired_physical_io_is_excluded_without_changing_hardware_and_returns_a .as_millis(), ) .unwrap() - + crowdb_kv_client::SERVICE_INSTANCE_TTL_MS + + crowdb_paxos_kv_client::SERVICE_INSTANCE_TTL_MS + 1; assert!(before.healthy_disk_groups_at_for_tests(expired).is_empty()); let stopped = &groups[0]; @@ -50,12 +50,12 @@ async fn expired_physical_io_is_excluded_without_changing_hardware_and_returns_a 0, 0, key.as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await .unwrap(); - let crowdb_kv_client::GetOutcome::Found { value, .. } = value else { + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = value else { panic!("missing IO registration") }; let mut observation: InstanceValue = serde_json::from_slice(&value).unwrap(); diff --git a/app/crowdb-cli/Cargo.toml b/app/crowdb-cli/Cargo.toml index 6fe0b758a..25d8bc84a 100644 --- a/app/crowdb-cli/Cargo.toml +++ b/app/crowdb-cli/Cargo.toml @@ -18,7 +18,7 @@ path = "src/main.rs" futures = "0.3" uuid = { version = "1", features = ["v4"] } crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } -crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../../lib/crowdb-paxos-kv-client" } crowdb-common = { path = "../../lib/crowdb-common/rust" } crowdb-protocol = { path = "../../lib/crowdb-protocol" } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi" } @@ -27,7 +27,7 @@ crowdb-chunkdb-client = { path = "../../lib/crowdb-chunkdb-client" } crowdb-chunk-client = { path = "../../lib/crowdb-chunk-client" } flatbuffers = { workspace = true } clap = { version = "4", features = ["derive", "env"] } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "sync", "net"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "sync", "net", "process"] } serde = { version = "1", features = ["derive"] } serde_json = "1" hex = "0.4" @@ -84,3 +84,7 @@ path = "tests/cluster_lifecycle_test.rs" [[test]] name = "cluster_workloads_test" path = "tests/cluster_workloads_test.rs" + +[[test]] +name = "kv_bench_test" +path = "tests/kv_bench_test.rs" diff --git a/app/crowdb-cli/src/commands.rs b/app/crowdb-cli/src/commands.rs index 0dfcc48aa..15ed6088a 100644 --- a/app/crowdb-cli/src/commands.rs +++ b/app/crowdb-cli/src/commands.rs @@ -9,15 +9,15 @@ pub(crate) mod bench; pub(crate) mod cluster; -pub(crate) mod kv; +pub(crate) mod paxos_kv; pub(crate) mod port_alloc; pub(crate) mod s3; pub(crate) use bench::{run_bench_verb, BenchVerb}; pub(crate) use cluster::{run_cluster_verb, ClusterVerb}; -pub(crate) use kv::{ - run_group_verb, run_kv_data_verb, run_replica_verb, run_store_verb, GroupVerb, KvDataVerb, ReplicaVerb, - StoreVerb, +pub(crate) use paxos_kv::{ + run_group_verb, run_paxos_kv_data_verb, run_replica_verb, run_store_verb, GroupVerb, PaxosKvDataVerb, + ReplicaVerb, StoreVerb, }; pub(crate) use port_alloc::{run as run_port_alloc, PortAllocArgs}; pub(crate) use s3::{run_s3_verb, S3Verb}; @@ -47,7 +47,7 @@ pub(crate) async fn group0_connection(cli: &Cli) -> Result<(Vec<String>, String) eprintln!("error: cluster native transport: {error}"); ExitCode::from(2) })?; - let native = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let native = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( target.identity.group0_management_endpoints.clone(), )); native.seed_leader(0, 0, target.identity.group0_rpc_endpoints[0].clone()); diff --git a/app/crowdb-cli/src/commands/bench.rs b/app/crowdb-cli/src/commands/bench.rs index 2919f81da..6db01d143 100644 --- a/app/crowdb-cli/src/commands/bench.rs +++ b/app/crowdb-cli/src/commands/bench.rs @@ -9,13 +9,14 @@ //! under `bench/`. pub mod chunk; +mod client; pub mod disk; pub mod io; -pub mod kv; pub mod loader; pub mod metrics; -pub mod result; +pub mod paxos_kv; pub mod s3; pub mod verb; +pub mod workload; pub use verb::{run_bench_verb, BenchVerb}; diff --git a/app/crowdb-cli/src/commands/bench/chunk.rs b/app/crowdb-cli/src/commands/bench/chunk.rs index a562fcc31..a31fab99a 100644 --- a/app/crowdb-cli/src/commands/bench/chunk.rs +++ b/app/crowdb-cli/src/commands/bench/chunk.rs @@ -10,7 +10,7 @@ use std::time::{Duration, Instant}; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport, RangeBindingClient, RetryConfig}; use crowdb_diskdb_client::{DiskdbClient, DiskdbClientError, DiskdbRpcTransport}; -use crowdb_kv_client::{ReadEndpointPolicy, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ReadEndpointPolicy, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ AllocateChunkRequest, AppendChunkRequest, Chunk, ChunkState, ChunkType, DeleteChunkRequest, QueryChunkRequest, SealChunkRequest, Strip, StripType, @@ -21,7 +21,7 @@ use crowdb_protocol::generate_chunk_id; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; -use super::kv::{build_kv_client, KvClientTunables}; +use super::client::{build_kv_client, KvClientTunables}; use super::metrics::BenchMetrics; use super::verb::{ChunkdbArgs, ChunkdbBenchVerb, ChunkdbStripMode}; use crate::Cli; diff --git a/app/crowdb-cli/src/commands/bench/kv/client.rs b/app/crowdb-cli/src/commands/bench/client.rs similarity index 83% rename from app/crowdb-cli/src/commands/bench/kv/client.rs rename to app/crowdb-cli/src/commands/bench/client.rs index d8c5031e2..a6102c0dc 100644 --- a/app/crowdb-cli/src/commands/bench/kv/client.rs +++ b/app/crowdb-cli/src/commands/bench/client.rs @@ -1,14 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Shared helper for building a [`CrowdbKvClient`] from the CLI endpoint +//! Shared helper for building a [`PaxosKvClient`] from the CLI endpoint //! with a bench-specific [`ReadEndpointPolicy`]. The standard `op_context` //! helper always uses the default `Leader` policy; bench read/scan //! commands need `AnyReplica` for distributed `MinSlot` reads. use std::process::ExitCode; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ReadEndpointPolicy}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ReadEndpointPolicy}; use crate::Cli; @@ -38,7 +38,7 @@ impl Default for KvClientTunables { } } -/// Build a `CrowdbKvClient` from the resolved cluster target with the given +/// Build a `PaxosKvClient` from the resolved cluster target with the given /// `read_endpoint_policy`. /// /// # Errors @@ -47,7 +47,7 @@ pub(crate) async fn build_kv_client( cli: &Cli, read_endpoint_policy: ReadEndpointPolicy, tunables: &KvClientTunables, -) -> Result<CrowdbKvClient, ExitCode> { +) -> Result<PaxosKvClient, ExitCode> { let (seeds, effective_g0) = crate::commands::group0_connection(cli).await?; let mut client_config = ClientConfig::new(seeds); @@ -60,7 +60,7 @@ pub(crate) async fn build_kv_client( client_config.pool_size_per_endpoint = tunables.pool_size; eprintln!( - "bench kv client: read_endpoint_policy={:?} event_write={} rpc_workers={} nagle={} quickack={} send_queue={}", + "bench paxos-kv client: read_endpoint_policy={:?} event_write={} rpc_workers={} nagle={} quickack={} send_queue={}", client_config.read_endpoint_policy, client_config.event_write, client_config.rpc_workers, @@ -69,7 +69,7 @@ pub(crate) async fn build_kv_client( client_config.send_queue_capacity, ); - let client = CrowdbKvClient::new(client_config); + let client = PaxosKvClient::new(client_config); client.seed_leader(0, 0, effective_g0); Ok(client) } diff --git a/app/crowdb-cli/src/commands/bench/disk/db.rs b/app/crowdb-cli/src/commands/bench/disk/db.rs index da0ca92aa..b6e168496 100644 --- a/app/crowdb-cli/src/commands/bench/disk/db.rs +++ b/app/crowdb-cli/src/commands/bench/disk/db.rs @@ -10,14 +10,14 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use crowdb_diskdb_client::{DiskdbClient, DiskdbClientError, DiskdbRpcTransport}; -use crowdb_kv_client::{ReadEndpointPolicy, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ReadEndpointPolicy, ServiceRegistryClient}; use crowdb_protocol::common::ChunkId; use crowdb_protocol::diskdb::rpc::{AllocateBlocksRequest, CompactZoneRequest, FreeBlocksRequest, Segment}; use rand::rngs::SmallRng; use rand::SeedableRng; use super::workload::select_free; -use crate::commands::bench::kv::{build_kv_client, KvClientTunables}; +use crate::commands::bench::client::{build_kv_client, KvClientTunables}; use crate::commands::bench::loader::BenchRecorder; use crate::commands::bench::metrics::BenchMetrics; use crate::commands::bench::verb::{DiskdbArgs, DiskdbBenchVerb}; diff --git a/app/crowdb-cli/src/commands/bench/kv.rs b/app/crowdb-cli/src/commands/bench/kv.rs deleted file mode 100644 index 2f6fff9eb..000000000 --- a/app/crowdb-cli/src/commands/bench/kv.rs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. - -//! KV-layer bench workloads (prepare / read / write / scan) against a -//! crowdb-kv store. The shared client builder is re-exported for the -//! `disk` and `chunk` sub-modules, which also drive a KV client. - -pub mod client; -pub mod prepare; -pub mod read; -pub mod scan; -pub mod write; - -pub(crate) use client::{build_kv_client, KvClientTunables}; diff --git a/app/crowdb-cli/src/commands/bench/kv/prepare.rs b/app/crowdb-cli/src/commands/bench/kv/prepare.rs deleted file mode 100644 index b0ddecfed..000000000 --- a/app/crowdb-cli/src/commands/bench/kv/prepare.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. - -//! `bench kv prepare` writes only the requested dataset into the selected group. -//! -//! Concurrent bulk-put loop with `--concurrency` tasks. No latency -//! histogram — this is a setup step, not a measurement. Reports total -//! keys written, elapsed time, and errors to stderr. - -use std::process::ExitCode; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use super::client::{build_kv_client, KvClientTunables}; -use crate::commands::bench::verb::PrepareArgs; -use crate::Cli; - -pub async fn run(cli: &Cli, args: PrepareArgs) -> ExitCode { - let store_id = args.store; - let group_id = args.group; - let client = match build_kv_client( - cli, - crowdb_kv_client::ReadEndpointPolicy::Leader, - &KvClientTunables::default(), - ) - .await - { - Ok(c) => Arc::new(c), - Err(c) => return c, - }; - - let value_size = args.value_size.max(1); - - let next_id = Arc::new(AtomicU64::new(0)); - let ok_count = Arc::new(AtomicU64::new(0)); - let err_count = Arc::new(AtomicU64::new(0)); - let total = args.keys; - - let concurrency = args.concurrency.max(1); - let start = Instant::now(); - - let mut handles = Vec::with_capacity(concurrency); - for _ in 0..concurrency { - let client = Arc::clone(&client); - let next_id = Arc::clone(&next_id); - let ok_count = Arc::clone(&ok_count); - let err_count = Arc::clone(&err_count); - handles.push(tokio::spawn(async move { - loop { - let id = next_id.fetch_add(1, Ordering::Relaxed); - if id >= total { - break; - } - let key = format!("k{id:020}"); - let value = build_value(id, value_size); - // The client's put() already retries 4 times, but the - // leader may still be stabilizing. Add one extra retry - // with a short backoff for the first few keys. - let mut succeeded = false; - let mut last_err = None; - for attempt in 0..3u32 { - match client.put(store_id, group_id, key.as_bytes(), &value, None).await { - Ok(_) => { - ok_count.fetch_add(1, Ordering::Relaxed); - succeeded = true; - break; - } - Err(e) => { - last_err = Some(e); - if attempt < 2 { - tokio::time::sleep(Duration::from_millis(50 * u64::from(attempt + 1))).await; - } - } - } - } - if !succeeded { - err_count.fetch_add(1, Ordering::Relaxed); - if err_count.load(Ordering::Relaxed) <= 3 { - if let Some(e) = &last_err { - eprintln!("prepare: put key={key} failed: {e}"); - } - } - } - } - })); - } - - for h in handles { - let _ = h.await; - } - - let elapsed = start.elapsed(); - let ok = ok_count.load(Ordering::Relaxed); - let err = err_count.load(Ordering::Relaxed); - eprintln!( - "bench kv prepare: {ok} keys written, {err} errors, {:.1}s", - elapsed.as_secs_f64() - ); - - if err > 0 { - ExitCode::from(2) - } else { - ExitCode::SUCCESS - } -} - -/// Build a deterministic value for key `id`: byte `i` = `(id + i) % 256`. -/// This pattern is reused by `bench kv read --verify-bytes` for -/// correctness verification. -fn build_value(id: u64, size: usize) -> Vec<u8> { - (0..size) - .map(|i| u8::try_from((id + i as u64) % 256).unwrap_or(0)) - .collect() -} diff --git a/app/crowdb-cli/src/commands/bench/kv/read.rs b/app/crowdb-cli/src/commands/bench/kv/read.rs deleted file mode 100644 index 169987c36..000000000 --- a/app/crowdb-cli/src/commands/bench/kv/read.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. - -//! `bench kv read` — point-get workload against store 0 / group 0. -//! -//! Spawns `--loader-num` tokio tasks, each looping random `get` calls -//! for `--duration-secs`. Supports `--read-mode` (linearizable|minslot), -//! `--min-slot` (auto|zero), `--read-endpoint-policy` (leader|any-replica), -//! and `--verify-bytes` correctness checking. - -use std::process::ExitCode; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadEndpointPolicy, ReadMode}; -use rand::rngs::SmallRng; -use rand::{Rng, SeedableRng}; - -use super::client::{build_kv_client, KvClientTunables}; -use crate::commands::bench::loader::{run_workload, BenchRecorder}; -use crate::commands::bench::metrics::BenchMetrics; -use crate::commands::bench::result::{BenchOps, BenchResult}; -use crate::commands::bench::verb::{BenchMinSlot, BenchReadEndpoint, BenchReadMode, ReadArgs}; -use crate::Cli; - -pub async fn run(cli: &Cli, args: ReadArgs) -> ExitCode { - let store_id = args.store; - let group_id = args.group; - let read_mode = match args.read_mode { - BenchReadMode::Linearizable => ReadMode::Linearizable, - BenchReadMode::Minslot => ReadMode::MinSlot, - }; - let endpoint_policy = match args.read_endpoint_policy { - BenchReadEndpoint::Leader => ReadEndpointPolicy::Leader, - BenchReadEndpoint::AnyReplica => ReadEndpointPolicy::AnyReplica, - }; - - let client = match build_kv_client( - cli, - endpoint_policy, - &KvClientTunables { - pool_size: args.connections, - ..Default::default() - }, - ) - .await - { - Ok(c) => Arc::new(c), - Err(c) => return c, - }; - - let key_space = args.key_space.max(1); - let verify_bytes = args.verify_bytes; - let value_size = args.value_size; - let min_slot_arg = args.min_slot; - - let mut metrics = BenchMetrics::new(&cli.log_dir, args.metrics_interval); - metrics.start(); - - let duration = Duration::from_secs(args.duration_secs); - let start = Instant::now(); - let recorder = Arc::clone(&metrics.recorder); - let run = run_workload( - recorder, - args.loader_num, - duration, - make_read_workload( - client, - key_space, - verify_bytes, - value_size, - read_mode, - min_slot_arg, - store_id, - group_id, - ), - || {}, - ) - .await; - - let duration_ms: u64 = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX); - metrics.stop().await; - - let snap = run.recorder.hist_snapshot(); - eprintln!( - "bench kv read: {} ops, {} errors, {} corr errors, {}ms", - run.recorder.ops(), - run.recorder.errors(), - run.recorder.correctness_errors(), - duration_ms, - ); - - let result = BenchResult { - total_ops: run.recorder.ops(), - duration_ms, - total_errors: run.recorder.errors(), - correctness_errors: Some(run.recorder.correctness_errors()), - by_op: BenchOps::default().read(snap), - client_transport_stats: None, - server_metrics: None, - }; - let json = serde_json::to_value(&result).unwrap_or_default(); - tracing::info!(report = %json, "bench_report"); - result.finish(&cli.log_dir) -} - -#[allow(clippy::too_many_arguments)] -fn make_read_workload( - client: Arc<CrowdbKvClient>, - key_space: u64, - verify_bytes: usize, - value_size: usize, - read_mode: ReadMode, - min_slot_arg: BenchMinSlot, - store_id: u64, - group_id: u64, -) -> impl Fn(Arc<BenchRecorder>) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> { - move |rec: Arc<BenchRecorder>| { - let client = Arc::clone(&client); - Box::pin(async move { - let mut rng = SmallRng::from_entropy(); - let id = rng.gen_range(0..key_space); - let key = format!("k{id:020}"); - let min_slot = match min_slot_arg { - BenchMinSlot::Auto => None, - BenchMinSlot::Zero => Some(0), - }; - let t0 = Instant::now(); - match client - .get(store_id, group_id, key.as_bytes(), read_mode, min_slot) - .await - { - Ok(GetOutcome::Found { value, .. }) => { - rec.record_ok( - t0.elapsed().as_micros().try_into().unwrap_or(u64::MAX), - value.len() as u64, - ); - if verify_bytes > 0 { - let expected = expected_value(id, value_size, verify_bytes); - if value.len() < verify_bytes || value[..verify_bytes] != expected[..] { - rec.record_correctness_err(); - } - } - } - Ok(GetOutcome::NotFound) => { - rec.record_err(); - if verify_bytes > 0 { - rec.record_correctness_err(); - } - } - Err(_) => rec.record_err(), - } - }) - } -} - -/// Build the expected first `verify` bytes for key `id`: byte `i` = -/// `(id + i) % 256` (same pattern as `super::prepare::build_value`). -fn expected_value(id: u64, size: usize, verify: usize) -> Vec<u8> { - let len = verify.min(size); - (0..len) - .map(|i| u8::try_from((id + i as u64) % 256).unwrap_or(0)) - .collect() -} diff --git a/app/crowdb-cli/src/commands/bench/kv/scan.rs b/app/crowdb-cli/src/commands/bench/kv/scan.rs deleted file mode 100644 index 3c43826f9..000000000 --- a/app/crowdb-cli/src/commands/bench/kv/scan.rs +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. - -//! `bench kv scan` — list/range workload against store 0 / group 0. -//! -//! Spawns `--loader-num` tokio tasks, each looping scan calls for -//! `--duration-secs`. Supports `--scan-limit`, `--scan-prefix`, -//! `--scan-start-after`, `--read-mode`, `--min-slot`, -//! `--read-endpoint-policy`, and `--value-size-mix`. - -use std::process::ExitCode; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use crowdb_kv_client::{ReadEndpointPolicy, ReadMode}; - -use super::client::{build_kv_client, KvClientTunables}; -use crate::commands::bench::loader::{run_workload, BenchRecorder}; -use crate::commands::bench::metrics::BenchMetrics; -use crate::commands::bench::result::{BenchOps, BenchResult}; -use crate::commands::bench::verb::{ - BenchMinSlot, BenchReadEndpoint, BenchReadMode, BenchScanDirection, ScanArgs, -}; -use crate::Cli; - -#[allow(clippy::too_many_lines)] -pub async fn run(cli: &Cli, args: ScanArgs) -> ExitCode { - let store_id = args.store; - let group_id = args.group; - let read_mode = match args.read_mode { - BenchReadMode::Linearizable => ReadMode::Linearizable, - BenchReadMode::Minslot => ReadMode::MinSlot, - }; - let endpoint_policy = match args.read_endpoint_policy { - BenchReadEndpoint::Leader => ReadEndpointPolicy::Leader, - BenchReadEndpoint::AnyReplica => ReadEndpointPolicy::AnyReplica, - }; - - let client = match build_kv_client( - cli, - endpoint_policy, - &KvClientTunables { - pool_size: args.connections, - ..Default::default() - }, - ) - .await - { - Ok(c) => Arc::new(c), - Err(c) => return c, - }; - - let prefix = args.scan_prefix.as_bytes().to_vec(); - let start_after = args.scan_start_after.as_bytes().to_vec(); - let limit = args.scan_limit; - let min_slot_arg = args.min_slot; - let value_size = args.value_size; - let value_size_mix = parse_value_size_mix(args.value_size_mix.as_deref()); - let direction = args.scan_direction; - - let mut metrics = BenchMetrics::new(&cli.log_dir, args.metrics_interval); - metrics.start(); - - let duration = Duration::from_secs(args.duration_secs); - let start = Instant::now(); - let recorder = Arc::clone(&metrics.recorder); - let run = run_workload( - recorder, - args.loader_num, - duration, - move |rec: Arc<BenchRecorder>| { - let client = Arc::clone(&client); - let prefix = prefix.clone(); - let start_after = start_after.clone(); - let limit = limit; - let read_mode = read_mode; - let min_slot_arg = min_slot_arg; - let value_size = value_size; - let value_size_mix = value_size_mix.clone(); - let direction = direction; - async move { - let min_slot = match min_slot_arg { - BenchMinSlot::Auto => None, - BenchMinSlot::Zero => Some(0), - }; - let t0 = Instant::now(); - let scan = match direction { - BenchScanDirection::Forward => { - client - .scan( - store_id, - group_id, - &prefix, - &start_after, - &[], - limit, - read_mode, - min_slot, - false, - None, - ) - .await - } - BenchScanDirection::Reverse => { - client - .scan_reverse( - store_id, - group_id, - &prefix, - &start_after, - &[], - limit, - read_mode, - min_slot, - false, - None, - ) - .await - } - }; - match scan { - Ok(outcome) => { - // Touch value bytes to prevent lazy-load optimization - // skewing latency. Both --value-size and --value-size-mix - // paths touch the same bytes. - let total_bytes: u64 = outcome.items.iter().map(|(_, v)| v.len() as u64).sum(); - rec.record_ok( - t0.elapsed().as_micros().try_into().unwrap_or(u64::MAX), - total_bytes, - ); - let _ = outcome.items.len(); - if value_size > 0 || !value_size_mix.is_empty() { - for (_, v) in &outcome.items { - let _ = v.len(); - } - } - } - Err(_) => rec.record_err(), - } - } - }, - || {}, - ) - .await; - - let duration_ms: u64 = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX); - metrics.stop().await; - - let snap = run.recorder.hist_snapshot(); - eprintln!( - "bench kv scan: {} ops, {} errors, {}ms", - run.recorder.ops(), - run.recorder.errors(), - duration_ms, - ); - - let result = BenchResult { - total_ops: run.recorder.ops(), - duration_ms, - total_errors: run.recorder.errors(), - correctness_errors: Some(run.recorder.correctness_errors()), - by_op: BenchOps::default().list(snap), - client_transport_stats: None, - server_metrics: None, - }; - let json = serde_json::to_value(&result).unwrap_or_default(); - tracing::info!(report = %json, "bench_report"); - result.finish(&cli.log_dir) -} - -/// Parse a `--value-size-mix` string like `64:70,1024:20,16384:10` -/// into a list of `(size, percent)` pairs. Empty/None → empty vec. -fn parse_value_size_mix(s: Option<&str>) -> Vec<(usize, u8)> { - let Some(s) = s else { return Vec::new() }; - s.split(',') - .filter_map(|pair| { - let mut parts = pair.split(':'); - let size = parts.next()?.trim().parse::<usize>().ok()?; - let pct = parts.next()?.trim().parse::<u8>().ok()?; - Some((size, pct)) - }) - .collect() -} diff --git a/app/crowdb-cli/src/commands/bench/kv/write.rs b/app/crowdb-cli/src/commands/bench/kv/write.rs deleted file mode 100644 index 58ff06a39..000000000 --- a/app/crowdb-cli/src/commands/bench/kv/write.rs +++ /dev/null @@ -1,460 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. - -//! `bench kv write` — put workload against store 0 / group 0. -//! -//! Spawns `--loader-num` tokio tasks, each looping random `put` calls -//! for `--duration-secs`. After the workload, fetches server-side -//! metrics from every node's `/metrics` endpoint and aggregates them -//! into the `server_metrics` JSON section. - -use std::process::ExitCode; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use crowdb_console_shared::clients::http::ServerClient; -use crowdb_console_shared::snapshot::MetricFieldView; -use rand::rngs::SmallRng; -use rand::{Rng, SeedableRng}; - -use super::client::{build_kv_client, KvClientTunables}; -use crate::commands::authority_context; -use crate::commands::bench::loader::{run_workload, BenchRecorder}; -use crate::commands::bench::metrics::BenchMetrics; -use crate::commands::bench::result::{ - BenchOps, BenchResult, ReplicaStats, ServerMetrics, ServerRpcLatency, SnapshotStats, TransportStats, -}; -use crate::commands::bench::verb::WriteArgs; -use crate::Cli; - -#[allow(clippy::too_many_lines)] -pub async fn run(cli: &Cli, args: WriteArgs) -> ExitCode { - let store_id = args.store; - let group_id = args.group; - let client = match build_kv_client( - cli, - crowdb_kv_client::ReadEndpointPolicy::Leader, - &KvClientTunables { - event_write: args.event_write, - rpc_workers: args.rpc_workers, - send_queue_capacity: if args.send_queue_capacity > 0 { - args.send_queue_capacity - } else { - 4096 - }, - pool_size: args.connections, - ..Default::default() - }, - ) - .await - { - Ok(c) => Arc::new(c), - Err(c) => return c, - }; - - let key_space = args.key_space.max(1); - let value_size = args.value_size; - - let mut metrics = BenchMetrics::new(&cli.log_dir, args.metrics_interval); - metrics.start(); - - let server_metrics_baseline = fetch_server_metrics(cli, store_id, group_id).await; - - let duration = Duration::from_secs(args.duration_secs); - tracing::info!( - duration_secs = duration.as_secs(), - loaders = args.loader_num, - connections = args.connections, - key_space, - value_size, - event_write = args.event_write, - rpc_workers = args.rpc_workers, - send_queue_capacity = if args.send_queue_capacity > 0 { - args.send_queue_capacity - } else { - 4096 - }, - metrics_interval_secs = args.metrics_interval, - "bench kv write: starting" - ); - let start = Instant::now(); - let recorder = Arc::clone(&metrics.recorder); - let stats_client = Arc::clone(&client); - let dump_client = Arc::clone(&client); - let run = run_workload( - recorder, - args.loader_num, - duration, - move |rec: Arc<BenchRecorder>| { - let client = Arc::clone(&client); - let key_space = key_space; - let value_size = value_size; - async move { - let mut rng = SmallRng::from_entropy(); - let id = rng.gen_range(0..key_space); - let key = format!("k{id:020}"); - let value = build_value(id, value_size); - let t0 = Instant::now(); - match client.put(store_id, group_id, key.as_bytes(), &value, None).await { - Ok(_) => rec.record_ok( - t0.elapsed().as_micros().try_into().unwrap_or(u64::MAX), - value.len() as u64, - ), - Err(_) => rec.record_err(), - } - } - }, - move || { - let next_id = dump_client.next_req_id(); - tracing::info!( - next_req_id = next_id, - "bench: dumping pending RPC requests at stop time" - ); - dump_client.dump_pending_requests(); - }, - ) - .await; - - let duration_ms: u64 = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX); - tracing::info!( - duration_ms, - ops = run.recorder.ops(), - errors = run.recorder.errors(), - "bench kv write: workload complete" - ); - - metrics.stop().await; - - // Fetch server-side metrics from every node. - tracing::info!("bench kv write: fetching server metrics"); - let server_metrics = fetch_server_metrics(cli, store_id, group_id) - .await - .map(|current| match server_metrics_baseline.as_ref() { - Some(baseline) => current.delta_since(baseline), - None => current, - }); - tracing::info!( - elapsed_ms = start.elapsed().as_millis(), - "bench kv write: server metrics fetched" - ); - - let snap = run.recorder.hist_snapshot(); - eprintln!( - "bench kv write: {} ops, {} errors, {}ms", - run.recorder.ops(), - run.recorder.errors(), - duration_ms, - ); - - // Client-side crowdb-rpc transport stats (submit_to_writev latency). - let client_transport_stats = stats_client.transport_stats().map(|s| TransportStats { - submit_to_writev_avg_us: (s.submit_to_writev.sum_ns / 1000) - .checked_div(s.submit_to_writev.count) - .unwrap_or(0), - ..Default::default() - }); - - let result = BenchResult { - total_ops: run.recorder.ops(), - duration_ms, - total_errors: run.recorder.errors(), - correctness_errors: Some(run.recorder.correctness_errors()), - by_op: BenchOps::default().write(snap), - client_transport_stats, - server_metrics, - }; - metrics.write_report(&result); - result.finish(&cli.log_dir) -} - -/// Build a deterministic value for key `id`: byte `i` = `(id + i) % 256`. -fn build_value(id: u64, size: usize) -> Vec<u8> { - (0..size) - .map(|i| u8::try_from((id + i as u64) % 256).unwrap_or(0)) - .collect() -} - -/// Fetch `/metrics` from confirmed replica hosts and aggregate into -/// `ServerMetrics`. Metrics are summed across nodes except for averages -/// (which are averaged across nodes that report them). -#[allow(clippy::too_many_lines)] -async fn fetch_server_metrics(cli: &Cli, store_id: u64, group_id: u64) -> Option<ServerMetrics> { - let ctx = authority_context(cli).await.ok()?; - let replicas = ctx - .sysmd() - .list_replicas_in_group(store_id, group_id) - .await - .ok()?; - let group_prefix = format!("s.{store_id}.g.{group_id}."); - let store_prefix = format!("s.{store_id}.rpc."); - - let mut mgmt_urls = Vec::with_capacity(replicas.len()); - for replica in replicas { - mgmt_urls.push(ctx.live_node_mgmt_url(replica.node_id).await.ok()?); - } - mgmt_urls.sort(); - mgmt_urls.dedup(); - if mgmt_urls.is_empty() { - return None; - } - - let mut wal_append_count = 0u64; - let mut inflight_enqueued = 0u64; - let mut inflight_wait_sum = 0u64; - let mut inflight_wait_count = 0u64; - let mut election_count = 0u64; - let mut snapshot_completed = 0u64; - let mut snapshot_failed = 0u64; - let mut snapshot_max_latency_us = 0u64; - let mut rpc_s2w_sum = 0u64; - let mut rpc_s2w_count = 0u64; - // Server-side per-op RPC latency accumulators (averaged across nodes). - let mut rpc_lat = RpcLatencyAcc::default(); - // Replica RPC: collect all `rpc.l@<peer>` summaries across nodes. - // The leader sends accept rounds to peers; followers only send - // fetchgap/chosen_notice. We aggregate by peer id and report the - // two peers with the highest `total` (round-trips) as r2/r3. - let mut peer_totals: Vec<(u64, u64, u64)> = Vec::new(); // (peer_id, avg_us, total) - - for url in &mgmt_urls { - let Ok(sc) = ServerClient::new(url.clone()) else { - continue; - }; - // Group-level metrics: WAL, inflight, replica RPC. - if let Ok(resp) = sc.metrics(&group_prefix).await { - for point in &resp.metrics { - let fields = field_map(&point.fields); - match point.name.as_str() { - // WAL append summary: `total` = cumulative count. - n if n.ends_with(".wal.mem.append.l") - || n.ends_with(".wal.file.append.l") - || n.ends_with(".wal.uring.append.l") => - { - wal_append_count += get_u64(&fields, "total"); - } - n if n.ends_with(".paxos.elections.c") => { - election_count += get_u64(&fields, "total"); - } - n if n.ends_with(".maintenance.snapshot.success.c") => { - snapshot_completed += get_u64(&fields, "total"); - } - n if n.ends_with(".maintenance.snapshot.failure.c") => { - snapshot_failed += get_u64(&fields, "total"); - } - n if n.ends_with(".maintenance.snapshot.max_us.g") => { - snapshot_max_latency_us = snapshot_max_latency_us.max(get_u64(&fields, "value")); - } - n if n.ends_with(".write.inflight_enqueued.c") => { - inflight_enqueued += get_u64(&fields, "total"); - } - n if n.ends_with(".write.inflight_wait.l") => { - if get_u64(&fields, "count") > 0 { - inflight_wait_sum += get_u64(&fields, "avg_ns") / 1000; - inflight_wait_count += 1; - } - } - // Replica RPC latency: s.0.g.0.rpc.l@<peer_id> - n if n.contains(".rpc.l@") => { - let peer_id = n - .rsplit('@') - .next() - .and_then(|s| s.parse::<u64>().ok()) - .unwrap_or(0); - let avg_us = get_u64(&fields, "avg_ns") / 1000; - let total = get_u64(&fields, "total"); - peer_totals.push((peer_id, avg_us, total)); - } - _ => {} - } - } - } - // Store-level RPC transport stats. - if let Ok(resp) = sc.metrics(&store_prefix).await { - for point in &resp.metrics { - let fields = field_map(&point.fields); - match point.name.as_str() { - n if n.ends_with(".rpc.submit_to_writev.avg_us.g") => { - rpc_s2w_sum += get_u64(&fields, "value"); - rpc_s2w_count += 1; - } - n if n.ends_with(".rpc.put.lh") => { - rpc_lat.put.add(&fields); - } - n if n.ends_with(".rpc.get.lh") => { - rpc_lat.get.add(&fields); - } - n if n.ends_with(".rpc.scan.lh") => { - rpc_lat.scan.add(&fields); - } - n if n.ends_with(".rpc.delete.lh") => { - rpc_lat.delete.add(&fields); - } - _ => {} - } - } - } - } - - Some(build_server_metrics( - wal_append_count, - inflight_enqueued, - inflight_wait_sum, - inflight_wait_count, - rpc_s2w_sum, - rpc_s2w_count, - &rpc_lat, - peer_totals, - election_count, - SnapshotStats { - completed: snapshot_completed, - failed: snapshot_failed, - max_latency_us: snapshot_max_latency_us, - }, - )) -} - -#[allow(clippy::too_many_arguments)] -fn build_server_metrics( - wal_append_count: u64, - inflight_enqueued: u64, - inflight_wait_sum: u64, - inflight_wait_count: u64, - rpc_s2w_sum: u64, - rpc_s2w_count: u64, - rpc_lat: &RpcLatencyAcc, - mut peer_totals: Vec<(u64, u64, u64)>, - election_count: u64, - snapshot: SnapshotStats, -) -> ServerMetrics { - // Aggregate peer totals by peer_id (sum across nodes), then pick - // the top 2 by total round-trips as r2/r3. - peer_totals.sort_by_key(|&(_, _, total)| std::cmp::Reverse(total)); - let (r2_avg, r2_tps) = peer_totals - .first() - .map_or((0, 0), |(_, avg, total)| (*avg, *total)); - let (r3_avg, r3_tps) = peer_totals - .get(1) - .map_or((0, 0), |(_, avg, total)| (*avg, *total)); - - let inflight_wait_avg_us = inflight_wait_sum.checked_div(inflight_wait_count).unwrap_or(0); - let submit_to_writev_avg_us = rpc_s2w_sum.checked_div(rpc_s2w_count).unwrap_or(0); - - // RPC transport stats (writev_calls, frames_sent, etc.) are not - // exposed via the metrics registry — they live in the crowdb-rpc - // FFI transport and are not yet surfaced to the management API. - // Reported as 0; the regression script tolerates this via jq `// 0`. - ServerMetrics { - wal_append_count, - rpc: TransportStats { - writev_calls: 0, - frames_sent: 0, - read_calls: 0, - frames_parsed: 0, - submit_to_writev_avg_us, - }, - replica: ReplicaStats { - r2: r2_avg, - r2_tps, - r3: r3_avg, - r3_tps, - }, - inflight_enqueued, - inflight_wait_avg_us, - election_count, - snapshot, - rpc_latency: rpc_lat.finalize(), - } -} - -fn field_map(fields: &[MetricFieldView]) -> std::collections::HashMap<&str, f64> { - fields.iter().map(|f| (f.key.as_str(), f.value)).collect() -} - -fn get_u64(map: &std::collections::HashMap<&str, f64>, key: &str) -> u64 { - // Metric values are non-negative counts/latencies; the filter - // guards against NaN/negative before the truncating cast. - map.get(key) - .filter(|v| v.is_finite() && **v >= 0.0) - .map_or(0, |v| { - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - { - *v as u64 - } - }) -} - -fn get_f64(map: &std::collections::HashMap<&str, f64>, key: &str) -> f64 { - // avg_ns is now a true f64; preserve fractional precision. The - // filter guards against NaN/negative. - map.get(key) - .filter(|v| v.is_finite() && **v >= 0.0) - .map_or(0.0, |v| *v) -} - -/// Accumulator for one op-type's RPC latency across nodes. Histograms -/// report `avg_ns`/`p50_ns`/`p99_ns`; we sum and divide by node count. -#[derive(Default)] -struct RpcLatencyAccOp { - avg_ns: f64, - p50_ns: u64, - p99_ns: u64, - count: u64, -} - -impl RpcLatencyAccOp { - fn add(&mut self, fields: &std::collections::HashMap<&str, f64>) { - if get_u64(fields, "count") == 0 { - return; - } - self.avg_ns += get_f64(fields, "avg_ns"); - self.p50_ns += get_u64(fields, "p50_ns"); - self.p99_ns += get_u64(fields, "p99_ns"); - self.count += 1; - } - fn finalize_us(&self) -> (f64, u64, u64) { - let n = self.count.max(1); - ( - #[allow(clippy::cast_precision_loss)] - { - self.avg_ns / n as f64 / 1000.0 - }, - self.p50_ns / n / 1000, - self.p99_ns / n / 1000, - ) - } -} - -/// Accumulator for all op-type RPC latencies across nodes. -#[derive(Default)] -struct RpcLatencyAcc { - put: RpcLatencyAccOp, - get: RpcLatencyAccOp, - scan: RpcLatencyAccOp, - delete: RpcLatencyAccOp, -} - -impl RpcLatencyAcc { - fn finalize(&self) -> Option<ServerRpcLatency> { - if self.put.count == 0 && self.get.count == 0 && self.scan.count == 0 && self.delete.count == 0 { - return None; - } - let (put_avg, put_p50, put_p99) = self.put.finalize_us(); - let (get_avg, get_p50, get_p99) = self.get.finalize_us(); - let (scan_avg, scan_p50, scan_p99) = self.scan.finalize_us(); - let (delete_avg, delete_p50, delete_p99) = self.delete.finalize_us(); - Some(ServerRpcLatency { - put_avg_us: put_avg, - put_p50_us: put_p50, - put_p99_us: put_p99, - get_avg_us: get_avg, - get_p50_us: get_p50, - get_p99_us: get_p99, - scan_avg_us: scan_avg, - scan_p50_us: scan_p50, - scan_p99_us: scan_p99, - delete_avg_us: delete_avg, - delete_p50_us: delete_p50, - delete_p99_us: delete_p99, - }) - } -} diff --git a/app/crowdb-cli/src/commands/bench/loader.rs b/app/crowdb-cli/src/commands/bench/loader.rs index 1ad30f7db..442d7d878 100644 --- a/app/crowdb-cli/src/commands/bench/loader.rs +++ b/app/crowdb-cli/src/commands/bench/loader.rs @@ -1,30 +1,11 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Shared loader-loop helper for tokio-based bench workloads. -//! -//! `run_workload` spawns `num_loaders` async tasks, each looping the -//! supplied `op` closure until a deadline, plus one timer task that -//! flips a shared `running` flag. The closure receives the shared -//! [`BenchRecorder`] and records each op's latency / errors itself. -//! The coroutine RPC path does not use this (the C++ `co_spawn` manages -//! its own loop); it records into a `BenchRecorder` directly from the -//! `on_response` callback. -//! -//! `BenchRecorder` wraps the project's `Counter` + `LatencyHistogram` + -//! `Bandwidth` handles from `crowdb_common::metrics`, all registered on -//! the `MetricsRunner`'s registry. The `LatencyHistogram` tracks both -//! window state (flushed/reset each interval for the periodic metrics -//! log) and cumulative state (never reset, for the final JSON report) -//! in a single `observe()` call — no double recording. Its `count` is -//! the success-ops count, so a separate success counter is not needed. +//! Metrics recorder retained by the chunk and disk workload adapters. -use std::future::Future; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use std::time::{Duration, Instant}; -use crowdb_common::metrics::{Bandwidth, Counter, HistogramSnapshot, LatencyHistogram, MetricsRegistry}; +use crowdb_common::metrics::{Bandwidth, Counter, LatencyHistogram, MetricsRegistry}; /// Shared error counters + latency histogram + bandwidth + running flag /// for one bench run. All fields are lock-free atomics — safe to update @@ -38,7 +19,6 @@ pub struct BenchRecorder { latency: Arc<LatencyHistogram>, /// Payload bandwidth (bytes/s, registered → flushed each interval). bytes: Arc<Bandwidth>, - running: AtomicBool, } impl BenchRecorder { @@ -51,7 +31,6 @@ impl BenchRecorder { correctness_errors: reg.register_counter("bench.ops.correctness_error.c"), latency: reg.register_histogram("bench.latency.lh"), bytes: reg.register_bandwidth("bench.bytes.bw"), - running: AtomicBool::new(true), } } @@ -76,119 +55,4 @@ impl BenchRecorder { pub fn record_correctness_err(&self) { self.correctness_errors.inc(); } - - /// Whether loader tasks should keep running. - pub fn is_running(&self) -> bool { - self.running.load(Ordering::Relaxed) - } - - /// Stop all loader tasks (set by the timer task at the deadline). - pub fn stop(&self) { - self.running.store(false, Ordering::Relaxed); - } - - /// Total successful ops (cumulative count from the latency histogram). - #[must_use] - pub fn ops(&self) -> u64 { - self.latency.snapshot_total().total_count - } - #[must_use] - pub fn errors(&self) -> u64 { - self.errors.snapshot().total - } - #[must_use] - #[allow(dead_code)] - pub fn correctness_errors(&self) -> u64 { - self.correctness_errors.snapshot().total - } - /// Cumulative latency snapshot (avg/p50/p99/max in ns) across all - /// observations — for the final JSON report. Window state may have - /// been reset by periodic flushes; cumulative state is preserved. - #[must_use] - pub fn hist_snapshot(&self) -> HistogramSnapshot { - self.latency.snapshot_total() - } -} - -/// The recorder + measured wall-clock duration of a workload run. -pub struct WorkloadRun { - pub recorder: Arc<BenchRecorder>, - #[allow(dead_code)] - pub duration_ms: u64, -} - -/// Spawn `num_loaders` tasks each looping `op` until `duration` elapses. -/// `op` performs exactly one operation and records its outcome into the -/// shared recorder. Returns the recorder + the measured run duration. -/// -/// `on_stop` is called once, immediately after the timer fires and -/// `running` is set to false, but before the loader tasks have exited — -/// so any in-flight requests are still pending. Use it to dump diagnostics. -/// -/// `num_loaders` is clamped to >= 1. -pub async fn run_workload<F, Fut, S>( - recorder: Arc<BenchRecorder>, - num_loaders: usize, - duration: Duration, - op: F, - on_stop: S, -) -> WorkloadRun -where - F: Fn(Arc<BenchRecorder>) -> Fut + Send + Sync + 'static, - Fut: Future<Output = ()> + Send + 'static, - S: FnOnce() + Send + 'static, -{ - let loaders = num_loaders.max(1); - let start = Instant::now(); - let op = Arc::new(op); - - let mut tasks = Vec::with_capacity(loaders); - for _ in 0..loaders { - let rec = Arc::clone(&recorder); - let op = Arc::clone(&op); - tasks.push(tokio::spawn(async move { - while rec.is_running() { - (*op)(Arc::clone(&rec)).await; - } - })); - } - - // Timer task: flip running=false at the deadline, then call on_stop - // while in-flight requests are still pending (before loaders exit). - let timer_rec = Arc::clone(&recorder); - let timer = tokio::spawn(async move { - tokio::time::sleep(duration).await; - let ops_at_deadline = timer_rec.ops(); - timer_rec.stop(); - tracing::info!( - deadline_secs = duration.as_secs(), - ops_at_deadline, - "bench: stop signaled — loaders draining in-flight ops" - ); - on_stop(); - }); - - for (i, t) in tasks.into_iter().enumerate() { - let _ = t.await; - if i == 0 { - tracing::info!( - task = i, - elapsed_ms = start.elapsed().as_millis(), - "bench: first loader exited" - ); - } - } - let elapsed_ms = start.elapsed().as_millis(); - tracing::info!( - elapsed_ms, - ops = recorder.ops(), - errors = recorder.errors(), - "bench: all loaders joined" - ); - let _ = timer.await; - - WorkloadRun { - recorder, - duration_ms: elapsed_ms.try_into().unwrap_or(u64::MAX), - } } diff --git a/app/crowdb-cli/src/commands/bench/metrics.rs b/app/crowdb-cli/src/commands/bench/metrics.rs index 03793ec3e..1fb1b3d2d 100644 --- a/app/crowdb-cli/src/commands/bench/metrics.rs +++ b/app/crowdb-cli/src/commands/bench/metrics.rs @@ -10,7 +10,7 @@ //! counters (e.g. `rpc.client.*`) are flushed into the same file via a //! `set_cpp_flush` callback that calls //! `crowdb_rpc_ffi::flush_cpp_global_metrics` — the same pattern -//! `crowdb-kv-server` uses. After the workload, the final JSON report +//! `crowdb-paxos-kv-server` uses. After the workload, the final JSON report //! is appended to the metrics log via `write_raw`. use std::path::Path; @@ -78,7 +78,7 @@ impl BenchMetrics { let mut runner = MetricsRunner::new(file, metrics_interval); // Merge C++ crowdb-rpc process-level metrics (rpc.client.*) into - // the same log file. Mirrors crowdb-kv-server's engine_collector. + // the same log file. Mirrors crowdb-paxos-kv-server's engine_collector. runner.set_cpp_flush(|writer, window_secs, timestamp, rust_width, count_w, tps_w| { let cpp_max = crowdb_rpc_ffi::cpp_global_metrics_max_name_len(); let shared_width = rust_width.max(cpp_max); @@ -119,15 +119,4 @@ impl BenchMetrics { r.stop().await; } } - - /// Write the final report to the metrics log file as readable text. - /// Falls back to `eprintln!` if the runner is disabled (no metrics log). - pub fn write_report(&self, result: &super::result::BenchResult) { - let text = result.to_string(); - if let Some(ref r) = self.runner { - r.write_raw(&text); - } else { - eprint!("{text}"); - } - } } diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv.rs b/app/crowdb-cli/src/commands/bench/paxos_kv.rs new file mode 100644 index 000000000..a3442a64f --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv.rs @@ -0,0 +1,13 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Read-only deployment discovery followed by host-generated KV load. +pub mod args; +mod client; +mod collect; +mod counters; +mod dataset; +mod deployment; +mod run; +mod verification; +pub use run::run; diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/args.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/args.rs new file mode 100644 index 000000000..c6b8cdbbd --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/args.rs @@ -0,0 +1,127 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use clap::{Args, Subcommand, ValueEnum}; +use serde::Serialize; + +use super::super::workload::LoadOptions; + +#[derive(Args, Debug, Serialize)] +#[allow(clippy::struct_excessive_bools)] +pub struct PaxosKvArgs { + #[arg(long, default_value = "kv", global = true)] + pub label: String, + /// Management origins of all three running nodes; repeat three times. + #[arg(long = "endpoint", global = true, value_parser = endpoint)] + pub endpoints: Vec<String>, + /// Local running containers in the same order as endpoints (host network). + #[arg(long = "container", global = true, conflicts_with = "server_pids")] + pub containers: Vec<String>, + /// Native diagnostic: three existing server PIDs in separate cgroups. + #[arg(long = "server-pid", global = true, conflicts_with = "containers")] + pub server_pids: Vec<u32>, + /// Snapshot success > 0, failure = 0, elections = 0 over the complete load. + #[arg(long, global = true)] + pub require_snapshot: bool, + #[arg(long, global = true)] + pub expect_window: Option<u64>, + #[arg(long, global = true)] + pub expect_coalesce: Option<u64>, + /// 70% 64B, 20% 1KiB, 10% 16KiB, assigned deterministically by key ID. + #[arg(long, global = true)] + pub mixed_values: bool, + #[arg(long, default_value_t = 0, global = true)] + pub store: u64, + /// Existing workload group. Group 0 is reserved for control-plane data. + #[arg(long, default_value_t = 1, global = true)] + pub group: u64, + #[arg(long, default_value_t = 1_000_000, global = true)] + pub key_space: u64, + #[arg(long, default_value_t = 64, global = true)] + pub value_size: usize, + #[arg(long, default_value_t = 2 * 1024 * 1024 * 1024, global = true)] + pub max_dataset_bytes: u64, + /// Stop injecting requests at this percentage of any node's memory limit. + #[arg(long, default_value_t = 90, global = true, value_parser = clap::value_parser!(u8).range(1..=99))] + pub memory_guard_percent: u8, + /// Upper bound on concurrent materialized scan responses in the host CLI. + #[arg(long, default_value_t = 512 * 1024 * 1024, global = true)] + pub max_inflight_bytes: u64, + #[arg(long, default_value_t = 4, global = true)] + pub connections: usize, + #[arg(long, default_value_t = 2, global = true)] + pub rpc_workers: u32, + #[arg(long, default_value_t = false, global = true)] + pub event_write: bool, + #[arg(long, default_value_t = 4096, global = true)] + pub send_queue_capacity: u32, + #[arg(long, default_value_t = 1, global = true)] + pub seed: u64, + #[arg(long, value_enum, default_value_t = ReadMode::Linearizable, global = true)] + pub read_mode: ReadMode, + #[arg(long, value_enum, default_value_t = ReadEndpoint::Leader, global = true)] + pub read_endpoint_policy: ReadEndpoint, + #[arg(long, default_value_t = 100, global = true)] + pub scan_limit: u32, + #[arg(long, default_value = "", global = true)] + pub scan_start_after: String, + #[arg(long, default_value = "", global = true)] + pub scan_prefix: String, + #[arg(long, default_value_t = false, global = true)] + pub reverse: bool, + #[command(flatten)] + pub load: LoadOptions, + #[command(subcommand)] + pub operation: Operation, +} + +#[derive(Subcommand, Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Operation { + /// Populate and verify the bounded dataset; no performance verdict. + Prepare, + Read, + Write, + Scan, +} + +#[derive(ValueEnum, Debug, Clone, Copy, Serialize)] +pub enum ReadMode { + Linearizable, + Minslot, +} + +#[derive(ValueEnum, Debug, Clone, Copy, Serialize)] +pub enum ReadEndpoint { + Leader, + AnyReplica, +} + +fn endpoint(value: &str) -> Result<String, String> { + let url = reqwest::Url::parse(value).map_err(|error| error.to_string())?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || url.port() == Some(0) + || url.path() != "/" + || url.query().is_some() + || url.fragment().is_some() + || !url.username().is_empty() + || url.password().is_some() + { + return Err("endpoint must be an HTTP(S) management origin without credentials or path".into()); + } + Ok(value.trim_end_matches('/').to_owned()) +} + +impl PaxosKvArgs { + pub(super) fn dataset_bytes(&self) -> Option<u64> { + let values = if self.mixed_values { + let complete = (self.key_space / 10).checked_mul(7 * 64 + 2 * 1024 + 16384)?; + let rest = self.key_space % 10; + complete.checked_add(rest.min(7) * 64 + rest.saturating_sub(7) * 1024)? + } else { + self.key_space.checked_mul(self.value_size as u64)? + }; + values.checked_add(self.key_space.checked_mul(21)?) + } +} diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/client.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/client.rs new file mode 100644 index 000000000..fbd580215 --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/client.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::args::{PaxosKvArgs, ReadEndpoint}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ReadEndpointPolicy}; +use crowdb_protocol::mgmt::TopologyResponse; +use serde_json::Value; +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, +}; + +pub(super) async fn fetch(endpoint: &str, path: &str) -> Result<Value, String> { + let response = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build() + .map_err(|e| e.to_string())? + .get(format!("{endpoint}{path}")) + .send() + .await + .map_err(|e| format!("{endpoint}{path}: {e}"))? + .error_for_status() + .map_err(|e| e.to_string())?; + serde_json::from_slice(&response.bytes().await.map_err(|e| e.to_string())?).map_err(|e| e.to_string()) +} + +pub(super) async fn connect(args: &PaxosKvArgs) -> Result<(PaxosKvClient, Vec<Value>), String> { + if args.endpoints.len() != 3 || args.endpoints.iter().collect::<BTreeSet<_>>().len() != 3 { + return Err("provide exactly three distinct --endpoint management origins".into()); + } + let mut topologies = Vec::new(); + let mut replicas = BTreeMap::new(); + let mut memberships = Vec::new(); + let mut leaders = BTreeSet::new(); + for endpoint in &args.endpoints { + let raw = fetch(endpoint, "/topology").await?; + let topology: TopologyResponse = serde_json::from_value(raw.clone()).map_err(|e| e.to_string())?; + let store = topology + .stores + .iter() + .find(|s| s.store_id == args.store) + .ok_or("store absent on a node")?; + let group = store + .groups + .iter() + .find(|g| g.group_id == args.group) + .ok_or("group absent on a node")?; + let address = store.listen_addr.clone().ok_or("RPC listener absent")?; + let mut socket: std::net::SocketAddr = + address.parse().map_err(|e| format!("invalid RPC address: {e}"))?; + if socket.ip().is_unspecified() { + let origin = reqwest::Url::parse(endpoint).map_err(|e| e.to_string())?; + let ip = origin + .host_str() + .ok_or("missing host")? + .parse() + .map_err(|_| "use a numeric endpoint IP")?; + socket.set_ip(ip); + } + let address = socket.to_string(); + tokio::time::timeout(Duration::from_secs(3), tokio::net::TcpStream::connect(socket)) + .await + .map_err(|_| format!("RPC endpoint unreachable: {socket}"))? + .map_err(|e| e.to_string())?; + if replicas.insert(group.local_replica_id, address.clone()).is_some() { + return Err("duplicate replica identity".into()); + } + let mut membership = BTreeMap::from([(group.local_replica_id, address)]); + for remote in &group.remotes { + if !remote.voting || membership.insert(remote.id, remote.endpoint.clone()).is_some() { + return Err("expected three distinct voting replicas".into()); + } + } + memberships.push(membership); + leaders.insert(group.leader_id); + topologies.push(raw); + } + if memberships.iter().any(|members| members != &replicas) || leaders.len() != 1 { + return Err("nodes disagree on replica membership or leader; wait for deployment readiness".into()); + } + let leader = leaders + .first() + .and_then(|id| replicas.get(id)) + .ok_or("no elected leader")?; + let mut config = ClientConfig::new(args.endpoints.clone()); + config.pool_size_per_endpoint = args.connections; + config.rpc_workers = args.rpc_workers; + config.event_write = args.event_write; + config.send_queue_capacity = args.send_queue_capacity; + config.read_endpoint_policy = match args.read_endpoint_policy { + ReadEndpoint::Leader => ReadEndpointPolicy::Leader, + ReadEndpoint::AnyReplica => ReadEndpointPolicy::AnyReplica, + }; + let client = PaxosKvClient::new(config); + client.refresh_topology().await.map_err(|e| e.to_string())?; + client.seed_leader(args.store, args.group, leader.clone()); + Ok((client, topologies)) +} diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/collect.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/collect.rs new file mode 100644 index 000000000..6e009a74a --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/collect.rs @@ -0,0 +1,17 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::{args::PaxosKvArgs, client::fetch, deployment::ServerProcess}; +use serde_json::{json, Value}; + +pub(super) async fn sample(args: &PaxosKvArgs, nodes: &[ServerProcess]) -> Result<Value, String> { + let start = chrono::Utc::now(); + let metrics = futures::future::try_join_all(args.endpoints.iter().map(|e| fetch(e, "/metrics"))).await?; + let resources: Vec<_> = nodes + .iter() + .map(ServerProcess::sample) + .collect::<Result<_, _>>()?; + Ok(json!({"started_at":start,"finished_at":chrono::Utc::now(),"metrics":metrics,"resources":resources})) +} + +pub(super) use super::counters::{deltas, sentinel}; diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/counters.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/counters.rs new file mode 100644 index 000000000..b122030c8 --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/counters.rs @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use serde_json::{json, Value}; + +pub fn counter(node: &Value, name: &str) -> Result<f64, String> { + node["metrics"] + .as_array() + .ok_or("metrics response has no points")? + .iter() + .find(|v| v["name"] == name) + .and_then(|v| v["fields"].as_array()) + .and_then(|f| f.iter().find(|v| v["key"] == "total")) + .and_then(|v| v["value"].as_f64()) + .ok_or_else(|| format!("missing required cumulative counter {name}")) +} + +pub fn sentinel(store: u64, group: u64, before: &Value, after: &Value) -> Result<Value, String> { + let mut sums = [0.0; 3]; + for (i, suffix) in [ + "paxos.elections.c", + "maintenance.snapshot.success.c", + "maintenance.snapshot.failure.c", + ] + .iter() + .enumerate() + { + let name = format!("s.{store}.g.{group}.{suffix}"); + for node in 0..3 { + let delta = counter(&after["metrics"][node], &name)? - counter(&before["metrics"][node], &name)?; + if delta < 0.0 { + return Err(format!("counter reset: {name}")); + } + sums[i] += delta; + } + } + Ok( + json!({"elections":sums[0],"snapshot_completed":sums[1],"snapshot_failed":sums[2], + "passed":sums[0]==0.0 && sums[1]>0.0 && sums[2]==0.0, + "window":"entire load including warmup, tail and drain; preparation excluded"}), + ) +} + +fn stat(sample: &Value, file: &str, key: &str) -> Result<u64, String> { + sample[file] + .as_str() + .and_then(|s| { + s.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + (fields.next() == Some(key)) + .then(|| fields.next()?.parse().ok()) + .flatten() + }) + }) + .ok_or_else(|| format!("missing cgroup {file}/{key}")) +} + +#[allow(clippy::cast_precision_loss)] +pub fn deltas(before: &Value, after: &Value) -> Result<Value, String> { + let mut nodes = Vec::new(); + for index in 0..3 { + let mut counters = serde_json::Map::new(); + let points = before["metrics"][index]["metrics"] + .as_array() + .ok_or("metrics missing")?; + for point in points { + let Some(name) = point["name"] + .as_str() + .filter(|s| s.rsplit('.').next() == Some("c")) + else { + continue; + }; + let delta = counter(&after["metrics"][index], name)? - counter(&before["metrics"][index], name)?; + if delta < 0.0 { + return Err(format!("counter reset: {name}")); + } + counters.insert(name.into(), json!(delta)); + } + let b = &before["resources"][index]; + let a = &after["resources"][index]; + let elapsed = (chrono::DateTime::parse_from_rfc3339(a["at"].as_str().ok_or("sample timestamp")?) + .map_err(|e| e.to_string())? + - chrono::DateTime::parse_from_rfc3339(b["at"].as_str().ok_or("sample timestamp")?) + .map_err(|e| e.to_string())?) + .num_microseconds() + .ok_or("invalid sample interval")?; + if elapsed <= 0 { + return Err("nonpositive resource interval".into()); + } + let delta = |file, key| { + stat(a, file, key)? + .checked_sub(stat(b, file, key)?) + .ok_or_else(|| format!("cgroup counter reset {file}/{key}")) + }; + let usage = delta("cpu_stat", "usage_usec")?; + let oom = delta("memory_events", "oom")?; + let oom_kill = delta("memory_events", "oom_kill")?; + if oom > 0 || oom_kill > 0 { + return Err("container memory exhaustion during load".into()); + } + nodes.push(json!({"counters":counters,"cpu_usage_usec":usage,"cpu_percent":usage as f64/elapsed as f64*100.0, + "cpu_throttled_usec":delta("cpu_stat","throttled_usec")?,"memory_current":a["memory_current"], + "memory_peak":a["memory_peak"],"memory_peak_scope":"container lifetime", "sample_elapsed_us":elapsed})); + } + Ok(json!(nodes)) +} diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/dataset.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/dataset.rs new file mode 100644 index 000000000..cb67fc40c --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/dataset.rs @@ -0,0 +1,204 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::args::{Operation, PaxosKvArgs, ReadMode as Mode}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode}; +use futures::{stream, StreamExt, TryStreamExt}; +use std::sync::Arc; + +pub(super) fn key(id: u64) -> String { + format!("k{id:020}") +} +pub(super) fn size(args: &PaxosKvArgs, id: u64) -> usize { + if args.mixed_values { + match id % 10 { + 0..=6 => 64, + 7..=8 => 1024, + _ => 16384, + } + } else { + args.value_size + } +} +fn value(args: &PaxosKvArgs, id: u64) -> Vec<u8> { + (0..size(args, id)) + .map(|offset| id.wrapping_add(offset as u64).to_le_bytes()[0]) + .collect() +} +fn verify_value(args: &PaxosKvArgs, id: u64, actual: &[u8]) -> Result<(), String> { + if actual.len() != size(args, id) + || actual + .iter() + .enumerate() + .any(|(offset, b)| *b != id.wrapping_add(offset as u64).to_le_bytes()[0]) + { + return Err(format!("incorrect bytes for key {}", key(id))); + } + Ok(()) +} + +pub(super) async fn prepare(client: Arc<PaxosKvClient>, args: Arc<PaxosKvArgs>) -> Result<(), String> { + stream::iter(0..args.key_space) + .map(|id| { + let client = client.clone(); + let args = args.clone(); + async move { + client + .put( + args.store, + args.group, + key(id).as_bytes(), + &value(&args, id), + None, + ) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) + } + }) + .buffer_unordered(args.load.loader_num) + .try_collect::<Vec<_>>() + .await?; + verify(client, args).await +} +pub(super) async fn verify(client: Arc<PaxosKvClient>, args: Arc<PaxosKvArgs>) -> Result<(), String> { + stream::iter(0..args.key_space) + .map(|id| { + let client = client.clone(); + let args = args.clone(); + async move { + match client + .get( + args.store, + args.group, + key(id).as_bytes(), + ReadMode::Linearizable, + None, + ) + .await + .map_err(|e| e.to_string())? + { + GetOutcome::Found { value, .. } => verify_value(&args, id, &value), + GetOutcome::NotFound => Err(format!("missing key {}", key(id))), + } + } + }) + .buffer_unordered(args.load.loader_num.max(16)) + .try_collect::<Vec<_>>() + .await?; + Ok(()) +} + +pub(super) async fn operation(client: &PaxosKvClient, args: &PaxosKvArgs, id: u64) -> Result<u64, String> { + let mode = match args.read_mode { + Mode::Linearizable => ReadMode::Linearizable, + Mode::Minslot => ReadMode::MinSlot, + }; + match args.operation { + Operation::Write => { + client + .put(args.store, args.group, key(id).as_bytes(), &value(args, id), None) + .await + .map_err(|e| e.to_string())?; + Ok(size(args, id) as u64) + } + Operation::Read => match client + .get(args.store, args.group, key(id).as_bytes(), mode, Some(0)) + .await + .map_err(|e| e.to_string())? + { + GetOutcome::Found { value, .. } if value.len() == size(args, id) => Ok(value.len() as u64), + GetOutcome::Found { .. } => Err("incorrect value length".into()), + GetOutcome::NotFound => Err(format!("missing key {}", key(id))), + }, + Operation::Scan => scan(client, args, mode, false).await, + Operation::Prepare => Err("prepare cannot be timed".into()), + } +} + +pub(super) async fn scan( + client: &PaxosKvClient, + args: &PaxosKvArgs, + mode: ReadMode, + verify: bool, +) -> Result<u64, String> { + // Keep scans inside this bounded dataset even after a larger earlier run. + let upper = key(args.key_space); + let prefix = if args.scan_prefix.is_empty() { + "k" + } else { + &args.scan_prefix + }; + let start = if args.reverse && (args.scan_start_after.is_empty() || args.scan_start_after > upper) { + &upper + } else { + &args.scan_start_after + }; + let end = if args.reverse { "" } else { &upper }; + let result = if args.reverse { + client + .scan_reverse( + args.store, + args.group, + prefix.as_bytes(), + start.as_bytes(), + end.as_bytes(), + args.scan_limit, + mode, + Some(0), + false, + None, + ) + .await + } else { + client + .scan( + args.store, + args.group, + prefix.as_bytes(), + start.as_bytes(), + end.as_bytes(), + args.scan_limit, + mode, + Some(0), + false, + None, + ) + .await + } + .map_err(|e| e.to_string())?; + if result.items.is_empty() { + return Err("scan returned no items; choose a nonempty dataset range".into()); + } + if result.timed_out { + return Err("partial timed-out scan".into()); + } + if verify { + let mut ids: Vec<_> = (0..args.key_space) + .filter(|id| { + let k = key(*id); + k.starts_with(&args.scan_prefix) + && (args.scan_start_after.is_empty() + || if args.reverse { + k < args.scan_start_after + } else { + k > args.scan_start_after + }) + }) + .collect(); + if args.reverse { + ids.reverse(); + } + let expected_len = ids.len().min(args.scan_limit as usize); + if result.items.len() != expected_len || result.truncated != (ids.len() > expected_len) { + return Err("scan count/truncated mismatch".into()); + } + for ((k, v), id) in result.items.iter().zip(ids) { + if k.as_ref() != key(id).as_bytes() { + return Err("scan key ordering/bound mismatch".into()); + } + verify_value(args, id, v)?; + } + } + Ok(result.items.iter().map(|(k, v)| (k.len() + v.len()) as u64).sum()) +} diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/deployment.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/deployment.rs new file mode 100644 index 000000000..eb202c5fd --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/deployment.rs @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Read-only deployment evidence and cgroup v2 resource accounting. +mod memory; +mod native; +use super::args::PaxosKvArgs; +use serde_json::{json, Value}; +use std::{collections::BTreeSet, path::PathBuf, time::Duration}; + +pub(super) struct ServerProcess { + pub evidence: Value, + cgroup: PathBuf, +} +impl ServerProcess { + pub fn sample(&self) -> Result<Value, String> { + let read = + |name| std::fs::read_to_string(self.cgroup.join(name)).map_err(|e| format!("cgroup {name}: {e}")); + Ok(json!({"at":chrono::Utc::now(),"cpu_stat":read("cpu.stat")?, + "memory_current":read("memory.current")?,"memory_peak":read("memory.peak").ok(), + "memory_events":read("memory.events")?,"cpu_pressure":read("cpu.pressure").ok(), + "memory_pressure":read("memory.pressure").ok()})) + } +} + +fn memory(node: &ServerProcess) -> Result<(Option<u64>, u64), String> { + let read = |file| -> Result<String, String> { + Ok(std::fs::read_to_string(node.cgroup.join(file)) + .map_err(|e| format!("cgroup {file}: {e}"))? + .trim() + .to_owned()) + }; + Ok(( + memory::parse_limit(&read("memory.max")?)?, + read("memory.current")? + .parse() + .map_err(|_| "invalid cgroup memory.current")?, + )) +} + +pub(super) fn validate_budget(args: &PaxosKvArgs, nodes: &[ServerProcess]) -> Result<(), String> { + let payload = args.dataset_bytes().ok_or("dataset size overflow")?; + for node in nodes { + let (limit, current) = memory(node)?; + let Some(limit) = limit else { continue }; + // Startup estimate for replicas' WAL, Tree and working buffers; + // the runtime guard remains necessary because write amplification is variable. + if (u128::from(current) + u128::from(payload) * 2) * 100 + > u128::from(limit) * u128::from(args.memory_guard_percent) + { + return Err(format!("{}: dataset {payload} bytes exceeds memory budget: current={current}, limit={limit}; reduce --key-space or --value-size", node.evidence["name"])); + } + } + Ok(()) +} + +pub(super) async fn memory_guard(nodes: &[ServerProcess], percent: u8) -> String { + loop { + for node in nodes { + match memory(node) { + Ok((None, _)) => {} + Ok((Some(limit), current)) + if u128::from(current) * 100 < u128::from(limit) * u128::from(percent) => {} + Ok((Some(limit), current)) => { + return format!( + "{}: memory guard stopped load at {current}/{limit} bytes ({percent}% limit)", + node.evidence["name"] + ) + } + Err(error) => return error, + } + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +pub(super) async fn inspect(args: &PaxosKvArgs) -> Result<Vec<ServerProcess>, String> { + if !args.server_pids.is_empty() { + return native::inspect(args); + } + if args.containers.len() != 3 || args.containers.iter().collect::<BTreeSet<_>>().len() != 3 { + return Err("provide three distinct --container names in endpoint order".into()); + } + let mut result = Vec::new(); + for (name, endpoint) in args.containers.iter().zip(&args.endpoints) { + if name.starts_with('-') { + return Err("invalid container name".into()); + } + let output = tokio::time::timeout( + Duration::from_secs(10), + tokio::process::Command::new("docker") + // Instrument the load client/server, not the Docker Go process. + .env_remove("LD_PRELOAD") + .env_remove("ASAN_OPTIONS") + .args(["inspect", name]) + .kill_on_drop(true) + .output(), + ) + .await + .map_err(|_| "docker inspect timeout")? + .map_err(|e| e.to_string())?; + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).into_owned()); + } + let data: Vec<Value> = serde_json::from_slice(&output.stdout).map_err(|e| e.to_string())?; + let node = data.first().ok_or("container absent")?; + if node["State"]["Running"] != true || node["HostConfig"]["NetworkMode"] != "host" { + return Err(format!( + "{name}: benchmark requires a running host-network container" + )); + } + let cmd = node["Config"]["Cmd"] + .as_array() + .ok_or("container has no command")?; + let flag = |name: &str| cmd.windows(2).find(|v| v[0] == name).and_then(|v| v[1].as_str()); + if flag("--kv-backend") != Some("mem-block") || flag("--wal-backend") != Some("mem-block") { + return Err("benchmark requires explicit mem-block KV and WAL backends".into()); + } + for (expected, name) in [ + (args.expect_window, "--max-inflight"), + (args.expect_coalesce, "--coalesce-max-keys"), + ] { + if let Some(expected) = expected { + if flag(name).and_then(|s| s.parse::<u64>().ok()) != Some(expected) { + return Err(format!("container {name} does not match expected {expected}")); + } + } + } + let origin = reqwest::Url::parse(endpoint).map_err(|e| e.to_string())?; + if !matches!(origin.host_str(), Some("127.0.0.1" | "localhost" | "[::1]")) { + return Err("local host-network benchmark endpoints must use loopback".into()); + } + if flag("--management-port").and_then(|v| v.parse::<u16>().ok()) != origin.port_or_known_default() { + return Err(format!( + "{name}: endpoint port does not match the container command" + )); + } + let pid = node["State"]["Pid"].as_u64().ok_or("missing container PID")?; + let groups = std::fs::read_to_string(format!("/proc/{pid}/cgroup")).map_err(|e| e.to_string())?; + let path = groups + .lines() + .find_map(|l| l.strip_prefix("0::/")) + .ok_or("cgroup v2 is required")?; + let cgroup = PathBuf::from("/sys/fs/cgroup").join(path); + result.push(ServerProcess { + cgroup, + evidence: json!({"kind":"container","name":name,"id":node["Id"],"image":node["Image"], + "image_labels":node["Config"]["Labels"],"command":cmd, + "pid":pid,"started_at":node["State"]["StartedAt"],"endpoint":endpoint, + "network":"host","memory_limit":node["HostConfig"]["Memory"], + "nano_cpus":node["HostConfig"]["NanoCpus"],"cpu_quota":node["HostConfig"]["CpuQuota"], + "cpu_period":node["HostConfig"]["CpuPeriod"],"cpuset":node["HostConfig"]["CpusetCpus"]}), + }); + } + Ok(result) +} diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/deployment/memory.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/deployment/memory.rs new file mode 100644 index 000000000..0ca67b62b --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/deployment/memory.rs @@ -0,0 +1,12 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +pub(super) fn parse_limit(raw: &str) -> Result<Option<u64>, String> { + if raw.trim() == "max" { + return Ok(None); + } + raw.trim() + .parse() + .map(Some) + .map_err(|_| "invalid cgroup memory.max".into()) +} diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/deployment/native.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/deployment/native.rs new file mode 100644 index 000000000..af7ecc792 --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/deployment/native.rs @@ -0,0 +1,106 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::{json, BTreeSet, PathBuf, PaxosKvArgs, ServerProcess}; + +pub(super) fn inspect(args: &PaxosKvArgs) -> Result<Vec<ServerProcess>, String> { + if !args.containers.is_empty() + || args.server_pids.len() != 3 + || args.endpoints.len() != 3 + || args.server_pids.iter().collect::<BTreeSet<_>>().len() != 3 + { + return Err("native diagnostic requires three distinct --server-pid values and three endpoints, without --container".into()); + } + let mut nodes = Vec::new(); + let mut cgroups = BTreeSet::new(); + for (&pid, endpoint) in args.server_pids.iter().zip(&args.endpoints) { + let node = inspect_process(args, pid, endpoint)?; + if !cgroups.insert(node.cgroup.clone()) { + return Err("native server PIDs must have separate cgroups".into()); + } + nodes.push(node); + } + Ok(nodes) +} + +fn inspect_process(args: &PaxosKvArgs, pid: u32, endpoint: &str) -> Result<ServerProcess, String> { + let proc = PathBuf::from(format!("/proc/{pid}")); + let read = |name| std::fs::read_to_string(proc.join(name)).map_err(|e| format!("PID {pid}/{name}: {e}")); + let stat = read("stat")?; + let fields: Vec<_> = stat + .rsplit_once(')') + .ok_or("invalid process stat")? + .1 + .split_whitespace() + .collect(); + if matches!(fields.first(), Some(&"Z" | &"X")) { + return Err(format!("PID {pid} has exited")); + } + let started = fields.get(19).ok_or("missing process start time")?; + let exe = std::fs::read_link(proc.join("exe")).map_err(|e| e.to_string())?; + if exe.file_name().and_then(|n| n.to_str()) != Some("crowdb-paxos-kv-server") { + return Err(format!("PID {pid} is not crowdb-paxos-kv-server")); + } + let command: Vec<String> = read("cmdline")? + .split('\0') + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .collect(); + validate_command(args, &command, endpoint)?; + let group = read("cgroup")?; + let relative = group + .lines() + .find_map(|l| l.strip_prefix("0::/")) + .ok_or("cgroup v2 required")?; + if relative.is_empty() { + return Err("native server requires a dedicated cgroup".into()); + } + let cgroup = PathBuf::from("/sys/fs/cgroup").join(relative); + let policy = |file| std::fs::read_to_string(cgroup.join(file)).map_err(|e| format!("cgroup {file}: {e}")); + let pids = policy("cgroup.procs")?; + if pids.split_whitespace().collect::<Vec<_>>() != [pid.to_string()] { + return Err(format!("PID {pid}: cgroup must contain only this server process")); + } + let boot = std::fs::read_to_string("/proc/sys/kernel/random/boot_id").map_err(|e| e.to_string())?; + // User services may not have the cpuset controller delegated. The process + // status still reports the effective CPU affinity inherited from ancestors. + let status = read("status")?; + let affinity = status + .lines() + .find_map(|line| line.strip_prefix("Cpus_allowed_list:")) + .ok_or("missing process CPU affinity")? + .trim(); + let evidence = json!({"kind":"native-process","name":format!("pid-{pid}"), + "id":format!("{}:{pid}:{started}",boot.trim()),"started_at":started, + "pid":pid,"exe":exe,"command":command,"endpoint":endpoint,"network":"host", + "cgroup":cgroup,"memory_limit":policy("memory.max")?,"cpu_max":policy("cpu.max")?, + "cpuset":std::fs::read_to_string(cgroup.join("cpuset.cpus.effective")).ok(), + "cpu_affinity_list":affinity}); + Ok(ServerProcess { evidence, cgroup }) +} + +fn validate_command(args: &PaxosKvArgs, command: &[String], endpoint: &str) -> Result<(), String> { + let flag = |name: &str| command.windows(2).find(|v| v[0] == name).map(|v| v[1].as_str()); + for name in ["--kv-backend", "--wal-backend"] { + if flag(name) != Some("mem-block") { + return Err("native benchmark requires explicit mem-block KV and WAL backends".into()); + } + } + for (expected, name) in [ + (args.expect_window, "--max-inflight"), + (args.expect_coalesce, "--coalesce-max-keys"), + ] { + if let Some(expected) = expected { + if flag(name).and_then(|s| s.parse::<u64>().ok()) != Some(expected) { + return Err(format!("native server {name} does not match expected {expected}")); + } + } + } + let origin = reqwest::Url::parse(endpoint).map_err(|e| e.to_string())?; + if !matches!(origin.host_str(), Some("127.0.0.1" | "localhost" | "[::1]")) + || flag("--management-port").and_then(|p| p.parse::<u16>().ok()) != origin.port_or_known_default() + { + return Err("native endpoint must match the server's loopback management port".into()); + } + Ok(()) +} diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/run.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/run.rs new file mode 100644 index 000000000..798b481b9 --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/run.rs @@ -0,0 +1,272 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::super::workload::{self, Window}; +use super::{ + args::{Operation, PaxosKvArgs}, + client, collect, dataset, deployment, +}; +use crate::Cli; +use crowdb_common::metrics::system::SystemCollector; +use serde_json::{json, Value}; +use std::{process::ExitCode, sync::Arc, time::Duration}; + +pub async fn run(_cli: &Cli, args: PaxosKvArgs) -> ExitCode { + let output = args.load.output.clone(); + let result = execute(Arc::new(args)).await; + let (report, success) = match result { + Ok(r) => { + let ok = r["passed"] == true; + (r, ok) + } + Err(e) => { + eprintln!("bench paxos-kv failed: {e}"); + (json!({"passed":false,"error":e}), false) + } + }; + let mut report = report; + if report.get("load").is_some() { + report["history_line"] = json!(format!( + "# {} {} {} ops/s={} latency_ns={} passed={}", + chrono::Utc::now().format("%Y-%m-%d"), + report["config"]["label"].as_str().unwrap_or("kv"), + report["deployment_kind"].as_str().unwrap_or("host-container"), + report["load"]["ops_per_sec"], + report["load"]["latency_ns"], + success + )); + } + let text = serde_json::to_string_pretty(&report).expect("JSON value"); + if let Some(path) = output { + if let Err(error) = std::fs::write(path, &text) { + eprintln!("write benchmark report: {error}"); + return ExitCode::from(2); + } + } + println!("{text}"); + if success { + ExitCode::SUCCESS + } else { + ExitCode::from(2) + } +} + +fn validate(args: &PaxosKvArgs) -> Result<(), String> { + args.load.validate()?; + let size = if args.mixed_values { 16384 } else { args.value_size }; + if args.group == 0 + || args.key_space == 0 + || args.key_space > 10_000_000 + || size == 0 + || size > 1024 * 1024 + || args.dataset_bytes().map_or(true, |n| n > args.max_dataset_bytes) + || args.connections == 0 + || args.rpc_workers == 0 + || args.send_queue_capacity == 0 + || args.scan_limit == 0 + { + return Err("invalid group, dataset budget, value size or client capacity".into()); + } + if matches!(args.operation, Operation::Scan) + && args + .key_space + .min(u64::from(args.scan_limit)) + .checked_mul(size as u64 + 21) + .and_then(|n| n.checked_mul(args.load.loader_num as u64)) + .map_or(true, |n| n > args.max_inflight_bytes) + { + return Err("scan responses exceed --max-inflight-bytes".into()); + } + if args.require_snapshot && (!matches!(args.operation, Operation::Write) || args.value_size < 16384) { + return Err("snapshot sentinel requires a write workload with values >=16KiB".into()); + } + Ok(()) +} + +async fn execute(args: Arc<PaxosKvArgs>) -> Result<Value, String> { + validate(&args)?; + let containers = deployment::inspect(&args).await?; + deployment::validate_budget(&args, &containers)?; + let (client, topology) = client::connect(&args).await?; + let client = Arc::new(client); + eprintln!("bench paxos-kv: checking deterministic dataset before timing"); + prepare_dataset(client.clone(), args.clone(), &containers).await?; + if matches!(args.operation, Operation::Prepare) { + return Ok(json!({"passed":true,"operation":"prepare","keys":args.key_space})); + } + if matches!(args.operation, Operation::Scan) { + dataset::scan( + &client, + &args, + crowdb_paxos_kv_client::ReadMode::Linearizable, + true, + ) + .await?; + } + let before = collect::sample(&args, &containers).await?; + let window = Window::new(&args.load); + let operation_args = args.clone(); + let operation_client = client.clone(); + let load = workload::run_until( + &args.load, + window, + move |worker, sequence| { + let args = operation_args.clone(); + let client = operation_client.clone(); + async move { + // Stateless SplitMix64 avoids shared RNG contention and repeating adjacent keys. + let mut n = args + .seed + .wrapping_add((worker as u64) << 32) + .wrapping_add(sequence) + .wrapping_add(0x9e37_79b9_7f4a_7c15); + n = (n ^ (n >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + n = (n ^ (n >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + let id = (n ^ (n >> 31)) % args.key_space; + dataset::operation(&client, &args, id).await + } + }, + deployment::memory_guard(&containers, args.memory_guard_percent), + ); + let measurement = async { + tokio::time::sleep_until(window.measure).await; + let mut system = SystemCollector::new(); + let start = collect::sample(&args, &containers).await?; + tokio::time::sleep_until(window.tail).await; + let cli_system = system.collect(); + let end = collect::sample(&args, &containers).await?; + Ok::<_, String>( + json!({"delta":collect::deltas(&start,&end)?,"before":start,"after":end,"cli_system":cli_system, + "note":"resource snapshots carry actual sampling timestamps; latency uses exact monotonic window; host TCP/DRAM are shared host metrics"}), + ) + }; + eprintln!( + "bench paxos-kv: warmup={}s measurement={}s tail={}s", + args.load.warmup_secs, args.load.duration_secs, args.load.tail_secs + ); + let (mut result, measurement) = tokio::join!(load, measurement); + let metrics = match measurement { + Ok(v) => v, + Err(e) => { + result.errors.push(e); + Value::Null + } + }; + let (after, sentinel) = finish_collection(&args, &containers, &before, &mut result).await; + let verified = + super::verification::after_load(&mut result.errors, verify_dataset(client, args.clone())).await; + let verification = if verified { + if result.errors.is_empty() { + "passed" + } else { + "failed" + } + } else { + eprintln!("bench paxos-kv: skipping dataset verification after failed load or service validation"); + "skipped_after_failure" + }; + let evidence: Vec<_> = containers.iter().map(|c| &c.evidence).collect(); + Ok( + json!({"schema_version":1,"cli_version":env!("CARGO_PKG_VERSION"),"debug_build":cfg!(debug_assertions),"at":chrono::Utc::now(),"passed":result.errors.is_empty(),"errors":result.errors, + "kind":if args.load.duration_secs>=20 {"benchmark"}else{"smoke"},"config":&*args,"load":result.report, + "deployment_kind":if args.server_pids.is_empty() {"host-container"} else {"native-process"}, + "containers":if args.server_pids.is_empty() {json!(evidence)} else {json!([])}, + "processes":if args.server_pids.is_empty() {json!([])} else {json!(evidence)},"initial_topology":topology,"measurement_resources":metrics, + "load_before":before,"load_after":after,"sentinel":sentinel,"verification":verification, + "dataset_policy":"bounded deterministic overwrite dataset; preparation and full verification outside timing", + "host_cpu":std::fs::read_to_string("/proc/cpuinfo").ok().and_then(|s|s.lines().find(|l|l.starts_with("model name")).map(str::to_owned))}), + ) +} + +async fn verify_dataset( + client: Arc<crowdb_paxos_kv_client::PaxosKvClient>, + args: Arc<PaxosKvArgs>, +) -> Result<(), String> { + eprintln!("bench paxos-kv: verifying all dataset bytes outside the timing window"); + tokio::time::timeout( + Duration::from_secs(300), + dataset::verify(client.clone(), args.clone()), + ) + .await + .map_err(|_| "verification timeout".to_owned())??; + if matches!(args.operation, Operation::Scan) { + dataset::scan( + &client, + &args, + crowdb_paxos_kv_client::ReadMode::Linearizable, + true, + ) + .await?; + } + Ok(()) +} + +async fn prepare_dataset( + client: Arc<crowdb_paxos_kv_client::PaxosKvClient>, + args: Arc<PaxosKvArgs>, + containers: &[deployment::ServerProcess], +) -> Result<(), String> { + let prepare = matches!(args.operation, Operation::Prepare | Operation::Write); + let preparation = async { + if prepare { + dataset::prepare(client.clone(), args.clone()).await + } else { + dataset::verify(client.clone(), args.clone()).await + } + }; + tokio::select! { + result = preparation => result?, + error = deployment::memory_guard(containers, args.memory_guard_percent) => return Err(error), + } + Ok(()) +} + +async fn finish_collection( + args: &PaxosKvArgs, + containers: &[deployment::ServerProcess], + before: &Value, + result: &mut workload::LoadResult, +) -> (Option<Value>, Value) { + let after = collect::sample(args, containers).await; + if let Ok(after) = after.as_ref() { + if let Err(error) = collect::deltas(before, after) { + result.errors.push(error); + } + } + match deployment::inspect(args).await { + Ok(current) + if current.iter().zip(containers).all(|(a, b)| { + a.evidence["id"] == b.evidence["id"] && a.evidence["started_at"] == b.evidence["started_at"] + }) => {} + Ok(_) => result + .errors + .push("server process incarnation changed during load".into()), + Err(error) => result.errors.push(error), + } + if result.report["measured_operations"] == 0 { + result + .errors + .push("no successful operations in measurement window".into()); + } + let sentinel = match after.as_ref() { + Ok(after) => match collect::sentinel(args.store, args.group, before, after) { + Ok(v) => { + if args.require_snapshot && v["passed"] != true { + result.errors.push("snapshot sentinel failed".into()); + } + v + } + Err(e) => { + if args.require_snapshot { + result.errors.push(e.clone()); + } + json!({"unsupported":e}) + } + }, + Err(e) => { + result.errors.push(e.clone()); + Value::Null + } + }; + (after.ok(), sentinel) +} diff --git a/app/crowdb-cli/src/commands/bench/paxos_kv/verification.rs b/app/crowdb-cli/src/commands/bench/paxos_kv/verification.rs new file mode 100644 index 000000000..369f74d43 --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/paxos_kv/verification.rs @@ -0,0 +1,18 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::future::Future; + +// A failed load must not inject a second full dataset's worth of reads. +pub(super) async fn after_load<F>(errors: &mut Vec<String>, verification: F) -> bool +where + F: Future<Output = Result<(), String>>, +{ + if !errors.is_empty() { + return false; + } + if let Err(error) = verification.await { + errors.push(error); + } + true +} diff --git a/app/crowdb-cli/src/commands/bench/result.rs b/app/crowdb-cli/src/commands/bench/result.rs deleted file mode 100644 index ecea040fd..000000000 --- a/app/crowdb-cli/src/commands/bench/result.rs +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. - -//! `BenchResult` — the structured result shared by `bench` subcommands. -//! -//! The CLI prints its human-readable [`fmt::Display`] representation and -//! retains the serializable shape for metrics reports. Optional sections -//! (`correctness_errors`, `client_transport_stats`, `server_metrics`) -//! are present only on the subcommands that produce them; `jq`'s `// 0` -//! fallback tolerates their absence. - -use std::fmt; - -use serde::Serialize; - -use crowdb_common::metrics::HistogramSnapshot; - -/// Top-level bench result. `by_op` carries one entry per op kind -/// actually exercised (`read` / `write` / `list`). -#[derive(Debug, Serialize)] -pub struct BenchResult { - pub total_ops: u64, - pub duration_ms: u64, - pub total_errors: u64, - #[serde(skip_serializing_if = "Option::is_none")] - pub correctness_errors: Option<u64>, - pub by_op: BenchOps, - #[serde(skip_serializing_if = "Option::is_none")] - pub client_transport_stats: Option<TransportStats>, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_metrics: Option<ServerMetrics>, -} - -impl BenchResult { - /// Keep the console report and a structured latest-run artifact for measurement tools. - #[must_use] - pub fn finish(&self, directory: &std::path::Path) -> std::process::ExitCode { - println!("{self}"); - let path = directory.join("report.json"); - let result = (|| -> Result<(), Box<dyn std::error::Error>> { - let file = std::fs::File::create(&path)?; - serde_json::to_writer_pretty(file, self)?; - Ok(()) - })(); - if let Err(error) = result { - eprintln!("error: retain benchmark report {}: {error}", path.display()); - return std::process::ExitCode::from(2); - } - std::process::ExitCode::SUCCESS - } -} - -#[derive(Debug, Default, Serialize)] -pub struct BenchOps { - #[serde(skip_serializing_if = "Option::is_none")] - pub read: Option<OpStats>, - #[serde(skip_serializing_if = "Option::is_none")] - pub write: Option<OpStats>, - #[serde(skip_serializing_if = "Option::is_none")] - pub list: Option<OpStats>, -} - -impl BenchOps { - /// Attach a read op section. - #[allow(dead_code)] - pub fn read(mut self, s: HistogramSnapshot) -> Self { - self.read = Some(OpStats::from(s)); - self - } - /// Attach a write op section. - pub fn write(mut self, s: HistogramSnapshot) -> Self { - self.write = Some(OpStats::from(s)); - self - } - /// Attach a list (scan) op section. - #[allow(dead_code)] - pub fn list(mut self, s: HistogramSnapshot) -> Self { - self.list = Some(OpStats::from(s)); - self - } -} - -#[derive(Debug, Serialize)] -pub struct OpStats { - pub latency_us: LatencyUs, - pub latency_ns: LatencyNs, -} - -impl From<HistogramSnapshot> for OpStats { - fn from(s: HistogramSnapshot) -> Self { - // LatencyHistogram records in ns; convert to µs for the JSON report. - Self { - latency_us: LatencyUs { - avg: s.avg / 1000.0, - p50: s.p50 / 1000, - p99: s.p99 / 1000, - }, - latency_ns: LatencyNs { - count: s.count, - avg: s.avg, - min: s.min, - max: s.max, - p50: s.p50, - p90: s.p90, - p95: s.p95, - p99: s.p99, - p999: s.p999, - zero_count: s.zero_count, - overflow_count: s.overflow_count, - }, - } - } -} - -/// Nanosecond values preserve sub-microsecond precision in machine reports. -#[derive(Debug, Serialize)] -pub struct LatencyNs { - pub count: u64, - pub avg: f64, - pub min: u64, - pub max: u64, - pub p50: u64, - pub p90: u64, - pub p95: u64, - pub p99: u64, - pub p999: u64, - pub zero_count: u64, - pub overflow_count: u64, -} - -impl fmt::Display for LatencyNs { - #[allow(clippy::cast_precision_loss)] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "avg={:.3}us p50={:.3}us p90={:.3}us p95={:.3}us p99={:.3}us p99.9={:.3}us max={:.3}us overflow={}", - self.avg / 1000.0, - self.p50 as f64 / 1000.0, - self.p90 as f64 / 1000.0, - self.p95 as f64 / 1000.0, - self.p99 as f64 / 1000.0, - self.p999 as f64 / 1000.0, - self.max as f64 / 1000.0, - self.overflow_count, - ) - } -} - -/// Per-op latency percentiles. Field names are serialized as -/// `*_us` (the contract the regression scripts parse); the Rust names -/// drop the shared `_us` postfix to satisfy `clippy::struct_field_names`. -#[derive(Debug, Serialize)] -pub struct LatencyUs { - #[serde(rename = "avg_us")] - pub avg: f64, - #[serde(rename = "p50_us")] - pub p50: u64, - #[serde(rename = "p99_us")] - pub p99: u64, -} - -/// crowdb-rpc transport stats (client-side or server-side). Fields not -/// exposed by the FFI for a given side are reported as 0. -#[derive(Debug, Default, Serialize)] -pub struct TransportStats { - pub writev_calls: u64, - pub frames_sent: u64, - pub read_calls: u64, - pub frames_parsed: u64, - pub submit_to_writev_avg_us: u64, -} - -/// Server-side metrics fetched from the KV server management API after -/// a write run. Aggregated across the cluster where noted. -#[derive(Debug, Default, Serialize)] -pub struct ServerMetrics { - /// WAL append count summed across nodes. - pub wal_append_count: u64, - pub rpc: TransportStats, - pub replica: ReplicaStats, - pub inflight_enqueued: u64, - pub inflight_wait_avg_us: u64, - /// Election counter delta for the measured interval. - pub election_count: u64, - /// Maintenance snapshot evidence for the measured interval. - pub snapshot: SnapshotStats, - /// Server-side per-op RPC latency (averaged across nodes). - pub rpc_latency: Option<ServerRpcLatency>, -} - -#[derive(Debug, Default, Serialize)] -pub struct SnapshotStats { - pub completed: u64, - pub failed: u64, - pub max_latency_us: u64, -} - -impl ServerMetrics { - /// Convert cumulative counters into values for one measured interval. - #[must_use] - pub fn delta_since(mut self, baseline: &Self) -> Self { - self.wal_append_count = self.wal_append_count.saturating_sub(baseline.wal_append_count); - self.inflight_enqueued = self.inflight_enqueued.saturating_sub(baseline.inflight_enqueued); - self.election_count = self.election_count.saturating_sub(baseline.election_count); - self.snapshot.completed = self - .snapshot - .completed - .saturating_sub(baseline.snapshot.completed); - self.snapshot.failed = self.snapshot.failed.saturating_sub(baseline.snapshot.failed); - self.replica.r2_tps = self.replica.r2_tps.saturating_sub(baseline.replica.r2_tps); - self.replica.r3_tps = self.replica.r3_tps.saturating_sub(baseline.replica.r3_tps); - self - } -} - -/// Server-side per-request-type RPC latency, averaged across nodes. -#[derive(Debug, Default, Serialize)] -#[allow(clippy::struct_field_names)] -pub struct ServerRpcLatency { - pub put_avg_us: f64, - pub put_p50_us: u64, - pub put_p99_us: u64, - pub get_avg_us: f64, - pub get_p50_us: u64, - pub get_p99_us: u64, - pub scan_avg_us: f64, - pub scan_p50_us: u64, - pub scan_p99_us: u64, - pub delete_avg_us: f64, - pub delete_p50_us: u64, - pub delete_p99_us: u64, -} - -#[derive(Debug, Default, Serialize)] -pub struct ReplicaStats { - /// Round-trip latency (µs) to replica 2. - pub r2: u64, - /// Round-trips/s to replica 2. - pub r2_tps: u64, - /// Round-trip latency (µs) to replica 3. - pub r3: u64, - /// Round-trips/s to replica 3. - pub r3_tps: u64, -} - -impl fmt::Display for BenchResult { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let ops_s = (self.total_ops * 1000).checked_div(self.duration_ms).unwrap_or(0); - writeln!(f, "=== bench report ===")?; - writeln!( - f, - " ops: {} duration: {}ms ops/s: {} errors: {}", - self.total_ops, self.duration_ms, ops_s, self.total_errors, - )?; - if let Some(ce) = self.correctness_errors { - writeln!(f, " correctness_errors: {ce}")?; - } - if let Some(s) = &self.by_op.read { - writeln!(f, " read: {}", s.latency_ns)?; - } - if let Some(s) = &self.by_op.write { - writeln!(f, " write: {}", s.latency_ns)?; - } - if let Some(s) = &self.by_op.list { - writeln!(f, " list: {}", s.latency_ns)?; - } - if let Some(cts) = &self.client_transport_stats { - writeln!( - f, - " client_transport: writev={} frames_sent={} read={} frames_parsed={} s2w_avg={}us", - cts.writev_calls, - cts.frames_sent, - cts.read_calls, - cts.frames_parsed, - cts.submit_to_writev_avg_us, - )?; - } - if let Some(sm) = &self.server_metrics { - let wal_per_node = sm.wal_append_count / 3; - writeln!( - f, - " server: wal_append={} ({} /node) inflight_enq={} inflight_wait_avg={}us", - sm.wal_append_count, wal_per_node, sm.inflight_enqueued, sm.inflight_wait_avg_us, - )?; - writeln!( - f, - " server_rpc: writev={} frames_sent={} read={} frames_parsed={} s2w_avg={}us", - sm.rpc.writev_calls, - sm.rpc.frames_sent, - sm.rpc.read_calls, - sm.rpc.frames_parsed, - sm.rpc.submit_to_writev_avg_us, - )?; - writeln!( - f, - " replica: r2={}us/{}tps r3={}us/{}tps", - sm.replica.r2, sm.replica.r2_tps, sm.replica.r3, sm.replica.r3_tps, - )?; - writeln!( - f, - " maintenance: snapshots={} failed={} max={}us elections={}", - sm.snapshot.completed, sm.snapshot.failed, sm.snapshot.max_latency_us, sm.election_count, - )?; - if let Some(rl) = &sm.rpc_latency { - writeln!(f, " server_rpc_latency (avg/p50/p99 us):")?; - writeln!( - f, - " put: {}/{}/{}", - rl.put_avg_us, rl.put_p50_us, rl.put_p99_us, - )?; - writeln!( - f, - " get: {}/{}/{}", - rl.get_avg_us, rl.get_p50_us, rl.get_p99_us, - )?; - writeln!( - f, - " scan: {}/{}/{}", - rl.scan_avg_us, rl.scan_p50_us, rl.scan_p99_us, - )?; - writeln!( - f, - " delete: {}/{}/{}", - rl.delete_avg_us, rl.delete_p50_us, rl.delete_p99_us, - )?; - } - } - write!(f, "=== end report ===") - } -} diff --git a/app/crowdb-cli/src/commands/bench/verb.rs b/app/crowdb-cli/src/commands/bench/verb.rs index 6c193de46..b04ba4226 100644 --- a/app/crowdb-cli/src/commands/bench/verb.rs +++ b/app/crowdb-cli/src/commands/bench/verb.rs @@ -13,9 +13,8 @@ use crate::Cli; /// Workloads attached to a selected application cluster. #[derive(Subcommand, Debug)] pub enum BenchVerb { - /// KV-layer workload (prepare / read / write / scan). - #[command(subcommand)] - Kv(KvBenchVerb), + /// Load an already running three-container Paxos-KV deployment. + PaxosKv(super::paxos_kv::args::PaxosKvArgs), /// Distributed disk-block allocation benchmark. #[command(subcommand)] Diskdb(DiskdbBenchVerb), @@ -290,191 +289,12 @@ pub struct DiskdbArgs { pub metrics_interval: u64, } -#[derive(Subcommand, Debug)] -pub enum KvBenchVerb { - /// Pre-populate `--keys` keys into store 0 / group 0. - Prepare(PrepareArgs), - /// Point-get workload. - Read(ReadArgs), - /// Put workload. - Write(WriteArgs), - /// Scan (list) workload. - Scan(ScanArgs), -} - -#[derive(clap::Args, Debug)] -pub struct PrepareArgs { - /// Number of keys to write into the target store/group. - #[arg(long, default_value_t = 100_000)] - pub keys: u64, - /// Value size in bytes. - #[arg(long, default_value_t = 64)] - pub value_size: usize, - /// Concurrent put tasks. - #[arg(long, default_value_t = 16)] - pub concurrency: usize, - /// Target store ID (default 0 = system store). - #[arg(long, default_value_t = 0)] - pub store: u64, - /// Target group ID (default 0 = system group; use 1+ for bench groups). - #[arg(long, default_value_t = 0)] - pub group: u64, -} - -#[derive(clap::Args, Debug)] -pub struct ReadArgs { - #[arg(long, default_value_t = 10)] - pub duration_secs: u64, - #[arg(long, default_value_t = 1)] - pub loader_num: usize, - #[arg(long, default_value_t = 1)] - pub connections: usize, - /// Read consistency: `linearizable` (lease barrier) or `minslot`. - #[arg(long, value_enum, default_value_t = BenchReadMode::Linearizable)] - pub read_mode: BenchReadMode, - /// `MinSlot` lower bound: `auto` (client high-watermark) or `zero`. - #[arg(long, value_enum, default_value_t = BenchMinSlot::Auto)] - pub min_slot: BenchMinSlot, - /// `MinSlot` endpoint selection: `leader` or `any-replica`. - #[arg(long, value_enum, default_value_t = BenchReadEndpoint::Leader)] - pub read_endpoint_policy: BenchReadEndpoint, - /// Keyspace size (keys are `k{id:020}`, id in `[0, key_space)`). - #[arg(long, default_value_t = 100_000)] - pub key_space: u64, - /// Value size in bytes (for correctness verification). - #[arg(long, default_value_t = 64)] - pub value_size: usize, - /// Verify the first N value bytes match the expected pattern - /// (`0..N` mod 256). 0 disables verification. - #[arg(long, default_value_t = 0)] - pub verify_bytes: usize, - /// Metrics flush interval in seconds. 0 disables the metrics log. - #[arg(long, default_value_t = 1)] - pub metrics_interval: u64, - /// Target store ID (default 0 = system store). - #[arg(long, default_value_t = 0)] - pub store: u64, - /// Target group ID (default 0 = system group; use 1+ for bench groups). - #[arg(long, default_value_t = 0)] - pub group: u64, -} - -#[derive(clap::Args, Debug)] -pub struct WriteArgs { - #[arg(long, default_value_t = 10)] - pub duration_secs: u64, - #[arg(long, default_value_t = 1)] - pub loader_num: usize, - #[arg(long, default_value_t = 1)] - pub connections: usize, - #[arg(long, default_value_t = 1_000_000)] - pub key_space: u64, - #[arg(long, default_value_t = 512)] - pub value_size: usize, - #[arg(long, default_value_t = 0)] - pub verify_bytes: usize, - /// Client transport: event-write mode (coalesce frames via I/O worker). - #[arg(long, default_value_t = false)] - pub event_write: bool, - /// Client transport: I/O worker threads (match server --rpc-workers). - #[arg(long, default_value_t = 2)] - pub rpc_workers: u32, - /// Client transport: per-connection send queue capacity. 0 = default (4096). - #[arg(long, default_value_t = 0)] - pub send_queue_capacity: u32, - /// Metrics flush interval in seconds. 0 disables the metrics log. - #[arg(long, default_value_t = 1)] - pub metrics_interval: u64, - /// Target store ID (default 0 = system store). - #[arg(long, default_value_t = 0)] - pub store: u64, - /// Target group ID (default 0 = system group; use 1+ for bench groups). - #[arg(long, default_value_t = 0)] - pub group: u64, -} - -#[derive(clap::Args, Debug)] -pub struct ScanArgs { - #[arg(long, default_value_t = 10)] - pub duration_secs: u64, - #[arg(long, default_value_t = 1)] - pub loader_num: usize, - #[arg(long, default_value_t = 1)] - pub connections: usize, - #[arg(long, value_enum, default_value_t = BenchReadMode::Linearizable)] - pub read_mode: BenchReadMode, - #[arg(long, value_enum, default_value_t = BenchMinSlot::Auto)] - pub min_slot: BenchMinSlot, - #[arg(long, value_enum, default_value_t = BenchReadEndpoint::Leader)] - pub read_endpoint_policy: BenchReadEndpoint, - /// Maximum items per scan request. - #[arg(long, default_value_t = 1000)] - pub scan_limit: u32, - /// Prefix filter (empty = no prefix). - #[arg(long, default_value = "")] - pub scan_prefix: String, - /// Exclusive lower bound (empty = start from beginning). - #[arg(long, default_value = "")] - pub scan_start_after: String, - /// Scan traversal direction. - #[arg(long, value_enum, default_value_t = BenchScanDirection::Forward)] - pub scan_direction: BenchScanDirection, - #[arg(long, default_value_t = 64)] - pub value_size: usize, - #[arg(long, default_value_t = 100_000)] - pub key_space: u64, - #[arg(long, default_value_t = 0)] - pub verify_bytes: usize, - /// Mixed value-size distribution, e.g. `64:70,1024:20,16384:10` - /// (size:percent). When set, overrides `--value-size`. - #[arg(long)] - pub value_size_mix: Option<String>, - /// Metrics flush interval in seconds. 0 disables the metrics log. - #[arg(long, default_value_t = 1)] - pub metrics_interval: u64, - /// Target store ID (default 0 = system store). - #[arg(long, default_value_t = 0)] - pub store: u64, - /// Target group ID (default 0 = system group; use 1+ for bench groups). - #[arg(long, default_value_t = 0)] - pub group: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -pub enum BenchReadMode { - Linearizable, - Minslot, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -pub enum BenchMinSlot { - Auto, - Zero, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -pub enum BenchReadEndpoint { - Leader, - AnyReplica, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -pub enum BenchScanDirection { - Forward, - Reverse, -} - pub async fn run_bench_verb(cli: &Cli, verb: BenchVerb) -> ExitCode { match verb { BenchVerb::Diskdb(verb) => super::disk::db::run(cli, verb).await, BenchVerb::Chunkdb(verb) => super::chunk::run(cli, verb).await, BenchVerb::Chunkio(verb) => super::io::run(cli, verb).await, BenchVerb::S3(verb) => super::s3::run(cli, verb).await, - BenchVerb::Kv(kv) => match kv { - KvBenchVerb::Prepare(args) => super::kv::prepare::run(cli, args).await, - KvBenchVerb::Read(args) => super::kv::read::run(cli, args).await, - KvBenchVerb::Write(args) => super::kv::write::run(cli, args).await, - KvBenchVerb::Scan(args) => super::kv::scan::run(cli, args).await, - }, + BenchVerb::PaxosKv(args) => super::paxos_kv::run(cli, args).await, } } diff --git a/app/crowdb-cli/src/commands/bench/workload.rs b/app/crowdb-cli/src/commands/bench/workload.rs new file mode 100644 index 000000000..2c5738167 --- /dev/null +++ b/app/crowdb-cli/src/commands/bench/workload.rs @@ -0,0 +1,252 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Continuous closed-loop load with one shared monotonic measurement window. +use clap::Args; +use crowdb_common::metrics::histogram::{HistogramData, LatencyHistogram}; +use serde::Serialize; +use serde_json::{json, Value}; +use std::{ + future::Future, + path::PathBuf, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + task::JoinSet, + time::{sleep_until, Instant}, +}; + +#[derive(Args, Debug, Clone, Serialize)] +pub struct LoadOptions { + #[arg(long, default_value_t = 5, global = true)] + pub warmup_secs: u64, + #[arg(long, default_value_t = 20, global = true)] + pub duration_secs: u64, + #[arg(long, default_value_t = 5, global = true)] + pub tail_secs: u64, + #[arg(long, default_value_t = 30, global = true)] + pub drain_timeout_secs: u64, + #[arg(long, default_value_t = 1, global = true)] + pub loader_num: usize, + #[arg(long, global = true)] + pub output: Option<PathBuf>, +} + +impl LoadOptions { + pub fn validate(&self) -> Result<(), String> { + if self.duration_secs == 0 + || self.duration_secs > 86400 + || self.warmup_secs > 3600 + || self.tail_secs > 3600 + || self.drain_timeout_secs == 0 + || self.drain_timeout_secs > 3600 + || self.loader_num == 0 + || self.loader_num > 4096 + { + return Err( + "invalid load window or concurrency (measurement 1..86400s, concurrency 1..4096)".into(), + ); + } + Ok(()) + } +} + +#[derive(Clone, Copy)] +pub struct Window { + pub start: Instant, + pub measure: Instant, + pub tail: Instant, + pub stop: Instant, +} +impl Window { + pub fn new(options: &LoadOptions) -> Self { + let start = Instant::now(); + let measure = start + Duration::from_secs(options.warmup_secs); + let tail = measure + Duration::from_secs(options.duration_secs); + Self { + start, + measure, + tail, + stop: tail + Duration::from_secs(options.tail_secs), + } + } + pub fn phase(self, time: Instant) -> usize { + if time < self.measure { + 0 + } else if time < self.tail { + 1 + } else { + 2 + } + } +} + +struct Counts { + admitted: AtomicU64, + completed: AtomicU64, + failed: AtomicU64, +} +impl Counts { + fn new() -> Self { + Self { + admitted: AtomicU64::new(0), + completed: AtomicU64::new(0), + failed: AtomicU64::new(0), + } + } + fn json(&self) -> Value { + let admitted = self.admitted.load(Ordering::Relaxed); + let completed = self.completed.load(Ordering::Relaxed); + let failed = self.failed.load(Ordering::Relaxed); + json!({"admitted": admitted, "completed": completed, "failed": failed, + "incomplete": admitted - completed - failed}) + } +} +struct Worker { + phases: [Counts; 3], + measured: LatencyHistogram, + bytes: AtomicU64, + crossing: AtomicU64, +} +impl Worker { + fn new() -> Self { + Self { + phases: std::array::from_fn(|_| Counts::new()), + measured: LatencyHistogram::new("bench.latency".into()), + bytes: AtomicU64::new(0), + crossing: AtomicU64::new(0), + } + } +} + +pub struct LoadResult { + pub report: Value, + pub errors: Vec<String>, +} + +/// Stop and join all workers if a resource guard reports a failure. +pub async fn run_until<F, Fut, Stop>( + options: &LoadOptions, + window: Window, + operation: F, + stop: Stop, +) -> LoadResult +where + F: Fn(usize, u64) -> Fut + Send + Sync + 'static, + Fut: Future<Output = Result<u64, String>> + Send + 'static, + Stop: Future<Output = String>, +{ + tokio::pin!(stop); + let operation = Arc::new(operation); + let workers: Vec<_> = (0..options.loader_num).map(|_| Arc::new(Worker::new())).collect(); + let mut tasks = JoinSet::new(); + for (index, worker) in workers.iter().enumerate() { + let worker = worker.clone(); + let operation = operation.clone(); + tasks.spawn(async move { + let mut sequence = 0; + let mut first_error = None; + loop { + let admitted = Instant::now(); + if admitted >= window.stop { + break; + } + let phase = window.phase(admitted); + worker.phases[phase].admitted.fetch_add(1, Ordering::Relaxed); + let outcome = operation(index, sequence).await; + let completed = Instant::now(); + if window.phase(completed) != phase || completed >= window.stop { + worker.crossing.fetch_add(1, Ordering::Relaxed); + } + match outcome { + Ok(bytes) => { + worker.phases[phase].completed.fetch_add(1, Ordering::Relaxed); + if phase == 1 && completed < window.tail { + worker.measured.observe( + u64::try_from((completed - admitted).as_nanos()).unwrap_or(u64::MAX), + ); + worker.bytes.fetch_add(bytes, Ordering::Relaxed); + } + } + Err(error) => { + worker.phases[phase].failed.fetch_add(1, Ordering::Relaxed); + if first_error.is_none() { + first_error = Some(error); + } + // Yield even for immediately rejected operations. + tokio::task::yield_now().await; + } + } + sequence += 1; + } + first_error + }); + } + let mut errors = Vec::new(); + let deadline = window.stop + Duration::from_secs(options.drain_timeout_secs); + loop { + tokio::select! { + reason = &mut stop => { + errors.push(reason); + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + break; + }, + result = tasks.join_next() => match result { + None => break, + Some(Ok(Some(error))) => errors.push(error), + Some(Err(error)) => errors.push(format!("load worker: {error}")), + Some(Ok(None)) => {}, + }, + () = sleep_until(deadline) => { + errors.push("drain timeout; unfinished operations have unknown outcomes".into()); + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + break; + } + } + } + let report = summarize(options, window, &workers); + if report["failed"].as_u64().unwrap_or(0) > 0 && errors.is_empty() { + errors.push("operation failures in an aborted worker".into()); + } + LoadResult { report, errors } +} + +#[allow(clippy::cast_precision_loss)] +fn summarize(options: &LoadOptions, window: Window, workers: &[Arc<Worker>]) -> Value { + let mut histogram = HistogramData::default(); + let mut phases = [[0_u64; 4]; 3]; + let mut bytes = 0; + let mut crossing = 0; + for worker in workers { + histogram.merge(&worker.measured.capture_total()); + bytes += worker.bytes.load(Ordering::Relaxed); + crossing += worker.crossing.load(Ordering::Relaxed); + for (sum, counts) in phases.iter_mut().zip(&worker.phases) { + let counts = counts.json(); + for (i, field) in ["admitted", "completed", "failed", "incomplete"] + .iter() + .enumerate() + { + sum[i] += counts[field].as_u64().unwrap_or(0); + } + } + } + let h = histogram.snapshot(0); + let phases_json: Vec<_> = phases.iter().zip(["warmup", "measurement", "tail"]).map(|(v, name)| + json!({"phase":name,"admitted":v[0],"completed":v[1],"failed":v[2],"incomplete":v[3]})).collect(); + json!({"phases":phases_json,"measured_operations":h.count,"measured_bytes":bytes, + "ops_per_sec":h.count as f64 / options.duration_secs as f64, + "bytes_per_sec":bytes as f64 / options.duration_secs as f64, + "failed":phases.iter().map(|p|p[2]).sum::<u64>(), + "incomplete":phases.iter().map(|p|p[3]).sum::<u64>(), + "boundary_crossings":crossing,"duration_secs":options.duration_secs, + "elapsed_secs_including_drain":window.start.elapsed().as_secs_f64(), + "selection":"successful requests admitted and completed within measurement; crossings excluded", + "latency_ns":{"count":h.count,"avg":h.avg,"min":h.min,"max":h.max,"p50":h.p50,"p99":h.p99,"p999":h.p999}}) +} diff --git a/app/crowdb-cli/src/commands/cluster/local.rs b/app/crowdb-cli/src/commands/cluster/local.rs index b8e87764c..dc899a3d4 100644 --- a/app/crowdb-cli/src/commands/cluster/local.rs +++ b/app/crowdb-cli/src/commands/cluster/local.rs @@ -29,7 +29,7 @@ pub struct StartArgs { /// Already loaded CROWDB node image; start never builds or pulls. #[arg(long)] image: Option<String>, - /// First localhost console port; each node reserves four ports, ten apart. + /// First localhost console port; Access uses the default relative offsets, nodes add 0/1/2. #[arg(long)] port_base: Option<u16>, #[arg(long)] diff --git a/app/crowdb-cli/src/commands/cluster/local/host.rs b/app/crowdb-cli/src/commands/cluster/local/host.rs index b1025a621..115f3d18e 100644 --- a/app/crowdb-cli/src/commands/cluster/local/host.rs +++ b/app/crowdb-cli/src/commands/cluster/local/host.rs @@ -113,7 +113,7 @@ pub(super) fn selected(name: &str, args: &StartArgs) -> Result<bool> { } if args.port_base.is_some() { return Err(invalid( - "host networking uses port 9090; --port-base is for a local bridge", + "host networking uses the default Console Web port; --port-base is for a local bridge", )); } return Ok(true); diff --git a/app/crowdb-cli/src/commands/cluster/local/host/args.rs b/app/crowdb-cli/src/commands/cluster/local/host/args.rs index e369022ab..a564708ca 100644 --- a/app/crowdb-cli/src/commands/cluster/local/host/args.rs +++ b/app/crowdb-cli/src/commands/cluster/local/host/args.rs @@ -37,7 +37,7 @@ pub(crate) struct HostArgs { physical_host_id: Option<String>, #[arg(long)] interface: Option<String>, - /// This host's management IP on the selected interface; port 9090. + /// This host's management IP on the selected interface; uses the default Console Web port. #[arg(long)] management_address: Option<IpAddr>, #[arg(long)] diff --git a/app/crowdb-cli/src/commands/cluster/local/launch.rs b/app/crowdb-cli/src/commands/cluster/local/launch.rs index 3582497c8..a6b6cf939 100644 --- a/app/crowdb-cli/src/commands/cluster/local/launch.rs +++ b/app/crowdb-cli/src/commands/cluster/local/launch.rs @@ -9,6 +9,7 @@ use super::{ StartArgs, }; use crowdb_console_shared::error::Result; +use crowdb_protocol::WEB_BASE; use serde_json::{json, Value}; use std::time::Duration; @@ -41,12 +42,12 @@ pub(super) async fn accepted(docker: &Docker, cluster: &str, args: &StartArgs) - return Err(invalid("local start requires the Linux amd64 CROWDB node image")); } let record = Launch { - version: 2, + version: 3, cluster: cluster.into(), owner: uuid::Uuid::new_v4().to_string(), runtime: docker.identity().await?, nodes: args.nodes.unwrap_or(1), - port_base: args.port_base.unwrap_or(9090), + port_base: args.port_base.unwrap_or(WEB_BASE), image: image.into(), image_id: metadata["Id"] .as_str() @@ -110,11 +111,16 @@ fn arguments(record: &Launch, index: u8) -> Vec<String> { "--mount", &format!("type=volume,source={volume},target=/opt/crowdb/data"), ])); - let base = record.port_base + u16::from(index - 1) * 10; + let base = record.port_base + u16::from(index - 1) * if record.version < 3 { 10 } else { 1 }; for ((offset, protocol), internal) in [(0_u16, ""), (1, "S3"), (2, "ICEBERG"), (3, "DATASET")] .into_iter() .zip(record.container_ports()) { + let offset = if record.version < 3 { + offset + } else { + internal - WEB_BASE + }; args.extend(strings(&[ "-p", &format!("127.0.0.1:{}:{internal}", base + offset), @@ -138,9 +144,14 @@ fn arguments(record: &Launch, index: u8) -> Vec<String> { pub(super) fn verify(item: &Value, record: &Launch, index: u8) -> Result<()> { require_owned(item, record, "container")?; - let base = record.port_base + u16::from(index - 1) * 10; + let base = record.port_base + u16::from(index - 1) * if record.version < 3 { 10 } else { 1 }; let mut ports = serde_json::Map::new(); for (offset, internal) in (0_u16..).zip(record.container_ports()) { + let offset = if record.version < 3 { + offset + } else { + internal - WEB_BASE + }; ports.insert( format!("{internal}/tcp"), json!([{"HostIp":"127.0.0.1", "HostPort":(base+offset).to_string()}]), @@ -176,7 +187,15 @@ pub(super) fn verify(item: &Value, record: &Launch, index: u8) -> Result<()> { format!("CROWDB_CLUSTER={}", record.cluster), format!("CROWDB_INITIAL_VOTERS={}", record.nodes), ]; - for (offset, protocol) in [(1, "S3"), (2, "ICEBERG"), (3, "DATASET")] { + for ((offset, protocol), internal) in [(1_u16, "S3"), (2, "ICEBERG"), (3, "DATASET")] + .into_iter() + .zip(record.container_ports().into_iter().skip(1)) + { + let offset = if record.version < 3 { + offset + } else { + internal - WEB_BASE + }; expected.push(format!( "CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{}", base + offset diff --git a/app/crowdb-cli/src/commands/cluster/local/state.rs b/app/crowdb-cli/src/commands/cluster/local/state.rs index 20769bb0c..3aa4432ca 100644 --- a/app/crowdb-cli/src/commands/cluster/local/state.rs +++ b/app/crowdb-cli/src/commands/cluster/local/state.rs @@ -152,12 +152,18 @@ impl Launch { pub fn validate(&self) -> Result<()> { let owner = uuid::Uuid::parse_str(&self.owner).map_err(|_| invalid("invalid cluster ownership token"))?; - if !matches!(self.version, 1 | 2) + if !matches!(self.version, 1..=3) || !name_valid(&self.cluster) || owner.is_nil() || owner.to_string() != self.owner || !matches!(self.nodes, 1 | 3) - || !(1..=65512).contains(&self.port_base) + || !(1..=u16::MAX + - if self.version < 3 { + 23 + } else { + crowdb_protocol::ACCESS_SERVER_DATASET_HTTP_BASE - crowdb_protocol::WEB_BASE + 2 + }) + .contains(&self.port_base) || !self.cpus.is_finite() || self.cpus <= 0.0 || self.cpus > f64::from(u32::MAX) @@ -179,21 +185,27 @@ impl Launch { } pub fn origins(&self) -> Vec<String> { (0..self.nodes) - .map(|index| format!("http://127.0.0.1:{}", self.port_base + u16::from(index) * 10)) + .map(|index| { + format!( + "http://127.0.0.1:{}", + self.port_base + u16::from(index) * if self.version < 3 { 10 } else { 1 } + ) + }) .collect() } pub fn container_ports(&self) -> [u16; 4] { - if self.version == 1 { - [9090, 9091, 9092, 9093] - } else { - [ - 9090, + match self.version { + 1 => [9090, 9091, 9092, 9093], + 2 => [9090, 16000, 17000, 18000], + _ => [ + crowdb_protocol::WEB_BASE, crowdb_protocol::ACCESS_SERVER_HTTP_BASE, crowdb_protocol::ACCESS_SERVER_ICEBERG_HTTP_BASE, crowdb_protocol::ACCESS_SERVER_DATASET_HTTP_BASE, - ] + ], } } + pub fn resource_name(&self, kind: &str, index: u8) -> String { let base = format!("crowdb-{}", self.cluster); if kind == "network" { diff --git a/app/crowdb-cli/src/commands/cluster/target.rs b/app/crowdb-cli/src/commands/cluster/target.rs index 1c9b05229..f3c5f5cb0 100644 --- a/app/crowdb-cli/src/commands/cluster/target.rs +++ b/app/crowdb-cli/src/commands/cluster/target.rs @@ -10,6 +10,19 @@ use crowdb_console_shared::deployment::target::{self, ClusterTargetBinding, Reso use crate::Cli; pub(crate) async fn resolve(cli: &Cli) -> Result<Option<ResolvedClusterTarget>, ExitCode> { + if matches!( + cli.command, + crate::Domain::Bench { + verb: crate::commands::BenchVerb::PaxosKv(_) + } + ) { + if cli.cluster.is_some() || !cli.management_endpoints.is_empty() { + return Err(failure( + "bench paxos-kv uses container --endpoint addresses, not a Console cluster target", + )); + } + return Ok(None); + } if matches!( cli.command, crate::Domain::Cluster { @@ -49,7 +62,7 @@ fn validate_command(command: &crate::Domain) -> Result<(), ExitCode> { fn supports_attachment(command: &crate::Domain) -> bool { use crate::{ commands::{BenchVerb, ClusterVerb}, - Domain, KvVerb, + Domain, PaxosKvVerb, }; matches!( command, @@ -61,15 +74,14 @@ fn supports_attachment(command: &crate::Domain) -> bool { | ClusterVerb::Node { .. } | ClusterVerb::DiskGroup { .. } | ClusterVerb::Disk { .. } - } | Domain::Kv { - verb: KvVerb::Store(_) | KvVerb::Group(_) | KvVerb::Replica(_) | KvVerb::Data(_) + } | Domain::PaxosKv { + verb: PaxosKvVerb::Store(_) + | PaxosKvVerb::Group(_) + | PaxosKvVerb::Replica(_) + | PaxosKvVerb::Data(_) } | Domain::S3 { .. } | Domain::Bench { - verb: BenchVerb::Kv(_) - | BenchVerb::Diskdb(_) - | BenchVerb::Chunkdb(_) - | BenchVerb::Chunkio(_) - | BenchVerb::S3(_) + verb: BenchVerb::Diskdb(_) | BenchVerb::Chunkdb(_) | BenchVerb::Chunkio(_) | BenchVerb::S3(_) } ) } diff --git a/app/crowdb-cli/src/commands/kv.rs b/app/crowdb-cli/src/commands/paxos_kv.rs similarity index 72% rename from app/crowdb-cli/src/commands/kv.rs rename to app/crowdb-cli/src/commands/paxos_kv.rs index 0b41af25c..6cbdcdd9d 100644 --- a/app/crowdb-cli/src/commands/kv.rs +++ b/app/crowdb-cli/src/commands/paxos_kv.rs @@ -1,14 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! `kv` domain — logical concepts (store/group/replica), +//! `paxos-kv` domain — logical concepts (store/group/replica), //! and data-plane operations. Each sub-module owns one slice of the //! domain. pub mod data; pub mod logical; -pub(crate) use data::{run_kv_data_verb, KvDataVerb}; +pub(crate) use data::{run_paxos_kv_data_verb, PaxosKvDataVerb}; pub(crate) use logical::{ run_group_verb, run_replica_verb, run_store_verb, GroupVerb, ReplicaVerb, StoreVerb, }; diff --git a/app/crowdb-cli/src/commands/kv/data.rs b/app/crowdb-cli/src/commands/paxos_kv/data.rs similarity index 94% rename from app/crowdb-cli/src/commands/kv/data.rs rename to app/crowdb-cli/src/commands/paxos_kv/data.rs index 4e22f9b1e..9cf478863 100644 --- a/app/crowdb-cli/src/commands/kv/data.rs +++ b/app/crowdb-cli/src/commands/paxos_kv/data.rs @@ -1,18 +1,18 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! `kv put/get/delete/scan/snapshot` command handlers — data-plane. +//! `paxos-kv data put/get/delete/scan/snapshot` command handlers — data-plane. use std::process::ExitCode; use clap::Subcommand; -use crowdb_kv_client::GetOutcome; +use crowdb_paxos_kv_client::GetOutcome; use crate::commands::authority_context; use crate::Cli; #[derive(Subcommand, Debug)] -pub enum KvDataVerb { +pub enum PaxosKvDataVerb { Put { #[arg(short = 's', long)] store: String, @@ -78,9 +78,9 @@ pub enum SnapshotVerb { } #[allow(clippy::too_many_lines)] -pub async fn run_kv_data_verb(cli: &Cli, verb: KvDataVerb) -> ExitCode { +pub async fn run_paxos_kv_data_verb(cli: &Cli, verb: PaxosKvDataVerb) -> ExitCode { match verb { - KvDataVerb::Put { + PaxosKvDataVerb::Put { store, group, key, @@ -114,7 +114,7 @@ pub async fn run_kv_data_verb(cli: &Cli, verb: KvDataVerb) -> ExitCode { } } } - KvDataVerb::Get { store, group, key } => { + PaxosKvDataVerb::Get { store, group, key } => { let (store_id, group_id) = match parse_store_group(&store, &group) { Ok(ids) => ids, Err(c) => return c, @@ -142,7 +142,7 @@ pub async fn run_kv_data_verb(cli: &Cli, verb: KvDataVerb) -> ExitCode { } } } - KvDataVerb::Delete { store, group, key } => { + PaxosKvDataVerb::Delete { store, group, key } => { let (store_id, group_id) = match parse_store_group(&store, &group) { Ok(ids) => ids, Err(c) => return c, @@ -164,7 +164,7 @@ pub async fn run_kv_data_verb(cli: &Cli, verb: KvDataVerb) -> ExitCode { } } } - KvDataVerb::Scan { + PaxosKvDataVerb::Scan { store, group, prefix, @@ -200,7 +200,7 @@ pub async fn run_kv_data_verb(cli: &Cli, verb: KvDataVerb) -> ExitCode { } } } - KvDataVerb::Snapshot(sv) => run_snapshot_verb(cli, sv).await, + PaxosKvDataVerb::Snapshot(sv) => run_snapshot_verb(cli, sv).await, } } diff --git a/app/crowdb-cli/src/commands/kv/logical.rs b/app/crowdb-cli/src/commands/paxos_kv/logical.rs similarity index 100% rename from app/crowdb-cli/src/commands/kv/logical.rs rename to app/crowdb-cli/src/commands/paxos_kv/logical.rs diff --git a/app/crowdb-cli/src/commands/port_alloc.rs b/app/crowdb-cli/src/commands/port_alloc.rs index 2785e8352..cc1e212a5 100644 --- a/app/crowdb-cli/src/commands/port_alloc.rs +++ b/app/crowdb-cli/src/commands/port_alloc.rs @@ -18,16 +18,12 @@ use crowdb_protocol::ServicePort; pub enum ServiceArg { #[value(name = "kv-mgmt")] KvMgmt, - #[value(name = "kv-listen")] - KvListen, - #[value(name = "diskdb-listen")] - DiskdbListen, + #[value(name = "kv-rpc")] + KvRpc, #[value(name = "diskdb-http")] DiskdbHttp, #[value(name = "diskdb-rpc")] DiskdbRpc, - #[value(name = "chunkdb-listen")] - ChunkdbListen, #[value(name = "chunkdb-http")] ChunkdbHttp, #[value(name = "chunkdb-rpc")] @@ -49,12 +45,10 @@ pub enum ServiceArg { impl From<ServiceArg> for ServicePort { fn from(arg: ServiceArg) -> Self { match arg { - ServiceArg::KvMgmt => Self::KvServerMgmt, - ServiceArg::KvListen => Self::KvServerListen, - ServiceArg::DiskdbListen => Self::DiskdbListen, + ServiceArg::KvMgmt => Self::PaxosKvServerMgmt, + ServiceArg::KvRpc => Self::PaxosKvServerRpc, ServiceArg::DiskdbHttp => Self::DiskdbHttp, ServiceArg::DiskdbRpc => Self::DiskdbRpc, - ServiceArg::ChunkdbListen => Self::ChunkdbListen, ServiceArg::ChunkdbHttp => Self::ChunkdbHttp, ServiceArg::ChunkdbRpc => Self::ChunkdbRpc, ServiceArg::DiskioRpc => Self::DiskioRpc, @@ -79,7 +73,7 @@ pub struct PortAllocArgs { #[arg(long, default_value_t = 0)] pub offset: u16, - /// Service type (e.g. "kv-mgmt", "kv-listen", "diskdb-rpc"). + /// Service type (e.g. "kv-mgmt", "kv-rpc", "diskdb-rpc"). #[arg(long)] pub service: Option<ServiceArg>, diff --git a/app/crowdb-cli/src/main.rs b/app/crowdb-cli/src/main.rs index 7e777130d..f4d5792fb 100644 --- a/app/crowdb-cli/src/main.rs +++ b/app/crowdb-cli/src/main.rs @@ -33,9 +33,9 @@ use std::process::ExitCode; use clap::{Parser, Subcommand}; use commands::{ - run_bench_verb, run_cluster_verb, run_group_verb, run_kv_data_verb, run_port_alloc, run_replica_verb, - run_s3_verb, run_store_verb, BenchVerb, ClusterVerb, GroupVerb, KvDataVerb, PortAllocArgs, ReplicaVerb, - S3Verb, StoreVerb, + run_bench_verb, run_cluster_verb, run_group_verb, run_paxos_kv_data_verb, run_port_alloc, + run_replica_verb, run_s3_verb, run_store_verb, BenchVerb, ClusterVerb, GroupVerb, PaxosKvDataVerb, + PortAllocArgs, ReplicaVerb, S3Verb, StoreVerb, }; #[derive(Parser, Debug)] @@ -66,7 +66,7 @@ impl Cli { fn benchmark_slug(&self) -> Option<&'static str> { match &self.command { Domain::Bench { verb } => Some(match verb { - BenchVerb::Kv(_) => "bench-kv", + BenchVerb::PaxosKv(_) => "bench-paxos-kv", BenchVerb::Diskdb(_) => "bench-diskdb", BenchVerb::Chunkdb(_) => "bench-chunkdb", BenchVerb::Chunkio(_) => "bench-chunkio", @@ -85,10 +85,10 @@ enum Domain { #[command(subcommand)] verb: ClusterVerb, }, - /// Logical KV topology and data operations. - Kv { + /// Logical Paxos-KV topology and data operations. + PaxosKv { #[command(subcommand)] - verb: KvVerb, + verb: PaxosKvVerb, }, /// Load injection only. Bench { @@ -110,7 +110,7 @@ enum Domain { } #[derive(Subcommand, Debug)] -enum KvVerb { +enum PaxosKvVerb { #[command(subcommand)] Store(StoreVerb), #[command(subcommand)] @@ -118,7 +118,7 @@ enum KvVerb { #[command(subcommand)] Replica(ReplicaVerb), #[command(subcommand)] - Data(KvDataVerb), + Data(PaxosKvDataVerb), } fn main() -> ExitCode { @@ -161,8 +161,8 @@ fn main() -> ExitCode { "crowdb-cli", 30, 1, - "warn,crowdb_cli=info,crowdb_console_shared=info,crowdb_kv_client=info", - "warn,crowdb_cli=info,crowdb_console_shared=info,crowdb_kv_client=info", + "warn,crowdb_cli=info,crowdb_console_shared=info,crowdb_paxos_kv_client=info", + "warn,crowdb_cli=info,crowdb_console_shared=info,crowdb_paxos_kv_client=info", ) .ok(); crowdb_rpc_ffi::init_logging( @@ -238,11 +238,11 @@ async fn dispatch(mut cli: Cli) -> ExitCode { ); match command { Domain::Cluster { verb } => run_cluster_verb(&cli, verb).await, - Domain::Kv { verb } => match verb { - KvVerb::Store(sv) => run_store_verb(&cli, sv).await, - KvVerb::Group(gv) => run_group_verb(&cli, gv).await, - KvVerb::Replica(rv) => run_replica_verb(&cli, rv).await, - KvVerb::Data(dv) => run_kv_data_verb(&cli, dv).await, + Domain::PaxosKv { verb } => match verb { + PaxosKvVerb::Store(sv) => run_store_verb(&cli, sv).await, + PaxosKvVerb::Group(gv) => run_group_verb(&cli, gv).await, + PaxosKvVerb::Replica(rv) => run_replica_verb(&cli, rv).await, + PaxosKvVerb::Data(dv) => run_paxos_kv_data_verb(&cli, dv).await, }, Domain::Bench { verb } => run_bench_verb(&cli, verb).await, Domain::S3 { verb } => run_s3_verb(&cli, verb).await, diff --git a/app/crowdb-cli/tests/cluster_cli_test.rs b/app/crowdb-cli/tests/cluster_cli_test.rs index e0cac2801..4fae1790f 100644 --- a/app/crowdb-cli/tests/cluster_cli_test.rs +++ b/app/crowdb-cli/tests/cluster_cli_test.rs @@ -2,7 +2,7 @@ //! CLI e2e for cluster observation: `cluster status` and `cluster //! topology` through a system-group endpoint against a real -//! `crowdb-kv-server` with the system group initialized. +//! `crowdb-paxos-kv-server` with the system group initialized. mod common; @@ -68,7 +68,7 @@ fn retired_storage_process_commands_are_absent_and_measurements_remain() { assert_eq!(output.status.code(), Some(2), "{arguments:?}: {output:?}"); assert!(String::from_utf8_lossy(&output.stderr).contains("unrecognized subcommand")); } - for family in ["kv", "diskdb", "chunkdb", "chunkio", "s3"] { + for family in ["paxos-kv", "diskdb", "chunkdb", "chunkio", "s3"] { let output = Command::new(&cli) .args(["bench", family, "--help"]) .output() diff --git a/app/crowdb-cli/tests/cluster_lifecycle_test.rs b/app/crowdb-cli/tests/cluster_lifecycle_test.rs index c871e83ab..0f72e947b 100644 --- a/app/crowdb-cli/tests/cluster_lifecycle_test.rs +++ b/app/crowdb-cli/tests/cluster_lifecycle_test.rs @@ -188,8 +188,14 @@ async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_ ); let original = fs::read(test.receipt("local")).unwrap(); let accepted: Value = serde_json::from_slice(&original).unwrap(); - assert_eq!(accepted["version"], 2); - for (internal, offset) in [(9090, 0), (16000, 1), (17000, 2), (18000, 3)] { + assert_eq!(accepted["version"], 3); + for internal in [ + crowdb_protocol::port::ports::WEB_BASE, + crowdb_protocol::port::ports::ACCESS_SERVER_HTTP_BASE, + crowdb_protocol::port::ports::ACCESS_SERVER_ICEBERG_HTTP_BASE, + crowdb_protocol::port::ports::ACCESS_SERVER_DATASET_HTTP_BASE, + ] { + let offset = internal - crowdb_protocol::port::ports::WEB_BASE; assert_eq!( test.state()["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"] [format!("{internal}/tcp")][0]["HostPort"], @@ -225,43 +231,59 @@ async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn legacy_receipts_retain_protocol_ports_on_resume_and_recreation() { - let test = TestDocker::new(); - let (port, server) = serve("local", 1, true).await; - test.run( - None, - &["start", "--port-base", &port.to_string(), "--wait", "0"], - 0, - ); - let path = test.receipt("local"); - let mut legacy: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - legacy["version"] = json!(1); - let accepted = serde_json::to_vec(&legacy).unwrap(); - fs::write(&path, &accepted).unwrap(); - let ports: serde_json::Map<_, _> = (0_u16..4) - .map(|offset| { - ( - format!("{}/tcp", 9090 + offset), - json!([{"HostIp":"127.0.0.1","HostPort":(port+offset).to_string()}]), - ) - }) - .collect(); - let mut state = test.state(); - state["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"] = json!(ports); - test.save(&state); - test.run(None, &["stop"], 0); - test.run(None, &["start", "--wait", "0"], 0); - assert_eq!(fs::read(&path).unwrap(), accepted); - let mut state = test.state(); - state["container"] = json!({}); - test.save(&state); - test.run(None, &["start", "--wait", "0"], 0); - assert_eq!(fs::read(&path).unwrap(), accepted); - assert_eq!( - test.state()["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"], - json!(ports) - ); - assert_eq!(test.state()["volume"].as_object().unwrap().len(), 1); - server.abort(); + for version in [1, 2] { + let test = TestDocker::new(); + let (port, server) = serve("local", 1, true).await; + test.run( + None, + &["start", "--port-base", &port.to_string(), "--wait", "0"], + 0, + ); + let path = test.receipt("local"); + let mut legacy: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + legacy["version"] = json!(version); + let accepted = serde_json::to_vec(&legacy).unwrap(); + fs::write(&path, &accepted).unwrap(); + let internal = if version == 1 { + [9090, 9091, 9092, 9093] + } else { + [9090, 16000, 17000, 18000] + }; + let ports: serde_json::Map<_, _> = internal + .into_iter() + .zip(0_u16..) + .map(|(internal, offset)| { + ( + format!("{internal}/tcp"), + json!([{"HostIp":"127.0.0.1","HostPort":(port+offset).to_string()}]), + ) + }) + .collect(); + let mut state = test.state(); + state["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"] = json!(ports); + state["container"]["crowdb-local-1"]["Config"]["Env"] = json!([ + "CROWDB_CLUSTER=local", + "CROWDB_INITIAL_VOTERS=1", + format!("CROWDB_S3_PUBLIC_URI=http://127.0.0.1:{}", port + 1), + format!("CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:{}", port + 2), + format!("CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:{}", port + 3), + ]); + test.save(&state); + test.run(None, &["stop"], 0); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!(fs::read(&path).unwrap(), accepted); + let mut state = test.state(); + state["container"] = json!({}); + test.save(&state); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!(fs::read(&path).unwrap(), accepted); + assert_eq!( + test.state()["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"], + json!(ports) + ); + assert_eq!(test.state()["volume"].as_object().unwrap().len(), 1); + server.abort(); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/app/crowdb-cli/tests/cluster_target_test.rs b/app/crowdb-cli/tests/cluster_target_test.rs index 049d2f0db..db8ff14bb 100644 --- a/app/crowdb-cli/tests/cluster_target_test.rs +++ b/app/crowdb-cli/tests/cluster_target_test.rs @@ -20,7 +20,7 @@ fn run(root: &Path, selected: &[&str]) -> String { .env_remove("CROWDB_SYSTEM_PORT") .args(selected) .args([ - "kv", + "paxos-kv", "data", "put", "--store", @@ -117,7 +117,6 @@ fn cli_rejects_removed_system_address_flags() { #[test] fn retained_workloads_require_a_target_before_any_runtime_or_native_effect() { for args in [ - vec!["bench", "kv", "prepare", "--keys", "1"], vec!["bench", "diskdb", "allocate"], vec!["bench", "chunkdb", "allocate"], vec!["bench", "chunkio", "write"], diff --git a/app/crowdb-cli/tests/cluster_workloads_test.rs b/app/crowdb-cli/tests/cluster_workloads_test.rs index 999e66937..956f1fe75 100644 --- a/app/crowdb-cli/tests/cluster_workloads_test.rs +++ b/app/crowdb-cli/tests/cluster_workloads_test.rs @@ -7,7 +7,7 @@ use crowdb_console_shared::deployment::target; use crowdb_diskdb_client::{DiskdbClient, DiskdbRpcTransport}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::{ common::ChunkId, diskdb::rpc::{AllocateBlocksRequest, FreeBlocksRequest}, @@ -23,7 +23,7 @@ async fn mixed_disk_workload_normalizes_frees_predating_its_baseline() { .await .unwrap(); selected.require_native_transport().await.unwrap(); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( selected.identity.group0_management_endpoints.clone(), ))); kv.seed_leader(0, 0, selected.identity.group0_rpc_endpoints[0].clone()); diff --git a/app/crowdb-cli/tests/common/target_commands.rs b/app/crowdb-cli/tests/common/target_commands.rs index c1efeaa40..dd4433e01 100644 --- a/app/crowdb-cli/tests/common/target_commands.rs +++ b/app/crowdb-cli/tests/common/target_commands.rs @@ -10,13 +10,13 @@ pub fn test_operations() -> Vec<Vec<&'static str>> { vec!["cluster", "disk-group", "list"], vec!["cluster", "disk", "list"], vec!["cluster", "topology", "--node", "1"], - vec!["kv", "store", "list"], - vec!["kv", "group", "list", "--store", "0"], + vec!["paxos-kv", "store", "list"], + vec!["paxos-kv", "group", "list", "--store", "0"], vec![ - "kv", "replica", "add", "--store", "0", "--group", "1", "--node", "2", + "paxos-kv", "replica", "add", "--store", "0", "--group", "1", "--node", "2", ], vec![ - "kv", + "paxos-kv", "data", "put", "--store", @@ -29,7 +29,6 @@ pub fn test_operations() -> Vec<Vec<&'static str>> { "untouched", ], vec!["s3", "bucket", "put", "target-marker"], - vec!["bench", "kv", "prepare", "--keys", "1"], vec!["bench", "diskdb", "allocate"], vec!["bench", "chunkdb", "allocate"], vec!["bench", "chunkio", "write"], diff --git a/app/crowdb-cli/tests/direct_cli_test.rs b/app/crowdb-cli/tests/direct_cli_test.rs index b7559e462..e2846bba4 100644 --- a/app/crowdb-cli/tests/direct_cli_test.rs +++ b/app/crowdb-cli/tests/direct_cli_test.rs @@ -8,7 +8,7 @@ fn retired_process_commands_and_selectors_fail_before_runtime_access() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("cli-removed-paths"); for args in [ vec!["launch", "list"], - vec!["kv", "server", "list"], + vec!["paxos-kv", "server", "list"], vec!["cluster", "init"], vec!["cluster", "local-deploy"], vec!["cluster", "destroy"], diff --git a/app/crowdb-cli/tests/kv_bench_test.rs b/app/crowdb-cli/tests/kv_bench_test.rs new file mode 100644 index 000000000..51263b3d8 --- /dev/null +++ b/app/crowdb-cli/tests/kv_bench_test.rs @@ -0,0 +1,237 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#[path = "../src/commands/bench/paxos_kv/counters.rs"] +mod counters; +#[path = "../src/commands/bench/paxos_kv/deployment/memory.rs"] +mod memory; +#[path = "../src/commands/bench/paxos_kv/verification.rs"] +mod verification; +#[path = "../src/commands/bench/workload.rs"] +mod workload; +use serde_json::{json, Value}; +use std::time::Duration; +use workload::{LoadOptions, Window}; + +#[test] +fn cgroup_memory_limit_accepts_unlimited_without_hiding_invalid_values() { + assert_eq!(memory::parse_limit("max\n").unwrap(), None); + assert_eq!( + memory::parse_limit("21474836480\n").unwrap(), + Some(21_474_836_480) + ); + assert!(memory::parse_limit("").is_err()); + assert!(memory::parse_limit("unknown").is_err()); + assert!(memory::parse_limit("-1").is_err()); +} + +fn options() -> LoadOptions { + LoadOptions { + warmup_secs: 1, + duration_secs: 1, + tail_secs: 1, + drain_timeout_secs: 1, + loader_num: 2, + output: None, + } +} + +#[tokio::test] +async fn continuous_load_separates_boundaries_and_drains() { + let options = options(); + options.validate().unwrap(); + let result = workload::run_until( + &options, + Window::new(&options), + |_, _| async { + tokio::time::sleep(Duration::from_millis(120)).await; + Ok(64) + }, + std::future::pending(), + ) + .await; + assert!(result.errors.is_empty(), "{:?}", result.errors); + assert_eq!(result.report["incomplete"], 0); + assert!(result.report["boundary_crossings"].as_u64().unwrap() >= 4); + let middle = &result.report["phases"][1]; + assert!(result.report["measured_operations"].as_u64().unwrap() < middle["completed"].as_u64().unwrap()); + for phase in result.report["phases"].as_array().unwrap() { + assert_eq!(phase["admitted"], phase["completed"]); + } +} + +#[tokio::test] +async fn failures_in_warmup_and_tail_invalidate_run() { + let options = options(); + let window = Window::new(&options); + let result = workload::run_until( + &options, + window, + move |_, _| async move { + tokio::time::sleep(Duration::from_millis(50)).await; + if window.phase(tokio::time::Instant::now()) == 1 { + Ok(1) + } else { + Err("injected phase error".into()) + } + }, + std::future::pending(), + ) + .await; + assert!(!result.errors.is_empty()); + assert!(result.report["phases"][0]["failed"].as_u64().unwrap() > 0); + assert!(result.report["phases"][2]["failed"].as_u64().unwrap() > 0); +} + +#[tokio::test] +async fn drain_timeout_preserves_incomplete_and_joins_workers() { + let mut options = options(); + options.warmup_secs = 0; + options.tail_secs = 0; + let result = workload::run_until( + &options, + Window::new(&options), + |_, _| async { std::future::pending::<Result<u64, String>>().await }, + std::future::pending(), + ) + .await; + assert_eq!(result.report["incomplete"], 2); + assert!(result.errors.iter().any(|e| e.contains("drain timeout"))); +} + +#[tokio::test] +async fn worker_panics_are_failures() { + let options = options(); + let result = workload::run_until( + &options, + Window::new(&options), + |_, _| async { + panic!("injected worker panic"); + #[allow(unreachable_code)] + Ok(0) + }, + std::future::pending(), + ) + .await; + assert_eq!(result.errors.len(), 2); + assert_eq!(result.report["incomplete"], 2); +} + +fn snapshot(elections: u64, completed: u64, failed: u64) -> Value { + let points:Vec<_>=[("paxos.elections.c",elections),("maintenance.snapshot.success.c",completed),("maintenance.snapshot.failure.c",failed)] + .into_iter().map(|(suffix,value)|json!({"name":format!("s.0.g.1.{suffix}"),"fields":[{"key":"total","value":value}]})).collect(); + json!({"metrics":[{"metrics":points},{"metrics":points},{"metrics":points}]}) +} +#[test] +fn snapshot_sentinel_uses_deltas_and_rejects_missing_or_reset_counters() { + let before = snapshot(2, 10, 1); + assert_eq!( + counters::sentinel(0, 1, &before, &snapshot(2, 11, 1)).unwrap()["passed"], + true + ); + for after in [snapshot(2, 10, 1), snapshot(3, 11, 1), snapshot(2, 11, 2)] { + assert_eq!( + counters::sentinel(0, 1, &before, &after).unwrap()["passed"], + false + ); + } + assert!(counters::sentinel(0, 1, &before, &snapshot(1, 11, 1)).is_err()); + assert!(counters::sentinel(0, 1, &before, &json!({})).is_err()); + assert!(counters::deltas(&json!({}), &json!({})).is_err()); +} + +#[test] +fn container_bench_rejects_missing_deployment_and_console_target() { + for (extra, message) in [ + (vec![], "three distinct --container"), + ( + vec!["--cluster", "missing"], + "bench paxos-kv uses container --endpoint", + ), + ] { + let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("kv-bench-contract"); + let output = std::process::Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) + .env_remove("CROWDB_CLUSTER") + .env("CROWDB_RUNTIME_ROOT", dir.path()) + .args(extra) + .args(["bench", "paxos-kv", "read"]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!( + String::from_utf8_lossy(&output.stderr).contains(message), + "{output:?}" + ); + } +} + +#[tokio::test] +async fn resource_guard_cancels_inflight_work_and_fails_the_run() { + let options = options(); + let result = workload::run_until( + &options, + Window::new(&options), + |_, _| async { std::future::pending::<Result<u64, String>>().await }, + async { + tokio::time::sleep(Duration::from_millis(30)).await; + "memory guard stopped load".to_owned() + }, + ) + .await; + assert_eq!(result.errors, ["memory guard stopped load"]); + assert_eq!(result.report["incomplete"], 2); + assert_eq!(result.report["measured_operations"], 0); + assert!(result.report["elapsed_secs_including_drain"].as_f64().unwrap() < 1.0); +} + +#[tokio::test] +async fn failed_load_does_not_poll_the_dataset_verification() { + let mut errors = vec!["memory guard stopped load".to_owned()]; + let mut requests = 0; + let started = verification::after_load(&mut errors, async { + requests += 1; + Ok(()) + }) + .await; + assert!(!started); + assert_eq!(requests, 0); + assert_eq!(errors, ["memory guard stopped load"]); + + errors.clear(); + assert!( + verification::after_load(&mut errors, async { + requests += 1; + Err("incorrect dataset bytes".to_owned()) + }) + .await + ); + assert_eq!(requests, 1); + assert_eq!(errors, ["incorrect dataset bytes"]); +} + +#[test] +fn native_bench_rejects_duplicate_pids_and_mixed_deployments() { + for (extra, message) in [ + ( + vec!["--server-pid", "1", "--server-pid", "1", "--server-pid", "1"], + "three distinct --server-pid", + ), + ( + vec!["--server-pid", "1", "--container", "node1"], + "cannot be used with", + ), + ] { + let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("kv-bench-native-contract"); + let output = std::process::Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) + .env_remove("CROWDB_CLUSTER") + .env("CROWDB_RUNTIME_ROOT", dir.path()) + .args(["bench", "paxos-kv", "read"]) + .args(extra) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!( + String::from_utf8_lossy(&output.stderr).contains(message), + "{output:?}" + ); + } +} diff --git a/app/crowdb-diskdb/Cargo.toml b/app/crowdb-diskdb/Cargo.toml index 546028912..18e6397a4 100644 --- a/app/crowdb-diskdb/Cargo.toml +++ b/app/crowdb-diskdb/Cargo.toml @@ -28,7 +28,7 @@ workspace = true [dependencies] crowdb-protocol = { path = "../../lib/crowdb-protocol" } -crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../../lib/crowdb-paxos-kv-client" } crowdb-common = { path = "../../lib/crowdb-common/rust" } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi" } crowdb-chunkdb-client = { path = "../../lib/crowdb-chunkdb-client" } @@ -55,7 +55,7 @@ rand = "0.9.5" futures = "0.3" [dev-dependencies] -crowdb-kv = { path = "../../lib/crowdb-kv" } +crowdb-paxos-kv = { path = "../../lib/crowdb-paxos-kv" } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } serde_json = "1" diff --git a/app/crowdb-diskdb/conf/crowdb_diskdb_config.toml b/app/crowdb-diskdb/conf/crowdb_diskdb_config.toml index 5caf605fe..c2b9b4c64 100644 --- a/app/crowdb-diskdb/conf/crowdb_diskdb_config.toml +++ b/app/crowdb-diskdb/conf/crowdb_diskdb_config.toml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. # Canonical diskdb server config template. Tracked so the # `tracked_config_file_loads_and_validates` test guards against # stale/mismatched template edits. All sections are explicit; omitting @@ -5,10 +6,9 @@ [server] rpc_workers = 2 -listen_addr = "0.0.0.0:11000" -http_listen_addr = "0.0.0.0:11100" -rpc_listen_addr = "0.0.0.0:11200" -kv_server_mgmt_seeds = ["http://127.0.0.1:10000"] +http_listen_addr = "0.0.0.0:13135" +rpc_listen_addr = "0.0.0.0:14135" +kv_server_mgmt_seeds = ["http://127.0.0.1:10135"] [storage] zone_size_bytes = 17179869184 diff --git a/app/crowdb-diskdb/conf/crowdb_diskdb_config.toml.ports.in b/app/crowdb-diskdb/conf/crowdb_diskdb_config.toml.ports.in new file mode 100644 index 000000000..e0eeddaf5 --- /dev/null +++ b/app/crowdb-diskdb/conf/crowdb_diskdb_config.toml.ports.in @@ -0,0 +1,71 @@ +# Canonical diskdb server config template. Tracked so the +# `tracked_config_file_loads_and_validates` test guards against +# stale/mismatched template edits. All sections are explicit; omitting +# any non-`[server]` section is also valid (serde fills from `Default`). + +[server] +rpc_workers = 2 +http_listen_addr = "0.0.0.0:@DISKDB_HTTP_BASE@" +rpc_listen_addr = "0.0.0.0:@DISKDB_RPC_BASE@" +kv_server_mgmt_seeds = ["http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@"] + +[storage] +zone_size_bytes = 17179869184 +block_size_bytes = 1048576 +allocate_granularity = 1048576 +zone_rotate_count = 4 +cas_retry_limit = 100 + +[heartbeat] +interval_secs = 10 +miss_threshold = 3 +temp_failure_timeout_secs = 900 + +[persistence] +# Coalesce concurrent frees without delaying the first request. +free_batch_enabled = false +# Maximum records per KV proposal; one oversized RPC remains atomic. +free_flush_max_batch = 256 +compaction_cadence_secs = 300 +snapshot_compaction_threshold = 4096 +load_concurrency = 16 + +[scanner] +scan_interval_secs = 600 +# Reconcile tentative ownership independently and at a deliberately low rate. +tentative_owner_scan_interval_secs = 600 +# Traverse disk-group, disk, and zone serially, pausing after every zone. +tentative_owner_zone_delay_secs = 3 +# Require a durable Absent observation to age for 24 hours before reclaiming. +tentative_owner_grace_secs = 86400 +reverify_delay_ms = 1000 + +[scanner.ghost] +detect = true +auto_correct = false + +[scanner.integrity] +verify = true +detect_owner_mismatch = false + +[rebalance] +enabled = true +plan_interval_secs = 300 +imbalance_threshold_pct = 20 +max_jobs_per_cycle = 1 +zone_delay_secs = 3 + +[allocator] +load_aware = true +load_aware_weight = "free_bytes" + +[sync] +group0_store_id = 0 +group0_group_id = 0 +sync_interval_secs = 10 + +[reporting] +interval_secs = 10 + +[notify] +notify_enabled = false diff --git a/app/crowdb-diskdb/src/ddb_config.rs b/app/crowdb-diskdb/src/ddb_config.rs index 9bc85a74d..3e0891111 100644 --- a/app/crowdb-diskdb/src/ddb_config.rs +++ b/app/crowdb-diskdb/src/ddb_config.rs @@ -7,7 +7,7 @@ use std::net::SocketAddr; use std::time::Duration; use crowdb_common::config::BaseConfig; -use crowdb_protocol::{DISKDB_HTTP_BASE, DISKDB_LISTEN_BASE, DISKDB_RPC_BASE, KV_SERVER_MGMT_BASE}; +use crowdb_protocol::{DISKDB_HTTP_BASE, DISKDB_RPC_BASE, PAXOS_KV_SERVER_MGMT_BASE}; use serde::{Deserialize, Serialize}; /// Top-level configuration for a diskdb instance. @@ -40,24 +40,21 @@ impl BaseConfig for DdbConfig { } } -/// main listener + HTTP + crowdb-rpc listen addresses. +/// HTTP management and crowdb-rpc listen addresses. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct ServerConfig { /// static: crowdb-rpc I/O workers serving this process. #[serde(default = "default_rpc_workers")] pub rpc_workers: u32, - /// static: main listen address. - pub listen_addr: String, /// static: HTTP management listen address. pub http_listen_addr: String, - /// static: crowdb-rpc listen address (R115 migration — runs alongside - /// the main listener during the mixed-rollout window). + /// static: crowdb-rpc listen address. pub rpc_listen_addr: String, /// static: unique instance ID (auto-generated UUID if absent). pub instance_id: Option<String>, /// static: HTTP management-API seed endpoints (`http://host:port`) - /// of the crowdb-kv-server(s) used to discover the system group + /// of the crowdb-paxos-kv-server(s) used to discover the system group /// (store 0, group 0) leader and, via `/topology`, the data-group /// leaders. The client refreshes this lazily on first use, so the /// leader does not need to be pre-seeded. At least one must be @@ -87,11 +84,10 @@ impl Default for ServerConfig { fn default() -> Self { Self { rpc_workers: default_rpc_workers(), - listen_addr: format!("0.0.0.0:{DISKDB_LISTEN_BASE}"), http_listen_addr: format!("0.0.0.0:{DISKDB_HTTP_BASE}"), rpc_listen_addr: format!("0.0.0.0:{DISKDB_RPC_BASE}"), instance_id: None, - kv_server_mgmt_seeds: vec![format!("http://127.0.0.1:{KV_SERVER_MGMT_BASE}")], + kv_server_mgmt_seeds: vec![format!("http://127.0.0.1:{PAXOS_KV_SERVER_MGMT_BASE}")], kv_pool_size: default_kv_pool_size(), kv_rpc_workers: default_kv_rpc_workers(), } @@ -462,12 +458,6 @@ pub fn validate(config: &DdbConfig) -> Result<(), String> { if config.persistence.load_concurrency == 0 { return Err("load_concurrency must be > 0".to_string()); } - if config.server.listen_addr.parse::<SocketAddr>().is_err() { - return Err(format!( - "listen_addr {:?} is not a valid SocketAddr", - config.server.listen_addr, - )); - } if config.server.http_listen_addr.parse::<SocketAddr>().is_err() { return Err(format!( "http_listen_addr {:?} is not a valid SocketAddr", diff --git a/app/crowdb-diskdb/src/ddb_kv_client.rs b/app/crowdb-diskdb/src/ddb_kv_client.rs index 5a0e7d06d..2904a7b16 100644 --- a/app/crowdb-diskdb/src/ddb_kv_client.rs +++ b/app/crowdb-diskdb/src/ddb_kv_client.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! [`DdbKvClient`] — wraps [`CrowdbKvClient`] for put/delete/scan on +//! [`DdbKvClient`] — wraps [`PaxosKvClient`] for put/delete/scan on //! the disk-group's bound paxos data group. //! //! Uses `(store_id, group_id)` from `DdbDiskGroup.bind` (set by the @@ -13,7 +13,7 @@ use std::sync::Arc; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, CrowdbKvClient, GetOutcome, JournalOp, ReadMode, Result}; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome, JournalOp, PaxosKvClient, ReadMode, Result}; use crowdb_protocol::common::DiskId; use crowdb_protocol::diskdb::rpc::{ BusyBlockValue, FreeBlockValue, RecoveryScanProgressValue, RelocationJournalValue, ZoneValue, @@ -39,11 +39,11 @@ pub type Bind = (u64, u64); /// Client for the disk-group's bound paxos data group. /// /// All methods take `(store_id, group_id)` from `DdbDiskGroup.bind`. The -/// wrapped `CrowdbKvClient` must have its topology seeded with the +/// wrapped `PaxosKvClient` must have its topology seeded with the /// data-group leader endpoint. #[derive(Clone)] pub struct DdbKvClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, metrics: Option<Arc<crate::metrics::DiskdbMetrics>>, require_fence: bool, disks: Option<Arc<Vec<Arc<crate::model::disk::DdbDisk>>>>, @@ -113,7 +113,7 @@ impl DdbKvClient { let mut journals = Vec::with_capacity(outcome.items.len()); for (raw_key, raw_value) in outcome.items { let key = RelocationJournalKey::from_bytes(&raw_key).map_err(|error| { - crowdb_kv_client::Error::SysdataDecode { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{raw_key:02x?}"), reason: error.to_string(), } @@ -137,7 +137,7 @@ impl DdbKvClient { .await? { GetOutcome::Found { value, .. } => bincode::deserialize(&value).map(Some).map_err(|error| { - crowdb_kv_client::Error::SysdataDecode { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{:02x?}", key.to_bytes()), reason: error.to_string(), } @@ -179,9 +179,9 @@ impl DdbKvClient { .map(|_| ()) } - /// Wrap a `CrowdbKvClient` for data-group access. + /// Wrap a `PaxosKvClient` for data-group access. #[must_use] - pub fn new(kv: CrowdbKvClient) -> Self { + pub fn new(kv: PaxosKvClient) -> Self { Self { kv: Arc::new(kv), metrics: None, @@ -191,9 +191,9 @@ impl DdbKvClient { } } - /// Wrap an already-shared `CrowdbKvClient`. + /// Wrap an already-shared `PaxosKvClient`. #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv, metrics: None, @@ -210,9 +210,9 @@ impl DdbKvClient { self } - /// Access the underlying `CrowdbKvClient`. + /// Access the underlying `PaxosKvClient`. #[must_use] - pub fn kv(&self) -> &CrowdbKvClient { + pub fn kv(&self) -> &PaxosKvClient { &self.kv } @@ -238,11 +238,12 @@ impl DdbKvClient { .await?; match outcome { GetOutcome::Found { value, revision } => { - let bv = - bincode::deserialize(&value).map_err(|e| crowdb_kv_client::Error::SysdataDecode { + let bv = bincode::deserialize(&value).map_err(|e| { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{:02x?}", key.to_bytes()), reason: e.to_string(), - })?; + } + })?; Ok(Some((bv, revision))) } GetOutcome::NotFound => Ok(None), @@ -435,13 +436,12 @@ impl DdbKvClient { .get(store_id, group_id, &zone_bytes, ReadMode::Linearizable, None) .await?; if let GetOutcome::Found { value, .. } = zone_get { - records.zone_value = - Some( - ZoneValue::from_bytes(&value).map_err(|e| crowdb_kv_client::Error::SysdataDecode { - key: format!("{zone_bytes:02x?}"), - reason: e, - })?, - ); + records.zone_value = Some(ZoneValue::from_bytes(&value).map_err(|e| { + crowdb_paxos_kv_client::Error::SysdataDecode { + key: format!("{zone_bytes:02x?}"), + reason: e, + } + })?); } // 2. BusyBlock records @@ -521,19 +521,20 @@ impl DdbKvClient { .await?; let cutoff = free_scan.scan_cutoff; if free_scan.items.len() != free_scan.commit_slots.len() { - return Err(crowdb_kv_client::Error::SysdataDecode { + return Err(crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{free_prefix:02x?}"), reason: "bounded free scan item/commit-slot count mismatch".to_string(), }); } for ((key, value), commit_slot) in free_scan.items.iter().zip(&free_scan.commit_slots) { - let decoded_key = - FreeBlockKey::from_bytes(key).map_err(|error| crowdb_kv_client::Error::SysdataDecode { + let decoded_key = FreeBlockKey::from_bytes(key).map_err(|error| { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{key:02x?}"), reason: error.to_string(), - })?; + } + })?; let decoded_value = bincode::deserialize::<FreeBlockValue>(value).map_err(|error| { - crowdb_kv_client::Error::SysdataDecode { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{key:02x?}"), reason: error.to_string(), } @@ -551,19 +552,20 @@ impl DdbKvClient { .scan_bounded_at(store_id, group_id, &busy_prefix, &[], &[], 0, false, None, cutoff) .await?; if busy_scan.items.len() != busy_scan.commit_slots.len() { - return Err(crowdb_kv_client::Error::SysdataDecode { + return Err(crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{busy_prefix:02x?}"), reason: "bounded busy scan item/commit-slot count mismatch".to_string(), }); } for ((key, value), commit_slot) in busy_scan.items.iter().zip(&busy_scan.commit_slots) { - let decoded_key = - BusyBlockKey::from_bytes(key).map_err(|error| crowdb_kv_client::Error::SysdataDecode { + let decoded_key = BusyBlockKey::from_bytes(key).map_err(|error| { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{key:02x?}"), reason: error.to_string(), - })?; + } + })?; let decoded_value = bincode::deserialize::<BusyBlockValue>(value).map_err(|error| { - crowdb_kv_client::Error::SysdataDecode { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{key:02x?}"), reason: error.to_string(), } @@ -646,11 +648,12 @@ impl DdbKvClient { .await?; match outcome { GetOutcome::Found { value, .. } => { - let zv = - ZoneValue::from_bytes(&value).map_err(|e| crowdb_kv_client::Error::SysdataDecode { + let zv = ZoneValue::from_bytes(&value).map_err(|e| { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{:02x?}", key.to_bytes()), reason: e, - })?; + } + })?; Ok(Some(zv)) } GetOutcome::NotFound => Ok(None), @@ -684,7 +687,7 @@ impl DdbKvClient { /// Slot-ordered journal scan over `BusyBlockKey` ops for one zone /// (R73 recovery strategy 2 step c, scan 1). Wraps - /// `CrowdbKvClient::journal_scan` with `BusyBlockKey::prefix_for_zone`. + /// `PaxosKvClient::journal_scan` with `BusyBlockKey::prefix_for_zone`. /// Returns ops in slot order; the caller replays them to rebuild /// the bitmap. pub async fn journal_scan_busy( @@ -716,7 +719,7 @@ impl DdbKvClient { /// Slot-ordered journal scan over `FreeBlockKey` ops for one zone /// (R73 recovery strategy 2 step c, scan 2). Wraps - /// `CrowdbKvClient::journal_scan` with `FreeBlockKey::prefix_for_zone`. + /// `PaxosKvClient::journal_scan` with `FreeBlockKey::prefix_for_zone`. pub async fn journal_scan_free( &self, bind: Bind, @@ -785,7 +788,7 @@ impl DdbKvClient { match outcome { GetOutcome::Found { value, .. } => { let v = RecoveryScanProgressValue::from_bytes(&value).map_err(|e| { - crowdb_kv_client::Error::SysdataDecode { + crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{:02x?}", key.to_bytes()), reason: e, } @@ -814,7 +817,7 @@ impl DdbKvClient { } fn decode_relocation_journal(key: &RelocationJournalKey, bytes: &[u8]) -> Result<RelocationJournalValue> { - bincode::deserialize(bytes).map_err(|error| crowdb_kv_client::Error::SysdataDecode { + bincode::deserialize(bytes).map_err(|error| crowdb_paxos_kv_client::Error::SysdataDecode { key: format!("{:02x?}", key.to_bytes()), reason: error.to_string(), }) diff --git a/app/crowdb-diskdb/src/ddb_kv_client/disk_writes.rs b/app/crowdb-diskdb/src/ddb_kv_client/disk_writes.rs index e7fb192ad..c0624f9cd 100644 --- a/app/crowdb-diskdb/src/ddb_kv_client/disk_writes.rs +++ b/app/crowdb-diskdb/src/ddb_kv_client/disk_writes.rs @@ -42,7 +42,7 @@ impl DdbKvClient { .filter_map(disk_id) .any(|id| !disks.iter().any(|disk| disk.disk_id == id)) { - return Err(crowdb_kv_client::Error::Server( + return Err(crowdb_paxos_kv_client::Error::Server( "disk is absent from the captured membership".into(), )); } @@ -58,7 +58,7 @@ impl DdbKvClient { for permit in permits { permit.complete(); } - return Err(crowdb_kv_client::Error::Server( + return Err(crowdb_paxos_kv_client::Error::Server( "disk is administratively excluded; metadata write rejected".into(), )); }; diff --git a/app/crowdb-diskdb/src/ddb_kv_client/ownership.rs b/app/crowdb-diskdb/src/ddb_kv_client/ownership.rs index b078228f7..02ebb3944 100644 --- a/app/crowdb-diskdb/src/ddb_kv_client/ownership.rs +++ b/app/crowdb-diskdb/src/ddb_kv_client/ownership.rs @@ -3,7 +3,7 @@ //! Data-group fencing for `DiskGroup` management handovers. -use crowdb_kv_client::{BatchOp, Error, GetOutcome, ReadMode, Result}; +use crowdb_paxos_kv_client::{BatchOp, Error, GetOutcome, ReadMode, Result}; use crowdb_protocol::{ common::{DiskId, HwStatus}, key::DiskKey, @@ -44,7 +44,7 @@ impl DdbKvClient { group: &crate::model::disk_group::DdbDiskGroup, disk: &DiskId, ) -> Result<()> { - let hardware = crowdb_kv_client::HardwareClient::from_shared(self.kv.clone()); + let hardware = crowdb_paxos_kv_client::HardwareClient::from_shared(self.kv.clone()); let Some(fence) = &self.fence else { if self.require_fence { return Err(fenced()); diff --git a/app/crowdb-diskdb/src/ddb_kv_client/retained.rs b/app/crowdb-diskdb/src/ddb_kv_client/retained.rs index 26018bed2..996b86744 100644 --- a/app/crowdb-diskdb/src/ddb_kv_client/retained.rs +++ b/app/crowdb-diskdb/src/ddb_kv_client/retained.rs @@ -5,7 +5,7 @@ use super::{BinaryKey, Bind, BusyBlockKey, DdbKvClient, DiskId, FreeBlockKey, ZoneKey}; use crate::model::records::{BusyRecord, FreeRecord, ZoneRecords}; use bincode::Options; -use crowdb_kv_client::{Error, Result}; +use crowdb_paxos_kv_client::{Error, Result}; use crowdb_protocol::diskdb::rpc::ZoneValue; use crowdb_protocol::ZoneValueExt; use serde::de::DeserializeOwned; @@ -36,7 +36,7 @@ impl DdbKvClient { &self, key: &crowdb_protocol::key::DiskKey, ) -> Result<bool> { - let hardware = crowdb_kv_client::HardwareClient::from_shared(self.kv.clone()); + let hardware = crowdb_paxos_kv_client::HardwareClient::from_shared(self.kv.clone()); Ok(hardware.disk_rejoin_receipt(key).await?.is_some() || hardware.disk_online_operation(key).await?.is_some()) } diff --git a/app/crowdb-diskdb/src/ddb_kv_client/zone_init.rs b/app/crowdb-diskdb/src/ddb_kv_client/zone_init.rs index 6b5ed431b..858cc4d1b 100644 --- a/app/crowdb-diskdb/src/ddb_kv_client/zone_init.rs +++ b/app/crowdb-diskdb/src/ddb_kv_client/zone_init.rs @@ -8,7 +8,8 @@ use super::{ ZoneKey, ZoneValue, ZoneValueExt, }; -pub const MAX_ZONE_INIT_BATCH: usize = 128; +// Each proposal must also fit the native tree's reserved gap-repair window. +pub const MAX_ZONE_INIT_BATCH: usize = 64; impl DdbKvClient { /// Any snapshot, live allocation or free fact requires ordinary recovery. @@ -49,9 +50,9 @@ impl DdbKvClient { /// Rejects an empty or oversized batch and propagates ownership or write failures. pub async fn put_zone_batch(&self, bind: Bind, disk: &DiskId, values: &[(u32, ZoneValue)]) -> Result<()> { if values.is_empty() || values.len() > MAX_ZONE_INIT_BATCH { - return Err(crowdb_kv_client::Error::Server( - "zone initialization batch must contain 1..=128 records".into(), - )); + return Err(crowdb_paxos_kv_client::Error::Server(format!( + "zone initialization batch must contain 1..={MAX_ZONE_INIT_BATCH} records" + ))); } let ops: Vec<_> = values .iter() diff --git a/app/crowdb-diskdb/src/liveness/keepalive/observation.rs b/app/crowdb-diskdb/src/liveness/keepalive/observation.rs index ba36ef506..418cd4813 100644 --- a/app/crowdb-diskdb/src/liveness/keepalive/observation.rs +++ b/app/crowdb-diskdb/src/liveness/keepalive/observation.rs @@ -130,7 +130,7 @@ impl KeepAlive { &self, owner: &crowdb_protocol::DiskdbOwnerEntry, bind: (u64, u64), - ) -> crowdb_kv_client::Result<std::sync::Arc<crate::ddb_kv_client::OwnershipFence>> { + ) -> crowdb_paxos_kv_client::Result<std::sync::Arc<crate::ddb_kv_client::OwnershipFence>> { use crowdb_protocol::key::OwnerMapKey; let epoch = self.ensure_process_session().await?; let generation = self diff --git a/app/crowdb-diskdb/src/liveness/keepalive/online.rs b/app/crowdb-diskdb/src/liveness/keepalive/online.rs index 6543a0b38..8abd9a802 100644 --- a/app/crowdb-diskdb/src/liveness/keepalive/online.rs +++ b/app/crowdb-diskdb/src/liveness/keepalive/online.rs @@ -132,7 +132,7 @@ impl KeepAlive { // published before its strict background loader can expose allocations. group.remove_disk_from_memory(&old.disk_id); self.disk_add_init(group, old.disk_id, &disk); - Ok::<(), crowdb_kv_client::Error>(()) + Ok::<(), crowdb_paxos_kv_client::Error>(()) } .await; if let Err(error) = result { diff --git a/app/crowdb-diskdb/src/liveness/keepalive/scheduler.rs b/app/crowdb-diskdb/src/liveness/keepalive/scheduler.rs index 34e29e249..3016f28cc 100644 --- a/app/crowdb-diskdb/src/liveness/keepalive/scheduler.rs +++ b/app/crowdb-diskdb/src/liveness/keepalive/scheduler.rs @@ -12,7 +12,7 @@ use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering} use std::sync::{Arc, RwLock}; use crowdb_common::metrics::Counter; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::common::{DiskGroupUsageSummary, DiskId, HwStatus}; use crowdb_protocol::diskdb::rpc::DiskValue; use crowdb_protocol::{DiskGroupId, NodeId, RackId}; diff --git a/app/crowdb-diskdb/src/liveness/keepalive/session.rs b/app/crowdb-diskdb/src/liveness/keepalive/session.rs index 1c6d341e5..15f47b988 100644 --- a/app/crowdb-diskdb/src/liveness/keepalive/session.rs +++ b/app/crowdb-diskdb/src/liveness/keepalive/session.rs @@ -3,7 +3,7 @@ //! Freeze one predecessor before claiming a new per-process epoch. use super::{KeepAlive, Ordering}; -use crowdb_kv_client::{Error, Result}; +use crowdb_paxos_kv_client::{Error, Result}; use std::sync::Arc; pub(super) struct Predecessor { diff --git a/app/crowdb-diskdb/src/liveness/notify.rs b/app/crowdb-diskdb/src/liveness/notify.rs index 61980d442..b1d862a54 100644 --- a/app/crowdb-diskdb/src/liveness/notify.rs +++ b/app/crowdb-diskdb/src/liveness/notify.rs @@ -10,7 +10,7 @@ use tokio::sync::mpsc; use tracing::{info, warn}; use crate::bg_task::StopHandle; -use crowdb_kv_client::WatchNotifyClient; +use crowdb_paxos_kv_client::WatchNotifyClient; use crowdb_protocol::DISKDB_WATCH_PREFIXES; /// Group-0 store + group ids (system group). diff --git a/app/crowdb-diskdb/src/main.rs b/app/crowdb-diskdb/src/main.rs index e1e76bfcb..2863448f4 100644 --- a/app/crowdb-diskdb/src/main.rs +++ b/app/crowdb-diskdb/src/main.rs @@ -24,8 +24,8 @@ use crowdb_diskdb::recovery::ZoneLoader; use crowdb_diskdb::scanner::{BusyBlockOwnerScanner, ScanState, ScannerTask, SegmentOwnerQuery}; use crowdb_diskdb::service::DiskdbRpcService; use crowdb_diskdb_client::{DiskdbClient, DiskdbRpcTransport}; -use crowdb_kv_client::{ - ClientConfig, CrowdbKvClient, DomainMonitorClient, HardwareClient, +use crowdb_paxos_kv_client::{ + ClientConfig, DomainMonitorClient, HardwareClient, PaxosKvClient, RangeBindingClient as KvRangeBindingClient, ServiceRegistryClient, WatchNotifyClient, }; use crowdb_protocol::chunk_kv::{EnsureDomainMonitorOutcome, EnsureDomainMonitorRequest}; @@ -39,10 +39,6 @@ struct Cli { #[arg(long)] config: String, - /// crowdb-rpc listen address (overrides config). - #[arg(long)] - listen_addr: Option<String>, - /// HTTP management listen address (overrides config). #[arg(long)] http_addr: Option<String>, @@ -51,10 +47,6 @@ struct Cli { #[arg(long)] rpc_listen_addr: Option<String>, - /// Main listener port (overrides the port in config `listen_addr`). - #[arg(long, value_parser = clap::value_parser!(u16).range(1..))] - listen_port: Option<u16>, - /// HTTP management port (overrides the port in config `http_listen_addr`). #[arg(long, value_parser = clap::value_parser!(u16).range(1..))] http_port: Option<u16>, @@ -198,7 +190,7 @@ async fn main() { // leader and data-group leaders are discovered lazily via the // kv-server HTTP management API seeds from config — no pre-seeding. // One client is shared across all service classes (hardware, - // service-registry, data-group) since `CrowdbKvClient` is fully + // service-registry, data-group) since `PaxosKvClient` is fully // interior-mutable; each service class takes it via `from_shared`. let kv_config = { let loaded = config.load(); @@ -207,7 +199,7 @@ async fn main() { client.rpc_workers = loaded.server.kv_rpc_workers; client }; - let kv_client = Arc::new(CrowdbKvClient::new(kv_config)); + let kv_client = Arc::new(PaxosKvClient::new(kv_config)); let (sys_store, sys_group) = kv_client.system_group(); info!( store_id = sys_store, @@ -323,14 +315,6 @@ async fn main() { let relocation_worker = Arc::new( RelocationWorker::new(owner_client.clone(), relocation_io).with_source_free(relocation_source_free), ); - let listen_addr: SocketAddr = config - .load() - .server - .listen_addr - .parse() - .expect("valid listen_addr"); - info!(%listen_addr, "RPC server listening (zone load pending)"); - // Build the crowdb-rpc server. The RpcServer listens on the RPC // port and dispatches to DiskdbRpcService handlers. let rpc_listen_addr: SocketAddr = config @@ -616,18 +600,12 @@ fn load_config(args: &Cli) -> DdbConfig { panic!("failed to load config file {config_path}: {e}"); }); - if let Some(addr) = &args.listen_addr { - config.server.listen_addr.clone_from(addr); - } if let Some(addr) = &args.http_addr { config.server.http_listen_addr.clone_from(addr); } if let Some(addr) = &args.rpc_listen_addr { config.server.rpc_listen_addr.clone_from(addr); } - if let Some(port) = args.listen_port { - config.server.listen_addr = replace_port(&config.server.listen_addr, port); - } if let Some(port) = args.http_port { config.server.http_listen_addr = replace_port(&config.server.http_listen_addr, port); } @@ -693,7 +671,7 @@ fn resolve_instance_id(config: &arc_swap::ArcSwap<DdbConfig>) -> u64 { } // 3. Generate new and persist. - let id = crowdb_kv_client::new_client_id(); + let id = crowdb_paxos_kv_client::new_client_id(); if let Err(e) = std::fs::create_dir_all(state_dir) { warn!(error = %e, "failed to create state dir; instance_id will not persist across restarts"); } else if let Err(e) = std::fs::write(&state_file, id.to_string()) { diff --git a/app/crowdb-diskdb/src/model/alloc.rs b/app/crowdb-diskdb/src/model/alloc.rs index 5b9187716..92868352a 100644 --- a/app/crowdb-diskdb/src/model/alloc.rs +++ b/app/crowdb-diskdb/src/model/alloc.rs @@ -35,7 +35,7 @@ fn matching_tentative(dg: &DdbDiskGroup, segment: &Segment, disk_id: DiskId) -> #[derive(Clone, Debug)] pub enum FreeError { /// KV client error during lookup or persist. - Kv(Arc<crowdb_kv_client::Error>), + Kv(Arc<crowdb_paxos_kv_client::Error>), /// Block is not busy (no `BusyBlockKey` exists) — double-free or /// never allocated. NotBusy { @@ -71,10 +71,10 @@ impl std::fmt::Display for FreeError { impl std::error::Error for FreeError {} -impl From<crowdb_kv_client::Error> for FreeError { - fn from(e: crowdb_kv_client::Error) -> Self { +impl From<crowdb_paxos_kv_client::Error> for FreeError { + fn from(e: crowdb_paxos_kv_client::Error) -> Self { match e { - crowdb_kv_client::Error::OutcomeUnknown => Self::OutcomeUnknown, + crowdb_paxos_kv_client::Error::OutcomeUnknown => Self::OutcomeUnknown, error => Self::Kv(Arc::new(error)), } } @@ -389,7 +389,7 @@ pub async fn free_block( let scoped_kv = kv.for_group(dg); let kv = &scoped_kv; let disk_id = segment.disk_id.ok_or_else(|| { - FreeError::Kv(Arc::new(crowdb_kv_client::Error::SysdataDecode { + FreeError::Kv(Arc::new(crowdb_paxos_kv_client::Error::SysdataDecode { key: "segment.disk_id".to_string(), reason: "missing disk_id in Segment".to_string(), })) @@ -529,7 +529,7 @@ pub fn prepare_free( let mut records = Vec::with_capacity(unique.len()); for segment in &unique { let disk_id = segment.disk_id.ok_or_else(|| { - FreeError::Kv(Arc::new(crowdb_kv_client::Error::SysdataDecode { + FreeError::Kv(Arc::new(crowdb_paxos_kv_client::Error::SysdataDecode { key: "segment.disk_id".to_string(), reason: "missing disk_id in Segment".to_string(), })) @@ -630,7 +630,7 @@ pub async fn commit_blocks( let mut unique = Vec::with_capacity(segments.len()); for seg in segments { let disk_id = seg.disk_id.ok_or_else(|| { - FreeError::Kv(Arc::new(crowdb_kv_client::Error::SysdataDecode { + FreeError::Kv(Arc::new(crowdb_paxos_kv_client::Error::SysdataDecode { key: "segment.disk_id".to_string(), reason: "missing disk_id in Segment".to_string(), })) @@ -699,7 +699,7 @@ pub async fn mark_blocks_corrupt( let mut marked = 0u32; for segment in segments { let disk_id = segment.disk_id.ok_or_else(|| { - FreeError::Kv(Arc::new(crowdb_kv_client::Error::SysdataDecode { + FreeError::Kv(Arc::new(crowdb_paxos_kv_client::Error::SysdataDecode { key: "segment.disk_id".into(), reason: "missing disk ID".into(), })) diff --git a/app/crowdb-diskdb/src/rebalance/relocation.rs b/app/crowdb-diskdb/src/rebalance/relocation.rs index a36b393e3..616399922 100644 --- a/app/crowdb-diskdb/src/rebalance/relocation.rs +++ b/app/crowdb-diskdb/src/rebalance/relocation.rs @@ -12,7 +12,7 @@ use bytes::Bytes; use crowdb_chunkdb_client::ChunkdbClient; use crowdb_diskdb_client::DiskdbClient; use crowdb_diskio_client::{DiskId as IoDiskId, DiskioClient, DiskioClientConfig, Durability, SegmentTarget}; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::chunk_task::relocation_operation_id; use crowdb_protocol::chunkdb::rpc::{RelocateSegmentHandoffRequest, RelocationHandoffDisposition}; use crowdb_protocol::common::DiskId; diff --git a/app/crowdb-diskdb/src/recovery.rs b/app/crowdb-diskdb/src/recovery.rs index a17bb93fc..49aa9b54f 100644 --- a/app/crowdb-diskdb/src/recovery.rs +++ b/app/crowdb-diskdb/src/recovery.rs @@ -45,7 +45,7 @@ pub use full_scan::rebuild_zone_bitmap_full_scan; #[derive(Debug)] pub enum ZoneLoadError { /// KV client error. - Kv(crowdb_kv_client::Error), + Kv(crowdb_paxos_kv_client::Error), /// A `JournalScan` returned `KV_ERROR_JOURNAL_SCAN_GC_GAP` — slots /// already GC'd below the WAL trim point. The caller should fall /// back to strategy 1 (full scan) for this zone. @@ -70,8 +70,8 @@ impl std::fmt::Display for ZoneLoadError { impl std::error::Error for ZoneLoadError {} -impl From<crowdb_kv_client::Error> for ZoneLoadError { - fn from(e: crowdb_kv_client::Error) -> Self { +impl From<crowdb_paxos_kv_client::Error> for ZoneLoadError { + fn from(e: crowdb_paxos_kv_client::Error) -> Self { Self::Kv(e) } } diff --git a/app/crowdb-diskdb/src/recovery/disk_recovery.rs b/app/crowdb-diskdb/src/recovery/disk_recovery.rs index 8174a2eed..17856edbb 100644 --- a/app/crowdb-diskdb/src/recovery/disk_recovery.rs +++ b/app/crowdb-diskdb/src/recovery/disk_recovery.rs @@ -291,7 +291,7 @@ impl RecoveryScanTask { /// Scan one zone: read zone records, extract live busy blocks, /// call the placeholder recovery function. Returns the count of /// impacted blocks in this zone. - async fn scan_zone(&self, zone_index: u32) -> Result<u32, crowdb_kv_client::Error> { + async fn scan_zone(&self, zone_index: u32) -> Result<u32, crowdb_paxos_kv_client::Error> { let records = self .kv .read_zone_records(self.bind, &self.disk.disk_id, zone_index) diff --git a/app/crowdb-diskdb/src/recovery/journal_replay.rs b/app/crowdb-diskdb/src/recovery/journal_replay.rs index b49e3e6b4..ed0de7271 100644 --- a/app/crowdb-diskdb/src/recovery/journal_replay.rs +++ b/app/crowdb-diskdb/src/recovery/journal_replay.rs @@ -68,7 +68,7 @@ pub async fn load_zone_inner( let busy_ops = kv.journal_scan_busy(bind, min_slot, 0, &disk_id, zone_idx).await; let busy_ops = match busy_ops { Ok(ops) => ops, - Err(crowdb_kv_client::Error::JournalScanGcGap) => { + Err(crowdb_paxos_kv_client::Error::JournalScanGcGap) => { return Err(ZoneLoadError::JournalScanGcGap); } Err(e) => return Err(ZoneLoadError::Kv(e)), @@ -120,7 +120,7 @@ pub async fn load_zone_inner( let free_ops = kv.journal_scan_free(bind, min_slot, 0, &disk_id, zone_idx).await; let free_ops = match free_ops { Ok(ops) => ops, - Err(crowdb_kv_client::Error::JournalScanGcGap) => { + Err(crowdb_paxos_kv_client::Error::JournalScanGcGap) => { return Err(ZoneLoadError::JournalScanGcGap); } Err(e) => return Err(ZoneLoadError::Kv(e)), diff --git a/app/crowdb-diskdb/src/scanner/integrity.rs b/app/crowdb-diskdb/src/scanner/integrity.rs index d458c89fe..37d8eea07 100644 --- a/app/crowdb-diskdb/src/scanner/integrity.rs +++ b/app/crowdb-diskdb/src/scanner/integrity.rs @@ -164,13 +164,13 @@ async fn scan_zone_records( // ZoneValue. let zone_key = crowdb_protocol::key::ZoneKey { disk_id, zone_index }; let zone_bytes = zone_key.to_bytes(); - if let Ok(crowdb_kv_client::GetOutcome::Found { value, .. }) = kv + if let Ok(crowdb_paxos_kv_client::GetOutcome::Found { value, .. }) = kv .kv() .get( store_id, group_id, &zone_bytes, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await @@ -203,7 +203,7 @@ async fn scan_zone_records( &[], &[], 0, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, false, None, @@ -230,7 +230,7 @@ async fn scan_zone_records( &[], &[], 0, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, false, None, diff --git a/app/crowdb-diskdb/tests/bg_task_test.rs b/app/crowdb-diskdb/tests/bg_task_test.rs index b59b6616c..aa2395ea8 100644 --- a/app/crowdb-diskdb/tests/bg_task_test.rs +++ b/app/crowdb-diskdb/tests/bg_task_test.rs @@ -63,8 +63,8 @@ impl BackgroundTask for MockTask { fn make_ctx() -> Arc<BgCtx> { let container = Arc::new(DdbDiskGroupContainer::new(1)); - let kv = Arc::new(DdbKvClient::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(vec!["127.0.0.1:0".to_string()]), + let kv = Arc::new(DdbKvClient::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(vec!["127.0.0.1:0".to_string()]), ))); let mut registry = crowdb_common::metrics::MetricsRegistry::new(); let metrics = DiskdbMetrics::register(&mut registry); diff --git a/app/crowdb-diskdb/tests/common/journal_recovery.rs b/app/crowdb-diskdb/tests/common/journal_recovery.rs index 08b481885..6f69ea64a 100644 --- a/app/crowdb-diskdb/tests/common/journal_recovery.rs +++ b/app/crowdb-diskdb/tests/common/journal_recovery.rs @@ -3,10 +3,10 @@ use bytes::Bytes; use crowdb_diskdb::model::zone::DdbZone; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::kv::{Batch, BatchOp, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, Op}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::{index::SegmentIndex, record::WALRecord, replay::ReplayResult}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{Batch, BatchOp, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, Op}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::{index::SegmentIndex, record::WALRecord, replay::ReplayResult}; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::BusyBlockValue; use crowdb_protocol::key::{BinaryKey, BusyBlockKey, ZoneKey}; diff --git a/app/crowdb-diskdb/tests/common/metadata.rs b/app/crowdb-diskdb/tests/common/metadata.rs index 20a1668ef..06b98dd61 100644 --- a/app/crowdb-diskdb/tests/common/metadata.rs +++ b/app/crowdb-diskdb/tests/common/metadata.rs @@ -5,10 +5,10 @@ //! Election, server packaging and persisted KV recovery belong to container E2E. use crowdb_diskdb::ddb_kv_client::DdbKvClient; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -40,8 +40,8 @@ impl TestMetadata { } } - fn client(&self) -> CrowdbKvClient { - let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + fn client(&self) -> PaxosKvClient { + let client = PaxosKvClient::new(ClientConfig::new(vec![])); client.seed_leader(0, 0, self.group0_leader_endpoint.clone()); client.seed_leader(0, 1, self.group1_leader_endpoint.clone()); client diff --git a/app/crowdb-diskdb/tests/common/online.rs b/app/crowdb-diskdb/tests/common/online.rs index d6cb4fa36..c3f031c13 100644 --- a/app/crowdb-diskdb/tests/common/online.rs +++ b/app/crowdb-diskdb/tests/common/online.rs @@ -7,7 +7,7 @@ use crowdb_diskdb::{ liveness::keepalive::KeepAlive, model::{disk::DdbDisk, disk_group_container::DdbDiskGroupContainer}, }; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::{ common::{DiskId, DiskioHardwareScope, HwStatus, NodeValue, RackValue}, disk_storage::{DiskExclusionRecord, DiskExclusionReporter, DiskOnlineState, DiskStorageRecord}, diff --git a/app/crowdb-diskdb/tests/config_reload_test.rs b/app/crowdb-diskdb/tests/config_reload_test.rs index 95d3abc3a..7411e9d15 100644 --- a/app/crowdb-diskdb/tests/config_reload_test.rs +++ b/app/crowdb-diskdb/tests/config_reload_test.rs @@ -17,7 +17,7 @@ use crowdb_diskdb::liveness::keepalive::KeepAlive; use crowdb_diskdb::metrics::DiskdbMetrics; use crowdb_diskdb::model::disk_group_container::DdbDiskGroupContainer; use crowdb_diskdb::recovery::compaction::CompactionEngine; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; fn make_config_handle(interval_secs: u32) -> Arc<ArcSwap<DdbConfig>> { let mut config = DdbConfig::default(); @@ -28,7 +28,7 @@ fn make_config_handle(interval_secs: u32) -> Arc<ArcSwap<DdbConfig>> { } fn make_keepalive(handle: &Arc<ArcSwap<DdbConfig>>) -> KeepAlive { - let kv_client = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let kv_client = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![ "127.0.0.1:0".to_string() ]))); let hw = HardwareClient::from_shared(Arc::clone(&kv_client)); @@ -66,7 +66,7 @@ fn keepalive_trigger_uses_timer_fn_when_config_handle_set() { #[test] fn keepalive_trigger_uses_fixed_timer_without_config_handle() { - let kv_client = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let kv_client = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![ "127.0.0.1:0".to_string() ]))); let hw = HardwareClient::from_shared(Arc::clone(&kv_client)); @@ -89,7 +89,7 @@ fn keepalive_trigger_uses_fixed_timer_without_config_handle() { #[test] fn compaction_engine_trigger_uses_timer_fn_when_config_handle_set() { let handle = make_config_handle(30); - let kv_client = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let kv_client = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![ "127.0.0.1:0".to_string() ]))); let ddb_kv = Arc::new(DdbKvClient::from_shared(kv_client)); @@ -113,7 +113,7 @@ fn compaction_engine_trigger_uses_timer_fn_when_config_handle_set() { #[test] fn compaction_engine_trigger_uses_fixed_timer_without_config_handle() { - let kv_client = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let kv_client = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![ "127.0.0.1:0".to_string() ]))); let ddb_kv = Arc::new(DdbKvClient::from_shared(kv_client)); @@ -136,7 +136,7 @@ async fn compaction_engine_run_cycle_reads_threshold_from_ctx_config() { config.persistence.snapshot_compaction_threshold = 7777; handle.store(Arc::new(config)); - let kv_client = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let kv_client = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![ "127.0.0.1:0".to_string() ]))); let ddb_kv = Arc::new(DdbKvClient::from_shared(kv_client)); diff --git a/app/crowdb-diskdb/tests/ddb_config_test.rs b/app/crowdb-diskdb/tests/ddb_config_test.rs index 1678c8521..3c90135f1 100644 --- a/app/crowdb-diskdb/tests/ddb_config_test.rs +++ b/app/crowdb-diskdb/tests/ddb_config_test.rs @@ -46,9 +46,9 @@ fn config_validate_rejects_granularity_not_equal_to_block() { } #[test] -fn config_validate_rejects_bad_listen_addr() { +fn config_validate_rejects_bad_rpc_listen_addr() { let mut config = DdbConfig::default(); - config.server.listen_addr = "not-an-addr".to_string(); + config.server.rpc_listen_addr = "not-an-addr".to_string(); assert!(validate(&config).is_err()); } @@ -93,14 +93,13 @@ fn config_defaults_match_design() { #[test] fn partial_persistence_config_keeps_defaults() { let config: DdbConfig = toml::from_str( - r#" + r" [server] - listen_addr = "127.0.0.1:1" [persistence] free_batch_enabled = true free_flush_max_batch = 32 - "#, + ", ) .unwrap(); assert!(config.persistence.free_batch_enabled); @@ -114,7 +113,6 @@ fn allocator_weight_policy_parses_and_keeps_other_defaults() { let config: DdbConfig = toml::from_str( r#" [server] - listen_addr = "127.0.0.1:1" [allocator] load_aware_weight = "inverse_used_pct" @@ -198,7 +196,6 @@ fn config_load_from_file_roundtrip() { std::fs::write(&tmp, &toml).expect("write temp"); let loaded = crowdb_common::config::load_from_file::<DdbConfig>(&tmp).expect("load"); assert_eq!(loaded.storage.block_size_bytes, config.storage.block_size_bytes); - assert_eq!(loaded.server.listen_addr, config.server.listen_addr); let _ = std::fs::remove_file(&tmp); } @@ -209,7 +206,6 @@ fn config_load_from_file_roundtrip() { #[test] fn minimal_server_only_config_uses_section_defaults() { let toml = "[server]\n\ - listen_addr = \"0.0.0.0:11000\"\n\ http_listen_addr = \"0.0.0.0:11100\"\n\ rpc_listen_addr = \"0.0.0.0:11200\"\n\ kv_server_mgmt_seeds = [\"http://127.0.0.1:10000\"]\n"; diff --git a/app/crowdb-diskdb/tests/disk_initialization_test.rs b/app/crowdb-diskdb/tests/disk_initialization_test.rs index 4dc6e9a13..6793cf923 100644 --- a/app/crowdb-diskdb/tests/disk_initialization_test.rs +++ b/app/crowdb-diskdb/tests/disk_initialization_test.rs @@ -149,7 +149,10 @@ async fn atomic_multi_disk_rejection_does_not_poison_unsubmitted_writes() { assert!(second.write_gate.quiescent()); } -async fn prepare_hardware(metadata: &TestMetadata, status: HwStatus) -> crowdb_kv_client::HardwareClient { +async fn prepare_hardware( + metadata: &TestMetadata, + status: HwStatus, +) -> crowdb_paxos_kv_client::HardwareClient { let hw = metadata.make_hardware_client(); let id = DiskId { high: 4, low: 5 }; hw.add_rack( diff --git a/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs b/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs index bf46047f1..954d501e7 100644 --- a/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs +++ b/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs @@ -2,10 +2,10 @@ // Licensed under the Apache License, Version 2.0. use crowdb_diskdb::{ddb_kv_client::DdbKvClient, recovery::ZoneLoader}; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::common::{DiskId, HwStatus}; use crowdb_protocol::diskdb::rpc::DiskValue; use std::sync::Arc; @@ -24,7 +24,7 @@ async fn restart_with_reclaimed_journal_rebuilds_busy_units_before_allocating() )); store.add_group(PxGroup::new(1, restored)); store.start().await.unwrap(); - let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + let client = PaxosKvClient::new(ClientConfig::new(vec![])); let endpoint = store.listen_addr().unwrap().to_string(); for group in [0, 1] { client.seed_leader(0, group, endpoint.clone()); diff --git a/app/crowdb-diskdb/tests/keepalive_sync_test.rs b/app/crowdb-diskdb/tests/keepalive_sync_test.rs index c1793247c..fd1811fd3 100644 --- a/app/crowdb-diskdb/tests/keepalive_sync_test.rs +++ b/app/crowdb-diskdb/tests/keepalive_sync_test.rs @@ -32,7 +32,10 @@ async fn owner_without_bind_does_not_publish_disk_group() { .set_owner(RACK_ID, NODE_ID, DG_ID, INSTANCE_ID + 1, u64::MAX) .await; assert!( - matches!(replacement, Err(crowdb_kv_client::Error::OwnerConflict { .. })), + matches!( + replacement, + Err(crowdb_paxos_kv_client::Error::OwnerConflict { .. }) + ), "different owner must be rejected" ); diff --git a/app/crowdb-diskdb/tests/metadata_flow_test.rs b/app/crowdb-diskdb/tests/metadata_flow_test.rs index ad2bb5ae4..663ed3d8d 100644 --- a/app/crowdb-diskdb/tests/metadata_flow_test.rs +++ b/app/crowdb-diskdb/tests/metadata_flow_test.rs @@ -19,7 +19,7 @@ use crowdb_diskdb::ddb_kv_client::DdbKvClient; use crowdb_diskdb::liveness::keepalive::KeepAlive; use crowdb_diskdb::model::alloc; use crowdb_diskdb::model::disk_group_container::DdbDiskGroupContainer; -use crowdb_kv_client::{GetOutcome, HardwareClient}; +use crowdb_paxos_kv_client::{GetOutcome, HardwareClient}; use crowdb_protocol::common::{ChunkId, DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{BlockState, CommitState, DiskGroupValue, DiskType, DiskValue}; use crowdb_protocol::key::{BinaryKey, BusyBlockKey, FreeBlockKey}; @@ -56,7 +56,7 @@ async fn kv_get(kv: &DdbKvClient, key: &[u8]) -> Option<bytes::Bytes> { STORE_ID, DATA_GROUP_ID, key, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await diff --git a/app/crowdb-diskdb/tests/ownership_fence_test.rs b/app/crowdb-diskdb/tests/ownership_fence_test.rs index 52371fa63..eccc708da 100644 --- a/app/crowdb-diskdb/tests/ownership_fence_test.rs +++ b/app/crowdb-diskdb/tests/ownership_fence_test.rs @@ -15,7 +15,7 @@ use std::sync::Arc; #[tokio::test] async fn stale_zone_loader_cannot_mark_a_successor_disk_offline() { - use crowdb_kv_client::ReadMode; + use crowdb_paxos_kv_client::ReadMode; use crowdb_protocol::{ common::{HwStatus, OwnerMapValue}, key::{OwnerMapKey, TextKey}, @@ -62,7 +62,7 @@ async fn stale_zone_loader_cannot_mark_a_successor_disk_offline() { HwStatus::Offline as i32 ); hardware.add_disk(1, 1, 1, &disk, &online).await.unwrap(); - let crowdb_kv_client::GetOutcome::Found { revision, .. } = kv + let crowdb_paxos_kv_client::GetOutcome::Found { revision, .. } = kv .kv() .get(0, 0, owner_key.as_bytes(), ReadMode::Linearizable, None) .await @@ -174,11 +174,15 @@ async fn concurrent_disk_initialization_persists_every_zone_under_one_owner() { ) }) .collect(); - assert!(scoped.put_zone_batch((0, 1), &disk, &records).await.is_err()); + let limit = crowdb_diskdb::ddb_kv_client::MAX_ZONE_INIT_BATCH; + assert!(scoped + .put_zone_batch((0, 1), &disk, &records[..=limit]) + .await + .is_err()); for batch in records.chunks(crowdb_diskdb::ddb_kv_client::MAX_ZONE_INIT_BATCH) { scoped.put_zone_batch((0, 1), &disk, batch).await?; } - Ok::<_, crowdb_kv_client::Error>(disk) + Ok::<_, crowdb_paxos_kv_client::Error>(disk) } })) .await; @@ -195,12 +199,12 @@ async fn concurrent_disk_initialization_persists_every_zone_under_one_owner() { 0, 1, &key.to_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await .unwrap(); - let crowdb_kv_client::GetOutcome::Found { value, .. } = record else { + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = record else { panic!("missing zone baseline") }; let snapshot = crowdb_protocol::diskdb::rpc::ZoneValue::from_bytes(&value).unwrap(); @@ -213,7 +217,7 @@ async fn concurrent_disk_initialization_persists_every_zone_under_one_owner() { #[tokio::test] async fn ordinary_writes_cannot_bypass_ownership_handover() { use bytes::Bytes; - use crowdb_kv_client::{BatchOp, GetOutcome, ReadMode}; + use crowdb_paxos_kv_client::{BatchOp, GetOutcome, ReadMode}; let cluster = TestMetadata::start().await; let ddb = cluster.make_ddb_kv_client(); let kv = ddb.kv(); diff --git a/app/crowdb-diskdb/tests/query_handler_test.rs b/app/crowdb-diskdb/tests/query_handler_test.rs index 54ec58f49..4d9bec0b6 100644 --- a/app/crowdb-diskdb/tests/query_handler_test.rs +++ b/app/crowdb-diskdb/tests/query_handler_test.rs @@ -194,13 +194,13 @@ async fn disk_rpc_reports_effective_status() { use crowdb_diskdb::scanner::ScanState; use crowdb_diskdb::service::DiskdbRpcService; use crowdb_diskdb_client::rpc_transport::DiskdbRpcTransport; - use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; + use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; let dg = make_dg(); let disk = dg.get_disk(disk_id(1)).unwrap(); let container = Arc::new(DdbDiskGroupContainer::new(1)); container.replace_disk_group(&dg); - let kv = Arc::new(DdbKvClient::new(CrowdbKvClient::new(ClientConfig::new(vec![])))); + let kv = Arc::new(DdbKvClient::new(PaxosKvClient::new(ClientConfig::new(vec![])))); let service = Arc::new(DiskdbRpcService::new( container.clone(), kv.clone(), diff --git a/app/crowdb-diskdb/tests/recovery_failure_test.rs b/app/crowdb-diskdb/tests/recovery_failure_test.rs index 92c7845d1..74d0dad92 100644 --- a/app/crowdb-diskdb/tests/recovery_failure_test.rs +++ b/app/crowdb-diskdb/tests/recovery_failure_test.rs @@ -6,7 +6,7 @@ use std::time::Duration; use crowdb_diskdb::ddb_kv_client::DdbKvClient; use crowdb_diskdb::recovery::ZoneLoader; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, RetryConfig}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, RetryConfig}; use crowdb_protocol::common::DiskId; use crowdb_protocol::diskdb::rpc::{DiskType, DiskValue}; @@ -19,8 +19,8 @@ async fn failed_recovery_does_not_return_writable_disk_group() { backoff_base: Duration::from_millis(10), backoff_max: Duration::from_millis(10), }; - let kv = CrowdbKvClient::new(config); - let port = crowdb_protocol::port::alloc::alloc_test_port(crowdb_protocol::ServicePort::KvServerListen); + let kv = PaxosKvClient::new(config); + let port = crowdb_protocol::port::alloc::alloc_test_port(crowdb_protocol::ServicePort::PaxosKvServerRpc); kv.seed_leader(0, 999, format!("127.0.0.1:{port}")); let loader = ZoneLoader::new(Arc::new(DdbKvClient::new(kv)), 1); let disk = DiskValue { diff --git a/app/crowdb-diskdb/tests/recovery_test.rs b/app/crowdb-diskdb/tests/recovery_test.rs index 6017629dc..6c22ddea6 100644 --- a/app/crowdb-diskdb/tests/recovery_test.rs +++ b/app/crowdb-diskdb/tests/recovery_test.rs @@ -25,7 +25,7 @@ use crowdb_diskdb::model::disk_group_container::DdbDiskGroupContainer; use crowdb_diskdb::model::zone::DdbZone; use crowdb_diskdb::recovery::compaction::CompactionEngine; use crowdb_diskdb::recovery::ZoneLoader; -use crowdb_kv_client::{GetOutcome, HardwareClient}; +use crowdb_paxos_kv_client::{GetOutcome, HardwareClient}; use crowdb_protocol::common::{ChunkId, DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{DiskGroupValue, DiskType, DiskValue}; use crowdb_protocol::key::BinaryKey; @@ -639,7 +639,7 @@ async fn compaction_compact_zone_writes_snapshot_and_deletes_free_records() { STORE_ID, DATA_GROUP_ID, &free_key.to_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await @@ -666,7 +666,7 @@ async fn compaction_compact_zone_writes_snapshot_and_deletes_free_records() { STORE_ID, DATA_GROUP_ID, &free_key.to_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await @@ -781,7 +781,7 @@ async fn compaction_watermark_prevents_double_free_after_crashed_compaction() { STORE_ID, DATA_GROUP_ID, &free_key.to_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await @@ -887,7 +887,7 @@ async fn compaction_watermark_prevents_double_free_after_crashed_compaction() { STORE_ID, DATA_GROUP_ID, &free_key.to_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await diff --git a/app/crowdb-diskdb/tests/relocation_journal_test.rs b/app/crowdb-diskdb/tests/relocation_journal_test.rs index 87ae11cf9..073c95d3c 100644 --- a/app/crowdb-diskdb/tests/relocation_journal_test.rs +++ b/app/crowdb-diskdb/tests/relocation_journal_test.rs @@ -26,7 +26,7 @@ use crowdb_diskdb::recovery::ZoneLoader; use crowdb_diskdb::scanner::ScanState; use crowdb_diskdb::service::DiskdbRpcService; use crowdb_diskdb_client::DiskdbRpcTransport; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::chunkdb::rpc::{RelocateSegmentHandoffRequest, RelocationHandoffDisposition}; use crowdb_protocol::common::{ChunkId, DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{ diff --git a/app/crowdb-diskdb/tests/scanner_test.rs b/app/crowdb-diskdb/tests/scanner_test.rs index 1476935a1..f7479f954 100644 --- a/app/crowdb-diskdb/tests/scanner_test.rs +++ b/app/crowdb-diskdb/tests/scanner_test.rs @@ -31,7 +31,7 @@ use crowdb_diskdb::model::disk_group_container::DdbDiskGroupContainer; use crowdb_diskdb::scanner::ghost::diff_bitmaps; use crowdb_diskdb::scanner::integrity::{is_zero_chunk, is_zero_owner}; use crowdb_diskdb::scanner::{BusyBlockOwnerScanner, ScanState, ScanSummary, SegmentOwnerQuery, ZonePacer}; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::chunkdb::rpc::{QuerySegmentOwnerRequest, SegmentOwnerDisposition}; use crowdb_protocol::common::{ChunkId, DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{BusyBlockValue, CommitState, DiskGroupValue, DiskType, DiskValue}; diff --git a/app/crowdb-diskdb/tests/startup_sync_test.rs b/app/crowdb-diskdb/tests/startup_sync_test.rs index ac2ca9176..d0b14c10d 100644 --- a/app/crowdb-diskdb/tests/startup_sync_test.rs +++ b/app/crowdb-diskdb/tests/startup_sync_test.rs @@ -34,7 +34,7 @@ fn recovered_group_replaces_only_the_retained_owner_generation() { #[tokio::test] async fn startup_reconstruction_keeps_the_original_administrative_write_gate() { use crowdb_diskdb::{ddb_kv_client::DdbKvClient, model::disk::DdbDisk, recovery::ZoneLoader}; - use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; + use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::{ common::{DiskId, HwStatus}, diskdb::rpc::DiskValue, @@ -51,7 +51,7 @@ async fn startup_reconstruction_keeps_the_original_administrative_write_gate() { let group = DdbDiskGroup::new(3, 2, 1); let disk = Arc::new(DdbDisk::new(id, 3, 2, 1, value.clone())); group.add_disk(disk.clone()); - let scoped = DdbKvClient::new(CrowdbKvClient::new(ClientConfig::new(vec![]))).for_group(&group); + let scoped = DdbKvClient::new(PaxosKvClient::new(ClientConfig::new(vec![]))).for_group(&group); let loader = ZoneLoader::new(Arc::new(scoped), 1); // The loader captured an Up value before exclusion, but it must neither // manufacture an empty baseline nor publish a new writable incarnation. diff --git a/app/crowdb-diskio/CMakeLists.txt b/app/crowdb-diskio/CMakeLists.txt index 534fc83b6..c94d17f1c 100644 --- a/app/crowdb-diskio/CMakeLists.txt +++ b/app/crowdb-diskio/CMakeLists.txt @@ -19,11 +19,11 @@ add_compile_options(-Wall -Wextra -Werror) # add crowdb-common separately here (avoids duplicate target collision). add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../lib/crowdb-rpc crowdb-rpc-build) -# ── crowdb-kv-client FFI (Rust cdylib with --features ffi) ────────── -# crowdb-kv-client produces both an rlib (for Rust dependents) and a +# ── crowdb-paxos-kv-client FFI (Rust cdylib with --features ffi) ────────── +# crowdb-paxos-kv-client produces both an rlib (for Rust dependents) and a # cdylib (for C++ consumers). The cdylib exports C ABI symbols only # when built with the `ffi` feature. -set(CROWDB_KV_CLIENT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/crowdb-kv-client) +set(CROWDB_PAXOS_KV_CLIENT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/crowdb-paxos-kv-client) set(CROWDB_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../..) if(CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") set(CARGO_PROFILE release) @@ -31,16 +31,16 @@ else() set(CARGO_PROFILE debug) endif() if(APPLE) - set(CROWDB_KV_CLIENT_FFI_LIB ${CROWDB_ROOT_DIR}/target/${CARGO_PROFILE}/libcrowdb_kv_client.dylib) + set(CROWDB_PAXOS_KV_CLIENT_FFI_LIB ${CROWDB_ROOT_DIR}/target/${CARGO_PROFILE}/libcrowdb_paxos_kv_client.dylib) else() - set(CROWDB_KV_CLIENT_FFI_LIB ${CROWDB_ROOT_DIR}/target/${CARGO_PROFILE}/libcrowdb_kv_client.so) + set(CROWDB_PAXOS_KV_CLIENT_FFI_LIB ${CROWDB_ROOT_DIR}/target/${CARGO_PROFILE}/libcrowdb_paxos_kv_client.so) endif() -add_custom_target(crowdb-kv-client-ffi - BYPRODUCTS ${CROWDB_KV_CLIENT_FFI_LIB} - COMMAND pixi run cargo build -p crowdb-kv-client --features ffi +add_custom_target(crowdb-paxos-kv-client-ffi + BYPRODUCTS ${CROWDB_PAXOS_KV_CLIENT_FFI_LIB} + COMMAND pixi run cargo build -p crowdb-paxos-kv-client --features ffi $<$<STREQUAL:${CARGO_PROFILE},release>:--release> WORKING_DIRECTORY ${CROWDB_ROOT_DIR} - COMMENT "Building crowdb-kv-client (Rust cdylib with ffi feature)" + COMMENT "Building crowdb-paxos-kv-client (Rust cdylib with ffi feature)" VERBATIM ) @@ -53,7 +53,7 @@ target_include_directories(crowdb-diskio-lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/crowdb-common/cpp/include ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/crowdb-rpc/include - ${CROWDB_KV_CLIENT_DIR}/include + ${CROWDB_PAXOS_KV_CLIENT_DIR}/include PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/crowdb-rpc-build/generated ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/crowdb-protocol/cpp/include ) @@ -67,8 +67,8 @@ target_include_directories(crowdb-diskio PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}/crowdb-rpc-build/generated ) -target_link_libraries(crowdb-diskio PRIVATE crowdb-diskio-lib ${CROWDB_KV_CLIENT_FFI_LIB} ${CMAKE_DL_LIBS}) -add_dependencies(crowdb-diskio crowdb-kv-client-ffi) +target_link_libraries(crowdb-diskio PRIVATE crowdb-diskio-lib ${CROWDB_PAXOS_KV_CLIENT_FFI_LIB} ${CMAKE_DL_LIBS}) +add_dependencies(crowdb-diskio crowdb-paxos-kv-client-ffi) set_target_properties(crowdb-diskio PROPERTIES BUILD_RPATH ${CROWDB_ROOT_DIR}/target/${CARGO_PROFILE} ) diff --git a/app/crowdb-diskio/src/dio_config.cpp b/app/crowdb-diskio/src/dio_config.cpp index d81196471..fe04c341c 100644 --- a/app/crowdb-diskio/src/dio_config.cpp +++ b/app/crowdb-diskio/src/dio_config.cpp @@ -282,7 +282,7 @@ bool DioConfig::parse_args(int argc, char *argv[], DioConfig &out, std::string & } else if (arg == "--kv-seeds" && i + 1 < argc) { // Comma-separated list of kv-server management endpoints. - // e.g. --kv-seeds http://127.0.0.1:10000,http://127.0.0.1:10001 + // e.g. --kv-seeds http://127.0.0.1:10135,http://127.0.0.1:10136 std::string seeds = argv[++i]; if (!cli_kv_seeds) { out.kv_seeds.clear(); diff --git a/app/crowdb-diskio/src/dio_config.h b/app/crowdb-diskio/src/dio_config.h index d582a9907..dad7ba995 100644 --- a/app/crowdb-diskio/src/dio_config.h +++ b/app/crowdb-diskio/src/dio_config.h @@ -6,6 +6,7 @@ #pragma once #include "crowdb-common/runtime_path.h" +#include "crowdb-protocol/ports.h" #include "crowdb-protocol/timing.h" #include "disk/disk_properties.h" #include "disk/types.h" @@ -35,10 +36,9 @@ struct DiskEntry // Server configuration. struct DioConfig { - // RPC listen. Default port 13000 matches DISKIO_RPC_BASE in - // lib/crowdb-protocol/src/ports.rs — keep in sync. + // RPC listen; defaults are generated from protocol ports.rs. std::string bind_address = "127.0.0.1"; - int listen_port = 13000; + int listen_port = protocol::DISKIO_RPC_BASE; uint32_t rpc_workers = 4; // Node identity (64-bit; each diskio service manages one node). diff --git a/app/crowdb-diskio/src/dio_main.cpp b/app/crowdb-diskio/src/dio_main.cpp index d86744bce..fa6841d3f 100644 --- a/app/crowdb-diskio/src/dio_main.cpp +++ b/app/crowdb-diskio/src/dio_main.cpp @@ -16,7 +16,7 @@ #include "crowdb-common/metrics/metrics.h" #include "crowdb-common/metrics/system_metrics.h" -#include "crowdb-kv-client/c_api.h" +#include "crowdb-paxos-kv-client/c_api.h" #include "crowdb-rpc/scheduled_executor.h" #include "crowdb-rpc/server/server.h" #include "crowdb-rpc/transport/socket_transport.h" diff --git a/app/crowdb-diskio/src/group0/group0_exclusion.cpp b/app/crowdb-diskio/src/group0/group0_exclusion.cpp index 5f9d60211..152fe2766 100644 --- a/app/crowdb-diskio/src/group0/group0_exclusion.cpp +++ b/app/crowdb-diskio/src/group0/group0_exclusion.cpp @@ -1,6 +1,6 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -#include "crowdb-kv-client/c_api.h" +#include "crowdb-paxos-kv-client/c_api.h" #include "group0/ffi_result.h" #include "group0/group0_sync.h" diff --git a/app/crowdb-diskio/src/group0/group0_sync.cpp b/app/crowdb-diskio/src/group0/group0_sync.cpp index 70c11509a..2d694d1cc 100644 --- a/app/crowdb-diskio/src/group0/group0_sync.cpp +++ b/app/crowdb-diskio/src/group0/group0_sync.cpp @@ -3,7 +3,7 @@ #include "group0/group0_sync.h" -#include "crowdb-kv-client/c_api.h" +#include "crowdb-paxos-kv-client/c_api.h" #include "disk/block_disk.h" #include "disk/mem_disk.h" #include "disk/null_disk.h" diff --git a/app/crowdb-diskio/src/group0/group0_sync.h b/app/crowdb-diskio/src/group0/group0_sync.h index c748d723d..68ffbc255 100644 --- a/app/crowdb-diskio/src/group0/group0_sync.h +++ b/app/crowdb-diskio/src/group0/group0_sync.h @@ -4,7 +4,7 @@ // Group0Sync: periodic group-0 synchronization for crowdb-diskio. // // When kv_seeds are configured, Group0Sync connects to group-0 via -// the crowdb-kv-client FFI C ABI, fetches the disk list for this node's +// the crowdb-paxos-kv-client FFI C ABI, fetches the disk list for this node's // disk-group, reconciles the DiskSet, and heartbeats to the service // registry. // @@ -81,7 +81,7 @@ class Group0Sync DiskExclusion exclusions_; bool registered_ = false; - // FFI handles (opaque pointers from crowdb-kv-client FFI). + // FFI handles (opaque pointers from crowdb-paxos-kv-client FFI). void *hw_client_ = nullptr; void *svc_client_ = nullptr; }; diff --git a/app/crowdb-diskio/src/rpc/dio_read.cpp b/app/crowdb-diskio/src/rpc/dio_read.cpp index 88f7698d5..2eb42bcfe 100644 --- a/app/crowdb-diskio/src/rpc/dio_read.cpp +++ b/app/crowdb-diskio/src/rpc/dio_read.cpp @@ -4,6 +4,7 @@ #include "crowdb-common/metrics/metrics.h" #include "rpc/dio_read_buffer.h" #include "rpc/dio_request.h" +#include "rpc/dio_request_deadline.h" #include "rpc/dio_server.h" #include <msg_type_generated.h> @@ -23,14 +24,30 @@ crowdb::rpc::OutFrame *DiskioServer::handle_read(crowdb::rpc::Frame *request, cr { uint64_t req_id = request->request_id; uint64_t create_nano = request->rpc_create_nano; - uint16_t msg_type = static_cast<uint16_t>(rproto::FBMsgType_EDiskReadResponse); + auto msg_type = static_cast<uint16_t>(rproto::FBMsgType_EDiskReadResponse); - auto *fb_req = decode_disk_request<dproto::FBDiskReadRequest>(*request); + const auto *fb_req = decode_disk_request<dproto::FBDiskReadRequest>(*request); if (fb_req == nullptr || (request->data_buf != nullptr && request->data_buf->len != 0)) { delete request; send_error_response(conn, req_id, create_nano, msg_type, static_cast<int16_t>(dproto::FBDiskIoRetCode_IoError)); return nullptr; } + const bool read_age_checked = fb_req->read_create_time_ms() != 0; + const auto deadline = read_age_checked ? request_deadline(fb_req->read_create_time_ms(), max_write_request_age_ms_, + max_clock_skew_ms_) + : std::nullopt; + if ((read_age_checked && !deadline) || (fb_req->fence_probe() && (!read_age_checked || fb_req->size() != 0))) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, + static_cast<int16_t>(dproto::FBDiskIoRetCode_OldRequest)); + return nullptr; + } + if (fb_req->fence_probe()) { + delete request; + send_response(transport_, conn->weak_from_this().lock(), req_id, create_nano, msg_type, + static_cast<int16_t>(dproto::FBDiskIoRetCode_Success), nullptr, false, true); + return nullptr; + } DiskId did = parse_disk_id(fb_req->disk_id()); uint32_t zone_index = fb_req->zone_index(); uint64_t zone_offset = fb_req->zone_offset(); @@ -67,7 +84,15 @@ crowdb::rpc::OutFrame *DiskioServer::handle_read(crowdb::rpc::Frame *request, cr root->ownership.load(std::memory_order_acquire) == crowdb::rpc::BufferOwnership::CallerOwned; // Control has been decoded, and READ carries no input payload. Retain its // existing receive data region until disk and response SEND both finish. - auto *read_buf = registered ? (receive_owned ? root->slice(0, size) : nullptr) : conn->pool()->alloc(size); + crowdb::rpc::Buffer *read_buf = nullptr; + if (registered) { + if (receive_owned) { + read_buf = root->slice(0, size); + } + } + else { + read_buf = conn->pool()->alloc(size); + } if (read_buf == nullptr) { delete request; send_error_response(conn, req_id, create_nano, msg_type, static_cast<int16_t>(dproto::FBDiskIoRetCode_IoError)); @@ -82,26 +107,36 @@ crowdb::rpc::OutFrame *DiskioServer::handle_read(crowdb::rpc::Frame *request, cr send_error_response(conn, req_id, create_nano, msg_type, static_cast<int16_t>(aligned)); return nullptr; } + if (deadline && std::chrono::steady_clock::now() >= *deadline) { + read_buf->release(); + send_error_response(conn, req_id, create_nano, msg_type, + static_cast<int16_t>(dproto::FBDiskIoRetCode_OldRequest)); + return nullptr; + } if (!io.begin()) { conn->close(); return nullptr; } - Disk *disk_ptr = disk.get(); - auto started = std::chrono::steady_clock::now(); - auto *io_data = io.data(); - disk_ptr->engine()->submit_read(disk_ptr, io.offset, io_data, io.io_size, test_pattern_offset, + Disk *disk_ptr = disk.get(); + auto started = std::chrono::steady_clock::now(); + auto *io_data = io.data(); + const auto io_offset = io.offset; + const auto io_size = io.io_size; + disk_ptr->engine()->submit_read(disk_ptr, io_offset, io_data, io_size, test_pattern_offset, [transport = transport_, connection = conn->weak_from_this().lock(), latency = read_latency_, req_id, create_nano, msg_type, io = std::move(io), - disk = std::move(disk), started](int result) mutable { + disk = std::move(disk), started, read_age_checked](int result) mutable { latency->observe(elapsed_nanos(started)); const auto completed = io.finish(result); if (!completed.valid_owner) { - if (connection != nullptr) + if (connection != nullptr) { connection->close(); + } return; } send_response(transport, connection, req_id, create_nano, msg_type, - static_cast<int16_t>(completed.code), completed.data); + static_cast<int16_t>(completed.code), completed.data, false, + read_age_checked); }); return nullptr; diff --git a/app/crowdb-diskio/src/rpc/dio_request_deadline.h b/app/crowdb-diskio/src/rpc/dio_request_deadline.h new file mode 100644 index 000000000..3123ac425 --- /dev/null +++ b/app/crowdb-diskio/src/rpc/dio_request_deadline.h @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <chrono> +#include <cstdint> +#include <optional> +#include <utility> + +namespace crowdb::diskio +{ +inline std::optional<std::chrono::steady_clock::time_point> request_deadline(uint64_t created_ms, uint64_t max_age_ms, + uint64_t skew_ms) +{ + const auto steady_now = std::chrono::steady_clock::now(); + const auto now_ms = static_cast<uint64_t>( + std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()) + .count()); + if (created_ms == 0 || max_age_ms == 0 || (created_ms > now_ms && created_ms - now_ms > skew_ms) || + max_age_ms > UINT64_MAX - skew_ms) { + return std::nullopt; + } + const auto allowance = max_age_ms + skew_ms; + if (created_ms > UINT64_MAX - allowance || created_ms + allowance <= now_ms) { + return std::nullopt; + } + const auto remaining_ms = created_ms + allowance - now_ms; + const auto max_wait_ms = + std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::time_point::max() - steady_now) + .count(); + if (std::cmp_greater(remaining_ms, max_wait_ms)) { + return std::nullopt; + } + return steady_now + std::chrono::milliseconds(static_cast<int64_t>(remaining_ms)); +} +} // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/rpc/dio_server.cpp b/app/crowdb-diskio/src/rpc/dio_server.cpp index 2515f0171..f5fbae4dc 100644 --- a/app/crowdb-diskio/src/rpc/dio_server.cpp +++ b/app/crowdb-diskio/src/rpc/dio_server.cpp @@ -57,7 +57,6 @@ DiskioServer::DiskioServer(std::shared_ptr<DiskSet> disk_set, std::shared_ptr<cr uint64_t max_write_request_age_ms, uint64_t max_clock_skew_ms) : disk_set_(std::move(disk_set)), transport_(std::move(transport)), - aligned_writer_(), read_latency_(&engine_read_latency_metric()), write_latency_(&engine_write_latency_metric()), fsync_latency_(&engine_fsync_latency_metric()), @@ -69,7 +68,7 @@ DiskioServer::DiskioServer(std::shared_ptr<DiskSet> disk_set, std::shared_ptr<cr // Build a diskio response control buffer (flatbuffer). crowdb::rpc::Buffer *DiskioServer::build_response_ctrl(crowdb::rpc::BufferPool *pool, uint64_t request_id, uint64_t rpc_create_nano, int16_t ret_code, uint16_t msg_type, - bool sync_completed) + bool sync_completed, bool read_age_checked) { auto fb_ret = static_cast<dproto::FBDiskIoRetCode>(ret_code); flatbuffers::FlatBufferBuilder fbb(64); @@ -78,7 +77,7 @@ crowdb::rpc::Buffer *DiskioServer::build_response_ctrl(crowdb::rpc::BufferPool * fbb.Finish(off); } else if (msg_type == static_cast<uint16_t>(rproto::FBMsgType_EDiskReadResponse)) { - auto off = dproto::CreateFBDiskReadResponse(fbb, request_id, rpc_create_nano, fb_ret); + auto off = dproto::CreateFBDiskReadResponse(fbb, request_id, rpc_create_nano, fb_ret, read_age_checked); fbb.Finish(off); } else { // fsync response @@ -103,16 +102,19 @@ void DiskioServer::send_error_response(crowdb::rpc::Connection *conn, uint64_t r void DiskioServer::send_response(const std::shared_ptr<crowdb::rpc::Transport> &transport, const std::shared_ptr<crowdb::rpc::Connection> &connection, uint64_t request_id, uint64_t created, uint16_t type, int16_t code, crowdb::rpc::Buffer *data, - bool sync_completed) + bool sync_completed, bool read_age_checked) { const auto release = [](crowdb::rpc::Buffer *buffer) { - if (buffer != nullptr) + if (buffer != nullptr) { buffer->release(); + } }; std::unique_ptr<crowdb::rpc::Buffer, decltype(release)> payload(data, release); - if (connection == nullptr || !connection->is_open()) + if (connection == nullptr || !connection->is_open()) { return; - auto *control = build_response_ctrl(connection->pool(), request_id, created, code, type, sync_completed); + } + auto *control = + build_response_ctrl(connection->pool(), request_id, created, code, type, sync_completed, read_age_checked); if (control == nullptr) { connection->close(); return; diff --git a/app/crowdb-diskio/src/rpc/dio_server.h b/app/crowdb-diskio/src/rpc/dio_server.h index e235c031f..9d84245d0 100644 --- a/app/crowdb-diskio/src/rpc/dio_server.h +++ b/app/crowdb-diskio/src/rpc/dio_server.h @@ -69,7 +69,7 @@ class DiskioServer // pulling the generated header into this .h). static crowdb::rpc::Buffer *build_response_ctrl(crowdb::rpc::BufferPool *pool, uint64_t request_id, uint64_t rpc_create_nano, int16_t ret_code, uint16_t msg_type, - bool sync_completed = false); + bool sync_completed = false, bool read_age_checked = false); // Send an error response synchronously (disk not found, parse error). void send_error_response(crowdb::rpc::Connection *conn, uint64_t request_id, uint64_t rpc_create_nano, @@ -78,7 +78,7 @@ class DiskioServer static void send_response(const std::shared_ptr<crowdb::rpc::Transport> &transport, const std::shared_ptr<crowdb::rpc::Connection> &connection, uint64_t request_id, uint64_t created, uint16_t type, int16_t code, crowdb::rpc::Buffer *data = nullptr, - bool sync_completed = false); + bool sync_completed = false, bool read_age_checked = false); }; } // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/rpc/dio_server_write.cpp b/app/crowdb-diskio/src/rpc/dio_server_write.cpp index 8362b90ef..74033ddbd 100644 --- a/app/crowdb-diskio/src/rpc/dio_server_write.cpp +++ b/app/crowdb-diskio/src/rpc/dio_server_write.cpp @@ -4,6 +4,7 @@ #include "crowdb-common/metrics/metrics.h" #include "crowdb-rpc/server/message.h" #include "disk/disk.h" +#include "rpc/dio_request_deadline.h" #include "rpc/dio_server.h" #include <diskio_generated.h> @@ -23,41 +24,12 @@ namespace rproto = crowdb::rpc::proto; namespace { -uint64_t unix_time_ms() -{ - return static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()) - .count()); -} - uint64_t elapsed_nanos(std::chrono::steady_clock::time_point started) { return static_cast<uint64_t>( std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count()); } -std::optional<std::chrono::steady_clock::time_point> write_deadline(uint64_t created_ms, uint64_t max_age_ms, - uint64_t skew_ms) -{ - const auto steady_now = std::chrono::steady_clock::now(); - const auto now_ms = unix_time_ms(); - if (created_ms == 0 || max_age_ms == 0 || (created_ms > now_ms && created_ms - now_ms > skew_ms) || - max_age_ms > UINT64_MAX - skew_ms) { - return std::nullopt; - } - const auto allowance = max_age_ms + skew_ms; - if (created_ms > UINT64_MAX - allowance || created_ms + allowance <= now_ms) { - return std::nullopt; - } - const auto remaining_ms = created_ms + allowance - now_ms; - const auto max_wait_ms = - std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::time_point::max() - steady_now) - .count(); - if (std::cmp_greater(remaining_ms, max_wait_ms)) { - return std::nullopt; - } - return steady_now + std::chrono::milliseconds(static_cast<int64_t>(remaining_ms)); -} } // namespace crowdb::rpc::OutFrame *DiskioServer::handle_write(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) @@ -82,7 +54,7 @@ crowdb::rpc::OutFrame *DiskioServer::handle_write(crowdb::rpc::Frame *request, c uint64_t write_create_time_ms = fb_req->write_create_time_ms(); const bool sync = fb_req->sync(); - const auto deadline = write_deadline(write_create_time_ms, max_write_request_age_ms_, max_clock_skew_ms_); + const auto deadline = request_deadline(write_create_time_ms, max_write_request_age_ms_, max_clock_skew_ms_); if (!deadline) { delete request; send_error_response(conn, req_id, create_nano, msg_type, diff --git a/app/crowdb-diskio/tests/dio_lifetime_test.cpp b/app/crowdb-diskio/tests/dio_lifetime_test.cpp index 86f55a4a2..f4e64dbfd 100644 --- a/app/crowdb-diskio/tests/dio_lifetime_test.cpp +++ b/app/crowdb-diskio/tests/dio_lifetime_test.cpp @@ -32,19 +32,21 @@ class TestDeferredEngine final : public IoEngine std::vector<Pending> pending; - void submit_write(Disk *, off_t, const uint8_t *data, size_t size, std::function<void(int)> done) override + void submit_write(Disk * /*unused*/, off_t /*unused*/, const uint8_t *data, size_t size, + std::function<void(int)> done) override { - pending.push_back({data, nullptr, size, std::move(done)}); + pending.push_back({.data = data, .output = nullptr, .size = size, .done = std::move(done)}); } - void submit_read(Disk *, off_t, uint8_t *data, size_t size, uint64_t, std::function<void(int)> done) override + void submit_read(Disk * /*unused*/, off_t /*unused*/, uint8_t *data, size_t size, uint64_t /*unused*/, + std::function<void(int)> done) override { - pending.push_back({nullptr, data, size, std::move(done)}); + pending.push_back({.data = nullptr, .output = data, .size = size, .done = std::move(done)}); } - void submit_fsync(Disk *, std::function<void(int)> done) override + void submit_fsync(Disk * /*unused*/, std::function<void(int)> done) override { - pending.push_back({nullptr, nullptr, 0, std::move(done)}); + pending.push_back({.data = nullptr, .output = nullptr, .size = 0, .done = std::move(done)}); } void complete(int result) @@ -52,8 +54,9 @@ class TestDeferredEngine final : public IoEngine ASSERT_FALSE(pending.empty()); auto io = std::move(pending.front()); pending.erase(pending.begin()); - if (io.output != nullptr && result > 0) + if (io.output != nullptr && result > 0) { std::memset(io.output, 0x5a, std::min(io.size, static_cast<size_t>(result))); + } io.done(result); } }; @@ -66,22 +69,22 @@ class TestDeferredDisk final : public Disk zones_.push_back({.zone_index = 0, .base_offset = 0, .capacity = 8 << 20}); } - DiskType type() const override + [[nodiscard]] DiskType type() const override { return DiskType::Mem; } - int fd() const override + [[nodiscard]] int fd() const override { return 1; } - bool is_o_direct() const override + [[nodiscard]] bool is_o_direct() const override { return false; } - size_t block_size() const override + [[nodiscard]] size_t block_size() const override { return 1; } @@ -91,14 +94,14 @@ class TestDeferredDisk final : public Disk return engine_; } - DiskId id() const override + [[nodiscard]] DiskId id() const override { - return {7, 9}; + return {.high = 7, .low = 9}; } Zone *find_zone(uint32_t index) override { - return index == 0 ? &zones_[0] : nullptr; + return index == 0 ? zones_.data() : nullptr; } private: @@ -142,11 +145,11 @@ class DioLifetimeTest : public ::testing::Test return frame; } - Frame *read(uint32_t size) + Frame *read(uint32_t size, uint64_t created_ms = 0, bool probe = false) { flatbuffers::FlatBufferBuilder builder; crowdb::rpc::proto::FBInt128 disk(7, 9); - builder.Finish(proto::CreateFBDiskReadRequest(builder, 19, 0, &disk, 0, 0, size)); + builder.Finish(proto::CreateFBDiskReadRequest(builder, 19, 0, &disk, 0, 0, size, 0, created_ms, probe)); auto *frame = new Frame; frame->request_id = 19; frame->control.assign(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); @@ -154,6 +157,24 @@ class DioLifetimeTest : public ::testing::Test } }; +TEST_F(DioLifetimeTest, ExpiredProtectedReadDoesNotSubmitDiskIo) +{ + EXPECT_EQ(service->handle_read(read(4096, 1), connection.get()), nullptr); + EXPECT_TRUE(engine.pending.empty()); +} + +TEST_F(DioLifetimeTest, ProtectedReadProbeNeverAllocatesOrSubmitsDataIo) +{ + const auto now = static_cast<uint64_t>( + std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()) + .count()); + EXPECT_EQ(service->handle_read(read(0, now, true), connection.get()), nullptr); + EXPECT_TRUE(engine.pending.empty()); + EXPECT_EQ(service->handle_read(read(4096, now), connection.get()), nullptr); + ASSERT_EQ(engine.pending.size(), 1U); + engine.complete(4096); +} + TEST_F(DioLifetimeTest, DiskOwnsExactPayloadUntilCompletionWithoutReplacementAdmission) { RegisteredBufferPool buffers(4096, 4); diff --git a/app/crowdb-diskio/tests/ordered_drain_test.cpp b/app/crowdb-diskio/tests/ordered_drain_test.cpp new file mode 100644 index 000000000..380c465c5 --- /dev/null +++ b/app/crowdb-diskio/tests/ordered_drain_test.cpp @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/aligned_writer.h" + +#include <gtest/gtest.h> + +#include <cerrno> +#include <chrono> + +namespace crowdb::diskio +{ +TEST(OrderedDiskDrain, ZeroByteBarrierWaitsForActiveWriteAndExpiredLateWriteCannotFollow) +{ + test::TestRecordingEngine engine(8192, true); + auto disk = std::make_shared<test::TestDisk>(&engine, 1); + AlignedWriter writer; + std::vector<uint8_t> data(32, 9); + std::vector<int> completions; + writer.submit_ordered(disk, 0, data.data(), data.size(), 0, [&](int result) { completions.push_back(result); }); + writer.submit_ordered(disk, 0, nullptr, 0, 0, [&](int result) { completions.push_back(result); }); + EXPECT_TRUE(completions.empty()); + EXPECT_EQ(engine.pending_writes.size(), 1U); + writer.submit_ordered( + disk, 32, data.data(), data.size(), 0, [&](int result) { completions.push_back(result); }, + std::chrono::steady_clock::now() - std::chrono::seconds(1)); + engine.complete_one_write(); + EXPECT_EQ(completions, (std::vector<int>{32, 0, -ETIMEDOUT})); + EXPECT_EQ(engine.write_sizes.size(), 1U); + EXPECT_TRUE(engine.pending_writes.empty()); +} +} // namespace crowdb::diskio diff --git a/app/crowdb-kv-server/conf/crowdb_kv_server_config.toml b/app/crowdb-kv-server/conf/crowdb_kv_server_config.toml deleted file mode 100644 index 815121855..000000000 --- a/app/crowdb-kv-server/conf/crowdb_kv_server_config.toml +++ /dev/null @@ -1,8 +0,0 @@ -# Canonical crowdb-kv-server startup tunables. -# Runtime paths are derived from --root and are not file settings. - -wal_early_ack = true -async_engine_apply = true - -[server] -rpc_workers = 2 diff --git a/app/crowdb-kv-server/Cargo.toml b/app/crowdb-paxos-kv-server/Cargo.toml similarity index 79% rename from app/crowdb-kv-server/Cargo.toml rename to app/crowdb-paxos-kv-server/Cargo.toml index 06389f473..b2f398706 100644 --- a/app/crowdb-kv-server/Cargo.toml +++ b/app/crowdb-paxos-kv-server/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "crowdb-kv-server" +name = "crowdb-paxos-kv-server" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -9,7 +9,7 @@ description = "CROWDB server binary: single-group, multi-group, and cluster mode autotests = false [[bin]] -name = "crowdb-kv-server" +name = "crowdb-paxos-kv-server" path = "src/main.rs" [lints] @@ -19,8 +19,8 @@ workspace = true default = [] [dependencies] -crowdb-kv = { path = "../../lib/crowdb-kv" } -crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } +crowdb-paxos-kv = { path = "../../lib/crowdb-paxos-kv" } +crowdb-paxos-kv-client = { path = "../../lib/crowdb-paxos-kv-client" } crowdb-common = { path = "../../lib/crowdb-common/rust" } crowdb-protocol = { path = "../../lib/crowdb-protocol", features = ["schema"] } crowdb-tree-ffi = { path = "../../lib/crowdb-tree/ffi" } @@ -39,7 +39,7 @@ sha2 = "0.10" uuid = { version = "1", features = ["v4"] } [dev-dependencies] -crowdb-kv = { path = "../../lib/crowdb-kv", features = ["test-util"] } +crowdb-paxos-kv = { path = "../../lib/crowdb-paxos-kv", features = ["test-util"] } reqwest = { version = "0.12", features = ["json"] } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } @@ -107,3 +107,19 @@ path = "tests/startup_test.rs" [[test]] name = "diskdb_placement_test" path = "tests/diskdb_placement_test.rs" + +[[test]] +name = "tree_runtime_host_test" +path = "tests/tree_runtime_host_test.rs" + +[[test]] +name = "tree_config_test" +path = "tests/tree_config_test.rs" + +[[test]] +name = "engine_resource_metrics_test" +path = "tests/engine_resource_metrics_test.rs" + +[[test]] +name = "runtime_shutdown_test" +path = "tests/runtime_shutdown_test.rs" diff --git a/app/crowdb-kv-server/build.rs b/app/crowdb-paxos-kv-server/build.rs similarity index 100% rename from app/crowdb-kv-server/build.rs rename to app/crowdb-paxos-kv-server/build.rs diff --git a/app/crowdb-paxos-kv-server/conf/crowdb_paxos_kv_server_config.toml b/app/crowdb-paxos-kv-server/conf/crowdb_paxos_kv_server_config.toml new file mode 100644 index 000000000..95cba275d --- /dev/null +++ b/app/crowdb-paxos-kv-server/conf/crowdb_paxos_kv_server_config.toml @@ -0,0 +1,199 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# +# Paxos KV Server +# crowdb-paxos-kv-server --root <data-dir> --config <this-file> +# Read at startup. Paths, backends and log locations are supplied through the CLI. +# These are the existing defaults. Find maximum sustainable flush throughput with +# ample resources first, then reduce resources using the measured curves. +# Sizes are integer bytes; 1 MiB = 1,048,576 bytes. Tune threads, queues and budgets +# together for the whole process and all of its Trees. +# --election-profile replaces the entire [election] table below. Omit that flag +# when tuning maintenance intervals in this file. + +# Preserve the default ACK/apply modes when introducing a configuration file. +wal_early_ack = true +async_engine_apply = true + +# Keep Tree field additions, removals, renames and explanations in sync with: +# app/crowdb-chunk-kv-server/conf/crowdb_chunk_kv_server_config.toml +# Each server owns its values independently; no shared include or override is loaded. + +# Process-wide CPU/backend pools. Check CPU quotas and utilization when scaling. +# Resource diagnostics sample every 100 ms and flush at the server metrics-log +# interval. Each host/Tree resource reports its limit, current usage, average +# held sampled usage, and sampled maximum for that log window. The maximum is +# not an exact transient peak; sample count, age, gaps and errors show coverage. +# Averages retain fractional counts: worker avg=0.750 means 75% of one worker. +# Reserved capacity and allocated storage are reported separately. Budget +# denial counters distinguish Tree limits from shared host limits. Queue-slot +# usage includes prepaid progress registrations even when no job is runnable. +# Tree cache rows cover decoded bytes, page slots and request slots; they +# overlap the storage charged to budget partitions and must not be added to it. +[tree_runtime.host] +workers = 4 +# Workers for synchronous PageStore calls; async concurrency also depends on the backend. +backend_workers = 2 +# Larger queues retain more work without increasing service rate. Recheck +# lifecycle/Routing reservations when increasing queue capacity. +cpu_queue_limit = 256 +backend_queue_limit = 256 +# Maximum pending waiters per resource partition. +max_waiters_per_class = 256 +# Retained queue/control reservation charged to host Routing, not an extra partition. +lifecycle_bytes = 4_194_304 + +# Shared limits for all Trees in this process. Resource classes cannot borrow +# from each other. These limits do not bound total process RSS. +[tree_runtime.host.limits] +# Source capture, run/range construction and complete-batch handoff. +handoff_bytes = 268_435_456 +# Routing, publication, cache/load and controls, including lifecycle reservations. +routing_bytes = 536_870_912 +# Consolidation, split/merge and checkpoint/writeback work. +reclaim_bytes = 134_217_728 +# Ordinary writes, payload owners and retained COW allocations. +writer_bytes = 536_870_912 +# Independent progress capacity for chosen/replay/gap apply work. +gap_repair_bytes = 67_108_864 +# In-flight I/O tickets. More tickets alone do not parallelize a serial write pipeline. +io_operations = 128 + +# Per-Tree limits. All Trees still compete for the shared host resources above. +[tree_runtime.tree] +# The producer window must fit this CPU queue. Local queues must fit host queues. +# This also bounds ordinary apply/backend operation owners; replay has two spare slots. +cpu_queue_limit = 64 +backend_queue_limit = 64 +# Fixed queue/core/cache descriptors, charged to this Tree's Routing partition. +# Increase this together with cpu_queue_limit and tuning.cache_max_pages; a +# larger Routing limit alone does not enlarge this retained control reserve. +lifecycle_bytes = 1_048_576 + +[tree_runtime.tree.limits] +# Same categories as the host limits. Check both levels when changing a partition. +handoff_bytes = 67_108_864 +routing_bytes = 268_435_456 +# 64 MiB is an adjustable default, not a fixed checkpoint memory target. +reclaim_bytes = 67_108_864 +writer_bytes = 134_217_728 +gap_repair_bytes = 16_777_216 +io_operations = 16 + +[tree] +# Durable frame size; the current public format supports at most 64 KiB. +# A logical leaf may exceed one frame. +# Must be at least 512 bytes and satisfy backend IU alignment. Logical working +# set size is independent of durable frame size. +frame_bytes = 65_536 +# Decoded-cache byte watermark; tuning.cache_max_pages also limits page count. +buffer_pool_bytes = 67_108_864 +# 0 = frame_bytes/4. Values larger than this spill to overflow; exactly 16 KiB +# remains inline with the default frame size. +max_inline_value = 0 +# LZ4 trades CPU for storage bandwidth/space. Keep compression fixed in comparisons. +compression_lz4 = false +# Local block allocation/reclamation unit; ignored by file/mem-block and injected Chunk storage. +block_size = 67_108_864 +# 0 = backend default: block 4 KiB, mem-block 1 byte, Chunk 64 KiB. A nonzero value +# must be a power of two that divides frame_bytes. +iu_size = 0 + +# Engine work granularity. Select these values from measurements. +[tree.tuning] +# Soft per-source rotation thresholds. Crossing either requests rotation; +# 0 requests rotation after every nonempty write. +# The source_max_* hard admission capacities below also apply. +memtable_flush_bytes = 16_777_216 +memtable_flush_entries = 100_000 +# Includes the writable source; the current read/handoff window requires 2..64. +max_memtable_count = 10 +# Per-source admitted version/key capacities, including writes not yet applied. +# Raising soft thresholds alone may not enlarge batches if these limits bind first. +source_max_entries = 4_096 +source_max_key_bytes = 1_048_576 +# Additional accepted/replay allowance. Size it for accepted apply bursts. +# Only the next contiguous slot may use gap source capacity; higher holes use ordinary capacity or wait. +source_max_gap_entries = 64 +source_max_gap_key_bytes = 131_072 +# 0 = min(host workers, Tree io_operations). Explicit values must fit both limits. +flush_parallelism = 0 +# Per-handoff-run entry/key/work-byte limits. Value owners are retained independently. +run_max_entries = 128 +# Effective key capacity is max(run_max_key_bytes, max_key_size). +run_max_key_bytes = 262_144 +run_max_bytes = 1_048_576 +# Key-arena allocation granularity; must not exceed run_max_key_bytes. +run_key_block_bytes = 1_024 +# Initial run-install window; later windows double. Balances early visible +# progress against prefix-array copying cost. +install_run_window = 64 +# Visits per step/pass; the step size must not exceed the pass limit. +maintenance_visits_per_step = 64 +maintenance_max_visits = 4_096 +# Candidates/steps per pass. Larger limits increase work and reservations; +# they do not repair starvation caused by an incorrect scan cursor. +maintenance_max_candidates = 256 +maintenance_max_steps = 1_024 +# Local scratch per maintenance operation; independent of the total Reclaim budget. +maintenance_metadata_bytes = 1_048_576 +# Decoded-cache page limit. Check this together with buffer_pool_bytes. +cache_max_pages = 256 +# Split/merge thresholds for the physical primary record region, not logical leaf limits. +# Split is capped by frame/format capacity. Merge=0 disables merging; merged +# output must also fit the hysteresis target of about 75% of the split threshold. +leaf_split_bytes = 65_502 +leaf_merge_bytes = 16_384 +# Inner separator limit; encoded output must also fit the frame byte capacity. +inner_max_keys = 256 +# 0 = max(1, inner_max_keys/4). +inner_merge_keys = 0 +# Maximum key size; 0 = frame_bytes/2. Oversized keys are rejected before admission. +max_key_size = 0 + +[paxos] +# Upstream proposal window, held through native apply even after an async Chosen +# reply; must not exceed tree_runtime.tree.cpu_queue_limit. +max_inflight_proposals = 32 +# Keys per proposal; 0 disables coalescing. The current descriptor limit is 64; +# this must also fit the source gap allowance. +coalesce_max_keys = 32 +# Skip draining a pending batch when at least this many other coalesced rounds exist. +coalesce_drain_threshold = 1 +# false = await a window permit; true = return Busy when full. Changes backpressure +# behavior without increasing service rate. +reject_on_inflight_window_full = false + +[server] +# RPC I/O workers are separate from Tree CPU and backend workers. +rpc_workers = 2 +# Connections per replica peer. +peer_pool_size = 2 +# Enqueue to I/O workers to coalesce writev calls; adds wakeup latency. +event_write = false +# Per-connection send queue. More queued work does not increase service rate. +send_queue_capacity = 4_096 +# Unary scan payload limit for mixed workloads; independent of checkpoint frame size. +scan_byte_budget = 3_670_016 + +[election] +# Watchdog/idle maintenance interval; source rotation also signals maintenance. +maintenance_tick_ms = 30_000 +# Any of the next three thresholds triggers a durable snapshot. Frequent snapshots +# add overhead; infrequent snapshots retain dirty state and WAL longer. +snapshot_slot_threshold = 100_000 +snapshot_time_threshold_ms = 600_000 +# 0 disables the flush-count trigger. +snapshot_flush_count_threshold = 10 +# Maintenance-forced WAL flush interval. 0 disables this trigger, not the WAL +# writer's own flush pipeline. +wal_flush_interval_ms = 60_000 +# Sparse-block relocation interval; 0 disables relocation, not snapshot folding. +merge_gc_interval_ms = 0 + +[wal] +# Segment rotation/GC granularity. +wal_segment_size = 67_108_864 +# WAL batch size and low-traffic flush watchdog; affect consensus input cost. +wal_flush_batch_bytes = 65_536 +wal_flush_watchdog_ms = 100 diff --git a/app/crowdb-kv-server/examples/openapi-export.rs b/app/crowdb-paxos-kv-server/examples/openapi-export.rs similarity index 91% rename from app/crowdb-kv-server/examples/openapi-export.rs rename to app/crowdb-paxos-kv-server/examples/openapi-export.rs index d1ae55940..feebdf83e 100644 --- a/app/crowdb-kv-server/examples/openapi-export.rs +++ b/app/crowdb-paxos-kv-server/examples/openapi-export.rs @@ -11,7 +11,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { if let Some(parent) = out.parent() { std::fs::create_dir_all(parent)?; } - let json = crowdb_kv_server::mgmt::openapi_json(); + let json = crowdb_paxos_kv_server::mgmt::openapi_json(); std::fs::write(out, serde_json::to_string_pretty(&json)?)?; println!("{}", out.display()); Ok(()) diff --git a/app/crowdb-kv-server/src/background.rs b/app/crowdb-paxos-kv-server/src/background.rs similarity index 100% rename from app/crowdb-kv-server/src/background.rs rename to app/crowdb-paxos-kv-server/src/background.rs diff --git a/app/crowdb-kv-server/src/background/discovery.rs b/app/crowdb-paxos-kv-server/src/background/discovery.rs similarity index 100% rename from app/crowdb-kv-server/src/background/discovery.rs rename to app/crowdb-paxos-kv-server/src/background/discovery.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor.rs similarity index 96% rename from app/crowdb-kv-server/src/background/domain_monitor.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor.rs index fafc63bbd..0d6d9880a 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor.rs +++ b/app/crowdb-paxos-kv-server/src/background/domain_monitor.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use crowdb_kv::cluster::group_operations::{ +use crowdb_paxos_kv::cluster::group_operations::{ KvGroupOperationError, KvGroupOperations, KvGroupScanRequest, KvReadConsistency, }; use crowdb_protocol::chunk_kv::DomainMonitorDescriptor; @@ -18,7 +18,7 @@ use crowdb_protocol::key::{DomainMonitorKey, TextKey}; use tracing::{debug, info, warn}; use crate::group0_control_plane::Group0ControlPlane; -use crate::store_registry::KvStoreRegistry; +use crate::store_registry::PaxosKvStoreRegistry; mod chunk_kv; mod chunkdb; @@ -136,7 +136,7 @@ struct RunningMonitor { /// group-0 leader tenure; followers therefore stay prepared but idle. #[must_use] pub fn spawn_domain_monitor_supervisor( - registry: Arc<KvStoreRegistry>, + registry: Arc<PaxosKvStoreRegistry>, drivers: DomainMonitorDrivers, discovery_interval: Duration, ) -> DomainMonitorSupervisorHandle { @@ -178,7 +178,7 @@ pub fn spawn_domain_monitor_supervisor( } async fn discover_descriptors( - registry: &KvStoreRegistry, + registry: &PaxosKvStoreRegistry, ) -> Result<Vec<DomainMonitorDescriptor>, KvGroupOperationError> { let store = registry .get_store(0) @@ -204,7 +204,7 @@ async fn discover_descriptors( deadline_ms: 0, bounded: true, requested_scan_cutoff: scan_cutoff, - direction: crowdb_kv::kv::ScanDirection::Forward, + direction: crowdb_paxos_kv::kv::ScanDirection::Forward, }) .await?; scan_cutoff = scan.scan_cutoff; @@ -238,7 +238,7 @@ async fn discover_descriptors( } fn reconcile_descriptors( - registry: &Arc<KvStoreRegistry>, + registry: &Arc<PaxosKvStoreRegistry>, drivers: &DomainMonitorDrivers, descriptors: Vec<DomainMonitorDescriptor>, running: &mut HashMap<String, RunningMonitor>, @@ -281,7 +281,7 @@ fn reconcile_descriptors( } fn spawn_driver( - registry: &Arc<KvStoreRegistry>, + registry: &Arc<PaxosKvStoreRegistry>, descriptor: DomainMonitorDescriptor, driver: Arc<dyn DomainMonitorDriver>, ) -> RunningMonitor { @@ -315,7 +315,7 @@ fn spawn_driver( } async fn run_driver_tick( - registry: &KvStoreRegistry, + registry: &PaxosKvStoreRegistry, driver: &dyn DomainMonitorDriver, descriptor: &DomainMonitorDescriptor, ) { diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv.rs similarity index 99% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv.rs index 7c757f980..db4176a31 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs +++ b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv.rs @@ -7,7 +7,7 @@ use std::collections::{HashMap, HashSet}; use std::time::{SystemTime, UNIX_EPOCH}; use bytes::Bytes; -use crowdb_kv::cluster::group_operations::{KvGroupOperationError, KvGroupScanItem}; +use crowdb_paxos_kv::cluster::group_operations::{KvGroupOperationError, KvGroupScanItem}; use crowdb_protocol::chunk_kv::{ AuthorityReleaseProof, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, DomainFailurePolicy, DomainMonitorDescriptor, Id128, OwnerDescriptor, ServingAssignment, ServingGrant, SplitPhase, diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance.rs similarity index 99% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance.rs index 61a3700f3..e143a1830 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs +++ b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance.rs @@ -6,7 +6,7 @@ use std::collections::{HashMap, HashSet}; use bytes::Bytes; -use crowdb_kv::cluster::group_operations::KvGroupOperationError; +use crowdb_paxos_kv::cluster::group_operations::KvGroupOperationError; use crowdb_protocol::chunk_kv::{ ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, DomainMonitorDescriptor, Id128, OwnerDescriptor, SplitPhase, TransferPhase, TransferTransition, diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/observation.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/observation.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/observation.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/observation.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/selection.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/selection.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/candidates.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/candidates.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/candidates.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/candidates.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/diagnostics.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/diagnostics.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/diagnostics.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/selection/diagnostics.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/split.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/split.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/split.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/balance/split.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs similarity index 99% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs index c340cc986..85cfcabd5 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs +++ b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs @@ -4,7 +4,7 @@ //! Immutable chunk-KV range-catalog loading and transition cutover. use bytes::Bytes; -use crowdb_kv::cluster::group_operations::KvGroupOperationError; +use crowdb_paxos_kv::cluster::group_operations::KvGroupOperationError; use crowdb_protocol::chunk_kv::{ AuthorityReleaseProof, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, SplitPhase, SplitTransition, diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog/endpoints.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/catalog/endpoints.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog/endpoints.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/catalog/endpoints.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/grants.rs similarity index 97% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/grants.rs index aa95956f9..530db8cf8 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs +++ b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/grants.rs @@ -6,7 +6,7 @@ use super::{catalog, operation_error, read_transfers, wall_time_ms}; use crate::group0_control_plane::Group0ControlPlane; use bytes::Bytes; -use crowdb_kv::cluster::group_operations::KvGroupMutation; +use crowdb_paxos_kv::cluster::group_operations::KvGroupMutation; use crowdb_protocol::{ chunk_kv::{ChunkKvRangeCatalogPartitionState, DomainMonitorDescriptor, ServingAssignment, ServingGrant}, chunk_stream::{StreamProcessLease, StreamPurpose}, @@ -109,7 +109,7 @@ async fn publish( if lease.identity.purpose != StreamPurpose::Wal || lease.identity.instance_id != instance_id { return Err("WAL process lease does not match its key".into()); } - if lease.identity.process_epoch != process_epoch || now >= lease.expires_at_ms { + if lease.revoked || lease.identity.process_epoch != process_epoch || now >= lease.expires_at_ms { return Ok(()); } let mut grant = ServingGrant { diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunkdb.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunkdb.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunkdb_dynamic.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb_dynamic.rs similarity index 98% rename from app/crowdb-kv-server/src/background/domain_monitor/chunkdb_dynamic.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb_dynamic.rs index 3c97172ff..ffeda12ea 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunkdb_dynamic.rs +++ b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb_dynamic.rs @@ -8,7 +8,7 @@ use super::{DomainMonitorDriver, DomainMonitorFuture}; mod owners; mod snapshot; use crate::group0_control_plane::Group0ControlPlane; -use crowdb_kv::cluster::group_operations::KvGroupMutation; +use crowdb_paxos_kv::cluster::group_operations::KvGroupMutation; use crowdb_protocol::chunk_kv::{DomainFailurePolicy, DomainMonitorDescriptor}; use crowdb_protocol::chunk_slot::{ ChunkSlot, ChunkSlotAuthority, ChunkSlotBinding, ChunkSlotBitmap, ChunkSlotMap, diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunkdb_dynamic/owners.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb_dynamic/owners.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunkdb_dynamic/owners.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb_dynamic/owners.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunkdb_dynamic/snapshot.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb_dynamic/snapshot.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/chunkdb_dynamic/snapshot.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/chunkdb_dynamic/snapshot.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/diskdb.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/diskdb.rs similarity index 100% rename from app/crowdb-kv-server/src/background/domain_monitor/diskdb.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/diskdb.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs b/app/crowdb-paxos-kv-server/src/background/domain_monitor/diskdb_placement.rs similarity index 99% rename from app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs rename to app/crowdb-paxos-kv-server/src/background/domain_monitor/diskdb_placement.rs index e8660dd5c..b28413359 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs +++ b/app/crowdb-paxos-kv-server/src/background/domain_monitor/diskdb_placement.rs @@ -6,7 +6,7 @@ use super::{DomainMonitorDriver, DomainMonitorFuture}; use crate::group0_control_plane::Group0ControlPlane; use bytes::Bytes; -use crowdb_kv::cluster::group_operations::KvGroupMutation; +use crowdb_paxos_kv::cluster::group_operations::KvGroupMutation; use crowdb_protocol::chunk_kv::{DomainFailurePolicy, DomainMonitorDescriptor}; use crowdb_protocol::common::{BindMapValue, InstanceValue, OwnerMapValue}; use crowdb_protocol::key::{BindMapKey, DiskGroupKey, InstanceKey, OwnerMapKey, RackKey, TextKey}; diff --git a/app/crowdb-kv-server/src/background/identity.rs b/app/crowdb-paxos-kv-server/src/background/identity.rs similarity index 97% rename from app/crowdb-kv-server/src/background/identity.rs rename to app/crowdb-paxos-kv-server/src/background/identity.rs index 20fba4614..a5d027a8e 100644 --- a/app/crowdb-kv-server/src/background/identity.rs +++ b/app/crowdb-paxos-kv-server/src/background/identity.rs @@ -33,7 +33,7 @@ pub fn load_or_create( Err(error) if error.kind() == io::ErrorKind::NotFound => {} Err(error) => return Err(error), } - let generated = crowdb_kv_client::new_client_id(); + let generated = crowdb_paxos_kv_client::new_client_id(); let stored = StoredIdentity { version: 1, instance_id: requested.unwrap_or(generated), diff --git a/app/crowdb-kv-server/src/background/keepalive.rs b/app/crowdb-paxos-kv-server/src/background/keepalive.rs similarity index 96% rename from app/crowdb-kv-server/src/background/keepalive.rs rename to app/crowdb-paxos-kv-server/src/background/keepalive.rs index 784109955..cc42f74c6 100644 --- a/app/crowdb-kv-server/src/background/keepalive.rs +++ b/app/crowdb-paxos-kv-server/src/background/keepalive.rs @@ -9,12 +9,12 @@ use std::sync::Arc; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::common::HostedGroup; use tokio::task::JoinHandle; use tracing::{debug, info, info_span, warn, Instrument}; -use crate::store_registry::KvStoreRegistry; +use crate::store_registry::PaxosKvStoreRegistry; /// Keep-alive loop handle. Call `stop` to unregister and terminate the /// background task. @@ -30,7 +30,7 @@ impl KeepAliveLoop { /// `interval_secs`. It reads `hosted_stores` / `hosted_groups` /// from the registry each tick so the record reflects live state. pub fn spawn( - registry: Arc<KvStoreRegistry>, + registry: Arc<PaxosKvStoreRegistry>, identity: crowdb_protocol::common::KvServerIdentity, rpc_endpoint: String, group0_endpoint: &str, @@ -56,7 +56,7 @@ impl KeepAliveLoop { mgmt_seeds.insert(0, rpc_endpoint.clone()); } let handle = tokio::spawn(async move { - let kv_client = CrowdbKvClient::new(ClientConfig::new(mgmt_seeds.clone())); + let kv_client = PaxosKvClient::new(ClientConfig::new(mgmt_seeds.clone())); if bootstrap_local { kv_client.seed_leader(0, 0, ep); } @@ -166,7 +166,7 @@ impl KeepAliveLoop { } async fn refresh_discovery( - registry: &KvStoreRegistry, + registry: &PaxosKvStoreRegistry, service: &ServiceRegistryClient, bootstrap_seeds: &[String], current: &mut Vec<String>, @@ -219,7 +219,7 @@ impl Drop for KeepAliveLoop { } /// Collect hosted store IDs and group IDs from the registry. -fn hosted_summary(registry: &KvStoreRegistry) -> (Vec<u64>, Vec<HostedGroup>) { +fn hosted_summary(registry: &PaxosKvStoreRegistry) -> (Vec<u64>, Vec<HostedGroup>) { let stores: Vec<u64> = registry.store_ids(); let mut groups: Vec<HostedGroup> = Vec::new(); for &sid in &stores { diff --git a/app/crowdb-kv-server/src/cli.rs b/app/crowdb-paxos-kv-server/src/cli.rs similarity index 95% rename from app/crowdb-kv-server/src/cli.rs rename to app/crowdb-paxos-kv-server/src/cli.rs index d7e63ad0a..7987280e8 100644 --- a/app/crowdb-kv-server/src/cli.rs +++ b/app/crowdb-paxos-kv-server/src/cli.rs @@ -2,15 +2,15 @@ // Licensed under the Apache License, Version 2.0. use clap::Parser; -use crowdb_protocol::KV_SERVER_MGMT_BASE; +use crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE; -/// `CrowDB` server — reference implementation wrapping the `crowdb_kv` library. +/// `CrowDB` server — reference implementation wrapping the `crowdb_paxos_kv` library. #[derive(Parser, Debug)] #[allow(clippy::struct_excessive_bools)] -#[command(name = "crowdb-kv-server", about = "CrowDB server daemon")] +#[command(name = "crowdb-paxos-kv-server", about = "CrowDB server daemon")] pub struct Cli { - /// HTTP management API listen port. Default: 10000. - #[arg(long, default_value_t = KV_SERVER_MGMT_BASE, value_parser = clap::value_parser!(u16).range(1..))] + /// HTTP management API listen port. Default: 10135. + #[arg(long, default_value_t = PAXOS_KV_SERVER_MGMT_BASE, value_parser = clap::value_parser!(u16).range(1..))] pub management_port: u16, /// HTTP management API bind address. @@ -18,7 +18,7 @@ pub struct Cli { pub management_addr: String, /// Reachable address advertised in the Group 0 service registry. - #[arg(long, env = "CROWDB_KV_MANAGEMENT_ADVERTISE_ADDR")] + #[arg(long, env = "CROWDB_PAXOS_KV_MANAGEMENT_ADVERTISE_ADDR")] pub management_advertise_addr: Option<std::net::IpAddr>, /// Node root directory. Derives `wal_root`=`<root>`/waldata, @@ -31,11 +31,11 @@ pub struct Cli { pub node_recovery_root: Option<std::path::PathBuf>, /// Optional TOML config for startup tunables in first-boot and restore modes. - /// When omitted, tunables use `CrowDBConfig::default()`. + /// When omitted, tunables use `PaxosKvConfig::default()`. #[arg(long)] pub config: Option<std::path::PathBuf>, - /// Port pool for crowdb-rpc `PxKvStore` listeners (comma/range format, e.g. "10100,10101,10110..10120"). + /// Port pool for crowdb-rpc `PxKvStore` listeners (comma/range format, e.g. "11135,11136,11145..11155"). #[arg(long)] pub ports: Option<String>, @@ -194,9 +194,9 @@ impl Cli { /// Returns the first complete-config validation error. pub fn apply_config_overrides( &self, - config: &mut crowdb_kv::common::config::CrowDBConfig, + config: &mut crowdb_paxos_kv::common::config::PaxosKvConfig, ) -> Result<(), String> { - use crowdb_kv::common::config::PxElectionConfig; + use crowdb_paxos_kv::common::config::PxElectionConfig; if let Some(profile) = self.election_profile.as_deref() { config.election = match profile { diff --git a/app/crowdb-kv-server/src/engine_collector.rs b/app/crowdb-paxos-kv-server/src/engine_collector.rs similarity index 91% rename from app/crowdb-kv-server/src/engine_collector.rs rename to app/crowdb-paxos-kv-server/src/engine_collector.rs index de35606dd..8f9387cc4 100644 --- a/app/crowdb-kv-server/src/engine_collector.rs +++ b/app/crowdb-paxos-kv-server/src/engine_collector.rs @@ -11,11 +11,13 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; -use crate::store_registry::KvStoreRegistry; +use crate::store_registry::PaxosKvStoreRegistry; use arc_swap::ArcSwap; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::kv::CrowdbTreeEngine; -use crowdb_kv::metrics::{Counter, Gauge, MetricsRegistry, MetricsRunner}; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::kv::CrowdbTreeEngine; +use crowdb_paxos_kv::metrics::{Counter, Gauge, MetricsRegistry, MetricsRunner}; + +mod resources; /// Registered Rust handles for one (store, group): Paxos gauges + /// snapshot.pages.c bridge counter. @@ -113,10 +115,11 @@ fn read_paxos_gauges_per_group(store: &Arc<PxKvStore>) -> std::collections::Hash #[allow(clippy::too_many_lines)] pub fn setup_engine_collector( registry: &Arc<Mutex<MetricsRegistry>>, - store_registry: &Arc<KvStoreRegistry>, + store_registry: &Arc<PaxosKvStoreRegistry>, runner: &mut MetricsRunner, ) { type Key = (u64, u64); + runner.set_sampler(resources::Resources::new(Arc::clone(store_registry))); let mut handles: Vec<(Key, Arc<EngineHandles>)> = Vec::new(); let mut rpc_handles: Vec<(u64, Arc<RpcTransportHandles>)> = Vec::new(); @@ -218,6 +221,14 @@ pub fn setup_engine_collector( let replica = group.local_replica(); let engine = replica.learner.engine(); if let Some(e) = engine.as_any().downcast_ref::<CrowdbTreeEngine>() { + if let Err(error) = e.capture_work_metrics() { + tracing::warn!( + store_id = *store_id, + group_id = key.1, + ?error, + "Tree metrics capture failed; retaining the prior window" + ); + } let s = e.stats(); let current = s.snapshot_pages_total; let previous = hd.last_snapshot_pages.swap(current, Ordering::Relaxed); @@ -269,11 +280,16 @@ pub fn setup_engine_collector( let stores2 = Arc::clone(store_registry); runner.set_cpp_flush( move |writer, window_secs, timestamp, rust_width, count_w, tps_w| { - for store in stores2.stores_snapshot().values() { + for (store_id, store) in stores2.stores_snapshot().iter() { store.for_each_group(|group| { let replica = group.local_replica(); let engine = replica.learner.engine(); if let Some(e) = engine.as_any().downcast_ref::<CrowdbTreeEngine>() { + let prefix = format!("s.{store_id}.g.{}", group.group_id()); + let work = e.format_work_metrics_window(&prefix, timestamp); + if !work.is_empty() { + let _ = std::io::Write::write_all(writer, work.as_bytes()); + } let cpp_max = e.max_name_len(); let shared_width = rust_width.max(cpp_max); let str = diff --git a/app/crowdb-paxos-kv-server/src/engine_collector/resources.rs b/app/crowdb-paxos-kv-server/src/engine_collector/resources.rs new file mode 100644 index 000000000..115f00eb9 --- /dev/null +++ b/app/crowdb-paxos-kv-server/src/engine_collector/resources.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; +use std::time::Instant; + +use crowdb_common::metrics::MetricsSampler; +use crowdb_paxos_kv::kv::CrowdbTreeEngine; +use crowdb_tree_ffi::runtime::{ResourceUsageLog, ResourceUsageSnapshot}; + +use crate::store_registry::PaxosKvStoreRegistry; + +pub(super) struct Resources { + stores: Arc<PaxosKvStoreRegistry>, + windows: ResourceUsageLog, +} + +impl Resources { + pub fn new(stores: Arc<PaxosKvStoreRegistry>) -> Self { + Self { + stores, + windows: ResourceUsageLog::default(), + } + } + + fn observe(&mut self, name: &str, value: &Result<ResourceUsageSnapshot, crowdb_tree_ffi::CtError>) { + match value { + Ok(value) => self.windows.observe(name, *value, Instant::now()), + Err(_) => self.windows.sample_failed(name), + } + } +} + +impl MetricsSampler for Resources { + fn sample(&mut self) { + self.windows.begin_sample(); + self.observe("host.tree", &self.stores.tree_runtime.resource_usage()); + let stores = self.stores.stores_snapshot(); + for (store_id, store) in stores.iter() { + store.for_each_group(|group| { + let replica = group.local_replica(); + let engine = replica.learner.engine(); + if let Some(engine) = engine.as_any().downcast_ref::<CrowdbTreeEngine>() { + let name = format!("s.{store_id}.g.{}.tree", group.group_id()); + self.observe(&name, &engine.handle().resource_usage()); + } + }); + } + self.windows.end_sample(Instant::now()); + } + + fn flush(&mut self, writer: &mut dyn std::io::Write, timestamp: &str) -> std::io::Result<()> { + self.windows.flush(writer, timestamp, Instant::now()) + } +} diff --git a/app/crowdb-kv-server/src/group0_control_plane.rs b/app/crowdb-paxos-kv-server/src/group0_control_plane.rs similarity index 97% rename from app/crowdb-kv-server/src/group0_control_plane.rs rename to app/crowdb-paxos-kv-server/src/group0_control_plane.rs index 846a45f5a..693f05ffa 100644 --- a/app/crowdb-kv-server/src/group0_control_plane.rs +++ b/app/crowdb-paxos-kv-server/src/group0_control_plane.rs @@ -8,11 +8,11 @@ use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use bytes::Bytes; -use crowdb_kv::cluster::group_operations::{ +use crowdb_paxos_kv::cluster::group_operations::{ KvGroupMutation, KvGroupOperationError, KvGroupOperations, KvGroupRead, KvGroupScanItem, KvGroupScanRequest, KvReadConsistency, KvRequestIdentity, }; -use crowdb_kv::cluster::PxKvStore; +use crowdb_paxos_kv::cluster::PxKvStore; static CONTROL_PLANE_INSTANCE_NONCE: AtomicU64 = AtomicU64::new(1); @@ -106,7 +106,7 @@ impl Group0ControlPlane { deadline_ms: 0, bounded: true, requested_scan_cutoff: scan_cutoff, - direction: crowdb_kv::kv::ScanDirection::Forward, + direction: crowdb_paxos_kv::kv::ScanDirection::Forward, }) .await?; Ok((scan.items, scan.truncated, scan.scan_cutoff)) diff --git a/app/crowdb-kv-server/src/lib.rs b/app/crowdb-paxos-kv-server/src/lib.rs similarity index 71% rename from app/crowdb-kv-server/src/lib.rs rename to app/crowdb-paxos-kv-server/src/lib.rs index ad87c785f..bd5b57515 100644 --- a/app/crowdb-kv-server/src/lib.rs +++ b/app/crowdb-paxos-kv-server/src/lib.rs @@ -1,12 +1,12 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! `crowdb-kv-server` library — exposes the binary's modules so integration +//! `crowdb-paxos-kv-server` library — exposes the binary's modules so integration //! tests under `tests/` can exercise CLI parsing, the management router, //! and the registry without spawning a process. //! //! The binary entry (`main.rs`) imports from this lib via -//! `use crowdb_kv_server::{cli, mgmt, startup, store_registry};`. +//! `use crowdb_paxos_kv_server::{cli, mgmt, startup, store_registry};`. pub mod background; pub mod cli; @@ -14,4 +14,5 @@ pub mod engine_collector; pub mod group0_control_plane; pub mod mgmt; pub mod recovery; +pub mod shutdown; pub mod store_registry; diff --git a/app/crowdb-kv-server/src/main.rs b/app/crowdb-paxos-kv-server/src/main.rs similarity index 76% rename from app/crowdb-kv-server/src/main.rs rename to app/crowdb-paxos-kv-server/src/main.rs index e60dc75d9..c7c8fabc7 100644 --- a/app/crowdb-kv-server/src/main.rs +++ b/app/crowdb-paxos-kv-server/src/main.rs @@ -1,9 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! `crowdb-kv-server` — `CrowDB` daemon entry point. +//! `crowdb-paxos-kv-server` — `CrowDB` daemon entry point. //! -//! Wraps the `crowdb_kv` library into a runnable server process with: +//! Wraps the `crowdb_paxos_kv` library into a runnable server process with: //! - CLI-driven startup of `PxKvStore` instances with `PxGroup`s. //! - HTTP management API for runtime topology control. //! - Graceful shutdown on SIGINT/SIGTERM. @@ -13,18 +13,18 @@ use std::net::SocketAddr; use std::sync::Arc; use clap::Parser; -use tracing::{debug, info, warn}; +use tracing::{debug, error, info, warn}; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::PxLocalReplicaRole; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::common::config::{CrowDBConfig, ServerConfig}; -use crowdb_kv::metrics::MetricsRunner; +use crowdb_paxos_kv::cluster::local_replica::PxLocalReplicaRole; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv::metrics::MetricsRunner; -use crowdb_kv_server::cli::{parse_id_list, parse_port_list, Cli}; -use crowdb_kv_server::mgmt::{self, persisted_port_for_store}; -use crowdb_kv_server::recovery::startup::create_group_with_wal; -use crowdb_kv_server::store_registry::KvStoreRegistry; +use crowdb_paxos_kv_server::cli::{parse_id_list, parse_port_list, Cli}; +use crowdb_paxos_kv_server::mgmt::{self, persisted_port_for_store}; +use crowdb_paxos_kv_server::recovery::startup::create_group_with_wal_on_runtime; +use crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry; #[tokio::main] #[allow(clippy::too_many_lines)] @@ -46,21 +46,21 @@ async fn main() { let _guards = if args.log { crowdb_common::logging::init_file_and_console_logging_split( &log_dir, - "crowdb-kv-server", + "crowdb-paxos-kv-server", args.log_max_file_mb, args.log_max_files, - crowdb_kv::common::logging::CROWDB_KV_DEFAULT_FILTER, + crowdb_paxos_kv::common::logging::CROWDB_PAXOS_KV_DEFAULT_FILTER, "warn", ) - .expect("failed to initialize crowdb-kv-server logging") + .expect("failed to initialize crowdb-paxos-kv-server logging") } else { - crowdb_kv::common::logging::init_file_logging( + crowdb_paxos_kv::common::logging::init_file_logging( &log_dir, - "crowdb-kv-server", + "crowdb-paxos-kv-server", args.log_max_file_mb, args.log_max_files, ) - .expect("failed to initialize crowdb-kv-server logging") + .expect("failed to initialize crowdb-paxos-kv-server logging") }; // Initialize the C++ spdlog async logger as a process-global resource. @@ -71,7 +71,7 @@ async fn main() { &cpp_level, args.log_max_file_mb, args.log_max_files, - "crowdb-kv-server-tree", + "crowdb-paxos-kv-server-tree", ); // Initialize the crowdb-rpc C++ spdlog logger (connection failures, @@ -82,7 +82,7 @@ async fn main() { &cpp_level, args.log_max_file_mb, args.log_max_files, - "crowdb-kv-server-rpc", + "crowdb-paxos-kv-server-rpc", ); // Optional err-to-stderr mirror for the C++ stacks. @@ -91,7 +91,7 @@ async fn main() { crowdb_rpc_ffi::add_log_stderr(stderr_level); } - info!("crowdb-kv-server starting..."); + info!("crowdb-paxos-kv-server starting..."); // Metrics runner: periodic flush to a dedicated metrics log file. let mut metrics_runner = create_metrics_runner( @@ -127,13 +127,13 @@ async fn main() { // omitted, use defaults. Paths are always derived from // --root via apply_root (fixed on-disk layout). let mut config = match args.config.as_ref() { - Some(path) => CrowDBConfig::load_from_file(path) + Some(path) => PaxosKvConfig::load_from_file(path) .unwrap_or_else(|e| panic!("failed to load config from {}: {e}", path.display())), - None => CrowDBConfig::default(), + None => PaxosKvConfig::default(), }; config.apply_root(&args.root); if let Some(root) = &args.node_recovery_root { - crowdb_kv_server::recovery::member_routes::configure(&config.config_root, root) + crowdb_paxos_kv_server::recovery::member_routes::configure(&config.config_root, root) .expect("invalid monitor member recovery source"); } @@ -145,7 +145,7 @@ async fn main() { .unwrap_or_else(|e| panic!("invalid config after CLI overrides: {e}")); let service_identity = (args.keepalive_interval > 0).then(|| { - crowdb_kv_server::background::identity::load_or_create( + crowdb_paxos_kv_server::background::identity::load_or_create( &config.config_root, args.instance_id, args.node_id, @@ -154,21 +154,25 @@ async fn main() { }); let registry = Arc::new( - KvStoreRegistry::try_with_config(config.clone()) + PaxosKvStoreRegistry::try_with_config(config.clone()) .unwrap_or_else(|error| panic!("failed to initialize WAL backend: {error}")) .with_metrics_registry(metrics_runner.as_ref().map_or_else( - || Arc::new(std::sync::Mutex::new(crowdb_kv::metrics::MetricsRegistry::new())), + || { + Arc::new(std::sync::Mutex::new( + crowdb_paxos_kv::metrics::MetricsRegistry::new(), + )) + }, |r| r.registry().clone(), )), ); - let _member_routes = crowdb_kv_server::recovery::member_routes::start(Arc::clone(®istry)); + let member_routes = crowdb_paxos_kv_server::recovery::member_routes::start(Arc::clone(®istry)); // Spawn a config file watcher for diff logging. Only when --config is // passed; with no toml there is nothing to watch. let watcher_old_config = Arc::new(std::sync::Mutex::new(config)); let _config_watcher = match args.config.as_ref() { Some(path) => { - match crowdb_common::config::watch::<CrowDBConfig, _>(std::path::Path::new(path), move |new| { + match crowdb_common::config::watch::<PaxosKvConfig, _>(std::path::Path::new(path), move |new| { let mut old = watcher_old_config.lock().unwrap(); crowdb_common::config::log_diff(&*old, new); *old = new.clone(); @@ -206,9 +210,9 @@ async fn main() { std::process::exit(1); }); - let router = mgmt::router(crowdb_kv_server::mgmt::operation_registry::AppState::new( - registry.clone(), - )); + let app_state = crowdb_paxos_kv_server::mgmt::operation_registry::AppState::new(registry.clone()); + let operation_cleanup = app_state.start_operation_cleanup(); + let router = mgmt::router(app_state); let listener = tokio::net::TcpListener::bind(mgmt_addr) .await .unwrap_or_else(|e| { @@ -243,12 +247,18 @@ async fn main() { // - First-boot mode: no group 0 on disk. Use --stores/--groups CLI // args (if given) to create stores; otherwise boot empty so the // operator can call POST /system/init. - let mut local_groups = crowdb_kv_server::recovery::restore::scan_local_groups(®istry.config.wal_root) - .await - .unwrap_or_else(|e| { - warn!(error = %e, wal_root = %registry.config.wal_root.display(), "scan_local_groups failed; treating as empty"); - Vec::new() - }); + let mut local_groups = + match crowdb_paxos_kv_server::recovery::restore::scan_local_groups(®istry.config.wal_root).await { + Ok(groups) => groups, + Err(error) => { + error!( + error = %error, + wal_root = %registry.config.wal_root.display(), + "scan_local_groups failed; refusing to continue startup" + ); + std::process::exit(1); + } + }; let cleanup_pending = mgmt::system_cleanup_pending(®istry.config.config_root) .expect("cannot inspect system cleanup fence"); if cleanup_pending { @@ -257,14 +267,18 @@ async fn main() { if local_groups.is_empty() { info!("first-boot mode: no group 0 on disk"); if let Some(b) = bootstrap.as_ref().filter(|_| !cleanup_pending) { - create_and_start_stores( + if let Err(error) = create_and_start_stores( &b.store_ids, &b.group_ids, b.replica_id, b.ports.clone(), registry.clone(), ) - .await; + .await + { + error!(%error, "first-boot store creation failed; refusing to continue with partial state"); + std::process::exit(1); + } } // Reconcile is a no-op without group 0; skip the call. } else { @@ -275,8 +289,17 @@ async fn main() { if bootstrap.is_some() { warn!("restore mode: --stores/--groups ignored (local disk is the source of truth)"); } - crowdb_kv_server::recovery::restore::load_local_groups(&local_groups, args.replica, ®istry).await; - crowdb_kv_server::recovery::reconcile::reconcile_with_group0(®istry).await; + if let Err(error) = crowdb_paxos_kv_server::recovery::restore::load_local_groups( + &local_groups, + args.replica, + ®istry, + ) + .await + { + error!(%error, "restore failed; refusing to continue with partial local state"); + std::process::exit(1); + } + crowdb_paxos_kv_server::recovery::reconcile::reconcile_with_group0(®istry).await; } // Start the keep-alive loop (registers under /srv/kv-server/<id>). @@ -297,20 +320,22 @@ async fn main() { .and_then(|s| s.listen_addr().map(|a| a.to_string())) .or_else(|| registry.first_port().map(|p| format!("{display_ip}:{p}"))) .unwrap_or_else(|| format!("http://{display_addr}")); - Some(crowdb_kv_server::background::keepalive::KeepAliveLoop::spawn( - registry.clone(), - identity, - mgmt_endpoint, - &group0_ep, - args.group0_management_seeds.clone(), - registry - .config - .node_root - .as_ref() - .map(|p| p.to_string_lossy().into_owned()) - .unwrap_or_default(), - args.keepalive_interval, - )) + Some( + crowdb_paxos_kv_server::background::keepalive::KeepAliveLoop::spawn( + registry.clone(), + identity, + mgmt_endpoint, + &group0_ep, + args.group0_management_seeds.clone(), + registry + .config + .node_root + .as_ref() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(), + args.keepalive_interval, + ), + ) } else { None }; @@ -319,7 +344,7 @@ async fn main() { // their compiled drivers prepared. Driver ticks acquire an exact local // leader tenure, so followers remain idle without loopback RPC. let domain_monitors = if args.binding_monitor_interval > 0 { - use crowdb_kv_server::background::domain_monitor::{ + use crowdb_paxos_kv_server::background::domain_monitor::{ spawn_domain_monitor_supervisor, ChunkKvRangeMonitorDriver, ChunkdbDynamicMonitorDriver, ChunkdbRangeMonitorDriver, DiskdbOwnershipMonitorDriver, DomainMonitorDrivers, }; @@ -330,7 +355,7 @@ async fn main() { Arc::new(ChunkdbDynamicMonitorDriver), Arc::new(ChunkKvRangeMonitorDriver::new()), Arc::new(DiskdbOwnershipMonitorDriver::new()), - Arc::new(crowdb_kv_server::background::domain_monitor::DiskdbPlacementMonitorDriver), + Arc::new(crowdb_paxos_kv_server::background::domain_monitor::DiskdbPlacementMonitorDriver), ]), std::time::Duration::from_secs(args.binding_monitor_interval), )) @@ -344,7 +369,7 @@ async fn main() { // appear in the metrics log alongside the Rust-side metrics. if let Some(ref mut runner) = metrics_runner { let reg = runner.registry().clone(); - crowdb_kv_server::engine_collector::setup_engine_collector(®, ®istry, runner); + crowdb_paxos_kv_server::engine_collector::setup_engine_collector(®, ®istry, runner); runner.start(); info!(interval_secs = args.metrics_interval, "metrics runner started"); } @@ -357,6 +382,9 @@ async fn main() { tracing::error!("management HTTP server error: {e}"); }); + operation_cleanup.abort(); + let _ = operation_cleanup.await; + // Graceful cascade shutdown (PxKvStore → PxGroup → replicas) if let Some(ref mut runner) = metrics_runner { runner.stop().await; @@ -370,7 +398,7 @@ async fn main() { monitors.stop_and_wait().await; info!("domain monitor supervisor stopped"); } - graceful_shutdown(registry).await; + graceful_shutdown(registry, member_routes, metrics_runner).await; } /// Create a metrics runner if interval > 0, else None. Does NOT start @@ -384,9 +412,13 @@ fn create_metrics_runner( if interval_secs == 0 { return None; } - let metrics_file = - crowdb_kv::common::logging::open_metrics_log(log_dir, "crowdb-kv-server", max_file_mb, max_files) - .expect("failed to open metrics log file"); + let metrics_file = crowdb_paxos_kv::common::logging::open_metrics_log( + log_dir, + "crowdb-paxos-kv-server", + max_file_mb, + max_files, + ) + .expect("failed to open metrics log file"); let runner = MetricsRunner::new(metrics_file, interval_secs); Some(runner) } @@ -470,8 +502,8 @@ async fn create_and_start_stores( group_ids: &[u64], replica_id: u64, ports: Vec<u16>, - registry: Arc<KvStoreRegistry>, -) { + registry: Arc<PaxosKvStoreRegistry>, +) -> Result<(), String> { debug!( store_count = store_ids.len(), group_count = group_ids.len(), @@ -515,7 +547,7 @@ async fn create_and_start_stores( store_id, group_id, replica_id, "creating PxGroup with local replica" ); - let group = match create_group_with_wal( + let group = match create_group_with_wal_on_runtime( store_id, group_id, replica_id, @@ -523,21 +555,22 @@ async fn create_and_start_stores( ®istry.config, registry.wal_backend.clone(), registry.crowtree_backend, + registry.tree_runtime.clone(), ) .await { Ok(group) => group, Err(e) => { - tracing::error!(s = store_id, g = group_id, error = %e, "failed to create WAL-backed group"); - continue; + return Err(format!( + "failed to create WAL-backed store {store_id} group {group_id}: {e}" + )); } }; store.add_group(group); } if let Err(e) = store.start().await { - tracing::error!(s = store_id, port, error = %e, "failed to start store, skipping"); - continue; + return Err(format!("failed to start store {store_id} on port {port}: {e}")); } // Wire the shared transport into all existing remote replicas. @@ -556,12 +589,17 @@ async fn create_and_start_stores( store_count = registry.store_count(), "all stores started, management API ready" ); + Ok(()) } /// Gracefully cascade-shutdown every store via [`PxKvStore::shutdown`]. /// /// Continues on errors; aggregates `critical:` messages to the operator. -async fn graceful_shutdown(registry: Arc<KvStoreRegistry>) { +async fn graceful_shutdown( + registry: Arc<PaxosKvStoreRegistry>, + member_routes: crowdb_paxos_kv_server::recovery::member_routes::Worker, + metrics_runner: Option<MetricsRunner>, +) { info!( store_count = registry.store_count(), "initiating graceful shutdown of crowdb-rpc stores" @@ -572,28 +610,24 @@ async fn graceful_shutdown(registry: Arc<KvStoreRegistry>) { crowdb_tree_ffi::ct_flush_logging(); crowdb_rpc_ffi::flush_logging(); - let mut total_errors = 0usize; - for (store_id, store) in registry.stores_snapshot().iter() { - let report = store - .shutdown(std::time::Duration::from_millis( - ServerConfig::DEFAULT.shutdown_timeout_ms, - )) - .await; - if !report.is_clean() { - total_errors += report.errors.len(); - for err in &report.errors { - tracing::error!(s = *store_id, "{err}"); - } - } - } - if total_errors == 0 { - info!("crowdb-kv-server shut down cleanly"); + let report = crowdb_paxos_kv_server::shutdown::shutdown_stores(®istry).await; + if report.is_clean() { + info!("crowdb-paxos-kv-server shut down cleanly"); } else { + for error in &report.errors { + tracing::error!(error = %error, "server shutdown stage failed"); + } tracing::warn!( - error_count = total_errors, - "crowdb-kv-server shut down with errors (see critical: logs above)" + error_count = report.errors.len(), + "crowdb-paxos-kv-server shut down with errors" ); } + member_routes.stop().await; + drop(metrics_runner); + registry.tree_runtime.stop_admission(); + // Drop stores and their Trees before the shared runtime and loggers. An + // incomplete accepted drain above never reaches this destruction point. + drop(registry); // Final flush + stop the C++ spdlog async loggers. All Crowdbtree // instances are now dropped (or about to be), so this is safe. diff --git a/app/crowdb-kv-server/src/mgmt.rs b/app/crowdb-paxos-kv-server/src/mgmt.rs similarity index 96% rename from app/crowdb-kv-server/src/mgmt.rs rename to app/crowdb-paxos-kv-server/src/mgmt.rs index 749a931cc..ed29e5d14 100644 --- a/app/crowdb-kv-server/src/mgmt.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt.rs @@ -15,6 +15,7 @@ pub(crate) mod group_creation; mod group_ops; pub mod operation_registry; mod replica_ops; +mod runtime_shutdown; mod store_ops; mod system_bootstrap; pub use system_bootstrap::system_cleanup_pending; @@ -28,9 +29,9 @@ use axum::{Json, Router}; use serde::Serialize; use utoipa::{OpenApi, ToSchema}; -use crowdb_kv::cluster::group_config::GroupConfigStore; -use crowdb_kv::cluster::node_config::NodeConfigStore; -use crowdb_kv::cluster::status::{GroupStatus, RemoteStatus, ReplicaStatus, StoreStatus}; +use crowdb_paxos_kv::cluster::group_config::GroupConfigStore; +use crowdb_paxos_kv::cluster::node_config::NodeConfigStore; +use crowdb_paxos_kv::cluster::status::{GroupStatus, RemoteStatus, ReplicaStatus, StoreStatus}; use crowdb_protocol::mgmt::{ AddGroupRequest, AddStoreRequest, GroupSummary, HealthResponse, MetricField, MetricPoint, MetricsResponse, RemoteListResponse, RemoteReplicaInfo, StepDownRequest, StepDownResult, StoreDetail, @@ -125,7 +126,9 @@ pub async fn persisted_port_for_store(config_root: &std::path::Path, store_id: u } /// Extract the first non-zero port from a list of group members. -fn extract_port_from_members(members: &[crowdb_kv::cluster::group_config::PxGroupMember]) -> Option<u16> { +fn extract_port_from_members( + members: &[crowdb_paxos_kv::cluster::group_config::PxGroupMember], +) -> Option<u16> { members .iter() .find_map(|m| m.endpoint.rsplit(':').next().and_then(|p| p.parse::<u16>().ok())) diff --git a/app/crowdb-kv-server/src/mgmt/discovery.rs b/app/crowdb-paxos-kv-server/src/mgmt/discovery.rs similarity index 100% rename from app/crowdb-kv-server/src/mgmt/discovery.rs rename to app/crowdb-paxos-kv-server/src/mgmt/discovery.rs diff --git a/app/crowdb-kv-server/src/mgmt/group_creation.rs b/app/crowdb-paxos-kv-server/src/mgmt/group_creation.rs similarity index 99% rename from app/crowdb-kv-server/src/mgmt/group_creation.rs rename to app/crowdb-paxos-kv-server/src/mgmt/group_creation.rs index b55412479..c44d69ce3 100644 --- a/app/crowdb-kv-server/src/mgmt/group_creation.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/group_creation.rs @@ -69,7 +69,7 @@ pub(super) async fn accept( )) }; } - let persisted = crowdb_kv::cluster::node_config::NodeConfigStore::new(root) + let persisted = crowdb_paxos_kv::cluster::node_config::NodeConfigStore::new(root) .load() .await .map_err(storage_error)?; diff --git a/app/crowdb-kv-server/src/mgmt/group_ops.rs b/app/crowdb-paxos-kv-server/src/mgmt/group_ops.rs similarity index 95% rename from app/crowdb-kv-server/src/mgmt/group_ops.rs rename to app/crowdb-paxos-kv-server/src/mgmt/group_ops.rs index f0800c4b5..ad8f91db6 100644 --- a/app/crowdb-kv-server/src/mgmt/group_ops.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/group_ops.rs @@ -12,8 +12,8 @@ use serde::{Deserialize, Serialize}; use tracing::{debug, info, info_span, Instrument}; use utoipa::ToSchema; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::local_replica::PxLocalReplicaRole; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::PxLocalReplicaRole; use crowdb_protocol::mgmt::{ AddGroupInitialRole, AddGroupRequest, GroupSummary, JoinGroupRequest, StepDownRequest, StepDownResult, }; @@ -235,7 +235,7 @@ pub(super) async fn add_group( AddGroupInitialRole::Leader => PxLocalReplicaRole::Leader, AddGroupInitialRole::Follower => PxLocalReplicaRole::Follower, }; - let group = crate::recovery::startup::create_group_with_wal( + let group = crate::recovery::startup::create_group_with_wal_on_runtime( sid, req.group_id, req.replica_id, @@ -243,6 +243,7 @@ pub(super) async fn add_group( &state.config, state.wal_backend.clone(), state.crowtree_backend, + state.tree_runtime.clone(), ) .await .map_err(|e| { @@ -384,7 +385,7 @@ pub(super) async fn join_snapshot( peer_endpoint = %req.peer_endpoint, "joining PxGroup via snapshot pull" ); - let group = crate::recovery::startup::create_group_with_wal( + let group = crate::recovery::startup::create_group_with_wal_on_runtime( sid, gid, req.replica_id, @@ -392,6 +393,7 @@ pub(super) async fn join_snapshot( &state.config, state.wal_backend.clone(), state.crowtree_backend, + state.tree_runtime.clone(), ) .await .map_err(|e| { @@ -423,7 +425,7 @@ pub(super) async fn join_snapshot( } }; // The frontier moved from 0 to `at_slot` (or further, if a concurrent - // catch-up already advanced it) after `create_group_with_wal` computed + // catch-up already advanced it) after `create_group_with_wal_on_runtime` computed // `next_slot` from a still-empty replica; recompute so a future // proposal (once this replica becomes leader-eligible) doesn't clash // with an already-applied slot. @@ -474,14 +476,17 @@ pub(super) async fn remove_group( .get_store(sid) .ok_or_else(|| err_json(StatusCode::NOT_FOUND, format!("store {sid} not found")))?; + let membership_guard = store.membership_guard(gid); + let _membership_guard = membership_guard.lock().await; + // Load the group only after serializing membership changes. Otherwise a + // request that waited behind a prior mutation could retain an old Arc and + // then remove or overwrite the newly published topology. let group = store.get_group(gid).ok_or_else(|| { err_json( StatusCode::NOT_FOUND, format!("group {gid} not found in store {sid}"), ) })?; - let membership_guard = store.membership_guard(gid); - let _membership_guard = membership_guard.lock().await; let expected_epoch = super::replica_ops::check_expected_epoch(&headers, group.membership_epoch())?; if expected_epoch == Some(group.membership_epoch()) { return Err(err_json( @@ -493,6 +498,7 @@ pub(super) async fn remove_group( )); } info!(s = sid, g = gid, "removing PxGroup via management API"); + super::runtime_shutdown::before_group_removal(&state, &group).await?; super::group_creation::retire(&state, sid, gid)?; if !store.remove_group(gid) { return Err(err_json( @@ -519,7 +525,8 @@ pub(super) async fn remove_group( } // Update node-config.json so the group does not resurrect on restart. - let node_config_store = crowdb_kv::cluster::node_config::NodeConfigStore::new(&state.config.config_root); + let node_config_store = + crowdb_paxos_kv::cluster::node_config::NodeConfigStore::new(&state.config.config_root); if let Err(e) = node_config_store.remove_group(sid, gid).await { tracing::warn!(s = sid, g = gid, error = %e, "failed to update node_config; continuing"); } @@ -643,7 +650,7 @@ pub(super) async fn group_readiness( if group.local_replica().voting { voting_count += 1; - if status.local_replica.status != crowdb_kv::cluster::status::StatusLevel::Unhealthy { + if status.local_replica.status == crowdb_paxos_kv::cluster::status::StatusLevel::Ok { reachable_count += 1; } } @@ -651,7 +658,7 @@ pub(super) async fn group_readiness( for remote in &status.remotes { if remote.voting { voting_count += 1; - if remote.status != crowdb_kv::cluster::status::StatusLevel::Unhealthy { + if remote.status == crowdb_paxos_kv::cluster::status::StatusLevel::Ok { reachable_count += 1; } } @@ -741,13 +748,12 @@ pub(super) async fn flush_group( ) })?; - // `KVEngine::flush` forces a freeze + drain of every memtable up to - // `contiguous_slot` into L1 (in-memory only; never touches the page - // store). Cheap when L0 is empty. Used by the bench's - // `--flush-after-prepopulate` flag to produce a clean L1-only scan - // baseline; also useful as an admin drain. - group.local_replica().learner.engine().flush(); - info!(s = sid, g = gid, "engine flush requested via management API"); + let outcome = crowdb_paxos_kv::kv::flush_engine(group.local_replica().learner.engine_arc()) + .await + .map_err(|error| err_json(StatusCode::SERVICE_UNAVAILABLE, error))?; + outcome + .status() + .map_err(|error| err_json(StatusCode::SERVICE_UNAVAILABLE, error.to_owned()))?; Ok(Json(FlushResult { store_id: sid, group_id: gid, @@ -764,7 +770,7 @@ pub(super) type WipeResult = crowdb_protocol::mgmt::WipeResult; /// group0 sysdata + store/group/replica topology. /// /// Flow: resolve → failed-WAL reject → unwired no-op → step down if -/// leader → delete on-disk WAL + engine dirs → `create_group_with_wal` +/// leader → delete on-disk WAL + engine dirs → `create_group_with_wal_on_runtime` /// (replays empty WAL → fresh state, restores remotes from /// node-config) → quorum-1 self-election → `store.add_group` (replaces /// old group, cancels old tenure, spawns fresh election driver). The @@ -829,6 +835,8 @@ pub(super) async fn wipe_user_data( })); } + super::runtime_shutdown::before_group_removal(&state, &group).await?; + // Step down if leader so the group stops accepting writes before // the data is destroyed. `accepted: false` (not leader) is fine. let step = group.step_down_if_leader("wipe-user-data"); @@ -848,7 +856,7 @@ pub(super) async fn wipe_user_data( // above, the old group's Arc refcount reaches zero (assuming no // in-flight tasks hold strong clones — they observe cancel within // one RPC timeout) and the old WalEngine + MemBlockDevice + engine - // are freed. Doing this BEFORE create_group_with_wal ensures the old + // are freed. Doing this BEFORE create_group_with_wal_on_runtime ensures the old // group's memory is released even if WAL replay is slow. store.remove_group(gid); @@ -856,16 +864,16 @@ pub(super) async fn wipe_user_data( // not tokio::fs::remove_dir_all: the mem-block backend stores // segments in-memory (BTreeMap<PathBuf, Vec<u8>>), and // tokio::fs::remove_dir_all is a no-op on it. Without this, the old - // WAL segments persist and create_group_with_wal replays them, + // WAL segments persist and create_group_with_wal_on_runtime replays them, // causing multi-second stalls and preventing the wipe from actually // wiping anything. wipe_wal_and_engine_dirs(&state, sid, gid).await?; - // Recreate the group with a fresh WAL + engine. `create_group_with_wal` + // Recreate the group with a fresh WAL + engine. `create_group_with_wal_on_runtime` // replays the now-empty WAL (→ slot 0), creates fresh WalEngine + // engine, and `maybe_apply_persisted_config` restores the remote // membership from node-config.json (which the wipe did not touch). - let new_group = crate::recovery::startup::create_group_with_wal( + let new_group = crate::recovery::startup::create_group_with_wal_on_runtime( sid, gid, replica_id, @@ -873,6 +881,7 @@ pub(super) async fn wipe_user_data( &state.config, state.wal_backend.clone(), state.crowtree_backend, + state.tree_runtime.clone(), ) .await .map_err(|e| { diff --git a/app/crowdb-kv-server/src/mgmt/operation_registry.rs b/app/crowdb-paxos-kv-server/src/mgmt/operation_registry.rs similarity index 80% rename from app/crowdb-kv-server/src/mgmt/operation_registry.rs rename to app/crowdb-paxos-kv-server/src/mgmt/operation_registry.rs index a21aa8254..6778a03f3 100644 --- a/app/crowdb-kv-server/src/mgmt/operation_registry.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/operation_registry.rs @@ -153,6 +153,26 @@ impl OperationRegistry { true }); } + + /// Start the process-local cleanup loop for completed operations. + /// + /// Pending and running operations are retained across cleanup ticks. The + /// returned task is owned by the server and must be aborted during + /// shutdown before the registry is dropped. + pub(crate) fn start_cleanup( + self: &Arc<Self>, + interval: Duration, + ttl: Duration, + ) -> tokio::task::JoinHandle<()> { + let registry = Arc::clone(self); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + loop { + ticker.tick().await; + registry.cleanup_expired(ttl); + } + }) + } } impl Default for OperationRegistry { @@ -169,16 +189,16 @@ fn now_ms() -> u64 { /// Wrapper for shared state passed to HTTP handlers — bundles the store /// registry and the operation registry. Implements `Deref` to -/// `KvStoreRegistry` so existing handlers that call `state.stores`, +/// `PaxosKvStoreRegistry` so existing handlers that call `state.stores`, /// `state.get_store()`, etc. work unchanged. #[derive(Clone)] pub struct AppState { - pub(crate) registry: Arc<crate::store_registry::KvStoreRegistry>, + pub(crate) registry: Arc<crate::store_registry::PaxosKvStoreRegistry>, pub(crate) operations: Arc<OperationRegistry>, } impl std::ops::Deref for AppState { - type Target = crate::store_registry::KvStoreRegistry; + type Target = crate::store_registry::PaxosKvStoreRegistry; fn deref(&self) -> &Self::Target { &self.registry } @@ -186,19 +206,28 @@ impl std::ops::Deref for AppState { impl AppState { #[must_use] - pub fn new(registry: Arc<crate::store_registry::KvStoreRegistry>) -> Self { + pub fn new(registry: Arc<crate::store_registry::PaxosKvStoreRegistry>) -> Self { Self { registry, operations: Arc::new(OperationRegistry::new()), } } + /// Start cleanup for completed management operations. Operation IDs are + /// process-local and expire after the retention window; callers receive + /// 404 once a completed record has been collected. + #[must_use] + pub fn start_operation_cleanup(&self) -> tokio::task::JoinHandle<()> { + self.operations + .start_cleanup(Duration::from_secs(300), Duration::from_secs(3600)) + } + /// Construct with an explicit operation registry (for testing or when /// the registry must be shared across routers). #[must_use] #[allow(dead_code)] pub(crate) fn with_operations( - registry: Arc<crate::store_registry::KvStoreRegistry>, + registry: Arc<crate::store_registry::PaxosKvStoreRegistry>, operations: Arc<OperationRegistry>, ) -> Self { Self { registry, operations } diff --git a/app/crowdb-kv-server/src/mgmt/replica_ops.rs b/app/crowdb-paxos-kv-server/src/mgmt/replica_ops.rs similarity index 96% rename from app/crowdb-kv-server/src/mgmt/replica_ops.rs rename to app/crowdb-paxos-kv-server/src/mgmt/replica_ops.rs index b1cc1bd6b..333a4f06a 100644 --- a/app/crowdb-kv-server/src/mgmt/replica_ops.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/replica_ops.rs @@ -8,9 +8,9 @@ use axum::http::{HeaderMap, StatusCode}; use axum::Json; use tracing::{debug, info}; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::remote_replica::PxRemoteReplica; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::remote_replica::PxRemoteReplica; use crowdb_protocol::mgmt::{RemoteListResponse, RemoteReplicaInfo, TopologyResponse}; use super::{err_json, ErrorResponse, RegistryArc}; @@ -79,14 +79,18 @@ pub(super) async fn add_remote_replicas( let store = state .get_store(sid) .ok_or_else(|| err_json(StatusCode::NOT_FOUND, format!("store {sid} not found")))?; + let membership_guard = store.membership_guard(gid); + let _membership_guard = membership_guard.lock().await; + // Resolve the group after taking the membership guard. A concurrent + // membership mutation may have published a replacement while this + // request was waiting; continuing with the earlier Arc would apply the + // update to stale topology and can overwrite the newer membership. let group = store.get_group(gid).ok_or_else(|| { err_json( StatusCode::NOT_FOUND, format!("group {gid} not found in store {sid}"), ) })?; - let membership_guard = store.membership_guard(gid); - let _membership_guard = membership_guard.lock().await; let expected_epoch = check_expected_epoch(&headers, group.membership_epoch())?; let local_id = group.local_replica().id; @@ -186,14 +190,14 @@ pub(super) async fn remove_remote_replica( let store = state .get_store(sid) .ok_or_else(|| err_json(StatusCode::NOT_FOUND, format!("store {sid} not found")))?; + let membership_guard = store.membership_guard(gid); + let _membership_guard = membership_guard.lock().await; let group = store.get_group(gid).ok_or_else(|| { err_json( StatusCode::NOT_FOUND, format!("group {gid} not found in store {sid}"), ) })?; - let membership_guard = store.membership_guard(gid); - let _membership_guard = membership_guard.lock().await; let expected_epoch = check_expected_epoch(&headers, group.membership_epoch())?; let local_id = group.local_replica().id; @@ -306,14 +310,14 @@ pub(super) async fn batch_add_remote_replicas( let store = state .get_store(sid) .ok_or_else(|| err_json(StatusCode::NOT_FOUND, format!("store {sid} not found")))?; + let membership_guard = store.membership_guard(gid); + let _membership_guard = membership_guard.lock().await; let group = store.get_group(gid).ok_or_else(|| { err_json( StatusCode::NOT_FOUND, format!("group {gid} not found in store {sid}"), ) })?; - let membership_guard = store.membership_guard(gid); - let _membership_guard = membership_guard.lock().await; let expected_epoch = check_expected_epoch(&headers, group.membership_epoch())?; let local_id = group.local_replica().id; @@ -433,7 +437,10 @@ pub(super) fn check_expected_epoch( Ok(Some(expected)) } -fn remotes_match_existing(group: &crowdb_kv::cluster::group::PxGroup, remotes: &[RemoteReplicaInfo]) -> bool { +fn remotes_match_existing( + group: &crowdb_paxos_kv::cluster::group::PxGroup, + remotes: &[RemoteReplicaInfo], +) -> bool { let existing = group.remote_replica_info(); remotes.iter().all(|requested| { existing.iter().any(|(id, endpoint, voting)| { diff --git a/app/crowdb-paxos-kv-server/src/mgmt/runtime_shutdown.rs b/app/crowdb-paxos-kv-server/src/mgmt/runtime_shutdown.rs new file mode 100644 index 000000000..f34c91922 --- /dev/null +++ b/app/crowdb-paxos-kv-server/src/mgmt/runtime_shutdown.rs @@ -0,0 +1,28 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{err_json, ErrorResponse, RegistryArc}; +use axum::{http::StatusCode, Json}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use std::time::Duration; + +pub(super) async fn before_group_removal( + state: &RegistryArc, + group: &PxGroup, +) -> Result<(), (StatusCode, Json<ErrorResponse>)> { + if !group.local_replica().learner.engine().requires_contiguous_apply() { + return Ok(()); + } + let report = group + .shutdown(Duration::from_millis(state.config.server.shutdown_timeout_ms)) + .await; + if report.is_clean() { + return Ok(()); + } + Err(err_json( + StatusCode::SERVICE_UNAVAILABLE, + format!( + "runtime group retained; shutdown incomplete: {}", + report.errors.join("; ") + ), + )) +} diff --git a/app/crowdb-kv-server/src/mgmt/store_ops.rs b/app/crowdb-paxos-kv-server/src/mgmt/store_ops.rs similarity index 90% rename from app/crowdb-kv-server/src/mgmt/store_ops.rs rename to app/crowdb-paxos-kv-server/src/mgmt/store_ops.rs index ec6d89446..be48c8c7d 100644 --- a/app/crowdb-kv-server/src/mgmt/store_ops.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/store_ops.rs @@ -11,9 +11,8 @@ use axum::http::StatusCode; use axum::Json; use tracing::{debug, info}; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::common::config::ServerConfig; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; use crowdb_protocol::mgmt::{AddStoreRequest, StoreDetail, StoreListResponse, StoreSummary}; use super::{err_json, ErrorResponse, RegistryArc}; @@ -184,11 +183,15 @@ pub(super) async fn remove_store( ) -> Result<StatusCode, (StatusCode, Json<ErrorResponse>)> { info!(s = sid, "removing PxKvStore via management API"); let store = state - .remove_store(sid) + .get_store(sid) .ok_or_else(|| err_json(StatusCode::NOT_FOUND, format!("store {sid} not found")))?; + let runtime = store.has_runtime_engine(); + if !runtime { + let _ = state.remove_store(sid); + } let report = store .shutdown(std::time::Duration::from_millis( - ServerConfig::DEFAULT.shutdown_timeout_ms, + state.config.server.shutdown_timeout_ms, )) .await; if !report.is_clean() { @@ -197,6 +200,19 @@ pub(super) async fn remove_store( } } + if runtime { + if !report.is_clean() { + return Err(err_json( + StatusCode::SERVICE_UNAVAILABLE, + format!( + "runtime store retained; shutdown incomplete: {}", + report.errors.join("; ") + ), + )); + } + let _ = state.remove_store(sid); + } + // Delete the engine store dir (cascades all group subdirs). let engine_store_dir = state.config.data_root.join(format!("store{sid}")); if let Err(e) = tokio::fs::remove_dir_all(&engine_store_dir).await { @@ -214,7 +230,8 @@ pub(super) async fn remove_store( } // Update node-config.json so the store does not resurrect on restart. - let node_config_store = crowdb_kv::cluster::node_config::NodeConfigStore::new(&state.config.config_root); + let node_config_store = + crowdb_paxos_kv::cluster::node_config::NodeConfigStore::new(&state.config.config_root); if let Err(e) = node_config_store.remove_store(sid).await { tracing::warn!(s = sid, error = %e, "failed to update node_config; continuing"); } diff --git a/app/crowdb-kv-server/src/mgmt/system_bootstrap.rs b/app/crowdb-paxos-kv-server/src/mgmt/system_bootstrap.rs similarity index 99% rename from app/crowdb-kv-server/src/mgmt/system_bootstrap.rs rename to app/crowdb-paxos-kv-server/src/mgmt/system_bootstrap.rs index 4a5cef094..383426002 100644 --- a/app/crowdb-kv-server/src/mgmt/system_bootstrap.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/system_bootstrap.rs @@ -29,7 +29,7 @@ struct AcceptedBootstrap { identity: SystemBootstrapIdentity, } -pub(super) struct BootstrapExecution(Arc<crate::store_registry::KvStoreRegistry>); +pub(super) struct BootstrapExecution(Arc<crate::store_registry::PaxosKvStoreRegistry>); impl Drop for BootstrapExecution { fn drop(&mut self) { diff --git a/app/crowdb-kv-server/src/mgmt/system_bootstrap/cleanup.rs b/app/crowdb-paxos-kv-server/src/mgmt/system_bootstrap/cleanup.rs similarity index 98% rename from app/crowdb-kv-server/src/mgmt/system_bootstrap/cleanup.rs rename to app/crowdb-paxos-kv-server/src/mgmt/system_bootstrap/cleanup.rs index 0f1fc9ed9..028f37273 100644 --- a/app/crowdb-kv-server/src/mgmt/system_bootstrap/cleanup.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/system_bootstrap/cleanup.rs @@ -86,7 +86,7 @@ pub(crate) async fn cleanup( if state.contains_store(0) { super::super::store_ops::remove_store(State(state.clone()), axum::extract::Path(0)).await?; } - let node_config = crowdb_kv::cluster::node_config::NodeConfigStore::new(root); + let node_config = crowdb_paxos_kv::cluster::node_config::NodeConfigStore::new(root); node_config .remove_store(0) .await diff --git a/app/crowdb-kv-server/src/mgmt/system_init.rs b/app/crowdb-paxos-kv-server/src/mgmt/system_init.rs similarity index 94% rename from app/crowdb-kv-server/src/mgmt/system_init.rs rename to app/crowdb-paxos-kv-server/src/mgmt/system_init.rs index 7ab6e60fe..c3b979652 100644 --- a/app/crowdb-kv-server/src/mgmt/system_init.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/system_init.rs @@ -8,10 +8,10 @@ use axum::http::StatusCode; use axum::Json; use tracing::info; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::PxLocalReplicaRole; -use crowdb_kv::cluster::status::{StatusLevel, StoreStatus}; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::PxLocalReplicaRole; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::status::{StatusLevel, StoreStatus}; use crowdb_protocol::mgmt::{HealthResponse, SystemInitRequest, SystemInitResponse}; use super::{err_json, ErrorResponse, RegistryArc}; @@ -122,7 +122,7 @@ pub(super) async fn system_init( )); } - let group = crate::recovery::startup::create_group_with_wal( + let group = crate::recovery::startup::create_group_with_wal_on_runtime( SYSTEM_STORE_ID, SYSTEM_GROUP_ID, req.replica_id, @@ -134,6 +134,7 @@ pub(super) async fn system_init( &state.config, state.wal_backend.clone(), state.crowtree_backend, + state.tree_runtime.clone(), ) .await .map_err(|e| { diff --git a/app/crowdb-kv-server/src/mgmt/system_join.rs b/app/crowdb-paxos-kv-server/src/mgmt/system_join.rs similarity index 100% rename from app/crowdb-kv-server/src/mgmt/system_join.rs rename to app/crowdb-paxos-kv-server/src/mgmt/system_join.rs diff --git a/app/crowdb-kv-server/src/mgmt/system_store.rs b/app/crowdb-paxos-kv-server/src/mgmt/system_store.rs similarity index 94% rename from app/crowdb-kv-server/src/mgmt/system_store.rs rename to app/crowdb-paxos-kv-server/src/mgmt/system_store.rs index 2ae1e5e47..8d76e169f 100644 --- a/app/crowdb-kv-server/src/mgmt/system_store.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/system_store.rs @@ -5,8 +5,8 @@ use super::{err_json, system_bootstrap::ManagementError, RegistryArc}; use axum::http::StatusCode; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; use std::{net::SocketAddr, sync::Arc}; use tracing::info; diff --git a/app/crowdb-kv-server/src/mgmt/topology.rs b/app/crowdb-paxos-kv-server/src/mgmt/topology.rs similarity index 94% rename from app/crowdb-kv-server/src/mgmt/topology.rs rename to app/crowdb-paxos-kv-server/src/mgmt/topology.rs index 277cdd899..f97a1e509 100644 --- a/app/crowdb-kv-server/src/mgmt/topology.rs +++ b/app/crowdb-paxos-kv-server/src/mgmt/topology.rs @@ -8,7 +8,7 @@ use axum::response::IntoResponse; use serde::Deserialize; use utoipa::ToSchema; -use crowdb_kv::cluster::status::StoreStatus; +use crowdb_paxos_kv::cluster::status::StoreStatus; use crowdb_protocol::mgmt::{MetricField, MetricPoint, MetricsResponse, TopologyResponse}; use super::RegistryArc; @@ -53,7 +53,7 @@ pub(super) async fn metrics( State(state): State<RegistryArc>, Query(q): Query<MetricsQuery>, ) -> impl IntoResponse { - let timestamp = crowdb_kv::metrics::iso8601_now(); + let timestamp = crowdb_paxos_kv::metrics::iso8601_now(); let window_secs = 5.0; // approximate — snapshot path does not track elapsed let metrics: Vec<MetricPoint> = state .metrics_registry @@ -76,8 +76,8 @@ pub(super) async fn metrics( } #[allow(clippy::cast_precision_loss)] -fn metric_point_to_dto(p: &crowdb_kv::metrics::MetricPoint) -> MetricPoint { - use crowdb_kv::metrics::MetricPoint as KvMetricPoint; +fn metric_point_to_dto(p: &crowdb_paxos_kv::metrics::MetricPoint) -> MetricPoint { + use crowdb_paxos_kv::metrics::MetricPoint as KvMetricPoint; let kind = p.kind().to_string(); let fields = match p { KvMetricPoint::Counter { diff --git a/app/crowdb-kv-server/src/recovery.rs b/app/crowdb-paxos-kv-server/src/recovery.rs similarity index 100% rename from app/crowdb-kv-server/src/recovery.rs rename to app/crowdb-paxos-kv-server/src/recovery.rs diff --git a/app/crowdb-kv-server/src/recovery/group_rebuild.rs b/app/crowdb-paxos-kv-server/src/recovery/group_rebuild.rs similarity index 94% rename from app/crowdb-kv-server/src/recovery/group_rebuild.rs rename to app/crowdb-paxos-kv-server/src/recovery/group_rebuild.rs index 3e3f9dc4e..01487bc73 100644 --- a/app/crowdb-kv-server/src/recovery/group_rebuild.rs +++ b/app/crowdb-paxos-kv-server/src/recovery/group_rebuild.rs @@ -13,10 +13,10 @@ //! `PxLocalReplica::new_inheriting_election_state` so a rebuild does //! not reset the cluster's election term and trigger a fresh race. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::local_replica::PxLocalReplica; -use crowdb_kv::cluster::remote_replica::PxRemoteReplica; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::PxLocalReplica; +use crowdb_paxos_kv::cluster::remote_replica::PxRemoteReplica; /// Rebuild `group` with `new_remotes` merged into its remote list, /// applying the membership-epoch bump correctly. diff --git a/app/crowdb-kv-server/src/recovery/member_routes.rs b/app/crowdb-paxos-kv-server/src/recovery/member_routes.rs similarity index 94% rename from app/crowdb-kv-server/src/recovery/member_routes.rs rename to app/crowdb-paxos-kv-server/src/recovery/member_routes.rs index d4fab9d4a..3cec2b2f8 100644 --- a/app/crowdb-kv-server/src/recovery/member_routes.rs +++ b/app/crowdb-paxos-kv-server/src/recovery/member_routes.rs @@ -3,8 +3,8 @@ //! Monitor-owned endpoint evidence changes connections, never membership. -use crate::store_registry::KvStoreRegistry; -use crowdb_kv::cluster::group::PxGroup; +use crate::store_registry::PaxosKvStoreRegistry; +use crowdb_paxos_kv::cluster::group::PxGroup; use crowdb_protocol::mgmt::{ node::recovery::{MemberEndpoint, MemberEndpoints, RecoveryGroups}, GroupCreationStatus, SystemBootstrapIdentity, @@ -181,6 +181,14 @@ fn resolve( } pub struct Worker(tokio::task::JoinHandle<()>); +impl Worker { + /// Stop endpoint updates and release the registry after accepted work has + /// drained. The host awaits this before dropping Trees and native loggers. + pub async fn stop(mut self) { + self.0.abort(); + let _ = (&mut self.0).await; + } +} impl Drop for Worker { fn drop(&mut self) { self.0.abort(); @@ -188,7 +196,7 @@ impl Drop for Worker { } #[must_use] -pub fn start(registry: Arc<KvStoreRegistry>) -> Worker { +pub fn start(registry: Arc<PaxosKvStoreRegistry>) -> Worker { Worker(tokio::spawn(async move { let mut tick = tokio::time::interval(Duration::from_secs(1)); loop { diff --git a/app/crowdb-kv-server/src/recovery/reconcile.rs b/app/crowdb-paxos-kv-server/src/recovery/reconcile.rs similarity index 94% rename from app/crowdb-kv-server/src/recovery/reconcile.rs rename to app/crowdb-paxos-kv-server/src/recovery/reconcile.rs index 6e55c37df..9ab290e5c 100644 --- a/app/crowdb-kv-server/src/recovery/reconcile.rs +++ b/app/crowdb-paxos-kv-server/src/recovery/reconcile.rs @@ -24,16 +24,16 @@ use std::collections::HashMap; use tracing::{info, warn}; -use crowdb_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::kv_store::KvStore; use super::group_rebuild::rebuild_group_with_new_remotes; -use crate::store_registry::KvStoreRegistry; +use crate::store_registry::PaxosKvStoreRegistry; /// Peers for one group, keyed by `(store_id, group_id)`. type PeersByGroup = HashMap<(u64, u64), Vec<(u64, String, bool)>>; /// Run reconciliation against group 0 if it is locally available. -pub async fn reconcile_with_group0(registry: &KvStoreRegistry) { +pub async fn reconcile_with_group0(registry: &PaxosKvStoreRegistry) { let Some(store0) = registry.get_store(0) else { info!("reconcile: store 0 not found locally; skipping"); return; @@ -115,7 +115,7 @@ pub struct ReconcileAction { /// Pure decision logic: given decoded group-0 replica records and the /// local registry, compute what to do for each local group. No side /// effects — the caller executes the plan via [`execute_reconcile`]. -pub fn plan_reconcile(records: &[ReplicaRecord], registry: &KvStoreRegistry) -> Vec<ReconcileAction> { +pub fn plan_reconcile(records: &[ReplicaRecord], registry: &PaxosKvStoreRegistry) -> Vec<ReconcileAction> { // Group records by (store_id, group_id) -> Vec<(replica_id, endpoint, voting)>. let mut by_group: PeersByGroup = HashMap::new(); for r in records { @@ -176,7 +176,7 @@ pub fn plan_reconcile(records: &[ReplicaRecord], registry: &KvStoreRegistry) -> /// Execute a reconcile plan: rebuild groups for seed actions, log warn /// for mismatch actions. -fn execute_reconcile(plan: &[ReconcileAction], registry: &KvStoreRegistry) { +fn execute_reconcile(plan: &[ReconcileAction], registry: &PaxosKvStoreRegistry) { let mut seeded = 0usize; let mut mismatches = 0usize; for action in plan { @@ -220,7 +220,7 @@ fn execute_reconcile(plan: &[ReconcileAction], registry: &KvStoreRegistry) { /// Prefix-scan group 0 for complete membership records. If no authoritative /// records exist yet, retain the old projection scan for pre-R229 groups. async fn scan_authoritative_records( - store0: &crowdb_kv::cluster::px_kv_store::PxKvStore, + store0: &crowdb_paxos_kv::cluster::px_kv_store::PxKvStore, ) -> Option<Vec<ReplicaRecord>> { let response = store0 .kv_scan( @@ -236,7 +236,7 @@ async fn scan_authoritative_records( 0, false, 0, - crowdb_kv::kv::ScanDirection::Forward, + crowdb_paxos_kv::kv::ScanDirection::Forward, 0, 0, ) @@ -278,7 +278,7 @@ async fn scan_authoritative_records( /// unreachable / not yet led); returns `Some(vec)` (possibly empty) on /// success. async fn scan_replica_records( - store0: &crowdb_kv::cluster::px_kv_store::PxKvStore, + store0: &crowdb_paxos_kv::cluster::px_kv_store::PxKvStore, ) -> Option<Vec<ReplicaRecord>> { let prefix = b"/kv/replica/"; let resp = store0 @@ -295,7 +295,7 @@ async fn scan_replica_records( 0, false, 0, - crowdb_kv::kv::ScanDirection::Forward, + crowdb_paxos_kv::kv::ScanDirection::Forward, 0, 0, ) diff --git a/app/crowdb-kv-server/src/recovery/restore.rs b/app/crowdb-paxos-kv-server/src/recovery/restore.rs similarity index 78% rename from app/crowdb-kv-server/src/recovery/restore.rs rename to app/crowdb-paxos-kv-server/src/recovery/restore.rs index a1e6f1971..76cbead13 100644 --- a/app/crowdb-kv-server/src/recovery/restore.rs +++ b/app/crowdb-paxos-kv-server/src/recovery/restore.rs @@ -6,7 +6,7 @@ //! When group 0 is present on disk (`<wal_root>/store0/group0`), the //! server boots in restore mode: it scans `<wal_root>` for every //! `store{S}/group{G}` directory, loads each via -//! [`crate::recovery::startup::create_group_with_wal`] (which replays the WAL, +//! [`crate::recovery::startup::create_group_with_wal_on_runtime`] (which replays the WAL, //! opens the crowdb-tree engine, and applies the persisted membership //! from `node-config.json` — including remote-replica endpoints), and //! starts the stores. No `--stores`/`--groups` CLI args are needed; @@ -20,16 +20,16 @@ use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::PxLocalReplicaRole; -use crowdb_kv::cluster::node_config::NodeConfigStore; -use crowdb_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::cluster::local_replica::PxLocalReplicaRole; +use crowdb_paxos_kv::cluster::node_config::NodeConfigStore; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; -use super::startup::create_group_with_wal; +use super::startup::create_group_with_wal_on_runtime; use crate::mgmt::persisted_port_for_store; -use crate::store_registry::KvStoreRegistry; +use crate::store_registry::PaxosKvStoreRegistry; /// A local `(store_id, group_id)` pair discovered by scanning `waldata`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -90,18 +90,24 @@ pub fn group0_exists(wal_root: &Path) -> bool { /// /// Groups `local_groups` by `store_id`, creates each `PxKvStore` /// (port from the `--ports` pool, else `persisted_port_for_store`, -/// else OS-assigned 0), then calls [`create_group_with_wal`] per group +/// else OS-assigned 0), then calls [`create_group_with_wal_on_runtime`] per group /// with [`PxLocalReplicaRole::Follower`] (the election driver /// promotes), `add_group`, `store.start()`, and registers the store. -/// Reuses `create_and_start_stores`' skip-on-error policy: a failed -/// group is logged and skipped, the store still starts with its other -/// groups. +/// Restore is all-or-fail for the scanned local state: a failed group or +/// store start returns an error before the partially restored store is +/// published to the registry. This prevents a partial local restore from +/// being reported as a healthy server. /// /// The local `replica_id` for each group is read from `node-config.json` /// (where it was persisted when the group was first created). If the /// entry is missing (e.g. first boot after migrate, or stale config), /// falls back to the `replica_id` CLI arg. /// +/// # Errors +/// Returns an error when a persisted group identity cannot be read, a group +/// cannot be restored, or a store cannot be started. No partially restored +/// store is published. +/// /// # Panics /// Panics if the computed bind address `"0.0.0.0:{port}"` fails to /// parse as a `SocketAddr` (only possible if `port > 65535`, which @@ -109,8 +115,8 @@ pub fn group0_exists(wal_root: &Path) -> bool { pub async fn load_local_groups( local_groups: &[LocalGroup], replica_id: u64, - registry: &Arc<KvStoreRegistry>, -) { + registry: &Arc<PaxosKvStoreRegistry>, +) -> Result<(), String> { // Group by store_id, preserving scan (sorted) order. let mut by_store: Vec<(u64, Vec<u64>)> = Vec::new(); for lg in local_groups { @@ -154,6 +160,12 @@ pub async fn load_local_groups( // fall back to the CLI arg if not found. let effective_replica_id = persisted_replica_id(®istry.config.config_root, store_id, group_id) .await + .map_err(|error| { + format!( + "restore: failed to read persisted replica identity for store {store_id} \ + group {group_id}: {error}" + ) + })? .unwrap_or(replica_id); debug!( s = store_id, @@ -161,7 +173,7 @@ pub async fn load_local_groups( replica = effective_replica_id, "restore: creating group from disk" ); - let group = match create_group_with_wal( + let group = match create_group_with_wal_on_runtime( store_id, group_id, effective_replica_id, @@ -169,26 +181,24 @@ pub async fn load_local_groups( ®istry.config, registry.wal_backend.clone(), registry.crowtree_backend, + registry.tree_runtime.clone(), ) .await { Ok(group) => group, - Err(e) => { - warn!( - store_id, - group_id, - error = %e, - "restore: failed to load group from disk; skipping" - ); - continue; + Err(error) => { + return Err(format!( + "restore: failed to load store {store_id} group {group_id} from disk: {error}" + )); } }; store.add_group(group); } if let Err(e) = store.start().await { - warn!(s = store_id, port, error = %e, "restore: failed to start store; skipping"); - continue; + return Err(format!( + "restore: failed to start store {store_id} on port {port}: {e}" + )); } info!( @@ -199,6 +209,8 @@ pub async fn load_local_groups( ); registry.add_store(store_id, &store); } + + Ok(()) } /// Parse `store{S}` → `S`. Returns `None` if the name does not match. @@ -213,8 +225,12 @@ fn parse_group_dir(name: &str) -> Option<u64> { /// Read the persisted `replica_id` for `(store_id, group_id)` from /// `node-config.json`. Returns `None` if the file or entry is missing. -async fn persisted_replica_id(config_root: &Path, store_id: u64, group_id: u64) -> Option<u64> { +/// +/// A present but unreadable or invalid file is an error. Falling back to the +/// CLI replica id in that case could restore a WAL with the wrong identity and +/// make a damaged node appear healthy. +async fn persisted_replica_id(config_root: &Path, store_id: u64, group_id: u64) -> io::Result<Option<u64>> { let store = NodeConfigStore::new(config_root); - let config = store.load().await.ok()?; - config.group(store_id, group_id).map(|g| g.replica_id) + let config = store.load().await?; + Ok(config.group(store_id, group_id).map(|g| g.replica_id)) } diff --git a/app/crowdb-kv-server/src/recovery/startup.rs b/app/crowdb-paxos-kv-server/src/recovery/startup.rs similarity index 72% rename from app/crowdb-kv-server/src/recovery/startup.rs rename to app/crowdb-paxos-kv-server/src/recovery/startup.rs index 932116dc6..7729fd201 100644 --- a/app/crowdb-kv-server/src/recovery/startup.rs +++ b/app/crowdb-paxos-kv-server/src/recovery/startup.rs @@ -7,15 +7,15 @@ use std::sync::Arc; use tracing::info; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_config::{GroupConfigStore, PxGroupConfig, PxGroupMember}; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::node_config::NodeConfigStore; -use crowdb_kv::common::config::{CrowDBConfig, WalConfig}; -use crowdb_kv::kv::{CrowdbTreeBackend, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_config::{GroupConfigStore, PxGroupConfig, PxGroupMember}; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::node_config::NodeConfigStore; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv::kv::{CrowdbTreeBackend, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine}; /// Load persisted group config and apply it to the group. /// @@ -121,22 +121,26 @@ async fn open_crowdb_tree_engine( store_id: u64, group_id: u64, backend: CrowdbTreeBackend, - log_dir: &str, + config: &PaxosKvConfig, + runtime: Arc<crowdb_tree_ffi::TreeRuntime>, ) -> io::Result<Box<dyn KVEngine>> { let path = store_crowdb_tree_path(data_root, store_id, group_id); if let Some(parent) = path.parent() { tokio::fs::create_dir_all(parent).await?; } tokio::fs::create_dir_all(&path).await?; - let opt = CrowdbTreeConfig { + let mut opt = CrowdbTreeConfig { path: Some(path.to_string_lossy().into_owned()), backend, store_id: u32::try_from(store_id).unwrap_or(0), group_id: u32::try_from(group_id).unwrap_or(0), - log_dir: log_dir.to_string(), - log_file_prefix: "crowdb-kv-server-tree".to_string(), + log_dir: config.log_dir.clone(), + runtime: Some(runtime), + runtime_binding: config.tree_runtime.binding_options(), + log_file_prefix: "crowdb-paxos-kv-server-tree".to_string(), ..Default::default() }; + config.tree.apply_to(&mut opt); info!( store_id, group_id, @@ -144,12 +148,9 @@ async fn open_crowdb_tree_engine( path = %path.display(), "opening crowdb-tree engine" ); - // `CrowdbTreeEngine::open` is a synchronous FFI call; called here inline - // (not `spawn_blocking`) consistent with `CrowdbTreeEngine`'s own - // documented policy of calling the still-fully-synchronous crowdb-tree - // core directly rather than adding a thread-pool hop with no genuine - // asynchrony behind it (see `crowdb_kv::kv::CrowdbTreeEngine`'s docs). This - // runs once per group at boot, not on a hot path. + // Open waits synchronously for recovery on the shared native runtime. + // This boot-time call runs once per group; subsequent reads, apply and + // maintenance use native asynchronous operation futures. let engine = CrowdbTreeEngine::open(&opt).map_err(|e| { io::Error::other(format!( "CrowdbTreeEngine::open({}) failed: {e:?}", @@ -161,7 +162,8 @@ async fn open_crowdb_tree_engine( /// Create a live group by replaying any existing WAL, restoring the local /// replica state, attaching a fresh `WalEngine`, and seeding the next proposal -/// slot / segment id. +/// slot / segment id using a standalone host. Production groups share the +/// registry host through `create_group_with_wal_on_runtime`. /// /// # Errors /// @@ -174,15 +176,57 @@ pub async fn create_group_with_wal( group_id: u64, replica_id: u64, initial_role: PxLocalReplicaRole, - config: &CrowDBConfig, + config: &PaxosKvConfig, wal_backend: Arc<IoBackend>, crowtree_backend: CrowdbTreeBackend, ) -> io::Result<PxGroup> { + config + .validate_tree_config() + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + let runtime = Arc::new( + crowdb_tree_ffi::TreeRuntime::new(config.tree_runtime.runtime_options()) + .map_err(|error| io::Error::other(format!("standalone Tree runtime: {error}")))?, + ); + create_group_with_wal_on_runtime( + store_id, + group_id, + replica_id, + initial_role, + config, + wal_backend, + crowtree_backend, + runtime, + ) + .await +} + +/// Create or restore a group with its host-owned shared runtime. +/// # Errors +/// Returns an error for invalid limits, backend recovery failures or failed +/// accepted-log application. +#[allow(clippy::too_many_arguments)] +pub async fn create_group_with_wal_on_runtime( + store_id: u64, + group_id: u64, + replica_id: u64, + initial_role: PxLocalReplicaRole, + config: &PaxosKvConfig, + wal_backend: Arc<IoBackend>, + crowtree_backend: CrowdbTreeBackend, + runtime: Arc<crowdb_tree_ffi::TreeRuntime>, +) -> io::Result<PxGroup> { + config + .validate_tree_config() + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; crate::mgmt::group_creation::check_restore(&config.config_root, store_id, group_id, replica_id)?; persist_creation_identity(&config.config_root, store_id, group_id, replica_id).await?; - let mut wal_config = WalConfig::with_root(store_wal_root(&config.wal_root, store_id)); - if std::env::var("CROWDB_KV_WAL_TEXT").as_deref() == Ok("1") { - wal_config.wal_record_format = crowdb_kv::wal::WalRecordFormat::TextLine; + // Preserve the typed WAL tunables from the effective server config while + // deriving the per-store root. Rebuilding a fresh `WalConfig` here would + // silently discard segment, flush, retention, and alignment settings. + let mut wal_config = config.wal.clone(); + wal_config.set_root(store_wal_root(&config.wal_root, store_id)); + if std::env::var("CROWDB_PAXOS_KV_WAL_TEXT").as_deref() == Ok("1") { + wal_config.wal_record_format = crowdb_paxos_kv::wal::WalRecordFormat::TextLine; } wal_config.wal_skip_fsync = config.wal_skip_fsync; let mut replay = replay_group(&wal_backend, &wal_config.wal_disks, group_id).await?; @@ -201,7 +245,8 @@ pub async fn create_group_with_wal( store_id, group_id, crowtree_backend, - &config.log_dir, + config, + runtime, ) .await?; wal.set_snapshot_slot(engine.resume_from_slot()); diff --git a/app/crowdb-paxos-kv-server/src/shutdown.rs b/app/crowdb-paxos-kv-server/src/shutdown.rs new file mode 100644 index 000000000..5affb9731 --- /dev/null +++ b/app/crowdb-paxos-kv-server/src/shutdown.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +//! Host-wide shutdown keeps every consensus listener alive until every +//! accepted runtime prefix has drained, including peers hosted in this process. +use crate::store_registry::PaxosKvStoreRegistry; +use crowdb_paxos_kv::common::OperationReport; +use std::time::Duration; + +/// Perform one bounded shutdown attempt. An incomplete accepted drain returns +/// an error before any store is torn down, so callers may repair and retry. +pub async fn shutdown_stores_once(registry: &PaxosKvStoreRegistry) -> OperationReport { + let stores = registry.stores_snapshot(); + let timeout = Duration::from_millis(registry.config.server.shutdown_timeout_ms); + for store in stores.values() { + store.close_runtime_proposals(); + } + let mut report = OperationReport::new(); + for store in stores.values() { + report.merge(store.prepare_runtime_shutdown(timeout).await); + } + if !report.is_clean() { + return report; + } + for store in stores.values() { + report.merge(store.shutdown(timeout).await); + } + report +} + +/// Complete host shutdown. A timed-out accepted prefix is reported while the +/// process retains its communication/repair owners and awaits real progress. +/// Backend checkpoint failures remain explicit terminal errors in the result. +pub async fn shutdown_stores(registry: &PaxosKvStoreRegistry) -> OperationReport { + loop { + let report = shutdown_stores_once(registry).await; + let stores = registry.stores_snapshot(); + if !stores.values().any(|store| store.runtime_shutdown_pending()) { + return report; + } + for error in &report.errors { + tracing::error!(error = %error, "shutdown incomplete; consensus remains live; next step: restore peer progress to drain accepted work"); + } + for store in stores.values() { + store.await_runtime_shutdown_ready().await; + } + } +} diff --git a/app/crowdb-kv-server/src/store_registry.rs b/app/crowdb-paxos-kv-server/src/store_registry.rs similarity index 86% rename from app/crowdb-kv-server/src/store_registry.rs rename to app/crowdb-paxos-kv-server/src/store_registry.rs index 1c4ab0d3d..dc130a395 100644 --- a/app/crowdb-kv-server/src/store_registry.rs +++ b/app/crowdb-paxos-kv-server/src/store_registry.rs @@ -2,11 +2,11 @@ // Licensed under the Apache License, Version 2.0. use arc_swap::ArcSwap; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::common::config::CrowDBConfig; -use crowdb_kv::kv::CrowdbTreeBackend; -use crowdb_kv::metrics::MetricsRegistry; -use crowdb_kv::wal::IoBackend; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv::kv::CrowdbTreeBackend; +use crowdb_paxos_kv::metrics::MetricsRegistry; +use crowdb_paxos_kv::wal::IoBackend; use std::collections::HashMap; use std::sync::Arc; use std::sync::Mutex; @@ -35,15 +35,17 @@ pub(crate) fn parse_crowtree_backend(s: &str) -> CrowdbTreeBackend { } } -pub struct KvStoreRegistry { +pub struct PaxosKvStoreRegistry { stores: ArcSwap<HashMap<u64, Arc<PxKvStore>>>, pub(crate) bootstrap_executing: std::sync::atomic::AtomicBool, /// Unified cluster configuration (all sub-configs + flags + paths). - pub config: CrowDBConfig, + pub config: PaxosKvConfig, /// Parsed WAL I/O backend (derived from `config.wal_backend`). pub wal_backend: Arc<IoBackend>, /// Parsed crowdb-tree storage backend (derived from `config.crowtree_backend`). pub crowtree_backend: CrowdbTreeBackend, + /// Shared by normal startup, restore, management creation and recreation. + pub tree_runtime: Arc<crowdb_tree_ffi::TreeRuntime>, /// Port pool for KV server listeners, populated from `--ports` CLI arg. /// Used by `add_store` as a fallback before `persisted_port_for_store`. port_pool: Mutex<Vec<u16>>, @@ -54,16 +56,16 @@ pub struct KvStoreRegistry { pub rpc_workers: u32, } -impl Default for KvStoreRegistry { +impl Default for PaxosKvStoreRegistry { fn default() -> Self { Self::new() } } -impl KvStoreRegistry { +impl PaxosKvStoreRegistry { #[must_use] pub(crate) fn new() -> Self { - Self::with_config(CrowDBConfig::default()) + Self::with_config(PaxosKvConfig::default()) } /// Construct a registry from an already validated configuration. @@ -72,7 +74,7 @@ impl KvStoreRegistry { /// Panics when the configured backend cannot initialize. Startup code /// should use [`Self::try_with_config`] to return an actionable error. #[must_use] - pub fn with_config(config: CrowDBConfig) -> Self { + pub fn with_config(config: PaxosKvConfig) -> Self { Self::try_with_config(config).expect("default WAL backend must initialize") } @@ -81,7 +83,14 @@ impl KvStoreRegistry { /// # Errors /// Returns an actionable initialization error when `uring` was explicitly /// selected but liburing or the running kernel cannot create a ring. - pub fn try_with_config(config: CrowDBConfig) -> std::io::Result<Self> { + pub fn try_with_config(config: PaxosKvConfig) -> std::io::Result<Self> { + config + .validate_tree_config() + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?; + let tree_runtime = Arc::new( + crowdb_tree_ffi::TreeRuntime::new(config.tree_runtime.runtime_options()) + .map_err(|error| std::io::Error::other(format!("requested Tree runtime: {error}")))?, + ); let wal_backend = Arc::new(parse_wal_backend(&config.wal_backend).map_err(|error| { std::io::Error::new( error.kind(), @@ -95,6 +104,7 @@ impl KvStoreRegistry { bootstrap_executing: std::sync::atomic::AtomicBool::new(false), wal_backend, crowtree_backend, + tree_runtime, config, port_pool: Mutex::new(Vec::new()), metrics_registry: None, diff --git a/app/crowdb-kv-server/tests/chunk_dynamic_monitor_test.rs b/app/crowdb-paxos-kv-server/tests/chunk_dynamic_monitor_test.rs similarity index 95% rename from app/crowdb-kv-server/tests/chunk_dynamic_monitor_test.rs rename to app/crowdb-paxos-kv-server/tests/chunk_dynamic_monitor_test.rs index af0f2b310..89998c2b1 100644 --- a/app/crowdb-kv-server/tests/chunk_dynamic_monitor_test.rs +++ b/app/crowdb-paxos-kv-server/tests/chunk_dynamic_monitor_test.rs @@ -4,7 +4,7 @@ #[path = "common/dynamic_slots.rs"] mod dynamic_slots; use bytes::Bytes; -use crowdb_kv_server::background::domain_monitor::{ChunkdbDynamicMonitorDriver, DomainMonitorDriver}; +use crowdb_paxos_kv_server::background::domain_monitor::{ChunkdbDynamicMonitorDriver, DomainMonitorDriver}; use crowdb_protocol::key::{ChunkSlotMapHeadKey, TextKey}; use dynamic_slots::{control, descriptor, entry, initialize, TestDynamicOwners}; @@ -137,8 +137,8 @@ async fn corrupt_dynamic_snapshot_is_rejected_without_overwriting_it() { #[tokio::test] async fn replacement_group_zero_tenure_resumes_and_obsolete_control_cannot_publish() { - use crowdb_kv::cluster::group_election::LeaderElection; - use crowdb_kv_server::group0_control_plane::Group0ControlPlane; + use crowdb_paxos_kv::cluster::group_election::LeaderElection; + use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; let (store, control) = control().await; initialize(&control).await; for owner in 1..=4 { diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs b/app/crowdb-paxos-kv-server/tests/chunk_kv_balance_planning_test.rs similarity index 97% rename from app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs rename to app/crowdb-paxos-kv-server/tests/chunk_kv_balance_planning_test.rs index 9f9503ab8..74dffbd6c 100644 --- a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs +++ b/app/crowdb-paxos-kv-server/tests/chunk_kv_balance_planning_test.rs @@ -11,9 +11,9 @@ mod committed_transfer; mod recovering_owner; use bytes::Bytes; -use crowdb_kv::cluster::{group::PxGroup, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_server::background::domain_monitor::{ChunkKvRangeMonitorDriver, DomainMonitorDriver}; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv::cluster::{group::PxGroup, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_server::background::domain_monitor::{ChunkKvRangeMonitorDriver, DomainMonitorDriver}; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; use crowdb_protocol::chunk_kv::{ ChunkKvRangeBalancePolicy, DomainFailurePolicy, DomainMonitorDescriptor, Id128, TransferTransition, }; diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs b/app/crowdb-paxos-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs similarity index 100% rename from app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs rename to app/crowdb-paxos-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs b/app/crowdb-paxos-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs similarity index 99% rename from app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs rename to app/crowdb-paxos-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs index 3fb2585a2..6d55907b3 100644 --- a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs +++ b/app/crowdb-paxos-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs @@ -45,6 +45,7 @@ async fn renewing_recovery_process_is_not_dead_when_discovery_is_stale() { prior_exclusion_deadline_ms: now - 1, grant_sequence: 0, grant_expires_at_ms: 0, + revoked: false, }; lease.validate().unwrap(); for expiry in [now + 60_000, now - 1] { diff --git a/app/crowdb-kv-server/tests/chunk_slot_monitor_test.rs b/app/crowdb-paxos-kv-server/tests/chunk_slot_monitor_test.rs similarity index 95% rename from app/crowdb-kv-server/tests/chunk_slot_monitor_test.rs rename to app/crowdb-paxos-kv-server/tests/chunk_slot_monitor_test.rs index 0f8b122f7..4d0131306 100644 --- a/app/crowdb-kv-server/tests/chunk_slot_monitor_test.rs +++ b/app/crowdb-paxos-kv-server/tests/chunk_slot_monitor_test.rs @@ -4,10 +4,10 @@ use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_server::background::domain_monitor::{ChunkdbRangeMonitorDriver, DomainMonitorDriver}; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_server::background::domain_monitor::{ChunkdbRangeMonitorDriver, DomainMonitorDriver}; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; use crowdb_protocol::chunk_kv::{DomainFailurePolicy, DomainMonitorDescriptor}; use crowdb_protocol::chunk_slot::{ChunkSlot, ChunkSlotBinding, ChunkSlotMapHead, ChunkStorageGroup}; use crowdb_protocol::common::InstanceValue; diff --git a/app/crowdb-kv-server/tests/cli_parse_test.rs b/app/crowdb-paxos-kv-server/tests/cli_parse_test.rs similarity index 85% rename from app/crowdb-kv-server/tests/cli_parse_test.rs rename to app/crowdb-paxos-kv-server/tests/cli_parse_test.rs index 711ad4d8b..38c13c7f3 100644 --- a/app/crowdb-kv-server/tests/cli_parse_test.rs +++ b/app/crowdb-paxos-kv-server/tests/cli_parse_test.rs @@ -2,13 +2,13 @@ // Licensed under the Apache License, Version 2.0. //! CLI argument parsing tests. Migrated from the inline `#[cfg(test)] mod -//! tests` in `crowdb-kv-server/src/cli.rs` per `.windsurf/workflows/coding.md` +//! tests` in `crowdb-paxos-kv-server/src/cli.rs` per `.windsurf/workflows/coding.md` //! §3 ("integration tests only — do not add new inline test modules"). use clap::Parser; -use crowdb_kv::common::config::CrowDBConfig; -use crowdb_kv_server::cli::{parse_id_list, parse_port_list, Cli}; -use crowdb_kv_server::store_registry::KvStoreRegistry; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv_server::cli::{parse_id_list, parse_port_list, Cli}; +use crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry; #[test] fn parse_single() { @@ -65,7 +65,7 @@ fn parse_port_rejects_zero() { #[test] fn parse_root_cli_option() { let cli = Cli::parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", "/data/N-1", "--management-port", @@ -78,7 +78,7 @@ fn parse_root_cli_option() { assert_eq!(cli.kv_backend, "block"); assert_eq!(cli.wal_backend, "block-device"); assert!(!cli.no_fsync); - let defaults = CrowDBConfig::default(); + let defaults = PaxosKvConfig::default(); assert_eq!(defaults.crowtree_backend, cli.kv_backend); assert_eq!(defaults.wal_backend, cli.wal_backend); assert!(!defaults.wal_skip_fsync); @@ -87,7 +87,7 @@ fn parse_root_cli_option() { #[test] fn parse_root_is_required() { // Omitting --root should fail (clap rejects the invocation). - let result = Cli::try_parse_from(["crowdb-kv-server", "--management-port", "10000"]); + let result = Cli::try_parse_from(["crowdb-paxos-kv-server", "--management-port", "10000"]); assert!(result.is_err()); } @@ -96,7 +96,7 @@ fn loaded_configuration_uses_durable_block_defaults() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("block-default-config"); let path = root.path().join("server.toml"); std::fs::write(&path, "").unwrap(); - let loaded = CrowDBConfig::load_from_file(&path).unwrap(); + let loaded = PaxosKvConfig::load_from_file(&path).unwrap(); assert_eq!(loaded.crowtree_backend, "block"); assert_eq!(loaded.wal_backend, "block-device"); assert!(!loaded.wal_skip_fsync); @@ -105,7 +105,7 @@ fn loaded_configuration_uses_durable_block_defaults() { #[test] fn management_port_rejects_zero() { let result = Cli::try_parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", "--management-port", @@ -117,7 +117,7 @@ fn management_port_rejects_zero() { #[test] fn wal_backend_accepts_explicit_uring() { let cli = Cli::parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", "--wal-backend", @@ -125,7 +125,7 @@ fn wal_backend_accepts_explicit_uring() { ]); assert_eq!(cli.wal_backend, "uring"); let invalid = Cli::try_parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", "--wal-backend", @@ -136,9 +136,9 @@ fn wal_backend_accepts_explicit_uring() { #[test] fn explicit_uring_initialization_never_silently_falls_back() { - let mut config = CrowDBConfig::for_tests(); + let mut config = PaxosKvConfig::for_tests(); config.wal_backend = "uring".to_owned(); - match KvStoreRegistry::try_with_config(config) { + match PaxosKvStoreRegistry::try_with_config(config) { Ok(registry) => assert_eq!(format!("{:?}", registry.wal_backend), "Uring"), Err(error) => assert!( error.to_string().contains("requested WAL backend 'uring'"), @@ -150,7 +150,7 @@ fn explicit_uring_initialization_never_silently_falls_back() { #[test] fn rpc_workers_is_only_an_explicit_override() { let defaults = Cli::parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", ]); @@ -161,7 +161,7 @@ fn rpc_workers_is_only_an_explicit_override() { assert_eq!(defaults.send_queue_capacity, None); let overridden = Cli::parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", "--rpc-workers", @@ -170,7 +170,7 @@ fn rpc_workers_is_only_an_explicit_override() { assert_eq!(overridden.rpc_workers, Some(7)); let invalid = Cli::try_parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", "--rpc-workers", @@ -182,15 +182,15 @@ fn rpc_workers_is_only_an_explicit_override() { #[test] fn production_tree_backend_defaults_to_block() { let defaults = Cli::parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", ]); assert_eq!(defaults.kv_backend, "block"); - assert_eq!(CrowDBConfig::default().crowtree_backend, "block"); + assert_eq!(PaxosKvConfig::default().crowtree_backend, "block"); let file = Cli::parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", "--kv-backend", @@ -202,11 +202,11 @@ fn production_tree_backend_defaults_to_block() { #[test] fn config_values_survive_absent_cli_and_explicit_values_override() { let defaults = Cli::parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", ]); - let mut config = CrowDBConfig::default(); + let mut config = PaxosKvConfig::default(); config.paxos.max_inflight_proposals = 19; config.server.peer_pool_size = 7; config.server.send_queue_capacity = 1234; @@ -224,7 +224,7 @@ fn config_values_survive_absent_cli_and_explicit_values_override() { assert!(config.server.event_write); let overrides = Cli::parse_from([ - "crowdb-kv-server", + "crowdb-paxos-kv-server", "--root", ".crowdb-runtime/ephemeral/cli-parse/n1", "--max-inflight", diff --git a/app/crowdb-kv-server/tests/common/balance_distribution.rs b/app/crowdb-paxos-kv-server/tests/common/balance_distribution.rs similarity index 98% rename from app/crowdb-kv-server/tests/common/balance_distribution.rs rename to app/crowdb-paxos-kv-server/tests/common/balance_distribution.rs index 875077518..5da4c13b2 100644 --- a/app/crowdb-kv-server/tests/common/balance_distribution.rs +++ b/app/crowdb-paxos-kv-server/tests/common/balance_distribution.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; use crowdb_protocol::{ chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, diff --git a/app/crowdb-kv-server/tests/common/balance_planning.rs b/app/crowdb-paxos-kv-server/tests/common/balance_planning.rs similarity index 98% rename from app/crowdb-kv-server/tests/common/balance_planning.rs rename to app/crowdb-paxos-kv-server/tests/common/balance_planning.rs index d98251c4b..45f5deb0c 100644 --- a/app/crowdb-kv-server/tests/common/balance_planning.rs +++ b/app/crowdb-paxos-kv-server/tests/common/balance_planning.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, HostedPartition, Id128, KeyRange, OwnerDescriptor, PartitionArtifact, diff --git a/app/crowdb-kv-server/tests/common/catalog_publication.rs b/app/crowdb-paxos-kv-server/tests/common/catalog_publication.rs similarity index 99% rename from app/crowdb-kv-server/tests/common/catalog_publication.rs rename to app/crowdb-paxos-kv-server/tests/common/catalog_publication.rs index 5599d7a60..b0adaac12 100644 --- a/app/crowdb-kv-server/tests/common/catalog_publication.rs +++ b/app/crowdb-paxos-kv-server/tests/common/catalog_publication.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::*; -use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; +use crowdb_paxos_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; struct TestPublication { control: Group0ControlPlane, diff --git a/app/crowdb-kv-server/tests/common/dynamic_slots.rs b/app/crowdb-paxos-kv-server/tests/common/dynamic_slots.rs similarity index 96% rename from app/crowdb-kv-server/tests/common/dynamic_slots.rs rename to app/crowdb-paxos-kv-server/tests/common/dynamic_slots.rs index 73b785636..7e4f1f746 100644 --- a/app/crowdb-kv-server/tests/common/dynamic_slots.rs +++ b/app/crowdb-paxos-kv-server/tests/common/dynamic_slots.rs @@ -4,9 +4,9 @@ use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; use crowdb_protocol::chunk_kv::{DomainFailurePolicy, DomainMonitorDescriptor}; use crowdb_protocol::chunk_slot::{ChunkSlot, ChunkSlotBinding, ChunkSlotMapHead, ChunkStorageGroup}; use crowdb_protocol::common::InstanceValue; diff --git a/app/crowdb-kv-server/tests/common/endpoint_publication.rs b/app/crowdb-paxos-kv-server/tests/common/endpoint_publication.rs similarity index 98% rename from app/crowdb-kv-server/tests/common/endpoint_publication.rs rename to app/crowdb-paxos-kv-server/tests/common/endpoint_publication.rs index c87bfe346..0bc31b444 100644 --- a/app/crowdb-kv-server/tests/common/endpoint_publication.rs +++ b/app/crowdb-paxos-kv-server/tests/common/endpoint_publication.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::*; -use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; +use crowdb_paxos_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; #[tokio::test] async fn recovered_owner_address_advances_catalog_without_transferring_data() { diff --git a/app/crowdb-kv-server/tests/common/grant_isolation.rs b/app/crowdb-paxos-kv-server/tests/common/grant_isolation.rs similarity index 98% rename from app/crowdb-kv-server/tests/common/grant_isolation.rs rename to app/crowdb-paxos-kv-server/tests/common/grant_isolation.rs index d8e2c251a..eccf1707b 100644 --- a/app/crowdb-kv-server/tests/common/grant_isolation.rs +++ b/app/crowdb-paxos-kv-server/tests/common/grant_isolation.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::*; -use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; +use crowdb_paxos_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; use crowdb_protocol::{ chunk_stream::{StreamProcessLease, StreamPurpose}, key::StreamProcessLeaseKey, diff --git a/app/crowdb-kv-server/tests/common/grant_process.rs b/app/crowdb-paxos-kv-server/tests/common/grant_process.rs similarity index 97% rename from app/crowdb-kv-server/tests/common/grant_process.rs rename to app/crowdb-paxos-kv-server/tests/common/grant_process.rs index 830f0c73b..a291094b4 100644 --- a/app/crowdb-kv-server/tests/common/grant_process.rs +++ b/app/crowdb-paxos-kv-server/tests/common/grant_process.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::*; -use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; +use crowdb_paxos_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; use crowdb_protocol::{ chunk_stream::{StreamProcessLease, StreamPurpose}, key::StreamProcessLeaseKey, diff --git a/app/crowdb-kv-server/tests/common/group_creation.rs b/app/crowdb-paxos-kv-server/tests/common/group_creation.rs similarity index 89% rename from app/crowdb-kv-server/tests/common/group_creation.rs rename to app/crowdb-paxos-kv-server/tests/common/group_creation.rs index f27d8f227..88e908dbd 100644 --- a/app/crowdb-kv-server/tests/common/group_creation.rs +++ b/app/crowdb-paxos-kv-server/tests/common/group_creation.rs @@ -1,20 +1,20 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::{ - cluster::{KvServer, PxKvStore}, - common::config::CrowDBConfig, +use crowdb_paxos_kv::{ + cluster::{PaxosKvServer, PxKvStore}, + common::config::PaxosKvConfig, }; -use crowdb_kv_server::{ +use crowdb_paxos_kv_server::{ mgmt::{self, operation_registry::AppState}, - store_registry::KvStoreRegistry, + store_registry::PaxosKvStoreRegistry, }; use serde_json::{json, Value}; use std::{path::PathBuf, sync::Arc}; pub struct TestGroupCreation { _root: crowdb_test_harness::test_dirs::TestDir, - pub registry: Arc<KvStoreRegistry>, + pub registry: Arc<PaxosKvStoreRegistry>, pub store: Arc<PxKvStore>, pub origin: String, server: tokio::task::JoinHandle<()>, @@ -23,18 +23,18 @@ pub struct TestGroupCreation { impl TestGroupCreation { pub async fn start(block_data: bool) -> Self { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("native-group-creation"); - let config = CrowDBConfig { + let config = PaxosKvConfig { data_root: root.path().join("data"), wal_root: root.path().join("wal"), config_root: root.path().join("config"), crowtree_backend: "file".into(), wal_backend: "file".into(), - ..CrowDBConfig::for_tests() + ..PaxosKvConfig::for_tests() }; if block_data { std::fs::write(&config.data_root, b"blocked").unwrap(); } - let registry = Arc::new(KvStoreRegistry::with_config(config)); + let registry = Arc::new(PaxosKvStoreRegistry::with_config(config)); let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); store.start().await.unwrap(); registry.add_store(0, &store); diff --git a/app/crowdb-kv-server/tests/common/process_lease.rs b/app/crowdb-paxos-kv-server/tests/common/process_lease.rs similarity index 94% rename from app/crowdb-kv-server/tests/common/process_lease.rs rename to app/crowdb-paxos-kv-server/tests/common/process_lease.rs index f867005a8..a34cadd1b 100644 --- a/app/crowdb-kv-server/tests/common/process_lease.rs +++ b/app/crowdb-paxos-kv-server/tests/common/process_lease.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; use crowdb_protocol::{ chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}, key::{StreamProcessLeaseKey, TextKey}, @@ -39,6 +39,7 @@ impl TestProcessLease { prior_exclusion_deadline_ms: 0, grant_sequence: 0, grant_expires_at_ms: 0, + revoked: false, }; control .compare_and_put( diff --git a/app/crowdb-paxos-kv-server/tests/common/runtime_shutdown.rs b/app/crowdb-paxos-kv-server/tests/common/runtime_shutdown.rs new file mode 100644 index 000000000..52a225507 --- /dev/null +++ b/app/crowdb-paxos-kv-server/tests/common/runtime_shutdown.rs @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use bytes::Bytes; +use crowdb_paxos_kv::{ + cluster::{ + group::PxGroup, + replica::{HeartbeatRequestPayload, ReplicaHandler}, + PaxosKvServer, PxKvStore, PxLocalReplicaRole, + }, + common::config::PaxosKvConfig, + paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry}, +}; +use crowdb_paxos_kv_server::{ + mgmt::{self, operation_registry::AppState}, + recovery::startup::{create_group_with_wal_on_runtime, store_crowdb_tree_path, store_wal_root}, + store_registry::PaxosKvStoreRegistry, +}; +use std::{path::PathBuf, sync::Arc, time::Duration}; +pub const DEADLINE: Duration = Duration::from_secs(3); +pub struct TestRuntimeShutdown { + _root: crowdb_test_harness::test_dirs::TestDir, + pub registry: Arc<PaxosKvStoreRegistry>, + pub store: Arc<PxKvStore>, + pub group: Arc<PxGroup>, + pub engine_marker: PathBuf, + pub wal_marker: PathBuf, + origin: String, + http: tokio::task::JoinHandle<()>, +} +impl TestRuntimeShutdown { + pub async fn start() -> Self { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("runtime-host-shutdown"); + let mut config = PaxosKvConfig { + wal_backend: "file".into(), + crowtree_backend: "mem-block".into(), + wal_root: root.path().join("wal"), + data_root: root.path().join("data"), + config_root: root.path().join("config"), + ..PaxosKvConfig::for_tests() + }; + config.server.shutdown_timeout_ms = 20; + config.tree_runtime.host.workers = Some(1); + let registry = Arc::new(PaxosKvStoreRegistry::try_with_config(config).unwrap()); + let group = create_group_with_wal_on_runtime( + 1, + 1, + 1, + PxLocalReplicaRole::Follower, + ®istry.config, + registry.wal_backend.clone(), + registry.crowtree_backend, + registry.tree_runtime.clone(), + ) + .await + .unwrap(); + let store = Arc::new(PxKvStore::new(1, "127.0.0.1:0".parse().unwrap())); + store.add_group_without_election(group); + store.start().await.unwrap(); + registry.add_store(1, &store); + let group = store.get_group(1).unwrap(); + accept(&group, 2).await; + group + .local_replica() + .on_heartbeat( + HeartbeatRequestPayload { + term: 0, + leader_id: 2, + prev_log_slot: 0, + prev_log_term: 0, + committed_safe_slot: 2, + lease_grant_until_ms_mono: 0, + t_send_ms_mono: 0, + gc_before_slot: 0, + gc_membership_epoch: 0, + }, + 1, + ) + .await + .unwrap(); + let engine_dir = store_crowdb_tree_path(®istry.config.data_root, 1, 1); + let wal_dir = store_wal_root(®istry.config.wal_root, 1).join("group1"); + std::fs::create_dir_all(&engine_dir).unwrap(); + std::fs::create_dir_all(&wal_dir).unwrap(); + let engine_marker = engine_dir.join("retained-marker"); + let wal_marker = wal_dir.join("retained-marker"); + std::fs::write(&engine_marker, b"engine retained").unwrap(); + std::fs::write(&wal_marker, b"WAL retained").unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let router = mgmt::router(AppState::new(registry.clone())); + let http = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + Self { + _root: root, + registry, + store, + group, + engine_marker, + wal_marker, + origin, + http, + } + } + pub async fn request(&self, method: reqwest::Method, path: &str) -> (reqwest::StatusCode, String) { + let client = reqwest::Client::builder() + .no_proxy() + .timeout(DEADLINE) + .build() + .unwrap(); + let result = client + .request(method, format!("{}{path}", self.origin)) + .send() + .await + .unwrap(); + (result.status(), result.text().await.unwrap()) + } + pub fn assert_retained(&self) { + assert!(self.registry.get_store(1).is_some()); + assert!(self.store.get_group(1).is_some()); + assert!(self.engine_marker.exists()); + assert!(self.wal_marker.exists()); + assert_eq!(self.group.local_replica().contiguous_applied(), 0); + assert_eq!(self.group.local_replica().learner.engine().resume_from_slot(), 0); + } + pub async fn repair(&self) { + accept(&self.group, 1).await; + tokio::time::timeout(DEADLINE, self.group.local_replica().await_apply_fence(2)) + .await + .unwrap(); + } +} +impl Drop for TestRuntimeShutdown { + fn drop(&mut self) { + self.http.abort(); + self.store.stop(); + } +} +async fn accept(group: &PxGroup, slot: u64) { + let mut payload = vec![1, 0, 0]; + payload.extend_from_slice(&8u32.to_le_bytes()); + payload.extend_from_slice(&slot.to_le_bytes()); + payload.extend_from_slice(&8u32.to_le_bytes()); + payload.extend_from_slice(&slot.to_le_bytes()); + let entry = PxLogEntry { + slot, + term: 0, + ballot: PxBallot::new(1, 1), + payload: Bytes::from(payload), + }; + assert!(matches!( + group.local_replica().on_accept(&entry).await, + PxAcceptReply::Accepted { .. } + )); +} diff --git a/app/crowdb-kv-server/tests/common/split_materialization.rs b/app/crowdb-paxos-kv-server/tests/common/split_materialization.rs similarity index 100% rename from app/crowdb-kv-server/tests/common/split_materialization.rs rename to app/crowdb-paxos-kv-server/tests/common/split_materialization.rs diff --git a/app/crowdb-kv-server/tests/common/split_materialization/transfer.rs b/app/crowdb-paxos-kv-server/tests/common/split_materialization/transfer.rs similarity index 100% rename from app/crowdb-kv-server/tests/common/split_materialization/transfer.rs rename to app/crowdb-paxos-kv-server/tests/common/split_materialization/transfer.rs diff --git a/app/crowdb-kv-server/tests/common/system_bootstrap.rs b/app/crowdb-paxos-kv-server/tests/common/system_bootstrap.rs similarity index 87% rename from app/crowdb-kv-server/tests/common/system_bootstrap.rs rename to app/crowdb-paxos-kv-server/tests/common/system_bootstrap.rs index 5bcad0b50..079aa41a8 100644 --- a/app/crowdb-kv-server/tests/common/system_bootstrap.rs +++ b/app/crowdb-paxos-kv-server/tests/common/system_bootstrap.rs @@ -1,10 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::common::config::CrowDBConfig; -use crowdb_kv_server::{ +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv_server::{ mgmt::{self, operation_registry::AppState}, - store_registry::KvStoreRegistry, + store_registry::PaxosKvStoreRegistry, }; use crowdb_protocol::mgmt::{SystemBootstrapIdentity, SystemPrepareRequest}; use reqwest::StatusCode; @@ -13,7 +13,7 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; pub struct TestSystemBootstrap { _root: crowdb_test_harness::test_dirs::TestDir, - pub registry: Arc<KvStoreRegistry>, + pub registry: Arc<PaxosKvStoreRegistry>, pub origin: String, server: tokio::task::JoinHandle<()>, } @@ -21,13 +21,13 @@ pub struct TestSystemBootstrap { impl TestSystemBootstrap { pub async fn start() -> Self { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("system-prepare"); - let config = CrowDBConfig { + let config = PaxosKvConfig { data_root: root.path().join("data"), wal_root: root.path().join("wal"), config_root: root.path().join("config"), - ..CrowDBConfig::for_tests() + ..PaxosKvConfig::for_tests() }; - let registry = Arc::new(KvStoreRegistry::with_config(config)); + let registry = Arc::new(PaxosKvStoreRegistry::with_config(config)); let (origin, server) = Self::serve(®istry).await; Self { _root: root, @@ -37,7 +37,7 @@ impl TestSystemBootstrap { } } - async fn serve(registry: &Arc<KvStoreRegistry>) -> (String, tokio::task::JoinHandle<()>) { + async fn serve(registry: &Arc<PaxosKvStoreRegistry>) -> (String, tokio::task::JoinHandle<()>) { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let origin = format!("http://{}", listener.local_addr().unwrap()); let router = mgmt::router(AppState::new(registry.clone())); @@ -49,7 +49,7 @@ impl TestSystemBootstrap { pub async fn restart(&mut self) { self.server.abort(); - self.registry = Arc::new(KvStoreRegistry::with_config(self.registry.config.clone())); + self.registry = Arc::new(PaxosKvStoreRegistry::with_config(self.registry.config.clone())); (self.origin, self.server) = Self::serve(&self.registry).await; } diff --git a/app/crowdb-kv-server/tests/diskdb_placement_test.rs b/app/crowdb-paxos-kv-server/tests/diskdb_placement_test.rs similarity index 92% rename from app/crowdb-kv-server/tests/diskdb_placement_test.rs rename to app/crowdb-paxos-kv-server/tests/diskdb_placement_test.rs index 495cc9ba3..db376bed2 100644 --- a/app/crowdb-kv-server/tests/diskdb_placement_test.rs +++ b/app/crowdb-paxos-kv-server/tests/diskdb_placement_test.rs @@ -1,12 +1,12 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::CrowDBConfig; -use crowdb_kv_server::background::domain_monitor::{DiskdbPlacementMonitorDriver, DomainMonitorDriver}; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; -use crowdb_kv_server::store_registry::KvStoreRegistry; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv_server::background::domain_monitor::{DiskdbPlacementMonitorDriver, DomainMonitorDriver}; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry; use crowdb_protocol::chunk_kv::{DomainFailurePolicy, DomainMonitorDescriptor}; use crowdb_protocol::common::{BindMapValue, InstanceValue, OwnerMapValue}; use crowdb_protocol::key::{BindMapKey, DiskGroupKey, InstanceKey, OwnerMapKey, RackKey, TextKey}; @@ -29,8 +29,8 @@ fn descriptor() -> DomainMonitorDescriptor { } } -fn registry_with_group_zero(role: PxLocalReplicaRole) -> (Arc<KvStoreRegistry>, Arc<PxKvStore>) { - let registry = Arc::new(KvStoreRegistry::with_config(CrowDBConfig::for_tests())); +fn registry_with_group_zero(role: PxLocalReplicaRole) -> (Arc<PaxosKvStoreRegistry>, Arc<PxKvStore>) { + let registry = Arc::new(PaxosKvStoreRegistry::with_config(PaxosKvConfig::for_tests())); let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); store.add_group(PxGroup::new(0, PxLocalReplica::new(1, role))); registry.add_store(0, &store); diff --git a/app/crowdb-kv-server/tests/domain_monitor_test.rs b/app/crowdb-paxos-kv-server/tests/domain_monitor_test.rs similarity index 97% rename from app/crowdb-kv-server/tests/domain_monitor_test.rs rename to app/crowdb-paxos-kv-server/tests/domain_monitor_test.rs index 1b1307719..3dd46c045 100644 --- a/app/crowdb-kv-server/tests/domain_monitor_test.rs +++ b/app/crowdb-paxos-kv-server/tests/domain_monitor_test.rs @@ -6,15 +6,15 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::CrowDBConfig; -use crowdb_kv_server::background::domain_monitor::{ +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv_server::background::domain_monitor::{ spawn_domain_monitor_supervisor, DiskdbOwnershipMonitorDriver, DomainMonitorDriver, DomainMonitorDrivers, DomainMonitorFuture, }; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; -use crowdb_kv_server::store_registry::KvStoreRegistry; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry; use crowdb_protocol::chunk_kv::{ AuthorityReleaseProof, ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, HostedPartition, @@ -86,8 +86,8 @@ fn descriptor() -> DomainMonitorDescriptor { } } -fn registry_with_group_zero(role: PxLocalReplicaRole) -> (Arc<KvStoreRegistry>, Arc<PxKvStore>) { - let registry = Arc::new(KvStoreRegistry::with_config(CrowDBConfig::for_tests())); +fn registry_with_group_zero(role: PxLocalReplicaRole) -> (Arc<PaxosKvStoreRegistry>, Arc<PxKvStore>) { + let registry = Arc::new(PaxosKvStoreRegistry::with_config(PaxosKvConfig::for_tests())); let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); store.add_group(PxGroup::new(0, PxLocalReplica::new(1, role))); registry.add_store(0, &store); @@ -208,7 +208,7 @@ async fn chunk_kv_driver_publishes_ready_transfer_then_issues_matching_grant() { #[allow(clippy::too_many_lines)] async fn assert_ready_transfer_publication() -> (Group0ControlPlane, TransferTransition) { - use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; + use crowdb_paxos_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); let control = Group0ControlPlane::acquire(&store).await.unwrap(); @@ -455,7 +455,7 @@ async fn assert_ready_transfer_publication() -> (Group0ControlPlane, TransferTra #[tokio::test] async fn chunk_kv_driver_plans_dead_owner_recovery_once_and_waits_for_target() { - use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; + use crowdb_paxos_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); let control = Group0ControlPlane::acquire(&store).await.unwrap(); @@ -556,7 +556,7 @@ async fn assert_chunk_kv_split_plan( target_partitions_per_owner: u32, target_partition_bytes: u64, ) -> (Group0ControlPlane, SplitTransition) { - use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; + use crowdb_paxos_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); let control = Group0ControlPlane::acquire(&store).await.unwrap(); @@ -753,7 +753,7 @@ async fn chunk_kv_driver_plans_size_driven_split() { #[tokio::test] async fn chunk_kv_driver_enables_child_owner_balance_from_policy() { - use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; + use crowdb_paxos_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); let control = Group0ControlPlane::acquire(&store).await.unwrap(); diff --git a/app/crowdb-paxos-kv-server/tests/engine_resource_metrics_test.rs b/app/crowdb-paxos-kv-server/tests/engine_resource_metrics_test.rs new file mode 100644 index 000000000..48157acf9 --- /dev/null +++ b/app/crowdb-paxos-kv-server/tests/engine_resource_metrics_test.rs @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; +use std::time::Duration; + +use crowdb_common::metrics::MetricsRunner; +use crowdb_paxos_kv::cluster::{px_kv_store::PxKvStore, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv::kv::CrowdbTreeEngine; +use crowdb_paxos_kv_server::engine_collector::setup_engine_collector; +use crowdb_paxos_kv_server::recovery::startup::create_group_with_wal_on_runtime; +use crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry; + +#[tokio::test] +async fn service_collector_logs_host_and_tree_resource_windows() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("paxos-tree-resources"); + let config = PaxosKvConfig { + wal_backend: "mem-block".into(), + crowtree_backend: "mem-block".into(), + wal_root: root.path().join("wal"), + config_root: root.path().join("config"), + data_root: root.path().join("tree"), + log_dir: root.path().join("log").to_string_lossy().into_owned(), + ..Default::default() + }; + let registry = Arc::new(PaxosKvStoreRegistry::try_with_config(config).unwrap()); + let store = Arc::new(PxKvStore::new(1, "127.0.0.1:0".parse().unwrap())); + let group = create_group_with_wal_on_runtime( + 1, + 1, + 1, + PxLocalReplicaRole::Leader, + ®istry.config, + registry.wal_backend.clone(), + registry.crowtree_backend, + registry.tree_runtime.clone(), + ) + .await + .unwrap(); + let engine = group.local_replica().learner.engine_arc(); + store.add_group_without_election(group); + registry.add_store(1, &store); + let dir = root.path().join("metrics"); + let file = crowdb_common::logging::open_metrics_log(&dir, "test", 10, 2).unwrap(); + let mut runner = MetricsRunner::new(file, 1); + setup_engine_collector(&runner.registry().clone(), ®istry, &mut runner); + runner.start(); + let tree = engine.as_any().downcast_ref::<CrowdbTreeEngine>().unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + while tree.work_metrics_window().is_none() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + runner.stop().await; + let path = std::fs::read_dir(&dir).unwrap().next().unwrap().unwrap().path(); + let log = std::fs::read_to_string(path).unwrap(); + assert!(log.contains("host.tree.resource.cpu_workers"), "{log}"); + assert!(log.contains("s.1.g.1.tree.resource.routing"), "{log}"); + assert!(log.contains("reserved_sample_max="), "{log}"); + assert!(log.contains("max_sample_gap_ns="), "{log}"); +} diff --git a/app/crowdb-kv-server/tests/group0_control_plane_test.rs b/app/crowdb-paxos-kv-server/tests/group0_control_plane_test.rs similarity index 85% rename from app/crowdb-kv-server/tests/group0_control_plane_test.rs rename to app/crowdb-paxos-kv-server/tests/group0_control_plane_test.rs index 423c6f0c4..c2cb3d1c7 100644 --- a/app/crowdb-kv-server/tests/group0_control_plane_test.rs +++ b/app/crowdb-paxos-kv-server/tests/group0_control_plane_test.rs @@ -4,11 +4,11 @@ use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::group_operations::KvGroupOperationError; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::group_operations::KvGroupOperationError; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_server::group0_control_plane::Group0ControlPlane; fn group_zero_store() -> Arc<PxKvStore> { let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); diff --git a/app/crowdb-kv-server/tests/group0_discovery_test.rs b/app/crowdb-paxos-kv-server/tests/group0_discovery_test.rs similarity index 92% rename from app/crowdb-kv-server/tests/group0_discovery_test.rs rename to app/crowdb-paxos-kv-server/tests/group0_discovery_test.rs index e1a96efc4..c5b04766c 100644 --- a/app/crowdb-kv-server/tests/group0_discovery_test.rs +++ b/app/crowdb-paxos-kv-server/tests/group0_discovery_test.rs @@ -5,7 +5,7 @@ use crowdb_test_harness::test_dirs::TestDir; #[test] fn persisted_identity_rejects_conflicting_configuration_and_corruption() { - use crowdb_kv_server::background::identity::load_or_create; + use crowdb_paxos_kv_server::background::identity::load_or_create; let root = TestDir::new("service-identity").unwrap(); let first = load_or_create(root.path(), Some(42), Some(2)).unwrap(); diff --git a/app/crowdb-kv-server/tests/group_creation_test.rs b/app/crowdb-paxos-kv-server/tests/group_creation_test.rs similarity index 98% rename from app/crowdb-kv-server/tests/group_creation_test.rs rename to app/crowdb-paxos-kv-server/tests/group_creation_test.rs index ae4b09aba..94a9b2b48 100644 --- a/app/crowdb-kv-server/tests/group_creation_test.rs +++ b/app/crowdb-paxos-kv-server/tests/group_creation_test.rs @@ -7,7 +7,7 @@ mod common; mod system_bootstrap; use common::{request, TestGroupCreation}; -use crowdb_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole}; use reqwest::{Method, StatusCode}; use serde_json::Value; @@ -91,7 +91,7 @@ async fn explicit_removal_keeps_the_deployment_tombstone_and_restore_cannot_recr true ); fixture.create(&body, StatusCode::CONFLICT).await; - let restored = crowdb_kv_server::recovery::startup::create_group_with_wal( + let restored = crowdb_paxos_kv_server::recovery::startup::create_group_with_wal( 0, 1, 7, diff --git a/app/crowdb-kv-server/tests/keepalive_recovery_test.rs b/app/crowdb-paxos-kv-server/tests/keepalive_recovery_test.rs similarity index 87% rename from app/crowdb-kv-server/tests/keepalive_recovery_test.rs rename to app/crowdb-paxos-kv-server/tests/keepalive_recovery_test.rs index ff7f45341..bc7e93f81 100644 --- a/app/crowdb-kv-server/tests/keepalive_recovery_test.rs +++ b/app/crowdb-paxos-kv-server/tests/keepalive_recovery_test.rs @@ -2,10 +2,12 @@ // Licensed under the Apache License, Version 2.0. use axum::{extract::State, routing::get, Json, Router}; -use crowdb_kv::cluster::{group::PxGroup, KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::CrowDBConfig; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; -use crowdb_kv_server::{background::keepalive::KeepAliveLoop, store_registry::KvStoreRegistry}; +use crowdb_paxos_kv::cluster::{ + group::PxGroup, PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; +use crowdb_paxos_kv_server::{background::keepalive::KeepAliveLoop, store_registry::PaxosKvStoreRegistry}; use crowdb_protocol::common::KvServerIdentity; use std::{sync::Arc, time::Duration}; @@ -32,7 +34,7 @@ async fn slow_registration_does_not_add_its_latency_to_the_first_heartbeat_inter PxLocalReplica::new(1, PxLocalReplicaRole::Leader), )); store.start().await.unwrap(); - let registry = Arc::new(KvStoreRegistry::with_config(CrowDBConfig::for_tests())); + let registry = Arc::new(PaxosKvStoreRegistry::with_config(PaxosKvConfig::for_tests())); registry.add_store(0, &store); let delay = Arc::new(TestDelayedTopology { store: store.clone(), @@ -64,9 +66,9 @@ async fn slow_registration_does_not_add_its_latency_to_the_first_heartbeat_inter // Inject discovery latency after register() has stamped its first value. tokio::time::sleep(Duration::from_millis(900)).await; delay.release.notify_one(); - let client = CrowdbKvClient::new(ClientConfig::new(Vec::new())); + let client = PaxosKvClient::new(ClientConfig::new(Vec::new())); client.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); - let observations = crowdb_kv_client::ServiceRegistryClient::new(client); + let observations = crowdb_paxos_kv_client::ServiceRegistryClient::new(client); let initial = tokio::time::timeout(Duration::from_millis(250), async { loop { if let Some(value) = observations.read_instance("kv-server", 7).await.unwrap() { @@ -104,7 +106,7 @@ async fn a_recovered_singleton_registers_its_current_address_with_unreachable_re PxLocalReplica::new(1, PxLocalReplicaRole::Leader), )); store.start().await.unwrap(); - let registry = Arc::new(KvStoreRegistry::with_config(CrowDBConfig::for_tests())); + let registry = Arc::new(PaxosKvStoreRegistry::with_config(PaxosKvConfig::for_tests())); registry.add_store(0, &store); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let current = format!("http://{}", listener.local_addr().unwrap()); @@ -126,7 +128,7 @@ async fn a_recovered_singleton_registers_its_current_address_with_unreachable_re String::new(), 1, ); - let client = CrowdbKvClient::new(ClientConfig::new(vec![current.clone()])); + let client = PaxosKvClient::new(ClientConfig::new(vec![current.clone()])); client.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); let value = tokio::time::timeout(Duration::from_secs(3), async { loop { diff --git a/app/crowdb-kv-server/tests/member_routes_test.rs b/app/crowdb-paxos-kv-server/tests/member_routes_test.rs similarity index 95% rename from app/crowdb-kv-server/tests/member_routes_test.rs rename to app/crowdb-paxos-kv-server/tests/member_routes_test.rs index 4b8f65d72..ba14e119b 100644 --- a/app/crowdb-kv-server/tests/member_routes_test.rs +++ b/app/crowdb-paxos-kv-server/tests/member_routes_test.rs @@ -1,8 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv_server::recovery::member_routes::{apply, configure}; +use crowdb_paxos_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv_server::recovery::member_routes::{apply, configure}; use crowdb_protocol::mgmt::{ node::recovery::{MemberEndpoint, MemberEndpoints, RecoveryGroup, RecoveryGroups, RecoveryReplica}, SystemBootstrapIdentity, diff --git a/app/crowdb-kv-server/tests/monitor_supervisor_test.rs b/app/crowdb-paxos-kv-server/tests/monitor_supervisor_test.rs similarity index 94% rename from app/crowdb-kv-server/tests/monitor_supervisor_test.rs rename to app/crowdb-paxos-kv-server/tests/monitor_supervisor_test.rs index d8ac1c5f7..f683420a5 100644 --- a/app/crowdb-kv-server/tests/monitor_supervisor_test.rs +++ b/app/crowdb-paxos-kv-server/tests/monitor_supervisor_test.rs @@ -4,7 +4,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use crowdb_kv_server::background::domain_monitor::spawn_restarting_monitor; +use crowdb_paxos_kv_server::background::domain_monitor::spawn_restarting_monitor; #[tokio::test] async fn failed_monitor_task_is_restarted_and_shutdown_is_drained() { diff --git a/app/crowdb-kv-server/tests/reconcile_test.rs b/app/crowdb-paxos-kv-server/tests/reconcile_test.rs similarity index 89% rename from app/crowdb-kv-server/tests/reconcile_test.rs rename to app/crowdb-paxos-kv-server/tests/reconcile_test.rs index 8bce17559..b6e1220d1 100644 --- a/app/crowdb-kv-server/tests/reconcile_test.rs +++ b/app/crowdb-paxos-kv-server/tests/reconcile_test.rs @@ -5,20 +5,20 @@ use std::sync::Arc; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::common::config::CrowDBConfig; -use crowdb_kv_server::recovery::reconcile::{expand_membership_records, plan_reconcile, ReplicaRecord}; -use crowdb_kv_server::store_registry::KvStoreRegistry; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv_server::recovery::reconcile::{expand_membership_records, plan_reconcile, ReplicaRecord}; +use crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry; use crowdb_protocol::kv_membership::{GroupMember, GroupMembership, GroupMembershipState}; // ── Unit tests: plan_reconcile (pure decision logic) ──────────── /// Build a registry with one store (id 0) containing a group (`gid`) /// whose local replica is `rid` and has `remotes` wired. -fn registry_with_group(gid: u64, rid: u64, remotes: &[(u64, &str, bool)]) -> Arc<KvStoreRegistry> { - let registry = Arc::new(KvStoreRegistry::with_config(CrowDBConfig::for_tests())); +fn registry_with_group(gid: u64, rid: u64, remotes: &[(u64, &str, bool)]) -> Arc<PaxosKvStoreRegistry> { + let registry = Arc::new(PaxosKvStoreRegistry::with_config(PaxosKvConfig::for_tests())); let store = PxKvStore::new(0, "0.0.0.0:0".parse().unwrap()); let local = PxLocalReplica::new(rid, PxLocalReplicaRole::Follower); let mut group = PxGroup::new(gid, local); @@ -27,7 +27,7 @@ fn registry_with_group(gid: u64, rid: u64, remotes: &[(u64, &str, bool)]) -> Arc remotes .iter() .map(|(id, ep, v)| { - crowdb_kv::cluster::remote_replica::PxRemoteReplica::new(*id, (*ep).to_string()) + crowdb_paxos_kv::cluster::remote_replica::PxRemoteReplica::new(*id, (*ep).to_string()) .with_voting(*v) }) .collect(), diff --git a/app/crowdb-kv-server/tests/restore_test.rs b/app/crowdb-paxos-kv-server/tests/restore_test.rs similarity index 89% rename from app/crowdb-kv-server/tests/restore_test.rs rename to app/crowdb-paxos-kv-server/tests/restore_test.rs index 03946382d..9647eece0 100644 --- a/app/crowdb-kv-server/tests/restore_test.rs +++ b/app/crowdb-paxos-kv-server/tests/restore_test.rs @@ -3,7 +3,7 @@ //! Tests for restore mode (R104): local-disk scan + restart rebuild. -use crowdb_kv_server::recovery::restore::{group0_exists, scan_local_groups}; +use crowdb_paxos_kv_server::recovery::restore::{group0_exists, scan_local_groups}; // ── Unit tests: scan_local_groups / group0_exists ─────────────── @@ -35,15 +35,15 @@ async fn scan_local_groups_multi_store() { assert_eq!( groups, vec![ - crowdb_kv_server::recovery::restore::LocalGroup { + crowdb_paxos_kv_server::recovery::restore::LocalGroup { store_id: 0, group_id: 0 }, - crowdb_kv_server::recovery::restore::LocalGroup { + crowdb_paxos_kv_server::recovery::restore::LocalGroup { store_id: 1, group_id: 1 }, - crowdb_kv_server::recovery::restore::LocalGroup { + crowdb_paxos_kv_server::recovery::restore::LocalGroup { store_id: 1, group_id: 2 }, diff --git a/app/crowdb-paxos-kv-server/tests/runtime_shutdown_test.rs b/app/crowdb-paxos-kv-server/tests/runtime_shutdown_test.rs new file mode 100644 index 000000000..f8f8622e7 --- /dev/null +++ b/app/crowdb-paxos-kv-server/tests/runtime_shutdown_test.rs @@ -0,0 +1,104 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#[path = "common/runtime_shutdown.rs"] +mod common; +use common::{TestRuntimeShutdown, DEADLINE}; +use crowdb_paxos_kv::cluster::{ + group::PxGroup, PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxPrepareReply}; +use crowdb_paxos_kv::rpc::ReplicaDestination; +use crowdb_paxos_kv_server::shutdown::{shutdown_stores, shutdown_stores_once}; +use reqwest::{Method, StatusCode}; +use std::sync::Arc; + +async fn removal_preserves_accepted_work(method: Method, path: &str) { + let fixture = TestRuntimeShutdown::start().await; + let (status, body) = fixture.request(method.clone(), path).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert!(body.contains("shutdown incomplete"), "{body}"); + fixture.assert_retained(); + fixture.repair().await; + let (status, body) = fixture.request(method, path).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + fixture.group.local_replica().learner.engine().resume_from_slot(), + 2 + ); + assert!(!fixture.engine_marker.exists()); + assert!(!fixture.wal_marker.exists()); + if fixture.registry.get_store(1).is_some() { + assert!(fixture.store.shutdown(DEADLINE).await.is_clean()); + } +} +#[tokio::test(flavor = "current_thread")] +async fn store_removal_retains_registry_and_wal_until_accepted_gap_drains() { + removal_preserves_accepted_work(Method::DELETE, "/stores/1").await; +} +#[tokio::test(flavor = "current_thread")] +async fn group_removal_retains_registry_and_wal_until_accepted_gap_drains() { + removal_preserves_accepted_work(Method::DELETE, "/stores/1/groups/1").await; +} +#[tokio::test(flavor = "current_thread")] +async fn wipe_retains_registry_and_wal_until_accepted_gap_drains() { + removal_preserves_accepted_work(Method::POST, "/stores/1/groups/1/wipe-user-data").await; +} +#[tokio::test(flavor = "current_thread")] +async fn host_timeout_retains_all_peer_servers_and_waits_for_real_progress() { + let fixture = TestRuntimeShutdown::start().await; + let peer = Arc::new(PxKvStore::new(2, "127.0.0.1:0".parse().unwrap())); + peer.add_group_without_election(PxGroup::new( + 2, + PxLocalReplica::new(2, PxLocalReplicaRole::Follower), + )); + peer.start().await.unwrap(); + fixture.registry.add_store(2, &peer); + let report = shutdown_stores_once(&fixture.registry).await; + assert!(!report.is_clean()); + fixture.assert_retained(); + let shutdown = tokio::spawn({ + let registry = fixture.registry.clone(); + async move { shutdown_stores(®istry).await } + }); + let endpoint = peer.listen_addr().unwrap().to_string(); + let reply = tokio::time::timeout( + DEADLINE, + fixture.store.rpc_transport().unwrap().send_prepare( + ReplicaDestination { + endpoint: &endpoint, + replica_id: 2, + }, + 2, + PxBallot::new(1, 1), + 0, + 2, + 0, + ), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!(reply, PxPrepareReply::Promised { .. })); + assert!(!shutdown.is_finished()); + fixture.repair().await; + let report = tokio::time::timeout(DEADLINE, shutdown).await.unwrap().unwrap(); + assert!(report.is_clean(), "{:?}", report.errors); + assert_eq!( + fixture.group.local_replica().learner.engine().resume_from_slot(), + 2 + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn member_route_shutdown_releases_its_registry_before_returning() { + let registry = Arc::new( + crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry::with_config( + crowdb_paxos_kv::common::config::PaxosKvConfig::for_tests(), + ), + ); + let weak = Arc::downgrade(®istry); + let worker = crowdb_paxos_kv_server::recovery::member_routes::start(registry.clone()); + drop(registry); + worker.stop().await; + assert!(weak.upgrade().is_none()); +} diff --git a/app/crowdb-kv-server/tests/startup_test.rs b/app/crowdb-paxos-kv-server/tests/startup_test.rs similarity index 64% rename from app/crowdb-kv-server/tests/startup_test.rs rename to app/crowdb-paxos-kv-server/tests/startup_test.rs index 170380d1a..8a955a733 100644 --- a/app/crowdb-kv-server/tests/startup_test.rs +++ b/app/crowdb-paxos-kv-server/tests/startup_test.rs @@ -4,16 +4,18 @@ use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::group::ProposeResult; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::local_replica::PxLocalReplicaRole; -use crowdb_kv::common::config::{CrowDBConfig, WalConfig}; -use crowdb_kv::kv::CrowdbTreeBackend; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine}; -use crowdb_kv_server::recovery::startup::{create_group_with_wal, store_wal_root}; +use crowdb_paxos_kv::cluster::group::ProposeResult; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::PxLocalReplicaRole; +use crowdb_paxos_kv::common::config::{PaxosKvConfig, WalConfig}; +use crowdb_paxos_kv::kv::CrowdbTreeBackend; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine}; +use crowdb_paxos_kv_server::recovery::restore::{load_local_groups, LocalGroup}; +use crowdb_paxos_kv_server::recovery::startup::{create_group_with_wal, store_wal_root}; +use crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry; #[path = "startup_test/reclamation.rs"] mod reclamation; @@ -36,11 +38,11 @@ async fn incomplete_group_creation_retains_replica_identity_before_wal_is_discov let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("startup-identity"); let blocked_tree = temp.path().join("blocked-tree"); std::fs::write(&blocked_tree, b"not a directory").unwrap(); - let config = CrowDBConfig { + let config = PaxosKvConfig { wal_root: temp.path().join("wal"), config_root: temp.path().join("conf"), data_root: blocked_tree, - ..CrowDBConfig::for_tests() + ..PaxosKvConfig::for_tests() }; let result = create_group_with_wal( 0, @@ -53,13 +55,13 @@ async fn incomplete_group_creation_retains_replica_identity_before_wal_is_discov ) .await; assert!(result.is_err()); - let groups = crowdb_kv_server::recovery::restore::scan_local_groups(&config.wal_root) + let groups = crowdb_paxos_kv_server::recovery::restore::scan_local_groups(&config.wal_root) .await .unwrap(); assert!(groups .iter() .any(|group| group.store_id == 0 && group.group_id == 1)); - let persisted = crowdb_kv::cluster::node_config::NodeConfigStore::new(&config.config_root) + let persisted = crowdb_paxos_kv::cluster::node_config::NodeConfigStore::new(&config.config_root) .load() .await .unwrap(); @@ -70,6 +72,98 @@ async fn incomplete_group_creation_retains_replica_identity_before_wal_is_discov ); } +#[tokio::test] +async fn restore_rejects_corrupt_node_config_instead_of_using_cli_replica() { + let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("startup-corrupt-config"); + let config = PaxosKvConfig { + wal_root: temp.path().join("wal"), + config_root: temp.path().join("conf"), + data_root: temp.path().join("data"), + ..PaxosKvConfig::for_tests() + }; + let backend = Arc::new(IoBackend::detect()); + + create_group_with_wal( + 0, + 1, + 7, + PxLocalReplicaRole::Follower, + &config, + backend, + CrowdbTreeBackend::File, + ) + .await + .unwrap(); + tokio::fs::write(config.config_root.join("node-config.json"), b"{broken") + .await + .unwrap(); + + let registry = Arc::new(PaxosKvStoreRegistry::with_config(config)); + let error = load_local_groups( + &[LocalGroup { + store_id: 0, + group_id: 1, + }], + 99, + ®istry, + ) + .await + .expect_err("corrupt node config must block restore"); + assert!( + error.contains("failed to read persisted replica identity"), + "{error}" + ); +} + +#[tokio::test] +async fn startup_passes_wal_segment_tuning_to_the_group_writer() { + let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("startup-wal-tuning"); + let mut config = PaxosKvConfig { + wal_root: temp.path().join("wal"), + config_root: temp.path().join("conf"), + data_root: temp.path().join("data"), + ..PaxosKvConfig::for_tests() + }; + config.wal.wal_segment_size = 512; + + let group = create_group_with_wal( + 0, + 1, + 1, + PxLocalReplicaRole::Follower, + &config, + Arc::new(IoBackend::detect()), + CrowdbTreeBackend::File, + ) + .await + .unwrap(); + let wal = group.local_replica().wal().unwrap().clone(); + for slot in 1..=8 { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(slot, 1), + term: slot, + payload: Bytes::from(vec![u8::try_from(slot).unwrap(); 1024]), + }; + wal.append(&WALRecord::from_accepted(group.group_id(), &entry)) + .await + .unwrap(); + } + wal.seal_all().await.unwrap(); + + let replay = replay_group( + &Arc::new(IoBackend::detect()), + &[store_wal_root(&config.wal_root, 0)], + 1, + ) + .await + .unwrap(); + assert!( + replay.max_segment_id > 1, + "configured 512-byte segments must rotate" + ); +} + #[tokio::test] async fn create_group_with_wal_restores_and_resumes_at_next_slot() { let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("startup"); @@ -94,6 +188,20 @@ async fn create_group_with_wal_restores_and_resumes_at_next_slot() { .await .unwrap(); + // Replay may apply slot 2 only after a real accepted entry completes slot 1. + // A promise alone does not establish an empty slot. + wal.append(&WALRecord::from_accepted( + group_id, + &PxLogEntry { + slot: 1, + ballot: PxBallot::new(2, replica_id), + term: 2, + payload: Bytes::new(), + }, + )) + .await + .unwrap(); + let accepted_entry = PxLogEntry { slot: 2, ballot: PxBallot::new(3, replica_id), @@ -109,11 +217,11 @@ async fn create_group_with_wal_restores_and_resumes_at_next_slot() { wal.seal_all().await.unwrap(); let data_root = temp.path().join("data-root"); - let config = CrowDBConfig { + let config = PaxosKvConfig { wal_root: wal_root.clone(), config_root: config_root.clone(), data_root: data_root.clone(), - ..CrowDBConfig::for_tests() + ..PaxosKvConfig::for_tests() }; let group = create_group_with_wal( store_id, @@ -152,7 +260,11 @@ async fn create_group_with_wal_restores_and_resumes_at_next_slot() { let replay_wal_dir = store_wal_root(&config.wal_root, store_id); let replay = replay_group(&backend, &[replay_wal_dir], group_id).await.unwrap(); assert!(replay.records.iter().any(|record| { - record.slot == 3 && matches!(record.record_type, crowdb_kv::wal::record::RecordType::Accepted) + record.slot == 3 + && matches!( + record.record_type, + crowdb_paxos_kv::wal::record::RecordType::Accepted + ) })); } @@ -176,11 +288,11 @@ async fn crowdb_tree_engine_persists_across_restart(crowtree_backend: CrowdbTree let group_id = 5; let replica_id = 1; - let config = CrowDBConfig { + let config = PaxosKvConfig { wal_root: wal_root.clone(), config_root: config_root.clone(), data_root: data_root.clone(), - ..CrowDBConfig::for_tests() + ..PaxosKvConfig::for_tests() }; let group = create_group_with_wal( store_id, @@ -196,7 +308,8 @@ async fn crowdb_tree_engine_persists_across_restart(crowtree_backend: CrowdbTree // The durable crowdb-tree file was created under data_root, not left at the // default in-memory (no file) path. - let ct_path = crowdb_kv_server::recovery::startup::store_crowdb_tree_path(&data_root, store_id, group_id); + let ct_path = + crowdb_paxos_kv_server::recovery::startup::store_crowdb_tree_path(&data_root, store_id, group_id); assert!( ct_path.exists(), "expected a durable crowdb-tree file at {}", diff --git a/app/crowdb-kv-server/tests/startup_test/reclamation.rs b/app/crowdb-paxos-kv-server/tests/startup_test/reclamation.rs similarity index 96% rename from app/crowdb-kv-server/tests/startup_test/reclamation.rs rename to app/crowdb-paxos-kv-server/tests/startup_test/reclamation.rs index b999207c7..7ff16a367 100644 --- a/app/crowdb-kv-server/tests/startup_test/reclamation.rs +++ b/app/crowdb-paxos-kv-server/tests/startup_test/reclamation.rs @@ -2,16 +2,16 @@ // Licensed under the Apache License, Version 2.0. use super::*; -use crowdb_kv::wal::gc::run_gc_pass; +use crowdb_paxos_kv::wal::gc::run_gc_pass; #[tokio::test] async fn restored_wal_keeps_old_segments_visible_to_safe_reclamation() { let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("startup-wal-reclamation"); - let config = CrowDBConfig { + let config = PaxosKvConfig { wal_root: temp.path().join("wal"), config_root: temp.path().join("conf"), data_root: temp.path().join("data"), - ..CrowDBConfig::for_tests() + ..PaxosKvConfig::for_tests() }; let backend = Arc::new(IoBackend::File); let mut wal_config = WalConfig::with_root(store_wal_root(&config.wal_root, 0)); diff --git a/app/crowdb-paxos-kv-server/tests/tree_config_test.rs b/app/crowdb-paxos-kv-server/tests/tree_config_test.rs new file mode 100644 index 000000000..1323da3d8 --- /dev/null +++ b/app/crowdb-paxos-kv-server/tests/tree_config_test.rs @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_tree_ffi::{Config, Crowdbtree, CtError}; + +const CANONICAL: &str = include_str!("../conf/crowdb_paxos_kv_server_config.toml"); + +fn load(text: &str) -> Result<PaxosKvConfig, String> { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-config"); + let path = root.path().join("server.toml"); + std::fs::write(&path, text).unwrap(); + PaxosKvConfig::load_from_file(&path) +} + +#[test] +fn canonical_config_preserves_engine_and_runtime_defaults() { + let config = load(CANONICAL).unwrap(); + let defaults = PaxosKvConfig::default(); + assert_eq!( + serde_json::to_value(&config.tree).unwrap(), + serde_json::to_value(&defaults.tree).unwrap() + ); + let host = config.tree_runtime.runtime_options(); + let expected = defaults.tree_runtime.runtime_options(); + assert_eq!(host.workers, expected.workers); + assert_eq!(host.backend_workers, expected.backend_workers); + assert_eq!(host.backend_queue_limit, expected.backend_queue_limit); + assert_eq!(host.limits.reclaim_bytes, expected.limits.reclaim_bytes); + assert_eq!( + config.tree_runtime.binding_options().backend_queue_limit, + defaults.tree_runtime.binding_options().backend_queue_limit + ); + assert!(config.wal_early_ack); + assert!(config.async_engine_apply); +} + +#[test] +fn toml_key_limit_is_enforced_by_native_admission() { + let config = load(&CANONICAL.replace("max_key_size = 0", "max_key_size = 4")).unwrap(); + let mut options = Config::default(); + config.tree.apply_to(&mut options); + let tree = Crowdbtree::open(&options).unwrap(); + assert_eq!(tree.put(b"abcde", b"value"), Err(CtError::InvalidArgument)); + tree.put(b"abcd", b"value").unwrap(); + assert_eq!( + tree.get(b"abcd").unwrap().map(|(_, value)| value), + Some(b"value".to_vec()) + ); +} + +#[test] +fn enlarged_source_capacity_is_included_in_scan_admission() { + let config = + load(&CANONICAL.replace("source_max_entries = 4_096", "source_max_entries = 8_192")).unwrap(); + let mut options = Config::default(); + config.tree.apply_to(&mut options); + let tree = Crowdbtree::open(&options).unwrap(); + for index in 0..4200 { + tree.put(format!("key{index:05}").as_bytes(), b"value").unwrap(); + } + assert_eq!(tree.frozen_table_count(), 0); + let (entries, truncated) = tree.scan(b"", b"", b"", 0, 1024 * 1024, true, 0, false).unwrap(); + assert!(!truncated); + assert_eq!(entries.len(), 4200); + assert_eq!(entries.first().unwrap().key.as_ref(), b"key00000"); + assert_eq!(entries.last().unwrap().key.as_ref(), b"key04199"); +} + +#[test] +fn incompatible_or_unknown_work_windows_are_rejected_on_load() { + for (input, field) in [ + ( + CANONICAL.replace("flush_parallelism = 0", "flush_parallelism = 5"), + "flush_parallelism", + ), + ( + CANONICAL.replace("cache_max_pages = 256", "cache_max_pages = 0"), + "cache_max_pages", + ), + ( + CANONICAL.replace("source_max_gap_entries = 64", "source_max_gap_entries = 16"), + "coalesce_max_keys", + ), + ( + CANONICAL.replace("run_max_entries = 128", "run_entries_typo = 128"), + "run_entries_typo", + ), + ] { + assert!(load(&input).unwrap_err().contains(field), "{field}"); + } +} + +#[test] +fn partial_tables_keep_omitted_defaults() { + let config = load("[paxos]\ncoalesce_max_keys = 16\n[election]\nmaintenance_tick_ms = 3000\n[wal]\nwal_flush_batch_bytes = 131072\n").unwrap(); + assert_eq!(config.paxos.coalesce_max_keys, 16); + assert_eq!(config.paxos.max_inflight_proposals, 32); + assert_eq!(config.election.snapshot_time_threshold_ms, 600_000); + assert_eq!(config.wal.wal_segment_size, 64 * 1024 * 1024); + assert!(config.wal_early_ack); + assert!(config.async_engine_apply); +} diff --git a/app/crowdb-paxos-kv-server/tests/tree_runtime_host_test.rs b/app/crowdb-paxos-kv-server/tests/tree_runtime_host_test.rs new file mode 100644 index 000000000..1e534f3f9 --- /dev/null +++ b/app/crowdb-paxos-kv-server/tests/tree_runtime_host_test.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_paxos_kv::cluster::PxLocalReplicaRole; +use crowdb_paxos_kv::common::config::PaxosKvConfig; +use crowdb_paxos_kv_server::recovery::startup::{create_group_with_wal, create_group_with_wal_on_runtime}; +use crowdb_paxos_kv_server::store_registry::PaxosKvStoreRegistry; + +fn config(root: &std::path::Path) -> PaxosKvConfig { + let mut config = PaxosKvConfig { + wal_backend: "mem-block".to_owned(), + crowtree_backend: "mem-block".to_owned(), + wal_root: root.join("wal"), + config_root: root.join("config"), + data_root: root.join("tree"), + log_dir: root.join("log").to_string_lossy().into_owned(), + ..Default::default() + }; + config.tree_runtime.host.workers = Some(1); + config +} + +#[test] +fn incompatible_proposal_window_is_rejected_before_startup() { + let mut config = PaxosKvConfig::default(); + config.tree_runtime.tree.cpu_queue_limit = Some(16); + let Err(error) = PaxosKvStoreRegistry::try_with_config(config) else { + panic!("proposal window must fit its paid Tree lane"); + }; + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + assert!(error.to_string().contains("max_inflight_proposals")); +} + +#[tokio::test(flavor = "current_thread")] +async fn standalone_constructor_uses_runtime_without_a_mode_flag() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-runtime-host"); + let registry = PaxosKvStoreRegistry::try_with_config(config(root.path())).unwrap(); + let result = create_group_with_wal( + 1, + 1, + 1, + PxLocalReplicaRole::Leader, + ®istry.config, + Arc::clone(®istry.wal_backend), + registry.crowtree_backend, + ) + .await; + let group = result.unwrap(); + let engine = group.local_replica().learner.engine(); + assert!(engine.requires_contiguous_apply()); + let permit = engine.admit_proposal(Bytes::new()).await.unwrap().unwrap(); + permit.apply(1).await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn every_group_uses_the_same_registry_owned_native_host() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-runtime-host"); + let registry = PaxosKvStoreRegistry::try_with_config(config(root.path())).unwrap(); + let mut groups = Vec::new(); + for group_id in 1..=2 { + groups.push( + create_group_with_wal_on_runtime( + 1, + group_id, + 1, + PxLocalReplicaRole::Leader, + ®istry.config, + Arc::clone(®istry.wal_backend), + registry.crowtree_backend, + registry.tree_runtime.clone(), + ) + .await + .unwrap(), + ); + } + for group in &groups { + let engine = group.local_replica().learner.engine(); + assert!(engine.requires_contiguous_apply()); + let permit = engine.admit_proposal(Bytes::new()).await.unwrap().unwrap(); + permit.apply(1).await.unwrap(); + } + registry.tree_runtime.stop_admission(); + for group in &groups { + // This one host transition must be visible in both independent Trees. + assert!(group + .local_replica() + .learner + .engine() + .admit_proposal(Bytes::new()) + .await + .is_err()); + } +} diff --git a/app/crowdb-web/Cargo.toml b/app/crowdb-web/Cargo.toml index 67c848a14..f22c271f8 100644 --- a/app/crowdb-web/Cargo.toml +++ b/app/crowdb-web/Cargo.toml @@ -23,7 +23,7 @@ crowdb-chunkdb-client = { path = "../../lib/crowdb-chunkdb-client" } crowdb-common = { path = "../../lib/crowdb-common/rust" } crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } crowdb-diskdb-client = { path = "../../lib/crowdb-diskdb-client" } -crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../../lib/crowdb-paxos-kv-client" } crowdb-monitor = { path = "../../container/crowdb-monitor" } crowdb-protocol = { path = "../../lib/crowdb-protocol" } bincode = "1" @@ -56,6 +56,6 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "ne reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } serde_json = "1" tower = { version = "0.5", features = ["util"] } -crowdb-kv = { path = "../../lib/crowdb-kv" } +crowdb-paxos-kv = { path = "../../lib/crowdb-paxos-kv" } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } diff --git a/app/crowdb-web/src/chunk.rs b/app/crowdb-web/src/chunk.rs index 3249407fa..13036e054 100644 --- a/app/crowdb-web/src/chunk.rs +++ b/app/crowdb-web/src/chunk.rs @@ -14,7 +14,7 @@ use axum::http::StatusCode; use axum::Json; use crowdb_chunkdb_client::ChunkdbClient; use crowdb_console_shared::ops::hardware; -use crowdb_kv_client::{RangeBindingClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{Chunk, ListChunksRequest, QueryChunkRequest}; use crowdb_protocol::common::ChunkId; use serde::Deserialize; diff --git a/app/crowdb-web/src/chunk/slots.rs b/app/crowdb-web/src/chunk/slots.rs index edbb34239..1c19b973d 100644 --- a/app/crowdb-web/src/chunk/slots.rs +++ b/app/crowdb-web/src/chunk/slots.rs @@ -4,7 +4,7 @@ //! Bounded owner-specific windows over validated fixed slot maps. use axum::extract::{Query, State}; -use crowdb_kv_client::ChunkSlotMapClient; +use crowdb_paxos_kv_client::ChunkSlotMapClient; use crowdb_protocol::chunk_slot::{ChunkSlotBitmap, ChunkStorageGroup, CHUNK_SLOT_COUNT}; use serde::Deserialize; use serde_json::json; diff --git a/app/crowdb-web/src/chunk_kv.rs b/app/crowdb-web/src/chunk_kv.rs index 43a516434..efe2d893a 100644 --- a/app/crowdb-web/src/chunk_kv.rs +++ b/app/crowdb-web/src/chunk_kv.rs @@ -14,7 +14,7 @@ use axum::{ http::StatusCode, Json, }; -use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage}; use crowdb_protocol::key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, TextKey}; use serde::{de::DeserializeOwned, Deserialize}; @@ -48,7 +48,7 @@ pub(crate) async fn catalog( .map_err(|_| err_502("Chunk-KV catalog observation timed out"))? } -async fn read<T: DeserializeOwned>(kv: &CrowdbKvClient, key: &str, max_bytes: usize) -> Result<T, Failure> { +async fn read<T: DeserializeOwned>(kv: &PaxosKvClient, key: &str, max_bytes: usize) -> Result<T, Failure> { let value = match kv .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) .await diff --git a/app/crowdb-web/src/chunk_kv/balance.rs b/app/crowdb-web/src/chunk_kv/balance.rs index 8aac86b6f..5cc015209 100644 --- a/app/crowdb-web/src/chunk_kv/balance.rs +++ b/app/crowdb-web/src/chunk_kv/balance.rs @@ -3,7 +3,7 @@ //! Read the monitor's bounded diagnostic snapshot without scanning catalog/data. -use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_kv::{ balance::{BalanceObservation, MAX_OBSERVATION_BYTES, OBSERVATION_KEY}, ChunkKvRangeCatalogEntry, @@ -16,7 +16,7 @@ pub(super) struct Observation { reason: String, } -pub(super) async fn observe(kv: &CrowdbKvClient, generation: u64) -> Observation { +pub(super) async fn observe(kv: &PaxosKvClient, generation: u64) -> Observation { match load(kv, generation).await { Ok(value) => Observation { reason: value.reason.clone(), @@ -26,7 +26,7 @@ pub(super) async fn observe(kv: &CrowdbKvClient, generation: u64) -> Observation } } -async fn load(kv: &CrowdbKvClient, generation: u64) -> Result<BalanceObservation, String> { +async fn load(kv: &PaxosKvClient, generation: u64) -> Result<BalanceObservation, String> { let outcome = kv .get(0, 0, OBSERVATION_KEY.as_bytes(), ReadMode::Linearizable, None) .await diff --git a/app/crowdb-web/src/chunk_kv/runtime/discovery.rs b/app/crowdb-web/src/chunk_kv/runtime/discovery.rs index 68adcaa0f..92fc1f394 100644 --- a/app/crowdb-web/src/chunk_kv/runtime/discovery.rs +++ b/app/crowdb-web/src/chunk_kv/runtime/discovery.rs @@ -6,7 +6,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use crowdb_console_shared::config::ServiceType; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::{common::InstanceValue, key::InstanceKey}; use super::super::Failure; diff --git a/app/crowdb-web/src/corr_id.rs b/app/crowdb-web/src/corr_id.rs index 7d5d56188..7a1ceea21 100644 --- a/app/crowdb-web/src/corr_id.rs +++ b/app/crowdb-web/src/corr_id.rs @@ -1,14 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Axum middleware that propagates the `x-crowdb-kv-corr-id` header. +//! Axum middleware that propagates the `x-crowdb-paxos-kv-corr-id` header. //! //! For every inbound request: -//! 1. Read `x-crowdb-kv-corr-id` from the request, or generate a new one +//! 1. Read `x-crowdb-paxos-kv-corr-id` from the request, or generate a new one //! via `shared::corr_id::new()` if absent. //! 2. Open a `shared::corr_id::scope` so any outbound client call made //! inside the handler attaches the same id to its -//! `x-crowdb-kv-corr-id` header (see `ConsoleClient` / +//! `x-crowdb-paxos-kv-corr-id` header (see `ConsoleClient` / //! `ServerClient` plumbing). //! 3. Add the id to the response headers so the SPA / CLI can echo it //! in user-facing error messages and operators can grep one log diff --git a/app/crowdb-web/src/diskdb/lifecycle.rs b/app/crowdb-web/src/diskdb/lifecycle.rs index 09c77c58c..d38438157 100644 --- a/app/crowdb-web/src/diskdb/lifecycle.rs +++ b/app/crowdb-web/src/diskdb/lifecycle.rs @@ -20,15 +20,9 @@ use crowdb_console_shared::lifecycle::{self, DiskdbDeployRequest}; #[derive(Debug, Deserialize)] pub struct DeployDiskdbBody { pub rpc_port: u16, - /// Main listener port. When omitted, derived from `rpc_port`. - #[serde(default)] - pub listen_port: Option<u16>, - /// Internal HTTP health port. When omitted, derived from `rpc_port`. + /// HTTP listener; omitted values use the protocol default. #[serde(default)] pub http_port: Option<u16>, - /// Explicit RPC listener for policies with independent listener ranges. - #[serde(default)] - pub rpc_listen_port: Option<u16>, /// Normal `DiskDB` heartbeat cadence, retained across service restart. #[serde(default)] pub keepalive_interval_secs: Option<std::num::NonZeroU32>, @@ -36,24 +30,16 @@ pub struct DeployDiskdbBody { pub(super) fn validate_diskdb_ports( body: &DeployDiskdbBody, -) -> Result<(u16, u16, u16), (StatusCode, Json<ErrorBody>)> { - let listen_port = body.listen_port.unwrap_or(body.rpc_port); - let http_port = body.http_port.unwrap_or_else(|| body.rpc_port.saturating_add(1)); - let rpc_listen_port = body - .rpc_listen_port - .unwrap_or_else(|| body.rpc_port.saturating_add(2)); - let ports = [listen_port, http_port, rpc_listen_port]; - if ports.contains(&0) || (body.rpc_listen_port.is_none() && body.rpc_port > 65_533) { - return Err(err_400("DiskDB ports must be in the range 1..=65535")); - } - if ports[0] == ports[1] || ports[0] == ports[2] || ports[1] == ports[2] { - return Err(err_400("DiskDB listener ports must be distinct")); +) -> Result<(u16, u16), (StatusCode, Json<ErrorBody>)> { + let http = body.http_port.unwrap_or(crowdb_protocol::DISKDB_HTTP_BASE); + if http == 0 || body.rpc_port == 0 || http == body.rpc_port { + return Err(err_400("DiskDB HTTP and RPC ports must be distinct and nonzero")); } - Ok((listen_port, http_port, rpc_listen_port)) + Ok((http, body.rpc_port)) } #[cfg(target_os = "macos")] -fn ensure_local_ports_free(ports: [u16; 3]) -> Result<(), (StatusCode, Json<ErrorBody>)> { +fn ensure_local_ports_free(ports: [u16; 2]) -> Result<(), (StatusCode, Json<ErrorBody>)> { for port in ports { if std::net::TcpListener::bind(("127.0.0.1", port)).is_err() { return Err(err_409(format!("DiskDB listener port {port} is unavailable"))); @@ -105,10 +91,10 @@ async fn deploy_diskdb( "Group 0 is not ready; deploy Paxos-KV and initialize Group 0 before starting DiskDB", )); } - let (listen_port, http_port, rpc_listen_port) = validate_diskdb_ports(&body)?; + let (http_port, rpc_listen_port) = validate_diskdb_ports(&body)?; #[cfg(target_os = "macos")] - ensure_local_ports_free([listen_port, http_port, rpc_listen_port])?; - let _ports = crate::services::defaults::claim_ports(&state, &[listen_port, http_port, rpc_listen_port])?; + ensure_local_ports_free([http_port, rpc_listen_port])?; + let _ports = crate::services::defaults::claim_ports(&state, &[http_port, rpc_listen_port])?; let node = { let cfg = state.config.read().unwrap(); // Check for existing diskdb instance on this node. @@ -155,7 +141,6 @@ async fn deploy_diskdb( free_batch_enabled: None, free_flush_max_batch: None, server_id: format!("diskdb-{node_id}"), - listen_port, http_port, rpc_port: rpc_listen_port, kv_server_mgmt_seeds, @@ -176,7 +161,7 @@ async fn deploy_diskdb( url: deployed.endpoint.clone(), node_id: Some(node_id), rpc_url: Some(deployed.endpoint.clone()), - rest_port: None, + rest_port: Some(http_port), rpc_port: Some(body.rpc_port), auto_start: true, binary: None, @@ -247,8 +232,10 @@ async fn restart_diskdb( let (entry, node) = diskdb_restart_inputs(&state, node_id)?; let rpc_port = entry - .rpc_port - .or_else(|| entry.rpc_url.as_deref().and_then(crate::mgmt::port_of)) + .rpc_url + .as_deref() + .and_then(crate::mgmt::port_of) + .or(entry.rpc_port) .ok_or_else(|| err_500(format!("diskdb entry has malformed rpc_url: {:?}", entry.rpc_url)))?; // Stop existing process. @@ -269,10 +256,21 @@ async fn restart_diskdb( .collect() }; - // Backward-compat: derive from the old paired-port scheme. - let listen_port = rpc_port; - let http_port = rpc_port.saturating_add(1); - let rpc_listen_port = rpc_port.saturating_add(2); + // Retain the configured listeners across restart. + let (http_port, rpc_listen_port) = validate_diskdb_ports(&DeployDiskdbBody { + rpc_port, + http_port: entry.rest_port.or_else(|| { + state + .config + .read() + .unwrap() + .local_launches + .get(&entry.id) + .and_then(|launch| launch.readiness_url.as_deref()) + .and_then(crate::mgmt::port_of) + }), + keepalive_interval_secs: None, + })?; let req = DiskdbDeployRequest { instance_id: None, metrics_interval: None, @@ -283,7 +281,6 @@ async fn restart_diskdb( free_batch_enabled: None, free_flush_max_batch: None, server_id: format!("diskdb-{node_id}"), - listen_port, http_port, rpc_port: rpc_listen_port, kv_server_mgmt_seeds, @@ -303,7 +300,7 @@ async fn restart_diskdb( url: deployed.endpoint.clone(), node_id: Some(node_id), rpc_url: Some(deployed.endpoint.clone()), - rest_port: None, + rest_port: Some(http_port), rpc_port: Some(rpc_port), auto_start: entry.auto_start, binary: None, diff --git a/app/crowdb-web/src/diskdb/proxy.rs b/app/crowdb-web/src/diskdb/proxy.rs index 63097731b..6d5297941 100644 --- a/app/crowdb-web/src/diskdb/proxy.rs +++ b/app/crowdb-web/src/diskdb/proxy.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use tracing::warn; use crowdb_diskdb_client::{DiskdbClient, DiskdbRpcTransport}; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::common::{DiskGroupUsageSummary, DiskdbExtra, HwStatus, InstanceValue}; use crowdb_protocol::common_type::InstanceId; use crowdb_protocol::diskdb::rpc::{ @@ -354,7 +354,7 @@ pub async fn http_diskdb_usage( /// Returns `502` if group-0 is not available. pub async fn http_hardware_capacity( State(state): State<AppState>, -) -> Result<Json<crowdb_kv_client::HardwareCapacitySummary>, (StatusCode, Json<ErrorBody>)> { +) -> Result<Json<crowdb_paxos_kv_client::HardwareCapacitySummary>, (StatusCode, Json<ErrorBody>)> { let hw = build_hardware_client(&state) .await .ok_or_else(|| err_502("no group-0 endpoint; cluster not initialized"))?; @@ -612,7 +612,7 @@ pub async fn http_set_disk_group_owner( hw.set_owner(rack_id, node_id, dg_id, instance_id, body.lease_expiry_ms) .await .map_err(|e| match e { - crowdb_kv_client::Error::OwnerConflict { .. } => ( + crowdb_paxos_kv_client::Error::OwnerConflict { .. } => ( StatusCode::CONFLICT, Json(ErrorBody { error: format!("set_owner: {e}"), diff --git a/app/crowdb-web/src/diskdb/remote.rs b/app/crowdb-web/src/diskdb/remote.rs index 542aa022c..145370260 100644 --- a/app/crowdb-web/src/diskdb/remote.rs +++ b/app/crowdb-web/src/diskdb/remote.rs @@ -15,7 +15,7 @@ pub(super) async fn deploy( node_id: u64, body: &DeployDiskdbBody, ) -> Result<(StatusCode, Json<DiskdbDeployResult>), crate::services::Failure> { - let (_, _, rpc) = super::lifecycle::validate_diskdb_ports(body)?; + let (_, rpc) = super::lifecycle::validate_diskdb_ports(body)?; let binding = crate::node::cluster_binding(state)?.ok_or_else(|| err_409("Node is unbound"))?; let mut node = state .config @@ -65,9 +65,9 @@ pub fn configuration( seeds: &[String], body: &DeployDiskdbBody, ) -> Result<String, crate::services::Failure> { - let (listen, http, rpc) = super::lifecycle::validate_diskdb_ports(body)?; - let mut configuration = format!("[server]\nrpc_workers = 2\nlisten_addr = {:?}\nhttp_listen_addr = {:?}\nrpc_listen_addr = {:?}\ninstance_id = {:?}\nkv_server_mgmt_seeds = {:?}\n", - format!("{}:{listen}", node.host), format!("{}:{http}", node.host), format!("{}:{rpc}", node.host), node.id.to_string(), seeds); + let (http, rpc) = super::lifecycle::validate_diskdb_ports(body)?; + let mut configuration = format!("[server]\nrpc_workers = 2\nhttp_listen_addr = {:?}\nrpc_listen_addr = {:?}\ninstance_id = {:?}\nkv_server_mgmt_seeds = {:?}\n", + format!("{}:{http}", node.host), format!("{}:{rpc}", node.host), node.id.to_string(), seeds); if let Some(interval) = body.keepalive_interval_secs { writeln!(configuration, "\n[heartbeat]\ninterval_secs = {interval}") .expect("writing configuration to a String cannot fail"); diff --git a/app/crowdb-web/src/error.rs b/app/crowdb-web/src/error.rs index eaeb2382c..31f02d7f7 100644 --- a/app/crowdb-web/src/error.rs +++ b/app/crowdb-web/src/error.rs @@ -59,7 +59,7 @@ pub fn map_config_err(e: Error) -> (StatusCode, Json<ErrorBody>) { Error::UpstreamRpc { ref status, .. } if status.contains("HTTP 409") => { (StatusCode::CONFLICT, Json(ErrorBody { error: msg })) } - Error::KvClient(crowdb_kv_client::Error::MembershipConflict { .. }) => { + Error::KvClient(crowdb_paxos_kv_client::Error::MembershipConflict { .. }) => { (StatusCode::CONFLICT, Json(ErrorBody { error: msg })) } Error::NotFound { .. } => (StatusCode::NOT_FOUND, Json(ErrorBody { error: msg })), diff --git a/app/crowdb-web/src/kv.rs b/app/crowdb-web/src/kv.rs index b403cd05f..f1a62a4c3 100644 --- a/app/crowdb-web/src/kv.rs +++ b/app/crowdb-web/src/kv.rs @@ -11,7 +11,7 @@ use axum::http::StatusCode; use axum::Json; use crowdb_console_shared::cluster::{GroupHealth, NodeId}; use crowdb_console_shared::ops; -use crowdb_kv_client::{GetOutcome, ScanOutcome}; +use crowdb_paxos_kv_client::{GetOutcome, ScanOutcome}; use hex; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; diff --git a/app/crowdb-web/src/lib.rs b/app/crowdb-web/src/lib.rs index 510e09246..c1c6af70d 100644 --- a/app/crowdb-web/src/lib.rs +++ b/app/crowdb-web/src/lib.rs @@ -476,7 +476,7 @@ pub fn router(state: AppState) -> axum::Router { health::require_editable, )) .with_state(state) - // Propagate `x-crowdb-kv-corr-id` through every request: read it + // Propagate `x-crowdb-paxos-kv-corr-id` through every request: read it // (or mint one), open a task-local scope so outbound clients // attach it to their own headers, echo it back on the response. .layer(axum::middleware::from_fn(corr_id::corr_id_layer)) diff --git a/app/crowdb-web/src/lifecycle.rs b/app/crowdb-web/src/lifecycle.rs index 0d54dec0a..0bc59a807 100644 --- a/app/crowdb-web/src/lifecycle.rs +++ b/app/crowdb-web/src/lifecycle.rs @@ -220,7 +220,7 @@ pub async fn http_remove_node( let t0 = std::time::Instant::now(); // Cascade-stop the server process and drop its deployment record + // topology before removing the node, so a direct DELETE /api/nodes/:id - // does not orphan a running crowdb-kv-server. No-op when no server is + // does not orphan a running crowdb-paxos-kv-server. No-op when no server is // deployed (e.g. the UI already called DELETE .../server first). stop_and_remove_server_for_node(&state, id).await; let t_stop = t0.elapsed().as_millis(); @@ -620,7 +620,7 @@ pub async fn http_stop_node_server( /// deployed on the node. Does NOT persist — the caller persists. Does NOT /// remove the node itself. Used by both `DELETE /api/nodes/:id/server` /// and the node-delete cascade so a direct `DELETE /api/nodes/:id` does -/// not orphan a running crowdb-kv-server. +/// not orphan a running crowdb-paxos-kv-server. /// /// # Panics /// Panics if the `RwLock` is poisoned. @@ -945,13 +945,13 @@ async fn shutdown_kv_data(state: &AppState) -> Vec<String> { tracing::warn!(rack_id = rid, error = %e, "reset: remove_rack_cascade failed"); } } - let meta = crowdb_kv_client::KVClusterMetaClient::from_shared(hw.shared_kv()); + let meta = crowdb_paxos_kv_client::PaxosKvClusterMetaClient::from_shared(hw.shared_kv()); for sid in &stores { if let Err(e) = meta.remove_store(*sid).await { tracing::warn!(store_id = sid, error = %e, "reset: remove_store from sysdata failed"); } } - let svc = crowdb_kv_client::ServiceRegistryClient::from_shared(hw.shared_kv()); + let svc = crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(hw.shared_kv()); match svc.read_all_diskdb_instances().await { Ok(instances) => { for (instance_id, _) in &instances { diff --git a/app/crowdb-web/src/lifecycle/kv_process.rs b/app/crowdb-web/src/lifecycle/kv_process.rs index a5f8f360f..8a8694825 100644 --- a/app/crowdb-web/src/lifecycle/kv_process.rs +++ b/app/crowdb-web/src/lifecycle/kv_process.rs @@ -62,10 +62,10 @@ pub struct DeployNodeServerBody { /// `--send-queue-capacity` value for per-connection send queue. #[serde(default)] send_queue_capacity: Option<u32>, - /// Optional `--config` JSON path passed to the spawned `crowdb-kv-server`. + /// Optional `--config` JSON path passed to the spawned `crowdb-paxos-kv-server`. #[serde(default)] config: Option<String>, - /// `--rpc-workers` value for the spawned `crowdb-kv-server`. + /// `--rpc-workers` value for the spawned `crowdb-paxos-kv-server`. #[serde(default)] rpc_workers: Option<u32>, } @@ -105,7 +105,7 @@ pub async fn http_get_node_server( Ok(Json(entry)) } -/// `POST /api/nodes/:node_id/server/deploy`. Spawn `crowdb-kv-server` on +/// `POST /api/nodes/:node_id/server/deploy`. Spawn `crowdb-paxos-kv-server` on /// the node (local fork for `ssh_user=""`, SSH otherwise), wait for /// health, persist the deployment record. /// @@ -179,9 +179,9 @@ async fn deploy_node_server( } /// `POST /api/nodes/:node_id/server/restart`. Stop the tracked -/// `crowdb-kv-server` process on this node (if any) and immediately +/// `crowdb-paxos-kv-server` process on this node (if any) and immediately /// re-deploy on the same ports recorded in the `ServerEntry`. The -/// binary path falls back to `CROWDB_KV_SERVER_BIN` / `"crowdb-kv-server"` +/// binary path falls back to `CROWDB_PAXOS_KV_SERVER_BIN` / `"crowdb-paxos-kv-server"` /// the same way the initial deploy does when no `binary` override /// is supplied. Returns the new `DeployResult`. /// @@ -313,7 +313,7 @@ fn deployment_request( group0_management_seeds: (*state.authority_seeds).clone(), election_profile: body .election_profile - .or_else(|| std::env::var("CROWDB_KV_SERVER_ELECTION_PROFILE").ok()), + .or_else(|| std::env::var("CROWDB_PAXOS_KV_SERVER_ELECTION_PROFILE").ok()), binary: body.binary.map(std::path::PathBuf::from), kv_backend: body.kv_backend, wal_backend: body.wal_backend, diff --git a/app/crowdb-web/src/main.rs b/app/crowdb-web/src/main.rs index 89e4eab1a..e418e4412 100644 --- a/app/crowdb-web/src/main.rs +++ b/app/crowdb-web/src/main.rs @@ -17,7 +17,7 @@ struct Args { #[arg(long, conflicts_with = "config")] bind: Option<String>, - /// Port for the web server (default: 9090) + /// Port for the web server (default: 6135) #[arg(long, conflicts_with = "config", value_parser = clap::value_parser!(u16).range(1..))] port: Option<u16>, @@ -49,7 +49,7 @@ struct Args { #[arg(long, conflicts_with = "config")] skip_startup_restore: bool, - /// Log directory. Default: ~/.crowdb-kv/log. + /// Log directory. Default: ~/.crowdb-paxos-kv/log. #[arg(long, conflicts_with = "config")] log_dir: Option<std::path::PathBuf>, @@ -90,14 +90,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> { let launch_registry = args.registry.as_deref().map(LaunchRegistry::load).transpose()?; let _log_guards = init_logging(&args, process_config.as_ref())?; - let bind = process_config.as_ref().map_or_else( - || args.bind.as_deref().unwrap_or("0.0.0.0"), - |config| config.bind.as_str(), - ); - let port = process_config - .as_ref() - .map_or_else(|| args.port.unwrap_or(9090), |config| config.port); - let addr: SocketAddr = format!("{bind}:{port}").parse()?; + let addr = listen_address(&args, process_config.as_ref())?; info!(%addr, "crowdb-web starting"); let mut test_namespace = if args.test_mode { @@ -183,6 +176,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> { Ok(()) } +fn listen_address( + args: &Args, + process_config: Option<&WebProcessConfig>, +) -> Result<SocketAddr, std::net::AddrParseError> { + let bind = process_config.map_or_else( + || args.bind.as_deref().unwrap_or("0.0.0.0"), + |config| config.bind.as_str(), + ); + let port = process_config.map_or_else( + || args.port.unwrap_or(crowdb_protocol::WEB_BASE), + |config| config.port, + ); + format!("{bind}:{port}").parse() +} + fn init_logging( args: &Args, process_config: Option<&WebProcessConfig>, diff --git a/app/crowdb-web/src/managed.rs b/app/crowdb-web/src/managed.rs index 9a30d54d8..cd22b56ff 100644 --- a/app/crowdb-web/src/managed.rs +++ b/app/crowdb-web/src/managed.rs @@ -6,8 +6,8 @@ use axum::extract::State; use axum::http::StatusCode; use axum::Json; use crowdb_console_shared::config::web::WebMode; -use crowdb_kv_client::CrowdbSysmdClient; use crowdb_monitor::{MonitorStatus, ServiceStatus, StatusStore}; +use crowdb_paxos_kv_client::CrowdbSysmdClient; use crowdb_protocol::common::{ReplicaValue, StoreValue}; use serde::Serialize; use serde_json::{json, Value}; @@ -110,7 +110,7 @@ async fn validate_live_store_nodes( sysmd: &CrowdbSysmdClient, stores: &[StoreValue], replicas: &[ReplicaValue], -) -> Result<(), crowdb_kv_client::Error> { +) -> Result<(), crowdb_paxos_kv_client::Error> { let mut live_nodes = BTreeSet::new(); for (_, instance) in sysmd.read_all_kv_server_instances().await? { let Some(node_id) = instance @@ -122,7 +122,7 @@ async fn validate_live_store_nodes( continue; }; if instance.rpc_endpoint.is_empty() || !live_nodes.insert(node_id) { - return Err(crowdb_kv_client::Error::Topology( + return Err(crowdb_paxos_kv_client::Error::Topology( "live KV management registration is ambiguous".into(), )); } @@ -133,7 +133,7 @@ async fn validate_live_store_nodes( .chain(replicas.iter().map(|replica| &replica.node_id)) .any(|node_id| !live_nodes.contains(node_id)) { - return Err(crowdb_kv_client::Error::Topology( + return Err(crowdb_paxos_kv_client::Error::Topology( "store node has no live KV management registration".into(), )); } @@ -164,7 +164,7 @@ async fn load_snapshot(state: &AppState) -> Result<ManagedSnapshot, SnapshotFail let disks = sysmd.list_all_disks().await?; let stores = sysmd.list_stores().await?; if racks.is_empty() || nodes.is_empty() || !stores.iter().any(|store| store.store_id == 0) { - return Err(crowdb_kv_client::Error::Topology("managed topology is incomplete".into())); + return Err(crowdb_paxos_kv_client::Error::Topology("managed topology is incomplete".into())); } let disk_group_nodes = disk_groups diff --git a/app/crowdb-web/src/mgmt.rs b/app/crowdb-web/src/mgmt.rs index 9ba883a38..8b831f59e 100644 --- a/app/crowdb-web/src/mgmt.rs +++ b/app/crowdb-web/src/mgmt.rs @@ -159,7 +159,7 @@ pub(crate) fn rpc_is_conflict(err: &SharedError) -> bool { } /// Return the bare `host:port` of the crowdb-rpc listener that hosts `store_id` -/// on `node_id`. Each `PxKvStore` on a `crowdb-kv-server` binds its own +/// on `node_id`. Each `PxKvStore` on a `crowdb-paxos-kv-server` binds its own /// random port, so the bootstrap `ServerEntry::rpc_url` only points at /// the store created at process start (id 1). Operator-created stores /// must be looked up via the monitor cache, which carries the actual @@ -193,7 +193,7 @@ pub(crate) fn strip_scheme(s: String) -> String { } } -/// Parse `port` out of a URL like `http://host:10000` or `host:10000`. +/// Parse `port` out of a URL like `http://host:10135` or `host:10135`. /// Returns `None` on any shape we don't recognise. #[must_use] pub(crate) fn port_of(url: &str) -> Option<u16> { @@ -219,11 +219,13 @@ pub(crate) fn remap_zero_host(addr: &str) -> String { /// leader instead of failing with "no seeds configured". /// Returns `None` when no group-0 endpoint is known (e.g. cluster not /// yet initialized). -pub(crate) async fn build_hardware_client(state: &AppState) -> Option<crowdb_kv_client::HardwareClient> { +pub(crate) async fn build_hardware_client( + state: &AppState, +) -> Option<crowdb_paxos_kv_client::HardwareClient> { if state.managed_mode { let kv = state.kv_client().await; kv.refresh_topology().await.ok()?; - return Some(crowdb_kv_client::HardwareClient::from_shared(kv)); + return Some(crowdb_paxos_kv_client::HardwareClient::from_shared(kv)); } if state.node_monitor_url.is_some() { let binding = crate::node::cluster_binding(state).ok()??; @@ -232,7 +234,7 @@ pub(crate) async fn build_hardware_client(state: &AppState) -> Option<crowdb_kv_ // Admitted remote services have no PID in this console's namespace. // A persisted binding supplies discovery before any browser warms the // process/topology observation cache, including after a web restart. - return Some(crowdb_kv_client::HardwareClient::from_shared(kv)); + return Some(crowdb_paxos_kv_client::HardwareClient::from_shared(kv)); } let snap = state.monitor_cache.snapshot().await; if snap.is_empty() { @@ -283,7 +285,7 @@ pub(crate) async fn build_hardware_client(state: &AppState) -> Option<crowdb_kv_ } } } - // Reuse the shared CrowdbKvClient — its topology cache persists + // Reuse the shared PaxosKvClient — its topology cache persists // across requests so the group-0 leader is already known from prior // op_context / clusterInit calls. Creating a new client here would // trigger a 5s leader-discovery retry on every poll, saturating the @@ -324,7 +326,7 @@ pub(crate) async fn build_hardware_client(state: &AppState) -> Option<crowdb_kv_ // refresh retry. Log at warn so this is visible. warn!("build_hardware_client: no rpc endpoint resolved; relying on shared client's cached leader"); } - Some(crowdb_kv_client::HardwareClient::from_shared(kv)) + Some(crowdb_paxos_kv_client::HardwareClient::from_shared(kv)) } /// Check whether group-0 (store 0) is available for a sysdata read. @@ -388,7 +390,7 @@ mod tests { NodeStore { node_id: 7, store_id: 0, - listen_addr: Some("127.0.0.1:10001".into()), + listen_addr: Some("127.0.0.1:10136".into()), groups: Vec::new(), }, ); diff --git a/app/crowdb-web/src/mgmt/group_ops.rs b/app/crowdb-web/src/mgmt/group_ops.rs index 646aac5e2..7f80421f9 100644 --- a/app/crowdb-web/src/mgmt/group_ops.rs +++ b/app/crowdb-web/src/mgmt/group_ops.rs @@ -180,7 +180,7 @@ fn project_group( gid: u64, members: Vec<ReplicaValue>, reports: Vec<Option<NodeGroup>>, - membership: Option<&crowdb_kv_client::GroupMembershipSnapshot>, + membership: Option<&crowdb_paxos_kv_client::GroupMembershipSnapshot>, ) -> GroupView { let mut replicas = Vec::with_capacity(members.len()); let mut observed = 0usize; diff --git a/app/crowdb-web/src/node/admission.rs b/app/crowdb-web/src/node/admission.rs index 69e67d691..d5ffe1c1e 100644 --- a/app/crowdb-web/src/node/admission.rs +++ b/app/crowdb-web/src/node/admission.rs @@ -343,7 +343,10 @@ pub(super) fn update_config( if config.server_for_node(record.node_id).is_none() { let (management_port, rpc_port) = setup .and_then(crowdb_protocol::mgmt::node::setup::NodeSetupChoices::paxos_kv_ports) - .unwrap_or((10000, 10100)); + .unwrap_or(( + crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE, + crowdb_protocol::PAXOS_KV_SERVER_RPC_BASE, + )); let mut server = ServerEntry::new( format!("kv-{}", record.node_id), format!("http://{}:{management_port}", bracket_host(&record.host)), diff --git a/app/crowdb-web/src/node/admission/recovery.rs b/app/crowdb-web/src/node/admission/recovery.rs index 9f3dbd26f..53539e7e5 100644 --- a/app/crowdb-web/src/node/admission/recovery.rs +++ b/app/crowdb-web/src/node/admission/recovery.rs @@ -13,7 +13,7 @@ use crowdb_console_shared::{ }, ops::OpContext, }; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::mgmt::node::{NodeBinding, NodeControl}; use std::path::Path; diff --git a/app/crowdb-web/src/node/authority.rs b/app/crowdb-web/src/node/authority.rs index c562d3a86..8a0a91ab7 100644 --- a/app/crowdb-web/src/node/authority.rs +++ b/app/crowdb-web/src/node/authority.rs @@ -9,7 +9,7 @@ use axum::{ Json, }; use crowdb_console_shared::deployment::PreparedBootstrap; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use serde_json::{json, Value}; use crate::{ diff --git a/app/crowdb-web/src/node/authority/projection.rs b/app/crowdb-web/src/node/authority/projection.rs index 63d576306..bcac3da5e 100644 --- a/app/crowdb-web/src/node/authority/projection.rs +++ b/app/crowdb-web/src/node/authority/projection.rs @@ -30,22 +30,22 @@ pub(crate) fn project(servers: &mut Vec<ServerEntry>, bootstrap: Vec<ServerEntry let management_port = reqwest::Url::parse(&server.url) .ok() .and_then(|url| url.port_or_known_default()) - .unwrap_or(10000); + .unwrap_or(crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE); let rpc_port = server .rpc_url .as_deref() .and_then(|url| reqwest::Url::parse(&format!("http://{url}")).ok()) .and_then(|url| url.port_or_known_default()) - .unwrap_or(10100); + .unwrap_or(crowdb_protocol::PAXOS_KV_SERVER_RPC_BASE); server.url = format!("http://{host}:{management_port}"); server.rpc_url = Some(format!("{host}:{rpc_port}")); } else { let mut server = crowdb_console_shared::config::ServerEntry::new( format!("kv-{}", node.node_id), - format!("http://{host}:10000"), + format!("http://{host}:{}", crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE), ); server.node_id = Some(node.node_id); - server.rpc_url = Some(format!("{host}:10100")); + server.rpc_url = Some(format!("{host}:{}", crowdb_protocol::PAXOS_KV_SERVER_RPC_BASE)); servers.push(server); } } diff --git a/app/crowdb-web/src/node/auto_join/runner.rs b/app/crowdb-web/src/node/auto_join/runner.rs index ab4ae78e5..d22533b5b 100644 --- a/app/crowdb-web/src/node/auto_join/runner.rs +++ b/app/crowdb-web/src/node/auto_join/runner.rs @@ -260,7 +260,7 @@ async fn admit_candidate( discovery_id: id.clone(), rack_id: record.map_or(default_rack, |record| record.rack_id), ssh_user: record.map_or_else(|| "crowdb".into(), |record| record.ssh_user.clone()), - ssh_port: record.map_or(2222, |record| record.ssh_port), + ssh_port: record.map_or(crowdb_protocol::SSH_PORT, |record| record.ssh_port), ssh_password: password, preflight_operation_id: None, setup: None, diff --git a/app/crowdb-web/src/node/cluster_status.rs b/app/crowdb-web/src/node/cluster_status.rs index 3d8c19ed3..a28ebc5bd 100644 --- a/app/crowdb-web/src/node/cluster_status.rs +++ b/app/crowdb-web/src/node/cluster_status.rs @@ -12,7 +12,7 @@ use crowdb_console_shared::{ ops::{cluster::readiness, OpContext}, ConsoleConfig, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::mgmt::{ cluster::{ClusterStatus, DependencyKind, DependencyObservation, DependencyState, ReadinessReason}, node::observation::NodeRuntimeObservation, @@ -37,7 +37,7 @@ pub(crate) async fn read(State(state): State<AppState>) -> Result<Json<ClusterSt .binding .as_ref() .map_or_else(Vec::new, |binding| binding.management_seeds.clone()); - let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); + let client = Arc::new(PaxosKvClient::new(ClientConfig::new(seeds.clone()))); let context = OpContext::with_shared_client_preserving_hint(client, &seeds, ConsoleConfig::default()); let id = local.binding.as_ref().map_or_else( || "local-node/observation".into(), diff --git a/app/crowdb-web/src/node/storage/authority.rs b/app/crowdb-web/src/node/storage/authority.rs index cb40c72c2..8508fb91e 100644 --- a/app/crowdb-web/src/node/storage/authority.rs +++ b/app/crowdb-web/src/node/storage/authority.rs @@ -8,7 +8,7 @@ use crate::{ error::{err_400, err_404, err_409, err_502}, state::AppState, }; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::{key::DiskKey, DiskIdExt}; pub(super) async fn disk( diff --git a/app/crowdb-web/src/node/storage/lifecycle.rs b/app/crowdb-web/src/node/storage/lifecycle.rs index e75c12088..40080c536 100644 --- a/app/crowdb-web/src/node/storage/lifecycle.rs +++ b/app/crowdb-web/src/node/storage/lifecycle.rs @@ -12,7 +12,7 @@ use axum::{ extract::{Path, State}, Json, }; -use crowdb_kv_client::{Error, HardwareClient}; +use crowdb_paxos_kv_client::{Error, HardwareClient}; use crowdb_protocol::{common::HwStatus, disk_storage::DiskRemovalRecord, key::DiskKey, DiskIdExt}; use serde::{Deserialize, Serialize}; diff --git a/app/crowdb-web/src/node/target.rs b/app/crowdb-web/src/node/target.rs index 952b40eea..6b8c29d1b 100644 --- a/app/crowdb-web/src/node/target.rs +++ b/app/crowdb-web/src/node/target.rs @@ -5,7 +5,7 @@ use axum::{extract::State, http::StatusCode, Json}; use crowdb_console_shared::deployment::{registry, PreparedBootstrap, CLUSTER_KEY}; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::mgmt::cluster::ClusterTargetIdentity; use crate::{ diff --git a/app/crowdb-web/src/physical/disk.rs b/app/crowdb-web/src/physical/disk.rs index a97f8b50a..6b86cac2d 100644 --- a/app/crowdb-web/src/physical/disk.rs +++ b/app/crowdb-web/src/physical/disk.rs @@ -5,7 +5,7 @@ use axum::{http::StatusCode, Json}; use crowdb_console_shared::config::{DiskEntry, DiskGroupEntry}; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crate::{ error::{err_409, err_502, map_config_err, map_persist_err, ErrorBody}, diff --git a/app/crowdb-web/src/physical/node.rs b/app/crowdb-web/src/physical/node.rs index 74c656956..cb064af3b 100644 --- a/app/crowdb-web/src/physical/node.rs +++ b/app/crowdb-web/src/physical/node.rs @@ -4,7 +4,7 @@ //! Per-node store / group / remote primitives (A4). //! //! Key work: read endpoints served from the monitor cache, low-level -//! mutators that proxy to the upstream `crowdb-kv-server` management API +//! mutators that proxy to the upstream `crowdb-paxos-kv-server` management API //! and invalidate the affected node in the monitor cache on success. use crate::error::{err_500, map_err, ErrorBody}; diff --git a/app/crowdb-web/src/services/defaults.rs b/app/crowdb-web/src/services/defaults.rs index 976320edd..3245a06cd 100644 --- a/app/crowdb-web/src/services/defaults.rs +++ b/app/crowdb-web/src/services/defaults.rs @@ -8,6 +8,7 @@ use crate::{ }; use axum::{extract::State, Json}; use crowdb_console_shared::{config::ServiceType, ConsoleConfig}; +use crowdb_protocol::ServicePort; use serde_json::{json, Value}; use std::collections::{BTreeMap, HashSet}; @@ -28,11 +29,6 @@ fn occupied(config: &ConsoleConfig) -> HashSet<u16> { ports.extend(server.rest_port); ports.extend(server.access_origins.values().filter_map(|origin| port(origin))); ports.extend(server.rpc_port); - if server.service_type == ServiceType::Diskdb { - if let Some(rpc) = server.rpc_url.as_deref().and_then(port) { - ports.extend([rpc.saturating_sub(2), rpc.saturating_sub(1)]); - } - } } for launch in config.local_launches.values() { for name in [ @@ -51,15 +47,16 @@ fn available(port: u16) -> bool { std::net::TcpListener::bind((std::net::Ipv4Addr::UNSPECIFIED, port)).is_ok() } -fn next(used: &mut HashSet<u16>, start: u16, width: u16) -> Result<u16, Failure> { - for candidate in start..=32767 - width { - if (candidate..candidate + width).all(|port| !used.contains(&port) && available(port)) { - used.extend(candidate..candidate + width); +fn next(used: &mut HashSet<u16>, service: ServicePort) -> Result<u16, Failure> { + for instance in 0..service.range_size() { + let candidate = service.port(instance); + if !used.contains(&candidate) && available(candidate) { + used.insert(candidate); return Ok(candidate); } } Err(err_409( - "No free listener range is available; release a port before deploying", + "No free port remains in the service's default range; specify a free listener explicitly", )) } @@ -68,13 +65,45 @@ pub(super) async fn get(State(state): State<AppState>) -> Result<Json<Value>, Fa let mut used = occupied(&config); used.extend(super::plans::reserved_ports(&state)?); let mut result = BTreeMap::new(); - for (kind, base) in [ - ("paxos-kv", 19910), - ("diskdb", 29920), - ("chunkdb", 12010), - ("diskio", 13010), - ("chunk-kv", 15010), - ("access-server", 9091), + for (kind, listeners) in [ + ( + "paxos-kv", + vec![ + ("http_port", ServicePort::PaxosKvServerMgmt), + ("rpc_port", ServicePort::PaxosKvServerRpc), + ], + ), + ( + "diskdb", + vec![ + ("http_port", ServicePort::DiskdbHttp), + ("rpc_port", ServicePort::DiskdbRpc), + ], + ), + ( + "chunkdb", + vec![ + ("http_port", ServicePort::ChunkdbHttp), + ("rpc_port", ServicePort::ChunkdbRpc), + ], + ), + ("diskio", vec![("rpc_port", ServicePort::DiskioRpc)]), + ( + "chunk-kv", + vec![ + ("http_port", ServicePort::ChunkKvHttp), + ("rpc_port", ServicePort::ChunkKvRpc), + ], + ), + ( + "access-server", + vec![ + ("s3_port", ServicePort::AccessServerHttp), + ("http_port", ServicePort::AccessServerIcebergHttp), + ("health_port", ServicePort::AccessServerHealthHttp), + ("dataset_port", ServicePort::AccessServerDatasetHttp), + ], + ), ] { let instance = config .servers @@ -85,20 +114,9 @@ pub(super) async fn get(State(state): State<AppState>) -> Result<Json<Value>, Fa .checked_add(1) .filter(|id| i64::try_from(*id).is_ok()) .ok_or_else(|| err_409("Service instance ID space exhausted"))?; - let first = next(&mut used, base, if kind == "diskdb" { 3 } else { 1 })?; let mut value = json!({"instance_id":instance.to_string()}); - match kind { - "diskdb" | "diskio" => value["rpc_port"] = json!(first), - "access-server" => { - value["s3_port"] = json!(first); - value["http_port"] = json!(next(&mut used, base + 1, 1)?); - value["health_port"] = json!(next(&mut used, base + 2, 1)?); - value["dataset_port"] = json!(next(&mut used, base + 3, 1)?); - } - _ => { - value["http_port"] = json!(first); - value["rpc_port"] = json!(next(&mut used, base + 100, 1)?); - } + for (field, service) in listeners { + value[field] = json!(next(&mut used, service)?); } if kind == "chunkdb" { if let Some(server) = config diff --git a/app/crowdb-web/src/services/dependencies.rs b/app/crowdb-web/src/services/dependencies.rs index d438c5c4a..889cbf1e1 100644 --- a/app/crowdb-web/src/services/dependencies.rs +++ b/app/crowdb-web/src/services/dependencies.rs @@ -7,14 +7,14 @@ pub(crate) async fn group0_ready(state: &AppState) -> bool { if state.service_operations.load().contains("cluster/init") { return false; } - let sysmd = crowdb_kv_client::CrowdbSysmdClient::from_shared(state.kv_client().await); + let sysmd = crowdb_paxos_kv_client::CrowdbSysmdClient::from_shared(state.kv_client().await); tokio::time::timeout(std::time::Duration::from_secs(2), sysmd.get_group(0, 0)) .await .is_ok_and(|result| result.is_ok_and(|group| group.is_some())) } use axum::{extract::State, Json}; use crowdb_console_shared::config::ServiceType; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use serde_json::{json, Value}; use std::collections::HashSet; diff --git a/app/crowdb-web/src/services/deployment/chunk_slots.rs b/app/crowdb-web/src/services/deployment/chunk_slots.rs index 4995f673e..ffa4babf8 100644 --- a/app/crowdb-web/src/services/deployment/chunk_slots.rs +++ b/app/crowdb-web/src/services/deployment/chunk_slots.rs @@ -8,7 +8,7 @@ use crate::{ services::Failure, state::AppState, }; -use crowdb_kv_client::{ChunkSlotMapClient, CrowdbSysmdClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, CrowdbSysmdClient, GetOutcome, ReadMode}; use crowdb_protocol::{ chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, key::{ChunkSlotMapHeadKey, TextKey}, diff --git a/app/crowdb-web/src/services/deployment/credentials.rs b/app/crowdb-web/src/services/deployment/credentials.rs index fe3963958..c6be8a4d8 100644 --- a/app/crowdb-web/src/services/deployment/credentials.rs +++ b/app/crowdb-web/src/services/deployment/credentials.rs @@ -13,7 +13,7 @@ pub(super) async fn prepare( root: &Path, spec: &LocalLaunchSpec, seeds: &[String], - kv: &crowdb_kv_client::CrowdbKvClient, + kv: &crowdb_paxos_kv_client::PaxosKvClient, ) -> Result<(), Failure> { let s3_endpoint = spec .env diff --git a/app/crowdb-web/src/services/deployment/geometry.rs b/app/crowdb-web/src/services/deployment/geometry.rs index 3d15b7ca6..8e2ddd119 100644 --- a/app/crowdb-web/src/services/deployment/geometry.rs +++ b/app/crowdb-web/src/services/deployment/geometry.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::chunk_allocation_geometry::uniform_allocation_unit; use crowdb_protocol::common::HwStatus; diff --git a/app/crowdb-web/src/services/deployment/launch.rs b/app/crowdb-web/src/services/deployment/launch.rs index c5282e5bf..2805b8175 100644 --- a/app/crowdb-web/src/services/deployment/launch.rs +++ b/app/crowdb-web/src/services/deployment/launch.rs @@ -189,15 +189,16 @@ pub(super) async fn chunk_kv_config( config["storage"]["stream_chunk_capacity_bytes"] = json!(bytes); } if let Some(group) = body.bootstrap_group_id { - let client = - crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new(seeds.to_vec())); + let client = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( + seeds.to_vec(), + )); tokio::time::timeout( std::time::Duration::from_secs(5), client.get( store, group, b"console-deployment-probe", - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ), ) diff --git a/app/crowdb-web/src/services/deployment/planned.rs b/app/crowdb-web/src/services/deployment/planned.rs index 0dfe87e3b..c47679e1e 100644 --- a/app/crowdb-web/src/services/deployment/planned.rs +++ b/app/crowdb-web/src/services/deployment/planned.rs @@ -148,9 +148,6 @@ async fn build( let name = service_name(selected.kind); let id = format!("{name}-{}", node.id); let (configuration, environment) = if selected.kind == SetupServiceKind::Diskdb { - if selected.listeners.listen_port.is_some() { - choices["rpc_listen_port"] = json!(selected.listeners.rpc_port); - } let body: crate::diskdb::DeployDiskdbBody = serde_json::from_value(choices).map_err(|error| err_400(error.to_string()))?; ( diff --git a/app/crowdb-web/src/services/observation.rs b/app/crowdb-web/src/services/observation.rs index f3fe88a34..6a38cb820 100644 --- a/app/crowdb-web/src/services/observation.rs +++ b/app/crowdb-web/src/services/observation.rs @@ -27,7 +27,7 @@ use axum::{extract::State, Json}; use crowdb_console_shared::{cluster::NodeHealth, config::ServiceType}; use serde::Serialize; -/// One row of `GET /api/servers`: a deployed `crowdb-kv-server` projected +/// One row of `GET /api/servers`: a deployed `crowdb-paxos-kv-server` projected /// from the persisted config plus the live monitor cache. #[derive(Debug, Serialize)] pub struct ServerSummary { diff --git a/app/crowdb-web/src/services/plans.rs b/app/crowdb-web/src/services/plans.rs index 947461e24..e9572b6d6 100644 --- a/app/crowdb-web/src/services/plans.rs +++ b/app/crowdb-web/src/services/plans.rs @@ -89,16 +89,11 @@ fn valid_overrides(overrides: &BTreeMap<String, ListenerOverrides>) -> bool { && (kind == "access-server" || (value.s3.is_none() && value.health.is_none() && value.dataset.is_none())) && (kind != "access-server" || value.rpc.is_none()) - && (!matches!(kind.as_str(), "diskdb" | "diskio") || value.http.is_none()) - && (kind != "diskdb" || value.rpc.map_or(true, |port| port <= 65533)) + && (kind != "diskio" || value.http.is_none()) && [value.http, value.rpc, value.s3, value.health, value.dataset] .into_iter() .flatten() .all(|port| port != 0 && ports.insert(port)) - && (kind != "diskdb" - || value - .rpc - .map_or(true, |port| ports.insert(port + 1) && ports.insert(port + 2))) }) } @@ -253,11 +248,6 @@ pub(super) fn reserved_ports(state: &AppState) -> Result<Vec<u16>, Failure> { .into_iter() .flatten(), ); - if kind == "diskdb" { - if let Some(port) = value.rpc.filter(|port| *port <= 65533) { - ports.extend([port + 1, port + 2]); - } - } } } } diff --git a/app/crowdb-web/src/standalone/readiness.rs b/app/crowdb-web/src/standalone/readiness.rs index b06273cd1..7778ab204 100644 --- a/app/crowdb-web/src/standalone/readiness.rs +++ b/app/crowdb-web/src/standalone/readiness.rs @@ -8,7 +8,7 @@ use std::time::Duration; use crowdb_console_shared::config::ServiceType; use crowdb_console_shared::error::{Error, Result}; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crate::state::AppState; diff --git a/app/crowdb-web/src/standalone/recovery.rs b/app/crowdb-web/src/standalone/recovery.rs index c3fda27ab..95da8270a 100644 --- a/app/crowdb-web/src/standalone/recovery.rs +++ b/app/crowdb-web/src/standalone/recovery.rs @@ -127,7 +127,7 @@ impl AppState { ..Default::default() }; let deployed = if node.ssh_enabled() { - let binary = server.binary.as_deref().unwrap_or("crowdb-kv-server"); + let binary = server.binary.as_deref().unwrap_or("crowdb-paxos-kv-server"); crowdb_console_shared::ssh::deploy_via_ssh(&request, node, binary).await? } else { let workspace = self.prepare_node_workspace(node.id)?; diff --git a/app/crowdb-web/src/state.rs b/app/crowdb-web/src/state.rs index 2120606b8..c422fccd2 100644 --- a/app/crowdb-web/src/state.rs +++ b/app/crowdb-web/src/state.rs @@ -32,19 +32,20 @@ pub struct AppState { pub diskdb_client: Arc<tokio::sync::RwLock<Option<crowdb_diskdb_client::DiskdbClient>>>, /// Cached crowdb-rpc transport reused across KV requests to avoid /// spawning 6+ threads per request. Shared by the cached `kv_client`. - pub kv_rpc_transport: Arc<tokio::sync::RwLock<Option<Arc<crowdb_kv_client::KvRpcTransport>>>>, + pub kv_rpc_transport: Arc<tokio::sync::RwLock<Option<Arc<crowdb_paxos_kv_client::PaxosKvRpcTransport>>>>, /// Immutable transport shared by bounded Chunk diagnostics requests. pub(crate) rpc_health: Arc<crate::services::rpc_health::RpcHealth>, pub(crate) chunk_rpc_transport: Arc<crowdb_chunkdb_client::ChunkdbRpcTransport>, - /// Cached `CrowdbKvClient` reused across KV requests so the topology + /// Cached `PaxosKvClient` reused across KV requests so the topology /// cache persists — avoids re-discovering the leader from seeds on /// every put/get/delete. Invalidated on `/internal/reset`. - pub kv_client: Arc<tokio::sync::RwLock<Option<Arc<crowdb_kv_client::CrowdbKvClient>>>>, + pub kv_client: Arc<tokio::sync::RwLock<Option<Arc<crowdb_paxos_kv_client::PaxosKvClient>>>>, /// Cached `ServiceDiscoveryClient` reused across requests so the /// discovery cache (service registry entries) persists. Built from - /// the same `CrowdbKvClient` as `kv_client` so topology + connection + /// the same `PaxosKvClient` as `kv_client` so topology + connection /// pool are shared. - pub discovery_client: Arc<tokio::sync::RwLock<Option<Arc<crowdb_kv_client::ServiceDiscoveryClient>>>>, + pub discovery_client: + Arc<tokio::sync::RwLock<Option<Arc<crowdb_paxos_kv_client::ServiceDiscoveryClient>>>>, /// Rate-limiter for repeated crowdb-rpc failure warnings: maps /// `endpoint` → last-warned timestamp. Prevents flooding the /// console with identical "instance query failed" warnings every @@ -383,11 +384,11 @@ impl AppState { std::fs::canonicalize(base).map_err(Error::Io) } - /// Get or create a cached `KvRpcTransport`. The transport (and its + /// Get or create a cached `PaxosKvRpcTransport`. The transport (and its /// crowdb-rpc server/client + I/O thread) is reused across KV requests /// to avoid spawning threads per request. Shared by the cached /// `kv_client`. - pub async fn kv_rpc_transport(&self) -> Arc<crowdb_kv_client::KvRpcTransport> { + pub async fn kv_rpc_transport(&self) -> Arc<crowdb_paxos_kv_client::PaxosKvRpcTransport> { if let Some(t) = self.kv_rpc_transport.read().await.as_ref() { return Arc::clone(t); } @@ -395,16 +396,16 @@ impl AppState { if let Some(t) = guard.as_ref() { return Arc::clone(t); } - let t = Arc::new(crowdb_kv_client::KvRpcTransport::new()); + let t = Arc::new(crowdb_paxos_kv_client::PaxosKvRpcTransport::new()); *guard = Some(Arc::clone(&t)); t } - /// Get or create a cached `CrowdbKvClient` sharing the cached RPC + /// Get or create a cached `PaxosKvClient` sharing the cached RPC /// transport. The topology cache persists across requests so the /// leader endpoint is not re-discovered from seeds on every call. /// Call `seed_leader` before each use to keep the hint fresh. - pub async fn kv_client(&self) -> Arc<crowdb_kv_client::CrowdbKvClient> { + pub async fn kv_client(&self) -> Arc<crowdb_paxos_kv_client::PaxosKvClient> { if let Some(c) = self.kv_client.read().await.as_ref() { return Arc::clone(c); } @@ -413,8 +414,8 @@ impl AppState { return Arc::clone(c); } let transport = self.kv_rpc_transport().await; - let c = Arc::new(crowdb_kv_client::CrowdbKvClient::new_with_rpc_transport( - crowdb_kv_client::ClientConfig::new(self.authority_seeds.as_ref().clone()), + let c = Arc::new(crowdb_paxos_kv_client::PaxosKvClient::new_with_rpc_transport( + crowdb_paxos_kv_client::ClientConfig::new(self.authority_seeds.as_ref().clone()), transport, )); *guard = Some(Arc::clone(&c)); @@ -423,7 +424,7 @@ impl AppState { /// Drop every client that retains cluster topology so the next /// request rebuilds from the post-reset cluster. The discovery and - /// `DiskDB` clients both retain the shared `CrowdbKvClient`, so + /// `DiskDB` clients both retain the shared `PaxosKvClient`, so /// clearing only `kv_client` would leave its old group-0 leader hint /// reachable through those wrappers. pub async fn clear_cluster_clients(&self) { @@ -436,10 +437,10 @@ impl AppState { } /// Get or lazily build the cached `ServiceDiscoveryClient`. Shares - /// the same `CrowdbKvClient` as `kv_client()` so topology + connection + /// the same `PaxosKvClient` as `kv_client()` so topology + connection /// pool are shared. Used by web handlers that need to discover living /// service instances (diskdb, chunkdb) from the group-0 registry. - pub async fn discovery_client(&self) -> Arc<crowdb_kv_client::ServiceDiscoveryClient> { + pub async fn discovery_client(&self) -> Arc<crowdb_paxos_kv_client::ServiceDiscoveryClient> { if let Some(d) = self.discovery_client.read().await.as_ref() { return Arc::clone(d); } @@ -448,12 +449,12 @@ impl AppState { return Arc::clone(d); } let kv = self.kv_client().await; - let d = Arc::new(crowdb_kv_client::ServiceDiscoveryClient::from_shared_kv(kv)); + let d = Arc::new(crowdb_paxos_kv_client::ServiceDiscoveryClient::from_shared_kv(kv)); *guard = Some(Arc::clone(&d)); d } - /// Re-seed the cached `CrowdbKvClient`'s mgmt seed list from the + /// Re-seed the cached `PaxosKvClient`'s mgmt seed list from the /// current config. Called after cluster init (or server deploy) so /// the shared client can discover leaders for the new topology. /// No-op if no client is cached (it will be created with config @@ -481,7 +482,7 @@ impl AppState { } /// Build an [`OpContext`] for a single request, sharing the cached - /// `CrowdbKvClient` (topology cache + connection pool) and a + /// `PaxosKvClient` (topology cache + connection pool) and a /// snapshot of the persisted [`ConsoleConfig`]. /// /// The group-0 endpoint is resolved from the config's store-0 @@ -737,11 +738,11 @@ mod tests { let _ = config.add_server(ServerEntry { access_origins: std::collections::BTreeMap::new(), id: "s1".into(), - url: "http://127.0.0.1:10000".into(), + url: "http://127.0.0.1:10135".into(), node_id: Some(1), rpc_url: None, - rest_port: Some(10000), - rpc_port: Some(10100), + rest_port: Some(crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE), + rpc_port: Some(crowdb_protocol::PAXOS_KV_SERVER_RPC_BASE), auto_start: true, binary: None, election_profile: None, @@ -752,7 +753,7 @@ mod tests { }); let (endpoint, seeds) = resolve_group0_endpoint(&config); assert!(endpoint.is_none(), "no store 0 → no RPC endpoint to seed"); - assert_eq!(seeds, vec!["http://127.0.0.1:10000"]); + assert_eq!(seeds, vec!["http://127.0.0.1:10135"]); } #[test] @@ -761,11 +762,11 @@ mod tests { let _ = config.add_server(ServerEntry { access_origins: std::collections::BTreeMap::new(), id: "s1".into(), - url: "http://127.0.0.1:10000".into(), + url: "http://127.0.0.1:10135".into(), node_id: Some(1), - rpc_url: Some("http://127.0.0.1:10100".into()), - rest_port: Some(10000), - rpc_port: Some(10100), + rpc_url: Some("http://127.0.0.1:11135".into()), + rest_port: Some(crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE), + rpc_port: Some(crowdb_protocol::PAXOS_KV_SERVER_RPC_BASE), auto_start: true, binary: None, election_profile: None, @@ -776,8 +777,8 @@ mod tests { }); config.record_store(0, vec![1]); let (endpoint, seeds) = resolve_group0_endpoint(&config); - assert_eq!(endpoint.as_deref(), Some("http://127.0.0.1:10100")); - assert_eq!(seeds, vec!["http://127.0.0.1:10000"]); + assert_eq!(endpoint.as_deref(), Some("http://127.0.0.1:11135")); + assert_eq!(seeds, vec!["http://127.0.0.1:10135"]); } #[tokio::test] @@ -796,11 +797,11 @@ mod tests { let _ = config.add_server(ServerEntry { access_origins: std::collections::BTreeMap::new(), id: "s1".into(), - url: "http://127.0.0.1:10000".into(), + url: "http://127.0.0.1:10135".into(), node_id: Some(1), - rpc_url: Some("http://127.0.0.1:10100".into()), - rest_port: Some(10000), - rpc_port: Some(10100), + rpc_url: Some("http://127.0.0.1:11135".into()), + rest_port: Some(crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE), + rpc_port: Some(crowdb_protocol::PAXOS_KV_SERVER_RPC_BASE), auto_start: true, binary: None, election_profile: None, @@ -819,7 +820,7 @@ mod tests { let cached = state.kv_client().await; assert!( Arc::ptr_eq(ctx.kv_arc(), &cached), - "OpContext must share the cached CrowdbKvClient" + "OpContext must share the cached PaxosKvClient" ); } @@ -852,11 +853,11 @@ mod tests { let _ = config.add_server(ServerEntry { access_origins: std::collections::BTreeMap::new(), id: "s1".into(), - url: "http://127.0.0.1:10000".into(), + url: "http://127.0.0.1:10135".into(), node_id: Some(1), - rpc_url: Some("http://127.0.0.1:10100".into()), - rest_port: Some(10000), - rpc_port: Some(10100), + rpc_url: Some("http://127.0.0.1:11135".into()), + rest_port: Some(crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE), + rpc_port: Some(crowdb_protocol::PAXOS_KV_SERVER_RPC_BASE), auto_start: true, binary: None, election_profile: None, diff --git a/app/crowdb-web/tests/bound_client_test.rs b/app/crowdb-web/tests/bound_client_test.rs index 1d2433ad3..de06cfd3a 100644 --- a/app/crowdb-web/tests/bound_client_test.rs +++ b/app/crowdb-web/tests/bound_client_test.rs @@ -5,14 +5,14 @@ mod memory_kv; use crowdb_console_shared::ConsoleConfig; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::mgmt::{node::NodeBinding, SystemBootstrapIdentity}; use crowdb_web::AppState; #[tokio::test] async fn bound_operations_discover_group0_without_a_warmed_console_projection() { let cluster = memory_kv::TestKvCluster::start().await; - let kv = CrowdbKvClient::new(ClientConfig::new(cluster.mgmt_endpoints.clone())); + let kv = PaxosKvClient::new(ClientConfig::new(cluster.mgmt_endpoints.clone())); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); kv.put(0, 0, b"binding-probe", b"committed", None).await.unwrap(); let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("bound-client"); diff --git a/app/crowdb-web/tests/chunk_kv_catalog_test.rs b/app/crowdb-web/tests/chunk_kv_catalog_test.rs index 0c66f7dbf..6dda00054 100644 --- a/app/crowdb-web/tests/chunk_kv_catalog_test.rs +++ b/app/crowdb-web/tests/chunk_kv_catalog_test.rs @@ -13,7 +13,7 @@ use axum::{ body::Body, http::{Request, StatusCode}, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, PartitionArtifact, @@ -44,7 +44,7 @@ async fn get(app: axum::Router, query: &str) -> (StatusCode, Value) { #[tokio::test] async fn catalog_windows_pin_generation_validate_checksums_and_keep_exact_ids() { let cluster = memory_kv::TestKvCluster::start().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); @@ -303,7 +303,7 @@ fn catalog_fixture() -> (ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage) { (head, page) } -async fn verify_balance(kv: &CrowdbKvClient, app: &axum::Router, page: &ChunkKvRangeCatalogPage) { +async fn verify_balance(kv: &PaxosKvClient, app: &axum::Router, page: &ChunkKvRangeCatalogPage) { use crowdb_protocol::chunk_kv::balance::OBSERVATION_KEY; catalog_balance::seed(kv, page).await; let (status, first) = get(app.clone(), "").await; @@ -319,12 +319,12 @@ async fn verify_balance(kv: &CrowdbKvClient, app: &axum::Router, page: &ChunkKvR first["entries"][0]["balance"]["partition"]["weight"]["byte_units"], "7619047" ); - let crowdb_kv_client::GetOutcome::Found { value, .. } = kv + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = kv .get( 0, 0, OBSERVATION_KEY.as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await diff --git a/app/crowdb-web/tests/cluster_target_test.rs b/app/crowdb-web/tests/cluster_target_test.rs index 17fdcefe0..16c6a1183 100644 --- a/app/crowdb-web/tests/cluster_target_test.rs +++ b/app/crowdb-web/tests/cluster_target_test.rs @@ -12,7 +12,7 @@ use axum::{ }; use cluster_target::TestClusterTarget; use crowdb_console_shared::deployment::CLUSTER_KEY; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use serde_json::Value; use tower::ServiceExt; diff --git a/app/crowdb-web/tests/common/catalog_balance.rs b/app/crowdb-web/tests/common/catalog_balance.rs index 41c3bbfe1..4a644c7fe 100644 --- a/app/crowdb-web/tests/common/catalog_balance.rs +++ b/app/crowdb-web/tests/common/catalog_balance.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_protocol::chunk_kv::{ balance::{ BalanceObservation, BalanceOwnerObservation, BalancePartitionObservation, BalanceWeights, @@ -10,7 +10,7 @@ use crowdb_protocol::chunk_kv::{ ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogPage, }; -pub async fn seed(kv: &CrowdbKvClient, page: &ChunkKvRangeCatalogPage) { +pub async fn seed(kv: &PaxosKvClient, page: &ChunkKvRangeCatalogPage) { let weights = BalanceWeights::new(&[(105, 1_050)], 80).unwrap(); let first = &page.entries[0]; let now = u64::try_from( diff --git a/app/crowdb-web/tests/common/cluster_target.rs b/app/crowdb-web/tests/common/cluster_target.rs index 5ffdee81e..0c44dca3f 100644 --- a/app/crowdb-web/tests/common/cluster_target.rs +++ b/app/crowdb-web/tests/common/cluster_target.rs @@ -7,7 +7,7 @@ use crowdb_console_shared::{ deployment::{PreparedBootstrap, CLUSTER_KEY}, ConsoleConfig, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::mgmt::node::NodeBinding; use crowdb_web::AppState; use std::sync::Arc; @@ -16,7 +16,7 @@ pub struct TestClusterTarget { pub state: AppState, pub prepared: PreparedBootstrap, pub binding: NodeBinding, - pub kv: Arc<CrowdbKvClient>, + pub kv: Arc<PaxosKvClient>, pub root: crowdb_test_harness::test_dirs::TestDir, _cluster: TestKvCluster, } @@ -24,7 +24,7 @@ pub struct TestClusterTarget { impl TestClusterTarget { pub async fn start() -> Self { let cluster = TestKvCluster::start().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); diff --git a/app/crowdb-web/tests/common/memory_kv.rs b/app/crowdb-web/tests/common/memory_kv.rs index 3e49a78f9..b3f680e57 100644 --- a/app/crowdb-web/tests/common/memory_kv.rs +++ b/app/crowdb-web/tests/common/memory_kv.rs @@ -7,8 +7,8 @@ #![allow(dead_code)] use axum::{extract::State, routing::get, Json, Router}; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; use std::sync::Arc; pub(crate) struct TestKvCluster { diff --git a/app/crowdb-web/tests/common/storage_lifecycle.rs b/app/crowdb-web/tests/common/storage_lifecycle.rs index e8b5b79c4..41b7fb4f1 100644 --- a/app/crowdb-web/tests/common/storage_lifecycle.rs +++ b/app/crowdb-web/tests/common/storage_lifecycle.rs @@ -10,7 +10,7 @@ use crowdb_console_shared::{ }, ConsoleConfig, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::{ common::{DiskId, DiskioHardwareScope, HwStatus, NodeValue, RackValue}, disk_storage::DiskStorageRecord, @@ -32,7 +32,7 @@ pub struct TestStorageApi { impl TestStorageApi { pub async fn start() -> Self { let cluster = TestKvCluster::start().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); diff --git a/app/crowdb-web/tests/common/storage_readiness.rs b/app/crowdb-web/tests/common/storage_readiness.rs index 2c3d490ca..4c503aefb 100644 --- a/app/crowdb-web/tests/common/storage_readiness.rs +++ b/app/crowdb-web/tests/common/storage_readiness.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::{ common::{DiskId, HwStatus, NodeValue, RackValue}, diskdb::rpc::DiskValue, diff --git a/app/crowdb-web/tests/corr_id_propagation_test.rs b/app/crowdb-web/tests/corr_id_propagation_test.rs index 9074a7953..f91ed9035 100644 --- a/app/crowdb-web/tests/corr_id_propagation_test.rs +++ b/app/crowdb-web/tests/corr_id_propagation_test.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Verify `x-crowdb-kv-corr-id` propagates inbound → handler → response. +//! Verify `x-crowdb-paxos-kv-corr-id` propagates inbound → handler → response. //! //! Each test spins up the real `crowdb-web` router. The middleware //! should: diff --git a/app/crowdb-web/tests/deployment_cancellation_test.rs b/app/crowdb-web/tests/deployment_cancellation_test.rs index 2735ce68d..447b2d0dd 100644 --- a/app/crowdb-web/tests/deployment_cancellation_test.rs +++ b/app/crowdb-web/tests/deployment_cancellation_test.rs @@ -133,20 +133,19 @@ async fn cancel_and_reset(kind: &str, restart: bool) { TestChild::new(root.path()) }; let (mut path, mut body, service) = if kind == "diskdb" { - let port = crowdb_protocol::port::alloc::alloc_test_port_range( - crowdb_protocol::ServicePort::DiskdbListen, - 3, - )[0]; + let port = + crowdb_protocol::port::alloc::alloc_test_port_range(crowdb_protocol::ServicePort::DiskdbRpc, 3) + [0]; ( "/api/nodes/1/diskdb/deploy", - json!({"rpc_port": port}), + json!({"rpc_port": port, "http_port": free_port()}), "crowdb-diskdb", ) } else { ( "/api/nodes/1/server/deploy", json!({"rest_port": free_port(), "rpc_port": free_port(), "binary": stub.binary, "election_profile": "test"}), - "crowdb-kv-server", + "crowdb-paxos-kv-server", ) }; if restart { @@ -203,11 +202,11 @@ async fn cancel_and_reset(kind: &str, restart: bool) { async fn prepare_diskdb_authority(state: &AppState) -> memory_kv::TestKvCluster { let metadata = memory_kv::TestKvCluster::start().await; - let kv = std::sync::Arc::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(metadata.mgmt_endpoints.clone()), + let kv = std::sync::Arc::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(metadata.mgmt_endpoints.clone()), )); kv.seed_leader(0, 0, metadata.group0_leader_endpoint.clone()); - crowdb_kv_client::CrowdbSysmdClient::from_shared(kv.clone()) + crowdb_paxos_kv_client::CrowdbSysmdClient::from_shared(kv.clone()) .add_group(0, 0) .await .unwrap(); diff --git a/app/crowdb-web/tests/diskdb_binding_test.rs b/app/crowdb-web/tests/diskdb_binding_test.rs index c9e179224..100dcd874 100644 --- a/app/crowdb-web/tests/diskdb_binding_test.rs +++ b/app/crowdb-web/tests/diskdb_binding_test.rs @@ -21,7 +21,7 @@ async fn disk_drilldown_uses_persisted_binding_without_local_process_observation 0, 0, b"/deployment/cluster", - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None ) .await diff --git a/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs b/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs index 335a7c1a2..37bf6f125 100644 --- a/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs +++ b/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs @@ -10,7 +10,7 @@ use serde_json::{json, Value}; use tower::ServiceExt; #[test] -fn explicit_diskdb_ranges_keep_adjacent_host_instances_disjoint_and_preserve_legacy_inputs() { +fn explicit_diskdb_ranges_keep_adjacent_host_instances_disjoint() { use crowdb_console_shared::config::NodeEntry; use crowdb_web::diskdb::{configuration_for_tests, DeployDiskdbBody}; let node: NodeEntry = serde_json::from_value(json!({ @@ -20,17 +20,16 @@ fn explicit_diskdb_ranges_keep_adjacent_host_instances_disjoint_and_preserve_leg let mut used = std::collections::BTreeSet::new(); for offset in 0..3 { let body: DeployDiskdbBody = serde_json::from_value(json!({ - "listen_port":11000+offset,"http_port":11100+offset, - "rpc_port":11200+offset,"rpc_listen_port":11200+offset + "http_port":crowdb_protocol::DISKDB_HTTP_BASE+offset, + "rpc_port":crowdb_protocol::DISKDB_RPC_BASE+offset })) .unwrap(); let config = configuration_for_tests(&node, &["http://127.0.0.1:10000".into()], &body) .unwrap_or_else(|(_, body)| panic!("{}", body.error)); let config: toml::Value = toml::from_str(&config).unwrap(); for (field, base) in [ - ("listen_addr", 11000), - ("http_listen_addr", 11100), - ("rpc_listen_addr", 11200), + ("http_listen_addr", crowdb_protocol::DISKDB_HTTP_BASE), + ("rpc_listen_addr", crowdb_protocol::DISKDB_RPC_BASE), ] { let address = config["server"][field].as_str().unwrap(); assert_eq!(address, format!("127.0.0.1:{}", base + offset)); @@ -43,7 +42,7 @@ fn explicit_diskdb_ranges_keep_adjacent_host_instances_disjoint_and_preserve_leg let config: toml::Value = toml::from_str(&config).unwrap(); assert_eq!( config["server"]["rpc_listen_addr"].as_str(), - Some("127.0.0.1:7102") + Some("127.0.0.1:7100") ); } diff --git a/app/crowdb-web/tests/diskdb_routes_test.rs b/app/crowdb-web/tests/diskdb_routes_test.rs index c733c7df8..81896bca6 100644 --- a/app/crowdb-web/tests/diskdb_routes_test.rs +++ b/app/crowdb-web/tests/diskdb_routes_test.rs @@ -18,7 +18,7 @@ use std::time::Duration; mod memory_kv; use crowdb_console_shared::{ConsoleConfig, DiskGroupEntry, NodeEntry, RackEntry, ServerEntry}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::{NodeValue, RackValue}, diskdb::rpc::DiskGroupValue, @@ -70,7 +70,7 @@ async fn spawn_web_with_config(mut cfg: ConsoleConfig, with_group0: bool) -> Soc let kv = if let Some(cluster) = &cluster { cfg.servers .push(ServerEntry::new("metadata", &cluster.mgmt_endpoints[0])); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); diff --git a/app/crowdb-web/tests/lifecycle_routes_test.rs b/app/crowdb-web/tests/lifecycle_routes_test.rs index 5e08d4874..934654a51 100644 --- a/app/crowdb-web/tests/lifecycle_routes_test.rs +++ b/app/crowdb-web/tests/lifecycle_routes_test.rs @@ -304,11 +304,11 @@ async fn multiple_racks_and_nodes_create_expected_workspaces() { assert!(std::fs::read_dir(dir.join("N-1/bin")).unwrap().next().is_some()); assert!(dir - .join(format!("N-1/log/crowdb-kv-server-{pid1}.out.log")) + .join(format!("N-1/log/crowdb-paxos-kv-server-{pid1}.out.log")) .exists()); assert!(std::fs::read_dir(dir.join("N-10/bin")).unwrap().next().is_some()); assert!(dir - .join(format!("N-10/log/crowdb-kv-server-{pid10}.out.log")) + .join(format!("N-10/log/crowdb-paxos-kv-server-{pid10}.out.log")) .exists()); assert!(std::fs::read_dir(dir.join("N-2/bin")).unwrap().next().is_none()); diff --git a/app/crowdb-web/tests/managed_mode_test.rs b/app/crowdb-web/tests/managed_mode_test.rs index 150446462..e30537d99 100644 --- a/app/crowdb-web/tests/managed_mode_test.rs +++ b/app/crowdb-web/tests/managed_mode_test.rs @@ -6,8 +6,8 @@ mod memory_kv; use axum::body::Body; use axum::http::{Method, Request, StatusCode}; use crowdb_console_shared::config::web::{WebMode, WebProcessConfig}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, CrowdbSysmdClient}; use crowdb_monitor::{MonitorPhase, MonitorStatus, ServiceStatus, StatusStore}; +use crowdb_paxos_kv_client::{ClientConfig, CrowdbSysmdClient, PaxosKvClient}; use crowdb_protocol::common::{HwStatus, KvServerIdentity, NodeValue, RackValue, ServiceExtra}; use crowdb_web::{router, AppState}; use tower::ServiceExt; @@ -195,7 +195,7 @@ async fn managed_readonly_mode_does_not_expose_local_topology_or_mutations() { #[tokio::test] async fn managed_snapshot_uses_group0_and_monitor_without_local_fallback() { let cluster = memory_kv::TestKvCluster::start().await; - let kv = CrowdbKvClient::new(ClientConfig::new(cluster.mgmt_endpoints.clone())); + let kv = PaxosKvClient::new(ClientConfig::new(cluster.mgmt_endpoints.clone())); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); let sysmd = CrowdbSysmdClient::new(kv); sysmd diff --git a/app/crowdb-web/tests/node_storage_authority_test.rs b/app/crowdb-web/tests/node_storage_authority_test.rs index 293deb394..57a56a2d9 100644 --- a/app/crowdb-web/tests/node_storage_authority_test.rs +++ b/app/crowdb-web/tests/node_storage_authority_test.rs @@ -9,7 +9,7 @@ use axum::{ http::{Request, StatusCode}, }; use crowdb_console_shared::ConsoleConfig; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::{DiskId, NodeValue, RackValue}, diskdb::rpc::{DiskGroupValue, DiskValue}, @@ -32,7 +32,7 @@ async fn read(app: &axum::Router, path: &str) -> serde_json::Value { #[tokio::test] async fn bound_storage_reads_observe_monitor_publication_without_console_cache_refresh() { let cluster = memory_kv::TestKvCluster::start().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); diff --git a/app/crowdb-web/tests/remote_kv_projection_test.rs b/app/crowdb-web/tests/remote_kv_projection_test.rs index 864cec16e..bdac9ae4e 100644 --- a/app/crowdb-web/tests/remote_kv_projection_test.rs +++ b/app/crowdb-web/tests/remote_kv_projection_test.rs @@ -6,7 +6,7 @@ mod memory_kv; use axum::{body::Body, http::Request}; use crowdb_console_shared::config::{ConsoleConfig, ServerEntry}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::mgmt::{ node::{NodeBinding, NodeServiceAction, NodeServiceIntent}, SystemBootstrapIdentity, @@ -19,7 +19,7 @@ use tower::ServiceExt; #[tokio::test] async fn diskdb_restart_claims_the_remote_node_once_before_target_validation() { let cluster = memory_kv::TestKvCluster::start().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); @@ -71,7 +71,7 @@ async fn diskdb_restart_claims_the_remote_node_once_before_target_validation() { #[tokio::test] async fn kv_lifecycle_intents_preserve_admitted_endpoints_while_auxiliary_services_refresh() { let cluster = memory_kv::TestKvCluster::start().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); @@ -135,7 +135,7 @@ async fn kv_lifecycle_intents_preserve_admitted_endpoints_while_auxiliary_servic .any(|row| row["id"] == "chunkdb-1")); } -async fn publish_intents(kv: &CrowdbKvClient, cluster_id: &str) { +async fn publish_intents(kv: &PaxosKvClient, cluster_id: &str) { for (id, kind, configuration) in [ ("kv-1", "kv", "node_id = 1\n"), ( diff --git a/app/crowdb-web/tests/service_lifecycle_test.rs b/app/crowdb-web/tests/service_lifecycle_test.rs index d8d561515..bf1582b52 100644 --- a/app/crowdb-web/tests/service_lifecycle_test.rs +++ b/app/crowdb-web/tests/service_lifecycle_test.rs @@ -290,7 +290,7 @@ async fn reset_removes_auxiliary_launches_before_nodes_and_racks() { #[tokio::test] async fn deployment_defaults_avoid_cross_service_and_live_listener_conflicts() { let mut config = config(); - let occupied = std::net::TcpListener::bind("0.0.0.0:12010").ok(); + let occupied = std::net::TcpListener::bind(("0.0.0.0", crowdb_protocol::CHUNKDB_HTTP_BASE)).ok(); let mut access = ServerEntry::new("access-server-7", "http://127.0.0.1:9092"); access.service_type = ServiceType::AccessServer; config.servers.push(access); @@ -306,18 +306,14 @@ async fn deployment_defaults_avoid_cross_service_and_live_listener_conflicts() { assert_eq!(status, StatusCode::OK, "{body}"); assert_eq!(body["access-server"]["instance_id"], "8"); let mut ports = std::collections::HashSet::new(); - for (kind, values) in body.as_object().unwrap() { + for values in body.as_object().unwrap().values() { for name in ["http_port", "rpc_port", "s3_port", "health_port"] { if let Some(port) = values[name].as_u64() { assert!(![9091, 9092, 19191].contains(&port)); if occupied.is_some() { - assert_ne!(port, 12010); + assert_ne!(port, u64::from(crowdb_protocol::CHUNKDB_HTTP_BASE)); } assert!(ports.insert(port)); - if kind == "diskdb" { - assert!(ports.insert(port + 1)); - assert!(ports.insert(port + 2)); - } } } } diff --git a/app/crowdb-web/tests/service_plans_test.rs b/app/crowdb-web/tests/service_plans_test.rs index c33e4b252..4c416cd5c 100644 --- a/app/crowdb-web/tests/service_plans_test.rs +++ b/app/crowdb-web/tests/service_plans_test.rs @@ -137,8 +137,8 @@ async fn plan_rejects_legacy_kinds_and_invalid_listener_configuration() { json!({"access-server":{"dataset_port":9093,"health_port":9093}}), json!({"chunkdb":{"dataset_port":9093}}), json!({"diskio":{"http_port":13010}}), - json!({"diskdb":{"rpc_port":65534}}), - json!({"diskdb":{"rpc_port":12000},"diskio":{"rpc_port":12001}}), + json!({"diskdb":{"rpc_port":0}}), + json!({"diskdb":{"rpc_port":12000},"diskio":{"rpc_port":12000}}), json!({"chunkdb":{"health_port":9093}}), json!({"kv":{"rpc_port":19920}}), json!({"access-server":{"instance_id":"1"}}), diff --git a/app/crowdb-web/tests/storage_readiness_test.rs b/app/crowdb-web/tests/storage_readiness_test.rs index 5832be703..bce27ec16 100644 --- a/app/crowdb-web/tests/storage_readiness_test.rs +++ b/app/crowdb-web/tests/storage_readiness_test.rs @@ -10,7 +10,7 @@ use crowdb_console_shared::{ config::{ServerEntry, ServiceType}, ConsoleConfig, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::diskdb::rpc::DiskGroupValue; use crowdb_web::{router, AppState}; use serde_json::Value; @@ -35,7 +35,7 @@ async fn readiness(app: &axum::Router) -> Value { #[tokio::test] async fn chunk_storage_requires_live_complete_ownership_even_when_diskio_pid_is_alive() { let cluster = memory_kv::TestKvCluster::start().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); diff --git a/app/crowdb-web/ui/src/App.tsx b/app/crowdb-web/ui/src/App.tsx index cf02db29b..3257782a6 100644 --- a/app/crowdb-web/ui/src/App.tsx +++ b/app/crowdb-web/ui/src/App.tsx @@ -1,5 +1,6 @@ import { PanelDivider } from './components/PanelDivider'; import { ClusterBreadcrumb } from './shell/ClusterBreadcrumb'; +import { PAXOS_KV_SERVER_MGMT_BASE, PAXOS_KV_SERVER_RPC_BASE, DISKDB_RPC_BASE } from './ports'; import { useClusterLayout } from './shell/useClusterLayout'; // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. @@ -367,13 +368,13 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma const deployDialogDefaults = useMemo(() => { if (!dialog.deployServer?.nodeId) { - return { defaultRestPort: '19910', defaultRpcPort: '19920' }; + return { defaultRestPort: String(PAXOS_KV_SERVER_MGMT_BASE), defaultRpcPort: String(PAXOS_KV_SERVER_RPC_BASE) }; } return deployPortDefaultsForNode( servers, dialog.deployServer.nodeId, - 19910, - 19920, + PAXOS_KV_SERVER_MGMT_BASE, + PAXOS_KV_SERVER_RPC_BASE, rememberedDeployPorts.mgmt, rememberedDeployPorts.rpc, ); @@ -386,8 +387,8 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma ...deployPortDefaultsForNode( servers, nextNodeId, - 19910, - 19920, + PAXOS_KV_SERVER_MGMT_BASE, + PAXOS_KV_SERVER_RPC_BASE, rememberedDeployPorts.mgmt, rememberedDeployPorts.rpc, ), @@ -403,7 +404,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma ); const deployDiskdbDefaults = useMemo(() => { - if (!dialog.deployDiskdb?.nodeId) return '29920'; + if (!dialog.deployDiskdb?.nodeId) return String(DISKDB_RPC_BASE); return diskdbPortDefaultsForNode( diskdbInstances, dialog.deployDiskdb.nodeId, diff --git a/app/crowdb-web/ui/src/api.ts b/app/crowdb-web/ui/src/api.ts index e6bf57ee5..7decc19c9 100644 --- a/app/crowdb-web/ui/src/api.ts +++ b/app/crowdb-web/ui/src/api.ts @@ -433,7 +433,7 @@ export async function pingNode(nodeId: number, options?: RequestOptions): Promis } /** - * Deploy a crowdb-kv-server instance to a node + * Deploy a crowdb-paxos-kv-server instance to a node */ export async function deployServer( nodeId: number, @@ -512,7 +512,7 @@ export async function removeServer(nodeId: number, options?: RequestOptions): Pr } /** - * Get the OpenAPI spec for a node's crowdb-kv-server instance + * Get the OpenAPI spec for a node's crowdb-paxos-kv-server instance */ export async function getNodeOpenApi(nodeId: number, options?: RequestOptions): Promise<Record<string, any>> { const url = `/api/nodes/${encodeURIComponent(nodeId)}/openapi.json`; @@ -592,7 +592,7 @@ export interface AddStoreRequest { /** * Create a new cluster-wide store. The target nodes must already have a - * running `crowdb-kv-server` (deployed via `deployServer`). + * running `crowdb-paxos-kv-server` (deployed via `deployServer`). */ export async function addStore( req: AddStoreRequest, @@ -901,6 +901,7 @@ export interface AddDisksBatchResult { export interface DeployDiskdbRequest { rpc_port: number; + http_port?: number; keepalive_interval_secs?: number; } diff --git a/app/crowdb-web/ui/src/components/dialogs/AddNodeDialog.tsx b/app/crowdb-web/ui/src/components/dialogs/AddNodeDialog.tsx index efca22f0c..71e7539a1 100644 --- a/app/crowdb-web/ui/src/components/dialogs/AddNodeDialog.tsx +++ b/app/crowdb-web/ui/src/components/dialogs/AddNodeDialog.tsx @@ -69,11 +69,10 @@ export function AddNodeDialog({ isOpen, onClose, racks, defaultRackId, }))); }, [defaults.values]); - const allPorts = selected.flatMap(kind => [...Object.values(ports[kind] ?? {}).filter((value): value is number => typeof value === 'number'), ...(kind === 'diskdb' && ports[kind]?.rpc_port ? [ports[kind]!.rpc_port! + 1, ports[kind]!.rpc_port! + 2] : [])]); + const allPorts = selected.flatMap(kind => Object.values(ports[kind] ?? {}).filter((value): value is number => typeof value === 'number')); const validPorts = selected.every(kind => listenerFields(kind).every(([field]) => ports[kind]?.[field] != null)) && allPorts.every(port => Number.isInteger(port) && port > 0 && port <= 65535) - && new Set(allPorts).size === allPorts.length - && (!selected.includes('diskdb') || (ports.diskdb?.rpc_port ?? 0) <= 65533); + && new Set(allPorts).size === allPorts.length; const valid = /^\d+$/.test(nodeId) && Number.isSafeInteger(Number(nodeId)) && Number(nodeId) > 0 && !!rackId && !!host.trim() && validPorts && !!defaults.values; diff --git a/app/crowdb-web/ui/src/components/dialogs/DeployDiskdbDialog.tsx b/app/crowdb-web/ui/src/components/dialogs/DeployDiskdbDialog.tsx index 39ee6c8ed..206db773c 100644 --- a/app/crowdb-web/ui/src/components/dialogs/DeployDiskdbDialog.tsx +++ b/app/crowdb-web/ui/src/components/dialogs/DeployDiskdbDialog.tsx @@ -22,11 +22,12 @@ export function DeployDiskdbDialog({ onClose, nodes, defaultNodeId, - defaultRpcPort = '29920', + defaultRpcPort = '', onSuccess, }: DeployDiskdbDialogProps) { const [nodeId, setNodeId] = useState(''); const [rpcPort, setRpcPort] = useState(defaultRpcPort); + const [httpPort, setHttpPort] = useState(''); const [isLoading, setIsLoading] = useState(false); const wasOpenRef = useRef(false); const { success, error } = useToast(); @@ -46,10 +47,11 @@ export function DeployDiskdbDialog({ useEffect(() => { if (!defaults.values) return; setRpcPort(String(defaults.values.diskdb.rpc_port)); + setHttpPort(String(defaults.values.diskdb.http_port)); }, [defaults.values]); const isPort = (v: string) => /^\d+$/.test(v) && Number(v) > 0 && Number(v) < 65536; - const valid = nodeId !== '' && isPort(rpcPort); + const valid = nodeId !== '' && isPort(rpcPort) && isPort(httpPort) && rpcPort !== httpPort; const handleSubmit = async () => { if (!valid) return; @@ -57,6 +59,7 @@ export function DeployDiskdbDialog({ try { await deployDiskdb(Number(nodeId), { rpc_port: Number(rpcPort), + http_port: Number(httpPort), }); success(`DiskDB deployed on node ${nodeId}`); onClose(); @@ -99,6 +102,7 @@ export function DeployDiskdbDialog({ ))} </select> </div> + <Input label="HTTP Port (management)" inputMode="numeric" value={httpPort} onChange={e => setHttpPort(e.target.value)} /> <Input label="RPC Port (crowdb-rpc)" inputMode="numeric" diff --git a/app/crowdb-web/ui/src/components/dialogs/DeployServerDialog.tsx b/app/crowdb-web/ui/src/components/dialogs/DeployServerDialog.tsx index c693cf412..73291677b 100644 --- a/app/crowdb-web/ui/src/components/dialogs/DeployServerDialog.tsx +++ b/app/crowdb-web/ui/src/components/dialogs/DeployServerDialog.tsx @@ -18,7 +18,7 @@ export interface DeployServerDialogProps { } /** - * Deploy a `crowdb-kv-server` instance on a node. Required before that + * Deploy a `crowdb-paxos-kv-server` instance on a node. Required before that * node can host any store / group / replica. Backend contract: * `crowdb-console/web/src/lifecycle.rs::DeployNodeServerBody`. */ @@ -26,8 +26,8 @@ export function DeployServerDialog({ isOpen, onClose, nodeId, - defaultRestPort = '19910', - defaultRpcPort = '19920', + defaultRestPort = '', + defaultRpcPort = '', onSuccess, }: DeployServerDialogProps) { const [restPort, setRestPort] = useState(defaultRestPort); diff --git a/app/crowdb-web/ui/src/components/dialogs/createFlows.test.tsx b/app/crowdb-web/ui/src/components/dialogs/createFlows.test.tsx index 15a076567..dafbe0096 100644 --- a/app/crowdb-web/ui/src/components/dialogs/createFlows.test.tsx +++ b/app/crowdb-web/ui/src/components/dialogs/createFlows.test.tsx @@ -10,6 +10,7 @@ import { AddNodeDialog } from './AddNodeDialog'; import { AddStoreDialog } from './AddStoreDialog'; import { AddGroupDialog } from './AddGroupDialog'; import { AddReplicaDialog } from './AddReplicaDialog'; +import { DeployDiskdbDialog } from './DeployDiskdbDialog'; import { DeployServerDialog } from './DeployServerDialog'; import { serviceOrder, serviceLabels } from '../../services/useNodeServicePlans'; import { NodeHealth, ProcState } from '../../types'; @@ -36,7 +37,7 @@ interface CapturedRequest { let captured: CapturedRequest[] = []; const deploymentDefaults = { 'paxos-kv': { instance_id: '1', http_port: 19911, rpc_port: 19921 }, - diskdb: { instance_id: '1', rpc_port: 29921 }, + diskdb: { instance_id: '1', rpc_port: 14135, http_port: 13135 }, diskio: { instance_id: '1', rpc_port: 13010 }, chunkdb: { instance_id: '1', http_port: 12010, rpc_port: 12110 }, 'chunk-kv': { instance_id: '1', http_port: 15010, rpc_port: 15110 }, @@ -166,7 +167,7 @@ describe('Add Node dialog', () => { const cards = screen.getByRole('list', { name: 'Default services' }); expect(within(cards).getAllByRole('checkbox').map(input => input.getAttribute('aria-label'))).toEqual(serviceOrder.map(kind => serviceLabels[kind])); for (const kind of serviceOrder.filter(kind => kind !== 'access-server')) { - expect(within(screen.getByTestId(`service-card-${kind}`)).getAllByRole('textbox')).toHaveLength(1); + expect(within(screen.getByTestId(`service-card-${kind}`)).getAllByRole('textbox')).toHaveLength(kind === 'diskio' ? 1 : 2); expect(screen.getByLabelText(`${serviceLabels[kind]} RPC port`)).toBeEnabled(); } fireEvent.click(screen.getByLabelText('crowdb-access-server')); @@ -282,17 +283,17 @@ describe('Deploy Server dialog', () => { [ { id: 1, - server: { mgmt_url: 'http://127.0.0.1:19910', rpc_url: 'http://127.0.0.1:19920' }, + server: { mgmt_url: 'http://127.0.0.1:10135', rpc_url: 'http://127.0.0.1:11135' }, }, { id: 2, - server: { mgmt_url: 'http://127.0.0.1:19910', rpc_url: 'http://127.0.0.1:19920' }, + server: { mgmt_url: 'http://127.0.0.1:10135', rpc_url: 'http://127.0.0.1:11135' }, }, ], 1, ); - expect(defaults).toEqual({ defaultRestPort: '19911', defaultRpcPort: '19921' }); + expect(defaults).toEqual({ defaultRestPort: '10136', defaultRpcPort: '11136' }); }); it('increments globally when a different node already uses the base ports', () => { @@ -300,77 +301,38 @@ describe('Deploy Server dialog', () => { [ { id: 2, - server: { mgmt_url: 'http://127.0.0.1:19910', rpc_url: 'http://127.0.0.1:19920' }, + server: { mgmt_url: 'http://127.0.0.1:10135', rpc_url: 'http://127.0.0.1:11135' }, }, ], 1, ); - expect(defaults).toEqual({ defaultRestPort: '19911', defaultRpcPort: '19921' }); + expect(defaults).toEqual({ defaultRestPort: '10136', defaultRpcPort: '11136' }); }); it('derives defaults from the node id suffix before checking collisions', () => { - const defaults = deployPortDefaultsForNode([], 2, 19910, 19920); + const defaults = deployPortDefaultsForNode([], 2, 10135, 11135); - expect(defaults).toEqual({ defaultRestPort: '19912', defaultRpcPort: '19922' }); + expect(defaults).toEqual({ defaultRestPort: '10136', defaultRpcPort: '11136' }); }); it('can increment from remembered same-node ports even after the server is gone', () => { - const defaults = deployPortDefaultsForNode([], 1, 19910, 19920, [19910], [19920]); + const defaults = deployPortDefaultsForNode([], 1, 10135, 11135, [10135], [11135]); - expect(defaults).toEqual({ defaultRestPort: '19911', defaultRpcPort: '19921' }); + expect(defaults).toEqual({ defaultRestPort: '10136', defaultRpcPort: '11136' }); }); }); describe('diskdbPortDefaultsForNode', () => { - it('returns the base port when no instances or remembered ports collide', () => { - expect(diskdbPortDefaultsForNode([], 1)).toBe('29921'); + it('uses node offsets zero, one and two', () => { + expect([1, 2, 3].map(id => diskdbPortDefaultsForNode([], id))).toEqual(['14135', '14136', '14137']); }); - - it('derives defaults from the node id suffix before checking collisions', () => { - expect(diskdbPortDefaultsForNode([], 2)).toBe('29922'); - }); - - it('increments past ports already assigned to other diskdb instances', () => { - const instances = [{ rpc_endpoint: 'http://127.0.0.1:29921' }]; - expect(diskdbPortDefaultsForNode(instances, 1)).toBe('29922'); - }); - - it('skips the complete range for remembered ports even when no instances exist', () => { - expect(diskdbPortDefaultsForNode([], 1, undefined, [29921])).toBe('29924'); - }); - - it('allocates disjoint 3-port ranges across nodes 1, 2, and 3', () => { - // Simulate sequential allocation: each node's base is remembered so - // the next node skips past the full derived range (base, base+1, base+2). - const node1 = Number(diskdbPortDefaultsForNode([], 1)); - const node2 = Number(diskdbPortDefaultsForNode([], 2, undefined, [node1])); - const node3 = Number(diskdbPortDefaultsForNode([], 3, undefined, [node1, node2])); - - const ranges = [node1, node2, node3].map((b) => [b, b + 1, b + 2]); - const all = ranges.flat(); - expect(new Set(all).size).toBe(all.length); - // Node 1 starts at its suffix offset (29921). - expect(node1).toBe(29921); - // Node 2's suffix offset (29922) falls inside node 1's range, so it - // skips to the first free 3-port window. - expect(node2).toBe(29924); - // Node 3's suffix offset (29923) falls inside node 1's range, and - // 29924-29926 are node 2's, so it skips further. - expect(node3).toBe(29927); - }); - - it('detects collisions on each derived listener port (main, HTTP, crowdb-rpc)', () => { - // An existing instance with crowdb-rpc endpoint 29923 means its main - // listener is 29921 and HTTP is 29922 — all three collide with node 1's - // default window [29921, 29922, 29923]. - expect(diskdbPortDefaultsForNode([{ rpc_endpoint: 'http://127.0.0.1:29923' }], 1)).toBe('29924'); - // Endpoint 29922 → main 29920, HTTP 29921, rpc 29922. Node 1's window - // [29921, 29922, 29923] collides on HTTP and rpc. - expect(diskdbPortDefaultsForNode([{ rpc_endpoint: 'http://127.0.0.1:29922' }], 1)).toBe('29923'); - // Endpoint 29921 → main 29919, HTTP 29920, rpc 29921. Node 1's window - // collides only on the main port (29921). - expect(diskdbPortDefaultsForNode([{ rpc_endpoint: 'http://127.0.0.1:29921' }], 1)).toBe('29922'); + it('skips active RPC endpoints and remembered ports', () => { + expect(diskdbPortDefaultsForNode([{ rpc_endpoint: 'http://127.0.0.1:14135' }], 1)).toBe('14136'); + expect(diskdbPortDefaultsForNode([], 1, undefined, [14135, 14136])).toBe('14137'); + }); + it('does not reserve imaginary main or HTTP listeners adjacent to RPC', () => { + expect(diskdbPortDefaultsForNode([{ rpc_endpoint: 'http://127.0.0.1:14137' }], 1)).toBe('14135'); }); }); @@ -888,3 +850,19 @@ describe('end-to-end create flow', () => { expect(captured[5].body).toEqual({ node_id: 2, replica_id: 701 }); }); }); + + +describe('Deploy DiskDB independent listeners', () => { + it('submits HTTP and the exact RPC port returned by the server', async () => { + installFetchMock({ node_id: 1, endpoint: 'http://127.0.0.1:14135', pid: 1 }); + render(<DeployDiskdbDialog isOpen nodes={[{ id: 1 }]} onClose={() => {}} />, { wrapper }); + await waitFor(() => expect(screen.getByLabelText('RPC Port (crowdb-rpc)')).toHaveValue('14135')); + expect(screen.getByLabelText('HTTP Port (management)')).toHaveValue('13135'); + fireEvent.change(screen.getByLabelText('HTTP Port (management)'), { target: { value: '14135' } }); + expect(screen.getByRole('button', { name: /^Deploy$/ })).toBeDisabled(); + fireEvent.change(screen.getByLabelText('HTTP Port (management)'), { target: { value: '13135' } }); + fireEvent.click(screen.getByRole('button', { name: /^Deploy$/ })); + await waitFor(() => expect(captured).toHaveLength(1)); + expect(captured[0]).toMatchObject({ url: '/api/nodes/1/diskdb/deploy', body: { rpc_port: 14135, http_port: 13135 } }); + }); +}); diff --git a/app/crowdb-web/ui/src/components/dialogs/defaults.ts b/app/crowdb-web/ui/src/components/dialogs/defaults.ts index 9bb324a1f..346fb23ae 100644 --- a/app/crowdb-web/ui/src/components/dialogs/defaults.ts +++ b/app/crowdb-web/ui/src/components/dialogs/defaults.ts @@ -1,13 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// Per-service port bases. Ports are generated dynamically from these -// bases (offset by the node-id suffix, then incremented past collisions -// against ports already assigned in the console config). The bases are -// dev defaults; production should override via a deployment config story. -export const KV_SERVER_REST_PORT_BASE = 19910; -export const KV_SERVER_RPC_PORT_BASE = 19920; -export const DISKDB_RPC_PORT_BASE = 29920; +import { PAXOS_KV_SERVER_MGMT_BASE, PAXOS_KV_SERVER_RPC_BASE, DISKDB_RPC_BASE } from '../../ports'; +export const KV_SERVER_REST_PORT_BASE = PAXOS_KV_SERVER_MGMT_BASE; +export const KV_SERVER_RPC_PORT_BASE = PAXOS_KV_SERVER_RPC_BASE; +export const DISKDB_RPC_PORT_BASE = DISKDB_RPC_BASE; const DIGIT_SUFFIX = /(\d+)$/; @@ -125,7 +122,7 @@ function preferredPortStart(base: number, nodeId: number): number { const suffix = Number(match[1]); if (!Number.isFinite(suffix) || suffix < 0) return base; - return base + suffix; + return base + Math.max(0, suffix - 1); } export function deployPortDefaultsForNode( @@ -178,26 +175,6 @@ export function diskdbPortDefaultsForNode( rpcStart = DISKDB_RPC_PORT_BASE, extraUsedRpcPorts: number[] = [], ): string { - // The public input is the base listener. DiskDB also binds base + 1 for - // HTTP readiness and base + 2 for crowdb-rpc, so reserve the whole range. - const usedPorts = new Set<number>(); - for (const base of extraUsedRpcPorts) { - usedPorts.add(base); - usedPorts.add(base + 1); - usedPorts.add(base + 2); - } - for (const inst of instances) { - const endpoint = extractPort(inst.rpc_endpoint); - if (endpoint) { - usedPorts.add(endpoint - 2); - usedPorts.add(endpoint - 1); - usedPorts.add(endpoint); - } - } - - let candidate = preferredPortStart(rpcStart, nodeId); - while (candidate < 65534 && [candidate, candidate + 1, candidate + 2].some((port) => usedPorts.has(port))) { - candidate += 1; - } - return String(candidate); + const usedPorts = [...extraUsedRpcPorts, ...instances.map(instance => extractPort(instance.rpc_endpoint)).filter((port): port is number => port !== null)]; + return nextAvailablePort(usedPorts, preferredPortStart(rpcStart, nodeId)); } diff --git a/app/crowdb-web/ui/src/embed.ts b/app/crowdb-web/ui/src/embed.ts index 4ea968798..ebce9eb0a 100644 --- a/app/crowdb-web/ui/src/embed.ts +++ b/app/crowdb-web/ui/src/embed.ts @@ -6,7 +6,7 @@ * * Usage from a host React app: * import { CrowdbConsole } from 'crowdb-console'; - * <CrowdbConsole apiPrefix="/storage/crowdb-kv/api" readonly /> + * <CrowdbConsole apiPrefix="/storage/crowdb-paxos-kv/api" readonly /> */ export { default as CrowdbConsole, type CrowdbConsoleProps } from './App'; export { Domain } from './types'; diff --git a/app/crowdb-web/ui/src/ports.ts b/app/crowdb-web/ui/src/ports.ts new file mode 100644 index 000000000..cf7362374 --- /dev/null +++ b/app/crowdb-web/ui/src/ports.ts @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +// Generated from ports.rs by tools/ports/generate.py; do not edit. +export const PAXOS_KV_SERVER_MGMT_BASE = 10135; +export const PAXOS_KV_SERVER_RPC_BASE = 11135; +export const DISKDB_HTTP_BASE = 13135; +export const DISKDB_RPC_BASE = 14135; +export const CHUNKDB_HTTP_BASE = 16135; +export const CHUNKDB_RPC_BASE = 17135; +export const DISKIO_RPC_BASE = 18135; +export const WEB_BASE = 6135; +export const CHUNK_KV_HTTP_BASE = 20135; +export const CHUNK_KV_RPC_BASE = 21135; +export const ACCESS_SERVER_HTTP_BASE = 7135; +export const ACCESS_SERVER_ICEBERG_HTTP_BASE = 8135; +export const ACCESS_SERVER_DATASET_HTTP_BASE = 9135; +export const ACCESS_SERVER_HEALTH_HTTP_BASE = 25135; +export const MONITOR_HTTP_BASE = 26135; +export const SSH_PORT = 2222; +export const HOST_SSH_PORT = 22; diff --git a/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx b/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx index 6cedf86f8..274fd025b 100644 --- a/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx +++ b/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx @@ -19,12 +19,11 @@ export function DeployServiceDialog({ nodeId, kind, servers, stores, diskGroups, const value = /^\d+$/.test(suffix) ? BigInt(suffix) : 0n; return value > maximum ? value : maximum; }, 0n) + 1n)); - const base = { chunkdb: 43200, diskio: 44000, 'chunk-kv': 45100, 'access-server': 9092 }[kind]; - const [httpPort, setHttpPort] = useState(String(base)); - const [rpcPort, setRpcPort] = useState(String(kind === 'diskio' ? base : base + 100)); - const [s3Port, setS3Port] = useState('9091'); - const [datasetPort, setDatasetPort] = useState('9093'); - const [healthPort, setHealthPort] = useState('9094'); + const [httpPort, setHttpPort] = useState(''); + const [rpcPort, setRpcPort] = useState(''); + const [s3Port, setS3Port] = useState(''); + const [datasetPort, setDatasetPort] = useState(''); + const [healthPort, setHealthPort] = useState(''); const [diskGroup, setDiskGroup] = useState(String(diskGroups[0]?.id ?? '')); const [store, setStore] = useState(String(stores.find(store => String(store.store_id) !== '0')?.store_id ?? stores[0]?.store_id ?? '')); const firstGroup = stores.find(entry => String(entry.store_id) === store)?.groups.find(entry => String(entry.group_id) !== '0'); diff --git a/app/crowdb-web/ui/src/services/ServicePlanCard.tsx b/app/crowdb-web/ui/src/services/ServicePlanCard.tsx index d28ac86bf..d756d7112 100644 --- a/app/crowdb-web/ui/src/services/ServicePlanCard.tsx +++ b/app/crowdb-web/ui/src/services/ServicePlanCard.tsx @@ -7,7 +7,7 @@ type Ports = Omit<NonNullable<ServiceOverrides[ServiceKind]>, 'dynamic_ownership export function listenerFields(kind: ServiceKind): [keyof Ports, string][] { return kind === 'access-server' ? [['http_port', 'Iceberg'], ['s3_port', 'S3'], ['dataset_port', 'Dataset'], ['health_port', 'Health']] - : [['rpc_port', 'RPC']]; + : kind === 'diskio' ? [['rpc_port', 'RPC']] : [['http_port', 'HTTP'], ['rpc_port', 'RPC']]; } export function ServicePlanCard({ kind, enabled, ports, onToggle, onChange }: { kind: ServiceKind; enabled: boolean; ports: Ports; onToggle: () => void; diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts index 079ea5fc8..f557526d1 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts @@ -242,7 +242,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< if (disposed || stopped.current) break; const value = { ...defaults[kind], ...(overrides.current[id]?.[kind] ?? {}) }; if (kind === 'paxos-kv') await deployServer(id, { rest_port: value.http_port!, rpc_port: value.rpc_port! }); - else if (kind === 'diskdb') await deployDiskdb(id, { rpc_port: value.rpc_port!, ...(value.keepalive_interval_secs === undefined ? {} : { keepalive_interval_secs: value.keepalive_interval_secs }) }); + else if (kind === 'diskdb') await deployDiskdb(id, { rpc_port: value.rpc_port!, ...(value.http_port === undefined ? {} : { http_port: value.http_port }), ...(value.keepalive_interval_secs === undefined ? {} : { keepalive_interval_secs: value.keepalive_interval_secs }) }); else await serviceRequest(`/nodes/${id}/services/deploy`, 'POST', { kind, ...value, ...(kind === 'diskio' ? { disk_group_id: diskGroup?.id ?? 0 } : {}), diff --git a/app/crowdb-web/ui/src/shell/NodeSetupOptions.tsx b/app/crowdb-web/ui/src/shell/NodeSetupOptions.tsx index 6e68d0c83..02607f34f 100644 --- a/app/crowdb-web/ui/src/shell/NodeSetupOptions.tsx +++ b/app/crowdb-web/ui/src/shell/NodeSetupOptions.tsx @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +import { PAXOS_KV_SERVER_MGMT_BASE, PAXOS_KV_SERVER_RPC_BASE } from '../ports'; import { Input, Select } from '../components/ui/Input'; import { serviceLabels } from '../services/useNodeServicePlans'; import type { DiskInventory } from '../storage/types'; @@ -23,7 +24,7 @@ export function NodeSetupOptions({ choices, inventory, disabled, onChange }: Pro return <fieldset disabled={disabled} className="tw-space-y-4"> <section aria-label="Node service choices" className="tw-space-y-2"> <h3 className="tw-font-semibold">Services</h3> - <p className="tw-text-xs tw-text-muted">Selected services start on the node after their prerequisites are ready. Blank listeners use available defaults. KV bootstrap retains ports 10000 / 10100.</p> + <p className="tw-text-xs tw-text-muted">Selected services start on the node after their prerequisites are ready. Blank listeners use available defaults. KV bootstrap uses ports {PAXOS_KV_SERVER_MGMT_BASE} / {PAXOS_KV_SERVER_RPC_BASE}.</p> <p className="tw-text-xs tw-text-muted">Unselecting a service retains any existing deployment, including the admitted KV process.</p> <div className="tw-grid tw-grid-cols-1 sm:tw-grid-cols-2 tw-gap-2">{choices.services.map((service, index) => <div key={service.kind} className="tw-rounded tw-border tw-border-border tw-p-2 tw-space-y-2"> <label className="tw-flex tw-gap-2 tw-text-sm"><input type="checkbox" checked={service.enabled} onChange={event => updateService(index, { enabled: event.target.checked })} />{serviceLabels[service.kind]}</label> diff --git a/app/crowdb-web/ui/src/shell/nodeSetup.ts b/app/crowdb-web/ui/src/shell/nodeSetup.ts index 631816fa7..220549af6 100644 --- a/app/crowdb-web/ui/src/shell/nodeSetup.ts +++ b/app/crowdb-web/ui/src/shell/nodeSetup.ts @@ -46,11 +46,7 @@ export function validNodeSetup(choices: NodeSetupChoices): boolean { ports.add(Number(value)); } } - if (service.kind === 'diskdb' && service.listeners.rpc_port != null) { - const base = service.listeners.rpc_port; - if (base > 65533 || ports.has(base + 1) || ports.has(base + 2)) return false; - ports.add(base + 1); ports.add(base + 2); - } + } const storage = choices.storage; if (storage.mode === 'diskless') return true; diff --git a/app/crowdb-web/ui/src/types/index.ts b/app/crowdb-web/ui/src/types/index.ts index ee72c3b5d..24fa3b770 100644 --- a/app/crowdb-web/ui/src/types/index.ts +++ b/app/crowdb-web/ui/src/types/index.ts @@ -86,7 +86,7 @@ export interface NodeStore { } // Batched crowdb-tree engine diagnostics (doc/todo-sm.md Step 6); mirrors -// crowdb-kv's `CrowdbTreeStatsView`. Present only when the group's engine is +// crowdb-paxos-kv's `CrowdbTreeStatsView`. Present only when the group's engine is // `CrowdbTreeEngine` (absent/undefined for `InMemKV`). export interface CrowdbTreeStats { last_applied_slot: number; @@ -257,7 +257,7 @@ export interface ApiError { details?: any; } -// Election/lease state (mirrors crowdb-kv's ElectionStateView) +// Election/lease state (mirrors crowdb-paxos-kv's ElectionStateView) export interface ElectionState { election_count?: number; current_term: number; @@ -269,7 +269,7 @@ export interface ElectionState { step_downs_admin?: number; } -// Read-path state gauges (mirrors crowdb-kv's ReadStateView) +// Read-path state gauges (mirrors crowdb-paxos-kv's ReadStateView) export interface ReadState { lease_valid: number; contiguous_applied: number; diff --git a/app/crowdb-web/ui/vite.config.ts b/app/crowdb-web/ui/vite.config.ts index 9f2386542..dc7e2a272 100644 --- a/app/crowdb-web/ui/vite.config.ts +++ b/app/crowdb-web/ui/vite.config.ts @@ -1,27 +1,22 @@ /// <reference types="vitest" /> import { defineConfig } from "vite"; import react from "@vitejs/plugin-react"; +import { WEB_BASE } from "./src/ports"; // During `npm run dev`, Vite serves the SPA on port 5173 and proxies -// `/api/*` and `/healthz` to the local Axum backend on 9090 (the -// crowdb-web default). For production, `npm run build` +// `/api/*` and `/healthz` to the default local Axum backend. +// For production, `npm run build` // emits to `dist/`, which Axum serves directly via ServeDir. +const backend = `http://127.0.0.1:${WEB_BASE}`; +const proxy = { "/api": backend, "/healthz": backend, "/internal": backend }; export default defineConfig({ plugins: [react()], server: { port: 5173, - proxy: { - "/api": "http://127.0.0.1:9090", - "/healthz": "http://127.0.0.1:9090", - "/internal": "http://127.0.0.1:9090", - }, + proxy, }, preview: { - proxy: { - "/api": "http://127.0.0.1:9090", - "/healthz": "http://127.0.0.1:9090", - "/internal": "http://127.0.0.1:9090", - }, + proxy, }, build: { outDir: "dist", diff --git a/container/crowdb-monitor/Cargo.toml b/container/crowdb-monitor/Cargo.toml index d2b6b6e53..71c56da5e 100644 --- a/container/crowdb-monitor/Cargo.toml +++ b/container/crowdb-monitor/Cargo.toml @@ -18,7 +18,7 @@ test-util = [] crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } clap = { version = "4", features = ["derive", "env"] } crowdb-chunk-kv-client = { path = "../../lib/crowdb-chunk-kv-client" } -crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../../lib/crowdb-paxos-kv-client" } crowdb-diskio-client = { path = "../../lib/crowdb-diskio-client" } crowdb-protocol = { path = "../../lib/crowdb-protocol" } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi" } @@ -42,7 +42,7 @@ if-addrs = "0.13" [dev-dependencies] ssh-key = { version = "0.6", features = ["ed25519"] } -crowdb-kv = { path = "../../lib/crowdb-kv" } +crowdb-paxos-kv = { path = "../../lib/crowdb-paxos-kv" } crowdb-monitor = { path = ".", features = ["test-util"] } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } diff --git a/container/crowdb-monitor/src/bootstrap/chunk.rs b/container/crowdb-monitor/src/bootstrap/chunk.rs index a909d0030..aa1e6274c 100644 --- a/container/crowdb-monitor/src/bootstrap/chunk.rs +++ b/container/crowdb-monitor/src/bootstrap/chunk.rs @@ -1,7 +1,7 @@ use std::fs; use std::time::Duration; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::chunk_kv::Id128; use serde::Deserialize; use thiserror::Error; @@ -17,7 +17,7 @@ pub enum ChunkBootstrapError { #[error("chunk service configuration failed: {0}")] Io(#[from] std::io::Error), #[error("chunk service registry lookup failed: {0}")] - Registry(#[from] crowdb_kv_client::Error), + Registry(#[from] crowdb_paxos_kv_client::Error), #[error("chunk service configuration is invalid: {0}")] Invalid(&'static str), } @@ -106,7 +106,7 @@ pub async fn verify_chunk_services( if expected.is_empty() { return Ok(()); } - let registry = ServiceRegistryClient::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let registry = ServiceRegistryClient::new(PaxosKvClient::new(ClientConfig::new(vec![ management_seed.to_owned() ]))); registry.kv().refresh_topology().await?; diff --git a/container/crowdb-monitor/src/bootstrap/hardware.rs b/container/crowdb-monitor/src/bootstrap/hardware.rs index 77f7a3d28..bc4680d15 100644 --- a/container/crowdb-monitor/src/bootstrap/hardware.rs +++ b/container/crowdb-monitor/src/bootstrap/hardware.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::common::{DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{DiskGroupValue, DiskType, DiskValue}; use thiserror::Error; @@ -16,7 +16,7 @@ const UNIT_BYTES: u64 = 1024 * 1024; #[derive(Debug, Error)] pub enum HardwareBootstrapError { #[error("Group 0 hardware request failed: {0}")] - Client(#[from] crowdb_kv_client::Error), + Client(#[from] crowdb_paxos_kv_client::Error), #[error("bootstrap manifest failed: {0}")] Manifest(#[from] ManifestError), #[error("monitor lifecycle log failed: {0}")] @@ -69,7 +69,7 @@ pub struct HardwareBootstrap { impl HardwareBootstrap { #[must_use] pub fn new(management_seed: String) -> Self { - let kv = CrowdbKvClient::new(ClientConfig::new(vec![management_seed])); + let kv = PaxosKvClient::new(ClientConfig::new(vec![management_seed])); Self { client: HardwareClient::new(kv), } @@ -444,7 +444,7 @@ fn disk_matches(actual: &DiskValue, expected: &DiskValue) -> bool { } fn verify_written( - write: Result<(), crowdb_kv_client::Error>, + write: Result<(), crowdb_paxos_kv_client::Error>, visible: bool, ) -> Result<(), HardwareBootstrapError> { if visible { diff --git a/container/crowdb-monitor/src/bootstrap/iceberg.rs b/container/crowdb-monitor/src/bootstrap/iceberg.rs index 0a16f4b60..dd13d2e95 100644 --- a/container/crowdb-monitor/src/bootstrap/iceberg.rs +++ b/container/crowdb-monitor/src/bootstrap/iceberg.rs @@ -40,7 +40,7 @@ pub enum IcebergBootstrapError { #[error("Iceberg bootstrap manifest failed: {0}")] Manifest(#[from] ManifestError), #[error("Iceberg bootstrap authority failed: {0}")] - Authority(#[from] crowdb_kv_client::Error), + Authority(#[from] crowdb_paxos_kv_client::Error), #[error("monitor lifecycle log failed: {0}")] MonitorLog(#[from] MonitorLogError), } @@ -95,7 +95,7 @@ impl IcebergBootstrap { program: &std::path::Path, seeds: &str, credentials: &ServerCredentials, - kv: &crowdb_kv_client::CrowdbKvClient, + kv: &crowdb_paxos_kv_client::PaxosKvClient, ) -> Result<(), IcebergBootstrapError> { let operations = shared_console::operations(root, kv).await?; console::ensure_with_operations( @@ -115,7 +115,7 @@ impl IcebergBootstrap { /// Propagates the production journal and Group-0 reservation errors. pub async fn shared_console_operations_for_tests( root: &std::path::Path, - kv: &crowdb_kv_client::CrowdbKvClient, + kv: &crowdb_paxos_kv_client::PaxosKvClient, ) -> Result<(Uuid, Uuid), IcebergBootstrapError> { shared_console::operations(root, kv).await } diff --git a/container/crowdb-monitor/src/bootstrap/iceberg/shared_console.rs b/container/crowdb-monitor/src/bootstrap/iceberg/shared_console.rs index 47c676442..82cf0d421 100644 --- a/container/crowdb-monitor/src/bootstrap/iceberg/shared_console.rs +++ b/container/crowdb-monitor/src/bootstrap/iceberg/shared_console.rs @@ -4,7 +4,7 @@ //! One cluster-scoped Console catalog operation, without per-node races. use super::{console, IcebergBootstrapError}; -use crowdb_kv_client::{CrowdbKvClient, Error, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{Error, GetOutcome, PaxosKvClient, ReadMode}; use serde::{Deserialize, Serialize}; use std::{path::Path, time::Duration}; use uuid::Uuid; @@ -21,14 +21,14 @@ struct Operations { pub(super) async fn operations( root: &Path, - kv: &CrowdbKvClient, + kv: &PaxosKvClient, ) -> Result<(Uuid, Uuid), IcebergBootstrapError> { tokio::time::timeout(Duration::from_secs(3), reserve(root, kv)) .await .map_err(|_| IcebergBootstrapError::Command("Console catalog authority reservation timed out"))? } -async fn reserve(root: &Path, kv: &CrowdbKvClient) -> Result<(Uuid, Uuid), IcebergBootstrapError> { +async fn reserve(root: &Path, kv: &PaxosKvClient) -> Result<(Uuid, Uuid), IcebergBootstrapError> { let local = console::load(&root.join("console-iceberg-bootstrap.json"))?; let candidate = Operations { version: 1, diff --git a/container/crowdb-monitor/src/bootstrap/logical.rs b/container/crowdb-monitor/src/bootstrap/logical.rs index ac6b595e9..c049daa61 100644 --- a/container/crowdb-monitor/src/bootstrap/logical.rs +++ b/container/crowdb-monitor/src/bootstrap/logical.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, CrowdbSysmdClient}; +use crowdb_paxos_kv_client::{ClientConfig, CrowdbSysmdClient, PaxosKvClient}; use crowdb_protocol::common::{GroupValue, ReplicaValue, StoreValue}; use thiserror::Error; @@ -14,7 +14,7 @@ const STEP: &str = "logical-topology"; #[derive(Debug, Error)] pub enum LogicalBootstrapError { #[error("Group 0 logical topology request failed: {0}")] - Client(#[from] crowdb_kv_client::Error), + Client(#[from] crowdb_paxos_kv_client::Error), #[error("bootstrap manifest failed: {0}")] Manifest(#[from] ManifestError), #[error("monitor lifecycle log failed: {0}")] @@ -40,7 +40,7 @@ impl LogicalBootstrap { #[must_use] pub fn new(management_seed: String) -> Self { Self { - client: CrowdbSysmdClient::new(CrowdbKvClient::new(ClientConfig::new(vec![management_seed]))), + client: CrowdbSysmdClient::new(PaxosKvClient::new(ClientConfig::new(vec![management_seed]))), } } @@ -221,7 +221,7 @@ fn expected(profile: &DeploymentProfile) -> Expected { } fn verify_write( - write: Result<(), crowdb_kv_client::Error>, + write: Result<(), crowdb_paxos_kv_client::Error>, matches: bool, ) -> Result<(), LogicalBootstrapError> { if matches { diff --git a/container/crowdb-monitor/src/bootstrap/storage_probe.rs b/container/crowdb-monitor/src/bootstrap/storage_probe.rs index eae7d7e26..8bb4e9d92 100644 --- a/container/crowdb-monitor/src/bootstrap/storage_probe.rs +++ b/container/crowdb-monitor/src/bootstrap/storage_probe.rs @@ -1,7 +1,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, DiskioError, OperationOptions}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; use thiserror::Error; use tokio::time::{sleep, Instant}; @@ -15,7 +15,7 @@ pub enum StorageProbeError { #[error("DiskIO probe failed: {0}")] Diskio(#[from] DiskioError), #[error("DiskIO registration lookup failed: {0}")] - Registry(#[from] crowdb_kv_client::Error), + Registry(#[from] crowdb_paxos_kv_client::Error), #[error("DiskIO profile is invalid: {0}")] Invalid(&'static str), #[error("DiskIO registration did not become ready")] @@ -69,7 +69,7 @@ async fn wait_for_registration( .first() .ok_or(StorageProbeError::Invalid("disk is absent"))? .disk_group_id; - let registry = ServiceRegistryClient::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let registry = ServiceRegistryClient::new(PaxosKvClient::new(ClientConfig::new(vec![ management_seed.to_owned() ]))); registry.kv().refresh_topology().await?; diff --git a/container/crowdb-monitor/src/main.rs b/container/crowdb-monitor/src/main.rs index ca27814eb..c6aae9359 100644 --- a/container/crowdb-monitor/src/main.rs +++ b/container/crowdb-monitor/src/main.rs @@ -24,7 +24,7 @@ enum Command { seed: Vec<String>, #[arg(long)] data_root: PathBuf, - #[arg(long, default_value = "0.0.0.0:9095")] + #[arg(long, default_value_t = std::net::SocketAddr::from(([0, 0, 0, 0], crowdb_protocol::MONITOR_HTTP_BASE)))] bind: std::net::SocketAddr, #[arg(long, required = true)] interface: Vec<String>, diff --git a/container/crowdb-monitor/src/node/runtime.rs b/container/crowdb-monitor/src/node/runtime.rs index ea54bcfe8..b9f87ab7f 100644 --- a/container/crowdb-monitor/src/node/runtime.rs +++ b/container/crowdb-monitor/src/node/runtime.rs @@ -251,7 +251,7 @@ async fn verify_binding(binding: &NodeBinding) -> Result<()> { if binding.management_seeds.is_empty() || binding.management_seeds.len() > 16 { return Err("invalid authority seeds".into()); } - let client = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let client = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( binding.management_seeds.clone(), )); let found = client @@ -259,11 +259,11 @@ async fn verify_binding(binding: &NodeBinding) -> Result<()> { 0, 0, b"/deployment/cluster", - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await?; - let crowdb_kv_client::GetOutcome::Found { value, .. } = found else { + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = found else { return Err("cluster publication not complete".into()); }; let publication: Value = serde_json::from_slice(&value)?; @@ -275,11 +275,11 @@ async fn verify_binding(binding: &NodeBinding) -> Result<()> { 0, 0, b"/deployment/nodes", - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await?; - let crowdb_kv_client::GetOutcome::Found { value, .. } = registry else { + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = registry else { return Err("node registry publication incomplete".into()); }; let registry: Value = serde_json::from_slice(&value)?; @@ -334,7 +334,7 @@ fn web_service(profile: &DeploymentProfile, monitor_port: u16) -> Result<Service profile.paths.log_root.join("web").to_string_lossy().into_owned(), ]; web.env.insert( - "CROWDB_KV_KNOWN_HOSTS".into(), + "CROWDB_PAXOS_KV_KNOWN_HOSTS".into(), root.join("ssh/console-known_hosts") .to_string_lossy() .into_owned(), diff --git a/container/crowdb-monitor/src/node/runtime/admission.rs b/container/crowdb-monitor/src/node/runtime/admission.rs index 2699e4e23..0000adaeb 100644 --- a/container/crowdb-monitor/src/node/runtime/admission.rs +++ b/container/crowdb-monitor/src/node/runtime/admission.rs @@ -25,7 +25,7 @@ pub(super) async fn validate( return Err("admission operation cancelled".into()); } let discovery_id = NodeIdentity::load_or_create(root)?.uuid().to_string(); - let client = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let client = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( grant.management_seeds.clone(), )); let publication = tokio::time::timeout( @@ -34,12 +34,12 @@ pub(super) async fn validate( 0, 0, crowdb_console_shared::deployment::CLUSTER_KEY, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ), ) .await??; - let crowdb_kv_client::GetOutcome::Found { value, .. } = publication else { + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = publication else { return Err("cluster publication unavailable".into()); }; let publication: crowdb_console_shared::deployment::PreparedBootstrap = serde_json::from_slice(&value)?; diff --git a/container/crowdb-monitor/src/node/runtime/file_admission.rs b/container/crowdb-monitor/src/node/runtime/file_admission.rs index e0359c61a..817a2a2b2 100644 --- a/container/crowdb-monitor/src/node/runtime/file_admission.rs +++ b/container/crowdb-monitor/src/node/runtime/file_admission.rs @@ -5,7 +5,7 @@ use std::path::Path; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::{DiskId, HwStatus}, disk_storage::{DiskStorageRecord, PhysicalBacking}, @@ -25,7 +25,7 @@ pub(super) fn authority(root: &Path, request: &PrepareDiskFile) -> Result<Hardwa if binding.node_id != request.node_id || binding.bootstrap.cluster_id != request.cluster_id { return Err("file admission belongs to another node or cluster".into()); } - Ok(HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + Ok(HardwareClient::new(PaxosKvClient::new(ClientConfig::new( binding.management_seeds, )))) } diff --git a/container/crowdb-monitor/src/node/runtime/file_preparation.rs b/container/crowdb-monitor/src/node/runtime/file_preparation.rs index 0f62eef96..2168e31bc 100644 --- a/container/crowdb-monitor/src/node/runtime/file_preparation.rs +++ b/container/crowdb-monitor/src/node/runtime/file_preparation.rs @@ -119,8 +119,8 @@ impl FileAllocator { FilePreparationPhase::WaitingForDiskIO } else if current == Some(FilePreparationPhase::Unknown) || matches!( - error.downcast_ref::<crowdb_kv_client::Error>(), - Some(crowdb_kv_client::Error::OutcomeUnknown) + error.downcast_ref::<crowdb_paxos_kv_client::Error>(), + Some(crowdb_paxos_kv_client::Error::OutcomeUnknown) ) { FilePreparationPhase::Unknown diff --git a/container/crowdb-monitor/src/node/runtime/kv.rs b/container/crowdb-monitor/src/node/runtime/kv.rs index 8af9ed7de..716ec45de 100644 --- a/container/crowdb-monitor/src/node/runtime/kv.rs +++ b/container/crowdb-monitor/src/node/runtime/kv.rs @@ -51,7 +51,10 @@ pub(super) fn node_profile(config: &super::NodeRuntimeConfig) -> Result<Deployme .find(|service| service.id == "kv") .ok_or("node profile requires KV service")? .env - .insert("CROWDB_KV_MANAGEMENT_ADVERTISE_ADDR".into(), address.to_string()); + .insert( + "CROWDB_PAXOS_KV_MANAGEMENT_ADVERTISE_ADDR".into(), + address.to_string(), + ); Ok(profile) } @@ -173,7 +176,7 @@ async fn spawn( } let mut kv = service(profile, "kv")?; kv.env.insert( - "CROWDB_KV_CLUSTER_ID".into(), + "CROWDB_PAXOS_KV_CLUSTER_ID".into(), accepted.bootstrap.cluster_id.clone(), ); let root = profile diff --git a/container/crowdb-monitor/src/node/runtime/recovery/confirmed.rs b/container/crowdb-monitor/src/node/runtime/recovery/confirmed.rs index 19c060f23..74ac55315 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/confirmed.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/confirmed.rs @@ -11,7 +11,7 @@ use crowdb_console_shared::{ }, ConsoleConfig, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::mgmt::node::NodeBinding; use ssh_key::PublicKey; use std::{path::Path, time::Duration}; @@ -31,7 +31,7 @@ async fn authority( }) .ok_or("confirmed recovery requires authority contacts")?; tokio::time::timeout(Duration::from_secs(3), async { - let client = CrowdbKvClient::new(ClientConfig::new(seeds)); + let client = PaxosKvClient::new(ClientConfig::new(seeds)); let GetOutcome::Found { value, .. } = client .get( 0, diff --git a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs index a72a043fc..e49d2104d 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs @@ -9,7 +9,7 @@ use crowdb_console_shared::{ ops::OpContext, ConsoleConfig, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::mgmt::node::{recovery::MemberEndpoints, NodeServiceAction}; use std::{path::Path, sync::Arc}; @@ -23,7 +23,7 @@ pub(super) async fn reconcile( let local = super::identity(root)?.endpoint; let mut config = topology.clone(); let seeds = apply_member_endpoints(&mut config, endpoints)?; - let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); + let client = Arc::new(PaxosKvClient::new(ClientConfig::new(seeds.clone()))); let GetOutcome::Found { value, .. } = client .get( 0, @@ -135,7 +135,7 @@ fn apply_member_endpoints(config: &mut ConsoleConfig, endpoints: &MemberEndpoint async fn rebind( root: &Path, - client: &CrowdbKvClient, + client: &PaxosKvClient, operation: &PreparedBootstrap, node: u64, current: &str, @@ -240,6 +240,6 @@ pub async fn rebind_recovery_services_for_tests( seeds: &[String], complete: bool, ) -> Result<bool> { - let client = CrowdbKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:9".into()])); + let client = PaxosKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:9".into()])); rebind(root, &client, operation, node, current, seeds, complete).await } diff --git a/container/crowdb-monitor/src/node/runtime/retained_storage.rs b/container/crowdb-monitor/src/node/runtime/retained_storage.rs index 532933987..a8935a7a8 100644 --- a/container/crowdb-monitor/src/node/runtime/retained_storage.rs +++ b/container/crowdb-monitor/src/node/runtime/retained_storage.rs @@ -3,7 +3,7 @@ //! Verify bound authority before inspecting an explicitly retained disk. use super::{durable, Result}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::HwStatus, disk_storage::DiskStorageRecord, @@ -35,7 +35,7 @@ pub(super) async fn check(root: &Path, request: CheckRetainedDisk) -> Result<Ret disk_group_id: request.disk_group_id, disk_id, }; - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new(binding.management_seeds))); + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new(binding.management_seeds))); let nodes = hardware.list_nodes().await?; let locations: Vec<_> = nodes.iter().filter(|(_, id, _)| *id == request.node_id).collect(); if locations.len() != 1 || locations[0].0 != request.rack_id { diff --git a/container/crowdb-monitor/src/node/runtime/services.rs b/container/crowdb-monitor/src/node/runtime/services.rs index d242000b1..7ccad59f7 100644 --- a/container/crowdb-monitor/src/node/runtime/services.rs +++ b/container/crowdb-monitor/src/node/runtime/services.rs @@ -85,8 +85,9 @@ pub(super) fn kv_paused(profile: &DeploymentProfile) -> Result<bool> { pub(super) async fn desired(profile: &DeploymentProfile) -> Result<Vec<NodeServiceIntent>> { let binding: NodeBinding = durable::read(&profile.paths.data_root.join("node-binding.json"))?.ok_or("node unbound")?; - let client = - crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new(binding.management_seeds)); + let client = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( + binding.management_seeds, + )); let records = tokio::time::timeout( Duration::from_secs(3), client.scan_bounded(0, 0, b"/deployment/services/", b"", b"", 4096, false, None), diff --git a/container/crowdb-monitor/src/node/runtime/services/dependencies.rs b/container/crowdb-monitor/src/node/runtime/services/dependencies.rs index 6db53985b..575a02130 100644 --- a/container/crowdb-monitor/src/node/runtime/services/dependencies.rs +++ b/container/crowdb-monitor/src/node/runtime/services/dependencies.rs @@ -13,7 +13,7 @@ pub(super) async fn committed_ready( intent: &NodeServiceIntent, binding: &NodeBinding, ) -> Result<()> { - let kv = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let kv = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( binding.management_seeds.clone(), )); let intents = tokio::time::timeout( @@ -88,8 +88,8 @@ pub(in crate::node::runtime) async fn ready( // has finished WAL replay. Probe the same routed read Access needs. let binding: NodeBinding = super::durable::read(&profile.paths.data_root.join("node-binding.json"))? .ok_or("node is unbound")?; - let kv = Arc::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(binding.management_seeds), + let kv = Arc::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(binding.management_seeds), )); let catalog = Arc::new(crowdb_chunk_kv_client::Group0ChunkKvRangeCatalogSource::from_shared(kv)); let (_, pages) = catalog.load().await?; @@ -130,8 +130,8 @@ pub(in crate::node::runtime) async fn ready( async fn diskdb_routes_ready(profile: &DeploymentProfile) -> Result<()> { let binding: NodeBinding = super::durable::read(&profile.paths.data_root.join("node-binding.json"))?.ok_or("node is unbound")?; - let kv = Arc::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(binding.management_seeds), + let kv = Arc::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(binding.management_seeds), )); crowdb_console_shared::deployment::services::diskdb_storage_ready(kv).await?; Ok(()) diff --git a/container/crowdb-monitor/src/node/runtime/services/execution.rs b/container/crowdb-monitor/src/node/runtime/services/execution.rs index 283a46a52..caf9aaef6 100644 --- a/container/crowdb-monitor/src/node/runtime/services/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/services/execution.rs @@ -315,7 +315,7 @@ async fn wait_ready( } async fn verify_intent(binding: &NodeBinding, intent: &NodeServiceIntent) -> Result<()> { - let client = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let client = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( binding.management_seeds.clone(), )); let key = format!("/deployment/services/{}/{}", intent.node_id, intent.service_id); @@ -325,12 +325,12 @@ async fn verify_intent(binding: &NodeBinding, intent: &NodeServiceIntent) -> Res 0, 0, key.as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ), ) .await??; - let crowdb_kv_client::GetOutcome::Found { value, .. } = outcome else { + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = outcome else { return Err("service intent is not committed".into()); }; let current: NodeServiceIntent = serde_json::from_slice(&value)?; diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution.rs b/container/crowdb-monitor/src/node/runtime/setup/execution.rs index d4c42e1d6..cf57862fc 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution.rs @@ -9,7 +9,7 @@ use super::{ }; use crate::DeploymentProfile; use crowdb_console_shared::deployment::registry; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::mgmt::node::{setup::NodeSetupPhase, NodeBinding}; use std::time::Duration; @@ -55,7 +55,7 @@ async fn advance(profile: &DeploymentProfile, state: &mut AcceptedSetup) -> Resu return Err("node setup admission was retired or rebound".into()); } tokio::time::timeout(Duration::from_secs(3), super::super::verify_binding(&binding)).await??; - let kv = std::sync::Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = std::sync::Arc::new(PaxosKvClient::new(ClientConfig::new( binding.management_seeds.clone(), ))); let (registry, _) = tokio::time::timeout(Duration::from_secs(3), registry::read(&kv)) diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/metadata.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/metadata.rs index 92a24e089..faeea9239 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/metadata.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/metadata.rs @@ -10,7 +10,7 @@ use crowdb_console_shared::{ ops::{kv_logical, OpContext}, ConsoleConfig, }; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_protocol::mgmt::node::NodeBinding; use super::Result; @@ -18,7 +18,7 @@ use super::Result; pub(super) async fn ready( root: &std::path::Path, binding: &NodeBinding, - kv: &Arc<CrowdbKvClient>, + kv: &Arc<PaxosKvClient>, ) -> Result<bool> { let ctx = OpContext::with_shared_client_preserving_hint( Arc::clone(kv), diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs index cb267f82f..ccb502d0d 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs @@ -6,7 +6,7 @@ use super::{journal, AcceptedSetup, Result}; use crate::DeploymentProfile; use crowdb_console_shared::deployment::services; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_protocol::mgmt::node::{ setup::{SetupServiceKind, SetupStepPhase}, NodeBinding, NodeControl, NodeServiceAction, @@ -18,7 +18,7 @@ pub(super) async fn advance( profile: &DeploymentProfile, state: &mut AcceptedSetup, binding: &NodeBinding, - kv: &CrowdbKvClient, + kv: &PaxosKvClient, storage_ready: bool, ) -> Result<bool> { if state.status.services.iter().all(|step| { diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/storage.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/storage.rs index 9e17e18ad..77c9344c1 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/storage.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/storage.rs @@ -5,7 +5,7 @@ use super::{journal, AcceptedSetup, Result}; use crate::DeploymentProfile; -use crowdb_kv_client::{CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{HardwareClient, PaxosKvClient}; use crowdb_protocol::mgmt::node::{ setup::{SetupDiskStatus, SetupStepPhase, SetupStorage}, storage::{managed_file_zone_bytes, DiskInput, FileDiskAdmission, FilePreparationPhase, PrepareDiskFile}, @@ -16,7 +16,7 @@ pub(super) async fn advance( profile: &DeploymentProfile, state: &mut AcceptedSetup, binding: &NodeBinding, - kv: &std::sync::Arc<CrowdbKvClient>, + kv: &std::sync::Arc<PaxosKvClient>, ) -> Result<bool> { let (capacity_bytes, reserve_bytes, sparse) = match &state.request.choices.storage { SetupStorage::Automatic { @@ -121,7 +121,7 @@ async fn advance_file( } pub(super) async fn ensure_group( - kv: &std::sync::Arc<CrowdbKvClient>, + kv: &std::sync::Arc<PaxosKvClient>, binding: &NodeBinding, group_id: u64, data_binding: Option<crowdb_protocol::common::BindMapValue>, @@ -170,7 +170,7 @@ pub(super) async fn ensure_group( pub(super) async fn select_group( hardware: &HardwareClient, - kv: &std::sync::Arc<CrowdbKvClient>, + kv: &std::sync::Arc<PaxosKvClient>, rack: u64, binding: &NodeBinding, media_type: Option<i32>, @@ -197,7 +197,7 @@ pub(super) async fn select_group( .await { Ok(()) => {} - Err(crowdb_kv_client::Error::Server(reason)) + Err(crowdb_paxos_kv_client::Error::Server(reason)) if reason == "DiskGroup media class or allocation unit is incompatible with this disk" => { if automatic { diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/supplied.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/supplied.rs index 08367ef76..b9306f5bb 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/supplied.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/supplied.rs @@ -8,7 +8,7 @@ use super::{ storage, Result, }; use crate::DeploymentProfile; -use crowdb_kv_client::{CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::{DiskId, HwStatus}, disk_storage::{DiskStorageRecord, PhysicalBacking}, @@ -29,7 +29,7 @@ pub(super) async fn advance( profile: &DeploymentProfile, state: &mut AcceptedSetup, binding: &NodeBinding, - kv: &Arc<CrowdbKvClient>, + kv: &Arc<PaxosKvClient>, ) -> Result<bool> { if state.supplied.is_empty() { prepare(profile, state, binding, kv).await?; @@ -102,7 +102,7 @@ async fn prepare( profile: &DeploymentProfile, state: &mut AcceptedSetup, binding: &NodeBinding, - kv: &Arc<CrowdbKvClient>, + kv: &Arc<PaxosKvClient>, ) -> Result<()> { let SetupStorage::Supplied { disks } = &state.request.choices.storage else { return Err("supplied setup choices missing".into()); diff --git a/container/crowdb-monitor/src/node/runtime/storage_health.rs b/container/crowdb-monitor/src/node/runtime/storage_health.rs index db16b261d..e4ab8d9f6 100644 --- a/container/crowdb-monitor/src/node/runtime/storage_health.rs +++ b/container/crowdb-monitor/src/node/runtime/storage_health.rs @@ -9,7 +9,7 @@ use std::{ }; use crowdb_console_shared::deployment::registry; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{common::HwStatus, key::DiskKey, mgmt::node::NodeBinding}; use super::{durable, Result}; @@ -61,7 +61,7 @@ async fn observe(root: PathBuf, after: Option<String>) -> Result<Option<String>> let binding: NodeBinding = durable::read(&root.join("node-binding.json"))?.ok_or("disk health binding is absent")?; tokio::time::timeout(Duration::from_secs(3), super::verify_binding(&binding)).await??; - let client = CrowdbKvClient::new(ClientConfig::new(binding.management_seeds.clone())); + let client = PaxosKvClient::new(ClientConfig::new(binding.management_seeds.clone())); let (registry, _) = registry::read(&client) .await? .ok_or("disk health registry is absent")?; diff --git a/container/crowdb-monitor/src/node/runtime/supplied_storage.rs b/container/crowdb-monitor/src/node/runtime/supplied_storage.rs index 72ddb008f..b1127b47f 100644 --- a/container/crowdb-monitor/src/node/runtime/supplied_storage.rs +++ b/container/crowdb-monitor/src/node/runtime/supplied_storage.rs @@ -112,8 +112,8 @@ impl SuppliedStorage { eprintln!("supplied disk provisioning failed; explicit retry required: {error}"); if let Ok(mut state) = journal::status(&root, &id) { state.phase = if matches!( - error.downcast_ref::<crowdb_kv_client::Error>(), - Some(crowdb_kv_client::Error::OutcomeUnknown) + error.downcast_ref::<crowdb_paxos_kv_client::Error>(), + Some(crowdb_paxos_kv_client::Error::OutcomeUnknown) ) { SuppliedPreparationPhase::Unknown } else { diff --git a/container/crowdb-monitor/src/node/runtime/supplied_storage/execution.rs b/container/crowdb-monitor/src/node/runtime/supplied_storage/execution.rs index d322665b0..94486723e 100644 --- a/container/crowdb-monitor/src/node/runtime/supplied_storage/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/supplied_storage/execution.rs @@ -6,7 +6,7 @@ use super::{authorize, journal, Result}; use crate::node::storage::blank; use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, OperationOptions, SegmentTarget}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::HwStatus, key::DiskKey, @@ -26,7 +26,7 @@ pub(super) async fn run_with_inputs( let mut state = journal::status(root, id)?; let binding = authorize(root, &state.request).await?; let retirement = super::retirement(root, &binding); - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new( binding.management_seeds.clone(), ))); let record = &state.request.record; diff --git a/container/crowdb-monitor/src/node/startup/policy.rs b/container/crowdb-monitor/src/node/startup/policy.rs index 0148669f7..8571437da 100644 --- a/container/crowdb-monitor/src/node/startup/policy.rs +++ b/container/crowdb-monitor/src/node/startup/policy.rs @@ -108,7 +108,6 @@ pub fn resolve_node_policy_with_port_offset( fn shift_service_ports(services: &mut [SetupService], offset: u16) -> io::Result<()> { for service in services { for port in [ - &mut service.listeners.listen_port, &mut service.listeners.http_port, &mut service.listeners.rpc_port, &mut service.listeners.s3_port, diff --git a/container/crowdb-monitor/tests/access_bootstrap_test.rs b/container/crowdb-monitor/tests/access_bootstrap_test.rs index 2c0fefd0c..50b3c3dc4 100644 --- a/container/crowdb-monitor/tests/access_bootstrap_test.rs +++ b/container/crowdb-monitor/tests/access_bootstrap_test.rs @@ -9,6 +9,7 @@ use crowdb_monitor::{ s3_step_names, show_client_credentials, BootstrapSession, DeploymentProfile, MonitorLog, S3Bootstrap, ServerCredentials, }; +use crowdb_protocol::port::ports::{ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE}; use uuid::Uuid; struct TestRoot(PathBuf); @@ -69,8 +70,12 @@ async fn s3_bootstrap_reuses_user_and_validates_ready_without_creation() { .unwrap(); assert_eq!(session.manifest().step_complete("s3-user"), Some(true)); let client = show_client_credentials(&data_root).unwrap(); - assert!(client.contains("AWS_ENDPOINT_URL=http://localhost:9091\n")); - assert!(client.contains("ICEBERG_URI=http://localhost:9092\n")); + assert!(client.contains(&format!( + "AWS_ENDPOINT_URL=http://localhost:{ACCESS_SERVER_HTTP_BASE}\n" + ))); + assert!(client.contains(&format!( + "ICEBERG_URI=http://localhost:{ACCESS_SERVER_ICEBERG_HTTP_BASE}\n" + ))); session.mark_ready().unwrap(); let mut restarted = BootstrapSession::open(&data_root, b"profile", b"config", &s3_step_names()).unwrap(); diff --git a/container/crowdb-monitor/tests/blank_medium_test.rs b/container/crowdb-monitor/tests/blank_medium_test.rs index 17c5d9fb8..31469e2cc 100644 --- a/container/crowdb-monitor/tests/blank_medium_test.rs +++ b/container/crowdb-monitor/tests/blank_medium_test.rs @@ -180,8 +180,8 @@ async fn authority(test: &TestMedium) -> node_setup::TestAuthority { }; let authority = node_setup::TestAuthority::start(test.root.path(), &request).await; assert_eq!(authority.registry.nodes[0].physical_host_id, "owned-host"); - let hardware = crowdb_kv_client::HardwareClient::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), + let hardware = crowdb_paxos_kv_client::HardwareClient::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), )); hardware .add_rack( @@ -232,13 +232,13 @@ async fn verified_medium_publishes_init_and_waits_for_real_diskio_without_reinit .await .unwrap(); assert_eq!(first.phase, SuppliedPreparationPhase::WaitingForDiskio); - let crowdb_kv_client::GetOutcome::Found { value, .. } = authority + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = authority .kv .get( 0, 0, test.record.key().as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await @@ -250,8 +250,8 @@ async fn verified_medium_publishes_init_and_waits_for_real_diskio_without_reinit serde_json::from_slice::<DiskStorageRecord>(&value).unwrap(), test.record ); - let hardware = crowdb_kv_client::HardwareClient::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), + let hardware = crowdb_paxos_kv_client::HardwareClient::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), )); let key = DiskKey { rack_id: 1, @@ -292,8 +292,8 @@ async fn nonblank_provisioning_never_publishes_membership_and_changed_retry_cann .await .unwrap_err(); assert!(error.to_string().contains("not blank"), "{error}"); - let hardware = crowdb_kv_client::HardwareClient::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), + let hardware = crowdb_paxos_kv_client::HardwareClient::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), )); let key = DiskKey { rack_id: 1, @@ -391,12 +391,12 @@ async fn missing_target_group_fails_before_blank_scan_and_global_resource_claim( 0, 0, record.key().as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await .unwrap(); - assert!(matches!(resources, crowdb_kv_client::GetOutcome::NotFound)); + assert!(matches!(resources, crowdb_paxos_kv_client::GetOutcome::NotFound)); let physical = record.physical_backing.as_ref().unwrap(); let encode = |text: &str| { text.as_bytes().iter().fold(String::new(), |mut encoded, byte| { @@ -415,12 +415,12 @@ async fn missing_target_group_fails_before_blank_scan_and_global_resource_claim( 0, 0, path.as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await .unwrap(); - assert!(matches!(claim, crowdb_kv_client::GetOutcome::NotFound)); + assert!(matches!(claim, crowdb_paxos_kv_client::GetOutcome::NotFound)); assert!( fs::metadata(test.root.path().join("disks/owned.img")) .unwrap() diff --git a/container/crowdb-monitor/tests/common/node_setup.rs b/container/crowdb-monitor/tests/common/node_setup.rs index 31f8a27e4..5e5d02a70 100644 --- a/container/crowdb-monitor/tests/common/node_setup.rs +++ b/container/crowdb-monitor/tests/common/node_setup.rs @@ -9,17 +9,17 @@ use crowdb_console_shared::{ ops::OpContext, ConsoleConfig, }; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::mgmt::node::{setup::NodeSetupRequest, NodeBinding}; use std::{os::unix::fs::PermissionsExt, path::Path, sync::Arc}; pub struct TestAuthority { store: Arc<PxKvStore>, task: tokio::task::JoinHandle<()>, - pub kv: CrowdbKvClient, + pub kv: PaxosKvClient, pub binding: NodeBinding, pub registry: NodeRegistry, } @@ -43,7 +43,7 @@ impl TestAuthority { let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); - let kv = CrowdbKvClient::new(ClientConfig::new(vec![seed.clone()])); + let kv = PaxosKvClient::new(ClientConfig::new(vec![seed.clone()])); kv.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); let bootstrap = bootstrap(root, &kv, &seed, &store.listen_addr().unwrap().to_string()).await; let binding = NodeBinding { @@ -84,7 +84,7 @@ impl TestAuthority { } } -async fn bootstrap(root: &Path, kv: &CrowdbKvClient, management: &str, rpc: &str) -> PreparedBootstrap { +async fn bootstrap(root: &Path, kv: &PaxosKvClient, management: &str, rpc: &str) -> PreparedBootstrap { let mut server = ServerEntry::new("kv-2", management); server.node_id = Some(2); server.rpc_url = Some(rpc.into()); diff --git a/container/crowdb-monitor/tests/common/node_setup_file.rs b/container/crowdb-monitor/tests/common/node_setup_file.rs index 41ca5117a..a778eb51e 100644 --- a/container/crowdb-monitor/tests/common/node_setup_file.rs +++ b/container/crowdb-monitor/tests/common/node_setup_file.rs @@ -53,11 +53,11 @@ pub async fn prepare( } async fn publish_storage(authority: &super::node_setup::TestAuthority) { - use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; + use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::common::{NodeValue, RackValue}; super::node_setup_groups::publish_group(authority, 1).await; let node = &authority.registry.nodes[0]; - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new( authority.binding.management_seeds.clone(), ))); hardware diff --git a/container/crowdb-monitor/tests/common/service_dependency.rs b/container/crowdb-monitor/tests/common/service_dependency.rs index 6f96d9c93..9b6170e7a 100644 --- a/container/crowdb-monitor/tests/common/service_dependency.rs +++ b/container/crowdb-monitor/tests/common/service_dependency.rs @@ -1,14 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::{DiskId, HwStatus, NodeValue, RackValue}, diskdb::rpc::{DiskGroupValue, DiskValue}, }; use std::sync::Arc; -pub async fn publish_online_disk(kv: Arc<CrowdbKvClient>) { +pub async fn publish_online_disk(kv: Arc<PaxosKvClient>) { let hardware = HardwareClient::from_shared(kv.clone()); hardware .add_rack( @@ -74,7 +74,7 @@ pub async fn ready_listener() -> (std::net::SocketAddr, tokio::task::JoinHandle< } pub async fn assert_initial_start_waits( - kv: &CrowdbKvClient, + kv: &PaxosKvClient, profile: &crowdb_monitor::DeploymentProfile, local: &crowdb_protocol::mgmt::node::NodeServiceIntent, consumer: &crowdb_protocol::mgmt::node::NodeServiceIntent, diff --git a/container/crowdb-monitor/tests/console_catalog_shared_test.rs b/container/crowdb-monitor/tests/console_catalog_shared_test.rs index bce87b447..0b0ad6c87 100644 --- a/container/crowdb-monitor/tests/console_catalog_shared_test.rs +++ b/container/crowdb-monitor/tests/console_catalog_shared_test.rs @@ -3,8 +3,8 @@ #[path = "common/node_setup.rs"] mod node_setup; -use crowdb_kv_client::{GetOutcome, ReadMode}; use crowdb_monitor::IcebergBootstrap; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::mgmt::node::setup::{NodeSetupChoices, NodeSetupRequest}; use serde_json::json; use std::{fs, path::Path}; diff --git a/container/crowdb-monitor/tests/file_admission_test.rs b/container/crowdb-monitor/tests/file_admission_test.rs index cb372ce6b..5adb4af91 100644 --- a/container/crowdb-monitor/tests/file_admission_test.rs +++ b/container/crowdb-monitor/tests/file_admission_test.rs @@ -1,8 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, HardwareClient, ReadMode}; use crowdb_monitor::{accept_file_for_tests, prepare_file_for_tests, publish_file_for_tests}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, HardwareClient, PaxosKvClient, ReadMode}; use crowdb_protocol::{ common::{HwStatus, NodeValue, RackValue}, disk_storage::DiskStorageRecord, @@ -93,7 +93,7 @@ async fn fixture(root: &std::path::Path) -> (node_setup::TestAuthority, Hardware }; let authority = node_setup::TestAuthority::start(root, &setup).await; assert_eq!(authority.registry.nodes[0].physical_host_id, "owned-host"); - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new( authority.binding.management_seeds.clone(), ))); hardware diff --git a/container/crowdb-monitor/tests/node_policy_test.rs b/container/crowdb-monitor/tests/node_policy_test.rs index 0184deb97..8569f23c6 100644 --- a/container/crowdb-monitor/tests/node_policy_test.rs +++ b/container/crowdb-monitor/tests/node_policy_test.rs @@ -44,13 +44,9 @@ fn host_network_policy_shifts_every_diskdb_listener_once_and_keeps_instances_dis crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &omitted, &omitted, offset) .unwrap(); let diskdb = &policy.setup.services[1].listeners; - assert_eq!(diskdb.listen_port, Some(11000 + offset)); - assert_eq!(diskdb.http_port, Some(11100 + offset)); - assert_eq!(diskdb.rpc_port, Some(11200 + offset)); - for port in [diskdb.listen_port, diskdb.http_port, diskdb.rpc_port] - .into_iter() - .flatten() - { + assert_eq!(diskdb.http_port, Some(crowdb_protocol::DISKDB_HTTP_BASE + offset)); + assert_eq!(diskdb.rpc_port, Some(crowdb_protocol::DISKDB_RPC_BASE + offset)); + for port in [diskdb.http_port, diskdb.rpc_port].into_iter().flatten() { assert!(listeners.insert(port)); } assert_eq!( diff --git a/container/crowdb-monitor/tests/node_profile_test.rs b/container/crowdb-monitor/tests/node_profile_test.rs index 2089dd209..2dcfda5a0 100644 --- a/container/crowdb-monitor/tests/node_profile_test.rs +++ b/container/crowdb-monitor/tests/node_profile_test.rs @@ -2,6 +2,9 @@ // Licensed under the Apache License, Version 2.0. use crowdb_monitor::DeploymentProfile; +use crowdb_protocol::port::ports::{ + ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE, WEB_BASE, +}; use std::{collections::BTreeMap, path::Path}; fn catalog() -> DeploymentProfile { @@ -34,7 +37,12 @@ fn node_catalog_has_no_prescribed_members_media_or_rendered_service_configuratio .collect(); assert_eq!( endpoints, - BTreeMap::from([("web", 9090), ("s3", 9091), ("iceberg", 9092), ("dataset", 9093)]) + BTreeMap::from([ + ("web", WEB_BASE), + ("s3", ACCESS_SERVER_HTTP_BASE), + ("iceberg", ACCESS_SERVER_ICEBERG_HTTP_BASE), + ("dataset", ACCESS_SERVER_DATASET_HTTP_BASE), + ]) ); } diff --git a/container/crowdb-monitor/tests/node_run_inputs_test.rs b/container/crowdb-monitor/tests/node_run_inputs_test.rs index d3f62fc6b..fe8fb6603 100644 --- a/container/crowdb-monitor/tests/node_run_inputs_test.rs +++ b/container/crowdb-monitor/tests/node_run_inputs_test.rs @@ -41,12 +41,12 @@ fn container_preflight_reuses_explicit_file_ports_with_a_nonzero_offset() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-file-offset"); let profile = profile(root.path()); let config = root.path().join("node.toml"); + let mut services = crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default() + .setup + .services; + services[0].listeners.rpc_port = Some(30101); let inputs = NodeDeploymentOverrides { - services: Some( - crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default() - .setup - .services, - ), + services: Some(services), ..Default::default() }; fs::write(&config, toml::to_string(&inputs).unwrap()).unwrap(); @@ -83,7 +83,7 @@ fn container_preflight_reuses_explicit_file_ports_with_a_nonzero_offset() { let accepted = crowdb_monitor::read_node_policy(&root.path().join("install/data")) .unwrap() .unwrap(); - assert_eq!(accepted.setup.services[0].listeners.rpc_port, Some(10101)); + assert_eq!(accepted.setup.services[0].listeners.rpc_port, Some(30102)); } #[test] diff --git a/container/crowdb-monitor/tests/node_service_dependencies_test.rs b/container/crowdb-monitor/tests/node_service_dependencies_test.rs index a8184d96e..174c92583 100644 --- a/container/crowdb-monitor/tests/node_service_dependencies_test.rs +++ b/container/crowdb-monitor/tests/node_service_dependencies_test.rs @@ -84,7 +84,7 @@ mod service_dependency; #[tokio::test] async fn healthy_diskdb_listener_must_publish_the_registered_disks_ownership_before_consumers_start() { - use crowdb_kv_client::ServiceRegistryClient; + use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::mgmt::node::setup::{NodeSetupChoices, NodeSetupRequest}; let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("diskdb-route-gate"); let request = NodeSetupRequest { @@ -96,8 +96,8 @@ async fn healthy_diskdb_listener_must_publish_the_registered_disks_ownership_bef let authority = node_setup::TestAuthority::start(root.path(), &request).await; assert_eq!(authority.registry.nodes[0].node_id, 2); authority.kv.refresh_topology().await.unwrap(); - let kv = std::sync::Arc::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), + let kv = std::sync::Arc::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), )); service_dependency::publish_online_disk(kv.clone()).await; let (address, server) = service_dependency::ready_listener().await; @@ -283,7 +283,7 @@ fn redirecting_rpc() -> (crowdb_rpc_ffi::RpcServer, Arc<AtomicUsize>) { (rpc, requests) } -async fn publish_catalog(kv: &crowdb_kv_client::CrowdbKvClient, port: i32, second: Option<u16>) { +async fn publish_catalog(kv: &crowdb_paxos_kv_client::PaxosKvClient, port: i32, second: Option<u16>) { let mut page = ChunkKvRangeCatalogPage { generation: 1, page_index: 0, diff --git a/container/crowdb-monitor/tests/node_setup_supplied_test.rs b/container/crowdb-monitor/tests/node_setup_supplied_test.rs index f7fe53ce1..09944f5c7 100644 --- a/container/crowdb-monitor/tests/node_setup_supplied_test.rs +++ b/container/crowdb-monitor/tests/node_setup_supplied_test.rs @@ -1,8 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeSetupForTests}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::{BindMapValue, NodeValue, RackValue}, diskdb::rpc::DiskGroupValue, @@ -109,7 +109,7 @@ impl TestSetup { async fn authority(&self) -> node_setup::TestAuthority { let authority = node_setup::TestAuthority::start(self.root.path(), &self.request).await; assert_eq!(authority.registry.nodes[0].physical_host_id, "owned-host"); - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new( authority.binding.management_seeds.clone(), ))); hardware @@ -249,7 +249,7 @@ async fn media_class_placement_and_disk_operations_survive_repeated_background_p disk_group_id: request.record.disk_group_id, disk_id: request.record.disk_id, }; - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new( authority.binding.management_seeds.clone(), ))); assert!( diff --git a/container/crowdb-monitor/tests/node_setup_test.rs b/container/crowdb-monitor/tests/node_setup_test.rs index b6dcc339b..91c6a0256 100644 --- a/container/crowdb-monitor/tests/node_setup_test.rs +++ b/container/crowdb-monitor/tests/node_setup_test.rs @@ -227,12 +227,12 @@ async fn assert_diskless_setup(kv_enabled: bool) { 0, 0, preparation_key, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None ) .await .unwrap(), - crowdb_kv_client::GetOutcome::NotFound, + crowdb_paxos_kv_client::GetOutcome::NotFound, )); assert_eq!(status.node_id, Some(2)); assert_eq!( @@ -300,7 +300,7 @@ async fn explicit_cleanup_retires_only_the_matching_setup_and_never_replays_it() #[tokio::test] async fn default_file_waits_for_metadata_then_reuses_one_durable_request_and_group() { - use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; + use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::common::{NodeValue, RackValue}; let directory = run_root::TestRunRoot::new("setup"); let root = directory.path(); @@ -313,7 +313,7 @@ async fn default_file_waits_for_metadata_then_reuses_one_durable_request_and_gro .accept_for_tests(root, intent.clone()) .unwrap(); let authority = node_setup::TestAuthority::start(root, &intent).await; - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new( authority.binding.management_seeds.clone(), ))); hardware @@ -370,7 +370,7 @@ async fn default_file_waits_for_metadata_then_reuses_one_durable_request_and_gro #[tokio::test] async fn a_conflicting_reserved_disk_group_binding_cannot_redirect_default_storage() { - use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; + use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::{NodeValue, RackValue}, diskdb::rpc::DiskGroupValue, @@ -388,7 +388,7 @@ async fn a_conflicting_reserved_disk_group_binding_cannot_redirect_default_stora let authority = node_setup::TestAuthority::start(root, &intent).await; node_setup_groups::publish_group(&authority, 1).await; node_setup_groups::publish_group(&authority, 8).await; - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new( authority.binding.management_seeds.clone(), ))); hardware diff --git a/container/crowdb-monitor/tests/profile_test.rs b/container/crowdb-monitor/tests/profile_test.rs index 1416a6995..eb8d78863 100644 --- a/container/crowdb-monitor/tests/profile_test.rs +++ b/container/crowdb-monitor/tests/profile_test.rs @@ -60,7 +60,7 @@ port = 14000 [[services]] id = "kv" -program = "/opt/crowdb/bin/crowdb-kv-server" +program = "/opt/crowdb/bin/crowdb-paxos-kv-server" args = [] dependencies = [] config_template = "/opt/crowdb/etc/templates/kv.toml" diff --git a/container/crowdb-monitor/tests/recovery_owners_test.rs b/container/crowdb-monitor/tests/recovery_owners_test.rs index 872532656..0c57c6c76 100644 --- a/container/crowdb-monitor/tests/recovery_owners_test.rs +++ b/container/crowdb-monitor/tests/recovery_owners_test.rs @@ -5,7 +5,7 @@ mod node_setup; use crowdb_console_shared::{ops::OpContext, ConsoleConfig}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::{ common::ReplicaValue, kv_membership::GroupMember, @@ -28,7 +28,7 @@ async fn installing_membership_authorizes_catchup_routes_before_replica_promotio let authority = node_setup::TestAuthority::start(root.path(), &request).await; authority.kv.refresh_topology().await.unwrap(); assert_eq!(authority.registry.nodes[0].node_id, 2); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( authority.binding.management_seeds.clone(), ))); let ctx = OpContext::with_shared_client_preserving_hint( diff --git a/container/crowdb-monitor/tests/single_node_profile_test.rs b/container/crowdb-monitor/tests/single_node_profile_test.rs index 812844cf8..4e8f343f3 100644 --- a/container/crowdb-monitor/tests/single_node_profile_test.rs +++ b/container/crowdb-monitor/tests/single_node_profile_test.rs @@ -37,7 +37,7 @@ fn single_node_preview_has_exact_topology_and_endpoints() { .collect::<BTreeMap<_, _>>(); assert_eq!( endpoints, - BTreeMap::from([("iceberg", 9092), ("s3", 9091), ("web", 9090)]) + BTreeMap::from([("iceberg", 8135), ("s3", 7135), ("web", 6135)]) ); let access_service = profile .services @@ -46,13 +46,13 @@ fn single_node_preview_has_exact_topology_and_endpoints() { .unwrap(); assert_eq!( access_service.env.get("CROWDB_ICEBERG_PUBLIC_URI"), - Some(&"http://localhost:9092".to_owned()) + Some(&"http://localhost:8135".to_owned()) ); - assert_eq!(access_service.probe.target, "http://127.0.0.1:9092/v1/config"); + assert_eq!(access_service.probe.target, "http://127.0.0.1:8135/v1/config"); assert_eq!(access_service.additional_probes.len(), 1); assert_eq!( access_service.additional_probes[0].target, - "http://127.0.0.1:9094/_crowdb/health/ready" + "http://127.0.0.1:25135/_crowdb/health/ready" ); assert_eq!( access_service.additional_probes[0].failure_threshold, @@ -60,16 +60,16 @@ fn single_node_preview_has_exact_topology_and_endpoints() { ); assert_eq!( access_service.env.get("CROWDB_MANAGEMENT_SEEDS"), - Some(&"http://127.0.0.1:10000".to_owned()) + Some(&"http://127.0.0.1:10135".to_owned()) ); assert_eq!(access_service.args[1], "/opt/crowdb/run/config/access.toml"); assert_eq!( access_service.fence_listeners, [ - "127.0.0.1:9092", - "127.0.0.1:9091", - "127.0.0.1:9093", - "127.0.0.1:9094" + "127.0.0.1:8135", + "127.0.0.1:7135", + "127.0.0.1:9135", + "127.0.0.1:25135" ] ); let access = std::fs::read_to_string( @@ -77,25 +77,25 @@ fn single_node_preview_has_exact_topology_and_endpoints() { ) .unwrap(); let access: toml::Value = toml::from_str(&access).unwrap(); - assert_eq!(access["iceberg"]["listen"].as_str(), Some("0.0.0.0:9092")); - assert_eq!(access["s3"]["listen"].as_str(), Some("0.0.0.0:9091")); - assert_eq!(access["dataset"]["listen"].as_str(), Some("0.0.0.0:9093")); + assert_eq!(access["iceberg"]["listen"].as_str(), Some("0.0.0.0:8135")); + assert_eq!(access["s3"]["listen"].as_str(), Some("0.0.0.0:7135")); + assert_eq!(access["dataset"]["listen"].as_str(), Some("0.0.0.0:9135")); assert_eq!( access_service.env.get("CROWDB_DATASET_PUBLIC_URI"), - Some(&"http://localhost:9093".to_owned()) + Some(&"http://localhost:9135".to_owned()) ); let web = profile .services .iter() .find(|service| service.id == "web") .unwrap(); - assert_eq!(web.probe.target, "http://127.0.0.1:9090/healthz"); + assert_eq!(web.probe.target, "http://127.0.0.1:6135/healthz"); let template = std::fs::read_to_string( Path::new(env!("CARGO_MANIFEST_DIR")).join("../single-node-container/templates/crowdb-web.toml"), ) .unwrap(); let config: toml::Value = toml::from_str(&template).unwrap(); - assert_eq!(config["port"].as_integer(), Some(9090)); + assert_eq!(config["port"].as_integer(), Some(6135)); } #[test] diff --git a/container/oci-image/Dockerfile b/container/oci-image/Dockerfile index 47a2e030a..71493b5f7 100644 --- a/container/oci-image/Dockerfile +++ b/container/oci-image/Dockerfile @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. FROM ubuntu:24.04@sha256:496754492fb28b4d3049432f2ca787449331e23fb14f0dd3fffea86bf5a93eb4 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates openssh-server util-linux curl tini libibverbs1 librdmacm1 ibverbs-providers ibverbs-utils && rm -rf /var/lib/apt/lists/* \ && groupadd --system --gid 10001 crowdb \ @@ -45,6 +46,6 @@ ENV PATH="/opt/crowdb/bin:${PATH}" \ CROWDB_RUNTIME_ROOT="/opt/crowdb/run" USER root VOLUME ["/opt/crowdb/data"] -EXPOSE 9090 16000 17000 18000 9095 2222 +EXPOSE 10135 11135 13135 14135 16135 17135 18135 6135 20135 21135 7135 8135 9135 25135 26135 2222 STOPSIGNAL SIGTERM ENTRYPOINT ["/usr/bin/tini", "--", "/opt/crowdb/bin/entrypoint"] diff --git a/container/oci-image/Dockerfile.ports.in b/container/oci-image/Dockerfile.ports.in new file mode 100644 index 000000000..a52b79341 --- /dev/null +++ b/container/oci-image/Dockerfile.ports.in @@ -0,0 +1,50 @@ +FROM ubuntu:24.04@sha256:496754492fb28b4d3049432f2ca787449331e23fb14f0dd3fffea86bf5a93eb4 +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates openssh-server util-linux curl tini libibverbs1 librdmacm1 ibverbs-providers ibverbs-utils && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 10001 crowdb \ + && useradd --system --uid 10001 --gid 10001 --home-dir /opt/crowdb --shell /bin/sh crowdb \ + && echo 'crowdb:crowdb' | chpasswd \ + && mkdir -p /run/sshd +ARG SOURCE_REVISION +ARG SOURCE_TREE +ARG RUNTIME_SHA256 +ARG PREVIEW_VERSION +ARG DEBUG_INFO=stripped +RUN --mount=type=bind,source=.,target=/staged,ro \ + test -n "$SOURCE_REVISION" && test -n "$PREVIEW_VERSION" \ + && test "$(cat /staged/SOURCE_REVISION)" = "$SOURCE_REVISION" \ + && { test -z "$SOURCE_TREE" || test "$(cat /staged/SOURCE_TREE)" = "$SOURCE_TREE"; } \ + && test "$(cat /staged/VERSION)" = "$PREVIEW_VERSION" \ + && mkdir -p /opt/crowdb/etc /opt/crowdb/data /opt/crowdb/run \ + && cp -a /staged/bin /staged/lib /staged/ui /opt/crowdb/ \ + && if test -d /staged/etc; then cp -a /staged/etc /opt/crowdb/; fi \ + && cp /staged/profile.toml /opt/crowdb/etc/profile.toml \ + && cp /staged/entrypoint.sh /opt/crowdb/bin/entrypoint \ + && chown -R crowdb:crowdb /opt/crowdb/data /opt/crowdb/run \ + && chmod 0755 /opt/crowdb/bin/entrypoint \ + && for binary in /opt/crowdb/bin/crowdb-* /opt/crowdb/lib/* /opt/crowdb/lib/libibverbs/*.so; do \ + test -f "$binary" || continue; \ + LD_LIBRARY_PATH=/opt/crowdb/lib ldd -r "$binary" > /tmp/dependencies.txt 2>&1 || { cat /tmp/dependencies.txt; exit 1; }; \ + if grep -Eq 'not found|undefined symbol' /tmp/dependencies.txt; then cat /tmp/dependencies.txt; exit 1; fi; \ + done && rm /tmp/dependencies.txt + +LABEL org.opencontainers.image.title="CROWDB Node" \ + org.opencontainers.image.description="CROWDB node with discovery, management UI and storage services." \ + org.opencontainers.image.url="https://crowdb.dev/" \ + org.opencontainers.image.source="https://github.com/buzzcrow/crowdb" \ + org.opencontainers.image.authors="Gian <crow.db@outlook.com>" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.revision="$SOURCE_REVISION" \ + org.crowdb.source.tree="$SOURCE_TREE" \ + org.opencontainers.image.version="$PREVIEW_VERSION" \ + org.crowdb.runtime.sha256="$RUNTIME_SHA256" \ + org.crowdb.debug-info="$DEBUG_INFO" \ + org.crowdb.startup.default="local" +ENV PATH="/opt/crowdb/bin:${PATH}" \ + LD_LIBRARY_PATH="/opt/crowdb/lib" \ + RDMAV_DRIVERS_DIR="/opt/crowdb/lib/libibverbs" \ + CROWDB_RUNTIME_ROOT="/opt/crowdb/run" +USER root +VOLUME ["/opt/crowdb/data"] +EXPOSE @PAXOS_KV_SERVER_MGMT_BASE@ @PAXOS_KV_SERVER_RPC_BASE@ @DISKDB_HTTP_BASE@ @DISKDB_RPC_BASE@ @CHUNKDB_HTTP_BASE@ @CHUNKDB_RPC_BASE@ @DISKIO_RPC_BASE@ @WEB_BASE@ @CHUNK_KV_HTTP_BASE@ @CHUNK_KV_RPC_BASE@ @ACCESS_SERVER_HTTP_BASE@ @ACCESS_SERVER_ICEBERG_HTTP_BASE@ @ACCESS_SERVER_DATASET_HTTP_BASE@ @ACCESS_SERVER_HEALTH_HTTP_BASE@ @MONITOR_HTTP_BASE@ @SSH_PORT@ +STOPSIGNAL SIGTERM +ENTRYPOINT ["/usr/bin/tini", "--", "/opt/crowdb/bin/entrypoint"] diff --git a/container/oci-image/README.md b/container/oci-image/README.md index 9d634992d..b174b7765 100644 --- a/container/oci-image/README.md +++ b/container/oci-image/README.md @@ -188,7 +188,7 @@ pixi run oci-clean --cluster demo-three Use the built `crowdb-cli` executable on PATH, or substitute its absolute path after `pixi run`. Start defaults to cluster `local`, one node, 1 CPU, 1536 MiB, -and console port 9090. It accepts `--image`, `--cpus`, `--memory-mib` and +and console port 6135. It accepts `--image`, `--cpus`, `--memory-mib` and `--port-base` before sealing the first launch inputs. Subsequent starts verify those inputs and resume existing state. `--wait` bounds readiness observation; an unmet check returns 1 and leaves accepted work resumable. @@ -214,7 +214,7 @@ pixi run oci-clean --cluster production The existing data directory must be writable by UID 10001. The node policy must select production mode, the same cluster and initial voter count, supplied media and reachable public protocol URIs. `--nodes 3` selects the three-voter -formation policy; run one node on each host. Host networking uses port 9090 and +formation policy; run one node on each host. Host networking uses port 6135 and rejects `--port-base`. The management address must belong to the selected local interface. Disk files use `--disk-file`; files and devices are explicit existing inputs and are never created by the CLI. @@ -274,11 +274,11 @@ A direct single-container cluster needs a volume and four published ports: ```bash pixi run docker run -d --name crowdb-one --cpus 1 --memory 1536m \ - -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9091 \ - -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9092 \ - -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9093 \ - -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:16000 \ - -p 127.0.0.1:9092:17000 -p 127.0.0.1:9093:18000 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:7135 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:8135 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9135 \ + -p 127.0.0.1:6135:6135 -p 127.0.0.1:7135:7135 \ + -p 127.0.0.1:8135:8135 -p 127.0.0.1:9135:9135 \ --mount type=volume,source=crowdb-one-data,target=/opt/crowdb/data \ crowdb-node:dev ``` @@ -290,27 +290,27 @@ in any order. No node addresses or membership list are supplied: pixi run docker network create crowdb-demo pixi run docker run -d --name crowdb-demo-1 --network crowdb-demo --cpus 1 --memory 1536m \ -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ - -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9091 \ - -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9092 \ - -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9093 \ - -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:16000 \ - -p 127.0.0.1:9092:17000 -p 127.0.0.1:9093:18000 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:7135 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:8135 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9135 \ + -p 127.0.0.1:6135:6135 -p 127.0.0.1:7135:7135 \ + -p 127.0.0.1:8135:8135 -p 127.0.0.1:9135:9135 \ --mount type=volume,source=crowdb-demo-1-data,target=/opt/crowdb/data crowdb-node:dev pixi run docker run -d --name crowdb-demo-2 --network crowdb-demo --cpus 1 --memory 1536m \ -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ - -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9101 \ - -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9102 \ - -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9103 \ - -p 127.0.0.1:9100:9090 -p 127.0.0.1:9101:16000 \ - -p 127.0.0.1:9102:17000 -p 127.0.0.1:9103:18000 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:7136 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:8136 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9136 \ + -p 127.0.0.1:6136:6135 -p 127.0.0.1:7136:7135 \ + -p 127.0.0.1:8136:8135 -p 127.0.0.1:9136:9135 \ --mount type=volume,source=crowdb-demo-2-data,target=/opt/crowdb/data crowdb-node:dev pixi run docker run -d --name crowdb-demo-3 --network crowdb-demo --cpus 1 --memory 1536m \ -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ - -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9111 \ - -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9112 \ - -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9113 \ - -p 127.0.0.1:9110:9090 -p 127.0.0.1:9111:16000 \ - -p 127.0.0.1:9112:17000 -p 127.0.0.1:9113:18000 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:7137 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:8137 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9137 \ + -p 127.0.0.1:6137:6135 -p 127.0.0.1:7137:7135 \ + -p 127.0.0.1:8137:8135 -p 127.0.0.1:9137:9135 \ --mount type=volume,source=crowdb-demo-3-data,target=/opt/crowdb/data crowdb-node:dev ``` @@ -376,8 +376,8 @@ pixi run --manifest-path container/oci-image/pixi.toml -e runtime run --runtime --password-file /private/crowdb-password ``` -Host networking uses port 9090 directly; no `-p` mapping is needed. Fixed listener -ports mean one node per host. Use `--seed http://host:9095` when multicast is unavailable. +Host networking uses port 6135 directly; no `-p` mapping is needed. Fixed listener +ports mean one node per host. Use `--seed http://host:26135` when multicast is unavailable. Explicit `--device /dev/...` permits individual block devices; the host must grant UID/GID 10001 access. The launcher checks device type and effective read/write permissions, actual cgroup v2 CPU/memory limits and the host network namespace. @@ -527,3 +527,6 @@ checked-in Pixi source recipes supply missing tools from SHA256-pinned upstream releases. All six upstream projects use Apache-2.0 licensing; recipes retain upstream sources/checksums and license metadata. The dedicated Linux workspace leaves the repository's macOS dependency solve unchanged. + +Default listener ports are defined in [protocol ports.rs](../../lib/crowdb-protocol/src/port/ports.rs). +Generated packaging defaults are maintained with [tools/ports](../../tools/ports/README.md). diff --git a/container/oci-image/collect-libs.sh b/container/oci-image/collect-libs.sh index 5b454781e..76dec9e47 100644 --- a/container/oci-image/collect-libs.sh +++ b/container/oci-image/collect-libs.sh @@ -24,7 +24,7 @@ if [[ -d "$provider_root" && -d "$provider_config" ]]; then fi for binary in \ - crowdb-monitor crowdb-kv-server crowdb-diskdb crowdb-diskio \ + crowdb-monitor crowdb-paxos-kv-server crowdb-diskdb crowdb-diskio \ crowdb-chunkdb crowdb-chunk-kv-server crowdb-access-server \ crowdb-web; do if [[ "$binary" == crowdb-diskio ]]; then @@ -66,7 +66,7 @@ for binary in "$output"/bin/* "$output"/lib/libibverbs/*.so; do done < "$output/dependencies.txt" patchelf --set-rpath '/opt/crowdb/lib' "$binary" done -if [[ ! -f "$output/lib/libcrowdb_kv_client.so" ]]; then +if [[ ! -f "$output/lib/libcrowdb_paxos_kv_client.so" ]]; then echo 'DiskIO FFI library was not collected' >&2 exit 1 fi diff --git a/container/oci-image/compose-three.yaml b/container/oci-image/compose-three.yaml index 01af56fff..aadaf6e0f 100644 --- a/container/oci-image/compose-three.yaml +++ b/container/oci-image/compose-three.yaml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. # Copyright 2026-present Gian <crow.db@outlook.com> # Licensed under the Apache License, Version 2.0. name: crowdb-${CROWDB_CLUSTER:-local-three} @@ -12,14 +13,14 @@ services: environment: CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} CROWDB_INITIAL_VOTERS: "3" - CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_S3_PORT:-9091} - CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-9092} - CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093} + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_S3_PORT:-7135} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-8135} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9135} ports: - - "127.0.0.1:${CROWDB_NODE1_UI_PORT:-9090}:9090" - - "127.0.0.1:${CROWDB_NODE1_S3_PORT:-9091}:${CROWDB_S3_CONTAINER_PORT:-16000}" - - "127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-9092}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" - - "127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" + - "127.0.0.1:${CROWDB_NODE1_UI_PORT:-6135}:6135" + - "127.0.0.1:${CROWDB_NODE1_S3_PORT:-7135}:${CROWDB_S3_CONTAINER_PORT:-7135}" + - "127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-8135}:${CROWDB_ICEBERG_CONTAINER_PORT:-8135}" + - "127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9135}:${CROWDB_DATASET_CONTAINER_PORT:-9135}" volumes: - node1-data:/opt/crowdb/data node2: @@ -27,14 +28,14 @@ services: environment: CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} CROWDB_INITIAL_VOTERS: "3" - CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_S3_PORT:-9101} - CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-9102} - CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103} + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_S3_PORT:-7136} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-8136} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9136} ports: - - "127.0.0.1:${CROWDB_NODE2_UI_PORT:-9100}:9090" - - "127.0.0.1:${CROWDB_NODE2_S3_PORT:-9101}:${CROWDB_S3_CONTAINER_PORT:-16000}" - - "127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-9102}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" - - "127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" + - "127.0.0.1:${CROWDB_NODE2_UI_PORT:-6136}:6135" + - "127.0.0.1:${CROWDB_NODE2_S3_PORT:-7136}:${CROWDB_S3_CONTAINER_PORT:-7135}" + - "127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-8136}:${CROWDB_ICEBERG_CONTAINER_PORT:-8135}" + - "127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9136}:${CROWDB_DATASET_CONTAINER_PORT:-9135}" volumes: - node2-data:/opt/crowdb/data node3: @@ -42,14 +43,14 @@ services: environment: CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} CROWDB_INITIAL_VOTERS: "3" - CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_S3_PORT:-9111} - CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-9112} - CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113} + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_S3_PORT:-7137} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-8137} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9137} ports: - - "127.0.0.1:${CROWDB_NODE3_UI_PORT:-9110}:9090" - - "127.0.0.1:${CROWDB_NODE3_S3_PORT:-9111}:${CROWDB_S3_CONTAINER_PORT:-16000}" - - "127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-9112}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" - - "127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" + - "127.0.0.1:${CROWDB_NODE3_UI_PORT:-6137}:6135" + - "127.0.0.1:${CROWDB_NODE3_S3_PORT:-7137}:${CROWDB_S3_CONTAINER_PORT:-7135}" + - "127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-8137}:${CROWDB_ICEBERG_CONTAINER_PORT:-8135}" + - "127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9137}:${CROWDB_DATASET_CONTAINER_PORT:-9135}" volumes: - node3-data:/opt/crowdb/data volumes: diff --git a/container/oci-image/compose-three.yaml.ports.in b/container/oci-image/compose-three.yaml.ports.in new file mode 100644 index 000000000..d8027c831 --- /dev/null +++ b/container/oci-image/compose-three.yaml.ports.in @@ -0,0 +1,58 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +name: crowdb-${CROWDB_CLUSTER:-local-three} +x-node: &node + image: ${CROWDB_IMAGE:-crowdb-node:dev} + pull_policy: never + cpus: 1 + mem_limit: 1536m +services: + node1: + <<: *node + environment: + CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} + CROWDB_INITIAL_VOTERS: "3" + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_S3_PORT:-@ACCESS_SERVER_HTTP_BASE@} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE@} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE@} + ports: + - "127.0.0.1:${CROWDB_NODE1_UI_PORT:-@WEB_BASE@}:@WEB_BASE@" + - "127.0.0.1:${CROWDB_NODE1_S3_PORT:-@ACCESS_SERVER_HTTP_BASE@}:${CROWDB_S3_CONTAINER_PORT:-@ACCESS_SERVER_HTTP_BASE@}" + - "127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE@}:${CROWDB_ICEBERG_CONTAINER_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE@}" + - "127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE@}:${CROWDB_DATASET_CONTAINER_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE@}" + volumes: + - node1-data:/opt/crowdb/data + node2: + <<: *node + environment: + CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} + CROWDB_INITIAL_VOTERS: "3" + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_S3_PORT:-@ACCESS_SERVER_HTTP_BASE+1@} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE+1@} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE+1@} + ports: + - "127.0.0.1:${CROWDB_NODE2_UI_PORT:-@WEB_BASE+1@}:@WEB_BASE@" + - "127.0.0.1:${CROWDB_NODE2_S3_PORT:-@ACCESS_SERVER_HTTP_BASE+1@}:${CROWDB_S3_CONTAINER_PORT:-@ACCESS_SERVER_HTTP_BASE@}" + - "127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE+1@}:${CROWDB_ICEBERG_CONTAINER_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE@}" + - "127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE+1@}:${CROWDB_DATASET_CONTAINER_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE@}" + volumes: + - node2-data:/opt/crowdb/data + node3: + <<: *node + environment: + CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} + CROWDB_INITIAL_VOTERS: "3" + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_S3_PORT:-@ACCESS_SERVER_HTTP_BASE+2@} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE+2@} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE+2@} + ports: + - "127.0.0.1:${CROWDB_NODE3_UI_PORT:-@WEB_BASE+2@}:@WEB_BASE@" + - "127.0.0.1:${CROWDB_NODE3_S3_PORT:-@ACCESS_SERVER_HTTP_BASE+2@}:${CROWDB_S3_CONTAINER_PORT:-@ACCESS_SERVER_HTTP_BASE@}" + - "127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE+2@}:${CROWDB_ICEBERG_CONTAINER_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE@}" + - "127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE+2@}:${CROWDB_DATASET_CONTAINER_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE@}" + volumes: + - node3-data:/opt/crowdb/data +volumes: + node1-data: + node2-data: + node3-data: diff --git a/container/oci-image/compose.yaml b/container/oci-image/compose.yaml index 96e5e608f..1b3848577 100644 --- a/container/oci-image/compose.yaml +++ b/container/oci-image/compose.yaml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. # Copyright 2026-present Gian <crow.db@outlook.com> # Licensed under the Apache License, Version 2.0. name: crowdb-${CROWDB_CLUSTER:-local} @@ -10,14 +11,14 @@ services: environment: CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local} CROWDB_INITIAL_VOTERS: "1" - CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_S3_PORT:-9091} - CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_ICEBERG_PORT:-9092} - CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_DATASET_PORT:-9093} + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_S3_PORT:-7135} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_ICEBERG_PORT:-8135} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_DATASET_PORT:-9135} ports: - - "127.0.0.1:${CROWDB_UI_PORT:-9090}:9090" - - "127.0.0.1:${CROWDB_S3_PORT:-9091}:${CROWDB_S3_CONTAINER_PORT:-16000}" - - "127.0.0.1:${CROWDB_ICEBERG_PORT:-9092}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" - - "127.0.0.1:${CROWDB_DATASET_PORT:-9093}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" + - "127.0.0.1:${CROWDB_UI_PORT:-6135}:6135" + - "127.0.0.1:${CROWDB_S3_PORT:-7135}:${CROWDB_S3_CONTAINER_PORT:-7135}" + - "127.0.0.1:${CROWDB_ICEBERG_PORT:-8135}:${CROWDB_ICEBERG_CONTAINER_PORT:-8135}" + - "127.0.0.1:${CROWDB_DATASET_PORT:-9135}:${CROWDB_DATASET_CONTAINER_PORT:-9135}" volumes: - node-data:/opt/crowdb/data volumes: diff --git a/container/oci-image/compose.yaml.ports.in b/container/oci-image/compose.yaml.ports.in new file mode 100644 index 000000000..ecb30283f --- /dev/null +++ b/container/oci-image/compose.yaml.ports.in @@ -0,0 +1,24 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +name: crowdb-${CROWDB_CLUSTER:-local} +services: + node: + image: ${CROWDB_IMAGE:-crowdb-node:dev} + pull_policy: never + cpus: 1 + mem_limit: 1536m + environment: + CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local} + CROWDB_INITIAL_VOTERS: "1" + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_S3_PORT:-@ACCESS_SERVER_HTTP_BASE@} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_ICEBERG_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE@} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_DATASET_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE@} + ports: + - "127.0.0.1:${CROWDB_UI_PORT:-@WEB_BASE@}:@WEB_BASE@" + - "127.0.0.1:${CROWDB_S3_PORT:-@ACCESS_SERVER_HTTP_BASE@}:${CROWDB_S3_CONTAINER_PORT:-@ACCESS_SERVER_HTTP_BASE@}" + - "127.0.0.1:${CROWDB_ICEBERG_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE@}:${CROWDB_ICEBERG_CONTAINER_PORT:-@ACCESS_SERVER_ICEBERG_HTTP_BASE@}" + - "127.0.0.1:${CROWDB_DATASET_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE@}:${CROWDB_DATASET_CONTAINER_PORT:-@ACCESS_SERVER_DATASET_HTTP_BASE@}" + volumes: + - node-data:/opt/crowdb/data +volumes: + node-data: diff --git a/container/oci-image/docker_local_launch.py b/container/oci-image/docker_local_launch.py index 99222ace3..ee0feb2f7 100644 --- a/container/oci-image/docker_local_launch.py +++ b/container/oci-image/docker_local_launch.py @@ -6,15 +6,23 @@ import math import uuid +from ports import WEB_BASE, ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE, ACCESS_SERVER_DATASET_HTTP_BASE from docker_local_state import CLUSTER, OWNER, check_daemon, load, require_owned, seal CONTAINER_PORTS = (9090, 16000, 17000, 18000) def container_ports(record): - return (9090, 9091, 9092, 9093) if record["version"] == 1 else CONTAINER_PORTS + if record["version"] == 1: + return (9090, 9091, 9092, 9093) + if record["version"] == 2: + return CONTAINER_PORTS + return tuple(port for port, _ in PUBLIC_PORTS) +PUBLIC_PORTS = ((WEB_BASE, None), (ACCESS_SERVER_HTTP_BASE, "S3"), + (ACCESS_SERVER_ICEBERG_HTTP_BASE, "ICEBERG"), (ACCESS_SERVER_DATASET_HTTP_BASE, "DATASET")) + def arguments(record): return ["--label", f"{CLUSTER}={record['cluster']}", "--label", f"{OWNER}={record['owner']}"] @@ -49,11 +57,11 @@ def accepted(docker, args): raise ValueError("local node image must be Linux amd64") if metadata.get("Config", {}).get("Labels", {}).get("org.opencontainers.image.source") != "https://github.com/buzzcrow/crowdb": raise ValueError("image is not a CROWDB node image") - record = {"version": 2, "cluster": args.cluster, "owner": str(uuid.uuid4()), "runtime": docker.identity(), - "nodes": args.nodes if args.nodes is not None else 1, "port_base": args.port_base if args.port_base is not None else 9090, "image": image, + record = {"version": 3, "cluster": args.cluster, "owner": str(uuid.uuid4()), "runtime": docker.identity(), + "nodes": args.nodes if args.nodes is not None else 1, "port_base": args.port_base if args.port_base is not None else WEB_BASE, "image": image, "image_id": metadata["Id"], "cpus": args.cpus if args.cpus is not None else 1, "memory_mib": args.memory_mib if args.memory_mib is not None else 1536} - if record["nodes"] not in (1, 3) or not 1 <= record["port_base"] <= 65512: + if record["nodes"] not in (1, 3) or not 1 <= record["port_base"] <= 65535 - (ACCESS_SERVER_DATASET_HTTP_BASE - WEB_BASE) - 2: raise ValueError("local cluster requires one or three nodes and a valid port range") if not math.isfinite(record["cpus"]) or record["cpus"] <= 0 or record["memory_mib"] < 512: raise ValueError("CPU must be positive and memory at least 512 MiB") @@ -64,12 +72,13 @@ def accepted(docker, args): def verify_container(item, record, index, volume, network): require_owned(item, record, "container") host = item["HostConfig"] - base = record["port_base"] + (index - 1) * 10 - ports = {f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(base+offset)}] + base = record["port_base"] + (index - 1) * (10 if record["version"] < 3 else 1) + ports = {f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(base + (offset if record["version"] < 3 else port - WEB_BASE))}] for offset, port in enumerate(container_ports(record))} expected_environment = {"CROWDB_CLUSTER": record["cluster"], "CROWDB_INITIAL_VOTERS": str(record["nodes"])} for offset, protocol in enumerate((None, "S3", "ICEBERG", "DATASET")): if protocol: + offset = offset if record["version"] < 3 else container_ports(record)[offset] - WEB_BASE expected_environment[f"CROWDB_{protocol}_PUBLIC_URI"] = f"http://127.0.0.1:{base+offset}" environment = dict(value.split("=", 1) for value in item["Config"].get("Env", []) if "=" in value) mounts = [mount for mount in item["Mounts"] if mount["Destination"] == "/opt/crowdb/data"] @@ -102,13 +111,15 @@ def run(docker, record): check_daemon(docker, record) docker.call("start", item["Id"]) continue - base = record["port_base"] + (index - 1) * 10 + base = record["port_base"] + (index - 1) * (10 if record["version"] < 3 else 1) command = ["run", "-d", "--pull", "never", "--name", name, *arguments(record), "--network", network, "--cpus", record["cpus"], "--memory", f"{record['memory_mib']}m", "--mount", f"type=volume,source={volume},target=/opt/crowdb/data", "-e", "CROWDB_CLUSTER=" + record["cluster"], "-e", f"CROWDB_INITIAL_VOTERS={record['nodes']}"] for offset, protocol in enumerate((None, "S3", "ICEBERG", "DATASET")): - command += ["-p", f"127.0.0.1:{base+offset}:{container_ports(record)[offset]}"] + internal = container_ports(record)[offset] + offset = offset if record["version"] < 3 else internal - WEB_BASE + command += ["-p", f"127.0.0.1:{base+offset}:{internal}"] if protocol: command += ["-e", f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{base+offset}"] check_daemon(docker, record) diff --git a/container/oci-image/docker_local_state.py b/container/oci-image/docker_local_state.py index 960492c6f..171a8fcc7 100644 --- a/container/oci-image/docker_local_state.py +++ b/container/oci-image/docker_local_state.py @@ -11,6 +11,8 @@ import stat import uuid +from ports import WEB_BASE, ACCESS_SERVER_DATASET_HTTP_BASE + CLUSTER = "org.crowdb.cluster" OWNER = "org.crowdb.cluster.owner" ROOT = Path(__file__).resolve().parents[2] @@ -71,7 +73,7 @@ def load(cluster, directory=RECORDS): if not stat.S_ISREG(metadata.st_mode) or metadata.st_mode & 0o077 or metadata.st_size > 1024 * 1024: raise ValueError("cluster receipt must be a bounded private regular file") record = json.loads(path.read_text()) - if record.get("version") not in (1, 2) or record.get("cluster") != cluster: + if record.get("version") not in (1, 2, 3) or record.get("cluster") != cluster: raise ValueError("invalid cluster receipt") required = {"version", "cluster", "owner", "runtime", "nodes", "port_base", "image", "image_id", "cpus", "memory_mib"} if set(record) != required or not isinstance(record["runtime"], dict): @@ -79,10 +81,11 @@ def load(cluster, directory=RECORDS): owner = uuid.UUID(record["owner"]) if owner.int == 0 or str(owner) != record["owner"]: raise ValueError("invalid cluster receipt owner") + spread = (ACCESS_SERVER_DATASET_HTTP_BASE - WEB_BASE) + 2 if record["version"] == 3 else 23 if (set(record["runtime"]) != {"daemon", "context", "endpoint"} or not all(isinstance(value, str) and value for value in record["runtime"].values()) or record["nodes"] not in (1, 3) or not isinstance(record["port_base"], int) - or not 1 <= record["port_base"] <= 65512 + or not 1 <= record["port_base"] <= 65535 - spread or not isinstance(record["cpus"], (float, int)) or not math.isfinite(record["cpus"]) or record["cpus"] <= 0 or not isinstance(record["memory_mib"], int) or record["memory_mib"] < 512 or not all(isinstance(record[key], str) and record[key] for key in ("image", "image_id"))): diff --git a/container/oci-image/host_inputs.py b/container/oci-image/host_inputs.py index 117ce7bb4..1b020757a 100644 --- a/container/oci-image/host_inputs.py +++ b/container/oci-image/host_inputs.py @@ -2,6 +2,8 @@ # Licensed under the Apache License, Version 2.0. """Reuse the ordinary host launch validator before accepting runtime ownership.""" +from ports import WEB_BASE + import hashlib import ipaddress import json @@ -81,7 +83,7 @@ def management(options): if str(address) not in {value['local'] for interface in interfaces for value in interface.get('addr_info', [])}: raise ValueError('management address is not assigned to the selected interface') - return 'http://' + (f'[{address}]' if address.version == 6 else str(address)) + ':9090' + return 'http://' + (f'[{address}]' if address.version == 6 else str(address)) + f':{WEB_BASE}' def validate(record, running): diff --git a/container/oci-image/node-profile.toml b/container/oci-image/node-profile.toml index 76ac257b4..ee925762e 100644 --- a/container/oci-image/node-profile.toml +++ b/container/oci-image/node-profile.toml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. # Copyright 2026-present Gian <crow.db@outlook.com> # Licensed under the Apache License, Version 2.0. # Programs and supervision defaults only. Accepted node policy and committed @@ -26,31 +27,31 @@ mirror_warnings_to_stderr = true [[public_endpoints]] id = "s3" bind = "0.0.0.0" -port = 9091 +port = 7135 [[public_endpoints]] id = "iceberg" bind = "0.0.0.0" -port = 9092 +port = 8135 [[public_endpoints]] id = "web" bind = "0.0.0.0" -port = 9090 +port = 6135 [[public_endpoints]] id = "dataset" bind = "0.0.0.0" -port = 9093 +port = 9135 [[services]] id = "kv" -program = "/opt/crowdb/bin/crowdb-kv-server" +program = "/opt/crowdb/bin/crowdb-paxos-kv-server" args = [] dependencies = [] [services.probe] kind = "http" -target = "http://127.0.0.1:10000/health" +target = "http://127.0.0.1:10135/health" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -65,7 +66,7 @@ args = [] dependencies = ["kv"] [services.probe] kind = "http" -target = "http://127.0.0.1:7101/ready" +target = "http://127.0.0.1:13135/ready" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -80,7 +81,7 @@ args = [] dependencies = ["kv", "diskdb"] [services.probe] kind = "rpc-ping" -target = "127.0.0.1:7200" +target = "127.0.0.1:18135" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -95,7 +96,7 @@ args = [] dependencies = ["kv", "diskdb", "diskio"] [services.probe] kind = "http" -target = "http://127.0.0.1:7300/ready" +target = "http://127.0.0.1:16135/ready" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -110,7 +111,7 @@ args = [] dependencies = ["kv", "chunkdb", "diskio"] [services.probe] kind = "http" -target = "http://127.0.0.1:7400/ready" +target = "http://127.0.0.1:20135/ready" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -125,7 +126,7 @@ args = [] dependencies = ["kv", "chunk-kv", "chunkdb", "diskio"] [services.probe] kind = "http" -target = "http://127.0.0.1:9094/_crowdb/health/ready" +target = "http://127.0.0.1:25135/_crowdb/health/ready" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -140,7 +141,7 @@ args = [] dependencies = [] [services.probe] kind = "http" -target = "http://127.0.0.1:9090/healthz" +target = "http://127.0.0.1:6135/healthz" timeout_ms = 1000 failure_threshold = 5 [services.restart] diff --git a/container/oci-image/node-profile.toml.ports.in b/container/oci-image/node-profile.toml.ports.in new file mode 100644 index 000000000..dbe23e92b --- /dev/null +++ b/container/oci-image/node-profile.toml.ports.in @@ -0,0 +1,149 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# Programs and supervision defaults only. Accepted node policy and committed +# service intents supply membership, media, configuration and actual listeners. + +version = 1 +name = "node-container" +display_name = "CROWDB Node" +placement_mode = "node-policy" +s3_tenant = "default" +iceberg_catalog = "default" + +[paths] +install_root = "/opt/crowdb" +bin_root = "/opt/crowdb/bin" +template_root = "/opt/crowdb/etc/templates" +data_root = "/opt/crowdb/data" +run_root = "/opt/crowdb/run" +log_root = "/opt/crowdb/data/log" + +[logs] +max_file_bytes = 31457280 +max_files = 5 +mirror_warnings_to_stderr = true + +[[public_endpoints]] +id = "s3" +bind = "0.0.0.0" +port = @ACCESS_SERVER_HTTP_BASE@ + +[[public_endpoints]] +id = "iceberg" +bind = "0.0.0.0" +port = @ACCESS_SERVER_ICEBERG_HTTP_BASE@ + +[[public_endpoints]] +id = "web" +bind = "0.0.0.0" +port = @WEB_BASE@ + +[[public_endpoints]] +id = "dataset" +bind = "0.0.0.0" +port = @ACCESS_SERVER_DATASET_HTTP_BASE@ + +[[services]] +id = "kv" +program = "/opt/crowdb/bin/crowdb-paxos-kv-server" +args = [] +dependencies = [] +[services.probe] +kind = "http" +target = "http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@/health" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "diskdb" +program = "/opt/crowdb/bin/crowdb-diskdb" +args = [] +dependencies = ["kv"] +[services.probe] +kind = "http" +target = "http://127.0.0.1:@DISKDB_HTTP_BASE@/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "diskio" +program = "/opt/crowdb/bin/crowdb-diskio" +args = [] +dependencies = ["kv", "diskdb"] +[services.probe] +kind = "rpc-ping" +target = "127.0.0.1:@DISKIO_RPC_BASE@" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "chunkdb" +program = "/opt/crowdb/bin/crowdb-chunkdb" +args = [] +dependencies = ["kv", "diskdb", "diskio"] +[services.probe] +kind = "http" +target = "http://127.0.0.1:@CHUNKDB_HTTP_BASE@/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "chunk-kv" +program = "/opt/crowdb/bin/crowdb-chunk-kv-server" +args = [] +dependencies = ["kv", "chunkdb", "diskio"] +[services.probe] +kind = "http" +target = "http://127.0.0.1:@CHUNK_KV_HTTP_BASE@/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "access" +program = "/opt/crowdb/bin/crowdb-access-server" +args = [] +dependencies = ["kv", "chunk-kv", "chunkdb", "diskio"] +[services.probe] +kind = "http" +target = "http://127.0.0.1:@ACCESS_SERVER_HEALTH_HTTP_BASE@/_crowdb/health/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "web" +program = "/opt/crowdb/bin/crowdb-web" +args = [] +dependencies = [] +[services.probe] +kind = "http" +target = "http://127.0.0.1:@WEB_BASE@/healthz" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 diff --git a/container/oci-image/paxos-kv-cluster.py b/container/oci-image/paxos-kv-cluster.py new file mode 100644 index 000000000..92a0a358b --- /dev/null +++ b/container/oci-image/paxos-kv-cluster.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +"""Explicit standalone Paxos-KV container deployment; never called by bench.""" +from ports import PAXOS_KV_SERVER_MGMT_BASE, PAXOS_KV_SERVER_RPC_BASE + +import argparse +import json +import os +import socket +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.request import build_opener, ProxyHandler, Request + +LABEL = "crowdb.paxos-kv.deployment" +HTTP = build_opener(ProxyHandler({})) + + +def docker(*args): + p = subprocess.run(["docker", *args], text=True, capture_output=True, timeout=60) + if p.returncode: + raise RuntimeError(p.stdout + p.stderr) + return (p.stdout + p.stderr if args[0] == "logs" else p.stdout).strip() + + +def request(origin, path, body=None): + data = None if body is None else json.dumps(body).encode() + with HTTP.open(Request(origin + path, data=data, headers={"Content-Type": "application/json"}), timeout=5) as r: + raw = r.read() + return json.loads(raw) if raw else None + + +def wait_ready(origins): + deadline = time.monotonic() + 30 + last = None + while time.monotonic() < deadline: + try: + result = [request(origin, "/topology") for origin in origins] + if all(t.get("stores") and t["stores"][0].get("groups") for t in result): + return result + except (OSError, ValueError) as error: + last = str(error) + time.sleep(.1) + raise RuntimeError(f"node readiness timeout: {last}") + + +def stop(state, artifacts=None, graceful=False): + # Check every identity before mutating any deployment member. + inspected = {} + for name in state["containers"]: + found = json.loads(docker("inspect", name))[0] + if (found["Config"].get("Labels") or {}).get(LABEL) != state["id"]: + raise RuntimeError(f"ownership mismatch: {name}") + inspected[name] = found + failures = [] + remaining = [] + for name in state["containers"]: + try: + failures.extend(stop_node(name, inspected[name], state, artifacts, graceful)) + except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error: + # Retain this node for another collection attempt, but still stop + # and collect the other owned nodes. Never report missing evidence + # as a successful sanitizer run. + failures.append(f"{name}: cleanup incomplete: {error}") + remaining.append(name) + state["containers"] = remaining + return failures + + +def stop_node(name, found, state, artifacts, graceful): + failures = [] + if graceful: + docker("stop", "--time", "30", name) + found = json.loads(docker("inspect", name))[0] + if found["State"]["ExitCode"] != 0 or found["State"]["OOMKilled"]: + failures.append(f"{name}: {found['State']}") + if artifacts is not None: + target = artifacts / name + target.mkdir(parents=True, exist_ok=True) + (target / "inspect.json").write_text(json.dumps(found, indent=2)) + (target / "stdout.log").write_text(docker("logs", name)) + docker("cp", f"{name}:/opt/crowdb/data/kv/log/.", str(target / "log")) + if state.get("sanitizer"): + docker("cp", f"{name}:/opt/crowdb/data/asan/.", str(target / "asan")) + for report in (target / "asan").glob("report*"): + failures.append(f"{name}: sanitizer report {report}\n{report.read_text()}") + docker("rm", "-f", "-v", name) + return failures + + +def start(args): + if not args.image: + raise ValueError("up requires --image") + if args.sanitizer: + image = json.loads(docker("image", "inspect", args.image))[0] + if image["Config"].get("Labels", {}).get("crowdb.sanitizer") != "address": + raise ValueError("--sanitizer requires a staged ASan image") + if args.state.exists(): + raise ValueError("state file exists; stop its deployment first") + if len(args.offsets) != 3 or len(set(args.offsets)) != 3 or any(o < 0 or o > 999 for o in args.offsets): + raise ValueError("three distinct offsets in 0..999 required") + if args.cpuset and len(args.cpuset) != 3: + raise ValueError("supply one cpuset per node") + ports = [port + offset for offset in args.offsets for port in [PAXOS_KV_SERVER_MGMT_BASE, PAXOS_KV_SERVER_RPC_BASE]] + if len(set(ports)) != len(ports): + raise ValueError("offset port pools overlap") + for port in ports: + with socket.socket() as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("0.0.0.0", port)) + state = {"id":uuid.uuid4().hex,"containers":[],"endpoints":[],"network":"host","offsets":args.offsets, + "sanitizer":args.sanitizer} + args.state.parent.mkdir(parents=True, exist_ok=True) + try: + for i, offset in enumerate(args.offsets): + name = f"paxos-kv-{state['id'][:12]}-{i+1}" + command = ["run", "-d", "--name", name, "--network", "host", "--label", f"{LABEL}={state['id']}", + "--pids-limit", "256", "-e", f"TOKIO_WORKER_THREADS={len(os.sched_getaffinity(0))}"] + if args.memory: + command += ["--memory", args.memory, "--memory-swap", args.memory] + if args.cpus is not None: + command += ["--cpus", str(args.cpus)] + if args.cpuset: + command += ["--cpuset-cpus", args.cpuset[i]] + command += [args.image] + if not args.sanitizer: + command += ["server", "paxos-kv"] + command += ["--root","/opt/crowdb/data/kv", + "--management-port",str(PAXOS_KV_SERVER_MGMT_BASE+offset),"--management-addr","0.0.0.0", + "--management-advertise-addr","127.0.0.1","--ports",str(PAXOS_KV_SERVER_RPC_BASE+offset), + "--stores","0","--groups","1","--replica",str(i+1), + "--kv-backend","mem-block","--wal-backend","mem-block","--event-write", + "--election-profile", args.election_profile, + "--max-inflight",str(args.window),"--coalesce-max-keys",str(args.coalesce), + "--peer-pool-size","4","--rpc-workers",str(args.workers),"--log"] + docker(*command) + state["containers"].append(name) + state["endpoints"].append(f"http://127.0.0.1:{PAXOS_KV_SERVER_MGMT_BASE+offset}") + args.state.write_text(json.dumps(state, indent=2)) + topologies = wait_ready(state["endpoints"]) + stores = [] + for topology, offset in zip(topologies, args.offsets, strict=True): + store = topology["stores"][0] + store["listen_addr"] = f"127.0.0.1:{PAXOS_KV_SERVER_RPC_BASE+offset}" + stores.append(store) + for origin in state["endpoints"]: + request(origin,"/stores/0/groups/1/remotes/batch",{"stores":stores}) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + groups = [request(o,"/topology")["stores"][0]["groups"][0] for o in state["endpoints"]] + leaders = {g["leader_id"] for g in groups} + if len(leaders) == 1 and next(iter(leaders)) in (1,2,3): + print(json.dumps(state, indent=2)) + return + time.sleep(.1) + raise RuntimeError("leader convergence timeout") + except BaseException: + for name in state["containers"]: + try: + print(docker("logs", name), file=sys.stderr) + except RuntimeError as error: + print(f"Cannot collect logs for {name}: {error}", file=sys.stderr) + failures = stop(state, args.state.parent / "startup-failure" if args.sanitizer else None, args.sanitizer) + for failure in failures: + print(failure, file=sys.stderr) + if state["containers"]: + args.state.write_text(json.dumps(state, indent=2)) + else: + args.state.unlink(missing_ok=True) + raise + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("action",choices=("up","down")) + p.add_argument("--state",type=Path,required=True) + p.add_argument("--image") + p.add_argument("--sanitizer", action="store_true", help="direct-server ASan image, staged by sanitizer tooling") + p.add_argument("--graceful", action="store_true", help="SIGTERM shutdown; fail on nonzero exit or OOM") + p.add_argument("--artifacts", type=Path, help="retain shutdown inspect, logs and sanitizer reports") + p.add_argument("--offsets",type=int,nargs=3,default=[0,1,2]) + p.add_argument("--memory", help="optional per-node memory limit; omitted for unrestricted performance baseline") + p.add_argument("--cpus",type=float,help="optional per-node CPU quota; omitted for unrestricted performance baseline") + p.add_argument("--cpuset",action="append",default=[]) + p.add_argument("--workers",type=int,default=2) + p.add_argument("--window",type=int,default=32) + p.add_argument("--coalesce",type=int,default=16) + p.add_argument("--election-profile", choices=("default", "e2e"), default="default") + args = p.parse_args() + if args.action == "up": + start(args) + else: + state = json.loads(args.state.read_text()) + if state.get("sanitizer") and (not args.graceful or args.artifacts is None): + raise ValueError("sanitizer shutdown requires --graceful --artifacts") + original = dict(state) + failures = stop(state, args.artifacts, args.graceful) + args.state.with_suffix(".stopped.json").write_text(json.dumps( + {"state":original, "failures":failures, "remaining_containers":state["containers"]}, indent=2)) + if state["containers"]: + args.state.write_text(json.dumps(state, indent=2)) + else: + args.state.unlink() + if failures: + raise RuntimeError("\n".join(failures)) + + +if __name__ == "__main__": + main() diff --git a/container/oci-image/ports.py b/container/oci-image/ports.py new file mode 100644 index 000000000..a8a2f56e5 --- /dev/null +++ b/container/oci-image/ports.py @@ -0,0 +1,22 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. + +"""Generated from ports.rs by tools/ports/generate.py; do not edit.""" + +PAXOS_KV_SERVER_MGMT_BASE = 10135 +PAXOS_KV_SERVER_RPC_BASE = 11135 +DISKDB_HTTP_BASE = 13135 +DISKDB_RPC_BASE = 14135 +CHUNKDB_HTTP_BASE = 16135 +CHUNKDB_RPC_BASE = 17135 +DISKIO_RPC_BASE = 18135 +WEB_BASE = 6135 +CHUNK_KV_HTTP_BASE = 20135 +CHUNK_KV_RPC_BASE = 21135 +ACCESS_SERVER_HTTP_BASE = 7135 +ACCESS_SERVER_ICEBERG_HTTP_BASE = 8135 +ACCESS_SERVER_DATASET_HTTP_BASE = 9135 +ACCESS_SERVER_HEALTH_HTTP_BASE = 25135 +MONITOR_HTTP_BASE = 26135 +SSH_PORT = 2222 +HOST_SSH_PORT = 22 diff --git a/container/oci-image/run-node.py b/container/oci-image/run-node.py index fe5a94d39..6618874eb 100755 --- a/container/oci-image/run-node.py +++ b/container/oci-image/run-node.py @@ -11,6 +11,8 @@ import stat from pathlib import Path +import ports + from runtime import Runtime, node_policy_file from storage import inputs @@ -20,13 +22,16 @@ def digest_reference(reference): return digest.startswith("sha256:") and len(digest) == 71 and all(c in "0123456789abcdef" for c in digest[7:]) -# Base listeners used by one node. Host-network instances add -# ``CROWDB_PORT_OFFSET`` to every listener before the policy is sealed. These -# values mirror the canonical service blocks in -# ``lib/crowdb-protocol/src/port/ports.rs``. -HOST_PORTS = (2222, 9090, 9095, 10000, 10100, 10101, 11000, 11100, 11200, - 12000, 12100, 12200, 13000, 14000, 15100, 15200, 16000, 17000, - 18000, 19000) +# Default node-policy listeners. Explicitly configured service endpoints remain authoritative. +HOST_PORTS = (ports.SSH_PORT, ports.PAXOS_KV_SERVER_MGMT_BASE, + ports.PAXOS_KV_SERVER_RPC_BASE, ports.PAXOS_KV_SERVER_RPC_BASE + 1, + ports.DISKDB_HTTP_BASE, ports.DISKDB_RPC_BASE, + ports.CHUNKDB_HTTP_BASE, ports.CHUNKDB_RPC_BASE, + ports.DISKIO_RPC_BASE, ports.WEB_BASE, + ports.CHUNK_KV_HTTP_BASE, ports.CHUNK_KV_RPC_BASE, + ports.ACCESS_SERVER_HTTP_BASE, ports.ACCESS_SERVER_ICEBERG_HTTP_BASE, + ports.ACCESS_SERVER_DATASET_HTTP_BASE, ports.ACCESS_SERVER_HEALTH_HTTP_BASE, + ports.MONITOR_HTTP_BASE) def check_host_ports(ports=HOST_PORTS): @@ -96,7 +101,7 @@ def validate(args, parser, check_ports=True): parser.error("management interface and physical host identity are required") if args.network == "host": if args.ui_port is not None: - parser.error("host networking uses the node's port 9090 directly") + parser.error(f"host networking uses the node's port {ports.WEB_BASE} directly") try: socket.if_nametoindex(args.interface) if check_ports: diff --git a/container/oci-image/runtime.py b/container/oci-image/runtime.py index db627149d..43317c553 100644 --- a/container/oci-image/runtime.py +++ b/container/oci-image/runtime.py @@ -2,6 +2,8 @@ # Licensed under the Apache License, Version 2.0. """Host Docker/containerd commands and verified container resource boundaries.""" +from ports import WEB_BASE + import json import math import os @@ -134,13 +136,13 @@ def create(self, args): if args.seed: command += ["-e", "CROWDB_DISCOVERY_SEEDS=" + ",".join(args.seed)] if args.ui_port: - command += ["-p", f"127.0.0.1:{args.ui_port}:9090"] + command += ["-p", f"127.0.0.1:{args.ui_port}:{WEB_BASE}"] try: container = self.invoke(*command, args.image).strip() self.verify(container, args.cpus, args.memory_mib, args.network == "host", shares) for disk in disks: kind = "-f" if disk["backend"] == "file" else "-b" - self.invoke("exec", "--user", "10001:10001", container, "sh", "-c", + self.invoke("exec", "--user", "10001:10136", container, "sh", "-c", 'test "$1" "$2" && test -r "$2" && test -w "$2"', "sh", kind, disk["path"]) record_inputs(args.data_root, disks) return container diff --git a/container/oci-image/stage.sh b/container/oci-image/stage.sh index d15bd84be..794698fb9 100644 --- a/container/oci-image/stage.sh +++ b/container/oci-image/stage.sh @@ -4,6 +4,7 @@ set -euo pipefail [[ $(uname -sm) == "Linux x86_64" ]] || { echo "Linux amd64 build host required; macOS VM backend is reserved" >&2; exit 1; } cd "$(git rev-parse --show-toplevel)" +python tools/ports/generate.py --check source_tree=$(python container/oci-image/source_tree.py) case "${CROWDB_KEEP_DEBUG_INFO:-0}" in 0) cmake_debug_args=() ;; @@ -14,11 +15,11 @@ case "${CROWDB_KEEP_DEBUG_INFO:-0}" in *) echo 'CROWDB_KEEP_DEBUG_INFO must be 0 or 1' >&2; exit 1 ;; esac # Build on the host, reusing the existing Cargo, CMake and npm artifacts. -cargo build --locked --release -p crowdb-kv-client --features ffi +cargo build --locked --release -p crowdb-paxos-kv-client --features ffi cmake -S app/crowdb-diskio -B app/crowdb-diskio/build -DCMAKE_BUILD_TYPE=Release "${cmake_debug_args[@]}" cmake --build app/crowdb-diskio/build --parallel "$(getconf _NPROCESSORS_ONLN)" --target crowdb-diskio -cargo build --locked --release --features crowdb-kv-client/ffi \ - -p crowdb-kv-client -p crowdb-monitor -p crowdb-kv-server -p crowdb-diskdb \ +cargo build --locked --release --features crowdb-paxos-kv-client/ffi \ + -p crowdb-paxos-kv-client -p crowdb-monitor -p crowdb-paxos-kv-server -p crowdb-diskdb \ -p crowdb-chunkdb -p crowdb-chunk-kv-server \ -p crowdb-access-server -p crowdb-web (cd app/crowdb-web/ui && npm ci --prefer-offline && npm run build) diff --git a/container/oci-image/tests/docker_local_test.py b/container/oci-image/tests/docker_local_test.py index 858062fcd..b6a36491a 100644 --- a/container/oci-image/tests/docker_local_test.py +++ b/container/oci-image/tests/docker_local_test.py @@ -15,15 +15,16 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from docker_local_cleanup import clean, image_garbage, selected from docker_local_launch import ensure, verify_container +from ports import WEB_BASE, ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE, ACCESS_SERVER_DATASET_HTTP_BASE from docker_local_state import CLUSTER, OWNER, Docker, load, seal class DockerLocalTest(unittest.TestCase): def setUp(self): - self.record = {"version": 2, "cluster": "first", "owner": str(uuid.uuid4()), + self.record = {"version": 3, "cluster": "first", "owner": str(uuid.uuid4()), "runtime": {"daemon": "owned", "context": "default", "endpoint": "unix:///owned"}, "image_id": "sha256:retained", "image": "crowdb-node:dev", "nodes": 1, - "port_base": 9090, "cpus": 1, "memory_mib": 1536} + "port_base": WEB_BASE, "cpus": 1, "memory_mib": 1536} self.labels = {CLUSTER: "first", OWNER: self.record["owner"]} self.docker = MagicMock() self.docker.identity.return_value = self.record["runtime"] @@ -120,35 +121,58 @@ def test_create_cannot_adopt_a_foreign_named_network(self): self.docker.call.assert_not_called() def test_changed_container_inputs_are_rejected_before_resume(self): - record = dict(self.record, nodes=1, port_base=9090, cpus=1, memory_mib=1536) + record = dict(self.record, nodes=1, port_base=WEB_BASE, cpus=1, memory_mib=1536) environment = ["CROWDB_CLUSTER=first", "CROWDB_INITIAL_VOTERS=1"] - for offset, name in enumerate(("S3", "ICEBERG", "DATASET"), start=1): - environment.append(f"CROWDB_{name}_PUBLIC_URI=http://127.0.0.1:{9090+offset}") + for port, name in ((ACCESS_SERVER_HTTP_BASE, "S3"), (ACCESS_SERVER_ICEBERG_HTTP_BASE, "ICEBERG"), (ACCESS_SERVER_DATASET_HTTP_BASE, "DATASET")): + environment.append(f"CROWDB_{name}_PUBLIC_URI=http://127.0.0.1:{port}") item = {"Name": "/crowdb-first-1", "Image": record["image_id"], "Config": {"Labels": self.labels, "Env": environment}, "HostConfig": {"NanoCpus": 1_000_000_000, "Memory": 1536 * 1024 * 1024, "NetworkMode": "crowdb-first", "PortBindings": { - f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(9090+offset)}] - for offset, port in enumerate((9090, 16000, 17000, 18000))}}, + f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}] + for port in (WEB_BASE, ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE, ACCESS_SERVER_DATASET_HTTP_BASE)}}, "Mounts": [{"Destination": "/opt/crowdb/data", "Name": "owned-volume", "Type": "volume", "RW": True}]} verify_container(item, record, 1, "owned-volume", "crowdb-first") - legacy = dict(item, HostConfig=dict(item["HostConfig"], PortBindings={ - f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}] - for port in range(9090, 9094)})) - verify_container(legacy, dict(record, version=1), 1, "owned-volume", "crowdb-first") - with self.assertRaisesRegex(ValueError, "endpoints"): - verify_container(legacy, record, 1, "owned-volume", "crowdb-first") for field, value in (("Image", "foreign"), ("Mounts", []), ("HostConfig", {})): with self.subTest(field=field), self.assertRaises((ValueError, KeyError)): verify_container(dict(item, **{field: value}), record, 1, "owned-volume", "crowdb-first") + def test_retained_versions_preserve_exact_ports_and_public_endpoints(self): + for version, internal in ((1, (9090, 9091, 9092, 9093)), + (2, (9090, 16000, 17000, 18000)), + (3, (WEB_BASE, ACCESS_SERVER_HTTP_BASE, + ACCESS_SERVER_ICEBERG_HTTP_BASE, ACCESS_SERVER_DATASET_HTTP_BASE))): + with self.subTest(version=version), tempfile.TemporaryDirectory() as temporary: + record = dict(self.record, version=version, nodes=3, + port_base=65000 if version < 3 else WEB_BASE+37) + seal(record, Path(temporary)) + self.assertEqual(load("first", Path(temporary)), record) + base = record["port_base"] + (20 if version < 3 else 2) + offsets = range(4) if version < 3 else [port-WEB_BASE for port in internal] + published = [base+offset for offset in offsets] + environment = ["CROWDB_CLUSTER=first", "CROWDB_INITIAL_VOTERS=3"] + for name, port in zip(("S3", "ICEBERG", "DATASET"), published[1:]): + environment.append(f"CROWDB_{name}_PUBLIC_URI=http://127.0.0.1:{port}") + item = {"Name": "/crowdb-first-3", "Image": record["image_id"], + "Config": {"Labels": self.labels, "Env": environment}, + "HostConfig": {"NanoCpus": 1_000_000_000, "Memory": 1536*1024*1024, + "NetworkMode": "crowdb-first", "PortBindings": { + f"{internal_port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}] + for internal_port, port in zip(internal, published)}}, + "Mounts": [{"Destination": "/opt/crowdb/data", "Name": "owned-volume", + "Type": "volume", "RW": True}]} + verify_container(item, record, 3, "owned-volume", "crowdb-first") + item["Config"]["Env"][-1] = "CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:1" + with self.assertRaisesRegex(ValueError, "endpoints"): + verify_container(item, record, 3, "owned-volume", "crowdb-first") + def test_compose_examples_publish_the_standard_node_listeners(self): for filename in ("compose.yaml", "compose-three.yaml"): document = yaml.safe_load((Path(__file__).resolve().parents[1] / filename).read_text()) for service in document["services"].values(): self.assertEqual([port.split("}:", 1)[1] for port in service["ports"]], - ["9090", "${CROWDB_S3_CONTAINER_PORT:-16000}", - "${CROWDB_ICEBERG_CONTAINER_PORT:-17000}", - "${CROWDB_DATASET_CONTAINER_PORT:-18000}"]) + [str(WEB_BASE), f"${{CROWDB_S3_CONTAINER_PORT:-{ACCESS_SERVER_HTTP_BASE}}}", + f"${{CROWDB_ICEBERG_CONTAINER_PORT:-{ACCESS_SERVER_ICEBERG_HTTP_BASE}}}", + f"${{CROWDB_DATASET_CONTAINER_PORT:-{ACCESS_SERVER_DATASET_HTTP_BASE}}}"]) def test_receipts_are_immutable_scoped_and_reject_symlinks(self): with tempfile.TemporaryDirectory() as temporary: diff --git a/container/oci-image/tests/host_inputs_test.py b/container/oci-image/tests/host_inputs_test.py index 77205ebbd..db01c20da 100644 --- a/container/oci-image/tests/host_inputs_test.py +++ b/container/oci-image/tests/host_inputs_test.py @@ -19,8 +19,8 @@ class HostInputsTest(unittest.TestCase): def test_port_offset_shifts_every_host_listener_for_additional_instances(self): shifted = run_node.shifted_ports(1) self.assertEqual(shifted[0], 2223) - self.assertIn(10001, shifted) - self.assertIn(10101, shifted) + self.assertIn(run_node.ports.PAXOS_KV_SERVER_MGMT_BASE + 1, shifted) + self.assertIn(run_node.ports.PAXOS_KV_SERVER_RPC_BASE + 1, shifted) with self.assertRaisesRegex(ValueError, "beyond port 65535"): run_node.shifted_ports(65535) diff --git a/container/oci-image/tests/ports_test.py b/container/oci-image/tests/ports_test.py new file mode 100644 index 000000000..58a9ce926 --- /dev/null +++ b/container/oci-image/tests/ports_test.py @@ -0,0 +1,49 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +"""Port defaults and deployment arguments agree with the protocol source.""" + +from pathlib import Path +import subprocess +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(ROOT / "container/oci-image")) +import ports +from docker_local_launch import PUBLIC_PORTS, verify_container + + +class PortsTest(unittest.TestCase): + def test_generated_defaults_are_current(self): + subprocess.run([sys.executable, str(ROOT / "tools/ports/generate.py"), "--check"], check=True) + + def test_default_bridge_ports_use_node_offsets_zero_one_two(self): + all_ports = set() + for offset in range(3): + for base, _ in PUBLIC_PORTS: + self.assertNotIn(base + offset, all_ports) + all_ports.add(base + offset) + self.assertEqual([port for port, _ in PUBLIC_PORTS], [6135, 7135, 8135, 9135]) + + def test_second_container_verification_uses_service_spacing_and_node_offset(self): + record = dict(version=3, cluster="ports", owner="owner", nodes=3, image_id="sha256:fixture", + cpus=1, memory_mib=1536, port_base=ports.WEB_BASE + 100) + env = ["CROWDB_CLUSTER=ports", "CROWDB_INITIAL_VOTERS=3"] + for base, protocol in PUBLIC_PORTS: + if protocol: + env.append(f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{base + 101}") + item = { + "Name": "/crowdb-ports-2", "Image": record["image_id"], + "Config": {"Labels": {"org.crowdb.local.cluster": "ports", "org.crowdb.local.owner": "owner"}, "Env": env}, + "HostConfig": {"NanoCpus": 1_000_000_000, "Memory": 1536 * 1024 * 1024, + "NetworkMode": "crowdb-ports", "PortBindings": { + f"{base}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(base + 101)}] + for base, _ in PUBLIC_PORTS}}, + "Mounts": [{"Destination": "/opt/crowdb/data", "Type": "volume", "Name": "data", "RW": True}], + } + from docker_local_state import CLUSTER, OWNER + item["Config"]["Labels"] = {CLUSTER: record["cluster"], OWNER: record["owner"]} + verify_container(item, record, 2, "data", "crowdb-ports") + item["HostConfig"]["PortBindings"][f"{ports.WEB_BASE}/tcp"][0]["HostPort"] = str(ports.WEB_BASE + 110) + with self.assertRaises(ValueError): + verify_container(item, record, 2, "data", "crowdb-ports") diff --git a/container/oci-image/tests/staging_test.py b/container/oci-image/tests/staging_test.py index 74500f739..194aedfec 100644 --- a/container/oci-image/tests/staging_test.py +++ b/container/oci-image/tests/staging_test.py @@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[3] PIXI = ["pixi", "run", "--as-is", "--manifest-path", str(ROOT / "pixi.toml")] -BINARIES = ("crowdb-monitor", "crowdb-kv-server", "crowdb-diskdb", "crowdb-diskio", +BINARIES = ("crowdb-monitor", "crowdb-paxos-kv-server", "crowdb-diskdb", "crowdb-diskio", "crowdb-chunkdb", "crowdb-chunk-kv-server", "crowdb-access-server", "crowdb-web") @@ -50,8 +50,8 @@ def stage_fixture(self, directory, aliased): for name, function in definitions.items(): self.compile(f"int {function}(void) {{ return 0; }}", libraries / name, "-shared", "-fPIC", "-Wl,-soname," + name) - self.compile("int ffi(void) { return 0; }", release / "libcrowdb_kv_client.so", - "-shared", "-fPIC", "-Wl,-soname,libcrowdb_kv_client.so") + self.compile("int ffi(void) { return 0; }", release / "libcrowdb_paxos_kv_client.so", + "-shared", "-fPIC", "-Wl,-soname,libcrowdb_paxos_kv_client.so") self.compile("int provider_dependency(void); int provider(void) { return provider_dependency(); }", libraries / "libfixture.so.1", "-shared", "-fPIC", "-L" + str(libraries), "-lprovider-dependency", "-Wl,-rpath," + str(libraries)) @@ -61,7 +61,7 @@ def stage_fixture(self, directory, aliased): executable = directory / "node" self.compile(source, executable, "-L" + str(libraries), "-L" + str(release), "-Wl,-l:libibverbs.so.1", "-Wl,-l:librdmacm.so.1", "-Wl,-l:libcrypto.so.3", - "-lcrowdb_kv_client") + "-lcrowdb_paxos_kv_client") for name in BINARIES: target = root / "app/crowdb-diskio/build" if name == "crowdb-diskio" else release target.mkdir(parents=True, exist_ok=True) @@ -76,7 +76,7 @@ def stage_fixture(self, directory, aliased): command = [*PIXI, "bash", "-c", 'PATH="$1:$PATH"; exec bash "$2" "$3"', "_", str(tools), str(ROOT / "container/oci-image/collect-libs.sh"), str(staged)] subprocess.run(command, check=True, env=dict(os.environ, CROWDB_STAGING_TEST_ROOT=str(root))) - for name in (*definitions, "libcrowdb_kv_client.so"): + for name in (*definitions, "libcrowdb_paxos_kv_client.so"): self.assertTrue((staged / "lib" / name).is_file(), name) provider = staged / "lib/libibverbs/libfixture-rdmav99.so" self.assertTrue(provider.is_file()) diff --git a/container/oci-image/tests/task_entrypoints_test.py b/container/oci-image/tests/task_entrypoints_test.py index 7e1cd3b6e..3e6ac69ba 100644 --- a/container/oci-image/tests/task_entrypoints_test.py +++ b/container/oci-image/tests/task_entrypoints_test.py @@ -45,8 +45,8 @@ def test_build_compiles_cpp_release_frontend_and_tests_without_running_tests(sel result, commands = self.execute('build-release.sh') self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertTrue(commands[0].startswith('cmake -S lib/crowdb-tree')) - self.assertIn('cargo build --release --workspace --exclude crowdb-kv-client', commands) - self.assertIn('cargo build --release -p crowdb-kv-client --features ffi', commands) + self.assertIn('cargo build --release --workspace --exclude crowdb-paxos-kv-client', commands) + self.assertIn('cargo build --release -p crowdb-paxos-kv-client --features ffi', commands) self.assertIn('npm run build', commands) self.assertEqual(commands[-1], 'cargo test --workspace --all-targets --no-run') diff --git a/container/single-node-container/README.md b/container/single-node-container/README.md index b0ba77b41..5c86a2705 100644 --- a/container/single-node-container/README.md +++ b/container/single-node-container/README.md @@ -28,7 +28,7 @@ reserve physical host space. Production does not fall back to this policy. is retained for diagnosis; restarting is not permission to replace its data. The node-first local client origins default to `http://127.0.0.1` on ports -9091 (S3), 9092 (Iceberg) and 9093 (Dataset). When publishing different host +7135 (S3), 8135 (Iceberg) and 9135 (Dataset). When publishing different host ports or addresses, supply `CROWDB_S3_PUBLIC_URI`, `CROWDB_ICEBERG_PUBLIC_URI` and `CROWDB_DATASET_PUBLIC_URI` to each node. These HTTP origins are sealed in the node policy before startup; omitted restart values retain the accepted @@ -49,8 +49,8 @@ into logs or cluster status. ## Client listeners and readiness -The same Access service exposes S3 at 9091, Iceberg at 9092 and Dataset at 9093. -The Web UI listens at 9090. Publish these ports when using a bridge. A listener +The same Access service exposes S3 at 7135, Iceberg at 8135 and Dataset at 9135. +The Web UI listens at 6135. Publish these ports when using a bridge. A listener or an old completed setup record is not proof of current storage availability. Use the client origins returned by the credential command. Iceberg metadata and @@ -98,7 +98,7 @@ s3 = ``` Use that profile in `AWS_CONFIG_FILE`, export the generated credentials, then -run `aws --endpoint-url http://127.0.0.1:9091 ...`. For server copy, supply +run `aws --endpoint-url http://127.0.0.1:7135 ...`. For server copy, supply `--metadata-directive COPY` to `aws s3 cp`. The repeatable repository gate is `pixi run test-e2e-s3 --case s3.protocol::test_aws_cli_workflow`; container acceptance runs the same recipe and checks persisted bytes after recovery before publication credentials become @@ -137,3 +137,6 @@ The OCI image carries the same server binaries for all protocols. It neither mounts the Docker socket nor delegates bootstrap or storage preparation to the host CLI. Monitor-owned accepted operations continue after the launching shell or browser exits. Changing an image does not reset node identity or storage. + +Default listener ports are defined in [protocol ports.rs](../../lib/crowdb-protocol/src/port/ports.rs). +Generated packaging defaults are maintained with [tools/ports](../../tools/ports/README.md). diff --git a/container/single-node-container/dev-container.sh b/container/single-node-container/dev-container.sh index 85e82f4d5..e1bdcdab8 100644 --- a/container/single-node-container/dev-container.sh +++ b/container/single-node-container/dev-container.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. # Copyright 2026-present Gian <crow.db@outlook.com> # Licensed under the Apache License, Version 2.0. set -euo pipefail @@ -49,7 +50,7 @@ start_container() { if ! docker run --detach --name "$name" -e CROWDB_STARTUP_MODE=single \ --mount "type=volume,source=$data_volume,target=/opt/crowdb/data" \ - --expose 9090 --expose 9091 --expose 9092 --expose 9093 --expose 9094 \ + --expose 6135 --expose 7135 --expose 8135 --expose 9135 --expose 25135 \ "$image"; then if docker container inspect "$name" >/dev/null 2>&1; then docker rm --force "$name" >/dev/null @@ -128,10 +129,10 @@ inject_container() { fi echo "Injecting into CROWDB container $name at $container_ip" pixi run -e iceberg-e2e python tools/load-tpch.py \ - --console-url "http://$container_ip:9090" \ + --console-url "http://$container_ip:6135" \ --credentials-file "$credentials_file" \ - --catalog-uri "http://$container_ip:9092" \ - --s3-endpoint "http://$container_ip:9091" + --catalog-uri "http://$container_ip:8135" \ + --s3-endpoint "http://$container_ip:7135" } clean_container() { diff --git a/container/single-node-container/dev-container.sh.ports.in b/container/single-node-container/dev-container.sh.ports.in new file mode 100644 index 000000000..5e06316e8 --- /dev/null +++ b/container/single-node-container/dev-container.sh.ports.in @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +set -euo pipefail + +root=$(git rev-parse --show-toplevel) +cd "$root" + +image=${CROWDB_CONTAINER_IMAGE:-crowdb-node:dev} +name=${CROWDB_CONTAINER_NAME:-crowdb-single-node} +data_volume=${CROWDB_CONTAINER_DATA_VOLUME:-${name}-data} + +usage() { + echo 'Usage: dev-container.sh {start|clean|inject [container IP|ID|name]}' >&2 + exit 2 +} + +wait_until_healthy() { + local status + for _ in $(seq 1 240); do + status=$(docker inspect --format '{{.State.Status}}' "$name" 2>/dev/null || true) + if [[ "$status" != running ]]; then + docker logs "$name" >&2 || true + echo "Container $name stopped before becoming healthy" >&2 + return 1 + fi + if docker exec "$name" crowdb-monitor readiness; then + echo "Container $name is healthy" + return 0 + fi + sleep 1 + done + docker logs "$name" >&2 || true + echo "Timed out waiting for container $name to become healthy" >&2 + return 1 +} + +start_container() { + if docker container inspect "$name" >/dev/null 2>&1; then + if [[ $(docker inspect --format '{{.State.Status}}' "$name") == running ]]; then + echo "Container $name is already running" + wait_until_healthy + return + fi + docker start "$name" + wait_until_healthy + return + fi + + if ! docker run --detach --name "$name" -e CROWDB_STARTUP_MODE=single \ + --mount "type=volume,source=$data_volume,target=/opt/crowdb/data" \ + --expose @WEB_BASE@ --expose @ACCESS_SERVER_HTTP_BASE@ --expose @ACCESS_SERVER_ICEBERG_HTTP_BASE@ --expose @ACCESS_SERVER_DATASET_HTTP_BASE@ --expose @ACCESS_SERVER_HEALTH_HTTP_BASE@ \ + "$image"; then + if docker container inspect "$name" >/dev/null 2>&1; then + docker rm --force "$name" >/dev/null + fi + return 1 + fi + if ! wait_until_healthy; then + docker rm --force "$name" >/dev/null 2>&1 || true + return 1 + fi + docker exec "$name" crowdb-monitor credentials show --format env +} + +select_inject_container() { + local target=${1:-${CROWDB_CONTAINER_NAME:-}} + local rows id candidate_name candidate_image source ips ip + local -a candidates=() matches=() + rows=$(docker ps --no-trunc --format '{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Label "org.opencontainers.image.source"}}') || return + while IFS=$'\t' read -r id candidate_name candidate_image source; do + [[ -n "$id" ]] || continue + if [[ "$source" != https://github.com/buzzcrow/crowdb && + "$candidate_image" != "$image" && "$candidate_image" != *crowdb* ]]; then + continue + fi + ips=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{if .IPAddress}}{{.IPAddress}} {{end}}{{end}}' "$id") || return + candidates+=("$id $candidate_name $candidate_image ${ips:-<no IP>}") + if [[ -z "$target" || "$target" == "$candidate_name" || "$id" == "$target"* ]]; then + matches+=("$id") + else + for ip in $ips; do + if [[ "$target" == "$ip" ]]; then + matches+=("$id") + break + fi + done + fi + done <<< "$rows" + if ((${#matches[@]} != 1)); then + if [[ -n "$target" ]]; then + echo "Cannot uniquely select a running CROWDB container for '$target'." >&2 + else + echo 'Expected one running CROWDB container; specify its IP, ID, or name.' >&2 + fi + if ((${#candidates[@]})); then + echo 'Candidates (ID NAME IMAGE IP):' >&2 + printf '%s\n' "${candidates[@]}" >&2 + else + echo "No running CROWDB containers found; use 'pixi run oci-run-1c'." >&2 + fi + echo 'Usage: pixi run bash container/single-node-container/dev-container.sh inject <container IP|ID|name>' >&2 + return 1 + fi + name=${matches[0]} +} + +inject_container() { + select_inject_container "${1:-}" || return + local credentials_file + credentials_file=$(mktemp "${TMPDIR:-/tmp}/crowdb-container-credentials.XXXXXX") + trap 'rm -f "$credentials_file"' RETURN + docker exec "$name" crowdb-monitor credentials show --format env > "$credentials_file" + local container_ip ip addresses + local -a container_ips + addresses=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{if .IPAddress}}{{.IPAddress}} {{end}}{{end}}' "$name") || return + read -r -a container_ips <<< "$addresses" + container_ip=${container_ips[0]:-} + for ip in "${container_ips[@]}"; do + if [[ "$ip" == "${1:-${CROWDB_CONTAINER_NAME:-}}" ]]; then + container_ip=$ip + break + fi + done + if [[ -z "$container_ip" ]]; then + echo "Container $name has no Docker network address" >&2 + return 1 + fi + echo "Injecting into CROWDB container $name at $container_ip" + pixi run -e iceberg-e2e python tools/load-tpch.py \ + --console-url "http://$container_ip:@WEB_BASE@" \ + --credentials-file "$credentials_file" \ + --catalog-uri "http://$container_ip:@ACCESS_SERVER_ICEBERG_HTTP_BASE@" \ + --s3-endpoint "http://$container_ip:@ACCESS_SERVER_HTTP_BASE@" +} + +clean_container() { + if docker container inspect "$name" >/dev/null 2>&1; then + docker rm --force "$name" + fi + + echo "Preserved data volume $data_volume; images and other containers were not removed." +} + +case "${1:-}" in + start) start_container ;; + inject) + (($# <= 2)) || usage + inject_container "${2:-}" + ;; + clean) clean_container ;; + *) usage ;; +esac diff --git a/container/single-node-container/entrypoint.sh b/container/single-node-container/entrypoint.sh index 6c65fa3fc..e20bb7adc 100644 --- a/container/single-node-container/entrypoint.sh +++ b/container/single-node-container/entrypoint.sh @@ -1,4 +1,5 @@ #!/bin/sh +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. set -eu if ! grep -q ' /opt/crowdb/data ' /proc/self/mountinfo; then @@ -30,7 +31,7 @@ if test "$#" -gt 0; then shift service=${1:-} case "$service" in - kv) program=crowdb-kv-server ;; + paxos-kv) program=crowdb-paxos-kv-server ;; diskdb) program=crowdb-diskdb ;; diskio) program=crowdb-diskio ;; chunkdb) program=crowdb-chunkdb ;; @@ -69,15 +70,14 @@ if test "$port_offset" -gt 0; then # The preview profile is intentionally self-contained. Shift every # declared listener and endpoint together so host-network containers can # coexist while retaining the same relative service layout. - for base in 19000 18000 17000 16000 15200 15100 14000 13000 12200 12100 12000 11200 11100 11000 10101 10100 10000 9095 9094 9093 9092 9091 9090 7401 7400 7301 7300 7200 7102 7101 7100; do + for base in 10135 11135 11136 13135 14135 16135 17135 18135 6135 20135 21135 7135 8135 9135 25135 26135; do + sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1CROWDB_PORT_${base}_END\\2/g" \ + "$offset_root/profile.toml" "$offset_templates"/*.toml + done + for base in 10135 11135 11136 13135 14135 16135 17135 18135 6135 20135 21135 7135 8135 9135 25135 26135; do next=$((base + port_offset)) - sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" "$offset_root/profile.toml" - if test -d "$offset_templates"; then - for template in "$offset_templates"/*.toml; do - test -f "$template" || continue - sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" "$template" - done - fi + sed -i "s/CROWDB_PORT_${base}_END/${next}/g" \ + "$offset_root/profile.toml" "$offset_templates"/*.toml done sed -i "s#/opt/crowdb/etc/templates#$offset_templates#g" "$offset_root/profile.toml" profile="$offset_root/profile.toml" diff --git a/container/single-node-container/entrypoint.sh.ports.in b/container/single-node-container/entrypoint.sh.ports.in new file mode 100644 index 000000000..bef144af1 --- /dev/null +++ b/container/single-node-container/entrypoint.sh.ports.in @@ -0,0 +1,175 @@ +#!/bin/sh +set -eu + +if ! grep -q ' /opt/crowdb/data ' /proc/self/mountinfo; then + echo 'CROWDB preview requires one volume mounted at /opt/crowdb/data' >&2 + exit 1 +fi + +echo 'CROWDB preview data: Docker creates an anonymous volume when none is specified. For data you want to keep across container recreation, use --mount type=volume,source=crowdb-data,target=/opt/crowdb/data.' + +CROWDB_CORE_DIR=/opt/crowdb/data/crash +export CROWDB_CORE_DIR + +case "$(cat /proc/sys/kernel/core_pattern)" in + core|core.*) + if [ "$(ulimit -c)" = 0 ]; then + echo 'CROWDB core files require a nonzero Docker --ulimit core setting.' >&2 + fi + ;; + '|'*) + echo 'CROWDB core dumps use the Docker host collector; inspect the host for dumps.' >&2 + ;; + *) + echo 'CROWDB core_pattern does not use a relative core filename; inspect the Docker host for dumps.' >&2 + ;; +esac + +if test "$#" -gt 0; then + test "$1" = server || { echo 'Expected server SERVICE [arguments]' >&2; exit 1; } + shift + service=${1:-} + case "$service" in + paxos-kv) program=crowdb-paxos-kv-server ;; + diskdb) program=crowdb-diskdb ;; + diskio) program=crowdb-diskio ;; + chunkdb) program=crowdb-chunkdb ;; + chunk-kv) program=crowdb-chunk-kv-server ;; + access) program=crowdb-access-server ;; + web) program=crowdb-web ;; + *) echo 'Unknown server service' >&2; exit 1 ;; + esac + shift + if test -e /opt/crowdb/data/accepted-node.json || test -e /opt/crowdb/data/bootstrap/manifest.json; then + echo 'Standalone server mode requires a separate data root from a managed node' >&2 + exit 1 + fi + mkdir -p "$CROWDB_CORE_DIR" "/opt/crowdb/data/log/$service" + chown crowdb:crowdb /opt/crowdb/data /opt/crowdb/data/log "$CROWDB_CORE_DIR" "/opt/crowdb/data/log/$service" + cd "$CROWDB_CORE_DIR" + exec setpriv --reuid=crowdb --regid=crowdb --init-groups "/opt/crowdb/bin/$program" "$@" +fi + +profile=${CROWDB_PROFILE_PATH:-/opt/crowdb/etc/profile.toml} +port_offset=${CROWDB_PORT_OFFSET:-0} +case "$port_offset" in + ''|*[!0-9]*) echo 'CROWDB_PORT_OFFSET must be a non-negative integer' >&2; exit 1 ;; +esac +if test "$port_offset" -gt 0; then + offset_root=/opt/crowdb/run/port-offset + offset_templates=$offset_root/templates + mkdir -p "$offset_templates" + cp "$profile" "$offset_root/profile.toml" + if test -d /opt/crowdb/etc/templates; then + for template in /opt/crowdb/etc/templates/*.toml; do + test -f "$template" || continue + cp "$template" "$offset_templates/" + done + fi + # The preview profile is intentionally self-contained. Shift every + # declared listener and endpoint together so host-network containers can + # coexist while retaining the same relative service layout. + for base in @PAXOS_KV_SERVER_MGMT_BASE@ @PAXOS_KV_SERVER_RPC_BASE@ @PAXOS_KV_SERVER_RPC_BASE+1@ @DISKDB_HTTP_BASE@ @DISKDB_RPC_BASE@ @CHUNKDB_HTTP_BASE@ @CHUNKDB_RPC_BASE@ @DISKIO_RPC_BASE@ @WEB_BASE@ @CHUNK_KV_HTTP_BASE@ @CHUNK_KV_RPC_BASE@ @ACCESS_SERVER_HTTP_BASE@ @ACCESS_SERVER_ICEBERG_HTTP_BASE@ @ACCESS_SERVER_DATASET_HTTP_BASE@ @ACCESS_SERVER_HEALTH_HTTP_BASE@ @MONITOR_HTTP_BASE@; do + sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1CROWDB_PORT_${base}_END\\2/g" \ + "$offset_root/profile.toml" "$offset_templates"/*.toml + done + for base in @PAXOS_KV_SERVER_MGMT_BASE@ @PAXOS_KV_SERVER_RPC_BASE@ @PAXOS_KV_SERVER_RPC_BASE+1@ @DISKDB_HTTP_BASE@ @DISKDB_RPC_BASE@ @CHUNKDB_HTTP_BASE@ @CHUNKDB_RPC_BASE@ @DISKIO_RPC_BASE@ @WEB_BASE@ @CHUNK_KV_HTTP_BASE@ @CHUNK_KV_RPC_BASE@ @ACCESS_SERVER_HTTP_BASE@ @ACCESS_SERVER_ICEBERG_HTTP_BASE@ @ACCESS_SERVER_DATASET_HTTP_BASE@ @ACCESS_SERVER_HEALTH_HTTP_BASE@ @MONITOR_HTTP_BASE@; do + next=$((base + port_offset)) + sed -i "s/CROWDB_PORT_${base}_END/${next}/g" \ + "$offset_root/profile.toml" "$offset_templates"/*.toml + done + sed -i "s#/opt/crowdb/etc/templates#$offset_templates#g" "$offset_root/profile.toml" + profile="$offset_root/profile.toml" +fi +ssh_port=$((@SSH_PORT@ + port_offset)) +/opt/crowdb/bin/crowdb-monitor validate "$profile" + +if test -n "${CROWDB_STARTUP_MODE:-}"; then + echo 'CROWDB_STARTUP_MODE was removed; use CROWDB_DEPLOYMENT_MODE and CROWDB_INITIAL_VOTERS.' >&2 + exit 1 +fi +accepted_mode=$(setpriv --reuid=crowdb --regid=crowdb --init-groups \ + /opt/crowdb/bin/crowdb-monitor accept-policy --profile "$profile") +CROWDB_DEPLOYMENT_MODE=$accepted_mode +export CROWDB_DEPLOYMENT_MODE +disks=/opt/crowdb/data/disks +if test -L "$disks" || { test -e "$disks" && ! test -d "$disks"; }; then + echo 'Disk directory must be a real directory, not a symlink or file.' >&2 + exit 1 +fi +mkdir -p "$disks" +chown -h crowdb:crowdb "$disks" +if test -L /opt/crowdb/data/ssh || { test -e /opt/crowdb/data/ssh && ! test -d /opt/crowdb/data/ssh; }; then + echo 'SSH identity directory must be a real directory.' >&2 + exit 1 +fi +mkdir -p /opt/crowdb/data/ssh +chmod 700 /opt/crowdb/data/ssh +ssh_password_auth=yes +if test -n "${CROWDB_SSH_PASSWORD_FILE:-}"; then + test -f "$CROWDB_SSH_PASSWORD_FILE" || { echo 'SSH password file missing' >&2; exit 1; } + CROWDB_SSH_PASSWORD=$(cat "$CROWDB_SSH_PASSWORD_FILE") +fi +if test "$accepted_mode" = local; then + local_password=/opt/crowdb/data/ssh/local-admission-password + if test -L "$local_password" || { test -e "$local_password" && ! test -f "$local_password"; }; then + echo 'Local admission credential must be a regular private file.' >&2 + exit 1 + fi + if test -f "$local_password"; then + test -s "$local_password" && test "$(stat -c %s "$local_password")" -le 4096 && + test "$(stat -c %a "$local_password")" = 600 || { + echo 'Retained local admission credential is empty, oversized or not private.' >&2 + exit 1 + } + fi + if test -z "${CROWDB_SSH_PASSWORD:-}" && test -f "$local_password"; then + CROWDB_SSH_PASSWORD=$(cat "$local_password") + test -n "$CROWDB_SSH_PASSWORD" || { echo 'Retained local admission credential is invalid.' >&2; exit 1; } + fi + CROWDB_SSH_PASSWORD=${CROWDB_SSH_PASSWORD:-crowdb} + password_tmp=$(mktemp /opt/crowdb/data/ssh/.local-admission.XXXXXX) + printf '%s' "$CROWDB_SSH_PASSWORD" > "$password_tmp" + chmod 600 "$password_tmp" + chown crowdb:crowdb "$password_tmp" + mv "$password_tmp" "$local_password" +fi +if test -n "${CROWDB_SSH_PASSWORD:-}"; then + printf 'crowdb:%s\n' "$CROWDB_SSH_PASSWORD" | chpasswd + unset CROWDB_SSH_PASSWORD +elif test "$accepted_mode" = production; then + if ! test -s /opt/crowdb/data/ssh/authorized_keys; then + echo 'Production requires explicit SSH credentials or preinstalled authorized_keys' >&2 + exit 1 + fi + passwd -d crowdb + ssh_password_auth=no +fi +mkdir -p /opt/crowdb/data/ssh /opt/crowdb/run +chmod 700 /opt/crowdb/data/ssh +for name in id_ed25519 ssh_host_ed25519_key; do + if ! test -f "/opt/crowdb/data/ssh/$name"; then + ssh-keygen -q -t ed25519 -N '' -f "/opt/crowdb/data/ssh/$name" + fi +done +if ! test -f /opt/crowdb/data/ssh/authorized_keys; then + cp /opt/crowdb/data/ssh/id_ed25519.pub /opt/crowdb/data/ssh/authorized_keys +fi +chmod 600 /opt/crowdb/data/ssh/authorized_keys +chown -R crowdb:crowdb /opt/crowdb/data/ssh /opt/crowdb/run +if ! test -e /opt/crowdb/.ssh; then ln -s /opt/crowdb/data/ssh /opt/crowdb/.ssh; fi +cat > /opt/crowdb/run/sshd_config <<EOF +Port $ssh_port +HostKey /opt/crowdb/data/ssh/ssh_host_ed25519_key +AuthorizedKeysFile /opt/crowdb/data/ssh/authorized_keys +PermitRootLogin no +AllowUsers crowdb +PasswordAuthentication $ssh_password_auth +KbdInteractiveAuthentication no +UsePAM no +PidFile /opt/crowdb/run/sshd.pid +Subsystem sftp internal-sftp +EOF +/usr/sbin/sshd -f /opt/crowdb/run/sshd_config +exec setpriv --reuid=crowdb --regid=crowdb --init-groups /opt/crowdb/bin/crowdb-monitor run \ + --profile "$profile" --interface "${CROWDB_MANAGEMENT_INTERFACE:-eth0}" diff --git a/container/single-node-container/profile.toml b/container/single-node-container/profile.toml index 0b5cbcba0..707852fb4 100644 --- a/container/single-node-container/profile.toml +++ b/container/single-node-container/profile.toml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. version = 1 name = "single-node-container" display_name = "CROWDB Single-Node Container" @@ -27,7 +28,7 @@ store_id = 0 group_id = 0 replica_id = 1 node_id = 1 -rpc_endpoint = "127.0.0.1:10100" +rpc_endpoint = "127.0.0.1:11135" role = "system" [[groups]] @@ -35,7 +36,7 @@ store_id = 0 group_id = 1 replica_id = 2 node_id = 1 -rpc_endpoint = "127.0.0.1:10100" +rpc_endpoint = "127.0.0.1:11135" role = "data" [[disks]] @@ -73,29 +74,29 @@ zone_size_bytes = 17179869184 [[public_endpoints]] id = "s3" bind = "0.0.0.0" -port = 9091 +port = 7135 [[public_endpoints]] id = "iceberg" bind = "0.0.0.0" -port = 9092 +port = 8135 [[public_endpoints]] id = "web" bind = "0.0.0.0" -port = 9090 +port = 6135 [[services]] id = "kv" env = { CROWDB_RDMA_SERVICE_SLOT = "1" } -program = "/opt/crowdb/bin/crowdb-kv-server" -args = ["--root", "/opt/crowdb/data/kv/node-1", "--config", "/opt/crowdb/run/config/kv.toml", "--management-port", "10000", "--ports", "10100,10101", "--instance-id", "1", "--node-id", "1", "--binding-monitor-interval", "1", "--log-dir", "/opt/crowdb/data/log/kv", "--log"] +program = "/opt/crowdb/bin/crowdb-paxos-kv-server" +args = ["--root", "/opt/crowdb/data/kv/node-1", "--config", "/opt/crowdb/run/config/kv.toml", "--management-port", "10135", "--ports", "11135,11136", "--instance-id", "1", "--node-id", "1", "--binding-monitor-interval", "1", "--log-dir", "/opt/crowdb/data/log/kv", "--log"] dependencies = [] -fence_listeners = ["127.0.0.1:10000", "127.0.0.1:10100", "127.0.0.1:10101"] +fence_listeners = ["127.0.0.1:10135", "127.0.0.1:11135", "127.0.0.1:11136"] config_template = "/opt/crowdb/etc/templates/kv.toml" [services.probe] kind = "http" -target = "http://127.0.0.1:10000/health" +target = "http://127.0.0.1:10135/health" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -109,11 +110,11 @@ env = { CROWDB_RDMA_SERVICE_SLOT = "2" } program = "/opt/crowdb/bin/crowdb-diskdb" args = ["--config", "/opt/crowdb/run/config/diskdb.toml", "--log-dir", "/opt/crowdb/data/log/diskdb", "--log"] dependencies = ["kv"] -fence_listeners = ["127.0.0.1:11000", "127.0.0.1:11100", "127.0.0.1:11200"] +fence_listeners = ["127.0.0.1:13135", "127.0.0.1:14135"] config_template = "/opt/crowdb/etc/templates/diskdb.toml" [services.probe] kind = "http" -target = "http://127.0.0.1:11100/health" +target = "http://127.0.0.1:13135/health" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -127,11 +128,11 @@ env = { CROWDB_RDMA_SERVICE_SLOT = "3" } program = "/opt/crowdb/bin/crowdb-diskio" args = ["--config", "/opt/crowdb/run/config/diskio.toml"] dependencies = ["kv", "diskdb"] -fence_listeners = ["127.0.0.1:13000"] +fence_listeners = ["127.0.0.1:18135"] config_template = "/opt/crowdb/etc/templates/diskio.toml" [services.probe] kind = "rpc-ping" -target = "127.0.0.1:13000" +target = "127.0.0.1:18135" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -145,11 +146,11 @@ env = { CROWDB_RDMA_SERVICE_SLOT = "4" } program = "/opt/crowdb/bin/crowdb-chunkdb" args = ["--config", "/opt/crowdb/run/config/chunkdb.toml", "--log-dir", "/opt/crowdb/data/log/chunkdb", "--log"] dependencies = ["kv", "diskdb", "diskio"] -fence_listeners = ["127.0.0.1:12100", "127.0.0.1:12200"] +fence_listeners = ["127.0.0.1:16135", "127.0.0.1:17135"] config_template = "/opt/crowdb/etc/templates/chunkdb.toml" [services.probe] kind = "http" -target = "http://127.0.0.1:12100/ready" +target = "http://127.0.0.1:16135/ready" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -163,11 +164,11 @@ env = { CROWDB_RDMA_SERVICE_SLOT = "5" } program = "/opt/crowdb/bin/crowdb-chunk-kv-server" args = ["--config", "/opt/crowdb/run/config/chunk-kv.toml", "--log-dir", "/opt/crowdb/data/log/chunk-kv", "--log"] dependencies = ["kv", "chunkdb", "diskio"] -fence_listeners = ["127.0.0.1:15100", "127.0.0.1:15200"] +fence_listeners = ["127.0.0.1:20135", "127.0.0.1:21135"] config_template = "/opt/crowdb/etc/templates/chunk-kv.toml" [services.probe] kind = "http" -target = "http://127.0.0.1:15100/health" +target = "http://127.0.0.1:20135/health" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -179,19 +180,19 @@ backoff_max_ms = 5000 id = "access" program = "/opt/crowdb/bin/crowdb-access-server" args = ["--config", "/opt/crowdb/run/config/access.toml"] -env = { CROWDB_S3_PUBLIC_URI = "http://localhost:9091", CROWDB_S3_REGION = "us-east-1", CROWDB_ICEBERG_PUBLIC_URI = "http://localhost:9092", CROWDB_DATASET_PUBLIC_URI = "http://localhost:9093", CROWDB_ICEBERG_GC_ENABLED = "0", CROWDB_MANAGEMENT_SEEDS = "http://127.0.0.1:10000", CROWDB_ACCESS_HEALTH_LISTEN = "127.0.0.1:9094", CROWDB_ACCESS_LOG_DIR = "/opt/crowdb/data/log/access", CROWDB_RDMA_SERVICE_SLOT = "6" } +env = { CROWDB_S3_PUBLIC_URI = "http://localhost:7135", CROWDB_S3_REGION = "us-east-1", CROWDB_ICEBERG_PUBLIC_URI = "http://localhost:8135", CROWDB_DATASET_PUBLIC_URI = "http://localhost:9135", CROWDB_ICEBERG_GC_ENABLED = "0", CROWDB_MANAGEMENT_SEEDS = "http://127.0.0.1:10135", CROWDB_ACCESS_HEALTH_LISTEN = "127.0.0.1:25135", CROWDB_ACCESS_LOG_DIR = "/opt/crowdb/data/log/access", CROWDB_RDMA_SERVICE_SLOT = "6" } dependencies = ["kv", "chunk-kv", "chunkdb", "diskio"] -fence_listeners = ["127.0.0.1:9092", "127.0.0.1:9091", "127.0.0.1:9093", "127.0.0.1:9094"] +fence_listeners = ["127.0.0.1:8135", "127.0.0.1:7135", "127.0.0.1:9135", "127.0.0.1:25135"] config_template = "/opt/crowdb/etc/templates/access.toml" [services.probe] kind = "http" -target = "http://127.0.0.1:9092/v1/config" +target = "http://127.0.0.1:8135/v1/config" bearer_env = "CROWDB_ICEBERG_READ_TOKEN" timeout_ms = 1000 failure_threshold = 5 [[services.additional_probes]] kind = "http" -target = "http://127.0.0.1:9094/_crowdb/health/ready" +target = "http://127.0.0.1:25135/_crowdb/health/ready" timeout_ms = 1000 failure_threshold = 5 [services.restart] @@ -203,13 +204,13 @@ backoff_max_ms = 5000 id = "web" env = { CROWDB_RDMA_SERVICE_SLOT = "7" } program = "/opt/crowdb/bin/crowdb-web" -args = ["--config", "/opt/crowdb/run/config/crowdb-web.toml", "--node-monitor", "http://127.0.0.1:9095"] +args = ["--config", "/opt/crowdb/run/config/crowdb-web.toml", "--node-monitor", "http://127.0.0.1:26135"] dependencies = ["access"] -fence_listeners = ["127.0.0.1:9090"] +fence_listeners = ["127.0.0.1:6135"] config_template = "/opt/crowdb/etc/templates/crowdb-web.toml" [services.probe] kind = "http" -target = "http://127.0.0.1:9090/healthz" +target = "http://127.0.0.1:6135/healthz" timeout_ms = 1000 failure_threshold = 5 [services.restart] diff --git a/container/single-node-container/profile.toml.ports.in b/container/single-node-container/profile.toml.ports.in new file mode 100644 index 000000000..7e62f04e9 --- /dev/null +++ b/container/single-node-container/profile.toml.ports.in @@ -0,0 +1,218 @@ +version = 1 +name = "single-node-container" +display_name = "CROWDB Single-Node Container" +placement_mode = "unsafe-colocated" +s3_tenant = "preview" +iceberg_catalog = "preview" + +[paths] +install_root = "/opt/crowdb" +bin_root = "/opt/crowdb/bin" +template_root = "/opt/crowdb/etc/templates" +data_root = "/opt/crowdb/data" +run_root = "/opt/crowdb/run" +log_root = "/opt/crowdb/data/log" + +[logs] +max_file_bytes = 31457280 +max_files = 5 +mirror_warnings_to_stderr = true + +[[nodes]] +node_id = 1 +rack_id = 1 + +[[groups]] +store_id = 0 +group_id = 0 +replica_id = 1 +node_id = 1 +rpc_endpoint = "127.0.0.1:@PAXOS_KV_SERVER_RPC_BASE@" +role = "system" + +[[groups]] +store_id = 0 +group_id = 1 +replica_id = 2 +node_id = 1 +rpc_endpoint = "127.0.0.1:@PAXOS_KV_SERVER_RPC_BASE@" +role = "data" + +[[disks]] +disk_id = "00000000000000010000000000000001" +disk_group_id = 101 +node_id = 1 +path = "/opt/crowdb/data/disks/disk-0001.img" +capacity_bytes = 17179869184 +zone_size_bytes = 17179869184 + +[[disks]] +disk_id = "00000000000000010000000000000002" +disk_group_id = 101 +node_id = 1 +path = "/opt/crowdb/data/disks/disk-0002.img" +capacity_bytes = 17179869184 +zone_size_bytes = 17179869184 + +[[disks]] +disk_id = "00000000000000010000000000000003" +disk_group_id = 101 +node_id = 1 +path = "/opt/crowdb/data/disks/disk-0003.img" +capacity_bytes = 17179869184 +zone_size_bytes = 17179869184 + +[[disks]] +disk_id = "00000000000000010000000000000004" +disk_group_id = 101 +node_id = 1 +path = "/opt/crowdb/data/disks/disk-0004.img" +capacity_bytes = 17179869184 +zone_size_bytes = 17179869184 + +[[public_endpoints]] +id = "s3" +bind = "0.0.0.0" +port = @ACCESS_SERVER_HTTP_BASE@ + +[[public_endpoints]] +id = "iceberg" +bind = "0.0.0.0" +port = @ACCESS_SERVER_ICEBERG_HTTP_BASE@ + +[[public_endpoints]] +id = "web" +bind = "0.0.0.0" +port = @WEB_BASE@ + +[[services]] +id = "kv" +env = { CROWDB_RDMA_SERVICE_SLOT = "1" } +program = "/opt/crowdb/bin/crowdb-paxos-kv-server" +args = ["--root", "/opt/crowdb/data/kv/node-1", "--config", "/opt/crowdb/run/config/kv.toml", "--management-port", "@PAXOS_KV_SERVER_MGMT_BASE@", "--ports", "@PAXOS_KV_SERVER_RPC_BASE@,@PAXOS_KV_SERVER_RPC_BASE+1@", "--instance-id", "1", "--node-id", "1", "--binding-monitor-interval", "1", "--log-dir", "/opt/crowdb/data/log/kv", "--log"] +dependencies = [] +fence_listeners = ["127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@", "127.0.0.1:@PAXOS_KV_SERVER_RPC_BASE@", "127.0.0.1:@PAXOS_KV_SERVER_RPC_BASE+1@"] +config_template = "/opt/crowdb/etc/templates/kv.toml" +[services.probe] +kind = "http" +target = "http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@/health" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "diskdb" +env = { CROWDB_RDMA_SERVICE_SLOT = "2" } +program = "/opt/crowdb/bin/crowdb-diskdb" +args = ["--config", "/opt/crowdb/run/config/diskdb.toml", "--log-dir", "/opt/crowdb/data/log/diskdb", "--log"] +dependencies = ["kv"] +fence_listeners = ["127.0.0.1:@DISKDB_HTTP_BASE@", "127.0.0.1:@DISKDB_RPC_BASE@"] +config_template = "/opt/crowdb/etc/templates/diskdb.toml" +[services.probe] +kind = "http" +target = "http://127.0.0.1:@DISKDB_HTTP_BASE@/health" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "diskio" +env = { CROWDB_RDMA_SERVICE_SLOT = "3" } +program = "/opt/crowdb/bin/crowdb-diskio" +args = ["--config", "/opt/crowdb/run/config/diskio.toml"] +dependencies = ["kv", "diskdb"] +fence_listeners = ["127.0.0.1:@DISKIO_RPC_BASE@"] +config_template = "/opt/crowdb/etc/templates/diskio.toml" +[services.probe] +kind = "rpc-ping" +target = "127.0.0.1:@DISKIO_RPC_BASE@" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "chunkdb" +env = { CROWDB_RDMA_SERVICE_SLOT = "4" } +program = "/opt/crowdb/bin/crowdb-chunkdb" +args = ["--config", "/opt/crowdb/run/config/chunkdb.toml", "--log-dir", "/opt/crowdb/data/log/chunkdb", "--log"] +dependencies = ["kv", "diskdb", "diskio"] +fence_listeners = ["127.0.0.1:@CHUNKDB_HTTP_BASE@", "127.0.0.1:@CHUNKDB_RPC_BASE@"] +config_template = "/opt/crowdb/etc/templates/chunkdb.toml" +[services.probe] +kind = "http" +target = "http://127.0.0.1:@CHUNKDB_HTTP_BASE@/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "chunk-kv" +env = { CROWDB_RDMA_SERVICE_SLOT = "5" } +program = "/opt/crowdb/bin/crowdb-chunk-kv-server" +args = ["--config", "/opt/crowdb/run/config/chunk-kv.toml", "--log-dir", "/opt/crowdb/data/log/chunk-kv", "--log"] +dependencies = ["kv", "chunkdb", "diskio"] +fence_listeners = ["127.0.0.1:@CHUNK_KV_HTTP_BASE@", "127.0.0.1:@CHUNK_KV_RPC_BASE@"] +config_template = "/opt/crowdb/etc/templates/chunk-kv.toml" +[services.probe] +kind = "http" +target = "http://127.0.0.1:@CHUNK_KV_HTTP_BASE@/health" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "access" +program = "/opt/crowdb/bin/crowdb-access-server" +args = ["--config", "/opt/crowdb/run/config/access.toml"] +env = { CROWDB_S3_PUBLIC_URI = "http://localhost:@ACCESS_SERVER_HTTP_BASE@", CROWDB_S3_REGION = "us-east-1", CROWDB_ICEBERG_PUBLIC_URI = "http://localhost:@ACCESS_SERVER_ICEBERG_HTTP_BASE@", CROWDB_DATASET_PUBLIC_URI = "http://localhost:@ACCESS_SERVER_DATASET_HTTP_BASE@", CROWDB_ICEBERG_GC_ENABLED = "0", CROWDB_MANAGEMENT_SEEDS = "http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@", CROWDB_ACCESS_HEALTH_LISTEN = "127.0.0.1:@ACCESS_SERVER_HEALTH_HTTP_BASE@", CROWDB_ACCESS_LOG_DIR = "/opt/crowdb/data/log/access", CROWDB_RDMA_SERVICE_SLOT = "6" } +dependencies = ["kv", "chunk-kv", "chunkdb", "diskio"] +fence_listeners = ["127.0.0.1:@ACCESS_SERVER_ICEBERG_HTTP_BASE@", "127.0.0.1:@ACCESS_SERVER_HTTP_BASE@", "127.0.0.1:@ACCESS_SERVER_DATASET_HTTP_BASE@", "127.0.0.1:@ACCESS_SERVER_HEALTH_HTTP_BASE@"] +config_template = "/opt/crowdb/etc/templates/access.toml" +[services.probe] +kind = "http" +target = "http://127.0.0.1:@ACCESS_SERVER_ICEBERG_HTTP_BASE@/v1/config" +bearer_env = "CROWDB_ICEBERG_READ_TOKEN" +timeout_ms = 1000 +failure_threshold = 5 +[[services.additional_probes]] +kind = "http" +target = "http://127.0.0.1:@ACCESS_SERVER_HEALTH_HTTP_BASE@/_crowdb/health/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "web" +env = { CROWDB_RDMA_SERVICE_SLOT = "7" } +program = "/opt/crowdb/bin/crowdb-web" +args = ["--config", "/opt/crowdb/run/config/crowdb-web.toml", "--node-monitor", "http://127.0.0.1:@MONITOR_HTTP_BASE@"] +dependencies = ["access"] +fence_listeners = ["127.0.0.1:@WEB_BASE@"] +config_template = "/opt/crowdb/etc/templates/crowdb-web.toml" +[services.probe] +kind = "http" +target = "http://127.0.0.1:@WEB_BASE@/healthz" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 diff --git a/container/single-node-container/templates/access.toml b/container/single-node-container/templates/access.toml index 961a58459..c7b0b2bd7 100644 --- a/container/single-node-container/templates/access.toml +++ b/container/single-node-container/templates/access.toml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. # Access configuration shared by the S3 and Iceberg listeners. # Secrets stay in /opt/crowdb/data/secrets/server.env. @@ -6,7 +7,7 @@ mode = "local" max_node_failures = 0 [common] -management_seeds = ["http://127.0.0.1:10000"] +management_seeds = ["http://127.0.0.1:10135"] diskio_connections_per_endpoint = 1 diskio_rpc_workers = 1 @@ -32,7 +33,7 @@ chunk_capacity_bytes = 67108864 [s3] large_mirror_copies = 1 -listen = "0.0.0.0:9091" +listen = "0.0.0.0:7135" tenant = "preview" region = "us-east-1" small_object_limit = 8388608 @@ -55,7 +56,7 @@ chunk_capacity_bytes = 67108864 [iceberg] large_mirror_copies = 1 -listen = "0.0.0.0:9092" +listen = "0.0.0.0:8135" native_budget_bytes = 268435456 max_chunk_size = 67108864 @@ -86,7 +87,7 @@ minimum_retention_ms = 604800000 catalogs = [] [dataset] -listen = "0.0.0.0:9093" +listen = "0.0.0.0:9135" max_in_flight = 8 request_timeout_seconds = 60 max_batch_size = 10000 diff --git a/container/single-node-container/templates/access.toml.ports.in b/container/single-node-container/templates/access.toml.ports.in new file mode 100644 index 000000000..69007bc6d --- /dev/null +++ b/container/single-node-container/templates/access.toml.ports.in @@ -0,0 +1,92 @@ +# Access configuration shared by the S3 and Iceberg listeners. +# Secrets stay in /opt/crowdb/data/secrets/server.env. + +[deployment] +mode = "local" +max_node_failures = 0 + +[common] +management_seeds = ["http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@"] +diskio_connections_per_endpoint = 1 +diskio_rpc_workers = 1 + +[read] +stream_window_bytes = 1048576 +stream_slots = 3 +global_stream_bytes = 268435456 +recovery_memory_bytes = 268435456 + +[small_write] +threshold_ratio = 0.9 +disk_block_bytes = 1048576 +conversion_enabled = false +mirror_copies = 1 +ec_data = 2 +ec_code = 1 +memory_budget_bytes = 1342177280 +queue_capacity = 1024 +min_pipelines = 1 +max_pipelines = 32 +max_batch_bytes = 1048576 +chunk_capacity_bytes = 67108864 + +[s3] +large_mirror_copies = 1 +listen = "0.0.0.0:@ACCESS_SERVER_HTTP_BASE@" +tenant = "preview" +region = "us-east-1" +small_object_limit = 8388608 +list_scan_items = 1001 +list_scan_bytes = 4194304 +continuation_ttl_seconds = 900 +native_budget_bytes = 268435456 +cleanup_backlog_limit = 10000 +ec_data = 2 +ec_code = 1 +max_chunk_size = 67108864 + +[s3.small_write] +conversion_enabled = false +mirror_copies = 1 +disk_block_bytes = 1048576 +ec_data = 2 +ec_code = 1 +chunk_capacity_bytes = 67108864 + +[iceberg] +large_mirror_copies = 1 +listen = "0.0.0.0:@ACCESS_SERVER_ICEBERG_HTTP_BASE@" +native_budget_bytes = 268435456 +max_chunk_size = 67108864 + +[iceberg.small_write] +conversion_enabled = false +mirror_copies = 1 +disk_block_bytes = 1048576 +ec_data = 2 +ec_code = 1 +chunk_capacity_bytes = 33554432 + +[iceberg.gc] +enabled = false +interval_ms = 60000 +step_bytes = 8388608 +step_ms = 1000 +page_items = 64 +page_bytes = 49152 +concurrency = 1 +kv_bytes = 67108864 +kv_requests = 128 +chunk_bytes = 8388608 +chunk_requests = 128 +retry_base_ms = 1000 +retry_max_ms = 60000 +corruption_attempts = 3 +minimum_retention_ms = 604800000 +catalogs = [] + +[dataset] +listen = "0.0.0.0:@ACCESS_SERVER_DATASET_HTTP_BASE@" +max_in_flight = 8 +request_timeout_seconds = 60 +max_batch_size = 10000 diff --git a/container/single-node-container/templates/chunk-kv.toml b/container/single-node-container/templates/chunk-kv.toml index 6c94e9c27..035a58e29 100644 --- a/container/single-node-container/templates/chunk-kv.toml +++ b/container/single-node-container/templates/chunk-kv.toml @@ -1,10 +1,11 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. instance_id = 1 node_id = {{node.0.id}} -rpc_listen_addr = "127.0.0.1:15200" -rpc_advertise_addr = "127.0.0.1:15200" -http_listen_addr = "127.0.0.1:15100" -http_advertise_addr = "127.0.0.1:15100" -group0_mgmt_seeds = ["http://127.0.0.1:10000"] +rpc_listen_addr = "127.0.0.1:21135" +rpc_advertise_addr = "127.0.0.1:21135" +http_listen_addr = "127.0.0.1:20135" +http_advertise_addr = "127.0.0.1:20135" +group0_mgmt_seeds = ["http://127.0.0.1:10135"] catalog_refresh_interval_ms = 200 rpc_workers = 1 diff --git a/container/single-node-container/templates/chunk-kv.toml.ports.in b/container/single-node-container/templates/chunk-kv.toml.ports.in new file mode 100644 index 000000000..7849d209f --- /dev/null +++ b/container/single-node-container/templates/chunk-kv.toml.ports.in @@ -0,0 +1,32 @@ +instance_id = 1 +node_id = {{node.0.id}} +rpc_listen_addr = "127.0.0.1:@CHUNK_KV_RPC_BASE@" +rpc_advertise_addr = "127.0.0.1:@CHUNK_KV_RPC_BASE@" +http_listen_addr = "127.0.0.1:@CHUNK_KV_HTTP_BASE@" +http_advertise_addr = "127.0.0.1:@CHUNK_KV_HTTP_BASE@" +group0_mgmt_seeds = ["http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@"] +catalog_refresh_interval_ms = 200 +rpc_workers = 1 + +[balance] +enabled = false +target_partitions_per_owner = 1 +target_partition_bytes = 9223372036854775807 +minimum_weighted_improvement_percent = 100 +cooldown_ms = 9223372036854775807 +max_owner_request_rate = 0 + +[storage] +metadata_store_id = 0 +stream_mirror_copies = 1 +tree_chunk_capacity_bytes = 16777216 +stream_chunk_capacity_bytes = 16777216 +diskio_connections_per_endpoint = 1 +diskio_rpc_workers = 1 + +[bootstrap_partition] +partition_id = { high = 1, low = 1 } +tree_id = 1 +stream_name = { high = 2, low = 1 } +owner_epoch = 1 +metadata_group_id = {{group.1.id}} diff --git a/container/single-node-container/templates/chunkdb.toml b/container/single-node-container/templates/chunkdb.toml index 96ca1a04a..836fb26cf 100644 --- a/container/single-node-container/templates/chunkdb.toml +++ b/container/single-node-container/templates/chunkdb.toml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. [slot_bootstrap] service_instances = [1] storage_groups = [{ store_id = 0, group_id = 1 }] @@ -8,10 +9,10 @@ max_node_failures = 0 [server] rpc_workers = 1 -http_listen_addr = "127.0.0.1:12100" -rpc_listen_addr = "127.0.0.1:12200" +http_listen_addr = "127.0.0.1:16135" +rpc_listen_addr = "127.0.0.1:17135" instance_id = "1" -kv_server_mgmt_seeds = ["http://127.0.0.1:10000"] +kv_server_mgmt_seeds = ["http://127.0.0.1:10135"] keepalive_interval_secs = 1 kv_pool_size = 1 kv_rpc_workers = 1 diff --git a/container/single-node-container/templates/chunkdb.toml.ports.in b/container/single-node-container/templates/chunkdb.toml.ports.in new file mode 100644 index 000000000..e3961096d --- /dev/null +++ b/container/single-node-container/templates/chunkdb.toml.ports.in @@ -0,0 +1,38 @@ +[slot_bootstrap] +service_instances = [1] +storage_groups = [{ store_id = 0, group_id = 1 }] + +[deployment] +mode = "local" +max_node_failures = 0 + +[server] +rpc_workers = 1 +http_listen_addr = "127.0.0.1:@CHUNKDB_HTTP_BASE@" +rpc_listen_addr = "127.0.0.1:@CHUNKDB_RPC_BASE@" +instance_id = "1" +kv_server_mgmt_seeds = ["http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@"] +keepalive_interval_secs = 1 +kv_pool_size = 1 +kv_rpc_workers = 1 +diskdb_pool_size = 1 +diskdb_rpc_workers = 1 + +[conversion_io] +normal_connections_per_endpoint = 1 +priority_connections_per_endpoint = 1 +rpc_workers = 1 + +[topology] +refresh_interval_secs = 1 + +[range_guard] +allow_all_when_empty = false + +[lifecycle] +cache_capacity = 10000 +sweep_chunk_lock_interval_secs = 60 +lock_hold_warn_threshold_ms = 1000 + +[placement] +mode = "unsafe_colocated" diff --git a/container/single-node-container/templates/crowdb-web.toml b/container/single-node-container/templates/crowdb-web.toml index e5b500aaa..fa812c214 100644 --- a/container/single-node-container/templates/crowdb-web.toml +++ b/container/single-node-container/templates/crowdb-web.toml @@ -1,8 +1,9 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. version = 1 mode = "docker" bind = "0.0.0.0" -port = 9090 -group0_management_seeds = ["http://127.0.0.1:10000"] +port = 6135 +group0_management_seeds = ["http://127.0.0.1:10135"] ui_root = "{{install_root}}/ui" monitor_status = "{{run_root}}/status/monitor.json" log_dir = "{{log_root}}/web" diff --git a/container/single-node-container/templates/crowdb-web.toml.ports.in b/container/single-node-container/templates/crowdb-web.toml.ports.in new file mode 100644 index 000000000..9e1bd69d9 --- /dev/null +++ b/container/single-node-container/templates/crowdb-web.toml.ports.in @@ -0,0 +1,10 @@ +version = 1 +mode = "docker" +bind = "0.0.0.0" +port = @WEB_BASE@ +group0_management_seeds = ["http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@"] +ui_root = "{{install_root}}/ui" +monitor_status = "{{run_root}}/status/monitor.json" +log_dir = "{{log_root}}/web" +log_max_file_mb = 30 +log_max_files = 5 diff --git a/container/single-node-container/templates/diskdb.toml b/container/single-node-container/templates/diskdb.toml index 039ab6e49..f1dfddf00 100644 --- a/container/single-node-container/templates/diskdb.toml +++ b/container/single-node-container/templates/diskdb.toml @@ -1,12 +1,12 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. [server] rpc_workers = 1 kv_pool_size = 1 kv_rpc_workers = 1 -listen_addr = "127.0.0.1:11000" -http_listen_addr = "127.0.0.1:11100" -rpc_listen_addr = "127.0.0.1:11200" +http_listen_addr = "127.0.0.1:13135" +rpc_listen_addr = "127.0.0.1:14135" instance_id = "{{node.0.id}}" -kv_server_mgmt_seeds = ["http://127.0.0.1:10000"] +kv_server_mgmt_seeds = ["http://127.0.0.1:10135"] [storage] zone_size_bytes = {{disk.0.capacity_bytes}} diff --git a/container/single-node-container/templates/diskdb.toml.ports.in b/container/single-node-container/templates/diskdb.toml.ports.in new file mode 100644 index 000000000..ccea9e6ec --- /dev/null +++ b/container/single-node-container/templates/diskdb.toml.ports.in @@ -0,0 +1,64 @@ +[server] +rpc_workers = 1 +kv_pool_size = 1 +kv_rpc_workers = 1 +http_listen_addr = "127.0.0.1:@DISKDB_HTTP_BASE@" +rpc_listen_addr = "127.0.0.1:@DISKDB_RPC_BASE@" +instance_id = "{{node.0.id}}" +kv_server_mgmt_seeds = ["http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@"] + +[storage] +zone_size_bytes = {{disk.0.capacity_bytes}} +block_size_bytes = 1048576 +allocate_granularity = 1048576 +zone_rotate_count = 4 +cas_retry_limit = 100 + +[heartbeat] +interval_secs = 10 +miss_threshold = 3 +temp_failure_timeout_secs = 900 + +[persistence] +free_batch_enabled = false +free_flush_max_batch = 256 +compaction_cadence_secs = 300 +snapshot_compaction_threshold = 4096 +load_concurrency = 16 + +[scanner] +scan_interval_secs = 600 +tentative_owner_scan_interval_secs = 600 +tentative_owner_zone_delay_secs = 3 +tentative_owner_grace_secs = 86400 +reverify_delay_ms = 1000 + +[scanner.ghost] +detect = true +auto_correct = false + +[scanner.integrity] +verify = true +detect_owner_mismatch = false + +[rebalance] +enabled = true +plan_interval_secs = 300 +imbalance_threshold_pct = 20 +max_jobs_per_cycle = 1 +zone_delay_secs = 3 + +[allocator] +load_aware = true +load_aware_weight = "free_bytes" + +[sync] +group0_store_id = 0 +group0_group_id = 0 +sync_interval_secs = 10 + +[reporting] +interval_secs = 10 + +[notify] +notify_enabled = false diff --git a/container/single-node-container/templates/diskio.toml b/container/single-node-container/templates/diskio.toml index 854290858..cc62eda7b 100644 --- a/container/single-node-container/templates/diskio.toml +++ b/container/single-node-container/templates/diskio.toml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. [server] bind_address = "127.0.0.1" listen_port = 13000 @@ -11,7 +12,7 @@ thread_pool_size = 1 sq_entries = 256 [group0] -kv_seeds = ["http://127.0.0.1:10000"] +kv_seeds = ["http://127.0.0.1:10135"] instance_id = {{node.0.id}} rack_id = {{node.0.rack_id}} disk_group_id = 101 diff --git a/container/single-node-container/templates/diskio.toml.ports.in b/container/single-node-container/templates/diskio.toml.ports.in new file mode 100644 index 000000000..fdf18b4ac --- /dev/null +++ b/container/single-node-container/templates/diskio.toml.ports.in @@ -0,0 +1,43 @@ +[server] +bind_address = "127.0.0.1" +listen_port = 13000 +rpc_workers = 1 +node_id = {{node.0.id}} +dummy_disk_type = "null" +o_direct = true + +[engine] +thread_pool_size = 1 +sq_entries = 256 + +[group0] +kv_seeds = ["http://127.0.0.1:@PAXOS_KV_SERVER_MGMT_BASE@"] +instance_id = {{node.0.id}} +rack_id = {{node.0.rack_id}} +disk_group_id = 101 +sync_interval_ms = 5000 +auto_discover_disks = false + +[metrics] +log_dir = "{{log_root}}/diskio" +interval_secs = 5 + +[[disk]] +id = "1:1" +path = "{{disk.0.path}}" +zone_capacity = {{disk.0.capacity_bytes}} + +[[disk]] +id = "1:2" +path = "{{disk.1.path}}" +zone_capacity = {{disk.1.capacity_bytes}} + +[[disk]] +id = "1:3" +path = "{{disk.2.path}}" +zone_capacity = {{disk.2.capacity_bytes}} + +[[disk]] +id = "1:4" +path = "{{disk.3.path}}" +zone_capacity = {{disk.3.capacity_bytes}} diff --git a/container/single-node-container/templates/kv.toml b/container/single-node-container/templates/kv.toml index 54006f518..3485509ab 100644 --- a/container/single-node-container/templates/kv.toml +++ b/container/single-node-container/templates/kv.toml @@ -1,3 +1,4 @@ +# Generated by tools/ports/generate.py from ports.rs; edit the .in template. wal_early_ack = true async_engine_apply = true diff --git a/container/single-node-container/templates/kv.toml.ports.in b/container/single-node-container/templates/kv.toml.ports.in new file mode 100644 index 000000000..54006f518 --- /dev/null +++ b/container/single-node-container/templates/kv.toml.ports.in @@ -0,0 +1,6 @@ +wal_early_ack = true +async_engine_apply = true + +[server] +rpc_workers = 1 +peer_pool_size = 1 diff --git a/container/single-node-container/tests/image-smoke.sh b/container/single-node-container/tests/image-smoke.sh index cd10a3fa9..7c862233f 100644 --- a/container/single-node-container/tests/image-smoke.sh +++ b/container/single-node-container/tests/image-smoke.sh @@ -17,12 +17,17 @@ test "$(docker image inspect --format '{{index .Config.Labels "org.opencontainer volumes=$(docker image inspect --format '{{json .Config.Volumes}}' "$image") jq -e 'has("/opt/crowdb/data")' <<<"$volumes" >/dev/null exposed=$(docker image inspect --format '{{json .Config.ExposedPorts}}' "$image") -for port in 9092 9091 9090 9095 2222; do - jq -e --arg port "$port/tcp" 'has($port)' <<<"$exposed" >/dev/null -done -for port in 10000 13000 15200; do - jq -e --arg port "$port/tcp" 'has($port) | not' <<<"$exposed" >/dev/null -done +PYTHONPATH=tools/ports python - "$exposed" <<'PY' +import json +import sys +import default_ports + +expected = {f"{port}/tcp" for name, port in vars(default_ports).items() + if name.endswith("_BASE") or name == "SSH_PORT"} +actual = set(json.loads(sys.argv[1])) +if actual != expected: + raise SystemExit(f"image ports differ: missing={expected - actual}, extra={actual - expected}") +PY docker run --rm --network none --entrypoint /opt/crowdb/bin/crowdb-monitor "$image" validate /opt/crowdb/etc/profile.toml docker run --rm --network none --entrypoint /bin/sh "$image" -ec ' diff --git a/doc/backlog/R102-diskdb-dynamic-binding-migration.md b/doc/backlog/R102-diskdb-dynamic-binding-migration.md deleted file mode 100644 index 1c4991137..000000000 --- a/doc/backlog/R102-diskdb-dynamic-binding-migration.md +++ /dev/null @@ -1,412 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R102: diskdb — Dynamic Disk-Group Binding Migration - -## Problem - -### Current behavior + impact - -diskdb disk-group → KV-group binding is operator-manual today: the -operator writes `BindMapValue` (disk-group → `(store_id, group_id)`) -and `OwnerMapValue` (disk-group → diskdb instance) via the console -through `HardwareClient` (design §5 "Map semantics"). Three gaps -remain for cluster-expansion scenarios: - -1. **No safe migration flow on bind change.** When an existing - disk-group is rebound to a different KV-group, the operator - writes a new `BindMapValue` directly — but the disk-group's zone - records (busy/free/snapshot) are physically stored on the old - KV-group, keyed by `disk_id` (no `disk_group_id`/`group_id` in - the key). The diskdb sync loop silently overwrites `dg.bind`, so - recovery/compaction/scanner immediately scan the new KV-group and - conclude old busy blocks are free (data corruption), and frees of - old blocks target the wrong group (leaks). The existing disk-move - code (`http_move_disk` + `copy_disk_records`) has the right shape - (Maintenance quiescence + copy + flip) but uses a fixed 10 s sleep - with a latent free/compaction race during the copy, and is - disk-scope only — there is no disk-group-scope bind-change flow. -2. **No binding client or enforcement.** Clients route zone-record - writes to a KV-group with no check that the disk-group is actually - bound there; a stale route silently corrupts. diskdb instances - accept RPCs for any disk-group with no `NotMyBinding` reject — a - misrouted write goes to the wrong KV-group undetected. -3. **No imbalance visibility.** The operator has no warning when the - disk-group → KV-group distribution is significantly imbalanced - (e.g. one KV-group hosts far more disk-groups than others after - uneven growth). Rebalancing is left to operator vigilance. - -R99 built the common binding framework (binding client, binding -monitor, reject-and-retry protocol) for chunkdb instance sharding. -GAP-R99-5 decided that diskdb disk-group → KV-group rebinding should -be a separate follow-up requirement. This is R102: diskdb reuses the -binding client + enforcement + reject-and-retry protocol (work items -1-3) and the binding monitor (work item 4, warning-only — diskdb bind -changes are rare; auto-rebalancing is appropriate for chunkdb but -not for diskdb), and adds the safe migration flow (work item 5) -shared between disk-group bind change and disk move. - -### Design pointers - -- `doc/design/diskdb/design-crowdb-diskdb.md` §3.2 (disk-group → paxos - group binding via a table, not hash — dynamic scaling without - rehashing), §5 ("Map semantics" — `OwnerMapValue`, - `BindMapValue` are operator-manual today). -- `doc/design/kv/design-crowdb-kv-group0.md` §2.1 (`crowdb-kv-client` is - the single sysdata API surface), §2.6 (two monitoring models: push - + pull). -- `doc/backlog/R99-kv-dynamic-range-binding-framework.md` — common - binding framework (`RangeBindingClient`, binding monitor, - reject-and-retry protocol). - -### Use scenarios - -diskdb bind changes are rare in practice — the common capacity- -expansion patterns do not require rebinding: - -- **New node + new KV-group (common, no migration)** — operator adds - a new node, creates a new KV-group, adds the node's disk-groups, - and binds them to the new KV-group at creation time. The disk-groups - are new (no records); the bind is set before any writes. No - migration needed. R102's binding client + enforcement (work items - 1-3) ensure clients route to the correct KV-group from the start. -- **New node, KV-group replica moved (common, no diskdb migration)** - — KV-groups are pre-created (replicas on existing nodes); adding a - node means moving a KV-group replica to the new node (Paxos - reconfiguration — journal replication to the new replica). The - disk-group → KV-group binding does not change; diskdb records stay - on the same KV-group. This is a KV-layer migration, transparent to - diskdb. Not R102's scope. -- **Rebind existing disk-group to a different KV-group (rare, - requires migration)** — an existing disk-group with live records is - rebound from KV-group A to KV-group B (e.g. to evacuate a KV-group - that is being retired, or to rebalance after uneven growth). The - operator triggers the rebind via the console; the console drives - the five-step migration flow (work item 5). This is the case that - requires copying records. -- **diskdb instance crashes (rare, ownership reassignment)** — a - diskdb instance crashes (service registry expiry); the monitor - warns about the lost instance. The operator reassigns its - disk-groups to a surviving instance (updating `OwnerMapValue`). If - the surviving instance is on the same KV-group, no record migration - is needed (just an ownership change). If on a different KV-group, - the operator triggers the five-step migration flow. -- **Monitor warns about imbalance** — the binding monitor observes - the disk-group → KV-group distribution and emits a warning when it - is significantly imbalanced (e.g. one KV-group hosts far more - disk-groups than others). The operator decides whether to act - (trigger rebinds via the console) or ignore. The monitor does NOT - auto-trigger rebinding for diskdb — unlike chunkdb (R99), where - auto-rebalancing is appropriate. - -## Solution - -Reuse R99's common binding framework for the binding client + -enforcement + reject-and-retry protocol (work items 1-3). The binding -monitor (work item 4) observes the disk-group → KV-group distribution -and emits imbalance warnings, but does NOT auto-trigger rebinding for -diskdb — rebinding is operator-triggered via the console (diskdb bind -changes are rare; auto-rebalancing is appropriate for chunkdb but not -for diskdb). Bind change and disk move share one five-step migration -flow (Maintenance → copy → switch → cleanup → Up) at different scopes -(work item 5), reusing the existing `http_move_disk` + -`copy_disk_records` pattern. - -### Work items - -1. **diskdb binding schema** — `lib/crowdb-protocol/src/fbs/common_type.fbs` - (extend): add `DiskdbBindingValue` (disk-group-id → paxos-group-id - `(store_id, group_id)` + diskdb instance endpoint); stored in - group-0 with key pattern `/diskdb/dg_bind/<disk_group_id>`. Add a - `MigrationIntentValue` carrying the migration scope (disk-group or - single disk), `old_bind` `(store_id, group_id)`, `new_bind` - `(store_id, group_id)`, and optional `disk_id` (present for - disk-level moves, absent for disk-group bind changes); stored at - `/diskdb/migrate/<disk_group_id>` (disk-group scope) or - `/diskdb/migrate/<disk_group_id>/<disk_id_hex>` (disk scope). The - intent serves double duty: a write-quiescence signal (the diskdb - instance reads it and suspends frees + compaction for the affected - disk/disk-group during the copy, before the bind flip) and a - crash-recovery marker (a new monitor leader reads it and resumes). - No `Copying`/`Cutover`/`Complete` state enum — progress is derived - from comparing the current `BindMapValue` to `new_bind` (copy done - iff bind already points to `new_bind`). Add - `ERROR_CODE_NOT_MY_BINDING` to `ret_code.fbs` for the - reject-and-retry protocol. -2. **diskdb binding client** — `lib/crowdb-kv-client/src/` (extend): - add `DiskdbBindingClient` (fetch + cache + watch/notify), - following R99's `RangeBindingClient` pattern. Caches - `DashMap<disk_group_id, (store_id, group_id, endpoint)>`, - subscribes to watch/notify for real-time updates. Provides - `route(disk_group_id) -> (store_id, group_id, endpoint)` for - clients to route zone-record writes to the correct paxos group. - Reject-and-retry: on `NotMyBinding` error, refresh the binding - cache, re-route, retry — follows the `NotLeaderHint` retry pattern - in `crowdb-kv-client/src/config.rs`. -3. **diskdb server binding enforcement** — `app/crowdb-diskdb/src/` - (extend): add a binding guard (following `range_guard.rs` pattern - from chunkdb). On every RPC, check the disk-group is bound to this - instance's paxos group; reject with `NotMyBinding` error if not. - Binding is read from group-0 at startup + updated via - watch/notify. The existing `DdbDiskGroup.bind` field - (`model/disk_group.rs:32`, `RwLock<(u64, u64)>`) is populated from - the binding client instead of operator config. -4. **Monitor integration (warning-only)** — the binding monitor - (relocated to `crowdb-kv-server` per R99 rework, - `app/crowdb-kv-server/src/`) observes the disk-group → KV-group - distribution alongside chunkdb range binding; same leader-gated - background task. For diskdb, the monitor computes the current - distribution and emits an imbalance warning (log + metric) when - one KV-group hosts significantly more disk-groups than others, or - when a diskdb instance is absent from the service registry (crash - suspicion). The monitor does NOT auto-trigger rebinding or - migration for diskdb — unlike chunkdb (R99) where auto-rebalancing - is appropriate. Rebinding is operator-triggered via the console - (work item 5's flow). The monitor's warning gives the operator the - signal to act; the console provides the safe migration path. -5. **Migration flow** — disk-group bind change and disk move share - one flow at different scopes (whole disk-group vs one disk), reusing - the existing `http_move_disk` + `copy_disk_records` pattern - (`app/crowdb-web/src/lifecycle.rs`). Both are operator-triggered via - the console (the monitor only warns; it does not drive migration). - The flow is five steps: - (a) **Quiesce** — the console sets the disk-group (or disk) to - `Maintenance` in group 0 and writes a `MigrationIntentValue`; - allocates block (`permits(Maintenance, Allocate) = false`); the - diskdb instance reads the intent on the next sync tick (or via - watch/notify) and suspends frees + compaction for the affected - disk/disk-group; reads still serve from the old bind (`dg.bind` - unchanged). - (b) **Copy** — `copy_disk_records(old_bind, new_bind, disk_id)` - per disk (one disk for disk move; every disk in the group for - bind change): prefix-scan `ZoneKey`/`BusyBlockKey`/`FreeBlockKey` - by `DiskId` from the old bind, batch-write to the new bind. - Idempotent — re-running overwrites the same keys, so crash-resume - is safe. - (c) **Switch** — write the new `BindMapValue` to group 0 (bind - change) or add the disk to its new disk-group placement (disk - move, whose new disk-group already has its own bind). The next - sync tick updates `dg.bind` → new bind; reads + writes now target - the new bind. The diskdb instance clears its in-memory migrating - flag (frees + compaction resume on the new bind). - (d) **Cleanup** — delete the migrated records from the old bind - (per-disk prefix scan + `batch_write` of `Delete` ops). For disk - move, only the moved disk's records (other disks still use the - old bind); for bind change, all disks in the group. - (e) **Resume** — set the disk-group (or disk) to `Up` and delete - the `MigrationIntentValue`; allocates resume on the new bind. - No `Copying`/`Cutover`/`Complete` state machine, no dual-reads — - `Maintenance` is the quiescence signal, the bind write is the - cutover, the intent deletion is the completion marker. - -### Flow diagram - -``` - driver: console (crowdb-web), operator-triggered - (monitor only warns — does not drive migration) - │ - (a) QUIESCE - ├── set disk-group (or disk) to Maintenance in group 0 - ├── write MigrationIntentValue { old_bind, new_bind, disk_id? } to group 0 - ▼ - watch/notify / next sync tick → diskdb instance - ├── allocates blocked (Maintenance) - ├── suspends frees + compaction for affected disk/disk-group (intent) - └── reads still serve from old bind (dg.bind unchanged) - │ - (b) COPY — copy_disk_records(old_bind, new_bind, disk_id) per disk - ├── prefix-scan ZoneKey / BusyBlockKey / FreeBlockKey by DiskId from old bind - └── batch_write Put to new bind (idempotent — safe to re-run on resume) - │ - (c) SWITCH - ├── bind change: write new BindMapValue to group 0 - ├── disk move: add disk to new placement (new disk-group's bind already set) - ▼ - next sync tick → diskdb updates dg.bind → new bind - ├── reads + writes now target new bind - └── diskdb clears migrating flag (frees + compaction resume on new bind) - │ - (d) CLEANUP — delete migrated records from old bind (per-disk prefix scan + batch_delete) - │ - (e) RESUME - ├── set disk-group (or disk) to Up - └── delete MigrationIntentValue → allocates resume on new bind -``` - -### Edge cases at a glance - -- **disk-group with no data** — fast flip: copy is a no-op (no zone - records to scan), switch + cleanup + resume proceed immediately. -- **Console crash mid-migration** — the `MigrationIntentValue` - persists in group-0; the operator retries (or a new console - session reads the intent) and resumes from the appropriate step - (re-run copy — idempotent; then switch; then cleanup; then - resume). The diskdb instance keeps the migrating flag set (frees - + compaction stay suspended) until the intent is deleted. -- **Free or compaction during copy** — the diskdb instance's - migrating flag (from the intent) suspends frees + compaction for - the affected disk/disk-group before the bind flip, so no - `Delete BusyBlockKey + Put FreeBlockKey` or compaction - `batch_write` races with the copy. After the flip, frees + - compaction resume on the new bind and are correct. -- **Concurrent rebinding requests** — rebinding is operator-triggered - via the console; the console serializes bind changes and disk moves - on the same disk-group via the `MigrationIntentValue` (one intent - per scope key; the console rejects a second intent while one - exists). The monitor does not trigger concurrent migrations (it - only warns). -- **Disk move to a disk-group on the same kv-group** — copy is a - no-op (old_bind == new_bind); the flow still quiesces, flips the - placement, and resumes. Detected by comparing `old_bind` and - `new_bind` in the intent. - -## Dependencies - -- Depends on **R99** — common binding framework (`RangeBindingClient` - pattern, binding monitor, reject-and-retry protocol). -- **R99 rework** (Phase 4-5 of `doc/working/plan-gaps-r99-r100.md`) - must land first — the binding monitor must be relocated to - `crowdb-kv-server` (leader-gated background task on group-0 - replicas) before diskdb rebinding can be added to the same - monitor. -- **R103** (chunkdb range migration) is a sibling requirement. R102 - and R103 no longer share a migration state machine — chunkdb holds - user chunk data and may need online dual-read migration (R103), - while diskdb is a metadata allocator whose brief write pause with - retry is tolerable, so R102 uses the simpler quiesce + copy + flip - flow. They can still proceed in parallel. -- **Disk move** (`app/crowdb-web/src/lifecycle.rs:1630` - `http_move_disk` + `copy_disk_records`) is the existing - disk-scope migration; R102 generalizes the same flow to - disk-group scope and adds the `MigrationIntentValue` for - write-quiescence + crash safety (the current disk-move code uses - a fixed 10 s sleep and has a latent free/compaction race during - the copy — R102 fixes both by sharing the intent-based flow). - -## Acceptance - -### diskdb binding schema - -- `DiskdbBindingValue` encodes `disk_group_id` → `(store_id, - group_id)` + instance endpoint → round-trip serialize/deserialize - preserves all fields. `[Unit test]` -- `MigrationIntentValue` encodes scope (disk-group or disk) + - `old_bind` + `new_bind` + optional `disk_id` → round-trip - preserves all fields. `[Unit test]` -- Binding value stored at `/diskdb/dg_bind/<disk_group_id>` in - group-0 → `DiskdbBindingClient` fetches and decodes it correctly. - `[Integration test]` -- Migration intent stored at `/diskdb/migrate/<disk_group_id>` (or - `.../<disk_id_hex>`) → console + diskdb instance fetch and decode - it correctly. `[Integration test]` - -### diskdb binding client - -- `DiskdbBindingClient::route(disk_group_id)` with a cached binding - → returns the correct `(store_id, group_id, endpoint)`. `[Unit test]` -- Binding table updated in group-0 → watch/notify fires → - `DiskdbBindingClient` cache updates within 1s → next `route` - returns the new binding. `[Integration test]` -- Client receives `NotMyBinding` → refreshes binding cache → - re-routes → retries against the correct paxos group → succeeds. - `[Integration test]` - -### diskdb server binding enforcement - -- diskdb instance receives an RPC for a disk-group bound to its - paxos group → binding guard allows → RPC processes normally. - `[Unit test]` -- diskdb instance receives an RPC for a disk-group NOT bound to its - paxos group → binding guard rejects with `NotMyBinding` including - the current binding (new paxos group + endpoint). `[Integration test]` -- `DdbDiskGroup.bind` is populated from the binding client at startup - (not operator config) → assert `bind` matches group-0 value. - `[Integration test]` - -### Monitor integration (warning-only) - -- The disk-group → KV-group distribution is significantly imbalanced - (one KV-group hosts far more disk-groups than others) → monitor - emits an imbalance warning (log + metric) within the polling - interval → assert no `BindMapValue` or `OwnerMapValue` writes from - the monitor. `[Integration test]` -- A diskdb instance is absent from the service registry (crash - suspicion) → monitor emits a warning → assert no auto-rebind of - its disk-groups (the operator triggers reassignment via the - console). `[Integration test]` -- Monitor follower (non-leader) does not emit warnings or perform - writes → assert no binding writes + no warning metrics from - followers. `[Integration test]` - -### Migration flow - -- disk-group bind change (A → B): console sets disk-group to - `Maintenance` + writes `MigrationIntentValue` → diskdb suspends - frees + compaction → `copy_disk_records` copies all disks' - records from A to B → console writes new `BindMapValue` → diskdb - updates `dg.bind` → B → console deletes old records from A → - console sets disk-group to `Up` + deletes intent → allocates - resume on B. `[Integration test]` -- disk move (old disk-group → new disk-group on a different - kv-group): same five-step flow at disk scope — disk set to - `Maintenance`, one disk's records copied, disk added to new - placement, old records deleted, disk set to `Up`. Reuses - `copy_disk_records`. `[Integration test]` -- disk-group with no data: copy is a no-op (no records to scan) → - switch + cleanup + resume proceed immediately → new bind serves - allocates. `[Integration test]` -- disk move to a disk-group on the same kv-group (`old_bind == - new_bind`): copy is skipped → placement flip + resume proceed. - `[Integration test]` -- E2E test: full bind-change lifecycle driven by the console with - live clients → assert no permanent request failures beyond - transient `NotMyBinding` redirects, and no busy-block records lost - (a block allocated before the migration is still busy on the new - bind after resume). `[E2E test]` - -### Edge cases - -- Console crashes mid-migration → `MigrationIntentValue` persists → - operator retries (or a new console session reads the intent) → - re-runs copy (idempotent) → completes switch + cleanup + resume. - `[Integration test]` -- Free or compaction attempted during copy (before bind flip) → - diskdb migrating flag suspends it → no `Delete BusyBlockKey` or - compaction `batch_write` races with the copy → records on new - bind match old bind exactly. `[Integration test]` -- Concurrent rebinding requests for different disk-groups → console - serializes via per-scope intents → both complete without conflict. - `[Integration test]` -- Second migration intent for the same scope while one exists → - console rejects (one intent per scope key) → no conflicting - concurrent migrations. `[Integration test]` -- Client with stale binding cache sends write to old paxos group - after the flip → `NotMyBinding` → client refreshes cache + - retries on new paxos group successfully. `[Integration test]` - -### Build & style - -- `pixi run test-diskdb` -- `pixi run test-kv-server` -- `pixi run cargo fmt --all -- --check` -- `pixi run cargo clippy --all-targets -- -D warnings` - -## Open Questions - -- **Copy data or just rebind?** Resolved: **copy is required.** Zone - records are physically stored on the disk-group's bound kv-group - (`dg.bind` routes every `persist_busy`/`persist_free`/`put_zone`/ - recovery/compaction/scanner read+write). Record keys are - `disk_id`-keyed (no `disk_group_id`/`store_id`/`group_id`), so a - bare bind change to a different kv-group leaves the records - stranded on the old kv-group — the sync loop silently overwrites - `dg.bind`, after which recovery/compaction/scanner scan the new - kv-group and conclude old busy blocks are free (data corruption) - and frees of old blocks target the wrong group (leaks). The - five-step flow copies records before the flip. No cross-group - replication is assumed in CROWDB v1. -- **Imbalance warning threshold** — what ratio triggers the - monitor's imbalance warning (e.g. max/min disk-groups per KV-group - > 2×, or absolute deviation from the mean)? Needs a tunable - default + a design decision on whether to warn on disk-group count - alone or also factor in per-disk-group capacity/usage. diff --git a/doc/backlog/R103-chunkdb-range-migration.md b/doc/backlog/R103-chunkdb-range-migration.md deleted file mode 100644 index 90cc9cafa..000000000 --- a/doc/backlog/R103-chunkdb-range-migration.md +++ /dev/null @@ -1,287 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R103: chunkdb — Dynamic Slot Ownership and KV-Group Expansion/Shrink - -Status: Deferred by user decision on 2026-10-03. The implemented fixed topology -uses three nodes and three selected nonzero Chunk-storage KV groups, plus -control-plane group 0. Start this follow-up when dynamic ownership or storage -capacity changes are requested. - -#### Problem - -[fixed slot routing](../design/chunkdb/design-crowdb-chunkdb-range-binding.md) -establishes two independent maps over 1024 fixed chunk-ID hash slots: - -- An ordinary client resolves slot to ChunkDB server. Group 0 stores one bitmap - record per server, including servers owning zero slots. -- ChunkDB resolves slot to a selected nonzero Paxos KV group (pxgroup). Group 0 - stores one bitmap record per group. All records belonging to a chunk follow - its chunk ID into that group. - -The initial R202 delivery freezes these assignments. It supports fixed-owner -restart and recovery, but does not make changing either bitmap safe. ChunkDB -servers perform chunk lifecycle and heavy task/IO orchestration without an -authoritative local chunk database. Changing a server owner therefore requires -execution-authority handoff, while changing a storage group requires moving -durable state. These operations have different costs and failure modes. - -The current [slot routing design](../design/chunkdb/design-crowdb-chunkdb-range-binding.md) -rejects ownership changes and describes the fixed service/storage maps. -Service-only handoff avoids metadata copying only when storage placement -remains unchanged. Existing ChunkDB routing, range guards and task -claims are useful foundations, but neither a routing refresh nor a grace-period -timer proves exclusive authority or complete storage transfer. - -Concrete scenarios are adding a ChunkDB server for more processing capacity, -draining a server, taking over slots after a failed owner, adding a KV group for -capacity, evacuating a group before removal, and resuming an interrupted move. -A source group may hold slots served by several servers; a server may own slots -stored in several groups. A one-to-one server/group assumption breaks both -handoff and migration. - -#### Solution - -##### Goal and scope - -Make the fixed R202 layout safely changeable without changing chunk identity or -losing acknowledged chunk/task state. Deliver two independently usable operations: - -1. **Service slot handoff.** Transfer selected slots between ChunkDB server - bitmaps. Keep storage-group bindings and persistent records in place. - Revoke old execution authority and recover tasks on the new server. -2. **Storage slot migration.** Transfer selected slots between pxgroup bitmaps - when adding, draining, removing or rebalancing storage groups. Move the - complete associated record set before activating the new destination. - Keep service assignments unchanged unless a separate handoff is requested. - -A supported combined change must coordinate these operations explicitly; it -must never infer one mapping from the other. Initial implementation may serialize -overlapping changes rather than execute both concurrently. - -In scope: - -- Explicit move requests, destination validation, durable progress, independent - authority generations, retry/recovery and scoped cleanup. -- Service join/leave/failure handoff and stale client/worker rejection. -- KV-group expansion/shrink and slot redistribution within the direct-KV backend. -- Migration of canonical Chunk records, all associated tasks and task indexes, - reservations and other per-chunk transaction participants. -- Safe refusal when source completeness or exclusive authority cannot be proven. -- Observable move status and an unambiguous completion condition for operators. - -Out of scope: - -- R202's initial fixed layout, chunk-type cleanup and PageIndex implementation. -- Changing the 1024-slot space, rehashing IDs, or hash-by-current-group-count. -- Moving DiskIO payload or reallocating DiskDB blocks as a side effect. -- Moving stream catalogs, tree roots or unrelated records merely because they - share a KV group; ownership is determined by the per-chunk record contract. -- Direct-KV-to-chunk-kv conversion, which belongs to - [R207](R207-chunkdb-repo-metadata-chunk-kv.md). -- Automatic cluster-wide capacity planning, creating/removing Paxos replicas, - or deleting a shared KV group. Removing a group from Chunk storage is distinct - from destroying the group. -- Mandatory conversion of pre-R202 layouts or legacy Repo/Stream/PageIndex data. - R202 uses fresh test state. A legacy group-0 source may be supported later only - with an explicit compatible record/layout conversion contract; it is not a - prerequisite for this requirement's R202-layout migration. - -##### Fixed granularity and publication - -- The move unit is one or more existing logical slots, selected by the owning - chunk ID. Related task IDs and encoded key order do not choose the destination. - Unmoved slots retain their hash meaning and bindings. -- Keep one steady-state bitmap record per server and per pxgroup. Do not create - a binding record, KV group or worker for each of the 1024 slots. Durable - per-operation progress is separate from these owner records. -- Publish each changed map as one complete committed generation. Readers cannot - observe independently cleared source bits and set target bits as active - authority. Service and storage maps have separate generations. -- Use group-0 atomic publication for routing/control state, but do not mistake it - for an atomic transaction with copied records in other groups. Preparation, - authority revocation, validation and activation form a recoverable protocol. -- Fence mutations at their authoritative acceptance/publication boundary. A - cached ownership check before asynchronous work is insufficient. Cover late - completions, task claims, reservation updates and operations already admitted - before revocation; authority cannot depend only on a process-local lock. -- Preserve group-local conditional atomic chunk operations. Source and target - revisions belong to their own groups; transferred values cannot make an old - source revision a valid target CAS token. - -##### Service handoff - -1. Validate source/target servers and requested slots against the current map; - persist operation identity and expected service generation. -2. Stop or revoke old admission and task authority for those slots. Resolve - admitted work and persisted claims using a durable fence/recovery contract. - Failure detection alone is not proof that an old server stopped running. -3. Publish source/target server bitmaps as one committed service generation. - Activate the new owner only under the new authority. -4. Refresh ordinary clients and recover task execution from the unchanged KV - groups. Preserve claim generations and prevent stale completion publication. -5. Complete when the new owner serves the slots and obsolete execution grants - cannot commit. Do not copy a local ChunkDB database or change storage bitmaps. - -An empty source bitmap means no remaining partition work, not allow-all. -An unreachable owner may delay handoff until fencing is proven; it must not -force a second active owner for availability. - -##### Storage migration and group drain - -1. **Prepare.** Validate a provisioned nonzero eligible target and persist the - slot set, source/target groups, expected storage generation and progress. - A newly added group has no authority simply because it is eligible. -2. **Copy.** Enumerate complete per-chunk state for the selected slots, including - indexes ordered by time/priority rather than chunk ID. Choose a bounded - slot-write pause or a consistent snapshot plus reliable change capture. - Handle creations, updates, deletes and claims without omissions or resurrected - records. A partial target copy is never serving authority. -3. **Fence and reconcile.** Coordinate every server that owns affected slots; - prevent old-group writes and delayed task publication. Bring the target to - the final acknowledged source state and validate all transaction participants. -4. **Cut over.** Publish source/target pxgroup bitmaps as one storage generation - only after completeness and fencing are proven. Enable operations against - the target; leave service bitmaps unchanged for a storage-only move. -5. **Clean up.** Retire old reads/claims safely and remove only moved source - records. Cleanup and retries are idempotent. Lost replies are reconciled - from durable authority/progress rather than interpreted as failure or success. - -For shrink, mark a group draining, stop new assignments to it, move all its slots -and finish relevant cleanup. Remove it from the Chunk-storage eligible set only -when no ownership or migration obligation needs it. Unrelated group contents -remain intact. Group expansion/shrink never changes the fixed hash function. - -##### Failure and cancellation outcomes - -- Before source revocation, a failed target leaves the source authoritative. - Retry or abandon a partial copy without exposing it to clients. -- During the fenced interval, operations may fail or retry; never acknowledge - writes that have no durable authoritative destination. -- If the source is unavailable before target completeness is proven, wait for - source recovery or a proven complete recovery source. Do not promote a partial - copy to finish a drain. -- After cutover, target unavailability does not reactivate the old source. - Restore the target or perform a new fenced move. -- Controller restart or an uncertain cutover reply resumes from persisted - operation identity and mapping generation before granting authority or cleanup. -- Cancellation before cutover must reconcile fences and partial target data; - cancellation after cutover cannot roll back by restoring old bitmap bits. -- Serialize or explicitly reject conflicting requests on the same slots. - Independent slots may progress within bounded copy/scan/task resources. -- Unavailable owner metadata is uncertainty, not evidence that payload is unused. - Preserve system/repo task separation throughout failures and recovery. - -##### Invariants - -- **M1 — Exclusive authority.** Only the current service and storage authorities - can commit chunk mutations or task outcomes, including delayed work. -- **M2 — Complete movement.** All acknowledged per-chunk state and maintenance - discoverability survive; atomic-operation participants share one active group. -- **M3 — Recoverable progress.** Retries, restart and uncertain replies cannot - create authority from an incomplete copy or reactivate an obsolete source. -- **M4 — Safe retirement.** Cleanup is scoped and idempotent; drain completion - requires zero remaining ownership and migration obligations. -- **M5 — Independent layers.** Service-only changes copy no metadata; - storage-only changes preserve service assignments. Neither moves payload, - changes slot identity nor merges system/repo task authority. -- **M6 — Coherent maps.** Each active slot has one owner in each committed map; - intermediate records and partial publications grant no authority. - -##### Work items - -1. Extend the KV domain monitor and ChunkdbRangeStrategy beyond fixed-layout - initialization with explicit service/storage move admission, conflict checks, - durable progress and separately published bitmap generations. -2. Extend RangeGuard, lifecycle handlers and task manager/scanner/executor with - service revocation and durable stale-publication rejection; integrate the - native tree and Rust client routing paths completed by R202. -3. Extend ChunkStore, TaskStore and reservation persistence with complete - slot-state enumeration/copy/validation under the selected change-capture - strategy. Preserve destination-local CAS semantics. -4. Implement storage fencing, recovery, cancellation and idempotent cleanup; - expose move/drain progress and actionable failure state. -5. Verify independent and combined changes, resource bounds and failure cases; - update permanent ChunkDB routing/migration documentation. - -#### Dependencies - -- [fixed slot routing](../design/chunkdb/design-crowdb-chunkdb-range-binding.md) supplies tested fixed slots, - independent bitmap maps, typed operations, task isolation, all client paths - and group-local atomicity. Start from its completed fixed-layout contract. - Until R103 is implemented, keep initialized assignments unchanged; do not - enable the old automatic rebalance path as a substitute. -- Existing direct-KV scan/snapshot/transfer facilities are candidate primitives. - Confirm snapshot consistency, delete capture and hash-slot enumeration before - choosing a copy strategy. Ordered key-range transfer alone is insufficient. -- [R207](R207-chunkdb-repo-metadata-chunk-kv.md) may reuse this authority/recovery - model later but needs its own backend transaction and placement contract. - R207 does not block either operation in R103. - -#### Acceptance - -- Given the completed R202 fixed topology, request a server-only slot move; - assert new-owner service, unchanged group bindings/data, no payload copy and - zero-slot old-owner rejection (M1, M5, M6). Integration test. -- Given a failed or partitioned old server, request takeover and reconnect its - delayed workers; assert takeover waits for proven fencing and stale writes, - reservations and task completions fail after activation (M1, M3). Integration test. -- Given an eligible additional group, move selected slots while handling allowed - foreground/task traffic; assert complete target state, preserved acknowledgements, - unchanged service map and unmoved slots (M1, M2, M5). E2E test. -- Given a group serving slots owned by several servers, drain it; assert all - affected owners follow storage cutover and removal waits for cleanup, while - unrelated data and DiskIO payload remain intact (M2, M4, M5). E2E test. -- Given non-contiguous chunk/task/index/reservation keys, copy with creates, - deletes and claims; assert complete enumeration, no resurrected deletes, - discoverable tasks and valid destination-local CAS (M2). Integration test. -- Given interrupted source/target bitmap publication, reload both maps; assert - readers use complete generations with one authority per slot and no partial - target activation (M3, M6). Integration test. -- Given source/target/controller failure or lost replies at every move stage, - restart and retry; assert acknowledged data survives, incomplete targets are - not promoted and obsolete sources never reactivate (M1–M3). Integration test. -- Given cancellation before and after cutover, reconcile the operation; assert - safe pre-cutover termination or forward recovery after cutover, with no bitmap - rollback that restores obsolete authority (M1, M3, M4). Integration test. -- Given conflicting service/storage changes for the same slots, submit both; - assert explicit serialization/rejection or a proven coordinated transition, - never inferred mapping changes or two writers (M1, M5, M6). Integration test. -- Given interrupted/repeated cleanup alongside unrelated slots, resume cleanup; - assert no duplicate release, foreign deletion or premature group retirement - (M3, M4). Integration test. -- Given invalid group 0 targets, incompatible layouts or attempted slot resizing, - submit a move; assert rejection without modifying current authority (M5, M6). - Unit test. -- Given both operation domains and unavailable ownership metadata during a move, - scan tasks and query payload ownership; assert no cross-domain claims and - uncertainty does not authorize reclamation (M1, M2, M5). Integration test. -- Given bounded migration concurrency under foreground load, run independent - moves and inspect status; assert configured resource limits, observable progress - and eventual completion after recoverable faults clear (M3, M4). E2E test. - -#### Open Questions - -- Prefer a bounded write pause per moved slot, or online snapshot plus reliable - catch-up? The former reduces protocol complexity but pauses affected writes; - the latter needs proven change capture and increases implementation cost. -- Should group-add select slots automatically from measured load, or initially - require explicit slot selection? Manual selection keeps the first migration - contract smaller; automatic policy needs agreed load/capacity objectives. -- What pause/latency and migration resource budgets are acceptable? Establish - measured limits before claiming online migration performance. - -Implementation selects the durable fence and state schema against R202's actual -interfaces; these are required technical proofs, not permission to weaken the -invariants. This rewrite ran no runtime tests. Extend the named suites with the -acceptance cases before treating these commands as migration coverage: - -```sh -pixi run cargo test -p crowdb-chunkdb --test routing_test --test lifecycle_test --test owner_metadata_test --test conversion_test -pixi run cargo test -p crowdb-chunkdb-client --test client_test -pixi run cargo test -p crowdb-kv-client --test chunkdb_partition_test -pixi run cargo test -p crowdb-kv-server --test domain_monitor_test -pixi run test-tree-ct -pixi run rs-fmt-check -pixi run cargo clippy -p crowdb-chunkdb -p crowdb-chunkdb-client -p crowdb-kv-client -p crowdb-kv-server --all-targets -- -D warnings -``` diff --git a/doc/backlog/R139-group0-service-config.md b/doc/backlog/R139-group0-service-config.md deleted file mode 100644 index d7b568b3e..000000000 --- a/doc/backlog/R139-group0-service-config.md +++ /dev/null @@ -1,147 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R139: config — Deployment profiles and Group-0 service configuration - -## Status - -Deferred until the file-backed service configuration contract is established. - -## Problem - -Service configuration is local to each node. Changing a cluster-wide policy -requires editing files or launch arguments on every host, and operators cannot -see which revision each process has accepted. Blind live reload is unsafe -because some fields can be applied dynamically while others define listener, -thread, storage, or durability state and require a restart. - -Single-container deployment, multi-node clusters and test fixtures currently -select tuning through different paths: service TOML files, command-line presets -such as KV's `e2e` election profile, and deployment request overrides such as -DiskDB's keepalive interval. A single container and a cluster require different -settings; clusters also need low- and high-performance profiles. These choices -must be explicit and reproducible without building separate images or scattering -special timing values through test clients. - -The group-0 architecture in -`doc/design/kv/design-crowdb-kv-group0.md` already provides durable replicated -sysdata and service registration, but it has no versioned configuration model, -target selection, acknowledgement state, or restart contract. - -## Solution - -Provide named, versioned configuration profiles through the normal deployment -and service configuration contract. Resolve the distributed record and rollout -details below before implementation. - -Profile invariants: - -- **I1 — Explicit selection:** distinguish single-container and cluster profiles; - cluster profiles include low- and high-performance settings. Selection is - recorded and remains stable across restart and cluster expansion. -- **I2 — One runtime:** profiles configure the same release image and binaries. - Tests may select a documented test tuning profile or explicit timing overrides - using the same schema, validation and startup mechanism as operators. -- **I3 — Complete validation:** validate the resolved configuration and its - cross-field constraints before starting or applying changes. Performance - settings never relax placement, quorum, lease safety or durability guarantees. -- **I4 — Observable configuration:** expose the profile name/version, resolved - values and accepted/applied revision for each service; errors identify the - conflicting setting and target. - -Numbered work items: - -1. Define single-container and cluster profiles, with low- and high-performance - cluster choices. Document the intended resource envelope and service settings - for each rather than treating a higher number as universally faster. -2. Unify profile selection at container/cluster creation and service startup; - support typed caller overrides and persist the resolved choice. Cover KV - election/heartbeat/lease settings, DiskDB keepalive/synchronization intervals, - and applicable worker, concurrency and batching limits. Test startup must be - able to request, for example, a 3-second DiskDB cadence through this contract. -3. Define deterministic precedence between profile defaults, cluster defaults, - service/node/instance overrides and explicitly supplied startup values. - Existing CLI presets and deployment fields must resolve through this contract - rather than silently overwrite one another. Distinguish heartbeat cadence - from metadata synchronization cadence and specify their actual consumers. -4. Keep bootstrap selection available before Group 0 exists. Persist profile - identity and effective local configuration, then reconcile with the validated - distributed configuration when authority is available. Joining nodes inherit - the cluster selection unless an explicit validated override applies. -5. Define the distributed configuration and rollout model: - -- Define a versioned group-0 record for cluster defaults plus service-, - instance-, and node-scoped overrides, with deterministic precedence. -- Reuse the file-backed schemas as typed payloads or define an evolvable - envelope that preserves unknown fields during mixed-version rollout. -- Classify every field as dynamic or restart-required and make each service - validate a complete candidate before atomically applying any dynamic subset. -- Publish per-instance observed, accepted, applied, and restart-required - revisions through the service registry so operators can detect drift. -- Define bootstrap and outage behavior: local file remains the seed and - fallback until group 0 is reachable; the last validated group-0 snapshot is - cached locally for restart. -- Add an authenticated administrative write path with compare-and-set revision - fencing, validation, staged rollout, and rollback semantics. - -Edge cases requiring design include an older binary receiving unknown fields, -group-0 loss during rollout, a process restart between acceptance and apply, -conflicting scope overrides, invalid cross-field combinations, and a static -field changed while a service is live. - -## Dependencies - -- Depends on the unified file-backed server configuration contract. -- Depends on group-0 service registry and watch/notify support. -- A first version may poll if watch delivery is unavailable; correctness must - not depend on notifications. - -## Acceptance - -- Given the same release image, starting with a single-container profile and - with each cluster performance profile produces the documented effective - settings and records the selected version. Invariants: I1, I2, I4. E2E test. -- Given profile defaults plus scoped and explicit overrides, resolution produces - one deterministic validated configuration; an unknown profile, zero interval - or unsafe election/lease combination fails before partial startup. Invariants: - I1, I3. Unit test. -- Given a test tuning selection and a 3-second DiskDB cadence override, normal - deployment creates the effective configuration and the actual timer consumes - that value; no test server build or hidden fixture mutation is used. - Invariants: I2, I4. Integration test. -- Given a selected cluster profile, service restart and admission of a new node - preserve or inherit the profile and report effective overrides accurately. - Invariants: I1, I4. E2E test. -- Given heartbeat and metadata synchronization settings differ, each runtime - timer uses its documented setting; updates follow their dynamic or - restart-required classification. Invariants: I3, I4. Integration test. - -- Given cluster, service, node, and instance records, resolving a candidate is - deterministic and produces the documented precedence. Invariant: every - instance resolves the same revision from the same records. Unit test. -- Given a candidate containing unknown fields during a mixed-version rollout, - the old service reports unsupported configuration without corrupting or - partially applying it. Invariant: candidate validation is atomic. E2E test. -- Given dynamic and restart-required changes in one revision, the service - applies only the validated dynamic subset and reports the static subset as - pending restart. Invariant: runtime state and reported revision agree. - Integration test. -- Given group 0 becomes unavailable, a running service retains its last applied - config and a restarting service uses its last validated cache or local file - according to the bootstrap contract. Invariant: control-plane loss does not - silently reset tunables to compiled defaults. E2E test. -- Given concurrent publishers, compare-and-set revision fencing permits one - winner and produces an auditable rejected update. Invariant: published - revisions form one ordered history. Integration test. -- Given a staged rollout and an unhealthy acknowledgement, rollback restores a - prior valid revision and all targeted instances converge or report a precise - blocker. Invariant: rollout status exposes all configuration drift. E2E test. - -Required gates: - -- `pixi run -- cargo fmt --all -- --check` -- `pixi run rs-lint` -- `pixi run test-core` -- `pixi run test-storage` -- `pixi run clean --runtime-only && pixi run test-container-e2e --scope console.operations` -- `pixi run test-suite` diff --git a/doc/backlog/R144-chunk-kv-partition-merge.md b/doc/backlog/R144-chunk-kv-partition-merge.md deleted file mode 100644 index d2c570682..000000000 --- a/doc/backlog/R144-chunk-kv-partition-merge.md +++ /dev/null @@ -1,261 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R144: chunk-kv — Adjacent partition merge - -**Status: Deferred.** Implement after R140, R142, R143, and R145 establish -stable manifest reuse, split/transfer fencing, group-0 transition recovery, and -routed retry semantics. - -## Problem - -R142 can split a partition and transfer ownership, but it intentionally exposes -no merge operation. Without merge, many formerly hot ranges can remain as -permanent small partitions after their load or data volume falls, increasing -routing, checkpoint, stream, and per-partition memory overhead. - -Merge is not the inverse of split at the control-plane level. Two adjacent -parents can have different owners, ownership epochs, tree manifests, WAL -streams, mutation sequences, checkpoints, and in-flight operations. Publishing -a merged child after fencing only one parent can lose acknowledged writes or -leave an overlapping writer. Reusing both trees also requires a valid combined -B+tree root and a leaf sibling chain that crosses the old boundary without -escaping the merged range. - -The existing crowdb-tree snapshot APIs rebuild or export one tree and R140 -rebuilds one manifest into a subrange; neither composes two roots. Copying every -key through Rust would lose page-reference reuse and make time and memory -proportional to the combined data set. The merged partition also cannot reset -its logical mutation sequence below a resolved sequence already stored in a -reused page. - -The root design links are R140's chunk page-store design, R142's partition -lifecycle design, and R143's group-0 range-map and transition design. - -## Solution - -Add an explicit two-parent merge transition that prepares one combined child -from adjacent immutable manifests, briefly fences both parents for catch-up, -and activates the child through one epoch-fenced group-0 range-map revision. - -1. Extend the permanent tree and chunk-KV designs with adjacent-manifest - composition, dual-parent lifecycle fencing, destination sequence seeding, - publication, rollback, recovery, retention, and failure semantics. Only two - exactly adjacent partitions may merge in one transition; repeated or - multi-way merge is composed from successful pairwise transitions. -2. Add a native crowdb-tree operation that builds a destination manifest for - `[left.start, right.end)` from the left and right pinned manifests. Reuse - verified interior pages from both sources, build a combined root, and - rewrite boundary ancestors plus the last left leaf so its sibling reference - leads to the first right leaf. It must not materialize all keys in Rust. -3. Accept an idempotent merge plan containing `transition_id`, both parent - identities, exact adjacent bounds, epochs and manifests, and the destination - identity/range/epoch. Reject gaps, overlaps, reversed order, stale inputs, - changed plans under one transition ID, or a parent already participating in - another lifecycle transition. -4. Enter `MergePreparing` on both parents, publish and pin independent base - checkpoints at sequences `b_left` and `b_right`, and build the destination - base while both parents continue serving. Seed its destination-local - sequence at `max(b_left, b_right)`, then replay both source deltas while the - parents remain writable. Preserve order within each source, assign each - replayed mutation a new contiguous destination sequence, and checkpoint the - two source cursors with that sequence so restart can resume without relying - on colliding source-local numbers. Preserve source WAL prefixes and every - reused page-pack reference until commit or proven abort. -5. Close mutation admission on both parents only after destination replay lag - is below configured record, byte, and estimated-time limits. If either - source cannot catch up, keep both serving and cancel or retry preparation. - Once fenced, drain all operations admitted on either side, reconcile their - durable streams through `c_left` and `c_right`, and assign the bounded final - delta contiguous destination sequences. Sequence overflow fails before the - dual fence. -6. Persist and validate one destination manifest at its final destination - sequence `d`, and create a distinct empty destination stream whose next - mutation sequence is `d + 1`. Return one immutable merge artifact binding - both exact parents, source cutover cursors, recorded conditional outcomes, - and retained request-result windows to that destination. Coalesce an exact - duplicate `(client_instance_id, client_sequence, operation_digest, result)`; - a repeated request identity with a different digest or result rejects merge - before fencing as `RequestConflict`. Reads may continue from the authoritative - parents during preparation and fencing; new mutations receive a retryable - merge error after the dual fence. -7. Extend R143's catalog transition so one new immutable range-map generation - replaces both still-current parents with the exact destination and publishes - one head. Current group-0 sysdata exposes blind puts rather than general - compare-and-swap, so publication must not depend on coordinated updates to - two live parent keys. Commit proof retires both parents and permits - destination activation. Abort proof resumes both unchanged parents. An - ambiguous head outcome keeps both parents mutation-fenced until an - authoritative head read resolves it. -8. Make preparation, fencing, and resolution restart-idempotent. A replacement - group-0 domain monitor reconstructs the transition, validates manifests and - stream tails, and either reuses the exact artifact or restarts an unpublished - build. If either parent owner dies, it waits for the old serving lease fence - before recovering that parent or aborting; it never infers commit from local - files, memory, or a service heartbeat. -9. Select a healthy destination owner and open the merged artifact as - `Prepared`. Publish its serving lease only after the catalog head commits and - both parent grants are fenced. A balance or split planner cannot start a - second lifecycle transition on either parent or the destination until merge - commit or abort is proven. -10. Reclaim parent manifests, streams, request-result records, and abandoned - destination objects only after catalog references and transition pins allow - it. Shared page packs survive as long as the destination or another manifest - references them. -11. Add merge metrics for preparation, dual-fence drain, delta replay, reused - and rewritten pages, publication/abort, ambiguous resolution, source pins, - orphan bytes, owner failures, and write-fence duration. - -Edge outcomes are explicit: non-adjacent partitions are rejected; either stale -parent epoch fails the whole transition; failure before publication leaves both -parents authoritative; partial parent replacement is invalid; sequence -overflow fails before fencing; and the destination never serves concurrently -with either writable parent. - -Operation outcomes are explicit during merge: - -- Gets, ordered seeks, and bounded scans already admitted to a parent complete - on that authoritative parent view. A new request after catalog publication is - routed to the destination; a stale request receives `NotMyRange`. -- Successful and condition-failed mutations admitted before the fence remain in - source order, including their recorded CAS result and request ID. Mutations - after the fence receive a retryable lifecycle result without WAL append. -- A lost pre-fence mutation response can be retried on the destination after - commit and returns the retained original result rather than executing again. -- Parent scan tokens are invalid after commit and return refresh-required. The - client cannot resume a parent token against the wider destination range. -- A journal, tree, owner, or group-0 failure follows R142/R143 containment and - lease rules; it never makes the unfenced parent or uncommitted destination - authoritative by inference. - -## Dependencies - -- Depends on R140's generation-addressed manifests, verified page fences, - immutable reference retention, and native async chunk backend. R144 extends - its range rebuild machinery with two-manifest adjacent composition. -- Depends on R141's independent stream replay, fencing, stable positions, and - prefix pins for both parents and the destination stream. -- Depends on R142's partition manager, logical mutation sequences, lock-free - admission fence, lifecycle proof boundary, checkpoint tuples, and fault - behavior. R142 remains complete without R144. -- Depends on R143's authoritative paged range map, monotonic epochs, persisted - transitions, domain monitor, owner serving grants, and atomic catalog-head - publication. R143 must add a two-parent merge plan and exact commit/abort - proof before R144 can ship. -- Depends on R145 for routed retry, request-identity persistence, scan replanning, - and real-process merge E2E coverage. - -## Acceptance - -- Given adjacent parents `[a, m)` and `[m, z)` with verified manifests, when - native composition completes, assert the destination scan equals their - ordered union, its root covers `[a, z)`, and the rewritten sibling chain does - not escape that range. Invariant: tree composition preserves exact key and - navigation bounds. Integration test. -- Given fully contained interior pages and pages touching `m`, when composition - runs, assert eligible interiors retain their page references while boundary - ancestors and the final left leaf are rewritten. Invariant: merge reuses only - pages whose references remain structurally valid. Unit test. -- Given parents with a gap, overlap, reversed order, stale epoch, or changed - plan under the same transition ID, when merge preparation is requested, - assert it fails before either mutation fence. Invariant: only one exact, - adjacent, current parent pair can enter a merge. Unit test. -- Given writes continue on both parents during destination base construction, - when preparation runs, assert the writes remain acknowledged and both base - manifests and WAL prefixes stay pinned. Invariant: base construction does not - impose the merge write outage. Integration test. -- Given both parents contain colliding source-local sequence numbers and later - mutations, when replay and the dual fence drain through `c_left` and - `c_right`, assert every acknowledged mutation appears once, destination - sequences are contiguous above `max(b_left, b_right)`, and the next sequence - is greater than every reused-page sequence. Invariant: independent parent - histories converge without sequence collision or rollback. E2E test. -- Given retained request windows contain an exact duplicate identity/result or - the same `(client_instance_id, client_sequence)` with conflicting digests, - when preparation combines them, assert the exact duplicate is coalesced and - the conflict aborts before either parent fence. Invariant: merge cannot make - request retry identity ambiguous. Unit test. -- Given either source write rate keeps destination replay lag above the fence - limits, when merge preparation reaches its catch-up deadline, assert both - parents remain serving and no dual fence begins. Invariant: merge never - begins a predictably unbounded final outage. Integration test. -- Given one parent faults or fails to drain, when finalization runs, assert no - destination artifact becomes publishable and the other parent does not - resume until group 0 proves abort. Invariant: dual-parent fencing is one - transition, not two independent decisions. Integration test. -- Given one parent owner dies during preparation or after the dual fence, when - the group-0 monitor resumes the transition, assert it waits for the old lease - fence, recovers the exact durable parent suffix, and commits or aborts without - granting overlapping authority. Invariant: owner death cannot shortcut a - dual-parent fence. E2E test. -- Given a valid merge artifact, when group 0 publishes its epoch-fenced - range-map activation record, assert the visible revision replaces both - parents or neither and only an exact commit proof activates the destination. - Invariant: the range map never exposes a gap, overlap, or partial merge. E2E - test. -- Given a timeout after publication is submitted, when its result is unknown, - assert both parents remain mutation-fenced until an authoritative read - returns commit or abort proof. Invariant: ambiguity cannot create concurrent - writable ownership. E2E test. -- Given the destination is prepared on a healthy server, when the catalog head - has not committed or either parent grant remains valid, assert it rejects all - data operations; after commit and both fences, assert one destination serving - lease is issued. Invariant: preparation and storage readiness do not grant - ownership. E2E test. -- Given a crash during preparation, fencing, commit, or abort, when a new - coordinator resumes the persisted transition, assert the final range map and - manifests match one valid all-parent or one valid destination outcome. - Invariant: merge is restart-idempotent. E2E test. -- Given source manifests, streams, and page packs referenced by a live or - unresolved transition, when GC runs, assert they remain; after committed or - aborted references and pins clear, assert only unreachable objects become - reclaimable. Invariant: reclamation never precedes merge resolution. - Integration test. -- Given concurrent reads during preparation and fencing, when the merge commits - or aborts, assert every read is served by an authoritative parent snapshot or - the published destination and never observes keys outside its routed range. - Invariant: merge preserves read range isolation. Integration test. -- Given a pre-fence successful or condition-failed mutation response is lost, - when its request ID is retried against the committed destination, assert the - original result is returned and no condition is re-evaluated. Invariant: - merge preserves the deduplication and CAS-result retention boundary. E2E test. -- Given a parent scan token before merge commit, when it is resumed after the - destination becomes current, assert it returns refresh-required rather than - scanning the widened range. Invariant: merge cannot silently change a scan's - range or pagination history. Integration test. -- Given merge preparation, fencing, replay, publication, abort, ambiguity, and - GC, when metrics are collected, assert each outcome is attributed to the - transition and both parents. Invariant: all material merge states are - observable. Unit test. - -Required gates: - -- `pixi run tree-fmt` -- `pixi run tree-lint` -- `pixi run test-tree-ct` -- `pixi run test-tree-ffi` -- `pixi run -- cargo fmt --all -- --check` -- `pixi run rs-lint` -- `pixi run -- cargo test -p crowdb-chunk-stream --all-targets` -- `pixi run -- cargo test -p crowdb-chunk-kv --all-targets` -- `pixi run -- cargo test -p crowdb-chunk-kv-client --all-targets` -- `pixi run -- cargo test -p crowdb-chunk-kv-server --all-targets` -- `pixi run clean --runtime-only && pixi run test-server` - -## Open Questions - -- Should the destination normally stay on the left owner, the less-loaded - parent owner, or a third server selected by the balance score? Reusing one - parent owner reduces warm-up; choosing by load can avoid an immediate follow- - up transfer. -- Should merge initially be operator-triggered only, or may the balance monitor - propose and automatically execute merges after a long low-load window? - Automatic merge controls partition overhead but adds a second autonomous - topology-changing policy beside split. -- Is native two-manifest composition required for the first merge release, or - may it fall back to a bounded background rebuild when page topology prevents - safe reuse? Composition minimizes IO; a rebuild fallback is simpler but may - make large merges impractically slow. -- What maximum dual-fence duration is acceptable before merge must abort? A - short bound protects foreground availability; a longer bound tolerates slow - final replay and checkpoint publication. diff --git a/doc/backlog/R147-tree-chunk-gc.md b/doc/backlog/R147-tree-chunk-gc.md deleted file mode 100644 index c45c91fd8..000000000 --- a/doc/backlog/R147-tree-chunk-gc.md +++ /dev/null @@ -1,139 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R147: crowdb-tree / chunkdb — Reclaim B+tree chunk strips - -**Status: Deferred.** Implement after R140 establishes immutable page packs, -manifest retention, and durable reclaim candidates. R146 must make abandoned -active chunks sealable before they can enter physical reclamation. - -## Problem - -The existing crowdb-tree GC removes unreachable page IDs from the mapping table -and eventually releases local page-store extents. For the R140 backend, that is -only logical reclamation: page packs reside in chunk strips whose DiskDB blocks -remain allocated until chunkdb releases them. Deleting a mapping entry without -this second step leaks chunk capacity; releasing a strip too early can destroy -pages still referenced by another manifest, split child, or pinned snapshot. - -A strip can also contain both dead and live page packs. Chunkdb's proposed -in-chunk GC in `doc/backlog/R92-chunkdb-in-chunk-gc.md` is the correct ownership -boundary for releasing strip storage, but crowdb-tree must first remove or -relocate every live reference and publish that fact durably. The release flow -must survive tree, KV-server, and chunkdb restart and must tolerate an ambiguous -release response. - -The root storage contracts are -`doc/design/tree/design-crowdb-tree-storage.md` and -`doc/design/chunkdb/design-crowdb-chunkdb.md`. - -## Solution - -Use a two-phase, manifest-fenced flow: crowdb-tree proves a strip unreachable, -then chunkdb idempotently reclaims it through a durable cleanup intent. - -1. Extend R140 logical GC to emit a durable reclaim candidate containing tree - identity, source manifest generation, chunk ID and revision, strip sequence - and byte range, and the page-pack ordinals formerly stored there. Emit it - only after a newer manifest that removes those references is durable. -2. Advance a candidate to releasable only after every retained manifest, - split-child manifest, and in-memory snapshot pin that can name any listed - page pack is past the reclamation watermark. Recompute reachability from the - current manifest before dispatch; a reachable ordinal cancels the candidate - as stale. -3. Add or complete the generic R92 `ReclaimStrip` chunkdb operation for both - `BtreePage` and `PageIndex` chunks. Its request carries a deterministic - reclaim ID, chunk ID, expected chunk revision, and exact - strip identities; under the existing lifecycle guard it verifies that the - chunk is Sealed and the identities still match, removes the strips from the - published layout, and persists the reclaim ID plus qualified DiskDB cleanup - intents. Retired segments are freed only after the layout-validity window. - A retry with the same reclaim ID returns the persisted result. -4. Delete an entire sealed chunk when all of its strips are releasable. For a - partially dead strip, do not punch arbitrary byte holes in the first - release. Copy its live page packs to fresh R140 B+tree chunks, verify the - copies, atomically publish a new manifest with replacement references, wait - for the old generation and reader-layout window to expire, and then reclaim - the whole old strip. This gives a workable compaction flow without changing - stable chunk offsets or requiring R92 `CollapseStrip` and `MergeStrips`. -5. Persist GC state as `Prepared`, `Releasing`, or `Complete` in the injected - root catalog before issuing chunkdb mutations. On startup, the current tree - owner resumes non-complete records; chunkdb independently resumes its - segment cleanup intents. An ambiguous RPC result is resolved by querying - the chunk revision and strip identities before retrying. -6. Bound candidates, bytes copied, concurrent chunk mutations, and work per - maintenance pass. GC and repacking run outside page lookup, apply, and - mapping-resolution hot paths and add no lock to them. Foreground checkpoint - publication wins the existing manifest gate; stale GC output becomes - orphan work and is retried from the newer generation. -7. Expose logical bytes retired, physical bytes released, pending and oldest - candidate age, live bytes copied, reclaim conflicts, ambiguous outcomes, - cleanup retries, and bytes retained by manifest or snapshot pins. - -## Dependencies - -- Depends on R140 for generation-addressed page packs, immutable manifests, - reference retention, logical GC, and the injected root catalog. -- Depends on R146 so crashed writers eventually produce Sealed chunks. GC skips - Active chunks and remains safe while sealing is unavailable. -- Uses R92's generic in-chunk `ReclaimStrip` boundary. If the broader collapse - and merge work is still unlanded, R147 implements only the idempotent - whole-strip operation needed here and leaves those compaction operations - deferred. -- R142 supplies the group-backed production root catalog and schedules tree - maintenance. Tests may use R140's in-memory epoch-fenced catalog. -- DiskDB qualified frees and chunkdb cleanup-intent reconciliation remain the - physical block-release authority. - -## Acceptance - -- Given tree GC removes a page from the current mapping but an older manifest - or pinned snapshot still references its pack, when GC evaluates the strip, - assert no chunkdb reclaim is issued and retained-byte metrics identify the - pin. Invariant: logical death does not imply physical unreachability. - Integration test. -- Given all packs in one strip are unreachable past the watermark, when the - persisted candidate is dispatched, assert chunkdb removes that exact strip, - waits out layout validity, frees its qualified segments, and marks the record - Complete. Invariant: physical release follows durable reference removal. - Integration test. -- Given every strip of a sealed chunk is releasable, when GC runs, assert it - uses whole-chunk deletion and all cleanup intents complete idempotently. - Invariant: an empty sealed chunk retains no allocated strips. Integration - test. -- Given a strip mixes live and dead packs, when GC compacts it, assert live - packs are copied and verified, the replacement manifest is published before - the old strip is retired, and reads remain correct across the cutover. - Invariant: partial-strip reclamation never creates a page-reference hole. - E2E test. -- Given a checkpoint publishes generation `g + 1` while GC works from `g`, when - GC reaches publication, assert it cannot replace `g + 1`, its unreferenced - copied packs become orphan candidates, and work restarts from the current - manifest. Invariant: GC never rolls back acknowledged tree state. - Integration test. -- Given the KV-server restarts in each of `Prepared` and `Releasing`, when the - partition reopens, assert it resumes the same candidate without an early or - duplicate free. Invariant: tree-side GC progress is restart-safe. E2E test. -- Given chunkdb restarts after layout retirement but before DiskDB cleanup, - when reconciliation resumes, assert the retired segments are freed after the - validity window and the cleanup intent clears. Invariant: physical release - survives chunkdb restart. E2E test. -- Given a reclaim RPC response is lost, when the tree owner queries and retries - the same reclaim ID, assert the persisted result identifies the exact retired - strips; a different ID or replacement identity conflicts. Invariant: - ambiguous completion cannot reclaim a replacement strip. Integration test. -- Given bounded GC budgets and sustained foreground traffic, when many strips - become reclaimable, assert each pass stays within its byte, candidate, and - concurrency limits while candidates eventually complete. Invariant: physical - GC is bounded and makes progress without entering tree hot paths. Integration - test. - -Required gates: - -- `pixi run tree-fmt` -- `pixi run tree-lint` -- `pixi run test-tree-ct` -- `pixi run test-tree-ffi` -- `pixi run -- cargo test -p crowdb-chunkdb --all-targets` -- `pixi run -- cargo fmt --all -- --check` -- `pixi run rs-lint` diff --git a/doc/backlog/R148-chunk-stream-scale-out.md b/doc/backlog/R148-chunk-stream-scale-out.md deleted file mode 100644 index 56a5b1a94..000000000 --- a/doc/backlog/R148-chunk-stream-scale-out.md +++ /dev/null @@ -1,154 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R148: chunk-stream/chunk-kv — Partition metadata scale-out and sealed-chunk EC - -**Status: Deferred.** Implement after the R141/R142/R143 single-group, -three-way-mirror production path is complete and measured. All features in -this requirement are disabled by default. - -## Problem - -R141 deliberately binds one stream's metadata to one nonzero KV group and -writes each active stream chunk as three mirrors. R142 stores the same -partition's R140 tree root catalog in that metadata group: tree page packs and -mapping data remain in ChunkDB/DiskIO chunks, while the Paxos KV records hold -the owner authority, current root manifest, immutable manifest generations, -and reference-segment indexes that locate those chunks. Group 0 contains only -the authoritative partition/range ownership and metadata-group binding. - -That baseline keeps one partition's checkpoint metadata together and keeps -high-frequency root publication out of group 0, but it does not move a hot or -oversized partition metadata namespace between groups, shard its stream extent -index and tree reference indexes across groups, or reduce the long-term -storage cost of sealed chunks. - -These changes require explicit cutover protocols. Moving keys without fencing -the authoritative binding can expose mixed metadata generations, while -converting an active chunk to erasure coding can race acknowledged-cursor -advancement and make the writer's durability contract ambiguous. - -## Solution - -1. Treat each partition's metadata placement as one binding-generation unit. - The selected nonzero KV group owns both the R141 stream namespace and the - R142 tree root namespace. The tree namespace contains - `authority(tree_id) = (owner_epoch, current_generation)`, the current - manifest, immutable generation manifests, and reference segments. A tree - root publish CASes the authority record and atomically writes the new - authority value, current manifest, and immutable generation in one Paxos KV - batch. Old epochs and same-generation concurrent publishers fail closed. -2. Add a binding-generation migration state machine for moving the complete - partition metadata unit from one KV group to another. Bulk-copy immutable - stream pages, tree manifest generations, and reference segments while the - source serves. Then fence and drain partition mutation admission, publish a - final checkpoint, copy the remaining current stream/root records, and - verify the destination contains the exact tree generation, applied - mutation sequence, acknowledged stream cursor, and replay-retention floor. - Atomically publish the destination group, higher binding generation, and - higher ownership epoch in group 0 before the destination can serve. Retain - the source read-only for a grace interval and delete it through bounded, - idempotent cleanup. Append and root publication never dual-write groups. -3. Make reopen resolve group 0 first and accept stream and tree metadata only - from the group named by the same binding generation. A prepared destination - may verify copied metadata and prewarm chunk routes but cannot append, - checkpoint, reclaim, or serve until the exact group-0 proof arrives. A - timeout around group-0 publication keeps the source fenced and both sides - non-writable until an authoritative read proves commit or abort. -4. Add optional per-partition metadata sharding only after whole-unit migration - works. Keep one small generation-fenced directory in the partition's - primary metadata group. Route stream extent pages by logical offset and - tree reference segments by stable object ID; keep the tree authority and - current root manifest together in the primary group so one CAS remains the - publication point. Random stream seek and tree-root open remain logarithmic, - and trim/reclaim can advance nonzero shard floors without scanning or - rewriting earlier shards. -5. Keep new and active chunks three-way mirrored. After a chunk is sealed and - no writer can advance its cursor, optionally submit it to chunkdb's existing - mirror-to-EC conversion workflow. Verify the EC layout before atomically - switching the chunk layout; retry or crash leaves the mirror layout - readable. Chunk-stream metadata continues to name the chunk, not its strips - or current layout. -6. Gate migration, sharding, and EC conversion behind separate configuration - switches that default to disabled. Old bindings and mirror-only chunks keep - their current meaning, and disabling a switch stops new background work - without invalidating already published layouts. -7. Bound copy, verification, conversion, and cleanup work across both metadata - namespaces. Expose binding age, primary and shard group IDs, stream/tree - metadata bytes, migration phase/lag, copied and verified bytes, source - retention, sealed mirror backlog, EC conversion latency/failure, retained - mirror bytes, and cleanup retries. - -## Dependencies - -- Depends on R141 for the production stream metadata and mirrored chunk - baseline, including stable logical offsets and sealed-chunk lifecycle. -- Depends on R142 for the chunk-backed tree, generation-CAS root catalog, - checkpoint tuple, and epoch-fenced partition lifecycle. -- Depends on R143 for authoritative group-0 binding publication and operator - configuration. -- Reuses chunkdb's durable layout-conversion state machine and R103's - copy/verify/cutover principles, but metadata-group migration is a distinct - namespace operation. -- The metadata publication fence selected in R141 must apply to binding and - shard-directory generations before this requirement begins. - -## Acceptance - -- Given a populated stream, when its metadata binding migrates to another KV - group with a checkpointed chunk-backed tree, and failures are injected before - bulk copy, final fence, verification, group-0 publication, and source cleanup, - assert every reopen selects one metadata group and recovers the exact same - tree generation, applied sequence, stream cursor, retained retry results, and - key/value contents. Invariant: migration never combines stream metadata and - a tree root from different binding generations. E2E test. -- Given an old source owner and a destination owner, when group 0 publishes the - higher epoch and destination group, assert the source cannot append, publish - a tree root, or reclaim metadata, while the destination cannot do any of - those operations before exact publication proof. Invariant: metadata - placement migration never creates two writable authorities. Integration test. -- Given two same-epoch checkpoint publishers with the same expected tree - generation, when both publish concurrently, assert exactly one authority CAS - and its matching current and immutable manifests commit. Repeat after moving - ownership to a higher epoch and assert the old epoch always fails. Invariant: - a root generation and its authority fence are one atomic Paxos decision. - Integration test. -- Given a sharded partition metadata namespace spanning several KV groups, when - readers seek near the stream head, middle, and tail, open the current tree - root, resolve reference segments, trim complete leading extent shards, and - reclaim old tree generations, assert lookup remains logarithmic and no live - metadata page is rewritten or lost. Invariant: sharding preserves logical - addressing and one primary root publication point. Integration test. -- Given a sealed mirrored chunk, when EC conversion succeeds, fails, or crashes - around layout publication, assert readers see either the verified mirror - layout or the verified EC layout and the stream's logical extent is - unchanged. Invariant: conversion cannot expose a partial layout. E2E test. -- Given default configuration, when streams append, roll, and reopen, assert no - metadata migration, sharding, or EC background task starts. Invariant: R141's - measured mirror-only behavior remains the default. Integration test. -- Given configured work limits and a large backlog, when maintenance runs, - assert foreground append/tree-read latency remains within the selected - regression threshold and stream copy, tree metadata copy, verification, EC, - and cleanup queues each report bounded progress. Invariant: coordinated - scale-out work cannot create unbounded foreground interference. Benchmark. - -Required gates: - -- `pixi run -- cargo fmt --all -- --check` -- `pixi run rs-lint` -- `pixi run -- cargo test -p crowdb-chunk-stream --all-targets` -- `pixi run -- cargo test -p crowdb-chunk-kv --all-targets` -- `pixi run -- cargo test -p crowdb-chunkdb --all-targets` -- `pixi run clean --runtime-only && pixi run test-server` - -## Open Issues - -- The CI log for - `small_object_writer_e2e::eight_closed_mirror_strips_become_one_durable_ec_strip_without_reread` - failed at `location strip`, while the exact test, its 16-test suite and the - full `test-server` task pass locally at default test concurrency. CI run - `36449749925` completed its "Upload test logs on failure" step, but the run - artifact list contains only `docker-preview-1`; the `runtime-server` artifact - is absent. The test now prints the queried chunk and location on this failure. - Wait for a recurrence and use those diagnostics to identify the first - divergent state before changing the assertion or retry policy. diff --git a/doc/backlog/R168-s3-shared-object-reclamation.md b/doc/backlog/R168-s3-shared-object-reclamation.md deleted file mode 100644 index e44291a0f..000000000 --- a/doc/backlog/R168-s3-shared-object-reclamation.md +++ /dev/null @@ -1,73 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R168: access server / S3 — Shared small-object reclamation - -## Status - -**Deferred until R95 qualified chunk-range deletion is implemented and basic -S3 deletion is stable.** Before then, the basic delete path removes metadata -and records exact -pending ranges as safe logical garbage. - -## Problem - -Small objects use the shared chunk writer to avoid private-chunk waste. Their -bytes cannot be reclaimed by deleting the whole chunk, and an unqualified byte -range could erase neighboring live objects or a reused generation. The basic -delete path needs -a restart-safe way to turn pending logical garbage into safe range deletion. - -The lifecycle boundary is -`doc/design/access-server/s3/design-crowdb-access-s3.md` §5. - -## Solution - -1. Persist a range-cleanup record containing object generation, chunk identity, - writer/layout generation, exact offset/length, reader-validity deadline, and - stable cleanup identity before invoking physical deletion. -2. After metadata invisibility and the grace period, call R95's idempotent - `DeleteChunkRange(chunk_id, offset, size)` interface. The RPC exists before - R95 but returns not-implemented and performs no mutation until chunkdb owns - the required range validation and used-bitmap lifecycle. -3. Reconcile unknown results from chunk state, retry bounded transient failures, - and quarantine generation/ownership mismatches for inspection. Never widen, - merge, or align a range across live neighbors merely to reclaim space. -4. Track logical garbage, eligible bytes, completed bytes, retry age, and - quarantine without high-cardinality labels. Apply backpressure to cleanup - independently of foreground DELETE. -5. Mark the S3 cleanup complete only after the qualified range is known deleted; - absence of cleanup never changes the object's already-absent visibility. - -## Dependencies - -- Depends on R95 and the basic S3 deletion contract. -- Uses immutable generation/data references and bounded cleanup admission. -- R169 may compact fragmented shared chunks after individual dead ranges are - marked; it is not required for correctness here. - -## Acceptance - -- Given adjacent live and deleted small objects in one shared chunk, when the - qualified range deletion runs after grace, assert only the deleted object's - exact bytes become reclaimable. Invariant: shared reclamation cannot damage - a neighbor. E2E test. -- Given stale chunk/writer generation or a range that has been reused, when - cleanup runs, assert R95 rejects/quarantines it without mutation. Invariant: - offset and length alone are never deletion authority. Integration test. -- Given lost replies, restart, and repeated cleanup delivery, when the - reconciler settles, assert one terminal result is recorded and visibility - stays absent throughout. Invariant: range cleanup is idempotent and separate - from logical deletion. E2E test. -- Given a sustained delete backlog, when configured cleanup limits are reached, - assert foreground object operations retain their reserved resources and - garbage metrics remain bounded-cardinality. Invariant: reclamation cannot - starve the data path. Integration test. - -Required gates: - -- `pixi run -- cargo test -p crowdb-access-s3 --all-targets` -- `pixi run -- cargo test -p crowdb-chunk-client --all-targets` -- `pixi run -- cargo test -p crowdb-chunkdb --all-targets` -- `pixi run -- cargo fmt --all -- --check` -- `pixi run rs-lint` diff --git a/doc/backlog/R169-s3-shared-chunk-tree-gc.md b/doc/backlog/R169-s3-shared-chunk-tree-gc.md deleted file mode 100644 index 00b747303..000000000 --- a/doc/backlog/R169-s3-shared-chunk-tree-gc.md +++ /dev/null @@ -1,77 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R169: access server / S3 — B+tree and shared-chunk garbage collection - -## Status - -**Deferred until R147, R168, and the basic S3 workload provide measured garbage -and fragmentation data.** It is unblocked when qualified dead ranges and -metadata tombstones can be enumerated safely and compaction benefit is known. - -## Problem - -Qualified range deletion can mark shared object bytes dead, but mixed-live -chunks may remain fragmented and S3 metadata trees accumulate obsolete object -generations, tombstones, completed upload intents, and cleanup records. Simple -age-based deletion can race snapshots/readers or discard the only recovery -evidence for an ambiguous operation. - -The lifecycle scope is -`doc/design/access-server/s3/design-crowdb-access-s3.md` §5; generic tree -strip reclamation is R147. - -## Solution - -1. Measure dead/live ratios and plan bounded compaction of shared object chunks. - Copy only still-reachable ranges into new shared chunks, verify contents, - atomically replace generation references, then retire old layouts after the - reader-validity fence. -2. Feed unreachable storage strips and chunks into the qualified generic - in-chunk/whole-chunk lifecycle rather than freeing DiskDB ranges directly - from S3. -3. Add S3 metadata retention rules for obsolete generations, tombstones, - completed/aborted upload intents, and completed cleanup records. Prove they - are no longer visibility, retry, reader, or recovery authority before - deleting their Chunk-KV keys. -4. Integrate tree logical-GC output with R147's manifest-fenced strip - reclamation. Compaction and metadata GC are restart-safe, idempotent, - throttled, and lower priority than foreground traffic. -5. Expose reclaimable, copied, freed, pinned, quarantined, and amplification - metrics. Disable compaction when its projected benefit does not exceed the - configured threshold. - -## Dependencies - -- Depends on R147 and R168; R92/R95 provide qualified in-chunk/range lifecycle - mechanisms where applicable. -- Uses the basic milestone's generation metadata, retry identities, tombstones, - and background admission. -- It is not a prerequisite for basic S3 visibility or large-object deletion. - -## Acceptance - -- Given a shared chunk with interleaved live/dead ranges and pinned old readers, - when compaction runs, assert live bytes remain exact, metadata switches - atomically, and the old layout is reclaimed only after pins expire. - Invariant: compaction never changes visible object bytes. E2E test. -- Given obsolete and still-authoritative generations, intents, tombstones, and - cleanup records, when metadata GC evaluates them, assert only records proven - irrelevant to visibility/retry/recovery are removed. Invariant: GC cannot - erase recovery authority. Integration test. -- Given a crash after copy, metadata switch, or old-layout retirement, when GC - restarts, assert it resumes to one valid layout without leaks outside the - recorded candidate set. Invariant: every compaction phase is idempotent. - E2E test. -- Given low dead ratio or foreground pressure, when the planner evaluates work, - assert it skips/throttles compaction and respects reserved resources. - Invariant: GC benefit and load gates protect foreground service. Integration - test. - -Required gates: - -- `pixi run -- cargo test -p crowdb-access-s3 --all-targets` -- `pixi run test-tree-ct` -- `pixi run -- cargo test -p crowdb-chunkdb --all-targets` -- `pixi run -- cargo fmt --all -- --check` -- `pixi run rs-lint` diff --git a/doc/backlog/R185-access-iceberg-cache-invalidation.md b/doc/backlog/R185-access-iceberg-cache-invalidation.md deleted file mode 100644 index cf2f3d28d..000000000 --- a/doc/backlog/R185-access-iceberg-cache-invalidation.md +++ /dev/null @@ -1,129 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R185: access server / Iceberg — Bounded cache and invalidation - -## Status - -Deferred until R178 through R184 stabilize identities, generations, digests, -lifecycles, clear fences, and the measured uncached correctness baseline. Cache is -not a dependency of the first correct Iceberg milestone. - -## Problem - -Authoritative reads on every request create an active-catalog hot key and repeat -name, head, metadata, manifest, and format parsing work. Independent caches would -multiply memory budgets and stale-data rules. Cross-server notification can reduce -staleness but cannot be authority because instances disconnect, register late, and -receive duplicated or reordered messages. - -The native Iceberg design resolves lease-plus-grace clear semantics and requires per-class limits chosen -by focused benchmarks. This requirement adds one Iceberg-owned cache manager while -preserving correct behavior when notifications or the complete cache are disabled. - -## Solution - -- **CACHE-I1 — Authority isolation:** a hit can avoid I/O but cannot create, - publish, rename, drop, clear, recover, authorize, or reclaim a resource. -- **CACHE-I2 — Bounded ownership:** total and class bytes, entries, one-entry size, - fills, queues, batches, fanout, retries, and retained owners have separate caps. -- **CACHE-I3 — Qualified identity:** mutable entries carry CatalogId plus epoch or - generation; immutable entries carry stable identity plus generation/digest. -- **CACHE-I4 — Optional notification:** loss, duplication, reorder, late join, and - unreachable receivers preserve behavior through expiry and authoritative refresh. -- **CACHE-I5 — Lock-free hit:** a hot hit takes no global or cross-class lock. -- **CACHE-I6 — Rename is not aliasing:** an old table name can retain a bounded - tombstone hint but never resolves as a normal alias. - -1. Add `cache/manager.rs`, `budget.rs`, `policy.rs`, `entry.rs`, `fill.rs`, and - `metrics.rs`. Reserve a validated global byte budget and class budgets for active - catalog, names, authorities, heads, negative/tombstone entries, parsed metadata, - serialized responses, manifests, format structures, and file records. Oversized - entries bypass cache. -2. Configure maximum entries, bytes, one-entry bytes, idle TTL, absolute lease, - negative TTL, fill concurrency, and retained-owner accounting per class. Derive - checked-in initial defaults from focused cold/hot and adversarial benchmarks; a - missing, zero-invalid, or unbounded setting fails startup. -3. Treat active catalog, names, namespace authorities, and TableHeads as mutable. - An expired entry must refresh authoritatively or fail closed. A mutation always - validates current durable revision and cannot succeed from cache alone. -4. Cache metadata projections under CatalogId, TableId, generation, JSON digest, - and projection version. Cache manifests and parsed Parquet/ORC/Avro/Puffin - structures under FileId, digest, and parser version. Whole large files never - enter this cache. -5. Add one bounded typed invalidation coalescer. After a durable mutation completes, - coalesce transitions by resource to the newest generation or strongest state. - Immutable files emit no event. Queue saturation drops optimization events rather - than delaying the mutation. -6. Start one internal `crowdb-rpc` listener per Access Server and publish its - endpoint through the Group-0 service registry. Use service-specific FlatBuffer - messages, bounded control size, deadlines, connection/fanout concurrency, and - retries. External Iceberg HTTP never accepts invalidation RPC. -7. Receivers validate the complete batch before applying any entry, then apply it - idempotently. Older generations are ignored. Table rename converts an existing - old-name entry to an authorization-neutral tombstone; only request-time current - authorization may disclose the destination. -8. Integrate clear with the native Iceberg contract: notification prompts eviction, but completion waits - for R178's persisted maintenance deadline and admitted/delegated grace. Start - lease age before the authoritative root read, never when a delayed reply arrives; - maintenance prevents fresh leases, while an existing lease may admit old-context - requests only until its original expiry. New-domain admission opens after the - grace proof. Retired-catalog GC uses - durable fences and never waits for physical cache eviction acknowledgements. -9. Expose per-class hit, miss, stale, fill, bypass, bytes, entries, eviction, expiry, - rebuild, notification, fanout, drop, and refresh-failure metrics. Benchmark hit - latency and contention for unrelated cache classes before enabling by default. - -## Dependencies - -- Depends on R177 through R184 and their final identities and measured baseline. -- Reuses Group-0 service discovery and `crowdb-rpc`; if registration is unavailable, - notification is disabled and TTL-only correctness remains. Static instance lists - are forbidden. -- Does not depend on R82 watch coalescing. This coalescer batches typed post-commit - cache transitions, not KV watch keys. -- Canonical JSON and file bytes remain the fallback for every missing, corrupt, - unknown-version, or evicted projection/cache entry. - -## Acceptance - -- Given mixed entries and fill pressure beyond every total and class limit, when - admission, eviction, and bypass run concurrently, assert retained owners, bytes, - entries, queues, and work stay within each configured cap. Invariant: CACHE-I2. - Integration test. -- Given an expired active-catalog or TableHead entry while Chunk-KV is unreachable, - when a request arrives, assert it fails closed and never reuses stale authority. - Invariants: CACHE-I1 and CACHE-I4. Integration test. -- Given duplicate, reordered, dropped, oversized, and malformed heterogeneous - invalidation batches, when receivers apply them, assert validation is atomic, - current entries only advance, and TTL refresh converges missed entries. Invariants: - CACHE-I3 and CACHE-I4. Integration test. -- Given a successful table rename and later authorized and unauthorized old-name - requests, when a cached tombstone exists and then expires, assert it never loads - the table, disclosure is authorization-filtered, and an uncached old name is - ordinary not-found. Invariant: CACHE-I6. E2E test. -- Given valid and corrupt metadata/format projections, when cached loads execute, - assert qualifiers prevent cross-generation reuse and every invalid entry falls - back to canonical bytes without authority mutation. Invariants: CACHE-I1 and - CACHE-I3. Integration test. -- Given a hot commit stream, unrelated mutations, queue saturation, disconnected - instances, and a late server, when fanout runs, assert work remains bounded, - newest generations converge, committed mutation latency is unaffected, and the - late server loads authority before readiness. Invariant: CACHE-I4. E2E test. -- Given a disconnected old-root lease holder and delayed cache fills, when clear - enters maintenance and publishes a new pointer, assert no lease extension and - no old-context response after the persisted completion boundary, even across a - recovering clear coordinator. Invariants: CACHE-I3 and CACHE-I4. E2E test. -- Given concurrent hits, fills, invalidations, expiry, and eviction across classes, - when contention benchmarks run, assert hot lookup takes no global lock and its - latency is independent of unrelated class activity. Invariant: CACHE-I5. - Integration test. - -Required gates: - -- `pixi run -- cargo test -p crowdb-access-iceberg --all-targets` -- `pixi run -- cargo test -p crowdb-access-server --all-targets` -- `pixi run -- cargo test -p crowdb-kv-client --all-targets` -- `pixi run -- cargo test -p crowdb-rpc-ffi --all-targets` -- `pixi run -- cargo fmt --all -- --check` -- `pixi run rs-lint` diff --git a/doc/backlog/R186-access-iceberg-orc-validation.md b/doc/backlog/R186-access-iceberg-orc-validation.md deleted file mode 100644 index 9e850c092..000000000 --- a/doc/backlog/R186-access-iceberg-orc-validation.md +++ /dev/null @@ -1,68 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R186: access server / Iceberg — Selected ORC validation - -## Status - -Deferred by user decision as an independent, unimplemented ORC follow-up. -Resume when ORC support is selected for implementation; it is not a completion -gate for the completed Parquet container/client workflows. Those workflows do -not absorb selected ORC validation. Selected-file validation continues to reject ORC explicitly -rather than representing an unchecked file as validated. - -## Problem - -Native FileIO can retain immutable ORC bytes, but container recognition does not -prove selected schema, row counts or delete semantics. Accepting a container hint -as proof would violate the canonical-file contract in -[the root design](../design/access-server/iceberge/design-crowdb-iceberg.md). -An official client configured to write ORC needs a distinct, tested validation -capability rather than an implicit fallback to Parquet checks. - -## Solution - -1. Extend `crowdb-access-iceberg::file` with bounded canonical ORC decoding. Use - the official ORC protobuf and pinned Iceberg mappings and SDK as authorities; - decode compression framing correctly and reject unsupported codecs or - encryption explicitly. Independently limit encoded/decoded bytes, protobuf - work, type depth/count and stripe count. -2. Extend `crowdb-access-iceberg::manifest` selected-file validation with field-ID, - historical schema, row-count and supported delete checks. Bind the manifest's - semantic kind without changing immutable file authority. Never trust hints - instead of canonical bytes. -3. Integrate ORC into complete selected snapshot validation only after its format - gates pass. Failure or cancellation publishes no validation result or head. - Keep unsupported selected ORC explicit before this capability lands; ordinary - immutable upload is not a table-selection or commit proof. - -## Dependencies - -- R180 supplies immutable files and canonical range reads. -- R181/R182 supply trusted table metadata and selected snapshot validation. -- Extend the completed REST/SDK conformance profile with ORC after this - requirement passes; initial Parquet-only acceptance is already complete. -- R185 caches are optional; uncached canonical reads remain correct. - -## Acceptance - -- Given official SDK ORC fixtures and supported compression variants, when - canonical metadata is decoded, assert schemas and row counts match the writer. - Invariant: canonical authority. Integration test. -- Given malformed framing, excessive expansion, deep types, invalid stripes, - encryption or unsupported codecs, when decoded, assert bounded failure with no - successful proof. Invariant: independent resource limits. Unit test. -- Given data and delete manifests with historical schemas and false counts, when - selected, assert compatible files pass and mismatched kind, identity, fields or - counts fail. Invariant: manifest-to-file binding. Integration test. -- Given a complete snapshot containing ORC and a failure or cancellation during - validation, when publication is attempted, assert no head advances; before ORC - support, assert selection fails explicitly. Invariant: fail-closed publication. - Integration test. - -Required gates: - -- `pixi run -- cargo test -p crowdb-access-iceberg --all-targets` -- `pixi run -- cargo test -p crowdb-access-server --all-targets` -- `pixi run rs-fmt-check` -- `pixi run rs-lint` diff --git a/doc/backlog/R193-chunkdb-node-failure-budget.md b/doc/backlog/R193-chunkdb-node-failure-budget.md deleted file mode 100644 index e2c5159f5..000000000 --- a/doc/backlog/R193-chunkdb-node-failure-budget.md +++ /dev/null @@ -1,47 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R193: chunkdb — Configurable node failure budget and EC placement - -#### Status - -Planned. The explicit single-node and three-node profiles, persisted strip layouts, and degraded-placement repair baseline are complete. - -#### Problem - -The current system defines two deployment contracts: one test node with no node-failure tolerance and three production nodes tolerating one failed node. The current EC selector checks the maximum fragments on one node against the parity count. That check cannot express a larger failure budget: with six nodes and two allowed failures, `4+2` and `8+4` can survive any two nodes, while `2+1` cannot. Mirror copy counts and journal/tree write policies are also configured independently rather than derived from one system protection contract. See [chunkdb placement](../design/chunkdb/design-crowdb-chunkdb.md) and [chunk IO](../design/chunkio/design-crowdb-chunkio.md). - -Operators need to choose a node failure budget for a deployment without accidentally admitting an EC shape or mirror layout that loses committed data within that budget. When nodes fail, new placement must use the remaining budget and record any loss of the full-cluster protection target for repair after recovery. - -#### Solution - -The system property `max_node_failures` is the number of unavailable nodes the configured deployment promises to tolerate from its complete topology. It is a protection target, not an automatic instruction to reduce copies each time a node fails. The remaining budget is the target minus the nodes already unavailable. A production profile must have enough voting KV replicas to retain quorum at that target and enough distinct storage nodes to place its selected layouts. The explicit test-single-node profile has a zero budget. Startup and allocation reject mismatched service policy, impossible budgets, and unsafe layouts; failure never silently switches deployment mode. - -For a mirror strip, the full protection target requires at least `max_node_failures + 1` copies on distinct nodes. Continue to use the full copy count after a failure when placement permits it. For an EC strip with `k` data and `m` parity fragments, sort per-node fragment counts descending; the sum of the largest `max_node_failures` counts must be at most `m` in a healthy topology. Apply the corresponding remaining-budget check to new allocations after failures. Keep fragments spread across available nodes even when no further node-failure budget remains. Persist actual strip geometry and its full protection target separately so readers use the real layout and repair can restore the target. - -1. Validate the deployment property and KV/storage topology consistently in deployment configuration, ChunkDB startup, and access/chunk writer policy. Preserve the existing one-node/zero-failure and three-node/one-failure profiles. -2. Replace the one-node EC bound in ChunkDB placement and physical validation with the worst-case sum across the configured number of failed nodes. Select mirror copy counts from the deployment contract, while retaining explicit per-strip policy only when it meets or exceeds the target. -3. During an outage within the configured budget, try full protection first. If it cannot fit, permit a layout that meets the remaining budget, persist a degraded-placement marker, and create a durable placement task. Reject allocation if even the remaining-budget layout or KV quorum is unavailable. Never claim full protection for a degraded strip. -4. After capacity returns, use fenced placement tasks to move or rebuild fragments until the original target holds. Reads and writes follow each strip's persisted geometry throughout migration; a restart resumes unfinished tasks without accepting stale placement. - -#### Dependencies - -- The existing two deployment profiles, strip dispatch, and placement-repair baseline are the starting point for generalized validation. -- R103 is responsible for ChunkDB range-owner migration after a ChunkDB instance failure. This requirement's storage placement budget does not replace metadata service failover; end-to-end availability depends on both. -- R139 may later distribute the system property through Group 0. Until then, startup must reject inconsistent local configuration rather than assume a remote configuration service exists. - -#### Acceptance - -- Given six voting KV/storage nodes and `max_node_failures = 2`, start services; startup accepts the budget and reports the configured target. Given an impossible budget or too few voters, startup rejects it before writes. Integration test. -- Given six healthy nodes with budget two, allocate `4+2` and `8+4` EC strips; every pair of nodes owns at most two and four fragments respectively. Request `2+1`; allocation rejects it because a pair can own more than one fragment. Integration test. -- Given six healthy nodes with budget two, allocate mirror strips used by object data, journal, and tree pages; each new strip has at least three copies on distinct nodes and reads from its persisted layout. Integration test. -- Given a six-node budget-two cluster, stop one node and allocate new mirror and EC strips; placement retains the full target where possible, otherwise uses the remaining one-failure budget and persists a repair task without changing deployment mode. E2E test. -- Given the same cluster with two nodes unavailable, allocate while KV quorum and a valid remaining-budget placement exist; operations succeed without claiming that a third failure is tolerated. Remove enough further capacity or quorum; allocation returns an error. E2E test. -- Given a degraded EC strip and recovered nodes, restart the repair worker and complete its task; data remains readable during movement, the task survives restart, and the final placement again passes the full two-node-failure check. E2E test. -- Given the existing single-node and three-node configurations, run their write/read and one-node-out suites after introducing the generalized policy; their configured budgets and persisted layouts remain valid. E2E test. - -Run `pixi run rs-fmt-check`, `pixi run rs-lint`, `pixi run cargo test -p crowdb-kv`, `pixi run cargo test -p crowdb-chunkdb`, `pixi run cargo test -p crowdb-chunk-client`, and `pixi run cargo test -p crowdb-chunk-stream` for the implemented scope. - -#### Open Questions - -None. diff --git a/doc/backlog/R196-refine-benchmark-tests.md b/doc/backlog/R196-refine-benchmark-tests.md deleted file mode 100644 index a5de1d89d..000000000 --- a/doc/backlog/R196-refine-benchmark-tests.md +++ /dev/null @@ -1,282 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R196: benchmark — Refine benchmark tests - -#### Problem - -Service benchmarks currently mix native regression fixtures, standalone Rust -bench binaries, and `crowdb-cli bench` commands. Result parsing, phase checks, -CPU accounting, and reports are repeated in shell/Python drivers. The existing -S3 CLI already sends authenticated HTTP requests to a selected cluster, but -does not observe server resource usage. Iceberg lacks an equivalent reusable -upload regression. These differences make storage and access measurements hard -to reproduce or relate to one another. - -The current Stream measurements demonstrate why deployment and CPU conditions -belong with every result: a client and services sharing CPUs 0–3 behave very -differently from the same binaries allowed on CPUs 0–31. Chunk-KV uses Stream -for its WAL, so Stream waits must also be observable in the KV workload. Object -performance needs its own measurements above ChunkIO. Neither an isolated -Stream microbenchmark nor an HTTP upload result describes all these layers. - -This requirement unifies the service benchmark regressions under -`tools/benchmark/`, including RPC, KV, DiskDB, ChunkDB, ChunkIO/object, -Chunk-Stream, Chunk-KV, S3, and Iceberg. Relevant contracts are the -[OCI cluster lifecycle](../../container/oci-image/README.md), -[Stream design](../design/chunkds/design-crowdb-chunk-stream.md), -[Chunk-KV design](../design/chunkds/design-crowdb-chunk-kv.md), -[object writer design](../design/chunkio/design-crowdb-chunkio-small-object-writer.md), -and [upload flow](../design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md). - -#### Solution - -The default regression deployment is a **three-node cluster with one node per -container**. A host-side `crowdb-cli bench` process is the external benchmark -tool. Publish the test servers' required ports and advertise endpoints reachable -by that client. All service regression drivers adopt this deployment and CLI -path; the current native Stream fixture remains only until this work lands. - -Responsibilities are explicit: - -- Existing OCI/cluster lifecycle code builds and starts the three containers, - prepares storage, checks full application readiness, exposes endpoints, and - stops only the invocation's owned resources. -- `crowdb-cli bench` owns workload execution, sampling, resource/metrics - collection, accounting, validation, and the common structured report. Reuse - common-library system counters and existing management/status APIs. -- Each driving script selects cases and deployment/CPU profiles, invokes the - external CLI, retains its exit status and artifacts, and records measured - results in that same script's comments. Do not maintain separate tracked - result documents or baseline JSON files, or duplicate collectors in scripts. - -The following invariants define the benchmark: - -- **I1 — Actual service path.** The external CLI reaches the selected cluster - over published ports. Storage workloads use the deployed services; S3 and - Iceberg use authenticated HTTP. The CLI does not create an in-process cluster - as a substitute. A successful write means the layer's configured durability - and publication contract is satisfied. -- **I2 — Controlled input and timing.** Record payload seed, size, concurrency, - count/duration, warmup, read-cache policy, and media/durability profile. Bound - client payload memory independently of large object size. Separate cluster - setup, dataset preparation, warmup, measurement, drain, verification, and - cleanup. A duration-limited case cannot silently finish early due to a small - operation limit. Timed 4-KiB single-/32-producer write cases run at least - 60 seconds by default; short pipeline smoke tests are labelled separately. -- **I3 — Complete accounting.** Track admitted, completed, failed, and incomplete - operations, drain admitted work, and verify deterministic bytes outside the - timed interval. A timeout, missing result/required metric, failed cleanup, - or unverified write fails the case. Missing cases cannot disappear from a - passing summary. Bounded memory and queue limits remain regression gates. -- **I4 — Attributable measurements.** Identify client, all three nodes, service - instances, and measurement windows. Counts and sums are interval deltas; - interval percentiles come from matching samples/histogram buckets, never - subtraction of cumulative percentiles. Do not average node percentiles into - a cluster percentile. Concurrent phases overlap and cannot be summed as if - they partitioned request latency. Counter resets or replaced processes are - explicit. Optional unavailable counters are `unsupported`, never zero. -- **I5 — Comparable conditions.** A result includes exact commands and software, - image, topology, CPU, runtime, storage, and durability identities. Compare - only matching profiles with documented tolerances. Default placement and a - constrained cpuset are separate profiles; native MemDisk and container disk - results are also distinct. The accepted implementation is the baseline; - the runner does not build or run an older binary to produce a comparison. -- **I6 — Owned three-node deployment.** Readiness requires all three members - and usable storage. Port conflicts, missing peers, or unreachable advertised - endpoints fail before measurement, without singleton/native fallback. - Failure/interruption retains diagnostics and cleans only resources owned by - this invocation. Image builds, compilation, tests, and other workloads run - outside performance measurement windows. -- **I7 — Results live with the driver.** Record the dated measured summary and - its conditions directly in each driving script. The CLI can emit JSON/TSV - and raw artifacts into ignored runtime directories and generate a comment - block for the driver. Retain useful previous measurements there with their - profile/revision so comparisons have context. Do not create an additional - tracked result file; optional raw traces do not replace the script summary. - -Work items: - -1. Reuse `container/oci-image/` and the existing CLI cluster lifecycle to run - the owned three-container cluster. Publish HTTP and the management/RPC/data - endpoints needed by each selected workload; verify client reachability of - every address returned by discovery, routing, and redirects. Supply the - accepted public origins when creating the deployment. The CLI runs outside - all node containers, using explicit target bindings and private credentials. - Record exact container/image identities and port bindings. Do not implement - another bootstrap or silently attach to an unrelated running cluster. -2. Refine the commands under `app/crowdb-cli/src/commands/bench/`, extending its - existing `metrics` and `result` responsibilities and reusing workload code - in `lib/crowdb-console-shared/src/ops/` and owning storage libraries. Add - Stream and Chunk-KV entry points where absent; move the reusable Stream - workload behind the CLI. Preserve S3's existing real HTTP path and extend - it for bounded large uploads and multipart. Add the Iceberg FileIO adapter. - All workload families share accounting, collector lifetimes, output schema, - and failure handling instead of parsing human-readable metrics in scripts. -3. Implement one CLI collection lifecycle around preparation, measurement, - drain, and teardown. Take service snapshots before/after the measured window - and flush final client metrics. Use bounded histograms and existing periodic - system metrics; make per-operation traces optional. Report collection cost - and window boundaries, with no request-path file/process polling. Retrieve - server process/cgroup evidence through the owned deployment or management - APIs, mapping container identities explicitly; never interpret a container - PID as a host PID or report CLI-only CPU as cluster CPU. -4. Capture the report contents and workload matrix below. Reuse shared system - counters for CPU and C6; do not add another hardware counter implementation - to the scripts or CLI. Permission failures leave optional fields explicitly - `unsupported`; default benchmarks need no extra perf privileges or power - policy changes. A missing required workload/phase field still fails. -5. Convert every service regression driver under `tools/benchmark/` to the - three-node container plus external CLI path, retaining case filters, bounded - timeouts, and complete failure output. The scripts carry named case/gate - configuration and measured result comments; collection, latency math, and - report validation belong in the CLI. Preserve correctness gates during the - conversion. Recapture results for the new container profile rather than - treating native measurements as directly comparable. Support an explicitly - selected remote cluster through the same CLI, with unavailable local-host - measurements marked `unsupported` and no remote deployment cleanup. - -Required report contents: - -- **Conditions:** UTC date; CLI command; source revision and dirty status; - client/service binary hashes and image digest; host CPU model, kernel, - physical/logical cores, SMT/NUMA topology; three node identities, placement, - ports, mirror/EC policy, media type/device, KV/WAL backend, fsync/durability, - dataset/cache policy, and runtime/RPC worker counts. Retained MemDisk, null - data sources, sparse files, and real disks must be named accurately. -- **CPU placement and usage:** requested and effective cpusets for client, - containers, and each service's worker threads; cgroup CPU quota/period and - throttling deltas; client and per-node/service user/system CPU seconds, - average CPU %, RSS/peak/growth, context switches and migrations where - available. State that 100% means one logical CPU. Separate timed-window - usage from whole-command usage that includes dataset preparation. -- **System evidence:** disk and network bytes/operations and resource pressure - when available; per-core C6 and per-package C6 residency with units, elapsed - interval and availability. Counter support/permission/reset states stay - distinct. Optional detailed perf profiles have explicit collection settings - and their own comparable profile; no per-operation privileged probes. -- **Workload outcome:** logical/physical bytes, operations/s and MiB/s, actual - duration, admission/drain counts, errors/retries/timeouts, error classes, - count/mean/p50/p95/p99/max latency and a mergeable histogram. State sample - counts and resolution; omit unsupported percentiles with a reason. Separate - client payload generation/signing/hash work from transfer/server stages. -- **Latency attribution:** queue depth/bytes and wait; RPC round-trip and - response scheduling; append batch/mirror waits; asynchronous capacity and - candidate preparation versus foreground wait; seal/rollover, StreamChunkRecord - publication/build/get/scan/CAS, checkpoint and replay; cache/physical reads - and reclaimed bytes. Chunk-KV adds WAL append/commit, tree read/write, - flush/snapshot/compaction, and read-cache evidence. Object adds admission, - batching, reservation/preparation, payload write, parity, and metadata - publication. Use the actual available stage counters and identify required - fields by workload; capture count, sum, distribution, and max separately. -- **Protocol stages:** S3/Iceberg direct PUT, UploadPart, multipart completion, - and optional table commit are separate operations. A successful part does - not count as a published object. Upload completion includes protocol metadata - publication; FileIO latency does not include an unrelated snapshot commit. - -Minimum workload matrix: - -- **Existing RPC/KV/DiskDB/ChunkDB cases:** retain their meaningful workload and - correctness coverage while moving startup and measurement to the common path. -- **Stream:** preserve the current seven cases: 4-KiB append at 1/32 producers, - 1-MiB append at 8, 4-MiB rollover, random 4-KiB read at 32, 320-MiB replay, - and 576-MiB GC. Ordinary append must not checkpoint the cursor; metadata - publication scales with chunk transitions, not historical record count. -- **Chunk-KV:** 4-KiB values at 1/32 clients for put/get, plus batch and ordered - scan; record key/value sizes, batch size, read/write mix and cache state. - Include a run crossing WAL chunk rollover and a bounded restart/replay case. - A sustained case must expose background flush/snapshot/compaction work and - cannot end before queued durable work drains. Correlate end-to-end KV latency - with its Stream WAL metrics in the same report. -- **Object/ChunkIO:** small and large write/read plus mixed reads through the - existing object writers/readers; include 4-KiB at 1/32 clients, 1-MiB at 8, - and the configured small/large boundary. Record physical write amplification, - batching, placement and verification rather than inferring performance from - Stream or KV alone. -- **S3/Iceberg:** small-path and threshold-boundary uploads, direct 100-MiB PUT, - and multipart 100-MiB upload, at single and concurrent clients. Reuse bounded - deterministic payload blocks and precompute expected integrity data outside - timing; record the integrity mode and part size. Preserve S3 read/range/list - and mix coverage through the same collector. Final bounded byte/range checks - occur outside upload timing. - -#### Dependencies - -- Existing three-node OCI cluster lifecycle, readiness, target bindings, and - port/public-origin configuration. Add missing external RPC/data exposure in - that lifecycle; an unreachable route blocks the affected benchmark rather - than falling back to native services. -- Existing storage benchmark workloads, Stream/Chunk-KV APIs, object writers - and readers, and S3/Iceberg authentication/publication paths. Reuse these - contracts and existing native results as recorded evidence; this requirement - owns their CLI/container regression conversion and new profile measurements. -- Shared metrics/system-status counters and management snapshots provide - collection. Hardware-dependent counters remain optional if unavailable; - required stage coverage is implemented or its case fails explicitly. -- Official-client compatibility remains in the existing E2E suites. A - client-inclusive profile can use the common result schema without replacing - the controlled CLI workload or mixing their results. - -#### Acceptance - -- Given an owned three-container cluster, run every converted driver with the - host CLI; assert three ready members, published reachable endpoints and real - service requests, with no native/in-process replacement (I1, I6). E2E test. -- Given a port collision, absent third member, or an unreachable advertised - RPC/HTTP address, start a case; assert failure before timing, useful - diagnostics, and no singleton fallback (I1, I6). E2E test. -- Given an existing unrelated cluster and an interrupted/failed benchmark, - stop the run; assert only owned containers/data are cleaned under the declared - retention policy and all failure logs remain available (I3, I6). E2E test. -- Given each minimum workload and fixed deterministic inputs, execute through - the CLI; assert complete operation/byte accounting, bounded memory/queues, - real stored-byte verification, and a common report (I1–I4). E2E test. -- Given S3 and Iceberg multipart uploads, complete both; assert distinct part, - completion and optional table-commit measurements, correct integrity checks, - and no part counted as a final object (I1, I3, I4). E2E test. -- Given a 60-second 4-KiB case and a small operation cap, execute the duration - profile; assert it cannot receive a passing full-duration verdict early and - that setup/warmup/drain have distinct windows (I2, I3). Integration test. -- Given permission-denied hardware counters and remote endpoints, collect - metrics; assert per-core and package C6/local-resource availability is - `unsupported` where appropriate, with no fabricated zeros or default sudo - requirement (I4). Integration test. -- Given independent client/container cpusets and a throttled node, run a case; - assert effective per-service/thread placement, quota, throttling, CPU time, - RSS and timing windows are attributed correctly (I4, I5). E2E test. -- Given overlapping stage histograms, low sample counts, process replacement - and counter reset, build the report; assert interval aggregation is valid, - unsupported percentiles are identified, and neither percentile subtraction - nor exclusive summation of overlapping phases occurs (I4). Unit test. -- Given Chunk-KV WAL rollover/replay and object small/large-boundary workloads, - run the CLI cases; assert their own verified outcomes, latency distributions, - Stream WAL/object stages, and drained durable work (I1–I4). E2E test. -- Given a failed request, missing phase/result/case, malformed report, timeout - or cleanup failure, summarize the run; assert nonzero status and retained - diagnostic evidence instead of a partial passing result (I3, I4). Integration test. -- Given matching and mismatched measurement profiles, assess recorded results; - assert only matching profiles receive a tolerance verdict and that the current - baseline does not require running an older binary (I5). Unit test. -- Given a completed run, generate its driver comment block; assert it includes - results, date, exact command, image/revision, cpusets, CPU usage and artifact - references, and can update the driving script without another tracked result - file. Assert scripts delegate collection/validation to the CLI (I7). Integration test. - -Run `pixi run rs-fmt-check`, `pixi run rs-lint`, -`pixi run cargo test -p crowdb-console-shared`, -`pixi run cargo test -p crowdb-cli`, and -`pixi run test-container-e2e-fixture`. -Run each converted service regression serially through -`pixi run bash tools/benchmark/bench-chunk-stream-regression.sh`, -`pixi run bash tools/benchmark/bench-chunk-kv-regression.sh`, -`pixi run bash tools/benchmark/bench-chunkio-write-regression.sh`, -`pixi run bash tools/benchmark/bench-chunkio-small-write-regression.sh`, -`pixi run bash tools/benchmark/bench-chunkio-read-regression.sh`, -`pixi run bash tools/benchmark/bench-rpc-regression.sh`, -`pixi run bash tools/benchmark/bench-kv-write-regression.sh`, -`pixi run bash tools/benchmark/bench-kv-read-regression.sh`, -`pixi run bash tools/benchmark/bench-kv-scan-regression.sh`, -`pixi run bash tools/benchmark/bench-diskdb-regression.sh`, and -`pixi run bash tools/benchmark/bench-chunkdb-regression.sh`. -Verify the S3/Iceberg adapters with `pixi run test-e2e-s3` and -`pixi run test-e2e-iceberg`, including their new CLI regression cases. diff --git a/doc/backlog/R207-chunkdb-repo-metadata-chunk-kv.md b/doc/backlog/R207-chunkdb-repo-metadata-chunk-kv.md deleted file mode 100644 index 0703d6da3..000000000 --- a/doc/backlog/R207-chunkdb-repo-metadata-chunk-kv.md +++ /dev/null @@ -1,318 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R207: chunkdb — Repo chunk metadata and tasks on chunk-kv - -Status: Deferred by user beyond the current direct-KV scope. This document -records the selected future architecture and its unresolved contracts. Start -implementation only when this follow-up is selected and its publication, -range-layout and migration policies have been settled. R202 does not depend -on completing this requirement. - -#### Problem - -All production ChunkDB Chunk records, maintenance tasks, task indexes and -reservations currently use `CrowdbKvClient` and direct Paxos KV groups. -Their routing cache loads an independently published 1024-slot storage map -whose eligible destinations are selected nonzero groups; group 0 holds only -control-plane bindings. System and user-data lifecycle/task runtimes have -separate authority and execution capacity. S3 and IcebergTable have distinct -chunk types, -but neither has a chunk-kv metadata backend. Existing chunk-kv functionality -must not be mistaken for an already connected ChunkDB persistence path. -See [ChunkStore](../../app/crowdb-chunkdb/src/storage.rs), -[TaskStore](../../app/crowdb-chunkdb/src/task/store.rs), -[routing](../../app/crowdb-chunkdb/src/routing.rs) and -[startup](../../app/crowdb-chunkdb/src/main.rs). - -Growing user-data metadata and maintenance traffic therefore grow Paxos -storage and write load. The goal is to move that user-data state onto chunk-kv -while retaining the system state needed to operate and recover chunk-kv on -direct KV. Changing the client alone cannot preserve the current publication, -task and recovery contracts. - -The [ChunkDB root design](../design/chunkdb/design-crowdb-chunkdb.md) describes -the existing lifecycle; [fixed slot routing](../design/chunkdb/design-crowdb-chunkdb-range-binding.md) owns the -current type/layer definitions and direct-KV partition contract. The -[chunk-kv design](../design/chunkds/design-crowdb-chunk-kv.md) supplies ordered -storage ranges. This follow-up owns their integration, not a replacement -partition engine or a global transaction manager. - -##### Audited gaps and potential failures - -Source inspection on 2026-10-03 found the following; no future-path performance -or runtime correctness is claimed by this audit. - -- **Backend and lifecycle dispatch:** ChunkStore and TaskStore use concrete - direct-KV clients. Allocation, updates, listing, reservations, task scanning - and owner queries have no complete repo backend path. A shared backend switch - would not establish the required separate system/repo operation logic. -- **Recursive dependency:** chunk-kv's journal uses chunk-stream. Allocation, - owner validation and reopen now preserve an explicit system `Wal` or business - `Stream` purpose. Backend dispatch must preserve that distinction: routing - system WAL metadata to chunk-kv would make journal metadata depend on the - storage it must recover. The binding's `owner_kind="chunk-kv-partition"` - does not itself select the allocated type. See - [Stream allocation](../../lib/crowdb-chunk-stream/src/production_chunk/allocation.rs), - [owner validation](../../lib/crowdb-protocol/src/chunk_stream.rs) and - [storage assembly](../../app/crowdb-chunk-kv-server/src/storage.rs). -- **Atomic publication:** chunk creation writes a Chunk, finalize task and - index using direct-KV `batch_write_cas`. Task transitions update canonical - state and secondary indexes together; reservation operations conditionally - publish chunk and reservation records together. Chunk-kv's current - [batch_mutate](../../lib/crowdb-chunk-kv-client/src/compose/batch.rs) is - explicitly non-transactional; its point mutations do not expose an equivalent - conditional multi-key commit. Same-range placement alone does not add it. -- **Ordered key placement:** `/chunk/`, `/reservation/` and binary task tags - have different key prefixes; task indexes order priority/time before chunk - identity. Direct KV explicitly selects their destination group. Routing the - same bytes independently by ordered range may separate related records. - Even a shared prefix does not prevent a later split inside that prefix. - This is a future chunk-kv integration concern, not a cross-group transaction - gap in R202: its hash of the owning chunk ID routes all associated records - to the same direct-KV group regardless of their encoded key order. - `ChunkTaskKey.partition_id` currently carries a chunk identity, not a - chunk-kv catalog partition ID. See - [task keys](../../lib/crowdb-protocol/src/key/chunk_task.rs) and - [reservation storage](../../app/crowdb-chunkdb/src/storage/reservation.rs). -- **Task isolation:** current startup wires a shared task store/manager/scanner/ - executor pipeline. Common ready/finalize/expired-lease scans do not define - separate persistent system/repo task domains. Moving only Chunk records - would leave tasks behind or make them undiscoverable. -- **Unsafe reclamation:** `SegmentOwnerResolver` reads both chunk references - and active task targets. An unavailable repo backend must remain an error; - interpreting it as absence could free a repair target or live payload. - See [owner resolver](../../app/crowdb-chunkdb/src/task/owner.rs). -- **Residual Paxos traffic:** stream cursor advancement still calls - `advance_chunk_write`, persisting lower-layer Chunk metadata through direct - KV. Stream manifests/extents and tree root catalogs also use direct KV. - Offloading upper-layer records does not prove that total Paxos operations - per user mutation fall. See - [cursor advancement](../../lib/crowdb-chunk-stream/src/production_chunk.rs), - [stream metadata](../../lib/crowdb-chunk-stream/src/kv.rs) and storage assembly. -- **Ambiguous migration:** S3/Iceberg records and tasks already in direct KV - cannot be switched merely by changing a type-routing constant. Old and new - clients, expired task claims, incomplete copies and lost cutover replies - need one durable storage authority. Source and destination revisions are - backend-local and must not be treated as interchangeable CAS tokens. - -#### Solution - -##### Selected future distribution - -Use [AGENTS.md terminology](../../AGENTS.md#terminology): repo chunk is a -collective discussion term for user-data chunks, never a generic implementation -type. Inherit the type and separate operation contracts from -[ChunkDB type and operation domains](../design/chunkdb/design-crowdb-chunkdb.md#39-chunk-types-for-different-use-cases). - -- **System:** Wal journal chunks, BtreePage data chunks and PageIndex chunks - holding persistent tree page mapping tables keep their Chunk metadata, - associated maintenance tasks, indexes and reservations in hash-selected - nonzero direct Paxos KV groups under the R202 routing contract. -- **Repo:** S3, IcebergTable and business Stream chunk metadata, together with - their tasks, indexes and reservations, move to chunk-kv. Future Dataset chunks - follow this layer with their own type; this requirement does not create a - Dataset API or claim that its type is already implemented. -- **Payload:** all chunk bytes remain on DiskIO-managed storage allocated by - DiskDB. This requirement changes the location of metadata and task state, - not the physical payload placement. -- **Bootstrap/control state:** group-0 bindings/catalogs, stream manifests and - extent pages, and tree root catalogs keep their explicit lower-layer - authorities. Page mapping payload in PageIndex chunks is distinct from - those chunks' metadata and from the root references used to locate them. - Relocating these control collections is outside this requirement. - -The dependency direction is: - -```text -repo operation domain -> chunk-kv metadata range -> system WAL/tree/PageIndex chunks - -> direct KV metadata + DiskIO payload -``` - -System recovery must not traverse the repo operation domain. A system task -may repair a physical tree chunk containing repo records; it remains a system -chunk task. It must not claim or execute the repo tasks stored inside that tree. - -##### Operation and range contracts - -- Dispatch by explicit type/purpose into separate system and repo lifecycle - orchestration. Reads, mutations, listing, task RPCs, reservations and owner - reconciliation all retain the layer. Codecs, wire envelopes, payload IO and - task algorithms may be reused; persistence authority and admission remain - independent. A repo outage never triggers direct-KV fallback after cutover. -- Chunk-kv already partitions by ordered range. Map the repo domain to those - ranges while preserving ChunkDB service routing as a separate authority. - R202 owns service partition topology; this requirement consumes it rather - than redefining a storage range whenever a service owner changes. -- Define the unit of conditional publication within one storage range. Either - add explicit range-local multi-record atomic commit with split-safe placement, - or use canonical single-key publication plus a reviewed recoverable index/ - task-discovery design. The latter must retain required lifecycle outcomes. - Do not wrap non-transactional batch_mutate and label it atomic. -- Define range-local task namespaces, due/lease scan scopes, claim ownership - and dispatch admission. Ordinary repo lifecycle does not require a global - due-time queue or cross-range transactions. Split/transfer must preserve the - chosen publication unit and task discoverability; a key prefix alone is not - a split fence. Namespace routing must not accidentally claim existing S3/ - Iceberg object or catalog keys hosted by chunk-kv. -- Separate system/repo task runtimes so upper-layer backlog or unavailable - storage cannot exhaust system task admission. Same-process versus separate - processes and exact capacity budgets remain open implementation choices. -- Coordinate a tree snapshot's page and mapping generations using the R202 - PageIndex contract. Do not publish a root that requires its own unavailable - mapping table to find the mapping chunks during cold recovery. - -##### Upgrade and failure outcomes - -- Bootstrap the lower system path before enabling repo operations. Verify the - target catalog, publication contract and task namespace before activation. - Until explicit enablement, the current direct-KV behavior remains unchanged. -- Backend migration covers the canonical chunk, tasks, indexes and reservations - as one recoverable logical unit. Persist migration identity and generation, - define source/destination authority at each stage, fence old writes/claims, - reconcile lost replies, and retire source records only after recovery proves - the destination authoritative. Exact online/offline policy is still open. -- Repo storage unavailable after cutover: return explicit retryable failures; - do not accept writes in direct KV or report data absent for reclamation. - Independently provisioned system operations and task admission remain usable. -- Task execution or client response is uncertain: preserve operation identity - and reconcile durable state before retrying; duplicate work must not publish - two state transitions or release a resource twice. -- Old service/storage generations after split or handoff: reject stale mutation - authority and refresh routing. Read fallback is allowed only by the reviewed - migration/read-consistency contract, not by endpoint reachability alone. - -##### Invariants - -- **I1 — Explicit distribution.** System metadata/task state stays in direct KV; - enabled repo metadata/task state uses chunk-kv, with unchanged payload placement. -- **I2 — Acyclic recovery.** System lookup, journal progress, mapping recovery - and maintenance never require repo metadata service availability. -- **I3 — Separate operation domains.** System and repo lifecycles, task scans, - claims and admission retain independent authority despite shared code. -- **I4 — Range-local consistency.** Publication and legal split boundaries - preserve required chunk/task/reservation outcomes and complete task discovery. -- **I5 — Durable authority.** Cutover, restart, stale caches and lost replies - retain acknowledged state and admit only the current writer/task authority. -- **I6 — Safe reclamation.** Unavailable metadata is not absence; current chunk - references and active task targets protect their exact allocation incarnation. -- **I7 — Measured relief.** Report total Paxos bytes, records and operations per - user mutation, including lower-layer cursor/checkpoint/task traffic, at equal - durability and workload. Do not claim savings from record relocation alone. - -##### Work items - -1. In ChunkDB lifecycle/storage/routing and startup, add the repo chunk-kv - operation path and explicit enablement while retaining the R202 system path. - Cover point operations, listing, reservations and administrative/task RPCs. -2. In the protocol key model and chunk-kv client/server, select and implement - range-local publication, namespace placement, legal split boundaries and - recovery. Define backend-specific revision and retry identity semantics. -3. In TaskStore/manager/scanner/executor and maintenance coordinators, connect - the isolated repo task domain to chunk-kv. Preserve claim fencing, finalize - discovery, conversion, repair, relocation and reservation cleanup. -4. In SegmentOwnerResolver and DiskDB ownership queries, retain the complete - chunk/task proof across both domains, failures, storage transfers and retry. -5. In routing catalogs and migration control, implement the selected backend - conversion and restart protocol. Reuse R103's service handoff and direct-KV - partition migration authority/recovery principles; never - use a service binding flip as proof that records were copied or fenced. -6. In chunk-kv startup/storage assembly, verify typed system WAL and PageIndex - prerequisites, bootstrap order and cold recovery before serving repo data. - Existing legacy identities follow the reviewed R202 compatibility policy. -7. Measure the current direct-KV baseline and complete integrated path under - equivalent S3/Iceberg and business-stream workloads. Include task load, - outage/recovery behavior, memory, latency and all residual Paxos traffic. - Update permanent designs only with the implemented contract. - -#### Dependencies - -- [fixed slot routing](../design/chunkdb/design-crowdb-chunkdb-range-binding.md) owns current direct-KV routing, - type cleanup, typed Wal/PageIndex and separate operation/task domains. This - requirement consumes those boundaries. It must not block R202 completion or - force a premature change to its all-KV-group scope. -- [R103](R103-chunkdb-range-migration.md) owns service handoff and direct-KV - partition data migration for group expansion/shrink. This requirement extends - its authority/recovery principles to a different backend; direct-KV group - migration and direct-KV-to-chunk-kv conversion remain separate operations. -- Existing chunk-kv client/server ordered-range split/transfer are primitives. - [R144](R144-chunk-kv-partition-merge.md) merge remains deferred; do not promise - merge for repo metadata before a reviewed implementation exists. -- Publication semantics and key/split rules are prerequisites to enabling the - new path. Their absence keeps this follow-up disabled; it does not justify - weakened atomicity or an implicit runtime fallback. -- Preserve the implemented [tree engine](../design/tree/design-crowdb-tree-engine.md) - concurrent admission and prefix-safe flush handoff. Backend migration must - retain its acknowledged-write and recovery guarantees. - -#### Acceptance - -- Given current deployment configuration without repo backend enablement, - allocate/update chunks and tasks; assert all remain in direct KV and require - no new repo backend connection (I1). Integration test. -- Given enabled system/repo domains, exercise allocation, query, listing, - mutations and reservations for supported types; assert selected metadata/ - task destinations, independent operation paths and unchanged payload route - (I1, I3). Integration test. -- Given chunk-kv cold startup and system Wal/BtreePage/PageIndex chunks, recover - tree and journal before repo activation; assert no recursion into repo - metadata and consistent page/mapping recovery (I2). Integration test. -- Given repo storage outage and task backlog, run system lookup/maintenance; - assert system admission remains usable and repo operations fail explicitly - without direct-KV fallback or cross-domain claims (I1–I3). Integration test. -- Given chunk/task/reservation publication, interrupt each selected commit or - recovery boundary and lose replies; restart/retry and assert preserved - lifecycle outcomes and discoverable required tasks (I4, I5). Integration test. -- Given related canonical/index keys and existing non-ChunkDB application keys, - route them and attempt legal/illegal splits; assert namespace isolation and - preservation of the selected publication unit (I3, I4). Integration test. -- Given ready and leased tasks in a transferred or split range, resume scanning - and stale workers; assert complete discovery, current claims only and no - duplicate resource release (I3–I5). Integration test. -- Given a DiskDB segment named by a repo repair task, query ownership while - chunk-kv is unavailable and after recovery; assert uncertainty retains the - segment and the exact task target remains protected (I6). Integration test. -- Given existing direct-KV repo chunks and associated state, run the selected - migration with crashes before/after cutover and lost replies; assert one - authoritative backend, preserved acknowledgements and safe cleanup (I1, I5, - I6). Integration test. -- Given concurrent service handoff and a supported storage transfer, issue - stale client/task operations; assert generations are reconciled independently - and service ownership changes do not invent storage authority (I5). - Integration test. -- Given invalid target configuration or an unsupported legacy migration, enable - the backend; assert rejection before authority changes and preservation of - source records (I1, I5). Integration test. -- Given equal-durability representative workloads, compare baseline and - integrated paths; assert the user-approved performance budget and report all - residual Paxos traffic and task/recovery overhead (I7). Integration test. - -#### Open Questions - -- Range-local conditional multi-key commit, or canonical single-key state with - recoverable derived indexes/tasks? Choose based on publication/recovery - complexity and storage-engine support; same-range routing alone is insufficient. -- What exact key namespace and indivisible split unit preserve publication and - due/lease discovery? Prefer a range-local design; compare the cost of aligned - index placement with explicit recoverable index maintenance. -- Should existing repo state move through online fenced migration, a maintenance - window, or an explicitly approved reset for development deployments? Define - rollback only where authority has not been irrevocably transferred. -- Separate processes or separately admitted runtimes in one process, and what - capacity budgets keep system repair available under repo backlog? -- What measured Paxos byte/operation and latency budget constitutes success? - Backend separation and lower metadata cardinality alone do not quantify it. - -The following are implementation verification commands, not tests run by this -documentation-only change. Add reviewed migration/outage cases to the affected -suites when implementing this requirement. - -```sh -pixi run cargo test -p crowdb-chunk-kv --test partition_test --test journal_failure_test -pixi run cargo test -p crowdb-chunk-kv-client --test routing_test --test compose_test -pixi run cargo test -p crowdb-chunk-stream --test production_chunk_test --test stream_test -pixi run cargo test -p crowdb-chunkdb --test routing_test --test lifecycle_test --test owner_metadata_test --test conversion_test -pixi run cargo test -p crowdb-protocol --test chunk_task_key_test --test chunk_task_value_test -pixi run rs-fmt-check -pixi run cargo clippy -p crowdb-protocol -p crowdb-chunk-client -p crowdb-chunk-stream -p crowdb-chunk-kv -p crowdb-chunk-kv-client -p crowdb-chunk-kv-server -p crowdb-chunkdb -p crowdb-chunkdb-client --all-targets -- -D warnings -``` diff --git a/doc/backlog/R233-sdk-local-cache-checkpoint.md b/doc/backlog/R233-sdk-local-cache-checkpoint.md deleted file mode 100644 index 27dd8e85d..000000000 --- a/doc/backlog/R233-sdk-local-cache-checkpoint.md +++ /dev/null @@ -1,91 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R233: SDK — Local disk cache and checkpoint staging - -Status: Deferred until dedicated HTTP upload/read workload measurements. -The public Rust/Python HTTP SDK and real-service functional acceptance are -available; their delivery did not include performance baselines. - -Problem: - -- Training reads may reuse immutable payloads; checkpoint writers may need a - short local-write pause before background upload. The - [Dataset contract](../design/access-server/dataset/spec-crowdb-dataset.md) - defines remote Snapshot publication, not local checkpoint completion. -- Treating a local write as a remotely durable checkpoint would report success - before another machine can recover it. Local staging can absorb bursts but - cannot compensate indefinitely for remote bandwidth below sustained writes. - -Solution: - -1. Add optional Rust cache/staging components under `sdk/`, exposed through the - same Python wrapper as the HTTP client. Rust owns journal recovery, disk - quotas, eviction and background upload; Python exposes configuration and - completion state without duplicating these mechanisms. Keep default HTTP - clients diskless. Key read cache entries by endpoint/security scope, Dataset, - immutable Snapshot, field/range and digest. Validate bytes before admission; - never cache mutable latest as immutable identity. A cache hit does not extend - remote retention or bypass server authorization/lease policy. -2. Separate evictable clean cache from uncommitted staging. Bound disk usage, - memory and upload concurrency. Evict clean entries first; apply explicit - backpressure or fail admission when staging reaches quota. Never silently - evict the only pending copy. Keep hot reads lock-free; any proposed lock - requires the repository's explicit contention/trade-off review. -3. Use a recoverable local journal and durable file/directory synchronization. - Expose distinct Staging, LocalDurable, Uploading, RemoteCommitted and Failed - states. LocalDurable means local bytes and replay information are durable; - it does not survive loss of that disk. RemoteCommitted requires payload - durability and successful Snapshot publication. `flush`/remote wait returns - that Snapshot or an error/ambiguous operation identity, never local success. -4. Resume after process restart using the same upload and publication tokens. - Local corrupt/incomplete files fail validation; partial remote uploads remain - invisible. Retain pending local data after transient failure and report quota, - disk-full and shutdown outcomes. Cleanup follows remote commitment or explicit - discard; remote ownership rules still govern uploaded bytes. -5. Keep model serialization and coordinated multi-rank checkpoint assembly in - adapters. Publish one complete Snapshot only after all declared checkpoint - components exist. Measure pause time, remote completion latency, sustained - throughput, disk/memory bounds and restart recovery against direct streaming. - Do not claim a performance improvement without these workload measurements. - -Dependencies: - -- Existing upload sessions, publication recovery and the public HTTP SDK; - dedicated workload metrics remain a prerequisite. - HTTP streaming remains the fallback without local storage or when caching is - disabled. No RDMA or fat client is required for this feature. -- Native Dataset integration may reuse these components after proving the - same visibility contract. - -Acceptance: - -1. Given identical fields in different snapshots/security scopes -> populate - and corrupt cache entries and advance latest -> assert no cross-scope reuse, - checksum rejection and fixed Snapshot identity. Cache invariant. Integration test. -2. Given a slow server and small disk quota -> stage repeatedly and exhaust the - disk -> assert bounded usage, clean eviction, explicit backpressure and - preservation of pending bytes. Capacity invariant. Integration test. -3. Given a local checkpoint -> stop after each durable state and restart -> - assert LocalDurable never implies RemoteCommitted, journal replay preserves - identity, and remote wait returns only a published Snapshot. Durability invariant. Integration test. -4. Given response loss, partial upload and corrupt local data -> recover/retry - or shut down -> assert no partial publication, duplicate Snapshot or silent - discard and explicit failure outcomes. Recovery invariant. Integration test. -5. Given a declared multi-rank checkpoint and measured baseline -> delay one - component and compare staged/direct runs -> assert complete-only publication - and record pause/completion latency, throughput and resource bounds. - Checkpoint completeness invariant. E2E test. -6. Given the same staged operation -> inspect and resume it through Rust and - the Python wrapper -> assert identical journal identity, durability states, - quota errors and remote completion. Shared implementation invariant. Integration test. - -Verification commands: - -- `pixi run -- cargo test --manifest-path sdk/Cargo.toml --workspace` -- `pixi run -- python -m unittest discover -s sdk -p 'test_*.py'` -- `pixi run clean --runtime-only && pixi run test-e2e-dataset` -- `pixi run -- cargo fmt --manifest-path sdk/Cargo.toml --all -- --check` -- `pixi run -- cargo clippy --manifest-path sdk/Cargo.toml --workspace --all-targets -- -D warnings` -- `pixi run rs-fmt-check` -- `pixi run rs-lint` diff --git a/doc/backlog/R4-bounded-mempool.md b/doc/backlog/R4-bounded-mempool.md deleted file mode 100644 index baca1936b..000000000 --- a/doc/backlog/R4-bounded-mempool.md +++ /dev/null @@ -1,27 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R4: Bounded memory pool for `buffer` allocations - -**Problem**: `buffer::allocate` uses unbounded `std::malloc`. When crowdbtree is -embedded in crowdb-kv-server, a burst of large writes can spike RSS without -backpressure. - -**Approach**: Admission control at `Crowdbtree::apply()`/`apply_batch()` entry -via `Config.mem_budget_bytes` (0 = unlimited). Track outstanding buffer -bytes atomically; reject with `Status::resource_exhausted()` when over -budget. Flush/snapshot path is exempt (must always succeed). - -**Priority**: Low — needed only when memory-bound deployment is a -requirement. - -**Complexity**: Medium — atomic counter, budget accounting in apply path, -test for rejection + recovery. - -**Files**: `lib/crowdb-tree/include/crowdb-tree/config.h`, -`lib/crowdb-tree/include/crowdb-tree/buffer.h`, -`lib/crowdb-tree/src/btree/crowdb-tree.cpp` - -**Acceptance**: Unit test: set `mem_budget_bytes=1MiB`, apply 2 MiB of -values, verify `resource_exhausted` after budget exceeded, verify success -after budget freed (entries applied + flushed). diff --git a/doc/backlog/R60-tree-scan-sibling-leaf-readahead.md b/doc/backlog/R60-tree-scan-sibling-leaf-readahead.md deleted file mode 100644 index 061f5fe7a..000000000 --- a/doc/backlog/R60-tree-scan-sibling-leaf-readahead.md +++ /dev/null @@ -1,137 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R60: Scan — Sibling-Leaf Readahead on Cold Scans - -**Status**: Deferred pending a cold file/block-backed scan benchmark that -forces leaf eviction and records page reads, scan retries, latency, and -throughput. Implement readahead only if that baseline shows serialized leaf -loads are a material part of scan latency or limit NVMe throughput. - -**Problem**: the scan path demand-loads each L1 leaf inline. The sync -path (`Crowdbtree::scan`) resolves a leaf via the page cache when the -merge loop reaches it — the read stalls the loop until the leaf is -resident. The async path (`scan_async_attempt`) resolves one pending -page per reactor round trip and retries the whole scan, so a multi-leaf -cold range pays one reactor round trip per cold leaf, serialized with -the merge work on prior leaves. - -A scan knows its next leaf before finishing the current one: the merge -loop reads `base->right_sibling()` in `crowdb-tree.cpp` right after -descending into a leaf, before iterating that leaf's entries. So -the page id of the next leaf is available while the current leaf is -still being merged. Today nothing is done with it until the current -leaf exhausts and `refill_l1` walks to the next — at which point the -read stalls. - -**Solution**: issue a readahead (prefetch) for the right-sibling leaf -as soon as its page id is known, overlapping the next leaf's I/O with -the current leaf's merge work. - -- **Sync path** (`Crowdbtree::scan` / `try_scan_no_load`): after reading - `right_sibling`, call the page cache's prefetch/async-resolve seam - for that page id (non-blocking — enqueues the read, does not wait). - When `refill_l1` later walks to that leaf, the read is already in - flight or complete, so the stall is hidden. If the leaf is already - resident the prefetch is a no-op. -- **Async path** (`scan_async_attempt`): instead of resolving one pending page - per reactor round trip and waiting until the current leaf is exhausted to - discover the next miss, submit the right-sibling read as soon as the current - resident leaf exposes its page id. This fixed one-leaf lookahead overlaps - the next NVMe read with current-leaf merge and packing work while bounding - memory and unused reads. It does not create multiple in-flight reads within - one scan because the following sibling id is not known yet. -- **Prefetch depth**: start with a fixed depth of one. Tunable 2–4-leaf windows - are out of scope until the cold benchmark shows one-leaf lookahead is useful - but insufficient. -- **Eligibility invariant**: only a range scan may request sibling readahead. - Issue it only when the scan can continue beyond the current leaf, has not - exhausted its item limit or byte budget, has time remaining before its - deadline, and uses a file/block-backed async page store. A resident sibling - needs only the existing CPU prefetch; an unloaded or already-loading sibling - must not receive a duplicate read. Point `get` operations and memory-mode - scans never request sibling I/O. -- **I/O mechanism**: use a request-scoped engine-managed asynchronous read, - not a process-wide operating-system sequential-access hint. The block path - uses `O_DIRECT`, and a global hint could be ineffective or pollute unrelated - buffered-file workloads. At most one speculative sibling page may be loaded - and then left unused when the current leaf satisfies the scan. - -**Scope**: -- `lib/crowdb-tree/src/btree/crowdb-tree.cpp` — `scan` / `try_scan_no_load`: - after `page_id = base->right_sibling()`, issue a prefetch. The - prefetch seam must already exist or be added to the page cache - using the configured `AsyncPageStore` and reactor. If no request-scoped - async-resolve seam exists, this is blocked on adding one. -- `lib/crowdb-tree/src/btree/crowdb-tree.cpp` — `scan_async_attempt`: submit - the right-sibling read while merging the current resident leaf and reuse or - await that read if the scan reaches the sibling before completion. -- Benchmark prerequisite: add a file/block-backed multi-leaf scan case that - forces eviction and records page reads, scan retries, latency, and - throughput before changing the read path. -- Tests: `test-tree-ct` scan tests must pass unchanged (readahead is - observational — output is identical). Extend or mirror - `AsyncScan.MissAfterEvictionCompletesViaReactor` for fixed one-leaf - readahead only after the benchmark meets the implementation threshold. - -**Complexity**: Medium. The prerequisite benchmark is required because the -maintained scan regression runs in memory mode and cannot establish a cold -NVMe benefit. The sync-path prefetch is small IF a -prefetch/async-resolve seam exists in the page cache (likely needs a -small addition — the cache today resolves on demand). The async-path -coordination is the bulk: the retry loop must distinguish a sibling read that -is already in flight from an unloaded page, avoid duplicate submissions, and -resume from the correct key when the read completes. -Measurement is required to confirm the win (cold ranges only — the -bench today runs mem-mode where leaves are resident, so a cold/disk -bench config is a prerequisite for validation). - -**Dependencies**: -- Blocked on the cold file/block-backed benchmark and baseline measurements. -- May need a page-cache prefetch seam (check first). If the cache has - no non-blocking resolve, add one as a sub-task. -- Independent of R57 (staging copies) and R58 (merge loop compares) — - they touch different parts of the scan path. R60 overlaps I/O with - merge work; R57/R58 speed up the merge work itself. Complementary. -- If the existing cold-scan regression measurements expose a regression - without identifying its source, profile the affected case on demand before - investing in readahead. - -**Acceptance**: -- Scan output is byte-identical to today across all `test-tree-ct` scan - tests — Integration test. -- With a cold file/block-backed tree, perform a point `get`, a memory-mode - scan, and a scan whose range ends in the current leaf; assert that none - submits a sibling read, preserving the scan-only eligibility invariant — - Integration test. -- With an unloaded sibling already being loaded, scan through the current - leaf; assert that the scan reuses or awaits the existing operation and does - not submit a duplicate read, preserving the one-read-per-page invariant — - Integration test. -- With an expired scan deadline or an exhausted item/byte budget, finish the - current leaf; assert that no sibling read is submitted, preserving the - request-bound work invariant — Integration test. -- A cold file/block-backed benchmark forces eviction and reports page reads, - scan retries, latency, and throughput for the no-readahead baseline — - Integration test. -- Before the production read path changes, a fixed one-leaf experiment shows - a material reduction in latency or increase in throughput against that - baseline; otherwise R60 is closed without implementation — Integration - test. -- Readahead memory is bounded (per-scan in-flight cap, default window - = 1); a full-keyspace cold scan does not grow unbounded RSS — Integration - test. -- No regression on `tools/benchmark/bench-kv-scan-regression.sh` (mem-mode configs - unchanged — readahead is a no-op when leaves are resident) — Integration - test. - -**Note**: the gap lives in -`doc/design/kv/kv-scan-flow-analysis.md` Gap Analysis → Performance → -"No sibling-leaf readahead on cold scans". - -Verification commands: -- `pixi run test-tree-ct` -- `pixi run tree-fmt` -- `pixi run tree-lint` -- `pixi run rs-fmt -- --check` -- `pixi run rs-lint` diff --git a/doc/backlog/R80-diskdb-rebalance.md b/doc/backlog/R80-diskdb-rebalance.md deleted file mode 100644 index 75dd7f5d4..000000000 --- a/doc/backlog/R80-diskdb-rebalance.md +++ /dev/null @@ -1,127 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R80: diskdb — Disk Space Rebalance Convergence - -**Problem**: DiskDB can now place new allocations on the least-used eligible -disk and can move one committed block through its durable relocation journal. -The current intra-disk-group planner, however, reacts to one utilization -sample, chooses the hottest and coldest disks without projecting the selected -block's effect, and has no end-to-end proof that repeated cycles converge and -then stop. A large block can overshoot the balance point, transient skew can -start unnecessary movement, and passing handoff or restart tests does not show -that a newly added or recovered empty disk actually absorbs old allocations. - -The implemented allocation, relocation, and owner-fencing baseline is defined -by the [DiskDB design](../design/diskdb/design-crowdb-diskdb.md), especially -sections 3.2, 9, and 10. ChunkDB's protection-preserving cross-disk-group -planner is defined by the -[ChunkDB design](../design/chunkdb/design-crowdb-chunkdb.md#74-active-cross-domain-rebalance). -This requirement is limited to completing and proving DiskDB's space-balance -behavior within one disk-group. - -**Solution**: - -1. **Define an improving move**. `RebalancePlannerTask` computes normalized - utilization as `busy_bytes / capacity_bytes` for every allocatable disk. It - selects the hottest source and coldest eligible target deterministically, - but admits a source block only when projecting that exact block onto the - target strictly reduces the disk-group's utilization spread. Require the - target to retain `rebalance.min_target_free_bytes` after reservation. If no - committed block improves the spread, emit no relocation and report the - group as stalled; never oscillate a block across the balance point merely - because the current spread exceeds the threshold. - -2. **Require sustained skew**. `RebalancePlannerTask` tracks when each owned - disk-group first exceeds `rebalance.imbalance_threshold_pct` and starts new - work only after it remains above the threshold for - `rebalance.hysteresis_secs`. A balanced observation clears the timer. This - observation state is an optimization and may reset on restart; - `RelocationJournalValue` remains the durable recovery authority and resumes - before any new decision. - -3. **Converge at a bounded rate**. `RebalancePlannerTask` re-evaluates live - usage after each completed relocation, starts no more than - `rebalance.max_jobs_per_cycle`, and keeps at most one active relocation per - disk-group. `RebalanceZonePacer` keeps disk-groups and zones serially paced - by `rebalance.zone_delay_secs`. The planner stops creating journals once the - spread is below the configured threshold. A source or target that is no - longer allocatable invalidates the candidate without weakening - `RelocationWorker` safety rules. - -4. **Make convergence observable**. `DiskdbMetrics` keeps the existing - imbalance and relocation metrics and distinguishes groups that are observing - hysteresis, actively moving, balanced, or stalled because no safe improving - block fits. Status is derived from `DdbDiskGroup::aggregate_usage`, the - hysteresis observation, and durable journals, not from a process-local job - set. Per-disk-group keepalive summaries remain the cluster-wide source for - placement and operator inspection. - -5. **Prove passive and active balance together**. Preserve the lock-free - `DdbDiskGroup::allocate_block` load-aware policy and verify its behavior over - sequences rather than a single allocation. Add a full-stack fixture with - real DiskIO and the ChunkDB owner path that repeatedly runs the production - planner until a hot disk-group converges or truthfully reports that block - granularity prevents further improvement. - -The relocation invariants remain unchanged: ChunkDB alone publishes layout -changes; the exact target is copied, fsynced, published, and confirmed before -the source is freed; and timeout, age, or utilization never authorizes a free. - -**Dependencies**: - -- Uses DiskDB's existing load-aware allocator, per-disk usage snapshots, - relocation journal, DiskIO copy/fsync path, and tentative-owner scanner. -- Uses ChunkDB's existing exact-source revision fence and durable relocation - task. No new metadata publication path is introduced. -- Cross-disk-group selection is already owned by ChunkDB. This requirement - does not change rack, node, or physical-disk placement policy. -- Dynamic disk membership and status come from the existing group-0 sync. A - disk leaving `Up` is excluded on the next planning decision. - -**Acceptance**: - -- Given equal-capacity disks at 90% and 0%, when 100 single-block allocations - run with load-aware allocation enabled, assert the colder disk receives the - allocations until its utilization catches up; repeat with the policy - disabled and assert cursor-order round-robin, and with equal utilization and - exclusions to assert deterministic ties and strict anti-affinity. Invariant: - passive balancing changes selection only and preserves allocation safety. - Integration test. -- Given unequal-capacity disks, when allocation uses `free_bytes` and then - `inverse_used_pct`, assert each policy follows its documented absolute or - normalized ordering without division-by-zero or overflow. Invariant: mixed - capacity produces deterministic, configured behavior. Unit test. -- Given a disk-group whose spread crosses the threshold for less than the - hysteresis interval, when planner cycles run, assert no relocation journal is - created; after sustained skew, assert exactly one improving journal is - admitted. Invariant: transient skew cannot trigger data movement. - Integration test. -- Given a candidate block whose projected move would preserve or increase the - spread, or leave less than the configured target headroom, when the planner - evaluates it, assert no target is reserved and the group reports stalled. - Invariant: every admitted move strictly improves usable balance. Unit test. -- Given a hot disk and a newly added or recovered empty peer, when production - planner cycles execute through real DiskIO and ChunkDB, assert each - `SourceFreed` move lowers projected spread, restart once with a non-terminal - journal, and eventually observe spread below threshold with no subsequent - journal. Invariant: active rebalance converges, survives restart, and stops. - E2E test. -- Given balanced disks, a single-disk group, all disks non-allocatable, or a - source/target status change between cycles, when the planner runs, assert it - creates no unsafe new journal and retains or safely resumes any existing - journal. Invariant: topology edge cases are no-op or retry, never unsafe - relocation. Integration test. -- Given several owned disk-groups, when some are observing, moving, balanced, - and stalled, assert metrics/status report each state consistently with - current usage and non-terminal journals. Invariant: operators can distinguish - progress from inability to improve. Integration test. - -Verification commands: - -- `pixi run test-diskdb` -- `pixi run test-chunkdb` -- `pixi run test-diskdb-client` -- `pixi run test-chunkdb-client` -- `pixi run rs-fmt -- --check` -- `pixi run rs-lint` diff --git a/doc/backlog/R82-kv-watch-notify-coalescing.md b/doc/backlog/R82-kv-watch-notify-coalescing.md deleted file mode 100644 index 37714c440..000000000 --- a/doc/backlog/R82-kv-watch-notify-coalescing.md +++ /dev/null @@ -1,191 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R82: kv — Watch/Notify Coalescing (Debounce) - -**Problem**: - -- **Current behavior + impact** — the watch/notify extension ships - without coalescing: `PxLearner::apply_entry` calls `WatchRegistry::emit` - directly, producing one notify frame per changed key per matching - prefix. A burst write to a watched prefix (e.g. a diskdb - `batch_write` touching 10 disks under `/hw/disk/`) generates 10 - separate `WatchNotify` frames, each waking the subscriber's - keepalive and triggering a re-read via the normal `Get` path. Under - sustained burst load (periodic discovery scans, bulk status - updates), this amplifies both notify traffic and downstream read - load — the notify mechanism can be strictly worse than the - fixed-interval polling it replaces. Not a correctness issue: the - safety-net poller covers any missed notifies. The impact is purely - load/efficiency under bursty workloads. -- **Design pointers** — - [`doc/design/kv/design-crowdb-kv-watch-notify.md`](../design/kv/design-crowdb-kv-watch-notify.md) - §3 (Coalescing — deferred) states the rationale and points - back to this backlog item. The original coalescer design (struct - shape, timer-task lifecycle, per-prefix debounce, step-down flush) - was specified in the same design draft but removed from the - watch/notify extension's scope after the timer-task wiring proved - incomplete (the spawned timer captured no registry/coalescer refs, so - buffered keys were silently dropped). New work; the reference model - is etcd watch (which coalesces at the mvcc backend, not the apply - path). -- **Use scenarios** — - - **Disk discovery burst** — diskdb runs a periodic discovery - scan; one `batch_write` updates 10+ disk records under - `/hw/disk/`; the owning diskdb's keepalive is woken once (not 10 - times) and performs one re-read sweep. - - **Bulk ownership transfer** — an operator reassigns 5 - disk-groups in one batch; the `/hw/dg_owner/` watcher receives - one coalesced notify with all 5 changed keys, not 5 separate - frames. - - **Continuous churn** — a prefix under continuous write pressure - (writes arriving faster than the debounce window) is - rate-limited to one notify per `debounce_ms` interval, bounding - subscriber wakeups. - -**Solution**: - -- **No clear solution yet — deferred to design.** The core design - questions (timer-task ownership model, Arc/Weak capture, per-prefix - vs per-watcher debounce, step-down flush semantics) need a design - draft to resolve. The high-level shape is known from the original - watch/notify design draft, but the timer-task wiring gap that caused - the extension to drop the feature must be addressed properly. - -- **One-line summary**: per-prefix debounce coalescer with - timer-task flush, wired into the apply path between - `apply_entry` and `WatchRegistry::emit`. - -- **Numbered work items**: - 1. **`WatchCoalescer` struct** - (`lib/crowdb-kv/src/cluster/watch_registry.rs`) — new struct - holding `debounce_ms: AtomicU64` + `pending: - Mutex<HashMap<Vec<u8>, (HashSet<Vec<u8>>, - Option<JoinHandle<()>>)>>`. The apply-path hook calls - `record_chosen` instead of `emit` directly; `record_chosen` - either emits immediately (`debounce_ms == 0`) or buffers keys - per prefix and spawns a timer task. - 2. **Timer task wiring** — the spawned `tokio::time::sleep` task - must capture `Weak<WatchRegistry>` + `Weak<WatchCoalescer>` so - it survives group drop mid-debounce (weak upgrade fails → - no-op). On wake: lock `pending`, drain the prefix's key set, - call `registry.emit`. This is the gap that caused the - coalescer's removal from the initial watch/notify extension — - the original code captured no refs and silently dropped buffered - keys. - 3. **`PxLearner` wiring** (`lib/crowdb-kv/src/paxos/learner.rs`) — - the `watch_registry` `OnceLock` gains a third element - (`Arc<WatchCoalescer>`); `set_watch_registry` takes the - coalescer; the apply-path hook calls `coalescer.record_chosen` - instead of `registry.emit`. - 4. **`PxGroup` wiring** (`lib/crowdb-kv/src/cluster/group.rs`) — - `watch_coalescer: Arc<WatchCoalescer>` field; constructed in - `PxGroup::new` with the configured `debounce_ms`; wired into - the learner via `set_watch_registry`. - 5. **Step-down flush** - (`lib/crowdb-kv/src/cluster/group_election.rs`, - `lib/crowdb-kv/src/cluster/group_propose.rs`) — `step_down` and - the two propose-path direct `become_follower` calls call - `watch_coalescer.flush_and_clear()` before - `watch_registry.clear()`, emitting any pending notifies so no - notify is lost on leader change. - 6. **Configuration** (`lib/crowdb-kv/src/common/config.rs`, - `app/crowdb-diskdb/src/ddb_config.rs`) — add - `watch_notify_debounce_ms` to `CrowDBConfig` (default 100, - dynamic) and `notify_debounce_ms` to `NotifyConfig` (default - 100, documentation/operator intent — the crowdb-kv config is - authoritative). Live reload via `set_from_config` propagates - to the coalescer's `AtomicU64`. - -- **Flow diagram**: - -``` - apply_entry (PxLearner) - │ - ▼ - ┌─────────────────────┐ - │ WatchCoalescer │ - │ debounce_ms == 0? │ - │ ├─ yes → emit now │ - │ └─ no → buffer │ - │ + spawn │ - │ timer │ - └────────┬────────────┘ - │ timer fires (debounce_ms) - ▼ - ┌─────────────────────┐ - │ drain pending[pfx] │ - │ registry.emit(...) │ - └─────────────────────┘ - │ - ▼ - ┌─────────────────────┐ - │ WatchRegistry.emit │ - │ → watcher channels │ - └─────────────────────┘ -``` - -- **Edge cases at a glance**: - - Group dropped mid-debounce → timer task's `Weak` upgrade fails → - no-op (no dangling emit). - - Leader step-down with pending keys → `flush_and_clear` emits - final notifies before registry clears. - - Second write to same prefix within window → adds to existing - `HashSet`, does not reset timer (fixed trailing-edge). - - Write to unwatched key → no pending entry created, no timer - spawned. - - `debounce_ms` changed live → new calls read the new value; - existing timers fire at their original schedule. - -**Dependencies**: - -- Depends on the watch/notify extension (`WatchRegistry`, - `WatchCoalescer` apply-path hook, `WatchNotifyClient` — see - [`doc/design/kv/design-crowdb-kv-watch-notify.md`](../design/kv/design-crowdb-kv-watch-notify.md)). - That extension must be landed first; this item adds the coalescer - layer on top. -- No items depend on R82 — it is a pure load optimization. - -**Acceptance**: - -**WatchCoalescer** (`lib/crowdb-kv/src/cluster/watch_registry.rs`): -- `debounce_ms=0`, `record_chosen` a payload with 2 keys → - immediate emit (no timer). Unit test. -- `debounce_ms=50`, `record_chosen` 3 payloads to the same prefix - within 10 ms → one `WatchNotify` with all keys after ~50 ms (use - `tokio::time::pause`). Unit test. -- `record_chosen` writes to two prefixes → two separate notifies - (one per prefix timer). Unit test. -- `flush_and_clear` with 2 buffered keys → one final notify with - the buffered keys + no timers remain. Unit test. -- `record_chosen` a payload whose key matches no registered prefix - → no pending entry created, no notify. Unit test. -- Group dropped mid-debounce (drop all `Arc`s, advance timer) → no - panic, no emit (weak upgrade fails). Unit test. - -**Step-down flush**: -- Buffer 2 keys, call `flush_and_clear` → assert one notify with - both keys, then `registry.clear()` → channels closed. Unit test. - -**E2E (diskdb)**: -- `notify_debounce_ms=100`, burst-write 10 disks in one - `batch_write` → one coalesced notify wakes keepalive once, not 10 - times. E2E test. -- `notify_debounce_ms=0`, burst-write 10 disks → 10 separate - notifies (proves coalescing is the debounce path, not emit). - E2E test. - -- `pixi run cargo fmt --all -- --check` -- `pixi run cargo clippy --all-targets -- -D warnings` - -**Open Questions**: - -- **Q1 — `debounce_ms` ownership: crowdb-kv config vs per-subscription.** - The coalescer runs on the crowdb-kv leader (group 0). Options: (a) - one global `debounce_ms` on `CrowDBConfig` — simple, but all - watchers share the same window; (b) per-subscription debounce sent - as a `WatchSubscribe` parameter — more ergonomic, adds per-watcher - debounce timer state. Recommendation: (a) for v1 — one global - debounce on the leader; revisit if per-subscription debounce is - needed. Cannot be resolved autonomously — it is an API-ergonomics - trade-off that needs a human decision. diff --git a/doc/backlog/R83-chunkdb-complete-recovery-flow.md b/doc/backlog/R83-chunkdb-complete-recovery-flow.md deleted file mode 100644 index cfff04f5e..000000000 --- a/doc/backlog/R83-chunkdb-complete-recovery-flow.md +++ /dev/null @@ -1,287 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R83: chunkdb — Complete Recovery Flow (Real Data Recovery + Speed Control) - -**Problem**: - -- **Current behavior + impact** — diskdb's recovery story is - disk-layer only and stops at the hand-off boundary. diskdb has zone - bitmap reconstruction (zone-management §6, strategies 1/2/3) and the - per-disk failure recovery scan (R76, `RecoveryScanTask` in - `app/crowdb-diskdb/src/recovery/disk_recovery.rs`). The R76 scan does - the right *discovery* work — it iterates a `Bad` disk's zones, lists - live `BusyBlockValue`s, and collects each impacted block's - `owner_chunk` (`ChunkId`, 192-bit, §3.4) — but the *repair* step is a - placeholder: `RecoveryAction` has exactly one variant `LogOnly`, and - `recover_zone_blocks` logs the impacted blocks + owner chunks and - does no data repair or relocation (disk_recovery.rs:378). The - doc-comment states the intent: "Future versions add - `Relocate`/`RebuildFromEc` when the `diskio` service exists" - (disk_recovery.rs:37). There is **no chunkdb yet** — chunks (logical - blocks composed of mirror or EC strips, each strip a set of diskdb - `Segment`s) are only a reserved fbs schema surface - (`chunkdb.fbs`). So when a disk goes - `Bad`, its impacted blocks are identified and handed to a - "future recovery/relocation path" (§8) that does not exist: the owner - is notified nowhere, no surviving replica/parity is read, no rebuilt - data is written to new segments, and the chunk's strips are never - updated. Full data recovery is missing. Recovery speed is also - uncontrolled — once real rebuild I/O ships, an unthrottled rebuild - would starve foreground allocate/free/read traffic; the throttle - belongs at the chunkdb layer (the component that owns the - chunk→strip→segment mapping and drives rebuild I/O), not at diskdb - (which has no data I/O envelope, §2). -- **Design pointers** — - [`doc/design/diskdb/design-crowdb-diskdb.md`](../design/diskdb/design-crowdb-diskdb.md) - §2 (Non-Goals — no data I/O; "a future diskio-like component does - data I/O"), §3.4 (records are the source of truth; - `BusyBlockValue` carries `owner_chunk` — the recovery hand-off - handle), §8 (disk failure detection + bad-disk handling — - "relocation/rebuild is a follow-up requirement"; the impacted list - is "handed to a future recovery/relocation path: the data-IO layer - rebuilds from EC/mirror, or the owner is notified to re-allocate - elsewhere"), - [`design-crowdb-diskdb-zone-management.md`](../design/diskdb/design-crowdb-diskdb-zone-management.md) - §6 (crash recovery strategies 1/2/3 — diskdb-layer bitmap - reconstruction only). Fbs schema surfaces: - `chunkdb.fbs` (`Chunk`, `ChunkStrip`, `MirrorStrip`, - `EcStrip`, `StripType`, `ECState`, `ChunkState`, - `ChunkdbService` — `AllocateChunk` / - `AppendChunk` / `SealChunk` / `DeleteChunk` / `ReplaceChunkStripRange` / - `ListChunks`), `diskio.fbs` (`DiskWrite` / `DiskRead`). - CROWDB's chunkdb owns disk-failure recovery: it rebuilds lost blocks - from mirror/EC at the chunk layer, driving rebuild I/O and pacing it. -- **Use scenarios** — - - **Mirror replica rebuild after disk failure** — a disk goes `Bad`; - diskdb's R76 scan lists the impacted busy blocks + their - `owner_chunk`. chunkdb (the owner) reads each impacted chunk's - strip layout, finds the surviving mirror replica(s) on healthy - disks, reads the surviving data via `diskio` `DiskRead`, allocates - a new segment on a healthy disk via diskdb `AllocateBlocks`, writes - the rebuilt replica via `diskio` `DiskWrite`, and updates the - chunk's strip (`ReplaceChunkStripRange`) to reference the new `Segment`. - The old segment on the `Bad` disk is freed (`FreeBlocks`) once the - disk is replaced/removed. The chunk is whole again. - - **EC data-block reconstruction** — an EC strip (`data_num` + `code_num`) - loses one data block when its disk goes `Bad`. chunkdb reads the - surviving data + parity blocks via `diskio`, reconstructs the lost - data block, writes it to a newly-allocated segment, and updates - the strip. Parity is recomputed if a parity block was lost. - - **Recovery speed control during foreground load** — a cluster is - serving foreground writes while a disk is `Bad`. The operator sets - a recovery throttle (bandwidth / IOps / concurrent rebuilds). - chunkdb paces the rebuild reads + writes so foreground - allocate/free/read latency is unaffected; recovery progress + - throttle state are visible to the operator. - - **Restart mid-recovery** — chunkdb crashes or restarts while a - rebuild is in progress. On restart it reads persisted per-chunk / - per-strip recovery progress from KV and resumes the rebuild from - the last completed strip, without re-rebuilding completed strips - or losing the in-flight one. - - **Unrecoverable data loss** — a mirror strip loses *all* replicas, - or an EC strip loses more than `code_num` blocks. chunkdb detects - the loss is unrecoverable, reports it (metric + log), and does not - attempt a rebuild that would silently corrupt data. - -**Solution**: - -- **No clear solution yet — deferred to design.** chunkdb does not - exist yet — the chunkdb server component (`ChunkdbService` - implementation) is itself unbuilt, and the recovery flow's shape - depends on chunkdb's architecture (chunk→strip→segment storage, - ownership model, KV schema) plus the `diskio` service (also unbuilt). - The high-level shape is known from R76's placeholder + the reserved - fbs schema surfaces, but the detailed design (rebuild orchestration, - hand-off mechanism, throttle mechanism, progress schema) needs a - design draft once chunkdb's core is defined. - -- **One-line summary**: chunkdb owns real data recovery — rebuild lost - mirror replicas / EC data+parity from surviving strips via `diskio`, - pace rebuild I/O at the chunkdb layer so foreground traffic is not - starved, and persist per-chunk/per-strip progress for restart-resume. - -- **Numbered work items**: - 1. **chunkdb server component (prerequisite)** — - `app/crowdb-chunkdb` (new crate, not yet existing) implementing - `ChunkdbService` (`AllocateChunk` / `AppendChunk` / `SealChunk` / - `DeleteChunk` / `ReplaceChunkStripRange` / `ListChunks`). This is a - prerequisite for R83, not part of it — it should be filed as its - own backlog item. R83 assumes chunkdb exists and owns the - chunk→strip→segment mapping persisted to CROWDB KV. - 2. **Recovery orchestration** (`app/crowdb-chunkdb/src/recovery/`) — - consume diskdb's impacted-blocks + `owner_chunk` hand-off (R76 - `RecoveryScanTask` output); for each impacted chunk, read its - strip layout, classify each impacted strip (`MirrorStrip` vs - `EcStrip`), and build a rebuild plan (which surviving - replicas/blocks to read, how to reconstruct, where to write the - new segment). Replaces R76's `RecoveryAction::LogOnly` with - `Relocate` / `RebuildFromEc` variants. - 3. **`diskio` integration** — read surviving blocks via - `DiskioService::DiskRead` and write rebuilt blocks via - `DiskioService::DiskWrite` to newly-allocated segments (diskdb - `AllocateBlocks` on healthy disks, with `exclude_disks` anti- - affinity against the `Bad` disk). Prerequisite: the `diskio` - server component (also unbuilt; should be its own backlog item). - 4. **Strip update + old-segment free** — after a rebuilt block is - durably written to its new segment, `ReplaceChunkStripRange` the chunk's - strip to reference the new `Segment`, then `FreeBlocks` the old - segment on the `Bad` disk (after the disk is replaced/removed, or - once the strip no longer references it). - 5. **Recovery speed control** (`app/crowdb-chunkdb/src/recovery/throttle.rs`) - — pace rebuild I/O at the chunkdb layer: configurable - `recovery.max_bandwidth_bytes`, `recovery.max_iops`, - `recovery.max_concurrent_rebuilds` (live-reloadable). The - orchestrator gates each rebuild read/write through the throttle so - foreground allocate/free/read traffic is not starved. Emit - recovery-progress + throttle metrics (`recovery.rebuilt_blocks`, - `recovery.pending_chunks`, `recovery.throttle_utilization`). - 6. **Recovery progress persistence** — per-chunk / per-strip - recovery progress written to CROWDB KV (schema follows R76's - `RecoveryScanProgressValue` precedent, keyed by `ChunkId` / - strip index). On chunkdb restart, resume from the last completed - strip; no double-rebuild of completed strips. - -- **Flow diagram**: - -``` - disk goes Bad - │ - ▼ - diskdb RecoveryScanTask (R76) - │ lists impacted BusyBlockValue + owner_chunk per zone - ▼ - chunkdb recovery orchestrator (item 2) - │ for each impacted chunk: - │ read strip layout → classify Mirror / EC - │ build rebuild plan (surviving blocks + reconstruct fn) - ▼ - ┌──────────────────────────────────────────┐ - │ throttle gate (item 5) │ - │ max_bandwidth / max_iops / max_concurrent │ - └──────────────┬───────────────────────────┘ - ▼ - diskio DiskRead (surviving blocks) ──► reconstruct (mirror copy / EC) - │ - ▼ - diskdb AllocateBlocks (healthy disk, exclude Bad) ──► diskio DiskWrite (new segment) - │ - ▼ - ReplaceChunkStripRange (new Segment) ──► FreeBlocks (old segment on Bad disk) - │ - ▼ - persist per-chunk/strip progress (item 6) ──► next strip / next chunk -``` - -- **Edge cases at a glance**: - - Mirror strip loses all replicas → unrecoverable; report data loss, - do not rebuild. - - EC strip loses more than `code_num` blocks → unrecoverable; report - data loss, do not rebuild. - - Rebuild target disk goes `Bad` mid-rebuild → re-pick a healthy - target on the next cycle; the in-flight write to the now-`Bad` - target is abandoned. - - chunkdb restart mid-rebuild → resume from persisted per-strip - progress; no double-rebuild, no lost in-flight strip (the strip is - re-attempted). - - Recovery throttle set very low → recovery takes long; operator - sees progress via metrics; no foreground impact. - - `owner_chunk` missing on a busy block (legacy / corrupt record) → - skip the block, report it (cannot rebuild without the owner). - - EC parity not yet computed (`EC_STATE_NO_PARITY`) on a strip that - loses a data block → cannot reconstruct from parity; rebuild from - mirror fallback or report unrecoverable (design decision). - - Disk moved (R81 Part 2) with segments still on the original paxos - group → recovery must read the moved disk's records via the - per-disk bind, not the disk-group's default bind. - -**Dependencies**: - -- **chunkdb server component (prerequisite, unlanded)** — R83 assumes - the chunkdb server exists (`ChunkdbService` impl, chunk→strip→segment - KV schema). This must be filed as its own backlog item and landed - first; without it there is no chunk owner to drive recovery. -- **`diskio` service (prerequisite, unlanded)** — real data recovery - needs `DiskRead` (surviving blocks) + `DiskWrite` (rebuilt blocks). - The `diskio` server is a future component (§2, `diskio.fbs` - reserved). Must be filed as its own backlog item. Fallback without - diskio: R76's `LogOnly` placeholder stays (current state). -- **R76** — `RecoveryScanTask`, impacted-blocks + `owner_chunk` - collection, `RecoveryAction` placeholder (`LogOnly` → `Relocate` / - `RebuildFromEc`), `RecoveryScanProgressValue` schema as the progress - precedent. -- **R72** — diskdb `AllocateBlocks` / `FreeBlocks`, `Segment` record - model, `BusyBlockValue` / `owner_chunk`. -- **R81 Part 2** — disk move with per-disk bind; recovery must account - for moved disks whose records live on the original paxos group - (per-disk bind routing, not the disk-group's default bind). -- Nothing depends on R83 yet. - -**Acceptance**: - -- **Mirror replica rebuild**: a disk goes `Bad`; a mirror strip with 2 - replicas (one on the `Bad` disk, one on a healthy disk) → chunkdb - reads the surviving replica via `diskio`, allocates a new segment on - a healthy disk (excluding the `Bad` disk), writes the rebuilt - replica, `ReplaceChunkStripRange` updates to the new `Segment`, and frees the old - segment → the chunk reads back correctly from the new replica. E2E - test (pending chunkdb + diskio). -- **EC data-block reconstruction**: an EC strip (`data_num=4`, - `code_num=2`) loses 1 data block → chunkdb reconstructs the lost - block from surviving data + parity, writes it to a new segment, and - updates the strip → the chunk reads back correctly. E2E test. -- **EC parity-block reconstruction**: an EC strip loses 1 parity block - → chunkdb recomputes parity from the surviving data blocks, writes - it to a new segment, updates the strip → `EC_STATE_PARITY` restored. - E2E test. -- **Unrecoverable — mirror all replicas lost**: a mirror strip with 2 - replicas both on `Bad` disks → chunkdb reports data loss (metric + - log), does not attempt a rebuild, no silent corruption. E2E test. -- **Unrecoverable — EC loses more than `code_num`**: an EC strip - (`code_num=2`) loses 3 blocks → chunkdb reports data loss, does not - rebuild. E2E test. -- **Recovery speed control**: `recovery.max_iops` set to N → rebuild - read/write rate stays ≤ N; foreground allocate/free/read latency - stays within the pre-recovery baseline (no starvation). E2E test. -- **Restart resume**: chunkdb restarts mid-rebuild → on restart it - resumes from the persisted per-strip progress; completed strips are - not re-rebuilt, the in-flight strip is re-attempted and completes. - E2E test. -- **`owner_chunk` missing**: a busy block with no `owner_chunk` → - chunkdb skips it and reports it (metric); no crash, no rebuild - attempt. E2E test. -- **Moved-disk recovery**: a disk moved via R81 Part 2 (records on the - original paxos group via per-disk bind) goes `Bad` → chunkdb reads - the impacted blocks via the per-disk bind and rebuilds correctly. - E2E test. -- `pixi run cargo fmt --all -- --check` and - `pixi run cargo clippy --all-targets -- -D warnings` clean. -- `pixi run test-chunkdb` (relevant integration tests pass, pending - chunkdb crate) + `pixi run test-diskdb` (R76 hand-off integration). - -**Open Questions**: - -- **chunkdb + diskio as separate backlog items?** R83 depends on both - the chunkdb server and the diskio server existing. These are - prerequisites that should be filed as their own backlog items - (chunkdb core, diskio core) before R83 can be designed in detail. - Cannot be resolved autonomously — it is a scope/sequencing decision - that needs a human to confirm the prerequisite items should be - filed and in what order. -- **Recovery hand-off mechanism** — how does diskdb's - `RecoveryScanTask` hand the impacted-blocks + `owner_chunk` list to - chunkdb? Options: (a) push — diskdb notifies chunkdb (the owner) via - an owner-notification mechanism (does not exist yet); (b) pull — - chunkdb subscribes to disk `Bad` transitions (watch/notify, R78) and - pulls the impacted list from diskdb. Trade-off: (a) couples diskdb - to chunkdb; (b) keeps diskdb unaware of chunkdb but adds latency. - Design decision. -- **Recovery speed control granularity** — per-chunk, per-disk, or - per-instance throttle? Per-instance is simplest (one global budget); - per-disk allows prioritizing hot disks; per-chunk is the finest. - Recommendation: per-instance for v1, revisit if per-disk priority is - needed. Design decision. -- **EC strip with no parity yet (`EC_STATE_NO_PARITY`) loses a data - block** — cannot reconstruct from parity. Rebuild from a mirror - fallback (if the chunk has one), or report unrecoverable? Design - decision. diff --git a/doc/backlog/R84-chunkdb-post-disk-move-placement-scanner.md b/doc/backlog/R84-chunkdb-post-disk-move-placement-scanner.md deleted file mode 100644 index cc559582d..000000000 --- a/doc/backlog/R84-chunkdb-post-disk-move-placement-scanner.md +++ /dev/null @@ -1,246 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R84: chunkdb — Post-Disk-Move Chunk Placement Scanner - -**Problem**: - -- **Current behavior + impact** — R81 Part 2 adds disk move with a - stable `DiskId` (UUID): a physical disk is relocated from - `(old_rack, old_node, old_dg)` to `(new_rack, new_node, new_dg)`, - its zone/busy/free records are copied from the old disk-group's bind - to the new disk-group's bind during Maintenance (no concurrent - writes), and the disk becomes available at the new placement without - a full recovery scan. The move is placement-only — the disk's block - data is intact on the physical medium, and the records are keyed - only by `DiskId` (globally unique, §3.9), so copying them between - paxos groups is a literal key-value copy. What is missing is any - verification that chunk placement is still consistent after a move. - Chunks reference the blocks they own via `Segment { disk_id, - zone_index, unit_offset, unit_count }` (one `Segment` per allocated - range, embedded in `MirrorStrip` / `EcStrip` inside each - `ChunkStrip`, `chunkdb.fbs`). After a disk move, every chunk - that has a `Segment` on the moved disk must still be able to reach - that segment: the `disk_id` must resolve to a disk that is `Up` at - its new placement. There is no scanner that walks - chunk→strip→segment placement after a move and verifies - reachability + health. A move that leaves a chunk's segments on a - disk that is now unreachable (e.g. the move partially failed, the - disk went `Missing` at the new placement) would silently break chunk - reads with no detection until a client hits the broken segment. This - is the chunk-placement analog of diskdb's background scanner (§10) — - but diskdb's scanner checks disk-layer record/bitmap drift, not - chunk→segment placement integrity, which is chunkdb's domain. -- **Design pointers** — - [`doc/backlog/R81-sysdata-id-reuse-safety-and-disk-move.md`](R81-sysdata-id-reuse-safety-and-disk-move.md) - Part 2 (disk move, record copy during Maintenance, records keyed - by `DiskId`, no full scan), - [`doc/design/diskdb/design-crowdb-diskdb.md`](../design/diskdb/design-crowdb-diskdb.md) - §3.9 (unit-based sizes; disk-id key routing — record keys carry - `DiskId`, no `node_id`/`disk_group_id`), §8 (disk status management - — effective status = `max(node, group, disk)`; `Missing` detection - by absence from a group-0 sync response), §10 (Background Scanner — - `ScannerTask` / `BgRunner` / `BackgroundTask` pattern, KV-persisted - progress, resume after restart, admin RPCs `TriggerScan` / - `GetScanStatus`). Fbs schema surfaces: `chunkdb.fbs` (`Chunk`, - `ChunkStrip`, `MirrorStrip`, `EcStrip`, `Segment` reference), - `chunkdb.fbs` (`ListChunks` — paginated chunk scan). - CROWDB's disk-move model (R81) is new, so the post-move chunk - placement scanner is new work shaped on diskdb's `ScannerTask` - precedent. -- **Use scenarios** — - - **Post-move placement verification** — an operator moves a disk - from disk-group A to disk-group B (R81 Part 2). The move completes - and the disk is `Up` at the new placement. A scanner walks every - chunk that has a `Segment` on the moved `DiskId`, resolves each - segment's disk to its current placement (group-0 `DiskValue`), - and verifies the disk is `Up`. All segments reachable → the move - is placement-clean; - the scanner reports 0 inconsistencies. - - **Detect partially-failed move** — a disk move left the disk - `Missing` at the new placement (the new disk-group's instance has - not picked it up via keepalive reconcile). The scanner finds - chunks whose segments reference the moved `DiskId`, sees the disk - is not `Up` at the new placement, and reports the unreachable - segments + their owning chunks so the operator can fix the move - or trigger recovery (R83). - - **Detect orphaned segment after disk removal (not move)** — a - segment references a `DiskId` that no longer exists in group 0 - (the disk was removed, not moved). The scanner reports the - orphaned segment + owning chunk as a data-loss risk; the chunk - needs recovery (R83) if it has surviving replicas/parity. - - **Periodic placement integrity sweep** — independent of any - specific move, a periodic sweep verifies that every chunk's - segments resolve to a live, `Up` disk, catching placement drift - from any cause (move, removal, status change). - -**Solution**: - -- **No clear solution yet — deferred to design.** chunkdb does not - exist yet (see R83 prerequisites), so the scanner's host (chunkdb-side - vs diskdb-side), the chunk iteration mechanism, and the trigger - model need a design draft once chunkdb's core is defined. The - high-level shape is known from diskdb's `ScannerTask` precedent (§10) - + the disk-move model (R81). - -- **One-line summary**: a placement-integrity scanner that, after a - disk move (and periodically), walks chunk→strip→segment placement, - resolves each segment's `DiskId` to its current group-0 placement, - and reports any segment that is unreachable or orphaned. - -- **Numbered work items**: - 1. **Placement scanner task** (`app/crowdb-chunkdb/src/scanner/placement.rs`, - new — or `app/crowdb-diskdb/src/scanner/` if the design picks - diskdb as the host) — a background task following the - `BgRunner` + `BackgroundTask` pattern (R75 / §10 `ScannerTask`). - Runs on a configurable interval (`scanner.placement_interval_secs`, - default 600) and is triggerable on-demand after a disk move - (admin RPC `TriggerPlacementScan`). - 2. **Chunk→strip→segment walk** — iterate chunks via - `ChunkdbService::ListChunks` (paginated, `ChunkId` order); for - each chunk, iterate its `ChunkStrip`s and the `Segment`s inside - each `MirrorStrip` / `EcStrip`. Collect the set of `DiskId`s - referenced by live (non-deleted) chunks. - 3. **Segment reachability check** — for each referenced `DiskId`, - resolve its current placement from group 0 (`HardwareClient` - `DiskValue` lookup). Verify: the disk exists in group 0 (not - orphaned); the disk's effective status is `Up` (not - `Missing`/`Bad`/`Offline`). A segment whose disk is - `Bad`/`Missing` is reported + handed to recovery (R83). - 4. **Placement integrity report + metrics** — emit a - `PlacementScanSummary` (counts: `reachable_segments`, - `unreachable_segments`, `orphaned_segments`) + gauges - (`scanner.placement.unreachable`, - `scanner.placement.orphaned`). Admin RPCs `TriggerPlacementScan` - / `GetPlacementScanStatus` mirror diskdb's scanner admin surface. - 5. **Move trigger** — the scanner is triggered on disk-move - completion via watch/notify (R78, `/hw/disk/` prefix) when a - `DiskValue` moves path, with periodic sweep as a safety net - (same pattern as diskdb's watch/notify + polling safety net, §8). - 6. **Progress persistence + resume** — KV-persisted scan progress - (last scanned `ChunkId`, following the existing - `RecoveryScanProgressValue` checkpoint pattern); on restart the scanner - resumes from the last checkpoint, no full re-walk. - -- **Flow diagram**: - -``` - disk move completes (R81 Part 2) ──or── periodic interval - │ - ▼ - PlacementScannerTask (item 1) - │ - ▼ - ListChunks (paginated by ChunkId) ──► for each chunk: - │ iterate ChunkStrips → MirrorStrip / EcStrip → Segments - │ collect referenced DiskId set - ▼ - for each referenced DiskId (item 3): - ├─ group-0 DiskValue lookup (HardwareClient) - │ ├─ disk missing in group 0 → orphaned segment (report) - │ ├─ effective status != Up → unreachable (report + hand to R83) - │ └─ status Up → segment OK - ▼ - PlacementScanSummary + gauges (item 4) - │ - ▼ - persist progress (item 6) ──► next ChunkId page / done -``` - -- **Edge cases at a glance**: - - Segment on a moved disk, disk `Up` at new placement → reachable; - scanner reports 0 issues for that segment. - - Segment on a moved disk, disk `Missing` at new placement (move - partially failed) → unreachable; report + hand to recovery (R83). - - Segment on a disk removed from group 0 (not moved) → orphaned; - report data-loss risk + hand to recovery (R83). - - Segment on a `Bad` disk → unreachable; hand to R76/R83 recovery - (the placement scanner does not rebuild, it detects + routes). - - chunkdb restart mid-scan → resume from persisted - `last_scanned_chunk_id`; no full re-walk. - - Chunk deleted mid-scan (`ChunkState::Deleted`) → skip its segments - (freed); no false orphan. - - No chunks reference the moved disk → scanner reports 0 segments - scanned for that disk; no overhead beyond the chunk walk. - - Disk moved multiple times → only the final placement matters; the - scanner resolves the current `DiskValue` each run. - -**Dependencies**: - -- **R81 Part 2** — disk move with record copy during Maintenance; - the scanner verifies placement integrity after a move. Without R81 - there is no disk move to scan. -- **chunkdb server component (prerequisite, unlanded)** — the scanner - walks chunks via `ListChunks`; chunkdb must exist (same prerequisite - as R83). Must be filed as its own backlog item. -- **R75** — `BgRunner` + `BackgroundTask` pattern; the placement - scanner follows `ScannerTask`'s structure (interval trigger, - on-demand `TriggerScan`, `ScanState`, in-progress flag). -- **R78** — watch/notify on `/hw/disk/` for the move trigger (optional; - polling fallback if watch/notify is not landed). -- **R83** — recovery flow; the placement scanner detects unreachable - / orphaned segments and hands them to R83 for rebuild. R83 must - exist for the hand-off to do real work; without R83 the scanner - reports only (same placeholder shape as R76's `LogOnly`). -- **R72** — `Segment` record model; the scanner reads `Segment`s from - chunk strips. -- Nothing depends on R84 yet. - -**Acceptance**: - -- **Post-move reachability**: move a disk via R81 Part 2 to a new - disk-group, disk `Up` at the new placement → the scanner walks all - chunks with segments on the moved `DiskId`, verifies each segment - is reachable, reports 0 unreachable / 0 orphaned. E2E test (pending - chunkdb). -- **Partially-failed move detection**: move a disk but leave it - `Missing` at the new placement → the scanner reports the chunks + - segments referencing the moved `DiskId` as unreachable. E2E test. -- **Orphaned segment after removal**: a chunk's segment references a - `DiskId` removed from group 0 (not moved) → the scanner reports the - segment as orphaned with its owning chunk. E2E test. -- **`Bad`-disk segment routing**: a chunk's segment references a disk - that is `Bad` → the scanner reports the segment as unreachable and - hands it to recovery (R83); the scanner does not rebuild. E2E test - (pending R83). -- **Deleted chunk skipped**: a chunk transitions to - `ChunkState::Deleted` mid-scan → its segments are not reported as - orphaned (they are freed). E2E test. -- **Restart resume**: chunkdb restarts mid-scan → on restart the - scanner resumes from the persisted `last_scanned_chunk_id`; no full - re-walk, no missed chunks. E2E test. -- **On-demand trigger**: `TriggerPlacementScan` after a move → the - scan runs immediately (not waiting for the interval) and returns the - current summary + `scan_in_progress` flag. E2E test. -- **No chunks reference the moved disk**: a disk is moved but no chunk - has a segment on it → the scanner completes with 0 segments scanned - for that disk; no false positives. E2E test. -- `pixi run cargo fmt --all -- --check` and - `pixi run cargo clippy --all-targets -- -D warnings` clean. -- `pixi run test-chunkdb` (relevant integration tests pass, pending - chunkdb crate). - -**Open Questions**: - -- **Scanner host: chunkdb-side or diskdb-side?** The scanner walks - chunk→strip→segment placement (chunkdb's data model) but checks disk - reachability (diskdb's / group-0's domain). Options: - (a) chunkdb-side — chunkdb owns the chunk→segment mapping, so it is - the natural host; it calls `HardwareClient` to resolve disk - placement. (b) diskdb-side — diskdb has the scanner infrastructure - (`ScannerTask`) and the disk records, but it does not know which - chunks reference which disks (that is chunkdb's domain), so it would - need a reverse lookup chunkdb does not expose. Recommendation: - chunkdb-side (a) — the chunk→segment walk is the core of the scan - and only chunkdb has it. Design decision. -- **Trigger model: watch/notify vs periodic-only?** Watch/notify - (R78) on `/hw/disk/` gives immediate post-move triggering; periodic- - only is simpler but adds detection latency. Recommendation: - watch/notify trigger + periodic safety net (same as diskdb's model, - §8). Depends on R78 being landed. Design decision. -- **Should the placement scanner be bundled with diskdb's existing - background scanner (§10) or kept separate?** Bundling reuses one - `ScannerTask` loop + admin surface; keeping it separate respects - the chunkdb/diskdb component boundary. Recommendation: separate - (chunkdb-owned), mirroring diskdb's scanner structure. Design - decision. diff --git a/doc/backlog/R92-chunkdb-in-chunk-gc.md b/doc/backlog/R92-chunkdb-in-chunk-gc.md deleted file mode 100644 index 08b377e4b..000000000 --- a/doc/backlog/R92-chunkdb-in-chunk-gc.md +++ /dev/null @@ -1,17 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R92: chunkdb — In-Chunk GC Operations - -**Problem**: Shared chunks accumulate unused space as objects are deleted. -Without in-chunk GC, this space is never reclaimed, leading to storage -waste. CROWDB needs localized GC operations confined to individual chunks -to avoid global merge overhead. - -**Solution**: Implement in-chunk GC operations (ReclaimStrip, CollapseStrip, -MergeStrips) for shared chunks. Add logical-to-physical offset mapping -to support GC while keeping chunk IDs stable. R95 owns the chunk-centered -orphan scan and qualified range deletion; this requirement provides the -in-chunk reclamation operations after R95 proves a range unreachable. - -**Scope**: Placeholder - detailed design to be refined before implementation. diff --git a/doc/backlog/R95-chunkdb-chunk-range-delete.md b/doc/backlog/R95-chunkdb-chunk-range-delete.md deleted file mode 100644 index d3994c0f5..000000000 --- a/doc/backlog/R95-chunkdb-chunk-range-delete.md +++ /dev/null @@ -1,201 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R95: chunkdb — Qualified chunk range deletion and orphan scan - -## Problem - -Shared chunks contain ranges owned by different objects or multipart parts. -Deleting a whole chunk for one unreachable range can erase live neighbors. -Uploads may also write a chunk and fail before their part or final object -reference is recorded. A cleanup queue populated by each MPU mutation would -add metadata writes to the upload path and still miss those pre-record crashes. -The [ChunkDB design](../design/chunkdb/design-crowdb-chunkdb.md) separates -logical range retirement from physical strip release. A successful notification -does not supply independent orphan discovery or publication fencing. - -Current user-data frames contain length and write time but no final resource -identity. Dataset upload expiry discovers ranges through upload/part records; -Iceberg discovers pre-publication ranges through block intents. Removing these -records before an independent discovery path exists would leak writes that -crash before publishing their result. A second failure window is publication -admission before expiry followed by an indefinitely delayed final write after -GC has reclaimed its physical range. A local clock check and a long grace -period do not fence that delayed write. - -## Solution - -1. Complete `DeleteChunkRange(chunk_id, offset, size)` in the chunkdb protocol, - client and server. Access submits one notification without a preliminary - QueryChunk or whole-chunk deletion. An optional observed chunk revision - adds a strict layout precondition; otherwise ChunkDB obtains and atomically - conditions on its current layout. Only sealed user-data chunks accept exact - byte ranges; a dedicated chunk additionally requires its complete sealed - object extent. Stale explicit layouts and active writers are rejected or - deferred. Persist merged dead intervals and the sealed physical - identity as resource metadata, atomically fenced by the chunk revision. - This background resource state is approved; per-upload process records are - not. Successful retirement completes the caller's notification responsibility; - Access must not wait for physical reclamation or poll the returned physical - status. ChunkDB owns subsequent release and restart recovery independently. - Never round a dead frame outward to the usage bitmap's coarser granularity. -2. Use a bounded, chunk-centered scanner to find old chunks and ranges whose - bytes have no live owner. Do not create per-MPU cleanup intents, candidate - records or a durable cleanup queue. Start with a configurable age threshold - of one day; a candidate must be older than the threshold after its last - write. Age alone never authorizes deletion. Enumerate complete, CRC-validated - frames only within a sealed chunk's acknowledged extent. A malformed frame, - incomplete boundary or uncertain authority defers its range; never scan - past corruption by guessing the next object boundary. - New frame formats must carry enough resource identity to resolve the exact - final publication authority and its fixed eligibility deadline. Encode this - in the existing data write, without a separate foreground intent/index RPC. - A digest of a variable key is insufficient unless it directly addresses the - final authority. Header extensions change per-frame payload capacity: the - location format and native reader/writer arithmetic must select the layout - unambiguously. Continue reading old frames and retain their existing minimum - discovery facts until their ranges can be independently qualified. -3. Compare each candidate against published S3 objects, Iceberg file - descriptors, committed Dataset/Split manifests and still-eligible completed - parts. Unpublished parts have an explicit commit deadline. Final publication - must be fenced against expiry reclamation, including a request admitted - before expiry whose publication arrives later. A lost part-publication - response or missing intermediate MPU record is not proof that a chunk is - unused. Refuse deletion when references or the publication fence cannot be confirmed. - Dataset's immutable part authority permits a direct boundary: an expired - missing part and its delayed final creation compete on the same part key. - GC installs a small permanent retirement value before retiring the range; - the delayed create-if-absent must then fail. Do not delete this value after - the range notification, which would reopen the key to an arbitrarily late - creation. If a final part exists with different exact locations, a losing - physical write cannot replace that immutable result. Existing owners and - complete final references protect the matching locations beyond the temporary - deadline. This eliminates process progress, not final resource facts. -4. Keep S3 MPU session, current-part and immutable part-generation keys under - a common upload prefix in Chunk-KV so the scanner can enumerate one upload's - references with a bounded prefix scan. The S3 multipart authority owns that - key layout and Abort/expiry semantics. Only completed data resources and - published references are required by this contract; it does not authorize - durable upload workflows. Old part generations remain available until the - scanner proves their bytes are unreachable. An aborted or expired upload becomes a candidate only after - the age gate and reference checks. -5. Recheck chunk identity, layout generation and exact range against current - authority immediately before reclaim. Treat lost delete replies - idempotently and keep an in-memory scan cursor and bounded work budget. - Restart may rescan old chunks. Report examined, - eligible, retired, deferred and physically reclaimed bytes without per-upload - metric labels. Snapshot deletion denies new requests and waits its persisted - grace period; readers are not registered or renewed. Layout retirement uses - its bounded validity interval; overdue readers may fail explicitly. -6. Establish overwrite publication authority before removing S3/Iceberg - discovery records. S3 ordinary objects are directly overwritten at their - tenant/bucket/object key and have no globally addressed immutable file - authority. Current Iceberg files publish at a canonical table path and file - name; the catalog/file-ID lookup is a legacy path, not the authority for - every new file. Neither can safely use a permanent tombstone at a mutable - name alone: a legitimate replacement must remain possible, while an old - generation must never return. A separate generation index alone also leaves - a cross-key race between checking eligibility and publishing the final name. - Select an authoritative conditional mutation or an immutable payload - generation plus an atomic reference boundary before implementing this part. - Specify the additional resource fields and foreground RPC cost for review; - do not add a request ledger, journal or response receipt by default. -7. Keep complete-result expiry discoverable. Dataset Snapshot candidate and - Split revision roots need fixed creation/eligibility facts in their existing - resource writes; retries must not extend their original deadline. Complete - unreferenced results remain readable by ID until retirement. Discover and - fence both partial and complete expired resources, preserving latest/stable, - retention, live children and inherited/shared data. An owner acquired before - the final root completes must not pin an abandoned payload forever. These - authority facts are a dependency of orphan qualification, not a reason to - restore editing/evaluation workflows. - -## Dependencies - -- R92 supplies in-chunk strip reclamation after R95 qualifies dead ranges. -- The S3 multipart authority supplies grouped MPU keys and completed part - and final object references. The scanner also recognizes Iceberg's frozen - MPU selection. -- Published references, part deadlines and deletion/layout grace must be - verifiable before physical deletion is enabled; R168 may use the qualified range-delete - interface for ordinary S3 object deletion. -- Access retains the existing minimum physical-range discovery facts until - the frame scanner and final publication fences are verified. Access cleanup - does not implement background physical scanning or wait for physical free. -- The overwrite authority choice below and complete-resource expiry must land - before removing their only orphan-discovery path. Without them, defer unsafe - ranges and retain the existing facts; do not report the scanner as complete. - -## Acceptance - -- Given a shared chunk with live and unreachable ranges, when the scanner - evaluates a candidate older than one day, assert only the exact unreachable - range is passed to `DeleteChunkRange`. Invariant: a live neighbor is never - reclaimed. E2E test. -- Given an MPU part that was replaced, aborted or expired, when its grouped KV - prefix is scanned, assert old locations remain protected by any active or - frozen selection and become eligible only after the age and reference - checks. Invariant: no extra MPU cleanup record is required. Integration test. -- Given a write that crashed before part metadata publication, when the chunk - scan runs, assert it defers the chunk until the age gate and in-flight writer - proof settle, then discovers the orphan without a part record. Invariant: - missing upload metadata alone cannot reclaim data. Integration test. -- Given an unexpired deletion/layout grace, changed layout generation or uncertain authority, - when range deletion is attempted, assert no bytes are reused. After the - grace expires and identity is confirmed, repeat the same delete through a - lost reply and assert one idempotent result. Invariant: reclamation is fenced - by current chunk and publication authority, without reader sessions. Integration test. -- Given a large backlog and foreground load, when scanning runs, assert it - respects item, byte and time budgets and reports deferred/reclaimed progress. - Invariant: cleanup cannot monopolize the data path. Integration test. - -- Given adjacent, overlapping and unaligned byte ranges, when the server records - their retirement and restarts, assert exact union, unchanged live gaps and - idempotent replay. A new stale-layout range fails without mutation; an already - retired range can be queried again. Invariant: monotonic exact retirement. - Integration test. -- Given retirement success with blocks still allocated, when the client receives - that response, assert Accepted without further polling. Invariant: notification - completion is independent of physical capacity recovery. Unit test. -- Given an expired Dataset part whose final creation is paused in transit, when - GC seals the same resource key and notifies range retirement, then resume the - write and retry it after worker restart, assert both creations fail and the - permanent resource fence remains. Invariant: retirement cannot resurrect. - Integration test. -- Given a complete part shared by live Snapshot owners and an abandoned root - that acquired an owner before completing, when expiry scans run after restart, - assert only the abandoned owner's reference is retired, matching live bytes - survive, and a complete resource's original deadline is unchanged by retry. - Invariant: final reference facts determine liveness. Integration test. -- Given an old S3 PUT or Iceberg file publication paused after its eligibility - check, when its generation expires and a new generation is published, resume - the old mutation and assert it cannot replace the new value or reference the - retired bytes. Repeat with a lost mutation reply. Invariant: overwrite - publication and generation retirement share an authoritative boundary. - Integration test. -- Given old frames and new frames with extended resource headers, when native - and HTTP readers request first, interior and final frame ranges before and - after restart, assert exact logical bytes without collecting a copy or using - the old fixed-header offset arithmetic for the new layout. Corrupt or - incomplete resource headers defer reclamation. Invariant: format-compatible - physical discovery preserves readable data. Integration test. - -## Open Questions - -- Which authoritative overwrite boundary should S3 and current Iceberg file - publication use? An in-authority conditional mutation must fence the exact - retired generation without an extra eligibility-read race. An immutable - payload-generation resource adds at least one resource write per generation - and still requires an atomic final reference boundary. Describe the concrete - failure flow and RPC cost before selecting either; an independent index or a - local expiry check is insufficient. - -Required gates: - -- `pixi run clean-env && pixi run cargo test -p crowdb-chunkdb --all-targets` -- `pixi run cargo test -p crowdb-chunk-client --all-targets` -- `pixi run cargo test -p crowdb-access-s3 --all-targets` -- `pixi run cargo test -p crowdb-access-dataset --all-targets` -- `pixi run cargo test -p crowdb-access-iceberg --all-targets` -- `pixi run cargo fmt --all -- --check` -- `pixi run rs-lint` diff --git a/doc/backlog/R96-chunkdb-console-cli-integration.md b/doc/backlog/R96-chunkdb-console-cli-integration.md deleted file mode 100644 index 0c6d2a3a9..000000000 --- a/doc/backlog/R96-chunkdb-console-cli-integration.md +++ /dev/null @@ -1,12 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -### R96: chunkdb — Console/CLI Integration - -**Problem**: chunkdb has no management interface for operators. There is no way to view chunk status, query chunk distribution, or manage chunk lifecycle via web UI or CLI. - -**Solution**: Add chunkdb panel to crowdb-web console with chunk overview, distribution visualization, and query capabilities. Implement crowdb-cli chunkdb subcommands for chunk management operations. Integrate with group-0 for topology display. - -**Scope**: The complete Web UI, including Chunk listing and Strip placement, -is specified by [Console UI specification](../design/console/design-crowdb-console-ui.md). This item retains CLI -integration and shared operation reuse; do not implement a second Web panel. diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md deleted file mode 100644 index 26c83c222..000000000 --- a/doc/backlog/backlog.md +++ /dev/null @@ -1,95 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# CROWDB Backlog Index - -This index contains only requirements that are not complete. Completed -requirements and their temporary execution plans are removed during the final -`implement-requirement` cleanup. Requirement detail and acceptance criteria -live in the linked document; implementation follows -[`implement-requirement`](../../.agents/skills/implement-requirement/SKILL.md). - -**Next R number: R245** - -## Dataset and access - -- **[R233](R233-sdk-local-cache-checkpoint.md)** — optional local disk read cache - and recoverable checkpoint staging with explicit remote durability. - **Area:** SDK. **Complexity:** High. **Status:** Deferred until dedicated HTTP workload measurements. - -## Access server and external data - -- **[R168](R168-s3-shared-object-reclamation.md)** — shared small-object - reclamation. **Area:** access server / S3 / chunkdb. **Complexity:** High. - **Status:** Deferred until R95 and the basic S3 delete path are stable. -- **[R169](R169-s3-shared-chunk-tree-gc.md)** — shared-chunk and B+tree garbage - collection. **Area:** access server / S3 / crowdb-tree / chunkdb. - **Complexity:** High. **Status:** Deferred until R147, R168, and workload - measurements are complete. -- **[R185](R185-access-iceberg-cache-invalidation.md)** — bounded Iceberg cache - and invalidation. **Area:** access server / Iceberg / Group 0 / Chunk-KV. - **Complexity:** Medium. **Status:** Deferred pending focused cache measurements. -- **[R186](R186-access-iceberg-orc-validation.md)** — selected ORC validation. - **Area:** access server / Iceberg. **Complexity:** Medium. **Status:** - Deferred as an independent follow-up by user decision. -- **[R196](R196-refine-benchmark-tests.md)** — refine benchmark tests: external - CLI workloads and shared collection against a three-node container cluster; - record results in each driver. **Area:** CLI / storage / access / benchmarks. - **Complexity:** Medium. **Dependencies:** existing OCI and service contracts. - -## Chunk, Chunk-KV, and recovery - -- **[R83](R83-chunkdb-complete-recovery-flow.md)** — complete data recovery and - recovery-speed control. **Area:** chunkdb / diskdb / DiskIO. - **Complexity:** High. **Dependencies:** chunkdb server and DiskIO recovery path. -- **[R84](R84-chunkdb-post-disk-move-placement-scanner.md)** — post-move chunk - placement scanner. **Area:** chunkdb / diskdb. **Complexity:** Medium. - **Dependencies:** R81 Part 2 and the chunkdb server. -- **[R92](R92-chunkdb-in-chunk-gc.md)** — in-chunk garbage-collection operations. - **Area:** chunkdb / chunk IO. **Complexity:** High. **Dependencies:** chunk - lifecycle and qualified range ownership. -- **[R95](R95-chunkdb-chunk-range-delete.md)** — qualified chunk-range deletion - and orphan scanning. **Area:** chunkdb / chunk IO / S3. - **Complexity:** High. **Dependencies:** chunk-range ownership records. -- **[R96](R96-chunkdb-console-cli-integration.md)** — chunkdb CLI management - integration. **Area:** chunkdb / console / CLI. **Complexity:** Medium. - **Dependencies:** stable chunkdb management surface. -- **[R147](R147-tree-chunk-gc.md)** — reclaim B+tree chunk strips. **Area:** - crowdb-tree / chunkdb / diskdb. **Complexity:** High. **Status:** Deferred - until immutable page packs and durable reclaim candidates are available. -- **[R148](R148-chunk-stream-scale-out.md)** — stream metadata scale-out and - sealed-chunk EC. **Area:** chunk-stream / chunk-kv / KV / chunkdb. - **Complexity:** High. **Status:** Deferred until the measured single-group - mirror path is complete. -- **[R193](R193-chunkdb-node-failure-budget.md)** — configurable node-failure - budget and EC placement. **Area:** KV / chunkdb / chunk IO / deployment. - **Complexity:** High. **Status:** Planned. -- **[R207](R207-chunkdb-repo-metadata-chunk-kv.md)** — repo metadata and tasks on - chunk-kv. **Area:** chunkdb / chunk-kv. **Complexity:** High. - **Status:** Deferred beyond the direct-KV stage. - -## KV, DiskDB, and topology - -- **[R80](R80-diskdb-rebalance.md)** — disk space rebalance convergence. - **Area:** diskdb / DiskIO. **Complexity:** Medium. -- **[R82](R82-kv-watch-notify-coalescing.md)** — watch/notify coalescing. - **Area:** KV / diskdb. **Complexity:** Medium. -- **[R102](R102-diskdb-dynamic-binding-migration.md)** — dynamic disk-group - binding migration. **Area:** diskdb / KV. **Complexity:** High. -- **[R103](R103-chunkdb-range-migration.md)** — dynamic slot ownership and - KV-group expansion/shrink. **Area:** chunkdb / KV. **Complexity:** High. - **Status:** Deferred by user decision until dynamic ownership is requested. -- **[R139](R139-group0-service-config.md)** — single-container and cluster - performance profiles, test tuning, and Group-0 distributed service configuration. **Area:** configuration / control plane. **Complexity:** Medium. - **Status:** Deferred until the file-backed service configuration contract. -- **[R144](R144-chunk-kv-partition-merge.md)** — adjacent partition merge. - **Area:** chunk-kv / KV / group 0. **Complexity:** High. **Status:** Deferred - until manifest reuse, transfer fencing, and transition recovery are stable. - -## Tree and memory - -- **[R4](R4-bounded-mempool.md)** — bounded memory pool for tree allocations. - **Area:** crowdb-tree engine. **Complexity:** Medium. -- **[R60](R60-tree-scan-sibling-leaf-readahead.md)** — sibling-leaf readahead on - cold scans. **Area:** crowdb-tree / scan / DiskIO. **Complexity:** Medium. - **Status:** Deferred pending cold file/block-backed measurements. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md b/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md index dc32b404e..f474897da 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md @@ -4,7 +4,7 @@ # CROWDB - Design: ChunkDB Slot Routing Depends on: [ChunkDB architecture](design-crowdb-chunkdb.md), -[KV group 0](../kv/design-crowdb-kv-group0.md). +[KV group 0](../kv/design-crowdb-paxos-kv-group0.md). Satisfies: ChunkDB stateless execution and explicit nonzero persistence. ChunkDB uses independent service and storage maps over the same fixed hash @@ -64,9 +64,21 @@ endpoints are registered separately and can refresh after same-instance restart. ## 3. Publication and startup -`ChunkSlotMapClient` validates the full bootstrap and publishes each map's -head and owner records in one head-conditional group-0 batch. A retry reads and -reconciles the committed result; a conflicting initializer fails. Both maps +`ChunkSlotMapClient` validates the full bootstrap. Up to 63 owner records and +their head publish in one head-conditional group-0 batch. Larger maps first +CAS the absent head to an initialization intent containing the complete layout, +canonically ordered by owner key, and a durable progress offset. Each page +writes at most 63 owners and advances that intent under its current head +revision. The final page replaces the intent with the regular head; readers +reject an intent and therefore never expose a partial map. Every proposal +fits the 64-record replay window. + +A replacement initializer can resume the identical intent, including with a +different input ordering. Conflicting layouts fail without writing owners. +An uncertain page result is reconciled only when the head proves durable +progress; otherwise the error remains visible. Stale pages cannot write after +publication because the head revision has advanced. A retry reads and +reconciles the committed result. Both maps must exist before ChunkDB accepts allocations, even though the maps have independent publication boundaries. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb.md b/doc/design/chunkdb/design-crowdb-chunkdb.md index f8d04e00c..2e7a4997d 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb.md @@ -103,9 +103,9 @@ and decides when a completed write group requires a durability barrier. - **Flexible EC** — support multiple EC schemes simultaneously, selected per-strip based on cluster size and fault tolerance requirements. -**Design philosophy:** "chunkdb is a thin, stateless client of crowdb-kv and +**Design philosophy:** "chunkdb is a thin, stateless client of crowdb-paxos-kv and diskdb." All consensus, replication, and durability are delegated to -crowdb-kv; block allocation is delegated to diskdb. chunkdb's job is chunk +crowdb-paxos-kv; block allocation is delegated to diskdb. chunkdb's job is chunk lifecycle management, placement policy, and EC coordination, nothing more. ## 2. Non-Goals (Design Envelope) @@ -113,8 +113,8 @@ lifecycle management, placement policy, and EC coordination, nothing more. - **No local block engine.** chunkdb does not own devices or implement block reads and writes. Maintenance handlers route I/O through DiskIO services. - **No local WAL.** CROWDB KV's WAL is the sole durability mechanism. -- **No consensus code.** chunkdb is a client of crowdb-kv; all interaction - uses the existing crowdb-kv API. +- **No consensus code.** chunkdb is a client of crowdb-paxos-kv; all interaction + uses the existing crowdb-paxos-kv API. - **No GC operations in v1.** Garbage collection (reclaim, collapse, merge) is deferred to a future requirement. Chunks are allocated and deleted as whole units in v1. @@ -175,11 +175,11 @@ periodic refresh provides a good balance between freshness and performance. ### 3.4a Watch/notify for real-time updates -chunkdb uses crowdb-kv's watch/notify mechanism for real-time topology updates +chunkdb uses crowdb-paxos-kv's watch/notify mechanism for real-time topology updates in addition to periodic refresh: - **Watch registration**: chunkdb registers for group-0 watch on disk-group - and node status keys via the crowdb-kv watch/notify client. + and node status keys via the crowdb-paxos-kv watch/notify client. - **Immediate updates**: A notification coalesces into a complete group-0 topology rebuild. The immutable snapshot is published with one `ArcSwap` operation, so allocation never observes a mixture of generations. @@ -298,7 +298,7 @@ either mirror or EC strips based on configuration and access patterns. │ │ │ ▼ ▼ ▼ ┌─────────┐ ┌─────────┐ ┌─────────┐ - │ crowdb-kv │ │ diskdb │ │ group-0 │ + │ crowdb-paxos-kv │ │ diskdb │ │ group-0 │ │ (KV) │ │ │ │Hardware │ └─────────┘ └─────────┘ └─────────┘ ``` @@ -494,7 +494,7 @@ allocations account for projected utilization without a placement lock. default 30s) fetches full topology from group-0 via `HardwareClient` for consistency and missed notification recovery. - **Watch/notify integration**: Register for group-0 watch on disk-group and - node status keys via crowdb-kv watch/notify client. Notifications coalesce + node status keys via crowdb-paxos-kv watch/notify client. Notifications coalesce into a complete refresh; no point mutation is published. - Fetch all nodes with status and rack assignment - Fetch all disk-groups with capacity and node assignment diff --git a/doc/design/chunkds/design-crowdb-chunk-kv.md b/doc/design/chunkds/design-crowdb-chunk-kv.md index e9090788e..f1d7d8daf 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv.md @@ -6,7 +6,7 @@ `crowdb-chunk-kv` is the embeddable range-partitioned state machine for the shared-storage metadata index. It composes crowdb-tree's runtime-selected chunk page store with a private chunk-stream WAL. It does not depend on the -Paxos-facing `crowdb-kv::KVEngine`, and partition count is independent of node +Paxos-facing `crowdb-paxos-kv::KVEngine`, and partition count is independent of node count. ## 1. Identity and Ownership @@ -152,43 +152,75 @@ Split preparation follows these ordered steps: 1. Enter `SplitPreparing` and range-rebuild the child base from one exact pinned parent tree view. The parent remains the only mutation sequencer and - continues its normal WAL and memtable path while the child tree, WAL, and - live memtable are prepared. -2. Prepare the child storage and establish an ordered parent frontier `C`. - Fixing this frontier does not stop parent journal appends or reads. Persist - a handoff status in the split transition, binding the parent and child - identities, epochs, pinned base and WAL replay boundaries. Successful durable - status update is the local handoff commit point. Resolve an uncertain result - by reading that exact transition; an in-memory `SplitCutover` message is not - durable evidence. Do not await this update in the mutation worker or add a - split-owned request waiting queue. -3. After the handoff status is committed, install in-memory key dispatch to - the child. Until dispatch changes, writes continue to the original parent - WAL, including writes after `C`; afterward, left-range writes continue to - that same WAL and right-range writes enter the child WAL. The handoff proof - and replay rules must cover the parent suffix between fixing `C` and changing - dispatch exactly once. `C` alone cannot be treated as the final inherited - parent frontier if parent writes continued beyond it. The child view must - include those acknowledged writes for reads and conditional mutations. - Seed its bounded request result cache and recover any acknowledged child-WAL - tail without reevaluating conditions. A mutation racing dispatch must have - exactly one writer and preserve its logical request identity. -4. The route change does not seal, publish, checkpoint, or wait for a - memtable. The child immediately reads its base, its live memtable, and the - right-range inherited parent view, including the covered handoff interval. - Conditional mutations consult that same merged view. -5. Complete inherited-view publication in background, preserving the original - parent tree and its normal checkpoint path. The child materializes only its - range; newer child revisions win over older inherited entries. Retain the - shared view and replay dependencies until the complete inherited frontier - and durable storage proofs are recorded in one immutable `SplitArtifact`. - Handoff status is not final readiness and does not replace this artifact. -6. Publish one catalog generation that shrinks the retained parent (same ID, + continues its normal WAL and memtable path while the child tree and WAL are + prepared. +2. Establish an ordered parent frontier `C` and persist a handoff status in + the split transition, binding the parent and child identities, epochs, + pinned base and WAL replay boundaries. Successful durable status update is + the local handoff commit point. Resolve an uncertain result by reading that + exact transition. Parent appends and reads continue while this control-plane + operation awaits persistence; the mutation worker does not wait for it. + Do not add a split-owned request waiting queue. An in-memory cutover message + is not durable handoff evidence. +3. After durable handoff, register a bounded observer of parent mutations. + Background registration joins only writers admitted before the observer; + those writers retain their registration guard through frontier completion. + Capture `A = max_seen` afterward, including pre-registration out-of-order + writes that may exceed the contiguous frontier. Subsequent writers record + shared cell references in an independent delta, so parent source rotation + and flush cannot remove the observed suffix. +4. Replay the child through a fixed contiguous prefix `P >= A`. In background, + prepare its complete L1 view through `P`, replacement sources and publication + storage, then acquire the child's generation lease. The lease closes only + the private child's ingress. Parent mutations continue into the observer. + This prepared view does not replace the pinned durable base used for crash + recovery. +5. At the ordered mutation-worker boundary, select the applied frontier `F` + and its exact synchronized WAL frame end `endF`. The durable append frontier + must equal `F`; `endF` advances only after each frame's apply or no-op succeeds. + Freeze the observer and install the prepaid child prefix through `F`, then + publish local key dispatch. This short commit performs no parent source seal, + replay, checkpoint or backend I/O and does not wait for CPU work. A stale, + busy or expired preparation fails before installation. Once installed, the + token is idempotent only for that exact `F`. +6. Complete readiness in background. Publish the parent journal metadata and + require its recoverable end to cover `endF`, then publish the child's + inherited view. A later checkpoint may cover additional parent writes but + cannot change the fixed `F` or `endF`. Retain the exact artifact and shared + replay dependencies across publication failures until the complete prefix + and durable storage proofs form one immutable `SplitArtifact`. Handoff + status alone is not final readiness. +7. Publish one catalog generation that shrinks the retained parent (same ID, new epoch) and inserts exactly one new child. The parent owner already has both writers active, so refresh only publishes this catalog and serving grant to external routing; it does not gate local handoff, recover the retained parent, or replay a cutover. +The observer and replacement controls are prepaid on the child's account with +finite record and byte limits. Parent and child share one host so cell payloads +retain their existing allocation identity; new reference metadata is charged before +insertion. Exhaustion expires the split attempt without rejecting the parent +mutation or growing the delta further. The child reads its prepared prefix and +frozen suffix with range filtering and its own live memtable; newer child +revisions win over inherited entries. Conditional mutations consult that same +merged view. Writes after dispatch use the child WAL for the child range and +the unchanged parent WAL for the retained range. The child inherits bounded +request outcomes so retries do not reevaluate conditions, and a mutation racing +dispatch keeps exactly one writer and its original request identity. + +The token owns both Trees and the old child-generation controls through commit. +Explicit child close may revoke a prepared lease; if close wins, installation +fails before changing the child. If installation wins, close joins that short +commit. Returning unused credit and destroying old controls can run retirement +or budget callbacks, so they belong to host cleanup, not the route boundary. +Before enqueueing cutover, a started host cleanup task holds the final token +owner. The worker releases its reference before completing the request; a +cancelled, unqueued request releases that reference before closing completion. +Cleanup therefore remains off the mutation worker even if the control caller +is cancelled. Cancellation after dispatch retains the installed route and exact +artifact: retry finalizes that same prefix instead of reinstalling over an +acknowledged child WAL. + The child base checkpoint records two independent counters: its logical tree snapshot sequence and its chunk root-catalog generation. Immediately after the child base is durable, preparation persists @@ -202,13 +234,21 @@ split and retains its base, WAL and pin even if final readiness is absent. A prepared or restarted child recovers in three layers: open the exact base manifest, replay the retained parent suffix through the final inherited frontier with range filtering, then replay its own WAL from the recorded child start. -The proof must include parent writes accepted while handoff status was being -persisted; the earlier fixed `C` is not automatically that final frontier. +Before final readiness exists, the first child WAL sequence determines +`F = first_child_sequence - 1`. If no child frame exists, recovery can extend +inheritance through the recovered parent tail because there is no acknowledged +child suffix to overwrite. The recovered parent writer first scans its +synchronized tail and publishes the required prefix before an independent +metadata-based reader is used. The earlier `C` alone cannot establish final +parent coverage, and a stale read-only metadata cursor is not evidence of data +loss. + Records at or below the base sequence restore retained request outcomes without -reapplying tree mutations. Failed -conditions remain no-ops with their original result. Sequence gaps, a wrong -source stream or epoch, a mismatched range, or a child WAL beginning anywhere -other than the proof's exact child start reject recovery. +reapplying tree mutations. Failed conditions remain no-ops with their original +result. Sequence gaps, a wrong source stream or epoch, a mismatched range, or a +child WAL beginning anywhere other than the proof's exact child start reject +recovery. Parent writes acknowledged between `C` and dispatch remain covered; +left-range writes after `F` cannot extend an already installed child's artifact. The child recognizes two minimum-position namespaces. A retained parent-stream position covered by the inherited suffix is satisfied by the inherited overlay @@ -236,25 +276,28 @@ The split invariants are: owner, and lower range boundary and creates exactly one new child; - **SPLIT-ONE-WRITER:** before in-memory dispatch changes the parent sequences the full range; afterward parent and child sequence disjoint ranges; -- **SPLIT-ROUTE-FRONTIER:** the fixed preparation frontier, durable handoff - commit and in-memory dispatch are distinct boundaries. Recovery includes all - parent writes through the final inherited frontier and the complete child - tail; no acknowledged mutation falls between them or executes twice; +- **SPLIT-ROUTE-FRONTIER:** durable handoff `C`, observer high-watermark `A`, + prepared prefix `P` and dispatch frontier `F` are distinct boundaries. The + child's complete `P` view and observed `(P, F]` suffix cover every parent + mutation through `F`; recovery also includes the complete child WAL. No + acknowledged mutation falls between them or executes twice; - **SPLIT-HANDOFF-DURABLE:** persist handoff before child write admission; restart after commit resumes the same split even before catalog publication; - **SPLIT-OVERLAY-DURABLE:** base plus parent suffix plus child WAL reconstructs values and request outcomes before activation; - **SPLIT-NO-FOREGROUND-WAIT:** routing does not wait for memtable seal, parent-history replay, filtered publication, checkpoint, catalog publication, - or ownership materialization; and + ownership materialization, budget callbacks or old-owner destruction; and - **SPLIT-PIN-BEFORE-RECLAIM:** physical deletion never precedes the last catalog, snapshot, forwarding, or retry reference. -Implementation verification remains required for the durable handoff status and -the exact replay/sequence representation covering continued parent appends -between fixing `C` and installing dispatch. It must not be resolved by pausing -parent writes, queueing requests behind status persistence, or retaining a -second destination tree/WAL for the parent. +The no-wait boundary covers route installation and token cleanup ownership. +The background parent Stream checkpoint can still serialize later appends to +that same Stream; the independently routed child WAL does not wait for that +publication. This distinction does not permit pausing parent writes during +observer preparation or durable handoff persistence. In-memory routing success +is not a substitute for the Stream and storage backends' durability and fencing +proofs, which readiness and crash recovery must validate separately. ## 6. Remote Balance Storage Contract diff --git a/doc/design/chunkds/design-crowdb-chunk-stream.md b/doc/design/chunkds/design-crowdb-chunk-stream.md index 786b92104..d4880209d 100644 --- a/doc/design/chunkds/design-crowdb-chunk-stream.md +++ b/doc/design/chunkds/design-crowdb-chunk-stream.md @@ -13,7 +13,7 @@ Depends on: [chunk IO](../chunkio/design-crowdb-chunkio.md), [chunk reader](../chunkio/design-crowdb-chunkio-reader.md), [DiskIO](../diskio/design-crowdb-diskio.md), [chunkdb](../chunkdb/design-crowdb-chunkdb.md), and -[group-0 sysdata](../kv/design-crowdb-kv-group0.md). +[group-0 sysdata](../kv/design-crowdb-paxos-kv-group0.md). ## Contents diff --git a/doc/design/config/design-crowdb-config.md b/doc/design/config/design-crowdb-config.md index 1f58d9296..aa53a842e 100644 --- a/doc/design/config/design-crowdb-config.md +++ b/doc/design/config/design-crowdb-config.md @@ -22,7 +22,7 @@ validated, and activated. ## 1. Scope -The `crowdb-kv-server`, `crowdb-diskdb`, `crowdb-chunkdb`, `crowdb-diskio`, +The `crowdb-paxos-kv-server`, `crowdb-diskdb`, `crowdb-chunkdb`, `crowdb-diskio`, and `crowdb-access-server` processes accept typed TOML startup configuration. A service may require a file or make it optional, but a supplied file follows the same resolution and failure rules in every process. @@ -153,7 +153,7 @@ backend. Manual node startup runs only monitor, SSH and Web until an operator prepares nodes and initializes Group 0; then shared authority enables management. Both startup policies use the same image and services. Discovery UUID, SSH keys, host keys, binding and operation recovery records reside on the persistent root. -`CROWDB_KV_MANAGEMENT_ADVERTISE_ADDR` specifies the reachable management address +`CROWDB_PAXOS_KV_MANAGEMENT_ADVERTISE_ADDR` specifies the reachable management address for service registration independently of the local listening address. See the [deployment architecture](../deploy/design-crowdb-deploy.md) for ownership and recovery ordering. diff --git a/doc/design/console/design-crowdb-console-ui.md b/doc/design/console/design-crowdb-console-ui.md index 243492726..970e168be 100644 --- a/doc/design/console/design-crowdb-console-ui.md +++ b/doc/design/console/design-crowdb-console-ui.md @@ -4,7 +4,7 @@ # CROWDB - Design: Console UI Specification Depends on: [Console architecture](design-crowdb-console.md), -[KV architecture](../kv/design-crowdb-kv.md). +[KV architecture](../kv/design-crowdb-paxos-kv.md). Satisfies: [Console architecture](design-crowdb-console.md). This is the authoritative UI design and acceptance specification. Each numbered diff --git a/doc/design/console/design-crowdb-console.md b/doc/design/console/design-crowdb-console.md index 67a027abc..88abadbdc 100644 --- a/doc/design/console/design-crowdb-console.md +++ b/doc/design/console/design-crowdb-console.md @@ -4,7 +4,7 @@ # CROWDB - Design: Console (Overview) The `crowdb-console` component provides a Web UI and a CLI that share one -Rust core for managing `crowdb-kv-server` clusters. This overview covers +Rust core for managing `crowdb-paxos-kv-server` clusters. This overview covers the backend, data model, CLI, and hosting concerns; the frontend SPA design is detailed in the sub-design `design-crowdb-console-ui.md`. @@ -46,11 +46,11 @@ design is detailed in the sub-design `design-crowdb-console-ui.md`. ### Goals - Single workspace project `crowdb-console` delivering a Web UI and a CLI that share one Rust core. -- Operate against any number of `crowdb-kv-server` instances via their public surfaces (HTTP management API + crowdb-rpc KV / health). +- Operate against any number of `crowdb-paxos-kv-server` instances via their public surfaces (HTTP management API + crowdb-rpc KV / health). - Model a **Rack → Node → Server Instance → Store → Group → Replica** hierarchy, including a "simulated hardware" mode that runs entirely on `127.0.0.1`. ### Non-Goals -- Bypassing `crowdb-kv-server` to talk to Paxos / WAL / storage internals. +- Bypassing `crowdb-paxos-kv-server` to talk to Paxos / WAL / storage internals. - Multi-tenancy and a general audit-log service. - Making a console-local file authoritative for cluster topology. @@ -88,8 +88,8 @@ Targets: The web frontend (SPA backed by Axum) and the CLI follow the same path through the shared `ops` module: -- **Web**: `user → crowdb-web (Axum) → shared (ops module) → group-0 sysdata + crowdb-kv-server` -- **CLI**: `user → crowdb-cli → shared (ops module) → group-0 sysdata + crowdb-kv-server mgmt` +- **Web**: `user → crowdb-web (Axum) → shared (ops module) → group-0 sysdata + crowdb-paxos-kv-server` +- **CLI**: `user → crowdb-cli → shared (ops module) → group-0 sysdata + crowdb-paxos-kv-server mgmt` Both frontends build an `OpContext` with Group 0 discovery seeds. Attached CLI commands select `--cluster` or `--management-endpoint`, verify the confirmed @@ -102,7 +102,7 @@ configured auxiliary Web process may retain a local launch registry; it is not required for node formation or endpoint-only attachment. The CLI talks directly to group-0 system metadata via -`CrowdbSysmdClient` and to individual `crowdb-kv-server` management +`CrowdbSysmdClient` and to individual `crowdb-paxos-kv-server` management APIs via `ServerClient` — no `crowdb-web` intermediary. The `ops` module in `shared` holds the operation logic that both frontends call. @@ -126,7 +126,7 @@ call. │ │ │ ▼ ▼ ▼ ┌────────────────────────────────────┐ - │ crowdb-kv-server │ (one per node) + │ crowdb-paxos-kv-server │ (one per node) └────────────────────────────────────┘ ``` @@ -136,11 +136,11 @@ call. `ops::kv_server::deploy`, `ops::kv_data::put`, `ops::hardware::add_rack`). - `web` (Axum) and `cli` (clap) only parse input and render output. - The web SPA **does not** reimplement business logic; it calls - `shared` via the Axum backend, never `crowdb-kv-server` directly. + `shared` via the Axum backend, never `crowdb-paxos-kv-server` directly. - Both frontends build an `OpContext` and call `shared`'s `ops` module directly — the CLI from its verified application target, the web backend via `AppState::op_context()` (sharing the - cached `CrowdbKvClient` and Group 0 management seeds). + cached `PaxosKvClient` and Group 0 management seeds). - Shared hardware and logical operations provide the same authority and conditional publication rules to both frontends. Node count and runtime do not select UI permissions; required authority and resource ownership still apply. @@ -158,7 +158,7 @@ direction from which the cluster is observed. Rooted at **Rack → Node → Server → PxStore → PxGroup → {LocalReplica, RemoteReplica…}**. Every entity below `Node` is described from that node's vantage point. A `PxGroup` has exactly one local replica plus -N−1 remote-replica proxies. This mirrors the `crowdb-kv-server` internal +N−1 remote-replica proxies. This mirrors the `crowdb-paxos-kv-server` internal data structure, which is why this view is also the "debugging view": the API surfaces the remote-list explicitly so an operator can spot bugs where a node failed to register all of its peers. @@ -204,7 +204,7 @@ Identity is `(store_id[, group_id[, replica_id]])`. logical view collapses replicas into a unified list so cluster-level operations can ignore placement. The physical view keeps the split for debugging missing peer registrations. -- `StoreView` / `GroupView` / `ReplicaView` reuse `crowdb_kv::cluster::info` +- `StoreView` / `GroupView` / `ReplicaView` reuse `crowdb_paxos_kv::cluster::info` where possible; the console-side wrapper adds the `node_id` projection that the per-server protocol does not encode. @@ -303,7 +303,7 @@ namespaces. ### 5.1 Two transports per node | Purpose | Transport | | --- | --- | -| Deploy / start / stop `crowdb-kv-server` process; copy binary | SSH | +| Deploy / start / stop `crowdb-paxos-kv-server` process; copy binary | SSH | | Runtime mgmt API (add store/group, list, health) | HTTP | | Runtime KV ops, paxos health | crowdb-rpc | @@ -477,9 +477,9 @@ backend-facing contract here: - `tracing` everywhere; `--vv` switches CLI to debug. - Web backend exposes `/healthz`. **`/metrics` is deferred**. The Rust Prometheus story has multiple competing crates; we will pick one when - broader observability work for `crowdb-kv-server` begins. + broader observability work for `crowdb-paxos-kv-server` begins. - All console-issued operations attach a correlation id propagated as - `x-crowdb-kv-corr-id` to `crowdb-kv-server` request headers. + `x-crowdb-paxos-kv-corr-id` to `crowdb-paxos-kv-server` request headers. - Regression service metrics have a content contract: KV emits `rust`, `cpp-rpc`, `cpp-tree`, and `misc`; DiskDB and ChunkDB emit `rust`, `cpp-rpc`, and `misc`; DiskIO emits `cpp-rpc` and `misc`. A tracing or RPC diff --git a/doc/design/deploy/design-crowdb-deploy.md b/doc/design/deploy/design-crowdb-deploy.md index 18038b3c5..8a93f0ace 100644 --- a/doc/design/deploy/design-crowdb-deploy.md +++ b/doc/design/deploy/design-crowdb-deploy.md @@ -10,9 +10,9 @@ containerd/nerdctl. The [OCI image design](design-crowdb-oci-image.md) defines construction, runtime resources and publication. The [console architecture](../console/design-crowdb-console.md) defines the -shared operation layer; the [Group-0 architecture](../kv/design-crowdb-kv-group0.md) +shared operation layer; the [Group-0 architecture](../kv/design-crowdb-paxos-kv-group0.md) defines system metadata; the -[reconfiguration design](../kv/design-crowdb-kv-reconfiguration.md) governs +[reconfiguration design](../kv/design-crowdb-paxos-kv-reconfiguration.md) governs Paxos voting membership. This design connects those responsibilities without making node discovery a membership protocol. diff --git a/doc/design/diskdb/design-crowdb-diskdb-space-metrics.md b/doc/design/diskdb/design-crowdb-diskdb-space-metrics.md index 13d7469ff..69a045eeb 100644 --- a/doc/design/diskdb/design-crowdb-diskdb-space-metrics.md +++ b/doc/design/diskdb/design-crowdb-diskdb-space-metrics.md @@ -5,7 +5,7 @@ Depends on: [`design-crowdb-diskdb.md`](design-crowdb-diskdb.md) §3.4 (records are source of truth), §7 (record model + recovery strategies), §3.7 (reuse -`crowdb-common` metrics), §8 (allocation algorithm); [`design-crowdb-kv-group0.md`](../kv/design-crowdb-kv-group0.md) +`crowdb-common` metrics), §8 (allocation algorithm); [`design-crowdb-paxos-kv-group0.md`](../kv/design-crowdb-paxos-kv-group0.md) (hardware hierarchy + service registry). Satisfies: `design-crowdb-diskdb.md` §11 (Space Metrics). @@ -588,7 +588,7 @@ summaries joined with the hardware hierarchy. This is the stale (≤1 sync interval) cluster-wide view; live per-disk/per-zone drill-down is via the diskdb-client (§11). -`SpaceUsageClient` in `lib/crowdb-kv-client/src/space_usage.rs` wraps +`SpaceUsageClient` in `lib/crowdb-paxos-kv-client/src/space_usage.rs` wraps `HardwareClient` (which already has the hierarchy scans + `scan_prefix`). Re-exported from `lib.rs`. @@ -626,7 +626,7 @@ impl SpaceUsageClient { hierarchy subtree. `ClusterUsage`/`RackUsage`/`NodeUsage` structs hold capacity/used/free -bytes + disk-group/disk counts. Mirror `crowdb-kv-client`'s retry pattern +bytes + disk-group/disk counts. Mirror `crowdb-paxos-kv-client`'s retry pattern (the underlying `HardwareClient` already retries). Edge cases: @@ -639,7 +639,7 @@ Edge cases: `crowdb-diskdb-client` is the primary client surface for all diskdb rpc operations (allocate/free/query with retry + endpoint caching), -mirroring `crowdb-kv-client`'s pattern. Without it, callers must use raw +mirroring `crowdb-paxos-kv-client`'s pattern. Without it, callers must use raw rpc stubs and do endpoint discovery manually. `DiskdbClient` in `lib/crowdb-diskdb-client/src/client.rs`: @@ -703,7 +703,7 @@ impl DiskdbClient { - **`recalc_disk_usage`**: wraps the admin RPC. - **Error model**: `DiskdbClientError` with `Rpc(crowdb_rpc::Status)` / `NoSpace` / `NotFound` / `InvalidArgument` mappings. `RetryConfig` - mirrors `crowdb-kv-client` (max retries, backoff). + mirrors `crowdb-paxos-kv-client` (max retries, backoff). Edge cases: - Cache miss (unknown `dg_id`) → lazy refresh + retry; if still not @@ -721,7 +721,7 @@ Edge cases: New rpc-level e2e tests in `lib/crowdb-diskdb-client/tests/` using `DiskdbClient` against an in-process diskdb rpc server + `KvCluster` -(mirroring `crowdb-kv-client/tests/` pattern): +(mirroring `crowdb-paxos-kv-client/tests/` pattern): - `allocate_free_e2e.rs` — allocate → verify busy record → free → verify free. - `query_e2e.rs` — allocate → @@ -780,7 +780,7 @@ lib/crowdb-protocol/src/fbs/ diskdb_type.fbs # ZoneUsage, extend DiskInfo/DiskGroupInfo diskdb_op.fbs # extend QueryCapacityStatsRequest, RecalcDiskUsage* diskdb_service.fbs # RecalcDiskUsage RPC -lib/crowdb-kv-client/src/ +lib/crowdb-paxos-kv-client/src/ space_usage.rs # SpaceUsageClient + ClusterUsage/RackUsage/NodeUsage lib.rs # re-export lib/crowdb-diskdb-client/src/ diff --git a/doc/design/diskdb/design-crowdb-diskdb-zone-management.md b/doc/design/diskdb/design-crowdb-diskdb-zone-management.md index b920af0fe..f85bf7bb1 100644 --- a/doc/design/diskdb/design-crowdb-diskdb-zone-management.md +++ b/doc/design/diskdb/design-crowdb-diskdb-zone-management.md @@ -8,7 +8,7 @@ exclusive ownership), §3.9 (unit-based sizes, disk-id key routing), §6 (hierarchy); [`design-crowdb-protocol-key.md`](../protocol/design-crowdb-protocol-key.md) (binary key encoding); -[`design-crowdb-kv-state-machine.md`](../kv/design-crowdb-kv-state-machine.md) +[`design-crowdb-paxos-kv-state-machine.md`](../kv/design-crowdb-paxos-kv-state-machine.md) (slot tracking, apply semantics). Satisfies: `design-crowdb-diskdb.md` §7 (Zone Records and Crash Recovery), §8 (Allocation Algorithm), §9 zone allocation state, §14 zone-level @@ -212,7 +212,7 @@ three complementary strategies (§6). **Note on scan ordering:** a key prefix scan returns records in **lexicographic key order** (= `unit_offset` order), not slot order. -`min_slot` on the ordinary crowdb-kv scan request is a read-freshness floor, +`min_slot` on the ordinary crowdb-paxos-kv scan request is a read-freshness floor, not a record-slot filter. Compaction uses the R132 fixed-cutoff bounded scan, which returns each current record's `commit_slot`. Strategy 2 uses `JournalScan` for slot-ordered replay. @@ -289,7 +289,7 @@ rolled back together on allocation or persistence failure. 1. **Phase 1 (sync)**: bitmap-scan allocate (nanoseconds). Bits are set in `usage_bits` via per-bit CAS. No zone-level lock. Other threads can allocate concurrently. -2. **Phase 2 (async)**: `.await` on crowdb-kv `batch_write` that Puts +2. **Phase 2 (async)**: `.await` on crowdb-paxos-kv `batch_write` that Puts the `BusyBlockValue` with a new `allocation_ts` at `BusyBlockKey { disk_id, zone_idx, unit_offset }`. On success, return a `Segment` carrying the same incarnation. On failure: @@ -440,7 +440,7 @@ atomic `batch_write`. 5. **One atomic `batch_write`**: Put the new `ZoneValue`, delete busy records for matched frees, and delete every processed free fact at or below the cutoff. This is a single - `batch_write` on the bound data group, atomic via crowdb-kv paxos. + `batch_write` on the bound data group, atomic via crowdb-paxos-kv paxos. The snapshot and the free-record deletion succeed or fail together; there is no window where the snapshot is written but the free records survive (the race that caused double-free in the two-op @@ -510,8 +510,8 @@ Load the `ZoneValue` snapshot, then replay only the operations (Put / Delete) written after `snapshot_slot`, in slot order, and apply them to the snapshot bitmap. One scan per disk-group (or per disk) covers all its zones, batch recovery, low overhead. **Requires a `JournalScan` -crowdb-kv RPC** (slot-range + key-prefix filter, returns ops in slot -order). This is the sole crowdb-kv extension diskdb needs. Fast because +crowdb-paxos-kv RPC** (slot-range + key-prefix filter, returns ops in slot +order). This is the sole crowdb-paxos-kv extension diskdb needs. Fast because compaction (strategy 3) keeps the uncompacted record set small. **Strategy 3 — compaction (ongoing maintenance).** @@ -591,8 +591,9 @@ bitmap, with `snapshot_slot` (replay start point), `compact_slot` `bitmap_codec` + compressed `usage_bitmap` + `compact_slot`). The baseline `ZoneValue` is written during disk-add initialization (root §8). A disk without snapshot, busy or free records uses three bounded prefix reads to prove it is -fresh, then publishes initial zone records in batches of at most 128 through -its ownership fence. Its baseline `snapshot_slot` captures the applied frontier +fresh, then publishes initial zone records in batches of at most 64 through +its ownership fence, fitting the tree's reserved gap-repair window. Its baseline +`snapshot_slot` captures the applied frontier before the disk becomes allocatable. Existing records use ordinary recovery. Empty zones keep only geometry and allocation counters; bitmap words are materialized by atomic publication on first allocation. The standard empty diff --git a/doc/design/diskdb/design-crowdb-diskdb.md b/doc/design/diskdb/design-crowdb-diskdb.md index 2dc994d4e..266403d65 100644 --- a/doc/design/diskdb/design-crowdb-diskdb.md +++ b/doc/design/diskdb/design-crowdb-diskdb.md @@ -78,13 +78,13 @@ ChunkDB owner to publish the new layout. capacity and busy/free, with a recalculation path to verify correctness. -**Design philosophy:** "diskdb is a thin, stateless client of crowdb-kv." -All consensus, replication, and durability are delegated to crowdb-kv. +**Design philosophy:** "diskdb is a thin, stateless client of crowdb-paxos-kv." +All consensus, replication, and durability are delegated to crowdb-paxos-kv. diskdb's job is allocation policy, disk health, and space accounting — -nothing more. One crowdb-kv extension is required: a `JournalScan` RPC +nothing more. One crowdb-paxos-kv extension is required: a `JournalScan` RPC (slot-range + key-prefix filter, returns ops in slot order) for fast crash recovery (zone-management §6, strategy 2). This is the sole extension; all other -diskdb ↔ crowdb-kv interaction uses the existing scan / get / put / +diskdb ↔ crowdb-paxos-kv interaction uses the existing scan / get / put / batch_write API. ## 2. Non-Goals (Design Envelope) @@ -93,9 +93,9 @@ batch_write API. implement a device engine. Background relocation delegates copy and fsync to DiskIO and persists only allocation and relocation metadata. - **No local WAL.** CROWDB KV's WAL is the sole durability mechanism. -- **No consensus code.** diskdb is a client of crowdb-kv, with one +- **No consensus code.** diskdb is a client of crowdb-paxos-kv, with one extension: a `JournalScan` RPC for fast crash recovery (zone-management §6). All - other interaction uses the existing crowdb-kv API. + other interaction uses the existing crowdb-paxos-kv API. - **No native SMR / zoned-namespace SSD support (v1).** Start with conventional HDD/SSD (`ZoneBlockDisk`); the zone is a logical concept that can later adopt to native zone APIs. @@ -197,7 +197,7 @@ a multiple of 64) ensures no bitmap masking or padding. Details in ### 3.6 Common protocol crate; crowdb-rpc transport A new `lib/crowdb-protocol` crate holds flatbuffers definitions for all CROWDB -components. diskdb uses it first; crowdb-kv's existing schemas stay where +components. diskdb uses it first; crowdb-paxos-kv's existing schemas stay where they are (unchanged). The flatbuffers messages are reused across the crowdb-rpc transport. @@ -242,7 +242,7 @@ routing (§3.2); it is not stored in record keys or `Segment`. │ │ disk-grp4│ │ disk-grp5│ │ disk-grp6│ │ │ └────┬─────┘ └────┬─────┘ └────┬─────┘ │ └───────┼─────────────────────┼─────────────────────┼────────────┘ - │ crowdb-kv-client │ │ + │ crowdb-paxos-kv-client │ │ v v v ┌────────────────────────────────────────────────────────────────┐ │ CROWDB KV (Multi-Paxos) │ @@ -291,12 +291,12 @@ The storage path uses three RPC services: free update the durable allocator records. - **Hardware admin** (no rpc surface): rack/node/disk-group/disk add/remove and `set_*_status` are writes to group-0 sysdata - performed through `HardwareClient` in `crowdb-kv-client`, invoked by + performed through `HardwareClient` in `crowdb-paxos-kv-client`, invoked by the console (`crowdb-web` / `crowdb-cli`). Hardware reads use `HardwareClient` prefix scans; liveness uses `ServiceRegistryClient.heartbeat`. The diskdb server reads hardware state from group 0 via `HardwareClient` in its sync loop; it does not - serve hardware admin. See `doc/design/kv/design-crowdb-kv-group0.md` + serve hardware admin. See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §2.8. - **`ChunkdbService`**: `AllocateChunk`, `AppendChunk`, `QueryChunk`, `SealChunk`, `DeleteChunk`, @@ -316,9 +316,9 @@ status codes with `ErrorInfo` details (`error_code.fbs`), not Group 0 stores the diskdb subsystem metadata as KV entries. The authoritative schema is defined in -[`doc/design/kv/design-crowdb-kv-group0.md`](../kv/design-crowdb-kv-group0.md) -(text-path keys + JSON values, owned by `crowdb-kv-client`'s -`HardwareClient` / `ServiceRegistryClient` / `KVClusterMetaClient`). +[`doc/design/kv/design-crowdb-paxos-kv-group0.md`](../kv/design-crowdb-paxos-kv-group0.md) +(text-path keys + JSON values, owned by `crowdb-paxos-kv-client`'s +`HardwareClient` / `ServiceRegistryClient` / `PaxosKvClusterMetaClient`). This section summarizes the diskdb-relevant parts; see the group-0 design doc for the full key/value layout and scan patterns. @@ -346,7 +346,7 @@ each key for scan narrowing: structs live in `lib/crowdb-protocol/src/key/` and implement both `BinaryKey` (kept for data-group records) and `TextKey` (group 0). See `doc/design/protocol/design-crowdb-protocol-key.md` §5 and -`doc/design/kv/design-crowdb-kv-group0.md` §3. +`doc/design/kv/design-crowdb-paxos-kv-group0.md` §3. diskdb's own data groups (zone records) keep the `BinaryKey` encoding: `ZoneKey`, `BusyBlockKey`, `FreeBlockKey` (binary + flatbuffers, unchanged). @@ -355,7 +355,7 @@ Value types live in `crowdb-protocol` (flatbuffers `*Value` types used directly; new sysdata values `OwnerMapValue`, `BindMapValue`, `InstanceValue` added as schema messages; Entry return types `DiskGroupEntry`/`DiskdbOwnerEntry`/`KVGroupBindEntry` are plain serde -structs). See `design-crowdb-kv-group0.md` §3.3. +structs). See `design-crowdb-paxos-kv-group0.md` §3.3. **Note**: zones are NOT stored in group 0. A zone is created when its disk is added, and its state (records + snapshot) lives on the @@ -535,7 +535,7 @@ This replaces polling for status changes as the primary change-detection mechanism; polling stays as a safety net with an increased interval. The detailed design (schema, registry, apply-path trigger, client, diskdb handler, configuration) lives in -[`design-crowdb-kv-watch-notify.md`](../kv/design-crowdb-kv-watch-notify.md). +[`design-crowdb-paxos-kv-watch-notify.md`](../kv/design-crowdb-paxos-kv-watch-notify.md). ## 9. Space Metrics @@ -646,7 +646,7 @@ persist round-trip. Latency metrics use `LatencyHistogram` (percentile precision) for hot paths (allocate/free bitmap scan, KV persist) and `LatencySummary` (count + sum + max + avg) for cold paths (sync, compaction, zone -rotate), matching crowdb-kv's convention. Gauges are derived snapshots +rotate), matching crowdb-paxos-kv's convention. Gauges are derived snapshots updated on the reporting interval, not hot-path writes. `degraded` and `last_sync_age_secs` are the key health indicators for alerting. @@ -803,7 +803,7 @@ lib/crowdb-diskdb-client (client library for callers: allocate/free/query; cr (allocate/free/query) + extension traits for schema types (`diskdb_type_util.rs`: `DiskIdExt`, `HwStatusExt`, `ZoneAllocationStateExt`, `ZoneValueExt`, `effective_status`). - crowdb-kv's existing schemas stay in `crowdb-kv` (unchanged). The + crowdb-paxos-kv's existing schemas stay in `crowdb-paxos-kv` (unchanged). The flatbuffers messages are reused across the crowdb-rpc transport. - **`app/crowdb-diskdb`** — server crate (package name `crowdb-diskdb`). Combined library + binary: contains all diskdb logic (types, zone @@ -840,7 +840,7 @@ lib/crowdb-diskdb-client (client library for callers: allocate/free/query; cr heartbeat → observation → reconciliation. `DiskdbRpcService` remains the public registration facade; all mutations pass through `mutation_gate`. - **`lib/crowdb-diskdb-client`** — client library for easy access to the server - (allocate/free/query), mirroring `crowdb-kv-client`'s retry + + (allocate/free/query), mirroring `crowdb-paxos-kv-client`'s retry + topology-cache pattern. crowdb-rpc transport; flatbuffers framing. ### Dependencies @@ -848,10 +848,10 @@ lib/crowdb-diskdb-client (client library for callers: allocate/free/query; cr ``` crowdb-common (metrics, logging, time) ↑ -crowdb-kv-client (sole durable store — group 0 + data groups) +crowdb-paxos-kv-client (sole durable store — group 0 + data groups) ↑ -crowdb-diskdb (server) ──depends──> crowdb-kv-client, crowdb-common, protocol -crowdb-diskdb-client ──depends──> protocol, crowdb-kv-client (for group-0 discovery) +crowdb-diskdb (server) ──depends──> crowdb-paxos-kv-client, crowdb-common, protocol +crowdb-diskdb-client ──depends──> protocol, crowdb-paxos-kv-client (for group-0 discovery) protocol ──> (no internal deps beyond flatbuffers/crowdb-rpc) ``` @@ -894,7 +894,7 @@ All public and inter-module APIs are `async`. Runtime is `tokio` These design assumptions map cleanly onto CROWDB and need no design work, just implementation: -- **Durability model**: crowdb-kv's WAL is the sole durable log. DiskDB's blind +- **Durability model**: crowdb-paxos-kv's WAL is the sole durable log. DiskDB's blind allocation/free writes and conditional commit batches become durable through the same Paxos/WAL path. - **Consensus semantics**: Multi-Paxos retains parallel blind allocation and @@ -969,7 +969,7 @@ the full diskdb server. - **Console + CLI integration** — disk/disk-group management UI, zone busy/free visualization, CLI command design. - **Group-0 notify/watch** — replace polling with watch/notify; see - [`design-crowdb-kv-watch-notify.md`](../kv/design-crowdb-kv-watch-notify.md). + [`design-crowdb-paxos-kv-watch-notify.md`](../kv/design-crowdb-paxos-kv-watch-notify.md). Polling stays as safety net. - **Free batch** — size-threshold batching, no timer; graceful-shutdown drain + flush. @@ -980,9 +980,9 @@ the diskdb component area. ## 16. References -- CROWDB root KV design: `doc/design/kv/design-crowdb-kv.md` -- CROWDB WAL design: `doc/design/kv/design-crowdb-kv-wal.md` -- CROWDB metrics design: `doc/design/kv/design-crowdb-kv-observability.md` +- CROWDB root KV design: `doc/design/kv/design-crowdb-paxos-kv.md` +- CROWDB WAL design: `doc/design/kv/design-crowdb-paxos-kv-wal.md` +- CROWDB metrics design: `doc/design/kv/design-crowdb-paxos-kv-observability.md` - CROWDB console design: `doc/design/console/design-crowdb-console.md` --- diff --git a/doc/design/diskdb/diskdb-allocate-flow-analysis.md b/doc/design/diskdb/diskdb-allocate-flow-analysis.md index d0ce7fd99..9607537fb 100644 --- a/doc/design/diskdb/diskdb-allocate-flow-analysis.md +++ b/doc/design/diskdb/diskdb-allocate-flow-analysis.md @@ -30,7 +30,7 @@ crowdb-cli workload task -> lock-free bitmap claim -> busy-record construction -> DdbKvClient::persist_busy_batch - -> KvRpcTransport connection pool + -> PaxosKvRpcTransport connection pool -> KV batch_write -> proposal coalescing and inflight admission -> Paxos accept quorum diff --git a/doc/design/kv/design-crowdb-kv-group0.md b/doc/design/kv/design-crowdb-paxos-kv-group0.md similarity index 90% rename from doc/design/kv/design-crowdb-kv-group0.md rename to doc/design/kv/design-crowdb-paxos-kv-group0.md index 7d02c0e18..5656af0c6 100644 --- a/doc/design/kv/design-crowdb-kv-group0.md +++ b/doc/design/kv/design-crowdb-paxos-kv-group0.md @@ -3,10 +3,10 @@ # CROWDB - Design: System Group (Group 0) -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §3.3 -[`design-crowdb-kv-server.md`](design-crowdb-kv-server.md), -[`design-crowdb-kv-reconfiguration.md`](design-crowdb-kv-reconfiguration.md) -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §3.3 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3.3 +[`design-crowdb-paxos-kv-server.md`](design-crowdb-paxos-kv-server.md), +[`design-crowdb-paxos-kv-reconfiguration.md`](design-crowdb-paxos-kv-reconfiguration.md) +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3.3 --- @@ -15,7 +15,7 @@ Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §3.3 - [1. Overview](#1-overview) - [2. Design Decisions](#2-design-decisions) - [2.1 External and in-process group-0 API surfaces](#21-external-and-in-process-group-0-api-surfaces) - - [2.2 `crowdb-kv-server` mgmt API is internal](#22-crowdb-kv-server-mgmt-api-is-internal) + - [2.2 `crowdb-paxos-kv-server` mgmt API is internal](#22-crowdb-paxos-kv-server-mgmt-api-is-internal) - [2.3 Unified key concept with two encodings](#23-unified-key-concept-with-two-encodings) - [2.4 All cross-component protocol types live in `crowdb-protocol`](#24-all-cross-component-protocol-types-live-in-crowdb-protocol) - [2.5 ID types defined in `crowdb-protocol`](#25-id-types-defined-in-crowdb-protocol) @@ -72,25 +72,25 @@ how the circular-dependency between kv-server and group 0 is handled. ### 2.1 External and in-process group-0 API surfaces -External processes use `crowdb-kv-client` for group-0 sysdata. Code already -running inside `crowdb-kv-server` uses `KvGroupOperations` and the fixed +External processes use `crowdb-paxos-kv-client` for group-0 sysdata. Code already +running inside `crowdb-paxos-kv-server` uses `KvGroupOperations` and the fixed `Group0ControlPlane` facade instead of opening a loopback RPC connection to itself. RPC handlers also delegate to `KvGroupOperations`, so internal and external paths share the same read fences, conditional-write semantics, and error mapping without coupling a monitor to the RPC transport. -`crowdb-kv-client` provides service classes wrapping a client pinned to group +`crowdb-paxos-kv-client` provides service classes wrapping a client pinned to group 0: - **`HardwareClient`** — hardware hierarchy + per-disk-group maps (rack/node/disk-group/disk CRUD, ownership map, binding map). - **`ServiceRegistryClient`** — service instance registry + keep-alive (register, heartbeat, unregister, discover). -- **`KVClusterMetaClient`** — KV-cluster topology records +- **`PaxosKvClusterMetaClient`** — KV-cluster topology records (store/group/replica metadata read/write). -- **`KVClusterAdmin`** — control-plane surface for cluster management +- **`PaxosKvClusterAdmin`** — control-plane surface for cluster management (lifecycle HTTP calls to kv-server + delegates metadata writes to - `KVClusterMetaClient`). Contains a `KVClusterMetaClient` internally. + `PaxosKvClusterMetaClient`). Contains a `PaxosKvClusterMetaClient` internally. Values are small and JSON-encoded. Conditional mutations use revision CAS where concurrent ownership or lifecycle changes must be preserved. Disk status @@ -99,19 +99,19 @@ when writing the new status; missing disks are never recreated by a status update. A matching status is a read-only no-op. Conflicts and unknown outcomes require reconciliation before retry. See §3 for the key/value schema. -### 2.2 `crowdb-kv-server` mgmt API is internal +### 2.2 `crowdb-paxos-kv-server` mgmt API is internal The kv-server HTTP management API (lifecycle endpoints: `add_store`, `add_group`, `add_remote_replicas`, `step_down`, `system_init`) stays in kv-server. These create/destroy actual `PxKvStore`/`PxGroup` objects with WALs and election drivers, which cannot move to a client. -But the API is now **internal**: only `crowdb-kv-client`'s -`KVClusterAdmin` calls it. All other code (`crowdb-web`, `crowdb-cli`) -goes through `KVClusterAdmin`. +But the API is now **internal**: only `crowdb-paxos-kv-client`'s +`PaxosKvClusterAdmin` calls it. All other code (`crowdb-web`, `crowdb-cli`) +goes through `PaxosKvClusterAdmin`. Removed from kv-server: `POST /topology/finalize`, `GET /topology/ready`. Persistent topology-record management moves -to `KVClusterMetaClient` / `HardwareClient` in `crowdb-kv-client`. +to `PaxosKvClusterMetaClient` / `HardwareClient` in `crowdb-paxos-kv-client`. Retained in kv-server (internal): lifecycle endpoints, `GET /topology` (live runtime state export), `GET /health`, `GET /metrics`. @@ -131,7 +131,7 @@ uses text keys + JSON values; the full group-0 key/value schema is in `crowdb-protocol` is the single home for all cross-component protocol types: data structures, RPC messages, key types, and HTTP mgmt API contracts. When `ServerClient`'s mgmt methods are absorbed into -`KVClusterAdmin`, the +`PaxosKvClusterAdmin`, the HTTP request/response types (`AddStoreRequest`, `AddGroupRequest`, `RemoteReplicaInfo`, `StepDownRequest`, `StoreSummary`, `TopologyResponse`) move to `crowdb-protocol` under a new `mgmt` @@ -202,7 +202,7 @@ extended to periodic polling. model as application services. Each kv-server instance registers under `/srv/kv-server/<instance_id>` and heartbeats with its hosted stores/groups and health status. This is implemented (a -background keep-alive loop in `crowdb-kv-server` writing via +background keep-alive loop in `crowdb-paxos-kv-server` writing via `ServiceRegistryClient`); it is not deferred to a follow-up. **Circular-dependency analysis:** The concern is that group 0 is @@ -212,8 +212,8 @@ itself." This is not actually harmful: - **Group 0 is replicated.** If one node hosting a group-0 replica goes down, the remaining replicas maintain quorum. The down node's heartbeat expires, which is the desired behavior. -- **kv-server is already a client of group 0.** `KVClusterMetaClient` - writes topology records to group 0; `KVClusterAdmin` delegates to +- **kv-server is already a client of group 0.** `PaxosKvClusterMetaClient` + writes topology records to group 0; `PaxosKvClusterAdmin` delegates to it. A heartbeat is just another client write. - **If ALL group-0 replicas are down, the cluster is down.** Heartbeats don't matter in that scenario. No component can read group 0 @@ -253,7 +253,7 @@ through the Paxos consensus path. Hardware admin operations (add/remove rack/node/disk-group/disk, set `*_status`) are writes to group-0 sysdata. They are performed -through `HardwareClient` in `crowdb-kv-client`, invoked by the console +through `HardwareClient` in `crowdb-paxos-kv-client`, invoked by the console (`crowdb-web` / `crowdb-cli`). There is **no `DiskdbAdminService` crowdb-rpc surface**: the previous `diskdb_sys_service.fbs` / `diskdb_sys_op.fbs` admin RPCs (`AddRack`, `SetDiskStatus`, @@ -309,7 +309,7 @@ namespace choice) is in All group-0 values are JSON-encoded (`serde_json::to_vec` / `serde_json::from_slice`). Value types live in `crowdb-protocol` (the single home for cross-component data structures) and are used by -`crowdb-kv-client` directly, with no per-crate redefinition: +`crowdb-paxos-kv-client` directly, with no per-crate redefinition: - **Existing flatbuffer `*Value` types** (`RackValue`, `NodeValue`, `DiskGroupValue`, `DiskValue`, `HwStatus`, `DiskId`) — used directly. @@ -322,7 +322,7 @@ single home for cross-component data structures) and are used by (`{ instance_id, lease_expiry_ms }`), `BindMapValue` (`{ store_id, group_id }`). `build.rs` derives serde on each. - **Entry return types** (key fields parsed from the path + value, - produced by `HardwareClient`/`KVClusterMetaClient` reads) are plain + produced by `HardwareClient`/`PaxosKvClusterMetaClient` reads) are plain `#[derive(serde)]` Rust structs in `crowdb-protocol`: `DiskGroupEntry` (`{ rack_id, node_id, dg_id, value: DiskGroupValue }`), `DiskdbOwnerEntry` (`{ rack_id, node_id, dg_id, instance_id, @@ -377,7 +377,7 @@ accordingly (greenfield). Every long-running service instance registers itself in group 0 under `/srv/<service>/<instance_id>` and heartbeats periodically. -`ServiceRegistryClient` in `crowdb-kv-client` is **generic across +`ServiceRegistryClient` in `crowdb-paxos-kv-client` is **generic across services** (diskdb and kv-server now; chunkdb and future services later). It takes a `service` name that selects the path namespace (`/srv/diskdb/`, `/srv/kv-server/`, …) and the value shape. The API: @@ -415,7 +415,7 @@ with the same accepted scope update liveness without advancing the Rack. `hosted_stores`, `hosted_groups`, and aggregate health. Used by the console and other components for kv-server discovery and health visibility. **Implemented** (a background keep-alive - loop in `crowdb-kv-server` writing via `ServiceRegistryClient`). + loop in `crowdb-paxos-kv-server` writing via `ServiceRegistryClient`). - **Future services** (chunkdb, etc.) — same pattern under `/srv/<service>/<instance_id>`. @@ -429,7 +429,7 @@ clean shutdown. ### 4.4 Client discovery -`ServiceDiscoveryClient` (in `crowdb-kv-client`) wraps +`ServiceDiscoveryClient` (in `crowdb-paxos-kv-client`) wraps `ServiceRegistryClient` with a per-service state map and TTL-based refresh. Each state contains an atomically published cached result, an atomic round-robin cursor, and an async single-flight refresh gate. The sharded-map @@ -452,7 +452,7 @@ round-robin doesn't cover); the generic discovery client handles the "pick any living instance" case. The bootstrap flow: a client knows group-0's mgmt port (10000, from -R118) → constructs `CrowdbKvClient` with that seed → +R118) → constructs `PaxosKvClient` with that seed → `ServiceDiscoveryClient::from_shared_kv` → `discover_all("diskdb")` queries `/srv/diskdb/` via group-0 → cache → client connects to the discovered RPC endpoint. No per-service address configuration needed @@ -474,11 +474,11 @@ a clear "no living instances" error. Phase 1 uses console TOML as the topology source of truth (operator-managed). Phase 2 cuts over to group 0 authoritative: `http_cluster_init` writes hardware records via `HardwareClient` and -KV-cluster topology records via `KVClusterMetaClient` into group 0, -and `crowdb-kv-server` restores its stores/groups from local disk on +KV-cluster topology records via `PaxosKvClusterMetaClient` into group 0, +and `crowdb-paxos-kv-server` restores its stores/groups from local disk on restart with group 0 as the verification and fallback source for remote-replica wiring (see -[`design-crowdb-kv-server.md`](design-crowdb-kv-server.md) §2.2). The +[`design-crowdb-paxos-kv-server.md`](design-crowdb-paxos-kv-server.md) §2.2). The topology content in TOML is bootstrap-only: `--root` is required on every start, and restart with `--root` alone rebuilds every local store/group from WAL and rejoins the cluster. An optional `--config` remains the local authority @@ -505,7 +505,7 @@ old `/topology/...` records are replaced by `/hw/...` and ### 5.4 Leader readiness before writing -`HardwareClient` and `KVClusterMetaClient` write via crowdb-rpc to the +`HardwareClient` and `PaxosKvClusterMetaClient` write via crowdb-rpc to the group-0 leader. `http_cluster_init` can only write sysdata after a group-0 leader is elected and reachable. For single-node init this is immediate (self-elect). For multi-node, the init flow must wait @@ -516,14 +516,14 @@ existing flow doesn't already wait. ## 6. Relationship to Existing Design Docs -- **`design-crowdb-kv.md` §3.3** — defines the system group concept. +- **`design-crowdb-paxos-kv.md` §3.3** — defines the system group concept. This doc expands it with the sysdata schema, service registry, and keep-alive model. -- **`design-crowdb-kv-server.md` §2.4** — defines the HTTP management +- **`design-crowdb-paxos-kv-server.md` §2.4** — defines the HTTP management API. This doc clarifies that the API is internal (only - `crowdb-kv-client` calls it) and that `topology_finalize` / + `crowdb-paxos-kv-client` calls it) and that `topology_finalize` / `topology_ready` are removed. -- **`design-crowdb-kv-reconfiguration.md`** — group 0 membership +- **`design-crowdb-paxos-kv-reconfiguration.md`** — group 0 membership evolves using the shipped Model B reconfiguration. No new consensus primitive required. - **`design-crowdb-protocol-key.md` §5** — documents the unified key concept diff --git a/doc/design/kv/design-crowdb-kv-leader-election.md b/doc/design/kv/design-crowdb-paxos-kv-leader-election.md similarity index 96% rename from doc/design/kv/design-crowdb-kv-leader-election.md rename to doc/design/kv/design-crowdb-paxos-kv-leader-election.md index 5566f47aa..c0e5f0a14 100644 --- a/doc/design/kv/design-crowdb-kv-leader-election.md +++ b/doc/design/kv/design-crowdb-paxos-kv-leader-election.md @@ -3,8 +3,8 @@ # CROWDB - Design: Leader Election, Term, and Lease -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §3, §4.2, §6.2, §7 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §3, §4.2, §6.2, §7 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3, §4.2, §6.2, §7 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3, §4.2, §6.2, §7 This document specifies leader election, term management, the `PxBallot`/`PxTerm` separation, and the leader lease used for fast linearizable reads. The design follows Raft very closely; only the per-slot Paxos parts differ. @@ -145,7 +145,7 @@ These re-Accepts are pipelined; the new leader does not wait for any one to be c ### 4.4 Steady state begins -Once the bulk Phase 1 has been *issued* (not necessarily completed), the leader is free to start assigning new slots starting at `ceiling + 1`. The repair work for `[floor+1, ceiling]` proceeds in parallel and reuses the same machinery as routine gap repair (see [`design-crowdb-kv-slot.md`](design-crowdb-kv-slot.md) §9). +Once the bulk Phase 1 has been *issued* (not necessarily completed), the leader is free to start assigning new slots starting at `ceiling + 1`. The repair work for `[floor+1, ceiling]` proceeds in parallel and reuses the same machinery as routine gap repair (see [`design-crowdb-paxos-kv-slot.md`](design-crowdb-paxos-kv-slot.md) §9). --- @@ -203,14 +203,14 @@ Engine apply is decoupled from the heartbeat handler via a **background apply lo ## 6. Leader Lease -A lease lets the leader serve `Get(mode=Linearizable)` without a per-read quorum round-trip ([§6.1 of design-crowdb-kv.md](design-crowdb-kv.md#61-linearizable-leader-read)). The lease is the standard Raft-style approach with a clock-skew bound. +A lease lets the leader serve `Get(mode=Linearizable)` without a per-read quorum round-trip ([§6.1 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#61-linearizable-leader-read)). The lease is the standard Raft-style approach with a clock-skew bound. ### 6.1 What the lease grants While its lease is valid, a leader may serve linearizable reads from local state under two assumptions: - No other node was elected leader during the lease window. -- The leader's `contiguous_applied` slot reflects all writes acked through this leader (which is true by Invariant I3 from [`design-crowdb-kv-slot.md`](design-crowdb-kv-slot.md) §2). +- The leader's `contiguous_applied` slot reflects all writes acked through this leader (which is true by Invariant I3 from [`design-crowdb-paxos-kv-slot.md`](design-crowdb-paxos-kv-slot.md) §2). The first assumption is enforced by: @@ -234,7 +234,7 @@ With the default `heartbeat_interval = 500 ms` and `lease_duration = 9 × heartb ### 6.3 Clock-skew assumption -design-crowdb-kv.md §3](design-crowdb-kv.md) caps clock skew at `max_clock_skew` (default `500 ms`, see §10) per heartbeat interval. The lease formula is: +design-crowdb-paxos-kv.md §3](design-crowdb-paxos-kv.md) caps clock skew at `max_clock_skew` (default `500 ms`, see §10) per heartbeat interval. The lease formula is: ``` effective_lease = lease_duration - max_clock_skew @@ -355,7 +355,7 @@ If the clock-skew assumption is *violated*, lease-based reads can return stale d | `heartbeat_interval` | 500 ms | 10 ms – 30 s | Should be ≪ `lease_duration` | | `lease_duration` | 4500 ms | 100 ms – 60 s | Should be ≫ `heartbeat_interval` + `max_clock_skew` (rule of thumb: `9 × heartbeat_interval`) | | `effective_lease` | derived | — | `= lease_duration - max_clock_skew` | -| `max_clock_skew` | 500 ms | 1 ms – 5 s | Architectural bound from design-crowdb-kv.md §3](design-crowdb-kv.md) | +| `max_clock_skew` | 500 ms | 1 ms – 5 s | Architectural bound from design-crowdb-paxos-kv.md §3](design-crowdb-paxos-kv.md) | | `election_min` | 4000 ms | ≥ 8 × `heartbeat_interval` | Avoid spurious elections | | `election_max` | 8000 ms | ≤ 60 s | Bounds time to elect after leader loss | | `prevote_enabled` | true | bool | Reduces disruption from rejoining nodes | @@ -366,8 +366,8 @@ Notes: - **PreVote** (Raft optimization, ON by default): a candidate first asks "would you vote for me?" without bumping the term. This avoids spurious term increments from a partitioned-and-rejoined node. - **Pre-emptive step-down on heartbeat loss** is governed by `lease_duration`, not `election_min`. A leader that loses contact with quorum gives up its lease at `lease_duration` and stops serving fast reads even before any follower starts an election. -- **Default choice rationale.** The defaults above are tuned for general-purpose deployments (mix of single-DC and modest cross-AZ latency). They give ~5–8 s failover detection, which matches the operational expectations of most online KV workloads while keeping heartbeat chatter low. See [`design-crowdb-kv.md`](design-crowdb-kv.md) §13 (Open Design Questions) for the analysis. +- **Default choice rationale.** The defaults above are tuned for general-purpose deployments (mix of single-DC and modest cross-AZ latency). They give ~5–8 s failover detection, which matches the operational expectations of most online KV workloads while keeping heartbeat chatter low. See [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §13 (Open Design Questions) for the analysis. - **Operational profiles:** - *Low-latency single datacenter:* `heartbeat_interval = 100 ms`, `election_min/max = 800/1500 ms`, `lease_duration = 900 ms`, `max_clock_skew = 100 ms`. Sub-second failover; higher message rate. - *Cross-region / WAN:* `heartbeat_interval = 3 s`, `election_min/max = 24/48 s`, `lease_duration = 27 s`. Matches CockroachDB-style geo-replicated tunings; tolerates wider RTT variance at the cost of failover latency. - - *Tests:* `PxElectionConfig::for_tests()` produces `heartbeat = 5 ms`, `election = 30–60 ms`, `lease = 25 ms` for use under `tokio::time::pause()`. Not exposed on `crowdb-kv-server` CLI. + - *Tests:* `PxElectionConfig::for_tests()` produces `heartbeat = 5 ms`, `election = 30–60 ms`, `lease = 25 ms` for use under `tokio::time::pause()`. Not exposed on `crowdb-paxos-kv-server` CLI. diff --git a/doc/design/kv/design-crowdb-kv-observability.md b/doc/design/kv/design-crowdb-paxos-kv-observability.md similarity index 96% rename from doc/design/kv/design-crowdb-kv-observability.md rename to doc/design/kv/design-crowdb-paxos-kv-observability.md index 5e4231632..a8ea1219e 100644 --- a/doc/design/kv/design-crowdb-kv-observability.md +++ b/doc/design/kv/design-crowdb-paxos-kv-observability.md @@ -3,8 +3,8 @@ # CROWDB - Design: Observability -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §16 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §16 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §16 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §16 ## Table of Contents @@ -94,7 +94,7 @@ registry has `start(interval_secs)` (spawns flush thread/task), `stop()` (final flush + join), and `flush()` (iterate all metrics, snapshot, format, reset window state). Interval is typically 5s or 10s. -- Rust (`lib/crowdb-kv/src/metrics/mod.rs`): `MetricsRegistry` with type-grouped +- Rust (`lib/crowdb-paxos-kv/src/metrics/mod.rs`): `MetricsRegistry` with type-grouped `Vec<T>` collections, `Arc`-shared, metric handles stored on service/store structs. `MetricsRunner` spawns a tokio interval task, computes real elapsed `window_secs` per tick, flushes Rust metrics, then invokes a @@ -282,12 +282,12 @@ All five are `LatencySummary` (count + sum + max), matching the read-path `barrier` / `engine_get` pattern. Percentile precision is not needed for phase-level attribution. The bench client already has `PreciseHistogram` for client-observed p99. The `accept_quorum_rpc` timer is meaningful only -after the quorum short-circuit (§6.1 of `design-crowdb-kv-rpc.md`); it records the +after the quorum short-circuit (§6.1 of `design-crowdb-paxos-kv-rpc.md`); it records the quorum-th-fastest remote latency, not the full fan-out tail. ### 2.12 Client Metrics -`crowdb-kv-client` exposes its own `ClientMetrics` (lock-free `AtomicU64` +`crowdb-paxos-kv-client` exposes its own `ClientMetrics` (lock-free `AtomicU64` counters, snapshotted via `ClientMetricsSnapshot`) for retry, topology, and read-distribution observability. They are not part of the server's `MetricsRegistry`. The client is a separate process and has no C++ @@ -368,11 +368,11 @@ server binaries. Every server binary accepts a `--log-dir <DIR>` CLI flag. All Rust and C++ logs for a process land under that directory. Defaults: -- `crowdb-kv-server`, `crowdb-diskdb`, `crowdb-chunkdb` — `"log"` - (relative to CWD). `crowdb-kv-server` also falls back to - `CrowDBConfig.log_dir` (config field, default `root/log`) when +- `crowdb-paxos-kv-server`, `crowdb-diskdb`, `crowdb-chunkdb` — `"log"` + (relative to CWD). `crowdb-paxos-kv-server` also falls back to + `PaxosKvConfig.log_dir` (config field, default `root/log`) when `--log-dir` is not given. -- `crowdb-web` — `~/.crowdb-kv/log`. +- `crowdb-web` — `~/.crowdb-paxos-kv/log`. - `crowdb-cli` — multi-service lifecycle, server deployment, and benchmark commands use `<cwd>/cli-log/cli-{command-slug}-{timestamp}/`; short inspection and data operations are console-only and create no file log. @@ -401,11 +401,11 @@ start in the same second. Rust and C++ write to separate files in the same directory (D1). Prefixes: -- `{server}` — Rust tracing log (e.g. `crowdb-kv-server`). +- `{server}` — Rust tracing log (e.g. `crowdb-paxos-kv-server`). - `{server}-tree` — C++ crowdb-tree spdlog (e.g. - `crowdb-kv-server-tree`). + `crowdb-paxos-kv-server-tree`). - `{server}-rpc` — C++ crowdb-rpc spdlog (e.g. - `crowdb-kv-server-rpc`). + `crowdb-paxos-kv-server-rpc`). - `{server}-metrics` — metrics log (Rust, via `MetricsRunner`). The shared timestamp + PID suffix lets the operator interleave files @@ -426,7 +426,7 @@ Both Rust and C++ stacks default to `info` (D3). The `--log-level` CLI flag drives both stacks. If `RUST_LOG` is set, the Rust `EnvFilter` uses it directly; for the C++ side, the first global directive is extracted via `cpp_level_from_rust_log()` (e.g. -`RUST_LOG=debug` → C++ `"debug"`, `RUST_LOG=crowdb_kv=info` → C++ +`RUST_LOG=debug` → C++ `"debug"`, `RUST_LOG=crowdb_paxos_kv=info` → C++ `"info"`). Valid levels: `trace`, `debug`, `info`, `warn`, `error`, `off`. @@ -438,7 +438,7 @@ for failures without grepping the file. `crowdb-web` defaults this to ### 3.6 C++ Logger Additivity -When `ct_init_logging` runs first (as in `crowdb-kv-server`), +When `ct_init_logging` runs first (as in `crowdb-paxos-kv-server`), `crowdb_rpc_init_logging` calls `add_log_file` — it adds a second file sink to the same `crowdb::common` logger rather than creating a new one. The rpc sink inherits the tree logger's level. The diff --git a/doc/design/kv/design-crowdb-kv-reconfiguration.md b/doc/design/kv/design-crowdb-paxos-kv-reconfiguration.md similarity index 95% rename from doc/design/kv/design-crowdb-kv-reconfiguration.md rename to doc/design/kv/design-crowdb-paxos-kv-reconfiguration.md index 3009e747f..943cfd3e2 100644 --- a/doc/design/kv/design-crowdb-kv-reconfiguration.md +++ b/doc/design/kv/design-crowdb-paxos-kv-reconfiguration.md @@ -3,10 +3,10 @@ # CROWDB - Design: Reconfiguration -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §9.1, §9.2, [`design-crowdb-kv-leader-election.md`](design-crowdb-kv-leader-election.md) -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §9.1, §9.2 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §9.1, §9.2, [`design-crowdb-paxos-kv-leader-election.md`](design-crowdb-paxos-kv-leader-election.md) +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §9.1, §9.2 -This document specifies how a CROWDB group safely changes its membership while preserving consensus safety. The mechanism is direct per-node HTTP mutation of each replica's remote-replica list, persisted to the local `GroupConfigStore`, with a `membership_epoch` exact-match fence. This model applies to all groups including the system group (group 0, which stores cluster topology metadata, see `design-crowdb-kv.md` §3.3). +This document specifies how a CROWDB group safely changes its membership while preserving consensus safety. The mechanism is direct per-node HTTP mutation of each replica's remote-replica list, persisted to the local `GroupConfigStore`, with a `membership_epoch` exact-match fence. This model applies to all groups including the system group (group 0, which stores cluster topology metadata, see `design-crowdb-paxos-kv.md` §3.3). ## Table of Contents @@ -31,7 +31,7 @@ CROWDB supports membership changes within a single group. Specifically: - **Replace a member.** Implemented as add-then-remove (or vice versa), each as a single-member change. - **Change the leadership of the group.** Triggered as a side effect when removing the current leader. -Out of scope ([design-crowdb-kv.md §2](design-crowdb-kv.md)): +Out of scope ([design-crowdb-paxos-kv.md §2](design-crowdb-paxos-kv.md)): - Changing `num_groups` (the total number of groups in the cluster) — fixed at cluster creation. - Splitting or merging groups — not supported. @@ -102,7 +102,7 @@ By requiring catch-up while the member is non-voting, we ensure that when it fir ### 3.3 Snapshot install protocol -Defined in [§8.4 of design-crowdb-kv.md](design-crowdb-kv.md#84-snapshot-and-install) and `design-crowdb-kv-state-machine.md` §6 (snapshot import). Resumable, throttled, end-to-end CRC. +Defined in [§8.4 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#84-snapshot-and-install) and `design-crowdb-paxos-kv-state-machine.md` §6 (snapshot import). Resumable, throttled, end-to-end CRC. ### 3.4 Catch-up termination criterion @@ -197,12 +197,12 @@ topology metadata. It uses the same direct HTTP mutation + groups. No joint-consensus primitive is needed. Group 0 stores cluster topology as KV entries under text-path keys (`/hw/rack/...`, `/hw/node/...`, `/kv/store/...`, `/kv/group/...`) with JSON -values, written by `HardwareClient` and `KVClusterMetaClient` in -`crowdb-kv-client`. It is created via `POST /system/init`. See -`design-crowdb-kv.md` §3.3, `design-crowdb-kv-group0.md`, and +values, written by `HardwareClient` and `PaxosKvClusterMetaClient` in +`crowdb-paxos-kv-client`. It is created via `POST /system/init`. See +`design-crowdb-paxos-kv.md` §3.3, `design-crowdb-paxos-kv-group0.md`, and `../console/design-crowdb-console.md` §4.3 for details. -A system embedding `crowdb-kv`'s primitives may choose to build a +A system embedding `crowdb-paxos-kv`'s primitives may choose to build a joint-consensus layer on top of the `membership_epoch` fence. --- diff --git a/doc/design/kv/design-crowdb-kv-rpc-client.md b/doc/design/kv/design-crowdb-paxos-kv-rpc-client.md similarity index 89% rename from doc/design/kv/design-crowdb-kv-rpc-client.md rename to doc/design/kv/design-crowdb-paxos-kv-rpc-client.md index 21b0ca707..94a662a02 100644 --- a/doc/design/kv/design-crowdb-kv-rpc-client.md +++ b/doc/design/kv/design-crowdb-paxos-kv-rpc-client.md @@ -3,8 +3,8 @@ # CROWDB - Design: Client-Facing KV RPC -Depends on: [`design-crowdb-kv-rpc.md`](design-crowdb-kv-rpc.md) §3, §5, §10, §11, §12; [`design-crowdb-rpc.md`](../rpc/design-crowdb-rpc.md) §3, §4, §6 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §3, §10.1 +Depends on: [`design-crowdb-paxos-kv-rpc.md`](design-crowdb-paxos-kv-rpc.md) §3, §5, §10, §11, §12; [`design-crowdb-rpc.md`](../rpc/design-crowdb-rpc.md) §3, §4, §6 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3, §10.1 The client-facing KV service (Put/Get/Delete/BatchWrite/Scan/ JournalScan/CreateSnapshot/ListSnapshots/SnapshotScan/ReleaseSnapshot + @@ -19,10 +19,10 @@ programmatic (`CrowdbClient::with_rpc_transport`). - [2. Message Surface](#2-message-surface) - [3. Flatbuffer Schema (kv_client.fbs)](#3-flatbuffer-schema-kv_clientfbs) - [4. Port Allocation](#4-port-allocation) -- [5. Server-Side Handler (KvRpcService)](#5-server-side-handler-kvrpcservice) +- [5. Server-Side Handler (PaxosKvRpcService)](#5-server-side-handler-kvrpcservice) - [6. Transparent Leader-Forwarding](#6-transparent-leader-forwarding) - [7. WatchNotify Server-Push](#7-watchnotify-server-push) -- [8. Client-Side Transport (KvRpcTransport)](#8-client-side-transport-kvrpctransport) +- [8. Client-Side Transport (PaxosKvRpcTransport)](#8-client-side-transport-kvrpctransport) - [9. CrowdbClient Transport Selection](#9-crowkvclient-transport-selection) - [10. WatchNotifyClient](#10-watchnotifyclient) - [11. Zero-Copy Wrapper Classes](#11-zero-copy-wrapper-classes) @@ -83,14 +83,14 @@ re-exported as `crowdb_protocol::kv_client_fb`. - **`forwarded: bool` field.** On `FBKvGetRequest`, `FBKvScanRequest`, and `FBKvJournalScanRequest` — the loop-guard for transparent leader-forwarding (the `forwarded` field replaces the - `x-crowdb-kv-forwarded` metadata header). + `x-crowdb-paxos-kv-forwarded` metadata header). ## 4. Port Allocation The client-facing crowdb-rpc port is derived from the base port: ``` -client_rpc_port = rpc_port + (KV_CLIENT_RPC_BASE - KV_SERVER_RPC_BASE) +client_rpc_port = rpc_port + (KV_CLIENT_RPC_BASE - PAXOS_KV_SERVER_RPC_BASE) = rpc_port + 200 ``` @@ -98,10 +98,10 @@ This is parallel to the consensus-side offset (`rpc_port + 100`). The `KV_CLIENT_RPC_BASE` constant lives in `crowdb-protocol::ports`. The port mapping is registered as `KvServerClientRpc` in the port-claim registry. -## 5. Server-Side Handler (KvRpcService) +## 5. Server-Side Handler (PaxosKvRpcService) -`KvRpcService` (in `lib/crowdb-kv/src/rpc/kv_rpc_service.rs`) holds -`Arc<PxKvStore>` + `Handle` (tokio runtime) + `Arc<KvClientRpcForwarder>`. +`PaxosKvRpcService` (in `lib/crowdb-paxos-kv/src/rpc/kv_rpc_service.rs`) holds +`Arc<PxKvStore>` + `Handle` (tokio runtime) + `Arc<PaxosKvRpcForwarder>`. `register_handlers` wires one handler per request `msg_type` into the client-facing `RpcServer`. @@ -119,7 +119,7 @@ c. Build the response flatbuffer via `FlatBufferBuilder`, `finish`, Server startup: `start_client_rpc_server` (in `kv_server.rs`) creates the `RpcServer`, binds the client-facing port, constructs the -`KvClientRpcForwarder` + `KvRpcService`, registers handlers, and stores +`PaxosKvRpcForwarder` + `PaxosKvRpcService`, registers handlers, and stores the server state in `PxKvStore::client_rpc_server_state`. `main.rs` calls it after `start_rpc_server`. `shutdown_server` stops both crowdb-rpc servers. @@ -128,18 +128,18 @@ crowdb-rpc servers. `Get`/`Scan`/`JournalScan` preserve the forward step (linearizable reads only; `MinSlot` served locally). The loop-guard -`x-crowdb-kv-forwarded` metadata header becomes the `forwarded: bool` +`x-crowdb-paxos-kv-forwarded` metadata header becomes the `forwarded: bool` field on the request flatbuffer. The handler: a. If `read_mode == Linearizable && !req.forwarded` and `forward_target_for(group_id)` returns `Some(endpoint)` → re-issue - the request to the leader via `KvClientRpcForwarder`. Set + the request to the leader via `PaxosKvRpcForwarder`. Set `forwarded = true` on the re-issued request. On success, submit the leader's response. On forward failure, serve stale local + set `not_leader_hint = endpoint`. b. Else serve locally. -`KvClientRpcForwarder` lives in `crowdb-kv` itself (not `crowdb-kv-client`) +`PaxosKvRpcForwarder` lives in `crowdb-paxos-kv` itself (not `crowdb-paxos-kv-client`) to avoid a crate cycle. It holds an `Arc<RpcServer>` + `Arc<RpcClient>` + connection cache, builds the request flatbuffer with `forwarded = true`, calls `rpc.call()`, and returns the raw response @@ -171,9 +171,9 @@ callback to the handler. The `WatchRegistry::emit` path detects a dead push target when `RpcClient::send` returns `ConnectionClosed`. The safety-net poller covers missed notifications during the gap. -## 8. Client-Side Transport (KvRpcTransport) +## 8. Client-Side Transport (PaxosKvRpcTransport) -`KvRpcTransport` mirrors `PxRpcTransport`: it holds `Arc<RpcServer>`, +`PaxosKvRpcTransport` mirrors `PxRpcTransport`: it holds `Arc<RpcServer>`, `Arc<RpcClient>`, a shared `ConnectionPoolIndex`, and an atomic request-ID source. `conn_for` derives the client-facing crowdb-rpc port from the base port via the `KV_CLIENT_RPC_BASE` offset. Pool selection returns its @@ -200,7 +200,7 @@ retry + topology-cache logic. ## 9. CrowdbClient Transport Selection -`CrowdbClient` holds an `Option<Arc<KvRpcTransport>>` field +`CrowdbClient` holds an `Option<Arc<PaxosKvRpcTransport>>` field (`rpc_transport`). `with_rpc_transport(transport)` sets it. Each public method (`put`/`get`/`delete`/`batch_write`/`scan`/`scan_count`/ `journal_scan`) checks `self.rpc_transport` first: when set, delegate @@ -213,7 +213,7 @@ to the transport's `send_*` (with the existing retry/topology/ - **crowdb-rpc path**: persistent connection + client-side handler. The reader loop resolves the leader, - opens a connection via `KvRpcTransport::get_conn`, registers + opens a connection via `PaxosKvRpcTransport::get_conn`, registers handlers for `FBWatchNotify` + `FBWatchNotifyError` via `RpcClient::register_handler`, sends `FBWatchSubscribe` as fire-and-forget `send()`. On `FBWatchNotifyError` with non-empty @@ -258,12 +258,12 @@ convention. ## References -- [`design-crowdb-kv-rpc.md`](design-crowdb-kv-rpc.md) — consensus RPC wire +- [`design-crowdb-paxos-kv-rpc.md`](design-crowdb-paxos-kv-rpc.md) — consensus RPC wire protocol (same engine, different port + schema). - [`design-crowdb-rpc.md`](../rpc/design-crowdb-rpc.md) — RPC engine overview (framing, connection pool, handler pattern, wrapper convention). -- [`design-crowdb-kv-watch-notify.md`](design-crowdb-kv-watch-notify.md) — +- [`design-crowdb-paxos-kv-watch-notify.md`](design-crowdb-paxos-kv-watch-notify.md) — WatchNotify design (trie, prefix matching, safety-net poller). ## 13. Topology cache eviction diff --git a/doc/design/kv/design-crowdb-kv-rpc.md b/doc/design/kv/design-crowdb-paxos-kv-rpc.md similarity index 93% rename from doc/design/kv/design-crowdb-kv-rpc.md rename to doc/design/kv/design-crowdb-paxos-kv-rpc.md index 06212f2c9..7bb87353d 100644 --- a/doc/design/kv/design-crowdb-kv-rpc.md +++ b/doc/design/kv/design-crowdb-paxos-kv-rpc.md @@ -3,8 +3,8 @@ # CROWDB - Design: RPC Wire Protocol -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §2, §3, §9.2, §10 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §3, §9.2, §10.1 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §2, §3, §9.2, §10 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3, §9.2, §10.1 This document defines the wire-serialization contract for all node-to-node and client-to-node RPC communication. The consensus hot @@ -14,7 +14,7 @@ Heartbeat/StepDown, ChosenNotification, FetchGap) uses the library with flatbuffer framing and request/response correlation via oneshot channels. Client-to-node KV RPCs (Get/Set/Delete/Batch/Scan) use the same crowdb-rpc engine on a separate port with a dedicated -schema — see [`design-crowdb-kv-rpc-client.md`](design-crowdb-kv-rpc-client.md) +schema — see [`design-crowdb-paxos-kv-rpc-client.md`](design-crowdb-paxos-kv-rpc-client.md) for the client-facing transport design. Every message carries a `version: u32` field for forward/backward compatibility; no `required` fields; field numbers are append-only. @@ -49,12 +49,12 @@ a `version: u32` field for forward/backward compatibility; no port with a dedicated schema for client library ergonomics and `Bytes` mapping. 2. **Pipelined unary calls on persistent connections.** Steady-state - data traffic (`Accept`, `ChosenNotification`) multiplexes over + request-response traffic (`Accept`) multiplexes over pipelined unary `call()`s on a persistent crowdb-rpc connection per `(group_id, peer_id)` pair. Connection setup cost is paid once per server lifetime. Fire-and-forget frames (`ChosenNotification`, - `BatchChosenNotification`) use `send()` with a no-op completion - callback — no reply is awaited. + `BatchChosenNotification`) use `send_one_way()` without registering + a reply completion or timeout. They share the persistent connections. 3. **Heartbeats on the same connection.** Steady-state `Heartbeat` traffic flows over the same crowdb-rpc connection as `Accept` and `Chosen`. The crowdb-rpc transport multiplexes frames at the I/O @@ -91,7 +91,7 @@ kinds are designed but not yet implemented. The full flatbuffer schema is in `lib/crowdb-protocol/src/fbs/kv_consensus.fbs`; the crowdb-rpc schema definitions for client KV RPCs are in -`lib/crowdb-kv/src/rpc/fbs/`. This doc covers design decisions only. +`lib/crowdb-paxos-kv/src/rpc/fbs/`. This doc covers design decisions only. --- @@ -101,7 +101,7 @@ The consensus RPC path uses the R104 `crowdb-rpc` engine: a C++ epoll/kqueue transport with flatbuffer framing and request/response correlation. The Rust facade (`crowdb-rpc-ffi`) exposes `RpcServer` (listen + register handlers), `RpcClient` (connect + `call()` for -request-response, `send()` for fire-and-forget), and `Connection` +request-response, `send_one_way()` for fire-and-forget), and `Connection` (per-peer persistent connection). **Why crowdb-rpc for consensus:** @@ -122,7 +122,7 @@ request-response, `send()` for fire-and-forget), and `Connection` **Client-facing KV RPCs:** Client-facing KV RPCs (`Get`, `Set`, `Delete`, `Batch`, `Scan`, `Watch`) run on the same crowdb-rpc engine on a separate port with a dedicated schema. The client library -(`crowdb-kv-client`) uses crowdb-rpc for retry, topology cache, and +(`crowdb-paxos-kv-client`) uses crowdb-rpc for retry, topology cache, and `NotLeaderHint` handling. The crowdb-rpc server also serves bounded snapshot-session handlers for unpublished new-member install. @@ -212,11 +212,12 @@ the control buffer. The response is parsed via the zero-copy `FB<Type>ResponseRef` wrapper (§11). **Fire-and-forget path (`send_chosen`, `send_batch_chosen`):** Builds -the flatbuffer, calls `rpc.send(server, conn, req_id, control, None, -msg_type, noop_completion(), null_mut())`. The no-op completion -callback satisfies the C++ side's non-null `on_complete` requirement. -No reply is awaited; failures are returned for caller-side -observability but treated as best-effort. +the flatbuffer, calls `rpc.send_one_way(server, conn, req_id, control, +None, msg_type)`. The frame carries the one-way flag and creates no +pending response entry, completion slot, or reply deadline. Successful +submission does not acknowledge remote delivery or apply. Send failures +are returned for caller-side observability but treated as best-effort; +the existing gap recovery handles missed notifications. **Port derivation:** `conn_for(endpoint)` parses the server endpoint (host:port) and connects to `port + RPC_PORT_OFFSET` (see §12). @@ -249,7 +250,7 @@ transport. ## 7. Flow Control and Parallelism The crowdb-rpc transport enables the parallel-slot pipelining described -in `design-crowdb-kv-slot.md` §5: +in `design-crowdb-paxos-kv-slot.md` §5: - **Multiple in-flight `Accept` frames per peer.** Each `send_accept` call creates a new `CallFuture` and returns immediately; the caller @@ -486,9 +487,9 @@ Edge cases: The crowdb-rpc port is derived from the base port via a fixed offset: ``` -KV_SERVER_RPC_BASE = 28001 (rpc port base for KV server) +PAXOS_KV_SERVER_RPC_BASE = 28001 (rpc port base for KV server) KV_RPC_BASE = 28101 (crowdb-rpc port base for KV server) -RPC_PORT_OFFSET = KV_RPC_BASE - KV_SERVER_RPC_BASE = 100 +RPC_PORT_OFFSET = KV_RPC_BASE - PAXOS_KV_SERVER_RPC_BASE = 100 ``` When a `PxKvStore` binds on port P, the crowdb-rpc server listens @@ -511,7 +512,7 @@ making every clone an O(n) heap allocate + memcpy. For hot-path KV fields that are cloned across retry loops or fanout, this is avoidable. -`lib/crowdb-kv/build.rs` configures selected +`lib/crowdb-paxos-kv/build.rs` configures selected flatbuffer `[ubyte]` fields to map to `bytes::Bytes` instead, turning clones into O(1) atomic ref-count bumps: @@ -526,7 +527,7 @@ clones into O(1) atomic ref-count bumps: - `KvScanRequest.prefix`, `KvScanRequest.start_after` — scan request. - `KvScanItem.key`, `KvScanItem.value` — scan response items. -The client `BatchOp` type (`lib/crowdb-kv-client/src/client.rs`) also uses +The client `BatchOp` type (`lib/crowdb-paxos-kv-client/src/client.rs`) also uses `Bytes` for `key` and `value`, so `BatchOp` → `KvBatchItem` conversion and the per-retry `items.clone()` are both O(1) per item. @@ -539,10 +540,10 @@ payload contract, out of scope). ## References -- `design-crowdb-kv-leader-election.md` §6 — heartbeat/lease interaction with +- `design-crowdb-paxos-kv-leader-election.md` §6 — heartbeat/lease interaction with stream ordering -- `design-crowdb-kv-slot.md` §5 — pipelined fanout and per-peer flow control -- `design-crowdb-kv-observability.md` — write-path phase metrics +- `design-crowdb-paxos-kv-slot.md` §5 — pipelined fanout and per-peer flow control +- `design-crowdb-paxos-kv-observability.md` — write-path phase metrics (`paxos.propose.e2e.l`, `paxos.classic.prepare.l`, `paxos.accept.quorum_rpc.l`, `paxos.learn.apply.l`) - `design-crowdb-rpc.md` §6 — Flatbuffer Wrapper Convention diff --git a/doc/design/kv/design-crowdb-kv-server.md b/doc/design/kv/design-crowdb-paxos-kv-server.md similarity index 89% rename from doc/design/kv/design-crowdb-kv-server.md rename to doc/design/kv/design-crowdb-paxos-kv-server.md index 8c50b9504..1a9dc07b0 100644 --- a/doc/design/kv/design-crowdb-kv-server.md +++ b/doc/design/kv/design-crowdb-paxos-kv-server.md @@ -3,8 +3,8 @@ # CROWDB - Design: KV Server -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §15.2 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §15.2 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §15.2 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §15.2 --- @@ -26,11 +26,11 @@ Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §15.2 ## 1. Overview -`crowdb-kv-server` is a thin binary that wires the `crowdb-kv` library into a +`crowdb-paxos-kv-server` is a thin binary that wires the `crowdb-paxos-kv` library into a runnable process. It adds CLI argument parsing, an HTTP management server for runtime topology control, and graceful startup/shutdown orchestration. The server does **not** contain business logic — all -consensus, KV operations, and replication are handled by the `crowdb-kv` +consensus, KV operations, and replication are handled by the `crowdb-paxos-kv` library. ## 2. Design Decisions @@ -64,12 +64,12 @@ file-backend deployments continue to specify `file` explicitly on restart. The management API starts **before** stores so the server is observable even if store creation fails. `--root <dir>` is required on every -start; it derives the four node paths via `CrowDBConfig::apply_root`: +start; it derives the four node paths via `PaxosKvConfig::apply_root`: `wal_root = <root>/waldata`, `config_root = <root>/conf`, `data_root = <root>/ctdata`, `log_dir = <root>/log` (fixed subfolder names, the only supported layout). `--config <toml>` is optional and supplies startup tunables in both first-boot and restore modes; when omitted, -tunables come from `CrowDBConfig::default()`. Explicit CLI settings override +tunables come from `PaxosKvConfig::default()`. Explicit CLI settings override the file, as defined by the shared [`service configuration design`](../config/design-crowdb-config.md) §3. The config-file watcher runs only when `--config` is passed. Static settings such @@ -114,7 +114,7 @@ persisted local identity fails creation; it is never overwritten by a retry. ### 2.3 Concurrency model -`KvStoreRegistry` and each `PxKvStore` publish immutable store/group maps +`PaxosKvStoreRegistry` and each `PxKvStore` publish immutable store/group maps through `ArcSwap`. Readers clone an `Arc` from one snapshot without taking a shard lock. Replacement uses compare-and-swap, so a reader sees the old or new group and never a missing intermediate entry; successful replacement cancels @@ -139,11 +139,11 @@ Key endpoint groups: - **System group** — `POST /system/init` bootstraps store 0 + group 0 on this node. (`POST /topology/finalize` and `GET /topology/ready` are removed. Persistent topology-record management moves to - `crowdb-kv-client`'s `KVClusterMetaClient` / `HardwareClient`. See - [`design-crowdb-kv-group0.md`](design-crowdb-kv-group0.md) for the + `crowdb-paxos-kv-client`'s `PaxosKvClusterMetaClient` / `HardwareClient`. See + [`design-crowdb-paxos-kv-group0.md`](design-crowdb-paxos-kv-group0.md) for the group-0 sysdata architecture.) The lifecycle endpoints (`add_store`, - `add_group`) are now **internal**; only `crowdb-kv-client`'s - `KVClusterAdmin` calls them. See `../console/design-crowdb-console.md` §4.3 for the + `add_group`) are now **internal**; only `crowdb-paxos-kv-client`'s + `PaxosKvClusterAdmin` calls them. See `../console/design-crowdb-console.md` §4.3 for the full persistent cluster config design. - **Admin operations** — `step-down` (force leader step-down), `join` (new-member snapshot join), `flush` (drain the local replica's L0 @@ -180,9 +180,13 @@ On SIGINT/SIGTERM, Axum stops accepting new HTTP requests, then 1. **`PxKvStore::shutdown`** — stops the crowdb-rpc server (cuts frontend load), then cascades into each group. 2. **`PxGroup::shutdown`** — cancels the tenure token and awaits the - election driver + maintenance loop, closes remote crowdb-rpc channels, - then cascades into the local replica. -3. **`PxLocalReplica::shutdown`** — calls `KVEngine::flush` (drain L0 + election driver, maintenance loop, FetchGap driver, and coalescer watchdog. + The watchdog keeps only a weak group reference. Each owned task + has a per-layer deadline; a timeout aborts and then joins the task before + the shutdown report is returned. It then closes remote crowdb-rpc + channels and cascades into the local replica. +3. **`PxLocalReplica::shutdown`** — cancels and joins the background apply + loop before calling `KVEngine::flush` (drain L0 memtable into L1 B+tree, in-memory) then `KVEngine::persist_snapshot` (write dirty L1 pages + superblock to the page store, durable). For `InMemKV` both are no-ops. Acceptor diff --git a/doc/design/kv/design-crowdb-kv-slot.md b/doc/design/kv/design-crowdb-paxos-kv-slot.md similarity index 94% rename from doc/design/kv/design-crowdb-kv-slot.md rename to doc/design/kv/design-crowdb-paxos-kv-slot.md index 29b010a89..9aea48d49 100644 --- a/doc/design/kv/design-crowdb-kv-slot.md +++ b/doc/design/kv/design-crowdb-paxos-kv-slot.md @@ -3,8 +3,8 @@ # CROWDB - Design: Slots — Parallel Pipelining & Acceptor Retention -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §5.1, §6.5, §7.3 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §5.1, §6.5, §7.3 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §5.1, §6.5, §7.3 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §5.1, §6.5, §7.3 This document covers two aspects of CROWDB's slot mechanism: @@ -51,14 +51,14 @@ CROWDB exploits this by running many slots in parallel on the leader. Throughput - **Gaps.** A slot may remain undecided long after later slots are decided. We need a mechanism to resolve gaps without stalling the hot path. - **Conservative cross-key reads.** A `Scan` must wait for a no-gap prefix; point reads do not. -The blind-ops premise from design-crowdb-kv.md §5.2](design-crowdb-kv.md) makes the trade-off cheap: out-of-order *apply* is safe because no operation reads before writing. +The blind-ops premise from design-crowdb-paxos-kv.md §5.2](design-crowdb-paxos-kv.md) makes the trade-off cheap: out-of-order *apply* is safe because no operation reads before writing. --- ## 2. Concepts and Invariants - **I1 — Single slot counter.** On a leader, slot assignment is performed by exactly one logical worker. Two writes never receive the same slot, and no two slots are assigned out of arrival order. -- **I2 — Slot determines linearization.** The slot number assigned to an op is the op's position in the global linearization order (design-crowdb-kv.md §6.1](design-crowdb-kv.md)). +- **I2 — Slot determines linearization.** The slot number assigned to an op is the op's position in the global linearization order (design-crowdb-paxos-kv.md §6.1](design-crowdb-paxos-kv.md)). - **I3 — Quorum-fsync before ack.** A client write is acked only after a quorum of acceptors (including the leader) have fsynced their `Accepted(slot, ballot, value)` records. This is the durability hook that makes I2 robust to failures. - **I4 — Apply-order independence for blind ops.** For any key *k*, the engine's final value is `value(max{ slot | slot writes k })`. Apply order between non-overlapping keys is irrelevant; for the same key, the higher slot wins regardless of arrival order ([§13](#13-correctness-analysis-for-parallel-slot-writes)). - **I5 — Per-key resolved-slot is monotone.** A learner's per-key tracker only ever advances. This is the basis for read-your-writes from followers. @@ -76,7 +76,12 @@ The blind-ops premise from design-crowdb-kv.md §5.2](design-crowdb-kv.md) makes | `Applied` | learner.apply() returned | terminal | Used by metrics; safe to evict | | `OrphanRepair` | leader changed before reaching `Chosen`; repair task takes over | `Chosen` (via repair) | New leader's responsibility | -**When does the ack happen?** When the slot reaches `Chosen` *and* the leader's own learner has applied it. The leader's own learner application is required so that an immediately-following `Get` on the leader sees the new value (see [§6.1 of design-crowdb-kv.md](design-crowdb-kv.md#61-linearizable-leader-read)). +**When does the ack happen?** The synchronous path acknowledges after local +application. With `async_engine_apply`, reaching `Chosen` publishes the chosen +frontier and request result, starts an owned apply task, and acknowledges +immediately. Linearizable reads wait for their chosen-slot apply fence before +reading the engine, preserving read-your-writes (see +[§6.1 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#61-linearizable-leader-read)). **Eviction.** Acceptor entries and WAL segments become eligible below the snapshot-backed reclamation watermark in §11. Local application alone does not authorize eviction. @@ -84,15 +89,22 @@ The blind-ops premise from design-crowdb-kv.md §5.2](design-crowdb-kv.md) makes ## 4. Sliding Window and Backpressure -The number of in-flight slots is capped at the **window size** (`max_inflight_proposals`, default 32). The proposer uses an `InflightAdmission` gate backed by one or more `Semaphore`s. If a permit is available, the request is admitted, slot is assigned, and proposing begins. The permit is held for the entire proposal duration. +The number of in-flight slots is capped at the **window size** +(`max_inflight_proposals`, default 32). `InflightAdmission` owns one semaphore. +A permit is acquired before engine admission and slot allocation. For an engine +that returns an admitted apply owner, the permit transfers with that owner to +the chosen apply task and remains occupied through apply completion and the +applied-frontier update, including after an asynchronous `Chosen` response. +Early proposal exit returns the permit; cancellation of a chosen wait does not +cancel its owned native apply. Engines using compatibility admission retain the +permit for the proposal call's duration. Replay and gap repair use their +independent progress admission, so a full proposal window cannot block repair. **Admission policy** (`reject_on_inflight_window_full: bool`, internal config, not exposed via CLI): - **`false` (default)** — If no permit is available, the caller blocks on `acquire().await` until a permit is freed. This eliminates client-side reject-retry storms under contention (e.g., window=1 with 16 writers). Wait time and queue depth are tracked as metrics (`inflight_total_enqueued`, `inflight_total_wait_us`, `inflight_queue_depth`). - **`true`** — If no permit is available (`try_acquire` fails), the leader immediately returns `Busy` — a retryable error. No queuing. Used only in tests that need to verify fail-fast behavior. -**Multi-queue routing** (`--inflight-queues`, default 1): The window is split across N semaphores, each sized `ceil(max_inflight / N)`. Proposals are routed round-robin. Multiple queues reduce semaphore contention under high concurrency without affecting correctness (each slot is an independent Paxos instance; admission ordering does not influence consensus safety). - --- ## 5. Pipelined Fanout @@ -120,7 +132,7 @@ budget applies backoff without invalidating the healthy connection. Each learner maintains, per key it has applied, the highest slot that has touched that key. This enables: -- **Read-your-writes** ([§6.2 of design-crowdb-kv.md](design-crowdb-kv.md#62-read-your-writes-follower-read)): a follower can serve `Get(k, slot=N)` as soon as `resolved_slot[k] ≥ N`. +- **Read-your-writes** ([§6.2 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#62-read-your-writes-follower-read)): a follower can serve `Get(k, slot=N)` as soon as `resolved_slot[k] ≥ N`. - **Linearizability** ([§14](#14-parallel-slot-linearizability-analysis)): per-key tracking makes "highest slot wins" correct under out-of-order apply. **Update rule.** On `apply(slot, batch)`: for each `(k, op, v?)` in the batch, if `slot > resolved_slot[k]`, update `resolved_slot[k] = slot` and write/tombstone `v`. Otherwise drop. Then advance `max_applied` and the contiguous-applied frontier. @@ -141,7 +153,7 @@ A peer that has never reported is treated as `0`, so the safe-slot only rises on **Propagation.** The leader includes `safe_slot` in heartbeats, write responses, read responses, and the describe-cluster RPC. -`Scan(Linearizable)` uses the leader's *own* contiguous frontier rather than the safe-slot. It is strictly ≥ safe-slot at all times ([§6.4 of design-crowdb-kv.md](design-crowdb-kv.md#64-scan-modes)). +`Scan(Linearizable)` uses the leader's *own* contiguous frontier rather than the safe-slot. It is strictly ≥ safe-slot at all times ([§6.4 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#64-scan-modes)). --- @@ -152,7 +164,7 @@ A "gap" is a slot N < max-chosen-slot for which the leader has no `Chosen` decis **Repair mechanism:** 1. **Opportunistic repair.** After each heartbeat round in the leader-state loop, `repair_once()` is called. It finds the lowest gap (`contiguous_chosen + 1`) and runs classic Paxos (Phase 1 + Phase 2) to close it. A no-gap leader returns immediately without any RPCs. -2. **On leader change:** the new leader runs bulk Phase 1 over `[contiguous_chosen+1, ceiling]` as a one-shot sweep (see [`design-crowdb-kv-leader-election.md`](design-crowdb-kv-leader-election.md) §4). +2. **On leader change:** the new leader runs bulk Phase 1 over `[contiguous_chosen+1, ceiling]` as a one-shot sweep (see [`design-crowdb-paxos-kv-leader-election.md`](design-crowdb-paxos-kv-leader-election.md) §4). There is no separate repair task with its own tick or concurrency cap; repair is interleaved with heartbeats at the heartbeat cadence. @@ -331,7 +343,7 @@ A new group or leader tenure clears progress evidence. Membership replacement also clears evidence, including when it adds a non-voting learner. A joining replica that needs a prefix already reclaimed uses snapshot catch-up. -See [WAL garbage collection](design-crowdb-kv-wal.md#7-garbage-collection). +See [WAL garbage collection](design-crowdb-paxos-kv-wal.md#7-garbage-collection). --- @@ -616,7 +628,7 @@ again. the drain at high load so the `max_keys` overflow path produces full batches. -CLI: `--coalesce-max-keys` on `crowdb-kv-server`, applied in `main.rs` +CLI: `--coalesce-max-keys` on `crowdb-paxos-kv-server`, applied in `main.rs` into `config.paxos`. Wired into the group via `set_from_config` (the coalescer reads `self.config.paxos.*`). diff --git a/doc/design/kv/design-crowdb-kv-state-machine.md b/doc/design/kv/design-crowdb-paxos-kv-state-machine.md similarity index 92% rename from doc/design/kv/design-crowdb-kv-state-machine.md rename to doc/design/kv/design-crowdb-paxos-kv-state-machine.md index 84e1d3c39..7900a9d04 100644 --- a/doc/design/kv/design-crowdb-kv-state-machine.md +++ b/doc/design/kv/design-crowdb-paxos-kv-state-machine.md @@ -3,8 +3,8 @@ # CROWDB - Design: State Machine -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §5.2, §8.3, §8.4 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §8.3, §8.4, §14.1 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §5.2, §8.3, §8.4 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §8.3, §8.4, §14.1 This document specifies the storage engine abstraction used by CROWDB learners. The engine is the **only** consumer of consensus output; it owns the materialized key-value state and serves all reads. The WAL is the durable log; the engine is the materialized projection. @@ -78,7 +78,7 @@ queried through `KVEngine::resume_from_slot()`. This replaces the former `DurableCommitWatermark` WAL record design. **Restart behavior** (`PxLocalReplica::restore_from_replay_with_engine`, -`lib/crowdb-kv/src/cluster/local_replica.rs`): +`lib/crowdb-paxos-kv/src/cluster/local_replica.rs`): 1. WAL replay rebuilds acceptor state (`Promised`, `Accepted`, `VoteGranted`), Pass 1, unconditional. 2. The engine reports `resume_from = resume_from_slot()`: the highest slot it @@ -92,7 +92,7 @@ queried through `KVEngine::resume_from_slot()`. This replaces the former re-`learn()` of the already-durable prefix. 4. Slots above the local WAL's own highest accepted slot are re-learned via new-leader recovery (bulk Phase 1) or steady-state heartbeat catch-up (§6 - of this doc's parent flows, `design-crowdb-kv-slot.md`). + of this doc's parent flows, `design-crowdb-paxos-kv-slot.md`). Step 3's skip is a pure **optimization**, not a correctness requirement: `KVEngine::apply` is idempotent (highest-slot-wins per key) and @@ -100,7 +100,7 @@ Step 3's skip is a pure **optimization**, not a correctness requirement: local WAL through `learn()` (`resume_from = 0`, `InMemKV`'s permanent case) always converges to the same correct state. It's just extra, safe, no-op work. `persist_snapshot()` (called periodically by the group's maintenance -loop, `lib/crowdb-kv/src/cluster/group_maintenance.rs::run_pass`) is what advances +loop, `lib/crowdb-paxos-kv/src/cluster/group_maintenance.rs::run_pass`) is what advances the durable floor `resume_from_slot()` will report on the *next* restart. --- @@ -131,11 +131,11 @@ can reflect different times and partial later batches. This does not establish one linearization point for the whole scan. Stable exports use the separate covered-prefix snapshot interface. -If true historical snapshots are ever required, MVCC is a future extension. The single-version restriction comes from design-crowdb-kv.md §1 / §5.2](design-crowdb-kv.md). +If true historical snapshots are ever required, MVCC is a future extension. The single-version restriction comes from design-crowdb-paxos-kv.md §1 / §5.2](design-crowdb-paxos-kv.md). ### 3.3 Resolved-slot is monotone per key -The engine never accepts a write at slot `s` for key `k` if `s ≤ resolved_slot(k)`. This is the runtime expression of [Invariant I5 in `design-crowdb-kv-slot.md`](design-crowdb-kv-slot.md#2-concepts-and-invariants). +The engine never accepts a write at slot `s` for key `k` if `s ≤ resolved_slot(k)`. This is the runtime expression of [Invariant I5 in `design-crowdb-paxos-kv-slot.md`](design-crowdb-paxos-kv-slot.md#2-concepts-and-invariants). Implication: replays and out-of-order applies are naturally idempotent. If WAL replay tries to apply slot 7 for key `k` and `resolved_slot(k)` is already 9, the apply is a no-op for `k`, consistent with the parallel-slot semantics. @@ -169,7 +169,7 @@ In-memory engines can hold a write lock for the duration of the batch. File-base ### 4.4 Intra-batch order -For a key `k` appearing multiple times in a batch (rare but legal; see [`design-crowdb-kv-slot.md` §13](design-crowdb-kv-slot.md#13-correctness-analysis-for-parallel-slot-writes)), the *last* occurrence in batch order wins. The earlier ones are folded into the apply procedure naturally (each tuple in turn updates `current`; the loop's final state is what persists). +For a key `k` appearing multiple times in a batch (rare but legal; see [`design-crowdb-paxos-kv-slot.md` §13](design-crowdb-paxos-kv-slot.md#13-correctness-analysis-for-parallel-slot-writes)), the *last* occurrence in batch order wins. The earlier ones are folded into the apply procedure naturally (each tuple in turn updates `current`; the loop's final state is what persists). ### 4.5 Failure during apply @@ -177,7 +177,7 @@ For a key `k` appearing multiple times in a batch (rare but legal; see [`design- ### 4.6 Zero-copy batch decode -`Batch::decode` extracts keys and values from the Paxos payload using +`Batch::try_decode` validates the complete Paxos payload, then extracts keys and values using `Bytes::slice(range)`, which creates zero-copy views into the same allocation as `PxLogEntry.payload` (an O(1) ref-count bump). This eliminates the per-key and per-value `to_vec()` heap allocations that @@ -190,9 +190,16 @@ implementations use `Bytes::as_ref()` to obtain `&[u8]` for FFI or internal storage. `Cell` and `EngineDiff` remain `Vec<u8>` since they represent engine-internal storage, not the decode path. -Truncated payloads (where a key or value length exceeds the remaining -bytes) yield empty `Bytes` for that field, matching the previous -`unwrap_or(&[])` behavior. +Proposal admission and learner apply reject truncated fields, unknown operation +kinds, nonempty delete values, and bytes after the declared operation count. +Malformed chosen entries do not mutate the engine or advance the applied +frontier. They remain chosen and require recovery of valid durable data; an +invalid entry never becomes a partial batch or an implicit `NoOp`. + +An empty payload and a zero-operation payload represent empty batches. Their +chosen slots still pass through fallible gap completion. `Batch::decode` retains +its permissive field behavior only for explicit compatibility consumers; the +proposal and learner paths use `Batch::try_decode`. ### 4.7 Async engine apply and the apply fence @@ -292,7 +299,7 @@ This minimal surface keeps multi-engine compatibility easy. ### 6.1 Purpose -Snapshot install ([§8.4 of design-crowdb-kv.md](design-crowdb-kv.md#84-snapshot-and-install)) is the bootstrap path for new or far-lagging members. The engine provides export/import primitives that the snapshot module wraps in a chunked, resumable, throttled transfer. +Snapshot install ([§8.4 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#84-snapshot-and-install)) is the bootstrap path for new or far-lagging members. The engine provides export/import primitives that the snapshot module wraps in a chunked, resumable, throttled transfer. In addition to peer transfer, the engine uses snapshot export to persist its own state locally (see §2.1). The local snapshot file stores the KV state and `last_applied_slot`, enabling fast restart without re-applying the entire WAL. @@ -363,7 +370,7 @@ When a key `k` is overwritten with a higher slot value, the old value is immedia ## 8. Compare for Cross-Learner Validation -A `compare(other) -> diff` operation is required so that `crowdbbench` can verify state equality across learners after a test run (design-crowdb-kv.md §14.1](design-crowdb-kv.md)). +A `compare(other) -> diff` operation is required so that `crowdbbench` can verify state equality across learners after a test run (design-crowdb-paxos-kv.md §14.1](design-crowdb-paxos-kv.md)). ### 8.1 Semantics @@ -445,4 +452,4 @@ crowdb-tree portable format; cross-engine transfer is not supported. - Manual debugging or operations exercises → ordered file. - Production → crowdb-tree. -**Per-key memory cost considerations:** the per-key resolved-slot adds 8 bytes per live key. For 10⁹ live keys this is 8 GiB on every learner, accepted in the requirement ([§7.3.1](design-crowdb-kv.md)). If memory pressure dictates, a future optimization could compress recently-applied resolved-slots into a "below safe-slot" bit (a single bit replacing the 8 bytes when the per-key slot is no longer needed for read-your-writes). +**Per-key memory cost considerations:** the per-key resolved-slot adds 8 bytes per live key. For 10⁹ live keys this is 8 GiB on every learner, accepted in the requirement ([§7.3.1](design-crowdb-paxos-kv.md)). If memory pressure dictates, a future optimization could compress recently-applied resolved-slots into a "below safe-slot" bit (a single bit replacing the 8 bytes when the per-key slot is no longer needed for read-your-writes). diff --git a/doc/design/kv/design-crowdb-kv-test.md b/doc/design/kv/design-crowdb-paxos-kv-test.md similarity index 80% rename from doc/design/kv/design-crowdb-kv-test.md rename to doc/design/kv/design-crowdb-paxos-kv-test.md index 7a93ef557..0f919d789 100644 --- a/doc/design/kv/design-crowdb-kv-test.md +++ b/doc/design/kv/design-crowdb-paxos-kv-test.md @@ -3,8 +3,8 @@ # CROWDB - Design: Test Strategy -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §17 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §17 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §17 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §17 ## Table of Contents @@ -35,12 +35,12 @@ results live in [`kv-write-flow-analysis.md`](kv-write-flow-analysis.md). ## 2. Architecture Stack ``` -store (PxKvStore: many groups, one node identity, routing) <- lib/crowdb-kv/tests/store.rs - group (PxGroup: 1 local + N remote replicas, full Paxos) <- lib/crowdb-kv/tests/group.rs - replica (PxLocalReplica: acceptor + learner + WAL + slots) <- lib/crowdb-kv/tests/replica.rs - election (PxLocalReplica election state machine, pure logic) <- lib/crowdb-kv/tests/election.rs - wal (WalEngine: durable log) slot (PxSlotList) <- lib/crowdb-kv/tests/wal.rs, slot.rs - unit (pure modules: codec, classifier, kv engine, roles) <- lib/crowdb-kv/tests/{paxos,kv}.rs + wal/slot codec tests +store (PxKvStore: many groups, one node identity, routing) <- lib/crowdb-paxos-kv/tests/store.rs + group (PxGroup: 1 local + N remote replicas, full Paxos) <- lib/crowdb-paxos-kv/tests/group.rs + replica (PxLocalReplica: acceptor + learner + WAL + slots) <- lib/crowdb-paxos-kv/tests/replica.rs + election (PxLocalReplica election state machine, pure logic) <- lib/crowdb-paxos-kv/tests/election.rs + wal (WalEngine: durable log) slot (PxSlotList) <- lib/crowdb-paxos-kv/tests/wal.rs, slot.rs + unit (pure modules: codec, classifier, kv engine, roles) <- lib/crowdb-paxos-kv/tests/{paxos,kv}.rs + wal/slot codec tests deployment (release containers + host HTTP/RPC clients) <- e2e-test/tests/{kv,deployment}/* console (mgmt API server + CLI: Axum REST, CLI commands) <- app/crowdb-{web,cli}/tests/* ui e2e (host Playwright browser + release node containers) <- e2e-test/clients/console-ui/e2e/* @@ -50,17 +50,17 @@ ui e2e (host Playwright browser + release node containers) <- e2e-test/cli | Binary | Layer | Drives | | ------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| `lib/crowdb-kv/tests/paxos.rs` | unit | `acceptor`, `learner`, `error` classifier, `roles` in isolation | -| `lib/crowdb-kv/tests/kv.rs` | unit | `InMemKV` / `KVEngine` / `CrowdbTreeEngine` apply semantics | -| `lib/crowdb-kv/tests/election.rs` | unit | `PxLocalReplica` election state machine (role transitions, vote granting, heartbeat, lease, term fencing) | -| `lib/crowdb-kv/tests/wal.rs` | subsystem | `WalEngine` append/replay/gc/segment + record codec + backends | -| `lib/crowdb-kv/tests/slot.rs` | subsystem | `PxSlotList` / `PxSlotNode` | -| `lib/crowdb-kv/tests/replica.rs` | replica | single `PxLocalReplica` (no peers) | -| `lib/crowdb-kv/tests/group.rs` | group | `PxGroup` multi-node clusters (real loopback crowdb-rpc, no mocks) | -| `lib/crowdb-kv/tests/store.rs` | store | `PxKvStore` routing / lifecycle / status / health | -| `app/crowdb-kv-server/tests/*` | component | management handlers, validation and deterministic local state | +| `lib/crowdb-paxos-kv/tests/paxos.rs` | unit | `acceptor`, `learner`, `error` classifier, `roles` in isolation | +| `lib/crowdb-paxos-kv/tests/kv.rs` | unit | `InMemKV` / `KVEngine` / `CrowdbTreeEngine` apply semantics | +| `lib/crowdb-paxos-kv/tests/election.rs` | unit | `PxLocalReplica` election state machine (role transitions, vote granting, heartbeat, lease, term fencing) | +| `lib/crowdb-paxos-kv/tests/wal.rs` | subsystem | `WalEngine` append/replay/gc/segment + record codec + backends | +| `lib/crowdb-paxos-kv/tests/slot.rs` | subsystem | `PxSlotList` / `PxSlotNode` | +| `lib/crowdb-paxos-kv/tests/replica.rs` | replica | single `PxLocalReplica` (no peers) | +| `lib/crowdb-paxos-kv/tests/group.rs` | group | `PxGroup` multi-node clusters (real loopback crowdb-rpc, no mocks) | +| `lib/crowdb-paxos-kv/tests/store.rs` | store | `PxKvStore` routing / lifecycle / status / health | +| `app/crowdb-paxos-kv-server/tests/*` | component | management handlers, validation and deterministic local state | | `e2e-test/tests/kv/*`, `e2e-test/tests/deployment/*` | container E2E | packaged server startup, public management/RPC, persistence and recovery | -| `lib/crowdb-kv-client/tests/*` | client component | `CrowdbClient` retry, topology cache, `NotLeaderHint` follow, `AnyReplica` read distribution + fallback against embedded crowdb-rpc servers | +| `lib/crowdb-paxos-kv-client/tests/*` | client component | `CrowdbClient` retry, topology cache, `NotLeaderHint` follow, `AnyReplica` read distribution + fallback against embedded crowdb-rpc servers | | `app/crowdb-web/tests/*` | console mgmt API | Axum REST API server: node management, OpenAPI proxy, API forwarding | | `lib/crowdb-console-shared/tests/*`, `app/crowdb-cli/tests/*` | console mgmt API | shared core (config, API client, health aggregation) + CLI commands | | `app/crowdb-cli/tests/*` | CLI component | argument validation, protocol stubs and comparisons; real cluster lifecycle has a container owner | @@ -93,7 +93,7 @@ per boundary flow. Component tests require neither Docker nor server binaries. - Persistence round-trip: all above operations survive WAL replay + restart **Cluster verification rule:** every tier that creates a real `PxKvStore`, -`PxGroup`, or `crowdb-kv-server` process must verify two things before +`PxGroup`, or `crowdb-paxos-kv-server` process must verify two things before proceeding to tier-specific assertions: 1. **Leader election succeeded within a bounded timeout.** If leader @@ -210,17 +210,17 @@ lowest broken layer. Detailed per-layer coverage checklists live in - **Client E2E** — `CrowdbClient` against embedded `PxKvStore` crowdb-rpc servers: retry, topology cache refresh, `NotLeaderHint` follow, `AnyReplica` read distribution, `MinSlot` lagging-follower fallback. No - `crowdb-kv-server` binary is booted. -- **Deployment** — `crowdb-kv-server` binary + HTTP management API + + `crowdb-paxos-kv-server` binary is booted. +- **Deployment** — `crowdb-paxos-kv-server` binary + HTTP management API + multi-process clusters: boots the actual binary, exercises the HTTP API, verifies multi-process cluster formation and lifecycle. - **Console mgmt API** — Console management API server (Axum) + CLI frontend: REST endpoints, OpenAPI proxy, API forwarding to - crowdb-kv-server nodes, shared core (config, API client, health + crowdb-paxos-kv-server nodes, shared core (config, API client, health aggregation), CLI commands. KV correctness is delegated to the deployment or UI E2E layer. - **UI E2E** — Playwright browser tests against a real `crowdb-web` + - `crowdb-kv-server` backend: drives the SPA as an operator would (clicks, + `crowdb-paxos-kv-server` backend: drives the SPA as an operator would (clicks, context menus, dialogs, KV panel), verifies both UI feedback and backend state. No mocks; the only network interception is route aborting for the backend-unreachable test. @@ -240,8 +240,8 @@ from the Rust test binaries above. They run as `test-tree-ct` in CI. | C++ unit | `liblib/crowdb-tree/tests/unit` | Single component correctness: cell encoding, leaf page build/read, delta replay, consolidation triggers, mapping table, epoch manager, split point. | | C++ integration | `liblib/crowdb-tree/tests/integration` | Multi-component flows over `InMemoryPageStore` and `FilePageStore`: basic CRUD, batch apply, scan, split/merge, consistent view, snapshot roundtrip, GC. | | Crash/recovery | `liblib/crowdb-tree/tests/recovery` | Durability: snapshot + recover, torn-page, superblock A/B, double-apply idempotency. Uses a `FaultyPageStore` with fault points for the FI matrix. | -| Rust FFI | `crowdb-kv` `tests/` | `CrowdbTreeEngine` trait conformance (shared parametrized tests with `InMemKV`), async bridge, buffer ownership, error mapping. | -| Cross-engine parity | `crowdb-kv` `tests/` | `InMemKV` and `CrowdbTreeEngine` produce identical state via `compare()` after random op streams, snapshot export/import round-trips, and mid-stream restart. | +| Rust FFI | `crowdb-paxos-kv` `tests/` | `CrowdbTreeEngine` trait conformance (shared parametrized tests with `InMemKV`), async bridge, buffer ownership, error mapping. | +| Cross-engine parity | `crowdb-paxos-kv` `tests/` | `InMemKV` and `CrowdbTreeEngine` produce identical state via `compare()` after random op streams, snapshot export/import round-trips, and mid-stream restart. | | Concurrency | sanitizer CI | No data races / UAF under reader+writer load (TSan/ASan/UBSan); epoch reclamation under load; version pin + GC. | The authoritative correctness oracle is **`compare()` against `InMemKV`**: for @@ -264,7 +264,7 @@ Every feature or component milestone includes: 1. **Unit invariants** from the matching test layer (property-based or deterministic). -2. **Failure-injection** matching [`design-crowdb-kv.md`](design-crowdb-kv.md) §9 +2. **Failure-injection** matching [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §9 scenarios. 3. **Benchmark integration test** (end-to-end correctness + performance) once the RPC/client layer is reached. diff --git a/doc/design/kv/design-crowdb-kv-wal.md b/doc/design/kv/design-crowdb-paxos-kv-wal.md similarity index 96% rename from doc/design/kv/design-crowdb-kv-wal.md rename to doc/design/kv/design-crowdb-paxos-kv-wal.md index b49b67819..915ff2e54 100644 --- a/doc/design/kv/design-crowdb-kv-wal.md +++ b/doc/design/kv/design-crowdb-paxos-kv-wal.md @@ -3,8 +3,8 @@ # CROWDB - Design: Write-Ahead Log -Depends on: [`design-crowdb-kv.md`](design-crowdb-kv.md) §8.1, §8.2 -Satisfies: [`design-crowdb-kv.md`](design-crowdb-kv.md) §8.1, §8.2 +Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §8.1, §8.2 +Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §8.1, §8.2 This document specifies CROWDB's write-ahead log. **There is exactly one durable log per group: the replica's consensus log (the per-slot acceptor log).** @@ -288,7 +288,7 @@ logical/physical bytes written, and RMW counts for observability. The reads cumulative counters for the engine collector to compute per-window deltas. -The WAL bench (`lib/crowdb-kv/benches/wal.rs`) exercises four backends: `Mem` +The WAL bench (`lib/crowdb-paxos-kv/benches/wal.rs`) exercises four backends: `Mem` (in-memory `MemBlockDevice`), `File` (`tokio::fs`), `Uring` (buffered CQE-backed file I/O), and `Block` (`BlockDevice::new()` with `wal_skip_fsync: true`). The `Block` case hits all @@ -312,7 +312,7 @@ crowdb-tree side for this refactor. > An `Accepted` response is sent to the leader only after the corresponding WAL record's backend durable flush has completed. > A client write is acked only after a quorum of acceptors have sent `Accepted`. -This is repeated from design-crowdb-kv.md §8.1](design-crowdb-kv.md) because everything else in this section is a consequence of it. This is the **strict** mode. The **early-ack** mode (§5.4) relaxes the leader's own durability ordering but keeps the quorum-durability requirement; both modes are described in §5.4. +This is repeated from design-crowdb-paxos-kv.md §8.1](design-crowdb-paxos-kv.md) because everything else in this section is a consequence of it. This is the **strict** mode. The **early-ack** mode (§5.4) relaxes the leader's own durability ordering but keeps the quorum-durability requirement; both modes are described in §5.4. ### 5.2 Failure cases @@ -360,7 +360,7 @@ The split matters: replay/restore are purely *local* (this node's WAL), but `Acc - New-leader recovery / steady-state heartbeat catch-up re-learning the value from quorum-confirmed consensus (§6.4 / §6.5). -**No `DurableCommitWatermark` WAL record.** The state machine owns its own persistence boundary, but that boundary is the *engine's* durability (crowdb-tree's on-disk snapshot), not a separate state-machine-level snapshot file; see §6.2 below and `design-crowdb-kv-state-machine.md §2.1`. The WAL does not duplicate this information. +**No `DurableCommitWatermark` WAL record.** The state machine owns its own persistence boundary, but that boundary is the *engine's* durability (crowdb-tree's on-disk snapshot), not a separate state-machine-level snapshot file; see §6.2 below and `design-crowdb-paxos-kv-state-machine.md §2.1`. The WAL does not duplicate this information. ### 6.1 Replay — rebuild acceptor state (`replay_group`) @@ -395,7 +395,7 @@ Pass 1 rebuilds acceptor state without assuming any local accept was ever chosen The learner / `KVEngine` is rebuilt in a second pass over the *same* replayed WAL records (`PxLocalReplica::restore_from_replay_with_engine`), not from a separate state-machine snapshot file; see -[`design-crowdb-kv-state-machine.md §2.1`](design-crowdb-kv-state-machine.md#21-state-machine-restart-engine-durability--wal-replay) +[`design-crowdb-paxos-kv-state-machine.md §2.1`](design-crowdb-paxos-kv-state-machine.md#21-state-machine-restart-engine-durability--wal-replay) for the full mechanics. In short: - Pass 2 walks every slot with an accepted entry (rebuilt in Pass 1 above) @@ -451,7 +451,7 @@ On winning election a leader sweeps slots `(floor, ceiling]`: is **re-confirmed, not overwritten**. This repairs a leader that restored an incomplete or merely-accepted prefix. -(Steady-state gap repair below the frontier uses the same adopt-from-quorum logic, one slot at a time; see [`design-crowdb-kv-slot.md`](design-crowdb-kv-slot.md).) +(Steady-state gap repair below the frontier uses the same adopt-from-quorum logic, one slot at a time; see [`design-crowdb-paxos-kv-slot.md`](design-crowdb-paxos-kv-slot.md).) Parallel slot assignment means holes are normal after crash / restart. Recovery resolves every slot in `(floor, ceiling]` independently: @@ -517,7 +517,7 @@ term and known voting leader. A missing certificate or local snapshot gives zero Only records with `slot < gc_slot` are eligible. The same boundary trims in-memory acceptor history; retained readers keep their owners alive. The full -contract is in [slot retention](design-crowdb-kv-slot.md#11-interaction-with-snapshot-and-wal-gc). +contract is in [slot retention](design-crowdb-paxos-kv-slot.md#11-interaction-with-snapshot-and-wal-gc). ### 7.2 GC granularity @@ -557,9 +557,9 @@ When the backend durable-flush operation returns an error, the OS reports the di 1. Stop using the disk for new writes (the `failed` AtomicBool is set; all subsequent `append` calls return `Err`). 2. Mark the group affected (a multi-disk WAL with one disk lost has incomplete state for slots that landed on the lost disk). 3. **Fail the node out of the group** (not yet implemented): the node would send a step-out RPC to the leader, the leader records the failed acceptor as not-eligible, and (if necessary) triggers a reconfiguration to maintain quorum. -4. Rebuild from peers via **snapshot install** ([§8.4 of design-crowdb-kv.md](design-crowdb-kv.md#84-snapshot-and-install)). +4. Rebuild from peers via **snapshot install** ([§8.4 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#84-snapshot-and-install)). -This is the **fail-out semantics** decided in design-crowdb-kv.md §8.1](design-crowdb-kv.md). We do not try to keep operating with partial WAL state on the surviving disks; that path requires per-slot replication across disks and adds disproportionate complexity. +This is the **fail-out semantics** decided in design-crowdb-paxos-kv.md §8.1](design-crowdb-paxos-kv.md). We do not try to keep operating with partial WAL state on the surviving disks; that path requires per-slot replication across disks and adds disproportionate complexity. ### 8.2 All-disk failure diff --git a/doc/design/kv/design-crowdb-kv-watch-notify.md b/doc/design/kv/design-crowdb-paxos-kv-watch-notify.md similarity index 96% rename from doc/design/kv/design-crowdb-kv-watch-notify.md rename to doc/design/kv/design-crowdb-paxos-kv-watch-notify.md index 0ef9f8fef..6d8cc1c4e 100644 --- a/doc/design/kv/design-crowdb-kv-watch-notify.md +++ b/doc/design/kv/design-crowdb-paxos-kv-watch-notify.md @@ -1,12 +1,12 @@ <!-- Copyright 2026-present Gian <crow.db@outlook.com> --> <!-- Licensed under the Apache License, Version 2.0. --> -# CROWDB - Design: crowdb-kv Watch/Notify Extension +# CROWDB - Design: crowdb-paxos-kv Watch/Notify Extension -Depends on: [`design-crowdb-kv-rpc.md`](design-crowdb-kv-rpc.md) §3, §5, §6; [`design-crowdb-kv.md`](design-crowdb-kv.md) §3; [`design-crowdb-kv-state-machine.md`](design-crowdb-kv-state-machine.md) §2 +Depends on: [`design-crowdb-paxos-kv-rpc.md`](design-crowdb-paxos-kv-rpc.md) §3, §5, §6; [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3; [`design-crowdb-paxos-kv-state-machine.md`](design-crowdb-paxos-kv-state-machine.md) §2 Satisfies: [`design-crowdb-diskdb.md`](../diskdb/design-crowdb-diskdb.md) §8 (Follow-up — group-0 notify/watch), §15 -This document covers the crowdb-kv watch/notify extension, the diskdb +This document covers the crowdb-paxos-kv watch/notify extension, the diskdb server-side notify handler, and the client endpoint cache proactive refresh that together replace fixed-interval polling as the primary change-detection mechanism for group-0 sysdata. The notify is a @@ -16,7 +16,7 @@ notifies over that stream. No separate notify-endpoint registration is needed for notify delivery. Polling stays as a safety net at a raised interval. Architecture decisions and rationale live here; the root design context is `design-crowdb-diskdb.md` §8 and -`design-crowdb-kv-rpc.md` §3. +`design-crowdb-paxos-kv-rpc.md` §3. ## Table of Contents @@ -56,7 +56,7 @@ design context is `design-crowdb-diskdb.md` §8 and ### 1.1 Why -crowdb-kv has no watch/notify capability without this extension. The only +crowdb-paxos-kv has no watch/notify capability without this extension. The only change-detection mechanism available to external components is polling (prefix scans of group 0). A push mechanism requires a long-lived stream from the group-0 leader to each watcher, fired when a watched @@ -66,7 +66,7 @@ handlers for notify and error frames on that connection. ### 1.2 Schema -`lib/crowdb-kv/src/rpc/fbs/kv_client.fbs` carries the watch/notify messages +`lib/crowdb-paxos-kv/src/rpc/fbs/kv_client.fbs` carries the watch/notify messages and one bidi RPC on `KvService`: ```fbs @@ -127,7 +127,7 @@ Add to the `KvService` service: ``` - **Field numbers are append-only** — matches the RPC design's - compatibility rule (`design-crowdb-kv-rpc.md` §5). + compatibility rule (`design-crowdb-paxos-kv-rpc.md` §5). - **`bytes` fields** — `prefix` and `keys` are raw key bytes as stored in the engine. For group-0 sysdata these are UTF-8 text paths (e.g. `/hw/disk/1/100/5/abcd`) because `HardwareClient` puts keys via @@ -147,7 +147,7 @@ Add to the `KvService` service: ### 1.3 Server-side handler -`KvRpcService` registers fire-and-forget handlers for `FBWatchSubscribe` and +`PaxosKvRpcService` registers fire-and-forget handlers for `FBWatchSubscribe` and `FBWatchUnsubscribe`. Subscribe resolves the group, checks leadership, and registers the connection plus prefix in `WatchRegistry`. A non-leader sends `FBWatchNotifyError` with the current leader hint. Apply-path notifications @@ -195,7 +195,7 @@ point with no cross-struct lookup. ### 2.2 WatchRegistry -`lib/crowdb-kv/src/cluster/watch_registry.rs`: +`lib/crowdb-paxos-kv/src/cluster/watch_registry.rs`: ```rust /// One watcher: a prefix + an outbound channel to push notify frames. @@ -286,7 +286,7 @@ impl WatchRegistry { ### 2.3 PxLearner integration -`PxLearner` (`lib/crowdb-kv/src/paxos/learner.rs`) holds: +`PxLearner` (`lib/crowdb-paxos-kv/src/paxos/learner.rs`) holds: ```rust /// Optional per-group watch registry. Set when the group is @@ -343,7 +343,7 @@ if let Some((group_id, registry)) = self.watch_registry.read().unwrap().as_ref() ### 2.4 PxGroup integration -`PxGroup` (`lib/crowdb-kv/src/cluster/group.rs`) holds: +`PxGroup` (`lib/crowdb-paxos-kv/src/cluster/group.rs`) holds: ```rust pub(crate) watch_registry: Arc<WatchRegistry>, @@ -401,10 +401,9 @@ watcher wakeup + re-read amplification under burst load. The watch/notify extension ships without coalescing. `apply_entry` calls `registry.emit` directly (one notify per changed key per matching -prefix). Coalescing is deferred to a follow-up requirement (see -`doc/backlog/backlog.md` — watch/notify coalescing / debounce). The -safety-net poller covers any notify-drop scenarios; the missing -coalescing is a load optimization, not a correctness gap. +prefix). Coalescing remains deferred. The safety-net poller covers any +notify-drop scenarios; the missing coalescing is a load optimization, not a +correctness gap. ## 4. WatchNotify Client @@ -417,7 +416,7 @@ and delivers notify frames via a channel: the client-side mirror of ### 4.2 WatchNotifyClient -`lib/crowdb-kv-client/src/watch_notify.rs`: +`lib/crowdb-paxos-kv-client/src/watch_notify.rs`: ```rust /// A live watch subscription. Dropping it unsubscribes and closes the @@ -434,7 +433,7 @@ impl Drop for WatchSubscription { fn drop(&mut self) { /* send WatchUnsubscribe, or drop the stream */ } } -/// Client for the crowdb-kv `WatchNotify` bidi stream. +/// Client for the crowdb-paxos-kv `WatchNotify` bidi stream. pub struct WatchNotifyClient { kv: Arc<CrowdbClient>, } @@ -741,7 +740,7 @@ pub watch_notify_enabled: bool, ## 8. Module Structure ``` -lib/crowdb-kv/src/ +lib/crowdb-paxos-kv/src/ rpc/ fbs/kv_client.fbs # WatchNotify messages + RPC kv_service.rs # watch_notify handler @@ -754,7 +753,7 @@ lib/crowdb-kv/src/ paxos/ learner.rs # watch_registry RwLock<Option> + set_* + apply-path hook -lib/crowdb-kv-client/src/ +lib/crowdb-paxos-kv-client/src/ watch_notify.rs # WatchNotifyClient, WatchSubscription lib.rs # export watch_notify @@ -793,7 +792,7 @@ lib/crowdb-diskdb-client/src/ ### Unit tests (UT) -**WatchRegistry** (`lib/crowdb-kv/src/cluster/watch_registry.rs`): +**WatchRegistry** (`lib/crowdb-paxos-kv/src/cluster/watch_registry.rs`): - `subscribe_emit` — register a watcher for prefix `/hw/disk/`, emit a changed key `/hw/disk/1/2/3/abcd`, assert the watcher channel receives a `WatchNotify` with the key. Assert a key outside the @@ -815,7 +814,7 @@ lib/crowdb-diskdb-client/src/ matching the prefix; assert the watcher receives a notify with the key. -**WatchNotify server handler** (`lib/crowdb-kv/src/rpc/kv_service.rs`): +**WatchNotify server handler** (`lib/crowdb-paxos-kv/src/rpc/kv_service.rs`): - `subscribe_not_leader` — open a stream to a non-leader node, send `WatchSubscribe` for a group it does not lead; assert `WatchNotifyError { not_leader_hint }` is received; stream stays @@ -878,7 +877,7 @@ lib/crowdb-diskdb-client/src/ `allocate_blocks` to a moved instance fails once, triggers `refresh_endpoints` (on-demand safety net), retries successfully. -**Integration tests** (`lib/crowdb-kv/tests/group_test/watch_notify_test.rs`): +**Integration tests** (`lib/crowdb-paxos-kv/tests/group_test/watch_notify_test.rs`): - `watch_notify_put_receives_key_and_value` — put a key under the watched prefix, assert the notify arrives with the correct key and value. diff --git a/doc/design/kv/design-crowdb-kv.md b/doc/design/kv/design-crowdb-paxos-kv.md similarity index 92% rename from doc/design/kv/design-crowdb-kv.md rename to doc/design/kv/design-crowdb-paxos-kv.md index e65f50eaf..ba65f6a0b 100644 --- a/doc/design/kv/design-crowdb-kv.md +++ b/doc/design/kv/design-crowdb-paxos-kv.md @@ -72,7 +72,7 @@ the leader before their resulting ordinary batch enters Paxos. - **No client-side transactions.** Each request is independent and idempotent. - **No authn/authz.** Trusted-network assumption. Consumers needing - auth should wrap `crowdb-kv` in a server that adds it. + auth should wrap `crowdb-paxos-kv` in a server that adds it. - **No full Jepsen-style linearizability checking.** Testing verifies same-state comparison and controlled-order verification. @@ -103,23 +103,23 @@ mechanism that protects user data. No external coordinator needed. **Two-phase bootstrap:** Phase 1 uses console TOML as the topology source of truth (operator-managed, existing behavior). Phase 2 cuts over to group 0 authoritative by writing hardware hierarchy via -`HardwareClient` and KV-cluster topology via `KVClusterMetaClient` -(text-path keys, JSON values) in `crowdb-kv-client`. No readiness flag +`HardwareClient` and KV-cluster topology via `PaxosKvClusterMetaClient` +(text-path keys, JSON values) in `crowdb-paxos-kv-client`. No readiness flag is needed — diskdb's sync loop treats an empty group 0 as "nothing assigned yet" and retries. Console restart uses a two-way fallback: group 0 missing → TOML mode; group 0 exists → group 0 authoritative. -See [`design-crowdb-kv-group0.md`](design-crowdb-kv-group0.md) for the +See [`design-crowdb-paxos-kv-group0.md`](design-crowdb-paxos-kv-group0.md) for the group-0 sysdata architecture. Group 0 membership evolves using the shipped Model B reconfiguration (direct HTTP mutation + `membership_epoch` fence). No new consensus primitive required. -Full design: [`design-crowdb-kv-group0.md`](design-crowdb-kv-group0.md) (sysdata schema, service registry, keep-alive, monitoring models), `../console/design-crowdb-console.md` §4.3, `design-crowdb-kv-server.md` §2.2/§2.4. +Full design: [`design-crowdb-paxos-kv-group0.md`](design-crowdb-paxos-kv-group0.md) (sysdata schema, service registry, keep-alive, monitoring models), `../console/design-crowdb-console.md` §4.3, `design-crowdb-paxos-kv-server.md` §2.2/§2.4. ### 3.4 Explicit group_id on every RPC -`crowdb-kv` performs no key-to-group mapping. An application built on top +`crowdb-paxos-kv` performs no key-to-group mapping. An application built on top is free to implement `hash(key) % num_groups` or any other policy. That policy is layered on top of, not inside, the core library. @@ -189,7 +189,7 @@ single-field overrides without rebuilding the whole struct. The └─────────┘ ``` -- **KvStore** — KV-facing runtime on one physical node (`crowdb-kv-server` +- **KvStore** — KV-facing runtime on one physical node (`crowdb-paxos-kv-server` process). Hosts multiple `PxGroup`s; routes by explicit `group_id`. - **PxGroup** — independent Paxos ensemble. Contains one `PxLocalReplica` (acceptor + learner) and N−1 `PxRemoteReplica` @@ -266,8 +266,8 @@ returns smaller keys. With no reverse continuation, `end_key` is the exclusive upper bound, or the prefix successor supplies that bound when `end_key` is empty. -Full read-flow details: `design-crowdb-kv-leader-election.md`, -`design-crowdb-kv-state-machine.md`. +Full read-flow details: `design-crowdb-paxos-kv-leader-election.md`, +`design-crowdb-paxos-kv-state-machine.md`. ## 7. Consensus @@ -283,9 +283,9 @@ Full read-flow details: `design-crowdb-kv-leader-election.md`, - **Partition behavior** — minority partition rejects all requests; majority continues serving. -Full design: `design-crowdb-kv-slot.md` (parallel slots, gap repair, -correctness proof), `design-crowdb-kv-leader-election.md` (election, lease, -ReadIndex), `design-crowdb-kv-rpc.md` (wire protocol and peer transport). +Full design: `design-crowdb-paxos-kv-slot.md` (parallel slots, gap repair, +correctness proof), `design-crowdb-paxos-kv-leader-election.md` (election, lease, +ReadIndex), `design-crowdb-paxos-kv-rpc.md` (wire protocol and peer transport). ## 8. Storage and Durability @@ -301,12 +301,12 @@ ReadIndex), `design-crowdb-kv-rpc.md` (wire protocol and peer transport). Resumable, checksum-verified, throttleable install for new-node bootstrap. -Full design: `design-crowdb-kv-wal.md`, `design-crowdb-kv-state-machine.md`, +Full design: `design-crowdb-paxos-kv-wal.md`, `design-crowdb-paxos-kv-state-machine.md`, `../tree/design-crowdb-tree.md`. ## 9. Cluster Lifecycle -- **Bootstrap** — operator starts each `crowdb-kv-server`, deploys via +- **Bootstrap** — operator starts each `crowdb-paxos-kv-server`, deploys via console, then calls `POST /api/cluster/init` to create the system group (store 0, group 0). Data store/group creation is blocked (`409`) until the cluster is initialized. Each node persists its @@ -332,11 +332,11 @@ Full design: `design-crowdb-kv-wal.md`, `design-crowdb-kv-state-machine.md`, behavior for backward compatibility. A `GET /stores/:sid/groups/:gid/ready` endpoint checks cluster readiness (leader elected, quorum reachable, applied-slot lag). The operation - registry is an in-memory sharded-lock map in `crowdb-kv-server`; + registry is an in-memory sharded-lock map in `crowdb-paxos-kv-server`; this retained use is limited to low-frequency management work, and entry guards are dropped before background polling or other awaits. -Full design: `design-crowdb-kv-reconfiguration.md`, `design-crowdb-kv-server.md`. +Full design: `design-crowdb-paxos-kv-reconfiguration.md`, `design-crowdb-paxos-kv-server.md`. ## 10. Client Interaction @@ -430,9 +430,9 @@ Single-leader hot path: **Proposer → WAL → Replicator → Learner → ack.** ``` lib/crowdb-common/rust (shared Rust crate: metrics, logging, time, report) lib/crowdb-common/cpp (shared C++ static lib: crc32c, log, compressing_sink, gzip, metrics) -lib/crowdb-kv (core library: consensus, engine, wal, rpc, cluster) -lib/crowdb-kv-client (client library: topology cache, retry, NotLeader handling) -app/crowdb-kv-server (binary: CLI, HTTP mgmt API, store/group wiring) +lib/crowdb-paxos-kv (core library: consensus, engine, wal, rpc, cluster) +lib/crowdb-paxos-kv-client (client library: topology cache, retry, NotLeader handling) +app/crowdb-paxos-kv-server (binary: CLI, HTTP mgmt API, store/group wiring) lib/crowdb-console-shared (console core: API client, models) app/crowdb-web (Axum web server + React SPA) app/crowdb-cli (CLI binary: cluster management command) @@ -442,7 +442,7 @@ lib/crowdb-tree/ffi (C++ B+tree engine, FFI bridge to Rust) `crowdb-common` holds project-agnostic utilities shared across the storage-system stack. The Rust crate (`crowdb-common`) provides the metrics registry, `tracing`-based logging wrapper, monotonic-time -helpers, and multi-step error aggregator; `crowdb-kv` re-exports them at +helpers, and multi-step error aggregator; `crowdb-paxos-kv` re-exports them at their original module paths so call sites compile unchanged. The C++ static library (`libcrowdbcommon.a`, namespace `crow::common`) provides CRC32C, the spdlog logging facade (`CR_LOG_*` macros), the compressing @@ -463,20 +463,20 @@ production). 4. Tests use `#[tokio::test(flavor = "current_thread", start_paused = true)]`. The async disk I/O substrate (`AsyncFile`: io_uring on Linux ≥ 5.11, -`tokio::fs` fallback otherwise) is detailed in `design-crowdb-kv-wal.md`. +`tokio::fs` fallback otherwise) is detailed in `design-crowdb-paxos-kv-wal.md`. ## 14. Components -- **`crowdb-kv`** — core library: consensus, engine, WAL, I/O, RPC, reconfiguration. +- **`crowdb-paxos-kv`** — core library: consensus, engine, WAL, I/O, RPC, reconfiguration. - **`crowdb-common`** — shared utilities (Rust crate + C++ static lib): metrics, logging, time, report (Rust); CRC32C, spdlog facade, compressing sink, gzip, atomic metrics (C++). -- **`crowdb-kv-server`** — reference server binary. Design: `design-crowdb-kv-server.md`. +- **`crowdb-paxos-kv-server`** — reference server binary. Design: `design-crowdb-paxos-kv-server.md`. - **`crowdb-console`** — unified management project (Web UI + CLI). Design: `../console/design-crowdb-console.md`, `../console/design-crowdb-console-ui.md`. - **`crowdbbench`** — benchmark tool, fulfilled by `crowdb-console` CLI `bench` subcommand. -- **RPC** — flatbuffers over crowdb-rpc. Design: `design-crowdb-kv-rpc.md`. +- **RPC** — flatbuffers over crowdb-rpc. Design: `design-crowdb-paxos-kv-rpc.md`. ## 15. Performance Targets @@ -494,7 +494,7 @@ consensus events. Full metrics module design (metric types, registry lifecycle, naming convention, instrumentation points, system collector, log format, in-memory -access, FFI boundary): `design-crowdb-kv-observability.md`. +access, FFI boundary): `design-crowdb-paxos-kv-observability.md`. ## 17. Testing @@ -507,7 +507,7 @@ access, FFI boundary): `design-crowdb-kv-observability.md`. step-down, split-brain prevention, leader failure mid-proposal, lagging learner catch-up, missing slot repair, snapshot recovery. -Full test strategy: `design-crowdb-kv-test.md`. +Full test strategy: `design-crowdb-paxos-kv-test.md`. ## 18. References diff --git a/doc/design/protocol/design-crowdb-protocol-key.md b/doc/design/protocol/design-crowdb-protocol-key.md index 2f746ab7c..3a81f9842 100644 --- a/doc/design/protocol/design-crowdb-protocol-key.md +++ b/doc/design/protocol/design-crowdb-protocol-key.md @@ -9,7 +9,7 @@ Satisfies: [`design-crowdb-protocol.md`](design-crowdb-protocol.md) key encoding This is the sub-design for **key encoding** within the protocol area. Architecture decisions and the area envelope live in the root [`design-crowdb-protocol.md`](design-crowdb-protocol.md); this doc covers -the detailed design: how every CROWDB key that is stored in crowdb-kv is +the detailed design: how every CROWDB key that is stored in crowdb-paxos-kv is serialized to and parsed from bytes, via two encoding traits: `BinaryKey` (binary, for data groups) and `TextKey` (text-path, for group 0). It is shared across all components (diskdb first; future @@ -22,7 +22,7 @@ rules, frozen layouts, and evolution policy. **Which keys are persisted where, their value types, who writes them, and their scan patterns** are persistence concerns, defined in: -- `doc/design/kv/design-crowdb-kv-group0.md` §3 — group-0 sysdata schema +- `doc/design/kv/design-crowdb-paxos-kv-group0.md` §3 — group-0 sysdata schema (text keys + JSON values). - `doc/design/diskdb/design-crowdb-diskdb.md` §5 and §7 — data-group zone records (binary keys + flatbuffer values). @@ -44,7 +44,7 @@ patterns** are persistence concerns, defined in: ## 1. Problem -crowdb-kv's `KVEngine` treats keys as raw `&[u8]` and runs prefix/range +crowdb-paxos-kv's `KVEngine` treats keys as raw `&[u8]` and runs prefix/range scans by **lexicographic byte order** (`get(&[u8])`, `scan(prefix, start_after, end_key, …)`). A key encoding must therefore satisfy two rules: @@ -84,7 +84,7 @@ bytes.** CROWDB controls its own binary key format. the sole producers/consumers of KV key bytes. No second encoder in any component. - **Cross-component** — diskdb, and any future component stored in - crowdb-kv, share the same magic, trait, and field-encoding rules. + crowdb-paxos-kv, share the same magic, trait, and field-encoding rules. ## 3. Key Design Decisions @@ -249,7 +249,7 @@ to bytes: **This doc defines the encoding protocol**: the rules, the frozen layouts, and the evolution policy. **Which keys are persisted where, their value types, and their scan patterns are defined in the -persistence docs**: `doc/design/kv/design-crowdb-kv-group0.md` §3 +persistence docs**: `doc/design/kv/design-crowdb-paxos-kv-group0.md` §3 (group-0 sysdata schema) and `doc/design/diskdb/design-crowdb-diskdb.md` §5/§7 (data-group zone records). @@ -359,7 +359,7 @@ Text magic namespaces: `/hw` (hardware), `/srv` (service registry), `/kv` (KV-cluster topology). Frozen text key path shapes (the encoding spec; value types and scan -patterns are in `design-crowdb-kv-group0.md` §3): +patterns are in `design-crowdb-paxos-kv-group0.md` §3): - **RackKey** — `/hw/rack/<rack_id>`. - **NodeKey** — `/hw/node/<rack_id>/<node_id>`. @@ -398,7 +398,7 @@ KV keys and RPC messages are separate concerns: - **KV key bytes** — produced and consumed only by the Rust `BinaryKey` types in `crowdb-protocol`. These bytes go to - `crowdb-kv-client`'s `put` / `get` / `scan` and never appear on the + `crowdb-paxos-kv-client`'s `put` / `get` / `scan` and never appear on the crowdb-rpc wire as a serialized key message. - **RPC responses/requests** — use flatbuffer `**Info` messages that **flatten key fields and value fields into one message** @@ -478,7 +478,7 @@ string-field kinds; not needed while all keys are fixed-width.) ## 10. References -- crowdb-kv `KVEngine` trait (key bytes, prefix scan). +- crowdb-paxos-kv `KVEngine` trait (key bytes, prefix scan). - diskdb key kinds and their hierarchy: `doc/design/diskdb/design-crowdb-diskdb.md` §5 (group-0 sysdata) and §7 (zone records). diff --git a/doc/design/protocol/design-crowdb-protocol-types.md b/doc/design/protocol/design-crowdb-protocol-types.md index e3979d860..3f523ad93 100644 --- a/doc/design/protocol/design-crowdb-protocol-types.md +++ b/doc/design/protocol/design-crowdb-protocol-types.md @@ -20,7 +20,7 @@ key encoding. This doc covers everything else in `crowdb-protocol`: ID aliases, HTTP management API wire types, and group-0 sysdata entry types. -See `doc/design/kv/design-crowdb-kv-group0.md` §2.4 (single home +See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §2.4 (single home decision) and §2.5 (ID alias decision) for the upstream mandates. ## Table of Contents @@ -36,8 +36,8 @@ decision) and §2.5 (ID alias decision) for the upstream mandates. ## 1. Problem -CROWDB has multiple consumers of the same wire shapes — `crowdb-kv-server` -(producer), `crowdb-kv-client` (consumer), `crowdb-console-shared` +CROWDB has multiple consumers of the same wire shapes — `crowdb-paxos-kv-server` +(producer), `crowdb-paxos-kv-client` (consumer), `crowdb-console-shared` (consumer), `crowdb-web` / `crowdb-cli` (via the console). If each crate defines its own copy, the copies drift: one crate adds a field, the others silently drop it (serde leaves them at default). The same @@ -114,7 +114,7 @@ Every mgmt wire type derives `utoipa::ToSchema` behind a feature-gated pub struct TopologyResponse { … } ``` -Only `crowdb-kv-server` (which builds the OpenAPI spec) enables the +Only `crowdb-paxos-kv-server` (which builds the OpenAPI spec) enables the `schema` feature. All other consumers depend on `crowdb-protocol` without it and never pull in `utoipa`. @@ -134,7 +134,7 @@ hosts the conversions. It defines zero structs. ### 3.6 Server-local types stay server-local -Types used only inside `crowdb-kv-server` (e.g. `JoinGroupRequest`, +Types used only inside `crowdb-paxos-kv-server` (e.g. `JoinGroupRequest`, `FlushResult`, `ReadinessResponse`, `ApiDoc`) stay in the server. They are not cross-component; promoting them to `crowdb-protocol` would be premature. The rule: a type moves to `crowdb-protocol` only @@ -156,7 +156,7 @@ types are re-exported from the crate root. state, read state, metrics points). - **`sysdata`** — group-0 sysdata entry return types: the decoded form of a text-path key + its JSON value, produced by - `HardwareClient` / `KVClusterMetaClient` reads. + `HardwareClient` / `PaxosKvClusterMetaClient` reads. - **`key`** — key encoding traits and structs (covered in [`design-crowdb-protocol-key.md`](design-crowdb-protocol-key.md)). - **`common` / `diskdb::rpc` / `chunkdb::rpc` / `diskio::rpc`** — @@ -164,13 +164,18 @@ types are re-exported from the crate root. - **`diskdb_type_util`** — extension traits and utility functions for diskdb flatbuffer types. - **`bitmap`** — usage bitmap utilities for disk space accounting. -- **`ports`** — default port allocation for CROWDB services. Each - service type has a base (start) port; multiple instances of the - same service type on one node increment by a per-type stride. Port - ranges are non-overlapping across service types so different - services never collide. Consumers (kv-server, diskdb, web, cli) - reference the base constants for clap `default_value_t` and config - defaults instead of hardcoding port numbers. +- **`ports`** — the single definition of default listener ports. Rust + consumers use the constants or `ServicePort`; Python, TypeScript, C++ and + packaged configuration use generated defaults. Each active listener kind + owns a disjoint 1000-port range, with instance stride 1. Console Web uses + 6135; S3, Iceberg and Dataset use 7135, 8135 and 9135. Management and RPC + defaults for the same server are 1000 apart. Explicit deployment endpoints + remain authoritative. An unused main-listener field does not reserve a + port or create a listener. DiskDB HTTP and RPC are independent endpoints; + its deployment RPC input is the actual RPC port without an implicit offset. + Paxos-KV's RPC pool serves both consensus and client requests. A deployment + with multiple stores must allocate distinct RPC ports for every store. + Field-level detail for every type lives in the source files (`lib/crowdb-protocol/src/{common_type,mgmt,sysdata}.rs`), not in this @@ -187,9 +192,9 @@ Every consumer follows the same pattern: ## 6. References -- `design-crowdb-kv-group0.md` §2.4 — single home mandate. -- `design-crowdb-kv-group0.md` §2.5 — ID alias mandate. -- `design-crowdb-kv-server.md` §2.4 — HTTP management API endpoints. +- `design-crowdb-paxos-kv-group0.md` §2.4 — single home mandate. +- `design-crowdb-paxos-kv-group0.md` §2.5 — ID alias mandate. +- `design-crowdb-paxos-kv-server.md` §2.4 — HTTP management API endpoints. - [`design-crowdb-protocol-key.md`](design-crowdb-protocol-key.md) — key encoding (companion sub-design). - The `crowdb-protocol` crate modules — type definitions (the diff --git a/doc/design/protocol/design-crowdb-protocol.md b/doc/design/protocol/design-crowdb-protocol.md index 367ec4c6c..9581a5b72 100644 --- a/doc/design/protocol/design-crowdb-protocol.md +++ b/doc/design/protocol/design-crowdb-protocol.md @@ -5,7 +5,7 @@ The **protocol** area covers the shared cross-component types hosted in the `crowdb-protocol` crate: the key encoding that every component uses -to store and scan rows in crowdb-kv, the wire types that cross crate +to store and scan rows in crowdb-paxos-kv, the wire types that cross crate boundaries (HTTP management API DTOs, group-0 sysdata entry types), and the identifier aliases (`RackId`, `NodeId`, `DiskId`, …) that give every crate one named, numeric definition of each ID. `crowdb-protocol` is the @@ -43,7 +43,7 @@ that keeps one source of truth across crates. - **No flatbuffer service definitions.** `.fbs` files and generated code live in `crowdb-protocol` for shared access, but their design is driven by each component's crowdb-rpc service doc, not here. -- **No server-local types.** Types used only inside `crowdb-kv-server` +- **No server-local types.** Types used only inside `crowdb-paxos-kv-server` stay in the server; they are not cross-component. - **No compression.** Keys are small and fixed-width; compression would break lexicographic order. @@ -57,7 +57,7 @@ that keeps one source of truth across crates. No other crate defines its own copy of a type that crosses a boundary. The crate is lightweight (no tokio runtime, no engine code) so any crate can depend on it. -- **Keys are self-sorting, prefix-stable bytes.** crowdb-kv's +- **Keys are self-sorting, prefix-stable bytes.** crowdb-paxos-kv's `KVEngine` treats keys as raw `&[u8]` and scans by lexicographic byte order. The key encoding produces deterministic, self-sorting, prefix-stable bytes — flatbuffer `*Key` messages are never used as KV diff --git a/doc/design/rpc/design-crowdb-rpc.md b/doc/design/rpc/design-crowdb-rpc.md index 56602be86..d0f980812 100644 --- a/doc/design/rpc/design-crowdb-rpc.md +++ b/doc/design/rpc/design-crowdb-rpc.md @@ -38,7 +38,7 @@ detail is in the sub-designs. the caller chooses how to serialize values (flatbuffers, bincode, raw). - **No TLS.** CROWDB runs on a trusted internal cluster network. - **No client-side retry or topology.** Retry, topology cache, and - `NotLeaderHint` handling live in `crowdb-kv-client`, not here. + `NotLeaderHint` handling live in `crowdb-paxos-kv-client`, not here. - **No streaming RPC.** v1 supports request-response and one-way messages only. Bidirectional streaming is a future addition. - **No HTTP/2.** The custom framing layer is simpler and faster for @@ -322,6 +322,11 @@ server side and the Rust client side. deserialize the flatbuffer into a separate owned Rust struct or C++ class that copies each field out of the buffer. The buffer is the object; accessors read from it in place. + C++ controls may start immediately after the 14-byte wire header, at an + unaligned address. After verification, use + `crowdb::protocol::flatbuffer_root<FBT>(ptr, size)` to load the root offset + with `memcpy`, then use the generated table accessors. This borrows the + original frame without copying its payload or allocating heap storage. - **Wrapper classes in `crowdb-protocol`, defined when required.** When a service needs a typed API over a flatbuffer buffer (to encapsulate parse + null-check + field access, to add domain logic, or to hide @@ -331,7 +336,7 @@ server side and the Rust client side. — it reads through the flatbuffer root pointer on every accessor call. Wrappers are defined when a service needs them (not preemptively for every flatbuffer type), and they live in - `crowdb-protocol` so every project (crowdb-kv, crowdb-diskdb, crowdb-chunkdb, + `crowdb-protocol` so every project (crowdb-paxos-kv, crowdb-diskdb, crowdb-chunkdb, crowdb-diskio) shares one definition. - **Wrapper naming.** Wrapper types use the `FB` prefix + the domain type name, without a `View`/`Ref` suffix — the `FB` prefix already diff --git a/doc/design/tree/design-crowdb-tree-chunk-storage.md b/doc/design/tree/design-crowdb-tree-chunk-storage.md index 7960713ff..09f3378e7 100644 --- a/doc/design/tree/design-crowdb-tree-chunk-storage.md +++ b/doc/design/tree/design-crowdb-tree-chunk-storage.md @@ -10,333 +10,274 @@ Depends on: [`design-crowdb-tree.md`](design-crowdb-tree.md), Satisfies: [`design-crowdb-tree-storage.md`](design-crowdb-tree-storage.md) §1 (page-store abstraction) -This document specifies the chunk-backed `PageStore`: immutable mirrored page -packs, generation-fenced manifests, recovery, range rebuild, ownership -materialization, and the boundaries for later physical reclamation. +This document defines typed immutable Chunk backing, root publication and +protection, physical collection, and explicit migration of logical-address +manifests. Host ownership and WAL recovery remain in the Chunk-KV design. ## Table of Contents -- [1. Architecture](#1-architecture) -- [2. Page Packs and Addressing](#2-page-packs-and-addressing) -- [3. Async Execution and Mirroring](#3-async-execution-and-mirroring) -- [4. Manifest Publication and Recovery](#4-manifest-publication-and-recovery) -- [5. Reads and Cache Validity](#5-reads-and-cache-validity) -- [6. Range Rebuild and Sharing](#6-range-rebuild-and-sharing) -- [7. Materialization and Reclamation](#7-materialization-and-reclamation) -- [8. Correctness Invariants](#8-correctness-invariants) -- [9. Configuration and Metrics](#9-configuration-and-metrics) -- [10. Performance Evidence](#10-performance-evidence) - ---- - -## 1. Architecture - -`ChunkPageStore` implements both `PageStore` and `AsyncPageStore`. Tree -algorithms depend only on those interfaces. The C API owns stores through an -opaque `ct_page_store`; `ct_open` accepts an injected store and does not name -or construct the chunk backend. The Rust chunk-KV owner selects the backend, -provides the ownership epoch, and injects production catalog and transport -dependencies. - -`ChunkTransport` supplies chunk allocation, append, layout lookup, mirror I/O, -and sealing. `RootCatalog` stores immutable reference-segment images and -publishes one current manifest per tree. The production transport speaks the -ChunkDB and DiskIO protocols directly; deterministic in-memory implementations -exercise the same contracts in tests. - -The common static tree archive contains the chunk and RPC-backed implementation, -but an ordinary local-tree link extracts neither archive member nor its RPC -symbols. Chunk implementation headers remain private, while the stable C ABI -exposes construction, transport routes, statistics, and orphan accounting. - -## 2. Page Packs and Addressing - -The tree writes a logical byte image. The chunk backend groups its dirty ranges -into immutable packs of at most `pack_bytes`. Every pack has a logical offset -and a `ChunkPageRef` containing chunk identity, physical offset, logical length, -and frame revision. Frame footers own payload integrity and expected chunk -identity; high-cardinality reference entries do not duplicate their checksums. - -Pending writes form a sparse byte overlay on one captured immutable manifest. -Starting a write does not materialize the published logical image or its holes. -Partial writes capture only their intersected published packs, retaining the -verified untouched bytes even if those mirrors become unavailable before commit. -Later overlapping writes replace earlier bytes; overlay fragments are bounded -to 64 KiB so replacing a small range cannot copy an arbitrarily large fragment. -Each fragment retains its full logical coverage while storing trailing zeros -implicitly. Zero padding therefore still overwrites base bytes and participates -in coverage checks without retaining an allocation-unit buffer per padded page. -Publication assembles one pack from verified base bytes and the overlay at a -time. Unchanged packs still undergo frame verification before reuse. Both -publication paths discard the overlay after commit, releasing its buffers. -The tree releases each encoded snapshot page as soon as its store write -completes, including asynchronous completion. It retains the page's publication -identity through the final anchor barrier, without retaining a second copy of -all bytes already accepted into the backend overlay. -Snapshot preparation retains logical encoded page bytes. It reserves aligned -addresses immediately but adds zero padding to the current page only at write -submission, avoiding an additional allocation-unit padding copy of every page. - -Mapping-table slots do not store physical chunk offsets. Their tagged 64-bit -page-reference words name a logical location whose covering pack is resolved -through immutable segmented reference tables. Untagged words remain valid for -local stores and legacy mapping images. Mixed mapping images use their versioned -codec so the two forms cannot be confused. - -Manifest formats 6 and 7 require concrete BtreePage/PageIndex chunk identities, -retain the WAL replay offset and permit sorted, non-overlapping sparse pack -layouts. A gap means that no live page references that logical pack range. -Recovery validates arithmetic bounds, reference-segment checksums and complete -pack coverage for every addressed page. Unsupported formats fail visibly. - -New manifests use format 7 and packs use ordinary revision-2 common frames. -Each frame has a 15-byte header and 20-byte footer, with at most 65,501 payload -bytes; Stream grouping roles do not apply. Revision-1 packs remain readable. -The serialized 28-byte reference uses its u32 length's high bit as the revision-2 -marker. Valid logical pack lengths are bounded by 256 MiB, so retained -revision-1 lengths have that bit clear. In-memory logical lengths exclude the -marker. Manifest and reference-segment CRCs cover the encoded marker, and -format-6 manifests accept only legacy references. Exact reads derive physical -length from the reference's revision without a separate header probe. Reused -packs and immutable reference segments retain their original revisions. - -The backend uses the configured number of mirror copies. One B+tree chunk carries -at most 256 MiB of logical pack data; this is a ceiling, not an initial reservation. -BtreePage and PageIndex chunks initially allocate at most 4 MiB of logical strip -capacity with the default 1 MiB strips. The ordered pack planner appends strips -in 4 MiB growth windows as writes require them, before submitting mirror I/O. -Physical allocation counts every replica of the allocated strips, not replicas -of the maximum chunk size. Growth uses the observed ChunkDB revision, preserves -writer authority, and cannot advance the acknowledged cursor. A failed or -uncertain append never authorizes a write beyond the confirmed physical layout. - -A pack never crosses the logical limit: the current chunk -is sealed and the whole pack is appended to a new chunk. A reopened store starts -with a fresh writable chunk. DiskIO owns device alignment; tree page records are -tail-padded to the configured page alignment while logical lengths and checksums -exclude padding. - -Snapshot page bytes use BtreePage chunks. Persistent mapping segments, -mapping directory, and retained tree anchors use PageIndex chunks through -`write_typed_at` / `submit_typed_write`. Staging records purpose boundaries; -packs split at a purpose boundary and chunks rotate when purpose changes. -Adjacent equal-purpose staging ranges coalesce, so pack planning scans purpose -transitions rather than every historical write boundary. Separate purpose -chunks can increase allocation and frame overhead for small mapping images. -Physical frame overhead also participates in capacity checks. Reuse and -materialization preserve purpose encoded in each chunk identity. - -The direct-KV root catalog and immutable reference segments locate the logical -image, including its PageIndex directory and anchors. Reading the mapping's own -chunk therefore does not require that mapping to be loaded first. Snapshot -publication makes page and mapping pack references visible together. Both -purposes follow the same generation pins and reachability rules during split, -sharing, materialization, and reclaim; purpose alone never authorizes freeing. - -## 3. Async Execution and Mirroring - -All backend operations enter a bounded lock-free queue. One ordered continuation -worker preserves submission order, assigns stable nonzero operation IDs, skips -cancelled queued work, and drains accepted completions during shutdown. Queue -exhaustion completes immediately with `ResourceExhausted` and still follows the -exactly-once callback contract. - -Pack construction admits at most `max_concurrent_packs`. Each pack fans out -three mirror writes through sender composition. The pack is durable only after -all mirrors acknowledge it. Failure or cancellation stops new admission, drains -launched work, records possible unpublished objects, and prevents manifest -publication. - -The tree uses one backend-neutral completion descriptor. Every terminal future -transition signals it exactly once after publishing completion state with -release ordering. Linux uses `eventfd`; the portable fallback uses a -nonblocking pipe. Rust polls the descriptor through its existing async reactor. - -## 4. Manifest Publication and Recovery - -`ChunkManifest` records tree identity, generation, owner epoch, logical size, -the snapshot's WAL replay offset, pack entries, reference-segment directory, -sharing counters, and checksums. Manifest directories and reference-segment -images are immutable. The backend accepts supported typed manifests and -rejects unknown formats before using their references. - -A checkpoint captures the current catalog generation before it constructs any -packs. Its new manifest is numbered `expected_generation + 1`. After pack and -metadata durability barriers, `RootCatalog::publish` compares the same expected -generation and ownership epoch. Only that compare-and-publish operation makes -the checkpoint visible. Stale, failed, or ambiguous work remains unreachable -and is accounted as orphan data. - -Once a tree has written an anchor, a failed final durability barrier leaves -that anchor pending under the existing snapshot serialization gate. Before -preparing another snapshot, synchronous and asynchronous callers retry the -pending barrier over the same staged image. Continued unavailability remains -a typed failure; no new snapshot data is written until the pending barrier -succeeds. Writes may continue into the live tree and are captured by the next -snapshot. Failed publication never advances the tree's durable snapshot or WAL -retention frontier. - -Recovery normally opens the newest epoch-valid, fully verified manifest. It -may fall back only to the preceding complete retained generation. A caller -holding topology recovery evidence may instead request one exact immutable -root-catalog generation. Exact open never falls back to current, never accepts -a cached generation with a different identity, and retains that bootstrap -layout until the opened store successfully publishes its own successor. A -successor publication first proves that the bootstrap is still current, so an -old exact view cannot create a divergent root branch. Corruption, absence, and -temporary availability remain distinct typed outcomes; an unavailable mirror -does not poison mappings or convert a retryable read into corruption. - -The root-catalog manifest generation and the tree snapshot sequence are -different identities. The former orders publication of page-pack reachability; -the latter is stored inside the tree image and orders logical tree snapshots. -Recovery evidence records and validates both rather than using either counter -as a substitute for the other. - -The tree owner sets a monotonic WAL replay offset before snapshot publication. -The offset and tree snapshot become visible in the same root manifest. WAL -retention advances only after that root is current, so recovery can always open -the latest root and replay from its offset through the stream's latest durable -tail without consulting a duplicated group-0 frontier. - -## 5. Reads and Cache Validity - -A read resolves its mapping word and immutable reference segment, obtains a -chunk layout, then reads and verifies the covering pack. Layouts are cached only -until their `valid_until` deadline and are refreshed before later use. - -The ordered worker owns one immutable verified pack cache. Adjacent asynchronous -page reads covered by that pack reuse its bytes and avoid another DiskIO call. -A different pack replaces the cache entry. Synchronous maintenance reads do not -mutate this executor-owned cache. Cancellation is observed before submission, -after transport completion, and before bytes or roots become visible. - -## 6. Range Rebuild and Sharing - -Each persisted page carries checksummed lower and upper reachability fences. -Range rebuild pins one source generation and walks native pages in bounded -batches. Separator fences skip disjoint subtrees before their pages are loaded. -Wholly contained pages reuse immutable pack and metadata references; boundary -leaves, crossing ancestors, escaping siblings, and overflow chains are rebuilt. - -Each child has an independent root, mapping table, high-water allocation range, -manifest lineage, and mutable state. Immutable reference-segment and mapping -images may be shared until a changed entry triggers segment-level copy-on-write. -Concurrent rebuild workers share only the immutable pinned source. - -## 7. Materialization and Reclamation - -Materialization first snapshots all valid retained tree anchors and derives the -live anchor, directory, mapping-image, and page extents. The snapshot generation -gate remains held while the chunk backend repacks those extents, so a foreground -snapshot cannot invalidate the reachability image mid-pass. - -Packs outside the live extents are omitted. Shared live packs are copied into -child-owned chunks under a byte budget, reference ordinals are made dense, and -a sparse typed manifest is published through the normal generation fence. -Later checkpoints preserve absent sparse ranges unless a new write dirties them. -A failed reuse verification rewrites already materialized bytes; it never turns -a live pack into a hole. - -Historical manifests are immutable. Catalog retention and in-memory pins decide -when manifest objects become logically reclaimable. The backend reports orphan -and reclaim candidates but does not physically release strips. Expired-writer -sealing and physical strip GC are independent maintenance services so either can -resume after a tree or ChunkDB restart. - -A topology transition that publishes an exact root generation uses a durable -generation pin, not only a process-local reader reference. Its logical record is -`root/<tree_id>/pin/<transition_id> = generation`; the durable chunk-KV catalog -encodes each numeric identity in big-endian bytes below its root-catalog prefix. -Creating the same transition-to-generation mapping is idempotent, while trying -to reuse the transition identity for another generation fails closed. Pin -creation also verifies that the named manifest still exists. - -If a process crashes after pinning but before persisting transition readiness, -the durable phase still proves that no artifact can name the abandoned root. -The repeated preparation removes that stale unpublished pin and installs the -new attempt's generation before recording readiness. Once readiness is durable, -the pin identity is immutable until catalog cleanup. - -The ordered pin protocol is: - -1. Produce and durably publish root generation `g`. -2. Persist the transition pin for `g` before making any split or transfer - artifact that names `g` publishable. -3. Exact-open `g` on every recovery or target preparation; absence is an error - and never falls back to latest. -4. Materialize and checkpoint an independently recoverable successor, then - publish catalog state that no longer names the exact-root overlay and clears - its transition marker. -5. Only after observing that authoritative catalog generation, delete the pin. - Deletion is idempotent, so recovery may repeat it after an ambiguous result. - -`reclaim_before(tree_id, n)` scans the durable pins for that tree and caps its -effective upper bound at the oldest pinned generation. Manifest metadata and -packs reachable from that generation therefore survive process and server -restart. A crash may temporarily leak a pin, but cannot permit early deletion; -the transition/materialization reconciler retries the final idempotent unpin. - -## 8. Correctness Invariants - -- **I1 — Durable before visible:** every referenced mirror and metadata image is - durable before its manifest can become current. -- **I2 — One fenced successor:** generation `g + 1` publishes only by comparing - generation `g` and the current ownership epoch. -- **I13 — Typed storage:** page and mapping payload never share a chunk; - mapping bootstrap references remain independently recoverable. -- **I3 — Whole-pack rotation:** no page pack crosses the 256-MiB logical chunk - limit. -- **I4 — Verified addressing:** every tagged page location resolves through a - checksummed immutable segment to complete manifest coverage. -- **I5 — Sparse absence is stable:** an omitted pack range remains absent until - an overlapping logical write dirties it. -- **I6 — Sharing is immutable:** lineages share only immutable packs and metadata - images; their first mutation uses copy-on-write. -- **I7 — Rebuild isolation:** a child contains exactly its half-open key range - without foreign keys, sibling edges, or overflow references. -- **I8 — Retention precedes release:** no physical strip is released while a - retained manifest or in-memory pin can name it. -- **I9 — Exactly-once completion:** every admitted async operation completes its - callback once, including cancellation, overload, and shutdown. -- **I10 — Root/WAL checkpoint:** a manifest's WAL replay offset becomes visible - atomically with its tree snapshot and never regresses within one lineage; - WAL trimming follows root publication. -- **I11 — Exact root identity:** an exact-generation open reads only the named - root-catalog manifest and cannot publish a successor after another current - generation has won. -- **I12 — Persistent transition pin:** a root generation named by a publishable - topology artifact remains reclaim-ineligible across restart until an - authoritative overlay-free catalog generation has been observed. - -## 9. Configuration and Metrics - -The bounded defaults are: - -- `pack_bytes`: 4 MiB; -- `max_chunk_bytes`: 256 MiB hard ceiling; -- `page_alignment` and `iu_size`: 64 KiB; -- `max_concurrent_packs`: 8; -- `max_pending_ops`: 256; -- `layout_validity_ms`: at most 30 seconds; and -- `materialization_bytes_per_pass`: 64 MiB. - -`ChunkPageStoreStats` exposes publication generations, pack writes and reuse, -pack bytes, reads and cache hits, layout queries, mirror attempts and failures, -retention pins, orphan bytes, materialization work, shared ownership, RPC and -DiskIO operation latency, coalesced reads, completion wakeups, metadata sharing, -manifest publication latency, and recovery latency. The C ABI and Rust wrapper -preserve the same counter meanings. - -## 10. Performance Evidence - -A fixed 4-MiB in-memory transport workload measured these aggregate means on -11 September 2026: - -- 64-KiB cold read: 0.79 ms real time, 89.6 MiB/s CPU throughput; -- coalesced read: 4.4 us real time, 15.6 GiB/s CPU throughput; and -- 4-MiB snapshot: 26.2 ms real time, 153.8 MiB/s. - -The benchmark validates bounded staging and cache behavior without claiming -production network latency. Runtime RPC and DiskIO counters provide the -production evidence needed to revisit the defaults. +- [1. Runtime and Backend Ownership](#1-runtime-and-backend-ownership) +- [2. Fixed Packs and Root Identity](#2-fixed-packs-and-root-identity) +- [3. Publication and Recovery](#3-publication-and-recovery) +- [4. Readers and Process Fencing](#4-readers-and-process-fencing) +- [5. Range Rebuild and Immutable Snapshots](#5-range-rebuild-and-immutable-snapshots) +- [6. Typed Pack Collection](#6-typed-pack-collection) +- [7. Explicit Legacy Migration](#7-explicit-legacy-migration) +- [8. Legacy Physical Retirement](#8-legacy-physical-retirement) +- [9. Accounting and Configuration](#9-accounting-and-configuration) +- [10. Correctness Invariants](#10-correctness-invariants) + +## 1. Runtime and Backend Ownership + +`ChunkRuntimePageStore` is a backend facade injected into `TreeCore` with a +shared `TreeRuntime`. It owns no private execution pool. The host shares CPU, +backend execution, and the resource budget; each Tree retains its own lane, +account, immutable publication state, and cache. Construction requires a +fixed-pack transport, a staging/protection catalog, and a host pin-release +service. Unsupported capabilities fail before an ordinary Tree is opened. + +The C opaque store and Rust `PageStore::open_chunk_runtime` retain the catalog +and transport domain. One catalog claim denotes one writer incarnation. Closing +that writer drains accepted work and publishes its quiescence; reopening +requires a fresh claim. Reader and snapshot owners can outlive the Tree. +Their final releases enqueue paid pin work on the host service. The host keeps +that service and the transports available until late owners and accepted +release work have drained. + +`ChunkTransport` provides fixed allocation identity, mirrored immutable I/O, +request-age fencing, ordered drain, and deletion receipts. The production +adapter speaks ChunkDB and DiskIO protocols directly. Local-only links do not +extract the Chunk/RPC implementation from the common archive. + +## 2. Fixed Packs and Root Identity + +`BackingRef` identifies an immutable typed backing. A Chunk pack reference +contains its stage identity, Chunk identity, byte offset, length, checksum, and +purpose. It is not an address into a mutable logical byte image. BtreePage and +PageIndex payloads have distinct Chunk purposes. Purpose alone never proves +ownership or authorizes reuse or deletion. + +A stage has a stable operation identity and a fixed allocation identity before +allocation is submitted. Its durable state records allocation and reloadability +progress. Mirrored bytes become reloadable only after all required writes and +barriers succeed. A cancelled or failed attempt retains sufficient stage and +allocation information for later reconciliation; it does not publish partial +logical coverage. + +`ChunkRootManifest` contains Tree identity, root generation, owner epoch, +snapshot sequence, page generation, coverage, GC floor, root PID, next PID, +WAL replay offset, and a bounded directory of immutable index packs. Index +entries contain exact frame/backing and dependency references. Root and index +records have independent magic, version, length, identity, and checksum checks. +Root generation and snapshot sequence are separate monotonic identities. + +## 3. Publication and Recovery + +A checkpoint captures a finite source frontier and immutable page state. It +writes or reuses verified primary backing, prepares the complete reference +set, and commits one manifest candidate against the captured root generation +and writer authority. Candidate preparation is windowed. Reference ownership +precedes the root CAS so a crash cannot expose a root whose dependencies are +collectible. + +Only successful durable root publication advances the durable frontier and +WAL replay offset. Handoff coverage and staged writeback do not authorize WAL +trimming. A lost commit reply is resolved by looking up the same operation and +root identity; it is not retried as a different publication. + +Ordinary recovery loads the current root. Exact-generation recovery loads only +the named retained root and never falls back to another generation. Absence at +generation zero denotes a new Tree; malformed or unsupported existing bytes +never silently open an empty Tree. Recovery validates the root, each index, +frame identity, graph summaries, and overflow dependencies before publication. +Bounded windows retain backend and protection owners through the final I/O. + +A later checkpoint may reuse immutable primary and overflow backing when its +captured identity still matches. Reuse does not imply logical liveness outside +the captured graph and does not turn an uncommitted stage into a durable root. + +## 4. Readers and Process Fencing + +Reads use the immutable captured mapping and its typed backing domain. Frame +and overflow verification check the requested identity and stored checksums. +The per-Tree cache pins verified immutable frames; cache eviction changes +residency, not publication or durable reachability. Layout validity and read +request-age checks are enforced at the transport boundary. + +The catalog binds writer registrations and live pins to the host process +lease. Pin, candidate, root, and collection mutations share an authority +revision. A definite CAS conflict discards derived state and rereads within a +finite retry bound. Transport ambiguity is returned for durable lookup; it is +not treated as a CAS conflict. + +Writer quiescence excludes future writes after accepted work drains. It does +not exclude readers. Removing a crashed process's pins requires authoritative +process exclusion, including the applicable request-age boundary, and the +recovery evidence required by the catalog. A new owner epoch alone cannot +erase live old readers. The same-epoch writer incarnation is also checked. + +## 5. Range Rebuild and Immutable Snapshots + +`rebuild_range` captures a fixed immutable source and copies the selected native +image into independently owned target backing. The target has an independent +root lineage, mapping, PID allocation state, writer claim, and mutable source +set. Source and target mutations cannot affect each other. Logical bounds are +intersected at get/scan entry, including when a physically complete imported +page contains keys outside the target range. + +Native and byte snapshot windows carry their allocation and backend owners. +The immutable snapshot interface retains a captured root for repeated get, +scan, and export operations without recapturing live Tree state. Closing the +source Tree does not revoke that captured view. Final owners release their +protection through the host service. + +Topology artifacts that name an exact root retain durable generation pins. +The pin is installed before the artifact becomes publishable and is released +only after authoritative catalog state no longer names that root. A retry or +crash may retain the pin longer; it cannot shorten the protected lifetime. + +## 6. Typed Pack Collection + +Collection reconciles candidate states, retained root references, live pins, +and stage ownership. Retired stages pass through a durable deleting state +before physical side effects. That transition excludes new references and +pins for the reclaimed identity. Unresolved candidate outcomes remain +protected until the same operation can be resolved. + +Before deletion, the collector verifies the exact allocation receipt and +excludes late writers and delayed allocation requests. DiskIO ordered drain +completes before storage is released. ChunkDB's durable Deleted record retains +strips while cleanup is incomplete; completion requires the expected identity, +zero capacity, and no remaining strips. A deletion retry resumes the same +identity. The reclaimed stage and physical accounting change only after that +proof, including recovery of a previously acknowledged deletion. + +Catalog history pruning and physical deletion are separate operations. A +manifest disappearing from metadata is never by itself proof that its bytes +have been freed. Shared physical accounting deduplicates the immutable +allocation identity across roots and observers. + +## 7. Explicit Legacy Migration + +Logical-address format-7 roots are accepted only by the explicit read-only +migration adapter. Ordinary runtime open reports an unsupported legacy format +and directs callers to `migrate_legacy_chunk`. The old page-store constructor +is not a fallback writer for runtime Trees. + +Migration requires stopped legacy writers and a registered, fenced migration +owner. It pins the exact source generation and validates the full legacy +manifest, segmented references, frame revisions, bounds, checksums, and graph. +It copies the native image into new fixed packs and publishes a typed root +through the normal candidate and authority protocol. + +An immutable receipt containing the original manifest and pack directory is +persisted before the root CAS. Until the CAS succeeds the old root remains +recoverable. After an ambiguous reply, migration resolves the current typed +root and that receipt without reissuing an unrelated publication. The exact +source-generation pin remains held while the commit outcome is unknown. It is +released idempotently after the committed typed root is established, or after a +definite failure leaves the original root authoritative. Migration data +completion does not imply legacy physical retirement. + +## 8. Legacy Physical Retirement + +`ChunkKvStorage::collect_legacy_storage` is an explicit host maintenance entry +for one production metadata namespace. Each call accepts a work limit from 1 +through 64; its returned reclaimed counts cover that invocation, while +`complete` covers only persisted, known manifest receipts. Production split +descendants inherit that namespace. Arbitrary sharing across independently managed catalogs is +not covered by this proof. + +The collector first installs a durable namespace marker that permanently +rejects new format-7 publication. It then checks each Tree's current root and +freezes legacy reader and generation-pin admission under the same per-Tree +authority CAS used by publication. A current legacy root returns `LegacyRoot` +until migration completes. Existing generation pins must be released through +their real transition protocol. + +Publication provenance and each admitted legacy reader are recorded durably. +Complete lineage inventory is established only with its first publication; +a later registration cannot retroactively establish missing history. Pack +owner Trees must belong to that proved namespace inventory. +Every relevant process must have authoritative exclusion evidence; a closed +writer is insufficient. The namespace scan and reader-proof scans retain +bounded cursors. No deletion begins until the namespace proof completes. +History pruning preserves an immutable legacy migration receipt before +removing the corresponding manifest, with at most one newly copied legacy +image in a pruning pass. + +For each known Chunk, retirement persists its exact allocation receipt before +ordered drain and deletion. Completion requires the same creation identity and +an acknowledged physical release. Unknown replies retain the claim; a later +invocation queries and resumes that identity. Missing or changed allocation +evidence remains typed pending. Receipts survive completed deletion for +idempotence and inspection. The host maintenance loop resumes only namespaces +whose explicit start marker already exists; a normal maintenance pass cannot +start an upgrade. Pending owner/pin proof waits for the next scheduled pass +instead of creating an immediate busy retry. + +Historical manifests without complete publication/reader provenance remain +`MissingOwnerProof`. Their receipts and physical data are retained. The latest +writer registration cannot reconstruct older reader owners. Similarly, a +historical allocation whose manifest was already removed and has no receipt +is outside this known-manifest census: enumerating an unattributed Chunk does +not prove which Tree owned it. Neither case is reported as reclaimed. These +limits do not prevent data migration or normal typed-pack collection. + +## 9. Accounting and Configuration + +Host resource admission covers runtime state, accepted operations, metadata, +frame/payload storage, I/O windows, and retained output owners. Fixed-pack +identity and physical allocation receipts underpin the backend ledger. +`TreeBackingStats` exposes allocated physical capacity, including mirrors; +`TreeWorkCollector` exports immutable collection windows and cumulative work +counters. Host consumers deduplicate observations with the same scope. Capacity +includes old or staged output until physical deletion is acknowledged, so it +does not measure only the latest root's useful contents. Classification metrics +do not create separate ownership records or scan the backing graph. + +The host samples `chunk_pin_release_slots` through the existing resource log +windows. `limit` is the queue capacity; `used` and `allocated` count registered +pins, including pins awaiting final release. `waiters` counts those pending +releases, and `denied_host` counts capacity rejections. Samples do not pump the +queue or reset counters. Averages and maxima describe sampled occupancy, while +release latency remains unmeasured by this row. Chunk-KV configures this shared +capacity with `storage.tree_pin_release_slots`, defaulting to 1,048,576. Local +memory/block hosts have no Chunk pin-release queue and emit no such row. + +Legacy `ChunkPageStoreStats` has no invented runtime equivalent. Its old C/Rust +getter reports `NotSupported` for a runtime facade. Callers use the typed +backing and work interfaces. + +`ChunkRuntimeOptions` specifies Tree and owner identity, optional exact root +generation, I/O alignment, mirror copies, and maximum Chunk bytes. Defaults are +64-KiB alignment, two mirrors, and a 256-MiB Chunk limit. Host worker, operation, +byte-budget, and cache limits remain separate configuration. A configured hard +limit is checked before work that depends on it is accepted. + +## 10. Correctness Invariants + +- **I1 — Durable before visible:** every reachable fixed pack and dependency is + durable and protected before its root candidate commits. +- **I2 — One fenced successor:** publication compares root generation, owner + epoch, writer incarnation, and process authority. +- **I3 — Immutable identity:** backing references name fixed verified content; + purpose or logical offset is never sufficient identity. +- **I4 — Exact recovery:** exact-root open cannot silently substitute another + generation or interpret an unsupported format as an empty Tree. +- **I5 — Independent rebuild:** target ownership and mutations are independent + of the captured source, and reads enforce the logical range. +- **I6 — Readers survive writer close:** retained snapshots keep their backend + and protection owners until final release is processed. +- **I7 — Process exclusion is explicit:** epoch advancement and writer + quiescence do not prove that old readers are gone. +- **I8 — Physical proof precedes accounting release:** neither metadata removal + nor an ambiguous deletion reply decrements confirmed physical ownership. +- **I9 — Bounded accepted work:** operations retain their prepaid CPU, metadata, + I/O, and terminal ownership until the real backend work completes. +- **I10 — Durable frontier controls WAL:** handoff and staged writeback cannot + independently advance WAL retention. +- **I11 — Migration preserves recovery:** original-root receipts precede CAS; + ambiguous outcomes resolve the same generation and operation. +- **I12 — Missing legacy evidence retains bytes:** unknown historical owners or + unattributed allocations are explicit limits, never inferred GC permission. diff --git a/doc/design/tree/design-crowdb-tree-engine-flush-flow.md b/doc/design/tree/design-crowdb-tree-engine-flush-flow.md index a38246785..0e3248d94 100644 --- a/doc/design/tree/design-crowdb-tree-engine-flush-flow.md +++ b/doc/design/tree/design-crowdb-tree-engine-flush-flow.md @@ -1,141 +1,163 @@ -# Flush Flow Analysis - -Flush drains frozen MemTables (L0) into the B+tree (L1) in memory. It does -not touch disk — that is snapshot's job. The `tree.flush.l` metric measures -the C++ `Crowdbtree::flush()` call only. - -## Call chain - -1. **Maintenance loop** (`group_maintenance.rs:194-211`) — every tick, - `spawn_blocking(|| engine_arc.flush())`. -2. **Rust bridge** (`crowdb_tree_engine.rs:284-293`) — calls - `handle().flush()` via FFI. -3. **C++ entry** (`crowdb-tree.cpp:1278-1337`) — `Crowdbtree::flush()`. - -## What `flush()` does - -``` -flush(): - t0 = steady_clock::now() - lock(write_mutex_) - maybe_freeze_active(force=true) // freeze active if past threshold - for iter in 0..max_memtable_count: - swap frozen_ -> to_drain // under memtable_mutex_ - if to_drain.empty(): break - cs = contiguous_slot_.load() // re-read each pass - active = current_active() - wrote_any |= drain_all_frozen_locked(to_drain, active, cs) - active->set_durable_floor(cs) - last_applied_slot_ = cs - version_++ - if frozen_ still non-empty: warn (iteration cap hit) - if !wrote_any: advance last_applied_slot; observe flush_l; return - maybe_evict_locked() // once, after all drains - observe flush_l -``` - -The re-check loop drains memtables that freeze *during* an in-flight drain -in the same `flush()` call — `apply()` on other threads keeps writing and -may trip the freeze threshold mid-drain. Re-reading `cs` each pass lets the -next pass drain entries that became contiguous during the prior pass. Capped -at `max_memtable_count` iterations so a sustained write rate faster than -drain throughput exits cleanly (remaining tables drain on the next tick). - -### `drain_all_frozen_locked` (`crowdb-tree.cpp:1137-1276`) - -Three phases: - -1. **Open cursors** on all frozen MemTables. -2. **K-way merge** via LoserTree. For each key run: - - `find_leaf_page_id(root, key)` — B-tree descent. - - `resident(page_id)` — may demand-load the leaf from disk if evicted - (`crowdb-tree.cpp:314-378`, blocking `read_at` + decompress under - `write_mutex_`). - - `publish_group_to_leaf_locked(page_id, cs, group)` — append deltas - or consolidate. -3. **Drain L0** — `mt->drain_up_to(cs)` on each frozen table. - -### `publish_group_to_leaf_locked` (`crowdb-tree.cpp:1105-1135`) - -Two paths: -- **In-frame delta** (fast): append deltas directly into the LeafBase - frame if `delta_count <= max_inframe_delta`. Consolidate when the - frame fills or `data_bytes > leaf_split_bytes`. -- **Heap delta**: build a `BatchDelta` node, prepend to the chain. - Consolidate when `delta_len > max_delta_len` or - `chain_bytes > max_delta_bytes`. - -### `consolidate_locked` (`crowdb-tree.cpp:1347-1389`) - -Folds the entire delta chain into a fresh LeafBase: -1. `resolve_leaf_chain_for_rebuild` — merge all base + delta entries into - an `std::map` by highest-slot-wins, drop tombstones `<= gc_floor`. -2. `build_leaf_spilling_locked` — re-encode every entry, spill large - values to overflow chains. -3. Retire old pages (delta nodes, old base, dead overflow chains). -4. `maybe_split_or_merge_locked` — may cascade splits/merges up the tree. - -### `maybe_evict_locked` (`crowdb-tree.cpp:721-737`) - -If buffer pool usage > 85% of frames, evict clean leaves to 70%. -`evict_clean_leaves_locked` (`crowdb-tree.cpp:555-630`) collects all -resident leaves, sorts by `last_touch_tick`, re-tags as unloaded. -This runs **inside `write_mutex_`**. - -## Metric observation points - -- **Start**: `crowdb-tree.cpp:1280` (`auto t0 = steady_clock::now()`) -- **End (no-op)**: `crowdb-tree.cpp:1319-1323` -- **End (normal)**: `crowdb-tree.cpp:1332-1335` -- **Registration**: `crowdb-tree.cpp:3655` (`prefix + ".flush.l"`) - -## Bottleneck analysis (2026-09-01 bench) - -Observed: `tree.flush.l` avg=643ms, max=1422ms. 20,026 frozen queue full -errors. The flush cannot keep up with the write rate. - -### Primary causes (in order of likelihood) - -1. **Synchronous demand-page loads under `write_mutex_`**. When the L1 - working set exceeds the buffer pool, `resident()` does a blocking - `read_at` + decompress for each cold leaf. This serializes all writes - behind the I/O. The buffer pool had `buf.resident.g=0` at the end of - the run — everything was evicted, so every flush demand-loads. - -2. **Large per-leaf consolidations**. A hot leaf receiving thousands of - updates per flush triggers `consolidate_locked`, which builds an - `std::map` of all entries, re-encodes them, and may split. The - `flush.entries.c` shows 1,035,551 entries drained across 5 flushes - (~207K entries/flush). - -3. **`maybe_evict_locked` inside `write_mutex_`**. The eviction scan - touches every resident leaf. With a large resident set this adds - hundreds of ms to each flush. - -4. **`write_mutex_` contention**. `flush()` holds the mutex for the - entire duration. Concurrent `apply()` calls are blocked, creating - a backlog that makes the next flush larger. - -### Levers - -- **Re-check loop in flush()** (implemented): drains memtables that freeze - during an in-flight drain in the same `flush()` call, reducing the frozen- - queue-full errors that occur when the backlog grows between flushes. -- **Increase buffer pool size** so the L1 working set stays resident. - This eliminates demand-load stalls — the dominant cost. -- **Pre-warm leaves** before acquiring `write_mutex_`. The frozen - MemTables know which keys they touch; pin those leaves first. -- **Move eviction out of `write_mutex_`**. Eviction can run after - releasing the lock, or in a separate background pass. -- **Tune delta thresholds** (`max_inframe_delta`, `max_delta_len`, - `max_delta_bytes`, `leaf_split_bytes`) to reduce consolidation - frequency vs. cost per consolidation. -- **Parallelize the k-way merge** across leaves. Currently the merge - is sequential; independent leaf writes could be parallelized. - -## Sub-phase instrumentation gap - -`tree.flush.l` is a single wall-clock summary. There are no per-phase -metrics for demand-load, merge, consolidation, split/merge, or eviction. -Adding local `steady_clock` probes and emitting per-phase times would -pinpoint which phase dominates the 643ms average. +<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> +<!-- Licensed under the Apache License, Version 2.0. --> + +# CROWDB - Design: Tree Flush Flow + +Depends on: [Tree engine](design-crowdb-tree-engine.md) +Satisfies: [Tree overview](design-crowdb-tree.md) §2 + +Flush transfers a finite applied source prefix into immutable L1 ownership. +This document describes the completion and resource contracts of that transfer. + +## Table of Contents + +- [1. Capture and Admission](#1-capture-and-admission) +- [2. Bounded Range Handoff](#2-bounded-range-handoff) +- [3. Maintenance and Completion](#3-maintenance-and-completion) +- [4. Progress and Observation](#4-progress-and-observation) + +## 1. Capture and Admission + +`FlushController` prepares a `FlushOperation` using the current published root, +source catalog, finite frontier and accepted writer bounds. Resource admission +reserves complete Handoff, Routing, Reclaim and I/O windows, plus operation, +publication and CPU continuation controls. Overflow in the bound or insufficient +credit rejects preparation before sources are closed; soft maintenance can +retain a valid plan and await temporary credit pressure asynchronously. + +Ordinary source admission holds a short atomic binder guard while recording its +entry and key-byte footprint. A reversible `SourcePause` prevents new ordinary +binders and observes completion of existing binders asynchronously. This fixes +the ordinary footprint, including admitted writes that have not applied yet, +without waiting for those writers to finish. Gap repair remains independently +admissible, so the bound adds its complete configured allowance. Returning +unused ordinary credit can only reduce this bound. + +The finite gap source allowance is reserved for the next contiguous missing +slot. Higher missing slots retain their prepared ordinary credit and wait for +frontier progress before using that allowance. Filling the gap window with +higher holes must not prevent the oldest hole from advancing the prefix. + +**I4 — Stable preparation:** the paid pause owner remains alive from planning +through actual capture and source closure. The plan is valid only for its +expected source catalog; a stale catalog requires replanning before closure. +Abandonment, cancellation or failed reservation releases the pause. Once the +outstanding binders have left, ordinary admission resumes and progress waiters +are notified. A permanently closed source never reopens. + +An open source without such a pause uses its full configured ordinary and gap +bounds. Soft rotation stores the stable bound before permanently closing its +source. That sealed bound covers outstanding admitted writers; once a closed +source is drained, later preparation can use its actual record/key footprint. +A source without a known stable or drained bound retains the configured bound. + +Before publishing a soft-rotation successor, `SourceRotation` checks that a +complete future flush fits every Handoff, Routing, Reclaim and I/O partition. +The check includes the sealed current footprint and a successor's full ordinary +and gap allowance. Source-count limits also leave room for the capture +successor and a split child's active table. If another source would exceed +these limits, maintenance hands off the existing prefix instead of publishing +that successor. + +`SourceAdmission` closes the captured sources and publishes a successor. +Already admitted writers can finish from their own credits. The operation +waits for their terminal state asynchronously; a CPU worker does not block while +those writers or child tasks are pending. The captured catalog and contiguous +frontier remain fixed. Source handoff does not infer coverage from the largest +key slot: NoOp and missing-slot progress come from the apply frontier. + +## 2. Bounded Range Handoff + +`FlushBatch` routes captured sources through immutable inner descriptors and +runs bounded range jobs. Each job uses independent lower/upper source cursors, +with a stable source tie order and the captured prefix/floor. It creates sorted +`OverlayBatch` entries with independent `CellVersion` owners and bounded key +arena blocks. Payload bytes remain shared; the output does not keep the whole +MemTable index alive. + +Entry and key-byte limits bound each installation window. Exhausting a key +arena pauses before consuming the next candidate. The next window continues +from the first unconsumed key with fresh thread-local guards. Parallelism, +range queue slots and inline dispatch all have explicit limits. + +The total range count is bounded by the captured input entry allowance and +the captured mapping's leaf count. Handoff prepays this complete metadata +bound; it does not impose a fixed ceiling on the Tree's leaf count. Routing +yields after a bounded number of leaves, and the dynamic join limits active +range jobs independently of the total admitted range count. + +Prepared replacements include expected page identities and an install token. +The coordinator merges a valid transaction with the latest root, preserving +unrelated sibling updates. Commit publishes mapping, source floor, coverage +and aggregate logical statistics together. Its final publication segment does +not allocate, cancel or invoke a caller callback. + +**I1 — Prefix safety:** a successful installation may precede a later error. +`FlushResult` reports actual published coverage; uncovered sources remain +available for retry and reads. Repeated tokens cannot duplicate installation. + +## 3. Maintenance and Completion + +A sealed dynamic join waits for every accepted range terminal. First error +stops further dispatch and drains already accepted children. Only after range +handoff has joined does the operation acknowledge detached source ownership +and execute its bounded maintenance phase. + +Maintenance performs coalescing, layout analysis, local split and eligible +merge publication. Unique page/pair candidates are indexed in the published +state; retry does not turn queue length into a page count. Cold maintenance +loads one bounded leaf and the required cold routing descriptors from one +captured view, then validates their original identities at publication. +Overflow payloads remain referenced during this primary-page work. + +Overlay coalescing can remove a tombstone at or below the captured GC floor +only when that leaf has no base references. A resident or cold base still +requires its masking tombstone; checkpoint encoding merges the complete leaf +before applying retention to its durable representation. + +The maintenance budget includes actual captured base-array bounds, configured +work/metadata limits and callback lifetimes. A continuation cannot consume an +unreserved host remainder merely because handoff has finished. All read and +I/O reservations are sealed only after their last required callback. + +**I2 — Distinct outcomes:** handoff success and maintenance success are reported +separately. A later maintenance error does not undo already installed coverage. +**I3 — No durability inference:** flush and representation-only maintenance do +not publish a backend root or advance the durable WAL replay point. + +## 4. Progress and Observation + +The host schedules finite maintenance/writeback windows even after writes stop. +Each window advances the oldest eligible candidate or reports a bounded reason +for no progress. Dirty pages, merge candidates and checkpoint-pending pages are +separate published quantities; checkpoint completion is not inferred from +writeback completion. + +Soft source maintenance uses the existing single dispatcher and prepaid +continuation controls. It either joins an accepted flush or retains a stable +plan while requesting the four resource partitions. Budget release resumes the +waiting continuation without another write. A changed immutable plan releases +its stale preparation and replans through the dispatcher; workers do not spin +or synchronously wait for credit. A bound that exceeds a partition's capacity +remains a rejection. + +A synchronous host apply may wait for an identified reversible source pause or +source-publication transition before consuming its input or claiming an +automatic slot. Synchronous host flush joins finite accepted work or waits for +an identified controller preparation. CPU and backend callbacks return typed +unavailability rather than waiting on their own executor. Ordinary writer +credit rejection retains its error; historical payload or old-view retention +does not create an unbounded host wait. + +Close and generation replacement first stop new ingress and stop unaccepted +soft-maintenance waits, then join ingress. This releases host callers awaiting +a pause before close waits for those callers. Already accepted chosen applies +have separate lifetimes and retain their prepaid handoff path until terminal; +close drains them before closing source admission and the flush controller. +Replacement may restart soft maintenance only while the Tree remains open. + +Stage counters measure capture, writer drain, queueing, execution, publication, +maintenance and terminal cleanup. Dispatch statistics record actual inline and +queued jobs and actual concurrency. Ages report outstanding work with an +explicit completeness flag when bounded tracking capacity is exhausted. +Metric saturation never rejects otherwise admissible storage work. diff --git a/doc/design/tree/design-crowdb-tree-engine-snapshot-flow.md b/doc/design/tree/design-crowdb-tree-engine-snapshot-flow.md index 6ca792072..f417d1efb 100644 --- a/doc/design/tree/design-crowdb-tree-engine-snapshot-flow.md +++ b/doc/design/tree/design-crowdb-tree-engine-snapshot-flow.md @@ -1,149 +1,131 @@ -# Snapshot Flow Analysis - -Snapshot makes the engine's materialized state durable on disk up to -`last_applied_slot`. The `tree.snapshot.l` metric measures the full -`Crowdbtree::snapshot()` call — CPU preparation plus synchronous I/O. - -## Call chain - -1. **Maintenance loop** (`group_maintenance.rs:234-257`) — decides - whether to snapshot based on `slot_advance`, `time_elapsed`, or - `flush_count` thresholds. -2. **Blocking wrapper** (`group_maintenance.rs:110-173`) — - `spawn_blocking(|| engine_arc.persist_snapshot())`. -3. **Rust bridge** (`crowdb_tree_engine.rs:321-339`) — calls - `handle().snapshot()` via FFI, logs `snapshot_ms` at INFO. -4. **C++ entry** (`persist.cpp:709-842`) — `Crowdbtree::snapshot()`. - -## What `snapshot()` does - -### A. Serialization gate - -`acquire_snapshot_slot()` (`persist.cpp:697-701`) — atomic spin on -`snapshot_inflight_`; only one snapshot at a time. - -### B. `prepare_snapshot_locked()` (`persist.cpp:367-663`) — CPU phase - -Runs under `write_mutex_`. Measured by `tree.mem.snapshot.apply.l`. - -1. **Allocator rebuild** (`persist.cpp:389-395`): read the best A/B - anchor, collect live extents from the previous segment directory, - build a `SpaceAllocator` of reusable gaps + append cursor. -2. **Pass 1 — fold deltas, queue dirty pages** (`persist.cpp:463-538`): - - Scan every dirty mapping-table segment (not a full tree walk). - - For each dirty leaf chain with `kBatchDelta`s: walk to the - `kLeafBase`, fold the whole chain into a fresh consolidated leaf - (`build_leaf_spilling_locked`), retire old delta/overflow pages. - - For each page: `persist_one` (`persist.cpp:406-431`) — if dirty - (`durable_addr == kNoAddr`), `encode_durable_page()` (compress + - CRC), allocate on-disk address, push to `page_writes`. If clean, - reuse existing address, increment `snapshot.page.write.cache.c`. -3. **Pass 2 — segment images + directory** (`persist.cpp:541-635`): - - For each dirty segment: pack slot words, `encode_segment_image()` - (`mapping_persist.cpp:73-96`), allocate address, push to - `segment_writes`. - - `encode_segment_directory()` (`mapping_persist.cpp:145-166`). -4. **Anchor encode** (`persist.cpp:638-655`): build `CommitAnchor` with - `last_applied_slot`, `root_page_id`, `snapshot_seq`. Select A or B - slot by parity. - -### C. Synchronous I/O (`persist.cpp:733-802`) - -1. `page_store->write_at()` for every dirty page blob (sequential). -2. `page_store->write_at()` for every dirty segment image. -3. `page_store->write_at()` for the segment directory. -4. **`page_store->sync()`** — first fsync/fdatasync barrier. -5. `page_store->write_at()` for the anchor (commit point). -6. **`page_store->sync()`** — second fsync/fdatasync barrier. - -### D. Commit bookkeeping (`persist.cpp:665-695`) - -Re-take `write_mutex_`. Update `durable_addr` / `durable_plen` / -`generation` for each page/segment. Publish new `version_`, increment -`snapshot_total_`. - -### E. Block compaction (`persist.cpp:810-830`) - -If `BlockPageStore`: delete blocks empty in two consecutive snapshots. - -## Metric observation points - -| Metric | Start | End | What | -|--------|-------|-----|------| -| `tree.snapshot.l` | `persist.cpp:714` | `persist.cpp:836-839` | Full wall time | -| `tree.mem.snapshot.apply.l` | `persist.cpp:719` | `persist.cpp:722-725` | `prepare_snapshot_locked` under `write_mutex_` | -| `tree.mem.snapshot.page.write.io.l` | `persist.cpp:734` | `persist.cpp:736-739` | Each `write_at` per page | -| `tree.mem.fsync.l` | `persist.cpp:764` / `781` | `767-769` / `784-786` | Two `sync()` calls | -| `tree.mem.snapshot.page.write.bw` | `persist.cpp:741-742` | — | Bytes per page write | -| `tree.mem.snapshot.meta.write.bw` | `persist.cpp:794-801` | — | Segment + directory + anchor bytes | -| `tree.snapshot.pages.c` | `persist.cpp:536-537` | — | Dirty page count | - -Registration: `crowdb-tree.cpp:3686-3714`. - -## Bottleneck analysis (2026-09-01 bench) - -Observed: `tree.snapshot.l` = 2027ms. Sub-phase breakdown from metrics: - -| Snapshot | `snapshot.apply.l` | `page.write.io.l` (avg/max) | `fsync.l` | pages | -|----------|-------------------:|-----------------------------:|----------:|------:| -| seq=1 (slot=9) | 532us | 67us / 67us | 0us | 1 | -| seq=1 (slot=36263) | 733ms | 388us / 129ms | 0us | 1045 | -| seq=2 (slot=159321) | 1309ms | 79us / 257us | 0us | 4167 | -| seq=3 (slot=239344) | 1304ms | 146us / 432ms | 0us | 4145 | -| seq=4 (slot=239349) | 1526us | 8us / 8us | 0us | 1 | - -Rust log wall times: 0ms, 1141ms, 2029ms, 1914ms, 1ms. - -### Primary bottleneck: `snapshot.apply.l` (CPU phase) - -The `prepare_snapshot_locked` phase dominates: 733-1309ms for snapshots -with 1045-4167 dirty pages. This is the delta-chain folding + -`encode_durable_page` (LZ4 compression + CRC) for every dirty page, -all under `write_mutex_`. - -The 2027ms total for seq=2 breaks down as: -- ~1309ms `snapshot.apply.l` (CPU: fold + compress + encode) -- ~432ms worst-case `page.write.io.l` (one slow pwrite, likely - read-modify-write bounce on O_DIRECT misalignment) -- ~286ms remaining (other pwrites + segment/directory writes + overhead) - -### Why `fsync.l` is 0ms - -The bench uses `--mode mem` (in-memory page store). `sync()` is a no-op. -On `--mode file` or `--mode block`, fsync would dominate instead. - -### Secondary: sequential pwrite with O_DIRECT bounce - -`BlockPageStore` opens with `O_DIRECT` when `iu_size_ > 1` -(`block_page_store.cpp:126-142`). Misaligned writes trigger a -read-modify-write through a scratch buffer (`block_page_store.cpp:561-582`), -explaining the 432ms outlier on a single page write. - -### `write_mutex_` blocking - -`prepare_snapshot_locked` holds `write_mutex_` for the entire CPU phase -(733-1309ms). All `apply()` calls are blocked during this time, -contributing to the frozen queue buildup (20,026 full errors). - -## Levers - -- **Reduce `snapshot.apply.l`**: the delta folding + compression is the - bottleneck. Options: - - Parallelize `encode_durable_page` across pages (CPU-bound, no - ordering constraint between independent pages). - - Skip compression for small pages or when the compression ratio is - poor (wastes CPU for no gain). - - Pre-consolidate during flush instead of deferring to snapshot. If - flush already folds delta chains, snapshot's Pass 1 has less work. -- **Overlap I/O with CPU**: write pages to disk while still encoding - the next batch. The current design is fully sequential: encode all, - then write all, then sync. -- **Reduce snapshot frequency**: increase `snapshot_slot_threshold` or - `snapshot_time_threshold_ms` so fewer snapshots occur. Each snapshot - with 4000+ pages is expensive; fewer snapshots = less total stall. -- **Move `prepare_snapshot_locked` off `write_mutex_`**: the delta - folding and encoding don't need to block `apply()`. A copy-on-write - snapshot of the dirty set could be prepared outside the lock, with - only the final commit under the lock. -- **Use async I/O** (`io_uring` backend): overlap pwrite + fsync with - the next maintenance tick instead of blocking the loop. +<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> +<!-- Licensed under the Apache License, Version 2.0. --> + +# CROWDB - Design: Tree Checkpoint Flow + +Depends on: [durable storage](design-crowdb-tree-storage.md), [Tree engine](design-crowdb-tree-engine.md) +Satisfies: [Tree overview](design-crowdb-tree.md) §3.1 + +A checkpoint turns one captured immutable Tree view into a durable backend +root. Reloadable writeback and logical handoff use related preparation but have +different completion contracts. + +## Table of Contents + +- [1. Capture and Plan](#1-capture-and-plan) +- [2. Writeback and Root Commit](#2-writeback-and-root-commit) +- [3. Rebase, Failure and Recovery](#3-rebase-failure-and-recovery) +- [4. Observation](#4-observation) + +## 1. Capture and Plan + +A checkpoint establishes a finite applied prefix through flush, then captures +an immutable published view. The operation retains its mapping, sources and +backing capabilities while later writes continue. Checkpoint serialization +uses the existing operation token; CPU and backend waits release their worker. + +`CapturedFrameStream` chooses valid durable-frame reuse or bounded page +materialization. Reuse requires matching logical content, route, retained +floor and primary layout proof. New overlays or an incompatible retention +floor invalidate the receipt. Advancing the floor remains compatible when an +exact leaf summary proves that every encoded entry is a live key: encoding +keeps one winner per key, and no tombstone remains to collect. Receipts with +tombstones or unknown live counts still require the original floor. Overflow +dependencies are retained through the +captured primary capability, not resolved against a mutable current Tree. + +The layout plan counts primaries whose existing representation already uses +the exact durable receipt. These pages need no replacement publication, so +checkpoint admission excludes them from the rebase replacement budget. +Relocation keeps the full allowance because it may change dependency owners. + +If a captured page cannot fit the durable layout, checkpoint planning releases +its current layout and phase reservations before admitting a bounded structural +split. Each split must publish progress. Planning then restarts against that +result while preserving the original generation and applied frontier; it does +not perform another handoff or include later L0 writes. A competing L1 +publication invalidates the attempt instead of silently changing its input. +The fence uses a per-generation L1 sequence covering the immutable mapping, +root, PID authority, coverage, GC floor and handoff ownership. Publishing a +successor L0 catalog changes the general commit sequence but leaves this fence +unchanged, so later source writes cannot invalidate the fixed layout proof. + +A native export returns real frames even when a checkpoint could reuse them +without reading. Portable export instead yields the captured logical entries. +Import validates the complete staged graph before publishing a replacement +generation; failure leaves the old generation usable. + +## 2. Writeback and Root Commit + +Primary-page and overflow encoding have explicit input, output, scratch and +I/O bounds. Backend tickets retain their buffers and permits until the actual +callback. Inline, delayed, duplicate and failing completions must produce one +operation terminal. + +Each accepted Chunk write retains independent prepaid credit so it can finish +after the enclosing reservation seals. Unused credit and later frees return +to the enclosing window while it is open, allowing a long checkpoint to reuse +the same transient capacity across frames; otherwise they return to the host. + +Writeback stores selected page representations and returns reloadable backing. +It may rebase a matching page while leaving checkpoint-pending membership and +durable coverage unchanged. Host idle maintenance uses finite candidate windows +and does not wait for new writes to trigger this work. + +A checkpoint writes or reuses every captured dependency, completes the required +backend barriers, then commits its root: + +- Local commits the directory and anchor through `LocalCheckpoint`. +- Chunk publishes the fenced immutable root through the catalog generation CAS. + +**I1 — Durable coverage:** only a successful root commit establishes the replay +point. Failed or ambiguous publication cannot be treated as successful merely +because some pages were written; an ambiguous result must be resolved through +the backend authority contract. + +## 3. Rebase, Failure and Recovery + +Representation rebase compares the captured page identity against the current +mapping. It may replace a captured suffix while retaining newer overlays. +Changed route/base/run identities defer the rebase instead of overwriting newer +content. Deferral does not invalidate an already successful durable checkpoint. + +An exact reused receipt needs no rebase: the captured representation already +owns that backing. This leaves the current mapping untouched, including any +later overlays, route changes or replacement pages. + +`SnapshotBaseline` retains committed primary receipts and their backing owners +without retaining the source mapping or covered overlay payload. Publication +checks its generation, capture token, coverage, GC floor and backend identity +before installing it. Representation rebase uses the existing page identity +and primary receipt; it does not install a separate durable range topology. + +Recovery validates root metadata, frame identities, ranges and dependencies +before exposing a writable generation. Readers retain the original backing +owner even if later checkpoints or collection replace the current backend root. +Physical retirement waits for those owners and the backend's actual deletion. + +Local cleanup holds an exclusive free-range lease from the extent ledger +through physical deletion, preventing concurrent address reuse. Memory block +storage releases only complete, unprotected 64 KiB backing pages; anchors, +partly protected pages and partial tails remain allocated. Cleanup advances in +bounded background windows through the same accepted operation lifetime. A +memory cleanup continues across windows without issuing another checkpoint, +preserving the logical device size and the bytes retained by live owners. +Backing capacity released by the allocator is distinct from process RSS. + +**I2 — Reader safety:** old captured views remain readable after Tree close or a +new root commit. **I3 — Failure isolation:** rejected import, failed write or +stale rebase cannot partially replace the live generation. **I4 — WAL safety:** +the host replays after the durable coverage and applies its own safe-retention +policy before truncating consensus WAL. + +## 4. Observation + +Checkpoint reports distinguish encoded/written work, durable coverage and +terminal status. Backing gauges distinguish dirty logical pages, reloadable +backing, checkpoint-pending membership and canonical physical capacity. + +A collector that outlives the Tree retains metrics only, so it does not keep +page or backend resources alive. Capture retains the immutable mapping and +its actual owners without enumerating allocations for statistics. diff --git a/doc/design/tree/design-crowdb-tree-engine.md b/doc/design/tree/design-crowdb-tree-engine.md index 5c9ad8203..45ed82e2e 100644 --- a/doc/design/tree/design-crowdb-tree-engine.md +++ b/doc/design/tree/design-crowdb-tree-engine.md @@ -3,877 +3,363 @@ # CROWDB - Design: crowdb-tree In-Memory Engine -Depends on: [`design-crowdb-tree.md`](design-crowdb-tree.md), [`../kv/design-crowdb-kv-state-machine.md`](../kv/design-crowdb-kv-state-machine.md) (apply semantics, slot rules) -Satisfies: [`design-crowdb-tree.md`](design-crowdb-tree.md) §2 (in-memory engine architecture) +Depends on: [Tree overview](design-crowdb-tree.md), [state machine](../kv/design-crowdb-paxos-kv-state-machine.md) +Satisfies: [Tree overview](design-crowdb-tree.md) §2 -This document specifies crowdb-tree's in-memory engine: the bounded 2-level -structure (concurrent MemTable over a COW B+tree), the slot-aware value cell, -the write path (apply → delta → consolidate → split/merge), the versioned root -for consistent snapshots, epoch-based reclamation, and the read path. Then the -two supporting layers that make that write/read path zero-copy: the `buffer` -memory-ownership model, and the io_uring async FFI bridge that exposes it to -Rust without blocking. - -On-disk format, backends, snapshot persistence, and recovery are in -[`design-crowdb-tree-storage.md`](design-crowdb-tree-storage.md). +This document defines source versions, immutable page publication, read +ownership, resource accounting and async completion. Frame formats and durable +root publication are specified by [durable storage](design-crowdb-tree-storage.md). ## Table of Contents - [1. The Tree Engine](#1-the-tree-engine) - - [1.1 Logical Model and the Slot-Aware Cell](#11-logical-model-and-the-slot-aware-cell) - - [1.2 Pages and Delta Records](#12-pages-and-delta-records) - - [1.3 Write Path: MemTable Ingest + Flush](#13-write-path-memtable-ingest--flush) - - [1.4 Consolidation, Split, and Merge](#14-consolidation-split-and-merge) - - [1.5 Versioned Root (Consistent Snapshots)](#15-versioned-root-consistent-snapshots) - - [1.6 Epoch-Based Reclamation](#16-epoch-based-reclamation) - - [1.7 Read Path](#17-read-path) - - [1.8 Concurrency Summary](#18-concurrency-summary) - - [1.9 Epoch-Protected MemTable (L0)](#19-epoch-protected-memtable-l0) - - [1.10 Scan Path Perf Baseline](#110-scan-path-perf-baseline) - [2. Memory and Buffer Management](#2-memory-and-buffer-management) - - [2.1 The `buffer` Abstraction](#21-the-buffer-abstraction) - - [2.2 Write and Read Pipelines](#22-write-and-read-pipelines) - - [2.3 Rust FFI Ownership](#23-rust-ffi-ownership) - [3. Async FFI Bridge](#3-async-ffi-bridge) - - [3.1 Problem and Design Principle](#31-problem-and-design-principle) - - [3.2 Architecture](#32-architecture) - - [3.3 Fast Path vs Slow Path](#33-fast-path-vs-slow-path) - - [3.4 Zero-Copy Value Across FFI](#34-zero-copy-value-across-ffi) - - [3.5 Decision Log](#35-decision-log) - ---- +- [4. Rust-Side KVEngine Async Trait Shape](#4-rust-side-kvengine-async-trait-shape) ## 1. The Tree Engine ### 1.1 Logical Model and the Slot-Aware Cell -crowdb-tree is an ordered map `key → (slot, cell)`, where `key` is an opaque byte -string compared lexicographically, `slot` is the resolved consensus slot, and -`cell` is a live value or a tombstone. It is a **bounded 2-level** structure -(`design-crowdb-tree.md` D3): - -- **L0 — MemTable.** A concurrent skip list with one node per key. CAS - publishes nodes and immutable version descriptors. Ordinary reads resolve the - highest observed slot; retained prefix versions support contiguous flush. - Payloads may borrow external buffers without copying (§2). -- **L1 — B+tree.** A **single-writer / multi-reader** B+tree with - copy-on-write base pages and a per-leaf delta chain. Inner pages hold - separator keys + child PIDs; leaf pages hold sorted `(key, slot, cell)` - entries and a `right_sibling` link for range scans. Reachable from a - `root_pid` through the mapping table - ([`design-crowdb-tree-storage.md §8`](design-crowdb-tree-storage.md#8-mapping-table)). - -L1 mutation is serialized by the existing tree write mutex. Flush publishes -only the captured contiguous prefix into L1; snapshot persistence establishes -the durable replay point. Reads merge retained L0 sources with L1 (§1.7). - -Every value carries the slot and a kind flag: - -``` -cell_payload := [slot: u64 LE][flags: u8][value bytes...] -flags bit0 = tombstone (1 → value bytes empty) -``` - -- **Highest-slot-wins:** on write, if the incoming `slot <= existing slot` for - a key, the write is skipped (idempotent). -- **Tombstone:** a delete is a cell with the tombstone flag; it occupies space - until GC removes it below the watermark - ([`design-crowdb-tree-storage.md §9`](design-crowdb-tree-storage.md#9-garbage-collection)). - `get` on a tombstone returns `None`; `iter_all` returns the tombstone (for - `compare`). - -This inlining is the single difference from a generic ordered KV engine; it is -why a stock bw-tree cannot be used unmodified. - -### 1.2 Pages and Delta Records - -A page is the unit of indexing, consolidation, and (when flushed) I/O. Leaf -pages hold sorted key/cell entries plus an optional bloom filter and -`right_sibling` link; inner pages hold separator keys + child PIDs only (no -values), so they stay small and are written only at snapshot. The concrete -on-disk/in-memory frame layout (identical representation, zero-copy) is -specified in -[`design-crowdb-tree-storage.md §3`](design-crowdb-tree-storage.md#3-on-disk-page-format-zero-copy-frame); -this doc only needs the logical shape above. - -A **flush** prepends an immutable delta to each affected leaf's chain instead -of rewriting the whole leaf. crowdb-tree needs only one data delta type, the -**batch delta**: one slot's worth of mutations targeting a single leaf, -entries sorted by key, each carrying its own cell. A single flushed slot may -touch several leaves; the Flusher produces one batch delta per affected leaf -(all stamped with the same `slot`; the slot fan-out is the *common* case, not -an edge case). Put and Delete are not separate delta types — a delete is a -tombstone cell inside the batch. SMO deltas (split/merge/index-insert) appear -only transiently during a writer-exclusive split/merge (§1.4); there is no -abort delta because there is no competing writer. - -### 1.3 Write Path: MemTable Ingest + Flush - -- `apply(slot, batch)` deduplicates each batch by key, with its last operation - winning. It selects Active, samples a pruning bound, and conditionally - registers one outstanding batch in that table's combined closed/count word. - Losing admission to closure retries against the successor. -- Independent writers publish skip-list links and version descriptors by CAS. - Writers hold epoch protection through publication, accounting and retirement - submission. Successful slot bookkeeping precedes admission release; a failed - batch cannot complete its slot. Empty batches use the same admission path. -- Encoded and external-value paths share this contract. Encoded cells must carry - the batch slot. External values retain their buffer owner; reads reconstruct - the header from the immutable slot/flags tuple. -- A validated bound D allows retaining the highest version at or below D and - every version above D. A writer samples D before successful admission. After - slot completion it may refresh D only through another successful open-state - RMW. Descriptor bounds are monotonic. Allocation failure in optional merging - leaves the successful apply intact. -- Rotation prepares the successor and pending capacity before closing Active. - Flush closes A, captures frontier F, then publishes B under source-selection - synchronization. Thus every pruning permission issued for A satisfies D <= F. - It captures a finite source set and waits only for its admitted writers. -- Prefix cursors choose each key's highest slot <= F across the captured tables, - including retained versions. L1 publication uses sorted leaf groups. Only - after every group is published does the source catalog advance its floor to F - and detach wholly covered tables. Above-F records stay in their source table; - there is no drain-and-reinsert relocation. -- A partially published flush retains its sources and old coverage floor. - Snapshot preparation is rejected until a successful flush repairs publication. - L1 publication alone does not authorize WAL reclamation; the persisted snapshot - frontier remains the durable recovery boundary. -- Size and pending-table thresholds are soft. A stalled writer, reader or slot - gap can retain memory; this path does not impose a hard write-admission limit. - Native async flush uses a completion worker, and maintenance/split callers - offload waits from executor threads. +A key maps to the highest eligible `(slot, kind, value)` cell. Slot is carried +in the value cell rather than the ordered key. Tombstones participate in slot +selection and remain visible to retention-aware iteration until the external +GC floor permits their removal. + +L0 contains concurrent MemTables with retained versions. L1 contains immutable +base references and overlay runs. A finite read/flush view fixes its source +catalog, floor and mapping; selection cannot mix sources from one publication +with pages from another. + +**I1 — Prefix retention:** an out-of-order future value or tombstone must not +hide the greatest value needed by an incomplete earlier prefix. Retention +keeps that boundary value and versions above the retained prefix. + +### 1.2 Immutable Pages and Overlay Runs + +`PageState` holds a half-open range, base references, overlay runs, content +identity, route generation and sibling information. `BaseRef` separates the +logical source from its resident representation and typed backing capability. +A range proxy may share a base without encoding or copying its payloads. + +`OverlayBatch` owns sorted `EntryRef` descriptors and bounded key arena blocks. +Each entry independently retains its `CellVersion`; keeping an overlay alive +does not require retaining the entire source skip list or version descriptor. +A leaf with only overlays is valid. Inner descriptors contain separators and +child page identities; cold inner backing remains independently reloadable. + +### 1.3 Write Path: MemTable Ingest and Flush + +A writer prepares the complete batch before publishing its first key. Its +credit covers keys, payload capacity, nodes, version descriptors, history and +reference/control storage. Failed preparation leaves keys and automatic slot +allocation unchanged. Duplicate keys in one API batch use the last operation. + +`SourceAdmission` registers writers against a source close/count word. A writer +that loses admission to closure retries against the successor; an already +admitted writer retains the resources needed to finish. Ordinary proposals and +accepted gap repair use distinct resource capacity. Replay does not discard +accepted work because the ordinary proposal queue is full. + +Prepared ordinary proposals use temporary admission status when their apply +or backend operation window is occupied. No consensus slot is chosen for that +attempt. Once armed, their lifetime tickets retain accepted work through its +last callback; a full pre-consensus window is distinct from allocation failure. + +Flush planning can reversibly pause ordinary source admission to obtain a +stable bound for all admitted writers, including those that have not applied. +The pause precedes input or automatic-slot consumption; gap repair keeps its +separate allowance. A paid owner holds the pause until source closure or +rollback. Soft rotation checks the complete flush budget for the prospective +source set and resumes on released credit without requiring another write. + +Flush captures a finite contiguous prefix, closes captured sources, publishes +the successor and asynchronously drains accepted writers. Parallel bounded +range jobs transfer independently owned cell references to L1. Installation +uses the current coordinator root and expected identities, with idempotent +install tokens. See [flush flow](design-crowdb-tree-engine-flush-flow.md). ### 1.4 Consolidation, Split, and Merge -When a leaf's delta chain exceeds `max_delta_len` (default 8) or -`max_delta_bytes` (default 256 KiB), the chain is folded into a fresh leaf -base page: replay the chain (highest slot wins per key, the authoritative -point where the slot rule is enforced; tombstones are preserved, removed only -by GC), then either build a new base page, or split/merge if the result -crosses a size threshold. - -Because there is exactly one writer, the multi-phase cooperative SMO protocol -of the pagetree reference is **dropped**: split/merge is a writer-exclusive -in-place restructure followed by atomic mapping-table stores. - -- **Split** (leaf exceeds the target page size): partition entries at a - cumulative-byte midpoint, build left/right leaves, splice `right` into the - sibling chain, insert a new separator into the parent (may recurse/split - the parent). -- **Merge** (leaf falls below `merge_threshold`, default target/4, - non-root): absorb into the left sibling if the combined size still fits, - rebuild, remove the parent's separator, free the old PID. - -Inner pages split/merge by the same logic (separator keys + child PIDs, no -values). Root collapses when it has a single child; root grows a new level -when an inner split propagates past the current root. Thresholds use a -hysteresis gap (split at target, merge at target/4) to avoid split-merge -oscillation. +Coalescing merges sorted overlay runs without materializing value bytes. Layout +analysis computes representable primary pages and bounded overflow dependencies. +Local split may retain shared base/range proxies; encoding is performed when a +representation actually requires it. Candidate pages and eligible sibling pairs +are indexed by unique identity in the immutable published state. -### 1.5 Versioned Root (Consistent Snapshots) +Maintenance uses finite visit, entry, metadata, page-output and path bounds. +Cold analysis loads one selected leaf and required routing descriptors, retains +original expected identities and publishes only if they still match. A stale +plan cannot replace newer logical contents. Repeated bounded invocations can +advance classification, coalescing, split and merge without rescanning the +entire tree from the beginning. -Ordinary point/range reads traverse the live overlay. `snapshot_view()` instead -serializes with L1 publication, records the published slot and version, and pins -L1 page chains for a stable `PinnedSnapshot`. It excludes unpublished L0 data. -Pinned pages remain alive independently of subsequent COW publication and epoch -retirement. Historical queries at an arbitrary older slot are not supported. +**I2 — Atomic accounting:** a committed structural transaction updates mapping, +root, logical contributions and candidate membership together. Retries and +unchanged classification do not duplicate counts or publish empty work. + +### 1.5 Versioned Root (Consistent Snapshots) -Snapshot persistence writes the covered L1 state and its recovery frontier to -the configured store. A stable in-memory view and a successfully persisted -snapshot have different durability guarantees. Import builds a private candidate -mapping before closing engine-generation admission. It drains old operations, -publishes mappings, sources and slot state together, then admits new operations. -Preparation failure preserves the old logical generation; borrowed allocations -survive replacement through their lifetime owners. +`PublishedTreeState` contains the mapping, source catalog/floor, coverage, +generation, commit sequence, root identity and aggregate logical statistics. +Persistent radix COW copies only touched paths and segments. A retained state +continues resolving its old page identities after later publications. + +`PublicationCoordinator` owns the bounded commit lane. Prepared transactions +validate expected pages and generation against the latest root, merge unrelated +updates and publish one new state. The final commit segment has no allocation, +cancellation or caller callbacks; completion notification follows publication. + +`PublishedSnapshot` owns a source-free view of the last complete L1 mapping +and its coverage, independent of the facade lifetime. During partial handoff, +the publication root retains that completed view alongside the changing live +mapping; capture selects both mapping and coverage in one epoch-protected +root read. Capturing a snapshot does not advance the flush frontier. Native +and portable export, range rebuild and generation replacement retain their +captured ownership through the same publication boundary. + +A split session fixes the complete L1 baseline paired with its captured source +floor. A synchronous host capture joins an in-flight flush without blocking a +runtime worker. A new session fixes its baseline only after capturing at least +the frontier observed when capture began; an older pending operation may finish +first. Later parent flushes do not change the fixed baseline. + +An existing session may already back a child's installed overlay. If intervening +parent flushes leave a gap after that session's retained sources, beginning its +shared view returns the older, continuously retained frontier. It does not claim +the parent's newer frontier is available. Callers publish the fixed cutover and +replay later slots; an explicit overlay request beyond the retained prefix is +rejected before changing the child. + +An empty, unwritten child can inherit the missing prefix by copying the captured +baseline through bounded portable work windows, reserving its complete output +shape first. The +replacement publishes independent L1 storage and retained L0 sources together +after rechecking the child root under its ingress fence. Preparation failure +leaves the child root and frontier unchanged. A nonempty child must already +cover the required L1 prefix; missing history is rejected rather than credited. ### 1.6 Epoch-Based Reclamation -Page lifetime (deletion / references / safe concurrent access) is governed by -an epoch manager **owned by each `Crowdbtree` instance** (not shared): because -the buffer pool and all retired pages are already tree-private, a per-tree -epoch is simpler and lets `get`/`scan` hand back **borrowed zero-copy views** -into resident frames that stay valid for the read guard's lifetime (§2.2). - -- A reader does `Enter()` (a single atomic store) before touching any page and - exits when its guard drops. -- The writer `Retire()`s pages it replaces (old delta chains, old base pages, - freed leaf/inner pages from split/merge, unreferenced `RootVersion` pages). -- A retired page is freed only after every thread that could hold a pointer - has left the epoch in which it was retired. - -L1 page retirement uses `retire()` under the reclamation mutex. Concurrent -MemTable writers enqueue preallocated retirement tickets through `defer()`'s -lock-free CAS list; per-record publication never takes that mutex. Maintenance -collects tickets, assigns retirement epochs and frees entries only below the -minimum active participant epoch. Idle maintenance also drives reclamation. - -**Guard entry and exit do not acquire a mutex.** First use on a thread allocates -and CAS-registers a participant, then caches it by manager identity. Subsequent -outermost entry publishes the current epoch; the outermost exit clears it. -Nested guards share that participant. Participants are dynamically registered, -not drawn from a fixed-capacity writer registry. The lifetime owner preserves -participants and pending retirements while borrowed results outlive the tree. - -**Why not sharding instead?** Sharding (multiple `EpochManager` instances, -reader hashed by thread) reduces contention but does not eliminate it. Each -shard still needs a mutex, and retire/reclaim must coordinate across shards -(take the min epoch across all of them). Lock-free EBR eliminates the mutex -entirely on the reader path, which is strictly better, for less complexity. - -**Why epoch-based reclamation and not pin counts or hazard pointers?** The -hard part of eviction/reclamation is not picking a victim; it is answering -*"when is it safe to actually reuse a page's frame bytes?"* Readers are -lock-free and read `Slice`s that point **directly into a frame**, so freeing -or reusing memory a reader still holds a pointer into is a use-after-free. -Three disciplines solve this: - -1. **Pin counts / refcounting** (textbook buffer pool): `pin++` before - touching a page, `pin--` after; evict only at `pin == 0`. Correct, but it - puts an atomic RMW on a shared cacheline on **every page touch on the read - hot path**, a real regression versus lock-free reads. -2. **Hazard pointers:** a reader publishes each pointer it dereferences into - a per-thread slot before touching it; the reclaimer scans all threads' - slots before freeing. Lock-free and tightly memory-bounded, but the reader - pays a publish (store + fence + re-validation) per pointer, and a - root→…→leaf descent needs one hazard per level. -3. **Epoch-based reclamation (EBR)**, crowdb-tree's choice for the sync read - hot path (`get`/`scan`/`get_view`). A reader brackets its **whole** - operation in one `Guard`; it never publishes which pointers it holds. - Reader cost is one enter/exit *per operation*, not per pointer. Trade-off - vs. hazard pointers: a single stalled guard blocks *all* reclamation - (worst-case unbounded retained memory), accepted because reads are short - (one `get`/`scan`) and there is a single writer, so the active-epoch - window is tiny. -4. **Per-page refcount on handoff paths**, composes with EBR as an - orthogonal cross-thread lifetime mechanism. EBR protects the same-thread - walk (the `Guard` is thread-bound); refcount extends page lifetime across - threads after the walk hands off a borrowed `Slice` (`get_async` slow - path) or a pinned snapshot (`snapshot_view` → `PinnedSnapshot`). The - per-page `pin_state_` atomic on `PageBase` is only touched on the - handoff paths (one `fetch_add` on pin, one `fetch_sub` on unpin), NOT on - the sync `get`/`scan` hot path. The §1.6 rejection of refcount ("atomic - RMW on every page touch on the read hot path") stands for the hot path. - The epoch deleter sets a `kRetiredBit` via `fetch_or`; if pins are - outstanding (count > 0), the deleter defers and the last `unpin` frees. - This decouples page lifetime from the entering thread's participant slot, - eliminating the copy in `materialize_owned` and the stale-root GC delay - when a snapshot is handed to another thread. +MemTable cursors and writers use a source's epoch lifetime to safely acquire +raw immutable nodes/version descriptors. Cross-thread handoff obtains an +independent `CellVersion` owner while still protected; it never reconstructs +ownership from a raw pointer after leaving the guard. + +Epoch participants register through CAS and cache the participant by manager +identity. Nested guards share the participant. A source retains participants +and pending retirements until its outstanding readers have left. Replaced +version descriptors use preallocated retirement tickets; per-key mutation does +not take a reclamation mutex. + +Published roots, descriptors, arrays and frame owners have independent shared +ownership. Releasing the live root only starts retirement; retained snapshots, +values, cache entries and callbacks can keep allocations alive. Host retirement +services perform bounded disposal after the last relevant owner exits. + +**I3 — Actual release:** neither logical removal nor queueing retirement reduces +physical allocation counters. The allocator/deleter or confirmed physical +backend delete is the release event. ### 1.7 Read Path -- **L0 overlay.** Capture source owners and their L1 coverage floor together, - before loading L1 pointers. Ignore L0 cells at or below that captured floor, - including stale replays admitted before a new floor was published. Among - eligible L0 candidates choose the highest observed slot; a tombstone wins by - the same comparison. A hit can borrow its value while retaining its source - epoch. If L0 has no eligible candidate, resolve the L1 chain. -- `scan(prefix, start_after, limit)` uses a **merge cursor** over L0 and the - L1 leaf chain: at each step take the smaller key, and on a key tie take the - highest observed slot. Within L1, walk the current leaf's live entries with a - `LeafChainCursor` (resolving the delta chain by highest slot), then follow - `right_sibling`. - Each cursor caches a coherent key/slot/value candidate. Ordinary scans do - not pin a common version, regardless of result size: different keys may - reflect different times or part of a concurrent batch. `start_after` - (empty = start from beginning) is an **exclusive lower bound** pushed down - into the engine: the descent targets the leaf that would contain - `start_after` (instead of the prefix start), and the merge loop skips keys - `<= start_after` natively, so a deep-pagination scan starts at the cursor - and applies the limit without over-fetching the prefix range. O(limit) - FFI + decode cost instead of O(prefix range) for a page near the end of a - large prefix. -- `scan_directional_async(..., Reverse)` is the descending order-dual. It - chooses the greatest live L0/L1 key below the exclusive continuation, - resolves collisions by highest slot, suppresses tombstones, and repeatedly - descends to predecessor leaves. Its probes never demand-load a cold page on - the caller thread: the page ID is returned to the same reactor retry loop as - forward scan. With an empty continuation, `end_key` or the prefix successor - supplies the exclusive upper bound. No previous-leaf pointer is required. -- **Lazy leaf resolution (`LeafChainCursor`).** A leaf chain is *never* - materialized for a scan. Every chain input is already key-sorted: each - `BatchDelta`'s entries, and the terminal `LeafBase`'s main frame slots, - except the in-frame delta overlay, which is bounded by `max_inframe_delta` - and pre-sorted once at cursor setup. The cursor merges those `k` streams - (`k <= max_delta_len + 2`) by linear min-key selection over the stream - heads, resolving highest-slot-wins per key as it goes, so producing one - entry costs O(k) and a `limit`-bounded scan never touches the rest of the - leaf. On an equal slot the winner is the stream visited earliest in chain - order (deltas head→tail, then the base's main entries, then its in-frame - overlay), the same resolution order a whole-chain fold would apply. - `seek()` binary-searches each stream, so the descent leaf's entries before - `start_after` / `prefix` are skipped rather than stepped over. Keys and - cells come out as Slices borrowed from the chain's own storage and are - copied only when an entry is actually emitted to the caller. - The whole-chain form (`resolve_chain_sorted`) is the same cursor drained - into an owned vector; it remains the right shape for the full-set callers - (`iter_all` / `compare`, `PinnedSnapshot::materialize`, GC's live walk). -- **Zero-copy value returns.** An **L1** hit returns a *borrowed* `buffer` - pointing into the resident leaf frame (valid only for the guard's - lifetime, §2.2). An **L0** hit also returns a *borrowed* `buffer` - pointing directly into the MemTable's skip-list node cell version. The - epoch guard keeps the node alive past any concurrent overwrite/drain, - exactly as it keeps an L1 frame resident. An overflow value (assembled - from multiple pages, no single frame to borrow) is materialized into an - owned `buffer`. -- `iter_all` traverses the live overlay and includes tombstones. Stable snapshot - export uses the separately pinned L1 view. - -Ordinary mutation does not hold a table-wide read/write token. Generation -replacement fences new operations and waits for old operations to finish; -borrowed results retain their allocation lifetime independently. +Point reads and scans capture a consistent mapping/source view, choose matching +half-open ranges, and merge eligible source/run/base cells by slot with stable +tie ordering. Tombstones suppress older values; keys-only scans avoid value +materialization. Forward and reverse cursors preserve inclusive/exclusive +bounds, prefix/end limits and resumable byte budgets. + +Cold frames are loaded asynchronously through the captured typed backing. +Reads use bounded retained-source and result windows; a scan does not reserve +or load every leaf in the mapping. Overflow chains use the captured primary +capability and have independent traversal, byte and cycle-detection bounds. +Malformed frames fail before becoming trusted descriptors. + +A fixed snapshot entry cursor has no L0 sources. Its admission bound uses the +captured mapping shape and known L1 value bytes; unknown summaries retain the +configured maximum. Later live writes cannot enlarge that fixed window, and +decoder validation limits remain unchanged. + +Owned read results retain only their necessary payload/frame owners. Packed +scan output carries its allocation owner through C and Rust, so dropping the +Tree or advancing a generation cannot invalidate returned data. + +The public result limit and byte budget apply across internal work windows; +an unlimited public limit does not stop at the first window. A strict byte +budget that cannot fit the first entry returns an empty, truncated result. +Packed output uses an independent Routing allocation account. Growth pays for +both buffers during replacement and returns an explicit resource error if the +account cannot cover the new capacity; retained output remains charged until +the caller frees it. ### 1.8 Concurrency Summary -- Ingest: per-batch table admission, CAS node/version publication and epoch - protection. Existing source-selection and slot-bookkeeping locks remain. -- Flush: serialized L1 mutation, finite old-writer wait and prefix publication. -- Reads: owned source bundle, captured coverage floor and coherent cursor - candidates. Forward/reverse and async retry paths use observed versions. -- Replacement: engine-generation admission excludes old apply/NoOp/read - operations while publishing privately prepared state. It does not reuse or - reset the outstanding count of an older table. -- Split: a journal cutover is not L1 coverage. Overlay sources remain queryable - until prefix publication; an unpublished overlay prevents destination flush - from advertising that inherited frontier. Uncovered parent sources remain - pending when split ownership is released. - -Demand loading retains the existing cold-path mutex. Allocation, source -selection and slot bookkeeping mean the complete apply operation is not claimed -to be lock-free merely because skip-list mutation is concurrent. - -- **I1 — Lifetime:** epoch borrowers and source owners jointly protect nodes, - descriptors, payloads and page chains through their final access. -- **I2 — Completion:** Frozen means admission is closed and the batch count is - zero. Reader lifetime does not delay this transition. -- **I3 — Resolution:** one node per key and coherent immutable descriptors - preserve highest-slot-wins, tombstones and equal-slot replay deduplication. -- **I4 — Scan:** observed-version scans preserve ordering and uniqueness; a - stable key cannot disappear due to flush. No common snapshot slot is promised. -- **I5 — Coverage filter:** a source bundle's floor excludes already-covered L0 - versions without changing the traversal of older bundles. -- **I6 — Publication:** L1 receives only versions within the captured contiguous - target. Complete L1 coverage and durable snapshot coverage remain separate. +Standalone C and Rust opens share the process runtime. Their per-Tree binding +still validates and enforces queue, lifecycle and resource limits; omitting an +explicit runtime does not discard that binding. Range rebuild uses the +destination binding under the same rule. + +- The host shares bounded CPU workers and backend execution; each Tree has an + independent lane/account. New nested work yields while children are pending. +- Source insertion uses CAS. Prepared writers can finish after source closure; + finite flush capture prevents new writers from extending a batch indefinitely. +- Range preparation, reads, page encoding and I/O can overlap. The publication + lane serializes only validated root transitions. +- Backend callbacks own their operation/buffer/permit state until one terminal. + Stop requests fence further dispatch and drain already accepted callbacks. +- Tree close fences ingress and stops unaccepted soft-rotation waits before + joining ingress participants. It separately drains accepted chosen/replay + work and preserves its operation owners. Generation replacement follows the + same ingress ordering; finishing replacement cannot reopen a closed Tree. + Host shutdown drains Trees, backend and retirement work before destroying + their shared runtime resources. ### 1.9 Epoch-Protected MemTable (L0) -- **Structure:** nodes contain immutable inline keys, atomic forward links and - an atomic `VersionSet*`. Level-zero CAS establishes membership; upper links - are an optimization. A paused insertion does not own unrelated keys. Height - selection uses thread-local state. Published nodes are never unlinked. -- **Versions:** each immutable descriptor owns its current `CellVersion` and - retained history through shared payload references. It stores a monotonic - pruning bound. The common one-version descriptor has no history allocation. - CAS failure recomputes selection from the winning descriptor; an older writer - cannot restore a discarded redundant prefix version. -- **Retention:** D=100 with versions 2, 80, 102 and 105 retains 80, 102 and 105. - Prefix targets 100, 102 and 105 therefore choose 80, 102 and 105. Reversed - arrival and future tombstones follow the same rule. A slot gap can retain many - versions; an admission window is not a bound on the global slot span. -- **Reclamation:** retired descriptors use embedded, preallocated queue tickets. - Apply submits ownership by lock-free push; maintenance collects tickets into - epoch retirement. It does not acquire the reclamation mutex per overwrite or - run a collection pass on the apply path. Shared payload ownership prevents - descriptor copies from freeing values still used by newer descriptors. -- **Readers:** cursor creation and advancement capture one candidate pointer. - Ranking and materialization use that same immutable tuple. Frozen tables stay - intact until fully covered and source owners are detached. Destruction queues - remaining nodes; outstanding guards retain the detached reclamation lifetime. -- **Metrics:** successful overwrite, history retention and merging events are - aggregated once per batch. Retention distinguishes a gap at admission from - temporary pending-batch retention; these are event counts, not a gap rate. - Separate gauges report live history, descriptor/node/payload charges and - allocations awaiting reclamation. Managed byte charges include object sizes - and retained buffer lengths; allocator overhead and external-owner sharing - require process-memory measurements. Concurrent gauge samples are approximate. -- **Descriptor copying:** `mt.version.copy.l` is a latency counter for rebuilding - an existing descriptor's payload-reference set, including pruning, candidates - discarded after a CAS conflict and aborted preparation. Its count is the - number of reconstruction attempts; sum/maximum measure individual attempt - durations. Samples are aggregated locally and submitted once per batch, - including failed batches. Payload bytes are shared, not copied. This is not - an I/O bandwidth metric or a count of successfully published versions. - -### 1.10 Scan Path Perf Baseline - -Scan is part of the read flow but a separate perf track from random -point reads (different cost shapes: per-entry overhead vs leaf-chain -traversal vs per-byte copy). The regression sentinel is -`tools/benchmark/bench-kv-scan-regression.sh` driving `crowdb-cli bench run --workload -list` with `--scan-limit`, `--scan-prefix`, `--scan-start-after` flags -against a 3-node mem-mode cluster, mirroring the write/read regression -sentinels. The sync `scan` `start_after` pushdown correctness is -covered by `ReadPath.ScanStartAfterCursorSkipsEarlierEntries` in -`read_path_test.cpp`. Full baseline numbers and analysis are in -`doc/design/kv/kv-scan-flow-analysis.md`; the raw TSV is in -`doc/working/bench-scan-regression.tsv`. - -Key findings from the baseline (Apple M5 Pro, macOS, 1T:1C, 100k -pre-populated keys): - -- **§1.7 O(limit) deep-pagination claim: confirmed.** Deep pagination - (`start_after` near end, limit=10) is 1.7x slower than from-start - (7084us vs 4236us): the O(log N) B+tree descent to a deeper leaf, - not O(prefix) over-fetch. If the engine over-fetched the prefix, - deep pagination would cost ~1.75s (the full-100k number), not 7ms. - The over-fetch proxy ratio is 1.7x; the etcd-style "fetch all then - truncate" regression would show ~400x. -- **Per-scan fixed cost** (~4.2ms): ReadIndex consensus round + crowdb-rpc - roundtrip + B+tree descent. Dominates at small limit (10/100). -- **Per-entry cost** (~0.3us/entry at 64B values): visible at limit - >= 10k. At limit=10k, avg=7.3ms vs 4.4ms at limit=10. -- **crowdb-rpc 4 MiB message size limit**: scans returning > 4 MiB (full - 100k keyspace at 64B values, or limit=1000 at 16KiB values) hit - crowdb-rpc's default `max_recv_message_length` and fail with transport - errors. This is the crowdb-rpc-message-size analog of etcd's range-read - OOM risk (issue #12342). A streaming scan response (mirroring etcd - PR #19766) or a raised max-message-size config is needed for - large-payload scans. -- **Value-size anomaly (resolved)**: 1KiB values scanned 3.8x faster - than 64B despite returning 16x more data. The L0-snapshot hypothesis - was refuted by per-step measurement: `l0_snapshot` is 0us in - production (the maintenance loop drains L0 before measurement), and - even a full 100k-entry 64B snapshot is only ~120us against a ~3900us - scan. The real cause was eager whole-leaf resolution: `l1_resolve` - was 99.5% of the 64B C++ scan (3985us of 4004us) and 93% of the 1KiB - scan, because a leaf's entire live entry set was rebuilt per scan - regardless of `limit`. 64B values pack ~640 entries per 64 KiB leaf - vs ~58 for 1KiB, so the dense-leaf case paid ~11x more per leaf. - The lazy `LeafChainCursor` (§1.7) makes per-leaf cost O(limit): - microbench 100k keys, limit=1000, L1-only: 64B 1183us → 19.9us, - 1KiB 476us → 32.2us; limit=10 at 64B 1163us → 0.4us. Cost now tracks - bytes returned rather than entries per leaf, so 64B is cheaper than - 1KiB, as expected. - -Per-step scan instrumentation (`Crowdbtree::scan_profile()`, the `scan.*` -metrics, and the `scan_step_profile` microbench under -`-DCROWDB_TREE_BENCH=ON`) is what made this attributable: `l0_snapshot`, -`l0_skip`, `l1_descent`, `l1_resolve`, `merge`, `total`. `l1_resolve` -now covers per-leaf cursor setup + seek only; the per-entry cursor step -is counted under `merge`, since timing each step would cost more than -the step. - -Cost-split prioritization: with leaf resolution O(limit), the remaining -scan cost is per-entry decode/copy plus the per-scan fixed cost -(ReadIndex consensus round + crowdb-rpc roundtrip + descent, ~0.4-0.6ms), -which dominates every bounded scan. The crowdb-rpc 4 MiB limit caps practical -scan width before per-byte copy becomes dominant, so response-size work -(pagination + byte budget) gates large-value scans more than per-byte -copy does. - -Future gaps (not measured by the baseline): high-concurrency -read-mode split (MinSlot vs Linearizable at > 1T:1C), and reverse scan -(forward-only today). The L0 snapshot copy (O(N_l0) per scan) was closed -by the L0 path; see §1.9. - ---- +Nodes contain immutable inline keys, atomic forward links and an atomic +`VersionSet*`. Level-zero CAS establishes membership; upper links accelerate +search. Published nodes remain linked until source retirement. Immutable version +descriptors own their current cell and retained history through shared owners. +A monotonic pruning bound prevents a losing older writer from restoring an +obsolete version set after a CAS conflict. + +For a retained prefix of 100, versions 2, 80, 102 and 105 retain 80, 102 and 105. +Views at 100, 102 and 105 therefore choose the corresponding highest eligible +version. Gaps can retain many versions; a proposal count limit is not a bound +on the total slot span. + +Source summaries and physical allocation counters have different +meanings. Logical history/frozen bytes describe source contents; physical +bytes include allocations retained by readers, history descriptors and +independent payload owners. Concurrent samples can be approximate. ## 2. Memory and Buffer Management -The write path used to copy key/value bytes **three times**: `encode_cell()` -into a `std::string`, `MemTable::upsert` storing the string, `flush()` -draining into a vector, and the frame builder writing into the frame. **Goal: -allocate key/value memory once, at the earliest point (the Rust/C API -boundary), then move it down to the MemTable and into the B+tree without -copying.** The only unavoidable copy is the final placement into the slotted -frame layout. That copy *is* page construction, not a redundant one. On the -read side, `get`/`scan` hand back a **borrowed** view into the resident frame -instead of a freshly-allocated `std::string`; the caller copies only if it -needs to outlive the read guard. - -### 2.1 The `buffer` Abstraction - -`buffer` (`lib/crowdb-tree/include/lib/crowdb-tree/buffer.h`) is a move-only byte container -that is *either* owned (frees on destruction) *or* borrowed (a non-owning view -whose lifetime is guaranteed elsewhere — a resident frame under an epoch -guard). Design rules: - -- **Move-only.** A `buffer` is never implicitly copied; passing it down the - write path is a `std::move`. Deep copies are explicit (`clone()`). -- **Two modes, one type.** The same type serves an owned write-path value and - a borrowed read-path view. A borrowed `buffer` never frees. -- **Header reserve.** `alloc(cap, header_reserve)` over-allocates by - `header_reserve` so the cell header (`[slot][flags]`, §1.1) is written in - the reserved prefix and the value bytes follow contiguously — the encoded - cell is one contiguous buffer with no second allocation or copy. -- **Small-buffer optimization (SBO) — required, not optional.** An owned - `buffer` whose total length fits `kInlineCap` (24 B) stores its bytes - **inline**, with no heap allocation, mirroring `std::string`'s SSO. This is - a *correctness-for-performance* rule: without it, replacing `std::string` - (which inlines ~15 B) with `buffer` on the write path would *regress* the - common small-key/small-value case by forcing a `malloc` where there was - none. A 9-byte cell header + up to 15 B of value stays inline. -- **Allocator seam.** `alloc()` routes owned allocations larger than - `kInlineCap` through a single internal allocator hook (today: glibc - `malloc`); a size-classed pool or RDMA-pinned allocator could slot in here - later with no call-site changes. See [`todo_code.md`](../../working/todo_code.md) for - why that hasn't been done speculatively. -- **MemTable = `absl::btree_map<std::string, cell_entry>`.** The KEY stays - `std::string`, the VALUE is a `cell_entry{slot, flags, cell}`. The - `cell` buffer is either **contiguous** (`kOwned`, full `[header][value]`, - used by snapshot import and the direct C API) or **split** (`kExternal`, - value-only borrowed from a Rust `Bytes`, the zero-copy consensus apply - path). The contiguous form is materialized at the API boundary - (`get`/`drain`/`snapshot`). **Why the key is not a `buffer`:** a B-tree stores - `pair<const Key, Value>` and *relocates* slots on node split/merge, which - requires moving the `const` key. A move-only `buffer` key falls back to - its deleted copy ctor and fails to compile. `std::string`'s SSO already - inlines small keys, so it is the correct key type; `buffer`'s SBO gives - the same inline benefit on the value side plus the borrowed read path. - `std::less<>` is transparent → heterogeneous `Slice`/`string_view` - lookup with no temporary key. +### 2.1 The buffer Abstraction + +`buffer` is move-only and supports inline, owned and externally owned storage. +Owned allocation capacity is the accounting unit, including header reserve; +an explicit clone copies bytes. External slices retain a validated full backing +identity/capacity and final drop owner. Incompatible host identity or unknown +capacity uses an admitted owned copy. + +Keys are copied into bounded node/arena storage. Payloads can remain shared +from the public API through source versions and L1 overlays. Building a durable +frame necessarily copies its selected bytes into that representation; this +encoding event is distinct from ownership transfer. ### 2.2 Write and Read Pipelines -``` -Write (consensus apply path — zero-copy): - Rust: Batch::decode produces Bytes slices (O(1) Arc ref-bump) - → CrowdbTreeEngine::apply: per-op Bytes::clone → BytesRef handle (Arc bump) - → ct_apply_batch_external: C++ wraps value as kExternal buffer (NO memcpy) - → apply_external → MemTable::upsert_external: cell_entry{slot, flags, value:kExternal} - → [payload Bytes stays alive, pinned by external buffers' Arc clones] - -Write (contiguous-cell path — snapshot import / direct C API): - → allocate one buffer (value + reserved cell header) ← the ONE allocation - → C FFI: ct_apply_put(tree, slot, key_ptr,klen, val_ptr,vlen) - → C++ wraps as buffer (copy at the boundary; §2.3) - → encode cell: write slot+flags into the reserved header (no alloc, no copy) - → MemTable::upsert(key_buf, cell_buf): std::move into the map (no copy) - -Flush (off the apply critical path): - → prefix snapshot: materialize selected cells → contiguous [header][value] (value memcpy HERE) - → frame builder copies bytes into the slotted frame layout ← unavoidable (page construction) - -Read: get(key): - guard = epoch.enter() // keeps resident frames alive (§1.6) - L0 (MemTable) hit: must COPY (materializes split cell into [header][value]; one copy) - L1 (B+tree) hit: return buffer::wrap(cell_value_ptr_in_frame, len) // BORROWED, zero copy - valid only while `guard` is held AND the frame stays resident -``` - -- **Apply critical path: zero value memcpy.** The value bytes are borrowed - from the Paxos payload `Bytes` via `kExternal` buffers; the `memcpy` is - deferred to flush (off the critical path). The total value-copy count is - 2 (materialization at flush + frame construction), both on the Flusher - thread, none on the apply thread. -- **L1 reads are zero-copy:** the returned `buffer` borrows the cell's value - bytes directly from the resident frame. **The epoch guard alone keeps the - frame resident.** Eviction does not free a page's frame directly, it - re-tags the mapping slot unloaded and epoch-retires the page, whose - destructor (which releases the frame) only runs after the epoch reclaims - it. A reader holding a guard therefore keeps the page alive, so the frame - stays pinned and the buffer pool's CLOCK sweep skips it; no separate pin - needs to be bundled into the returned handle. -- **Caller contract:** a borrowed `buffer` is valid only while the caller - holds the read guard. To retain a value past the guard, the caller calls - `clone()` (owned copy) and releases the guard. +`AllocationLease`, allocation reservations and typed credits associate real +storage with a Tree account and its host budget. Reserved credit converts into +allocated bytes without charging both simultaneously. Unused credit returns on +rollback or sealing; allocated bytes remain until their actual deallocation. + +Handoff, Routing, Reclaim, admitted-writer and gap-repair partitions prevent +ordinary growth from consuming all capacity needed to complete accepted work. +A complete reclaim operation receives its inputs, output, scratch, publication +and callback allowance before starting. I/O operations use separate permits. ### 2.3 Rust FFI Ownership -Three options for how Rust hands value memory to C++: - -- **Option A — copy at the boundary.** Rust passes `*const u8` + len; C++ - `buffer::alloc()`s and copies once at the FFI boundary. Simpler; one - copy remains at the boundary but the *internal* pipeline is zero-copy - (§2.2). Used by the simple single-op C API (`ct_apply_put`, - `ct_apply_batch`, `ct_apply_batch_slices`). -- **Option B — handle-based ownership transfer.** The caller - allocates crowdb-tree-owned memory via `ct_alloc(key_len, val_len)`, which - returns an opaque `ct_write_handle` plus two writable pointers (key + - value). The caller writes key/value bytes directly into that memory, - then `ct_apply_put_owned(tree, slot, handle)` writes the cell header - (slot + flags) into the pre-allocated cell and moves key+cell into - `apply_encoded` — zero value memcpy. `ct_free_handle(handle)` is the - error/cancel path. The cell header layout (`kCellHeaderSize`) is - internal — the C API never exposes it, sidestepping the ABI-stability - concern that originally blocked this option. The Rust adapter wraps the - handle in a `WriteHandle` struct with RAII `Drop` (frees if not - consumed) and `apply(self, slot)` (consumes). `!Send + !Sync`. SBO is - preserved: small values (≤ 15 B) use inline storage, same as Option A. - Intended for direct API callers that want zero-copy single-op apply. -- **Option C — external borrow from Rust `Bytes`.** The consensus - apply path (`CrowdbTreeEngine::apply`) uses `ct_apply_batch_external`: - each Put op's value `Bytes` is cloned (O(1) `Arc` bump) into a - `BytesRef` handle handed to C++. C++ wraps the value bytes as a - `kExternal` `buffer` (borrowed, no memcpy) and stores a **split cell** - in the MemTable — `cell_entry{slot, flags, value:kExternal}` — where - the 9-byte cell header is stored as fields, not as adjacent bytes. The - contiguous `[header][value]` cell is materialized at the MemTable API - boundary (`get`/prefix snapshot/`snapshot`), where a copy already exists, - all off the apply critical path. When the `kExternal` buffer is freed - (drain/overwrite), C++ calls back into Rust (`ct_release_bytes`) to - drop the `Arc<Bytes>` clone, keeping the payload allocation alive until - every borrowing buffer is freed. The `kExternal` mode adds zero size - overhead to `buffer` (the drop-control fields overlap the SBO `inbuf_` - array via a union). See §2.4. - -Ownership rule (extends `design-crowdb-tree.md §4`): a buffer *yielded* to -`ct_apply_*` is owned by the tree and freed by it once flushed into a frame; -a buffer *borrowed* into a call (not yielded) follows the existing "borrowed -for the call's duration" rule. An **external** buffer (Option C) is -borrowed from Rust and kept alive by an `Arc<Bytes>` refcount — C++ calls -back into Rust to drop the ref when the buffer is freed (drain/overwrite), -so the payload allocation survives until every borrowing buffer is released. - -### 2.4 Zero-Copy Apply: Split-Cell External Buffers - -**Design challenge.** The consensus apply path receives value bytes as -`Bytes` slices into the packed Paxos payload, a single refcounted -allocation shared by all ops in the batch. The contiguous-cell apply path -(Option A/B) calls `encode_cell_buf`, which performs a value `memcpy` -(64 KiB for a large value) on the apply thread, the dominant per-op cost -for large values. The goal is to eliminate this copy for the consensus -apply path without changing the contiguous cell representation downstream. - -**Why a naive external buffer doesn't work.** The cell is stored as one -contiguous `buffer` `[9-byte header][value]` everywhere (`mem_entry.cell`, -`leaf_entry.cell`, B+tree frames, snapshot I/O). `CellView` reads -`slot()`/`flags()`/`value()` by slicing this contiguous memory. A -`Bytes`-backed external buffer cannot satisfy this contiguity: the value -bytes live in the packed payload `Bytes`, and the 9-byte header (known only -at apply time) is not adjacent to them. - -**Design.** Split the cell **only while it lives in the MemTable**; -materialize the contiguous form at the memtable API boundary -(`get`/prefix snapshot/`snapshot`), points where a copy already exists, all -off the apply critical path. Everything downstream (flush, delta, frame, -`leaf_entry`, `CellView`, snapshot I/O) sees contiguous cells unchanged. - -- **`buffer::mode::kExternal`** — a buffer mode that borrows value bytes - from a Rust `Bytes` and calls `drop_fn(owner)` on destruction to - decrement the Rust refcount. The drop-control fields (`drop_fn`, `owner`) - overlap the SBO `inbuf_` array via a union — zero size overhead. - `clone()` deep-copies the borrowed bytes into an owned buffer; move - transfers `ext_` and releases the source without calling `drop_fn`. -- **`cell_entry`** — the MemTable's internal map value: `{slot, flags, cell}`. - Contiguous: `cell` is `kOwned` (full `[header][value]`). Split: `cell` is - `kExternal` (value-only, borrowed). Tag: `cell.ownership() == kExternal`. - `materialize()` builds a contiguous `[header][value]` buffer from the - fields + borrowed value (one value `memcpy`, off the apply hot path). -- **`apply_external(slot, vector<external_op>)`** — same semantics as - `apply_encoded` (oversized-key rejection, slot bookkeeping, - `maybe_swap_active`, intra-batch last-key-wins) but builds `cell_entry` - directly — no `encode_cell_buf`, no value `memcpy`. -- **`ct_apply_batch_external`** — C API: `ct_ext_op{key, value, kind, - bytes_ref, drop_fn}`. Rust hands a `BytesRef` (boxed `Arc<Bytes>`) per Put - op; C++ wraps the value as `kExternal` and calls `drop_fn(bytes_ref)` when - the buffer is freed. -- **`CrowdbTreeEngine::apply`** — unchanged `KVEngine::apply` trait; builds - `ExtOp`s from the `Batch`'s `Bytes` slices (O(1) `Arc::clone` per op) and - calls `apply_batch_external`. WAL replay goes through the same - `learner.learn` → `apply_entry` path, so it benefits automatically. - ---- +Known Rust `Bytes` allocations cross the ABI with a stable backing identity, +capacity and drop callback. Same-host slices share the payload charge; external +control blocks, foreign owner/context and slice holders remain separately +charged allocations. Failure/cancellation drops each transferred owner once. + +`PinnedValue::into_bytes` retains a native result owner through `Bytes::from_owner`. +Packed scan owners follow the same principle. A shared output may be dropped +on another thread or after the facade closes without retaining an unrelated +source index or changing the charged capacity to the slice length. + +### 2.4 Allocation Accounting and Ownership + +Allocation owners charge their actual capacity once and return it on final +release. Shared cell, frame and external-buffer owners preserve that charge +across source handoff, immutable views and Tree closure. Epoch guards protect +borrowed accesses until reclamation is safe. + +Snapshot capture and split source installation retain existing immutable +owners. They do not enumerate allocations or create per-reference statistics +tokens. Source summaries describe current logical contents; allocation and +budget counters describe physical occupancy and admission. + +### 2.5 Work and Runtime Metrics + +Each Tree records fixed stage summaries, terminal outcomes, event counters and +bounded observations of unfinished work. Queue wait, execution and end-to-end +pipeline durations describe different intervals; durations of parallel children +must not be interpreted as elapsed time for their parent. When age observation +slots fill, the sample reports incomplete coverage without rejecting work. + +Budget request waits are separated into Handoff, Routing, Reclaim, admitted +writer, gap-repair and I/O classes. A wait runs from its first submission to its +single grant, error or cancellation terminal; direct non-waiting reservations +are separate. Shared-cache latency counts requests joining a load already in +flight, ending after each waiter's own proof and pin validation. First loaders +and ready cache hits are excluded from that wait summary. + +One collector owns destructive window sampling and shares an immutable result +with logs and diagnostics. Cumulative samples never drain. Every supported C +capture version advances the same window, including enabled distributions that +an older output structure cannot expose. Versioned structures validate all +headers before changing a window or caller output. + +Histograms are explicitly enabled for one Tree and one stage. They allocate no +buckets by default; enabling charges the real fixed bucket storage to that +Tree's Routing account. Concurrent enable calls retain one instance, and failed +enable leaves business state unchanged. Terminal observations use atomic updates +without allocating. Enabled storage survives with the metrics owners and is +released with the last owner. Quantiles come from the distribution; they are not +estimated from the summary average or maximum. + +The shared host runtime also exposes cumulative CPU and backend executor queue, +execution and rejection counters, alongside reserved/allocated resources and +budget waits. These samples cover all Trees using that runtime. They are relaxed +cross-field observations, require no backend I/O, and remain readable after its +Trees close while the runtime handle is retained. + +Periodic resource windows integrate non-draining samples to report average +and sampled maximum occupancy. Tree samples include backend, ordinary apply +and replay operation slots independently of runnable CPU queues. They also +include the active source's ordinary/gap entry and key-byte allowances, +including admitted writers that have not materialized records. These source +counts reset on source rotation and do not measure retained older sources. +Operation capacity denials are cumulative; source contention counters are not +measured. Sampling age, gaps and errors accompany the occupancy windows. ## 3. Async FFI Bridge -### 3.1 Problem and Design Principle - -Wrapping every synchronous C++ engine call in `tokio::task::spawn_blocking` -has two problems: (1) every `get`/`apply`/`scan` hops through Tokio's blocking -thread pool, and the ~5–10 μs scheduling overhead is significant relative to -a μs-level in-memory operation; (2) a high-performance engine must not block -async workers or rely on large thread pools for I/O. `spawn_blocking` is a -workaround for synchronous I/O, not a long-term architecture. - -**Design principle: completion-based async I/O via io_uring. No blocking, no -thread pools, no C++ coroutine dependency.** - -- **C++ coroutine (`co_await`) is not used.** It would add compiler/runtime - surface and lifetime complexity at the C ABI without improving the kernel - I/O path. The boundary still needs an opaque handle Rust can poll, so - exposing coroutine state machines across FFI is the wrong abstraction. -- **Folly futures are not used.** They bring a large dependency stack and an - executor model that duplicates Tokio on the Rust side; crowdb-tree only needs - a small completion object plus a notification fd. -- **epoll is only the readiness fallback.** io_uring handles disk/block I/O; - `eventfd` handles C++→Rust wakeup, and Tokio already integrates that fd via - `AsyncFd` (epoll internally), so no separate epoll event loop for storage I/O. - -### 3.2 Architecture - -``` - Tokio async runtime C++ engine - ┌──────────────────┐ ┌──────────────────────────┐ - │ async fn get() │ │ ct_get_async() │ - │ CtGetFuture │──poll()───────►│ fast path? ──yes──► done│ - │ .poll() │ │ │ no │ - │ │ pending │◄──done=0───────│ submit io_uring SQE │ - │ │ register │ │ return pending │ - │ │ waker on │ │ │ - │ │ AsyncFd │ │ polling thread (1) │ - │ │ (eventfd) │◄──eventfd──────│ io_uring_enter loop │ - │ │ wake │ │ CQE → callback │ - │ .poll() again │──poll()───────►│ ct_future_poll → done=1 │ - │ → Ready(result) │ │ return ct_buf │ - └──────────────────┘ └──────────────────────────┘ -``` - -One DiskIOUring per `Crowdbtree` instance, on a dedicated C++ thread: it calls -`io_uring_enter` (blocking until a CQE arrives or a timeout fires), peeks -CQEs, dispatches each completion callback, then writes to its `eventfd` to -wake the Rust side. The Rust side wraps that `eventfd` in Tokio's `AsyncFd`; -a `ct_future` handle carries the pending state, result, and (for the fast -path) an `EpochManager::Guard` keeping the frame resident (§3.4). - -### 3.3 Fast Path vs Slow Path - -| Operation | Fast path (sync) | Slow path (io_uring) | -|-----------|-----------------|---------------------| -| `get` | L0 MemTable hit, L1 resident page hit | L1 cache miss → demand-load from disk | -| `scan` | All pages resident | Any page needs demand-load | -| `apply_put` / `apply_delete` | MemTable insert (no flush triggered) | Triggers flush → write to disk | -| `flush` / `snapshot` | — | Always: write dirty pages to disk | -| `snapshot_view` | All pages resident | Any page needs demand-load | - -The C++ engine determines the path internally: if the operation can complete -without I/O, it fills the future synchronously (`state = kDone`) and returns; -otherwise it submits SQE(s) to the DiskIOUring (`state = kPending`) and the -DiskIOUring completes the future when I/O finishes. A `CtFuture`'s first -`poll()` on the fast path resolves immediately, zero scheduling overhead, -no thread switch; on the slow path it registers a waker on `AsyncFd` and -resolves on the next DiskIOUring-driven wakeup. - -### 3.4 Zero-Copy Value Across FFI - -- **Fast path (PinnedValue):** the `ct_future` holds an - `EpochManager::Guard` that keeps the frame resident; - `ct_future_poll` returns a `ct_buf` pointing directly into the frame - bytes (borrowed, not owned). `AsyncCrowdbtree::try_get_pinned` wraps - this in a `PinnedValue`, a `!Send` RAII type that holds the - `ct_future` handle so the epoch guard stays alive until `PinnedValue` - is dropped. `PinnedValue::as_bytes()` borrows directly from the C++ - frame with no `copy_buf` allocation. The `KVEngine::get_bytes` trait - method (default impl delegates to `get` + `Bytes::from(vec)`; - `CrowdbTreeEngine` overrides to use `try_get_pinned`) copies from - `PinnedValue` into `Bytes` before dropping the pin, eliminating the - intermediate `Vec<u8>` allocation. Total: one copy (frame → `Bytes`), - one allocation (`Bytes`), no `Vec<u8>`. -- **Slow path:** the I/O read fills a C++-owned buffer; on completion - the future returns it as an owned `ct_buf` (C++ allocates, Rust - frees). For frame-borrowed values, the zero-copy path replaces the copy with a per-page - refcount pin: the polling thread pins the leaf base page(s) under its - epoch guard, releases the guard, and hands the `GetView` (still borrowing - the frame) across threads; `ct_future_free` unpins from the dropping - thread. `materialize_owned` remains only for overflow-chain values - (assembled from multiple pages, no single frame to borrow). The Rust - side then copies from the borrowed/owned buffer into `Vec<u8>` / `Bytes`. -- **Key copy elimination:** `AsyncCrowdbtree::try_get` / - `try_get_pinned` accept `&[u8]` instead of `Vec<u8>`, since - `ct_get_async` copies the key internally into a - `std::shared_ptr<std::string>`. The Rust-side `key.to_vec()` is - eliminated, no C++ changes required. -- **True zero-copy:** `PinnedValue::into_bytes()` creates a `Bytes` - via `Bytes::from_owner` backed by the C++ frame, no copy. The - `ct_future` handle (and its page refcount pins) is held by the `Bytes` - owner; when the last `Bytes` ref clone is dropped on any thread, the - owner's `Drop` runs, which drops the `PinnedValue`, which calls - `ct_future_free` to unpin. `PinnedValue` is `Send` (the per-page - refcount is thread-independent), enabling this cross-thread zero-copy - handoff. `CrowdbTreeEngine::get_bytes` uses this path directly. - -### 3.5 Decision Log - -| ID | Decision | Rationale | -|----|----------|-----------| -| D-Async1 | **Async FFI via io_uring + completion-based futures.** No `spawn_blocking`, no large thread pools, no C++ coroutine/Folly dependency. | A `ct_future` handle is the correct FFI abstraction for Rust polling; C++ coroutine and Folly executor state must not cross the C ABI. Fast path completes synchronously with zero scheduling overhead. | -| D-Async2 | **One polling thread per `Crowdbtree`.** | Each tree has its own buffer pool and I/O; a per-tree DiskIOUring keeps I/O completions local and avoids cross-tree coordination. The DiskIOUring does no application logic — only CQE dispatch. | -| D-Async3 | **`eventfd` for Rust↔C++ notification.** | Simplest kernel-level notification: one `write(8 bytes)` wakes Tokio's `AsyncFd`. Level-triggered, no missed events. Falls back to a pipe + `kqueue` on macOS. | -| D-Async4 | **macOS dev path: in-memory store, no io_uring.** | io_uring is Linux-only. For dev/testing on macOS, the in-memory store completes synchronously (fast path only). Production runs on Linux with real io_uring. | - ---- - -## 4. Rust-Side `KVEngine` Async Trait Shape - -> **Merged from `design-crowdb-kv-async-kvengine.md` (2026-07).** -> **Status:** implemented (landed 2026-07-09). -> This section records the design of the Rust-side `KVEngine` trait's async -> shape (`KVFuture<T>`) and why it looks the way it does, kept as the -> rationale record for a decision that is easy to get wrong (the naive -> `async-trait` conversion), not as a live plan. - -### 4.1 The Problem - -`KVEngine` is consumed as `Box<dyn KVEngine>` (`PxLearner::engine`), chosen at -runtime via a CLI flag (`--kv-engine {memory,crowdb-tree}`). Its `get`/`scan`/ -`apply` need an async-capable return type so that a genuine I/O path -(crowdb-tree demand-load miss, served by the io_uring engine, §3 "Async FFI -Bridge") can suspend instead of blocking a Tokio worker thread on a -synchronous `pread`. That is the exact anti-pattern the DiskIOUring exists to -avoid at the C++/FFI layer, which would otherwise resurface immediately one -layer up in Rust. - -### 4.2 The Central Tension: `dyn KVEngine` vs. `async fn` in Traits - -Native `async fn` in traits is **not `dyn`-compatible**. Three ways to square -that with runtime engine selection: - -| Option | Cost | -| --- | --- | -| **(a) `async-trait` crate** | Boxes every async call into a `Pin<Box<dyn Future>>` via macro, one heap allocation **per call, including the fast in-memory path with no I/O**. Undoes the DiskIOUring's "fast path costs nothing" property one layer up. | -| **(b) Generic `PxLearner<E: KVEngine>`** | Zero overhead, but `E` would need to propagate through `PxLocalReplica`, `PxGroup`, `PxKvStore`, and `DashMap<GroupId, PxGroup>`, too invasive for an engine chosen at runtime. | -| **(c) Hybrid fast-path/slow-path future (chosen)** | Plain (non-`async`) `fn`s return a small custom future enum that resolves immediately (no allocation) for the fast path and only boxes a real future for the rare slow (I/O) path. Fully `dyn`-compatible; mirrors the exact fast/slow split the C++ layer already makes. | - -### 4.3 `KVFuture<T>` and the Trait Shape - -```rust -pub enum KVFuture<T> { - Ready(Option<T>), // taken on first poll; re-polling panics - Pending(Pin<Box<dyn Future<Output = T> + Send>>), -} - -impl<T> KVFuture<T> { - pub fn ready(v: T) -> Self { KVFuture::Ready(Some(v)) } -} - -pub trait KVEngine: Send + Sync { - fn apply(&self, slot: u64, batch: &Batch) -> KVFuture<()>; - fn get(&self, key: &[u8]) -> KVFuture<Option<(u64, Vec<u8>)>>; - fn scan(&self, prefix: &[u8], limit: usize) -> KVFuture<(Vec<(Vec<u8>, u64, Vec<u8>)>, bool)>; - // iter_all / clear / compare / resume_from_slot / - // persist_snapshot / set_gc_watermark / compact_sparse_blocks: plain sync fns. - // Diagnostic/maintenance-path only (compare/iter_all: tests + snapshot - // export; the rest: restore path + the periodic group-maintenance - // task), never on the hot Paxos-accept / crowdb-rpc-read path, so a brief - // blocking call there is an acceptable trade-off. -} -``` - -`Ready` costs nothing beyond the enum tag + inline value, no allocation, no -`Pin<Box<..>>`. `InMemKV` always returns `Ready`. `CrowdbTreeEngine::get` -(`lib/crowdb-kv/src/kv/crowdb_tree_engine.rs`) does the same fast-path check the C++ -layer does first, via `crowdb_tree_ffi::AsyncCrowdbtree::try_get`; on a resident -hit/miss it returns `Ready` at zero extra cost, and only on a genuine -demand-load miss does it construct `Pending`, wrapping the DiskIOUring-driven -future `try_get` already builds. `CrowdbTreeEngine::scan`/`apply` always -resolve `Ready` today (no async `scan`/`apply` C API exists yet, an honest -gap, not an oversight; see `CrowdbTreeEngine`'s doc comment). - -### 4.4 Caller-Side Wiring - -`Learner::learn` is a native `async fn` (mirroring `Acceptor`'s existing -`async fn accept`/`prepare` convention; neither trait is ever used as -`dyn`, so native `async fn` is safe for both). `PxLearner::apply_entry`/ -`engine_get`/`engine_scan` are `async fn` and `.await` the `KVFuture` -directly. `PxLocalReplica::learn_chosen`/`apply_committed_up_to` and -`PxKvStore::kv_get`/`kv_scan` were already `async fn` (already awaited by -every caller) and now genuinely `.await` a future that can suspend, instead -of always resolving on the first poll. - -`engine_scan`/`apply_entry` stay `async fn` for signature uniformity with -`engine_get`, but never actually suspend today, matching §4.3's note that -`scan`/`apply` have no async C API yet. - -### 4.5 Testing - -- `kv/kv_future_test.rs`: `KVFuture::ready(v)` polls to `Ready(v)` on the - first poll without registering a waker; a `Pending` variant polls through - to the wrapped future's result; polling a completed `Ready` again panics. -- `InMemKV` regression guard (`mem_kv_test.rs`): every call returns `Ready` - (asserted via `matches!`, not just the unwrapped value, so an accidental - switch to `Pending` fails loudly here first). -- `CrowdbTreeEngine` regression tests (`crowdb_tree_engine_test.rs`): a resident - hit/miss returns `Ready`; `get_constructs_pending_for_genuine_demand_load_miss` - evicts a durable engine's resident leaf and asserts `get` returns `Pending` - and resolves to the right value. -- `paxos/learner_async_test.rs`: the same fast/slow property one layer up, - through `PxLearner::engine_get`. +Private sender operations compose bounded CPU jobs and asynchronous backend +callbacks. Public headers expose opaque handles, status/result structures and +callbacks, without leaking sender templates. The synchronous facade waits for +its operation completion; async callers use native futures directly. + +A native future retains accepted state and uses a notification fd to wake Rust. +Rust checks readiness, registers an independently owned `AsyncFd`, and rechecks +completion to avoid lost wakeups. Closing a former reactor cannot deregister a +new native descriptor that reuses the same fd number. + +The first poll may already find a completed operation; callers must also handle +pending CPU dispatch even for resident data. Tree work is not implemented by +creating a new OS thread per flush or by wrapping all calls in `spawn_blocking`. + +**I4 — Terminal ownership:** cancellation is a request, not permission to free +accepted buffers. Exactly one success/error/stopped terminal releases the +operation's remaining credits after its children and backend callbacks drain. + +## 4. Rust-Side KVEngine Async Trait Shape + +`KVFuture<T>` represents either an immediate result or a pending boxed future, +allowing object-safe `KVEngine` calls without requiring a heap future for every +immediate completion. `CrowdbTreeEngine` adapts owned native operations to that +contract and retains one metrics collector for its lifetime. + +Proposal admission produces a grant that survives the consensus decision and +subsequent apply. Accepted apply/replay and NoOps drive the contiguous frontier; +ordinary proposal backpressure cannot consume the gap-repair allowance. + +Typed maintenance reports distinguish handoff, writeback, checkpoint and +retirement progress. The host's finite idle loop checks actual candidates and +backend availability, so stopped writes can still converge without advancing +WAL durability from representation-only work. diff --git a/doc/design/tree/design-crowdb-tree-storage.md b/doc/design/tree/design-crowdb-tree-storage.md index e5cbc9058..c6df17a15 100644 --- a/doc/design/tree/design-crowdb-tree-storage.md +++ b/doc/design/tree/design-crowdb-tree-storage.md @@ -3,823 +3,329 @@ # CROWDB - Design: crowdb-tree Durable Storage -Depends on: [`design-crowdb-tree.md`](design-crowdb-tree.md), [`design-crowdb-tree-engine.md`](design-crowdb-tree-engine.md), [`../kv/design-crowdb-kv.md`](../kv/design-crowdb-kv.md) §12.1, [`../kv/design-crowdb-kv-state-machine.md`](../kv/design-crowdb-kv-state-machine.md), [`../kv/design-crowdb-kv-wal.md`](../kv/design-crowdb-kv-wal.md), [`../kv/design-crowdb-kv-reconfiguration.md`](../kv/design-crowdb-kv-reconfiguration.md) -Satisfies: [`design-crowdb-tree.md`](design-crowdb-tree.md) §2 (durable storage architecture) +Depends on: [Tree overview](design-crowdb-tree.md), [Tree engine](design-crowdb-tree-engine.md) +Satisfies: [Tree overview](design-crowdb-tree.md) §3.1 -This document specifies how crowdb-tree pages reach durable media and how that -durability composes with the rest of CROWDB: the `PageStore` backend -abstraction, the on-disk page format and alignment, the buffer pool (frame -cache), snapshot + the internal-WAL decision + recovery, snapshot -export/import, the mapping table (the B+tree's PID indirection layer and its -own persistence), and the snapshot/GC flow integration with the learner and -consensus WAL. +This document specifies page storage, durable frame encoding, cache ownership, +checkpoint/recovery and physical reclamation. The [Chunk storage design](design-crowdb-tree-chunk-storage.md) +defines its catalog and mirrored pack protocol. ## Table of Contents - [1. PageStore Abstraction](#1-pagestore-abstraction) - [2. Backends](#2-backends) - - [2.1 TextPageStore — On-Disk Layout (Debug)](#21-textpagestore--on-disk-layout-debug) - - [2.2 BlockPageStore — On-Disk Layout (Production)](#22-blockpagestore--on-disk-layout-production) - - [2.3 Async I/O Architecture](#23-async-io-architecture) - - [2.4 fsync Policy](#24-fsync-policy) - - [2.5 Block Compaction](#25-block-compaction) -- [3. On-Disk Page Format (Zero-Copy Frame)](#3-on-disk-page-format-zero-copy-frame) -- [4. Buffer Pool (Frame Cache)](#4-buffer-pool-frame-cache) +- [3. On-Disk Page Format](#3-on-disk-page-format-zero-copy-frame) +- [4. Buffer Pool and Cache Residency](#4-buffer-pool-and-cache-residency) - [5. Internal WAL Decision](#5-internal-wal-decision) - [6. Snapshot, Recovery, and Export/Import](#6-snapshot-recovery-and-exportimport) - [7. Interaction with Consensus WAL GC](#7-interaction-with-consensus-wal-gc) - [8. Mapping Table](#8-mapping-table) - [9. Garbage Collection](#9-garbage-collection) - [10. New-Member Install Flow](#10-new-member-install-flow) - ---- +- [11. Root and Physical Metrics](#11-root-and-physical-metrics) ## 1. PageStore Abstraction -crowdb-tree's tree logic references pages by `PID` and is unaware of the storage -medium. A `PageStore` maps a durable page slot to bytes; it is the only part -of crowdb-tree that does I/O, and it is page-granular and **always asynchronous** -(`read_page`/`write_page` complete via callback/future, matching -[`../kv/design-crowdb-kv-wal.md`](../kv/design-crowdb-kv-wal.md)). -The upper layer always uses the async API, regardless of whether the -underlying platform has `io_uring` (Linux) or not (macOS/fallback). On -Linux, `BlockAsyncPageStore` + `DiskIOUring` submit genuine `io_uring` SQEs; -on macOS or without liburing, the `*_async` methods fall back to -synchronous blocking I/O wrapped as immediately-ready completions, so the -upper layer has a **unified async interface** with no sync/async split. -No FFI on the I/O hot path. An injected store may use immutable chunk page -packs; its manifest, recovery, and range-sharing rules are specified in -[`design-crowdb-tree-chunk-storage.md`](design-crowdb-tree-chunk-storage.md). - -**IU = Indivisible Unit.** The minimum atomically-writable size. Leaf base -pages are padded to a multiple of it so a page write cannot tear (§3). The IU -is backend-configurable and may be as small as **1 byte** for byte-addressable -media (in-memory test store, SCM) or a flash page (SSD); see §2. A small -**page-allocation map** (free IUs in the backing file/device) is persisted -alongside the root pointer in the commit anchor (§6). - ---- - -## 2. Backends +The engine identifies logical pages independently of their stored address. +`BackingRef` carries the storage domain, address/length, content/fence proof and +an owning capability. A captured reader resolves dependencies through that +capability, including overflow owned by an older checkpoint. -There are **three** `PageStore` implementations: a text-encoded debug store, -a block-device store, and an injected chunk store. RDMA is **not** a separate -backend: a remote page region is just a block device reached over the network, -served by the same `BlockPageStore` with an RDMA medium driver. +`AsyncPageStore` expresses accepted asynchronous work through completion +callbacks. `ScheduledPageStore` accounts host I/O and keeps backend ownership +through callbacks. Synchronous stores use the host's backend executor; native +asynchronous stores use their completion interface. A Tree does not create a +private executor for each operation. -| Backend | Medium | IU | Notes | -| --- | --- | --- | --- | -| `TextPageStore` | Local filesystem directory | 1 byte (always) | Debug/test backend. Each page, anchor, and segment image is a separate human-readable text file. No compression. Implements `PageStore` with `iu_size()=1`; internally maps addresses to filenames. | -| `BlockPageStore` | Raw block device or regular file, served by a pluggable medium driver: **SSD** (`O_DIRECT`), **SCM** (byte-addressable), **mem** (test), **RDMA-remote** (one-sided verbs to a remote region) | SSD: 16/64 KiB; SCM/mem: down to **1 byte**; RDMA: the remote region's IU | Production backend. Array-of-blocks growth: a group owns multiple fixed-size block files (`{path}.blk-{NNNN}`), allocated on demand. No filesystem; the allocation map owns the whole device/region. Byte-IU media skip page padding. | -| `ChunkPageStore` | ChunkDB mirror strips through DiskIO | 64 KiB | Injected backend. Writes immutable checksummed packs, resolves tagged mapping references through immutable manifests, and publishes roots through a generation-fenced catalog. See the [chunk-storage design](design-crowdb-tree-chunk-storage.md). | +`CheckpointBackend` separates reloadable page writeback from durable root +commit. A stored page can support eviction before it belongs to a committed +checkpoint. Only root commit establishes a durable replay point. -All implement the same async `PageStore`. Local backends are selected at -`ct_open` -via `ct_options.backend` (0 = text debug, 1 = block). When `path` is -null/empty, an in-memory `BlockPageStore` with IU=1 is used (test path). -The chunk backend is constructed separately and injected through the opaque -page-store handle. +**I1 — Domain ownership:** incompatible runtime/backend bindings are rejected. +Opening a second writable Tree on the same owned backend domain cannot bypass +the first owner merely by constructing another facade. -### 2.3 Async I/O Architecture +## 2. Backends -`BlockPageStore` has an async twin, `BlockAsyncPageStore`, that submits all -I/O through a `DiskIOUring` (io_uring event loop). The async path is used by -`ct_get_async`, `ct_flush_async`, and `ct_snapshot_async`. +- `TextPageStore` is a human-readable Local debug store with byte IU. +- `MemPageStore` keeps its byte-addressed data in sparse fixed-size 64 KiB + memory pages. Growth allocates only touched pages and never relocates existing + data. Reads of unwritten holes return zeroes within the logical high-water + mark. Its backing data is storage capacity, separate from Tree work credits; + `resident_bytes()` reports allocated data-page capacity without counting holes. +- `BlockPageStore` allocates aligned extents in memory or local block files. + Its allocation metadata, directories and anchors are retained by the backend + independently of the facade. +- `ChunkRuntimePageStore` uses immutable fixed page packs, typed backing and a + process-fenced root catalog. Its physical mirror ledger is canonical across + compatible facades for the same transport, retirement host and logical tree. -| Platform | Mechanism | Status | -| --- | --- | --- | -| Linux (liburing) | `BlockAsyncPageStore` + `DiskIOUring` (io_uring SQE/CQE) | Production | -| macOS / no liburing | Synchronous fallback (blocking `pwrite`/`pread`/`fdatasync` wrapped as immediately-ready completions) | Dev | +### 2.1 TextPageStore — On-Disk Layout (Debug) -**Linux (liburing)**: `BlockAsyncPageStore` maps global byte offsets to -per-extent fds via `BlockPageStore::fd_for_offset()`, then submits -`io_uring` SQEs through the `DiskIOUring`. Writes spanning multiple extents are -split and chained; fsync chains across all dirty extent fds. Requires -`CROWDB_TREE_HAVE_LIBURING` build flag. +The debug backend stores page/metadata images and anchors as text records and +maps logical addresses to those records. It implements the same Local +checkpoint ordering and frame validation contracts; byte IU removes alignment +padding but does not remove checksums or root publication ordering. -**macOS / no liburing**: `CROWDB_TREE_HAVE_LIBURING` is not defined. `ct_open` -does not wire a `DiskIOUring` or `AsyncPageStore`. The `*_async` C API methods -fall back to completing synchronously (blocking I/O wrapped as an -immediately-ready completion). The upper-layer API is identical regardless -of platform. +### 2.2 BlockPageStore — On-Disk Layout (Production) -**`FilePageStore`, `FileAsyncPageStore`, `IoEngine`, and `DirectIoEngine` -are removed.** `TextPageStore` serves the testing role. `BlockAsyncPageStore` -(array-of-blocks `io_uring`) is the production async backend. The -platform-level fallback described above provides the async abstraction. +Block storage grows through fixed-size `.blk-*` files or in-memory blocks. +`SpaceAllocator` owns allocation/exclusion state. Frame addresses identify aligned +IU extents; directory and anchor images identify the selected durable graph. +Local recovery and compaction retain the backend domain so delayed readers and +callbacks cannot access a destroyed allocator or file handle. -### 2.4 fsync Policy +### 2.3 Async I/O Architecture -`fsync`/`fdatasync` is **configurable** via `ct_options.sync_mode`: +`BlockAsyncPageStore` can submit Linux io_uring operations through its owned I/O +backend. A synchronous store is adapted by the host backend executor. Both +paths retain aligned buffers, I/O tickets and backend capabilities through the +real terminal callback. CPU decoding runs on the bounded Tree CPU runtime. -| Mode | Behavior | Use case | -| --- | --- | --- | -| `FullSync` (default) | `fdatasync` after every flush | Production durability | -| `SkipSync` | No fsync; writes are flushed by OS page cache only | Testing/benchmarks | -| `BatchSync` | fsync once per snapshot, not per flush | Throughput-sensitive testing | +Inline completion is allowed. A transport error after uncertain acceptance +cannot release a ticket while a late callback may still access its buffer. +Duplicate completion does not decrement resource or work counters twice. -On macOS, `fsync` costs ~3ms per call (stable). `SkipSync` or `BatchSync` -eliminates this overhead for test/CI runs. The same policy applies to WAL -file sync (see `../kv/design-crowdb-kv-wal.md`). +### 2.4 fsync Policy -**RDMA-remote is deferred**, like any remote block device: it needs a local -page cache (§4) with pinning + epoch-gated eviction, and remote allocation -coordination. The v1 target is `TextPageStore` for debugging and -`BlockPageStore` (in-memory and SSD) for production/tests. +Local checkpoint writes all required page and directory data before publishing +its anchor. The selected sync policy determines the backend durability barriers; +skip-sync configurations intentionally provide weaker crash durability. +Logical flush and representation writeback do not claim checkpoint durability, +even if their individual page writes have completed. ### 2.5 Block Compaction -Block compaction reclaims physical space from sparse block files (`.blk-{NNNN}`) -through two cooperating mechanisms: **gap filtering** controls where new -allocations land during every snapshot, and **cadence-driven relocation** -(`compact_sparse_blocks`) force-moves live extents out of the sparsest blocks -on a configured interval. Both share the same free-ratio threshold and operate -only in array-of-blocks mode (`open_blocks`, `block_size > 0`). - -**Gap filtering in `SpaceAllocator`.** - -`SpaceAllocator` is rebuilt every snapshot from the committed anchor's live -extents (`build_allocator()` in `persist.cpp`). Its gap list is the complement -of live extents within `[region_base, file_size)`. When `block_size > 0`, -compute each block's free ratio (`gap_bytes / block_size`) and exclude gaps in -blocks above `merge_gc_block_free_threshold` (default 70%) from the allocator's -gap list. - -Filtered-out gaps are invisible to `alloc()`, so new writes never reuse space -in sparse blocks. They land in dense blocks' gaps or append past EOF (which -may trigger `allocate_new_block()`). Over successive snapshots, as dirty pages -are rewritten, sparse blocks lose their live pages. - -**Cadence-driven relocation — `compact_sparse_blocks`.** - -Gap filtering drains sparse blocks gradually through normal write churn. When -disk space is urgent or churn is low, `compact_sparse_blocks` force-relocates -live extents out of the sparsest blocks in a single snapshot pass. - -Selection: compute each block's current free ratio from the newest valid -anchor's live extents. Blocks whose free ratio is strictly greater than -`merge_gc_block_free_threshold` are candidates. Sort by descending free ratio, -then ascending block index. Accumulate candidate live bytes until the sum -exceeds `merge_gc_max_relocation_bytes`; the first eligible block is always -allowed even when it exceeds the budget so progress is possible. Block zero -(anchor slots) is never selected. - -Prefetch: before taking `write_mutex_`, read every unloaded mapping slot in -the selected blocks from the store. Retain the original packed mapping word -with the decoded frame. Preparation rechecks that word under `write_mutex_`; -a mismatch discards the prefetched candidate and the page is skipped for this -pass. This keeps disk reads out of the writer critical section and adds no -lock. - -Relocation: for each selected page (resident or validated prefetched -unloaded), encode its durable blob, allocate a new address outside every -source block, and queue the write through the normal prepared-snapshot -pipeline. The owning mapping segment image is forced into the prepared -generation. Unloaded relocations update the mapping word in the segment image -directly. `PreparedSnapshot` tracks `blocks_selected`, `pages_relocated`, and -`bytes_relocated`. - -Commit: the standard two-barrier snapshot protocol applies — page writes, -segment images, directory, durability barrier, anchor, durability barrier. -`finalize_prepared_snapshot` rereads both valid anchors, derives the protected -block set, and deletes only nonzero blocks referenced by neither generation. -`blocks_deleted` is recorded in the `PreparedSnapshot` and reported in -`MergeGcStats`. - -**Block deletion.** - -`BlockPageStore::delete_block(idx)` closes the fd, removes the `BlockExtent` -from `extents_`, and unlinks the `.blk-{NNNN}` file. The global address space -shrinks; future `write_at` / `read_at` calls never target the deleted block. -On reopen, `open_blocks()` scans the directory; the deleted file is gone and -its index is not re-opened. - -**Crash safety.** - -Identical to normal snapshot crash safety. If the process dies mid-compaction: -- Old blocks still exist; old addresses still valid. -- New block has copied pages; new addresses valid. -- Recovery uses whichever anchor is valid; the other references the old blocks. - -Block deletion only happens after the finalizer confirms neither valid anchor -references the block. A crash before the finalizer leaves all blocks in place; -the next snapshot or compaction pass will retry. - -**Cost.** - -- Gap filtering: zero additional I/O. O(gaps) for per-block free ratio - computation during `build_allocator()`. -- Cadence-driven relocation: one snapshot pass worth of I/O for the relocated - pages, plus O(live_extents) for selection and O(anchors) for finalization. - The byte budget bounds the per-pass I/O burst. - -**Scope.** - -Only applies to `BlockPageStore` in array-of-blocks mode (`open_blocks`). -Single-medium mode (`open`, `open_mem`) and `TextPageStore` have no multiple -blocks; `block_size()` returns 0, gap filtering is skipped, and -`compact_sparse_blocks` returns an empty `MergeGcStats` with no snapshot -write. - -### 2.1 TextPageStore — On-Disk Layout (Debug) - -`TextPageStore` writes a directory of human-readable text files, one per -durable object. The directory layout at `{path}/{store_id}-{partition_id}/`: - -``` -{path}/{store_id}-{partition_id}/ - manifest.ck # addr → (type, filename) mapping (text, one line per entry) - anchor-A.ck # CommitAnchor A (text key=value format) - anchor-B.ck # CommitAnchor B (text key=value format) - page-{addr}.ck # One file per B-tree page blob (debug_codec annotated text) - seg-{N}.ck # One file per mapping-table segment image (text) - segdir.ck # Segment directory (text, one line per DirEntry) -``` - -All files use the `.ck` extension, the CROWDB-specific file suffix. The -filename prefix (`anchor-`, `page-`, `seg-`, `segdir`, `manifest`) distinguishes -the type; the `.ck` suffix identifies the file as CROWDB-owned. Editors open -`.ck` files as text by default. - -**Manifest file**: lists `(addr, len, type, filename)` for every blob written. -On open, `TextPageStore` reads the manifest to reconstruct the addr→file -mapping. The manifest itself is human-readable text. - -**Anchor text format** (mirrors WAL's text segment header): -``` -CROWDB_CT_ANCHOR magic=0x41435443 format_version=2 snapshot_seq=123 root_page_id=42 -last_applied_slot=99 next_page_id=100 segment_slots=1024 segdir_addr=4096 -segdir_len=2048 segdir_crc=deadbeef anchor_crc=cafebabe -``` - -**Page text format**: uses existing `encode_frame_text()` / `decode_frame_text()` -from `debug_codec.h`. Each `page-{addr}` file contains the annotated frame text -with `crowdb-tree-frame-text` header, `type leaf`/`type inner`/`type overflow`, -per-slot fields, and a raw hex line for exact reconstruction. - -**Segment image text format**: `CROWDB_CT_SEGIMG` header with `seg_idx`, -`generation`, `slot_count`, `live_count`, followed by one line per slot word -(`slot[N] = (iu_index, iu_count)` or `empty`). - -**Segment directory text format**: `CROWDB_CT_SEGDIR` header, followed by one -line per `DirEntry`: `seg_idx=N generation=G image_addr=A image_len=L -crc=C`. - -**Limitations**: Compression is always `kNone` (text mode is for debugging). -The `debug_codec` operates on uncompressed frames, so compressed blobs cannot -round-trip through text. - -### 2.2 BlockPageStore — On-Disk Layout (Production) +Sparse-block compaction selects a bounded relocation set and reserves complete +read, output, metadata and I/O resources. It writes relocated frames, publishes +the updated checkpoint representation and retires old extents through actual +backend ownership. A live reader or captured older root can keep an old extent +excluded from reuse after the current root moves elsewhere. -A group's storage is an **array of fixed-size block files**, not one -pre-allocated file. When the current block fills up, `allocate_new_block()` -creates the next file. - -**Block file naming**: `{store_id}-{partition_id}.blk-{NNNN}` (e.g., -`1-3.blk-0000`, `1-3.blk-0001`). `store_id` and `partition_id` are passed -via `ct_options`. Block files use the `.blk` extension (binary, fixed-size); -text/debug files use `.ck` (see §2.1). - -**Internal state**: -``` -struct BlockExtent { - std::unique_ptr<FileMedium> medium; // one FileMedium per block file - uint64_t base_offset; // global offset = block_idx * block_size - uint64_t used; // high-water mark within this block - bool dirty; -}; -std::vector<BlockExtent> extents_; -``` - -**Address space**: The global address space is linear: -`global_off = block_idx * block_size + local_off`. `write_at(global_off, buf, -len)` maps to `(extent_idx, local_off)`, splitting writes that cross extent -boundaries. If `global_off + len > total_capacity`, `allocate_new_block()` is -called first. - -**On-disk binary layout** (within each `.blk-*` file): -``` -block file (block_size bytes): -+--------------------------------------------------+ 0 -| [Anchor A slot] (superblock_slot_bytes, IU-aligned) | -+--------------------------------------------------+ slot_bytes -| [Anchor B slot] (superblock_slot_bytes, IU-aligned) | -+--------------------------------------------------+ 2 * slot_bytes -| [Page/Segment image region] | -| ... page blobs, segment images, segment dir ... | -| ... allocated by SpaceAllocator, may have gaps ..| -+--------------------------------------------------+ block_size -``` - -- **Anchor A/B**: Two IU-aligned slots at offsets 0 and `superblock_slot_bytes`. - Binary `CommitAnchor` struct (10 fields + CRC32C), zero-padded to IU boundary. - Alternating A/B by `snapshot_seq` parity. -- **Page blobs**: Durable page frames (optionally LZ4-compressed), written at - addresses allocated by `SpaceAllocator`. Each blob is `[plen u32][payload] - [crc32c u32]`. -- **Segment images**: `SegmentImageHeader` + packed slot words + CRC. -- **Segment directory**: `DirEntry[]` + header + CRC. -- **Gaps**: Free space between live extents. Reused by `SpaceAllocator` for - future allocations. - -**Recovery**: On `open()`, scan the directory for `{store_id}-{partition_id}.blk-*` -files, sort by index, open all. The anchor in block 0 determines which blocks -are live (via segment directory → segment images → page addresses). - -**Sync**: `fdatasync`/`fsync` all dirty extents (tracked per-extent dirty flag). - -**Garbage collection**: Dead pages mark gaps in `SpaceAllocator` but block -files are not deleted (deferred optimization; see plan Task 14, block -compaction design). - ---- +Physical accounting distinguishes committed, candidate, retired and free +capacity. Relocation does not count as free; the block release service updates +capacity only after the actual extent-release transition. ## 3. On-Disk Page Format (Zero-Copy Frame) -**Decision: the in-memory and on-disk page representations are identical.** -A page is a fixed-size *frame*; loading is one `read_page` into a frame with -**no decode/copy**, and the B+tree reads, binary-searches, and compares keys -directly on the frame bytes. Persisting is the reverse: write the frame bytes -(optionally compressed) back. There is no separate "C++ object" form for base -pages; leaf/inner pages become thin **views** over a frame, so the same -comparison (`memcmp`) orders keys identically in memory, on disk, and across -nodes. +Encoded base pages use the same byte layout on disk and after loading. Leaf +views binary-search keys and read cells directly from an owned validated frame. +The live logical page is separate: it may combine several base ranges and +immutable overlay runs. Persistence materializes that logical state into a +new frame; ordinary writes do not insert into the captured base frame. ### 3.1 Frame geometry -A frame is `page_bytes` long (default **16 KiB**, configurable; a multiple of -the backend IU). All frames in a pool are the same size, so the buffer pool -(§4) is a flat array with O(1) indexing. An entry larger than a frame's usable -space spills to an **overflow chain**; the overflow policy is tiered: -inline (≤ frame payload, zero-copy), small overflow (frame limit < v ≤ 1 MB, -spill + warn), large overflow (1 MB < v ≤ 16 MB, spill + warn), rejected -(> 16 MB, reject as a likely bug). Defaults: `Config.max_overflow_value` = -1 MB, `Config.max_value_hard_limit` = 16 MB. +`Config.frame_bytes` defaults to 64 KiB. Primary page capacity is capped by +both that configuration and the public frame protocol payload limit. Leaf +split/merge thresholds count record-region bytes; planning adds the 128-byte +header and 8-byte trailer. A merged leaf must fit below three quarters of the +split target, preserving hysteresis. + +Values above `max_inline_value` use overflow frames with independent backing +owners. A zero configured threshold derives the limit from one quarter of the +configured frame size. The leaf retains a pointer cell and logical value +length; the full overflow chain is not copied into every read or captured leaf. +Key size and complete operation allocation limits are checked before mutation. ### 3.2 Slotted layout -The frame layout is the standard "sorted slot directory growing forward + -records growing backward" so keys stay binary-searchable in place and inserts -don't rewrite record bytes: - -``` -frame (page_bytes): -+--------------------------------------------------+ 0 -| FrameHeader (fixed, 64 B) | magic, type, version, flags, -| slot_count u16, free_lo u16, free_hi u16, | self_pid, right_sibling, -| logical_len u32, lsn/last_applied_slot u64, | page_bytes, crc32c (trailer) -| self_pid u64, right_sibling u64 (leaf only) | -+--------------------------------------------------+ 64 -| Slot directory: Slot[slot_count] | grows forward (free_lo) -| Slot = { rec_off u16, key_len u16, cell_len u16 } (sorted by key) -+--------------------------------------------------+ -| ... free space ... | -+--------------------------------------------------+ free_hi -| Records: [key bytes][cell bytes] ... | grows backward from end -+--------------------------------------------------+ page_bytes - trailer -| Trailer: { logical_len u32, crc32c u32 } | CRC over [0, logical_len) -+--------------------------------------------------+ -``` - -Inner pages hold `n+1` child PIDs (fixed array) plus `n` separator keys (same -sorted slot-directory style, no cells). `ChildIndexFor(key)` is an -`upper_bound` binary search over the separators, reading them directly from -the frame. - -The slot directory is **kept sorted by key**, so lookup is a binary search -reading each key directly as a `Slice` into the frame (zero copy). Insert -appends the record at `free_hi` and splices a slot into sorted position (a -small `memmove` of the slot array only, not the records); delete removes the -slot (records become dead space, reclaimed by rewriting a fresh frame when -dead space crosses a threshold, and because L1 is single-writer COW, most -leaf mutation already produces a fresh frame anyway). - -### 3.3 Durability framing, alignment, compression - -- **`logical_len` + CRC32C trailer.** `logical_len` lets a reader ignore IU - zero padding; CRC32C covers `[0, logical_len)`. A CRC mismatch means a torn - page, and recovery falls back to the previous snapshot (§6). -- **Alignment.** `BlockPageStore` with `iu_size > 1` (e.g. 4096 for NVMe) - writes frames as IU multiples with bounce-buffer read-modify-write for - unaligned offsets. `TextPageStore` always uses IU=1 (byte-addressable, no - alignment) since it writes human-readable text files. `BlockPageStore::open_mem` - also uses IU=1 for in-memory/SCM. -- **Compression is per-page, on-disk only.** The frame body is optionally - compressed with LZ4 (a flag bit + algorithm id in the header), decompressed - into a full frame on load so in-memory access stays zero-copy and uniform. - A page whose compressed size ≥ its raw size is stored uncompressed. CRC is - computed over the *stored* bytes so torn-write detection is independent of - the codec. - -### 3.6 Compression details - -`Config.compression` defaults to `kNone`; LZ4 is opt-in (`kLz4`) so behavior -is deterministic across build environments rather than depending on what the -build machine happens to have installed. The algorithm id is recorded per -page, so mixed pages decode correctly regardless of the option in force when -written. **LZ4 is a system dependency, not vendored:** `CMakeLists.txt` -discovers a system LZ4 dev package and defines `CROWDB_TREE_HAVE_LZ4` when -found; the FFI `cc`-based Rust build does not link LZ4 at all, and the -compressor degrades to an identity codec when the library isn't linked, so -correctness never depends on LZ4 being present, only the compression ratio -does. - ---- - -## 4. Buffer Pool (Frame Cache) - -The buffer pool is crowdb-tree's **only** holder of base-page memory: a -fixed-capacity arena of equal-size frames, explicitly managed (not the OS page -cache) so memory is bounded and eviction is epoch-correct for RDMA/SCM/SSD -alike. This is the "4 GiB / 8 GiB btree cache" knob (`capacity_bytes` default -`min(8 GiB, 25% RAM)`). - -- **No per-page heap objects.** Frames live in one contiguous arena - (`capacity_bytes / page_bytes` frames); pin/unpin update compact frame - metadata under the pool mutex. - The page table (`pid -> frame index`) is an open-addressing array, not - `std::unordered_map`, to keep the hot path branch-light and cache-friendly. -- **Unused capacity.** Arena allocation preserves the configured frame count - without initializing every unused frame. Fresh and anonymous frames are - zeroed when acquired. A read miss loads and validates the complete frame - before publishing it, preserving initialization and stable frame addresses. -- **Pinning and eviction.** A reader pins the leaf/inner frames along its - descent, reads via the view, then releases. Pinned frames are never - evicted. Eviction uses a **CLOCK (second-chance)** sweep: skip pinned and - dirty frames, clear the ref bit on the first pass, evict a clean unpinned - frame whose ref bit is 0. A dirty frame is written back before reuse. - Capacity is a hard cap; if every frame is pinned/dirty, new pins block - briefly and an eager dirty flush triggers. -- **COW + dirty tracking.** L1 stays single-writer COW: to mutate page `pid`, - the Flusher builds the new page in a fresh frame, atomically publishes it - (mapping store), marks it dirty, and epoch-retires the old frame's logical - version. The **dirty set** (frames with `dirty == true`) is what snapshot - walks (§6) instead of the whole tree. This is what makes snapshots - incremental (only dirty frames are written) rather than a full rewrite. -- **Anonymous frames.** A newly built page (flush/consolidate/split/merge) - has no durable address yet; it is dirty and pinned-resident until - snapshot assigns it one and clears the pin, becoming evictable. Between - snapshots, memory is bounded by *(working set since last snapshot) + - (resident clean cache)*; a write storm that outruns snapshot triggers an - eager snapshot (back-pressure, §6). - -The pool mutex protects the page table, CLOCK hand, and frame metadata, but is -never held across `PageStore` I/O. A miss reserves its victim as `Loading`; a -dirty eviction first reserves it as `Writeback`. The mapping stays visible -during either operation, so another pin for that page waits on the frame state -while hits on unrelated ready frames continue. Read failure removes the loading -mapping and returns the frame to the free set. Writeback failure restores the -old frame to `Ready` without eviction. `flush_dirty` uses the same reservation -protocol one frame at a time. - -### 4.1 Why epoch reclamation, not pin counts, governs eviction safety - -Picking a CLOCK victim is the easy part; the hard part is *"when is it safe -to reuse a page's frame bytes?"* Readers are lock-free and hold `Slice`s -pointing directly into a frame with no per-page refcount, so this is -identical to the general reclamation problem already solved by -[`design-crowdb-tree-engine.md §1.6`](design-crowdb-tree-engine.md#16-epoch-based-reclamation). -crowdb-tree deliberately reuses that mechanism rather than introducing a second -one: dropping a resident page is structurally the same as a COW replace -(`mapping.StoreUnloaded(pid, addr, len); retire(old_page)`), and the evicted -frame is returned to the pool's free list only when the epoch reclaimer frees -the page, i.e. after every overlapping reader guard has drained. So frame -reuse is epoch-safe for free, with no extra read-path pins and no hazard -scan. This is also why the pool's own CLOCK must never evict a base frame -directly: the pool knows nothing about mapping slots or epochs and would -reuse bytes a lock-free reader still points at. Base frames are therefore -pinned so the pool's victim search skips them; eviction is instead driven by -the single writer, which does the retire + re-tag above. - -One precondition matters for correctness: re-tagging a slot `unloaded(addr, -len)` is only valid once the page's durable bytes at `addr` equal its live -frame exactly. A demand-loaded page already satisfies this; a freshly -built/consolidated page does not until a snapshot assigns it an address and -writes that exact frame, so general eviction of freshly-written pages is -gated on the snapshot pipeline (§6) having run at least once for that page. - ---- +A leaf frame contains: + +- a 128-byte header with type, format, logical page identity, range fences, + slot/record offsets and subtree summary; +- a sorted directory of 12-byte slots, each containing three `u32` values: + record offset, key length and cell length; +- record bytes growing backward from the trailer, containing key and encoded + cell (slot, flags and inline value or overflow reference); +- an 8-byte trailer containing logical length and CRC32C. + +Inner frames contain child PIDs and an 8-byte separator slot directory. +Routing uses the separator upper bound. Builders consume sorted bounded input +and stamp a complete immutable frame; later logical mutation publishes a new +page descriptor or replacement frame. + +### 3.3 Validation, alignment and compression + +Frame validation checks type/version, lengths, slot bounds, identity, ranges +and checksum before exposing views. A corrupt referenced frame is an error; +recovery must not silently discard committed content or create an empty Tree. +Backend IU alignment and stored-envelope padding are separate from the logical +frame capacity. Readers retain the actual stored allocation and buffer owner. + +`Config.compression` defaults to `kNone`. With LZ4 enabled, the stored envelope +records the codec, original/stored lengths and CRC of stored bytes. Incompressible +frames remain raw. Decoding restores a validated frame before leaf access. +CMake discovers system LZ4; the Rust FFI build can enable it through +`CROWDB_TREE_LZ4_LIB`. Without the codec, new output remains raw and an existing +LZ4 envelope returns `NotSupported`, preserving the stored data. + +## 4. Buffer Pool and Cache Residency + +`SharedPageCache` stores reloadable immutable frames under bounded page, byte +and waiter limits. It deduplicates in-flight loads and separates demand and +maintenance work. Bindings and waiter state are prepaid before accepted work +can need them; a queue full of ordinary reads does not invalidate an accepted +flush's routing credit. + +Logical page identity, current residency and physical frame lifetime are +separate. Evicting a cache entry or publishing a proxy removes that residency +reference; a read result or snapshot can still own the frame. The allocation +charge persists until the last frame owner releases it. + +Dirty logical pages require a reloadable backing representation before their +resident data can be dropped. Bounded writeback installs such backing without +advancing durable coverage. Idle host windows continue writeback and trimming +after writes stop. Clean captured backing can be reused by checkpoint without +loading its primary frame when its proof is still valid. + +**I2 — Eviction safety:** a proxy retains a readable typed backing capability. +No cache eviction rewrites logical content, advances the source floor or +invalidates an already returned owner. ## 5. Internal WAL Decision -**Decision: crowdb-tree does NOT keep a redo WAL of data operations. Recovery is -snapshot + consensus replay**, consistent with `../kv/design-crowdb-kv-state-machine.md §2.1` -and `../kv/design-crowdb-kv.md §8`. - -- The consensus layer already has a durable WAL of chosen entries - ([`../kv/design-crowdb-kv-wal.md`](../kv/design-crowdb-kv-wal.md)). Adding a second op-log in crowdb-tree - duplicates durability machinery. -- crowdb-tree persists a **snapshot** at `last_applied_slot`. On restart, slots - `> last_applied_slot` are re-applied from the consensus WAL / re-learned via - consensus recovery. Worst-case redo work is bounded by the snapshot - interval, not unbounded. - -A crash between snapshots loses only the in-memory deltas since the last -snapshot, which the consensus layer re-applies. The engine reports its -restored `last_applied_slot` so the learner knows where to resume (§6.2). A -structural mini-journal for in-progress split/merge is unnecessary because -SMOs are writer-exclusive and only their consolidated result is ever flushed; -a crash mid-SMO simply reverts to the last snapshot. - ---- +crowdb-tree has no operation redo log. The host consensus WAL owns the durable +chosen-operation sequence; Tree checkpoints own durable materialized state. +Recovery loads the selected root and replays the external WAL suffix after its +coverage. Chunk catalog metadata is a root/ownership protocol, not a second +user-operation WAL. ## 6. Snapshot, Recovery, and Export/Import -`persist_snapshot()` makes the engine's materialized state durable up to -`last_applied_slot` and produces a `RootVersion` -([`design-crowdb-tree-engine.md §1.5`](design-crowdb-tree-engine.md#15-versioned-root-consistent-snapshots)). -Two staged movements, no leveled compaction (crowdb-tree is not an LSM): - -``` -apply(slot,batch) flush (Flusher, 1/tree) snapshot (cadence) - L0 MemTable ───────────▶ L1 frames (buffer pool) ───────────▶ PageStore - (concurrent map) batched, single-writer COW incremental, dirty - hot-key collapse into frame builders frames only + fsync -``` - -1. **L0 → L1 (flush).** The Flusher builds a **new frame** per touched leaf - directly from the old frame's entries merged with the batch - (highest-slot-wins); no serialization step, the builder writes the final - on-disk bytes directly, and the old frame is epoch-retired. This is fast - because hot keys already collapsed in L0, so it's one frame rebuild per - touched leaf per flush (not per key), and the rebuilt frame **is** the - durable image (snapshot never re-encodes it). -2. **L1 → PageStore (snapshot).** Walks the dirty set (§4), not the whole - tree, and writes only dirty frames (optionally compressed), remaps - `pid -> PageAddr`, fsyncs, then commits the anchor (below). Cost is - proportional to pages dirtied since the last snapshot, not tree size. -3. **Durability barrier ordering.** Dirty frames + directory fsync **before** - the commit write; a crash never exposes a commit referencing a - half-written frame (CRC also guards this). - -**Back-pressure:** if L0 grows faster than the Flusher drains (e.g. a slow -device), `apply` throttles once L0 crosses a high-water mark, and the buffer -pool's dirty ratio triggers an eager snapshot, keeping memory bounded under -write storms. - -**Recovery** is lazy: only the commit anchor and the mapping-table directory -(§8) are read eagerly; base pages are demand-loaded on first access, so -restart stays fast even for large trees. A page failing CRC on first read -indicates media corruption (the anchor that referenced it was already -committed). The engine fails the node out of the group, and it rejoins via -snapshot install (§10) from a healthy peer. After recovery the learner sets -`contiguous_applied = last_applied_slot` and resumes consensus catch-up for -higher slots. +A checkpoint captures one immutable published view after finite prefix handoff. +It retains sources, mapping, typed primary backing and overflow dependencies. +`CapturedFrameStream` reuses valid durable receipts or materializes bounded frames, +then `LocalCheckpoint` or the Chunk root protocol commits the complete graph. +See [checkpoint flow](design-crowdb-tree-engine-snapshot-flow.md). + +Snapshot persistence is page-granular. Changed Tree pages and pages explicitly +selected for GC or relocation receive new backing; unchanged pages and +mapping/reference segment images retain their stored identities. Local's small +segment directory may be rewritten when segment entries change. With no +persistent-state change, the existing durable root is reused without data or +metadata writes or a new snapshot sequence. Root publication changes +reachability; Tree GC reclaims old physical data only after retained roots, +candidates, readers and accepted I/O no longer require it. + +The current runtime metadata path still rebuilds Local segment images and Chunk +indexes. It does not yet satisfy this incremental metadata contract; zero +data-frame writes alone do not establish a no-op checkpoint. + +Local recovery validates anchor/directory metadata, frame identity and +structural relationships before exposing a Tree. Logical descriptors may +retain cold frames and load them through the owned backend as required. +Corrupt or incompatible roots fail explicitly; they do not silently produce +an empty writable database. + +A checkpoint can commit while newer writes exist. Representation rebase checks +the captured content/route/run identities and retains newer overlays. A stale +rebase is deferred; it never overwrites current content or revokes the +successfully committed captured root. ### 6.1 Snapshot Export / Import -**Decision: export is a byte stream, with a thin "to a file" convenience -wrapper.** The streaming form is the primitive; it feeds the network -snapshot-transfer service for new-member install (§10) without ever touching -local disk. Dumping to a `.ctsnap` file is just streaming into a file writer. -The same code serves both "send to a peer" and "dump to a bin file." - -Two on-the-wire formats, selected at export-begin: - -| Format | Layout | Use | -| --- | --- | --- | -| **Portable** (default) | versioned header `{magic, format, slot}`, then key-sorted `(klen,key,slot,kind,vlen,value)` tuples in fixed ≤1 MiB chunks, end marker, whole-stream CRC32C | cross-engine parity (in-mem ↔ crowdb-tree), durable archival; deterministic chunk boundaries ⇒ resumable | -| **Native** | header + raw frame images + a remapped manifest | fast crowdb-tree→crowdb-tree transfer; skips tuple re-encode | - -The portable format is deterministic and engine-independent; an exported -`.ctsnap` re-imports identically on any backend/page size. The header's -`slot` is the engine's `last_applied_slot` at export time (there is no -`at_slot` parameter; historical snapshot export is not supported; a -snapshot always exports the current, latest durable state). - -- **Export** pins the current `RootVersion` and streams over the pinned, - immutable tree; the pin releases at the end. Newer snapshots may supersede - it meanwhile (refcount keeps it alive). -- **Import** bulk-loads bottom-up into fully-consolidated immutable frames in - a staging area (never touching the live tree), verifies the end-to-end - CRC, then atomically swaps in a new `RootVersion` with `last_applied_slot` - = the slot from the export stream header; the old version is - epoch-retired. Readers see the previous state until the swap. -- Resumability/throttling live **above** the engine; the engine only - guarantees deterministic, chunk-boundary-stable export and atomic import. - -The exact C ABI (`ct_snapshot_export_begin/next/end`, -`ct_snapshot_import*`, and the rest of the surface: `ct_open`, `ct_apply`, -`ct_get`, `ct_scan`, `ct_snapshot_view`, `ct_set_gc_watermark`, -`ct_compact_sparse_blocks`, ...) is specified in -[`lib/crowdb-tree/include/lib/crowdb-tree/c_api.h`](../../../lib/crowdb-tree/include/lib/crowdb-tree/c_api.h). -That header is the single source of truth for signatures; this document -only records the shape and rationale. - ---- +Native export emits actual captured frames, including cold data that a normal +checkpoint could reuse without reading. Portable export emits logical entries. +Both iterate a stable immutable view using bounded materialization windows; +outputs retain their actual owner after iterator or Tree destruction. + +Import validates frames, root/child graph, ranges, slot fences and overflow +dependencies before replacing a generation through the coordinator. Corruption, +admission failure or cancellation during staging leaves the old generation and +its writer usable. Imported resident overflow retains independent owners and +can later be checkpointed without a global mutable resolver. + +Local compatibility migration reads the supported durable format into a staged +runtime generation, validates it and publishes through the ordinary checkpoint +protocol. Chunk compatibility migration has a distinct explicit stop-writes +contract, described by the Chunk storage design. ## 7. Interaction with Consensus WAL GC -crowdb-tree consumes two slot watermarks the learner already tracks, plus its -own `last_applied_slot`: - -| Watermark | Source | Meaning | -| --- | --- | --- | -| `last_applied_slot` | crowdb-tree (per snapshot) | Highest slot whose effects are durable in the engine's last snapshot. | -| `snapshot_slot` | learner / replicator | State at this slot is durable on leader + ≥1 peer. | -| `safe_slot` | learner (`contiguous_applied` min across members) | Every learner has applied through here. | - -`set_gc_watermark(snapshot_slot, safe_slot)` is called by the learner -whenever these advance; crowdb-tree stores them and uses -`gc_slot = min(snapshot_slot, safe_slot)` to gate reclamation (§9), matching -`../kv/design-crowdb-kv-state-machine.md §7`. - -The consensus WAL's own GC ([`../kv/design-crowdb-kv-wal.md`](../kv/design-crowdb-kv-wal.md)) uses this same -`gc_slot` rule: a WAL segment may be unlinked only when all its records have -`slot < min(last_applied_slot across members, safe_slot, snapshot_slot)`. -Because a restarting node resumes at its own `last_applied_slot` and -re-learns the rest, the consensus WAL must retain entries above the -**minimum** `last_applied_slot` of any member that might restart and replay -locally. This couples the two GC watermarks: crowdb-tree advancing -`last_applied_slot` (via snapshots) is what eventually lets the consensus WAL -drop old segments. - -**Restart flow:** - -``` -node restart: - 1. consensus WAL replay rebuilds ACCEPTOR state only - (Promised / Accepted / VoteGranted) — ../kv/design-crowdb-kv-wal.md - 2. crowdb-tree.recover() loads the commit anchor -> last_applied_slot = L - 3. learner sets contiguous_applied = L - 4. slots (L, max_chosen] are re-learned (heartbeat catch-up or new-leader - bulk Phase 1) and re-applied to crowdb-tree via apply(slot, batch) -``` - -Re-applying slots `<= L` is harmless: highest-slot-wins makes them no-ops. So -the consensus WAL need only retain entries `> min(last_applied_slot across -members)`. - ---- +The host separately tracks applied, handed-off and durable coverage. A Tree +writeback report cannot authorize WAL truncation. `persist_snapshot` returns +the committed durable prefix; the learner combines that prefix with its quorum +and safe-slot policy before advancing its external GC floor. + +Logical retention is bounded by the supplied safe/snapshot floor. Old snapshots +continue owning the representation they captured even after the current Tree +prunes tombstones or superseded values. Logical GC and physical backend +reclamation therefore have different completion times. + +**I3 — Replay safety:** after restart, operations above the selected durable +coverage remain necessary even if their former process had completed handoff +or staged writeback. ## 8. Mapping Table -The mapping table is the B+tree's indirection layer: every structural -reference (root, sibling, inner child) is a logical `PID`, and the table -translates a `PID` to the page's current location, a resident in-memory -page or an unloaded durable address. It follows the Bw-Tree/LLAMA idea -(tree links are logical PIDs so moving a page between memory and disk changes -only one entry; page state is installed by replacing one entry) but -simplifies it for crowdb-tree's constraints: **single-writer atomic store** -instead of a multi-writer CAS (the Flusher/reclaimer is the only writer; -readers only load), COW root versions, and no internal data WAL. So the -durable mapping table is a **snapshoted metadata image of `PID -> PageAddr`**, -not a second redo log. - -**Decisions:** - -| Decision | Rationale | -|----------|-----------| -| **No PID recycling** (`next_page_id_` is monotonic). | A reused PID could be seen by a stale reader as the new page — silent wrong data. Data integrity outweighs the memory saved. | -| **Segment recycling — yes.** | Slots are grouped into fixed-size segments (default 1024 slots/segment); when every slot in a segment is empty, its memory is freed. Bounds growth to live segments, not total split/merge count. Safe because PIDs are never reused, so a freed interior segment is never re-created. | -| **Sparse segments are acceptable.** | One live slot pins an ~8 KB segment. Migrating the slot to compact it would change the PID (referenced everywhere) — far worse. Segment size is tunable. | -| **Segment-level persistence, not a full manifest.** | Persist only *dirty* segments as self-describing images; the mapping table itself is the durable structure, avoiding an O(N) DFS-walk-and-write-everything on every snapshot (the old full-manifest approach cost ~120 MB of I/O per snapshot for a 100 GB tree even if few pages changed). | -| **Backend-neutral format.** | All I/O via `PageStore` (`allocate`/`write_page`/`read_page`/`free`/`flush`) — no fixed superblock/manifest region tied to the file backend, so block/RDMA work the same way. | -| **Tiny fixed commit anchor + a separate segment directory.** | The anchor is a small fixed A/B record (the atomic commit point); it points to a segment-directory image, so the commit stays atomic even for thousands of segments — the directory (cost ~195 KB for a 100 GB tree) decouples the possibly-large segment list from the tiny atomic anchor. | - -**In-memory structure:** each `PID` maps to a 64-bit **packed slot word**, -the same encoding in memory and on disk. `0` = empty; `bit0=0` = resident -(an in-memory pointer, never persisted); `bit0=1` = unloaded (a durable -`(iu_index, iu_count)` descriptor). On disk a slot is only "empty" or a -tagged unloaded descriptor, so a segment image is literally the array of -packed words; recovery installs them with **zero decode**. `Get(pid)` is a -lock-free atomic load under an epoch guard; an unloaded slot demand-loads via -the buffer pool (§4), publishes the resident tag, and returns the frame. - -**Segment recycling** runs in the epoch deleter, after all guards that could -see the page have drained: clear the slot, decrement the segment's live -count, and if it hits zero, atomically swap the segment pointer to null and -epoch-retire the old segment. A reader that loads a null segment or an empty -slot treats the PID as gone and retries from the root, never wrong data. - -**On-disk format** (all records are `PageStore` allocations, self-describing, -CRC-protected): a **segment image** per dirty segment per snapshot (header + -the raw packed-word array, ~8 KB for 1024 slots); a **segment directory -image**, rewritten whenever any segment's generation changes, mapping -`seg_idx -> (generation, image_addr, image_len)`; and a **commit anchor** -(fixed size, A/B double-buffered at reserved IU 0/1) holding -`{snapshot_seq, root_pid, last_applied_slot, next_page_id, segment_slots, -segdir_addr/len/crc}` plus a CRC. The anchor is the commit point; its small -fixed size makes the A/B swap atomic on every backend, and a snapshot always -writes to the slot *not* named by the current highest-seq anchor, so a torn -write never destroys the last committed one. - -**Binary layout** (BlockPageStore, see §2.2): The anchor, segment images, -and segment directory are stored as binary blobs at addresses allocated by -`SpaceAllocator` within `.blk-*` files. The anchor occupies IU slots 0 and 1 -in block 0. - -**Text layout** (TextPageStore, see §2.1): The anchor is stored as -`anchor-A.ck`/`anchor-B.ck` text files. Segment images are `seg-{N}.ck` text -files. The segment directory is a `segdir.ck` text file. A `manifest.ck` file -maps addresses to filenames. All are human-readable for debugging. - -**Snapshot** integrates with §6's pipeline: write dirty frames and assign -durable addresses, serialize each dirty segment (pack `unloaded` or the -frame's new durable address per slot), rewrite the directory if any -generation changed, `flush()` (frames + images + directory durable), then -write the commit anchor and `flush()` again (the actual commit point), then -clear dirty bits and schedule old-image cleanup. **Recovery** picks the -highest-`snapshot_seq` anchor with a valid CRC, reads the segment directory -it points to, and for each entry reads that segment's image and installs its -packed words directly; no page bytes are read eagerly. - -**Old image cleanup** uses a two-generation rule: after anchor `N` commits, -images referenced only by anchors `< N-1` are freed. This tolerates a crash -during cleanup, since the still-referenced generation is always intact. - -**Concurrency invariants:** the Flusher (and the epoch reclaimer it drives) -are the only slot writers; no CAS needed for slot stores, readers only -atomic-load. Segment install/retire is a single atomic store/CAS. Slot -clearing and segment retire happen in the epoch deleter, after all -overlapping reader guards drain. - ---- +`MappingView` is a persistent radix COW structure from logical page IDs to +immutable published descriptors. Transactions specify unique ordered page IDs +and expected old owners. Root acquisition retains the entire view needed by +that reader; later publication cannot redirect its IDs to another generation. + +COW copies touched paths and segments. Each path aggregates logical page +contributions and bounded maintenance summaries, including maximum captured +leaf descriptor/base-array size and oldest candidate timestamps. Removing or +replacing a child updates those summaries, including shrinking maxima. + +Local durable metadata uses CRC-protected mapping segment images, a segment +directory and an anchor. In-memory pointers are never persisted. Recovery +translates validated stored references into typed backing and immutable +descriptors; it does not expose a mutable tagged-slot mapping to ordinary Tree +writers. Chunk roots instead describe fixed immutable packs and their indexes. + +Generation replacement, import, split, maintenance and checkpoint rebase all +publish through `PublicationCoordinator`. Cache readiness changes only a +representation's residency and does not independently mutate the logical root. + +**I4 — Publication safety:** a stale expected identity cannot clear or replace a +newer page. Metadata summaries and mapping contents belong to the same +published state. ## 9. Garbage Collection -Logical retention GC folds tombstones and stale versions during snapshot -preparation, gated by `gc_slot = min(snapshot_slot, safe_slot)` (§7). There -is no standalone resident-tree sweep; `set_gc_watermark` advances the floor -and the next snapshot does the folding. - -1. **Tombstones.** A tombstone cell written at slot `t` is folded during - snapshot preparation when `t <= gc_slot`. The snapshot walk inspects - clean resident `LeafBase` pages and rebuilds only those containing - eligible tombstones via `resolve_leaf_chain_for_rebuild`. -2. **Stale root versions.** A `RootVersion` with `refcount == 0` and - `last_applied_slot < gc_slot` is retired, and pages reachable only from - it are freed. - -Triggers: the maintenance loop calls `set_gc_watermark` on every tick -(default ~50ms) with `group_snapshot_slot` and `group_safe_slot`. Snapshot -preparation folds eligible tombstones whenever a snapshot runs (either the -periodic persist-snapshot cadence or a `compact_sparse_blocks` pass, §2.5). -Physical block reclamation is handled by `compact_sparse_blocks` on a -separate cadence (`merge_gc_interval_ms`, default 0 = disabled). EBR -remains automatic memory reclamation for retired pages. - -`MergeGcStats` reports blocks selected, pages relocated, bytes relocated, -and blocks deleted from `compact_sparse_blocks`. Tombstone folding counts -are not separately reported; the snapshot's `pages_written` reflects -rebuilt leaves. - ---- +Logical tombstone/version retention follows the external floor. Physical +reclamation follows real ownership and backend proof: + +- Epoch retirement frees source nodes/descriptors only after protected readers + leave, while independently shared cell payloads may outlive the source. +- Root/page/array/frame retirement waits for old views, output handles and cache + owners. A logical retirement marker does not itself free allocation bytes. +- Local extent release waits for exclusion and retained backend owners. +- Chunk collection requires candidate identity, process/catalog authority, + reference protection and confirmed drain/delete. Shared mirror capacity is + deduplicated by canonical physical identity. + +Bounded host maintenance drives these queues when writes stop. A retained user +handle can legitimately delay convergence; diagnostics expose that reason and +the age of unfinished work instead of reporting those bytes as already free. ## 10. New-Member Install Flow -For a new or far-lagging member: - -``` -1. Leader picks a snapshot slot S (>= its last_applied_slot). -2. Leader engine: snapshot_export() streams chunks via the snapshot service. - S = leader's last_applied_slot at export time (always the latest durable state). -3. New member engine: snapshot_import(stream) builds + swaps in the tree at S. -4. New member sets contiguous_applied = S, then streams consensus WAL - (S+1, current_max_chosen] and applies via apply(slot, batch). -5. compare(view) against an existing learner -> empty diff (parity gate). -``` - -Resumability and throttling are handled by the snapshot module above the -engine; crowdb-tree only provides deterministic, chunk-boundary-stable export -and atomic import (§6.1). - -**Steady-state write + periodic durability, for reference:** - -``` -learner.learn(entry) -> engine.apply(slot, batch) // in-memory deltas -... every snapshot_every_slots / dirty threshold ... -engine.persist_snapshot() -> last_applied_slot advances // durable + new RootVersion -learner observes safe_slot/snapshot_slot advance -> engine.set_gc_watermark(...) -background -> engine.compact_sparse_blocks() // block relocation + deletion -``` - -These flows require no change to the learner's public contract beyond the -async `KVEngine` surface (`design-crowdb-tree.md §3`). `InMemKV` implements the -same methods (snapshot/GC are near-no-ops in memory) so tests exercise the -same code paths on both engines. -## Root metrics - -The tree publishes a cached summary for the selected durable root. Its -`root_version` and `covered_slot` fence the observation; WAL and memtable -entries after that slot are excluded. An empty durable root reports exact zero -live keys and bytes while retaining its reachable root page cost. A tree that -has no selected durable root reports `exact = false`; its numeric fields are -diagnostic only and must not authorize a size based split or move. - -`live_kv`, `live_key_bytes`, `live_value_bytes`, and `live_logical_bytes` count -the current visible value once. Tombstones and overwritten versions do not -count. Overflow values contribute their logical length; their pointer cells and -overflow frames are structural storage. `reachable_leaf_pages`, -`reachable_inner_pages`, and `reachable_overflow_pages` classify pages reachable -from the serving root. `reachable_page_capacity_bytes` is page capacity, kept -separate from logical data bytes. The summary is replaced only after the page, -directory, and root anchor have passed their durability barrier and is rebuilt -from the selected root during reopen. - -The current unpublished frame format keeps `format_version = 1`; its 128-byte -header reserves the summary fields for every leaf and inner page. Inner-page -fields are the sum of the reachable child summaries, and the page CRC covers -those fields. A root is exact only when every reachable child summary is exact; -an unavailable or partial child never contributes as zero. This is an internal -layout extension of the existing format, not a version-2 compatibility path. +The sender captures a stable durable/logical prefix and streams native frames +or portable entries. The receiver stages and validates an independent +generation, commits its durable root, then replays the consensus WAL suffix. +Installation failure preserves the prior generation. Range rebuild and split +handoff honor their half-open range and captured slot floor; child writes do +not invalidate the parent's retained captured owners. + +## 11. Root and Physical Metrics + +`TreeStatsSnapshot` reports incrementally maintained logical contributions with +version/commit identity. Known and unknown page sizes remain explicit: an +unmaterialized overlay does not invent exact encoded bytes. Dirty, reloadable, +checkpoint-pending and merge-candidate membership have separate meanings. + +The durable root summary is fenced by root version and covered slot. It excludes +later WAL/L0 writes; unknown child summaries propagate unknown rather than zero. +Live keys/value bytes exclude tombstones and overwritten values. Overflow +payloads contribute logical length, while overflow frames contribute structural +capacity. + +Physical backing metrics expose scope identity for host deduplication. A +metrics-only sampler may outlive the Tree without pinning backend or page +owners. Reference-reason byte counts overlap and must not be summed as physical +capacity. Ordinary collectors use incremental state; a complete graph/owner +reconciliation is an explicit diagnostic with completeness and quiescence +reported in its result. diff --git a/doc/design/tree/design-crowdb-tree.md b/doc/design/tree/design-crowdb-tree.md index 781a470b4..7de737f87 100644 --- a/doc/design/tree/design-crowdb-tree.md +++ b/doc/design/tree/design-crowdb-tree.md @@ -3,17 +3,12 @@ # CROWDB - Design: crowdb-tree Storage Engine (Overview) -Depends on: [`../kv/design-crowdb-kv.md`](../kv/design-crowdb-kv.md), [`../kv/design-crowdb-kv-state-machine.md`](../kv/design-crowdb-kv-state-machine.md) -Satisfies: [`../kv/design-crowdb-kv.md`](../kv/design-crowdb-kv.md) §8.3 (learner storage) +Depends on: [Paxos KV](../kv/design-crowdb-paxos-kv.md), [state machine](../kv/design-crowdb-paxos-kv-state-machine.md) +Satisfies: [Paxos KV](../kv/design-crowdb-paxos-kv.md) §8.3 -This is the parent document for **crowdb-tree**, the production storage engine that -backs CROWDB learners: an embeddable, ordered key-value engine implementing the -CROWDB `KVEngine` contract, built as a standalone C++ library (`libcrowdb-tree`) and -consumed from the Rust `crowdb-kv` crate over a C ABI. It records the decisions -behind that design and maps the sub-design documents. `libcrowdb-tree` is fully -implemented, wired into `crowdb-kv` (`CrowdbTreeEngine`), and shipped; this document -set is the durable record of *why* it looks the way it does, not a build plan. -See [`todo_code.md`](../../working/todo_code.md) for anything still open. +crowdb-tree is the ordered, slot-aware storage engine used by CROWDB learners. +The C++ engine owns publication, memory, page I/O and recovery. Rust supplies +consensus, external WAL policy and the host lifecycle through an opaque C ABI. ## Table of Contents @@ -22,283 +17,145 @@ See [`todo_code.md`](../../working/todo_code.md) for anything still open. - [3. Engine Abstraction](#3-engine-abstraction) - [4. FFI Boundary](#4-ffi-boundary) - [5. Sub-Design Document Map](#5-sub-design-document-map) -- [6. Decision Log](#6-decision-log) - ---- +- [6. Correctness Invariants](#6-correctness-invariants) ## 1. Goals and Non-Goals -**Goals** - -- One ordered, **single version per key** KV engine that serves all learner - reads. Only the latest write per key is kept (no multi-version / time-travel). -- **Slot-aware storage:** every live key carries the consensus slot (= WAL - position) of the write that produced it, stored **in the value cell, not the - key**. Putting it in the key would make each key sort into multiple ordered - versions — multi-version storage — which is explicitly not wanted. -- Pluggable persistence behind one page-granular backend: a **file** store, a - **block-device** store, and an immutable **chunk** store. The block-device - store covers raw SSD, SCM, an - in-memory store for tests, and RDMA-remote (a remote block device); it is - parameterized by an **IU (indivisible-unit) alignment** as small as 1 byte - (mem / SCM) or a flash page (SSD). -- Consistent point-in-time read snapshots (`scan`, `compare`, snapshot export) - without stopping writes: an immutable COW B+tree root tagged with the slot it - reflects (§3.1). -- Compose with the **external** durability/GC flow: crowdb-tree exposes - `last_applied_slot` and accepts a GC watermark from the slot/WAL layer; it - keeps **no operation log of its own**. There are two distinct GCs: - 1. **Data-retention GC — logical, slot-driven.** Dropping tombstones and - superseded values once the slot/WAL layer says they are durable - everywhere. crowdb-tree is *told* the floor via `set_gc_watermark`; the - *policy* (which slot is safe) is owned by the learner/WAL layer. - 2. **Page reclamation — physical, internal.** Freeing B+tree pages and - retired root versions no longer referenced by any reader. - Crowdbtree-internal (epoch-based), automatic, **not** slot-driven. -- A structure the team fully implements and controls — no third-party storage - library. - -**Non-Goals** - -- **Multi-version / MVCC time-travel reads.** Single version per key only — - not worth the cost for CROWDB. -- **Tree-level range split/merge in v1.** This is *tree*-level (splitting a - whole crowdb-tree into two), distinct from internal B+tree *page* split/merge - (always done). One crowdb-tree per consensus group; partitioning happens at the - consensus-group layer. **The design must not preclude it** — a future - large-KV-cluster sharding feature will split/merge whole trees. -- **A second operation log.** crowdb-tree has no redo/replay log of its own. - Durability of consensus output is the consensus WAL; crowdb-tree persists only - its materialized state (a snapshot = immutable root + `last_applied_slot`). - This lets crowdb-tree compose with the slot/WAL system (or any other log) for - recovery. -- **Lock-free multi-writer B+tree.** The B+tree has a single writer (the flush - thread). Write concurrency is provided *in front of* the tree by a - concurrent MemTable; a single Flusher merges the contiguous-applied prefix - into the tree ("flush = the persistent write"). - -The pagetree design (page layout, delta records, consolidation, epoch -GC, bloom filters, IU alignment) was an **algorithmic reference only** -during design. It is not linked and not a dependency. What crowdb-tree -reused vs. dropped vs. simplified: - -| Mechanism | Decision | -| --- | --- | -| Leaf page layout + bloom + IU alignment + CRC32C | Reuse | -| Delta records (batch upsert/delete) + consolidation by length/bytes | Reuse (deltas carry `slot`) | -| Mapping table (PID indirection) | Reuse, **drop CAS retry** (single writer stores directly) | -| Epoch-based GC | Reuse (readers lightweight enter/exit) | -| Page-level split / merge | Simplify: **writer-exclusive**, drop multi-phase cooperative help-along | -| Immutable versioned root table | **New** (MVCC snapshot + export + recovery anchor) | -| Inline `(slot, cell)` per value | **New** (slot single-version semantics) | -| Tree-level split / merge | **Dropped in v1** (partitioning is in the consensus layer); design must not preclude future sharding | -| Internal WAL | **Dropped** — no redo log; recovery = snapshot + external-WAL replay from `last_applied_slot+1` | - ---- +- Expose the latest value or tombstone for each key, using the highest eligible + consensus slot. Retained versions support incomplete prefixes and captured + views; the public API does not provide arbitrary historical queries. +- Keep L0 ingestion concurrent and L1 preparation parallel. Bound memory, + queue slots, CPU work and I/O before accepting work that must complete. +- Hand cell ownership from sources to immutable page overlays without copying + value payloads or pinning the whole source index. +- Support Local file/block storage and immutable mirrored Chunk storage. + Logical publication, reloadable writeback and durable checkpoint are + separate transitions. +- Preserve captured views and owned results across later writes, generation + replacement and Tree close. Last-owner release determines physical reuse. +- Compose with the consensus WAL. Tree persistence does not create a second + operation log and does not independently decide when WAL data is safe to drop. ## 2. Architecture -``` -crowdb-kv (Rust) - PxLearner ──drives──► dyn KVEngine - ├─ InMemKV (Rust, tests) - └─ CrowdbTreeEngine (Rust, FFI adapter; §4) - │ C ABI: ct_open / ct_apply / ct_get / ct_scan / ct_get_async / ... - ▼ - libcrowdb-tree (C++) - └─ Crowdbtree (one per consensus group) - ├─ EpochManager page reclamation, tree-private - ├─ MemTable (L0) concurrent ordered buffer; absorbs concurrent/out-of-order apply - ├─ Flusher (1 thread) flushes the contiguous-applied prefix L0 → B+tree - ├─ MappingTable PID → resident page | unloaded durable address - ├─ BufferPool tree-private frame cache - ├─ DiskIOUring (1 thread) io_uring event loop for async I/O - ├─ root_pid / leftmost_leaf_pid - ├─ RootVersion versioned root + refcount for consistent snapshots - └─ PageStore (backend) TextPageStore | BlockPageStore | ChunkPageStore - (local aligned pages or immutable mirrored packs) -``` - -- **One crowdb-tree per consensus group.** A node hosting many groups owns many - lightweight `Crowdbtree` instances. Each owns its epoch manager, buffer pool, - and DiskIOUring; no shared process-wide state. -- **Two-level write path.** `apply` writes into the concurrent **MemTable - (L0)**; a single **Flusher** thread merges the contiguous-applied prefix - into the COW **B+tree (L1)**. The B+tree therefore has exactly one writer; - ingestion can be concurrent and out-of-order. Full design in - [`design-crowdb-tree-engine.md`](design-crowdb-tree-engine.md). -- **Concurrent readers.** Reads take an epoch guard, overlay the MemTable on - the B+tree (read amplification 2), and walk immutable pages with lock-free - atomic pointer loads. -- **Async I/O.** A single-thread DiskIOUring per `Crowdbtree` handles demand-load - reads and flush/snapshot writes via io_uring - ([`design-crowdb-tree-engine.md §3`](design-crowdb-tree-engine.md#3-async-ffi-bridge)). - Fast path (in-memory hit) completes synchronously; slow path (I/O) submits - an SQE and returns pending. - ---- +A host owns a `TreeRuntime`: a bounded CPU pool, resource partitions and +retirement services. Multiple Trees bind independent accounts and execution +lanes to that host. A standalone Tree without an injected runtime uses a +shared process runtime; servers inject their explicitly configured host. + +A `Crowdbtree` facade owns a `TreeCore` whose live state consists of: + +- `SourceAdmission`, with active and captured MemTables, writer grants and + the contiguous apply frontier; +- `PublishedTreeState`, an immutable root containing the mapping, source + catalog/floor, generation, coverage, commit sequence and logical statistics; +- a `PublicationCoordinator`, which serializes bounded prepared transactions + while range jobs, readers, encoding and I/O execute independently; +- a shared page cache and typed backing references, with physical ownership + separate from logical page identity and cache residency; +- an optional checkpoint backend, whose successful root commit establishes + durable coverage. + +L0 is an epoch-protected skip list. L1 is a persistent COW mapping of immutable +`PageState` and inner descriptors. Leaves compose base references and sorted +`OverlayBatch` runs. Split and merge preserve range ownership; page layout is +materialized only when required by bounded maintenance or persistence. + +The CPU pool does not wait synchronously for nested jobs. Continuations return +their lane while awaiting child work or I/O. Backend callbacks retain operation +state until terminal completion, including after cancellation or facade close. ## 3. Engine Abstraction -The Rust `KVEngine` trait exposes durability, snapshots, GC, and consistent -views; both `InMemKV` and `CrowdbTreeEngine` implement it: - -```rust -#[async_trait] -pub trait KVEngine: Send + Sync { - async fn apply(&self, slot: u64, batch: &Batch) -> Result<(), EngineError>; - async fn get(&self, key: &[u8]) -> Result<Option<(u64, Vec<u8>)>, EngineError>; - async fn scan(&self, prefix: &[u8], limit: usize) - -> Result<(Vec<(Vec<u8>, u64, Vec<u8>)>, bool), EngineError>; - - fn snapshot_view(&self) -> Arc<dyn EngineView>; // pin a consistent point-in-time version +`CrowdbTreeEngine` implements `KVEngine` with owned async apply/read/scan, +immutable views, typed maintenance reports and durable snapshot operations. +Proposal admission returns a native grant before consensus accepts a batch. +Follower and WAL replay use the accepted-work path and the reserved gap lane. +A failed preparation publishes no key and does not consume an automatic slot. - fn last_applied_slot(&self) -> u64; - async fn persist_snapshot(&self) -> Result<u64, EngineError>; - fn set_gc_watermark(&self, snapshot_slot: u64, safe_slot: u64); - async fn compact_sparse_blocks(&self) -> Result<MergeGcStats, EngineError>; - - fn snapshot_export(&self) -> BoxStream<'_, Result<Chunk, EngineError>>; - async fn snapshot_import(&self, chunks: BoxStream<'_, Chunk>) -> Result<(), EngineError>; - async fn clear(&self) -> Result<(), EngineError>; -} - -pub trait EngineView: Send + Sync { // compare / iter_all / range read on a fixed version - fn get(&self, key: &[u8]) -> Option<(u64, Vec<u8>)>; - fn iter_all(&self) -> Box<dyn Iterator<Item = (Vec<u8>, u64, Cell)> + '_>; - fn compare(&self, other: &dyn EngineView) -> Vec<EngineDiff>; - fn at_slot(&self) -> u64; -} -``` - -- **Async + fallible.** Persistence is async and can fail; the in-memory - engine implements the methods trivially (always `Ok`). -- **`snapshot_view()`** removes the "stop client traffic and wait for - quiescence" requirement `compare`/`iter_all`/range reads previously needed — - they run on a pinned version instead. -- **`last_applied_slot` / `persist_snapshot` / `set_gc_watermark` / - `compact_sparse_blocks`** make the state-machine self-persistence and the - logical retention-GC policy explicit interface methods (semantics in §3.1). +`EngineView` retains a captured immutable view. Reads, iteration and comparison +use that view's source catalog and mapping, including demand-loading cold +frames through their captured backend capabilities. Range rebuild and split +handoff publish through the same coordinator as ordinary writes. ### 3.1 Out-of-order apply, snapshots, and the two GCs -**Out-of-order apply is required.** `learn()` applies each chosen entry to the -engine immediately, so `apply(slot, batch)` can arrive out of slot order, e.g. -slot 7 before slot 6 in the parallel-slot window. The final materialized state -is **order-independent and idempotent** thanks to per-key highest-slot-wins. -The **learner**, not the engine, tracks the contiguous applied frontier. So the -engine contract is: accept `apply` at any slot; never corrupt; converge to -highest-slot-wins. *How* it absorbs out-of-order writes internally is the -two-level write path: `apply` lands in the concurrent MemTable; a single -Flusher merges the *contiguous-applied prefix* into the B+tree, so the B+tree -only ever sees ordered, contiguous, single-writer batches. +Apply can arrive out of slot order. Versions retain the greatest value needed +below the captured prefix and values above it; a future write must not hide a +value needed by an earlier contiguous flush. NoOp slots advance the frontier +explicitly and cannot be inferred from MemTable contents. -**Contiguous frontier comes from the learner.** A NoOp / repair-fill slot -carries an empty batch and leaves no MemTable entry, so crowdb-tree must **not** -infer the contiguous prefix from MemTable contents. It would block forever at -the gap left by a NoOp. Instead the learner passes its `contiguous_applied` -watermark down (`ct_advance_contiguous`), and the Flusher flushes only -MemTable entries with `slot ≤ contiguous_slot`. +A flush captures a finite contiguous frontier and source catalog, closes +source admission, waits for accepted writers and hands bounded ranges into L1. +Coverage advances only for published work. Failure may leave an installed +prefix: its report identifies that prefix, and retry preserves uncovered +sources. Flush does not itself advance the durable replay point. -**Flush trigger is dual.** The Flusher runs when the MemTable crosses a byte / -entry limit (**primary**) **or** a long time interval elapses since the last -flush (**secondary safety net**, default ~2 h). The size trigger keeps -per-flush work roughly constant; the time trigger bounds how long a -slow-write workload can leave L0 un-flushed (unbounded L0 ⇒ unbounded -crash-recovery replay). Each flush produces a new immutable COW root tagged -with the flushed slot = a snapshot, so `flush` *is* snapshot creation; the -public surface is unified under snapshot terminology (`create_snapshot`). +Writeback creates reloadable backing for selected dirty pages and performs +identity-checked representation rebases. It does not publish a durable root +or allow the host to advance its WAL checkpoint. A checkpoint captures one +view, completes its page/dependency writes and commits the backend root before +reporting durable coverage. Recovery starts from that coverage and replays +the external WAL suffix. -**`last_applied_slot` ownership is split.** The *learner* owns the in-memory -applied frontier (`contiguous_applied`); the *engine* owns only the -**durable** one. `last_applied_slot` = the highest contiguous slot whose -MemTable entries have been flushed and whose root is persisted -(`last_applied_slot ≤ contiguous_applied`). A snapshot = flush + persist root; -it needs only this single watermark plus a per-root slot tag, **not** a -per-key slot index. Because the B+tree only ever holds a clean contiguous -prefix, a snapshot is an exact point-in-time state and new-member install is -trivial: the receiver imports the root at `S`, then replays the WAL from -`S+1`. +Two distinct reclamation decisions apply: -**The two GCs, concretely.** - -1. *Data-retention GC.* The learner/replicator advances `safe_slot` (every - member applied through here) and `snapshot_slot` (state durable on a - quorum) and calls `set_gc_watermark(snapshot_slot, safe_slot)`. crowdb-tree - computes `gc_slot = min(safe_slot, snapshot_slot)` and, during the next - snapshot preparation, physically folds tombstones and - superseded cells whose slot `<= gc_slot`. **Policy is owned by the WAL/slot - layer; crowdb-tree only enforces the floor.** This is the same `gc_slot` the - consensus-WAL GC uses ([`design-crowdb-tree-storage.md §7`](design-crowdb-tree-storage.md#7-interaction-with-consensus-wal-gc)). -2. *Page reclamation.* Freeing B+tree pages and retired root versions once no - reader epoch references them — epoch-based, automatic, not slot-driven, no - external policy. - ---- +- Logical retention uses the externally supplied safe/snapshot floor to + discard obsolete values and tombstones without changing retained views. +- Physical reclamation waits for real reader, source, snapshot, cache, + candidate and backend owners to release. Logical removal alone does not + release a byte charge or authorize pack/block reuse. ## 4. FFI Boundary -The boundary is **coarse** (engine-level). The Rust `CrowdbTreeEngine` -(`lib/crowdb-kv/src/kv/crowdb_tree_engine.rs`) is a thin adapter that: - -- Owns an opaque `*mut ct_tree` handle returned by `ct_open`. -- Translates `Batch` / keys / ranges to `(ptr, len)` pairs across the C ABI. -- Bridges async↔sync via the io_uring engine + completion-based `ct_future` - protocol (fast path completes synchronously with zero scheduling overhead; - slow path parks on `AsyncFd` until the C++ DiskIOUring signals completion). - Full protocol in [`design-crowdb-tree-engine.md §3`](design-crowdb-tree-engine.md#3-async-ffi-bridge). -- Maps C status codes to `EngineError`. +The public ABI exposes opaque Tree, runtime, operation and result handles. +Private sender types remain within C++. Versioned structures validate their +size and version before reading or writing caller memory. -Ownership rules (enforced by convention, documented at the C API in -`lib/crowdb-tree/include/lib/crowdb-tree/c_api.h`, the single source of truth for exact -signatures): +Async work completes through an owned native future and a notification fd. +Rust checks completion around readiness registration and owns its registered +fd independently, so dropping a previous reactor cannot deregister a reused +native descriptor. `PinnedValue` and packed scan owners retain their actual +allocation or immutable frame owner across threads. -- Buffers passed *into* C are borrowed for the call's duration only. -- Buffers returned *from* C are either copied immediately by Rust, or - returned as an opaque owned handle freed by a matching `ct_free_*`. -- No C++ exceptions cross the boundary; everything is `noexcept` + status - code. +Known external `Bytes` backing is identified at the full allocation boundary. +Slices share payload capacity once within a host; slice/control holders have +separate charges. Unknown capacity or incompatible host ownership uses a +bounded owned copy. No C++ exception crosses the C boundary. ---- +A Tree permits one destructive work-window collector. It captures one immutable +sample and fans it out to logs and diagnostics. Ordinary gauges do not scan +the full mapping. Allocation counters follow actual allocation and release; +capturing a view does not require a separate statistics-registration pass. ## 5. Sub-Design Document Map -The crowdb-tree design is split into focused documents: - -| Doc | Covers | -| --- | --- | -| `design-crowdb-tree.md` (this) | Goals, architecture, `KVEngine` trait, FFI boundary, decision log. | -| [`design-crowdb-tree-engine.md`](design-crowdb-tree-engine.md) | **In-memory engine.** MemTable (L0) + COW B+tree (L1), slot-aware value cell, delta records + consolidation, split/merge, versioned root (MVCC snapshots), epoch-based reclamation, read path; the `buffer` memory-ownership model (zero-copy write/read pipelines); the io_uring async FFI bridge. | -| [`design-crowdb-tree-storage.md`](design-crowdb-tree-storage.md) | **Durable storage.** `PageStore` backends, on-disk zero-copy frame format, buffer pool (frame cache) + eviction safety, snapshot + internal-WAL decision + recovery, snapshot export/import; the mapping table (PID indirection, segment persistence, recycling); snapshot/GC flow integration with the learner and consensus WAL. | -| [`design-crowdb-tree-chunk-storage.md`](design-crowdb-tree-chunk-storage.md) | **Chunk storage.** Immutable mirrored page packs, async execution, fenced manifests and recovery, range rebuild, sharing, ownership materialization, and reclamation boundaries. | - -Test strategy for crowdb-tree (C++ unit, integration, crash/recovery, Rust FFI, -cross-engine parity, sanitizer) is documented in [`../kv/design-crowdb-kv-test.md`](../kv/design-crowdb-kv-test.md) § -"crowdb-tree C++ Test Layers". - ---- - -## 6. Decision Log - -| # | Decision | Rationale | -| --- | --- | --- | -| D1 | **Build crowdb-tree in C++** as `libcrowdb-tree`, consume from Rust over a C API. | The lowest storage layer (block device / RDMA) is C++. Placing the FFI boundary at the *top* (engine level) keeps the page-I/O hot path entirely in C++ and crosses FFI only at coarse `apply`/`get`/`scan`/`snapshot` calls. | -| D2 | **B+tree mutation is single-writer (the flush thread); no bw-tree lock-free CAS.** | The B+tree has exactly one mutator — the flush thread — holding the writer lock; readers do lock-free atomic loads. A bw-tree's lock-free multi-writer CAS adds large complexity that conflicts with COW + persistence + snapshots. Ingestion concurrency is absorbed *in front of* the tree by the MemTable (D3). | -| D3 | **Bounded 2-level: a concurrent memtable (L0) over one COW B+tree (L1); no multi-run LSM, no global compaction.** | Concurrent / out-of-order `apply` lands in the memtable; a single flush thread merges the *contiguous-applied prefix* into the B+tree ("flush = the persistent write"). Read amplification is bounded at **2** (memtable + tree). A classic multi-run LSM (many runs + per-run bloom + global compaction) is rejected for hurting linearizable `get`/`scan`. | -| D4 | **Tree family = COW B+tree + per-leaf delta chain + local consolidation.** | A "mini bw-tree without the lock-free machinery": leaf-level deltas give LSM-like write batching with low write amplification, while a single B+tree locate keeps reads cheap and snapshots clean. | -| D5 | **Slot is inlined into the value cell.** | Single-version, highest-slot-wins semantics require each entry to carry `(slot, kind, value)`. Slot is **not** in the key (that would create multiple versions). A batch at slot `S` fans out into one cell per key, scattered across possibly many leaves/pages, all carrying the same `S` — this many-cells-share-one-slot case is the *common* case (§3.1). | -| D6 | **Page lifecycle via epoch-based reclamation, owned per-`Crowdbtree`.** | This is the *physical* page-reclamation GC, internal and automatic: readers do a nanosecond-scale epoch enter/exit; the writer retires replaced pages; a GC thread reclaims after readers drain. The epoch manager is per-tree because the buffer pool (and therefore all retired pages) is already tree-private — there is no cross-tree page sharing, so a per-tree epoch is simpler and makes zero-copy borrowed reads fully tree-scoped. There is no shared cross-tree environment object. | -| D7 | **Engine abstraction is async and adds snapshot/retention-GC/`last_applied_slot`/consistent views.** | Persistence can fail and is async (io_uring / RDMA); `compare`/`iter_all` move onto a pinned consistent view instead of a quiescent global stop. `set_gc_watermark`/`compact_sparse_blocks` drive the **logical, slot-driven data-retention GC** (tombstone folding during snapshot) and **physical block compaction** (sparse-block relocation), *not* internal page reclamation (§3.1). | -| D8 | **Unified `buffer` memory model; single-allocation zero-copy pipeline.** | Key/value bytes are allocated once at the API boundary and moved down to the MemTable and into the frame; reads return borrowed views into resident frames (L1) or copies (L0). Replaces per-write `std::string`. Full design in [`design-crowdb-tree-engine.md §2`](design-crowdb-tree-engine.md#2-memory-and-buffer-management). | -| D9 | **MemTable ordered map = `absl::btree_map`.** | Chosen over `std::map` (poor cache locality) and skip list (cache-miss-heavy at MemTable scale). B-tree fanout gives 2–3× faster point lookups; ordered iteration is preserved for drain/snapshot. | -| D10 | **Snapshot/flush unified terminology + dual flush trigger.** | `flush` (drain L0→L1 + publish a new COW root) *is* snapshot creation. Trigger = MemTable size (primary) **OR** a long time interval (secondary safety net, default ~2 h) so a slow-write workload cannot leave L0 un-flushed and make crash recovery replay unbounded (§3.1). | -| D11 | **C++ logging via `spdlog`, hot-path-silent.** | Async ring-buffer file logger matching the Rust `tracing` file format. Hot paths (`apply`/`get`/`scan`) emit no info/warn logs; only structural events (flush/snapshot/recover) and errors log at info+. Off by default (`Config.log_dir`). | -| D12 | **Async FFI via io_uring + completion-based futures; no `spawn_blocking`, no large thread pools.** | Fast path (in-memory hit) completes synchronously with zero scheduling overhead; slow path (I/O) submits an io_uring SQE and returns pending. A single-thread C++ DiskIOUring per `Crowdbtree` processes completions and notifies the Rust `Future` via `eventfd` + Tokio `AsyncFd`. Full design in [`design-crowdb-tree-engine.md §3`](design-crowdb-tree-engine.md#3-async-ffi-bridge). | -| D13 | **Flush trigger = MemTable byte/entry limit OR time limit; flush the contiguous prefix only.** | The contiguous frontier is supplied by the learner (NoOp / repair-fill slots leave no MemTable entry, so the frontier must not be inferred from MemTable contents — otherwise the Flusher blocks at a NoOp gap). Flush entries with `slot ≤ contiguous_slot` (§3.1). | -| D14 | **One block-device backend covers raw SSD / SCM / mem-for-test / RDMA-remote; RDMA is not a separate backend.** | All are IU-aligned with a configurable IU that can be **1 byte** (mem / SCM) up to a flash page (SSD). RDMA-remote is just a remote block device; its cache/eviction details are deferred with the rest of the block backend. Full design in [`design-crowdb-tree-storage.md §2`](design-crowdb-tree-storage.md#2-backends). | -| D15 | **No multi-version; slot in the value cell, not the key.** | Single version per key, highest-slot-wins (§1, D5). | -| D16 | **Tree-level split/merge: not in v1, but not precluded.** | Designed so a future large-cluster *sharding* feature can split/merge whole trees (§1). | -| D17 | **No internal redo-WAL; snapshot-only recovery.** | crowdb-tree persists a snapshot = immutable root + `last_applied_slot`. On restart it composes with the external WAL: replay starts from `last_applied_slot+1`. Full rationale in [`design-crowdb-tree-storage.md §5`](design-crowdb-tree-storage.md#5-internal-wal-decision). | -| D18 | **Snapshot is implicit (a COW root version).** | Every flush/snapshot yields a new immutable root tagged with its slot = a snapshot, no explicit "create snapshot" API; callers obtain `(version, root, slot)` via `snapshot_view()`. `snapshot_export` iterates a pinned root (§3.1). | -| D19 | **Compression implemented (LZ4), off by default.** | LZ4 on-disk page compression is opt-in (`Config.compression = kLz4`); see [`design-crowdb-tree-storage.md §3.6`](design-crowdb-tree-storage.md#36-compression-details). | -| D20 | **Chunk storage is an injected private backend built from immutable mirrored packs.** | Tree algorithms keep the `PageStore` contract while chunk-KV supplies topology, ownership epochs, and root publication. Immutable pack and metadata sharing makes range rebuild cheap; generation fencing and later materialization preserve independent ownership. Full design in [`design-crowdb-tree-chunk-storage.md`](design-crowdb-tree-chunk-storage.md). | +- [Engine](design-crowdb-tree-engine.md): source versions, immutable pages, + publication, read ownership, resource accounting and async FFI. +- [Flush](design-crowdb-tree-engine-flush-flow.md): capture, bounded range + handoff, maintenance join, retry and reports. +- [Durable storage](design-crowdb-tree-storage.md): frames, cache residency, + Local backends, checkpoint, recovery and physical reclamation. +- [Checkpoint flow](design-crowdb-tree-engine-snapshot-flow.md): capture, + encoding/reuse, write completion, root commit and rebase. +- [Chunk storage](design-crowdb-tree-chunk-storage.md): fixed immutable packs, + process fencing, catalog pins, physical collection and explicit migration. + +## 6. Correctness Invariants + +- **I1 — Single published state:** mapping, source floor, coverage and logical + counters change in one coordinator transaction. Preparation may be parallel; + stale expected identities cannot overwrite a newer publication. +- **I2 — Accepted work is prepaid:** an admitted writer owns its complete + descriptor/payload allowance. Flush owns its handoff/routing/reclaim/I/O + windows before closing sources, including the capacity needed to free work. +- **I3 — Capture is finite:** new writes use the successor; they cannot extend + an already captured flush or checkpoint indefinitely. +- **I4 — Durability is explicit:** reloadable backing and successful handoff + are insufficient to advance durable coverage. Backend root commit is required. +- **I5 — Ownership is physical:** shared payload/backing bytes are charged once + to their canonical owner. Counters fall + only when the real owner releases or the physical backend confirms deletion. +- **I6 — Close preserves accepted lifetimes:** stop new admission, finish or + cancel accepted operations with exactly one terminal callback, drain backend + and retirement work, then destroy the host resources they require. diff --git a/doc/dev/crash_debugging.md b/doc/dev/crash_debugging.md index 3b39d281c..ceaa3e1bd 100644 --- a/doc/dev/crash_debugging.md +++ b/doc/dev/crash_debugging.md @@ -30,7 +30,7 @@ cat /proc/sys/fs/suid_dumpable The container's data volume does not override a host collector. During the 2026-09-29 Ubuntu development-host check, Apport received the crash but could -not resolve `/opt/crowdb/bin/crowdb-kv-server` in the host filesystem. +not resolve `/opt/crowdb/bin/crowdb-paxos-kv-server` in the host filesystem. The image contains the executable; the failed lookup happens in Apport. No CROWDB core was saved by that test. @@ -108,7 +108,7 @@ Use the exact image that ran the crashed process. Export its binary and shared libraries to a private directory, then run: ```sh -pixi run -- gdb -q /private/runtime/bin/crowdb-kv-server /private/core +pixi run -- gdb -q /private/runtime/bin/crowdb-paxos-kv-server /private/core (gdb) set solib-search-path /private/runtime/lib (gdb) sharedlibrary (gdb) set print frame-arguments none @@ -125,7 +125,7 @@ Bare-metal deployment keeps the binary's debug information. Use the exact binary and shared libraries that were running when the core was made: ```sh -pixi run -- gdb -q /path/to/exact/crowdb-kv-server /private/core +pixi run -- gdb -q /path/to/exact/crowdb-paxos-kv-server /private/core (gdb) set solib-search-path /path/to/exact/lib (gdb) sharedlibrary (gdb) set print frame-arguments none diff --git a/doc/dev/env_setup.md b/doc/dev/env_setup.md index 077bfa922..0b7d1b71c 100644 --- a/doc/dev/env_setup.md +++ b/doc/dev/env_setup.md @@ -89,7 +89,7 @@ so perf time-shares them. ## How CROWDB reads DRAM bandwidth in-process -CROWDB services (`crowdb-cli`, `crowdb-kv-server`, `crowdb-diskdb`, +CROWDB services (`crowdb-cli`, `crowdb-paxos-kv-server`, `crowdb-diskdb`, `crowdb-chunkdb`, `crowdb-diskio`) read the **same hardware uncore counters** directly via `perf_event_open` — no `perf` CLI or `pcm` needed at runtime. Two implementations exist (kept in sync): @@ -168,9 +168,9 @@ sentinel needs first: ```bash # KV / RPC / diskdb / chunkdb sentinels -pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server +pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server # chunkio write sentinel also needs diskdb + chunkdb + diskio (C++) -pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server \ +pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server \ -p crowdb-diskdb -p crowdb-chunkdb pixi run build ``` diff --git a/doc/doc_index.md b/doc/doc_index.md index 8497de03a..c2375c284 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -10,7 +10,7 @@ listed document or section needed by the task. | Doc | When to read | | --------------------------------------------------------- | ---------------------------------------------------------- | -| `doc/design/kv/design-crowdb-kv.md` | KV architecture and sub-design map. | +| `doc/design/kv/design-crowdb-paxos-kv.md` | KV architecture and sub-design map. | | `doc/design/protocol/design-crowdb-protocol.md` | Protocol and key encoding architecture. | | `doc/design/diskdb/design-crowdb-diskdb.md` | Diskdb lifecycle, placement, and concurrency. | | `doc/design/diskio/design-crowdb-diskio.md` | Disk I/O engine, fallbacks, cancellation, and RPC. | @@ -20,30 +20,20 @@ listed document or section needed by the task. | `doc/design/rpc/design-crowdb-rpc.md` | RPC engine, wire format, FFI, and transport. | | `doc/design/console/design-crowdb-console.md` | Console architecture and service lifecycle. | | `doc/design/config/design-crowdb-config.md` | Configuration ownership, precedence, validation, reload. | -| `doc/design/deploy/design-crowdb-deploy.md` | Node discovery, Group-0 bootstrap, multi-node deployment. | +| `doc/design/deploy/design-crowdb-deploy.md` | Node discovery, Group-0 bootstrap, multi-node deployment. | | `doc/design/access-server/design-crowdb-access-server.md` | S3, Iceberg, Dataset and shared native storage access. | -## Backlog (`doc/backlog/`) +## Flow-Analysis Docs -| Doc | When to read | -| ---------------------------------------- | ------------------------------------------------------------ | -| `doc/backlog/backlog.md` | Requirement index, priority, and status. | -| `doc/backlog/R**-<component>-<topic>.md` | One requirement's high-level design and acceptance contract. | - -## Working & Flow-Analysis Docs - -Temporary plans live under `doc/working/`; flow analyses live under -`doc/design/{access-server,kv,chunkio,rpc}/`. - -| Doc | When to read | -| ------------------------------------------------------------------------ | ------------------------------------------------------- | -| `doc/design/kv/kv-read-flow-analysis.md` | KV point-read flow and benchmarks. | -| `doc/design/kv/kv-scan-flow-analysis.md` | KV scan flow and benchmarks. | -| `doc/design/kv/kv-write-flow-analysis.md` | KV write flow and optimization evidence. | -| `doc/design/chunkio/chunkio-write-flow-analysis.md` | Chunk I/O large-write flow and benchmarks. | -| `doc/design/chunkio/chunkio-small-io-flow-analysis.md` | Chunk I/O small-I/O flow and benchmarks. | +| Doc | When to read | +| ------------------------------------------------------------------------ | -------------------------------------------------------- | +| `doc/design/kv/kv-read-flow-analysis.md` | KV point-read flow and benchmarks. | +| `doc/design/kv/kv-scan-flow-analysis.md` | KV scan flow and benchmarks. | +| `doc/design/kv/kv-write-flow-analysis.md` | KV write flow and optimization evidence. | +| `doc/design/chunkio/chunkio-write-flow-analysis.md` | Chunk I/O large-write flow and benchmarks. | +| `doc/design/chunkio/chunkio-small-io-flow-analysis.md` | Chunk I/O small-I/O flow and benchmarks. | | `doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md` | Native injection, SDK upload and container measurements. | -| `doc/design/rpc/rpc-flow-analysis.md` | RPC flow, benchmarks, and performance history. | +| `doc/design/rpc/rpc-flow-analysis.md` | RPC flow, benchmarks, and performance history. | ## Dev Environment (`doc/dev/`) @@ -70,7 +60,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | File | When to read | | -------------------- | ------------------------------------------- | -| `AGENTS.md` | Always-on project rules and skill dispatch. | +| `AGENTS.md` | Always-on project and validation rules. | | `VERSION` | Canonical project development version. | | `CONTRIBUTING.md` | PR setup, conventions, and process. | | `CHANGELOG.md` | Release history. | @@ -79,59 +69,59 @@ Temporary plans live under `doc/working/`; flow analyses live under ## Sub-Designs (`doc/design/{access-server,kv,tree,console,protocol,diskdb,diskio,chunkdb,chunkio,chunkds,rpc}/`) -| Doc | Read when working on | -| ----------------------------------------------------------------------------- | ------------------------------------------------------- | -| `doc/design/deploy/design-crowdb-oci-image.md` | Pixi OCI build, Docker/containerd, CI and macOS slot. | -| `doc/design/kv/design-crowdb-kv-leader-election.md` | Election, lease, ReadIndex, step-down. | -| `doc/design/kv/design-crowdb-kv-slot.md` | Slot pipelining, catch-up, coalescing, safe retention. | -| `doc/design/kv/design-crowdb-kv-rpc.md` | Paxos RPC transport, schema, and errors. | -| `doc/design/kv/design-crowdb-kv-rpc-client.md` | Client KV RPC, watch push, rollout, topology cache. | -| `doc/design/kv/design-crowdb-kv-reconfiguration.md` | Membership change, leader transfer, epoch fence. | -| `doc/design/kv/design-crowdb-kv-group0.md` | Group-0 schema, registry, topology, cleanup. | -| `doc/design/kv/design-crowdb-kv-state-machine.md` | Apply semantics, per-key slots, snapshot, compaction. | -| `doc/design/kv/design-crowdb-kv-wal.md` | WAL flush, replay, recovery, index, GC. | -| `doc/design/kv/design-crowdb-kv-watch-notify.md` | Watch registry, apply trigger, client, fallback. | -| `doc/design/kv/design-crowdb-kv-server.md` | KV server startup, API, lifecycle, cleanup. | -| `doc/design/kv/design-crowdb-kv-test.md` | KV test strategy and coverage. | -| `doc/design/kv/design-crowdb-kv-observability.md` | Metrics, collectors, instrumentation, logs. | -| `doc/design/chunkds/design-crowdb-chunk-kv.md` | Chunk-backed range KV, recovery, transfer, split. | -| `doc/design/chunkds/design-crowdb-chunk-kv-server.md` | Range catalog, fencing, balance, lifecycle. | -| `doc/design/chunkds/design-crowdb-chunk-kv-client.md` | Routed client, cache, composition, scans. | -| `doc/design/tree/design-crowdb-tree-engine.md` | In-memory engine, versioned root, reclamation, I/O. | -| `doc/design/tree/design-crowdb-tree-storage.md` | Durable pages, buffer pool, snapshot, mapping, GC. | -| `doc/design/tree/design-crowdb-tree-chunk-storage.md` | Mirrored page packs, manifests, rebuild, reclaim. | -| `doc/design/tree/design-crowdb-tree-engine-flush-flow.md` | L0→L1 flush path and bottlenecks. | -| `lib/crowdb-tree/bench/README.md` | MemTable handoff benchmark results and reproduction. | -| `doc/design/tree/design-crowdb-tree-engine-snapshot-flow.md` | Snapshot persist path and bottlenecks. | -| `doc/design/console/design-crowdb-console-ui.md` | UI contracts, resource menus, layout and inspection. | -| `doc/design/protocol/design-crowdb-protocol-key.md` | Binary/text key encoding and evolution. | -| `doc/design/protocol/design-crowdb-protocol-types.md` | Wire types, ID aliases, schemas, re-exports. | -| `doc/design/chunkdb/design-crowdb-chunkdb-mirror-to-ec.md` | Mirror-to-EC tasks, leases, publication, recovery. | -| `doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md` | Slot maps, dynamic ownership epochs, task authority. | -| `doc/design/chunkdb/design-crowdb-chunkdb-rpc.md` | Chunkdb RPC schema, service, transport, errors. | -| `doc/design/chunkdb/chunkdb-allocate-flow-analysis.md` | EC allocation benchmark and bottlenecks. | -| `doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md` | Shared-chunk admission, routing, recovery, elasticity. | -| `doc/design/chunkio/design-crowdb-chunkio-reader.md` | Mirror/EC reads, recovery, fencing, repair. | -| `doc/design/chunkds/design-crowdb-chunk-stream.md` | Frame groups, mirrored append, ordered chunk records. | -| `doc/design/diskdb/design-crowdb-diskdb-zone-management.md` | Zone allocation, free, compaction, recovery. | -| `doc/design/diskdb/design-crowdb-diskdb-space-metrics.md` | Capacity counters, reporting, aggregation. | -| `doc/design/diskdb/diskdb-allocate-flow-analysis.md` | Durable allocation flow and benchmarks. | -| `doc/design/rpc/design-crowdb-rpc-tcp.md` | TCP engines, worker loop, zero-copy I/O, scaling. | -| `doc/design/rpc/design-crowdb-rpc-rdma.md` | RDMA setup, CQ polling, registered buffers. | -| `doc/design/rpc/design-crowdb-rpc-diskdb-migration.md` | Diskdb RPC migration, rollout, connection lifetime. | -| `doc/design/access-server/s3/design-crowdb-access-s3.md` | S3 namespace, publication, integrity, list, delete. | -| `doc/design/access-server/s3/design-crowdb-access-s3-rdma.md` | Optional RDMA and direct-to-GPU data planes. | -| `doc/design/access-server/iceberge/design-crowdb-iceberg.md` | Iceberg ID paths, direct files and metadata scale. | -| `doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md` | Backed-up normative Iceberg table format specification. | -| `doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml` | Backed-up normative Iceberg REST Catalog OpenAPI. | -| `doc/design/access-server/dataset/design-crowdb-access-dataset.md` | Dataset authority, payload lifecycle, REST and SDK. | -| `doc/design/access-server/dataset/design-dataset-control.md` | Composition, Split revisions, bounded reads and cleanup. | -| `doc/design/access-server/dataset/spec-crowdb-dataset.md` | Normative Dataset behavior and compatibility. | -| `doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml` | Dataset REST v1 schemas, framing, bounds and retries. | -| `doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml` | Dataset REST v2 capability-gated control operations. | -| `doc/design/access-server/design-crowdb-native-sdk.md` | Trusted runtime, topology, S3 and host-lease ownership. | -| `lib/crowdb-sdk-native/README.md` | Native S3/Dataset configuration, streaming and HCA. | -| `sdk/README.md` | Rust/Python SDK installation and resource ownership. | +| Doc | Read when working on | +| ----------------------------------------------------------------------------- | --------------------------------------------------------- | +| `doc/design/deploy/design-crowdb-oci-image.md` | Pixi OCI build, Docker/containerd, CI and macOS slot. | +| `doc/design/kv/design-crowdb-paxos-kv-leader-election.md` | Election, lease, ReadIndex, step-down. | +| `doc/design/kv/design-crowdb-paxos-kv-slot.md` | Slot pipelining, catch-up, coalescing, safe retention. | +| `doc/design/kv/design-crowdb-paxos-kv-rpc.md` | Paxos RPC transport, schema, and errors. | +| `doc/design/kv/design-crowdb-paxos-kv-rpc-client.md` | Client KV RPC, watch push, rollout, topology cache. | +| `doc/design/kv/design-crowdb-paxos-kv-reconfiguration.md` | Membership change, leader transfer, epoch fence. | +| `doc/design/kv/design-crowdb-paxos-kv-group0.md` | Group-0 schema, registry, topology, cleanup. | +| `doc/design/kv/design-crowdb-paxos-kv-state-machine.md` | Apply semantics, per-key slots, snapshot, compaction. | +| `doc/design/kv/design-crowdb-paxos-kv-wal.md` | WAL flush, replay, recovery, index, GC. | +| `doc/design/kv/design-crowdb-paxos-kv-watch-notify.md` | Watch registry, apply trigger, client, fallback. | +| `doc/design/kv/design-crowdb-paxos-kv-server.md` | KV server startup, API, lifecycle, cleanup. | +| `doc/design/kv/design-crowdb-paxos-kv-test.md` | KV test strategy and coverage. | +| `doc/design/kv/design-crowdb-paxos-kv-observability.md` | Metrics, collectors, instrumentation, logs. | +| `doc/design/chunkds/design-crowdb-chunk-kv.md` | Chunk-backed range KV, recovery, transfer, split. | +| `doc/design/chunkds/design-crowdb-chunk-kv-server.md` | Range catalog, fencing, balance, lifecycle. | +| `doc/design/chunkds/design-crowdb-chunk-kv-client.md` | Routed client, cache, composition, scans. | +| `doc/design/tree/design-crowdb-tree-engine.md` | Sources, publication, owned reads, runtime and memory. | +| `doc/design/tree/design-crowdb-tree-storage.md` | Durable frames, shared cache, checkpoint and recovery. | +| `doc/design/tree/design-crowdb-tree-chunk-storage.md` | Mirrored page packs, manifests, rebuild, reclaim. | +| `doc/design/tree/design-crowdb-tree-engine-flush-flow.md` | Finite capture, parallel handoff and bounded maintenance. | +| `lib/crowdb-tree/bench/README.md` | MemTable handoff benchmark results and reproduction. | +| `doc/design/tree/design-crowdb-tree-engine-snapshot-flow.md` | Captured encoding, root commit and representation rebase. | +| `doc/design/console/design-crowdb-console-ui.md` | UI domains, operator flow, Chunk and Chunk-KV inspection. | +| `doc/design/protocol/design-crowdb-protocol-key.md` | Binary/text key encoding and evolution. | +| `doc/design/protocol/design-crowdb-protocol-types.md` | Wire types, ID aliases, schemas, re-exports. | +| `doc/design/chunkdb/design-crowdb-chunkdb-mirror-to-ec.md` | Mirror-to-EC tasks, leases, publication, recovery. | +| `doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md` | Slot maps, dynamic ownership epochs, task authority. | +| `doc/design/chunkdb/design-crowdb-chunkdb-rpc.md` | Chunkdb RPC schema, service, transport, errors. | +| `doc/design/chunkdb/chunkdb-allocate-flow-analysis.md` | EC allocation benchmark and bottlenecks. | +| `doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md` | Shared-chunk admission, routing, recovery, elasticity. | +| `doc/design/chunkio/design-crowdb-chunkio-reader.md` | Mirror/EC reads, recovery, fencing, repair. | +| `doc/design/chunkds/design-crowdb-chunk-stream.md` | Frame groups, mirrored append, ordered chunk records. | +| `doc/design/diskdb/design-crowdb-diskdb-zone-management.md` | Zone allocation, free, compaction, recovery. | +| `doc/design/diskdb/design-crowdb-diskdb-space-metrics.md` | Capacity counters, reporting, aggregation. | +| `doc/design/diskdb/diskdb-allocate-flow-analysis.md` | Durable allocation flow and benchmarks. | +| `doc/design/rpc/design-crowdb-rpc-tcp.md` | TCP engines, worker loop, zero-copy I/O, scaling. | +| `doc/design/rpc/design-crowdb-rpc-rdma.md` | RDMA setup, CQ polling, registered buffers. | +| `doc/design/rpc/design-crowdb-rpc-diskdb-migration.md` | Diskdb RPC migration, rollout, connection lifetime. | +| `doc/design/access-server/s3/design-crowdb-access-s3.md` | S3 namespace, publication, integrity, list, delete. | +| `doc/design/access-server/s3/design-crowdb-access-s3-rdma.md` | Optional RDMA and direct-to-GPU data planes. | +| `doc/design/access-server/iceberge/design-crowdb-iceberg.md` | Iceberg ID paths, direct files and metadata scale. | +| `doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md` | Backed-up normative Iceberg table format specification. | +| `doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml` | Backed-up normative Iceberg REST Catalog OpenAPI. | +| `doc/design/access-server/dataset/design-crowdb-access-dataset.md` | Dataset authority, payload lifecycle, REST and SDK. | +| `doc/design/access-server/dataset/design-dataset-control.md` | Composition, Split revisions, bounded reads and cleanup. | +| `doc/design/access-server/dataset/spec-crowdb-dataset.md` | Normative Dataset behavior and compatibility. | +| `doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml` | Dataset REST v1 schemas, framing, bounds and retries. | +| `doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml` | Dataset REST v2 capability-gated control operations. | +| `doc/design/access-server/design-crowdb-native-sdk.md` | Trusted runtime, topology, S3 and host-lease ownership. | +| `lib/crowdb-sdk-native/README.md` | Native S3/Dataset configuration, streaming and HCA. | +| `sdk/README.md` | Rust/Python SDK installation and resource ownership. | Prefer the most specific match. Open a root design only for cross-topic work; update its row when a permanent document is renamed or materially rescoped. diff --git a/doc/working/plan-rdma-multi-node.md b/doc/working/plan-rdma-multi-node.md deleted file mode 100644 index c189da8e1..000000000 --- a/doc/working/plan-rdma-multi-node.md +++ /dev/null @@ -1,117 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# RDMA physical multi-node validation Plan - -Upstream: [RDMA transport](../design/rpc/design-crowdb-rpc-rdma.md), -[native SDK](../design/access-server/design-crowdb-native-sdk.md). - -Goal: validate the existing RPC, storage and native SDK across distinct physical -hosts, with reproducible fabric, resource and failure evidence. - -Status: independently scheduled follow-up. This request refines the plan only; -no remote host, fabric or cross-host benchmark is changed or executed. Completion -of the local RDMA implementation and Dataset client work does not depend on this -plan. Hardware inventory must be supplied when execution is scheduled. - -## Scope and prerequisites - -- At least two distinct physical hosts with reachable peer HCA addresses. - Record host identity, NIC model/firmware, provider/kernel versions, link rate, - MTU, port/GID/IP mapping, NUMA placement and switch/fabric settings. -- Use the same source revision, native library hashes and immutable OCI image - digest on every host. Keep data directories, service IDs, endpoints and - cleanup ownership separate for this run. -- Host configuration supplies devices, memlock and non-root access. Application - and test code do not move links or enter other network namespaces. Changes - to shared fabric settings require separately authorized operator work. -- Each container receives an explicit device/address and nonoverlapping service - and CM ports. Benchmark port offsets and real-service explicit endpoints must - produce a recorded complete endpoint map; HCA physical port remains unchanged. -- Missing hosts, HCA access or peer reachability leaves this plan pending. It - cannot be replaced by localhost, two containers on one host or a TCP-only run. - -## Phase 1: inventory and remote fixture ownership - -- [ ] **Capture inventory**: write one machine-readable manifest per host and - a run manifest tying physical hosts, NICs and peer endpoints together. Probe - device access under the application UID, available registration capacity and - bidirectional reachability. Files: `tools/rdma/`, `e2e-test/`. -- [ ] **Extend the explicit launcher**: reuse immutable image/client bundles, - source checks, container labels, resource limits, reports and cleanup. Record - the ownership of every remote process and directory; cleanup only this run. - Preserve logs on startup failure and interruption. Files: - `tools/benchmark/rpc_bench_process.py`, `tools/rdma/`, `e2e-test/`. -- [ ] **Verify actual peer placement**: correlate TCP preflight/CM addresses, - QP identity and NIC port counters with the distinct hosts. Reject a result - whose endpoints accidentally connect within one host. Files: launcher and - structured transport observations. - -## Phase 2: transport correctness - -- [ ] **Run exact-byte corpus**: cover empty/small payloads, 64 KiB/256 KiB/1 MiB - classes, framing boundaries and logical controls beyond one receive window. - Run TCP, strict RDMA, single/multiple QPs and producers, packing off/on, idle - singleton and several batches in flight. Preserve client request counts and - per-request validation; SEND/CQE counts are separate observations. Files: - `tools/benchmark/rpc_regression.py`, `lib/crowdb-rpc/tests/`. -- [ ] **Exercise bounded pressure**: hold receive leases, saturate WR and byte - windows, then release capacity. Report posted receives, SQ occupancy, - registered bytes, held bytes and queue high water. All admitted owners must - remain valid until completion/release, including cancellation and shutdown. -- [ ] **Inject owned failures**: stop only this run's peer or selected service; - cover setup failure, stale generation and accepted partial delivery. Eligible - fallback preserves identity; partial reads fail terminally and ambiguous - mutations are not replayed. Do not flap a shared physical link to simulate - failure. Retain exact failing case and transport counters. - -## Phase 3: real services and embedded consumers - -- [ ] **Run distributed service corpus**: place real metadata/Chunk-KV/DiskIO - services and clients on distinct hosts, using service-discovered routing. - Verify HTTP, native Rust and unprivileged OCI results before and after owned - service restarts. Cover conditional errors, generation selection, copy, - multipart, mixed sizes and physical read integrity. Files: - `e2e-test/tests/access_s3/native/`, `tools/rdma/run-native-s3.py` and fixtures. -- [ ] **Verify caller lifetime**: retain registered read leases across stream - close and runtime shutdown while simulating asynchronous host-copy completion; - verify bounded producer/held capacity and final release. Record actual RDMA - readiness, fallback, QPs, allocation classes and copy counters per process. -- [ ] **Attach accepted Dataset corpus when available**: reuse native Dataset - acceptance without reimplementing its authority, Snapshot/Split selection or - acknowledgement. Keep this optional subcase explicit if Dataset is not yet - available; it cannot silently count as executed. - -## Phase 4: comparable performance and report - -- [ ] **Run pinned baselines**: use fixed physical cores/NUMA, the same message - distribution/seed, loader concurrency, QP count, warmup/duration/repeats and - payload-copy behavior for TCP and RDMA. Start with the checked-in mixed-size - profile, then include recorded application-size distributions when supplied. - Files: `tools/benchmark/profiles/rdma-mixed-size.json`, regression runners. -- [ ] **Report throughput and tails**: count completed client requests, bytes/s, - p50/p99/p999 by size, client+server CPU/request, memory and errors. Separate - closed-loop concurrency tests from intended-arrival open-load latency. Never - compare unrestricted cores against pinned runs as one performance gain. -- [ ] **Publish reproducible evidence**: retain source/image/binary identity, - full commands, host/fabric manifest, actual path/counters, failures and raw - trials. Explain remaining copies and hardware limitations. Do not impose an - invented TPS gate; record repeatability and latency/resource trade-offs. - -## Verification and exit - -- Unit: remote fixture selection, ownership and cleanup; incompatible/missing - prerequisites; truthful report classification and endpoint maps. -- Integration: bounded transport corpus, pre-payload fallback, partial failures, - shared-owner retention and no-device TCP regression. -- E2E: at least two physical hosts, real services, native/non-root OCI consumers, - exact results, restart/failure cases and retained-lease pressure. -- All strict-RDMA success cases require zero workload fallback, validation and - transport errors. Expected injected failures are identified separately, with - no successful partial result or premature owner reuse. -- Final report must prove cross-host execution and explain each skipped scope. - Missing environmental prerequisites remain pending with their exact evidence. - -Use the current gates through `pixi run`; launcher changes require fixture tests -and affected Rust/C++ formatting/lint/tests. Concrete remote commands are added -after inventory and endpoint selection, rather than embedding assumed hosts. diff --git a/doc/working/test.md b/doc/working/test.md deleted file mode 100644 index 1023664a3..000000000 --- a/doc/working/test.md +++ /dev/null @@ -1,220 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# Test tracking - -Track remaining issues and environment measurements. One row per domain; -record a complete run and keep its previous wall time. Detailed timings and -redacted diagnostics stay in the recorded artifact roots. - -## intel7960 - -Linux Docker bridge; clients run on the host and use inspected container IPs -and fixed ports. Wall times include startup, clients and teardown, excluding -shared builds. Record complete domain runs, including failures. Requested -concurrency and actual case capacity are recorded separately. New runs use 1024 -CPU shares per container without a CPU quota; complete native nodes retain -2048 MiB / 512 PIDs. Exact image and resource settings stay in run artifacts. - -### Container domains - -| Domain | Date | Passed / failed / skipped | Wall s | Previous s | -| ------------ | ---------- | ------------------------- | -------- | ---------- | -| KV | 2026-10-16 | 50 / 0 / 0 | 167.177 | 126.685 | -| Diskdb | 2026-10-16 | 5 / 0 / 0 | 12.531 | 12.679 | -| Diskio | 2026-10-16 | 8 / 0 / 0 | 20.851 | 27.230 | -| Chunkdb | 2026-10-16 | 11 / 0 / 0 | 145.752 | 87.330 | -| Chunk client | 2026-10-16 | 40 / 0 / 0 | 538.994 | 349.746 | -| Chunk stream | 2026-10-16 | 4 / 0 / 0 | 60.849 | 42.558 | -| Chunk-KV | 2026-10-16 | 2 / 0 / 0 | 35.266 | 30.160 | -| S3 | 2026-10-16 | 43 / 0 / 0 | 1009.182 | 877.167 | -| Iceberg | 2026-10-16 | 32 / 0 / 0 | 1984.972 | 2187.829 | -| Dataset | 2026-10-16 | 5 / 0 / 0 | 121.596 | 23.609 | -| Console | 2026-10-16 | 159 / 0 / 0 | 3211.651 | 2223.972 | -| Deployment | 2026-10-16 | 33 / 0 / 0 | 307.108 | 310.503 | - -- The 2026-10-16 full run completed on the working tree based on `664f04a1`: - all twelve domains passed, with 392 scenario executions and no failures or - skips. KV includes four additional isolation/recovery probes beyond the - 388 named flows. The first ten domains used optimized, stripped image - `5ede9f3a4f76` and source tree `03c0eb9e7f40`; Console and Deployment used - image `7a176ef2218b` and source tree `8bba22a39c62` after the UI repair. - Container wall time uses `timings.scenarios_wall`, excluding shared client - and image builds. Each task requests two case slots and a 4-CPU admission - budget; per-domain memory budgets and actual slot capacity stay in artifacts. -- Previous rows used the `9c0f8805` debug release. Iceberg uses one case slot - and includes the approximately 20-minute retirement grace in both runs. Build - profile, cache state and concurrency differences prevent direct performance - comparison. Historical diagnostics: `/tmp/crowdb-r230-shares-public`. -- Historical special profiles passed on that earlier image: bridge SSH - 148.600 s, Linux host 7.759 s, containerd host/cluster 111.815 / 112.784 s, - five-engine Iceberg 127.531 s and Python/DuckDB 20.458 s. These profiles - are separate from the standard twelve-domain full suite. -- Current container results and logs: - `.crowdb-runtime/artifacts/measure-tests/20261016T031530.418485Z/` and - `.crowdb-runtime/artifacts/measure-tests/20261016T054410.910103Z/`. - Consolidated verification and source/resource identities: - `.crowdb-runtime/artifacts/measure-tests/20261016T031530.418485Z/full-test-summary.json`. -- The first full attempt exposed a lost RPC write-interest rearm during - ChunkDB conversion recovery. Its Chunk Client result was 39 passed / 1 failed. - After fixing the event ordering, the complete domain passes 40 / 0 with the - original timeout. Initial diagnostics remain under - `.crowdb-runtime/artifacts/measure-tests/20261016T014707.773546Z/`. -- The next full Console attempt completed 159 flows with 152 passed / 7 failed. - It exposed stale storage/menu contracts, an authority recovery ordering gap - in a browser test and a scroll restoration race. After repair, the complete - Console and Deployment reruns passed 159 / 0 and 33 / 0 respectively - under `.crowdb-runtime/artifacts/measure-tests/20261016T054410.910103Z/`, - using image `7a176ef2218b` and source tree `8bba22a39c62`. The runtime source - difference from the earlier image is confined to UI scroll restoration. - -### Component tests - -One row per owning domain, combining its Rust/C++ packages and SDK protocol -peers. The 2026-10-16 component run passes all 4,658 cases, including an explicit run -of the installed-wheel Dataset Split HTTP test skipped by the standard Access -task. All twelve container domains also pass. - -Wall times include compilation inside each component command; the shared C++ -prerequisite and OCI builds are excluded. The shared image was rebuilt after the RPC fix, -and component tasks ran sequentially with existing compilation caches. Cache -state and dependency recompilation affect comparisons. - -| Domain | Date | Passed / failed / skipped | Wall s | Previous s | -| ------------------- | ---------- | ------------------------- | ------- | ---------- | -| access | 2026-10-16 | 154 / 0 / 0 | 209.090 | 125.050 | -| chunk-client | 2026-10-16 | 125 / 0 / 0 | 13.000 | 41.324 | -| chunk-kv | 2026-10-16 | 153 / 0 / 0 | 46.565 | 213.867 | -| chunk-stream | 2026-10-16 | 50 / 0 / 0 | 12.421 | 26.958 | -| chunkdb | 2026-10-16 | 160 / 0 / 0 | 137.869 | 113.006 | -| common | 2026-10-16 | 112 / 0 / 0 | 10.097 | 12.033 | -| console | 2026-10-16 | 293 / 0 / 0 | 177.636 | 190.015 | -| console-ui | 2026-10-16 | 275 / 0 / 0 | 18.570 | 17.968 | -| dataset | 2026-10-16 | 175 / 0 / 0 | 83.589 | 7.199 | -| deployment | 2026-10-16 | 130 / 0 / 0 | 89.446 | 45.902 | -| diskdb | 2026-10-16 | 201 / 0 / 0 | 37.024 | 40.249 | -| diskio | 2026-10-16 | 70 / 0 / 0 | 9.451 | 20.891 | -| iceberg | 2026-10-16 | 741 / 0 / 0 | 352.876 | 359.299 | -| kv | 2026-10-16 | 790 / 0 / 0 | 92.181 | 144.680 | -| protocol | 2026-10-16 | 240 / 0 / 0 | 9.585 | 23.567 | -| rpc | 2026-10-16 | 83 / 0 / 0 | 77.733 | 4.793 | -| s3 | 2026-10-16 | 114 / 0 / 0 | 9.400 | 13.606 | -| test-infrastructure | 2026-10-16 | 191 / 0 / 0 | 7.779 | 5.041 | -| tree | 2026-10-16 | 601 / 0 / 0 | 69.870 | 70.605 | - -- C++/FFI passes 779 cases. Common/RPC cases inherited through nested CTest - projects are counted only under their owning project. -- Dataset includes the standalone SDK workspace and ten installed Python wheel - consumer tests. The SDK source-distribution build reruns the same twenty Rust - tests; those repeated executions are not counted twice. -- Test infrastructure includes four Rust cases, 155 fixture/dispatch cases and - 32 OCI checks. CI ownership covers 28 packages and 388 named container flows. -- UI passes its type checks. Workspace Rust fmt/Clippy, standalone SDK boundary, - fmt/Clippy and CI test-task coverage checks pass. -- Component command logs: `.crowdb-runtime/artifacts/unit/`. Aggregated results: - `.crowdb-runtime/artifacts/measure-tests/20261016T031530.418485Z/component-summary.json`. - -## amd5950 - -AMD Ryzen 9 5950X, Linux Docker bridge; same component tasks and container -domains as Intel. Image revision `2529950f`; release binaries are optimized -and stripped, while the Intel container baseline used a debug release. -Timing comparisons therefore include a build-profile difference. - -### Container domains - -| Domain | Date | Passed / failed / skipped | Wall s | Previous s | -| ------------ | ---------- | ------------------------- | -------- | ---------- | -| kv | 2026-10-13 | 50 / 0 / 0 | 174.310 | — | -| diskdb | 2026-10-13 | 5 / 0 / 0 | 9.290 | — | -| diskio | 2026-10-13 | 8 / 0 / 0 | 15.500 | — | -| chunkdb | 2026-10-13 | 11 / 0 / 0 | 86.940 | — | -| chunk-client | 2026-10-13 | 40 / 0 / 0 | 274.390 | — | -| chunk-stream | 2026-10-13 | 4 / 0 / 0 | 31.490 | — | -| chunk-kv | 2026-10-13 | 2 / 0 / 0 | 19.140 | — | -| iceberg | 2026-10-13 | 31 / 1 / 0 | 1965.300 | — | -| dataset | 2026-10-13 | 2 / 0 / 0 | 16.200 | — | -| console | 2026-10-13 | 148 / 5 / 0 | 1955.340 | — | -| deployment | 2026-10-13 | 33 / 0 / 0 | 328.790 | — | -| s3 | 2026-10-13 | 43 / 0 / 0 | 1053.930 | — | - -- Wall times are the complete suite elapsed times, including startup, clients and teardown; shared builds are excluded. -- 1024 CPU shares per container, no CPU quota, one Tokio worker per service; - 4-CPU budget, two requested case slots; actual capacity is recorded per run. -- The first matrix stopped after Iceberg; Dataset and Deployment were rerun separately and passed. Console recorded five failures; the single-case retry reproduced the `io_uring_queue_init: Operation not permitted` / missing DiskIO owner failure under the host Docker seccomp profile. - Iceberg failed one case (`apache_rest_compatibility_kit_supported_catalog_surface`); - 31 cases passed. The first failure was Gradle distribution download timeout; after the cache - was populated, the unchanged case built successfully but exceeded its existing 960-second - client budget during the cold RCK build. The complete traces are in the recorded logs. -- Stability fix: native Console fixtures now launch their node containers with - `seccomp=unconfined`, allowing the required `io_uring_setup` path; the generic - bridge fixture remains unchanged. The focused fixture regression tests pass; the - The rebuilt-image six-node rerun reached service startup; the original missing-root race now - publishes the initial partition, but a later retry still sees a transient `disk group 3 has no - live DiskIO owner` while Chunk-KV connects. The remaining trace is in - `.crowdb-runtime/artifacts/console-retry-six-bootstrap-fix/`; the DiskIO ownership race remains - to be fixed before marking Console stable. -- Container artifacts: `.crowdb-runtime/artifacts/measure-tests/20261013T092119.686610Z/`, `.crowdb-runtime/artifacts/measure-tests/20261013T102929.186483Z/`, `.crowdb-runtime/artifacts/measure-tests/20261013T113259.109082Z/`, and `.crowdb-runtime/artifacts/console-retry-six/`. - -### Component tests - -| Domain | Date | Passed / failed / skipped | Wall s | Previous s | -| ------------------- | ---------- | ------------------------- | -------- | ---------- | -| access | 2026-10-13 | 117 / 0 / 0 | 95.677 | — | -| chunk-client | 2026-10-13 | 125 / 0 / 0 | 27.105 | — | -| chunk-kv | 2026-10-13 | 148 / 0 / 0 | 180.001 | — | -| chunk-stream | 2026-10-13 | 50 / 0 / 0 | 19.845 | — | -| chunkdb | 2026-10-13 | 158 / 0 / 0 | 146.525 | — | -| common | 2026-10-13 | 112 / 0 / 0 | 22.158 | — | -| console | 2026-10-13 | 264 / 0 / 0 | 196.454 | — | -| console-ui | 2026-10-13 | 183 / 0 / 0 | 9.391 | — | -| dataset | 2026-10-13 | 37 / 0 / 0 | 23.330 | — | -| deployment | 2026-10-13 | 75 / 0 / 0 | 55.542 | — | -| diskdb | 2026-10-13 | 183 / 0 / 0 | 39.004 | — | -| diskio | 2026-10-13 | 58 / 0 / 0 | 14.720 | — | -| iceberg | 2026-10-13 | 716 / 0 / 0 | 427.969 | — | -| kv | 2026-10-13 | 738 / 0 / 0 | 250.138 | — | -| protocol | 2026-10-13 | 221 / 0 / 0 | 12.738 | — | -| rpc | 2026-10-13 | 81 / 0 / 0 | 63.609 | — | -| s3 | 2026-10-13 | 114 / 0 / 0 | 31.354 | — | -| test-infrastructure | 2026-10-13 | 4 / 0 / 0 | 1.130 | — | -| tree | 2026-10-13 | 598 / 0 / 0 | 127.933 | — | - -- All seven component tasks pass; UI lint/type checks also pass. - The explicitly rerun boto3 case is counted once. Shared C++ prerequisite - build time is excluded; SDK dependency preparation is included. -- Test infrastructure currently records its 4 Rust cases; fixture/dispatch - and OCI checks are pending to match the Intel row. -- Component artifacts: `.crowdb-runtime/artifacts/measure-tests/20261013T073206.432338Z/` - and `.crowdb-runtime/artifacts/unit/`. -- Image construction succeeds after installing host `uidmap`; Docker import - loads the image but reports a config-digest mismatch. The container runner - validates source identity and uses the loaded image; retain this limitation. - -## macm5pro - -No container measurements. Linux VM routing remains unverified. - -## Remaining issues - -- [ ] **Remote CI acceptance**: CI #214 identifies a stale Docker socket after - daemon shutdown/package removal. Cleanup now verifies connection refusal - before removing that inode; active listeners and non-socket files fail safely. - Require a complete successful remote run before merge. -- [ ] **Bridge cleanup timing**: a run overlapping special profiles exceeded - the unchanged 45-second cleanup response bound. Independent diagnosis and - final uninstrumented acceptance pass, including all cleanup assertions. - Retain the failure evidence and investigate if it recurs; remote four-core - CI performance remains unmeasured. -- [ ] **macOS Iceberg SDK**: Maven `dependency:go-offline` stalled for more than - eight minutes. Rerun with the pinned dependency cache available. -- [ ] **WAL disk loss**: full fail-out and snapshot rejoin coverage awaits the - fail-out implementation (`design-crowdb-kv-wal.md`, section 8.1). -- [ ] **Per-group WAL disks**: startup currently derives disks from store-level - config; isolation coverage awaits per-group `wal_disks` configuration. -- [ ] **Network partition**: add sever/restore acceptance with an owned partition - mechanism. Existing crash/restart acceptance does not prove partition behavior. -- [ ] **CLI command and benchmark review**: redesign command definitions and - benchmark regression ownership in a later review. Current CLI lifecycle moves - to container clusters; keep `tools/benchmark/bench-chunk-kv-regression.sh` - until that review decides its scope. diff --git a/doc/working/todo_code.md b/doc/working/todo_code.md deleted file mode 100644 index a960b8ae8..000000000 --- a/doc/working/todo_code.md +++ /dev/null @@ -1,52 +0,0 @@ -<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> -<!-- Licensed under the Apache License, Version 2.0. --> - -# Code TODOs / FIXMEs - -Tracked `TODO` and `FIXME` comments across the codebase. Update this file -when adding or resolving a tracked item. - -## crowdb-rpc - -- **`lib/crowdb-rpc/src/transport/rdma/rdma_transport.cpp`** (lines 81, 102, - 138, 149, 159, 170, 177, 184, 199) — RDMA transport is a work-in-progress - stub. Major items: proper slab allocation from the registered memory - region (currently malloc + reg per buffer), completion queue polling, - send/recv path, connection management. All tracked inline; consolidate - here when RDMA work resumes. - -## crowdb-kv - -- **`lib/crowdb-kv/src/wal/file_backend.rs:45`** — switch `write_vectored_at` - to `FileExt::write_vectored_at` (`pwritev`) once it stabilizes - (rust tracking issue #89517). Would halve syscall count (no `lseek`). - -## crowdb-tree - -- **`lib/crowdb-tree/include/crowdb-tree/maptable/mapping_table.h:120`** — - `Config::mapping_segment_slots` is fixed at `kSegmentSize`; needs - parameterization for variable segment slot counts. -## crowdb-chunk-client - -- **`lib/crowdb-chunk-client/tests/common/mod.rs`** — `LocalFileDiskWriter` - uses synchronous `std::fs` I/O inside async `DiskWriter` methods (blocks - the executor). Acceptable for tests; if tests need true async I/O, wrap - in `spawn_blocking`. Also: `fsync` silently skips non-existent files - (partial strips where not all disks were written). The production - `DiskioBlockWriter` should handle this correctly via the diskio server. -- **`lib/crowdb-chunk-client/src/chunk/ec_strip_writer.rs`** — Whole-strip - retry is not implemented. A data or parity write failure currently aborts - the object; fresh placement retry needs a replacement allocation flow and - deterministic fault-injection coverage. - -## crowdb-chunkdb - -- **`app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs`** — - `DeleteChunkRange` is wired end to end but deliberately returns - `Unimplemented` without mutation until R95 adds exact used-range validation, - persistence, idempotency, and GC integration. -- **`lib/crowdb-chunk-client/tests/write_stream.rs`** — - `push_mode_drop_mid_write_deletes_partial` has no assertion on - `delete_calls` — `LargeObjectWriter` doesn't implement `Drop` cleanup - yet. Need a `Drop` impl that aborts the pipeline + deletes partial - chunks. diff --git a/e2e-test/Cargo.toml b/e2e-test/Cargo.toml index ce25102b1..866123aac 100644 --- a/e2e-test/Cargo.toml +++ b/e2e-test/Cargo.toml @@ -40,8 +40,8 @@ flatbuffers = { workspace = true } crowdb-diskio-client = { path = "../lib/crowdb-diskio-client", features = ["test-util"] } crowdb-rpc-ffi = { path = "../lib/crowdb-rpc/ffi" } crowdb-diskdb-client = { path = "../lib/crowdb-diskdb-client" } -crowdb-kv = { path = "../lib/crowdb-kv" } -crowdb-kv-client = { path = "../lib/crowdb-kv-client" } +crowdb-paxos-kv = { path = "../lib/crowdb-paxos-kv" } +crowdb-paxos-kv-client = { path = "../lib/crowdb-paxos-kv-client" } crowdb-protocol = { path = "../lib/crowdb-protocol" } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time"] } reqwest = { version = "0.12", features = ["json", "stream"] } diff --git a/e2e-test/README.md b/e2e-test/README.md index 3c957ac21..7bc433f85 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -8,8 +8,8 @@ is `e2e-test`; server domains use separate test modules within this crate. The Python runner owns container lifecycle and scheduling. Browser and SDK clients with independent language dependencies live under `clients/<client>/` and consume the same fixture endpoints and ownership contract. They do not -introduce separate server launchers. Domain selection and measured remaining issues are tracked in -[the test record](../doc/working/test.md). +introduce separate server launchers. The runner's domain registry defines +domain selection; each run retains its results and logs. Test clients run on the host against inspected node-container IPs and fixed ports, with three node containers for a three-node cluster. Fresh one- through eight-node profiles exercise management, membership, @@ -26,7 +26,7 @@ they are not test execution environments. Domain profiles configure that same server artifact rather than introducing separate test-server builds. This component owns real-server acceptance. The KV -profile uses three packaged `crowdb-kv-server` processes with block tree storage, +profile uses three packaged `crowdb-paxos-kv-server` processes with block tree storage, block-device WAL and the existing `e2e` election preset. It consumes the shared `crowdb-node` OCI runtime image; it never starts a host server substitute. @@ -229,8 +229,8 @@ and crash data remain available for investigation. launchers and overlapping acceptance files are removed. - Component tests retain internal invariants, planning, protocol peers and injected hardware/storage behavior. They do not duplicate the full server flow. -- Current complete-run results and remaining coverage gaps live only in - `doc/working/test.md`. +- Complete-run results and failures are retained in each run's artifacts. + A successful selected-domain run certifies only that domain. The cluster flows also cover exact follower hints, nontrivial topology identities, dynamic group creation/removal with isolated same-key values, and a five-node @@ -419,10 +419,9 @@ connection after the upstream commits; it never substitutes a service response. through label/PID/executable guards. Default anonymous-volume policy is an explicit exception to data bind mounts; logs/crash files are copied to host artifacts before its owned volume is removed. -- Domain wall times are recorded in [test.md](../doc/working/test.md); individual - client/fixture times remain in each run's `results.json`. - Manual admission, native host and ecosystem - profiles and final Pixi/CI classification remain in the working plan. +- Domain, client and fixture timings remain in each run's `results.json`. + Manual admission, native host and ecosystem profiles require explicit + selection; the checked Pixi/CI map defines automatic coverage. `bridge-admission` runs manual release nodes on an owned bridge, using inspected IPs and fixed Web/SSH listeners without host port publishing. Its five-node peak diff --git a/e2e-test/admission_profile.py b/e2e-test/admission_profile.py index 22401ba03..86076f3a4 100644 --- a/e2e-test/admission_profile.py +++ b/e2e-test/admission_profile.py @@ -79,7 +79,7 @@ def run_profile(args, parser): (run_root / "identity.json").write_text(json.dumps({"profile":"bridge-admission", "id":identity["Id"], "labels":labels, "sources":source, "cpus":cpus, "memory_mib":memory, "node_cpus":args.node_cpus, "host_data":str(data), "container_data":"/opt/crowdb/data", - "ports":{"web":9090,"ssh":2222}, + "ports":{"web":6135,"ssh":2222}, "core_policy":Path("/proc/sys/kernel/core_pattern").read_text().strip()}, indent=2)) bundle = client_bundle(run_root / "host-client") env = os.environ.copy() diff --git a/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts b/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts index af03a5d1b..62e83b1aa 100644 --- a/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts @@ -49,8 +49,14 @@ async function deployDiskDb(page: Page, request: APIRequestContext, kv: LiveServ await expect(dialog).toHaveCount(0); await diskMenu(page, node, 'Deploy DiskDB'); await expect(dialog.getByRole('button', { name: 'Deploy', exact: true })).toBeEnabled(); + await expect(dialog.getByLabel('HTTP Port (management)')).toHaveValue('13135'); await dialog.getByLabel('RPC Port (crowdb-rpc)').fill('7100'); + const deployed = page.waitForResponse(response => response.url().endsWith('/api/nodes/1/diskdb/deploy') + && response.request().method() === 'POST'); await dialog.getByRole('button', { name: 'Deploy', exact: true }).click(); + const response = await deployed; + expect(response.ok(), await response.text()).toBe(true); + expect(response.request().postDataJSON()).toMatchObject({ http_port: 13135, rpc_port: 7100 }); await expect(dialog).toHaveCount(0, { timeout: 5_000 }); const ddb = await liveService(request, 'diskdb'); await unchangedService(request, kv); diff --git a/e2e-test/clients/ecosystem/README.md b/e2e-test/clients/ecosystem/README.md index 2da77e958..b157ad18a 100644 --- a/e2e-test/clients/ecosystem/README.md +++ b/e2e-test/clients/ecosystem/README.md @@ -12,7 +12,7 @@ `pixi run -e iceberg-ecosystem test-iceberg-container-ecosystem`. No default task or default CI job depends on this environment or test. - Every run owns an isolated bridge network and normal release node with private - mounted data. Host SDK/JVM clients access its inspected IP and fixed port 9092; + mounted data. Host SDK/JVM clients access its inspected IP and fixed port 8135; the optional pinned Trino engine shares that bridge on port 8080. No host ports are published. Node recreation preserves the same data, IP and credentials. Cleanup removes owned containers/networks and retains data/log/crash files diff --git a/e2e-test/clients/ecosystem/acceptance.py b/e2e-test/clients/ecosystem/acceptance.py index 065b90762..390068844 100644 --- a/e2e-test/clients/ecosystem/acceptance.py +++ b/e2e-test/clients/ecosystem/acceptance.py @@ -46,7 +46,7 @@ def verify(cluster, data, all_engines): env = os.environ.copy() engine_env, trino = cluster.engine_environment() if all_engines else ({}, "") env.update(engine_env, ICEBERG_TOKEN=token, - CROWDB_PREVIEW_ICEBERG_URI=f"http://{cluster.addresses[0]}:9092", + CROWDB_PREVIEW_ICEBERG_URI=f"http://{cluster.addresses[0]}:8135", CROWDB_ECOSYSTEM_RESULTS=str(results), CROWDB_ECOSYSTEM_STATE=str(data / "state.json")) prefix = ["pixi", "run", "--as-is", "-e", "iceberg-ecosystem"] timings = {} diff --git a/e2e-test/console_access_fixture.py b/e2e-test/console_access_fixture.py index 36e246ca4..fad75804a 100644 --- a/e2e-test/console_access_fixture.py +++ b/e2e-test/console_access_fixture.py @@ -22,11 +22,11 @@ def start(self): "server": [{"id": f"kv-{index + 1}", "url": origin, "node_id": index + 1, "auto_start": False} for index, origin in enumerate(self.origins[:3])]}) address = start_node(self, 12, data, - ("server", "web", "--bind", "0.0.0.0", "--port", "9090", + ("server", "web", "--bind", "0.0.0.0", "--port", "6135", "--ui-root", "/opt/crowdb/ui", "--log-dir", "/opt/crowdb/data/log/web"), environment=("CROWDB_RUNTIME_ROOT=/opt/crowdb/data",)) self.addresses.append(address) - wait(lambda: HTTP.open(f"http://{address}:9090/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{address}:6135/healthz", timeout=5).read(), lambda body: body == b"ok", "release Access deployment Web listener", seconds=20) return self diff --git a/e2e-test/console_bootstrap_fixture.py b/e2e-test/console_bootstrap_fixture.py index ddd8de34c..14baa6d1f 100644 --- a/e2e-test/console_bootstrap_fixture.py +++ b/e2e-test/console_bootstrap_fixture.py @@ -17,7 +17,7 @@ def start(self): super().start() data = node_data(self, 1) profile = self.artifacts / "web.toml" - profile.write_text('version = 1\nmode = "bare-metal"\nbind = "0.0.0.0"\nport = 9090\n' + profile.write_text('version = 1\nmode = "bare-metal"\nbind = "0.0.0.0"\nport = 6135\n' f'group0_management_seeds = {json.dumps(self.origins)}\n' 'ui_root = "/opt/crowdb/ui"\nlog_dir = "/opt/crowdb/data/log/web"\n' 'log_max_file_mb = 30\nlog_max_files = 5\nrequest_timeout_ms = 500\n') @@ -25,7 +25,7 @@ def start(self): ("server", "web", "--config", "/opt/crowdb/etc/e2e-web.toml"), environment=("CROWDB_RUNTIME_ROOT=/opt/crowdb/data",), mounts=(f"type=bind,source={profile},target=/opt/crowdb/etc/e2e-web.toml,readonly",))) - wait(lambda: HTTP.open(f"http://{self.addresses[1]}:9090/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{self.addresses[1]}:6135/healthz", timeout=5).read(), lambda body: body == b"ok", "bootstrap Web listener") return self diff --git a/e2e-test/console_cli_fixture.py b/e2e-test/console_cli_fixture.py index 63b4a8922..a9fafce44 100644 --- a/e2e-test/console_cli_fixture.py +++ b/e2e-test/console_cli_fixture.py @@ -12,6 +12,8 @@ from fixture import Cluster, redact, request from console_cli_recovery import terminal_writer_recovery from console_cli_client import CliClient +from default_ports import (WEB_BASE, ACCESS_SERVER_HTTP_BASE, + ACCESS_SERVER_ICEBERG_HTTP_BASE, ACCESS_SERVER_DATASET_HTTP_BASE) LABEL = "org.crowdb.cluster" OWNER_LABEL = "org.crowdb.cluster.owner" @@ -39,13 +41,22 @@ def start(self): self.root.mkdir(parents=True, mode=0o700) # Claims belong to this live runner in the shared allocator. Receipt roots # remain per case; separate roots must not allocate overlapping host ports. - result = subprocess.run([str(self.bundle / "crowdb-cli"), "port-alloc", - "--service", "web", "--count", "24", "--owner-pid", str(os.getpid())], - capture_output=True, text=True, timeout=10, check=True) - ports = [int(port) for port in result.stdout.split()] - if len(ports) != 24 or ports != list(range(ports[0], ports[0] + 24)): - raise AssertionError("CLI fixture needs 24 contiguous published ports") - self.port_base = ports[0] + services = (("web", WEB_BASE), ("access-http", ACCESS_SERVER_HTTP_BASE), + ("access-iceberg-http", ACCESS_SERVER_ICEBERG_HTTP_BASE), + ("access-dataset-http", ACCESS_SERVER_DATASET_HTTP_BASE)) + offset = 0 + for service, base in services: + result = subprocess.run([str(self.bundle / "crowdb-cli"), "port-alloc", + "--service", service, "--instance", str(offset), "--count", "3", + "--owner-pid", str(os.getpid())], + capture_output=True, text=True, timeout=10, check=True) + ports = [int(port) for port in result.stdout.split()] + if service == "web" and ports: + offset = ports[0] - WEB_BASE + self.port_base = ports[0] + expected = list(range(base + offset, base + offset + 3)) + if ports != expected: + raise AssertionError(f"CLI fixture needs aligned {service} ports {expected}, got {ports}") self.interrupt("storage" if self.checkpoint == "terminal-writer" else self.checkpoint) self.refresh() return self @@ -125,11 +136,11 @@ def lifecycle(self, cluster_id): assert self.invoke("s3", "object", "get", "cli-replay", "witness") == "durable-container-witness" # Alternate-member endpoint attachment uses no local runtime selector. report = json.loads(self.invoke("cluster", "check", "--json", - endpoint=f"http://{self.addresses[2]}:9090")) + endpoint=f"http://{self.addresses[2]}:6135")) assert report["phase"] == "ready" and report["observation"]["cluster_id"] == cluster_id def current_readiness(self, cluster_id): - origin = f"http://{self.addresses[0]}:9090" + origin = f"http://{self.addresses[0]}:6135" node_id = self.read_owned("node-binding.json")["node_id"] services = request(origin, "/api/servers") service = next(item for item in services if item["node_id"] == node_id @@ -178,7 +189,7 @@ def file_disks(self): observed = [] for index, address in enumerate(self.addresses): node_id = self.read_owned("node-binding.json", index)["node_id"] - page = request(f"http://{address}:9090", f"/api/nodes/{node_id}/disk-files") + page = request(f"http://{address}:6135", f"/api/nodes/{node_id}/disk-files") assert page["next"] is None and len(page["items"]) == 1, page disk = page["items"][0] assert disk["phase"] == "registered" and disk["preparation_complete"] and disk["backing_identity"], disk @@ -197,10 +208,10 @@ def refresh(self): self.servers.append(name) self.addresses.append(info["NetworkSettings"]["Networks"][network]["IPAddress"]) self.containers = self.servers.copy() - self.origins = [f"http://{address}:10000" for address in self.addresses] + self.origins = [f"http://{address}:10135" for address in self.addresses] (self.artifacts / "endpoints.json").write_text(json.dumps({ - "addresses": self.addresses, "web_port": 9090, - "published_console_ports": [self.port_base + index * 10 for index in range(3)]}, indent=2)) + "addresses": self.addresses, "web_port": 6135, + "published_console_ports": [self.port_base + index for index in range(3)]}, indent=2)) def read_owned(self, path, index=0): if path not in ("node-binding.json", "node-policy.json", "diskdb-execution"): @@ -232,7 +243,7 @@ def diagnostics(self): for path in ("/api/cluster/status", "/api/disk-groups", "/api/disks", "/api/servers"): name = path.removeprefix("/api/").replace("/", "-") try: - value = request(f"http://{address}:9090", path) + value = request(f"http://{address}:6135", path) (self.artifacts / f"{name}-{index}.json").write_text(redact(value)) except (OSError, ValueError) as error: (self.artifacts / f"{name}-{index}.error").write_text(redact(str(error))) diff --git a/e2e-test/console_cli_operations.py b/e2e-test/console_cli_operations.py index 3933cc0f9..ec37becef 100644 --- a/e2e-test/console_cli_operations.py +++ b/e2e-test/console_cli_operations.py @@ -32,33 +32,33 @@ def hardware(cluster): def logical(cluster): - cluster.invoke("kv", "store", "add", "--id", "9", "--nodes", "1") - stores = cluster.invoke("kv", "store", "list", endpoint=cluster.endpoint(1)) + cluster.invoke("paxos-kv", "store", "add", "--id", "9", "--nodes", "1") + stores = cluster.invoke("paxos-kv", "store", "list", endpoint=cluster.endpoint(1)) assert any(line.split()[0] == "9" for line in stores.splitlines() if line.split()), stores for group in (90, 91): - cluster.invoke("kv", "group", "add", "--store", "9", "--group", str(group), + cluster.invoke("paxos-kv", "group", "add", "--store", "9", "--group", str(group), "--replica-id", str(group * 10), "--nodes", "1") - groups = cluster.invoke("kv", "group", "list", "--store", "9", endpoint=cluster.endpoint(2)) + groups = cluster.invoke("paxos-kv", "group", "list", "--store", "9", endpoint=cluster.endpoint(2)) assert "90" in groups and "91" in groups, groups - cluster.invoke("kv", "data", "put", "--store", "9", "--group", "90", "--key", "witness", "--value", "owned") - assert "owned" in cluster.invoke("kv", "data", "get", "--store", "9", "--group", "90", "--key", "witness") - cluster.invoke("kv", "group", "remove", "--store", "9", "--group", "91") - groups = cluster.invoke("kv", "group", "list", "--store", "9") + cluster.invoke("paxos-kv", "data", "put", "--store", "9", "--group", "90", "--key", "witness", "--value", "owned") + assert "owned" in cluster.invoke("paxos-kv", "data", "get", "--store", "9", "--group", "90", "--key", "witness") + cluster.invoke("paxos-kv", "group", "remove", "--store", "9", "--group", "91") + groups = cluster.invoke("paxos-kv", "group", "list", "--store", "9") assert "90" in groups and "91" not in groups, groups - cluster.invoke("kv", "store", "remove", "--id", "9") - stores = cluster.invoke("kv", "store", "list") + cluster.invoke("paxos-kv", "store", "remove", "--id", "9") + stores = cluster.invoke("paxos-kv", "store", "list") assert not any(line.split()[0] == "9" for line in stores.splitlines() if line.split()), stores def data(cluster): prefix = "cli-data/" + cluster.run_id for suffix in ("a", "b"): - cluster.invoke("kv", "data", "put", "--store", "0", "--group", "0", + cluster.invoke("paxos-kv", "data", "put", "--store", "0", "--group", "0", "--key", prefix + suffix, "--value", suffix) - scanned = cluster.invoke("kv", "data", "scan", "--store", "0", "--group", "0", + scanned = cluster.invoke("paxos-kv", "data", "scan", "--store", "0", "--group", "0", "--prefix", prefix, endpoint=cluster.endpoint(2)) assert prefix + "a" in scanned and prefix + "b" in scanned, scanned for suffix in ("a", "b"): - cluster.invoke("kv", "data", "delete", "--store", "0", "--group", "0", "--key", prefix + suffix) - assert "not found" in cluster.invoke("kv", "data", "get", "--store", "0", "--group", "0", + cluster.invoke("paxos-kv", "data", "delete", "--store", "0", "--group", "0", "--key", prefix + suffix) + assert "not found" in cluster.invoke("paxos-kv", "data", "get", "--store", "0", "--group", "0", "--key", prefix + suffix, endpoint=cluster.endpoint(1)) diff --git a/e2e-test/console_cli_recovery.py b/e2e-test/console_cli_recovery.py index a384e682c..3f07b853e 100644 --- a/e2e-test/console_cli_recovery.py +++ b/e2e-test/console_cli_recovery.py @@ -71,7 +71,7 @@ def managed_processes(cluster): def terminal_writer_recovery(cluster, cluster_id): - origin = f"http://{cluster.addresses[0]}:9090" + origin = f"http://{cluster.addresses[0]}:6135" inputs = cluster.state() identities = [cluster.inspect_owned(name)["Id"] for name in cluster.servers] cluster.invoke("s3", "bucket", "put", "writer-recovery") @@ -128,7 +128,7 @@ def terminal_writer_recovery(cluster, cluster_id): for address in cluster.addresses: for key in ("before", "after"): value = cluster.invoke("s3", "object", "get", "writer-recovery", key, - endpoint=f"http://{address}:9090") + endpoint=f"http://{address}:6135") assert value == f"acknowledged-{key}-fault", value (cluster.artifacts / "recovery-proof.json").write_text(json.dumps({ "cluster_id": cluster_id, "old_pids": old, "new_pids": new, "restarted": restarted, diff --git a/e2e-test/console_cli_target.py b/e2e-test/console_cli_target.py index 5b0a4002f..32eeb1918 100644 --- a/e2e-test/console_cli_target.py +++ b/e2e-test/console_cli_target.py @@ -41,11 +41,11 @@ def start(self): f"CROWDB_CLUSTER={self.run_id}", "CROWDB_INITIAL_VOTERS=3", "CROWDB_DEPLOYMENT_MODE=local", f"CROWDB_PHYSICAL_HOST_ID={host}", - f"CROWDB_S3_PUBLIC_URI=http://{address}:16000", - f"CROWDB_ICEBERG_PUBLIC_URI=http://{address}:17000", - f"CROWDB_DATASET_PUBLIC_URI=http://{address}:18000", - "CROWDB_DISCOVERY_SEEDS=" + ",".join(f"http://{peer}:9095" for peer in peers)))) - self.origins = [f"http://{address}:10000" for address in self.addresses] + f"CROWDB_S3_PUBLIC_URI=http://{address}:7135", + f"CROWDB_ICEBERG_PUBLIC_URI=http://{address}:8135", + f"CROWDB_DATASET_PUBLIC_URI=http://{address}:9135", + "CROWDB_DISCOVERY_SEEDS=" + ",".join(f"http://{peer}:26135" for peer in peers)))) + self.origins = [f"http://{address}:10135" for address in self.addresses] for name in self.servers: if not json.loads(self.command("inspect", name))[0]["State"]["Running"]: raise AssertionError("plain Docker node exited: " + self.command("logs", name)) @@ -62,7 +62,7 @@ def start(self): return self def endpoint(self, index): - return f"http://{self.addresses[index]}:9090" + return f"http://{self.addresses[index]}:6135" def access_listeners(self, index): return {"s3_port": 16000, "health_port": 19000} @@ -149,17 +149,17 @@ def attachment(self): before = self.inventory() for domain in ("rack", "node", "disk-group", "disk"): assert self.invoke("cluster", domain, "list", endpoint=self.endpoint(1)).strip() - assert self.invoke("kv", "store", "list").strip() - assert self.invoke("kv", "group", "list", "--store", "0").strip() + assert self.invoke("paxos-kv", "store", "list").strip() + assert self.invoke("paxos-kv", "group", "list", "--store", "0").strip() key, value = "cli-attach/" + self.run_id, "verified-selected-cluster" - self.invoke("kv", "data", "put", + self.invoke("paxos-kv", "data", "put", "--store", "0", "--group", "0", "--key", key, "--value", value) for index in range(3): - found = self.invoke("kv", "data", "get", "--store", "0", "--group", "0", "--key", key, + found = self.invoke("paxos-kv", "data", "get", "--store", "0", "--group", "0", "--key", key, endpoint=self.endpoint(index)) assert value in found, found - self.invoke("kv", "data", "delete", "--store", "0", "--group", "0", "--key", key) - assert "not found" in self.invoke("kv", "data", "get", "--store", "0", "--group", "0", "--key", key) + self.invoke("paxos-kv", "data", "delete", "--store", "0", "--group", "0", "--key", key) + assert "not found" in self.invoke("paxos-kv", "data", "get", "--store", "0", "--group", "0", "--key", key) self.invoke("s3", "bucket", "put", "cli-attach", endpoint=self.endpoint(2)) self.invoke("s3", "object", "put", "cli-attach", "witness", "--text", value) for index in range(3): diff --git a/e2e-test/console_discovery_browser_fixture.py b/e2e-test/console_discovery_browser_fixture.py index c55b35bd7..2a93867a8 100644 --- a/e2e-test/console_discovery_browser_fixture.py +++ b/e2e-test/console_discovery_browser_fixture.py @@ -20,10 +20,10 @@ def __init__(self, image, artifacts, bundle, **kwargs): def start(self): subnet = create_network(self) provider = str(subnet.network_address + 2) - monitor = "http://127.0.0.1:9095" + monitor = "http://127.0.0.1:26135" host = json.loads(self.command("info", "--format", "{{json .}}"))["ID"] self.addresses.append(start_node(self, 0, node_data(self, 0), - ("server", "web", "--bind", "0.0.0.0", "--port", "9091", + ("server", "web", "--bind", "0.0.0.0", "--port", "7135", "--ui-root", "/opt/crowdb/ui", "--log-dir", "/opt/crowdb/data/log/web", "--node-monitor", monitor), address=provider, environment=("CROWDB_RUNTIME_ROOT=/opt/crowdb/data",))) @@ -40,9 +40,9 @@ def start(self): "CROWDB_SSH_PASSWORD=crowdb", f"CROWDB_CLUSTER={self.run_id}", f"CROWDB_PHYSICAL_HOST_ID={host}"), network_container=self.servers[0])) - wait(lambda: HTTP.open(f"http://{self.addresses[0]}:9091/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{self.addresses[0]}:7135/healthz", timeout=5).read(), lambda body: body == b"ok", "independent release Web", seconds=20) - wait(lambda: HTTP.open(f"http://{self.addresses[0]}:9091/api/node/candidates", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{self.addresses[0]}:7135/api/node/candidates", timeout=5).read(), lambda body: json.loads(body)["local"]["advertisement"]["monitor_endpoints"], "release monitor observations", seconds=10) return self @@ -51,7 +51,7 @@ def browser(self, name, *, config="shell.config.ts", **kwargs): with Control(self) as control: super().browser(name, config=config, extra_env={ "CROWDB_E2E_CONTROL": str(control.directory), - "CROWDB_WEB_E2E_BASE_URL": f"http://{self.addresses[0]}:9091"}) + "CROWDB_WEB_E2E_BASE_URL": f"http://{self.addresses[0]}:7135"}) def control(self, body): if body != {"action": "stop-discovery-provider"}: diff --git a/e2e-test/console_fixture.py b/e2e-test/console_fixture.py index a4157108e..d0e35dd95 100644 --- a/e2e-test/console_fixture.py +++ b/e2e-test/console_fixture.py @@ -31,7 +31,7 @@ def browser(self, name): env = os.environ.copy() bypass = ",".join(filter(None, [env.get("NO_PROXY", ""), env.get("no_proxy", ""), self.address])) env.update(NO_PROXY=bypass, no_proxy=bypass, - CROWDB_WEB_E2E_BASE_URL=f"http://{self.address}:9090", + CROWDB_WEB_E2E_BASE_URL=f"http://{self.address}:6135", CROWDB_NATIVE_KV_PID=str(self.observation()["services"]["kv"]["pid"]), CROWDB_E2E_CONTROL=str(self.artifacts / "control"), CROWDB_WEB_E2E_OUTPUT=str(self.artifacts / "browser"), diff --git a/e2e-test/console_lifetime_fixture.py b/e2e-test/console_lifetime_fixture.py index a6e8778d3..d7f143fb6 100644 --- a/e2e-test/console_lifetime_fixture.py +++ b/e2e-test/console_lifetime_fixture.py @@ -36,11 +36,11 @@ def start(self): self.command("exec", name, "chown", "-R", "crowdb:crowdb", RUNTIME) self.command("exec", "-d", name, "sh", "-c", "CROWDB_RUNTIME_ROOT=" + RUNTIME + " /opt/crowdb/bin/entrypoint server web " - "--bind 0.0.0.0 --port 9090 --test-mode --ui-root /opt/crowdb/ui " + "--bind 0.0.0.0 --port 6135 --test-mode --ui-root /opt/crowdb/ui " "--log-dir /opt/crowdb/data/log/web > /opt/crowdb/data/log/web-launch.log 2>&1 & " "child=$!; echo $child > /opt/crowdb/data/web.pid; " "wait $child; echo $? > /opt/crowdb/data/web.exit") - wait(lambda: HTTP.open(f"http://{address}:9090/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{address}:6135/healthz", timeout=5).read(), lambda body: body == b"ok", "disposable release Web readiness", seconds=3) self.web_pid = int(self.command("exec", name, "cat", "/opt/crowdb/data/web.pid")) self.identity(self.web_pid, "crowdb-web") @@ -84,7 +84,7 @@ def control(self, body): super().control({"action": "status", "node": body.get("node")}) if action == "record-disposable-child": pid = body.get("pid") - start = self.identity(pid, "crowdb-kv-server") + start = self.identity(pid, "crowdb-paxos-kv-server") owners = self.manifests() if not any(record.get("pid") == pid and record.get("start") == start for owner in owners for record in owner.get("processes", [])): @@ -107,7 +107,7 @@ def control(self, body): # Component clean-env tests verify stale/persistent identity and # claim pruning. Here the host controller verifies Web's real # manifest wiring before acting within its owned PID namespace. - if self.identity(self.child_pid, "crowdb-kv-server") != self.child_start: + if self.identity(self.child_pid, "crowdb-paxos-kv-server") != self.child_start: raise ValueError("disposable process identity changed before cleanup") if not any(record.get("pid") == self.child_pid and record.get("start") == self.child_start for owner in self.manifests() for record in owner.get("processes", [])): diff --git a/e2e-test/console_native_fixture.py b/e2e-test/console_native_fixture.py index 4581716de..96b9a7e63 100644 --- a/e2e-test/console_native_fixture.py +++ b/e2e-test/console_native_fixture.py @@ -4,6 +4,7 @@ import json import re +import default_ports as ports import urllib.request import urllib.error from urllib.parse import urlparse @@ -68,7 +69,7 @@ def __init__(self, image, artifacts, bundle, *, disk_gib=(80, 8, 8), def api(self, path, body=None, method=None, status=200): payload = None if body is None else json.dumps(body).encode() origin = getattr(self, "management_address", self.addresses[0]) - request = urllib.request.Request(f"http://{origin}:9090" + path, + request = urllib.request.Request(f"http://{origin}:6135" + path, data=payload, method=method, headers={"Content-Type": "application/json"}) try: with HTTP.open(request, timeout=20) as response: @@ -116,9 +117,9 @@ def start(self): "CROWDB_DISK_INPUTS=" + json.dumps(self.candidate_inputs if index == 0 else [])))) self.ready() for address in self.addresses: - with HTTP.open(f"http://{address}:9090/api/node/auto-join", timeout=5) as response: + with HTTP.open(f"http://{address}:6135/api/node/auto-join", timeout=5) as response: assert json.load(response)["enabled"] is False - request = urllib.request.Request(f"http://{address}:9090/api/node/auto-join", + request = urllib.request.Request(f"http://{address}:6135/api/node/auto-join", data=json.dumps({"enabled": False}).encode(), headers={"Content-Type": "application/json"}) with HTTP.open(request, timeout=5) as response: @@ -167,11 +168,12 @@ def start(self): if self.auto_bootstrap: self.verify_auto_bootstrap() verify_file_preparation(self) - self.origins = [f"http://{address}:10000" for address in self.addresses] + self.origins = [f"http://{address}:{ports.PAXOS_KV_SERVER_MGMT_BASE}" for address in self.addresses] (self.artifacts / "native-endpoints.json").write_text(json.dumps({ "nodes": self.node_ids, "addresses": self.addresses, "disk_gib": self.disk_gib, "zone_gib": self.zone_gib, "spare_disk_gib": self.spare_disk_gib, "racks": self.rack_ids, "ports": { - "web": 9090, "ssh": 2222, "kv": 10000, "diskdb": 7100, "diskio": 7200}}, indent=2)) + "web": ports.WEB_BASE, "ssh": ports.SSH_PORT, "kv": ports.PAXOS_KV_SERVER_MGMT_BASE, + "diskdb": 7100, "diskio": 7200}}, indent=2)) return self def verify_auto_bootstrap(self): @@ -185,7 +187,7 @@ def verify_auto_bootstrap(self): assert accepted["operation"]["identity"]["operation_id"] == operation["operation_id"] assert accepted["create_data_group"] is False, accepted for address in self.addresses: - with HTTP.open(f"http://{address}:10000/stores/0/groups/0/ready", timeout=5) as response: + with HTTP.open(f"http://{address}:{ports.PAXOS_KV_SERVER_MGMT_BASE}/stores/0/groups/0/ready", timeout=5) as response: readiness = json.load(response) assert readiness["ready"] and readiness["voting_replicas"] == 3, readiness self.api("/api/node/auto-join", {"enabled": False}) @@ -199,7 +201,7 @@ def initialization_status(self): def ready(self): for address in self.addresses: - wait(lambda: HTTP.open(f"http://{address}:9090/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{address}:6135/healthz", timeout=5).read(), lambda body: body == b"ok", "native release Web listener", seconds=20) def observed_browser(self, name, *, config): diff --git a/e2e-test/console_recovery_fixture.py b/e2e-test/console_recovery_fixture.py index 283e4e52a..71400575e 100644 --- a/e2e-test/console_recovery_fixture.py +++ b/e2e-test/console_recovery_fixture.py @@ -23,14 +23,14 @@ def start(self): "server": [{"id": f"kv-{index + 1}", "url": origin, "node_id": index + 1, "auto_start": False} for index, origin in enumerate(self.origins)]}) self.addresses.append(start_node(self, 3, data, - ("server", "web", "--bind", "0.0.0.0", "--port", "9090", + ("server", "web", "--bind", "0.0.0.0", "--port", "6135", "--ui-root", "/opt/crowdb/ui", "--log-dir", "/opt/crowdb/data/log/web"), environment=("CROWDB_RUNTIME_ROOT=/opt/crowdb/data",))) self.web_ready() return self def web_ready(self): - wait(lambda: HTTP.open(f"http://{self.addresses[3]}:9090/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{self.addresses[3]}:6135/healthz", timeout=5).read(), lambda body: body == b"ok", "standalone release recovery readiness", seconds=20) def control(self, body): diff --git a/e2e-test/console_registry_fixture.py b/e2e-test/console_registry_fixture.py index 6d6bce592..c91bf7018 100644 --- a/e2e-test/console_registry_fixture.py +++ b/e2e-test/console_registry_fixture.py @@ -18,7 +18,7 @@ def start(self): service_id = "kv" host = "localhost" ssh_port = 22 -binary_path = "/opt/crowdb/bin/crowdb-kv-server" +binary_path = "/opt/crowdb/bin/crowdb-paxos-kv-server" service_config_path = "/opt/crowdb/data/service.toml" workspace = "/opt/crowdb/data" auto_start = true @@ -28,7 +28,7 @@ def start(self): (data / "web.toml").write_text('''version = 1 mode = "bare-metal" bind = "0.0.0.0" -port = 9090 +port = 6135 group0_management_seeds = ["http://127.0.0.1:9"] ui_root = "/opt/crowdb/ui" log_dir = "/opt/crowdb/data/log/web" @@ -40,6 +40,6 @@ def start(self): ("server", "web", "--config", "/opt/crowdb/data/web.toml", "--registry", "/opt/crowdb/data/launches.toml"), address=str(subnet.network_address + 2))) - wait(lambda: HTTP.open(f"http://{self.addresses[0]}:9090/healthz", timeout=3).read(), + wait(lambda: HTTP.open(f"http://{self.addresses[0]}:6135/healthz", timeout=3).read(), lambda body: body == b"ok", "registry Web listener", seconds=5) return self diff --git a/e2e-test/console_shell_fixture.py b/e2e-test/console_shell_fixture.py index a619a6c7a..e57e40b4a 100644 --- a/e2e-test/console_shell_fixture.py +++ b/e2e-test/console_shell_fixture.py @@ -33,7 +33,7 @@ def browser(self, name, *, config="shell.config.ts", extra_env=None, expected=1) browser_output = (self.artifacts / "browser").resolve() bypass = ",".join(filter(None, [env.get("NO_PROXY", ""), env.get("no_proxy", ""), address])) env.update(NO_PROXY=bypass, no_proxy=bypass, - CROWDB_WEB_E2E_BASE_URL=f"http://{address}:9090", + CROWDB_WEB_E2E_BASE_URL=f"http://{address}:6135", CROWDB_WEB_E2E_OUTPUT=str(browser_output)) env.update(extra_env or {}) test_budget = scaled_timeout(max(getattr(self, "browser_timeout", 120), diff --git a/e2e-test/console_standalone_fixture.py b/e2e-test/console_standalone_fixture.py index 2be13b007..b2fe10806 100644 --- a/e2e-test/console_standalone_fixture.py +++ b/e2e-test/console_standalone_fixture.py @@ -39,7 +39,7 @@ def start(self): config.write_text("broken config") address = str(subnet.network_address + 2) self.addresses.append(start_node(self, 0, data, - ("server", "web", "--bind", "0.0.0.0", "--port", "9090", + ("server", "web", "--bind", "0.0.0.0", "--port", "6135", "--ui-root", "/opt/crowdb/ui", "--log-dir", "/opt/crowdb/data/log/web"), address=address, environment=("CROWDB_RUNTIME_ROOT=/opt/crowdb/data",))) self.ready() @@ -54,7 +54,7 @@ def ready(self): (self.artifacts / "standalone-rejection.json").write_text(json.dumps({ "state": state, "log": self.command("logs", self.servers[0])}, indent=2)) else: - wait(lambda: HTTP.open(f"http://{self.addresses[0]}:9090/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{self.addresses[0]}:6135/healthz", timeout=5).read(), lambda body: body == b"ok", "standalone Web listener", seconds=20) def chunkdb_launch(self, action, node): diff --git a/e2e-test/console_standalone_observation.py b/e2e-test/console_standalone_observation.py index dbeadde03..2c2adf9c3 100644 --- a/e2e-test/console_standalone_observation.py +++ b/e2e-test/console_standalone_observation.py @@ -20,7 +20,7 @@ def control(self, body): if body != {"action": "observe-native-services", "node": 0}: raise ValueError("unsupported standalone observation") identity = owned(self.cluster, node) - with HTTP.open(f"http://{self.cluster.addresses[0]}:9090/api/servers", timeout=5) as response: + with HTTP.open(f"http://{self.cluster.addresses[0]}:6135/api/servers", timeout=5) as response: rows = json.load(response) observed = {} known = getattr(self.cluster, "native_processes", set()) diff --git a/e2e-test/console_typed_fixture.py b/e2e-test/console_typed_fixture.py index c7e9c66fa..1ea4cd973 100644 --- a/e2e-test/console_typed_fixture.py +++ b/e2e-test/console_typed_fixture.py @@ -33,7 +33,7 @@ def start(self): "--mount", f"type=bind,source={data},target=/opt/crowdb/data", self.image, "-R", "crowdb:crowdb", "/opt/crowdb/data/persistent/console/default") address = start_node(self, 3, data, - ("server", "web", "--skip-startup-restore", "--bind", "0.0.0.0", "--port", "9090", + ("server", "web", "--skip-startup-restore", "--bind", "0.0.0.0", "--port", "6135", "--ui-root", "/opt/crowdb/ui", "--log-dir", "/opt/crowdb/data/log/web"), environment=("CROWDB_RUNTIME_ROOT=/opt/crowdb/data",)) self.addresses.append(address) @@ -43,5 +43,5 @@ def start(self): def ready(self): if len(self.servers) < 4: return Cluster.ready(self) - wait(lambda: HTTP.open(f"http://{self.addresses[3]}:9090/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{self.addresses[3]}:6135/healthz", timeout=5).read(), lambda body: body == b"ok", "typed release Web listener", seconds=20) diff --git a/e2e-test/console_wal_fixture.py b/e2e-test/console_wal_fixture.py index 2c5666912..39d2de599 100644 --- a/e2e-test/console_wal_fixture.py +++ b/e2e-test/console_wal_fixture.py @@ -44,10 +44,10 @@ def start(self): "--mount", f"type=bind,source={data},target=/opt/crowdb/data", self.image, "-R", "crowdb:crowdb", "/opt/crowdb/data/persistent/console/default") self.addresses.append(start_node(self, index, data, - ("server", "web", "--skip-startup-restore", "--bind", "0.0.0.0", "--port", "9090", + ("server", "web", "--skip-startup-restore", "--bind", "0.0.0.0", "--port", "6135", "--ui-root", "/opt/crowdb/ui", "--log-dir", "/opt/crowdb/data/log/web"), environment=("CROWDB_RUNTIME_ROOT=/opt/crowdb/data",))) - wait(lambda: HTTP.open(f"http://{self.addresses[-1]}:9090/healthz", timeout=5).read(), + wait(lambda: HTTP.open(f"http://{self.addresses[-1]}:6135/healthz", timeout=5).read(), lambda body: body == b"ok", "WAL recovery release Web listener", seconds=20) return self diff --git a/e2e-test/console_web_fixture.py b/e2e-test/console_web_fixture.py index cd47049bd..829c36220 100644 --- a/e2e-test/console_web_fixture.py +++ b/e2e-test/console_web_fixture.py @@ -18,7 +18,7 @@ def start(self): for index in (3, 4): data = node_data(self, index) profile = self.artifacts / f"web-{index}.toml" - profile.write_text('version = 1\nmode = "bare-metal"\nbind = "0.0.0.0"\nport = 9090\n' + profile.write_text('version = 1\nmode = "bare-metal"\nbind = "0.0.0.0"\nport = 6135\n' f'group0_management_seeds = {json.dumps(self.origins)}\n' 'ui_root = "/opt/crowdb/ui"\nlog_dir = "/opt/crowdb/data/log/web"\n' 'log_max_file_mb = 30\nlog_max_files = 5\nrequest_timeout_ms = 500\n') @@ -26,10 +26,10 @@ def start(self): ("server", "web", "--config", "/opt/crowdb/etc/e2e-web.toml"), mounts=(f"type=bind,source={profile},target=/opt/crowdb/etc/e2e-web.toml,readonly",)) self.addresses.append(address) - origin = f"http://{address}:9090" + origin = f"http://{address}:6135" opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) wait(lambda: opener.open(origin + "/healthz", timeout=5).read(), lambda body: body == b"ok", "release Web listener") (self.artifacts / "web-endpoints.json").write_text(json.dumps({ - "kv": self.origins, "web": [f"http://{address}:9090" for address in self.addresses[3:]]}, indent=2)) + "kv": self.origins, "web": [f"http://{address}:6135" for address in self.addresses[3:]]}, indent=2)) return self diff --git a/e2e-test/default_ports.py b/e2e-test/default_ports.py new file mode 100644 index 000000000..a8a2f56e5 --- /dev/null +++ b/e2e-test/default_ports.py @@ -0,0 +1,22 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. + +"""Generated from ports.rs by tools/ports/generate.py; do not edit.""" + +PAXOS_KV_SERVER_MGMT_BASE = 10135 +PAXOS_KV_SERVER_RPC_BASE = 11135 +DISKDB_HTTP_BASE = 13135 +DISKDB_RPC_BASE = 14135 +CHUNKDB_HTTP_BASE = 16135 +CHUNKDB_RPC_BASE = 17135 +DISKIO_RPC_BASE = 18135 +WEB_BASE = 6135 +CHUNK_KV_HTTP_BASE = 20135 +CHUNK_KV_RPC_BASE = 21135 +ACCESS_SERVER_HTTP_BASE = 7135 +ACCESS_SERVER_ICEBERG_HTTP_BASE = 8135 +ACCESS_SERVER_DATASET_HTTP_BASE = 9135 +ACCESS_SERVER_HEALTH_HTTP_BASE = 25135 +MONITOR_HTTP_BASE = 26135 +SSH_PORT = 2222 +HOST_SSH_PORT = 22 diff --git a/e2e-test/deployment_anonymous_fixture.py b/e2e-test/deployment_anonymous_fixture.py index 8e3a92a18..837b452b3 100644 --- a/e2e-test/deployment_anonymous_fixture.py +++ b/e2e-test/deployment_anonymous_fixture.py @@ -23,9 +23,9 @@ def start(self): raise AssertionError("anonymous-volume startup exited: " + self.command("logs", self.servers[0])) try: self.command("exec", self.servers[0], "/opt/crowdb/bin/crowdb-monitor", "readiness") - if HTTP.open(f"http://{address}:9090/healthz", timeout=5).read() != b"ok": + if HTTP.open(f"http://{address}:6135/healthz", timeout=5).read() != b"ok": raise RuntimeError("default release Web health response is invalid") - if request(f"http://{address}:9090", "/api/cluster/status")["phase"] != "ready": + if request(f"http://{address}:6135", "/api/cluster/status")["phase"] != "ready": raise RuntimeError("default node application is not ready") break except (RuntimeError, OSError): @@ -35,7 +35,7 @@ def start(self): info = json.loads(self.command("inspect", self.servers[0]))[0] accepted = json.loads(self.command("exec", self.servers[0], "cat", "/opt/crowdb/data/accepted-node.json")) - status = request(f"http://{address}:9090", "/api/cluster/status") + status = request(f"http://{address}:6135", "/api/cluster/status") (self.artifacts / "anonymous-runtime.json").write_text(json.dumps({ "mounts": info["Mounts"], "accepted": accepted, "status": status, "log": self.command("logs", self.servers[0])}, indent=2)) diff --git a/e2e-test/deployment_browser_fixture.py b/e2e-test/deployment_browser_fixture.py index 30a0554c4..a1cd4a4a9 100644 --- a/e2e-test/deployment_browser_fixture.py +++ b/e2e-test/deployment_browser_fixture.py @@ -29,7 +29,7 @@ def browser(self): CROWDB_PREVIEW_TEST_ARTIFACTS=str(self.artifacts / "browser")) with Control(self): self.invoke(["pixi", "run", "node", "release-web.cjs", - f"http://{self.address}:9090", "owned-monitor"], project, env, "browser") + f"http://{self.address}:6135", "owned-monitor"], project, env, "browser") def invoke(self, arguments, project, env, phase): started = time.monotonic() @@ -55,7 +55,7 @@ def control(self, body): if type(body.get("pid")) is not int or body["pid"] != pid or pid <= 1: raise ValueError("browser service PID changed") executable = self.command("exec", self.servers[0], "cat", f"/proc/{pid}/cmdline").split("\0")[0] - if Path(executable).name != "crowdb-kv-server": + if Path(executable).name != "crowdb-paxos-kv-server": raise ValueError("browser service executable changed") self.command("exec", self.servers[0], "kill", "-" + body["signal"], str(pid)) return {"pid": pid} diff --git a/e2e-test/deployment_listener_fixture.py b/e2e-test/deployment_listener_fixture.py index 50ff83d9e..6ec46cdc0 100644 --- a/e2e-test/deployment_listener_fixture.py +++ b/e2e-test/deployment_listener_fixture.py @@ -4,6 +4,7 @@ import json import tomllib +import default_ports as ports from managed_fixture import ManagedCluster @@ -15,7 +16,8 @@ def __init__(self, image, artifacts, protocol, services, **kwargs): def start(self): super().start() config = self.command("exec", self.servers[0], "cat", "/opt/crowdb/data/services/access-1/service.toml") - port = {"s3": 9091, "iceberg": 9092}[self.protocol] + port = {"s3": ports.ACCESS_SERVER_HTTP_BASE, + "iceberg": ports.ACCESS_SERVER_ICEBERG_HTTP_BASE}[self.protocol] expected = f"{self.addresses[0]}:{port}" if tomllib.loads(config)[self.protocol]["listen"] != expected: raise AssertionError(f"{self.protocol} listener was not kept on its fixed port {expected}") diff --git a/e2e-test/diskdb_fixture.py b/e2e-test/diskdb_fixture.py index c39a6ea49..0f6d29e39 100644 --- a/e2e-test/diskdb_fixture.py +++ b/e2e-test/diskdb_fixture.py @@ -27,7 +27,6 @@ def start(self): config = self.artifacts / "diskdb.toml" config.write_text(f'''[server] rpc_workers = 2 -listen_addr = "0.0.0.0:7100" http_listen_addr = "0.0.0.0:7101" rpc_listen_addr = "{address}:7102" instance_id = "999" diff --git a/e2e-test/image_identity.py b/e2e-test/image_identity.py index 531f63fd5..5abb6bfa3 100644 --- a/e2e-test/image_identity.py +++ b/e2e-test/image_identity.py @@ -83,6 +83,13 @@ ":(exclude)tools/sdk/native_package.py", # The pinned Dataset consumer/fixture runner never enters server staging. ":(exclude)tools/sdk/dataset_workload.py", + # The staged node never builds/packages the operator CLI or KV shell drivers. + ":(exclude)app/crowdb-cli/Cargo.toml", + ":(exclude)app/crowdb-cli/src/commands/bench.rs", + ":(glob,exclude)app/crowdb-cli/src/commands/bench/**", + ":(glob,exclude)tools/benchmark/bench-kv-*.sh", + ":(exclude)tools/benchmark/test-bench-kv-write-sentinel.sh", + ":(exclude)container/oci-image/paxos-kv-cluster.py", # This host-only RPC benchmark is not a dependency or staged server binary. ":(exclude)app/crowdb-cli/src/commands/bench/rpc.rs", ":(exclude)app/crowdb-cli/src/commands/bench/result.rs", diff --git a/e2e-test/kv_node.py b/e2e-test/kv_node.py index 619b72b45..e3d888520 100644 --- a/e2e-test/kv_node.py +++ b/e2e-test/kv_node.py @@ -8,7 +8,7 @@ def arguments(cluster, index, address): group_args = ("--groups", ",".join(map(str, groups))) if groups else () store_args = ("--stores", ",".join(map(str, stores))) if stores else () advertised = ("--management-addr", address, "--management-advertise-addr", address) if address else () - return ("server", "kv", "--root", "/opt/crowdb/data/kv", "--management-port", "7000", + return ("server", "paxos-kv", "--root", "/opt/crowdb/data/kv", "--management-port", "7000", "--ports", "7001..7011", *store_args, *group_args, "--replica", str(index + 1), "--kv-backend", "block", "--wal-backend", "block-device", "--election-profile", "e2e", "--log-dir", "/opt/crowdb/data/log/kv", "--log", *advertised, *cluster.node_args[index]) diff --git a/e2e-test/managed_fixture.py b/e2e-test/managed_fixture.py index cc6e78694..c304ae4e6 100644 --- a/e2e-test/managed_fixture.py +++ b/e2e-test/managed_fixture.py @@ -5,6 +5,7 @@ import json import time from pathlib import Path +import default_ports as ports from allocation import create_network, node_data, start_node from fixture import Cluster, LABEL, request, wait @@ -39,7 +40,7 @@ def start(self): profile = self.artifacts / "profile.toml" profile.write_text(self.profile()) self.addresses.append(self.launch(data, profile)) - self.origins = [f"http://{self.addresses[0]}:10000"] + self.origins = [f"http://{self.addresses[0]}:{ports.PAXOS_KV_SERVER_MGMT_BASE}"] self.before_ready(data, profile) self.ready() return self @@ -52,20 +53,20 @@ def profile(self): def launch(self, data, profile): listeners = { - "paxos-kv": ("kv", {"http_port": 10000, "rpc_port": 10100}), - "diskdb": ("diskdb", {"rpc_port": 7100}), + "paxos-kv": ("kv", {"http_port": ports.PAXOS_KV_SERVER_MGMT_BASE, "rpc_port": ports.PAXOS_KV_SERVER_RPC_BASE}), + "diskdb": ("diskdb", {"rpc_port": ports.DISKDB_RPC_BASE, "http_port": ports.DISKDB_HTTP_BASE}), "diskio": ("diskio", {"rpc_port": 7200}), "chunkdb": ("chunkdb", {"http_port": 7300, "rpc_port": 7301}), "chunk-kv": ("chunk-kv", {"http_port": 7400, "rpc_port": 7401}), - "access-server": ("access", {"http_port": 9092, "s3_port": 9091, - "dataset_port": 9093, "health_port": 9094}), + "access-server": ("access", {"http_port": 8135, "s3_port": 7135, + "dataset_port": 9135, "health_port": 25135}), } policy = self.artifacts / "node-policy.toml" choices = [] - for kind, (role, ports) in listeners.items(): - ports = ", ".join(f"{key} = {value}" for key, value in ports.items()) + for kind, (role, service_ports) in listeners.items(): + port_values = ", ".join(f"{key} = {value}" for key, value in service_ports.items()) enabled = str(role in self.services).lower() - choices.append(f'{{ kind = "{kind}", enabled = {enabled}, listeners = {{ {ports} }} }}') + choices.append(f'{{ kind = "{kind}", enabled = {enabled}, listeners = {{ {port_values} }} }}') choices = ", ".join(choices) storage = 'mode = "automatic"\ncapacity_bytes = 17179869184\nreserve_bytes = 1073741824\nsparse = true' \ if "diskio" in self.services else 'mode = "diskless"' @@ -89,7 +90,7 @@ def ready(self): if not state["Running"]: raise AssertionError("normal release monitor exited: " + self.command("logs", self.servers[0])) try: - report = request(f"http://{self.addresses[0]}:9090", "/api/cluster/status") + report = request(f"http://{self.addresses[0]}:6135", "/api/cluster/status") last = report if report["phase"] == "failed": raise AssertionError(f"normal selected release services failed: {report}") @@ -167,7 +168,7 @@ def control(self, body): if type(pid) is not int or pid <= 1: raise ValueError("service PID must identify a child of the release monitor") executable = self.command("exec", name, "cat", f"/proc/{pid}/cmdline").split("\0")[0] - expected = {"kv": "crowdb-kv-server", "diskdb": "crowdb-diskdb", + expected = {"kv": "crowdb-paxos-kv-server", "diskdb": "crowdb-diskdb", "diskio": "crowdb-diskio", "chunkdb": "crowdb-chunkdb", "chunk-kv": "crowdb-chunk-kv-server", "access": "crowdb-access-server", "web": "crowdb-web"}[service] diff --git a/e2e-test/oci_host_node.py b/e2e-test/oci_host_node.py index 1aa925bed..06fa3e3ba 100644 --- a/e2e-test/oci_host_node.py +++ b/e2e-test/oci_host_node.py @@ -40,7 +40,7 @@ def __init__(self, root, bundle, image, runtime, *, address=None, voters=1, meta self.cli = self.bundle / 'crowdb-cli' self.cli_digest = self.client_digest() self.interface, self.ip = interface_address() - self.origin = f'http://{self.ip}:9090' + self.origin = f'http://{self.ip}:6135' self.voters, self.metadata_only = voters, metadata_only self.private = self.root / 'private' self.private.mkdir(mode=0o700) @@ -71,10 +71,10 @@ def make_policy(self): for kind in ('paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server'): lines += ['[[services]]', f'kind = "{kind}"', f'enabled = {str(kind == "paxos-kv").lower()}'] if kind == 'paxos-kv': - lines += ['[services.listeners]', 'http_port = 10000', 'rpc_port = 10100'] + lines += ['[services.listeners]', 'http_port = 10135', 'rpc_port = 11135'] else: lines += ['[public_access]', *[f'{kind} = "http://{self.ip}:{port}"' - for kind, port in [('s3',9091), ('iceberg',9092), ('dataset',9093)]], '[storage]'] + for kind, port in [('s3',7135), ('iceberg',8135), ('dataset',9135)]], '[storage]'] if self.devices: for device in self.devices: os.chown(device, 10001, 10001) diff --git a/e2e-test/oci_local_host_node.py b/e2e-test/oci_local_host_node.py index 30e229993..ce249d012 100644 --- a/e2e-test/oci_local_host_node.py +++ b/e2e-test/oci_local_host_node.py @@ -20,7 +20,7 @@ def make_policy(self): self.devices, self.disk_files = [], [] lines = ['version = 1', 'mode = "local"', f'cluster = "{self.cluster}"', 'initial_voters = 1', '[public_access]', *[f'{kind} = "http://{self.ip}:{port}"' - for kind, port in [('s3',9091), ('iceberg',9092), ('dataset',9093)]], + for kind, port in [('s3',7135), ('iceberg',8135), ('dataset',9135)]], '[storage]', 'mode = "automatic"', 'capacity_bytes = 8589934592', 'reserve_bytes = 1073741824', 'sparse = true'] self.policy.write_text('\n'.join(lines) + '\n') diff --git a/e2e-test/oci_reference_recovery.py b/e2e-test/oci_reference_recovery.py index 537b8f65a..eff64c615 100644 --- a/e2e-test/oci_reference_recovery.py +++ b/e2e-test/oci_reference_recovery.py @@ -40,7 +40,7 @@ def create_nonvoter(cluster): '--mount', f'type=bind,source={policy.resolve()},target=/run/later-member.toml,readonly', '--env', 'CROWDB_NODE_POLICY_FILE=/run/later-member.toml', '--env', f'CROWDB_CLUSTER={cluster.run_id}', '--env', 'CROWDB_INITIAL_VOTERS=3', - '--publish', f'127.0.0.1:{cluster.port_base + 30}:9090', cluster.image) + '--publish', f'127.0.0.1:{cluster.port_base + 30}:6135', cluster.image) cluster.containers.append(name) @@ -85,7 +85,7 @@ def unreachable(endpoint): raise AssertionError(f'old management transport remains reachable: {endpoint}') except OSError: return endpoint - endpoints = [(address, port) for address in addresses for port in (2222, 9090, 9095, 10000, 10100)] + endpoints = [(address, port) for address in addresses for port in (2222, 6135, 26135, 10135, 11135)] with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: return list(executor.map(unreachable, endpoints)) diff --git a/e2e-test/oci_reference_runtime.py b/e2e-test/oci_reference_runtime.py index 5030c7bc4..07ba5e482 100644 --- a/e2e-test/oci_reference_runtime.py +++ b/e2e-test/oci_reference_runtime.py @@ -7,6 +7,8 @@ from pathlib import Path import subprocess +import default_ports + from fixture import LABEL, request, wait ROOT = Path(__file__).resolve().parents[1] @@ -90,7 +92,9 @@ def create_docker_nodes(cluster, indexes): "--env", f"CROWDB_CLUSTER={cluster.run_id}", "--env", f"CROWDB_INITIAL_VOTERS={cluster.node_count}"] for protocol, port in ports(cluster.port_base + index * 10).items(): - internal = {"UI": 9090, "S3": 16000, "ICEBERG": 17000, "DATASET": 18000}[protocol] + internal = {"UI": default_ports.WEB_BASE, "S3": default_ports.ACCESS_SERVER_HTTP_BASE, + "ICEBERG": default_ports.ACCESS_SERVER_ICEBERG_HTTP_BASE, + "DATASET": default_ports.ACCESS_SERVER_DATASET_HTTP_BASE}[protocol] arguments += ["--publish", f"127.0.0.1:{port}:{internal}"] if protocol != "UI": arguments += ["--env", f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{port}"] diff --git a/e2e-test/profiles/bridge_node.py b/e2e-test/profiles/bridge_node.py index e767b250e..8d1e9b3cc 100644 --- a/e2e-test/profiles/bridge_node.py +++ b/e2e-test/profiles/bridge_node.py @@ -26,7 +26,7 @@ def origin_for(name): address = node["NetworkSettings"]["Networks"][PREFIX]["IPAddress"] if not address: raise ValueError("owned bridge node has no inspected IP") - return f"http://{address}:9090" + return f"http://{address}:6135" def start_node(name, *, default_setup=False): diff --git a/e2e-test/profiles/containerd-cluster.py b/e2e-test/profiles/containerd-cluster.py index 06d76f47f..7d3efcb44 100644 --- a/e2e-test/profiles/containerd-cluster.py +++ b/e2e-test/profiles/containerd-cluster.py @@ -86,7 +86,7 @@ def main(): 'pixi run --as-is --manifest-path container/oci-image/pixi.toml -e runtime ' 'python e2e-test/profiles/containerd-host.py /opt/crowdb-e2e/archive.tar --serve') address = json.loads(docker("inspect", host))[0]["NetworkSettings"]["Networks"][prefix]["IPAddress"] - origins.append("http://" + address + ":9090") + origins.append("http://" + address + ":6135") # Host creation includes archive unpacking. Finish each owned host's # ordinary management startup before importing into another daemon. wait(lambda: request(origins[-1], '/api/node/candidates'), diff --git a/e2e-test/profiles/node-recovery.py b/e2e-test/profiles/node-recovery.py index 7344993f1..cdd2436fc 100644 --- a/e2e-test/profiles/node-recovery.py +++ b/e2e-test/profiles/node-recovery.py @@ -5,6 +5,7 @@ """Independent cluster cleanup, persistent endpoint changes and voting catch-up.""" import json from bridge_node import start_node, origin_for, browser +import default_ports as ports from setup_choices import diskless_setup @@ -62,11 +63,11 @@ def verify(docker, request, wait, image, network, names, origins, cluster): origins.append(origin) for gid, replica_id in [(0, 601), (3, 600)]: request(origins[0], f"/api/stores/0/groups/{gid}/replicas", {"node_id":joined["node_id"], "replica_id":replica_id}) - topology = json.loads(docker("exec", name, "curl", "-fsS", "http://127.0.0.1:10000/topology")) + topology = json.loads(docker("exec", name, "curl", "-fsS", f"http://127.0.0.1:{ports.PAXOS_KV_SERVER_MGMT_BASE}/topology")) group = next(group for store in topology["stores"] if store["store_id"] == 0 for group in store["groups"] if group["group_id"] == gid) assert group["local_replica_id"] == replica_id and group["read_state"]["contiguous_applied"] > 0, group for peer in names[:3]: - topology = json.loads(docker("exec", peer, "curl", "-fsS", "http://127.0.0.1:10000/topology")) + topology = json.loads(docker("exec", peer, "curl", "-fsS", f"http://127.0.0.1:{ports.PAXOS_KV_SERVER_MGMT_BASE}/topology")) group = next(group for store in topology["stores"] if store["store_id"] == 0 for group in store["groups"] if group["group_id"] == gid) assert any(remote["id"] == replica_id and remote["voting"] for remote in group["remotes"]), group assert request(origin, "/api/stores/0/groups/3/kv/get?key=remote-quorum")["value_utf8"] == "wired" diff --git a/e2e-test/release.py b/e2e-test/release.py index cfae06763..7971407e3 100644 --- a/e2e-test/release.py +++ b/e2e-test/release.py @@ -71,7 +71,7 @@ def check(case, arguments, expected, mounts=(), environment=(), entrypoint=None) ("version", "unknown field"), mounts=( f"type=bind,source={old_config.resolve()},target=/opt/crowdb/etc/old-web.toml,readonly",)) managed_config = root / "managed-web.toml" - managed_config.write_text("version = 1\nmode = 'docker'\nbind = '127.0.0.1'\nport = 9090\n" + managed_config.write_text("version = 1\nmode = 'docker'\nbind = '127.0.0.1'\nport = 6135\n" "group0_management_seeds = ['http://127.0.0.1:10000']\nui_root = '/opt/crowdb/ui'\n" "monitor_status = '/opt/crowdb/run/status/monitor.json'\nlog_dir = '/opt/crowdb/data/log/web'\n" "log_max_file_mb = 30\nlog_max_files = 5\n") @@ -83,7 +83,7 @@ def check(case, arguments, expected, mounts=(), environment=(), entrypoint=None) f"type=bind,source={registry.resolve()},target=/opt/crowdb/etc/registry.toml,readonly")) check("chunk-kv-missing-config", ["server", "chunk-kv", "--config", "/opt/crowdb/etc/missing.toml", "--log-dir", "/opt/crowdb/data/log/chunk-kv", "--log"], "failed to load configuration") - check("managed-root", ["server", "kv"], "requires a separate data root") + check("managed-root", ["server", "paxos-kv"], "requires a separate data root") profile = root / "invalid-profile.toml" profile.write_text("version = [\n") check("invalid-profile", [], "failed to decode deployment profile", mounts=( diff --git a/e2e-test/s3_node_fixture.py b/e2e-test/s3_node_fixture.py index 037ba4482..4828a5a9e 100644 --- a/e2e-test/s3_node_fixture.py +++ b/e2e-test/s3_node_fixture.py @@ -66,7 +66,6 @@ def diskdb(self, logical): address = str(ipaddress.ip_address(self.addresses[0]) + index) config = f'''[server] rpc_workers = 2 -listen_addr = "0.0.0.0:7100" http_listen_addr = "0.0.0.0:7101" rpc_listen_addr = "{address}:7102" instance_id = "{10001 + logical}" diff --git a/e2e-test/scenarios.py b/e2e-test/scenarios.py index 19edccc1b..17719e13b 100644 --- a/e2e-test/scenarios.py +++ b/e2e-test/scenarios.py @@ -124,6 +124,8 @@ class Case: Case("kv.metadata", "kv::slot_map::initialize_is_atomic_idempotent_and_layer_independent", groups=(0,)), Case("kv.metadata", "kv::slot_map::concurrent_initializers_cannot_combine_different_assignments", groups=(0,)), Case("kv.metadata", "kv::slot_map::paginated_owner_records_preserve_empty_instances", groups=(0,)), + Case("kv.metadata", "kv::slot_map_recovery::interrupted_large_bootstrap_resumes_after_crash", groups=(0,), + restart=True, persisted_paths=("kv/waldata/store0/group0",)), Case("kv.metadata", "kv::slot_map::service_client_needs_no_storage_map_and_refreshes_same_owner_endpoints", groups=(0,)), Case("kv.metadata", "kv::slot_map_validation::legacy_records_are_never_overwritten", groups=(0,)), Case("kv.metadata", "kv::slot_map_validation::legacy_chunk_state_without_range_bindings_is_not_reinterpreted", groups=(0,)), diff --git a/e2e-test/tests/access_iceberg/file_http/capacity.rs b/e2e-test/tests/access_iceberg/file_http/capacity.rs index 7c269bf58..9ac4b6360 100644 --- a/e2e-test/tests/access_iceberg/file_http/capacity.rs +++ b/e2e-test/tests/access_iceberg/file_http/capacity.rs @@ -60,8 +60,8 @@ async fn full_simulated_disk_preserves_file_authority_then_recovers_after_compac client.shutdown_small_writes().await.unwrap(); drop(client); let disk = DiskdbClient::new( - crowdb_kv_client::ServiceRegistryClient::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(stack.seeds.clone()), + crowdb_paxos_kv_client::ServiceRegistryClient::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(stack.seeds.clone()), )), Arc::new(DiskdbRpcTransport::new()), ); diff --git a/e2e-test/tests/access_iceberg/file_http/streaming.rs b/e2e-test/tests/access_iceberg/file_http/streaming.rs index 51ae1e086..2e4496b30 100644 --- a/e2e-test/tests/access_iceberg/file_http/streaming.rs +++ b/e2e-test/tests/access_iceberg/file_http/streaming.rs @@ -45,9 +45,11 @@ async fn ordinary_put_size_matrix_streams_and_reads_ranges() { assert!(!locations.is_empty()); if size == 12 * 1024 * 1024 { let chunkdb = crowdb_chunkdb_client::ChunkdbClient::new( - crowdb_kv_client::ServiceRegistryClient::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(stack.seeds.clone()), - )), + crowdb_paxos_kv_client::ServiceRegistryClient::new( + crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( + stack.seeds.clone(), + )), + ), std::sync::Arc::new(crowdb_chunkdb_client::ChunkdbRpcTransport::new()), ); let chunk = chunkdb diff --git a/e2e-test/tests/access_iceberg/gc/notifications/large.rs b/e2e-test/tests/access_iceberg/gc/notifications/large.rs index c536e2639..7a43c5c33 100644 --- a/e2e-test/tests/access_iceberg/gc/notifications/large.rs +++ b/e2e-test/tests/access_iceberg/gc/notifications/large.rs @@ -15,7 +15,7 @@ use crowdb_chunkdb::{ storage::{range_retirement::RetiredByteRange, ChunkStore}, }; use crowdb_common::ec::EcScheme; -use crowdb_kv_client::{ChunkSlotMapClient, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkType, Location}; use md5::{Digest, Md5}; use std::sync::Arc; @@ -62,7 +62,7 @@ fn large_policy() -> LargeWritePolicy { } async fn chunk_store(stack: &TestNativeStorage) -> ChunkStore { - let kv = Arc::new(CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( stack.seeds.clone(), ))); let bindings = BindingCache::new(); diff --git a/e2e-test/tests/access_iceberg/gc/notifications/proof.rs b/e2e-test/tests/access_iceberg/gc/notifications/proof.rs index 17458332c..2378e41e1 100644 --- a/e2e-test/tests/access_iceberg/gc/notifications/proof.rs +++ b/e2e-test/tests/access_iceberg/gc/notifications/proof.rs @@ -15,7 +15,7 @@ use crowdb_chunkdb::{ routing::{BindingCache, BindingTable}, storage::{range_retirement::RetiredByteRange, ChunkStore}, }; -use crowdb_kv_client::{ChunkSlotMapClient, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkType}; use super::storage::TestPayloads; @@ -29,7 +29,7 @@ pub(super) struct TestRangeProof { impl TestRangeProof { pub(super) async fn new(stack: &TestNativeStorage, data: &TestPayloads) -> Self { - let kv = Arc::new(CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( stack.seeds.clone(), ))); let maps = ChunkSlotMapClient::new(kv.clone()); diff --git a/e2e-test/tests/access_iceberg/native.rs b/e2e-test/tests/access_iceberg/native.rs index 7e7e3b94b..03a97086e 100644 --- a/e2e-test/tests/access_iceberg/native.rs +++ b/e2e-test/tests/access_iceberg/native.rs @@ -31,7 +31,9 @@ impl TestNativeStorage { pub async fn store(&self) -> Arc<RoutedCatalogStore> { let config = ClientConfig::default(); let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::new( - crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new(self.seeds.clone())), + crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( + self.seeds.clone(), + )), ))); let transport = Arc::new(ChunkKvRpcTransport::new(config.max_owner_connections, 1, 2)); let client = Arc::new(ChunkKvClient::new(config, source, transport).unwrap()); diff --git a/e2e-test/tests/access_s3.rs b/e2e-test/tests/access_s3.rs index 612ef7b92..b585471fa 100644 --- a/e2e-test/tests/access_s3.rs +++ b/e2e-test/tests/access_s3.rs @@ -20,8 +20,8 @@ use crowdb_access_server::credentials::CredentialAuthority; #[ignore = "requires the owned release access storage containers"] async fn credentials() { let ips = crate::common::management::addresses(12); - let kv = Arc::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new( + let kv = Arc::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new( ips[..3] .iter() .map(|ip| crate::common::management::origin(ip)) diff --git a/e2e-test/tests/access_s3/node_failure.rs b/e2e-test/tests/access_s3/node_failure.rs index 9781b6d5e..d1b426471 100644 --- a/e2e-test/tests/access_s3/node_failure.rs +++ b/e2e-test/tests/access_s3/node_failure.rs @@ -6,7 +6,7 @@ //! preview belong to the Console container flows. use crowdb_console_shared::ops::s3::S3HttpClient; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::common::{DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{DiskGroupValue, DiskType, DiskValue}; use reqwest::Method; @@ -73,9 +73,9 @@ async fn protected_restart_preserves_objects_and_fixed_endpoints() { assert_eq!(range, b"object"); } -fn kv_client() -> crowdb_kv_client::CrowdbKvClient { +fn kv_client() -> crowdb_paxos_kv_client::PaxosKvClient { let addresses: Vec<String> = serde_json::from_str(&std::env::var("CROWDB_E2E_RPC").unwrap()).unwrap(); - crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( addresses[..3] .iter() .map(|address| format!("http://{address}:7000")) @@ -92,7 +92,7 @@ async fn seed() { } if std::env::var("CROWDB_E2E_PHASE").as_deref() == Ok("seed") { crate::chunkdb::voting::register_actual_voters().await; - let maps = crowdb_kv_client::ChunkSlotMapClient::new(std::sync::Arc::new( + let maps = crowdb_paxos_kv_client::ChunkSlotMapClient::new(std::sync::Arc::new( crate::common::hardware::kv_client(), )); maps.initialize_layout(&crowdb_protocol::chunk_slot::ChunkSlotBootstrap { diff --git a/e2e-test/tests/access_s3/node_failure/recovery.rs b/e2e-test/tests/access_s3/node_failure/recovery.rs index fc2b78e91..ab3a6a1e9 100644 --- a/e2e-test/tests/access_s3/node_failure/recovery.rs +++ b/e2e-test/tests/access_s3/node_failure/recovery.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv_client::{HardwareClient, RangeBindingClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ChunkType, ListChunksRequest, QueryChunkRequest, Strip, StripType}; use crowdb_protocol::common::{ChunkId, HwStatus}; use std::collections::{HashMap, HashSet}; @@ -215,7 +215,7 @@ pub(super) async fn repaired(chunks: &[ChunkId]) { } pub(super) async fn ownership( - kv: Arc<crowdb_kv_client::CrowdbKvClient>, + kv: Arc<crowdb_paxos_kv_client::PaxosKvClient>, hardware: HardwareClient, failed_node: u64, ) { diff --git a/e2e-test/tests/access_s3/policy.rs b/e2e-test/tests/access_s3/policy.rs index e46a01be6..5e37a6f09 100644 --- a/e2e-test/tests/access_s3/policy.rs +++ b/e2e-test/tests/access_s3/policy.rs @@ -10,8 +10,8 @@ use crowdb_chunk_client::{ ChunkIoClient, ChunkIoWriter, ChunkReadPolicy, LargeWritePolicy, SmallWritePolicy, }; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv_client::{ - ChunkSlotMapClient, ClientConfig, CrowdbKvClient, GetOutcome, ReadMode, ServiceRegistryClient, +use crowdb_paxos_kv_client::{ + ChunkSlotMapClient, ClientConfig, GetOutcome, PaxosKvClient, ReadMode, ServiceRegistryClient, }; use crowdb_protocol::chunk_slot::ChunkSlot; use crowdb_protocol::chunkdb::rpc::{ChunkType, Location, QueryChunkRequest, Strip}; @@ -78,7 +78,7 @@ async fn assert_chunk_layouts( s3_large: Location, iceberg_large: Location, ) { - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(seeds))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(seeds))); let maps = ChunkSlotMapClient::new(Arc::clone(&kv)); let storage = maps.read_storage().await.unwrap(); assert_eq!(storage.bindings().len(), 3); @@ -130,7 +130,7 @@ async fn assert_chunk_layouts( } async fn assert_system_chunk_purposes( - kv: &CrowdbKvClient, + kv: &PaxosKvClient, storage: &crowdb_protocol::chunk_slot::ChunkSlotMap<crowdb_protocol::chunk_slot::ChunkStorageGroup>, ) { let mut purposes = std::collections::HashSet::new(); diff --git a/e2e-test/tests/admission_profile_test.py b/e2e-test/tests/admission_profile_test.py index 8eab5714f..db7bb8bfd 100644 --- a/e2e-test/tests/admission_profile_test.py +++ b/e2e-test/tests/admission_profile_test.py @@ -31,7 +31,7 @@ def test_required_browser_flow_cannot_succeed_with_a_skip(self): "expected":0,"unexpected":0,"flaky":0,"skipped":1}})) with patch.object(bridge.subprocess,"run"): with self.assertRaisesRegex(AssertionError,"did not pass exactly once"): - bridge.browser("http://owned:9090","Docker candidates","prepare") + bridge.browser("http://owned:6135","Docker candidates","prepare") def test_profile_reserves_every_node_and_host_browser_before_runtime_access(self): for arguments in (("--concurrency","2"), ("--concurrency","1","--cpus","1.49"), diff --git a/e2e-test/tests/chunk_client/large_repair.rs b/e2e-test/tests/chunk_client/large_repair.rs index aa24e9d65..d4d6cee79 100644 --- a/e2e-test/tests/chunk_client/large_repair.rs +++ b/e2e-test/tests/chunk_client/large_repair.rs @@ -16,7 +16,7 @@ use crowdb_chunk_client::{ }; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_common::ec::{encode_parity_from_shards, EcScheme}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, Location, Strip}; use crowdb_protocol::diskdb::rpc::Segment; use crowdb_protocol::frame::{ @@ -262,7 +262,7 @@ fn make_test_data(size: usize) -> Vec<u8> { } async fn real_parts(stack: &E2eStack) -> (Arc<ChunkdbClient>, Arc<RoutedDiskWriter>) { - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( stack.cluster.mgmt_endpoints.clone(), ))); let service = ServiceRegistryClient::from_shared(kv.clone()); diff --git a/e2e-test/tests/chunk_client/reader.rs b/e2e-test/tests/chunk_client/reader.rs index 45b8312fa..58f476af5 100644 --- a/e2e-test/tests/chunk_client/reader.rs +++ b/e2e-test/tests/chunk_client/reader.rs @@ -15,7 +15,7 @@ use crowdb_chunk_client::{ }; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_common::ec::EcScheme; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ AdHocEcRecoveryDisposition, AdHocEcRecoveryRequest, Location, ReplaceChunkStripRangeRequest, Strip, }; @@ -130,7 +130,7 @@ fn test_data(size: usize) -> Vec<u8> { } async fn real_parts(stack: &E2eStack) -> (Arc<ChunkdbClient>, Arc<dyn DiskWriter>) { - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( stack.cluster.mgmt_endpoints.clone(), ))); let service = ServiceRegistryClient::from_shared(kv.clone()); diff --git a/e2e-test/tests/chunk_client/small_flow.rs b/e2e-test/tests/chunk_client/small_flow.rs index 0cd58d485..132a6b721 100644 --- a/e2e-test/tests/chunk_client/small_flow.rs +++ b/e2e-test/tests/chunk_client/small_flow.rs @@ -15,7 +15,7 @@ use crowdb_chunk_client::{ }; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_common::ec::{decode, encode_parity_from_shards, EcScheme}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ Chunk, ChunkState, ChunkType, ConversionFilter, Location, Strip, TriggerConversionBatchRequest, TriggerConversionRequest, @@ -196,7 +196,7 @@ async fn read_data_image(stack: &E2eStack, chunk: &Chunk, location: &Location) - } async fn real_write_parts(stack: &E2eStack) -> (Arc<ChunkdbClient>, Arc<RoutedDiskWriter>) { - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( stack.cluster.mgmt_endpoints.clone(), ))); let service = ServiceRegistryClient::from_shared(kv.clone()); diff --git a/e2e-test/tests/chunk_client/stack.rs b/e2e-test/tests/chunk_client/stack.rs index d67a5433c..954b0c665 100644 --- a/e2e-test/tests/chunk_client/stack.rs +++ b/e2e-test/tests/chunk_client/stack.rs @@ -4,7 +4,7 @@ use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, SmallWritePolicy}; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, SegmentTarget}; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{Chunk, Location, QueryChunkRequest}; use crowdb_protocol::diskdb::rpc::Segment; use std::sync::Arc; diff --git a/e2e-test/tests/chunk_kv/recovery.rs b/e2e-test/tests/chunk_kv/recovery.rs index 27c9e3da6..cd563321b 100644 --- a/e2e-test/tests/chunk_kv/recovery.rs +++ b/e2e-test/tests/chunk_kv/recovery.rs @@ -14,8 +14,8 @@ fn client() -> ChunkKvClient { .iter() .map(|ip| crate::common::management::origin(ip)) .collect(); - let kv = Arc::new(crowdb_kv_client::CrowdbKvClient::new( - crowdb_kv_client::ClientConfig::new(seeds), + let kv = Arc::new(crowdb_paxos_kv_client::PaxosKvClient::new( + crowdb_paxos_kv_client::ClientConfig::new(seeds), )); ChunkKvClient::new( ClientConfig::default(), diff --git a/e2e-test/tests/chunkdb/allocation.rs b/e2e-test/tests/chunkdb/allocation.rs index 6e2dab7e9..81fcc0a09 100644 --- a/e2e-test/tests/chunkdb/allocation.rs +++ b/e2e-test/tests/chunkdb/allocation.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crowdb_chunkdb::allocator::DiskdbClientPool; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use std::sync::atomic::Ordering; use crate::common::rpc_response_proxy::TestResponseProxy; diff --git a/e2e-test/tests/chunkdb/conversion.rs b/e2e-test/tests/chunkdb/conversion.rs index cd175bc27..602145089 100644 --- a/e2e-test/tests/chunkdb/conversion.rs +++ b/e2e-test/tests/chunkdb/conversion.rs @@ -7,7 +7,7 @@ use bytes::Bytes; use crowdb_chunkdb::conversion::io::ConversionDiskIo; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_common::ec::{encode_parity_from_shards, EcScheme}; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ AllocateChunkRequest, Chunk, ChunkType, QueryChunkRequest, SealChunkRequest, Strip, StripType, }; diff --git a/e2e-test/tests/chunkdb/handoff.rs b/e2e-test/tests/chunkdb/handoff.rs index e54ec46ae..5bf2d16f4 100644 --- a/e2e-test/tests/chunkdb/handoff.rs +++ b/e2e-test/tests/chunkdb/handoff.rs @@ -3,7 +3,7 @@ use bytes::Bytes; use crowdb_diskdb_client::{DiskdbClient, DiskdbRpcTransport}; -use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode, ServiceRegistryClient}; use crowdb_protocol::chunk_task::relocation_operation_id; use crowdb_protocol::chunkdb::rpc::{ AllocateChunkRequest, AllocateReplacementSegmentRequest, ChunkType, QueryChunkRequest, Strip, StripType, @@ -103,7 +103,7 @@ async fn relocation_journal_copies_publishes_then_frees_exact_source() { assert!(source_freed(&kv, source).await); } -async fn source_freed(kv: &CrowdbKvClient, source: Segment) -> bool { +async fn source_freed(kv: &PaxosKvClient, source: Segment) -> bool { let key = FreeBlockKey { disk_id: source.disk_id.unwrap(), zone_index: source.zone_index, @@ -118,7 +118,7 @@ async fn source_freed(kv: &CrowdbKvClient, source: Segment) -> bool { } async fn retired( - kv: &CrowdbKvClient, + kv: &PaxosKvClient, source: Segment, target: Segment, operation: crowdb_protocol::common::ChunkId, diff --git a/e2e-test/tests/chunkdb/hardware.rs b/e2e-test/tests/chunkdb/hardware.rs index 43e0ed86f..19d747383 100644 --- a/e2e-test/tests/chunkdb/hardware.rs +++ b/e2e-test/tests/chunkdb/hardware.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::common::{DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{DiskGroupValue, DiskType, DiskValue}; const ZONE_SIZE_UNITS: u64 = 128; diff --git a/e2e-test/tests/chunkdb/io.rs b/e2e-test/tests/chunkdb/io.rs index 3f7159440..301afa655 100644 --- a/e2e-test/tests/chunkdb/io.rs +++ b/e2e-test/tests/chunkdb/io.rs @@ -3,7 +3,7 @@ use bytes::Bytes; use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, Durability, SegmentTarget}; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use std::time::{Duration, Instant}; mod readiness; diff --git a/e2e-test/tests/chunkdb/io/readiness.rs b/e2e-test/tests/chunkdb/io/readiness.rs index 2b9c80c8d..bef65ec13 100644 --- a/e2e-test/tests/chunkdb/io/readiness.rs +++ b/e2e-test/tests/chunkdb/io/readiness.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crowdb_diskio_client::{DiskioClient, DiskioClientConfig}; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::common::InstanceValue; fn current_owners(instances: &[(u64, InstanceValue)], started_at_ms: u64) -> bool { diff --git a/e2e-test/tests/chunkdb/lifecycle.rs b/e2e-test/tests/chunkdb/lifecycle.rs index 48a00b677..e11caf888 100644 --- a/e2e-test/tests/chunkdb/lifecycle.rs +++ b/e2e-test/tests/chunkdb/lifecycle.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError, ChunkdbRpcTransport}; -use crowdb_kv_client::{GetOutcome, ReadMode, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ AllocateChunkRequest, Chunk, ChunkState, ChunkType, DeleteChunkRangeRequest, DeleteChunkRequest, QueryChunkRequest, SealChunkRequest, Strip, StripType, diff --git a/e2e-test/tests/chunkdb/placement.rs b/e2e-test/tests/chunkdb/placement.rs index 1f46d582d..aa694ee68 100644 --- a/e2e-test/tests/chunkdb/placement.rs +++ b/e2e-test/tests/chunkdb/placement.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::chunkdb::rpc::{AllocateChunkRequest, ChunkType, DeleteChunkRequest, Strip, StripType}; use std::sync::Arc; diff --git a/e2e-test/tests/chunkdb/recovery.rs b/e2e-test/tests/chunkdb/recovery.rs index 26ea24c38..d5c21792c 100644 --- a/e2e-test/tests/chunkdb/recovery.rs +++ b/e2e-test/tests/chunkdb/recovery.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv_client::{GetOutcome, ReadMode, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ Chunk, ChunkState, DeleteChunkRequest, QueryChunkRequest, SealChunkRequest, }; diff --git a/e2e-test/tests/chunkdb/repair/topology.rs b/e2e-test/tests/chunkdb/repair/topology.rs index a3a01dfd9..e75958d73 100644 --- a/e2e-test/tests/chunkdb/repair/topology.rs +++ b/e2e-test/tests/chunkdb/repair/topology.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::chunkdb::rpc::{AllocateChunkRequest, ChunkType, QueryChunkRequest, Strip, StripType}; use crowdb_protocol::common::HwStatus; use std::time::{Duration, Instant}; diff --git a/e2e-test/tests/chunkdb/slots.rs b/e2e-test/tests/chunkdb/slots.rs index 6663a6309..5e80e9d73 100644 --- a/e2e-test/tests/chunkdb/slots.rs +++ b/e2e-test/tests/chunkdb/slots.rs @@ -6,7 +6,7 @@ use crowdb_chunkdb::routing::{BindingCache, BindingTable}; use crowdb_chunkdb::storage::ChunkStore; use crowdb_chunkdb::task::TaskStore; use crowdb_chunkdb_client::ChunkdbRpcTransport; -use crowdb_kv_client::{ChunkSlotMapClient, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_slot::ChunkSlot; use crowdb_protocol::chunk_stream::StreamName; use crowdb_protocol::chunkdb::rpc::{AllocateChunkRequest, ChunkType, StripType}; @@ -19,7 +19,7 @@ use std::sync::Arc; #[ignore = "requires three packaged Chunkdb owners and three KV voters"] async fn three_service_owners_preserve_all_purposes_across_paxos_failover() { let ips = management::addresses(7)[..3].to_vec(); - let kv = Arc::new(CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( ips.iter().map(|ip| format!("http://{ip}:7000")).collect(), ))); let maps = ChunkSlotMapClient::new(Arc::clone(&kv)); @@ -85,7 +85,7 @@ async fn allocate_owner( instance: u64, service: &crowdb_protocol::chunk_slot::ChunkSlotMap<u64>, storage: &crowdb_protocol::chunk_slot::ChunkSlotMap<crowdb_protocol::chunk_slot::ChunkStorageGroup>, - kv: &CrowdbKvClient, + kv: &PaxosKvClient, chunks: &ChunkStore, tasks: &TaskStore, ) -> Vec<(ChunkId, u64)> { @@ -139,13 +139,7 @@ async fn allocate_owner( allocated } -async fn assert_records( - kv: &CrowdbKvClient, - chunks: &ChunkStore, - tasks: &TaskStore, - id: ChunkId, - group: u64, -) { +async fn assert_records(kv: &PaxosKvClient, chunks: &ChunkStore, tasks: &TaskStore, id: ChunkId, group: u64) { let chunk = chunks.get_chunk(&id).await.unwrap(); assert_eq!(u64::try_from(chunk.chunk_type).unwrap(), id.high >> 56); let task = tasks diff --git a/e2e-test/tests/chunkdb/tasks.rs b/e2e-test/tests/chunkdb/tasks.rs index 39669a402..937e87e36 100644 --- a/e2e-test/tests/chunkdb/tasks.rs +++ b/e2e-test/tests/chunkdb/tasks.rs @@ -3,7 +3,7 @@ use crowdb_chunkdb::routing::{BindingCache, BindingTable}; use crowdb_chunkdb::task::TaskStore; -use crowdb_kv_client::ChunkSlotMapClient; +use crowdb_paxos_kv_client::ChunkSlotMapClient; use crowdb_protocol::chunk_task::{ChunkTaskState, ChunkTaskValue}; use crowdb_protocol::common::ChunkId; use std::sync::Arc; diff --git a/e2e-test/tests/chunkdb/voting.rs b/e2e-test/tests/chunkdb/voting.rs index 7ee73f8ab..8ab319a9d 100644 --- a/e2e-test/tests/chunkdb/voting.rs +++ b/e2e-test/tests/chunkdb/voting.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::KVClusterMetaClient; +use crowdb_paxos_kv_client::PaxosKvClusterMetaClient; use crowdb_protocol::common::ReplicaValue; pub(crate) async fn register_actual_voters() { @@ -11,7 +11,7 @@ pub(crate) async fn register_actual_voters() { 3, "production metadata needs three owned KV nodes" ); - let metadata = KVClusterMetaClient::new(crate::common::hardware::kv_client()); + let metadata = PaxosKvClusterMetaClient::new(crate::common::hardware::kv_client()); metadata.add_store(0, &[1, 2, 3]).await.unwrap(); for (index, address) in addresses.iter().enumerate() { let topology: serde_json::Value = reqwest::Client::new() diff --git a/e2e-test/tests/common.rs b/e2e-test/tests/common.rs index e66cfc8bf..f1e320cdf 100644 --- a/e2e-test/tests/common.rs +++ b/e2e-test/tests/common.rs @@ -10,8 +10,8 @@ pub(crate) mod rpc_response_proxy; mod test_client; use bytes::Bytes; -use crowdb_kv::rpc::{KvBatchWriteRequest, KvDeleteRequest, KvErrorCode, KvGetRequest, KvSetRequest}; -use crowdb_kv_client::KvRpcTransport; +use crowdb_paxos_kv::rpc::{KvBatchWriteRequest, KvDeleteRequest, KvErrorCode, KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv_client::PaxosKvRpcTransport; use serde_json::Value; use std::sync::Arc; use test_client::TestKvClient; @@ -103,9 +103,9 @@ pub async fn run_kv_op_with_retry( nodes: &[ServerNode], group_id: u64, op: &KvOp, -) -> crowdb_kv::rpc::KvResponse { +) -> crowdb_paxos_kv::rpc::KvResponse { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); - let transport = Arc::new(KvRpcTransport::new()); + let transport = Arc::new(PaxosKvRpcTransport::new()); let mut last_err = String::new(); while std::time::Instant::now() < deadline { let leader_idx = wait_for_leader(nodes, group_id, std::time::Duration::from_secs(10)).await; diff --git a/e2e-test/tests/common/hardware.rs b/e2e-test/tests/common/hardware.rs index b50d777d9..41206c5a0 100644 --- a/e2e-test/tests/common/hardware.rs +++ b/e2e-test/tests/common/hardware.rs @@ -3,7 +3,7 @@ //! Hardware metadata seeding for E2E tests. -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::common::{DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{DiskGroupValue, DiskType, DiskValue}; @@ -113,10 +113,10 @@ pub fn standard_disk_ids_4() -> Vec<DiskId> { ids } -pub(crate) fn kv_client() -> crowdb_kv_client::CrowdbKvClient { +pub(crate) fn kv_client() -> crowdb_paxos_kv_client::PaxosKvClient { super::logging::init(); let addresses: Vec<String> = serde_json::from_str(&std::env::var("CROWDB_E2E_RPC").unwrap()).unwrap(); - crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new(vec![format!( + crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new(vec![format!( "http://{}:7000", addresses[0] )])) diff --git a/e2e-test/tests/common/native_s3/signed_copy.rs b/e2e-test/tests/common/native_s3/signed_copy.rs index 45c0018f9..d69655b98 100644 --- a/e2e-test/tests/common/native_s3/signed_copy.rs +++ b/e2e-test/tests/common/native_s3/signed_copy.rs @@ -8,13 +8,13 @@ use std::sync::Arc; use crowdb_access_s3::auth::{CredentialCipher, MasterKey}; use crowdb_access_server::credentials::CredentialAuthority; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use hmac::{Hmac, Mac}; use reqwest::{Client, Response}; use sha2::{Digest, Sha256}; pub async fn copy(addresses: &[String], http: &Client, base: &str) -> Response { - let control = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let control = Arc::new(PaxosKvClient::new(ClientConfig::new( addresses[..3] .iter() .map(|ip| super::super::management::origin(ip)) diff --git a/e2e-test/tests/common/test_client.rs b/e2e-test/tests/common/test_client.rs index d2cd2c1cc..3d7704af3 100644 --- a/e2e-test/tests/common/test_client.rs +++ b/e2e-test/tests/common/test_client.rs @@ -3,7 +3,7 @@ //! Test-only client facade that preserves the former tonic //! `KvServiceClient` call shape (`.put(req).await.expect(..). -//! into_inner()`) but sends over the crowdb-rpc `KvRpcTransport`. +//! into_inner()`) but sends over the crowdb-rpc `PaxosKvRpcTransport`. //! Exists so the process-level e2e tests could be migrated off the //! deleted tonic client with minimal churn. @@ -12,11 +12,11 @@ use std::sync::Arc; -use crowdb_kv::rpc::{ +use crowdb_paxos_kv::rpc::{ KvBatchWriteRequest, KvDeleteRequest, KvGetRequest, KvResponse, KvScanRequest, KvScanResponse, KvSetRequest, ReadMode, }; -use crowdb_kv_client::KvRpcTransport; +use crowdb_paxos_kv_client::PaxosKvRpcTransport; /// Status-like error returned by the facade. Implements `Debug` so /// `Result<_, TestRpcStatus>::expect(..)` compiles, and exposes @@ -60,7 +60,7 @@ fn read_mode_from_i32(v: i32) -> ReadMode { /// with `TestKvClient::connect(endpoint).await` (the `.await` mirrors the /// old `KvServiceClient::connect(..).await` shape). pub struct TestKvClient { - transport: Arc<KvRpcTransport>, + transport: Arc<PaxosKvRpcTransport>, endpoint: String, } @@ -68,13 +68,13 @@ impl TestKvClient { #[allow(clippy::unused_async_trait_impl)] pub async fn connect(endpoint: String) -> Self { Self { - transport: Arc::new(KvRpcTransport::new()), + transport: Arc::new(PaxosKvRpcTransport::new()), endpoint, } } #[must_use] - pub fn with_transport(transport: Arc<KvRpcTransport>, endpoint: String) -> Self { + pub fn with_transport(transport: Arc<PaxosKvRpcTransport>, endpoint: String) -> Self { Self { transport, endpoint } } @@ -170,8 +170,12 @@ impl TestKvClient { false, 0, match req.direction { - crowdb_kv::rpc::KvScanDirection::Forward => crowdb_kv_client::ScanDirection::Forward, - crowdb_kv::rpc::KvScanDirection::Reverse => crowdb_kv_client::ScanDirection::Reverse, + crowdb_paxos_kv::rpc::KvScanDirection::Forward => { + crowdb_paxos_kv_client::ScanDirection::Forward + } + crowdb_paxos_kv::rpc::KvScanDirection::Reverse => { + crowdb_paxos_kv_client::ScanDirection::Reverse + } }, ) .await diff --git a/e2e-test/tests/console/bootstrap_publication.rs b/e2e-test/tests/console/bootstrap_publication.rs index 3f6241764..bd202c925 100644 --- a/e2e-test/tests/console/bootstrap_publication.rs +++ b/e2e-test/tests/console/bootstrap_publication.rs @@ -136,7 +136,7 @@ async fn bootstrap_resumes_missing_records_and_preserves_committed_revisions() { 0, 0, key.as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await @@ -148,14 +148,14 @@ async fn bootstrap_resumes_missing_records_and_preserves_committed_revisions() { 0, 0, key.as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await .unwrap(); let ( - crowdb_kv_client::GetOutcome::Found { revision: before, .. }, - crowdb_kv_client::GetOutcome::Found { revision: after, .. }, + crowdb_paxos_kv_client::GetOutcome::Found { revision: before, .. }, + crowdb_paxos_kv_client::GetOutcome::Found { revision: after, .. }, ) = (before, after) else { panic!("rack must remain present") diff --git a/e2e-test/tests/console/deployment.rs b/e2e-test/tests/console/deployment.rs index b7d93976d..6a9f71699 100644 --- a/e2e-test/tests/console/deployment.rs +++ b/e2e-test/tests/console/deployment.rs @@ -22,7 +22,7 @@ fn config() -> ConsoleConfig { #[ignore = "requires the owned release KV containers"] async fn concurrent_admissions_allocate_unique_stable_ids_through_group_zero() { let cluster = KvCluster::start().await; - let kv = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let kv = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( cluster.mgmt_endpoints.clone(), )); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); @@ -115,7 +115,7 @@ async fn service_retries_retain_identity_and_other_nodes_cannot_claim_the_same_s use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; let cluster = KvCluster::start().await; - let kv = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + let kv = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( cluster.mgmt_endpoints.clone(), )); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); diff --git a/e2e-test/tests/console/discovery.rs b/e2e-test/tests/console/discovery.rs index 29960110e..5acdd09e1 100644 --- a/e2e-test/tests/console/discovery.rs +++ b/e2e-test/tests/console/discovery.rs @@ -9,7 +9,7 @@ use std::time::Duration; use super::fixture::TestKvCluster as KvCluster; -use crowdb_kv_client::{Error, ServiceDiscoveryClient}; +use crowdb_paxos_kv_client::{Error, ServiceDiscoveryClient}; use crowdb_protocol::chunk_kv::{HostedPartition, Id128}; use crowdb_protocol::common::{ ChunkKvExtra, DiskGroupUsageSummary, DiskdbExtra, InstanceValue, ServiceExtra, @@ -70,7 +70,7 @@ async fn chunk_kv_raw_observation_preserves_load_and_partition_state() { }], partition_loads: Vec::new(), }; - let lease_client = crowdb_kv_client::StreamProcessLeaseClient::new(svc.shared_kv()); + let lease_client = crowdb_paxos_kv_client::StreamProcessLeaseClient::new(svc.shared_kv()); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() diff --git a/e2e-test/tests/console/fixture.rs b/e2e-test/tests/console/fixture.rs index 2ebed5b8a..66f7f032a 100644 --- a/e2e-test/tests/console/fixture.rs +++ b/e2e-test/tests/console/fixture.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; pub(super) struct TestKvCluster { pub mgmt_endpoints: Vec<String>, @@ -39,7 +39,7 @@ impl TestKvCluster { } pub fn make_service_registry_client(&self) -> ServiceRegistryClient { - let client = CrowdbKvClient::new(ClientConfig::new(self.mgmt_endpoints.clone())); + let client = PaxosKvClient::new(ClientConfig::new(self.mgmt_endpoints.clone())); client.seed_leader(0, 0, self.group0_leader_endpoint.clone()); ServiceRegistryClient::new(client) } diff --git a/e2e-test/tests/console/group_publication.rs b/e2e-test/tests/console/group_publication.rs index cb6992dbc..440434606 100644 --- a/e2e-test/tests/console/group_publication.rs +++ b/e2e-test/tests/console/group_publication.rs @@ -11,7 +11,7 @@ use axum::routing::{get, post}; use axum::Router; use crowdb_console_shared::ops::{kv_logical, OpContext}; use crowdb_console_shared::ConsoleConfig; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::common::{KvServerIdentity, ReplicaValue}; use crowdb_protocol::key::{KvGroupKey, KvReplicaKey, TextKey}; diff --git a/e2e-test/tests/console/native_balance.rs b/e2e-test/tests/console/native_balance.rs index 6008510ac..949fce404 100644 --- a/e2e-test/tests/console/native_balance.rs +++ b/e2e-test/tests/console/native_balance.rs @@ -8,7 +8,7 @@ use crowdb_chunk_kv_client::{ BatchItem, ChunkKvClient, ChunkKvRangeCatalogMap, ChunkKvRangeCatalogSource, ChunkKvRpcTransport, ClientConfig, Group0ChunkKvRangeCatalogSource, }; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::{ chunk_kv::{ChunkKvRangeCatalogPartitionState, OperationResult, PointOperation}, common::ChunkKvExtra, @@ -122,7 +122,7 @@ async fn verify_values(client: &ChunkKvClient) { async fn wait_balanced( source: &Group0ChunkKvRangeCatalogSource, registry: &ServiceRegistryClient, - kv: &crowdb_kv_client::CrowdbKvClient, + kv: &crowdb_paxos_kv_client::PaxosKvClient, started: Instant, ) -> ChunkKvRangeCatalogMap { let mut interval = tokio::time::interval(Duration::from_millis(500)); @@ -192,19 +192,19 @@ async fn wait_balanced( } } -async fn placement_ready(kv: &crowdb_kv_client::CrowdbKvClient, generation: u64) -> bool { +async fn placement_ready(kv: &crowdb_paxos_kv_client::PaxosKvClient, generation: u64) -> bool { use crowdb_protocol::chunk_kv::balance::{BalanceObservation, OBSERVATION_KEY}; let value = kv .get( 0, 0, OBSERVATION_KEY.as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await .unwrap(); - let crowdb_kv_client::GetOutcome::Found { value, .. } = value else { + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = value else { return false; }; let observation: BalanceObservation = serde_json::from_slice(&value).unwrap(); diff --git a/e2e-test/tests/console/native_binding.rs b/e2e-test/tests/console/native_binding.rs index 26de2aaf5..5c9a052f7 100644 --- a/e2e-test/tests/console/native_binding.rs +++ b/e2e-test/tests/console/native_binding.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::native_context::TestNativeContext; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use reqwest::Method; use serde_json::json; use std::time::Duration; diff --git a/e2e-test/tests/console/native_capacity.rs b/e2e-test/tests/console/native_capacity.rs index e9dcba390..e11240bbf 100644 --- a/e2e-test/tests/console/native_capacity.rs +++ b/e2e-test/tests/console/native_capacity.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::native_context::TestNativeContext; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::{ common::{DiskId, HwStatus}, disk_storage::DiskStorageRecord, diff --git a/e2e-test/tests/console/native_chunks.rs b/e2e-test/tests/console/native_chunks.rs index 0c0226a53..1b7ce7f22 100644 --- a/e2e-test/tests/console/native_chunks.rs +++ b/e2e-test/tests/console/native_chunks.rs @@ -3,7 +3,7 @@ use super::native_context::TestNativeContext; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::chunkdb::rpc::{AllocateChunkRequest, ChunkType, Strip, StripType}; use serde_json::{json, Value}; use std::sync::Arc; diff --git a/e2e-test/tests/console/native_context.rs b/e2e-test/tests/console/native_context.rs index dc48e816f..3d1eba332 100644 --- a/e2e-test/tests/console/native_context.rs +++ b/e2e-test/tests/console/native_context.rs @@ -3,7 +3,8 @@ //! Host clients of the normal three-node Console deployment surface. -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; +use crowdb_protocol::port::ports::{PAXOS_KV_SERVER_MGMT_BASE, PAXOS_KV_SERVER_RPC_BASE, WEB_BASE}; use reqwest::{Client, Method}; use serde_json::Value; use std::{sync::Arc, time::Duration}; @@ -11,7 +12,7 @@ use std::{sync::Arc, time::Duration}; pub(super) struct TestNativeContext { pub(super) http: Client, pub(super) base: String, - pub(super) kv: Arc<CrowdbKvClient>, + pub(super) kv: Arc<PaxosKvClient>, pub(super) addresses: Vec<String>, } @@ -23,7 +24,7 @@ impl TestNativeContext { .timeout(Duration::from_secs(20)) .build() .unwrap(); - let base = format!("http://{}:9090", addresses[0]); + let base = format!("http://{}:{WEB_BASE}", addresses[0]); let group = crate::common::management::request( &http, &base, @@ -43,13 +44,13 @@ impl TestNativeContext { .unwrap(); let index = usize::try_from(leader.checked_sub(1).unwrap()).unwrap(); assert!(index < addresses.len()); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( addresses .iter() - .map(|address| format!("http://{address}:10000")) + .map(|address| format!("http://{address}:{PAXOS_KV_SERVER_MGMT_BASE}")) .collect(), ))); - kv.seed_leader(0, 0, format!("{}:10100", addresses[index])); + kv.seed_leader(0, 0, format!("{}:{PAXOS_KV_SERVER_RPC_BASE}", addresses[index])); Self { http, base, diff --git a/e2e-test/tests/console/native_disk_lifecycle.rs b/e2e-test/tests/console/native_disk_lifecycle.rs index 75068b0dc..d97142097 100644 --- a/e2e-test/tests/console/native_disk_lifecycle.rs +++ b/e2e-test/tests/console/native_disk_lifecycle.rs @@ -9,10 +9,11 @@ use crowdb_diskdb_client::{DiskdbClient, DiskdbRpcTransport}; use crowdb_diskio_client::{ DiskId as IoDiskId, DiskioClient, DiskioClientConfig, Durability, OperationOptions, SegmentTarget, }; -use crowdb_kv_client::{GetOutcome, HardwareClient, ReadMode, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{GetOutcome, HardwareClient, ReadMode, ServiceRegistryClient}; use crowdb_protocol::{ diskdb::rpc::{AllocateBlocksRequest, CommitBlocksRequest, Segment}, key::{BinaryKey, BusyBlockKey, DiskKey}, + port::ports::PAXOS_KV_SERVER_MGMT_BASE, DiskIdExt, }; use reqwest::Method; @@ -213,7 +214,7 @@ async fn write_committed( management_seeds: context .addresses .iter() - .map(|address| format!("http://{address}:10000")) + .map(|address| format!("http://{address}:{PAXOS_KV_SERVER_MGMT_BASE}")) .collect(), normal_connections_per_endpoint: 1, priority_connections_per_endpoint: 1, diff --git a/e2e-test/tests/console/native_disk_usage.rs b/e2e-test/tests/console/native_disk_usage.rs index 350835092..b9a497be7 100644 --- a/e2e-test/tests/console/native_disk_usage.rs +++ b/e2e-test/tests/console/native_disk_usage.rs @@ -3,7 +3,7 @@ //! A restarted owner reports unknown occupancy until retained validation loads it. use super::native_context::TestNativeContext; -use crowdb_kv_client::{GetOutcome, HardwareClient, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, HardwareClient, ReadMode}; use crowdb_protocol::{ key::{DiskKey, OwnerMapKey, TextKey}, DiskIdExt, diff --git a/e2e-test/tests/console/native_load.rs b/e2e-test/tests/console/native_load.rs index e7f1185f5..cfb68986d 100644 --- a/e2e-test/tests/console/native_load.rs +++ b/e2e-test/tests/console/native_load.rs @@ -6,7 +6,7 @@ use crowdb_chunk_kv_client::{ BatchItem, ChunkKvClient, ChunkKvRangeCatalogMap, ChunkKvRangeCatalogSource, ChunkKvRpcTransport, ClientConfig, Group0ChunkKvRangeCatalogSource, }; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::chunk_kv::{ChunkKvInstanceObservation, OperationResult, PointOperation}; use reqwest::Method; use std::{ diff --git a/e2e-test/tests/console/native_s3.rs b/e2e-test/tests/console/native_s3.rs index f24d62c7f..38f20644d 100644 --- a/e2e-test/tests/console/native_s3.rs +++ b/e2e-test/tests/console/native_s3.rs @@ -150,7 +150,7 @@ pub(super) async fn assert_native_locations(app: &TestNativeContext) { } let id = first["locations"][0]["chunk_id"].as_str().unwrap(); let chunkdb = crowdb_chunkdb_client::ChunkdbClient::new( - crowdb_kv_client::ServiceRegistryClient::from_shared(app.kv.clone()), + crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(app.kv.clone()), std::sync::Arc::new(crowdb_chunkdb_client::ChunkdbRpcTransport::new()), ); let chunk = chunkdb diff --git a/e2e-test/tests/console/native_storage.rs b/e2e-test/tests/console/native_storage.rs index d532907b2..ec5d8a761 100644 --- a/e2e-test/tests/console/native_storage.rs +++ b/e2e-test/tests/console/native_storage.rs @@ -4,7 +4,8 @@ //! Normal three-node storage prerequisites shared by public Console clients. use super::native_context::TestNativeContext; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; +use crowdb_protocol::port::ports::DISKDB_HTTP_BASE; use reqwest::Method; use serde_json::{json, Value}; use std::{collections::BTreeSet, time::Duration}; @@ -121,12 +122,12 @@ async fn verify_manual_metadata(context: &TestNativeContext) { 0, 0, crowdb_console_shared::deployment::preparation::DEFAULT_PREPARATION_KEY, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None ) .await .unwrap(), - crowdb_kv_client::GetOutcome::NotFound, + crowdb_paxos_kv_client::GetOutcome::NotFound, )); } @@ -149,7 +150,7 @@ pub(super) async fn disks(context: &TestNativeContext, units: [u64; 3]) { async fn wait_diskdb(context: &TestNativeContext) { for node in 1..=3 { - ready(context, node, 7101, "/ready").await; + ready(context, node, DISKDB_HTTP_BASE, "/ready").await; } let registry = ServiceRegistryClient::from_shared(context.kv.clone()); tokio::time::timeout(Duration::from_secs(10), async { @@ -161,7 +162,7 @@ async fn wait_diskdb(context: &TestNativeContext) { context .addresses .iter() - .any(|address| value.rpc_endpoint == format!("{address}:7102")) + .any(|address| value.rpc_endpoint == format!("{address}:7100")) }) .flat_map(|(_, value)| { value diff --git a/e2e-test/tests/console/native_storage/media.rs b/e2e-test/tests/console/native_storage/media.rs index 2a5f19475..2ef149d8d 100644 --- a/e2e-test/tests/console/native_storage/media.rs +++ b/e2e-test/tests/console/native_storage/media.rs @@ -4,7 +4,7 @@ //! Trusted legacy bootstrap for freshly truncated, owned native fixture files. use super::TestNativeContext; -use crowdb_kv_client::HardwareClient; +use crowdb_paxos_kv_client::HardwareClient; use crowdb_protocol::{ common::{DiskId, HwStatus}, disk_storage::DiskStorageRecord, diff --git a/e2e-test/tests/console/server_data.rs b/e2e-test/tests/console/server_data.rs index 5d1672004..3be549334 100644 --- a/e2e-test/tests/console/server_data.rs +++ b/e2e-test/tests/console/server_data.rs @@ -4,7 +4,7 @@ use crowdb_console_shared::clients::http::ServerClient; use crowdb_console_shared::lifecycle; use crowdb_console_shared::mgmt::{AddGroupRequest, AddStoreRequest}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use std::time::Duration; async fn store_rpc_endpoint(mgmt_url: &str, store_id: u64) -> String { @@ -48,11 +48,11 @@ async fn put_get_delete_cycle() { .expect("wait_for_leader"); let endpoint = store_rpc_endpoint(&mgmt_url, store_id).await; - // `CrowdbKvClient` normally discovers leaders via `/topology`; here we + // `PaxosKvClient` normally discovers leaders via `/topology`; here we // already know the endpoint (just resolved it above), so seed it // directly rather than standing up topology discovery for a // single-node test. - let kv = CrowdbKvClient::new(ClientConfig::new(Vec::new())); + let kv = PaxosKvClient::new(ClientConfig::new(Vec::new())); kv.seed_leader(store_id, group_id, endpoint.clone()); // Put. @@ -182,11 +182,11 @@ async fn put_get_delete_cycle() { .expect_err("scan missing group"); assert!(format!("{err}").contains("not found"), "got: {err}"); - // A second, independently-constructed `CrowdbKvClient` seeded at the + // A second, independently-constructed `PaxosKvClient` seeded at the // same endpoint must also work end-to-end (no shared process-wide - // state to warm up -- `crowdb-kv-client`'s connection pool is + // state to warm up -- `crowdb-paxos-kv-client`'s connection pool is // per-instance and lazily connected). - let kv2 = CrowdbKvClient::new(ClientConfig::new(Vec::new())); + let kv2 = PaxosKvClient::new(ClientConfig::new(Vec::new())); kv2.seed_leader(store_id, group_id, endpoint.clone()); let _ = kv2 .put(store_id, group_id, b"cached", b"hit", None) diff --git a/e2e-test/tests/console/web_access.rs b/e2e-test/tests/console/web_access.rs index 91cd7026f..8c4312ad2 100644 --- a/e2e-test/tests/console/web_access.rs +++ b/e2e-test/tests/console/web_access.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::{Client, Method}; use serde_json::{json, Value}; use std::time::Duration; @@ -21,7 +22,7 @@ async fn call(http: &Client, base: &str, method: Method, path: &str, body: Value #[tokio::test] #[ignore = "fixture setup requires the owned storage authority"] async fn prepare() { - let hardware = crowdb_kv_client::HardwareClient::new(crate::common::hardware::kv_client()); + let hardware = crowdb_paxos_kv_client::HardwareClient::new(crate::common::hardware::kv_client()); hardware .add_rack( 1, @@ -51,7 +52,7 @@ async fn prepare() { #[ignore = "requires owned release Web and native storage containers"] async fn native_access_deployment_provisions_private_credentials_and_reuses_them() { let addresses = crate::common::management::addresses(13); - let base = format!("http://{}:9090", addresses[12]); + let base = format!("http://{}:{WEB_BASE}", addresses[12]); let http = Client::builder() .timeout(Duration::from_secs(30)) .build() diff --git a/e2e-test/tests/console/web_authority.rs b/e2e-test/tests/console/web_authority.rs index 36103026e..ada335cfa 100644 --- a/e2e-test/tests/console/web_authority.rs +++ b/e2e-test/tests/console/web_authority.rs @@ -2,9 +2,10 @@ // Licensed under the Apache License, Version 2.0. use super::fixture::{web_kv, TestKvCluster as KvCluster}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, CrowdbSysmdClient}; +use crowdb_paxos_kv_client::{ClientConfig, CrowdbSysmdClient, PaxosKvClient}; use crowdb_protocol::common::{HwStatus, KvServerIdentity, NodeValue, RackValue, ReplicaValue}; use crowdb_protocol::key::InstanceKey; +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::StatusCode; async fn snapshot(app: &str) -> (StatusCode, serde_json::Value) { @@ -29,7 +30,7 @@ async fn register(sysmd: &CrowdbSysmdClient, node: u64, instance: u64, endpoint: } async fn initialized_authority(cluster: &KvCluster) -> CrowdbSysmdClient { - let kv = CrowdbKvClient::new(ClientConfig::new(cluster.mgmt_endpoints.clone())); + let kv = PaxosKvClient::new(ClientConfig::new(cluster.mgmt_endpoints.clone())); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); let sysmd = CrowdbSysmdClient::new(kv); sysmd @@ -64,11 +65,11 @@ async fn initialized_authority(cluster: &KvCluster) -> CrowdbSysmdClient { } fn application(_cluster: &KvCluster) -> String { - format!("http://{}:9090", crate::common::management::addresses(5)[3]) + format!("http://{}:{WEB_BASE}", crate::common::management::addresses(5)[3]) } fn application_peer() -> String { - format!("http://{}:9090", crate::common::management::addresses(5)[4]) + format!("http://{}:{WEB_BASE}", crate::common::management::addresses(5)[4]) } async fn hardware_request( diff --git a/e2e-test/tests/console/web_catalog.rs b/e2e-test/tests/console/web_catalog.rs index d61983cb0..957661bb9 100644 --- a/e2e-test/tests/console/web_catalog.rs +++ b/e2e-test/tests/console/web_catalog.rs @@ -1,13 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, Id128, KeyRange, OwnerDescriptor, PartitionArtifact, }; use crowdb_protocol::chunk_stream::StreamName; use crowdb_protocol::key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, TextKey}; +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::StatusCode; use serde_json::Value; use std::sync::Arc; @@ -29,12 +30,12 @@ async fn runtime_get(origin: &str, query: &str) -> StatusCode { #[ignore = "requires owned release Web and KV containers"] async fn published_owner_change_invalidates_saved_catalog_and_runtime_cursors() { let cluster = super::fixture::web_kv().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); let addresses = crate::common::management::addresses(5); - let app = format!("http://{}:9090", addresses[3]); + let app = format!("http://{}:{WEB_BASE}", addresses[3]); let (mut head, mut page) = catalog_fixture(); for generation in [9, 10] { page.generation = generation; diff --git a/e2e-test/tests/console/web_cluster_recovery.rs b/e2e-test/tests/console/web_cluster_recovery.rs index 49780bee9..c544aa7db 100644 --- a/e2e-test/tests/console/web_cluster_recovery.rs +++ b/e2e-test/tests/console/web_cluster_recovery.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::{Client, Method}; use serde_json::{json, Value}; use std::{path::PathBuf, time::Duration}; @@ -11,12 +12,12 @@ async fn prepare() { super::fixture::TestKvCluster::start().await; super::super::chunkdb::voting::register_actual_voters().await; let kv = std::sync::Arc::new(crate::common::hardware::kv_client()); - let sysmd = crowdb_kv_client::CrowdbSysmdClient::from_shared(kv.clone()); + let sysmd = crowdb_paxos_kv_client::CrowdbSysmdClient::from_shared(kv.clone()); sysmd.add_store(0, &[1, 2, 3]).await.unwrap(); for group in [0, 1] { sysmd.add_group(0, group).await.unwrap(); } - let hardware = crowdb_kv_client::HardwareClient::from_shared(kv); + let hardware = crowdb_paxos_kv_client::HardwareClient::from_shared(kv); hardware .add_rack( 1, @@ -47,7 +48,7 @@ async fn prepare() { async fn call(http: &Client, method: Method, path: &str, body: Value) -> Value { let addresses = crate::common::management::addresses(4); let response = http - .request(method, format!("http://{}:9090{path}", addresses[3])) + .request(method, format!("http://{}:{WEB_BASE}{path}", addresses[3])) .json(&body) .send() .await @@ -69,7 +70,7 @@ async fn standalone_recovers_three_member_group0_and_diskdb() { &http, Method::POST, "/api/nodes/1/diskdb/deploy", - json!({"rpc_port":7100, "listen_port":7100, "http_port":7101}), + json!({"rpc_port":7102, "http_port":7101}), ) .await; assert!(deployed["pid"].as_u64().unwrap() > 1); diff --git a/e2e-test/tests/console/web_diskio_discovery.rs b/e2e-test/tests/console/web_diskio_discovery.rs index 1bb7d3e6d..6f36750b2 100644 --- a/e2e-test/tests/console/web_diskio_discovery.rs +++ b/e2e-test/tests/console/web_diskio_discovery.rs @@ -3,7 +3,8 @@ use super::web_legacy::{bootstrap, get}; use crate::common::management::request; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::Method; use serde_json::json; use std::sync::Arc; @@ -34,7 +35,7 @@ async fn owned_groups(registry: &ServiceRegistryClient) -> Option<Vec<u64>> { async fn diskio_discovers_disk_groups_created_after_web_deployment() { let address = crate::common::management::addresses(1).remove(0); let http = crate::common::management::client(); - let base = format!("http://{address}:9090"); + let base = format!("http://{address}:{WEB_BASE}"); assert_eq!(get(&http, &base, "/api/group0-readiness").await["ready"], false); let (http, base) = bootstrap().await; assert_eq!(get(&http, &base, "/api/group0-readiness").await["ready"], true); @@ -59,7 +60,7 @@ async fn diskio_discovers_disk_groups_created_after_web_deployment() { 201, ) .await; - let kv = CrowdbKvClient::new(ClientConfig::new(vec![format!("http://{address}:7000")])); + let kv = PaxosKvClient::new(ClientConfig::new(vec![format!("http://{address}:7000")])); kv.seed_leader(0, 0, format!("{address}:7001")); let registry = ServiceRegistryClient::from_shared(Arc::new(kv)); let initial = tokio::time::timeout(Duration::from_secs(10), async { diff --git a/e2e-test/tests/console/web_isolation.rs b/e2e-test/tests/console/web_isolation.rs index 79ee53656..c216db5c1 100644 --- a/e2e-test/tests/console/web_isolation.rs +++ b/e2e-test/tests/console/web_isolation.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::{Client, StatusCode}; use serde_json::{json, Value}; use std::time::Duration; @@ -10,7 +11,7 @@ use std::time::Duration; async fn survives_peer_fixture_retirement() { let http = Client::builder().timeout(Duration::from_secs(5)).build().unwrap(); let addresses = crate::common::management::addresses(1); - let base = format!("http://{}:9090", addresses[0]); + let base = format!("http://{}:{WEB_BASE}", addresses[0]); let token = std::env::var("CROWDB_E2E_TOKEN").unwrap(); let phase = std::env::var("CROWDB_E2E_PHASE").unwrap(); if phase != "verify" { diff --git a/e2e-test/tests/console/web_kv_lifecycle.rs b/e2e-test/tests/console/web_kv_lifecycle.rs index b109bd1b5..7aa7459d7 100644 --- a/e2e-test/tests/console/web_kv_lifecycle.rs +++ b/e2e-test/tests/console/web_kv_lifecycle.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::{Client, Method}; use serde_json::{json, Value}; use std::path::PathBuf; @@ -27,7 +28,7 @@ async fn deploy_restart_and_stop_retains_fixed_ports_and_workspace() { .build() .unwrap(); let addresses = crate::common::management::addresses(1); - let base = format!("http://{}:9090", addresses[0]); + let base = format!("http://{}:{WEB_BASE}", addresses[0]); request(&http, &base, Method::POST, "/api/racks", json!({"id":1}), 201).await; request( &http, @@ -60,7 +61,7 @@ async fn deploy_restart_and_stop_retains_fixed_ports_and_workspace() { } assert!(workspace.join("bin").read_dir().unwrap().next().is_some()); assert!(workspace - .join(format!("log/crowdb-kv-server-{first}.out.log")) + .join(format!("log/crowdb-paxos-kv-server-{first}.out.log")) .is_file()); let server = request(&http, &base, Method::GET, "/api/nodes/1/server", Value::Null, 200).await; assert_eq!(server["url"], "http://127.0.0.1:7000"); diff --git a/e2e-test/tests/console/web_legacy.rs b/e2e-test/tests/console/web_legacy.rs index 2d19900e1..c4495900f 100644 --- a/e2e-test/tests/console/web_legacy.rs +++ b/e2e-test/tests/console/web_legacy.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::Client; use serde_json::{json, Value}; use std::time::{Duration, Instant}; @@ -15,7 +16,7 @@ pub(super) async fn get(http: &Client, base: &str, path: &str) -> Value { pub(super) async fn bootstrap() -> (Client, String) { let addresses = crate::common::management::addresses(1); - let base = format!("http://{}:9090", addresses[0]); + let base = format!("http://{}:{WEB_BASE}", addresses[0]); let http = Client::builder().timeout(Duration::from_secs(5)).build().unwrap(); assert_eq!(get(&http, &base, "/api/mode").await["mode"], "legacy"); for (path, body, expected) in [ diff --git a/e2e-test/tests/console/web_lifecycle.rs b/e2e-test/tests/console/web_lifecycle.rs index 0d9b25a7d..19ff4d631 100644 --- a/e2e-test/tests/console/web_lifecycle.rs +++ b/e2e-test/tests/console/web_lifecycle.rs @@ -1,7 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::{Client, StatusCode}; use serde_json::{json, Value}; use std::path::PathBuf; @@ -28,7 +29,7 @@ async fn deploys_real_chunkdb_with_fixed_cluster_seeds_and_retained_launch() { .build() .unwrap(); let addresses = crate::common::management::addresses(4); - let base = format!("http://{}:9090", addresses[3]); + let base = format!("http://{}:{WEB_BASE}", addresses[3]); prepare_storage(&http, &base).await; let deployed = post( &http, @@ -65,17 +66,18 @@ async fn deploys_real_chunkdb_with_fixed_cluster_seeds_and_retained_launch() { async fn prepare_storage(http: &Client, base: &str) { let addresses = crate::common::management::addresses(4); let cluster = super::fixture::TestKvCluster::from_addresses(addresses[..3].to_vec()).await; - let kv = - crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new(cluster.mgmt_endpoints)); + let kv = crowdb_paxos_kv_client::PaxosKvClient::new(crowdb_paxos_kv_client::ClientConfig::new( + cluster.mgmt_endpoints, + )); kv.seed_leader(0, 0, cluster.group0_leader_endpoint); let kv = std::sync::Arc::new(kv); - let sysmd = crowdb_kv_client::CrowdbSysmdClient::from_shared(kv.clone()); + let sysmd = crowdb_paxos_kv_client::CrowdbSysmdClient::from_shared(kv.clone()); sysmd.add_store(0, &[1, 2, 3]).await.unwrap(); sysmd.add_group(0, 0).await.unwrap(); sysmd.add_group(0, 1).await.unwrap(); register_voters(http, &sysmd, &addresses[..3]).await; - let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(kv.clone()); - let hardware = crowdb_kv_client::HardwareClient::from_shared(kv); + let registry = crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(kv.clone()); + let hardware = crowdb_paxos_kv_client::HardwareClient::from_shared(kv); hardware .add_rack( 1, @@ -174,7 +176,11 @@ async fn prepare_diskdb( .unwrap(); } -async fn register_voters(http: &Client, sysmd: &crowdb_kv_client::CrowdbSysmdClient, addresses: &[String]) { +async fn register_voters( + http: &Client, + sysmd: &crowdb_paxos_kv_client::CrowdbSysmdClient, + addresses: &[String], +) { for (index, address) in addresses.iter().enumerate() { let topology: Value = http .get(format!("http://{address}:7000/topology")) diff --git a/e2e-test/tests/console/web_lifetime.rs b/e2e-test/tests/console/web_lifetime.rs index 2e669f765..084d70b4a 100644 --- a/e2e-test/tests/console/web_lifetime.rs +++ b/e2e-test/tests/console/web_lifetime.rs @@ -1,13 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::Client; use serde_json::{json, Value}; async fn post(http: &Client, path: &str, body: Value) -> Value { let addresses = crate::common::management::addresses(1); let response = http - .post(format!("http://{}:9090{path}", addresses[0])) + .post(format!("http://{}:{WEB_BASE}{path}", addresses[0])) .json(&body) .send() .await diff --git a/e2e-test/tests/console/web_recovery.rs b/e2e-test/tests/console/web_recovery.rs index 9e346f9fc..915f493db 100644 --- a/e2e-test/tests/console/web_recovery.rs +++ b/e2e-test/tests/console/web_recovery.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::Client; use serde_json::{json, Value}; use std::path::PathBuf; @@ -8,7 +9,7 @@ use std::time::Duration; async fn request(http: &Client, path: &str, body: Option<Value>) -> Value { let addresses = crate::common::management::addresses(1); - let url = format!("http://{}:9090{path}", addresses[0]); + let url = format!("http://{}:{WEB_BASE}{path}", addresses[0]); let response = if let Some(body) = body { http.post(url).json(&body).send().await.unwrap() } else { diff --git a/e2e-test/tests/console/web_registry.rs b/e2e-test/tests/console/web_registry.rs index bb05be94a..b467b49aa 100644 --- a/e2e-test/tests/console/web_registry.rs +++ b/e2e-test/tests/console/web_registry.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::{Client, StatusCode}; use serde_json::Value; use std::path::PathBuf; @@ -14,7 +15,7 @@ async fn bare_metal_web_loads_launch_policy_without_restoring_topology() { let data = PathBuf::from(mounts["host_data"][0].as_str().unwrap()); let identity_path = data.join("launches.runtime/701-kv.json"); let addresses = crate::common::management::addresses(1); - let base = format!("http://{}:9090", addresses[0]); + let base = format!("http://{}:{WEB_BASE}", addresses[0]); let http = Client::builder().timeout(Duration::from_secs(3)).build().unwrap(); let response = http.get(format!("{base}/api/launches")).send().await.unwrap(); assert_eq!(response.status(), StatusCode::OK); diff --git a/e2e-test/tests/console/web_sealed_bootstrap.rs b/e2e-test/tests/console/web_sealed_bootstrap.rs index c493c9e1b..af93c6215 100644 --- a/e2e-test/tests/console/web_sealed_bootstrap.rs +++ b/e2e-test/tests/console/web_sealed_bootstrap.rs @@ -4,6 +4,7 @@ use crowdb_console_shared::bootstrap_intent::BootstrapIntent; use crowdb_console_shared::config::{NodeEntry, RackEntry, ServerEntry}; use crowdb_console_shared::ConsoleConfig; +use crowdb_protocol::port::ports::WEB_BASE; use serde_json::{json, Value}; use std::path::PathBuf; @@ -37,7 +38,7 @@ async fn bare_metal_bootstrap_consumes_sealed_topology_without_local_topology() .unwrap(); let source = crate::common::control::node("upload-bootstrap-source", 1).await; let response = crate::common::management::client() - .post(format!("http://{}:9090/api/cluster/init", addresses[1])) + .post(format!("http://{}:{WEB_BASE}/api/cluster/init", addresses[1])) .bearer_auth("bare-metal-bootstrap-test-token-12345") .json(&json!({"nodes": [1], "bootstrap_file": source["path"]})) .send() diff --git a/e2e-test/tests/console/web_slots.rs b/e2e-test/tests/console/web_slots.rs index c1e745b7a..68c88d213 100644 --- a/e2e-test/tests/console/web_slots.rs +++ b/e2e-test/tests/console/web_slots.rs @@ -1,8 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ChunkSlotMapClient, ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, ClientConfig, PaxosKvClient}; use crowdb_protocol::chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}; +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::StatusCode; use serde_json::Value; use std::sync::Arc; @@ -21,12 +22,12 @@ async fn get(origin: &str, query: &str) -> (StatusCode, Value) { #[ignore = "requires owned release Web and KV containers"] async fn owner_windows_keep_disjoint_slots_generations_and_exact_ids() { let cluster = super::fixture::web_kv().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( cluster.mgmt_endpoints.clone(), ))); kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); let addresses = crate::common::management::addresses(5); - let state = format!("http://{}:9090", addresses[3]); + let state = format!("http://{}:{WEB_BASE}", addresses[3]); let bootstrap = ChunkSlotBootstrap { service_instances: vec![1, u64::MAX], storage_groups: vec![ diff --git a/e2e-test/tests/console/web_standalone.rs b/e2e-test/tests/console/web_standalone.rs index 2c409a803..a254b991a 100644 --- a/e2e-test/tests/console/web_standalone.rs +++ b/e2e-test/tests/console/web_standalone.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::Client; use serde_json::{json, Value}; use std::path::PathBuf; @@ -14,7 +15,7 @@ fn data() -> PathBuf { async fn request(http: &Client, path: &str, body: Option<Value>) -> Value { let addresses = crate::common::management::addresses(1); - let url = format!("http://{}:9090{path}", addresses[0]); + let url = format!("http://{}:{WEB_BASE}{path}", addresses[0]); let response = if let Some(body) = body { http.post(url).json(&body).send().await.unwrap() } else { @@ -35,7 +36,7 @@ async fn empty_ui_persists_rack_and_node_across_restart() { assert_eq!(request(&http, "/api/stores", None).await, json!([])); let addresses = crate::common::management::addresses(1); assert!(http - .get(format!("http://{}:9090", addresses[0])) + .get(format!("http://{}:{WEB_BASE}", addresses[0])) .send() .await .unwrap() @@ -43,7 +44,7 @@ async fn empty_ui_persists_rack_and_node_across_restart() { .is_success()); for path in ["/", "/some/spa/route"] { let response = http - .get(format!("http://{}:9090{path}", addresses[0])) + .get(format!("http://{}:{WEB_BASE}{path}", addresses[0])) .send() .await .unwrap(); @@ -51,7 +52,7 @@ async fn empty_ui_persists_rack_and_node_across_restart() { assert!(response.text().await.unwrap().contains("id=\"root\"")); } assert_eq!( - http.get(format!("http://{}:9090/healthz", addresses[0])) + http.get(format!("http://{}:{WEB_BASE}/healthz", addresses[0])) .send() .await .unwrap() @@ -61,7 +62,7 @@ async fn empty_ui_persists_rack_and_node_across_restart() { "ok" ); let racks = http - .get(format!("http://{}:9090/api/racks", addresses[0])) + .get(format!("http://{}:{WEB_BASE}/api/racks", addresses[0])) .send() .await .unwrap(); @@ -71,7 +72,7 @@ async fn empty_ui_persists_rack_and_node_across_restart() { .unwrap() .starts_with("application/json")); let traversal = http - .get(format!("http://{}:9090/../Cargo.toml", addresses[0])) + .get(format!("http://{}:{WEB_BASE}/../Cargo.toml", addresses[0])) .send() .await .unwrap() diff --git a/e2e-test/tests/console/web_voters.rs b/e2e-test/tests/console/web_voters.rs index 1f0a36e97..47e8624ae 100644 --- a/e2e-test/tests/console/web_voters.rs +++ b/e2e-test/tests/console/web_voters.rs @@ -2,13 +2,14 @@ // Licensed under the Apache License, Version 2.0. use crate::common::management::{addresses, client, request}; +use crowdb_protocol::port::ports::WEB_BASE; use reqwest::{Client, Method}; use serde_json::{json, Value}; pub(super) async fn setup(count: usize) -> (Client, String, Vec<String>) { let ips = addresses(count + 1); let http = client(); - let base = format!("http://{}:9090", ips[count]); + let base = format!("http://{}:{WEB_BASE}", ips[count]); request(&http, &base, Method::POST, "/api/racks", json!({"id":1}), 201).await; for (index, ip) in ips[..count].iter().enumerate() { request( diff --git a/e2e-test/tests/console/web_wal_restart.rs b/e2e-test/tests/console/web_wal_restart.rs index 6b2c632fb..8bc07fe63 100644 --- a/e2e-test/tests/console/web_wal_restart.rs +++ b/e2e-test/tests/console/web_wal_restart.rs @@ -6,6 +6,7 @@ mod recovery; mod state; mod wal; +use crowdb_protocol::port::ports::WEB_BASE; use http::*; use recovery::{restart_recovery, GroupSpec, REPLAY_PUTS}; use serde_json::json; @@ -27,7 +28,7 @@ async fn setup_cluster() -> Cluster { if ips.len() > 1 { return setup_remote_cluster(client, &ips).await; } - let base = format!("http://{}:9090", ips[0]); + let base = format!("http://{}:{WEB_BASE}", ips[0]); create_rack(&client, &base, 1).await; create_node(&client, &base, 1, 1, "127.0.0.1").await; let (status, body) = json_post( @@ -59,7 +60,7 @@ async fn setup_remote_cluster(client: reqwest::Client, ips: &[String]) -> Cluste .iter() .map(|data| PathBuf::from(data.as_str().unwrap()).join("kv/waldata")) .collect(); - let base = format!("http://{}:9090", ips[count]); + let base = format!("http://{}:{WEB_BASE}", ips[count]); create_rack(&client, &base, 1).await; if count > 3 { create_rack(&client, &base, 2).await; diff --git a/e2e-test/tests/console/web_wal_restart/wal.rs b/e2e-test/tests/console/web_wal_restart/wal.rs index f31ed6e7a..6aaa9e26b 100644 --- a/e2e-test/tests/console/web_wal_restart/wal.rs +++ b/e2e-test/tests/console/web_wal_restart/wal.rs @@ -1,9 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::wal::io_backend::IoBackend; -use crowdb_kv::wal::record::{RecordType, WALRecord}; -use crowdb_kv::wal::segment::SegmentReader; +use crowdb_paxos_kv::wal::io_backend::IoBackend; +use crowdb_paxos_kv::wal::record::{RecordType, WALRecord}; +use crowdb_paxos_kv::wal::segment::SegmentReader; use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::Arc; diff --git a/e2e-test/tests/console_lifetime_test.py b/e2e-test/tests/console_lifetime_test.py index 18de092e4..47a0a1912 100644 --- a/e2e-test/tests/console_lifetime_test.py +++ b/e2e-test/tests/console_lifetime_test.py @@ -34,5 +34,5 @@ def test_record_rejects_non_process_identity(self): cluster.command = Mock() for pid in (True, None, 0, 1, -1, "42"): with self.assertRaises(ValueError): - cluster.identity(pid, "crowdb-kv-server") + cluster.identity(pid, "crowdb-paxos-kv-server") cluster.command.assert_not_called() diff --git a/e2e-test/tests/deployment.rs b/e2e-test/tests/deployment.rs index 6183c9053..13254d766 100644 --- a/e2e-test/tests/deployment.rs +++ b/e2e-test/tests/deployment.rs @@ -1,7 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; +use crowdb_protocol::port::ports::PAXOS_KV_SERVER_MGMT_BASE; mod anonymous; mod balance; mod bootstrap; @@ -29,9 +30,10 @@ async fn selected_kv_services_recover_acknowledged_data() { async fn selected_storage_services_recover_acknowledged_data() { witness(false).await; let address = crate::common::management::addresses(1)[0].clone(); - let registry = crowdb_kv_client::ServiceRegistryClient::new(CrowdbKvClient::new(ClientConfig::new( - vec![format!("http://{address}:10000")], - ))); + let registry = + crowdb_paxos_kv_client::ServiceRegistryClient::new(PaxosKvClient::new(ClientConfig::new(vec![ + format!("http://{address}:{PAXOS_KV_SERVER_MGMT_BASE}"), + ]))); let instances = registry.read_all_diskdb_instances().await.unwrap(); assert_eq!(instances.len(), 1); assert_eq!(instances[0].0, 1); @@ -53,7 +55,10 @@ async fn witness(storage: bool) { async fn witness_with_write(storage: bool, write: bool) { let addresses: Vec<String> = serde_json::from_str(&std::env::var("CROWDB_E2E_RPC").unwrap()).unwrap(); - let client = CrowdbKvClient::new(ClientConfig::new(vec![format!("http://{}:10000", addresses[0])])); + let client = PaxosKvClient::new(ClientConfig::new(vec![format!( + "http://{}:{PAXOS_KV_SERVER_MGMT_BASE}", + addresses[0] + )])); let token = std::env::var("CROWDB_E2E_TOKEN").unwrap(); if write { let outcome = client @@ -63,8 +68,8 @@ async fn witness_with_write(storage: bool, write: bool) { assert!(outcome.revision > 0); } let hardware = - crowdb_kv_client::HardwareClient::new(CrowdbKvClient::new(ClientConfig::new(vec![format!( - "http://{}:10000", + crowdb_paxos_kv_client::HardwareClient::new(PaxosKvClient::new(ClientConfig::new(vec![format!( + "http://{}:{PAXOS_KV_SERVER_MGMT_BASE}", addresses[0] )]))); let disks = hardware.list_all_disks().await.unwrap(); diff --git a/e2e-test/tests/deployment/balance.rs b/e2e-test/tests/deployment/balance.rs index ae1ef3168..3bb2e88a0 100644 --- a/e2e-test/tests/deployment/balance.rs +++ b/e2e-test/tests/deployment/balance.rs @@ -1,9 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_kv::{ChunkKvRangeBalancePolicy, DomainMonitorDescriptor, SplitTransition}; use crowdb_protocol::key::{ChunkKvSplitKey, DomainMonitorKey, TextKey}; +use crowdb_protocol::port::ports::PAXOS_KV_SERVER_MGMT_BASE; use std::time::{Duration, Instant}; #[tokio::test] @@ -11,7 +12,9 @@ use std::time::{Duration, Instant}; async fn single_node_preserves_default_chunk_kv_balance_policy() { super::witness_with_write(true, true).await; let address = crate::common::management::addresses(1)[0].clone(); - let client = CrowdbKvClient::new(ClientConfig::new(vec![format!("http://{address}:10000")])); + let client = PaxosKvClient::new(ClientConfig::new(vec![format!( + "http://{address}:{PAXOS_KV_SERVER_MGMT_BASE}" + )])); let key = DomainMonitorKey { domain: "chunk-kv".into(), } diff --git a/e2e-test/tests/deployment/clients.rs b/e2e-test/tests/deployment/clients.rs index d07ddd097..1edff040f 100644 --- a/e2e-test/tests/deployment/clients.rs +++ b/e2e-test/tests/deployment/clients.rs @@ -3,6 +3,9 @@ use crowdb_chunkdb_client::ChunkdbRpcTransport; use crowdb_protocol::chunkdb::rpc::{ChunkType, ListChunksRequest, Strip, StripType}; +use crowdb_protocol::port::ports::{ + ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE, WEB_BASE, +}; use std::path::PathBuf; #[tokio::test] @@ -36,8 +39,14 @@ async fn official_clients_and_protocol_layout_survive_restart() { let (key, value) = line.split_once('=').expect("client environment assignment"); command.env(key, value); } - command.env("CROWDB_PREVIEW_S3_ENDPOINT", format!("http://{address}:9091")); - command.env("CROWDB_PREVIEW_ICEBERG_URI", format!("http://{address}:9092")); + command.env( + "CROWDB_PREVIEW_S3_ENDPOINT", + format!("http://{address}:{ACCESS_SERVER_HTTP_BASE}"), + ); + command.env( + "CROWDB_PREVIEW_ICEBERG_URI", + format!("http://{address}:{ACCESS_SERVER_ICEBERG_HTTP_BASE}"), + ); command.env( "CROWDB_E2E_SOURCE_ROOT", env!("CARGO_MANIFEST_DIR").trim_end_matches("/e2e-test"), @@ -56,7 +65,7 @@ async fn official_clients_and_protocol_layout_survive_restart() { async fn verify_public_services(address: &str) { let client = crate::common::management::client(); - let web = format!("http://{address}:9090"); + let web = format!("http://{address}:{WEB_BASE}"); let report: serde_json::Value = client .get(format!("{web}/api/cluster/status")) .send() @@ -132,7 +141,9 @@ async fn verify_public_services(address: &str) { } assert_eq!( client - .get(format!("http://{address}:9093/v1/dataset/open")) + .get(format!( + "http://{address}:{ACCESS_SERVER_DATASET_HTTP_BASE}/v1/dataset/open" + )) .send() .await .unwrap() diff --git a/e2e-test/tests/deployment/hardware_bootstrap.rs b/e2e-test/tests/deployment/hardware_bootstrap.rs index 6cefd3dbc..8ac9d8e60 100644 --- a/e2e-test/tests/deployment/hardware_bootstrap.rs +++ b/e2e-test/tests/deployment/hardware_bootstrap.rs @@ -4,10 +4,10 @@ use std::fs; use std::path::{Path, PathBuf}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; use crowdb_monitor::{ hardware_step_names, BootstrapSession, DeploymentProfile, HardwareBootstrap, MonitorLog, }; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::common::{HwStatus, RackValue}; use uuid::Uuid; @@ -84,7 +84,7 @@ async fn writes_and_validates_hardware_with_real_group_zero() { .reconcile(&mut restart, &profile, &mut events) .await .unwrap(); - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new(seeds.clone()))); let disks = hardware.list_all_disks().await.unwrap(); assert_eq!(disks.len(), 4); assert!(disks.iter().all(|disk| disk.value.zone_count == 1)); @@ -106,7 +106,7 @@ async fn conflicting_group_zero_record_rejects_without_creating_hardware() { .iter() .map(|ip| crate::common::management::origin(ip)) .collect::<Vec<_>>(); - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); + let hardware = HardwareClient::new(PaxosKvClient::new(ClientConfig::new(seeds.clone()))); hardware .add_rack( 1, diff --git a/e2e-test/tests/deployment/kv_bootstrap.rs b/e2e-test/tests/deployment/kv_bootstrap.rs index 295064038..7f222f8d2 100644 --- a/e2e-test/tests/deployment/kv_bootstrap.rs +++ b/e2e-test/tests/deployment/kv_bootstrap.rs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crate::common::management::{client, request}; +use crowdb_protocol::port::ports::PAXOS_KV_SERVER_MGMT_BASE; use reqwest::Method; use serde_json::{json, Value}; @@ -21,7 +22,10 @@ async fn monitor_creates_real_kv_groups_then_validates_after_restart() { let initial: Value = serde_json::from_slice(&std::fs::read(witness).unwrap()).unwrap(); assert_eq!(accepted, initial, "restart must preserve accepted initialization"); } - let origin = format!("http://{}:10000", crate::common::management::addresses(1)[0]); + let origin = format!( + "http://{}:{PAXOS_KV_SERVER_MGMT_BASE}", + crate::common::management::addresses(1)[0] + ); let http = client(); let result = request( &http, @@ -76,10 +80,10 @@ async fn monitor_creates_real_kv_groups_then_validates_after_restart() { } pub(super) async fn create_witness_group() { - use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; + use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; let origin = format!("http://{}:10000", crate::common::management::addresses(1)[0]); - let kv = CrowdbKvClient::new(ClientConfig::new(vec![origin.clone()])); + let kv = PaxosKvClient::new(ClientConfig::new(vec![origin.clone()])); assert!(matches!( kv.get( 0, diff --git a/e2e-test/tests/deployment/listeners.rs b/e2e-test/tests/deployment/listeners.rs index 5a5ffc7bc..0c6d72fd4 100644 --- a/e2e-test/tests/deployment/listeners.rs +++ b/e2e-test/tests/deployment/listeners.rs @@ -1,6 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +use crowdb_protocol::port::ports::{ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE}; + fn verify(protocol: &str, port: &str) { let root = std::path::PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").unwrap()); let result: serde_json::Value = @@ -19,7 +21,10 @@ fn verify(protocol: &str, port: &str) { fn fixed_s3_listener_recovers_after_explicit_termination() { verify( "s3", - &format!("{}:9091", crate::common::management::addresses(1)[0]), + &format!( + "{}:{ACCESS_SERVER_HTTP_BASE}", + crate::common::management::addresses(1)[0] + ), ); } @@ -28,6 +33,9 @@ fn fixed_s3_listener_recovers_after_explicit_termination() { fn fixed_iceberg_listener_recovers_after_explicit_termination() { verify( "iceberg", - &format!("{}:9092", crate::common::management::addresses(1)[0]), + &format!( + "{}:{ACCESS_SERVER_ICEBERG_HTTP_BASE}", + crate::common::management::addresses(1)[0] + ), ); } diff --git a/e2e-test/tests/deployment/storage.rs b/e2e-test/tests/deployment/storage.rs index 32d20fc67..2168fbec7 100644 --- a/e2e-test/tests/deployment/storage.rs +++ b/e2e-test/tests/deployment/storage.rs @@ -1,8 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, CrowdbSysmdClient}; use crowdb_monitor::{verify_chunk_services, DeploymentProfile}; +use crowdb_paxos_kv_client::{ClientConfig, CrowdbSysmdClient, PaxosKvClient}; +use crowdb_protocol::port::ports::{ACCESS_SERVER_ICEBERG_HTTP_BASE, PAXOS_KV_SERVER_MGMT_BASE}; use std::path::PathBuf; fn root() -> PathBuf { @@ -10,7 +11,10 @@ fn root() -> PathBuf { } fn seed() -> String { - format!("http://{}:10000", crate::common::management::addresses(1)[0]) + format!( + "http://{}:{PAXOS_KV_SERVER_MGMT_BASE}", + crate::common::management::addresses(1)[0] + ) } #[tokio::test] @@ -50,7 +54,7 @@ async fn chunk_services_validate_live_registration_and_logical_topology() { } async fn verify_logical_topology() { - let sysmd = CrowdbSysmdClient::new(CrowdbKvClient::new(ClientConfig::new(vec![seed()]))); + let sysmd = CrowdbSysmdClient::new(PaxosKvClient::new(ClientConfig::new(vec![seed()]))); assert_eq!(sysmd.list_stores().await.unwrap().len(), 1); let initial = sysmd.list_groups_in_store(0).await.unwrap(); assert_eq!(initial.len(), 2); @@ -70,7 +74,10 @@ async fn real_iceberg_catalog_and_listener_survive_restart() { .lines() .find_map(|line| line.strip_prefix("ICEBERG_TOKEN=")) .unwrap(); - let origin = format!("http://{}:9092", crate::common::management::addresses(1)[0]); + let origin = format!( + "http://{}:{ACCESS_SERVER_ICEBERG_HTTP_BASE}", + crate::common::management::addresses(1)[0] + ); let client = reqwest::Client::new(); let config = client .get(format!("{origin}/v1/config")) diff --git a/e2e-test/tests/deployment/supplied_media.rs b/e2e-test/tests/deployment/supplied_media.rs index d6351f69c..910de3b72 100644 --- a/e2e-test/tests/deployment/supplied_media.rs +++ b/e2e-test/tests/deployment/supplied_media.rs @@ -5,12 +5,12 @@ use bytes::Bytes; use crowdb_diskio_client::{ DiskId, DiskioClient, DiskioClientConfig, Durability, OperationOptions, SegmentTarget, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{common::HwStatus, key::DiskKey, mgmt::node::storage::DiskBackend, DiskIdExt}; use serde_json::Value; use std::time::Duration; -async fn assert_selected_host(kv: &CrowdbKvClient, specification: &Value, host: &str) { +async fn assert_selected_host(kv: &PaxosKvClient, specification: &Value, host: &str) { let (registry, _) = crowdb_console_shared::deployment::registry::read(kv) .await .unwrap() @@ -36,7 +36,7 @@ async fn retained_data_probe() { assert!(matches!(mode, "write" | "verify")); let host = specification["physical_host_id"].as_str().unwrap(); let seeds: Vec<String> = serde_json::from_value(specification["seeds"].clone()).unwrap(); - let kv = CrowdbKvClient::new(ClientConfig::new(seeds.clone())); + let kv = PaxosKvClient::new(ClientConfig::new(seeds.clone())); assert_selected_host(&kv, &specification, host).await; let hardware = HardwareClient::new(kv); let disks = specification["disks"].as_array().unwrap(); diff --git a/e2e-test/tests/diskdb.rs b/e2e-test/tests/diskdb.rs index a75b1940f..3f9e63709 100644 --- a/e2e-test/tests/diskdb.rs +++ b/e2e-test/tests/diskdb.rs @@ -9,12 +9,12 @@ mod recovery; mod space_reuse; use crowdb_diskdb_client::{DiskdbClient, DiskdbRpcTransport, RetryConfig}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; use std::sync::Arc; -fn kv_client() -> CrowdbKvClient { +fn kv_client() -> PaxosKvClient { let addresses: Vec<String> = serde_json::from_str(&std::env::var("CROWDB_E2E_RPC").unwrap()).unwrap(); - CrowdbKvClient::new(ClientConfig::new(vec![format!("http://{}:7000", addresses[0])])) + PaxosKvClient::new(ClientConfig::new(vec![format!("http://{}:7000", addresses[0])])) } fn registry() -> ServiceRegistryClient { diff --git a/e2e-test/tests/diskdb/hardware.rs b/e2e-test/tests/diskdb/hardware.rs index e89648d8f..1318428eb 100644 --- a/e2e-test/tests/diskdb/hardware.rs +++ b/e2e-test/tests/diskdb/hardware.rs @@ -8,6 +8,6 @@ pub(super) use crate::common::hardware::{ #[tokio::test] #[ignore = "requires the owned container fixture"] async fn seed() { - let hardware = crowdb_kv_client::HardwareClient::new(super::kv_client()); + let hardware = crowdb_paxos_kv_client::HardwareClient::new(super::kv_client()); crate::common::hardware::seed_hardware(&hardware, &standard_disk_ids_3()).await; } diff --git a/e2e-test/tests/diskdb/recovery.rs b/e2e-test/tests/diskdb/recovery.rs index 59bdcb9b9..49f8ebabe 100644 --- a/e2e-test/tests/diskdb/recovery.rs +++ b/e2e-test/tests/diskdb/recovery.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::hardware::{DG_ID, UNIT_SIZE_BYTES}; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::diskdb::rpc::{ AllocateBlocksRequest, CommitBlocksRequest, CompactZoneRequest, FreeBlocksRequest, Segment, }; diff --git a/e2e-test/tests/diskio.rs b/e2e-test/tests/diskio.rs index 1f9cd17b0..80e402fe7 100644 --- a/e2e-test/tests/diskio.rs +++ b/e2e-test/tests/diskio.rs @@ -68,9 +68,9 @@ async fn wait_for_disk( } } -fn registry() -> crowdb_kv_client::ServiceRegistryClient { - crowdb_kv_client::ServiceRegistryClient::new(crate::common::hardware::kv_client()) +fn registry() -> crowdb_paxos_kv_client::ServiceRegistryClient { + crowdb_paxos_kv_client::ServiceRegistryClient::new(crate::common::hardware::kv_client()) } -fn hardware_client() -> crowdb_kv_client::HardwareClient { - crowdb_kv_client::HardwareClient::new(crate::common::hardware::kv_client()) +fn hardware_client() -> crowdb_paxos_kv_client::HardwareClient { + crowdb_paxos_kv_client::HardwareClient::new(crate::common::hardware::kv_client()) } diff --git a/e2e-test/tests/diskio/hardware.rs b/e2e-test/tests/diskio/hardware.rs index 281d78a84..a95508a96 100644 --- a/e2e-test/tests/diskio/hardware.rs +++ b/e2e-test/tests/diskio/hardware.rs @@ -4,6 +4,6 @@ #[tokio::test] #[ignore = "requires the owned container fixture"] async fn seed() { - let hardware = crowdb_kv_client::HardwareClient::new(crate::common::hardware::kv_client()); + let hardware = crowdb_paxos_kv_client::HardwareClient::new(crate::common::hardware::kv_client()); crate::common::hardware::seed_hardware(&hardware, &crate::common::hardware::standard_disk_ids_4()).await; } diff --git a/e2e-test/tests/domain_test.rs b/e2e-test/tests/domain_test.rs index c41dce42a..8b3525f29 100644 --- a/e2e-test/tests/domain_test.rs +++ b/e2e-test/tests/domain_test.rs @@ -19,7 +19,7 @@ mod diskio; mod kv; use bytes::Bytes; -use crowdb_kv::rpc::{KvBatchItem, KvBatchWriteRequest, KvDeleteRequest, KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::rpc::{KvBatchItem, KvBatchWriteRequest, KvDeleteRequest, KvGetRequest, KvSetRequest}; use common::{run_kv_op_with_retry, KvOp}; diff --git a/e2e-test/tests/kv.rs b/e2e-test/tests/kv.rs index 605cf06c5..432fff3da 100644 --- a/e2e-test/tests/kv.rs +++ b/e2e-test/tests/kv.rs @@ -23,6 +23,7 @@ mod routing; mod snapshot; mod slot_map; +mod slot_map_recovery; mod slot_map_validation; mod handoff_fixture; diff --git a/e2e-test/tests/kv/authority.rs b/e2e-test/tests/kv/authority.rs index 1578d5c28..2c1d2609a 100644 --- a/e2e-test/tests/kv/authority.rs +++ b/e2e-test/tests/kv/authority.rs @@ -2,20 +2,20 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv::rpc::{KvBatchItem, KvErrorCode}; -use crowdb_kv_client::{BatchOp, Error, GetOutcome, KvRpcTransport, ReadMode}; +use crowdb_paxos_kv::rpc::{KvBatchItem, KvErrorCode}; +use crowdb_paxos_kv_client::{BatchOp, Error, GetOutcome, PaxosKvRpcTransport, ReadMode}; use crowdb_protocol::chunk_slot::{ChunkServiceIncarnation, ChunkSlotAuthority}; const FENCE: &[u8] = b"/chunkdb/ownership-fence/0"; use crate::common::management::{addresses, client as http_client, origin, wait_for_leader}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; -async fn kv() -> (CrowdbKvClient, String) { +async fn kv() -> (PaxosKvClient, String) { let ips = addresses(1); wait_for_leader(&http_client(), &origin(&ips[0])).await; ( - CrowdbKvClient::new(ClientConfig::new(vec![origin(&ips[0])])), + PaxosKvClient::new(ClientConfig::new(vec![origin(&ips[0])])), format!("{}:7001", ips[0]), ) } @@ -133,7 +133,7 @@ async fn ordinary_and_malformed_writes_cannot_change_slot_authority() { async fn raw_owner_rpc_rejects_malformed_authority_and_cross_namespace_mutation() { let (client, endpoint) = kv().await; let endpoint = &endpoint; - let transport = KvRpcTransport::new(); + let transport = PaxosKvRpcTransport::new(); let value = ChunkSlotAuthority::new(4, ChunkServiceIncarnation::try_from([1; 16]).unwrap(), 1) .unwrap() .to_fence_value(); diff --git a/e2e-test/tests/kv/client_data.rs b/e2e-test/tests/kv/client_data.rs index 08a6ae3f1..f12048eec 100644 --- a/e2e-test/tests/kv/client_data.rs +++ b/e2e-test/tests/kv/client_data.rs @@ -2,17 +2,17 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv_client::{BatchOp, ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; const STORE_ID: u64 = 1; const GROUP_ID: u64 = 1; -pub(super) async fn fixture_client() -> CrowdbKvClient { +pub(super) async fn fixture_client() -> PaxosKvClient { let ips = crate::common::management::addresses(1); let http = crate::common::management::client(); let seed = crate::common::management::origin(&ips[0]); crate::common::management::wait_for_leader(&http, &seed).await; - CrowdbKvClient::new(ClientConfig::new(vec![seed])) + PaxosKvClient::new(ClientConfig::new(vec![seed])) } #[tokio::test] @@ -69,7 +69,7 @@ async fn revision_cas_and_conditional_batch_round_trip() { .expect("create-if-absent CAS"); assert!(matches!( client.put_cas(STORE_ID, GROUP_ID, b"guard", b"stale", 0).await, - Err(crowdb_kv_client::Error::CasFailed { + Err(crowdb_paxos_kv_client::Error::CasFailed { current_revision, .. }) if current_revision == created.revision )); diff --git a/e2e-test/tests/kv/client_scan.rs b/e2e-test/tests/kv/client_scan.rs index ece70e973..08b289b09 100644 --- a/e2e-test/tests/kv/client_scan.rs +++ b/e2e-test/tests/kv/client_scan.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::client_data::fixture_client; -use crowdb_kv_client::ReadMode; +use crowdb_paxos_kv_client::ReadMode; const STORE_ID: u64 = 1; const GROUP_ID: u64 = 1; @@ -107,18 +107,20 @@ async fn scan_keys_only_and_scan_count() { async fn scan_deadline_returns_partial_result() { let client = fixture_client().await; - // Populate enough keys that the engine merge loop's periodic deadline - // check (every 1024 entries) fires mid-scan. + // Keep the full scan corpus while each setup proposal fits the native + // tree's reserved replay window. let items: Vec<_> = (0..3000u64) - .map(|i| crowdb_kv_client::BatchOp::Put { + .map(|i| crowdb_paxos_kv_client::BatchOp::Put { key: bytes::Bytes::from(format!("k{i:05}")), value: bytes::Bytes::from_static(b"v"), }) .collect(); - client - .batch_write(STORE_ID, GROUP_ID, &items) - .await - .expect("populate scan"); + for batch in items.chunks(64) { + client + .batch_write(STORE_ID, GROUP_ID, batch) + .await + .expect("populate scan"); + } // No deadline: full scan returns all 3000 entries. let full = client diff --git a/e2e-test/tests/kv/cluster.rs b/e2e-test/tests/kv/cluster.rs index d67938a76..62732a34e 100644 --- a/e2e-test/tests/kv/cluster.rs +++ b/e2e-test/tests/kv/cluster.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crate::common::management::{addresses, client, origin, request}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, KvRpcTransport, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, PaxosKvRpcTransport, ReadMode}; use reqwest::{Client, Method}; use serde_json::{json, Value}; use std::time::{Duration, Instant}; @@ -40,7 +40,7 @@ async fn topology_and_follower_hint() { assert!(Instant::now() < deadline, "follower lacks leader hint: {topo}"); tokio::time::sleep(Duration::from_millis(100)).await; } - let response = KvRpcTransport::new() + let response = PaxosKvRpcTransport::new() .send_put( &format!("http://{}:7001", ips[follower]), b"k", @@ -56,7 +56,7 @@ async fn topology_and_follower_hint() { assert!(!response.ok); assert_eq!(response.error, "not leader"); assert_eq!(response.not_leader_hint, format!("{}:7001", ips[leader])); - let kv = CrowdbKvClient::new(ClientConfig::new(Vec::new())); + let kv = PaxosKvClient::new(ClientConfig::new(Vec::new())); kv.seed_leader(10, 1, format!("{}:7001", ips[follower])); let first = kv.put(10, 1, b"redirect-one", b"one", None).await.unwrap(); let second = kv.put(10, 1, b"redirect-two", b"two", None).await.unwrap(); @@ -102,7 +102,7 @@ async fn add_group(http: &Client, ips: &[String]) { } } -async fn exact_value(kv: &CrowdbKvClient, group: u64, value: &[u8]) { +async fn exact_value(kv: &PaxosKvClient, group: u64, value: &[u8]) { assert!( matches!(kv.get(0, group, b"shared-key", ReadMode::Linearizable, None).await.unwrap(), GetOutcome::Found {value: found, ..} if found.as_ref() == value) @@ -117,7 +117,7 @@ async fn dynamic_groups_isolate_data() { add_group(&http, &ips).await; leader(&http, &ips, 1).await; leader(&http, &ips, 2).await; - let kv = CrowdbKvClient::new(ClientConfig::new(ips.iter().map(|ip| origin(ip)).collect())); + let kv = PaxosKvClient::new(ClientConfig::new(ips.iter().map(|ip| origin(ip)).collect())); for (group, value) in [(1, b"value-from-A"), (2, b"value-from-B")] { kv.put(0, group, b"shared-key", value, None).await.unwrap(); exact_value(&kv, group, value).await; diff --git a/e2e-test/tests/kv/discovery.rs b/e2e-test/tests/kv/discovery.rs index 9d0df1d23..e808252be 100644 --- a/e2e-test/tests/kv/discovery.rs +++ b/e2e-test/tests/kv/discovery.rs @@ -5,7 +5,7 @@ use crate::common::{ control, management::{addresses, client, origin, request}, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::common::KvServerIdentity; use reqwest::Method; use serde_json::{json, Value}; @@ -83,7 +83,7 @@ async fn nonmember_recovers_discovery_and_registration_identity() { 201, ) .await; - let service = ServiceRegistryClient::new(CrowdbKvClient::new(ClientConfig::new(vec![leader.clone()]))); + let service = ServiceRegistryClient::new(PaxosKvClient::new(ClientConfig::new(vec![leader.clone()]))); for seeds in [ json!([]), json!(["127.0.0.1:1"]), diff --git a/e2e-test/tests/kv/domain_monitor.rs b/e2e-test/tests/kv/domain_monitor.rs index d15ca9b72..6931d1977 100644 --- a/e2e-test/tests/kv/domain_monitor.rs +++ b/e2e-test/tests/kv/domain_monitor.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::client_data::fixture_client; -use crowdb_kv_client::DomainMonitorClient; +use crowdb_paxos_kv_client::DomainMonitorClient; use crowdb_protocol::chunk_kv::{ DomainFailurePolicy, DomainMonitorDescriptor, EnsureDomainMonitorOutcome, EnsureDomainMonitorRequest, }; diff --git a/e2e-test/tests/kv/failover.rs b/e2e-test/tests/kv/failover.rs index dfbbe6ebd..b79b0b039 100644 --- a/e2e-test/tests/kv/failover.rs +++ b/e2e-test/tests/kv/failover.rs @@ -7,15 +7,15 @@ use crate::common::{ control, management::{addresses, client, origin, request}, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use reqwest::{Client, Method}; use serde_json::{json, Value}; -fn kv(ips: &[String]) -> CrowdbKvClient { - CrowdbKvClient::new(ClientConfig::new(ips.iter().map(|ip| origin(ip)).collect())) +fn kv(ips: &[String]) -> PaxosKvClient { + PaxosKvClient::new(ClientConfig::new(ips.iter().map(|ip| origin(ip)).collect())) } -async fn exact(kv: &CrowdbKvClient, store: u64, key: &[u8], expected: &[u8]) { +async fn exact(kv: &PaxosKvClient, store: u64, key: &[u8], expected: &[u8]) { assert!( matches!(kv.get(store, 1, key, ReadMode::Linearizable, None).await.unwrap(), GetOutcome::Found {value, ..} if value.as_ref() == expected) diff --git a/e2e-test/tests/kv/handoff_fixture.rs b/e2e-test/tests/kv/handoff_fixture.rs index 00844155a..b034e65f5 100644 --- a/e2e-test/tests/kv/handoff_fixture.rs +++ b/e2e-test/tests/kv/handoff_fixture.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{ChunkSlotMapClient, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, PaxosKvClient}; use crowdb_protocol::chunk_slot::{ ChunkServiceHandoff, ChunkServiceIncarnation, ChunkSlot, ChunkSlotAuthority, ChunkSlotBootstrap, ChunkSlotTransfer, ChunkStorageGroup, @@ -10,7 +10,7 @@ use crowdb_protocol::key::{ChunkSlotFenceKey, TextKey}; use std::sync::Arc; pub(super) struct TestHandoffCluster { - pub(super) kv: Arc<CrowdbKvClient>, + pub(super) kv: Arc<PaxosKvClient>, } impl TestHandoffCluster { diff --git a/e2e-test/tests/kv/maintenance.rs b/e2e-test/tests/kv/maintenance.rs index a4bb83a2b..9e8921811 100644 --- a/e2e-test/tests/kv/maintenance.rs +++ b/e2e-test/tests/kv/maintenance.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crate::common::management::{addresses, client, origin, request, wait_for_leader}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use reqwest::Method; use serde_json::{json, Value}; @@ -46,7 +46,7 @@ async fn flush_and_wipe_preserve_group_and_resume_writes() { let client = client(); maintenance_errors(&client, &base).await; wait_for_leader(&client, &base).await; - let kv = CrowdbKvClient::new(ClientConfig::new(vec![base.clone()])); + let kv = PaxosKvClient::new(ClientConfig::new(vec![base.clone()])); kv.put(0, 1, b"wipe-test-key", b"wipe-test-value", None) .await .unwrap(); @@ -79,7 +79,7 @@ async fn flush_and_wipe_preserve_group_and_resume_writes() { let restored = wait_for_leader(&client, &base).await; assert_eq!(restored["stores"][0]["store_id"], 0); assert_eq!(restored["stores"][0]["groups"][0]["group_id"], 1); - let kv = CrowdbKvClient::new(ClientConfig::new(vec![base])); + let kv = PaxosKvClient::new(ClientConfig::new(vec![base])); assert!(matches!( kv.get(0, 1, b"wipe-test-key", ReadMode::Linearizable, None) .await diff --git a/e2e-test/tests/kv/membership_cas.rs b/e2e-test/tests/kv/membership_cas.rs index 066f8eca4..4f2e0ba3e 100644 --- a/e2e-test/tests/kv/membership_cas.rs +++ b/e2e-test/tests/kv/membership_cas.rs @@ -2,13 +2,13 @@ // Licensed under the Apache License, Version 2.0. use crate::common::management::{addresses, client as http_client, origin, wait_for_leader}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, Error, GroupMembershipClient}; +use crowdb_paxos_kv_client::{ClientConfig, Error, GroupMembershipClient, PaxosKvClient}; use crowdb_protocol::kv_membership::{GroupMember, GroupMembershipState}; async fn client() -> GroupMembershipClient { let ips = addresses(1); wait_for_leader(&http_client(), &origin(&ips[0])).await; - GroupMembershipClient::from_shared(std::sync::Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![ + GroupMembershipClient::from_shared(std::sync::Arc::new(PaxosKvClient::new(ClientConfig::new(vec![ origin(&ips[0]), ])))) } diff --git a/e2e-test/tests/kv/reconfiguration.rs b/e2e-test/tests/kv/reconfiguration.rs index 847aeceae..59d726974 100644 --- a/e2e-test/tests/kv/reconfiguration.rs +++ b/e2e-test/tests/kv/reconfiguration.rs @@ -3,13 +3,13 @@ use super::cluster::{leader, topology}; use crate::common::management::{addresses, client, origin, request}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, KvRpcTransport, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, PaxosKvRpcTransport, ReadMode}; use reqwest::{Client, Method}; use serde_json::{json, Value}; use std::time::{Duration, Instant}; -fn kv(ips: &[String]) -> CrowdbKvClient { - CrowdbKvClient::new(ClientConfig::new(ips.iter().map(|ip| origin(ip)).collect())) +fn kv(ips: &[String]) -> PaxosKvClient { + PaxosKvClient::new(ClientConfig::new(ips.iter().map(|ip| origin(ip)).collect())) } pub(super) async fn wire_initial(http: &Client, ips: &[String]) { @@ -52,14 +52,14 @@ async fn applied(http: &Client, ip: &str) -> u64 { } async fn imported_value(ip: &str, required: u64) { - let response = KvRpcTransport::new() + let response = PaxosKvRpcTransport::new() .send_get( &format!("http://{ip}:7001"), b"k1", 9201, 9201, 1, - crowdb_kv::rpc::ReadMode::MinSlot, + crowdb_paxos_kv::rpc::ReadMode::MinSlot, required, ) .await @@ -174,7 +174,7 @@ async fn shrink(http: &Client, ips: &[String], mut epoch: u64) { } } -async fn exact(kv: &CrowdbKvClient, store: u64, key: &[u8], value: &[u8]) { +async fn exact(kv: &PaxosKvClient, store: u64, key: &[u8], value: &[u8]) { let deadline = Instant::now() + Duration::from_secs(30); loop { match kv.get(store, 1, key, ReadMode::Linearizable, None).await.unwrap() { diff --git a/e2e-test/tests/kv/routing.rs b/e2e-test/tests/kv/routing.rs index 9f4b79a48..5e16005ba 100644 --- a/e2e-test/tests/kv/routing.rs +++ b/e2e-test/tests/kv/routing.rs @@ -2,13 +2,13 @@ // Licensed under the Apache License, Version 2.0. use crate::common::management::{addresses, origin}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; #[tokio::test] #[ignore = "requires three release-container nodes, each hosting stores one and two"] async fn multi_store_isolation_and_recovery() { let seeds = addresses(3).iter().map(|ip| origin(ip)).collect(); - let client = CrowdbKvClient::new(ClientConfig::new(seeds)); + let client = PaxosKvClient::new(ClientConfig::new(seeds)); if std::env::var("CROWDB_E2E_PHASE").unwrap() == "write" { for (store, key, value) in [(1, b"ms-a", b"val-a"), (2, b"ms-b", b"val-b")] { client.put(store, 10, key, value, None).await.unwrap(); diff --git a/e2e-test/tests/kv/slot_handoff.rs b/e2e-test/tests/kv/slot_handoff.rs index 0ae866706..fc0410879 100644 --- a/e2e-test/tests/kv/slot_handoff.rs +++ b/e2e-test/tests/kv/slot_handoff.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::handoff_fixture::TestHandoffCluster; -use crowdb_kv_client::{ChunkSlotMapClient, Error}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, Error}; use crowdb_protocol::chunk_slot::{ ChunkServiceHandoff, ChunkServiceHandoffPhase, ChunkSlot, ChunkSlotFenceReceipt, }; diff --git a/e2e-test/tests/kv/slot_map.rs b/e2e-test/tests/kv/slot_map.rs index c79fd50e7..385f29532 100644 --- a/e2e-test/tests/kv/slot_map.rs +++ b/e2e-test/tests/kv/slot_map.rs @@ -3,8 +3,8 @@ use std::sync::Arc; -use crowdb_kv_client::{ChunkSlotMapClient, CrowdbKvClient, GetOutcome, ReadMode}; -use crowdb_kv_client::{RangeBindingClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, GetOutcome, PaxosKvClient, ReadMode}; +use crowdb_paxos_kv_client::{RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunk_slot::{ ChunkSlot, ChunkSlotBinding, ChunkSlotBitmap, ChunkSlotMap, ChunkSlotMapHead, ChunkStorageGroup, CHUNK_SLOT_LAYOUT_VERSION, @@ -12,7 +12,7 @@ use crowdb_protocol::chunk_slot::{ use crowdb_protocol::key::{ChunkServiceSlotsKey, ChunkSlotMapHeadKey, TextKey}; pub(super) struct TestGroup0 { - pub(super) kv: Arc<CrowdbKvClient>, + pub(super) kv: Arc<PaxosKvClient>, } impl TestGroup0 { pub(super) async fn start() -> Self { @@ -134,7 +134,19 @@ async fn concurrent_initializers_cannot_combine_different_assignments() { async fn paginated_owner_records_preserve_empty_instances() { let test = TestGroup0::start().await; let client = ChunkSlotMapClient::new(Arc::clone(&test.kv)); - let map = ChunkSlotMap::new( + let map = paginated_services(); + client.initialize_service(&map).await.unwrap(); + let actual = client.read_service().await.unwrap(); + assert_eq!(actual.bindings().len(), 300); + assert_eq!( + actual.bindings().iter().filter(|b| b.slots.is_empty()).count(), + 299 + ); + assert!(ChunkSlot::all().all(|slot| actual.owner(slot) == 1)); +} + +pub(super) fn paginated_services() -> ChunkSlotMap<u64> { + ChunkSlotMap::new( ChunkSlotMapHead { layout_version: CHUNK_SLOT_LAYOUT_VERSION, generation: 1, @@ -152,15 +164,7 @@ async fn paginated_owner_records_preserve_empty_instances() { }) .collect(), ) - .unwrap(); - client.initialize_service(&map).await.unwrap(); - let actual = client.read_service().await.unwrap(); - assert_eq!(actual.bindings().len(), 300); - assert_eq!( - actual.bindings().iter().filter(|b| b.slots.is_empty()).count(), - 299 - ); - assert!(ChunkSlot::all().all(|slot| actual.owner(slot) == 1)); + .unwrap() } #[tokio::test] diff --git a/e2e-test/tests/kv/slot_map_recovery.rs b/e2e-test/tests/kv/slot_map_recovery.rs new file mode 100644 index 000000000..9a7ff9022 --- /dev/null +++ b/e2e-test/tests/kv/slot_map_recovery.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::slot_map::{paginated_services, TestGroup0}; +use bytes::Bytes; +use crowdb_paxos_kv_client::{BatchOp, ChunkSlotMapClient, GetOutcome, ReadMode}; +use crowdb_protocol::chunk_slot::ChunkSlot; +use crowdb_protocol::key::{ChunkServiceSlotsKey, ChunkSlotMapHeadKey, TextKey}; +use std::sync::Arc; + +#[tokio::test] +#[ignore = "requires the owned release-container Group-0 slot map profile"] +async fn interrupted_large_bootstrap_resumes_after_crash() { + let test = TestGroup0::start().await; + let client = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + let map = paginated_services(); + let head = ChunkSlotMapHeadKey::Service.to_path(); + let phase = std::env::var("CROWDB_E2E_PHASE").unwrap(); + if phase == "write" { + let mut bindings = map.bindings().to_vec(); + bindings.sort_by_key(|binding| { + ChunkServiceSlotsKey { + instance_id: binding.owner, + } + .to_path() + }); + let intent = serde_json::json!({ + "initializing": { "head": map.head(), "bindings": bindings }, "next": 63 + }); + let mut operations: Vec<_> = bindings[..63] + .iter() + .map(|binding| BatchOp::Put { + key: Bytes::from( + ChunkServiceSlotsKey { + instance_id: binding.owner, + } + .to_path(), + ), + value: Bytes::from(serde_json::to_vec(binding).unwrap()), + }) + .collect(); + operations.push(BatchOp::Put { + key: Bytes::from(head.clone()), + value: Bytes::from(serde_json::to_vec(&intent).unwrap()), + }); + test.kv + .batch_write_cas(0, 0, &operations, head.as_bytes(), 0) + .await + .unwrap(); + assert!(client.read_service().await.is_err()); + return; + } + assert_eq!(phase, "verify"); + let GetOutcome::Found { value, .. } = test + .kv + .get(0, 0, head.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!("acknowledged bootstrap progress was lost on restart"); + }; + let intent: serde_json::Value = serde_json::from_slice(&value).unwrap(); + assert_eq!(intent["next"], 63); + assert_eq!(intent["initializing"]["bindings"].as_array().unwrap().len(), 300); + assert!(client.read_service().await.is_err()); + client.initialize_service(&map).await.unwrap(); + client.initialize_service(&map).await.unwrap(); + let actual = client.read_service().await.unwrap(); + assert_eq!(actual.bindings().len(), 300); + assert_eq!( + actual + .bindings() + .iter() + .filter(|binding| binding.slots.is_empty()) + .count(), + 299 + ); + assert!(ChunkSlot::all().all(|slot| actual.owner(slot) == 1)); +} diff --git a/e2e-test/tests/kv/slot_map_validation.rs b/e2e-test/tests/kv/slot_map_validation.rs index f0a42a836..3d145ee38 100644 --- a/e2e-test/tests/kv/slot_map_validation.rs +++ b/e2e-test/tests/kv/slot_map_validation.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::slot_map::{services, TestGroup0}; -use crowdb_kv_client::{ChunkSlotMapClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, GetOutcome, ReadMode}; use crowdb_protocol::key::{ChunkServiceSlotsKey, ChunkSlotMapHeadKey, TextKey}; use std::sync::Arc; diff --git a/e2e-test/tests/kv/slot_publication.rs b/e2e-test/tests/kv/slot_publication.rs index 33d970de3..bafa91108 100644 --- a/e2e-test/tests/kv/slot_publication.rs +++ b/e2e-test/tests/kv/slot_publication.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::handoff_fixture::TestHandoffCluster; -use crowdb_kv_client::ChunkSlotMapClient; +use crowdb_paxos_kv_client::ChunkSlotMapClient; use crowdb_protocol::chunk_slot::{ ChunkServiceIncarnation, ChunkSlot, ChunkSlotAuthority, ChunkSlotBinding, ChunkSlotBitmap, ChunkSlotMap, }; @@ -15,7 +15,7 @@ async fn missing_cached_endpoint_refreshes_dynamic_owner_before_routing() { let test = TestHandoffCluster::start().await; let maps = ChunkSlotMapClient::new(Arc::clone(&test.kv)); maps.initialize_service_epochs().await.unwrap(); - crowdb_kv_client::ServiceRegistryClient::from_shared(Arc::clone(&test.kv)) + crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(Arc::clone(&test.kv)) .register_chunkdb(2, "127.0.0.1:17002") .await .unwrap(); @@ -25,11 +25,11 @@ async fn missing_cached_endpoint_refreshes_dynamic_owner_before_routing() { .find(|id| before.service().owner(ChunkSlot::for_chunk(id)) == 1) .unwrap(); let slot = ChunkSlot::for_chunk(&chunk); - let routes = crowdb_kv_client::RangeBindingClient::from_shared(Arc::clone(&test.kv)); + let routes = crowdb_paxos_kv_client::RangeBindingClient::from_shared(Arc::clone(&test.kv)); routes.refresh().await.unwrap(); assert!(matches!( routes.route_slot(slot), - Err(crowdb_kv_client::RangeRouteError::NoEndpoint(1)) + Err(crowdb_paxos_kv_client::RangeRouteError::NoEndpoint(1)) )); let after = before.authority().reassign(&[(slot, 2)]).unwrap(); maps.publish_service_epochs(&after).await.unwrap(); @@ -54,7 +54,7 @@ async fn stale_epoch_publications_are_conflicts_and_concurrent_regrants_preserve second.unwrap(); assert!(matches!( client.publish_service_epochs(&stale).await, - Err(crowdb_kv_client::Error::CasFailed { .. }) + Err(crowdb_paxos_kv_client::Error::CasFailed { .. }) )); let after = client.read_service_snapshot().await.unwrap(); assert_eq!( @@ -124,8 +124,8 @@ async fn complete_epoch_publication_is_atomic_idempotent_and_storage_independent assert!( matches!( (&a, &b), - (Ok(()), Ok(()) | Err(crowdb_kv_client::Error::CasBusy)) - | (Err(crowdb_kv_client::Error::CasBusy), Ok(())) + (Ok(()), Ok(()) | Err(crowdb_paxos_kv_client::Error::CasBusy)) + | (Err(crowdb_paxos_kv_client::Error::CasBusy), Ok(())) ), "first: {a:?}; second: {b:?}" ); @@ -198,12 +198,12 @@ async fn competing_publications_choose_one_complete_epoch_map_without_changing_d 0, 1, path.as_bytes(), - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await .unwrap(); assert!( - matches!(original, crowdb_kv_client::GetOutcome::Found { value, .. } if value.as_ref() == TestHandoffCluster::plan(2).record().transfers[0].previous.unwrap().to_fence_value()) + matches!(original, crowdb_paxos_kv_client::GetOutcome::Found { value, .. } if value.as_ref() == TestHandoffCluster::plan(2).record().transfers[0].previous.unwrap().to_fence_value()) ); } diff --git a/e2e-test/tests/kv/snapshot.rs b/e2e-test/tests/kv/snapshot.rs index eafc2408c..92a9169cf 100644 --- a/e2e-test/tests/kv/snapshot.rs +++ b/e2e-test/tests/kv/snapshot.rs @@ -3,8 +3,8 @@ use super::cluster::{leader, topology}; use crate::common::management::{addresses, client, origin, request}; -use crowdb_kv::rpc::ReadMode; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, KvRpcTransport}; +use crowdb_paxos_kv::rpc::ReadMode; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, PaxosKvRpcTransport}; use reqwest::{Client, Method}; use serde_json::{json, Value}; use std::time::{Duration, Instant}; @@ -21,7 +21,13 @@ async fn peers(http: &Client, ip: &str, store: usize, body: Value) { .await; } -async fn local_value(transport: &KvRpcTransport, ip: &str, key: &[u8], expected: &[u8], timeout: Duration) { +async fn local_value( + transport: &PaxosKvRpcTransport, + ip: &str, + key: &[u8], + expected: &[u8], + timeout: Duration, +) { let deadline = Instant::now() + timeout; loop { let response = transport @@ -65,7 +71,7 @@ async fn snapshot_import_precedes_wiring_and_catches_up_tail() { .await; } let leader = leader(&http, &ips[..2], 1).await; - let kv = CrowdbKvClient::new(ClientConfig::new(ips[..2].iter().map(|ip| origin(ip)).collect())); + let kv = PaxosKvClient::new(ClientConfig::new(ips[..2].iter().map(|ip| origin(ip)).collect())); for (key, value) in [(b"k1", b"v1"), (b"k2", b"v2")] { kv.put(0, 1, key, value, None).await.unwrap(); } @@ -80,7 +86,7 @@ async fn snapshot_import_precedes_wiring_and_catches_up_tail() { 201, ) .await; - let transport = KvRpcTransport::new(); + let transport = PaxosKvRpcTransport::new(); for (key, value) in [(b"k1", b"v1"), (b"k2", b"v2")] { local_value(&transport, &ips[2], key, value, Duration::from_secs(1)).await; } diff --git a/e2e-test/tests/managed_supervision_test.py b/e2e-test/tests/managed_supervision_test.py index 47e27dfce..0fd99d831 100644 --- a/e2e-test/tests/managed_supervision_test.py +++ b/e2e-test/tests/managed_supervision_test.py @@ -10,7 +10,7 @@ class ManagedSupervisionTest(unittest.TestCase): - def fixture(self, label="owned", pid=27, executable="/opt/crowdb/bin/crowdb-kv-server"): + def fixture(self, label="owned", pid=27, executable="/opt/crowdb/bin/crowdb-paxos-kv-server"): cluster = object.__new__(ManagedCluster) cluster.services, cluster.servers, cluster.run_id = ("kv",), ["node"], "owned" cluster.command = Mock(side_effect=[json.dumps([{ diff --git a/e2e-test/tests/native_containerd_test.py b/e2e-test/tests/native_containerd_test.py index dc8c90141..6b5adcd55 100644 --- a/e2e-test/tests/native_containerd_test.py +++ b/e2e-test/tests/native_containerd_test.py @@ -92,8 +92,11 @@ def test_owned_daemon_is_closed_after_body_failure_without_touching_other_proces process.wait.assert_called_once_with(timeout=10) def test_host_fixture_imports_cannot_shadow_the_e2e_artifact_publisher(self): - result=subprocess.run([sys.executable,'-c', - 'import native_containerd_profile; import publish; print(publish.__file__)'], - env=dict(os.environ,PYTHONPATH=str(Path(__file__).resolve().parents[1])), - capture_output=True,text=True,check=True,timeout=10) - self.assertEqual(Path(result.stdout.strip()).resolve(),Path(__file__).resolve().parents[1]/'publish.py') + directory = Path(__file__).resolve().parents[1] + for module in ('native_containerd_profile', 'paxos_kv_diagnostics_test'): + with self.subTest(module=module): + result=subprocess.run([sys.executable,'-c', + f'import {module}; import publish; print(publish.__file__)'], + env=dict(os.environ,PYTHONPATH=os.pathsep.join((str(directory), str(directory / 'tests')))), + capture_output=True,text=True,check=True,timeout=10) + self.assertEqual(Path(result.stdout.strip()).resolve(),directory/'publish.py') diff --git a/e2e-test/tests/oci_reference_test.py b/e2e-test/tests/oci_reference_test.py index 270fda706..7e2f30ce4 100644 --- a/e2e-test/tests/oci_reference_test.py +++ b/e2e-test/tests/oci_reference_test.py @@ -5,6 +5,8 @@ import unittest from unittest.mock import Mock, patch +import default_ports + from fixture import LABEL import oci_reference_runtime as runtime from scenarios import case_nodes, case_resources, select_cases @@ -127,8 +129,11 @@ def test_recreated_names_reuse_exact_retained_volumes_on_the_accepted_new_networ self.assertEqual(create[create.index('--network') + 1], 'owned-recovered') self.assertNotIn('owned-node1-cold-data', create) published = [create[index+1] for index, arg in enumerate(create) if arg == '--publish'] - self.assertEqual(published, ['127.0.0.1:20000:9090', '127.0.0.1:20001:16000', - '127.0.0.1:20002:17000', '127.0.0.1:20003:18000']) + self.assertEqual(published, [ + f'127.0.0.1:20000:{default_ports.WEB_BASE}', + f'127.0.0.1:20001:{default_ports.ACCESS_SERVER_HTTP_BASE}', + f'127.0.0.1:20002:{default_ports.ACCESS_SERVER_ICEBERG_HTTP_BASE}', + f'127.0.0.1:20003:{default_ports.ACCESS_SERVER_DATASET_HTTP_BASE}']) def test_cold_network_scope_still_rejects_foreign_names_and_attachments(self): cluster = self.cluster() diff --git a/e2e-test/tests/oci_workloads_test.py b/e2e-test/tests/oci_workloads_test.py index 8a022f378..a7218ba25 100644 --- a/e2e-test/tests/oci_workloads_test.py +++ b/e2e-test/tests/oci_workloads_test.py @@ -21,7 +21,7 @@ def services(self): def test_stops_producers_in_dependency_order_and_preserves_allocator_processes(self): services, actions = self.services(), [] def request(origin, path, body=None): - self.assertEqual(origin, "http://owned:9090") + self.assertEqual(origin, "http://owned:6135") if path == "/api/servers": return copy.deepcopy(services) service_id = path.split("/")[-2] @@ -31,7 +31,7 @@ def request(origin, path, body=None): next(item for item in services if item["id"] == service_id)["pid"] = None with tempfile.TemporaryDirectory() as directory: cluster = SimpleNamespace(node_count=1, artifacts=Path(directory), - endpoint=lambda _: "http://owned:9090") + endpoint=lambda _: "http://owned:6135") with patch("oci_reference_workloads.request", side_effect=request): quiesce_chunk_allocators(cluster) proof = json.loads((cluster.artifacts / "allocator-workload-precondition.json").read_text()) @@ -41,12 +41,12 @@ def request(origin, path, body=None): for index, kind in enumerate(("paxos-kv", "diskdb", "diskio"))]) def test_missing_producer_rejects_before_stopping_any_service(self): - cluster = SimpleNamespace(node_count=1, endpoint=lambda _: "http://owned:9090") + cluster = SimpleNamespace(node_count=1, endpoint=lambda _: "http://owned:6135") services = [item for item in self.services() if item["service_type"] != "chunkdb"] with patch("oci_reference_workloads.request", return_value=services) as request: with self.assertRaisesRegex(AssertionError, "one owned chunkdb"): quiesce_chunk_allocators(cluster) - request.assert_called_once_with("http://owned:9090", "/api/servers") + request.assert_called_once_with("http://owned:6135", "/api/servers") def test_other_topologies_reject_before_contacting_the_target(self): with patch("oci_reference_workloads.request", new=Mock()) as request: diff --git a/e2e-test/tests/paxos_kv_diagnostics_test.py b/e2e-test/tests/paxos_kv_diagnostics_test.py new file mode 100644 index 000000000..ec19a776e --- /dev/null +++ b/e2e-test/tests/paxos_kv_diagnostics_test.py @@ -0,0 +1,126 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +"""Container diagnostic ownership and failure propagation contracts.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.append(str(ROOT / 'container/oci-image')) +SPEC = importlib.util.spec_from_file_location('paxos_cluster_diagnostics', ROOT / 'container/oci-image/paxos-kv-cluster.py') +cluster = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(cluster) +PID_SPEC = importlib.util.spec_from_file_location('paxos_profile_pids', ROOT / 'tools/profiling/container-server-pids.py') +profile_pids = importlib.util.module_from_spec(PID_SPEC) +PID_SPEC.loader.exec_module(profile_pids) + + +class PaxosKvDiagnosticsTest(unittest.TestCase): + def test_profile_selects_server_instead_of_container_init(self): + table = ('PID COMMAND\n101 /usr/bin/tini -- /entrypoint server paxos-kv\n' + '102 /opt/crowdb/bin/crowdb-paxos-kv-server --root /data\n' + '103 logger crowdb-paxos-kv-server\n') + self.assertEqual(profile_pids.server_pid(table), 102) + + def test_profile_rejects_absent_or_ambiguous_server(self): + for table in ['PID COMMAND\n101 /usr/bin/tini\n', + 'PID COMMAND\n101 crowdb-paxos-kv-server\n102 crowdb-paxos-kv-server\n']: + with self.assertRaisesRegex(ValueError, 'exactly one live'): + profile_pids.server_pid(table) + + def test_foreign_container_is_never_stopped(self): + found = {'Config': {'Labels': {cluster.LABEL: 'foreign'}}} + with patch.object(cluster, 'docker', return_value=json.dumps([found])) as docker: + with self.assertRaisesRegex(RuntimeError, 'ownership mismatch'): + cluster.stop({'id': 'owned', 'containers': ['node']}, graceful=True) + self.assertEqual(docker.call_args_list[0].args, ('inspect', 'node')) + self.assertEqual(docker.call_count, 1) + + def check_shutdown(self, code=0, oom=False, report=False): + state = {'id': 'owned', 'containers': ['node'], 'sanitizer': True} + found = {'Config': {'Labels': {cluster.LABEL: 'owned'}}, + 'State': {'ExitCode': code, 'OOMKilled': oom}} + calls = [] + def docker(*args): + calls.append(args) + if args[0] == 'inspect': + return json.dumps([found]) + if args[0] == 'cp': + output = Path(args[-1]) + output.mkdir() + if report: + (output / 'report.1').write_text('ERROR: LeakSanitizer: detected memory leaks') + return 'service lifecycle' if args[0] == 'logs' else '' + with tempfile.TemporaryDirectory() as temporary, patch.object(cluster, 'docker', side_effect=docker): + artifacts = Path(temporary) + failures = cluster.stop(state, artifacts, graceful=True) + self.assertTrue((artifacts / 'node/inspect.json').is_file()) + self.assertTrue((artifacts / 'node/stdout.log').is_file()) + self.assertTrue((artifacts / 'node/log').is_dir()) + self.assertLess(calls.index(('stop', '--time', '30', 'node')), calls.index(('rm', '-f', '-v', 'node'))) + return failures + + def test_clean_shutdown_passes(self): + self.assertEqual(self.check_shutdown(), []) + + def test_collection_failure_preserves_node_and_stops_other_nodes(self): + state = {'id': 'owned', 'containers': ['first', 'second'], 'sanitizer': True} + found = {'Config': {'Labels': {cluster.LABEL: 'owned'}}, + 'State': {'ExitCode': 0, 'OOMKilled': False}} + calls = [] + def docker(*args): + calls.append(args) + if args[0] == 'inspect': + return json.dumps([found]) + if args[0] == 'cp': + if args[1].startswith('first:'): + raise RuntimeError('cannot read diagnostic directory') + Path(args[-1]).mkdir(exist_ok=True) + return '' + with tempfile.TemporaryDirectory() as temporary, patch.object(cluster, 'docker', side_effect=docker): + failures = cluster.stop(state, Path(temporary), graceful=True) + self.assertIn('cannot read diagnostic directory', '\n'.join(failures)) + self.assertEqual(state['containers'], ['first']) + self.assertIn(('stop', '--time', '30', 'second'), calls) + self.assertIn(('rm', '-f', '-v', 'second'), calls) + self.assertNotIn(('rm', '-f', '-v', 'first'), calls) + + def test_all_ownership_is_checked_before_any_stop(self): + def docker(*args): + owner = 'owned' if args[1] == 'first' else 'foreign' + return json.dumps([{'Config': {'Labels': {cluster.LABEL: owner}}}]) + with patch.object(cluster, 'docker', side_effect=docker) as mocked: + with self.assertRaisesRegex(RuntimeError, 'ownership mismatch: second'): + cluster.stop({'id': 'owned', 'containers': ['first', 'second']}, graceful=True) + self.assertTrue(all(call.args[0] == 'inspect' for call in mocked.call_args_list)) + + def test_killed_oom_and_sanitizer_report_each_fail(self): + self.assertTrue(self.check_shutdown(code=137)) + self.assertTrue(self.check_shutdown(oom=True)) + self.assertIn('LeakSanitizer', '\n'.join(self.check_shutdown(report=True))) + + def test_sampler_success_cannot_hide_failed_report(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + state = root / 'deployment.json' + state.write_text(json.dumps({'containers': ['a', 'b', 'c'], 'endpoints': ['http://localhost:1'] * 3})) + cli = root / 'fake-cli' + cli.write_text('#!/usr/bin/env python3\nimport sys\nfrom pathlib import Path\n' + 'Path(sys.argv[sys.argv.index("--output")+1]).write_text(\'{"passed":false}\')\n') + cli.chmod(0o700) + env = dict(os.environ, KV_BENCH_DEPLOYMENT=str(state), KV_BENCH_CLI=str(cli), + KV_BENCH_LOG_ROOT=str(root / 'logs')) + run = subprocess.run(['bash', '-c', 'source tools/benchmark/bench-kv-common.sh; invoke test write'], + cwd=ROOT, env=env, capture_output=True, text=True, timeout=10) + self.assertNotEqual(run.returncode, 0, run.stdout + run.stderr) + self.assertIn('failed benchmark report', run.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/e2e-test/tests/regression_fixture_test.py b/e2e-test/tests/regression_fixture_test.py index 934a3ea1f..4f549b24b 100644 --- a/e2e-test/tests/regression_fixture_test.py +++ b/e2e-test/tests/regression_fixture_test.py @@ -65,10 +65,10 @@ def test_dead_owner_cannot_spawn_a_fixture(self): def test_failed_client_drops_stale_report_and_does_not_inherit_cluster_target(self): self.write_descriptor() - report = self.root / "client-runtime/artifacts/cli/bench-kv/report.json" + report = self.root / "client-runtime/artifacts/cli/bench-paxos-kv/report.json" report.parent.mkdir(parents=True) report.write_text('{"total_ops": 999}') - args = argparse.Namespace(state=self.state, action="client", arguments=["bench", "kv", "bad"]) + args = argparse.Namespace(state=self.state, action="client", arguments=["bench", "paxos-kv", "bad"]) with patch.dict(os.environ, {"CROWDB_CLUSTER": "foreign"}), \ patch.object(fixture.subprocess, "call", return_value=2) as call: self.assertEqual(fixture.run(args), 2) diff --git a/e2e-test/tests/selection_test.py b/e2e-test/tests/selection_test.py index f652961c6..5c907e185 100644 --- a/e2e-test/tests/selection_test.py +++ b/e2e-test/tests/selection_test.py @@ -93,10 +93,13 @@ def test_remote_browser_flows_reserve_each_server_node_and_host_client(self): self.assertEqual(case_resources(case, 2), (1.5, count * 2048 + 512)) def test_client_and_metadata_scopes_expose_their_named_flows(self): - for scope, count in (("kv.client", 6), ("kv.metadata", 14)): + for scope, count in (("kv.client", 6), ("kv.metadata", 15)): cases = json.loads(self.invoke("--list", "--scope", scope)) self.assertEqual(len(cases), count) self.assertTrue(all(case["scope"] == scope for case in cases)) + if scope == "kv.metadata": + self.assertIn("kv::slot_map_recovery::interrupted_large_bootstrap_resumes_after_crash", + [case["case"] for case in cases]) def test_layers_select_only_their_domains_and_count_dependencies(self): diskdb = json.loads(self.invoke("--list", "--layer", "diskdb")) diff --git a/lib/crowdb-access-iceberg/Cargo.toml b/lib/crowdb-access-iceberg/Cargo.toml index 8883d347d..259e8df47 100644 --- a/lib/crowdb-access-iceberg/Cargo.toml +++ b/lib/crowdb-access-iceberg/Cargo.toml @@ -27,7 +27,7 @@ lz4_flex = { version = "0.11", default-features = false, features = ["std", "saf md-5 = "0.10" crowdb-chunk-kv-client = { path = "../crowdb-chunk-kv-client" } crowdb-chunk-client = { path = "../crowdb-chunk-client" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-access-multipart = { path = "../crowdb-access-multipart" } crowdb-common = { workspace = true } crowdb-protocol = { path = "../crowdb-protocol" } diff --git a/lib/crowdb-access-iceberg/src/storage.rs b/lib/crowdb-access-iceberg/src/storage.rs index 9edec2727..452658f0f 100644 --- a/lib/crowdb-access-iceberg/src/storage.rs +++ b/lib/crowdb-access-iceberg/src/storage.rs @@ -15,7 +15,7 @@ use crowdb_chunk_kv_client::{ Group0ChunkKvRangeCatalogSource, }; use crowdb_common::ec::EcScheme; -use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig as KvConfig, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::{ChunkType, Location}; use crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES; @@ -259,7 +259,7 @@ pub async fn connect_with_chunk_kv_config( chunk_kv_rpc: &ChunkKvRpcTransportConfig, ) -> Result<(Arc<CatalogRepository>, Arc<RoutedCatalogStore>, ChunkIoClient), IcebergStorageError> { small_write.chunk_type = ChunkType::IcebergTable; - let control = Arc::new(CrowdbKvClient::new(KvConfig::new(seeds.clone()))); + let control = Arc::new(PaxosKvClient::new(KvConfig::new(seeds.clone()))); let client_config = ClientConfig::default(); let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&control))); let transport = Arc::new(ChunkKvRpcTransport::with_config( diff --git a/lib/crowdb-access-s3/Cargo.toml b/lib/crowdb-access-s3/Cargo.toml index 6213ac116..00922d79f 100644 --- a/lib/crowdb-access-s3/Cargo.toml +++ b/lib/crowdb-access-s3/Cargo.toml @@ -23,7 +23,7 @@ crowdb-chunk-client = { path = "../crowdb-chunk-client" } crowdb-common = { workspace = true } crowdb-access-multipart = { path = "../crowdb-access-multipart" } crowdb-chunk-kv-client = { path = "../crowdb-chunk-kv-client" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-protocol = { path = "../crowdb-protocol" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } flatbuffers = { workspace = true } diff --git a/lib/crowdb-access-s3/src/storage.rs b/lib/crowdb-access-s3/src/storage.rs index b878fd003..1732b114f 100644 --- a/lib/crowdb-access-s3/src/storage.rs +++ b/lib/crowdb-access-s3/src/storage.rs @@ -15,12 +15,12 @@ use crowdb_chunk_kv_client::{ Group0ChunkKvRangeCatalogSource, }; use crowdb_common::ec::EcScheme; -use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig as KvConfig, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::ChunkType; #[derive(Clone)] pub struct S3StorageClients { - pub control: Arc<CrowdbKvClient>, + pub control: Arc<PaxosKvClient>, pub metadata: Arc<ChunkKvMetadataStore>, pub chunks: Arc<ChunkIoClient>, } @@ -185,7 +185,7 @@ impl S3StorageClients { chunk_kv_rpc: &ChunkKvRpcTransportConfig, ) -> Result<Self, StorageConnectError> { small_write.chunk_type = ChunkType::S3; - let kv = Arc::new(CrowdbKvClient::new(KvConfig::new(management_seeds.clone()))); + let kv = Arc::new(PaxosKvClient::new(KvConfig::new(management_seeds.clone()))); let config = ChunkKvConfig::default(); let catalog = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&kv))); let transport = Arc::new( diff --git a/lib/crowdb-access-s3/tests/common/multipart_completion_request.rs b/lib/crowdb-access-s3/tests/common/multipart_completion_request.rs index d03eb58a6..6b5d00880 100644 --- a/lib/crowdb-access-s3/tests/common/multipart_completion_request.rs +++ b/lib/crowdb-access-s3/tests/common/multipart_completion_request.rs @@ -9,7 +9,7 @@ use crowdb_access_s3::{ storage::S3StorageClients, S3ErrorCode, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use super::retrieval_storage::TestReadStorage; use super::{actor::TestProbe, part, repository_with_probe, session, CompletionPart, MetadataKey, TenantId}; @@ -29,7 +29,7 @@ impl TestCompletion { let chunks = Arc::new(TestReadStorage { queries: 0.into() }); let executor = S3Executor::new( S3StorageClients { - control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + control: Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))), metadata: store.clone(), chunks: Arc::new(chunks.client()), }, diff --git a/lib/crowdb-access-s3/tests/common/multipart_expiry_request.rs b/lib/crowdb-access-s3/tests/common/multipart_expiry_request.rs index 54e9d3294..2f3e886aa 100644 --- a/lib/crowdb-access-s3/tests/common/multipart_expiry_request.rs +++ b/lib/crowdb-access-s3/tests/common/multipart_expiry_request.rs @@ -13,7 +13,7 @@ use crowdb_access_s3::{ storage::S3StorageClients, }; use crowdb_chunk_kv_client::ClientError; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::{chunkdb::rpc::Location, common::ChunkId}; use super::{actor::TestProbe, retrieval_storage::TestReadStorage}; @@ -122,7 +122,7 @@ impl TestExpiry { fn executor(store: Arc<ChunkKvMetadataStore>, chunks: &Arc<TestReadStorage>) -> S3Executor { S3Executor::new( S3StorageClients { - control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + control: Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))), metadata: store, chunks: Arc::new(chunks.client()), }, diff --git a/lib/crowdb-access-s3/tests/multipart_repository/independent.rs b/lib/crowdb-access-s3/tests/multipart_repository/independent.rs index 0216788c2..a5b67babe 100644 --- a/lib/crowdb-access-s3/tests/multipart_repository/independent.rs +++ b/lib/crowdb-access-s3/tests/multipart_repository/independent.rs @@ -226,7 +226,7 @@ async fn distinct_part_numbers_share_no_upload_cas_or_counter() { async fn modern_complete_reads_selected_generations_without_current_pointer_reloads() { use super::retrieval_storage::TestReadStorage; use crowdb_access_s3::{execution::S3Executor, storage::S3StorageClients}; - use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; + use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; let mut session = modern_session(); session.expires_ms = u64::MAX; @@ -238,7 +238,7 @@ async fn modern_complete_reads_selected_generations_without_current_pointer_relo let chunks = Arc::new(TestReadStorage { queries: 0.into() }); let executor = S3Executor::new( S3StorageClients { - control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + control: Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))), metadata: test.store.clone(), chunks: Arc::new(chunks.client()), }, diff --git a/lib/crowdb-chunk-client/Cargo.toml b/lib/crowdb-chunk-client/Cargo.toml index 04d7036c1..7b23e6b4e 100644 --- a/lib/crowdb-chunk-client/Cargo.toml +++ b/lib/crowdb-chunk-client/Cargo.toml @@ -17,7 +17,7 @@ test-util = [] [dependencies] crowdb-chunkdb-client = { path = "../crowdb-chunkdb-client" } crowdb-diskio-client = { path = "../crowdb-diskio-client" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-common = { path = "../crowdb-common/rust" } crowdb-protocol = { path = "../crowdb-protocol" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } diff --git a/lib/crowdb-chunk-client/src/client.rs b/lib/crowdb-chunk-client/src/client.rs index 638b33b13..4414666f0 100644 --- a/lib/crowdb-chunk-client/src/client.rs +++ b/lib/crowdb-chunk-client/src/client.rs @@ -21,7 +21,7 @@ use async_trait::async_trait; use bytes::Bytes; use crowdb_chunkdb_client::ChunkdbClient; use crowdb_common::ec::EcScheme; -use crowdb_kv_client::{CrowdbKvClient, HardwareClient, RangeBindingClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, PaxosKvClient, RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ AdvanceChunkWriteRequest, AdvanceChunkWriteResponse, AllocateChunkRequest, AllocateChunkResponse, AllocateReplacementSegmentRequest, AllocateReplacementSegmentResponse, AppendChunkRequest, @@ -595,7 +595,7 @@ impl ChunkAllocator for MetricsChunkAllocator { async fn discover_current_range_bindings( service: &ServiceRegistryClient, - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, ) -> Result<Option<RangeBindingClient>> { let deadline = Instant::now() + Duration::from_secs(10); let bindings = RangeBindingClient::from_shared(kv); diff --git a/lib/crowdb-chunk-client/src/client/connection.rs b/lib/crowdb-chunk-client/src/client/connection.rs index a9dea1eef..6863e3bc7 100644 --- a/lib/crowdb-chunk-client/src/client/connection.rs +++ b/lib/crowdb-chunk-client/src/client/connection.rs @@ -9,13 +9,13 @@ use crate::negative_list::FailedDiskList; use crate::writer::small_pool::SmallWritePool; use crate::{ChunkIoClient, ChunkIoClientConfig, ChunkReadPolicy, ChunkReader, Result, RoutedDiskWriter}; use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use std::sync::Arc; impl ChunkIoClient { /// Discover services and build lock-free DiskIO routing. pub async fn connect(config: ChunkIoClientConfig) -> Result<Self> { - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( config.management_seeds.clone(), ))); Self::connect_with_kv(config, kv).await @@ -23,14 +23,14 @@ impl ChunkIoClient { /// Discover services using an existing KV topology client shared with the /// embedding process. - pub async fn connect_with_kv(config: ChunkIoClientConfig, kv: Arc<CrowdbKvClient>) -> Result<Self> { + pub async fn connect_with_kv(config: ChunkIoClientConfig, kv: Arc<PaxosKvClient>) -> Result<Self> { Self::connect_with_kv_read_policy(config, kv, ChunkReadPolicy::default()).await } /// Connect with explicit shared read resource limits. pub async fn connect_with_kv_read_policy( config: ChunkIoClientConfig, - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, read_policy: ChunkReadPolicy, ) -> Result<Self> { Self::connect_with_kv_transport(config, kv, read_policy, None).await @@ -39,7 +39,7 @@ impl ChunkIoClient { /// Connect using explicit data transport settings; None retains environment defaults. pub async fn connect_with_kv_transport( config: ChunkIoClientConfig, - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, read_policy: ChunkReadPolicy, rdma: Option<crowdb_rpc_ffi::RdmaConfig>, ) -> Result<Self> { diff --git a/lib/crowdb-chunk-client/src/client/readiness.rs b/lib/crowdb-chunk-client/src/client/readiness.rs index 239ab1ef1..97690723b 100644 --- a/lib/crowdb-chunk-client/src/client/readiness.rs +++ b/lib/crowdb-chunk-client/src/client/readiness.rs @@ -3,7 +3,7 @@ //! Read-only ChunkDB and physical DiskIO evidence through current client routes. -use crowdb_kv_client::DiskRecord; +use crowdb_paxos_kv_client::DiskRecord; use crowdb_protocol::{ common::{DiskId, HwStatus}, diskdb::rpc::Segment, diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index 17ba4af27..428c57ec1 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -11,7 +11,7 @@ use crowdb_diskio_client::{ DiskId, DiskioClient, DiskioClientConfig, DiskioError, Durability, NativeDiskIoRoutes, SegmentTarget, TrafficLane, }; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::diskdb::rpc::Segment; use crate::{DiskWriter, IoError, Result}; diff --git a/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs b/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs index c1822fc8d..4bcb92a03 100644 --- a/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs +++ b/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use bytes::Bytes; use crowdb_chunk_client::{DiskWriter, IoError, Result}; -use crowdb_kv_client::DiskRecord; +use crowdb_paxos_kv_client::DiskRecord; use crowdb_protocol::{ common::{DiskId, HwStatus}, diskdb::rpc::{DiskValue, Segment}, diff --git a/lib/crowdb-chunk-kv-client/Cargo.toml b/lib/crowdb-chunk-kv-client/Cargo.toml index 24a2e2aec..859d0bf38 100644 --- a/lib/crowdb-chunk-kv-client/Cargo.toml +++ b/lib/crowdb-chunk-kv-client/Cargo.toml @@ -14,7 +14,7 @@ workspace = true arc-swap = "1.9" async-trait = "0.1" crowdb-protocol = { path = "../crowdb-protocol" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } bytes = "1" clap = { version = "4", features = ["derive"] } diff --git a/lib/crowdb-chunk-kv-client/src/bin/crowdb-chunk-kv-cli.rs b/lib/crowdb-chunk-kv-client/src/bin/crowdb-chunk-kv-cli.rs index e4ff3fe4a..198e86035 100644 --- a/lib/crowdb-chunk-kv-client/src/bin/crowdb-chunk-kv-cli.rs +++ b/lib/crowdb-chunk-kv-client/src/bin/crowdb-chunk-kv-cli.rs @@ -9,13 +9,13 @@ use clap::{Parser, Subcommand}; use crowdb_chunk_kv_client::{ ChunkKvClient, ChunkKvRpcTransport, ClientConfig, Group0ChunkKvRangeCatalogSource, }; -use crowdb_kv_client::{ClientConfig as KvClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig as KvClientConfig, PaxosKvClient}; use futures::{stream, StreamExt}; #[derive(Debug, Parser)] #[command(name = "crowdb-chunk-kv-cli")] struct Args { - #[arg(long = "mgmt-seed", default_value = "http://127.0.0.1:25000")] + #[arg(long = "mgmt-seed", default_values_t = [format!("http://127.0.0.1:{}", crowdb_protocol::CHUNK_KV_HTTP_BASE)])] mgmt_seeds: Vec<String>, #[command(subcommand)] @@ -105,7 +105,7 @@ async fn main() { } fn production_client(mgmt_seeds: Vec<String>) -> Result<ChunkKvClient, String> { - let kv = Arc::new(CrowdbKvClient::new(KvClientConfig::new(mgmt_seeds))); + let kv = Arc::new(PaxosKvClient::new(KvClientConfig::new(mgmt_seeds))); let config = ClientConfig::default(); let catalog = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(kv)); let transport = Arc::new(ChunkKvRpcTransport::new(config.max_owner_connections, 1, 2)); diff --git a/lib/crowdb-chunk-kv-client/src/catalog/store.rs b/lib/crowdb-chunk-kv-client/src/catalog/store.rs index 4c01a1ab1..7d25b76dc 100644 --- a/lib/crowdb-chunk-kv-client/src/catalog/store.rs +++ b/lib/crowdb-chunk-kv-client/src/catalog/store.rs @@ -8,7 +8,7 @@ use async_trait::async_trait; use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage}; use crowdb_protocol::key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, TextKey}; -use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode}; use crate::{ClientError, Result}; @@ -24,12 +24,12 @@ pub trait ChunkKvRangeCatalogSource: Send + Sync { /// Production catalog source reading immutable pages from group 0. pub struct Group0ChunkKvRangeCatalogSource { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } impl Group0ChunkKvRangeCatalogSource { #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } diff --git a/lib/crowdb-chunk-kv/Cargo.toml b/lib/crowdb-chunk-kv/Cargo.toml index 094711af0..6e6d8dcc3 100644 --- a/lib/crowdb-chunk-kv/Cargo.toml +++ b/lib/crowdb-chunk-kv/Cargo.toml @@ -30,6 +30,7 @@ tokio = { workspace = true, features = ["rt", "sync", "time"] } tracing = { workspace = true } [dev-dependencies] +crowdb-tree-ffi = { path = "../crowdb-tree/ffi", features = ["test-util"] } crowdb-chunk-kv = { path = ".", features = ["test-util"] } crowdb-chunk-stream = { path = "../crowdb-chunk-stream", features = ["test-util"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "test-util"] } diff --git a/lib/crowdb-chunk-kv/src/lib.rs b/lib/crowdb-chunk-kv/src/lib.rs index bbeb5f52b..b6d4832ef 100644 --- a/lib/crowdb-chunk-kv/src/lib.rs +++ b/lib/crowdb-chunk-kv/src/lib.rs @@ -17,10 +17,10 @@ pub use manager::PartitionManager; pub use metrics::{PartitionMetrics, PartitionMetricsSnapshot}; pub use partition::{ decode_frame, encode_frame, CheckpointReclaim, CrowdbPartitionTree, DecodedFrame, FrameDecode, - JournalFrontier, MaterializationProgress, MutationResponse, Partition, PartitionConfig, PartitionJournal, - PartitionSnapshot, PartitionTree, PreparedSplit, PreparedSplitWriter, ScanEntry, ScanPage, - SplitHandoffStore, SplitIngress, SplitWriterTarget, StreamPartitionJournal, TreeObservation, - MAX_FRAME_BYTES, + InstalledSplit, JournalFrontier, MaterializationProgress, MutationResponse, Partition, PartitionConfig, + PartitionJournal, PartitionSnapshot, PartitionTree, PreparedSplit, PreparedSplitOverlay, + PreparedSplitWriter, ScanEntry, ScanPage, SplitHandoffStore, SplitIngress, SplitOverlayLimits, + SplitWriterTarget, StreamPartitionJournal, TreeObservation, MAX_FRAME_BYTES, }; pub use types::{ canonical_operation_digest, Checkpoint, CompareCondition, JournalPosition, MutationOperation, diff --git a/lib/crowdb-chunk-kv/src/memory.rs b/lib/crowdb-chunk-kv/src/memory.rs index 53f744876..5c3f71df8 100644 --- a/lib/crowdb-chunk-kv/src/memory.rs +++ b/lib/crowdb-chunk-kv/src/memory.rs @@ -13,11 +13,15 @@ use tokio::sync::{Mutex, Notify, RwLock}; use crate::{MutationOperation, PartitionTree, Result, ScanEntry, ValueRevision}; +mod prepared_split; +use prepared_split::MemoryValues; + #[derive(Debug)] pub struct MemoryPartitionTree { tree_id: u64, - values: RwLock<BTreeMap<Vec<u8>, ValueRevision>>, - last_applied: AtomicU64, + range: crate::PartitionRange, + values: Arc<RwLock<MemoryValues>>, + last_applied: Arc<AtomicU64>, checkpoint_applied: AtomicU64, fail_next_apply: AtomicBool, rebuild_paused: Arc<AtomicBool>, @@ -28,14 +32,18 @@ pub struct MemoryPartitionTree { split_publish_resume: Arc<Notify>, split_views: Mutex<BTreeMap<u64, BTreeMap<Vec<u8>, ValueRevision>>>, next_split_view: AtomicU64, + collection_backlog: AtomicU64, + collection_calls: AtomicU64, + fail_collection: AtomicBool, } impl Default for MemoryPartitionTree { fn default() -> Self { Self { tree_id: 1, - values: RwLock::default(), - last_applied: AtomicU64::new(0), + range: crate::PartitionRange::default(), + values: Arc::new(RwLock::new(MemoryValues::new(BTreeMap::new()))), + last_applied: Arc::new(AtomicU64::new(0)), checkpoint_applied: AtomicU64::new(u64::MAX), fail_next_apply: AtomicBool::new(false), rebuild_paused: Arc::new(AtomicBool::new(false)), @@ -46,11 +54,28 @@ impl Default for MemoryPartitionTree { split_publish_resume: Arc::new(Notify::new()), split_views: Mutex::default(), next_split_view: AtomicU64::new(0), + collection_backlog: AtomicU64::new(0), + collection_calls: AtomicU64::new(0), + fail_collection: AtomicBool::new(false), } } } impl MemoryPartitionTree { + pub async fn prepared_overlay_footprint_for_tests(&self) -> (usize, u64, bool) { + self.values.read().await.prepared_footprint() + } + + pub fn configure_collection_for_tests(&self, backlog: u64, fail: bool) { + self.collection_backlog.store(backlog, Ordering::Release); + self.fail_collection.store(fail, Ordering::Release); + } + + #[must_use] + pub fn collection_calls_for_tests(&self) -> u64 { + self.collection_calls.load(Ordering::Acquire) + } + #[must_use] pub fn with_tree_id(tree_id: u64) -> Self { Self { @@ -98,6 +123,22 @@ impl MemoryPartitionTree { #[async_trait] impl PartitionTree for MemoryPartitionTree { + async fn collect_storage(&self) -> Result<bool> { + self.collection_calls.fetch_add(1, Ordering::AcqRel); + if self.fail_collection.load(Ordering::Acquire) { + return Err(crate::ChunkKvError::MaintenanceDegraded( + "injected collection failure".into(), + )); + } + let previous = self + .collection_backlog + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + Some(value.saturating_sub(1)) + }) + .expect("collection update always succeeds"); + Ok(previous > 1) + } + fn as_any(&self) -> &dyn std::any::Any { self } @@ -232,20 +273,13 @@ impl PartitionTree for MemoryPartitionTree { return Err(crate::ChunkKvError::ApplyStateUnknown); } let mut values = self.values.write().await; - match operation.successful_value() { - Some(value) => { - values.insert( - operation.key().to_vec(), - ValueRevision { - revision: mutation_seq, - value: value.to_vec(), - }, - ); - } - None => { - values.remove(operation.key()); - } - } + values.apply( + operation.key(), + operation.successful_value().map(|value| ValueRevision { + revision: mutation_seq, + value: value.to_vec(), + }), + ); self.last_applied.store(mutation_seq, Ordering::Release); Ok(()) } @@ -270,8 +304,9 @@ impl PartitionTree for MemoryPartitionTree { let applied = self.last_applied.load(Ordering::Acquire); let snapshot = Self { tree_id: self.tree_id, - values: RwLock::new(values), - last_applied: AtomicU64::new(applied), + range: self.range.clone(), + values: Arc::new(RwLock::new(MemoryValues::new(values))), + last_applied: Arc::new(AtomicU64::new(applied)), checkpoint_applied: AtomicU64::new(applied), fail_next_apply: AtomicBool::new(false), rebuild_paused: Arc::clone(&self.rebuild_paused), @@ -282,6 +317,9 @@ impl PartitionTree for MemoryPartitionTree { split_publish_resume: Arc::clone(&self.split_publish_resume), split_views: Mutex::default(), next_split_view: AtomicU64::new(0), + collection_backlog: AtomicU64::new(0), + collection_calls: AtomicU64::new(0), + fail_collection: AtomicBool::new(false), }; Ok((applied, applied, Arc::new(snapshot))) } @@ -312,8 +350,9 @@ impl PartitionTree for MemoryPartitionTree { }; let rebuilt = Self { tree_id, - values: RwLock::new(filtered), - last_applied: AtomicU64::new(self.last_applied.load(Ordering::Acquire)), + range: range.clone(), + values: Arc::new(RwLock::new(MemoryValues::new(filtered))), + last_applied: Arc::new(AtomicU64::new(self.last_applied.load(Ordering::Acquire))), checkpoint_applied: AtomicU64::new(u64::MAX), fail_next_apply: AtomicBool::new(false), rebuild_paused: Arc::clone(&self.rebuild_paused), @@ -324,6 +363,9 @@ impl PartitionTree for MemoryPartitionTree { split_publish_resume: Arc::clone(&self.split_publish_resume), split_views: Mutex::default(), next_split_view: AtomicU64::new(0), + collection_backlog: AtomicU64::new(0), + collection_calls: AtomicU64::new(0), + fail_collection: AtomicBool::new(false), }; Ok((Arc::new(rebuilt), stats)) } @@ -350,6 +392,14 @@ impl PartitionTree for MemoryPartitionTree { Ok((generation, self.last_applied.load(Ordering::Acquire))) } + async fn arm_split_overlay( + &self, + destination: &dyn PartitionTree, + limits: crate::SplitOverlayLimits, + ) -> Result<(Arc<dyn crate::PreparedSplitOverlay>, u64)> { + prepared_split::arm(self, destination, limits).await + } + async fn install_split_memtable_overlay( &self, source: &dyn PartitionTree, @@ -360,8 +410,11 @@ impl PartitionTree for MemoryPartitionTree { })?; let inherited = source.values.read().await.clone(); let mut values = self.values.write().await; + // The inherited view replaces the older base in this child's range, + // including deletions. Independently newer child revisions still win. + values.retain(|key, value| !self.range.contains(key) || value.revision > journal_frontier); for (key, value) in inherited { - if value.revision <= journal_frontier { + if self.range.contains(&key) && value.revision <= journal_frontier { values.entry(key).or_insert(value); } } @@ -389,14 +442,13 @@ impl PartitionTree for MemoryPartitionTree { let destination = destination.as_any().downcast_ref::<Self>().ok_or_else(|| { crate::ChunkKvError::InvalidRequest("memory split view requires a memory destination".into()) })?; - let view = self - .split_views - .lock() - .await - .get(&generation) - .cloned() - .ok_or_else(|| crate::ChunkKvError::SplitRetry("split memtable view is stale".into()))?; + let view = if let Some(view) = self.split_views.lock().await.get(&generation).cloned() { + view + } else { + self.values.read().await.prepared_view(generation)? + }; let mut values = destination.values.write().await; + values.retain(|key, value| !range.contains(key) || value.revision > journal_frontier); for (key, value) in view .into_iter() .filter(|(key, value)| range.contains(key) && value.revision <= journal_frontier) @@ -419,7 +471,9 @@ impl PartitionTree for MemoryPartitionTree { } async fn release_split_memtable_view(&self, generation: u64) -> Result<()> { - if self.split_views.lock().await.remove(&generation).is_none() { + if self.split_views.lock().await.remove(&generation).is_none() + && !self.values.write().await.release_prepared(generation) + { return Err(crate::ChunkKvError::SplitRetry( "split memtable view is stale".into(), )); diff --git a/lib/crowdb-chunk-kv/src/memory/prepared_split.rs b/lib/crowdb-chunk-kv/src/memory/prepared_split.rs new file mode 100644 index 000000000..de2c0f5b9 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/memory/prepared_split.rs @@ -0,0 +1,291 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Test backend preparation follows writes before its constant-size cutover. + +use super::MemoryPartitionTree; +use crate::{ + ChunkKvError, InstalledSplit, PartitionRange, PartitionTree, PreparedSplitOverlay, Result, + SplitOverlayLimits, ValueRevision, +}; +use async_trait::async_trait; +use std::{ + collections::BTreeMap, + ops::{Deref, DerefMut}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Weak, + }, +}; +use tokio::sync::RwLock; + +type Values = BTreeMap<Vec<u8>, ValueRevision>; +const UNCOMMITTED: u64 = u64::MAX; + +#[derive(Debug)] +pub(super) struct MemoryValues { + current: Arc<Values>, + prepared: BTreeMap<u64, PreparedView>, +} + +#[derive(Debug)] +struct PreparedView { + values: Arc<Values>, + previous: Option<Arc<Values>>, + range: PartitionRange, + committed: Arc<AtomicU64>, + alive: Weak<()>, + changed_bytes: u64, + changed_records: usize, + limits: SplitOverlayLimits, + expired: bool, +} + +impl MemoryValues { + pub(super) fn new(values: Values) -> Self { + Self { + current: Arc::new(values), + prepared: BTreeMap::new(), + } + } + + pub(super) fn apply(&mut self, key: &[u8], value: Option<ValueRevision>) { + for view in self.prepared.values_mut() { + if view.expired + || view.committed.load(Ordering::Acquire) != UNCOMMITTED + || !view.range.contains(key) + { + continue; + } + let bytes = + (key.len() as u64).saturating_add(value.as_ref().map_or(0, |value| value.value.len() as u64)); + if view.changed_records == view.limits.max_records + || bytes > view.limits.max_bytes.saturating_sub(view.changed_bytes) + { + view.expired = true; + continue; + } + view.changed_records += 1; + view.changed_bytes += bytes; + update(Arc::make_mut(&mut view.values), key, value.clone()); + } + update(Arc::make_mut(&mut self.current), key, value); + } + + pub(super) fn prepared_view(&self, generation: u64) -> Result<Values> { + self.prepared + .get(&generation) + .filter(|view| view.committed.load(Ordering::Acquire) != UNCOMMITTED) + .map(|view| (*view.values).clone()) + .ok_or_else(|| ChunkKvError::SplitRetry("prepared memory capture is stale".into())) + } + + pub(super) fn prepared_footprint(&self) -> (usize, u64, bool) { + self.prepared.values().next().map_or((0, 0, false), |view| { + (view.changed_records, view.changed_bytes, view.expired) + }) + } + + pub(super) fn release_prepared(&mut self, generation: u64) -> bool { + self.prepared.remove(&generation).is_some() + } +} + +impl Deref for MemoryValues { + type Target = Values; + fn deref(&self) -> &Self::Target { + &self.current + } +} +impl DerefMut for MemoryValues { + fn deref_mut(&mut self) -> &mut Self::Target { + Arc::make_mut(&mut self.current) + } +} + +fn update(values: &mut Values, key: &[u8], value: Option<ValueRevision>) { + if let Some(value) = value { + values.insert(key.to_vec(), value); + } else { + values.remove(key); + } +} + +struct MemoryPreparedOverlay { + source: Arc<RwLock<MemoryValues>>, + destination: Arc<RwLock<MemoryValues>>, + source_frontier: Arc<AtomicU64>, + destination_frontier: Arc<AtomicU64>, + committed: Arc<AtomicU64>, + _alive: Arc<()>, + generation: u64, + start_frontier: u64, + replayed_prefix: AtomicU64, + limits: SplitOverlayLimits, +} + +pub(super) async fn arm( + source: &MemoryPartitionTree, + destination: &dyn PartitionTree, + limits: SplitOverlayLimits, +) -> Result<(Arc<dyn PreparedSplitOverlay>, u64)> { + let destination = destination + .as_any() + .downcast_ref::<MemoryPartitionTree>() + .ok_or_else(|| { + ChunkKvError::InvalidRequest("memory preparation requires a memory destination".into()) + })?; + if Arc::ptr_eq(&source.values, &destination.values) || limits.max_records == 0 || limits.max_bytes == 0 { + return Err(ChunkKvError::SplitRetry( + "memory preparation has no private replayed prefix".into(), + )); + } + let generation = source + .next_split_view + .fetch_add(1, Ordering::AcqRel) + .checked_add(1) + .ok_or_else(|| ChunkKvError::Faulted("split view generation exhausted".into()))?; + let committed = Arc::new(AtomicU64::new(UNCOMMITTED)); + let alive = Arc::new(()); + if !source.split_views.lock().await.is_empty() { + return Err(ChunkKvError::SplitRetry( + "another memory split capture is active".into(), + )); + } + let mut values = source.values.write().await; + values.prepared.retain(|_, view| { + view.alive.strong_count() != 0 || view.committed.load(Ordering::Acquire) != UNCOMMITTED + }); + if !values.prepared.is_empty() { + return Err(ChunkKvError::SplitRetry( + "another memory split capture is active".into(), + )); + } + let inherited = values + .iter() + .filter(|(key, _)| destination.range.contains(key)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + values.prepared.insert( + generation, + PreparedView { + values: Arc::new(inherited), + previous: None, + range: destination.range.clone(), + committed: committed.clone(), + alive: Arc::downgrade(&alive), + changed_bytes: 0, + changed_records: 0, + limits, + expired: false, + }, + ); + let start_frontier = source.last_applied.load(Ordering::Acquire); + Ok(( + Arc::new(MemoryPreparedOverlay { + source: source.values.clone(), + destination: destination.values.clone(), + source_frontier: source.last_applied.clone(), + destination_frontier: destination.last_applied.clone(), + committed, + _alive: alive, + generation, + start_frontier, + replayed_prefix: AtomicU64::new(UNCOMMITTED), + limits, + }), + start_frontier, + )) +} + +#[async_trait] +impl PreparedSplitOverlay for MemoryPreparedOverlay { + async fn prepare(&self, replayed_prefix: u64) -> Result<()> { + if replayed_prefix < self.start_frontier + || self.destination_frontier.load(Ordering::Acquire) != replayed_prefix + { + return Err(ChunkKvError::SplitRetry( + "memory replay does not cover observer start".into(), + )); + } + match self.replayed_prefix.compare_exchange( + UNCOMMITTED, + replayed_prefix, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => Ok(()), + Err(previous) if previous == replayed_prefix => Ok(()), + Err(_) => Err(ChunkKvError::SplitRetry("memory prepared prefix is fixed".into())), + } + } + + fn try_install(&self, frontier: u64) -> Result<InstalledSplit> { + if let Some(previous) = self.committed_frontier() { + return if previous == frontier { + Ok(InstalledSplit { + generation: self.generation, + frontier, + }) + } else { + Err(ChunkKvError::SplitRetry( + "prepared memory frontier is already fixed".into(), + )) + }; + } + let mut source = self + .source + .try_write() + .map_err(|_| ChunkKvError::SplitRetry("memory source preparation is busy".into()))?; + let mut destination = self + .destination + .try_write() + .map_err(|_| ChunkKvError::SplitRetry("memory destination preparation is busy".into()))?; + // A concurrent installer may have committed after the optimistic + // query and released these locks. Never extend its frozen prefix. + if let Some(previous) = self.committed_frontier() { + return if previous == frontier { + Ok(InstalledSplit { + generation: self.generation, + frontier, + }) + } else { + Err(ChunkKvError::SplitRetry( + "prepared memory frontier is already fixed".into(), + )) + }; + } + let view = source + .prepared + .get_mut(&self.generation) + .ok_or_else(|| ChunkKvError::SplitRetry("prepared memory capture is absent".into()))?; + let replayed_prefix = self.replayed_prefix.load(Ordering::Acquire); + if view.expired + || replayed_prefix == UNCOMMITTED + || frontier == UNCOMMITTED + || frontier < replayed_prefix + || frontier.saturating_sub(self.start_frontier) > self.limits.max_records as u64 + || view.changed_bytes > self.limits.max_bytes + || self.source_frontier.load(Ordering::Acquire) != frontier + || self.destination_frontier.load(Ordering::Acquire) != replayed_prefix + { + return Err(ChunkKvError::SplitRetry( + "prepared memory tail no longer fits its admission".into(), + )); + } + // Retain the old map for background release. Neither replacing the + // route's map nor acknowledging its frontier copies or frees entries. + view.previous = Some(std::mem::replace(&mut destination.current, view.values.clone())); + self.destination_frontier.store(frontier, Ordering::Release); + self.committed.store(frontier, Ordering::Release); + Ok(InstalledSplit { + generation: self.generation, + frontier, + }) + } + + fn committed_frontier(&self) -> Option<u64> { + let value = self.committed.load(Ordering::Acquire); + (value != UNCOMMITTED).then_some(value) + } +} diff --git a/lib/crowdb-chunk-kv/src/partition.rs b/lib/crowdb-chunk-kv/src/partition.rs index c5b9db0c0..f4b0af5b9 100644 --- a/lib/crowdb-chunk-kv/src/partition.rs +++ b/lib/crowdb-chunk-kv/src/partition.rs @@ -6,6 +6,7 @@ //! [`tree`] (ordered storage). mod checkpoint; +mod config; mod drain; mod frame; mod frontier; @@ -19,8 +20,10 @@ mod scan; mod split; mod transfer; mod tree; +mod worker_completion; mod worker_sender; +pub use config::PartitionConfig; pub use frame::{decode_frame, encode_frame, DecodedFrame, FrameDecode, MAX_FRAME_BYTES}; pub use frontier::JournalFrontier; pub use ingress::SplitIngress; @@ -29,7 +32,9 @@ use native::native_storage_parts; pub use observation::TreeObservation; use replay::{replay_suffix, validate_replay_record, ReplayState}; pub use split::{PreparedSplit, PreparedSplitWriter, SplitHandoffStore, SplitWriterTarget}; -pub use tree::{CrowdbPartitionTree, PartitionTree}; +pub use tree::{ + CrowdbPartitionTree, InstalledSplit, PartitionTree, PreparedSplitOverlay, SplitOverlayLimits, +}; use worker_sender::WorkerSender; use std::collections::HashMap; @@ -51,52 +56,6 @@ use crate::{ SplitPlan, TransitionId, ValueRevision, WalRecord, }; -#[derive(Clone, Debug)] -pub struct PartitionConfig { - pub queue_requests: usize, - pub queue_bytes: u64, - pub batch_requests: usize, - pub batch_bytes: u64, - pub max_key_bytes: usize, - pub max_value_bytes: usize, - pub retained_results: usize, - pub metadata_reclaim_records_per_pass: usize, -} - -impl Default for PartitionConfig { - fn default() -> Self { - Self { - queue_requests: 1_024, - queue_bytes: 64 * 1024 * 1024, - batch_requests: 64, - batch_bytes: 4 * 1024 * 1024, - max_key_bytes: 64 * 1024, - max_value_bytes: 16 * 1024 * 1024, - retained_results: 65_536, - metadata_reclaim_records_per_pass: 128, - } - } -} - -impl PartitionConfig { - fn validate(&self) -> Result<()> { - if self.queue_requests == 0 - || self.queue_bytes == 0 - || self.batch_requests == 0 - || self.batch_bytes == 0 - || self.max_key_bytes == 0 - || self.max_value_bytes == 0 - || self.retained_results == 0 - || self.metadata_reclaim_records_per_pass == 0 - { - return Err(ChunkKvError::InvalidRequest( - "partition bounds must be nonzero".into(), - )); - } - Ok(()) - } -} - #[derive(Clone, Debug, PartialEq, Eq)] pub struct MutationResponse { pub mutation_seq: u64, @@ -185,6 +144,7 @@ pub struct Partition { prepared_artifact: Arc<ArcSwapOption<PreparedSplitWriterArtifact>>, inherited_position: Arc<ArcSwapOption<JournalPosition>>, initialization: Shared<BoxFuture<'static, Result<()>>>, + worker_completion: worker_completion::WorkerCompletion, metrics: Arc<PartitionMetrics>, config: Arc<PartitionConfig>, } @@ -200,6 +160,7 @@ struct WorkerState { journal_durable_seq: Arc<AtomicU64>, applied_seq: Arc<AtomicU64>, applied_position: Arc<AtomicU64>, + applied_end: u64, retry_replay_offset: Arc<AtomicU64>, applied_notify: Arc<Notify>, admission_notify: Arc<Notify>, @@ -225,6 +186,7 @@ struct RetainedResult { struct RecoverySeed { applied_seq: u64, applied_position: u64, + applied_end: u64, retry_replay_offset: u64, results: HashMap<RequestId, RetainedResult>, result_order: std::collections::VecDeque<RequestId>, @@ -235,6 +197,7 @@ struct RecoverySeed { struct SplitTransition { plan: SplitPlan, artifact: Option<SplitArtifact>, + inheritance: Option<split::inheritance::ChildInheritance>, } struct PreparedMutation { @@ -278,6 +241,7 @@ impl Partition { RecoverySeed { applied_seq: applied, applied_position: 0, + applied_end: 0, retry_replay_offset: 0, results: HashMap::new(), result_order: std::collections::VecDeque::new(), @@ -800,6 +764,7 @@ impl Partition { journal_durable_seq: Arc::clone(&journal_durable_seq), applied_seq: Arc::clone(&applied_seq), applied_position: Arc::clone(&applied_position), + applied_end: seed.applied_end, retry_replay_offset: Arc::clone(&retry_replay_offset), applied_notify: Arc::clone(&applied_notify), admission_notify: Arc::clone(&admission_notify), @@ -814,7 +779,7 @@ impl Partition { result_order: seed.result_order, expired_floor: seed.expired_floor, }; - tokio::spawn(run_worker(state, receiver)); + let worker_completion = worker_completion::spawn(state, receiver); Ok(Self { id: partition_id, range: Arc::new(ArcSwap::from_pointee(range)), @@ -841,6 +806,7 @@ impl Partition { prepared_artifact, inherited_position, initialization, + worker_completion, metrics, config, }) @@ -1182,7 +1148,11 @@ impl Partition { Ordering::Acquire, ) .map_err(|observed| write_state_error(lifecycle_from_code(observed)))?; - *transition = Some(SplitTransition { plan, artifact: None }); + *transition = Some(SplitTransition { + plan, + artifact: None, + inheritance: None, + }); self.metrics.split_begin(); Ok(()) } @@ -1575,8 +1545,8 @@ impl Partition { Ok(()) } - /// Returns native chunk page-store counters when this partition uses the - /// R140 backend. + /// Returns legacy mutable page-store counters. Runtime-backed Chunk trees + /// reject this schema; use `backing_storage_stats` for physical storage. /// /// # Errors /// @@ -1585,7 +1555,24 @@ impl Partition { self.tree.chunk_stats() } - /// Estimates current retained pack bytes without scanning data or storage metadata. + /// Samples the actual physical backing ledger. Equal scope IDs identify the + /// same ledger and must be deduplicated when aggregating hosted partitions. + /// + /// # Errors + /// Returns a typed Tree error when the backing ledger is unavailable. + pub fn backing_storage_stats(&self) -> Result<Option<crowdb_tree_ffi::BackingStatsSnapshot>> { + self.tree.backing_stats() + } + + /// Samples the Tree admission ledger without resetting periodic windows. + /// # Errors + /// Returns a Tree diagnostic error when the ledger is unavailable. + pub fn tree_resource_usage(&self) -> Result<Option<crowdb_tree_ffi::runtime::ResourceUsageSnapshot>> { + self.tree.resource_usage() + } + + /// Samples allocated physical bytes, including all mirrors and old or staged + /// output that has not yet been physically deleted. /// /// # Errors /// @@ -1824,6 +1811,7 @@ async fn replay_parent_overlay( .await?; buffered.advance(decoded.bytes_consumed); frame_offset += decoded.bytes_consumed as u64; + replay.seed.applied_end = frame_offset; } if buffered.len() > crate::MAX_FRAME_BYTES { return Err(ChunkKvError::JournalCorruption( @@ -1842,6 +1830,7 @@ async fn replay_parent_overlay( )); } replay.seed.applied_position = 0; + replay.seed.applied_end = 0; Ok(replay.seed) } @@ -1854,6 +1843,8 @@ async fn replay_child_overlay( journal: &dyn PartitionJournal, seed: RecoverySeed, ) -> Result<RecoverySeed> { + let mut seed = seed; + seed.applied_end = 0; let mut replay = ReplayState { seed, checkpoint_applied_seq: cutover_seq, @@ -1880,6 +1871,7 @@ async fn replay_child_overlay( replay.process(tree, frame_offset, decoded.record, true).await?; buffered.advance(decoded.bytes_consumed); frame_offset += decoded.bytes_consumed as u64; + replay.seed.applied_end = frame_offset; } if buffered.len() > crate::MAX_FRAME_BYTES { return Err(ChunkKvError::JournalCorruption( @@ -1954,7 +1946,7 @@ async fn run_worker(mut state: WorkerState, mut receiver: mpsc::Receiver<WorkerR continue; } WorkerRequest::ChildSplitCutover(request) => { - split::stable_session::install_child_cutover(&mut state, *request).await; + split::stable_session::install_child_cutover(&mut state, *request); continue; } WorkerRequest::TransferCatchUp(request) => { @@ -2167,6 +2159,20 @@ async fn append_and_apply(state: &mut WorkerState, prepared: Vec<PreparedMutatio return; } }; + let mut previous_end = None; + for (entry, position) in prepared.iter().zip(&positions) { + let end = position.offset.checked_add(entry.frame.len() as u64); + if position.stream_name != state.journal.stream_name() + || end.is_none() + || previous_end.is_some_and(|previous| previous != position.offset) + { + let error = ChunkKvError::JournalCorruption("appended WAL frame positions are invalid".into()); + journal_append_failed(state, &error); + fail_prepared(state, prepared, &error); + return; + } + previous_end = end; + } if let Some(last) = prepared.last() { state .journal_durable_seq @@ -2204,6 +2210,8 @@ async fn append_and_apply(state: &mut WorkerState, prepared: Vec<PreparedMutatio .applied_seq .store(entry.record.mutation_seq, Ordering::Release); state.applied_position.store(position.offset, Ordering::Release); + // Logical frame positions and encoded lengths were checked before applying. + state.applied_end = position.offset + entry.frame.len() as u64; state.applied_notify.notify_waiters(); state.metrics.mutation_result(entry.response.result.applied()); retain_result( diff --git a/lib/crowdb-chunk-kv/src/partition/config.rs b/lib/crowdb-chunk-kv/src/partition/config.rs new file mode 100644 index 000000000..d8b1673d2 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/config.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Partition admission, batching, and retention bounds. + +use crate::{ChunkKvError, Result}; + +#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct PartitionConfig { + pub queue_requests: usize, + pub queue_bytes: u64, + pub batch_requests: usize, + pub batch_bytes: u64, + pub max_key_bytes: usize, + pub max_value_bytes: usize, + pub retained_results: usize, + pub metadata_reclaim_records_per_pass: usize, +} + +impl Default for PartitionConfig { + fn default() -> Self { + Self { + queue_requests: 1_024, + queue_bytes: 64 * 1024 * 1024, + batch_requests: 64, + batch_bytes: 4 * 1024 * 1024, + max_key_bytes: 64 * 1024, + max_value_bytes: 16 * 1024 * 1024, + retained_results: 65_536, + metadata_reclaim_records_per_pass: 128, + } + } +} + +impl PartitionConfig { + /// Validate partition admission and maintenance bounds. + /// + /// # Errors + /// Returns an invalid request error when any capacity is zero. + pub fn validate(&self) -> Result<()> { + if self.queue_requests == 0 + || self.queue_bytes == 0 + || self.batch_requests == 0 + || self.batch_bytes == 0 + || self.max_key_bytes == 0 + || self.max_value_bytes == 0 + || self.retained_results == 0 + || self.metadata_reclaim_records_per_pass == 0 + { + return Err(ChunkKvError::InvalidRequest( + "partition bounds must be nonzero".into(), + )); + } + Ok(()) + } +} diff --git a/lib/crowdb-chunk-kv/src/partition/ingress.rs b/lib/crowdb-chunk-kv/src/partition/ingress.rs index 99dae214b..6eee49e7a 100644 --- a/lib/crowdb-chunk-kv/src/partition/ingress.rs +++ b/lib/crowdb-chunk-kv/src/partition/ingress.rs @@ -32,6 +32,26 @@ impl Partition { /// # Errors /// Rejects a child that does not match the active split plan. pub async fn install_child_split_ingress(&self, plan: &SplitPlan, child: Partition) -> Result<()> { + let ingress = self.prepare_child_split_ingress(plan, child)?; + self.install_split_ingress(ingress.retained_parent, ingress.child) + .await + } + + pub(super) fn prepare_child_split_ingress( + &self, + plan: &SplitPlan, + child: Partition, + ) -> Result<SplitIngress> { + let snapshot = child.snapshot(); + if plan.parent_id != self.id + || snapshot.partition_id != plan.child.partition_id + || snapshot.ownership_epoch != plan.child.ownership_epoch + || snapshot.range != plan.child.range + { + return Err(crate::ChunkKvError::SplitRetry( + "split ingress does not match its prepared plan".into(), + )); + } let mut retained = self.clone(); retained.range = Arc::new(ArcSwap::from_pointee(PartitionRange { start: plan.parent_range.start.clone(), @@ -43,7 +63,11 @@ impl Partition { // The worker owns this ingress. A strong sender here would keep its // channel alive after every external partition handle was dropped. retained.sender = self.sender.borrowed(); - self.install_split_ingress(retained, child).await + Ok(SplitIngress { + split_key: Arc::from(plan.split_key.clone()), + retained_parent: retained, + child, + }) } } diff --git a/lib/crowdb-chunk-kv/src/partition/maintenance.rs b/lib/crowdb-chunk-kv/src/partition/maintenance.rs index 754376d47..a2f7b2526 100644 --- a/lib/crowdb-chunk-kv/src/partition/maintenance.rs +++ b/lib/crowdb-chunk-kv/src/partition/maintenance.rs @@ -52,6 +52,9 @@ impl Partition { let pending = reclaimed > 0; self.metadata_reclaim .store(Some(Arc::new(MetadataReclaim { generation, pending }))); - Ok(pending) + // Keep the two finite cursors independent. A tree error leaves the + // completed WAL metadata watermark intact and does not authorize trim. + let tree_pending = self.tree.collect_storage().await?; + Ok(pending || tree_pending) } } diff --git a/lib/crowdb-chunk-kv/src/partition/native.rs b/lib/crowdb-chunk-kv/src/partition/native.rs index ec71219f2..107b29e7a 100644 --- a/lib/crowdb-chunk-kv/src/partition/native.rs +++ b/lib/crowdb-chunk-kv/src/partition/native.rs @@ -48,6 +48,9 @@ pub(super) async fn open_latest( stream: ChunkStream, ) -> Result<(Arc<dyn PartitionTree>, Arc<dyn PartitionJournal>, Checkpoint)> { tokio::task::spawn_blocking(move || { + let stream_name = stream.stream_name(); + let stream_generation = stream.record_generation(); + let (tree, journal) = native_storage_parts(tree_id, &range, tree_config, page_store.clone(), stream)?; let replay_offset = if page_store.is_chunk_backed() && tree::chunk_root_generation(&page_store)? == 0 { 0 @@ -57,9 +60,6 @@ pub(super) async fn open_latest( _ => ChunkKvError::TreeUnavailable(error.to_string()), })? }; - let stream_name = stream.stream_name(); - let stream_generation = stream.record_generation(); - let (tree, journal) = native_storage_parts(tree_id, &range, tree_config, page_store, stream)?; let (tree_manifest, applied_seq) = tree.checkpoint_state()?; let root_manifest_generation = tree.root_manifest_generation()?; Ok(( diff --git a/lib/crowdb-chunk-kv/src/partition/replay.rs b/lib/crowdb-chunk-kv/src/partition/replay.rs index ace9e3ac0..ce9869404 100644 --- a/lib/crowdb-chunk-kv/src/partition/replay.rs +++ b/lib/crowdb-chunk-kv/src/partition/replay.rs @@ -67,6 +67,7 @@ pub(super) async fn replay_suffix( frame_offset = frame_offset .checked_add(consumed as u64) .ok_or_else(|| ChunkKvError::JournalCorruption("journal frame offset overflows".into()))?; + replay.seed.applied_end = frame_offset; } if buffered.len() > crate::MAX_FRAME_BYTES { return Err(ChunkKvError::JournalCorruption( @@ -86,6 +87,7 @@ impl ReplayState { seed: RecoverySeed { applied_seq: checkpoint.applied_seq, applied_position: checkpoint.replay_offset, + applied_end: checkpoint.replay_offset, retry_replay_offset: checkpoint.replay_offset, results: HashMap::new(), result_order: std::collections::VecDeque::new(), diff --git a/lib/crowdb-chunk-kv/src/partition/split.rs b/lib/crowdb-chunk-kv/src/partition/split.rs index 2093650e1..ee5f274f8 100644 --- a/lib/crowdb-chunk-kv/src/partition/split.rs +++ b/lib/crowdb-chunk-kv/src/partition/split.rs @@ -23,6 +23,8 @@ pub struct SplitWriterTarget { pub journal: Arc<dyn PartitionJournal>, } +pub(super) mod inheritance; +mod journal_proof; pub(super) mod preparation; mod recovery; pub(super) mod stable_session; @@ -105,6 +107,7 @@ impl PreparedSplitWriter { super::RecoverySeed { applied_seq: self.artifact.applied_seq, applied_position: 0, + applied_end: 0, retry_replay_offset: 0, results: std::collections::HashMap::new(), result_order: std::collections::VecDeque::new(), @@ -116,7 +119,7 @@ impl PreparedSplitWriter { ) } - async fn start_live( + fn prepare_live_cutover( &mut self, config: PartitionConfig, split: &SplitArtifact, @@ -130,36 +133,27 @@ impl PreparedSplitWriter { self.parent_journal.as_ref(), )?; config.validate()?; - if self.tree.last_applied_seq() != self.artifact.applied_seq { - return Err(ChunkKvError::TreeCorruption( - "live split writer does not reach its cutover frontier".into(), + if self.journal.tail() != 0 { + return Err(ChunkKvError::SplitRetry( + "a child WAL tail requires recovered split activation".into(), )); } - let mut seed = super::replay_child_overlay( - self.artifact.partition_id, - self.artifact.ownership_epoch, - self.artifact.applied_seq, - config.retained_results, - self.tree.as_ref(), - self.journal.as_ref(), - seed, - ) - .await?; - seed.retry_replay_offset = 0; - let partition = Partition::start( + let child = Partition::start( self.artifact.partition_id, self.artifact.range.clone(), self.artifact.ownership_epoch, config, - Arc::clone(&self.tree), - Arc::clone(&self.journal), + self.tree.clone(), + self.journal.clone(), seed, PartitionLifecycle::Prepared, Some(self.artifact.clone()), )?; - partition.activate_local_split_writer(split)?; - self.live = Some(partition.clone()); - Ok(partition) + // This writer is still private. No route or external handle can send + // work until the prepared native prefix is committed below. + child.activate_local_split_writer(split)?; + self.live = Some(child.clone()); + Ok(child) } } @@ -588,13 +582,27 @@ async fn apply_writer_record( } fn prepare_writer( + writer: &SplitChild, + target: SplitWriterTarget, + tree: Arc<dyn PartitionTree>, + base: (u64, u64, u64), + source: &SplitSourceFrontier, +) -> Result<PreparedSplitWriter> { + prepare_writer_at(writer, target, tree, base, source, source.cutover_seq) +} + +fn prepare_writer_at( writer: &SplitChild, target: SplitWriterTarget, tree: Arc<dyn PartitionTree>, (tree_manifest, root_manifest_generation, base_applied_seq): (u64, u64, u64), source: &SplitSourceFrontier, + replayed_prefix: u64, ) -> Result<PreparedSplitWriter> { - if tree.last_applied_seq() != source.cutover_seq || base_applied_seq > source.cutover_seq { + if tree.last_applied_seq() != replayed_prefix + || base_applied_seq > replayed_prefix + || replayed_prefix > source.cutover_seq + { return Err(ChunkKvError::ApplyStateUnknown); } let artifact = PreparedSplitWriterArtifact { diff --git a/lib/crowdb-chunk-kv/src/partition/split/inheritance.rs b/lib/crowdb-chunk-kv/src/partition/split/inheritance.rs new file mode 100644 index 000000000..81c610e2b --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/split/inheritance.rs @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Retains one native source capture across background publication retries. + +use super::{ChunkKvError, Partition, PartitionTree, Result, SplitArtifact, SplitPlan}; + +pub(in crate::partition) struct ChildInheritance { + generation: Option<u64>, + artifact: SplitArtifact, + published: bool, + journal_published: bool, +} + +impl ChildInheritance { + pub(super) fn installed(artifact: SplitArtifact, installed: crate::InstalledSplit) -> Self { + debug_assert_eq!(artifact.cutover_seq, installed.frontier); + Self { + generation: Some(installed.generation), + artifact, + published: false, + journal_published: false, + } + } + + pub(super) fn replayed(artifact: SplitArtifact) -> Self { + Self { + generation: None, + artifact, + published: true, + journal_published: true, + } + } +} + +impl Partition { + pub(super) async fn retained_child_artifact(&self, plan: &SplitPlan) -> Result<Option<SplitArtifact>> { + let transition = self.split_transition.lock().await; + let active = transition + .as_ref() + .filter(|active| active.plan == *plan) + .ok_or_else(|| ChunkKvError::SplitRetry("split inheritance has no matching transition".into()))?; + Ok(active + .inheritance + .as_ref() + .map(|inheritance| inheritance.artifact.clone())) + } + + pub(super) async fn finish_child_inheritance( + &self, + plan: &SplitPlan, + artifact: &SplitArtifact, + child_tree: &dyn PartitionTree, + ) -> Result<SplitArtifact> { + // This existing control-plane mutex serializes finalizers. Foreground + // reads and mutations use the installed ingress without taking it. + let mut transition = self.split_transition.lock().await; + let active = transition + .as_mut() + .filter(|active| active.plan == *plan) + .ok_or_else(|| ChunkKvError::SplitRetry("split inheritance has no matching transition".into()))?; + let inheritance = active.inheritance.get_or_insert_with(|| ChildInheritance { + generation: None, + artifact: artifact.clone(), + published: false, + journal_published: false, + }); + if inheritance.artifact.cutover_seq != artifact.cutover_seq + || inheritance.artifact.child != artifact.child + { + return Err(ChunkKvError::SplitRetry("split inheritance proof changed".into())); + } + if !inheritance.journal_published { + self.publish_split_journal_prefix(inheritance.artifact.child.parent_cutover_offset) + .await?; + inheritance.journal_published = true; + } + if inheritance.generation.is_none() && !inheritance.published { + let (generation, captured) = self.tree.begin_split_memtable_view().await?; + if captured < artifact.cutover_seq { + self.tree.release_split_memtable_view(generation).await?; + return Err(ChunkKvError::ApplyStateUnknown); + } + inheritance.generation = Some(generation); + } + if let Some(generation) = inheritance.generation { + if !inheritance.published { + self.tree + .publish_split_memtable_view( + generation, + inheritance.artifact.cutover_seq, + child_tree, + &plan.child.range, + ) + .await?; + inheritance.published = true; + } + // A failed publication retains the same source generation. If + // release alone fails, the next attempt only releases that owner. + self.tree.release_split_memtable_view(generation).await?; + inheritance.generation = None; + } + Ok(inheritance.artifact.clone()) + } +} diff --git a/lib/crowdb-chunk-kv/src/partition/split/journal_proof.rs b/lib/crowdb-chunk-kv/src/partition/split/journal_proof.rs new file mode 100644 index 000000000..1d74108cc --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/split/journal_proof.rs @@ -0,0 +1,24 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! WAL publication is a background readiness proof, separate from route installation. + +use crate::{ChunkKvError, Partition, Result}; + +impl Partition { + /// Publish the parent journal prefix required by a split recovery reader. + /// This control-plane operation does not occupy the partition mutation worker. + /// The stream's own ordered checkpoint can still delay its later appends. + /// + /// # Errors + /// Returns publication failures or a journal that does not cover the fixed end. + pub async fn publish_split_journal_prefix(&self, minimum_end: u64) -> Result<u64> { + let published_end = self.journal.checkpoint().await?; + if published_end < minimum_end { + return Err(ChunkKvError::JournalCorruption( + "published parent WAL does not cover the installed split prefix".into(), + )); + } + Ok(published_end) + } +} diff --git a/lib/crowdb-chunk-kv/src/partition/split/preparation.rs b/lib/crowdb-chunk-kv/src/partition/split/preparation.rs index a111d3438..fd2e2934e 100644 --- a/lib/crowdb-chunk-kv/src/partition/split/preparation.rs +++ b/lib/crowdb-chunk-kv/src/partition/split/preparation.rs @@ -104,20 +104,21 @@ impl Partition { // An error can mean the CAS committed but its response was lost. Keep // this exact base and pins; rebuilding would change the durable proof. handoff.commit(&prepared.artifact).await?; - let (artifact, child) = self + let (artifact, child, tail_records) = self .install_child_cutover_session( plan, prepared.target.clone(), prepared.base_checkpoint.clone(), prepared.child_tree.clone(), prepared.child_base, + prepared.artifact.parent_cutover_offset, ) .await?; Ok(PreparedSplit { artifact, child, child_rebuild: prepared.child_rebuild, - delta_records: prepared.delta_records, + delta_records: prepared.delta_records.saturating_add(tail_records), }) } diff --git a/lib/crowdb-chunk-kv/src/partition/split/recovery.rs b/lib/crowdb-chunk-kv/src/partition/split/recovery.rs index 28d21e605..3b8173641 100644 --- a/lib/crowdb-chunk-kv/src/partition/split/recovery.rs +++ b/lib/crowdb-chunk-kv/src/partition/split/recovery.rs @@ -71,10 +71,14 @@ impl Partition { child: child_artifact.clone(), }; super::super::validate_split_artifact(&plan, &artifact, artifact.cutover_seq)?; - child - .tree - .force_advance_split_frontier(artifact.cutover_seq) - .await?; + // Recovery already replayed the complete inherited suffix and the + // child's own WAL. Do not install an unwritten-base overlay over + // those real child writes, or synthesize a missing replay frontier. + if child.snapshot().applied_seq < artifact.cutover_seq + || child.tree.last_applied_seq() < artifact.cutover_seq + { + return Err(ChunkKvError::ApplyStateUnknown); + } child.activate_local_split_writer(&artifact)?; { let mut current = self.split_transition.lock().await; @@ -86,16 +90,10 @@ impl Partition { *current = Some(super::super::SplitTransition { plan: plan.clone(), artifact: None, + inheritance: Some(super::inheritance::ChildInheritance::replayed(artifact.clone())), }); } self.install_child_split_ingress(&plan, child.clone()).await?; - super::stable_session::finish_child_inheritance( - self.tree.as_ref(), - &plan, - &artifact, - child.tree.as_ref(), - ) - .await?; self.lifecycle.store( super::super::lifecycle_code(crate::PartitionLifecycle::SplitFinalizing), std::sync::atomic::Ordering::Release, @@ -152,7 +150,7 @@ impl Partition { tree_config: crowdb_tree_ffi::Config::default(), journal: child.journal.clone(), }; - let (artifact, child) = self + let (artifact, child, delta_records) = self .install_child_cutover_session( plan, target, @@ -163,13 +161,14 @@ impl Partition { proof.root_manifest_generation, proof.base_applied_seq, ), + proof.parent_cutover_offset, ) .await?; Ok(super::PreparedSplit { artifact, child, child_rebuild: crowdb_tree_ffi::RangeRebuildStats::default(), - delta_records: 0, + delta_records, }) } @@ -185,13 +184,14 @@ impl Partition { stream: crowdb_chunk_stream::ChunkStream, parent_stream: crowdb_chunk_stream::ChunkStream, ) -> Result<(Self, PreparedSplitWriterArtifact)> { - let (tree, journal) = super::super::native_storage_parts( + let (tree, journal) = super::super::native::open_storage( artifact.tree_id, - &artifact.range, + artifact.range.clone(), tree_config, page_store, stream, - )?; + ) + .await?; let source = Arc::new(crate::StreamPartitionJournal::new( parent_stream, artifact.parent_stream_name, diff --git a/lib/crowdb-chunk-kv/src/partition/split/stable_session.rs b/lib/crowdb-chunk-kv/src/partition/split/stable_session.rs index 083537066..c62a2e298 100644 --- a/lib/crowdb-chunk-kv/src/partition/split/stable_session.rs +++ b/lib/crowdb-chunk-kv/src/partition/split/stable_session.rs @@ -4,7 +4,7 @@ //! Online child preparation with an unchanged parent tree and journal. use super::{ - prepare_retained_parent, prepare_writer, Checkpoint, ChunkKvError, Partition, PartitionLifecycle, + prepare_retained_parent, prepare_writer_at, Checkpoint, ChunkKvError, Partition, PartitionLifecycle, PartitionTree, PreparedSplit, PreparedSplitWriter, PreparedSplitWriterArtifact, Result, SplitArtifact, SplitPlan, SplitSourceFrontier, SplitWriterTarget, }; @@ -24,6 +24,8 @@ pub(crate) struct ChildSplitCutoverRequest { base_checkpoint: Checkpoint, child_tree: Arc<dyn PartitionTree>, child_base: (u64, u64, u64), + replayed_prefix: u64, + overlay: Arc<dyn crate::PreparedSplitOverlay>, completion: tokio::sync::oneshot::Sender<Result<(SplitArtifact, PreparedSplitWriter)>>, } @@ -41,43 +43,49 @@ impl Partition { .prepared_artifact .load_full() .ok_or_else(|| ChunkKvError::SplitRetry("installed child handoff proof is absent".into()))?; - let (manifest, base_seq) = self.tree.checkpoint_state()?; - let checkpoint = Checkpoint { - tree_id: self.tree_id(), - tree_manifest: manifest, - root_manifest_generation: self.tree.root_manifest_generation()?, - applied_seq: base_seq, - stream_name: self.journal.stream_name(), - stream_generation: self.journal.record_generation(), - replay_offset: proof.parent_replay_offset, - }; - let frontier = SplitSourceFrontier { - parent_id: plan.parent_id, - parent_epoch: plan.parent_epoch, - checkpoint: checkpoint.clone(), - cutover_offset: proof.parent_cutover_offset, - cutover_seq: proof.applied_seq, - journal: self.journal.clone(), - }; - let mut retained = prepare_retained_parent( - &plan, - (manifest, checkpoint.root_manifest_generation, base_seq), - &frontier, - )?; - retained.applied_seq = proof.applied_seq; - retained.child_stream_start_seq = proof.child_stream_start_seq; - let artifact = SplitArtifact { - transition_id: plan.transition_id, - parent_id: plan.parent_id, - parent_epoch: plan.parent_epoch, - parent_next_epoch: plan.parent_next_epoch, - shared_view_generation: 0, - cutover_seq: proof.applied_seq, - retained_parent: retained, - child: (*proof).clone(), + let artifact = if let Some(retained) = self.retained_child_artifact(&plan).await? { + retained + } else { + let (manifest, base_seq) = self.tree.checkpoint_state()?; + let checkpoint = Checkpoint { + tree_id: self.tree_id(), + tree_manifest: manifest, + root_manifest_generation: self.tree.root_manifest_generation()?, + applied_seq: base_seq, + stream_name: self.journal.stream_name(), + stream_generation: self.journal.record_generation(), + replay_offset: proof.parent_replay_offset, + }; + let frontier = SplitSourceFrontier { + parent_id: plan.parent_id, + parent_epoch: plan.parent_epoch, + checkpoint: checkpoint.clone(), + cutover_offset: proof.parent_cutover_offset, + cutover_seq: proof.applied_seq, + journal: self.journal.clone(), + }; + let mut retained = prepare_retained_parent( + &plan, + (manifest, checkpoint.root_manifest_generation, base_seq), + &frontier, + )?; + retained.applied_seq = proof.applied_seq; + retained.child_stream_start_seq = proof.child_stream_start_seq; + SplitArtifact { + transition_id: plan.transition_id, + parent_id: plan.parent_id, + parent_epoch: plan.parent_epoch, + parent_next_epoch: plan.parent_next_epoch, + shared_view_generation: 0, + cutover_seq: proof.applied_seq, + retained_parent: retained, + child: (*proof).clone(), + } }; super::super::validate_split_artifact(&plan, &artifact, proof.applied_seq)?; - finish_child_inheritance(self.tree.as_ref(), &plan, &artifact, child.tree.as_ref()).await?; + let artifact = self + .finish_child_inheritance(&plan, &artifact, child.tree.as_ref()) + .await?; self.record_split_artifact(artifact.clone()).await?; let writer = PreparedSplitWriter { artifact: (*proof).clone(), @@ -110,67 +118,92 @@ impl Partition { base_checkpoint: Checkpoint, child_tree: Arc<dyn PartitionTree>, child_base: (u64, u64, u64), - ) -> Result<(SplitArtifact, PreparedSplitWriter)> { - let (completion, installed) = tokio::sync::oneshot::channel(); - self.sender - .send(super::super::WorkerRequest::ChildSplitCutover(Box::new( - ChildSplitCutoverRequest { - parent: self.clone(), - plan: plan.clone(), - target, - base_checkpoint, - child_tree: Arc::clone(&child_tree), - child_base, - completion, + replay_offset: u64, + ) -> Result<(SplitArtifact, PreparedSplitWriter, u64)> { + let permit = self.sender.reserve().await?; + let mut cursor = super::DeltaCursor { + offset: replay_offset, + applied_seq: child_tree.last_applied_seq(), + delta_records: 0, + }; + let (overlay, start_frontier) = self + .tree + .arm_split_overlay( + child_tree.as_ref(), + crate::SplitOverlayLimits { + max_records: self.config.queue_requests, + max_bytes: self.config.queue_bytes, }, - ))) + ) .await?; + let target_prefix = start_frontier.max(self.applied_seq.load(Ordering::Acquire)); + super::replay_writer_until( + self, + &mut cursor, + target_prefix, + child_tree.as_ref(), + &plan.parent_range, + &plan.child, + ) + .await?; + overlay.prepare(cursor.applied_seq).await?; + let (completion, installed) = tokio::sync::oneshot::channel(); + // Cancellation drops the request's token before its completion sender, + // including while waiting for the host cleanup task to start. + let request = Box::new(ChildSplitCutoverRequest { + parent: self.clone(), + plan: plan.clone(), + target, + base_checkpoint, + child_tree: Arc::clone(&child_tree), + child_base, + replayed_prefix: cursor.applied_seq, + overlay: overlay.clone(), + completion, + }); + let (started, running) = tokio::sync::oneshot::channel(); + // A started blocking task survives caller cancellation and runtime + // shutdown. All native preparation is finished before it occupies a + // host thread, so even a one-thread blocking pool can make progress. + let installed = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + complete_prepared_cutover(overlay, installed) + }); + running.await.map_err(|_| ChunkKvError::WriteStalled)?; + // The observer was registered before replay. A full observer expires + // this installation, without rejecting any parent mutation. + permit.send(super::super::WorkerRequest::ChildSplitCutover(request)); let (artifact, child) = installed.await.map_err(|_| ChunkKvError::WriteStalled)??; - self.pending_child_preparation.store(None); - finish_child_inheritance(self.tree.as_ref(), &plan, &artifact, child_tree.as_ref()).await?; + let artifact = self + .finish_child_inheritance(&plan, &artifact, child_tree.as_ref()) + .await?; self.record_split_artifact(artifact.clone()).await?; self.metrics.split_finalization(); - Ok((artifact, child)) - } -} - -pub(super) async fn finish_child_inheritance( - parent_tree: &dyn PartitionTree, - plan: &SplitPlan, - artifact: &SplitArtifact, - child_tree: &dyn PartitionTree, -) -> Result<()> { - parent_tree - .force_advance_split_frontier(artifact.cutover_seq) - .await?; - let (generation, captured) = parent_tree.begin_split_memtable_view().await?; - if captured < artifact.cutover_seq { - parent_tree.release_split_memtable_view(generation).await?; - return Err(ChunkKvError::ApplyStateUnknown); + Ok((artifact, child, cursor.delta_records)) } - let publication = parent_tree - .publish_split_memtable_view(generation, artifact.cutover_seq, child_tree, &plan.child.range) - .await; - let release = parent_tree.release_split_memtable_view(generation).await; - publication?; - release?; - Ok(()) } -pub(crate) async fn install_child_cutover( +pub(crate) fn install_child_cutover( state: &mut super::super::WorkerState, request: ChildSplitCutoverRequest, ) { - let result = install_child_cutover_inner(state, &request).await; - let _ = request.completion.send(result); + let result = install_child_cutover_inner(state, &request); + let ChildSplitCutoverRequest { + overlay, completion, .. + } = request; + drop(overlay); + let _ = completion.send(result); } -async fn install_child_cutover_inner( +fn install_child_cutover_inner( state: &mut super::super::WorkerState, request: &ChildSplitCutoverRequest, ) -> Result<(SplitArtifact, PreparedSplitWriter)> { let cutover_seq = state.applied_seq.load(Ordering::Acquire); - let cutover_offset = state.journal.checkpoint().await?; + if state.journal_durable_seq.load(Ordering::Acquire) != cutover_seq { + return Err(ChunkKvError::ApplyStateUnknown); + } + let cutover_offset = state.applied_end; let source = SplitSourceFrontier { parent_id: request.plan.parent_id, parent_epoch: request.plan.parent_epoch, @@ -179,16 +212,64 @@ async fn install_child_cutover_inner( cutover_seq, journal: Arc::clone(&request.parent.journal), }; - request - .child_tree - .force_advance_split_frontier(cutover_seq) - .await?; - let mut child = prepare_writer( + let (artifact, mut child) = prepare_cutover_writers(request, &source)?; + let seed = super::super::RecoverySeed { + applied_seq: cutover_seq, + applied_position: 0, + applied_end: 0, + retry_replay_offset: 0, + results: state.results.clone(), + result_order: state.result_order.clone(), + expired_floor: state.expired_floor.clone(), + recovered: false, + }; + let live = child.prepare_live_cutover((*request.parent.config).clone(), &artifact, seed)?; + let ingress = Arc::new(request.parent.prepare_child_split_ingress(&request.plan, live)?); + let retained_artifact = artifact.clone(); + let mut transition = + request.parent.split_transition.try_lock().map_err(|_| { + ChunkKvError::SplitRetry("split transition is busy before route installation".into()) + })?; + let active = transition + .as_mut() + .filter(|active| active.plan == request.plan) + .ok_or_else(|| ChunkKvError::SplitRetry("cutover has no matching prepared transition".into()))?; + if active.inheritance.is_some() || request.parent.split_ingress.load().is_some() { + return Err(ChunkKvError::SplitRetry("cutover is already installed".into())); + } + let installed = request.overlay.try_install(cutover_seq)?; + // Every fallible validation and allocation precedes this commit. The + // private child is now complete, so route publication cannot strand a + // committed native prefix behind a still-writing parent. + active.inheritance = Some(super::inheritance::ChildInheritance::installed( + retained_artifact, + installed, + )); + request.parent.split_ingress.store(Some(ingress)); + // Once the route exists, retries must finalize this installed prefix even + // if the control caller is cancelled before receiving its acknowledgement. + request.parent.pending_child_preparation.store(None); + state.split_ingress = Arc::clone(&request.parent.split_ingress); + state.ownership_epoch = Arc::clone(&request.parent.ownership_epoch); + state.lifecycle = Arc::clone(&request.parent.lifecycle); + state.lifecycle.store( + super::super::lifecycle_code(PartitionLifecycle::SplitFinalizing), + Ordering::Release, + ); + Ok((artifact, child)) +} + +fn prepare_cutover_writers( + request: &ChildSplitCutoverRequest, + source: &SplitSourceFrontier, +) -> Result<(SplitArtifact, PreparedSplitWriter)> { + let child = prepare_writer_at( &request.plan.child, request.target.clone(), Arc::clone(&request.child_tree), request.child_base, - &source, + source, + request.replayed_prefix, )?; let mut retained = prepare_retained_parent( &request.plan, @@ -197,10 +278,11 @@ async fn install_child_cutover_inner( request.base_checkpoint.root_manifest_generation, request.base_checkpoint.applied_seq, ), - &source, + source, )?; - retained.applied_seq = cutover_seq; - retained.child_stream_start_seq = cutover_seq + retained.applied_seq = source.cutover_seq; + retained.child_stream_start_seq = source + .cutover_seq .checked_add(1) .ok_or_else(|| ChunkKvError::InvalidRequest("split cutover sequence overflows".into()))?; let artifact = SplitArtifact { @@ -209,32 +291,20 @@ async fn install_child_cutover_inner( parent_epoch: request.plan.parent_epoch, parent_next_epoch: request.plan.parent_next_epoch, shared_view_generation: 0, - cutover_seq, + cutover_seq: source.cutover_seq, retained_parent: retained, child: child.artifact.clone(), }; - let seed = super::super::RecoverySeed { - applied_seq: cutover_seq, - applied_position: 0, - retry_replay_offset: 0, - results: state.results.clone(), - result_order: state.result_order.clone(), - expired_floor: state.expired_floor.clone(), - recovered: false, - }; - let live = child - .start_live((*request.parent.config).clone(), &artifact, seed) - .await?; - request - .parent - .install_child_split_ingress(&request.plan, live) - .await?; - state.split_ingress = Arc::clone(&request.parent.split_ingress); - state.ownership_epoch = Arc::clone(&request.parent.ownership_epoch); - state.lifecycle = Arc::clone(&request.parent.lifecycle); - state.lifecycle.store( - super::super::lifecycle_code(PartitionLifecycle::SplitFinalizing), - Ordering::Release, - ); Ok((artifact, child)) } + +fn complete_prepared_cutover( + overlay: Arc<dyn crate::PreparedSplitOverlay>, + installed: tokio::sync::oneshot::Receiver<Result<(SplitArtifact, PreparedSplitWriter)>>, +) -> Result<(SplitArtifact, PreparedSplitWriter)> { + let result = installed.blocking_recv().map_err(|_| ChunkKvError::WriteStalled); + // The worker (or a cancelled request) drops its token before completing + // this channel. Native capability destruction therefore stays on the host. + drop(overlay); + result? +} diff --git a/lib/crowdb-chunk-kv/src/partition/transfer.rs b/lib/crowdb-chunk-kv/src/partition/transfer.rs index 989a00730..3dd7fd1e7 100644 --- a/lib/crowdb-chunk-kv/src/partition/transfer.rs +++ b/lib/crowdb-chunk-kv/src/partition/transfer.rs @@ -220,6 +220,7 @@ async fn catch_up_inner(state: &mut super::WorkerState, request: &TransferCatchU let seed = super::RecoverySeed { applied_seq: state.applied_seq.load(Ordering::Acquire), applied_position: 0, + applied_end: 0, retry_replay_offset: state.retry_replay_offset.load(Ordering::Acquire), results: state.results.clone(), result_order: state.result_order.clone(), @@ -264,6 +265,7 @@ async fn catch_up_inner(state: &mut super::WorkerState, request: &TransferCatchU .journal_durable_seq .store(seed.applied_seq, Ordering::Release); state.applied_seq.store(seed.applied_seq, Ordering::Release); + state.applied_end = seed.applied_end; state .applied_position .store(seed.applied_position, Ordering::Release); diff --git a/lib/crowdb-chunk-kv/src/partition/tree.rs b/lib/crowdb-chunk-kv/src/partition/tree.rs index 6dcd17fd0..a6e48c7d4 100644 --- a/lib/crowdb-chunk-kv/src/partition/tree.rs +++ b/lib/crowdb-chunk-kv/src/partition/tree.rs @@ -3,6 +3,10 @@ use std::any::Any; +mod snapshot; +mod split_overlay; +pub use split_overlay::{InstalledSplit, PreparedSplitOverlay, SplitOverlayLimits}; + use async_trait::async_trait; use crate::{ChunkKvError, MutationOperation, Result, ScanEntry, ValueRevision}; @@ -77,6 +81,19 @@ pub trait PartitionTree: Send + Sync { "partition tree does not support split memtable views".into(), )) } + /// Prepare a private replayed destination while the parent keeps serving. + /// + /// # Errors + /// Rejects incompatible trees, insufficient capacity, or an expired prefix. + async fn arm_split_overlay( + &self, + _destination: &dyn PartitionTree, + _limits: SplitOverlayLimits, + ) -> Result<(std::sync::Arc<dyn PreparedSplitOverlay>, u64)> { + Err(ChunkKvError::InvalidRequest( + "partition tree does not support prepared split overlays".into(), + )) + } async fn install_split_memtable_overlay( &self, _source: &dyn PartitionTree, @@ -128,6 +145,13 @@ pub trait PartitionTree: Send + Sync { fn runtime_stats(&self) -> Option<crowdb_tree_ffi::Stats> { None } + /// Samples admission resources without page traversal or backend I/O. + /// Non-runtime trees have no admission resource ledger. + /// # Errors + /// Returns a Tree diagnostic error when its resource ledger is unavailable. + fn resource_usage(&self) -> Result<Option<crowdb_tree_ffi::runtime::ResourceUsageSnapshot>> { + Ok(None) + } fn tree_summary(&self) -> Option<crowdb_tree_ffi::TreeSummary> { None } @@ -144,7 +168,8 @@ pub trait PartitionTree: Send + Sync { "page inspection is unavailable for this tree backend".into(), )) } - /// Estimates retained pack bytes without reading keys, pages or remote metadata. + /// Samples allocated physical bytes, including mirrors, without I/O. + /// Old and staged output remains counted until physical deletion completes. /// /// # Errors /// Returns a storage error when the cached estimate cannot be read. @@ -158,11 +183,20 @@ pub trait PartitionTree: Send + Sync { fn approximate_split_key(&self) -> Result<Option<Vec<u8>>> { Ok(None) } - /// Returns chunk-backend counters, or `None` for another backend. + /// Samples the physical backing ledger, or `None` for another backend. /// /// # Errors /// /// Returns a storage error when native counters cannot be read. + fn backing_stats(&self) -> Result<Option<crowdb_tree_ffi::BackingStatsSnapshot>> { + Ok(None) + } + + /// Returns legacy mutable page-store counters, or `None` for another backend. + /// Runtime implementations reject this obsolete schema. + /// + /// # Errors + /// Returns a typed storage error when the schema is unsupported. fn chunk_stats(&self) -> Result<Option<crowdb_tree_ffi::ChunkPageStoreStats>> { Ok(None) } @@ -199,6 +233,15 @@ pub trait PartitionTree: Send + Sync { fn reclaim_orphans(&self) -> Result<u64> { Ok(0) } + /// Continue one finite physical tree-storage collection pass, without + /// advancing root or WAL retention. Non-staged backends have no work here. + /// + /// # Errors + /// Returns a catalog authority or backend maintenance failure. + async fn collect_storage(&self) -> Result<bool> { + Ok(false) + } + /// Materializes one bounded pass of storage inherited from another tree. /// /// The returned tuple is `(bytes_written, complete)`. Backends without @@ -275,23 +318,14 @@ impl PartitionTree for CrowdbPartitionTree { limit: usize, byte_budget: usize, ) -> Result<(Vec<ScanEntry>, bool)> { - let scan = match (start_key, start_inclusive) { - // A continuation is always exclusive and uses ct_scan's native - // lower-bound contract. - (Some(start_after), false) => self.tree.scan( - b"", - start_after, - end_key.unwrap_or_default(), - limit, - byte_budget, - false, - 0, - false, - ), - (Some(start_key), true) => self.tree.scan_from( + if end_key.is_some_and(<[u8]>::is_empty) { + return Ok((Vec::new(), false)); + } + let scan = match start_key { + Some(start_key) => self.tree.scan_from( b"", start_key, - true, + start_inclusive, end_key.unwrap_or_default(), limit, byte_budget, @@ -299,7 +333,7 @@ impl PartitionTree for CrowdbPartitionTree { 0, false, ), - (None, _) => self.tree.scan( + None => self.tree.scan( b"", b"", end_key.unwrap_or_default(), @@ -334,6 +368,9 @@ impl PartitionTree for CrowdbPartitionTree { limit: usize, bytes: usize, ) -> Result<(Vec<ScanEntry>, bool)> { + if end.is_some_and(<[u8]>::is_empty) { + return Ok((Vec::new(), false)); + } let (entries, truncated) = self .tree .scan_from_bounded( @@ -469,20 +506,20 @@ impl PartitionTree for CrowdbPartitionTree { wal_replay_offset: u64, ) -> Result<(u64, u64, std::sync::Arc<dyn PartitionTree>)> { let (generation, applied_seq) = self.checkpoint(wal_replay_offset).await?; - let config = self.config.clone().ok_or_else(|| { - ChunkKvError::InvalidRequest("native tree was not opened from a retained configuration".into()) - })?; - let tree_id = self.tree_id; - let snapshot = tokio::task::spawn_blocking(move || Self::open(tree_id, &config)) + let tree = self.tree.clone(); + let snapshot = tokio::task::spawn_blocking(move || tree.immutable_snapshot(generation, applied_seq)) .await - .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))??; - let observed = snapshot.tree.snapshot_state().map_err(map_tree_read_error)?; - if observed != (generation, applied_seq) { - return Err(ChunkKvError::TreeCorruption(format!( - "reopened split base is {observed:?}, expected ({generation}, {applied_seq})" - ))); - } - Ok((generation, applied_seq, std::sync::Arc::new(snapshot))) + .map_err(|error| ChunkKvError::MaintenanceDegraded(error.to_string()))? + .map_err(map_tree_read_error)?; + Ok(( + generation, + applied_seq, + std::sync::Arc::new(snapshot::CapturedPartitionTree::new( + self.tree_id, + self.root_manifest_generation()?, + snapshot, + )), + )) } async fn rebuild_range( @@ -535,6 +572,35 @@ impl PartitionTree for CrowdbPartitionTree { .map_err(map_tree_read_error) } + async fn arm_split_overlay( + &self, + destination: &dyn PartitionTree, + limits: SplitOverlayLimits, + ) -> Result<(std::sync::Arc<dyn PreparedSplitOverlay>, u64)> { + let destination = destination.as_any().downcast_ref::<Self>().ok_or_else(|| { + ChunkKvError::InvalidRequest("split preparation destination is not a native tree".into()) + })?; + let source = self.tree.clone(); + let destination = destination.tree.clone(); + let limits = crowdb_tree_ffi::SplitOverlayLimits { + max_records: u64::try_from(limits.max_records).map_err(|_| { + ChunkKvError::InvalidRequest("split request capacity exceeds native limits".into()) + })?, + max_bytes: limits.max_bytes, + }; + let prepared = tokio::task::spawn_blocking(move || source.arm_split_overlay(destination, limits)) + .await + .map_err(|error| ChunkKvError::MaintenanceDegraded(error.to_string()))? + .map_err(map_tree_read_error)?; + let start = prepared.start_frontier(); + Ok(( + std::sync::Arc::new(split_overlay::NativePreparedOverlay(std::sync::Arc::new( + prepared, + ))), + start, + )) + } + async fn install_split_memtable_overlay( &self, source: &dyn PartitionTree, @@ -543,9 +609,14 @@ impl PartitionTree for CrowdbPartitionTree { let source = source.as_any().downcast_ref::<Self>().ok_or_else(|| { ChunkKvError::InvalidRequest("split overlay source is not a native tree".into()) })?; - self.tree - .install_split_memtable_overlay(&source.tree, journal_frontier) - .map_err(map_tree_read_error) + let destination = std::sync::Arc::clone(&self.tree); + let source = std::sync::Arc::clone(&source.tree); + tokio::task::spawn_blocking(move || { + destination.install_split_memtable_overlay(&source, journal_frontier) + }) + .await + .map_err(|error| ChunkKvError::MaintenanceDegraded(error.to_string()))? + .map_err(map_tree_read_error) } async fn clear_split_memtable_overlay(&self, source: &dyn PartitionTree) -> Result<()> { @@ -588,6 +659,18 @@ impl PartitionTree for CrowdbPartitionTree { .map_err(map_tree_read_error) } + async fn collect_storage(&self) -> Result<bool> { + let tree = crowdb_tree_ffi::AsyncCrowdbtree::from_handle(std::sync::Arc::clone(&self.tree)); + let report = match tree.collect_chunk_storage().await { + Ok(report) => report, + // Non-Chunk backends have no Chunk storage to collect. + Err(crowdb_tree_ffi::CtError::NotSupported) => return Ok(false), + Err(error) => return Err(map_tree_read_error(error)), + }; + report.status.map_err(map_tree_read_error)?; + Ok(report.more) + } + fn last_applied_seq(&self) -> u64 { self.tree.stats().contiguous_slot } @@ -595,6 +678,9 @@ impl PartitionTree for CrowdbPartitionTree { fn runtime_stats(&self) -> Option<crowdb_tree_ffi::Stats> { Some(self.tree.stats()) } + fn resource_usage(&self) -> Result<Option<crowdb_tree_ffi::runtime::ResourceUsageSnapshot>> { + self.tree.resource_usage().map(Some).map_err(map_tree_read_error) + } fn tree_summary(&self) -> Option<crowdb_tree_ffi::TreeSummary> { Some(self.tree.tree_summary()) @@ -648,22 +734,29 @@ impl PartitionTree for CrowdbPartitionTree { } fn chunk_stats(&self) -> Result<Option<crowdb_tree_ffi::ChunkPageStoreStats>> { - self.config + if self + .config .as_ref() .and_then(|config| config.page_store.as_ref()) - .filter(|store| store.is_chunk_backed()) - .map(|store| store.chunk_stats().map_err(map_tree_read_error)) - .transpose() + .is_some_and(|store| store.is_chunk_backed()) + { + return Err(map_tree_read_error(crowdb_tree_ffi::CtError::NotSupported)); + } + Ok(None) } - fn estimated_bytes(&self) -> Result<u64> { + fn backing_stats(&self) -> Result<Option<crowdb_tree_ffi::BackingStatsSnapshot>> { self.config .as_ref() .and_then(|config| config.page_store.as_ref()) .filter(|store| store.is_chunk_backed()) - .map_or(Ok(0), |store| { - store.chunk_estimated_bytes().map_err(map_tree_read_error) - }) + .map(|_| self.tree.backing_stats().map_err(map_tree_read_error)) + .transpose() + } + + fn estimated_bytes(&self) -> Result<u64> { + self.backing_stats() + .map(|snapshot| snapshot.map_or(0, |stats| stats.allocated_bytes)) } fn approximate_split_key(&self) -> Result<Option<Vec<u8>>> { diff --git a/lib/crowdb-chunk-kv/src/partition/tree/snapshot.rs b/lib/crowdb-chunk-kv/src/partition/tree/snapshot.rs new file mode 100644 index 000000000..1ccd8b0a3 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/tree/snapshot.rs @@ -0,0 +1,200 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! The split base owns a fixed native mapping, never another backend writer. +use super::{map_tree_read_error, CrowdbPartitionTree, PartitionTree}; +use crate::{ChunkKvError, MutationOperation, PartitionRange, Result, ScanEntry, ValueRevision}; +use async_trait::async_trait; +use crowdb_tree_ffi::{Config, ImmutableTreeSnapshot, RangeRebuildStats, ScanDirection, ScanOptions}; +use std::{any::Any, sync::Arc}; + +pub(super) struct CapturedPartitionTree { + tree_id: u64, + manifest: u64, + snapshot: Arc<ImmutableTreeSnapshot>, +} + +impl CapturedPartitionTree { + pub(super) fn new(tree_id: u64, manifest: u64, snapshot: ImmutableTreeSnapshot) -> Self { + Self { + tree_id, + manifest, + snapshot: Arc::new(snapshot), + } + } + + async fn scan(&self, options: ScanOptions) -> Result<(Vec<ScanEntry>, bool)> { + let snapshot = self.snapshot.clone(); + let (entries, truncated) = tokio::task::spawn_blocking(move || snapshot.scan(&options)) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? + .map_err(map_tree_read_error)?; + Ok(( + entries + .into_iter() + .map(|entry| ScanEntry { + key: entry.key, + value: ValueRevision { + revision: entry.slot, + value: entry.value.to_vec(), + }, + }) + .collect(), + truncated, + )) + } +} + +fn readonly() -> ChunkKvError { + ChunkKvError::InvalidRequest("immutable partition snapshot is read-only".into()) +} + +#[async_trait] +impl PartitionTree for CapturedPartitionTree { + fn as_any(&self) -> &dyn Any { + self + } + fn tree_id(&self) -> u64 { + self.tree_id + } + async fn get(&self, key: &[u8]) -> Result<Option<ValueRevision>> { + let snapshot = self.snapshot.clone(); + let key = key.to_vec(); + tokio::task::spawn_blocking(move || snapshot.get(&key)) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? + .map(|entry| entry.map(|(revision, value)| ValueRevision { revision, value })) + .map_err(map_tree_read_error) + } + async fn scan_forward( + &self, + start: Option<&[u8]>, + inclusive: bool, + end: Option<&[u8]>, + limit: usize, + byte_budget: usize, + ) -> Result<(Vec<ScanEntry>, bool)> { + if end.is_some_and(<[u8]>::is_empty) { + return Ok((Vec::new(), false)); + } + self.scan(ScanOptions { + start: start.map(<[u8]>::to_vec), + start_inclusive: inclusive, + end: end.unwrap_or_default().to_vec(), + limit, + byte_budget, + ..ScanOptions::default() + }) + .await + } + async fn scan_forward_bounded( + &self, + start: Option<&[u8]>, + inclusive: bool, + end: Option<&[u8]>, + limit: usize, + byte_budget: usize, + ) -> Result<(Vec<ScanEntry>, bool)> { + if end.is_some_and(<[u8]>::is_empty) { + return Ok((Vec::new(), false)); + } + self.scan(ScanOptions { + start: start.map(<[u8]>::to_vec), + start_inclusive: inclusive, + end: end.unwrap_or_default().to_vec(), + limit, + byte_budget, + strict_byte_budget: true, + ..ScanOptions::default() + }) + .await + } + async fn seek_reverse( + &self, + start: &[u8], + inclusive: bool, + begin: Option<&[u8]>, + ) -> Result<Option<ScanEntry>> { + let (entries, _) = self + .scan(ScanOptions { + start: Some(start.to_vec()), + start_inclusive: inclusive, + begin: begin.unwrap_or_default().to_vec(), + direction: ScanDirection::Reverse, + limit: 1, + ..ScanOptions::default() + }) + .await?; + Ok(entries.into_iter().next()) + } + async fn scan_reverse( + &self, + start: Option<&[u8]>, + begin: Option<&[u8]>, + limit: usize, + byte_budget: usize, + ) -> Result<(Vec<ScanEntry>, bool)> { + self.scan(ScanOptions { + start: start.map(<[u8]>::to_vec), + start_inclusive: false, + begin: begin.unwrap_or_default().to_vec(), + direction: ScanDirection::Reverse, + limit, + byte_budget, + ..ScanOptions::default() + }) + .await + } + async fn apply(&self, _sequence: u64, _operation: &MutationOperation) -> Result<()> { + Err(readonly()) + } + async fn advance_noop(&self, _sequence: u64) -> Result<()> { + Err(readonly()) + } + async fn checkpoint(&self, _wal: u64) -> Result<(u64, u64)> { + Err(readonly()) + } + async fn checkpoint_snapshot(&self, _wal: u64) -> Result<(u64, u64, Arc<dyn PartitionTree>)> { + Err(readonly()) + } + + async fn rebuild_range( + &self, + tree_id: u64, + range: &PartitionRange, + mut config: Config, + ) -> Result<(Arc<dyn PartitionTree>, RangeRebuildStats)> { + if tree_id == 0 || config.page_store.is_none() { + return Err(ChunkKvError::InvalidRequest( + "snapshot rebuild needs a child tree identity and page store".into(), + )); + } + config.key_range = crowdb_tree_ffi::KeyRange::Bounded { + start: range.start.clone(), + end: range.end.clone(), + }; + let retained_config = config.clone(); + let snapshot = self.snapshot.clone(); + let (tree, stats) = tokio::task::spawn_blocking(move || snapshot.rebuild_range(&config)) + .await + .map_err(|error| ChunkKvError::MaintenanceDegraded(error.to_string()))? + .map_err(map_tree_read_error)?; + Ok(( + Arc::new(CrowdbPartitionTree { + tree_id, + tree: Arc::new(tree), + config: Some(retained_config), + }), + stats, + )) + } + fn checkpoint_state(&self) -> Result<(u64, u64)> { + Ok(self.snapshot.checkpoint_state()) + } + fn root_manifest_generation(&self) -> Result<u64> { + Ok(self.manifest) + } + fn last_applied_seq(&self) -> u64 { + self.snapshot.checkpoint_state().1 + } +} diff --git a/lib/crowdb-chunk-kv/src/partition/tree/split_overlay.rs b/lib/crowdb-chunk-kv/src/partition/tree/split_overlay.rs new file mode 100644 index 000000000..daa48a570 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/tree/split_overlay.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! An admitted source prefix ready for a non-waiting route cutover. + +use crate::{ChunkKvError, Result}; +use async_trait::async_trait; +use std::sync::Arc; + +/// Complete mutation work admitted ahead of the cutover control message. +#[derive(Clone, Copy, Debug)] +pub struct SplitOverlayLimits { + pub max_records: usize, + pub max_bytes: u64, +} + +/// The immutable native capture retained until background publication finishes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InstalledSplit { + pub generation: u64, + pub frontier: u64, +} + +/// Owns all preparation needed before publishing one inherited source prefix. +#[async_trait] +pub trait PreparedSplitOverlay: Send + Sync { + /// Prepare the destination lease after replay covers the registered start. + /// + /// # Errors + /// A changed destination or expired observer leaves the child uninstalled. + async fn prepare(&self, replayed_prefix: u64) -> Result<()>; + + /// Commit the exact completed parent frontier without waiting or allocating. + /// + /// # Errors + /// A stale or exhausted preparation leaves the destination unchanged. + fn try_install(&self, frontier: u64) -> Result<InstalledSplit>; + + /// Returns the sole committed frontier, including after an idempotent retry. + fn committed_frontier(&self) -> Option<u64>; +} + +pub(super) struct NativePreparedOverlay(pub(super) Arc<crowdb_tree_ffi::PreparedSplitOverlay>); + +#[async_trait] +impl PreparedSplitOverlay for NativePreparedOverlay { + async fn prepare(&self, replayed_prefix: u64) -> Result<()> { + let prepared = self.0.clone(); + tokio::task::spawn_blocking(move || prepared.prepare(replayed_prefix)) + .await + .map_err(|error| ChunkKvError::MaintenanceDegraded(error.to_string()))? + .map_err(super::map_tree_read_error) + } + + fn try_install(&self, frontier: u64) -> Result<InstalledSplit> { + let installed = self.0.try_install(frontier).map_err(super::map_tree_read_error)?; + Ok(InstalledSplit { + generation: installed.generation, + frontier: installed.frontier, + }) + } + + fn committed_frontier(&self) -> Option<u64> { + self.0.committed_frontier() + } +} diff --git a/lib/crowdb-chunk-kv/src/partition/worker_completion.rs b/lib/crowdb-chunk-kv/src/partition/worker_completion.rs new file mode 100644 index 000000000..f450f4c07 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/worker_completion.rs @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use futures::{ + future::{BoxFuture, Shared}, + FutureExt, +}; +use tokio::sync::{mpsc, oneshot}; + +use super::{run_worker, Partition, WorkerRequest, WorkerState}; +use crate::{ChunkKvError, Result}; + +pub(super) type WorkerCompletion = Shared<BoxFuture<'static, Result<()>>>; + +pub(super) fn spawn(state: WorkerState, receiver: mpsc::Receiver<WorkerRequest>) -> WorkerCompletion { + let (completed, completion) = oneshot::channel(); + tokio::spawn(async move { + run_worker(state, receiver).await; + let _ = completed.send(()); + }); + async move { completion.await.map_err(|_| ChunkKvError::WriteStalled) } + .boxed() + .shared() +} + +impl Partition { + /// Consumes this handle and waits until the mutation worker releases its + /// Tree and WAL owners. Other owning partition clones must also be dropped. + /// Complete the transfer or shutdown protocol before joining a serving owner; + /// this join does not change fencing, checkpoints, or repair state. + /// + /// # Errors + /// Returns `WriteStalled` if the worker was cancelled or panicked. + pub async fn close(self) -> Result<()> { + let completion = self.worker_completion.clone(); + drop(self); + completion.await + } +} diff --git a/lib/crowdb-chunk-kv/src/partition/worker_sender.rs b/lib/crowdb-chunk-kv/src/partition/worker_sender.rs index cacf68006..bc1d33aff 100644 --- a/lib/crowdb-chunk-kv/src/partition/worker_sender.rs +++ b/lib/crowdb-chunk-kv/src/partition/worker_sender.rs @@ -47,6 +47,13 @@ impl WorkerSender { } } + pub(super) async fn reserve(&self) -> Result<mpsc::OwnedPermit<WorkerRequest>> { + self.upgrade()? + .reserve_owned() + .await + .map_err(|_| ChunkKvError::WriteStalled) + } + pub(super) async fn send(&self, request: WorkerRequest) -> Result<()> { match self { Self::Owned(sender) => sender.send(request).await.map_err(|_| ChunkKvError::WriteStalled), diff --git a/lib/crowdb-chunk-kv/tests/bootstrap_recovery_test.rs b/lib/crowdb-chunk-kv/tests/bootstrap_recovery_test.rs index ac01d4e76..cc025d6a4 100644 --- a/lib/crowdb-chunk-kv/tests/bootstrap_recovery_test.rs +++ b/lib/crowdb-chunk-kv/tests/bootstrap_recovery_test.rs @@ -11,7 +11,7 @@ use crowdb_chunk_stream::{ ChunkStream, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamPurpose, StreamRecord, StreamRecordState, StreamRegistry, }; -use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, PageStore}; +use crowdb_tree_ffi::chunk::test_util::TestChunkBackend; #[tokio::test] async fn interrupted_bootstrap_replays_wal_before_the_first_tree_checkpoint() { @@ -26,19 +26,8 @@ async fn interrupted_bootstrap_replays_wal_before_the_first_tree_checkpoint() { }; let store = Arc::new(MemoryStreamStore::new(4096)); let stream = create_stream(name, &store).await; - let catalog = Arc::new(ChunkRootCatalog::open_memory(4).unwrap()); - let options = ChunkPageStoreOptions { - tree_id: identity.low, - owner_epoch: 4, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - max_chunk_bytes: 0, - mirror_copies: 0, - materialization_bytes_per_pass: 4096, - }; - let pages = Arc::new(PageStore::open_chunk(options, catalog.clone(), None).unwrap()); + let mut backend = TestChunkBackend::new(identity.low, 4).unwrap(); + let pages = backend.page_store(0).unwrap(); let range = PartitionRange { start: Some(Vec::new()), end: None, @@ -49,7 +38,7 @@ async fn interrupted_bootstrap_replays_wal_before_the_first_tree_checkpoint() { 4, PartitionConfig::default(), identity.low, - crowdb_tree_ffi::Config::default(), + backend.config(), pages.clone(), stream, ) @@ -65,17 +54,18 @@ async fn interrupted_bootstrap_replays_wal_before_the_first_tree_checkpoint() { pages.chunk_manifest_generation(), Ok(0) | Err(crowdb_tree_ffi::CtError::NotFound) )); - drop(partition); + partition.close().await.unwrap(); drop(pages); let stream = open_stream(name, &store).await; - let pages = Arc::new(PageStore::open_chunk(options, catalog, None).unwrap()); + backend.next_writer().unwrap(); + let pages = backend.page_store(0).unwrap(); let recovered = Partition::recover_native_latest_prepared_assignment( identity, range, 4, identity.low, PartitionConfig::default(), - crowdb_tree_ffi::Config::default(), + backend.config(), pages.clone(), stream, ) @@ -100,6 +90,8 @@ async fn interrupted_bootstrap_replays_wal_before_the_first_tree_checkpoint() { assert!(recovered.get(4, b"", None).await.unwrap().is_none()); recovered.checkpoint(4).await.unwrap(); assert!(pages.chunk_manifest_generation().unwrap() > 0); + recovered.suspend_for_shutdown(4).await.unwrap(); + recovered.close().await.unwrap(); } } @@ -156,3 +148,30 @@ async fn open_stream(name: StreamName, store: &Arc<MemoryStreamStore>) -> ChunkS .await .unwrap() } + +#[tokio::test] +async fn close_waits_for_the_last_partition_owner_and_releases_the_worker_tree() { + let store = Arc::new(MemoryStreamStore::new(4096)); + let name = StreamName { high: 3, low: 4 }; + let stream = create_stream(name, &store).await; + let tree = Arc::new(crowdb_chunk_kv::memory::MemoryPartitionTree::with_tree_id(7)); + let weak_tree = Arc::downgrade(&tree); + let partition = Partition::open( + PartitionId { high: 3, low: 4 }, + PartitionRange::default(), + 4, + PartitionConfig::default(), + tree, + Arc::new(crowdb_chunk_kv::StreamPartitionJournal::new(stream, name).unwrap()), + ) + .unwrap(); + partition.suspend_for_shutdown(4).await.unwrap(); + let other_owner = partition.clone(); + let completion = partition.close(); + futures::pin_mut!(completion); + assert!(futures::poll!(&mut completion).is_pending()); + assert!(weak_tree.upgrade().is_some()); + drop(other_owner); + completion.await.unwrap(); + assert!(weak_tree.upgrade().is_none()); +} diff --git a/lib/crowdb-chunk-kv/tests/common/root_catalog.rs b/lib/crowdb-chunk-kv/tests/common/root_catalog.rs index 927a0585e..2c31a611c 100644 --- a/lib/crowdb-chunk-kv/tests/common/root_catalog.rs +++ b/lib/crowdb-chunk-kv/tests/common/root_catalog.rs @@ -1,32 +1,19 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use arc_swap::ArcSwap; -use crowdb_tree_ffi::{CtError, RootCatalogObject, RootCatalogStore}; -use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; #[derive(Default)] -pub struct TestRootCatalog { - objects: ArcSwap<BTreeMap<(u64, u8, u64), Vec<u8>>>, - next_reference: AtomicU64, +pub struct TestCatalogLoad { pub armed: AtomicBool, pub entered: tokio::sync::Notify, pub async_progress: AtomicBool, pub observed_progress: AtomicBool, } -impl TestRootCatalog { - fn key(tree_id: u64, object: RootCatalogObject) -> (u64, u8, u64) { - match object { - RootCatalogObject::CurrentManifest => (tree_id, 0, 0), - RootCatalogObject::Manifest(generation) => (tree_id, 1, generation), - RootCatalogObject::ReferenceSegment(id) => (tree_id, 2, id), - } - } - - fn wait_for_async_progress(&self) { +impl TestCatalogLoad { + pub fn wait_for_async_progress(&self) { if !self.armed.swap(false, Ordering::AcqRel) { return; } @@ -39,29 +26,3 @@ impl TestRootCatalog { .store(self.async_progress.load(Ordering::Acquire), Ordering::Release); } } - -impl RootCatalogStore for TestRootCatalog { - fn load(&self, tree_id: u64, object: RootCatalogObject) -> Result<Option<Vec<u8>>, CtError> { - self.wait_for_async_progress(); - Ok(self.objects.load().get(&Self::key(tree_id, object)).cloned()) - } - - fn store(&self, tree_id: u64, object: RootCatalogObject, data: &[u8]) -> Result<(), CtError> { - self.objects.rcu(|previous| { - let mut next = (**previous).clone(); - next.insert(Self::key(tree_id, object), data.to_vec()); - next - }); - Ok(()) - } - - fn publish(&self, tree_id: u64, _: u64, _: u64, generation: u64, manifest: &[u8]) -> Result<(), CtError> { - self.wait_for_async_progress(); - self.store(tree_id, RootCatalogObject::Manifest(generation), manifest)?; - self.store(tree_id, RootCatalogObject::CurrentManifest, manifest) - } - - fn allocate_reference_segment_id(&self, _: u64) -> Result<u64, CtError> { - Ok(self.next_reference.fetch_add(1, Ordering::AcqRel) + 1) - } -} diff --git a/lib/crowdb-chunk-kv/tests/common/split_publish_tree.rs b/lib/crowdb-chunk-kv/tests/common/split_publish_tree.rs new file mode 100644 index 000000000..657e77052 --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/common/split_publish_tree.rs @@ -0,0 +1,349 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv::{ + ChunkKvError, MutationOperation, PartitionRange, PartitionTree, Result, ScanEntry, TransitionId, + ValueRevision, +}; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}, + Arc, OnceLock, +}; + +/// All storage operations use the real native Tree. Only the publication +/// boundary is paused or failed, before transferring that call to the Tree. +pub struct TestPublishTree { + inner: Arc<dyn PartitionTree>, + pub entered: tokio::sync::Notify, + pub release: tokio::sync::Notify, + pause_once: AtomicBool, + fail_publish: AtomicBool, + fail_release: AtomicBool, + generation: AtomicU64, + begins: AtomicU64, + publications: AtomicU64, + releases: AtomicU64, + stage: Arc<AtomicU8>, + preparations: AtomicU64, + installations: Arc<AtomicU64>, + cancel_on_install: Arc<OnceLock<tokio::task::AbortHandle>>, + pause_prepare: AtomicBool, + cleanup: Arc<OnceLock<Arc<TestCleanupGate>>>, + pub prepare_entered: tokio::sync::Notify, + pub prepare_release: tokio::sync::Notify, +} + +impl TestPublishTree { + pub fn new(inner: Arc<dyn PartitionTree>, fail_publish: bool, fail_release: bool) -> Self { + Self { + inner, + entered: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + pause_once: AtomicBool::new(true), + fail_publish: AtomicBool::new(fail_publish), + fail_release: AtomicBool::new(fail_release), + generation: AtomicU64::new(0), + begins: AtomicU64::new(0), + publications: AtomicU64::new(0), + releases: AtomicU64::new(0), + stage: Arc::new(AtomicU8::new(0)), + preparations: AtomicU64::new(0), + installations: Arc::new(AtomicU64::new(0)), + cancel_on_install: Arc::new(OnceLock::new()), + pause_prepare: AtomicBool::new(false), + cleanup: Arc::new(OnceLock::new()), + prepare_entered: tokio::sync::Notify::new(), + prepare_release: tokio::sync::Notify::new(), + } + } + + pub fn stage(&self) -> &'static str { + match self.stage.load(Ordering::Acquire) { + 0 => "before parent snapshot", + 1 => "parent snapshot", + 2 => "after parent snapshot, before native capture", + 3 => "native capture", + 4 => "native capture complete", + 5 => "publication gate", + 6 => "native publication", + 7 => "native publication complete", + 8 => "native release", + 9 => "native release complete", + 10 => "background preparation gate", + 11 => "native preparation", + 12 => "native preparation complete", + 13 => "native token install", + 14 => "native token installed", + _ => "unknown test stage", + } + } + + pub fn cancel_control_after_install(&self, abort: tokio::task::AbortHandle) { + self.cancel_on_install.set(abort).unwrap(); + } + + pub fn pause_cleanup(&self) -> TestCleanupPause { + let gate = Arc::new(TestCleanupGate { + entered: tokio::sync::Notify::new(), + release: tokio::sync::Semaphore::new(0), + }); + assert!(self.cleanup.set(gate.clone()).is_ok()); + TestCleanupPause { gate } + } + + pub fn pause_preparation(&self) { + self.pause_prepare.store(true, Ordering::Release); + } + + pub fn prepared_calls(&self) -> (u64, u64) { + ( + self.preparations.load(Ordering::Acquire), + self.installations.load(Ordering::Acquire), + ) + } + + pub fn calls(&self) -> (u64, u64, u64) { + ( + self.begins.load(Ordering::Acquire), + self.publications.load(Ordering::Acquire), + self.releases.load(Ordering::Acquire), + ) + } +} + +#[async_trait::async_trait] +impl PartitionTree for TestPublishTree { + fn as_any(&self) -> &dyn std::any::Any { + self.inner.as_any() + } + fn tree_id(&self) -> u64 { + self.inner.tree_id() + } + async fn get(&self, key: &[u8]) -> Result<Option<ValueRevision>> { + self.inner.get(key).await + } + async fn scan_forward( + &self, + start: Option<&[u8]>, + inclusive: bool, + end: Option<&[u8]>, + limit: usize, + bytes: usize, + ) -> Result<(Vec<ScanEntry>, bool)> { + self.inner.scan_forward(start, inclusive, end, limit, bytes).await + } + async fn scan_forward_bounded( + &self, + start: Option<&[u8]>, + inclusive: bool, + end: Option<&[u8]>, + limit: usize, + bytes: usize, + ) -> Result<(Vec<ScanEntry>, bool)> { + self.inner + .scan_forward_bounded(start, inclusive, end, limit, bytes) + .await + } + async fn seek_reverse( + &self, + start: &[u8], + inclusive: bool, + begin: Option<&[u8]>, + ) -> Result<Option<ScanEntry>> { + self.inner.seek_reverse(start, inclusive, begin).await + } + async fn scan_reverse( + &self, + start: Option<&[u8]>, + begin: Option<&[u8]>, + limit: usize, + bytes: usize, + ) -> Result<(Vec<ScanEntry>, bool)> { + self.inner.scan_reverse(start, begin, limit, bytes).await + } + async fn apply(&self, sequence: u64, operation: &MutationOperation) -> Result<()> { + self.inner.apply(sequence, operation).await + } + async fn advance_noop(&self, sequence: u64) -> Result<()> { + self.inner.advance_noop(sequence).await + } + async fn checkpoint(&self, offset: u64) -> Result<(u64, u64)> { + self.inner.checkpoint(offset).await + } + async fn checkpoint_snapshot(&self, offset: u64) -> Result<(u64, u64, Arc<dyn PartitionTree>)> { + self.stage.store(1, Ordering::Release); + let snapshot = self.inner.checkpoint_snapshot(offset).await?; + self.stage.store(2, Ordering::Release); + Ok(snapshot) + } + async fn rebuild_range( + &self, + tree: u64, + range: &PartitionRange, + config: crowdb_tree_ffi::Config, + ) -> Result<(Arc<dyn PartitionTree>, crowdb_tree_ffi::RangeRebuildStats)> { + self.inner.rebuild_range(tree, range, config).await + } + async fn force_advance_split_frontier(&self, frontier: u64) -> Result<()> { + self.inner.force_advance_split_frontier(frontier).await + } + async fn begin_split_memtable_view(&self) -> Result<(u64, u64)> { + self.begins.fetch_add(1, Ordering::AcqRel); + self.stage.store(3, Ordering::Release); + let captured = self.inner.begin_split_memtable_view().await?; + self.stage.store(4, Ordering::Release); + Ok(captured) + } + async fn arm_split_overlay( + &self, + destination: &dyn PartitionTree, + limits: crowdb_chunk_kv::SplitOverlayLimits, + ) -> Result<(Arc<dyn crowdb_chunk_kv::PreparedSplitOverlay>, u64)> { + self.preparations.fetch_add(1, Ordering::AcqRel); + if self.pause_prepare.swap(false, Ordering::AcqRel) { + self.stage.store(10, Ordering::Release); + self.prepare_entered.notify_one(); + self.prepare_release.notified().await; + } + self.stage.store(11, Ordering::Release); + let (inner, start) = self.inner.arm_split_overlay(destination, limits).await?; + self.stage.store(12, Ordering::Release); + Ok(( + Arc::new(TestPreparedOverlay { + inner, + stage: self.stage.clone(), + installations: self.installations.clone(), + cancel_on_install: self.cancel_on_install.clone(), + cleanup: self.cleanup.get().cloned(), + }), + start, + )) + } + + async fn install_split_memtable_overlay(&self, source: &dyn PartitionTree, frontier: u64) -> Result<()> { + self.inner.install_split_memtable_overlay(source, frontier).await + } + async fn clear_split_memtable_overlay(&self, source: &dyn PartitionTree) -> Result<()> { + self.inner.clear_split_memtable_overlay(source).await + } + async fn publish_split_memtable_view( + &self, + generation: u64, + frontier: u64, + destination: &dyn PartitionTree, + range: &PartitionRange, + ) -> Result<()> { + self.publications.fetch_add(1, Ordering::AcqRel); + if let Err(previous) = + self.generation + .compare_exchange(0, generation, Ordering::AcqRel, Ordering::Acquire) + { + assert_eq!( + generation, previous, + "retry must retain the exact native source generation" + ); + } + if self.pause_once.swap(false, Ordering::AcqRel) { + self.stage.store(5, Ordering::Release); + self.entered.notify_one(); + self.release.notified().await; + } + if self.fail_publish.swap(false, Ordering::AcqRel) { + return Err(ChunkKvError::SplitRetry("injected publication failure".into())); + } + self.stage.store(6, Ordering::Release); + self.inner + .publish_split_memtable_view(generation, frontier, destination, range) + .await?; + self.stage.store(7, Ordering::Release); + Ok(()) + } + async fn release_split_memtable_view(&self, generation: u64) -> Result<()> { + self.releases.fetch_add(1, Ordering::AcqRel); + if self.fail_release.swap(false, Ordering::AcqRel) { + return Err(ChunkKvError::SplitRetry("injected source release failure".into())); + } + self.stage.store(8, Ordering::Release); + self.inner.release_split_memtable_view(generation).await?; + self.stage.store(9, Ordering::Release); + Ok(()) + } + fn checkpoint_state(&self) -> Result<(u64, u64)> { + self.inner.checkpoint_state() + } + fn root_manifest_generation(&self) -> Result<u64> { + self.inner.root_manifest_generation() + } + fn last_applied_seq(&self) -> u64 { + self.inner.last_applied_seq() + } + fn pin_generation(&self, transition: TransitionId, generation: u64) -> Result<()> { + self.inner.pin_generation(transition, generation) + } + fn unpin_generation(&self, transition: TransitionId) -> Result<()> { + self.inner.unpin_generation(transition) + } +} + +struct TestCleanupGate { + entered: tokio::sync::Notify, + release: tokio::sync::Semaphore, +} + +pub struct TestCleanupPause { + gate: Arc<TestCleanupGate>, +} + +impl TestCleanupPause { + pub async fn entered(&self) { + self.gate.entered.notified().await; + } +} + +impl Drop for TestCleanupPause { + fn drop(&mut self) { + // Also unblock cleanup if a test assertion unwinds. + self.gate.release.add_permits(1); + } +} + +struct TestPreparedOverlay { + inner: Arc<dyn crowdb_chunk_kv::PreparedSplitOverlay>, + stage: Arc<AtomicU8>, + installations: Arc<AtomicU64>, + cancel_on_install: Arc<OnceLock<tokio::task::AbortHandle>>, + cleanup: Option<Arc<TestCleanupGate>>, +} + +impl Drop for TestPreparedOverlay { + fn drop(&mut self) { + if let Some(gate) = &self.cleanup { + gate.entered.notify_one(); + futures::executor::block_on(gate.release.acquire()) + .unwrap() + .forget(); + } + } +} + +#[async_trait::async_trait] +impl crowdb_chunk_kv::PreparedSplitOverlay for TestPreparedOverlay { + async fn prepare(&self, replayed_prefix: u64) -> Result<()> { + self.inner.prepare(replayed_prefix).await + } + + fn try_install(&self, frontier: u64) -> Result<crowdb_chunk_kv::InstalledSplit> { + self.installations.fetch_add(1, Ordering::AcqRel); + self.stage.store(13, Ordering::Release); + let installed = self.inner.try_install(frontier)?; + self.stage.store(14, Ordering::Release); + if let Some(abort) = self.cancel_on_install.get() { + abort.abort(); + } + Ok(installed) + } + + fn committed_frontier(&self) -> Option<u64> { + self.inner.committed_frontier() + } +} diff --git a/lib/crowdb-chunk-kv/tests/history_maintenance_test.rs b/lib/crowdb-chunk-kv/tests/history_maintenance_test.rs index bf5641073..1ca797f31 100644 --- a/lib/crowdb-chunk-kv/tests/history_maintenance_test.rs +++ b/lib/crowdb-chunk-kv/tests/history_maintenance_test.rs @@ -22,6 +22,7 @@ struct TestPartition { store: Arc<MemoryStreamStore>, journal: Arc<dyn PartitionJournal>, config: PartitionConfig, + tree: Arc<MemoryPartitionTree>, } impl TestPartition { @@ -60,6 +61,7 @@ impl TestPartition { store, journal, config, + tree, } } @@ -264,3 +266,42 @@ async fn one_retired_record_per_pass_keeps_same_generation_maintenance_progressi fixture.drain().await; assert!(fixture.store.chunk_records_for_tests(NAME).await.unwrap().len() < before - 2); } + +#[tokio::test] +async fn serving_maintenance_drives_tree_backlog_without_advancing_wal_or_checkpoint() { + let fixture = TestPartition::open().await; + fixture.tree.configure_collection_for_tests(2, false); + let before = fixture.partition.snapshot(); + assert!(fixture.partition.maintain_serving_metadata(EPOCH).await.unwrap()); + assert!(!fixture.partition.maintain_serving_metadata(EPOCH).await.unwrap()); + assert_eq!(fixture.tree.collection_calls_for_tests(), 2); + let after = fixture.partition.snapshot(); + assert_eq!(before.applied_seq, after.applied_seq); + assert_eq!(before.journal_durable_seq, after.journal_durable_seq); + assert_eq!(fixture.partition.metrics().snapshot().checkpoints, 0); +} + +#[tokio::test] +async fn collection_failure_preserves_retry_and_transition_pin_suppresses_collection() { + let fixture = TestPartition::open().await; + fixture.write(1).await; + fixture.tree.configure_collection_for_tests(2, true); + assert!(matches!( + fixture.partition.maintain_serving_metadata(EPOCH).await, + Err(ChunkKvError::MaintenanceDegraded(_)) + )); + fixture.tree.configure_collection_for_tests(2, false); + let transition = TransitionId { high: 70, low: 71 }; + let checkpoint = fixture + .partition + .checkpoint_with_pin(EPOCH, transition) + .await + .unwrap(); + assert_ne!(checkpoint.root_manifest_generation, 0); + assert!(!fixture.partition.maintain_serving_metadata(EPOCH).await.unwrap()); + assert_eq!(fixture.tree.collection_calls_for_tests(), 1); + fixture.partition.release_generation_pin(transition).unwrap(); + assert!(fixture.partition.maintain_serving_metadata(EPOCH).await.unwrap()); + assert!(!fixture.partition.maintain_serving_metadata(EPOCH).await.unwrap()); + assert_eq!(fixture.tree.collection_calls_for_tests(), 3); +} diff --git a/lib/crowdb-chunk-kv/tests/immutable_runtime_split_test.rs b/lib/crowdb-chunk-kv/tests/immutable_runtime_split_test.rs new file mode 100644 index 000000000..a862823a9 --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/immutable_runtime_split_test.rs @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_chunk_kv::{ChunkKvError, CrowdbPartitionTree, MutationOperation, PartitionRange, PartitionTree}; +use crowdb_tree_ffi::{Config, PageStore, TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +fn config(runtime: Arc<TreeRuntime>) -> Config { + Config { + runtime: Some(runtime), + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + frame_bytes: 4096, + max_inline_value: 64, + ..Config::default() + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_split_base_is_read_only_and_rebuilds_after_source_writer_drop() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let tree = CrowdbPartitionTree::open(41, &config(runtime.clone())).unwrap(); + tree.apply( + 1, + &MutationOperation::Put { + key: b"a".to_vec(), + value: b"left".to_vec(), + }, + ) + .await + .unwrap(); + tree.apply( + 2, + &MutationOperation::Put { + key: b"z".to_vec(), + value: vec![b'v'; 9000], + }, + ) + .await + .unwrap(); + let (generation, coverage, snapshot) = tree.checkpoint_snapshot(0).await.unwrap(); + assert_eq!(coverage, 2); + tree.apply( + 3, + &MutationOperation::Put { + key: b"z".to_vec(), + value: b"new".to_vec(), + }, + ) + .await + .unwrap(); + drop(tree); + assert_eq!(snapshot.checkpoint_state().unwrap(), (generation, coverage)); + assert!(matches!( + snapshot.advance_noop(3).await, + Err(ChunkKvError::InvalidRequest(_)) + )); + assert!(matches!( + snapshot + .apply(3, &MutationOperation::Delete { key: b"z".to_vec() }) + .await, + Err(ChunkKvError::InvalidRequest(_)) + )); + assert!(matches!( + snapshot.checkpoint(0).await, + Err(ChunkKvError::InvalidRequest(_)) + )); + assert_eq!(snapshot.get(b"z").await.unwrap().unwrap().value, vec![b'v'; 9000]); + let (entries, _) = snapshot.scan_reverse(None, None, 8, 0).await.unwrap(); + assert_eq!(entries[0].key.as_ref(), b"z"); + let (child, stats) = snapshot + .rebuild_range( + 42, + &PartitionRange { + start: Some(b"m".to_vec()), + end: None, + }, + config(runtime), + ) + .await + .unwrap(); + assert_eq!(child.last_applied_seq(), coverage); + assert_eq!(stats.entries_emitted, 1); + assert_eq!(child.get(b"z").await.unwrap().unwrap().value, vec![b'v'; 9000]); +} diff --git a/lib/crowdb-chunk-kv/tests/native_checkpoint_test.rs b/lib/crowdb-chunk-kv/tests/native_checkpoint_test.rs index dc2c4549c..b6dd15a0e 100644 --- a/lib/crowdb-chunk-kv/tests/native_checkpoint_test.rs +++ b/lib/crowdb-chunk-kv/tests/native_checkpoint_test.rs @@ -2,33 +2,15 @@ // Licensed under the Apache License, Version 2.0. use crowdb_chunk_kv::{CrowdbPartitionTree, MutationOperation, PartitionTree}; -use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, Config, PageStore}; -use std::sync::Arc; +use crowdb_tree_ffi::chunk::test_util::TestChunkBackend; -#[tokio::test] -async fn native_checkpoint_drives_chunk_io_completions_and_publishes_the_exact_frontier() { - let store = Arc::new( - PageStore::open_chunk( - ChunkPageStoreOptions { - tree_id: 91, - owner_epoch: 1, - open_generation: 0, - pack_bytes: 65_501, - iu_size: 1, - max_concurrent_packs: 4, - materialization_bytes_per_pass: 1024 * 1024, - max_chunk_bytes: 1024 * 1024, - mirror_copies: 0, - }, - Arc::new(ChunkRootCatalog::open_memory(1).unwrap()), - None, - ) - .unwrap(), - ); - let config = Config { - page_store: Some(store.clone()), - ..Config::default() - }; +async fn check_native_checkpoint(frame_bytes: u32, reclaim_bytes: usize) { + let mut backend = TestChunkBackend::new(91, 1).unwrap(); + let store = backend.page_store(0).unwrap(); + let mut config = backend.config(); + config.page_store = Some(store.clone()); + config.frame_bytes = frame_bytes; + config.runtime_binding.limits.reclaim_bytes = reclaim_bytes; let tree = CrowdbPartitionTree::open(91, &config).unwrap(); for sequence in 1_u64..=32 { tree.apply( @@ -41,17 +23,24 @@ async fn native_checkpoint_drives_chunk_io_completions_and_publishes_the_exact_f .await .unwrap(); } - assert_eq!(store.chunk_stats().unwrap().completion_wakeups, 0); + let completed_before = backend.runtime.snapshot().unwrap().backend.completed; let (generation, applied) = tree.checkpoint(4096).await.unwrap(); assert!(generation > 0); assert_eq!(applied, 32); assert_eq!(store.wal_replay_offset().unwrap(), 4096); assert!( - store.chunk_stats().unwrap().completion_wakeups > 0, + backend.runtime.snapshot().unwrap().backend.completed > completed_before, "native checkpoints must drive the asynchronous chunk IO completion path" ); - assert!(store.chunk_estimated_bytes().unwrap() >= 2 * 1024 * 1024); + assert!(tree.backing_stats().unwrap().unwrap().allocated_bytes >= 2 * 1024 * 1024); drop(tree); + drop(config); + drop(store); + backend.next_writer().unwrap(); + let mut config = backend.config(); + config.page_store = Some(backend.page_store(0).unwrap()); + config.frame_bytes = frame_bytes; + config.runtime_binding.limits.reclaim_bytes = reclaim_bytes; let recovered = CrowdbPartitionTree::open(91, &config).unwrap(); assert_eq!(recovered.checkpoint_state().unwrap(), (generation, applied)); @@ -64,3 +53,13 @@ async fn native_checkpoint_drives_chunk_io_completions_and_publishes_the_exact_f ); } } + +#[tokio::test] +async fn native_checkpoint_drives_chunk_io_completions_and_publishes_the_exact_frontier() { + check_native_checkpoint(64 * 1024, 64 * 1024 * 1024).await; +} + +#[tokio::test] +async fn small_frames_checkpoint_with_sufficient_host_pin_capacity() { + check_native_checkpoint(4096, 64 * 1024 * 1024).await; +} diff --git a/lib/crowdb-chunk-kv/tests/native_runtime_test.rs b/lib/crowdb-chunk-kv/tests/native_runtime_test.rs index d21253a94..9e6475fb8 100644 --- a/lib/crowdb-chunk-kv/tests/native_runtime_test.rs +++ b/lib/crowdb-chunk-kv/tests/native_runtime_test.rs @@ -10,40 +10,24 @@ use crowdb_chunk_stream::{ memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, StreamRecordState, }; -use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, Config, PageStore}; -use root_catalog::TestRootCatalog; +use crowdb_tree_ffi::chunk::test_util::TestChunkBackend; +use crowdb_tree_ffi::Config; +use root_catalog::TestCatalogLoad; use std::sync::atomic::Ordering; use std::sync::Arc; -fn config(tree_id: u64, catalog: &Arc<ChunkRootCatalog>) -> Config { +fn config(backend: &TestChunkBackend) -> Config { Config { - page_store: Some(Arc::new( - PageStore::open_chunk( - ChunkPageStoreOptions { - tree_id, - owner_epoch: 1, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - materialization_bytes_per_pass: 4096, - max_chunk_bytes: 1024 * 1024, - mirror_copies: 0, - }, - catalog.clone(), - None, - ) - .unwrap(), - )), - ..Config::default() + page_store: Some(backend.page_store(0).unwrap()), + ..backend.config() } } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn range_rebuild_keeps_the_runtime_available_during_catalog_io() { - let backend = Arc::new(TestRootCatalog::default()); - let catalog = Arc::new(ChunkRootCatalog::open_callback(backend.clone()).unwrap()); - let source = CrowdbPartitionTree::open(91, &config(91, &catalog)).unwrap(); + let observed = Arc::new(TestCatalogLoad::default()); + let source_backend = TestChunkBackend::new(91, 1).unwrap(); + let source = CrowdbPartitionTree::open(91, &config(&source_backend)).unwrap(); source .apply( 1, @@ -55,13 +39,18 @@ async fn range_rebuild_keeps_the_runtime_available_during_catalog_io() { .await .unwrap(); source.checkpoint(0).await.unwrap(); - let child_config = config(92, &catalog); - let progress = backend.clone(); + let mut child_backend = TestChunkBackend::new(92, 1).unwrap(); + let hook = observed.clone(); + child_backend + .on_root_load_for_tests(move || hook.wait_for_async_progress()) + .unwrap(); + let child_config = config(&child_backend); + let progress = observed.clone(); let heartbeat = tokio::spawn(async move { progress.entered.notified().await; progress.async_progress.store(true, Ordering::Release); }); - backend.armed.store(true, Ordering::Release); + observed.armed.store(true, Ordering::Release); let (child, _) = tokio::spawn(async move { source .rebuild_range(92, &PartitionRange::default(), child_config) @@ -72,7 +61,7 @@ async fn range_rebuild_keeps_the_runtime_available_during_catalog_io() { .unwrap(); heartbeat.await.unwrap(); assert!( - backend.observed_progress.load(Ordering::Acquire), + observed.observed_progress.load(Ordering::Acquire), "catalog IO blocked the only async worker" ); assert_eq!(child.get(b"key").await.unwrap().unwrap().value, b"value"); @@ -80,9 +69,13 @@ async fn range_rebuild_keeps_the_runtime_available_during_catalog_io() { #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn native_recovery_keeps_the_runtime_available_during_catalog_io() { - let backend = Arc::new(TestRootCatalog::default()); - let catalog = Arc::new(ChunkRootCatalog::open_callback(backend.clone()).unwrap()); - let config = config(93, &catalog); + let observed = Arc::new(TestCatalogLoad::default()); + let mut backend = TestChunkBackend::new(93, 1).unwrap(); + let hook = observed.clone(); + backend + .on_root_load_for_tests(move || hook.wait_for_async_progress()) + .unwrap(); + let config = config(&backend); let pages = config.page_store.clone().unwrap(); let store = Arc::new(MemoryStreamStore::new(4096)); let stream = ChunkStream::create( @@ -104,12 +97,12 @@ async fn native_recovery_keeps_the_runtime_available_during_catalog_io() { ) .await .unwrap(); - let progress = backend.clone(); + let progress = observed.clone(); let heartbeat = tokio::spawn(async move { progress.entered.notified().await; progress.async_progress.store(true, Ordering::Release); }); - backend.armed.store(true, Ordering::Release); + observed.armed.store(true, Ordering::Release); let recovered = tokio::spawn(async move { Partition::recover_native_latest_prepared_assignment( PartitionId { high: 1, low: 93 }, @@ -128,8 +121,9 @@ async fn native_recovery_keeps_the_runtime_available_during_catalog_io() { .unwrap(); heartbeat.await.unwrap(); assert!( - backend.observed_progress.load(Ordering::Acquire), + observed.observed_progress.load(Ordering::Acquire), "native recovery blocked the only async worker" ); assert_eq!(recovered.snapshot().applied_seq, 0); + recovered.close().await.unwrap(); } diff --git a/lib/crowdb-chunk-kv/tests/native_split_handoff_test.rs b/lib/crowdb-chunk-kv/tests/native_split_handoff_test.rs index 20ee17860..7f20597e5 100644 --- a/lib/crowdb-chunk-kv/tests/native_split_handoff_test.rs +++ b/lib/crowdb-chunk-kv/tests/native_split_handoff_test.rs @@ -4,6 +4,9 @@ #[path = "common/split_storage.rs"] mod split_storage; +#[path = "common/split_publish_tree.rs"] +mod split_publish_tree; + use crowdb_chunk_kv::{ CrowdbPartitionTree, MutationOperation, PartitionId, PartitionRange, RequestId, SplitChild, SplitHandoffStore, SplitPlan, SplitWriterTarget, TransitionId, @@ -48,8 +51,9 @@ impl SplitHandoffStore for TestCommittedHandoff { } } -fn native_config() -> crowdb_tree_ffi::Config { +fn native_config(runtime: &Arc<crowdb_tree_ffi::TreeRuntime>) -> crowdb_tree_ffi::Config { crowdb_tree_ffi::Config { + runtime: Some(runtime.clone()), page_store: Some(Arc::new(crowdb_tree_ffi::PageStore::open_mem(65_536).unwrap())), ..crowdb_tree_ffi::Config::default() } @@ -57,10 +61,12 @@ fn native_config() -> crowdb_tree_ffi::Config { #[tokio::test] async fn consecutive_native_splits_preserve_every_original_record() { + let runtime = + Arc::new(crowdb_tree_ffi::TreeRuntime::new(crowdb_tree_ffi::TreeRuntimeOptions::default()).unwrap()); let storage = TestSplitStorage { store: Arc::new(crowdb_chunk_stream::memory::MemoryStreamStore::new(128 * 1024)), }; - let tree = Arc::new(CrowdbPartitionTree::open(80, &native_config()).unwrap()); + let tree = Arc::new(CrowdbPartitionTree::open(80, &native_config(&runtime)).unwrap()); let (parent, _) = storage .partition( PartitionId { high: 80, low: 1 }, @@ -97,7 +103,7 @@ async fn consecutive_native_splits_preserve_every_original_record() { }; let target = SplitWriterTarget { tree_id: 80 + round, - tree_config: native_config(), + tree_config: native_config(&runtime), journal: storage .journal(crowdb_chunk_stream::StreamName { high: 81, low: round }, 1) .await, @@ -159,20 +165,50 @@ async fn verify_native_records(partitions: &[crowdb_chunk_kv::Partition]) { #[tokio::test] async fn native_child_inherits_writes_accepted_while_handoff_status_is_pending() { + native_child_publication(false, false).await; +} + +#[tokio::test] +async fn native_publication_retry_retains_capture_and_new_child_writes() { + native_child_publication(true, false).await; +} + +#[tokio::test] +async fn native_release_retry_does_not_republish_after_child_writes() { + native_child_publication(false, true).await; +} + +struct TestNativeSplit { + store: Arc<crowdb_chunk_stream::memory::MemoryStreamStore>, + parent: crowdb_chunk_kv::Partition, + publication: Arc<split_publish_tree::TestPublishTree>, + gate: Arc<TestNativeHandoffGate>, + plan: SplitPlan, + preparing: tokio::task::JoinHandle<crowdb_chunk_kv::Result<crowdb_chunk_kv::PreparedSplit>>, +} + +async fn setup_native_publication( + fail_publish: bool, + fail_release: bool, + pause_prepare: bool, +) -> TestNativeSplit { + // Each independent test has the default host budget. The parent and its + // children share that host without competing with another test's workload. + let runtime = + Arc::new(crowdb_tree_ffi::TreeRuntime::new(crowdb_tree_ffi::TreeRuntimeOptions::default()).unwrap()); let storage = TestSplitStorage::new(); let parent_id = PartitionId { high: 60, low: 1 }; - let parent_tree = Arc::new( - CrowdbPartitionTree::open( - 60, - &crowdb_tree_ffi::Config { - page_store: Some(Arc::new(crowdb_tree_ffi::PageStore::open_mem(65_536).unwrap())), - ..crowdb_tree_ffi::Config::default() - }, - ) - .unwrap(), - ); + let parent_tree = Arc::new(CrowdbPartitionTree::open(60, &native_config(&runtime)).unwrap()); + let publication = Arc::new(split_publish_tree::TestPublishTree::new( + parent_tree, + fail_publish, + fail_release, + )); + if pause_prepare { + publication.pause_preparation(); + } let (parent, _) = storage - .partition(parent_id, PartitionRange::default(), 1, parent_tree) + .partition(parent_id, PartitionRange::default(), 1, publication.clone()) .await; let (request, operation) = put(1, b"n", b"base"); parent.mutate(1, request, operation).await.unwrap(); @@ -194,10 +230,7 @@ async fn native_child_inherits_writes_accepted_while_handoff_status_is_pending() }; let target = SplitWriterTarget { tree_id: 61, - tree_config: crowdb_tree_ffi::Config { - page_store: Some(Arc::new(crowdb_tree_ffi::PageStore::open_mem(65_536).unwrap())), - ..crowdb_tree_ffi::Config::default() - }, + tree_config: native_config(&runtime), journal: storage .journal(crowdb_chunk_stream::StreamName { high: 61, low: 1 }, 1) .await, @@ -207,13 +240,40 @@ async fn native_child_inherits_writes_accepted_while_handoff_status_is_pending() release: tokio::sync::Notify::new(), }); let source = parent.clone(); + let retry_plan = plan.clone(); let waiting_gate = gate.clone(); let preparing = tokio::spawn(async move { source .prepare_split_child_session(plan, target, 8, waiting_gate) .await }); - gate.entered.notified().await; + TestNativeSplit { + store: storage.store, + parent, + publication, + gate, + plan: retry_plan, + preparing, + } +} + +async fn native_child_publication(fail_publish: bool, fail_release: bool) { + let TestNativeSplit { + parent, + publication, + gate, + plan: retry_plan, + mut preparing, + .. + } = setup_native_publication(fail_publish, fail_release, false).await; + tokio::select! { + () = gate.entered.notified() => {}, + result = &mut preparing => match result { + Ok(Err(error)) => panic!("native split failed before reaching the handoff gate: {error}; stage={}; calls={:?}", publication.stage(), publication.calls()), + Err(error) => panic!("native split task failed before reaching the handoff gate: {error}; stage={}; calls={:?}", publication.stage(), publication.calls()), + Ok(Ok(_)) => panic!("native split completed without entering the handoff gate"), + }, + } tokio::time::timeout(std::time::Duration::from_secs(3), async { for (sequence, key) in [(2, b"b"), (3, b"n"), (4, b"o")] { let (request, operation) = put(sequence, key, b"during-status"); @@ -227,24 +287,358 @@ async fn native_child_inherits_writes_accepted_while_handoff_status_is_pending() .await .unwrap(); gate.release.notify_one(); - let prepared = preparing.await.unwrap().unwrap(); + tokio::select! { + () = publication.entered.notified() => {}, + result = &mut preparing => match result { + Ok(Err(error)) => panic!("split failed before inherited publication: {error}; stage={}; calls={:?}", publication.stage(), publication.calls()), + Err(error) => panic!("split task failed before inherited publication: {error}; stage={}; calls={:?}", publication.stage(), publication.calls()), + Ok(Ok(_)) => panic!("split completed without the publication gate"), + }, + } + for key in [b"n", b"o"] { + assert_eq!( + parent.get(1, key, None).await.unwrap().unwrap().value, + b"during-status", + "the child must read the inherited update while publication is paused" + ); + } + let (request, operation) = put(5, b"n", b"child-own-wal"); + let written = parent.mutate(1, request, operation).await.unwrap(); + let child_stream = parent.split_ingress().unwrap().child().snapshot().stream_name; + assert_eq!(written.journal_position.stream_name, child_stream); + publication.release.notify_one(); + verify_native_publication( + &parent, + &publication, + retry_plan, + preparing.await.unwrap(), + fail_publish, + fail_release, + ) + .await; +} + +async fn verify_native_publication( + parent: &crowdb_chunk_kv::Partition, + publication: &split_publish_tree::TestPublishTree, + retry_plan: SplitPlan, + attempt: crowdb_chunk_kv::Result<crowdb_chunk_kv::PreparedSplit>, + fail_publish: bool, + fail_release: bool, +) { + let prepared = if fail_publish || fail_release { + assert!(matches!( + attempt, + Err(crowdb_chunk_kv::ChunkKvError::SplitRetry(_)) + )); + for (sequence, key, value) in [ + (6, b"o", b"after-error".as_slice()), + (7, b"b", b"left-after-error".as_slice()), + ] { + let (request, operation) = put(sequence, key, value); + parent.mutate(1, request, operation).await.unwrap(); + } + parent + .finalize_installed_split_child_session(retry_plan.clone()) + .await + .unwrap() + } else { + attempt.unwrap() + }; assert_eq!(prepared.artifact.cutover_seq, 4); assert_eq!(prepared.artifact.retained_parent.tree_id, 60); let child = prepared .child .open_warmed(crowdb_chunk_kv::PartitionConfig::default()) .unwrap(); + assert_eq!( + child.get(1, b"n", None).await.unwrap().unwrap().value, + b"child-own-wal" + ); + assert_eq!( + child.get(1, b"o", None).await.unwrap().unwrap().value, + if fail_publish || fail_release { + b"after-error".as_slice() + } else { + b"during-status".as_slice() + } + ); + assert_eq!( + parent.get(1, b"b", None).await.unwrap().unwrap().value, + if fail_publish || fail_release { + b"left-after-error".as_slice() + } else { + b"during-status".as_slice() + } + ); + assert_eq!( + publication.calls(), + ( + 0, + if fail_publish { 2 } else { 1 }, + if fail_release { 2 } else { 1 } + ) + ); + assert_eq!(publication.prepared_calls(), (1, 1)); + let repeated = parent + .finalize_installed_split_child_session(retry_plan) + .await + .unwrap(); + assert_eq!(repeated.artifact, prepared.artifact); + assert_eq!( + publication.calls(), + ( + 0, + if fail_publish { 2 } else { 1 }, + if fail_release { 2 } else { 1 } + ) + ); +} + +#[tokio::test] +async fn native_overlay_preparation_keeps_parent_mutation_worker_serving() { + let TestNativeSplit { + parent, + publication, + gate, + plan: _, + mut preparing, + .. + } = setup_native_publication(false, false, true).await; + tokio::select! { + () = gate.entered.notified() => {}, + result = &mut preparing => panic!("failed before handoff: {}; stage={}; calls={:?}", preparation_failure(result), publication.stage(), publication.calls()), + } + gate.release.notify_one(); + tokio::select! { + () = publication.prepare_entered.notified() => {}, + result = &mut preparing => panic!("failed before preparation: {}; stage={}; calls={:?}", preparation_failure(result), publication.stage(), publication.calls()), + } + let old_stream = parent.snapshot().stream_name; + tokio::time::timeout(std::time::Duration::from_secs(3), async { + for (sequence, key) in [(2, b"n"), (3, b"o")] { + let (request, operation) = put(sequence, key, b"during-prepare"); + let response = parent.mutate(1, request, operation).await.unwrap(); + assert_eq!(response.journal_position.stream_name, old_stream); + assert_eq!( + parent.get(1, key, None).await.unwrap().unwrap().value, + b"during-prepare" + ); + } + }) + .await + .unwrap(); + publication.prepare_release.notify_one(); + tokio::select! { + () = publication.entered.notified() => {}, + result = &mut preparing => panic!("failed before publication: {}; stage={}; calls={:?}", preparation_failure(result), publication.stage(), publication.calls()), + } for key in [b"n", b"o"] { assert_eq!( - child.get(1, key, None).await.unwrap().unwrap().value, - b"during-status" + parent.get(1, key, None).await.unwrap().unwrap().value, + b"during-prepare" ); } - let (request, operation) = put(5, b"n", b"child-own-wal"); - let written = parent.mutate(1, request, operation).await.unwrap(); - assert_eq!(written.journal_position.stream_name, child.snapshot().stream_name); + let (request, operation) = put(4, b"n", b"child-own-wal"); + let response = parent.mutate(1, request, operation).await.unwrap(); + assert_ne!(response.journal_position.stream_name, old_stream); + publication.release.notify_one(); + let prepared = preparing.await.unwrap().unwrap(); + assert_eq!(prepared.artifact.cutover_seq, 3); + assert_eq!(prepared.artifact.child.base_applied_seq, 1); + assert_eq!(publication.prepared_calls(), (1, 1)); + assert_eq!(publication.calls(), (0, 1, 1)); + let child = prepared + .child + .open_warmed(crowdb_chunk_kv::PartitionConfig::default()) + .unwrap(); + assert_eq!( + child.get(1, b"n", None).await.unwrap().unwrap().value, + b"child-own-wal" + ); + assert_eq!( + child.get(1, b"o", None).await.unwrap().unwrap().value, + b"during-prepare" + ); +} + +fn preparation_failure( + result: Result<crowdb_chunk_kv::Result<crowdb_chunk_kv::PreparedSplit>, tokio::task::JoinError>, +) -> String { + match result { + Ok(Err(error)) => error.to_string(), + Err(error) => error.to_string(), + Ok(Ok(_)) => "completed without reaching its gate".into(), + } +} + +#[tokio::test] +async fn native_route_and_child_wal_do_not_wait_for_parent_journal_publication() { + let TestNativeSplit { + store, + parent, + publication, + gate, + mut preparing, + .. + } = setup_native_publication(false, false, false).await; + tokio::select! { + () = gate.entered.notified() => {}, + result = &mut preparing => panic!("failed before durable handoff: {}", preparation_failure(result)), + } + let (request, operation) = put(2, b"o", b"synced-before-route"); + parent.mutate(1, request, operation).await.unwrap(); + let end = parent.snapshot().journal_durable_offset; + store.pause_publication_for_tests(); + gate.release.notify_one(); + tokio::select! { + () = store.wait_for_publication_for_tests() => {}, + result = &mut preparing => panic!("failed before parent WAL proof: {}; stage={}", preparation_failure(result), publication.stage()), + } + assert!(!preparing.is_finished()); + assert_eq!(publication.calls(), (0, 0, 0)); + assert_eq!(publication.prepared_calls(), (1, 1)); + let ingress = parent + .split_ingress() + .expect("route must precede WAL publication proof"); + let child = ingress.child(); + assert_eq!( + child.get(1, b"o", None).await.unwrap().unwrap().value, + b"synced-before-route" + ); + let (request, operation) = put(3, b"n", b"independent-child-wal"); + let response = tokio::time::timeout( + std::time::Duration::from_secs(3), + parent.mutate(1, request, operation), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.mutation_seq, 3); + assert_eq!( + response.journal_position.stream_name, + child.snapshot().stream_name + ); + // ChunkStream serializes its own checkpoint with later parent appends. + // This case proves child routing, not progress through that WAL barrier. + assert!(!preparing.is_finished()); + store.resume_publication_for_tests(); + tokio::select! { + () = publication.entered.notified() => {}, + result = &mut preparing => panic!("failed before inherited publication: {}", preparation_failure(result)), + } + publication.release.notify_one(); + let prepared = preparing.await.unwrap().unwrap(); + assert_eq!(prepared.artifact.cutover_seq, 2); + assert_eq!(prepared.artifact.child.parent_cutover_offset, end); + assert_eq!(prepared.artifact.child.child_stream_start_seq, 3); + assert_eq!( + child.get(1, b"n", None).await.unwrap().unwrap().value, + b"independent-child-wal" + ); +} + +#[tokio::test] +async fn cancelled_cutover_receiver_finalizes_installed_route_without_reinstalling() { + let TestNativeSplit { + parent, + publication, + gate, + plan, + mut preparing, + .. + } = setup_native_publication(false, false, false).await; + tokio::select! { + () = gate.entered.notified() => {}, + result = &mut preparing => panic!("failed before handoff: {}", preparation_failure(result)), + } + publication.cancel_control_after_install(preparing.abort_handle()); + gate.release.notify_one(); + match preparing.await { + Err(error) => assert!(error.is_cancelled()), + Ok(_) => panic!("control future must be cancelled immediately after native commit"), + } + assert!(parent.split_ingress().is_some()); + assert!(parent.pending_split_child_target(&plan).unwrap().is_none()); + assert_eq!(publication.calls(), (0, 0, 0)); + let (request, operation) = put(2, b"n", b"after-cancel"); + let response = parent.mutate(1, request, operation).await.unwrap(); + assert_eq!(response.mutation_seq, 2); + assert_eq!( + response.journal_position.stream_name, + parent.split_ingress().unwrap().child().snapshot().stream_name + ); + let retry_source = parent.clone(); + let mut retry = + tokio::spawn(async move { retry_source.finalize_installed_split_child_session(plan).await }); + tokio::select! { + () = publication.entered.notified() => {}, + result = &mut retry => panic!("finalizer failed: {}", preparation_failure(result)), + } + publication.release.notify_one(); + let prepared = retry.await.unwrap().unwrap(); + assert_eq!(prepared.artifact.cutover_seq, 1); + assert_eq!(publication.prepared_calls(), (1, 1)); + assert_eq!(publication.calls(), (0, 1, 1)); + assert_eq!( + parent.get(1, b"n", None).await.unwrap().unwrap().value, + b"after-cancel" + ); +} + +#[tokio::test] +async fn native_token_cleanup_never_blocks_parent_or_child_mutation_workers() { + let TestNativeSplit { + parent, + publication, + gate, + mut preparing, + .. + } = setup_native_publication(false, false, false).await; + tokio::select! { + () = gate.entered.notified() => {}, + result = &mut preparing => panic!("failed before handoff: {}", preparation_failure(result)), + } + let cleanup = publication.pause_cleanup(); + gate.release.notify_one(); + tokio::select! { + () = cleanup.entered() => {}, + result = &mut preparing => panic!("failed before cleanup: {}", preparation_failure(result)), + } + assert!(parent.split_ingress().is_some()); + assert!(!preparing.is_finished()); + assert_eq!(publication.calls(), (0, 0, 0)); + let parent_stream = parent.snapshot().stream_name; + let child_stream = parent.split_ingress().unwrap().child().snapshot().stream_name; + tokio::time::timeout(std::time::Duration::from_secs(3), async { + for (sequence, key, expected_stream) in [(2, b"b", parent_stream), (3, b"n", child_stream)] { + let (request, operation) = put(sequence, key, b"during-host-cleanup"); + let response = parent.mutate(1, request, operation).await.unwrap(); + assert_eq!(response.journal_position.stream_name, expected_stream); + assert_eq!( + parent.get(1, key, None).await.unwrap().unwrap().value, + b"during-host-cleanup" + ); + } + }) + .await + .unwrap(); + drop(cleanup); + tokio::select! { + () = publication.entered.notified() => {}, + result = &mut preparing => panic!("failed before publication: {}", preparation_failure(result)), + } + publication.release.notify_one(); + let prepared = preparing.await.unwrap().unwrap(); + assert_eq!(prepared.artifact.cutover_seq, 1); + assert_eq!(publication.prepared_calls(), (1, 1)); + assert_eq!(publication.calls(), (0, 1, 1)); assert_eq!( parent.get(1, b"b", None).await.unwrap().unwrap().value, - b"during-status" + b"during-host-cleanup" + ); + assert_eq!( + parent.get(1, b"n", None).await.unwrap().unwrap().value, + b"during-host-cleanup" ); } diff --git a/lib/crowdb-chunk-kv/tests/partition_empty_cursor_test.rs b/lib/crowdb-chunk-kv/tests/partition_empty_cursor_test.rs new file mode 100644 index 000000000..917f288aa --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/partition_empty_cursor_test.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv::{CrowdbPartitionTree, MutationOperation, PartitionTree}; +use crowdb_tree_ffi::{Config, PageStore}; +use std::sync::Arc; + +async fn tree() -> CrowdbPartitionTree { + let tree = CrowdbPartitionTree::open( + 43, + &Config { + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + frame_bytes: 4096, + ..Config::default() + }, + ) + .unwrap(); + for (slot, key) in [(1, b"".as_slice()), (2, b"a".as_slice())] { + tree.apply( + slot, + &MutationOperation::Put { + key: key.to_vec(), + value: b"value".to_vec(), + }, + ) + .await + .unwrap(); + } + tree +} + +async fn check_empty_key_continuation(tree: &dyn PartitionTree) { + for (limit, budget) in [(0, 0), (1, 1024)] { + let empty = tree + .scan_forward(None, true, Some(b""), limit, budget) + .await + .unwrap(); + assert_eq!(empty, (Vec::new(), false)); + let empty = tree + .scan_forward_bounded(None, true, Some(b""), limit, budget) + .await + .unwrap(); + assert_eq!(empty, (Vec::new(), false)); + } + let (first, more) = tree.scan_forward(None, true, None, 1, 1024).await.unwrap(); + assert!(more); + assert_eq!(first.len(), 1); + assert_eq!(first[0].key.as_ref(), b""); + let (next, more) = tree + .scan_forward(Some(first[0].key.as_ref()), false, None, 1, 1024) + .await + .unwrap(); + assert!(!more); + assert_eq!(next.len(), 1); + assert_eq!(next[0].key.as_ref(), b"a"); + let (inclusive, more) = tree.scan_forward(Some(b""), true, None, 1, 1024).await.unwrap(); + assert!(more); + assert_eq!(inclusive.len(), 1); + assert_eq!(inclusive[0].key.as_ref(), b""); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn live_default_tree_resumes_exclusively_after_an_empty_key() { + let tree = tree().await; + check_empty_key_continuation(&tree).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn captured_default_tree_preserves_the_empty_key_cursor_after_writer_close() { + let tree = tree().await; + let (_, coverage, snapshot) = tree.checkpoint_snapshot(0).await.unwrap(); + assert_eq!(coverage, 2); + drop(tree); + check_empty_key_continuation(snapshot.as_ref()).await; +} diff --git a/lib/crowdb-chunk-kv/tests/partition_test.rs b/lib/crowdb-chunk-kv/tests/partition_test.rs index d3c0c37c7..612762167 100644 --- a/lib/crowdb-chunk-kv/tests/partition_test.rs +++ b/lib/crowdb-chunk-kv/tests/partition_test.rs @@ -15,7 +15,7 @@ use crowdb_chunk_stream::{ ChunkId, ChunkStream, CursorAdvance, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamRecord, StreamRecordState, StreamRegistry, }; -use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, PageStore}; +use crowdb_tree_ffi::chunk::test_util::TestChunkBackend; fn request(sequence: u64) -> RequestId { RequestId { @@ -289,28 +289,6 @@ async fn split_ingress_routes_old_parent_requests_to_both_new_writers() { ); } -fn chunk_page_store(tree_id: u64, owner_epoch: u64) -> Arc<PageStore> { - let catalog = Arc::new(ChunkRootCatalog::open_memory(owner_epoch).unwrap()); - Arc::new( - PageStore::open_chunk( - ChunkPageStoreOptions { - tree_id, - owner_epoch, - open_generation: 0, - pack_bytes: 4_096, - iu_size: 1, - max_concurrent_packs: 2, - max_chunk_bytes: 0, - mirror_copies: 0, - materialization_bytes_per_pass: 4_096, - }, - catalog, - None, - ) - .unwrap(), - ) -} - async fn finish_materialization(partition: &Partition, ownership_epoch: u64) -> u64 { let mut passes = 0; loop { @@ -561,19 +539,8 @@ async fn chunk_root_checkpoint_supplies_the_wal_replay_offset() { ) .await .unwrap(); - let catalog = Arc::new(ChunkRootCatalog::open_memory(4).unwrap()); - let options = ChunkPageStoreOptions { - tree_id: 45, - owner_epoch: 4, - open_generation: 0, - pack_bytes: 4_096, - iu_size: 1, - max_concurrent_packs: 2, - max_chunk_bytes: 0, - mirror_copies: 0, - materialization_bytes_per_pass: 4_096, - }; - let page_store = Arc::new(PageStore::open_chunk(options, Arc::clone(&catalog), None).unwrap()); + let mut backend = TestChunkBackend::new(45, 4).unwrap(); + let page_store = backend.page_store(0).unwrap(); let config = PartitionConfig { retained_results: 1, ..PartitionConfig::default() @@ -587,7 +554,7 @@ async fn chunk_root_checkpoint_supplies_the_wal_replay_offset() { 4, config.clone(), 45, - crowdb_tree_ffi::Config::default(), + backend.config(), Arc::clone(&page_store), stream, ) @@ -609,7 +576,7 @@ async fn chunk_root_checkpoint_supplies_the_wal_replay_offset() { let checkpoint = partition.checkpoint_quiesced(4).await.unwrap(); assert!(checkpoint.replay_offset > 0); assert_eq!(page_store.wal_replay_offset().unwrap(), checkpoint.replay_offset); - drop(partition); + partition.close().await.unwrap(); let stream = ChunkStream::open( stream_name, @@ -621,7 +588,8 @@ async fn chunk_root_checkpoint_supplies_the_wal_replay_offset() { ) .await .unwrap(); - let page_store = Arc::new(PageStore::open_chunk(options, catalog, None).unwrap()); + backend.next_writer().unwrap(); + let page_store = backend.page_store(0).unwrap(); let recovered = Partition::recover_native_latest_prepared_assignment( PartitionId { high: 1, low: 11 }, PartitionRange { @@ -631,7 +599,7 @@ async fn chunk_root_checkpoint_supplies_the_wal_replay_offset() { 4, 45, config, - crowdb_tree_ffi::Config::default(), + backend.config(), page_store, stream, ) @@ -639,6 +607,8 @@ async fn chunk_root_checkpoint_supplies_the_wal_replay_offset() { .unwrap(); recovered.activate_recovered(4).unwrap(); assert_eq!(recovered.get(4, b"b", None).await.unwrap().unwrap().value, b"2"); + recovered.suspend_for_shutdown(4).await.unwrap(); + recovered.close().await.unwrap(); } #[tokio::test] @@ -2175,6 +2145,8 @@ async fn native_online_split_rebuilds_one_child_from_the_exact_parent_manifest() ) .await .unwrap(); + let parent_backend = TestChunkBackend::new(190, 18).unwrap(); + let child_backend = TestChunkBackend::new(192, 19).unwrap(); let parent = Partition::open_native( PartitionId { high: 120, low: 120 }, PartitionRange { @@ -2184,8 +2156,8 @@ async fn native_online_split_rebuilds_one_child_from_the_exact_parent_manifest() 18, PartitionConfig::default(), 190, - crowdb_tree_ffi::Config::default(), - chunk_page_store(190, 18), + parent_backend.config(), + parent_backend.page_store(0).unwrap(), parent_stream, ) .unwrap(); @@ -2212,8 +2184,8 @@ async fn native_online_split_rebuilds_one_child_from_the_exact_parent_manifest() SplitWriterTarget { tree_id: 192, tree_config: crowdb_tree_ffi::Config { - page_store: Some(chunk_page_store(192, 19)), - ..crowdb_tree_ffi::Config::default() + page_store: Some(child_backend.page_store(0).unwrap()), + ..child_backend.config() }, journal: empty_journal(&store, StreamName { high: 192, low: 1 }, 19).await, }, @@ -2232,7 +2204,15 @@ async fn native_online_split_rebuilds_one_child_from_the_exact_parent_manifest() child.activate_prepared(&proof).unwrap(); let passes = finish_materialization(&child, 19).await; assert_eq!(child.metrics().snapshot().materialization_passes, passes); - assert_eq!(child.chunk_storage_stats().unwrap().unwrap().shared_packs, 0); + let backing = child.backing_storage_stats().unwrap().unwrap(); + assert!(backing.allocated_bytes > 0); + assert_eq!(child.estimated_bytes().unwrap(), backing.allocated_bytes); + assert_ne!( + backing.scope_id, + parent.backing_storage_stats().unwrap().unwrap().scope_id + ); + parent.close().await.unwrap(); + child.close().await.unwrap(); } async fn overlay_base_tree(tree_id: u64) -> (Arc<MemoryPartitionTree>, u64) { diff --git a/lib/crowdb-chunk-kv/tests/prepared_split_overlay_test.rs b/lib/crowdb-chunk-kv/tests/prepared_split_overlay_test.rs new file mode 100644 index 000000000..059e126fb --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/prepared_split_overlay_test.rs @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, MutationOperation, PartitionRange, PartitionTree, SplitOverlayLimits, +}; + +fn put(key: &[u8], value: &[u8]) -> MutationOperation { + MutationOperation::Put { + key: key.to_vec(), + value: value.to_vec(), + } +} + +#[tokio::test] +async fn observer_covers_registration_replay_gap_and_detaches_at_exact_frontier() { + let source = MemoryPartitionTree::with_tree_id(1); + let destination = MemoryPartitionTree::with_tree_id(2); + source.apply(1, &put(b"a", b"base")).await.unwrap(); + destination.apply(1, &put(b"a", b"base")).await.unwrap(); + let (token, start) = source + .arm_split_overlay( + &destination, + SplitOverlayLimits { + max_records: 8, + max_bytes: 1024, + }, + ) + .await + .unwrap(); + assert_eq!(start, 1); + source.apply(2, &put(b"b", b"before-replay")).await.unwrap(); + destination.apply(2, &put(b"b", b"before-replay")).await.unwrap(); + token.prepare(2).await.unwrap(); + source.apply(3, &put(b"c", b"after-replay")).await.unwrap(); + let installed = token.try_install(3).unwrap(); + assert_eq!(token.committed_frontier(), Some(3)); + assert_eq!(token.try_install(3).unwrap(), installed); + assert!(token.try_install(4).is_err()); + source + .apply(4, &put(b"d", b"parent-after-cutover")) + .await + .unwrap(); + destination + .apply(4, &put(b"b", b"child-after-cutover")) + .await + .unwrap(); + source + .publish_split_memtable_view(installed.generation, 3, &destination, &PartitionRange::default()) + .await + .unwrap(); + assert_eq!(destination.get(b"a").await.unwrap().unwrap().value, b"base"); + assert_eq!( + destination.get(b"b").await.unwrap().unwrap().value, + b"child-after-cutover" + ); + assert_eq!( + destination.get(b"c").await.unwrap().unwrap().value, + b"after-replay" + ); + assert!(destination.get(b"d").await.unwrap().is_none()); + source + .release_split_memtable_view(installed.generation) + .await + .unwrap(); + assert_eq!(source.prepared_overlay_footprint_for_tests().await, (0, 0, false)); +} + +#[tokio::test] +async fn expired_observer_stops_growth_without_rejecting_parent_writes() { + let source = MemoryPartitionTree::with_tree_id(1); + let destination = MemoryPartitionTree::with_tree_id(2); + let (token, start) = source + .arm_split_overlay( + &destination, + SplitOverlayLimits { + max_records: 2, + max_bytes: 4, + }, + ) + .await + .unwrap(); + assert_eq!(start, 0); + token.prepare(0).await.unwrap(); + source.apply(1, &put(b"a", b"1")).await.unwrap(); + source.apply(2, &put(b"b", b"2")).await.unwrap(); + source.apply(3, &put(b"c", b"3")).await.unwrap(); + let exhausted = source.prepared_overlay_footprint_for_tests().await; + assert_eq!(exhausted, (2, 4, true)); + for sequence in 4_u64..100 { + source + .apply(sequence, &put(&sequence.to_be_bytes(), b"still-serving")) + .await + .unwrap(); + } + assert_eq!(source.prepared_overlay_footprint_for_tests().await, exhausted); + assert_eq!(source.last_applied_seq(), 99); + assert!(token.try_install(99).is_err()); + assert_eq!(destination.last_applied_seq(), 0); + assert!(destination.get(b"a").await.unwrap().is_none()); + assert_eq!( + source.get(&99_u64.to_be_bytes()).await.unwrap().unwrap().value, + b"still-serving" + ); +} diff --git a/lib/crowdb-chunk-kv/tests/split_cutover_journal_test.rs b/lib/crowdb-chunk-kv/tests/split_cutover_journal_test.rs new file mode 100644 index 000000000..73e26683a --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/split_cutover_journal_test.rs @@ -0,0 +1,306 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#[path = "common/split_storage.rs"] +mod split_storage; + +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, Checkpoint, MutationOperation, Partition, PartitionConfig, PartitionId, + PartitionJournal, PartitionRange, PartitionTree, PreparedSplitWriterArtifact, RequestId, SplitChild, + SplitHandoffStore, SplitPlan, SplitWriterTarget, StreamPartitionJournal, TransitionId, +}; +use crowdb_chunk_stream::{memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName}; +use split_storage::TestSplitStorage; +use std::sync::Arc; +use tokio::sync::{Mutex, Notify}; + +#[derive(Default)] +struct TestHandoff { + proof: Mutex<Option<PreparedSplitWriterArtifact>>, + entered: Notify, + release: Notify, +} + +#[async_trait::async_trait] +impl SplitHandoffStore for TestHandoff { + async fn commit(&self, artifact: &PreparedSplitWriterArtifact) -> crowdb_chunk_kv::Result<()> { + *self.proof.lock().await = Some(artifact.clone()); + self.entered.notify_one(); + self.release.notified().await; + Ok(()) + } +} + +struct TestCrashState { + parent_store: Arc<MemoryStreamStore>, + child_store: Arc<MemoryStreamStore>, + proof: PreparedSplitWriterArtifact, + checkpoint: Checkpoint, + plan: SplitPlan, + published_base_end: u64, + end: u64, +} + +fn operation(key: &[u8], value: &[u8]) -> MutationOperation { + MutationOperation::Put { + key: key.to_vec(), + value: value.to_vec(), + } +} + +async fn mutate(parent: &Partition, sequence: u64, key: &[u8], value: &[u8]) { + parent + .mutate( + 1, + RequestId { + client_high: 92, + client_low: 1, + client_sequence: sequence, + }, + operation(key, value), + ) + .await + .unwrap(); +} + +async fn interrupt_after_route() -> TestCrashState { + let parent_storage = TestSplitStorage::new(); + let child_storage = TestSplitStorage::new(); + let parent_tree = Arc::new(MemoryPartitionTree::with_tree_id(92)); + let (parent, journal) = parent_storage + .partition( + PartitionId { high: 92, low: 1 }, + PartitionRange::default(), + 1, + parent_tree.clone(), + ) + .await; + mutate(&parent, 1, b"b", b"base-left").await; + let base_end = journal.tail(); + let plan = SplitPlan { + transition_id: TransitionId { high: 92, low: 2 }, + parent_id: parent.snapshot().partition_id, + parent_range: PartitionRange::default(), + parent_epoch: 1, + parent_next_epoch: 2, + split_key: b"m".to_vec(), + child: SplitChild { + partition_id: PartitionId { high: 93, low: 1 }, + ownership_epoch: 1, + range: PartitionRange { + start: Some(b"m".to_vec()), + end: None, + }, + }, + }; + let child_journal = child_storage.journal(StreamName { high: 93, low: 1 }, 1).await; + let target = SplitWriterTarget { + tree_id: 93, + tree_config: crowdb_tree_ffi::Config::default(), + journal: child_journal, + }; + let handoff = Arc::new(TestHandoff::default()); + let source = parent.clone(); + let pending_plan = plan.clone(); + let gate = handoff.clone(); + let mut preparing = tokio::spawn(async move { + source + .prepare_split_child_session(pending_plan, target, 8, gate) + .await + }); + tokio::select! { + () = handoff.entered.notified() => {}, + result = &mut preparing => panic!("handoff failed before commit: {}", failure(result)), + } + let published_base = open_published(&parent_storage.store, journal.stream_name()).await; + let published_base_end = published_base.tail(); + assert_eq!(published_base_end, base_end); + // Memory's live facade has no persisted root identity. Capture C before + // this fixture advances the live parent beyond the immutable base. + let (manifest, applied) = parent_tree.checkpoint_state().unwrap(); + let checkpoint = Checkpoint { + tree_id: 92, + tree_manifest: manifest, + root_manifest_generation: parent_tree.root_manifest_generation().unwrap(), + applied_seq: applied, + stream_name: journal.stream_name(), + stream_generation: journal.record_generation(), + replay_offset: base_end, + }; + assert_eq!(checkpoint.applied_seq, 1); + mutate(&parent, 2, b"n", b"parent-tail").await; + mutate(&parent, 3, b"c", b"left-tail").await; + let end = journal.tail(); + parent_storage.store.pause_publication_for_tests(); + handoff.release.notify_one(); + tokio::select! { + () = parent_storage.store.wait_for_publication_for_tests() => {}, + result = &mut preparing => panic!("handoff failed before WAL proof: {}", failure(result)), + } + assert!(parent.split_ingress().is_some()); + let child = parent.split_ingress().unwrap().child(); + assert_eq!(child.snapshot().applied_seq, 3); + mutate(&parent, 4, b"n", b"child-tail").await; + assert_eq!(child.snapshot().applied_seq, 4); + assert!(!preparing.is_finished()); + let proof = handoff.proof.lock().await.clone().unwrap(); + assert_eq!(proof.applied_seq, 1); + assert_eq!(proof.base_applied_seq, 1); + // The fixed Tree already includes C. With no catch-up records, its replay + // interval stays empty at the retained retry floor; it is not end(C). + assert_eq!(proof.parent_replay_offset, 0); + assert_eq!(proof.parent_cutover_offset, proof.parent_replay_offset); + TestCrashState { + parent_store: parent_storage.store, + child_store: child_storage.store, + proof, + checkpoint, + plan, + published_base_end, + end, + } +} + +fn failure( + result: Result<crowdb_chunk_kv::Result<crowdb_chunk_kv::PreparedSplit>, tokio::task::JoinError>, +) -> String { + match result { + Ok(Err(error)) => error.to_string(), + Err(error) => error.to_string(), + Ok(Ok(_)) => "completed before the expected gate".into(), + } +} + +async fn open_published(store: &Arc<MemoryStreamStore>, name: StreamName) -> ChunkStream { + ChunkStream::open_read_only( + name, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap() +} + +async fn reopen_writer(store: &Arc<MemoryStreamStore>, name: StreamName) -> ChunkStream { + ChunkStream::open( + name, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap() +} + +async fn recover_after_interrupt(state: TestCrashState) { + state.parent_store.resume_publication_for_tests(); + let source_name = state.checkpoint.stream_name; + let stale_reader = open_published(&state.parent_store, source_name).await; + assert_eq!(stale_reader.tail(), state.published_base_end); + assert!(stale_reader.tail() < state.end); + // The old runtime was dropped while its publication was paused. Writer + // recovery scans synchronized frames before a separate read-only view opens. + let source_stream = reopen_writer(&state.parent_store, source_name).await; + assert_eq!(source_stream.tail(), state.end); + let source_journal: Arc<dyn PartitionJournal> = + Arc::new(StreamPartitionJournal::new(source_stream, source_name).unwrap()); + let source_tree = Arc::new(MemoryPartitionTree::with_tree_id(92)); + source_tree + .apply(1, &operation(b"b", b"base-left")) + .await + .unwrap(); + source_tree + .checkpoint(state.checkpoint.replay_offset) + .await + .unwrap(); + let parent = Partition::recover_prepared_assignment( + state.plan.parent_id, + PartitionRange::default(), + 1, + state.checkpoint, + PartitionConfig::default(), + source_tree, + source_journal, + ) + .await + .unwrap(); + assert!( + parent + .publish_split_journal_prefix(state.proof.parent_cutover_offset) + .await + .unwrap() + >= state.end + ); + let reader = open_published(&state.parent_store, source_name).await; + assert!(reader.tail() >= state.end); + let child_stream = reopen_writer(&state.child_store, state.proof.stream_name).await; + let child_journal = Arc::new(StreamPartitionJournal::new(child_stream, state.proof.stream_name).unwrap()); + let child_tree = Arc::new(MemoryPartitionTree::with_tree_id(93)); + child_tree.advance_noop(1).await.unwrap(); + child_tree.checkpoint(0).await.unwrap(); + let source_reader = Arc::new(StreamPartitionJournal::new(reader, source_name).unwrap()); + let (child, artifact) = Partition::recover_split_handoff( + state.proof, + PartitionConfig::default(), + child_tree.clone(), + child_journal, + source_reader, + ) + .await + .unwrap(); + assert_eq!(artifact.applied_seq, 3); + assert_eq!(artifact.parent_cutover_offset, state.end); + assert_eq!(artifact.child_stream_start_seq, 4); + assert_eq!(child.snapshot().applied_seq, 4); + assert_eq!( + child.snapshot().lifecycle, + crowdb_chunk_kv::PartitionLifecycle::Prepared + ); + let prepared = parent + .resume_split_child_session(state.plan, child.clone(), artifact) + .await + .unwrap(); + assert_eq!(prepared.artifact.cutover_seq, 3); + assert_eq!( + child.get(1, b"n", None).await.unwrap().unwrap().value, + b"child-tail" + ); + assert!(matches!( + child.get(1, b"c", None).await, + Err(crowdb_chunk_kv::ChunkKvError::OutOfRange) + )); + assert!(child_tree.get(b"c").await.unwrap().is_none()); + mutate(&parent, 5, b"o", b"recovered-route").await; + assert_eq!( + parent + .split_ingress() + .unwrap() + .child() + .get(1, b"o", None) + .await + .unwrap() + .unwrap() + .value, + b"recovered-route" + ); +} + +#[test] +fn crash_after_route_before_parent_publication_recovers_first_child_wal() { + let old_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let state = old_runtime.block_on(interrupt_after_route()); + drop(old_runtime); + let recovered_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + recovered_runtime.block_on(recover_after_interrupt(state)); +} diff --git a/lib/crowdb-chunk-kv/tests/split_handoff_recovery_test.rs b/lib/crowdb-chunk-kv/tests/split_handoff_recovery_test.rs index a449b3fcb..ae0b2d71c 100644 --- a/lib/crowdb-chunk-kv/tests/split_handoff_recovery_test.rs +++ b/lib/crowdb-chunk-kv/tests/split_handoff_recovery_test.rs @@ -27,7 +27,7 @@ fn put(key: &[u8], value: &[u8]) -> MutationOperation { } } -async fn recover_after_handoff(child_has_write: bool, checkpoint_after_dispatch: bool) { +async fn recover_after_handoff(child_has_write: bool, checkpoint_after_dispatch: bool, native: bool) { let storage = TestSplitStorage::new(); let parent_id = PartitionId { high: 50, low: 1 }; let child_id = PartitionId { high: 51, low: 1 }; @@ -76,28 +76,7 @@ async fn recover_after_handoff(child_has_write: bool, checkpoint_after_dispatch: .await .unwrap(); if child_has_write { - let live_tree = Arc::new(MemoryPartitionTree::with_tree_id(51)); - live_tree.advance_noop(3).await.unwrap(); - let child = Partition::open( - child_id, - child_range, - 1, - PartitionConfig::default(), - live_tree, - child_journal.clone(), - ) - .unwrap(); - child - .mutate(1, request(4), put(b"n", b"child-tail")) - .await - .unwrap(); - // Parent and child sequences diverge after dispatch. The parent's - // later sequence 4 must not move the inherited child frontier to 4. - parent - .mutate(1, request(5), put(b"d", b"later-left")) - .await - .unwrap(); - drop(child); + write_child_tail(&parent, child_id, child_range, child_journal.clone()).await; } if checkpoint_after_dispatch { let (checkpoint, tree) = @@ -106,7 +85,19 @@ async fn recover_after_handoff(child_has_write: bool, checkpoint_after_dispatch: cold_parent_tree = tree; } drop(parent); - let cold_tree = Arc::new(MemoryPartitionTree::with_tree_id(51)); + if native { + assert!(!checkpoint_after_dispatch); + let native_parent = native_tree(50); + native_parent.apply(1, &put(b"b", b"base-left")).await.unwrap(); + native_parent.checkpoint(0).await.unwrap(); + assert_eq!(native_parent.checkpoint_state().unwrap(), (1, 1)); + cold_parent_tree = native_parent; + } + let cold_tree: Arc<dyn PartitionTree> = if native { + native_tree(51) + } else { + Arc::new(MemoryPartitionTree::with_tree_id(51)) + }; cold_tree.advance_noop(1).await.unwrap(); cold_tree.checkpoint(0).await.unwrap(); let (recovered, artifact) = Partition::recover_split_handoff( @@ -118,7 +109,7 @@ async fn recover_after_handoff(child_has_write: bool, checkpoint_after_dispatch: ) .await .unwrap(); - verify_recovery(&recovered, &artifact, &cold_tree, child_has_write).await; + verify_recovery(&recovered, &artifact, cold_tree.as_ref(), child_has_write).await; verify_restored_dispatch( recovered, artifact, @@ -129,6 +120,36 @@ async fn recover_after_handoff(child_has_write: bool, checkpoint_after_dispatch: .await; } +async fn write_child_tail( + parent: &Partition, + child_id: PartitionId, + child_range: PartitionRange, + child_journal: Arc<dyn crowdb_chunk_kv::PartitionJournal>, +) { + let live_tree = Arc::new(MemoryPartitionTree::with_tree_id(51)); + live_tree.advance_noop(3).await.unwrap(); + let child = Partition::open( + child_id, + child_range, + 1, + PartitionConfig::default(), + live_tree, + child_journal, + ) + .unwrap(); + child + .mutate(1, request(4), put(b"n", b"child-tail")) + .await + .unwrap(); + // Parent and child sequences diverge after dispatch. The parent's + // later sequence 4 must not move the inherited child frontier to 4. + parent + .mutate(1, request(5), put(b"d", b"later-left")) + .await + .unwrap(); + drop(child); +} + async fn verify_restored_dispatch( child: Partition, artifact: PreparedSplitWriterArtifact, @@ -191,7 +212,7 @@ async fn verify_restored_dispatch( async fn verify_recovery( recovered: &Partition, artifact: &PreparedSplitWriterArtifact, - cold_tree: &MemoryPartitionTree, + cold_tree: &dyn PartitionTree, child_has_write: bool, ) { assert_eq!(artifact.applied_seq, 3); @@ -215,17 +236,17 @@ async fn verify_recovery( #[tokio::test] async fn status_committed_before_dispatch_recovers_parent_suffix() { - recover_after_handoff(false, false).await; + recover_after_handoff(false, false, false).await; } #[tokio::test] async fn child_wal_identifies_inherited_frontier_and_replays_after_parent() { - recover_after_handoff(true, false).await; + recover_after_handoff(true, false, false).await; } #[tokio::test] async fn recovered_parent_checkpoint_can_be_ahead_of_child_inherited_frontier() { - recover_after_handoff(true, true).await; + recover_after_handoff(true, true, false).await; } async fn checkpoint_source_after_dispatch( @@ -258,3 +279,21 @@ fn source_checkpoint_at_base( replay_offset: base_offset, } } + +fn native_tree(id: u64) -> Arc<dyn PartitionTree> { + Arc::new( + crowdb_chunk_kv::CrowdbPartitionTree::open( + id, + &crowdb_tree_ffi::Config { + page_store: Some(Arc::new(crowdb_tree_ffi::PageStore::open_mem(65_536).unwrap())), + ..crowdb_tree_ffi::Config::default() + }, + ) + .unwrap(), + ) +} + +#[tokio::test] +async fn native_replayed_child_wal_resumes_without_reinstalling_parent_overlay() { + recover_after_handoff(true, false, true).await; +} diff --git a/lib/crowdb-chunk-kv/tests/split_ingress_test.rs b/lib/crowdb-chunk-kv/tests/split_ingress_test.rs index d700586d8..8bdbc1415 100644 --- a/lib/crowdb-chunk-kv/tests/split_ingress_test.rs +++ b/lib/crowdb-chunk-kv/tests/split_ingress_test.rs @@ -37,7 +37,7 @@ async fn handoff_persistence_does_not_block_parent_reads_or_journal_appends() { let (parent, _) = storage .partition(parent_id, range.clone(), 1, parent_tree.clone()) .await; - parent.mutate(1, request(10), put(b"b", b"base")).await.unwrap(); + parent.mutate(1, request(10), put(b"n", b"base")).await.unwrap(); let plan = SplitPlan { transition_id: TransitionId { high: 10, low: 2 }, parent_id, @@ -92,6 +92,11 @@ async fn handoff_persistence_does_not_block_parent_reads_or_journal_appends() { parent_tree.pause_split_publish(); gate.release.notify_one(); parent_tree.wait_for_split_publish().await; + assert_eq!( + parent.get(1, b"n", None).await.unwrap().unwrap().value, + b"during-handoff", + "the inherited update must be readable before publication finishes" + ); verify_live_dispatch_during_publication(&parent).await; parent_tree.resume_split_publish(); let prepared = preparing.await.unwrap().unwrap(); diff --git a/lib/crowdb-chunk-stream/Cargo.toml b/lib/crowdb-chunk-stream/Cargo.toml index 45a1fa88a..737f911bd 100644 --- a/lib/crowdb-chunk-stream/Cargo.toml +++ b/lib/crowdb-chunk-stream/Cargo.toml @@ -24,7 +24,7 @@ async-trait = "0.1" bytes = "1" crc32fast = "1" crowdb-common = { path = "../crowdb-common/rust" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-chunk-client = { path = "../crowdb-chunk-client" } crowdb-protocol = { path = "../crowdb-protocol" } crossbeam-skiplist = "0.1" @@ -38,5 +38,5 @@ tracing = { workspace = true } [dev-dependencies] crowdb-chunk-stream = { path = ".", features = ["test-util"] } crowdb-test-harness = { path = "../crowdb-test-harness" } -crowdb-kv = { path = "../crowdb-kv", features = ["test-util"] } +crowdb-paxos-kv = { path = "../crowdb-paxos-kv", features = ["test-util"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "test-util"] } diff --git a/lib/crowdb-chunk-stream/benches/production_bounds.rs b/lib/crowdb-chunk-stream/benches/production_bounds.rs index 841696ac6..9d71d4d87 100644 --- a/lib/crowdb-chunk-stream/benches/production_bounds.rs +++ b/lib/crowdb-chunk-stream/benches/production_bounds.rs @@ -18,7 +18,7 @@ use crowdb_chunk_client::{ }; use crowdb_chunk_stream::{ProductionStreamRuntime, StreamConfig}; use crowdb_common::metrics; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use config::BenchConfig; @@ -62,7 +62,7 @@ async fn run(config: BenchConfig) -> ExitCode { return ExitCode::FAILURE; } }; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![config + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![config .management_seed .clone()]))); let stream_runtime = match ProductionStreamRuntime::new( diff --git a/lib/crowdb-chunk-stream/src/kv.rs b/lib/crowdb-chunk-stream/src/kv.rs index 2f7e5409e..9ce1db51f 100644 --- a/lib/crowdb-chunk-stream/src/kv.rs +++ b/lib/crowdb-chunk-stream/src/kv.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_trait::async_trait; -use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::chunk_stream::{StreamName, StreamRecord}; use crowdb_protocol::key::{BinaryKey, StreamRecordKey}; @@ -17,13 +17,13 @@ use crate::{Result, StreamError, StreamMetadataStore, StreamRegistry}; /// Group-0 readable stream record registry. pub struct KvStreamRegistry { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, processes: owner::OwnerProcesses, } impl KvStreamRegistry { #[must_use] - pub fn new(kv: Arc<CrowdbKvClient>) -> Self { + pub fn new(kv: Arc<PaxosKvClient>) -> Self { Self { kv, processes: owner::OwnerProcesses::default(), @@ -81,14 +81,16 @@ impl StreamRegistry for KvStreamRegistry { .map_err(|error| StreamError::Internal(format!("stream record encode failed: {error}")))?; match self.kv.put_cas(0, 0, &key, &value, 0).await { Ok(_) => Ok(()), - Err(crowdb_kv_client::Error::CasFailed { .. }) => { + Err(crowdb_paxos_kv_client::Error::CasFailed { .. }) => { Err(StreamError::InvalidRequest("stream record already exists".into())) } - Err(crowdb_kv_client::Error::CasBusy) => Err(StreamError::Backpressure), - Err(crowdb_kv_client::Error::OutcomeUnknown) => match self.load(record.stream_name).await? { - Some(observed) if observed == record => Ok(()), - _ => Err(StreamError::WriteStalled), - }, + Err(crowdb_paxos_kv_client::Error::CasBusy) => Err(StreamError::Backpressure), + Err(crowdb_paxos_kv_client::Error::OutcomeUnknown) => { + match self.load(record.stream_name).await? { + Some(observed) if observed == record => Ok(()), + _ => Err(StreamError::WriteStalled), + } + } Err(error) => Err(kv_error(&error)), } } @@ -96,14 +98,14 @@ impl StreamRegistry for KvStreamRegistry { /// KV adapter for one nonzero Stream metadata group. pub struct KvStreamMetadataStore { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, store_id: u64, group_id: u64, } impl KvStreamMetadataStore { /// # Errors /// Rejects group zero, reserved for Stream identity and ownership records. - pub fn new(kv: Arc<CrowdbKvClient>, store_id: u64, group_id: u64) -> Result<Self> { + pub fn new(kv: Arc<PaxosKvClient>, store_id: u64, group_id: u64) -> Result<Self> { if group_id == 0 { return Err(StreamError::InvalidRequest( "Stream chunk records require a nonzero KV group".into(), @@ -133,6 +135,6 @@ impl StreamMetadataStore for KvStreamMetadataStore { ) } } -fn kv_error(error: &crowdb_kv_client::Error) -> StreamError { +fn kv_error(error: &crowdb_paxos_kv_client::Error) -> StreamError { StreamError::ReadUnavailable(format!("Stream KV operation failed: {error}")) } diff --git a/lib/crowdb-chunk-stream/src/kv/owner.rs b/lib/crowdb-chunk-stream/src/kv/owner.rs index 3b86c8241..3689701ce 100644 --- a/lib/crowdb-chunk-stream/src/kv/owner.rs +++ b/lib/crowdb-chunk-stream/src/kv/owner.rs @@ -7,8 +7,8 @@ use super::{kv_error, KvStreamRegistry}; use crate::{Result, StreamError, StreamName, StreamOwnerAuthority, StreamWriterSession}; use arc_swap::ArcSwapOption; use async_trait::async_trait; -use crowdb_kv_client::{ - CrowdbKvClient, GetOutcome, ReadMode, StreamProcessAuthority, StreamProcessLeaseClient, +use crowdb_paxos_kv_client::{ + GetOutcome, PaxosKvClient, ReadMode, StreamProcessAuthority, StreamProcessLeaseClient, StreamProcessLeaseError, StreamProcessLeaseTiming, }; use crowdb_protocol::{ @@ -176,7 +176,7 @@ impl KvStreamRegistry { } struct AssignedOwner { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, process: Arc<ProcessOwner>, owner: StreamWriterOwner, } @@ -209,7 +209,7 @@ impl StreamOwnerAuthority for AssignedOwner { } async fn exclude_previous( - kv: &Arc<CrowdbKvClient>, + kv: &Arc<PaxosKvClient>, current: &StreamProcessAuthority, owner: &StreamWriterOwner, ) -> Result<u64> { @@ -245,14 +245,23 @@ async fn exclude_previous( let safe = expiry .saturating_add(MAX_WRITE_CLOCK_SKEW_MS) .saturating_add(MAX_WRITE_REQUEST_AGE_MS); - if wall_ms() >= safe { + if wall_ms() >= safe + && client + .confirm_exclusion(owner.process, wall_ms()) + .await + .map_err(|error| StreamError::ReadUnavailable(error.to_string()))? + .is_some() + { return Ok(revision); } - tokio::time::sleep(Duration::from_millis((safe - wall_ms()).min(500))).await; + tokio::time::sleep(Duration::from_millis( + safe.saturating_sub(wall_ms()).clamp(1, 500), + )) + .await; } } -async fn read_owner(kv: &CrowdbKvClient, name: StreamName) -> Result<Option<(StreamWriterOwner, u64)>> { +async fn read_owner(kv: &PaxosKvClient, name: StreamName) -> Result<Option<(StreamWriterOwner, u64)>> { let key = StreamWriterOwnerKey { stream_name: name }.to_path(); match kv .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) @@ -271,7 +280,7 @@ async fn read_owner(kv: &CrowdbKvClient, name: StreamName) -> Result<Option<(Str } } -async fn write_owner(kv: &CrowdbKvClient, owner: &StreamWriterOwner, revision: u64) -> Result<()> { +async fn write_owner(kv: &PaxosKvClient, owner: &StreamWriterOwner, revision: u64) -> Result<()> { let key = StreamWriterOwnerKey { stream_name: owner.stream_name, } diff --git a/lib/crowdb-chunk-stream/src/kv/record_backend.rs b/lib/crowdb-chunk-stream/src/kv/record_backend.rs index 437de03a4..752259c72 100644 --- a/lib/crowdb-chunk-stream/src/kv/record_backend.rs +++ b/lib/crowdb-chunk-stream/src/kv/record_backend.rs @@ -4,11 +4,13 @@ use crate::perf::{perf, PhaseTimer}; use crate::records::RecordBackend; use async_trait::async_trait; -use crowdb_kv_client::{BatchOp, CrowdbKvClient, GetOutcome, ReadMode, Result, ScanOutcome, WriteOutcome}; +use crowdb_paxos_kv_client::{ + BatchOp, GetOutcome, PaxosKvClient, ReadMode, Result, ScanOutcome, WriteOutcome, +}; use std::sync::Arc; pub(super) struct KvRecordBackend { - pub(super) kv: Arc<CrowdbKvClient>, + pub(super) kv: Arc<PaxosKvClient>, pub(super) store: u64, pub(super) group: u64, } @@ -76,7 +78,7 @@ impl KvRecordBackend { .await }?; if outcome.timed_out || outcome.items.len() != outcome.commit_slots.len() { - return Err(crowdb_kv_client::Error::Server( + return Err(crowdb_paxos_kv_client::Error::Server( "incomplete Stream chunk record scan".into(), )); } diff --git a/lib/crowdb-chunk-stream/src/memory/record_backend.rs b/lib/crowdb-chunk-stream/src/memory/record_backend.rs index 0b3104104..85e782398 100644 --- a/lib/crowdb-chunk-stream/src/memory/record_backend.rs +++ b/lib/crowdb-chunk-stream/src/memory/record_backend.rs @@ -8,7 +8,7 @@ use std::sync::{ use async_trait::async_trait; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, Error, GetOutcome, Result, ScanOutcome, WriteOutcome}; +use crowdb_paxos_kv_client::{BatchOp, Error, GetOutcome, Result, ScanOutcome, WriteOutcome}; use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; use tokio::sync::{Mutex, Notify}; @@ -163,10 +163,12 @@ impl RecordBackend for MemoryRecordBackend { if current != revision { return Err(Error::CasFailed { current_revision: current, - observation: Some(Box::new(crowdb_kv_client::KvCompareObservation::from_versioned( - Bytes::copy_from_slice(key), - observed.map(|record| (record.revision, record.value.clone())), - ))), + observation: Some(Box::new( + crowdb_paxos_kv_client::KvCompareObservation::from_versioned( + Bytes::copy_from_slice(key), + observed.map(|record| (record.revision, record.value.clone())), + ), + )), }); } let next = state diff --git a/lib/crowdb-chunk-stream/src/production.rs b/lib/crowdb-chunk-stream/src/production.rs index 0020dae29..5e64f1e5f 100644 --- a/lib/crowdb-chunk-stream/src/production.rs +++ b/lib/crowdb-chunk-stream/src/production.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use crossbeam_skiplist::SkipMap; use crowdb_chunk_client::{ChunkIoClient, ChunkReadPolicy}; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use crowdb_protocol::chunk_stream::{StreamName, StreamRecord}; use crate::{ @@ -17,7 +17,7 @@ use crate::{ /// Shared production dependencies used to create and reopen stream handles. pub struct ProductionStreamRuntime { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, registry: Arc<KvStreamRegistry>, chunks: Arc<ProductionStreamChunkStore>, config: StreamConfig, @@ -32,7 +32,7 @@ impl ProductionStreamRuntime { /// Returns an error for an invalid writer lease, read policy, or stream /// configuration. pub fn new( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, chunk_io: &ChunkIoClient, writer_lease_ms: u64, read_policy: ChunkReadPolicy, @@ -47,7 +47,7 @@ impl ProductionStreamRuntime { /// /// Returns an error for an invalid writer lease, mirror count, read policy, or stream configuration. pub fn new_with_mirror_copies( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, chunk_io: &ChunkIoClient, writer_lease_ms: u64, read_policy: ChunkReadPolicy, diff --git a/lib/crowdb-chunk-stream/src/records.rs b/lib/crowdb-chunk-stream/src/records.rs index 9784eaeda..c8f85f6c7 100644 --- a/lib/crowdb-chunk-stream/src/records.rs +++ b/lib/crowdb-chunk-stream/src/records.rs @@ -4,7 +4,7 @@ //! Ordered Stream chunk records over conditional KV operations. use crate::{Result, StreamError}; -use crowdb_kv_client::GetOutcome; +use crowdb_paxos_kv_client::GetOutcome; use crowdb_protocol::chunk_stream::{ decode_stream_chunk_record, StreamChunkRecord, StreamChunkState, StreamMappingIndex, StreamName, }; @@ -182,12 +182,12 @@ fn codec_error(error: impl std::fmt::Display) -> StreamError { fn corrupt(reason: &str) -> StreamError { StreamError::Corruption(format!("invalid Stream chunk record: {reason}")) } -fn storage_error(error: crowdb_kv_client::Error) -> StreamError { +fn storage_error(error: crowdb_paxos_kv_client::Error) -> StreamError { match error { - crowdb_kv_client::Error::CasBusy | crowdb_kv_client::Error::CasFailed { .. } => { + crowdb_paxos_kv_client::Error::CasBusy | crowdb_paxos_kv_client::Error::CasFailed { .. } => { StreamError::Backpressure } - crowdb_kv_client::Error::OutcomeUnknown => StreamError::WriteStalled, + crowdb_paxos_kv_client::Error::OutcomeUnknown => StreamError::WriteStalled, other => StreamError::ReadUnavailable(format!("Stream chunk record KV operation failed: {other}")), } } diff --git a/lib/crowdb-chunk-stream/src/records/backend.rs b/lib/crowdb-chunk-stream/src/records/backend.rs index b07f3b562..5300cb16a 100644 --- a/lib/crowdb-chunk-stream/src/records/backend.rs +++ b/lib/crowdb-chunk-stream/src/records/backend.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use async_trait::async_trait; -use crowdb_kv_client::{BatchOp, GetOutcome, Result, ScanOutcome, WriteOutcome}; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome, Result, ScanOutcome, WriteOutcome}; #[async_trait] pub(crate) trait RecordBackend: Send + Sync { diff --git a/lib/crowdb-chunk-stream/src/records/candidate.rs b/lib/crowdb-chunk-stream/src/records/candidate.rs index f942b9378..13cb662b2 100644 --- a/lib/crowdb-chunk-stream/src/records/candidate.rs +++ b/lib/crowdb-chunk-stream/src/records/candidate.rs @@ -4,7 +4,7 @@ use super::{corrupt, publication::put, storage_error, StreamChunkRecordSnapshot, StreamChunkRecordStore}; use crate::{ChunkId, Result, StreamError}; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, Error}; +use crowdb_paxos_kv_client::{BatchOp, Error}; use crowdb_protocol::chunk_stream::{StreamChunkRecord, StreamChunkState, StreamName}; use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; diff --git a/lib/crowdb-chunk-stream/src/records/collection.rs b/lib/crowdb-chunk-stream/src/records/collection.rs index 36f623cf4..801b0ce81 100644 --- a/lib/crowdb-chunk-stream/src/records/collection.rs +++ b/lib/crowdb-chunk-stream/src/records/collection.rs @@ -10,7 +10,7 @@ use super::{ storage_error, StreamChunkRecordStore, }; use crate::{Result, StreamChunkStore, StreamError}; -use crowdb_kv_client::{BatchOp, GetOutcome}; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome}; use crowdb_protocol::chunk_stream::{StreamChunkState, StreamName}; use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; diff --git a/lib/crowdb-chunk-stream/src/records/pin.rs b/lib/crowdb-chunk-stream/src/records/pin.rs index ed56ccd19..4a980d202 100644 --- a/lib/crowdb-chunk-stream/src/records/pin.rs +++ b/lib/crowdb-chunk-stream/src/records/pin.rs @@ -7,7 +7,7 @@ use super::{corrupt, storage_error, StreamChunkRecordSnapshot, StreamChunkRecord use crate::{ChunkId, Result, StreamError}; use arc_swap::ArcSwap; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, GetOutcome}; +use crowdb_paxos_kv_client::{BatchOp, GetOutcome}; use crowdb_protocol::chunk_stream::{StreamName, StreamProcessLease}; use serde::{Deserialize, Serialize}; use std::{ diff --git a/lib/crowdb-chunk-stream/src/records/publication.rs b/lib/crowdb-chunk-stream/src/records/publication.rs index c5a655006..86db81737 100644 --- a/lib/crowdb-chunk-stream/src/records/publication.rs +++ b/lib/crowdb-chunk-stream/src/records/publication.rs @@ -6,7 +6,7 @@ use super::{ }; use crate::{Result, StreamError}; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, Error}; +use crowdb_paxos_kv_client::{BatchOp, Error}; use crowdb_protocol::chunk_stream::{encode_stream_chunk_record, StreamChunkRecord, StreamChunkState}; use crowdb_protocol::key::BinaryKey; diff --git a/lib/crowdb-chunk-stream/tests/common/metadata_server.rs b/lib/crowdb-chunk-stream/tests/common/metadata_server.rs index 875dce289..c8fda3727 100644 --- a/lib/crowdb-chunk-stream/tests/common/metadata_server.rs +++ b/lib/crowdb-chunk-stream/tests/common/metadata_server.rs @@ -3,15 +3,15 @@ use std::sync::Arc; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; pub struct TestMetadataServer { pub server: Arc<PxKvStore>, - pub client: Arc<CrowdbKvClient>, + pub client: Arc<PaxosKvClient>, } impl TestMetadataServer { @@ -24,7 +24,7 @@ impl TestMetadataServer { server.start().await.unwrap(); // This fixture explicitly seeds its one real RPC endpoint; discovery is // outside the metadata adapter contract exercised here. - let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))); + let client = Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))); client.seed_leader(1, 1, server.listen_addr().unwrap().to_string()); Self { server, client } } diff --git a/lib/crowdb-chunk-stream/tests/common/runtime_kv.rs b/lib/crowdb-chunk-stream/tests/common/runtime_kv.rs index 357519e24..cac825a93 100644 --- a/lib/crowdb-chunk-stream/tests/common/runtime_kv.rs +++ b/lib/crowdb-chunk-stream/tests/common/runtime_kv.rs @@ -1,10 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use std::sync::Arc; pub struct TestRuntimeKv(Arc<PxKvStore>); @@ -15,7 +15,7 @@ impl Drop for TestRuntimeKv { } } -pub async fn runtime_kv() -> (TestRuntimeKv, Arc<CrowdbKvClient>) { +pub async fn runtime_kv() -> (TestRuntimeKv, Arc<PaxosKvClient>) { let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); for group in 0..=1 { store.add_group(PxGroup::new( @@ -24,7 +24,7 @@ pub async fn runtime_kv() -> (TestRuntimeKv, Arc<CrowdbKvClient>) { )); } store.start().await.unwrap(); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![]))); for group in 0..=1 { kv.seed_leader(0, group, store.listen_addr().unwrap().to_string()); } diff --git a/lib/crowdb-chunk-stream/tests/ordered_kv_record_test.rs b/lib/crowdb-chunk-stream/tests/ordered_kv_record_test.rs index b9f8c75e9..a8ada40bc 100644 --- a/lib/crowdb-chunk-stream/tests/ordered_kv_record_test.rs +++ b/lib/crowdb-chunk-stream/tests/ordered_kv_record_test.rs @@ -8,7 +8,7 @@ use crowdb_chunk_stream::{ memory::MemoryStreamStore, ChunkStream, KvStreamMetadataStore, StreamConfig, StreamMetadataStore, StreamName, StreamPurpose, StreamRecord, StreamRecordState, }; -use crowdb_kv_client::ReadMode; +use crowdb_paxos_kv_client::ReadMode; use crowdb_protocol::{ chunk_stream::{decode_stream_chunk_record, StreamChunkState}, key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}, @@ -77,7 +77,7 @@ async fn actual_kv_reverse_seek_uses_ordered_keys_and_never_updates_historical_r .get(1, 1, &first.0, ReadMode::Linearizable, None) .await .unwrap(); - let crowdb_kv_client::GetOutcome::Found { value, .. } = first_again else { + let crowdb_paxos_kv_client::GetOutcome::Found { value, .. } = first_again else { panic!("sealed record missing") }; assert_eq!(value, first.1); @@ -140,7 +140,7 @@ async fn actual_kv_collection_uses_bounded_keys_and_releases_only_trimmed_chunks #[tokio::test] async fn checkpoint_publication_waits_for_an_inflight_tail_guard_update() { - use crowdb_kv_client::GetOutcome; + use crowdb_paxos_kv_client::GetOutcome; use std::time::Duration; let server = metadata_server::TestMetadataServer::start().await; @@ -213,7 +213,7 @@ async fn checkpoint_publication_waits_for_an_inflight_tail_guard_update() { #[tokio::test] async fn read_pin_waits_for_an_inflight_tail_guard_update() { - use crowdb_kv_client::GetOutcome; + use crowdb_paxos_kv_client::GetOutcome; use std::time::Duration; let server = metadata_server::TestMetadataServer::start().await; diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs index 16272acd4..fac5c7306 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs @@ -13,7 +13,7 @@ use crowdb_chunk_stream::{ ProductionStreamRuntime, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamRecord, StreamRecordState, StreamRegistry, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::{ AdvanceChunkWriteRequest, AdvanceChunkWriteResponse, AllocateChunkRequest, AllocateChunkResponse, AllocateReplacementSegmentRequest, AllocateReplacementSegmentResponse, AppendChunkRequest, @@ -465,7 +465,7 @@ fn production_runtime_shares_connected_chunk_io_parts() { let disks: Arc<dyn DiskWriter> = Arc::new(Disks::default()); let chunk_io = ChunkIoClient::from_parts(allocator, disks, crowdb_protocol::chunkdb::rpc::ChunkType::Stream); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![ + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![ "http://127.0.0.1:1".into() ]))); diff --git a/lib/crowdb-chunkdb-client/Cargo.toml b/lib/crowdb-chunkdb-client/Cargo.toml index 9e50a8dfe..097e2d516 100644 --- a/lib/crowdb-chunkdb-client/Cargo.toml +++ b/lib/crowdb-chunkdb-client/Cargo.toml @@ -13,7 +13,7 @@ workspace = true [dependencies] crowdb-protocol = { path = "../crowdb-protocol" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } tokio = { workspace = true, features = ["rt", "macros", "time", "sync"] } bytes = "1" diff --git a/lib/crowdb-chunkdb-client/src/client.rs b/lib/crowdb-chunkdb-client/src/client.rs index 77232e373..332092eb4 100644 --- a/lib/crowdb-chunkdb-client/src/client.rs +++ b/lib/crowdb-chunkdb-client/src/client.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use arc_swap::ArcSwap; -use crowdb_kv_client::{RangeBindingClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ AdvanceChunkWriteRequest, AdvanceChunkWriteResponse, AllocateChunkRequest, AllocateChunkResponse, AllocateReplacementSegmentRequest, AllocateReplacementSegmentResponse, AppendChunkRequest, diff --git a/lib/crowdb-chunkdb-client/src/lib.rs b/lib/crowdb-chunkdb-client/src/lib.rs index 4072dad89..678431c9e 100644 --- a/lib/crowdb-chunkdb-client/src/lib.rs +++ b/lib/crowdb-chunkdb-client/src/lib.rs @@ -19,8 +19,8 @@ pub use client::{ChunkdbClient, NativeChunkRoutes, RetryConfig}; pub use rpc_transport::ChunkdbRpcTransport; // Re-export RangeBindingClient so callers can construct it without a -// direct crowdb-kv-client dependency for this one type. -pub use crowdb_kv_client::RangeBindingClient; +// direct crowdb-paxos-kv-client dependency for this one type. +pub use crowdb_paxos_kv_client::RangeBindingClient; use thiserror::Error; diff --git a/lib/crowdb-chunkdb-client/src/rpc_transport.rs b/lib/crowdb-chunkdb-client/src/rpc_transport.rs index b70a7da74..088713811 100644 --- a/lib/crowdb-chunkdb-client/src/rpc_transport.rs +++ b/lib/crowdb-chunkdb-client/src/rpc_transport.rs @@ -552,6 +552,7 @@ impl ChunkdbRpcTransport { id: req_id, rpc_create_nano: 0, chunk_id: chunk_id_off.as_ref(), + fence_missing: false, }; let fb_req = FBDeleteChunkRequest::create(&mut builder, &args); builder.finish(fb_req, None); diff --git a/lib/crowdb-common/cpp/CMakeLists.txt b/lib/crowdb-common/cpp/CMakeLists.txt index 211b5a338..7571a9db1 100644 --- a/lib/crowdb-common/cpp/CMakeLists.txt +++ b/lib/crowdb-common/cpp/CMakeLists.txt @@ -48,13 +48,17 @@ endif() # Gated behind CROWDB_HAVE_SPDLOG (PUBLIC so includers of log.h see the # spdlog branch). Linked PUBLIC because crowdb-common's public headers # (log.h, compressing_sink.h) include spdlog headers — consumers must -# inherit spdlog's INTERFACE_COMPILE_DEFINITIONS (SPDLOG_COMPILED_LIB / -# SPDLOG_SHARED_LIB / SPDLOG_FMT_EXTERNAL) so they use the dylib's -# declarations and registry rather than compiling spdlog in -# SPDLOG_HEADER_ONLY mode (which would give each TU its own inline -# registry, disjoint from crowdbcommon's set_default_logger calls). +# inherit the selected spdlog mode and share its registry with +# crowdbcommon's set_default_logger calls. find_package(spdlog REQUIRED) -target_link_libraries(crowdbcommon PUBLIC spdlog::spdlog) +option(CROWDB_COMMON_SPDLOG_HEADER_ONLY "Compile spdlog with the consumer's instrumentation" OFF) +if(CROWDB_COMMON_SPDLOG_HEADER_ONLY) + # A prebuilt library hides shared_ptr release/acquire operations from TSan. + # Propagate one mode to every consumer so the registry stays consistent. + target_link_libraries(crowdbcommon PUBLIC spdlog::spdlog_header_only) +else() + target_link_libraries(crowdbcommon PUBLIC spdlog::spdlog) +endif() target_compile_definitions(crowdbcommon PUBLIC CROWDB_HAVE_SPDLOG=1) # zlib for gzip compression of rotated log files (compressing_sink). diff --git a/lib/crowdb-common/cpp/include/crowdb-common/diskio_uring.h b/lib/crowdb-common/cpp/include/crowdb-common/diskio_uring.h index 198c69804..c9d90c14a 100644 --- a/lib/crowdb-common/cpp/include/crowdb-common/diskio_uring.h +++ b/lib/crowdb-common/cpp/include/crowdb-common/diskio_uring.h @@ -87,6 +87,31 @@ struct Topology bool attach_wq = false; // IORING_SETUP_ATTACH_WQ }; +// Caller-owned completion storage avoids hidden callable allocations. Hooks +// claim actual entry storage before allocation and release it after free. +struct IOAllocation +{ + std::shared_ptr<const void> owner = nullptr; + void *context = nullptr; + void *(*allocate)(void *, size_t, size_t) = nullptr; + void (*deallocate)(void *, void *, size_t, size_t) noexcept = nullptr; +}; + +struct IOCompletion +{ + void *context = nullptr; + void (*callback)(void *, int) noexcept = nullptr; + std::shared_ptr<const void> owner = nullptr; + IOAllocation allocation{}; + + void complete(int result) const noexcept + { + if (callback != nullptr) { + callback(context, result); + } + } +}; + // DiskIOUring: multi-pipeline io_uring engine. // // fd→pipeline routing: register_fd(fd) assigns fd to a pipeline (auto or @@ -172,6 +197,11 @@ class DiskIOUring submit_fsync(fd, false, std::move(on_complete)); } + void submit_read_owned(int fd, void *buf, size_t len, off_t offset, IOCompletion complete); + void submit_write_owned(int fd, const void *buf, size_t len, off_t offset, IOCompletion complete); + void submit_fsync_owned(int fd, bool data_only, IOCompletion complete); + [[nodiscard]] static size_t owned_submission_allocation_bytes() noexcept; + // Returns one eventfd per pipeline, for the Rust FFI to register with // tokio::io::AsyncFd. Each eventfd becomes readable after the poll // thread dispatches a batch of CQEs on that pipeline. The caller @@ -186,6 +216,7 @@ class DiskIOUring struct CallbackEntry { std::function<void(int)> cb; + IOCompletion owned{}; int fd{-1}; // for in_flight decrement on dispatch CallbackEntry *next_free{nullptr}; // poll-thread-only: deferred-delete list uint64_t queued_nanos{0}; @@ -258,6 +289,13 @@ class DiskIOUring // Lock-free SQE claim on a specific pipeline. void submit_lockfree(Pipeline &p, int fd, std::function<void(int)> on_complete, const Prep &prep); + void submit_entry(Pipeline &pipeline, int fd, CallbackEntry *entry, const void *context, + void (*prepare)(const void *, struct io_uring_sqe *)); + void submit_owned(int fd, void *buffer, size_t length, off_t offset, uint8_t opcode, bool data_only, + IOCompletion complete); + static void complete_entry(CallbackEntry *entry, int result); + static void delete_entry(CallbackEntry *entry) noexcept; + // Publish contiguous filled SQE slots to the kernel for one pipeline. static void publish_ready_sqes(Pipeline &p); diff --git a/lib/crowdb-common/cpp/include/crowdb-common/metrics/inline_callback_gauge.h b/lib/crowdb-common/cpp/include/crowdb-common/metrics/inline_callback_gauge.h new file mode 100644 index 000000000..5418da3e8 --- /dev/null +++ b/lib/crowdb-common/cpp/include/crowdb-common/metrics/inline_callback_gauge.h @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <array> +#include <bit> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <memory_resource> +#include <string> +#include <string_view> +#include <type_traits> + +namespace crowdb::common::metrics +{ +// Trivial callback descriptors stay in the paid registry node. No callable +// target, copied metric name or function wrapper is allocated on the side. +class InlineCallbackGauge +{ + public: + class Callback + { + public: + template <class F> explicit Callback(F function) + { + static_assert(std::is_trivially_copyable_v<F>); + static_assert(sizeof(F) <= kCapacity); + static_assert(std::is_invocable_r_v<uint64_t, const F &>); + std::memcpy(storage_.data(), &function, sizeof(F)); + invoke_ = [](const std::byte *data) { + std::array<std::byte, sizeof(F)> bytes; + std::memcpy(bytes.data(), data, bytes.size()); + const auto function = std::bit_cast<F>(bytes); + return static_cast<uint64_t>(function()); + }; + } + + [[nodiscard]] uint64_t operator()() const + { + return invoke_(storage_.data()); + } + + private: + static constexpr size_t kCapacity = 32; + std::array<std::byte, kCapacity> storage_{}; + uint64_t (*invoke_)(const std::byte *) = nullptr; + }; + + InlineCallbackGauge(std::pmr::string name, Callback callback) : name_(std::move(name)), callback_(callback) + { + } + + [[nodiscard]] uint64_t get() const + { + return callback_(); + } + + [[nodiscard]] const std::pmr::string &name() const + { + return name_; + } + + void replace(Callback callback) noexcept + { + callback_ = callback; + } + + private: + std::pmr::string name_; + Callback callback_; +}; +} // namespace crowdb::common::metrics diff --git a/lib/crowdb-common/cpp/include/crowdb-common/metrics/metrics.h b/lib/crowdb-common/cpp/include/crowdb-common/metrics/metrics.h index 2953f5c98..8078d6fc4 100644 --- a/lib/crowdb-common/cpp/include/crowdb-common/metrics/metrics.h +++ b/lib/crowdb-common/cpp/include/crowdb-common/metrics/metrics.h @@ -22,6 +22,7 @@ #include "crowdb-common/metrics/callback_gauge.h" #include "crowdb-common/metrics/counter.h" #include "crowdb-common/metrics/gauge.h" +#include "crowdb-common/metrics/inline_callback_gauge.h" #include "crowdb-common/metrics/latency_histogram.h" #include "crowdb-common/metrics/latency_summary.h" @@ -29,7 +30,9 @@ #include <condition_variable> #include <cstdint> #include <cstdio> +#include <list> #include <memory> +#include <memory_resource> #include <mutex> #include <string> #include <thread> @@ -43,7 +46,21 @@ namespace crowdb::common::metrics class MetricsRegistry { public: - MetricsRegistry() = default; + MetricsRegistry(); + // The owner outlives every PMR name, node and flush scratch allocation. + explicit MetricsRegistry(std::shared_ptr<std::pmr::memory_resource> resource); + + template <class F> + InlineCallbackGauge *register_callback_gauge_inline(std::string_view prefix, std::string_view suffix, F callback) + { + return register_inline(prefix, suffix, InlineCallbackGauge::Callback(callback)); + } + + template <class F> InlineCallbackGauge *register_callback_gauge_inline(std::string_view name, F callback) + { + return register_callback_gauge_inline(name, {}, callback); + } + ~MetricsRegistry(); MetricsRegistry(const MetricsRegistry &) = delete; @@ -81,6 +98,11 @@ class MetricsRegistry void stop(); private: + InlineCallbackGauge *register_inline(std::string_view prefix, std::string_view suffix, + InlineCallbackGauge::Callback callback); + std::shared_ptr<std::pmr::memory_resource> resource_owner_; + std::pmr::memory_resource *resource_; + std::pmr::list<InlineCallbackGauge> inline_gauges_; std::vector<std::unique_ptr<Counter>> counters_; std::vector<std::unique_ptr<Gauge>> gauges_; std::vector<std::unique_ptr<CallbackGauge>> callback_gauges_; diff --git a/lib/crowdb-common/cpp/include/crowdb-common/metrics/system_metrics.h b/lib/crowdb-common/cpp/include/crowdb-common/metrics/system_metrics.h index 5672c6410..20cfc37df 100644 --- a/lib/crowdb-common/cpp/include/crowdb-common/metrics/system_metrics.h +++ b/lib/crowdb-common/cpp/include/crowdb-common/metrics/system_metrics.h @@ -2,7 +2,7 @@ // OS-level system metrics: CPU time, memory RSS, TCP retransmits, and // DRAM read/write bandwidth. Mirrors the Rust `crowdb_common::metrics::system` -// module so C++ servers (crowdb-diskio) and Rust servers (crowdb-kv-server) +// module so C++ servers (crowdb-diskio) and Rust servers (crowdb-paxos-kv-server) // report the same `sys.*` metric lines. // // On Linux, reads /proc/self/stat for CPU jiffies, /proc/self/status for diff --git a/lib/crowdb-common/cpp/src/diskio_owned.cpp b/lib/crowdb-common/cpp/src/diskio_owned.cpp new file mode 100644 index 000000000..144e29b5f --- /dev/null +++ b/lib/crowdb-common/cpp/src/diskio_owned.cpp @@ -0,0 +1,125 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#ifdef CROWDB_HAVE_LIBURING +# include "crowdb-common/diskio_uring.h" + +# include <cerrno> +# include <chrono> +# include <climits> +# include <new> + +namespace crowdb::common +{ +namespace +{ +struct OwnedCommand +{ + int fd; + void *buffer; + size_t length; + off_t offset; + uint8_t opcode; + bool data_only; +}; + +void prepare_owned(const void *context, struct io_uring_sqe *sqe) +{ + const auto &command = *static_cast<const OwnedCommand *>(context); + if (command.opcode == IORING_OP_READ) { + ::io_uring_prep_read(sqe, command.fd, command.buffer, static_cast<unsigned>(command.length), + static_cast<__u64>(command.offset)); + } + else if (command.opcode == IORING_OP_WRITE) { + ::io_uring_prep_write(sqe, command.fd, command.buffer, static_cast<unsigned>(command.length), + static_cast<__u64>(command.offset)); + } + else { + ::io_uring_prep_fsync(sqe, command.fd, command.data_only ? IORING_FSYNC_DATASYNC : 0); + } +} +} // namespace + +size_t DiskIOUring::owned_submission_allocation_bytes() noexcept +{ + return sizeof(CallbackEntry); +} + +void DiskIOUring::complete_entry(CallbackEntry *entry, int result) +{ + if (entry->owned.callback != nullptr) { + entry->owned.complete(result); + } + else if (entry->cb) { + entry->cb(result); + } +} + +void DiskIOUring::delete_entry(CallbackEntry *entry) noexcept +{ + if (entry->owned.allocation.deallocate == nullptr) { + delete entry; + return; + } + auto allocation = std::move(entry->owned.allocation); + std::destroy_at(entry); + allocation.deallocate(allocation.context, entry, sizeof(CallbackEntry), alignof(CallbackEntry)); +} + +void DiskIOUring::submit_owned(int fd, void *buffer, size_t length, off_t offset, uint8_t opcode, bool data_only, + IOCompletion complete) +{ + if (fd < 0 || fd >= fd_table_size_ || pipelines_.empty()) { + complete.complete(-EBADF); + return; + } + if (complete.callback == nullptr || !complete.allocation.owner || complete.allocation.allocate == nullptr || + complete.allocation.deallocate == nullptr || length > INT_MAX || offset < 0) { + complete.complete(-EINVAL); + return; + } + auto &pipeline = *pipelines_[fd_table_[fd].registered ? fd_table_[fd].pipeline : 0]; + if (!pipeline.valid) { + complete.complete(-EIO); + return; + } + CallbackEntry *entry = nullptr; + try { + void *storage = + complete.allocation.allocate(complete.allocation.context, sizeof(CallbackEntry), alignof(CallbackEntry)); + if (storage == nullptr) { + complete.complete(-ENOMEM); + return; + } + entry = ::new (storage) CallbackEntry{}; + entry->owned = std::move(complete); + entry->fd = fd; + entry->queued_nanos = static_cast<uint64_t>( + std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()) + .count()); + } + catch (const std::bad_alloc &) { + complete.complete(-ENOMEM); + return; + } + const OwnedCommand command{ + .fd = fd, .buffer = buffer, .length = length, .offset = offset, .opcode = opcode, .data_only = data_only}; + submit_entry(pipeline, fd, entry, &command, &prepare_owned); +} + +void DiskIOUring::submit_read_owned(int fd, void *buffer, size_t length, off_t offset, IOCompletion complete) +{ + submit_owned(fd, buffer, length, offset, IORING_OP_READ, false, std::move(complete)); +} + +void DiskIOUring::submit_write_owned(int fd, const void *buffer, size_t length, off_t offset, IOCompletion complete) +{ + submit_owned(fd, const_cast<void *>(buffer), length, offset, IORING_OP_WRITE, false, std::move(complete)); +} + +void DiskIOUring::submit_fsync_owned(int fd, bool data_only, IOCompletion complete) +{ + submit_owned(fd, nullptr, 0, 0, IORING_OP_FSYNC, data_only, std::move(complete)); +} +} // namespace crowdb::common + +#endif // CROWDB_HAVE_LIBURING diff --git a/lib/crowdb-common/cpp/src/diskio_uring.cpp b/lib/crowdb-common/cpp/src/diskio_uring.cpp index 24fbe0c28..f37fd30d4 100644 --- a/lib/crowdb-common/cpp/src/diskio_uring.cpp +++ b/lib/crowdb-common/cpp/src/diskio_uring.cpp @@ -163,8 +163,9 @@ DiskIOUring::DiskIOUring(Topology topo) for (size_t pi : pt->pipelines) { int rc = ::io_uring_register_eventfd(&pipelines_[pi]->ring, pt->wake_fd); if (rc < 0) { - CRB_LOG_ERROR("DiskIOUring: pipeline {} completion eventfd registration failed: {}", pi, - std::strerror(-rc)); + CRB_LOG_ERROR("DiskIOUring: pipeline {} completion eventfd " + "registration failed: {}", + pi, std::strerror(-rc)); poll_threads_.push_back(std::move(pt)); return; } @@ -219,7 +220,7 @@ DiskIOUring::~DiskIOUring() if (p->valid) { while (p->free_list != nullptr) { CallbackEntry *next = p->free_list->next_free; - delete p->free_list; + delete_entry(p->free_list); p->free_list = next; } ::io_uring_queue_exit(&p->ring); @@ -315,7 +316,9 @@ void DiskIOUring::submit_read(int fd, void *buf, size_t len, off_t offset, std:: { if (fd < 0 || fd >= fd_table_size_ || !fd_table_[fd].registered) { if (fd >= 0 && fd < fd_table_size_ && !fd_table_[fd].registered) { - CRB_LOG_WARN("DiskIOUring::submit_read: fd {} not registered, routing to pipeline 0", fd); + CRB_LOG_WARN("DiskIOUring::submit_read: fd {} not registered, routing to " + "pipeline 0", + fd); if (!pipelines_.empty() && pipelines_[0]->valid) { submit_lockfree( *pipelines_[0], fd, std::move(on_complete), [fd, buf, len, offset](struct io_uring_sqe *sqe) { @@ -339,7 +342,9 @@ void DiskIOUring::submit_write(int fd, const void *buf, size_t len, off_t offset { if (fd < 0 || fd >= fd_table_size_ || !fd_table_[fd].registered) { if (fd >= 0 && fd < fd_table_size_ && !fd_table_[fd].registered) { - CRB_LOG_WARN("DiskIOUring::submit_write: fd {} not registered, routing to pipeline 0", fd); + CRB_LOG_WARN("DiskIOUring::submit_write: fd {} not registered, routing " + "to pipeline 0", + fd); if (!pipelines_.empty() && pipelines_[0]->valid) { submit_lockfree( *pipelines_[0], fd, std::move(on_complete), [fd, buf, len, offset](struct io_uring_sqe *sqe) { @@ -424,7 +429,9 @@ void DiskIOUring::submit_fsync(int fd, bool data_only, std::function<void(int)> { if (fd < 0 || fd >= fd_table_size_ || !fd_table_[fd].registered) { if (fd >= 0 && fd < fd_table_size_ && !fd_table_[fd].registered) { - CRB_LOG_WARN("DiskIOUring::submit_fsync: fd {} not registered, routing to pipeline 0", fd); + CRB_LOG_WARN("DiskIOUring::submit_fsync: fd {} not registered, routing " + "to pipeline 0", + fd); if (!pipelines_.empty() && pipelines_[0]->valid) { submit_lockfree(*pipelines_[0], fd, std::move(on_complete), [fd, data_only](struct io_uring_sqe *sqe) { ::io_uring_prep_fsync(sqe, fd, data_only ? IORING_FSYNC_DATASYNC : 0); @@ -463,6 +470,18 @@ void DiskIOUring::submit_lockfree(Pipeline &p, int fd, std::function<void(int)> return; } + // Allocation failure must leave both the queue and in-flight counts + // unchanged; callers may handle it as a rejected submission. + auto *entry = + new CallbackEntry{.cb = std::move(on_complete), .fd = fd, .next_free = {}, .queued_nanos = steady_nanos()}; + + submit_entry(p, fd, entry, &prep, + [](const void *context, struct io_uring_sqe *sqe) { (*static_cast<const Prep *>(context))(sqe); }); +} + +void DiskIOUring::submit_entry(Pipeline &p, int fd, CallbackEntry *entry, const void *context, + void (*prepare)(const void *, struct io_uring_sqe *)) +{ // Increment in-flight count for this fd. if (fd >= 0 && fd < fd_table_size_) { fd_in_flight_[fd].fetch_add(1, std::memory_order_acq_rel); @@ -470,9 +489,6 @@ void DiskIOUring::submit_lockfree(Pipeline &p, int fd, std::function<void(int)> total_in_flight_.fetch_add(1, std::memory_order_relaxed); uring_in_flight.fetch_add(1, std::memory_order_relaxed); - auto *entry = - new CallbackEntry{.cb = std::move(on_complete), .fd = fd, .next_free = {}, .queued_nanos = steady_nanos()}; - // CAS loop: check capacity BEFORE claiming a slot. bool observed_full = false; for (int attempt = 0; attempt < 1000; ++attempt) { @@ -495,7 +511,7 @@ void DiskIOUring::submit_lockfree(Pipeline &p, int fd, std::function<void(int)> unsigned idx = tail & p.ring.sq.ring_mask; struct io_uring_sqe *sqe = &p.ring.sq.sqes[idx << p.sq_shift]; io_uring_initialize_sqe(sqe); - prep(sqe); + prepare(context, sqe); entry->opcode = sqe->opcode; io_uring_sqe_set_data(sqe, entry); p.sqe_ready[idx].store(true, std::memory_order_release); @@ -505,10 +521,8 @@ void DiskIOUring::submit_lockfree(Pipeline &p, int fd, std::function<void(int)> } // All retries failed (SQ persistently full). - if (entry->cb) { - entry->cb(-ENOMEM); - } - delete entry; + complete_entry(entry, -ENOMEM); + delete_entry(entry); if (fd >= 0 && fd < fd_table_size_) { fd_in_flight_[fd].fetch_sub(1, std::memory_order_acq_rel); } @@ -569,7 +583,7 @@ void DiskIOUring::poll_thread_run(PollThread &pt) } while (p.free_list != nullptr) { CallbackEntry *next = p.free_list->next_free; - delete p.free_list; + delete_entry(p.free_list); p.free_list = next; } // Publish contiguous filled SQE slots. @@ -679,11 +693,17 @@ bool DiskIOUring::drain_cqes(Pipeline &p) } total_in_flight_.fetch_sub(1, std::memory_order_relaxed); uring_in_flight.fetch_sub(1, std::memory_order_relaxed); - if (entry->cb) { - entry->cb(res); + const bool owned = entry->owned.callback != nullptr; + complete_entry(entry, res); + if (owned) { + // Returning paid storage cannot wait for an arbitrarily long + // batch of completions before the poll loop becomes idle. + delete_entry(entry); + } + else { + entry->next_free = p.free_list; + p.free_list = entry; } - entry->next_free = p.free_list; - p.free_list = entry; } cqe = nullptr; diff --git a/lib/crowdb-common/cpp/src/metrics/metrics.cpp b/lib/crowdb-common/cpp/src/metrics/metrics.cpp index b3581c5d6..5feb46179 100644 --- a/lib/crowdb-common/cpp/src/metrics/metrics.cpp +++ b/lib/crowdb-common/cpp/src/metrics/metrics.cpp @@ -291,21 +291,33 @@ void MetricsRegistry::flush_to(FILE *fp, double window_secs, const char * /*time // Gauges (always printed, even if 0). Regular gauges and callback // gauges are merged into one sorted list for a unified output section. - if (!gauges_.empty() || !callback_gauges_.empty()) { + if (!gauges_.empty() || !callback_gauges_.empty() || !inline_gauges_.empty()) { // Collect (name, value) pairs from both gauge types, then sort. - std::vector<std::pair<std::string, uint64_t>> all_gauges; - all_gauges.reserve(gauges_.size() + callback_gauges_.size()); + struct GaugeValue + { + const char *name; + size_t length; + uint64_t value; + }; + + std::pmr::vector<GaugeValue> all_gauges(resource_); + all_gauges.reserve(gauges_.size() + callback_gauges_.size() + inline_gauges_.size()); for (const auto &g : gauges_) { - all_gauges.emplace_back(g->name(), g->get()); + all_gauges.push_back({.name = g->name().c_str(), .length = g->name().size(), .value = g->get()}); } for (const auto &g : callback_gauges_) { - all_gauges.emplace_back(g->name(), g->get()); + all_gauges.push_back({.name = g->name().c_str(), .length = g->name().size(), .value = g->get()}); } - std::sort(all_gauges.begin(), all_gauges.end(), [](const auto &a, const auto &b) { return a.first < b.first; }); + for (const auto &g : inline_gauges_) { + all_gauges.push_back({.name = g.name().c_str(), .length = g.name().size(), .value = g.get()}); + } + std::sort(all_gauges.begin(), all_gauges.end(), [](const auto &a, const auto &b) { + return std::string_view(a.name, a.length) < std::string_view(b.name, b.length); + }); std::fprintf(fp, "%-*s value\n", static_cast<int>(name_w), ""); - for (const auto &[name, value] : all_gauges) { - std::fprintf(fp, "%-*s %5llu\n", static_cast<int>(name_w), name.c_str(), - static_cast<unsigned long long>(value)); + for (const auto &entry : all_gauges) { + std::fprintf(fp, "%-*s %5llu\n", static_cast<int>(name_w), entry.name, + static_cast<unsigned long long>(entry.value)); } } @@ -333,6 +345,9 @@ size_t MetricsRegistry::max_name_len() const for (const auto &e : callback_gauges_) { max_len = std::max(max_len, e->name().size()); } + for (const auto &entry : inline_gauges_) { + max_len = std::max(max_len, entry.name().size()); + } return max_len; } diff --git a/lib/crowdb-common/cpp/src/metrics/metrics_inline.cpp b/lib/crowdb-common/cpp/src/metrics/metrics_inline.cpp new file mode 100644 index 000000000..b3a8178af --- /dev/null +++ b/lib/crowdb-common/cpp/src/metrics/metrics_inline.cpp @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/metrics/metrics.h" + +#include <limits> +#include <stdexcept> + +namespace crowdb::common::metrics +{ +MetricsRegistry::MetricsRegistry() : resource_(std::pmr::get_default_resource()), inline_gauges_(resource_) +{ +} + +MetricsRegistry::MetricsRegistry(std::shared_ptr<std::pmr::memory_resource> resource) + : resource_owner_(std::move(resource)), + resource_(resource_owner_ ? resource_owner_.get() : std::pmr::get_default_resource()), + inline_gauges_(resource_) +{ +} + +InlineCallbackGauge *MetricsRegistry::register_inline(std::string_view prefix, std::string_view suffix, + InlineCallbackGauge::Callback callback) +{ + if (suffix.size() > std::numeric_limits<size_t>::max() - prefix.size()) { + throw std::length_error("metrics name length overflow"); + } + std::scoped_lock lock(flush_mutex_); + for (auto &entry : inline_gauges_) { + const std::string_view name(entry.name()); + if (name.size() == prefix.size() + suffix.size() && name.starts_with(prefix) && name.ends_with(suffix)) { + entry.replace(callback); + return &entry; + } + } + std::pmr::string name(resource_); + name.reserve(prefix.size() + suffix.size()); + name.append(prefix); + name.append(suffix); + inline_gauges_.emplace_back(std::move(name), callback); + return &inline_gauges_.back(); +} +} // namespace crowdb::common::metrics diff --git a/lib/crowdb-common/cpp/tests/diskio_owned_test.cpp b/lib/crowdb-common/cpp/tests/diskio_owned_test.cpp new file mode 100644 index 000000000..8be0d9485 --- /dev/null +++ b/lib/crowdb-common/cpp/tests/diskio_owned_test.cpp @@ -0,0 +1,168 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#ifdef CROWDB_HAVE_LIBURING +# include "crowdb-common/diskio_uring.h" +# include "crowdb-common/runtime_path.h" + +# include <fcntl.h> +# include <gtest/gtest.h> +# include <unistd.h> + +# include <array> +# include <chrono> +# include <filesystem> + +namespace crowdb::common +{ +namespace +{ +struct TestAllocation +{ + size_t limit = 0; + std::atomic<size_t> used{0}; + std::atomic<size_t> acquired{0}; + std::atomic<size_t> freed{0}; + + static void *allocate(void *context, size_t bytes, size_t alignment) + { + auto &self = *static_cast<TestAllocation *>(context); + if (bytes > self.limit) { + throw std::bad_alloc(); + } + auto *storage = ::operator new(bytes, std::align_val_t{alignment}); + self.used.fetch_add(bytes); + self.acquired.fetch_add(1); + return storage; + } + + static void deallocate(void *context, void *storage, size_t bytes, size_t alignment) noexcept + { + auto &self = *static_cast<TestAllocation *>(context); + ::operator delete(storage, std::align_val_t{alignment}); + self.used.fetch_sub(bytes); + self.freed.fetch_add(1); + } +}; + +struct TestCompletion +{ + std::atomic<int> result{INT_MIN}; + std::atomic<size_t> live_at_callback{0}; + std::shared_ptr<TestAllocation> allocations; + + static void done(void *context, int result) noexcept + { + auto &self = *static_cast<TestCompletion *>(context); + self.live_at_callback.store(self.allocations->used.load()); + self.result.store(result, std::memory_order_release); + } + + IOCompletion completion() + { + return { + .context = this, + .callback = &done, + .owner = {}, + .allocation = {.owner = allocations, + .context = allocations.get(), + .allocate = &TestAllocation::allocate, + .deallocate = &TestAllocation::deallocate} + }; + } +}; + +struct TestFile +{ + std::filesystem::path path; + int fd = -1; + + TestFile() + { + auto root = test_runtime_path("owned-uring"); + std::filesystem::create_directories(root); + auto name = (root / "io-XXXXXX").string(); + fd = ::mkstemp(name.data()); + path = name; + } + + ~TestFile() + { + if (fd >= 0) { + ::close(fd); + } + std::filesystem::remove(path); + } +}; + +template <class F> bool wait_until(F predicate) +{ + for (size_t i = 0; i < 200; ++i) { + if (predicate()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return predicate(); +} + +TEST(DiskIOOwned, ExactEntryStorageRemainsChargedThroughCallbackAndFreesAfterwards) +{ + TestFile file; + ASSERT_GE(file.fd, 0); + Topology topology; + topology.pipelines.push_back({.entries = 16, .mode = PollingMode::Classic}); + DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + uring.register_fd(file.fd); + auto budget = std::make_shared<TestAllocation>(); + budget->limit = DiskIOUring::owned_submission_allocation_bytes(); + TestCompletion completed; + completed.allocations = budget; + const std::array<uint8_t, 3> expected{1, 2, 3}; + uring.submit_write_owned(file.fd, expected.data(), expected.size(), 0, completed.completion()); + ASSERT_TRUE(wait_until([&] { return completed.result.load(std::memory_order_acquire) != INT_MIN; })); + EXPECT_EQ(completed.result.load(), 3); + EXPECT_EQ(completed.live_at_callback.load(), budget->limit); + ASSERT_TRUE(wait_until([&] { return budget->used.load() == 0; })); + EXPECT_EQ(budget->acquired.load(), 1U); + EXPECT_EQ(budget->freed.load(), 1U); + completed.result.store(INT_MIN); + std::array<uint8_t, 3> actual{}; + uring.submit_read_owned(file.fd, actual.data(), actual.size(), 0, completed.completion()); + ASSERT_TRUE(wait_until([&] { return completed.result.load(std::memory_order_acquire) != INT_MIN; })); + EXPECT_EQ(completed.result.load(), 3); + EXPECT_EQ(actual, expected); + ASSERT_TRUE(wait_until([&] { return budget->used.load() == 0; })); + completed.result.store(INT_MIN); + uring.submit_fsync_owned(file.fd, true, completed.completion()); + ASSERT_TRUE(wait_until([&] { return completed.result.load(std::memory_order_acquire) != INT_MIN; })); + EXPECT_EQ(completed.result.load(), 0); + ASSERT_TRUE(wait_until([&] { return budget->used.load() == 0; })); + EXPECT_EQ(budget->acquired.load(), 3U); + EXPECT_EQ(budget->freed.load(), 3U); +} + +TEST(DiskIOOwned, EntryQuotaFailureDoesNotPublishOrLeakInflight) +{ + TestFile file; + ASSERT_GE(file.fd, 0); + Topology topology; + topology.pipelines.push_back({.entries = 16, .mode = PollingMode::Classic}); + DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + uring.register_fd(file.fd); + TestCompletion completed; + completed.allocations = std::make_shared<TestAllocation>(); + completed.allocations->limit = DiskIOUring::owned_submission_allocation_bytes() - 1; + const uint8_t byte = 1; + uring.submit_write_owned(file.fd, &byte, 1, 0, completed.completion()); + EXPECT_EQ(completed.result.load(), -ENOMEM); + EXPECT_EQ(completed.allocations->used.load(), 0U); + EXPECT_EQ(completed.allocations->acquired.load(), 0U); + EXPECT_EQ(uring.total_in_flight_count(), 0U); + EXPECT_EQ(uring.in_flight_count(file.fd), 0U); + EXPECT_EQ(::lseek(file.fd, 0, SEEK_END), 0); +} +} // namespace +} // namespace crowdb::common +#endif // CROWDB_HAVE_LIBURING diff --git a/lib/crowdb-common/rust/src/lib.rs b/lib/crowdb-common/rust/src/lib.rs index 29e002fbe..89323bf33 100644 --- a/lib/crowdb-common/rust/src/lib.rs +++ b/lib/crowdb-common/rust/src/lib.rs @@ -2,10 +2,10 @@ // Licensed under the Apache License, Version 2.0. //! Shared utilities for the `crow` storage-system components -//! (`crowdb_kv`, `crowdb-tree`, future `crowdb-*` libs). +//! (`crowdb_paxos_kv`, `crowdb-tree`, future `crowdb-*` libs). //! //! Re-exports the generic, project-agnostic primitives extracted out of -//! `crowdb_kv` and `crowdb-tree` so each component depends on `crowdb-common` +//! `crowdb_paxos_kv` and `crowdb-tree` so each component depends on `crowdb-common` //! instead of re-implementing or vendoring them: //! //! - [`metrics`] — lightweight atomic counters, gauges, bandwidth, diff --git a/lib/crowdb-common/rust/src/logging.rs b/lib/crowdb-common/rust/src/logging.rs index fc0a41487..1ac03f1c0 100644 --- a/lib/crowdb-common/rust/src/logging.rs +++ b/lib/crowdb-common/rust/src/logging.rs @@ -172,7 +172,7 @@ pub fn timestamp_secs() -> String { /// /// If `RUST_LOG` is set and non-empty, the first global directive (the /// part before any `=`) is used — e.g. `RUST_LOG=debug` → `"debug"`, -/// `RUST_LOG=crowdb_kv=info` → `"info"`. If `RUST_LOG` is unset or +/// `RUST_LOG=crowdb_paxos_kv=info` → `"info"`. If `RUST_LOG` is unset or /// empty, `fallback` is used. If the derived level is not a valid /// spdlog level, `"info"` is returned. #[must_use] @@ -250,7 +250,7 @@ fn civil_from_days(days: u64) -> (i64, i64, i64) { /// /// `default_filter` is the `EnvFilter` directive string used when /// `RUST_LOG` is unset — callers pass their own project-specific -/// default (e.g. `crowdb_kv` passes its `crowdb_kv`/`crowdb_kv_server`/... list) +/// default (e.g. `crowdb_paxos_kv` passes its `crowdb_paxos_kv`/`crowdb_paxos_kv_server`/... list) /// so the shared library does not bake in one project's targets. /// /// # Errors @@ -424,7 +424,7 @@ pub fn init_file_and_console_logging_split( /// /// `default_filter` is the `EnvFilter` directive string used when /// `RUST_LOG` is unset — callers pass their own project-specific -/// default (e.g. `crowdb_kv` passes its `crowdb_kv`/`crowdb_kv_server`/... list) +/// default (e.g. `crowdb_paxos_kv` passes its `crowdb_paxos_kv`/`crowdb_paxos_kv_server`/... list) /// so the shared library does not bake in one project's targets. /// /// # Errors @@ -495,7 +495,7 @@ mod tests { #[test] fn cpp_level_target_directive_uses_fallback() { let _guard = rust_log_lock().lock().unwrap(); - std::env::set_var("RUST_LOG", "crowdb_kv=info"); + std::env::set_var("RUST_LOG", "crowdb_paxos_kv=info"); assert_eq!(cpp_level_from_rust_log("info"), "info"); std::env::remove_var("RUST_LOG"); } diff --git a/lib/crowdb-common/rust/src/metrics.rs b/lib/crowdb-common/rust/src/metrics.rs index 63ff8003c..78d62584b 100644 --- a/lib/crowdb-common/rust/src/metrics.rs +++ b/lib/crowdb-common/rust/src/metrics.rs @@ -20,6 +20,8 @@ pub mod name; pub mod point; pub mod registry; pub mod runner; +mod sampler; +pub use sampler::MetricsSampler; pub mod timestamp; pub use bandwidth::{Bandwidth, BandwidthSnapshot}; diff --git a/lib/crowdb-common/rust/src/metrics/runner.rs b/lib/crowdb-common/rust/src/metrics/runner.rs index 4fc7b3180..a5ad1ff47 100644 --- a/lib/crowdb-common/rust/src/metrics/runner.rs +++ b/lib/crowdb-common/rust/src/metrics/runner.rs @@ -8,6 +8,7 @@ use std::time::Instant; use super::registry::MetricsRegistry; use super::system::{flush_system, SystemCollector}; use super::timestamp::iso8601_now; +use super::MetricsSampler; /// Lifecycle manager for the metrics registry. Owns the registry, /// the metrics log file writer, and the tokio interval task. @@ -17,7 +18,9 @@ use super::timestamp::iso8601_now; pub struct MetricsRunner { registry: Arc<Mutex<MetricsRegistry>>, writer: Arc<Mutex<crate::logging::RotatingLogWriter>>, - task: Option<tokio::task::JoinHandle<()>>, + task: Option<tokio::task::JoinHandle<Option<Box<dyn MetricsSampler>>>>, + sampler: Option<Box<dyn MetricsSampler>>, + stop: Option<tokio::sync::oneshot::Sender<()>>, interval_secs: f64, system_collector: Arc<std::sync::Mutex<SystemCollector>>, collector: Option<Arc<dyn Fn() + Send + Sync>>, @@ -37,6 +40,8 @@ impl MetricsRunner { registry: Arc::new(Mutex::new(MetricsRegistry::new())), writer: Arc::new(Mutex::new(file)), task: None, + sampler: None, + stop: None, #[allow(clippy::cast_precision_loss)] interval_secs: interval_secs as f64, system_collector: Arc::new(std::sync::Mutex::new(SystemCollector::new())), @@ -80,6 +85,15 @@ impl MetricsRunner { self.cpp_flush = Some(Arc::new(f)); } + /// Install a sampler before `start`. It owns its windows on the metrics + /// task and runs every 100 ms independently of the log flush interval. + /// # Panics + /// Panics if this runner is already running. + pub fn set_sampler(&mut self, sampler: impl MetricsSampler + 'static) { + assert!(self.task.is_none(), "install the sampler before starting metrics"); + self.sampler = Some(Box::new(sampler)); + } + /// Set a negotiate callback to query C++ for its preferred column /// widths. Called before each flush. Returns (`count_w`, `tps_w`). pub fn set_cpp_negotiate(&mut self, f: impl Fn() -> (usize, usize) + Send + Sync + 'static) { @@ -89,6 +103,8 @@ impl MetricsRunner { /// Start the periodic flush task. The first tick fires immediately /// and is skipped (no data yet); subsequent ticks flush with the /// real elapsed time since the previous flush as the window. + /// # Panics + /// Panics if this runner is already running or the log interval is zero. pub fn start(&mut self) { let registry = Arc::clone(&self.registry); let writer = Arc::clone(&self.writer); @@ -99,7 +115,13 @@ impl MetricsRunner { let cpp_negotiate = self.cpp_negotiate.clone(); let last_flush = Arc::clone(&self.last_flush); + assert!(self.task.is_none(), "metrics runner already started"); + let mut sampler = self.sampler.take(); + let (stop, mut stopped) = tokio::sync::oneshot::channel(); + self.stop = Some(stop); self.task = Some(tokio::spawn(async move { + let mut sample_ticks = tokio::time::interval(std::time::Duration::from_millis(100)); + sample_ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut ticker = tokio::time::interval(std::time::Duration::from_secs_f64(interval)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); ticker.tick().await; @@ -107,7 +129,14 @@ impl MetricsRunner { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Instant::now()); loop { - ticker.tick().await; + tokio::select! { + _ = &mut stopped => break, + _ = sample_ticks.tick(), if sampler.is_some() => { + if let Some(sampler) = &mut sampler { sampler.sample(); } + continue; + } + _ = ticker.tick() => {} + } let now = Instant::now(); let window_secs = last_flush .lock() @@ -135,6 +164,9 @@ impl MetricsRunner { let snap = sc.collect(); flush_system(&mut *w, &snap); } + if let Some(sampler) = &mut sampler { + let _ = sampler.flush(&mut *w, &ts); + } if let Some(ref cpp) = cpp_flush { cpp(&mut *w, window_secs, &ts, rust_max, count_w, tps_w); let _ = w.flush(); @@ -142,15 +174,18 @@ impl MetricsRunner { } } } + sampler })); } /// Stop the flush task and perform a final flush with the real /// elapsed time since the last periodic flush. pub async fn stop(&mut self) { + if let Some(stop) = self.stop.take() { + let _ = stop.send(()); + } if let Some(task) = self.task.take() { - task.abort(); - let _ = task.await; + self.sampler = task.await.ok().flatten(); } if let Some(ref col) = self.collector { col(); @@ -176,6 +211,9 @@ impl MetricsRunner { let snap = sc.collect(); flush_system(&mut *w, &snap); } + if let Some(sampler) = &mut self.sampler { + let _ = sampler.flush(&mut *w, &ts); + } if let Some(ref cpp) = self.cpp_flush { cpp(&mut *w, window_secs, &ts, rust_max, count_w, tps_w); } diff --git a/lib/crowdb-common/rust/src/metrics/sampler.rs b/lib/crowdb-common/rust/src/metrics/sampler.rs new file mode 100644 index 000000000..1f8a1bc3c --- /dev/null +++ b/lib/crowdb-common/rust/src/metrics/sampler.rs @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +/// Owned by the metrics task, sampled every 100 ms independently of the log +/// period. Sampling only reads bounded diagnostics, without backend I/O or +/// waiting for application work. Flushing writes the metrics log. No +/// application hot path accesses this state. +pub trait MetricsSampler: Send { + fn sample(&mut self); + /// Writes and advances the owned observation window. + /// # Errors + /// Returns an output error if the metrics writer rejects the window. + fn flush(&mut self, writer: &mut dyn std::io::Write, timestamp: &str) -> std::io::Result<()>; +} diff --git a/lib/crowdb-common/rust/tests/metrics_sampler_test.rs b/lib/crowdb-common/rust/tests/metrics_sampler_test.rs new file mode 100644 index 000000000..16a9a8c48 --- /dev/null +++ b/lib/crowdb-common/rust/tests/metrics_sampler_test.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_common::metrics::{MetricsRunner, MetricsSampler}; +use crowdb_test_harness::test_dirs::tempdir_in_test_data; + +struct TestSampler { + samples: usize, + ready: Arc<tokio::sync::Notify>, +} + +impl MetricsSampler for TestSampler { + fn sample(&mut self) { + self.samples += 1; + if self.samples >= 2 { + self.ready.notify_one(); + } + } + + fn flush(&mut self, writer: &mut dyn std::io::Write, _: &str) -> std::io::Result<()> { + writeln!(writer, "resource_test samples={}", self.samples) + } +} + +#[tokio::test] +async fn sampling_runs_between_log_periods_and_shutdown_preserves_owned_window() { + let dir = tempdir_in_test_data("metrics-resource-sampler"); + let file = crowdb_common::logging::open_metrics_log(dir.path(), "test", 10, 2).unwrap(); + let ready = Arc::new(tokio::sync::Notify::new()); + let mut runner = MetricsRunner::new(file, 60); + runner.set_sampler(TestSampler { + samples: 0, + ready: ready.clone(), + }); + runner.start(); + tokio::time::timeout(std::time::Duration::from_secs(5), ready.notified()) + .await + .unwrap(); + runner.stop().await; + let path = std::fs::read_dir(dir.path()) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let log = std::fs::read_to_string(path).unwrap(); + let samples: usize = log + .lines() + .find_map(|line| line.strip_prefix("resource_test samples=")) + .unwrap() + .parse() + .unwrap(); + assert!(samples >= 2, "{log}"); + assert_eq!(log.matches("resource_test").count(), 1, "{log}"); +} diff --git a/lib/crowdb-console-shared/Cargo.toml b/lib/crowdb-console-shared/Cargo.toml index b89b527da..cef6fdc81 100644 --- a/lib/crowdb-console-shared/Cargo.toml +++ b/lib/crowdb-console-shared/Cargo.toml @@ -32,20 +32,20 @@ dirs = "5" russh = "0.45" async-trait = "0.1" crowdb-protocol = { path = "../crowdb-protocol" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } [dev-dependencies] bincode = "1" clap = { version = "4", features = ["derive"] } -crowdb-kv-server = { path = "../../app/crowdb-kv-server" } -crowdb-kv = { path = "../crowdb-kv" } +crowdb-paxos-kv-server = { path = "../../app/crowdb-paxos-kv-server" } +crowdb-paxos-kv = { path = "../crowdb-paxos-kv" } crowdb-console-shared = { path = ".", features = ["test-util"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "net", "signal"] } axum = "0.7" # KV data-plane crowdb-rpc client for `tests/kv_e2e_test.rs` (C6's own crowdb-rpc # `KvClient` wrapper is gone; that test now exercises the real -# `crowdb-kv-client` crate end-to-end instead). -crowdb-kv-client = { path = "../crowdb-kv-client" } +# `crowdb-paxos-kv-client` crate end-to-end instead). +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-chunkdb-client = { path = "../crowdb-chunkdb-client" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../crowdb-test-harness" } diff --git a/lib/crowdb-console-shared/src/clients.rs b/lib/crowdb-console-shared/src/clients.rs index 840d7cf96..a8d22f9f4 100644 --- a/lib/crowdb-console-shared/src/clients.rs +++ b/lib/crowdb-console-shared/src/clients.rs @@ -3,17 +3,17 @@ //! Transport clients used across the console. //! -//! - `http`: low-level management API on a single `crowdb-kv-server`. +//! - `http`: low-level management API on a single `crowdb-paxos-kv-server`. //! Used by `crowdb-web` to fan out per-node primitives. //! - `console`: high-level two-tree API on a `crowdb-web`. Used by -//! the CLI, which never talks to a `crowdb-kv-server` directly. +//! the CLI, which never talks to a `crowdb-paxos-kv-server` directly. //! //! KV data-plane access used to live here too (`rpc::KvClient`, //! C6). It's gone: `crowdb-web`'s KV handlers and `crowdb-cli`'s `kv` -//! commands and bench runner all depend on the standalone `crowdb-kv-client` +//! commands and bench runner all depend on the standalone `crowdb-paxos-kv-client` //! crate instead for topology //! discovery, retry, and connection pooling on top of the same generated -//! `crowdb_kv::rpc` types. +//! `crowdb_paxos_kv::rpc` types. pub mod console; pub mod http; diff --git a/lib/crowdb-console-shared/src/clients/console.rs b/lib/crowdb-console-shared/src/clients/console.rs index 9687ee96f..bbdc73bf1 100644 --- a/lib/crowdb-console-shared/src/clients/console.rs +++ b/lib/crowdb-console-shared/src/clients/console.rs @@ -3,7 +3,7 @@ //! HTTP client for `crowdb-web`'s two-tree API contract. //! -//! `ServerClient` (sibling) targets a single `crowdb-kv-server`'s +//! `ServerClient` (sibling) targets a single `crowdb-paxos-kv-server`'s //! management API and is used by `crowdb-web` itself. `ConsoleClient` //! lives one layer higher: it talks to `crowdb-web` over the public //! `/api/...` surface that the SPA and CLI both consume. @@ -24,7 +24,7 @@ use crate::config::{NodeEntry, RackEntry, ServerEntry}; use crate::error::{Error, Result}; /// Thin wrapper around `reqwest::Client` bound to one `crowdb-web` -/// console base URL (default `http://127.0.0.1:9090`). +/// console base URL (default `http://127.0.0.1:6135`). #[derive(Debug, Clone)] pub struct ConsoleClient { base_url: String, @@ -115,7 +115,7 @@ pub struct DeployNodeServerBody { /// `--send-queue-capacity` value for per-connection send queue. #[serde(default, skip_serializing_if = "Option::is_none")] pub send_queue_capacity: Option<u32>, - /// Optional `--config` JSON path passed to the spawned `crowdb-kv-server`. + /// Optional `--config` JSON path passed to the spawned `crowdb-paxos-kv-server`. #[serde(default, skip_serializing_if = "Option::is_none")] pub config: Option<String>, } @@ -202,7 +202,7 @@ pub struct KvScanResponse { pub truncated: bool, } -/// One row of `GET /api/servers` — a deployed `crowdb-kv-server` projected +/// One row of `GET /api/servers` — a deployed `crowdb-paxos-kv-server` projected /// from the console config plus the monitor cache. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ServerSummary { @@ -441,7 +441,7 @@ impl ConsoleClient { self.get_json("/api/servers").await } - /// Restart the `crowdb-kv-server` running on `node_id`. Stops the + /// Restart the `crowdb-paxos-kv-server` running on `node_id`. Stops the /// tracked process (if any) and re-deploys on the same ports /// recorded in the console config. /// @@ -728,7 +728,7 @@ impl ConsoleClient { // ── HTTP plumbing ────────────────────────────────────────────── // - // Every helper attaches the current `x-crowdb-kv-corr-id` header (see + // Every helper attaches the current `x-crowdb-paxos-kv-corr-id` header (see // `crate::corr_id`) and emits one `log_ops_http` tracing record on // completion. The id comes from the task-local if a `corr_id::scope` // wraps the call (web handler / CLI main), otherwise we generate @@ -907,7 +907,7 @@ mod tests { #[test] fn trims_trailing_slashes_in_base_url() { - let c = ConsoleClient::new("http://127.0.0.1:14000///").unwrap(); - assert_eq!(c.base_url(), "http://127.0.0.1:14000"); + let c = ConsoleClient::new("http://127.0.0.1:6135///").unwrap(); + assert_eq!(c.base_url(), "http://127.0.0.1:6135"); } } diff --git a/lib/crowdb-console-shared/src/clients/http.rs b/lib/crowdb-console-shared/src/clients/http.rs index afdf1cc2c..67bbca084 100644 --- a/lib/crowdb-console-shared/src/clients/http.rs +++ b/lib/crowdb-console-shared/src/clients/http.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! HTTP client for `crowdb-kv-server`'s management API. +//! HTTP client for `crowdb-paxos-kv-server`'s management API. //! //! Endpoints used in C1: `/health`, `/topology`. Other endpoints are //! exposed as additional methods so later phases (`store add`, etc.) can @@ -12,8 +12,8 @@ use std::time::Duration; use crate::error::{Error, Result}; use crate::snapshot::{HealthInfo, MetricsResponse, StoreView, TopologyResponse, WipeResult}; -/// Thin wrapper around a `reqwest::Client` bound to one `crowdb-kv-server` -/// management base URL (e.g. `http://127.0.0.1:10000`). +/// Thin wrapper around a `reqwest::Client` bound to one `crowdb-paxos-kv-server` +/// management base URL (e.g. `http://127.0.0.1:10135`). #[derive(Debug, Clone)] pub struct ServerClient { base_url: String, diff --git a/lib/crowdb-console-shared/src/cluster.rs b/lib/crowdb-console-shared/src/cluster.rs index ba47c203f..3b6c66a0a 100644 --- a/lib/crowdb-console-shared/src/cluster.rs +++ b/lib/crowdb-console-shared/src/cluster.rs @@ -60,12 +60,12 @@ pub struct Node { /// `127.0.0.1` by default for simulated clusters. pub host: String, pub ssh: SshCreds, - /// 0 or 1 `crowdb-kv-server` per node, enforced console-side. + /// 0 or 1 `crowdb-paxos-kv-server` per node, enforced console-side. #[serde(default, skip_serializing_if = "Option::is_none")] pub server: Option<ServerProcess>, } -/// Console's view of one deployed `crowdb-kv-server` process. +/// Console's view of one deployed `crowdb-paxos-kv-server` process. /// /// `mgmt_url` / `rpc_url` are the *intended* endpoints (persisted); /// `pid` / `state` / `health` / `last_seen_ms` are live cache filled by @@ -86,7 +86,7 @@ pub struct ServerProcess { pub last_seen_ms: u64, } -/// Lifecycle state of the `crowdb-kv-server` process (console's view). +/// Lifecycle state of the `crowdb-paxos-kv-server` process (console's view). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ProcState { @@ -114,7 +114,7 @@ pub enum NodeHealth { // ── Per-node store / group / replica (physical "debugging" view) ──── -/// What `crowdb-kv-server` reports for one local `PxStore` on one node. +/// What `crowdb-paxos-kv-server` reports for one local `PxStore` on one node. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct NodeStore { pub node_id: NodeId, @@ -152,7 +152,7 @@ pub struct LocalReplicaInfo { pub replica_id: ReplicaId, pub role: ReplicaRole, pub state: ReplicaState, - /// Mirrors `crowdb_kv`'s `KvStoreStatus::engine_healthy`, threaded through + /// Mirrors `crowdb_paxos_kv`'s `KvStoreStatus::engine_healthy`, threaded through /// from `crate::snapshot::KvStoreView` by /// `crate::monitor::legacy_topology_to_node_stores`. #[serde(default = "default_engine_healthy")] @@ -186,7 +186,7 @@ pub enum ReplicaRole { Unknown, } -/// Operational status of a replica (mirrors `crowdb-kv-server`'s reporting). +/// Operational status of a replica (mirrors `crowdb-paxos-kv-server`'s reporting). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ReplicaState { diff --git a/lib/crowdb-console-shared/src/cluster_deployer.rs b/lib/crowdb-console-shared/src/cluster_deployer.rs index 9b8b9b1d1..bed318626 100644 --- a/lib/crowdb-console-shared/src/cluster_deployer.rs +++ b/lib/crowdb-console-shared/src/cluster_deployer.rs @@ -354,12 +354,12 @@ impl CrowdbClusterDeployer { ) -> Result<Vec<NodeInfo>> { let n = u16::try_from(nodes.len()).unwrap_or(u16::MAX); let rest_ports = - assign_process_ports(ServicePort::KvServerMgmt, 0, n).map_err(|e| Error::Validation { + assign_process_ports(ServicePort::PaxosKvServerMgmt, 0, n).map_err(|e| Error::Validation { field: "port_alloc".into(), message: e.to_string(), })?; let rpc_ports = - assign_process_ports(ServicePort::KvServerListen, 0, n).map_err(|e| Error::Validation { + assign_process_ports(ServicePort::PaxosKvServerRpc, 0, n).map_err(|e| Error::Validation { field: "port_alloc".into(), message: e.to_string(), })?; @@ -464,11 +464,6 @@ impl CrowdbClusterDeployer { let mut diskdb_instances = Vec::new(); if topo.deploy_diskdb { let n = u16::try_from(nodes.len()).unwrap_or(u16::MAX); - let listen_ports = - assign_process_ports(ServicePort::DiskdbListen, 0, n).map_err(|e| Error::Validation { - field: "port_alloc".into(), - message: e.to_string(), - })?; let http_ports = assign_process_ports(ServicePort::DiskdbHttp, 0, n).map_err(|e| Error::Validation { field: "port_alloc".into(), @@ -481,7 +476,6 @@ impl CrowdbClusterDeployer { })?; for (i, &node_id) in nodes.iter().enumerate() { let rpc_port = rpc_ports[i]; - let listen_port = listen_ports[i]; let http_port = http_ports[i]; let result = self .client @@ -489,7 +483,6 @@ impl CrowdbClusterDeployer { node_id, &DeployDiskdbBody { rpc_port, - listen_port: Some(listen_port), http_port: Some(http_port), }, ) diff --git a/lib/crowdb-console-shared/src/config.rs b/lib/crowdb-console-shared/src/config.rs index ef1408ebc..10f065c6f 100644 --- a/lib/crowdb-console-shared/src/config.rs +++ b/lib/crowdb-console-shared/src/config.rs @@ -123,7 +123,7 @@ pub struct NodeEntry { } fn default_ssh_port() -> u16 { - 22 + crowdb_protocol::port::ports::HOST_SSH_PORT } impl NodeEntry { @@ -185,14 +185,14 @@ pub enum ServiceType { pub struct ServerEntry { /// Console-side identifier; must be unique within an operation context. pub id: String, - /// Service URL. For KV this is the `crowdb-kv-server` management base + /// Service URL. For KV this is the `crowdb-paxos-kv-server` management base /// URL; for `DiskDB` this is its public crowdb-rpc endpoint. pub url: String, /// Owning node id; populated for console-deployed instances. `None` /// for plain "registered external server" entries from C2. #[serde(default, skip_serializing_if = "Option::is_none")] pub node_id: Option<NodeId>, - /// crowdb-rpc base URL, e.g. `http://127.0.0.1:10100`. Populated for + /// crowdb-rpc base URL, e.g. `http://127.0.0.1:11135`. Populated for /// console-deployed instances. #[serde(default, skip_serializing_if = "Option::is_none")] pub rpc_url: Option<String>, @@ -214,11 +214,11 @@ pub struct ServerEntry { /// Service type discriminator. KV is the default for local fixtures. #[serde(default, skip_serializing_if = "is_default_service_type")] pub service_type: ServiceType, - /// `--rpc-workers` value passed to the spawned `crowdb-kv-server`. + /// `--rpc-workers` value passed to the spawned `crowdb-paxos-kv-server`. /// `None` means the server's default (2) is used. #[serde(default, skip_serializing_if = "Option::is_none")] pub rpc_workers: Option<u32>, - /// `--no-fsync` flag passed to the spawned `crowdb-kv-server`. + /// `--no-fsync` flag passed to the spawned `crowdb-paxos-kv-server`. /// Persisted so restart reuses the same value. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub no_fsync: bool, diff --git a/lib/crowdb-console-shared/src/corr_id.rs b/lib/crowdb-console-shared/src/corr_id.rs index 8dc351e0e..cd54f2b10 100644 --- a/lib/crowdb-console-shared/src/corr_id.rs +++ b/lib/crowdb-console-shared/src/corr_id.rs @@ -6,8 +6,8 @@ //! Every console request (CLI invocation, web handler call) runs inside //! a `tokio` task-local that carries an opaque 16-hex-char correlation //! id. Outbound HTTP/crowdb-rpc clients attach this id as the -//! `x-crowdb-kv-corr-id` header so a single user action shows up linked -//! across console + every upstream `crowdb-kv-server` it touched. +//! `x-crowdb-paxos-kv-corr-id` header so a single user action shows up linked +//! across console + every upstream `crowdb-paxos-kv-server` it touched. //! //! Key work: task-local storage with `scope`/`scope_in`, generators //! (`new`, `current_or_new`), and the canonical header constant. @@ -20,9 +20,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; /// HTTP header name carrying the correlation id between console -/// components and upstream `crowdb-kv-server`. Lowercase per RFC 7230; -/// `crowdb-kv-server` will recognise it once the tracing middleware lands. -pub const HEADER: &str = "x-crowdb-kv-corr-id"; +/// components and upstream `crowdb-paxos-kv-server`. Lowercase per RFC 7230; +/// `crowdb-paxos-kv-server` will recognise it once the tracing middleware lands. +pub const HEADER: &str = "x-crowdb-paxos-kv-corr-id"; tokio::task_local! { /// Per-task correlation id. Reading via `try_with` returns `None` diff --git a/lib/crowdb-console-shared/src/deployment/bootstrap.rs b/lib/crowdb-console-shared/src/deployment/bootstrap.rs index 3ca62f1a1..8829790b0 100644 --- a/lib/crowdb-console-shared/src/deployment/bootstrap.rs +++ b/lib/crowdb-console-shared/src/deployment/bootstrap.rs @@ -6,7 +6,7 @@ use std::io::{Read, Write}; use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; use std::path::Path; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::mgmt::SystemBootstrapIdentity; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; diff --git a/lib/crowdb-console-shared/src/deployment/node_update.rs b/lib/crowdb-console-shared/src/deployment/node_update.rs index 629382737..6212bb273 100644 --- a/lib/crowdb-console-shared/src/deployment/node_update.rs +++ b/lib/crowdb-console-shared/src/deployment/node_update.rs @@ -8,7 +8,7 @@ use crate::{ error::{Error, Result}, ops::OpContext, }; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::common::NodeValue; use serde::{Deserialize, Serialize}; @@ -152,9 +152,9 @@ async fn publish(ctx: &OpContext, cluster: &str, source: &NodeRecord, target: &N { Ok(_) => return Ok(()), Err( - crowdb_kv_client::Error::CasFailed { .. } - | crowdb_kv_client::Error::CasBusy - | crowdb_kv_client::Error::OutcomeUnknown, + crowdb_paxos_kv_client::Error::CasFailed { .. } + | crowdb_paxos_kv_client::Error::CasBusy + | crowdb_paxos_kv_client::Error::OutcomeUnknown, ) => {} Err(error) => return Err(error.into()), } @@ -170,7 +170,7 @@ fn config_error(error: impl std::fmt::Display) -> Error { /// # Errors /// Rejects concurrent admission while a node relocation is incomplete. -pub async fn check_admission(kv: &crowdb_kv_client::CrowdbKvClient, id: &str) -> Result<()> { +pub async fn check_admission(kv: &crowdb_paxos_kv_client::PaxosKvClient, id: &str) -> Result<()> { if let GetOutcome::Found { value, .. } = kv .get( 0, diff --git a/lib/crowdb-console-shared/src/deployment/preparation.rs b/lib/crowdb-console-shared/src/deployment/preparation.rs index 86a3022cf..086b9c894 100644 --- a/lib/crowdb-console-shared/src/deployment/preparation.rs +++ b/lib/crowdb-console-shared/src/deployment/preparation.rs @@ -11,7 +11,7 @@ pub use record::{ DefaultChunkKvBootstrap, DefaultMetadataGroup, DefaultPreparation, DefaultReplica, DefaultStorageBinding, }; -use crowdb_kv_client::{Error as KvError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{Error as KvError, GetOutcome, ReadMode}; use crowdb_protocol::mgmt::SystemBootstrapIdentity; use super::{PreparedBootstrap, CLUSTER_KEY}; diff --git a/lib/crowdb-console-shared/src/deployment/recovery/confirmed.rs b/lib/crowdb-console-shared/src/deployment/recovery/confirmed.rs index fbb7b0df9..ecab47b2c 100644 --- a/lib/crowdb-console-shared/src/deployment/recovery/confirmed.rs +++ b/lib/crowdb-console-shared/src/deployment/recovery/confirmed.rs @@ -119,10 +119,15 @@ impl ConfirmedMemberTrust { .map_err(|_| invalid("invalid recovery contact address"))?; let mut server = ServerEntry::new( format!("kv-{}", node.node_id), - format!("http://{}", std::net::SocketAddr::new(address, 10000)), + format!( + "http://{}", + std::net::SocketAddr::new(address, crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE) + ), ); server.node_id = Some(node.node_id); - server.rpc_url = Some(std::net::SocketAddr::new(address, 10100).to_string()); + server.rpc_url = Some( + std::net::SocketAddr::new(address, crowdb_protocol::PAXOS_KV_SERVER_RPC_BASE).to_string(), + ); config.servers.push(server); } Ok(config) diff --git a/lib/crowdb-console-shared/src/deployment/registry.rs b/lib/crowdb-console-shared/src/deployment/registry.rs index 1ebe3b1db..5f972bfd2 100644 --- a/lib/crowdb-console-shared/src/deployment/registry.rs +++ b/lib/crowdb-console-shared/src/deployment/registry.rs @@ -4,7 +4,7 @@ //! Conditional UUID allocation and public admission state in Group 0. use crate::error::{Error, Result}; -use crowdb_kv_client::{CrowdbKvClient, Error as KvError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{Error as KvError, GetOutcome, PaxosKvClient, ReadMode}; use serde::{Deserialize, Serialize}; pub const NODES_KEY: &[u8] = b"/deployment/nodes"; @@ -64,7 +64,7 @@ impl NodeRecord { /// # Errors /// Rejects conflicting bootstrap mapping or unavailable quorum. -pub async fn publish(kv: &CrowdbKvClient, registry: &NodeRegistry) -> Result<()> { +pub async fn publish(kv: &PaxosKvClient, registry: &NodeRegistry) -> Result<()> { let bytes = serde_json::to_vec(registry).map_err(config_error)?; match kv.put_cas(0, 0, NODES_KEY, &bytes, 0).await { Ok(_) => Ok(()), @@ -91,7 +91,7 @@ pub async fn publish(kv: &CrowdbKvClient, registry: &NodeRegistry) -> Result<()> /// /// # Errors /// Rejects duplicate UUID endpoints, changed clusters and unavailable authority. -pub async fn admit(kv: &CrowdbKvClient, cluster: &str, mut node: NodeRecord) -> Result<NodeRecord> { +pub async fn admit(kv: &PaxosKvClient, cluster: &str, mut node: NodeRecord) -> Result<NodeRecord> { super::node_update::check_admission(kv, &node.discovery_id).await?; let mut last_conflict = None; for attempt in 0..32 { @@ -175,7 +175,7 @@ fn cancellation_key(operation: &str) -> Vec<u8> { format!("/deployment/cancelled-admissions/{operation}").into_bytes() } -async fn cancellation_complete(kv: &CrowdbKvClient, cluster: &str, node: &NodeRecord) -> Result<bool> { +async fn cancellation_complete(kv: &PaxosKvClient, cluster: &str, node: &NodeRecord) -> Result<bool> { let expected = serde_json::to_vec(&(cluster, &node.discovery_id, &node.operation_id)).map_err(config_error)?; Ok( @@ -188,7 +188,7 @@ async fn cancellation_complete(kv: &CrowdbKvClient, cluster: &str, node: &NodeRe /// /// # Errors /// Rejects replaced operations, confirmed nodes and unavailable authority. -pub async fn complete_cancellation(kv: &CrowdbKvClient, cluster: &str, node: &NodeRecord) -> Result<()> { +pub async fn complete_cancellation(kv: &PaxosKvClient, cluster: &str, node: &NodeRecord) -> Result<()> { let (registry, _) = read(kv) .await? .ok_or_else(|| Error::Config("cluster mapping absent".into()))?; @@ -218,12 +218,7 @@ pub async fn complete_cancellation(kv: &CrowdbKvClient, cluster: &str, node: &No /// /// # Errors /// Rejects committed nodes, changed operation identity and unavailable authority. -pub async fn cancel( - kv: &CrowdbKvClient, - cluster: &str, - discovery_id: &str, - operation_id: &str, -) -> Result<()> { +pub async fn cancel(kv: &PaxosKvClient, cluster: &str, discovery_id: &str, operation_id: &str) -> Result<()> { for attempt in 0..32 { let (mut registry, revision) = read(kv) .await? @@ -256,7 +251,7 @@ pub async fn cancel( /// # Errors /// Returns an explicit authority error rather than an empty registry on failure. -pub async fn read(kv: &CrowdbKvClient) -> Result<Option<(NodeRegistry, u64)>> { +pub async fn read(kv: &PaxosKvClient) -> Result<Option<(NodeRegistry, u64)>> { match kv.get(0, 0, NODES_KEY, ReadMode::Linearizable, None).await? { GetOutcome::Found { value, revision, .. } => Ok(Some(( serde_json::from_slice(&value).map_err(config_error)?, diff --git a/lib/crowdb-console-shared/src/deployment/services.rs b/lib/crowdb-console-shared/src/deployment/services.rs index 92d87e041..36c0ce935 100644 --- a/lib/crowdb-console-shared/src/deployment/services.rs +++ b/lib/crowdb-console-shared/src/deployment/services.rs @@ -5,7 +5,7 @@ use crate::{ config::NodeEntry, error::{Error, Result}, }; -use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::mgmt::node::{NodeControl, NodeServiceIntent}; use serde_json::Value; @@ -20,7 +20,7 @@ pub use readiness::diskdb_storage_ready; /// /// # Errors /// Rejects lost quorum, concurrent supersession and failed remote execution. -pub async fn execute(kv: &CrowdbKvClient, node: &NodeEntry, mut intent: NodeServiceIntent) -> Result<Value> { +pub async fn execute(kv: &PaxosKvClient, node: &NodeEntry, mut intent: NodeServiceIntent) -> Result<Value> { claim_service_id(kv, &intent).await?; let key = format!("/deployment/services/{}/{}", intent.node_id, intent.service_id); let current = kv.get(0, 0, key.as_bytes(), ReadMode::Linearizable, None).await?; @@ -45,7 +45,7 @@ pub async fn execute(kv: &CrowdbKvClient, node: &NodeEntry, mut intent: NodeServ /// /// # Errors /// Rejects unavailable authority or invalid stored records. -pub async fn current(kv: &CrowdbKvClient, node: u64, service: &str) -> Result<Option<NodeServiceIntent>> { +pub async fn current(kv: &PaxosKvClient, node: u64, service: &str) -> Result<Option<NodeServiceIntent>> { let key = format!("/deployment/services/{node}/{service}"); match kv.get(0, 0, key.as_bytes(), ReadMode::Linearizable, None).await? { GetOutcome::Found { value, .. } => serde_json::from_slice(&value) @@ -59,7 +59,7 @@ pub async fn current(kv: &CrowdbKvClient, node: u64, service: &str) -> Result<Op /// /// # Errors /// Rejects unavailable authority, oversized views or invalid stored records. -pub async fn list(kv: &CrowdbKvClient) -> Result<Vec<NodeServiceIntent>> { +pub async fn list(kv: &PaxosKvClient) -> Result<Vec<NodeServiceIntent>> { let result = kv .scan_bounded(0, 0, b"/deployment/services/", b"", b"", 4096, false, None) .await?; @@ -73,7 +73,7 @@ pub async fn list(kv: &CrowdbKvClient) -> Result<Vec<NodeServiceIntent>> { .collect() } -async fn claim_service_id(kv: &CrowdbKvClient, intent: &NodeServiceIntent) -> Result<()> { +async fn claim_service_id(kv: &PaxosKvClient, intent: &NodeServiceIntent) -> Result<()> { let key = format!("/deployment/service-owners/{}", intent.service_id); let owner = serde_json::to_vec(&(intent.cluster_id.as_str(), intent.node_id, intent.kind.as_str())) .map_err(|error| Error::Config(error.to_string()))?; diff --git a/lib/crowdb-console-shared/src/deployment/services/planned_start.rs b/lib/crowdb-console-shared/src/deployment/services/planned_start.rs index c3542e33e..b422bb72a 100644 --- a/lib/crowdb-console-shared/src/deployment/services/planned_start.rs +++ b/lib/crowdb-console-shared/src/deployment/services/planned_start.rs @@ -3,7 +3,7 @@ //! Accepted node plans cannot supersede later explicit service controls. -use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; use crate::error::{Error, Result}; @@ -15,7 +15,7 @@ use crate::error::{Error, Result}; /// Rejects invalid starts, an existing different intent, changed service ownership /// and unavailable authority. In particular a later Stop/Restart/Delete can never /// be overwritten by an old accepted plan or its delayed retry. -pub async fn publish_planned_start(kv: &CrowdbKvClient, intent: &NodeServiceIntent) -> Result<()> { +pub async fn publish_planned_start(kv: &PaxosKvClient, intent: &NodeServiceIntent) -> Result<()> { let operation = uuid::Uuid::parse_str(&intent.operation_id) .map_err(|error| Error::Config(format!("planned Start operation: {error}")))?; if intent.action != NodeServiceAction::Start diff --git a/lib/crowdb-console-shared/src/deployment/services/readiness.rs b/lib/crowdb-console-shared/src/deployment/services/readiness.rs index 0758ae363..57d57dd9e 100644 --- a/lib/crowdb-console-shared/src/deployment/services/readiness.rs +++ b/lib/crowdb-console-shared/src/deployment/services/readiness.rs @@ -2,15 +2,15 @@ // Licensed under the Apache License, Version 2.0. use crate::error::{Error, Result}; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use std::{collections::BTreeMap, sync::Arc, time::Duration}; /// Confirm that loaded `DiskDB` group counts cover the registered online disks. /// # Errors /// Rejects unavailable registry/hardware reads and incomplete storage publication. -pub async fn diskdb_storage_ready(kv: Arc<CrowdbKvClient>) -> Result<()> { - let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(kv.clone()); - let hardware = crowdb_kv_client::HardwareClient::from_shared(kv); +pub async fn diskdb_storage_ready(kv: Arc<PaxosKvClient>) -> Result<()> { + let registry = crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(kv.clone()); + let hardware = crowdb_paxos_kv_client::HardwareClient::from_shared(kv); let (instances, disks) = tokio::time::timeout(Duration::from_secs(3), async { tokio::join!(registry.read_all_diskdb_instances(), hardware.list_all_disks()) }) diff --git a/lib/crowdb-console-shared/src/deployment/target.rs b/lib/crowdb-console-shared/src/deployment/target.rs index 19a60fb4a..fa5af4824 100644 --- a/lib/crowdb-console-shared/src/deployment/target.rs +++ b/lib/crowdb-console-shared/src/deployment/target.rs @@ -123,9 +123,9 @@ impl ResolvedClusterTarget { /// cluster, even if its TCP listener accepts connections. pub async fn verify_native_authority( &self, - kv: &crowdb_kv_client::CrowdbKvClient, + kv: &crowdb_paxos_kv_client::PaxosKvClient, ) -> Result<(), TargetError> { - use crowdb_kv_client::{GetOutcome, ReadMode}; + use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; let result = tokio::time::timeout( Duration::from_secs(3), kv.get(0, 0, super::CLUSTER_KEY, ReadMode::Linearizable, None), diff --git a/lib/crowdb-console-shared/src/diskdb.rs b/lib/crowdb-console-shared/src/diskdb.rs index e16f3d094..466978642 100644 --- a/lib/crowdb-console-shared/src/diskdb.rs +++ b/lib/crowdb-console-shared/src/diskdb.rs @@ -205,8 +205,6 @@ pub struct SetStatusBody { pub struct DeployDiskdbBody { pub rpc_port: u16, #[serde(skip_serializing_if = "Option::is_none")] - pub listen_port: Option<u16>, - #[serde(skip_serializing_if = "Option::is_none")] pub http_port: Option<u16>, } diff --git a/lib/crowdb-console-shared/src/error.rs b/lib/crowdb-console-shared/src/error.rs index dafe300a3..586f6b458 100644 --- a/lib/crowdb-console-shared/src/error.rs +++ b/lib/crowdb-console-shared/src/error.rs @@ -42,7 +42,7 @@ pub enum Error { NotImplemented(String), #[error("kv client error: {0}")] - KvClient(#[from] crowdb_kv_client::Error), + KvClient(#[from] crowdb_paxos_kv_client::Error), } pub type Result<T> = std::result::Result<T, Error>; diff --git a/lib/crowdb-console-shared/src/lib.rs b/lib/crowdb-console-shared/src/lib.rs index 76b9e861a..97794c87c 100644 --- a/lib/crowdb-console-shared/src/lib.rs +++ b/lib/crowdb-console-shared/src/lib.rs @@ -4,7 +4,7 @@ //! Shared core for the `CrowDB Storage` Console (web + CLI). //! //! Key work: data models for racks/nodes/servers/stores/groups/replicas, -//! HTTP and crowdb-rpc clients to `crowdb-kv-server`, registry persistence, +//! HTTP and crowdb-rpc clients to `crowdb-paxos-kv-server`, registry persistence, //! topology aggregation, error model, structured operation log. //! //! C0 status: skeleton only. Real modules land in C1+. diff --git a/lib/crowdb-console-shared/src/lifecycle.rs b/lib/crowdb-console-shared/src/lifecycle.rs index 586c4c0fd..f19532eee 100644 --- a/lib/crowdb-console-shared/src/lifecycle.rs +++ b/lib/crowdb-console-shared/src/lifecycle.rs @@ -4,7 +4,7 @@ //! Server-instance lifecycle (deploy / start / stop). //! //! C3 status: **local-spawn placeholder**. `deploy_local` runs -//! `tokio::process::Command` against the `crowdb-kv-server` binary on the +//! `tokio::process::Command` against the `crowdb-paxos-kv-server` binary on the //! current host; the `node.host` is honored for URL construction but //! ignored for transport. C4 replaces this module's body with `russh`, //! preserving the public API. @@ -50,8 +50,8 @@ pub struct DeployRequest { pub rpc_port: u16, pub group0_management_seeds: Vec<String>, /// Optional override of the binary path. Defaults via - /// `crowdb_kv_server_bin()` resolution: `$CROWDB_KV_SERVER_BIN` → - /// `$PATH` → `target/{debug,release}/crowdb-kv-server` next to the + /// `crowdb_paxos_kv_server_bin()` resolution: `$CROWDB_PAXOS_KV_SERVER_BIN` → + /// `$PATH` → `target/{debug,release}/crowdb-paxos-kv-server` next to the /// current executable. pub binary: Option<PathBuf>, pub election_profile: Option<String>, @@ -86,10 +86,10 @@ pub struct DeployRequest { /// `--send-queue-capacity` value. `None` leaves the spawned /// server's own default in effect. pub send_queue_capacity: Option<u32>, - /// Optional `--config` TOML path for `crowdb-kv-server` (first-boot + /// Optional `--config` TOML path for `crowdb-paxos-kv-server` (first-boot /// tunable overrides only; ignored in restore mode). pub config: Option<PathBuf>, - /// `--rpc-workers` value for the spawned `crowdb-kv-server`. `None` + /// `--rpc-workers` value for the spawned `crowdb-paxos-kv-server`. `None` /// leaves the server's default (2) in effect. pub rpc_workers: Option<u32>, } @@ -169,7 +169,7 @@ pub struct DiskioLocalDisk { pub zone_capacity: u64, } -/// Spawn `crowdb-kv-server` locally. The `node.host` is folded into the +/// Spawn `crowdb-paxos-kv-server` locally. The `node.host` is folded into the /// returned URLs so the rest of the console can address the instance /// uniformly with the SSH path coming in C4. /// @@ -195,7 +195,7 @@ pub async fn deploy_local_in_dir( } /// Deploys a server in a specific workspace directory, passing extra -/// CLI arguments to the spawned `crowdb-kv-server` binary. Used by tests +/// CLI arguments to the spawned `crowdb-paxos-kv-server` binary. Used by tests /// to bootstrap previously-created stores/groups on restart so the /// server recovers from WAL and rejoins the cluster. /// @@ -213,7 +213,7 @@ pub async fn deploy_local_in_dir_with_extra_args( } /// Append `--kv-backend`/`--wal-backend`/`--no-fsync`/`--metrics-interval`/ -/// `--max-inflight` flags to the spawned `crowdb-kv-server` command per `req`. +/// `--max-inflight` flags to the spawned `crowdb-paxos-kv-server` command per `req`. /// Split out of `deploy_local_in_workspace` to keep it under the line-count /// lint. fn apply_benchmark_flags(cmd: &mut Command, req: &DeployRequest) { @@ -266,16 +266,16 @@ fn apply_node_identity_and_seeds(cmd: &mut Command, req: &DeployRequest) { /// Resolve the `--config` path for a deploy. When `req.config` is set, /// it is used verbatim. When unset, returns `None` — the server boots -/// with `CrowDBConfig::default()` tunables (no toml needed). The +/// with `PaxosKvConfig::default()` tunables (no toml needed). The /// `--config` flag is now optional and only used for first-boot /// tunable overrides. fn resolve_config_path(req: &DeployRequest) -> Option<PathBuf> { req.config.clone() } -// stage_server_binary creates a symlink at dir/bin/crowdb-kv-server. +// stage_server_binary creates a symlink at dir/bin/crowdb-paxos-kv-server. // We set current_dir(dir) below, so use a path relative to the CWD -// (just "bin/crowdb-kv-server") — an absolute or full-relative path +// (just "bin/crowdb-paxos-kv-server") — an absolute or full-relative path // would be resolved relative to the new CWD and fail with ENOENT. fn resolve_launch_binary(binary: &Path, workspace_dir: Option<&Path>) -> Result<PathBuf> { #[cfg(target_os = "macos")] @@ -293,7 +293,7 @@ fn resolve_launch_binary(binary: &Path, workspace_dir: Option<&Path>) -> Result< Ok(PathBuf::from("bin").join( binary .file_name() - .unwrap_or_else(|| std::ffi::OsStr::new("crowdb-kv-server")), + .unwrap_or_else(|| std::ffi::OsStr::new("crowdb-paxos-kv-server")), )) } else { Ok(binary.to_path_buf()) @@ -321,9 +321,9 @@ async fn deploy_local_in_workspace( let binary = match &req.binary { Some(p) => p.clone(), - None => crowdb_kv_server_bin().ok_or_else(|| Error::Validation { + None => crowdb_paxos_kv_server_bin().ok_or_else(|| Error::Validation { field: "binary".into(), - message: "could not locate crowdb-kv-server binary; set $CROWDB_KV_SERVER_BIN".into(), + message: "could not locate crowdb-paxos-kv-server binary; set $CROWDB_PAXOS_KV_SERVER_BIN".into(), })?, }; let launch_binary = resolve_launch_binary(&binary, workspace_dir)?; @@ -343,7 +343,7 @@ async fn deploy_local_in_workspace( req.election_profile .as_deref() .map(str::to_owned) - .or_else(|| std::env::var("CROWDB_KV_SERVER_ELECTION_PROFILE").ok()) + .or_else(|| std::env::var("CROWDB_PAXOS_KV_SERVER_ELECTION_PROFILE").ok()) .unwrap_or_else(|| "default".into()), ) .kill_on_drop(false); @@ -362,7 +362,7 @@ async fn deploy_local_in_workspace( // Merge stdout and stderr into one file. We open a temp file before // spawn (PID unknown), then rename it with the PID after spawn. let log_dir = dir.join("log"); - let tmp_path = log_dir.join("crowdb-kv-server.stdout.log"); + let tmp_path = log_dir.join("crowdb-paxos-kv-server.stdout.log"); let out = std::fs::OpenOptions::new() .create(true) .append(true) @@ -378,7 +378,7 @@ async fn deploy_local_in_workspace( // first-boot mode, not restore mode. let root = crowdb_protocol::port::namespace::runtime_root() .join("ephemeral") - .join(format!("crowdb-kv-server-deploy-{}", req.rest_port)); + .join(format!("crowdb-paxos-kv-server-deploy-{}", req.rest_port)); let _ = std::fs::remove_dir_all(&root); std::fs::create_dir_all(&root).map_err(Error::Io)?; cmd.arg("--root") @@ -403,10 +403,10 @@ async fn deploy_local_in_workspace( // confirmed by polling /health. if workspace_dir.is_none() { if let Some(stdout) = child.stdout.take() { - spawn_log_pipe(stdout, "crowdb-kv-server stdout"); + spawn_log_pipe(stdout, "crowdb-paxos-kv-server stdout"); } if let Some(stderr) = child.stderr.take() { - spawn_log_pipe(stderr, "crowdb-kv-server stderr"); + spawn_log_pipe(stderr, "crowdb-paxos-kv-server stderr"); } } @@ -414,7 +414,7 @@ async fn deploy_local_in_workspace( // Keep ownership of the child in a reaper task so exited children do not // become zombies under the console process. - tokio::spawn(reap_child(child, pid, "crowdb-kv-server")); + tokio::spawn(reap_child(child, pid, "crowdb-paxos-kv-server")); Ok(DeployedServer { server_id: req.server_id.clone(), @@ -433,11 +433,11 @@ async fn reap_child(mut child: Child, pid: u32, service: &'static str) { /// Send SIGTERM to a tracked pid on the **local** host. Returns /// `Ok(false)` if the pid is already gone. Implemented by shelling out -/// to `/bin/kill`, matching `crowdb-kv-server/tests/testkit/process.rs` so +/// to `/bin/kill`, matching `crowdb-paxos-kv-server/tests/testkit/process.rs` so /// that both paths behave identically and the workspace /// `unsafe_code = deny` lint is kept. /// -/// For SSH-deployed servers, use `crowdb_kv_ssh` to run the +/// For SSH-deployed servers, use `crowdb_paxos_kv_ssh` to run the /// equivalent command on the remote host. /// /// # Errors @@ -526,7 +526,7 @@ pub fn process_is_alive(pid: u32) -> bool { } } -/// Render the shell command that brings up `crowdb-kv-server` on the remote +/// Render the shell command that brings up `crowdb-paxos-kv-server` on the remote /// host. Public so the SSH path can reuse it without duplicating arg /// formatting. #[must_use] @@ -569,8 +569,8 @@ fn record_kv_child( record_workspace_child(child, workspace, pid)?; let log_dir = workspace.join("log"); let _ = std::fs::rename( - log_dir.join("crowdb-kv-server.stdout.log"), - log_dir.join(format!("crowdb-kv-server-{pid}.out.log")), + log_dir.join("crowdb-paxos-kv-server.stdout.log"), + log_dir.join(format!("crowdb-paxos-kv-server-{pid}.out.log")), ); } Ok(()) @@ -658,7 +658,7 @@ fn stage_server_binary(binary: &std::path::Path, workspace_dir: &std::path::Path let staged = bin_dir.join( source .file_name() - .unwrap_or_else(|| std::ffi::OsStr::new("crowdb-kv-server")), + .unwrap_or_else(|| std::ffi::OsStr::new("crowdb-paxos-kv-server")), ); if staged.exists() { std::fs::remove_file(&staged).map_err(Error::Io)?; @@ -689,17 +689,17 @@ fn resolve_binary_path(binary: &std::path::Path) -> Option<PathBuf> { find_in_path(binary.as_os_str()) } -/// Resolve the path to the `crowdb-kv-server` binary. +/// Resolve the path to the `crowdb-paxos-kv-server` binary. /// /// Search order: -/// 1. `$CROWDB_KV_SERVER_BIN`. -/// 2. A sibling named `crowdb-kv-server` next to the current executable +/// 1. `$CROWDB_PAXOS_KV_SERVER_BIN`. +/// 2. A sibling named `crowdb-paxos-kv-server` next to the current executable /// (covers `cargo run -p crowdb-console-cli`). -/// 3. `crowdb-kv-server` on `$PATH` (returned as a relative path so the OS +/// 3. `crowdb-paxos-kv-server` on `$PATH` (returned as a relative path so the OS /// resolves it at exec time). #[must_use] -pub fn crowdb_kv_server_bin() -> Option<PathBuf> { - if let Ok(p) = std::env::var("CROWDB_KV_SERVER_BIN") { +pub fn crowdb_paxos_kv_server_bin() -> Option<PathBuf> { + if let Ok(p) = std::env::var("CROWDB_PAXOS_KV_SERVER_BIN") { return Some(PathBuf::from(p)); } if let Ok(exe) = std::env::current_exe() { @@ -707,7 +707,7 @@ pub fn crowdb_kv_server_bin() -> Option<PathBuf> { // dir is e.g. target/debug/. Walk up if we are deeper (deps/). let mut p = dir.to_path_buf(); for _ in 0..3 { - let candidate = p.join("crowdb-kv-server"); + let candidate = p.join("crowdb-paxos-kv-server"); if is_executable(&candidate) { return Some(candidate); } @@ -726,7 +726,7 @@ pub fn crowdb_kv_server_bin() -> Option<PathBuf> { } } if target.file_name().is_some_and(|name| name == "target") { - let candidate = target.join("release").join("crowdb-kv-server"); + let candidate = target.join("release").join("crowdb-paxos-kv-server"); if is_executable(&candidate) { return Some(candidate); } @@ -734,10 +734,10 @@ pub fn crowdb_kv_server_bin() -> Option<PathBuf> { } } // Fall back to PATH. - if let Some(path) = find_in_path(std::ffi::OsStr::new("crowdb-kv-server")) { + if let Some(path) = find_in_path(std::ffi::OsStr::new("crowdb-paxos-kv-server")) { return Some(path); } - warn!("crowdb-kv-server binary not found via env, sibling, or $PATH"); + warn!("crowdb-paxos-kv-server binary not found via env, sibling, or $PATH"); None } @@ -792,8 +792,6 @@ pub struct DiskdbDeployRequest { pub free_batch_enabled: Option<bool>, /// Maximum free records per coalesced KV proposal. pub free_flush_max_batch: Option<u32>, - /// Main listener port (diskdb `listen_addr`). - pub listen_port: u16, /// HTTP management port (diskdb `http_listen_addr`). pub http_port: u16, /// crowdb-rpc listener port (diskdb `rpc_listen_addr`). @@ -885,7 +883,7 @@ pub fn crowdb_diskio_bin() -> Option<PathBuf> { /// pre-copied config at `<workspace>/conf/crowdb_diskdb_config.toml`. /// Falls back to a minimal auto-generated config with all required /// sections if the file is missing. The auto-generated config sets -/// `listen_addr`, `http_listen_addr`, and `rpc_listen_addr` +/// `http_listen_addr` and `rpc_listen_addr` /// independently (no paired-port invariant). When /// `kv_server_mgmt_seeds` is non-empty, the config is always /// (re)written so the diskdb can discover group-0 on the actual @@ -928,7 +926,10 @@ fn resolve_diskdb_config_path( return Ok(path); } let seeds = if request.kv_server_mgmt_seeds.is_empty() { - format!("\"http://127.0.0.1:{}\"", crowdb_protocol::KV_SERVER_MGMT_BASE) + format!( + "\"http://127.0.0.1:{}\"", + crowdb_protocol::PAXOS_KV_SERVER_MGMT_BASE + ) } else { request .kv_server_mgmt_seeds @@ -960,14 +961,12 @@ fn resolve_diskdb_config_path( let config = format!( "[server]\n\ rpc_workers = {}\n\ - listen_addr = \"0.0.0.0:{}\"\n\ http_listen_addr = \"0.0.0.0:{}\"\n\ rpc_listen_addr = \"0.0.0.0:{}\"\n\ {instance_id}\ kv_server_mgmt_seeds = [{seeds}]\n\ {persistence}{heartbeat}", request.rpc_workers.unwrap_or(2), - request.listen_port, request.http_port, request.rpc_port, ); @@ -1071,8 +1070,7 @@ async fn wait_for_service_ready( /// Spawn `crowdb-diskdb` locally. The binary and config file are /// pre-copied to the node workspace (`<workspace>/bin/crowdb-diskdb` /// and `<workspace>/conf/crowdb_diskdb_config.toml`). The deploy only -/// overrides `--listen-addr` with the crowdb-rpc port; the HTTP port comes -/// from the config file. +/// sets the HTTP and crowdb-rpc endpoints explicitly. /// /// # Errors /// Returns `Error::Validation` for bad inputs and `Error::Io` for @@ -1082,10 +1080,10 @@ pub async fn deploy_diskdb_local( node: &NodeEntry, workspace_dir: &std::path::Path, ) -> Result<DeployedDiskdb> { - if req.listen_port == 0 || req.http_port == 0 || req.rpc_port == 0 { + if req.http_port == 0 || req.rpc_port == 0 || req.http_port == req.rpc_port { return Err(Error::Validation { field: "port".into(), - message: "listen_port, http_port, and rpc_port must all be non-zero".into(), + message: "http_port and rpc_port must be distinct and non-zero".into(), }); } std::fs::create_dir_all(workspace_dir.join("log"))?; @@ -1104,8 +1102,7 @@ pub async fn deploy_diskdb_local( }; let config_path = resolve_diskdb_config_path(workspace_dir, req)?; - // The public endpoint is the crowdb-rpc listener. The main listener is - // an internal compatibility endpoint and is not exposed as DiskDB identity. + // The public endpoint is the crowdb-rpc listener. let endpoint = format!("http://{}:{}", node.host, req.rpc_port); // Read the HTTP listen address from the config file for readiness @@ -1130,8 +1127,6 @@ pub async fn deploy_diskdb_local( if let Some(workers) = req.kv_client_rpc_workers { cmd.arg("--kv-client-rpc-workers").arg(workers.to_string()); } - cmd.arg("--listen-addr") - .arg(format!("{}:{}", node.host, req.listen_port)); cmd.arg("--http-addr") .arg(format!("{}:{}", node.host, req.http_port)); cmd.arg("--rpc-listen-addr") @@ -1181,8 +1176,6 @@ fn diskdb_launch_args(req: &DiskdbDeployRequest, config_path: &Path, workspace_d config_path.to_string_lossy().into_owned(), "--log-dir".into(), workspace_dir.join("log").to_string_lossy().into_owned(), - "--listen-addr".into(), - format!("127.0.0.1:{}", req.listen_port), "--http-addr".into(), format!("127.0.0.1:{}", req.http_port), "--rpc-listen-addr".into(), @@ -1432,12 +1425,12 @@ fn diskio_ffi_lib_dir(workspace_dir: &Path) -> Option<PathBuf> { } for mut root in roots { for _ in 0..8 { - if root.join("libcrowdb_kv_client.so").exists() { + if root.join("libcrowdb_paxos_kv_client.so").exists() { return Some(root); } for profile in ["debug", "release"] { let candidate = root.join("target").join(profile); - if candidate.join("libcrowdb_kv_client.so").exists() { + if candidate.join("libcrowdb_paxos_kv_client.so").exists() { return Some(candidate); } } diff --git a/lib/crowdb-console-shared/src/mgmt.rs b/lib/crowdb-console-shared/src/mgmt.rs index 219f2f32f..3e27cbf2e 100644 --- a/lib/crowdb-console-shared/src/mgmt.rs +++ b/lib/crowdb-console-shared/src/mgmt.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Typed bindings for `crowdb-kv-server`'s management API (C5). +//! Typed bindings for `crowdb-paxos-kv-server`'s management API (C5). //! //! The HTTP request/response DTOs live in `crowdb-protocol::mgmt` (the //! single home for cross-component protocol types). This module re- @@ -10,7 +10,7 @@ //! stays in one place. // Re-export all DTOs so existing callers (`use crate::mgmt::*`) keep -// working until Stage 4 migrates them to `KVClusterAdmin`. +// working until Stage 4 migrates them to `PaxosKvClusterAdmin`. pub use crowdb_protocol::mgmt::*; use crate::clients::http::ServerClient; diff --git a/lib/crowdb-console-shared/src/monitor.rs b/lib/crowdb-console-shared/src/monitor.rs index ecdabfd10..4eb947288 100644 --- a/lib/crowdb-console-shared/src/monitor.rs +++ b/lib/crowdb-console-shared/src/monitor.rs @@ -147,7 +147,7 @@ impl MonitorCache { /// to fail with "no known leader". /// /// Used by `AppState::op_context` to seed the shared - /// `CrowdbKvClient` with the actual group-0 endpoint so sysdata + /// `PaxosKvClient` with the actual group-0 endpoint so sysdata /// writes reach the right listener (the configured `rpc_url` is /// only correct for the bootstrap store, not for store 0 once the /// system group has been initialized on a different port). diff --git a/lib/crowdb-console-shared/src/ops/chunk.rs b/lib/crowdb-console-shared/src/ops/chunk.rs index e2199425a..04e71c399 100644 --- a/lib/crowdb-console-shared/src/ops/chunk.rs +++ b/lib/crowdb-console-shared/src/ops/chunk.rs @@ -5,7 +5,7 @@ use std::sync::Arc; -use crowdb_kv_client::ServiceDiscoveryClient; +use crowdb_paxos_kv_client::ServiceDiscoveryClient; use crate::error::{Error, Result}; use crate::ops::OpContext; diff --git a/lib/crowdb-console-shared/src/ops/cluster.rs b/lib/crowdb-console-shared/src/ops/cluster.rs index 57ebefe80..13fa344c2 100644 --- a/lib/crowdb-console-shared/src/ops/cluster.rs +++ b/lib/crowdb-console-shared/src/ops/cluster.rs @@ -13,7 +13,7 @@ mod chunk_slots; use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use crowdb_kv_client::RangeBindingClient; +use crowdb_paxos_kv_client::RangeBindingClient; use crowdb_protocol::common::{HwStatus, NodeValue, RackValue}; use crowdb_protocol::port::namespace::{assign_process_ports, RuntimeNamespace}; use crowdb_protocol::ServicePort; @@ -22,7 +22,8 @@ use crate::clients::http::ServerClient; use crate::config::{NodeEntry, RackEntry, ServerEntry, ServiceType}; use crate::error::{Error, Result}; use crate::lifecycle::{ - self, crowdb_kv_server_bin, ChunkdbDeployRequest, DeployRequest, DiskdbDeployRequest, DiskioDeployRequest, + self, crowdb_paxos_kv_server_bin, ChunkdbDeployRequest, DeployRequest, DiskdbDeployRequest, + DiskioDeployRequest, }; use crate::ops::hardware::{self, AddDiskInput}; use crate::ops::OpContext; @@ -1037,7 +1038,6 @@ async fn provision_diskdb_topology( } struct DiskdbPorts { - listen: Vec<u16>, http: Vec<u16>, rpc: Vec<u16>, } @@ -1047,7 +1047,6 @@ fn alloc_diskdb_ports(workspace: &std::path::Path, node_count: usize) -> Result< let count = u16::try_from(node_count).unwrap_or(u16::MAX); let alloc = |service| alloc_workspace_ports(workspace, service, 0, count); Ok(DiskdbPorts { - listen: alloc(ServicePort::DiskdbListen)?, http: alloc(ServicePort::DiskdbHttp)?, rpc: alloc(ServicePort::DiskdbRpc)?, }) @@ -1084,7 +1083,6 @@ async fn deploy_diskdb_instances( keepalive_interval_secs: config.keepalive_interval_secs, free_batch_enabled: config.free_batch_enabled, free_flush_max_batch: config.free_flush_max_batch, - listen_port: ports.listen[index], http_port: ports.http[index], rpc_port: ports.rpc[index], kv_server_mgmt_seeds: seeds.clone(), @@ -1118,7 +1116,7 @@ async fn deploy_diskdb_instances( } /// Deploy a local N-node KV cluster on `127.0.0.1`: creates rack 1, -/// nodes 1..=N in the config, forks a `crowdb-kv-server` on each node +/// nodes 1..=N in the config, forks a `crowdb-paxos-kv-server` on each node /// with auto-allocated ports, then bootstraps group 0 via [`init`]. /// /// The `workspace_dir` is used as the runtime root for the spawned @@ -1155,7 +1153,7 @@ pub struct KvDeployTunables { } /// Deploy a local N-node KV cluster on `127.0.0.1`: creates rack 1, -/// nodes 1..=N in the config, forks a `crowdb-kv-server` on each node +/// nodes 1..=N in the config, forks a `crowdb-paxos-kv-server` on each node /// with auto-allocated ports, then bootstraps group 0 via [`init`]. /// /// The `workspace_dir` is used as the runtime root for the spawned @@ -1165,7 +1163,7 @@ pub struct KvDeployTunables { /// `--peer-pool-size`, etc.). `None` uses server defaults. /// /// # Errors -/// Returns [`Error::NotFound`] if the `crowdb-kv-server` binary cannot +/// Returns [`Error::NotFound`] if the `crowdb-paxos-kv-server` binary cannot /// be located. Returns [`Error::Io`] on spawn/readiness failures. pub async fn local_deploy( ctx: &OpContext, @@ -1224,9 +1222,9 @@ async fn prepare_local_deploy_with_layout( }); } - let bin = crowdb_kv_server_bin().ok_or_else(|| Error::NotFound { + let bin = crowdb_paxos_kv_server_bin().ok_or_else(|| Error::NotFound { kind: "binary".into(), - id: "crowdb-kv-server".into(), + id: "crowdb-paxos-kv-server".into(), })?; if !bin.exists() { return Err(Error::NotFound { @@ -1342,7 +1340,7 @@ fn write_rack_and_nodes(ctx: &OpContext, rack_id: u64, node_ids: &[u64], distinc } } -/// Phase 2: fork a `crowdb-kv-server` on each node (skips nodes that +/// Phase 2: fork a `crowdb-paxos-kv-server` on each node (skips nodes that /// already have a deployed server). /// /// # Errors @@ -1357,8 +1355,8 @@ async fn deploy_servers( distinct_racks: bool, ) -> Result<()> { let n = u16::try_from(node_ids.len()).unwrap_or(u16::MAX); - let rest_ports = alloc_workspace_ports(workspace, ServicePort::KvServerMgmt, 0, n)?; - let rpc_ports = alloc_workspace_ports(workspace, ServicePort::KvServerListen, 0, n)?; + let rest_ports = alloc_workspace_ports(workspace, ServicePort::PaxosKvServerMgmt, 0, n)?; + let rpc_ports = alloc_workspace_ports(workspace, ServicePort::PaxosKvServerRpc, 0, n)?; for (i, nid) in node_ids.iter().enumerate() { let rack_id = if distinct_racks { *nid } else { rack_id }; let rest_port = rest_ports[i]; diff --git a/lib/crowdb-console-shared/src/ops/cluster/bootstrap.rs b/lib/crowdb-console-shared/src/ops/cluster/bootstrap.rs index c3f202256..d1bc20091 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/bootstrap.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/bootstrap.rs @@ -162,7 +162,7 @@ async fn init_internal( // Phase 4: seed the KV client with the group-0 leader endpoint so // `write_topology_to_sysdata` (which uses `ctx.sysmd()`) can reach - // group 0. Without this, the shared `CrowdbKvClient` may have a + // group 0. Without this, the shared `PaxosKvClient` may have a // stale or dummy leader hint from before init. let mgmt_seeds: Vec<String> = succeeded .iter() diff --git a/lib/crowdb-console-shared/src/ops/cluster/bootstrap/publication.rs b/lib/crowdb-console-shared/src/ops/cluster/bootstrap/publication.rs index 3fb855a4a..fe05b1811 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/bootstrap/publication.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/bootstrap/publication.rs @@ -5,7 +5,7 @@ use crate::error::{Error, Result}; use crate::ops::OpContext; -use crowdb_kv_client::{Error as KvError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{Error as KvError, GetOutcome, ReadMode}; use crowdb_protocol::common::{GroupValue, HwStatus, NodeValue, RackValue, ReplicaValue, StoreValue}; use crowdb_protocol::key::{KvGroupKey, KvReplicaKey, KvStoreKey, NodeKey, RackKey, TextKey}; diff --git a/lib/crowdb-console-shared/src/ops/cluster/chunk_slots.rs b/lib/crowdb-console-shared/src/ops/cluster/chunk_slots.rs index 2208166f5..45a67b74d 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/chunk_slots.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/chunk_slots.rs @@ -29,7 +29,7 @@ pub(super) async fn initialize(ctx: &OpContext, config: &LocalChunkdbDeployConfi }) .collect(), }; - let maps = crowdb_kv_client::ChunkSlotMapClient::new(Arc::clone(ctx.kv_arc())); + let maps = crowdb_paxos_kv_client::ChunkSlotMapClient::new(Arc::clone(ctx.kv_arc())); maps.initialize_layout(&bootstrap).await?; if config.dynamic_ownership { maps.initialize_service_epochs().await?; diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs index d96d3477e..1800fb47d 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs @@ -10,7 +10,7 @@ use crate::{ error::{Error, Result}, ops::OpContext, }; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::mgmt::{ cluster::{ ClusterObservation, ClusterStatus, DependencyKind, DependencyObservation, DependencyState, diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage.rs index cdad0557a..0f43bdcc9 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage.rs @@ -12,7 +12,7 @@ use crate::{ error::{Error, Result}, ops::OpContext, }; -use crowdb_kv_client::{DiskRecord, HardwareClient, ReadMode, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{DiskRecord, HardwareClient, ReadMode, ServiceRegistryClient}; use crowdb_protocol::mgmt::cluster::{DependencyKind, DependencyObservation, DependencyState}; use crowdb_protocol::{ common::InstanceValue, @@ -93,11 +93,11 @@ pub(super) async fn read(ctx: &OpContext, bootstrap: &PreparedBootstrap) -> Resu pub(super) struct StorageSnapshot { accepted: preparation::DefaultPreparation, revision: u64, - groups: crowdb_kv_client::Result<Vec<DiskGroupEntry>>, - disks: crowdb_kv_client::Result<Vec<DiskRecord>>, - owners: crowdb_kv_client::Result<Vec<DiskdbOwnerEntry>>, - binds: crowdb_kv_client::Result<Vec<KVGroupBindEntry>>, - instances: crowdb_kv_client::Result<Vec<(u64, InstanceValue)>>, + groups: crowdb_paxos_kv_client::Result<Vec<DiskGroupEntry>>, + disks: crowdb_paxos_kv_client::Result<Vec<DiskRecord>>, + owners: crowdb_paxos_kv_client::Result<Vec<DiskdbOwnerEntry>>, + binds: crowdb_paxos_kv_client::Result<Vec<KVGroupBindEntry>>, + instances: crowdb_paxos_kv_client::Result<Vec<(u64, InstanceValue)>>, } impl StorageSnapshot { diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/checks.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/checks.rs index 21f3cd08e..f7de2a559 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/checks.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/checks.rs @@ -5,14 +5,14 @@ use super::super::fact; use crate::deployment::preparation::DefaultStorageBinding; -use crowdb_kv_client::DiskRecord; +use crowdb_paxos_kv_client::DiskRecord; use crowdb_protocol::{ common::{HwStatus, InstanceValue}, mgmt::cluster::{DependencyKind, DependencyObservation, DependencyState}, sysdata::{DiskGroupEntry, DiskdbOwnerEntry, KVGroupBindEntry}, }; -type Read<'a, T> = Result<&'a [T], &'a crowdb_kv_client::Error>; +type Read<'a, T> = Result<&'a [T], &'a crowdb_paxos_kv_client::Error>; type Check = Result<String, (&'static str, String)>; pub(super) fn observation(id: &str, result: Check) -> DependencyObservation { diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/supplied.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/supplied.rs index 0d327d230..9fa13f3ee 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/supplied.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/supplied.rs @@ -4,7 +4,7 @@ //! Resolve every sealed supplied medium through its actual Group-0 placement. use crate::deployment::preparation::DefaultStorageBinding; -use crowdb_kv_client::DiskRecord; +use crowdb_paxos_kv_client::DiskRecord; use crowdb_protocol::{ common::DiskId, mgmt::node::{ @@ -21,8 +21,8 @@ type Failure = (&'static str, String); pub(super) fn bindings( expected: &DefaultStorageBinding, node: &NodeRuntimeObservation, - disks: Result<&[DiskRecord], &crowdb_kv_client::Error>, - groups: Result<&[DiskGroupEntry], &crowdb_kv_client::Error>, + disks: Result<&[DiskRecord], &crowdb_paxos_kv_client::Error>, + groups: Result<&[DiskGroupEntry], &crowdb_paxos_kv_client::Error>, ) -> Result<Vec<DefaultStorageBinding>, Failure> { let SetupStorage::Supplied { disks: media } = &node.policy.setup.storage else { unreachable!("caller selected supplied storage") diff --git a/lib/crowdb-console-shared/src/ops/cluster/storage_readiness.rs b/lib/crowdb-console-shared/src/ops/cluster/storage_readiness.rs index cb225e5ad..ca1ad4636 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/storage_readiness.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/storage_readiness.rs @@ -10,8 +10,8 @@ use crate::{ use std::sync::Arc; pub(super) async fn wait_for_diskdb_registration(ctx: &OpContext, expected: usize) -> Result<()> { - let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(Arc::clone(ctx.kv_arc())); - let hardware = crowdb_kv_client::HardwareClient::from_shared(Arc::clone(ctx.kv_arc())); + let registry = crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(Arc::clone(ctx.kv_arc())); + let hardware = crowdb_paxos_kv_client::HardwareClient::from_shared(Arc::clone(ctx.kv_arc())); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); loop { let instances = registry.read_all_diskdb_instances().await?; diff --git a/lib/crowdb-console-shared/src/ops/context.rs b/lib/crowdb-console-shared/src/ops/context.rs index 26f426d62..c42e732d8 100644 --- a/lib/crowdb-console-shared/src/ops/context.rs +++ b/lib/crowdb-console-shared/src/ops/context.rs @@ -5,8 +5,8 @@ use std::sync::{Arc, RwLock}; -use crowdb_kv_client::{ - ClientConfig, CrowdbKvClient, CrowdbSysmdClient, GroupMembershipClient, ServiceDiscoveryClient, +use crowdb_paxos_kv_client::{ + ClientConfig, CrowdbSysmdClient, GroupMembershipClient, PaxosKvClient, ServiceDiscoveryClient, }; use crate::config::{ConsoleConfig, NodeEntry, ServerEntry}; @@ -17,7 +17,7 @@ use crate::error::{Error, Result}; /// Holds: /// - **`sysmd`** — a [`CrowdbSysmdClient`] for group-0 system metadata /// (hardware hierarchy, KV-cluster topology, service registry). -/// - **`kv`** — a [`CrowdbKvClient`] for the KV data-plane (put/get/ +/// - **`kv`** — a [`PaxosKvClient`] for the KV data-plane (put/get/ /// delete/scan on user stores/groups). /// - **`config`** — ephemeral [`ConsoleConfig`] inputs for bootstrap and /// local benchmark deployment. Group 0 remains the topology authority. @@ -26,11 +26,11 @@ use crate::error::{Error, Result}; /// the group-0 service registry. `None` when the caller (e.g. a unit /// test) passes explicit addresses instead of using discovery. /// -/// Both `sysmd` and `kv` wrap the same underlying `Arc<CrowdbKvClient>` +/// Both `sysmd` and `kv` wrap the same underlying `Arc<PaxosKvClient>` /// so the topology cache and connection pool are shared. pub struct OpContext { sysmd: CrowdbSysmdClient, - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, config: RwLock<ConsoleConfig>, discovery: Option<Arc<ServiceDiscoveryClient>>, test_scenario: bool, @@ -40,7 +40,7 @@ impl std::fmt::Debug for OpContext { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OpContext") .field("sysmd", &"CrowdbSysmdClient") - .field("kv", &"Arc<CrowdbKvClient>") + .field("kv", &"Arc<PaxosKvClient>") .field("config", &self.config) .field("discovery", &self.discovery.is_some()) .field("test_scenario", &self.test_scenario) @@ -50,7 +50,7 @@ impl std::fmt::Debug for OpContext { impl OpContext { /// Build an `OpContext` from a group-0 endpoint (e.g. - /// `127.0.0.1:10100`) and an initial [`ConsoleConfig`]. + /// `127.0.0.1:11135`) and an initial [`ConsoleConfig`]. /// /// The `group0_endpoint` is used to seed the topology cache for /// group 0 (store 0, group 0). The mgmt URLs of all group-0 hosting @@ -78,7 +78,7 @@ impl OpContext { if !seeds.iter().any(|s| s == &group0_endpoint) { seeds.push(group0_endpoint.clone()); } - let kv = CrowdbKvClient::new(ClientConfig::new(seeds)); + let kv = PaxosKvClient::new(ClientConfig::new(seeds)); kv.seed_leader(0, 0, group0_endpoint); let shared = Arc::new(kv); let sysmd = CrowdbSysmdClient::from_shared(Arc::clone(&shared)); @@ -94,7 +94,7 @@ impl OpContext { } } - /// Build an `OpContext` from a pre-built shared [`CrowdbKvClient`]. + /// Build an `OpContext` from a pre-built shared [`PaxosKvClient`]. /// /// Used by the web backend (`AppState::op_context`) to share the /// cached client (topology cache + connection pool) across requests @@ -104,7 +104,7 @@ impl OpContext { /// client's config — accepted for API symmetry with [`Self::new`]). #[must_use] pub fn with_shared_client( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, group0_endpoint: String, mgmt_seeds: &[String], config: ConsoleConfig, @@ -124,7 +124,7 @@ impl OpContext { } } - /// Build an `OpContext` from a pre-built shared [`CrowdbKvClient`] + /// Build an `OpContext` from a pre-built shared [`PaxosKvClient`] /// without overwriting the existing group-0 leader hint. /// /// Like [`Self::with_shared_client`] but does NOT call @@ -136,7 +136,7 @@ impl OpContext { /// new nodes. #[must_use] pub fn with_shared_client_preserving_hint( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, mgmt_seeds: &[String], config: ConsoleConfig, ) -> Self { @@ -167,9 +167,9 @@ impl OpContext { self.test_scenario } - /// Access the [`CrowdbKvClient`] for the KV data-plane. + /// Access the [`PaxosKvClient`] for the KV data-plane. #[must_use] - pub fn kv(&self) -> &CrowdbKvClient { + pub fn kv(&self) -> &PaxosKvClient { &self.kv } @@ -186,11 +186,11 @@ impl OpContext { self.kv.seed_leader(0, 0, endpoint); } - /// Access the shared [`Arc<CrowdbKvClient>`] for the KV data-plane. + /// Access the shared [`Arc<PaxosKvClient>`] for the KV data-plane. /// Used by the web backend to verify the client is shared (not /// duplicated) between `AppState` and `OpContext`. #[must_use] - pub fn kv_arc(&self) -> &Arc<CrowdbKvClient> { + pub fn kv_arc(&self) -> &Arc<PaxosKvClient> { &self.kv } @@ -240,8 +240,8 @@ impl OpContext { } /// Resolve the HTTP management URL for a node's deployed - /// `crowdb-kv-server`. This is the `ServerEntry.url` field (e.g. - /// `http://127.0.0.1:10000`). + /// `crowdb-paxos-kv-server`. This is the `ServerEntry.url` field (e.g. + /// `http://127.0.0.1:10135`). /// /// # Errors /// Returns [`Error::NotFound`] if no server is deployed on the node. diff --git a/lib/crowdb-console-shared/src/ops/hardware/authority.rs b/lib/crowdb-console-shared/src/ops/hardware/authority.rs index 6cedce19e..b976d3368 100644 --- a/lib/crowdb-console-shared/src/ops/hardware/authority.rs +++ b/lib/crowdb-console-shared/src/ops/hardware/authority.rs @@ -3,7 +3,7 @@ //! Conditional Group 0 hardware publication with confirmed outcomes. -use crowdb_kv_client::{BatchOp, Error as KvError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, Error as KvError, GetOutcome, ReadMode}; use crowdb_protocol::key::TextKey; use crowdb_protocol::{ common::{NodeValue, RackValue}, diff --git a/lib/crowdb-console-shared/src/ops/hardware/authority_storage.rs b/lib/crowdb-console-shared/src/ops/hardware/authority_storage.rs index cec5dbda3..6aead6f48 100644 --- a/lib/crowdb-console-shared/src/ops/hardware/authority_storage.rs +++ b/lib/crowdb-console-shared/src/ops/hardware/authority_storage.rs @@ -3,7 +3,7 @@ //! Rack-fenced disk-group and disk mutations in Group 0. -use crowdb_kv_client::{BatchOp, Error as KvError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, Error as KvError, GetOutcome, ReadMode}; use crowdb_protocol::common::{DiskId, HwStatus, NodeValue, RackValue}; use crowdb_protocol::diskdb::rpc::{DiskGroupValue, DiskValue}; use crowdb_protocol::key::{BindMapKey, DiskGroupKey, DiskKey, NodeKey, RackKey, TextKey}; diff --git a/lib/crowdb-console-shared/src/ops/hardware/relocation.rs b/lib/crowdb-console-shared/src/ops/hardware/relocation.rs index 633686522..2b5006b17 100644 --- a/lib/crowdb-console-shared/src/ops/hardware/relocation.rs +++ b/lib/crowdb-console-shared/src/ops/hardware/relocation.rs @@ -8,7 +8,7 @@ use crate::{ error::{Error, Result}, ops::OpContext, }; -use crowdb_kv_client::{BatchOp, Error as KvError, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, Error as KvError, GetOutcome, ReadMode}; use crowdb_protocol::{ common::{NodeValue, RackValue}, key::{NodeKey, RackKey, TextKey}, diff --git a/lib/crowdb-console-shared/src/ops/kv_data.rs b/lib/crowdb-console-shared/src/ops/kv_data.rs index 1242e162a..cefa33444 100644 --- a/lib/crowdb-console-shared/src/ops/kv_data.rs +++ b/lib/crowdb-console-shared/src/ops/kv_data.rs @@ -3,11 +3,11 @@ //! KV data-plane operations: put, get, delete, scan, snapshot. //! -//! Thin wrappers around [`CrowdbKvClient`] that resolve the leader from +//! Thin wrappers around [`PaxosKvClient`] that resolve the leader from //! the topology cache and translate the client's `Error` into the //! console's `Error`. -use crowdb_kv_client::{ +use crowdb_paxos_kv_client::{ CreateSnapshotResponse, GetOutcome, ReadMode, ReleaseSnapshotResponse, ScanOutcome, SnapshotInfo, SnapshotScanResponse, WriteOutcome, }; diff --git a/lib/crowdb-console-shared/src/ops/kv_logical.rs b/lib/crowdb-console-shared/src/ops/kv_logical.rs index 57088a747..ce6e36a0f 100644 --- a/lib/crowdb-console-shared/src/ops/kv_logical.rs +++ b/lib/crowdb-console-shared/src/ops/kv_logical.rs @@ -10,7 +10,7 @@ use std::collections::HashSet; -use crowdb_kv_client::GroupMembershipSnapshot; +use crowdb_paxos_kv_client::GroupMembershipSnapshot; use crowdb_protocol::key::{KvGroupKey, KvReplicaKey, KvStoreKey}; use crowdb_protocol::kv_membership::{canonical_members, GroupMember, GroupMembershipState}; use crowdb_protocol::mgmt::{ @@ -49,9 +49,9 @@ pub async fn create_local_group_for_tests( /// Returns conflicts for other members or epochs and propagates storage errors. #[cfg(feature = "test-util")] pub async fn initial_membership_for_tests( - authority: &crowdb_kv_client::GroupMembershipClient, + authority: &crowdb_paxos_kv_client::GroupMembershipClient, members: Vec<GroupMember>, -) -> Result<crowdb_kv_client::GroupMembershipSnapshot> { +) -> Result<crowdb_paxos_kv_client::GroupMembershipSnapshot> { publication::initial_membership(authority, 0, 1, members).await } @@ -61,8 +61,8 @@ pub async fn initial_membership_for_tests( /// Returns conflicts for superseded membership and propagates storage errors. #[cfg(feature = "test-util")] pub async fn complete_initial_membership_for_tests( - authority: &crowdb_kv_client::GroupMembershipClient, - snapshot: &crowdb_kv_client::GroupMembershipSnapshot, + authority: &crowdb_paxos_kv_client::GroupMembershipClient, + snapshot: &crowdb_paxos_kv_client::GroupMembershipSnapshot, ) -> Result<()> { publication::complete_initial(authority, snapshot).await } diff --git a/lib/crowdb-console-shared/src/ops/kv_logical/default_group.rs b/lib/crowdb-console-shared/src/ops/kv_logical/default_group.rs index d8201442b..38cb8f46c 100644 --- a/lib/crowdb-console-shared/src/ops/kv_logical/default_group.rs +++ b/lib/crowdb-console-shared/src/ops/kv_logical/default_group.rs @@ -86,7 +86,7 @@ pub async fn ensure(ctx: &OpContext, accepted: &DefaultPreparation) -> Result<bo group.store_id, group.group_id, b"/deployment/default-readiness", - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ), ) diff --git a/lib/crowdb-console-shared/src/ops/kv_logical/publication.rs b/lib/crowdb-console-shared/src/ops/kv_logical/publication.rs index caf02720e..c6042687e 100644 --- a/lib/crowdb-console-shared/src/ops/kv_logical/publication.rs +++ b/lib/crowdb-console-shared/src/ops/kv_logical/publication.rs @@ -3,7 +3,7 @@ //! Publish logical membership without overwriting a concurrent winner. -use crowdb_kv_client::{ +use crowdb_paxos_kv_client::{ BatchOp, Error as KvError, GetOutcome, GroupMembershipClient, GroupMembershipSnapshot, ReadMode, }; use crowdb_protocol::common::{GroupValue, ReplicaValue, StoreValue}; diff --git a/lib/crowdb-console-shared/src/ops/kv_logical/replica_join.rs b/lib/crowdb-console-shared/src/ops/kv_logical/replica_join.rs index b8418ef61..8c501e713 100644 --- a/lib/crowdb-console-shared/src/ops/kv_logical/replica_join.rs +++ b/lib/crowdb-console-shared/src/ops/kv_logical/replica_join.rs @@ -19,17 +19,17 @@ pub(super) async fn bootstrap( 0, 0, crate::deployment::CLUSTER_KEY, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, ) .await? { - crowdb_kv_client::GetOutcome::Found { value, .. } => { + crowdb_paxos_kv_client::GetOutcome::Found { value, .. } => { let operation: crate::deployment::PreparedBootstrap = serde_json::from_slice(&value).map_err(|error| Error::Config(error.to_string()))?; Ok(Some(operation.identity)) } - crowdb_kv_client::GetOutcome::NotFound => Ok(None), + crowdb_paxos_kv_client::GetOutcome::NotFound => Ok(None), } } diff --git a/lib/crowdb-console-shared/src/ops/kv_server.rs b/lib/crowdb-console-shared/src/ops/kv_server.rs index 933e6f403..a0212a09f 100644 --- a/lib/crowdb-console-shared/src/ops/kv_server.rs +++ b/lib/crowdb-console-shared/src/ops/kv_server.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! `crowdb-kv-server` lifecycle: deploy, restart, stop, delete, list. +//! `crowdb-paxos-kv-server` lifecycle: deploy, restart, stop, delete, list. //! //! Each function mutates the local TOML [`ConsoleConfig`] (adding / //! removing [`ServerEntry`] records) and spawns / stops the server @@ -81,7 +81,7 @@ fn restart_seed_hints(ctx: &OpContext, configured_seeds: &[String]) -> Result<Ve Ok(seeds) } -/// Deploy a `crowdb-kv-server` on a node. +/// Deploy a `crowdb-paxos-kv-server` on a node. /// /// `workspace_dir` is used for local-fork deploys (the server's data /// directory); `None` uses the current directory. SSH deploys ignore it. @@ -111,7 +111,7 @@ pub async fn deploy( let binary = req.binary.as_ref().map(|p| p.to_string_lossy().to_string()); let deployed = if node.ssh_enabled() { let server_bin = binary.clone().unwrap_or_else(|| { - std::env::var("CROWDB_KV_SERVER_BIN").unwrap_or_else(|_| "crowdb-kv-server".into()) + std::env::var("CROWDB_PAXOS_KV_SERVER_BIN").unwrap_or_else(|_| "crowdb-paxos-kv-server".into()) }); crate::ssh::deploy_via_ssh(&launch, &node, &server_bin) .await @@ -153,7 +153,7 @@ pub async fn deploy( Ok(deployed) } -/// Stop the `crowdb-kv-server` on a node (keep the deployment record). +/// Stop the `crowdb-paxos-kv-server` on a node (keep the deployment record). /// /// `pid_override` lets callers supply an in-memory PID (e.g. the web's /// `runtime_pid`) that takes precedence over the persisted `entry.pid`. @@ -192,7 +192,7 @@ pub async fn stop(ctx: &OpContext, node_id: NodeId, pid_override: Option<u32>) - Ok(sent) } -/// Restart the `crowdb-kv-server` on a node: stop (if running) and +/// Restart the `crowdb-paxos-kv-server` on a node: stop (if running) and /// re-deploy on the same recorded ports. /// /// `workspace_dir` is used for local-fork redeploys; `None` uses the @@ -253,7 +253,8 @@ pub async fn restart( }; let deployed = if node.ssh_enabled() { - let server_bin = std::env::var("CROWDB_KV_SERVER_BIN").unwrap_or_else(|_| "crowdb-kv-server".into()); + let server_bin = + std::env::var("CROWDB_PAXOS_KV_SERVER_BIN").unwrap_or_else(|_| "crowdb-paxos-kv-server".into()); crate::ssh::deploy_via_ssh(&req, &node, &server_bin) .await .map_err(|e| Error::NodeUnreachable { diff --git a/lib/crowdb-console-shared/src/ssh.rs b/lib/crowdb-console-shared/src/ssh.rs index faa9816ab..4efdd2fb7 100644 --- a/lib/crowdb-console-shared/src/ssh.rs +++ b/lib/crowdb-console-shared/src/ssh.rs @@ -234,7 +234,7 @@ impl Session { } } -/// Run a no-op command (`echo crowdb-kv-ping`) and assert the round trip. +/// Run a no-op command (`echo crowdb-paxos-kv-ping`) and assert the round trip. /// /// # Errors /// `Error::UpstreamRpc` on connect, auth, or exec failure; or if the @@ -242,7 +242,7 @@ impl Session { pub async fn probe(node: &NodeEntry) -> Result<()> { let creds = SshCreds::resolve(node)?; let mut session = Session::connect(node, &creds).await?; - let out = session.exec("echo crowdb-kv-ping").await?; + let out = session.exec("echo crowdb-paxos-kv-ping").await?; session.close().await; if !out.success() { @@ -251,7 +251,7 @@ pub async fn probe(node: &NodeEntry) -> Result<()> { status: format!("probe exit={:?}, stderr={}", out.exit, out.stderr_str()), }); } - if !out.stdout_str().contains("crowdb-kv-ping") { + if !out.stdout_str().contains("crowdb-paxos-kv-ping") { return Err(Error::UpstreamRpc { node_id: node.host.clone(), status: format!("probe got unexpected stdout: {}", out.stdout_str()), @@ -311,7 +311,7 @@ impl client::Handler for ClientHandler { } /// Resolve the process-wide `known_hosts` store. The first call computes -/// the path (`$CROWDB_KV_KNOWN_HOSTS` or `~/.crowdb-kv/known_hosts`) and +/// the path (`$CROWDB_PAXOS_KV_KNOWN_HOSTS` or `~/.crowdb-paxos-kv/known_hosts`) and /// memoizes the resulting `KnownHostsStore` for the rest of the /// process. If neither path is resolvable we still hand back an /// in-memory store so connections can proceed under TOFU rules within @@ -325,7 +325,7 @@ fn known_hosts_store_for_session() -> Result<Arc<KnownHostsStore>> { let path = KnownHostsStore::default_path().unwrap_or_else(|| { // No HOME or override: keep the fallback inside the workspace runtime // namespace so cleanup and inspection remain deterministic. - warn!("ssh: cannot resolve $HOME or $CROWDB_KV_KNOWN_HOSTS; using in-memory known_hosts (keys will not persist)"); + warn!("ssh: cannot resolve $HOME or $CROWDB_PAXOS_KV_KNOWN_HOSTS; using in-memory known_hosts (keys will not persist)"); crowdb_protocol::port::namespace::runtime_root() .join("persistent") .join("ssh") diff --git a/lib/crowdb-console-shared/src/ssh/known_hosts.rs b/lib/crowdb-console-shared/src/ssh/known_hosts.rs index 77ed50eaf..1b309c3d1 100644 --- a/lib/crowdb-console-shared/src/ssh/known_hosts.rs +++ b/lib/crowdb-console-shared/src/ssh/known_hosts.rs @@ -74,14 +74,14 @@ pub(crate) struct KnownHostsStore { } impl KnownHostsStore { - /// Default file path: `$CROWDB_KV_KNOWN_HOSTS` if set, else - /// `~/.crowdb-kv/known_hosts`. + /// Default file path: `$CROWDB_PAXOS_KV_KNOWN_HOSTS` if set, else + /// `~/.crowdb-paxos-kv/known_hosts`. #[must_use] pub(crate) fn default_path() -> Option<PathBuf> { - if let Ok(p) = std::env::var("CROWDB_KV_KNOWN_HOSTS") { + if let Ok(p) = std::env::var("CROWDB_PAXOS_KV_KNOWN_HOSTS") { return Some(PathBuf::from(p)); } - dirs::home_dir().map(|h| h.join(".crowdb-kv").join("known_hosts")) + dirs::home_dir().map(|h| h.join(".crowdb-paxos-kv").join("known_hosts")) } /// Open (or create) the store at `path`. A missing file is diff --git a/lib/crowdb-console-shared/tests/admission_control_failure_test.rs b/lib/crowdb-console-shared/tests/admission_control_failure_test.rs index ad1c33c19..2a463a525 100644 --- a/lib/crowdb-console-shared/tests/admission_control_failure_test.rs +++ b/lib/crowdb-console-shared/tests/admission_control_failure_test.rs @@ -10,7 +10,7 @@ mod ssh_server; #[tokio::test] async fn dependency_wait_survives_the_remote_monitor_control_boundary() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("monitor-dependency-wait"); - std::env::set_var("CROWDB_KV_KNOWN_HOSTS", root.path().join("known-hosts")); + std::env::set_var("CROWDB_PAXOS_KV_KNOWN_HOSTS", root.path().join("known-hosts")); let key = russh::keys::key::KeyPair::generate_ed25519().unwrap(); let path = root.path().join("operator-key"); russh::keys::encode_pkcs8_pem(&key, std::fs::File::create(&path).unwrap()).unwrap(); diff --git a/lib/crowdb-console-shared/tests/bootstrap_hardware_membership_test.rs b/lib/crowdb-console-shared/tests/bootstrap_hardware_membership_test.rs index 127cc54b9..f6180a7ac 100644 --- a/lib/crowdb-console-shared/tests/bootstrap_hardware_membership_test.rs +++ b/lib/crowdb-console-shared/tests/bootstrap_hardware_membership_test.rs @@ -7,8 +7,8 @@ use crowdb_console_shared::{ config::ConsoleConfig, ops::{cluster::publish_topology_for_tests, OpContext}, }; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; use serde_json::json; diff --git a/lib/crowdb-console-shared/tests/bootstrap_intent_test.rs b/lib/crowdb-console-shared/tests/bootstrap_intent_test.rs index f9703b2a5..67e2fd9f0 100644 --- a/lib/crowdb-console-shared/tests/bootstrap_intent_test.rs +++ b/lib/crowdb-console-shared/tests/bootstrap_intent_test.rs @@ -33,7 +33,7 @@ fn config() -> ConsoleConfig { rest_port: Some(10000), rpc_port: Some(10100), auto_start: true, - binary: Some("/private/bin/crowdb-kv-server".into()), + binary: Some("/private/bin/crowdb-paxos-kv-server".into()), election_profile: None, pid: Some(42), service_type: ServiceType::PaxosKv, diff --git a/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs index 2bf45d7eb..ed01bbab8 100644 --- a/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs +++ b/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs @@ -153,7 +153,7 @@ async fn storage_diagnostics_follow_disk_owner_space_dependencies() { owner.lease_expiry_ms = u64::try_from(now).unwrap() + 120_000; put(&authority, "/hw/dg_owner/4/2/1", &owner).await; storage_layer(&authority, "/owner", Some("storage-owner-not-live")).await; - let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()); + let registry = crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()); registry .register_diskdb(42, "127.0.0.1:7200", &[], &[]) .await @@ -231,7 +231,7 @@ async fn space_must_come_from_the_current_owner_instead_of_a_detached_summary() }, ) .await; - let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()); + let registry = crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()); registry .register_diskdb(42, "127.0.0.1:7200", &[1], &[]) .await diff --git a/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs index 837f72c94..f2d9fb0cb 100644 --- a/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs +++ b/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs @@ -88,7 +88,7 @@ async fn setup() -> (TestPreparationAuthority, NodeRuntimeObservation) { statuses.push(status); usage.push(summary); } - crowdb_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()) + crowdb_paxos_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()) .register_diskdb(42, "127.0.0.1:7200", &[7, 9], &usage) .await .unwrap(); diff --git a/lib/crowdb-console-shared/tests/common/default_metadata.rs b/lib/crowdb-console-shared/tests/common/default_metadata.rs index c47c0a884..5b6bab137 100644 --- a/lib/crowdb-console-shared/tests/common/default_metadata.rs +++ b/lib/crowdb-console-shared/tests/common/default_metadata.rs @@ -3,13 +3,13 @@ use super::default_preparation::TestPreparationAuthority; use crowdb_console_shared::deployment::preparation::{self, DefaultPreparation}; -use crowdb_kv::{ - cluster::{KvServer, PxKvStore}, - common::config::CrowDBConfig, +use crowdb_paxos_kv::{ + cluster::{PaxosKvServer, PxKvStore}, + common::config::PaxosKvConfig, }; -use crowdb_kv_server::{ +use crowdb_paxos_kv_server::{ mgmt::{self, operation_registry::AppState}, - store_registry::KvStoreRegistry, + store_registry::PaxosKvStoreRegistry, }; use crowdb_protocol::common::KvServerIdentity; use std::sync::Arc; @@ -35,15 +35,15 @@ impl TestDefaultMetadata { let mut endpoints = Vec::new(); for (index, node) in members.iter().enumerate() { let path = root.path().join(format!("node-{node}")); - let config = CrowDBConfig { + let config = PaxosKvConfig { config_root: path.join("config"), data_root: path.join("data"), wal_root: path.join("wal"), wal_backend: "file".into(), crowtree_backend: "file".into(), - ..CrowDBConfig::for_tests() + ..PaxosKvConfig::for_tests() }; - let registry = Arc::new(KvStoreRegistry::with_config(config)); + let registry = Arc::new(PaxosKvStoreRegistry::with_config(config)); let store = if index == 0 { Arc::clone(&authority.store) } else { diff --git a/lib/crowdb-console-shared/tests/common/default_preparation.rs b/lib/crowdb-console-shared/tests/common/default_preparation.rs index de5954657..5a4dfebea 100644 --- a/lib/crowdb-console-shared/tests/common/default_preparation.rs +++ b/lib/crowdb-console-shared/tests/common/default_preparation.rs @@ -7,8 +7,10 @@ use crowdb_console_shared::{ ops::OpContext, ConsoleConfig, }; -use crowdb_kv::cluster::{group::PxGroup, KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv::cluster::{ + group::PxGroup, PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use std::sync::Arc; pub struct TestPreparationAuthority { @@ -53,7 +55,7 @@ impl TestPreparationAuthority { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("default-preparation"); let bootstrap = PreparedBootstrap::open(&root.path().join("prepared.json"), &config, members).unwrap(); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![endpoint.clone()]))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![endpoint.clone()]))); kv.seed_leader(0, 0, endpoint); kv.put(0, 0, CLUSTER_KEY, &serde_json::to_vec(&bootstrap).unwrap(), None) .await diff --git a/lib/crowdb-console-shared/tests/common/regression/inspection.rs b/lib/crowdb-console-shared/tests/common/regression/inspection.rs index 5f4b9a151..980d6b448 100644 --- a/lib/crowdb-console-shared/tests/common/regression/inspection.rs +++ b/lib/crowdb-console-shared/tests/common/regression/inspection.rs @@ -3,7 +3,7 @@ use super::TestResult; use crowdb_console_shared::ops::OpContext; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::{ chunk_kv::{ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage}, chunk_stream::{decode_stream_chunk_record, StreamName, StreamRecord}, diff --git a/lib/crowdb-console-shared/tests/config_web_test.rs b/lib/crowdb-console-shared/tests/config_web_test.rs index 98502fbb2..398d798b1 100644 --- a/lib/crowdb-console-shared/tests/config_web_test.rs +++ b/lib/crowdb-console-shared/tests/config_web_test.rs @@ -63,7 +63,7 @@ node_id = 1 service_id = "kv" host = "localhost" ssh_credential_ref = "operator-key" -binary_path = "/opt/crowdb/bin/crowdb-kv-server" +binary_path = "/opt/crowdb/bin/crowdb-paxos-kv-server" service_config_path = "/opt/crowdb/run/config/kv.toml" workspace = "/opt/crowdb/run" auto_start = false diff --git a/lib/crowdb-console-shared/tests/default_metadata_test.rs b/lib/crowdb-console-shared/tests/default_metadata_test.rs index 4efc8379c..d77bd6dd4 100644 --- a/lib/crowdb-console-shared/tests/default_metadata_test.rs +++ b/lib/crowdb-console-shared/tests/default_metadata_test.rs @@ -7,8 +7,8 @@ mod default_metadata; mod default_preparation; use crowdb_console_shared::deployment::preparation; -use crowdb_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use default_metadata::TestDefaultMetadata; async fn ready(fixture: &TestDefaultMetadata) { @@ -70,7 +70,7 @@ async fn three_owned_native_replicas_publish_native_listeners_and_allow_lineariz .zip(&fixture.stores) .zip(&fixture.endpoints) { - use crowdb_kv::cluster::KvServer; + use crowdb_paxos_kv::cluster::PaxosKvServer; assert_eq!(member.endpoint, store.listen_addr().unwrap().to_string()); assert_ne!(format!("http://{}", member.endpoint), *management); } diff --git a/lib/crowdb-console-shared/tests/default_preparation_test.rs b/lib/crowdb-console-shared/tests/default_preparation_test.rs index abe1b0090..ba51ed898 100644 --- a/lib/crowdb-console-shared/tests/default_preparation_test.rs +++ b/lib/crowdb-console-shared/tests/default_preparation_test.rs @@ -9,7 +9,7 @@ use crowdb_console_shared::deployment::{ preparation::{self, DEFAULT_PREPARATION_KEY}, CLUSTER_KEY, }; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::{ common::GroupValue, key::{KvGroupKey, TextKey}, diff --git a/lib/crowdb-console-shared/tests/group_publication_replay_test.rs b/lib/crowdb-console-shared/tests/group_publication_replay_test.rs index 674ec057e..60736166f 100644 --- a/lib/crowdb-console-shared/tests/group_publication_replay_test.rs +++ b/lib/crowdb-console-shared/tests/group_publication_replay_test.rs @@ -4,13 +4,13 @@ use crowdb_console_shared::ops::kv_logical::{ complete_initial_membership_for_tests, initial_membership_for_tests, }; -use crowdb_kv::cluster::{ +use crowdb_paxos_kv::cluster::{ group::PxGroup, - kv_server::KvServer, local_replica::{PxLocalReplica, PxLocalReplicaRole}, + paxos_kv_server::PaxosKvServer, px_kv_store::PxKvStore, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GroupMembershipClient}; +use crowdb_paxos_kv_client::{ClientConfig, GroupMembershipClient, PaxosKvClient}; use crowdb_protocol::kv_membership::{GroupMember, GroupMembershipState}; use std::sync::Arc; @@ -22,7 +22,7 @@ async fn authority() -> (Arc<PxKvStore>, GroupMembershipClient, Vec<GroupMember> )); store.start().await.unwrap(); let endpoint = store.listen_addr().unwrap().to_string(); - let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))); + let client = Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))); client.seed_leader(0, 0, endpoint.clone()); let member = GroupMember { node_id: 1, diff --git a/lib/crowdb-console-shared/tests/launch_ssh_test.rs b/lib/crowdb-console-shared/tests/launch_ssh_test.rs index 8c055d6c9..d0cac0527 100644 --- a/lib/crowdb-console-shared/tests/launch_ssh_test.rs +++ b/lib/crowdb-console-shared/tests/launch_ssh_test.rs @@ -26,7 +26,7 @@ impl Drop for TestProcess { #[tokio::test] async fn ssh_launch_uses_referenced_key_and_preserves_literal_arguments() { let dir = tempdir_in_test_data("launch-ssh"); - std::env::set_var("CROWDB_KV_KNOWN_HOSTS", dir.path().join("known-hosts")); + std::env::set_var("CROWDB_PAXOS_KV_KNOWN_HOSTS", dir.path().join("known-hosts")); let key = russh::keys::key::KeyPair::generate_ed25519().unwrap(); let credentials = dir.path().join("credentials"); std::fs::create_dir_all(&credentials).unwrap(); diff --git a/lib/crowdb-console-shared/tests/member_recovery_test.rs b/lib/crowdb-console-shared/tests/member_recovery_test.rs index 7a0a420b4..0b888c898 100644 --- a/lib/crowdb-console-shared/tests/member_recovery_test.rs +++ b/lib/crowdb-console-shared/tests/member_recovery_test.rs @@ -123,7 +123,7 @@ async fn pinned_identity_survives_endpoint_changes_without_address_tofu() { let dir = tempdir_in_test_data("member-recovery-move"); let address_cache = dir.path().join("address-cache"); std::fs::write(&address_cache, "deliberately invalid address-keyed trust").unwrap(); - std::env::set_var("CROWDB_KV_KNOWN_HOSTS", &address_cache); + std::env::set_var("CROWDB_PAXOS_KV_KNOWN_HOSTS", &address_cache); let key = KeyPair::generate_ed25519().unwrap(); let host = KeyPair::generate_ed25519().unwrap(); let trust = recovery::trust(&host); diff --git a/lib/crowdb-console-shared/tests/ops_context_test.rs b/lib/crowdb-console-shared/tests/ops_context_test.rs index 6795c5f02..0e0087df5 100644 --- a/lib/crowdb-console-shared/tests/ops_context_test.rs +++ b/lib/crowdb-console-shared/tests/ops_context_test.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. //! Construction tests for [`OpContext`]. The sysmd + kv clients share -//! the same underlying `Arc<CrowdbKvClient>`; the config is held behind +//! the same underlying `Arc<PaxosKvClient>`; the config is held behind //! an `RwLock` and accessible via `config()` / `config_mut()`. use crowdb_console_shared::config::ConsoleConfig; @@ -15,7 +15,7 @@ fn op_context_shares_arc_between_sysmd_and_kv() { vec!["http://127.0.0.1:10000".into()], ConsoleConfig::default(), ); - // Both sysmd and kv wrap the same Arc<CrowdbKvClient>, so the + // Both sysmd and kv wrap the same Arc<PaxosKvClient>, so the // references returned by kv() must point to the same allocation. assert!(std::ptr::eq(ctx.sysmd().kv(), ctx.kv())); } diff --git a/lib/crowdb-console-shared/tests/ops_kv_server_test.rs b/lib/crowdb-console-shared/tests/ops_kv_server_test.rs index a41a03bc7..a5067cc3d 100644 --- a/lib/crowdb-console-shared/tests/ops_kv_server_test.rs +++ b/lib/crowdb-console-shared/tests/ops_kv_server_test.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. //! Tests for [`ops::kv_server`] validation paths. The actual deploy / -//! stop requires a real `crowdb-kv-server` binary and is covered by E2E +//! stop requires a real `crowdb-paxos-kv-server` binary and is covered by E2E //! tests in Phase 4; here we verify the guard clauses. use crowdb_console_shared::config::{ConsoleConfig, NodeEntry, ServerEntry, ServiceType}; diff --git a/lib/crowdb-console-shared/tests/planned_service_start_test.rs b/lib/crowdb-console-shared/tests/planned_service_start_test.rs index 48db9c47d..4aa5a4df8 100644 --- a/lib/crowdb-console-shared/tests/planned_service_start_test.rs +++ b/lib/crowdb-console-shared/tests/planned_service_start_test.rs @@ -2,16 +2,16 @@ // Licensed under the Apache License, Version 2.0. use crowdb_console_shared::deployment::services::{current, publish_planned_start}; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; use std::sync::Arc; struct TestAuthority { store: Arc<PxKvStore>, - client: CrowdbKvClient, + client: PaxosKvClient, } impl TestAuthority { async fn start() -> Self { @@ -21,7 +21,7 @@ impl TestAuthority { PxLocalReplica::new(1, PxLocalReplicaRole::Leader), )); store.start().await.unwrap(); - let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + let client = PaxosKvClient::new(ClientConfig::new(vec![])); client.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); Self { store, client } } diff --git a/lib/crowdb-console-shared/tests/topology_aggregate_test.rs b/lib/crowdb-console-shared/tests/topology_aggregate_test.rs index d09b2a7eb..93f285761 100644 --- a/lib/crowdb-console-shared/tests/topology_aggregate_test.rs +++ b/lib/crowdb-console-shared/tests/topology_aggregate_test.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. //! Aggregator integration test using a stand-in HTTP server that mimics -//! `crowdb-kv-server`'s `/health` + `/topology` endpoints. +//! `crowdb-paxos-kv-server`'s `/health` + `/topology` endpoints. use std::net::SocketAddr; diff --git a/lib/crowdb-diskdb-client/Cargo.toml b/lib/crowdb-diskdb-client/Cargo.toml index ba6fc286c..f94eef3a0 100644 --- a/lib/crowdb-diskdb-client/Cargo.toml +++ b/lib/crowdb-diskdb-client/Cargo.toml @@ -18,7 +18,7 @@ test-util = [] [dependencies] arc-swap = "1.9" crowdb-protocol = { path = "../crowdb-protocol" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } tokio = { workspace = true, features = ["rt", "macros", "time", "sync"] } bytes = "1" @@ -28,7 +28,7 @@ tracing = { workspace = true } [dev-dependencies] crowdb-diskdb-client = { path = ".", features = ["test-util"] } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../crowdb-test-harness" } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time"] } diff --git a/lib/crowdb-diskdb-client/src/client.rs b/lib/crowdb-diskdb-client/src/client.rs index c76390401..90a212cac 100644 --- a/lib/crowdb-diskdb-client/src/client.rs +++ b/lib/crowdb-diskdb-client/src/client.rs @@ -18,7 +18,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tracing::warn; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::common::DiskId; use crowdb_protocol::diskdb::rpc::{ AllocateBlocksRequest, AllocateResponse, CommitBlocksRequest, CommitBlocksResponse, CompactZoneRequest, diff --git a/lib/crowdb-diskdb-client/src/lib.rs b/lib/crowdb-diskdb-client/src/lib.rs index 60195d971..77299ead5 100644 --- a/lib/crowdb-diskdb-client/src/lib.rs +++ b/lib/crowdb-diskdb-client/src/lib.rs @@ -4,7 +4,7 @@ //! Client library for CROWDB diskdb. //! //! Provides allocate/free/query APIs with retry and endpoint caching, -//! mirroring `crowdb-kv-client`'s pattern. The client discovers diskdb +//! mirroring `crowdb-paxos-kv-client`'s pattern. The client discovers diskdb //! instances via the service registry (group 0), caches //! `disk_group_id -> rpc_endpoint`, and lazily refreshes on cache //! miss or `Unavailable`. diff --git a/lib/crowdb-diskdb-client/tests/routing_snapshot_test.rs b/lib/crowdb-diskdb-client/tests/routing_snapshot_test.rs index cf7ed34d2..ba88dbbc2 100644 --- a/lib/crowdb-diskdb-client/tests/routing_snapshot_test.rs +++ b/lib/crowdb-diskdb-client/tests/routing_snapshot_test.rs @@ -4,11 +4,11 @@ use std::sync::Arc; use crowdb_diskdb_client::{DiskdbClient, DiskdbRpcTransport}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::common::DiskId; fn client() -> DiskdbClient { - let service = ServiceRegistryClient::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))); + let service = ServiceRegistryClient::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))); DiskdbClient::new(service, Arc::new(DiskdbRpcTransport::new())) } diff --git a/lib/crowdb-diskio-client/Cargo.toml b/lib/crowdb-diskio-client/Cargo.toml index 906bf33c9..77f29a354 100644 --- a/lib/crowdb-diskio-client/Cargo.toml +++ b/lib/crowdb-diskio-client/Cargo.toml @@ -19,7 +19,7 @@ workspace = true bytes = "1" arc-swap = "1.9" crowdb-common = { path = "../crowdb-common/rust" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-protocol = { path = "../crowdb-protocol" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } flatbuffers = { workspace = true } @@ -29,7 +29,7 @@ thiserror = { workspace = true } tracing = { workspace = true } [dev-dependencies] -crowdb-kv = { path = "../crowdb-kv" } +crowdb-paxos-kv = { path = "../crowdb-paxos-kv" } crowdb-diskio-client = { path = ".", features = ["test-util"] } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "net", "io-util"] } diff --git a/lib/crowdb-diskio-client/src/client.rs b/lib/crowdb-diskio-client/src/client.rs index 8ce2d0042..11a6cf7ae 100644 --- a/lib/crowdb-diskio-client/src/client.rs +++ b/lib/crowdb-diskio-client/src/client.rs @@ -210,7 +210,7 @@ impl WireClient { sync: false, write_create_time_ms: unix_time_ms(), }, - Buffer::from_owned_bytes(data), + Buffer::try_from_owned_bytes(data).map_err(|error| WireError::Protocol(error.to_string()))?, ) } @@ -240,7 +240,7 @@ impl WireClient { sync: target.sync, write_create_time_ms: target.write_create_time_ms, }, - Buffer::from_owned_bytes(data), + Buffer::try_from_owned_bytes(data).map_err(|error| WireError::Protocol(error.to_string()))?, ) } @@ -358,6 +358,8 @@ impl WireClient { zone_offset, size, test_pattern_offset, + read_create_time_ms: 0, + fence_probe: false, }, ); fbb.finish(off, None); diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index 36388c453..c1b92c84d 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -12,7 +12,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use arc_swap::ArcSwap; use bytes::Bytes; -use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_rpc_ffi::{ ConnectionPoolError, ConnectionPoolIndex, OwnedClientRoute, OwnedPayload, RpcError, RpcServer, SelectedConnection, diff --git a/lib/crowdb-diskio-client/src/semantic/connection.rs b/lib/crowdb-diskio-client/src/semantic/connection.rs index b02109384..02ac1c2e7 100644 --- a/lib/crowdb-diskio-client/src/semantic/connection.rs +++ b/lib/crowdb-diskio-client/src/semantic/connection.rs @@ -7,7 +7,7 @@ use super::{ Counters, DiskioClient, DiskioClientConfig, DiskioError, DiskioResult, RouteSnapshot, WireClient, }; use arc_swap::ArcSwap; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_rpc_ffi::{ConnectionPoolIndex, RdmaWireLayout, RpcServer}; use std::sync::{atomic::AtomicU64, Arc}; @@ -61,7 +61,7 @@ impl DiskioClient { scope: Option<crowdb_protocol::key::DiskKey>, ) -> DiskioResult<Self> { config.validate(true)?; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new( config.management_seeds.clone(), ))); kv.refresh_topology() diff --git a/lib/crowdb-diskio-client/src/topology.rs b/lib/crowdb-diskio-client/src/topology.rs index 6ab709e34..e7bacf7f9 100644 --- a/lib/crowdb-diskio-client/src/topology.rs +++ b/lib/crowdb-diskio-client/src/topology.rs @@ -6,7 +6,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use crowdb_kv_client::{DiskRecord, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{DiskRecord, HardwareClient, ServiceRegistryClient}; use crowdb_protocol::common::InstanceValue; use crate::{DiskId, DiskioError, DiskioResult}; diff --git a/lib/crowdb-diskio-client/tests/scoped_discovery_test.rs b/lib/crowdb-diskio-client/tests/scoped_discovery_test.rs index d52dabceb..4e46e1768 100644 --- a/lib/crowdb-diskio-client/tests/scoped_discovery_test.rs +++ b/lib/crowdb-diskio-client/tests/scoped_discovery_test.rs @@ -4,10 +4,10 @@ use std::sync::Arc; use crowdb_diskio_client::{DiskioClient, DiskioClientConfig}; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::{ common::{DiskId, HwStatus}, diskdb::rpc::DiskValue, @@ -16,7 +16,7 @@ use crowdb_protocol::{ struct TestDiscovery { store: Arc<PxKvStore>, - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } impl TestDiscovery { async fn start() -> Self { @@ -26,7 +26,7 @@ impl TestDiscovery { PxLocalReplica::new(1, PxLocalReplicaRole::Leader), )); store.start().await.unwrap(); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![]))); kv.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); Self { store, kv } } diff --git a/lib/crowdb-kv-client/Cargo.toml b/lib/crowdb-paxos-kv-client/Cargo.toml similarity index 71% rename from lib/crowdb-kv-client/Cargo.toml rename to lib/crowdb-paxos-kv-client/Cargo.toml index 7097babb8..55bf67811 100644 --- a/lib/crowdb-kv-client/Cargo.toml +++ b/lib/crowdb-paxos-kv-client/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "crowdb-kv-client" +name = "crowdb-paxos-kv-client" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -13,10 +13,10 @@ description = "Standalone client library for CROWDB: topology cache, retry/idemp crate-type = ["rlib", "cdylib"] [features] -# Enables test-only seams on `crowdb-kv` (e.g. `PxKvStore::set_get_delay_for_tests`) +# Enables test-only seams on `crowdb-paxos-kv` (e.g. `PxKvStore::set_get_delay_for_tests`) # for this crate's own test build. Auto-enabled via the self dev-dependency below; -# no `--features` flag needed for `cargo test -p crowdb-kv-client`. -test-util = ["crowdb-kv/test-util"] +# no `--features` flag needed for `cargo test -p crowdb-paxos-kv-client`. +test-util = ["crowdb-paxos-kv/test-util"] # Enables the C ABI FFI module (ffi.rs) for C++ consumers like crowdb-diskio. # Adds tokio rt-multi-thread for the FFI's dedicated runtime. ffi = ["tokio/rt-multi-thread"] @@ -25,11 +25,11 @@ ffi = ["tokio/rt-multi-thread"] workspace = true [dependencies] -# `crowdb-kv` gives us the KV RPC types (`crowdb_kv::rpc::*`, hand-written +# `crowdb-paxos-kv` gives us the KV RPC types (`crowdb_paxos_kv::rpc::*`, hand-written # in `crowdb-protocol`) and the `/topology` response shape -# (`crowdb_kv::cluster::status::*`, already `Serialize`/`Deserialize`) -# without depending on `crowdb-kv-server` or `crowdb-console`. -crowdb-kv = { path = "../crowdb-kv" } +# (`crowdb_paxos_kv::cluster::status::*`, already `Serialize`/`Deserialize`) +# without depending on `crowdb-paxos-kv-server` or `crowdb-console`. +crowdb-paxos-kv = { path = "../crowdb-paxos-kv" } crowdb-protocol = { path = "../crowdb-protocol" } crowdb-common = { path = "../crowdb-common/rust" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } @@ -52,5 +52,5 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "te axum = "0.7" crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } # Self dev-dependency: auto-enables the `test-util` feature for this -# crate's own test build (same pattern as `crowdb-kv`'s self dev-dep). -crowdb-kv-client = { path = ".", features = ["test-util"] } +# crate's own test build (same pattern as `crowdb-paxos-kv`'s self dev-dep). +crowdb-paxos-kv-client = { path = ".", features = ["test-util"] } diff --git a/lib/crowdb-kv-client/include/crowdb-kv-client/c_api.h b/lib/crowdb-paxos-kv-client/include/crowdb-paxos-kv-client/c_api.h similarity index 98% rename from lib/crowdb-kv-client/include/crowdb-kv-client/c_api.h rename to lib/crowdb-paxos-kv-client/include/crowdb-paxos-kv-client/c_api.h index 9975c8c85..e2eaa38a6 100644 --- a/lib/crowdb-kv-client/include/crowdb-kv-client/c_api.h +++ b/lib/crowdb-paxos-kv-client/include/crowdb-paxos-kv-client/c_api.h @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// C ABI for crowdb-kv-client (ffi feature): exposes HardwareClient and +// C ABI for crowdb-paxos-kv-client (ffi feature): exposes HardwareClient and // ServiceRegistryClient operations to C++ consumers (crowdb-diskio). // // Async operations use a callback pattern: the caller provides a diff --git a/lib/crowdb-kv-client/src/binding.rs b/lib/crowdb-paxos-kv-client/src/binding.rs similarity index 100% rename from lib/crowdb-kv-client/src/binding.rs rename to lib/crowdb-paxos-kv-client/src/binding.rs diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots.rs b/lib/crowdb-paxos-kv-client/src/binding/chunk_slots.rs similarity index 95% rename from lib/crowdb-kv-client/src/binding/chunk_slots.rs rename to lib/crowdb-paxos-kv-client/src/binding/chunk_slots.rs index 20cc9a803..6b28b679f 100644 --- a/lib/crowdb-kv-client/src/binding/chunk_slots.rs +++ b/lib/crowdb-paxos-kv-client/src/binding/chunk_slots.rs @@ -4,6 +4,7 @@ //! Fixed-layout bitmap publication and consistent group-0 reads. mod handoff; +mod initialization; mod publication; pub use handoff::ChunkServiceHandoffSnapshot; pub use publication::ChunkServiceSnapshot; @@ -19,16 +20,16 @@ use crowdb_protocol::key::{ }; use serde::{de::DeserializeOwned, Serialize}; -use crate::{BatchOp, CrowdbKvClient, Error, GetOutcome, ReadMode, Result}; +use crate::{BatchOp, Error, GetOutcome, PaxosKvClient, ReadMode, Result}; /// Reads each layer independently; ordinary chunk clients only need service slots. pub struct ChunkSlotMapClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } impl ChunkSlotMapClient { #[must_use] - pub fn new(kv: Arc<CrowdbKvClient>) -> Self { + pub fn new(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } @@ -133,14 +134,17 @@ impl ChunkSlotMapClient { async fn initialize<O: MapOwner>(&self, map: &ChunkSlotMap<O>) -> Result<()> { self.reject_legacy().await?; - if self.read_head::<O>().await?.is_some() { - return self.check_initialized(map).await; + if let Some(pending) = self.initialization_state(map).await? { + return self.finish_initialization(map, pending).await; } self.reject_legacy_chunk_state().await?; let key = O::head().to_path(); if !self.scan_prefix(&O::prefix()).await?.is_empty() { return Err(invalid(&key, "orphan slot bindings require explicit recovery")); } + if map.bindings().len() >= initialization::MAX_BATCH_RECORDS { + return self.begin_initialization(map).await; + } let mut ops = Vec::with_capacity(map.bindings().len() + 1); for binding in map.bindings() { ops.push(put(binding.owner.key(), binding)?); diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots/handoff.rs b/lib/crowdb-paxos-kv-client/src/binding/chunk_slots/handoff.rs similarity index 100% rename from lib/crowdb-kv-client/src/binding/chunk_slots/handoff.rs rename to lib/crowdb-paxos-kv-client/src/binding/chunk_slots/handoff.rs diff --git a/lib/crowdb-paxos-kv-client/src/binding/chunk_slots/initialization.rs b/lib/crowdb-paxos-kv-client/src/binding/chunk_slots/initialization.rs new file mode 100644 index 000000000..873bd3bd1 --- /dev/null +++ b/lib/crowdb-paxos-kv-client/src/binding/chunk_slots/initialization.rs @@ -0,0 +1,154 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Head-fenced bootstrap pages, with the complete intent retained across restart. + +use serde::{Deserialize, Serialize}; + +use super::{decode, invalid, put, ChunkSlotMapClient, MapOwner}; +use crate::{Error, GetOutcome, ReadMode, Result}; +use crowdb_protocol::chunk_slot::{ChunkSlotBinding, ChunkSlotMap, ChunkSlotMapHead}; +use crowdb_protocol::key::TextKey; + +// Every proposal must fit the native tree's reserved replay entry window. +pub(super) const MAX_BATCH_RECORDS: usize = 64; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Intent<O> { + initializing: Layout<O>, + next: usize, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Layout<O> { + head: ChunkSlotMapHead, + bindings: Vec<ChunkSlotBinding<O>>, +} + +pub(super) struct Pending<O> { + intent: Option<Intent<O>>, + revision: u64, +} + +impl ChunkSlotMapClient { + // A regular head is already committed. An intent is deliberately not a + // ChunkSlotMapHead, so ordinary readers fail closed until publication. + pub(super) async fn initialization_state<O: MapOwner>( + &self, + expected: &ChunkSlotMap<O>, + ) -> Result<Option<Pending<O>>> { + let key = O::head().to_path(); + let GetOutcome::Found { value, revision } = self + .kv + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await? + else { + return Ok(None); + }; + if serde_json::from_slice::<ChunkSlotMapHead>(&value).is_ok() { + self.check_initialized(expected).await?; + return Ok(Some(Pending { + intent: None, + revision, + })); + } + let intent: Intent<O> = decode(key.as_bytes(), &value)?; + let mut expected_bindings = expected.bindings().to_vec(); + expected_bindings.sort_by_key(|binding| binding.owner.key()); + // The offset refers to this canonical order, independent of caller order. + if intent.initializing.head != *expected.head() + || intent.initializing.bindings != expected_bindings + || intent.next >= expected_bindings.len() + || intent.next % (MAX_BATCH_RECORDS - 1) != 0 + { + return Err(invalid(&key, "conflicting or corrupt slot map initialization")); + } + Ok(Some(Pending { + intent: Some(intent), + revision, + })) + } + + pub(super) async fn begin_initialization<O: MapOwner>(&self, map: &ChunkSlotMap<O>) -> Result<()> { + let key = O::head().to_path(); + let mut bindings = map.bindings().to_vec(); + bindings.sort_by_key(|binding| binding.owner.key()); + let intent = Intent { + initializing: Layout { + head: map.head().clone(), + bindings, + }, + next: 0, + }; + let operation = put(key.clone(), &intent)?; + let pending = match self + .kv + .batch_write_cas(0, 0, &[operation], key.as_bytes(), 0) + .await + { + Ok(written) => Pending { + intent: Some(intent), + revision: written.revision, + }, + Err(error @ (Error::OutcomeUnknown | Error::CasFailed { .. } | Error::CasBusy)) => { + self.initialization_state(map).await?.ok_or(error)? + } + Err(error) => return Err(error), + }; + self.finish_initialization(map, pending).await + } + + pub(super) async fn finish_initialization<O: MapOwner>( + &self, + map: &ChunkSlotMap<O>, + mut pending: Pending<O>, + ) -> Result<()> { + let key = O::head().to_path(); + while let Some(mut intent) = pending.intent { + let previous = intent.next; + let end = (previous + MAX_BATCH_RECORDS - 1).min(intent.initializing.bindings.len()); + let mut operations = Vec::with_capacity(end - previous + 1); + for binding in &intent.initializing.bindings[previous..end] { + operations.push(put(binding.owner.key(), binding)?); + } + intent.next = end; + let complete = end == intent.initializing.bindings.len(); + operations.push(if complete { + put(key.clone(), map.head())? + } else { + put(key.clone(), &intent)? + }); + match self + .kv + .batch_write_cas(0, 0, &operations, key.as_bytes(), pending.revision) + .await + { + Ok(written) => { + pending = Pending { + intent: (!complete).then_some(intent), + revision: written.revision, + }; + } + Err(error @ (Error::OutcomeUnknown | Error::CasFailed { .. } | Error::CasBusy)) => { + let Some(observed) = self.initialization_state(map).await? else { + return Err(error); + }; + // Resume only proven durable progress by an identical + // initializer. An unchanged/unknown outcome remains an error. + if observed + .intent + .as_ref() + .is_some_and(|state| state.next <= previous) + { + return Err(error); + } + pending = observed; + } + Err(error) => return Err(error), + } + } + Ok(()) + } +} diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots/publication.rs b/lib/crowdb-paxos-kv-client/src/binding/chunk_slots/publication.rs similarity index 100% rename from lib/crowdb-kv-client/src/binding/chunk_slots/publication.rs rename to lib/crowdb-paxos-kv-client/src/binding/chunk_slots/publication.rs diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs b/lib/crowdb-paxos-kv-client/src/binding/chunk_slots/publication/epochs.rs similarity index 100% rename from lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs rename to lib/crowdb-paxos-kv-client/src/binding/chunk_slots/publication/epochs.rs diff --git a/lib/crowdb-kv-client/src/binding/chunkdb_strategy.rs b/lib/crowdb-paxos-kv-client/src/binding/chunkdb_strategy.rs similarity index 98% rename from lib/crowdb-kv-client/src/binding/chunkdb_strategy.rs rename to lib/crowdb-paxos-kv-client/src/binding/chunkdb_strategy.rs index 5b1220391..7b6c4d53b 100644 --- a/lib/crowdb-kv-client/src/binding/chunkdb_strategy.rs +++ b/lib/crowdb-paxos-kv-client/src/binding/chunkdb_strategy.rs @@ -10,7 +10,7 @@ use crowdb_protocol::common::{ChunkdbRangeBindingValue, InstanceValue, RangeStat use crowdb_protocol::key::ChunkdbRangeBindingKey; use super::framework::BindingStrategy; -use crate::{CrowdbKvClient, Error, ReadMode, Result}; +use crate::{Error, PaxosKvClient, ReadMode, Result}; const G0_STORE: u64 = 0; const G0_GROUP: u64 = 0; @@ -57,7 +57,7 @@ impl BindingStrategy for ChunkdbRangeStrategy { fn write_bindings( &self, - _kv: &CrowdbKvClient, + _kv: &PaxosKvClient, _bindings: &[Self::Binding], ) -> impl std::future::Future<Output = Result<()>> { std::future::ready(Err(Error::SysdataDecode { @@ -66,7 +66,7 @@ impl BindingStrategy for ChunkdbRangeStrategy { })) } - async fn read_bindings(&self, kv: &CrowdbKvClient) -> Result<Vec<Self::Binding>> { + async fn read_bindings(&self, kv: &PaxosKvClient) -> Result<Vec<Self::Binding>> { let prefix = ChunkdbRangeBindingKey::text_prefix_all(); let mut bindings: Vec<ChunkdbRangeBindingValue> = Vec::new(); let mut start_after: Vec<u8> = Vec::new(); diff --git a/lib/crowdb-kv-client/src/binding/framework.rs b/lib/crowdb-paxos-kv-client/src/binding/framework.rs similarity index 97% rename from lib/crowdb-kv-client/src/binding/framework.rs rename to lib/crowdb-paxos-kv-client/src/binding/framework.rs index 1b2e8eefc..2305bc572 100644 --- a/lib/crowdb-kv-client/src/binding/framework.rs +++ b/lib/crowdb-paxos-kv-client/src/binding/framework.rs @@ -12,7 +12,7 @@ use crowdb_protocol::common::InstanceValue; -use crate::{CrowdbKvClient, Result}; +use crate::{PaxosKvClient, Result}; /// A binding strategy — maps a key to an owning instance. pub trait BindingStrategy: Send + Sync { @@ -24,14 +24,14 @@ pub trait BindingStrategy: Send + Sync { /// Write the assignment to group-0. fn write_bindings( &self, - kv: &CrowdbKvClient, + kv: &PaxosKvClient, bindings: &[Self::Binding], ) -> impl std::future::Future<Output = Result<()>> + Send; /// Read the current assignment from group-0. fn read_bindings( &self, - kv: &CrowdbKvClient, + kv: &PaxosKvClient, ) -> impl std::future::Future<Output = Result<Vec<Self::Binding>>> + Send; /// Compute an incremental assignment, preserving transition state @@ -58,7 +58,7 @@ pub trait BindingStrategy: Send + Sync { /// writes it to group-0. Only the leader should write; followers /// compute but skip the write phase. pub struct BindingMonitor<S: BindingStrategy> { - kv: std::sync::Arc<CrowdbKvClient>, + kv: std::sync::Arc<PaxosKvClient>, svc: crate::ServiceRegistryClient, strategy: S, interval: std::time::Duration, @@ -69,7 +69,7 @@ impl<S: BindingStrategy> BindingMonitor<S> { /// Create a new binding monitor. #[must_use] pub fn new( - kv: std::sync::Arc<CrowdbKvClient>, + kv: std::sync::Arc<PaxosKvClient>, svc: crate::ServiceRegistryClient, strategy: S, interval: std::time::Duration, diff --git a/lib/crowdb-kv-client/src/binding/range.rs b/lib/crowdb-paxos-kv-client/src/binding/range.rs similarity index 96% rename from lib/crowdb-kv-client/src/binding/range.rs rename to lib/crowdb-paxos-kv-client/src/binding/range.rs index 3c1c41aee..dcd3a1412 100644 --- a/lib/crowdb-kv-client/src/binding/range.rs +++ b/lib/crowdb-paxos-kv-client/src/binding/range.rs @@ -3,7 +3,7 @@ //! Client-to-service slot routing. Storage placement is never read here. -use crate::{ChunkSlotMapClient, CrowdbKvClient, Error, Result, ServiceRegistryClient}; +use crate::{ChunkSlotMapClient, Error, PaxosKvClient, Result, ServiceRegistryClient}; use arc_swap::ArcSwapOption; use crowdb_protocol::chunk_slot::{ChunkSlot, ChunkSlotBitmap, ChunkSlotMap}; use crowdb_protocol::common::ChunkId; @@ -35,13 +35,13 @@ struct ServiceSnapshot { } pub struct RangeBindingClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, snapshot: ArcSwapOption<ServiceSnapshot>, } impl RangeBindingClient { #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv, snapshot: ArcSwapOption::empty(), @@ -49,7 +49,7 @@ impl RangeBindingClient { } #[must_use] - pub fn kv(&self) -> &CrowdbKvClient { + pub fn kv(&self) -> &PaxosKvClient { &self.kv } diff --git a/lib/crowdb-kv-client/src/client.rs b/lib/crowdb-paxos-kv-client/src/client.rs similarity index 74% rename from lib/crowdb-kv-client/src/client.rs rename to lib/crowdb-paxos-kv-client/src/client.rs index 00680d65c..c4a6b90e2 100644 --- a/lib/crowdb-kv-client/src/client.rs +++ b/lib/crowdb-paxos-kv-client/src/client.rs @@ -9,6 +9,6 @@ pub mod retry; pub mod topology; pub use core::{ - new_client_id, BatchOp, CrowdbKvClient, GetOutcome, JournalOp, JournalScanOutcome, ScanDirection, + new_client_id, BatchOp, GetOutcome, JournalOp, JournalScanOutcome, PaxosKvClient, ScanDirection, ScanOutcome, WriteOutcome, }; diff --git a/lib/crowdb-kv-client/src/client/admin.rs b/lib/crowdb-paxos-kv-client/src/client/admin.rs similarity index 96% rename from lib/crowdb-kv-client/src/client/admin.rs rename to lib/crowdb-paxos-kv-client/src/client/admin.rs index e06b10692..665786114 100644 --- a/lib/crowdb-kv-client/src/client/admin.rs +++ b/lib/crowdb-paxos-kv-client/src/client/admin.rs @@ -1,17 +1,17 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Admin/management operations for [`CrowdbKvClient`]: snapshot lifecycle +//! Admin/management operations for [`PaxosKvClient`]: snapshot lifecycle //! (create, list, scan, release). -use crowdb_kv::rpc::{ +use crowdb_paxos_kv::rpc::{ CreateSnapshotResponse, ReadMode, ReleaseSnapshotResponse, SnapshotInfo, SnapshotScanResponse, }; -use crate::client::CrowdbKvClient; +use crate::client::PaxosKvClient; use crate::error::{Error, Result}; -impl CrowdbKvClient { +impl PaxosKvClient { /// Create a point-in-time-consistent snapshot. Flushes L0 → L1 /// and pins the durable view at `last_applied_slot`. Returns a /// snapshot handle for use with `snapshot_scan`/`release_snapshot`. diff --git a/lib/crowdb-kv-client/src/client/core.rs b/lib/crowdb-paxos-kv-client/src/client/core.rs similarity index 95% rename from lib/crowdb-kv-client/src/client/core.rs rename to lib/crowdb-paxos-kv-client/src/client/core.rs index 53a821425..0a228d90b 100644 --- a/lib/crowdb-kv-client/src/client/core.rs +++ b/lib/crowdb-paxos-kv-client/src/client/core.rs @@ -1,9 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! [`CrowdbKvClient`]: the C1-C3 client library (— +//! [`PaxosKvClient`]: the C1-C3 client library (— //! topology cache, retry/idempotency, and `ReadMode` routing on top of -//! `crowdb_kv`'s generated `KvService` client. +//! `crowdb_paxos_kv`'s generated `KvService` client. #![allow(clippy::cast_possible_truncation)] @@ -19,7 +19,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use bytes::Bytes; use crowdb_common::RequestIdGen; -use crowdb_kv::rpc::ReadMode; +use crowdb_paxos_kv::rpc::ReadMode; use super::topology::{EndpointStats, TopologyCache}; use crate::config::{ClientConfig, ReadEndpointPolicy, RetryConfig}; @@ -118,7 +118,7 @@ impl Drop for InFlightGuard { /// API, per-group leader cache, retry loop reusing `(client_id, seq)` across /// retries of one logical write, and `ReadMode` routing including /// `MinSlot` client-side slot tracking. -pub struct CrowdbKvClient { +pub struct PaxosKvClient { pub(crate) topology: TopologyCache, pub(crate) retry: RetryConfig, client_id: u64, @@ -135,10 +135,10 @@ pub struct CrowdbKvClient { /// `batch_write`/`scan`/`scan_count`/`journal_scan`) send via /// via crowdb-rpc. The retry/topology/`NotLeaderHint` /// logic is unchanged — only the wire send changes. - rpc_transport: Option<Arc<crate::KvRpcTransport>>, + rpc_transport: Option<Arc<crate::PaxosKvRpcTransport>>, } -impl CrowdbKvClient { +impl PaxosKvClient { #[must_use] pub fn new(config: ClientConfig) -> Self { Self::build(config, None, false) @@ -153,13 +153,13 @@ impl CrowdbKvClient { /// Create a client using an existing shared RPC transport. #[must_use] - pub fn new_with_rpc_transport(config: ClientConfig, transport: Arc<crate::KvRpcTransport>) -> Self { + pub fn new_with_rpc_transport(config: ClientConfig, transport: Arc<crate::PaxosKvRpcTransport>) -> Self { Self::build(config, Some(transport), false) } fn build( config: ClientConfig, - transport: Option<Arc<crate::KvRpcTransport>>, + transport: Option<Arc<crate::PaxosKvRpcTransport>>, isolated_point_reads: bool, ) -> Self { // A standalone client is normal at process startup. Keep the @@ -168,12 +168,12 @@ impl CrowdbKvClient { if transport.is_none() { tracing::debug!( seed_count = config.mgmt_seeds.len(), - "CrowdbKvClient: new standalone instance created" + "PaxosKvClient: new standalone instance created" ); } else { tracing::info!( seed_count = config.mgmt_seeds.len(), - "CrowdbKvClient: new shared instance created" + "PaxosKvClient: new shared instance created" ); } Self { @@ -185,7 +185,7 @@ impl CrowdbKvClient { metrics: Arc::new(ClientMetrics::default()), read_endpoint_policy: config.read_endpoint_policy, rpc_transport: Some(transport.unwrap_or_else(|| { - let transport = crate::KvRpcTransport::with_pool_size( + let transport = crate::PaxosKvRpcTransport::with_pool_size( config.pool_size_per_endpoint, config.enable_nagle, config.quickack, @@ -207,7 +207,7 @@ impl CrowdbKvClient { /// `scan_count`/`journal_scan` send via the transport instead of /// via crowdb-rpc. The retry/topology/`NotLeaderHint` logic is unchanged. #[must_use] - pub fn with_rpc_transport(mut self, transport: Arc<crate::KvRpcTransport>) -> Self { + pub fn with_rpc_transport(mut self, transport: Arc<crate::PaxosKvRpcTransport>) -> Self { self.rpc_transport = Some(transport); self } @@ -215,7 +215,7 @@ impl CrowdbKvClient { /// The crowdb-rpc transport, if set via `with_rpc_transport`. /// Used by `WatchNotifyClient` to select the crowdb-rpc push path. #[must_use] - pub(crate) fn rpc_transport(&self) -> Option<&Arc<crate::KvRpcTransport>> { + pub(crate) fn rpc_transport(&self) -> Option<&Arc<crate::PaxosKvRpcTransport>> { self.rpc_transport.as_ref() } @@ -299,7 +299,7 @@ impl CrowdbKvClient { /// group, bypassing `/topology` discovery entirely. For callers that /// already resolved an endpoint through some other discovery path /// (e.g. `crowdb-console`'s own management API) and just want - /// `CrowdbKvClient`'s retry/pool machinery on top of it. + /// `PaxosKvClient`'s retry/pool machinery on top of it. #[allow(clippy::needless_pass_by_value)] pub fn seed_leader(&self, store_id: u64, group_id: u64, endpoint: String) { self.topology.set_leader(store_id, group_id, &endpoint); @@ -316,7 +316,7 @@ impl CrowdbKvClient { /// 0, the fixed directory holding hardware/service-registry/ /// KV-cluster-topology records. Group-0 service classes /// (`HardwareClient`, `ServiceRegistryClient`, - /// `KVClusterMetaClient`) target this group; callers can use this + /// `PaxosKvClusterMetaClient`) target this group; callers can use this /// instead of hardcoding `(0, 0)`. #[must_use] pub fn system_group(&self) -> (u64, u64) { diff --git a/lib/crowdb-kv-client/src/client/core/conditional.rs b/lib/crowdb-paxos-kv-client/src/client/core/conditional.rs similarity index 96% rename from lib/crowdb-kv-client/src/client/core/conditional.rs rename to lib/crowdb-paxos-kv-client/src/client/core/conditional.rs index 536bd8726..408eeccd5 100644 --- a/lib/crowdb-kv-client/src/client/core/conditional.rs +++ b/lib/crowdb-paxos-kv-client/src/client/core/conditional.rs @@ -1,13 +1,13 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use super::{now_ms, BatchOp, CrowdbKvClient, WriteOutcome}; +use super::{now_ms, BatchOp, PaxosKvClient, WriteOutcome}; use crate::error::{Error, Result}; use bytes::Bytes; -use crowdb_kv::rpc::{KvBatchItem, KvErrorCode}; +use crowdb_paxos_kv::rpc::{KvBatchItem, KvErrorCode}; use std::sync::atomic::Ordering; -impl CrowdbKvClient { +impl PaxosKvClient { /// Conditionally put `key` when its current revision equals /// `expected_revision`. Revision zero means create-if-absent. /// @@ -165,7 +165,7 @@ impl CrowdbKvClient { } } -fn comparison_error(response: crowdb_kv::rpc::KvResponse, key: &[u8], expected_revision: u64) -> Error { +fn comparison_error(response: crowdb_paxos_kv::rpc::KvResponse, key: &[u8], expected_revision: u64) -> Error { if response.compare_observation.as_ref().is_some_and(|observation| { observation.key.as_ref() != key || observation.revision == expected_revision }) { diff --git a/lib/crowdb-kv-client/src/client/core/operations.rs b/lib/crowdb-paxos-kv-client/src/client/core/operations.rs similarity index 97% rename from lib/crowdb-kv-client/src/client/core/operations.rs rename to lib/crowdb-paxos-kv-client/src/client/core/operations.rs index 97481b626..1873d0551 100644 --- a/lib/crowdb-kv-client/src/client/core/operations.rs +++ b/lib/crowdb-paxos-kv-client/src/client/core/operations.rs @@ -1,14 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use super::{now_ms, BatchOp, CrowdbKvClient, GetOutcome, WriteOutcome}; +use super::{now_ms, BatchOp, GetOutcome, PaxosKvClient, WriteOutcome}; use crate::error::{Error, Result}; use bytes::Bytes; -use crowdb_kv::rpc::{KvBatchItem, ReadMode}; +use crowdb_paxos_kv::rpc::{KvBatchItem, ReadMode}; use std::sync::atomic::Ordering; use std::time::Instant; -impl CrowdbKvClient { +impl PaxosKvClient { /// `Put` a single key/value. /// /// `ids`: override `(client_id, seq)` for callers that manage their own @@ -44,7 +44,7 @@ impl CrowdbKvClient { let request_id = self.request_ids.next().as_u64(); let request_create_ms = now_ms(); let t0 = Instant::now(); - let send_result: std::result::Result<crowdb_kv::rpc::KvResponse, String> = t + let send_result: std::result::Result<crowdb_paxos_kv::rpc::KvResponse, String> = t .send_put( &endpoint, key, @@ -131,7 +131,7 @@ impl CrowdbKvClient { let request_create_ms = now_ms(); let t0 = Instant::now(); let _in_flight = self.incr_in_flight(store_id, group_id, &endpoint); - let send_result: std::result::Result<crowdb_kv::rpc::KvResponse, String> = t + let send_result: std::result::Result<crowdb_paxos_kv::rpc::KvResponse, String> = t .send_get( &endpoint, key, @@ -257,7 +257,7 @@ impl CrowdbKvClient { let request_id = self.request_ids.next().as_u64(); let request_create_ms = now_ms(); let t0 = Instant::now(); - let send_result: std::result::Result<crowdb_kv::rpc::KvResponse, String> = t + let send_result: std::result::Result<crowdb_paxos_kv::rpc::KvResponse, String> = t .send_delete( &endpoint, key, @@ -364,7 +364,7 @@ impl CrowdbKvClient { let request_id = self.request_ids.next().as_u64(); let request_create_ms = now_ms(); let t0 = Instant::now(); - let send_result: std::result::Result<crowdb_kv::rpc::KvResponse, String> = t + let send_result: std::result::Result<crowdb_paxos_kv::rpc::KvResponse, String> = t .send_batch_write( &endpoint, &items, diff --git a/lib/crowdb-kv-client/src/client/core/owned.rs b/lib/crowdb-paxos-kv-client/src/client/core/owned.rs similarity index 96% rename from lib/crowdb-kv-client/src/client/core/owned.rs rename to lib/crowdb-paxos-kv-client/src/client/core/owned.rs index a7db7423e..226302e6e 100644 --- a/lib/crowdb-kv-client/src/client/core/owned.rs +++ b/lib/crowdb-paxos-kv-client/src/client/core/owned.rs @@ -1,14 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use super::{now_ms, BatchOp, CrowdbKvClient, WriteOutcome}; +use super::{now_ms, BatchOp, PaxosKvClient, WriteOutcome}; use crate::error::{Error, Result}; use bytes::Bytes; -use crowdb_kv::rpc::{KvBatchItem, KvErrorCode}; +use crowdb_paxos_kv::rpc::{KvBatchItem, KvErrorCode}; use crowdb_protocol::owner_fence::{is_owner_fence_key, valid_owner_fence, valid_owner_fence_scope}; use std::sync::atomic::Ordering; -impl CrowdbKvClient { +impl PaxosKvClient { /// Write under a read-only owner identity; KV drains writes only at handover. /// /// # Errors diff --git a/lib/crowdb-kv-client/src/client/core/scans.rs b/lib/crowdb-paxos-kv-client/src/client/core/scans.rs similarity index 98% rename from lib/crowdb-kv-client/src/client/core/scans.rs rename to lib/crowdb-paxos-kv-client/src/client/core/scans.rs index a69eef351..42e13979b 100644 --- a/lib/crowdb-kv-client/src/client/core/scans.rs +++ b/lib/crowdb-paxos-kv-client/src/client/core/scans.rs @@ -1,13 +1,13 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use super::{now_ms, CrowdbKvClient, JournalOp, JournalScanOutcome, ScanDirection, ScanOutcome}; +use super::{now_ms, JournalOp, JournalScanOutcome, PaxosKvClient, ScanDirection, ScanOutcome}; use crate::error::{Error, Result}; use bytes::Bytes; -use crowdb_kv::rpc::ReadMode; +use crowdb_paxos_kv::rpc::ReadMode; use std::time::Instant; -impl CrowdbKvClient { +impl PaxosKvClient { /// Prefix-scan a group's key space. Uses S3-style pagination /// (`start_after` + `truncated`): the server applies a byte budget to /// each unary response so every page is provably bounded regardless of @@ -288,7 +288,7 @@ impl CrowdbKvClient { let request_create_ms = now_ms(); let t0 = Instant::now(); let _in_flight = self.incr_in_flight(store_id, group_id, &endpoint); - let send_result: std::result::Result<crowdb_kv::rpc::KvScanResponse, String> = t + let send_result: std::result::Result<crowdb_paxos_kv::rpc::KvScanResponse, String> = t .send_scan_directional( &endpoint, prefix, @@ -315,7 +315,7 @@ impl CrowdbKvClient { // Follow a `not_leader_hint` before honoring `ok`: a // linearizable scan that fails mid-forward to the leader // falls back to a stale local read + hint (see - // `patch_scan_not_leader_hint` in kv_rpc_service). That + // `patch_scan_not_leader_hint` in paxos_kv_rpc_service). That // stale read returns `ok=true` with empty items, so // checking `ok` first would surface empty data to the // caller. Mirrors the get/delete hint-first fix. @@ -501,7 +501,7 @@ impl CrowdbKvClient { let request_create_ms = now_ms(); let t0 = Instant::now(); let _in_flight = self.incr_in_flight(store_id, group_id, &endpoint); - let send_result: std::result::Result<crowdb_kv::rpc::KvScanResponse, String> = t + let send_result: std::result::Result<crowdb_paxos_kv::rpc::KvScanResponse, String> = t .send_scan( &endpoint, prefix, @@ -681,7 +681,7 @@ impl CrowdbKvClient { self.metrics.record_scan_error(); // GC gap is deterministic — do not retry; the caller // (diskdb recovery) falls back to a full-scan rebuild. - if resp.error_code == crowdb_kv::rpc::KvErrorCode::KvErrorJournalScanGcGap as i32 { + if resp.error_code == crowdb_paxos_kv::rpc::KvErrorCode::KvErrorJournalScanGcGap as i32 { return Err(Error::JournalScanGcGap); } if let Some(next) = self diff --git a/lib/crowdb-kv-client/src/client/retry.rs b/lib/crowdb-paxos-kv-client/src/client/retry.rs similarity index 97% rename from lib/crowdb-kv-client/src/client/retry.rs rename to lib/crowdb-paxos-kv-client/src/client/retry.rs index baa80abb0..6288cb05d 100644 --- a/lib/crowdb-kv-client/src/client/retry.rs +++ b/lib/crowdb-paxos-kv-client/src/client/retry.rs @@ -2,14 +2,14 @@ // Licensed under the Apache License, Version 2.0. //! Retry logic, topology cache refresh, and `NotLeaderHint` handling -//! for [`CrowdbKvClient`]. +//! for [`PaxosKvClient`]. use std::time::Duration; -use crowdb_kv::rpc::{KvErrorCode, KvResponse}; +use crowdb_paxos_kv::rpc::{KvErrorCode, KvResponse}; use tracing::warn; -use crate::client::CrowdbKvClient; +use crate::client::PaxosKvClient; use crate::error::{Error, Result}; #[derive(Default)] @@ -18,7 +18,7 @@ pub(crate) struct Redirects { repeats: u32, } -impl CrowdbKvClient { +impl PaxosKvClient { /// Follow new hints immediately; repeated or cyclic hints consume a /// bounded election-wait budget rather than spinning on stale authority. pub(crate) async fn follow_not_leader( diff --git a/lib/crowdb-kv-client/src/client/topology.rs b/lib/crowdb-paxos-kv-client/src/client/topology.rs similarity index 99% rename from lib/crowdb-kv-client/src/client/topology.rs rename to lib/crowdb-paxos-kv-client/src/client/topology.rs index fee39b06b..df31b52d8 100644 --- a/lib/crowdb-kv-client/src/client/topology.rs +++ b/lib/crowdb-paxos-kv-client/src/client/topology.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. //! Topology cache: `(store_id, group_id) -> leader_endpoint`, sourced from -//! `crowdb-kv-server`'s HTTP management API (`GET /topology`). There is no crowdb-rpc +//! `crowdb-paxos-kv-server`'s HTTP management API (`GET /topology`). There is no crowdb-rpc //! `DescribeCluster` RPC — this is the only discovery mechanism. use std::collections::{HashMap, HashSet}; diff --git a/lib/crowdb-kv-client/src/client/topology/address.rs b/lib/crowdb-paxos-kv-client/src/client/topology/address.rs similarity index 100% rename from lib/crowdb-kv-client/src/client/topology/address.rs rename to lib/crowdb-paxos-kv-client/src/client/topology/address.rs diff --git a/lib/crowdb-kv-client/src/client/topology/recovery.rs b/lib/crowdb-paxos-kv-client/src/client/topology/recovery.rs similarity index 100% rename from lib/crowdb-kv-client/src/client/topology/recovery.rs rename to lib/crowdb-paxos-kv-client/src/client/topology/recovery.rs diff --git a/lib/crowdb-kv-client/src/config.rs b/lib/crowdb-paxos-kv-client/src/config.rs similarity index 100% rename from lib/crowdb-kv-client/src/config.rs rename to lib/crowdb-paxos-kv-client/src/config.rs diff --git a/lib/crowdb-kv-client/src/error.rs b/lib/crowdb-paxos-kv-client/src/error.rs similarity index 95% rename from lib/crowdb-kv-client/src/error.rs rename to lib/crowdb-paxos-kv-client/src/error.rs index 49d7b6ca8..91226aef6 100644 --- a/lib/crowdb-kv-client/src/error.rs +++ b/lib/crowdb-paxos-kv-client/src/error.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Error type for [`crate::CrowdbKvClient`]. +//! Error type for [`crate::PaxosKvClient`]. /// Errors surfaced by the client library. Transport/server errors that the /// retry loop can recover from are handled internally and never reach the @@ -63,7 +63,7 @@ pub enum Error { /// No mgmt seed URLs are configured — the client cannot discover /// any leader. Returned immediately (no retry) so the caller gets /// a clear, fast error instead of a 5s timeout. If you see this, - /// the code that created the `CrowdbKvClient` forgot to call + /// the code that created the `PaxosKvClient` forgot to call /// `set_mgmt_seeds` with the cluster's server URLs. #[error("no mgmt seeds configured — cannot discover leader; call set_mgmt_seeds first")] NoSeeds, @@ -87,7 +87,7 @@ pub enum Error { NoLivingInstances { service: String }, /// The group-0 service registry is unreachable — the underlying - /// `CrowdbKvClient` exhausted its retry budget trying to read + /// `PaxosKvClient` exhausted its retry budget trying to read /// `/srv/<service>/`. The cache is not invalidated on this error /// (a stale cache is better than no cache if group-0 is transiently /// down). diff --git a/lib/crowdb-kv-client/src/ffi.rs b/lib/crowdb-paxos-kv-client/src/ffi.rs similarity index 97% rename from lib/crowdb-kv-client/src/ffi.rs rename to lib/crowdb-paxos-kv-client/src/ffi.rs index 2ae52de69..844581881 100644 --- a/lib/crowdb-kv-client/src/ffi.rs +++ b/lib/crowdb-paxos-kv-client/src/ffi.rs @@ -27,7 +27,7 @@ use std::os::raw::{c_char, c_int, c_void}; use std::ptr; use std::sync::{Arc, OnceLock}; -use crate::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegistryClient}; +use crate::{ClientConfig, HardwareClient, PaxosKvClient, ServiceRegistryClient}; use crowdb_protocol::common::DiskGroupUsageSummary; // ── Opaque handle types ─────────────────────────────────────────── @@ -85,7 +85,7 @@ where let rt = match runtime() { Ok(rt) => rt, Err(e) => { - eprintln!("crowdb-kv-client (ffi): runtime unavailable: {e}"); + eprintln!("crowdb-paxos-kv-client (ffi): runtime unavailable: {e}"); callback(-1, ptr::null(), user_data); return; } @@ -101,7 +101,7 @@ where } Err(err) => { cb_fn(-1, ptr::null(), ud_ptr); - eprintln!("crowdb-kv-client (ffi): op failed: {err}"); + eprintln!("crowdb-paxos-kv-client (ffi): op failed: {err}"); } } }); @@ -134,7 +134,7 @@ pub unsafe extern "C" fn crowdb_hw_client_create( if seed_vec.is_empty() { return ptr::null_mut(); } - let kv = CrowdbKvClient::new(ClientConfig::new(seed_vec)); + let kv = PaxosKvClient::new(ClientConfig::new(seed_vec)); let hw = HardwareClient::from_shared(Arc::new(kv)); Box::into_raw(Box::new(hw)) as crowdb_hw_client_t } @@ -259,7 +259,7 @@ pub unsafe extern "C" fn crowdb_svc_client_create( if seed_vec.is_empty() { return ptr::null_mut(); } - let kv = CrowdbKvClient::new(ClientConfig::new(seed_vec)); + let kv = PaxosKvClient::new(ClientConfig::new(seed_vec)); let svc = ServiceRegistryClient::from_shared(Arc::new(kv)); Box::into_raw(Box::new(svc)) as crowdb_svc_client_t } diff --git a/lib/crowdb-kv-client/src/ffi/disk_exclusion.rs b/lib/crowdb-paxos-kv-client/src/ffi/disk_exclusion.rs similarity index 100% rename from lib/crowdb-kv-client/src/ffi/disk_exclusion.rs rename to lib/crowdb-paxos-kv-client/src/ffi/disk_exclusion.rs diff --git a/lib/crowdb-kv-client/src/ffi/diskio_scope.rs b/lib/crowdb-paxos-kv-client/src/ffi/diskio_scope.rs similarity index 100% rename from lib/crowdb-kv-client/src/ffi/diskio_scope.rs rename to lib/crowdb-paxos-kv-client/src/ffi/diskio_scope.rs diff --git a/lib/crowdb-kv-client/src/hardware.rs b/lib/crowdb-paxos-kv-client/src/hardware.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware.rs rename to lib/crowdb-paxos-kv-client/src/hardware.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy.rs similarity index 96% rename from lib/crowdb-kv-client/src/hardware/hierarchy.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy.rs index 30da8378c..021b00b09 100644 --- a/lib/crowdb-kv-client/src/hardware/hierarchy.rs +++ b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy.rs @@ -5,14 +5,14 @@ //! [`HardwareClient`]: group-0 hardware hierarchy + per-disk-group maps. //! -//! Wraps a [`CrowdbKvClient`] pinned to group 0 (store 0, group 0). All +//! Wraps a [`PaxosKvClient`] pinned to group 0 (store 0, group 0). All //! keys are text-path keys (see `design-crowdb-protocol-key.md` §5.2); all values //! are JSON-encoded proto `*Value` types from `crowdb-protocol`. //! //! Most hierarchy writes are blind puts; values are small (< 1 KB). //! Disk status updates use revision CAS and never create missing disks. //! -//! See `doc/design/kv/design-crowdb-kv-group0.md` §2.1, §3. +//! See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §2.1, §3. mod disk_exclusion; mod disk_identity; @@ -47,7 +47,7 @@ use crowdb_protocol::key::{BindMapKey, DiskGroupKey, DiskGroupUsageKey, DiskKey, use crowdb_protocol::sysdata::{DiskGroupEntry, KVGroupBindEntry}; use crate::client::{GetOutcome, ScanOutcome}; -use crate::{CrowdbKvClient, Error, Result}; +use crate::{Error, PaxosKvClient, Result}; /// Group-0 store/group IDs (the system group). const G0_STORE: u64 = 0; @@ -132,12 +132,12 @@ pub struct HardwareCapacitySummary { /// Client for the hardware hierarchy and per-disk-group maps in /// group 0. /// -/// All methods target store 0, group 0. The wrapped `CrowdbKvClient` +/// All methods target store 0, group 0. The wrapped `PaxosKvClient` /// must have its topology seeded with a group-0 leader endpoint /// (via `seed_leader(0, 0, endpoint)` or `/topology` discovery). #[derive(Clone)] pub struct HardwareClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } fn now_ms() -> u64 { @@ -148,7 +148,7 @@ fn now_ms() -> u64 { // ── helpers ───────────────────────────────────────────────────── -async fn get_json<T: serde::de::DeserializeOwned>(kv: &CrowdbKvClient, key: &str) -> Result<Option<T>> { +async fn get_json<T: serde::de::DeserializeOwned>(kv: &PaxosKvClient, key: &str) -> Result<Option<T>> { match kv .get( G0_STORE, @@ -171,7 +171,7 @@ async fn get_json<T: serde::de::DeserializeOwned>(kv: &CrowdbKvClient, key: &str } pub(crate) async fn scan_prefix<T: serde::de::DeserializeOwned>( - kv: &CrowdbKvClient, + kv: &PaxosKvClient, prefix: &str, ) -> Result<Vec<(String, T)>> { let mut out: Vec<(String, T)> = Vec::new(); @@ -220,27 +220,27 @@ pub(crate) async fn scan_prefix<T: serde::de::DeserializeOwned>( // ── rack ──────────────────────────────────────────────────────── impl HardwareClient { - /// Wrap a `CrowdbKvClient` for group-0 hardware access. + /// Wrap a `PaxosKvClient` for group-0 hardware access. #[must_use] - pub fn new(kv: CrowdbKvClient) -> Self { + pub fn new(kv: PaxosKvClient) -> Self { Self { kv: Arc::new(kv) } } - /// Wrap an already-shared `CrowdbKvClient` for group-0 hardware access. + /// Wrap an already-shared `PaxosKvClient` for group-0 hardware access. #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } - /// Access the underlying `CrowdbKvClient`. + /// Access the underlying `PaxosKvClient`. #[must_use] - pub fn kv(&self) -> &CrowdbKvClient { + pub fn kv(&self) -> &PaxosKvClient { &self.kv } - /// Access the underlying `CrowdbKvClient` as a shared `Arc`. + /// Access the underlying `PaxosKvClient` as a shared `Arc`. #[must_use] - pub fn shared_kv(&self) -> Arc<CrowdbKvClient> { + pub fn shared_kv(&self) -> Arc<PaxosKvClient> { Arc::clone(&self.kv) } diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/disk_exclusion.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/disk_exclusion.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/disk_exclusion.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/disk_exclusion.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/disk_identity.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/disk_identity.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/disk_identity.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/disk_identity.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/disk_status.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/disk_status.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/disk_status.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/disk_status.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/group_identity.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/group_identity.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/group_identity.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/group_identity.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/legacy_cascade.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/legacy_cascade.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/legacy_cascade.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/legacy_cascade.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/legacy_media.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/legacy_media.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/legacy_media.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/legacy_media.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/mutation.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/mutation.rs similarity index 92% rename from lib/crowdb-kv-client/src/hardware/hierarchy/mutation.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/mutation.rs index a644321cc..f7d34e1ee 100644 --- a/lib/crowdb-kv-client/src/hardware/hierarchy/mutation.rs +++ b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/mutation.rs @@ -6,9 +6,9 @@ use crowdb_protocol::key::{RackKey, TextKey}; use std::hash::{DefaultHasher, Hash, Hasher}; -use crate::{BatchOp, CrowdbKvClient, Error, GetOutcome, ReadMode, Result}; +use crate::{BatchOp, Error, GetOutcome, PaxosKvClient, ReadMode, Result}; -pub(super) async fn read(kv: &CrowdbKvClient, path: &str) -> Result<Option<(bytes::Bytes, u64)>> { +pub(super) async fn read(kv: &PaxosKvClient, path: &str) -> Result<Option<(bytes::Bytes, u64)>> { match kv .get(0, 0, path.as_bytes(), ReadMode::Linearizable, None) .await? @@ -36,7 +36,7 @@ pub(super) fn put<T: serde::Serialize>(path: &str, value: &T) -> Result<BatchOp> }) } -pub(super) async fn update<T, F>(kv: &CrowdbKvClient, rack_id: u64, path: &str, change: F) -> Result<()> +pub(super) async fn update<T, F>(kv: &PaxosKvClient, rack_id: u64, path: &str, change: F) -> Result<()> where T: serde::de::DeserializeOwned + serde::Serialize + PartialEq, F: Fn(&mut T) -> Result<()>, diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/owner_session.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/owner_session.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/owner_session.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/owner_session.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/ownership.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/ownership.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/ownership.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/ownership.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_activation.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_activation.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_activation.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_activation.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_admission.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_admission.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_admission.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_admission.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_compatibility.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_compatibility.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_compatibility.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_compatibility.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_guard.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_guard.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_guard.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_guard.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_health.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_health.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_health.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_health.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_observation.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_observation.rs similarity index 99% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_observation.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_observation.rs index b2b38421b..1daf07b34 100644 --- a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_observation.rs +++ b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_observation.rs @@ -15,7 +15,7 @@ use super::{mutation, HardwareClient}; use crate::{Error, Result}; pub(super) async fn bounded_inventory( - kv: &crate::CrowdbKvClient, + kv: &crate::PaxosKvClient, prefix: &str, ) -> Result<Vec<(bytes::Bytes, bytes::Bytes)>> { let mut rows = Vec::new(); diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_online.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_online.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_online.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_online.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_online/completion.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_online/completion.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_online/completion.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_online/completion.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_page.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_page.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_page.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_page.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_rejoin.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_rejoin.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_rejoin.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_rejoin.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_removal.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_removal.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_removal.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_removal.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_removal/conditions.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_removal/conditions.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_removal/conditions.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_removal/conditions.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_resource.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_resource.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_resource.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_resource.rs diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_resource/authority.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_resource/authority.rs similarity index 93% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_resource/authority.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_resource/authority.rs index 4de704db8..2e23a4fde 100644 --- a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_resource/authority.rs +++ b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_resource/authority.rs @@ -3,7 +3,7 @@ //! Project only the immutable physical-host fields from authoritative node admission. -use crate::{CrowdbKvClient, Error, GetOutcome, ReadMode, Result}; +use crate::{Error, GetOutcome, PaxosKvClient, ReadMode, Result}; use crowdb_protocol::disk_storage::DiskStorageRecord; use serde::Deserialize; use std::collections::BTreeMap; @@ -24,7 +24,7 @@ struct AdmissionNode { cancelled: bool, } -pub(super) async fn hosts(kv: &CrowdbKvClient, record: &DiskStorageRecord) -> Result<Hosts> { +pub(super) async fn hosts(kv: &PaxosKvClient, record: &DiskStorageRecord) -> Result<Hosts> { let GetOutcome::Found { value, .. } = kv .get(0, 0, b"/deployment/nodes", ReadMode::Linearizable, None) .await? diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/storage_resource/legacy.rs b/lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_resource/legacy.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/hierarchy/storage_resource/legacy.rs rename to lib/crowdb-paxos-kv-client/src/hardware/hierarchy/storage_resource/legacy.rs diff --git a/lib/crowdb-kv-client/src/hardware/space_usage.rs b/lib/crowdb-paxos-kv-client/src/hardware/space_usage.rs similarity index 100% rename from lib/crowdb-kv-client/src/hardware/space_usage.rs rename to lib/crowdb-paxos-kv-client/src/hardware/space_usage.rs diff --git a/lib/crowdb-kv-client/src/hardware/sysmd.rs b/lib/crowdb-paxos-kv-client/src/hardware/sysmd.rs similarity index 94% rename from lib/crowdb-kv-client/src/hardware/sysmd.rs rename to lib/crowdb-paxos-kv-client/src/hardware/sysmd.rs index b59c3a97d..187f75204 100644 --- a/lib/crowdb-kv-client/src/hardware/sysmd.rs +++ b/lib/crowdb-paxos-kv-client/src/hardware/sysmd.rs @@ -11,11 +11,11 @@ //! (diskdb/kv-server/diskio/chunkdb registrations). //! //! Historically these were three separate client types -//! ([`HardwareClient`], [`KVClusterMetaClient`], +//! ([`HardwareClient`], [`PaxosKvClusterMetaClient`], //! [`ServiceRegistryClient`]), each wrapping the same -//! [`CrowdbKvClient`] pinned to group 0. `CrowdbSysmdClient` brings +//! [`PaxosKvClient`] pinned to group 0. `CrowdbSysmdClient` brings //! the `sysmd` concept — defined in -//! `doc/design/kv/design-crowdb-kv-group0.md` — into the client layer +//! `doc/design/kv/design-crowdb-paxos-kv-group0.md` — into the client layer //! as a single facade with domain methods (`add_rack`, `list_nodes`, //! `add_store`, …). New code should use this type; the three //! sub-clients remain public for existing callers until they migrate. @@ -31,20 +31,20 @@ use crowdb_protocol::diskdb::rpc::{DiskGroupValue, DiskValue}; use crowdb_protocol::sysdata::{DiskGroupEntry, DiskdbOwnerEntry, KVGroupBindEntry}; use super::hierarchy::{DiskRecord, HardwareCapacitySummary, HardwareClient}; -use crate::KVClusterMetaClient; +use crate::PaxosKvClusterMetaClient; use crate::ServiceRegistryClient; -use crate::{CrowdbKvClient, Result}; +use crate::{PaxosKvClient, Result}; /// Unified client for group-0 system metadata. /// -/// Wraps a single `Arc<CrowdbKvClient>` (seeded with a group-0 leader +/// Wraps a single `Arc<PaxosKvClient>` (seeded with a group-0 leader /// endpoint) and exposes domain methods across the three sysmd /// sub-areas: hardware hierarchy, KV-cluster topology, and service /// registry. Each method delegates to the corresponding sub-client. #[derive(Clone)] pub struct CrowdbSysmdClient { hw: HardwareClient, - meta: KVClusterMetaClient, + meta: PaxosKvClusterMetaClient, svc: ServiceRegistryClient, } @@ -74,38 +74,38 @@ impl CrowdbSysmdClient { .await } - /// Build a `CrowdbSysmdClient` from an owned `CrowdbKvClient`. + /// Build a `CrowdbSysmdClient` from an owned `PaxosKvClient`. /// The client must have its topology seeded with a group-0 leader /// endpoint (via `seed_leader(0, 0, endpoint)` or `/topology` /// discovery). #[must_use] - pub fn new(kv: CrowdbKvClient) -> Self { + pub fn new(kv: PaxosKvClient) -> Self { let shared = Arc::new(kv); Self::from_shared(shared) } /// Build a `CrowdbSysmdClient` from an already-shared - /// `CrowdbKvClient`. All three sub-clients share the same `Arc`, + /// `PaxosKvClient`. All three sub-clients share the same `Arc`, /// so topology cache and connection pool are reused. #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { hw: HardwareClient::from_shared(Arc::clone(&kv)), - meta: KVClusterMetaClient::from_shared(Arc::clone(&kv)), + meta: PaxosKvClusterMetaClient::from_shared(Arc::clone(&kv)), svc: ServiceRegistryClient::from_shared(kv), } } - /// Access the underlying `CrowdbKvClient` (data-plane + topology + /// Access the underlying `PaxosKvClient` (data-plane + topology /// cache shared by all sub-clients). #[must_use] - pub fn kv(&self) -> &CrowdbKvClient { + pub fn kv(&self) -> &PaxosKvClient { self.hw.kv() } - /// Access the shared `Arc<CrowdbKvClient>`. + /// Access the shared `Arc<PaxosKvClient>`. #[must_use] - pub fn shared_kv(&self) -> Arc<CrowdbKvClient> { + pub fn shared_kv(&self) -> Arc<PaxosKvClient> { self.hw.shared_kv() } diff --git a/lib/crowdb-kv-client/src/lib.rs b/lib/crowdb-paxos-kv-client/src/lib.rs similarity index 81% rename from lib/crowdb-kv-client/src/lib.rs rename to lib/crowdb-paxos-kv-client/src/lib.rs index f0aa40e1c..b6f84ca8d 100644 --- a/lib/crowdb-kv-client/src/lib.rs +++ b/lib/crowdb-paxos-kv-client/src/lib.rs @@ -3,9 +3,9 @@ //! Standalone client library for `CrowDB`. //! -//! Wraps `crowdb_kv`'s generated `KvService` crowdb-rpc client with: +//! Wraps `crowdb_paxos_kv`'s generated `KvService` crowdb-rpc client with: //! - **Topology cache** (`(store_id, group_id) -> leader_endpoint`) sourced -//! from `crowdb-kv-server`'s HTTP management API `/topology` (no crowdb-rpc +//! from `crowdb-paxos-kv-server`'s HTTP management API `/topology` (no crowdb-rpc //! `DescribeCluster`). //! - **Retry policy** on `NotLeaderHint` / timeout / other errors, reusing //! the same `(client_id, seq)` across retries of one logical write so the @@ -38,7 +38,7 @@ pub use binding::{ RangeBindingClient, RangeRouteError, DEFAULT_SUB_RANGE_COUNT, }; pub use client::{ - new_client_id, BatchOp, CrowdbKvClient, GetOutcome, JournalOp, JournalScanOutcome, ScanDirection, + new_client_id, BatchOp, GetOutcome, JournalOp, JournalScanOutcome, PaxosKvClient, ScanDirection, ScanOutcome, WriteOutcome, }; pub use config::{ClientConfig, ReadEndpointPolicy, RetryConfig}; @@ -56,12 +56,12 @@ pub use service::{ StreamProcessContext, StreamProcessLeaseClient, StreamProcessLeaseError, StreamProcessLeaseStatus, StreamProcessLeaseTiming, WatchNotify, WatchNotifyClient, WatchSubscription, }; -pub use transport::{KVClusterAdmin, KVClusterMetaClient, KvRpcTransport}; +pub use transport::{PaxosKvClusterAdmin, PaxosKvClusterMetaClient, PaxosKvRpcTransport}; -pub use crowdb_kv::rpc::KvCompareObservation; -/// Re-exported so callers don't need a direct `crowdb_kv` dependency just to +pub use crowdb_paxos_kv::rpc::KvCompareObservation; +/// Re-exported so callers don't need a direct `crowdb_paxos_kv` dependency just to /// pick a read mode or use snapshot DTOs. -pub use crowdb_kv::rpc::ReadMode; -pub use crowdb_kv::rpc::{ +pub use crowdb_paxos_kv::rpc::ReadMode; +pub use crowdb_paxos_kv::rpc::{ CreateSnapshotResponse, ReleaseSnapshotResponse, SnapshotInfo, SnapshotScanResponse, }; diff --git a/lib/crowdb-kv-client/src/membership.rs b/lib/crowdb-paxos-kv-client/src/membership.rs similarity index 98% rename from lib/crowdb-kv-client/src/membership.rs rename to lib/crowdb-paxos-kv-client/src/membership.rs index 68e2c4019..66084ff64 100644 --- a/lib/crowdb-kv-client/src/membership.rs +++ b/lib/crowdb-paxos-kv-client/src/membership.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use crowdb_protocol::key::{KvGroupMembersKey, TextKey}; use crowdb_protocol::kv_membership::{canonical_members, GroupMember, GroupMembership, GroupMembershipState}; -use crate::{CrowdbKvClient, Error, GetOutcome, ReadMode, Result}; +use crate::{Error, GetOutcome, PaxosKvClient, ReadMode, Result}; /// An exact authoritative configuration and its Group-0 CAS revision. #[derive(Clone, Debug)] @@ -27,12 +27,12 @@ impl GroupMembershipSnapshot { /// Shared membership authority; independent UI servers use the same CAS key. #[derive(Clone)] pub struct GroupMembershipClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } impl GroupMembershipClient { #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } diff --git a/lib/crowdb-kv-client/src/metrics.rs b/lib/crowdb-paxos-kv-client/src/metrics.rs similarity index 99% rename from lib/crowdb-kv-client/src/metrics.rs rename to lib/crowdb-paxos-kv-client/src/metrics.rs index a5c118c86..7a90704f3 100644 --- a/lib/crowdb-kv-client/src/metrics.rs +++ b/lib/crowdb-paxos-kv-client/src/metrics.rs @@ -145,7 +145,7 @@ impl Default for WindowLatency { } } -/// Metrics embedded in [`crate::CrowdbKvClient`]. Hot-path error counters +/// Metrics embedded in [`crate::PaxosKvClient`]. Hot-path error counters /// are lock-free atomics; per-op latency histograms are sharded by thread; /// leader-change tracking uses a `Mutex` (rare event, not hot path). #[derive(Debug)] diff --git a/lib/crowdb-kv-client/src/service.rs b/lib/crowdb-paxos-kv-client/src/service.rs similarity index 100% rename from lib/crowdb-kv-client/src/service.rs rename to lib/crowdb-paxos-kv-client/src/service.rs diff --git a/lib/crowdb-kv-client/src/service/discovery.rs b/lib/crowdb-paxos-kv-client/src/service/discovery.rs similarity index 95% rename from lib/crowdb-kv-client/src/service/discovery.rs rename to lib/crowdb-paxos-kv-client/src/service/discovery.rs index 768a49db5..063743a05 100644 --- a/lib/crowdb-kv-client/src/service/discovery.rs +++ b/lib/crowdb-paxos-kv-client/src/service/discovery.rs @@ -11,7 +11,7 @@ //! addresses. The cache is poll-on-demand: the first call queries //! group-0, subsequent calls within the TTL return the cached result. //! -//! See `doc/design/kv/design-crowdb-kv-group0.md` §4.4. +//! See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §4.4. use std::future::Future; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; @@ -25,7 +25,7 @@ use crowdb_protocol::common::InstanceValue; use crowdb_protocol::common_type::InstanceId; use crate::error::{Error, Result}; -use crate::{CrowdbKvClient, ServiceRegistryClient}; +use crate::{PaxosKvClient, ServiceRegistryClient}; /// Default cache TTL: 5 seconds. Matches the service heartbeat /// interval — the discovery client sees new registrations within one @@ -62,7 +62,7 @@ struct ServiceState { /// TTL. Round-robin selection for `discover_one`. /// /// All reads target store 0, group 0 (group-0 sysdata). The wrapped -/// `ServiceRegistryClient` shares the same `Arc<CrowdbKvClient>` as +/// `ServiceRegistryClient` shares the same `Arc<PaxosKvClient>` as /// the rest of the application, so topology re-seeding and connection /// pooling are shared. #[derive(Clone)] @@ -85,11 +85,11 @@ impl ServiceDiscoveryClient { } } - /// Construct from a shared `CrowdbKvClient`. The discovery client + /// Construct from a shared `PaxosKvClient`. The discovery client /// wraps a new `ServiceRegistryClient` that shares the same /// underlying KV client + connection pool. #[must_use] - pub fn from_shared_kv(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared_kv(kv: Arc<PaxosKvClient>) -> Self { Self::new(ServiceRegistryClient::from_shared(kv)) } @@ -239,7 +239,7 @@ mod tests { #[test] fn default_cache_ttl_is_5s() { - let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(CrowdbKvClient::new( + let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(PaxosKvClient::new( ClientConfig::new(vec!["http://127.0.0.1:10000".into()]), ))); assert_eq!(client.cache_ttl_ms.load(Ordering::Relaxed), 5_000); @@ -247,7 +247,7 @@ mod tests { #[test] fn with_cache_ttl_sets_ttl() { - let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(CrowdbKvClient::new( + let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(PaxosKvClient::new( ClientConfig::new(vec!["http://127.0.0.1:10000".into()]), ))) .with_cache_ttl(Duration::from_millis(500)); @@ -256,7 +256,7 @@ mod tests { #[test] fn invalidate_removes_single_service() { - let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(CrowdbKvClient::new( + let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(PaxosKvClient::new( ClientConfig::new(vec!["http://127.0.0.1:10000".into()]), ))); let diskdb = client.service_state("diskdb"); @@ -276,7 +276,7 @@ mod tests { #[test] fn invalidate_all_clears_cache() { - let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(CrowdbKvClient::new( + let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(PaxosKvClient::new( ClientConfig::new(vec!["http://127.0.0.1:10000".into()]), ))); let diskdb = client.service_state("diskdb"); @@ -290,7 +290,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn simultaneous_expiry_uses_one_refresh() { - let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(CrowdbKvClient::new( + let client = ServiceDiscoveryClient::new(ServiceRegistryClient::new(PaxosKvClient::new( ClientConfig::new(vec!["http://127.0.0.1:10000".into()]), ))) .with_cache_ttl(Duration::from_secs(1)); diff --git a/lib/crowdb-kv-client/src/service/domain_monitor.rs b/lib/crowdb-paxos-kv-client/src/service/domain_monitor.rs similarity index 95% rename from lib/crowdb-kv-client/src/service/domain_monitor.rs rename to lib/crowdb-paxos-kv-client/src/service/domain_monitor.rs index bd8e7d806..3cbe86318 100644 --- a/lib/crowdb-kv-client/src/service/domain_monitor.rs +++ b/lib/crowdb-paxos-kv-client/src/service/domain_monitor.rs @@ -10,18 +10,18 @@ use crowdb_protocol::chunk_kv::{ }; use crowdb_protocol::key::{DomainMonitorKey, TextKey}; -use crate::{CrowdbKvClient, Error, GetOutcome, ReadMode, Result}; +use crate::{Error, GetOutcome, PaxosKvClient, ReadMode, Result}; const GROUP_ZERO: u64 = 0; #[derive(Clone)] pub struct DomainMonitorClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } impl DomainMonitorClient { #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } diff --git a/lib/crowdb-kv-client/src/service/registry.rs b/lib/crowdb-paxos-kv-client/src/service/registry.rs similarity index 96% rename from lib/crowdb-kv-client/src/service/registry.rs rename to lib/crowdb-paxos-kv-client/src/service/registry.rs index 8d0fff807..84701e22b 100644 --- a/lib/crowdb-kv-client/src/service/registry.rs +++ b/lib/crowdb-paxos-kv-client/src/service/registry.rs @@ -10,7 +10,7 @@ //! heartbeats periodically. Readers filter expired entries by //! `last_heartbeat_ms` against a configurable TTL. //! -//! See `doc/design/kv/design-crowdb-kv-group0.md` §4. +//! See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §4. mod diskio; mod session; @@ -23,7 +23,7 @@ use crowdb_protocol::common_type::InstanceId; use crowdb_protocol::key::InstanceKey; use crate::client::GetOutcome; -use crate::{CrowdbKvClient, Error, Result}; +use crate::{Error, PaxosKvClient, Result}; const G0_STORE: u64 = 0; const G0_GROUP: u64 = 0; @@ -39,35 +39,35 @@ fn now_ms() -> u64 { /// Client for the service instance registry in group 0. /// -/// All methods target store 0, group 0. The wrapped `CrowdbKvClient` +/// All methods target store 0, group 0. The wrapped `PaxosKvClient` /// must have its topology seeded with a group-0 leader endpoint. #[derive(Clone)] pub struct ServiceRegistryClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } impl ServiceRegistryClient { - /// Wrap a `CrowdbKvClient` for group-0 service registry access. + /// Wrap a `PaxosKvClient` for group-0 service registry access. #[must_use] - pub fn new(kv: CrowdbKvClient) -> Self { + pub fn new(kv: PaxosKvClient) -> Self { Self { kv: Arc::new(kv) } } - /// Wrap an already-shared `CrowdbKvClient` for group-0 service registry access. + /// Wrap an already-shared `PaxosKvClient` for group-0 service registry access. #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } - /// Access the underlying `CrowdbKvClient`. + /// Access the underlying `PaxosKvClient`. #[must_use] - pub fn kv(&self) -> &CrowdbKvClient { + pub fn kv(&self) -> &PaxosKvClient { &self.kv } /// Share the group-0 transport with typed service routing clients. #[must_use] - pub fn shared_kv(&self) -> Arc<CrowdbKvClient> { + pub fn shared_kv(&self) -> Arc<PaxosKvClient> { Arc::clone(&self.kv) } diff --git a/lib/crowdb-kv-client/src/service/registry/diskio.rs b/lib/crowdb-paxos-kv-client/src/service/registry/diskio.rs similarity index 100% rename from lib/crowdb-kv-client/src/service/registry/diskio.rs rename to lib/crowdb-paxos-kv-client/src/service/registry/diskio.rs diff --git a/lib/crowdb-kv-client/src/service/registry/session.rs b/lib/crowdb-paxos-kv-client/src/service/registry/session.rs similarity index 100% rename from lib/crowdb-kv-client/src/service/registry/session.rs rename to lib/crowdb-paxos-kv-client/src/service/registry/session.rs diff --git a/lib/crowdb-kv-client/src/service/stream_lease.rs b/lib/crowdb-paxos-kv-client/src/service/stream_lease.rs similarity index 96% rename from lib/crowdb-kv-client/src/service/stream_lease.rs rename to lib/crowdb-paxos-kv-client/src/service/stream_lease.rs index 9e2debea5..7a7a85582 100644 --- a/lib/crowdb-kv-client/src/service/stream_lease.rs +++ b/lib/crowdb-paxos-kv-client/src/service/stream_lease.rs @@ -8,9 +8,10 @@ use std::sync::Arc; use crowdb_protocol::chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}; use crowdb_protocol::key::{StreamProcessLeaseKey, TextKey}; -use crate::{CrowdbKvClient, Error, GetOutcome, ReadMode}; +use crate::{Error, GetOutcome, PaxosKvClient, ReadMode}; mod authority; +mod exclusion; mod observation; pub use authority::{ StreamProcessAuthority, StreamProcessContext, StreamProcessLeaseStatus, StreamProcessLeaseTiming, @@ -33,12 +34,12 @@ pub enum StreamProcessLeaseError { type Result<T> = std::result::Result<T, StreamProcessLeaseError>; pub struct StreamProcessLeaseClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } impl StreamProcessLeaseClient { #[must_use] - pub fn new(kv: Arc<CrowdbKvClient>) -> Self { + pub fn new(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } @@ -72,7 +73,7 @@ impl StreamProcessLeaseClient { let current = self.read(purpose, instance_id).await?; if let Some((record, _)) = ¤t { if record.identity.boot_nonce == boot_nonce && record.identity.process_epoch == epoch { - if now_ms >= record.expires_at_ms { + if record.revoked || now_ms >= record.expires_at_ms { return Err(StreamProcessLeaseError::Expired); } return Ok(record.clone()); @@ -92,6 +93,7 @@ impl StreamProcessLeaseClient { prior_exclusion_deadline_ms: 0, grant_sequence: 0, grant_expires_at_ms: 0, + revoked: false, }; if let Some((previous, _)) = ¤t { record.prior_exclusion_deadline_ms = previous @@ -127,7 +129,7 @@ impl StreamProcessLeaseClient { if record.identity != identity { return Err(StreamProcessLeaseError::Replaced); } - if now_ms >= record.expires_at_ms { + if record.revoked || now_ms >= record.expires_at_ms { return Err(StreamProcessLeaseError::Expired); } record.expires_at_ms = record.expires_at_ms.max(expires_at_ms); diff --git a/lib/crowdb-kv-client/src/service/stream_lease/authority.rs b/lib/crowdb-paxos-kv-client/src/service/stream_lease/authority.rs similarity index 97% rename from lib/crowdb-kv-client/src/service/stream_lease/authority.rs rename to lib/crowdb-paxos-kv-client/src/service/stream_lease/authority.rs index 876e6e31b..ccbeab3a4 100644 --- a/lib/crowdb-kv-client/src/service/stream_lease/authority.rs +++ b/lib/crowdb-paxos-kv-client/src/service/stream_lease/authority.rs @@ -102,6 +102,10 @@ impl StreamProcessAuthority { self.fence(); return Err(StreamProcessLeaseError::Replaced); } + if lease.revoked { + self.fence(); + return Err(StreamProcessLeaseError::Expired); + } let deadline = match project(&lease, self.timing, wall_ms, received) { Ok(deadline) => deadline, Err(error) => { @@ -213,6 +217,9 @@ fn project( received: Instant, ) -> Result<Instant> { lease.validate().map_err(|error| invalid(&error.to_string()))?; + if lease.revoked { + return Err(StreamProcessLeaseError::Expired); + } if timing.max_clock_skew_ms > MAX_WRITE_CLOCK_SKEW_MS { return Err(invalid("owner clock skew exceeds shared recovery bound")); } diff --git a/lib/crowdb-paxos-kv-client/src/service/stream_lease/exclusion.rs b/lib/crowdb-paxos-kv-client/src/service/stream_lease/exclusion.rs new file mode 100644 index 000000000..0517606af --- /dev/null +++ b/lib/crowdb-paxos-kv-client/src/service/stream_lease/exclusion.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Irreversible process exclusion competes with renewal on the lease revision. + +use super::{invalid, Result, StreamProcessLeaseClient, StreamProcessLeaseError}; +use crowdb_protocol::{ + chunk_stream::StreamProcessIdentity, + timing::{MAX_WRITE_CLOCK_SKEW_MS, MAX_WRITE_REQUEST_AGE_MS}, +}; + +impl StreamProcessLeaseClient { + /// Confirm that this fixed process can no longer renew or submit a valid + /// late request. The returned boundary still requires physical IO drain. + /// An active or not-yet-excluded identity returns None without mutation. + /// # Errors + /// Rejects missing/malformed identities and unresolved authority updates. + pub async fn confirm_exclusion( + &self, + identity: StreamProcessIdentity, + now_ms: u64, + ) -> Result<Option<u64>> { + for attempt in 0..8 { + let Some((mut record, revision)) = self.read(identity.purpose, identity.instance_id).await? + else { + return Err(StreamProcessLeaseError::Replaced); + }; + let expires = if record.identity == identity { + record.expires_at_ms.max(record.grant_expires_at_ms) + } else if record.identity.process_epoch > identity.process_epoch { + record.prior_exclusion_deadline_ms + } else { + return Err(StreamProcessLeaseError::Replaced); + }; + let safe = expires + .checked_add(MAX_WRITE_CLOCK_SKEW_MS) + .and_then(|value| value.checked_add(MAX_WRITE_REQUEST_AGE_MS)) + .ok_or_else(|| invalid("process exclusion deadline overflow"))?; + if now_ms < safe { + return Ok(None); + } + if record.identity != identity || record.revoked { + return Ok(Some(safe)); + } + record.revoked = true; + if self.write(&record, revision, attempt).await? { + return Ok(Some(safe)); + } + } + Err(StreamProcessLeaseError::OutcomeUnknown) + } +} diff --git a/lib/crowdb-kv-client/src/service/stream_lease/observation.rs b/lib/crowdb-paxos-kv-client/src/service/stream_lease/observation.rs similarity index 98% rename from lib/crowdb-kv-client/src/service/stream_lease/observation.rs rename to lib/crowdb-paxos-kv-client/src/service/stream_lease/observation.rs index 1e7e5cfed..2cf384d5b 100644 --- a/lib/crowdb-kv-client/src/service/stream_lease/observation.rs +++ b/lib/crowdb-paxos-kv-client/src/service/stream_lease/observation.rs @@ -91,7 +91,7 @@ impl StreamProcessLeaseClient { if lease.identity != identity { return Err(StreamProcessLeaseError::Replaced); } - if now_ms >= lease.expires_at_ms { + if lease.revoked || now_ms >= lease.expires_at_ms { return Err(StreamProcessLeaseError::Expired); } let ops = [ diff --git a/lib/crowdb-kv-client/src/service/watch_notify.rs b/lib/crowdb-paxos-kv-client/src/service/watch_notify.rs similarity index 95% rename from lib/crowdb-kv-client/src/service/watch_notify.rs rename to lib/crowdb-paxos-kv-client/src/service/watch_notify.rs index 39616efd8..5a0ec781d 100644 --- a/lib/crowdb-kv-client/src/service/watch_notify.rs +++ b/lib/crowdb-paxos-kv-client/src/service/watch_notify.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Client for the crowdb-kv `WatchNotify` push. Opens a connection to +//! Client for the crowdb-paxos-kv `WatchNotify` push. Opens a connection to //! the group leader, subscribes to prefixes, and delivers notify //! frames via a channel. Automatically reconnects on leader change. //! @@ -15,7 +15,7 @@ use std::time::Duration; use flatbuffers::FlatBufferBuilder; use tokio::sync::mpsc; -use crate::client::CrowdbKvClient; +use crate::client::PaxosKvClient; use crate::error::Result; use crowdb_protocol::fb::FBMsgType; use crowdb_protocol::fb_wrappers::kv_client::FBWatchNotifyRef; @@ -25,7 +25,7 @@ use crowdb_protocol::kv_client_fb::{ use crowdb_rpc_ffi::{Buffer, RpcClient, RpcServer}; /// Re-export of the `WatchNotify` frame for callers. -pub use crowdb_kv::rpc::WatchNotify; +pub use crowdb_paxos_kv::rpc::WatchNotify; /// A live watch subscription. Dropping it unsubscribes and closes the /// stream. @@ -47,16 +47,16 @@ impl Drop for WatchSubscription { } } -/// Client for the crowdb-kv `WatchNotify` push. +/// Client for the crowdb-paxos-kv `WatchNotify` push. pub struct WatchNotifyClient { - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, } impl WatchNotifyClient { - /// Create from a shared `CrowdbKvClient`. Reuses the client's + /// Create from a shared `PaxosKvClient`. Reuses the client's /// topology discovery. Each subscription owns its push-handler transport. #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } @@ -108,7 +108,7 @@ const CROWDB_RPC_LIVENESS_CHECK: Duration = Duration::from_secs(5); #[allow(clippy::too_many_lines)] #[tracing::instrument(level = "debug", skip_all, fields(s = store_id, g = group_id))] async fn crowdb_rpc_reader_loop( - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, store_id: u64, group_id: u64, prefix: Vec<u8>, @@ -118,7 +118,7 @@ async fn crowdb_rpc_reader_loop( let mut backoff = Duration::from_millis(50); // Push handlers belong to an RPC client, not to a connection. A shared // client's next subscription would replace this subscription's handlers. - let transport = crate::KvRpcTransport::with_pool_size(1, false, false, false, 4096, 1); + let transport = crate::PaxosKvRpcTransport::with_pool_size(1, false, false, false, 4096, 1); loop { if notify_tx.is_closed() { return; @@ -243,7 +243,7 @@ fn register_crowdb_rpc_handlers( _server: &Arc<RpcServer>, notify_tx: &Arc<mpsc::Sender<WatchNotify>>, reconnect_tx: tokio::sync::mpsc::Sender<()>, - kv: Arc<CrowdbKvClient>, + kv: Arc<PaxosKvClient>, store_id: u64, ) { let runtime = tokio::runtime::Handle::current(); diff --git a/lib/crowdb-kv-client/src/transport.rs b/lib/crowdb-paxos-kv-client/src/transport.rs similarity index 65% rename from lib/crowdb-kv-client/src/transport.rs rename to lib/crowdb-paxos-kv-client/src/transport.rs index 593222e1b..654295973 100644 --- a/lib/crowdb-kv-client/src/transport.rs +++ b/lib/crowdb-paxos-kv-client/src/transport.rs @@ -7,5 +7,5 @@ pub mod cluster; mod comparison; pub mod rpc_transport; -pub use cluster::{KVClusterAdmin, KVClusterMetaClient}; -pub use rpc_transport::KvRpcTransport; +pub use cluster::{PaxosKvClusterAdmin, PaxosKvClusterMetaClient}; +pub use rpc_transport::PaxosKvRpcTransport; diff --git a/lib/crowdb-kv-client/src/transport/cluster.rs b/lib/crowdb-paxos-kv-client/src/transport/cluster.rs similarity index 92% rename from lib/crowdb-kv-client/src/transport/cluster.rs rename to lib/crowdb-paxos-kv-client/src/transport/cluster.rs index 73248a47e..66b89861f 100644 --- a/lib/crowdb-kv-client/src/transport/cluster.rs +++ b/lib/crowdb-paxos-kv-client/src/transport/cluster.rs @@ -3,16 +3,16 @@ #![allow(clippy::missing_errors_doc, clippy::cast_possible_truncation)] -//! [`KVClusterMetaClient`] and [`KVClusterAdmin`]. +//! [`PaxosKvClusterMetaClient`] and [`PaxosKvClusterAdmin`]. //! -//! `KVClusterMetaClient` reads/writes the KV-cluster topology +//! `PaxosKvClusterMetaClient` reads/writes the KV-cluster topology //! records (store/group/replica) in group 0 under `/kv/...` text-path -//! keys. `KVClusterAdmin` is the control-plane surface for cluster +//! keys. `PaxosKvClusterAdmin` is the control-plane surface for cluster //! management: each lifecycle method calls the kv-server HTTP endpoint //! AND writes the `/kv/...` record; query methods delegate to the //! kv-server HTTP mgmt API. //! -//! See `doc/design/kv/design-crowdb-kv-group0.md` §2.1, §2.2, §3. +//! See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §2.1, §2.2, §3. use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; @@ -27,7 +27,7 @@ use crowdb_protocol::mgmt::{ }; use crate::client::{GetOutcome, ScanOutcome}; -use crate::{CrowdbKvClient, Error, Result}; +use crate::{Error, PaxosKvClient, Result}; const G0_STORE: u64 = 0; const G0_GROUP: u64 = 0; @@ -40,7 +40,7 @@ fn now_ms() -> u64 { // ── helpers ───────────────────────────────────────────────────── -async fn put_json<T: serde::Serialize>(kv: &CrowdbKvClient, key: &str, value: &T) -> Result<()> { +async fn put_json<T: serde::Serialize>(kv: &PaxosKvClient, key: &str, value: &T) -> Result<()> { let payload = serde_json::to_vec(value).map_err(|e| Error::SysdataDecode { key: key.to_string(), reason: e.to_string(), @@ -50,7 +50,7 @@ async fn put_json<T: serde::Serialize>(kv: &CrowdbKvClient, key: &str, value: &T .map(|_| ()) } -async fn get_json<T: serde::de::DeserializeOwned>(kv: &CrowdbKvClient, key: &str) -> Result<Option<T>> { +async fn get_json<T: serde::de::DeserializeOwned>(kv: &PaxosKvClient, key: &str) -> Result<Option<T>> { match kv .get( G0_STORE, @@ -73,7 +73,7 @@ async fn get_json<T: serde::de::DeserializeOwned>(kv: &CrowdbKvClient, key: &str } async fn scan_prefix<T: serde::de::DeserializeOwned>( - kv: &CrowdbKvClient, + kv: &PaxosKvClient, prefix: &str, ) -> Result<Vec<(String, T)>> { let mut out: Vec<(String, T)> = Vec::new(); @@ -118,34 +118,34 @@ async fn scan_prefix<T: serde::de::DeserializeOwned>( Ok(out) } -// ── KVClusterMetaClient ───────────────────────────────────────── +// ── PaxosKvClusterMetaClient ───────────────────────────────────────── /// Client for the KV-cluster topology records (store/group/replica) /// in group 0. /// -/// All methods target store 0, group 0. The wrapped `CrowdbKvClient` +/// All methods target store 0, group 0. The wrapped `PaxosKvClient` /// must have its topology seeded with a group-0 leader endpoint. #[derive(Clone)] -pub struct KVClusterMetaClient { - kv: Arc<CrowdbKvClient>, +pub struct PaxosKvClusterMetaClient { + kv: Arc<PaxosKvClient>, } -impl KVClusterMetaClient { - /// Wrap a `CrowdbKvClient` for group-0 KV-cluster topology access. +impl PaxosKvClusterMetaClient { + /// Wrap a `PaxosKvClient` for group-0 KV-cluster topology access. #[must_use] - pub fn new(kv: CrowdbKvClient) -> Self { + pub fn new(kv: PaxosKvClient) -> Self { Self { kv: Arc::new(kv) } } - /// Wrap an already-shared `CrowdbKvClient`. + /// Wrap an already-shared `PaxosKvClient`. #[must_use] - pub fn from_shared(kv: Arc<CrowdbKvClient>) -> Self { + pub fn from_shared(kv: Arc<PaxosKvClient>) -> Self { Self { kv } } - /// Access the underlying `CrowdbKvClient`. + /// Access the underlying `PaxosKvClient`. #[must_use] - pub fn kv(&self) -> &CrowdbKvClient { + pub fn kv(&self) -> &PaxosKvClient { &self.kv } @@ -331,30 +331,30 @@ impl KVClusterMetaClient { } } -// ── KVClusterAdmin ────────────────────────────────────────────── +// ── PaxosKvClusterAdmin ────────────────────────────────────────────── /// Control-plane surface for KV-cluster management. /// /// Each lifecycle method (`add_store`, `add_group`, `add_remote`, /// `remove_*`, `step_down`, `system_init`) calls the kv-server HTTP /// management endpoint AND writes the corresponding `/kv/...` record -/// via the embedded [`KVClusterMetaClient`]. Query methods +/// via the embedded [`PaxosKvClusterMetaClient`]. Query methods /// (`list_stores`, `list_groups`, `list_remote_replicas`) delegate to /// the kv-server HTTP mgmt API for live runtime state. /// /// The `base_url` is the kv-server's HTTP management base URL /// (e.g. `http://127.0.0.1:10000`). -pub struct KVClusterAdmin { - meta: KVClusterMetaClient, +pub struct PaxosKvClusterAdmin { + meta: PaxosKvClusterMetaClient, http: reqwest::Client, base_url: String, } -impl KVClusterAdmin { - /// Create a new `KVClusterAdmin` wrapping a `KVClusterMetaClient` +impl PaxosKvClusterAdmin { + /// Create a new `PaxosKvClusterAdmin` wrapping a `PaxosKvClusterMetaClient` /// and an HTTP client bound to `base_url`. #[must_use] - pub fn new(meta: KVClusterMetaClient, base_url: &str) -> Self { + pub fn new(meta: PaxosKvClusterMetaClient, base_url: &str) -> Self { Self { meta, http: reqwest::Client::new(), @@ -362,9 +362,9 @@ impl KVClusterAdmin { } } - /// Access the embedded [`KVClusterMetaClient`]. + /// Access the embedded [`PaxosKvClusterMetaClient`]. #[must_use] - pub fn meta(&self) -> &KVClusterMetaClient { + pub fn meta(&self) -> &PaxosKvClusterMetaClient { &self.meta } diff --git a/lib/crowdb-kv-client/src/transport/comparison.rs b/lib/crowdb-paxos-kv-client/src/transport/comparison.rs similarity index 100% rename from lib/crowdb-kv-client/src/transport/comparison.rs rename to lib/crowdb-paxos-kv-client/src/transport/comparison.rs diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport.rs b/lib/crowdb-paxos-kv-client/src/transport/rpc_transport.rs similarity index 98% rename from lib/crowdb-kv-client/src/transport/rpc_transport.rs rename to lib/crowdb-paxos-kv-client/src/transport/rpc_transport.rs index b76d03732..b240391fa 100644 --- a/lib/crowdb-kv-client/src/transport/rpc_transport.rs +++ b/lib/crowdb-paxos-kv-client/src/transport/rpc_transport.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. #![allow(clippy::missing_errors_doc)] -#![allow(dead_code)] // Wired in Phase 7 (CrowdbKvClient transport selection) +#![allow(dead_code)] // Wired in Phase 7 (PaxosKvClient transport selection) //! crowdb-rpc client transport for the KV client-facing service (R117 //! migration). Mirrors `px_rpc_transport.rs` (consensus side): builds @@ -13,7 +13,7 @@ //! retry/topology/`NotLeaderHint` logic in `client.rs` is unchanged. //! //! The only transport path (the former transport `ConnectionPool` was removed). -//! window; `CrowdbKvClient` selects the transport via +//! window; `PaxosKvClient` selects the transport via //! `with_rpc_transport`. mod connections; @@ -28,7 +28,7 @@ use std::sync::Arc; use bytes::Bytes; use flatbuffers::FlatBufferBuilder; -use crowdb_kv::rpc::{ +use crowdb_paxos_kv::rpc::{ CreateSnapshotResponse, KvErrorCode, KvJournalOp, KvJournalScanResponse, KvResponse, KvScanItem, KvScanResponse, ListSnapshotsResponse, ReadMode, ReleaseSnapshotResponse, SnapshotInfo, SnapshotScanResponse, @@ -59,7 +59,7 @@ use crate::ScanDirection; /// client-side `RpcServer` (manages connections), `RpcClient` /// (request/response correlation), and a connection pool per /// endpoint (round-robin across `pool_size` connections). -pub struct KvRpcTransport { +pub struct PaxosKvRpcTransport { server: Arc<RpcServer>, rpc: Arc<RpcClient>, connections: ConnectionPoolIndex, @@ -67,16 +67,16 @@ pub struct KvRpcTransport { next_req_id: AtomicU64, } -impl std::fmt::Debug for KvRpcTransport { +impl std::fmt::Debug for PaxosKvRpcTransport { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("KvRpcTransport") + f.debug_struct("PaxosKvRpcTransport") .field("next_req_id", &self.next_req_id.load(Ordering::Relaxed)) .field("isolated_point_reads", &self.point_connections.is_some()) .finish_non_exhaustive() } } -impl KvRpcTransport { +impl PaxosKvRpcTransport { fn next_id(&self) -> u64 { self.next_req_id.fetch_add(1, Ordering::Relaxed) } @@ -252,7 +252,7 @@ impl KvRpcTransport { pub async fn send_batch_write( &self, rpc_endpoint: &str, - items: &[crowdb_kv::rpc::KvBatchItem], + items: &[crowdb_paxos_kv::rpc::KvBatchItem], client_id: u64, seq: u64, request_id: u64, @@ -315,7 +315,7 @@ impl KvRpcTransport { pub async fn send_batch_write_cas( &self, rpc_endpoint: &str, - items: &[crowdb_kv::rpc::KvBatchItem], + items: &[crowdb_paxos_kv::rpc::KvBatchItem], precondition_key: &[u8], expected_revision: u64, client_id: u64, @@ -817,7 +817,7 @@ impl KvRpcTransport { } } -impl Default for KvRpcTransport { +impl Default for PaxosKvRpcTransport { fn default() -> Self { Self::new() } @@ -852,7 +852,7 @@ fn check_client_ret_code(code: FBKvClientRetCode, msg: Option<&str>, endpoint: & /// Parse a `FBKvResponse` flatbuffer into a `KvResponse`. #[cfg(feature = "test-util")] -impl KvRpcTransport { +impl PaxosKvRpcTransport { /// Decode a synthetic peer reply through the production parser. /// # Errors /// Rejects malformed control messages and inconsistent comparisons. diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs b/lib/crowdb-paxos-kv-client/src/transport/rpc_transport/connections.rs similarity index 98% rename from lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs rename to lib/crowdb-paxos-kv-client/src/transport/rpc_transport/connections.rs index 4031b992f..23fc79b04 100644 --- a/lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs +++ b/lib/crowdb-paxos-kv-client/src/transport/rpc_transport/connections.rs @@ -4,11 +4,11 @@ //! Shared transport owners and independently invalidated point/ordinary pools. use super::{ - Arc, AtomicU64, ConnectionPoolError, ConnectionPoolIndex, Error, KvRpcTransport, Result, RpcClient, + Arc, AtomicU64, ConnectionPoolError, ConnectionPoolIndex, Error, PaxosKvRpcTransport, Result, RpcClient, RpcError, RpcServer, SelectedConnection, }; -impl KvRpcTransport { +impl PaxosKvRpcTransport { /// Create a new crowdb-rpc transport with `pool_size` connections /// per endpoint, using the given RPC tunables and `workers` I/O /// worker threads. The `RpcServer` is the client-side transport — diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport/owned.rs b/lib/crowdb-paxos-kv-client/src/transport/rpc_transport/owned.rs similarity index 96% rename from lib/crowdb-kv-client/src/transport/rpc_transport/owned.rs rename to lib/crowdb-paxos-kv-client/src/transport/rpc_transport/owned.rs index 3326391bc..c290819b3 100644 --- a/lib/crowdb-kv-client/src/transport/rpc_transport/owned.rs +++ b/lib/crowdb-paxos-kv-client/src/transport/rpc_transport/owned.rs @@ -4,16 +4,16 @@ use super::{ parse_kv_response, Buffer, Error, FBKvBatchItem, FBKvBatchItemArgs, FBKvBatchWriteRequest, FBKvBatchWriteRequestArgs, FBKvRevisionPrecondition, FBKvRevisionPreconditionArgs, FBMsgType, - FlatBufferBuilder, KvResponse, KvRpcTransport, Result, + FlatBufferBuilder, KvResponse, PaxosKvRpcTransport, Result, }; use crowdb_protocol::kv_client_fb::{FBKvOwnerFence, FBKvOwnerFenceArgs}; -impl KvRpcTransport { +impl PaxosKvRpcTransport { #[allow(clippy::too_many_arguments)] pub async fn send_batch_write_owned( &self, rpc_endpoint: &str, - items: &[crowdb_kv::rpc::KvBatchItem], + items: &[crowdb_paxos_kv::rpc::KvBatchItem], precondition_key: &[u8], expected_value: &[u8], record_condition: Option<(&[u8], u64)>, diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs b/lib/crowdb-paxos-kv-client/src/transport/rpc_transport/point_reads.rs similarity index 96% rename from lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs rename to lib/crowdb-paxos-kv-client/src/transport/rpc_transport/point_reads.rs index 6174dc7a6..cf4c573db 100644 --- a/lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs +++ b/lib/crowdb-paxos-kv-client/src/transport/rpc_transport/point_reads.rs @@ -5,10 +5,10 @@ use super::{ parse_kv_response, read_mode_to_fb, Buffer, Error, FBKvGetRequest, FBKvGetRequestArgs, FBMsgType, - FlatBufferBuilder, KvResponse, KvRpcTransport, ReadMode, Result, + FlatBufferBuilder, KvResponse, PaxosKvRpcTransport, ReadMode, Result, }; -impl KvRpcTransport { +impl PaxosKvRpcTransport { /// Send a `Get` request via crowdb-rpc. #[allow(clippy::too_many_arguments)] pub async fn send_get( diff --git a/lib/crowdb-paxos-kv-client/tests/chunk_slot_initialization_test.rs b/lib/crowdb-paxos-kv-client/tests/chunk_slot_initialization_test.rs new file mode 100644 index 000000000..4f1e0f203 --- /dev/null +++ b/lib/crowdb-paxos-kv-client/tests/chunk_slot_initialization_test.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#[path = "common/slot_map.rs"] +mod slot_map; + +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_paxos_kv_client::{BatchOp, ChunkSlotMapClient, Error}; +use crowdb_protocol::chunk_slot::{ChunkSlot, ChunkSlotMap}; +use crowdb_protocol::key::{ChunkServiceSlotsKey, ChunkSlotMapHeadKey, TextKey}; +use slot_map::{services, TestSlotMap}; + +#[tokio::test] +async fn large_bootstrap_preserves_all_paginated_owners_and_is_idempotent() { + let test = TestSlotMap::start().await; + let client = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + let map = services(1, 300); + client.initialize_service(&map).await.unwrap(); + client.initialize_service(&map).await.unwrap(); + let actual = client.read_service().await.unwrap(); + assert_eq!(actual.bindings().len(), 300); + assert_eq!( + actual + .bindings() + .iter() + .filter(|entry| entry.slots.is_empty()) + .count(), + 299 + ); + assert!(ChunkSlot::all().all(|slot| actual.owner(slot) == 1)); +} + +#[tokio::test] +async fn interrupted_bootstrap_is_hidden_resumable_and_fences_stale_pages() { + let test = TestSlotMap::start().await; + let map = services(1, 300); + let mut bindings = map.bindings().to_vec(); + bindings.sort_by_key(|binding| { + ChunkServiceSlotsKey { + instance_id: binding.owner, + } + .to_path() + }); + let head = ChunkSlotMapHeadKey::Service.to_path(); + let pending = serde_json::json!({"initializing": {"head": map.head(), "bindings": bindings}, "next": 63}); + let mut operations: Vec<_> = bindings[..63] + .iter() + .map(|binding| BatchOp::Put { + key: Bytes::from( + ChunkServiceSlotsKey { + instance_id: binding.owner, + } + .to_path(), + ), + value: Bytes::from(serde_json::to_vec(binding).unwrap()), + }) + .collect(); + operations.push(BatchOp::Put { + key: Bytes::from(head.clone()), + value: Bytes::from(serde_json::to_vec(&pending).unwrap()), + }); + let partial = test + .kv + .batch_write_cas(0, 0, &operations, head.as_bytes(), 0) + .await + .unwrap(); + let first = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + assert!(first.read_service().await.is_err()); + assert!(first.initialize_service(&services(1000, 300)).await.is_err()); + drop(first); + let resumed = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + bindings.reverse(); + let reordered = ChunkSlotMap::new(map.head().clone(), bindings).unwrap(); + resumed.initialize_service(&reordered).await.unwrap(); + assert_eq!(resumed.read_service().await.unwrap().bindings().len(), 300); + assert!(matches!( + test.kv + .batch_write_cas(0, 0, &operations, head.as_bytes(), partial.revision) + .await, + Err(Error::CasFailed { .. }) + )); + assert_eq!(resumed.read_service().await.unwrap().bindings().len(), 300); +} + +#[tokio::test] +async fn competing_large_and_small_bootstraps_cannot_mix_assignments() { + for count in [2, 300] { + let test = TestSlotMap::start().await; + let client = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + let large = services(1, 300); + let competitor = services(1000, count); + let (a, b) = tokio::join!( + client.initialize_service(&large), + client.initialize_service(&competitor) + ); + assert_ne!(a.is_ok(), b.is_ok(), "large={a:?}, competitor={b:?}"); + let actual = client.read_service().await.unwrap(); + let expected = if a.is_ok() { &large } else { &competitor }; + assert_eq!(actual.head(), expected.head()); + assert!(actual + .bindings() + .iter() + .all(|binding| expected.bindings().contains(binding))); + } +} diff --git a/lib/crowdb-kv-client/tests/chunk_slot_owner_fence_test.rs b/lib/crowdb-paxos-kv-client/tests/chunk_slot_owner_fence_test.rs similarity index 95% rename from lib/crowdb-kv-client/tests/chunk_slot_owner_fence_test.rs rename to lib/crowdb-paxos-kv-client/tests/chunk_slot_owner_fence_test.rs index 93504c96c..728cb17e0 100644 --- a/lib/crowdb-kv-client/tests/chunk_slot_owner_fence_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/chunk_slot_owner_fence_test.rs @@ -5,7 +5,7 @@ mod owner_fence_server; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, Error, GetOutcome, KvRpcTransport, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, Error, GetOutcome, PaxosKvRpcTransport, ReadMode}; use crowdb_protocol::chunk_slot::{ChunkServiceIncarnation, ChunkSlotAuthority}; use owner_fence_server::TestOwnerFenceServer; @@ -28,7 +28,7 @@ async fn cancelled_handover_drains_one_slot_without_blocking_another() { let admitted = group.hold_owner_write_for_tests(Bytes::from_static(FENCE)); let endpoint = servers.endpoint.clone(); let claim = tokio::spawn(async move { - KvRpcTransport::new() + PaxosKvRpcTransport::new() .send_put_cas(&endpoint, FENCE, &new, initial.revision, 901, 1, 1, 0, 1) .await }); diff --git a/lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs b/lib/crowdb-paxos-kv-client/tests/chunk_slot_publication_test.rs similarity index 96% rename from lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs rename to lib/crowdb-paxos-kv-client/tests/chunk_slot_publication_test.rs index 3fdd5b416..30c82712e 100644 --- a/lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/chunk_slot_publication_test.rs @@ -6,8 +6,8 @@ mod fixture; #[path = "common/publication_peer.rs"] mod peer; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv_client::{ChunkServiceSnapshot, ChunkSlotMapClient, ClientConfig, CrowdbKvClient, Error}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv_client::{ChunkServiceSnapshot, ChunkSlotMapClient, ClientConfig, Error, PaxosKvClient}; use crowdb_protocol::{ chunk_slot::{ChunkSlot, ChunkSlotAuthority, ChunkSlotMap}, key::{ChunkSlotMapHeadKey, TextKey}, @@ -21,7 +21,7 @@ use std::{ #[tokio::test] async fn ambiguous_publication_does_not_rebase_after_busy_reconciliation() { let peer = peer::TestPublicationPeer::start().await; - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![]))); kv.seed_leader(0, 0, peer.endpoint.clone()); let maps = ChunkSlotMapClient::new(kv); let before = maps.read_service_snapshot().await.unwrap(); diff --git a/lib/crowdb-kv-client/tests/chunkdb_partition_test.rs b/lib/crowdb-paxos-kv-client/tests/chunkdb_partition_test.rs similarity index 83% rename from lib/crowdb-kv-client/tests/chunkdb_partition_test.rs rename to lib/crowdb-paxos-kv-client/tests/chunkdb_partition_test.rs index e1c7c5e12..ccfe19020 100644 --- a/lib/crowdb-kv-client/tests/chunkdb_partition_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/chunkdb_partition_test.rs @@ -1,13 +1,13 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::{compute_sub_range_assignment, DEFAULT_SUB_RANGE_COUNT}; +use crowdb_paxos_kv_client::{compute_sub_range_assignment, DEFAULT_SUB_RANGE_COUNT}; use crowdb_protocol::common::InstanceValue; #[tokio::test] async fn legacy_partition_writer_cannot_publish_assignments() { - use crowdb_kv_client::{BindingStrategy, ChunkdbRangeStrategy, ClientConfig, CrowdbKvClient}; - let client = CrowdbKvClient::new(ClientConfig::new(Vec::new())); + use crowdb_paxos_kv_client::{BindingStrategy, ChunkdbRangeStrategy, ClientConfig, PaxosKvClient}; + let client = PaxosKvClient::new(ClientConfig::new(Vec::new())); let strategy = ChunkdbRangeStrategy::new(); let error = strategy.write_bindings(&client, &[]).await.unwrap_err(); assert!(error.to_string().contains("legacy range assignment is disabled")); diff --git a/lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs b/lib/crowdb-paxos-kv-client/tests/common/chunk_slot_publication.rs similarity index 83% rename from lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs rename to lib/crowdb-paxos-kv-client/tests/common/chunk_slot_publication.rs index d16bed7d0..d3730d7a0 100644 --- a/lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs +++ b/lib/crowdb-paxos-kv-client/tests/common/chunk_slot_publication.rs @@ -3,10 +3,10 @@ use std::sync::Arc; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ChunkSlotMapClient, ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ChunkSlotMapClient, ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; use crowdb_protocol::{ chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, key::{ChunkSlotMapHeadKey, TextKey}, @@ -14,7 +14,7 @@ use crowdb_protocol::{ pub struct TestChunkSlotPublication { pub store: Arc<PxKvStore>, - pub kv: Arc<CrowdbKvClient>, + pub kv: Arc<PaxosKvClient>, } impl TestChunkSlotPublication { @@ -27,7 +27,7 @@ impl TestChunkSlotPublication { )); } store.start().await.unwrap(); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![]))); for group in [0, 1] { kv.seed_leader(0, group, store.listen_addr().unwrap().to_string()); } diff --git a/lib/crowdb-kv-client/tests/common/conditional_servers.rs b/lib/crowdb-paxos-kv-client/tests/common/conditional_servers.rs similarity index 88% rename from lib/crowdb-kv-client/tests/common/conditional_servers.rs rename to lib/crowdb-paxos-kv-client/tests/common/conditional_servers.rs index e06d67f6a..f3f9c8c33 100644 --- a/lib/crowdb-kv-client/tests/common/conditional_servers.rs +++ b/lib/crowdb-paxos-kv-client/tests/common/conditional_servers.rs @@ -5,12 +5,12 @@ use std::sync::Arc; use std::time::Duration; use axum::{routing::get, Json, Router}; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::cluster::remote_replica::PxRemoteReplica; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::cluster::remote_replica::PxRemoteReplica; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::mgmt::{GroupStatus, ReplicaStatus, StoreStatus, TopologyResponse}; pub struct TestServers { @@ -86,8 +86,8 @@ impl TestServers { config } - pub fn client(&self) -> CrowdbKvClient { - let client = CrowdbKvClient::new(self.client_config()); + pub fn client(&self) -> PaxosKvClient { + let client = PaxosKvClient::new(self.client_config()); client.seed_leader(1, 1, self.follower.listen_addr().unwrap().to_string()); client } diff --git a/lib/crowdb-kv-client/tests/common/owner_fence_server.rs b/lib/crowdb-paxos-kv-client/tests/common/owner_fence_server.rs similarity index 70% rename from lib/crowdb-kv-client/tests/common/owner_fence_server.rs rename to lib/crowdb-paxos-kv-client/tests/common/owner_fence_server.rs index 89f8e2398..3f4d02925 100644 --- a/lib/crowdb-kv-client/tests/common/owner_fence_server.rs +++ b/lib/crowdb-paxos-kv-client/tests/common/owner_fence_server.rs @@ -3,14 +3,14 @@ use std::sync::Arc; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; pub struct TestOwnerFenceServer { pub store: Arc<PxKvStore>, - pub client: CrowdbKvClient, + pub client: PaxosKvClient, pub endpoint: String, } @@ -23,7 +23,7 @@ impl TestOwnerFenceServer { )); store.start().await.unwrap(); let endpoint = store.listen_addr().unwrap().to_string(); - let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + let client = PaxosKvClient::new(ClientConfig::new(vec![])); client.seed_leader(1, 1, endpoint.clone()); Self { store, diff --git a/lib/crowdb-kv-client/tests/common/point_read_peer.rs b/lib/crowdb-paxos-kv-client/tests/common/point_read_peer.rs similarity index 100% rename from lib/crowdb-kv-client/tests/common/point_read_peer.rs rename to lib/crowdb-paxos-kv-client/tests/common/point_read_peer.rs diff --git a/lib/crowdb-kv-client/tests/common/publication_peer.rs b/lib/crowdb-paxos-kv-client/tests/common/publication_peer.rs similarity index 100% rename from lib/crowdb-kv-client/tests/common/publication_peer.rs rename to lib/crowdb-paxos-kv-client/tests/common/publication_peer.rs diff --git a/lib/crowdb-paxos-kv-client/tests/common/slot_map.rs b/lib/crowdb-paxos-kv-client/tests/common/slot_map.rs new file mode 100644 index 000000000..58fbe50b2 --- /dev/null +++ b/lib/crowdb-paxos-kv-client/tests/common/slot_map.rs @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; +use crowdb_protocol::chunk_slot::{ + ChunkSlot, ChunkSlotBinding, ChunkSlotBitmap, ChunkSlotMap, ChunkSlotMapHead, CHUNK_SLOT_LAYOUT_VERSION, +}; + +pub struct TestSlotMap { + store: Arc<PxKvStore>, + pub kv: Arc<PaxosKvClient>, +} + +impl TestSlotMap { + pub async fn start() -> Self { + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new( + 0, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + store.start().await.unwrap(); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![]))); + kv.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); + Self { store, kv } + } +} + +impl Drop for TestSlotMap { + fn drop(&mut self) { + self.store.stop(); + } +} + +pub fn services(first: u64, count: u32) -> ChunkSlotMap<u64> { + ChunkSlotMap::new( + ChunkSlotMapHead { + layout_version: CHUNK_SLOT_LAYOUT_VERSION, + generation: 1, + owner_count: count, + }, + (first..first + u64::from(count)) + .map(|owner| ChunkSlotBinding { + generation: 1, + owner, + slots: if owner == first { + ChunkSlot::all().collect() + } else { + ChunkSlotBitmap::default() + }, + }) + .collect(), + ) + .unwrap() +} diff --git a/lib/crowdb-kv-client/tests/common/storage.rs b/lib/crowdb-paxos-kv-client/tests/common/storage.rs similarity index 90% rename from lib/crowdb-kv-client/tests/common/storage.rs rename to lib/crowdb-paxos-kv-client/tests/common/storage.rs index 7b6a26e40..85d1f1638 100644 --- a/lib/crowdb-kv-client/tests/common/storage.rs +++ b/lib/crowdb-paxos-kv-client/tests/common/storage.rs @@ -3,10 +3,10 @@ use std::sync::Arc; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; +use crowdb_paxos_kv_client::{ClientConfig, HardwareClient, PaxosKvClient}; use crowdb_protocol::{ common::{DiskId, HwStatus, NodeValue, RackValue}, disk_storage::DiskStorageRecord, @@ -30,7 +30,7 @@ impl TestStorage { )); } store.start().await.unwrap(); - let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + let client = PaxosKvClient::new(ClientConfig::new(vec![])); client.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); client.seed_leader(0, 1, store.listen_addr().unwrap().to_string()); let hardware = HardwareClient::new(client); diff --git a/lib/crowdb-kv-client/tests/common/stream_lease.rs b/lib/crowdb-paxos-kv-client/tests/common/stream_lease.rs similarity index 70% rename from lib/crowdb-kv-client/tests/common/stream_lease.rs rename to lib/crowdb-paxos-kv-client/tests/common/stream_lease.rs index db95b5286..935b25727 100644 --- a/lib/crowdb-kv-client/tests/common/stream_lease.rs +++ b/lib/crowdb-paxos-kv-client/tests/common/stream_lease.rs @@ -3,14 +3,14 @@ use std::sync::Arc; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; pub struct TestStreamLeaseKv { pub store: Arc<PxKvStore>, - pub kv: Arc<CrowdbKvClient>, + pub kv: Arc<PaxosKvClient>, } impl TestStreamLeaseKv { @@ -21,7 +21,7 @@ impl TestStreamLeaseKv { PxLocalReplica::new(1, PxLocalReplicaRole::Leader), )); store.start().await.unwrap(); - let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + let kv = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![]))); kv.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); Self { store, kv } } diff --git a/lib/crowdb-kv-client/tests/common/stream_lease_clock.rs b/lib/crowdb-paxos-kv-client/tests/common/stream_lease_clock.rs similarity index 96% rename from lib/crowdb-kv-client/tests/common/stream_lease_clock.rs rename to lib/crowdb-paxos-kv-client/tests/common/stream_lease_clock.rs index abf8edb72..b36132087 100644 --- a/lib/crowdb-kv-client/tests/common/stream_lease_clock.rs +++ b/lib/crowdb-paxos-kv-client/tests/common/stream_lease_clock.rs @@ -18,6 +18,7 @@ impl TestStreamLeaseRecord { prior_exclusion_deadline_ms: 0, grant_sequence: 0, grant_expires_at_ms: 0, + revoked: false, } } } diff --git a/lib/crowdb-kv-client/tests/common/topology_recovery.rs b/lib/crowdb-paxos-kv-client/tests/common/topology_recovery.rs similarity index 90% rename from lib/crowdb-kv-client/tests/common/topology_recovery.rs rename to lib/crowdb-paxos-kv-client/tests/common/topology_recovery.rs index c15614182..e6849f833 100644 --- a/lib/crowdb-kv-client/tests/common/topology_recovery.rs +++ b/lib/crowdb-paxos-kv-client/tests/common/topology_recovery.rs @@ -2,10 +2,10 @@ // Licensed under the Apache License, Version 2.0. use axum::{routing::get, Json, Router}; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use std::{ sync::{ atomic::{AtomicUsize, Ordering}, @@ -17,7 +17,7 @@ use std::{ pub struct TestDiscovery { stores: Vec<Arc<PxKvStore>>, servers: Vec<tokio::task::JoinHandle<()>>, - pub client: Arc<CrowdbKvClient>, + pub client: Arc<PaxosKvClient>, pub fetches: Arc<AtomicUsize>, } @@ -63,7 +63,7 @@ impl TestDiscovery { config.retry.max_retries = 2; config.retry.unknown_leader_wait = Duration::from_millis(10); config.retry.backoff_base = Duration::from_secs(1); - let client = Arc::new(CrowdbKvClient::new(config)); + let client = Arc::new(PaxosKvClient::new(config)); client.seed_leader(11, 1, failed); Self { stores, diff --git a/lib/crowdb-kv-client/tests/comparison_wire_test.rs b/lib/crowdb-paxos-kv-client/tests/comparison_wire_test.rs similarity index 75% rename from lib/crowdb-kv-client/tests/comparison_wire_test.rs rename to lib/crowdb-paxos-kv-client/tests/comparison_wire_test.rs index 506413c14..66764c8bf 100644 --- a/lib/crowdb-kv-client/tests/comparison_wire_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/comparison_wire_test.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv_client::KvRpcTransport; +use crowdb_paxos_kv_client::PaxosKvRpcTransport; use crowdb_protocol::kv_client_fb::{ FBKvClientRetCode, FBKvCompareObservation, FBKvCompareObservationArgs, FBKvResponse, FBKvResponseArgs, }; @@ -38,22 +38,28 @@ fn reply(code: FBKvClientRetCode, revision: u64, observed: Option<(u64, &[u8])>) #[test] fn legacy_failure_is_not_misread_as_an_observed_absent_key() { let legacy = - KvRpcTransport::parse_response_for_tests(&reply(FBKvClientRetCode::CasFailed, 8, None)).unwrap(); + PaxosKvRpcTransport::parse_response_for_tests(&reply(FBKvClientRetCode::CasFailed, 8, None)).unwrap(); assert_eq!(legacy.revision, 8); assert!(legacy.compare_observation.is_none()); - let absent = - KvRpcTransport::parse_response_for_tests(&reply(FBKvClientRetCode::CasFailed, 0, Some((0, b"")))) - .unwrap() - .compare_observation - .unwrap(); + let absent = PaxosKvRpcTransport::parse_response_for_tests(&reply( + FBKvClientRetCode::CasFailed, + 0, + Some((0, b"")), + )) + .unwrap() + .compare_observation + .unwrap(); assert_eq!(absent.revision, 0); assert_eq!(absent.key.as_ref(), b"key\0\xff"); assert!(absent.value.is_empty()); - let empty = - KvRpcTransport::parse_response_for_tests(&reply(FBKvClientRetCode::CasFailed, 9, Some((9, b"")))) - .unwrap() - .compare_observation - .unwrap(); + let empty = PaxosKvRpcTransport::parse_response_for_tests(&reply( + FBKvClientRetCode::CasFailed, + 9, + Some((9, b"")), + )) + .unwrap() + .compare_observation + .unwrap(); assert_eq!(empty.revision, 9); assert!(empty.value.is_empty()); } @@ -67,7 +73,9 @@ fn invalid_observations_never_become_definite_comparison_results() { (FBKvClientRetCode::OutcomeUnknown, 8, (8, b"value".as_slice())), (FBKvClientRetCode::Success, 8, (8, b"value".as_slice())), ] { - assert!(KvRpcTransport::parse_response_for_tests(&reply(code, revision, Some(observed))).is_err()); + assert!( + PaxosKvRpcTransport::parse_response_for_tests(&reply(code, revision, Some(observed))).is_err() + ); } let mut builder = FlatBufferBuilder::new(); let comparison = FBKvCompareObservation::create(&mut builder, &FBKvCompareObservationArgs::default()); @@ -80,5 +88,5 @@ fn invalid_observations_never_become_definite_comparison_results() { }, ); builder.finish(response, None); - assert!(KvRpcTransport::parse_response_for_tests(builder.finished_data()).is_err()); + assert!(PaxosKvRpcTransport::parse_response_for_tests(builder.finished_data()).is_err()); } diff --git a/lib/crowdb-kv-client/tests/conditional_observation_test.rs b/lib/crowdb-paxos-kv-client/tests/conditional_observation_test.rs similarity index 97% rename from lib/crowdb-kv-client/tests/conditional_observation_test.rs rename to lib/crowdb-paxos-kv-client/tests/conditional_observation_test.rs index bdfd05226..cf9b4d5f3 100644 --- a/lib/crowdb-kv-client/tests/conditional_observation_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/conditional_observation_test.rs @@ -5,7 +5,7 @@ mod owner_fence_server; use bytes::Bytes; -use crowdb_kv_client::{BatchOp, Error, KvCompareObservation, KvRpcTransport}; +use crowdb_paxos_kv_client::{BatchOp, Error, KvCompareObservation, PaxosKvRpcTransport}; use crowdb_protocol::chunk_slot::{ChunkServiceIncarnation, ChunkSlotAuthority}; use owner_fence_server::TestOwnerFenceServer; use std::time::Duration; @@ -56,7 +56,7 @@ async fn four_real_conflicts_reuse_versioned_values_without_any_get() { async fn absent_and_existing_empty_values_remain_distinct_through_batch_delete() { let test = TestOwnerFenceServer::start().await; let key = b"empty"; - let raw = KvRpcTransport::new() + let raw = PaxosKvRpcTransport::new() .send_put_cas(&test.endpoint, key, b"bad", 99, 901, 1, 1, 0, 1) .await .unwrap(); diff --git a/lib/crowdb-kv-client/tests/conditional_retry_test.rs b/lib/crowdb-paxos-kv-client/tests/conditional_retry_test.rs similarity index 88% rename from lib/crowdb-kv-client/tests/conditional_retry_test.rs rename to lib/crowdb-paxos-kv-client/tests/conditional_retry_test.rs index cdc46190e..be1cbca62 100644 --- a/lib/crowdb-kv-client/tests/conditional_retry_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/conditional_retry_test.rs @@ -5,8 +5,8 @@ mod conditional_servers; use conditional_servers::TestServers; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv_client::{BatchOp, CrowdbKvClient, Error, KvRpcTransport}; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv_client::{BatchOp, Error, PaxosKvClient, PaxosKvRpcTransport}; #[tokio::test] async fn a_removed_replica_refreshes_its_stale_route_before_reading_or_writing() { @@ -19,11 +19,17 @@ async fn a_removed_replica_refreshes_its_stale_route_before_reading_or_writing() .unwrap(); let value = servers .client() - .get(1, 1, b"value", crowdb_kv_client::ReadMode::Linearizable, None) + .get( + 1, + 1, + b"value", + crowdb_paxos_kv_client::ReadMode::Linearizable, + None, + ) .await .unwrap(); assert!( - matches!(value, crowdb_kv_client::GetOutcome::Found { value, .. } if value.as_ref() == b"retained") + matches!(value, crowdb_paxos_kv_client::GetOutcome::Found { value, .. } if value.as_ref() == b"retained") ); servers .client() @@ -39,7 +45,7 @@ async fn a_removed_replica_refreshes_its_stale_route_before_reading_or_writing() b"", b"", 10, - crowdb_kv_client::ReadMode::Linearizable, + crowdb_paxos_kv_client::ReadMode::Linearizable, None, false, None, @@ -53,7 +59,7 @@ async fn a_removed_replica_refreshes_its_stale_route_before_reading_or_writing() #[tokio::test] async fn conditional_writes_discover_leader_after_explicit_rejection_without_a_hint() { let servers = TestServers::start(true).await; - let raw = KvRpcTransport::new() + let raw = PaxosKvRpcTransport::new() .send_put_cas( &servers.follower.listen_addr().unwrap().to_string(), b"sanity", @@ -134,7 +140,7 @@ async fn discovering_a_replacement_endpoint_does_not_wait_on_the_failed_endpoint let mut config = servers.client_config(); config.retry.max_retries = 1; config.retry.backoff_base = std::time::Duration::from_secs(1); - let client = CrowdbKvClient::new(config); + let client = PaxosKvClient::new(config); client.seed_leader(1, 1, "127.0.0.1:1".into()); let result = tokio::time::timeout( std::time::Duration::from_millis(250), diff --git a/lib/crowdb-kv-client/tests/disk_identity_test.rs b/lib/crowdb-paxos-kv-client/tests/disk_identity_test.rs similarity index 100% rename from lib/crowdb-kv-client/tests/disk_identity_test.rs rename to lib/crowdb-paxos-kv-client/tests/disk_identity_test.rs diff --git a/lib/crowdb-kv-client/tests/disk_resource_test.rs b/lib/crowdb-paxos-kv-client/tests/disk_resource_test.rs similarity index 99% rename from lib/crowdb-kv-client/tests/disk_resource_test.rs rename to lib/crowdb-paxos-kv-client/tests/disk_resource_test.rs index e0f652ce1..ca155fbab 100644 --- a/lib/crowdb-kv-client/tests/disk_resource_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/disk_resource_test.rs @@ -3,7 +3,7 @@ #[path = "common/storage.rs"] mod storage; -use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::{ common::{NodeValue, RackValue}, disk_storage::{DiskStorageRecord, PhysicalBacking}, diff --git a/lib/crowdb-kv-client/tests/disk_status_test.rs b/lib/crowdb-paxos-kv-client/tests/disk_status_test.rs similarity index 95% rename from lib/crowdb-kv-client/tests/disk_status_test.rs rename to lib/crowdb-paxos-kv-client/tests/disk_status_test.rs index 94a0a79d5..c49f4a6b4 100644 --- a/lib/crowdb-kv-client/tests/disk_status_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/disk_status_test.rs @@ -3,10 +3,10 @@ use std::sync::Arc; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{BatchOp, ClientConfig, CrowdbKvClient, GetOutcome, HardwareClient, ReadMode}; +use crowdb_paxos_kv_client::{BatchOp, ClientConfig, GetOutcome, HardwareClient, PaxosKvClient, ReadMode}; use crowdb_protocol::{ common::{DiskId, HwStatus, RackValue}, diskdb::rpc::{DiskGroupValue, DiskValue}, @@ -129,7 +129,7 @@ async fn status_change_invalidates_an_already_prepared_hierarchy_removal() { hw.kv() .batch_write_cas(0, 0, &stale_remove, rack.as_bytes(), revision) .await, - Err(crowdb_kv_client::Error::CasFailed { .. }) + Err(crowdb_paxos_kv_client::Error::CasFailed { .. }) )); assert_eq!( hw.get_disk_with_revision(&key).await.unwrap().unwrap().0.status, @@ -145,7 +145,7 @@ impl TestHardware { PxLocalReplica::new(1, PxLocalReplicaRole::Leader), )); store.start().await.unwrap(); - let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + let client = PaxosKvClient::new(ClientConfig::new(vec![])); client.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); let hardware = HardwareClient::new(client); hardware.add_rack(1, &RackValue::default()).await.unwrap(); diff --git a/lib/crowdb-kv-client/tests/diskdb_session_test.rs b/lib/crowdb-paxos-kv-client/tests/diskdb_session_test.rs similarity index 98% rename from lib/crowdb-kv-client/tests/diskdb_session_test.rs rename to lib/crowdb-paxos-kv-client/tests/diskdb_session_test.rs index beb004f1c..b9fd4380c 100644 --- a/lib/crowdb-kv-client/tests/diskdb_session_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/diskdb_session_test.rs @@ -3,7 +3,7 @@ #[path = "common/storage.rs"] mod storage; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::key::OwnerMapKey; use storage::TestStorage; diff --git a/lib/crowdb-kv-client/tests/e2e_follower_read_test.rs b/lib/crowdb-paxos-kv-client/tests/e2e_follower_read_test.rs similarity index 95% rename from lib/crowdb-kv-client/tests/e2e_follower_read_test.rs rename to lib/crowdb-paxos-kv-client/tests/e2e_follower_read_test.rs index 8c9d1647a..de11528f9 100644 --- a/lib/crowdb-kv-client/tests/e2e_follower_read_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/e2e_follower_read_test.rs @@ -42,14 +42,14 @@ use axum::extract::State; use axum::routing::get; use axum::{Json, Router}; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::cluster::remote_replica::PxRemoteReplica; -use crowdb_kv::cluster::status::RemoteStatus; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::cluster::remote_replica::PxRemoteReplica; +use crowdb_paxos_kv::cluster::status::RemoteStatus; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadEndpointPolicy, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadEndpointPolicy, ReadMode}; const STORE_ID: u64 = 1; const GROUP_ID: u64 = 1; @@ -245,7 +245,7 @@ async fn any_replica_distributes_minslot_reads_with_lagging_follower() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(any_replica_config(seed)); + let client = PaxosKvClient::new(any_replica_config(seed)); let write = client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -297,7 +297,7 @@ async fn any_replica_falls_back_to_leader_when_follower_lags() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(any_replica_config(seed)); + let client = PaxosKvClient::new(any_replica_config(seed)); let write = client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -340,7 +340,7 @@ async fn any_replica_linearizable_still_targets_leader() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(any_replica_config(seed)); + let client = PaxosKvClient::new(any_replica_config(seed)); client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -379,7 +379,7 @@ async fn leader_policy_unchanged_for_minslot() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(ClientConfig::new(vec![seed])); + let client = PaxosKvClient::new(ClientConfig::new(vec![seed])); client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -416,7 +416,7 @@ async fn any_replica_scan_distributes_with_lagging_follower() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(any_replica_config(seed)); + let client = PaxosKvClient::new(any_replica_config(seed)); client .put(STORE_ID, GROUP_ID, b"prefix_k1", b"v1", None) @@ -478,7 +478,7 @@ async fn any_replica_scan_falls_back_when_follower_lags() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(any_replica_config(seed)); + let client = PaxosKvClient::new(any_replica_config(seed)); let write = client .put(STORE_ID, GROUP_ID, b"prefix_k1", b"v1", None) @@ -530,13 +530,13 @@ async fn any_replica_scan_falls_back_when_follower_lags() { /// error, exact prefix boundary. #[tokio::test] async fn follow_scan_not_leader_parser_extracts_endpoint() { - use crowdb_kv_client::CrowdbKvClient; + use crowdb_paxos_kv_client::PaxosKvClient; // The parser is a private associated function; reach it through the - // public `ClientConfig` -> `CrowdbKvClient` constructor (no I/O needed + // public `ClientConfig` -> `PaxosKvClient` constructor (no I/O needed // since we call a pure function). let cfg = ClientConfig::new(Vec::new()); - let _client = CrowdbKvClient::new(cfg); // construct to satisfy borrow + let _client = PaxosKvClient::new(cfg); // construct to satisfy borrow // Direct parser checks via the associated function path. The // function is private, so we exercise the same logic inline to @@ -585,7 +585,7 @@ async fn least_connections_distributes_minslot_reads() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(least_connections_config(seed)); + let client = PaxosKvClient::new(least_connections_config(seed)); let write = client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -631,7 +631,7 @@ async fn latency_distributes_minslot_reads() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(latency_config(seed)); + let client = PaxosKvClient::new(latency_config(seed)); client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -681,7 +681,7 @@ async fn least_connections_routes_away_from_slow_replica() { lagging.set_get_delay_for_tests(Duration::from_millis(50)); let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = Arc::new(CrowdbKvClient::new(least_connections_config(seed))); + let client = Arc::new(PaxosKvClient::new(least_connections_config(seed))); client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -746,7 +746,7 @@ async fn latency_routes_away_from_slow_replica() { lagging.set_get_delay_for_tests(Duration::from_millis(50)); let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(latency_config(seed)); + let client = PaxosKvClient::new(latency_config(seed)); client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -785,7 +785,7 @@ async fn least_connections_falls_back_to_leader() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(least_connections_config(seed)); + let client = PaxosKvClient::new(least_connections_config(seed)); let write = client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -828,7 +828,7 @@ async fn latency_falls_back_to_leader() { let (leader, follower, lagging) = start_three_node_cluster().await; let lagging_ep = lagging.listen_addr().expect("lagging server started").to_string(); let seed = spawn_topology_server(leader.clone(), lagging_ep).await; - let client = CrowdbKvClient::new(latency_config(seed)); + let client = PaxosKvClient::new(latency_config(seed)); let write = client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) @@ -874,7 +874,7 @@ async fn new_policies_linearizable_still_targets_leader() { for policy in [ReadEndpointPolicy::LeastConnections, ReadEndpointPolicy::Latency] { let mut cfg = ClientConfig::new(vec![seed.clone()]); cfg.read_endpoint_policy = policy; - let client = CrowdbKvClient::new(cfg); + let client = PaxosKvClient::new(cfg); client .put(STORE_ID, GROUP_ID, b"k1", b"v1", None) diff --git a/lib/crowdb-kv-client/tests/e2e_single_node_test.rs b/lib/crowdb-paxos-kv-client/tests/e2e_single_node_test.rs similarity index 90% rename from lib/crowdb-kv-client/tests/e2e_single_node_test.rs rename to lib/crowdb-paxos-kv-client/tests/e2e_single_node_test.rs index 96cfc631c..c23ea2d37 100644 --- a/lib/crowdb-kv-client/tests/e2e_single_node_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/e2e_single_node_test.rs @@ -10,13 +10,13 @@ use axum::extract::State; use axum::routing::get; use axum::{Json, Router}; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::metrics::MetricsRegistry; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::metrics::MetricsRegistry; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{ClientConfig, GetOutcome, PaxosKvClient, ReadMode}; const STORE_ID: u64 = 1; const GROUP_ID: u64 = 1; @@ -32,8 +32,8 @@ async fn concurrent_watch_subscriptions_keep_independent_push_handlers() { } store.start().await.unwrap(); let seed = spawn_topology_server(Arc::clone(&store)).await; - let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![seed]))); - let watches = crowdb_kv_client::WatchNotifyClient::from_shared(Arc::clone(&client)); + let client = Arc::new(PaxosKvClient::new(ClientConfig::new(vec![seed]))); + let watches = crowdb_paxos_kv_client::WatchNotifyClient::from_shared(Arc::clone(&client)); let mut first = watches.subscribe(STORE_ID, 1, b"catalog/").unwrap(); let mut second = watches.subscribe(STORE_ID, 2, b"grant/").unwrap(); tokio::time::timeout(std::time::Duration::from_secs(3), async { @@ -69,7 +69,7 @@ async fn concurrent_watch_subscriptions_keep_independent_push_handlers() { } /// Starts a single-node, single-replica group already believing itself -/// leader (mirrors `crowdb_kv/tests/testkit/cluster.rs::start_cluster_inner` +/// leader (mirrors `crowdb_paxos_kv/tests/testkit/cluster.rs::start_cluster_inner` /// for a 1-node cluster — no election driver needed). async fn start_single_node_store() -> Arc<PxKvStore> { let replica = PxLocalReplica::new(STORE_ID, PxLocalReplicaRole::Leader); @@ -106,7 +106,7 @@ async fn reverse_scan_paginates_without_repeating_boundaries() { store.add_group(PxGroup::new(GROUP_ID, replica)); store.start().await.expect("start store"); let seed = spawn_topology_server(store.clone()).await; - let client = CrowdbKvClient::new(ClientConfig::new(vec![seed])); + let client = PaxosKvClient::new(ClientConfig::new(vec![seed])); for i in 0..8 { let key = format!("r:{i}"); @@ -148,7 +148,7 @@ async fn reverse_scan_paginates_without_repeating_boundaries() { async fn read_your_writes_uses_auto_tracked_watermark() { let store = start_single_node_store().await; let seed = spawn_topology_server(store.clone()).await; - let client = CrowdbKvClient::new(ClientConfig::new(vec![seed])); + let client = PaxosKvClient::new(ClientConfig::new(vec![seed])); assert_eq!(client.read_your_writes_slot(STORE_ID, GROUP_ID), 0); @@ -194,7 +194,7 @@ async fn linearizable_multi_page_scan_pays_barrier_once() { server.start().await.expect("failed to start KvStore"); let seed = spawn_topology_server(server.clone()).await; - let client = CrowdbKvClient::new(ClientConfig::new(vec![seed])); + let client = PaxosKvClient::new(ClientConfig::new(vec![seed])); // Write 12 keys: "k00".."k11", each with a 20-byte value. for i in 0..12u32 { diff --git a/lib/crowdb-kv-client/tests/ffi_exclusion_test.rs b/lib/crowdb-paxos-kv-client/tests/ffi_exclusion_test.rs similarity index 93% rename from lib/crowdb-kv-client/tests/ffi_exclusion_test.rs rename to lib/crowdb-paxos-kv-client/tests/ffi_exclusion_test.rs index bebec3f5c..0b1292d99 100644 --- a/lib/crowdb-kv-client/tests/ffi_exclusion_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/ffi_exclusion_test.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. #![cfg(feature = "ffi")] -use crowdb_kv_client::ffi::parse_diskio_exclusions_for_tests as parse; +use crowdb_paxos_kv_client::ffi::parse_diskio_exclusions_for_tests as parse; #[test] fn exclusion_wire_keeps_every_identity_bit_and_rejects_invalid_batches() { diff --git a/lib/crowdb-kv-client/tests/group_identity_test.rs b/lib/crowdb-paxos-kv-client/tests/group_identity_test.rs similarity index 100% rename from lib/crowdb-kv-client/tests/group_identity_test.rs rename to lib/crowdb-paxos-kv-client/tests/group_identity_test.rs diff --git a/lib/crowdb-kv-client/tests/journal_scan_test.rs b/lib/crowdb-paxos-kv-client/tests/journal_scan_test.rs similarity index 88% rename from lib/crowdb-kv-client/tests/journal_scan_test.rs rename to lib/crowdb-paxos-kv-client/tests/journal_scan_test.rs index d7a4c8ed1..d55ed07f4 100644 --- a/lib/crowdb-kv-client/tests/journal_scan_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/journal_scan_test.rs @@ -4,8 +4,8 @@ #[path = "common/owner_fence_server.rs"] mod owner_fence_server; -use crowdb_kv::paxos::roles::Acceptor; -use crowdb_kv_client::{Error, KvRpcTransport, ReadMode}; +use crowdb_paxos_kv::paxos::roles::Acceptor; +use crowdb_paxos_kv_client::{Error, PaxosKvRpcTransport, ReadMode}; use owner_fence_server::TestOwnerFenceServer; use std::time::Duration; @@ -14,7 +14,7 @@ async fn trimmed_journal_returns_a_typed_gap_without_transport_retries() { let test = TestOwnerFenceServer::start().await; test.client.put(1, 1, b"retained", b"value", None).await.unwrap(); test.store.get_group(1).unwrap().local_replica().acceptor.trim(2); - let raw = KvRpcTransport::new() + let raw = PaxosKvRpcTransport::new() .send_journal_scan(&test.endpoint, 1, 0, b"", 0, 1, 1, 1, ReadMode::Linearizable) .await; assert!(matches!(raw, Err(Error::JournalScanGcGap)), "{raw:?}"); diff --git a/lib/crowdb-kv-client/tests/metrics_lock_test.rs b/lib/crowdb-paxos-kv-client/tests/metrics_lock_test.rs similarity index 94% rename from lib/crowdb-kv-client/tests/metrics_lock_test.rs rename to lib/crowdb-paxos-kv-client/tests/metrics_lock_test.rs index 5a9baf925..af789d8f3 100644 --- a/lib/crowdb-kv-client/tests/metrics_lock_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/metrics_lock_test.rs @@ -3,7 +3,7 @@ use std::sync::Arc; -use crowdb_kv_client::ClientMetrics; +use crowdb_paxos_kv_client::ClientMetrics; #[test] fn concurrent_latency_shards_preserve_every_sample() { diff --git a/lib/crowdb-kv-client/tests/multi_seed_discovery_test.rs b/lib/crowdb-paxos-kv-client/tests/multi_seed_discovery_test.rs similarity index 90% rename from lib/crowdb-kv-client/tests/multi_seed_discovery_test.rs rename to lib/crowdb-paxos-kv-client/tests/multi_seed_discovery_test.rs index 9e8c28909..5caaa9250 100644 --- a/lib/crowdb-kv-client/tests/multi_seed_discovery_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/multi_seed_discovery_test.rs @@ -2,10 +2,10 @@ // Licensed under the Apache License, Version 2.0. use axum::{extract::State, routing::get, Json, Router}; -use crowdb_kv::cluster::{ - group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +use crowdb_paxos_kv::cluster::{ + group::PxGroup, paxos_kv_server::PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use std::sync::Arc; async fn seed(store: Arc<PxKvStore>) -> (String, tokio::task::JoinHandle<()>) { @@ -35,7 +35,7 @@ async fn group_missing_on_first_seed_is_discovered_on_another_member() { } let (first, first_task) = seed(stores[0].clone()).await; let (second, second_task) = seed(stores[1].clone()).await; - let client = CrowdbKvClient::new(ClientConfig::new(vec![first, second])); + let client = PaxosKvClient::new(ClientConfig::new(vec![first, second])); client.put(11, 1, b"first", b"member-one", None).await.unwrap(); let written = client.put(11, 2, b"overlap", b"member-two", None).await; first_task.abort(); @@ -80,7 +80,7 @@ async fn an_unverified_empty_leader_route_does_not_hide_another_members_local_li ); let first_task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); let (second, second_task) = seed(store.clone()).await; - let client = CrowdbKvClient::new(ClientConfig::new(vec![first, second])); + let client = PaxosKvClient::new(ClientConfig::new(vec![first, second])); let written = tokio::time::timeout( std::time::Duration::from_secs(2), client.put(11, 2, b"verified", b"leader", None), diff --git a/lib/crowdb-kv-client/tests/point_read_isolation_test.rs b/lib/crowdb-paxos-kv-client/tests/point_read_isolation_test.rs similarity index 86% rename from lib/crowdb-kv-client/tests/point_read_isolation_test.rs rename to lib/crowdb-paxos-kv-client/tests/point_read_isolation_test.rs index 4dcf9f5c8..5546005fd 100644 --- a/lib/crowdb-kv-client/tests/point_read_isolation_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/point_read_isolation_test.rs @@ -1,8 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::rpc::{KvErrorCode, KvResponse}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, Error, GetOutcome, KvRpcTransport, ReadMode, Result}; +use crowdb_paxos_kv::rpc::{KvErrorCode, KvResponse}; +use crowdb_paxos_kv_client::{ + ClientConfig, Error, GetOutcome, PaxosKvClient, PaxosKvRpcTransport, ReadMode, Result, +}; use std::{ sync::{atomic::Ordering, Arc}, time::Duration, @@ -15,7 +17,7 @@ use point_read_peer::{TestPointReadPeer, TestPointReply}; const BUDGET: Duration = Duration::from_millis(750); -async fn scan_value(transport: &KvRpcTransport, endpoint: &str) { +async fn scan_value(transport: &PaxosKvRpcTransport, endpoint: &str) { let page = transport .send_scan( endpoint, @@ -40,7 +42,7 @@ async fn scan_value(transport: &KvRpcTransport, endpoint: &str) { assert_eq!(page.items[0].value.len(), 65536); } -fn start_scan(transport: Arc<KvRpcTransport>, endpoint: String) -> JoinHandle<()> { +fn start_scan(transport: Arc<PaxosKvRpcTransport>, endpoint: String) -> JoinHandle<()> { tokio::spawn(async move { scan_value(&transport, &endpoint).await }) } @@ -49,7 +51,7 @@ async fn finish_scan(peer: &TestPointReadPeer, scan: JoinHandle<()>) { tokio::time::timeout(BUDGET, scan).await.unwrap().unwrap(); } -async fn point(transport: &KvRpcTransport, endpoint: &str) -> Result<KvResponse> { +async fn point(transport: &PaxosKvRpcTransport, endpoint: &str) -> Result<KvResponse> { transport .send_get(endpoint, b"/chunk/point", 2, 0, 1, ReadMode::Linearizable, 0) .await @@ -58,7 +60,7 @@ async fn point(transport: &KvRpcTransport, endpoint: &str) -> Result<KvResponse> #[tokio::test] async fn point_read_finishes_while_a_scan_frame_is_incomplete() { let peer = TestPointReadPeer::start(TestPointReply::Ready).await; - let client = Arc::new(CrowdbKvClient::new_with_isolated_point_reads(ClientConfig::new( + let client = Arc::new(PaxosKvClient::new_with_isolated_point_reads(ClientConfig::new( vec![], ))); client.seed_leader(0, 1, peer.endpoint.clone()); @@ -105,7 +107,7 @@ async fn point_read_finishes_while_a_scan_frame_is_incomplete() { #[tokio::test] async fn ordinary_transport_keeps_the_shared_pool_without_opt_in() { let peer = TestPointReadPeer::start(TestPointReply::Ready).await; - let transport = Arc::new(KvRpcTransport::new()); + let transport = Arc::new(PaxosKvRpcTransport::new()); let scan = start_scan(transport.clone(), peer.endpoint.clone()); tokio::time::timeout(BUDGET, peer.state.bulk_started.notified()) .await @@ -123,7 +125,7 @@ async fn ordinary_transport_keeps_the_shared_pool_without_opt_in() { #[tokio::test] async fn isolated_point_reads_preserve_rejection() { let peer = TestPointReadPeer::start(TestPointReply::Rejected).await; - let transport = KvRpcTransport::new().with_isolated_point_reads(); + let transport = PaxosKvRpcTransport::new().with_isolated_point_reads(); let result = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)) .await .unwrap() @@ -137,7 +139,7 @@ async fn isolated_point_reads_preserve_rejection() { #[tokio::test] async fn isolated_point_reads_keep_the_original_deadline() { let peer = TestPointReadPeer::start(TestPointReply::Held).await; - let transport = KvRpcTransport::new().with_isolated_point_reads(); + let transport = PaxosKvRpcTransport::new().with_isolated_point_reads(); assert!(tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)) .await .is_err()); @@ -147,7 +149,7 @@ async fn isolated_point_reads_keep_the_original_deadline() { #[tokio::test] async fn failed_point_generation_does_not_evict_a_healthy_scan_pool() { let peer = TestPointReadPeer::start(TestPointReply::ClosedOnce).await; - let transport = Arc::new(KvRpcTransport::new().with_isolated_point_reads()); + let transport = Arc::new(PaxosKvRpcTransport::new().with_isolated_point_reads()); let scan = start_scan(transport.clone(), peer.endpoint.clone()); tokio::time::timeout(BUDGET, peer.state.bulk_started.notified()) .await diff --git a/lib/crowdb-kv-client/tests/storage_admission_test.rs b/lib/crowdb-paxos-kv-client/tests/storage_admission_test.rs similarity index 99% rename from lib/crowdb-kv-client/tests/storage_admission_test.rs rename to lib/crowdb-paxos-kv-client/tests/storage_admission_test.rs index 85d325633..01b52f486 100644 --- a/lib/crowdb-kv-client/tests/storage_admission_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/storage_admission_test.rs @@ -312,7 +312,7 @@ async fn duplicate_checks_cover_every_inventory_page() { retained.backing_identity = format!("file:10:{id}"); retained.geometry.device_path = format!("/opt/crowdb/data/disks/{id}.img"); retained.removed = true; - operations.push(crowdb_kv_client::BatchOp::Put { + operations.push(crowdb_paxos_kv_client::BatchOp::Put { key: retained.key().into_bytes().into(), value: serde_json::to_vec(&retained).unwrap().into(), }); diff --git a/lib/crowdb-kv-client/tests/storage_compatibility_test.rs b/lib/crowdb-paxos-kv-client/tests/storage_compatibility_test.rs similarity index 99% rename from lib/crowdb-kv-client/tests/storage_compatibility_test.rs rename to lib/crowdb-paxos-kv-client/tests/storage_compatibility_test.rs index b2af2c5db..f2060c887 100644 --- a/lib/crowdb-kv-client/tests/storage_compatibility_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/storage_compatibility_test.rs @@ -90,7 +90,7 @@ async fn backend_does_not_split_compatible_media_but_legacy_type_is_not_unknown( #[tokio::test] async fn owner_status_write_cannot_resurrect_concurrently_removed_membership() { - use crowdb_kv_client::{GetOutcome, ReadMode}; + use crowdb_paxos_kv_client::{GetOutcome, ReadMode}; use crowdb_protocol::key::{OwnerMapKey, TextKey}; let fixture = TestStorage::start().await; let hw = &fixture.hardware; diff --git a/lib/crowdb-kv-client/tests/storage_exclusion_test.rs b/lib/crowdb-paxos-kv-client/tests/storage_exclusion_test.rs similarity index 98% rename from lib/crowdb-kv-client/tests/storage_exclusion_test.rs rename to lib/crowdb-paxos-kv-client/tests/storage_exclusion_test.rs index 08568c263..9caefec20 100644 --- a/lib/crowdb-kv-client/tests/storage_exclusion_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/storage_exclusion_test.rs @@ -3,7 +3,7 @@ #[path = "common/storage.rs"] mod storage; -use crowdb_kv_client::{GetOutcome, ReadMode, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode, ServiceRegistryClient}; use crowdb_protocol::{ common::{HwStatus, ServiceExtra}, disk_storage::{DiskExclusionRecord, DiskExclusionReporter}, @@ -11,7 +11,7 @@ use crowdb_protocol::{ }; use storage::{key, record, TestStorage}; -async fn revision(hw: &crowdb_kv_client::HardwareClient, path: &str) -> u64 { +async fn revision(hw: &crowdb_paxos_kv_client::HardwareClient, path: &str) -> u64 { let GetOutcome::Found { revision, .. } = hw .kv() .get(0, 0, path.as_bytes(), ReadMode::Linearizable, None) diff --git a/lib/crowdb-kv-client/tests/storage_health_test.rs b/lib/crowdb-paxos-kv-client/tests/storage_health_test.rs similarity index 100% rename from lib/crowdb-kv-client/tests/storage_health_test.rs rename to lib/crowdb-paxos-kv-client/tests/storage_health_test.rs diff --git a/lib/crowdb-kv-client/tests/storage_rejoin_test.rs b/lib/crowdb-paxos-kv-client/tests/storage_rejoin_test.rs similarity index 98% rename from lib/crowdb-kv-client/tests/storage_rejoin_test.rs rename to lib/crowdb-paxos-kv-client/tests/storage_rejoin_test.rs index 9be272971..b392ff33b 100644 --- a/lib/crowdb-kv-client/tests/storage_rejoin_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/storage_rejoin_test.rs @@ -3,7 +3,7 @@ #[path = "common/storage.rs"] mod storage; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::{ common::{DiskioHardwareScope, HwStatus}, disk_storage::{DiskExclusionRecord, DiskExclusionReporter, DiskRemovalRecord}, diff --git a/lib/crowdb-kv-client/tests/storage_removal_test.rs b/lib/crowdb-paxos-kv-client/tests/storage_removal_test.rs similarity index 98% rename from lib/crowdb-kv-client/tests/storage_removal_test.rs rename to lib/crowdb-paxos-kv-client/tests/storage_removal_test.rs index d41025be4..10c63ffd3 100644 --- a/lib/crowdb-kv-client/tests/storage_removal_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/storage_removal_test.rs @@ -3,7 +3,7 @@ #[path = "common/storage.rs"] mod storage; -use crowdb_kv_client::{GetOutcome, HardwareClient, ReadMode, ServiceRegistryClient}; +use crowdb_paxos_kv_client::{GetOutcome, HardwareClient, ReadMode, ServiceRegistryClient}; use crowdb_protocol::{ common::{ChunkId, DiskioHardwareScope, HwStatus, InstanceValue}, disk_storage::{DiskExclusionRecord, DiskExclusionReporter, DiskRemovalRecord}, diff --git a/lib/crowdb-kv-client/tests/stream_process_authority_test.rs b/lib/crowdb-paxos-kv-client/tests/stream_process_authority_test.rs similarity index 98% rename from lib/crowdb-kv-client/tests/stream_process_authority_test.rs rename to lib/crowdb-paxos-kv-client/tests/stream_process_authority_test.rs index 0b8a52b83..a9081c3f3 100644 --- a/lib/crowdb-kv-client/tests/stream_process_authority_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/stream_process_authority_test.rs @@ -4,7 +4,7 @@ #[path = "common/stream_lease_clock.rs"] mod fixture; -use crowdb_kv_client::{StreamProcessAuthority, StreamProcessLeaseError, StreamProcessLeaseTiming}; +use crowdb_paxos_kv_client::{StreamProcessAuthority, StreamProcessLeaseError, StreamProcessLeaseTiming}; use fixture::TestStreamLeaseRecord as Test; use std::time::{Duration, Instant}; diff --git a/lib/crowdb-paxos-kv-client/tests/stream_process_exclusion_test.rs b/lib/crowdb-paxos-kv-client/tests/stream_process_exclusion_test.rs new file mode 100644 index 000000000..3bbc6d619 --- /dev/null +++ b/lib/crowdb-paxos-kv-client/tests/stream_process_exclusion_test.rs @@ -0,0 +1,141 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#[path = "common/stream_lease.rs"] +mod fixture; + +use crowdb_paxos_kv_client::{ + GetOutcome, ReadMode, StreamProcessAuthority, StreamProcessLeaseClient, StreamProcessLeaseError, + StreamProcessLeaseTiming, +}; +use crowdb_protocol::{ + chunk_stream::StreamPurpose, + key::{StreamProcessLeaseKey, TextKey}, + timing::{MAX_WRITE_CLOCK_SKEW_MS, MAX_WRITE_REQUEST_AGE_MS}, +}; +use fixture::TestStreamLeaseKv; + +#[tokio::test] +async fn confirmed_exclusion_fences_paused_cas_stale_renewal_and_same_boot_retry() { + let fixture = TestStreamLeaseKv::start().await; + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let first = client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_000, 100_000) + .await + .unwrap(); + let owner = StreamProcessAuthority::new( + first.clone(), + StreamProcessLeaseTiming::default(), + 1_000, + std::time::Instant::now(), + ) + .unwrap(); + let context = owner.capture(std::time::Instant::now()).unwrap(); + let key = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id: 7, + } + .to_path(); + let GetOutcome::Found { revision, .. } = fixture + .kv + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!("lease is absent"); + }; + let mut paused = first.clone(); + paused.expires_at_ms += 100_000; + let safe = first.expires_at_ms + MAX_WRITE_CLOCK_SKEW_MS + MAX_WRITE_REQUEST_AGE_MS; + assert_eq!( + client.confirm_exclusion(first.identity, safe - 1).await.unwrap(), + None + ); + assert_eq!( + client.confirm_exclusion(first.identity, safe).await.unwrap(), + Some(safe) + ); + assert!(fixture + .kv + .put_cas( + 0, + 0, + key.as_bytes(), + &serde_json::to_vec(&paused).unwrap(), + revision + ) + .await + .is_err()); + assert!(matches!( + client.renew(first.identity, 1_001, 100_000).await, + Err(StreamProcessLeaseError::Expired) + )); + assert!(matches!( + client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_001, 100_000) + .await, + Err(StreamProcessLeaseError::Expired) + )); + let excluded = client.load(StreamPurpose::Wal, 7).await.unwrap().unwrap(); + assert!(excluded.revoked); + assert!(owner.observe(excluded, 1_001, std::time::Instant::now()).is_err()); + assert!(owner + .validate_context(&context, std::time::Instant::now()) + .is_err()); + assert_eq!( + client.confirm_exclusion(first.identity, safe).await.unwrap(), + Some(safe) + ); + let next = client + .claim( + StreamPurpose::Wal, + 7, + [2; 16], + Some(first.identity), + safe, + 100_000, + ) + .await + .unwrap(); + assert!(!next.revoked); + assert_eq!(next.identity.process_epoch, 2); + assert_eq!(next.prior_exclusion_deadline_ms, first.expires_at_ms); + assert_eq!( + client.confirm_exclusion(first.identity, safe).await.unwrap(), + Some(safe) + ); +} + +#[tokio::test] +async fn renewal_that_wins_authority_moves_the_exclusion_boundary() { + let fixture = TestStreamLeaseKv::start().await; + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let first = client + .claim(StreamPurpose::Stream, 8, [1; 16], None, 1_000, 1_000) + .await + .unwrap(); + let renewed = client.renew(first.identity, 1_001, 100_000).await.unwrap(); + let margin = MAX_WRITE_CLOCK_SKEW_MS + MAX_WRITE_REQUEST_AGE_MS; + assert_eq!( + client + .confirm_exclusion(first.identity, first.expires_at_ms + margin) + .await + .unwrap(), + None + ); + assert!( + !client + .load(StreamPurpose::Stream, 8) + .await + .unwrap() + .unwrap() + .revoked + ); + assert_eq!( + client + .confirm_exclusion(first.identity, renewed.expires_at_ms + margin) + .await + .unwrap(), + Some(renewed.expires_at_ms + margin) + ); +} diff --git a/lib/crowdb-kv-client/tests/stream_process_lease_test.rs b/lib/crowdb-paxos-kv-client/tests/stream_process_lease_test.rs similarity index 99% rename from lib/crowdb-kv-client/tests/stream_process_lease_test.rs rename to lib/crowdb-paxos-kv-client/tests/stream_process_lease_test.rs index 927583235..ca8c1030c 100644 --- a/lib/crowdb-kv-client/tests/stream_process_lease_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/stream_process_lease_test.rs @@ -4,7 +4,7 @@ #[path = "common/stream_lease.rs"] mod fixture; -use crowdb_kv_client::{ +use crowdb_paxos_kv_client::{ StreamProcessAuthority, StreamProcessLeaseClient, StreamProcessLeaseError, StreamProcessLeaseTiming, }; use crowdb_protocol::{ diff --git a/lib/crowdb-kv-client/tests/stream_process_observation_test.rs b/lib/crowdb-paxos-kv-client/tests/stream_process_observation_test.rs similarity index 98% rename from lib/crowdb-kv-client/tests/stream_process_observation_test.rs rename to lib/crowdb-paxos-kv-client/tests/stream_process_observation_test.rs index 61fbde58c..fbb5a21b0 100644 --- a/lib/crowdb-kv-client/tests/stream_process_observation_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/stream_process_observation_test.rs @@ -4,7 +4,7 @@ #[path = "common/stream_lease.rs"] mod fixture; -use crowdb_kv_client::{GetOutcome, ReadMode, StreamProcessLeaseClient, StreamProcessLeaseError}; +use crowdb_paxos_kv_client::{GetOutcome, ReadMode, StreamProcessLeaseClient, StreamProcessLeaseError}; use crowdb_protocol::{chunk_stream::StreamPurpose, common::ChunkKvExtra, key::InstanceKey}; use fixture::TestStreamLeaseKv; diff --git a/lib/crowdb-kv-client/tests/sysmd_client_test.rs b/lib/crowdb-paxos-kv-client/tests/sysmd_client_test.rs similarity index 76% rename from lib/crowdb-kv-client/tests/sysmd_client_test.rs rename to lib/crowdb-paxos-kv-client/tests/sysmd_client_test.rs index 493e4f95f..516b64856 100644 --- a/lib/crowdb-kv-client/tests/sysmd_client_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/sysmd_client_test.rs @@ -4,15 +4,15 @@ //! Construction tests for [`CrowdbSysmdClient`]. Delegation is trivial //! (each method forwards to the wrapped sub-client), so we only verify //! that the facade builds from both `new` and `from_shared` and that -//! all three sub-clients share the same underlying `Arc<CrowdbKvClient>`. +//! all three sub-clients share the same underlying `Arc<PaxosKvClient>`. use std::sync::Arc; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, CrowdbSysmdClient}; +use crowdb_paxos_kv_client::{ClientConfig, CrowdbSysmdClient, PaxosKvClient}; #[test] fn sysmd_client_new_shares_single_arc() { - let kv = CrowdbKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:1".into()])); + let kv = PaxosKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:1".into()])); let sysmd = CrowdbSysmdClient::new(kv); // All sub-clients wrap the same Arc; kv() returns a reference to // the shared inner client. @@ -26,7 +26,7 @@ fn sysmd_client_new_shares_single_arc() { #[test] fn sysmd_client_from_shared_preserves_arc() { - let kv = CrowdbKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:1".into()])); + let kv = PaxosKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:1".into()])); let shared = Arc::new(kv); let original = Arc::clone(&shared); let sysmd = CrowdbSysmdClient::from_shared(shared); @@ -37,7 +37,7 @@ fn sysmd_client_from_shared_preserves_arc() { #[test] fn sysmd_client_is_clone() { - let kv = CrowdbKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:1".into()])); + let kv = PaxosKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:1".into()])); let sysmd = CrowdbSysmdClient::new(kv); let cloned = sysmd.clone(); // Clone shares the same underlying Arc. diff --git a/lib/crowdb-kv-client/tests/topology_publication_test.rs b/lib/crowdb-paxos-kv-client/tests/topology_publication_test.rs similarity index 93% rename from lib/crowdb-kv-client/tests/topology_publication_test.rs rename to lib/crowdb-paxos-kv-client/tests/topology_publication_test.rs index 058fd3ae6..da3c3e62d 100644 --- a/lib/crowdb-kv-client/tests/topology_publication_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/topology_publication_test.rs @@ -2,13 +2,13 @@ // Licensed under the Apache License, Version 2.0. use axum::{routing::get, Json, Router}; -use crowdb_kv::cluster::{ +use crowdb_paxos_kv::cluster::{ group::PxGroup, - kv_server::KvServer, local_replica::{PxLocalReplica, PxLocalReplicaRole}, + paxos_kv_server::PaxosKvServer, px_kv_store::PxKvStore, }; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::mgmt::{GroupStatus, ReplicaStatus, StoreStatus, TopologyResponse}; use std::sync::Arc; use tokio::sync::Notify; @@ -60,7 +60,7 @@ async fn repeated_identical_hint_does_not_discard_an_inflight_topology_refresh() let management = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); let mut config = ClientConfig::new(vec![seed]); config.retry.max_retries = 0; - let client = Arc::new(CrowdbKvClient::new(config)); + let client = Arc::new(PaxosKvClient::new(config)); client.seed_leader(1, 1, "127.0.0.1:1".into()); let refreshing = client.clone(); let refresh = tokio::spawn(async move { refreshing.refresh_topology().await }); diff --git a/lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs b/lib/crowdb-paxos-kv-client/tests/topology_retry_discovery_test.rs similarity index 97% rename from lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs rename to lib/crowdb-paxos-kv-client/tests/topology_retry_discovery_test.rs index 332da0162..7c199f531 100644 --- a/lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs +++ b/lib/crowdb-paxos-kv-client/tests/topology_retry_discovery_test.rs @@ -4,7 +4,7 @@ #[path = "common/topology_recovery.rs"] mod topology_recovery; -use crowdb_kv_client::{Error, GetOutcome, ReadMode}; +use crowdb_paxos_kv_client::{Error, GetOutcome, ReadMode}; use std::time::Duration; use topology_recovery::TestDiscovery; diff --git a/lib/crowdb-kv/Cargo.toml b/lib/crowdb-paxos-kv/Cargo.toml similarity index 81% rename from lib/crowdb-kv/Cargo.toml rename to lib/crowdb-paxos-kv/Cargo.toml index d6fa5a6ec..31d98b9d7 100644 --- a/lib/crowdb-kv/Cargo.toml +++ b/lib/crowdb-paxos-kv/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "crowdb-kv" +name = "crowdb-paxos-kv" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -51,10 +51,10 @@ criterion = { version = "0.8", features = ["html_reports"] } # Self dev-dependency: turns on the `test-util` feature for this crate's own # test/bench build so integration tests can reach the test-only hooks under # plain `cargo test` (no `--features` needed). -crowdb-kv = { path = ".", features = ["test-util"] } +crowdb-paxos-kv = { path = ".", features = ["test-util"] } # Used by integration-test helpers (`tests/common/test_client.rs`) to build -# the crowdb-rpc `KvRpcTransport`/`PxRpcTransport` facades. -crowdb-kv-client = { path = "../crowdb-kv-client" } +# the crowdb-rpc `PaxosKvRpcTransport`/`PxRpcTransport` facades. +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../crowdb-test-harness" } @@ -74,10 +74,26 @@ path = "tests/group_test.rs" name = "rpc_migration_test" path = "tests/rpc_migration_test.rs" +[[test]] +name = "rpc_notification_test" +path = "tests/rpc_notification_test.rs" + [[test]] name = "kv_test" path = "tests/kv_test.rs" +[[test]] +name = "tree_idle_writeback_test" +path = "tests/tree_idle_writeback_test.rs" + +[[test]] +name = "tree_maintenance_test" +path = "tests/tree_maintenance_test.rs" + +[[test]] +name = "tree_snapshot_owner_test" +path = "tests/tree_snapshot_owner_test.rs" + [[test]] name = "metrics_test" path = "tests/metrics_test.rs" @@ -117,3 +133,11 @@ path = "tests/replay_compaction_memory_test.rs" [[bench]] name = "wal" harness = false + +[[test]] +name = "tree_admission_test" +path = "tests/tree_admission_test.rs" + +[[test]] +name = "tree_native_admission_test" +path = "tests/tree_native_admission_test.rs" diff --git a/lib/crowdb-kv/benches/slot_list.rs b/lib/crowdb-paxos-kv/benches/slot_list.rs similarity index 98% rename from lib/crowdb-kv/benches/slot_list.rs rename to lib/crowdb-paxos-kv/benches/slot_list.rs index 27a82d2c9..7878c3626 100644 --- a/lib/crowdb-kv/benches/slot_list.rs +++ b/lib/crowdb-paxos-kv/benches/slot_list.rs @@ -5,9 +5,9 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use crossbeam_skiplist::SkipMap; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::paxos::slot_list::PxSlotList; -use crowdb_kv::paxos::slot_node::PxSlotNode; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::paxos::slot_list::PxSlotList; +use crowdb_paxos_kv::paxos::slot_node::PxSlotNode; use dashmap::DashMap; use std::collections::BTreeMap; use std::ptr::null_mut; diff --git a/lib/crowdb-kv/benches/wal.rs b/lib/crowdb-paxos-kv/benches/wal.rs similarity index 96% rename from lib/crowdb-kv/benches/wal.rs rename to lib/crowdb-paxos-kv/benches/wal.rs index 6b79f0333..eff19ef3d 100644 --- a/lib/crowdb-kv/benches/wal.rs +++ b/lib/crowdb-paxos-kv/benches/wal.rs @@ -25,10 +25,10 @@ use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::wal_engine::WalEngine; -use crowdb_kv::wal::{BlockDevice, IoBackend, MemBlockDevice, WalConfig}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{BlockDevice, IoBackend, MemBlockDevice, WalConfig}; // ---------- helpers ---------- @@ -284,12 +284,12 @@ fn main() { // When this bench is run via `cargo test --all-targets`, skip the heavy // timed workload. Criterion-based benches do this automatically, but this // custom bench needs an explicit guard to avoid blocking commits. - let force_run = std::env::var("CROWDB_KV_RUN_WAL_BENCH") + let force_run = std::env::var("CROWDB_PAXOS_KV_RUN_WAL_BENCH") .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) .unwrap_or(false); if cfg!(test) && !force_run { eprintln!( - "Skipping wal bench in cargo test mode; set CROWDB_KV_RUN_WAL_BENCH=1 to run real results." + "Skipping wal bench in cargo test mode; set CROWDB_PAXOS_KV_RUN_WAL_BENCH=1 to run real results." ); return; } diff --git a/lib/crowdb-kv/benches/wal_bench_history.md b/lib/crowdb-paxos-kv/benches/wal_bench_history.md similarity index 100% rename from lib/crowdb-kv/benches/wal_bench_history.md rename to lib/crowdb-paxos-kv/benches/wal_bench_history.md diff --git a/lib/crowdb-kv/build.rs b/lib/crowdb-paxos-kv/build.rs similarity index 100% rename from lib/crowdb-kv/build.rs rename to lib/crowdb-paxos-kv/build.rs diff --git a/lib/crowdb-kv/src/cluster.rs b/lib/crowdb-paxos-kv/src/cluster.rs similarity index 92% rename from lib/crowdb-kv/src/cluster.rs rename to lib/crowdb-paxos-kv/src/cluster.rs index b596d3b10..be28c4286 100644 --- a/lib/crowdb-kv/src/cluster.rs +++ b/lib/crowdb-paxos-kv/src/cluster.rs @@ -21,30 +21,33 @@ pub mod group_prepare; pub mod group_propose; mod group_retention; mod group_routes; +mod group_shutdown; #[cfg(feature = "test-util")] mod group_tests; -pub mod kv_server; pub mod kv_store; pub mod local_replica; pub mod local_replica_accept; pub mod local_replica_apply; pub mod local_replica_election; pub mod local_replica_replay; +mod local_replica_shutdown; pub mod node_config; +pub mod paxos_kv_server; pub mod px_kv_store; mod px_kv_store_handler; mod px_kv_store_owner; pub mod remote_replica; pub mod replica; pub mod status; +mod store_shutdown; pub mod watch_registry; pub use group::ProposeResult; pub use group_config::{GroupConfigStore, PxGroupConfig, PxGroupMember}; pub use node_config::{NodeConfig, NodeConfigStore, NodeGroupEntry, NodeStoreEntry}; -pub use kv_server::*; pub use local_replica::*; +pub use paxos_kv_server::*; pub use px_kv_store::*; pub use remote_replica::PxRemoteReplica; pub use replica::{Replica, ReplicaClient, ReplicaHandler}; diff --git a/lib/crowdb-kv/src/cluster/group.rs b/lib/crowdb-paxos-kv/src/cluster/group.rs similarity index 97% rename from lib/crowdb-kv/src/cluster/group.rs rename to lib/crowdb-paxos-kv/src/cluster/group.rs index 3bd786527..8c9c02312 100644 --- a/lib/crowdb-kv/src/cluster/group.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group.rs @@ -25,7 +25,7 @@ use crate::cluster::local_replica::PxLocalReplica; use crate::cluster::node_config::NodeConfigStore; use crate::cluster::remote_replica::PxRemoteReplica; use crate::cluster::replica::Replica; -use crate::common::config::{CrowDBConfig, PaxosConfig}; +use crate::common::config::{PaxosConfig, PaxosKvConfig}; use crate::metrics::{Counter, Gauge, LatencySummary}; use crate::paxos::roles::{Acceptor, PxBallot, PxLogEntry, RequestIdentity, SlotIndex}; use crate::paxos::{PxGroupId, PxNodeId}; @@ -122,7 +122,7 @@ pub struct PxGroup { /// `async_engine_apply` bool fields and `election_cfg`. Set wholesale /// via [`Self::set_from_config`]; individual setters delegate into /// `self.config.*` so the held config stays the source of truth. - pub(crate) config: CrowDBConfig, + pub(crate) config: PaxosKvConfig, /// Per-leadership-tenure [`CancellationToken`]. Cancelled in /// [`Self::shutdown`] and by every step-down trigger. The bulk-Phase-1 /// sweep and the election driver both honor it. @@ -199,10 +199,9 @@ pub struct PxGroup { pub(crate) group_snapshot_slot: AtomicU64, /// Membership-fenced certificate received from the accepted leader. pub(super) leader_gc_certificate: arc_swap::ArcSwapOption<super::group_retention::ReclamationCertificate>, - /// In-flight proposal admission gate. Holds N semaphores (one per - /// queue), each sized to `ceil(max_inflight / N)` permits. Each - /// in-flight `propose` call holds one permit for its duration. - /// Depending on `policy`, a full queue either fails fast with + /// Proposal admission gate. Each proposal holds one semaphore permit + /// through consensus and completion of its admitted native apply. + /// Depending on the policy, a full window either fails fast with /// `ProposeResult::Busy` (Reject) or blocks on `acquire().await` /// (Queue). pub(crate) inflight: InflightAdmission, @@ -285,11 +284,8 @@ pub struct PxGroup { #[cfg(feature = "test-util")] pub(crate) repair_gate: std::sync::atomic::AtomicU8, /// R36 self-weak back-reference, set once the group is wrapped in an - /// [`Arc`] via [`Self::set_self_weak`]. The coalescer's per-batch timer - /// task upgrades this to spawn a flush without the group holding a - /// strong self-reference (which would leak). `None` until set (test - /// groups not wrapped in `Arc`); the coalescer falls back to the - /// direct single-op path when unset. + /// [`Arc`] via [`Self::set_self_weak`]. Coalescer tasks upgrade this only + /// for one operation, so a watchdog cannot keep the group alive. pub(crate) self_weak: OnceLock<Weak<PxGroup>>, /// R45 coalescer state. `None` when idle (no round in flight, no /// batch accumulating); `Some(PendingBatch)` while a Paxos round is @@ -309,9 +305,11 @@ pub struct PxGroup { /// Updated on every enqueue and round completion. The watchdog /// checks this to detect stuck batches. pub(crate) coalesce_last_activity_us: std::sync::atomic::AtomicU64, - /// Single long-running watchdog task handle. Started lazily on - /// first enqueue. - pub(crate) coalesce_watchdog_handle: OnceLock<tokio::task::JoinHandle<()>>, + /// Single long-running watchdog task handle. Started lazily on first + /// enqueue through a lock-free `OnceLock` fast path and taken/joined by + /// group shutdown through the cold-path async mutex. + pub(crate) coalesce_watchdog_handle: + OnceLock<Arc<tokio::sync::Mutex<Option<tokio::task::JoinHandle<()>>>>>, /// R59 snapshot versioning API: per-group snapshot handle registry. /// Each entry is a pinned `SnapshotHandle` with a lease/expiry. Reaped /// lazily on `create`/`list`/`scan` when the lease has elapsed. @@ -389,10 +387,10 @@ impl PxGroup { // Test path: wal_early_ack / async_engine_apply default false // for deterministic synchronous apply; production overwrites via // set_from_config. - config: CrowDBConfig { + config: PaxosKvConfig { wal_early_ack: false, async_engine_apply: false, - ..CrowDBConfig::default() + ..PaxosKvConfig::default() }, tenure_cancel: CancellationToken::new(), driver_handle: AsyncMutex::new(None), @@ -502,12 +500,12 @@ impl PxGroup { self.config.async_engine_apply = enabled; } - /// Apply all tunables from a `CrowDBConfig` wholesale: replaces the + /// Apply all tunables from a `PaxosKvConfig` wholesale: replaces the /// held config, mirrors the election lease onto the local replica, /// and reconstructs the inflight admission gate from the paxos /// params. Replaces the per-field setter calls at group creation /// and rebuild sites. - pub fn set_from_config(&mut self, config: &CrowDBConfig) { + pub fn set_from_config(&mut self, config: &PaxosKvConfig) { self.config = config.clone(); self.set_election_config(config.election); self.set_inflight_config( @@ -522,7 +520,7 @@ impl PxGroup { /// Borrow the unified config held by this group. #[must_use] - pub fn config(&self) -> &CrowDBConfig { + pub fn config(&self) -> &PaxosKvConfig { &self.config } @@ -1037,6 +1035,8 @@ pub enum ProposeResult { }, CasBusy, OutcomeUnknown, + /// Rejected before any slot or proposal was submitted. + Rejected(String), Err(String), } diff --git a/lib/crowdb-kv/src/cluster/group_accept.rs b/lib/crowdb-paxos-kv/src/cluster/group_accept.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_accept.rs rename to lib/crowdb-paxos-kv/src/cluster/group_accept.rs diff --git a/lib/crowdb-kv/src/cluster/group_cas.rs b/lib/crowdb-paxos-kv/src/cluster/group_cas.rs similarity index 97% rename from lib/crowdb-kv/src/cluster/group_cas.rs rename to lib/crowdb-paxos-kv/src/cluster/group_cas.rs index e4f199c9d..af4b9b242 100644 --- a/lib/crowdb-kv/src/cluster/group_cas.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_cas.rs @@ -88,7 +88,7 @@ impl PxGroup { required_term: Option<u64>, ) -> ProposeResult { if client_id == 0 { - return ProposeResult::Err("conditional write requires nonzero client_id".into()); + return ProposeResult::Rejected("conditional write requires nonzero client_id".into()); } if let Some(slot) = self.local_replica.learner.request_result_lookup(client_id, seq) { return ProposeResult::Chosen { slot }; @@ -131,7 +131,7 @@ impl PxGroup { .engine_get_versioned(&precondition_key) .await; let result = match observed { - Err(error) => ProposeResult::Err(error), + Err(error) => ProposeResult::Rejected(error), Ok(value) => { let current_revision = value.as_ref().map_or(0, |(revision, _)| *revision); if current_revision != expected_revision { diff --git a/lib/crowdb-kv/src/cluster/group_coalesce.rs b/lib/crowdb-paxos-kv/src/cluster/group_coalesce.rs similarity index 82% rename from lib/crowdb-kv/src/cluster/group_coalesce.rs rename to lib/crowdb-paxos-kv/src/cluster/group_coalesce.rs index 1cabb03fd..18897bcfc 100644 --- a/lib/crowdb-kv/src/cluster/group_coalesce.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_coalesce.rs @@ -3,7 +3,7 @@ #![allow(clippy::too_many_lines, clippy::cast_possible_truncation)] -use std::sync::Weak; +use std::sync::{Arc, Weak}; use crate::cluster::group::{PendingBatch, ProposeResult, PxGroup}; use crate::paxos::roles::RequestIdentity; @@ -35,20 +35,25 @@ impl PxGroup { /// Called lazily on first enqueue. The watchdog sleeps for /// `WATCHDOG_US`, then flushes any non-empty pending batch. fn coalesce_start_watchdog(&self) { + let Some(weak) = self.self_weak.get().cloned() else { + return; + }; + let g = self.group_id; + let replica = self.local_replica().id; + let span = self.log_store_id().map_or_else( + || info_span!("coalesce_watchdog", g, replica), + |s| info_span!("coalesce_watchdog", s, g, replica), + ); + let cancel = self.tenure_cancel.clone(); self.coalesce_watchdog_handle.get_or_init(|| { - let Some(group) = self.self_weak.get().and_then(Weak::upgrade) else { - return tokio::spawn(async {}); - }; - let g = group.group_id; - let replica = group.local_replica().id; - let span = group.log_store_id().map_or_else( - || info_span!("coalesce_watchdog", g, replica), - |s| info_span!("coalesce_watchdog", s, g, replica), - ); - tokio::spawn( + Arc::new(tokio::sync::Mutex::new(Some(tokio::spawn( async move { loop { - tokio::time::sleep(std::time::Duration::from_micros(WATCHDOG_US)).await; + tokio::select! { + () = cancel.cancelled() => break, + () = tokio::time::sleep(std::time::Duration::from_micros(WATCHDOG_US)) => {} + } + let Some(group) = weak.upgrade() else { break }; // Atomically swap the batch for a fresh empty one so // no op can sneak in and start a 1-op round during // the swap. @@ -67,7 +72,7 @@ impl PxGroup { } } .instrument(span), - ) + )))) }); } @@ -76,6 +81,12 @@ impl PxGroup { /// new ops accumulate for the next round). This function only /// builds the payload and spawns the round task. fn coalesce_spawn_round(&self, batch: PendingBatch) { + if self.tenure_cancel.is_cancelled() || self.inflight.closed() { + for waiter in batch.waiters { + let _ = waiter.send(ProposeResult::Err("coalescer shutting down".to_string())); + } + return; + } let mut payload = Vec::with_capacity(2 + batch.op_bodies.len()); payload.extend_from_slice(&batch.op_count.to_le_bytes()); payload.extend_from_slice(&batch.op_bodies); @@ -83,6 +94,9 @@ impl PxGroup { let identities = batch.identities; let waiters = batch.waiters; let Some(group) = self.self_weak.get().and_then(Weak::upgrade) else { + for waiter in waiters { + let _ = waiter.send(ProposeResult::Err("group dropped".to_string())); + } return; }; group @@ -105,7 +119,7 @@ impl PxGroup { ); } - /// Enqueue one single-key op into the coalescer. + /// Enqueue one atomic request into the coalescer. /// /// - Idle (no batch): start a 1-op round immediately. Open a pending /// batch for concurrent ops. @@ -120,6 +134,9 @@ impl PxGroup { payload: Vec<u8>, identity: Option<RequestIdentity>, ) -> ProposeResult { + if self.tenure_cancel.is_cancelled() { + return ProposeResult::Err("coalescer shutting down".to_string()); + } let (tx, rx) = tokio::sync::oneshot::channel(); let request_op_count = payload .get(..2) @@ -149,6 +166,13 @@ impl PxGroup { let round_identities: Vec<RequestIdentity> = identity.into_iter().collect(); Some((round_payload, round_identities, vec![tx])) } + Some(batch) + if u32::from(batch.op_count) + u32::from(request_op_count) > u32::from(max_keys) => + { + // Keep each caller's batch intact. This request can run + // independently; its completion will drain the pending batch. + Some((payload, identity.into_iter().collect(), vec![tx])) + } Some(batch) => { batch.op_bodies.extend_from_slice(op_body); batch.op_count = batch.op_count.saturating_add(request_op_count); @@ -181,6 +205,14 @@ impl PxGroup { // Start the round. Spawn so the caller is not pinned to the paxos // round and can return via the oneshot. + if self.tenure_cancel.is_cancelled() { + for waiter in round_waiters { + let _ = waiter.send(ProposeResult::Err("coalescer shutting down".to_string())); + } + return rx + .await + .unwrap_or_else(|_| ProposeResult::Err("coalescer round dropped".to_string())); + } let payload = bytes::Bytes::from(payload); let Some(group) = self.self_weak.get().and_then(Weak::upgrade) else { return ProposeResult::Err("group dropped".to_string()); @@ -228,6 +260,14 @@ impl PxGroup { /// would fragment the batch. fn coalesce_drain_after_round(&self) { self.coalesce_touch_activity(); + if self.tenure_cancel.is_cancelled() { + if let Some(batch) = self.coalesce_take_pending() { + for waiter in batch.waiters { + let _ = waiter.send(ProposeResult::Err("coalescer shutting down".to_string())); + } + } + return; + } let threshold = self.config.paxos.coalesce_drain_threshold; if threshold > 0 && self diff --git a/lib/crowdb-kv/src/cluster/group_config.rs b/lib/crowdb-paxos-kv/src/cluster/group_config.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_config.rs rename to lib/crowdb-paxos-kv/src/cluster/group_config.rs diff --git a/lib/crowdb-kv/src/cluster/group_election.rs b/lib/crowdb-paxos-kv/src/cluster/group_election.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_election.rs rename to lib/crowdb-paxos-kv/src/cluster/group_election.rs diff --git a/lib/crowdb-kv/src/cluster/group_election/recovery.rs b/lib/crowdb-paxos-kv/src/cluster/group_election/recovery.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_election/recovery.rs rename to lib/crowdb-paxos-kv/src/cluster/group_election/recovery.rs diff --git a/lib/crowdb-kv/src/cluster/group_election_candidate.rs b/lib/crowdb-paxos-kv/src/cluster/group_election_candidate.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_election_candidate.rs rename to lib/crowdb-paxos-kv/src/cluster/group_election_candidate.rs diff --git a/lib/crowdb-kv/src/cluster/group_election_leader.rs b/lib/crowdb-paxos-kv/src/cluster/group_election_leader.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_election_leader.rs rename to lib/crowdb-paxos-kv/src/cluster/group_election_leader.rs diff --git a/lib/crowdb-kv/src/cluster/group_fetchgap.rs b/lib/crowdb-paxos-kv/src/cluster/group_fetchgap.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_fetchgap.rs rename to lib/crowdb-paxos-kv/src/cluster/group_fetchgap.rs diff --git a/lib/crowdb-kv/src/cluster/group_inflight.rs b/lib/crowdb-paxos-kv/src/cluster/group_inflight.rs similarity index 86% rename from lib/crowdb-kv/src/cluster/group_inflight.rs rename to lib/crowdb-paxos-kv/src/cluster/group_inflight.rs index a10bd0c0a..cb30b652a 100644 --- a/lib/crowdb-kv/src/cluster/group_inflight.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_inflight.rs @@ -20,7 +20,7 @@ use crate::paxos::PxNodeId; /// `false`, the caller blocks on `acquire().await` until a permit /// is freed. pub(crate) struct InflightAdmission { - pub(crate) semaphore: tokio::sync::Semaphore, + pub(crate) semaphore: Arc<tokio::sync::Semaphore>, pub(crate) window: usize, pub(crate) reject_on_window_full: bool, /// Cumulative count of proposals that entered the queue (did not @@ -30,6 +30,7 @@ pub(crate) struct InflightAdmission { pub(crate) total_wait_us: AtomicU64, /// Current number of proposals waiting on `acquire().await`. pub(crate) waiting: AtomicU64, + idle: Arc<tokio::sync::Notify>, /// Registry handles for metrics-log export of window-full events. pub(crate) handles: OnceLock<InflightRegistryHandles>, } @@ -42,15 +43,35 @@ pub(crate) struct InflightRegistryHandles { pub(crate) wait_us: Arc<LatencySummary>, } +pub(crate) struct InflightPermit { + permit: Option<tokio::sync::OwnedSemaphorePermit>, + idle: Arc<tokio::sync::Notify>, +} +impl Drop for InflightPermit { + fn drop(&mut self) { + drop(self.permit.take()); + self.idle.notify_waiters(); + } +} + +struct InflightWait<'a>(&'a AtomicU64); + +impl Drop for InflightWait<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } +} + impl InflightAdmission { pub(crate) fn new(max_inflight: usize, reject_on_window_full: bool) -> Self { Self { - semaphore: tokio::sync::Semaphore::new(max_inflight), + semaphore: Arc::new(tokio::sync::Semaphore::new(max_inflight)), window: max_inflight, reject_on_window_full, total_enqueued: AtomicU64::new(0), total_wait_us: AtomicU64::new(0), waiting: AtomicU64::new(0), + idle: Arc::new(tokio::sync::Notify::new()), handles: OnceLock::new(), } } @@ -68,10 +89,13 @@ impl InflightAdmission { /// Acquire a permit. Returns `None` if `reject_on_window_full` and /// the window is full. Otherwise blocks until a permit is available. - pub(crate) async fn acquire_permit(&self) -> Option<tokio::sync::SemaphorePermit<'_>> { + pub(crate) async fn acquire_permit(&self) -> Option<InflightPermit> { // Fast path: try to acquire without blocking. - if let Ok(permit) = self.semaphore.try_acquire() { - return Some(permit); + if let Ok(permit) = Arc::clone(&self.semaphore).try_acquire_owned() { + return Some(InflightPermit { + permit: Some(permit), + idle: Arc::clone(&self.idle), + }); } // Slow path: reject or queue. if self.reject_on_window_full { @@ -79,17 +103,37 @@ impl InflightAdmission { } else { self.total_enqueued.fetch_add(1, Ordering::Relaxed); self.waiting.fetch_add(1, Ordering::Relaxed); + let waiting = InflightWait(&self.waiting); let t0 = std::time::Instant::now(); - let permit = self.semaphore.acquire().await.expect("inflight semaphore closed"); + let permit = Arc::clone(&self.semaphore).acquire_owned().await; let wait_us = t0.elapsed().as_micros(); - self.waiting.fetch_sub(1, Ordering::Relaxed); + drop(waiting); self.total_wait_us .fetch_add(u64::try_from(wait_us).unwrap_or(u64::MAX), Ordering::Relaxed); if let Some(h) = self.handles.get() { h.enqueued.inc(); h.wait_us.observe(u64::try_from(wait_us).unwrap_or(u64::MAX)); } - Some(permit) + permit.ok().map(|permit| InflightPermit { + permit: Some(permit), + idle: Arc::clone(&self.idle), + }) + } + } + + pub(crate) fn close(&self) { + self.semaphore.close(); + } + pub(crate) fn closed(&self) -> bool { + self.semaphore.is_closed() + } + pub(crate) async fn drain(&self) { + loop { + let idle = self.idle.notified(); + if self.occupied() == 0 { + return; + } + idle.await; } } diff --git a/lib/crowdb-kv/src/cluster/group_maintenance.rs b/lib/crowdb-paxos-kv/src/cluster/group_maintenance.rs similarity index 82% rename from lib/crowdb-kv/src/cluster/group_maintenance.rs rename to lib/crowdb-paxos-kv/src/cluster/group_maintenance.rs index c603e0aa1..d458c19cc 100644 --- a/lib/crowdb-kv/src/cluster/group_maintenance.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_maintenance.rs @@ -42,6 +42,7 @@ use tracing::{debug, error, info, info_span, warn, Instrument, Span}; use super::group::PxGroup; use super::group_election::LeaderElection; +use crate::kv::{checkpoint_engine, flush_engine}; use crate::paxos::roles::Acceptor; use crate::wal::gc::run_gc_with_watermark; @@ -109,11 +110,9 @@ async fn maintenance_loop(group: Weak<PxGroup>, tick: Duration, cancel: Cancella } } -/// Persist a durable snapshot on a blocking thread, update watermarks, -/// and return the snapshot slot. Runs on a blocking thread so a large -/// snapshot (e.g. 100k × 16 KiB = 1.6 GB) cannot stall the async -/// election driver / heartbeat task. -async fn persist_snapshot_blocking( +/// Await engine durability, preserving a committed watermark even if later +/// maintenance fails. The engine selects native async or legacy blocking I/O. +async fn persist_snapshot( group: &PxGroup, contiguous: u64, slot_advance: u64, @@ -127,17 +126,30 @@ async fn persist_snapshot_blocking( ); let snap_start = std::time::Instant::now(); let engine_arc = group.local_replica().learner.engine_arc(); - let span = Span::current(); - let at = tokio::task::spawn_blocking(move || span.in_scope(|| engine_arc.persist_snapshot())) - .await - .unwrap_or_else(|e| { - error!( - error = %e, - "maintenance: persist_snapshot blocking task panicked; \ - next step: inspect engine snapshot path for panic" - ); - 0 - }); + let (at, complete) = match checkpoint_engine(engine_arc).await { + Ok(report) => { + if let Err(error) = &report.status { + error!( + error, + durable_committed = report.durable_committed, + durable_coverage = report.durable_coverage, + "maintenance: checkpoint terminal failure" + ); + } + ( + if report.durable_committed { + report.durable_coverage + } else { + 0 + }, + report.status.is_ok(), + ) + } + Err(error) => { + error!(error, "maintenance: checkpoint admission or transport failed"); + (0, false) + } + }; let elapsed_ms = u64::try_from(snap_start.elapsed().as_millis()).unwrap_or(u64::MAX); if let Some(handles) = group.snapshot_handles.get() { let elapsed = snap_start.elapsed(); @@ -148,7 +160,7 @@ async fn persist_snapshot_blocking( handles .max_us .set(handles.max_us.snapshot().max(snapshot_latency_us)); - if at > 0 { + if complete && at > 0 { handles.success.inc(); } else { handles.failure.inc(); @@ -158,7 +170,7 @@ async fn persist_snapshot_blocking( info!( elapsed_ms, snapshot_slot = at, - "maintenance: persist_snapshot completed on blocking thread" + "maintenance: durable checkpoint completed" ); } // Gap 4: only advance watermarks if the snapshot actually succeeded @@ -201,28 +213,29 @@ pub(crate) async fn run_pass(group: &PxGroup, do_flush: bool) { // is empty). Advances last_applied_slot in memory only. Skipped on // tick wakes when a recent proactive flush already drained — avoids // force-freezing the active table and draining a large batch. - // Runs on a blocking thread because flush holds the C++ write_mutex_. if do_flush { let flush_start = std::time::Instant::now(); let engine_arc = group.local_replica().learner.engine_arc(); - let span = Span::current(); - tokio::task::spawn_blocking(move || span.in_scope(|| engine_arc.flush())) - .await - .unwrap_or_else(|e| { - error!( - error = %e, - "maintenance: flush blocking task panicked" - ); - }); + match flush_engine(engine_arc).await { + Ok(outcome) => { + if outcome.advances_flush_count() { + group.flushes_since_snapshot.fetch_add(1, Ordering::Relaxed); + } + if let Err(error) = outcome.status() { + error!( + error, + "maintenance: accepted flush failed; completed handoff remains visible" + ); + } + } + Err(error) => { + error!(error, "maintenance: flush admission or transport failed"); + } + } debug!( elapsed_ms = u64::try_from(flush_start.elapsed().as_millis()).unwrap_or(u64::MAX), "maintenance: flush phase complete" ); - - // Gap 5 step 4: increment flush counter for the snapshot trigger. - group - .flushes_since_snapshot - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } // 2. Conditionally persist a durable snapshot to disk (expensive: sync + @@ -262,7 +275,7 @@ pub(crate) async fn run_pass(group: &PxGroup, do_flush: bool) { } let engine_snapshot_at = if should_snapshot { - let at = persist_snapshot_blocking(group, contiguous, slot_advance, time_elapsed).await; + let at = persist_snapshot(group, contiguous, slot_advance, time_elapsed).await; // Gap 5 step 4: reset the flush counter on a successful snapshot. if at > 0 { group @@ -274,6 +287,21 @@ pub(crate) async fn run_pass(group: &PxGroup, do_flush: bool) { 0 }; + // An idle/pressure pass can make pages reloadable without advancing WAL + // durability. The following GC step only consumes durable checkpoints. + if !should_snapshot && engine.writeback_pending() { + if let Some(work) = engine.writeback_async() { + match work.await { + Ok(report) => { + if let Err(error) = report.status { + error!(error, work=?report.work, "maintenance: writeback failed"); + } + } + Err(error) => error!(error, "maintenance: writeback admission or transport failed"), + } + } + } + // 3. Conditionally force a durable WAL flush (real fsync, even with // --no-fsync). Gated on wal_flush_interval_ms so we don't fsync // every tick. The WAL writer's own pipeline mutex serializes @@ -312,8 +340,8 @@ pub(crate) async fn run_pass(group: &PxGroup, do_flush: bool) { // part of snapshot preparation (R129), so there is no standalone // resident-tree GC sweep. // `set_gc_watermark` is a cheap atomic store and stays inline; - // `compact_sparse_blocks` holds the C++ write_mutex_ and runs on a - // blocking thread. + // Runtime compaction awaits native continuations; compatibility engines + // retain their host blocking adapter. let safe_slot = group.group_safe_slot(); let snapshot_slot = group.group_snapshot_slot(); engine.set_gc_watermark(snapshot_slot, safe_slot); @@ -323,21 +351,18 @@ pub(crate) async fn run_pass(group: &PxGroup, do_flush: bool) { let last_compact_ms = group.last_merge_gc_time_ms.load(Ordering::Acquire); if now_ms.saturating_sub(last_compact_ms) >= merge_gc_interval_ms { let compact_start = std::time::Instant::now(); - let engine_arc = group.local_replica().learner.engine_arc(); - let span = Span::current(); - match tokio::task::spawn_blocking(move || span.in_scope(|| engine_arc.compact_sparse_blocks())) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => { - error!(error, "maintenance: compact_sparse_blocks failed"); - } - Err(error) => { - error!( - %error, - "maintenance: compact_sparse_blocks blocking task panicked" - ); - } + let result = if let Some(native) = engine.compact_sparse_blocks_async() { + native.await + } else { + let engine_arc = group.local_replica().learner.engine_arc(); + let span = Span::current(); + tokio::task::spawn_blocking(move || span.in_scope(|| engine_arc.compact_sparse_blocks())) + .await + .map_err(|error| error.to_string()) + .and_then(std::convert::identity) + }; + if let Err(error) = result { + error!(error, "maintenance: compact_sparse_blocks failed"); } group.last_merge_gc_time_ms.store( crate::common::time::instant_to_anchor_ms(std::time::Instant::now()), diff --git a/lib/crowdb-kv/src/cluster/group_membership.rs b/lib/crowdb-paxos-kv/src/cluster/group_membership.rs similarity index 91% rename from lib/crowdb-kv/src/cluster/group_membership.rs rename to lib/crowdb-paxos-kv/src/cluster/group_membership.rs index aed010ef6..dded4484a 100644 --- a/lib/crowdb-kv/src/cluster/group_membership.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_membership.rs @@ -6,6 +6,7 @@ use std::sync::atomic::Ordering; use std::sync::Arc; +use tokio::task::JoinHandle; use tokio::time::Duration; use tracing::{debug, error, info, warn}; @@ -28,7 +29,7 @@ use crate::cluster::replica::{Replica, StepDownReply, StepDownRequestPayload}; use crate::cluster::status::{GroupStatus, InflightStatus, ReadStateView, StatusLevel}; use crate::common::report::OperationReport; use crate::paxos::roles::SlotIndex; -use crate::paxos::PxNodeId; +use crate::paxos::{PxGroupId, PxNodeId}; impl PxGroup { /// Seed the membership epoch from a persisted/prior value, without @@ -442,6 +443,15 @@ impl PxGroup { .iter() .map(|msg| format!("local#{}: {msg}", self.local_replica.id)) .collect(); + if self.runtime_shutdown_pending() { + status = StatusLevel::Unhealthy; + messages.push(format!( + "shutdown waiting for accepted Tree work: applied {}, target {}, apply tasks {}", + self.local_replica.contiguous_applied(), + self.local_replica.shutdown_apply_target(), + self.local_replica.learner.applying_tasks() + )); + } let mut ok_voting = u32::from(self.local_replica.voting() && local_status != StatusLevel::Unhealthy); for remote in &remotes { @@ -507,6 +517,50 @@ impl PxGroup { // ── Shutdown ────────────────────────────────────────────────── + async fn join_owned_task( + group_id: PxGroupId, + task_name: &'static str, + mut handle: JoinHandle<()>, + timeout: Duration, + report: &mut OperationReport, + ) { + match tokio::time::timeout(timeout, &mut handle).await { + Ok(Ok(())) => {} + Ok(Err(error)) if error.is_cancelled() => {} + Ok(Err(error)) => { + warn!( + g = group_id, + task = task_name, + error = %error, + "owned group task panicked during shutdown" + ); + report.push_error(format!( + "critical: group {group_id} {task_name} join failed: {error}" + )); + } + Err(_) => { + handle.abort(); + let join_result = handle.await; + let detail = match join_result { + Ok(()) => "clean exit after abort".to_string(), + Err(error) if error.is_cancelled() => "cancelled after abort".to_string(), + Err(error) => format!("join failed after abort: {error}"), + }; + warn!( + g = group_id, + task = task_name, + timeout_ms = timeout.as_millis() as u64, + detail = %detail, + "owned group task did not exit within per-layer timeout; aborted and joined" + ); + report.push_error(format!( + "critical: group {group_id} {task_name} shutdown timed out after {} ms; task aborted and joined ({detail})", + timeout.as_millis() + )); + } + } + } + /// Cascade shutdown through this group's replicas. /// /// Iterates real remote replicas and closes their crowdb-rpc channels, then @@ -527,52 +581,61 @@ impl PxGroup { "PxGroup shutdown starting" ); + if let Err(error) = self.drain_runtime_before_shutdown(per_layer_timeout).await { + report.push_error(error); + return report; + } + // 0. Cancel the per-tenure token and await the election driver + // engine maintenance loop (both share this same cancel source). // Both are cooperative; a 100 ms scaffold tick is well within // `per_layer_timeout`. self.tenure_cancel.cancel(); if let Some(handle) = self.driver_handle.lock().await.take() { - match tokio::time::timeout(per_layer_timeout, handle).await { - Ok(Ok(())) => {} - Ok(Err(join_err)) => { - warn!( - g = self.group_id, - error = %join_err, - "election driver task panicked during shutdown" - ); - } - Err(_) => { - warn!( - g = self.group_id, - timeout_ms = per_layer_timeout.as_millis() as u64, - "election driver task did not exit within per-layer timeout" - ); - } - } + Self::join_owned_task( + self.group_id, + "election driver", + handle, + per_layer_timeout, + &mut report, + ) + .await; } if let Some(handle) = self.maintenance_handle.lock().await.take() { - match tokio::time::timeout(per_layer_timeout, handle).await { - Ok(Ok(())) => {} - Ok(Err(join_err)) => { - warn!( - g = self.group_id, - error = %join_err, - "engine maintenance task panicked during shutdown" - ); - } - Err(_) => { - warn!( - g = self.group_id, - timeout_ms = per_layer_timeout.as_millis() as u64, - "engine maintenance task did not exit within per-layer timeout" - ); - } - } + Self::join_owned_task( + self.group_id, + "engine maintenance task", + handle, + per_layer_timeout, + &mut report, + ) + .await; } // R65: stop the FetchGap driver. if let Some(handle) = self.fetchgap_handle.lock().await.take() { - let _ = handle.await; + Self::join_owned_task( + self.group_id, + "FetchGap driver", + handle, + per_layer_timeout, + &mut report, + ) + .await; + } + let coalesce_watchdog = self.coalesce_watchdog_handle.get().cloned(); + let coalesce_watchdog = match coalesce_watchdog { + Some(handle) => handle.lock().await.take(), + None => None, + }; + if let Some(handle) = coalesce_watchdog { + Self::join_owned_task( + self.group_id, + "coalescer watchdog", + handle, + per_layer_timeout, + &mut report, + ) + .await; } // 1. Close remote crowdb-rpc channels first so no in-flight RPCs spin. diff --git a/lib/crowdb-kv/src/cluster/group_operations.rs b/lib/crowdb-paxos-kv/src/cluster/group_operations.rs similarity index 97% rename from lib/crowdb-kv/src/cluster/group_operations.rs rename to lib/crowdb-paxos-kv/src/cluster/group_operations.rs index b0a3004cc..8d031fc11 100644 --- a/lib/crowdb-kv/src/cluster/group_operations.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_operations.rs @@ -145,7 +145,7 @@ impl KvGroupOperations { /// # Errors /// /// Returns a routing or availability error when the read barrier cannot - /// be established, or when a bound leader tenure has ended. + /// be established, the engine read fails, or a bound leader tenure has ended. pub async fn get( &self, key: &[u8], @@ -153,13 +153,14 @@ impl KvGroupOperations { ) -> Result<KvGroupRead, KvGroupOperationError> { let (read_slot, safe_slot) = self.resolve_read_point(consistency).await?; let engine_start = Instant::now(); - let value = self.group.local_replica().learner.engine_get_bytes(key).await; + let value = self.group.local_replica().learner.engine_get_versioned(key).await; if let Some(handles) = self.group.read_handles() { handles .engine_get .observe(engine_start.elapsed().as_nanos() as u64); } self.ensure_tenure()?; + let value = value.map_err(KvGroupOperationError::Unavailable)?; let (revision, value) = value.map_or((0, None), |(revision, value)| (revision, Some(value))); Ok(KvGroupRead { value, @@ -365,7 +366,9 @@ impl KvGroupOperations { } ProposeResult::CasBusy => Err(KvGroupOperationError::CompareBusy), ProposeResult::OutcomeUnknown => Err(KvGroupOperationError::OutcomeUnknown), - ProposeResult::Err(error) => Err(KvGroupOperationError::Internal(error)), + ProposeResult::Rejected(error) | ProposeResult::Err(error) => { + Err(KvGroupOperationError::Internal(error)) + } } } diff --git a/lib/crowdb-kv/src/cluster/group_owner_fence.rs b/lib/crowdb-paxos-kv/src/cluster/group_owner_fence.rs similarity index 99% rename from lib/crowdb-kv/src/cluster/group_owner_fence.rs rename to lib/crowdb-paxos-kv/src/cluster/group_owner_fence.rs index 3e100548c..ad772744e 100644 --- a/lib/crowdb-kv/src/cluster/group_owner_fence.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_owner_fence.rs @@ -196,7 +196,7 @@ impl PxGroup { observation: Box::new(crate::rpc::KvCompareObservation::from_versioned(key, value)), } } - Err(error) => return ProposeResult::Err(error), + Err(error) => return ProposeResult::Rejected(error), }; if !self.cas_admission_ready(tenure, Some(tenure)) { return self.cas_not_ready_result(); diff --git a/lib/crowdb-kv/src/cluster/group_prepare.rs b/lib/crowdb-paxos-kv/src/cluster/group_prepare.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_prepare.rs rename to lib/crowdb-paxos-kv/src/cluster/group_prepare.rs diff --git a/lib/crowdb-kv/src/cluster/group_propose.rs b/lib/crowdb-paxos-kv/src/cluster/group_propose.rs similarity index 91% rename from lib/crowdb-kv/src/cluster/group_propose.rs rename to lib/crowdb-paxos-kv/src/cluster/group_propose.rs index 8f9595f5c..6e77bb9b9 100644 --- a/lib/crowdb-kv/src/cluster/group_propose.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_propose.rs @@ -15,11 +15,12 @@ use crate::cluster::group_accept::AcceptAttempt; use crate::cluster::group_election::XorShift64; use crate::cluster::group_prepare::PrepareAttempt; use crate::common::config::PaxosConfig; +use crate::kv::ApplyAdmissionError; use crate::paxos::error::PxPaxosError; use crate::paxos::roles::{PxLogEntry, RequestIdentity}; impl PxGroup { - /// Propose an opaque payload through Paxos. Returns the slot if chosen, + /// Propose an encoded KV batch through Paxos. Returns the slot if chosen, /// or an error string. /// /// When R45 coalescing is enabled (`coalesce_max_keys > 0` and the @@ -59,6 +60,9 @@ impl PxGroup { required_term: Option<u64>, ) -> ProposeResult { let replica = &self.local_replica; + if self.inflight.closed() { + return ProposeResult::Busy; + } // Leadership gate. Checks BOTH: // * role == Leader -- captures the role atomic flipped by @@ -195,11 +199,11 @@ impl PxGroup { } // Sliding-window admission: cap concurrent in-flight proposals. The - // permit is held for the whole proposal (released on drop at every - // return path below). Depending on the admission policy, a full + // permit covers consensus and any admitted native apply, including + // background apply after returning Chosen. A full // window either fails fast with `Busy` (Reject) or blocks until a // permit is freed (Queue). - let Some(_window_permit) = self.inflight.acquire_permit().await else { + let Some(window_permit) = self.inflight.acquire_permit().await else { warn!( g = self.group_id, window = self.inflight.total_permits(), @@ -207,7 +211,24 @@ impl PxGroup { ); return ProposeResult::Busy; }; - + let mut window_permit = Some(window_permit); + + // This resource owner survives every slot/ballot retry. Only choosing + // this exact payload transfers it to apply; rejected attempts drop it. + let mut apply_admission = match replica.learner.admit_proposal(payload.clone()).await { + Ok(permit) => permit, + Err(ApplyAdmissionError::Busy) => return ProposeResult::Busy, + Err(ApplyAdmissionError::Rejected(error)) => return ProposeResult::Rejected(error), + }; + // Admission may suspend. Do not allocate a slot after a tenure change. + if replica.role() != crate::cluster::local_replica::PxLocalReplicaRole::Leader + || replica.current_term_snapshot() != current_term + || self.proposing_term.load(Ordering::Acquire) != current_term + { + return ProposeResult::NotLeader { + leader_hint: self.leader_endpoint().unwrap_or_default(), + }; + } let group_id = self.group_id; // Voting-only quorum (`self.quorum`/`cached_quorum`), *not* // `valid_replica_count + 1` -- the latter counts non-voting @@ -313,10 +334,28 @@ impl PxGroup { // the engine apply as a background task and return // Chosen immediately. The fan_out_chosen_notice // fires immediately too (non-blocking mpsc enqueue). + let admitted = if own_payload { apply_admission.take() } else { None }; + let apply_window = if admitted.is_some() { + window_permit.take() + } else { + None + }; if self.config.async_engine_apply { - replica.spawn_learn_chosen(entry.clone(), entry_identities); + replica.spawn_learn_chosen_admitted( + entry.clone(), + entry_identities, + admitted, + apply_window, + ); } else { - replica.learn_chosen(&entry, entry_identities).await; + replica + .learn_chosen_admitted( + entry.clone(), + entry_identities, + admitted, + apply_window, + ) + .await; } self.fan_out_chosen_notice(&entry, group_id); trace!( diff --git a/lib/crowdb-kv/src/cluster/group_retention.rs b/lib/crowdb-paxos-kv/src/cluster/group_retention.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_retention.rs rename to lib/crowdb-paxos-kv/src/cluster/group_retention.rs diff --git a/lib/crowdb-kv/src/cluster/group_routes.rs b/lib/crowdb-paxos-kv/src/cluster/group_routes.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_routes.rs rename to lib/crowdb-paxos-kv/src/cluster/group_routes.rs diff --git a/lib/crowdb-paxos-kv/src/cluster/group_shutdown.rs b/lib/crowdb-paxos-kv/src/cluster/group_shutdown.rs new file mode 100644 index 000000000..65348e8d2 --- /dev/null +++ b/lib/crowdb-paxos-kv/src/cluster/group_shutdown.rs @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::group::{PxGroup, RepairOutcome}; +use std::time::Duration; + +impl PxGroup { + pub(super) fn close_runtime_proposals(&self) { + if self.local_replica.learner.engine().requires_contiguous_apply() { + self.inflight.close(); + } + } + + pub(super) async fn drain_runtime_before_shutdown(&self, timeout: Duration) -> Result<(), String> { + if !self.local_replica.learner.engine().requires_contiguous_apply() { + return Ok(()); + } + self.inflight.close(); + let drained = tokio::time::timeout(timeout, self.drain_runtime_chosen()).await; + if drained.is_err() { + return Err(format!("critical: group {} accepted Tree work did not drain within {} ms; repair remains running, retry shutdown after restoring progress (applied {}, chosen {})", + self.group_id, timeout.as_millis(), self.local_replica.contiguous_applied(), self.local_replica.last_chosen_slot())); + } + Ok(()) + } + + pub(super) fn runtime_shutdown_pending(&self) -> bool { + self.local_replica.learner.engine().requires_contiguous_apply() + && self.inflight.closed() + && !self.tenure_cancel.is_cancelled() + } + + pub(super) async fn drain_runtime_chosen(&self) { + let replica = &self.local_replica; + loop { + let target = replica.shutdown_apply_target(); + replica.wake_apply_loop(); + if self.inflight.occupied() == 0 + && replica.contiguous_applied() >= target + && replica.learner.applying_tasks() == 0 + { + return; + } + // Use the existing Prepare/Accept protocol. It adopts accepted + // values and can only choose an empty gap after a real quorum. + if replica.is_leader() + && replica.contiguous_chosen() < replica.last_chosen_slot() + && matches!(self.repair_once().await, RepairOutcome::Filled { .. }) + { + continue; + } + tokio::select! { + () = replica.learner.await_applied(target), if replica.contiguous_applied() < target => {}, + () = self.inflight.drain(), if self.inflight.occupied() != 0 => {}, + () = replica.learner.drain_apply_tasks(), if replica.learner.applying_tasks() != 0 => {}, + () = tokio::time::sleep(Duration::from_millis(10)) => {}, + } + } + } +} diff --git a/lib/crowdb-kv/src/cluster/group_tests.rs b/lib/crowdb-paxos-kv/src/cluster/group_tests.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/group_tests.rs rename to lib/crowdb-paxos-kv/src/cluster/group_tests.rs diff --git a/lib/crowdb-kv/src/cluster/kv_store.rs b/lib/crowdb-paxos-kv/src/cluster/kv_store.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/kv_store.rs rename to lib/crowdb-paxos-kv/src/cluster/kv_store.rs diff --git a/lib/crowdb-kv/src/cluster/local_replica.rs b/lib/crowdb-paxos-kv/src/cluster/local_replica.rs similarity index 98% rename from lib/crowdb-kv/src/cluster/local_replica.rs rename to lib/crowdb-paxos-kv/src/cluster/local_replica.rs index e807d29c5..9a1062555 100644 --- a/lib/crowdb-kv/src/cluster/local_replica.rs +++ b/lib/crowdb-paxos-kv/src/cluster/local_replica.rs @@ -609,7 +609,7 @@ impl PxLocalReplica { /// page store (disk). For `InMemKV` both are no-ops. #[tracing::instrument(level = "debug", skip_all, fields(s = self.log_store_id.load(Ordering::Acquire), g = self.log_group_id.load(Ordering::Acquire), replica = self.id))] #[allow(clippy::unused_async)] // async kept for cascade uniformity - pub async fn shutdown(&self, _per_layer_timeout: Duration) -> OperationReport { + pub async fn shutdown(&self, per_layer_timeout: Duration) -> OperationReport { if self.shutdown_started.swap(true, Ordering::AcqRel) { debug!( replica = self.id, @@ -618,8 +618,17 @@ impl PxLocalReplica { return OperationReport::new(); } - // R63: stop the background apply loop before flushing the engine. - self.stop_apply_loop(); + if let Err(error) = self.drain_runtime_apply(per_layer_timeout).await { + self.shutdown_started.store(false, Ordering::Release); + let mut report = OperationReport::new(); + report.push_error(error); + return report; + } + + // R63: stop and join the background apply loop before flushing the + // engine. A timed abort is still joined so the report reflects the + // owned task's terminal state before shutdown proceeds. + let mut report = self.stop_apply_loop(per_layer_timeout).await; // Durably flush the WAL (real fsync, regardless of --no-fsync). // This ensures shutdown always persists data to disk. @@ -634,26 +643,12 @@ impl PxLocalReplica { debug!("WAL flush_all completed during shutdown"); } } - let engine = self.learner.engine(); - engine.flush(); - let snap_slot = engine.persist_snapshot(); - if snap_slot > 0 { - info!( - replica = self.id, - snapshot_slot = snap_slot, - "PxLocalReplica shutdown: engine snapshot persisted" - ); - } else { - debug!( - replica = self.id, - "PxLocalReplica shutdown: no durable snapshot (non-durable engine or no data)" - ); - } + self.finish_engine_shutdown(&mut report).await; info!( replica = self.id, "PxLocalReplica shutdown complete (acceptor/learner cleanup deferred to Drop)" ); - OperationReport::new() + report } #[must_use] diff --git a/lib/crowdb-kv/src/cluster/local_replica_accept.rs b/lib/crowdb-paxos-kv/src/cluster/local_replica_accept.rs similarity index 92% rename from lib/crowdb-kv/src/cluster/local_replica_accept.rs rename to lib/crowdb-paxos-kv/src/cluster/local_replica_accept.rs index ccc971f7b..02d0631c5 100644 --- a/lib/crowdb-kv/src/cluster/local_replica_accept.rs +++ b/lib/crowdb-paxos-kv/src/cluster/local_replica_accept.rs @@ -102,6 +102,14 @@ impl PxLocalReplica { ); } } + // The value may fill a known chosen gap while its apply loop is asleep. + // Acceptance alone does not establish chosen-ness or advance a frontier. + if self.learner.engine().requires_contiguous_apply() + && (entry.slot <= self.known_commit_slot.load(std::sync::atomic::Ordering::Acquire) + || self.learner.is_chosen(entry.slot)) + { + self.wake_apply_loop(); + } } /// R16b: spawn the WAL persist as a detached background task. diff --git a/lib/crowdb-kv/src/cluster/local_replica_apply.rs b/lib/crowdb-paxos-kv/src/cluster/local_replica_apply.rs similarity index 75% rename from lib/crowdb-kv/src/cluster/local_replica_apply.rs rename to lib/crowdb-paxos-kv/src/cluster/local_replica_apply.rs index b94e27950..4d5602afa 100644 --- a/lib/crowdb-kv/src/cluster/local_replica_apply.rs +++ b/lib/crowdb-paxos-kv/src/cluster/local_replica_apply.rs @@ -3,16 +3,20 @@ #![allow(clippy::cast_possible_truncation)] +use crate::cluster::group_inflight::InflightPermit; use crate::cluster::local_replica::PxLocalReplica; +use crate::common::report::OperationReport; +use crate::kv::AdmittedApply; use crate::metrics::Counter; use crate::paxos::acceptor::PxAcceptor; use crate::paxos::learner::PxLearner; -use crate::paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry, RequestIdentity, SlotIndex}; +use crate::paxos::roles::{Acceptor, PxBallot, PxLogEntry, RequestIdentity, SlotIndex}; use crate::paxos::PxTerm; use parking_lot::Mutex; use std::collections::BTreeSet; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use std::time::Duration; use tracing::{info_span, trace, Instrument}; /// R63: bounded batch size for the background apply loop. Limits the number @@ -30,7 +34,32 @@ impl PxLocalReplica { /// a coalesced batch passes one per client op; repair/election pass /// none. pub async fn learn_chosen(&self, entry: &PxLogEntry, request_identities: &[RequestIdentity]) { - self.learner.learn(entry.clone(), request_identities).await; + self.learn_chosen_admitted(entry.clone(), request_identities, None, None) + .await; + } + + pub(crate) async fn learn_chosen_admitted( + &self, + entry: PxLogEntry, + request_identities: &[RequestIdentity], + admission: Option<Box<dyn AdmittedApply>>, + window: Option<InflightPermit>, + ) { + if self.learner.engine().requires_contiguous_apply() { + // Cancelling this waiter detaches the owned task. Already chosen + // native work still advances the Rust frontier and drops its guard. + if let Err(error) = self + .spawn_owned_chosen(entry, request_identities, admission, window) + .await + { + tracing::error!(error = %error, "critical: accepted Tree apply task failed; next step: inspect the native operation and restore from WAL"); + } + } else { + let _window = window; + self.learner + .learn_admitted(entry, request_identities, admission) + .await; + } } /// R17: defer the engine apply (`apply_entry` + applied-frontier @@ -47,8 +76,29 @@ impl PxLocalReplica { /// learner's `apply_entry` is idempotent, and the frontier/cache /// updates are atomic, so a delayed apply is safe. pub fn spawn_learn_chosen(&self, entry: PxLogEntry, request_identities: &[RequestIdentity]) { + self.spawn_learn_chosen_admitted(entry, request_identities, None, None); + } + + pub(crate) fn spawn_learn_chosen_admitted( + &self, + entry: PxLogEntry, + request_identities: &[RequestIdentity], + admission: Option<Box<dyn AdmittedApply>>, + window: Option<InflightPermit>, + ) { + drop(self.spawn_owned_chosen(entry, request_identities, admission, window)); + } + + fn spawn_owned_chosen( + &self, + entry: PxLogEntry, + request_identities: &[RequestIdentity], + admission: Option<Box<dyn AdmittedApply>>, + window: Option<InflightPermit>, + ) -> tokio::task::JoinHandle<()> { let slot = entry.slot; let learner = Arc::clone(&self.learner); + let accepted = learner.track_apply_task(); // Sync: chosen frontier + request result (cheap atomics; must precede // `propose` returning `Chosen` for read-your-writes). learner.update_chosen_frontier(entry.slot, entry.term); @@ -65,12 +115,20 @@ impl PxLocalReplica { ); tokio::spawn( async move { - if learner.apply_entry(entry.slot, &entry.payload).await.is_ok() { + let _accepted = accepted; + // The Chosen reply may already be delivered. Keep native + // apply inside the proposal window through frontier advance. + let _window = window; + if learner + .apply_entry_admitted(entry.slot, &entry.payload, admission) + .await + .is_ok() + { learner.advance_applied_frontier(entry.slot); } } .instrument(span), - ); + ) } /// R63: advance `known_commit_slot` to at least `slot` via `fetch_max`. @@ -94,9 +152,9 @@ impl PxLocalReplica { /// R63: lazily spawn the background apply loop if it hasn't been started /// yet. The loop runs for the replica's lifetime (cancelled in /// [`PxLocalReplica::shutdown`]); it reads `known_commit_slot`, collects entries - /// from the acceptor, and applies them via the learner. Missing slots - /// are skipped (skip-and-continue) so a single gap doesn't block the - /// entire apply backlog. + /// from the acceptor, and applies them via the learner. Engines that + /// require contiguous admission pause at the first gap until a repair + /// or chosen notification arrives; other engines may apply later slots. fn ensure_apply_loop(&self) { if self.apply_loop_handle.lock().is_some() { return; @@ -202,12 +260,41 @@ impl PxLocalReplica { result } - /// R63: stop the background apply loop. Called in [`PxLocalReplica::shutdown`]. - pub(super) fn stop_apply_loop(&self) { + /// R63: stop and join the background apply loop. Called in + /// [`PxLocalReplica::shutdown`]. A timeout first requests cooperative + /// cancellation, then aborts and joins the task so shutdown never + /// returns while the owned apply task is still detached. + pub(super) async fn stop_apply_loop(&self, timeout: Duration) -> OperationReport { + let mut report = OperationReport::new(); self.apply_loop_cancel.cancel(); - if let Some(handle) = self.apply_loop_handle.lock().take() { - handle.abort(); + let Some(mut handle) = self.apply_loop_handle.lock().take() else { + return report; + }; + match tokio::time::timeout(timeout, &mut handle).await { + Ok(Ok(())) => {} + Ok(Err(error)) if error.is_cancelled() => {} + Ok(Err(error)) => { + report.push_error(format!( + "critical: replica {} apply loop join failed: {error}", + self.id + )); + } + Err(_) => { + handle.abort(); + let join_result = handle.await; + report.push_error(format!( + "critical: replica {} apply loop shutdown timed out after {} ms; task aborted and joined ({})", + self.id, + timeout.as_millis(), + match join_result { + Ok(()) => "clean exit after abort".to_string(), + Err(error) if error.is_cancelled() => "cancelled after abort".to_string(), + Err(error) => format!("join failed after abort: {error}"), + } + )); + } } + report } /// Receive a peer-side `ChosenNotice` for `(slot, term)`. @@ -315,10 +402,9 @@ impl PxLocalReplica { /// R63: background apply loop task. Reads `known_commit_slot`, collects /// entries from the acceptor for the contiguous range, and applies them via -/// the learner. Missing slots are skipped (skip-and-continue) so a single -/// gap doesn't block the entire apply backlog — `contiguous_applied` stays -/// at the gap until it's filled; subsequent available slots are applied via -/// `advance_applied_frontier` (which handles out-of-order applies). +/// the learner. Engines with contiguous admission pause at the first missing +/// or unapplied slot, allowing the same loop to consume its repair notification. +/// Other engines may apply higher slots while their contiguous frontier waits. async fn apply_loop_task( known: Arc<AtomicU64>, learner: Arc<PxLearner>, @@ -328,6 +414,7 @@ async fn apply_loop_task( apply_loop_skip: Option<Arc<Counter>>, cancel: tokio_util::sync::CancellationToken, ) { + let requires_contiguous = learner.engine().requires_contiguous_apply(); loop { if cancel.is_cancelled() { return; @@ -339,7 +426,10 @@ async fn apply_loop_task( let target = known.load(Ordering::Acquire).max(learner.last_chosen_slot()); let current = learner.contiguous_applied(); if current >= target { - notify.notified().await; + tokio::select! { + () = notify.notified() => {} + () = cancel.cancelled() => return, + } continue; } let mut next = current.saturating_add(1); @@ -373,13 +463,19 @@ async fn apply_loop_task( } } } + if requires_contiguous && learner.contiguous_applied() < next { + break; + } next += 1; } // If contiguous_applied didn't advance (stuck at a gap with only // out-of-order applies), wait for a wake-up before retrying — // avoids busy-looping on gaps that haven't been filled yet. if learner.contiguous_applied() == before { - notify.notified().await; + tokio::select! { + () = notify.notified() => {} + () = cancel.cancelled() => return, + } } } } diff --git a/lib/crowdb-kv/src/cluster/local_replica_election.rs b/lib/crowdb-paxos-kv/src/cluster/local_replica_election.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/local_replica_election.rs rename to lib/crowdb-paxos-kv/src/cluster/local_replica_election.rs diff --git a/lib/crowdb-kv/src/cluster/local_replica_replay.rs b/lib/crowdb-paxos-kv/src/cluster/local_replica_replay.rs similarity index 85% rename from lib/crowdb-kv/src/cluster/local_replica_replay.rs rename to lib/crowdb-paxos-kv/src/cluster/local_replica_replay.rs index 74eec8dfc..a6044a36b 100644 --- a/lib/crowdb-kv/src/cluster/local_replica_replay.rs +++ b/lib/crowdb-paxos-kv/src/cluster/local_replica_replay.rs @@ -61,6 +61,7 @@ impl PxLocalReplica { // contract only promises an accurate floor for a freshly-recovered // engine that hasn't taken any `apply` calls yet in this process. let resume_from = engine.resume_from_slot(); + let requires_contiguous = engine.requires_contiguous_apply(); let replica = Self::new_with_learner(id, role, PxLearner::with_engine(engine)); // Accepted history and later promises are independent durable facts. @@ -111,8 +112,11 @@ impl PxLocalReplica { // - `update_frontier` handles out-of-order slots via the // `out_of_order` BTreeMap, so watermarks stay correct even with // gaps. - // - NoOp entries (empty payload) are skipped by `apply_entry` and - // do not corrupt the KV state. + // - Empty payloads complete their real slot without adding a key. + // + // Engines with contiguous admission apply only the available prefix. + // A sparse suffix remains in the acceptor for consensus recovery; + // waiting for it here would prevent startup from accepting its repair. // // If the engine reported a resume floor (`resume_from > 0`), skip // re-`learn`ing that prefix and start the walk at `resume_from + @@ -144,7 +148,26 @@ impl PxLocalReplica { 1 }; for slot in start_slot..=highest { - if let Some(entry) = replica.acceptor.accepted_at(slot) { + let Some(entry) = replica.acceptor.accepted_at(slot) else { + if requires_contiguous { + break; + } + continue; + }; + if requires_contiguous { + replica + .learner + .apply_entry(entry.slot, &entry.payload) + .await + .map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("restore slot {slot}: {error}"), + ) + })?; + replica.learner.update_chosen_frontier(entry.slot, entry.term); + replica.learner.advance_applied_frontier(entry.slot); + } else { replica.learner.learn(entry, &[]).await; } } diff --git a/lib/crowdb-paxos-kv/src/cluster/local_replica_shutdown.rs b/lib/crowdb-paxos-kv/src/cluster/local_replica_shutdown.rs new file mode 100644 index 000000000..4da750fdc --- /dev/null +++ b/lib/crowdb-paxos-kv/src/cluster/local_replica_shutdown.rs @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crate::common::report::OperationReport; +use crate::kv::{checkpoint_engine, flush_engine}; + +use super::local_replica::PxLocalReplica; + +impl PxLocalReplica { + pub(super) fn shutdown_apply_target(&self) -> u64 { + self.last_chosen_slot() + .max(self.known_commit_slot.load(std::sync::atomic::Ordering::Acquire)) + } + + pub(super) async fn drain_runtime_apply(&self, timeout: std::time::Duration) -> Result<(), String> { + if !self.learner.engine().requires_contiguous_apply() { + return Ok(()); + } + self.wake_apply_loop(); + let drain = async { + loop { + let target = self.shutdown_apply_target(); + self.learner.await_applied(target).await; + self.learner.drain_apply_tasks().await; + if self.contiguous_applied() >= self.shutdown_apply_target() { + return; + } + } + }; + if tokio::time::timeout(timeout, drain).await.is_err() { + return Err(format!("critical: replica {} accepted Tree work did not drain within {} ms; apply/repair owners retained (applied {}, chosen {})", + self.id, timeout.as_millis(), self.contiguous_applied(), self.shutdown_apply_target())); + } + Ok(()) + } + + pub(super) async fn finish_engine_shutdown(&self, report: &mut OperationReport) { + let engine = self.learner.engine_arc(); + let flush = flush_engine(engine.clone()).await; + let flush_error = match &flush { + Ok(outcome) => outcome.status().err(), + Err(error) => Some(error.as_str()), + }; + if let Some(error) = flush_error { + self.record_engine_shutdown_error(report, "flush", error); + } + match checkpoint_engine(engine).await { + Ok(checkpoint) => { + if checkpoint.durable_committed { + tracing::info!( + replica = self.id, + snapshot_slot = checkpoint.durable_coverage, + "PxLocalReplica shutdown: engine snapshot persisted" + ); + } + if let Err(error) = checkpoint.status { + self.record_engine_shutdown_error(report, "checkpoint", &error); + } + } + Err(error) => self.record_engine_shutdown_error(report, "checkpoint", &error), + } + } + + fn record_engine_shutdown_error(&self, report: &mut OperationReport, phase: &str, error: &str) { + tracing::warn!( + replica = self.id, + phase, + error, + "engine shutdown stage failed; retaining WAL" + ); + report.push_error(format!( + "critical: replica {} engine {phase} failed: {error}", + self.id + )); + } +} diff --git a/lib/crowdb-kv/src/cluster/node_config.rs b/lib/crowdb-paxos-kv/src/cluster/node_config.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/node_config.rs rename to lib/crowdb-paxos-kv/src/cluster/node_config.rs diff --git a/lib/crowdb-kv/src/cluster/kv_server.rs b/lib/crowdb-paxos-kv/src/cluster/paxos_kv_server.rs similarity index 93% rename from lib/crowdb-kv/src/cluster/kv_server.rs rename to lib/crowdb-paxos-kv/src/cluster/paxos_kv_server.rs index 921d55a55..3e9aec599 100644 --- a/lib/crowdb-kv/src/cluster/kv_server.rs +++ b/lib/crowdb-paxos-kv/src/cluster/paxos_kv_server.rs @@ -3,10 +3,10 @@ //! Server lifecycle for a `PxKvStore`. //! -//! Defines the [`KvServer`] trait (start / join / stop / `listen_addr`) +//! Defines the [`PaxosKvServer`] trait (start / join / stop / `listen_addr`) //! and implements it on `Arc<PxKvStore>`. The trait's `start` binds a //! single crowdb-rpc server on the listen port that hosts both -//! consensus (`PxRpcService`) and client-facing (`KvRpcService`) +//! consensus (`PxRpcService`) and client-facing (`PaxosKvRpcService`) //! handlers. Server state (join handle, shutdown sender, bound //! address) lives on [`RpcTaskState`] inside the store so //! [`PxKvStore::shutdown_server`] can drive a timed graceful stop from @@ -15,7 +15,7 @@ #![allow(clippy::cast_possible_truncation)] use crate::cluster::px_kv_store::PxKvStore; -use crate::rpc::{KvClientRpcForwarder, KvRpcService, PxRpcService, PxRpcTransport}; +use crate::rpc::{PaxosKvRpcForwarder, PaxosKvRpcService, PxRpcService, PxRpcTransport}; use crowdb_rpc_ffi::RpcServer; use std::net::SocketAddr; use std::sync::Arc; @@ -23,7 +23,7 @@ use std::time::Duration; use tracing::{debug, error, info, info_span, Instrument}; #[allow(async_fn_in_trait)] -pub trait KvServer { +pub trait PaxosKvServer { async fn start(&self) -> Result<(), String>; async fn join(&self); @@ -48,7 +48,7 @@ pub(crate) struct RpcServerState { pub(crate) transport: Option<Arc<PxRpcTransport>>, } #[allow(async_fn_in_trait)] -impl KvServer for Arc<PxKvStore> { +impl PaxosKvServer for Arc<PxKvStore> { #[allow(clippy::unused_async_trait_impl)] async fn start(&self) -> Result<(), String> { { @@ -65,7 +65,7 @@ impl KvServer for Arc<PxKvStore> { let bound_addr = self.listen_addr; // Start a single crowdb-rpc server on the listen port. Both - // consensus (PxRpcService) and client (KvRpcService) handlers + // consensus (PxRpcService) and client (PaxosKvRpcService) handlers // are registered on the same server — their handler names // don't conflict. Worker count set from `--rpc-workers` CLI // (via `PxKvStore::rpc_workers`). @@ -91,8 +91,8 @@ impl KvServer for Arc<PxKvStore> { let px_service = Arc::new(PxRpcService::new(self.clone(), rt.clone())); px_service.register_handlers(&server); - let forwarder = Arc::new(KvClientRpcForwarder::with_workers(self.rpc_workers)); - let kv_service = Arc::new(KvRpcService::new(self.clone(), rt, forwarder)); + let forwarder = Arc::new(PaxosKvRpcForwarder::with_workers(self.rpc_workers)); + let kv_service = Arc::new(PaxosKvRpcService::new(self.clone(), rt, forwarder)); kv_service.register_handlers(&server); let transport = Arc::new(PxRpcTransport::with_pool_size( @@ -190,8 +190,8 @@ impl KvServer for Arc<PxKvStore> { impl PxKvStore { /// Stop the server task and await its join under a timeout. /// - /// Used by [`PxKvStore::shutdown`] as the first cascade step. Aborts the - /// task on hang; idempotent. + /// Used by [`PxKvStore::shutdown`] after runtime accepted work drains. + /// Aborts the task on hang; idempotent. pub(crate) async fn shutdown_server(&self, timeout: Duration) -> Result<(), String> { self.stop_rpc_server(); self.stop_client_rpc_server(); @@ -260,7 +260,7 @@ impl PxKvStore { /// Wire the shared `PxRpcTransport` into all existing remote /// replicas across all groups. Must be called after - /// [`KvServer::start`]. Remote replicas added later (via + /// [`PaxosKvServer::start`]. Remote replicas added later (via /// the management API) get the transport via /// [`Self::rpc_transport`] at construction time. pub fn wire_rpc_transport(&self) { diff --git a/lib/crowdb-kv/src/cluster/px_kv_store.rs b/lib/crowdb-paxos-kv/src/cluster/px_kv_store.rs similarity index 94% rename from lib/crowdb-kv/src/cluster/px_kv_store.rs rename to lib/crowdb-paxos-kv/src/cluster/px_kv_store.rs index fc5fb779d..a23158783 100644 --- a/lib/crowdb-kv/src/cluster/px_kv_store.rs +++ b/lib/crowdb-paxos-kv/src/cluster/px_kv_store.rs @@ -8,7 +8,7 @@ use crate::cluster::group_election::LeaderElection; use crate::cluster::group_operations::{ KvGroupOperationError, KvGroupOperations, KvReadConsistency, KvRequestIdentity, }; -use crate::cluster::kv_server::{RpcServerState, RpcTaskState}; +use crate::cluster::paxos_kv_server::{RpcServerState, RpcTaskState}; use crate::cluster::status::{GroupStatus, StatusLevel, StoreStatus}; use crate::common::config::ServerConfig; use crate::common::report::OperationReport; @@ -45,7 +45,7 @@ pub struct PxKvStore { pub(crate) metrics_registry: Option<Arc<std::sync::Mutex<MetricsRegistry>>>, /// Per-page byte budget for scan responses (see `ServerConfig::scan_byte_budget`). /// Defaults to `ServerConfig::DEFAULT.scan_byte_budget`; overridden via - /// `set_scan_byte_budget` from the loaded `CrowDBConfig` before `start()`. + /// `set_scan_byte_budget` from the loaded `PaxosKvConfig` before `start()`. pub(crate) scan_byte_budget: usize, /// Number of crowdb-rpc I/O worker threads for the server's `RpcServer`. /// Set from `--rpc-workers` CLI before `start()`. Default: 2. @@ -134,37 +134,37 @@ impl PxKvStore { } /// Override the per-page scan byte budget from the loaded - /// `CrowDBConfig.server.scan_byte_budget`. Called before `start()`. + /// `PaxosKvConfig.server.scan_byte_budget`. Called before `start()`. pub fn set_scan_byte_budget(&mut self, budget: usize) { self.scan_byte_budget = budget; } /// Override the peer connection pool size from the loaded - /// `CrowDBConfig.server.peer_pool_size`. Called before `start()`. + /// `PaxosKvConfig.server.peer_pool_size`. Called before `start()`. pub fn set_peer_pool_size(&mut self, size: usize) { self.peer_pool_size = size; } /// Override the Nagle setting from the loaded - /// `CrowDBConfig.server.enable_nagle`. Called before `start()`. + /// `PaxosKvConfig.server.enable_nagle`. Called before `start()`. pub fn set_enable_nagle(&mut self, enabled: bool) { self.enable_nagle = enabled; } /// Override `TCP_QUICKACK` from the loaded - /// `CrowDBConfig.server.quickack`. Called before `start()`. + /// `PaxosKvConfig.server.quickack`. Called before `start()`. pub fn set_quickack(&mut self, enabled: bool) { self.quickack = enabled; } /// Override the event-write mode from the loaded - /// `CrowDBConfig.server.event_write`. Called before `start()`. + /// `PaxosKvConfig.server.event_write`. Called before `start()`. pub fn set_event_write(&mut self, enabled: bool) { self.event_write = enabled; } /// Override the send queue capacity from the loaded - /// `CrowDBConfig.server.send_queue_capacity`. Called before `start()`. + /// `PaxosKvConfig.server.send_queue_capacity`. Called before `start()`. pub fn set_send_queue_capacity(&mut self, capacity: u32) { self.send_queue_capacity = capacity; } @@ -208,8 +208,10 @@ impl PxKvStore { *self.get_delay.lock() = Some(delay); } - /// Cascade shutdown: stop crowdb-rpc server (with timeout), then shut down each - /// group, cascading into every replica layer. + /// Cascade shutdown: close runtime proposals and drain accepted work while + /// consensus RPCs remain available, then stop the server and each group. + /// A runtime drain timeout leaves the servers and repair tasks alive for a + /// later retry. Stores with only legacy engines retain the server-first order. /// /// The shutdown contract across layers (`PxKvStore` → `PxGroup` → /// `PxLocalReplica` / `PxRemoteReplica` → `acceptor` / `learner` / `slot_list` @@ -222,8 +224,8 @@ impl PxKvStore { /// 4. Returns an [`OperationReport`](crate::common::report::OperationReport) /// with aggregated `critical:` errors. /// - /// Calls are **idempotent** — second and later calls return an empty clean - /// report and log at `debug`. Layers are responsible for their own + /// Completed calls are **idempotent**. An incomplete runtime drain can be + /// retried without recreating its server or repair tasks. Layers own their /// `AtomicBool` "already-shutdown" gate. /// /// ## Why this shape @@ -256,7 +258,15 @@ impl PxKvStore { let mut report = OperationReport::new(); - // 1. Stop crowdb-rpc server first so no new requests reach the groups. + // Close every runtime proposal window before waiting for any group. + // Consensus RPCs must remain available to repair an accepted gap. + let drained = self.drain_runtime_groups(per_layer_timeout).await; + if !drained.is_clean() { + self.shutdown_started.store(false, Ordering::Release); + return drained; + } + + // 1. Stop RPC ingress after the runtime accepted prefix is complete. if let Err(msg) = self.shutdown_server(per_layer_timeout).await { report.push_error(msg); } @@ -275,6 +285,16 @@ impl PxKvStore { report.merge(sub); } + // An incomplete accepted-work drain deliberately keeps repair and + // apply owners alive. A later shutdown must retry that same drain. + if self + .groups + .load() + .values() + .any(|group| group.runtime_shutdown_pending()) + { + self.shutdown_started.store(false, Ordering::Release); + } if report.is_clean() { info!("PxKvStore shutdown complete"); } else { diff --git a/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs b/lib/crowdb-paxos-kv/src/cluster/px_kv_store_handler.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/px_kv_store_handler.rs rename to lib/crowdb-paxos-kv/src/cluster/px_kv_store_handler.rs diff --git a/lib/crowdb-kv/src/cluster/px_kv_store_owner.rs b/lib/crowdb-paxos-kv/src/cluster/px_kv_store_owner.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/px_kv_store_owner.rs rename to lib/crowdb-paxos-kv/src/cluster/px_kv_store_owner.rs diff --git a/lib/crowdb-kv/src/cluster/remote_replica.rs b/lib/crowdb-paxos-kv/src/cluster/remote_replica.rs similarity index 100% rename from lib/crowdb-kv/src/cluster/remote_replica.rs rename to lib/crowdb-paxos-kv/src/cluster/remote_replica.rs diff --git a/lib/crowdb-kv/src/cluster/replica.rs b/lib/crowdb-paxos-kv/src/cluster/replica.rs similarity index 98% rename from lib/crowdb-kv/src/cluster/replica.rs rename to lib/crowdb-paxos-kv/src/cluster/replica.rs index 54a68e5ce..da1d78359 100644 --- a/lib/crowdb-kv/src/cluster/replica.rs +++ b/lib/crowdb-paxos-kv/src/cluster/replica.rs @@ -24,7 +24,7 @@ use crate::paxos::{PxGroupId, PxNodeId, PxTerm}; /// /// All [`ReplicaHandler`] and [`ReplicaClient`] methods return this. The crowdb-rpc /// adapter (`crate::rpc::px_rpc_service`) maps to/from `PxReplicaError` at the -/// network boundary so `crowdb_kv` library code never names `PxReplicaError` +/// network boundary so `crowdb_paxos_kv` library code never names `PxReplicaError` /// outside of `rpc/`. #[derive(Debug, thiserror::Error)] pub enum PxReplicaError { diff --git a/lib/crowdb-kv/src/cluster/status.rs b/lib/crowdb-paxos-kv/src/cluster/status.rs similarity index 97% rename from lib/crowdb-kv/src/cluster/status.rs rename to lib/crowdb-paxos-kv/src/cluster/status.rs index d4e796fa5..bfae7b18c 100644 --- a/lib/crowdb-kv/src/cluster/status.rs +++ b/lib/crowdb-paxos-kv/src/cluster/status.rs @@ -5,9 +5,9 @@ //! //! The wire types (`StoreStatus`, `GroupStatus`, `ReplicaStatus`, etc.) //! live in `crowdb_protocol::mgmt` — the single home for cross-component -//! protocol types (`design-crowdb-kv-group0.md` §2.4). This module re- +//! protocol types (`design-crowdb-paxos-kv-group0.md` §2.4). This module re- //! exports them and hosts the two conversions that must stay local to -//! `crowdb-kv`: +//! `crowdb-paxos-kv`: //! //! - `crowdb_tree_stats_to_view` — a free function converting //! `CrowdbTreeStats` (from `crowdb-tree-ffi`) to `CrowdbTreeStatsView` diff --git a/lib/crowdb-paxos-kv/src/cluster/store_shutdown.rs b/lib/crowdb-paxos-kv/src/cluster/store_shutdown.rs new file mode 100644 index 000000000..b0a913dde --- /dev/null +++ b/lib/crowdb-paxos-kv/src/cluster/store_shutdown.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::PxKvStore; +use crate::common::report::OperationReport; +use std::time::Duration; + +impl PxKvStore { + /// Whether this store hosts an engine whose accepted apply must be drained + /// before stopping its consensus transport or deleting its durable files. + #[must_use] + pub fn has_runtime_engine(&self) -> bool { + self.groups_snapshot() + .values() + .any(|group| group.local_replica().learner.engine().requires_contiguous_apply()) + } + + /// Close every hosted runtime proposal window without stopping consensus + /// communication. A host calls this for every store before draining any. + pub fn close_runtime_proposals(&self) { + for group in self.groups_snapshot().values() { + group.close_runtime_proposals(); + } + } + + /// Drain accepted runtime work while all peer transports remain live. + /// Failure reports the incomplete drain and retains every repair owner. + pub async fn prepare_runtime_shutdown(&self, timeout: Duration) -> OperationReport { + self.drain_runtime_groups(timeout).await + } + + /// True while accepted shutdown is waiting with repair/communication live. + #[must_use] + pub fn runtime_shutdown_pending(&self) -> bool { + self.groups_snapshot() + .values() + .any(|group| group.runtime_shutdown_pending()) + } + + /// Await actual accepted-prefix progress after a reported timeout. This + /// retains repair owners and suspends the caller without blocking workers. + /// It does not turn an unavailable peer into a successful shutdown. + pub async fn await_runtime_shutdown_ready(&self) { + for group in self.groups_snapshot().values() { + if group.local_replica().learner.engine().requires_contiguous_apply() { + group.drain_runtime_chosen().await; + } + } + } + + pub(super) async fn drain_runtime_groups(&self, timeout: Duration) -> OperationReport { + let groups = self.groups_snapshot(); + for group in groups.values() { + group.close_runtime_proposals(); + } + let mut report = OperationReport::new(); + for group in groups.values() { + if let Err(error) = group.drain_runtime_before_shutdown(timeout).await { + report.push_error(error); + } + } + // No teardown has started: a timeout in any group preserves every + // group's communication and repair owners, including legacy peers. + report + } +} diff --git a/lib/crowdb-kv/src/cluster/watch_registry.rs b/lib/crowdb-paxos-kv/src/cluster/watch_registry.rs similarity index 91% rename from lib/crowdb-kv/src/cluster/watch_registry.rs rename to lib/crowdb-paxos-kv/src/cluster/watch_registry.rs index f63663328..3d5fccc96 100644 --- a/lib/crowdb-kv/src/cluster/watch_registry.rs +++ b/lib/crowdb-paxos-kv/src/cluster/watch_registry.rs @@ -197,11 +197,13 @@ impl WatchRegistry { fn subscribe_with_target(&self, prefix: &[u8], target: PushTarget) -> u64 { let id = self.next_id.fetch_add(1, Ordering::Relaxed); let mut trie = self.trie.write(); - let mut node = &mut trie.root; - for &byte in prefix { - node = node.children.entry(byte).or_insert_with(TrieNode::new); + { + let mut node = &mut trie.root; + for &byte in prefix { + node = node.children.entry(byte).or_insert_with(TrieNode::new); + } + node.watchers.push((id, target)); } - node.watchers.push((id, target)); self.has_watchers.store(true, Ordering::Release); id } @@ -209,8 +211,8 @@ impl WatchRegistry { /// Remove a specific watcher by `(prefix, watcher_id)`. Updates /// `has_watchers` if the registry becomes empty. pub fn unsubscribe(&self, prefix: &[u8], watcher_id: u64) { - let should_check = { - let mut trie = self.trie.write(); + let mut trie = self.trie.write(); + { let mut node = &mut trie.root; for &byte in prefix { match node.children.get_mut(&byte) { @@ -219,11 +221,8 @@ impl WatchRegistry { } } node.watchers.retain(|(id, _)| *id != watcher_id); - node.watchers.is_empty() - }; - if should_check { - self.recompute_has_watchers(); } + self.publish_has_watchers_locked(&trie); } /// Remove all watchers whose `watcher_id` is in the list (stream- @@ -234,17 +233,9 @@ impl WatchRegistry { return; } let id_set: std::collections::HashSet<u64> = watcher_ids.iter().copied().collect(); - let removed = { - let mut trie = self.trie.write(); - let before = count_watchers(&trie.root); - remove_all_from_node(&mut trie.root, &id_set); - let after = count_watchers(&trie.root); - drop(trie); - before != after - }; - if removed { - self.recompute_has_watchers(); - } + let mut trie = self.trie.write(); + remove_all_from_node(&mut trie.root, &id_set); + self.publish_has_watchers_locked(&trie); } /// Clear all watchers (leader step-down). Drops all tx senders, @@ -252,8 +243,7 @@ impl WatchRegistry { pub fn clear(&self) { let mut trie = self.trie.write(); trie.root = TrieNode::new(); - drop(trie); - self.has_watchers.store(false, Ordering::Release); + self.publish_has_watchers_locked(&trie); } /// For a set of changed keys (from `Batch::decode`), find matching @@ -395,14 +385,12 @@ impl WatchRegistry { self.closed_watchers.load(Ordering::Relaxed) } - /// Recompute `has_watchers` by checking if the trie is empty. - /// Called after removals. - fn recompute_has_watchers(&self) { - let empty = { - let trie = self.trie.read(); - trie.root.watchers.is_empty() && trie.root.children.is_empty() - }; - self.has_watchers.store(!empty, Ordering::Release); + /// Publish the fast-path flag while the trie lock is held. All mutation + /// paths update the flag before releasing their write guard, so a clear + /// cannot overwrite a concurrent subscribe's `true` publication. + fn publish_has_watchers_locked(&self, trie: &PrefixTrie) { + let non_empty = !trie.root.watchers.is_empty() || !trie.root.children.is_empty(); + self.has_watchers.store(non_empty, Ordering::Release); } } @@ -426,12 +414,3 @@ fn remove_all_from_node(node: &mut TrieNode, id_set: &std::collections::HashSet< node.children.remove(&byte); } } - -/// Total watcher count across `node` and its children. -fn count_watchers(node: &TrieNode) -> usize { - let mut n = node.watchers.len(); - for child in node.children.values() { - n += count_watchers(child); - } - n -} diff --git a/lib/crowdb-kv/src/common.rs b/lib/crowdb-paxos-kv/src/common.rs similarity index 100% rename from lib/crowdb-kv/src/common.rs rename to lib/crowdb-paxos-kv/src/common.rs diff --git a/lib/crowdb-kv/src/common/config.rs b/lib/crowdb-paxos-kv/src/common/config.rs similarity index 90% rename from lib/crowdb-kv/src/common/config.rs rename to lib/crowdb-paxos-kv/src/common/config.rs index 064475d1b..84d79d68a 100644 --- a/lib/crowdb-kv/src/common/config.rs +++ b/lib/crowdb-paxos-kv/src/common/config.rs @@ -6,9 +6,13 @@ //! //! All values here are compile-time `const`s exposed via `DEFAULT` //! constants and `for_tests` constructors; runtime overrides happen at -//! the call sites (`crowdb-kv-server` CLI, testkit harness) before the +//! the call sites (`crowdb-paxos-kv-server` CLI, testkit harness) before the //! group is wrapped in an `Arc`. +pub use crowdb_tree_ffi::{ + TreeBindingConfig, TreeEngineConfig, TreeHostConfig, TreeResourceConfig, TreeRuntimeConfig, +}; + use std::path::{Path, PathBuf}; use crowdb_common::config::BaseConfig; @@ -19,6 +23,7 @@ use crate::wal::record::WalRecordFormat; /// Paxos retry configuration (all fields static — bind at group /// creation). #[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(default)] pub struct PaxosConfig { /// static: max paxos retries. pub max_paxos_retries: usize, @@ -26,13 +31,9 @@ pub struct PaxosConfig { pub max_slot_retries: usize, /// static: retry base backoff in milliseconds. pub retry_base_backoff_ms: u64, - /// Maximum number of in-flight (allocated-but-not-yet-chosen) proposals - /// static: maximum number of in-flight (allocated-but-not-yet-chosen) - /// proposals the leader admits concurrently. A proposal that cannot - /// acquire a window permit fails fast with `PxPaxosError::Busy` - /// (retryable) rather than blocking, so the leader never stalls - /// behind a saturated pipeline (parallel-slot window / performance - /// targets). + /// static: maximum concurrent proposals, including admitted native apply + /// after an asynchronous Chosen reply. A full window queues or rejects + /// according to `reject_on_inflight_window_full`, before slot allocation. pub max_inflight_proposals: usize, /// static: when `true`, a full inflight window fails fast with /// `ProposeResult::Busy` (test-only). When `false` (default), the @@ -142,6 +143,10 @@ impl Default for ServerConfig { } } +const fn default_true() -> bool { + true +} + const fn default_rpc_workers() -> u32 { ServerConfig::DEFAULT.rpc_workers } @@ -244,6 +249,7 @@ impl Default for WalConfig { /// clocks. Rationale and cross-DC / WAN override profile documented /// in the leader-election sub-design. #[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(default)] pub struct PxElectionConfig { /// Whether `PreVote` rounds protect against partition-rejoin disruption. pub prevote_enabled: bool, @@ -341,7 +347,7 @@ impl PxElectionConfig { /// Aggressive timings for `#[tokio::test(start_paused = true)]` suites. /// /// Heartbeat 5 ms / election 30–60 ms / lease 25 ms. Not exposed on the - /// `crowdb-kv-server` CLI. + /// `crowdb-paxos-kv-server` CLI. #[must_use] pub const fn for_tests() -> Self { Self { @@ -413,15 +419,20 @@ impl Default for PxElectionConfig { /// paths and backends are `#[serde(skip)]` — set from CLI args after /// loading. /// -/// Usage: `crowdb_common::config::load_from_file::<CrowDBConfig>(path)` -/// or `CrowDBConfig::default()`, then CLI args override individual -/// fields, then pass `&CrowDBConfig` to `create_group_with_wal`. +/// Usage: `crowdb_common::config::load_from_file::<PaxosKvConfig>(path)` +/// or `PaxosKvConfig::default()`, then CLI args override individual +/// fields, then pass `&PaxosKvConfig` to `create_group_with_wal`. /// /// All top-level fields are static (bind at startup / group creation). /// Dynamic fields are inside `WalConfig` (GC tunables). #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[allow(clippy::struct_excessive_bools)] -pub struct CrowDBConfig { +pub struct PaxosKvConfig { + /// One explicitly configured CPU/resource host shared by every local group. + #[serde(default)] + pub tree_runtime: TreeRuntimeConfig, + #[serde(default)] + pub tree: TreeEngineConfig, #[serde(default)] pub server: ServerConfig, #[serde(default)] @@ -434,10 +445,10 @@ pub struct CrowDBConfig { #[serde(default)] pub force_classic: bool, /// static: early WAL ack timing. - #[serde(default)] + #[serde(default = "default_true")] pub wal_early_ack: bool, /// static: async engine apply (R35). - #[serde(default)] + #[serde(default = "default_true")] pub async_engine_apply: bool, /// static: WAL root directory. `#[serde(skip)]` — set from /// `--wal-root` CLI. @@ -474,8 +485,9 @@ pub struct CrowDBConfig { pub node_root: Option<PathBuf>, } -impl BaseConfig for CrowDBConfig { +impl BaseConfig for PaxosKvConfig { fn validate(&self) -> Result<(), String> { + self.validate_tree_config()?; if self.server.rpc_workers == 0 { return Err("server.rpc_workers must be > 0".to_string()); } @@ -537,9 +549,11 @@ impl BaseConfig for CrowDBConfig { } } -impl Default for CrowDBConfig { +impl Default for PaxosKvConfig { fn default() -> Self { Self { + tree_runtime: TreeRuntimeConfig::default(), + tree: TreeEngineConfig::default(), server: ServerConfig::DEFAULT, paxos: PaxosConfig::DEFAULT, election: PxElectionConfig::DEFAULT, @@ -564,7 +578,25 @@ impl Default for CrowDBConfig { } } -impl CrowDBConfig { +impl PaxosKvConfig { + /// Validate Tree capacity and Paxos proposal/replay batch compatibility. + /// # Errors + /// Returns the incompatible configuration field before storage is opened. + pub fn validate_tree_config(&self) -> Result<(), String> { + self.tree_runtime.validate()?; + self.tree.validate(&self.tree_runtime)?; + if self.paxos.max_inflight_proposals > self.tree_runtime.binding_options().cpu_queue_limit { + return Err("paxos.max_inflight_proposals exceeds tree_runtime.tree.cpu_queue_limit".to_owned()); + } + if self.paxos.coalesce_max_keys > 64 { + return Err("paxos.coalesce_max_keys exceeds the runtime gap descriptor bound (64)".to_owned()); + } + if self.paxos.coalesce_max_keys > self.tree.tuning.source_max_gap_entries { + return Err("paxos.coalesce_max_keys exceeds tree.tuning.source_max_gap_entries".to_owned()); + } + Ok(()) + } + /// Load from a TOML file. Delegates to /// `crowdb_common::config::load_from_file` (fills skip defaults + /// validates). Kept for backward compatibility with existing @@ -632,19 +664,19 @@ mod tests { use super::*; #[test] - fn crowdb_kv_config_default_round_trip() { - let config = CrowDBConfig::default(); + fn crowdb_paxos_kv_config_default_round_trip() { + let config = PaxosKvConfig::default(); let toml_str = toml::to_string(&config).unwrap(); - let restored: CrowDBConfig = toml::from_str(&toml_str).unwrap(); + let restored: PaxosKvConfig = toml::from_str(&toml_str).unwrap(); assert_eq!(restored.max_inflight(), config.max_inflight()); assert_eq!(restored.wal_early_ack, config.wal_early_ack); assert_eq!(restored.election.election_min_ms, config.election.election_min_ms); } #[test] - fn crowdb_kv_config_partial_toml_uses_defaults() { + fn crowdb_paxos_kv_config_partial_toml_uses_defaults() { let toml_str = "wal_early_ack = true"; - let config: CrowDBConfig = toml::from_str(toml_str).unwrap(); + let config: PaxosKvConfig = toml::from_str(toml_str).unwrap(); assert!(config.wal_early_ack); assert_eq!( config.paxos.max_inflight_proposals, @@ -654,8 +686,8 @@ mod tests { } #[test] - fn crowdb_kv_config_rejects_zero_rpc_workers() { - let mut config = CrowDBConfig::default(); + fn crowdb_paxos_kv_config_rejects_zero_rpc_workers() { + let mut config = PaxosKvConfig::default(); config.server.rpc_workers = 0; assert_eq!( config.validate(), @@ -664,8 +696,8 @@ mod tests { } #[test] - fn crowdb_kv_config_rejects_zero_transport_capacity() { - let mut config = CrowDBConfig::default(); + fn crowdb_paxos_kv_config_rejects_zero_transport_capacity() { + let mut config = PaxosKvConfig::default(); config.server.peer_pool_size = 0; assert_eq!( config.validate(), @@ -682,7 +714,7 @@ mod tests { #[test] fn snapshot_transfer_defaults_are_bounded_and_validate() { - let config = CrowDBConfig::default(); + let config = PaxosKvConfig::default(); assert_eq!(config.server.snapshot_chunk_bytes, 1024 * 1024); assert_eq!(config.server.snapshot_source_sessions, 4); assert_eq!(config.server.snapshot_session_lease_ms, 30_000); @@ -697,7 +729,7 @@ mod tests { ); } - /// The tracked `app/crowdb-kv-server/conf/crowdb_kv_server_config.toml` + /// The tracked `app/crowdb-paxos-kv-server/conf/crowdb_paxos_kv_server_config.toml` /// must parse without error — guards against stale/mismatched /// template edits. #[test] @@ -707,14 +739,14 @@ mod tests { .join("..") .join("..") .join("app") - .join("crowdb-kv-server") + .join("crowdb-paxos-kv-server") .join("conf") - .join("crowdb_kv_server_config.toml"); + .join("crowdb_paxos_kv_server_config.toml"); if !config_path.exists() { // Running from a published crate (no workspace layout). return; } - let config = CrowDBConfig::load_from_file(&config_path).expect("load tracked config"); + let config = PaxosKvConfig::load_from_file(&config_path).expect("load tracked config"); assert_eq!(config.server.shutdown_timeout_ms, 10_000); assert_eq!(config.server.rpc_workers, 2); assert!(config.wal_early_ack); diff --git a/lib/crowdb-kv/src/common/logging.rs b/lib/crowdb-paxos-kv/src/common/logging.rs similarity index 72% rename from lib/crowdb-kv/src/common/logging.rs rename to lib/crowdb-paxos-kv/src/common/logging.rs index e1b3a6fad..b40311f6e 100644 --- a/lib/crowdb-kv/src/common/logging.rs +++ b/lib/crowdb-paxos-kv/src/common/logging.rs @@ -2,20 +2,20 @@ // Licensed under the Apache License, Version 2.0. //! Re-export of the logging primitives now living in `crowdb-common`. -//! The `crowdb_kv`-specific `init_file_logging` / +//! The `crowdb_paxos_kv`-specific `init_file_logging` / //! `init_file_and_console_logging` wrappers preserve the original 4-arg -//! signature and supply the `crowdb_kv` default `EnvFilter` so existing +//! signature and supply the `crowdb_paxos_kv` default `EnvFilter` so existing //! call sites compile unchanged; the implementation moved to //! `crowdb_common::logging` (R12). pub use crowdb_common::logging::{open_metrics_log, LogGuards, RotatingLogWriter}; -/// `crowdb_kv` default `EnvFilter` directive used when `RUST_LOG` is unset. -pub const CROWDB_KV_DEFAULT_FILTER: &str = - "warn,crowdb_kv=info,crowdb_kv_server=info,crowdb_web=info,crowdb_console_shared=info,crowdb_cli=info"; +/// `crowdb_paxos_kv` default `EnvFilter` directive used when `RUST_LOG` is unset. +pub const CROWDB_PAXOS_KV_DEFAULT_FILTER: &str = + "warn,crowdb_paxos_kv=info,crowdb_paxos_kv_server=info,crowdb_web=info,crowdb_console_shared=info,crowdb_cli=info"; /// Initializes file logging to the specified directory using the -/// `crowdb_kv` default `EnvFilter`. Thin wrapper over +/// `crowdb_paxos_kv` default `EnvFilter`. Thin wrapper over /// `crowdb_common::logging::init_file_logging`. /// /// # Errors @@ -31,12 +31,12 @@ pub fn init_file_logging( process_name, max_file_mb, max_files, - CROWDB_KV_DEFAULT_FILTER, + CROWDB_PAXOS_KV_DEFAULT_FILTER, ) } /// Initializes file and console logging to the specified directory using -/// the `crowdb_kv` default `EnvFilter`. Thin wrapper over +/// the `crowdb_paxos_kv` default `EnvFilter`. Thin wrapper over /// `crowdb_common::logging::init_file_and_console_logging`. /// /// # Errors @@ -52,6 +52,6 @@ pub fn init_file_and_console_logging( process_name, max_file_mb, max_files, - CROWDB_KV_DEFAULT_FILTER, + CROWDB_PAXOS_KV_DEFAULT_FILTER, ) } diff --git a/lib/crowdb-kv/src/common/report.rs b/lib/crowdb-paxos-kv/src/common/report.rs similarity index 80% rename from lib/crowdb-kv/src/common/report.rs rename to lib/crowdb-paxos-kv/src/common/report.rs index 62db5d160..017733091 100644 --- a/lib/crowdb-kv/src/common/report.rs +++ b/lib/crowdb-paxos-kv/src/common/report.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. //! Re-export of [`OperationReport`] now living in `crowdb-common`. The -//! module path `crowdb_kv::common::report::OperationReport` is preserved +//! module path `crowdb_paxos_kv::common::report::OperationReport` is preserved //! so existing call sites compile unchanged; the implementation moved //! to `crowdb_common::report` (R12). diff --git a/lib/crowdb-kv/src/common/time.rs b/lib/crowdb-paxos-kv/src/common/time.rs similarity index 81% rename from lib/crowdb-kv/src/common/time.rs rename to lib/crowdb-paxos-kv/src/common/time.rs index 5f05939cf..e350ce618 100644 --- a/lib/crowdb-kv/src/common/time.rs +++ b/lib/crowdb-paxos-kv/src/common/time.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. //! Re-export of the monotonic-time helpers now living in `crowdb-common`. -//! The module path `crowdb_kv::common::time::*` is preserved so existing +//! The module path `crowdb_paxos_kv::common::time::*` is preserved so existing //! call sites compile unchanged; the implementation moved to //! `crowdb_common::time` (R12). diff --git a/lib/crowdb-kv/src/io.rs b/lib/crowdb-paxos-kv/src/io.rs similarity index 100% rename from lib/crowdb-kv/src/io.rs rename to lib/crowdb-paxos-kv/src/io.rs diff --git a/lib/crowdb-kv/src/io/async_file.rs b/lib/crowdb-paxos-kv/src/io/async_file.rs similarity index 100% rename from lib/crowdb-kv/src/io/async_file.rs rename to lib/crowdb-paxos-kv/src/io/async_file.rs diff --git a/lib/crowdb-kv/src/kv.rs b/lib/crowdb-paxos-kv/src/kv.rs similarity index 84% rename from lib/crowdb-kv/src/kv.rs rename to lib/crowdb-paxos-kv/src/kv.rs index a2ff73836..31b7b8708 100644 --- a/lib/crowdb-kv/src/kv.rs +++ b/lib/crowdb-paxos-kv/src/kv.rs @@ -20,15 +20,22 @@ //! in later phases. `InMemKV` lives in `tests/` as a test-only reference //! implementation. +mod admission; mod crowdb_tree_engine; mod kv_engine; mod kv_future; +mod maintenance; mod op; +pub use admission::{AdmittedApply, ApplyAdmissionError, ApplyAdmissionResult}; pub use crowdb_tree_engine::{CrowdbTreeBackend, CrowdbTreeConfig, CrowdbTreeEngine, CrowdbTreeStats}; pub use kv_engine::{ KVEngine, ScanDirection, SnapshotChunk, SnapshotExporter, SnapshotFormat, SnapshotImporter, SnapshotMetadata, SnapshotViewEntry, }; pub use kv_future::KVFuture; +pub use maintenance::{ + checkpoint_engine, flush_engine, EngineCheckpointReport, EngineCheckpointWork, EngineFlushOutcome, + EngineFlushReport, +}; pub use op::{Batch, BatchOp, Cell, EngineDiff, Op}; diff --git a/lib/crowdb-paxos-kv/src/kv/admission.rs b/lib/crowdb-paxos-kv/src/kv/admission.rs new file mode 100644 index 000000000..bcb41c62c --- /dev/null +++ b/lib/crowdb-paxos-kv/src/kv/admission.rs @@ -0,0 +1,26 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::KVFuture; + +/// Rejection before a request can become chosen. +#[derive(Debug)] +pub enum ApplyAdmissionError { + /// Finite resource capacity is currently occupied by accepted work. + Busy, + /// The request violates the configured complete-batch bound or format. + Rejected(String), +} + +/// A prepaid apply operation for one exact proposal payload. +/// Dropping it before choosing returns unused resource credit. +pub trait AdmittedApply: Send { + /// Transfer the reserved bytes into the chosen slot's owned allocations. + /// + /// # Errors + /// Reports a local terminal storage failure, never a consensus rejection. + fn apply(self: Box<Self>, slot: u64) -> KVFuture<Result<(), String>>; +} + +/// `None` selects an engine's compatibility apply path. +pub type ApplyAdmissionResult = Result<Option<Box<dyn AdmittedApply>>, ApplyAdmissionError>; diff --git a/lib/crowdb-kv/src/kv/crowdb_tree_engine.rs b/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine.rs similarity index 56% rename from lib/crowdb-kv/src/kv/crowdb_tree_engine.rs rename to lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine.rs index 78a8fd78a..d2f3e518f 100644 --- a/lib/crowdb-kv/src/kv/crowdb_tree_engine.rs +++ b/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine.rs @@ -7,39 +7,38 @@ #[cfg(feature = "test-util")] use super::op::Cell; use super::{ - Batch, KVEngine, KVFuture, Op, ScanDirection, SnapshotChunk, SnapshotExporter, SnapshotFormat, + Batch, KVEngine, KVFuture, ScanDirection, SnapshotChunk, SnapshotExporter, SnapshotFormat, SnapshotImporter, SnapshotMetadata, SnapshotViewEntry, }; use bytes::Bytes; -use crowdb_tree_ffi::{ - AsyncCrowdbtree, Crowdbtree, CtError, ExtOp, GetOutcome, PinnedGetOutcome, ScanOutcome, -}; +use crowdb_tree_ffi::{AsyncCrowdbtree, Crowdbtree, CtError, GetOutcome, OwnedGetOutcome, ScanOutcome}; use std::sync::Arc; +mod admission; +mod maintenance; +mod metrics; + pub use crowdb_tree_ffi::Config as CrowdbTreeConfig; pub use crowdb_tree_ffi::PageStoreBackend as CrowdbTreeBackend; pub use crowdb_tree_ffi::Stats as CrowdbTreeStats; /// `KVEngine` backed by [`crowdb_tree_ffi::Crowdbtree`], via [`AsyncCrowdbtree`]. /// -/// `KVEngine::get`/`scan`/`apply` return [`KVFuture`]. `get` and `scan` both -/// genuinely construct [`KVFuture::Pending`] for a cold-leaf miss -/// , via [`AsyncCrowdbtree::try_get`]/ -/// [`AsyncCrowdbtree::try_scan`] respectively -- a resident hit/miss -/// (`GetOutcome`/`ScanOutcome`'s `Ready` variant, the overwhelmingly common -/// case) still costs nothing beyond the enum tag, same as before this -/// wiring landed. `apply` stays [`KVFuture::ready`]-only: crowdb-tree has no -/// `ct_apply_*_async` C API yet (only `ct_get_async`/`ct_scan_async`/ -/// `ct_flush_async`/`ct_snapshot_async` exist), so it can't -/// genuinely wait on the reactor today; a synchronous crowdb-tree call in its -/// place is an unchanged, honest reflection of what the C API actually -/// offers, not a shortcut taken here. +/// Runtime proposals acquire an owned native permit before consensus chooses +/// a slot. The permit's apply future awaits prepaid native continuations. +/// Reads dispatch through the shared native runtime and may complete on the +/// first poll or await CPU/backend progress. The +/// compatibility `apply` entry remains available for engines without permits. /// -/// **`iter_all`/`compare` note:** `iter_all` uses `scan(b"", 0, true)` -/// which merges L0+L1 internally and includes tombstones, so every `apply` -/// is visible immediately without an explicit `flush()`, matching `InMemKV`. +/// Test iteration scans L0 and L1 with tombstones included, so successful +/// writes are visible without an explicit `flush()`, matching `InMemKV`. pub struct CrowdbTreeEngine { + // Stop the collector before closing the native Tree. + work_collector: Option<crowdb_tree_ffi::TreeWorkCollector>, + work_window: arc_swap::ArcSwapOption<crowdb_tree_ffi::TreeWorkSnapshot>, inner: AsyncCrowdbtree, + has_backend: bool, + maintenance_pending: Arc<std::sync::atomic::AtomicU64>, } impl CrowdbTreeEngine { @@ -50,8 +49,14 @@ impl CrowdbTreeEngine { /// Returns the underlying [`CtError`] if the engine fails to open (e.g. a /// corrupt or unreadable durable file). pub fn open(opt: &CrowdbTreeConfig) -> Result<Self, CtError> { + let inner = AsyncCrowdbtree::open(opt)?; + let work_collector = Some(inner.handle().work_metrics_collector()?); Ok(Self { - inner: AsyncCrowdbtree::open(opt)?, + work_collector, + work_window: arc_swap::ArcSwapOption::empty(), + inner, + has_backend: opt.page_store.is_some() || opt.path.as_deref().is_some_and(|path| !path.is_empty()), + maintenance_pending: Arc::new(std::sync::atomic::AtomicU64::new(0)), }) } @@ -64,12 +69,10 @@ impl CrowdbTreeEngine { self.inner.handle() } - /// Batched diagnostics snapshot: durable/GC - /// watermarks, `io_failed`, last-snapshot page/segment counts, and - /// buffer-pool occupancy/hit-rate counters. Engine-specific (like - /// [`Self::handle`]) rather than on the generic [`KVEngine`] trait -- - /// `InMemKV` has no comparable internals to report. O(1); safe to poll - /// periodically for metrics/console display. + /// Compatibility diagnostics from Core publication and cache counters. + /// Sampling does not scan pool frames; historical field names remain for + /// callers of this schema. Use `handle().state_stats()` and the shared work + /// metrics window for the full runtime statistics. #[must_use] pub fn stats(&self) -> CrowdbTreeStats { self.inner.handle().stats() @@ -116,10 +119,9 @@ impl CrowdbTreeEngine { #[cfg(feature = "test-util")] #[must_use] pub fn iter_all_for_tests(&self) -> Vec<(Vec<u8>, u64, Cell)> { - // Use scan(b"", 0, true) which merges L0+L1 internally and includes - // tombstones — no flush() needed, matching InMemKV's immediate - // visibility for both live entries and tombstones. Scan returns - // Bytes; iter_all's Cell::Value(Vec<u8>) shape requires .to_vec(). + // Scan the complete L0+L1 keyspace, including tombstones. No flush + // is needed for visibility. The test-only Cell::Value(Vec<u8>) shape + // copies the returned shared Bytes. match self.inner.handle().scan(b"", b"", b"", 0, 0, false, 0, true) { Ok((entries, _)) => entries .into_iter() @@ -142,43 +144,22 @@ impl KVEngine for CrowdbTreeEngine { self } + fn requires_contiguous_apply(&self) -> bool { + true + } + + fn admit_proposal(&self, payload: Bytes) -> KVFuture<super::ApplyAdmissionResult> { + KVFuture::ready(admission::prepare(&self.inner.handle(), &payload)) + } + fn apply(&self, slot: u64, batch: &Batch) -> KVFuture<Result<(), String>> { - if batch.ops.is_empty() { - return KVFuture::ready(Ok(())); - } - // R30 zero-copy apply: the value Bytes are borrowed by crowdb-tree via - // kExternal buffers (no value memcpy on the apply critical path); the - // copy is deferred to MemTable drain (flush, off the critical path). - // Bytes::clone is an O(1) Arc ref-bump, so building ExtOps from the - // borrowed Batch is zero-copy. crowdb-tree enforces per-key - // highest-slot-wins internally, same contract InMemKV enforces itself, - // via one atomic multi-key apply so a concurrent reader never observes - // a partially-applied batch. - let ops: Vec<ExtOp> = batch - .ops - .iter() - .map(|b| match &b.op { - Op::Put(v) => ExtOp::Put { - key: b.key.clone(), - value: v.clone(), - }, - Op::Delete => ExtOp::Delete { key: b.key.clone() }, - }) - .collect(); - let result = self - .inner - .handle() - .apply_batch_external(slot, ops) - .map_err(|e| e.to_string()); - KVFuture::ready(result) + admission::replay(self.inner.handle(), slot, batch) } fn get(&self, key: &[u8]) -> KVFuture<Option<(u64, Vec<u8>)>> { - // AsyncCrowdbtree::try_get does the same fast-path check - // Crowdbtree::get itself does, first -- a resident hit/miss resolves - // right here, no allocation, same as the old always-`Ready` body. - // Only a genuine demand-load miss reaches `Pending`, wrapping the - // reactor-driven future `try_get` already built for us. + // Use the first native completion poll when ready. Resident reads + // can also await shared CPU dispatch; native state/result allocations + // remain charged in either case. match self.inner.try_get(key) { GetOutcome::Ready(result) => KVFuture::ready(result.ok().flatten()), GetOutcome::Pending(fut) => KVFuture::Pending(Box::pin(async move { fut.await.ok().flatten() })), @@ -186,39 +167,19 @@ impl KVEngine for CrowdbTreeEngine { } fn get_bytes(&self, key: &[u8]) -> KVFuture<Option<(u64, Bytes)>> { - // try_get_pinned: fast path returns a PinnedValue borrowing directly - // from the C++ frame. into_bytes() creates a Bytes backed by the - // C++ frame via Bytes::from_owner — zero-copy, no copy_from_slice. - // The page refcount pins stay alive until the Bytes is dropped (on - // any thread, R6). Slow path is identical to get's Pending arm. - match self.inner.try_get_pinned(key) { - PinnedGetOutcome::Ready(result) => { - let mapped = result - .ok() - .flatten() - .map(|(slot, pinned)| (slot, pinned.into_bytes())); - KVFuture::ready(mapped) + match self.inner.try_get_owned(key) { + OwnedGetOutcome::Ready(result) => KVFuture::ready(result.ok().flatten()), + OwnedGetOutcome::Pending(future) => { + KVFuture::Pending(Box::pin(async move { future.await.ok().flatten() })) } - PinnedGetOutcome::Pending(fut) => KVFuture::Pending(Box::pin(async move { - fut.await - .ok() - .flatten() - .map(|(slot, vec)| (slot, Bytes::from(vec))) - })), } } fn get_versioned(&self, key: &[u8]) -> KVFuture<Result<Option<(u64, Bytes)>, String>> { - match self.inner.try_get_pinned(key) { - PinnedGetOutcome::Ready(result) => KVFuture::ready( - result - .map(|value| value.map(|(slot, pinned)| (slot, pinned.into_bytes()))) - .map_err(|error| error.to_string()), - ), - PinnedGetOutcome::Pending(fut) => KVFuture::Pending(Box::pin(async move { - fut.await - .map(|value| value.map(|(slot, bytes)| (slot, Bytes::from(bytes)))) - .map_err(|error| error.to_string()) + match self.inner.try_get_owned(key) { + OwnedGetOutcome::Ready(result) => KVFuture::ready(result.map_err(|error| error.to_string())), + OwnedGetOutcome::Pending(future) => KVFuture::Pending(Box::pin(async move { + future.await.map_err(|error| error.to_string()) })), } } @@ -234,24 +195,12 @@ impl KVEngine for CrowdbTreeEngine { deadline_ms: u64, direction: ScanDirection, ) -> KVFuture<Result<(Vec<(Bytes, u64, Bytes)>, bool), String>> { - // start_after is pushed down into the C++ engine: the descent targets - // the leaf containing start_after (instead of the prefix start), and - // the merge loop skips keys <= start_after natively, so the engine - // applies the limit without over-fetching the prefix range. - // end_key is likewise pushed down: the merge loop early-stops once - // the winner key reaches end_key, so the engine never over-reads - // past the upper bound. - // byte_budget is likewise pushed down: the C++ merge loop accumulates - // key+value bytes and stops with truncated when the budget is - // exceeded, always returning at least one entry. - // keys_only is pushed down: the consider lambda skips value assembly - // (no overflow-chain walk) and stages empty values; the byte budget - // then accounts for key bytes only. - // deadline_ms is pushed down: the merge loop checks periodically - // (every 1024 entries) and breaks early with truncated when exceeded. - // ScanEntry holds zero-copy Bytes slices into the packed buffer, - // so decode_scan just re-packages them — no per-entry copy. - // FFI errors propagate as Err instead of being swallowed. + // Native traversal applies the direction, cursor, end, count and + // byte/deadline limits across bounded internal windows. Keys-only + // scans skip overflow value assembly. Non-strict byte limits allow + // one oversized first record when resource admission permits it. + // ScanEntry retains shared slices of the charged packed buffer; + // decode_scan re-packages them and propagates native errors. let prefix_owned = prefix.to_vec(); let start_after_owned = start_after.to_vec(); let end_key_owned = end_key.to_vec(); @@ -275,24 +224,8 @@ impl KVEngine for CrowdbTreeEngine { } } - fn clear(&self) { - // `Crowdbtree::clear` (crowdb-tree/ffi) wraps the new `ct_clear` C API - // entry point, which duplicates the same wipe sequence (epoch-safe - // resident-page retire + fresh empty root + memtable/watermark - // reset) `Crowdbtree::install_snapshot` already performs on a live - // tree before loading imported entries -- not a bespoke reset. - // Not durable by itself: a caller that needs the wipe to survive a - // crash must still call `persist_snapshot`/`handle.flush` - // afterward, same as snapshot import's own contract. - // - // `unwrap`: the only failure mode `Crowdbtree::clear` has is an - // invalid-argument on a null tree pointer, which can't happen - // through this safe wrapper -- matches this trait method's - // infallible signature. - self.inner - .handle() - .clear() - .expect("CrowdbTreeEngine::clear should never fail through the safe FFI wrapper"); + fn clear(&self) -> Result<(), String> { + self.inner.handle().clear().map_err(|error| error.to_string()) } fn is_healthy(&self) -> bool { @@ -316,6 +249,31 @@ impl KVEngine for CrowdbTreeEngine { } } + fn flush_async(&self) -> Option<KVFuture<Result<super::EngineFlushReport, String>>> { + Some(self.async_flush()) + } + + fn writeback_pending(&self) -> bool { + self.has_backend + && self + .inner + .handle() + .state_stats() + .is_ok_and(|state| state.logical.writeback_candidate_pages != 0) + } + + fn writeback_async(&self) -> Option<KVFuture<Result<super::EngineCheckpointReport, String>>> { + Some(self.async_checkpoint(false)) + } + + fn persist_snapshot_async(&self) -> Option<KVFuture<Result<super::EngineCheckpointReport, String>>> { + Some(self.async_checkpoint(true)) + } + + fn apply_noop(&self, slot: u64) -> KVFuture<Result<(), String>> { + admission::replay_empty(self.inner.handle(), slot) + } + fn noop(&self, slot: u64) { // Record this slot without declaring earlier, still-in-flight slots // applied. Forced advancement is reserved for recovery initialization. @@ -326,37 +284,24 @@ impl KVEngine for CrowdbTreeEngine { } fn flush_pending(&self) -> bool { - // Gap 5 step 2: check if any frozen memtables are waiting to be - // drained. `frozen_table_count()` returns the current queue depth; - // after `flush()` drains them all, it returns 0. This is a cheap - // shared-lock read, safe to poll after every maintenance pass. - self.inner.handle().frozen_table_count() > 0 + self.maintenance_pending + .load(std::sync::atomic::Ordering::Acquire) + & 1 + != 0 + || self.inner.handle().frozen_table_count() > 0 } fn resume_from_slot(&self) -> u64 { - // `Crowdbtree::last_applied_slot` is `contiguous_slot_` as of the last - // `flush` (see `Crowdbtree::apply`/`flush`/`recompute_contiguous_locked` - // in crowdb-tree.cpp) -- a durable, gap-free watermark, not just "the max - // slot ever seen". On a fresh in-memory engine (`path: None`) or an - // empty durable file this is `0`. On a recovered durable file it's - // whatever the on-disk superblock last recorded (`persist.cpp`), - // which only advances via an explicit `snapshot`/persist -- a - // conservative floor if no such call happened right before the - // process that wrote this file exited, which is fine (see the trait - // doc: under-reporting only means more, still-safe, replay work). - self.inner.handle().last_applied_slot() + // Recovery/WAL retention must use the committed checkpoint frontier. + // Publication can advance in memory without any durable backend. + self.inner.handle().durable_coverage() } fn persist_snapshot(&self) -> u64 { - // O7: callers must flush() before calling persist_snapshot() so L0 - // is already drained into L1. The maintenance loop flushes in step 1 - // before calling this in step 2; the shutdown path - // (local_replica.rs) also flushes first. This avoids a redundant - // write_mutex_ acquire inside persist_snapshot when the caller - // already flushed — the snapshot path itself does NOT hold - // write_mutex_ (it walks L1 pages which are immutable once - // published), so pipelining is possible: the next tick's flush can - // overlap with this snapshot's disk I/O. + // Synchronous compatibility entry: wait for the native checkpoint, + // which captures a finite handoff and commits its durable frontier. + // Native async maintenance uses checkpoint_report instead. A prior + // flush may advance publication but does not establish durability. let snap_t0 = std::time::Instant::now(); let snap_result = self.inner.handle().snapshot(); let snap_ms = snap_t0.elapsed().as_millis(); @@ -371,6 +316,17 @@ impl KVEngine for CrowdbTreeEngine { self.inner.handle().set_gc_watermark(snapshot_slot, safe_slot); } + fn compact_sparse_blocks_async(&self) -> Option<KVFuture<Result<(), String>>> { + let tree = self.inner.clone(); + Some(KVFuture::Pending(Box::pin(async move { + tree.compact_sparse_blocks() + .await + .map_err(|error| error.to_string())? + .status + .map_err(|error| error.to_string()) + }))) + } + fn compact_sparse_blocks(&self) -> Result<(), String> { self.inner .handle() @@ -401,7 +357,10 @@ impl KVEngine for CrowdbTreeEngine { fn snapshot_view(&self) -> Result<(u64, Vec<SnapshotViewEntry>), String> { // Flush first so the view reflects every `apply` up to now, not // just whatever an earlier `flush` already moved into L1. - let _ = self.inner.handle().flush(); + self.inner + .handle() + .flush() + .map_err(|error| format!("flush before snapshot_view: {error:?}"))?; let (at_slot, entries) = self .inner .handle() @@ -445,7 +404,7 @@ impl SnapshotExporter for CrowdbTreeSnapshotExporter { fn read(&mut self, offset: u64) -> Result<SnapshotChunk, String> { self.session - .read(offset) + .read_owned(offset) .map(|chunk| SnapshotChunk { offset: chunk.offset, bytes: chunk.bytes, diff --git a/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/admission.rs b/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/admission.rs new file mode 100644 index 000000000..5cfaad769 --- /dev/null +++ b/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/admission.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_tree_ffi::{ArmedApply, BatchOp as NativeOp, Crowdbtree, CtError, ExtOp}; + +use crate::kv::{AdmittedApply, ApplyAdmissionError, ApplyAdmissionResult, Batch, KVFuture, Op}; + +struct NativeAdmission(ArmedApply); + +impl AdmittedApply for NativeAdmission { + fn apply(self: Box<Self>, slot: u64) -> KVFuture<Result<(), String>> { + let operation = self.0; + KVFuture::Pending(Box::pin(async move { + operation.apply(slot).await.map_err(|error| error.to_string()) + })) + } +} + +pub(super) fn prepare(tree: &Arc<Crowdbtree>, payload: &Bytes) -> ApplyAdmissionResult { + let batch = + Batch::try_decode(payload).map_err(|error| ApplyAdmissionError::Rejected(error.to_owned()))?; + let operations: Vec<_> = batch + .ops + .iter() + .map(|entry| match &entry.op { + Op::Put(value) => NativeOp::Put { + key: &entry.key, + value, + }, + Op::Delete => NativeOp::Delete { key: &entry.key }, + }) + .collect(); + let prepared = tree.prepare_proposal(&operations).map_err(admission_error)?; + let armed = prepared.arm().map_err(admission_error)?; + Ok(Some(Box::new(NativeAdmission(armed)))) +} + +fn admission_error(error: CtError) -> ApplyAdmissionError { + match error { + CtError::Unavailable => ApplyAdmissionError::Busy, + other => ApplyAdmissionError::Rejected(format!("native apply admission: {other}")), + } +} + +pub(super) fn replay(tree: Arc<Crowdbtree>, slot: u64, batch: &Batch) -> KVFuture<Result<(), String>> { + let operations = batch + .ops + .iter() + .map(|entry| match &entry.op { + Op::Put(value) => ExtOp::Put { + key: entry.key.clone(), + value: value.clone(), + }, + Op::Delete => ExtOp::Delete { + key: entry.key.clone(), + }, + }) + .collect(); + KVFuture::Pending(Box::pin(async move { + tree.apply_replay(slot, operations) + .await + .map_err(|error| error.to_string()) + })) +} + +pub(super) fn replay_empty(tree: Arc<Crowdbtree>, slot: u64) -> KVFuture<Result<(), String>> { + KVFuture::Pending(Box::pin(async move { + tree.apply_replay(slot, Vec::new()) + .await + .map_err(|error| error.to_string()) + })) +} diff --git a/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/maintenance.rs b/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/maintenance.rs new file mode 100644 index 000000000..03297d0cf --- /dev/null +++ b/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/maintenance.rs @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use super::CrowdbTreeEngine; +use crate::kv::{EngineCheckpointReport, EngineCheckpointWork, EngineFlushReport, KVFuture}; + +// The low bit retains work after errors or abandonment. A later attempt owns +// a new stamp, so polling an older future cannot erase newer pending work. +fn begin(pending: &AtomicU64) -> u64 { + let previous = pending + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + Some(value.wrapping_add(2) | 1) + }) + .expect("unconditional update"); + previous.wrapping_add(2) | 1 +} +fn complete(pending: &AtomicU64, attempt: u64, success: bool) { + if success { + let _ = pending.compare_exchange(attempt, attempt & !1, Ordering::AcqRel, Ordering::Acquire); + } +} +fn checkpoint_report(report: crowdb_tree_ffi::CheckpointReport) -> EngineCheckpointReport { + EngineCheckpointReport { + status: report.status.map_err(|error| error.to_string()), + durable_committed: report.durable_committed, + durable_coverage: report.durable_coverage, + work: Some(EngineCheckpointWork { + writeback_completed: report.writeback_completed, + captured_coverage: report.captured_coverage, + written_frames: report.written_frames, + reused_frames: report.reused_frames, + rebase_deferred: report.rebase_deferred, + }), + } +} + +impl CrowdbTreeEngine { + pub(super) fn async_flush(&self) -> KVFuture<Result<EngineFlushReport, String>> { + let tree = self.inner.clone(); + let pending = Arc::clone(&self.maintenance_pending); + let attempt = begin(&pending); + KVFuture::Pending(Box::pin(async move { + let report = tree.flush_report().await.map_err(|error| error.to_string())?; + complete( + &pending, + attempt, + report.status.is_ok() && report.deferred_pages == 0, + ); + Ok(EngineFlushReport { + handoff_status: report.handoff_status.map_err(|error| error.to_string()), + maintenance_status: report.maintenance_status.map_err(|error| error.to_string()), + handoff_completed: report.handoff_completed, + handoff_advanced: report.handoff_advanced, + floor: report.floor, + frontier: report.frontier, + published_coverage: report.published_coverage, + deferred_pages: report.deferred_pages, + }) + })) + } + + pub(super) fn async_checkpoint(&self, durable: bool) -> KVFuture<Result<EngineCheckpointReport, String>> { + if durable && !self.has_backend { + // A volatile engine has no durability barrier. Preserve the trait's + // zero-coverage result without invoking unsupported native persistence. + return KVFuture::ready(Ok(EngineCheckpointReport { + status: Ok(()), + durable_committed: false, + durable_coverage: 0, + work: None, + })); + } + let tree = self.inner.clone(); + let pending = Arc::clone(&self.maintenance_pending); + let attempt = begin(&pending); + KVFuture::Pending(Box::pin(async move { + let report = if durable { + tree.checkpoint_report().await + } else { + tree.writeback_report().await + } + .map_err(|error| error.to_string())?; + complete( + &pending, + attempt, + report.status.is_ok() && report.rebase_deferred == 0, + ); + Ok(checkpoint_report(report)) + })) + } +} diff --git a/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/metrics.rs b/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/metrics.rs new file mode 100644 index 000000000..d2ec0672a --- /dev/null +++ b/lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine/metrics.rs @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::fmt::Write; +use std::sync::Arc; + +use crowdb_tree_ffi::{ + CtError, TreeWorkSnapshot, TREE_WORK_CAUSE_NAMES, TREE_WORK_COUNTER_NAMES, TREE_WORK_STAGE_NAMES, +}; + +use super::CrowdbTreeEngine; + +impl CrowdbTreeEngine { + /// Called once by the host collector per tick. Consumers read the shared + /// immutable result through `work_metrics_window` without draining again. + /// + /// # Errors + /// Returns `Unavailable` if another capture is already in progress. + pub fn capture_work_metrics(&self) -> Result<Option<Arc<TreeWorkSnapshot>>, CtError> { + let Some(collector) = &self.work_collector else { + return Ok(None); + }; + let snapshot = Arc::new(collector.capture()?); + self.work_window.rcu(|current| { + if matches!(current.as_ref(), Some(old) if old.window_sequence >= snapshot.window_sequence) { + current.clone() + } else { + Some(Arc::clone(&snapshot)) + } + }); + Ok(Some(snapshot)) + } + + /// Returns the latest collector window; sampling this never resets counts. + #[must_use] + pub fn work_metrics_window(&self) -> Option<Arc<TreeWorkSnapshot>> { + self.work_window.load_full() + } + + /// Formats the same immutable window used by diagnostics and other readers. + #[must_use] + pub fn format_work_metrics_window(&self, prefix: &str, timestamp: &str) -> String { + let Some(window) = self.work_metrics_window() else { + return String::new(); + }; + let mut output = String::new(); + for (name, stage) in TREE_WORK_STAGE_NAMES.iter().zip(window.stages) { + if stage.attempts == 0 { + continue; + } + let _ = writeln!( + output, + "{timestamp} {prefix}.tree.work.{name} window={} count={} total={} inflight={} sum_ns={} max_ns={} success={} error={} cancelled={} retry={} stale={} age_sampled_at_ns={} age_inflight={} age_tracked={} age_untracked={} observed_oldest_age_ns={} age_complete={}", + window.window_sequence, + stage.window_count, + stage.total_count, + stage.inflight, + stage.window_sum_ns, + stage.window_max_ns, + stage.outcomes[0], + stage.outcomes[1], + stage.outcomes[2], + stage.outcomes[3], + stage.outcomes[4], + window.age_sampled_at_ns, + stage.age.inflight, + stage.age.tracked, + stage.age.untracked, + stage.age.observed_oldest_age_ns, + stage.age.complete + ); + } + for (index, name) in TREE_WORK_STAGE_NAMES.iter().enumerate() { + if window.histograms.enabled_stages & (1 << index) == 0 { + continue; + } + let histogram = window.histograms.stages[index]; + let _ = writeln!( + output, + "{timestamp} {prefix}.tree.histogram.{name} window={} count={} total={} sum_ns={} min_ns={} max_ns={} p50_ns={} p90_ns={} p95_ns={} p99_ns={} p999_ns={} zero={} overflow={}", + window.histograms.window_sequence, + histogram.count, + histogram.total_count, + histogram.sum_ns, + histogram.min_ns, + histogram.max_ns, + histogram.p50_ns, + histogram.p90_ns, + histogram.p95_ns, + histogram.p99_ns, + histogram.p999_ns, + histogram.zero_count, + histogram.overflow_count + ); + } + for (cause, values) in TREE_WORK_CAUSE_NAMES.iter().zip(window.counters.values) { + for (counter, value) in TREE_WORK_COUNTER_NAMES.iter().zip(values) { + if value != 0 { + let _ = writeln!( + output, + "{timestamp} {prefix}.tree.work.{cause}.{counter} total={value} window={} sampled_at_ns={}", + window.window_sequence, window.counters.sampled_at_ns + ); + } + } + } + for (name, wait) in crowdb_tree_ffi::TREE_WAIT_CLASS_NAMES + .iter() + .zip(window.waits.classes) + { + if wait.attempts == 0 { + continue; + } + let _ = writeln!(output, + "{timestamp} {prefix}.tree.wait.{name} count={} sum_ns={} max_ns={} inflight={} granted={} error={} cancelled={} age_ns={} age_complete={} sampled_at_ns={}", + wait.total_count, wait.total_sum_ns, wait.total_max_ns, wait.inflight, + wait.outcomes[0], wait.outcomes[1], wait.outcomes[2], + wait.age.observed_oldest_age_ns, wait.age.complete, window.waits.sampled_at_ns); + } + output + } +} diff --git a/lib/crowdb-kv/src/kv/kv_engine.rs b/lib/crowdb-paxos-kv/src/kv/kv_engine.rs similarity index 85% rename from lib/crowdb-kv/src/kv/kv_engine.rs rename to lib/crowdb-paxos-kv/src/kv/kv_engine.rs index 67276004e..56423b51a 100644 --- a/lib/crowdb-kv/src/kv/kv_engine.rs +++ b/lib/crowdb-paxos-kv/src/kv/kv_engine.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::kv_future::KVFuture; -use super::Batch; +use super::{Batch, EngineCheckpointReport, EngineFlushReport}; use bytes::Bytes; @@ -28,7 +28,7 @@ pub struct SnapshotMetadata { #[derive(Clone, Debug, PartialEq, Eq)] pub struct SnapshotChunk { pub offset: u64, - pub bytes: Vec<u8>, + pub bytes: Bytes, pub done: bool, } @@ -73,8 +73,8 @@ pub enum ScanDirection { /// demand-load miss, via the `io_uring` reactor) returns a real `Pending` future. [`super::CrowdbTreeEngine::get`] and /// [`super::CrowdbTreeEngine::scan`] both construct `Pending` for a genuine /// cold-leaf miss (via `crowdb_tree_ffi::AsyncCrowdbtree::try_get`/`try_scan`); -/// `InMemKV` and `CrowdbTreeEngine::apply` always resolve `Ready` (no -/// `ct_apply_*_async` C API exists yet). +/// Runtime-backed `CrowdbTreeEngine::apply` returns a retained native future; +/// compatibility engines may complete an in-memory apply immediately. pub trait KVEngine: Send + Sync { /// Apply `batch` at `slot`. Atomic to readers and idempotent: an op for /// key `k` is skipped when `slot <= resolved_slot(k)`. The last occurrence @@ -89,6 +89,27 @@ pub trait KVEngine: Send + Sync { /// I/O path and always returns `Ok()`. fn apply(&self, slot: u64, batch: &Batch) -> KVFuture<Result<(), String>>; + /// Whether a single apply loop must stop at the first unavailable slot. + /// Such engines retain higher-slot work until the contiguous gap completes, + /// so awaiting a higher slot from the only gap-processing loop would deadlock. + fn requires_contiguous_apply(&self) -> bool { + false + } + + /// Reserve the complete apply bound before a payload can enter consensus. + /// `None` keeps the compatibility path for engines without admission. + /// The engine's permit owns this exact payload until commit or cancellation. + fn admit_proposal(&self, _payload: Bytes) -> KVFuture<super::ApplyAdmissionResult> { + KVFuture::ready(Ok(None)) + } + + /// Fallible gap completion. Engines with bounded frontiers must report + /// failure before the learner advances its applied watermark. + fn apply_noop(&self, slot: u64) -> KVFuture<Result<(), String>> { + self.noop(slot); + KVFuture::ready(Ok(())) + } + /// Advance the engine's internal contiguous-slot watermark for a /// `NoOp` slot (empty batch from `repair_once` gap-fill). Without this, /// `contiguous_slot_` (C++) stalls at the `NoOp`, permanently blocking @@ -124,8 +145,8 @@ pub trait KVEngine: Send + Sync { } } - /// Fallible live value lookup used by conditional admission. Unlike the - /// legacy read surface, storage errors must not be interpreted as absence. + /// Fallible live value lookup used by reads and conditional admission. + /// Storage errors must not be interpreted as absence. fn get_versioned(&self, key: &[u8]) -> KVFuture<Result<Option<(u64, Bytes)>, String>> { match self.get_bytes(key) { KVFuture::Ready(v) => KVFuture::ready(Ok(v.flatten())), @@ -194,7 +215,10 @@ pub trait KVEngine: Send + Sync { /// Drop all state. Used by snapshot-install reset (before importing a /// peer's snapshot) and by tests that need to simulate a wiped replica. - fn clear(&self); + /// # Errors + /// Returns an error if accepted work is still active or replacement + /// preparation cannot acquire its resources. The old state remains usable. + fn clear(&self) -> Result<(), String>; /// Type-erased downcast escape hatch. Lets a caller holding only /// `&dyn KVEngine` (e.g. `PxLearner::engine`) recover the concrete @@ -233,6 +257,29 @@ pub trait KVEngine: Send + Sync { /// distinction). fn flush(&self) {} + /// A true async handoff path, when the engine supports one. `None` keeps + /// the host's blocking compatibility adapter. Admission failures are outer + /// errors; accepted phase errors retain their report and progress. + fn flush_async(&self) -> Option<KVFuture<Result<EngineFlushReport, String>>> { + None + } + + /// Whether local pages currently need reloadable backing. + fn writeback_pending(&self) -> bool { + false + } + + /// Materialize reloadable local pages without advancing durable coverage. + fn writeback_async(&self) -> Option<KVFuture<Result<EngineCheckpointReport, String>>> { + None + } + + /// A true async checkpoint. A committed durable frontier remains usable + /// even if the report also contains a later reconciliation error. + fn persist_snapshot_async(&self) -> Option<KVFuture<Result<EngineCheckpointReport, String>>> { + None + } + /// Highest slot `S` such that every slot in `[1, S]` is durably reflected /// in this engine already — i.e. a caller rebuilding state from a WAL can /// safely skip re-`apply`ing slots `<= S` and start at `S + 1`. Must be @@ -282,6 +329,11 @@ pub trait KVEngine: Send + Sync { let _ = (snapshot_slot, safe_slot); } + /// Optional native continuation path. `None` selects the compatibility host worker. + fn compact_sparse_blocks_async(&self) -> Option<KVFuture<Result<(), String>>> { + None + } + /// Cadence-driven block compaction (R129). Selects sparse source blocks, /// relocates their live extents through a snapshot, and deletes blocks /// unreachable from any retained anchor. Non-block stores and disabled diff --git a/lib/crowdb-kv/src/kv/kv_future.rs b/lib/crowdb-paxos-kv/src/kv/kv_future.rs similarity index 82% rename from lib/crowdb-kv/src/kv/kv_future.rs rename to lib/crowdb-paxos-kv/src/kv/kv_future.rs index 446afdc4f..ebf62c8c5 100644 --- a/lib/crowdb-kv/src/kv/kv_future.rs +++ b/lib/crowdb-paxos-kv/src/kv/kv_future.rs @@ -36,13 +36,9 @@ impl<T> KVFuture<T> { KVFuture::Ready(Some(v)) } - /// Unwrap a `KVFuture` that is known, by construction, to never be - /// `Pending` — true for every [`super::KVEngine`] call site in this - /// codebase today, since no impl constructs `Pending` yet (no reactor - /// exists). Panics on `Pending` so the day a real `Pending` future shows - /// up here, it's a loud, unmissable signal that this call site now needs - /// converting to real `async`/`.await` instead of a silent wrong answer - /// or a hang. The deferred caller-side conversion this panic is meant to trigger. + /// Unwrap a `KVFuture` whose implementation guarantees `Ready`. + /// Engines that perform asynchronous admission or I/O may return + /// `Pending`; their callers must await the result instead. /// /// # Panics /// Panics if `self` is [`KVFuture::Pending`]. diff --git a/lib/crowdb-paxos-kv/src/kv/maintenance.rs b/lib/crowdb-paxos-kv/src/kv/maintenance.rs new file mode 100644 index 000000000..4801d2d4e --- /dev/null +++ b/lib/crowdb-paxos-kv/src/kv/maintenance.rs @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::sync::Arc; + +use super::KVEngine; + +/// Terminal progress from one accepted handoff and its maintenance phase. +#[derive(Debug)] +pub struct EngineFlushReport { + pub handoff_status: Result<(), String>, + pub maintenance_status: Result<(), String>, + pub handoff_completed: bool, + pub handoff_advanced: bool, + pub floor: u64, + pub frontier: u64, + pub published_coverage: u64, + pub deferred_pages: u64, +} +impl EngineFlushReport { + /// A maintenance error retains any successful handoff progress. + /// # Errors + /// Returns the first unsuccessful phase status. + pub fn status(&self) -> Result<(), &str> { + self.handoff_status.as_ref().map_err(String::as_str)?; + self.maintenance_status.as_ref().map_err(String::as_str).copied() + } +} + +/// Accepted checkpoint progress remains meaningful after post-commit errors. +#[derive(Debug)] +pub struct EngineCheckpointReport { + pub status: Result<(), String>, + pub durable_committed: bool, + pub durable_coverage: u64, + pub work: Option<EngineCheckpointWork>, +} + +/// Local materialization details, absent for a compatibility-only engine. +#[derive(Debug)] +pub struct EngineCheckpointWork { + pub writeback_completed: bool, + pub captured_coverage: u64, + pub written_frames: u64, + pub reused_frames: u64, + pub rebase_deferred: u64, +} + +/// The compatibility path has no per-stage progress to report. +#[derive(Debug)] +pub enum EngineFlushOutcome { + LegacyCompleted, + Reported(EngineFlushReport), +} +impl EngineFlushOutcome { + #[must_use] + pub fn advances_flush_count(&self) -> bool { + match self { + Self::LegacyCompleted => true, + Self::Reported(report) => report.handoff_advanced, + } + } + /// # Errors + /// Returns the first unsuccessful phase status. + pub fn status(&self) -> Result<(), &str> { + match self { + Self::LegacyCompleted => Ok(()), + Self::Reported(report) => report.status(), + } + } +} + +/// Await native maintenance where available; preserve the existing blocking +/// adapter for engines that only expose the compatibility API. +/// # Errors +/// Returns an admission error or a failed compatibility worker. +pub async fn flush_engine(engine: Arc<dyn KVEngine>) -> Result<EngineFlushOutcome, String> { + if let Some(future) = engine.flush_async() { + return future.await.map(EngineFlushOutcome::Reported); + } + let span = tracing::Span::current(); + tokio::task::spawn_blocking(move || span.in_scope(|| engine.flush())) + .await + .map_err(|error| format!("engine flush task failed: {error}"))?; + Ok(EngineFlushOutcome::LegacyCompleted) +} + +/// Await a checkpoint while retaining any committed durable progress. +/// +/// # Errors +/// Returns an admission error or a failed compatibility worker. +pub async fn checkpoint_engine(engine: Arc<dyn KVEngine>) -> Result<EngineCheckpointReport, String> { + if let Some(future) = engine.persist_snapshot_async() { + return future.await; + } + let span = tracing::Span::current(); + let coverage = tokio::task::spawn_blocking(move || span.in_scope(|| engine.persist_snapshot())) + .await + .map_err(|error| format!("engine checkpoint task failed: {error}"))?; + Ok(EngineCheckpointReport { + status: Ok(()), + durable_committed: coverage > 0, + durable_coverage: coverage, + work: None, + }) +} diff --git a/lib/crowdb-kv/src/kv/op.rs b/lib/crowdb-paxos-kv/src/kv/op.rs similarity index 59% rename from lib/crowdb-kv/src/kv/op.rs rename to lib/crowdb-paxos-kv/src/kv/op.rs index 9e1014c17..b43061de2 100644 --- a/lib/crowdb-kv/src/kv/op.rs +++ b/lib/crowdb-paxos-kv/src/kv/op.rs @@ -40,6 +40,39 @@ pub struct EngineDiff { } impl Batch { + /// Validate an entire consensus payload before admitting or applying it. + /// Empty payloads represent gap-repair `NoOp` entries. + /// + /// # Errors + /// Rejects truncated fields, unknown operations, nonempty delete values, + /// and bytes following the declared operation count. + pub fn try_decode(payload: &Bytes) -> Result<Self, &'static str> { + if payload.is_empty() { + return Ok(Self::default()); + } + let count = payload.get(..2).ok_or("truncated batch operation count")?; + let count = usize::from(u16::from_le_bytes([count[0], count[1]])); + let mut cursor = 2; + let mut ops = Vec::with_capacity(count); + for _ in 0..count { + let kind = *payload.get(cursor).ok_or("truncated operation kind")?; + cursor += 1; + let key = read_field(payload, &mut cursor)?; + let value = read_field(payload, &mut cursor)?; + let op = match kind { + 0 => Op::Put(value), + 1 if value.is_empty() => Op::Delete, + 1 => return Err("delete operation carries a value"), + _ => return Err("unknown batch operation kind"), + }; + ops.push(BatchOp { key, op }); + } + if cursor != payload.len() { + return Err("trailing bytes after batch operations"); + } + Ok(Self { ops }) + } + /// Decode the minimal KV payload emitted by /// `PxKvStore::encode_kv_payload` / `encode_kv_batch_items`. /// @@ -97,3 +130,17 @@ fn read_u32_le(buf: &Bytes, offset: usize) -> u32 { let b = buf.get(offset..offset + 4).unwrap_or(&[0; 4]); u32::from_le_bytes([b[0], b[1], b[2], b[3]]) } + +fn read_field(payload: &Bytes, cursor: &mut usize) -> Result<Bytes, &'static str> { + let end = cursor.checked_add(4).ok_or("batch field length overflow")?; + let length = payload.get(*cursor..end).ok_or("truncated batch field length")?; + let length = u32::from_le_bytes([length[0], length[1], length[2], length[3]]) as usize; + *cursor = end; + let end = cursor.checked_add(length).ok_or("batch field length overflow")?; + if end > payload.len() { + return Err("truncated batch field bytes"); + } + let field = payload.slice(*cursor..end); + *cursor = end; + Ok(field) +} diff --git a/lib/crowdb-kv/src/lib.rs b/lib/crowdb-paxos-kv/src/lib.rs similarity index 95% rename from lib/crowdb-kv/src/lib.rs rename to lib/crowdb-paxos-kv/src/lib.rs index 65ea7a562..8ff010b88 100644 --- a/lib/crowdb-kv/src/lib.rs +++ b/lib/crowdb-paxos-kv/src/lib.rs @@ -3,7 +3,7 @@ //! `CrowDB` core library. //! -//! All business logic lives here as modules. Binaries (`crowdb-kv-server`) +//! All business logic lives here as modules. Binaries (`crowdb-paxos-kv-server`) //! are thin wrappers that wire configuration and CLI parsing. #![allow(clippy::mod_module_files)] diff --git a/lib/crowdb-kv/src/metrics.rs b/lib/crowdb-paxos-kv/src/metrics.rs similarity index 92% rename from lib/crowdb-kv/src/metrics.rs rename to lib/crowdb-paxos-kv/src/metrics.rs index ad54d0fb5..eeaf47a17 100644 --- a/lib/crowdb-kv/src/metrics.rs +++ b/lib/crowdb-paxos-kv/src/metrics.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. //! Re-export of the generic metrics primitives now living in -//! `crowdb-common`. The module path `crowdb_kv::metrics::*` is preserved so +//! `crowdb-common`. The module path `crowdb_paxos_kv::metrics::*` is preserved so //! existing call sites compile unchanged; the implementation moved to //! `crowdb_common::metrics` (R12). diff --git a/lib/crowdb-kv/src/paxos.rs b/lib/crowdb-paxos-kv/src/paxos.rs similarity index 100% rename from lib/crowdb-kv/src/paxos.rs rename to lib/crowdb-paxos-kv/src/paxos.rs diff --git a/lib/crowdb-kv/src/paxos/acceptor.rs b/lib/crowdb-paxos-kv/src/paxos/acceptor.rs similarity index 100% rename from lib/crowdb-kv/src/paxos/acceptor.rs rename to lib/crowdb-paxos-kv/src/paxos/acceptor.rs diff --git a/lib/crowdb-kv/src/paxos/error.rs b/lib/crowdb-paxos-kv/src/paxos/error.rs similarity index 100% rename from lib/crowdb-kv/src/paxos/error.rs rename to lib/crowdb-paxos-kv/src/paxos/error.rs diff --git a/lib/crowdb-kv/src/paxos/learner.rs b/lib/crowdb-paxos-kv/src/paxos/learner.rs similarity index 93% rename from lib/crowdb-kv/src/paxos/learner.rs rename to lib/crowdb-paxos-kv/src/paxos/learner.rs index 7daab5f33..73d7663de 100644 --- a/lib/crowdb-kv/src/paxos/learner.rs +++ b/lib/crowdb-paxos-kv/src/paxos/learner.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +mod accepted_apply; +use accepted_apply::ApplyTasks; + use std::collections::VecDeque; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; @@ -14,7 +17,10 @@ use parking_lot::Mutex; use std::sync::Barrier; use tokio::sync::Notify; -use crate::kv::{Batch, CrowdbTreeBackend, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; +use crate::kv::{ + AdmittedApply, ApplyAdmissionResult, Batch, CrowdbTreeBackend, CrowdbTreeConfig, CrowdbTreeEngine, + KVEngine, KVFuture, +}; use crate::paxos::roles::{Learner, PxLogEntry, RequestIdentity, SlotIndex}; use crate::paxos::PxTerm; @@ -147,6 +153,7 @@ pub struct PxLearner { /// fast path (slot already applied) never awaits — `await_applied` does /// one `Acquire` load and returns. apply_notify: Notify, + apply_tasks: Arc<ApplyTasks>, /// Test-only gate that holds `apply_entry` until the test releases it, /// so the R35 apply-fence test can deterministically park the spawned /// R17 apply and prove the Linearizable read's fence waits for it. @@ -198,6 +205,7 @@ impl Default for PxLearner { applied_insert_barrier: Mutex::new(None), request_results: SkipMap::new(), apply_notify: Notify::new(), + apply_tasks: Arc::new(ApplyTasks::default()), #[cfg(feature = "test-util")] apply_gate: Mutex::new(None), engine_apply: OnceLock::new(), @@ -234,6 +242,7 @@ impl PxLearner { applied_insert_barrier: Mutex::new(None), request_results: SkipMap::new(), apply_notify: Notify::new(), + apply_tasks: Arc::new(ApplyTasks::default()), #[cfg(feature = "test-util")] apply_gate: Mutex::new(None), engine_apply: OnceLock::new(), @@ -716,27 +725,43 @@ impl PxLearner { self.request_result_entries_for_tests(client_id) } - /// Decode `payload` and apply it to the engine at `slot`. - /// - /// An empty payload (`NoOp` repair fill) decodes to an empty batch and - /// advances C++ `contiguous_slot_` via [`KVEngine::noop`] (Gap 1 fix) so - /// `flush()` can drain past it; the Rust frontier is advanced by the - /// caller in `learn`. Per-key highest-slot-wins is enforced inside - /// [`KVEngine::apply`]. `async fn` for the same reason as - /// [`Self::engine_get`]; `KVEngine::apply` has no genuine `Pending` path - /// yet either (no async apply C API), so this never actually suspends - /// today. - /// - /// Returns `Err` on a local apply failure (e.g. a durable I/O error on a - /// `CrowdbTreeEngine`). The error is logged at `ERROR` here; the caller - /// must NOT advance `contiguous_applied` on `Err` (Gap 2 fix) so - /// linearizable reads stall at the failed slot instead of reading - /// stale/missing data. The value is still Paxos-chosen (this is a local - /// durability fault, not a consensus outcome), so it must not be - /// re-proposed. Detecting and reacting to a persistently-unhealthy - /// local engine is the caller's job at a layer that can see engine - /// health across calls, not a single failed apply. + /// Reserves an engine apply owner before the proposer allocates a slot. + pub(crate) fn admit_proposal(&self, payload: Bytes) -> KVFuture<ApplyAdmissionResult> { + self.engine.admit_proposal(payload) + } + + pub(crate) async fn learn_admitted( + &self, + entry: PxLogEntry, + request_identities: &[RequestIdentity], + admission: Option<Box<dyn AdmittedApply>>, + ) { + // This value is already chosen. Publish that fact before awaiting a + // contiguous engine, so shutdown/repair can see an earlier missing slot. + self.update_chosen_frontier(entry.slot, entry.term); + let applied = self + .apply_entry_admitted(entry.slot, &entry.payload, admission) + .await + .is_ok(); + if applied { + self.advance_applied_frontier(entry.slot); + } + self.record_request_results(request_identities, entry.slot); + } + + /// Applies follower/replay work through the engine's independent progress path. pub(crate) async fn apply_entry(&self, slot: SlotIndex, payload: &Bytes) -> Result<(), String> { + self.apply_entry_admitted(slot, payload, None).await + } + + /// Applies one chosen payload. Failure must not advance `contiguous_applied`. + pub(crate) async fn apply_entry_admitted( + &self, + slot: SlotIndex, + payload: &Bytes, + admission: Option<Box<dyn AdmittedApply>>, + ) -> Result<(), String> { + let _accepted = self.track_apply_task(); // Test-only apply gate: park until the test releases, so the R35 // fence test can hold the spawned R17 apply deterministically. The // guard is dropped at the `;` so the `Notify` is awaited without a @@ -747,17 +772,14 @@ impl PxLearner { if let Some(gate) = gate { gate.notified().await; } - let batch = Batch::decode(payload); - if batch.ops.is_empty() { - // Gap 1 fix: NoOp slots (from `repair_once` gap-fill) must - // still advance C++ `contiguous_slot_` so `flush()` can drain - // past them and `last_applied_slot_` advances. Without this, - // a single NoOp permanently blocks the durable watermark. - self.engine.noop(slot); - return Ok(()); - } + let batch = Batch::try_decode(payload).map_err(str::to_owned)?; let apply_start = std::time::Instant::now(); - match self.engine.apply(slot, &batch).await { + let applied = match admission { + Some(permit) => permit.apply(slot).await, + None if batch.ops.is_empty() => self.engine.apply_noop(slot).await, + None => self.engine.apply(slot, &batch).await, + }; + match applied { Ok(()) => {} Err(error) => { tracing::error!( @@ -805,18 +827,6 @@ impl PxLearner { impl Learner for PxLearner { async fn learn(&self, entry: PxLogEntry, request_identities: &[RequestIdentity]) { - // V1 sync path (followers, restore, R17-off leader): apply, then - // advance both frontiers, then cache request results. With apply synchronous, - // `contiguous_applied` tracks `contiguous_chosen` exactly — the R35 - // apply fence is a no-op fast path on this path. - // Gap 2: only advance `contiguous_applied` if the apply succeeded. - // On error, `contiguous_applied` stalls at the failed slot so - // linearizable reads block (visible) instead of reading stale data. - let apply_ok = self.apply_entry(entry.slot, &entry.payload).await.is_ok(); - self.update_chosen_frontier(entry.slot, entry.term); - if apply_ok { - self.advance_applied_frontier(entry.slot); - } - self.record_request_results(request_identities, entry.slot); + self.learn_admitted(entry, request_identities, None).await; } } diff --git a/lib/crowdb-paxos-kv/src/paxos/learner/accepted_apply.rs b/lib/crowdb-paxos-kv/src/paxos/learner/accepted_apply.rs new file mode 100644 index 000000000..b821e63aa --- /dev/null +++ b/lib/crowdb-paxos-kv/src/paxos/learner/accepted_apply.rs @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::PxLearner; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use tokio::sync::Notify; + +#[derive(Default)] +pub(super) struct ApplyTasks { + active: AtomicUsize, + changed: Notify, +} +pub(crate) struct ApplyTask { + tasks: Arc<ApplyTasks>, +} +impl Drop for ApplyTask { + fn drop(&mut self) { + if self.tasks.active.fetch_sub(1, Ordering::AcqRel) == 1 { + self.tasks.changed.notify_waiters(); + } + } +} +impl PxLearner { + // Register detached work synchronously, before chosen frontier publication. + pub(crate) fn track_apply_task(&self) -> ApplyTask { + self.apply_tasks.active.fetch_add(1, Ordering::AcqRel); + ApplyTask { + tasks: self.apply_tasks.clone(), + } + } + pub(crate) fn applying_tasks(&self) -> usize { + self.apply_tasks.active.load(Ordering::Acquire) + } + pub(crate) async fn drain_apply_tasks(&self) { + loop { + let changed = self.apply_tasks.changed.notified(); + if self.applying_tasks() == 0 { + return; + } + changed.await; + } + } +} diff --git a/lib/crowdb-kv/src/paxos/roles.rs b/lib/crowdb-paxos-kv/src/paxos/roles.rs similarity index 100% rename from lib/crowdb-kv/src/paxos/roles.rs rename to lib/crowdb-paxos-kv/src/paxos/roles.rs diff --git a/lib/crowdb-kv/src/paxos/slot_list.rs b/lib/crowdb-paxos-kv/src/paxos/slot_list.rs similarity index 100% rename from lib/crowdb-kv/src/paxos/slot_list.rs rename to lib/crowdb-paxos-kv/src/paxos/slot_list.rs diff --git a/lib/crowdb-kv/src/paxos/slot_node.rs b/lib/crowdb-paxos-kv/src/paxos/slot_node.rs similarity index 100% rename from lib/crowdb-kv/src/paxos/slot_node.rs rename to lib/crowdb-paxos-kv/src/paxos/slot_node.rs diff --git a/lib/crowdb-kv/src/rpc.rs b/lib/crowdb-paxos-kv/src/rpc.rs similarity index 93% rename from lib/crowdb-kv/src/rpc.rs rename to lib/crowdb-paxos-kv/src/rpc.rs index 7cea2daff..340bdb923 100644 --- a/lib/crowdb-kv/src/rpc.rs +++ b/lib/crowdb-paxos-kv/src/rpc.rs @@ -17,18 +17,18 @@ pub use crowdb_protocol::kv_client::rpc::*; pub use crowdb_protocol::kv_consensus::rpc::*; -pub(crate) mod kv_rpc_service; +pub(crate) mod paxos_kv_rpc_service; pub(crate) mod px_rpc_service; pub(crate) mod px_rpc_transport; pub(crate) mod snapshot_registry; #[allow(unused_imports)] -pub(crate) use kv_rpc_service::{KvClientRpcForwarder, KvRpcService}; +pub(crate) use paxos_kv_rpc_service::{PaxosKvRpcForwarder, PaxosKvRpcService}; #[allow(unused_imports)] pub(crate) use px_rpc_service::PxRpcService; pub use px_rpc_transport::{PxRpcTransport, ReplicaDestination}; // Re-export the flatbuffer KV client request/response types so the -// server handler (`kv_rpc_service.rs`) can reference them via +// server handler (`paxos_kv_rpc_service.rs`) can reference them via // `crate::rpc::FB<Type>` without a direct `crowdb_protocol` import at // every call site. These are the types from `kv_client_fb` (R117). #[allow(unused_imports)] diff --git a/lib/crowdb-kv/src/rpc/kv_rpc_service.rs b/lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service.rs similarity index 98% rename from lib/crowdb-kv/src/rpc/kv_rpc_service.rs rename to lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service.rs index 467341e08..4f549e8a2 100644 --- a/lib/crowdb-kv/src/rpc/kv_rpc_service.rs +++ b/lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service.rs @@ -27,9 +27,9 @@ //! `Get`/`Scan`/`JournalScan` preserve the transparent leader-forward //! step (linearizable reads only). The loop-guard `forwarded: bool` //! field on the request flatbuffer replaces the former transport's -//! `x-crowdb-kv-forwarded` metadata header. The forwarder -//! (`KvClientRpcForwarder`) lives in `crowdb-kv` itself (not -//! `crowdb-kv-client`) to avoid a crate cycle. +//! `x-crowdb-paxos-kv-forwarded` metadata header. The forwarder +//! (`PaxosKvRpcForwarder`) lives in `crowdb-paxos-kv` itself (not +//! `crowdb-paxos-kv-client`) to avoid a crate cycle. mod comparison; mod owner_fence; @@ -69,23 +69,23 @@ use crate::rpc::{ FBKvDeleteRequest, FBKvGetRequest, FBKvGetRequestArgs, FBKvSetRequest, KvBatchItem, ReadMode, }; -// ── KvClientRpcForwarder ───────────────────────────────────────── +// ── PaxosKvRpcForwarder ───────────────────────────────────────── /// Minimal server-side forwarder for transparent leader-forwarding of -/// linearizable reads (R117). Lives in `crowdb-kv` (not `crowdb-kv-client`) +/// linearizable reads (R117). Lives in `crowdb-paxos-kv` (not `crowdb-paxos-kv-client`) /// to avoid a crate cycle. Holds an `Arc<RpcServer>`, an `Arc<RpcClient>`, /// and a connection cache. Builds the request flatbuffer with /// `forwarded = true`, calls `rpc.call()`, and returns the raw response /// control buffer for the handler to submit back to the original /// client. -pub(crate) struct KvClientRpcForwarder { +pub(crate) struct PaxosKvRpcForwarder { pub(crate) server: Arc<RpcServer>, pub(crate) rpc: Arc<RpcClient>, connections: ConnectionPoolIndex, next_req_id: AtomicU64, } -impl KvClientRpcForwarder { +impl PaxosKvRpcForwarder { pub(crate) fn new() -> Self { Self::with_workers(2) } @@ -257,16 +257,16 @@ impl KvClientRpcForwarder { } } -// ── KvRpcService ───────────────────────────────────────────────── +// ── PaxosKvRpcService ───────────────────────────────────────────────── /// crowdb-rpc handler set for the KV client-facing service. Holds the /// same dependencies as the former `KvStoreService` plus a tokio /// `Handle` for spawning async work from the C++ I/O thread callback, -/// and a `KvClientRpcForwarder` for transparent leader-forwarding. -pub struct KvRpcService { +/// and a `PaxosKvRpcForwarder` for transparent leader-forwarding. +pub struct PaxosKvRpcService { store: Arc<PxKvStore>, rt: Handle, - forwarder: Arc<KvClientRpcForwarder>, + forwarder: Arc<PaxosKvRpcForwarder>, /// Per-request-type end-to-end latency histograms. `None` when /// metrics are disabled (`--metrics-interval 0`). latency: Option<RpcLatencyHandles>, @@ -297,8 +297,8 @@ impl RpcLatencyHandles { } } -impl KvRpcService { - pub(crate) fn new(store: Arc<PxKvStore>, rt: Handle, forwarder: Arc<KvClientRpcForwarder>) -> Self { +impl PaxosKvRpcService { + pub(crate) fn new(store: Arc<PxKvStore>, rt: Handle, forwarder: Arc<PaxosKvRpcForwarder>) -> Self { let latency = store .metrics_registry .as_ref() diff --git a/lib/crowdb-kv/src/rpc/kv_rpc_service/comparison.rs b/lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/comparison.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/kv_rpc_service/comparison.rs rename to lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/comparison.rs diff --git a/lib/crowdb-kv/src/rpc/kv_rpc_service/owner_fence.rs b/lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/owner_fence.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/kv_rpc_service/owner_fence.rs rename to lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/owner_fence.rs diff --git a/lib/crowdb-kv/src/rpc/kv_rpc_service/response_correlation.rs b/lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/response_correlation.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/kv_rpc_service/response_correlation.rs rename to lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/response_correlation.rs diff --git a/lib/crowdb-kv/src/rpc/kv_rpc_service/watch.rs b/lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/watch.rs similarity index 97% rename from lib/crowdb-kv/src/rpc/kv_rpc_service/watch.rs rename to lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/watch.rs index 06182157c..8e63aee96 100644 --- a/lib/crowdb-kv/src/rpc/kv_rpc_service/watch.rs +++ b/lib/crowdb-paxos-kv/src/rpc/paxos_kv_rpc_service/watch.rs @@ -5,14 +5,14 @@ use super::{ debug, record_group_context, request_span, Arc, Buffer, FBMsgType, FlatBufferBuilder, - KvClientRpcForwarder, KvRpcService, RpcServer, + PaxosKvRpcForwarder, PaxosKvRpcService, RpcServer, }; use crowdb_protocol::kv_client_fb::{ FBWatchNotifyError, FBWatchNotifyErrorArgs, FBWatchSubscribe, FBWatchUnsubscribe, }; use crowdb_rpc_ffi::Connection; -impl KvClientRpcForwarder { +impl PaxosKvRpcForwarder { /// Send a fire-and-forget `WatchNotifyError` push frame on the /// given connection (server→client). Used when a subscribe arrives /// on a non-leader or for a missing group. @@ -50,7 +50,7 @@ impl KvClientRpcForwarder { } } -impl KvRpcService { +impl PaxosKvRpcService { // ── WatchSubscribe (fire-and-forget, no response) ───────────── #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service.rs similarity index 99% rename from lib/crowdb-kv/src/rpc/px_rpc_service.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_service.rs index 66f4ce9b8..ae45c9aa7 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_service.rs +++ b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service.rs @@ -128,7 +128,7 @@ impl PxRpcService { store, rt, snapshots, - cluster_id: std::env::var("CROWDB_KV_CLUSTER_ID").ok(), + cluster_id: std::env::var("CROWDB_PAXOS_KV_CLUSTER_ID").ok(), } } diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/accept.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service/accept.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/px_rpc_service/accept.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_service/accept.rs diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/election.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service/election.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/px_rpc_service/election.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_service/election.rs diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/heartbeat.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service/heartbeat.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/px_rpc_service/heartbeat.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_service/heartbeat.rs diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/notifications.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service/notifications.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/px_rpc_service/notifications.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_service/notifications.rs diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/prepare.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service/prepare.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/px_rpc_service/prepare.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_service/prepare.rs diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/responses.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service/responses.rs similarity index 95% rename from lib/crowdb-kv/src/rpc/px_rpc_service/responses.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_service/responses.rs index eb4b366bb..ba0772d2c 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_service/responses.rs +++ b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service/responses.rs @@ -89,10 +89,26 @@ pub(super) fn submit_snapshot_read( offset: u64, read: Option<crate::rpc::snapshot_registry::SnapshotRead>, ) { + let mut payload_crc32c = read.as_ref().map_or(0, |value| value.payload_crc32c); + let mut done = read.as_ref().is_some_and(|value| value.chunk.done); + let data = read + .map(|value| Buffer::try_from_owned_bytes(value.chunk.bytes)) + .transpose(); + let control_error = data.as_ref().err().map(ToString::to_string); + if control_error.is_some() { + payload_crc32c = 0; + done = false; + } + let ret_code = if control_error.is_some() { + FBKvRetCode::SnapshotBackpressure + } else { + ret_code + }; let mut builder = FlatBufferBuilder::new(); - let error_msg = error.map(|message| builder.create_string(message)); - let payload_crc32c = read.as_ref().map_or(0, |value| value.payload_crc32c); - let done = read.as_ref().is_some_and(|value| value.chunk.done); + let error_msg = control_error + .as_deref() + .or(error) + .map(|message| builder.create_string(message)); let response = FBSnapshotReadResponse::create( &mut builder, &FBSnapshotReadResponseArgs { @@ -109,7 +125,7 @@ pub(super) fn submit_snapshot_read( ); builder.finish(response, None); let control = builder.collapse(); - let data = read.map(|value| Buffer::from_vec_offset(value.chunk.bytes, 0)); + let data = data.ok().flatten(); let control = Buffer::from_vec_offset(control.0, control.1); unsafe { let _ = server.submit_response_buffer( diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/snapshots.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_service/snapshots.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/px_rpc_service/snapshots.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_service/snapshots.rs diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_transport.rs similarity index 97% rename from lib/crowdb-kv/src/rpc/px_rpc_transport.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_transport.rs index 40cc0f9bf..789b51a73 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_transport.rs +++ b/lib/crowdb-paxos-kv/src/rpc/px_rpc_transport.rs @@ -30,8 +30,7 @@ use crowdb_protocol::kv_consensus_fb::{ FBStepDownRequestArgs, }; use crowdb_rpc_ffi::{ - noop_completion, Buffer, ConnectionPoolError, ConnectionPoolIndex, RpcClient, RpcError, RpcServer, - SelectedConnection, + Buffer, ConnectionPoolError, ConnectionPoolIndex, RpcClient, RpcError, RpcServer, SelectedConnection, }; use crate::cluster::replica::{ @@ -96,7 +95,7 @@ impl PxRpcTransport { rpc, connections: ConnectionPoolIndex::new(pool_size, None), next_req_id: AtomicU64::new(1), - cluster_id: std::env::var("CROWDB_KV_CLUSTER_ID").ok(), + cluster_id: std::env::var("CROWDB_PAXOS_KV_CLUSTER_ID").ok(), } } @@ -547,16 +546,7 @@ impl PxRpcTransport { let control = Buffer::from_bytes(builder.finished_data()); let msg_type = FBMsgType::EChosenNotification.0 as u16; self.rpc - .send( - &self.server, - &conn, - req_id, - control, - None, - msg_type, - noop_completion(), - std::ptr::null_mut(), - ) + .send_one_way(&self.server, &conn, req_id, control, None, msg_type) .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation())) } @@ -596,16 +586,7 @@ impl PxRpcTransport { let control = Buffer::from_bytes(builder.finished_data()); let msg_type = FBMsgType::EBatchChosenNotification.0 as u16; self.rpc - .send( - &self.server, - &conn, - req_id, - control, - None, - msg_type, - noop_completion(), - std::ptr::null_mut(), - ) + .send_one_way(&self.server, &conn, req_id, control, None, msg_type) .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation())) } diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport/connection.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_transport/connection.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/px_rpc_transport/connection.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_transport/connection.rs diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs b/lib/crowdb-paxos-kv/src/rpc/px_rpc_transport/heartbeat.rs similarity index 100% rename from lib/crowdb-kv/src/rpc/px_rpc_transport/heartbeat.rs rename to lib/crowdb-paxos-kv/src/rpc/px_rpc_transport/heartbeat.rs diff --git a/lib/crowdb-kv/src/rpc/snapshot_registry.rs b/lib/crowdb-paxos-kv/src/rpc/snapshot_registry.rs similarity index 99% rename from lib/crowdb-kv/src/rpc/snapshot_registry.rs rename to lib/crowdb-paxos-kv/src/rpc/snapshot_registry.rs index 3231868ab..e94387792 100644 --- a/lib/crowdb-kv/src/rpc/snapshot_registry.rs +++ b/lib/crowdb-paxos-kv/src/rpc/snapshot_registry.rs @@ -413,7 +413,7 @@ mod tests { let end = self.data.len().min(self.offset + self.chunk_bytes); let chunk = SnapshotChunk { offset, - bytes: self.data[self.offset..end].to_vec(), + bytes: self.data[self.offset..end].to_vec().into(), done: end == self.data.len(), }; self.offset = end; diff --git a/lib/crowdb-kv/src/wal.rs b/lib/crowdb-paxos-kv/src/wal.rs similarity index 100% rename from lib/crowdb-kv/src/wal.rs rename to lib/crowdb-paxos-kv/src/wal.rs diff --git a/lib/crowdb-kv/src/wal/block_backend.rs b/lib/crowdb-paxos-kv/src/wal/block_backend.rs similarity index 100% rename from lib/crowdb-kv/src/wal/block_backend.rs rename to lib/crowdb-paxos-kv/src/wal/block_backend.rs diff --git a/lib/crowdb-kv/src/wal/file_backend.rs b/lib/crowdb-paxos-kv/src/wal/file_backend.rs similarity index 100% rename from lib/crowdb-kv/src/wal/file_backend.rs rename to lib/crowdb-paxos-kv/src/wal/file_backend.rs diff --git a/lib/crowdb-kv/src/wal/gc.rs b/lib/crowdb-paxos-kv/src/wal/gc.rs similarity index 100% rename from lib/crowdb-kv/src/wal/gc.rs rename to lib/crowdb-paxos-kv/src/wal/gc.rs diff --git a/lib/crowdb-kv/src/wal/index.rs b/lib/crowdb-paxos-kv/src/wal/index.rs similarity index 100% rename from lib/crowdb-kv/src/wal/index.rs rename to lib/crowdb-paxos-kv/src/wal/index.rs diff --git a/lib/crowdb-kv/src/wal/io_backend.rs b/lib/crowdb-paxos-kv/src/wal/io_backend.rs similarity index 100% rename from lib/crowdb-kv/src/wal/io_backend.rs rename to lib/crowdb-paxos-kv/src/wal/io_backend.rs diff --git a/lib/crowdb-kv/src/wal/pipeline.rs b/lib/crowdb-paxos-kv/src/wal/pipeline.rs similarity index 100% rename from lib/crowdb-kv/src/wal/pipeline.rs rename to lib/crowdb-paxos-kv/src/wal/pipeline.rs diff --git a/lib/crowdb-kv/src/wal/pipeline_backend.rs b/lib/crowdb-paxos-kv/src/wal/pipeline_backend.rs similarity index 100% rename from lib/crowdb-kv/src/wal/pipeline_backend.rs rename to lib/crowdb-paxos-kv/src/wal/pipeline_backend.rs diff --git a/lib/crowdb-kv/src/wal/pipeline_writer.rs b/lib/crowdb-paxos-kv/src/wal/pipeline_writer.rs similarity index 100% rename from lib/crowdb-kv/src/wal/pipeline_writer.rs rename to lib/crowdb-paxos-kv/src/wal/pipeline_writer.rs diff --git a/lib/crowdb-kv/src/wal/record.rs b/lib/crowdb-paxos-kv/src/wal/record.rs similarity index 100% rename from lib/crowdb-kv/src/wal/record.rs rename to lib/crowdb-paxos-kv/src/wal/record.rs diff --git a/lib/crowdb-kv/src/wal/replay.rs b/lib/crowdb-paxos-kv/src/wal/replay.rs similarity index 100% rename from lib/crowdb-kv/src/wal/replay.rs rename to lib/crowdb-paxos-kv/src/wal/replay.rs diff --git a/lib/crowdb-kv/src/wal/replay/records.rs b/lib/crowdb-paxos-kv/src/wal/replay/records.rs similarity index 100% rename from lib/crowdb-kv/src/wal/replay/records.rs rename to lib/crowdb-paxos-kv/src/wal/replay/records.rs diff --git a/lib/crowdb-kv/src/wal/segment.rs b/lib/crowdb-paxos-kv/src/wal/segment.rs similarity index 100% rename from lib/crowdb-kv/src/wal/segment.rs rename to lib/crowdb-paxos-kv/src/wal/segment.rs diff --git a/lib/crowdb-kv/src/wal/segment/read_buffer.rs b/lib/crowdb-paxos-kv/src/wal/segment/read_buffer.rs similarity index 100% rename from lib/crowdb-kv/src/wal/segment/read_buffer.rs rename to lib/crowdb-paxos-kv/src/wal/segment/read_buffer.rs diff --git a/lib/crowdb-kv/src/wal/uring_backend.rs b/lib/crowdb-paxos-kv/src/wal/uring_backend.rs similarity index 100% rename from lib/crowdb-kv/src/wal/uring_backend.rs rename to lib/crowdb-paxos-kv/src/wal/uring_backend.rs diff --git a/lib/crowdb-kv/src/wal/wal_engine.rs b/lib/crowdb-paxos-kv/src/wal/wal_engine.rs similarity index 100% rename from lib/crowdb-kv/src/wal/wal_engine.rs rename to lib/crowdb-paxos-kv/src/wal/wal_engine.rs diff --git a/lib/crowdb-kv/src/wal/wal_file.rs b/lib/crowdb-paxos-kv/src/wal/wal_file.rs similarity index 100% rename from lib/crowdb-kv/src/wal/wal_file.rs rename to lib/crowdb-paxos-kv/src/wal/wal_file.rs diff --git a/lib/crowdb-kv/tests/common.rs b/lib/crowdb-paxos-kv/tests/common.rs similarity index 82% rename from lib/crowdb-kv/tests/common.rs rename to lib/crowdb-paxos-kv/tests/common.rs index d367ed060..bea77187f 100644 --- a/lib/crowdb-kv/tests/common.rs +++ b/lib/crowdb-paxos-kv/tests/common.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Shared deterministic test harness for `crowdb_kv` integration tests. +//! Shared deterministic test harness for `crowdb_paxos_kv` integration tests. //! //! Real content lands incrementally with each phase: //! - P1 M2: `TestTimer`, `TestRouter`, `TestNode`. diff --git a/lib/crowdb-kv/tests/common/cluster.rs b/lib/crowdb-paxos-kv/tests/common/cluster.rs similarity index 96% rename from lib/crowdb-kv/tests/common/cluster.rs rename to lib/crowdb-paxos-kv/tests/common/cluster.rs index f2701a740..a97ca7b96 100644 --- a/lib/crowdb-kv/tests/common/cluster.rs +++ b/lib/crowdb-paxos-kv/tests/common/cluster.rs @@ -6,18 +6,20 @@ use std::time::Duration; use crate::common::logging::init_test_subscriber; use crate::common::net_lock::{lock, unique_port}; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::common::config::PxElectionConfig; -use crowdb_kv_client::KvRpcTransport; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::{ + PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica, +}; +use crowdb_paxos_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv_client::PaxosKvRpcTransport; use crate::common::test_client::{TestKvClient, TestPxClient}; pub struct TestCluster { nodes: Vec<Arc<PxKvStore>>, leader_id: u64, - kv_transport: Arc<KvRpcTransport>, + kv_transport: Arc<PaxosKvRpcTransport>, _net: tokio::sync::MutexGuard<'static, ()>, } @@ -26,7 +28,7 @@ impl TestCluster { Self { nodes, leader_id, - kv_transport: Arc::new(KvRpcTransport::new()), + kv_transport: Arc::new(PaxosKvRpcTransport::new()), _net: net, } } diff --git a/lib/crowdb-kv/tests/common/logging.rs b/lib/crowdb-paxos-kv/tests/common/logging.rs similarity index 86% rename from lib/crowdb-kv/tests/common/logging.rs rename to lib/crowdb-paxos-kv/tests/common/logging.rs index 0a07abd32..88a8c747a 100644 --- a/lib/crowdb-kv/tests/common/logging.rs +++ b/lib/crowdb-paxos-kv/tests/common/logging.rs @@ -19,7 +19,7 @@ pub fn init_test_subscriber() { crowdb_tree_ffi::ct_init_test_logging(); crowdb_rpc_ffi::init_test_logging(); - if std::env::var("CROWDB_KV_TEST_LOG").is_err() { + if std::env::var("CROWDB_PAXOS_KV_TEST_LOG").is_err() { // No debug logging requested — still install a minimal // error-level subscriber so Rust-side panics and error // spans are visible on stderr (not just the C++ side). @@ -31,7 +31,7 @@ pub fn init_test_subscriber() { } let filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("warn,crowdb_kv=debug,crowdb_kv_server=debug,crowdb_web=debug,crowdb_console_shared=debug,crowdb_cli=debug")); + .unwrap_or_else(|_| EnvFilter::new("warn,crowdb_paxos_kv=debug,crowdb_paxos_kv_server=debug,crowdb_web=debug,crowdb_console_shared=debug,crowdb_cli=debug")); let _ = tracing_subscriber::registry() .with(filter) .with(fmt::layer().with_test_writer().with_target(true)) diff --git a/lib/crowdb-kv/tests/common/net_lock.rs b/lib/crowdb-paxos-kv/tests/common/net_lock.rs similarity index 97% rename from lib/crowdb-kv/tests/common/net_lock.rs rename to lib/crowdb-paxos-kv/tests/common/net_lock.rs index 97c98463f..13b2008b5 100644 --- a/lib/crowdb-kv/tests/common/net_lock.rs +++ b/lib/crowdb-paxos-kv/tests/common/net_lock.rs @@ -23,7 +23,7 @@ static NET_LOCK: OnceLock<Mutex<()>> = OnceLock::new(); /// # Panics /// Panics when the shared port allocator cannot reserve a port. pub fn unique_port() -> u16 { - crowdb_protocol::port::alloc::alloc_test_port(crowdb_protocol::ServicePort::KvServerListen) + crowdb_protocol::port::alloc::alloc_test_port(crowdb_protocol::ServicePort::PaxosKvServerRpc) } /// Acquire the global network test lock. Hold the guard for the duration of diff --git a/lib/crowdb-kv/tests/common/payload_owner.rs b/lib/crowdb-paxos-kv/tests/common/payload_owner.rs similarity index 100% rename from lib/crowdb-kv/tests/common/payload_owner.rs rename to lib/crowdb-paxos-kv/tests/common/payload_owner.rs diff --git a/lib/crowdb-kv/tests/common/simdisk.rs b/lib/crowdb-paxos-kv/tests/common/simdisk.rs similarity index 100% rename from lib/crowdb-kv/tests/common/simdisk.rs rename to lib/crowdb-paxos-kv/tests/common/simdisk.rs diff --git a/lib/crowdb-kv/tests/common/test_client.rs b/lib/crowdb-paxos-kv/tests/common/test_client.rs similarity index 89% rename from lib/crowdb-kv/tests/common/test_client.rs rename to lib/crowdb-paxos-kv/tests/common/test_client.rs index 656fa268f..72db71927 100644 --- a/lib/crowdb-kv/tests/common/test_client.rs +++ b/lib/crowdb-paxos-kv/tests/common/test_client.rs @@ -4,7 +4,7 @@ //! Test-only client facades that preserve the former tonic //! `KvServiceClient`/`PxServiceClient` call shape (`.put(req).await. //! expect(..).into_inner()`) but send over the crowdb-rpc transports -//! (`KvRpcTransport` / `PxRpcTransport`). They exist so the many +//! (`PaxosKvRpcTransport` / `PxRpcTransport`). They exist so the many //! integration-test call sites could be migrated off the deleted tonic //! clients with minimal churn: the request/response structs are //! unchanged, only the wire transport changed. @@ -14,13 +14,13 @@ use std::sync::Arc; -use crowdb_kv::paxos::roles::{PxBallot, PxPrepareReply}; -use crowdb_kv::rpc::PxRpcTransport; -use crowdb_kv::rpc::{ +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxPrepareReply}; +use crowdb_paxos_kv::rpc::PxRpcTransport; +use crowdb_paxos_kv::rpc::{ KvBatchWriteRequest, KvDeleteRequest, KvGetRequest, KvResponse, KvScanRequest, KvScanResponse, KvSetRequest, PromiseResponse, ReadMode, }; -use crowdb_kv_client::KvRpcTransport; +use crowdb_paxos_kv_client::PaxosKvRpcTransport; /// Status-like error returned by the facades. Implements `Debug` so /// `Result<_, TestRpcStatus>::expect(..)` compiles, and exposes @@ -64,11 +64,11 @@ fn read_mode_from_i32(v: i32) -> ReadMode { /// with `TestKvClient::connect(endpoint).await` (the `.await` mirrors the /// old `KvServiceClient::connect(..).await` shape), or /// `TestKvClient::with_transport(transport, endpoint)` to share a -/// single `KvRpcTransport` (and its underlying `RpcServer`/`RpcClient`) +/// single `PaxosKvRpcTransport` (and its underlying `RpcServer`/`RpcClient`) /// across many clients — avoiding per-call `RpcServer` creation in /// crash/restart tests that issue hundreds of sequential reads. pub struct TestKvClient { - transport: Arc<KvRpcTransport>, + transport: Arc<PaxosKvRpcTransport>, endpoint: String, } @@ -76,7 +76,7 @@ impl TestKvClient { #[allow(clippy::unused_async_trait_impl)] pub async fn connect(endpoint: String) -> Self { Self { - transport: Arc::new(KvRpcTransport::new()), + transport: Arc::new(PaxosKvRpcTransport::new()), endpoint, } } @@ -85,7 +85,7 @@ impl TestKvClient { /// holds the `RpcServer` (epoll loop) and `RpcClient` (reaper) — /// sharing them avoids spawning a new event-loop thread per RPC. #[must_use] - pub fn with_transport(transport: Arc<KvRpcTransport>, endpoint: String) -> Self { + pub fn with_transport(transport: Arc<PaxosKvRpcTransport>, endpoint: String) -> Self { Self { transport, endpoint } } @@ -190,8 +190,12 @@ impl TestKvClient { req.bounded, req.scan_cutoff, match req.direction { - crowdb_kv::rpc::KvScanDirection::Forward => crowdb_kv_client::ScanDirection::Forward, - crowdb_kv::rpc::KvScanDirection::Reverse => crowdb_kv_client::ScanDirection::Reverse, + crowdb_paxos_kv::rpc::KvScanDirection::Forward => { + crowdb_paxos_kv_client::ScanDirection::Forward + } + crowdb_paxos_kv::rpc::KvScanDirection::Reverse => { + crowdb_paxos_kv_client::ScanDirection::Reverse + } }, ) .await @@ -219,12 +223,12 @@ impl TestPxClient { pub async fn prepare( &self, - req: crowdb_kv::rpc::PrepareRequest, + req: crowdb_paxos_kv::rpc::PrepareRequest, ) -> Result<TestResponse<PromiseResponse>, TestRpcStatus> { let reply = self .transport .send_prepare( - crowdb_kv::rpc::ReplicaDestination { + crowdb_paxos_kv::rpc::ReplicaDestination { endpoint: &self.endpoint, replica_id: 0, }, diff --git a/lib/crowdb-kv/tests/common/test_util.rs b/lib/crowdb-paxos-kv/tests/common/test_util.rs similarity index 82% rename from lib/crowdb-kv/tests/common/test_util.rs rename to lib/crowdb-paxos-kv/tests/common/test_util.rs index 0132aa1b7..6951c6e2d 100644 --- a/lib/crowdb-kv/tests/common/test_util.rs +++ b/lib/crowdb-paxos-kv/tests/common/test_util.rs @@ -8,7 +8,19 @@ //! keeping `iter_all`/`compare` off the main `KVEngine` trait. use crate::mem_kv::InMemKV; -use crowdb_kv::kv::{Cell, CrowdbTreeEngine, EngineDiff, KVEngine}; +use crowdb_paxos_kv::kv::{Cell, CrowdbTreeEngine, EngineDiff, KVEngine}; + +/// One valid native KV mutation for tests of consensus admission and ordering. +pub fn encode_put_payload(key: &[u8], value: &[u8]) -> Vec<u8> { + let mut payload = Vec::with_capacity(11 + key.len() + value.len()); + payload.extend_from_slice(&1_u16.to_le_bytes()); + payload.push(0); + payload.extend_from_slice(&u32::try_from(key.len()).unwrap().to_le_bytes()); + payload.extend_from_slice(key); + payload.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes()); + payload.extend_from_slice(value); + payload +} /// Full ordered stream including tombstones, via downcast to the concrete /// engine type. diff --git a/lib/crowdb-kv/tests/common/timer.rs b/lib/crowdb-paxos-kv/tests/common/timer.rs similarity index 100% rename from lib/crowdb-kv/tests/common/timer.rs rename to lib/crowdb-paxos-kv/tests/common/timer.rs diff --git a/lib/crowdb-kv/tests/election_test.rs b/lib/crowdb-paxos-kv/tests/election_test.rs similarity index 100% rename from lib/crowdb-kv/tests/election_test.rs rename to lib/crowdb-paxos-kv/tests/election_test.rs diff --git a/lib/crowdb-kv/tests/election_test/apply_loop_test.rs b/lib/crowdb-paxos-kv/tests/election_test/apply_loop_test.rs similarity index 73% rename from lib/crowdb-kv/tests/election_test/apply_loop_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/apply_loop_test.rs index cabeda31d..f375a9a78 100644 --- a/lib/crowdb-kv/tests/election_test/apply_loop_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/apply_loop_test.rs @@ -1,15 +1,18 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! R63: background apply loop + value-less catch-up tests. +//! Background apply loop and value-less catch-up tests. //! -//! Tests the background apply loop's skip-and-continue behavior, the +//! Tests the background apply loop's contiguous replay behavior, the //! `BatchChosenNotification` frontier advance, and the heartbeat reply //! not being delayed by engine apply. -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; +use crowdb_paxos_kv::metrics::MetricsRegistry; +use crowdb_paxos_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; fn heartbeat(term: u64, leader_id: u64, commit_slot: u64) -> HeartbeatRequestPayload { HeartbeatRequestPayload { @@ -43,18 +46,30 @@ fn write_entry(slot: u64, key: &[u8], value: &[u8]) -> PxLogEntry { } } -/// The background apply loop skips a missing slot and applies subsequent -/// available slots. `contiguous_applied` stays at the gap; the subsequent -/// slots are applied via `advance_applied_frontier` (out-of-order). +/// The native engine requires contiguous replay. Preserve the accepted suffix +/// while a slot is missing, then resume from the repair without a new heartbeat. #[tokio::test] -async fn apply_loop_skips_missing_slot_and_applies_subsequent() { +async fn apply_loop_preserves_missing_slot_and_resumes_after_repair() { let replica = PxLocalReplica::new(1, PxLocalReplicaRole::Follower); + let registry = Arc::new(Mutex::new(MetricsRegistry::new())); + replica.set_metrics_registry(®istry, 1, 1); + let skipped = registry + .lock() + .unwrap() + .register_counter("s.1.g.1.paxos.apply_loop.skip.c"); + assert!(replica.learner.engine().requires_contiguous_apply()); // Accept slots 1, 3 (skip 2 — simulates election-churn gap). let entry1 = write_entry(1, b"k1", b"v1"); - let _ = replica.on_accept(&entry1).await; + assert!(matches!( + replica.on_accept(&entry1).await, + PxAcceptReply::Accepted { .. } + )); let entry3 = write_entry(3, b"k3", b"v3"); - let _ = replica.on_accept(&entry3).await; + assert!(matches!( + replica.on_accept(&entry3).await, + PxAcceptReply::Accepted { .. } + )); // Heartbeat with commit_slot=3 signals the apply loop. let _ = <PxLocalReplica as ReplicaHandler>::on_heartbeat(&replica, heartbeat(1, 2, 3), 1) @@ -65,20 +80,32 @@ async fn apply_loop_skips_missing_slot_and_applies_subsequent() { replica.await_apply_fence(1).await; assert_eq!(replica.contiguous_applied(), 1, "slot 1 applied"); - // Slot 3 is applied out-of-order (skip-and-continue). Verify the engine - // has the value even though contiguous_applied is stuck at 1. - // Give the apply loop a moment to process slot 3. - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - assert_eq!( - replica.learner.engine_get(b"k3").await.map(|(_, v)| v), - Some(b"v3".to_vec()), - "slot 3 applied despite gap at slot 2" - ); + tokio::time::timeout(Duration::from_secs(2), async { + while skipped.snapshot().total == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("apply loop must observe the missing slot"); + assert!(replica.learner.engine_get(b"k3").await.is_none()); + assert_eq!(replica.accepted_at(3).await.unwrap().payload, entry3.payload); assert_eq!( replica.contiguous_applied(), 1, "contiguous_applied stays at gap (slot 2 missing)" ); + let entry2 = write_entry(2, b"k2", b"v2"); + assert!(matches!( + replica.on_accept(&entry2).await, + PxAcceptReply::Accepted { .. } + )); + tokio::time::timeout(Duration::from_secs(2), replica.await_apply_fence(3)) + .await + .expect("gap repair must wake the existing apply loop"); + assert_eq!(replica.contiguous_applied(), 3); + assert_eq!(replica.learner.engine_get(b"k2").await, Some((2, b"v2".to_vec()))); + assert_eq!(replica.learner.engine_get(b"k3").await, Some((3, b"v3".to_vec()))); + assert!(replica.shutdown(Duration::from_secs(2)).await.is_clean()); } /// Heartbeat reply returns immediately with the current `contiguous_applied`, diff --git a/lib/crowdb-kv/tests/election_test/frontier_test.rs b/lib/crowdb-paxos-kv/tests/election_test/frontier_test.rs similarity index 97% rename from lib/crowdb-kv/tests/election_test/frontier_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/frontier_test.rs index 21406ff79..37e5782c6 100644 --- a/lib/crowdb-kv/tests/election_test/frontier_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/frontier_test.rs @@ -12,9 +12,9 @@ //! public `ReplicaHandler` trait methods that embed the triple in their //! replies. -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::replica::{ReplicaHandler, VoteRequestPayload}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::replica::{ReplicaHandler, VoteRequestPayload}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; fn make_vote_req(term: u64, candidate_id: u64, tip_slot: u64, tip_term: u64) -> VoteRequestPayload { VoteRequestPayload { diff --git a/lib/crowdb-kv/tests/election_test/heartbeat_test.rs b/lib/crowdb-paxos-kv/tests/election_test/heartbeat_test.rs similarity index 96% rename from lib/crowdb-kv/tests/election_test/heartbeat_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/heartbeat_test.rs index 27e41b926..db7509fba 100644 --- a/lib/crowdb-kv/tests/election_test/heartbeat_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/heartbeat_test.rs @@ -9,9 +9,9 @@ //! the leader's commit point (R63: apply is async, so tests await the //! apply fence before checking `contiguous_applied`). -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; fn heartbeat(term: u64, leader_id: u64, commit_slot: u64) -> HeartbeatRequestPayload { HeartbeatRequestPayload { diff --git a/lib/crowdb-kv/tests/election_test/lease_test.rs b/lib/crowdb-paxos-kv/tests/election_test/lease_test.rs similarity index 96% rename from lib/crowdb-kv/tests/election_test/lease_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/lease_test.rs index 2f3282d61..6974faa3e 100644 --- a/lib/crowdb-kv/tests/election_test/lease_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/lease_test.rs @@ -9,8 +9,8 @@ use std::time::{Duration, Instant}; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PxElectionConfig; #[test] fn lease_read_valid_requires_leader() { diff --git a/lib/crowdb-kv/tests/election_test/metrics_test.rs b/lib/crowdb-paxos-kv/tests/election_test/metrics_test.rs similarity index 95% rename from lib/crowdb-kv/tests/election_test/metrics_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/metrics_test.rs index 332423de3..0cac2f9ae 100644 --- a/lib/crowdb-kv/tests/election_test/metrics_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/metrics_test.rs @@ -7,10 +7,10 @@ use std::sync::{Arc, Mutex}; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::replica::StepDownRequestPayload; -use crowdb_kv::cluster::status::ElectionCounters; -use crowdb_kv::metrics::{MetricPoint, MetricsRegistry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::replica::StepDownRequestPayload; +use crowdb_paxos_kv::cluster::status::ElectionCounters; +use crowdb_paxos_kv::metrics::{MetricPoint, MetricsRegistry}; fn replica_with_registry() -> (PxLocalReplica, Arc<Mutex<MetricsRegistry>>) { let replica = PxLocalReplica::new(1, PxLocalReplicaRole::Follower); diff --git a/lib/crowdb-kv/tests/election_test/role_test.rs b/lib/crowdb-paxos-kv/tests/election_test/role_test.rs similarity index 97% rename from lib/crowdb-kv/tests/election_test/role_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/role_test.rs index d6c984835..72031ed6f 100644 --- a/lib/crowdb-kv/tests/election_test/role_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/role_test.rs @@ -9,7 +9,7 @@ use std::time::Instant; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; // ── become_follower ────────────────────────────────────────── @@ -168,7 +168,7 @@ fn new_inheriting_election_state_preserves_term_and_voted_for() { #[tokio::test] async fn new_inheriting_election_state_shares_acceptor_and_learner() { use bytes::Bytes; - use crowdb_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; + use crowdb_paxos_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; let prior = PxLocalReplica::new(3, PxLocalReplicaRole::Leader); let entry = PxLogEntry { diff --git a/lib/crowdb-kv/tests/election_test/step_down_test.rs b/lib/crowdb-paxos-kv/tests/election_test/step_down_test.rs similarity index 97% rename from lib/crowdb-kv/tests/election_test/step_down_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/step_down_test.rs index 2211b6597..1074058d8 100644 --- a/lib/crowdb-kv/tests/election_test/step_down_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/step_down_test.rs @@ -9,8 +9,8 @@ //! (same term) and `admin_step_down_signal` is notified. On reject, no state //! changes. -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::replica::{ReplicaHandler, StepDownRequestPayload}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::replica::{ReplicaHandler, StepDownRequestPayload}; fn step_down_req(term: u64, target_leader_id: u64, reason: &str) -> StepDownRequestPayload { StepDownRequestPayload { diff --git a/lib/crowdb-kv/tests/election_test/term_fence_test.rs b/lib/crowdb-paxos-kv/tests/election_test/term_fence_test.rs similarity index 95% rename from lib/crowdb-kv/tests/election_test/term_fence_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/term_fence_test.rs index 67da748a4..290f8f315 100644 --- a/lib/crowdb-kv/tests/election_test/term_fence_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/term_fence_test.rs @@ -9,8 +9,8 @@ //! acceptor logic. use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply}; fn write_entry(slot: u64, term: u64, payload: &[u8]) -> PxLogEntry { PxLogEntry { diff --git a/lib/crowdb-kv/tests/election_test/vote_test.rs b/lib/crowdb-paxos-kv/tests/election_test/vote_test.rs similarity index 97% rename from lib/crowdb-kv/tests/election_test/vote_test.rs rename to lib/crowdb-paxos-kv/tests/election_test/vote_test.rs index 6abb023d2..7ee309098 100644 --- a/lib/crowdb-kv/tests/election_test/vote_test.rs +++ b/lib/crowdb-paxos-kv/tests/election_test/vote_test.rs @@ -7,10 +7,10 @@ //! bare `PxLocalReplica` — no peers, no crowdb-rpc. The decision is a pure function //! of `(current_term, voted_for, vote_lockout_until, candidate_log_up_to_date)`. -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::replica::{ReplicaHandler, VoteRequestPayload}; -use crowdb_kv::common::config::PxElectionConfig; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::replica::{ReplicaHandler, VoteRequestPayload}; +use crowdb_paxos_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; use std::time::Instant; #[tokio::test] diff --git a/lib/crowdb-kv/tests/group_test.rs b/lib/crowdb-paxos-kv/tests/group_test.rs similarity index 98% rename from lib/crowdb-kv/tests/group_test.rs rename to lib/crowdb-paxos-kv/tests/group_test.rs index f4a7aa059..99b0ab815 100644 --- a/lib/crowdb-kv/tests/group_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test.rs @@ -148,3 +148,6 @@ mod r65_replication; #[path = "group_test/forward_correlation_test.rs"] mod forward_correlation_test; + +#[path = "watch_registry_test.rs"] +mod watch_registry; diff --git a/lib/crowdb-kv/tests/group_test/a3_crash_restart_no_data_loss_test.rs b/lib/crowdb-paxos-kv/tests/group_test/a3_crash_restart_no_data_loss_test.rs similarity index 93% rename from lib/crowdb-kv/tests/group_test/a3_crash_restart_no_data_loss_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/a3_crash_restart_no_data_loss_test.rs index 5e9f2ca4d..eaeefd6ed 100644 --- a/lib/crowdb-kv/tests/group_test/a3_crash_restart_no_data_loss_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/a3_crash_restart_no_data_loss_test.rs @@ -15,12 +15,12 @@ use std::path::PathBuf; use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::wal_engine::WalEngine; -use crowdb_kv::wal::{IoBackend, WalConfig}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, WalConfig}; const GROUP: u64 = 1; diff --git a/lib/crowdb-kv/tests/group_test/cas_test.rs b/lib/crowdb-paxos-kv/tests/group_test/cas_test.rs similarity index 97% rename from lib/crowdb-kv/tests/group_test/cas_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/cas_test.rs index df558214b..3903e3a75 100644 --- a/lib/crowdb-kv/tests/group_test/cas_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/cas_test.rs @@ -4,8 +4,8 @@ use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; fn encode_put(key: &[u8], value: &[u8]) -> Vec<u8> { let mut payload = Vec::with_capacity(11 + key.len() + value.len()); diff --git a/lib/crowdb-kv/tests/group_test/coalesce_test.rs b/lib/crowdb-paxos-kv/tests/group_test/coalesce_test.rs similarity index 72% rename from lib/crowdb-kv/tests/group_test/coalesce_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/coalesce_test.rs index 4acded350..32676bc39 100644 --- a/lib/crowdb-kv/tests/group_test/coalesce_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/coalesce_test.rs @@ -5,9 +5,9 @@ use std::sync::Arc; -use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::CrowDBConfig; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PaxosKvConfig; #[allow(clippy::cast_possible_truncation)] fn encode_put(key: &[u8], value: &[u8]) -> Vec<u8> { @@ -31,14 +31,14 @@ fn encode_puts(items: &[(&[u8], &[u8])]) -> Vec<u8> { fn coalesce_group(max_keys: usize) -> Arc<PxGroup> { let local = PxLocalReplica::new(1, PxLocalReplicaRole::Leader); let mut group = PxGroup::new(1, local); - let config = CrowDBConfig { + let config = PaxosKvConfig { wal_early_ack: false, async_engine_apply: false, - paxos: crowdb_kv::common::config::PaxosConfig { + paxos: crowdb_paxos_kv::common::config::PaxosConfig { coalesce_max_keys: max_keys, - ..crowdb_kv::common::config::PaxosConfig::DEFAULT + ..crowdb_paxos_kv::common::config::PaxosConfig::DEFAULT }, - ..CrowDBConfig::default() + ..PaxosKvConfig::default() }; group.set_from_config(&config); let arc = Arc::new(group); @@ -46,6 +46,43 @@ fn coalesce_group(max_keys: usize) -> Arc<PxGroup> { arc } +#[tokio::test] +async fn individually_valid_batches_do_not_exceed_native_limit_when_coalesced() { + let group = coalesce_group(32); + let (release, gate) = tokio::sync::oneshot::channel(); + group.set_coalesce_round_gate_for_tests(gate); + let first_group = Arc::clone(&group); + let first = + tokio::spawn(async move { first_group.propose(encode_put(b"first", b"v"), None, None).await }); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !group.has_coalesce_pending_for_tests() { + tokio::task::yield_now().await; + } + let small_group = Arc::clone(&group); + let small = + tokio::spawn(async move { small_group.propose(encode_put(b"small", b"v"), None, None).await }); + while group.coalesce_pending_count_for_tests() != 1 { + tokio::task::yield_now().await; + } + let keys: Vec<_> = (0..64).map(|i| format!("large-{i}")).collect(); + let items: Vec<_> = keys.iter().map(|key| (key.as_bytes(), b"v".as_slice())).collect(); + let result = group.propose(encode_puts(&items), None, None).await; + let _ = release.send(()); + assert!( + matches!(result, ProposeResult::Chosen { .. }), + "large request: {result:?}" + ); + let result = small.await.unwrap(); + assert!( + matches!(result, ProposeResult::Chosen { .. }), + "pending request: {result:?}" + ); + assert!(matches!(first.await.unwrap(), ProposeResult::Chosen { .. })); + }) + .await + .expect("bounded coalescer requests did not complete"); +} + #[tokio::test] async fn coalesce_disabled_when_max_keys_zero() { let group = coalesce_group(0); @@ -62,6 +99,72 @@ async fn coalesce_disabled_when_max_keys_zero() { assert_ne!(slot1, slot2); } +#[tokio::test] +async fn coalescer_watchdog_is_joined_and_does_not_retain_group() { + let group = coalesce_group(32); + let weak = Arc::downgrade(&group); + let result = group + .propose(encode_put(b"watchdog", b"v1"), Some(1), Some(1)) + .await; + assert!(matches!(result, ProposeResult::Chosen { .. }), "got {result:?}"); + + let report = group.shutdown(std::time::Duration::from_millis(200)).await; + assert!(report.errors.is_empty(), "shutdown errors: {:?}", report.errors); + drop(group); + assert_eq!(weak.strong_count(), 0, "watchdog retained PxGroup after shutdown"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_resolves_pending_coalescer_waiter() { + let group = coalesce_group(32); + let (gate_tx, gate_rx) = tokio::sync::oneshot::channel(); + group.set_coalesce_round_gate_for_tests(gate_rx); + + let first_group = Arc::clone(&group); + let first = tokio::spawn(async move { + first_group + .propose(encode_put(b"shutdown-first", b"v1"), Some(10), Some(1)) + .await + }); + tokio::time::timeout(std::time::Duration::from_millis(500), async { + while !group.has_coalesce_pending_for_tests() { + tokio::task::yield_now().await; + } + }) + .await + .expect("first coalescer round did not start"); + + let second_group = Arc::clone(&group); + let second = tokio::spawn(async move { + second_group + .propose(encode_put(b"shutdown-second", b"v2"), Some(11), Some(1)) + .await + }); + tokio::time::timeout(std::time::Duration::from_millis(500), async { + while group.coalesce_pending_count_for_tests() < 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("second coalescer request did not become pending"); + + let report = group.shutdown(std::time::Duration::from_millis(200)).await; + assert!(report.errors.is_empty(), "shutdown errors: {:?}", report.errors); + let _ = gate_tx.send(()); + let _ = tokio::time::timeout(std::time::Duration::from_secs(1), first) + .await + .expect("first coalescer round did not finish") + .expect("first coalescer task panicked"); + let second_result = tokio::time::timeout(std::time::Duration::from_secs(1), second) + .await + .expect("pending coalescer waiter did not finish") + .expect("pending coalescer task panicked"); + assert!( + matches!(second_result, ProposeResult::Err(ref message) if message.contains("shutting down")), + "got {second_result:?}" + ); +} + #[tokio::test] async fn coalesce_first_op_starts_immediately() { let group = coalesce_group(32); diff --git a/lib/crowdb-kv/tests/group_test/election_test.rs b/lib/crowdb-paxos-kv/tests/group_test/election_test.rs similarity index 95% rename from lib/crowdb-kv/tests/group_test/election_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/election_test.rs index 8a8f5011d..f94ca33eb 100644 --- a/lib/crowdb-kv/tests/group_test/election_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/election_test.rs @@ -10,16 +10,16 @@ //! `common::cluster::start_cluster_no_leader`. //! //! 2. **Driver unit tests** (migrated from the inline `#[cfg(test)]` -//! module in `crowdb_kv/src/cluster/election.rs`) — exercise the +//! module in `crowdb_paxos_kv/src/cluster/election.rs`) — exercise the //! election driver, lease renewal, and admin step-down in //! isolation with `tokio::test(start_paused = true)`. use crate::common::cluster::start_cluster_no_leader; -use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; -use crowdb_kv::cluster::group_election::{spawn, LeaderElection, XorShift64}; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::replica::StepDownRequestPayload; -use crowdb_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::group_election::{spawn, LeaderElection, XorShift64}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::replica::StepDownRequestPayload; +use crowdb_paxos_kv::common::config::PxElectionConfig; use std::sync::Arc; use std::time::Duration; use tokio::time::sleep; @@ -417,7 +417,7 @@ fn xorshift_random_between_ms_stays_in_range() { #[tokio::test(flavor = "current_thread", start_paused = true)] async fn stalled_gap_repair_cannot_stop_the_leader_heartbeat_ticker() { - use crowdb_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; + use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; let cfg = PxElectionConfig::for_tests(); let group = Arc::new(PxGroup::new( 99, @@ -434,19 +434,15 @@ async fn stalled_gap_repair_cannot_stop_the_leader_heartbeat_ticker() { } assert!(group.leader_read_ready()); group.pause_repair_for_tests(); - group - .local_replica() - .learner - .learn( - PxLogEntry { - slot: 2, - ballot: PxBallot::new(1, 31), - term: group.local_replica().current_term_snapshot(), - payload: bytes::Bytes::new(), - }, - &[], - ) - .await; + group.local_replica().spawn_learn_chosen( + PxLogEntry { + slot: 2, + ballot: PxBallot::new(1, 31), + term: group.local_replica().current_term_snapshot(), + payload: bytes::Bytes::new(), + }, + &[], + ); for _ in 0..100 { tokio::time::advance(Duration::from_millis(cfg.heartbeat_interval_ms + 1)).await; tokio::task::yield_now().await; diff --git a/lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs b/lib/crowdb-paxos-kv/tests/group_test/fetch_gap_durability_test.rs similarity index 92% rename from lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/fetch_gap_durability_test.rs index 50422cdcb..a9d5d15d0 100644 --- a/lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/fetch_gap_durability_test.rs @@ -2,8 +2,8 @@ // Licensed under the Apache License, Version 2.0. use crate::common::cluster::start_cluster; -use crowdb_kv::{ - cluster::{group::PxGroup, KvServer, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}, +use crowdb_paxos_kv::{ + cluster::{group::PxGroup, PaxosKvServer, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}, paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}, wal::{replay::replay_group, IoBackend, MemBlockDevice, WalConfig, WalEngine}, }; @@ -52,12 +52,12 @@ async fn restart_fetched_value(higher_promise: bool) { ..entry.clone() }; replica.acceptor.accept(&stale).await; - wal.append(&crowdb_kv::wal::WALRecord::from_accepted(1, &stale)) + wal.append(&crowdb_paxos_kv::wal::WALRecord::from_accepted(1, &stale)) .await .unwrap(); let promise = PxBallot::new(9, 99); replica.acceptor.prepare(1, promise).await; - wal.append(&crowdb_kv::wal::WALRecord::from_promised(1, 7, 1, promise)) + wal.append(&crowdb_paxos_kv::wal::WALRecord::from_promised(1, 7, 1, promise)) .await .unwrap(); } @@ -133,7 +133,7 @@ async fn fetch_gap_does_not_advertise_an_unproven_accepted_value_as_chosen() { leader.local_replica().acceptor.accept(&entry).await; let transport = cluster.leader().rpc_transport().unwrap(); let endpoint = cluster.leader().listen_addr().unwrap().to_string(); - let destination = crowdb_kv::rpc::ReplicaDestination { + let destination = crowdb_paxos_kv::rpc::ReplicaDestination { endpoint: &endpoint, replica_id: 1, }; @@ -142,7 +142,7 @@ async fn fetch_gap_does_not_advertise_an_unproven_accepted_value_as_chosen() { "an accepted value without a chosen proof must not be advertised as chosen" ); leader.local_replica().learner.learn(entry.clone(), &[]).await; - let destination = crowdb_kv::rpc::ReplicaDestination { + let destination = crowdb_paxos_kv::rpc::ReplicaDestination { endpoint: &endpoint, replica_id: 1, }; diff --git a/lib/crowdb-kv/tests/group_test/forward_correlation_test.rs b/lib/crowdb-paxos-kv/tests/group_test/forward_correlation_test.rs similarity index 86% rename from lib/crowdb-kv/tests/group_test/forward_correlation_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/forward_correlation_test.rs index 727411c6f..8c290873a 100644 --- a/lib/crowdb-kv/tests/group_test/forward_correlation_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/forward_correlation_test.rs @@ -3,11 +3,11 @@ use std::time::Duration; -use crowdb_kv::cluster::KvServer; -use crowdb_kv::rpc::{KvJournalScanResponse, KvResponse, KvScanResponse}; -use crowdb_kv_client::{KvRpcTransport, ReadMode}; +use crowdb_paxos_kv::cluster::PaxosKvServer; +use crowdb_paxos_kv::rpc::{KvJournalScanResponse, KvResponse, KvScanResponse}; +use crowdb_paxos_kv_client::{PaxosKvRpcTransport, ReadMode}; -async fn missing_get(transport: &KvRpcTransport, endpoint: &str, business_id: u64) -> KvResponse { +async fn missing_get(transport: &PaxosKvRpcTransport, endpoint: &str, business_id: u64) -> KvResponse { tokio::time::timeout( Duration::from_secs(1), transport.send_get(endpoint, b"missing", business_id, 0, 1, ReadMode::Linearizable, 0), @@ -17,7 +17,7 @@ async fn missing_get(transport: &KvRpcTransport, endpoint: &str, business_id: u6 .unwrap() } -async fn empty_scan(transport: &KvRpcTransport, endpoint: &str, business_id: u64) -> KvScanResponse { +async fn empty_scan(transport: &PaxosKvRpcTransport, endpoint: &str, business_id: u64) -> KvScanResponse { tokio::time::timeout( Duration::from_secs(1), transport.send_scan( @@ -44,7 +44,7 @@ async fn empty_scan(transport: &KvRpcTransport, endpoint: &str, business_id: u64 } async fn empty_journal( - transport: &KvRpcTransport, + transport: &PaxosKvRpcTransport, endpoint: &str, business_id: u64, ) -> KvJournalScanResponse { @@ -74,7 +74,7 @@ async fn forwarded_reads_and_failed_forward_keep_caller_rpc_identity() { let follower = cluster.followers()[0]; let leader_endpoint = leader.listen_addr().unwrap().to_string(); let follower_endpoint = follower.listen_addr().unwrap().to_string(); - let transport = KvRpcTransport::new(); + let transport = PaxosKvRpcTransport::new(); // Offset the caller's wire IDs from the follower's independent counter. missing_get(&transport, &leader_endpoint, 9001).await; diff --git a/lib/crowdb-kv/tests/group_test/full_restart_delete_test.rs b/lib/crowdb-paxos-kv/tests/group_test/full_restart_delete_test.rs similarity index 94% rename from lib/crowdb-kv/tests/group_test/full_restart_delete_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/full_restart_delete_test.rs index 4a348a668..3bff421ec 100644 --- a/lib/crowdb-kv/tests/group_test/full_restart_delete_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/full_restart_delete_test.rs @@ -4,7 +4,7 @@ //! Full-cluster restart durability for deletes. //! //! Reproduces the web-console `cluster_restart_recovery_test` failure at the -//! `crowdb_kv` layer (no console / crowdb-rpc proxy): a 3-node WAL-backed group commits +//! `crowdb_paxos_kv` layer (no console / crowdb-rpc proxy): a 3-node WAL-backed group commits //! puts and deletes, then EVERY node is killed and restarted from its own WAL //! dir. After restart the deleted keys must stay deleted (no resurrection) and //! the surviving keys must remain readable. @@ -14,15 +14,15 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_config::GroupConfigStore; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::{PxElectionConfig, WalConfig}; -use crowdb_kv::rpc::{KvDeleteRequest, KvGetRequest, KvSetRequest}; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine}; -use crowdb_kv_client::KvRpcTransport; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_config::GroupConfigStore; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::{PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::{PxElectionConfig, WalConfig}; +use crowdb_paxos_kv::rpc::{KvDeleteRequest, KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine}; +use crowdb_paxos_kv_client::PaxosKvRpcTransport; use crate::common::test_client::TestKvClient; @@ -36,7 +36,7 @@ struct WalNode { struct WalCluster { nodes: Vec<WalNode>, - kv_transport: Arc<KvRpcTransport>, + kv_transport: Arc<PaxosKvRpcTransport>, _tmp: crowdb_test_harness::test_dirs::TestDir, _net: tokio::sync::MutexGuard<'static, ()>, } @@ -135,7 +135,7 @@ async fn start_wal_cluster(ids: &[u64]) -> WalCluster { WalCluster { nodes, - kv_transport: Arc::new(KvRpcTransport::new()), + kv_transport: Arc::new(PaxosKvRpcTransport::new()), _tmp: tmp, _net: net, } diff --git a/lib/crowdb-kv/tests/group_test/g1_step_down_survival_test.rs b/lib/crowdb-paxos-kv/tests/group_test/g1_step_down_survival_test.rs similarity index 97% rename from lib/crowdb-kv/tests/group_test/g1_step_down_survival_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/g1_step_down_survival_test.rs index bbbccdc48..f39588f01 100644 --- a/lib/crowdb-kv/tests/group_test/g1_step_down_survival_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/g1_step_down_survival_test.rs @@ -8,8 +8,8 @@ use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::replica::StepDownRequestPayload; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::cluster::replica::StepDownRequestPayload; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster}; diff --git a/lib/crowdb-kv/tests/group_test/g2_crash_restart_no_data_loss_test.rs b/lib/crowdb-paxos-kv/tests/group_test/g2_crash_restart_no_data_loss_test.rs similarity index 96% rename from lib/crowdb-kv/tests/group_test/g2_crash_restart_no_data_loss_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/g2_crash_restart_no_data_loss_test.rs index f0117c191..e5cd205aa 100644 --- a/lib/crowdb-kv/tests/group_test/g2_crash_restart_no_data_loss_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/g2_crash_restart_no_data_loss_test.rs @@ -21,15 +21,15 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_config::GroupConfigStore; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::{PxElectionConfig, WalConfig}; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; -use crowdb_kv_client::KvRpcTransport; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_config::GroupConfigStore; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::{PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::{PxElectionConfig, WalConfig}; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; +use crowdb_paxos_kv_client::PaxosKvRpcTransport; use crate::common::test_client::TestKvClient; @@ -44,7 +44,7 @@ struct WalNode { /// A 3-node cluster where every replica logs to its own `File`-backed WAL dir. struct WalCluster { nodes: Vec<WalNode>, - kv_transport: Arc<KvRpcTransport>, + kv_transport: Arc<PaxosKvRpcTransport>, _tmp: crowdb_test_harness::test_dirs::TestDir, _net: tokio::sync::MutexGuard<'static, ()>, } @@ -144,7 +144,7 @@ async fn start_wal_cluster(ids: &[u64]) -> WalCluster { WalCluster { nodes, - kv_transport: Arc::new(KvRpcTransport::new()), + kv_transport: Arc::new(PaxosKvRpcTransport::new()), _tmp: tmp, _net: net, } diff --git a/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs b/lib/crowdb-paxos-kv/tests/group_test/g3_leader_change_test.rs similarity index 95% rename from lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/g3_leader_change_test.rs index 1965f3d88..eabaa773e 100644 --- a/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/g3_leader_change_test.rs @@ -12,9 +12,9 @@ use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::replica::StepDownRequestPayload; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::replica::StepDownRequestPayload; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster}; @@ -221,10 +221,10 @@ async fn leader_wait_does_not_accept_role_before_a_current_quorum_lease() { #[tokio::test(flavor = "current_thread", start_paused = true)] async fn prior_tenure_admin_wakeup_cannot_step_down_a_newly_elected_leader() { - use crowdb_kv::cluster::group::PxGroup; - use crowdb_kv::cluster::group_election::spawn; - use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; - use crowdb_kv::common::config::PxElectionConfig; + use crowdb_paxos_kv::cluster::group::PxGroup; + use crowdb_paxos_kv::cluster::group_election::spawn; + use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; + use crowdb_paxos_kv::common::config::PxElectionConfig; use std::sync::Arc; let config = PxElectionConfig::for_tests(); diff --git a/lib/crowdb-kv/tests/group_test/g4_learner_stream_test.rs b/lib/crowdb-paxos-kv/tests/group_test/g4_learner_stream_test.rs similarity index 98% rename from lib/crowdb-kv/tests/group_test/g4_learner_stream_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/g4_learner_stream_test.rs index a5842b7a1..916b215b9 100644 --- a/lib/crowdb-kv/tests/group_test/g4_learner_stream_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/g4_learner_stream_test.rs @@ -21,7 +21,7 @@ use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster}; use crate::common::test_client::TestKvClient; diff --git a/lib/crowdb-kv/tests/group_test/g5_recovery_test.rs b/lib/crowdb-paxos-kv/tests/group_test/g5_recovery_test.rs similarity index 98% rename from lib/crowdb-kv/tests/group_test/g5_recovery_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/g5_recovery_test.rs index e4b71daac..16c893771 100644 --- a/lib/crowdb-kv/tests/group_test/g5_recovery_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/g5_recovery_test.rs @@ -16,8 +16,8 @@ use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::replica::StepDownRequestPayload; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::cluster::replica::StepDownRequestPayload; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader, TestCluster}; diff --git a/lib/crowdb-kv/tests/group_test/g6_reconfig_test.rs b/lib/crowdb-paxos-kv/tests/group_test/g6_reconfig_test.rs similarity index 96% rename from lib/crowdb-kv/tests/group_test/g6_reconfig_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/g6_reconfig_test.rs index bbcfecd8a..6491b1e3f 100644 --- a/lib/crowdb-kv/tests/group_test/g6_reconfig_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/g6_reconfig_test.rs @@ -18,11 +18,13 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::common::config::PxElectionConfig; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::{ + PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica, +}; +use crowdb_paxos_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster}; diff --git a/lib/crowdb-kv/tests/group_test/g7_reconfig_remove_leader_test.rs b/lib/crowdb-paxos-kv/tests/group_test/g7_reconfig_remove_leader_test.rs similarity index 97% rename from lib/crowdb-kv/tests/group_test/g7_reconfig_remove_leader_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/g7_reconfig_remove_leader_test.rs index 715efd1ea..4ed8c0815 100644 --- a/lib/crowdb-kv/tests/group_test/g7_reconfig_remove_leader_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/g7_reconfig_remove_leader_test.rs @@ -13,10 +13,10 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxRemoteReplica}; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxRemoteReplica}; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster}; diff --git a/lib/crowdb-kv/tests/group_test/group_config_test.rs b/lib/crowdb-paxos-kv/tests/group_test/group_config_test.rs similarity index 97% rename from lib/crowdb-kv/tests/group_test/group_config_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/group_config_test.rs index 3e554c0e4..e81824371 100644 --- a/lib/crowdb-kv/tests/group_test/group_config_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/group_config_test.rs @@ -4,7 +4,7 @@ //! Unit tests for `PxGroupConfig` encode/decode and `GroupConfigStore` //! file-based persistence. -use crowdb_kv::cluster::group_config::{GroupConfigStore, PxGroupConfig, PxGroupMember}; +use crowdb_paxos_kv::cluster::group_config::{GroupConfigStore, PxGroupConfig, PxGroupMember}; #[test] fn roundtrip_config() { diff --git a/lib/crowdb-kv/tests/group_test/group_propose_test.rs b/lib/crowdb-paxos-kv/tests/group_test/group_propose_test.rs similarity index 83% rename from lib/crowdb-kv/tests/group_test/group_propose_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/group_propose_test.rs index fc6bebef7..0e6f8a2bb 100644 --- a/lib/crowdb-kv/tests/group_test/group_propose_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/group_propose_test.rs @@ -3,11 +3,14 @@ //! `PxGroup` unit tests for single-local-only group proposal. +use crate::test_util::encode_put_payload; use std::sync::Arc; -use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::rpc::PxRpcTransport; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{ + PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica, +}; +use crowdb_paxos_kv::rpc::PxRpcTransport; fn single_leader_group() -> PxGroup { let local = PxLocalReplica::new(1, PxLocalReplicaRole::Leader); @@ -18,7 +21,10 @@ fn single_leader_group() -> PxGroup { async fn single_local_propose_succeeds() { let group = single_leader_group(); - match group.propose(b"payload-1".to_vec(), Some(1), Some(1)).await { + match group + .propose(encode_put_payload(b"key", b"payload-1"), Some(1), Some(1)) + .await + { ProposeResult::Chosen { slot } => { assert_eq!(slot, 1, "first proposal should get slot 1"); } @@ -26,7 +32,10 @@ async fn single_local_propose_succeeds() { } // Second proposal gets next slot - match group.propose(b"payload-2".to_vec(), Some(1), Some(2)).await { + match group + .propose(encode_put_payload(b"key", b"payload-2"), Some(1), Some(2)) + .await + { ProposeResult::Chosen { slot } => { assert_eq!(slot, 2, "second proposal should get slot 2"); } @@ -37,12 +46,17 @@ async fn single_local_propose_succeeds() { #[tokio::test] async fn single_local_propose_learns_entry() { let group = single_leader_group(); - let _ = group.propose(b"test-value".to_vec(), Some(1), Some(1)).await; + let _ = group + .propose(encode_put_payload(b"key", b"test-value"), Some(1), Some(1)) + .await; // Verify learner applied the payload let replica = group.local_replica(); let accepted = replica.accepted_at(1).await.expect("slot 1 accepted"); - assert_eq!(accepted.payload.as_ref(), b"test-value"); + assert_eq!( + accepted.payload.as_ref(), + encode_put_payload(b"key", b"test-value") + ); } #[tokio::test] @@ -53,7 +67,10 @@ async fn follower_group_rejects_proposal() { group.set_remote_replicas(remotes); group.local_replica().set_believed_leader(1); - match group.propose(b"payload".to_vec(), Some(1), Some(1)).await { + match group + .propose(encode_put_payload(b"key", b"payload"), Some(1), Some(1)) + .await + { ProposeResult::NotLeader { leader_hint } => { assert_eq!(leader_hint, "127.0.0.1:9999"); } @@ -67,7 +84,10 @@ async fn single_local_classic_propose_succeeds() { let mut group = PxGroup::new(1, local); group.set_force_classic(true); - match group.propose(b"classic-payload".to_vec(), Some(1), Some(1)).await { + match group + .propose(encode_put_payload(b"key", b"classic-payload"), Some(1), Some(1)) + .await + { ProposeResult::Chosen { slot } => { assert_eq!(slot, 1); } @@ -77,14 +97,20 @@ async fn single_local_classic_propose_succeeds() { // Verify entry was learned let replica = group.local_replica(); let accepted = replica.accepted_at(1).await.expect("slot 1 accepted"); - assert_eq!(accepted.payload.as_ref(), b"classic-payload"); + assert_eq!( + accepted.payload.as_ref(), + encode_put_payload(b"key", b"classic-payload") + ); } #[tokio::test] async fn propose_with_no_client_id() { let group = single_leader_group(); - match group.propose(b"no-client".to_vec(), None, None).await { + match group + .propose(encode_put_payload(b"key", b"no-client"), None, None) + .await + { ProposeResult::Chosen { slot } => { assert_eq!(slot, 1); } @@ -133,7 +159,10 @@ async fn non_voting_remote_accept_does_not_count_toward_quorum() { // non-voting and must not shrink the effective bar. assert_eq!(group.quorum(), 2); - if let ProposeResult::Chosen { .. } = group.propose(b"payload".to_vec(), Some(1), Some(1)).await { + if let ProposeResult::Chosen { .. } = group + .propose(encode_put_payload(b"key", b"payload"), Some(1), Some(1)) + .await + { panic!( "value must not be Chosen: only the local self-accept is a real \ voting ack, quorum is 2, and the non-voting remote's Accepted \ @@ -188,7 +217,10 @@ async fn membership_epoch_mismatch_fences_prepare_and_accept_until_epochs_match( assert_eq!(group.membership_epoch(), 0); assert_eq!(group.quorum(), 2); - match group.propose(b"payload".to_vec(), Some(1), Some(1)).await { + match group + .propose(encode_put_payload(b"key", b"payload"), Some(1), Some(1)) + .await + { ProposeResult::Chosen { .. } => {} other => panic!( "expected Chosen after epoch convergence, got {other:?}; \ @@ -204,7 +236,10 @@ async fn membership_epoch_mismatch_fences_prepare_and_accept_until_epochs_match( ); // A second proposal at the now-matching epoch must succeed normally. - match group.propose(b"payload2".to_vec(), Some(1), Some(2)).await { + match group + .propose(encode_put_payload(b"key", b"payload2"), Some(1), Some(2)) + .await + { ProposeResult::Chosen { .. } => {} other => panic!("expected Chosen once epochs match, got {other:?}"), } @@ -220,7 +255,11 @@ async fn sequential_proposals_allocate_increasing_slots() { let mut slots = Vec::new(); for i in 0..5 { match group - .propose(format!("val-{i}").into_bytes(), Some(1), Some(i)) + .propose( + encode_put_payload(b"key", format!("val-{i}").as_bytes()), + Some(1), + Some(i), + ) .await { ProposeResult::Chosen { slot } => slots.push(slot), diff --git a/lib/crowdb-kv/tests/group_test/group_test.rs b/lib/crowdb-paxos-kv/tests/group_test/group_test.rs similarity index 93% rename from lib/crowdb-kv/tests/group_test/group_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/group_test.rs index 09b23ce10..38786925b 100644 --- a/lib/crowdb-kv/tests/group_test/group_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/group_test.rs @@ -1,9 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::replica::Replica; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::replica::Replica; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; fn sample_group() -> PxGroup { let remote_replicas = vec![PxRemoteReplica::new(2, "127.0.0.1:2".to_string())]; diff --git a/lib/crowdb-kv/tests/group_test/kv_correctness_test.rs b/lib/crowdb-paxos-kv/tests/group_test/kv_correctness_test.rs similarity index 98% rename from lib/crowdb-kv/tests/group_test/kv_correctness_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/kv_correctness_test.rs index 83179410e..7eb1a897b 100644 --- a/lib/crowdb-kv/tests/group_test/kv_correctness_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/kv_correctness_test.rs @@ -12,7 +12,7 @@ use crate::common::cluster::{start_cluster, TestCluster}; use crate::common::test_client::TestKvClient; use bytes::Bytes; -use crowdb_kv::rpc::{KvBatchItem, KvBatchWriteRequest, KvDeleteRequest, KvSetRequest}; +use crowdb_paxos_kv::rpc::{KvBatchItem, KvBatchWriteRequest, KvDeleteRequest, KvSetRequest}; async fn put(client: &mut TestKvClient, key: &[u8], val: &[u8], req_id: u64) { let resp = client diff --git a/lib/crowdb-kv/tests/group_test/kv_edge_case_test.rs b/lib/crowdb-paxos-kv/tests/group_test/kv_edge_case_test.rs similarity index 99% rename from lib/crowdb-kv/tests/group_test/kv_edge_case_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/kv_edge_case_test.rs index 6f69b65aa..6da661a57 100644 --- a/lib/crowdb-kv/tests/group_test/kv_edge_case_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/kv_edge_case_test.rs @@ -10,7 +10,7 @@ use crate::common::cluster::{start_cluster, TestCluster}; use crate::common::test_client::TestKvClient; use bytes::Bytes; -use crowdb_kv::rpc::KvSetRequest; +use crowdb_paxos_kv::rpc::KvSetRequest; async fn put_raw(client: &mut TestKvClient, key: &[u8], val: &[u8], req_id: u64) { let resp = client diff --git a/lib/crowdb-kv/tests/group_test/kv_forward_test.rs b/lib/crowdb-paxos-kv/tests/group_test/kv_forward_test.rs similarity index 85% rename from lib/crowdb-kv/tests/group_test/kv_forward_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/kv_forward_test.rs index c79ba1842..459e75f7d 100644 --- a/lib/crowdb-kv/tests/group_test/kv_forward_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/kv_forward_test.rs @@ -9,7 +9,7 @@ //! 1. A follower whose local learner store is empty still returns the //! leader's committed value when the client calls `Get`/`Scan` — //! proving the forward path engaged. -//! 2. The `x-crowdb-kv-forwarded: 1` metadata header disables the +//! 2. The `x-crowdb-paxos-kv-forwarded: 1` metadata header disables the //! forwarder on the receiving node, so a misrouted request can hop //! at most once and never enters an infinite loop. //! @@ -22,7 +22,7 @@ use crate::common::cluster::start_cluster; use bytes::Bytes; -use crowdb_kv::rpc::{KvGetRequest, KvScanRequest, KvSetRequest}; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvScanRequest, KvSetRequest}; #[tokio::test] async fn follower_get_forwards_to_leader_after_local_clear() { @@ -54,30 +54,28 @@ async fn follower_get_forwards_to_leader_after_local_clear() { assert!(put.ok, "leader put should succeed"); // 2. Confirm Paxos has propagated the value to the follower. - // R65: follower apply is driven by ChosenNotice (async). Poll until - // the follower's engine has the value, with a bounded timeout. + // Wait for apply completion, including release of the native apply ticket. let follower_group = follower.get_group(1).expect("group 1 on follower"); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); - loop { - if follower_group + let target = leader.get_group(1).unwrap().local_replica().contiguous_chosen(); + tokio::time::timeout( + std::time::Duration::from_secs(3), + follower_group.local_replica().learner.await_applied(target), + ) + .await + .expect("follower must finish applying the leader's write"); + assert_eq!( + follower_group .local_replica() .learner .engine_get(b"fk".as_slice()) .await - .is_some() - { - break; - } - assert!( - std::time::Instant::now() < deadline, - "expected paxos to propagate value to follower's learner" - ); - tokio::time::sleep(std::time::Duration::from_millis(5)).await; - } + .map(|(_, value)| value), + Some(b"fv".to_vec()) + ); // 3. Clear the follower's local learner store. After this, a // local-only read on the follower would return not_found. - follower_group.local_replica().learner.engine().clear(); + follower_group.local_replica().learner.engine().clear().unwrap(); // 4. Read via the follower's RPC. The transparent forward should // return the leader's value, not not_found. @@ -148,7 +146,14 @@ async fn follower_scan_forwards_to_leader_after_local_clear() { // Wait for propagation, then clear follower's store. let follower_group = follower.get_group(1).expect("group"); - follower_group.local_replica().learner.engine().clear(); + let target = leader.get_group(1).unwrap().local_replica().contiguous_chosen(); + tokio::time::timeout( + std::time::Duration::from_secs(3), + follower_group.local_replica().learner.await_applied(target), + ) + .await + .expect("follower must finish applying the leader's writes"); + follower_group.local_replica().learner.engine().clear().unwrap(); let follower_client = cluster.isolated_kv_client(follower).await; let resp = follower_client @@ -168,7 +173,7 @@ async fn follower_scan_forwards_to_leader_after_local_clear() { deadline_ms: 0, bounded: false, scan_cutoff: 0, - direction: crowdb_kv::rpc::KvScanDirection::Forward, + direction: crowdb_paxos_kv::rpc::KvScanDirection::Forward, }) .await .expect("kv scan on follower") @@ -200,7 +205,7 @@ async fn follower_scan_forwards_to_leader_after_local_clear() { deadline_ms: 0, bounded: false, scan_cutoff: 0, - direction: crowdb_kv::rpc::KvScanDirection::Reverse, + direction: crowdb_paxos_kv::rpc::KvScanDirection::Reverse, }) .await .expect("reverse kv scan on follower") diff --git a/lib/crowdb-kv/tests/group_test/kv_slot_retry_test.rs b/lib/crowdb-paxos-kv/tests/group_test/kv_slot_retry_test.rs similarity index 93% rename from lib/crowdb-kv/tests/group_test/kv_slot_retry_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/kv_slot_retry_test.rs index 5df48d2a5..c29b7fa2a 100644 --- a/lib/crowdb-kv/tests/group_test/kv_slot_retry_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/kv_slot_retry_test.rs @@ -3,9 +3,9 @@ use crate::common::cluster::start_cluster_classic; use bytes::Bytes; -use crowdb_kv::cluster::group::ProposeResult; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::rpc::KvSetRequest; +use crowdb_paxos_kv::cluster::group::ProposeResult; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::rpc::KvSetRequest; fn encode_put_payload(key: &[u8], value: &[u8]) -> Vec<u8> { let mut payload = Vec::new(); @@ -33,7 +33,7 @@ async fn conditional_foreign_slot_returns_the_value_from_the_renewed_comparison( }; assert!(matches!( group.local_replica().on_accept(&entry).await, - crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. } + crowdb_paxos_kv::paxos::roles::PxAcceptReply::Accepted { .. } )); } let group = cluster.leader().get_group(1).unwrap(); @@ -71,7 +71,10 @@ async fn kv_put_retries_next_slot_when_slot_has_prior_accepted_value() { }; let reply = follower_replica.on_accept(&entry).await; assert!( - matches!(reply, crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. }), + matches!( + reply, + crowdb_paxos_kv::paxos::roles::PxAcceptReply::Accepted { .. } + ), "preload accept should succeed: {reply:?}" ); } diff --git a/lib/crowdb-kv/tests/group_test/kv_test.rs b/lib/crowdb-paxos-kv/tests/group_test/kv_test.rs similarity index 97% rename from lib/crowdb-kv/tests/group_test/kv_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/kv_test.rs index 9b3302130..70cc4150a 100644 --- a/lib/crowdb-kv/tests/group_test/kv_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/kv_test.rs @@ -5,7 +5,7 @@ use crate::common::cluster::{start_cluster, TestCluster}; use bytes::Bytes; -use crowdb_kv::rpc::{KvBatchItem, KvBatchWriteRequest, KvDeleteRequest, KvSetRequest}; +use crowdb_paxos_kv::rpc::{KvBatchItem, KvBatchWriteRequest, KvDeleteRequest, KvSetRequest}; #[tokio::test] async fn kv_mutations_apply_to_all_learners() { diff --git a/lib/crowdb-kv/tests/group_test/learner_heartbeat_test.rs b/lib/crowdb-paxos-kv/tests/group_test/learner_heartbeat_test.rs similarity index 92% rename from lib/crowdb-kv/tests/group_test/learner_heartbeat_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/learner_heartbeat_test.rs index b47493944..4552d666c 100644 --- a/lib/crowdb-kv/tests/group_test/learner_heartbeat_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/learner_heartbeat_test.rs @@ -1,8 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{ + PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica, +}; use std::sync::Arc; use std::time::Duration; diff --git a/lib/crowdb-kv/tests/group_test/maintenance_test.rs b/lib/crowdb-paxos-kv/tests/group_test/maintenance_test.rs similarity index 94% rename from lib/crowdb-kv/tests/group_test/maintenance_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/maintenance_test.rs index fc40638f8..c3350ee30 100644 --- a/lib/crowdb-kv/tests/group_test/maintenance_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/maintenance_test.rs @@ -14,17 +14,17 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::PxElectionConfig; -use crowdb_kv::kv::{Batch, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, KVFuture, ScanDirection}; -use crowdb_kv::metrics::MetricsRegistry; -use crowdb_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::wal_engine::WalEngine; -use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::kv::{Batch, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, KVFuture, ScanDirection}; +use crowdb_paxos_kv::metrics::MetricsRegistry; +use crowdb_paxos_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; use super::mem_kv::InMemKV; @@ -62,8 +62,8 @@ impl KVEngine for FailingSnapshotEngine { ) } - fn clear(&self) { - self.0.clear(); + fn clear(&self) -> Result<(), String> { + self.0.clear() } fn as_any(&self) -> &dyn std::any::Any { @@ -367,7 +367,7 @@ async fn maintenance_pass_does_not_gc_wal_when_safe_slot_lags_snapshot() { snapshot_slot_threshold: 1, ..PxElectionConfig::for_tests() }); - group.add_remote_replica(crowdb_kv::cluster::PxRemoteReplica::new( + group.add_remote_replica(crowdb_paxos_kv::cluster::PxRemoteReplica::new( 2, "127.0.0.1:2".to_string(), )); diff --git a/lib/crowdb-kv/tests/group_test/membership_epoch_fence_test.rs b/lib/crowdb-paxos-kv/tests/group_test/membership_epoch_fence_test.rs similarity index 98% rename from lib/crowdb-kv/tests/group_test/membership_epoch_fence_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/membership_epoch_fence_test.rs index 2a51bbee3..23d27f4a3 100644 --- a/lib/crowdb-kv/tests/group_test/membership_epoch_fence_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/membership_epoch_fence_test.rs @@ -14,7 +14,7 @@ //! processes-in-process, real crowdb-rpc wire messages, no mocks) rather than //! the console's HTTP mgmt layer: the epoch bump itself is what's under //! test here, not the HTTP fan-out mechanics already covered by -//! `crowdb-kv-server`'s `add_remote_replicas` tests. Calling +//! `crowdb-paxos-kv-server`'s `add_remote_replicas` tests. Calling //! `PxGroup::set_membership_epoch` directly on individual nodes is the //! same effect an `add_remote_replicas`/`remove_remote_replica` call has //! on that one node's own `PxGroup` (`bump_membership_epoch`), just @@ -25,7 +25,7 @@ use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster}; diff --git a/lib/crowdb-kv/tests/group_test/owner_fence_test.rs b/lib/crowdb-paxos-kv/tests/group_test/owner_fence_test.rs similarity index 93% rename from lib/crowdb-kv/tests/group_test/owner_fence_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/owner_fence_test.rs index 9f9e5177e..1003a7149 100644 --- a/lib/crowdb-kv/tests/group_test/owner_fence_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/owner_fence_test.rs @@ -2,8 +2,8 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; use std::sync::Arc; fn encode(key: &Bytes, value: &[u8]) -> Vec<u8> { @@ -18,10 +18,20 @@ fn encode(key: &Bytes, value: &[u8]) -> Vec<u8> { } async fn owner_group(key: &[u8]) -> (Arc<PxGroup>, Bytes) { - let group = Arc::new(PxGroup::new( - 1, - PxLocalReplica::new(1, PxLocalReplicaRole::Leader), - )); + // Each concurrency test owns its quota; unrelated tests must not consume + // the reservations whose availability this fixture assumes. + let engine = crowdb_paxos_kv::kv::CrowdbTreeEngine::open(&crowdb_paxos_kv::kv::CrowdbTreeConfig { + runtime: Some(Arc::new( + crowdb_tree_ffi::TreeRuntime::new(crowdb_tree_ffi::TreeRuntimeOptions::default()).unwrap(), + )), + ..Default::default() + }) + .unwrap(); + let mut local = PxLocalReplica::new(1, PxLocalReplicaRole::Leader); + local.learner = Arc::new(crowdb_paxos_kv::paxos::learner::PxLearner::with_engine(Box::new( + engine, + ))); + let group = Arc::new(PxGroup::new(1, local)); group.set_self_weak(); let key = Bytes::copy_from_slice(key); assert!(matches!( @@ -224,7 +234,7 @@ async fn topology_replacement_preserves_the_handover_barrier(key: &[u8]) { } async fn new_leader_tenure_does_not_reuse_a_closed_old_admission(key: &[u8]) { - use crowdb_kv::cluster::group_election::LeaderElection; + use crowdb_paxos_kv::cluster::group_election::LeaderElection; let (group, fence) = owner_group(key).await; let admitted = group.hold_owner_write_for_tests(fence.clone()); let old_claim = { diff --git a/lib/crowdb-kv/tests/group_test/paxos_error_test.rs b/lib/crowdb-paxos-kv/tests/group_test/paxos_error_test.rs similarity index 91% rename from lib/crowdb-kv/tests/group_test/paxos_error_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/paxos_error_test.rs index 6a591e82f..3c44552cf 100644 --- a/lib/crowdb-kv/tests/group_test/paxos_error_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/paxos_error_test.rs @@ -6,11 +6,12 @@ //! error-classifier unit tests live at `tests/paxos/error_test.rs`. use crate::common::cluster::start_cluster; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::paxos::roles::PxBallot; -use crowdb_kv::rpc::PrepareRequest; +use crate::test_util::encode_put_payload; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv::paxos::roles::PxBallot; +use crowdb_paxos_kv::rpc::PrepareRequest; #[tokio::test] async fn follower_request_maps_to_not_leader_with_hint() { @@ -66,9 +67,11 @@ async fn prepare_rejection_blocks_low_ballot_until_retry_uses_higher_ballot() { // Use PxGroup::propose() - it should detect preemption, bump ballot, and succeed let leader = cluster.leader(); let group = leader.get_group(1).expect("group exists"); - let result = group.propose(b"test-value".to_vec(), Some(1), Some(1)).await; + let result = group + .propose(encode_put_payload(b"key", b"test-value"), Some(1), Some(1)) + .await; match result { - crowdb_kv::cluster::group::ProposeResult::Chosen { slot: _ } => { + crowdb_paxos_kv::cluster::group::ProposeResult::Chosen { slot: _ } => { // Success - PxGroup handled the ballot bump internally } _ => panic!("Expected Chosen result, got {result:?}"), diff --git a/lib/crowdb-kv/tests/group_test/preemption_retry_test.rs b/lib/crowdb-paxos-kv/tests/group_test/preemption_retry_test.rs similarity index 77% rename from lib/crowdb-kv/tests/group_test/preemption_retry_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/preemption_retry_test.rs index 02f955dcb..f70f841cd 100644 --- a/lib/crowdb-kv/tests/group_test/preemption_retry_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/preemption_retry_test.rs @@ -2,8 +2,9 @@ // Licensed under the Apache License, Version 2.0. use crate::common::cluster::{assert_all_accepted, start_cluster}; -use crowdb_kv::paxos::roles::PxBallot; -use crowdb_kv::rpc::PrepareRequest; +use crate::test_util::encode_put_payload; +use crowdb_paxos_kv::paxos::roles::PxBallot; +use crowdb_paxos_kv::rpc::PrepareRequest; #[tokio::test] async fn integration_accept_preemption_then_ballot_bump_retry() { @@ -39,10 +40,12 @@ async fn integration_accept_preemption_then_ballot_bump_retry() { // Use PxGroup::propose() - it should detect preemption, bump ballot, and succeed let leader = cluster.leader(); let group = leader.get_group(1).expect("group exists"); - let result = group.propose(b"test-value".to_vec(), Some(1), Some(1)).await; + let result = group + .propose(encode_put_payload(b"key", b"test-value"), Some(1), Some(1)) + .await; match result { - crowdb_kv::cluster::group::ProposeResult::Chosen { slot } => { - assert_all_accepted(&cluster, slot, b"test-value").await; + crowdb_paxos_kv::cluster::group::ProposeResult::Chosen { slot } => { + assert_all_accepted(&cluster, slot, &encode_put_payload(b"key", b"test-value")).await; } _ => panic!("Expected Chosen result, got {result:?}"), } diff --git a/lib/crowdb-kv/tests/group_test/proposer_test.rs b/lib/crowdb-paxos-kv/tests/group_test/proposer_test.rs similarity index 73% rename from lib/crowdb-kv/tests/group_test/proposer_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/proposer_test.rs index 98d0c29ce..28929dd38 100644 --- a/lib/crowdb-kv/tests/group_test/proposer_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/proposer_test.rs @@ -8,10 +8,11 @@ //! using a single-voter leader group so quorum is 1 and no peer RPCs are //! needed. -use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::PaxosConfig; -use crowdb_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; +use crate::test_util::encode_put_payload; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PaxosConfig; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; /// Single-voter leader group: quorum is 1, so propose / repair complete /// against the local acceptor with no peer RPCs. @@ -33,14 +34,20 @@ async fn propose_returns_busy_when_window_is_full() { PaxosConfig::DEFAULT.max_inflight_proposals, "should acquire all default window permits" ); - match group.propose(b"v".to_vec(), Some(1), Some(1)).await { + match group + .propose(encode_put_payload(b"key", b"v"), Some(1), Some(1)) + .await + { ProposeResult::Busy => {} other => panic!("expected Busy with a full window, got {other:?}"), } // Releasing the permits reopens admission. drop(held); - match group.propose(b"v".to_vec(), Some(1), Some(2)).await { + match group + .propose(encode_put_payload(b"key", b"v"), Some(1), Some(2)) + .await + { ProposeResult::Chosen { .. } => {} other => panic!("expected Chosen after window drained, got {other:?}"), } @@ -52,19 +59,15 @@ async fn repair_once_fills_gap_and_advances_frontier() { // Learn slot 2 directly, leaving slot 1 as an abandoned gap: the // contiguous frontier stays at 0 while the highest-seen slot is 2. - group - .local_replica() - .learner - .learn( - PxLogEntry { - slot: 2, - ballot: PxBallot::new(1, 1), - term: 0, - payload: bytes::Bytes::from_static(b""), - }, - &[], - ) - .await; + group.local_replica().spawn_learn_chosen( + PxLogEntry { + slot: 2, + ballot: PxBallot::new(1, 1), + term: 0, + payload: bytes::Bytes::from_static(b""), + }, + &[], + ); assert_eq!(group.local_replica().contiguous_chosen(), 0, "gap below slot 2"); assert_eq!(group.local_replica().last_chosen_slot(), 2); @@ -80,6 +83,7 @@ async fn repair_once_fills_gap_and_advances_frontier() { 2, "frontier advances past the filled gap and the trailing learned slot" ); + group.local_replica().learner.await_applied(2).await; // A second repair has nothing to do. assert_eq!( @@ -110,7 +114,10 @@ async fn propose_queues_when_policy_is_queue() { // Launch a propose in the background — it should block. let g = group.clone(); - let task = tokio::spawn(async move { g.propose(b"v".to_vec(), Some(2), Some(1)).await }); + let task = tokio::spawn(async move { + g.propose(encode_put_payload(b"key", b"v"), Some(2), Some(1)) + .await + }); // Give it a moment to ensure it's blocked. tokio::time::sleep(std::time::Duration::from_millis(50)).await; @@ -130,8 +137,8 @@ async fn propose_queues_when_policy_is_queue() { #[tokio::test(flavor = "current_thread", start_paused = true)] async fn gap_backlog_drains_without_waiting_one_heartbeat_per_slot() { - use crowdb_kv::cluster::group_election::{spawn, LeaderElection}; - use crowdb_kv::common::config::PxElectionConfig; + use crowdb_paxos_kv::cluster::group_election::{spawn, LeaderElection}; + use crowdb_paxos_kv::common::config::PxElectionConfig; use std::{sync::Arc, time::Duration}; let cfg = PxElectionConfig::for_tests(); @@ -142,32 +149,30 @@ async fn gap_backlog_drains_without_waiting_one_heartbeat_per_slot() { tokio::task::yield_now().await; } assert!(group.leader_read_ready()); - group - .local_replica() - .learner - .learn( - PxLogEntry { - slot: 65, - ballot: PxBallot::new(0, 1), - term: group.local_replica().current_term_snapshot(), - payload: bytes::Bytes::new(), - }, - &[], - ) - .await; + group.local_replica().spawn_learn_chosen( + PxLogEntry { + slot: 65, + ballot: PxBallot::new(0, 1), + term: group.local_replica().current_term_snapshot(), + payload: bytes::Bytes::new(), + }, + &[], + ); assert_eq!(group.local_replica().contiguous_chosen(), 0); // Dispatch one repair pass, then keep the clock fixed. Recovery work // must drain the existing backlog independently of heartbeat cadence. tokio::time::advance(Duration::from_millis(cfg.heartbeat_interval_ms + 1)).await; - for _ in 0..10_000 { - if group.local_replica().contiguous_applied() == 65 { - break; - } + let fixed_clock = tokio::time::Instant::now(); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + // Keep the paused Tokio clock runnable while the native worker completes. + // Counting yields does not bound the real work scheduled on that worker. + while group.local_replica().contiguous_applied() != 65 && std::time::Instant::now() < deadline { tokio::task::yield_now().await; } let frontier = group.local_replica().contiguous_applied(); cancel.cancel(); handle.await.unwrap(); assert_eq!(frontier, 65, "repair left a backlog waiting for heartbeat ticks"); + assert_eq!(tokio::time::Instant::now(), fixed_clock); } diff --git a/lib/crowdb-kv/tests/group_test/quorum_drain_test.rs b/lib/crowdb-paxos-kv/tests/group_test/quorum_drain_test.rs similarity index 82% rename from lib/crowdb-kv/tests/group_test/quorum_drain_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/quorum_drain_test.rs index 5c26760fb..1b102d2a7 100644 --- a/lib/crowdb-kv/tests/group_test/quorum_drain_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/quorum_drain_test.rs @@ -4,9 +4,12 @@ use std::sync::Arc; use std::time::Duration; -use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::rpc::PxRpcTransport; +use crate::test_util::encode_put_payload; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{ + PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica, +}; +use crowdb_paxos_kv::rpc::PxRpcTransport; #[tokio::test] async fn completed_quorum_drains_release_groups_without_tenure_cancellation() { @@ -30,7 +33,9 @@ async fn completed_quorum_drains_release_groups_without_tenure_cancellation() { // Both voters must answer before quorum: the detached prepare and // accept drains are already empty, without any tenure cancellation. assert!(matches!( - group.propose_in_tenure(b"payload".to_vec(), None, None, 0).await, + group + .propose_in_tenure(encode_put_payload(b"key", b"payload"), None, None, 0) + .await, ProposeResult::Chosen { .. } )); let weak = Arc::downgrade(&group); diff --git a/lib/crowdb-kv/tests/group_test/r65_replication_test.rs b/lib/crowdb-paxos-kv/tests/group_test/r65_replication_test.rs similarity index 98% rename from lib/crowdb-kv/tests/group_test/r65_replication_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/r65_replication_test.rs index 6b88ad70f..b3dc3bac0 100644 --- a/lib/crowdb-kv/tests/group_test/r65_replication_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/r65_replication_test.rs @@ -8,7 +8,7 @@ use crate::common::cluster::{ start_cluster, start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster, }; use bytes::Bytes; -use crowdb_kv::rpc::{KvDeleteRequest, KvSetRequest}; +use crowdb_paxos_kv::rpc::{KvDeleteRequest, KvSetRequest}; use std::time::{Duration, Instant}; /// Helper: assert all nodes in the cluster eventually see `expected` for `key`. @@ -346,7 +346,7 @@ async fn large_value_converges() { /// R65-8: Batch writes converge across all nodes. #[tokio::test] async fn batch_writes_converge() { - use crowdb_kv::rpc::{KvBatchItem, KvBatchWriteRequest}; + use crowdb_paxos_kv::rpc::{KvBatchItem, KvBatchWriteRequest}; let cluster = start_cluster(&[0, 1, 2], 0).await; let leader = cluster.leader(); let client = cluster.kv_client(leader).await; diff --git a/lib/crowdb-kv/tests/group_test/recovery_backlog_test.rs b/lib/crowdb-paxos-kv/tests/group_test/recovery_backlog_test.rs similarity index 99% rename from lib/crowdb-kv/tests/group_test/recovery_backlog_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/recovery_backlog_test.rs index 14f5dbf41..1d261a81c 100644 --- a/lib/crowdb-kv/tests/group_test/recovery_backlog_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/recovery_backlog_test.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crate::common::cluster::start_cluster; -use crowdb_kv::{ +use crowdb_paxos_kv::{ cluster::group_election::LeaderElection, common::config::PxElectionConfig, paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}, diff --git a/lib/crowdb-kv/tests/group_test/remote_error_test.rs b/lib/crowdb-paxos-kv/tests/group_test/remote_error_test.rs similarity index 89% rename from lib/crowdb-kv/tests/group_test/remote_error_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/remote_error_test.rs index 178b23f4b..d919b4286 100644 --- a/lib/crowdb-kv/tests/group_test/remote_error_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/remote_error_test.rs @@ -3,9 +3,9 @@ //! `PxRemoteReplica` connection failure and error handling tests. -use crowdb_kv::cluster::replica::{PxReplicaError, ReplicaClient}; -use crowdb_kv::cluster::PxRemoteReplica; -use crowdb_kv::paxos::roles::PxBallot; +use crowdb_paxos_kv::cluster::replica::{PxReplicaError, ReplicaClient}; +use crowdb_paxos_kv::cluster::PxRemoteReplica; +use crowdb_paxos_kv::paxos::roles::PxBallot; #[tokio::test] async fn send_prepare_to_unreachable_endpoint_returns_error() { @@ -30,7 +30,7 @@ async fn send_prepare_to_unreachable_endpoint_returns_error() { #[tokio::test] async fn send_accept_to_unreachable_endpoint_returns_error() { let remote = PxRemoteReplica::new(99, "127.0.0.1:1".to_string()); - let entry = crowdb_kv::paxos::roles::PxLogEntry { + let entry = crowdb_paxos_kv::paxos::roles::PxLogEntry { slot: 1, ballot: PxBallot::new(1, 0), term: 0, diff --git a/lib/crowdb-kv/tests/group_test/retention_test.rs b/lib/crowdb-paxos-kv/tests/group_test/retention_test.rs similarity index 94% rename from lib/crowdb-kv/tests/group_test/retention_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/retention_test.rs index ebf2a921a..244914c70 100644 --- a/lib/crowdb-kv/tests/group_test/retention_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/retention_test.rs @@ -5,12 +5,14 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}; -use crowdb_kv::wal::wal_engine::WalEngine; -use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; +use crowdb_paxos_kv::cluster::{ + PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica, +}; +use crowdb_paxos_kv::paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; async fn local(id: u64, role: PxLocalReplicaRole, applied: u64, durable: u64) -> PxLocalReplica { let mut replica = PxLocalReplica::new(id, role); diff --git a/lib/crowdb-kv/tests/group_test/safe_slot_test.rs b/lib/crowdb-paxos-kv/tests/group_test/safe_slot_test.rs similarity index 91% rename from lib/crowdb-kv/tests/group_test/safe_slot_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/safe_slot_test.rs index 9c912663d..b768a27e0 100644 --- a/lib/crowdb-kv/tests/group_test/safe_slot_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/safe_slot_test.rs @@ -8,9 +8,9 @@ //! Drives the crate-internal peer-applied injection through the `test-util` //! feature hook on `PxGroup`. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::paxos::roles::{Learner, PxBallot, PxLogEntry, SlotIndex}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv::paxos::roles::{Learner, PxBallot, PxLogEntry, SlotIndex}; /// Drive the local learner's contiguous-applied watermark to `upto` by /// learning slots `1..=upto` (empty `NoOp` payloads; only the frontier diff --git a/lib/crowdb-kv/tests/group_test/snapshot_join_test.rs b/lib/crowdb-paxos-kv/tests/group_test/snapshot_join_test.rs similarity index 73% rename from lib/crowdb-kv/tests/group_test/snapshot_join_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/snapshot_join_test.rs index a39c45329..830f8f286 100644 --- a/lib/crowdb-kv/tests/group_test/snapshot_join_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/snapshot_join_test.rs @@ -10,10 +10,10 @@ use crate::common::cluster::start_cluster; use crate::test_util::compare_dyn; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{KvServer, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::kv::{Batch, BatchOp, Op}; -use crowdb_kv::rpc::KvSetRequest; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{PaxosKvServer, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{Batch, BatchOp, Op}; +use crowdb_paxos_kv::rpc::KvSetRequest; #[tokio::test] async fn fresh_replica_joins_via_snapshot_and_matches_leader_state() { @@ -119,23 +119,44 @@ async fn snapshot_larger_than_64_mib_joins_through_bounded_reads() { }) .collect(), }; - leader_group - .local_replica() - .learner - .engine() - .apply(1, &batch) - .await - .expect("large source batch applies"); + // Keep each proposal within native progress limits while retaining the + // complete >64 MiB snapshot that this transfer test exercises. + let mut last_slot = 0; + for ops in batch.ops.chunks(4) { + last_slot += 1; + leader_group + .local_replica() + .learner + .engine() + .apply(last_slot, &Batch { ops: ops.to_vec() }) + .await + .expect("bounded source batch applies"); + } drop(batch); - let new_group = PxGroup::new(1, PxLocalReplica::new(99, PxLocalReplicaRole::Follower)); + // The test deliberately imports more than the default 64 MiB staging + // partition. Provision the final image plus its construction workspace; + // the wire read window remains 1 MiB throughout the transfer. + let mut host = crowdb_tree_ffi::TreeRuntimeOptions::default(); + host.limits.reclaim_bytes = ENTRY_COUNT * VALUE_BYTES * 3; + let mut config = crowdb_paxos_kv::kv::CrowdbTreeConfig::default(); + config.runtime_binding.limits.reclaim_bytes = host.limits.reclaim_bytes; + config.runtime = Some(std::sync::Arc::new( + crowdb_tree_ffi::TreeRuntime::new(host).unwrap(), + )); + let engine = crowdb_paxos_kv::kv::CrowdbTreeEngine::open(&config).unwrap(); + let mut replica = PxLocalReplica::new(99, PxLocalReplicaRole::Follower); + replica.learner = std::sync::Arc::new(crowdb_paxos_kv::paxos::learner::PxLearner::with_engine(Box::new( + engine, + ))); + let new_group = PxGroup::new(1, replica); let endpoint = leader.listen_addr().expect("leader started").to_string(); assert_eq!( new_group .join_via_snapshot_with_config(&endpoint, 1024 * 1024, 3) .await .expect("large snapshot join succeeds"), - 1 + last_slot ); let (keys, truncated) = new_group .local_replica() diff --git a/lib/crowdb-kv/tests/group_test/snapshot_slot_test.rs b/lib/crowdb-paxos-kv/tests/group_test/snapshot_slot_test.rs similarity index 95% rename from lib/crowdb-kv/tests/group_test/snapshot_slot_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/snapshot_slot_test.rs index 7c967d024..c5e0f7466 100644 --- a/lib/crowdb-kv/tests/group_test/snapshot_slot_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/snapshot_slot_test.rs @@ -17,10 +17,10 @@ use std::path::PathBuf; use std::sync::Arc; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::wal::wal_engine::WalEngine; -use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; fn sim_backend() -> Arc<IoBackend> { Arc::new(IoBackend::MemBlock(MemBlockDevice::new())) diff --git a/lib/crowdb-kv/tests/group_test/t1_early_ack_crash_test.rs b/lib/crowdb-paxos-kv/tests/group_test/t1_early_ack_crash_test.rs similarity index 96% rename from lib/crowdb-kv/tests/group_test/t1_early_ack_crash_test.rs rename to lib/crowdb-paxos-kv/tests/group_test/t1_early_ack_crash_test.rs index e920b38df..ef2e1a6b2 100644 --- a/lib/crowdb-kv/tests/group_test/t1_early_ack_crash_test.rs +++ b/lib/crowdb-paxos-kv/tests/group_test/t1_early_ack_crash_test.rs @@ -29,16 +29,16 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_config::GroupConfigStore; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::{PxElectionConfig, WalConfig}; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; -use crowdb_kv_client::KvRpcTransport; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_config::GroupConfigStore; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::{PxElectionConfig, WalConfig}; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; +use crowdb_paxos_kv_client::PaxosKvRpcTransport; use crate::common::test_client::TestKvClient; @@ -53,7 +53,7 @@ struct WalNode { struct WalCluster { nodes: Vec<WalNode>, - kv_transport: Arc<KvRpcTransport>, + kv_transport: Arc<PaxosKvRpcTransport>, _tmp: crowdb_test_harness::test_dirs::TestDir, _net: tokio::sync::MutexGuard<'static, ()>, } @@ -156,7 +156,7 @@ async fn start_wal_cluster(ids: &[u64]) -> WalCluster { WalCluster { nodes, - kv_transport: Arc::new(KvRpcTransport::new()), + kv_transport: Arc::new(PaxosKvRpcTransport::new()), _tmp: tmp, _net: net, } diff --git a/lib/crowdb-kv/tests/kv_test.rs b/lib/crowdb-paxos-kv/tests/kv_test.rs similarity index 91% rename from lib/crowdb-kv/tests/kv_test.rs rename to lib/crowdb-paxos-kv/tests/kv_test.rs index 324a9d657..690ea7478 100644 --- a/lib/crowdb-kv/tests/kv_test.rs +++ b/lib/crowdb-paxos-kv/tests/kv_test.rs @@ -28,3 +28,6 @@ mod op_codec; #[path = "kv_test/kv_future_test.rs"] mod kv_future; + +#[path = "kv_test/tree_work_metrics_test.rs"] +mod tree_work_metrics; diff --git a/lib/crowdb-kv/tests/kv_test/conformance_test.rs b/lib/crowdb-paxos-kv/tests/kv_test/conformance_test.rs similarity index 75% rename from lib/crowdb-kv/tests/kv_test/conformance_test.rs rename to lib/crowdb-paxos-kv/tests/kv_test/conformance_test.rs index 0a680301e..1ac5a5a05 100644 --- a/lib/crowdb-kv/tests/kv_test/conformance_test.rs +++ b/lib/crowdb-paxos-kv/tests/kv_test/conformance_test.rs @@ -11,7 +11,7 @@ use crate::test_util::{compare_dyn, iter_all_dyn}; use bytes::Bytes; -use crowdb_kv::kv::{Batch, BatchOp, Cell, KVEngine, Op, ScanDirection}; +use crowdb_paxos_kv::kv::{Batch, BatchOp, Cell, KVEngine, Op, ScanDirection}; use std::time::{SystemTime, UNIX_EPOCH}; fn now_ms() -> u64 { @@ -42,92 +42,99 @@ pub fn batch(ops: Vec<BatchOp>) -> Batch { Batch { ops } } -pub fn highest_slot_wins_regardless_of_apply_order(e: &dyn KVEngine) { - // Apply the higher slot first, then a lower slot for the same key. - e.apply(5, &batch(vec![put(b"k", b"v5")])).into_ready().unwrap(); - e.apply(3, &batch(vec![put(b"k", b"v3")])).into_ready().unwrap(); +pub async fn highest_slot_wins_regardless_of_apply_order(e: &dyn KVEngine) { + // Only these slots are empty; slot 3 still carries the delayed write. + for slot in [1, 2, 4] { + e.noop(slot); + } + let higher = batch(vec![put(b"k", b"v5")]); + let lower = batch(vec![put(b"k", b"v3")]); + let (high_result, low_result) = tokio::join!(biased; e.apply(5, &higher), e.apply(3, &lower)); + high_result.unwrap(); + low_result.unwrap(); + e.apply(3, &lower).await.unwrap(); assert_eq!( - e.get(b"k").into_ready(), + e.get(b"k").await, Some((5, b"v5".to_vec())), "lower slot must not overwrite" ); - // A strictly higher slot wins. - e.apply(7, &batch(vec![put(b"k", b"v7")])).into_ready().unwrap(); - assert_eq!(e.get(b"k").into_ready(), Some((7, b"v7".to_vec()))); + // A strictly higher slot wins after the empty slot 6. + e.noop(6); + e.apply(7, &batch(vec![put(b"k", b"v7")])).await.unwrap(); + assert_eq!(e.get(b"k").await, Some((7, b"v7".to_vec()))); } -pub fn equal_slot_is_idempotent_noop(e: &dyn KVEngine) { - e.apply(4, &batch(vec![put(b"k", b"first")])) - .into_ready() - .unwrap(); +pub async fn equal_slot_is_idempotent_noop(e: &dyn KVEngine) { + for slot in 1..4 { + e.noop(slot); + } + e.apply(4, &batch(vec![put(b"k", b"first")])).await.unwrap(); // Re-applying the same slot must not change the stored value. - e.apply(4, &batch(vec![put(b"k", b"second")])) - .into_ready() - .unwrap(); - assert_eq!(e.get(b"k").into_ready(), Some((4, b"first".to_vec()))); + e.apply(4, &batch(vec![put(b"k", b"second")])).await.unwrap(); + assert_eq!(e.get(b"k").await, Some((4, b"first".to_vec()))); } -pub fn delete_writes_tombstone(e: &dyn KVEngine) { - e.apply(1, &batch(vec![put(b"k", b"v")])).into_ready().unwrap(); - e.apply(2, &batch(vec![del(b"k")])).into_ready().unwrap(); - assert_eq!(e.get(b"k").into_ready(), None, "tombstoned key is not live"); +pub async fn delete_writes_tombstone(e: &dyn KVEngine) { + e.apply(1, &batch(vec![put(b"k", b"v")])).await.unwrap(); + e.apply(2, &batch(vec![del(b"k")])).await.unwrap(); + assert_eq!(e.get(b"k").await, None, "tombstoned key is not live"); // The tombstone is retained internally (visible via iter_all) at its slot. let all = iter_all_dyn(e); assert_eq!(all, vec![(b"k".to_vec(), 2, Cell::Tombstone)]); } -pub fn intra_batch_last_occurrence_wins(e: &dyn KVEngine) { +pub async fn intra_batch_last_occurrence_wins(e: &dyn KVEngine) { e.apply(1, &batch(vec![put(b"k", b"a"), del(b"k"), put(b"k", b"final")])) - .into_ready() + .await .unwrap(); - assert_eq!(e.get(b"k").into_ready(), Some((1, b"final".to_vec()))); + assert_eq!(e.get(b"k").await, Some((1, b"final".to_vec()))); } -pub fn scan_is_ordered_prefix_filtered_and_truncates(e: &dyn KVEngine) { +pub async fn scan_is_ordered_prefix_filtered_and_truncates(e: &dyn KVEngine) { e.apply( 1, &batch(vec![put(b"a:1", b"1"), put(b"a:2", b"2"), put(b"a:3", b"3")]), ) - .into_ready() + .await .unwrap(); - e.apply(2, &batch(vec![put(b"b:1", b"x")])).into_ready().unwrap(); - e.apply(3, &batch(vec![del(b"a:2")])).into_ready().unwrap(); + e.apply(2, &batch(vec![put(b"b:1", b"x")])).await.unwrap(); + e.apply(3, &batch(vec![del(b"a:2")])).await.unwrap(); // Unlimited: only live "a:" keys, in order, tombstone excluded. - let (items, truncated) = e.scan(b"a:", b"", b"", 0, 0, false, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"a:", b"", b"", 0, 0, false, 0).await.unwrap(); let keys: Vec<Vec<u8>> = items.iter().map(|(k, _, _)| k.to_vec()).collect(); assert_eq!(keys, vec![b"a:1".to_vec(), b"a:3".to_vec()]); assert!(!truncated); // Limit smaller than the match count sets truncated. - let (items, truncated) = e.scan(b"a:", b"", b"", 1, 0, false, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"a:", b"", b"", 1, 0, false, 0).await.unwrap(); assert_eq!(items.len(), 1); assert_eq!(items[0].0.as_ref(), b"a:1"); assert!(truncated); // Empty prefix scans everything live. - let (all, _) = e.scan(b"", b"", b"", 0, 0, false, 0).into_ready().unwrap(); + let (all, _) = e.scan(b"", b"", b"", 0, 0, false, 0).await.unwrap(); assert_eq!(all.len(), 3); // a:1, a:3, b:1 // start_after: returns keys strictly greater than start_after. - let (page, truncated) = e.scan(b"a:", b"a:1", b"", 0, 0, false, 0).into_ready().unwrap(); + let (page, truncated) = e.scan(b"a:", b"a:1", b"", 0, 0, false, 0).await.unwrap(); let keys: Vec<Vec<u8>> = page.iter().map(|(k, _, _)| k.to_vec()).collect(); assert_eq!(keys, vec![b"a:3".to_vec()]); assert!(!truncated); // start_after with limit: pagination returns next page. - let (page1, trunc1) = e.scan(b"a:", b"", b"", 1, 0, false, 0).into_ready().unwrap(); + let (page1, trunc1) = e.scan(b"a:", b"", b"", 1, 0, false, 0).await.unwrap(); assert_eq!(page1.len(), 1); assert_eq!(page1[0].0.as_ref(), b"a:1"); assert!(trunc1); - let (page2, trunc2) = e.scan(b"a:", b"a:1", b"", 1, 0, false, 0).into_ready().unwrap(); + let (page2, trunc2) = e.scan(b"a:", b"a:1", b"", 1, 0, false, 0).await.unwrap(); assert_eq!(page2.len(), 1); assert_eq!(page2[0].0.as_ref(), b"a:3"); assert!(!trunc2); } -pub fn reverse_scan_is_descending_exclusive_and_prefix_bounded(e: &dyn KVEngine) { +pub async fn reverse_scan_is_descending_exclusive_and_prefix_bounded(e: &dyn KVEngine) { e.apply( 1, &batch(vec![ @@ -137,20 +144,20 @@ pub fn reverse_scan_is_descending_exclusive_and_prefix_bounded(e: &dyn KVEngine) put(b"q:1", b"outside"), ]), ) - .into_ready() + .await .unwrap(); - e.apply(2, &batch(vec![del(b"p:2")])).into_ready().unwrap(); + e.apply(2, &batch(vec![del(b"p:2")])).await.unwrap(); let (items, truncated) = e .scan_directional(b"p:", b"p:4", b"q:", 1, 0, false, 0, ScanDirection::Reverse) - .into_ready() + .await .unwrap(); assert!(truncated); assert_eq!(items[0].0.as_ref(), b"p:3"); let (items, truncated) = e .scan_directional(b"p:", b"p:3", b"q:", 0, 0, true, 0, ScanDirection::Reverse) - .into_ready() + .await .unwrap(); assert!(!truncated); assert_eq!(items.len(), 1); @@ -159,18 +166,21 @@ pub fn reverse_scan_is_descending_exclusive_and_prefix_bounded(e: &dyn KVEngine) let (items, truncated) = e .scan_directional(b"p:", b"", b"q:", 0, 6, false, 0, ScanDirection::Reverse) - .into_ready() + .await .unwrap(); assert_eq!(items.len(), 1, "one oversized item must still make progress"); assert_eq!(items[0].0.as_ref(), b"p:3"); assert!(truncated); } -pub fn scan_reports_each_live_records_commit_slot(e: &dyn KVEngine) { - e.apply(3, &batch(vec![put(b"k:a", b"a")])).into_ready().unwrap(); - e.apply(7, &batch(vec![put(b"k:b", b"b")])).into_ready().unwrap(); +pub async fn scan_reports_each_live_records_commit_slot(e: &dyn KVEngine) { + for slot in [1, 2, 4, 5, 6] { + e.noop(slot); + } + e.apply(3, &batch(vec![put(b"k:a", b"a")])).await.unwrap(); + e.apply(7, &batch(vec![put(b"k:b", b"b")])).await.unwrap(); - let (items, truncated) = e.scan(b"k:", b"", b"", 0, 0, false, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"k:", b"", b"", 0, 0, false, 0).await.unwrap(); assert!(!truncated); assert_eq!(items.len(), 2); @@ -183,40 +193,40 @@ pub fn scan_reports_each_live_records_commit_slot(e: &dyn KVEngine) { /// `end_key` is an exclusive upper bound: only keys strictly less than /// `end_key` are returned. Empty `end_key` = unbounded (today's behavior). /// `prefix` + `end_key` together intersect correctly. -pub fn scan_end_key_exclusive_upper_bound(e: &dyn KVEngine) { +pub async fn scan_end_key_exclusive_upper_bound(e: &dyn KVEngine) { // Keys: k00, k01, k02, k03, k04 let mut ops = Vec::new(); for i in 0..5 { ops.push(put(format!("k0{i}").as_bytes(), format!("v{i}").as_bytes())); } - e.apply(1, &batch(ops)).into_ready().unwrap(); + e.apply(1, &batch(ops)).await.unwrap(); // end_key="k03": returns k00, k01, k02 (strictly less than k03). - let (items, truncated) = e.scan(b"", b"", b"k03", 0, 0, false, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"", b"", b"k03", 0, 0, false, 0).await.unwrap(); let keys: Vec<Vec<u8>> = items.iter().map(|(k, _, _)| k.to_vec()).collect(); assert_eq!(keys, vec![b"k00".to_vec(), b"k01".to_vec(), b"k02".to_vec()]); assert!(!truncated); // start_after + end_key: (k01, k04) → k02, k03. - let (items, _) = e.scan(b"", b"k01", b"k04", 0, 0, false, 0).into_ready().unwrap(); + let (items, _) = e.scan(b"", b"k01", b"k04", 0, 0, false, 0).await.unwrap(); let keys: Vec<Vec<u8>> = items.iter().map(|(k, _, _)| k.to_vec()).collect(); assert_eq!(keys, vec![b"k02".to_vec(), b"k03".to_vec()]); // prefix + end_key: prefix="k0", end_key="k02" → k00, k01. - let (items, _) = e.scan(b"k0", b"", b"k02", 0, 0, false, 0).into_ready().unwrap(); + let (items, _) = e.scan(b"k0", b"", b"k02", 0, 0, false, 0).await.unwrap(); let keys: Vec<Vec<u8>> = items.iter().map(|(k, _, _)| k.to_vec()).collect(); assert_eq!(keys, vec![b"k00".to_vec(), b"k01".to_vec()]); // Empty end_key = unbounded (all 5 keys). - let (items, _) = e.scan(b"", b"", b"", 0, 0, false, 0).into_ready().unwrap(); + let (items, _) = e.scan(b"", b"", b"", 0, 0, false, 0).await.unwrap(); assert_eq!(items.len(), 5); // end_key at the very start: no keys are < "k00". - let (items, _) = e.scan(b"", b"", b"k00", 0, 0, false, 0).into_ready().unwrap(); + let (items, _) = e.scan(b"", b"", b"k00", 0, 0, false, 0).await.unwrap(); assert!(items.is_empty()); // end_key with limit: limit=1, end_key="k04" → k00, truncated. - let (items, truncated) = e.scan(b"", b"", b"k04", 1, 0, false, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"", b"", b"k04", 1, 0, false, 0).await.unwrap(); assert_eq!(items.len(), 1); assert_eq!(items[0].0.as_ref(), b"k00"); assert!(truncated); @@ -225,7 +235,7 @@ pub fn scan_end_key_exclusive_upper_bound(e: &dyn KVEngine) { /// `byte_budget` caps the total key+value bytes in a scan response. The scan /// stops with `truncated = true` when the budget is exceeded, always returning /// at least one entry (so a single oversized entry still makes progress). -pub fn scan_byte_budget_stops_and_truncates(e: &dyn KVEngine) { +pub async fn scan_byte_budget_stops_and_truncates(e: &dyn KVEngine) { // 5 entries: key=3B ("k00".."k04"), value=10B ("vvvvvvvvv0".."vvvvvvvvv4") // = 13B per entry. Budget=30B allows 2 entries (26B) before the 3rd // (39B) exceeds the budget. @@ -236,23 +246,23 @@ pub fn scan_byte_budget_stops_and_truncates(e: &dyn KVEngine) { format!("vvvvvvvvv{i}").as_bytes(), )); } - e.apply(1, &batch(ops)).into_ready().unwrap(); + e.apply(1, &batch(ops)).await.unwrap(); // Budget=30: 2 entries (26B) fit, 3rd would be 39B > 30. - let (items, truncated) = e.scan(b"", b"", b"", 0, 30, false, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"", b"", b"", 0, 30, false, 0).await.unwrap(); assert_eq!(items.len(), 2); assert!(truncated); assert_eq!(items[0].0.as_ref(), b"k00"); assert_eq!(items[1].0.as_ref(), b"k01"); // Budget=0 (unlimited): all 5 entries, no truncation. - let (items, truncated) = e.scan(b"", b"", b"", 0, 0, false, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"", b"", b"", 0, 0, false, 0).await.unwrap(); assert_eq!(items.len(), 5); assert!(!truncated); // Budget smaller than a single entry: always return >= 1 entry. // Entry "k00" = 3 + 10 = 13B; budget=5 < 13. - let (items, truncated) = e.scan(b"", b"", b"", 0, 5, false, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"", b"", b"", 0, 5, false, 0).await.unwrap(); assert_eq!(items.len(), 1); assert!(truncated); // more entries remain assert_eq!(items[0].0.as_ref(), b"k00"); @@ -262,7 +272,7 @@ pub fn scan_byte_budget_stops_and_truncates(e: &dyn KVEngine) { /// returned, in order, but every value is empty. The byte budget then accounts /// for key bytes only, so a `keys_only` page fits more entries than a full /// scan with the same budget. -pub fn scan_keys_only_skips_values(e: &dyn KVEngine) { +pub async fn scan_keys_only_skips_values(e: &dyn KVEngine) { // 5 entries: key=3B ("k00".."k04"), value=10B ("vvvvvvvvv0".."vvvvvvvvv4"). let mut ops = Vec::new(); for i in 0..5 { @@ -271,11 +281,11 @@ pub fn scan_keys_only_skips_values(e: &dyn KVEngine) { format!("vvvvvvvvv{i}").as_bytes(), )); } - e.apply(1, &batch(ops)).into_ready().unwrap(); + e.apply(1, &batch(ops)).await.unwrap(); // keys_only: same keys as a full scan, all values empty. - let (keys_items, keys_trunc) = e.scan(b"", b"", b"", 0, 0, true, 0).into_ready().unwrap(); - let (full_items, full_trunc) = e.scan(b"", b"", b"", 0, 0, false, 0).into_ready().unwrap(); + let (keys_items, keys_trunc) = e.scan(b"", b"", b"", 0, 0, true, 0).await.unwrap(); + let (full_items, full_trunc) = e.scan(b"", b"", b"", 0, 0, false, 0).await.unwrap(); let keys: Vec<Vec<u8>> = keys_items.iter().map(|(k, _, _)| k.to_vec()).collect(); let full_keys: Vec<Vec<u8>> = full_items.iter().map(|(k, _, _)| k.to_vec()).collect(); assert_eq!( @@ -300,12 +310,12 @@ pub fn scan_keys_only_skips_values(e: &dyn KVEngine) { ); // keys_only with prefix: only matching keys, values empty. - let (items, _) = e.scan(b"k0", b"", b"", 0, 0, true, 0).into_ready().unwrap(); + let (items, _) = e.scan(b"k0", b"", b"", 0, 0, true, 0).await.unwrap(); assert_eq!(items.len(), 5); assert!(items.iter().all(|(_, _, v)| v.is_empty())); // keys_only with start_after: pagination cursor works. - let (items, trunc) = e.scan(b"", b"k01", b"", 0, 0, true, 0).into_ready().unwrap(); + let (items, trunc) = e.scan(b"", b"k01", b"", 0, 0, true, 0).await.unwrap(); let keys: Vec<Vec<u8>> = items.iter().map(|(k, _, _)| k.to_vec()).collect(); assert_eq!(keys, vec![b"k02".to_vec(), b"k03".to_vec(), b"k04".to_vec()]); assert!(!trunc); @@ -313,7 +323,7 @@ pub fn scan_keys_only_skips_values(e: &dyn KVEngine) { // keys_only with byte_budget: accounts for key bytes only (3B per key), // so budget=7 fits 2 keys (6B) before the 3rd (9B) exceeds — vs a full // scan where 13B per entry would fit only 1 entry in 7B. - let (items, truncated) = e.scan(b"", b"", b"", 0, 7, true, 0).into_ready().unwrap(); + let (items, truncated) = e.scan(b"", b"", b"", 0, 7, true, 0).await.unwrap(); assert_eq!(items.len(), 2); assert!(truncated); assert_eq!(items[0].0.as_ref(), b"k00"); @@ -321,18 +331,18 @@ pub fn scan_keys_only_skips_values(e: &dyn KVEngine) { assert!(items.iter().all(|(_, _, v)| v.is_empty())); } -pub fn scan_deadline_returns_partial_result(e: &dyn KVEngine) { +pub async fn scan_deadline_returns_partial_result(e: &dyn KVEngine) { // Populate enough keys that the merge loop's periodic deadline check // (every 1024 entries) fires mid-scan. for i in 0..3000u64 { let key = format!("k{i:05}"); e.apply(i + 1, &batch(vec![put(key.as_bytes(), b"v")])) - .into_ready() + .await .unwrap(); } // deadline_ms = 0 (no deadline): full scan returns all 3000 entries. - let (all, all_trunc) = e.scan(b"", b"", b"", 0, 0, false, 0).into_ready().unwrap(); + let (all, all_trunc) = e.scan(b"", b"", b"", 0, 0, false, 0).await.unwrap(); assert_eq!(all.len(), 3000); assert!(!all_trunc); @@ -340,7 +350,7 @@ pub fn scan_deadline_returns_partial_result(e: &dyn KVEngine) { // (every 1024 entries) and breaks early with truncated = true. The // result is a partial, correctly-ordered prefix (no gaps). let deadline = now_ms(); - let (partial, partial_trunc) = e.scan(b"", b"", b"", 0, 0, false, deadline).into_ready().unwrap(); + let (partial, partial_trunc) = e.scan(b"", b"", b"", 0, 0, false, deadline).await.unwrap(); assert!(partial_trunc); assert!(partial.len() < all.len()); // The partial result is a correctly-ordered prefix of the full scan. @@ -349,12 +359,15 @@ pub fn scan_deadline_returns_partial_result(e: &dyn KVEngine) { } } -pub fn compare_is_empty_for_identical_state_and_detects_divergence(a: &dyn KVEngine, b: &dyn KVEngine) { +pub async fn compare_is_empty_for_identical_state_and_detects_divergence(a: &dyn KVEngine, b: &dyn KVEngine) { // Same ops, different apply order — must converge to identical state. - a.apply(2, &batch(vec![put(b"x", b"2")])).into_ready().unwrap(); - a.apply(1, &batch(vec![put(b"y", b"1")])).into_ready().unwrap(); - b.apply(1, &batch(vec![put(b"y", b"1")])).into_ready().unwrap(); - b.apply(2, &batch(vec![put(b"x", b"2")])).into_ready().unwrap(); + let second = batch(vec![put(b"x", b"2")]); + let first = batch(vec![put(b"y", b"1")]); + let (second_result, first_result) = tokio::join!(biased; a.apply(2, &second), a.apply(1, &first)); + second_result.unwrap(); + first_result.unwrap(); + b.apply(1, &batch(vec![put(b"y", b"1")])).await.unwrap(); + b.apply(2, &batch(vec![put(b"x", b"2")])).await.unwrap(); assert!(compare_dyn(a, b).is_empty(), "identical logical state"); // Divergent resolved-slot for the same value is a difference. Slot 3 @@ -364,7 +377,7 @@ pub fn compare_is_empty_for_identical_state_and_detects_divergence(a: &dyn KVEng // a gap wouldn't be visible yet even though `get`/`scan` would already // see it (an engine-specific difference from `InMemKV`, not something // this test is trying to exercise). - b.apply(3, &batch(vec![put(b"x", b"2")])).into_ready().unwrap(); + b.apply(3, &batch(vec![put(b"x", b"2")])).await.unwrap(); let diff = compare_dyn(a, b); assert_eq!(diff.len(), 1); assert_eq!(diff[0].key, b"x".to_vec()); @@ -376,16 +389,13 @@ pub fn compare_is_empty_for_identical_state_and_detects_divergence(a: &dyn KVEng /// (`compare` empty) and report the same `at_slot`. `target` must be freshly /// constructed and never previously `apply`ed to (`snapshot_import`'s /// documented precondition). -pub fn snapshot_export_import_round_trip(source: &dyn KVEngine, target: &dyn KVEngine) { +pub async fn snapshot_export_import_round_trip(source: &dyn KVEngine, target: &dyn KVEngine) { source .apply(1, &batch(vec![put(b"a", b"1"), put(b"b", b"2")])) - .into_ready() - .unwrap(); - source.apply(2, &batch(vec![del(b"a")])).into_ready().unwrap(); - source - .apply(3, &batch(vec![put(b"c", b"3")])) - .into_ready() + .await .unwrap(); + source.apply(2, &batch(vec![del(b"a")])).await.unwrap(); + source.apply(3, &batch(vec![put(b"c", b"3")])).await.unwrap(); let mut exporter = source .snapshot_export_begin(17) @@ -421,7 +431,7 @@ pub fn snapshot_export_import_round_trip(source: &dyn KVEngine, target: &dyn KVE compare_dyn(target, source).is_empty(), "imported state must be logically identical to the exporter's" ); - assert_eq!(target.get(b"a").into_ready(), None, "tombstoned key stays absent"); - assert_eq!(target.get(b"b").into_ready(), Some((1, b"2".to_vec()))); - assert_eq!(target.get(b"c").into_ready(), Some((3, b"3".to_vec()))); + assert_eq!(target.get(b"a").await, None, "tombstoned key stays absent"); + assert_eq!(target.get(b"b").await, Some((1, b"2".to_vec()))); + assert_eq!(target.get(b"c").await, Some((3, b"3".to_vec()))); } diff --git a/lib/crowdb-kv/tests/kv_test/crowdb_tree_engine_test.rs b/lib/crowdb-paxos-kv/tests/kv_test/crowdb_tree_engine_test.rs similarity index 72% rename from lib/crowdb-kv/tests/kv_test/crowdb_tree_engine_test.rs rename to lib/crowdb-paxos-kv/tests/kv_test/crowdb_tree_engine_test.rs index 3117111c1..af7f5678d 100644 --- a/lib/crowdb-kv/tests/kv_test/crowdb_tree_engine_test.rs +++ b/lib/crowdb-paxos-kv/tests/kv_test/crowdb_tree_engine_test.rs @@ -6,7 +6,7 @@ //! `InMemKV` satisfy the identical `KVEngine` contract. use crate::test_util::{compare_dyn, iter_all_dyn}; -use crowdb_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; use super::conformance; @@ -14,20 +14,33 @@ fn open() -> CrowdbTreeEngine { CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).unwrap() } -#[test] -fn highest_slot_wins_regardless_of_apply_order() { - conformance::highest_slot_wins_regardless_of_apply_order(&open()); +fn durable_config(path: &std::path::Path) -> CrowdbTreeConfig { + // Independent durable fixtures must not share the process host's finite + // checkpoint quotas. Reopens within a fixture retain the same host. + CrowdbTreeConfig { + path: Some(path.to_string_lossy().into_owned()), + runtime: Some(std::sync::Arc::new( + crowdb_tree_ffi::TreeRuntime::new(crowdb_tree_ffi::TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + ..Default::default() + } +} + +#[tokio::test] +async fn highest_slot_wins_regardless_of_apply_order() { + conformance::highest_slot_wins_regardless_of_apply_order(&open()).await; } #[tokio::test] async fn out_of_order_noop_does_not_discard_a_delayed_write() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("crowdb-tree-noop-gap"); - let config = CrowdbTreeConfig { - path: Some(tmp.path().to_string_lossy().into_owned()), - ..Default::default() - }; + let config = durable_config(tmp.path()); let engine = CrowdbTreeEngine::open(&config).unwrap(); engine .apply(1, &conformance::batch(vec![conformance::put(b"key", b"old")])) @@ -36,15 +49,17 @@ async fn out_of_order_noop_does_not_discard_a_delayed_write() { engine.flush(); engine.noop(3); engine.flush(); - assert_eq!(engine.resume_from_slot(), 1); + assert_eq!(engine.resume_from_slot(), 0); assert_eq!(engine.persist_snapshot(), 1); + assert_eq!(engine.resume_from_slot(), 1); engine .apply(2, &conformance::batch(vec![conformance::put(b"key", b"new")])) .await .unwrap(); engine.flush(); - assert_eq!(engine.resume_from_slot(), 3); + assert_eq!(engine.resume_from_slot(), 1); assert_eq!(engine.persist_snapshot(), 3); + assert_eq!(engine.resume_from_slot(), 3); drop(engine); let engine = CrowdbTreeEngine::open(&config).unwrap(); assert_eq!( @@ -53,24 +68,21 @@ async fn out_of_order_noop_does_not_discard_a_delayed_write() { ); } -#[test] -fn equal_slot_is_idempotent_noop() { - conformance::equal_slot_is_idempotent_noop(&open()); +#[tokio::test] +async fn equal_slot_is_idempotent_noop() { + conformance::equal_slot_is_idempotent_noop(&open()).await; } #[tokio::test] async fn versioned_lookup_reads_cold_tree_and_respects_newer_tombstone() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("crowdb-tree-cas-lookup"); - let opt = CrowdbTreeConfig { - path: Some(tmp.path().to_string_lossy().into_owned()), - ..Default::default() - }; + let opt = durable_config(tmp.path()); let engine = CrowdbTreeEngine::open(&opt).expect("open durable engine"); engine .apply(1, &conformance::batch(vec![conformance::put(b"key", b"tree")])) - .into_ready() + .await .unwrap(); engine.flush(); assert_eq!(engine.persist_snapshot(), 1); @@ -83,64 +95,64 @@ async fn versioned_lookup_reads_cold_tree_and_respects_newer_tombstone() { ); reopened .apply(2, &conformance::batch(vec![conformance::del(b"key")])) - .into_ready() + .await .unwrap(); assert_eq!(reopened.get_versioned(b"key").await.unwrap(), None); } -#[test] -fn delete_writes_tombstone() { - conformance::delete_writes_tombstone(&open()); +#[tokio::test] +async fn delete_writes_tombstone() { + conformance::delete_writes_tombstone(&open()).await; } -#[test] -fn intra_batch_last_occurrence_wins() { - conformance::intra_batch_last_occurrence_wins(&open()); +#[tokio::test] +async fn intra_batch_last_occurrence_wins() { + conformance::intra_batch_last_occurrence_wins(&open()).await; } -#[test] -fn scan_is_ordered_prefix_filtered_and_truncates() { - conformance::scan_is_ordered_prefix_filtered_and_truncates(&open()); +#[tokio::test] +async fn scan_is_ordered_prefix_filtered_and_truncates() { + conformance::scan_is_ordered_prefix_filtered_and_truncates(&open()).await; } -#[test] -fn reverse_scan_is_descending_exclusive_and_prefix_bounded() { - conformance::reverse_scan_is_descending_exclusive_and_prefix_bounded(&open()); +#[tokio::test] +async fn reverse_scan_is_descending_exclusive_and_prefix_bounded() { + conformance::reverse_scan_is_descending_exclusive_and_prefix_bounded(&open()).await; } -#[test] -fn scan_reports_each_live_records_commit_slot() { - conformance::scan_reports_each_live_records_commit_slot(&open()); +#[tokio::test] +async fn scan_reports_each_live_records_commit_slot() { + conformance::scan_reports_each_live_records_commit_slot(&open()).await; } -#[test] -fn scan_end_key_exclusive_upper_bound() { - conformance::scan_end_key_exclusive_upper_bound(&open()); +#[tokio::test] +async fn scan_end_key_exclusive_upper_bound() { + conformance::scan_end_key_exclusive_upper_bound(&open()).await; } -#[test] -fn scan_byte_budget_stops_and_truncates() { - conformance::scan_byte_budget_stops_and_truncates(&open()); +#[tokio::test] +async fn scan_byte_budget_stops_and_truncates() { + conformance::scan_byte_budget_stops_and_truncates(&open()).await; } -#[test] -fn scan_keys_only_skips_values() { - conformance::scan_keys_only_skips_values(&open()); +#[tokio::test] +async fn scan_keys_only_skips_values() { + conformance::scan_keys_only_skips_values(&open()).await; } -#[test] -fn scan_deadline_returns_partial_result() { - conformance::scan_deadline_returns_partial_result(&open()); +#[tokio::test] +async fn scan_deadline_returns_partial_result() { + conformance::scan_deadline_returns_partial_result(&open()).await; } -#[test] -fn compare_is_empty_for_identical_state_and_detects_divergence() { - conformance::compare_is_empty_for_identical_state_and_detects_divergence(&open(), &open()); +#[tokio::test] +async fn compare_is_empty_for_identical_state_and_detects_divergence() { + conformance::compare_is_empty_for_identical_state_and_detects_divergence(&open(), &open()).await; } -#[test] -fn snapshot_export_import_round_trip() { - conformance::snapshot_export_import_round_trip(&open(), &open()); +#[tokio::test] +async fn snapshot_export_import_round_trip() { + conformance::snapshot_export_import_round_trip(&open(), &open()).await; } /// No FFI fault-injection hook exists yet to genuinely trip @@ -151,31 +163,33 @@ fn snapshot_export_import_round_trip() { /// the trait default is. #[test] fn is_healthy_is_true_on_a_freshly_opened_engine() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; assert!(open().is_healthy()); } -/// Regression guard: an in-memory -/// `CrowdbTreeEngine` (`opt.path: None`, no page store, no reactor -- see -/// `CrowdbTreeConfig::default`) has no I/O path *at all*, so `get`/`scan`/ -/// `apply` must always resolve `Ready` -- proves the "fast path stays fast" -/// property holds for the durable engine's in-memory mode too, not just -/// `InMemKV`. -#[test] -fn get_scan_apply_always_resolve_ready() { - use crowdb_kv::kv::{KVEngine, KVFuture}; +/// Default in-memory engines use real runtime admission and asynchronous +/// completion even when the operation has no backend I/O. +#[tokio::test] +async fn default_engine_awaits_apply_get_and_scan_on_its_runtime() { + use crowdb_paxos_kv::kv::KVEngine; let e = open(); - assert!(matches!( - e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v")])), - KVFuture::Ready(_) - )); - assert!(matches!(e.get(b"k"), KVFuture::Ready(_))); - assert!(matches!( - e.scan(b"", b"", b"", 0, 0, false, 0), - KVFuture::Ready(_) - )); + assert!(e.requires_contiguous_apply()); + e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v")])) + .await + .unwrap(); + assert_eq!(e.get(b"k").await, Some((1, b"v".to_vec()))); + let (entries, truncated) = e.scan(b"", b"", b"", 0, 0, false, 0).await.unwrap(); + assert_eq!( + entries, + vec![( + bytes::Bytes::from_static(b"k"), + 1, + bytes::Bytes::from_static(b"v") + )] + ); + assert!(!truncated); } /// A durable engine reloads an evicted leaf and returns the correct value. @@ -183,20 +197,18 @@ fn get_scan_apply_always_resolve_ready() { /// `Pending` is valid after the miss has submitted asynchronous I/O. #[tokio::test] async fn get_reloads_an_evicted_leaf() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("crowdb-tree-engine"); - let e = CrowdbTreeEngine::open(&CrowdbTreeConfig { - path: Some(tmp.path().to_string_lossy().into_owned()), - ..Default::default() - }) - .expect("open durable engine"); + let e = CrowdbTreeEngine::open(&durable_config(tmp.path())).expect("open durable engine"); e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v")])) - .into_ready() + .await .unwrap(); e.handle().flush().expect("flush"); e.handle().snapshot().expect("snapshot"); + // Checkpoint installs reloadable bases without keeping encoded frames resident. + assert_eq!(e.get(b"k").await, Some((1, b"v".to_vec()))); let evicted = e.handle().evict_clean_leaves(0); assert!( evicted > 0, @@ -210,20 +222,18 @@ async fn get_reloads_an_evicted_leaf() { /// Same reload guard as [`get_reloads_an_evicted_leaf`], for `scan`. #[tokio::test] async fn scan_reloads_an_evicted_leaf() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("crowdb-tree-engine"); - let e = CrowdbTreeEngine::open(&CrowdbTreeConfig { - path: Some(tmp.path().to_string_lossy().into_owned()), - ..Default::default() - }) - .expect("open durable engine"); + let e = CrowdbTreeEngine::open(&durable_config(tmp.path())).expect("open durable engine"); e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v")])) - .into_ready() + .await .unwrap(); e.handle().flush().expect("flush"); e.handle().snapshot().expect("snapshot"); + // Populate the read cache before verifying eviction followed by a cold scan. + assert_eq!(e.get(b"k").await, Some((1, b"v".to_vec()))); let evicted = e.handle().evict_clean_leaves(0); assert!( evicted > 0, @@ -244,16 +254,16 @@ async fn scan_reloads_an_evicted_leaf() { /// `clear_drops_all_state` for the crowdb-tree-backed engine now that /// `CrowdbTreeEngine::clear` is wired to `Crowdbtree::clear` instead of /// panicking. -#[test] -fn clear_drops_all_state() { - use crowdb_kv::kv::KVEngine; +#[tokio::test] +async fn clear_drops_all_state() { + use crowdb_paxos_kv::kv::KVEngine; let e = open(); e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v")])) - .into_ready() + .await .unwrap(); - e.clear(); - assert_eq!(e.get(b"k").into_ready(), None); + e.clear().unwrap(); + assert_eq!(e.get(b"k").await, None); assert!(iter_all_dyn(&e).is_empty()); } @@ -263,19 +273,19 @@ fn clear_drops_all_state() { /// freshly-reset replica about to re-learn its whole log from scratch) /// would be silently ignored by `apply`'s idempotency check, even though /// this is logically a brand-new tree. -#[test] -fn apply_after_clear_accepts_the_same_slot_number_again() { - use crowdb_kv::kv::KVEngine; +#[tokio::test] +async fn apply_after_clear_accepts_the_same_slot_number_again() { + use crowdb_paxos_kv::kv::KVEngine; let e = open(); e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v1")])) - .into_ready() + .await .unwrap(); - e.clear(); + e.clear().unwrap(); e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v2")])) - .into_ready() + .await .unwrap(); - assert_eq!(e.get(b"k").into_ready(), Some((1, b"v2".to_vec()))); + assert_eq!(e.get(b"k").await, Some((1, b"v2".to_vec()))); } /// Crash-safety-adjacent check for a *durable* (file-backed) engine: @@ -286,21 +296,18 @@ fn apply_after_clear_accepts_the_same_slot_number_again() { /// just the in-memory tree. #[tokio::test] async fn clear_then_persist_survives_reopen() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("crowdb-tree-engine"); - let opt = CrowdbTreeConfig { - path: Some(tmp.path().to_string_lossy().into_owned()), - ..Default::default() - }; + let opt = durable_config(tmp.path()); let e = CrowdbTreeEngine::open(&opt).expect("open durable engine"); e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v")])) - .into_ready() + .await .unwrap(); e.flush(); assert_eq!(e.persist_snapshot(), 1, "sanity: pre-clear state persisted"); - e.clear(); + e.clear().unwrap(); e.flush(); assert_eq!( e.persist_snapshot(), @@ -310,12 +317,7 @@ async fn clear_then_persist_survives_reopen() { drop(e); let reopened = CrowdbTreeEngine::open(&opt).expect("reopen durable engine"); - // `.await`, not `.into_ready`: a just-reopened durable tree's root is - // installed as an *unloaded* descriptor (lazy recovery -- see - // `Crowdbtree::open`'s doc comment in persist.cpp), so this first read - // may genuinely demand-load it, constructing `KVFuture::Pending` -- - // exactly like `get_constructs_pending_for_genuine_demand_load_miss`, - // just incidentally rather than via a forced eviction. + // Recovery installs a cold root; await its read before checking the wipe. assert_eq!( reopened.get(b"k").await, None, @@ -326,10 +328,10 @@ async fn clear_then_persist_survives_reopen() { /// Cross-engine parity (design's strongest correctness gate): apply the same /// op stream to `InMemKV` and `CrowdbTreeEngine`, `compare` must be empty. -#[test] -fn parity_with_in_mem_kv_after_identical_op_stream() { +#[tokio::test] +async fn parity_with_in_mem_kv_after_identical_op_stream() { use crate::mem_kv::InMemKV; - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let mem = InMemKV::new(); let ct = open(); @@ -344,7 +346,7 @@ fn parity_with_in_mem_kv_after_identical_op_stream() { conformance::batch(vec![conformance::put(&key, &value)]) }; mem.apply(slot, &b).into_ready().unwrap(); - ct.apply(slot, &b).into_ready().unwrap(); + ct.apply(slot, &b).await.unwrap(); assert!(compare_dyn(&mem, &ct).is_empty(), "diverged after round {round}"); } } @@ -355,11 +357,11 @@ fn parity_with_in_mem_kv_after_identical_op_stream() { /// for both found and not-found cases. #[tokio::test] async fn get_bytes_fast_path_returns_correct_value() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let e = open(); e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"hello")])) - .into_ready() + .await .unwrap(); let result = e.get_bytes(b"k").await; @@ -371,19 +373,19 @@ async fn get_bytes_fast_path_returns_correct_value() { /// `snapshot_view` returns a frozen, key-sorted vector including /// tombstones, pinned at `last_applied_slot` after a flush. -#[test] -fn snapshot_view_returns_pinned_entries() { - use crowdb_kv::kv::KVEngine; +#[tokio::test] +async fn snapshot_view_returns_pinned_entries() { + use crowdb_paxos_kv::kv::KVEngine; let e = open(); e.apply(1, &conformance::batch(vec![conformance::put(b"a", b"v1")])) - .into_ready() + .await .unwrap(); e.apply(2, &conformance::batch(vec![conformance::put(b"b", b"v2")])) - .into_ready() + .await .unwrap(); e.apply(3, &conformance::batch(vec![conformance::del(b"a")])) - .into_ready() + .await .unwrap(); let (at_slot, entries) = e.snapshot_view().expect("snapshot_view"); @@ -399,13 +401,13 @@ fn snapshot_view_returns_pinned_entries() { /// `snapshot_view` is point-in-time-consistent: writes applied after /// the snapshot do not appear in the frozen view. -#[test] -fn snapshot_view_is_point_in_time_consistent() { - use crowdb_kv::kv::KVEngine; +#[tokio::test] +async fn snapshot_view_is_point_in_time_consistent() { + use crowdb_paxos_kv::kv::KVEngine; let e = open(); e.apply(1, &conformance::batch(vec![conformance::put(b"k", b"v1")])) - .into_ready() + .await .unwrap(); let (at_slot, entries) = e.snapshot_view().expect("snapshot_view"); @@ -416,10 +418,10 @@ fn snapshot_view_is_point_in_time_consistent() { // Apply more writes after the snapshot. e.apply(2, &conformance::batch(vec![conformance::put(b"k", b"v2")])) - .into_ready() + .await .unwrap(); e.apply(3, &conformance::batch(vec![conformance::put(b"new", b"v3")])) - .into_ready() + .await .unwrap(); // The frozen view is unchanged. @@ -439,12 +441,12 @@ fn snapshot_view_is_point_in_time_consistent() { /// correctly handles values that span the full frame. #[tokio::test] async fn get_bytes_fast_path_large_value() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let e = open(); let large: Vec<u8> = (0..4096u32).map(|i| u8::try_from(i % 256).unwrap()).collect(); e.apply(1, &conformance::batch(vec![conformance::put(b"k", &large)])) - .into_ready() + .await .unwrap(); let result = e.get_bytes(b"k").await; @@ -462,7 +464,7 @@ async fn get_bytes_fast_path_large_value() { /// stall at slot 1 and `last_applied_slot_` would never advance past 1. #[tokio::test] async fn noop_slot_does_not_block_contiguous_slot_advancement() { - use crowdb_kv::kv::KVEngine; + use crowdb_paxos_kv::kv::KVEngine; let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("crowdb-tree-engine"); let opt = CrowdbTreeConfig { @@ -473,13 +475,13 @@ async fn noop_slot_does_not_block_contiguous_slot_advancement() { // Slot 1: real write. e.apply(1, &conformance::batch(vec![conformance::put(b"k1", b"v1")])) - .into_ready() + .await .unwrap(); // Slot 2: NoOp (empty batch — simulates repair_once gap-fill). e.noop(2); // Slot 3: real write. e.apply(3, &conformance::batch(vec![conformance::put(b"k2", b"v2")])) - .into_ready() + .await .unwrap(); // Flush + snapshot to make the state durable. diff --git a/lib/crowdb-kv/tests/kv_test/kv_future_test.rs b/lib/crowdb-paxos-kv/tests/kv_test/kv_future_test.rs similarity index 97% rename from lib/crowdb-kv/tests/kv_test/kv_future_test.rs rename to lib/crowdb-paxos-kv/tests/kv_test/kv_future_test.rs index a8edfe65b..40d30febf 100644 --- a/lib/crowdb-kv/tests/kv_test/kv_future_test.rs +++ b/lib/crowdb-paxos-kv/tests/kv_test/kv_future_test.rs @@ -6,7 +6,7 @@ //! that a genuine `Pending` future (not constructed by any `KVEngine` impl //! yet, but exercised here directly) still polls/awaits through correctly. -use crowdb_kv::kv::KVFuture; +use crowdb_paxos_kv::kv::KVFuture; #[test] fn ready_holds_the_value_without_ever_being_pending() { diff --git a/lib/crowdb-kv/tests/kv_test/mem_kv_impl_test.rs b/lib/crowdb-paxos-kv/tests/kv_test/mem_kv_impl_test.rs similarity index 98% rename from lib/crowdb-kv/tests/kv_test/mem_kv_impl_test.rs rename to lib/crowdb-paxos-kv/tests/kv_test/mem_kv_impl_test.rs index fdabf4f8b..15513b63c 100644 --- a/lib/crowdb-kv/tests/kv_test/mem_kv_impl_test.rs +++ b/lib/crowdb-paxos-kv/tests/kv_test/mem_kv_impl_test.rs @@ -9,7 +9,7 @@ use dashmap::DashMap; use std::collections::BTreeMap; use std::sync::Arc; -use crowdb_kv::kv::{ +use crowdb_paxos_kv::kv::{ Batch, BatchOp, Cell, KVEngine, KVFuture, Op, ScanDirection, SnapshotChunk, SnapshotExporter, SnapshotFormat, SnapshotImporter, SnapshotMetadata, }; @@ -179,8 +179,9 @@ impl KVEngine for InMemKV { KVFuture::ready(Ok((items, truncated))) } - fn clear(&self) { + fn clear(&self) -> Result<(), String> { self.map.clear(); + Ok(()) } fn snapshot_export_begin(&self, chunk_bytes: usize) -> Result<Box<dyn SnapshotExporter>, String> { @@ -269,7 +270,7 @@ impl SnapshotExporter for InMemSnapshotExporter { let end = self.bytes.len().min(self.offset + self.metadata.chunk_bytes); let chunk = SnapshotChunk { offset, - bytes: self.bytes[self.offset..end].to_vec(), + bytes: self.bytes[self.offset..end].to_vec().into(), done: end == self.bytes.len(), }; self.offset = end; diff --git a/lib/crowdb-kv/tests/kv_test/mem_kv_test.rs b/lib/crowdb-paxos-kv/tests/kv_test/mem_kv_test.rs similarity index 72% rename from lib/crowdb-kv/tests/kv_test/mem_kv_test.rs rename to lib/crowdb-paxos-kv/tests/kv_test/mem_kv_test.rs index 3b6042fa4..ded4abfde 100644 --- a/lib/crowdb-kv/tests/kv_test/mem_kv_test.rs +++ b/lib/crowdb-paxos-kv/tests/kv_test/mem_kv_test.rs @@ -7,67 +7,68 @@ use crate::mem_kv::InMemKV; use crate::test_util::iter_all_dyn; use bytes::Bytes; -use crowdb_kv::kv::{Batch, Cell, KVEngine}; +use crowdb_paxos_kv::kv::{Batch, Cell, KVEngine}; use super::conformance; use super::conformance::{batch, del, put}; -#[test] -fn highest_slot_wins_regardless_of_apply_order() { - conformance::highest_slot_wins_regardless_of_apply_order(&InMemKV::new()); +#[tokio::test] +async fn highest_slot_wins_regardless_of_apply_order() { + conformance::highest_slot_wins_regardless_of_apply_order(&InMemKV::new()).await; } -#[test] -fn equal_slot_is_idempotent_noop() { - conformance::equal_slot_is_idempotent_noop(&InMemKV::new()); +#[tokio::test] +async fn equal_slot_is_idempotent_noop() { + conformance::equal_slot_is_idempotent_noop(&InMemKV::new()).await; } -#[test] -fn delete_writes_tombstone() { - conformance::delete_writes_tombstone(&InMemKV::new()); +#[tokio::test] +async fn delete_writes_tombstone() { + conformance::delete_writes_tombstone(&InMemKV::new()).await; } -#[test] -fn intra_batch_last_occurrence_wins() { - conformance::intra_batch_last_occurrence_wins(&InMemKV::new()); +#[tokio::test] +async fn intra_batch_last_occurrence_wins() { + conformance::intra_batch_last_occurrence_wins(&InMemKV::new()).await; } -#[test] -fn scan_is_ordered_prefix_filtered_and_truncates() { - conformance::scan_is_ordered_prefix_filtered_and_truncates(&InMemKV::new()); +#[tokio::test] +async fn scan_is_ordered_prefix_filtered_and_truncates() { + conformance::scan_is_ordered_prefix_filtered_and_truncates(&InMemKV::new()).await; } -#[test] -fn reverse_scan_is_descending_exclusive_and_prefix_bounded() { - conformance::reverse_scan_is_descending_exclusive_and_prefix_bounded(&InMemKV::new()); +#[tokio::test] +async fn reverse_scan_is_descending_exclusive_and_prefix_bounded() { + conformance::reverse_scan_is_descending_exclusive_and_prefix_bounded(&InMemKV::new()).await; } -#[test] -fn scan_reports_each_live_records_commit_slot() { - conformance::scan_reports_each_live_records_commit_slot(&InMemKV::new()); +#[tokio::test] +async fn scan_reports_each_live_records_commit_slot() { + conformance::scan_reports_each_live_records_commit_slot(&InMemKV::new()).await; } -#[test] -fn scan_end_key_exclusive_upper_bound() { - conformance::scan_end_key_exclusive_upper_bound(&InMemKV::new()); +#[tokio::test] +async fn scan_end_key_exclusive_upper_bound() { + conformance::scan_end_key_exclusive_upper_bound(&InMemKV::new()).await; } -#[test] -fn scan_byte_budget_stops_and_truncates() { - conformance::scan_byte_budget_stops_and_truncates(&InMemKV::new()); +#[tokio::test] +async fn scan_byte_budget_stops_and_truncates() { + conformance::scan_byte_budget_stops_and_truncates(&InMemKV::new()).await; } -#[test] -fn scan_keys_only_skips_values() { - conformance::scan_keys_only_skips_values(&InMemKV::new()); +#[tokio::test] +async fn scan_keys_only_skips_values() { + conformance::scan_keys_only_skips_values(&InMemKV::new()).await; } -#[test] -fn compare_is_empty_for_identical_state_and_detects_divergence() { +#[tokio::test] +async fn compare_is_empty_for_identical_state_and_detects_divergence() { conformance::compare_is_empty_for_identical_state_and_detects_divergence( &InMemKV::new(), &InMemKV::new(), - ); + ) + .await; } /// Regression guard: `InMemKV` never @@ -77,7 +78,7 @@ fn compare_is_empty_for_identical_state_and_detects_divergence() { /// wrong unwrapped value. #[test] fn get_scan_apply_always_resolve_ready() { - use crowdb_kv::kv::KVFuture; + use crowdb_paxos_kv::kv::KVFuture; let e = InMemKV::new(); assert!(matches!( @@ -91,9 +92,9 @@ fn get_scan_apply_always_resolve_ready() { )); } -#[test] -fn snapshot_export_import_round_trip() { - conformance::snapshot_export_import_round_trip(&InMemKV::new(), &InMemKV::new()); +#[tokio::test] +async fn snapshot_export_import_round_trip() { + conformance::snapshot_export_import_round_trip(&InMemKV::new(), &InMemKV::new()).await; } #[test] @@ -119,7 +120,7 @@ fn delete_nonexistent_key_is_noop() { fn clear_drops_all_state() { let e = InMemKV::new(); e.apply(1, &batch(vec![put(b"k", b"v")])).into_ready().unwrap(); - e.clear(); + e.clear().unwrap(); assert_eq!(e.get(b"k").into_ready(), None); assert!(iter_all_dyn(&e).is_empty()); } diff --git a/lib/crowdb-kv/tests/kv_test/op_codec_test.rs b/lib/crowdb-paxos-kv/tests/kv_test/op_codec_test.rs similarity index 77% rename from lib/crowdb-kv/tests/kv_test/op_codec_test.rs rename to lib/crowdb-paxos-kv/tests/kv_test/op_codec_test.rs index 8ceac9876..e27e481cd 100644 --- a/lib/crowdb-kv/tests/kv_test/op_codec_test.rs +++ b/lib/crowdb-paxos-kv/tests/kv_test/op_codec_test.rs @@ -8,7 +8,7 @@ //! payload, multi-op, large keys/values, and boundary conditions. use bytes::Bytes; -use crowdb_kv::kv::{Batch, BatchOp, Op}; +use crowdb_paxos_kv::kv::{Batch, BatchOp, Op}; fn put(key: &[u8], value: &[u8]) -> BatchOp { BatchOp { @@ -152,3 +152,41 @@ fn decode_max_op_count_u16() { let batch = Batch::decode(&Bytes::from(buf)); assert_eq!(batch.ops.len(), 300); } + +#[test] +fn checked_decode_rejects_every_truncated_prefix_without_partial_operations() { + let bytes = encode(&[(b"one", Some(b"first")), (b"two", None)]); + for length in 1..bytes.len() { + let payload = Bytes::copy_from_slice(&bytes[..length]); + assert!(Batch::try_decode(&payload).is_err(), "prefix {length}"); + } + assert_eq!( + Batch::try_decode(&Bytes::from(bytes)).unwrap().ops, + vec![put(b"one", b"first"), del(b"two")] + ); +} + +#[test] +fn checked_decode_rejects_unknown_kind_delete_payload_and_trailing_bytes() { + let mut unknown = encode(&[(b"key", None)]); + unknown[2] = 2; + assert!(Batch::try_decode(&Bytes::from(unknown)).is_err()); + let mut delete = encode(&[(b"key", Some(b"value"))]); + delete[2] = 1; + assert!(Batch::try_decode(&Bytes::from(delete)).is_err()); + let mut trailing = encode(&[(b"key", Some(b"value"))]); + trailing.push(0); + assert!(Batch::try_decode(&Bytes::from(trailing)).is_err()); + assert!(Batch::try_decode(&Bytes::from_static(&[0, 0, 1])).is_err()); +} + +#[test] +fn checked_decode_preserves_empty_gap_entries_and_empty_live_values() { + assert!(Batch::try_decode(&Bytes::new()).unwrap().ops.is_empty()); + assert!(Batch::try_decode(&Bytes::from_static(&[0, 0])) + .unwrap() + .ops + .is_empty()); + let batch = Batch::try_decode(&Bytes::from(encode(&[(b"", Some(b""))]))).unwrap(); + assert_eq!(batch.ops, vec![put(b"", b"")]); +} diff --git a/lib/crowdb-paxos-kv/tests/kv_test/tree_work_metrics_test.rs b/lib/crowdb-paxos-kv/tests/kv_test/tree_work_metrics_test.rs new file mode 100644 index 000000000..dee6fb609 --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/kv_test/tree_work_metrics_test.rs @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_tree_ffi::{TreeRuntime, TreeRuntimeOptions, TreeWorkStage}; +use std::sync::Arc; + +#[test] +fn logging_and_diagnostics_share_the_host_collector_window() { + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }) + .unwrap(); + let tree = engine.handle(); + tree.enable_work_histogram(TreeWorkStage::FlushBatch).unwrap(); + tree.apply_put(1, b"key", b"value").unwrap(); + tree.flush().unwrap(); + let window = engine.capture_work_metrics().unwrap().unwrap(); + let first = engine.format_work_metrics_window("s.7.g.9", "now"); + assert!(first.contains("s.7.g.9.tree.work.flush.batch")); + assert!(first.contains("window=1")); + let histogram = window.histograms.stage(TreeWorkStage::FlushBatch).unwrap(); + assert_eq!(histogram.count, 1); + assert_eq!(window.histograms.window_sequence, window.window_sequence); + assert!(first.contains("s.7.g.9.tree.histogram.flush.batch")); + assert!(first.contains(&format!("p50_ns={}", histogram.p50_ns))); + assert!(first.contains(&format!("p95_ns={}", histogram.p95_ns))); + assert!(first.contains(&format!("p99_ns={}", histogram.p99_ns))); + assert!(!first.contains("tree.histogram.page.load")); + assert!(window.age_sampled_at_ns > 0); + assert!(first.contains(&format!("age_sampled_at_ns={}", window.age_sampled_at_ns))); + assert!(first.contains("age_tracked=")); + assert!(first.contains("age_untracked=")); + assert!(first.contains("observed_oldest_age_ns=")); + assert!(first.contains("age_complete=")); + let diagnostics = engine.work_metrics_window().unwrap(); + assert!(Arc::ptr_eq(&window, &diagnostics)); + let second = engine.format_work_metrics_window("s.7.g.9", "now"); + assert_eq!(first, second); + let next = engine.capture_work_metrics().unwrap().unwrap(); + assert_eq!(next.window_sequence, 2); + assert_eq!(next.histograms.stage(TreeWorkStage::FlushBatch).unwrap().count, 0); + assert_eq!( + window.histograms.stage(TreeWorkStage::FlushBatch).unwrap().count, + 1 + ); + assert!(Arc::ptr_eq(&window, &diagnostics)); +} diff --git a/lib/crowdb-kv/tests/metrics_test.rs b/lib/crowdb-paxos-kv/tests/metrics_test.rs similarity index 96% rename from lib/crowdb-kv/tests/metrics_test.rs rename to lib/crowdb-paxos-kv/tests/metrics_test.rs index ea031b888..c411929f6 100644 --- a/lib/crowdb-kv/tests/metrics_test.rs +++ b/lib/crowdb-paxos-kv/tests/metrics_test.rs @@ -4,8 +4,8 @@ //! Integration tests for the metrics registry: lifecycle, window reset, //! snapshot filtering, and flush output format. -use crowdb_kv::common::logging::RotatingLogWriter; -use crowdb_kv::metrics::{MetricsRegistry, MetricsRunner}; +use crowdb_paxos_kv::common::logging::RotatingLogWriter; +use crowdb_paxos_kv::metrics::{MetricsRegistry, MetricsRunner}; use std::sync::Arc; use std::sync::Mutex; @@ -118,7 +118,7 @@ fn summary_avg_max_and_reset() { async fn registry_start_stop_lifecycle() { let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("metrics"); let mut runner = MetricsRunner::new( - crowdb_kv::common::logging::open_metrics_log(tmp.path(), "test", 30, 5).unwrap(), + crowdb_paxos_kv::common::logging::open_metrics_log(tmp.path(), "test", 30, 5).unwrap(), 1, // 1 second interval ); runner.start(); @@ -273,7 +273,7 @@ fn registry_shared_arc_mutex_usage() { #[tokio::test] async fn cpp_metrics_block_appears_with_matching_window() { let dir = crowdb_test_harness::test_dirs::test_data_dir().join(format!( - "crowdb_kv_metrics_test_{}_{}", + "crowdb_paxos_kv_metrics_test_{}_{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -366,11 +366,11 @@ fn no_bridged_cpp_names_in_rust_section() { #[cfg_attr(target_os = "macos", ignore = "per-write fsync is disabled on macOS")] #[tokio::test] async fn wal_fsync_and_write_bw_counts_match() { - use crowdb_kv::paxos::roles::{PxBallot, SlotIndex}; - use crowdb_kv::wal::{IoBackend, RecordType, WALRecord, WalConfig, WalEngine}; + use crowdb_paxos_kv::paxos::roles::{PxBallot, SlotIndex}; + use crowdb_paxos_kv::wal::{IoBackend, RecordType, WALRecord, WalConfig, WalEngine}; let dir = crowdb_test_harness::test_dirs::test_data_dir().join(format!( - "crowdb_kv_wal_metrics_test_{}_{}", + "crowdb_paxos_kv_wal_metrics_test_{}_{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -436,7 +436,7 @@ fn exact_name_lookup_returns_registered_metric_only() { .expect("selected metric must exist"); assert!(matches!( point, - crowdb_kv::metrics::MetricPoint::Counter { total: 3, .. } + crowdb_paxos_kv::metrics::MetricPoint::Counter { total: 3, .. } )); assert!(registry.snapshot_named("s.1.g.2.missing.c", 1.0).is_none()); } diff --git a/lib/crowdb-kv/tests/node_config_test.rs b/lib/crowdb-paxos-kv/tests/node_config_test.rs similarity index 98% rename from lib/crowdb-kv/tests/node_config_test.rs rename to lib/crowdb-paxos-kv/tests/node_config_test.rs index ec36dfcdf..c07a54366 100644 --- a/lib/crowdb-kv/tests/node_config_test.rs +++ b/lib/crowdb-paxos-kv/tests/node_config_test.rs @@ -3,8 +3,8 @@ //! Unit tests for `NodeConfig` and `NodeConfigStore`. -use crowdb_kv::cluster::group_config::{PxGroupConfig, PxGroupMember}; -use crowdb_kv::cluster::node_config::{NodeConfig, NodeConfigStore}; +use crowdb_paxos_kv::cluster::group_config::{PxGroupConfig, PxGroupMember}; +use crowdb_paxos_kv::cluster::node_config::{NodeConfig, NodeConfigStore}; #[test] fn node_config_default_is_empty() { diff --git a/lib/crowdb-kv/tests/paxos_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test.rs similarity index 75% rename from lib/crowdb-kv/tests/paxos_test.rs rename to lib/crowdb-paxos-kv/tests/paxos_test.rs index eb5327c45..5c752744b 100644 --- a/lib/crowdb-kv/tests/paxos_test.rs +++ b/lib/crowdb-paxos-kv/tests/paxos_test.rs @@ -24,8 +24,20 @@ mod learner_dedup; #[path = "paxos_test/learner_async_test.rs"] mod learner_async; +#[path = "paxos_test/runtime_gap_test.rs"] +mod runtime_gap; + +#[path = "paxos_test/runtime_shutdown_test.rs"] +mod runtime_shutdown; + +#[path = "paxos_test/native_apply_window_test.rs"] +mod native_apply_window; + #[path = "paxos_test/roles_test.rs"] mod roles; #[path = "paxos_test/error_test.rs"] mod error; + +#[path = "paxos_test/cas_rejection_test.rs"] +mod cas_rejection; diff --git a/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/acceptor_test.rs similarity index 98% rename from lib/crowdb-kv/tests/paxos_test/acceptor_test.rs rename to lib/crowdb-paxos-kv/tests/paxos_test/acceptor_test.rs index 9ad583701..06a9727f8 100644 --- a/lib/crowdb-kv/tests/paxos_test/acceptor_test.rs +++ b/lib/crowdb-paxos-kv/tests/paxos_test/acceptor_test.rs @@ -6,8 +6,10 @@ //! Covers acceptor state machine invariants and ballot monotonicity. //! Key work: prepare/promise, accept/accepted, ballot ordering, idempotency. -use crowdb_kv::paxos::acceptor::PxAcceptor; -use crowdb_kv::paxos::roles::{Acceptor, PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply, SlotIndex}; +use crowdb_paxos_kv::paxos::acceptor::PxAcceptor; +use crowdb_paxos_kv::paxos::roles::{ + Acceptor, PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply, SlotIndex, +}; fn entry(slot: SlotIndex, ballot: PxBallot, payload: &[u8]) -> PxLogEntry { PxLogEntry { diff --git a/lib/crowdb-paxos-kv/tests/paxos_test/cas_rejection_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/cas_rejection_test.rs new file mode 100644 index 000000000..f5f0765f6 --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/paxos_test/cas_rejection_test.rs @@ -0,0 +1,292 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::group_operations::{ + KvGroupOperationError, KvGroupOperations, KvReadConsistency, +}; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{ + ApplyAdmissionError, ApplyAdmissionResult, Batch, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, KVFuture, + ScanDirection, +}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::IoBackend; +use crowdb_tree_ffi::{TreeRuntime, TreeRuntimeOptions}; + +const FENCE: &[u8] = b"/diskdb/ownership-fence/read-admission"; +const RECORD: &[u8] = b"record"; + +#[derive(Default)] +struct TestFailures { + read: AtomicBool, + pending_read: AtomicBool, + proposal: AtomicBool, +} + +struct TestEngine { + inner: CrowdbTreeEngine, + failures: Arc<TestFailures>, +} + +impl KVEngine for TestEngine { + fn apply(&self, slot: u64, batch: &Batch) -> KVFuture<Result<(), String>> { + self.inner.apply(slot, batch) + } + + fn admit_proposal(&self, payload: Bytes) -> KVFuture<ApplyAdmissionResult> { + if self.failures.proposal.load(Ordering::Acquire) { + KVFuture::ready(Err(ApplyAdmissionError::Rejected( + "test proposal rejection".into(), + ))) + } else { + self.inner.admit_proposal(payload) + } + } + + fn get(&self, key: &[u8]) -> KVFuture<Option<(u64, Vec<u8>)>> { + self.inner.get(key) + } + + fn get_versioned(&self, key: &[u8]) -> KVFuture<Result<Option<(u64, Bytes)>, String>> { + if key == RECORD && self.failures.read.load(Ordering::Acquire) { + let error = "ResourceExhausted before proposal".to_owned(); + if self.failures.pending_read.load(Ordering::Acquire) { + KVFuture::Pending(Box::pin(async move { + tokio::task::yield_now().await; + Err(error) + })) + } else { + KVFuture::ready(Err(error)) + } + } else { + self.inner.get_versioned(key) + } + } + + fn scan_directional( + &self, + prefix: &[u8], + start: &[u8], + end: &[u8], + limit: usize, + bytes: usize, + keys: bool, + deadline: u64, + direction: ScanDirection, + ) -> KVFuture<Result<(Vec<(Bytes, u64, Bytes)>, bool), String>> { + self.inner + .scan_directional(prefix, start, end, limit, bytes, keys, deadline, direction) + } + + fn clear(&self) -> Result<(), String> { + self.inner.clear() + } + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +async fn replica() -> (PxLocalReplica, Arc<TestFailures>) { + let failures = Arc::new(TestFailures::default()); + let engine = TestEngine { + inner: CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }) + .unwrap(), + failures: failures.clone(), + }; + let replay = replay_group(&Arc::new(IoBackend::File), &[], 1).await.unwrap(); + let replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(engine), + ) + .await + .unwrap(); + (replica, failures) +} + +async fn group() -> (Arc<PxGroup>, Arc<TestFailures>) { + let (replica, failures) = replica().await; + let group = Arc::new(PxGroup::new(1, replica)); + group.set_self_weak(); + (group, failures) +} + +#[tokio::test] +async fn point_read_errors_never_become_successful_absence() { + let (replica, failures) = replica().await; + let store = PxKvStore::new(1, "127.0.0.1:0".parse().unwrap()); + store.add_group_without_election(PxGroup::new(1, replica)); + let operations = store.group_operations(1).unwrap(); + failures.read.store(true, Ordering::Release); + for pending in [false, true] { + failures.pending_read.store(pending, Ordering::Release); + let error = operations + .get(RECORD, KvReadConsistency::Linearizable) + .await + .unwrap_err(); + assert_eq!( + error, + KvGroupOperationError::Unavailable("ResourceExhausted before proposal".into()) + ); + let response = store.kv_get(1, RECORD, 0, 0, 1, 0).await; + assert!(!response.ok); + assert!(!response.not_found); + assert_eq!(response.error, "ResourceExhausted before proposal"); + } + failures.read.store(false, Ordering::Release); + let response = store.kv_get(1, RECORD, 0, 0, 2, 0).await; + assert!(response.not_found); +} + +fn payload(key: &[u8], value: &[u8]) -> Vec<u8> { + let mut payload = vec![1, 0, 0]; + payload.extend_from_slice(&u32::try_from(key.len()).unwrap().to_le_bytes()); + payload.extend_from_slice(key); + payload.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes()); + payload.extend_from_slice(value); + payload +} + +async fn readable(group: &Arc<PxGroup>, chosen: u64) { + assert_eq!(group.local_replica().last_chosen_slot(), chosen); + assert!(group.local_replica().accepted_at(chosen + 1).await.is_none()); + KvGroupOperations::new(group.clone(), 1024) + .bind_current_leader_tenure() + .await + .expect("definite rejection preserves read readiness"); +} + +#[tokio::test] +async fn failed_cas_read_does_not_poison_the_leader_or_consume_a_slot() { + let (group, failures) = group().await; + failures.read.store(true, Ordering::Release); + let key = Bytes::from_static(RECORD); + let result = group + .propose_cas(payload(RECORD, b"value"), key.clone(), 0, 1, 1) + .await; + assert!(matches!(result, ProposeResult::Rejected(_)), "{result:?}"); + readable(&group, 0).await; + failures.read.store(false, Ordering::Release); + assert!(matches!( + group.propose_cas(payload(RECORD, b"value"), key, 0, 1, 2).await, + ProposeResult::Chosen { slot: 1 } + )); +} + +#[tokio::test] +async fn rejected_owner_handover_releases_its_admission_without_poisoning_it() { + let (group, failures) = group().await; + let key = Bytes::from_static(FENCE); + failures.proposal.store(true, Ordering::Release); + let result = group + .propose_cas(payload(FENCE, b"owner"), key.clone(), 0, 1, 1) + .await; + assert!(matches!(result, ProposeResult::Rejected(_)), "{result:?}"); + assert!(!group.owner_change_pending_for_tests(&key)); + readable(&group, 0).await; + failures.proposal.store(false, Ordering::Release); + assert!(matches!( + group.propose_cas(payload(FENCE, b"owner"), key, 0, 1, 2).await, + ProposeResult::Chosen { slot: 1 } + )); +} + +#[tokio::test] +async fn nested_owner_cas_read_rejection_preserves_the_owner_and_leader() { + let (group, failures) = group().await; + let key = Bytes::from_static(FENCE); + assert!(matches!( + group + .propose_cas(payload(FENCE, b"owner"), key.clone(), 0, 1, 1) + .await, + ProposeResult::Chosen { slot: 1 } + )); + failures.read.store(true, Ordering::Release); + let result = group + .propose_owner_write_for_tests( + payload(RECORD, b"value"), + key.clone(), + Bytes::from_static(b"owner"), + Some((Bytes::from_static(RECORD), 0)), + 1, + 2, + ) + .await; + assert!(matches!(result, ProposeResult::Rejected(_)), "{result:?}"); + readable(&group, 1).await; + failures.read.store(false, Ordering::Release); + let result = group + .propose_owner_write_for_tests( + payload(RECORD, b"value"), + key, + Bytes::from_static(b"owner"), + Some((Bytes::from_static(RECORD), 0)), + 1, + 3, + ) + .await; + assert!(matches!(result, ProposeResult::Chosen { slot: 2 }), "{result:?}"); +} + +#[tokio::test] +async fn invalid_cas_identity_is_definitely_rejected_without_consuming_a_slot() { + let (group, _) = group().await; + let key = Bytes::from_static(RECORD); + let result = group + .propose_cas(payload(RECORD, b"value"), key.clone(), 0, 0, 1) + .await; + assert!(matches!(result, ProposeResult::Rejected(_)), "{result:?}"); + readable(&group, 0).await; + assert!(matches!( + group.propose_cas(payload(RECORD, b"value"), key, 0, 1, 2).await, + ProposeResult::Chosen { slot: 1 } + )); +} + +#[tokio::test] +async fn nested_owner_cas_invalid_identity_preserves_owner_and_leader_progress() { + let (group, _) = group().await; + let key = Bytes::from_static(FENCE); + assert!(matches!( + group + .propose_cas(payload(FENCE, b"owner"), key.clone(), 0, 1, 1) + .await, + ProposeResult::Chosen { slot: 1 } + )); + let result = group + .propose_owner_write_for_tests( + payload(RECORD, b"value"), + key.clone(), + Bytes::from_static(b"owner"), + Some((Bytes::from_static(RECORD), 0)), + 0, + 2, + ) + .await; + assert!(!matches!(result, ProposeResult::Chosen { .. }), "{result:?}"); + // Validation rejected before a proposal. This barrier checks that the + // nested owner guard did not turn the known rejection into unresolved work. + readable(&group, 1).await; + assert!(matches!(result, ProposeResult::Rejected(_)), "{result:?}"); + let result = group + .propose_owner_write_for_tests( + payload(RECORD, b"value"), + key, + Bytes::from_static(b"owner"), + Some((Bytes::from_static(RECORD), 0)), + 1, + 3, + ) + .await; + assert!(matches!(result, ProposeResult::Chosen { slot: 2 }), "{result:?}"); +} diff --git a/lib/crowdb-kv/tests/paxos_test/error_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/error_test.rs similarity index 96% rename from lib/crowdb-kv/tests/paxos_test/error_test.rs rename to lib/crowdb-paxos-kv/tests/paxos_test/error_test.rs index eb7f98bf0..8a384a354 100644 --- a/lib/crowdb-kv/tests/paxos_test/error_test.rs +++ b/lib/crowdb-paxos-kv/tests/paxos_test/error_test.rs @@ -6,8 +6,8 @@ //! preemption retry, crowdb-rpc boundary rejection) lives in the group layer at //! `tests/group/paxos_error_test.rs`. -use crowdb_kv::paxos::error::{PxPaxosError, PxPaxosPhase, PxRetryAction}; -use crowdb_kv::paxos::roles::PxBallot; +use crowdb_paxos_kv::paxos::error::{PxPaxosError, PxPaxosPhase, PxRetryAction}; +use crowdb_paxos_kv::paxos::roles::PxBallot; #[test] fn paxos_error_classifier_maps_prepare_rejection_to_same_slot_prepare() { diff --git a/lib/crowdb-kv/tests/paxos_test/learner_async_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/learner_async_test.rs similarity index 90% rename from lib/crowdb-kv/tests/paxos_test/learner_async_test.rs rename to lib/crowdb-paxos-kv/tests/paxos_test/learner_async_test.rs index 29d83da6d..85d833eff 100644 --- a/lib/crowdb-kv/tests/paxos_test/learner_async_test.rs +++ b/lib/crowdb-paxos-kv/tests/paxos_test/learner_async_test.rs @@ -9,9 +9,9 @@ //! covers that layer). This is the test the design doc's §6 explicitly //! deferred adding until `PxLearner` actually had an `async fn` to test. -use crowdb_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; -use crowdb_kv::paxos::learner::PxLearner; -use crowdb_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::paxos::learner::PxLearner; +use crowdb_paxos_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; fn encode_put_payload(key: &[u8], value: &[u8]) -> Vec<u8> { let mut payload = Vec::new(); @@ -66,6 +66,8 @@ async fn engine_get_resolves_correctly_across_both_ready_and_pending() { // this same property). handle.flush().expect("flush"); handle.snapshot().expect("snapshot"); + // Checkpoint installs reloadable bases without retaining resident frames. + assert_eq!(learner.engine_get(b"k").await, Some((1, b"v".to_vec()))); let evicted = handle.evict_clean_leaves(0); assert!( evicted > 0, diff --git a/lib/crowdb-kv/tests/paxos_test/learner_dedup_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/learner_dedup_test.rs similarity index 89% rename from lib/crowdb-kv/tests/paxos_test/learner_dedup_test.rs rename to lib/crowdb-paxos-kv/tests/paxos_test/learner_dedup_test.rs index 8d2ee6754..d26089e9b 100644 --- a/lib/crowdb-kv/tests/paxos_test/learner_dedup_test.rs +++ b/lib/crowdb-paxos-kv/tests/paxos_test/learner_dedup_test.rs @@ -9,9 +9,10 @@ //! watermark advancement including out-of-order gap fill. use bytes::Bytes; -use crowdb_kv::kv::CrowdbTreeEngine; -use crowdb_kv::paxos::learner::PxLearner; -use crowdb_kv::paxos::roles::{DedupTag, Learner, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::paxos::learner::PxLearner; +use crowdb_paxos_kv::paxos::roles::{DedupTag, Learner, PxBallot, PxLogEntry}; +use crowdb_tree_ffi::{TreeRuntime, TreeRuntimeOptions}; use std::sync::{Arc, Barrier}; fn tag(client_id: u64, seq: u64) -> [DedupTag; 1] { @@ -67,6 +68,9 @@ async fn dedup_lookup_returns_none_for_client_id_zero() { #[tokio::test] async fn dedup_lookup_returns_slot_for_already_applied_seq() { let learner = PxLearner::new(); + for slot in 1..5 { + learner.learn(noop_entry(slot), &[]).await; + } let entry = write_entry(5, &encode_put(b"k", b"v")); learner.learn(entry, &tag(42, 3)).await; assert_eq!( @@ -83,6 +87,9 @@ async fn dedup_lookup_returns_slot_for_already_applied_seq() { #[tokio::test] async fn dedup_lookup_returns_none_for_higher_seq() { let learner = PxLearner::new(); + for slot in 1..5 { + learner.learn(noop_entry(slot), &[]).await; + } let entry = write_entry(5, &encode_put(b"k", b"v")); learner.learn(entry, &tag(42, 3)).await; assert!( @@ -98,6 +105,7 @@ async fn dedup_records_each_seq_at_its_own_slot() { learner .learn(write_entry(1, &encode_put(b"a", b"1")), &tag(7, 1)) .await; + learner.learn(noop_entry(2), &[]).await; learner .learn(write_entry(3, &encode_put(b"a", b"5")), &tag(7, 5)) .await; @@ -184,9 +192,16 @@ async fn learn_advances_contiguous_chosen_and_applied_in_order() { #[tokio::test] async fn learn_out_of_order_does_not_advance_contiguous_until_gap_filled() { + use std::future::{poll_fn, Future}; + use std::task::Poll; let learner = PxLearner::new(); // Learn slot 3 before slot 1 and 2 — gap at 1 blocks contiguous. - learner.learn(write_entry(3, &encode_put(b"k", b"v3")), &[]).await; + let mut higher = Box::pin(learner.learn(write_entry(3, &encode_put(b"k", b"v3")), &[])); + poll_fn(|context| { + assert!(higher.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; assert_eq!(learner.contiguous_chosen(), 0); assert_eq!(learner.contiguous_applied(), 0); assert_eq!(learner.last_chosen_slot(), 3); @@ -197,13 +212,23 @@ async fn learn_out_of_order_does_not_advance_contiguous_until_gap_filled() { // Fill slot 2 — contiguous drains the out-of-order map and jumps to 3. learner.learn(write_entry(2, &encode_put(b"k", b"v2")), &[]).await; + higher.await; assert_eq!(learner.contiguous_chosen(), 3); assert_eq!(learner.contiguous_applied(), 3); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn concurrent_gap_fill_drains_both_frontiers() { - let learner = Arc::new(PxLearner::new()); + // This fixture submits 128 replay operations before any gap is repaired. + // Provision that registration window explicitly; the default is 64. + let mut config = CrowdbTreeConfig { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }; + config.runtime_binding.cpu_queue_limit = 128; + let learner = Arc::new(PxLearner::with_engine(Box::new( + CrowdbTreeEngine::open(&config).unwrap(), + ))); let mut tasks = Vec::new(); for slot in (1..=128u64).rev() { let learner = Arc::clone(&learner); diff --git a/lib/crowdb-kv/tests/paxos_test/learner_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/learner_test.rs similarity index 97% rename from lib/crowdb-kv/tests/paxos_test/learner_test.rs rename to lib/crowdb-paxos-kv/tests/paxos_test/learner_test.rs index ea8ca375a..3ca5aaad7 100644 --- a/lib/crowdb-kv/tests/paxos_test/learner_test.rs +++ b/lib/crowdb-paxos-kv/tests/paxos_test/learner_test.rs @@ -5,7 +5,7 @@ //! receipt should advance `last_chosen_slot` / `last_chosen_term` only, //! never the contiguous watermarks. -use crowdb_kv::paxos::learner::PxLearner; +use crowdb_paxos_kv::paxos::learner::PxLearner; #[test] fn note_chosen_advances_high_water_mark() { diff --git a/lib/crowdb-paxos-kv/tests/paxos_test/native_apply_window_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/native_apply_window_test.rs new file mode 100644 index 000000000..12971cf69 --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/paxos_test/native_apply_window_test.rs @@ -0,0 +1,165 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::path::PathBuf; +use std::sync::Arc; +use std::task::Poll; +use std::time::Duration; + +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice}; +use crowdb_tree_ffi::{TreeRuntime, TreeRuntimeOptions}; +use tokio::sync::Notify; + +async fn group() -> PxGroup { + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let replay = replay_group(&backend, &[PathBuf::from("/empty")], 1) + .await + .unwrap(); + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + ..Default::default() + }) + .unwrap(); + let replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(engine), + ) + .await + .unwrap(); + let mut group = PxGroup::new(1, replica); + group.set_inflight_config(1, false); + group.set_async_engine_apply(true); + group +} + +fn payload(value: u8) -> Vec<u8> { + vec![1, 0, 0, 1, 0, 0, 0, b'k', 1, 0, 0, 0, value] +} + +#[tokio::test] +async fn chosen_reply_keeps_the_native_apply_inside_the_proposal_window() { + let group = group().await; + let gate = Arc::new(Notify::new()); + group + .local_replica() + .learner + .set_apply_gate_for_tests(Arc::clone(&gate)); + let chosen = group.propose(payload(b'a'), None, None).await; + let available_before_apply = group.inflight_semaphore().available_permits(); + let applied_before_release = group.local_replica().contiguous_applied(); + gate.notify_one(); + tokio::time::timeout(Duration::from_secs(5), group.local_replica().await_apply_fence(1)) + .await + .unwrap(); + assert!(matches!(chosen, ProposeResult::Chosen { slot: 1 })); + assert_eq!( + applied_before_release, 0, + "Chosen must still return before native apply" + ); + assert_eq!( + available_before_apply, 0, + "the acknowledged native apply still owns the window" + ); + assert_eq!(group.inflight_semaphore().available_permits(), 1); +} + +#[tokio::test] +async fn queued_proposal_waits_for_native_apply_without_allocating_another_slot() { + let group = group().await; + let gate = Arc::new(Notify::new()); + group + .local_replica() + .learner + .set_apply_gate_for_tests(Arc::clone(&gate)); + let first = group.propose(payload(b'a'), None, None).await; + let mut second = Box::pin(group.propose(payload(b'b'), None, None)); + let queued = matches!(futures::poll!(&mut second), Poll::Pending); + let chosen_before_release = group.local_replica().last_chosen_slot(); + gate.notify_one(); + tokio::time::timeout(Duration::from_secs(5), group.local_replica().await_apply_fence(1)) + .await + .unwrap(); + let second = tokio::time::timeout(Duration::from_secs(5), second) + .await + .unwrap(); + gate.notify_one(); + tokio::time::timeout(Duration::from_secs(5), group.local_replica().await_apply_fence(2)) + .await + .unwrap(); + assert!(matches!(first, ProposeResult::Chosen { slot: 1 })); + assert!(queued, "queue policy must backpressure before native admission"); + assert_eq!(chosen_before_release, 1); + assert!(matches!(second, ProposeResult::Chosen { slot: 2 })); + assert_eq!( + group.local_replica().learner.engine_get(b"k").await, + Some((2, vec![b'b'])) + ); + assert_eq!(group.inflight_semaphore().available_permits(), 1); +} + +#[tokio::test] +async fn cancelling_a_queued_proposal_removes_its_waiter_without_releasing_the_active_apply() { + let group = group().await; + let gate = Arc::new(Notify::new()); + group + .local_replica() + .learner + .set_apply_gate_for_tests(Arc::clone(&gate)); + let first = group.propose(payload(b'a'), None, None).await; + let mut queued = Box::pin(group.propose(payload(b'b'), None, None)); + let pending = matches!(futures::poll!(&mut queued), Poll::Pending); + let before_cancel = group.status().inflight.unwrap(); + drop(queued); + let after_cancel = group.status().inflight.unwrap(); + gate.notify_one(); + tokio::time::timeout(Duration::from_secs(5), group.local_replica().await_apply_fence(1)) + .await + .unwrap(); + assert!(matches!(first, ProposeResult::Chosen { slot: 1 })); + assert!(pending); + assert_eq!(before_cancel.waiting, 1); + assert_eq!(after_cancel.waiting, 0); + assert_eq!(after_cancel.occupied, 1); + assert_eq!(group.local_replica().last_chosen_slot(), 1); + assert_eq!(group.inflight_semaphore().available_permits(), 1); +} + +#[tokio::test] +async fn cancelling_a_chosen_synchronous_wait_keeps_the_window_with_owned_native_apply() { + let mut group = group().await; + group.set_async_engine_apply(false); + let gate = Arc::new(Notify::new()); + group + .local_replica() + .learner + .set_apply_gate_for_tests(Arc::clone(&gate)); + let mut proposal = Box::pin(group.propose(payload(b'a'), None, None)); + let pending = matches!(futures::poll!(&mut proposal), Poll::Pending); + let chosen = group.local_replica().last_chosen_slot(); + drop(proposal); + let available_after_cancel = group.inflight_semaphore().available_permits(); + gate.notify_one(); + tokio::time::timeout(Duration::from_secs(5), group.local_replica().await_apply_fence(1)) + .await + .unwrap(); + assert!(pending); + assert_eq!(chosen, 1); + assert_eq!(available_after_cancel, 0); + assert_eq!(group.inflight_semaphore().available_permits(), 1); + assert_eq!( + group.local_replica().learner.engine_get(b"k").await, + Some((1, vec![b'a'])) + ); +} diff --git a/lib/crowdb-kv/tests/paxos_test/roles_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/roles_test.rs similarity index 97% rename from lib/crowdb-kv/tests/paxos_test/roles_test.rs rename to lib/crowdb-paxos-kv/tests/paxos_test/roles_test.rs index efedbf38d..77a6a38d4 100644 --- a/lib/crowdb-kv/tests/paxos_test/roles_test.rs +++ b/lib/crowdb-paxos-kv/tests/paxos_test/roles_test.rs @@ -8,7 +8,7 @@ //! semantics. use bytes::Bytes; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; // ── PxBallot ordering ──────────────────────────────────────── diff --git a/lib/crowdb-paxos-kv/tests/paxos_test/runtime_gap_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/runtime_gap_test.rs new file mode 100644 index 000000000..272b2baef --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/paxos_test/runtime_gap_test.rs @@ -0,0 +1,192 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use crowdb_paxos_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::WalConfig; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::metrics::MetricsRegistry; +use crowdb_paxos_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::{replay_group, ReplayResult}; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine}; +use crowdb_tree_ffi::{TreeRuntime, TreeRuntimeOptions}; + +fn engine() -> CrowdbTreeEngine { + CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + ..Default::default() + }) + .unwrap() +} + +fn entry(slot: u64) -> PxLogEntry { + let mut payload = vec![1, 0, 0]; + payload.extend_from_slice(&1u32.to_le_bytes()); + payload.push(b'k'); + payload.extend_from_slice(&8u32.to_le_bytes()); + payload.extend_from_slice(&slot.to_le_bytes()); + PxLogEntry { + slot, + term: 1, + ballot: PxBallot::new(1, 2), + payload: Bytes::from(payload), + } +} + +async fn heartbeat(replica: &PxLocalReplica, slot: u64) { + let reply = ReplicaHandler::on_heartbeat( + replica, + HeartbeatRequestPayload { + term: 1, + leader_id: 2, + prev_log_slot: 0, + prev_log_term: 0, + committed_safe_slot: slot, + lease_grant_until_ms_mono: 0, + t_send_ms_mono: 0, + gc_before_slot: 0, + gc_membership_epoch: 0, + }, + 1, + ) + .await + .unwrap(); + assert!(reply.success); +} + +async fn replay(entries: &[PxLogEntry]) -> ReplayResult { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("runtime-gap-replay"); + let backend = Arc::new(IoBackend::File); + let wal = WalEngine::create( + Arc::clone(&backend), + WalConfig::with_root(directory.path().to_path_buf()), + 1, + ) + .await + .unwrap(); + for entry in entries { + wal.append(&WALRecord::from_accepted(1, entry)).await.unwrap(); + } + wal.seal_all().await.unwrap(); + drop(wal); + replay_group(&backend, &[directory.path().to_path_buf()], 1) + .await + .unwrap() +} + +#[tokio::test(flavor = "current_thread")] +async fn background_gap_repair_wakes_the_original_runtime_apply_loop() { + let replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Follower, + &replay(&[]).await, + Box::new(engine()), + ) + .await + .unwrap(); + let registry = Arc::new(Mutex::new(MetricsRegistry::new())); + replica.set_metrics_registry(®istry, 1, 1); + let skipped = registry + .lock() + .unwrap() + .register_counter("s.1.g.1.paxos.apply_loop.skip.c"); + assert!(matches!( + replica.on_accept(&entry(2)).await, + PxAcceptReply::Accepted { .. } + )); + heartbeat(&replica, 2).await; + tokio::time::timeout(Duration::from_secs(2), async { + while skipped.snapshot().total == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("the real background loop must observe its first gap"); + assert_eq!(replica.contiguous_applied(), 0); + assert!(replica.learner.engine_get(b"k").await.is_none()); + + // No second heartbeat: acceptance of the missing chosen value must wake + // the loop already waiting for slot one, rather than leave it awaiting two. + assert!(matches!( + replica.on_accept(&entry(1)).await, + PxAcceptReply::Accepted { .. } + )); + tokio::time::timeout(Duration::from_secs(2), replica.await_apply_fence(2)) + .await + .expect("the repaired gap must allow both real entries to apply"); + assert_eq!( + replica.learner.engine_get(b"k").await, + Some((2, 2u64.to_le_bytes().to_vec())) + ); + assert!(replica.shutdown(Duration::from_secs(2)).await.is_clean()); +} + +#[tokio::test(flavor = "current_thread")] +async fn sparse_durable_replay_stops_at_gap_and_preserves_the_accepted_suffix() { + let entries = [entry(1), entry(3)]; + let recovered = replay(&entries).await; + let replica = tokio::time::timeout( + Duration::from_secs(2), + PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Follower, + &recovered, + Box::new(engine()), + ), + ) + .await + .expect("startup must not await a missing slot that only live repair can supply") + .unwrap(); + assert_eq!(replica.contiguous_applied(), 1); + assert_eq!(replica.contiguous_chosen(), 1); + assert_eq!(replica.last_chosen_slot(), 1); + assert_eq!(replica.learner.engine().resume_from_slot(), 0); + assert!(replica.accepted_at(2).await.is_none()); + assert_eq!(replica.accepted_at(3).await.unwrap().payload, entries[1].payload); + assert_eq!( + replica.learner.engine_get(b"k").await, + Some((1, 1u64.to_le_bytes().to_vec())) + ); + + heartbeat(&replica, 3).await; + assert!(matches!( + replica.on_accept(&entry(2)).await, + PxAcceptReply::Accepted { .. } + )); + tokio::time::timeout(Duration::from_secs(2), replica.await_apply_fence(3)) + .await + .expect("repair must resume the retained accepted suffix"); + assert_eq!( + replica.learner.engine_get(b"k").await, + Some((3, 3u64.to_le_bytes().to_vec())) + ); + assert!(replica.shutdown(Duration::from_secs(2)).await.is_clean()); +} + +#[tokio::test(flavor = "current_thread")] +async fn malformed_durable_runtime_replay_returns_the_real_apply_failure() { + let mut invalid = entry(1); + invalid.payload = Bytes::from_static(&[1, 0]); + let recovered = replay(&[invalid]).await; + let error = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Follower, + &recovered, + Box::new(engine()), + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!(error.to_string().contains("restore slot 1")); +} diff --git a/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_owner_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_owner_test.rs new file mode 100644 index 000000000..132d379d8 --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_owner_test.rs @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{entry, group, DEADLINE}; +use crowdb_paxos_kv::cluster::{group::ProposeResult, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::{PaxosConfig, PaxosKvConfig}; +use std::sync::Arc; +#[tokio::test(flavor = "current_thread")] +async fn runtime_coalescer_does_not_keep_tree_host_owned_after_shutdown() { + let mut group = group(PxLocalReplicaRole::Leader).await; + group.set_from_config(&PaxosKvConfig { + wal_early_ack: false, + async_engine_apply: false, + paxos: PaxosConfig { + coalesce_max_keys: 32, + ..PaxosConfig::DEFAULT + }, + ..PaxosKvConfig::default() + }); + let group = Arc::new(group); + group.set_self_weak(); + let weak = Arc::downgrade(&group); + let result = group.propose(entry(1).payload.to_vec(), Some(1), Some(1)).await; + assert!(matches!(result, ProposeResult::Chosen { .. }), "{result:?}"); + let report = group.shutdown(DEADLINE).await; + assert!(report.is_clean(), "{:?}", report.errors); + drop(group); + assert_eq!( + weak.strong_count(), + 0, + "coalescer watchdog retained the native Tree and its host" + ); +} diff --git a/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_rpc_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_rpc_test.rs new file mode 100644 index 000000000..b263e100c --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_rpc_test.rs @@ -0,0 +1,165 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::{accept, assert_applied, entry, group, DEADLINE}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::replica::HeartbeatRequestPayload; +use crowdb_paxos_kv::cluster::{ + PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica, +}; +use crowdb_paxos_kv::paxos::roles::{PxAcceptReply, PxBallot, PxPrepareReply}; +use crowdb_paxos_kv::rpc::ReplicaDestination; +use std::sync::Arc; +use std::time::Duration; + +async fn peer(role: PxLocalReplicaRole) -> Arc<PxKvStore> { + let store = Arc::new(PxKvStore::new(2, "127.0.0.1:0".parse().unwrap())); + store.add_group_without_election(PxGroup::new(1, PxLocalReplica::new(2, role))); + store.start().await.unwrap(); + store +} + +async fn runtime_store(role: PxLocalReplicaRole, peer: &Arc<PxKvStore>) -> Arc<PxKvStore> { + let mut runtime = group(role).await; + runtime.set_remote_replicas(vec![PxRemoteReplica::new( + 2, + peer.listen_addr().unwrap().to_string(), + )]); + let store = Arc::new(PxKvStore::new(1, "127.0.0.1:0".parse().unwrap())); + store.add_group_without_election(runtime); + store.start().await.unwrap(); + store +} + +#[tokio::test(flavor = "current_thread")] +async fn store_keeps_consensus_server_live_until_remote_quorum_repairs_the_gap() { + let peer = peer(PxLocalReplicaRole::Follower).await; + let peer_group = peer.get_group(1).unwrap(); + accept(peer_group.local_replica(), 1).await; + accept(peer_group.local_replica(), 2).await; + let store = runtime_store(PxLocalReplicaRole::Leader, &peer).await; + let group = store.get_group(1).unwrap(); + accept(group.local_replica(), 2).await; + group.local_replica().spawn_learn_chosen(entry(2), &[]); + group.pause_repair_for_tests(); + let shutdown = tokio::spawn({ + let store = store.clone(); + async move { store.shutdown(DEADLINE).await } + }); + tokio::time::timeout(DEADLINE, async { + while !group.repair_is_paused_for_tests() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let endpoint = store.listen_addr().unwrap().to_string(); + let transport = peer.rpc_transport().unwrap(); + // Both processes have real consensus servers. The draining server still + // receives peer RPCs while its own Prepare/Accept repair is paused. + let reply = tokio::time::timeout( + DEADLINE, + transport.send_prepare( + ReplicaDestination { + endpoint: &endpoint, + replica_id: 1, + }, + 1, + PxBallot::new(1, 1), + 0, + 1, + 0, + ), + ) + .await + .expect("shutdown closed consensus ingress before accepted work drained") + .unwrap(); + assert!(matches!(reply, PxPrepareReply::Promised { .. })); + group.resume_repair_for_tests(); + let report = shutdown.await.unwrap(); + assert!(report.is_clean(), "{:?}", report.errors); + assert_applied(group.local_replica()).await; + assert!(peer.shutdown(DEADLINE).await.is_clean()); +} + +#[tokio::test(flavor = "current_thread")] +async fn timed_out_store_keeps_rpc_and_fetch_gap_alive_until_retry() { + let peer = peer(PxLocalReplicaRole::Leader).await; + let peer_group = peer.get_group(1).unwrap(); + accept(peer_group.local_replica(), 2).await; + let store = runtime_store(PxLocalReplicaRole::Follower, &peer).await; + // Mixed-engine stores must not tear down a legacy group while another + // group's accepted native prefix is still awaiting repair. + store.add_group_without_election(PxGroup::new( + 2, + PxLocalReplica::new(3, PxLocalReplicaRole::Follower), + )); + let group = store.get_group(1).unwrap(); + group.local_replica().set_believed_leader(2); + group.start_fetchgap_driver().await; + let endpoint = store.listen_addr().unwrap().to_string(); + let transport = peer.rpc_transport().unwrap(); + let destination = ReplicaDestination { + endpoint: &endpoint, + replica_id: 1, + }; + let accepted = transport.send_accept(destination, &entry(2), 1, 0).await.unwrap(); + assert!(matches!(accepted, PxAcceptReply::Accepted { .. })); + transport + .send_heartbeat( + ReplicaDestination { + endpoint: &endpoint, + replica_id: 1, + }, + HeartbeatRequestPayload { + term: 0, + leader_id: 2, + prev_log_slot: 0, + prev_log_term: 0, + committed_safe_slot: 2, + lease_grant_until_ms_mono: 0, + t_send_ms_mono: 0, + gc_before_slot: 0, + gc_membership_epoch: 0, + }, + 1, + ) + .await + .unwrap(); + let first = store.shutdown(Duration::from_millis(20)).await; + assert!(!first.is_clean()); + assert!(first + .errors + .iter() + .any(|error| error.contains("accepted Tree work did not drain"))); + assert_eq!(group.local_replica().contiguous_applied(), 0); + let reply = tokio::time::timeout( + DEADLINE, + transport.send_accept( + ReplicaDestination { + endpoint: &endpoint, + replica_id: 3, + }, + &entry(1), + 2, + 0, + ), + ) + .await + .expect("timed-out shutdown closed the shared consensus server") + .unwrap(); + assert!(matches!(reply, PxAcceptReply::Accepted { .. })); + // The original FetchGap driver now obtains slot one over RPC and wakes + // the retained contiguous apply loop, without replacing either owner. + accept(peer_group.local_replica(), 1).await; + // FetchGap serves the leader's proven chosen prefix, not accepted values. + peer_group.local_replica().learn_chosen(&entry(1), &[]).await; + peer_group.local_replica().learn_chosen(&entry(2), &[]).await; + tokio::time::timeout(DEADLINE, group.local_replica().await_apply_fence(2)) + .await + .unwrap(); + let retried = store.shutdown(DEADLINE).await; + assert!(retried.is_clean(), "{:?}", retried.errors); + assert_applied(group.local_replica()).await; + assert!(peer.shutdown(DEADLINE).await.is_clean()); +} diff --git a/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_test.rs new file mode 100644 index 000000000..49323aee0 --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/paxos_test/runtime_shutdown_test.rs @@ -0,0 +1,212 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaHandler}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::IoBackend; +use crowdb_tree_ffi::{PageStore, SyncMode, TreeRuntime, TreeRuntimeOptions}; + +#[path = "runtime_shutdown_rpc_test.rs"] +mod rpc; + +#[path = "runtime_shutdown_owner_test.rs"] +mod owner; + +const DEADLINE: Duration = Duration::from_secs(2); + +async fn group(role: PxLocalReplicaRole) -> PxGroup { + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + frame_bytes: 4096, + max_inline_value: 64, + sync_mode: SyncMode::Skip, + ..Default::default() + }) + .unwrap(); + let replay = replay_group(&Arc::new(IoBackend::File), &[], 1).await.unwrap(); + let replica = PxLocalReplica::restore_from_replay_with_engine(1, role, &replay, Box::new(engine)) + .await + .unwrap(); + PxGroup::new(1, replica) +} + +fn entry(slot: u64) -> PxLogEntry { + let mut payload = vec![1, 0, 0]; + payload.extend_from_slice(&8u32.to_le_bytes()); + payload.extend_from_slice(&slot.to_le_bytes()); + payload.extend_from_slice(&8u32.to_le_bytes()); + payload.extend_from_slice(&slot.to_le_bytes()); + PxLogEntry { + slot, + term: 0, + ballot: PxBallot::new(1, 1), + payload: Bytes::from(payload), + } +} + +async fn accept(replica: &PxLocalReplica, slot: u64) { + assert!(matches!( + replica.on_accept(&entry(slot)).await, + PxAcceptReply::Accepted { .. } + )); +} + +async fn assert_applied(replica: &PxLocalReplica) { + assert_eq!(replica.contiguous_applied(), 2); + for slot in 1u64..=2 { + assert_eq!( + replica.learner.engine_get(&slot.to_le_bytes()).await, + Some((slot, slot.to_le_bytes().to_vec())) + ); + } + assert_eq!(replica.learner.engine().resume_from_slot(), 2); +} + +#[tokio::test(flavor = "current_thread")] +async fn leader_shutdown_repairs_accepted_gap_before_cancelling_owned_tasks() { + let group = Arc::new(group(PxLocalReplicaRole::Leader).await); + accept(group.local_replica(), 1).await; + accept(group.local_replica(), 2).await; + group.local_replica().spawn_learn_chosen(entry(2), &[]); + group.pause_repair_for_tests(); + let shutdown = tokio::spawn({ + let group = group.clone(); + async move { group.shutdown(DEADLINE).await } + }); + tokio::time::timeout(DEADLINE, async { + while !group.repair_is_paused_for_tests() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(matches!( + group.propose(entry(3).payload.to_vec(), None, None).await, + ProposeResult::Busy + )); + assert!(group + .status() + .messages + .iter() + .any(|message| message.contains("shutdown waiting for accepted Tree work"))); + assert_eq!(group.local_replica().contiguous_applied(), 0); + group.resume_repair_for_tests(); + let report = shutdown.await.unwrap(); + assert!(report.is_clean(), "{:?}", report.errors); + // Real Prepare/Accept must adopt slot one's accepted payload, never replace + // it with an invented no-op merely to unblock slot two's native apply. + assert_applied(group.local_replica()).await; +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelled_synchronous_chosen_wait_keeps_accepted_apply_owned() { + let group = Arc::new(group(PxLocalReplicaRole::Leader).await); + accept(group.local_replica(), 1).await; + accept(group.local_replica(), 2).await; + let apply = tokio::spawn({ + let group = group.clone(); + async move { group.local_replica().learn_chosen(&entry(2), &[]).await } + }); + tokio::time::timeout(DEADLINE, async { + while group.local_replica().last_chosen_slot() != 2 { + tokio::task::yield_now().await; + } + }) + .await + .expect("chosen must be visible while its native apply is awaiting the gap"); + assert_eq!(group.local_replica().contiguous_applied(), 0); + apply.abort(); + assert!(apply.await.unwrap_err().is_cancelled()); + let report = group.shutdown(DEADLINE).await; + assert!(report.is_clean(), "{:?}", report.errors); + assert_applied(group.local_replica()).await; +} + +#[tokio::test(flavor = "current_thread")] +async fn follower_store_timeout_keeps_gap_repair_alive_and_retry_drains() { + let store = PxKvStore::new(1, "127.0.0.1:0".parse().unwrap()); + store.add_group_without_election(group(PxLocalReplicaRole::Follower).await); + let group = store.get_group(1).unwrap(); + let replica = group.local_replica(); + accept(replica, 2).await; + let reply = ReplicaHandler::on_heartbeat( + replica, + HeartbeatRequestPayload { + term: 0, + leader_id: 2, + prev_log_slot: 0, + prev_log_term: 0, + committed_safe_slot: 2, + lease_grant_until_ms_mono: 0, + t_send_ms_mono: 0, + gc_before_slot: 0, + gc_membership_epoch: 0, + }, + 1, + ) + .await + .unwrap(); + assert!(reply.success); + let first = store.shutdown(Duration::from_millis(20)).await; + assert!(!first.is_clean()); + assert!(first + .errors + .iter() + .any(|error| error.contains("accepted Tree work did not drain"))); + assert_eq!(replica.contiguous_applied(), 0); + assert_eq!(replica.learner.engine().resume_from_slot(), 0); + assert!(group + .status() + .messages + .iter() + .any(|message| message.contains("shutdown waiting"))); + accept(replica, 1).await; + // No new heartbeat or new apply loop: accepted gap repair must wake the + // same retained loop after the first bounded shutdown attempt returned. + tokio::time::timeout(DEADLINE, replica.await_apply_fence(2)) + .await + .unwrap(); + let retried = store.shutdown(DEADLINE).await; + assert!(retried.is_clean(), "{:?}", retried.errors); + assert_applied(replica).await; +} + +#[tokio::test(flavor = "current_thread")] +async fn closing_runtime_proposal_window_wakes_queued_callers_without_panic() { + let mut group = group(PxLocalReplicaRole::Leader).await; + group.set_inflight_config(1, false); + let group = Arc::new(group); + let permit = group.inflight_semaphore().acquire().await.unwrap(); + let proposal = tokio::spawn({ + let group = group.clone(); + async move { group.propose(entry(1).payload.to_vec(), None, None).await } + }); + tokio::task::yield_now().await; + let shutdown = tokio::spawn({ + let group = group.clone(); + async move { group.shutdown(DEADLINE).await } + }); + assert!(matches!( + tokio::time::timeout(DEADLINE, proposal).await.unwrap().unwrap(), + ProposeResult::Busy + )); + drop(permit); + let report = shutdown.await.unwrap(); + assert!(report.is_clean(), "{:?}", report.errors); +} diff --git a/lib/crowdb-kv/tests/replay_compaction_memory_test.rs b/lib/crowdb-paxos-kv/tests/replay_compaction_memory_test.rs similarity index 93% rename from lib/crowdb-kv/tests/replay_compaction_memory_test.rs rename to lib/crowdb-paxos-kv/tests/replay_compaction_memory_test.rs index 55caf9424..554f54ac5 100644 --- a/lib/crowdb-kv/tests/replay_compaction_memory_test.rs +++ b/lib/crowdb-paxos-kv/tests/replay_compaction_memory_test.rs @@ -7,10 +7,10 @@ use std::io::Write; use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::IoBackend; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::IoBackend; fn peak_rss_kib() -> u64 { std::fs::read_to_string("/proc/self/status") diff --git a/lib/crowdb-kv/tests/replay_memory_test.rs b/lib/crowdb-paxos-kv/tests/replay_memory_test.rs similarity index 93% rename from lib/crowdb-kv/tests/replay_memory_test.rs rename to lib/crowdb-paxos-kv/tests/replay_memory_test.rs index 010ccadde..93ffda1e6 100644 --- a/lib/crowdb-kv/tests/replay_memory_test.rs +++ b/lib/crowdb-paxos-kv/tests/replay_memory_test.rs @@ -7,10 +7,10 @@ use std::io::Write; use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::IoBackend; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::IoBackend; fn peak_rss_kib() -> u64 { std::fs::read_to_string("/proc/self/status") diff --git a/lib/crowdb-kv/tests/replica_test.rs b/lib/crowdb-paxos-kv/tests/replica_test.rs similarity index 100% rename from lib/crowdb-kv/tests/replica_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test.rs diff --git a/lib/crowdb-kv/tests/replica_test/concurrent_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/concurrent_test.rs similarity index 97% rename from lib/crowdb-kv/tests/replica_test/concurrent_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/concurrent_test.rs index 39b04a66d..b7b0e43df 100644 --- a/lib/crowdb-kv/tests/replica_test/concurrent_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/concurrent_test.rs @@ -13,13 +13,13 @@ use std::sync::Arc; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry}; fn make_entry(slot: u64, term: u64, value: &[u8]) -> PxLogEntry { let mut payload = Vec::new(); - payload.push(1); // op = PUT - payload.push(0); // flags + payload.extend_from_slice(&1_u16.to_le_bytes()); + payload.push(0); // Put let key = b"k"; let key_len = u32::try_from(key.len()).expect("key len"); payload.extend_from_slice(&key_len.to_le_bytes()); diff --git a/lib/crowdb-kv/tests/replica_test/dedup_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/dedup_test.rs similarity index 93% rename from lib/crowdb-kv/tests/replica_test/dedup_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/dedup_test.rs index 532f2ff33..18850de36 100644 --- a/lib/crowdb-kv/tests/replica_test/dedup_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/dedup_test.rs @@ -9,8 +9,8 @@ //! `learn_chosen` and consulted via `learner.dedup_lookup` at the replica level. use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::paxos::roles::{DedupTag, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::paxos::roles::{DedupTag, PxBallot, PxLogEntry}; fn tag(client_id: u64, seq: u64) -> [DedupTag; 1] { [DedupTag { client_id, seq }] @@ -158,7 +158,7 @@ async fn dedup_does_not_false_positive_on_out_of_order_higher_seq() { // choice — e.g. seq=105's proposal happened to win its quorum round // before seq=100's did). let entry_b = write_entry(2, b"b", b"vb"); // slot 2, seq 105 - replica.learn_chosen(&entry_b, &tag(77, 105)).await; + replica.spawn_learn_chosen(entry_b, &tag(77, 105)); // BUG (pre-fix): seq=100 has never been recorded, but the old // single-entry "latest wins" dedup treats 100 <= 105 as a hit and @@ -176,6 +176,15 @@ async fn dedup_does_not_false_positive_on_out_of_order_higher_seq() { produce a false-positive hit against its slot" ); + // Repair the missing slot before awaiting contiguous native application. + let fill = PxLogEntry { + slot: 1, + ballot: PxBallot::new(0, 1), + term: 1, + payload: Bytes::new(), + }; + replica.learn_chosen(&fill, &[]).await; + replica.learner.await_applied(2).await; // Now seq=100 (key "a") is actually proposed and learned at slot 3. let entry_a = write_entry(3, b"a", b"va"); replica.learn_chosen(&entry_a, &tag(77, 100)).await; diff --git a/lib/crowdb-kv/tests/replica_test/election_metrics_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/election_metrics_test.rs similarity index 93% rename from lib/crowdb-kv/tests/replica_test/election_metrics_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/election_metrics_test.rs index c48beff43..8757a2322 100644 --- a/lib/crowdb-kv/tests/replica_test/election_metrics_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/election_metrics_test.rs @@ -3,7 +3,7 @@ //! Step 11: `ElectionMetrics` counter wiring on `PxLocalReplica`. -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; #[test] fn election_count_bumps_on_become_candidate() { diff --git a/lib/crowdb-kv/tests/replica_test/election_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/election_test.rs similarity index 93% rename from lib/crowdb-kv/tests/replica_test/election_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/election_test.rs index 77435ad15..c4055d4f5 100644 --- a/lib/crowdb-kv/tests/replica_test/election_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/election_test.rs @@ -4,15 +4,15 @@ //! Step 12a: election-related unit tests. //! //! These cover gaps not already exercised by inline `#[cfg(test)]` tests -//! in `crowdb_kv/src/cluster/election.rs`. Inline migration is left for a +//! in `crowdb_paxos_kv/src/cluster/election.rs`. Inline migration is left for a //! future pass per the workflow rule "migrate existing inline tests //! when you next touch the file". -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::replica::{ReplicaHandler, VoteRequestPayload}; -use crowdb_kv::kv::CrowdbTreeEngine; -use crowdb_kv::paxos::learner::PxLearner; -use crowdb_kv::paxos::roles::{Learner, PxAcceptReply, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::replica::{ReplicaHandler, VoteRequestPayload}; +use crowdb_paxos_kv::kv::CrowdbTreeEngine; +use crowdb_paxos_kv::paxos::learner::PxLearner; +use crowdb_paxos_kv::paxos::roles::{Learner, PxAcceptReply, PxBallot, PxLogEntry}; /// A `NoOp` entry should advance the chosen / applied /// watermarks without inserting any key into the store. (Bulk Phase-1 diff --git a/lib/crowdb-kv/tests/replica_test/lease_read_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/lease_read_test.rs similarity index 94% rename from lib/crowdb-kv/tests/replica_test/lease_read_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/lease_read_test.rs index ce426c193..dbd37583c 100644 --- a/lib/crowdb-kv/tests/replica_test/lease_read_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/lease_read_test.rs @@ -7,7 +7,7 @@ use std::time::{Duration, Instant}; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; #[test] fn lease_read_valid_requires_leader_and_unexpired_lease() { diff --git a/lib/crowdb-kv/tests/replica_test/op_correctness_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/op_correctness_test.rs similarity index 99% rename from lib/crowdb-kv/tests/replica_test/op_correctness_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/op_correctness_test.rs index 474fe1e32..a73660d0e 100644 --- a/lib/crowdb-kv/tests/replica_test/op_correctness_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/op_correctness_test.rs @@ -9,8 +9,8 @@ //! the right KV state. use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; fn encode_put(key: &[u8], value: &[u8]) -> Vec<u8> { let mut buf = Vec::new(); diff --git a/lib/crowdb-kv/tests/replica_test/persistence_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/persistence_test.rs similarity index 97% rename from lib/crowdb-kv/tests/replica_test/persistence_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/persistence_test.rs index 1c111b145..aacaecc8d 100644 --- a/lib/crowdb-kv/tests/replica_test/persistence_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/persistence_test.rs @@ -18,11 +18,11 @@ use std::path::PathBuf; use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::WalConfig; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::WalConfig; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; const GROUP: u64 = 1; const REPLICA_ID: u64 = 7; @@ -132,7 +132,10 @@ async fn wal_backed_replica_reloads_committed_kv_after_restart() { let entry = write_entry(slot, key, value); let reply = replica.on_accept(&entry).await; assert!( - matches!(reply, crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. }), + matches!( + reply, + crowdb_paxos_kv::paxos::roles::PxAcceptReply::Accepted { .. } + ), "slot {slot} should be accepted: {reply:?}" ); replica.learn_chosen(&entry, &[]).await; diff --git a/lib/crowdb-kv/tests/replica_test/prepare_accept_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/prepare_accept_test.rs similarity index 96% rename from lib/crowdb-kv/tests/replica_test/prepare_accept_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/prepare_accept_test.rs index 864b5b879..91090dc16 100644 --- a/lib/crowdb-kv/tests/replica_test/prepare_accept_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/prepare_accept_test.rs @@ -9,15 +9,15 @@ //! `accepted_at`, `promised_at`). use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply}; fn write_entry(slot: u64, ballot: PxBallot, term: u64, payload: &[u8]) -> PxLogEntry { PxLogEntry { slot, ballot, term, - payload: Bytes::copy_from_slice(payload), + payload: Bytes::from(crate::test_util::encode_put_payload(b"key", payload)), } } diff --git a/lib/crowdb-kv/tests/replica_test/replay_ordering_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/replay_ordering_test.rs similarity index 90% rename from lib/crowdb-kv/tests/replica_test/replay_ordering_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/replay_ordering_test.rs index 8069eb15b..7188ba67b 100644 --- a/lib/crowdb-kv/tests/replica_test/replay_ordering_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/replay_ordering_test.rs @@ -21,12 +21,12 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::WalConfig; -use crowdb_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::WalConfig; +use crowdb_paxos_kv::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; const GROUP: u64 = 1; const REPLICA_ID: u64 = 7; @@ -143,19 +143,17 @@ async fn restore_stops_at_hole_below_watermark() { } assert_eq!(replica.contiguous_applied(), 3); - // Now accept slot 5 (skipping 4) and learn it. + // Persist slot 5 across a gap. Native application requires slot 4, + // so the restart must leave this suffix available for gap repair. let entry5 = write_entry(5, b"k", b"5"); let _ = replica.on_accept(&entry5).await; - replica.learn_chosen(&entry5, &[]).await; wal.seal_all().await.expect("seal"); } let restored = replay_and_restore(&wal_dir).await; - // Replay applies slots 1-3 contiguously, then slot 5 is out-of-order - // (slot 4 is missing). contiguous_applied stops at 3, but the engine - // has slot 5's value (highest-slot-wins). + // Replay applies only slots 1-3; the sparse suffix remains accepted. assert_eq!( restored.contiguous_applied(), 3, @@ -163,12 +161,22 @@ async fn restore_stops_at_hole_below_watermark() { ); assert_eq!( restored.learner.engine_get(b"k").await.map(|(_, v)| v), - Some(b"5".to_vec()), - "k = slot-5 value (highest-slot-wins, out-of-order)" + Some(b"3".to_vec()), + "slot 5 remains unapplied until its missing predecessor is repaired" ); // Slot 5 is in the acceptor but not applied. assert!(restored.accepted_at(5).await.is_some(), "slot 5 in acceptor"); assert!(restored.accepted_at(4).await.is_none(), "slot 4 not accepted"); + let gap = write_entry(4, b"k", b"4"); + restored.on_accept(&gap).await; + restored.learn_chosen(&gap, &[]).await; + let suffix = restored.accepted_at(5).await.unwrap(); + restored.learn_chosen(&suffix, &[]).await; + assert_eq!(restored.contiguous_applied(), 5); + assert_eq!( + restored.learner.engine_get(b"k").await.map(|(_, v)| v), + Some(b"5".to_vec()) + ); } // ── Out-of-order WAL records ────────────────────────────────── diff --git a/lib/crowdb-kv/tests/replica_test/snapshot_test.rs b/lib/crowdb-paxos-kv/tests/replica_test/snapshot_test.rs similarity index 94% rename from lib/crowdb-kv/tests/replica_test/snapshot_test.rs rename to lib/crowdb-paxos-kv/tests/replica_test/snapshot_test.rs index 72ef2c76f..d328e062f 100644 --- a/lib/crowdb-kv/tests/replica_test/snapshot_test.rs +++ b/lib/crowdb-paxos-kv/tests/replica_test/snapshot_test.rs @@ -10,8 +10,8 @@ use crate::test_util::iter_all_dyn; use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; #[allow(clippy::cast_possible_truncation)] fn encode_put(key: &[u8], value: &[u8]) -> Vec<u8> { @@ -71,7 +71,7 @@ async fn clear_wipes_kv_state_but_preserves_accepted_log() { ); // Clear the engine (snapshot-install reset). - replica.learner.engine().clear(); + replica.learner.engine().clear().unwrap(); assert_eq!(replica.learner.engine_get(b"k1").await, None); assert_eq!(replica.learner.engine_get(b"k2").await, None); assert!(iter_all_dyn(replica.learner.engine()).is_empty()); @@ -106,7 +106,7 @@ async fn re_apply_after_clear_restores_kv_state() { ); // Clear and re-apply from the accepted log. - replica.learner.engine().clear(); + replica.learner.engine().clear().unwrap(); for slot in 1..=3u64 { let entry = replica.accepted_at(slot).await.expect("accepted entry"); replica.learn_chosen(&entry, &[]).await; @@ -137,7 +137,7 @@ async fn re_apply_after_clear_restores_watermarks() { assert_eq!(replica.contiguous_applied(), 3); // Clear and re-apply. - replica.learner.engine().clear(); + replica.learner.engine().clear().unwrap(); for slot in 1..=3u64 { let entry = replica.accepted_at(slot).await.expect("accepted entry"); replica.learn_chosen(&entry, &[]).await; @@ -164,6 +164,6 @@ async fn clear_drops_all_keys_including_tombstones() { assert_eq!(iter_all_dyn(replica.learner.engine()).len(), 1); // Clear removes everything. - replica.learner.engine().clear(); + replica.learner.engine().clear().unwrap(); assert!(iter_all_dyn(replica.learner.engine()).is_empty()); } diff --git a/lib/crowdb-kv/tests/rpc_migration_test.rs b/lib/crowdb-paxos-kv/tests/rpc_migration_test.rs similarity index 90% rename from lib/crowdb-kv/tests/rpc_migration_test.rs rename to lib/crowdb-paxos-kv/tests/rpc_migration_test.rs index 171d2fb33..e85bed6d4 100644 --- a/lib/crowdb-kv/tests/rpc_migration_test.rs +++ b/lib/crowdb-paxos-kv/tests/rpc_migration_test.rs @@ -8,13 +8,13 @@ mod common; use std::sync::Arc; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::replica::PxReplicaError; -use crowdb_kv::cluster::replica::ReplicaClient; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::paxos::roles::{PxBallot, PxPrepareReply}; -use crowdb_kv::rpc::PxRpcTransport; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::replica::PxReplicaError; +use crowdb_paxos_kv::cluster::replica::ReplicaClient; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxPrepareReply}; +use crowdb_paxos_kv::rpc::PxRpcTransport; use crowdb_rpc_ffi::RpcError; use common::logging::init_test_subscriber; @@ -158,7 +158,7 @@ async fn crowdb_rpc_prepare_accept_roundtrip() { } // Send an Accept for slot 1. - let entry = crowdb_kv::paxos::roles::PxLogEntry { + let entry = crowdb_paxos_kv::paxos::roles::PxLogEntry { slot: 1, ballot, term: 0, @@ -171,7 +171,7 @@ async fn crowdb_rpc_prepare_accept_roundtrip() { accept_result.err() ); match accept_result.unwrap() { - crowdb_kv::paxos::roles::PxAcceptReply::Accepted { slot, .. } => { + crowdb_paxos_kv::paxos::roles::PxAcceptReply::Accepted { slot, .. } => { assert_eq!(slot, 1, "accepted slot should be 1"); } other => panic!("expected Accepted, got {other:?}"), @@ -201,11 +201,12 @@ async fn crowdb_rpc_chosen_notification_fire_and_forget() { // First, Prepare + Accept slot 1 so the follower has a value. let ballot = PxBallot::new(1, 1); - let entry = crowdb_kv::paxos::roles::PxLogEntry { + let entry = crowdb_paxos_kv::paxos::roles::PxLogEntry { slot: 1, ballot, term: 0, - payload: bytes::Bytes::from_static(b"chosen-test"), + // One KV Put: count, kind, key length/key, value length/value. + payload: bytes::Bytes::from_static(b"\x01\x00\x00\x03\x00\x00\x00key\x05\x00\x00\x00value"), }; follower_remote .send_prepare(1, ballot, 0, 1, 0) @@ -242,6 +243,10 @@ async fn crowdb_rpc_chosen_notification_fire_and_forget() { tokio::time::sleep(std::time::Duration::from_millis(10)).await; } assert_eq!(follower_replica.learner.contiguous_chosen(), 1); + assert_eq!( + follower_replica.learner.engine_get(b"key").await, + Some((1, b"value".to_vec())) + ); cluster.shutdown().await; } @@ -273,7 +278,7 @@ async fn crowdb_rpc_fetch_gap() { // Prepare + Accept slot 1 on both leader (locally) and follower (via RPC). let ballot = PxBallot::new(1, 1); - let entry = crowdb_kv::paxos::roles::PxLogEntry { + let entry = crowdb_paxos_kv::paxos::roles::PxLogEntry { slot: 1, ballot, term: 0, @@ -292,7 +297,7 @@ async fn crowdb_rpc_fetch_gap() { assert!( matches!( accept_reply, - crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. } + crowdb_paxos_kv::paxos::roles::PxAcceptReply::Accepted { .. } ), "leader accept should succeed: {accept_reply:?}" ); @@ -309,7 +314,7 @@ async fn crowdb_rpc_fetch_gap() { .expect("follower accept"); assert!(matches!( accepted, - crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. } + crowdb_paxos_kv::paxos::roles::PxAcceptReply::Accepted { .. } )); // Both voters have accepted. Publish that quorum decision before serving // the chosen value; an Accepted response alone is not a learner proof. diff --git a/lib/crowdb-paxos-kv/tests/rpc_notification_test.rs b/lib/crowdb-paxos-kv/tests/rpc_notification_test.rs new file mode 100644 index 000000000..cc0ade0a4 --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/rpc_notification_test.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::time::Duration; + +use crowdb_paxos_kv::rpc::{PxRpcTransport, ReplicaDestination}; +use crowdb_protocol::fb::FBMsgType; +use crowdb_rpc_ffi::{flush_cpp_global_metrics, RpcServer}; + +/// A receiver deliberately sends no replies, as required for chosen notices. +/// Both notification variants must arrive without later timing out in the sender. +#[tokio::test] +async fn chosen_notifications_do_not_create_reply_timeouts() { + let server = RpcServer::with_engines(None, 1, 1); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + for kind in [ + FBMsgType::EChosenNotification, + FBMsgType::EBatchChosenNotification, + ] { + let tx = tx.clone(); + let msg_type = u16::try_from(kind.0).unwrap(); + server.register_handler(msg_type, move |_| { + tx.send(msg_type).unwrap(); + }); + } + server.listen("127.0.0.1", 0).unwrap(); + server.start(); + let endpoint = format!("127.0.0.1:{}", server.port()); + let destination = ReplicaDestination { + endpoint: &endpoint, + replica_id: 2, + }; + let transport = PxRpcTransport::new(); + for slot in 1..=32 { + transport.send_chosen(destination, 1, slot, 1, 1, 1).unwrap(); + transport + .send_batch_chosen(destination, 1, slot, slot + 1, 1, 1, 1) + .unwrap(); + } + let mut received = [0; 2]; + for _ in 0..64 { + let kind = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("notification delivery timeout") + .expect("notification receiver closed"); + let is_batch = kind == u16::try_from(FBMsgType::EBatchChosenNotification.0).unwrap(); + received[usize::from(is_batch)] += 1; + } + assert_eq!(received, [32, 32]); + + // The consensus client's reply deadline is 3s and its reaper scans every + // 500ms. Keep it alive beyond both so the old request-response path fails. + tokio::time::sleep(Duration::from_secs(4)).await; + let metrics = + flush_cpp_global_metrics(4.0, "notification-test", "cpp-rpc", 60, 10, 10).unwrap_or_default(); + let reaped = metrics.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + (fields.next()? == "rpc.request.reaped.c").then(|| fields.last().unwrap().parse::<u64>().unwrap()) + }); + server.stop(); + assert_eq!( + reaped.unwrap_or(0), + 0, + "one-way notices must not be reaped:\n{metrics}" + ); +} diff --git a/lib/crowdb-kv/tests/slot_test.rs b/lib/crowdb-paxos-kv/tests/slot_test.rs similarity index 100% rename from lib/crowdb-kv/tests/slot_test.rs rename to lib/crowdb-paxos-kv/tests/slot_test.rs diff --git a/lib/crowdb-kv/tests/slot_test/concurrent_test.rs b/lib/crowdb-paxos-kv/tests/slot_test/concurrent_test.rs similarity index 99% rename from lib/crowdb-kv/tests/slot_test/concurrent_test.rs rename to lib/crowdb-paxos-kv/tests/slot_test/concurrent_test.rs index 53103367f..fc85d1f4e 100644 --- a/lib/crowdb-kv/tests/slot_test/concurrent_test.rs +++ b/lib/crowdb-paxos-kv/tests/slot_test/concurrent_test.rs @@ -13,7 +13,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Barrier}; use std::thread; -use crowdb_kv::paxos::slot_list::PxSlotList; +use crowdb_paxos_kv::paxos::slot_list::PxSlotList; const SLOT_CHUNK_SIZE: usize = 1024; diff --git a/lib/crowdb-kv/tests/slot_test/reclaim_test.rs b/lib/crowdb-paxos-kv/tests/slot_test/reclaim_test.rs similarity index 98% rename from lib/crowdb-kv/tests/slot_test/reclaim_test.rs rename to lib/crowdb-paxos-kv/tests/slot_test/reclaim_test.rs index cc26e305a..d5f3b82f0 100644 --- a/lib/crowdb-kv/tests/slot_test/reclaim_test.rs +++ b/lib/crowdb-paxos-kv/tests/slot_test/reclaim_test.rs @@ -4,7 +4,7 @@ //! Reclamation watermark tests: long-lived read guards prevent chunk reclamation //! until the guard is dropped. -use crowdb_kv::paxos::slot_list::PxSlotList; +use crowdb_paxos_kv::paxos::slot_list::PxSlotList; const SLOT_CHUNK_SIZE: usize = 1024; diff --git a/lib/crowdb-kv/tests/slot_test/slot_list_test.rs b/lib/crowdb-paxos-kv/tests/slot_test/slot_list_test.rs similarity index 98% rename from lib/crowdb-kv/tests/slot_test/slot_list_test.rs rename to lib/crowdb-paxos-kv/tests/slot_test/slot_list_test.rs index 9692e8beb..b0666435f 100644 --- a/lib/crowdb-kv/tests/slot_test/slot_list_test.rs +++ b/lib/crowdb-paxos-kv/tests/slot_test/slot_list_test.rs @@ -7,9 +7,9 @@ //! //! Covers general-value tests and `PxSlotNode` integration scenarios. -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::paxos::slot_list::PxSlotList; -use crowdb_kv::paxos::slot_node::PxSlotNode; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::paxos::slot_list::PxSlotList; +use crowdb_paxos_kv::paxos::slot_node::PxSlotNode; const SLOT_CHUNK_SIZE: usize = 1024; use std::ptr::null_mut; diff --git a/lib/crowdb-kv/tests/store_test.rs b/lib/crowdb-paxos-kv/tests/store_test.rs similarity index 86% rename from lib/crowdb-kv/tests/store_test.rs rename to lib/crowdb-paxos-kv/tests/store_test.rs index 784391953..a748351c8 100644 --- a/lib/crowdb-kv/tests/store_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test.rs @@ -8,9 +8,9 @@ //! add/remove of groups, single-node KV/read modes, the topology `status` //! snapshot, the `health` report, and graceful `shutdown` cascade. //! -//! These exercise the `crowdb_kv` library `PxKvStore` directly (the embedded crowdb-rpc -//! server, no HTTP). Tests that boot the `crowdb-kv-server` binary / HTTP -//! management API live under `crowdb-kv-server/tests`. +//! These exercise the `crowdb_paxos_kv` library `PxKvStore` directly (the embedded crowdb-rpc +//! server, no HTTP). Tests that boot the `crowdb-paxos-kv-server` binary / HTTP +//! management API live under `crowdb-paxos-kv-server/tests`. mod common; diff --git a/lib/crowdb-kv/tests/store_test/apply_fence_test.rs b/lib/crowdb-paxos-kv/tests/store_test/apply_fence_test.rs similarity index 96% rename from lib/crowdb-kv/tests/store_test/apply_fence_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/apply_fence_test.rs index ac99cf337..430b3300e 100644 --- a/lib/crowdb-kv/tests/store_test/apply_fence_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/apply_fence_test.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use std::time::Duration; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; use tokio::sync::Notify; fn leader_group(group_id: u64, my_id: u64) -> PxGroup { diff --git a/lib/crowdb-kv/tests/store_test/bounded_scan_test.rs b/lib/crowdb-paxos-kv/tests/store_test/bounded_scan_test.rs similarity index 91% rename from lib/crowdb-kv/tests/store_test/bounded_scan_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/bounded_scan_test.rs index 365800520..936770d96 100644 --- a/lib/crowdb-kv/tests/store_test/bounded_scan_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/bounded_scan_test.rs @@ -3,11 +3,11 @@ //! Fixed-cutoff current-version scan coverage through `PxKvStore`. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::kv::ScanDirection; -use crowdb_kv::rpc::ReadMode; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::ScanDirection; +use crowdb_paxos_kv::rpc::ReadMode; fn store() -> PxKvStore { let store = PxKvStore::new(0, "127.0.0.1:0".parse().unwrap()); @@ -23,7 +23,7 @@ async fn bounded_scan( start_after: &[u8], limit: u32, cutoff: u64, -) -> crowdb_kv::rpc::KvScanResponse { +) -> crowdb_paxos_kv::rpc::KvScanResponse { store .kv_scan( 1, diff --git a/lib/crowdb-kv/tests/store_test/group_operations_test.rs b/lib/crowdb-paxos-kv/tests/store_test/group_operations_test.rs similarity index 93% rename from lib/crowdb-kv/tests/store_test/group_operations_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/group_operations_test.rs index a9318be34..6cd3d7473 100644 --- a/lib/crowdb-kv/tests/store_test/group_operations_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/group_operations_test.rs @@ -2,12 +2,12 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::group_operations::{ +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::group_operations::{ KvGroupMutation, KvGroupOperationError, KvGroupScanRequest, KvReadConsistency, KvRequestIdentity, }; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; fn group_zero_store() -> PxKvStore { let store = PxKvStore::new(0, "127.0.0.1:0".parse().unwrap()); @@ -57,7 +57,7 @@ async fn rpc_and_internal_operations_share_applied_reads_writes_and_scans() { deadline_ms: 0, bounded: true, requested_scan_cutoff: 0, - direction: crowdb_kv::kv::ScanDirection::Forward, + direction: crowdb_paxos_kv::kv::ScanDirection::Forward, }) .await .unwrap(); diff --git a/lib/crowdb-kv/tests/store_test/health_test.rs b/lib/crowdb-paxos-kv/tests/store_test/health_test.rs similarity index 85% rename from lib/crowdb-kv/tests/store_test/health_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/health_test.rs index 60ca30ee3..4e300ae64 100644 --- a/lib/crowdb-kv/tests/store_test/health_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/health_test.rs @@ -9,11 +9,11 @@ use std::sync::Arc; use std::time::Duration; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::cluster::status::StatusLevel; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::cluster::status::StatusLevel; fn make_store_single_replica() -> Arc<PxKvStore> { let store = Arc::new(PxKvStore::new(1, "127.0.0.1:0".parse().unwrap())); diff --git a/lib/crowdb-kv/tests/store_test/journal_recovery_test.rs b/lib/crowdb-paxos-kv/tests/store_test/journal_recovery_test.rs similarity index 70% rename from lib/crowdb-kv/tests/store_test/journal_recovery_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/journal_recovery_test.rs index 61cf2b043..d4d910b72 100644 --- a/lib/crowdb-kv/tests/store_test/journal_recovery_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/journal_recovery_test.rs @@ -2,19 +2,25 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::kv::{Batch, BatchOp, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, Op}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::rpc::{KvErrorCode, ReadMode}; -use crowdb_kv::wal::index::SegmentIndex; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::ReplayResult; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{Batch, BatchOp, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, Op}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::rpc::{KvErrorCode, ReadMode}; +use crowdb_paxos_kv::wal::index::SegmentIndex; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::ReplayResult; +use crowdb_tree_ffi::PageStore; +use std::sync::Arc; #[tokio::test] async fn restored_journal_rejects_missing_durable_history_before_maintenance() { - let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).unwrap(); + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig { + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + ..Default::default() + }) + .unwrap(); for slot in 1..=3 { engine .apply( @@ -30,6 +36,10 @@ async fn restored_journal_rejects_missing_durable_history_before_maintenance() { .unwrap(); } engine.handle().flush().unwrap(); + assert_eq!(engine.resume_from_slot(), 0); + let checkpoint = engine.persist_snapshot_async().unwrap().await.unwrap(); + assert!(checkpoint.status.is_ok(), "{:?}", checkpoint.status); + assert!(checkpoint.durable_committed); assert_eq!(engine.resume_from_slot(), 3); // Earlier WAL segments were reclaimed after the tree became durable. // The surviving tail still seeds the applied frontier on restart. diff --git a/lib/crowdb-kv/tests/store_test/kv_correctness_test.rs b/lib/crowdb-paxos-kv/tests/store_test/kv_correctness_test.rs similarity index 96% rename from lib/crowdb-kv/tests/store_test/kv_correctness_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/kv_correctness_test.rs index 5a646f146..9745a9bd4 100644 --- a/lib/crowdb-kv/tests/store_test/kv_correctness_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/kv_correctness_test.rs @@ -9,10 +9,10 @@ //! mixed ops across slots. Also covers edge-case keys. use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::rpc::KvBatchItem; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::rpc::KvBatchItem; fn leader_store() -> PxKvStore { let store = PxKvStore::new(0, "127.0.0.1:0".parse().unwrap()); diff --git a/lib/crowdb-kv/tests/store_test/multi_group_test.rs b/lib/crowdb-paxos-kv/tests/store_test/multi_group_test.rs similarity index 94% rename from lib/crowdb-kv/tests/store_test/multi_group_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/multi_group_test.rs index 15ae6feca..6c4d996fc 100644 --- a/lib/crowdb-kv/tests/store_test/multi_group_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/multi_group_test.rs @@ -3,10 +3,10 @@ //! Multi-group KV store routing and missing-group error tests. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Barrier}; @@ -70,7 +70,7 @@ async fn missing_group_returns_error() { assert!(!resp.ok); assert_eq!( resp.error_code, - crowdb_kv::rpc::KvErrorCode::KvErrorNotLeader as i32 + crowdb_paxos_kv::rpc::KvErrorCode::KvErrorNotLeader as i32 ); assert!( resp.error.contains("no kv group"), diff --git a/lib/crowdb-kv/tests/store_test/multi_node_multi_group_test.rs b/lib/crowdb-paxos-kv/tests/store_test/multi_node_multi_group_test.rs similarity index 95% rename from lib/crowdb-kv/tests/store_test/multi_node_multi_group_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/multi_node_multi_group_test.rs index 09088963f..7d0e9e075 100644 --- a/lib/crowdb-kv/tests/store_test/multi_node_multi_group_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/multi_node_multi_group_test.rs @@ -9,11 +9,11 @@ use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::{KvServer, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::common::config::PxElectionConfig; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::{PaxosKvServer, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster}; use crate::common::test_client::TestKvClient; diff --git a/lib/crowdb-kv/tests/store_test/node_test.rs b/lib/crowdb-paxos-kv/tests/store_test/node_test.rs similarity index 95% rename from lib/crowdb-kv/tests/store_test/node_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/node_test.rs index 4e441e108..64dd67a52 100644 --- a/lib/crowdb-kv/tests/store_test/node_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/node_test.rs @@ -2,11 +2,13 @@ // Licensed under the Apache License, Version 2.0. use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::rpc::KvBatchItem; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{ + PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica, +}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::rpc::KvBatchItem; use std::sync::Arc; fn sample_group(my_id: u64, leader_id: u64, leader_endpoint: &str) -> PxGroup { @@ -112,7 +114,7 @@ async fn classic_prepare_and_accept_track_state() { let prepare_reply = node.on_prepare(5, ballot, 0).await; assert!(matches!( prepare_reply, - crowdb_kv::paxos::roles::PxPrepareReply::Promised { .. } + crowdb_paxos_kv::paxos::roles::PxPrepareReply::Promised { .. } )); let entry = PxLogEntry { @@ -125,7 +127,7 @@ async fn classic_prepare_and_accept_track_state() { let accept_reply = node.on_accept(&entry).await; assert!(matches!( accept_reply, - crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. } + crowdb_paxos_kv::paxos::roles::PxAcceptReply::Accepted { .. } )); let accepted = node.accepted_at(5).await.expect("accepted entry present"); diff --git a/lib/crowdb-kv/tests/store_test/persistence_test.rs b/lib/crowdb-paxos-kv/tests/store_test/persistence_test.rs similarity index 90% rename from lib/crowdb-kv/tests/store_test/persistence_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/persistence_test.rs index 632db690b..9d9a890fa 100644 --- a/lib/crowdb-kv/tests/store_test/persistence_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/persistence_test.rs @@ -12,18 +12,18 @@ //! //! This sits one layer above `replica::persistence`: it proves the store's //! routing + KV API reload correctly on top of the replica's durable state. It -//! uses the embedded library only (no `crowdb-kv-server` HTTP process), so it lives -//! in `crowdb_kv`. +//! uses the embedded library only (no `crowdb-paxos-kv-server` HTTP process), so it lives +//! in `crowdb_paxos_kv`. use std::path::{Path, PathBuf}; use std::sync::Arc; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::WalConfig; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::WalConfig; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; const GROUP: u64 = 1; const REPLICA_ID: u64 = 1; diff --git a/lib/crowdb-kv/tests/store_test/read_metrics_test.rs b/lib/crowdb-paxos-kv/tests/store_test/read_metrics_test.rs similarity index 96% rename from lib/crowdb-kv/tests/store_test/read_metrics_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/read_metrics_test.rs index a33fa71fa..46b87f5c8 100644 --- a/lib/crowdb-kv/tests/store_test/read_metrics_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/read_metrics_test.rs @@ -5,10 +5,10 @@ //! `engine_get` latency, MinSlot-fallback counter, e2e read latency, //! and read-state gauges. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::metrics::MetricsRegistry; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::metrics::MetricsRegistry; use std::sync::{Arc, Mutex}; fn leader_group(group_id: u64, my_id: u64) -> PxGroup { diff --git a/lib/crowdb-kv/tests/store_test/readindex_batch_test.rs b/lib/crowdb-paxos-kv/tests/store_test/readindex_batch_test.rs similarity index 96% rename from lib/crowdb-kv/tests/store_test/readindex_batch_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/readindex_batch_test.rs index 84831a26a..2051d1469 100644 --- a/lib/crowdb-kv/tests/store_test/readindex_batch_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/readindex_batch_test.rs @@ -15,10 +15,10 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; -use crowdb_kv::metrics::MetricsRegistry; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_paxos_kv::metrics::MetricsRegistry; use tokio::sync::oneshot; fn leader_group(group_id: u64, my_id: u64) -> PxGroup { diff --git a/lib/crowdb-kv/tests/store_test/shutdown_test.rs b/lib/crowdb-paxos-kv/tests/store_test/shutdown_test.rs similarity index 88% rename from lib/crowdb-kv/tests/store_test/shutdown_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/shutdown_test.rs index 4e88becd5..ec3c7b9ec 100644 --- a/lib/crowdb-kv/tests/store_test/shutdown_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/shutdown_test.rs @@ -12,10 +12,10 @@ use std::sync::Arc; use std::time::Duration; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_server::KvServer; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::paxos_kv_server::PaxosKvServer; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; fn make_store() -> Arc<PxKvStore> { let store = Arc::new(PxKvStore::new(7, "127.0.0.1:0".parse().unwrap())); diff --git a/lib/crowdb-kv/tests/store_test/shutdown_under_load_test.rs b/lib/crowdb-paxos-kv/tests/store_test/shutdown_under_load_test.rs similarity index 95% rename from lib/crowdb-kv/tests/store_test/shutdown_under_load_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/shutdown_under_load_test.rs index 94b72d665..853bfb726 100644 --- a/lib/crowdb-kv/tests/store_test/shutdown_under_load_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/shutdown_under_load_test.rs @@ -10,10 +10,10 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use bytes::Bytes; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PaxosKvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::rpc::{KvGetRequest, KvSetRequest}; use crate::common::cluster::{start_cluster_no_leader_relaxed as start_cluster_no_leader, TestCluster}; diff --git a/lib/crowdb-kv/tests/store_test/snapshot_api_test.rs b/lib/crowdb-paxos-kv/tests/store_test/snapshot_api_test.rs similarity index 84% rename from lib/crowdb-kv/tests/store_test/snapshot_api_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/snapshot_api_test.rs index 5ada9fe61..338c843ce 100644 --- a/lib/crowdb-kv/tests/store_test/snapshot_api_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/snapshot_api_test.rs @@ -12,19 +12,32 @@ //! - `kv_release_snapshot` drops the handle; subsequent scan fails. //! - Backward-compat: live `kv_scan` is unchanged. -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; -use crowdb_kv::rpc::ReadMode; -use crowdb_kv::wal::io_backend::IoBackend; -use crowdb_kv::wal::replay::replay_group; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::rpc::ReadMode; +use crowdb_paxos_kv::wal::io_backend::IoBackend; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_tree_ffi::{PageStore, TreeRuntime, TreeRuntimeOptions}; use std::path::PathBuf; use std::sync::Arc; async fn crowdb_tree_store() -> PxKvStore { let store = PxKvStore::new(0, "127.0.0.1:0".parse().unwrap()); - let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).expect("open crowdb-tree engine"); + // These Stores represent independent hosts. Keep the native default + // capacities per fixture instead of competing with other tests' hosts. + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).expect("create tree runtime")); + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(runtime), + // An explicit runtime without a backend selects a volatile Tree. + // Retain a reloadable in-memory backend for snapshot coverage. + page_store: Some(Arc::new( + PageStore::open_mem(1).expect("create memory page store"), + )), + ..Default::default() + }) + .expect("open crowdb-tree engine"); // Empty replay (no WAL segments) — gives a fresh replica with the // caller-supplied engine wired in. let backend = Arc::new(IoBackend::mem_block()); @@ -44,6 +57,21 @@ async fn crowdb_tree_store() -> PxKvStore { store } +fn snapshot_failure(store: &PxKvStore, error: &str) -> String { + let group = store.get_group(1).unwrap(); + let replica = group.local_replica(); + let engine = replica + .learner + .engine() + .as_any() + .downcast_ref::<CrowdbTreeEngine>() + .unwrap(); + format!( + "{error}; native resources: {:?}", + engine.handle().resource_usage() + ) +} + #[tokio::test] async fn create_snapshot_returns_handle_and_at_slot() { let store = crowdb_tree_store().await; @@ -108,7 +136,7 @@ async fn snapshot_scan_is_point_in_time_consistent() { 0, false, 0, - crowdb_kv::kv::ScanDirection::Forward, + crowdb_paxos_kv::kv::ScanDirection::Forward, 200, 2000, ) @@ -143,7 +171,7 @@ async fn snapshot_scan_prefix_filter() { let snap = store .kv_create_snapshot(1, ReadMode::Linearizable as i32, 0) .await; - assert!(snap.ok); + assert!(snap.ok, "{}", snapshot_failure(&store, &snap.error)); let handle = snap.snapshot_handle; let resp = store.kv_snapshot_scan(1, handle, b"prefix_", b"", 100).await; @@ -216,11 +244,11 @@ async fn list_snapshots_returns_active_handles() { let s1 = store .kv_create_snapshot(1, ReadMode::Linearizable as i32, 0) .await; - assert!(s1.ok); + assert!(s1.ok, "{}", snapshot_failure(&store, &s1.error)); let s2 = store .kv_create_snapshot(1, ReadMode::Linearizable as i32, 0) .await; - assert!(s2.ok); + assert!(s2.ok, "{}", snapshot_failure(&store, &s2.error)); let list = store.kv_list_snapshots(1).await; assert!(list.ok); diff --git a/lib/crowdb-kv/tests/store_test/status_test.rs b/lib/crowdb-paxos-kv/tests/store_test/status_test.rs similarity index 95% rename from lib/crowdb-kv/tests/store_test/status_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/status_test.rs index 6932bb668..92e4874d6 100644 --- a/lib/crowdb-kv/tests/store_test/status_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/status_test.rs @@ -7,16 +7,16 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::group_election::LeaderElection; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::cluster::px_kv_store::PxKvStore; -use crowdb_kv::cluster::remote_replica::PxRemoteReplica; -use crowdb_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaClient}; -use crowdb_kv::cluster::KvServer; -use crowdb_kv::common::config::PxElectionConfig; -use crowdb_kv::metrics::{MetricPoint, MetricsRegistry}; -use crowdb_kv::rpc::PxRpcTransport; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::cluster::px_kv_store::PxKvStore; +use crowdb_paxos_kv::cluster::remote_replica::PxRemoteReplica; +use crowdb_paxos_kv::cluster::replica::{HeartbeatRequestPayload, ReplicaClient}; +use crowdb_paxos_kv::cluster::PaxosKvServer; +use crowdb_paxos_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::metrics::{MetricPoint, MetricsRegistry}; +use crowdb_paxos_kv::rpc::PxRpcTransport; #[test] fn status_empty_store() { diff --git a/lib/crowdb-kv/tests/store_test/wal_isolation_test.rs b/lib/crowdb-paxos-kv/tests/store_test/wal_isolation_test.rs similarity index 92% rename from lib/crowdb-kv/tests/store_test/wal_isolation_test.rs rename to lib/crowdb-paxos-kv/tests/store_test/wal_isolation_test.rs index 597a2760e..3819965da 100644 --- a/lib/crowdb-kv/tests/store_test/wal_isolation_test.rs +++ b/lib/crowdb-paxos-kv/tests/store_test/wal_isolation_test.rs @@ -13,12 +13,12 @@ use std::path::PathBuf; use std::sync::Arc; -use crowdb_kv::cluster::group::PxGroup; -use crowdb_kv::cluster::kv_store::KvStore; -use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::common::config::WalConfig; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::kv_store::KvStore; +use crowdb_paxos_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::WalConfig; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; const REPLICA_ID: u64 = 1; diff --git a/lib/crowdb-paxos-kv/tests/tree_admission_test.rs b/lib/crowdb-paxos-kv/tests/tree_admission_test.rs new file mode 100644 index 000000000..33105c6cc --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/tree_admission_test.rs @@ -0,0 +1,256 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_paxos_kv::cluster::group::{ProposeResult, PxGroup}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{ + AdmittedApply, ApplyAdmissionError, ApplyAdmissionResult, Batch, KVEngine, KVFuture, ScanDirection, +}; +use crowdb_paxos_kv::paxos::roles::{Learner, PxAcceptReply, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice}; +use tokio::sync::Notify; + +#[derive(Default)] +struct Calls { + busy: AtomicBool, + hold_admission: AtomicBool, + hold_apply: AtomicBool, + fail_noop: AtomicBool, + admitted: AtomicU64, + released: AtomicU64, + admitted_slot: AtomicU64, + fallback_slot: AtomicU64, + admission_entered: Notify, + admission_release: Notify, + apply_entered: Notify, + apply_release: Notify, +} + +struct TestEngine(Arc<Calls>); +struct TestPermit(Arc<Calls>); + +impl Drop for TestPermit { + fn drop(&mut self) { + self.0.released.fetch_add(1, Ordering::Relaxed); + } +} + +impl AdmittedApply for TestPermit { + fn apply(self: Box<Self>, slot: u64) -> KVFuture<Result<(), String>> { + KVFuture::Pending(Box::pin(async move { + self.0.apply_entered.notify_one(); + if self.0.hold_apply.load(Ordering::Acquire) { + self.0.apply_release.notified().await; + } + self.0.admitted_slot.store(slot, Ordering::Release); + Ok(()) + })) + } +} + +impl KVEngine for TestEngine { + fn admit_proposal(&self, _: Bytes) -> KVFuture<ApplyAdmissionResult> { + if self.0.busy.load(Ordering::Acquire) { + return KVFuture::ready(Err(ApplyAdmissionError::Busy)); + } + self.0.admitted.fetch_add(1, Ordering::Relaxed); + let permit: Box<dyn AdmittedApply> = Box::new(TestPermit(Arc::clone(&self.0))); + let calls = Arc::clone(&self.0); + KVFuture::Pending(Box::pin(async move { + calls.admission_entered.notify_one(); + if calls.hold_admission.load(Ordering::Acquire) { + calls.admission_release.notified().await; + } + Ok(Some(permit)) + })) + } + + fn apply(&self, slot: u64, _: &Batch) -> KVFuture<Result<(), String>> { + self.0.fallback_slot.store(slot, Ordering::Release); + KVFuture::ready(Ok(())) + } + + fn apply_noop(&self, _: u64) -> KVFuture<Result<(), String>> { + KVFuture::ready(if self.0.fail_noop.load(Ordering::Acquire) { + Err("gap progress unavailable".to_owned()) + } else { + Ok(()) + }) + } + + fn get(&self, _: &[u8]) -> KVFuture<Option<(u64, Vec<u8>)>> { + KVFuture::ready(None) + } + + fn scan_directional( + &self, + _: &[u8], + _: &[u8], + _: &[u8], + _: usize, + _: usize, + _: bool, + _: u64, + _: ScanDirection, + ) -> KVFuture<Result<(Vec<(Bytes, u64, Bytes)>, bool), String>> { + KVFuture::ready(Ok((vec![], false))) + } + + fn clear(&self) -> Result<(), String> { + Ok(()) + } + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +async fn group(calls: &Arc<Calls>) -> PxGroup { + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let replay = replay_group(&backend, &[PathBuf::from("/empty")], 1) + .await + .unwrap(); + let replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(TestEngine(Arc::clone(calls))), + ) + .await + .unwrap(); + PxGroup::new(1, replica) +} + +fn payload(value: u8) -> Vec<u8> { + vec![1, 0, 0, 1, 0, 0, 0, b'k', 1, 0, 0, 0, value] +} + +#[tokio::test] +async fn resource_rejection_precedes_slot_allocation_and_acceptance() { + let calls = Arc::new(Calls::default()); + calls.busy.store(true, Ordering::Release); + let group = group(&calls).await; + assert!(matches!( + group.propose(payload(b'a'), None, None).await, + ProposeResult::Busy + )); + assert_eq!(group.local_replica().last_chosen_slot(), 0); + assert!(group.local_replica().accepted_at(1).await.is_none()); + calls.busy.store(false, Ordering::Release); + assert!(matches!( + group.propose(payload(b'b'), None, None).await, + ProposeResult::Chosen { slot: 1 } + )); + assert_eq!(calls.admitted_slot.load(Ordering::Acquire), 1); + assert_eq!(calls.fallback_slot.load(Ordering::Acquire), 0); + assert_eq!(calls.released.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn adopted_foreign_value_does_not_consume_the_client_reservation() { + let calls = Arc::new(Calls::default()); + let mut group = group(&calls).await; + group.set_force_classic(true); + let entry = PxLogEntry { + slot: 1, + ballot: PxBallot::new(10, 99), + term: 0, + payload: Bytes::from(payload(b'f')), + }; + assert!(matches!( + group.local_replica().on_accept(&entry).await, + PxAcceptReply::Accepted { .. } + )); + assert!(matches!( + group.propose(payload(b'o'), None, None).await, + ProposeResult::Chosen { slot: 2 } + )); + assert_eq!(calls.admitted.load(Ordering::Acquire), 1); + assert_eq!(calls.fallback_slot.load(Ordering::Acquire), 1); + assert_eq!(calls.admitted_slot.load(Ordering::Acquire), 2); + assert_eq!(calls.released.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn chosen_background_apply_retains_credit_until_terminal() { + let calls = Arc::new(Calls::default()); + calls.hold_apply.store(true, Ordering::Release); + let mut group = group(&calls).await; + group.set_async_engine_apply(true); + assert!(matches!( + group.propose(payload(b'v'), None, None).await, + ProposeResult::Chosen { slot: 1 } + )); + calls.apply_entered.notified().await; + assert_eq!(group.local_replica().contiguous_chosen(), 1); + assert_eq!(group.local_replica().learner.contiguous_applied(), 0); + assert_eq!(calls.released.load(Ordering::Acquire), 0); + calls.apply_release.notify_one(); + group.local_replica().await_apply_fence(1).await; + assert_eq!(calls.released.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn cancelling_before_consensus_returns_the_reserved_credit() { + let calls = Arc::new(Calls::default()); + calls.hold_admission.store(true, Ordering::Release); + let group = group(&calls).await; + let mut future = Box::pin(group.propose(payload(b'v'), None, None)); + tokio::select! { + result = &mut future => panic!("admission unexpectedly completed: {result:?}"), + () = calls.admission_entered.notified() => {} + } + assert_eq!(calls.released.load(Ordering::Acquire), 0); + drop(future); + assert_eq!(calls.released.load(Ordering::Acquire), 1); + assert_eq!(group.local_replica().last_chosen_slot(), 0); +} + +#[tokio::test] +async fn failed_noop_cannot_advance_the_applied_frontier() { + let calls = Arc::new(Calls::default()); + calls.fail_noop.store(true, Ordering::Release); + let group = group(&calls).await; + group + .local_replica() + .learner + .learn( + PxLogEntry { + slot: 1, + ballot: PxBallot::new(0, 1), + term: 0, + payload: Bytes::new(), + }, + &[], + ) + .await; + assert_eq!(group.local_replica().contiguous_chosen(), 1); + assert_eq!(group.local_replica().learner.contiguous_applied(), 0); +} + +#[tokio::test] +async fn malformed_chosen_payload_cannot_become_a_noop_or_partial_apply() { + let calls = Arc::new(Calls::default()); + let group = group(&calls).await; + group + .local_replica() + .learner + .learn( + PxLogEntry { + slot: 1, + ballot: PxBallot::new(0, 1), + term: 0, + payload: Bytes::from_static(&[1, 0]), + }, + &[], + ) + .await; + assert_eq!(group.local_replica().contiguous_chosen(), 1); + assert_eq!(group.local_replica().learner.contiguous_applied(), 0); + assert_eq!(calls.fallback_slot.load(Ordering::Relaxed), 0); +} diff --git a/lib/crowdb-paxos-kv/tests/tree_idle_writeback_test.rs b/lib/crowdb-paxos-kv/tests/tree_idle_writeback_test.rs new file mode 100644 index 000000000..ca2790a8f --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/tree_idle_writeback_test.rs @@ -0,0 +1,142 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; + +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; +use crowdb_paxos_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; +use crowdb_tree_ffi::{PageStore, SyncMode, TreeRuntime, TreeRuntimeOptions}; + +fn native_engine() -> CrowdbTreeEngine { + let runtime = Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + ); + CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(runtime), + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + frame_bytes: 4096, + max_inline_value: 64, + sync_mode: SyncMode::Skip, + ..Default::default() + }) + .unwrap() +} + +async fn native_replica(engine: CrowdbTreeEngine) -> (PxLocalReplica, Arc<WalEngine>) { + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let replay = replay_group(&backend, &[PathBuf::from("/empty")], 1) + .await + .unwrap(); + let wal = WalEngine::create( + Arc::clone(&backend), + WalConfig { + wal_disks: vec![PathBuf::from("/wal")], + ..Default::default() + }, + 1, + ) + .await + .unwrap(); + let mut replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Follower, + &replay, + Box::new(engine), + ) + .await + .unwrap(); + replica.set_wal(Arc::clone(&wal)); + (replica, wal) +} + +async fn learn_prefix(replica: &PxLocalReplica) { + for slot in 1..=256_u64 { + let mut payload = vec![1, 0, 0]; + payload.extend_from_slice(&8_u32.to_le_bytes()); + payload.extend_from_slice(&slot.to_be_bytes()); + payload.extend_from_slice(&512_u32.to_le_bytes()); + payload.extend_from_slice(&[7; 512]); + replica + .learner + .learn( + PxLogEntry { + slot, + ballot: PxBallot::new(0, 1), + term: 1, + payload: Bytes::from(payload), + }, + &[], + ) + .await; + } +} + +#[tokio::test(flavor = "current_thread")] +async fn idle_host_pass_retries_abandoned_writeback_without_checkpoint_or_new_writes() { + let engine = native_engine(); + let tree = engine.handle(); + let (replica, wal) = native_replica(engine).await; + learn_prefix(&replica).await; + assert_eq!(replica.contiguous_applied(), 256); + let engine = replica.learner.engine(); + let flushed = engine.flush_async().unwrap().await.unwrap(); + assert_eq!(flushed.status(), Ok(())); + assert!(engine.writeback_pending()); + let before = tree.state_stats().unwrap(); + assert!(before.logical.writeback_candidate_pages > 1); + assert_eq!(tree.durable_coverage(), 0); + drop(engine.writeback_async().unwrap()); + assert_eq!( + tree.state_stats().unwrap().logical.writeback_candidate_pages, + before.logical.writeback_candidate_pages + ); + + let mut group = PxGroup::new(1, replica); + group.set_election_config(PxElectionConfig { + snapshot_slot_threshold: u64::MAX, + snapshot_time_threshold_ms: u64::MAX, + snapshot_flush_count_threshold: 0, + wal_flush_interval_ms: 0, + merge_gc_interval_ms: 0, + ..PxElectionConfig::for_tests() + }); + let mut passes = 0; + while tree.state_stats().unwrap().logical.writeback_candidate_pages != 0 && passes < 16 { + let candidates = tree.state_stats().unwrap().logical.writeback_candidate_pages; + group.run_maintenance_pass_for_tests().await; + passes += 1; + assert!(tree.state_stats().unwrap().logical.writeback_candidate_pages < candidates); + assert_eq!(tree.durable_coverage(), 0); + assert_eq!(wal.snapshot_slot(), 0); + assert_eq!(tree.last_applied_slot(), 256); + assert_eq!(group.local_replica().contiguous_applied(), 256); + } + assert!(passes > 1); + assert_eq!(tree.state_stats().unwrap().logical.writeback_candidate_pages, 0); + let settled = tree.state_stats().unwrap(); + for _ in 0..3 { + group.run_maintenance_pass_for_tests().await; + let idle = tree.state_stats().unwrap(); + assert_eq!(idle.logical, settled.logical); + assert_eq!(tree.durable_coverage(), 0); + assert_eq!(wal.snapshot_slot(), 0); + } + assert_eq!( + tree.get(&256_u64.to_be_bytes()).unwrap(), + Some((256, vec![7; 512])) + ); +} diff --git a/lib/crowdb-paxos-kv/tests/tree_maintenance_test.rs b/lib/crowdb-paxos-kv/tests/tree_maintenance_test.rs new file mode 100644 index 000000000..f52482fa7 --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/tree_maintenance_test.rs @@ -0,0 +1,324 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_paxos_kv::cluster::group::PxGroup; +use crowdb_paxos_kv::cluster::group_election::LeaderElection; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::common::config::PxElectionConfig; +use crowdb_paxos_kv::kv::{ + Batch, CrowdbTreeConfig, CrowdbTreeEngine, EngineCheckpointReport, EngineCheckpointWork, + EngineFlushReport, KVEngine, KVFuture, ScanDirection, +}; +use crowdb_paxos_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; +use crowdb_tree_ffi::{PageStore, SyncMode, TreeRuntime, TreeRuntimeOptions}; + +#[derive(Default)] +struct Calls { + flush: AtomicU64, + checkpoint: AtomicU64, + compaction: AtomicU64, +} +struct ScriptedMaintenance { + calls: Arc<Calls>, +} +impl KVEngine for ScriptedMaintenance { + fn apply(&self, _: u64, _: &Batch) -> KVFuture<Result<(), String>> { + KVFuture::ready(Ok(())) + } + fn get(&self, _: &[u8]) -> KVFuture<Option<(u64, Vec<u8>)>> { + KVFuture::ready(None) + } + fn scan_directional( + &self, + _: &[u8], + _: &[u8], + _: &[u8], + _: usize, + _: usize, + _: bool, + _: u64, + _: ScanDirection, + ) -> KVFuture<Result<(Vec<(Bytes, u64, Bytes)>, bool), String>> { + KVFuture::ready(Ok((vec![], false))) + } + fn clear(&self) -> Result<(), String> { + Ok(()) + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn flush(&self) { + panic!("runtime engine used blocking compatibility flush") + } + fn persist_snapshot(&self) -> u64 { + panic!("runtime engine used blocking compatibility checkpoint") + } + fn compact_sparse_blocks(&self) -> Result<(), String> { + panic!("runtime engine used blocking compatibility compaction") + } + fn compact_sparse_blocks_async(&self) -> Option<KVFuture<Result<(), String>>> { + let calls = Arc::clone(&self.calls); + Some(KVFuture::Pending(Box::pin(async move { + tokio::task::yield_now().await; + calls.compaction.fetch_add(1, Ordering::Relaxed); + Ok(()) + }))) + } + fn flush_async(&self) -> Option<KVFuture<Result<EngineFlushReport, String>>> { + let call = self.calls.flush.fetch_add(1, Ordering::Relaxed); + if call == 0 { + return Some(KVFuture::ready(Err("admission exhausted".to_owned()))); + } + Some(KVFuture::ready(Ok(EngineFlushReport { + handoff_status: Ok(()), + maintenance_status: if call == 1 { + Err("maintenance failed".to_owned()) + } else { + Ok(()) + }, + handoff_completed: true, + handoff_advanced: call == 1 || call == 3, + floor: u64::from(call >= 3), + frontier: if call < 3 { 1 } else { 2 }, + published_coverage: if call < 3 { 1 } else { 2 }, + deferred_pages: u64::from(call == 1), + }))) + } + fn persist_snapshot_async(&self) -> Option<KVFuture<Result<EngineCheckpointReport, String>>> { + self.calls.checkpoint.fetch_add(1, Ordering::Relaxed); + Some(KVFuture::ready(Ok(EngineCheckpointReport { + status: Err("post-commit rebase failed".to_owned()), + durable_committed: true, + durable_coverage: 2, + work: Some(EngineCheckpointWork { + writeback_completed: true, + captured_coverage: 2, + written_frames: 1, + reused_frames: 0, + rebase_deferred: 1, + }), + }))) + } +} + +#[tokio::test(flavor = "current_thread")] +async fn host_counts_only_new_handoffs_and_retains_post_commit_durability() { + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let wal = WalEngine::create( + Arc::clone(&backend), + WalConfig { + wal_disks: vec![PathBuf::from("/wal")], + ..Default::default() + }, + 1, + ) + .await + .unwrap(); + let replay = replay_group(&backend, &[PathBuf::from("/empty")], 1) + .await + .unwrap(); + let calls = Arc::new(Calls::default()); + let mut replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(ScriptedMaintenance { + calls: Arc::clone(&calls), + }), + ) + .await + .unwrap(); + replica.set_wal(Arc::clone(&wal)); + for slot in 1..=2 { + replica + .learner + .learn( + PxLogEntry { + slot, + ballot: PxBallot::new(0, 1), + term: 1, + payload: Bytes::new(), + }, + &[], + ) + .await; + } + let mut group = PxGroup::new(1, replica); + group.set_election_config(PxElectionConfig { + snapshot_slot_threshold: u64::MAX, + snapshot_time_threshold_ms: u64::MAX, + snapshot_flush_count_threshold: 2, + wal_flush_interval_ms: 0, + merge_gc_interval_ms: 0, + ..PxElectionConfig::for_tests() + }); + for _ in 0..3 { + group.run_maintenance_pass_for_tests().await; + assert_eq!(calls.checkpoint.load(Ordering::Relaxed), 0); + assert_eq!(wal.snapshot_slot(), 0); + } + group.run_maintenance_pass_for_tests().await; + assert_eq!(calls.checkpoint.load(Ordering::Relaxed), 1); + assert_eq!(wal.snapshot_slot(), 2); + group.run_maintenance_pass_for_tests().await; + assert_eq!(calls.checkpoint.load(Ordering::Relaxed), 1); +} + +fn native_engine() -> CrowdbTreeEngine { + let runtime = Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + ); + CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(runtime), + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + frame_bytes: 4096, + max_inline_value: 64, + sync_mode: SyncMode::Skip, + ..Default::default() + }) + .unwrap() +} + +#[tokio::test(flavor = "current_thread")] +async fn native_idle_writeback_and_checkpoint_use_async_progress() { + let engine = native_engine(); + engine.handle().apply_put(1, b"key", &vec![b'v'; 8192]).unwrap(); + let abandoned = engine.flush_async().unwrap(); + drop(abandoned); + assert!(engine.flush_pending()); + let handoff = engine.flush_async().unwrap().await.unwrap(); + assert!(handoff.handoff_advanced); + assert_eq!(handoff.status(), Ok(())); + assert!(engine.writeback_pending()); + let written = engine.writeback_async().unwrap().await.unwrap(); + assert_eq!(written.status, Ok(())); + assert!(written.work.as_ref().unwrap().writeback_completed); + assert!(!written.durable_committed); + assert_eq!(written.durable_coverage, 0); + assert!(!engine.writeback_pending()); + assert!(!engine.flush_pending()); + let durable = engine.persist_snapshot_async().unwrap().await.unwrap(); + assert_eq!(durable.status, Ok(())); + assert!(durable.durable_committed); + assert_eq!(durable.durable_coverage, 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn shutdown_awaits_async_stages_and_preserves_both_terminal_errors() { + let calls = Arc::new(Calls::default()); + let replica = shutdown_replica(Box::new(ScriptedMaintenance { + calls: Arc::clone(&calls), + })) + .await; + let report = replica.shutdown(std::time::Duration::from_secs(2)).await; + assert_eq!(calls.flush.load(Ordering::Acquire), 1); + assert_eq!(calls.checkpoint.load(Ordering::Acquire), 1); + assert_eq!(report.errors.len(), 2); + assert!(report.errors[0].contains("admission exhausted")); + assert!(report.errors[1].contains("post-commit rebase failed")); + assert!(replica + .shutdown(std::time::Duration::from_secs(2)) + .await + .is_clean()); + assert_eq!(calls.flush.load(Ordering::Acquire), 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn volatile_runtime_shutdown_has_no_false_durable_frontier() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(runtime), + ..Default::default() + }) + .unwrap(); + let tree = engine.handle(); + let replica = shutdown_replica(Box::new(engine)).await; + replica + .learner + .learn( + PxLogEntry { + slot: 1, + ballot: PxBallot::new(1, 1), + term: 1, + payload: Bytes::new(), + }, + &[], + ) + .await; + let report = replica.shutdown(std::time::Duration::from_secs(2)).await; + assert!(report.is_clean(), "{:?}", report.errors); + assert_eq!(tree.last_applied_slot(), 1); + assert_eq!(tree.durable_coverage(), 0); + assert_eq!(replica.learner.engine().resume_from_slot(), 0); +} + +#[tokio::test(flavor = "current_thread")] +async fn runtime_shutdown_waits_for_the_backend_checkpoint_before_returning() { + let engine = native_engine(); + let tree = engine.handle(); + let replica = shutdown_replica(Box::new(engine)).await; + replica + .learner + .learn( + PxLogEntry { + slot: 1, + ballot: PxBallot::new(1, 1), + term: 1, + payload: Bytes::from_static(&[ + 1, 0, 0, 3, 0, 0, 0, b'k', b'e', b'y', 5, 0, 0, 0, b'v', b'a', b'l', b'u', b'e', + ]), + }, + &[], + ) + .await; + assert_eq!(tree.last_applied_slot(), 0); + let flushed = replica.learner.engine().flush_async().unwrap().await.unwrap(); + assert!(flushed.status().is_ok()); + assert_eq!(tree.last_applied_slot(), 1); + assert_eq!(tree.durable_coverage(), 0); + assert_eq!(replica.learner.engine().resume_from_slot(), 0); + let report = replica.shutdown(std::time::Duration::from_secs(2)).await; + assert!(report.is_clean(), "{:?}", report.errors); + assert_eq!(tree.last_applied_slot(), 1); + assert_eq!(tree.durable_coverage(), 1); + assert_eq!(replica.learner.engine().resume_from_slot(), 1); + assert_eq!(tree.get(b"key").unwrap(), Some((1, b"value".to_vec()))); +} + +async fn shutdown_replica(engine: Box<dyn KVEngine>) -> PxLocalReplica { + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let replay = replay_group(&backend, &[PathBuf::from("/empty")], 1) + .await + .unwrap(); + PxLocalReplica::restore_from_replay_with_engine(1, PxLocalReplicaRole::Follower, &replay, engine) + .await + .unwrap() +} + +#[tokio::test(flavor = "current_thread")] +async fn configured_compaction_cadence_awaits_the_native_engine_path() { + let calls = Arc::new(Calls::default()); + let replica = shutdown_replica(Box::new(ScriptedMaintenance { + calls: Arc::clone(&calls), + })) + .await; + let mut group = PxGroup::new(1, replica); + group.set_election_config(PxElectionConfig { + merge_gc_interval_ms: 1, + ..PxElectionConfig::for_tests() + }); + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + group.run_maintenance_pass_for_tests().await; + assert_eq!(calls.compaction.load(Ordering::Acquire), 1); +} diff --git a/lib/crowdb-paxos-kv/tests/tree_native_admission_test.rs b/lib/crowdb-paxos-kv/tests/tree_native_admission_test.rs new file mode 100644 index 000000000..46ace72be --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/tree_native_admission_test.rs @@ -0,0 +1,153 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_paxos_kv::kv::{ApplyAdmissionError, Batch, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; +use crowdb_tree_ffi::{TreeRuntime, TreeRuntimeOptions}; + +fn engine() -> CrowdbTreeEngine { + CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + ..Default::default() + }) + .unwrap() +} + +fn put(value: &[u8]) -> Bytes { + let mut payload = vec![1, 0, 0]; + payload.extend_from_slice(&3u32.to_le_bytes()); + payload.extend_from_slice(b"key"); + payload.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes()); + payload.extend_from_slice(value); + Bytes::from(payload) +} + +#[tokio::test(flavor = "current_thread")] +async fn native_permit_owns_the_payload_and_applies_on_the_shared_runtime() { + let engine = engine(); + let payload = put(b"owned value"); + let permit = engine.admit_proposal(payload.clone()).await.unwrap().unwrap(); + drop(payload); + permit.apply(1).await.unwrap(); + assert_eq!(engine.get(b"key").await, Some((1, b"owned value".to_vec()))); +} + +#[tokio::test(flavor = "current_thread")] +async fn cancelling_native_admission_releases_the_pinned_generation() { + let engine = engine(); + let permit = engine.admit_proposal(put(b"cancelled")).await.unwrap().unwrap(); + drop(permit); + engine.handle().clear().unwrap(); + assert!(engine.get(b"key").await.is_none()); +} + +#[tokio::test(flavor = "current_thread")] +async fn clear_returns_busy_without_panicking_or_revoking_an_armed_proposal() { + let engine = engine(); + let permit = engine.admit_proposal(put(b"accepted")).await.unwrap().unwrap(); + assert!(engine.clear().is_err()); + permit.apply(1).await.unwrap(); + assert_eq!(engine.get(b"key").await, Some((1, b"accepted".to_vec()))); + engine.clear().unwrap(); + assert!(engine.get(b"key").await.is_none()); +} + +#[tokio::test(flavor = "current_thread")] +async fn malformed_native_proposal_is_rejected_before_it_has_a_permit() { + let engine = engine(); + assert!(matches!( + engine.admit_proposal(Bytes::from_static(&[1, 0])).await, + Err(ApplyAdmissionError::Rejected(_)) + )); + assert_eq!(engine.stats().contiguous_slot, 0); +} + +#[tokio::test(flavor = "current_thread")] +async fn native_empty_proposal_still_completes_its_real_slot() { + let engine = engine(); + let permit = engine.admit_proposal(Bytes::new()).await.unwrap().unwrap(); + permit.apply(1).await.unwrap(); + assert_eq!(engine.stats().contiguous_slot, 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn complete_gap_bound_rejects_an_oversized_proposal_before_consensus() { + let engine = engine(); + let permit = engine.admit_proposal(put(&vec![0; 8 * 1024 * 1024])).await; + assert!(matches!(permit, Err(ApplyAdmissionError::Rejected(_)))); +} + +#[tokio::test(flavor = "current_thread")] +async fn default_engine_returns_a_real_permit_and_applies_its_slot() { + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).unwrap(); + assert!(engine.requires_contiguous_apply()); + let permit = engine.admit_proposal(put(b"default")).await.unwrap().unwrap(); + assert_eq!(engine.stats().contiguous_slot, 0); + assert_eq!(engine.handle().durable_coverage(), 0); + assert!(engine.get(b"key").await.is_none()); + permit.apply(1).await.unwrap(); + assert_eq!(engine.stats().contiguous_slot, 1); + assert_eq!(engine.handle().durable_coverage(), 0); + assert_eq!(engine.get(b"key").await, Some((1, b"default".to_vec()))); +} + +#[tokio::test(flavor = "current_thread")] +async fn occupied_native_writer_credit_is_busy_and_cancellation_returns_it() { + let mut options = CrowdbTreeConfig { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }; + options.runtime_binding.limits.writer_bytes = 1024 * 1024; + let engine = CrowdbTreeEngine::open(&options).unwrap(); + let mut retained = Vec::new(); + let mut busy = false; + for _ in 0..16 { + match engine.admit_proposal(put(b"reserved")).await { + Ok(Some(permit)) => retained.push(permit), + Err(ApplyAdmissionError::Busy) => { + busy = true; + break; + } + other => panic!( + "unexpected native admission: {}", + match other { + Err(ApplyAdmissionError::Rejected(error)) => error, + _ => "missing permit".to_owned(), + } + ), + } + } + assert!(busy); + assert!(!retained.is_empty()); + drop(retained); + engine.handle().clear().unwrap(); + let permit = engine.admit_proposal(put(b"returned")).await.unwrap().unwrap(); + permit.apply(1).await.unwrap(); + assert_eq!(engine.get(b"key").await, Some((1, b"returned".to_vec()))); +} + +#[tokio::test(flavor = "current_thread")] +async fn chosen_follower_apply_waits_for_empty_gap_without_blocking_the_host() { + use std::future::{poll_fn, Future}; + use std::task::Poll; + let engine = engine(); + let batch = Batch::try_decode(&put(b"follower")).unwrap(); + let mut higher = Box::pin(engine.apply(2, &batch)); + poll_fn(|context| { + assert!(higher.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; + engine.apply_noop(1).await.unwrap(); + higher.await.unwrap(); + assert_eq!(engine.get(b"key").await, Some((2, b"follower".to_vec()))); + assert_eq!(engine.stats().contiguous_slot, 2); +} diff --git a/lib/crowdb-paxos-kv/tests/tree_snapshot_owner_test.rs b/lib/crowdb-paxos-kv/tests/tree_snapshot_owner_test.rs new file mode 100644 index 000000000..facdbfaf5 --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/tree_snapshot_owner_test.rs @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; +use crowdb_rpc_ffi::Buffer; +use crowdb_tree_ffi::{TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +#[test] +fn snapshot_view_propagates_failed_handoff_without_returning_an_old_prefix() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let mut config = CrowdbTreeConfig { + runtime: Some(runtime), + ..Default::default() + }; + config.runtime_binding.limits.handoff_bytes = 1024 * 1024; + let engine = CrowdbTreeEngine::open(&config).unwrap(); + engine.handle().apply_put(1, b"key", b"value").unwrap(); + assert!(engine.handle().flush().is_err()); + let result = engine.snapshot_view(); + assert!( + result.is_err(), + "a rejected handoff cannot expose an older successful snapshot: {result:?}" + ); + assert_eq!(engine.handle().get(b"key").unwrap(), Some((1, b"value".to_vec()))); +} + +#[test] +fn rpc_snapshot_buffer_retains_native_bytes_after_all_engine_handles_close() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig { + runtime: Some(Arc::clone(&runtime)), + ..Default::default() + }) + .unwrap(); + let tree = engine.handle(); + tree.apply_put(1, b"key", &vec![b'v'; 8192]).unwrap(); + tree.flush().unwrap(); + let mut export = engine.snapshot_export_begin(64).unwrap(); + let chunk = export.read(0).unwrap(); + assert_eq!(chunk.bytes.len(), 64); + let retained = chunk.bytes.slice(4..36); + let pointer = retained.as_ptr(); + let expected = retained.to_vec(); + assert!(retained.owner_allocation().is_some()); + let response = Buffer::try_from_owned_bytes(retained).unwrap(); + assert_eq!(response.as_slice().as_ptr(), pointer); + drop(chunk); + drop(export); + drop(tree); + drop(engine); + drop(runtime); + assert_eq!(response.as_slice(), expected); +} + +#[tokio::test(flavor = "current_thread")] +async fn versioned_and_byte_reads_keep_the_account_on_resident_and_cold_paths() { + let runtime = Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + ); + let config = CrowdbTreeConfig { + runtime: Some(runtime.clone()), + page_store: Some(Arc::new(crowdb_tree_ffi::PageStore::open_mem(4096).unwrap())), + ..Default::default() + }; + let engine = CrowdbTreeEngine::open(&config).unwrap(); + let expected = vec![b'v'; 8192]; + engine.handle().apply_put(1, b"key", &expected).unwrap(); + let (_, resident) = engine.get_bytes(b"key").await.unwrap(); + assert!(resident.owner_allocation().is_some()); + let checkpoint = engine.persist_snapshot_async().unwrap().await.unwrap(); + assert!(checkpoint.status.is_ok(), "{:?}", checkpoint.status); + assert!(checkpoint.durable_committed); + drop(engine); + let reopened = CrowdbTreeEngine::open(&config).unwrap(); + // No yield between old pump cancellation and the replacement Tree opening. + let (slot, cold) = + tokio::time::timeout(std::time::Duration::from_secs(2), reopened.get_versioned(b"key")) + .await + .expect("replacement Tree completion remains registered") + .unwrap() + .unwrap(); + assert_eq!(slot, 1); + assert!(cold.owner_allocation().is_some()); + let response = Buffer::try_from_owned_bytes(cold).unwrap(); + drop(reopened); + drop(config); + drop(runtime); + assert_eq!(resident.as_ref(), expected); + assert_eq!(response.as_slice(), expected); +} diff --git a/lib/crowdb-kv/tests/wal_test.rs b/lib/crowdb-paxos-kv/tests/wal_test.rs similarity index 100% rename from lib/crowdb-kv/tests/wal_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test.rs diff --git a/lib/crowdb-kv/tests/wal_test/block_backend_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/block_backend_test.rs similarity index 96% rename from lib/crowdb-kv/tests/wal_test/block_backend_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/block_backend_test.rs index 15cb01a27..23dee6c59 100644 --- a/lib/crowdb-kv/tests/wal_test/block_backend_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/block_backend_test.rs @@ -4,11 +4,11 @@ //! Tests for the simulated `BlockDevice` backend's two write implementations: //! byte-addressable (unaligned) vs block-aligned read-modify-write. -use crowdb_kv::wal::pipeline_backend::WalBlockAlignment; -use crowdb_kv::wal::{IoBackend, MemBlockDevice, OpenOptions}; +use crowdb_paxos_kv::wal::pipeline_backend::WalBlockAlignment; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, OpenOptions}; use std::path::Path; -async fn open_segment(backend: &IoBackend, path: &str) -> crowdb_kv::wal::WalFile { +async fn open_segment(backend: &IoBackend, path: &str) -> crowdb_paxos_kv::wal::WalFile { backend .open(Path::new(path), OpenOptions::create_rw()) .await diff --git a/lib/crowdb-kv/tests/wal_test/block_restore_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/block_restore_test.rs similarity index 93% rename from lib/crowdb-kv/tests/wal_test/block_restore_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/block_restore_test.rs index ed5c0f265..f5c7544d7 100644 --- a/lib/crowdb-kv/tests/wal_test/block_restore_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/block_restore_test.rs @@ -3,11 +3,11 @@ use std::sync::Arc; -use crowdb_kv::paxos::roles::PxBallot; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::segment::WalSegment; -use crowdb_kv::wal::{IoBackend, OpenOptions}; +use crowdb_paxos_kv::paxos::roles::PxBallot; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::segment::WalSegment; +use crowdb_paxos_kv::wal::{IoBackend, OpenOptions}; #[tokio::test] async fn direct_block_replay_recovers_sealed_and_unsealed_records() { diff --git a/lib/crowdb-kv/tests/wal_test/block_runtime_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/block_runtime_test.rs similarity index 95% rename from lib/crowdb-kv/tests/wal_test/block_runtime_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/block_runtime_test.rs index 2211f4829..8651ca59d 100644 --- a/lib/crowdb-kv/tests/wal_test/block_runtime_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/block_runtime_test.rs @@ -9,8 +9,8 @@ use std::sync::{ }; use std::time::Duration; -use crowdb_kv::paxos::roles::PxBallot; -use crowdb_kv::wal::{BlockDevice, IoBackend, OpenOptions, WALRecord, WalConfig, WalEngine}; +use crowdb_paxos_kv::paxos::roles::PxBallot; +use crowdb_paxos_kv::wal::{BlockDevice, IoBackend, OpenOptions, WALRecord, WalConfig, WalEngine}; async fn held_wal_io(sync: bool, expected_progress: bool) { let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("block-runtime-held"); @@ -66,7 +66,7 @@ async fn held_wal_io(sync: bool, expected_progress: bool) { assert!(device.fdatasync_count() >= 2); wal.flush_all().await.unwrap(); drop(wal); - let recovered = crowdb_kv::wal::replay::replay_group(&backend, &[directory.path().into()], 1) + let recovered = crowdb_paxos_kv::wal::replay::replay_group(&backend, &[directory.path().into()], 1) .await .unwrap(); assert_eq!( diff --git a/lib/crowdb-kv/tests/wal_test/file_backend_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/file_backend_test.rs similarity index 99% rename from lib/crowdb-kv/tests/wal_test/file_backend_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/file_backend_test.rs index b01f2b64e..a82557395 100644 --- a/lib/crowdb-kv/tests/wal_test/file_backend_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/file_backend_test.rs @@ -8,7 +8,7 @@ //! and `WalFile` methods. These tests cover the real-filesystem fallback //! path (open/append/flush/fsync/truncate) independent of `BlockDevice`. -use crowdb_kv::wal::{IoBackend, OpenOptions}; +use crowdb_paxos_kv::wal::{IoBackend, OpenOptions}; #[tokio::test] async fn file_backend_write_then_reopen_preserves_data() { diff --git a/lib/crowdb-kv/tests/wal_test/file_restore_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/file_restore_test.rs similarity index 96% rename from lib/crowdb-kv/tests/wal_test/file_restore_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/file_restore_test.rs index e69b5acaa..fb14b813f 100644 --- a/lib/crowdb-kv/tests/wal_test/file_restore_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/file_restore_test.rs @@ -15,11 +15,11 @@ use std::path::PathBuf; use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::common::config::WalConfig; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::{RecordType, WALRecord}; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; +use crowdb_paxos_kv::common::config::WalConfig; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::{RecordType, WALRecord}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; const GROUP: u64 = 1; diff --git a/lib/crowdb-kv/tests/wal_test/gc_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/gc_test.rs similarity index 97% rename from lib/crowdb-kv/tests/wal_test/gc_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/gc_test.rs index ed7b2079c..396217d56 100644 --- a/lib/crowdb-kv/tests/wal_test/gc_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/gc_test.rs @@ -4,12 +4,12 @@ //! GC worker tests (W14) — `SimDisk` backend. use bytes::Bytes; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::gc::{run_gc_pass, run_gc_with_watermark}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::wal_engine::WalEngine; -use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::gc::{run_gc_pass, run_gc_with_watermark}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; use std::path::PathBuf; use std::sync::Arc; diff --git a/lib/crowdb-kv/tests/wal_test/index_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/index_test.rs similarity index 98% rename from lib/crowdb-kv/tests/wal_test/index_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/index_test.rs index 0b23c488d..16138d984 100644 --- a/lib/crowdb-kv/tests/wal_test/index_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/index_test.rs @@ -4,7 +4,7 @@ //! `SegmentIndex` unit tests: slot→location lookup, segment registration, //! removal, and rebuild-from-scans. -use crowdb_kv::wal::index::{SegmentIndex, SegmentMeta, SlotLocation}; +use crowdb_paxos_kv::wal::index::{SegmentIndex, SegmentMeta, SlotLocation}; fn loc(slot: u64, seg: u64, off: u64) -> (u64, SlotLocation) { ( diff --git a/lib/crowdb-kv/tests/wal_test/io_backend_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/io_backend_test.rs similarity index 99% rename from lib/crowdb-kv/tests/wal_test/io_backend_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/io_backend_test.rs index 461b672c6..67cca410b 100644 --- a/lib/crowdb-kv/tests/wal_test/io_backend_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/io_backend_test.rs @@ -8,7 +8,7 @@ use std::sync::Arc; -use crowdb_kv::wal::{IoBackend, OpenOptions}; +use crowdb_paxos_kv::wal::{IoBackend, OpenOptions}; #[tokio::test] async fn detect_returns_file_backend() { diff --git a/lib/crowdb-kv/tests/wal_test/pipeline_backend_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/pipeline_backend_test.rs similarity index 94% rename from lib/crowdb-kv/tests/wal_test/pipeline_backend_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/pipeline_backend_test.rs index 2ab70ad49..3ed523210 100644 --- a/lib/crowdb-kv/tests/wal_test/pipeline_backend_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/pipeline_backend_test.rs @@ -1,7 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use crowdb_kv::wal::pipeline_backend::{WalBlockAlignment, WalBlockPipelineBackend, WalPipelineBackend}; +use crowdb_paxos_kv::wal::pipeline_backend::{ + WalBlockAlignment, WalBlockPipelineBackend, WalPipelineBackend, +}; #[test] fn mem_block_has_no_alignment_requirement() { diff --git a/lib/crowdb-kv/tests/wal_test/pipeline_writer_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/pipeline_writer_test.rs similarity index 97% rename from lib/crowdb-kv/tests/wal_test/pipeline_writer_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/pipeline_writer_test.rs index f06772d9b..65f8a57a4 100644 --- a/lib/crowdb-kv/tests/wal_test/pipeline_writer_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/pipeline_writer_test.rs @@ -18,10 +18,10 @@ use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::common::config::WalConfig; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; +use crowdb_paxos_kv::common::config::WalConfig; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::{IoBackend, WalEngine, WalRecordFormat}; const GROUP: u64 = 1; diff --git a/lib/crowdb-kv/tests/wal_test/record_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/record_test.rs similarity index 97% rename from lib/crowdb-kv/tests/wal_test/record_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/record_test.rs index 634c855c5..13d84a2d4 100644 --- a/lib/crowdb-kv/tests/wal_test/record_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/record_test.rs @@ -4,8 +4,8 @@ //! `WALRecord` codec tests (W2). use bytes::Bytes; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::{RecordError, RecordType, WALRecord, MIN_RECORD_SIZE}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::{RecordError, RecordType, WALRecord, MIN_RECORD_SIZE}; #[test] fn short_frame_marker_is_rejected() { diff --git a/lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/replay_compaction_test.rs similarity index 92% rename from lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/replay_compaction_test.rs index 6f87b83db..20ac57b2d 100644 --- a/lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/replay_compaction_test.rs @@ -5,11 +5,11 @@ use std::path::PathBuf; use std::sync::Arc; use bytes::Bytes; -use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::{RecordType, WALRecord}; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig, WalEngine}; +use crowdb_paxos_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::{RecordType, WALRecord}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, WalConfig, WalEngine}; fn accepted(ballot: u64, term: u64, value: &[u8]) -> WALRecord { let mut payload = vec![1, 0, 0, 1, 0, 0, 0, b'k']; diff --git a/lib/crowdb-kv/tests/wal_test/replay_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/replay_test.rs similarity index 84% rename from lib/crowdb-kv/tests/wal_test/replay_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/replay_test.rs index dafaf7c79..848ed04dc 100644 --- a/lib/crowdb-kv/tests/wal_test/replay_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/replay_test.rs @@ -4,13 +4,13 @@ //! Replay engine tests (W10-W13) — `SimDisk` backend. use bytes::Bytes; -use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; -use crowdb_kv::kv::{Batch, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::record::WALRecord; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::wal_engine::WalEngine; -use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; +use crowdb_paxos_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_paxos_kv::kv::{Batch, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::record::WALRecord; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, WalConfig}; use std::path::PathBuf; use std::sync::Arc; @@ -26,6 +26,20 @@ fn test_config(disks: &[PathBuf]) -> WalConfig { } } +fn durable_tree_config(path: &std::path::Path) -> CrowdbTreeConfig { + CrowdbTreeConfig { + path: Some(path.to_string_lossy().into_owned()), + runtime: Some(Arc::new( + crowdb_tree_ffi::TreeRuntime::new(crowdb_tree_ffi::TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + ..Default::default() + } +} + fn encode_put_payload(key: &[u8], value: &[u8]) -> Vec<u8> { let mut buf = Vec::new(); buf.extend_from_slice(&1u16.to_le_bytes()); @@ -172,7 +186,7 @@ async fn replay_1000_records_deterministic() { /// WAL replay now fully restores the learner: every accepted entry is /// replayed into the state machine. Both accepted slots are applied. #[tokio::test(flavor = "current_thread", start_paused = true)] -async fn restore_without_snapshot_does_not_apply_slots() { +async fn restore_without_snapshot_applies_contiguous_slots() { let backend = sim_backend(); let disks = vec![PathBuf::from("/wal")]; let config = test_config(&disks); @@ -245,6 +259,22 @@ async fn restore_from_replay_rebuilds_live_replica_state() { assert_eq!(restored.promised_at(1).await, Some(PxBallot::new(2, 7))); assert_eq!(restored.accepted_at(2).await, Some(accepted_entry.clone())); assert_eq!(restored.contiguous_chosen(), 0); + assert_eq!(restored.last_chosen_slot(), 0); + assert_eq!(restored.contiguous_applied(), 0); + assert!(restored.learner.engine_get(b"restore-key").await.is_none()); + + // Contiguous replay retains the accepted suffix until consensus repairs + // the missing prefix; startup must return so that repair can run. + let repair = PxLogEntry { + slot: 1, + ballot: PxBallot::new(6, 7), + term: 5, + payload: Bytes::new(), + }; + restored.learn_chosen(&repair, &[]).await; + restored.learn_chosen(&accepted_entry, &[]).await; + assert_eq!(restored.contiguous_chosen(), 2); + assert_eq!(restored.contiguous_applied(), 2); assert_eq!(restored.last_chosen_slot(), 2); assert_eq!( restored.learner.engine_get(b"restore-key").await.map(|(_, v)| v), @@ -254,8 +284,8 @@ async fn restore_from_replay_rebuilds_live_replica_state() { /// `restore_from_replay_with_engine` replays the full WAL into a /// caller-supplied engine (here an in-memory `CrowdbTreeEngine`, standing in -/// for the durable file-backed one `crowdb-kv-server` uses), not just the -/// default `InMemKV`. Otherwise identical restore semantics to +/// for the durable file-backed one `crowdb-paxos-kv-server` uses). Otherwise +/// identical restore semantics to /// `restore_from_replay` (same term/voted-for/acceptor state, same /// contiguous-chosen advance) -- the only difference is which `KVEngine` /// ends up behind `learner.engine_get`. @@ -280,7 +310,7 @@ async fn restore_from_replay_with_engine_uses_injected_engine() { let replay = replay_group(&backend, &disks, 1).await.unwrap(); // `path: None` selects an in-memory crowdb-tree store -- same restore path - // crowdb-kv-server's `--kv-engine crowdb-tree` uses, minus the on-disk file. + // crowdb-paxos-kv-server's `--kv-engine crowdb-tree` uses, minus the on-disk file. let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).expect("open in-memory crowdb-tree engine"); let restored = PxLocalReplica::restore_from_replay_with_engine( @@ -306,9 +336,8 @@ async fn restore_from_replay_with_engine_uses_injected_engine() { } /// `resume_from_slot() > 0`: an engine that already durably reflects a -/// prefix of the WAL (here, slot 1 pre-applied and `flush()`ed before the -/// engine is handed to `restore_from_replay_with_engine`, simulating what a -/// real restart sees from a durable `CrowdbTreeEngine`) skips re-`learn()`ing +/// prefix of the WAL (here, slot 1 checkpointed before reopening the engine) +/// skips re-`learn()`ing /// that prefix, but must land at the exact same `contiguous_chosen` / /// `contiguous_applied` / `last_chosen_slot` / `last_chosen_term` state (and /// the same final KV contents) a full sequential replay would have produced. @@ -335,19 +364,19 @@ async fn restore_from_replay_with_engine_resumes_from_last_applied_slot() { wal.seal_all().await.unwrap(); let replay = replay_group(&backend, &disks, 1).await.unwrap(); - // Pre-apply + flush slot 1 directly, matching what a durably-recovered - // CrowdbTreeEngine reports via `resume_from_slot()` on a real restart. - let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).expect("open crowdb-tree engine"); + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("wal-resume-prefix"); + let tree_config = durable_tree_config(directory.path()); + let engine = CrowdbTreeEngine::open(&tree_config).expect("open crowdb-tree engine"); engine .apply(1, &Batch::decode(&Bytes::from(encode_put_payload(b"k1", b"v1")))) - .into_ready() + .await .unwrap(); engine.handle().flush().expect("flush"); - assert_eq!( - engine.handle().last_applied_slot(), - 1, - "sanity: engine reports a resume floor of 1" - ); + assert_eq!(engine.resume_from_slot(), 0, "flush alone is not durable"); + assert_eq!(engine.persist_snapshot(), 1); + drop(engine); + let engine = CrowdbTreeEngine::open(&tree_config).expect("reopen checkpoint"); + assert_eq!(engine.resume_from_slot(), 1); let restored = PxLocalReplica::restore_from_replay_with_engine( 7, @@ -399,7 +428,7 @@ async fn durable_snapshot_prefix_survives_reclaimed_wal_boundary_records() { &[u8::try_from(slot).unwrap()], ))), ) - .into_ready() + .await .unwrap(); } engine.flush(); @@ -457,27 +486,25 @@ async fn restore_from_replay_with_engine_keeps_prefix_when_resume_slot_has_no_ac wal.seal_all().await.unwrap(); let replay = replay_group(&backend, &disks, 1).await.unwrap(); - // Engine reports a resume floor (slot 2) the WAL-rebuilt acceptor has no - // accepted entry for. `last_applied_slot()` is itself a contiguous - // watermark (crowdb-tree folds `received_slots_` forward from its own - // frontier), so reaching floor 2 legitimately requires applying slot 1 - // too. The retained WAL may omit the already durable boundary after - // reclamation; it need not corroborate that complete prefix again. - let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).expect("open crowdb-tree engine"); + // Checkpoint a contiguous prefix ending at slot 2, whose accepted WAL + // record is missing. Reopen it to exercise the durable resume floor. + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("wal-resume-missing-record"); + let tree_config = durable_tree_config(directory.path()); + let engine = CrowdbTreeEngine::open(&tree_config).expect("open crowdb-tree engine"); engine .apply(1, &Batch::decode(&Bytes::from(encode_put_payload(b"k1", b"v1")))) - .into_ready() + .await .unwrap(); engine .apply(2, &Batch::decode(&Bytes::from(encode_put_payload(b"k2", b"v2")))) - .into_ready() + .await .unwrap(); engine.handle().flush().expect("flush"); - assert_eq!( - engine.handle().last_applied_slot(), - 2, - "sanity: engine reports a resume floor of 2" - ); + assert_eq!(engine.resume_from_slot(), 0, "flush alone is not durable"); + assert_eq!(engine.persist_snapshot(), 2); + drop(engine); + let engine = CrowdbTreeEngine::open(&tree_config).expect("reopen checkpoint"); + assert_eq!(engine.resume_from_slot(), 2); let restored = PxLocalReplica::restore_from_replay_with_engine( 7, diff --git a/lib/crowdb-kv/tests/wal_test/segment_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/segment_test.rs similarity index 96% rename from lib/crowdb-kv/tests/wal_test/segment_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/segment_test.rs index b55f57dbb..d113e55d4 100644 --- a/lib/crowdb-kv/tests/wal_test/segment_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/segment_test.rs @@ -4,11 +4,11 @@ //! Segment file format tests (W3) — `SimDisk` backend. use bytes::Bytes; -use crowdb_kv::paxos::roles::PxBallot; -use crowdb_kv::wal::pipeline_backend::WalBlockAlignment; -use crowdb_kv::wal::record::{RecordType, WALRecord, WalRecordFormat}; -use crowdb_kv::wal::segment::{SegmentReader, WalSegment, SEG_HEADER_LEN}; -use crowdb_kv::wal::{IoBackend, MemBlockDevice}; +use crowdb_paxos_kv::paxos::roles::PxBallot; +use crowdb_paxos_kv::wal::pipeline_backend::WalBlockAlignment; +use crowdb_paxos_kv::wal::record::{RecordType, WALRecord, WalRecordFormat}; +use crowdb_paxos_kv::wal::segment::{SegmentReader, WalSegment, SEG_HEADER_LEN}; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice}; use std::path::PathBuf; fn sim_backend() -> IoBackend { diff --git a/lib/crowdb-kv/tests/wal_test/wal_engine_test.rs b/lib/crowdb-paxos-kv/tests/wal_test/wal_engine_test.rs similarity index 98% rename from lib/crowdb-kv/tests/wal_test/wal_engine_test.rs rename to lib/crowdb-paxos-kv/tests/wal_test/wal_engine_test.rs index f8a6421f7..3c9eda05b 100644 --- a/lib/crowdb-kv/tests/wal_test/wal_engine_test.rs +++ b/lib/crowdb-paxos-kv/tests/wal_test/wal_engine_test.rs @@ -4,13 +4,13 @@ //! `WalEngine` tests (W8) — `SimDisk` backend. use bytes::Bytes; -use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; -use crowdb_kv::wal::pipeline_backend::{WalBlockAlignment, WalPipelineBackend}; -use crowdb_kv::wal::record::{WALRecord, WalRecordFormat, WAL_MAGIC}; -use crowdb_kv::wal::replay::replay_group; -use crowdb_kv::wal::segment::SEG_HEADER_LEN; -use crowdb_kv::wal::wal_engine::WalEngine; -use crowdb_kv::wal::{IoBackend, MemBlockDevice, OpenOptions, WalConfig}; +use crowdb_paxos_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_paxos_kv::wal::pipeline_backend::{WalBlockAlignment, WalPipelineBackend}; +use crowdb_paxos_kv::wal::record::{WALRecord, WalRecordFormat, WAL_MAGIC}; +use crowdb_paxos_kv::wal::replay::replay_group; +use crowdb_paxos_kv::wal::segment::SEG_HEADER_LEN; +use crowdb_paxos_kv::wal::wal_engine::WalEngine; +use crowdb_paxos_kv::wal::{IoBackend, MemBlockDevice, OpenOptions, WalConfig}; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -599,7 +599,7 @@ async fn aligned_engine_append_rotate_seal_replays_all_records() { /// not locatable through the WAL index — only the previously-durable slots /// are. This verifies the "reads are consistent with the last durable state" /// half of the disk-loss recovery contract at the API level. The full -/// fail-out procedure (step-out RPC + reconfiguration, design-crowdb-kv-wal.md +/// fail-out procedure (step-out RPC + reconfiguration, design-crowdb-paxos-kv-wal.md /// §8.1) is not yet implemented and is tested separately once it lands. #[cfg_attr(target_os = "macos", ignore = "per-write fsync is disabled on macOS")] #[tokio::test(flavor = "current_thread")] diff --git a/lib/crowdb-paxos-kv/tests/watch_registry_test.rs b/lib/crowdb-paxos-kv/tests/watch_registry_test.rs new file mode 100644 index 000000000..70a0a7d4c --- /dev/null +++ b/lib/crowdb-paxos-kv/tests/watch_registry_test.rs @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_paxos_kv::cluster::watch_registry::{CrowdbRpcPushTarget, WatchRegistry}; +use crowdb_rpc_ffi::{RpcClient, RpcServer}; + +#[test] +fn fast_path_flag_tracks_watcher_mutations() { + let server = Arc::new(RpcServer::with_engines(None, 1, 1)); + server.listen("127.0.0.1", 0).unwrap(); + server.start(); + let connection = server.connect("127.0.0.1", server.port()).unwrap(); + let target = Arc::new(CrowdbRpcPushTarget::new( + connection, + Arc::new(RpcClient::new()), + Arc::clone(&server), + )); + + let registry = WatchRegistry::new(); + assert!(!registry.has_watchers()); + let first = registry.subscribe_crowdb_rpc(b"a", Arc::clone(&target)); + assert!(registry.has_watchers()); + let second = registry.subscribe_crowdb_rpc(b"ab", target); + registry.unsubscribe(b"a", first); + assert!(registry.has_watchers()); + registry.remove_all(&[second]); + assert!(!registry.has_watchers()); + + let replacement = Arc::new(RpcClient::new()); + let replacement_target = Arc::new(CrowdbRpcPushTarget::new( + server.connect("127.0.0.1", server.port()).unwrap(), + replacement, + Arc::clone(&server), + )); + let _ = registry.subscribe_crowdb_rpc(b"", replacement_target); + registry.clear(); + assert!(!registry.has_watchers()); + server.stop(); +} diff --git a/lib/crowdb-protocol/cpp/include/crowdb-protocol/flatbuffer.h b/lib/crowdb-protocol/cpp/include/crowdb-protocol/flatbuffer.h new file mode 100644 index 000000000..72b7ea5ec --- /dev/null +++ b/lib/crowdb-protocol/cpp/include/crowdb-protocol/flatbuffer.h @@ -0,0 +1,27 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <flatbuffers/base.h> + +#include <cstddef> +#include <cstdint> +#include <cstring> + +namespace crowdb::protocol +{ +// Call after VerifyBuffer<Table>. RPC controls can follow an unaligned wire +// header. Read the root offset without an aligned scalar dereference; generated +// table accessors retain their normal borrowed, unaligned field reads. +template <class Table> const Table *flatbuffer_root(const uint8_t *bytes, size_t size) noexcept +{ + static_assert(alignof(Table) == 1); + if (bytes == nullptr || size < sizeof(flatbuffers::uoffset_t)) { + return nullptr; + } + flatbuffers::uoffset_t offset = 0; + std::memcpy(&offset, bytes, sizeof(offset)); + offset = flatbuffers::EndianScalar(offset); + return offset < size ? reinterpret_cast<const Table *>(bytes + offset) : nullptr; +} +} // namespace crowdb::protocol diff --git a/lib/crowdb-protocol/cpp/include/crowdb-protocol/ports.h b/lib/crowdb-protocol/cpp/include/crowdb-protocol/ports.h new file mode 100644 index 000000000..461b3ad0b --- /dev/null +++ b/lib/crowdb-protocol/cpp/include/crowdb-protocol/ports.h @@ -0,0 +1,28 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +// Generated from ports.rs by tools/ports/generate.py; do not edit. +#pragma once + +#include <cstdint> + +namespace crowdb::protocol +{ +inline constexpr std::uint16_t PAXOS_KV_SERVER_MGMT_BASE = 10135; +inline constexpr std::uint16_t PAXOS_KV_SERVER_RPC_BASE = 11135; +inline constexpr std::uint16_t DISKDB_HTTP_BASE = 13135; +inline constexpr std::uint16_t DISKDB_RPC_BASE = 14135; +inline constexpr std::uint16_t CHUNKDB_HTTP_BASE = 16135; +inline constexpr std::uint16_t CHUNKDB_RPC_BASE = 17135; +inline constexpr std::uint16_t DISKIO_RPC_BASE = 18135; +inline constexpr std::uint16_t WEB_BASE = 6135; +inline constexpr std::uint16_t CHUNK_KV_HTTP_BASE = 20135; +inline constexpr std::uint16_t CHUNK_KV_RPC_BASE = 21135; +inline constexpr std::uint16_t ACCESS_SERVER_HTTP_BASE = 7135; +inline constexpr std::uint16_t ACCESS_SERVER_ICEBERG_HTTP_BASE = 8135; +inline constexpr std::uint16_t ACCESS_SERVER_DATASET_HTTP_BASE = 9135; +inline constexpr std::uint16_t ACCESS_SERVER_HEALTH_HTTP_BASE = 25135; +inline constexpr std::uint16_t MONITOR_HTTP_BASE = 26135; +inline constexpr std::uint16_t SSH_PORT = 2222; +inline constexpr std::uint16_t HOST_SSH_PORT = 22; +} // namespace crowdb::protocol diff --git a/lib/crowdb-protocol/src/chunk_slot/map.rs b/lib/crowdb-protocol/src/chunk_slot/map.rs index f51b0d421..536a84f2a 100644 --- a/lib/crowdb-protocol/src/chunk_slot/map.rs +++ b/lib/crowdb-protocol/src/chunk_slot/map.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use super::{ChunkSlot, ChunkSlotBitmap, ChunkSlotError, CHUNK_SLOT_COUNT, CHUNK_SLOT_LAYOUT_VERSION}; -/// Publish a head and all its owner records in one group-0 conditional batch. +/// A committed head makes the complete owner layout visible to readers. /// Service and storage heads have independent generations and namespaces. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/lib/crowdb-protocol/src/chunk_stream/ownership.rs b/lib/crowdb-protocol/src/chunk_stream/ownership.rs index e6680f010..a2a1974c3 100644 --- a/lib/crowdb-protocol/src/chunk_stream/ownership.rs +++ b/lib/crowdb-protocol/src/chunk_stream/ownership.rs @@ -25,6 +25,9 @@ pub struct StreamProcessLease { pub prior_exclusion_deadline_ms: u64, pub grant_sequence: u64, pub grant_expires_at_ms: u64, + /// Monotone for this identity; prevents late renewal after GC exclusion. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub revoked: bool, } #[derive(Debug, Error, PartialEq, Eq)] diff --git a/lib/crowdb-protocol/src/fbs/chunkdb.fbs b/lib/crowdb-protocol/src/fbs/chunkdb.fbs index d2f19621b..662c7ada7 100644 --- a/lib/crowdb-protocol/src/fbs/chunkdb.fbs +++ b/lib/crowdb-protocol/src/fbs/chunkdb.fbs @@ -409,6 +409,8 @@ table FBDeleteChunkRequest { id:uint64; rpc_create_nano:uint64; chunk_id:crowdb.rpc.proto.FBInt128; + // A tombstone must precede retiring an unknown fixed-ID allocation. + fence_missing:bool = false; } table FBDeleteChunkResponse { diff --git a/lib/crowdb-protocol/src/fbs/diskio.fbs b/lib/crowdb-protocol/src/fbs/diskio.fbs index d5bf1e03a..e339f8226 100644 --- a/lib/crowdb-protocol/src/fbs/diskio.fbs +++ b/lib/crowdb-protocol/src/fbs/diskio.fbs @@ -53,6 +53,10 @@ table FBDiskReadRequest { zone_offset:uint64; size:uint32; test_pattern_offset:uint64; + // Fixed before admission; retries must not refresh this timestamp. + read_create_time_ms:uint64; + // Zero-data negotiation for independently reclaimable immutable packs. + fence_probe:bool = false; } // Read response. The data payload follows as raw bytes (data_size in the @@ -61,6 +65,7 @@ table FBDiskReadResponse { id:uint64; rpc_create_nano:uint64; ret_code:FBDiskIoRetCode; + request_age_checked:bool = false; } table FBDiskFsyncRequest { diff --git a/lib/crowdb-protocol/src/fbs/kv_client.fbs b/lib/crowdb-protocol/src/fbs/kv_client.fbs index d1cef6c41..eaf2f0843 100644 --- a/lib/crowdb-protocol/src/fbs/kv_client.fbs +++ b/lib/crowdb-protocol/src/fbs/kv_client.fbs @@ -19,7 +19,7 @@ namespace crowdb.kv_client.proto; // cache. No Paxos term/epoch reasoning (unlike kv_consensus.fbs). // - Unary requests carry `forwarded: bool` (default false) — the // server-side leader-forward loop-guard that replaces the -// `x-crowdb-kv-forwarded` legacy metadata header (crowdb-rpc has no +// `x-crowdb-paxos-kv-forwarded` legacy metadata header (crowdb-rpc has no // metadata). Set to true when the server re-issues a Linearizable // read to the leader; the leader serves it locally without // re-forwarding. diff --git a/lib/crowdb-protocol/src/key.rs b/lib/crowdb-protocol/src/key.rs index 3a730eeba..6417a8a79 100644 --- a/lib/crowdb-protocol/src/key.rs +++ b/lib/crowdb-protocol/src/key.rs @@ -1,10 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Key encoding for crowdb-kv. +//! Key encoding for crowdb-paxos-kv. //! //! Defines the [`BinaryKey`] and [`TextKey`] traits and all key kinds -//! stored in crowdb-kv. A key kind is a flat struct with hierarchy +//! stored in crowdb-paxos-kv. A key kind is a flat struct with hierarchy //! fields; two encoding traits map the same struct to bytes: //! //! - [`BinaryKey`] — `magic_byte | type_tag:u16 BE | fields BE`, diff --git a/lib/crowdb-protocol/src/key/chunkdb.rs b/lib/crowdb-protocol/src/key/chunkdb.rs index f1474ae80..009f885b5 100644 --- a/lib/crowdb-protocol/src/key/chunkdb.rs +++ b/lib/crowdb-protocol/src/key/chunkdb.rs @@ -8,7 +8,7 @@ //! path keys. They implement [`TextKey`] only (no [`BinaryKey`] — //! these are group-0 only). //! -//! See `doc/design/kv/design-crowdb-kv-group0.md` §3.1 for the key +//! See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §3.1 for the key //! layout and `doc/working/design-r99-dynamic-range-binding.md` §1. use super::encoding::{ diff --git a/lib/crowdb-protocol/src/key/encoding.rs b/lib/crowdb-protocol/src/key/encoding.rs index e68e400f4..4f9de5cc4 100644 --- a/lib/crowdb-protocol/src/key/encoding.rs +++ b/lib/crowdb-protocol/src/key/encoding.rs @@ -45,7 +45,7 @@ impl std::error::Error for KeyError {} // ── Trait ─────────────────────────────────────────────────────── -/// A crowdb-kv binary key. +/// A crowdb-paxos-kv binary key. /// /// Each implementor is a flat struct with a fixed, positional binary /// layout. The layout is frozen once shipped — never change field @@ -195,7 +195,7 @@ pub(super) fn check_exact(buf: &[u8], consumed: usize) -> Result<(), KeyError> { // ── TextKey trait ─────────────────────────────────────────────── -/// A crowdb-kv text-path key (group-0 sysdata encoding). +/// A crowdb-paxos-kv text-path key (group-0 sysdata encoding). /// /// The text encoding is `/magic/type/<field1>/<field2>/...` — a slash- /// delimited path where `magic` is a namespace prefix (`/hw`, `/srv`, diff --git a/lib/crowdb-protocol/src/key/kv_cluster.rs b/lib/crowdb-protocol/src/key/kv_cluster.rs index be634fc26..a5d72fbef 100644 --- a/lib/crowdb-protocol/src/key/kv_cluster.rs +++ b/lib/crowdb-protocol/src/key/kv_cluster.rs @@ -8,7 +8,7 @@ //! `/kv/...` text-path keys. They implement [`TextKey`] only (no //! [`BinaryKey`] — these are group-0 only). //! -//! See `doc/design/kv/design-crowdb-kv-group0.md` §3.1 for the key +//! See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §3.1 for the key //! layout. use super::encoding::{ diff --git a/lib/crowdb-protocol/src/lib.rs b/lib/crowdb-protocol/src/lib.rs index bdf079739..36d1d2d2c 100644 --- a/lib/crowdb-protocol/src/lib.rs +++ b/lib/crowdb-protocol/src/lib.rs @@ -349,7 +349,7 @@ pub mod port; pub use port::alloc::{PortAllocConfig, PortAllocError}; pub use port::ports::{ ServicePort, ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HEALTH_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, - ACCESS_SERVER_ICEBERG_HTTP_BASE, CHUNKDB_HTTP_BASE, CHUNKDB_LISTEN_BASE, CHUNKDB_RPC_BASE, - CHUNK_KV_HTTP_BASE, CHUNK_KV_RPC_BASE, DISKDB_HTTP_BASE, DISKDB_LISTEN_BASE, DISKDB_RPC_BASE, - DISKIO_RPC_BASE, KV_SERVER_LISTEN_BASE, KV_SERVER_MGMT_BASE, WEB_BASE, + ACCESS_SERVER_ICEBERG_HTTP_BASE, CHUNKDB_HTTP_BASE, CHUNKDB_RPC_BASE, CHUNK_KV_HTTP_BASE, + CHUNK_KV_RPC_BASE, DISKDB_HTTP_BASE, DISKDB_RPC_BASE, DISKIO_RPC_BASE, MONITOR_HTTP_BASE, + PAXOS_KV_SERVER_MGMT_BASE, PAXOS_KV_SERVER_RPC_BASE, SSH_PORT, WEB_BASE, }; diff --git a/lib/crowdb-protocol/src/mgmt.rs b/lib/crowdb-protocol/src/mgmt.rs index 1ded4567a..c518424f7 100644 --- a/lib/crowdb-protocol/src/mgmt.rs +++ b/lib/crowdb-protocol/src/mgmt.rs @@ -1,17 +1,17 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! HTTP management API request/response types for `crowdb-kv-server`. +//! HTTP management API request/response types for `crowdb-paxos-kv-server`. //! //! These are the wire shapes for the kv-server's internal HTTP mgmt //! API (lifecycle endpoints, runtime state export). They live in //! `crowdb-protocol` (the single home for cross-component protocol -//! types) so that `crowdb-kv-client`'s `KVClusterAdmin`, +//! types) so that `crowdb-paxos-kv-client`'s `PaxosKvClusterAdmin`, //! `crowdb-console-shared`, `crowdb-web`, and `crowdb-cli` all import from //! one place. //! -//! See `doc/design/kv/design-crowdb-kv-server.md` §2.4 for the endpoint -//! list and `doc/design/kv/design-crowdb-kv-group0.md` §2.2 for the +//! See `doc/design/kv/design-crowdb-paxos-kv-server.md` §2.4 for the endpoint +//! list and `doc/design/kv/design-crowdb-paxos-kv-group0.md` §2.2 for the //! "internal API" decision. #![allow(clippy::struct_field_names)] @@ -265,8 +265,8 @@ pub struct SystemInitResponse { // // These types are the wire shapes for `GET /topology`, `GET /health`, // and `GET /metrics`. They are cross-component (consumed by -// `crowdb-kv-client`, `crowdb-console-shared`, `crowdb-web`, `crowdb-cli`) -// and therefore live here per `design-crowdb-kv-group0.md` §2.4. +// `crowdb-paxos-kv-client`, `crowdb-console-shared`, `crowdb-web`, `crowdb-cli`) +// and therefore live here per `design-crowdb-paxos-kv-group0.md` §2.4. /// Severity of a layer's runtime status. Serializes as a lowercase /// string (`"ok"`, `"degraded"`, `"unhealthy"`). diff --git a/lib/crowdb-protocol/src/mgmt/node/deployment.rs b/lib/crowdb-protocol/src/mgmt/node/deployment.rs index 32f6d460f..e7bf310bf 100644 --- a/lib/crowdb-protocol/src/mgmt/node/deployment.rs +++ b/lib/crowdb-protocol/src/mgmt/node/deployment.rs @@ -4,12 +4,7 @@ //! Accepted startup inputs, distinct from discovered or confirmed membership. use super::setup::{NodeSetupChoices, SetupService, SetupServiceKind, SetupStorage}; -use crate::{ - ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HEALTH_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, - ACCESS_SERVER_ICEBERG_HTTP_BASE, CHUNKDB_HTTP_BASE, CHUNKDB_RPC_BASE, CHUNK_KV_HTTP_BASE, - CHUNK_KV_RPC_BASE, DISKDB_HTTP_BASE, DISKDB_LISTEN_BASE, DISKDB_RPC_BASE, DISKIO_RPC_BASE, - KV_SERVER_LISTEN_BASE, KV_SERVER_MGMT_BASE, -}; +use crate::{ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE}; use serde::{Deserialize, Serialize}; mod public_access; @@ -79,28 +74,27 @@ impl Default for NodeDeploymentPolicy { let listeners = &mut service.listeners; match service.kind { SetupServiceKind::PaxosKv => { - listeners.http_port = Some(KV_SERVER_MGMT_BASE); - listeners.rpc_port = Some(KV_SERVER_LISTEN_BASE); + listeners.http_port = Some(crate::PAXOS_KV_SERVER_MGMT_BASE); + listeners.rpc_port = Some(crate::PAXOS_KV_SERVER_RPC_BASE); } SetupServiceKind::Diskdb => { - listeners.listen_port = Some(DISKDB_LISTEN_BASE); - listeners.http_port = Some(DISKDB_HTTP_BASE); - listeners.rpc_port = Some(DISKDB_RPC_BASE); + listeners.rpc_port = Some(crate::DISKDB_RPC_BASE); + listeners.http_port = Some(crate::DISKDB_HTTP_BASE); } - SetupServiceKind::Diskio => listeners.rpc_port = Some(DISKIO_RPC_BASE), + SetupServiceKind::Diskio => listeners.rpc_port = Some(crate::DISKIO_RPC_BASE), SetupServiceKind::Chunkdb => { - listeners.http_port = Some(CHUNKDB_HTTP_BASE); - listeners.rpc_port = Some(CHUNKDB_RPC_BASE); + listeners.http_port = Some(crate::CHUNKDB_HTTP_BASE); + listeners.rpc_port = Some(crate::CHUNKDB_RPC_BASE); } SetupServiceKind::ChunkKv => { - listeners.http_port = Some(CHUNK_KV_HTTP_BASE); - listeners.rpc_port = Some(CHUNK_KV_RPC_BASE); + listeners.http_port = Some(crate::CHUNK_KV_HTTP_BASE); + listeners.rpc_port = Some(crate::CHUNK_KV_RPC_BASE); } SetupServiceKind::AccessServer => { - listeners.s3_port = Some(ACCESS_SERVER_HTTP_BASE); - listeners.http_port = Some(ACCESS_SERVER_ICEBERG_HTTP_BASE); - listeners.dataset_port = Some(ACCESS_SERVER_DATASET_HTTP_BASE); - listeners.health_port = Some(ACCESS_SERVER_HEALTH_HTTP_BASE); + listeners.s3_port = Some(crate::ACCESS_SERVER_HTTP_BASE); + listeners.http_port = Some(crate::ACCESS_SERVER_ICEBERG_HTTP_BASE); + listeners.dataset_port = Some(crate::ACCESS_SERVER_DATASET_HTTP_BASE); + listeners.health_port = Some(crate::ACCESS_SERVER_HEALTH_HTTP_BASE); } } } @@ -152,7 +146,6 @@ impl NodeDeploymentPolicy { for service in self.setup.services.iter().filter(|service| service.enabled) { let ports = &service.listeners; if [ - ports.listen_port, ports.http_port, ports.rpc_port, ports.s3_port, @@ -161,7 +154,7 @@ impl NodeDeploymentPolicy { ] .into_iter() .flatten() - .any(|port| matches!(port, 2222 | 9090 | 9095)) + .any(|port| [crate::SSH_PORT, crate::WEB_BASE, crate::MONITOR_HTTP_BASE].contains(&port)) { return Err("service listener conflicts with node management or bootstrap transport"); } diff --git a/lib/crowdb-protocol/src/mgmt/node/setup.rs b/lib/crowdb-protocol/src/mgmt/node/setup.rs index 847c0de49..a39e9a01e 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup.rs @@ -33,8 +33,6 @@ impl SetupServiceKind { #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SetupListeners { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub listen_port: Option<u16>, pub http_port: Option<u16>, pub rpc_port: Option<u16>, pub s3_port: Option<u16>, diff --git a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs index 4de75f2d7..7e33e7fed 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs @@ -144,11 +144,7 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet<u16>) -> bool { || listeners.health_port.is_some() || listeners.dataset_port.is_some())) || (service.kind == AccessServer && listeners.rpc_port.is_some()) - || (service.kind != Diskdb && listeners.listen_port.is_some()) || (service.kind == Diskio && listeners.http_port.is_some()) - || (service.kind == Diskdb - && (listeners.listen_port.is_some() != listeners.http_port.is_some() - || (listeners.listen_port.is_some() && listeners.rpc_port.is_none()))) { return false; } @@ -156,7 +152,6 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet<u16>) -> bool { return true; } for port in [ - listeners.listen_port, listeners.http_port, listeners.rpc_port, listeners.s3_port, @@ -170,10 +165,5 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet<u16>) -> bool { return false; } } - if service.kind == Diskdb && listeners.listen_port.is_none() { - if let Some(port) = listeners.rpc_port { - return port <= 65533 && ports.insert(port + 1) && ports.insert(port + 2); - } - } true } diff --git a/lib/crowdb-protocol/src/port/alloc.rs b/lib/crowdb-protocol/src/port/alloc.rs index bc51d9297..83d70ceb1 100644 --- a/lib/crowdb-protocol/src/port/alloc.rs +++ b/lib/crowdb-protocol/src/port/alloc.rs @@ -389,8 +389,8 @@ mod tests { #[test] fn alloc_single_port() { let cfg = unique_cfg(); - let base = ServicePort::KvServerMgmt.base() + cfg.offset; - let port = alloc_port(ServicePort::KvServerMgmt, 0, &cfg).unwrap(); + let base = ServicePort::PaxosKvServerMgmt.base() + cfg.offset; + let port = alloc_port(ServicePort::PaxosKvServerMgmt, 0, &cfg).unwrap(); assert_eq!(port, base); let claimed = read_claimed_ports(&cfg).unwrap(); assert!(claimed.contains(&port)); @@ -400,9 +400,9 @@ mod tests { #[test] fn alloc_second_instance() { let cfg = unique_cfg(); - let base = ServicePort::KvServerMgmt.base() + cfg.offset; - let p0 = alloc_port(ServicePort::KvServerMgmt, 0, &cfg).unwrap(); - let p1 = alloc_port(ServicePort::KvServerMgmt, 1, &cfg).unwrap(); + let base = ServicePort::PaxosKvServerMgmt.base() + cfg.offset; + let p0 = alloc_port(ServicePort::PaxosKvServerMgmt, 0, &cfg).unwrap(); + let p1 = alloc_port(ServicePort::PaxosKvServerMgmt, 1, &cfg).unwrap(); assert_eq!(p0, base); assert_eq!(p1, base + 1); cleanup(&cfg); @@ -411,11 +411,11 @@ mod tests { #[test] fn alloc_skips_claimed() { let cfg = unique_cfg(); - let base = ServicePort::KvServerMgmt.base() + cfg.offset; - let p0 = alloc_port(ServicePort::KvServerMgmt, 0, &cfg).unwrap(); + let base = ServicePort::PaxosKvServerMgmt.base() + cfg.offset; + let p0 = alloc_port(ServicePort::PaxosKvServerMgmt, 0, &cfg).unwrap(); assert_eq!(p0, base); // Allocating instance 0 again should skip to instance 1. - let p1 = alloc_port(ServicePort::KvServerMgmt, 0, &cfg).unwrap(); + let p1 = alloc_port(ServicePort::PaxosKvServerMgmt, 0, &cfg).unwrap(); assert_eq!(p1, base + 1); cleanup(&cfg); } @@ -423,8 +423,8 @@ mod tests { #[test] fn alloc_range() { let cfg = unique_cfg(); - let base = ServicePort::KvServerListen.base() + cfg.offset; - let ports = alloc_port_range(ServicePort::KvServerListen, 0, 3, &cfg).unwrap(); + let base = ServicePort::PaxosKvServerRpc.base() + cfg.offset; + let ports = alloc_port_range(ServicePort::PaxosKvServerRpc, 0, 3, &cfg).unwrap(); assert_eq!(ports.len(), 3); assert_eq!(ports[0], base); assert_eq!(ports[1], base + 1); @@ -439,7 +439,7 @@ mod tests { #[test] fn alloc_range_zero_count() { let cfg = unique_cfg(); - let ports = alloc_port_range(ServicePort::KvServerListen, 0, 0, &cfg).unwrap(); + let ports = alloc_port_range(ServicePort::PaxosKvServerRpc, 0, 0, &cfg).unwrap(); assert!(ports.is_empty()); cleanup(&cfg); } @@ -447,9 +447,9 @@ mod tests { #[test] fn mark_failed_skips_port() { let cfg = unique_cfg(); - let base = ServicePort::KvServerMgmt.base() + cfg.offset; + let base = ServicePort::PaxosKvServerMgmt.base() + cfg.offset; mark_failed(base, &cfg).unwrap(); - let port = alloc_port(ServicePort::KvServerMgmt, 0, &cfg).unwrap(); + let port = alloc_port(ServicePort::PaxosKvServerMgmt, 0, &cfg).unwrap(); assert_ne!(port, base, "failed port must be skipped"); cleanup(&cfg); } @@ -457,7 +457,7 @@ mod tests { #[test] fn reset_claims_clears() { let cfg = unique_cfg(); - alloc_port(ServicePort::KvServerMgmt, 0, &cfg).unwrap(); + alloc_port(ServicePort::PaxosKvServerMgmt, 0, &cfg).unwrap(); let claimed = read_claimed_ports(&cfg).unwrap(); assert!(!claimed.is_empty()); reset_claims(&cfg).unwrap(); @@ -472,16 +472,16 @@ mod tests { let extra = 50; let base_offset = base_cfg.offset; let cfg = base_cfg.with_offset(base_offset + extra); - let port = alloc_port(ServicePort::KvServerMgmt, 0, &cfg).unwrap(); - assert_eq!(port, ServicePort::KvServerMgmt.base() + base_offset + extra); + let port = alloc_port(ServicePort::PaxosKvServerMgmt, 0, &cfg).unwrap(); + assert_eq!(port, ServicePort::PaxosKvServerMgmt.base() + base_offset + extra); cleanup(&cfg); } #[test] fn different_services_do_not_collide() { let cfg = unique_cfg(); - let kv_mgmt = alloc_port(ServicePort::KvServerMgmt, 0, &cfg).unwrap(); - let kv_listen = alloc_port(ServicePort::KvServerListen, 0, &cfg).unwrap(); + let kv_mgmt = alloc_port(ServicePort::PaxosKvServerMgmt, 0, &cfg).unwrap(); + let kv_listen = alloc_port(ServicePort::PaxosKvServerRpc, 0, &cfg).unwrap(); let diskdb = alloc_port(ServicePort::DiskdbRpc, 0, &cfg).unwrap(); assert_ne!(kv_mgmt, kv_listen); assert_ne!(kv_mgmt, diskdb); diff --git a/lib/crowdb-protocol/src/port/ports.rs b/lib/crowdb-protocol/src/port/ports.rs index b335fac3a..1995d064d 100644 --- a/lib/crowdb-protocol/src/port/ports.rs +++ b/lib/crowdb-protocol/src/port/ports.rs @@ -1,135 +1,94 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -//! Default port allocation for CROWDB services. +//! Default listener ports. Each listener kind has a disjoint 1000-port block, +//! Console web and Access start at 6135, 7135, 8135 and 9135. Management and RPC +//! listeners of the same server are 1000 ports apart. Instances use stride 1: +//! host-network nodes use base + node offset (0, 1, 2 for three nodes). +//! The allocator checks claims and bind availability; configured endpoints +//! remain authoritative and are not migrated by changing these defaults. //! -//! Each service type gets a **1000-port block** with a shared prefix -//! (same kind of server = same leading digits), all ports **>10000**. -//! Access protocols each have their own block. Other services split -//! their block into listener sub-ranges. All -//! services use **stride 1** — no paired-port logic. Port 0 is never -//! used (rejected everywhere by CLI parse and the port allocator). -//! -//! Sub-ranges within a block may overlap (e.g. kv-mgmt 10000-10499 -//! and kv-listen 10100-10599 share 10100-10499). This is safe because -//! the port allocator uses a shared per-process claim file plus bind -//! probes — a port claimed by one service type is never handed to -//! another. -//! -//! ## Port map -//! -//! - `10000`–`10999` — crowdb-kv-server (prefix 10) -//! - `10000`–`10499` — HTTP management API (stride 1) -//! - `10100`–`10599` — main `PxKvStore` listener (hosts both -//! consensus and client crowdb-rpc handlers; stride 1) -//! - `10600`–`10999` — spare -//! - `11000`–`11999` — crowdb-diskdb (prefix 11) -//! - `11000`–`11499` — main listener (stride 1) -//! - `11100`–`11599` — HTTP management API (stride 1; independent -//! of listen — no paired-port invariant) -//! - `11200`–`11699` — crowdb-rpc listener (stride 1) -//! - `11700`–`11999` — spare -//! - `12000`–`12999` — crowdb-chunkdb (prefix 12) -//! - `12100`–`12599` — HTTP management API (stride 1) -//! - `12200`–`12699` — crowdb-rpc listener (stride 1) -//! - `12700`–`12999` — spare -//! - `13000`–`13999` — crowdb-diskio (prefix 13) -//! - `13000`–`13499` — crowdb-rpc listener (stride 1) -//! - `13500`–`13999` — spare -//! - `14000`–`14999` — crowdb-web (prefix 14) -//! - `14000`–`14499` — HTTP service (stride 1) -//! - `14500`–`14999` — spare -//! - `15000`–`15999` — crowdb-chunk-kv-server (prefix 15) -//! - `15100`–`15599` — HTTP management API (stride 1) -//! - `15200`–`15699` — crowdb-rpc listener (stride 1) -//! - `15700`–`15999` — spare -//! - `16000`–`16999` — crowdb-access-server S3 HTTP (prefix 16) -//! - `17000`–`17999` — crowdb-access-server Iceberg HTTP (prefix 17) -//! - `18000`–`18999` — crowdb-access-server Dataset HTTP (prefix 18) -//! - `19000`–`19999` — crowdb-access-server health HTTP (prefix 19) -//! -//! The group-0 kv-server mgmt port (`10000`) is the famous bootstrap -//! discovery port — any client can contact group-0 to read the service -//! registry and learn all living services' IP + port. -//! -//! Future service types should pick a base outside these ranges (next -//! free prefix: 19xxx) and document it here. - -/// crowdb-kv-server HTTP management API — base port. Also the famous +//! Paxos-KV: 10135 management / 11135 RPC. +//! `DiskDB`: 13135 management / 14135 RPC. +//! `ChunkDB`: 16135 management / 17135 RPC. +//! `DiskIO`: 18135 RPC. Console web: 6135 HTTP. +//! Chunk-KV: 20135 management / 21135 RPC. +//! Access: 7135 S3 / 8135 Iceberg / 9135 Dataset / 25135 health. +//! Monitor: 26135 HTTP. SSH: 2222 (no instance block). +//! Rust callers import these constants. Non-Rust defaults are generated by +//! `pixi run python tools/ports/generate.py`; `--check` detects stale outputs. + +/// crowdb-paxos-kv-server HTTP management API — base port. Also the famous /// group-0 bootstrap discovery port. -pub const KV_SERVER_MGMT_BASE: u16 = 10000; +pub const PAXOS_KV_SERVER_MGMT_BASE: u16 = 10135; -/// crowdb-kv-server main `PxKvStore` listener — base port (port pool). +/// crowdb-paxos-kv-server main `PxKvStore` listener — base port (port pool). /// Hosts both consensus and client crowdb-rpc handlers on the same /// listener (RPC port collapse — no separate consensus/client ports). -pub const KV_SERVER_LISTEN_BASE: u16 = 10100; - -/// crowdb-diskdb main listener — base port. -pub const DISKDB_LISTEN_BASE: u16 = 11000; +pub const PAXOS_KV_SERVER_RPC_BASE: u16 = 11135; -/// crowdb-diskdb HTTP management API — base port. Independent of -/// `DISKDB_LISTEN_BASE` (no paired-port invariant). -pub const DISKDB_HTTP_BASE: u16 = 11100; +/// crowdb-diskdb HTTP management API — base port. +pub const DISKDB_HTTP_BASE: u16 = 13135; /// crowdb-diskdb crowdb-rpc listener — base port. -pub const DISKDB_RPC_BASE: u16 = 11200; - -/// crowdb-chunkdb main listener — base port. Vestigial: the chunkdb -/// server binds `rpc_listen_addr` and `http_listen_addr` only; this -/// range is reserved for future use. -pub const CHUNKDB_LISTEN_BASE: u16 = 12000; +pub const DISKDB_RPC_BASE: u16 = 14135; /// crowdb-chunkdb HTTP management API — base port. -pub const CHUNKDB_HTTP_BASE: u16 = 12100; +pub const CHUNKDB_HTTP_BASE: u16 = 16135; /// crowdb-chunkdb crowdb-rpc listener — base port. -pub const CHUNKDB_RPC_BASE: u16 = 12200; +pub const CHUNKDB_RPC_BASE: u16 = 17135; /// crowdb-diskio crowdb-rpc listener — base port. -pub const DISKIO_RPC_BASE: u16 = 13000; +pub const DISKIO_RPC_BASE: u16 = 18135; /// crowdb-web HTTP service — base port. -pub const WEB_BASE: u16 = 14000; +pub const WEB_BASE: u16 = 6135; /// crowdb-chunk-kv-server HTTP management API — base port. -pub const CHUNK_KV_HTTP_BASE: u16 = 15100; +pub const CHUNK_KV_HTTP_BASE: u16 = 20135; /// crowdb-chunk-kv-server crowdb-rpc listener — base port. -pub const CHUNK_KV_RPC_BASE: u16 = 15200; +pub const CHUNK_KV_RPC_BASE: u16 = 21135; /// crowdb-access-server S3 HTTP service — base port. -pub const ACCESS_SERVER_HTTP_BASE: u16 = 16000; +pub const ACCESS_SERVER_HTTP_BASE: u16 = 7135; /// crowdb-access-server Iceberg HTTP service — base port. -pub const ACCESS_SERVER_ICEBERG_HTTP_BASE: u16 = 17000; +pub const ACCESS_SERVER_ICEBERG_HTTP_BASE: u16 = 8135; /// crowdb-access-server Dataset HTTP service — base port. -pub const ACCESS_SERVER_DATASET_HTTP_BASE: u16 = 18000; +pub const ACCESS_SERVER_DATASET_HTTP_BASE: u16 = 9135; /// Independent Access health listener. -pub const ACCESS_SERVER_HEALTH_HTTP_BASE: u16 = 19000; +pub const ACCESS_SERVER_HEALTH_HTTP_BASE: u16 = 25135; + +/// Node monitor discovery and management HTTP listener. +pub const MONITOR_HTTP_BASE: u16 = 26135; + +/// Container SSH listener; outside service instance allocation. +pub const SSH_PORT: u16 = 2222; + +/// Standard SSH endpoint for externally managed hosts. +pub const HOST_SSH_PORT: u16 = 22; /// CROWDB service type for default port allocation. /// /// Use [`ServicePort::port`] to compute the listen port for the /// `instance`-th instance of a service type on a given node (0-based). -/// The base constants (e.g. [`KV_SERVER_MGMT_BASE`]) are re-exported +/// The base constants (e.g. [`PAXOS_KV_SERVER_MGMT_BASE`]) are re-exported /// for contexts that need a plain `const` value (clap `default_value_t`). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ServicePort { - /// crowdb-kv-server HTTP management API. - KvServerMgmt, - /// crowdb-kv-server main `PxKvStore` listener (port pool; hosts + /// crowdb-paxos-kv-server HTTP management API. + PaxosKvServerMgmt, + /// crowdb-paxos-kv-server main `PxKvStore` listener (port pool; hosts /// both consensus and client crowdb-rpc). - KvServerListen, - /// crowdb-diskdb main listener. - DiskdbListen, + PaxosKvServerRpc, /// crowdb-diskdb HTTP management API. DiskdbHttp, /// crowdb-diskdb crowdb-rpc listener. DiskdbRpc, - /// crowdb-chunkdb main listener (vestigial — range reserved). - ChunkdbListen, /// crowdb-chunkdb HTTP management API. ChunkdbHttp, /// crowdb-chunkdb crowdb-rpc listener. @@ -150,6 +109,8 @@ pub enum ServicePort { AccessServerDatasetHttp, /// Internal Access liveness/readiness HTTP service. AccessServerHealthHttp, + /// Node monitor discovery and management HTTP. + MonitorHttp, } impl ServicePort { @@ -157,12 +118,10 @@ impl ServicePort { #[must_use] pub const fn name(self) -> &'static str { match self { - Self::KvServerMgmt => "kv_server_mgmt", - Self::KvServerListen => "kv_server_listen", - Self::DiskdbListen => "diskdb_listen", + Self::PaxosKvServerMgmt => "kv_server_mgmt", + Self::PaxosKvServerRpc => "kv_server_rpc", Self::DiskdbHttp => "diskdb_http", Self::DiskdbRpc => "diskdb_rpc", - Self::ChunkdbListen => "chunkdb_listen", Self::ChunkdbHttp => "chunkdb_http", Self::ChunkdbRpc => "chunkdb_rpc", Self::DiskioRpc => "diskio_rpc", @@ -173,6 +132,7 @@ impl ServicePort { Self::AccessServerIcebergHttp => "access_server_iceberg_http", Self::AccessServerDatasetHttp => "access_server_dataset_http", Self::AccessServerHealthHttp => "access_server_health_http", + Self::MonitorHttp => "monitor_http", } } @@ -180,12 +140,10 @@ impl ServicePort { #[must_use] pub const fn base(self) -> u16 { match self { - Self::KvServerMgmt => KV_SERVER_MGMT_BASE, - Self::KvServerListen => KV_SERVER_LISTEN_BASE, - Self::DiskdbListen => DISKDB_LISTEN_BASE, + Self::PaxosKvServerMgmt => PAXOS_KV_SERVER_MGMT_BASE, + Self::PaxosKvServerRpc => PAXOS_KV_SERVER_RPC_BASE, Self::DiskdbHttp => DISKDB_HTTP_BASE, Self::DiskdbRpc => DISKDB_RPC_BASE, - Self::ChunkdbListen => CHUNKDB_LISTEN_BASE, Self::ChunkdbHttp => CHUNKDB_HTTP_BASE, Self::ChunkdbRpc => CHUNKDB_RPC_BASE, Self::DiskioRpc => DISKIO_RPC_BASE, @@ -196,6 +154,7 @@ impl ServicePort { Self::AccessServerIcebergHttp => ACCESS_SERVER_ICEBERG_HTTP_BASE, Self::AccessServerDatasetHttp => ACCESS_SERVER_DATASET_HTTP_BASE, Self::AccessServerHealthHttp => ACCESS_SERVER_HEALTH_HTTP_BASE, + Self::MonitorHttp => MONITOR_HTTP_BASE, } } @@ -214,18 +173,10 @@ impl ServicePort { self.base() + instance * self.stride() } - /// Sub-range size (number of ports) for this service type's - /// listener kind. Most listener kinds get 500 ports — large enough - /// for parallel E2E test suites that deploy 80+ nodes per service - /// type. Each access protocol gets a full independent 1000-port block. + /// Number of instance ports reserved for each listener kind. #[must_use] pub const fn range_size(self) -> u16 { - match self { - Self::AccessServerHttp - | Self::AccessServerIcebergHttp - | Self::AccessServerDatasetHttp - | Self::AccessServerHealthHttp => 1000, - _ => 500, - } + let _ = self; + 1000 } } diff --git a/lib/crowdb-protocol/src/sysdata.rs b/lib/crowdb-protocol/src/sysdata.rs index e2e423f68..d778ac838 100644 --- a/lib/crowdb-protocol/src/sysdata.rs +++ b/lib/crowdb-protocol/src/sysdata.rs @@ -4,11 +4,11 @@ //! Entry return types for group-0 sysdata reads. //! //! These are the decoded form of a text-path key + its JSON value, -//! produced by `HardwareClient` / `KVClusterMetaClient` reads. The key +//! produced by `HardwareClient` / `PaxosKvClusterMetaClient` reads. The key //! fields (parsed from the path) are included alongside the value so //! callers do not need to re-parse the path. //! -//! See `doc/design/kv/design-crowdb-kv-group0.md` §3.3. +//! See `doc/design/kv/design-crowdb-paxos-kv-group0.md` §3.3. use crate::common_type::{DiskGroupId, NodeId, RackId}; use crate::diskdb::rpc::DiskGroupValue; diff --git a/lib/crowdb-protocol/src/types/kv_client.rs b/lib/crowdb-protocol/src/types/kv_client.rs index dc59ff6a1..789623cf8 100644 --- a/lib/crowdb-protocol/src/types/kv_client.rs +++ b/lib/crowdb-protocol/src/types/kv_client.rs @@ -10,7 +10,7 @@ clippy::too_many_lines )] -//! Hand-written Rust types replacing the prost-generated `crowdb_kv.rpc` +//! Hand-written Rust types replacing the prost-generated `crowdb_paxos_kv.rpc` //! KV client types. API-compatible with the former proto-generated structs. use bytes::Bytes; diff --git a/lib/crowdb-protocol/src/types/kv_consensus.rs b/lib/crowdb-protocol/src/types/kv_consensus.rs index f504b1331..3c3a9c366 100644 --- a/lib/crowdb-protocol/src/types/kv_consensus.rs +++ b/lib/crowdb-protocol/src/types/kv_consensus.rs @@ -10,7 +10,7 @@ clippy::too_many_lines )] -//! Hand-written Rust types replacing the prost-generated `crowdb_kv.rpc` +//! Hand-written Rust types replacing the prost-generated `crowdb_paxos_kv.rpc` //! consensus types. API-compatible with the former //! proto-generated structs. diff --git a/lib/crowdb-protocol/tests/fb_schema_test.rs b/lib/crowdb-protocol/tests/fb_schema_test.rs index a4c09ce7e..1e1f440c5 100644 --- a/lib/crowdb-protocol/tests/fb_schema_test.rs +++ b/lib/crowdb-protocol/tests/fb_schema_test.rs @@ -198,6 +198,8 @@ fn disk_read_request_round_trip() { zone_offset: 8192, size: 2048, test_pattern_offset: 0x1234, + read_create_time_ms: 0, + fence_probe: false, }, ); fbb.finish(req, None); diff --git a/lib/crowdb-protocol/tests/node_deployment_test.rs b/lib/crowdb-protocol/tests/node_deployment_test.rs index e2a70df6b..3814a557f 100644 --- a/lib/crowdb-protocol/tests/node_deployment_test.rs +++ b/lib/crowdb-protocol/tests/node_deployment_test.rs @@ -23,9 +23,8 @@ fn fresh_local_policy_has_one_voter_and_all_public_access_listeners() { )); assert!(policy.setup.services.iter().all(|service| service.enabled)); let diskdb = &policy.setup.services[1].listeners; - assert_eq!(diskdb.listen_port, Some(11000)); - assert_eq!(diskdb.http_port, Some(11100)); - assert_eq!(diskdb.rpc_port, Some(11200)); + assert_eq!(diskdb.http_port, Some(crowdb_protocol::DISKDB_HTTP_BASE)); + assert_eq!(diskdb.rpc_port, Some(crowdb_protocol::DISKDB_RPC_BASE)); let access = policy .setup .services @@ -198,11 +197,16 @@ fn invalid_policies_are_rejected_before_any_resolution_can_replace_them() { .is_err()); } let mut duplicate_port = valid; - duplicate_port.setup.services[5].listeners.dataset_port = Some(ACCESS_SERVER_ICEBERG_HTTP_BASE); + duplicate_port.setup.services[5].listeners.dataset_port = + Some(crowdb_protocol::ACCESS_SERVER_ICEBERG_HTTP_BASE); assert!(duplicate_port.validate().is_err()); duplicate_port.setup.services[5].listeners.dataset_port = None; assert!(duplicate_port.validate().is_err()); - for reserved in [2222, 9090, 9095] { + for reserved in [ + crowdb_protocol::SSH_PORT, + crowdb_protocol::WEB_BASE, + crowdb_protocol::MONITOR_HTTP_BASE, + ] { let mut policy = NodeDeploymentPolicy::default(); policy.setup.services[5].listeners.dataset_port = Some(reserved); assert!(policy.validate().is_err()); diff --git a/lib/crowdb-protocol/tests/node_setup_test.rs b/lib/crowdb-protocol/tests/node_setup_test.rs index cf2bbbe2d..a802ca384 100644 --- a/lib/crowdb-protocol/tests/node_setup_test.rs +++ b/lib/crowdb-protocol/tests/node_setup_test.rs @@ -102,14 +102,16 @@ fn omitted_choices_select_six_services_and_preallocated_standard_capacity() { } #[test] -fn service_selection_and_reserved_diskdb_listener_range_cannot_be_ambiguous() { +fn service_selection_and_active_diskdb_listeners_cannot_be_ambiguous() { let mut plan = request(); plan.choices.services[1].listeners.rpc_port = Some(32000); - plan.choices.services[2].listeners.rpc_port = Some(32002); + plan.choices.services[2].listeners.rpc_port = Some(32000); assert!(plan.validate().is_err()); - plan.choices.services[2].listeners.rpc_port = Some(32003); + plan.choices.services[2].listeners.rpc_port = Some(32001); plan.validate().unwrap(); - plan.choices.services[1].listeners.rpc_port = Some(65534); + plan.choices.services[1].listeners.rpc_port = Some(65535); + plan.validate().unwrap(); + plan.choices.services[1].listeners.http_port = Some(65535); assert!(plan.validate().is_err()); let mut duplicated = request(); duplicated.choices.services[1] = duplicated.choices.services[0].clone(); diff --git a/lib/crowdb-protocol/tests/ports_test.rs b/lib/crowdb-protocol/tests/ports_test.rs index fa2fff335..53cb80b45 100644 --- a/lib/crowdb-protocol/tests/ports_test.rs +++ b/lib/crowdb-protocol/tests/ports_test.rs @@ -7,21 +7,19 @@ use crowdb_protocol::ServicePort; use crowdb_protocol::{ ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HEALTH_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, - ACCESS_SERVER_ICEBERG_HTTP_BASE, CHUNKDB_HTTP_BASE, CHUNKDB_LISTEN_BASE, CHUNKDB_RPC_BASE, - CHUNK_KV_HTTP_BASE, CHUNK_KV_RPC_BASE, DISKDB_HTTP_BASE, DISKDB_LISTEN_BASE, DISKDB_RPC_BASE, - DISKIO_RPC_BASE, KV_SERVER_LISTEN_BASE, KV_SERVER_MGMT_BASE, WEB_BASE, + ACCESS_SERVER_ICEBERG_HTTP_BASE, CHUNKDB_HTTP_BASE, CHUNKDB_RPC_BASE, CHUNK_KV_HTTP_BASE, + CHUNK_KV_RPC_BASE, DISKDB_HTTP_BASE, DISKDB_RPC_BASE, DISKIO_RPC_BASE, PAXOS_KV_SERVER_MGMT_BASE, + PAXOS_KV_SERVER_RPC_BASE, WEB_BASE, }; // ── base constants match enum ────────────────────────────────── #[test] fn base_constants_match_enum_base() { - assert_eq!(ServicePort::KvServerMgmt.base(), KV_SERVER_MGMT_BASE); - assert_eq!(ServicePort::KvServerListen.base(), KV_SERVER_LISTEN_BASE); - assert_eq!(ServicePort::DiskdbListen.base(), DISKDB_LISTEN_BASE); + assert_eq!(ServicePort::PaxosKvServerMgmt.base(), PAXOS_KV_SERVER_MGMT_BASE); + assert_eq!(ServicePort::PaxosKvServerRpc.base(), PAXOS_KV_SERVER_RPC_BASE); assert_eq!(ServicePort::DiskdbHttp.base(), DISKDB_HTTP_BASE); assert_eq!(ServicePort::DiskdbRpc.base(), DISKDB_RPC_BASE); - assert_eq!(ServicePort::ChunkdbListen.base(), CHUNKDB_LISTEN_BASE); assert_eq!(ServicePort::ChunkdbHttp.base(), CHUNKDB_HTTP_BASE); assert_eq!(ServicePort::ChunkdbRpc.base(), CHUNKDB_RPC_BASE); assert_eq!(ServicePort::DiskioRpc.base(), DISKIO_RPC_BASE); @@ -39,30 +37,28 @@ fn base_constants_match_enum_base() { ); } -// ── known defaults (new port map, all >10000) ────────────────── +// ── known defaults (default port map) ────────────────── #[test] fn known_base_ports() { - assert_eq!(KV_SERVER_MGMT_BASE, 10000); - assert_eq!(KV_SERVER_LISTEN_BASE, 10100); - assert_eq!(DISKDB_LISTEN_BASE, 11000); - assert_eq!(DISKDB_HTTP_BASE, 11100); - assert_eq!(DISKDB_RPC_BASE, 11200); - assert_eq!(CHUNKDB_LISTEN_BASE, 12000); - assert_eq!(CHUNKDB_HTTP_BASE, 12100); - assert_eq!(CHUNKDB_RPC_BASE, 12200); - assert_eq!(DISKIO_RPC_BASE, 13000); - assert_eq!(WEB_BASE, 14000); - assert_eq!(CHUNK_KV_HTTP_BASE, 15100); - assert_eq!(CHUNK_KV_RPC_BASE, 15200); - assert_eq!(ACCESS_SERVER_HTTP_BASE, 16000); - assert_eq!(ACCESS_SERVER_ICEBERG_HTTP_BASE, 17000); - assert_eq!(ACCESS_SERVER_DATASET_HTTP_BASE, 18000); + assert_eq!(PAXOS_KV_SERVER_MGMT_BASE, 10135); + assert_eq!(PAXOS_KV_SERVER_RPC_BASE, 11135); + assert_eq!(DISKDB_HTTP_BASE, 13135); + assert_eq!(DISKDB_RPC_BASE, 14135); + assert_eq!(CHUNKDB_HTTP_BASE, 16135); + assert_eq!(CHUNKDB_RPC_BASE, 17135); + assert_eq!(DISKIO_RPC_BASE, 18135); + assert_eq!(WEB_BASE, 6135); + assert_eq!(CHUNK_KV_HTTP_BASE, 20135); + assert_eq!(CHUNK_KV_RPC_BASE, 21135); + assert_eq!(ACCESS_SERVER_HTTP_BASE, 7135); + assert_eq!(ACCESS_SERVER_ICEBERG_HTTP_BASE, 8135); + assert_eq!(ACCESS_SERVER_DATASET_HTTP_BASE, 9135); assert_eq!( ServicePort::AccessServerHealthHttp.base(), ACCESS_SERVER_HEALTH_HTTP_BASE ); - assert_eq!(ACCESS_SERVER_HEALTH_HTTP_BASE, 19000); + assert_eq!(ACCESS_SERVER_HEALTH_HTTP_BASE, 25135); } // ── stride (all stride 1 — no paired-port logic) ─────────────── @@ -70,12 +66,10 @@ fn known_base_ports() { #[test] fn all_services_have_stride_one() { for svc in [ - ServicePort::KvServerMgmt, - ServicePort::KvServerListen, - ServicePort::DiskdbListen, + ServicePort::PaxosKvServerMgmt, + ServicePort::PaxosKvServerRpc, ServicePort::DiskdbHttp, ServicePort::DiskdbRpc, - ServicePort::ChunkdbListen, ServicePort::ChunkdbHttp, ServicePort::ChunkdbRpc, ServicePort::DiskioRpc, @@ -86,6 +80,7 @@ fn all_services_have_stride_one() { ServicePort::AccessServerIcebergHttp, ServicePort::AccessServerDatasetHttp, ServicePort::AccessServerHealthHttp, + ServicePort::MonitorHttp, ] { assert_eq!(svc.stride(), 1, "{svc:?} must have stride 1"); } @@ -96,12 +91,10 @@ fn all_services_have_stride_one() { #[test] fn port_instance_zero_is_base() { for svc in [ - ServicePort::KvServerMgmt, - ServicePort::KvServerListen, - ServicePort::DiskdbListen, + ServicePort::PaxosKvServerMgmt, + ServicePort::PaxosKvServerRpc, ServicePort::DiskdbHttp, ServicePort::DiskdbRpc, - ServicePort::ChunkdbListen, ServicePort::ChunkdbHttp, ServicePort::ChunkdbRpc, ServicePort::DiskioRpc, @@ -112,6 +105,7 @@ fn port_instance_zero_is_base() { ServicePort::AccessServerIcebergHttp, ServicePort::AccessServerDatasetHttp, ServicePort::AccessServerHealthHttp, + ServicePort::MonitorHttp, ] { assert_eq!(svc.port(0), svc.base()); } @@ -119,35 +113,33 @@ fn port_instance_zero_is_base() { #[test] fn port_increments_by_one() { - assert_eq!(ServicePort::KvServerMgmt.port(0), 10000); - assert_eq!(ServicePort::KvServerMgmt.port(1), 10001); - assert_eq!(ServicePort::KvServerMgmt.port(99), 10099); + assert_eq!(ServicePort::PaxosKvServerMgmt.port(0), 10135); + assert_eq!(ServicePort::PaxosKvServerMgmt.port(1), 10136); + assert_eq!(ServicePort::PaxosKvServerMgmt.port(99), 10234); - assert_eq!(ServicePort::KvServerListen.port(0), 10100); - assert_eq!(ServicePort::KvServerListen.port(1), 10101); - assert_eq!(ServicePort::KvServerListen.port(99), 10199); + assert_eq!(ServicePort::PaxosKvServerRpc.port(0), 11135); + assert_eq!(ServicePort::PaxosKvServerRpc.port(1), 11136); + assert_eq!(ServicePort::PaxosKvServerRpc.port(99), 11234); - assert_eq!(ServicePort::DiskdbRpc.port(0), 11200); - assert_eq!(ServicePort::DiskdbRpc.port(1), 11201); - assert_eq!(ServicePort::DiskdbRpc.port(99), 11299); + assert_eq!(ServicePort::DiskdbRpc.port(0), 14135); + assert_eq!(ServicePort::DiskdbRpc.port(1), 14136); + assert_eq!(ServicePort::DiskdbRpc.port(99), 14234); - assert_eq!(ServicePort::DiskioRpc.port(0), 13000); - assert_eq!(ServicePort::DiskioRpc.port(1), 13001); - assert_eq!(ServicePort::DiskioRpc.port(99), 13099); + assert_eq!(ServicePort::DiskioRpc.port(0), 18135); + assert_eq!(ServicePort::DiskioRpc.port(1), 18136); + assert_eq!(ServicePort::DiskioRpc.port(99), 18234); } // ── non-overlap across service types ─────────────────────────── #[test] fn port_ranges_do_not_overlap() { - // Each service type's first 100 instances (full sub-range). + // Check every declared instance, including the edges of adjacent blocks. let all_services = [ - ServicePort::KvServerMgmt, - ServicePort::KvServerListen, - ServicePort::DiskdbListen, + ServicePort::PaxosKvServerMgmt, + ServicePort::PaxosKvServerRpc, ServicePort::DiskdbHttp, ServicePort::DiskdbRpc, - ServicePort::ChunkdbListen, ServicePort::ChunkdbHttp, ServicePort::ChunkdbRpc, ServicePort::DiskioRpc, @@ -158,11 +150,12 @@ fn port_ranges_do_not_overlap() { ServicePort::AccessServerIcebergHttp, ServicePort::AccessServerDatasetHttp, ServicePort::AccessServerHealthHttp, + ServicePort::MonitorHttp, ]; let mut seen: std::collections::HashSet<u16> = std::collections::HashSet::new(); for svc in all_services { - let ports: std::collections::HashSet<u16> = (0..100_u16).map(|i| svc.port(i)).collect(); + let ports: std::collections::HashSet<u16> = (0..svc.range_size()).map(|i| svc.port(i)).collect(); assert!( seen.is_disjoint(&ports), "{svc:?} ports overlap with a prior service type", @@ -176,12 +169,10 @@ fn port_ranges_do_not_overlap() { #[test] fn range_sizes_match_listener_scopes() { for svc in [ - ServicePort::KvServerMgmt, - ServicePort::KvServerListen, - ServicePort::DiskdbListen, + ServicePort::PaxosKvServerMgmt, + ServicePort::PaxosKvServerRpc, ServicePort::DiskdbHttp, ServicePort::DiskdbRpc, - ServicePort::ChunkdbListen, ServicePort::ChunkdbHttp, ServicePort::ChunkdbRpc, ServicePort::DiskioRpc, @@ -192,14 +183,9 @@ fn range_sizes_match_listener_scopes() { ServicePort::AccessServerIcebergHttp, ServicePort::AccessServerDatasetHttp, ServicePort::AccessServerHealthHttp, + ServicePort::MonitorHttp, ] { - let expected = match svc { - ServicePort::AccessServerHttp - | ServicePort::AccessServerIcebergHttp - | ServicePort::AccessServerDatasetHttp - | ServicePort::AccessServerHealthHttp => 1000, - _ => 500, - }; + let expected = 1000; assert_eq!(svc.range_size(), expected, "{svc:?} range_size"); } } @@ -218,9 +204,25 @@ fn access_listener_scopes_have_disjoint_thousand_port_blocks() { assert!(names.insert(service.name())); for instance in 0..service.range_size() { let port = service.port(instance); - assert!((16000..20000).contains(&port)); + assert!((7135..10135).contains(&port) || (25135..26135).contains(&port)); assert!(seen.insert(port), "overlapping access listener port {port}"); } } assert_eq!(seen.len(), 4000); } + +#[test] +fn management_and_rpc_defaults_are_one_thousand_apart() { + for (management, rpc) in [ + (ServicePort::PaxosKvServerMgmt, ServicePort::PaxosKvServerRpc), + (ServicePort::DiskdbHttp, ServicePort::DiskdbRpc), + (ServicePort::ChunkdbHttp, ServicePort::ChunkdbRpc), + (ServicePort::ChunkKvHttp, ServicePort::ChunkKvRpc), + ] { + assert_eq!(rpc.base() - management.base(), 1000); + for offset in 0..3 { + assert_eq!(management.port(offset), management.base() + offset); + } + assert_eq!(management.base() % 1000, 135); + } +} diff --git a/lib/crowdb-protocol/tests/runtime_namespace_test.rs b/lib/crowdb-protocol/tests/runtime_namespace_test.rs index 05e30c934..23596b2e6 100644 --- a/lib/crowdb-protocol/tests/runtime_namespace_test.rs +++ b/lib/crowdb-protocol/tests/runtime_namespace_test.rs @@ -82,7 +82,7 @@ fn persistent_namespace_reopens_saved_assignments() { let mut persistent = RuntimeNamespace::persistent(&root, "mini-cluster").expect("create persistent namespace"); persistent - .assign_port(ServicePort::KvServerMgmt, 0) + .assign_port(ServicePort::PaxosKvServerMgmt, 0) .expect("assign persistent port") }; @@ -90,7 +90,7 @@ fn persistent_namespace_reopens_saved_assignments() { RuntimeNamespace::persistent(&root, "mini-cluster").expect("reopen persistent namespace"); assert_eq!( reopened - .assign_port(ServicePort::KvServerMgmt, 0) + .assign_port(ServicePort::PaxosKvServerMgmt, 0) .expect("reuse persistent port"), first_port ); diff --git a/lib/crowdb-protocol/tests/stream_process_lease_test.rs b/lib/crowdb-protocol/tests/stream_process_lease_test.rs index c50a6115a..8e11d28ca 100644 --- a/lib/crowdb-protocol/tests/stream_process_lease_test.rs +++ b/lib/crowdb-protocol/tests/stream_process_lease_test.rs @@ -25,9 +25,18 @@ fn owner_keys_and_records_keep_purpose_instance_and_process_identity() { prior_exclusion_deadline_ms: 0, grant_sequence: 0, grant_expires_at_ms: 0, + revoked: false, }; record.validate().unwrap(); let value = serde_json::to_value(&record).unwrap(); + assert!(!value.as_object().unwrap().contains_key("revoked")); + let mut excluded = record.clone(); + excluded.revoked = true; + let excluded_bytes = serde_json::to_vec(&excluded).unwrap(); + assert_eq!( + serde_json::from_slice::<StreamProcessLease>(&excluded_bytes).unwrap(), + excluded + ); assert_eq!( serde_json::from_value::<StreamProcessLease>(value.clone()).unwrap(), record @@ -59,6 +68,7 @@ fn business_stream_processes_cannot_claim_wal_grant_fencing_fields() { prior_exclusion_deadline_ms: 0, grant_sequence: 1, grant_expires_at_ms: 3_000, + revoked: false, }; assert!(record.validate().is_err()); record.identity.purpose = StreamPurpose::Wal; diff --git a/lib/crowdb-rpc/CMakeLists.txt b/lib/crowdb-rpc/CMakeLists.txt index a88fd4678..88ed2c964 100644 --- a/lib/crowdb-rpc/CMakeLists.txt +++ b/lib/crowdb-rpc/CMakeLists.txt @@ -18,6 +18,9 @@ if(CROWDB_RPC_SANITIZER) message(STATUS "crowdb-rpc sanitizer: ${CROWDB_RPC_SANITIZER}") add_compile_options(-fsanitize=${CROWDB_RPC_SANITIZER} -fno-omit-frame-pointer -g) add_link_options(-fsanitize=${CROWDB_RPC_SANITIZER}) + if(CROWDB_RPC_SANITIZER STREQUAL "thread") + set(CROWDB_COMMON_SPDLOG_HEADER_ONLY ON) + endif() endif() # ── Flatbuffer C++ codegen ──────────────────────────────────────── @@ -130,11 +133,15 @@ list(APPEND CMAKE_IGNORE_PATH /opt/boost) # ── folly (ConcurrentHashMap for pending map) ───────────────────── # Required: the pending map uses folly::ConcurrentHashMap (striped locks) # for the response hot path. No fallback to std::unordered_map. -find_package(folly CONFIG REQUIRED) -get_target_property(FOLLY_LINK_LIBRARIES Folly::folly INTERFACE_LINK_LIBRARIES) -list(REMOVE_ITEM FOLLY_LINK_LIBRARIES Boost::regex) -set_property(TARGET Folly::folly PROPERTY INTERFACE_LINK_LIBRARIES "${FOLLY_LINK_LIBRARIES}") -target_link_libraries(crowdb-rpc PUBLIC Folly::folly) +if(CROWDB_RPC_SANITIZER STREQUAL "thread" OR CROWDB_TREE_SANITIZER STREQUAL "thread") + include(cmake/folly-thread-sanitizer.cmake) +else() + find_package(folly CONFIG REQUIRED) + get_target_property(FOLLY_LINK_LIBRARIES Folly::folly INTERFACE_LINK_LIBRARIES) + list(REMOVE_ITEM FOLLY_LINK_LIBRARIES Boost::regex) + set_property(TARGET Folly::folly PROPERTY INTERFACE_LINK_LIBRARIES "${FOLLY_LINK_LIBRARIES}") + target_link_libraries(crowdb-rpc PUBLIC Folly::folly) +endif() message(STATUS "crowdb-rpc folly: enabled (ConcurrentHashMap)") if(CROWDB_RPC_HAVE_RDMA) diff --git a/lib/crowdb-rpc/cmake/folly-project.cmake b/lib/crowdb-rpc/cmake/folly-project.cmake new file mode 100644 index 000000000..d48997c6c --- /dev/null +++ b/lib/crowdb-rpc/cmake/folly-project.cmake @@ -0,0 +1,6 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. + +# Use the same pixi Boost lookup as the installed Folly consumer. Folly's +# project resets this policy before discovering its own dependencies. +cmake_policy(SET CMP0167 OLD) diff --git a/lib/crowdb-rpc/cmake/folly-thread-sanitizer.cmake b/lib/crowdb-rpc/cmake/folly-thread-sanitizer.cmake new file mode 100644 index 000000000..81c86d9a3 --- /dev/null +++ b/lib/crowdb-rpc/cmake/folly-thread-sanitizer.cmake @@ -0,0 +1,37 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. + +function(crowdb_rpc_instrument_folly) + include(FetchContent) + if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) + endif() + list(PREPEND CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX}") + # Match the pinned dependency. Its singleton publication and hazard-pointer + # synchronization must be visible to the same sanitizer as the consumer. + FetchContent_Declare(crowdb_folly + URL https://github.com/facebook/folly/archive/v2026.08.10.00.tar.gz + URL_HASH SHA256=add1cde8feb39f2f3777bf222f64e6ebf6d8107806e6b665afa3772a7acf8b74 + SOURCE_SUBDIR crowdb-download-only) + FetchContent_Declare(crowdb_fast_float + URL https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.0.tar.gz + URL_HASH SHA256=f312f2dc34c61e665f4b132c0307d6f70ad9420185fa831911bc24408acf625d + SOURCE_SUBDIR crowdb-download-only) + FetchContent_MakeAvailable(crowdb_folly crowdb_fast_float) + set(FASTFLOAT_INCLUDE_DIR "${crowdb_fast_float_SOURCE_DIR}/include") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + set(CMAKE_PROJECT_folly_INCLUDE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/folly-project.cmake") + set(BUILD_TESTS OFF) + set(BUILD_BENCHMARKS OFF) + set(BUILD_SHARED_LIBS OFF) + set(BOOST_LINK_STATIC OFF) + set(ENABLE_JEMALLOC OFF) + add_subdirectory("${crowdb_folly_SOURCE_DIR}" "${crowdb_folly_BINARY_DIR}" EXCLUDE_FROM_ALL) + # Link only the map and its real dependencies; all are compiled with TSan. + target_link_libraries(crowdb-rpc PUBLIC folly_concurrency_concurrent_hash_map) + target_include_directories(crowdb-rpc SYSTEM PUBLIC + "${crowdb_folly_SOURCE_DIR}" "${crowdb_folly_BINARY_DIR}") +endfunction() + +crowdb_rpc_instrument_folly() diff --git a/lib/crowdb-rpc/ffi/build.rs b/lib/crowdb-rpc/ffi/build.rs index 987685552..192a3dfd3 100644 --- a/lib/crowdb-rpc/ffi/build.rs +++ b/lib/crowdb-rpc/ffi/build.rs @@ -36,6 +36,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { .join("crowdb-common") .join("cpp") .join("include"); + let protocol_include = engine + .parent() + .ok_or("engine dir must have a parent lib dir")? + .join("crowdb-protocol/cpp/include"); let mut sources = Vec::new(); collect_cpp(&src, &mut sources)?; @@ -117,6 +121,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { .flag_if_supported("-Wno-sign-compare") .include(&include) .include(&common_include) + .include(&protocol_include) .files(sources.iter().map(|p| p.as_path()).collect::<Vec<_>>()); // Enable spdlog in the C++ build (gates CR_LOG_* macros). @@ -261,6 +266,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { println!("cargo:rerun-if-changed={}", src.display()); println!("cargo:rerun-if-changed={}", include.display()); println!("cargo:rerun-if-changed={}", common_include.display()); + println!("cargo:rerun-if-changed={}", protocol_include.display()); Ok(()) } diff --git a/lib/crowdb-rpc/ffi/src/buffer.rs b/lib/crowdb-rpc/ffi/src/buffer.rs index da001fca3..8ec219dbd 100644 --- a/lib/crowdb-rpc/ffi/src/buffer.rs +++ b/lib/crowdb-rpc/ffi/src/buffer.rs @@ -3,6 +3,9 @@ //! Safe wrappers for Buffer and BufferPool. +mod owned; +pub use owned::OwnedBytesError; + use crate::sys; use std::ptr; @@ -74,6 +77,7 @@ pub enum BufferChainError { TooManyViews { maximum: usize, actual: usize }, PayloadTooLarge, EmptyView, + OwnerAllocation(OwnedBytesError), } impl std::fmt::Display for BufferChainError { @@ -85,6 +89,7 @@ impl std::fmt::Display for BufferChainError { } Self::PayloadTooLarge => f.write_str("buffer chain payload exceeds u32"), Self::EmptyView => f.write_str("buffer chain views must not be empty"), + Self::OwnerAllocation(error) => error.fmt(f), } } } @@ -106,7 +111,7 @@ impl BufferChain { if view.is_empty() { Err(BufferChainError::EmptyView) } else { - Ok(Buffer::from_owned_bytes(view)) + Buffer::try_from_owned_bytes(view).map_err(BufferChainError::OwnerAllocation) } }) .collect::<Result<Vec<_>, _>>()?; @@ -306,17 +311,7 @@ impl Buffer { /// Wrap immutable owned bytes without copying their allocation. pub fn from_owned_bytes(data: bytes::Bytes) -> Self { - if data.is_empty() { - return Buffer { - handle: std::ptr::null_mut(), - }; - } - let data_ptr = data.as_ptr(); - let len = data.len(); - let ctx = Box::into_raw(Box::new(data)).cast::<std::ffi::c_void>(); - let handle = - unsafe { sys::crowdb_rpc_buffer_create_external(data_ptr, len as u32, Some(free_bytes_cb), ctx) }; - Buffer { handle } + Self::try_from_owned_bytes(data).unwrap_or_else(|_| Self::from_raw(ptr::null_mut())) } /// Create an external buffer wrapping a Vec allocation (zero-copy). @@ -433,11 +428,3 @@ extern "C" fn free_vec_cb(ctx: *mut std::ffi::c_void) { unsafe { drop(Box::from_raw(ctx.cast::<Vec<u8>>())) }; } } - -extern "C" fn free_bytes_cb(ctx: *mut std::ffi::c_void) { - if !ctx.is_null() { - // SAFETY: `ctx` came from `Box::into_raw(Box::new(Bytes))` above and - // the C++ buffer invokes its release callback exactly once. - unsafe { drop(Box::from_raw(ctx.cast::<bytes::Bytes>())) }; - } -} diff --git a/lib/crowdb-rpc/ffi/src/buffer/owned.rs b/lib/crowdb-rpc/ffi/src/buffer/owned.rs new file mode 100644 index 000000000..88a4d5183 --- /dev/null +++ b/lib/crowdb-rpc/ffi/src/buffer/owned.rs @@ -0,0 +1,123 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::Buffer; +use crate::sys; +use bytes::{Bytes, OwnerAllocationLease}; +use std::alloc::Layout; +use std::ffi::c_void; +use std::mem::ManuallyDrop; + +/// An immutable buffer could not admit its complete owner/control storage. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OwnedBytesError { + PayloadTooLarge, + BudgetExhausted, + AllocationFailed, +} +impl std::fmt::Display for OwnedBytesError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::PayloadTooLarge => "owned RPC payload exceeds its wire length", + Self::BudgetExhausted => "owned RPC control budget exhausted", + Self::AllocationFailed => "owned RPC control allocation failed", + }) + } +} +impl std::error::Error for OwnedBytesError {} + +struct OwnedBytes { + bytes: Bytes, + allocation: ManuallyDrop<Option<OwnerAllocationLease>>, +} + +impl Buffer { + /// Wrap bytes and charge every new native/Rust control allocation to the + /// immutable owner's budget, when present. Refund follows actual free, + /// including the Rust owner Box. Ordinary input Bytes remain caller-owned. + /// # Errors + /// Returns a length, budget, or allocation error without retaining input. + pub fn try_from_owned_bytes(data: Bytes) -> Result<Self, OwnedBytesError> { + if data.is_empty() { + return Ok(Self::from_raw(std::ptr::null_mut())); + } + let len = u32::try_from(data.len()).map_err(|_| OwnedBytesError::PayloadTooLarge)?; + let allocation = data + .try_owner_allocation(Layout::new::<OwnedBytes>()) + .map_err(|_| OwnedBytesError::BudgetExhausted)?; + let accounted = allocation.is_some(); + let mut owner = Box::new(OwnedBytes { + bytes: data, + allocation: ManuallyDrop::new(allocation), + }); + if let Some(allocation) = owner.allocation.as_mut() { + allocation.allocated(); + } + let data_ptr = owner.bytes.as_ptr(); + let context = Box::into_raw(owner).cast::<c_void>(); + let allocator = sys::crowdb_rpc_control_allocator_v1 { + version: 1, + context, + claim: Some(claim), + record: Some(record), + release: Some(release), + }; + let handle = unsafe { + sys::crowdb_rpc_buffer_create_external_accounted( + data_ptr, + len, + Some(free_owner), + context, + if accounted { + &raw const allocator + } else { + std::ptr::null() + }, + ) + }; + if handle.is_null() { + unsafe { free_owner(context) }; + return Err(OwnedBytesError::AllocationFailed); + } + Ok(Self::from_raw(handle)) + } +} + +unsafe extern "C" fn free_owner(context: *mut c_void) { + let mut owner = unsafe { Box::from_raw(context.cast::<OwnedBytes>()) }; + let allocation = unsafe { ManuallyDrop::take(&mut owner.allocation) }; + // The lease itself retains Bytes: native budget/context survive both the + // contained Bytes drop and the actual Box deallocation below. + drop(owner); + drop(allocation); +} +unsafe extern "C" fn claim(context: *mut c_void, size: usize, align: usize) -> i32 { + let Ok(layout) = Layout::from_size_align(size, align) else { + return 0; + }; + let owner = unsafe { &*context.cast::<OwnedBytes>() }; + i32::from( + owner + .bytes + .owner_allocation() + .is_some_and(|budget| budget.claim(layout).is_ok()), + ) +} +unsafe extern "C" fn record(context: *mut c_void, size: usize, align: usize, acquired: i32) { + let Ok(layout) = Layout::from_size_align(size, align) else { + return; + }; + let owner = unsafe { &*context.cast::<OwnedBytes>() }; + if let Some(budget) = owner.bytes.owner_allocation() { + budget.record(layout, acquired != 0); + } +} +unsafe extern "C" fn release(context: *mut c_void, size: usize, align: usize) { + let Ok(layout) = Layout::from_size_align(size, align) else { + return; + }; + let owner = unsafe { &*context.cast::<OwnedBytes>() }; + if let Some(budget) = owner.bytes.owner_allocation() { + budget.release(layout); + } +} diff --git a/lib/crowdb-rpc/ffi/src/lib.rs b/lib/crowdb-rpc/ffi/src/lib.rs index 333e60571..346128f54 100644 --- a/lib/crowdb-rpc/ffi/src/lib.rs +++ b/lib/crowdb-rpc/ffi/src/lib.rs @@ -28,7 +28,7 @@ mod transport; pub use buffer::{ Buffer, BufferAllocationMetadata, BufferChain, BufferChainError, BufferLeaseMetadata, BufferOwnership, - BufferPool, + BufferPool, OwnedBytesError, }; pub use client::{noop_completion, CallFuture, ClientRequest, Response, RpcClient}; pub use connection_pool::{ConnectionPoolError, ConnectionPoolIndex, SelectedConnection}; diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index c837679f5..81639b6cc 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -476,3 +476,6 @@ pub struct CrowdbRpcCoStats { pub min_latency_ns: u64, pub max_latency_ns: u64, } + +mod owner; +pub use owner::*; diff --git a/lib/crowdb-rpc/ffi/src/sys/owner.rs b/lib/crowdb-rpc/ffi/src/sys/owner.rs new file mode 100644 index 000000000..553282229 --- /dev/null +++ b/lib/crowdb-rpc/ffi/src/sys/owner.rs @@ -0,0 +1,20 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{c_void, crowdb_rpc_buffer_t}; +#[repr(C)] +pub struct crowdb_rpc_control_allocator_v1 { + pub version: u32, + pub context: *mut c_void, + pub claim: Option<unsafe extern "C" fn(*mut c_void, usize, usize) -> i32>, + pub record: Option<unsafe extern "C" fn(*mut c_void, usize, usize, i32)>, + pub release: Option<unsafe extern "C" fn(*mut c_void, usize, usize)>, +} +unsafe extern "C" { + pub fn crowdb_rpc_buffer_create_external_accounted( + data: *const u8, + len: u32, + free_cb: Option<unsafe extern "C" fn(*mut c_void)>, + free_ctx: *mut c_void, + allocator: *const crowdb_rpc_control_allocator_v1, + ) -> crowdb_rpc_buffer_t; +} diff --git a/lib/crowdb-rpc/ffi/tests/accounted_owner_test.rs b/lib/crowdb-rpc/ffi/tests/accounted_owner_test.rs new file mode 100644 index 000000000..8e23d5ae6 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/accounted_owner_test.rs @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use bytes::{Bytes, OwnerAllocation}; +use crowdb_rpc_ffi::{Buffer, BufferChain, BufferChainError}; +use std::{ + alloc::Layout, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; +#[derive(Default)] +struct TestLedger { + admitted: AtomicUsize, + allocated: AtomicUsize, + claims: AtomicUsize, + refuse: AtomicUsize, +} +struct TestAccount(Arc<TestLedger>); +impl OwnerAllocation for TestAccount { + type Error = (); + fn claim(&self, layout: Layout) -> Result<(), ()> { + let ordinal = self.0.claims.fetch_add(1, Ordering::SeqCst) + 1; + if ordinal == self.0.refuse.load(Ordering::SeqCst) { + return Err(()); + } + self.0.admitted.fetch_add(layout.size(), Ordering::SeqCst); + Ok(()) + } + fn record(&self, layout: Layout, acquired: bool) { + if acquired { + self.0.allocated.fetch_add(layout.size(), Ordering::SeqCst); + } else { + assert!(self.0.allocated.fetch_sub(layout.size(), Ordering::SeqCst) >= layout.size()); + } + } + fn release(&self, layout: Layout) { + assert!(self.0.admitted.fetch_sub(layout.size(), Ordering::SeqCst) >= layout.size()); + } +} +fn owner(ledger: &Arc<TestLedger>) -> Bytes { + Bytes::try_from_owner_accounted([7u8; 64], TestAccount(ledger.clone())).unwrap() +} +fn released(ledger: &TestLedger) { + assert_eq!(ledger.allocated.load(Ordering::SeqCst), 0); + assert_eq!(ledger.admitted.load(Ordering::SeqCst), 0); +} +#[test] +fn accounted_bytes_pay_all_rpc_controls_and_survive_original_owner_release() { + let ledger = Arc::new(TestLedger::default()); + let bytes = owner(&ledger); + let base = ledger.allocated.load(Ordering::SeqCst); + let original = bytes.as_ptr(); + let buffer = Buffer::try_from_owned_bytes(bytes.clone()).unwrap(); + assert_eq!(buffer.as_slice().as_ptr(), original); + assert_eq!(ledger.claims.load(Ordering::SeqCst), 5); // Bytes, Rust Box, Buffer, ref, handle. + assert!(ledger.allocated.load(Ordering::SeqCst) > base); + assert_eq!( + ledger.allocated.load(Ordering::SeqCst), + ledger.admitted.load(Ordering::SeqCst) + ); + drop(bytes); + assert_eq!(buffer.as_slice(), &[7; 64]); + drop(buffer); + released(&ledger); +} +#[test] +fn every_rpc_control_admission_failure_unwinds_the_partial_owner() { + for refused in 2..=5 { + let ledger = Arc::new(TestLedger::default()); + let bytes = owner(&ledger); + ledger.refuse.store(refused, Ordering::SeqCst); + assert!(Buffer::try_from_owned_bytes(bytes).is_err(), "claim {refused}"); + released(&ledger); + } +} +#[test] +fn slices_share_bytes_capacity_but_each_rpc_view_pays_its_controls() { + let ledger = Arc::new(TestLedger::default()); + let bytes = owner(&ledger); + let first = Buffer::try_from_owned_bytes(bytes.slice(1..9)).unwrap(); + let after_first = ledger.allocated.load(Ordering::SeqCst); + let second = Buffer::try_from_owned_bytes(bytes.slice(15..19)).unwrap(); + assert!(ledger.allocated.load(Ordering::SeqCst) > after_first); + drop(bytes); + drop(first); + assert_eq!(second.as_slice(), &[7; 4]); + drop(second); + released(&ledger); +} +#[test] +fn chain_reports_owner_admission_failure_without_submitting_empty_payload() { + let ledger = Arc::new(TestLedger::default()); + let bytes = owner(&ledger); + ledger.refuse.store(2, Ordering::SeqCst); + assert!(matches!( + BufferChain::from_owned_bytes([bytes]), + Err(BufferChainError::OwnerAllocation(_)) + )); + released(&ledger); +} diff --git a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h index b20f4ce0a..798a4c3dd 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/buffer.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/buffer.h @@ -3,6 +3,8 @@ #pragma once +#include "crowdb-rpc/control_allocator.h" + #include <array> #include <atomic> #include <cstdint> @@ -100,11 +102,12 @@ struct Buffer // External free callback: when set, release() calls free_cb(free_ctx) // instead of freeing `data` directly. Used for zero-copy buffers that // wrap externally-owned memory (e.g. a Rust Vec allocation). - void (*free_cb)(void *) = nullptr; - void *free_ctx = nullptr; + void (*free_cb)(void *) = nullptr; + void *free_ctx = nullptr; + ControlAllocator control_allocator{}; - // Transfer the final reference to an asynchronous receive recycler. - // The callback consumes that reference, including any repost attempt. + // Transfer the final reference to its owner-specific release handler. + // The callback consumes that reference, including any receive repost. void (*last_release_cb)(void *, Buffer *) = nullptr; void *last_release_ctx = nullptr; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 560bf9474..d3d3850ff 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -244,6 +244,8 @@ void crowdb_rpc_client_attach(crowdb_rpc_client_t client, crowdb_rpc_conn_t conn // Completion callback — invoked on the C++ I/O thread when the response // arrives or on error. Must be O(1) and non-blocking. +// Takes ownership of each non-null buffer and must release it with +// crowdb_rpc_buffer_release after use. typedef void (*crowdb_rpc_on_complete)(uint64_t request_id, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, void *user_data); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h index 07e1bb952..6d58d04c3 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api_internal.h @@ -64,3 +64,9 @@ void invoke_c_complete(crowdb_rpc_on_complete cb, void *user_data, uint64_t requ void invoke_c_handler(crowdb_rpc_handler_fn callback, void *user_data, Frame *request, Connection *conn); } // namespace crowdb::rpc + +namespace crowdb::rpc +{ +// Consumes one Buffer reference on both success and failure. +[[nodiscard]] crowdb_rpc_buffer_t wrap_owned_buffer(Buffer *buffer) noexcept; +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/connection.h b/lib/crowdb-rpc/include/crowdb-rpc/connection.h index 6de26267c..2164e131c 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/connection.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/connection.h @@ -164,7 +164,7 @@ class Connection : public std::enable_shared_from_this<Connection> // Check if the connection has pending send data (queue or partials). [[nodiscard]] bool has_pending_send() const { - return send_queue_.has_pending() || overflow_.has_pending() || pending_count_ > 0; + return send_queue_.has_pending() || overflow_.has_pending() || has_partials_.load(std::memory_order_relaxed); } // Push a frame to the overflow queue (called when send_queue is full). @@ -312,6 +312,9 @@ class Connection : public std::enable_shared_from_this<Connection> // try_send sends these first, then drains the queue for new frames. OutFrame *pending_frames_[BATCH_MAX]; int pending_count_{0}; + // Readers outside in_send_ only need a hint; the sender publishes it + // before releasing ownership and arms writable events for partial sends. + std::atomic<bool> has_partials_{false}; ConnectionCallback<OnFrameCallback> on_frame_callback_; ConnectionCallback<std::function<void()>> on_frame_ready_; diff --git a/lib/crowdb-rpc/include/crowdb-rpc/control_allocator.h b/lib/crowdb-rpc/include/crowdb-rpc/control_allocator.h new file mode 100644 index 000000000..69f16b9b6 --- /dev/null +++ b/lib/crowdb-rpc/include/crowdb-rpc/control_allocator.h @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-rpc/control_allocator_c_api.h" + +#include <utility> + +namespace crowdb::rpc +{ +// Borrowed immutable callbacks. Every object using them retains the payload +// owner until its own storage and admission have both been returned. +struct ControlAllocator +{ + crowdb_rpc_control_allocator_v1 callbacks{}; + + template <typename T, typename... Args> [[nodiscard]] T *create(Args &&...args) const noexcept + { + if (callbacks.claim != nullptr && callbacks.claim(callbacks.context, sizeof(T), alignof(T)) == 0) { + return nullptr; + } + T *object = nullptr; + try { + object = new T(std::forward<Args>(args)...); + } + catch (...) { + release<T>(); + return nullptr; + } + record<T>(true); + return object; + } + + template <typename T> void destroy(T *object) const noexcept + { + if (object == nullptr) { + return; + } + delete object; + record<T>(false); + release<T>(); + } + + private: + template <typename T> void record(bool acquired) const noexcept + { + if (callbacks.record != nullptr) { + callbacks.record(callbacks.context, sizeof(T), alignof(T), acquired ? 1 : 0); + } + } + + template <typename T> void release() const noexcept + { + if (callbacks.release != nullptr) { + callbacks.release(callbacks.context, sizeof(T), alignof(T)); + } + } +}; +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/include/crowdb-rpc/control_allocator_c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/control_allocator_c_api.h new file mode 100644 index 000000000..512b662f8 --- /dev/null +++ b/lib/crowdb-rpc/include/crowdb-rpc/control_allocator_c_api.h @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#ifdef __cplusplus +# include <cstddef> +# include <cstdint> +#else +# include <stddef.h> +# include <stdint.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct crowdb_rpc_buffer_s *crowdb_rpc_buffer_t; + +// Callbacks never throw and are safe on any final-owner thread. The external +// owner keeps context alive through every control deallocation; free_cb runs +// only after all native Buffer, refcount, and C handle storage has been freed. +// claim precedes allocation; record(acquired=0) and release follow actual free. +typedef struct crowdb_rpc_control_allocator_v1 +{ + uint32_t version; + void *context; + int32_t (*claim)(void *, size_t, size_t); + void (*record)(void *, size_t, size_t, int32_t); + void (*release)(void *, size_t, size_t); +} crowdb_rpc_control_allocator_v1; + +// On failure no ownership transfers and free_cb is not called. All partial +// native allocations and their claims have been released before returning. +crowdb_rpc_buffer_t crowdb_rpc_buffer_create_external_accounted(const uint8_t *data, uint32_t len, + void (*free_cb)(void *), void *free_ctx, + const crowdb_rpc_control_allocator_v1 *callbacks); + +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index 234836369..45bc1ebbc 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -27,7 +27,7 @@ namespace crowdb::rpc // references Worker for the owner-worker back-pointer. class SocketTransport; class Worker; -class Frame; +struct Frame; // ── TransportStats: aggregation + latency counters ──────────────── // diff --git a/lib/crowdb-rpc/src/buffer.cpp b/lib/crowdb-rpc/src/buffer.cpp index 23774ef0a..4b765069a 100644 --- a/lib/crowdb-rpc/src/buffer.cpp +++ b/lib/crowdb-rpc/src/buffer.cpp @@ -87,9 +87,12 @@ void Buffer::release() // the data is owned by an external allocator (e.g. a Rust Vec). // free_cb drops the external owner; data itself is NOT freed // here (the external owner's destructor handles that). - free_cb(free_ctx); - delete ref; - delete this; + auto callback = free_cb; + auto *context = free_ctx; + const auto allocator = control_allocator; + allocator.destroy(ref); + allocator.destroy(this); + callback(context); } else if (pool != nullptr) { pool->recycle(this); @@ -109,12 +112,20 @@ Buffer *Buffer::slice(uint32_t offset, uint32_t size) { if (offset > capacity || size > capacity - offset || data == nullptr || ref_clone() == nullptr) return nullptr; - auto *view = new Buffer; - view->data = payload_data() + offset; - view->len = size; - view->capacity = size; - view->type = type; - view->ref = new std::atomic<int32_t>(1); + const auto allocator = control_allocator; + auto *view = allocator.create<Buffer>(); + auto *references = view == nullptr ? nullptr : allocator.create<std::atomic<int32_t>>(1); + if (references == nullptr) { + allocator.destroy(view); + release(); + return nullptr; + } + view->control_allocator = allocator; + view->data = payload_data() + offset; + view->len = size; + view->capacity = size; + view->type = type; + view->ref = references; view->ownership.store(BufferOwnership::CallerOwned, std::memory_order_relaxed); view->capacity_class = capacity_class; view->registration_id = registration_id; diff --git a/lib/crowdb-rpc/src/buffer_arena.cpp b/lib/crowdb-rpc/src/buffer_arena.cpp index def30aca7..c9878eab7 100644 --- a/lib/crowdb-rpc/src/buffer_arena.cpp +++ b/lib/crowdb-rpc/src/buffer_arena.cpp @@ -187,22 +187,28 @@ bool BufferArena::detach(Buffer *buffer) const auto state = buffer->ownership.load(std::memory_order_acquire); if (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned) return false; - auto context = std::make_unique<DetachedRelease>(DetachedRelease{this, buffer->pool_index, buffer}); entries_[buffer->pool_index].buffer.store(nullptr, std::memory_order_release); - buffer->pool = nullptr; - buffer->free_ctx = context.release(); - buffer->free_cb = &BufferArena::release_detached; + buffer->pool = nullptr; + buffer->last_release_ctx = this; + buffer->last_release_cb = &BufferArena::release_detached; return true; } -void BufferArena::release_detached(void *context) +void BufferArena::release_detached(void *context, Buffer *buffer) { - const auto release = std::unique_ptr<DetachedRelease>(static_cast<DetachedRelease *>(context)); - const auto state = release->buffer->ownership.load(std::memory_order_acquire); + auto *arena = static_cast<BufferArena *>(context); + // The facade may already be gone. Retain the arena through descriptor + // destruction and the final return of its entry and outstanding credit. + const auto owner = std::move(buffer->pool_owner); + const auto index = buffer->pool_index; + const auto state = buffer->ownership.load(std::memory_order_acquire); if (state != BufferOwnership::Available && state != BufferOwnership::CallerOwned) - release->arena->quarantined.fetch_add(release->buffer->allocation_capacity, std::memory_order_relaxed); - release->arena->outstanding.fetch_sub(1, std::memory_order_release); - release->arena->push_entry(release->arena->unused_entries_, release->index); + arena->quarantined.fetch_add(buffer->allocation_capacity, std::memory_order_relaxed); + const auto allocator = buffer->control_allocator; + allocator.destroy(buffer->ref); + allocator.destroy(buffer); + arena->outstanding.fetch_sub(1, std::memory_order_release); + arena->push_entry(arena->unused_entries_, index); } } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/buffer_arena.h b/lib/crowdb-rpc/src/buffer_arena.h index d8f7a7550..66a7b61a9 100644 --- a/lib/crowdb-rpc/src/buffer_arena.h +++ b/lib/crowdb-rpc/src/buffer_arena.h @@ -58,13 +58,6 @@ class BufferArena : public BufferPool, public std::enable_shared_from_this<Buffe uint32_t pop_entry(std::atomic<uint64_t> &head); void push_entry(std::atomic<uint64_t> &head, uint32_t index); - struct DetachedRelease - { - BufferArena *arena; - uint32_t index; - Buffer *buffer; - }; - - static void release_detached(void *context); + static void release_detached(void *context, Buffer *buffer); }; } // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index 4823cee9a..61362d6ed 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -10,6 +10,7 @@ #include "crowdb-rpc/client/client.h" #include "crowdb-rpc/client/co_client.h" #include "crowdb-rpc/client/rpc_client_metrics.h" +#include "crowdb-rpc/control_allocator_c_api.h" #include "crowdb-rpc/rpc_metrics.h" #include "crowdb-rpc/server/message.h" #include "crowdb-rpc/server/server.h" @@ -75,6 +76,7 @@ void crowdb_rpc_buffer_write(crowdb_rpc_buffer_t buf, const uint8_t *data, uint3 buf->buf->write(data, len); } catch (...) { + return; } } @@ -185,9 +187,7 @@ crowdb_rpc_buffer_t crowdb_rpc_buffer_ref(crowdb_rpc_buffer_t buf) if (buf == nullptr || buf->buf == nullptr) { return nullptr; } - auto clone = std::make_unique<crowdb_rpc_buffer_s>(); - clone->buf = buf->buf->ref_clone(); - return clone->buf == nullptr ? nullptr : clone.release(); + return crowdb::rpc::wrap_owned_buffer(buf->buf->ref_clone()); } catch (...) { return nullptr; @@ -202,11 +202,13 @@ void crowdb_rpc_buffer_release(crowdb_rpc_buffer_t buf) } // All buffers now have a refcount (pool-allocated or standalone). // release() decrements and frees on last reference. - buf->buf->release(); - buf->buf = nullptr; - delete buf; + auto *buffer = buf->buf; + const auto allocator = buffer->control_allocator; + allocator.destroy(buf); + buffer->release(); } catch (...) { + return; } } @@ -243,23 +245,7 @@ crowdb_rpc_buffer_t crowdb_rpc_buffer_create(const uint8_t *data, uint32_t len) crowdb_rpc_buffer_t crowdb_rpc_buffer_create_external(const uint8_t *data, uint32_t len, void (*free_cb)(void *), void *free_ctx) { - try { - if (data == nullptr || len == 0 || free_cb == nullptr) { - return nullptr; - } - auto *buf = new crowdb::rpc::Buffer; - buf->data = const_cast<uint8_t *>(data); - buf->len = len; - buf->capacity = len; - buf->ref = new std::atomic<int32_t>(1); - buf->pool = nullptr; - buf->free_cb = free_cb; - buf->free_ctx = free_ctx; - return new crowdb_rpc_buffer_s{buf}; - } - catch (...) { - return nullptr; - } + return crowdb_rpc_buffer_create_external_accounted(data, len, free_cb, free_ctx, nullptr); } // ── Pool ────────────────────────────────────────────────────────── @@ -287,6 +273,7 @@ void crowdb_rpc_pool_destroy(crowdb_rpc_pool_t pool) delete pool; } catch (...) { + return; } } @@ -303,6 +290,7 @@ void crowdb_rpc_client_get_counters(crowdb_rpc_client_t /*client*/, crowdb_rpc_c out->reaped = cumulative(crowdb::rpc::rpc_reaped()); } catch (...) { + return; } } @@ -329,6 +317,7 @@ void crowdb_rpc_client_destroy(crowdb_rpc_client_t client) delete client; } catch (...) { + return; } } @@ -349,7 +338,6 @@ static crowdb_rpc_status rpc_error_to_status(RpcError err) case RpcError::SendQueueFull: return CROWDB_RPC_ERR_SEND_QUEUE; case RpcError::ConnectionError: - return CROWDB_RPC_ERR_CONN_ERROR; default: return CROWDB_RPC_ERR_CONN_ERROR; } @@ -383,6 +371,7 @@ void crowdb_rpc_client_attach(crowdb_rpc_client_t client, crowdb_rpc_conn_t conn }); } catch (...) { + return; } } @@ -395,6 +384,7 @@ void crowdb_rpc_client_set_completion_pool_size(crowdb_rpc_client_t client, uint client->client->set_completion_pool_size(max_in_flight); } catch (...) { + return; } } @@ -407,6 +397,7 @@ void crowdb_rpc_client_start_reaper(crowdb_rpc_client_t client, uint64_t timeout client->client->start_reaper(timeout_ns, scan_interval_ns); } catch (...) { + return; } } @@ -419,6 +410,7 @@ void crowdb_rpc_client_stop_reaper(crowdb_rpc_client_t client) client->client->stop_reaper(); } catch (...) { + return; } } @@ -431,6 +423,7 @@ void crowdb_rpc_client_dump_pending(crowdb_rpc_client_t client) client->client->dump_pending(); } catch (...) { + return; } } @@ -637,6 +630,7 @@ void crowdb_rpc_conn_destroy(crowdb_rpc_conn_t conn) delete conn; } catch (...) { + return; } } @@ -668,6 +662,7 @@ void crowdb_rpc_conn_close(crowdb_rpc_conn_t conn) } } catch (...) { + return; } } @@ -682,6 +677,7 @@ void crowdb_rpc_server_register_echo_handler(crowdb_rpc_server_t server, uint16_ server->server->register_handler(msg_type, crowdb::rpc::handle_echo); } catch (...) { + return; } } @@ -762,5 +758,6 @@ void crowdb_rpc_server_register_conn_count_gauge(crowdb_rpc_server_t server, con }); } catch (...) { + return; } } diff --git a/lib/crowdb-rpc/src/c_api/buffer_lease.cpp b/lib/crowdb-rpc/src/c_api/buffer_lease.cpp index 15cbf27c6..b26c29dcf 100644 --- a/lib/crowdb-rpc/src/c_api/buffer_lease.cpp +++ b/lib/crowdb-rpc/src/c_api/buffer_lease.cpp @@ -60,9 +60,7 @@ extern "C" crowdb_rpc_buffer_t crowdb_rpc_buffer_allocation_ref(crowdb_rpc_buffe if (buffer == nullptr || buffer->len == 0) { return nullptr; } - auto result = std::make_unique<crowdb_rpc_buffer_s>(); - result->buf = buffer->slice(0, buffer->len); - return result->buf == nullptr ? nullptr : result.release(); + return crowdb::rpc::wrap_owned_buffer(buffer->slice(0, buffer->len)); } catch (...) { return nullptr; diff --git a/lib/crowdb-rpc/src/c_api/buffer_owner.cpp b/lib/crowdb-rpc/src/c_api/buffer_owner.cpp new file mode 100644 index 000000000..f36a9278c --- /dev/null +++ b/lib/crowdb-rpc/src/c_api/buffer_owner.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-rpc/c_api_internal.h" +#include "crowdb-rpc/control_allocator_c_api.h" + +namespace crowdb::rpc +{ +crowdb_rpc_buffer_t wrap_owned_buffer(Buffer *buffer) noexcept +{ + if (buffer == nullptr) { + return nullptr; + } + auto *handle = buffer->control_allocator.create<crowdb_rpc_buffer_s>(); + if (handle == nullptr) { + buffer->release(); + return nullptr; + } + handle->buf = buffer; + return handle; +} +} // namespace crowdb::rpc + +extern "C" crowdb_rpc_buffer_t +crowdb_rpc_buffer_create_external_accounted(const uint8_t *data, uint32_t len, void (*free_cb)(void *), void *free_ctx, + const crowdb_rpc_control_allocator_v1 *callbacks) +{ + if (data == nullptr || len == 0 || free_cb == nullptr) { + return nullptr; + } + if (callbacks != nullptr && (callbacks->version != 1 || callbacks->claim == nullptr || + callbacks->record == nullptr || callbacks->release == nullptr)) { + return nullptr; + } + crowdb::rpc::ControlAllocator allocator; + if (callbacks != nullptr) { + allocator.callbacks = *callbacks; + } + auto *buffer = allocator.create<crowdb::rpc::Buffer>(); + if (buffer == nullptr) { + return nullptr; + } + auto *references = allocator.create<std::atomic<int32_t>>(1); + if (references == nullptr) { + allocator.destroy(buffer); + return nullptr; + } + auto *handle = allocator.create<crowdb_rpc_buffer_s>(); + if (handle == nullptr) { + allocator.destroy(references); + allocator.destroy(buffer); + return nullptr; + } + buffer->control_allocator = allocator; + buffer->data = const_cast<uint8_t *>(data); + buffer->len = len; + buffer->capacity = len; + buffer->ref = references; + buffer->free_cb = free_cb; + buffer->free_ctx = free_ctx; + handle->buf = buffer; + return handle; +} diff --git a/lib/crowdb-rpc/src/c_api/frame.cpp b/lib/crowdb-rpc/src/c_api/frame.cpp index 075606a8c..dd4762fd3 100644 --- a/lib/crowdb-rpc/src/c_api/frame.cpp +++ b/lib/crowdb-rpc/src/c_api/frame.cpp @@ -37,9 +37,7 @@ crowdb_rpc_buffer_t wrap_data(Buffer *buffer, uint64_t request_id) return nullptr; buffer->set_metadata(request_id, buffer->stream_id, buffer->sequence, buffer->logical_offset, buffer->registration_id); - auto handle = std::make_unique<crowdb_rpc_buffer_s>(); - handle->buf = buffer->ref_clone(); - return handle->buf == nullptr ? nullptr : handle.release(); + return wrap_owned_buffer(buffer->ref_clone()); } } // namespace diff --git a/lib/crowdb-rpc/src/c_api/producer.cpp b/lib/crowdb-rpc/src/c_api/producer.cpp index 74406a2b8..6b427180e 100644 --- a/lib/crowdb-rpc/src/c_api/producer.cpp +++ b/lib/crowdb-rpc/src/c_api/producer.cpp @@ -42,9 +42,7 @@ extern "C" crowdb_rpc_buffer_t crowdb_rpc_server_alloc_producer_buffer(crowdb_rp server->server->transport()->allocate_producer_buffer(capacity, require_registered != 0), release); if (buffer == nullptr) return nullptr; - auto *handle = new crowdb_rpc_buffer_s{buffer.get()}; - (void)buffer.release(); - return handle; + return crowdb::rpc::wrap_owned_buffer(buffer.release()); } catch (...) { return nullptr; diff --git a/lib/crowdb-rpc/src/connection.cpp b/lib/crowdb-rpc/src/connection.cpp index 506ae0264..8923459fe 100644 --- a/lib/crowdb-rpc/src/connection.cpp +++ b/lib/crowdb-rpc/src/connection.cpp @@ -7,6 +7,7 @@ #include "crowdb-rpc/rpc_metrics.h" #include "crowdb-rpc/transport/socket_transport.h" // TransportStats +#include <sys/socket.h> #include <sys/uio.h> #include <unistd.h> @@ -281,7 +282,16 @@ bool Connection::try_send(int fd, TransportStats *stats) break; } + // A disconnected peer is a connection error, regardless of the host + // application's process-wide SIGPIPE disposition. +#ifdef MSG_NOSIGNAL + msghdr message{}; + message.msg_iov = iovs; + message.msg_iovlen = static_cast<decltype(message.msg_iovlen)>(iov_count); + ssize_t written = ::sendmsg(fd, &message, MSG_NOSIGNAL); +#else ssize_t written = ::writev(fd, iovs, iov_count); +#endif if (stats != nullptr) stats->writev_calls.fetch_add(1, std::memory_order_relaxed); if (written > 0) { @@ -375,6 +385,7 @@ bool Connection::try_send(int fd, TransportStats *stats) discarded_count = drain_send_queue(discarded, BATCH_MAX); } } + has_partials_.store(pending_count_ != 0, std::memory_order_relaxed); in_send_.store(false, std::memory_order_release); // Race check: if more frames arrived while we were sending, retry. @@ -403,6 +414,7 @@ void Connection::close() release_frame(pending_frames_[i]); } pending_count_ = 0; + has_partials_.store(false, std::memory_order_relaxed); OutFrame *discarded[BATCH_MAX]; int discarded_count = drain_overflow(discarded, BATCH_MAX); while (discarded_count > 0) { diff --git a/lib/crowdb-rpc/src/server/message.cpp b/lib/crowdb-rpc/src/server/message.cpp index 8ef51ae33..41c325cb6 100644 --- a/lib/crowdb-rpc/src/server/message.cpp +++ b/lib/crowdb-rpc/src/server/message.cpp @@ -4,6 +4,7 @@ #include "crowdb-rpc/server/message.h" #include "common_msg_generated.h" +#include "crowdb-protocol/flatbuffer.h" #include "msg_type_generated.h" #include "ret_code_generated.h" @@ -30,7 +31,7 @@ void extract_control_fields(const uint8_t *control, uint32_t len, uint64_t &out_ // Use Verifier to safely access fields on untrusted input. ::flatbuffers::Verifier verifier(control, len); if (verifier.VerifyBuffer<proto::ConnectionPingResponse>()) { - const auto *resp = ::flatbuffers::GetRoot<proto::ConnectionPingResponse>(control); + const auto *resp = protocol::flatbuffer_root<proto::ConnectionPingResponse>(control, len); if (resp != nullptr) { out_request_id = resp->id(); out_rpc_create_nano = resp->rpc_create_nano(); @@ -39,7 +40,7 @@ void extract_control_fields(const uint8_t *control, uint32_t len, uint64_t &out_ return; } if (verifier.VerifyBuffer<proto::ConnectionPingRequest>()) { - const auto *req = ::flatbuffers::GetRoot<proto::ConnectionPingRequest>(control); + const auto *req = protocol::flatbuffer_root<proto::ConnectionPingRequest>(control, len); if (req != nullptr) { out_request_id = req->id(); out_rpc_create_nano = req->rpc_create_nano(); @@ -47,7 +48,7 @@ void extract_control_fields(const uint8_t *control, uint32_t len, uint64_t &out_ return; } if (verifier.VerifyBuffer<proto::UnknownMessage>()) { - const auto *unk = ::flatbuffers::GetRoot<proto::UnknownMessage>(control); + const auto *unk = protocol::flatbuffer_root<proto::UnknownMessage>(control, len); if (unk != nullptr) { out_request_id = unk->id(); out_rpc_create_nano = unk->rpc_create_nano(); diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index 609578df6..64b3f8683 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -186,7 +186,6 @@ void RpcServer::start() if (!transport_->listen_rdma_service(listen_addr_, listen_port_)) CRB_LOG_WARN("rpc server: RDMA listener unavailable; TCP remains active"); } - transport_->start(); // Register the listen fd with the I/O worker's epoll loop instead of // a dedicated acceptor thread. The worker's epoll loop is already // woken by I/O events, so accept is not starved under CPU contention. @@ -198,6 +197,9 @@ void RpcServer::start() transport_->set_accept_handler([this](int fd) { handle_accept(fd); }); transport_->add_listen_fd(listen_fd_); } + // Publish handlers and listener registration before any worker can read + // them. They stay immutable until all workers have joined in stop(). + transport_->start(); } void RpcServer::stop() @@ -209,10 +211,14 @@ void RpcServer::stop() if (was_running) { CRB_LOG_INFO("rpc server: stopping"); } - // Clear the accept handler before closing the listen fd so the - // worker doesn't call accept() on a closed fd during shutdown. + // Join workers before destroying their accept callback or closing its fd. + // running_ already prevents an active accept loop from draining more work. + if (was_running) { + transport_->stop(); + } transport_->set_accept_handler(nullptr); if (listen_fd_ >= 0) { + transport_->remove_listen_fd(listen_fd_); ::close(listen_fd_); listen_fd_ = -1; } @@ -225,9 +231,6 @@ void RpcServer::stop() ::close(listener_lock_fd_); listener_lock_fd_ = -1; } - if (was_running) { - transport_->stop(); - } } void RpcServer::handle_accept(int listen_fd) diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp index 682a0388d..eeab6772e 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_connection.cpp @@ -20,14 +20,15 @@ namespace crowdb::rpc bool RdmaTransport::listen(const std::string &addr, int port) { - if (addr.empty() || port <= 0 || port > 65535 || listen_id_ != nullptr) { + if (addr.empty() || port < 0 || port > 65535 || listen_id_ != nullptr) { CRB_LOG_WARN("rdma listen: transport unavailable pd={} addr={} port={} existing_listener={}", static_cast<void *>(pd_), addr, port, listen_id_ != nullptr); return false; } - if (cm_channel_ == nullptr) + if (cm_channel_ == nullptr) { cm_channel_ = rdma_create_event_channel(); + } if (cm_channel_ == nullptr || rdma_create_id(cm_channel_, &listen_id_, this, RDMA_PS_TCP) != 0) { CRB_LOG_WARN("rdma listen: create CM id failed addr={} port={} errno={}", addr, port, errno); return false; @@ -49,19 +50,21 @@ bool RdmaTransport::listen(const std::string &addr, int port) return false; } local_capability_.cm_address = addr; - local_capability_.port = static_cast<uint16_t>(port); + local_capability_.port = ntohs(rdma_get_src_port(listen_id_)); listen_active_.store(true, std::memory_order_release); return true; } std::shared_ptr<Connection> RdmaTransport::accept() { - if (cm_channel_ == nullptr) + if (cm_channel_ == nullptr) { return nullptr; + } rdma_cm_event *event = nullptr; while (rdma_get_cm_event(cm_channel_, &event) == 0) { - if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST && listen_id_ != nullptr) + if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST && listen_id_ != nullptr) { break; + } if (event != nullptr) { rdma_cm_id *event_id = event->id; const int event_type = event->event; @@ -81,22 +84,27 @@ std::shared_ptr<Connection> RdmaTransport::accept() } cm_ids_.erase(std::remove(cm_ids_.begin(), cm_ids_.end(), event_id), cm_ids_.end()); } - if (closed_connection == nullptr) + if (closed_connection == nullptr) { continue; + } close_after_drain(closed_connection.get()); - if (event_type == RDMA_CM_EVENT_DISCONNECTED || event_type == RDMA_CM_EVENT_CONNECT_ERROR) + if (event_type == RDMA_CM_EVENT_DISCONNECTED || event_type == RDMA_CM_EVENT_CONNECT_ERROR) { stats_.cm_disconnects.fetch_add(1, std::memory_order_relaxed); - if (closed_connection != nullptr) + } + if (closed_connection != nullptr) { destroy_connection_qp(closed_connection.get(), event_id); - else if (event_id->qp != nullptr) + } + else if (event_id->qp != nullptr) { rdma_destroy_qp(event_id); + } rdma_destroy_id(event_id); } } event = nullptr; } - if (event == nullptr) + if (event == nullptr) { return nullptr; + } rdma_cm_id *id = event->id; RdmaCmOffer peer_offer; const bool valid_peer = event->param.conn.private_data != nullptr && @@ -134,17 +142,19 @@ std::shared_ptr<Connection> RdmaTransport::accept() rdma_destroy_id(id); return nullptr; } - ibv_qp_init_attr qp_attr{}; - qp_attr.send_cq = reservation.queue->cq; - qp_attr.recv_cq = reservation.queue->cq; - qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = config_.sq_depth; - qp_attr.cap.max_recv_wr = config_.rq_depth; - qp_attr.cap.max_send_sge = config_.qp_max_send_sge; - qp_attr.cap.max_recv_sge = config_.qp_max_recv_sge; - qp_attr.cap.max_inline_data = 0; - qp_attr.sq_sig_all = 0; - qp_attr.qp_context = id; + ibv_qp_init_attr qp_attr{ + .qp_context = id, + .send_cq = reservation.queue->cq, + .recv_cq = reservation.queue->cq, + .srq = nullptr, + .cap = {.max_send_wr = config_.sq_depth, + .max_recv_wr = config_.rq_depth, + .max_send_sge = config_.qp_max_send_sge, + .max_recv_sge = config_.qp_max_recv_sge, + .max_inline_data = 0}, + .qp_type = IBV_QPT_RC, + .sq_sig_all = 0 + }; if (rdma_create_qp(id, pd_, &qp_attr) != 0) { rdma_destroy_id(id); @@ -198,16 +208,41 @@ std::shared_ptr<Connection> RdmaTransport::accept() if (event_rc != 0 || event == nullptr || event->event != RDMA_CM_EVENT_ESTABLISHED) { CRB_LOG_WARN("rdma accept: ESTABLISHED missing event={} status={} errno={}", event == nullptr ? -1 : event->event, event == nullptr ? 0 : event->status, errno); - if (event != nullptr) + if (event != nullptr) { rdma_ack_cm_event(event); + } destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); return nullptr; } rdma_ack_cm_event(event); - ibv_qp_attr receive_attributes{}; - receive_attributes.min_rnr_timer = config_.min_rnr_timer; + ibv_qp_attr receive_attributes{.qp_state = IBV_QPS_RESET, + .cur_qp_state = IBV_QPS_RESET, + .path_mtu = IBV_MTU_256, + .path_mig_state = IBV_MIG_MIGRATED, + .qkey = 0, + .rq_psn = 0, + .sq_psn = 0, + .dest_qp_num = 0, + .qp_access_flags = 0, + .cap = {}, + .ah_attr = {}, + .alt_ah_attr = {}, + .pkey_index = 0, + .alt_pkey_index = 0, + .en_sqd_async_notify = 0, + .sq_draining = 0, + .max_rd_atomic = 0, + .max_dest_rd_atomic = 0, + .min_rnr_timer = config_.min_rnr_timer, + .port_num = 0, + .timeout = 0, + .retry_cnt = 0, + .rnr_retry = 0, + .alt_port_num = 0, + .alt_timeout = 0, + .rate_limit = 0}; if (ibv_modify_qp(id->qp, &receive_attributes, IBV_QP_MIN_RNR_TIMER) != 0) { destroy_connection_qp(connection.get(), id); rdma_destroy_id(id); diff --git a/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.cpp b/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.cpp index 47c98d8a0..8d578253e 100644 --- a/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.cpp +++ b/lib/crowdb-rpc/src/transport/rdma/rdma_control_codec.cpp @@ -4,6 +4,7 @@ #include "rdma_control_codec.h" #include "common_msg_generated.h" +#include "crowdb-protocol/flatbuffer.h" #include "msg_type_generated.h" #include <algorithm> @@ -45,25 +46,30 @@ bool parse_control_fragment(std::span<const uint8_t> control, ControlFragment &f if (!verifier.VerifyBuffer<proto::RpcControlFragment>()) { return false; } - const auto *f = flatbuffers::GetRoot<proto::RpcControlFragment>(control.data()); - fragment = {.request_id = f->id(), - .rpc_create_nano = f->rpc_create_nano(), - .response_create_nano = f->response_create_nano(), - .stream_id = f->stream_id(), - .generation = f->generation(), - .sequence = f->sequence(), - .offset = f->logical_offset(), - .total = f->total_length(), - .capacity = f->data_capacity(), - .message_type = f->original_msg_type(), - .message_flags = f->original_flags(), - .stream_flags = f->stream_flags()}; + const auto *f = protocol::flatbuffer_root<proto::RpcControlFragment>(control.data(), control.size()); + if (f == nullptr) { + return false; + } + fragment = {.request_id = f->id(), + .rpc_create_nano = f->rpc_create_nano(), + .response_create_nano = f->response_create_nano(), + .stream_id = f->stream_id(), + .generation = f->generation(), + .sequence = f->sequence(), + .offset = f->logical_offset(), + .total = f->total_length(), + .capacity = f->data_capacity(), + .message_type = f->original_msg_type(), + .message_flags = f->original_flags(), + .stream_flags = f->stream_flags()}; return true; } bool control_identity_matches(const Frame &frame) { - uint64_t id = 0, created = 0, response = 0; + uint64_t id = 0; + uint64_t created = 0; + uint64_t response = 0; extract_control_fields(frame.control.data(), static_cast<uint32_t>(frame.control.size()), id, created, response); return id == frame.request_id && created == frame.rpc_create_nano && response == frame.response_create_nano; } diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index 376032f14..161c6379d 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -815,6 +815,10 @@ std::shared_ptr<Connection> SocketTransport::create_connection(int fd, const std Connection::OnFrameCallback on_frame, Connection::OnCloseCallback on_close) { +#ifdef SO_NOSIGPIPE + int nosigpipe = 1; + ::setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &nosigpipe, sizeof(nosigpipe)); +#endif int64_t id = next_conn_id_.fetch_add(1, std::memory_order_relaxed); auto conn = std::make_shared<Connection>(id, name, pool_, 4 << 20, send_queue_capacity_); conn->transport_handle = static_cast<uint64_t>(fd); @@ -833,7 +837,7 @@ std::shared_ptr<Connection> SocketTransport::create_connection(int fd, const std // Keep the identity registry until descriptor cleanup; a worker must // safely resolve an event even after an explicit caller-side close. conn->set_on_frame(std::move(on_frame)); - conn->set_on_close([this, on_close = std::move(on_close)](Connection *c) { + conn->set_on_close([on_close = std::move(on_close)](Connection *c) { if (auto *worker = static_cast<Worker *>(c->io_worker); worker != nullptr) { worker->request_close(c->shared_from_this()); } diff --git a/lib/crowdb-rpc/tests/buffer_test.cpp b/lib/crowdb-rpc/tests/buffer_test.cpp index 51a86a748..109cc38c4 100644 --- a/lib/crowdb-rpc/tests/buffer_test.cpp +++ b/lib/crowdb-rpc/tests/buffer_test.cpp @@ -178,6 +178,39 @@ TEST(BufferTest, RegisteredPoolDetachTransfersReleaseToCaller) buf->release(); } +TEST(BufferTest, DetachedRegisteredBufferOutlivesPoolAndRetainsSlices) +{ + auto pool = std::make_unique<RegisteredBufferPool>(crowdb::rpc::RDMA_CONTROL_CAPACITY, 1); + Buffer *buf = pool->alloc(4); + ASSERT_NE(buf, nullptr); + const uint8_t data[] = {1, 2, 3, 4}; + buf->write(data, 4); + std::weak_ptr<void> arena = buf->pool_owner; + ASSERT_TRUE(pool->detach(buf)); + Buffer *view = buf->slice(1, 2); + ASSERT_NE(view, nullptr); + pool.reset(); + buf->release(); + EXPECT_FALSE(arena.expired()); + EXPECT_EQ(std::memcmp(view->data, data + 1, 2), 0); + view->release(); + EXPECT_TRUE(arena.expired()); +} + +TEST(BufferTest, DetachedRegisteredBufferCountsQuarantineOnFinalRelease) +{ + RegisteredBufferPool pool(2 * crowdb::rpc::RDMA_CONTROL_CAPACITY, 1); + Buffer *buf = pool.alloc(1); + ASSERT_NE(buf, nullptr); + ASSERT_TRUE(pool.detach(buf)); + EXPECT_TRUE(buf->quarantine()); + buf->release(); + EXPECT_EQ(pool.quarantined_bytes(), crowdb::rpc::RDMA_CONTROL_CAPACITY); + Buffer *replacement = pool.alloc(1); + ASSERT_NE(replacement, nullptr); + replacement->release(); +} + TEST(BufferTest, ReleasedRegisteredBufferCannotBeReleasedTwice) { RegisteredBufferPool pool(crowdb::rpc::RDMA_CONTROL_CAPACITY, 1); diff --git a/lib/crowdb-rpc/tests/c_api_load_test.cpp b/lib/crowdb-rpc/tests/c_api_load_test.cpp index c18db368b..d44ea8a56 100644 --- a/lib/crowdb-rpc/tests/c_api_load_test.cpp +++ b/lib/crowdb-rpc/tests/c_api_load_test.cpp @@ -23,11 +23,13 @@ struct PendingReq bool ok = false; }; -extern "C" void on_complete_cb(uint64_t /*request_id*/, crowdb_rpc_buffer_t /*control*/, crowdb_rpc_buffer_t /*data*/, +extern "C" void on_complete_cb(uint64_t /*request_id*/, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, int status, void *user_data) { auto *pr = static_cast<PendingReq *>(user_data); pr->ok = (status == CROWDB_RPC_OK); + crowdb_rpc_buffer_release(control); + crowdb_rpc_buffer_release(data); pr->got_response.store(true, std::memory_order_release); // Reclaim the PendingReq via the user_data pointer — the caller // allocates it with new and deletes it after checking the flag. diff --git a/lib/crowdb-rpc/tests/client_disconnect_test.cpp b/lib/crowdb-rpc/tests/client_disconnect_test.cpp index 1b318b9a6..e6a28f70c 100644 --- a/lib/crowdb-rpc/tests/client_disconnect_test.cpp +++ b/lib/crowdb-rpc/tests/client_disconnect_test.cpp @@ -13,11 +13,14 @@ #include <atomic> #include <chrono> +#include <csignal> #include <future> #include <thread> namespace { +volatile std::sig_atomic_t broken_pipe_signal = 0; + int socketpair_nonblocking(int sockets[2]) { if (::socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) != 0) { @@ -127,6 +130,32 @@ TEST(ClientDisconnectTest, PeerCloseFailsSlabAndMapWithoutReaperAndPreservesOthe ::close(second[1]); } +TEST(ClientDisconnectTest, ClosedPeerFailsSendWithoutRaisingSigpipe) +{ + using namespace crowdb::rpc; + int sockets[2]{}; + ASSERT_EQ(socketpair_nonblocking(sockets), 0); + SocketTransport transport(1, 1); + auto connection = transport.create_connection(sockets[0], "closed-peer"); + ::close(sockets[1]); + auto *frame = new OutFrame{}; + frame->control = transport.pool()->alloc(1); + ASSERT_NE(frame->control, nullptr); + frame->control->write("x", 1); + ASSERT_TRUE(connection->enqueue_send(frame)); + + broken_pipe_signal = 0; + const auto previous = std::signal(SIGPIPE, [](int) { broken_pipe_signal = 1; }); + ASSERT_NE(previous, SIG_ERR); + const bool sent = connection->try_send(connection->write_fd, nullptr); + std::signal(SIGPIPE, previous); + EXPECT_FALSE(sent); + EXPECT_EQ(broken_pipe_signal, 0); + EXPECT_FALSE(connection->is_open()); + EXPECT_FALSE(connection->has_pending_send()); + transport.stop(); +} + TEST(ClientDisconnectTest, AttachedConnectionOutlivesFfiClientHandle) { auto *server = crowdb_rpc_server_create(nullptr); diff --git a/lib/crowdb-rpc/tests/client_pool_test.cpp b/lib/crowdb-rpc/tests/client_pool_test.cpp index ade4e10a9..616f4b2dc 100644 --- a/lib/crowdb-rpc/tests/client_pool_test.cpp +++ b/lib/crowdb-rpc/tests/client_pool_test.cpp @@ -187,7 +187,7 @@ struct CallState std::atomic<int> recv_status{CROWDB_RPC_OK}; }; -extern "C" void call_recv_cb(uint64_t /*request_id*/, crowdb_rpc_buffer_t /*control*/, crowdb_rpc_buffer_t data, +extern "C" void call_recv_cb(uint64_t /*request_id*/, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, void *user_data) { auto *s = static_cast<CallState *>(user_data); @@ -195,6 +195,7 @@ extern "C" void call_recv_cb(uint64_t /*request_id*/, crowdb_rpc_buffer_t /*cont if (data != nullptr) { crowdb_rpc_buffer_release(data); } + crowdb_rpc_buffer_release(control); s->got_response.store(true, std::memory_order_release); } @@ -270,8 +271,12 @@ TEST_F(CallerLoopbackTest, CallAndReceiveResponse) EXPECT_EQ(state.recv_status.load(), CROWDB_RPC_OK); // Late response (after the callback already fired) — should be discarded. - auto *late_frame = new Frame; - caller.on_response(req_id, late_frame); + auto *late_frame = new Frame; + const bool claimed = caller.on_response(req_id, late_frame); + EXPECT_FALSE(claimed); + if (!claimed) { + delete late_frame; + } // No crash, no double-callback. transport.stop(); @@ -300,13 +305,15 @@ struct SlabCallbackState std::atomic<uint64_t> last_request_id{0}; }; -extern "C" void slab_test_cb(uint64_t request_id, crowdb_rpc_buffer_t /*control*/, crowdb_rpc_buffer_t /*data*/, +extern "C" void slab_test_cb(uint64_t request_id, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, void *user_data) { auto *s = static_cast<SlabCallbackState *>(user_data); - s->call_count.fetch_add(1, std::memory_order_relaxed); + crowdb_rpc_buffer_release(control); + crowdb_rpc_buffer_release(data); s->last_status.store(status, std::memory_order_relaxed); s->last_request_id.store(request_id, std::memory_order_relaxed); + s->call_count.fetch_add(1, std::memory_order_release); } // Test: slab fallback to map when the slot is occupied by a slow request. @@ -510,7 +517,11 @@ TEST_F(CallerLoopbackTest, ReaperTimesOutSlabSlot) late_resp->request_id = 1; late_resp->header.msg_type = 42; late_resp->data_buf = nullptr; - caller.on_response(1, late_resp); + const bool claimed = caller.on_response(1, late_resp); + EXPECT_FALSE(claimed); + if (!claimed) { + delete late_resp; + } EXPECT_EQ(state.call_count.load(std::memory_order_acquire), 1); // still 1 EXPECT_EQ(rpc_resp_missed().window(), 1U); diff --git a/lib/crowdb-rpc/tests/common/test_rdma_control.cpp b/lib/crowdb-rpc/tests/common/test_rdma_control.cpp index 1016b6324..b8b2f47a8 100644 --- a/lib/crowdb-rpc/tests/common/test_rdma_control.cpp +++ b/lib/crowdb-rpc/tests/common/test_rdma_control.cpp @@ -38,7 +38,7 @@ OutFrame *test_control_frame(BufferPool &pool, uint64_t id, uint32_t size) return nullptr; } for (uint32_t index = 0; index < size; ++index) { - control->data[index] = static_cast<uint8_t>(index * 131U + id); + control->data[index] = static_cast<uint8_t>((index * 131U) + id); } auto *identity = build_ping_request(&pool, id, 11); if (identity == nullptr || size < identity->len) { @@ -72,7 +72,7 @@ std::unique_ptr<TestControlPeers> TestControlPeers::create(const RdmaConfig &cli auto pair = std::make_unique<TestControlPeers>(); pair->server = std::make_unique<RdmaTransport>(server_config, &pair->application); pair->client = std::make_unique<RdmaTransport>(client_config, &pair->application); - if (!pair->server->listen(server_config.source_address, 19401)) { + if (!pair->server->listen(server_config.source_address, 0)) { return nullptr; } std::thread acceptor([&] { @@ -82,7 +82,7 @@ std::unique_ptr<TestControlPeers> TestControlPeers::create(const RdmaConfig &cli std::this_thread::sleep_for(std::chrono::milliseconds(1)); } }); - pair->outgoing = pair->client->connect(server_config.source_address, 19401); + pair->outgoing = pair->client->connect(server_config.source_address, pair->server->local_capability().port); acceptor.join(); if (pair->incoming == nullptr || pair->outgoing == nullptr) { return nullptr; diff --git a/lib/crowdb-rpc/tests/control_allocator_test.cpp b/lib/crowdb-rpc/tests/control_allocator_test.cpp new file mode 100644 index 000000000..5ca8add12 --- /dev/null +++ b/lib/crowdb-rpc/tests/control_allocator_test.cpp @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-rpc/control_allocator.h" + +#include <gtest/gtest.h> + +#include <new> + +namespace crowdb::rpc +{ +namespace +{ +struct TestBudget +{ + size_t claimed = 0; + size_t allocated = 0; + bool released_after_delete = false; + bool deleted = false; + bool refuse = false; + + static int32_t claim(void *context, size_t size, size_t /*alignment*/) + { + auto &budget = *static_cast<TestBudget *>(context); + if (budget.refuse) { + return 0; + } + budget.claimed += size; + return 1; + } + + static void record(void *context, size_t size, size_t /*alignment*/, int32_t acquired) + { + auto &budget = *static_cast<TestBudget *>(context); + if (acquired != 0) { + budget.allocated += size; + } + else { + EXPECT_TRUE(budget.deleted); + budget.allocated -= size; + } + } + + static void release(void *context, size_t size, size_t /*alignment*/) + { + auto &budget = *static_cast<TestBudget *>(context); + budget.released_after_delete = budget.deleted; + budget.claimed -= size; + } + + ControlAllocator allocator() + { + return { + .callbacks = {.version = 1, .context = this, .claim = claim, .record = record, .release = release} + }; + } +}; + +struct TestControl +{ + static inline TestBudget *budget = nullptr; + + static void *operator new(size_t size) + { + return ::operator new(size); + } + + static void operator delete(void *memory) noexcept + { + ::operator delete(memory); + budget->deleted = true; + } +}; + +TEST(ControlAllocator, PhysicalAndAdmissionRefundFollowActualDelete) +{ + TestBudget budget; + TestControl::budget = &budget; + const auto allocator = budget.allocator(); + auto *control = allocator.create<TestControl>(); + ASSERT_NE(control, nullptr); + EXPECT_EQ(budget.allocated, sizeof(TestControl)); + EXPECT_EQ(budget.claimed, sizeof(TestControl)); + allocator.destroy(control); + EXPECT_TRUE(budget.released_after_delete); + EXPECT_EQ(budget.allocated, 0); + EXPECT_EQ(budget.claimed, 0); +} + +TEST(ControlAllocator, RefusalCreatesNoAllocationAndNoRefund) +{ + TestBudget budget; + budget.refuse = true; + const auto allocator = budget.allocator(); + EXPECT_EQ(allocator.create<TestControl>(), nullptr); + EXPECT_FALSE(budget.released_after_delete); + EXPECT_EQ(budget.allocated, 0); + EXPECT_EQ(budget.claimed, 0); +} +} // namespace +} // namespace crowdb::rpc diff --git a/lib/crowdb-rpc/tests/load_test.cpp b/lib/crowdb-rpc/tests/load_test.cpp index 297ccee9f..9fa136a30 100644 --- a/lib/crowdb-rpc/tests/load_test.cpp +++ b/lib/crowdb-rpc/tests/load_test.cpp @@ -42,7 +42,7 @@ struct PendingReq std::vector<uint8_t> payload; }; -extern "C" void load_on_complete(uint64_t /*request_id*/, crowdb_rpc_buffer_t /*control*/, crowdb_rpc_buffer_t data, +extern "C" void load_on_complete(uint64_t /*request_id*/, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, void *user_data) { auto *pr = static_cast<PendingReq *>(user_data); @@ -56,6 +56,7 @@ extern "C" void load_on_complete(uint64_t /*request_id*/, crowdb_rpc_buffer_t /* if (data != nullptr) { crowdb_rpc_buffer_release(data); } + crowdb_rpc_buffer_release(control); pr->got_response.store(true, std::memory_order_release); } diff --git a/lib/crowdb-rpc/tests/loopback_test.cpp b/lib/crowdb-rpc/tests/loopback_test.cpp index 5b6a78a40..a3c088472 100644 --- a/lib/crowdb-rpc/tests/loopback_test.cpp +++ b/lib/crowdb-rpc/tests/loopback_test.cpp @@ -40,7 +40,7 @@ struct LoopbackState std::vector<uint8_t> expected_payload; }; -extern "C" void loopback_on_complete(uint64_t request_id, crowdb_rpc_buffer_t /*control*/, crowdb_rpc_buffer_t data, +extern "C" void loopback_on_complete(uint64_t request_id, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, void *user_data) { auto *s = static_cast<LoopbackState *>(user_data); @@ -56,6 +56,7 @@ extern "C" void loopback_on_complete(uint64_t request_id, crowdb_rpc_buffer_t /* if (data != nullptr) { crowdb_rpc_buffer_release(data); } + crowdb_rpc_buffer_release(control); s->got_response.store(true, std::memory_order_release); } diff --git a/lib/crowdb-rpc/tests/rdma_completion_test.cpp b/lib/crowdb-rpc/tests/rdma_completion_test.cpp index ef013f820..8027e3481 100644 --- a/lib/crowdb-rpc/tests/rdma_completion_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_completion_test.cpp @@ -34,6 +34,7 @@ class RdmaCompletionTest : public ::testing::Test RegisteredBufferPool slots{16U << 20, 32}; SystemBufferPool buffers{64}; TestCompletionQueue queue; + EpollEngine engine; RdmaTransport transport{test_config()}; std::shared_ptr<Connection> connection = std::make_shared<Connection>(1, "test", &slots); @@ -258,9 +259,8 @@ TEST_F(RdmaCompletionTest, FixedPollerRetainsEarlyReceiveAndWakesOnHandlerPublic queue.ready.push_back(completion); TestRdmaTransport::set_cq(transport, &queue.cq); TestRdmaTransport::watch_frames(transport, connection, 0); - EpollEngine generic; - ASSERT_EQ(generic.init(), 0); - transport.register_events(&generic); + ASSERT_EQ(engine.init(), 0); + transport.register_events(&engine); transport.start(); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(1); while (transport.stats().pending_deferred_receives.load() == 0 && std::chrono::steady_clock::now() < deadline) @@ -268,7 +268,7 @@ TEST_F(RdmaCompletionTest, FixedPollerRetainsEarlyReceiveAndWakesOnHandlerPublic ASSERT_EQ(transport.stats().pending_deferred_receives.load(), 1U); EXPECT_EQ(transport.handle_external_event(3), 0); std::array<EngineEvent, 8> events{}; - EXPECT_EQ(generic.wait(events.data(), events.size(), 0), 0); + EXPECT_EQ(engine.wait(events.data(), events.size(), 0), 0); std::atomic<bool> received{false}; std::thread::id callback_thread; connection->set_on_frame([&](Frame *frame, Connection *) { @@ -535,7 +535,6 @@ TEST_F(RdmaCompletionTest, BoundedCqBatchContinuesThroughEpollUntilDrained) queue.ready.push_back(TestCompletionQueue::completion(token)); } TestRdmaTransport::set_cq(transport, &queue.cq); - EpollEngine engine; ASSERT_EQ(engine.init(), 0); transport.register_events(&engine); EXPECT_EQ(transport.drain_cq(2), 2); diff --git a/lib/crowdb-rpc/tests/rdma_control_codec_test.cpp b/lib/crowdb-rpc/tests/rdma_control_codec_test.cpp index 0cda7c83e..aab2eb383 100644 --- a/lib/crowdb-rpc/tests/rdma_control_codec_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_control_codec_test.cpp @@ -11,45 +11,50 @@ using namespace crowdb::rpc; TEST(ControlCodecTest, BoundedFragmentPreservesCommonCorrelationAndAllStreamMetadata) { - const ControlFragment source{.request_id = 43, - .rpc_create_nano = 11, - .response_create_nano = 12, - .stream_id = 5, - .generation = 7, - .sequence = 2, - .offset = 131072, - .total = 131089, - .capacity = RDMA_METADATA_CAPACITY, - .message_type = 3701, - .message_flags = FLAG_ONE_WAY, - .stream_flags = CONTROL_STREAM_END}; - std::array<uint8_t, RDMA_CONTROL_CAPACITY> prefix{}; - ASSERT_TRUE(write_control_fragment(prefix, source, 17)); - Header header; - ASSERT_TRUE(parse_rdma_direct_control(prefix, source.capacity, prefix.size() + 17, header)); - EXPECT_EQ(header.msg_type, RPC_CONTROL_FRAGMENT_MSG_TYPE); - EXPECT_EQ(header.flags, FLAG_ONE_WAY); - ControlFragment decoded; - const auto control = std::span(prefix).subspan(HEADER_SIZE); - ASSERT_TRUE(parse_control_fragment(control, decoded)); - EXPECT_EQ(decoded.request_id, source.request_id); - EXPECT_EQ(decoded.rpc_create_nano, source.rpc_create_nano); - EXPECT_EQ(decoded.response_create_nano, source.response_create_nano); - EXPECT_EQ(decoded.stream_id, source.stream_id); - EXPECT_EQ(decoded.generation, source.generation); - EXPECT_EQ(decoded.sequence, source.sequence); - EXPECT_EQ(decoded.offset, source.offset); - EXPECT_EQ(decoded.total, source.total); - EXPECT_EQ(decoded.capacity, source.capacity); - EXPECT_EQ(decoded.message_type, source.message_type); - EXPECT_EQ(decoded.message_flags, source.message_flags); - EXPECT_EQ(decoded.stream_flags, source.stream_flags); - uint64_t id = 0, created = 0, response = 0; - extract_control_fields(control.data(), control.size(), id, created, response); - EXPECT_EQ(id, source.request_id); - EXPECT_EQ(created, source.rpc_create_nano); - EXPECT_EQ(response, source.response_create_nano); - EXPECT_FALSE(write_control_fragment(prefix, source, source.capacity + 1)); + const ControlFragment source{.request_id = 43, + .rpc_create_nano = 11, + .response_create_nano = 12, + .stream_id = 5, + .generation = 7, + .sequence = 2, + .offset = 131072, + .total = 131089, + .capacity = RDMA_METADATA_CAPACITY, + .message_type = 3701, + .message_flags = FLAG_ONE_WAY, + .stream_flags = CONTROL_STREAM_END}; + alignas(8) std::array<uint8_t, RDMA_CONTROL_CAPACITY + 8> storage{}; + for (size_t shift = 0; shift < 8; ++shift) { + auto prefix = std::span(storage).subspan(shift, RDMA_CONTROL_CAPACITY); + ASSERT_TRUE(write_control_fragment(prefix, source, 17)); + Header header; + ASSERT_TRUE(parse_rdma_direct_control(prefix, source.capacity, prefix.size() + 17, header)); + EXPECT_EQ(header.msg_type, RPC_CONTROL_FRAGMENT_MSG_TYPE); + EXPECT_EQ(header.flags, FLAG_ONE_WAY); + ControlFragment decoded; + const auto control = std::span(prefix).subspan(HEADER_SIZE); + ASSERT_TRUE(parse_control_fragment(control, decoded)); + EXPECT_EQ(decoded.request_id, source.request_id); + EXPECT_EQ(decoded.rpc_create_nano, source.rpc_create_nano); + EXPECT_EQ(decoded.response_create_nano, source.response_create_nano); + EXPECT_EQ(decoded.stream_id, source.stream_id); + EXPECT_EQ(decoded.generation, source.generation); + EXPECT_EQ(decoded.sequence, source.sequence); + EXPECT_EQ(decoded.offset, source.offset); + EXPECT_EQ(decoded.total, source.total); + EXPECT_EQ(decoded.capacity, source.capacity); + EXPECT_EQ(decoded.message_type, source.message_type); + EXPECT_EQ(decoded.message_flags, source.message_flags); + EXPECT_EQ(decoded.stream_flags, source.stream_flags); + uint64_t id = 0; + uint64_t created = 0; + uint64_t response = 0; + extract_control_fields(control.data(), control.size(), id, created, response); + EXPECT_EQ(id, source.request_id); + EXPECT_EQ(created, source.rpc_create_nano); + EXPECT_EQ(response, source.response_create_nano); + EXPECT_FALSE(write_control_fragment(prefix, source, source.capacity + 1)); + } } TEST(ControlCodecTest, MalformedMetadataAndMismatchedOriginalControlDoNotPassTheBoundary) diff --git a/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp b/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp index 012055629..be59505f7 100644 --- a/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_direct_hardware_test.cpp @@ -27,8 +27,9 @@ RdmaConfig direct_config() config.send_arena_bytes = 4U * RDMA_DATAPATH_CAPACITY; config.receive_arena_bytes = uint64_t{4} * config.wire_slot_capacity; config.source_address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); - if (const auto *device = std::getenv("CROWDB_RDMA_DEVICE")) + if (const auto *device = std::getenv("CROWDB_RDMA_DEVICE")) { config.device = device; + } return config; } @@ -46,8 +47,9 @@ template <typename Ready> bool drain_until(RdmaTransport &client, RdmaTransport TEST(RdmaHardwareTest, DirectDataPreservesBytesAlignmentAndLeaseAdmission) { const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); - if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) { GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + } auto config = direct_config(); config.send_batch_size = 4; SystemBufferPool application; @@ -66,7 +68,8 @@ TEST(RdmaHardwareTest, DirectDataPreservesBytesAlignmentAndLeaseAdmission) acceptor.join(); ASSERT_NE(incoming, nullptr); ASSERT_NE(outgoing, nullptr); - std::vector<Buffer *> server_held, client_held; + std::vector<Buffer *> server_held; + std::vector<Buffer *> client_held; const std::array<uint32_t, 4> lengths{0, 13, RDMA_DATAPATH_CAPACITY, 65536}; incoming->set_on_frame([&](Frame *frame, Connection *connection) { EXPECT_GE(frame->request_id, 1U); @@ -83,8 +86,9 @@ TEST(RdmaHardwareTest, DirectDataPreservesBytesAlignmentAndLeaseAdmission) delete frame; const bool accepted = server->submit(connection, reply); EXPECT_TRUE(accepted); - if (!accepted) + if (!accepted) { release_rdma_frames(reply); + } }); outgoing->set_on_frame([&](Frame *frame, Connection *) { const auto expected = lengths[frame->request_id - 1]; @@ -103,8 +107,9 @@ TEST(RdmaHardwareTest, DirectDataPreservesBytesAlignmentAndLeaseAdmission) ASSERT_NE(data, nullptr); std::memset(data->data, static_cast<int>(index + 1), lengths[index]); data->len = lengths[index]; - if (index == 2) + if (index == 2) { source_held = data->ref_clone(); + } total += lengths[index]; } auto *frame = build_out_frame(index + 1, 100, build_ping_request(&application, index + 1, 0), data); @@ -137,10 +142,12 @@ TEST(RdmaHardwareTest, DirectDataPreservesBytesAlignmentAndLeaseAdmission) EXPECT_FALSE(client_device.expired()); EXPECT_FALSE(server_device.expired()); EXPECT_EQ(source_held->payload_data()[RDMA_DATAPATH_CAPACITY - 1], 3U); - for (auto *buffer : server_held) + for (auto *buffer : server_held) { buffer->release(); - for (auto *buffer : client_held) + } + for (auto *buffer : client_held) { buffer->release(); + } source_held->release(); EXPECT_TRUE(client_device.expired()); EXPECT_TRUE(server_device.expired()); @@ -149,8 +156,9 @@ TEST(RdmaHardwareTest, DirectDataPreservesBytesAlignmentAndLeaseAdmission) TEST(RdmaHardwareTest, LayoutMismatchFailsBeforeCmOrUsesExplicitTcpFallback) { const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); - if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) { GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + } auto config = direct_config(); auto contiguous = config; contiguous.wire_layout = RdmaWireLayout::Contiguous; @@ -181,15 +189,19 @@ TEST(RdmaHardwareTest, LayoutMismatchFailsBeforeCmOrUsesExplicitTcpFallback) static void test_mixed_staging(bool packed, uint32_t receive_batch = 1) { const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); - if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) { GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + } auto config = direct_config(); config.send_batch_size = 4; config.aggregation_enabled = packed; config.receive_batch_size = receive_batch; SystemBufferPool application; - RdmaTransport server(config, &application), client(config, &application); - ASSERT_TRUE(server.listen(address, 19392)); + RdmaTransport server(config, &application); + RdmaTransport client(config, &application); + ASSERT_TRUE(server.listen(address, 0)); + const auto port = server.local_capability().port; + ASSERT_GT(port, 0); std::shared_ptr<Connection> incoming; std::thread acceptor([&] { const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); @@ -198,7 +210,7 @@ static void test_mixed_staging(bool packed, uint32_t receive_batch = 1) std::this_thread::sleep_for(std::chrono::milliseconds(1)); } }); - auto outgoing = client.connect(address, 19392); + auto outgoing = client.connect(address, port); acceptor.join(); ASSERT_NE(outgoing, nullptr); ASSERT_NE(incoming, nullptr); @@ -210,8 +222,9 @@ static void test_mixed_staging(bool packed, uint32_t receive_batch = 1) delete frame; const bool accepted = server.submit(connection, reply); EXPECT_TRUE(accepted); - if (!accepted) + if (!accepted) { release_rdma_frames(reply); + } }); std::vector<Buffer *> held; outgoing->set_on_frame([&](Frame *frame, Connection *) { @@ -232,18 +245,21 @@ static void test_mixed_staging(bool packed, uint32_t receive_batch = 1) auto submit = [&](uint64_t id, uint32_t size) { auto *data = application.alloc(size); EXPECT_NE(data, nullptr); - if (data == nullptr) + if (data == nullptr) { return false; + } std::memset(data->data, static_cast<int>(id), size); data->len = size; auto *frame = build_out_frame(id, 100, build_ping_request(&application, id, 0), data); - if (client.submit(outgoing.get(), frame)) + if (client.submit(outgoing.get(), frame)) { return true; + } release_rdma_frames(frame); return false; }; - for (uint64_t id = 1; id <= 4; ++id) + for (uint64_t id = 1; id <= 4; ++id) { ASSERT_TRUE(submit(id, 128)); + } if (!packed) { ASSERT_EQ(TestRdmaTransport::flush_for_tests(client, outgoing.get(), 4), 4U); // Compatibility staging cannot consume the registered producer's @@ -259,9 +275,9 @@ static void test_mixed_staging(bool packed, uint32_t receive_batch = 1) ASSERT_TRUE(submit(5, 65536)); ASSERT_TRUE(drain_until(client, server, [&] { return replies == 5; })); EXPECT_EQ(client.stats().send_post_calls.load(), 2U); - EXPECT_EQ(client.stats().copied_payload_bytes.load(), 4U * 128 + 65536); + EXPECT_EQ(client.stats().copied_payload_bytes.load(), (4U * 128) + 65536); EXPECT_EQ(client.stats().copied_aggregate_payload_bytes.load(), packed ? 4U * 128 : 0U); - EXPECT_EQ(client.stats().copied_staging_payload_bytes.load(), packed ? 65536U : 4U * 128 + 65536); + EXPECT_EQ(client.stats().copied_staging_payload_bytes.load(), packed ? 65536U : (4U * 128) + 65536); if (packed) { const auto posted = client.stats().posted_receives.load(); ASSERT_EQ(held.size(), 4U); diff --git a/lib/crowdb-rpc/tests/rdma_listen_hardware_test.cpp b/lib/crowdb-rpc/tests/rdma_listen_hardware_test.cpp new file mode 100644 index 000000000..243dfc239 --- /dev/null +++ b/lib/crowdb-rpc/tests/rdma_listen_hardware_test.cpp @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-rpc/transport/rdma/rdma_transport.h" + +#include <gtest/gtest.h> + +#ifdef CROWDB_RPC_HAVE_RDMA +# include <cstdlib> + +using namespace crowdb::rpc; + +TEST(RdmaHardwareTest, EphemeralListenersAdvertiseDistinctBoundPorts) +{ + const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) { + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + } + RdmaConfig config; + config.use_environment = false; + if (const auto *device = std::getenv("CROWDB_RDMA_DEVICE")) { + config.device = device; + } + RdmaTransport first(config); + RdmaTransport second(config); + ASSERT_FALSE(first.listen(address, -1)); + ASSERT_FALSE(first.listen(address, 65536)); + ASSERT_TRUE(first.listen(address, 0)); + ASSERT_TRUE(second.listen(address, 0)); + const auto first_endpoint = first.local_capability(); + const auto second_endpoint = second.local_capability(); + EXPECT_EQ(first_endpoint.cm_address, address); + EXPECT_EQ(second_endpoint.cm_address, address); + EXPECT_GT(first_endpoint.port, 0); + EXPECT_GT(second_endpoint.port, 0); + EXPECT_NE(first_endpoint.port, second_endpoint.port); + EXPECT_FALSE(first.listen(address, 0)); +} +#endif diff --git a/lib/crowdb-rpc/tests/rdma_test.cpp b/lib/crowdb-rpc/tests/rdma_test.cpp index 25b8206ef..312c39147 100644 --- a/lib/crowdb-rpc/tests/rdma_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_test.cpp @@ -101,17 +101,33 @@ TEST(RdmaBufferPoolTest, SlicesOneRegisteredRegion) TEST(RdmaBufferPoolTest, TransportCopiesSystemBufferIntoRegisteredPool) { - RdmaTransport transport; + const auto *address = std::getenv("CROWDB_RDMA_E2E_LISTEN_ADDR"); + if (std::getenv("CROWDB_RDMA_E2E") == nullptr || address == nullptr) { + GTEST_SKIP() << "set CROWDB_RDMA_E2E and local HCA address"; + } + // Registered arenas use the device context selected by the CM connection. + RdmaTransport server; + RdmaTransport transport; + ASSERT_TRUE(server.listen(address, 19415)); + std::shared_ptr<Connection> incoming; + std::thread acceptor([&] { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (incoming == nullptr && std::chrono::steady_clock::now() < deadline) { + incoming = server.accept(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + }); + auto outgoing = transport.connect(address, 19415); + acceptor.join(); + ASSERT_NE(incoming, nullptr); + ASSERT_NE(outgoing, nullptr); SystemBufferPool system_pool(1); Buffer *source = system_pool.alloc(32); ASSERT_NE(source, nullptr); const uint8_t payload[] = {1, 2, 3, 4}; source->write(payload, sizeof(payload)); Buffer *registered = transport.register_buffer(source); - if (registered == nullptr) { - source->release(); - GTEST_SKIP() << "RDMA device registration is unavailable"; - } + ASSERT_NE(registered, nullptr); EXPECT_EQ(registered->type, crowdb::rpc::BufferType::Registered); EXPECT_EQ(registered->len, sizeof(payload)); EXPECT_EQ(std::memcmp(registered->data, payload, sizeof(payload)), 0); @@ -180,12 +196,12 @@ TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) return; } if (role != nullptr && std::strcmp(role, "client") == 0) { - std::fprintf(stderr, "R239 hardware client: device=%s source=%s target=%s port=%d\\n", + std::fprintf(stderr, "RDMA hardware client: device=%s source=%s target=%s port=%d\n", std::getenv("CROWDB_RDMA_DEVICE") == nullptr ? "" : std::getenv("CROWDB_RDMA_DEVICE"), std::getenv("CROWDB_RDMA_SOURCE_ADDR") == nullptr ? "" : std::getenv("CROWDB_RDMA_SOURCE_ADDR"), connect_address, port); RdmaTransport client; - std::fprintf(stderr, "R239 client transport ready=%d\\n", client.ready() ? 1 : 0); + std::fprintf(stderr, "RDMA client transport ready=%d\n", client.ready() ? 1 : 0); client.start(); auto outgoing = client.connect(connect_address, port); ASSERT_NE(outgoing, nullptr); @@ -195,7 +211,8 @@ TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) Buffer *raw = system_pool.alloc(HEADER_SIZE); ASSERT_NE(raw, nullptr); Header header; - header.msg_type = static_cast<uint16_t>(1 + sequence); + // Keep application frames out of the reserved transport types. + header.msg_type = 1; serialize_header(raw->data, header); raw->len = HEADER_SIZE; Buffer *control = client.register_buffer(raw); @@ -203,7 +220,7 @@ TEST(RdmaHardwareTest, CompleteCmQpSendReceiveFlow) auto *frame = new OutFrame; frame->header = header; frame->control = control; - ASSERT_TRUE(client.submit(outgoing.get(), frame)); + ASSERT_TRUE(client.submit(outgoing.get(), frame)) << "sequence " << sequence; raw->release(); } for (int i = 0; i < 300; ++i) { diff --git a/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp b/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp index f3abb7114..14a2a6f48 100644 --- a/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_timed_load_test.cpp @@ -72,6 +72,7 @@ OutFrame *test_echo(RpcServer &server, Frame *frame) void test_timed_server(const char *address, int port, const TestLoadSettings &settings) { RpcServer server(nullptr, 1, settings.io_workers); + ASSERT_TRUE(server.configure_rdma(RdmaConfig{}, true, false, true)); ASSERT_TRUE(server.listen(address, port)); std::atomic<uint64_t> received{0}; std::atomic<uint64_t> expected{0}; @@ -131,9 +132,10 @@ void test_report_load(const TestLoadSettings &settings, TestRdmaLoad &load) void test_timed_client(const char *address, int port, const TestLoadSettings &settings) { SocketTransport transport(1, settings.io_workers); - TestRdmaLoad load(settings.producer_count, settings.in_flight_per_producer, settings.duration_seconds); - RpcClient client; - TestLoadStop stop{transport}; + ASSERT_TRUE(transport.configure_rdma(RdmaConfig{}, true, false, false)); + TestRdmaLoad load(settings.producer_count, settings.in_flight_per_producer, settings.duration_seconds); + RpcClient client; + TestLoadStop stop{transport}; client.set_completion_pool_size(load.id_stride * 4); transport.start(); std::vector<std::shared_ptr<Connection>> connections; diff --git a/lib/crowdb-rpc/tests/rdma_worker_test.cpp b/lib/crowdb-rpc/tests/rdma_worker_test.cpp index 983bef89a..b39213e13 100644 --- a/lib/crowdb-rpc/tests/rdma_worker_test.cpp +++ b/lib/crowdb-rpc/tests/rdma_worker_test.cpp @@ -119,6 +119,7 @@ TEST(RdmaHardwareTest, RpcServerClientWorkerRoundTrip) if (role != nullptr && std::strcmp(role, "server") == 0) { RpcServer server(nullptr, 1, kWorkers); + ASSERT_TRUE(server.configure_rdma(RdmaConfig{}, true, false, true)); ASSERT_TRUE(server.listen(address, port)); std::atomic<uint32_t> received{0}; std::atomic<bool> client_finished{false}; @@ -153,6 +154,7 @@ TEST(RdmaHardwareTest, RpcServerClientWorkerRoundTrip) ASSERT_NE(role, nullptr); ASSERT_STREQ(role, "client"); SocketTransport transport(1, kWorkers); + ASSERT_TRUE(transport.configure_rdma(RdmaConfig{}, true, false, false)); transport.start(); std::vector<std::shared_ptr<Connection>> connections; std::unordered_set<uint32_t> qp_numbers; diff --git a/lib/crowdb-rpc/tests/write_progress_test.cpp b/lib/crowdb-rpc/tests/write_progress_test.cpp index bdaa61118..abb770158 100644 --- a/lib/crowdb-rpc/tests/write_progress_test.cpp +++ b/lib/crowdb-rpc/tests/write_progress_test.cpp @@ -128,6 +128,47 @@ class SocketWriteProgressTest : public ::testing::TestWithParam<bool> { }; +TEST(SocketWriteProgress, ConcurrentPendingInspectionDuringPartialSendAndClose) +{ + int sockets[2]{}; + ASSERT_EQ(::socketpair(AF_UNIX, SOCK_STREAM, 0, sockets), 0); + ASSERT_NE(::fcntl(sockets[0], F_SETFL, ::fcntl(sockets[0], F_GETFL, 0) | O_NONBLOCK), -1); + const int send_bytes = 4096; + ASSERT_EQ(::setsockopt(sockets[0], SOL_SOCKET, SO_SNDBUF, &send_bytes, sizeof(send_bytes)), 0); + crowdb::rpc::SystemBufferPool pool; + crowdb::rpc::Connection connection(1, "pending-inspection", &pool); + std::vector<uint8_t> bytes(PAYLOAD_BYTES, 0x5a); + auto *data = pool.alloc(PAYLOAD_BYTES); + data->write(bytes.data(), PAYLOAD_BYTES); + auto *control = crowdb::rpc::build_ping_request(&pool, 42, 1); + ASSERT_TRUE(connection.enqueue_send(crowdb::rpc::build_out_frame(42, 42, control, data))); + + std::atomic<uint64_t> observations{0}; + std::jthread observer([&](std::stop_token stop) { + while (!stop.stop_requested()) { + if (connection.has_pending_send()) { + observations.fetch_add(1, std::memory_order_relaxed); + } + std::this_thread::yield(); + } + }); + while (observations.load(std::memory_order_relaxed) == 0) { + std::this_thread::yield(); + } + // The peer does not read: the first send is partial and later calls hit + // EAGAIN while the reader inspects the connection without send ownership. + for (int i = 0; i < 100; ++i) { + EXPECT_FALSE(connection.try_send(sockets[0], nullptr)); + EXPECT_TRUE(connection.has_pending_send()); + } + connection.close(); + EXPECT_FALSE(connection.has_pending_send()); + observer.request_stop(); + observer.join(); + ::close(sockets[0]); + ::close(sockets[1]); +} + TEST_P(SocketWriteProgressTest, ConcurrentRearmSurvivesAnEarlierEmptyWritableEvent) { int sockets[2]{}; diff --git a/lib/crowdb-sdk-native/Cargo.toml b/lib/crowdb-sdk-native/Cargo.toml index 8c6cd98ad..cbcf82fb2 100644 --- a/lib/crowdb-sdk-native/Cargo.toml +++ b/lib/crowdb-sdk-native/Cargo.toml @@ -24,7 +24,7 @@ crowdb-access-multipart = { path = "../crowdb-access-multipart" } crowdb-chunk-client = { path = "../crowdb-chunk-client" } crowdb-chunk-kv-client = { path = "../crowdb-chunk-kv-client" } crowdb-diskio-client = { path = "../crowdb-diskio-client" } -crowdb-kv-client = { path = "../crowdb-kv-client" } +crowdb-paxos-kv-client = { path = "../crowdb-paxos-kv-client" } crowdb-protocol = { path = "../crowdb-protocol" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi" } futures = "0.3" diff --git a/lib/crowdb-sdk-native/src/runtime/attachment.rs b/lib/crowdb-sdk-native/src/runtime/attachment.rs index e5baefdbc..af4a27259 100644 --- a/lib/crowdb-sdk-native/src/runtime/attachment.rs +++ b/lib/crowdb-sdk-native/src/runtime/attachment.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use arc_swap::ArcSwap; use crowdb_chunk_client::ChunkIoClient; use crowdb_chunk_kv_client::{ChunkKvClient, ChunkKvRpcTransport}; -use crowdb_kv_client::CrowdbKvClient; +use crowdb_paxos_kv_client::PaxosKvClient; use super::{ClientRuntime, RuntimeLimits, RuntimeResources, Storage}; use crate::topology::CatalogObservation; @@ -15,7 +15,7 @@ use crate::{Error, Result}; /// Existing client owners attached by an embedding system. The catalog must /// already be validated; this path never reconnects or creates HTTP listeners. pub struct RuntimeClients { - pub control: Arc<CrowdbKvClient>, + pub control: Arc<PaxosKvClient>, pub metadata: Arc<ChunkKvClient>, pub metadata_transport: Arc<ChunkKvRpcTransport>, pub chunks: Arc<ChunkIoClient>, diff --git a/lib/crowdb-sdk-native/src/runtime/storage.rs b/lib/crowdb-sdk-native/src/runtime/storage.rs index e6398f227..1b771f611 100644 --- a/lib/crowdb-sdk-native/src/runtime/storage.rs +++ b/lib/crowdb-sdk-native/src/runtime/storage.rs @@ -10,7 +10,7 @@ use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, SmallWritePolicy}; use crowdb_chunk_kv_client::{ ChunkKvClient, ChunkKvRpcTransport, ClientConfig, Group0ChunkKvRangeCatalogSource, }; -use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig as KvConfig, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::ChunkType; use super::RuntimeConfig; @@ -20,7 +20,7 @@ use crate::{Error, Result}; pub(crate) struct Storage { pub owns_small_pool: bool, pub owns_transports: bool, - pub control: Arc<CrowdbKvClient>, + pub control: Arc<PaxosKvClient>, pub metadata: Arc<ChunkKvClient>, pub chunks: Arc<ChunkIoClient>, pub chunk_type: ChunkType, @@ -34,9 +34,7 @@ impl Storage { pub async fn connect(config: RuntimeConfig) -> Result<Self> { let policy = config.small_write.clone(); let chunk_type = config.small_write.chunk_type; - let control = Arc::new(CrowdbKvClient::new(KvConfig::new( - config.management_seeds.clone(), - ))); + let control = Arc::new(PaxosKvClient::new(KvConfig::new(config.management_seeds.clone()))); let catalog = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(Arc::clone(&control))); let metadata_config = ClientConfig::default(); let transport = Arc::new( diff --git a/lib/crowdb-sdk-native/src/topology.rs b/lib/crowdb-sdk-native/src/topology.rs index bc1625bb0..45374a547 100644 --- a/lib/crowdb-sdk-native/src/topology.rs +++ b/lib/crowdb-sdk-native/src/topology.rs @@ -11,7 +11,7 @@ use std::time::{Duration, Instant}; use crowdb_chunk_kv_client::ChunkKvRangeCatalogMap; use crowdb_diskio_client::DiskTopologyView; -use crowdb_kv_client::ServiceRegistryClient; +use crowdb_paxos_kv_client::ServiceRegistryClient; use crowdb_protocol::common::InstanceValue; use crate::error::{TopologyFailureKind, TopologySource}; diff --git a/lib/crowdb-sdk-native/tests/common/dataset_transport.rs b/lib/crowdb-sdk-native/tests/common/dataset_transport.rs index 911111801..8ef3c39ac 100644 --- a/lib/crowdb-sdk-native/tests/common/dataset_transport.rs +++ b/lib/crowdb-sdk-native/tests/common/dataset_transport.rs @@ -17,7 +17,7 @@ use crowdb_access_dataset::{ use crowdb_chunk_client::{ChunkIoClient, RoutedDiskWriter, SmallWriteIntent}; use crowdb_chunk_kv_client::{ChunkKvRpcTransport, ChunkKvRpcTransportConfig}; use crowdb_diskio_client::DiskioClient; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::{ chunkdb::rpc::{ChunkType, Location}, common::ChunkId, @@ -90,7 +90,7 @@ impl TestDatasetTransport { let authority = DatasetAuthority::new(store.clone()); let runtime = ClientRuntime::adopt_clients( RuntimeClients { - control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + control: Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))), metadata, metadata_transport: Arc::new( ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap(), diff --git a/lib/crowdb-sdk-native/tests/common/read_storage.rs b/lib/crowdb-sdk-native/tests/common/read_storage.rs index 79c360659..cfbebeb42 100644 --- a/lib/crowdb-sdk-native/tests/common/read_storage.rs +++ b/lib/crowdb-sdk-native/tests/common/read_storage.rs @@ -12,7 +12,7 @@ use crowdb_access_s3::integrity::SinglePartIntegrity; use crowdb_access_s3::metadata::{BucketId, ChunkKvMetadataStore, MetadataKey, ObjectRecord, TenantId}; use crowdb_chunk_client::{ChunkAllocator, ChunkIoClient, DiskWriter, Result}; use crowdb_chunk_kv_client::{ChunkKvRpcTransport, ChunkKvRpcTransportConfig}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::*; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::Segment; @@ -166,7 +166,7 @@ impl TestFixture { let metadata = super::metadata::client().await; let runtime = ClientRuntime::adopt_clients( RuntimeClients { - control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + control: Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))), metadata: metadata.clone(), metadata_transport: Arc::new( ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap(), diff --git a/lib/crowdb-sdk-native/tests/common/routed_write_batch.rs b/lib/crowdb-sdk-native/tests/common/routed_write_batch.rs index ad372ed6a..4148d188a 100644 --- a/lib/crowdb-sdk-native/tests/common/routed_write_batch.rs +++ b/lib/crowdb-sdk-native/tests/common/routed_write_batch.rs @@ -11,7 +11,7 @@ use crowdb_chunk_kv_client::{ BatchItem, ChunkKvClient, ChunkKvRpcTransport, ChunkKvRpcTransportConfig, ClientConfig, Group0ChunkKvRangeCatalogSource, }; -use crowdb_kv_client::{ClientConfig as ControlConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig as ControlConfig, PaxosKvClient}; use crowdb_protocol::chunk_kv::PointOperation; use crowdb_rpc_ffi::RdmaConfig; use futures::{stream::FuturesUnordered, StreamExt}; @@ -71,7 +71,7 @@ impl TestRoutedBatch { if seeds.is_empty() { return Err("measurement requires explicit management seeds".into()); } - let control = Arc::new(CrowdbKvClient::new(ControlConfig::new(seeds))); + let control = Arc::new(PaxosKvClient::new(ControlConfig::new(seeds))); let config = ClientConfig::default(); let partition_group_concurrency = config.max_inflight_partition_groups; let native = Arc::new(ChunkKvRpcTransport::with_config( diff --git a/lib/crowdb-sdk-native/tests/common/rpc_read.rs b/lib/crowdb-sdk-native/tests/common/rpc_read.rs index 91473f6e4..748f274b4 100644 --- a/lib/crowdb-sdk-native/tests/common/rpc_read.rs +++ b/lib/crowdb-sdk-native/tests/common/rpc_read.rs @@ -75,6 +75,7 @@ impl TestRpcReadServer { id: request.request_id, rpc_create_nano: 0, ret_code: FBDiskIoRetCode::Success, + request_age_checked: false, }, ); builder.finish(response, None); diff --git a/lib/crowdb-sdk-native/tests/common/storage.rs b/lib/crowdb-sdk-native/tests/common/storage.rs index b8ae465d9..5ddae0743 100644 --- a/lib/crowdb-sdk-native/tests/common/storage.rs +++ b/lib/crowdb-sdk-native/tests/common/storage.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use bytes::Bytes; use crowdb_chunk_client::{ChunkAllocator, ChunkIoClient, DiskWriter, Result}; use crowdb_chunk_kv_client::{ChunkKvRpcTransport, ChunkKvRpcTransportConfig}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::*; use crowdb_protocol::diskdb::rpc::Segment; use crowdb_sdk_native::runtime::RuntimeClients; @@ -61,7 +61,7 @@ pub fn runtime_with_limits( Arc::new(ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap()); ClientRuntime::from_clients( RuntimeClients { - control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + control: Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))), metadata, metadata_transport, chunks: Arc::new(ChunkIoClient::from_parts( diff --git a/lib/crowdb-sdk-native/tests/common/write_storage.rs b/lib/crowdb-sdk-native/tests/common/write_storage.rs index 430a8b110..7630d27fa 100644 --- a/lib/crowdb-sdk-native/tests/common/write_storage.rs +++ b/lib/crowdb-sdk-native/tests/common/write_storage.rs @@ -10,7 +10,7 @@ use bytes::Bytes; use crossbeam_skiplist::SkipMap; use crowdb_chunk_client::{ChunkAllocator, ChunkIoClient, DiskWriter, Result}; use crowdb_chunk_kv_client::{ChunkKvClient, ChunkKvRpcTransport, ChunkKvRpcTransportConfig}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_paxos_kv_client::{ClientConfig, PaxosKvClient}; use crowdb_protocol::chunkdb::rpc::*; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::Segment; @@ -95,7 +95,7 @@ impl TestWriteStorage { }; attach( RuntimeClients { - control: Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))), + control: Arc::new(PaxosKvClient::new(ClientConfig::new(Vec::new()))), metadata, metadata_transport: Arc::new( ChunkKvRpcTransport::with_config(8, 1, 1, &ChunkKvRpcTransportConfig::default()).unwrap(), diff --git a/lib/crowdb-tree/CMakeLists.txt b/lib/crowdb-tree/CMakeLists.txt index 26c54fce6..c3757d744 100644 --- a/lib/crowdb-tree/CMakeLists.txt +++ b/lib/crowdb-tree/CMakeLists.txt @@ -18,6 +18,9 @@ if(CROWDB_TREE_SANITIZER) message(STATUS "crowdb-tree sanitizer: ${CROWDB_TREE_SANITIZER}") add_compile_options(-fsanitize=${CROWDB_TREE_SANITIZER} -fno-omit-frame-pointer -g) add_link_options(-fsanitize=${CROWDB_TREE_SANITIZER}) + if(CROWDB_TREE_SANITIZER STREQUAL "thread") + set(CROWDB_COMMON_SPDLOG_HEADER_ONLY ON) + endif() endif() # ── liburing (io_uring reactor) ──────────────────────────────────── @@ -39,6 +42,21 @@ find_library(LIBURING_LIB NAMES uring HINTS ENV CONDA_PREFIX PATH_SUFFIXES lib) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../crowdb-common/cpp crowdb-common-build) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../crowdb-rpc crowdb-rpc-build EXCLUDE_FROM_ALL) +# Audited allocator hooks are generated outside the pinned vendor checkout. +find_package(Python3 REQUIRED COMPONENTS Interpreter) +set(CROWDB_STDEXEC_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/stdexec/include) +set(CROWDB_STDEXEC_OVERLAY ${CMAKE_CURRENT_BINARY_DIR}/stdexec-pool) +file(GLOB_RECURSE CROWDB_STDEXEC_HEADERS CONFIGURE_DEPENDS ${CROWDB_STDEXEC_SOURCE}/*) +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + ${CROWDB_STDEXEC_HEADERS} + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/prepare_stdexec.py + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/stdexec-pool.patch + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/stdexec-pool-base.json) +execute_process(COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/prepare_stdexec.py + ${CROWDB_STDEXEC_SOURCE} ${CROWDB_STDEXEC_OVERLAY} + COMMAND_ERROR_IS_FATAL ANY) + # ── Library ─────────────────────────────────────────────────────── file(GLOB_RECURSE CROWDB_TREE_SRC CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp) if(NOT (LIBURING_INCLUDE_DIR AND LIBURING_LIB)) @@ -55,7 +73,7 @@ endif() target_include_directories(crowdb-tree PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) target_include_directories(crowdb-tree PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) target_include_directories(crowdb-tree PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/crowdb-rpc-build/generated) -target_include_directories(crowdb-tree SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/stdexec/include) +target_include_directories(crowdb-tree SYSTEM PRIVATE ${CROWDB_STDEXEC_OVERLAY}/include) find_package(Threads REQUIRED) target_link_libraries(crowdb-tree PUBLIC Threads::Threads crowdbcommon PRIVATE crowdb-rpc) @@ -109,7 +127,7 @@ add_executable(crowdb_tree_tests ${CROWDB_TREE_TEST_SRC}) target_include_directories(crowdb_tree_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests ${CMAKE_CURRENT_SOURCE_DIR}/src) target_include_directories(crowdb_tree_tests PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/crowdb-rpc-build/generated) target_include_directories(crowdb_tree_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../crowdb-rpc/include) -target_include_directories(crowdb_tree_tests SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/stdexec/include) +target_include_directories(crowdb_tree_tests SYSTEM PRIVATE ${CROWDB_STDEXEC_OVERLAY}/include) target_link_libraries(crowdb_tree_tests PRIVATE crowdb-tree crowdb-rpc GTest::gtest) # GCC 13 -Warray-bounds/-Wstringop-overflow false positives: buffer::alloc # returns heap memory, but GCC thinks b.data() points into the stack object b. @@ -131,6 +149,7 @@ if(CROWDB_TREE_BENCH) find_package(benchmark REQUIRED) file(GLOB CROWDB_TREE_BENCH_SRC CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/bench/*.cpp) list(FILTER CROWDB_TREE_BENCH_SRC EXCLUDE REGEX "scan_step_bench\\.cpp$") + list(FILTER CROWDB_TREE_BENCH_SRC EXCLUDE REGEX "parallel_flush_profile\\.cpp$") add_executable(crowdb_tree_bench ${CROWDB_TREE_BENCH_SRC}) target_include_directories(crowdb_tree_bench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) target_link_libraries(crowdb_tree_bench PRIVATE benchmark::benchmark absl::btree crowdb-tree) @@ -149,4 +168,6 @@ if(CROWDB_TREE_BENCH) # dependency. Isolates the 1KiB anomaly's root cause via Crowtree::scan_profile. add_executable(scan_step_profile bench/scan_step_bench.cpp) target_link_libraries(scan_step_profile PRIVATE crowdb-tree absl::btree) + add_executable(parallel_flush_profile bench/parallel_flush_profile.cpp) + target_link_libraries(parallel_flush_profile PRIVATE crowdb-tree) endif() diff --git a/lib/crowdb-tree/bench/README.md b/lib/crowdb-tree/bench/README.md index 18fdb1d90..a403b0bea 100644 --- a/lib/crowdb-tree/bench/README.md +++ b/lib/crowdb-tree/bench/README.md @@ -3,6 +3,48 @@ # MemTable handoff benchmark +## Parallel flush profile + +`parallel_flush_profile.cpp` is a standalone checked C ABI workload for comparing +the engine before parallel flush with the current implementation. Build the +identical driver against each engine in separate Release directories, with +`CROWDB_TREE_BENCH=ON` and test hooks disabled: + +```sh +pixi run cmake -S lib/crowdb-tree -B lib/crowdb-tree/build-perf \ + -DCMAKE_BUILD_TYPE=Release -DCROWDB_TREE_BENCH=ON \ + -DCROWDB_TREE_TEST_UTIL=OFF -DCROWDB_RPC_TEST_UTIL=OFF +pixi run cmake --build lib/crowdb-tree/build-perf \ + --target parallel_flush_profile --parallel 4 +pixi run env CROWDB_PROFILE_CACHE_BYTES=268435456 \ + CROWDB_PROFILE_CACHE_PAGES=4096 \ + lib/crowdb-tree/build-perf/parallel_flush_profile 4 4096 16384 16 random 0 1 0 +``` + +- Arguments are workers, entries per Tree, value bytes, batch size, pattern, + cold flag, Tree count and histogram flag. Patterns are `sequential`, `random`, + `hot`, `delete` and `long-key`. Cold mode evicts clean leaf/inner pages before + point verification and before each scan repetition. +- Each process checks two write/flush/checkpoint passes, verifies every point + result, and performs 32 full scans per pass. Compare apply + flush + snapshot + together; a shorter flush can defer work to the checkpoint. Multiple Trees + are exercised sequentially on a shared runtime, so this mode does not prove + concurrent fairness or sustainable server throughput. +- `CROWDB_PROFILE_SOURCE_ENTRIES` defaults to 4096. Cache defaults are 64MiB + and 256 page slots. Byte capacity and page capacity both apply in the current + engine; the old engine only supports byte capacity. Set both explicitly when + comparing resident workloads. The JSON records the effective driver settings. +- Current-engine limits per Tree are Handoff 256MiB, Routing 512MiB, Reclaim + 256MiB and Writer 512MiB. The 8MiB lifecycle reserve includes fixed controls + for larger cache/queue profiles. Host byte limits scale with Tree count. + These are benchmark capacities, not production defaults or preallocated RSS. +- Run repetitions without concurrent builds or test suites. Record exact + source/diff and binary hashes, compiler flags, CPU quota, memory limits and + histogram settings. Keep resource-limited runs separate from ample-resource + runs. A microbenchmark does not replace the three-node backlog/idle-drain case. + +## Earlier MemTable comparison + Local comparison recorded on 2026-10-03 for implementation `139149b7`. The benchmark is `memtable_handoff_bench.cpp`; the current handoff contract is in [the engine design](../../../doc/design/tree/design-crowdb-tree-engine.md). diff --git a/lib/crowdb-tree/bench/parallel_flush_profile.cpp b/lib/crowdb-tree/bench/parallel_flush_profile.cpp new file mode 100644 index 000000000..856c35a7d --- /dev/null +++ b/lib/crowdb-tree/bench/parallel_flush_profile.cpp @@ -0,0 +1,358 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +// One identical checked workload for the pre-parallel engine and current runtime. +#include "crowdb-tree/c_api.h" +#if __has_include("crowdb-tree/tree_work_histograms_c_api.h") +# include "crowdb-tree/runtime_c_api.h" +# include "crowdb-tree/tree_stats_c_api.h" +# include "crowdb-tree/tree_work_histograms_c_api.h" +# include "crowdb-tree/tuning_c_api.h" +# define CROWDB_PROFILE_RUNTIME 1 +#endif + +#include <sys/resource.h> + +#include <algorithm> +#include <chrono> +#include <cstdlib> +#include <iostream> +#include <memory> +#include <stdexcept> +#include <string> +#include <vector> + +namespace +{ +using Clock = std::chrono::steady_clock; +using Tree = std::unique_ptr<ct_tree, decltype(&ct_close)>; + +void check(int32_t status, const char *operation) +{ + if (status != 0) { + throw std::runtime_error(std::string(operation) + ": " + std::to_string(status)); + } +} + +uint64_t elapsed(Clock::time_point start) +{ + return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(Clock::now() - start).count()); +} + +struct Options +{ + size_t workers, entries, value_bytes, batch, cold, trees, histogram; + std::string pattern; + size_t source_entries = 4096; + size_t cache_bytes = 64ULL * 1024 * 1024; + size_t cache_pages = 256; +}; + +#ifdef CROWDB_PROFILE_RUNTIME +ct_tree_runtime_options tree_resources() +{ + ct_tree_runtime_options resources{}; + ct_tree_runtime_options_default(&resources); + // Leave headroom for retained owners and the next admitted work window. + constexpr size_t mib = 1024 * 1024; + // Fixed cache/queue descriptors must fit even in the ample-cache profile. + resources.lifecycle_bytes = 8 * mib; + resources.limits.handoff_bytes = 256 * mib; + resources.limits.routing_bytes = 512 * mib; + resources.limits.reclaim_bytes = 256 * mib; + resources.limits.writer_bytes = 512 * mib; + return resources; +} +#endif + +std::string key(size_t id, const Options &options) +{ + auto result = std::string(20 - std::to_string(id).size(), '0') + std::to_string(id); + if (options.pattern == "long-key") { + result += std::string(1024, 'k'); + } + return result; +} + +std::vector<Tree> open_trees(const Options &options) +{ +#ifdef CROWDB_PROFILE_RUNTIME + ct_runtime_options settings{}; + ct_runtime_options_default(&settings); + settings.workers = options.workers; + const auto resources = tree_resources(); + settings.limits.handoff_bytes = resources.limits.handoff_bytes * options.trees; + settings.limits.routing_bytes = resources.limits.routing_bytes * options.trees; + settings.limits.reclaim_bytes = resources.limits.reclaim_bytes * options.trees; + settings.limits.writer_bytes = resources.limits.writer_bytes * options.trees; + ct_runtime *raw = nullptr; + check(ct_runtime_create(&settings, &raw), "runtime create"); + std::unique_ptr<ct_runtime, decltype(&ct_runtime_free)> runtime(raw, ct_runtime_free); +#endif + std::vector<Tree> trees; + trees.reserve(options.trees); + for (size_t index = 0; index < options.trees; ++index) { + ct_page_store *store = nullptr; + check(ct_page_store_open_mem(1, &store), "mem store"); + std::unique_ptr<ct_page_store, decltype(&ct_page_store_free)> owner(store, ct_page_store_free); + ct_options config{}; + config.page_store = store; + config.frame_bytes = 65536; + config.buffer_pool_bytes = options.cache_bytes; + ct_tree *tree = nullptr; +#ifdef CROWDB_PROFILE_RUNTIME + ct_tree_tuning tuning{}; + ct_tree_tuning_default(&tuning); + tuning.source_max_entries = options.source_entries; + tuning.cache_max_pages = options.cache_pages; + config.tuning = &tuning; + check(ct_open_with_runtime(&config, raw, &resources, &tree), "open"); +#else + check(ct_open(&config, &tree), "open"); +#endif + trees.emplace_back(tree, ct_close); +#ifdef CROWDB_PROFILE_RUNTIME + if (options.histogram != 0) { + for (uint32_t stage = 0; stage < CT_TREE_WORK_STAGES_V1; ++stage) { + check(ct_work_histogram_enable_v1(tree, stage), "enable histogram"); + } + } +#endif + } + return trees; +} + +void write_pass(ct_tree *tree, const Options &options, size_t pass, std::vector<uint64_t> *latencies) +{ + std::vector<std::string> keys(options.batch); + std::vector<ct_kv_ref> operations(options.batch); + const std::string value(options.value_bytes, pass == 0 ? 'v' : 'w'); + for (size_t begin = 0; begin < options.entries; begin += options.batch) { + const auto count = std::min(options.batch, options.entries - begin); + for (size_t item = 0; item < count; ++item) { + auto id = begin + item; + if (pass != 0 && options.pattern == "random") { + id = ((id * 2654435761ULL) + 31) % options.entries; + } + if (pass != 0 && options.pattern == "hot") { + id %= 64; + } + keys[item] = key(id, options); + const bool erase = pass != 0 && options.pattern == "delete" && id % 2 == 0; + operations[item] = {.key = reinterpret_cast<const uint8_t *>(keys[item].data()), + .key_len = keys[item].size(), + .value = reinterpret_cast<const uint8_t *>(value.data()), + .value_len = erase ? 0 : value.size(), + .kind = static_cast<uint8_t>(erase ? 1 : 0)}; + } + const auto start = Clock::now(); + const auto batches = (options.entries + options.batch - 1) / options.batch; + check(ct_apply_batch_slices(tree, (pass * batches) + (begin / options.batch) + 1, operations.data(), count), + "apply"); + latencies->push_back(elapsed(start)); + } +} + +void verify(ct_tree *tree, const Options &options, size_t pass, std::vector<uint64_t> *latencies) +{ + for (size_t id = 0; id < options.entries; ++id) { + const bool replaced = pass != 0 && (options.pattern != "hot" || id < 64); + const std::string expected(options.value_bytes, replaced ? 'w' : 'v'); + const auto name = key(id, options); + ct_buf value{}; + int32_t found = 0; + uint64_t slot = 0; + const auto start = Clock::now(); + check(ct_get(tree, reinterpret_cast<const uint8_t *>(name.data()), name.size(), &found, &slot, &value), "get"); + latencies->push_back(elapsed(start)); + const bool present = pass == 0 || options.pattern != "delete" || id % 2 != 0; + const bool valid = (found != 0) == present && + (!present || std::string(reinterpret_cast<char *>(value.data), value.len) == expected); + ct_free_buf(&value); + if (!valid) { + throw std::runtime_error("point value mismatch at " + std::to_string(id)); + } + } +} + +void scan(ct_tree *tree, const Options &options, size_t pass) +{ + // Cover the whole keyspace through bounded responses on both engines. + // A single 64MiB result plus its read window exceeds the runtime budget. + for (size_t first = 0; first < options.entries; first += 64) { + const auto last = std::min(first + 64, options.entries); + const auto lower = key(first, options); + const auto upper = key(last, options); + ct_buf packed{}; + uint64_t count = 0; + int32_t truncated = 0; + check(ct_scan(tree, nullptr, 0, reinterpret_cast<const uint8_t *>(lower.data()), lower.size(), 1, 1, + reinterpret_cast<const uint8_t *>(upper.data()), upper.size(), 65, 0, 0, 0, 0, &packed, &count, + &truncated), + "scan"); + ct_free_buf(&packed); + const auto expected = pass != 0 && options.pattern == "delete" ? (last - first) / 2 : last - first; + if (count != expected || truncated != 0) { + throw std::runtime_error("scan count/truncation mismatch"); + } + } +} + +void metrics(ct_tree *tree) +{ +#ifdef CROWDB_PROFILE_RUNTIME + ct_tree_stats_snapshot_v1 state{}; + state.struct_size = sizeof(state); + state.version = 1; + check(ct_get_tree_state_stats(tree, &state), "tree stats"); + ct_tree_flush_dispatch_v1 dispatch{}; + dispatch.struct_size = sizeof(dispatch); + dispatch.version = 1; + check(ct_flush_dispatch_totals(tree, &dispatch), "flush stats"); + std::cout << ",\"allocated_bytes\":" << state.resources.allocated_bytes + << ",\"retired_bytes\":" << state.resources.retired_bytes << ",\"cow_bytes\":" << dispatch.cow_bytes + << ",\"prepared_bytes\":" << dispatch.prepared_bytes << ",\"work_counters\":{"; + ct_tree_work_counters_v1 counters{}; + counters.struct_size = sizeof(counters); + counters.version = 1; + check(ct_work_counters_totals(tree, &counters), "work counters"); + for (uint32_t counter = 0; counter < CT_TREE_WORK_COUNTERS_V1; ++counter) { + uint64_t sum = 0; + for (auto &value : counters.values) { + sum += value[counter]; + } + std::cout << (counter == 0 ? "" : ",") << '"' << ct_work_counter_name_v1(counter) << "\":" << sum; + } + std::cout << '}'; + ct_tree_work_snapshot_v1 work{}; + work.struct_size = sizeof(work); + work.version = 1; + check(ct_work_metrics_totals(tree, &work), "work stages"); + std::cout << ",\"stage_totals\":["; + for (uint32_t stage = 0; stage < CT_TREE_WORK_STAGES_V1; ++stage) { + const auto &value = work.stages[stage]; + std::cout << (stage == 0 ? "" : ",") << "{\"count\":" << value.total_count + << ",\"sum_ns\":" << value.total_sum_ns << ",\"max_ns\":" << value.total_max_ns << '}'; + } + std::cout << ']'; + const auto begin = Clock::now(); + for (size_t index = 0; index < 10000; ++index) { + check(ct_get_tree_state_stats(tree, &state), "collect stats"); + } + std::cout << ",\"stats_10000_ns\":" << elapsed(begin); + ct_resource_usage_snapshot_v1 resources{}; + resources.struct_size = sizeof(resources); + resources.version = 1; + const auto resource_begin = Clock::now(); + for (size_t index = 0; index < 10000; ++index) { + check(ct_tree_resource_usage(tree, &resources), "collect resource usage"); + } + std::cout << ",\"resource_usage_10000_ns\":" << elapsed(resource_begin); +#else + (void)tree; +#endif +} +} // namespace + +int main(int argc, char **argv) +try { + if (argc != 9) { + throw std::runtime_error("args: workers entries value-bytes batch pattern cold trees histogram"); + } + Options options{.workers = std::stoull(argv[1]), + .entries = std::stoull(argv[2]), + .value_bytes = std::stoull(argv[3]), + .batch = std::stoull(argv[4]), + .cold = std::stoull(argv[6]), + .trees = std::stoull(argv[7]), + .histogram = std::stoull(argv[8]), + .pattern = argv[5]}; + // Diagnostic sweep of the existing source window; this is not a new server setting. + if (const auto *source_entries = std::getenv("CROWDB_PROFILE_SOURCE_ENTRIES")) { + options.source_entries = std::stoull(source_entries); + } + if (const auto *cache_bytes = std::getenv("CROWDB_PROFILE_CACHE_BYTES")) { + options.cache_bytes = std::stoull(cache_bytes); + } + if (const auto *cache_pages = std::getenv("CROWDB_PROFILE_CACHE_PAGES")) { + options.cache_pages = std::stoull(cache_pages); + } + if (options.entries < 64 || (options.entries & (options.entries - 1)) != 0 || options.batch == 0 || + options.batch > 64 || options.trees == 0 || options.workers == 0 || options.source_entries < options.batch || + options.cache_bytes < 65536 || options.cache_pages == 0) { + throw std::runtime_error("entries must be a power of two >=64, batch 1..64, positive trees/workers"); + } + ct_init_logging("/tmp/crowdb-parallel-flush-profile", "error", 1, 1, "profile"); + auto trees = open_trees(options); + std::vector<uint64_t> apply_latencies; + std::vector<uint64_t> read_latencies; + std::vector<uint64_t> scan_latencies; + uint64_t apply_ns = 0; + uint64_t flush_ns = 0; + uint64_t snapshot_ns = 0; + uint64_t get_ns = 0; + uint64_t scan_ns = 0; + for (size_t pass = 0; pass < 2; ++pass) { + for (auto &tree : trees) { + auto start = Clock::now(); + write_pass(tree.get(), options, pass, &apply_latencies); + apply_ns += elapsed(start); + start = Clock::now(); + check(ct_flush(tree.get()), "flush"); + flush_ns += elapsed(start); + start = Clock::now(); + uint64_t durable_slot = 0; + check(ct_snapshot(tree.get(), &durable_slot), "snapshot"); + ct_set_gc_watermark(tree.get(), durable_slot, durable_slot); + snapshot_ns += elapsed(start); + if (options.cold != 0) { + ct_evict_clean_leaves(tree.get(), 0); + ct_evict_clean_inner(tree.get(), 0); + } + start = Clock::now(); + verify(tree.get(), options, pass, &read_latencies); + get_ns += elapsed(start); + for (size_t repeat = 0; repeat < 32; ++repeat) { + if (options.cold != 0) { + ct_evict_clean_leaves(tree.get(), 0); + ct_evict_clean_inner(tree.get(), 0); + } + start = Clock::now(); + scan(tree.get(), options, pass); + const auto duration = elapsed(start); + scan_ns += duration; + scan_latencies.push_back(duration); + } + } + } + std::sort(apply_latencies.begin(), apply_latencies.end()); + std::sort(read_latencies.begin(), read_latencies.end()); + std::sort(scan_latencies.begin(), scan_latencies.end()); + rusage usage{}; + getrusage(RUSAGE_SELF, &usage); + std::cout << R"({"passed":true,"workers":)" << options.workers << ",\"entries_per_tree\":" << options.entries + << ",\"value_bytes\":" << options.value_bytes << ",\"batch\":" << options.batch << R"(,"pattern":")" + << options.pattern << R"(","cold":)" << options.cold << ",\"trees\":" << options.trees + << ",\"histogram\":" << options.histogram << ",\"source_entries\":" << options.source_entries + << ",\"cache_bytes\":" << options.cache_bytes << ",\"cache_pages\":" << options.cache_pages + << ",\"apply_ns\":" << apply_ns << ",\"flush_ns\":" << flush_ns << ",\"snapshot_ns\":" << snapshot_ns + << ",\"get_ns\":" << get_ns << ",\"scan_ns\":" << scan_ns + << ",\"batch_p50_ns\":" << apply_latencies[apply_latencies.size() / 2] + << ",\"batch_p99_ns\":" << apply_latencies[apply_latencies.size() * 99 / 100] + << ",\"get_p50_ns\":" << read_latencies[read_latencies.size() / 2] + << ",\"get_p99_ns\":" << read_latencies[read_latencies.size() * 99 / 100] + << ",\"scan_p50_ns\":" << scan_latencies[scan_latencies.size() / 2] + << ",\"scan_p99_ns\":" << scan_latencies[scan_latencies.size() * 99 / 100] + << ",\"peak_rss_kib\":" << usage.ru_maxrss; + metrics(trees.front().get()); + const auto closing = Clock::now(); + trees.clear(); + std::cout << ",\"close_ns\":" << elapsed(closing) << "}\n"; + ct_shutdown_logging(); + return 0; +} +catch (const std::exception &error) { + ct_shutdown_logging(); + std::cerr << "parallel flush profile failed: " << error.what() << '\n'; + return 1; +} diff --git a/lib/crowdb-tree/cmake/prepare_stdexec.py b/lib/crowdb-tree/cmake/prepare_stdexec.py new file mode 100644 index 000000000..52e7a295d --- /dev/null +++ b/lib/crowdb-tree/cmake/prepare_stdexec.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. + +"""Build an audited stdexec include overlay without modifying the submodule.""" +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +def prepare(source: Path, output: Path) -> None: + here = Path(__file__).resolve().parent + patch = here / "stdexec-pool.patch" + expected = json.loads((here / "stdexec-pool-base.json").read_text()) + for name, digest in expected.items(): + if hashlib.sha256((source / name).read_bytes()).hexdigest() != digest: + raise RuntimeError(f"stdexec pool baseline changed: {name}; review the allocator patch") + fingerprint = hashlib.sha256(patch.read_bytes()) + for path in sorted(source.rglob("*")): + if path.is_file(): + fingerprint.update(str(path.relative_to(source)).encode()) + fingerprint.update(path.read_bytes()) + signature = fingerprint.hexdigest() + stamp = output / "stamp" + if stamp.is_file() and stamp.read_text() == signature: + return + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="stdexec-pool-", dir=output.parent) as temporary: + staging = Path(temporary) / "overlay" + shutil.copytree(source, staging / "include") + subprocess.run(["git", "apply", "--check", "--unidiff-zero", str(patch)], cwd=staging, check=True) + subprocess.run(["git", "apply", "--unidiff-zero", str(patch)], cwd=staging, check=True) + (staging / "stamp").write_text(signature) + if output.exists(): + shutil.rmtree(output) + staging.rename(output) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + raise SystemExit("usage: prepare_stdexec.py VENDOR_INCLUDE OUTPUT") + prepare(Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()) diff --git a/lib/crowdb-tree/cmake/stdexec-pool-base.json b/lib/crowdb-tree/cmake/stdexec-pool-base.json new file mode 100644 index 000000000..29994c82e --- /dev/null +++ b/lib/crowdb-tree/cmake/stdexec-pool-base.json @@ -0,0 +1,4 @@ +{ + "exec/static_thread_pool.hpp": "672ecff626603ec62bfbfbf7c2ea7f424620543b66256d1fe2faeef005f13214", + "exec/detail/bwos_lifo_queue.hpp": "7aaafd413e97f9b4d8fa804f1fb18f9c1814ecba8e734ff667e63a1d3e6e49a2" +} diff --git a/lib/crowdb-tree/cmake/stdexec-pool.patch b/lib/crowdb-tree/cmake/stdexec-pool.patch new file mode 100644 index 000000000..956e2e19b --- /dev/null +++ b/lib/crowdb-tree/cmake/stdexec-pool.patch @@ -0,0 +1,258 @@ +--- a/include/exec/static_thread_pool.hpp ++++ b/include/exec/static_thread_pool.hpp +@@ -57,0 +58,2 @@ ++#include "detail/pool_resource.hpp" ++#include <optional> +@@ -129,2 +131,3 @@ +- explicit remote_queue(std::size_t nthreads) noexcept +- : queues_(nthreads) ++ using queue_type = __atomic_intrusive_queue<&task_base::next_>; ++ explicit remote_queue(std::size_t nthreads, std::pmr::memory_resource* resource) ++ : queues_(nthreads, pool_allocator<queue_type>(resource)) +@@ -133 +136 @@ +- explicit remote_queue(remote_queue* next, std::size_t nthreads) noexcept ++ explicit remote_queue(remote_queue* next, std::size_t nthreads, std::pmr::memory_resource* resource) +@@ -135 +138 @@ +- , queues_(nthreads) ++ , queues_(nthreads, pool_allocator<queue_type>(resource)) +@@ -139 +142 @@ +- std::vector<__atomic_intrusive_queue<&task_base::next_>> queues_{}; ++ pool_vector<queue_type> queues_; +@@ -157,0 +161 @@ ++ pool_allocator<remote_queue> allocator_; +@@ -160 +164 @@ +- explicit remote_queue_list(std::size_t nthreads) noexcept ++ explicit remote_queue_list(std::size_t nthreads, std::pmr::memory_resource* resource) +@@ -164 +168,2 @@ +- , this_remotes_(nthreads) ++ , this_remotes_(nthreads, resource) ++ , allocator_(resource) +@@ -173 +178,2 @@ +- delete tmp; ++ std::destroy_at(tmp); ++ allocator_.deallocate(tmp, 1); +@@ -207 +213,3 @@ +- auto* new_head = new remote_queue{head, nthreads_}; ++ auto* new_head = allocator_.allocate(1); ++ try { std::construct_at(new_head, head, nthreads_, this_remotes_.queues_.get_allocator().resource()); } ++ catch (...) { allocator_.deallocate(new_head, 1); throw; } +@@ -339 +347,4 @@ +- numa_policy numa = get_numa_policy()); ++ numa_policy numa = get_numa_policy(), ++ std::pmr::memory_resource* resource = nullptr, ++ pool_thread_factory factory = {}); ++ static auto allocation_bound(std::uint32_t threads, bwos_params params = {}) noexcept -> std::optional<std::size_t>; +@@ -419 +430 @@ +- nodemask const * mask = &nodemask::any()) noexcept ++ nodemask const * mask = &nodemask::any()) +@@ -475 +486 @@ +- auto get_scheduler() noexcept -> scheduler ++ auto get_scheduler() -> scheduler +@@ -480 +491 @@ +- auto get_scheduler_on_thread(std::size_t threadIndex) noexcept -> scheduler ++ auto get_scheduler_on_thread(std::size_t threadIndex) -> scheduler +@@ -486 +497 @@ +- auto get_constrained_scheduler(nodemask const * constraints) noexcept -> scheduler ++ auto get_constrained_scheduler(nodemask const * constraints) -> scheduler +@@ -491 +502 @@ +- auto get_remote_queue() noexcept -> remote_queue* ++ auto get_remote_queue() -> remote_queue* +@@ -495 +506 @@ +- for (std::thread& t: threads_) ++ for (pool_thread& t: threads_) +@@ -545 +556 @@ +- bwos::lifo_queue<task_base*, numa_allocator<task_base*>>* queue, ++ bwos::lifo_queue<task_base*, pool_allocator<task_base*>>* queue, +@@ -571 +582 @@ +- bwos::lifo_queue<task_base*, numa_allocator<task_base*>>* queue_; ++ bwos::lifo_queue<task_base*, pool_allocator<task_base*>>* queue_; +@@ -599 +610 @@ +- numa_policy const & numa) noexcept ++ numa_policy const & numa) +@@ -603 +614,3 @@ +- numa_allocator<task_base*>(this->numa_node_)) ++ pool_allocator<task_base*>(pool->resource_, this->numa_node_)) ++ , near_victims_(pool_allocator<workstealing_victim>(pool->resource_)) ++ , all_victims_(pool_allocator<workstealing_victim>(pool->resource_)) +@@ -607,2 +620,2 @@ +- std::random_device rd; +- rng_.seed(rd); ++ if (pool->resource_) rng_ = xorshift(0xc1f651c67c62c6e0ull ^ (std::uint64_t(index) + 1)); ++ else { std::random_device rd; rng_.seed(rd); } +@@ -618,2 +631,4 @@ +- void victims(std::vector<workstealing_victim> const & victims) +- { ++ void victims(pool_vector<workstealing_victim> const & victims) ++ { ++ near_victims_.reserve(victims.size() - 1); ++ all_victims_.reserve(victims.size() - 1); +@@ -673 +688 @@ +- bwos::lifo_queue<task_base*, numa_allocator<task_base*>> local_queue_; ++ bwos::lifo_queue<task_base*, pool_allocator<task_base*>> local_queue_; +@@ -678,2 +693,2 @@ +- std::vector<workstealing_victim> near_victims_{}; +- std::vector<workstealing_victim> all_victims_{}; ++ pool_vector<workstealing_victim> near_victims_{}; ++ pool_vector<workstealing_victim> all_victims_{}; +@@ -687,0 +703,2 @@ ++ std::pmr::memory_resource* resource_; ++ pool_thread_factory factory_; +@@ -693,2 +710,2 @@ +- std::vector<std::thread> threads_; +- std::vector<__optional<thread_state>> thread_states_; ++ pool_vector<pool_thread> threads_; ++ pool_vector<__optional<thread_state>> thread_states_; +@@ -716 +733 @@ +- std::vector<thread_index_by_numa_node> thread_index_by_numa_node_; ++ pool_vector<thread_index_by_numa_node> thread_index_by_numa_node_; +@@ -727,0 +745,32 @@ ++ inline auto _static_thread_pool::allocation_bound(std::uint32_t threads, bwos_params params) noexcept ++ -> std::optional<std::size_t> { ++ constexpr auto limit = (std::numeric_limits<std::size_t>::max)() / 2; ++ if (!threads || threads > UINT16_MAX || !params.numBlocks || !params.blockSize || params.numBlocks > limit / 2) return {}; ++ const auto blocks = std::bit_ceil(params.numBlocks); ++ std::size_t bytes = 0; ++ auto add = [&](std::size_t count, std::size_t size) { ++ if (size && count > (limit - bytes) / size) return false; ++ bytes += count * size; ++ return true; ++ }; ++ using queue = bwos::lifo_queue<task_base*, pool_allocator<task_base*>>; ++ // Includes the one cached submitter queue, construction scratch and both ++ // victim lists. Other caller-thread queues, if used, are still charged. ++ if (!add(2 * std::size_t(threads), sizeof(remote_queue::queue_type)) || ++ !add(1, sizeof(remote_queue)) || !add(threads, sizeof(pool_thread)) || ++ !add(threads, sizeof(__optional<thread_state>)) || ++ !add(threads, sizeof(thread_index_by_numa_node)) || ++ !add(threads, sizeof(workstealing_victim)) || ++ !add(params.blockSize, sizeof(task_base*))) return {}; ++ const auto fixed_bytes = bytes; ++ bytes = 0; ++ if (!add(blocks, queue::block_storage_size()) || ++ params.blockSize > limit / blocks || ++ !add(blocks * params.blockSize, sizeof(task_base*)) || ++ !add(2 * (std::size_t(threads) - 1), sizeof(workstealing_victim))) return {}; ++ const auto per_thread = bytes; ++ bytes = fixed_bytes; ++ if (!add(threads, per_thread)) return {}; ++ return bytes; ++ } ++ +@@ -734,2 +783,6 @@ +- numa_policy numa) +- : remotes_(thread_count) ++ numa_policy numa, ++ std::pmr::memory_resource* resource, ++ pool_thread_factory factory) ++ : resource_(resource) ++ , factory_(factory) ++ , remotes_(thread_count, resource) +@@ -738 +791,2 @@ +- , thread_states_(thread_count) ++ , threads_(pool_allocator<pool_thread>(resource)) ++ , thread_states_(thread_count, pool_allocator<__optional<thread_state>>(resource)) +@@ -739,0 +794 @@ ++ , thread_index_by_numa_node_(pool_allocator<thread_index_by_numa_node>(resource)) +@@ -741,0 +797 @@ ++ thread_index_by_numa_node_.reserve(thread_count); +@@ -752 +808,2 @@ +- std::vector<workstealing_victim> victims{}; ++ pool_vector<workstealing_victim> victims{pool_allocator<workstealing_victim>(resource)}; ++ victims.reserve(thread_count); +@@ -768 +825,3 @@ +- threads_.emplace_back([this, i] { run(i); }); ++ threads_.emplace_back(factory_, [](void* context, std::uint32_t index) noexcept { ++ static_cast<_static_thread_pool*>(context)->run(index); ++ }, this, i); +@@ -981 +1040 @@ +- bwos::lifo_queue<task_base*, numa_allocator<task_base*>>& local_queue) ++ bwos::lifo_queue<task_base*, pool_allocator<task_base*>>& local_queue) +@@ -1835,2 +1894,4 @@ +- numa_policy numa = get_numa_policy()) +- : _pool_::_static_thread_pool(thread_count, params, std::move(numa)) ++ numa_policy numa = get_numa_policy(), ++ std::pmr::memory_resource* resource = nullptr, ++ pool_thread_factory factory = {}) ++ : _pool_::_static_thread_pool(thread_count, params, std::move(numa), resource, factory) +@@ -1837,0 +1899,2 @@ ++ ++ using _pool_::_static_thread_pool::allocation_bound; +--- a/include/exec/detail/bwos_lifo_queue.hpp ++++ b/include/exec/detail/bwos_lifo_queue.hpp +@@ -102,0 +103,2 @@ ++ ++ static auto block_storage_size() noexcept -> std::size_t { return sizeof(block_type); } +--- /dev/null ++++ b/include/exec/detail/pool_resource.hpp +@@ -0,0 +1,71 @@ ++// Copyright 2026-present Gian <crow.db@outlook.com> ++// Licensed under the Apache License, Version 2.0. ++#pragma once ++#include "numa.hpp" ++#include <memory_resource> ++#include <limits> ++#include <thread> ++#include <utility> ++#include <vector> ++ ++namespace experimental::execution { ++struct pool_thread_factory { ++ void* context{}; ++ void* (*start)(void*, void (*)(void*, std::uint32_t) noexcept, void*, std::uint32_t){}; ++ void (*join)(void*) noexcept{}; ++ std::thread::id (*id)(void*) noexcept{}; ++}; ++namespace _pool_ { ++template<class T> class pool_allocator { ++public: ++ using value_type = T; ++ explicit pool_allocator(std::pmr::memory_resource* resource = nullptr, int node = 0) noexcept ++ : resource_(resource), node_(node) {} ++ template<class U> pool_allocator(const pool_allocator<U>& other) noexcept ++ : resource_(other.resource_), node_(other.node_) {} ++ std::pmr::memory_resource* resource() const noexcept { return resource_; } ++ T* allocate(std::size_t n) { ++ if (n > (std::numeric_limits<std::size_t>::max)() / sizeof(T)) throw std::bad_array_new_length(); ++ if (resource_) return static_cast<T*>(resource_->allocate(n * sizeof(T), alignof(T))); ++ return numa_allocator<T>(node_).allocate(n); ++ } ++ void deallocate(T* pointer, std::size_t n) noexcept { ++ if (resource_) resource_->deallocate(pointer, n * sizeof(T), alignof(T)); ++ else numa_allocator<T>(node_).deallocate(pointer, n); ++ } ++ friend bool operator==(const pool_allocator&, const pool_allocator&) noexcept = default; ++private: ++ template<class> friend class pool_allocator; ++ std::pmr::memory_resource* resource_; ++ int node_; ++}; ++template<class T> using pool_vector = std::vector<T, pool_allocator<T>>; ++ ++// A successful start transfers one handle; join releases it after the worker returns. ++class pool_thread { ++public: ++ pool_thread(pool_thread_factory factory, void (*run)(void*, std::uint32_t) noexcept, ++ void* context, std::uint32_t index) : factory_(factory) { ++ if (factory_.start) handle_ = factory_.start(factory_.context, run, context, index); ++ else thread_ = std::thread([=] { run(context, index); }); ++ } ++ pool_thread(const pool_thread&) = delete; ++ pool_thread& operator=(const pool_thread&) = delete; ++ pool_thread(pool_thread&& other) noexcept ++ : factory_(other.factory_), handle_(std::exchange(other.handle_, nullptr)), ++ thread_(std::move(other.thread_)) {} ++ ~pool_thread() { if (handle_) std::terminate(); } ++ std::thread::id get_id() const noexcept { ++ return handle_ ? factory_.id(handle_) : thread_.get_id(); ++ } ++ void join() noexcept { ++ if (handle_) factory_.join(std::exchange(handle_, nullptr)); ++ else thread_.join(); ++ } ++private: ++ pool_thread_factory factory_; ++ void* handle_{}; ++ std::thread thread_; ++}; ++} ++} diff --git a/lib/crowdb-tree/ffi/Cargo.toml b/lib/crowdb-tree/ffi/Cargo.toml index 4c364918c..81a62f502 100644 --- a/lib/crowdb-tree/ffi/Cargo.toml +++ b/lib/crowdb-tree/ffi/Cargo.toml @@ -19,11 +19,12 @@ chunk-rpc = ["dep:crowdb-rpc-ffi"] test-util = [] [dependencies] +serde = { version = "1", features = ["derive"] } bytes = "1" crowdb-rpc-ffi = { path = "../../crowdb-rpc/ffi", optional = true } # "net" is required for tokio::io::unix::AsyncFd, "sync" for tokio::sync:: # Notify. -tokio = { version = "1.40", features = ["rt", "rt-multi-thread", "macros", "net", "sync"] } +tokio = { version = "1.40", features = ["rt", "rt-multi-thread", "macros", "net", "sync", "time"] } [build-dependencies] cc = "1" @@ -33,3 +34,7 @@ crowdb-test-harness = { path = "../../crowdb-test-harness" } # Self dev-dependency: turns on `test-util` so integration tests auto-init # C++ logging to a temp dir on first Crowdbtree::open. crowdb-tree-ffi = { path = ".", features = ["test-util"] } + +[[test]] +name = "runtime_wait_metrics_test" +path = "tests/runtime_wait_metrics_test.rs" diff --git a/lib/crowdb-tree/ffi/build.rs b/lib/crowdb-tree/ffi/build.rs index 7b3066600..783aebd41 100644 --- a/lib/crowdb-tree/ffi/build.rs +++ b/lib/crowdb-tree/ffi/build.rs @@ -46,7 +46,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { .join("cpp"); let common_src = common.join("src"); let common_include = common.join("include"); - let stdexec_include = engine + let stdexec_source = engine .parent() .and_then(|p| p.parent()) .ok_or("crowdb-tree must be under the repository lib directory")? @@ -57,6 +57,28 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { .parent() .and_then(|path| path.parent()) .ok_or("crowdb-tree must be under the repository lib directory")?; + let stdexec_overlay = PathBuf::from(std::env::var("OUT_DIR")?).join("stdexec-pool"); + let prepare_stdexec = engine.join("cmake").join("prepare_stdexec.py"); + if !Command::new("python3") + .arg(&prepare_stdexec) + .arg(&stdexec_source) + .arg(&stdexec_overlay) + .status()? + .success() + { + return Err("stdexec allocator overlay preparation failed".into()); + } + let stdexec_include = stdexec_overlay.join("include"); + for name in [ + "prepare_stdexec.py", + "stdexec-pool.patch", + "stdexec-pool-base.json", + ] { + println!( + "cargo:rerun-if-changed={}", + engine.join("cmake").join(name).display() + ); + } let rpc = repository.join("lib").join("crowdb-rpc"); let protocol_cpp_include = repository .join("lib") @@ -185,6 +207,14 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { }); } + // Cross-language runtime fixtures are excluded from every ordinary build. + if std::env::var_os("CARGO_FEATURE_TEST_UTIL").is_some() { + let tests = engine.join("tests"); + files.push(tests.join("common/test_chunk_runtime_c_api.cpp")); + build.include(&tests); + println!("cargo:rerun-if-changed={}", tests.join("common").display()); + } + for f in &files { build.file(f); println!("cargo:rerun-if-changed={}", f.display()); @@ -279,7 +309,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { println!("cargo:rustc-link-arg=-Wl,-rpath,{dir}"); } println!("cargo:rerun-if-changed={}", include.display()); - println!("cargo:rerun-if-changed={}", stdexec_include.display()); + println!("cargo:rerun-if-changed={}", stdexec_source.display()); println!("cargo:rerun-if-env-changed=CROWDB_TREE_LZ4_LIB"); println!("cargo:rerun-if-env-changed=CONDA_PREFIX"); Ok(()) diff --git a/lib/crowdb-tree/ffi/src/admission.rs b/lib/crowdb-tree/ffi/src/admission.rs new file mode 100644 index 000000000..1cc1df154 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/admission.rs @@ -0,0 +1,174 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::ptr::NonNull; +use std::sync::{Arc, Weak}; + +use crate::error::check; +use crate::reactor::{drive_ct_future, FutureGuard, FutureKind}; +use crate::{sys, BatchOp, Crowdbtree, CtError, ExtOp}; + +/// An owned, charged copy of a complete batch. Preparation holds no source +/// writer, epoch guard, or Tree close ticket, and may be cancelled by dropping it. +/// Arm immediately before consensus may choose a slot for this proposal. +pub struct PreparedApply { + raw: NonNull<sys::ct_apply_permit>, + tree: Weak<Crowdbtree>, +} + +// SAFETY: this opaque handle is exclusively owned. Native prepare/arm/submit +// permit movement between threads but require serialized access to one handle. +unsafe impl Send for PreparedApply {} + +/// A generation-pinned proposal. Dropping it before submit cancels admission; +/// after submit, the accepted operation drains even if its future is dropped. +pub struct ArmedApply { + // Release the native permit before dropping the last possible Tree owner. + permit: PreparedApply, + tree: Arc<Crowdbtree>, +} + +impl Crowdbtree { + /// Copy and admit a complete local proposal before choosing its slot. + /// Resource pressure is reported before consensus accepts the proposal. + pub fn prepare_proposal(self: &Arc<Self>, ops: &[BatchOp<'_>]) -> Result<PreparedApply, CtError> { + self.prepare_in_partition(ops, false) + } + + /// Copy a follower or replay batch using its independent progress partition. + pub fn prepare_replay(self: &Arc<Self>, ops: &[BatchOp<'_>]) -> Result<PreparedApply, CtError> { + self.prepare_in_partition(ops, true) + } + + fn prepare_in_partition( + self: &Arc<Self>, + ops: &[BatchOp<'_>], + gap_repair: bool, + ) -> Result<PreparedApply, CtError> { + let refs: Vec<sys::ct_kv_ref> = ops + .iter() + .map(|op| match op { + BatchOp::Put { key, value } => sys::ct_kv_ref { + key: key.as_ptr(), + key_len: key.len(), + value: value.as_ptr(), + value_len: value.len(), + kind: 0, + }, + BatchOp::Delete { key } => sys::ct_kv_ref { + key: key.as_ptr(), + key_len: key.len(), + value: std::ptr::null(), + value_len: 0, + kind: 1, + }, + }) + .collect(); + let mut raw = std::ptr::null_mut(); + check(unsafe { + sys::ct_apply_prepare( + self.as_ptr(), + refs.as_ptr(), + refs.len(), + i32::from(gap_repair), + &mut raw, + ) + })?; + Ok(PreparedApply { + raw: NonNull::new(raw).ok_or(CtError::Internal)?, + tree: Arc::downgrade(self), + }) + } +} + +impl PreparedApply { + /// Pin this prepared generation before a proposal can become chosen. + /// Generation replacement or Tree closure makes an old permit unavailable. + pub fn arm(self) -> Result<ArmedApply, CtError> { + let tree = self.tree.upgrade().ok_or(CtError::Unavailable)?; + check(unsafe { sys::ct_apply_arm(self.raw.as_ptr()) })?; + Ok(ArmedApply { permit: self, tree }) + } +} + +impl Drop for PreparedApply { + fn drop(&mut self) { + unsafe { sys::ct_apply_permit_free(self.raw.as_ptr()) }; + } +} + +impl ArmedApply { + /// Consume this admission for exactly one chosen slot. CPU and budget + /// continuations run in the Tree runtime; this future only awaits completion. + pub async fn apply(self, slot: u64) -> Result<(), CtError> { + let guard = { + let mut raw = std::ptr::null_mut(); + check(unsafe { sys::ct_apply_submit(self.permit.raw.as_ptr(), slot, &mut raw) })?; + FutureGuard(raw) + }; + drive_ct_future(guard, &self.tree, FutureKind::Flush).await?; + Ok(()) + } +} + +// Native retains this bridge while reading the immutable caller input. Its Box +// allocation is charged separately from the controlled native batch copy. +struct ReplayOwner { + ops: Vec<ExtOp>, +} + +unsafe extern "C" fn read_replay(context: *mut std::ffi::c_void, index: usize, out: *mut sys::ct_kv_ref) { + // SAFETY: native retains this box, serializes reads, and uses index < count. + let owner = unsafe { &*context.cast::<ReplayOwner>() }; + let value = match &owner.ops[index] { + ExtOp::Put { key, value } => sys::ct_kv_ref { + key: key.as_ptr(), + key_len: key.len(), + value: value.as_ptr(), + value_len: value.len(), + kind: 0, + }, + ExtOp::Delete { key } => sys::ct_kv_ref { + key: key.as_ptr(), + key_len: key.len(), + value: std::ptr::null(), + value_len: 0, + kind: 1, + }, + }; + // SAFETY: native supplies an initialized writable ct_kv_ref for this call. + unsafe { out.write(value) }; +} + +unsafe extern "C" fn release_replay(context: *mut std::ffi::c_void) { + // SAFETY: ct_apply_replay consumes one Box and invokes this callback once. + drop(unsafe { Box::from_raw(context.cast::<ReplayOwner>()) }); +} + +impl Crowdbtree { + /// Apply one already-chosen follower or WAL batch. High slots retain the + /// immutable input while the missing contiguous slot owns the gap reserve. + /// Native quota waits and staged writeback continue if this future drops. + pub async fn apply_replay(self: &Arc<Self>, slot: u64, operations: Vec<ExtOp>) -> Result<(), CtError> { + let guard = { + let count = operations.len(); + let context = Box::into_raw(Box::new(ReplayOwner { ops: operations })).cast(); + let mut raw = std::ptr::null_mut(); + check(unsafe { + sys::ct_apply_replay( + self.as_ptr(), + slot, + count, + read_replay, + release_replay, + context, + std::mem::size_of::<ReplayOwner>(), + &mut raw, + ) + })?; + let raw = NonNull::new(raw).ok_or(CtError::Internal)?; + FutureGuard(raw.as_ptr()) + }; + drive_ct_future(guard, self, FutureKind::Flush).await?; + Ok(()) + } +} diff --git a/lib/crowdb-tree/ffi/src/async_tree.rs b/lib/crowdb-tree/ffi/src/async_tree.rs index ffe18ccb5..58cebe0ea 100644 --- a/lib/crowdb-tree/ffi/src/async_tree.rs +++ b/lib/crowdb-tree/ffi/src/async_tree.rs @@ -9,16 +9,23 @@ use crate::error::CtError; use crate::reactor::{ drive_ct_future, try_poll_ct_future, try_poll_ct_future_pinned, FutureGuard, FutureKind, PinnedValue, }; -use crate::scan::{decode_scan, ScanDirection, ScanEntry}; +use crate::scan::{ScanDirection, ScanEntry, ScanOptions}; use crate::sys; use crate::tree::Crowdbtree; use crate::Config; -/// Async facade. `get`/`flush`/`snapshot`/`scan` drive the engine's io_uring -/// reactor directly via [`drive_ct_future`] -- no thread pool hop. The -/// remaining methods have no async C API twin yet and are called via the -/// synchronous `Crowdbtree` handle (`handle()`). Cheap to clone (shares -/// one `Arc<Crowdbtree>`). +mod compaction; +pub use compaction::{CompactionOptions, CompactionReport}; +mod chunk_collection; +mod maintenance; +mod owned_get; +pub use chunk_collection::ChunkCollectionReport; +pub use owned_get::OwnedGetOutcome; +mod scan; + +/// Async facade over native operation futures. Runtime Trees execute CPU work +/// on their shared native pool; [`drive_ct_future`] polls completion and the +/// backend reactor without blocking the Rust executor. Clones share one Tree. #[derive(Clone, Debug)] pub struct AsyncCrowdbtree { inner: Arc<Crowdbtree>, @@ -31,6 +38,12 @@ impl AsyncCrowdbtree { }) } + /// Share an existing Tree without opening another writer or storage domain. + #[must_use] + pub fn from_handle(inner: Arc<Crowdbtree>) -> Self { + Self { inner } + } + pub fn from_sync(tree: Crowdbtree) -> Self { Self { inner: Arc::new(tree), @@ -48,40 +61,26 @@ impl AsyncCrowdbtree { /// Await the native flush completion. The native worker waits for the /// captured MemTable writers without blocking this executor thread. pub async fn flush(&self) -> Result<(), CtError> { - let fut = unsafe { sys::ct_flush_async(self.inner.as_ptr()) }; - drive_ct_future(FutureGuard(fut), &self.inner, FutureKind::Flush).await?; - Ok(()) + self.flush_report().await?.status } - /// Drives the engine's io_uring reactor directly (Phase - /// 3) -- no `spawn_blocking`; the write phase always waits on the - /// reactor, unlike `flush`/the fast `get` path. + /// Awaits the captured checkpoint, including its backend completion. pub async fn snapshot(&self) -> Result<u64, CtError> { - let fut = unsafe { sys::ct_snapshot_async(self.inner.as_ptr()) }; - Ok( - drive_ct_future(FutureGuard(fut), &self.inner, FutureKind::Snapshot) - .await? - .slot, - ) + let report = self.checkpoint_report().await?; + report.status?; + Ok(report.durable_coverage) } - /// Drives the engine's io_uring reactor directly (Phase - /// 3) -- no `spawn_blocking`. Resolves on the very first poll for a - /// resident hit (`get_view`'s existing fast path, `#5 B3`); only a - /// genuine demand-load miss waits on the reactor. + /// Awaits a native point read dispatched through the shared CPU pool. pub async fn get(&self, key: Vec<u8>) -> Result<Option<(u64, Vec<u8>)>, CtError> { let fut = unsafe { sys::ct_get_async(self.inner.as_ptr(), key.as_ptr(), key.len()) }; let out = drive_ct_future(FutureGuard(fut), &self.inner, FutureKind::Get).await?; Ok(out.found.then_some((out.slot, out.value))) } - /// Like [`Self::get`], but never allocates or boxes anything for the - /// fast (resident hit/miss) path -- only the genuine demand-load-miss - /// path (`GetOutcome::Pending`) does. Lets a caller with its own - /// fast-path/slow-path return type (`crowdb_kv`'s `KVFuture`, /// #11 Phase 6) mirror `ct_get_async`'s own C++-layer split one layer - /// up, instead of collapsing it back into a uniform `async fn` (which - /// would force boxing on every call, fast path included -- exactly what - /// `KVFuture::Ready` exists to avoid). + /// Returns a result immediately if the first native poll completes; + /// otherwise boxes a pending future. Native operation and result storage + /// still allocate; runtime operations charge their admitted storage. pub fn try_get(&self, key: &[u8]) -> GetOutcome { let fut = unsafe { sys::ct_get_async(self.inner.as_ptr(), key.as_ptr(), key.len()) }; let mut guard = FutureGuard(fut); @@ -113,13 +112,11 @@ impl AsyncCrowdbtree { })) } - /// Async twin of [`Crowdbtree::scan`] (follow-up). Drives - /// the reactor directly like `get`/`flush`/`snapshot` -- resolves on the - /// first poll whenever every leaf in range is already resident - /// (`scan`'s own fast path), only waiting on the reactor for a - /// genuine cold leaf (or the initial root->leaf descent). See - /// `Crowdbtree::scan_async`'s doc comment (crowdb-tree.h) for why a miss - /// retries the whole scan rather than resuming a cursor. + /// Async twin of [`Crowdbtree::scan`]. Runtime scans retain one captured + /// publication across internal windows and keep the native allocation + /// owner in every returned `Bytes` slice. Resume truncated output from + /// its last key, even when the requested limit is zero. Use + /// [`Self::scan_with_options`] to resume from an empty last key. #[allow(clippy::too_many_arguments)] pub async fn scan( &self, @@ -158,33 +155,22 @@ impl AsyncCrowdbtree { deadline_ms: u64, direction: ScanDirection, ) -> Result<(Vec<ScanEntry>, bool), CtError> { - let fut = unsafe { - sys::ct_scan_directional_async( - self.inner.as_ptr(), - prefix.as_ptr(), - prefix.len(), - start_after.as_ptr(), - start_after.len(), - end_key.as_ptr(), - end_key.len(), - limit, - byte_budget, - if keys_only { 1 } else { 0 }, - deadline_ms, - i32::from(direction == ScanDirection::Reverse), - ) - }; - let out = drive_ct_future(FutureGuard(fut), &self.inner, FutureKind::Scan).await?; - let entries = decode_scan(out.value, out.slot as usize)?; - Ok((entries, out.found)) + self.scan_with_options(ScanOptions { + prefix, + start: (!start_after.is_empty()).then_some(start_after), + end: end_key, + limit, + byte_budget, + keys_only, + deadline_ms, + direction, + ..Default::default() + }) + .await } - /// Like [`Self::try_get`], but for [`Self::scan`]: never allocates or - /// boxes anything for the fast (all-leaves-resident) path -- only a - /// genuine cold-leaf miss (`ScanOutcome::Pending`) does. Same - /// motivation as `try_get`'s doc comment: lets a caller with its own - /// fast-path/slow-path return type mirror `ct_scan_async`'s own - /// C++-layer split one layer up instead of forcing a box on every call. + /// Like [`Self::try_get`], returns ready scan output on the first native + /// completion poll and boxes a future only when that poll is pending. #[allow(clippy::too_many_arguments)] pub fn try_scan( &self, @@ -208,7 +194,7 @@ impl AsyncCrowdbtree { ) } - /// Nonallocating-fast-path directional scan. + /// Directional scan with an immediate first completion poll. #[allow(clippy::too_many_arguments)] pub fn try_scan_directional( &self, @@ -221,35 +207,17 @@ impl AsyncCrowdbtree { deadline_ms: u64, direction: ScanDirection, ) -> ScanOutcome { - let fut = unsafe { - sys::ct_scan_directional_async( - self.inner.as_ptr(), - prefix.as_ptr(), - prefix.len(), - start_after.as_ptr(), - start_after.len(), - end_key.as_ptr(), - end_key.len(), - limit, - byte_budget, - if keys_only { 1 } else { 0 }, - deadline_ms, - i32::from(direction == ScanDirection::Reverse), - ) - }; - let mut guard = FutureGuard(fut); - if let Some(result) = try_poll_ct_future(&mut guard, FutureKind::Scan) { - return ScanOutcome::Ready(result.and_then(|out| { - let entries = decode_scan(out.value, out.slot as usize)?; - Ok((entries, out.found)) - })); - } - let tree = self.inner.clone(); - ScanOutcome::Pending(Box::pin(async move { - let out = drive_ct_future(guard, &tree, FutureKind::Scan).await?; - let entries = decode_scan(out.value, out.slot as usize)?; - Ok((entries, out.found)) - })) + self.try_scan_with_options(ScanOptions { + prefix, + start: (!start_after.is_empty()).then_some(start_after), + end: end_key, + limit, + byte_budget, + keys_only, + deadline_ms, + direction, + ..Default::default() + }) } } @@ -257,11 +225,9 @@ impl AsyncCrowdbtree { /// exists instead of a single uniform `async fn`. #[allow(clippy::type_complexity)] pub enum GetOutcome { - /// Resolved on the very first (and only) poll attempt -- no allocation. + /// Resolved on the first poll without boxing a Rust future. Ready(Result<Option<(u64, Vec<u8>)>, CtError>), - /// A genuine demand-load miss, already registered with the reactor - /// (or, absent one, that will complete synchronously on the next poll - /// regardless -- `.await` this to completion. + /// Native CPU or I/O work is pending; await its completion. Pending(Pin<Box<dyn Future<Output = Result<Option<(u64, Vec<u8>)>, CtError>> + Send>>), } @@ -269,12 +235,9 @@ pub enum GetOutcome { /// this exists instead of a single uniform `async fn`. #[allow(clippy::type_complexity)] pub enum ScanOutcome { - /// Resolved on the very first (and only) poll attempt -- no allocation - /// beyond the returned entries themselves. + /// Resolved on the first poll without boxing a Rust future. Ready(Result<(Vec<ScanEntry>, bool), CtError>), - /// A genuine cold-leaf miss, already registered with the reactor (or, - /// absent one, that will complete synchronously on the next poll - /// regardless -- `.await` this to completion. + /// Native CPU or I/O work is pending; await its completion. Pending(Pin<Box<dyn Future<Output = Result<(Vec<ScanEntry>, bool), CtError>> + Send>>), } diff --git a/lib/crowdb-tree/ffi/src/async_tree/chunk_collection.rs b/lib/crowdb-tree/ffi/src/async_tree/chunk_collection.rs new file mode 100644 index 000000000..5940bbbd5 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/async_tree/chunk_collection.rs @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::AsyncCrowdbtree; +use crate::{ + error::check, + reactor::{drive_completion, FutureGuard}, + sys, CtError, +}; + +/// Progress for one bounded catalog and physical-chunk maintenance pass. +/// Root and WAL retention watermarks are unchanged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChunkCollectionReport { + pub status: Result<(), CtError>, + pub more: bool, + pub candidates_examined: u64, + pub reference_windows: u64, + pub stages_examined: u64, + pub pins_examined: u64, + pub pins_released: u64, + pub stages_removed: u64, + /// Requested logical pack capacity removed after acknowledged physical deletion. + /// Excludes physical strip rounding and mirror replication. + pub logical_capacity_bytes: u64, +} + +impl AsyncCrowdbtree { + /// Continue one finite sweep on this Tree's paid backend lane. Busy or + /// rejected admission is outer `Err`; accepted progress includes its status. + /// # Errors + /// Returns admission, unsupported-backend, or malformed native result errors. + pub async fn collect_chunk_storage(&self) -> Result<ChunkCollectionReport, CtError> { + let future = { + let mut raw = std::ptr::null_mut(); + check(unsafe { sys::ct_chunk_collect_async(self.inner.as_ptr(), &mut raw) })?; + FutureGuard(raw) + }; + drive_completion(future, &self.inner, poll_collection).await + } +} + +fn poll_collection(guard: &mut FutureGuard) -> Option<Result<ChunkCollectionReport, CtError>> { + let mut raw = sys::ct_chunk_collection_result_v1 { + struct_size: std::mem::size_of::<sys::ct_chunk_collection_result_v1>() as u32, + version: 1, + ..Default::default() + }; + let mut done = 0; + if let Err(error) = check(unsafe { sys::ct_future_poll_chunk_collection(guard.0, &mut done, &mut raw) }) { + return Some(Err(error)); + } + if done == 0 { + return None; + } + guard.0 = std::ptr::null_mut(); + Some(Ok(ChunkCollectionReport { + status: check(raw.terminal_status), + more: raw.more != 0, + candidates_examined: raw.candidates_examined, + reference_windows: raw.reference_windows, + stages_examined: raw.stages_examined, + pins_examined: raw.pins_examined, + pins_released: raw.pins_released, + stages_removed: raw.stages_removed, + logical_capacity_bytes: raw.logical_capacity_bytes, + })) +} diff --git a/lib/crowdb-tree/ffi/src/async_tree/compaction.rs b/lib/crowdb-tree/ffi/src/async_tree/compaction.rs new file mode 100644 index 000000000..34466039b --- /dev/null +++ b/lib/crowdb-tree/ffi/src/async_tree/compaction.rs @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::AsyncCrowdbtree; +use crate::error::{check, CtError}; +use crate::reactor::{drive_completion, FutureGuard}; +use crate::sys; + +/// Bounded source-block selection for one explicit compaction pass. +#[derive(Debug, Clone, Copy)] +pub struct CompactionOptions { + /// Select only blocks whose free fraction strictly exceeds this value. + pub free_threshold: f64, + /// Zero disables relocation. The first eligible block may exceed a + /// nonzero budget; each pass selects at most one block. + pub max_relocation_bytes: u64, +} +impl Default for CompactionOptions { + fn default() -> Self { + Self { + free_threshold: 0.70, + max_relocation_bytes: 1, + } + } +} +/// Progress survives a terminal failure after partial block relocation. +#[derive(Debug, Clone)] +pub struct CompactionReport { + pub status: Result<(), CtError>, + pub blocks_selected: u64, + pub pages_relocated: u64, + pub bytes_relocated: u64, + pub blocks_deleted: u64, +} +impl AsyncCrowdbtree { + /// Await admitted native CPU/backend continuations without blocking the host executor. + /// # Errors + /// Returns an admission or transport error before a progress report is available. + pub async fn compact_sparse_blocks(&self) -> Result<CompactionReport, CtError> { + self.compact_sparse_blocks_with(CompactionOptions::default()) + .await + } + /// Await one explicitly configured compaction pass. + /// # Errors + /// Returns an admission or transport error before a progress report is available. + pub async fn compact_sparse_blocks_with( + &self, + options: CompactionOptions, + ) -> Result<CompactionReport, CtError> { + let future = { + let options = sys::ct_compaction_options_v1 { + struct_size: std::mem::size_of::<sys::ct_compaction_options_v1>() as u32, + version: 1, + free_threshold: options.free_threshold, + max_relocation_bytes: options.max_relocation_bytes, + }; + let mut raw = std::ptr::null_mut(); + check(unsafe { sys::ct_compact_sparse_blocks_async(self.inner.as_ptr(), &options, &mut raw) })?; + FutureGuard(raw) + }; + drive_completion(future, &self.inner, poll).await + } +} +fn poll(guard: &mut FutureGuard) -> Option<Result<CompactionReport, CtError>> { + let mut raw = sys::ct_compaction_result_v1 { + struct_size: std::mem::size_of::<sys::ct_compaction_result_v1>() as u32, + version: 1, + ..Default::default() + }; + let mut done = 0; + if let Err(error) = check(unsafe { sys::ct_future_poll_compaction(guard.0, &mut done, &mut raw) }) { + return Some(Err(error)); + } + if done == 0 { + return None; + } + guard.0 = std::ptr::null_mut(); + Some(Ok(CompactionReport { + status: check(raw.terminal_status), + blocks_selected: raw.blocks_selected, + pages_relocated: raw.pages_relocated, + bytes_relocated: raw.bytes_relocated, + blocks_deleted: raw.blocks_deleted, + })) +} diff --git a/lib/crowdb-tree/ffi/src/async_tree/maintenance.rs b/lib/crowdb-tree/ffi/src/async_tree/maintenance.rs new file mode 100644 index 000000000..48906acee --- /dev/null +++ b/lib/crowdb-tree/ffi/src/async_tree/maintenance.rs @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::AsyncCrowdbtree; +use crate::error::{check, CtError}; +use crate::maintenance::{CheckpointReport, FlushReport}; +use crate::reactor::{drive_completion, FutureGuard}; +use crate::sys; + +impl AsyncCrowdbtree { + /// Await both flush phases. Admission errors are outer `Err`; accepted + /// operation errors remain in the returned report alongside progress. + pub async fn flush_report(&self) -> Result<FlushReport, CtError> { + let future = { + let mut raw = std::ptr::null_mut(); + check(unsafe { sys::ct_flush_report_async(self.inner.as_ptr(), &mut raw) })?; + FutureGuard(raw) + }; + drive_completion(future, &self.inner, poll_flush).await + } + + /// Materialize reloadable local pages without committing a durable anchor. + pub async fn writeback_report(&self) -> Result<CheckpointReport, CtError> { + self.checkpoint_with(sys::ct_writeback_async).await + } + + /// Await a durable checkpoint and retain its progress even on failure. + pub async fn checkpoint_report(&self) -> Result<CheckpointReport, CtError> { + self.checkpoint_with(sys::ct_checkpoint_report_async).await + } + + async fn checkpoint_with( + &self, + start: unsafe extern "C" fn(*mut sys::ct_tree, *mut *mut sys::ct_future) -> i32, + ) -> Result<CheckpointReport, CtError> { + let future = { + let mut raw = std::ptr::null_mut(); + check(unsafe { start(self.inner.as_ptr(), &mut raw) })?; + FutureGuard(raw) + }; + drive_completion(future, &self.inner, poll_checkpoint).await + } +} + +fn poll_flush(guard: &mut FutureGuard) -> Option<Result<FlushReport, CtError>> { + let mut raw = sys::ct_flush_result_v1 { + struct_size: std::mem::size_of::<sys::ct_flush_result_v1>() as u32, + version: 1, + ..Default::default() + }; + let mut done = 0; + if let Err(error) = check(unsafe { sys::ct_future_poll_flush_report(guard.0, &mut done, &mut raw) }) { + return Some(Err(error)); + } + if done == 0 { + return None; + } + guard.0 = std::ptr::null_mut(); + Some(Ok(raw.into())) +} + +fn poll_checkpoint(guard: &mut FutureGuard) -> Option<Result<CheckpointReport, CtError>> { + let mut raw = sys::ct_checkpoint_result_v1 { + struct_size: std::mem::size_of::<sys::ct_checkpoint_result_v1>() as u32, + version: 1, + ..Default::default() + }; + let mut done = 0; + if let Err(error) = check(unsafe { sys::ct_future_poll_checkpoint_report(guard.0, &mut done, &mut raw) }) + { + return Some(Err(error)); + } + if done == 0 { + return None; + } + guard.0 = std::ptr::null_mut(); + Some(Ok(raw.into())) +} diff --git a/lib/crowdb-tree/ffi/src/async_tree/owned_get.rs b/lib/crowdb-tree/ffi/src/async_tree/owned_get.rs new file mode 100644 index 000000000..fe450a42a --- /dev/null +++ b/lib/crowdb-tree/ffi/src/async_tree/owned_get.rs @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::future::Future; +use std::pin::Pin; + +use super::AsyncCrowdbtree; +use crate::error::CtError; +use crate::reactor::{drive_completion, try_poll_ct_future_pinned, FutureGuard, PinnedValue}; +use crate::sys; +use bytes::Bytes; + +type OwnedValue = Option<(u64, Bytes)>; + +/// Both immediate and pending reads retain the immutable native allocation. +pub enum OwnedGetOutcome { + Ready(Result<OwnedValue, CtError>), + Pending(Pin<Box<dyn Future<Output = Result<OwnedValue, CtError>> + Send>>), +} + +fn convert(value: Option<(u64, PinnedValue)>) -> Result<OwnedValue, CtError> { + value + .map(|(slot, owner)| owner.try_into_bytes().map(|bytes| (slot, bytes))) + .transpose() +} + +impl AsyncCrowdbtree { + /// Read without copying a pending native result into unaccounted storage. + /// The native future and the eventual Bytes control retain one paid owner. + pub fn try_get_owned(&self, key: &[u8]) -> OwnedGetOutcome { + let future = unsafe { sys::ct_get_async(self.inner.as_ptr(), key.as_ptr(), key.len()) }; + let mut guard = FutureGuard(future); + if let Some(result) = try_poll_ct_future_pinned(&mut guard) { + return OwnedGetOutcome::Ready(result.and_then(convert)); + } + let tree = self.inner.clone(); + OwnedGetOutcome::Pending(Box::pin(async move { + let value = drive_completion(guard, &tree, try_poll_ct_future_pinned).await?; + convert(value) + })) + } +} diff --git a/lib/crowdb-tree/ffi/src/async_tree/scan.rs b/lib/crowdb-tree/ffi/src/async_tree/scan.rs new file mode 100644 index 000000000..cba292997 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/async_tree/scan.rs @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::{AsyncCrowdbtree, ScanOutcome}; +use crate::error::CtError; +use crate::reactor::{drive_ct_future, try_poll_ct_future, FutureKind, RawOutcome}; +use crate::scan::{decode_scan_bytes, owner, ScanEntry, ScanOptions}; + +fn decode(out: RawOutcome) -> Result<(Vec<ScanEntry>, bool), CtError> { + let packed = out.packed.ok_or(CtError::Internal)?; + let count = usize::try_from(out.slot).map_err(|_| CtError::Corruption)?; + Ok((decode_scan_bytes(packed, count)?, out.found)) +} + +impl AsyncCrowdbtree { + /// Scans with explicit cursor, result limit and byte-budget semantics. + /// Internal work windows do not reduce the requested result limit. The + /// native allocation stays charged until the last returned `Bytes` owner drops. + pub async fn scan_with_options(&self, options: ScanOptions) -> Result<(Vec<ScanEntry>, bool), CtError> { + let guard = owner::start(self.inner.as_ptr(), &options)?; + decode(drive_ct_future(guard, &self.inner, FutureKind::ScanOwned).await?) + } + + /// Starts the same explicit-bound scan and polls its completion once. + pub fn try_scan_with_options(&self, options: ScanOptions) -> ScanOutcome { + let mut guard = match owner::start(self.inner.as_ptr(), &options) { + Ok(guard) => guard, + Err(error) => return ScanOutcome::Ready(Err(error)), + }; + if let Some(result) = try_poll_ct_future(&mut guard, FutureKind::ScanOwned) { + return ScanOutcome::Ready(result.and_then(decode)); + } + let tree = self.inner.clone(); + ScanOutcome::Pending(Box::pin(async move { + decode(drive_ct_future(guard, &tree, FutureKind::ScanOwned).await?) + })) + } +} diff --git a/lib/crowdb-tree/ffi/src/backend_release.rs b/lib/crowdb-tree/ffi/src/backend_release.rs new file mode 100644 index 000000000..b7bcb45b5 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/backend_release.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +//! Explicit host ownership for real backend close work after the last reader. +use crate::{error::check, sys, CtError}; +use std::ffi::c_void; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use tokio::sync::Notify; + +struct ServiceState { + native: NonNull<sys::ct_backend_release_service>, + wake: Arc<Notify>, + done: AtomicBool, + completion: Notify, +} +// Native final owners are lock-free producers. The host pump is serialized by +// the native queue and owns this handle through actual descriptor disposal. +unsafe impl Send for ServiceState {} +unsafe impl Sync for ServiceState {} +impl Drop for ServiceState { + fn drop(&mut self) { + unsafe { sys::ct_backend_release_service_free(self.native.as_ptr()) }; + } +} + +/// An independent host handle, also retained by the bounded background pump. +/// Keep the Tokio host alive until `shutdown` has completed. Dropping a Tree or +/// this handle never closes admission while retained snapshots can still read. +#[derive(Clone)] +pub struct BackendReleaseService { + state: Arc<ServiceState>, +} +impl BackendReleaseService { + pub(crate) fn attach(tree: NonNull<sys::ct_tree>) -> Result<Option<Self>, CtError> { + let Ok(executor) = tokio::runtime::Handle::try_current() else { + return Ok(None); + }; + let wake = Arc::new(Notify::new()); + let context = Arc::into_raw(wake.clone()).cast_mut().cast::<c_void>(); + let mut native = std::ptr::null_mut(); + let status = unsafe { + sys::ct_tree_backend_release_service(tree.as_ptr(), wake_queue, drop_wake, context, &mut native) + }; + if let Err(error) = check(status) { + unsafe { drop_wake(context) }; + return if matches!(error, CtError::NotFound) { + Ok(None) + } else { + Err(error) + }; + } + let state = Arc::new(ServiceState { + native: NonNull::new(native).ok_or(CtError::Internal)?, + wake, + done: AtomicBool::new(false), + completion: Notify::new(), + }); + executor.spawn(pump(state.clone())); + Ok(Some(Self { state })) + } + /// Call only after closing host ingress to all users of the backend. This + /// waits for retained readers and actual close jobs, without retaining a + /// Tree or its native CPU runtime. The native backend also stops admission + /// automatically when its final owner disappears. + pub async fn shutdown(&self) { + unsafe { sys::ct_backend_release_service_stop(self.state.native.as_ptr()) }; + loop { + let done = self.state.completion.notified(); + tokio::pin!(done); + done.as_mut().enable(); + if self.state.done.load(Ordering::Acquire) { + return; + } + done.await; + } + } +} +async fn pump(state: Arc<ServiceState>) { + loop { + let wake = state.wake.notified(); + tokio::pin!(wake); + wake.as_mut().enable(); + let owner = state.clone(); + let step = tokio::task::spawn_blocking(move || unsafe { + sys::ct_backend_release_service_pump(owner.native.as_ptr(), 8) + }) + .await; + let Ok(step) = step else { + return; + }; + if step.drained != 0 { + state.done.store(true, Ordering::Release); + state.completion.notify_waiters(); + return; + } + if step.more == 0 { + wake.await; + } + } +} +unsafe extern "C" fn wake_queue(context: *mut c_void) { + unsafe { &*context.cast::<Notify>() }.notify_one(); +} +unsafe extern "C" fn drop_wake(context: *mut c_void) { + unsafe { drop(Arc::from_raw(context.cast::<Notify>())) }; +} diff --git a/lib/crowdb-tree/ffi/src/batch.rs b/lib/crowdb-tree/ffi/src/batch.rs index 11beb0881..4baf512cb 100644 --- a/lib/crowdb-tree/ffi/src/batch.rs +++ b/lib/crowdb-tree/ffi/src/batch.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +mod allocation; +pub use allocation::{OwnedAllocation, OwnedOp}; + use bytes::Bytes; use crate::error::check; @@ -15,35 +18,9 @@ pub enum BatchOp<'a> { Delete { key: &'a [u8] }, } -/// Opaque Rust handle keeping a [`Bytes`] alive while crowdb-tree borrows its -/// bytes via a C++ `kExternal` buffer (R30). Created by -/// [`Crowdbtree::apply_batch_external`]; freed by the C++ `drop_fn` callback -/// ([`ct_release_bytes`]) when the borrowed buffer is destroyed (MemTable -/// drain/overwrite). The `Bytes`' refcount keeps the underlying allocation -/// (typically a Paxos payload) alive until every borrowing buffer is freed. -// The field is never read — its sole purpose is to be dropped (decrementing -// the `Bytes` Arc refcount) when the `Box<BytesRef>` is reclaimed by -// `ct_release_bytes`. Keeping the `Bytes` alive is the side effect we want. -#[allow(dead_code)] -struct BytesRef(Bytes); - -/// Drop callback handed to C++ for each `ct_ext_op` value (R30). Reclaims the -/// boxed [`BytesRef`] — dropping the [`Bytes`] inside, which decrements the -/// underlying `Arc` refcount. Called exactly once per handle by crowdb-tree when -/// the corresponding `kExternal` buffer is freed. -extern "C" fn ct_release_bytes(owner: *mut std::ffi::c_void) { - if !owner.is_null() { - // SAFETY: `owner` was created by `Box::into_raw(Box::new(BytesRef(..)))` - // in `apply_batch_external` and is handed back to us exactly once. - unsafe { drop(Box::from_raw(owner as *mut BytesRef)) }; - } -} - -/// One zero-copy op for [`Crowdbtree::apply_batch_external`] (R30). The value -/// [`Bytes`] is kept alive by a ref handle handed to C++; crowdb-tree borrows the -/// value bytes until MemTable drain, then calls back to drop the ref. The key -/// is copied into a C++ `std::string` during the call (small; SBO), so it can -/// be a cheap [`Bytes`] clone. +/// One operation for [`Crowdbtree::apply_batch_external`]. The key and value +/// are copied during the call. A `Bytes` slice has unknown backing capacity; +/// its length is never treated as the retained allocation's capacity. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ExtOp { Put { key: Bytes, value: Bytes }, @@ -77,60 +54,33 @@ impl Crowdbtree { check(unsafe { sys::ct_apply_batch_slices(self.as_ptr(), slot, refs.as_ptr(), refs.len() as u64) }) } - /// Zero-copy apply (R30): like [`Self::apply_batch`] but the value bytes - /// are borrowed from Rust-owned [`Bytes`] memory instead of copied at the - /// FFI boundary. Each Put op's `value` [`Bytes`] is kept alive by a ref - /// handle handed to C++; crowdb-tree borrows the value bytes via a C++ - /// `kExternal` buffer and calls [`ct_release_bytes`] when the buffer is - /// freed (MemTable drain/overwrite). The value `memcpy` is deferred to - /// flush (off the apply critical path). Keys are copied into C++ - /// `std::string`s during the call (small; SBO), same as [`Self::apply_batch`]. + /// Atomically apply operations consuming their `Bytes` owners. Backing + /// capacity and identity are unknown for arbitrary `Bytes`, so values are + /// copied into owned cells during this call. This prevents a small slice + /// from retaining an unaccounted large allocation until a later flush. /// - /// Ownership of every Put's value handle transfers to C++; on any error - /// return, C++ frees the handles via `drop_fn` (the `external_op` vector - /// destructors run on the error path). + /// Rust retains the inputs until the synchronous copy returns and then + /// drops every consumed `Bytes`, including on error. pub fn apply_batch_external(&self, slot: u64, ops: Vec<ExtOp>) -> Result<(), CtError> { - let mut ffi_ops: Vec<sys::ct_ext_op> = Vec::with_capacity(ops.len()); - // Hold key Bytes alive for the duration of the FFI call (C++ copies - // them into std::string during the call; after that the pointers are - // not needed). - let mut keys: Vec<Bytes> = Vec::with_capacity(ops.len()); - for op in ops { - match op { - ExtOp::Put { key, value } => { - let val_ptr = value.as_ptr(); - let val_len = value.len(); - // Move the value Bytes into a heap handle; C++ owns it now. - let handle = Box::into_raw(Box::new(BytesRef(value))) as *mut std::ffi::c_void; - keys.push(key); - let k = keys.last().expect("just pushed"); - ffi_ops.push(sys::ct_ext_op { - key: k.as_ptr(), - key_len: k.len(), - value: val_ptr, - value_len: val_len, - kind: 0, - bytes_ref: handle, - drop_fn: Some(ct_release_bytes), - }); - } - ExtOp::Delete { key } => { - keys.push(key); - let k = keys.last().expect("just pushed"); - ffi_ops.push(sys::ct_ext_op { - key: k.as_ptr(), - key_len: k.len(), - value: std::ptr::null(), - value_len: 0, - kind: 1, - bytes_ref: std::ptr::null_mut(), - drop_fn: None, - }); - } - } - } - check(unsafe { - sys::ct_apply_batch_external(self.as_ptr(), slot, ffi_ops.as_ptr(), ffi_ops.len() as u64) - }) + let refs: Vec<sys::ct_kv_ref> = ops + .iter() + .map(|op| match op { + ExtOp::Put { key, value } => sys::ct_kv_ref { + key: key.as_ptr(), + key_len: key.len(), + value: value.as_ptr(), + value_len: value.len(), + kind: 0, + }, + ExtOp::Delete { key } => sys::ct_kv_ref { + key: key.as_ptr(), + key_len: key.len(), + value: std::ptr::null(), + value_len: 0, + kind: 1, + }, + }) + .collect(); + check(unsafe { sys::ct_apply_batch_slices(self.as_ptr(), slot, refs.as_ptr(), refs.len() as u64) }) } } diff --git a/lib/crowdb-tree/ffi/src/batch/allocation.rs b/lib/crowdb-tree/ffi/src/batch/allocation.rs new file mode 100644 index 000000000..690d94964 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/batch/allocation.rs @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crate::{error::check, sys, Crowdbtree, CtError}; +use std::ops::Range; +use std::ptr::NonNull; + +/// Immutable slice of an allocation admitted at its complete `Vec` capacity. +/// Clones and slices share one payload charge, including spare capacity. The +/// native account survives the Tree without retaining its CPU executor. +#[derive(Debug)] +pub struct OwnedAllocation { + owner: NonNull<sys::ct_owned_allocation>, + data: NonNull<u8>, + offset: usize, + len: usize, + capacity: usize, +} + +impl OwnedAllocation { + /// Complete backing allocation capacity, not this slice's length. + pub fn capacity(&self) -> usize { + self.capacity + } + pub fn len(&self) -> usize { + self.len + } + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Return an initialized subrange without allocating another control block. + pub fn slice(&self, range: Range<usize>) -> Result<Self, CtError> { + if range.start > range.end || range.end > self.len { + return Err(CtError::InvalidArgument); + } + let mut result = self.clone(); + result.offset += range.start; + result.len = range.end - range.start; + Ok(result) + } + + /// Tests identity even when the visible byte ranges differ. + pub fn shares_allocation(&self, other: &Self) -> bool { + self.owner == other.owner + } +} + +impl Clone for OwnedAllocation { + fn clone(&self) -> Self { + unsafe { sys::ct_owned_allocation_retain(self.owner.as_ptr()) }; + Self { + owner: self.owner, + data: self.data, + offset: self.offset, + len: self.len, + capacity: self.capacity, + } + } +} +impl Drop for OwnedAllocation { + fn drop(&mut self) { + unsafe { sys::ct_owned_allocation_free(self.owner.as_ptr()) }; + } +} +impl AsRef<[u8]> for OwnedAllocation { + fn as_ref(&self) -> &[u8] { + // The complete immutable Vec and its initialized prefix remain alive + // under this handle. Every slice is checked against that prefix. + unsafe { std::slice::from_raw_parts(self.data.as_ptr().add(self.offset), self.len) } + } +} +// Native reference counting is atomic, and the foreign Vec is never mutated. +unsafe impl Send for OwnedAllocation {} +unsafe impl Sync for OwnedAllocation {} + +/// Borrowed keys and retained complete-allocation values for one atomic batch. +#[derive(Debug)] +pub enum OwnedOp<'a> { + Put { + key: &'a [u8], + value: &'a OwnedAllocation, + }, + Delete { + key: &'a [u8], + }, +} + +unsafe extern "C" fn drop_vec(context: *mut std::ffi::c_void) { + drop(unsafe { Box::from_raw(context.cast::<Vec<u8>>()) }); +} + +impl Crowdbtree { + /// Admit a complete `Vec`, charging its real capacity exactly once. Every + /// failure consumes and drops the input. + pub fn own_vec(&self, value: Vec<u8>) -> Result<OwnedAllocation, CtError> { + let len = value.len(); + let capacity = value.capacity(); + let data = NonNull::new(value.as_ptr().cast_mut()).ok_or(CtError::Internal)?; + let context = Box::into_raw(Box::new(value)).cast(); + let mut out = std::ptr::null_mut(); + // C consumes context on every return and also charges Box<Vec>'s size. + check(unsafe { + sys::ct_owned_allocation_create( + self.as_ptr(), + data.as_ptr(), + len, + capacity, + drop_vec, + context, + std::mem::size_of::<Vec<u8>>(), + &mut out, + ) + })?; + Ok(OwnedAllocation { + owner: NonNull::new(out).ok_or(CtError::Internal)?, + data, + offset: 0, + len, + capacity, + }) + } + + /// Borrow allocation owners for an atomic write. Values within one host + /// retain their existing allocation; a different host performs a budgeted + /// copy. The caller may retain, reuse, or cancel its owners independently. + pub fn apply_batch_owned(&self, slot: u64, operations: &[OwnedOp<'_>]) -> Result<(), CtError> { + check(unsafe { + sys::ct_apply_batch_owned_reader( + self.as_ptr(), + slot, + operations.len(), + read_operation, + std::ptr::from_ref(&operations).cast(), + ) + }) + } +} + +unsafe extern "C" fn read_operation(context: *const std::ffi::c_void, index: usize) -> sys::ct_owned_kv_ref { + // The call is synchronous. Native code only visits indexes below the + // supplied count, and this borrowed slice stays alive for the whole call. + let operations = unsafe { &*context.cast::<&[OwnedOp<'_>]>() }; + match &operations[index] { + OwnedOp::Put { key, value } => sys::ct_owned_kv_ref { + key: key.as_ptr(), + key_len: key.len(), + owner: value.owner.as_ptr(), + offset: value.offset, + length: value.len, + kind: 0, + }, + OwnedOp::Delete { key } => sys::ct_owned_kv_ref { + key: key.as_ptr(), + key_len: key.len(), + owner: std::ptr::null_mut(), + offset: 0, + length: 0, + kind: 1, + }, + } +} diff --git a/lib/crowdb-tree/ffi/src/chunk.rs b/lib/crowdb-tree/ffi/src/chunk.rs index 264269102..a5ea71e3a 100644 --- a/lib/crowdb-tree/ffi/src/chunk.rs +++ b/lib/crowdb-tree/ffi/src/chunk.rs @@ -8,6 +8,16 @@ use std::sync::Arc; use crate::error::{check, CtError}; use crate::{sys, PageStore}; +pub mod legacy; +pub mod pin_service; +mod runtime; +pub use runtime::ChunkRuntimeOptions; +pub mod protection; +mod protection_callbacks; +pub mod stage; +mod stage_callbacks; +use stage::{StageId, StageRecord}; + #[derive(Debug, Clone, Copy)] pub struct ChunkPageStoreOptions { pub tree_id: u64, @@ -73,6 +83,46 @@ pub enum RootCatalogObject { /// maintenance paths. Implementations may bridge to an asynchronous service /// on a dedicated worker; tree point reads never call this interface. pub trait RootCatalogStore: Send + Sync + 'static { + /// Immutable original manifest retained before replacing a legacy root. + fn prepare_legacy_migration( + &self, + _tree_id: u64, + _generation: u64, + _manifest: &[u8], + ) -> Result<(), CtError> { + Err(CtError::NotSupported) + } + fn load_legacy_migration(&self, _tree_id: u64, _generation: u64) -> Result<Option<Vec<u8>>, CtError> { + Err(CtError::NotSupported) + } + /// Durable reference intents share the same authority as pack deletion. + fn stage_protection(&self) -> Option<&dyn protection::StageProtectionCatalog> { + None + } + /// Schema support alone does not enable independent staged writeback. + fn supports_staging(&self) -> bool { + false + } + fn load_stage(&self, _tree_id: u64, _id: StageId) -> Result<Option<StageRecord>, CtError> { + Err(CtError::NotSupported) + } + fn compare_stage( + &self, + _tree_id: u64, + _owner_epoch: u64, + _expected: Option<&StageRecord>, + _next: Option<&StageRecord>, + ) -> Result<(), CtError> { + Err(CtError::NotSupported) + } + fn list_stages( + &self, + _tree_id: u64, + _after: Option<StageId>, + _limit: u32, + ) -> Result<Vec<StageRecord>, CtError> { + Err(CtError::NotSupported) + } fn load(&self, tree_id: u64, object: RootCatalogObject) -> Result<Option<Vec<u8>>, CtError>; fn store(&self, tree_id: u64, object: RootCatalogObject, data: &[u8]) -> Result<(), CtError>; fn publish( @@ -110,6 +160,11 @@ pub trait RootCatalogStore: Send + Sync + 'static { } impl ChunkRootCatalog { + /// Persistent intent/pin schema; the complete backend must also support fenced IO and drain. + #[must_use] + pub fn supports_protection_schema(&self) -> bool { + unsafe { sys::ct_root_catalog_protection_capabilities(self.ptr.as_ptr()) & 1 != 0 } + } pub fn open_memory(owner_epoch: u64) -> Result<Self, CtError> { let mut out = std::ptr::null_mut(); check(unsafe { sys::ct_memory_root_catalog_open(owner_epoch, &mut out) })?; @@ -124,6 +179,8 @@ impl ChunkRootCatalog { /// /// Returns an invalid callback configuration error. pub fn open_callback(store: Arc<dyn RootCatalogStore>) -> Result<Self, CtError> { + let supports_staging = store.supports_staging(); + let supports_protection = store.stage_protection().is_some(); let context = Box::into_raw(Box::new(store)).cast::<c_void>(); let callbacks = sys::ct_root_catalog_callbacks { load: Some(catalog_load), @@ -138,9 +195,27 @@ impl ChunkRootCatalog { drop_context: Some(catalog_drop_context), }; let mut out = std::ptr::null_mut(); - if let Err(error) = - check(unsafe { sys::ct_callback_root_catalog_open(&callbacks, context, &mut out) }) - { + let status = unsafe { + if supports_staging && supports_protection { + sys::ct_callback_root_catalog_open_v3( + &callbacks, + &stage_callbacks::callbacks(), + &protection_callbacks::callbacks(), + context, + &mut out, + ) + } else if supports_staging { + sys::ct_callback_root_catalog_open_v2( + &callbacks, + &stage_callbacks::callbacks(), + context, + &mut out, + ) + } else { + sys::ct_callback_root_catalog_open(&callbacks, context, &mut out) + } + }; + if let Err(error) = check(status) { unsafe { drop(Box::from_raw(context.cast::<Arc<dyn RootCatalogStore>>())) }; return Err(error); } @@ -221,11 +296,13 @@ unsafe extern "C" fn catalog_load( if context.is_null() || out.is_null() || len.is_null() { return -2; } - let object = match catalog_object(kind, object_id) { - Ok(object) => object, - Err(error) => return error_code(error), + let result = if kind == 4 { + catalog_store_ref(context).load_legacy_migration(tree_id, object_id) + } else { + catalog_object(kind, object_id) + .and_then(|object| catalog_store_ref(context).load(tree_id, object)) }; - match catalog_store_ref(context).load(tree_id, object) { + match result { Ok(Some(bytes)) if !bytes.is_empty() => { let bytes = bytes.into_boxed_slice(); *len = bytes.len(); @@ -260,18 +337,18 @@ unsafe extern "C" fn catalog_store( if context.is_null() || (data.is_null() && len != 0) { return -2; } - let object = match catalog_object(kind, object_id) { - Ok(object) => object, - Err(error) => return error_code(error), - }; let bytes = if len == 0 { &[] } else { std::slice::from_raw_parts(data, len) }; - catalog_store_ref(context) - .store(tree_id, object, bytes) - .map_or_else(error_code, |()| 0) + let result = if kind == 4 { + catalog_store_ref(context).prepare_legacy_migration(tree_id, object_id, bytes) + } else { + catalog_object(kind, object_id) + .and_then(|object| catalog_store_ref(context).store(tree_id, object, bytes)) + }; + result.map_or_else(error_code, |()| 0) })) .unwrap_or(-6) } @@ -420,6 +497,9 @@ pub use transport::{ChunkRpcDiskRoute, ChunkRpcRoute, ChunkRpcTransportOptions, pub use transport::{ChunkRpcRouteResolver, OwnedChunkRpcDiskRoute, OwnedChunkRpcTransportOptions}; impl PageStore { + /// Opens the legacy mutable-address storage adapter for explicit migration. + /// Ordinary Tree open rejects this adapter. New writers use + /// `PageStore::open_chunk_runtime`; old roots use `Crowdbtree::migrate_legacy_chunk`. pub fn open_chunk( options: ChunkPageStoreOptions, catalog: Arc<ChunkRootCatalog>, @@ -455,13 +535,16 @@ impl PageStore { }) } - /// Estimates retained pack bytes from cached layout metadata without I/O. + /// Legacy mutable-address layout estimate. Runtime-backed stores return + /// `NotSupported`; use `Crowdbtree::backing_stats` for actual physical bytes. pub fn chunk_estimated_bytes(&self) -> Result<u64, CtError> { let mut bytes = 0; check(unsafe { sys::ct_chunk_page_store_estimated_bytes(self.ptr.as_ptr(), &mut bytes) })?; Ok(bytes) } + /// Legacy mutable-address counters. Runtime-backed stores return + /// `NotSupported`; use Tree work and backing snapshots for current metrics. pub fn chunk_stats(&self) -> Result<ChunkPageStoreStats, CtError> { let mut raw = sys::ct_chunk_page_store_stats::default(); check(unsafe { sys::ct_chunk_page_store_get_stats(self.ptr.as_ptr(), &mut raw) })?; @@ -569,3 +652,6 @@ impl PageStore { }) } } + +#[cfg(feature = "test-util")] +pub mod test_util; diff --git a/lib/crowdb-tree/ffi/src/chunk/legacy.rs b/lib/crowdb-tree/ffi/src/chunk/legacy.rs new file mode 100644 index 000000000..b11722dbe --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/legacy.rs @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Read-only validation of the historical logical-address manifest. +use crate::{crc32c, crc32c_update, CtError}; + +const HEADER: usize = 80; +const SEGMENT: usize = 33; +const PACK: usize = 53; +const REFERENCES_PER_SEGMENT: usize = 256; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LegacyPack { + pub owner_tree_id: u64, + pub chunk_high: u64, + pub chunk_low: u64, + pub offset: u64, + pub length: u32, +} + +#[derive(Clone, Copy, Debug)] +pub struct LegacyManifest<'a> { + bytes: &'a [u8], + packs_offset: usize, +} + +impl<'a> LegacyManifest<'a> { + /// Validate the complete immutable manifest, including its CRC and geometry. + /// # Errors + /// Returns corruption for any truncated, unsupported or inconsistent descriptor. + pub fn decode(bytes: &'a [u8]) -> Result<Self, CtError> { + if bytes.len() < HEADER + 8 || bytes.len() > 16 * 1024 * 1024 || u32_at(bytes, 0) != 7 { + return Err(CtError::Corruption); + } + let segments = usize::try_from(u64_at(bytes, 72)).map_err(|_| CtError::Corruption)?; + if segments > (bytes.len() - HEADER - 8) / SEGMENT { + return Err(CtError::Corruption); + } + let packs_offset = HEADER + segments * SEGMENT + 8; + let packs = usize::try_from(u64_at(bytes, packs_offset - 8)).map_err(|_| CtError::Corruption)?; + if packs != (bytes.len() - packs_offset) / PACK + || !(bytes.len() - packs_offset).is_multiple_of(PACK) + || segments != packs.div_ceil(REFERENCES_PER_SEGMENT) + { + return Err(CtError::Corruption); + } + let view = Self { bytes, packs_offset }; + if view.tree_id() == 0 || view.owner_epoch() == 0 || matches!(view.generation(), 0 | u64::MAX) { + return Err(CtError::Corruption); + } + let crc = crc32c_update(crc32c(&bytes[..68]), &bytes[packs_offset..]); + if crc32c_update(crc, &bytes[HEADER..packs_offset - 8]) != u32_at(bytes, 68) { + return Err(CtError::Corruption); + } + let mut end = 0_u64; + let mut reused = 0_u64; + let mut reused_bytes = 0_u64; + for (index, pack) in view.packs().enumerate() { + let offset = packs_offset + index * PACK; + let logical = u64_at(bytes, offset + 16); + let length = u64::from(pack.length); + if pack.owner_tree_id == 0 + || u64_at(bytes, offset + 8) != index as u64 + || !matches!(pack.chunk_high >> 56, 2 | 3) + || length == 0 + || length > 256 * 1024 * 1024 + || logical < end + || logical > u64_at(bytes, 28) + || length > u64_at(bytes, 28) - logical + || pack.offset.checked_add(length).is_none() + || bytes[offset + 52] > 1 + { + return Err(CtError::Corruption); + } + end = logical + length; + if bytes[offset + 52] != 0 { + reused += 1; + reused_bytes += length; + } + } + if reused != u64_at(bytes, 44) || reused_bytes != u64_at(bytes, 52) { + return Err(CtError::Corruption); + } + for index in 0..segments { + let offset = HEADER + index * SEGMENT; + let first = index * REFERENCES_PER_SEGMENT; + let count = (packs - first).min(REFERENCES_PER_SEGMENT); + if u64_at(bytes, offset) == 0 + || u64_at(bytes, offset + 8) == 0 + || u64_at(bytes, offset + 16) != first as u64 + || u64::from(u32_at(bytes, offset + 24)) != count as u64 + || bytes[offset + 32] > 1 + { + return Err(CtError::Corruption); + } + } + Ok(view) + } + #[must_use] + pub fn tree_id(self) -> u64 { + u64_at(self.bytes, 4) + } + #[must_use] + pub fn generation(self) -> u64 { + u64_at(self.bytes, 12) + } + #[must_use] + pub fn owner_epoch(self) -> u64 { + u64_at(self.bytes, 20) + } + #[must_use] + pub fn packs(self) -> impl ExactSizeIterator<Item = LegacyPack> + 'a { + self.bytes[self.packs_offset..] + .as_chunks::<PACK>() + .0 + .iter() + .map(|bytes| LegacyPack { + owner_tree_id: u64_at(bytes, 0), + chunk_high: u64_at(bytes, 24), + chunk_low: u64_at(bytes, 32), + offset: u64_at(bytes, 40), + length: u32_at(bytes, 48), + }) + } +} + +fn u64_at(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes( + bytes[offset..offset + 8] + .try_into() + .expect("validated manifest field"), + ) +} +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes( + bytes[offset..offset + 4] + .try_into() + .expect("validated manifest field"), + ) +} diff --git a/lib/crowdb-tree/ffi/src/chunk/pin_service.rs b/lib/crowdb-tree/ffi/src/chunk/pin_service.rs new file mode 100644 index 000000000..b0d84e240 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/pin_service.rs @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Host-owned final pin release. No Tree or native CPU executor is retained. +use super::ChunkRootCatalog; +use crate::{error::check, sys, CtError, TreeRuntime}; +use std::ffi::c_void; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use tokio::sync::Notify; + +struct ServiceState { + native: NonNull<sys::ct_chunk_pin_service>, + wake: Arc<Notify>, + done: AtomicBool, + completion: Notify, +} +// The native queue has atomic producers and one serialized backend consumer. +// The task owns this handle through the last metadata callback and queue tail. +unsafe impl Send for ServiceState {} +unsafe impl Sync for ServiceState {} +impl Drop for ServiceState { + fn drop(&mut self) { + unsafe { sys::ct_chunk_pin_service_free(self.native.as_ptr()) }; + } +} + +/// Keep this service at the host boundary, independently of every Tree. +/// Tree close leaves retained snapshots readable. Host shutdown stops new pins, +/// then waits for those snapshots to release their pins and real unpin replies. +/// The surrounding Tokio runtime must stay alive until `shutdown` completes. +pub struct ChunkPinReleaseService { + state: Arc<ServiceState>, +} +impl ChunkPinReleaseService { + /// Reserve the native queue in the host ledger and start one bounded pump + /// on the existing Tokio blocking pool; this creates no private thread. + /// # Errors + /// Rejects missing Tokio context, zero capacity, or insufficient host credit. + pub fn new(runtime: &TreeRuntime, capacity: usize) -> Result<Self, CtError> { + let executor = tokio::runtime::Handle::try_current().map_err(|_| CtError::InvalidArgument)?; + let wake = Arc::new(Notify::new()); + let context = Arc::into_raw(wake.clone()).cast_mut().cast::<c_void>(); + let mut native = std::ptr::null_mut(); + let status = unsafe { + sys::ct_chunk_pin_service_create( + runtime.as_ptr(), + capacity, + wake_queue, + drop_wake, + context, + &mut native, + ) + }; + if let Err(error) = check(status) { + unsafe { drop_wake(context) }; + return Err(error); + } + let state = Arc::new(ServiceState { + native: NonNull::new(native).ok_or(CtError::Internal)?, + wake, + done: AtomicBool::new(false), + completion: Notify::new(), + }); + executor.spawn(pump(state.clone())); + Ok(Self { state }) + } + /// Bind only during catalog initialization, before opening its first Tree. + /// # Errors + /// Rejects catalogs without staged protection or a different prior service. + pub fn bind(&self, catalog: &ChunkRootCatalog) -> Result<(), CtError> { + check(unsafe { + sys::ct_root_catalog_bind_pin_service(catalog.ptr.as_ptr(), self.state.native.as_ptr()) + }) + } + /// Sample registered slots and pending releases without pumping the queue. + /// `denied_host` counts capacity rejections; `waiters` counts pending releases. + /// # Errors + /// Returns a native argument error if the service handle is invalid. + pub fn resource_usage(&self) -> Result<crate::runtime::ResourceUsage, CtError> { + let mut raw = sys::ct_resource_usage_v1::default(); + check(unsafe { sys::ct_chunk_pin_service_resource_usage(self.state.native.as_ptr(), &mut raw) })?; + Ok(raw.into()) + } + /// Stop admission, preserve existing readers, and join real unpin work. + pub async fn shutdown(&self) { + unsafe { sys::ct_chunk_pin_service_stop(self.state.native.as_ptr()) }; + loop { + let done = self.state.completion.notified(); + tokio::pin!(done); + done.as_mut().enable(); + if self.state.done.load(Ordering::Acquire) { + return; + } + done.await; + } + } +} +impl Drop for ChunkPinReleaseService { + fn drop(&mut self) { + // The existing host task owns accepted late releases. Explicit shutdown + // is required before dropping the surrounding Tokio runtime. + unsafe { sys::ct_chunk_pin_service_stop(self.state.native.as_ptr()) }; + } +} +async fn pump(state: Arc<ServiceState>) { + loop { + let wake = state.wake.notified(); + tokio::pin!(wake); + wake.as_mut().enable(); + let owner = state.clone(); + let step = tokio::task::spawn_blocking(move || unsafe { + sys::ct_chunk_pin_service_pump(owner.native.as_ptr(), 8) + }) + .await; + let Ok(step) = step else { + // No native step throws. A host runtime cancelling its blocking + // pool is an incomplete shutdown; never report pins as released. + return; + }; + if step.drained != 0 { + state.done.store(true, Ordering::Release); + state.completion.notify_waiters(); + return; + } + if step.failed != 0 { + // Retry the same durable pin identity after an unknown outcome. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } else if step.more == 0 { + wake.await; + } + } +} +unsafe extern "C" fn wake_queue(context: *mut c_void) { + unsafe { &*context.cast::<Notify>() }.notify_one(); +} +unsafe extern "C" fn drop_wake(context: *mut c_void) { + unsafe { drop(Arc::from_raw(context.cast::<Notify>())) }; +} diff --git a/lib/crowdb-tree/ffi/src/chunk/protection.rs b/lib/crowdb-tree/ffi/src/chunk/protection.rs new file mode 100644 index 000000000..19b0f0f59 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/protection.rs @@ -0,0 +1,143 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Bounded durable reference preparation before a root becomes visible. + +use super::stage::StageId; +use crate::{crc32c_update, CtError}; + +// Final release also writes the candidate, window, manifest and authority. +pub const MAX_REFERENCE_WINDOW: usize = 60; +pub const MAX_CANDIDATE_REFERENCES: u64 = 16 * 1024 * 1024; +pub const MAX_CANDIDATE_MANIFEST_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestCandidate { + pub id: StageId, + pub expected_generation: u64, + pub reference_count: u64, + pub reference_checksum: u32, + /// Immutable root descriptor; reference windows are persisted separately. + pub manifest: Vec<u8>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u64)] +pub enum CandidateState { + Preparing = 1, + Ready = 2, + Committed = 3, + Aborted = 4, + Reclaimed = 5, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CandidateStatus { + pub id: StageId, + pub state: CandidateState, + pub generation: u64, + pub reference_count: u64, + pub prepared_references: u64, + pub prepared_windows: u64, + pub released_windows: u64, +} + +pub fn validate_operation(id: StageId) -> Result<(), CtError> { + if id.owner_epoch == 0 || (id.operation_high == 0 && id.operation_low == 0) { + return Err(CtError::InvalidArgument); + } + Ok(()) +} + +/// Update in the same sorted order used by prepare_manifest_references. +#[must_use] +pub fn reference_checksum(mut checksum: u32, references: &[StageId]) -> u32 { + for id in references { + for value in [id.owner_epoch, id.operation_high, id.operation_low] { + checksum = crc32c_update(checksum, &value.to_le_bytes()); + } + } + checksum +} + +impl ManifestCandidate { + pub fn validate(&self) -> Result<(), CtError> { + validate_operation(self.id)?; + if self.expected_generation == u64::MAX + || self.reference_count > MAX_CANDIDATE_REFERENCES + || self.manifest.is_empty() + || self.manifest.len() > MAX_CANDIDATE_MANIFEST_BYTES + || (self.reference_count == 0 && self.reference_checksum != 0) + { + return Err(CtError::InvalidArgument); + } + Ok(()) + } +} + +/// Implementations serialize these operations with root publication and GC. +/// No method may infer an unknown outcome from a timeout or create a new ID. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PinPruneProgress { + pub examined: u32, + pub released: u32, + pub after: Option<StageId>, + pub more: bool, +} + +pub trait StageProtectionCatalog: Send + Sync { + /// Local fixed-process lease check before submission and after each read window. + /// # Errors + /// Returns unavailable for an expired owner or unsupported for legacy catalogs. + fn check_read_owner(&self) -> Result<(), CtError> { + Err(CtError::NotSupported) + } + /// Called after closing native write admission and draining every accepted backend operation. + /// A lost reply retries this same writer identity; it never releases live reader pins. + /// # Errors + /// Returns an authority, metadata availability, or unsupported error. + fn writer_quiesced(&self, _tree_id: u64, _owner_epoch: u64) -> Result<(), CtError> { + Err(CtError::NotSupported) + } + /// Resolves an earlier owner's uncommitted candidate after actual root and WAL recovery. + /// # Errors + /// Returns an authority, incomplete recovery, or unsupported error. + fn recover_manifest_candidate(&self, _tree_id: u64, _id: StageId) -> Result<(), CtError> { + Err(CtError::NotSupported) + } + /// Remove only pins whose process exclusion has been authoritatively confirmed. + /// # Errors + /// Rejects incomplete recovery, unavailable metadata, or unsupported catalogs. + fn prune_stage_pins( + &self, + _tree_id: u64, + _stage: StageId, + _after: Option<StageId>, + _limit: u32, + ) -> Result<PinPruneProgress, CtError> { + Err(CtError::NotSupported) + } + + fn load_manifest_candidate(&self, tree_id: u64, id: StageId) -> Result<Option<CandidateStatus>, CtError>; + fn list_manifest_candidates( + &self, + tree_id: u64, + after: Option<StageId>, + limit: u32, + ) -> Result<Vec<CandidateStatus>, CtError>; + fn begin_manifest_candidate(&self, tree_id: u64, candidate: &ManifestCandidate) -> Result<(), CtError>; + fn prepare_manifest_references( + &self, + tree_id: u64, + id: StageId, + window: u64, + references: &[StageId], + ) -> Result<(), CtError>; + fn commit_manifest_candidate(&self, tree_id: u64, id: StageId) -> Result<(), CtError>; + fn abort_manifest_candidate(&self, tree_id: u64, id: StageId) -> Result<(), CtError>; + /// Release one previously prepared window after abort or root reclamation. + fn release_manifest_references(&self, tree_id: u64, id: StageId, window: u64) -> Result<(), CtError>; + fn pin_stage(&self, tree_id: u64, stage: StageId, pin: StageId) -> Result<(), CtError>; + /// A fenced owner may release its own pin; it may never acquire a new one. + fn unpin_stage(&self, tree_id: u64, stage: StageId, pin: StageId) -> Result<(), CtError>; +} diff --git a/lib/crowdb-tree/ffi/src/chunk/protection_callbacks.rs b/lib/crowdb-tree/ffi/src/chunk/protection_callbacks.rs new file mode 100644 index 000000000..e986d4951 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/protection_callbacks.rs @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ + catalog_store_ref, error_code, + protection::{CandidateStatus, ManifestCandidate, StageProtectionCatalog, MAX_CANDIDATE_MANIFEST_BYTES}, + stage::{StageId, MAX_STAGE_SCAN}, + sys, CtError, +}; +use std::ffi::c_void; +mod mutation; + +fn internal(id: sys::ct_chunk_stage_id) -> StageId { + StageId { + owner_epoch: id.owner_epoch, + operation_high: id.operation_high, + operation_low: id.operation_low, + } +} +fn external(id: StageId) -> sys::ct_chunk_stage_id { + sys::ct_chunk_stage_id { + owner_epoch: id.owner_epoch, + operation_high: id.operation_high, + operation_low: id.operation_low, + } +} +fn status(value: CandidateStatus) -> sys::ct_chunk_candidate_status { + sys::ct_chunk_candidate_status { + id: external(value.id), + generation: value.generation, + reference_count: value.reference_count, + prepared_references: value.prepared_references, + prepared_windows: value.prepared_windows, + released_windows: value.released_windows, + state: value.state as u32, + } +} +unsafe fn with_catalog( + context: *mut c_void, + operation: impl FnOnce(&dyn StageProtectionCatalog) -> Result<(), CtError>, +) -> i32 { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if context.is_null() { + return Err(CtError::InvalidArgument); + } + let catalog = catalog_store_ref(context) + .stage_protection() + .ok_or(CtError::NotSupported)?; + operation(catalog) + })) + .unwrap_or(Err(CtError::Internal)) + .map_or_else(error_code, |()| 0) +} +pub(super) fn callbacks() -> sys::ct_root_catalog_protection_callbacks { + sys::ct_root_catalog_protection_callbacks { + version: 1, + struct_bytes: std::mem::size_of::<sys::ct_root_catalog_protection_callbacks>() + .try_into() + .expect("callback layout fits u32"), + capabilities: 1, + check_reader: Some(check_reader), + writer_quiesced: Some(writer_quiesced), + begin: Some(begin), + load: Some(load), + list: Some(list), + prepare: Some(mutation::prepare), + finish: Some(mutation::finish), + release: Some(mutation::release), + pin: Some(mutation::pin), + prune: Some(mutation::prune), + } +} +unsafe extern "C" fn check_reader(context: *mut c_void) -> i32 { + with_catalog(context, |catalog| catalog.check_read_owner()) +} +unsafe extern "C" fn writer_quiesced(context: *mut c_void, tree: u64, epoch: u64) -> i32 { + with_catalog(context, |catalog| catalog.writer_quiesced(tree, epoch)) +} +unsafe extern "C" fn begin( + context: *mut c_void, + tree: u64, + id: sys::ct_chunk_stage_id, + expected_generation: u64, + reference_count: u64, + reference_checksum: u32, + manifest: *const u8, + length: usize, +) -> i32 { + with_catalog(context, |catalog| { + if manifest.is_null() || length == 0 || length > MAX_CANDIDATE_MANIFEST_BYTES { + return Err(CtError::InvalidArgument); + } + let candidate = ManifestCandidate { + id: internal(id), + expected_generation, + reference_count, + reference_checksum, + manifest: std::slice::from_raw_parts(manifest, length).to_vec(), + }; + candidate.validate()?; + catalog.begin_manifest_candidate(tree, &candidate) + }) +} +unsafe extern "C" fn load( + context: *mut c_void, + tree: u64, + id: sys::ct_chunk_stage_id, + out: *mut sys::ct_chunk_candidate_status, +) -> i32 { + with_catalog(context, |catalog| { + if out.is_null() { + return Err(CtError::InvalidArgument); + } + *out = status( + catalog + .load_manifest_candidate(tree, internal(id))? + .ok_or(CtError::NotFound)?, + ); + Ok(()) + }) +} +unsafe extern "C" fn list( + context: *mut c_void, + tree: u64, + after: *const sys::ct_chunk_stage_id, + limit: u32, + out: *mut sys::ct_chunk_candidate_status, + length: *mut usize, +) -> i32 { + with_catalog(context, |catalog| { + if out.is_null() + || length.is_null() + || !(1..=MAX_STAGE_SCAN).contains(&limit) + || *length < limit as usize + { + return Err(CtError::InvalidArgument); + } + let records = catalog.list_manifest_candidates(tree, after.as_ref().copied().map(internal), limit)?; + if records.len() > limit as usize { + return Err(CtError::Corruption); + } + for (index, value) in records.iter().copied().enumerate() { + *out.add(index) = status(value); + } + *length = records.len(); + Ok(()) + }) +} diff --git a/lib/crowdb-tree/ffi/src/chunk/protection_callbacks/mutation.rs b/lib/crowdb-tree/ffi/src/chunk/protection_callbacks/mutation.rs new file mode 100644 index 000000000..c3989e91d --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/protection_callbacks/mutation.rs @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{external, internal, sys, with_catalog, CtError}; +use crate::chunk::protection::MAX_REFERENCE_WINDOW; +use crate::chunk::stage::{StageId, MAX_STAGE_SCAN}; +use std::ffi::c_void; + +pub(super) unsafe extern "C" fn prepare( + context: *mut c_void, + tree: u64, + id: sys::ct_chunk_stage_id, + window: u64, + references: *const sys::ct_chunk_stage_id, + length: usize, +) -> i32 { + with_catalog(context, |catalog| { + if references.is_null() || !(1..=MAX_REFERENCE_WINDOW).contains(&length) { + return Err(CtError::InvalidArgument); + } + let mut values = [StageId::default(); MAX_REFERENCE_WINDOW]; + for (slot, value) in values + .iter_mut() + .zip(std::slice::from_raw_parts(references, length)) + { + *slot = internal(*value); + } + catalog.prepare_manifest_references(tree, internal(id), window, &values[..length]) + }) +} +pub(super) unsafe extern "C" fn finish( + context: *mut c_void, + tree: u64, + id: sys::ct_chunk_stage_id, + action: u32, +) -> i32 { + with_catalog(context, |catalog| match action { + 1 => catalog.commit_manifest_candidate(tree, internal(id)), + 2 => catalog.abort_manifest_candidate(tree, internal(id)), + 3 => catalog.recover_manifest_candidate(tree, internal(id)), + _ => Err(CtError::InvalidArgument), + }) +} +pub(super) unsafe extern "C" fn release( + context: *mut c_void, + tree: u64, + id: sys::ct_chunk_stage_id, + window: u64, +) -> i32 { + with_catalog(context, |catalog| { + catalog.release_manifest_references(tree, internal(id), window) + }) +} +pub(super) unsafe extern "C" fn pin( + context: *mut c_void, + tree: u64, + stage: sys::ct_chunk_stage_id, + pin: sys::ct_chunk_stage_id, + acquire: u8, +) -> i32 { + with_catalog(context, |catalog| match acquire { + 0 => catalog.unpin_stage(tree, internal(stage), internal(pin)), + 1 => catalog.pin_stage(tree, internal(stage), internal(pin)), + _ => Err(CtError::InvalidArgument), + }) +} +pub(super) unsafe extern "C" fn prune( + context: *mut c_void, + tree: u64, + stage: sys::ct_chunk_stage_id, + after: *const sys::ct_chunk_stage_id, + limit: u32, + out: *mut sys::ct_chunk_pin_progress, +) -> i32 { + with_catalog(context, |catalog| { + if out.is_null() || !(1..=MAX_STAGE_SCAN).contains(&limit) { + return Err(CtError::InvalidArgument); + } + let result = catalog.prune_stage_pins( + tree, + internal(stage), + after.as_ref().copied().map(internal), + limit, + )?; + *out = sys::ct_chunk_pin_progress { + after: external(result.after.unwrap_or_default()), + examined: result.examined, + released: result.released, + has_after: u8::from(result.after.is_some()), + more: u8::from(result.more), + }; + Ok(()) + }) +} diff --git a/lib/crowdb-tree/ffi/src/chunk/runtime.rs b/lib/crowdb-tree/ffi/src/chunk/runtime.rs new file mode 100644 index 000000000..e49925ff3 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/runtime.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +//! Shared host runtime storage with exact immutable pack identities. + +use super::{ChunkRootCatalog, ChunkTransport}; +use crate::{error::check, sys, CtError, PageStore, TreeRuntime}; +use std::ptr::NonNull; +use std::sync::Arc; + +#[derive(Debug, Clone, Copy)] +pub struct ChunkRuntimeOptions { + pub tree_id: u64, + pub owner_epoch: u64, + /// Zero opens current; a nonzero value opens only that retained root. + pub open_generation: u64, + pub iu_size: u32, + pub mirror_copies: u32, + pub max_chunk_bytes: u64, +} + +impl Default for ChunkRuntimeOptions { + fn default() -> Self { + Self { + tree_id: 0, + owner_epoch: 0, + open_generation: 0, + iu_size: 64 * 1024, + mirror_copies: 2, + max_chunk_bytes: 256 * 1024 * 1024, + } + } +} + +impl PageStore { + /// Create a paid facade without opening storage or starting a private pool. + /// Inject it into a `Config` with the same shared `TreeRuntime`. The catalog + /// must already be bound to the host `ChunkPinReleaseService`. Each catalog + /// claim is one writer incarnation and cannot be reopened after close. + /// + /// # Errors + /// Rejects unsupported fixed-pack/protection capabilities, invalid geometry, + /// or insufficient host credit. Actual root validation happens at Tree open. + pub fn open_chunk_runtime( + runtime: &TreeRuntime, + options: ChunkRuntimeOptions, + catalog: Arc<ChunkRootCatalog>, + transport: &ChunkTransport, + ) -> Result<Self, CtError> { + let raw = sys::ct_chunk_runtime_options { + version: 1, + struct_bytes: u32::try_from(std::mem::size_of::<sys::ct_chunk_runtime_options>()) + .map_err(|_| CtError::InvalidArgument)?, + tree_id: options.tree_id, + owner_epoch: options.owner_epoch, + open_generation: options.open_generation, + iu_size: options.iu_size, + mirror_copies: options.mirror_copies, + max_chunk_bytes: options.max_chunk_bytes, + }; + let mut out = std::ptr::null_mut(); + check(unsafe { + sys::ct_chunk_runtime_page_store_open( + runtime.as_ptr(), + &raw, + catalog.ptr.as_ptr(), + transport.ptr.as_ptr(), + &mut out, + ) + })?; + Ok(Self { + ptr: NonNull::new(out).ok_or(CtError::Internal)?, + chunk_catalog: Some(catalog), + }) + } +} diff --git a/lib/crowdb-tree/ffi/src/chunk/stage.rs b/lib/crowdb-tree/ffi/src/chunk/stage.rs new file mode 100644 index 000000000..5697d7624 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/stage.rs @@ -0,0 +1,138 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Versioned protection of one independently allocated immutable pack window. + +use crate::CtError; + +mod codec; +pub(crate) use codec::{decode_stage_list, encode_stage_list}; +pub const MAX_STAGE_PACKS: usize = 1024; +pub const MAX_STAGE_SCAN: u32 = 64; +pub const MAX_STAGE_BYTES: usize = 100 + 16 * MAX_STAGE_PACKS; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct StageId { + pub owner_epoch: u64, + pub operation_high: u64, + pub operation_low: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] +pub enum StageState { + Reserved = 1, + Writing = 2, + Reloadable = 3, + Referenced = 4, + Retired = 5, + Deleting = 6, + Reclaimed = 7, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StagePack { + pub offset: u64, + pub length: u32, + pub checksum: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StageRecord { + pub tree_id: u64, + pub id: StageId, + pub revision: u64, + /// Fixed before allocation is submitted, including retries after lost replies. + pub chunk_high: u64, + pub chunk_low: u64, + pub capacity: u64, + /// Concrete Tree page/index purpose, matching the ChunkId prefix. + pub purpose: u32, + pub state: StageState, + pub manifest_generation: u64, + pub packs: Vec<StagePack>, +} + +impl StageRecord { + pub fn validate(&self) -> Result<(), CtError> { + if self.tree_id == 0 + || self.id.owner_epoch == 0 + || (self.id.operation_high == 0 && self.id.operation_low == 0) + || self.revision == 0 + || self.capacity == 0 + || self.capacity > u64::from(u32::MAX) + || !matches!(self.purpose, 2 | 3) + || self.chunk_high >> 56 != u64::from(self.purpose) + || self.packs.len() > MAX_STAGE_PACKS + || (matches!(self.state, StageState::Reloadable | StageState::Referenced) + && self.packs.is_empty()) + || (self.state == StageState::Referenced && self.manifest_generation == 0) + || (matches!( + self.state, + StageState::Reserved | StageState::Writing | StageState::Reloadable + ) && self.manifest_generation != 0) + { + return Err(CtError::InvalidArgument); + } + let mut end = 0; + for pack in &self.packs { + if pack.length == 0 || pack.length > 256 * 1024 * 1024 || pack.offset < end { + return Err(CtError::InvalidArgument); + } + end = pack + .offset + .checked_add(u64::from(pack.length)) + .ok_or(CtError::InvalidArgument)?; + if end > self.capacity { + return Err(CtError::InvalidArgument); + } + } + Ok(()) + } +} + +pub fn validate_stage_transition( + expected: Option<&StageRecord>, + next: Option<&StageRecord>, +) -> Result<(), CtError> { + if let Some(record) = expected { + record.validate()?; + } + if let Some(record) = next { + record.validate()?; + } + let (Some(old), Some(new)) = (expected, next) else { + return match (expected, next) { + (None, Some(record)) if record.revision == 1 && record.state == StageState::Reserved => Ok(()), + (Some(record), None) if record.state == StageState::Reclaimed => Ok(()), + _ => Err(CtError::InvalidArgument), + }; + }; + if old.tree_id != new.tree_id + || old.id != new.id + || old.chunk_high != new.chunk_high + || old.chunk_low != new.chunk_low + || old.capacity != new.capacity + || old.purpose != new.purpose + || old.revision.checked_add(1) != Some(new.revision) + || !new.packs.starts_with(&old.packs) + || (new.packs != old.packs && !matches!(old.state, StageState::Reserved | StageState::Writing)) + || new.manifest_generation < old.manifest_generation + { + return Err(CtError::InvalidArgument); + } + use StageState::*; + if matches!( + (old.state, new.state), + (Reserved, Writing | Retired) + | (Writing, Writing | Reloadable | Retired) + | (Reloadable, Referenced | Retired) + | (Referenced, Referenced | Retired) + | (Retired, Deleting) + | (Deleting, Reclaimed) + ) { + Ok(()) + } else { + Err(CtError::InvalidArgument) + } +} diff --git a/lib/crowdb-tree/ffi/src/chunk/stage/codec.rs b/lib/crowdb-tree/ffi/src/chunk/stage/codec.rs new file mode 100644 index 000000000..033a09a4e --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/stage/codec.rs @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::{StageId, StagePack, StageRecord, StageState, MAX_STAGE_BYTES, MAX_STAGE_PACKS}; +use crate::{crc32c, CtError}; + +const MAGIC: u32 = 0x4753_5443; + +impl StageRecord { + pub fn encode(&self) -> Result<Vec<u8>, CtError> { + self.validate()?; + let mut out = Vec::with_capacity(100 + 16 * self.packs.len()); + out.extend_from_slice(&MAGIC.to_le_bytes()); + out.extend_from_slice(&1_u32.to_le_bytes()); + for value in [ + self.tree_id, + self.id.owner_epoch, + self.id.operation_high, + self.id.operation_low, + self.revision, + self.chunk_high, + self.chunk_low, + self.capacity, + self.manifest_generation, + ] { + out.extend_from_slice(&value.to_le_bytes()); + } + for value in [self.purpose, self.state as u32, self.packs.len() as u32, 0] { + out.extend_from_slice(&value.to_le_bytes()); + } + for pack in &self.packs { + out.extend_from_slice(&pack.offset.to_le_bytes()); + out.extend_from_slice(&pack.length.to_le_bytes()); + out.extend_from_slice(&pack.checksum.to_le_bytes()); + } + let crc = crc32c(&out); + out.extend_from_slice(&crc.to_le_bytes()); + Ok(out) + } + + pub fn decode(bytes: &[u8]) -> Result<Self, CtError> { + if !(100..=MAX_STAGE_BYTES).contains(&bytes.len()) || u32_at(bytes, 0) != MAGIC { + return Err(CtError::Corruption); + } + if u32_at(bytes, 4) != 1 { + return Err(CtError::NotSupported); + } + let count = u32_at(bytes, 88) as usize; + if count > MAX_STAGE_PACKS + || bytes.len() != 100 + 16 * count + || u32_at(bytes, 92) != 0 + || crc32c(&bytes[..bytes.len() - 4]) != u32_at(bytes, bytes.len() - 4) + { + return Err(CtError::Corruption); + } + let state = match u32_at(bytes, 84) { + 1 => StageState::Reserved, + 2 => StageState::Writing, + 3 => StageState::Reloadable, + 4 => StageState::Referenced, + 5 => StageState::Retired, + 6 => StageState::Deleting, + 7 => StageState::Reclaimed, + _ => return Err(CtError::Corruption), + }; + let out = Self { + tree_id: u64_at(bytes, 8), + id: StageId { + owner_epoch: u64_at(bytes, 16), + operation_high: u64_at(bytes, 24), + operation_low: u64_at(bytes, 32), + }, + revision: u64_at(bytes, 40), + chunk_high: u64_at(bytes, 48), + chunk_low: u64_at(bytes, 56), + capacity: u64_at(bytes, 64), + manifest_generation: u64_at(bytes, 72), + purpose: u32_at(bytes, 80), + state, + packs: (0..count) + .map(|i| StagePack { + offset: u64_at(bytes, 96 + 16 * i), + length: u32_at(bytes, 104 + 16 * i), + checksum: u32_at(bytes, 108 + 16 * i), + }) + .collect(), + }; + out.validate().map_err(|_| CtError::Corruption)?; + Ok(out) + } +} + +fn u32_at(bytes: &[u8], at: usize) -> u32 { + u32::from_le_bytes(bytes[at..at + 4].try_into().expect("validated stage wire bounds")) +} +fn u64_at(bytes: &[u8], at: usize) -> u64 { + u64::from_le_bytes(bytes[at..at + 8].try_into().expect("validated stage wire bounds")) +} + +pub(crate) fn encode_stage_list(records: &[StageRecord]) -> Result<Vec<u8>, CtError> { + if records.len() > super::MAX_STAGE_SCAN as usize { + return Err(CtError::InvalidArgument); + } + let mut out = (records.len() as u32).to_le_bytes().to_vec(); + for record in records { + let bytes = record.encode()?; + out.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + out.extend_from_slice(&bytes); + } + Ok(out) +} + +pub(crate) fn decode_stage_list(bytes: &[u8]) -> Result<Vec<StageRecord>, CtError> { + if bytes.len() < 4 || bytes.len() > 4 + super::MAX_STAGE_SCAN as usize * (4 + MAX_STAGE_BYTES) { + return Err(CtError::Corruption); + } + let count = u32_at(bytes, 0) as usize; + if count > super::MAX_STAGE_SCAN as usize { + return Err(CtError::Corruption); + } + let mut records = Vec::with_capacity(count); + let mut at = 4; + for _ in 0..count { + if bytes.len() - at < 4 { + return Err(CtError::Corruption); + } + let length = u32_at(bytes, at) as usize; + at += 4; + if length > bytes.len() - at { + return Err(CtError::Corruption); + } + records.push(StageRecord::decode(&bytes[at..at + length])?); + at += length; + } + if at != bytes.len() { + return Err(CtError::Corruption); + } + Ok(records) +} diff --git a/lib/crowdb-tree/ffi/src/chunk/stage_callbacks.rs b/lib/crowdb-tree/ffi/src/chunk/stage_callbacks.rs new file mode 100644 index 000000000..0138fca5e --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/stage_callbacks.rs @@ -0,0 +1,190 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::stage::{ + decode_stage_list, encode_stage_list, StageId, StageRecord, MAX_STAGE_BYTES, MAX_STAGE_SCAN, +}; +use super::{catalog_store_ref, check, error_code, sys, ChunkRootCatalog, CtError}; +use std::ffi::c_void; + +fn external(id: StageId) -> sys::ct_chunk_stage_id { + sys::ct_chunk_stage_id { + owner_epoch: id.owner_epoch, + operation_high: id.operation_high, + operation_low: id.operation_low, + } +} +fn internal(id: sys::ct_chunk_stage_id) -> StageId { + StageId { + owner_epoch: id.owner_epoch, + operation_high: id.operation_high, + operation_low: id.operation_low, + } +} +impl ChunkRootCatalog { + /// Persistent stage-record schema support; not a complete writeback capability. + pub fn supports_staging(&self) -> bool { + unsafe { sys::ct_root_catalog_stage_capabilities(self.ptr.as_ptr()) & 1 != 0 } + } + pub fn load_stage(&self, tree_id: u64, id: StageId) -> Result<Option<StageRecord>, CtError> { + let mut out = sys::ct_buf { + data: std::ptr::null_mut(), + len: 0, + }; + let status = + unsafe { sys::ct_root_catalog_stage_load(self.ptr.as_ptr(), tree_id, external(id), &mut out) }; + let bytes = take_blob(status, out)?; + bytes.map(|bytes| StageRecord::decode(&bytes)).transpose() + } + pub fn compare_stage( + &self, + tree_id: u64, + owner_epoch: u64, + expected: Option<&StageRecord>, + next: Option<&StageRecord>, + ) -> Result<(), CtError> { + let expected = expected.map(StageRecord::encode).transpose()?.unwrap_or_default(); + let next = next.map(StageRecord::encode).transpose()?.unwrap_or_default(); + check(unsafe { + sys::ct_root_catalog_stage_compare( + self.ptr.as_ptr(), + tree_id, + owner_epoch, + expected.as_ptr(), + expected.len(), + next.as_ptr(), + next.len(), + ) + }) + } + pub fn list_stages( + &self, + tree_id: u64, + after: Option<StageId>, + limit: u32, + ) -> Result<Vec<StageRecord>, CtError> { + let cursor = after.map(external); + let mut out = sys::ct_buf { + data: std::ptr::null_mut(), + len: 0, + }; + let status = unsafe { + sys::ct_root_catalog_stage_list( + self.ptr.as_ptr(), + tree_id, + cursor.as_ref().map_or(std::ptr::null(), |id| id), + limit, + &mut out, + ) + }; + decode_stage_list(&take_blob(status, out)?.ok_or(CtError::Corruption)?) + } +} +fn take_blob(status: i32, mut out: sys::ct_buf) -> Result<Option<Vec<u8>>, CtError> { + let result = if status == -1 { + Ok(None) + } else { + check(status).and_then(|()| { + if out.data.is_null() || out.len > 4 + MAX_STAGE_SCAN as usize * (4 + MAX_STAGE_BYTES) { + return Err(CtError::Corruption); + } + Ok(Some( + unsafe { std::slice::from_raw_parts(out.data, out.len) }.to_vec(), + )) + }) + }; + unsafe { sys::ct_free_buf(&mut out) }; + result +} + +pub(super) fn callbacks() -> sys::ct_root_catalog_staging_callbacks { + sys::ct_root_catalog_staging_callbacks { + version: 1, + struct_bytes: std::mem::size_of::<sys::ct_root_catalog_staging_callbacks>() as u32, + capabilities: 1, + load: Some(load), + compare: Some(compare), + list: Some(list), + } +} +unsafe fn output(bytes: Vec<u8>, out: *mut *const u8, length: *mut usize) -> i32 { + let bytes = bytes.into_boxed_slice(); + *length = bytes.len(); + *out = Box::into_raw(bytes).cast::<u8>(); + 0 +} +unsafe fn optional(data: *const u8, length: usize) -> Result<Option<StageRecord>, CtError> { + if length == 0 { + return Ok(None); + } + if data.is_null() || length > MAX_STAGE_BYTES { + return Err(CtError::InvalidArgument); + } + StageRecord::decode(std::slice::from_raw_parts(data, length)).map(Some) +} +unsafe extern "C" fn load( + context: *mut c_void, + tree_id: u64, + id: sys::ct_chunk_stage_id, + out: *mut *const u8, + length: *mut usize, +) -> i32 { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if context.is_null() || out.is_null() || length.is_null() { + return -2; + } + match catalog_store_ref(context).load_stage(tree_id, internal(id)) { + Ok(Some(record)) => match record.encode() { + Ok(bytes) => output(bytes, out, length), + Err(error) => error_code(error), + }, + Ok(None) => -1, + Err(error) => error_code(error), + } + })) + .unwrap_or(-6) +} +unsafe extern "C" fn compare( + context: *mut c_void, + tree_id: u64, + owner_epoch: u64, + expected: *const u8, + expected_length: usize, + next: *const u8, + next_length: usize, +) -> i32 { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if context.is_null() { + return -2; + } + let result = (|| { + let expected = optional(expected, expected_length)?; + let next = optional(next, next_length)?; + catalog_store_ref(context).compare_stage(tree_id, owner_epoch, expected.as_ref(), next.as_ref()) + })(); + result.map_or_else(error_code, |()| 0) + })) + .unwrap_or(-6) +} +unsafe extern "C" fn list( + context: *mut c_void, + tree_id: u64, + after: *const sys::ct_chunk_stage_id, + limit: u32, + out: *mut *const u8, + length: *mut usize, +) -> i32 { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if context.is_null() || out.is_null() || length.is_null() || !(1..=MAX_STAGE_SCAN).contains(&limit) { + return -2; + } + let result = catalog_store_ref(context) + .list_stages(tree_id, after.as_ref().copied().map(internal), limit) + .and_then(|records| encode_stage_list(&records)); + match result { + Ok(bytes) => output(bytes, out, length), + Err(error) => error_code(error), + } + })) + .unwrap_or(-6) +} diff --git a/lib/crowdb-tree/ffi/src/chunk/test_util.rs b/lib/crowdb-tree/ffi/src/chunk/test_util.rs new file mode 100644 index 000000000..cae21aa87 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/chunk/test_util.rs @@ -0,0 +1,129 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +//! In-memory services exercising the ordinary fixed-pack runtime factory. +use super::{ChunkRootCatalog, ChunkRuntimeOptions, ChunkTransport}; +use crate::{error::check, sys, Config, CtError, PageStore, TreeRuntime, TreeRuntimeOptions}; +use std::ptr::NonNull; +use std::sync::Arc; + +extern "C" { + fn ct_test_chunk_runtime_create( + runtime: *mut sys::ct_runtime, + catalog: *mut *mut sys::ct_root_catalog, + transport: *mut *mut sys::ct_chunk_transport, + ) -> std::os::raw::c_int; + fn ct_test_chunk_runtime_next( + old: *mut sys::ct_root_catalog, + out: *mut *mut sys::ct_root_catalog, + ) -> std::os::raw::c_int; + fn ct_test_chunk_runtime_finish(catalog: *mut sys::ct_root_catalog) -> std::os::raw::c_int; + fn ct_test_chunk_runtime_root_load( + catalog: *mut sys::ct_root_catalog, + callback: extern "C" fn(*mut std::ffi::c_void) -> bool, + drop_context: extern "C" fn(*mut std::ffi::c_void), + context: *mut std::ffi::c_void, + ) -> std::os::raw::c_int; +} + +struct TestRootLoad(Box<dyn Fn() + Send + Sync>); + +extern "C" fn root_load(context: *mut std::ffi::c_void) -> bool { + // The host retains this stable box until its catalog and accepted work drain. + let hook = unsafe { &*context.cast::<TestRootLoad>() }; + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (hook.0)())).is_ok() +} + +extern "C" fn drop_root_load(context: *mut std::ffi::c_void) { + // Native catalog owners retain the context across writer replacement. + unsafe { drop(Box::from_raw(context.cast::<TestRootLoad>())) }; +} + +struct TestReleaseOwner(Arc<ChunkRootCatalog>); +impl Drop for TestReleaseOwner { + fn drop(&mut self) { + let result = check(unsafe { ct_test_chunk_runtime_finish(self.0.ptr.as_ptr()) }); + assert!( + result.is_ok(), + "test retained a Tree or snapshot past its Chunk host: {result:?}" + ); + } +} + +/// One persistent test tree with a shared CPU host and real candidate/pin rules. +/// Drop every Tree and snapshot before this service. Reopen through `next_writer` +/// only after the previous Tree closes; the catalog incarnation cannot be reused. +pub struct TestChunkBackend { + _release: TestReleaseOwner, + pub catalog: Arc<ChunkRootCatalog>, + transport: ChunkTransport, + options: ChunkRuntimeOptions, + pub runtime: Arc<TreeRuntime>, +} +impl TestChunkBackend { + pub fn new(tree: u64, epoch: u64) -> Result<Self, CtError> { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..TreeRuntimeOptions::default() + })?); + let mut catalog = std::ptr::null_mut(); + let mut transport = std::ptr::null_mut(); + check(unsafe { ct_test_chunk_runtime_create(runtime.as_ptr(), &mut catalog, &mut transport) })?; + let catalog = Arc::new(ChunkRootCatalog { + ptr: NonNull::new(catalog).ok_or(CtError::Internal)?, + }); + Ok(Self { + runtime, + catalog: catalog.clone(), + transport: ChunkTransport { + ptr: NonNull::new(transport).ok_or(CtError::Internal)?, + }, + options: ChunkRuntimeOptions { + tree_id: tree, + owner_epoch: epoch, + iu_size: 1, + ..ChunkRuntimeOptions::default() + }, + _release: TestReleaseOwner(catalog), + }) + } + pub fn page_store(&self, generation: u64) -> Result<Arc<PageStore>, CtError> { + PageStore::open_chunk_runtime( + &self.runtime, + ChunkRuntimeOptions { + open_generation: generation, + ..self.options + }, + self.catalog.clone(), + &self.transport, + ) + .map(Arc::new) + } + pub fn config(&self) -> Config { + Config { + runtime: Some(self.runtime.clone()), + frame_bytes: 4096, + ..Config::default() + } + } + /// Install while no Tree operation is running; the callback must not panic. + /// # Errors + /// Returns an invalid or unavailable native test catalog error. + pub fn on_root_load_for_tests( + &mut self, + callback: impl Fn() + Send + Sync + 'static, + ) -> Result<(), CtError> { + let hook = Box::into_raw(Box::new(TestRootLoad(Box::new(callback)))); + check(unsafe { + ct_test_chunk_runtime_root_load(self.catalog.ptr.as_ptr(), root_load, drop_root_load, hook.cast()) + })?; + Ok(()) + } + pub fn next_writer(&mut self) -> Result<(), CtError> { + let mut out = std::ptr::null_mut(); + check(unsafe { ct_test_chunk_runtime_next(self.catalog.ptr.as_ptr(), &mut out) })?; + self.catalog = Arc::new(ChunkRootCatalog { + ptr: NonNull::new(out).ok_or(CtError::Internal)?, + }); + Ok(()) + } +} diff --git a/lib/crowdb-tree/ffi/src/config.rs b/lib/crowdb-tree/ffi/src/config.rs index 3fe14ee64..3fbdde2e1 100644 --- a/lib/crowdb-tree/ffi/src/config.rs +++ b/lib/crowdb-tree/ffi/src/config.rs @@ -1,6 +1,11 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +mod engine; +mod runtime; +pub use engine::TreeEngineConfig; +pub use runtime::{TreeBindingConfig, TreeHostConfig, TreeResourceConfig, TreeRuntimeConfig}; + use std::ptr::NonNull; use std::sync::Arc; @@ -111,6 +116,14 @@ pub enum KeyRange { /// Engine configuration. `path = None` selects an in-memory store. #[derive(Debug, Clone, Default)] pub struct Config { + /// Shared host CPU/backend pools and allocation budgets. Each Tree keeps + /// its own publication state and page cache within this runtime. `None` + /// uses the process-wide standalone runtime. + pub runtime: Option<Arc<crate::TreeRuntime>>, + /// Per-Tree queue and memory limits, including on the process-wide host. + pub runtime_binding: crate::TreeBindingOptions, + /// Source, handoff, maintenance and cache work windows. + pub tuning: crate::TreeTuningOptions, /// Optional injected durable backend. When set, `path` and `backend` are /// ignored by the C++ constructor. pub page_store: Option<Arc<PageStore>>, diff --git a/lib/crowdb-tree/ffi/src/config/engine.rs b/lib/crowdb-tree/ffi/src/config/engine.rs new file mode 100644 index 000000000..7f256d031 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/config/engine.rs @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Startup engine settings. Native work-window defaults come through the FFI. +use super::TreeRuntimeConfig; +use crate::{Config, TreeTuningOptions}; + +#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct TreeEngineConfig { + pub frame_bytes: u32, + pub buffer_pool_bytes: u64, + pub max_inline_value: u64, + pub compression_lz4: bool, + pub block_size: u64, + pub iu_size: u32, + pub tuning: TreeTuningOptions, +} + +impl Default for TreeEngineConfig { + fn default() -> Self { + Self { + frame_bytes: 64 * 1024, + buffer_pool_bytes: 64 * 1024 * 1024, + max_inline_value: 0, + compression_lz4: false, + block_size: 64 * 1024 * 1024, + iu_size: 0, + tuning: TreeTuningOptions::default(), + } + } +} + +impl TreeEngineConfig { + pub fn apply_to(&self, options: &mut Config) { + options.frame_bytes = self.frame_bytes; + options.buffer_pool_bytes = self.buffer_pool_bytes; + options.max_inline_value = self.max_inline_value; + options.compression_lz4 = self.compression_lz4; + options.block_size = self.block_size; + options.iu_size = self.iu_size; + options.tuning = self.tuning; + } + + /// Validate before creating or recovering storage. + /// # Errors + /// Returns the incompatible configuration field or window. + pub fn validate(&self, runtime: &TreeRuntimeConfig) -> Result<(), String> { + if !(512..=65536).contains(&self.frame_bytes) || self.buffer_pool_bytes < u64::from(self.frame_bytes) + { + return Err("tree.frame_bytes must be in 512..=65536 and fit buffer_pool_bytes".to_owned()); + } + if self.iu_size != 0 + && (!self.iu_size.is_power_of_two() || self.frame_bytes.checked_rem(self.iu_size) != Some(0)) + { + return Err( + "tree.iu_size must be a power of two dividing frame_bytes, or zero for the backend default" + .to_owned(), + ); + } + let t = &self.tuning; + for (name, value) in [ + ("leaf_split_bytes", t.leaf_split_bytes), + ("inner_max_keys", t.inner_max_keys as usize), + ("source_max_entries", t.source_max_entries), + ("source_max_key_bytes", t.source_max_key_bytes), + ("source_max_gap_entries", t.source_max_gap_entries), + ("source_max_gap_key_bytes", t.source_max_gap_key_bytes), + ("run_max_entries", t.run_max_entries), + ("run_max_key_bytes", t.run_max_key_bytes), + ("run_key_block_bytes", t.run_key_block_bytes), + ("run_max_bytes", t.run_max_bytes), + ("install_run_window", t.install_run_window), + ("maintenance_visits_per_step", t.maintenance_visits_per_step), + ("maintenance_max_visits", t.maintenance_max_visits), + ("maintenance_max_candidates", t.maintenance_max_candidates), + ("maintenance_max_steps", t.maintenance_max_steps), + ("maintenance_metadata_bytes", t.maintenance_metadata_bytes), + ("cache_max_pages", t.cache_max_pages), + ] { + if value == 0 { + return Err(format!("tree.tuning.{name} must be positive")); + } + } + if t.max_memtable_count < 2 || t.max_memtable_count > 64 { + return Err("tree.tuning.max_memtable_count must be in 2..=64 (source window)".to_owned()); + } + if t.run_key_block_bytes > t.run_max_key_bytes + || t.maintenance_visits_per_step > t.maintenance_max_visits + { + return Err("tree.tuning step must fit its enclosing work window".to_owned()); + } + if t.flush_parallelism > runtime.runtime_options().workers + || t.flush_parallelism > runtime.binding_options().limits.io_operations + { + return Err("tree.tuning.flush_parallelism exceeds host workers or Tree I/O permits".to_owned()); + } + Ok(()) + } +} diff --git a/lib/crowdb-tree/ffi/src/config/runtime.rs b/lib/crowdb-tree/ffi/src/config/runtime.rs new file mode 100644 index 000000000..9f767d8de --- /dev/null +++ b/lib/crowdb-tree/ffi/src/config/runtime.rs @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Explicit host and per-Tree limits. Omitted values use native defaults. + +use crate::{TreeBindingOptions, TreeResourceLimits, TreeRuntimeOptions}; + +#[derive(Clone, Debug, PartialEq, Eq, Default, serde::Deserialize, serde::Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct TreeRuntimeConfig { + pub host: TreeHostConfig, + pub tree: TreeBindingConfig, +} + +#[derive(Clone, Debug, PartialEq, Eq, Default, serde::Deserialize, serde::Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct TreeHostConfig { + pub backend_workers: Option<usize>, + pub backend_queue_limit: Option<usize>, + pub workers: Option<usize>, + pub cpu_queue_limit: Option<usize>, + pub max_waiters_per_class: Option<usize>, + pub lifecycle_bytes: Option<usize>, + pub limits: TreeResourceConfig, +} + +#[derive(Clone, Debug, PartialEq, Eq, Default, serde::Deserialize, serde::Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct TreeBindingConfig { + pub backend_queue_limit: Option<usize>, + pub cpu_queue_limit: Option<usize>, + pub lifecycle_bytes: Option<usize>, + pub limits: TreeResourceConfig, +} + +#[derive(Clone, Debug, PartialEq, Eq, Default, serde::Deserialize, serde::Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct TreeResourceConfig { + pub handoff_bytes: Option<usize>, + pub routing_bytes: Option<usize>, + pub reclaim_bytes: Option<usize>, + pub writer_bytes: Option<usize>, + pub gap_repair_bytes: Option<usize>, + pub io_operations: Option<usize>, +} + +impl TreeResourceConfig { + fn apply(&self, defaults: TreeResourceLimits) -> TreeResourceLimits { + TreeResourceLimits { + handoff_bytes: self.handoff_bytes.unwrap_or(defaults.handoff_bytes), + routing_bytes: self.routing_bytes.unwrap_or(defaults.routing_bytes), + reclaim_bytes: self.reclaim_bytes.unwrap_or(defaults.reclaim_bytes), + writer_bytes: self.writer_bytes.unwrap_or(defaults.writer_bytes), + gap_repair_bytes: self.gap_repair_bytes.unwrap_or(defaults.gap_repair_bytes), + io_operations: self.io_operations.unwrap_or(defaults.io_operations), + } + } +} + +impl TreeRuntimeConfig { + #[must_use] + pub fn runtime_options(&self) -> TreeRuntimeOptions { + let defaults = TreeRuntimeOptions::default(); + TreeRuntimeOptions { + backend_workers: self.host.backend_workers.unwrap_or(defaults.backend_workers), + backend_queue_limit: self + .host + .backend_queue_limit + .unwrap_or(defaults.backend_queue_limit), + workers: self.host.workers.unwrap_or(defaults.workers), + cpu_queue_limit: self.host.cpu_queue_limit.unwrap_or(defaults.cpu_queue_limit), + max_waiters_per_class: self + .host + .max_waiters_per_class + .unwrap_or(defaults.max_waiters_per_class), + lifecycle_bytes: self.host.lifecycle_bytes.unwrap_or(defaults.lifecycle_bytes), + limits: self.host.limits.apply(defaults.limits), + } + } + + #[must_use] + pub fn binding_options(&self) -> TreeBindingOptions { + let defaults = TreeBindingOptions::default(); + TreeBindingOptions { + backend_queue_limit: self + .tree + .backend_queue_limit + .unwrap_or(defaults.backend_queue_limit), + cpu_queue_limit: self.tree.cpu_queue_limit.unwrap_or(defaults.cpu_queue_limit), + lifecycle_bytes: self.tree.lifecycle_bytes.unwrap_or(defaults.lifecycle_bytes), + limits: self.tree.limits.apply(defaults.limits), + } + } + + /// Validate host and per-Tree capacity before opening data. + /// # Errors + /// Returns the specific incompatible configuration field. + pub fn validate(&self) -> Result<(), String> { + let host = self.runtime_options(); + let tree = self.binding_options(); + if host.workers == 0 || host.cpu_queue_limit == 0 || host.max_waiters_per_class == 0 { + return Err("tree_runtime host workers, queue and waiter capacities must be positive".to_owned()); + } + if host.backend_workers == 0 + || host.backend_queue_limit == 0 + || tree.backend_queue_limit == 0 + || tree.backend_queue_limit > host.backend_queue_limit + { + return Err( + "tree_runtime backend workers/queues must be positive and the Tree queue must fit the host" + .to_owned(), + ); + } + if tree.cpu_queue_limit == 0 || tree.cpu_queue_limit > host.cpu_queue_limit { + return Err("tree_runtime.tree.cpu_queue_limit must be in 1..=host.cpu_queue_limit".to_owned()); + } + for (name, local, global) in [ + ( + "handoff_bytes", + tree.limits.handoff_bytes, + host.limits.handoff_bytes, + ), + ( + "routing_bytes", + tree.limits.routing_bytes, + host.limits.routing_bytes, + ), + ( + "reclaim_bytes", + tree.limits.reclaim_bytes, + host.limits.reclaim_bytes, + ), + ("writer_bytes", tree.limits.writer_bytes, host.limits.writer_bytes), + ( + "gap_repair_bytes", + tree.limits.gap_repair_bytes, + host.limits.gap_repair_bytes, + ), + ( + "io_operations", + tree.limits.io_operations, + host.limits.io_operations, + ), + ] { + if local == 0 || local > global { + return Err(format!( + "tree_runtime.tree.limits.{name} must be positive and at most its host limit" + )); + } + } + if host.lifecycle_bytes == 0 + || host.lifecycle_bytes > host.limits.routing_bytes + || tree.lifecycle_bytes == 0 + || tree.lifecycle_bytes > tree.limits.routing_bytes + { + return Err( + "tree_runtime lifecycle_bytes must be positive and fit its routing partition".to_owned(), + ); + } + Ok(()) + } +} diff --git a/lib/crowdb-tree/ffi/src/immutable_snapshot.rs b/lib/crowdb-tree/ffi/src/immutable_snapshot.rs new file mode 100644 index 000000000..39897a3be --- /dev/null +++ b/lib/crowdb-tree/ffi/src/immutable_snapshot.rs @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Independently owned immutable runtime view. No second writer is opened. +use crate::error::check; +use crate::reactor::{try_poll_ct_future, FutureGuard, FutureKind}; +use crate::{sys, Config, Crowdbtree, CtError, RangeRebuildStats, ScanDirection, ScanEntry, ScanOptions}; +use std::ptr::NonNull; + +#[derive(Debug)] +pub struct ImmutableTreeSnapshot { + handle: NonNull<sys::ct_immutable_snapshot>, + sequence: u64, + coverage: u64, +} + +// SAFETY: the native handle owns an immutable mapping; each read has independent +// paid operation state and backend ownership. Destruction follows host ownership. +unsafe impl Send for ImmutableTreeSnapshot {} +unsafe impl Sync for ImmutableTreeSnapshot {} + +impl Drop for ImmutableTreeSnapshot { + fn drop(&mut self) { + unsafe { sys::ct_immutable_snapshot_free(self.handle.as_ptr()) }; + } +} + +impl Crowdbtree { + /// Capture exactly the expected completed L1 coverage. A nonzero sequence + /// also requires the current durable checkpoint pair to match. + /// # Errors + /// Rejects changed coverage, unsupported backends, or unavailable budget. + pub fn immutable_snapshot(&self, sequence: u64, coverage: u64) -> Result<ImmutableTreeSnapshot, CtError> { + let mut raw = std::ptr::null_mut(); + check(unsafe { sys::ct_immutable_snapshot_capture(self.as_ptr(), sequence, coverage, &mut raw) })?; + Ok(ImmutableTreeSnapshot { + handle: NonNull::new(raw).ok_or(CtError::Internal)?, + sequence, + coverage, + }) + } +} + +impl ImmutableTreeSnapshot { + #[must_use] + pub const fn checkpoint_state(&self) -> (u64, u64) { + (self.sequence, self.coverage) + } + + /// Read a key from the fixed view. The copying result is application-owned. + /// # Errors + /// Returns a typed read or admission failure. + pub fn get(&self, key: &[u8]) -> Result<Option<(u64, Vec<u8>)>, CtError> { + let mut raw = std::ptr::null_mut(); + check(unsafe { + sys::ct_immutable_snapshot_get(self.handle.as_ptr(), key.as_ptr(), key.len(), &mut raw) + })?; + let result = + try_poll_ct_future(&mut FutureGuard(raw), FutureKind::Get).ok_or(CtError::Internal)??; + Ok(result.found.then_some((result.slot, result.value))) + } + + /// Read one bounded window with independently owned result bytes. + /// # Errors + /// Returns invalid boundaries, read failures, or insufficient budget. + pub fn scan(&self, options: &ScanOptions) -> Result<(Vec<ScanEntry>, bool), CtError> { + let start = options.start.as_deref().unwrap_or_default(); + let raw = sys::ct_scan_options_v2 { + struct_size: std::mem::size_of::<sys::ct_scan_options_v2>(), + prefix: options.prefix.as_ptr(), + prefix_len: options.prefix.len(), + start: start.as_ptr(), + start_len: start.len(), + end: options.end.as_ptr(), + end_len: options.end.len(), + begin: options.begin.as_ptr(), + begin_len: options.begin.len(), + limit: options.limit, + byte_budget: options.byte_budget, + deadline_ms: options.deadline_ms, + reverse: i32::from(options.direction == ScanDirection::Reverse), + keys_only: i32::from(options.keys_only), + include_tombstones: i32::from(options.include_tombstones), + has_start_bound: i32::from(options.start.is_some()), + start_inclusive: i32::from(options.start_inclusive), + strict_byte_budget: i32::from(options.strict_byte_budget), + }; + let mut result = std::ptr::null_mut(); + check(unsafe { sys::ct_immutable_snapshot_scan(self.handle.as_ptr(), &raw, &mut result) })?; + let (packed, count, truncated) = crate::scan::owner::take(result)?; + Ok((crate::scan::decode_scan_bytes(packed, count)?, truncated)) + } + + /// Creates a separate runtime destination/account and rebuilds only its + /// configured range. Captured pages remain valid after the source closes. + /// # Errors + /// Returns destination admission, read, import, or checkpoint failures. + pub fn rebuild_range(&self, options: &Config) -> Result<(Crowdbtree, RangeRebuildStats), CtError> { + if options.page_store.is_none() { + return Err(CtError::InvalidArgument); + } + let destination = Crowdbtree::open(options)?; + let mut stats = sys::ct_range_rebuild_stats::default(); + check(unsafe { + sys::ct_immutable_snapshot_rebuild(self.handle.as_ptr(), destination.as_ptr(), &mut stats) + })?; + Ok(( + destination, + RangeRebuildStats { + entries_examined: stats.entries_examined, + entries_emitted: stats.entries_emitted, + entries_filtered: stats.entries_filtered, + pages_reused: stats.pages_reused, + pages_rebuilt: stats.pages_rebuilt, + subtrees_skipped: stats.subtrees_skipped, + }, + )) + } +} diff --git a/lib/crowdb-tree/ffi/src/lib.rs b/lib/crowdb-tree/ffi/src/lib.rs index 9fe5e1a9e..359dab757 100644 --- a/lib/crowdb-tree/ffi/src/lib.rs +++ b/lib/crowdb-tree/ffi/src/lib.rs @@ -10,42 +10,73 @@ //! thread hop, Phase 3); the remaining methods (no async C API twin exists //! for them yet) are called via the synchronous `Crowdbtree` handle. +pub mod admission; pub mod async_tree; +pub mod backend_release; pub mod batch; pub mod chunk; pub mod config; pub mod cpp_global_metrics; pub mod crc; pub mod error; +pub mod immutable_snapshot; +pub mod maintenance; +mod owner_budget; pub mod page; pub mod reactor; +pub mod runtime; pub mod scan; pub mod snapshot; pub mod stats; pub(crate) mod sys; pub mod tree; +pub mod tuning; +pub use tuning::TreeTuningOptions; pub mod uring; pub mod write_handle; -pub use async_tree::{AsyncCrowdbtree, GetOutcome, PinnedGetOutcome, ScanOutcome}; -pub use batch::{BatchOp, ExtOp}; +pub use admission::{ArmedApply, PreparedApply}; +pub use async_tree::{AsyncCrowdbtree, GetOutcome, OwnedGetOutcome, PinnedGetOutcome, ScanOutcome}; +pub use backend_release::BackendReleaseService; +pub use batch::{BatchOp, ExtOp, OwnedAllocation, OwnedOp}; pub use chunk::{ ChunkPageStoreOptions, ChunkPageStoreStats, ChunkRootCatalog, ChunkRpcDiskRoute, ChunkRpcRoute, - ChunkRpcTransportOptions, ChunkTransport, RootCatalogObject, RootCatalogStore, + ChunkRpcTransportOptions, ChunkRuntimeOptions, ChunkTransport, RootCatalogObject, RootCatalogStore, }; #[cfg(feature = "chunk-rpc")] pub use chunk::{ChunkRpcRouteResolver, OwnedChunkRpcDiskRoute, OwnedChunkRpcTransportOptions}; -pub use config::{Compression, Config, KeyRange, PageStore, PageStoreBackend, SyncMode}; +pub use config::{ + Compression, Config, KeyRange, PageStore, PageStoreBackend, SyncMode, TreeBindingConfig, + TreeEngineConfig, TreeHostConfig, TreeResourceConfig, TreeRuntimeConfig, +}; pub use cpp_global_metrics::{cpp_global_metrics_max_name_len, flush_cpp_global_metrics}; pub use crc::{crc32c, crc32c_update}; pub use error::CtError; +pub use immutable_snapshot::ImmutableTreeSnapshot; +pub use maintenance::{CheckpointReport, FlushPhase, FlushReport}; pub use reactor::PinnedValue; -pub use scan::{ScanDirection, ScanEntry, ViewEntry}; -pub use snapshot::{SnapshotChunk, SnapshotExportSession, SnapshotImportSession, SnapshotMetadata}; -pub use stats::{MergeGcStats, RangeRebuildStats, Stats, TreeSummary}; +pub use runtime::{ + TreeBindingOptions, TreeExecutorSnapshot, TreeResourceLimits, TreeRuntime, TreeRuntimeOptions, + TreeRuntimeSnapshot, +}; +pub use scan::{ScanDirection, ScanEntry, ScanOptions, ViewEntry}; +pub use snapshot::{ + OwnedSnapshotChunk, SnapshotChunk, SnapshotExportSession, SnapshotImportSession, SnapshotMetadata, +}; +pub use stats::{ + BackingStatsSnapshot, MergeGcStats, RangeRebuildStats, Stats, TreeFlushDispatchStats, TreeHistogramStats, + TreeLogicalStats, TreeProgressSnapshot, TreeResourceStats, TreeStatsSnapshot, TreeSummary, + TreeWorkCollector, TreeWorkCounters, TreeWorkHistograms, TreeWorkSnapshot, TreeWorkStage, + TreeWorkStageAge, TreeWorkStageStats, TREE_WORK_CAUSE_NAMES, TREE_WORK_COUNTER_NAMES, + TREE_WORK_STAGE_NAMES, +}; pub use tree::{ ct_add_log_stderr, ct_flush_logging, ct_init_logging, ct_init_test_logging, ct_shutdown_logging, - Crowdbtree, + Crowdbtree, InstalledSplit, PreparedSplitOverlay, SplitOverlayLimits, }; pub use uring::{Uring, UringFile}; pub use write_handle::WriteHandle; + +pub use async_tree::{CompactionOptions, CompactionReport}; + +pub use stats::{TreeWaitSnapshot, TreeWaitStats, TREE_WAIT_CLASS_NAMES}; diff --git a/lib/crowdb-tree/ffi/src/maintenance.rs b/lib/crowdb-tree/ffi/src/maintenance.rs new file mode 100644 index 000000000..49827561e --- /dev/null +++ b/lib/crowdb-tree/ffi/src/maintenance.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crate::error::{check, CtError}; +use crate::sys; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FlushPhase { + Captured, + Handoff, + RepairRequired, + Covered, + Maintenance, + Complete, + Unknown(u32), +} + +/// An accepted flush retains both phase results, including partial progress. +/// A maintenance failure does not undo an already completed L0-to-L1 handoff. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FlushReport { + pub status: Result<(), CtError>, + pub handoff_status: Result<(), CtError>, + pub maintenance_status: Result<(), CtError>, + pub phase: FlushPhase, + pub batch_id: u64, + pub floor: u64, + pub frontier: u64, + pub published_coverage: u64, + pub deferred_pages: u64, + pub handoff_completed: bool, + /// This attempt first advanced coverage; retries never count it twice. + pub handoff_advanced: bool, +} + +impl From<sys::ct_flush_result_v1> for FlushReport { + fn from(raw: sys::ct_flush_result_v1) -> Self { + Self { + status: check(raw.terminal_status), + handoff_status: check(raw.handoff_status), + maintenance_status: check(raw.maintenance_status), + phase: match raw.phase { + 0 => FlushPhase::Captured, + 1 => FlushPhase::Handoff, + 2 => FlushPhase::RepairRequired, + 3 => FlushPhase::Covered, + 4 => FlushPhase::Maintenance, + 5 => FlushPhase::Complete, + other => FlushPhase::Unknown(other), + }, + batch_id: raw.batch_id, + floor: raw.floor, + frontier: raw.frontier, + published_coverage: raw.published_coverage, + deferred_pages: raw.deferred_pages, + handoff_completed: raw.handoff_completed != 0, + handoff_advanced: raw.handoff_advanced != 0, + } + } +} + +/// Writeback keeps durable coverage unchanged. Only a completed durable +/// checkpoint may advance recovery and WAL reclamation watermarks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CheckpointReport { + pub status: Result<(), CtError>, + pub durable_committed: bool, + pub writeback_completed: bool, + pub durable_coverage: u64, + pub snapshot_sequence: u64, + pub checkpoint_token: u64, + pub captured_coverage: u64, + pub primary_frames: u64, + pub written_frames: u64, + pub reused_frames: u64, + pub rebased_pages: u64, + pub rebase_deferred: u64, +} + +impl From<sys::ct_checkpoint_result_v1> for CheckpointReport { + fn from(raw: sys::ct_checkpoint_result_v1) -> Self { + Self { + status: check(raw.terminal_status), + durable_committed: raw.durable_committed != 0, + writeback_completed: raw.writeback_completed != 0, + durable_coverage: raw.durable_coverage, + snapshot_sequence: raw.snapshot_sequence, + checkpoint_token: raw.checkpoint_token, + captured_coverage: raw.captured_coverage, + primary_frames: raw.primary_frames, + written_frames: raw.written_frames, + reused_frames: raw.reused_frames, + rebased_pages: raw.rebased_pages, + rebase_deferred: raw.rebase_deferred, + } + } +} diff --git a/lib/crowdb-tree/ffi/src/owner_budget.rs b/lib/crowdb-tree/ffi/src/owner_budget.rs new file mode 100644 index 000000000..15d97e71e --- /dev/null +++ b/lib/crowdb-tree/ffi/src/owner_budget.rs @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::alloc::Layout; + +use crate::error::{check, CtError}; +use crate::sys; + +pub(crate) struct OwnerBudget(*mut sys::ct_owned_budget); + +// SAFETY: native budgets retain independent ownership and use atomic admission +// and accounting. The unique opaque handle is freed once after all byte clones. +unsafe impl Send for OwnerBudget {} +unsafe impl Sync for OwnerBudget {} + +impl OwnerBudget { + pub(crate) fn wrap<T: AsRef<[u8]> + Send + 'static>(self, owner: T) -> Result<bytes::Bytes, CtError> { + if self.0.is_null() { + Ok(bytes::Bytes::from_owner(owner)) + } else { + bytes::Bytes::try_from_owner_accounted(owner, self) + } + } + + pub(crate) fn future(handle: *mut sys::ct_future) -> Result<Self, CtError> { + let mut budget = std::ptr::null_mut(); + check(unsafe { sys::ct_future_owned_budget(handle, &mut budget) })?; + Ok(Self(budget)) + } + pub(crate) fn scan(handle: *mut sys::ct_scan_result) -> Result<Self, CtError> { + let mut budget = std::ptr::null_mut(); + check(unsafe { sys::ct_scan_result_owned_budget(handle, &mut budget) })?; + Ok(Self(budget)) + } + pub(crate) fn snapshot(handle: *mut sys::ct_snapshot_chunk) -> Result<Self, CtError> { + let mut budget = std::ptr::null_mut(); + check(unsafe { sys::ct_snapshot_chunk_owned_budget(handle, &mut budget) })?; + Ok(Self(budget)) + } +} +impl bytes::OwnerAllocation for OwnerBudget { + type Error = CtError; + fn claim(&self, layout: Layout) -> Result<(), CtError> { + check(unsafe { sys::ct_owned_budget_claim(self.0, layout.size()) }) + } + fn record(&self, layout: Layout, acquired: bool) { + unsafe { sys::ct_owned_budget_record(self.0, layout.size(), i32::from(acquired)) }; + } + fn release(&self, layout: Layout) { + unsafe { sys::ct_owned_budget_release(self.0, layout.size()) }; + } +} +impl Drop for OwnerBudget { + fn drop(&mut self) { + unsafe { sys::ct_owned_budget_free(self.0) }; + } +} diff --git a/lib/crowdb-tree/ffi/src/reactor.rs b/lib/crowdb-tree/ffi/src/reactor.rs index 790625303..b52b75ca0 100644 --- a/lib/crowdb-tree/ffi/src/reactor.rs +++ b/lib/crowdb-tree/ffi/src/reactor.rs @@ -1,21 +1,21 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use std::os::fd::{AsRawFd, RawFd}; +use std::os::fd::{BorrowedFd, OwnedFd, RawFd}; use std::os::raw::c_int; use std::sync::Arc; -use crate::error::{check, copy_buf, take_buf, CtError}; +use crate::error::{check, copy_buf, CtError}; use crate::sys; use crate::tree::Crowdbtree; // Reactor-driven async futures. // -// AsyncCrowdbtree::get/flush/snapshot/scan drive drive_ct_future below directly: -// no spawn_blocking, no OS thread hop. A fast-path completion (get_view's -// cached L0/L1 hit, or flush's always-in-memory work) resolves on the -// *first* poll without ever touching the reactor; only a genuine -// demand-load miss waits on the tree's eventfd. +// Native reads, scans, admission and maintenance operations use completion +// polling with eventfd notifications. CPU work runs on the shared native runtime; +// even resident reads may be pending on their first poll. An already completed +// operation returns immediately. Pending CPU work and backend I/O both wake the +// eventfd pump when completion becomes observable. // // Fan-out note: this deliberately does *not* have every pending future call // `AsyncFd::poll_read_ready` on a shared registration -- that method's @@ -27,17 +27,13 @@ use crate::tree::Crowdbtree; // waits on a `tokio::sync::Notify` the pump fans out to instead, which does // support any number of concurrent waiters. -/// Non-owning view of a raw fd for `AsyncFd` registration. The engine's -/// `DiskIOUring` owns the eventfds `ct_uring_eventfds` returns and closes -/// them in its own destructor (~`DiskIOUring`); Rust must wrap them -/// *without* taking closing ownership -- unlike `std::os::fd::OwnedFd`, -/// this type's `Drop` is a no-op. -pub(crate) struct RawFdView(pub(crate) RawFd); - -impl AsRawFd for RawFdView { - fn as_raw_fd(&self) -> RawFd { - self.0 - } +/// Give the pump an independent descriptor so native Tree closure cannot +/// recycle a descriptor that Tokio has yet to deregister after task abort. +/// +/// # Safety +/// `fd` is a live native eventfd and its owner outlives this call. +pub(crate) unsafe fn duplicate_eventfd(fd: RawFd) -> std::io::Result<OwnedFd> { + unsafe { BorrowedFd::borrow_raw(fd) }.try_clone_to_owned() } extern "C" { @@ -62,8 +58,8 @@ pub(crate) fn drain_eventfd(fd: RawFd) { /// The tree's lazily-spawned eventfd pump (see the module-level fan-out /// note above): `notify` is fanned out to every waiting `drive_ct_future` /// call each time the pump observes the eventfd fire; `task` is aborted by -/// `Crowdbtree`'s `Drop` before `ct_close` runs (the eventfd itself becomes -/// invalid once the Reactor is torn down). +/// `Crowdbtree`'s `Drop` before `ct_close` runs. The pump owns a duplicate fd +/// until Tokio deregisters it and drops the cancelled task. pub(crate) struct EventfdPump { pub(crate) notify: Arc<tokio::sync::Notify>, pub(crate) task: tokio::task::AbortHandle, @@ -74,24 +70,17 @@ pub(crate) struct RawOutcome { pub(crate) found: bool, pub(crate) slot: u64, pub(crate) value: Vec<u8>, + pub(crate) packed: Option<bytes::Bytes>, } -/// Which `ct_*_async` call produced a `FutureGuard` -- `ct_future_poll`'s -/// freeing contract differs for `Get`: -/// a resolved kGet future is deliberately *not* freed by `ct_future_poll` -/// itself (its `out_value` may still borrow from a resident frame, kept -/// alive by the future's own epoch guard), so the caller must free it -/// explicitly once done reading -- unlike Flush/Snapshot/Scan, which -/// `ct_future_poll` already frees on completion, same as before Phase 4. -/// `Scan`'s `out_value` (follow-up) is always a *malloc'd* -/// owned buffer (never borrowed, unlike Get) -- `try_poll_ct_future` must -/// free it via `take_buf`, not `copy_buf`. +/// Native future ownership kind. Get results borrow their bytes until the +/// handle is freed; admission completions free themselves on polling. Owned +/// scans use the versioned owner protocol and retain their result separately. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum FutureKind { Get, Flush, - Snapshot, - Scan, + ScanOwned, } /// RAII guard for one in-flight `ct_future`: frees it via `ct_future_free` @@ -121,7 +110,7 @@ unsafe impl Send for FutureGuard {} /// One synchronous `ct_future_poll` attempt. `None` if still pending; /// `Some` if done, in which case `guard` has already been nulled out -- /// either `ct_future_poll` itself freed the underlying `ct_future` -/// (Flush/Snapshot), or, for a `Get`, this function did so explicitly via +/// (admission completion), or, for a `Get`, this function did so explicitly via /// `ct_future_free` right after copying `out_value`'s bytes out (which may /// be a *borrowed* pointer into a still-live frame -- see `copy_buf`). /// @@ -132,6 +121,12 @@ pub(crate) fn try_poll_ct_future( guard: &mut FutureGuard, kind: FutureKind, ) -> Option<Result<RawOutcome, CtError>> { + if guard.0.is_null() { + return Some(Err(CtError::ResourceExhausted)); + } + if kind == FutureKind::ScanOwned { + return crate::scan::owner::poll(guard); + } let mut done: c_int = 0; let mut found: c_int = 0; let mut slot: u64 = 0; @@ -143,37 +138,25 @@ pub(crate) fn try_poll_ct_future( if done == 0 { return None; } - // Extracted unconditionally, before checking status: for Scan, `value` - // is always a malloc'd owned buffer (see FutureKind::Scan's doc - // comment) that must be freed via take_buf/ct_free_buf regardless of - // whether the underlying op errored (ct_future_poll's kScan branch - // populates *out_value with an owned buffer -- possibly empty, but - // still malloc'd -- either way; leaving `value` unhandled inside a - // match arm that only runs on Ok would leak it on an errored scan). - let value_bytes = if kind == FutureKind::Scan { - take_buf(value) - } else { - // copy_buf, not take_buf: for a Get, `value` may borrow from a - // still-live frame (zero-copy fast path) and must - // not be passed to ct_free_buf. Flush/Snapshot never populate - // `value` at all, so this is a no-op for them either way. - copy_buf(value) - }; + // The get handle retains the borrowed result until after this copy. + // Admission completions return an empty buffer. + let value_bytes = copy_buf(value); let result = match check(rc) { Ok(()) => Ok(RawOutcome { found: found != 0, slot, value: value_bytes, + packed: None, }), Err(e) => Err(e), }; if kind == FutureKind::Get { // ct_future_poll deliberately does *not* free a kGet future (see - // its doc comment in c_api.h) -- the epoch guard behind a - // zero-copy fast-path value must outlive the copy_buf call above. + // its doc comment in c_api.h). The owned read result must outlive + // the copy_buf call above. unsafe { sys::ct_future_free(guard.0) }; } - // Flush/Snapshot: already freed by ct_future_poll itself. Either way, + // Admission completions are freed by ct_future_poll itself. Either way, // the underlying ct_future is gone now -- don't let FutureGuard's Drop // free it again. guard.0 = std::ptr::null_mut(); @@ -215,9 +198,24 @@ impl PinnedValue { /// per-page refcount is thread-independent, so `PinnedValue` is `Send`). /// When the last `Bytes` ref clone is dropped, the owner's `Drop` runs, /// which drops the `PinnedValue`, which calls `ct_future_free` to unpin. + /// + /// # Panics + /// Panics if the pre-admitted control window cannot satisfy its fixed owner layout. #[must_use] pub fn into_bytes(self) -> bytes::Bytes { - bytes::Bytes::from_owner(PinnedBytesOwner { pv: self }) + self.try_into_bytes() + .expect("admitted immutable result control capacity") + } + + /// Preserve the native allocation account through the Rust control block. + /// The exact box layout is claimed before allocation and refunded after + /// the last byte clone is destroyed and that box has been deallocated. + /// + /// # Errors + /// Returns a native admission error if the fixed owner control cannot be claimed. + pub fn try_into_bytes(self) -> Result<bytes::Bytes, CtError> { + let budget = crate::owner_budget::OwnerBudget::future(self.handle)?; + budget.wrap(PinnedBytesOwner { pv: self }) } } @@ -251,6 +249,9 @@ impl Drop for PinnedValue { pub(crate) fn try_poll_ct_future_pinned( guard: &mut FutureGuard, ) -> Option<Result<Option<(u64, PinnedValue)>, CtError>> { + if guard.0.is_null() { + return Some(Err(CtError::ResourceExhausted)); + } let mut done: c_int = 0; let mut found: c_int = 0; let mut slot: u64 = 0; @@ -286,15 +287,22 @@ pub(crate) fn try_poll_ct_future_pinned( Some(result) } -/// Drives one `ct_get_async`/`ct_flush_async`/`ct_snapshot_async` handle to -/// completion: polls it, and if not yet done, waits for the tree's eventfd -/// pump to fan out a notification before polling again. A fast-path -/// completion never reaches the `notified.await` at all. +/// Drives a get, admission or owned-scan handle to completion: polls it and, +/// if pending, waits for the tree's eventfd pump to fan out a notification. +/// An operation already complete on the first poll needs no notification wait. pub(crate) async fn drive_ct_future( - mut guard: FutureGuard, + guard: FutureGuard, tree: &Arc<Crowdbtree>, kind: FutureKind, ) -> Result<RawOutcome, CtError> { + drive_completion(guard, tree, |guard| try_poll_ct_future(guard, kind)).await +} + +pub(crate) async fn drive_completion<T>( + mut guard: FutureGuard, + tree: &Arc<Crowdbtree>, + mut poll: impl FnMut(&mut FutureGuard) -> Option<Result<T, CtError>>, +) -> Result<T, CtError> { loop { // Construct (but do not yet await) the notification future *before* // checking ct_future_poll below, not after: Notify::notified @@ -308,18 +316,16 @@ pub(crate) async fn drive_ct_future( let notify_arc = tree.eventfd_notify(); let notified = notify_arc.as_ref().map(|n| n.notified()); - if let Some(result) = try_poll_ct_future(&mut guard, kind) { + if let Some(result) = poll(&mut guard) { return result; } match notified { Some(n) => n.await, None => { - // No reactor wired (or the pump failed to spawn): per - // ct_get_async/ct_flush_async/ct_snapshot_async's contract, - // no reactor means every op already completes - // synchronously, so done=0 here should be unreachable -- - // yield instead of busy-looping just in case. + // Without an available notification pump, yield before + // polling again. Native CPU work can still be pending even + // when no backend I/O is needed. tokio::task::yield_now().await; } } diff --git a/lib/crowdb-tree/ffi/src/runtime.rs b/lib/crowdb-tree/ffi/src/runtime.rs new file mode 100644 index 000000000..01e29549d --- /dev/null +++ b/lib/crowdb-tree/ffi/src/runtime.rs @@ -0,0 +1,178 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +//! Shared host CPU capacity and resource limits for budgeted runtime Trees. +//! Cloned runtime owners share the same CPU pool and allocation ledger. + +use std::ptr::NonNull; + +use crate::error::{check, CtError}; +use crate::sys; + +mod stats; +mod usage; +pub use stats::{TreeExecutorSnapshot, TreeRuntimeSnapshot}; +pub use usage::{ResourceUsage, ResourceUsageLog, ResourceUsageSnapshot, TREE_RESOURCE_NAMES}; + +/// Independent partitions; `io_operations` is a count, all other limits bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TreeResourceLimits { + pub handoff_bytes: usize, + pub routing_bytes: usize, + pub reclaim_bytes: usize, + pub writer_bytes: usize, + pub gap_repair_bytes: usize, + pub io_operations: usize, +} + +impl From<sys::ct_resource_limits> for TreeResourceLimits { + fn from(value: sys::ct_resource_limits) -> Self { + Self { + handoff_bytes: value.handoff_bytes, + routing_bytes: value.routing_bytes, + reclaim_bytes: value.reclaim_bytes, + writer_bytes: value.writer_bytes, + gap_repair_bytes: value.gap_repair_bytes, + io_operations: value.io_operations, + } + } +} + +impl From<TreeResourceLimits> for sys::ct_resource_limits { + fn from(value: TreeResourceLimits) -> Self { + Self { + handoff_bytes: value.handoff_bytes, + routing_bytes: value.routing_bytes, + reclaim_bytes: value.reclaim_bytes, + writer_bytes: value.writer_bytes, + gap_repair_bytes: value.gap_repair_bytes, + io_operations: value.io_operations, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct TreeRuntimeOptions { + pub backend_workers: usize, + pub backend_queue_limit: usize, + pub workers: usize, + pub cpu_queue_limit: usize, + pub max_waiters_per_class: usize, + pub lifecycle_bytes: usize, + pub limits: TreeResourceLimits, +} + +impl Default for TreeRuntimeOptions { + fn default() -> Self { + let mut raw = sys::ct_runtime_options::default(); + unsafe { sys::ct_runtime_options_default(&mut raw) }; + Self { + backend_workers: raw.backend_workers, + backend_queue_limit: raw.backend_queue_limit, + workers: raw.workers, + cpu_queue_limit: raw.cpu_queue_limit, + max_waiters_per_class: raw.max_waiters_per_class, + lifecycle_bytes: raw.lifecycle_bytes, + limits: raw.limits.into(), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct TreeBindingOptions { + pub backend_queue_limit: usize, + pub cpu_queue_limit: usize, + pub lifecycle_bytes: usize, + pub limits: TreeResourceLimits, +} + +impl Default for TreeBindingOptions { + fn default() -> Self { + let mut raw = sys::ct_tree_runtime_options::default(); + unsafe { sys::ct_tree_runtime_options_default(&mut raw) }; + Self { + backend_queue_limit: raw.backend_queue_limit, + cpu_queue_limit: raw.cpu_queue_limit, + lifecycle_bytes: raw.lifecycle_bytes, + limits: raw.limits.into(), + } + } +} + +impl TreeBindingOptions { + pub(crate) fn as_raw(self) -> sys::ct_tree_runtime_options { + sys::ct_tree_runtime_options { + struct_size: std::mem::size_of::<sys::ct_tree_runtime_options>(), + cpu_queue_limit: self.cpu_queue_limit, + lifecycle_bytes: self.lifecycle_bytes, + limits: self.limits.into(), + backend_queue_limit: self.backend_queue_limit, + } + } +} + +/// Share with `Arc<TreeRuntime>` in multiple [`crate::Config`] values. +/// Native bindings retain independent ownership; dropping this handle does +/// not stop live Trees. The last owner must be released at a host boundary. +#[derive(Debug)] +pub struct TreeRuntime { + ptr: NonNull<sys::ct_runtime>, +} + +impl TreeRuntime { + /// Read cumulative host counters without resetting a collector window. + pub fn snapshot(&self) -> Result<TreeRuntimeSnapshot, CtError> { + let mut raw = sys::ct_runtime_snapshot_v1 { + struct_size: std::mem::size_of::<sys::ct_runtime_snapshot_v1>(), + version: 1, + ..Default::default() + }; + check(unsafe { sys::ct_runtime_snapshot(self.as_ptr(), &mut raw) })?; + Ok(raw.into()) + } + + pub fn new(options: TreeRuntimeOptions) -> Result<Self, CtError> { + let raw = sys::ct_runtime_options { + struct_size: std::mem::size_of::<sys::ct_runtime_options>(), + workers: options.workers, + cpu_queue_limit: options.cpu_queue_limit, + max_waiters_per_class: options.max_waiters_per_class, + lifecycle_bytes: options.lifecycle_bytes, + limits: options.limits.into(), + backend_workers: options.backend_workers, + backend_queue_limit: options.backend_queue_limit, + }; + let mut out = std::ptr::null_mut(); + check(unsafe { sys::ct_runtime_create(&raw, &mut out) })?; + Ok(Self { + ptr: NonNull::new(out).ok_or(CtError::Internal)?, + }) + } + + /// Reject new bindings/reservations and cancel budget waiters. Accepted + /// operations can still submit CPU continuations using retained credit. + pub fn stop_admission(&self) { + unsafe { sys::ct_runtime_stop_admission(self.as_ptr()) }; + } + + /// Blocking host boundary after Tree operations and backend tickets end. + /// Closes CPU submission and waits for CPU jobs only, not external I/O. + pub fn drain_cpu(&self) { + unsafe { sys::ct_runtime_drain_cpu(self.as_ptr()) }; + } + + pub(crate) fn as_ptr(&self) -> *mut sys::ct_runtime { + self.ptr.as_ptr() + } +} + +// Native runtime methods share atomic admission/queue state. The opaque handle +// stays alive under Rust ownership for every call; Drop needs exclusive access. +unsafe impl Send for TreeRuntime {} +unsafe impl Sync for TreeRuntime {} + +impl Drop for TreeRuntime { + fn drop(&mut self) { + unsafe { sys::ct_runtime_free(self.as_ptr()) }; + } +} diff --git a/lib/crowdb-tree/ffi/src/runtime/stats.rs b/lib/crowdb-tree/ffi/src/runtime/stats.rs new file mode 100644 index 000000000..ee7578d74 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/runtime/stats.rs @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::TreeResourceLimits; +use crate::sys; + +/// Cumulative executor samples. Parallel durations may exceed elapsed time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TreeExecutorSnapshot { + pub queued: u64, + pub inflight: u64, + pub submitted: u64, + pub completed: u64, + pub rejected: u64, + pub queue_ns: u64, + pub execute_ns: u64, +} + +impl From<sys::ct_executor_snapshot_v1> for TreeExecutorSnapshot { + fn from(value: sys::ct_executor_snapshot_v1) -> Self { + Self { + queued: value.queued, + inflight: value.inflight, + submitted: value.submitted, + completed: value.completed, + rejected: value.rejected, + queue_ns: value.queue_ns, + execute_ns: value.execute_ns, + } + } +} + +/// Approximate host-wide resource and execution samples; sampling never drains counters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TreeRuntimeSnapshot { + pub reserved: TreeResourceLimits, + pub allocated: TreeResourceLimits, + pub bindings: u64, + pub stopped: bool, + pub cpu: TreeExecutorSnapshot, + pub backend: TreeExecutorSnapshot, + pub budget_waiters: u64, + pub budget_grants: u64, + pub budget_cancelled: u64, + pub budget_wait_ns: u64, +} + +impl From<sys::ct_runtime_snapshot_v1> for TreeRuntimeSnapshot { + fn from(value: sys::ct_runtime_snapshot_v1) -> Self { + Self { + reserved: value.reserved.into(), + allocated: value.allocated.into(), + bindings: value.bindings, + stopped: value.stopped != 0, + cpu: value.cpu.into(), + backend: value.backend.into(), + budget_waiters: value.budget_waiters, + budget_grants: value.budget_grants, + budget_cancelled: value.budget_cancelled, + budget_wait_ns: value.budget_wait_ns, + } + } +} diff --git a/lib/crowdb-tree/ffi/src/runtime/usage.rs b/lib/crowdb-tree/ffi/src/runtime/usage.rs new file mode 100644 index 000000000..0ee7bd90a --- /dev/null +++ b/lib/crowdb-tree/ffi/src/runtime/usage.rs @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crate::error::check; +use crate::{sys, Crowdbtree, CtError}; + +mod window; +pub use window::ResourceUsageLog; + +pub const TREE_RESOURCE_NAMES: [&str; 21] = [ + "handoff", + "routing", + "reclaim", + "writer", + "gap_repair", + "io", + "cpu_queue_slots", + "backend_queue_slots", + "cpu_workers", + "backend_workers", + "cache_bytes", + "cache_pages", + "cache_request_slots", + "backend_operation_slots", + "apply_operation_slots", + "replay_operation_slots", + "source_entry_slots", + "source_key_bytes", + "gap_source_entry_slots", + "gap_source_key_bytes", + "chunk_pin_release_slots", +]; + +/// Relaxed, non-draining admission gauges and cumulative contention counters. +/// Denials count failed capacity attempts, including waiter retries, rather +/// than failed user requests. Waits count submitted budget wait requests. +/// For host queues, waits/wait_ns mean submitted jobs and cumulative queue +/// time, while waiters means runnable queued jobs. Tree queue contention and +/// worker contention counters are not measured by this schema. Operation +/// rows track accepted owners, including callback tails. Source rows report +/// active-source admission, including unapplied writers; their contention +/// counters are not measured. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ResourceUsage { + pub limit: u64, + pub used: u64, + pub reserved: u64, + pub allocated: u64, + pub denied_tree: u64, + pub denied_host: u64, + pub waits: u64, + pub wait_ns: u64, + pub waiters: u64, +} + +impl From<sys::ct_resource_usage_v1> for ResourceUsage { + fn from(value: sys::ct_resource_usage_v1) -> Self { + Self { + limit: value.limit, + used: value.used, + reserved: value.reserved, + allocated: value.allocated, + denied_tree: value.denied_tree, + denied_host: value.denied_host, + waits: value.waits, + wait_ns: value.wait_ns, + waiters: value.waiters, + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ResourceUsageSnapshot { + /// Supported resource indices. Hosts expose shared workers; Trees expose + /// their cache capacities instead. See `TREE_RESOURCE_NAMES` for indices. + /// The Chunk host adds its pin-release service at index 20; the native + /// runtime/Tree v1 snapshots retain their original 20-row ABI. + pub resource_mask: u32, + pub scope_id: u64, + pub resources: [ResourceUsage; TREE_RESOURCE_NAMES.len()], +} + +fn sample( + f: impl FnOnce(*mut sys::ct_resource_usage_snapshot_v1) -> i32, +) -> Result<ResourceUsageSnapshot, CtError> { + let mut raw = sys::ct_resource_usage_snapshot_v1 { + struct_size: std::mem::size_of::<sys::ct_resource_usage_snapshot_v1>(), + version: 1, + ..Default::default() + }; + check(f(&mut raw))?; + if raw.resource_mask & !((1_u32 << raw.resources.len()) - 1) != 0 { + return Err(CtError::Internal); + } + let mut resources = [ResourceUsage::default(); TREE_RESOURCE_NAMES.len()]; + for (target, value) in resources.iter_mut().zip(raw.resources) { + *target = value.into(); + } + Ok(ResourceUsageSnapshot { + resource_mask: raw.resource_mask, + scope_id: raw.scope_id, + resources, + }) +} + +impl super::TreeRuntime { + /// Reads gauges without resetting any periodic sampler or work window. + pub fn resource_usage(&self) -> Result<ResourceUsageSnapshot, CtError> { + sample(|out| unsafe { sys::ct_runtime_resource_usage(self.as_ptr(), out) }) + } +} + +impl Crowdbtree { + /// Reads Tree admission and queue permits without page walks or I/O. + pub fn resource_usage(&self) -> Result<ResourceUsageSnapshot, CtError> { + sample(|out| unsafe { sys::ct_tree_resource_usage(self.as_ptr(), out) }) + } +} diff --git a/lib/crowdb-tree/ffi/src/runtime/usage/window.rs b/lib/crowdb-tree/ffi/src/runtime/usage/window.rs new file mode 100644 index 000000000..a5c78a57e --- /dev/null +++ b/lib/crowdb-tree/ffi/src/runtime/usage/window.rs @@ -0,0 +1,212 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::collections::BTreeMap; +use std::io::{Result, Write}; +use std::time::Instant; + +use super::{ResourceUsageSnapshot, TREE_RESOURCE_NAMES}; + +struct Average { + area: u128, + duration_ns: u128, +} + +impl std::fmt::Display for Average { + fn fmt(&self, writer: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let duration = self.duration_ns.max(1); + let whole = self.area / duration; + // Duration comes from Instant's u64 seconds, so the remainder times + // 1000 fits u128. Preserve fractional worker/permit occupancy. + let fraction = (self.area % duration) * 1000 / duration; + write!(writer, "{whole}.{fraction:03}") + } +} + +#[derive(Default)] +struct Row { + area: [u128; 4], + max: [u64; 4], +} + +struct Scope { + last: ResourceUsageSnapshot, + baseline: ResourceUsageSnapshot, + rows: [Row; TREE_RESOURCE_NAMES.len()], + integrated_at: Instant, + sampled_at: Instant, + duration_ns: u128, + max_gap_ns: u128, + samples: u64, + errors: u64, + seen: u64, + active: bool, +} + +impl Scope { + fn new(snapshot: ResourceUsageSnapshot, now: Instant, generation: u64) -> Self { + let mut result = Self { + last: snapshot, + baseline: snapshot, + rows: std::array::from_fn(|_| Row::default()), + integrated_at: now, + sampled_at: now, + duration_ns: 0, + max_gap_ns: 0, + samples: 1, + errors: 0, + seen: generation, + active: true, + }; + result.peaks(); + result + } + + fn integrate(&mut self, now: Instant) { + if self.active { + let elapsed = now.saturating_duration_since(self.integrated_at).as_nanos(); + self.duration_ns = self.duration_ns.saturating_add(elapsed); + self.max_gap_ns = self + .max_gap_ns + .max(now.saturating_duration_since(self.sampled_at).as_nanos()); + for (index, (row, value)) in self.rows.iter_mut().zip(self.last.resources).enumerate() { + if self.last.resource_mask & (1 << index) == 0 { + continue; + } + for (area, current) in + row.area + .iter_mut() + .zip([value.used, value.reserved, value.allocated, value.waiters]) + { + *area = area.saturating_add(u128::from(current).saturating_mul(elapsed)); + } + } + } + self.integrated_at = now; + } + + fn peaks(&mut self) { + for (index, (row, value)) in self.rows.iter_mut().zip(self.last.resources).enumerate() { + if self.last.resource_mask & (1 << index) == 0 { + continue; + } + for (peak, current) in + row.max + .iter_mut() + .zip([value.used, value.reserved, value.allocated, value.waiters]) + { + *peak = (*peak).max(current); + } + } + } + + fn flush(&mut self, writer: &mut dyn Write, timestamp: &str, name: &str, now: Instant) -> Result<()> { + self.integrate(now); + for (index, resource) in TREE_RESOURCE_NAMES.iter().enumerate() { + if self.last.resource_mask & (1 << index) == 0 { + continue; + } + let value = self.last.resources[index]; + let previous = self.baseline.resources[index]; + let row = &self.rows[index]; + let avg = row.area.map(|area| Average { + area, + duration_ns: self.duration_ns, + }); + let unit = if index < 5 || matches!(index, 10 | 17 | 19) { + "bytes" + } else { + "count" + }; + let contention_measured = index < 6 + || (13..16).contains(&index) + || (self.last.resource_mask & (1 << 8) != 0 && index < 8); + writeln!(writer, + "{timestamp} {name}.resource.{resource} scope_id={} unit={unit} active={} limit={} used={} reserved={} allocated={} avg={} sample_max={} reserved_avg={} reserved_sample_max={} allocated_avg={} allocated_sample_max={} denied_tree={} denied_host={} waits={} wait_ns={} waiters={} waiters_avg={} waiters_sample_max={} samples={} coverage_ns={} max_sample_gap_ns={} sample_age_ns={} sample_errors={} contention_measured={}", + self.last.scope_id, u8::from(self.active), value.limit, value.used, value.reserved, value.allocated, + avg[0], row.max[0], avg[1], row.max[1], avg[2], row.max[2], + value.denied_tree.saturating_sub(previous.denied_tree), value.denied_host.saturating_sub(previous.denied_host), + value.waits.saturating_sub(previous.waits), value.wait_ns.saturating_sub(previous.wait_ns), value.waiters, avg[3], row.max[3], + self.samples, self.duration_ns, self.max_gap_ns, now.saturating_duration_since(self.sampled_at).as_nanos(), self.errors, u8::from(contention_measured))?; + } + self.baseline = self.last; + self.rows = std::array::from_fn(|_| Row::default()); + self.duration_ns = 0; + self.max_gap_ns = 0; + self.samples = 0; + self.errors = 0; + // A held sample can occupy the entire next window without changing. + self.peaks(); + Ok(()) + } +} + +/// Metrics-task-owned windows; application hot paths do not update this state. +/// Averages integrate the last sampled value over its observed duration. +/// Maxima are sampled maxima, not exact allocation peaks. Sampling gaps and +/// errors are exported so delayed or missing observations remain visible. +#[derive(Default)] +pub struct ResourceUsageLog { + scopes: BTreeMap<(String, u64), Scope>, + generation: u64, + failed_samples: BTreeMap<String, u64>, +} + +impl ResourceUsageLog { + /// Begin one complete host/Tree enumeration, then call `end_sample`. + pub fn begin_sample(&mut self) { + self.generation = self.generation.wrapping_add(1); + } + + pub fn observe(&mut self, name: &str, snapshot: ResourceUsageSnapshot, now: Instant) { + let key = (name.to_owned(), snapshot.scope_id); + if let Some(scope) = self.scopes.get_mut(&key) { + scope.integrate(now); + scope.last = snapshot; + scope.sampled_at = now; + scope.seen = self.generation; + scope.active = true; + scope.samples = scope.samples.saturating_add(1); + scope.peaks(); + } else { + self.scopes + .insert(key, Scope::new(snapshot, now, self.generation)); + } + } + + /// A failed sample retains the last observation and reports degraded + /// coverage instead of silently treating resource use as zero. + pub fn sample_failed(&mut self, name: &str) { + let failures = self.failed_samples.entry(name.to_owned()).or_default(); + *failures = failures.saturating_add(1); + for ((scope_name, _), scope) in &mut self.scopes { + if scope_name == name { + scope.errors = scope.errors.saturating_add(1); + scope.seen = self.generation; + } + } + } + + pub fn end_sample(&mut self, now: Instant) { + for scope in self.scopes.values_mut() { + if scope.seen != self.generation && scope.active { + scope.integrate(now); + scope.active = false; + } + } + } + + /// Flush exactly once per log period. Ordinary native snapshot readers + /// neither advance nor reset these windows. + pub fn flush(&mut self, writer: &mut dyn Write, timestamp: &str, now: Instant) -> Result<()> { + for (name, errors) in &self.failed_samples { + writeln!(writer, "{timestamp} {name}.resource.sample_errors count={errors}")?; + } + self.failed_samples.clear(); + for ((name, _), scope) in &mut self.scopes { + scope.flush(writer, timestamp, name, now)?; + } + self.scopes.retain(|_, scope| scope.active); + Ok(()) + } +} diff --git a/lib/crowdb-tree/ffi/src/scan.rs b/lib/crowdb-tree/ffi/src/scan.rs index 063239dad..08c2e874e 100644 --- a/lib/crowdb-tree/ffi/src/scan.rs +++ b/lib/crowdb-tree/ffi/src/scan.rs @@ -1,12 +1,14 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -use bytes::Bytes; -use std::os::raw::c_int; - -use crate::error::{check, take_buf, CtError}; +use crate::error::CtError; use crate::sys; use crate::tree::Crowdbtree; +use bytes::Bytes; + +mod options; +pub(crate) mod owner; +pub use options::ScanOptions; /// A scan result entry. `key` and `value` are zero-copy `Bytes` slices /// into the packed result buffer, not per-entry `Vec<u8>` copies. @@ -35,14 +37,8 @@ pub struct ViewEntry { pub value: Vec<u8>, } -/// Unpack the C++ packed scan result buffer into `ScanEntry`s. -/// `bytes` is converted to a single `Bytes` once (zero-copy move of the -/// `Vec<u8>` allocation via `Bytes::from`), then each entry's `key` and -/// `value` is a `packed.slice(range)` — zero-copy, sharing the same -/// allocation. All slices keep it alive until the last one is dropped. -/// Replaces the prior per-entry `.to_vec()` copies (2N copies to 0). -pub(crate) fn decode_scan(bytes: Vec<u8>, count: usize) -> Result<Vec<ScanEntry>, CtError> { - let packed = Bytes::from(bytes); +/// Decode entries as slices retaining the native packed allocation owner. +pub(crate) fn decode_scan_bytes(packed: Bytes, count: usize) -> Result<Vec<ScanEntry>, CtError> { let bytes: &[u8] = packed.as_ref(); let mut out = Vec::with_capacity(count); let mut pos = 0usize; @@ -92,41 +88,11 @@ impl Crowdbtree { start_inclusive: bool, begin_key: &[u8], ) -> Result<Option<ScanEntry>, CtError> { - let mut found: c_int = 0; - let mut key = sys::ct_buf { - data: std::ptr::null_mut(), - len: 0, - }; - let mut value = sys::ct_buf { - data: std::ptr::null_mut(), - len: 0, - }; - let mut slot = 0_u64; - check(unsafe { - sys::ct_seek_reverse( - self.as_ptr(), - start_key.as_ptr(), - start_key.len(), - if start_inclusive { 1 } else { 0 }, - begin_key.as_ptr(), - begin_key.len(), - &mut found, - &mut key, - &mut slot, - &mut value, - ) - })?; - let key = take_buf(key); - let value = take_buf(value); - Ok((found != 0).then(|| ScanEntry { - key: Bytes::from(key), - slot, - value: Bytes::from(value), - tombstone: false, - })) + let (entries, _) = self.scan_reverse(Some(start_key), start_inclusive, begin_key, 1, 0)?; + Ok(entries.into_iter().next()) } - /// Returns a bounded page in descending key order. + /// Scans in descending key order under the supplied count and byte limits. pub fn scan_reverse( &self, start_key: Option<&[u8]>, @@ -135,29 +101,30 @@ impl Crowdbtree { limit: usize, byte_budget: usize, ) -> Result<(Vec<ScanEntry>, bool), CtError> { - let mut entries = sys::ct_buf { - data: std::ptr::null_mut(), - len: 0, - }; - let mut count = 0_u64; - let mut truncated: c_int = 0; - check(unsafe { - sys::ct_scan_reverse( - self.as_ptr(), - start_key.unwrap_or_default().as_ptr(), - start_key.map_or(0, <[u8]>::len), - if start_key.is_some() { 1 } else { 0 }, - if start_inclusive { 1 } else { 0 }, - begin_key.as_ptr(), - begin_key.len(), + let start = start_key.unwrap_or_default(); + owner::scan( + self.as_ptr(), + &sys::ct_scan_options_v2 { + struct_size: std::mem::size_of::<sys::ct_scan_options_v2>(), + prefix: std::ptr::null(), + prefix_len: 0, + start: start.as_ptr(), + start_len: start.len(), + end: std::ptr::null(), + end_len: 0, + begin: begin_key.as_ptr(), + begin_len: begin_key.len(), limit, byte_budget, - &mut entries, - &mut count, - &mut truncated, - ) - })?; - Ok((decode_scan(take_buf(entries), count as usize)?, truncated != 0)) + deadline_ms: 0, + reverse: 1, + keys_only: 0, + include_tombstones: 0, + has_start_bound: i32::from(start_key.is_some()), + start_inclusive: i32::from(start_inclusive), + strict_byte_budget: 0, + }, + ) } /// Range scan over `prefix` (empty = whole keyspace). @@ -168,6 +135,9 @@ impl Crowdbtree { /// strictly less than `end_key` are returned. When `keys_only` is true, /// values are not materialized (no overflow-chain assembly): entries /// carry empty values and the byte budget accounts for key bytes only. + /// A zero `limit` removes the total count limit across internal work windows. + /// Results still obey the byte/deadline limits and allocation budgets. Resume + /// truncated output from the last returned key; use `scan_from` for an empty key. #[allow(clippy::too_many_arguments)] pub fn scan( &self, @@ -273,41 +243,28 @@ impl Crowdbtree { include_tombstones: bool, strict_byte_budget: bool, ) -> Result<(Vec<ScanEntry>, bool), CtError> { - let mut buf = sys::ct_buf { - data: std::ptr::null_mut(), - len: 0, - }; - let mut count = 0u64; - let mut truncated: c_int = 0; - let scan = if strict_byte_budget { - sys::ct_scan_strict - } else { - sys::ct_scan - }; - let status = unsafe { - scan( - self.as_ptr(), - prefix.as_ptr(), - prefix.len(), - start_key.as_ptr(), - start_key.len(), - if has_start_bound { 1 } else { 0 }, - if start_inclusive { 1 } else { 0 }, - end_key.as_ptr(), - end_key.len(), + owner::scan( + self.as_ptr(), + &sys::ct_scan_options_v2 { + struct_size: std::mem::size_of::<sys::ct_scan_options_v2>(), + prefix: prefix.as_ptr(), + prefix_len: prefix.len(), + start: start_key.as_ptr(), + start_len: start_key.len(), + end: end_key.as_ptr(), + end_len: end_key.len(), + begin: std::ptr::null(), + begin_len: 0, limit, byte_budget, - if keys_only { 1 } else { 0 }, deadline_ms, - if include_tombstones { 1 } else { 0 }, - &mut buf, - &mut count, - &mut truncated, - ) - }; - check(status)?; - let bytes = take_buf(buf); - let entries = decode_scan(bytes, count as usize)?; - Ok((entries, truncated != 0)) + reverse: 0, + keys_only: i32::from(keys_only), + include_tombstones: i32::from(include_tombstones), + has_start_bound: i32::from(has_start_bound), + start_inclusive: i32::from(start_inclusive), + strict_byte_budget: i32::from(strict_byte_budget), + }, + ) } } diff --git a/lib/crowdb-tree/ffi/src/scan/options.rs b/lib/crowdb-tree/ffi/src/scan/options.rs new file mode 100644 index 000000000..9a45bb389 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/scan/options.rs @@ -0,0 +1,30 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::ScanDirection; + +/// A scan request with an explicit optional cursor and bounded internal work windows. +/// `None` starts without a cursor; `Some(Vec::new())` bounds the empty key. +/// Resume truncated output with the last returned key and `start_inclusive=false`. +#[derive(Debug, Clone, Default)] +pub struct ScanOptions { + pub prefix: Vec<u8>, + pub start: Option<Vec<u8>>, + pub start_inclusive: bool, + /// Exclusive forward upper bound; empty means unbounded. + pub end: Vec<u8>, + /// Inclusive reverse lower bound; empty includes the empty key. + pub begin: Vec<u8>, + pub direction: ScanDirection, + /// Total result count limit; zero means no count limit across internal windows. + /// Result allocation remains subject to the Tree and host resource budgets. + pub limit: usize, + /// Total key/value byte budget; zero disables this result byte limit. + pub byte_budget: usize, + pub keys_only: bool, + pub include_tombstones: bool, + /// Absolute Unix time in milliseconds; zero disables the deadline. + pub deadline_ms: u64, + /// Stops before an oversized first record, returning empty, truncated output. + pub strict_byte_budget: bool, +} diff --git a/lib/crowdb-tree/ffi/src/scan/owner.rs b/lib/crowdb-tree/ffi/src/scan/owner.rs new file mode 100644 index 000000000..17a25c5b3 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/scan/owner.rs @@ -0,0 +1,120 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; + +use crate::error::{check, CtError}; +use crate::reactor::{FutureGuard, RawOutcome}; +use crate::sys; + +struct ScanOwner { + handle: *mut sys::ct_scan_result, + data: *const u8, + len: usize, +} + +// SAFETY: the owner exposes immutable bytes and is destroyed once after all +// Bytes clones/slices drop; native allocation accounting is thread-safe. +unsafe impl Send for ScanOwner {} + +impl AsRef<[u8]> for ScanOwner { + fn as_ref(&self) -> &[u8] { + if self.len == 0 { + &[] + } else { + // SAFETY: ct_scan_result_data ties these bytes to the live handle. + unsafe { std::slice::from_raw_parts(self.data, self.len) } + } + } +} + +impl Drop for ScanOwner { + fn drop(&mut self) { + unsafe { sys::ct_scan_result_free(self.handle) }; + } +} + +pub(crate) fn take(handle: *mut sys::ct_scan_result) -> Result<(Bytes, usize, bool), CtError> { + if handle.is_null() { + return Err(CtError::Internal); + } + let mut owner = ScanOwner { + handle, + data: std::ptr::null(), + len: 0, + }; + let mut count = 0; + let mut truncated = 0; + check(unsafe { + sys::ct_scan_result_data( + handle, + &mut owner.data, + &mut owner.len, + &mut count, + &mut truncated, + ) + })?; + let count = usize::try_from(count).map_err(|_| CtError::Corruption)?; + let budget = crate::owner_budget::OwnerBudget::scan(handle)?; + let bytes = budget.wrap(owner)?; + Ok((bytes, count, truncated != 0)) +} + +pub(crate) fn poll(guard: &mut FutureGuard) -> Option<Result<RawOutcome, CtError>> { + let mut done = 0; + let mut result = std::ptr::null_mut(); + let status = unsafe { sys::ct_future_poll_scan_v2(guard.0, &mut done, &mut result) }; + if done == 0 { + return None; + } + guard.0 = std::ptr::null_mut(); + Some(check(status).and_then(|()| { + let (packed, count, truncated) = take(result)?; + Ok(RawOutcome { + found: truncated, + slot: count as u64, + value: Vec::new(), + packed: Some(packed), + }) + })) +} + +pub(crate) fn scan( + tree: *mut sys::ct_tree, + options: &sys::ct_scan_options_v2, +) -> Result<(Vec<super::ScanEntry>, bool), CtError> { + let mut result = std::ptr::null_mut(); + check(unsafe { sys::ct_scan_v2(tree, options, &mut result) })?; + let (packed, count, truncated) = take(result)?; + Ok((super::decode_scan_bytes(packed, count)?, truncated)) +} + +pub(crate) fn start(tree: *mut sys::ct_tree, options: &super::ScanOptions) -> Result<FutureGuard, CtError> { + let start = options.start.as_deref().unwrap_or_default(); + let raw = sys::ct_scan_options_v2 { + struct_size: std::mem::size_of::<sys::ct_scan_options_v2>(), + prefix: options.prefix.as_ptr(), + prefix_len: options.prefix.len(), + start: start.as_ptr(), + start_len: start.len(), + end: options.end.as_ptr(), + end_len: options.end.len(), + begin: options.begin.as_ptr(), + begin_len: options.begin.len(), + limit: options.limit, + byte_budget: options.byte_budget, + deadline_ms: options.deadline_ms, + reverse: i32::from(options.direction == super::ScanDirection::Reverse), + keys_only: i32::from(options.keys_only), + include_tombstones: i32::from(options.include_tombstones), + has_start_bound: i32::from(options.start.is_some()), + start_inclusive: i32::from(options.start_inclusive), + strict_byte_budget: i32::from(options.strict_byte_budget), + }; + let mut future = std::ptr::null_mut(); + check(unsafe { sys::ct_scan_async_v2(tree, &raw, &mut future) })?; + if future.is_null() { + return Err(CtError::Internal); + } + Ok(FutureGuard(future)) +} diff --git a/lib/crowdb-tree/ffi/src/snapshot.rs b/lib/crowdb-tree/ffi/src/snapshot.rs index 6f812e282..b24b73acd 100644 --- a/lib/crowdb-tree/ffi/src/snapshot.rs +++ b/lib/crowdb-tree/ffi/src/snapshot.rs @@ -10,6 +10,9 @@ use crate::scan::ViewEntry; use crate::sys; use crate::tree::Crowdbtree; +mod owner; +pub use owner::OwnedSnapshotChunk; + /// Immutable metadata captured when a portable snapshot export begins. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SnapshotMetadata { @@ -49,6 +52,12 @@ impl SnapshotExportSession { unsafe { sys::ct_snapshot_export_offset(self.ptr.as_ptr()) } } + /// Retains the native chunk allocation through every returned Bytes clone + /// and slice, even after this session and its Tree are closed. + pub fn read_owned(&mut self, offset: u64) -> Result<OwnedSnapshotChunk, CtError> { + owner::read(self.ptr.as_ptr(), offset) + } + /// Read the next sequential chunk. A source registry may cache this result /// to serve an exact retry without advancing the C encoder twice. pub fn read(&mut self, offset: u64) -> Result<SnapshotChunk, CtError> { diff --git a/lib/crowdb-tree/ffi/src/snapshot/owner.rs b/lib/crowdb-tree/ffi/src/snapshot/owner.rs new file mode 100644 index 000000000..58a3c47b6 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/snapshot/owner.rs @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use bytes::Bytes; + +use crate::error::{check, CtError}; +use crate::sys; + +/// A native chunk whose allocation remains charged through every byte slice. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OwnedSnapshotChunk { + pub offset: u64, + pub bytes: Bytes, + pub done: bool, +} + +struct ChunkOwner { + handle: *mut sys::ct_snapshot_chunk, + data: *const u8, + len: usize, +} + +// SAFETY: native chunks expose immutable bytes and retain an independent +// allocation owner. Bytes invokes this owner's destructor exactly once. +unsafe impl Send for ChunkOwner {} + +impl AsRef<[u8]> for ChunkOwner { + fn as_ref(&self) -> &[u8] { + if self.len == 0 { + &[] + } else { + // SAFETY: the native handle retains this immutable byte range. + unsafe { std::slice::from_raw_parts(self.data, self.len) } + } + } +} + +impl Drop for ChunkOwner { + fn drop(&mut self) { + unsafe { sys::ct_snapshot_chunk_free(self.handle) }; + } +} + +pub(super) fn read(export: *mut sys::ct_export, offset: u64) -> Result<OwnedSnapshotChunk, CtError> { + let mut handle = std::ptr::null_mut(); + let mut done = 0; + check(unsafe { sys::ct_snapshot_export_next_v2(export, offset, &mut handle, &mut done) })?; + if handle.is_null() { + return Err(CtError::Internal); + } + let mut owner = ChunkOwner { + handle, + data: std::ptr::null(), + len: 0, + }; + check(unsafe { sys::ct_snapshot_chunk_data(handle, &mut owner.data, &mut owner.len) })?; + let budget = crate::owner_budget::OwnerBudget::snapshot(handle)?; + Ok(OwnedSnapshotChunk { + offset, + bytes: budget.wrap(owner)?, + done: done != 0, + }) +} diff --git a/lib/crowdb-tree/ffi/src/stats.rs b/lib/crowdb-tree/ffi/src/stats.rs index 13fe8a8e8..d17ccf70b 100644 --- a/lib/crowdb-tree/ffi/src/stats.rs +++ b/lib/crowdb-tree/ffi/src/stats.rs @@ -6,9 +6,20 @@ use crate::sys; use crate::tree::Crowdbtree; use crate::CtError; -/// Result of a cadence-driven [`Crowdbtree::compact_sparse_blocks`] pass -/// (R129). Snapshot folding drops eligible tombstones during every snapshot; -/// this struct reports the block-level relocation and deletion outcome. +mod backing; +mod state; +pub use backing::BackingStatsSnapshot; +mod work; +pub use state::{TreeLogicalStats, TreeProgressSnapshot, TreeResourceStats, TreeStatsSnapshot}; +pub use work::{ + TreeFlushDispatchStats, TreeHistogramStats, TreeWaitSnapshot, TreeWaitStats, TreeWorkCollector, + TreeWorkCounters, TreeWorkHistograms, TreeWorkSnapshot, TreeWorkStage, TreeWorkStageAge, + TreeWorkStageStats, TREE_WAIT_CLASS_NAMES, TREE_WORK_CAUSE_NAMES, TREE_WORK_COUNTER_NAMES, + TREE_WORK_STAGE_NAMES, +}; + +/// Block relocation and deletion performed by one +/// [`Crowdbtree::compact_sparse_blocks`] pass. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct MergeGcStats { pub blocks_selected: u64, @@ -42,9 +53,9 @@ pub struct TreeSummary { pub exact: bool, } -/// Point-in-time diagnostics snapshot; see [`Crowdbtree::stats`]. Every field -/// is O(1) on the C++ side (an already-tracked atomic counter or -/// `BufferPool::stats`), so this is safe to poll periodically. +/// Compatibility diagnostics snapshot; see [`Crowdbtree::stats`]. Cache fields +/// retain their historical names. Use [`Crowdbtree::state_stats`] for the +/// publication statistics and resource gauges. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct Stats { pub last_applied_slot: u64, @@ -102,7 +113,7 @@ impl Crowdbtree { } } - /// Cadence-driven block compaction (R129). Selects sparse source blocks, + /// Selects sparse source blocks, /// relocates their live extents through a snapshot, and deletes blocks /// unreachable from any retained anchor. Non-block stores and disabled /// configurations return an empty stats result with no snapshot write. @@ -117,8 +128,9 @@ impl Crowdbtree { }) } - /// Batched diagnostics snapshot. O(1) -- safe to poll periodically for - /// metrics/console display. + /// Samples Core publication and cache counters without scanning pool frames. + /// Compatibility-only fields without a current counterpart remain zero; + /// use `state_stats` and work metrics for the full runtime statistics. pub fn stats(&self) -> Stats { let mut raw = sys::ct_stats::default(); unsafe { sys::ct_get_stats(self.as_ptr(), &mut raw) }; diff --git a/lib/crowdb-tree/ffi/src/stats/backing.rs b/lib/crowdb-tree/ffi/src/stats/backing.rs new file mode 100644 index 000000000..4af62b168 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/stats/backing.rs @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crate::error::check; +use crate::{sys, Crowdbtree, CtError}; + +/// Constant-cost backend-ledger gauges, independent of a writer incarnation. +/// Equal nonzero scopes are projections of the same ledger; deduplicate by +/// `scope_id` rather than summing those views. Allocated capacity includes old +/// and staged output until physical deletion is acknowledged. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct BackingStatsSnapshot { + pub scope_id: u64, + pub allocated_bytes: u64, +} + +impl Crowdbtree { + /// Samples a stable backing ledger without scanning pages or performing I/O. + /// + /// # Errors + /// Returns `Unavailable` when this backend has no independently owned ledger. + pub fn backing_stats(&self) -> Result<BackingStatsSnapshot, CtError> { + sample(|out| unsafe { sys::ct_get_backing_stats(self.as_ptr(), out) }) + } +} + +pub(super) fn sample( + get: impl FnOnce(*mut sys::ct_backing_stats_v1) -> i32, +) -> Result<BackingStatsSnapshot, CtError> { + let mut raw = sys::ct_backing_stats_v1 { + struct_size: u32::try_from(std::mem::size_of::<sys::ct_backing_stats_v1>()) + .map_err(|_| CtError::Internal)?, + version: 1, + ..Default::default() + }; + check(get(&mut raw))?; + Ok(BackingStatsSnapshot { + scope_id: raw.scope_id, + allocated_bytes: raw.allocated_bytes, + }) +} diff --git a/lib/crowdb-tree/ffi/src/stats/state.rs b/lib/crowdb-tree/ffi/src/stats/state.rs new file mode 100644 index 000000000..346548f97 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/stats/state.rs @@ -0,0 +1,163 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crate::error::{check, CtError}; +use crate::{sys, Crowdbtree}; + +/// Exact and unknown logical subtotals from one immutable publication. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeLogicalStats { + pub leaf_pages: u64, + pub inner_pages: u64, + pub height: u64, + pub dirty_pages: u64, + pub overlay_pages: u64, + pub overlay_runs: u64, + pub overlay_entries: u64, + pub page_size_known_bytes: u64, + pub page_size_unknown_pages: u64, + pub page_size_estimate_bytes: u64, + pub page_size_max_observed_bytes: u64, + pub kv_known_count: u64, + pub kv_known_bytes: u64, + pub kv_unknown_pages: u64, + pub consolidate_candidate_pages: u64, + pub split_candidate_pages: u64, + pub split_ready_pages: u64, + pub merge_candidate_pages: u64, + pub merge_candidate_pairs: u64, + pub merge_ready_pairs: u64, + pub writeback_candidate_pages: u64, + pub evict_candidate_pages: u64, + pub checkpoint_pending_pages: u64, + pub checkpoint_mapping_pending: bool, +} + +/// Approximate allocation/reference gauges. Referenced subsets overlap. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeResourceStats { + pub allocated_bytes: u64, + pub reserved_bytes: u64, + pub payload_bytes: u64, + pub frame_bytes: u64, + pub metadata_bytes: u64, + pub io_bytes: u64, + pub scratch_bytes: u64, + pub retired_bytes: u64, + pub resident_frames: u64, + pub cache_bytes: u64, + pub cache_pinned_bytes: u64, + pub proxy_pages: u64, + pub read_inflight_bytes: u64, + pub write_inflight_bytes: u64, + pub materialize_inflight_bytes: u64, + pub allocated_backing_bytes: u64, + pub maintenance_queued: u64, + pub maintenance_inflight: u64, +} + +/// Constant-cost immutable logical snapshot plus independent resource samples. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeStatsSnapshot { + pub instance_generation: u64, + pub commit_seq: u64, + pub sampled_at_ns: u64, + pub logical: TreeLogicalStats, + pub resources: TreeResourceStats, +} + +impl Crowdbtree { + /// Samples state in O(number of metrics), without loading or walking pages. + /// Unknown summaries stay explicit. + pub fn state_stats(&self) -> Result<TreeStatsSnapshot, CtError> { + let mut raw = sys::ct_tree_stats_snapshot_v1 { + struct_size: u32::try_from(std::mem::size_of::<sys::ct_tree_stats_snapshot_v1>()) + .map_err(|_| CtError::Internal)?, + version: 1, + ..Default::default() + }; + check(unsafe { sys::ct_get_tree_state_stats(self.as_ptr(), &mut raw) })?; + Ok(TreeStatsSnapshot { + instance_generation: raw.instance_generation, + commit_seq: raw.commit_seq, + sampled_at_ns: raw.sampled_at_ns, + logical: TreeLogicalStats { + leaf_pages: raw.logical.leaf_pages, + inner_pages: raw.logical.inner_pages, + height: raw.logical.height, + dirty_pages: raw.logical.dirty_pages, + overlay_pages: raw.logical.overlay_pages, + overlay_runs: raw.logical.overlay_runs, + overlay_entries: raw.logical.overlay_entries, + page_size_known_bytes: raw.logical.page_size_known_bytes, + page_size_unknown_pages: raw.logical.page_size_unknown_pages, + page_size_estimate_bytes: raw.logical.page_size_estimate_bytes, + page_size_max_observed_bytes: raw.logical.page_size_max_observed_bytes, + kv_known_count: raw.logical.kv_known_count, + kv_known_bytes: raw.logical.kv_known_bytes, + kv_unknown_pages: raw.logical.kv_unknown_pages, + consolidate_candidate_pages: raw.logical.consolidate_candidate_pages, + split_candidate_pages: raw.logical.split_candidate_pages, + split_ready_pages: raw.logical.split_ready_pages, + merge_candidate_pages: raw.logical.merge_candidate_pages, + merge_candidate_pairs: raw.logical.merge_candidate_pairs, + merge_ready_pairs: raw.logical.merge_ready_pairs, + writeback_candidate_pages: raw.logical.writeback_candidate_pages, + evict_candidate_pages: raw.logical.evict_candidate_pages, + checkpoint_pending_pages: raw.logical.checkpoint_pending_pages, + checkpoint_mapping_pending: raw.logical.checkpoint_mapping_pending != 0, + }, + resources: TreeResourceStats { + allocated_bytes: raw.resources.allocated_bytes, + reserved_bytes: raw.resources.reserved_bytes, + payload_bytes: raw.resources.payload_bytes, + frame_bytes: raw.resources.frame_bytes, + metadata_bytes: raw.resources.metadata_bytes, + io_bytes: raw.resources.io_bytes, + scratch_bytes: raw.resources.scratch_bytes, + retired_bytes: raw.resources.retired_bytes, + resident_frames: raw.resources.resident_frames, + cache_bytes: raw.resources.cache_bytes, + cache_pinned_bytes: raw.resources.cache_pinned_bytes, + proxy_pages: raw.resources.proxy_pages, + read_inflight_bytes: raw.resources.read_inflight_bytes, + write_inflight_bytes: raw.resources.write_inflight_bytes, + materialize_inflight_bytes: raw.resources.materialize_inflight_bytes, + allocated_backing_bytes: raw.resources.allocated_backing_bytes, + maintenance_queued: raw.resources.maintenance_queued, + maintenance_inflight: raw.resources.maintenance_inflight, + }, + }) + } +} + +/// Incrementally maintained backlog ages at one immutable publication. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct TreeProgressSnapshot { + pub instance_generation: u64, + pub commit_seq: u64, + pub sampled_at_ns: u64, + pub candidate_oldest_age_ns: u64, + pub checkpoint_oldest_pending_age_ns: u64, + pub maintenance_oldest_age_ns: u64, +} + +impl Crowdbtree { + /// Reads maintained timestamps without loading or walking pages. + pub fn progress_stats(&self) -> Result<TreeProgressSnapshot, CtError> { + let mut raw = sys::ct_tree_progress_snapshot_v1 { + struct_size: std::mem::size_of::<sys::ct_tree_progress_snapshot_v1>() as u32, + version: 1, + ..Default::default() + }; + check(unsafe { sys::ct_get_tree_progress_stats(self.as_ptr(), &mut raw) })?; + Ok(TreeProgressSnapshot { + instance_generation: raw.instance_generation, + commit_seq: raw.commit_seq, + sampled_at_ns: raw.sampled_at_ns, + candidate_oldest_age_ns: raw.candidate_oldest_age_ns, + checkpoint_oldest_pending_age_ns: raw.checkpoint_oldest_pending_age_ns, + maintenance_oldest_age_ns: raw.maintenance_oldest_age_ns, + }) + } +} diff --git a/lib/crowdb-tree/ffi/src/stats/work.rs b/lib/crowdb-tree/ffi/src/stats/work.rs new file mode 100644 index 000000000..b6501d732 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/stats/work.rs @@ -0,0 +1,261 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +mod waits; +pub use waits::{TreeWaitSnapshot, TreeWaitStats, TREE_WAIT_CLASS_NAMES}; +mod counters; +mod histograms; +use counters::raw_counters; +pub use counters::{TreeWorkCounters, TREE_WORK_CAUSE_NAMES, TREE_WORK_COUNTER_NAMES}; +use histograms::raw_histograms; +pub use histograms::{TreeHistogramStats, TreeWorkHistograms, TreeWorkStage}; + +use std::ptr::NonNull; + +use crate::error::{check, CtError}; +use crate::{sys, Crowdbtree}; + +/// Fixed v1 stage order; stages are shared across every window and totals sample. +pub const TREE_WORK_STAGE_NAMES: [&str; 21] = [ + "flush.batch", + "flush.admission_wait", + "flush.capture", + "flush.writer_drain", + "flush.handoff", + "flush.range", + "publish.queue_wait", + "publish.commit", + "maintenance.plan", + "overlay.coalesce", + "page.split", + "page.merge", + "page.layout", + "page.encode", + "budget.wait", + "page.load_pipeline", + "page.writeback_pipeline", + "snapshot.capture", + "snapshot.commit", + "memory.retire_wait", + "flush.repair", +]; + +/// Relaxed totals and one approximate collector window for a pipeline stage. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeWorkStageStats { + pub attempts: u64, + pub inflight: u64, + pub total_count: u64, + pub total_sum_ns: u64, + pub total_max_ns: u64, + /// Success, error, cancelled, retry, stale, in that order. + pub outcomes: [u64; 5], + pub window_count: u64, + pub window_sum_ns: u64, + pub window_max_ns: u64, + /// Independent non-draining age sample; see `age_sampled_at_ns`. + pub age: TreeWorkStageAge, +} + +/// Approximate fixed-capacity observation of active work. Saturation never +/// rejects work. Only a complete sample covers every active operation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeWorkStageAge { + pub inflight: u64, + pub tracked: u64, + pub untracked: u64, + /// Oldest age in the observed subset, zero when none was observed. + pub observed_oldest_age_ns: u64, + pub complete: bool, +} + +/// Immutable value shared by all consumers of one collector tick. +/// A totals-only sample has sequence zero and empty window fields. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeWorkSnapshot { + pub window_sequence: u64, + pub sampled_at_ns: u64, + /// The age sample and cumulative/window counters are not transactional. + pub age_sampled_at_ns: u64, + pub stages: [TreeWorkStageStats; 21], + pub counters: TreeWorkCounters, + pub histograms: TreeWorkHistograms, + pub waits: TreeWaitSnapshot, +} + +/// Cumulative flush dispatch counters, independent of stage collector windows. +/// Fields are relaxed samples. `active_steps` is current concurrency and +/// `peak_steps` is its lifetime high-water mark; neither counts waiting jobs. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeFlushDispatchStats { + pub sampled_at_ns: u64, + pub inline_probes: u64, + pub inline_ranges: u64, + pub queued_steps: u64, + pub prepared_runs: u64, + pub prepared_entries: u64, + pub prepared_bytes: u64, + pub cow_bytes: u64, + pub active_steps: u64, + pub peak_steps: u64, +} + +impl From<sys::ct_tree_work_snapshot_v1> for TreeWorkSnapshot { + fn from(raw: sys::ct_tree_work_snapshot_v1) -> Self { + Self { + window_sequence: raw.window_sequence, + sampled_at_ns: raw.sampled_at_ns, + age_sampled_at_ns: 0, + counters: TreeWorkCounters::default(), + histograms: TreeWorkHistograms::default(), + waits: TreeWaitSnapshot::default(), + stages: raw.stages.map(|stage| TreeWorkStageStats { + attempts: stage.attempts, + inflight: stage.inflight, + total_count: stage.total_count, + total_sum_ns: stage.total_sum_ns, + total_max_ns: stage.total_max_ns, + outcomes: stage.outcomes, + window_count: stage.window_count, + window_sum_ns: stage.window_sum_ns, + window_max_ns: stage.window_max_ns, + age: TreeWorkStageAge::default(), + }), + } + } +} + +impl TreeWorkSnapshot { + fn with_ages(mut self, raw: sys::ct_tree_work_ages_v1) -> Self { + self.age_sampled_at_ns = raw.sampled_at_ns; + for (stage, age) in self.stages.iter_mut().zip(raw.stages) { + stage.age = TreeWorkStageAge { + inflight: age.inflight, + tracked: age.tracked, + untracked: age.untracked, + observed_oldest_age_ns: age.observed_oldest_age_ns, + complete: age.complete == 1, + }; + } + self + } +} + +fn raw_ages() -> Result<sys::ct_tree_work_ages_v1, CtError> { + Ok(sys::ct_tree_work_ages_v1 { + struct_size: u32::try_from(std::mem::size_of::<sys::ct_tree_work_ages_v1>()) + .map_err(|_| CtError::Internal)?, + version: 1, + ..Default::default() + }) +} + +fn raw_snapshot() -> Result<sys::ct_tree_work_snapshot_v1, CtError> { + Ok(sys::ct_tree_work_snapshot_v1 { + struct_size: u32::try_from(std::mem::size_of::<sys::ct_tree_work_snapshot_v1>()) + .map_err(|_| CtError::Internal)?, + version: 1, + ..Default::default() + }) +} + +/// Exclusive draining owner. The ledger remains valid after the Tree closes; +/// observations retained by accepted work are released with their actual owners. +#[derive(Debug)] +pub struct TreeWorkCollector { + handle: NonNull<sys::ct_work_metrics_collector>, +} + +// SAFETY: native capture uses a nonblocking atomic claim; ledger counters are +// atomic and neither capture nor destruction accesses a raw Tree pointer. +unsafe impl Send for TreeWorkCollector {} +unsafe impl Sync for TreeWorkCollector {} + +impl TreeWorkCollector { + /// Samples backing counters after Tree close without retaining storage owners. + pub fn backing_stats(&self) -> Result<super::BackingStatsSnapshot, CtError> { + super::backing::sample(|out| unsafe { + sys::ct_work_metrics_collector_backing_stats(self.handle.as_ptr(), out) + }) + } + + /// Drains one window. Concurrent captures return `Unavailable` without + /// resetting any samples; callers distribute this immutable result. + pub fn capture(&self) -> Result<TreeWorkSnapshot, CtError> { + let mut raw = raw_snapshot()?; + let mut ages = raw_ages()?; + let mut counters = raw_counters()?; + let mut histograms = raw_histograms()?; + check(unsafe { + sys::ct_work_metrics_capture_v3( + self.handle.as_ptr(), + &mut raw, + &mut ages, + &mut counters, + &mut histograms, + ) + })?; + let mut snapshot = TreeWorkSnapshot::from(raw) + .with_ages(ages) + .with_counters(counters); + snapshot.histograms = histograms.into(); + snapshot.waits = self.wait_metrics_totals()?; + Ok(snapshot) + } +} + +impl Drop for TreeWorkCollector { + fn drop(&mut self) { + unsafe { sys::ct_work_metrics_collector_free(self.handle.as_ptr()) }; + } +} + +impl Crowdbtree { + /// Samples dispatch counters without taking a collector claim or draining it. + pub fn flush_dispatch_stats(&self) -> Result<TreeFlushDispatchStats, CtError> { + let mut raw = sys::ct_tree_flush_dispatch_v1 { + struct_size: u32::try_from(std::mem::size_of::<sys::ct_tree_flush_dispatch_v1>()) + .map_err(|_| CtError::Internal)?, + version: 1, + ..Default::default() + }; + check(unsafe { sys::ct_flush_dispatch_totals(self.as_ptr(), &mut raw) })?; + Ok(TreeFlushDispatchStats { + sampled_at_ns: raw.sampled_at_ns, + inline_probes: raw.inline_probes, + inline_ranges: raw.inline_ranges, + queued_steps: raw.queued_steps, + prepared_runs: raw.prepared_runs, + prepared_entries: raw.prepared_entries, + prepared_bytes: raw.prepared_bytes, + cow_bytes: raw.cow_bytes, + active_steps: raw.active_steps, + peak_steps: raw.peak_steps, + }) + } + + /// Claims the only window collector; another claim returns `Unavailable`. + pub fn work_metrics_collector(&self) -> Result<TreeWorkCollector, CtError> { + let mut handle = std::ptr::null_mut(); + check(unsafe { sys::ct_work_metrics_collector_create(self.as_ptr(), &mut handle) })?; + Ok(TreeWorkCollector { + handle: NonNull::new(handle).ok_or(CtError::Internal)?, + }) + } + + /// Samples cumulative counters without resetting the collector window. + pub fn work_metrics_totals(&self) -> Result<TreeWorkSnapshot, CtError> { + let mut raw = raw_snapshot()?; + let mut ages = raw_ages()?; + check(unsafe { sys::ct_work_metrics_ages(self.as_ptr(), &mut ages) })?; + check(unsafe { sys::ct_work_metrics_totals(self.as_ptr(), &mut raw) })?; + let mut counters = raw_counters()?; + check(unsafe { sys::ct_work_counters_totals(self.as_ptr(), &mut counters) })?; + let mut snapshot = TreeWorkSnapshot::from(raw) + .with_ages(ages) + .with_counters(counters); + snapshot.histograms = self.work_histogram_totals()?; + snapshot.waits = self.wait_metrics_totals()?; + Ok(snapshot) + } +} diff --git a/lib/crowdb-tree/ffi/src/stats/work/counters.rs b/lib/crowdb-tree/ffi/src/stats/work/counters.rs new file mode 100644 index 000000000..e15fac656 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/stats/work/counters.rs @@ -0,0 +1,106 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::TreeWorkSnapshot; +use crate::{error::CtError, sys}; + +/// Fixed cause and quantity order shared by native, C and immutable host samples. +pub const TREE_WORK_CAUSE_NAMES: [&str; 8] = [ + "other", + "flush", + "cache_pressure", + "checkpoint", + "gc", + "read", + "snapshot", + "allocation", +]; +pub const TREE_WORK_COUNTER_NAMES: [&str; 55] = [ + "flush.input.tables", + "flush.range.jobs", + "flush.candidate.versions", + "flush.winning.versions", + "flush.winning.key.bytes", + "flush.winning.value.bytes", + "flush.handoff.bytes", + "source.detached", + "candidate.checks", + "candidate.skips", + "publication.mismatches", + "split.operations", + "split.output.pages", + "merge.operations", + "merge.input.pages", + "merge.output.pages", + "coalesce.operations", + "coalesce.input.runs", + "coalesce.output.runs", + "coalesce.input.entries", + "coalesce.output.entries", + "payload.references.dropped", + "memory.reclaimed.bytes", + "key.copy.bytes", + "handoff.value.copy.bytes", + "cow.metadata.copy.bytes", + "materialize.copy.bytes", + "backend.bounce.copy.bytes", + "encode.frame.bytes", + "encode.primary.frames", + "encode.overflow.frames", + "encode.key.copy.bytes", + "encode.value.copy.bytes", + "read.operations", + "read.bytes", + "read.errors", + "write.operations", + "write.bytes", + "write.errors", + "reused.frames", + "reused.backing.bytes", + "proxy.avoided.loads", + "cache.hits", + "cache.misses", + "cache.joined.loads", + "cache.evictions", + "callbacks.stale", + "rebase.mismatches", + "admission.rejected", + "admission.waits", + "snapshot.retirement.delays", + "backing.deleted.bytes", + "checkpoint.failures", + "retries", + "memory.logical.retirement.bytes", +]; +/// Completed physical quantities and committed logical outcomes. Values are +/// cumulative relaxed samples, not transactionally consistent with stage ages. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TreeWorkCounters { + pub sampled_at_ns: u64, + pub values: [[u64; 55]; 8], +} +impl Default for TreeWorkCounters { + fn default() -> Self { + Self { + sampled_at_ns: 0, + values: [[0; 55]; 8], + } + } +} +impl TreeWorkSnapshot { + pub(super) fn with_counters(mut self, raw: sys::ct_tree_work_counters_v1) -> Self { + self.counters = TreeWorkCounters { + sampled_at_ns: raw.sampled_at_ns, + values: raw.values, + }; + self + } +} +pub(super) fn raw_counters() -> Result<sys::ct_tree_work_counters_v1, CtError> { + Ok(sys::ct_tree_work_counters_v1 { + struct_size: u32::try_from(std::mem::size_of::<sys::ct_tree_work_counters_v1>()) + .map_err(|_| CtError::Internal)?, + version: 1, + ..Default::default() + }) +} diff --git a/lib/crowdb-tree/ffi/src/stats/work/histograms.rs b/lib/crowdb-tree/ffi/src/stats/work/histograms.rs new file mode 100644 index 000000000..53c78ad24 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/stats/work/histograms.rs @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crate::error::{check, CtError}; +use crate::{sys, Crowdbtree}; + +/// Fixed work-v1 stage order. Enables only this Tree's selected distribution. +#[repr(u32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TreeWorkStage { + FlushBatch = 0, + FlushAdmission = 1, + FlushCapture = 2, + FlushWriterDrain = 3, + FlushHandoff = 4, + FlushRange = 5, + PublishQueue = 6, + PublishCommit = 7, + MaintenancePlan = 8, + OverlayCoalesce = 9, + PageSplit = 10, + PageMerge = 11, + PageLayout = 12, + PageEncode = 13, + BudgetWait = 14, + PageLoadPipeline = 15, + PageWritebackPipeline = 16, + SnapshotCapture = 17, + SnapshotCommit = 18, + MemoryRetireWait = 19, + FlushRepair = 20, +} + +/// Real nanosecond distribution from common LatencyHistogram. Quantiles are +/// bucket upper bounds, capped at observed max; overflow quantiles are u64::MAX. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeHistogramStats { + pub count: u64, + pub sum_ns: u128, + pub min_ns: u64, + pub max_ns: u64, + pub p50_ns: u64, + pub p90_ns: u64, + pub p95_ns: u64, + pub p99_ns: u64, + pub p999_ns: u64, + pub zero_count: u64, + pub overflow_count: u64, + pub total_count: u64, +} + +/// Small immutable results, not bucket storage. Default has no enabled stages. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeWorkHistograms { + pub window_sequence: u64, + pub sampled_at_ns: u64, + pub enabled_stages: u64, + pub stages: [TreeHistogramStats; 21], +} + +impl TreeWorkHistograms { + #[must_use] + pub fn stage(&self, stage: TreeWorkStage) -> Option<&TreeHistogramStats> { + let index = stage as usize; + (self.enabled_stages & (1 << index) != 0).then_some(&self.stages[index]) + } +} + +impl From<sys::ct_tree_work_histograms_v1> for TreeWorkHistograms { + fn from(raw: sys::ct_tree_work_histograms_v1) -> Self { + Self { + window_sequence: raw.window_sequence, + sampled_at_ns: raw.sampled_at_ns, + enabled_stages: raw.enabled_stages, + stages: raw.stages.map(|stage| TreeHistogramStats { + count: stage.count, + sum_ns: (u128::from(stage.sum_high) << 64) | u128::from(stage.sum_low), + min_ns: stage.min_ns, + max_ns: stage.max_ns, + p50_ns: stage.p50_ns, + p90_ns: stage.p90_ns, + p95_ns: stage.p95_ns, + p99_ns: stage.p99_ns, + p999_ns: stage.p999_ns, + zero_count: stage.zero_count, + overflow_count: stage.overflow_count, + total_count: stage.total_count, + }), + } + } +} + +pub(super) fn raw_histograms() -> Result<sys::ct_tree_work_histograms_v1, CtError> { + Ok(sys::ct_tree_work_histograms_v1 { + struct_size: u32::try_from(std::mem::size_of::<sys::ct_tree_work_histograms_v1>()) + .map_err(|_| CtError::Internal)?, + version: 1, + ..Default::default() + }) +} + +impl Crowdbtree { + /// Additively enables one distribution. Repeated calls are idempotent. + /// Bucket storage is charged to this Tree; a budget failure leaves the + /// selected set and business state unchanged. Terminal events allocate nothing. + pub fn enable_work_histogram(&self, stage: TreeWorkStage) -> Result<(), CtError> { + check(unsafe { sys::ct_work_histogram_enable_v1(self.as_ptr(), stage as u32) }) + } + + /// Cumulative distribution sampling never drains the collector window. + pub fn work_histogram_totals(&self) -> Result<TreeWorkHistograms, CtError> { + let mut raw = raw_histograms()?; + check(unsafe { sys::ct_work_histograms_totals_v1(self.as_ptr(), &mut raw) })?; + Ok(raw.into()) + } +} diff --git a/lib/crowdb-tree/ffi/src/stats/work/waits.rs b/lib/crowdb-tree/ffi/src/stats/work/waits.rs new file mode 100644 index 000000000..708114f03 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/stats/work/waits.rs @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{TreeWorkCollector, TreeWorkStageAge}; +use crate::{error::check, sys, Crowdbtree, CtError}; + +pub const TREE_WAIT_CLASS_NAMES: [&str; 7] = [ + "handoff", + "routing", + "reclaim", + "admitted_writer", + "gap_repair", + "io", + "shared_cache", +]; + +/// Cumulative per-Tree request summaries, independent of host reservation totals. +/// Shared-cache samples count joined waiters, excluding first loaders and hits. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeWaitStats { + pub attempts: u64, + pub inflight: u64, + pub total_count: u64, + pub total_sum_ns: u64, + pub total_max_ns: u64, + /// Granted/ready, error, cancelled. + pub outcomes: [u64; 3], + pub age: TreeWorkStageAge, +} + +/// Approximate non-draining cumulative sample. The age sample uses fixed paid +/// storage; saturation reports incomplete coverage without rejecting work. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TreeWaitSnapshot { + pub sampled_at_ns: u64, + pub classes: [TreeWaitStats; 7], +} + +fn sample(read: impl FnOnce(&mut sys::ct_tree_wait_snapshot_v1) -> i32) -> Result<TreeWaitSnapshot, CtError> { + let mut raw = sys::ct_tree_wait_snapshot_v1 { + struct_size: u32::try_from(std::mem::size_of::<sys::ct_tree_wait_snapshot_v1>()) + .map_err(|_| CtError::Internal)?, + version: 1, + ..Default::default() + }; + check(read(&mut raw))?; + Ok(TreeWaitSnapshot { + sampled_at_ns: raw.sampled_at_ns, + classes: raw.classes.map(|value| TreeWaitStats { + attempts: value.attempts, + inflight: value.inflight, + total_count: value.total_count, + total_sum_ns: value.total_sum_ns, + total_max_ns: value.total_max_ns, + outcomes: value.outcomes, + age: TreeWorkStageAge { + inflight: value.inflight, + tracked: value.tracked, + untracked: value.untracked, + observed_oldest_age_ns: value.observed_oldest_age_ns, + complete: value.complete == 1, + }, + }), + }) +} + +impl TreeWorkCollector { + /// Valid after Tree close and never resets the stage collector window. + pub fn wait_metrics_totals(&self) -> Result<TreeWaitSnapshot, CtError> { + sample(|out| unsafe { sys::ct_work_metrics_collector_wait_totals(self.handle.as_ptr(), out) }) + } +} + +impl Crowdbtree { + pub fn wait_metrics_totals(&self) -> Result<TreeWaitSnapshot, CtError> { + sample(|out| unsafe { sys::ct_tree_wait_totals(self.as_ptr(), out) }) + } +} diff --git a/lib/crowdb-tree/ffi/src/sys.rs b/lib/crowdb-tree/ffi/src/sys.rs index d3091deba..c2a60a31a 100644 --- a/lib/crowdb-tree/ffi/src/sys.rs +++ b/lib/crowdb-tree/ffi/src/sys.rs @@ -5,6 +5,35 @@ use std::os::raw::{c_char, c_int}; +mod owned_allocation; +mod owned_budget; +pub(crate) use owned_allocation::*; +pub use owned_budget::*; +mod backend_release; +pub(crate) use backend_release::*; +mod admission; +pub(crate) use admission::*; +mod maintenance; +pub(crate) use maintenance::*; +mod wait_metrics; +pub(crate) use wait_metrics::*; +mod backing_stats; +mod runtime; +mod state_stats; +pub(crate) use backing_stats::*; +pub(crate) use runtime::*; +pub(crate) use state_stats::*; +mod chunk_protection; +mod chunk_runtime; +mod chunk_stage; +pub(crate) use chunk_protection::*; +pub(crate) use chunk_runtime::*; +pub(crate) use chunk_stage::*; +mod split_overlay; +pub(crate) use split_overlay::*; +mod scan_owner; +pub(crate) use scan_owner::*; + pub type ct_root_catalog_load_fn = unsafe extern "C" fn( context: *mut std::ffi::c_void, kind: i32, @@ -200,22 +229,6 @@ pub struct ct_kv_ref { pub kind: u8, } -// One zero-copy op for ct_apply_batch_external. `bytes_ref` is an -// opaque Rust handle (a boxed Bytes); `drop_fn` decrements its refcount -// when crowdb-tree frees the borrowed buffer. -pub type ct_drop_fn = extern "C" fn(*mut std::ffi::c_void); - -#[repr(C)] -pub struct ct_ext_op { - pub key: *const u8, - pub key_len: usize, - pub value: *const u8, - pub value_len: usize, - pub kind: u8, - pub bytes_ref: *mut std::ffi::c_void, - pub drop_fn: Option<ct_drop_fn>, -} - #[repr(C)] #[derive(Default)] pub struct ct_merge_gc_stats { @@ -280,6 +293,40 @@ pub struct ct_tree_summary { pub exact: c_int, } +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_tuning { + pub struct_size: usize, + pub memtable_flush_bytes: usize, + pub memtable_flush_entries: u32, + pub max_memtable_count: u32, + pub leaf_split_bytes: usize, + pub leaf_merge_bytes: usize, + pub inner_max_keys: u32, + pub inner_merge_keys: u32, + pub max_key_size: usize, + pub source_max_entries: usize, + pub source_max_key_bytes: usize, + pub source_max_gap_entries: usize, + pub source_max_gap_key_bytes: usize, + pub flush_parallelism: usize, + pub run_max_entries: usize, + pub run_max_key_bytes: usize, + pub run_key_block_bytes: usize, + pub run_max_bytes: usize, + pub install_run_window: usize, + pub maintenance_visits_per_step: usize, + pub maintenance_max_visits: usize, + pub maintenance_max_candidates: usize, + pub maintenance_max_steps: usize, + pub maintenance_metadata_bytes: usize, + pub cache_max_pages: usize, +} + +extern "C" { + pub fn ct_tree_tuning_default(out: *mut ct_tree_tuning); +} + #[repr(C)] pub struct ct_options { pub page_store: *mut ct_page_store, @@ -304,6 +351,7 @@ pub struct ct_options { pub log_file_prefix: *const c_char, pub log_max_file_mb: usize, pub log_max_files: usize, + pub tuning: *const ct_tree_tuning, } extern "C" { @@ -365,13 +413,6 @@ extern "C" { transition_high: u64, transition_low: u64, ) -> c_int; - pub fn ct_open(opt: *const ct_options, out: *mut *mut ct_tree) -> c_int; - pub fn ct_rebuild_range( - source: *mut ct_tree, - destination_options: *const ct_options, - out: *mut *mut ct_tree, - stats: *mut ct_range_rebuild_stats, - ) -> c_int; pub fn ct_close(t: *mut ct_tree); pub fn ct_init_logging( log_dir: *const c_char, @@ -422,7 +463,6 @@ extern "C" { ) -> c_int; pub fn ct_apply_delete(t: *mut ct_tree, slot: u64, key: *const u8, klen: usize) -> c_int; pub fn ct_apply_batch_slices(t: *mut ct_tree, slot: u64, ops: *const ct_kv_ref, count: u64) -> c_int; - pub fn ct_apply_batch_external(t: *mut ct_tree, slot: u64, ops: *const ct_ext_op, count: u64) -> c_int; pub fn ct_force_advance_slot(t: *mut ct_tree, slot: u64); pub fn ct_alloc( t: *mut ct_tree, @@ -468,70 +508,6 @@ extern "C" { slot: *mut u64, value: *mut ct_buf, ) -> c_int; - pub fn ct_scan( - t: *mut ct_tree, - prefix: *const u8, - plen: usize, - start_key: *const u8, - sklen: usize, - has_start_bound: c_int, - start_inclusive: c_int, - end_key: *const u8, - elen: usize, - limit: usize, - byte_budget: usize, - keys_only: c_int, - deadline_ms: u64, - include_tombstones: c_int, - out_entries: *mut ct_buf, - out_count: *mut u64, - truncated: *mut c_int, - ) -> c_int; - pub fn ct_scan_strict( - t: *mut ct_tree, - prefix: *const u8, - plen: usize, - start_key: *const u8, - sklen: usize, - has_start_bound: c_int, - start_inclusive: c_int, - end_key: *const u8, - elen: usize, - limit: usize, - byte_budget: usize, - keys_only: c_int, - deadline_ms: u64, - include_tombstones: c_int, - out_entries: *mut ct_buf, - out_count: *mut u64, - truncated: *mut c_int, - ) -> c_int; - pub fn ct_seek_reverse( - t: *mut ct_tree, - start_key: *const u8, - sklen: usize, - start_inclusive: c_int, - begin_key: *const u8, - bklen: usize, - found: *mut c_int, - out_key: *mut ct_buf, - out_slot: *mut u64, - out_value: *mut ct_buf, - ) -> c_int; - pub fn ct_scan_reverse( - t: *mut ct_tree, - start_key: *const u8, - sklen: usize, - has_start_bound: c_int, - start_inclusive: c_int, - begin_key: *const u8, - bklen: usize, - limit: usize, - byte_budget: usize, - out_entries: *mut ct_buf, - out_count: *mut u64, - truncated: *mut c_int, - ) -> c_int; pub fn ct_snapshot_view(t: *mut ct_tree, out: *mut *mut ct_view) -> c_int; pub fn ct_view_at_slot(v: *const ct_view) -> u64; pub fn ct_view_iter(v: *mut ct_view, out: *mut *mut ct_iter) -> c_int; @@ -567,36 +543,6 @@ extern "C" { pub fn ct_evict_clean_leaves(t: *mut ct_tree, max_resident_leaves: u64) -> u64; pub fn ct_evict_clean_inner(t: *mut ct_tree, max_resident_inner: u64) -> u64; pub fn ct_get_async(t: *mut ct_tree, key: *const u8, klen: usize) -> *mut ct_future; - pub fn ct_flush_async(t: *mut ct_tree) -> *mut ct_future; - pub fn ct_snapshot_async(t: *mut ct_tree) -> *mut ct_future; - #[allow(dead_code)] - pub fn ct_scan_async( - t: *mut ct_tree, - prefix: *const u8, - plen: usize, - start_after: *const u8, - salen: usize, - end_key: *const u8, - elen: usize, - limit: usize, - byte_budget: usize, - keys_only: c_int, - deadline_ms: u64, - ) -> *mut ct_future; - pub fn ct_scan_directional_async( - t: *mut ct_tree, - prefix: *const u8, - plen: usize, - start_after: *const u8, - salen: usize, - end_key: *const u8, - elen: usize, - limit: usize, - byte_budget: usize, - keys_only: c_int, - deadline_ms: u64, - direction: c_int, - ) -> *mut ct_future; pub fn ct_future_poll( f: *mut ct_future, done: *mut c_int, @@ -668,3 +614,23 @@ extern "C" { pub fn crowdb_common_metrics_global_max_name_len() -> usize; pub fn crowdb_common_metrics_global_free(s: *mut c_char); } + +mod work_histograms; +pub use work_histograms::*; +mod work_metrics; +pub use work_metrics::*; + +mod snapshot_owner; +pub use snapshot_owner::*; + +mod chunk_pin_service; +pub(crate) use chunk_pin_service::*; + +mod immutable_snapshot; +pub use immutable_snapshot::*; + +mod compaction; +pub use compaction::*; + +mod chunk_migration; +pub use chunk_migration::*; diff --git a/lib/crowdb-tree/ffi/src/sys/admission.rs b/lib/crowdb-tree/ffi/src/sys/admission.rs new file mode 100644 index 000000000..09537234f --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/admission.rs @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_future, ct_kv_ref, ct_tree}; + +#[repr(C)] +pub struct ct_apply_permit { + _private: [u8; 0], +} + +extern "C" { + pub fn ct_apply_replay( + tree: *mut ct_tree, + slot: u64, + count: usize, + read: unsafe extern "C" fn(*mut std::ffi::c_void, usize, *mut ct_kv_ref), + drop_context: unsafe extern "C" fn(*mut std::ffi::c_void), + context: *mut std::ffi::c_void, + owner_control_bytes: usize, + out: *mut *mut ct_future, + ) -> i32; + pub fn ct_apply_prepare( + tree: *mut ct_tree, + operations: *const ct_kv_ref, + count: usize, + gap_repair: i32, + out: *mut *mut ct_apply_permit, + ) -> i32; + pub fn ct_apply_arm(permit: *mut ct_apply_permit) -> i32; + pub fn ct_apply_submit(permit: *mut ct_apply_permit, slot: u64, out: *mut *mut ct_future) -> i32; + pub fn ct_apply_permit_free(permit: *mut ct_apply_permit); +} diff --git a/lib/crowdb-tree/ffi/src/sys/backend_release.rs b/lib/crowdb-tree/ffi/src/sys/backend_release.rs new file mode 100644 index 000000000..3df8bf331 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/backend_release.rs @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::ct_tree; +use std::ffi::c_void; +#[repr(C)] +pub struct ct_backend_release_service { + _private: [u8; 0], +} +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct ct_backend_release_step { + pub visited: usize, + pub completed: usize, + pub more: u8, + pub drained: u8, +} +unsafe extern "C" { + pub fn ct_tree_backend_release_service( + tree: *mut ct_tree, + wake: unsafe extern "C" fn(*mut c_void), + drop_context: unsafe extern "C" fn(*mut c_void), + context: *mut c_void, + out: *mut *mut ct_backend_release_service, + ) -> i32; + pub fn ct_backend_release_service_pump( + service: *mut ct_backend_release_service, + limit: usize, + ) -> ct_backend_release_step; + pub fn ct_backend_release_service_stop(service: *mut ct_backend_release_service); + pub fn ct_backend_release_service_free(service: *mut ct_backend_release_service); +} diff --git a/lib/crowdb-tree/ffi/src/sys/backing_stats.rs b/lib/crowdb-tree/ffi/src/sys/backing_stats.rs new file mode 100644 index 000000000..180031f86 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/backing_stats.rs @@ -0,0 +1,18 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_tree, ct_work_metrics_collector}; +#[repr(C)] +#[derive(Default)] +pub struct ct_backing_stats_v1 { + pub struct_size: u32, + pub version: u32, + pub scope_id: u64, + pub allocated_bytes: u64, +} +unsafe extern "C" { + pub fn ct_work_metrics_collector_backing_stats( + collector: *mut ct_work_metrics_collector, + out: *mut ct_backing_stats_v1, + ) -> i32; + pub fn ct_get_backing_stats(tree: *mut ct_tree, out: *mut ct_backing_stats_v1) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/chunk_migration.rs b/lib/crowdb-tree/ffi/src/sys/chunk_migration.rs new file mode 100644 index 000000000..9c89d3f04 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/chunk_migration.rs @@ -0,0 +1,12 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_options, ct_runtime, ct_tree, ct_tree_runtime_options}; +extern "C" { + pub fn ct_migrate_legacy_chunk_with_runtime( + options: *const ct_options, + runtime: *mut ct_runtime, + binding: *const ct_tree_runtime_options, + expected_generation: u64, + out: *mut *mut ct_tree, + ) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/chunk_pin_service.rs b/lib/crowdb-tree/ffi/src/sys/chunk_pin_service.rs new file mode 100644 index 000000000..922e04d69 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/chunk_pin_service.rs @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_resource_usage_v1, ct_root_catalog, ct_runtime}; +use std::ffi::c_void; +#[repr(C)] +pub struct ct_chunk_pin_service { + _private: [u8; 0], +} +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct ct_chunk_pin_service_step { + pub completed: usize, + pub failed: usize, + pub more: u8, + pub drained: u8, +} +unsafe extern "C" { + pub fn ct_chunk_pin_service_create( + runtime: *mut ct_runtime, + capacity: usize, + wake: unsafe extern "C" fn(*mut c_void), + drop_context: unsafe extern "C" fn(*mut c_void), + context: *mut c_void, + out: *mut *mut ct_chunk_pin_service, + ) -> i32; + pub fn ct_root_catalog_bind_pin_service( + catalog: *mut ct_root_catalog, + service: *mut ct_chunk_pin_service, + ) -> i32; + pub fn ct_chunk_pin_service_pump( + service: *mut ct_chunk_pin_service, + limit: usize, + ) -> ct_chunk_pin_service_step; + pub fn ct_chunk_pin_service_stop(service: *mut ct_chunk_pin_service); + pub fn ct_chunk_pin_service_resource_usage( + service: *const ct_chunk_pin_service, + out: *mut ct_resource_usage_v1, + ) -> i32; + pub fn ct_chunk_pin_service_free(service: *mut ct_chunk_pin_service); +} diff --git a/lib/crowdb-tree/ffi/src/sys/chunk_protection.rs b/lib/crowdb-tree/ffi/src/sys/chunk_protection.rs new file mode 100644 index 000000000..68916304a --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/chunk_protection.rs @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ + ct_chunk_stage_id, ct_root_catalog, ct_root_catalog_callbacks, ct_root_catalog_staging_callbacks, +}; +use std::ffi::c_void; +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ct_chunk_candidate_status { + pub id: ct_chunk_stage_id, + pub generation: u64, + pub reference_count: u64, + pub prepared_references: u64, + pub prepared_windows: u64, + pub released_windows: u64, + pub state: u32, +} +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ct_chunk_pin_progress { + pub after: ct_chunk_stage_id, + pub examined: u32, + pub released: u32, + pub has_after: u8, + pub more: u8, +} +#[repr(C)] +pub struct ct_root_catalog_protection_callbacks { + pub version: u32, + pub struct_bytes: u32, + pub capabilities: u64, + pub check_reader: Option<unsafe extern "C" fn(*mut c_void) -> i32>, + pub writer_quiesced: Option<unsafe extern "C" fn(*mut c_void, u64, u64) -> i32>, + pub begin: Option< + unsafe extern "C" fn(*mut c_void, u64, ct_chunk_stage_id, u64, u64, u32, *const u8, usize) -> i32, + >, + pub load: Option< + unsafe extern "C" fn(*mut c_void, u64, ct_chunk_stage_id, *mut ct_chunk_candidate_status) -> i32, + >, + pub list: Option< + unsafe extern "C" fn( + *mut c_void, + u64, + *const ct_chunk_stage_id, + u32, + *mut ct_chunk_candidate_status, + *mut usize, + ) -> i32, + >, + pub prepare: Option< + unsafe extern "C" fn( + *mut c_void, + u64, + ct_chunk_stage_id, + u64, + *const ct_chunk_stage_id, + usize, + ) -> i32, + >, + pub finish: Option<unsafe extern "C" fn(*mut c_void, u64, ct_chunk_stage_id, u32) -> i32>, + pub release: Option<unsafe extern "C" fn(*mut c_void, u64, ct_chunk_stage_id, u64) -> i32>, + pub pin: Option<unsafe extern "C" fn(*mut c_void, u64, ct_chunk_stage_id, ct_chunk_stage_id, u8) -> i32>, + pub prune: Option< + unsafe extern "C" fn( + *mut c_void, + u64, + ct_chunk_stage_id, + *const ct_chunk_stage_id, + u32, + *mut ct_chunk_pin_progress, + ) -> i32, + >, +} +unsafe extern "C" { + pub fn ct_callback_root_catalog_open_v3( + callbacks: *const ct_root_catalog_callbacks, + staging: *const ct_root_catalog_staging_callbacks, + protection: *const ct_root_catalog_protection_callbacks, + context: *mut c_void, + out: *mut *mut ct_root_catalog, + ) -> i32; + pub fn ct_root_catalog_protection_capabilities(catalog: *const ct_root_catalog) -> u64; +} diff --git a/lib/crowdb-tree/ffi/src/sys/chunk_runtime.rs b/lib/crowdb-tree/ffi/src/sys/chunk_runtime.rs new file mode 100644 index 000000000..cd9ffbce7 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/chunk_runtime.rs @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_chunk_transport, ct_future, ct_page_store, ct_root_catalog, ct_runtime, ct_tree}; + +#[repr(C)] +pub struct ct_chunk_runtime_options { + pub version: u32, + pub struct_bytes: u32, + pub tree_id: u64, + pub owner_epoch: u64, + pub open_generation: u64, + pub iu_size: u32, + pub mirror_copies: u32, + pub max_chunk_bytes: u64, +} + +extern "C" { + pub fn ct_chunk_runtime_page_store_open( + runtime: *mut ct_runtime, + options: *const ct_chunk_runtime_options, + catalog: *mut ct_root_catalog, + transport: *mut ct_chunk_transport, + out: *mut *mut ct_page_store, + ) -> i32; +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_chunk_collection_result_v1 { + pub struct_size: u32, + pub version: u32, + pub terminal_status: i32, + pub more: u32, + pub candidates_examined: u64, + pub reference_windows: u64, + pub stages_examined: u64, + pub pins_examined: u64, + pub pins_released: u64, + pub stages_removed: u64, + pub logical_capacity_bytes: u64, +} +extern "C" { + pub fn ct_chunk_collect_async(tree: *mut ct_tree, out: *mut *mut ct_future) -> i32; + pub fn ct_future_poll_chunk_collection( + future: *mut ct_future, + done: *mut i32, + out: *mut ct_chunk_collection_result_v1, + ) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/chunk_stage.rs b/lib/crowdb-tree/ffi/src/sys/chunk_stage.rs new file mode 100644 index 000000000..c59f8832a --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/chunk_stage.rs @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::{ct_buf, ct_root_catalog, ct_root_catalog_callbacks}; +use std::ffi::c_void; +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ct_chunk_stage_id { + pub owner_epoch: u64, + pub operation_high: u64, + pub operation_low: u64, +} +#[repr(C)] +pub struct ct_root_catalog_staging_callbacks { + pub version: u32, + pub struct_bytes: u32, + pub capabilities: u64, + pub load: + Option<unsafe extern "C" fn(*mut c_void, u64, ct_chunk_stage_id, *mut *const u8, *mut usize) -> i32>, + pub compare: + Option<unsafe extern "C" fn(*mut c_void, u64, u64, *const u8, usize, *const u8, usize) -> i32>, + pub list: Option< + unsafe extern "C" fn( + *mut c_void, + u64, + *const ct_chunk_stage_id, + u32, + *mut *const u8, + *mut usize, + ) -> i32, + >, +} +extern "C" { + pub fn ct_callback_root_catalog_open_v2( + callbacks: *const ct_root_catalog_callbacks, + staging: *const ct_root_catalog_staging_callbacks, + context: *mut c_void, + out: *mut *mut ct_root_catalog, + ) -> i32; + pub fn ct_root_catalog_stage_capabilities(catalog: *const ct_root_catalog) -> u64; + pub fn ct_root_catalog_stage_load( + catalog: *mut ct_root_catalog, + tree_id: u64, + id: ct_chunk_stage_id, + out: *mut ct_buf, + ) -> i32; + pub fn ct_root_catalog_stage_compare( + catalog: *mut ct_root_catalog, + tree_id: u64, + owner_epoch: u64, + expected: *const u8, + expected_length: usize, + next: *const u8, + next_length: usize, + ) -> i32; + pub fn ct_root_catalog_stage_list( + catalog: *mut ct_root_catalog, + tree_id: u64, + after: *const ct_chunk_stage_id, + limit: u32, + out: *mut ct_buf, + ) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/compaction.rs b/lib/crowdb-tree/ffi/src/sys/compaction.rs new file mode 100644 index 000000000..1aa466c69 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/compaction.rs @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_future, ct_tree}; +#[repr(C)] +pub struct ct_compaction_options_v1 { + pub struct_size: u32, + pub version: u32, + pub free_threshold: f64, + pub max_relocation_bytes: u64, +} +#[repr(C)] +#[derive(Default)] +pub struct ct_compaction_result_v1 { + pub struct_size: u32, + pub version: u32, + pub terminal_status: i32, + pub blocks_selected: u64, + pub pages_relocated: u64, + pub bytes_relocated: u64, + pub blocks_deleted: u64, +} +extern "C" { + pub fn ct_compact_sparse_blocks_async( + tree: *mut ct_tree, + options: *const ct_compaction_options_v1, + out: *mut *mut ct_future, + ) -> i32; + pub fn ct_future_poll_compaction( + future: *mut ct_future, + done: *mut i32, + out: *mut ct_compaction_result_v1, + ) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/immutable_snapshot.rs b/lib/crowdb-tree/ffi/src/sys/immutable_snapshot.rs new file mode 100644 index 000000000..3071112aa --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/immutable_snapshot.rs @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_future, ct_range_rebuild_stats, ct_scan_options_v2, ct_scan_result, ct_tree}; +use std::ffi::c_int; + +#[repr(C)] +pub struct ct_immutable_snapshot { + _private: [u8; 0], +} +extern "C" { + pub fn ct_immutable_snapshot_capture( + tree: *mut ct_tree, + expected_sequence: u64, + expected_coverage: u64, + out: *mut *mut ct_immutable_snapshot, + ) -> c_int; + pub fn ct_immutable_snapshot_free(snapshot: *mut ct_immutable_snapshot); + pub fn ct_immutable_snapshot_get( + snapshot: *mut ct_immutable_snapshot, + key: *const u8, + size: usize, + out: *mut *mut ct_future, + ) -> c_int; + pub fn ct_immutable_snapshot_scan( + snapshot: *mut ct_immutable_snapshot, + options: *const ct_scan_options_v2, + out: *mut *mut ct_scan_result, + ) -> c_int; + pub fn ct_immutable_snapshot_rebuild( + snapshot: *mut ct_immutable_snapshot, + destination: *mut ct_tree, + stats: *mut ct_range_rebuild_stats, + ) -> c_int; +} diff --git a/lib/crowdb-tree/ffi/src/sys/maintenance.rs b/lib/crowdb-tree/ffi/src/sys/maintenance.rs new file mode 100644 index 000000000..58b5ba44b --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/maintenance.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_future, ct_tree}; + +#[repr(C)] +#[derive(Default)] +pub struct ct_flush_result_v1 { + pub struct_size: u32, + pub version: u32, + pub terminal_status: i32, + pub handoff_status: i32, + pub maintenance_status: i32, + pub phase: u32, + pub batch_id: u64, + pub floor: u64, + pub frontier: u64, + pub published_coverage: u64, + pub deferred_pages: u64, + pub handoff_completed: i32, + pub handoff_advanced: i32, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_checkpoint_result_v1 { + pub struct_size: u32, + pub version: u32, + pub terminal_status: i32, + pub durable_committed: i32, + pub writeback_completed: i32, + pub durable_coverage: u64, + pub snapshot_sequence: u64, + pub checkpoint_token: u64, + pub captured_coverage: u64, + pub primary_frames: u64, + pub written_frames: u64, + pub reused_frames: u64, + pub rebased_pages: u64, + pub rebase_deferred: u64, +} + +extern "C" { + pub fn ct_durable_coverage(tree: *mut ct_tree) -> u64; + pub fn ct_flush_report_async(tree: *mut ct_tree, out: *mut *mut ct_future) -> i32; + pub fn ct_writeback_async(tree: *mut ct_tree, out: *mut *mut ct_future) -> i32; + pub fn ct_checkpoint_report_async(tree: *mut ct_tree, out: *mut *mut ct_future) -> i32; + pub fn ct_future_poll_flush_report( + future: *mut ct_future, + done: *mut i32, + out: *mut ct_flush_result_v1, + ) -> i32; + pub fn ct_future_poll_checkpoint_report( + future: *mut ct_future, + done: *mut i32, + out: *mut ct_checkpoint_result_v1, + ) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/owned_allocation.rs b/lib/crowdb-tree/ffi/src/sys/owned_allocation.rs new file mode 100644 index 000000000..06cdd7b40 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/owned_allocation.rs @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{c_int, ct_tree}; + +#[repr(C)] +pub struct ct_owned_allocation { + _private: [u8; 0], +} + +#[repr(C)] +pub struct ct_owned_kv_ref { + pub key: *const u8, + pub key_len: usize, + pub owner: *mut ct_owned_allocation, + pub offset: usize, + pub length: usize, + pub kind: u8, +} + +extern "C" { + pub fn ct_owned_allocation_create( + tree: *mut ct_tree, + data: *const u8, + length: usize, + capacity: usize, + drop_context: unsafe extern "C" fn(*mut std::ffi::c_void), + context: *mut std::ffi::c_void, + control_bytes: usize, + out: *mut *mut ct_owned_allocation, + ) -> c_int; + pub fn ct_owned_allocation_retain(owner: *mut ct_owned_allocation); + pub fn ct_owned_allocation_free(owner: *mut ct_owned_allocation); + pub fn ct_apply_batch_owned_reader( + tree: *mut ct_tree, + slot: u64, + count: usize, + read: unsafe extern "C" fn(*const std::ffi::c_void, usize) -> ct_owned_kv_ref, + context: *const std::ffi::c_void, + ) -> c_int; +} diff --git a/lib/crowdb-tree/ffi/src/sys/owned_budget.rs b/lib/crowdb-tree/ffi/src/sys/owned_budget.rs new file mode 100644 index 000000000..e4fcef502 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/owned_budget.rs @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_future, ct_scan_result, ct_snapshot_chunk}; + +#[repr(C)] +pub struct ct_owned_budget { + _private: [u8; 0], +} + +extern "C" { + pub fn ct_future_owned_budget(future: *const ct_future, out: *mut *mut ct_owned_budget) -> i32; + pub fn ct_scan_result_owned_budget(result: *const ct_scan_result, out: *mut *mut ct_owned_budget) -> i32; + pub fn ct_snapshot_chunk_owned_budget( + chunk: *const ct_snapshot_chunk, + out: *mut *mut ct_owned_budget, + ) -> i32; + pub fn ct_owned_budget_claim(budget: *mut ct_owned_budget, bytes: usize) -> i32; + pub fn ct_owned_budget_record(budget: *mut ct_owned_budget, bytes: usize, acquired: i32); + pub fn ct_owned_budget_release(budget: *mut ct_owned_budget, bytes: usize); + pub fn ct_owned_budget_free(budget: *mut ct_owned_budget); +} diff --git a/lib/crowdb-tree/ffi/src/sys/runtime.rs b/lib/crowdb-tree/ffi/src/sys/runtime.rs new file mode 100644 index 000000000..6dd665bfe --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/runtime.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::{ct_options, ct_range_rebuild_stats, ct_tree}; + +#[repr(C)] +pub struct ct_runtime { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Default, Clone, Copy)] +pub struct ct_resource_limits { + pub handoff_bytes: usize, + pub routing_bytes: usize, + pub reclaim_bytes: usize, + pub writer_bytes: usize, + pub gap_repair_bytes: usize, + pub io_operations: usize, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_runtime_options { + pub struct_size: usize, + pub workers: usize, + pub cpu_queue_limit: usize, + pub max_waiters_per_class: usize, + pub lifecycle_bytes: usize, + pub limits: ct_resource_limits, + pub backend_workers: usize, + pub backend_queue_limit: usize, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_runtime_options { + pub struct_size: usize, + pub cpu_queue_limit: usize, + pub lifecycle_bytes: usize, + pub limits: ct_resource_limits, + pub backend_queue_limit: usize, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_executor_snapshot_v1 { + pub queued: u64, + pub inflight: u64, + pub submitted: u64, + pub completed: u64, + pub rejected: u64, + pub queue_ns: u64, + pub execute_ns: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_runtime_snapshot_v1 { + pub struct_size: usize, + pub version: u32, + pub stopped: u32, + pub reserved: ct_resource_limits, + pub allocated: ct_resource_limits, + pub bindings: u64, + pub cpu: ct_executor_snapshot_v1, + pub backend: ct_executor_snapshot_v1, + pub budget_waiters: u64, + pub budget_grants: u64, + pub budget_cancelled: u64, + pub budget_wait_ns: u64, +} + +extern "C" { + pub fn ct_runtime_snapshot(runtime: *const ct_runtime, out: *mut ct_runtime_snapshot_v1) -> i32; + pub fn ct_runtime_options_default(out: *mut ct_runtime_options); + pub fn ct_tree_runtime_options_default(out: *mut ct_tree_runtime_options); + pub fn ct_runtime_create(options: *const ct_runtime_options, out: *mut *mut ct_runtime) -> i32; + pub fn ct_runtime_stop_admission(runtime: *mut ct_runtime); + pub fn ct_runtime_drain_cpu(runtime: *mut ct_runtime); + pub fn ct_runtime_free(runtime: *mut ct_runtime); + pub fn ct_open_with_runtime( + options: *const ct_options, + runtime: *mut ct_runtime, + binding: *const ct_tree_runtime_options, + out: *mut *mut ct_tree, + ) -> i32; + pub fn ct_rebuild_range_with_runtime( + source: *mut ct_tree, + options: *const ct_options, + runtime: *mut ct_runtime, + binding: *const ct_tree_runtime_options, + out: *mut *mut ct_tree, + stats: *mut ct_range_rebuild_stats, + ) -> i32; +} + +#[repr(C)] +#[derive(Default, Clone, Copy)] +pub struct ct_resource_usage_v1 { + pub limit: u64, + pub used: u64, + pub reserved: u64, + pub allocated: u64, + pub denied_tree: u64, + pub denied_host: u64, + pub waits: u64, + pub wait_ns: u64, + pub waiters: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_resource_usage_snapshot_v1 { + pub struct_size: usize, + pub version: u32, + pub resource_mask: u32, + pub scope_id: u64, + pub resources: [ct_resource_usage_v1; 20], +} + +extern "C" { + pub fn ct_runtime_resource_usage( + runtime: *const ct_runtime, + out: *mut ct_resource_usage_snapshot_v1, + ) -> i32; + pub fn ct_tree_resource_usage(tree: *mut ct_tree, out: *mut ct_resource_usage_snapshot_v1) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/scan_owner.rs b/lib/crowdb-tree/ffi/src/sys/scan_owner.rs new file mode 100644 index 000000000..8150de037 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/scan_owner.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::{ct_future, ct_tree}; + +#[repr(C)] +pub struct ct_scan_result { + _private: [u8; 0], +} + +#[repr(C)] +pub struct ct_scan_options_v2 { + pub struct_size: usize, + pub prefix: *const u8, + pub prefix_len: usize, + pub start: *const u8, + pub start_len: usize, + pub end: *const u8, + pub end_len: usize, + pub begin: *const u8, + pub begin_len: usize, + pub limit: usize, + pub byte_budget: usize, + pub deadline_ms: u64, + pub reverse: i32, + pub keys_only: i32, + pub include_tombstones: i32, + pub has_start_bound: i32, + pub start_inclusive: i32, + pub strict_byte_budget: i32, +} + +extern "C" { + pub fn ct_scan_async_v2( + tree: *mut ct_tree, + options: *const ct_scan_options_v2, + out: *mut *mut ct_future, + ) -> i32; + pub fn ct_scan_v2( + tree: *mut ct_tree, + options: *const ct_scan_options_v2, + out: *mut *mut ct_scan_result, + ) -> i32; + pub fn ct_future_poll_scan_v2( + future: *mut ct_future, + done: *mut i32, + out: *mut *mut ct_scan_result, + ) -> i32; + pub fn ct_scan_result_data( + result: *const ct_scan_result, + data: *mut *const u8, + len: *mut usize, + count: *mut u64, + truncated: *mut i32, + ) -> i32; + pub fn ct_scan_result_free(result: *mut ct_scan_result); +} diff --git a/lib/crowdb-tree/ffi/src/sys/snapshot_owner.rs b/lib/crowdb-tree/ffi/src/sys/snapshot_owner.rs new file mode 100644 index 000000000..5a8ab249a --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/snapshot_owner.rs @@ -0,0 +1,24 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::ct_export; +use std::os::raw::c_int; + +#[repr(C)] +pub struct ct_snapshot_chunk { + _private: [u8; 0], +} + +extern "C" { + pub fn ct_snapshot_export_next_v2( + export: *mut ct_export, + offset: u64, + out: *mut *mut ct_snapshot_chunk, + done: *mut c_int, + ) -> c_int; + pub fn ct_snapshot_chunk_data( + chunk: *const ct_snapshot_chunk, + data: *mut *const u8, + len: *mut usize, + ) -> c_int; + pub fn ct_snapshot_chunk_free(chunk: *mut ct_snapshot_chunk); +} diff --git a/lib/crowdb-tree/ffi/src/sys/split_overlay.rs b/lib/crowdb-tree/ffi/src/sys/split_overlay.rs new file mode 100644 index 000000000..08889a8ef --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/split_overlay.rs @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::ct_tree; +use std::os::raw::c_int; + +#[repr(C)] +pub struct ct_prepared_split_overlay { + _private: [u8; 0], +} + +#[repr(C)] +pub struct ct_split_overlay_limits_v1 { + pub version: u32, + pub struct_size: u32, + pub max_records: u64, + pub max_bytes: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_installed_split_overlay_v1 { + pub version: u32, + pub struct_size: u32, + pub generation: u64, + pub frontier: u64, +} + +unsafe extern "C" { + pub fn ct_arm_split_overlay( + source: *mut ct_tree, + destination: *mut ct_tree, + limits: *const ct_split_overlay_limits_v1, + out: *mut *mut ct_prepared_split_overlay, + start_frontier: *mut u64, + ) -> c_int; + pub fn ct_prepared_split_overlay_prepare( + prepared: *mut ct_prepared_split_overlay, + replayed_prefix: u64, + ) -> c_int; + pub fn ct_prepared_split_overlay_try_install( + prepared: *mut ct_prepared_split_overlay, + frontier: u64, + out: *mut ct_installed_split_overlay_v1, + ) -> c_int; + pub fn ct_prepared_split_overlay_committed_frontier( + prepared: *const ct_prepared_split_overlay, + out: *mut u64, + ) -> i32; + pub fn ct_prepared_split_overlay_destroy(prepared: *mut ct_prepared_split_overlay); +} diff --git a/lib/crowdb-tree/ffi/src/sys/state_stats.rs b/lib/crowdb-tree/ffi/src/sys/state_stats.rs new file mode 100644 index 000000000..d68d0c58e --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/state_stats.rs @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use super::ct_tree; + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_logical_stats_v1 { + pub leaf_pages: u64, + pub inner_pages: u64, + pub height: u64, + pub dirty_pages: u64, + pub overlay_pages: u64, + pub overlay_runs: u64, + pub overlay_entries: u64, + pub page_size_known_bytes: u64, + pub page_size_unknown_pages: u64, + pub page_size_estimate_bytes: u64, + pub page_size_max_observed_bytes: u64, + pub kv_known_count: u64, + pub kv_known_bytes: u64, + pub kv_unknown_pages: u64, + pub consolidate_candidate_pages: u64, + pub split_candidate_pages: u64, + pub split_ready_pages: u64, + pub merge_candidate_pages: u64, + pub merge_candidate_pairs: u64, + pub merge_ready_pairs: u64, + pub writeback_candidate_pages: u64, + pub evict_candidate_pages: u64, + pub checkpoint_pending_pages: u64, + pub checkpoint_mapping_pending: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_resource_stats_v1 { + pub allocated_bytes: u64, + pub reserved_bytes: u64, + pub payload_bytes: u64, + pub frame_bytes: u64, + pub metadata_bytes: u64, + pub io_bytes: u64, + pub scratch_bytes: u64, + pub retired_bytes: u64, + pub resident_frames: u64, + pub cache_bytes: u64, + pub cache_pinned_bytes: u64, + pub proxy_pages: u64, + pub read_inflight_bytes: u64, + pub write_inflight_bytes: u64, + pub materialize_inflight_bytes: u64, + pub allocated_backing_bytes: u64, + pub maintenance_queued: u64, + pub maintenance_inflight: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_stats_snapshot_v1 { + pub struct_size: u32, + pub version: u32, + pub instance_generation: u64, + pub commit_seq: u64, + pub sampled_at_ns: u64, + pub logical: ct_tree_logical_stats_v1, + pub resources: ct_tree_resource_stats_v1, +} + +extern "C" { + pub fn ct_get_tree_state_stats(tree: *mut ct_tree, out: *mut ct_tree_stats_snapshot_v1) -> i32; +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_progress_snapshot_v1 { + pub struct_size: u32, + pub version: u32, + pub instance_generation: u64, + pub commit_seq: u64, + pub sampled_at_ns: u64, + pub candidate_oldest_age_ns: u64, + pub checkpoint_oldest_pending_age_ns: u64, + pub maintenance_oldest_age_ns: u64, +} + +extern "C" { + pub fn ct_get_tree_progress_stats(tree: *mut ct_tree, out: *mut ct_tree_progress_snapshot_v1) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/wait_metrics.rs b/lib/crowdb-tree/ffi/src/sys/wait_metrics.rs new file mode 100644 index 000000000..328a51796 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/wait_metrics.rs @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ct_tree, ct_work_metrics_collector}; +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct ct_tree_wait_summary_v1 { + pub attempts: u64, + pub inflight: u64, + pub total_count: u64, + pub total_sum_ns: u64, + pub total_max_ns: u64, + pub outcomes: [u64; 3], + pub tracked: u64, + pub untracked: u64, + pub observed_oldest_age_ns: u64, + pub complete: u64, +} +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct ct_tree_wait_snapshot_v1 { + pub struct_size: u32, + pub version: u32, + pub sampled_at_ns: u64, + pub classes: [ct_tree_wait_summary_v1; 7], +} +extern "C" { + pub fn ct_tree_wait_totals(tree: *mut ct_tree, out: *mut ct_tree_wait_snapshot_v1) -> i32; + pub fn ct_work_metrics_collector_wait_totals( + collector: *mut ct_work_metrics_collector, + out: *mut ct_tree_wait_snapshot_v1, + ) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/work_histograms.rs b/lib/crowdb-tree/ffi/src/sys/work_histograms.rs new file mode 100644 index 000000000..53cbbec84 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/work_histograms.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::{ + ct_tree, ct_tree_work_ages_v1, ct_tree_work_counters_v1, ct_tree_work_snapshot_v1, + ct_work_metrics_collector, +}; + +#[repr(C)] +#[derive(Default, Clone, Copy)] +pub struct ct_tree_histogram_v1 { + pub count: u64, + pub sum_low: u64, + pub sum_high: u64, + pub min_ns: u64, + pub max_ns: u64, + pub p50_ns: u64, + pub p90_ns: u64, + pub p95_ns: u64, + pub p99_ns: u64, + pub p999_ns: u64, + pub zero_count: u64, + pub overflow_count: u64, + pub total_count: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_work_histograms_v1 { + pub struct_size: u32, + pub version: u32, + pub window_sequence: u64, + pub sampled_at_ns: u64, + pub enabled_stages: u64, + pub stages: [ct_tree_histogram_v1; 21], +} + +extern "C" { + pub fn ct_work_histogram_enable_v1(tree: *mut ct_tree, stage: u32) -> i32; + pub fn ct_work_histograms_totals_v1(tree: *mut ct_tree, out: *mut ct_tree_work_histograms_v1) -> i32; + pub fn ct_work_metrics_capture_v3( + collector: *mut ct_work_metrics_collector, + out: *mut ct_tree_work_snapshot_v1, + ages: *mut ct_tree_work_ages_v1, + counters: *mut ct_tree_work_counters_v1, + histograms: *mut ct_tree_work_histograms_v1, + ) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/sys/work_metrics.rs b/lib/crowdb-tree/ffi/src/sys/work_metrics.rs new file mode 100644 index 000000000..39c2b3a08 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/sys/work_metrics.rs @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use super::ct_tree; + +#[repr(C)] +pub struct ct_work_metrics_collector { + _private: [u8; 0], +} + +#[repr(C)] +#[derive(Default, Clone, Copy)] +pub struct ct_tree_work_stage_v1 { + pub attempts: u64, + pub inflight: u64, + pub total_count: u64, + pub total_sum_ns: u64, + pub total_max_ns: u64, + pub outcomes: [u64; 5], + pub window_count: u64, + pub window_sum_ns: u64, + pub window_max_ns: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_work_snapshot_v1 { + pub struct_size: u32, + pub version: u32, + pub window_sequence: u64, + pub sampled_at_ns: u64, + pub stages: [ct_tree_work_stage_v1; 21], +} + +#[repr(C)] +#[derive(Default, Clone, Copy)] +pub struct ct_tree_work_stage_age_v1 { + pub inflight: u64, + pub tracked: u64, + pub untracked: u64, + pub observed_oldest_age_ns: u64, + pub complete: u64, +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_work_ages_v1 { + pub struct_size: u32, + pub version: u32, + pub sampled_at_ns: u64, + pub stages: [ct_tree_work_stage_age_v1; 21], +} + +#[repr(C)] +#[derive(Default)] +pub struct ct_tree_flush_dispatch_v1 { + pub struct_size: u32, + pub version: u32, + pub sampled_at_ns: u64, + pub inline_probes: u64, + pub inline_ranges: u64, + pub queued_steps: u64, + pub prepared_runs: u64, + pub prepared_entries: u64, + pub prepared_bytes: u64, + pub cow_bytes: u64, + pub active_steps: u64, + pub peak_steps: u64, +} + +extern "C" { + pub fn ct_work_metrics_collector_create( + tree: *mut ct_tree, + out: *mut *mut ct_work_metrics_collector, + ) -> i32; + pub fn ct_work_metrics_collector_free(collector: *mut ct_work_metrics_collector); + pub fn ct_work_metrics_totals(tree: *mut ct_tree, out: *mut ct_tree_work_snapshot_v1) -> i32; + pub fn ct_work_metrics_ages(tree: *mut ct_tree, out: *mut ct_tree_work_ages_v1) -> i32; + pub fn ct_flush_dispatch_totals(tree: *mut ct_tree, out: *mut ct_tree_flush_dispatch_v1) -> i32; +} + +#[repr(C)] +pub struct ct_tree_work_counters_v1 { + pub struct_size: u32, + pub version: u32, + pub sampled_at_ns: u64, + pub values: [[u64; 55]; 8], +} +impl Default for ct_tree_work_counters_v1 { + fn default() -> Self { + Self { + struct_size: 0, + version: 0, + sampled_at_ns: 0, + values: [[0; 55]; 8], + } + } +} +extern "C" { + pub fn ct_work_counters_totals(tree: *mut ct_tree, out: *mut ct_tree_work_counters_v1) -> i32; +} diff --git a/lib/crowdb-tree/ffi/src/tree.rs b/lib/crowdb-tree/ffi/src/tree.rs index 8f36a39d5..07ca23402 100644 --- a/lib/crowdb-tree/ffi/src/tree.rs +++ b/lib/crowdb-tree/ffi/src/tree.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +mod split_overlay; +pub use split_overlay::{InstalledSplit, PreparedSplitOverlay, SplitOverlayLimits}; + use std::ffi::CString; use std::os::fd::{AsRawFd, RawFd}; use std::os::raw::c_int; @@ -12,7 +15,7 @@ use tokio::io::unix::AsyncFd; use crate::config::{Config, KeyRange, PageStoreBackend}; use crate::error::{check, take_buf, CtError}; -use crate::reactor::{drain_eventfd, EventfdPump, RawFdView}; +use crate::reactor::{drain_eventfd, duplicate_eventfd, EventfdPump}; use crate::stats::RangeRebuildStats; use crate::sys; @@ -20,6 +23,7 @@ use crate::sys; /// writes internally and keeps reads lock-free. pub struct Crowdbtree { ptr: NonNull<sys::ct_tree>, + backend_release: Option<crate::BackendReleaseService>, // Lazily-spawned eventfd pumps for this tree's DiskIOUring eventfds // (one per pipeline), shared by every concurrently-pending // drive_ct_future call. Empty once resolved means no DiskIOUring is @@ -68,6 +72,21 @@ impl Crowdbtree { /// Open (recovering durable state when `path` is set, else fresh in-memory). pub fn open(opt: &Config) -> Result<Self, CtError> { + Self::open_impl(opt, None) + } + + /// Converts a legacy Chunk root using its runtime backend and shared host. + /// Stop and drain every old-format writer in the catalog namespace first. + /// Retry an unknown commit with the same generation and a fresh catalog owner; + /// the durable migration receipt resolves it without copying the data twice. + pub fn migrate_legacy_chunk(opt: &Config, expected_generation: u64) -> Result<Self, CtError> { + if expected_generation == 0 || expected_generation == u64::MAX || opt.runtime.is_none() { + return Err(CtError::InvalidArgument); + } + Self::open_impl(opt, Some(expected_generation)) + } + + fn open_impl(opt: &Config, migration_generation: Option<u64>) -> Result<Self, CtError> { #[cfg(feature = "test-util")] ct_init_test_logging(); @@ -95,6 +114,7 @@ impl Crowdbtree { KeyRange::Unbounded => (0, None, None), KeyRange::Bounded { start, end } => (1, start.as_deref(), end.as_deref()), }; + let tuning = opt.tuning.as_raw(); let copt = sys::ct_options { page_store: opt .page_store @@ -125,13 +145,50 @@ impl Crowdbtree { log_file_prefix: clog_prefix.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()), log_max_file_mb: opt.log_max_file_mb, log_max_files: opt.log_max_files, + tuning: &tuning, }; let mut out: *mut sys::ct_tree = std::ptr::null_mut(); - check(unsafe { sys::ct_open(&copt, &mut out) })?; - Ok(Self { + let status = match (&opt.runtime, migration_generation) { + (Some(runtime), Some(generation)) => unsafe { + sys::ct_migrate_legacy_chunk_with_runtime( + &copt, + runtime.as_ptr(), + &opt.runtime_binding.as_raw(), + generation, + &mut out, + ) + }, + (Some(runtime), None) => unsafe { + sys::ct_open_with_runtime(&copt, runtime.as_ptr(), &opt.runtime_binding.as_raw(), &mut out) + }, + (None, None) => unsafe { + sys::ct_open_with_runtime( + &copt, + std::ptr::null_mut(), + &opt.runtime_binding.as_raw(), + &mut out, + ) + }, + (None, Some(_)) => return Err(CtError::InvalidArgument), + }; + check(status)?; + Self::from_opened(out) + } + + fn from_opened(out: *mut sys::ct_tree) -> Result<Self, CtError> { + let mut tree = Self { ptr: NonNull::new(out).ok_or(CtError::Internal)?, + backend_release: None, eventfd_pumps: std::sync::OnceLock::new(), - }) + }; + tree.backend_release = crate::BackendReleaseService::attach(tree.ptr)?; + Ok(tree) + } + + /// Host release handle, present for filesystem Block backends opened from + /// a Tokio context. Retain it through host shutdown after all readers drop. + pub fn backend_release_service(&self) -> Option<crate::BackendReleaseService> { + self.backend_release.clone() } pub(crate) fn as_ptr(&self) -> *mut sys::ct_tree { @@ -159,6 +216,7 @@ impl Crowdbtree { KeyRange::Unbounded => (0, None, None), KeyRange::Bounded { start, end } => (1, start.as_deref(), end.as_deref()), }; + let tuning = opt.tuning.as_raw(); let copt = sys::ct_options { page_store: store.as_ptr(), range_bounded, @@ -182,14 +240,34 @@ impl Crowdbtree { log_file_prefix: std::ptr::null(), log_max_file_mb: 0, log_max_files: 0, + tuning: &tuning, }; let mut out = std::ptr::null_mut(); let mut raw = sys::ct_range_rebuild_stats::default(); - check(unsafe { sys::ct_rebuild_range(self.as_ptr(), &copt, &mut out, &mut raw) })?; - let tree = Self { - ptr: NonNull::new(out).ok_or(CtError::Internal)?, - eventfd_pumps: std::sync::OnceLock::new(), + let status = match &opt.runtime { + Some(runtime) => unsafe { + sys::ct_rebuild_range_with_runtime( + self.as_ptr(), + &copt, + runtime.as_ptr(), + &opt.runtime_binding.as_raw(), + &mut out, + &mut raw, + ) + }, + None => unsafe { + sys::ct_rebuild_range_with_runtime( + self.as_ptr(), + &copt, + std::ptr::null_mut(), + &opt.runtime_binding.as_raw(), + &mut out, + &mut raw, + ) + }, }; + check(status)?; + let tree = Self::from_opened(out)?; Ok(( tree, RangeRebuildStats { @@ -305,6 +383,12 @@ impl Crowdbtree { check(unsafe { sys::ct_release_split_memtable_view(self.as_ptr(), generation) }) } + /// Exact checkpoint coverage. A flush only publishes memory and does not + /// advance this value. Volatile Trees remain at zero. + pub fn durable_coverage(&self) -> u64 { + unsafe { sys::ct_durable_coverage(self.as_ptr()) } + } + pub fn last_applied_slot(&self) -> u64 { unsafe { sys::ct_last_applied_slot(self.as_ptr()) } } @@ -410,7 +494,7 @@ impl Crowdbtree { /// The tree's DiskIOUring eventfds (one per pipeline), or empty if /// none is wired (an in-memory tree, or a build without liburing). - /// DiskIOUring-owned -- see `RawFdView`'s doc comment for why callers + /// DiskIOUring-owned -- callers /// must never close them. fn uring_eventfds(&self) -> Vec<RawFd> { // Query the count first, then fetch the fds. @@ -435,8 +519,7 @@ impl Crowdbtree { /// Lazily spawns (once) and returns the `Notify` fanned out by this /// tree's eventfd pumps -- one pump per pipeline eventfd, all sharing - /// a single `Arc<Notify>`. See the module-level fan-out note above - /// `RawFdView` for why a single Notify (not a per-future registration) + /// a single `Arc<Notify>`. See the module-level fan-out note in the reactor module for why a single Notify (not a per-future registration) /// is required. `None` if there's no DiskIOUring wired, or all pumps /// failed to spawn. Must be called from within a Tokio runtime context; /// every call site is inside an async fn body driven by one, so this @@ -450,7 +533,12 @@ impl Crowdbtree { let notify = Arc::new(tokio::sync::Notify::new()); let mut pumps = Vec::with_capacity(fds.len()); for fd in fds { - let async_fd = match AsyncFd::new(RawFdView(fd)) { + // SAFETY: self retains the native owner throughout duplication. + let owned_fd = match unsafe { duplicate_eventfd(fd) } { + Ok(fd) => fd, + Err(_) => continue, + }; + let async_fd = match AsyncFd::new(owned_fd) { Ok(af) => af, Err(_) => continue, // skip this pump, try the rest }; @@ -510,10 +598,9 @@ impl Crowdbtree { impl Drop for Crowdbtree { fn drop(&mut self) { - // Stop all eventfd pumps (if any) before ct_close tears down the - // DiskIOUring -- once that happens the eventfds are closed and the - // raw fd numbers may be reused elsewhere in the process, so the - // pumps must not still be waiting on them. + // Cancel pumps before native teardown. Their independently owned fds + // stay live until task cancellation has deregistered them from Tokio, + // so opening another Tree immediately cannot reuse an active fd. if let Some(pumps) = self.eventfd_pumps.get() { for pump in pumps { pump.task.abort(); diff --git a/lib/crowdb-tree/ffi/src/tree/split_overlay.rs b/lib/crowdb-tree/ffi/src/tree/split_overlay.rs new file mode 100644 index 000000000..8c0a1a443 --- /dev/null +++ b/lib/crowdb-tree/ffi/src/tree/split_overlay.rs @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crate::{error::check, sys, Crowdbtree, CtError}; +use std::{ptr::NonNull, sync::Arc}; + +#[derive(Clone, Copy, Debug)] +pub struct SplitOverlayLimits { + pub max_records: u64, + pub max_bytes: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InstalledSplit { + pub generation: u64, + pub frontier: u64, +} + +/// An owned native installation capability; dropping it cancels an unused lease. +pub struct PreparedSplitOverlay { + raw: NonNull<sys::ct_prepared_split_overlay>, + start_frontier: u64, + _source: Arc<Crowdbtree>, + _destination: Arc<Crowdbtree>, +} + +// SAFETY: the native capability serializes state transitions atomically and +// supports concurrent read-only queries. Its Tree owners outlive every call. +unsafe impl Send for PreparedSplitOverlay {} +unsafe impl Sync for PreparedSplitOverlay {} + +impl Crowdbtree { + /// Prepare all source, destination, and allocation work on a host thread. + /// The destination's durable checkpoint stays unchanged. + /// + /// # Errors + /// Returns an incompatible destination, exhausted budget, or closed source. + pub fn arm_split_overlay( + self: &Arc<Self>, + destination: Arc<Self>, + limits: SplitOverlayLimits, + ) -> Result<PreparedSplitOverlay, CtError> { + let limits = sys::ct_split_overlay_limits_v1 { + version: 1, + struct_size: u32::try_from(std::mem::size_of::<sys::ct_split_overlay_limits_v1>()) + .map_err(|_| CtError::Internal)?, + max_records: limits.max_records, + max_bytes: limits.max_bytes, + }; + let mut raw = std::ptr::null_mut(); + let mut start_frontier = 0; + check(unsafe { + sys::ct_arm_split_overlay( + self.as_ptr(), + destination.as_ptr(), + &raw const limits, + &raw mut raw, + &raw mut start_frontier, + ) + })?; + Ok(PreparedSplitOverlay { + raw: NonNull::new(raw).ok_or(CtError::Internal)?, + start_frontier, + _source: self.clone(), + _destination: destination, + }) + } +} + +impl PreparedSplitOverlay { + #[must_use] + pub const fn start_frontier(&self) -> u64 { + self.start_frontier + } + + /// Finish destination preparation after replay reaches the observer start. + /// + /// # Errors + /// Returns an expired observer or changed destination. + pub fn prepare(&self, replayed_prefix: u64) -> Result<(), CtError> { + check(unsafe { sys::ct_prepared_split_overlay_prepare(self.raw.as_ptr(), replayed_prefix) }) + } + + /// Atomically commit one exact frontier without I/O, allocation, or waiting. + /// + /// # Errors + /// Returns a busy, expired, or conflicting installation without publishing it. + pub fn try_install(&self, frontier: u64) -> Result<InstalledSplit, CtError> { + let mut result = sys::ct_installed_split_overlay_v1 { + version: 1, + struct_size: u32::try_from(std::mem::size_of::<sys::ct_installed_split_overlay_v1>()) + .map_err(|_| CtError::Internal)?, + ..Default::default() + }; + check(unsafe { + sys::ct_prepared_split_overlay_try_install(self.raw.as_ptr(), frontier, &raw mut result) + })?; + Ok(InstalledSplit { + generation: result.generation, + frontier: result.frontier, + }) + } + + #[must_use] + pub fn committed_frontier(&self) -> Option<u64> { + let mut frontier = 0; + (unsafe { sys::ct_prepared_split_overlay_committed_frontier(self.raw.as_ptr(), &raw mut frontier) } + != 0) + .then_some(frontier) + } +} + +impl Drop for PreparedSplitOverlay { + fn drop(&mut self) { + unsafe { sys::ct_prepared_split_overlay_destroy(self.raw.as_ptr()) }; + } +} diff --git a/lib/crowdb-tree/ffi/src/tuning.rs b/lib/crowdb-tree/ffi/src/tuning.rs new file mode 100644 index 000000000..acd0df98a --- /dev/null +++ b/lib/crowdb-tree/ffi/src/tuning.rs @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +//! Engine work windows used by open and range rebuild. +use crate::sys; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct TreeTuningOptions { + pub memtable_flush_bytes: usize, + pub memtable_flush_entries: u32, + pub max_memtable_count: u32, + pub leaf_split_bytes: usize, + pub leaf_merge_bytes: usize, + pub inner_max_keys: u32, + pub inner_merge_keys: u32, + pub max_key_size: usize, + pub source_max_entries: usize, + pub source_max_key_bytes: usize, + pub source_max_gap_entries: usize, + pub source_max_gap_key_bytes: usize, + pub flush_parallelism: usize, + pub run_max_entries: usize, + pub run_max_key_bytes: usize, + pub run_key_block_bytes: usize, + pub run_max_bytes: usize, + pub install_run_window: usize, + pub maintenance_visits_per_step: usize, + pub maintenance_max_visits: usize, + pub maintenance_max_candidates: usize, + pub maintenance_max_steps: usize, + pub maintenance_metadata_bytes: usize, + pub cache_max_pages: usize, +} + +impl Default for TreeTuningOptions { + fn default() -> Self { + let mut raw = sys::ct_tree_tuning::default(); + unsafe { sys::ct_tree_tuning_default(&mut raw) }; + Self { + memtable_flush_bytes: raw.memtable_flush_bytes, + memtable_flush_entries: raw.memtable_flush_entries, + max_memtable_count: raw.max_memtable_count, + leaf_split_bytes: raw.leaf_split_bytes, + leaf_merge_bytes: raw.leaf_merge_bytes, + inner_max_keys: raw.inner_max_keys, + inner_merge_keys: raw.inner_merge_keys, + max_key_size: raw.max_key_size, + source_max_entries: raw.source_max_entries, + source_max_key_bytes: raw.source_max_key_bytes, + source_max_gap_entries: raw.source_max_gap_entries, + source_max_gap_key_bytes: raw.source_max_gap_key_bytes, + flush_parallelism: raw.flush_parallelism, + run_max_entries: raw.run_max_entries, + run_max_key_bytes: raw.run_max_key_bytes, + run_key_block_bytes: raw.run_key_block_bytes, + run_max_bytes: raw.run_max_bytes, + install_run_window: raw.install_run_window, + maintenance_visits_per_step: raw.maintenance_visits_per_step, + maintenance_max_visits: raw.maintenance_max_visits, + maintenance_max_candidates: raw.maintenance_max_candidates, + maintenance_max_steps: raw.maintenance_max_steps, + maintenance_metadata_bytes: raw.maintenance_metadata_bytes, + cache_max_pages: raw.cache_max_pages, + } + } +} + +impl TreeTuningOptions { + pub(crate) fn as_raw(self) -> sys::ct_tree_tuning { + sys::ct_tree_tuning { + struct_size: std::mem::size_of::<sys::ct_tree_tuning>(), + memtable_flush_bytes: self.memtable_flush_bytes, + memtable_flush_entries: self.memtable_flush_entries, + max_memtable_count: self.max_memtable_count, + leaf_split_bytes: self.leaf_split_bytes, + leaf_merge_bytes: self.leaf_merge_bytes, + inner_max_keys: self.inner_max_keys, + inner_merge_keys: self.inner_merge_keys, + max_key_size: self.max_key_size, + source_max_entries: self.source_max_entries, + source_max_key_bytes: self.source_max_key_bytes, + source_max_gap_entries: self.source_max_gap_entries, + source_max_gap_key_bytes: self.source_max_gap_key_bytes, + flush_parallelism: self.flush_parallelism, + run_max_entries: self.run_max_entries, + run_max_key_bytes: self.run_max_key_bytes, + run_key_block_bytes: self.run_key_block_bytes, + run_max_bytes: self.run_max_bytes, + install_run_window: self.install_run_window, + maintenance_visits_per_step: self.maintenance_visits_per_step, + maintenance_max_visits: self.maintenance_max_visits, + maintenance_max_candidates: self.maintenance_max_candidates, + maintenance_max_steps: self.maintenance_max_steps, + maintenance_metadata_bytes: self.maintenance_metadata_bytes, + cache_max_pages: self.cache_max_pages, + } + } +} diff --git a/lib/crowdb-tree/ffi/src/write_handle.rs b/lib/crowdb-tree/ffi/src/write_handle.rs index 7c4422531..6640ce493 100644 --- a/lib/crowdb-tree/ffi/src/write_handle.rs +++ b/lib/crowdb-tree/ffi/src/write_handle.rs @@ -14,9 +14,18 @@ use crate::tree::Crowdbtree; /// /// `!Send + !Sync`: the handle's internal pointers are not safe to /// share across threads. -pub struct WriteHandle { +/// The handle borrows its tree, which must remain alive through apply/cancel. +/// +/// ```compile_fail +/// use crowdb_tree_ffi::{Config, Crowdbtree}; +/// let tree = Crowdbtree::open(&Config::default()).unwrap(); +/// let handle = tree.alloc_put(1, 1).unwrap(); +/// drop(tree); +/// handle.apply(1).unwrap(); +/// ``` +pub struct WriteHandle<'tree> { ptr: *mut sys::ct_write_handle, - tree: *mut sys::ct_tree, + tree: &'tree Crowdbtree, key_ptr: *mut u8, val_ptr: *mut u8, key_len: usize, @@ -24,7 +33,7 @@ pub struct WriteHandle { consumed: bool, } -impl std::fmt::Debug for WriteHandle { +impl std::fmt::Debug for WriteHandle<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("WriteHandle") .field("key_len", &self.key_len) @@ -34,7 +43,7 @@ impl std::fmt::Debug for WriteHandle { } } -impl WriteHandle { +impl WriteHandle<'_> { /// Writable key slice `[0, key_len)`. pub fn key_mut(&mut self) -> &mut [u8] { if self.consumed || self.key_len == 0 || self.key_ptr.is_null() { @@ -58,11 +67,11 @@ impl WriteHandle { return Err(CtError::InvalidArgument); } self.consumed = true; - check(unsafe { sys::ct_apply_put_owned(self.tree, slot, self.ptr) }) + check(unsafe { sys::ct_apply_put_owned(self.tree.as_ptr(), slot, self.ptr) }) } } -impl Drop for WriteHandle { +impl Drop for WriteHandle<'_> { fn drop(&mut self) { if !self.consumed { unsafe { sys::ct_free_handle(self.ptr) }; @@ -75,7 +84,7 @@ impl Crowdbtree { /// [`WriteHandle`] whose `key_mut`/`value_mut` slices the caller writes /// into directly, then [`WriteHandle::apply`] consumes with zero value /// memcpy. - pub fn alloc_put(&self, key_len: usize, val_len: usize) -> Result<WriteHandle, CtError> { + pub fn alloc_put(&self, key_len: usize, val_len: usize) -> Result<WriteHandle<'_>, CtError> { let mut handle: *mut sys::ct_write_handle = std::ptr::null_mut(); let mut ptrs = sys::ct_write_ptrs { key: std::ptr::null_mut(), @@ -84,7 +93,7 @@ impl Crowdbtree { check(unsafe { sys::ct_alloc(self.as_ptr(), key_len, val_len, &mut handle, &mut ptrs) })?; Ok(WriteHandle { ptr: handle, - tree: self.as_ptr(), + tree: self, key_ptr: ptrs.key, val_ptr: ptrs.val, key_len, diff --git a/lib/crowdb-tree/ffi/tests/backend_release_test.rs b/lib/crowdb-tree/ffi/tests/backend_release_test.rs new file mode 100644 index 000000000..d6bdd574a --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/backend_release_test.rs @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{Config, Crowdbtree, PageStoreBackend, TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +#[tokio::test] +async fn block_host_driver_survives_tree_and_cpu_runtime_then_joins_real_release() { + let path = std::env::temp_dir().join(format!("crowdb-block-release-{}", std::process::id())); + std::fs::create_dir_all(&path).unwrap(); + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let tree = Arc::new( + Crowdbtree::open(&Config { + runtime: Some(runtime.clone()), + path: Some(path.to_string_lossy().into_owned()), + backend: PageStoreBackend::Block, + block_size: 1 << 20, + ..Default::default() + }) + .unwrap(), + ); + let service = tree.backend_release_service().unwrap(); + tree.apply_put(1, b"key", b"value").unwrap(); + tree.snapshot().unwrap(); + let mut snapshot = tree.snapshot_export_begin(64).unwrap(); + drop(tree); + drop(runtime); + assert!(!snapshot.read_owned(0).unwrap().bytes.is_empty()); + drop(snapshot); + service.shutdown().await; + service.shutdown().await; + std::fs::remove_dir_all(path).unwrap(); +} + +#[tokio::test] +async fn memory_backend_does_not_start_a_release_driver() { + let tree = Crowdbtree::open(&Config::default()).unwrap(); + assert!(tree.backend_release_service().is_none()); +} diff --git a/lib/crowdb-tree/ffi/tests/backing_stats_test.rs b/lib/crowdb-tree/ffi/tests/backing_stats_test.rs new file mode 100644 index 000000000..93ef0e217 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/backing_stats_test.rs @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{Config, Crowdbtree, PageStore, TreeRuntime}; +use std::sync::Arc; + +#[test] +fn collector_observes_local_backing_after_tree_close_without_pinning_storage() { + let runtime = Arc::new(TreeRuntime::new(Default::default()).unwrap()); + let mut config = Config { + runtime: Some(runtime.clone()), + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + frame_bytes: 4096, + max_inline_value: 64, + ..Default::default() + }; + let tree = Crowdbtree::open(&config).unwrap(); + tree.apply_put(1, b"key", &vec![b'v'; 9000]).unwrap(); + let (sequence, coverage) = tree.snapshot_info().unwrap(); + let snapshot = tree.immutable_snapshot(sequence, coverage).unwrap(); + let collector = tree.work_metrics_collector().unwrap(); + let before = tree.backing_stats().unwrap(); + assert_ne!(before.scope_id, 0); + assert!(before.allocated_bytes > 0); + assert_eq!(collector.backing_stats().unwrap(), before); + drop(tree); + let retained = collector.backing_stats().unwrap(); + assert_eq!(retained.scope_id, before.scope_id); + assert!(retained.allocated_bytes > 0); + assert_eq!(snapshot.get(b"key").unwrap().unwrap().1, vec![b'v'; 9000]); + drop(snapshot); + runtime.drain_cpu(); + assert_eq!(collector.backing_stats().unwrap().allocated_bytes, 0); + // Host drain stops admission. A new host can claim the same backend while + // the old sampler remains alive and owns no storage domain. + config.runtime = Some(Arc::new(TreeRuntime::new(Default::default()).unwrap())); + let reopened = Crowdbtree::open(&config).unwrap(); + assert_ne!(reopened.backing_stats().unwrap().scope_id, before.scope_id); + drop(reopened); + drop(config); + drop(runtime); + assert_eq!(collector.backing_stats().unwrap().allocated_bytes, 0); +} diff --git a/lib/crowdb-tree/ffi/tests/chunk_pin_service_test.rs b/lib/crowdb-tree/ffi/tests/chunk_pin_service_test.rs new file mode 100644 index 000000000..df800d5ad --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/chunk_pin_service_test.rs @@ -0,0 +1,28 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{chunk::pin_service::ChunkPinReleaseService, CtError, TreeRuntime}; +#[tokio::test] +async fn pin_service_has_no_native_runtime_owner_and_joins_its_host_task() { + let runtime = TreeRuntime::new(Default::default()).unwrap(); + let service = ChunkPinReleaseService::new(&runtime, 64).unwrap(); + drop(runtime); + let before = service.resource_usage().unwrap(); + assert_eq!(before.limit, 64); + assert_eq!(before.used, 0); + assert_eq!(before.waiters, 0); + assert_eq!(before.denied_host, 0); + assert_eq!(service.resource_usage().unwrap(), before); + service.shutdown().await; + assert_eq!(service.resource_usage().unwrap(), before); + service.shutdown().await; +} +#[tokio::test] +async fn invalid_pin_capacity_does_not_consume_host_queue_context() { + let runtime = TreeRuntime::new(Default::default()).unwrap(); + assert!(matches!( + ChunkPinReleaseService::new(&runtime, 0), + Err(CtError::InvalidArgument) + )); + let valid = ChunkPinReleaseService::new(&runtime, 1).unwrap(); + valid.shutdown().await; +} diff --git a/lib/crowdb-tree/ffi/tests/chunk_stage_test.rs b/lib/crowdb-tree/ffi/tests/chunk_stage_test.rs new file mode 100644 index 000000000..efce4d6f1 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/chunk_stage_test.rs @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_tree_ffi::chunk::stage::{validate_stage_transition, StageId, StagePack, StageRecord, StageState}; +use crowdb_tree_ffi::{ChunkRootCatalog, CtError}; + +fn reserved() -> StageRecord { + StageRecord { + tree_id: 42, + id: StageId { + owner_epoch: 7, + operation_high: 8, + operation_low: 9, + }, + revision: 1, + chunk_high: 2_u64 << 56, + chunk_low: 123, + capacity: 4096, + purpose: 2, + state: StageState::Reserved, + manifest_generation: 0, + packs: vec![], + } +} +#[test] +fn versioned_stage_wire_preserves_identity_and_rejects_corruption() { + let mut record = reserved(); + record.state = StageState::Writing; + record.revision = 2; + record.packs.push(StagePack { + offset: 0, + length: 4096, + checksum: 0x12345678, + }); + let bytes = record.encode().unwrap(); + assert_eq!(&bytes[..8], &[b'C', b'T', b'S', b'G', 1, 0, 0, 0]); + assert_eq!(bytes.len(), 116); + assert_eq!(StageRecord::decode(&bytes), Ok(record)); + for offset in [8, 40, 64, 84, 88, 96, 112] { + let mut corrupt = bytes.clone(); + corrupt[offset] ^= 1; + assert_eq!(StageRecord::decode(&corrupt), Err(CtError::Corruption)); + } + let mut version = bytes.clone(); + version[4] = 2; + assert_eq!(StageRecord::decode(&version), Err(CtError::NotSupported)); + assert_eq!( + StageRecord::decode(&bytes[..bytes.len() - 1]), + Err(CtError::Corruption) + ); +} +#[test] +fn transitions_keep_provisional_identity_and_never_treat_unknown_write_as_reclaimed() { + let first = reserved(); + assert!(validate_stage_transition(None, Some(&first)).is_ok()); + let mut next = first.clone(); + next.revision = 2; + next.state = StageState::Writing; + assert!(validate_stage_transition(Some(&first), Some(&next)).is_ok()); + assert_eq!( + validate_stage_transition(Some(&next), None), + Err(CtError::InvalidArgument) + ); + let mut changed = next.clone(); + changed.chunk_low += 1; + assert_eq!( + validate_stage_transition(Some(&first), Some(&changed)), + Err(CtError::InvalidArgument) + ); + let mut complete = next.clone(); + complete.revision = 3; + complete.state = StageState::Reloadable; + assert_eq!( + validate_stage_transition(Some(&next), Some(&complete)), + Err(CtError::InvalidArgument) + ); + complete.packs.push(StagePack { + offset: 0, + length: 4096, + checksum: 0, + }); + assert!(validate_stage_transition(Some(&next), Some(&complete)).is_ok()); + complete.packs.push(StagePack { + offset: 1, + length: 1, + checksum: 0, + }); + assert_eq!(complete.validate(), Err(CtError::InvalidArgument)); +} +#[test] +fn legacy_catalog_explicitly_disables_staging_records() { + let catalog = ChunkRootCatalog::open_memory(1).unwrap(); + assert!(!catalog.supports_staging()); + let first = reserved(); + assert_eq!(catalog.load_stage(42, first.id), Err(CtError::NotSupported)); + assert_eq!( + catalog.compare_stage(42, 7, None, Some(&first)), + Err(CtError::NotSupported) + ); + assert_eq!(catalog.list_stages(42, None, 1), Err(CtError::NotSupported)); +} diff --git a/lib/crowdb-tree/ffi/tests/common/external_owner.rs b/lib/crowdb-tree/ffi/tests/common/external_owner.rs new file mode 100644 index 000000000..58893d2f4 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/common/external_owner.rs @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use bytes::Bytes; + +struct TestOwner { + data: Vec<u8>, + drops: Arc<AtomicUsize>, +} + +impl AsRef<[u8]> for TestOwner { + fn as_ref(&self) -> &[u8] { + &self.data + } +} + +impl Drop for TestOwner { + fn drop(&mut self) { + self.drops.fetch_add(1, Ordering::SeqCst); + } +} + +pub fn payload(length: usize, drops: &Arc<AtomicUsize>) -> Bytes { + Bytes::from_owner(TestOwner { + data: vec![0xA7; length], + drops: Arc::clone(drops), + }) +} diff --git a/lib/crowdb-tree/ffi/tests/dispatch_metrics_test.rs b/lib/crowdb-tree/ffi/tests/dispatch_metrics_test.rs new file mode 100644 index 000000000..3d55f7c31 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/dispatch_metrics_test.rs @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_tree_ffi::{Config, Crowdbtree, TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +#[test] +fn dispatch_totals_follow_real_flushes_without_draining_collector_windows() { + let tree = Crowdbtree::open(&Config { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }) + .unwrap(); + let collector = tree.work_metrics_collector().unwrap(); + assert_eq!(tree.flush_dispatch_stats().unwrap().prepared_entries, 0); + tree.apply_put(1, b"key", b"first").unwrap(); + tree.flush().unwrap(); + let first = tree.flush_dispatch_stats().unwrap(); + assert_eq!(first.prepared_entries, 1); + assert!(first.prepared_runs > 0); + assert!(first.prepared_bytes >= 8); + assert!(first.cow_bytes > 0); + assert!(first.peak_steps > 0); + assert!(first.active_steps <= first.peak_steps); + let again = tree.flush_dispatch_stats().unwrap(); + assert_eq!(again.prepared_entries, first.prepared_entries); + let window = collector.capture().unwrap(); + assert_eq!(window.window_sequence, 1); + assert!(window.stages.iter().any(|stage| stage.window_count > 0)); + tree.apply_put(2, b"next", b"second").unwrap(); + tree.flush().unwrap(); + let second = tree.flush_dispatch_stats().unwrap(); + assert_eq!(second.prepared_entries, 2); + assert!(second.cow_bytes > first.cow_bytes); + assert!(second.sampled_at_ns >= first.sampled_at_ns); + let standalone = Crowdbtree::open(&Config::default()).unwrap(); + standalone.apply_put(1, b"default", b"value").unwrap(); + standalone.flush().unwrap(); + assert_eq!(standalone.flush_dispatch_stats().unwrap().prepared_entries, 1); +} diff --git a/lib/crowdb-tree/ffi/tests/external_owner_test.rs b/lib/crowdb-tree/ffi/tests/external_owner_test.rs new file mode 100644 index 000000000..47e60d77d --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/external_owner_test.rs @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_tree_ffi::{Config, Crowdbtree, CtError, ExtOp}; + +#[path = "common/external_owner.rs"] +mod external_owner; +use external_owner::payload; + +#[test] +fn unknown_capacity_slice_releases_large_backing_during_apply() { + let tree = Crowdbtree::open(&Config::default()).unwrap(); + let drops = Arc::new(AtomicUsize::new(0)); + let backing = payload(1024 * 1024, &drops); + let value = backing.slice(500_000..500_064); + drop(backing); + tree.apply_batch_external( + 1, + vec![ExtOp::Put { + key: Bytes::from_static(b"slice"), + value, + }], + ) + .unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(tree.get(b"slice").unwrap(), Some((1, vec![0xA7; 64]))); + tree.flush().unwrap(); + assert_eq!(tree.get(b"slice").unwrap(), Some((1, vec![0xA7; 64]))); + drop(tree); + assert_eq!(drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn cloned_slices_across_trees_release_last_rust_owner_once() { + let first = Crowdbtree::open(&Config::default()).unwrap(); + let second = Crowdbtree::open(&Config::default()).unwrap(); + let drops = Arc::new(AtomicUsize::new(0)); + let backing = payload(65_536, &drops); + let left = backing.slice(100..200); + let right = backing.slice(200..400); + drop(backing); + first + .apply_batch_external( + 1, + vec![ExtOp::Put { + key: Bytes::from_static(b"first"), + value: left, + }], + ) + .unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 0); + second + .apply_batch_external( + 1, + vec![ExtOp::Put { + key: Bytes::from_static(b"second"), + value: right, + }], + ) + .unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(first.get(b"first").unwrap(), Some((1, vec![0xA7; 100]))); + assert_eq!(second.get(b"second").unwrap(), Some((1, vec![0xA7; 200]))); +} + +#[test] +fn rejected_batch_releases_every_transferred_owner_without_partial_apply() { + let tree = Crowdbtree::open(&Config::default()).unwrap(); + let first = Arc::new(AtomicUsize::new(0)); + let second = Arc::new(AtomicUsize::new(0)); + let ops = vec![ + ExtOp::Put { + key: Bytes::from_static(b"valid"), + value: payload(1024, &first), + }, + ExtOp::Put { + key: Bytes::from(vec![b'k'; 65_536]), + value: payload(2048, &second), + }, + ]; + assert_eq!(tree.apply_batch_external(1, ops), Err(CtError::InvalidArgument)); + assert_eq!(first.load(Ordering::SeqCst), 1); + assert_eq!(second.load(Ordering::SeqCst), 1); + assert_eq!(tree.get(b"valid").unwrap(), None); +} + +#[test] +fn cancelling_unsubmitted_clones_and_write_handle_preserves_tree() { + let drops = Arc::new(AtomicUsize::new(0)); + let value = payload(1024, &drops); + let ops = vec![ExtOp::Put { + key: Bytes::from_static(b"cancel"), + value: value.clone(), + }]; + drop(value); + drop(ops); + assert_eq!(drops.load(Ordering::SeqCst), 1); + + let tree = Crowdbtree::open(&Config::default()).unwrap(); + let mut handle = tree.alloc_put(6, 1024).unwrap(); + handle.key_mut().copy_from_slice(b"cancel"); + handle.value_mut().fill(0xA7); + drop(handle); + assert_eq!(tree.get(b"cancel").unwrap(), None); + let mut handle = tree.alloc_put(2, 2).unwrap(); + handle.key_mut().copy_from_slice(b"ok"); + handle.value_mut().copy_from_slice(b"ok"); + handle.apply(1).unwrap(); + assert_eq!(tree.get(b"ok").unwrap(), Some((1, b"ok".to_vec()))); +} + +#[test] +fn empty_value_owner_is_released_before_returning() { + let tree = Crowdbtree::open(&Config::default()).unwrap(); + let drops = Arc::new(AtomicUsize::new(0)); + tree.apply_batch_external( + 1, + vec![ExtOp::Put { + key: Bytes::from_static(b"empty"), + value: payload(0, &drops), + }], + ) + .unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(tree.get(b"empty").unwrap(), Some((1, vec![]))); +} diff --git a/lib/crowdb-tree/ffi/tests/ffi_test.rs b/lib/crowdb-tree/ffi/tests/ffi_test.rs index 8f9f8c5f8..c2e14b417 100644 --- a/lib/crowdb-tree/ffi/tests/ffi_test.rs +++ b/lib/crowdb-tree/ffi/tests/ffi_test.rs @@ -11,6 +11,34 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; +// Independent test fixtures must not compete for the process-default host's +// finite quotas. Trees inside one fixture retain the same explicitly configured +// host; dedicated runtime tests exercise sharing and exhaustion. +fn isolated_config(config: &Config) -> Config { + let mut config = config.clone(); + if config.runtime.is_none() { + if config.path.is_none() && config.page_store.is_none() { + config.page_store = Some(Arc::new(PageStore::open_mem(1).unwrap())); + } + config.runtime = Some(Arc::new( + crowdb_tree_ffi::TreeRuntime::new(crowdb_tree_ffi::TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )); + } + config +} + +fn open_tree(config: &Config) -> Result<Crowdbtree, CtError> { + Crowdbtree::open(&isolated_config(config)) +} + +fn open_async_tree(config: &Config) -> Result<AsyncCrowdbtree, CtError> { + AsyncCrowdbtree::open(&isolated_config(config)) +} + struct FileRootCatalogStore { dir: PathBuf, generation: AtomicU64, @@ -186,7 +214,7 @@ fn owned_chunk_rpc_transport_retains_route_handles() { #[test] fn mem_apply_get_scan() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); for i in 0..40usize { let v = format!("v{i}").into_bytes(); t.apply_put((i + 1) as u64, &key(i), &v).unwrap(); @@ -211,7 +239,7 @@ fn mem_apply_get_scan() { #[test] fn scan_from_honors_inclusive_and_exclusive_lower_bounds() { - let tree = Crowdbtree::open(&Config::default()).unwrap(); + let tree = open_tree(&Config::default()).unwrap(); for (slot, key) in [b"".as_slice(), b"b"].into_iter().enumerate() { tree.apply_put(slot as u64 + 1, key, b"value").unwrap(); } @@ -236,7 +264,7 @@ fn scan_from_honors_inclusive_and_exclusive_lower_bounds() { #[test] fn scan_start_after_pages_with_the_unified_scan_ffi() { - let tree = Crowdbtree::open(&Config::default()).unwrap(); + let tree = open_tree(&Config::default()).unwrap(); for index in 0..300_u64 { tree.apply_put(index + 1, format!("key-{index:04}").as_bytes(), b"value") .unwrap(); @@ -258,7 +286,7 @@ fn scan_start_after_pages_with_the_unified_scan_ffi() { #[test] fn reverse_seek_merges_l0_l1_and_tombstones() { - let tree = Crowdbtree::open(&Config::default()).unwrap(); + let tree = open_tree(&Config::default()).unwrap(); for (slot, key) in [b"".as_slice(), b"b", b"d"].into_iter().enumerate() { tree.apply_put(slot as u64 + 1, key, b"value").unwrap(); } @@ -293,7 +321,7 @@ fn reverse_seek_merges_l0_l1_and_tombstones() { #[test] fn reverse_scan_is_descending_bounded_and_tombstone_aware() { - let tree = Crowdbtree::open(&Config::default()).unwrap(); + let tree = open_tree(&Config::default()).unwrap(); for (slot, key) in [b"a", b"b", b"c", b"d"].into_iter().enumerate() { tree.apply_put(slot as u64 + 1, key, key).unwrap(); } @@ -325,7 +353,7 @@ fn reverse_scan_is_descending_bounded_and_tombstone_aware() { #[test] fn reverse_scan_crosses_leaf_boundaries() { - let tree = Crowdbtree::open(&Config { + let tree = open_tree(&Config { frame_bytes: 4096, ..Config::default() }) @@ -345,107 +373,59 @@ fn reverse_scan_crosses_leaf_boundaries() { assert_eq!(entries.last().unwrap().key.as_ref(), b"key-0000"); } -#[test] -fn injected_chunk_store_round_trip_and_stats() { - let catalog = Arc::new(ChunkRootCatalog::open_memory(7).unwrap()); - let store = Arc::new( - PageStore::open_chunk( - ChunkPageStoreOptions { - tree_id: 41, - owner_epoch: 7, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - materialization_bytes_per_pass: 4096, - max_chunk_bytes: 0, - mirror_copies: 0, - }, - Arc::clone(&catalog), - None, - ) - .unwrap(), - ); - let tree = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&store)), - ..Config::default() +fn open_runtime_chunk( + backend: &crowdb_tree_ffi::chunk::test_util::TestChunkBackend, + generation: u64, +) -> (Arc<PageStore>, Crowdbtree) { + let store = backend.page_store(generation).unwrap(); + let tree = open_tree(&Config { + page_store: Some(store.clone()), + ..backend.config() }) .unwrap(); + (store, tree) +} + +#[test] +fn injected_chunk_runtime_round_trip_and_backing_stats() { + let backend = crowdb_tree_ffi::chunk::test_util::TestChunkBackend::new(41, 7).unwrap(); + let (store, tree) = open_runtime_chunk(&backend, 0); tree.apply_put(1, b"chunk-key", b"chunk-value").unwrap(); let published = tree.snapshot_info().unwrap(); assert_eq!(published.0, 1); assert_eq!(tree.snapshot_state().unwrap(), published); - assert_eq!(tree.snapshot_state().unwrap(), published); assert_eq!( tree.get(b"chunk-key").unwrap(), Some((1, b"chunk-value".to_vec())) ); - let stats = store.chunk_stats().unwrap(); - assert_eq!(stats.generations_published, 1); - assert!(stats.packs_written > 0); - assert!(stats.pack_bytes_written > 0); - assert!(stats.diskio_operations > 0); - assert!(stats.rpc_operations > 0); + let stats = tree.backing_stats().unwrap(); + assert!(stats.allocated_bytes > 0); assert_eq!(tree.materialize_ownership().unwrap(), (0, true)); + assert_eq!(store.chunk_manifest_generation().unwrap(), 1); + assert_eq!(store.chunk_estimated_bytes(), Err(CtError::NotSupported)); + assert!(matches!(store.chunk_stats(), Err(CtError::NotSupported))); } #[test] -fn callback_root_catalog_reopens_published_manifest() { - let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-callback-root"); - let backend = Arc::new(FileRootCatalogStore { - dir: dir.path().to_path_buf(), - generation: AtomicU64::new(0), - next_reference: AtomicU64::new(1), - hide_current: AtomicBool::new(false), - }); - let catalog = Arc::new(ChunkRootCatalog::open_callback(backend.clone()).unwrap()); - let options = ChunkPageStoreOptions { - tree_id: 42, - owner_epoch: 9, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - materialization_bytes_per_pass: 4096, - max_chunk_bytes: 0, - mirror_copies: 0, - }; - let store = Arc::new(PageStore::open_chunk(options, Arc::clone(&catalog), None).unwrap()); - assert_eq!(store.chunk_estimated_bytes().unwrap(), 0); - { - store.set_wal_replay_offset(4_096).unwrap(); - assert_eq!(store.set_wal_replay_offset(4_095), Err(CtError::InvalidArgument)); - let tree = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&store)), - ..Config::default() - }) - .unwrap(); - tree.apply_put(1, b"durable-root", b"chunk-value").unwrap(); - tree.flush().unwrap(); - assert_eq!(tree.snapshot_info().unwrap(), (1, 1)); - } - - let estimated = store.chunk_estimated_bytes().unwrap(); - assert!(estimated > 0); - let reopened_store = Arc::new(PageStore::open_chunk(options, catalog, None).unwrap()); - assert_eq!(reopened_store.chunk_stats().unwrap().pack_bytes_written, 0); - assert_eq!(reopened_store.wal_replay_offset().unwrap(), 4_096); +fn runtime_root_catalog_reopens_published_manifest_and_wal() { + let mut backend = crowdb_tree_ffi::chunk::test_util::TestChunkBackend::new(42, 9).unwrap(); + let (store, tree) = open_runtime_chunk(&backend, 0); + store.set_wal_replay_offset(4096).unwrap(); + store.set_wal_replay_offset(4095).unwrap(); + tree.apply_put(1, b"durable-root", b"chunk-value").unwrap(); + assert_eq!(tree.snapshot_info().unwrap(), (1, 1)); + let before = tree.backing_stats().unwrap(); + assert_eq!(store.wal_replay_offset().unwrap(), 4096); + drop(tree); + drop(store); + backend.next_writer().unwrap(); + let (store, reopened) = open_runtime_chunk(&backend, 0); + assert_eq!(store.wal_replay_offset().unwrap(), 4096); + assert_eq!(reopened.backing_stats().unwrap().scope_id, before.scope_id); assert_eq!( - reopened_store.set_wal_replay_offset(4_095), - Err(CtError::InvalidArgument) + reopened.backing_stats().unwrap().allocated_bytes, + before.allocated_bytes ); - let reopened = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&reopened_store)), - ..Config::default() - }) - .unwrap(); - assert_eq!(reopened_store.chunk_estimated_bytes().unwrap(), estimated); - // Observation must keep working when the remote catalog becomes unavailable. - backend.hide_current.store(true, Ordering::Release); - assert_eq!(reopened_store.chunk_estimated_bytes().unwrap(), estimated); - reopened_store.reclaim_chunk_orphans(); - assert_eq!(reopened_store.chunk_estimated_bytes().unwrap(), estimated); - backend.hide_current.store(false, Ordering::Release); assert_eq!( reopened.get(b"durable-root").unwrap(), Some((1, b"chunk-value".to_vec())) @@ -457,94 +437,45 @@ fn callback_root_catalog_persists_transition_generation_pin() { let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-callback-root-pin"); let backend = Arc::new(FileRootCatalogStore { dir: dir.path().to_path_buf(), - generation: AtomicU64::new(0), + generation: AtomicU64::new(1), next_reference: AtomicU64::new(1), hide_current: AtomicBool::new(false), }); - let catalog = Arc::new(ChunkRootCatalog::open_callback(backend.clone()).unwrap()); - let options = ChunkPageStoreOptions { - tree_id: 43, - owner_epoch: 9, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - materialization_bytes_per_pass: 4096, - max_chunk_bytes: 0, - mirror_copies: 0, - }; - let store = Arc::new(PageStore::open_chunk(options, Arc::clone(&catalog), None).unwrap()); - let tree = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&store)), - ..Config::default() - }) - .unwrap(); - tree.apply_put(1, b"key", b"value").unwrap(); - tree.flush().unwrap(); - tree.snapshot_info().unwrap(); - let generation = store.chunk_manifest_generation().unwrap(); - store.pin_chunk_generation(43, 7, 8, generation).unwrap(); - drop(tree); - drop(store); - drop(catalog); - + backend + .store(43, RootCatalogObject::Manifest(1), b"immutable retained object") + .unwrap(); + { + let catalog = ChunkRootCatalog::open_callback(backend.clone()).unwrap(); + catalog.pin_generation(43, 7, 8, 1).unwrap(); + } let reopened = ChunkRootCatalog::open_callback(backend).unwrap(); - reopened.pin_generation(43, 7, 8, generation).unwrap(); - assert_eq!( - reopened.pin_generation(43, 7, 8, generation + 1), - Err(CtError::NotFound) - ); + reopened.pin_generation(43, 7, 8, 1).unwrap(); + assert_eq!(reopened.pin_generation(43, 7, 8, 2), Err(CtError::NotFound)); reopened.unpin_generation(43, 7, 8).unwrap(); reopened.unpin_generation(43, 7, 8).unwrap(); } #[test] -fn retained_pack_estimate_survives_old_generation_reclamation_and_reopen() { - let catalog = Arc::new(ChunkRootCatalog::open_memory(9).unwrap()); - let options = ChunkPageStoreOptions { - tree_id: 46, - owner_epoch: 9, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - materialization_bytes_per_pass: 4096, - max_chunk_bytes: 0, - mirror_copies: 0, - }; - let store = Arc::new(PageStore::open_chunk(options, Arc::clone(&catalog), None).unwrap()); - let tree = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&store)), - ..Config::default() - }) - .unwrap(); +fn checkpoint_backing_survives_history_reclamation_and_reopen() { + let mut backend = crowdb_tree_ffi::chunk::test_util::TestChunkBackend::new(46, 9).unwrap(); + let (store, tree) = open_runtime_chunk(&backend, 0); for sequence in 1..=4 { tree.apply_put(sequence, b"retained-key", &vec![sequence as u8; 64 * 1024]) .unwrap(); - tree.flush().unwrap(); tree.snapshot_info().unwrap(); } - let estimate = store.chunk_estimated_bytes().unwrap(); - let written = store.chunk_stats().unwrap().pack_bytes_written; - assert!(estimate >= 64 * 1024); - assert!( - written > estimate, - "estimate must describe current retained packs rather than cumulative writes" - ); let generation = store.chunk_manifest_generation().unwrap(); - assert!(catalog.reclaim_before(46, generation) > 0); - store.reclaim_chunk_orphans(); - assert_eq!(store.chunk_estimated_bytes().unwrap(), estimate); + let before = tree.backing_stats().unwrap(); + assert!(before.allocated_bytes >= 64 * 1024); + assert!(backend.catalog.reclaim_before(46, generation) > 0); assert_eq!(tree.get(b"retained-key").unwrap(), Some((4, vec![4; 64 * 1024]))); drop(tree); drop(store); - let reopened = Arc::new(PageStore::open_chunk(options, catalog, None).unwrap()); - let recovered = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&reopened)), - ..Config::default() - }) - .unwrap(); - assert_eq!(reopened.chunk_estimated_bytes().unwrap(), estimate); + backend.next_writer().unwrap(); + let (_, recovered) = open_runtime_chunk(&backend, 0); + let after = recovered.backing_stats().unwrap(); + assert_eq!(after.scope_id, before.scope_id); + assert!(after.allocated_bytes >= 64 * 1024); assert_eq!( recovered.get(b"retained-key").unwrap(), Some((4, vec![4; 64 * 1024])) @@ -552,198 +483,128 @@ fn retained_pack_estimate_survives_old_generation_reclamation_and_reopen() { } #[test] -fn memory_root_catalog_pin_blocks_generation_reclaim_until_unpin() { - let catalog = Arc::new(ChunkRootCatalog::open_memory(9).unwrap()); - let options = ChunkPageStoreOptions { - tree_id: 45, - owner_epoch: 9, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - materialization_bytes_per_pass: 4096, - max_chunk_bytes: 0, - mirror_copies: 0, - }; - let store = Arc::new(PageStore::open_chunk(options, Arc::clone(&catalog), None).unwrap()); - let tree = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&store)), - ..Config::default() - }) - .unwrap(); +fn runtime_catalog_pin_blocks_generation_reclaim_until_unpin() { + let mut backend = crowdb_tree_ffi::chunk::test_util::TestChunkBackend::new(45, 9).unwrap(); + let (store, tree) = open_runtime_chunk(&backend, 0); for sequence in 1..=3 { tree.apply_put(sequence, b"key", &[sequence as u8]).unwrap(); - tree.flush().unwrap(); tree.snapshot_info().unwrap(); if sequence == 1 { store.pin_chunk_generation(45, 9, 10, 1).unwrap(); } } - assert_eq!(catalog.reclaim_before(45, 4), 0); - PageStore::open_chunk( - ChunkPageStoreOptions { - open_generation: 1, - ..options - }, - Arc::clone(&catalog), - None, - ) - .unwrap(); - + assert_eq!(backend.catalog.reclaim_before(45, 4), 0); + drop(tree); + drop(store); + backend.next_writer().unwrap(); + let (store, exact) = open_runtime_chunk(&backend, 1); + assert_eq!(exact.get(b"key").unwrap(), Some((1, vec![1]))); store.unpin_chunk_generation(45, 9, 10).unwrap(); - assert!(catalog.reclaim_before(45, 4) > 0); - assert_eq!( - PageStore::open_chunk( - ChunkPageStoreOptions { - open_generation: 1, - ..options - }, - catalog, - None, - ) - .unwrap_err(), - CtError::NotFound - ); + assert!(backend.catalog.reclaim_before(45, 4) > 0); + drop(exact); + drop(store); + backend.next_writer().unwrap(); + let store = backend.page_store(1).unwrap(); + assert!(matches!( + open_tree(&Config { + page_store: Some(store), + ..backend.config() + }), + Err(CtError::NotFound) + )); } #[test] -fn callback_root_catalog_opens_exact_manifest_without_latest_fallback() { - let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-callback-exact-root"); - let backend = Arc::new(FileRootCatalogStore { - dir: dir.path().to_path_buf(), - generation: AtomicU64::new(0), - next_reference: AtomicU64::new(1), - hide_current: AtomicBool::new(false), - }); - let catalog_backend: Arc<dyn RootCatalogStore> = backend.clone(); - let catalog = Arc::new(ChunkRootCatalog::open_callback(catalog_backend).unwrap()); - let latest_options = ChunkPageStoreOptions { - tree_id: 44, - owner_epoch: 11, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - materialization_bytes_per_pass: 4096, - max_chunk_bytes: 0, - mirror_copies: 0, - }; - let latest_store = Arc::new(PageStore::open_chunk(latest_options, Arc::clone(&catalog), None).unwrap()); - let latest = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&latest_store)), - ..Config::default() - }) - .unwrap(); +fn runtime_root_catalog_opens_exact_manifest_without_latest_fallback() { + let mut backend = crowdb_tree_ffi::chunk::test_util::TestChunkBackend::new(44, 11).unwrap(); + let (store, latest) = open_runtime_chunk(&backend, 0); latest.apply_put(1, b"key", b"generation-one").unwrap(); - latest.flush().unwrap(); assert_eq!(latest.snapshot_info().unwrap(), (1, 1)); latest.apply_put(2, b"key", b"generation-two").unwrap(); - latest.flush().unwrap(); assert_eq!(latest.snapshot_info().unwrap(), (2, 2)); - assert_eq!(latest_store.chunk_manifest_generation().unwrap(), 2); drop(latest); - drop(latest_store); - - let exact_options = ChunkPageStoreOptions { - open_generation: 1, - ..latest_options - }; - let exact_store = Arc::new(PageStore::open_chunk(exact_options, Arc::clone(&catalog), None).unwrap()); - let exact = Crowdbtree::open(&Config { - page_store: Some(Arc::clone(&exact_store)), - ..Config::default() - }) - .unwrap(); - assert_eq!(exact_store.chunk_manifest_generation().unwrap(), 1); + drop(store); + backend.next_writer().unwrap(); + let (store, exact) = open_runtime_chunk(&backend, 1); + assert_eq!(store.chunk_manifest_generation().unwrap(), 1); assert_eq!(exact.snapshot_state().unwrap(), (1, 1)); assert_eq!(exact.get(b"key").unwrap(), Some((1, b"generation-one".to_vec()))); exact.apply_put(2, b"key", b"stale-branch").unwrap(); - exact.flush().unwrap(); assert_eq!(exact.snapshot_info(), Err(CtError::Unavailable)); - - let missing_options = ChunkPageStoreOptions { - open_generation: 3, - ..latest_options - }; - assert_eq!( - PageStore::open_chunk(missing_options, catalog, None).unwrap_err(), - CtError::NotFound - ); + drop(exact); + drop(store); + backend.next_writer().unwrap(); + let store = backend.page_store(3).unwrap(); + assert!(matches!( + open_tree(&Config { + page_store: Some(store), + ..backend.config() + }), + Err(CtError::NotFound) + )); } #[test] fn exact_manifest_tracks_durable_snapshot_after_range_rebuild() { - let catalog = Arc::new(ChunkRootCatalog::open_memory(12).unwrap()); - let source = Crowdbtree::open(&Config::default()).unwrap(); + let source = open_tree(&Config { + frame_bytes: 4096, + ..Config::default() + }) + .unwrap(); source.apply_put(1, b"b", b"base").unwrap(); source.flush().unwrap(); - - let options = ChunkPageStoreOptions { - tree_id: 46, - owner_epoch: 12, - open_generation: 0, - pack_bytes: 4096, - iu_size: 1, - max_concurrent_packs: 2, - materialization_bytes_per_pass: 4096, - max_chunk_bytes: 0, - mirror_copies: 0, - }; - let store = Arc::new(PageStore::open_chunk(options, Arc::clone(&catalog), None).unwrap()); - let config = Config { - page_store: Some(Arc::clone(&store)), - ..Config::default() - }; - let (rebuilt, _) = source.rebuild_range(&config).unwrap(); + let mut backend = crowdb_tree_ffi::chunk::test_util::TestChunkBackend::new(46, 12).unwrap(); + let store = backend.page_store(0).unwrap(); + let (rebuilt, _) = source + .rebuild_range(&Config { + page_store: Some(store.clone()), + ..backend.config() + }) + .unwrap(); assert_eq!(rebuilt.snapshot_state().unwrap(), (1, 1)); - rebuilt.apply_put(2, b"c", b"tail").unwrap(); - rebuilt.flush().unwrap(); let checkpoint = rebuilt.snapshot_info().unwrap(); let generation = store.chunk_manifest_generation().unwrap(); assert_eq!(rebuilt.snapshot_state().unwrap(), checkpoint); drop(rebuilt); drop(store); - - let exact_store = Arc::new( - PageStore::open_chunk( - ChunkPageStoreOptions { - open_generation: generation, - ..options - }, - Arc::clone(&catalog), - None, - ) - .unwrap(), - ); - let exact = Crowdbtree::open(&Config { - page_store: Some(exact_store), - ..Config::default() - }) - .unwrap(); + backend.next_writer().unwrap(); + let (_, exact) = open_runtime_chunk(&backend, generation); assert_eq!(exact.snapshot_state().unwrap(), checkpoint); assert_eq!(exact.get(b"c").unwrap(), Some((2, b"tail".to_vec()))); } #[test] -fn published_manifest_is_visible_through_the_same_page_store() { - let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-callback-read-own-write"); - let backend = Arc::new(FileRootCatalogStore { - dir: dir.path().to_path_buf(), - generation: AtomicU64::new(0), - next_reference: AtomicU64::new(1), - hide_current: AtomicBool::new(false), - }); - let catalog_backend: Arc<dyn RootCatalogStore> = backend.clone(); - let catalog = Arc::new(ChunkRootCatalog::open_callback(catalog_backend).unwrap()); +fn runtime_root_publication_is_visible_and_reusing_a_writer_is_rejected() { + let backend = crowdb_tree_ffi::chunk::test_util::TestChunkBackend::new(43, 10).unwrap(); + let (store, tree) = open_runtime_chunk(&backend, 0); + tree.apply_put(1, b"durable-root", b"chunk-value").unwrap(); + let published = tree.snapshot_info().unwrap(); + assert_eq!(tree.snapshot_state().unwrap(), published); + assert_eq!( + tree.get(b"durable-root").unwrap(), + Some((1, b"chunk-value".to_vec())) + ); + assert!(matches!( + open_tree(&Config { + page_store: Some(store), + ..backend.config() + }), + Err(CtError::Unavailable) + )); +} + +#[test] +fn legacy_chunk_store_is_explicitly_rejected_by_ordinary_tree_open() { + let catalog = Arc::new(ChunkRootCatalog::open_memory(7).unwrap()); let store = Arc::new( PageStore::open_chunk( ChunkPageStoreOptions { - tree_id: 43, - owner_epoch: 10, + tree_id: 47, + owner_epoch: 7, open_generation: 0, pack_bytes: 4096, - iu_size: 65_536, + iu_size: 1, max_concurrent_packs: 2, materialization_bytes_per_pass: 4096, max_chunk_bytes: 0, @@ -754,27 +615,18 @@ fn published_manifest_is_visible_through_the_same_page_store() { ) .unwrap(), ); - let config = Config { - page_store: Some(Arc::clone(&store)), - ..Config::default() - }; - let tree = Crowdbtree::open(&config).unwrap(); - tree.apply_put(1, b"durable-root", b"chunk-value").unwrap(); - tree.flush().unwrap(); - let published = tree.snapshot_info().unwrap(); - backend.hide_current.store(true, Ordering::Release); - - let reopened = Crowdbtree::open(&config).unwrap(); - assert_eq!(reopened.snapshot_state().unwrap(), published); - assert_eq!( - reopened.get(b"durable-root").unwrap(), - Some((1, b"chunk-value".to_vec())) - ); + assert!(matches!( + open_tree(&Config { + page_store: Some(store), + ..Config::default() + }), + Err(CtError::NotSupported) + )); } #[test] fn range_rebuild_returns_independent_bounded_tree() { - let source = Crowdbtree::open(&Config::default()).unwrap(); + let source = open_tree(&Config::default()).unwrap(); for (slot, key) in [b"a", b"b", b"m", b"z"].into_iter().enumerate() { source.apply_put(slot as u64 + 1, key, b"v").unwrap(); } @@ -791,9 +643,9 @@ fn range_rebuild_returns_independent_bounded_tree() { ..Default::default() }) .unwrap(); - assert_eq!(stats.entries_examined, 4); + assert_eq!(stats.entries_examined, 1); assert_eq!(stats.entries_emitted, 1); - assert_eq!(stats.entries_filtered, 3); + assert_eq!(stats.entries_filtered, 0); assert_eq!(rebuilt.get(b"b").unwrap(), Some((2, b"v".to_vec()))); assert_eq!(rebuilt.get(b"a"), Err(CtError::InvalidArgument)); } @@ -801,7 +653,7 @@ fn range_rebuild_returns_independent_bounded_tree() { #[test] fn injected_mem_store_survives_caller_handle_drop() { let store = std::sync::Arc::new(PageStore::open_mem(1).unwrap()); - let t = Crowdbtree::open(&Config { + let t = open_tree(&Config { page_store: Some(store.clone()), frame_bytes: 4096, ..Default::default() @@ -816,7 +668,7 @@ fn injected_mem_store_survives_caller_handle_drop() { #[test] fn bounded_tree_rejects_foreign_keys_and_filters_scans() { - let t = Crowdbtree::open(&Config { + let t = open_tree(&Config { key_range: KeyRange::Bounded { start: Some(b"b".to_vec()), end: Some(b"d".to_vec()), @@ -838,7 +690,7 @@ fn bounded_tree_rejects_foreign_keys_and_filters_scans() { #[test] fn mem_gc_watermark_and_snapshot_folding() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); t.apply_put(1, b"a", b"A").unwrap(); t.apply_delete(2, b"a").unwrap(); t.flush().unwrap(); @@ -868,7 +720,7 @@ fn mem_gc_watermark_and_snapshot_folding() { #[test] fn mem_compact_sparse_blocks_noop_on_mem_store() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); t.apply_put(1, b"a", b"A").unwrap(); t.flush().unwrap(); t.snapshot().unwrap(); @@ -880,7 +732,7 @@ fn mem_compact_sparse_blocks_noop_on_mem_store() { #[test] fn mem_apply_batch_multi_key_and_dup_last_wins() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); t.apply_batch( 1, &[ @@ -934,7 +786,7 @@ fn file_snapshot_reopen_smoke() { }; { - let t = Crowdbtree::open(&opt).unwrap(); + let t = open_tree(&opt).unwrap(); for i in 0..50usize { let v = format!("value{i}").into_bytes(); t.apply_put((i + 1) as u64, &key(i), &v).unwrap(); @@ -944,7 +796,7 @@ fn file_snapshot_reopen_smoke() { assert_eq!(durable, 50); } // Reopen the same file and verify recovery. - let t = Crowdbtree::open(&opt).unwrap(); + let t = open_tree(&opt).unwrap(); assert_eq!(t.last_applied_slot(), 50); for i in 0..50usize { assert_eq!( @@ -970,7 +822,7 @@ fn block_device_snapshot_reopen_smoke() { }; { - let t = Crowdbtree::open(&opt).unwrap(); + let t = open_tree(&opt).unwrap(); for i in 0..50usize { let v = format!("value{i}").into_bytes(); t.apply_put((i + 1) as u64, &key(i), &v).unwrap(); @@ -980,7 +832,7 @@ fn block_device_snapshot_reopen_smoke() { assert_eq!(durable, 50); } // Reopen the same file and verify recovery. - let t = Crowdbtree::open(&opt).unwrap(); + let t = open_tree(&opt).unwrap(); assert_eq!(t.last_applied_slot(), 50); for i in 0..50usize { assert_eq!( @@ -992,7 +844,7 @@ fn block_device_snapshot_reopen_smoke() { #[test] fn snapshot_sessions_stream_with_metadata_and_abort_without_installing() { - let source = Arc::new(Crowdbtree::open(&Config::default()).unwrap()); + let source = Arc::new(open_tree(&Config::default()).unwrap()); for i in 0..30usize { source .apply_put((i + 1) as u64, &key(i), format!("v{i}").as_bytes()) @@ -1008,7 +860,7 @@ fn snapshot_sessions_stream_with_metadata_and_abort_without_installing() { assert_ne!(metadata.final_crc32c, 0); assert_eq!(export.read(1), Err(CtError::InvalidArgument)); - let target = Arc::new(Crowdbtree::open(&Config::default()).unwrap()); + let target = Arc::new(open_tree(&Config::default()).unwrap()); let mut import = target.snapshot_import_begin().unwrap(); let mut offset = 0; loop { @@ -1025,7 +877,7 @@ fn snapshot_sessions_stream_with_metadata_and_abort_without_installing() { assert_eq!(import.finish().unwrap(), metadata.at_slot); assert_eq!(target.snapshot_view().unwrap(), source.snapshot_view().unwrap()); - let empty_target = Arc::new(Crowdbtree::open(&Config::default()).unwrap()); + let empty_target = Arc::new(open_tree(&Config::default()).unwrap()); let mut abandoned = empty_target.snapshot_import_begin().unwrap(); let mut second_export = source.snapshot_export_begin(19).unwrap(); abandoned.feed(&second_export.read(0).unwrap().bytes).unwrap(); @@ -1035,7 +887,7 @@ fn snapshot_sessions_stream_with_metadata_and_abort_without_installing() { #[test] fn io_failed_clean_on_healthy_engine() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); for i in 0..10usize { t.apply_put((i + 1) as u64, &key(i), b"v").unwrap(); t.flush().unwrap(); @@ -1054,7 +906,7 @@ fn open_rejects_path_with_nul() { path: Some("bad\0path".to_string()), ..Default::default() }; - assert_eq!(Crowdbtree::open(&opt).unwrap_err(), CtError::InvalidArgument); + assert_eq!(open_tree(&opt).unwrap_err(), CtError::InvalidArgument); } #[tokio::test] @@ -1066,7 +918,7 @@ async fn async_bridge_apply_get_snapshot() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); for i in 0..20usize { t.handle() .apply_put((i + 1) as u64, &key(i), format!("a{i}").as_bytes()) @@ -1087,10 +939,7 @@ async fn async_bridge_apply_get_snapshot() { // without ever registering on the reactor's eventfd (unlike a timing-based // assertion, this is deterministic). #[tokio::test] -async fn async_get_fast_path_completes_on_first_poll() { - use std::future::Future; - use std::task::{Context, Poll, Waker}; - +async fn async_get_resident_value_completes_on_shared_runtime() { let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-ffi"); let opt = Config { path: Some(dir.path().to_string_lossy().into_owned()), @@ -1098,19 +947,11 @@ async fn async_get_fast_path_completes_on_first_poll() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); t.handle().apply_put(1, &key(0), b"v0").unwrap(); t.flush().await.unwrap(); - // Resident (never evicted): get_async's fast path (try_get_view_no_load, - // design #5 B3) must resolve this on the very first poll. - let mut fut = Box::pin(t.get(key(0))); - let waker = Waker::noop(); - let mut cx = Context::from_waker(waker); - assert_eq!( - fut.as_mut().poll(&mut cx), - Poll::Ready(Ok(Some((1u64, b"v0".to_vec())))) - ); + assert_eq!(t.get(key(0)).await.unwrap(), Some((1u64, b"v0".to_vec()))); } #[tokio::test] @@ -1122,7 +963,7 @@ async fn async_get_slow_path_completes_after_eviction() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); t.handle().apply_put(1, &key(0), b"v0").unwrap(); t.flush().await.unwrap(); t.snapshot().await.unwrap(); @@ -1143,7 +984,7 @@ async fn concurrent_async_gets_all_resolve_correctly() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); const N: usize = 16; for i in 0..N { t.handle() @@ -1172,10 +1013,7 @@ async fn concurrent_async_gets_all_resolve_correctly() { // path_completes_on_first_poll's regression shape for scan, which also // has a resident-hit fast path (try_scan_no_load). #[tokio::test] -async fn async_scan_fast_path_completes_on_first_poll() { - use std::future::Future; - use std::task::{Context, Poll, Waker}; - +async fn async_scan_resident_values_complete_on_shared_runtime() { let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-ffi"); let opt = Config { path: Some(dir.path().to_string_lossy().into_owned()), @@ -1183,7 +1021,7 @@ async fn async_scan_fast_path_completes_on_first_poll() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); for i in 0..10usize { t.handle() .apply_put((i + 1) as u64, &key(i), &format!("v{i}").into_bytes()) @@ -1191,16 +1029,12 @@ async fn async_scan_fast_path_completes_on_first_poll() { } t.flush().await.unwrap(); - let mut fut = Box::pin(t.scan(Vec::new(), Vec::new(), Vec::new(), 0, 0, false, 0)); - let waker = Waker::noop(); - let mut cx = Context::from_waker(waker); - match fut.as_mut().poll(&mut cx) { - Poll::Ready(Ok((entries, truncated))) => { - assert_eq!(entries.len(), 10); - assert!(!truncated); - } - other => panic!("expected an immediately-ready resolved scan, got {other:?}"), - } + let (entries, truncated) = t + .scan(Vec::new(), Vec::new(), Vec::new(), 0, 0, false, 0) + .await + .unwrap(); + assert_eq!(entries.len(), 10); + assert!(!truncated); } // Mirrors async_get_slow_path_completes_after_eviction for scan: forcing @@ -1216,7 +1050,7 @@ async fn async_scan_slow_path_completes_after_eviction() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); for i in 0..20usize { t.handle() .apply_put((i + 1) as u64, &key(i), &format!("v{i}").into_bytes()) @@ -1251,7 +1085,7 @@ async fn async_reverse_scan_crosses_evicted_leaves_and_respects_cursor() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); for i in 0..80usize { t.handle() .apply_put((i + 1) as u64, &key(i), &[b'v'; 128]) @@ -1291,7 +1125,7 @@ async fn async_scan_respects_limit_and_truncated_flag() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); for i in 0..15usize { t.handle() .apply_put((i + 1) as u64, &key(i), &format!("v{i}").into_bytes()) @@ -1319,7 +1153,7 @@ async fn async_scan_keys_only_skips_values() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); for i in 0..10usize { t.handle() .apply_put((i + 1) as u64, &key(i), &format!("v{i}").into_bytes()) @@ -1352,7 +1186,7 @@ async fn async_scan_keys_only_skips_values() { } #[tokio::test] -async fn try_get_pinned_fast_path_returns_borrowed_value() { +async fn try_get_pinned_resident_value_completes_with_owned_bytes() { let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-ffi"); let opt = Config { path: Some(dir.path().to_string_lossy().into_owned()), @@ -1360,7 +1194,7 @@ async fn try_get_pinned_fast_path_returns_borrowed_value() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); t.handle().apply_put(1, &key(0), b"v0").unwrap(); t.flush().await.unwrap(); @@ -1369,13 +1203,17 @@ async fn try_get_pinned_fast_path_returns_borrowed_value() { assert_eq!(slot, 1); assert_eq!(pinned.as_bytes(), b"v0"); } - _ => panic!("expected Ready(Ok(Some)) on resident hit"), + PinnedGetOutcome::Pending(future) => { + assert_eq!(future.await.unwrap(), Some((1, b"v0".to_vec()))); + } + _ => panic!("resident value must complete successfully"), } - // Not-found case: should return Ready(Ok(None)). + // Absence may also complete after the shared worker runs. match t.try_get_pinned(&key(99)) { PinnedGetOutcome::Ready(Ok(None)) => {} - _ => panic!("expected Ready(Ok(None)) for missing key"), + PinnedGetOutcome::Pending(future) => assert_eq!(future.await.unwrap(), None), + _ => panic!("missing key must complete without a value"), } } @@ -1388,7 +1226,7 @@ async fn try_get_pinned_slow_path_resolves_after_eviction() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); t.handle().apply_put(1, &key(0), b"v0").unwrap(); t.flush().await.unwrap(); t.snapshot().await.unwrap(); @@ -1411,7 +1249,7 @@ async fn try_get_pinned_slow_path_resolves_after_eviction() { } #[tokio::test] -async fn try_get_pinned_fast_path_with_large_value() { +async fn try_get_pinned_resident_large_value_completes() { let dir = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-ffi"); let opt = Config { path: Some(dir.path().to_string_lossy().into_owned()), @@ -1419,7 +1257,7 @@ async fn try_get_pinned_fast_path_with_large_value() { frame_bytes: 4096, ..Default::default() }; - let t = AsyncCrowdbtree::open(&opt).unwrap(); + let t = open_async_tree(&opt).unwrap(); let large: Vec<u8> = (0..4096u32).map(|i| u8::try_from(i % 256).unwrap()).collect(); t.handle().apply_put(1, &key(0), &large).unwrap(); t.flush().await.unwrap(); @@ -1429,13 +1267,16 @@ async fn try_get_pinned_fast_path_with_large_value() { assert_eq!(slot, 1); assert_eq!(pinned.as_bytes(), large.as_slice()); } - _ => panic!("expected Ready(Ok(Some)) on resident hit"), + PinnedGetOutcome::Pending(future) => { + assert_eq!(future.await.unwrap(), Some((1, large))); + } + _ => panic!("resident large value must complete successfully"), } } #[test] fn zero_copy_alloc_apply_round_trip() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); let mut h = t.alloc_put(3, 5).unwrap(); h.key_mut().copy_from_slice(b"abc"); h.value_mut().copy_from_slice(b"hello"); @@ -1447,7 +1288,7 @@ fn zero_copy_alloc_apply_round_trip() { #[test] fn zero_copy_large_value_round_trip() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); let big: Vec<u8> = (0..8192u32).map(|i| u8::try_from(i % 256).unwrap()).collect(); let mut h = t.alloc_put(4, big.len()).unwrap(); h.key_mut().copy_from_slice(b"bigk"); @@ -1460,7 +1301,7 @@ fn zero_copy_large_value_round_trip() { #[test] fn zero_copy_empty_value_round_trip() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); let mut h = t.alloc_put(2, 0).unwrap(); h.key_mut().copy_from_slice(b"ev"); assert!(h.value_mut().is_empty()); @@ -1472,7 +1313,7 @@ fn zero_copy_empty_value_round_trip() { #[test] fn zero_copy_handle_drop_without_apply_no_leak() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); let mut h = t.alloc_put(10, 4096).unwrap(); h.key_mut().fill(0xAB); h.value_mut().fill(0xCD); @@ -1485,16 +1326,16 @@ fn zero_copy_oversized_key_rejected() { frame_bytes: 4096, ..Default::default() }; - let t = Crowdbtree::open(&opt).unwrap(); + let t = open_tree(&opt).unwrap(); let result = t.alloc_put(3000, 4); assert_eq!(result.err(), Some(CtError::InvalidArgument)); } -// ── R30: zero-copy apply_batch_external ─────────────────────────── +// ── External Bytes batch input ─────────────────────────────────── #[test] fn apply_batch_external_round_trip_before_flush() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); let big = bytes::Bytes::from(vec![0xF0; 4096]); let ops = vec![ ExtOp::Put { @@ -1513,7 +1354,7 @@ fn apply_batch_external_round_trip_before_flush() { #[test] fn apply_batch_external_round_trip_after_flush() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); let big = bytes::Bytes::from(vec![0xE1; 8192]); let ops = vec![ExtOp::Put { key: bytes::Bytes::from_static(b"k2"), @@ -1526,7 +1367,7 @@ fn apply_batch_external_round_trip_after_flush() { #[test] fn apply_batch_external_multi_key_atomicity() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); let ops = vec![ ExtOp::Put { key: bytes::Bytes::from_static(b"a"), @@ -1549,7 +1390,7 @@ fn apply_batch_external_multi_key_atomicity() { #[test] fn apply_batch_external_intra_batch_last_key_wins() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); let ops = vec![ ExtOp::Put { key: bytes::Bytes::from_static(b"k"), @@ -1566,7 +1407,7 @@ fn apply_batch_external_intra_batch_last_key_wins() { #[test] fn apply_batch_external_large_value_round_trip() { - let t = Crowdbtree::open(&Config::default()).unwrap(); + let t = open_tree(&Config::default()).unwrap(); // 64 KiB values — the workload R30 targets (eliminate the apply-path copy). let v1 = bytes::Bytes::from(vec![0x11; 65536]); let v2 = bytes::Bytes::from(vec![0x22; 65536]); @@ -1593,20 +1434,17 @@ fn apply_batch_external_large_value_round_trip() { } #[test] -fn apply_batch_external_bytes_kept_alive_until_drain() { - // The payload Bytes is kept alive by the external buffers' Arc clones; - // it must not be freed until the MemTable drains (flush). This test - // verifies the ref handle lifecycle: the Bytes stays valid through apply - // and flush, and the drop callback fires at drain (not before). - let t = Crowdbtree::open(&Config::default()).unwrap(); +fn apply_batch_external_preserves_caller_clone_through_flush() { + // Unknown Bytes backing capacity takes a controlled copy. The caller's + // clone remains valid independently of the engine's owned cell. + let t = open_tree(&Config::default()).unwrap(); let payload = bytes::Bytes::from(vec![0xAB; 1024]); let ops = vec![ExtOp::Put { key: bytes::Bytes::from_static(b"k"), value: payload.clone(), }]; t.apply_batch_external(1, ops).unwrap(); - // payload is still alive here (we hold a clone); the engine's clone is - // pinned by the memtable entry. After flush, the engine's clone is freed. + // The bridge released its clone during apply; the caller still owns one. assert_eq!(t.get(b"k").unwrap(), Some((1, payload.to_vec()))); t.flush().unwrap(); assert_eq!(t.get(b"k").unwrap(), Some((1, payload.to_vec()))); diff --git a/lib/crowdb-tree/ffi/tests/immutable_snapshot_test.rs b/lib/crowdb-tree/ffi/tests/immutable_snapshot_test.rs new file mode 100644 index 000000000..6ab046932 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/immutable_snapshot_test.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{ + Config, Crowdbtree, CtError, KeyRange, PageStore, ScanDirection, ScanOptions, TreeRuntime, +}; +use std::sync::Arc; + +fn config(runtime: Arc<TreeRuntime>) -> Config { + Config { + runtime: Some(runtime), + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + frame_bytes: 4096, + max_inline_value: 64, + ..Config::default() + } +} + +#[test] +fn fixed_snapshot_scans_both_directions_and_keeps_results_after_all_handles_close() { + let runtime = Arc::new(TreeRuntime::new(Default::default()).unwrap()); + let tree = Crowdbtree::open(&config(runtime.clone())).unwrap(); + tree.apply_put(1, b"", b"empty").unwrap(); + tree.apply_put(2, b"b", b"old").unwrap(); + tree.apply_put(3, b"c", b"deleted").unwrap(); + tree.apply_delete(4, b"c").unwrap(); + let (sequence, coverage) = tree.snapshot_info().unwrap(); + let snapshot = tree.immutable_snapshot(sequence, coverage).unwrap(); + assert_eq!( + tree.immutable_snapshot(sequence, coverage + 1).unwrap_err(), + CtError::Unavailable + ); + tree.apply_put(5, b"b", b"new").unwrap(); + tree.snapshot_info().unwrap(); + drop(tree); + drop(runtime); + assert_eq!(snapshot.get(b"b").unwrap().unwrap(), (2, b"old".to_vec())); + let (forward, truncated) = snapshot + .scan(&ScanOptions { + start: Some(vec![]), + start_inclusive: true, + limit: 8, + ..ScanOptions::default() + }) + .unwrap(); + assert!(!truncated); + assert_eq!( + forward.iter().map(|e| e.key.as_ref()).collect::<Vec<_>>(), + vec![b"".as_slice(), b"b"] + ); + let (reverse, truncated) = snapshot + .scan(&ScanOptions { + direction: ScanDirection::Reverse, + include_tombstones: true, + keys_only: true, + limit: 8, + ..ScanOptions::default() + }) + .unwrap(); + assert!(!truncated); + assert_eq!( + reverse.iter().map(|e| e.key.as_ref()).collect::<Vec<_>>(), + vec![b"c".as_slice(), b"b", b""] + ); + assert!(reverse[0].tombstone); + assert!(reverse.iter().all(|e| e.value.is_empty())); + let (bounded, truncated) = snapshot + .scan(&ScanOptions { + byte_budget: 1, + strict_byte_budget: true, + ..ScanOptions::default() + }) + .unwrap(); + assert!(bounded.is_empty()); + assert!(truncated); + drop(snapshot); + assert_eq!(forward[1].value.as_ref(), b"old"); +} + +#[test] +fn immutable_rebuild_copies_only_target_range_and_keeps_captured_frontier() { + let runtime = Arc::new(TreeRuntime::new(Default::default()).unwrap()); + let tree = Crowdbtree::open(&config(runtime.clone())).unwrap(); + tree.apply_put(1, b"a", b"left").unwrap(); + tree.apply_put(2, b"z", &vec![b'v'; 9000]).unwrap(); + let (sequence, coverage) = tree.snapshot_info().unwrap(); + let snapshot = tree.immutable_snapshot(sequence, coverage).unwrap(); + tree.apply_put(3, b"z", b"changed").unwrap(); + drop(tree); + let options = Config { + key_range: KeyRange::Bounded { + start: Some(b"m".to_vec()), + end: None, + }, + ..config(runtime) + }; + let (child, stats) = snapshot.rebuild_range(&options).unwrap(); + assert_eq!(stats.entries_emitted, 1); + assert_eq!(stats.pages_reused, 0); + assert!(stats.pages_rebuilt > 0); + assert_eq!(child.snapshot_state().unwrap().1, coverage); + assert_eq!(child.get(b"z").unwrap().unwrap().1, vec![b'v'; 9000]); + assert_eq!(child.get(b"a").unwrap_err(), CtError::InvalidArgument); + drop(snapshot); + drop(child); + let reopened = Crowdbtree::open(&options).unwrap(); + assert_eq!(reopened.get(b"z").unwrap().unwrap().1, vec![b'v'; 9000]); +} diff --git a/lib/crowdb-tree/ffi/tests/legacy_migration_test.rs b/lib/crowdb-tree/ffi/tests/legacy_migration_test.rs new file mode 100644 index 000000000..88a974de9 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/legacy_migration_test.rs @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{chunk::legacy::LegacyManifest, Config, Crowdbtree, CtError}; + +fn manifest() -> &'static [u8] { + let fixture = include_bytes!("../../tests/data/legacy-chunk-v7.bin"); + let length = usize::try_from(u64::from_le_bytes(fixture[8..16].try_into().unwrap())).unwrap(); + &fixture[16..16 + length] +} + +#[test] +fn actual_old_writer_manifest_has_complete_pack_identities() { + let original = LegacyManifest::decode(manifest()).unwrap(); + assert_eq!( + (original.tree_id(), original.owner_epoch(), original.generation()), + (42, 7, 1) + ); + assert_eq!(original.packs().len(), 29); + assert!(original + .packs() + .all(|pack| pack.owner_tree_id == 42 && pack.length != 0)); +} + +#[test] +fn corrupted_and_truncated_old_manifests_cannot_become_retirement_proofs() { + for bytes in [&manifest()[..80], &manifest()[..manifest().len() - 1]] { + assert!(matches!(LegacyManifest::decode(bytes), Err(CtError::Corruption))); + } + let mut damaged = manifest().to_vec(); + let last = damaged.len() - 1; + damaged[last] ^= 1; + assert!(matches!( + LegacyManifest::decode(&damaged), + Err(CtError::Corruption) + )); +} + +#[test] +fn explicit_migration_requires_a_host_and_nonzero_original_generation() { + let config = Config { + runtime: None, + ..Config::default() + }; + assert!(matches!( + Crowdbtree::migrate_legacy_chunk(&config, 1), + Err(CtError::InvalidArgument) + )); + assert!(matches!( + Crowdbtree::migrate_legacy_chunk(&config, 0), + Err(CtError::InvalidArgument) + )); + assert!(matches!( + Crowdbtree::migrate_legacy_chunk(&config, u64::MAX), + Err(CtError::InvalidArgument) + )); +} diff --git a/lib/crowdb-tree/ffi/tests/owned_allocation_test.rs b/lib/crowdb-tree/ffi/tests/owned_allocation_test.rs new file mode 100644 index 000000000..e79bedbe5 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/owned_allocation_test.rs @@ -0,0 +1,166 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{Config, Crowdbtree, CtError, OwnedOp, TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +fn runtime() -> Arc<TreeRuntime> { + Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()) +} +fn tree(runtime: &Arc<TreeRuntime>) -> Crowdbtree { + Crowdbtree::open(&Config { + runtime: Some(runtime.clone()), + ..Default::default() + }) + .unwrap() +} +fn payload(tree: &Crowdbtree, capacity: usize) -> crowdb_tree_ffi::OwnedAllocation { + let mut data = Vec::with_capacity(capacity); + data.extend_from_slice(&[0xA7; 256]); + tree.own_vec(data).unwrap() +} + +#[test] +fn spare_capacity_is_charged_once_across_clones_and_slices() { + let host = runtime(); + let first = tree(&host); + // Opening publishes a new root and retires its temporary bootstrap state. + // Exclude that asynchronous release from the owned-allocation delta. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(1); + let before = loop { + let resources = first.state_stats().unwrap().resources; + if resources.retired_bytes == 0 { + break resources; + } + assert!( + std::time::Instant::now() < deadline, + "bootstrap retirement did not complete" + ); + std::thread::yield_now(); + }; + let owner = payload(&first, 1 << 20); + let left = owner.slice(32..96).unwrap(); + let right = owner.slice(96..160).unwrap(); + assert!(left.shares_allocation(&right)); + assert_eq!(left.capacity(), owner.capacity()); + let after = first.state_stats().unwrap().resources; + assert_eq!( + after.payload_bytes - before.payload_bytes, + owner.capacity() as u64 + ); + assert!( + after.metadata_bytes >= before.metadata_bytes + std::mem::size_of::<Vec<u8>>() as u64, + "owned allocation must charge its foreign control metadata" + ); + assert_eq!(left.as_ref(), &[0xA7; 64]); + assert_eq!(owner.slice(0..257).unwrap_err(), CtError::InvalidArgument); + drop(owner); + drop(left); + assert_eq!( + first.state_stats().unwrap().resources.payload_bytes, + after.payload_bytes + ); + drop(right); + assert_eq!( + first.state_stats().unwrap().resources.payload_bytes, + before.payload_bytes + ); +} + +#[test] +fn same_host_trees_share_charge_and_different_host_copies() { + let host = runtime(); + let first = tree(&host); + let second = tree(&host); + let other = tree(&runtime()); + let owner = payload(&first, 1 << 20); + let slice = owner.slice(0..128).unwrap(); + let second_before = second.state_stats().unwrap().resources.payload_bytes; + second + .apply_batch_owned( + 1, + &[OwnedOp::Put { + key: b"shared", + value: &slice, + }], + ) + .unwrap(); + assert_eq!( + second.state_stats().unwrap().resources.payload_bytes, + second_before + ); + let other_before = other.state_stats().unwrap().resources.payload_bytes; + other + .apply_batch_owned( + 1, + &[OwnedOp::Put { + key: b"copied", + value: &slice, + }], + ) + .unwrap(); + assert_eq!( + other.state_stats().unwrap().resources.payload_bytes - other_before, + 128 + 9 + ); + drop(first); + drop(owner); + drop(slice); + assert_eq!(second.get(b"shared").unwrap(), Some((1, vec![0xA7; 128]))); + assert_eq!(other.get(b"copied").unwrap(), Some((1, vec![0xA7; 128]))); +} + +#[test] +fn rejected_batch_has_no_partial_write_and_caller_retains_owner() { + let tree = tree(&runtime()); + let owner = payload(&tree, 4096); + let invalid_key = vec![0; 65536]; + let operations = [ + OwnedOp::Put { + key: b"valid", + value: &owner, + }, + OwnedOp::Put { + key: &invalid_key, + value: &owner, + }, + ]; + assert_eq!( + tree.apply_batch_owned(1, &operations), + Err(CtError::InvalidArgument) + ); + assert_eq!(tree.get(b"valid").unwrap(), None); + assert_eq!(owner.as_ref(), &[0xA7; 256]); + tree.apply_batch_owned( + 1, + &[OwnedOp::Put { + key: b"valid", + value: &owner, + }], + ) + .unwrap(); + assert_eq!(tree.get(b"valid").unwrap(), Some((1, vec![0xA7; 256]))); +} + +#[test] +fn unsubmitted_owner_outlives_tree_and_host_handles() { + let host = runtime(); + let tree = tree(&host); + let owner = payload(&tree, 4096); + drop(tree); + drop(host); + let slice = owner.slice(128..256).unwrap(); + drop(owner); + assert_eq!(slice.as_ref(), &[0xA7; 128]); +} + +#[test] +fn ownership_admission_supports_standalone_and_rejects_stopped_host() { + let standalone = Crowdbtree::open(&Config::default()).unwrap(); + let owned = standalone.own_vec(vec![1]).unwrap(); + drop(standalone); + assert_eq!(owned.as_ref(), &[1]); + let host = runtime(); + let tree = tree(&host); + host.stop_admission(); + assert!(tree.own_vec(vec![1]).is_err()); +} diff --git a/lib/crowdb-tree/ffi/tests/page_inspection_test.rs b/lib/crowdb-tree/ffi/tests/page_inspection_test.rs index 85142acbf..868f063df 100644 --- a/lib/crowdb-tree/ffi/tests/page_inspection_test.rs +++ b/lib/crowdb-tree/ffi/tests/page_inspection_test.rs @@ -6,6 +6,7 @@ use crowdb_tree_ffi::{Config, Crowdbtree, CtError}; #[test] fn follows_real_root_children_and_preserves_binary_keys() { let tree = Crowdbtree::open(&Config { + page_store: Some(crowdb_tree_ffi::PageStore::open_mem(4096).unwrap().into()), frame_bytes: 4096, ..Config::default() }) diff --git a/lib/crowdb-tree/ffi/tests/resource_usage_test.rs b/lib/crowdb-tree/ffi/tests/resource_usage_test.rs new file mode 100644 index 000000000..7cbbe2dec --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/resource_usage_test.rs @@ -0,0 +1,271 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::time::{Duration, Instant}; + +use crowdb_tree_ffi::runtime::{ResourceUsage, ResourceUsageLog, ResourceUsageSnapshot, TREE_RESOURCE_NAMES}; +use crowdb_tree_ffi::{Config, Crowdbtree, TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +fn snapshot(id: u64, used: u64) -> ResourceUsageSnapshot { + let mut resources = [ResourceUsage::default(); TREE_RESOURCE_NAMES.len()]; + resources[0] = ResourceUsage { + used, + reserved: used / 2, + allocated: used / 2, + limit: 100, + ..ResourceUsage::default() + }; + ResourceUsageSnapshot { + scope_id: id, + resource_mask: 1, + resources, + } +} + +fn flush(log: &mut ResourceUsageLog, at: Instant) -> String { + let mut bytes = Vec::new(); + log.flush(&mut bytes, "timestamp", at).unwrap(); + String::from_utf8(bytes).unwrap() +} + +#[test] +fn averages_weight_observed_duration_and_keep_unchanged_usage_in_later_windows() { + let mut log = ResourceUsageLog::default(); + let start = Instant::now(); + log.begin_sample(); + log.observe("tree", snapshot(1, 10), start); + log.end_sample(start); + log.begin_sample(); + let at = start + Duration::from_millis(100); + log.observe("tree", snapshot(1, 80), at); + log.end_sample(at); + let first = flush(&mut log, start + Duration::from_millis(400)); + assert!(first.contains(" avg=62.500 sample_max=80 "), "{first}"); + assert!( + first.contains("samples=2 coverage_ns=400000000 max_sample_gap_ns=300000000"), + "{first}" + ); + let second = flush(&mut log, start + Duration::from_millis(700)); + assert!(second.contains(" avg=80.000 sample_max=80 "), "{second}"); + assert!( + second.contains("samples=0 coverage_ns=300000000 max_sample_gap_ns=600000000"), + "{second}" + ); +} + +#[test] +fn removed_and_reopened_scopes_do_not_mix_or_grow_forever() { + let mut log = ResourceUsageLog::default(); + let start = Instant::now(); + log.begin_sample(); + log.observe("tree", snapshot(1, 40), start); + log.end_sample(start); + log.begin_sample(); + let at = start + Duration::from_millis(100); + log.observe("tree", snapshot(2, 20), at); + log.end_sample(at); + let first = flush(&mut log, start + Duration::from_millis(200)); + assert_eq!(first.lines().count(), 2); + assert!(first.contains("scope_id=1 unit=bytes active=0"), "{first}"); + assert!(first.contains("scope_id=2 unit=bytes active=1"), "{first}"); + let second = flush(&mut log, start + Duration::from_millis(300)); + assert_eq!(second.lines().count(), 1); + assert!(!second.contains("scope_id=1"), "{second}"); +} + +#[test] +fn failed_samples_keep_last_usage_and_expose_missing_coverage() { + let mut log = ResourceUsageLog::default(); + let start = Instant::now(); + log.begin_sample(); + log.observe("tree", snapshot(1, 90), start); + log.end_sample(start); + log.begin_sample(); + log.sample_failed("tree"); + log.end_sample(start + Duration::from_millis(100)); + let text = flush(&mut log, start + Duration::from_millis(200)); + assert!(text.contains(" avg=90.000 sample_max=90 "), "{text}"); + assert!(text.contains("sample_age_ns=200000000 sample_errors=1"), "{text}"); +} + +#[test] +fn contention_deltas_are_per_window_and_raw_queries_do_not_drain() { + let mut log = ResourceUsageLog::default(); + let start = Instant::now(); + log.begin_sample(); + log.observe("tree", snapshot(1, 10), start); + let mut next = snapshot(1, 10); + next.resources[0].denied_tree = 3; + next.resources[0].denied_host = 5; + next.resources[0].waits = 7; + log.observe("tree", next, start + Duration::from_millis(100)); + log.end_sample(start + Duration::from_millis(100)); + let first = flush(&mut log, start + Duration::from_millis(200)); + assert!(first.contains("denied_tree=3 denied_host=5 waits=7"), "{first}"); + let second = flush(&mut log, start + Duration::from_millis(300)); + assert!(second.contains("denied_tree=0 denied_host=0 waits=0"), "{second}"); +} + +#[test] +fn native_host_and_tree_resource_samples_keep_distinct_limits_and_scopes() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let tree = Crowdbtree::open(&Config { + runtime: Some(runtime.clone()), + ..Config::default() + }) + .unwrap(); + let host = runtime.resource_usage().unwrap(); + let first = tree.resource_usage().unwrap(); + let second = tree.resource_usage().unwrap(); + assert_eq!(host.resource_mask, 0x3ff); + assert_eq!(first.resource_mask, 0xffcff); + assert_eq!(first.resources[13].limit, first.resources[6].limit); + assert_eq!(first.resources[14].limit, first.resources[6].limit); + assert_eq!(first.resources[15].limit, first.resources[6].limit + 2); + assert_eq!(first.resources[16].limit, 4096); + assert_eq!(first.resources[18].limit, 64); + assert_eq!(first.scope_id, second.scope_id); + assert_ne!(first.scope_id, 0); + assert!(first.resources[1].used > 0); + assert!(host.resources[1].used >= first.resources[1].used); + assert!(first.resources[6].used > 0); + assert_eq!(host.resources[8].limit, 4); + assert_eq!(first.resources[0].denied_host, second.resources[0].denied_host); +} + +#[test] +fn operation_and_source_windows_keep_units_peaks_and_capacity_denials() { + let mut log = ResourceUsageLog::default(); + let start = Instant::now(); + let mut held = snapshot(7, 0); + held.resource_mask = (1 << 13) | (1 << 17) | (1 << 18); + held.resources[13].limit = 256; + held.resources[13].used = 256; + held.resources[17].limit = 1024; + held.resources[17].used = 512; + held.resources[18].limit = 64; + held.resources[18].used = 64; + log.begin_sample(); + log.observe("tree", held, start); + log.end_sample(start); + let at = start + Duration::from_millis(100); + held.resources[13].used = 0; + held.resources[13].denied_tree = 3; + held.resources[17].used = 0; + held.resources[18].used = 0; + log.begin_sample(); + log.observe("tree", held, at); + log.end_sample(at); + let text = flush(&mut log, start + Duration::from_millis(200)); + let operation = text + .lines() + .find(|line| line.contains(".backend_operation_slots ")) + .unwrap(); + assert!(operation.contains("unit=count"), "{operation}"); + assert!(operation.contains("avg=128.000 sample_max=256"), "{operation}"); + assert!(operation.contains("denied_tree=3"), "{operation}"); + let keys = text + .lines() + .find(|line| line.contains(".source_key_bytes ")) + .unwrap(); + assert!(keys.contains("unit=bytes"), "{keys}"); + assert!(keys.contains("avg=256.000 sample_max=512"), "{keys}"); + let gaps = text + .lines() + .find(|line| line.contains(".gap_source_entry_slots ")) + .unwrap(); + assert!(gaps.contains("unit=count"), "{gaps}"); + assert!(gaps.contains("avg=32.000 sample_max=64"), "{gaps}"); + assert!(gaps.contains("contention_measured=0"), "{gaps}"); +} + +#[test] +fn transient_waiter_usage_remains_visible_after_the_queue_drains() { + let mut log = ResourceUsageLog::default(); + let start = Instant::now(); + log.begin_sample(); + log.observe("tree", snapshot(1, 0), start); + let mut waiting = snapshot(1, 0); + waiting.resources[0].waiters = 10; + log.observe("tree", waiting, start + Duration::from_millis(100)); + log.observe("tree", snapshot(1, 0), start + Duration::from_millis(300)); + log.end_sample(start + Duration::from_millis(300)); + let text = flush(&mut log, start + Duration::from_millis(400)); + assert!( + text.contains("waiters=0 waiters_avg=5.000 waiters_sample_max=10"), + "{text}" + ); +} + +#[test] +fn fractional_worker_occupancy_does_not_round_down_to_idle() { + let mut log = ResourceUsageLog::default(); + let start = Instant::now(); + let mut busy = ResourceUsageSnapshot { + resource_mask: 1 << 8, + ..Default::default() + }; + busy.resources[8] = ResourceUsage { + limit: 1, + used: 1, + ..Default::default() + }; + log.begin_sample(); + log.observe("host.tree", busy, start); + busy.resources[8].used = 0; + log.observe("host.tree", busy, start + Duration::from_millis(300)); + log.end_sample(start + Duration::from_millis(300)); + let text = flush(&mut log, start + Duration::from_millis(400)); + assert!( + text.contains("used=0 reserved=0 allocated=0 avg=0.750 sample_max=1"), + "{text}" + ); +} + +#[test] +fn pin_release_windows_report_registered_pending_and_capacity_denials() { + let mut log = ResourceUsageLog::default(); + let start = Instant::now(); + let mut held = ResourceUsageSnapshot { + resource_mask: 1 << 20, + ..Default::default() + }; + held.resources[20] = ResourceUsage { + limit: 10, + used: 8, + allocated: 8, + waiters: 2, + ..Default::default() + }; + log.begin_sample(); + log.observe("host.tree", held, start); + held.resources[20] = ResourceUsage { + limit: 10, + used: 4, + allocated: 4, + denied_host: 3, + waiters: 0, + ..Default::default() + }; + log.observe("host.tree", held, start + Duration::from_millis(100)); + log.end_sample(start + Duration::from_millis(100)); + let text = flush(&mut log, start + Duration::from_millis(200)); + assert!( + text.contains("host.tree.resource.chunk_pin_release_slots"), + "{text}" + ); + assert!(text.contains("unit=count"), "{text}"); + assert!( + text.contains("limit=10 used=4 reserved=0 allocated=4 avg=6.000 sample_max=8"), + "{text}" + ); + assert!(text.contains("denied_host=3"), "{text}"); + assert!( + text.contains("waiters=0 waiters_avg=1.000 waiters_sample_max=2"), + "{text}" + ); + let next = flush(&mut log, start + Duration::from_millis(300)); + assert!(next.contains("denied_host=0"), "{next}"); + assert!(next.contains("avg=4.000 sample_max=4"), "{next}"); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_admission_test.rs b/lib/crowdb-tree/ffi/tests/runtime_admission_test.rs new file mode 100644 index 000000000..e32244ea3 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_admission_test.rs @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::sync::Arc; + +use crowdb_tree_ffi::{BatchOp, Config, Crowdbtree, CtError, TreeRuntime, TreeRuntimeOptions}; + +fn tree() -> Arc<Crowdbtree> { + Arc::new( + Crowdbtree::open(&Config { + runtime: Some(Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + ..Default::default() + }) + .unwrap(), + ) +} + +#[test] +fn unarmed_admission_does_not_keep_a_tree_alive() { + let tree = tree(); + let weak = Arc::downgrade(&tree); + let prepared = tree + .prepare_proposal(&[BatchOp::Put { + key: b"key", + value: b"value", + }]) + .unwrap(); + drop(tree); + assert!(weak.upgrade().is_none()); + assert!(matches!(prepared.arm(), Err(CtError::Unavailable))); +} + +#[test] +fn generation_replacement_invalidates_unarmed_admission() { + let tree = tree(); + let prepared = tree + .prepare_proposal(&[BatchOp::Put { + key: b"old", + value: b"generation", + }]) + .unwrap(); + tree.clear().unwrap(); + assert!(matches!(prepared.arm(), Err(CtError::Unavailable))); + assert!(tree.get(b"old").unwrap().is_none()); +} + +#[test] +fn cancelled_armed_proposal_releases_close_tickets_and_future_is_send() { + fn require_send<T: Send>(_: T) {} + let tree = tree(); + let armed = tree.prepare_proposal(&[]).unwrap().arm().unwrap(); + require_send(armed.apply(1)); + // Dropping an unpolled Rust future cancels the still unsubmitted permit. + tree.clear().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn admitted_apply_owns_bytes_and_preserves_last_operation() { + let tree = tree(); + let mut key = b"key".to_vec(); + let mut value = b"owned".to_vec(); + let prepared = tree + .prepare_proposal(&[ + BatchOp::Put { + key: &key, + value: b"first", + }, + BatchOp::Delete { key: &key }, + BatchOp::Put { + key: &key, + value: &value, + }, + ]) + .unwrap(); + key.fill(b'x'); + value.fill(b'y'); + prepared.arm().unwrap().apply(1).await.unwrap(); + assert_eq!(tree.get(b"key").unwrap(), Some((1, b"owned".to_vec()))); + tree.prepare_replay(&[BatchOp::Delete { key: b"key" }]) + .unwrap() + .arm() + .unwrap() + .apply(2) + .await + .unwrap(); + assert!(tree.get(b"key").unwrap().is_none()); +} + +#[tokio::test(flavor = "current_thread")] +async fn replay_noop_advances_the_same_frontier() { + let tree = tree(); + tree.prepare_replay(&[]) + .unwrap() + .arm() + .unwrap() + .apply(1) + .await + .unwrap(); + tree.prepare_proposal(&[BatchOp::Put { + key: b"after", + value: b"noop", + }]) + .unwrap() + .arm() + .unwrap() + .apply(2) + .await + .unwrap(); + assert_eq!(tree.get(b"after").unwrap(), Some((2, b"noop".to_vec()))); + tree.flush().unwrap(); +} + +#[tokio::test] +async fn standalone_tree_uses_the_same_prepared_admission() { + let tree = Arc::new(Crowdbtree::open(&Config::default()).unwrap()); + tree.prepare_proposal(&[BatchOp::Put { + key: b"key", + value: b"value", + }]) + .unwrap() + .arm() + .unwrap() + .apply(1) + .await + .unwrap(); + assert_eq!(tree.get(b"key").unwrap(), Some((1, b"value".to_vec()))); + tree.flush().unwrap(); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_async_scan_test.rs b/lib/crowdb-tree/ffi/tests/runtime_async_scan_test.rs new file mode 100644 index 000000000..3d9fb1270 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_async_scan_test.rs @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_tree_ffi::{ + AsyncCrowdbtree, Config, Crowdbtree, CtError, ScanDirection, ScanOptions, ScanOutcome, TreeRuntime, + TreeRuntimeOptions, +}; +use std::sync::Arc; + +fn tree() -> AsyncCrowdbtree { + let tree = Crowdbtree::open(&Config { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }) + .unwrap(); + tree.apply_put(1, b"", b"empty").unwrap(); + tree.apply_put(2, b"a", b"value").unwrap(); + AsyncCrowdbtree::from_sync(tree) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn explicit_empty_cursor_advances_bounded_async_windows() { + let tree = tree(); + let (first, truncated) = tree + .scan_with_options(ScanOptions { + limit: 1, + ..Default::default() + }) + .await + .unwrap(); + assert!(truncated); + assert_eq!(first.len(), 1); + assert!(first[0].key.is_empty()); + let cursor = first[0].key.to_vec(); + let (second, truncated) = tree + .scan_with_options(ScanOptions { + start: Some(cursor), + limit: 1, + ..Default::default() + }) + .await + .unwrap(); + assert!(!truncated); + assert_eq!(second.len(), 1); + assert_eq!(&second[0].key[..], b"a"); + let value = second[0].value.slice(1..4); + drop(second); + drop(first); + drop(tree); + assert_eq!(&value[..], b"alu"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn immediate_scan_supports_an_inclusive_empty_reverse_bound() { + let tree = tree(); + let outcome = tree.try_scan_with_options(ScanOptions { + start: Some(Vec::new()), + start_inclusive: true, + direction: ScanDirection::Reverse, + ..Default::default() + }); + let (entries, truncated) = match outcome { + ScanOutcome::Ready(result) => result, + ScanOutcome::Pending(future) => future.await, + } + .unwrap(); + assert!(!truncated); + assert_eq!(entries.len(), 1); + assert!(entries[0].key.is_empty()); + drop(tree); + assert_eq!(&entries[0].value[..], b"empty"); +} + +#[tokio::test] +async fn explicit_scan_rejects_invalid_budget_and_supports_standalone_tree() { + let tree = tree(); + assert_eq!( + tree.scan_with_options(ScanOptions { + strict_byte_budget: true, + ..Default::default() + }) + .await + .unwrap_err(), + CtError::InvalidArgument + ); + let standalone = AsyncCrowdbtree::open(&Config::default()).unwrap(); + standalone.handle().apply_put(1, b"default", b"value").unwrap(); + let (entries, truncated) = standalone + .scan_with_options(ScanOptions::default()) + .await + .unwrap(); + assert!(!truncated); + drop(standalone); + assert_eq!(entries.len(), 1); + assert_eq!(&entries[0].key[..], b"default"); + assert_eq!(&entries[0].value[..], b"value"); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_bytes_owner_test.rs b/lib/crowdb-tree/ffi/tests/runtime_bytes_owner_test.rs new file mode 100644 index 000000000..bb645c91d --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_bytes_owner_test.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{ + AsyncCrowdbtree, Config, Crowdbtree, OwnedGetOutcome, TreeRuntime, TreeRuntimeOptions, +}; +use std::{alloc::Layout, sync::Arc}; +fn tree() -> (Arc<TreeRuntime>, Arc<Crowdbtree>) { + let runtime = Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + ); + let tree = Arc::new( + Crowdbtree::open(&Config { + runtime: Some(runtime.clone()), + ..Default::default() + }) + .unwrap(), + ); + tree.apply_put(1, b"key", b"value").unwrap(); + (runtime, tree) +} +fn accounted(bytes: &bytes::Bytes) { + assert!( + bytes.owner_allocation().is_some(), + "runtime output lost its allocation account" + ); + let too_large = Layout::from_size_align(isize::MAX as usize, 1).unwrap(); + assert!(bytes.try_owner_allocation(too_large).is_err()); +} +#[tokio::test(flavor = "current_thread")] +async fn owned_get_retains_paid_bytes_after_tree_and_runtime_close() { + let (runtime, tree) = tree(); + let asynchronous = AsyncCrowdbtree::from_sync(Arc::try_unwrap(tree).ok().unwrap()); + let value = match asynchronous.try_get_owned(b"key") { + OwnedGetOutcome::Ready(value) => value, + OwnedGetOutcome::Pending(value) => value.await, + } + .unwrap() + .unwrap() + .1; + accounted(&value); + let slice = value.slice(1..4); + drop(value); + drop(asynchronous); + drop(runtime); + assert_eq!(&slice[..], b"alu"); + accounted(&slice); +} +#[test] +fn packed_scan_clones_and_slices_keep_the_paid_control_owner() { + let (runtime, tree) = tree(); + let (entries, _) = tree.scan(b"", b"", b"", 10, 1024, false, 0, false).unwrap(); + accounted(&entries[0].key); + accounted(&entries[0].value); + let value = entries[0].value.clone(); + drop(entries); + drop(tree); + drop(runtime); + assert_eq!(&value[..], b"value"); + accounted(&value); +} +#[test] +fn snapshot_window_retains_its_own_control_budget_after_stream_close() { + let (runtime, tree) = tree(); + tree.flush().unwrap(); + let mut export = tree.snapshot_export_begin(64).unwrap(); + let chunk = export.read_owned(0).unwrap(); + accounted(&chunk.bytes); + let clone = chunk.bytes.clone(); + let expected = clone.to_vec(); + drop(chunk); + drop(export); + drop(tree); + drop(runtime); + assert_eq!(&clone[..], expected); + accounted(&clone); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_compaction_test.rs b/lib/crowdb-tree/ffi/tests/runtime_compaction_test.rs new file mode 100644 index 000000000..1d6f6484c --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_compaction_test.rs @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{ + AsyncCrowdbtree, CompactionOptions, Config, CtError, PageStoreBackend, TreeRuntime, TreeRuntimeOptions, +}; +use std::sync::Arc; + +#[tokio::test(flavor = "current_thread")] +async fn explicit_block_pass_is_send_and_runs_with_bounded_operation_slots() { + fn require_send<T: Send>(_: T) {} + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-async-compaction"); + let runtime = Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + ); + let mut config = Config { + runtime: Some(runtime), + path: Some(directory.path().to_string_lossy().into_owned()), + backend: PageStoreBackend::Block, + block_size: 64 * 1024, + frame_bytes: 4096, + ..Default::default() + }; + config.runtime_binding.cpu_queue_limit = 2; + let tree = AsyncCrowdbtree::open(&config).unwrap(); + require_send(tree.compact_sparse_blocks()); + let report = tree.compact_sparse_blocks().await.unwrap(); + assert_eq!(report.status, Ok(())); + assert_eq!(report.blocks_selected, 0); + assert_eq!(tree.handle().durable_coverage(), 0); + let service = tree.handle().backend_release_service().unwrap(); + drop(tree); + drop(config); + service.shutdown().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn explicit_policy_rejects_invalid_threshold_and_skips_volatile_backend() { + let volatile = AsyncCrowdbtree::open(&Config::default()).unwrap(); + let skipped = volatile.compact_sparse_blocks().await.unwrap(); + assert_eq!(skipped.status, Ok(())); + assert_eq!(skipped.blocks_selected, 0); + assert_eq!(skipped.blocks_deleted, 0); + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("tree-async-compaction-invalid"); + let tree = AsyncCrowdbtree::open(&Config { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + path: Some(directory.path().to_string_lossy().into_owned()), + backend: PageStoreBackend::Block, + block_size: 64 * 1024, + ..Default::default() + }) + .unwrap(); + assert_eq!( + tree.compact_sparse_blocks_with(CompactionOptions { + free_threshold: f64::NAN, + ..Default::default() + }) + .await + .unwrap_err(), + CtError::InvalidArgument + ); + let report = tree + .compact_sparse_blocks_with(CompactionOptions { + max_relocation_bytes: 0, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(report.status, Ok(())); + assert_eq!(report.blocks_deleted, 0); + let service = tree.handle().backend_release_service().unwrap(); + drop(tree); + service.shutdown().await; +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_maintenance_test.rs b/lib/crowdb-tree/ffi/tests/runtime_maintenance_test.rs new file mode 100644 index 000000000..dbb45b2f4 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_maintenance_test.rs @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::sync::Arc; + +use crowdb_tree_ffi::{ + AsyncCrowdbtree, Config, FlushPhase, PageStore, SyncMode, TreeRuntime, TreeRuntimeOptions, +}; + +fn tree() -> AsyncCrowdbtree { + let runtime = Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + ); + AsyncCrowdbtree::open(&Config { + runtime: Some(runtime), + page_store: Some(Arc::new(PageStore::open_mem(4096).unwrap())), + frame_bytes: 4096, + max_inline_value: 64, + sync_mode: SyncMode::Skip, + ..Default::default() + }) + .unwrap() +} + +#[test] +fn maintenance_futures_can_move_to_the_host_executor() { + fn require_send<T: Send>(_: T) {} + let tree = tree(); + require_send(tree.flush_report()); + require_send(tree.writeback_report()); + require_send(tree.checkpoint_report()); + require_send(tree.flush()); + require_send(tree.snapshot()); +} + +#[tokio::test(flavor = "current_thread")] +async fn flush_report_keeps_both_stages_and_actual_frontiers() { + let tree = tree(); + tree.handle().apply_put(1, b"a", b"first").unwrap(); + tree.handle().apply_put(2, b"b", b"second").unwrap(); + let report = tree.flush_report().await.unwrap(); + assert_eq!(report.status, Ok(())); + assert_eq!(report.handoff_status, Ok(())); + assert_eq!(report.maintenance_status, Ok(())); + assert!(report.handoff_completed); + assert!(report.handoff_advanced); + assert_eq!(report.floor, 0); + assert_eq!(report.frontier, 2); + assert_eq!(report.published_coverage, 2); + assert_eq!(report.phase, FlushPhase::Complete); + let repeated = tree.flush_report().await.unwrap(); + assert!(repeated.handoff_completed); + assert!(!repeated.handoff_advanced); + tree.flush().await.unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn writeback_does_not_advance_the_durable_checkpoint() { + let tree = tree(); + tree.handle().apply_put(1, b"key", &vec![b'v'; 8192]).unwrap(); + let written = tree.writeback_report().await.unwrap(); + assert_eq!(written.status, Ok(())); + assert!(written.writeback_completed); + assert!(!written.durable_committed); + assert_eq!(written.captured_coverage, 1); + assert_eq!(written.durable_coverage, 0); + assert!(written.written_frames > written.primary_frames); + assert!(written.rebased_pages > 0); + let durable = tree.checkpoint_report().await.unwrap(); + assert_eq!(durable.status, Ok(())); + assert!(durable.durable_committed); + assert_eq!(durable.durable_coverage, 1); + assert!(durable.snapshot_sequence > 0); + assert_eq!(tree.snapshot().await.unwrap(), 1); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_replay_test.rs b/lib/crowdb-tree/ffi/tests/runtime_replay_test.rs new file mode 100644 index 000000000..6ce894bac --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_replay_test.rs @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use std::future::{poll_fn, Future}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::Poll; + +use bytes::Bytes; +use crowdb_tree_ffi::{Config, Crowdbtree, CtError, ExtOp, TreeRuntime, TreeRuntimeOptions}; + +fn tree() -> Arc<Crowdbtree> { + Arc::new( + Crowdbtree::open(&Config { + runtime: Some(Arc::new( + TreeRuntime::new(TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + ..Default::default() + }) + .unwrap(), + ) +} + +struct InputOwner { + bytes: Vec<u8>, + drops: Arc<AtomicUsize>, +} +impl AsRef<[u8]> for InputOwner { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} +impl Drop for InputOwner { + fn drop(&mut self) { + self.drops.fetch_add(1, Ordering::AcqRel); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn dropped_waiter_retains_chosen_bytes_until_gap_completion() { + let tree = tree(); + let drops = Arc::new(AtomicUsize::new(0)); + let input = Bytes::from_owner(InputOwner { + bytes: b"retained".to_vec(), + drops: Arc::clone(&drops), + }); + let mut high = Box::pin(tree.apply_replay( + 2, + vec![ExtOp::Put { + key: Bytes::from_static(b"high"), + value: input, + }], + )); + poll_fn(|context| { + assert!(high.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; + drop(high); + assert_eq!(drops.load(Ordering::Acquire), 0); + tree.apply_replay(1, Vec::new()).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while tree.stats().contiguous_slot != 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(tree.get(b"high").unwrap(), Some((2, b"retained".to_vec()))); + assert_eq!(drops.load(Ordering::Acquire), 1); +} + +#[tokio::test(flavor = "current_thread")] +async fn replay_empty_and_duplicate_slots_keep_the_same_contiguous_frontier() { + let tree = tree(); + tree.apply_replay(1, Vec::new()).await.unwrap(); + tree.apply_replay( + 1, + vec![ExtOp::Put { + key: Bytes::from_static(b"ignored"), + value: Bytes::from_static(b"old"), + }], + ) + .await + .unwrap(); + tree.apply_replay( + 2, + vec![ExtOp::Put { + key: Bytes::from_static(b"new"), + value: Bytes::from_static(b"value"), + }], + ) + .await + .unwrap(); + assert_eq!(tree.stats().contiguous_slot, 2); + assert!(tree.get(b"ignored").unwrap().is_none()); +} + +#[tokio::test(flavor = "current_thread")] +async fn invalid_replay_releases_foreign_input_once() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let tree = Arc::new( + Crowdbtree::open(&Config { + runtime: Some(Arc::clone(&runtime)), + key_range: crowdb_tree_ffi::KeyRange::Bounded { + start: Some(b"m".to_vec()), + end: None, + }, + ..Default::default() + }) + .unwrap(), + ); + let drops = Arc::new(AtomicUsize::new(0)); + let input = Bytes::from_owner(InputOwner { + bytes: b"owned".to_vec(), + drops: Arc::clone(&drops), + }); + let result = tree + .apply_replay( + 1, + vec![ExtOp::Put { + key: Bytes::from_static(b"key"), + value: input, + }], + ) + .await; + assert_eq!(result, Err(CtError::InvalidArgument)); + assert_eq!(drops.load(Ordering::Acquire), 1); + assert_eq!(tree.stats().contiguous_slot, 0); +} + +#[test] +fn replay_future_is_send_before_and_after_admission() { + fn require_send<T: Send>(_: T) {} + let tree = tree(); + require_send(tree.apply_replay(1, Vec::new())); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_scan_owner_test.rs b/lib/crowdb-tree/ffi/tests/runtime_scan_owner_test.rs new file mode 100644 index 000000000..cc505189a --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_scan_owner_test.rs @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_tree_ffi::{AsyncCrowdbtree, Config, Crowdbtree, TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +fn tree() -> Crowdbtree { + Crowdbtree::open(&Config { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }) + .unwrap() +} + +#[test] +fn scan_owner_keeps_clones_and_slices_alive_after_tree_close() { + let tree = tree(); + tree.apply_put(1, b"key", b"value").unwrap(); + let (entries, _) = tree.scan(b"", b"", b"", 10, 1024, false, 0, false).unwrap(); + assert_eq!(entries.len(), 1); + let value = entries[0].value.slice(1..4); + let clone = entries[0].value.clone(); + drop(entries); + drop(tree); + std::thread::spawn(move || { + assert_eq!(&value[..], b"alu"); + assert_eq!(&clone[..], b"value"); + }) + .join() + .unwrap(); +} + +#[test] +fn runtime_scan_preserves_empty_key_bounds_and_strict_budget() { + let tree = tree(); + tree.apply_put(1, b"", b"empty").unwrap(); + tree.apply_put(2, b"a", b"value").unwrap(); + let (entries, _) = tree.scan_reverse(Some(b""), true, b"", 10, 1024).unwrap(); + assert_eq!(entries.len(), 1); + assert!(entries[0].key.is_empty()); + let (entries, _) = tree.scan_from_bounded(b"", false, b"", 10, 1024).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(&entries[0].key[..], b"a"); +} + +#[test] +fn reverse_seek_retains_the_runtime_scan_owner() { + let tree = tree(); + tree.apply_put(1, b"key", b"value").unwrap(); + let found = tree.seek_reverse(b"key", true, b"").unwrap().unwrap(); + drop(tree); + assert_eq!(&found.key[..], b"key"); + assert_eq!(&found.value[..], b"value"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn async_scan_owner_survives_async_tree_and_future_drop() { + let tree = AsyncCrowdbtree::from_sync(tree()); + tree.handle().apply_put(1, b"key", b"value").unwrap(); + let (entries, _) = tree + .scan(Vec::new(), Vec::new(), Vec::new(), 10, 1024, false, 0) + .await + .unwrap(); + let key = entries[0].key.clone(); + drop(entries); + drop(tree); + assert_eq!(&key[..], b"key"); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_snapshot_owner_test.rs b/lib/crowdb-tree/ffi/tests/runtime_snapshot_owner_test.rs new file mode 100644 index 000000000..4b7285525 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_snapshot_owner_test.rs @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{Config, Crowdbtree, CtError, TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +#[test] +fn retained_chunk_slices_survive_the_stream_tree_and_runtime() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let tree = Arc::new( + Crowdbtree::open(&Config { + runtime: Some(Arc::clone(&runtime)), + ..Default::default() + }) + .unwrap(), + ); + tree.apply_put(1, b"key", &vec![b'v'; 8192]).unwrap(); + tree.flush().unwrap(); + let mut export = tree.snapshot_export_begin(64).unwrap(); + let first = export.read_owned(0).unwrap(); + assert_eq!(first.bytes.len(), 64); + assert!(!first.done); + let retained = first.bytes.slice(2..32); + let expected = retained.to_vec(); + let copied = export.read(64).unwrap(); + assert_eq!(copied.bytes.len(), 64); + assert_eq!(export.offset(), 128); + drop(first); + drop(export); + drop(tree); + drop(runtime); + assert_eq!(retained.as_ref(), expected); + let cloned = retained.clone(); + drop(retained); + assert_eq!(cloned.as_ref(), expected); +} + +#[test] +fn invalid_offset_preserves_the_next_owned_chunk() { + let tree = Arc::new(Crowdbtree::open(&Config::default()).unwrap()); + tree.apply_put(1, b"key", b"value").unwrap(); + tree.flush().unwrap(); + let mut export = tree.snapshot_export_begin(64).unwrap(); + assert_eq!(export.read_owned(1).unwrap_err(), CtError::InvalidArgument); + assert_eq!(export.offset(), 0); + assert!(!export.read_owned(0).unwrap().bytes.is_empty()); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_source_retention_test.rs b/lib/crowdb-tree/ffi/tests/runtime_source_retention_test.rs new file mode 100644 index 000000000..7bbd88856 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_source_retention_test.rs @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{Config, Crowdbtree, TreeRuntime}; +use std::sync::Arc; + +#[test] +fn inherited_source_outlives_parent_and_collector_outlives_child() { + let runtime = Arc::new(TreeRuntime::new(Default::default()).unwrap()); + let config = Config { + runtime: Some(runtime.clone()), + frame_bytes: 4096, + ..Default::default() + }; + let parent = Crowdbtree::open(&config).unwrap(); + let child = Crowdbtree::open(&config).unwrap(); + parent.apply_put(1, b"key", &vec![b'v'; 4096]).unwrap(); + child.install_split_memtable_overlay(&parent, 1).unwrap(); + let collector = parent.work_metrics_collector().unwrap(); + drop(parent); + assert_eq!(child.get(b"key").unwrap().unwrap().1, vec![b'v'; 4096]); + // This is the terminal host boundary; no new reads are admitted after it. + runtime.drain_cpu(); + let window = Arc::new(collector.capture().unwrap()); + let consumer = Arc::clone(&window); + drop(child); + runtime.drain_cpu(); + drop(config); + drop(runtime); + let closed = collector.capture().unwrap(); + assert!(closed.window_sequence > consumer.window_sequence); + assert!(Arc::ptr_eq(&consumer, &window)); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_stats_test.rs b/lib/crowdb-tree/ffi/tests/runtime_stats_test.rs new file mode 100644 index 000000000..5f2510512 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_stats_test.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{Config, Crowdbtree, TreeRuntime, TreeRuntimeOptions}; +use std::sync::Arc; + +#[test] +fn publication_stats_are_constant_cost_and_account_local() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let config = Config { + runtime: Some(runtime), + ..Default::default() + }; + let first = Crowdbtree::open(&config).unwrap(); + let second = Crowdbtree::open(&config).unwrap(); + let before = first.state_stats().unwrap(); + assert_eq!(before.logical.leaf_pages, 1); + assert!(before.resources.allocated_bytes > 0); + assert!(before.resources.reserved_bytes > 0); + let progress = first.progress_stats().unwrap(); + assert_eq!(progress.instance_generation, before.instance_generation); + assert_eq!(progress.commit_seq, before.commit_seq); + assert!(progress.sampled_at_ns >= before.sampled_at_ns); + assert_eq!(progress.candidate_oldest_age_ns, 0); + assert_eq!(progress.checkpoint_oldest_pending_age_ns, 0); + let second_before = second.state_stats().unwrap(); + let value = vec![b'v'; 64 * 1024]; + first.apply_put(1, b"key", &value).unwrap(); + // Resource fields are independent relaxed samples. Finish the host CPU + // tails before comparing account ownership; initialization can return + // unused metadata from allocated bytes to reserved bytes asynchronously. + config.runtime.as_ref().unwrap().drain_cpu(); + let first_after = first.state_stats().unwrap(); + let second_after = second.state_stats().unwrap(); + assert!(first_after.resources.allocated_bytes > before.resources.allocated_bytes); + assert!( + first_after.resources.allocated_bytes > second_after.resources.allocated_bytes + value.len() as u64 + ); + assert_eq!(second_before.logical, second_after.logical); + assert_eq!(second_before.commit_seq, second_after.commit_seq); +} + +#[test] +fn standalone_stats_sample_the_same_runtime_state() { + let tree = Crowdbtree::open(&Config::default()).unwrap(); + let state = tree.state_stats().unwrap(); + assert_eq!(state.logical.leaf_pages, 1); + assert!(state.resources.allocated_bytes > 0); + let progress = tree.progress_stats().unwrap(); + assert_eq!(progress.instance_generation, state.instance_generation); + assert_eq!(progress.commit_seq, state.commit_seq); + tree.backing_stats().unwrap(); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_test.rs b/lib/crowdb-tree/ffi/tests/runtime_test.rs new file mode 100644 index 000000000..a51cab2ad --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_test.rs @@ -0,0 +1,127 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_tree_ffi::{Config, Crowdbtree, CtError, PageStore, TreeRuntime, TreeRuntimeOptions}; + +#[test] +fn runtime_rejects_invalid_or_unfunded_storage() { + let options = TreeRuntimeOptions { + workers: 0, + ..Default::default() + }; + assert_eq!(TreeRuntime::new(options).unwrap_err(), CtError::InvalidArgument); + let options = TreeRuntimeOptions { + lifecycle_bytes: 1, + ..Default::default() + }; + assert_eq!(TreeRuntime::new(options).unwrap_err(), CtError::ResourceExhausted); +} + +#[test] +fn runtime_snapshot_preserves_cumulative_work_after_tree_close() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let before = runtime.snapshot().unwrap(); + let config = Config { + runtime: Some(runtime.clone()), + ..Default::default() + }; + let tree = Crowdbtree::open(&config).unwrap(); + tree.apply_put(1, b"key", b"value").unwrap(); + assert_eq!(tree.get(b"key").unwrap().unwrap().1, b"value"); + drop(tree); + runtime.stop_admission(); + runtime.drain_cpu(); + let after = runtime.snapshot().unwrap(); + assert_eq!(after.bindings, 0); + assert!(after.stopped); + assert_eq!(after.cpu.queued, 0); + assert_eq!(after.cpu.inflight, 0); + assert!(after.cpu.completed > before.cpu.completed); + assert!(after.cpu.execute_ns > before.cpu.execute_ns); + assert!(after.budget_grants > before.budget_grants); + assert_eq!(runtime.snapshot().unwrap(), after); +} + +#[test] +fn shared_runtime_opens_owned_core_and_survives_caller_handle_drop() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let config = Config { + runtime: Some(runtime.clone()), + ..Default::default() + }; + drop(runtime); + let tree = Crowdbtree::open(&config).unwrap(); + tree.apply_put(1, b"key", b"value").unwrap(); + assert_eq!(tree.get(b"key").unwrap().unwrap().1, b"value"); + drop(tree); + let clone = config.clone(); + drop(config); + let child = std::thread::spawn(move || { + let tree = Crowdbtree::open(&clone).unwrap(); + tree.apply_put(1, b"other", b"value").unwrap(); + drop(tree); + clone.runtime.as_ref().unwrap().stop_admission(); + clone.runtime.as_ref().unwrap().drain_cpu(); + }); + child.join().unwrap(); +} + +#[test] +fn standalone_source_rebuilds_into_an_explicit_host() { + let source = Crowdbtree::open(&Config::default()).unwrap(); + source.apply_put(1, b"key", b"value").unwrap(); + let config = Config { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + page_store: Some(Arc::new(PageStore::open_mem(1).unwrap())), + ..Default::default() + }; + let (child, stats) = source.rebuild_range(&config).unwrap(); + assert_eq!(stats.entries_emitted, 1); + assert!(stats.pages_rebuilt > 0); + drop(source); + assert_eq!(child.get(b"key").unwrap(), Some((1, b"value".to_vec()))); + assert_eq!(child.snapshot_state().unwrap().1, 1); +} + +#[test] +fn standalone_custom_writer_limit_rejects_before_accepting() { + let mut config = Config::default(); + config.runtime_binding.limits.writer_bytes = 1024 * 1024; + let tree = Crowdbtree::open(&config).unwrap(); + assert_eq!( + tree.apply_put(1, b"key", &vec![0; 2 * 1024 * 1024]), + Err(CtError::Unavailable) + ); + assert_eq!(tree.stats().contiguous_slot, 0); +} + +#[test] +fn standalone_binding_rejects_an_unfunded_open() { + let mut config = Config::default(); + config.runtime_binding.lifecycle_bytes = 1; + assert_eq!(Crowdbtree::open(&config).unwrap_err(), CtError::ResourceExhausted); +} + +#[test] +fn standalone_live_and_snapshot_rebuilds_honor_destination_binding() { + let source = Crowdbtree::open(&Config::default()).unwrap(); + source.apply_put(1, b"key", b"value").unwrap(); + let (generation, coverage) = source.snapshot_info().unwrap(); + let snapshot = source.immutable_snapshot(generation, coverage).unwrap(); + let mut destination = Config { + page_store: Some(Arc::new(PageStore::open_mem(1).unwrap())), + ..Default::default() + }; + destination.runtime_binding.lifecycle_bytes = 1; + assert_eq!( + source.rebuild_range(&destination).unwrap_err(), + CtError::ResourceExhausted + ); + assert_eq!( + snapshot.rebuild_range(&destination).unwrap_err(), + CtError::ResourceExhausted + ); + assert_eq!(source.get(b"key").unwrap(), Some((1, b"value".to_vec()))); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_wait_metrics_test.rs b/lib/crowdb-tree/ffi/tests/runtime_wait_metrics_test.rs new file mode 100644 index 000000000..aaf10c20a --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_wait_metrics_test.rs @@ -0,0 +1,19 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{Config, Crowdbtree}; +#[test] +fn wait_totals_are_non_draining_and_collector_outlives_tree() { + let tree = Crowdbtree::open(&Config::default()).unwrap(); + let collector = tree.work_metrics_collector().unwrap(); + let initial = tree.wait_metrics_totals().unwrap(); + assert_eq!(initial.classes.len(), 7); + assert!(initial.classes.iter().all(|value| value.age.complete)); + let first = collector.capture().unwrap(); + let read = collector.wait_metrics_totals().unwrap(); + let second = collector.capture().unwrap(); + assert_eq!(second.window_sequence, first.window_sequence + 1); + assert_eq!(read.classes, second.waits.classes); + drop(tree); + let after_close = collector.wait_metrics_totals().unwrap(); + assert!(after_close.classes.iter().all(|value| value.inflight == 0)); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_work_histograms_test.rs b/lib/crowdb-tree/ffi/tests/runtime_work_histograms_test.rs new file mode 100644 index 000000000..1f334c1c0 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_work_histograms_test.rs @@ -0,0 +1,69 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +use crowdb_tree_ffi::{Config, Crowdbtree, TreeRuntime, TreeRuntimeOptions, TreeWorkStage}; +use std::sync::Arc; + +#[test] +fn selected_tree_stage_has_one_paid_histogram_and_immutable_collector_fanout() { + let runtime = Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap()); + let tree = Crowdbtree::open(&Config { + runtime: Some(Arc::clone(&runtime)), + ..Default::default() + }) + .unwrap(); + let other = Crowdbtree::open(&Config { + runtime: Some(Arc::clone(&runtime)), + ..Default::default() + }) + .unwrap(); + assert_eq!(tree.work_histogram_totals().unwrap().enabled_stages, 0); + tree.enable_work_histogram(TreeWorkStage::FlushBatch).unwrap(); + tree.enable_work_histogram(TreeWorkStage::FlushBatch).unwrap(); + assert_eq!(other.work_histogram_totals().unwrap().enabled_stages, 0); + let collector = tree.work_metrics_collector().unwrap(); + tree.apply_put(1, b"key", b"value").unwrap(); + tree.flush().unwrap(); + let totals = tree.work_histogram_totals().unwrap(); + let stage = totals.stage(TreeWorkStage::FlushBatch).unwrap(); + assert_eq!(stage.count, 1); + assert!(stage.sum_ns > 0); + assert!(stage.p50_ns <= stage.p95_ns && stage.p95_ns <= stage.p99_ns); + assert!(stage.p99_ns <= stage.max_ns); + assert!(totals.stage(TreeWorkStage::PageLoadPipeline).is_none()); + let first = Arc::new(collector.capture().unwrap()); + let fanout = Arc::clone(&first); + assert_eq!(first.window_sequence, first.histograms.window_sequence); + assert_eq!( + first.histograms.stage(TreeWorkStage::FlushBatch).unwrap().count, + 1 + ); + let empty = collector.capture().unwrap(); + assert_eq!( + empty.histograms.stage(TreeWorkStage::FlushBatch).unwrap().count, + 0 + ); + assert_eq!( + empty + .histograms + .stage(TreeWorkStage::FlushBatch) + .unwrap() + .total_count, + 1 + ); + drop(other); + drop(tree); + let final_window = collector.capture().unwrap(); + assert_eq!( + final_window + .histograms + .stage(TreeWorkStage::FlushBatch) + .unwrap() + .total_count, + 1 + ); + assert!(Arc::ptr_eq(&first, &fanout)); + assert_eq!( + fanout.histograms.stage(TreeWorkStage::FlushBatch).unwrap().count, + 1 + ); +} diff --git a/lib/crowdb-tree/ffi/tests/runtime_work_metrics_test.rs b/lib/crowdb-tree/ffi/tests/runtime_work_metrics_test.rs new file mode 100644 index 000000000..85153d470 --- /dev/null +++ b/lib/crowdb-tree/ffi/tests/runtime_work_metrics_test.rs @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +use crowdb_tree_ffi::{ + Config, Crowdbtree, CtError, TreeRuntime, TreeRuntimeOptions, TREE_WORK_CAUSE_NAMES, + TREE_WORK_COUNTER_NAMES, +}; +use std::sync::Arc; + +#[test] +fn one_collector_distributes_a_window_and_totals_never_drain_it() { + let tree = Crowdbtree::open(&Config { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }) + .unwrap(); + let collector = tree.work_metrics_collector().unwrap(); + assert_eq!(tree.work_metrics_collector().unwrap_err(), CtError::Unavailable); + tree.apply_put(1, b"key", b"value").unwrap(); + tree.flush().unwrap(); + let before = tree.work_metrics_totals().unwrap(); + let first = Arc::new(collector.capture().unwrap()); + let second_reader = Arc::clone(&first); + let after = tree.work_metrics_totals().unwrap(); + assert_eq!(first.window_sequence, 1); + let flush = TREE_WORK_CAUSE_NAMES + .iter() + .position(|name| *name == "flush") + .unwrap(); + let values = TREE_WORK_COUNTER_NAMES + .iter() + .position(|name| *name == "flush.winning.value.bytes") + .unwrap(); + assert_eq!(before.counters.values[flush][values], 5); + assert_eq!(first.counters.values[flush][values], 5); + assert_eq!(after.counters.values[flush][values], 5); + assert!(first.counters.sampled_at_ns > 0); + assert!(first.age_sampled_at_ns > 0); + assert!(before.age_sampled_at_ns > 0); + assert_eq!(first.stages[0].age.inflight, 0); + assert_eq!(first.stages[0].age.tracked, 0); + assert!(first.stages[0].age.complete); + assert!(first.stages.iter().any(|stage| stage.window_count > 0)); + assert!(Arc::ptr_eq(&first, &second_reader)); + assert!(after + .stages + .iter() + .zip(before.stages) + .all(|(a, b)| a.total_count >= b.total_count)); + drop(tree); + let final_window = collector.capture().unwrap(); + assert_eq!(final_window.window_sequence, 2); + assert_eq!(final_window.counters.values[flush][values], 5); + assert!(final_window.age_sampled_at_ns >= first.age_sampled_at_ns); + assert_eq!(final_window.stages[0].age.untracked, 0); + assert!(final_window.stages[0].age.complete); + assert_eq!(first, second_reader); +} + +#[test] +fn releasing_the_collector_allows_a_new_single_owner() { + let tree = Crowdbtree::open(&Config { + runtime: Some(Arc::new(TreeRuntime::new(TreeRuntimeOptions::default()).unwrap())), + ..Default::default() + }) + .unwrap(); + drop(tree.work_metrics_collector().unwrap()); + assert!(tree.work_metrics_collector().is_ok()); + let standalone = Crowdbtree::open(&Config::default()).unwrap(); + let standalone_collector = standalone.work_metrics_collector().unwrap(); + assert_eq!( + standalone.work_metrics_collector().unwrap_err(), + CtError::Unavailable + ); + drop(standalone_collector); + assert!(standalone.work_metrics_collector().is_ok()); +} diff --git a/lib/crowdb-tree/ffi/tests/split_hint_test.rs b/lib/crowdb-tree/ffi/tests/split_hint_test.rs index 3975e0e7a..d0a55455a 100644 --- a/lib/crowdb-tree/ffi/tests/split_hint_test.rs +++ b/lib/crowdb-tree/ffi/tests/split_hint_test.rs @@ -13,7 +13,7 @@ fn resident_index_hint_is_interior_without_reading_values() { assert!(tree.approximate_split_key().unwrap().is_none()); for sequence in 1..=5000 { tree.apply_put(sequence, format!("key-{sequence:05}").as_bytes(), &[9; 100]) - .unwrap(); + .unwrap_or_else(|error| panic!("apply slot {sequence}: {error:?}")); } tree.flush().unwrap(); let before = tree.stats(); @@ -40,10 +40,14 @@ fn leaf_hint_respects_bounded_range_and_empty_keys() { tree.apply_put(index as u64 + 1, key, b"value").unwrap(); } tree.flush().unwrap(); - // Small flushes can leave a delta over the initially empty leaf base. - // A snapshot materializes the leaf; structural hints never force that work. - tree.snapshot().unwrap(); + // Flush publishes a resident overlay. Hints inspect it without forcing + // checkpoint encoding or loading a cold checkpoint base. + let before = tree.stats(); let key = tree.approximate_split_key().unwrap().unwrap(); + let after = tree.stats(); + assert_eq!(after.buffer_pool_misses, before.buffer_pool_misses); + assert_eq!(after.l1_get_total, before.l1_get_total); + assert_eq!(after.mt_get_total, before.mt_get_total); assert!(!key.is_empty()); assert!(key.as_slice() < b"z".as_slice()); assert!(tree.get(&key).unwrap().is_some()); diff --git a/lib/crowdb-tree/include/crowdb-tree/apply_c_api.h b/lib/crowdb-tree/include/crowdb-tree/apply_c_api.h new file mode 100644 index 000000000..e25c380b8 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/apply_c_api.h @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/c_api.h" + +extern "C" { +struct ct_apply_permit; +// Copies and charges the complete raw batch, but retains no Tree close ticket. +// gap_repair selects the independent replay/follower progress partition. +int32_t ct_apply_prepare(ct_tree *tree, const ct_kv_ref *operations, size_t count, int32_t gap_repair, + ct_apply_permit **out); +// Call before allowing consensus to choose. Pins the prepared generation; +// dropping an unsubmitted armed permit releases its accepted-operation tickets. +int32_t ct_apply_arm(ct_apply_permit *permit); +// Transfer an armed permit into one already chosen slot. The future handle was +// allocated by prepare; freeing either handle cannot cancel accepted work. +// A future completes exactly once via ct_future_poll. +// Each handle may move between threads; calls on one handle are serialized. +int32_t ct_apply_submit(ct_apply_permit *permit, uint64_t slot, ct_future **out); +// Reader returns immutable caller-owned bytes for index < count. Calls on the +// reader are serialized, may occur on runtime workers, and must not throw. +using ct_replay_read = void (*)(void *context, size_t index, ct_kv_ref *out); +// Consume context, invoking drop_context once even on immediate errors. The +// optional owner_control_bytes is the actual allocation of the foreign bridge +// owner, charged as Metadata until drop_context frees it; input storage remains +// caller-owned until native makes its charged copy. No descriptor array is made. +// Accepted work awaits its complete gap grant even after future handle release. +int32_t ct_apply_replay(ct_tree *tree, uint64_t slot, size_t count, ct_replay_read read, void (*drop_context)(void *), + void *context, size_t owner_control_bytes, ct_future **out); +void ct_apply_permit_free(ct_apply_permit *permit); +} diff --git a/lib/crowdb-tree/include/crowdb-tree/backend/async_page_store.h b/lib/crowdb-tree/include/crowdb-tree/backend/async_page_store.h index 0a8951702..1150ce56d 100644 --- a/lib/crowdb-tree/include/crowdb-tree/backend/async_page_store.h +++ b/lib/crowdb-tree/include/crowdb-tree/backend/async_page_store.h @@ -12,6 +12,7 @@ // frame (no uring, no I/O) needs no dedicated type. #pragma once +#include "crowdb-tree/btree/work_cause.h" #include "crowdb-tree/maptable/page_types.h" // PageAddr #include "crowdb-tree/status.h" @@ -20,6 +21,7 @@ #endif #include "crowdb-tree/backend/page_purpose.h" +#include "crowdb-tree/memory_budget.h" #include <cstddef> #include <cstdint> @@ -35,6 +37,11 @@ struct AsyncCompletion { void *context = nullptr; AsyncCompleteFn complete_fn = nullptr; + // Optional context/buffer lifetime, copied with the callback through a + // transport. This also keeps uncertain submissions safe until drain. + std::shared_ptr<const void> owner = nullptr; + std::shared_ptr<AllocationBudget> allocations = nullptr; + TreeWorkCause cause = TreeWorkCause::Other; void complete(Status status) const { @@ -54,13 +61,22 @@ class AsyncPageStore public: virtual ~AsyncPageStore() = default; + // A successful write can be read by its supplied address without a later + // checkpoint/seal. Address reclamation additionally requires an extent + // protection owner supplied by the allocator. Staging-only stores keep + // the default until their pack-reference read path is available. + [[nodiscard]] virtual bool write_completion_is_reloadable() const noexcept + { + return false; + } + // Submit an async read/write of `len` bytes at durable offset `addr` // (the same PageAddr/byte-offset domain as PageStore::read_at/write_at). // `on_complete` fires exactly once with the outcome. Immediate results // and rejected submissions may complete inline; accepted I/O completes - // from the backend's worker. Returns an opaque nonzero operation id when the backend - // supports per-operation cancellation; 0 means no cancellable operation - // was accepted. + // from the backend's worker. Returns an opaque nonzero operation id when the + // backend supports per-operation cancellation; 0 means no cancellable + // operation was accepted. virtual uint64_t submit_read(PageAddr addr, void *buf, size_t len, AsyncCompletion on_complete) = 0; virtual uint64_t submit_write(PageAddr addr, const void *buf, size_t len, AsyncCompletion on_complete) = 0; @@ -97,6 +113,11 @@ class BlockAsyncPageStore : public AsyncPageStore // for at least as long as this object. BlockAsyncPageStore(BlockPageStore *store, ::crowdb::common::DiskIOUring *uring); + [[nodiscard]] bool write_completion_is_reloadable() const noexcept override + { + return true; + } + uint64_t submit_read(PageAddr addr, void *buf, size_t len, AsyncCompletion on_complete) override; uint64_t submit_write(PageAddr addr, const void *buf, size_t len, AsyncCompletion on_complete) override; Status submit_fsync(AsyncCompletion on_complete) override; diff --git a/lib/crowdb-tree/include/crowdb-tree/backend/backend_release_service.h b/lib/crowdb-tree/include/crowdb-tree/backend/backend_release_service.h new file mode 100644 index 000000000..ae3cdf917 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/backend/backend_release_service.h @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" +#include "crowdb-tree/status.h" + +#include <atomic> +#include <memory> + +namespace crowdb::tree +{ +// A host-owned backend service has no executor or Tree reference. The host +// retains and pumps it until stop_admission() followed by drained(). Stopping +// ingress does not invalidate previously registered readers or release nodes. +class BackendReleaseService : public std::enable_shared_from_this<BackendReleaseService> +{ + public: + using Wake = void (*)(void *) noexcept; + + struct Node + { + // Each false result yields the same paid node for a later bounded step. + // A true result must have destroyed/freed the node and its resource. + bool (*dispose)(Node *) noexcept = nullptr; + std::shared_ptr<BackendReleaseService> service; + Node *next = nullptr; + bool queued = false; + Node() = default; + Node(const Node &) = delete; + Node &operator=(const Node &) = delete; + }; + + struct Snapshot + { + size_t registered = 0; + size_t pending = 0; + bool closed = false; + }; + + struct Step + { + size_t visited = 0; + size_t completed = 0; + bool more = false; + }; + + [[nodiscard]] static Status create(const std::shared_ptr<AllocationBudget> &budget, size_t capacity, + std::shared_ptr<const void> wake_owner, Wake wake, void *context, + std::shared_ptr<BackendReleaseService> *out); + ~BackendReleaseService(); + // Bind once before admitting backend users; a host driver owns the wake + // context and continues pumping after Tree/runtime shutdown. + [[nodiscard]] Status bind_wake(std::shared_ptr<const void> owner, Wake wake, void *context); + + [[nodiscard]] bool externally_driven() const noexcept + { + return wake_ != nullptr; + } + + [[nodiscard]] const std::shared_ptr<AllocationBudget> &allocation_budget() const noexcept + { + return budget_; + } + + [[nodiscard]] static size_t allocation_bound() noexcept; + // Registration precedes resource publication and never allocates. The node + // is embedded in an independently paid owner that survives final release. + [[nodiscard]] Status register_node(Node &node); + static void enqueue(Node &node) noexcept; + // Only the host backend consumer may pump. Concurrent pumps do not wait. + [[nodiscard]] Step pump(size_t limit) noexcept; + void stop_admission() noexcept; + [[nodiscard]] Snapshot snapshot() const noexcept; + [[nodiscard]] bool drained() const noexcept; + + private: + void wake() const noexcept; + static constexpr size_t kClosed = size_t{1} << ((sizeof(size_t) * 8) - 1); + std::shared_ptr<AllocationBudget> budget_; + std::shared_ptr<const void> wake_owner_; + Wake wake_ = nullptr; + void *context_ = nullptr; + size_t capacity_ = 0; + std::atomic<size_t> state_{0}; + std::atomic<size_t> pending_{0}; + std::atomic<Node *> incoming_{nullptr}; + Node *active_ = nullptr; + Node *deferred_ = nullptr; + bool prefer_deferred_ = false; + std::atomic_flag consumer_ = ATOMIC_FLAG_INIT; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/backend/block_page_store.h b/lib/crowdb-tree/include/crowdb-tree/backend/block_page_store.h index fec0f9474..46f853085 100644 --- a/lib/crowdb-tree/include/crowdb-tree/backend/block_page_store.h +++ b/lib/crowdb-tree/include/crowdb-tree/backend/block_page_store.h @@ -15,7 +15,9 @@ #pragma once #include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/memory_budget.h" +#include <atomic> #include <cstdint> #include <memory> #include <mutex> @@ -25,6 +27,21 @@ namespace crowdb::tree { +class BlockDirectory; +class BackendReleaseService; + +namespace detail +{ +class AllocationAccount; +struct BlockAccountAccess; +} // namespace detail + +struct BlockDirectoryDeleter +{ + std::shared_ptr<AllocationBudget> budget; + void operator()(BlockDirectory *directory) const noexcept; +}; + // ── Medium abstraction ──────────────────────────────────────────── // BlockPageStore delegates all raw I/O to a Medium. This allows the same // alignment/bounce logic to work over files, block devices, and in-memory @@ -92,6 +109,7 @@ class FileMedium : public BlockPageStoreMedium { public: ~FileMedium() override; + static void operator delete(void *pointer) noexcept; FileMedium(const FileMedium &) = delete; FileMedium &operator=(const FileMedium &) = delete; @@ -99,23 +117,48 @@ class FileMedium : public BlockPageStoreMedium // Open (creating if absent) a regular file. `o_direct` selects O_DIRECT // on Linux / F_NOCACHE on macOS. static Status open(const std::string &path, bool o_direct, std::unique_ptr<FileMedium> *out); + static Status open_owned(const char *path, bool direct, const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<FileMedium> *out, + const std::shared_ptr<BackendReleaseService> &release = {}); Status pwrite_at(uint64_t off, const uint8_t *buf, size_t len) override; Status pread_partial(uint64_t off, uint8_t *buf, size_t len, size_t *out_read) const override; Status fsync() override; [[nodiscard]] uint64_t size() const override; + [[nodiscard]] static size_t close_allocation_bound() noexcept; + [[nodiscard]] int fd() const { return fd_; } + // A barrier acknowledges only the generation observed before submission. + // Writers mark both submission and completion so a racing barrier cannot + // consume a write that has not yet reached the device. + void mark_dirty() noexcept; + [[nodiscard]] uint64_t dirty_generation() const noexcept; + [[nodiscard]] bool dirty() const noexcept; + void acknowledge_sync(uint64_t generation) noexcept; + private: explicit FileMedium(int fd) : fd_(fd) { } - int fd_ = -1; + struct CloseNode; + [[nodiscard]] Status prepare_close(const std::shared_ptr<AllocationBudget> &budget, + const std::shared_ptr<BackendReleaseService> &service); + CloseNode *close_ = nullptr; + int fd_ = -1; + std::atomic<uint64_t> written_{0}; + std::atomic<uint64_t> synced_{0}; +}; + +struct BlockSyncTarget +{ + std::shared_ptr<FileMedium> medium; + uint64_t generation = 0; }; // ── BlockPageStore ──────────────────────────────────────────────── @@ -124,6 +167,16 @@ class BlockPageStore : public PageStore public: ~BlockPageStore() override; + // A standalone C++ host retains this service and pumps it until drained + // after releasing the store and any retained file readers. A bound wake + // driver owns this responsibility across Tree/runtime close. + [[nodiscard]] const std::shared_ptr<BackendReleaseService> &release_service() const noexcept + { + return release_; + } + + static void operator delete(void *pointer) noexcept; + BlockPageStore(const BlockPageStore &) = delete; BlockPageStore &operator=(const BlockPageStore &) = delete; @@ -133,9 +186,10 @@ class BlockPageStore : public PageStore // sector size. static Status open(const std::string &path, uint32_t iu_size, std::unique_ptr<BlockPageStore> *out); - // Open an array-of-blocks store. Creates `{dir}/{store_id}-{group_id}.blk-{NNNN}` - // files on demand when the current block fills up. On reopen, scans the - // directory for existing block files and opens them all. + // Open an array-of-blocks store. Creates + // `{dir}/{store_id}-{group_id}.blk-{NNNN}` files on demand when the current + // block fills up. On reopen, scans the directory for existing block files and + // opens them all. static Status open_blocks(const std::string &dir, uint32_t store_id, uint32_t group_id, uint64_t block_size, uint32_t iu_size, std::unique_ptr<BlockPageStore> *out); @@ -146,6 +200,10 @@ class BlockPageStore : public PageStore Status read_at(uint64_t off, uint8_t *buf, size_t len) const override; Status sync() override; [[nodiscard]] uint64_t size() const override; + Status write_at(uint64_t off, const uint8_t *buf, size_t len, const std::shared_ptr<AllocationBudget> &budget); + Status read_at(uint64_t off, uint8_t *buf, size_t len, const std::shared_ptr<AllocationBudget> &budget) const; + Status sync(const std::shared_ptr<AllocationBudget> &budget); + [[nodiscard]] uint64_t size(const std::shared_ptr<AllocationBudget> &budget) const; [[nodiscard]] uint32_t iu_size() const override { @@ -157,10 +215,11 @@ class BlockPageStore : public PageStore return block_size_; } - // Delete a block file (array-of-blocks mode only). Closes the fd, - // removes the BlockExtent, and unlinks the .blk-{NNNN} file. + // Delete a block file (array-of-blocks mode only). Unlinks its current + // incarnation; already accepted I/O retains the old fd until completion. // Safe only after snapshot commit confirms zero live pages in the block. Status delete_block(uint32_t block_idx) override; + Status delete_block(uint32_t block_idx, const std::shared_ptr<AllocationBudget> &budget); [[nodiscard]] bool is_block_device() const { @@ -179,84 +238,69 @@ class BlockPageStore : public PageStore // (e.g. MemoryMedium). Used by BlockAsyncPageStore. int fd_for_offset(uint64_t off, uint64_t *out_local) const; + // Retain the actual fd incarnation through an asynchronous operation. + [[nodiscard]] std::shared_ptr<FileMedium> + file_for_offset(uint64_t off, uint64_t *out_local, const std::shared_ptr<AllocationBudget> &budget = {}) const; + // Ensures block files are allocated for the byte range [off, off+len), // mirroring write_at_extents' allocation logic without writing any data. // Also marks the covered extents as dirty. Call before fd_for_offset() // in the async write path so the fd exists. - Status ensure_extents(uint64_t off, size_t len); + Status ensure_extents(uint64_t off, size_t len, const std::shared_ptr<AllocationBudget> &budget = {}); // Returns fds of all dirty, non-deleted extents (array-of-blocks) or the // single medium's fd (single-file mode), for async fsync via DiskIOUring. - // Clears the dirty flag on each returned extent (mirroring sync()). + // Enumeration does not acknowledge durability. Failed or unknown barriers + // keep their captured generation dirty; newer writes survive older success. // Returns an empty vector if no fd is fsync-able (e.g. MemoryMedium or // SyncMode::kSkip). - std::vector<int> dirty_fds(); + [[nodiscard]] std::vector<int> dirty_fds() const; + + [[nodiscard]] std::vector<BlockSyncTarget> sync_targets() const; + using BlockSyncTargets = std::vector<BlockSyncTarget, MetadataAllocator<BlockSyncTarget>>; + [[nodiscard]] BlockSyncTargets sync_targets(const std::shared_ptr<AllocationBudget> &budget) const; // Returns fds of all live, non-deleted extents (array-of-blocks) or the // single medium's fd (single-file mode), for fd registration with - // DiskIOUring at startup. Unlike dirty_fds(), does not clear the dirty - // flag. Returns an empty vector if no fd is usable (e.g. MemoryMedium). + // DiskIOUring at startup. Returns an empty vector if no fd is usable + // (e.g. MemoryMedium). [[nodiscard]] std::vector<int> all_extent_fds() const; - // Number of live block files in an array-of-blocks store (0 for single-medium). - [[nodiscard]] size_t num_extents() const - { - size_t n = 0; - for (const auto &ext : extents_) { - if (!ext.deleted) { - ++n; - } - } - return n; - } + // Number of live block files in an array-of-blocks store (0 for + // single-medium). + [[nodiscard]] size_t num_extents() const; private: - // Single-medium constructor (open / open_mem) - BlockPageStore(std::unique_ptr<BlockPageStoreMedium> medium, uint32_t iu_size, bool is_block_device) - : medium_(std::move(medium)), - iu_size_(iu_size), - is_block_device_(is_block_device) - { - } - - // Array-of-blocks constructor - BlockPageStore(std::string dir, uint32_t store_id, uint32_t group_id, uint64_t block_size, uint32_t iu_size) - : iu_size_(iu_size), - is_block_device_(false), - block_size_(block_size), - store_id_(store_id), - group_id_(group_id), - dir_(std::move(dir)) - { - } + friend struct detail::BlockAccountAccess; + BlockPageStore(std::shared_ptr<detail::AllocationAccount> allocations, std::shared_ptr<BlockPageStoreMedium> medium, + uint32_t iu_size, bool is_block_device, std::shared_ptr<BackendReleaseService> release = {}); + BlockPageStore(std::shared_ptr<detail::AllocationAccount> allocations, const std::string &dir, uint32_t store_id, + uint32_t group_id, uint64_t block_size, uint32_t iu_size, + std::shared_ptr<BackendReleaseService> release); [[nodiscard]] int fd() const; // returns -1 if no FileMedium uint64_t capacity_ = 0; // probed fixed device size; 0 = unknown // Array-of-blocks management - Status allocate_new_block(); - Status write_at_extents(uint64_t off, const uint8_t *buf, size_t len); - Status read_at_extents(uint64_t off, uint8_t *buf, size_t len) const; - - struct BlockExtent - { - std::unique_ptr<FileMedium> medium; - uint64_t base_offset = 0; // block_idx * block_size - uint64_t used = 0; // high-water mark - bool dirty = false; - bool deleted = false; // block file unlinked, extent kept for index stability - }; - - std::unique_ptr<BlockPageStoreMedium> medium_; // single-medium mode - uint32_t iu_size_; - bool is_block_device_; + using BlockPath = std::basic_string<char, std::char_traits<char>, MetadataAllocator<char>>; + [[nodiscard]] BlockPath block_path(uint32_t index, const std::shared_ptr<AllocationBudget> &budget = {}) const; + Status write_at_extents(uint64_t off, const uint8_t *buf, size_t len, + const std::shared_ptr<AllocationBudget> &budget); + Status read_at_extents(uint64_t off, uint8_t *buf, size_t len, + const std::shared_ptr<AllocationBudget> &budget) const; + + std::shared_ptr<detail::AllocationAccount> allocations_; + std::shared_ptr<BackendReleaseService> release_; + std::shared_ptr<BlockPageStoreMedium> medium_; // single-medium mode + uint32_t iu_size_; + bool is_block_device_; // Array-of-blocks mode - uint64_t block_size_ = 0; - uint32_t store_id_ = 0; - uint32_t group_id_ = 0; - std::string dir_; - std::vector<BlockExtent> extents_; + uint64_t block_size_ = 0; + uint32_t store_id_ = 0; + uint32_t group_id_ = 0; + BlockPath dir_; + std::unique_ptr<BlockDirectory, BlockDirectoryDeleter> blocks_; }; // Dump utility: annotated hex dump of a block file. Parses known structures diff --git a/lib/crowdb-tree/include/crowdb-tree/backend/page_store.h b/lib/crowdb-tree/include/crowdb-tree/backend/page_store.h index a8474316a..c27a78537 100644 --- a/lib/crowdb-tree/include/crowdb-tree/backend/page_store.h +++ b/lib/crowdb-tree/include/crowdb-tree/backend/page_store.h @@ -20,8 +20,10 @@ #include "crowdb-tree/status.h" #include <algorithm> +#include <atomic> #include <cstdint> #include <functional> +#include <memory> #include <mutex> #include <string> #include <vector> @@ -43,11 +45,28 @@ inline uint64_t round_up_to_iu(uint64_t v, uint32_t iu) return iu <= 1 ? v : ((v + iu - 1) / iu) * iu; } +struct Config; +class TreeCore; + class PageStore { public: virtual ~PageStore() = default; + // One runtime allocator/recovery domain per backend object. The caller + // retains this lease through all accepted operations and export owners. + // Legacy callers must still obey the backend's exclusive-writer contract. + [[nodiscard]] bool try_claim_runtime_domain() noexcept + { + bool unclaimed = false; + return runtime_claimed_.compare_exchange_strong(unclaimed, true, std::memory_order_acq_rel); + } + + void release_runtime_domain() noexcept + { + runtime_claimed_.store(false, std::memory_order_release); + } + // Write/read `len` bytes at byte offset `off`. The store grows on write as // needed (up to capacity for fixed-size backends). // Unwritten ranges may return not_found. Other read failures must remain @@ -200,8 +219,18 @@ class PageStore return sync_mode_; } + using RuntimeFactory = Status (*)(const Config &, std::shared_ptr<TreeCore> *); + + [[nodiscard]] virtual RuntimeFactory runtime_factory() const noexcept + { + return nullptr; + } + protected: SyncMode sync_mode_ = SyncMode::kFull; + + private: + std::atomic<bool> runtime_claimed_{false}; }; // In-memory block device. Durable only for the lifetime of the object; @@ -210,9 +239,10 @@ class PageStore class MemPageStore : public PageStore { public: - explicit MemPageStore(uint32_t iu_size = 1) : iu_size_(iu_size) - { - } + static constexpr size_t kBackingPageBytes = 64 * 1024; + + explicit MemPageStore(uint32_t iu_size = 1); + ~MemPageStore() override; Status write_at(uint64_t off, const uint8_t *buf, size_t len) override; Status read_at(uint64_t off, uint8_t *buf, size_t len) const override; @@ -224,15 +254,23 @@ class MemPageStore : public PageStore [[nodiscard]] uint64_t size() const override; + // Allocated byte capacity of the memory data backing, excluding metadata. + [[nodiscard]] size_t resident_bytes() const; + + // The caller must hold the ledger's exclusive free-range lease for this + // entire backing page until removal completes. The logical size is unchanged. + [[nodiscard]] bool release_backing_page(uint64_t index); + [[nodiscard]] uint32_t iu_size() const override { return iu_size_; } private: - mutable std::mutex mu_; - std::vector<uint8_t> data_; - uint32_t iu_size_; + struct Data; + mutable std::mutex mu_; + std::unique_ptr<Data> data_; + uint32_t iu_size_; }; // Debug wrapper (PT9-B): a byte-transparent pass-through over an inner store diff --git a/lib/crowdb-tree/include/crowdb-tree/backend_release_c_api.h b/lib/crowdb-tree/include/crowdb-tree/backend_release_c_api.h new file mode 100644 index 000000000..7fdbd7469 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/backend_release_c_api.h @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/c_api.h" +#ifdef __cplusplus +extern "C" { +#endif +typedef struct ct_backend_release_service ct_backend_release_service; + +typedef struct ct_backend_release_step +{ + size_t visited; + size_t completed; + uint8_t more; + uint8_t drained; +} ct_backend_release_step; + +// Bind once before publishing the opened Tree to other callers. Returns +// NotFound for backends without deferred file release. Context transfers only +// on success. Wake must neither block nor allocate. This service owns neither +// the Tree, its runtime, nor the backend; the host retains it through drained. +ct_status ct_tree_backend_release_service(ct_tree *tree, void (*wake)(void *), void (*drop_context)(void *), + void *context, ct_backend_release_service **out); +// Pump only on a host blocking-backend executor. Limit is in [1,64]. +ct_backend_release_step ct_backend_release_service_pump(ct_backend_release_service *service, size_t limit); +// Stop new backend files only after host ingress is closed. Existing readers +// remain valid and may enqueue their final release after this call. +void ct_backend_release_service_stop(ct_backend_release_service *service); +void ct_backend_release_service_free(ct_backend_release_service *service); +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/backing_stats_c_api.h b/lib/crowdb-tree/include/crowdb-tree/backing_stats_c_api.h new file mode 100644 index 000000000..136d639c7 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/backing_stats_c_api.h @@ -0,0 +1,30 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#ifdef __cplusplus +# include <cstdint> +#else +# include <stdint.h> +#endif +#ifdef __cplusplus +extern "C" { +#endif +struct ct_tree; +struct ct_work_metrics_collector; + +// Equal nonzero scopes identify the same backend ledger. Do not sum views of +// one scope. This additive API does not change ct_tree_stats_snapshot_v1. +typedef struct ct_backing_stats_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t scope_id; + uint64_t allocated_bytes; +} ct_backing_stats_v1; + +int32_t ct_get_backing_stats(struct ct_tree *tree, ct_backing_stats_v1 *out); +// The collector retains only the ledger counters, including after Tree close. +int32_t ct_work_metrics_collector_backing_stats(struct ct_work_metrics_collector *collector, ct_backing_stats_v1 *out); +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/diagnostics.h b/lib/crowdb-tree/include/crowdb-tree/btree/diagnostics.h index 2c38dd3fb..f7f9c57cc 100644 --- a/lib/crowdb-tree/include/crowdb-tree/btree/diagnostics.h +++ b/lib/crowdb-tree/include/crowdb-tree/btree/diagnostics.h @@ -6,6 +6,93 @@ namespace crowdb::tree { +// Published with the mapping that produced these counts. Unknown summaries +// contribute to their unknown-page count, never to the exact subtotal. +struct TreeLogicalStats +{ + uint64_t leaf_pages = 0; + uint64_t inner_pages = 0; + uint64_t height = 0; + uint64_t dirty_pages = 0; + uint64_t overlay_pages = 0; + uint64_t overlay_runs = 0; + uint64_t overlay_entries = 0; + uint64_t page_size_known_bytes = 0; + uint64_t page_size_unknown_pages = 0; + uint64_t page_size_estimate_bytes = 0; + uint64_t page_size_max_observed_bytes = 0; + uint64_t kv_known_count = 0; + uint64_t kv_known_bytes = 0; + uint64_t kv_unknown_pages = 0; + uint64_t consolidate_candidate_pages = 0; + uint64_t split_candidate_pages = 0; + uint64_t split_ready_pages = 0; + uint64_t merge_candidate_pages = 0; + uint64_t merge_candidate_pairs = 0; + uint64_t merge_ready_pairs = 0; + uint64_t writeback_candidate_pages = 0; + uint64_t evict_candidate_pages = 0; + uint64_t checkpoint_pending_pages = 0; + bool checkpoint_mapping_pending = false; + + [[nodiscard]] uint64_t page_count() const + { + return leaf_pages + inner_pages; + } +}; + +// Allocation and reference gauges are sampled independently with relaxed +// atomics. Referenced/pinned subsets overlap; only allocated + reserved is +// meaningful as physical occupancy, and admission uses its own exact ledger. +struct TreeResourceStats +{ + uint64_t allocated_bytes = 0; + uint64_t reserved_bytes = 0; + uint64_t payload_bytes = 0; + uint64_t frame_bytes = 0; + uint64_t metadata_bytes = 0; + uint64_t io_bytes = 0; + uint64_t scratch_bytes = 0; + uint64_t retired_bytes = 0; + uint64_t resident_frames = 0; + uint64_t cache_bytes = 0; + uint64_t cache_pinned_bytes = 0; + uint64_t proxy_pages = 0; + uint64_t read_inflight_bytes = 0; + uint64_t write_inflight_bytes = 0; + uint64_t materialize_inflight_bytes = 0; + // Backing gauges may project a shared backend ledger. Use backing_stats() + // scope_id to deduplicate those views; RAM and admission remain per Tree. + uint64_t allocated_backing_bytes = 0; + uint64_t maintenance_queued = 0; + uint64_t maintenance_inflight = 0; +}; + +// Backend-ledger projection. Equal nonzero scope IDs name the same physical +// accounting domain; observers must deduplicate scopes rather than sum views. +struct BackingStatsSnapshot +{ + uint64_t scope_id = 0; + uint64_t allocated_bytes = 0; +}; + +struct TreeProgressStats +{ + uint64_t maintenance_oldest_age_ns = 0; + uint64_t candidate_oldest_age_ns = 0; + uint64_t checkpoint_oldest_pending_age_ns = 0; +}; + +struct TreeStatsSnapshot +{ + uint64_t instance_generation = 0; + uint64_t commit_seq = 0; + uint64_t sampled_at_ns = 0; + TreeLogicalStats logical; + TreeResourceStats resources; + TreeProgressStats progress; +}; + struct TreeSummary { uint64_t root_version = 0; @@ -33,22 +120,21 @@ struct MergeGcStats uint64_t blocks_deleted = 0; // source blocks unlinked after commit }; -// Point-in-time diagnostics snapshot: batches every -// cheap (O(1)) internal counter worth exposing to an operator into one -// struct, so a caller/FFI/console poll costs one call instead of many -// small ones. Deliberately excludes anything that requires walking the -// tree (height()/leaf_count()) or the full keyspace -- every field -// here is already an atomic counter or BufferPool::stats(), also O(1). +// Compatibility diagnostics. Runtime cache values are sampled from maintained +// counters; no page, keyspace or frame walk occurs. Deprecated fields below +// return zero. TreeStatsSnapshot and the work collector expose current schemas. struct EngineStats { uint64_t last_applied_slot = 0; // in-memory L1 coverage (see last_applied_slot()) uint64_t contiguous_slot = 0; // gap-free-applied watermark (see contiguous_slot()) uint64_t gc_watermark = 0; // min(snapshot_slot, safe_slot) (see gc_watermark()) bool io_failed = false; // latched media fault (see io_failed()) - uint64_t snapshot_pages_written = 0; // last snapshot()'s dirty base pages written + uint64_t snapshot_pages_written = 0; // frames physically written by the last successful durable checkpoint uint64_t snapshot_pages_total = 0; // cumulative pages written across all snapshots - uint64_t snapshot_segments_written = 0; // last snapshot()'s dirty mapping segments written - // BufferPool::Stats as of this call -- see buffer_pool.h. + uint64_t snapshot_segments_written = 0; // deprecated dirty-segment counter; always zero + // Runtime decoded-page cache hits/misses/evictions and resident pages. + // Dirty counts use the published logical pages; used aliases resident. + // writebacks and num_frames describe the removed arena and return zero. uint64_t buffer_pool_hits = 0; uint64_t buffer_pool_misses = 0; uint64_t buffer_pool_evictions = 0; @@ -57,7 +143,9 @@ struct EngineStats uint32_t buffer_pool_dirty = 0; uint32_t buffer_pool_used = 0; uint32_t buffer_pool_num_frames = 0; - // MemTable (L0) / flush / L1 cumulative counters (monotonic since open). + // Deprecated legacy MemTable/flush/lookup events; these fields return zero. + // The work collector reports the new fixed quantity/cause schema, whose + // units deliberately do not alias the historical counters below. uint64_t mt_upsert_total = 0; // apply() writes into L0 uint64_t mt_overwrite_total = 0; uint64_t mt_history_keep_total = 0; @@ -77,13 +165,8 @@ struct EngineStats uint64_t inner_count = 0; // live inner pages (O(1) atomic) }; -// Per-step scan profile: each step's aggregate over the window since the last -// scan_profile() call (the underlying LatencySummary handles are flushed, so -// this is a destructive read -- the window resets on each call). `count` is the -// number of scans in the window; `entries` is the total entries returned. Each -// step's `sum_ns` / `max_ns` cover only that step; `avg_ns` is sum_ns / count. -// Steps: l1 (B-tree descent + per-leaf resolve), merge (L0+L1 advance + -// winner + decode), total (whole scan). +// Deprecated empty compatibility profile. Runtime stage measurements are +// available through the explicit work metrics collector. struct ScanProfile { uint64_t count = 0; // scans in the window diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/range_rebuild.h b/lib/crowdb-tree/include/crowdb-tree/btree/range_rebuild.h index c2a08f0e9..020f4879c 100644 --- a/lib/crowdb-tree/include/crowdb-tree/btree/range_rebuild.h +++ b/lib/crowdb-tree/include/crowdb-tree/btree/range_rebuild.h @@ -14,11 +14,17 @@ namespace crowdb::tree struct RangeRebuildStats { + // Logical keys enumerated after route/range selection. Skipped subtrees + // and binary-search-elided leaf records are not examined or filtered. uint64_t entries_examined = 0; uint64_t entries_emitted = 0; uint64_t entries_filtered = 0; - uint64_t pages_reused = 0; - uint64_t pages_rebuilt = 0; + // Shared physical storage only; independently paid frame copies count as + // rebuilt even when their encoded bytes can be copied unchanged. + uint64_t pages_reused = 0; + uint64_t pages_rebuilt = 0; + // Maximal child roots skipped by routing, including one root for an empty + // selection. Descendants are not visited just to count them. uint64_t subtrees_skipped = 0; }; diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/read_result.h b/lib/crowdb-tree/include/crowdb-tree/btree/read_result.h index 61bf14263..472b11198 100644 --- a/lib/crowdb-tree/include/crowdb-tree/btree/read_result.h +++ b/lib/crowdb-tree/include/crowdb-tree/btree/read_result.h @@ -2,9 +2,7 @@ // Licensed under the Apache License, Version 2.0. #pragma once -#include "crowdb-tree/buffer.h" -#include "crowdb-tree/epoch.h" -#include "crowdb-tree/maptable/page.h" +#include "crowdb-tree/slice.h" #include <memory> #include <string> @@ -27,75 +25,27 @@ struct get_result std::string value; }; -// Zero-copy point-read result (plan-tree #5 B3 remaining). `value()` is a -// borrowed `Slice` for an L1 hit resolved to a non-overflow cell -- it -// points directly into the resident leaf's frame, kept alive for this -// object's lifetime by the epoch guard it owns (no copy). An L0 hit (R50: -// the MemTable's skip-list node is epoch-protected the same way an L1 frame -// is) also borrows directly off the node's cell version. An overflow value -// (assembled from multiple pages, no single frame to borrow) is materialized -// into an owned `buffer` instead; `value()` is transparent to the caller -// either way. -// -// Move-only (like `EpochManager::Guard`): copying would either double-free -// the guard or silently let a caller outlive it. `get()`/`multi_get()` are -// thin wrappers over `get_view()` that clone `value()` into a `std::string` -// and let the guard drop before returning, preserving their existing owned- -// copy contract for every other caller. +// Move-only read result. The independent owner keeps immutable cell or frame +// bytes and their allocation charge alive across callbacks, generation changes, +// and Tree close. Overflow results own their assembled value through the same +// interface. No thread-bound epoch guard crosses the callback boundary. class GetView { public: - GetView() = default; + GetView() = default; + GetView(GetView &&) noexcept = default; + GetView &operator=(GetView &&) noexcept = default; + GetView(const GetView &) = delete; + GetView &operator=(const GetView &) = delete; - // Not defaulted (found this the hard way, via - // ASan): `owned_` (a `buffer`) relocates its bytes on move when small - // enough to be inline (SBO, buffer::kInlineCap) -- but `value_` is a - // *separate* field, a plain Slice pointer+len that a defaulted move - // would blindly copy byte-for-byte, still aliasing the just-moved-from - // `owned_`'s old (now-stale, for an inline buffer) storage. Any - // resolved GetView whose value is owned (owned_ non-empty) must have - // `value_` re-derived from *this* object's own (possibly relocated) - // `owned_` after the move -- a borrowed (frame-pointing) value_ is - // untouched either way, since it aliases external storage the move - // never touches. - GetView(GetView &&o) noexcept - : guard_(std::move(o.guard_)), - source_guard_(std::move(o.source_guard_)), - found_(o.found_), - slot_(o.slot_), - value_(o.value_), - owned_(std::move(o.owned_)), - pins_(std::move(o.pins_)) + [[nodiscard]] static GetView from_owner(uint64_t slot, Slice value, std::shared_ptr<const void> owner) { - if (!owned_.empty()) { - value_ = owned_.slice(); - } - } - - GetView &operator=(GetView &&o) noexcept - { - if (this != &o) { - release_pins(); - guard_ = std::move(o.guard_); - source_guard_ = std::move(o.source_guard_); - found_ = o.found_; - slot_ = o.slot_; - value_ = o.value_; - owned_ = std::move(o.owned_); - pins_ = std::move(o.pins_); - if (!owned_.empty()) { - value_ = owned_.slice(); - } - } - return *this; - } - - GetView(const GetView &) = delete; - GetView &operator=(const GetView &) = delete; - - ~GetView() - { - release_pins(); + GetView result; + result.found_ = static_cast<bool>(owner); + result.slot_ = slot; + result.value_ = value; + result.owner_ = std::move(owner); + return result; } [[nodiscard]] bool found() const @@ -108,44 +58,23 @@ class GetView return slot_; } - // Valid only while this GetView is alive. + // Valid while this result retains its owner. [[nodiscard]] Slice value() const { return value_; } - // R6 debug-only: the frame address a borrowed value points into, or - // nullptr for an owned (L0 / overflow) value. Used by tests to verify - // the get_async slow path returns a borrowed Slice (no copy). + // Compatibility diagnostic: address of the independently owned bytes. + // It is not the beginning of a physical frame and also covers overflow. [[nodiscard]] const uint8_t *frame_base() const { - return owned_.empty() ? value_.bytes() : nullptr; + return value_.bytes(); } private: - friend class Crowdbtree; - EpochManager::Guard guard_; // keeps an L1 hit's frame resident - std::shared_ptr<EpochManager::Guard> source_guard_; - bool found_ = false; - uint64_t slot_ = 0; - Slice value_; // borrowed (L1) or owned_.slice() (L0 / overflow) - buffer owned_; // backing storage when the value can't be borrowed - // R6: cross-thread pins holding the borrowed value's chain alive after - // the epoch guard is released (get_async slow path). Empty on the fast - // path (guard_ alone keeps the frame resident) and for owned values. - std::vector<PageBase *> pins_; - // R6: the chain head whose frame/entries back the borrowed value_, set - // by try_get_view_no_load when the value is borrowed (not L0/overflow). - // Used by the slow path to walk + pin the chain before releasing guard_. - PageBase *borrowed_chain_head_ = nullptr; - - void release_pins() - { - for (PageBase *p : pins_) { - p->unpin(); - } - pins_.clear(); - } + std::shared_ptr<const void> owner_; + bool found_ = false; + uint64_t slot_ = 0; + Slice value_; }; - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/scan_packed.h b/lib/crowdb-tree/include/crowdb-tree/btree/scan_packed.h index 57ab28d58..548b7dff2 100644 --- a/lib/crowdb-tree/include/crowdb-tree/btree/scan_packed.h +++ b/lib/crowdb-tree/include/crowdb-tree/btree/scan_packed.h @@ -3,10 +3,13 @@ #pragma once +#include "crowdb-tree/memory_budget.h" #include "crowdb-tree/slice.h" #include <cstdint> +#include <cstdlib> #include <cstring> +#include <limits> #include <utility> namespace crowdb::tree @@ -22,6 +25,10 @@ class ScanPackedBuf public: ScanPackedBuf() = default; + explicit ScanPackedBuf(std::shared_ptr<AllocationBudget> budget) : budget_(std::move(budget)) + { + } + ~ScanPackedBuf() { std::free(data_); @@ -30,7 +37,12 @@ class ScanPackedBuf ScanPackedBuf(const ScanPackedBuf &) = delete; ScanPackedBuf &operator=(const ScanPackedBuf &) = delete; - ScanPackedBuf(ScanPackedBuf &&o) noexcept : data_(o.data_), size_(o.size_), cap_(o.cap_) + ScanPackedBuf(ScanPackedBuf &&o) noexcept + : budget_(std::move(o.budget_)), + charge_(std::move(o.charge_)), + data_(o.data_), + size_(o.size_), + cap_(o.cap_) { o.data_ = nullptr; o.size_ = 0; @@ -41,6 +53,8 @@ class ScanPackedBuf { if (this != &o) { std::free(data_); + charge_ = std::move(o.charge_); + budget_ = std::move(o.budget_); data_ = o.data_; size_ = o.size_; cap_ = o.cap_; @@ -51,7 +65,7 @@ class ScanPackedBuf return *this; } - void reserve(size_t n) // NOLINT(readability-convert-member-functions-to-static) modifies cap_ via grow + void reserve(size_t n) { if (n <= cap_) { return; @@ -59,7 +73,7 @@ class ScanPackedBuf grow(n); } - void pack_u32(uint32_t v) // NOLINT(readability-convert-member-functions-to-static) modifies data_/size_ via ensure + void pack_u32(uint32_t v) { ensure(4); for (int i = 0; i < 4; ++i) { @@ -67,7 +81,7 @@ class ScanPackedBuf } } - void pack_u64(uint64_t v) // NOLINT(readability-convert-member-functions-to-static) modifies data_/size_ via ensure + void pack_u64(uint64_t v) { ensure(8); for (int i = 0; i < 8; ++i) { @@ -81,8 +95,7 @@ class ScanPackedBuf data_[size_++] = b; } - void append(const char *p, - size_t n) // NOLINT(readability-convert-member-functions-to-static) modifies data_/size_ via ensure + void append(const char *p, size_t n) { if (n == 0) { return; @@ -107,6 +120,11 @@ class ScanPackedBuf return size_; } + [[nodiscard]] size_t capacity() const + { + return cap_; + } + [[nodiscard]] const uint8_t *data() const { return data_; @@ -117,6 +135,22 @@ class ScanPackedBuf // must std::free the returned pointer. [[nodiscard]] uint8_t *release() { + if (budget_) { + // The legacy raw-buffer ABI transfers an application-owned copy. + // Budgeted consumers keep this object instead of using release(). + auto *copy = static_cast<uint8_t *>(std::malloc(size_)); + if (copy == nullptr && size_ != 0) { + throw std::bad_alloc(); + } + if (size_ != 0) { + std::memcpy(copy, data_, size_); + } + std::free(data_); + data_ = nullptr; + size_ = cap_ = 0; + charge_.reset(); + return copy; + } uint8_t *p = data_; data_ = nullptr; size_ = 0; @@ -125,8 +159,11 @@ class ScanPackedBuf } private: - void ensure(size_t need) // NOLINT(readability-convert-member-functions-to-static) modifies data_/cap_ via grow + void ensure(size_t need) { + if (need > std::numeric_limits<size_t>::max() - size_) { + throw std::bad_array_new_length(); + } if (size_ + need > cap_) { grow(size_ + need); } @@ -134,18 +171,33 @@ class ScanPackedBuf void grow(size_t min_cap) { - size_t new_cap = (cap_ == 0) ? 256 : cap_ * 2; + size_t new_cap = budget_ ? min_cap : 256; while (new_cap < min_cap) { + if (new_cap > std::numeric_limits<size_t>::max() / 2) { + new_cap = min_cap; + break; + } new_cap *= 2; } - auto *p = static_cast<uint8_t *>(std::realloc(data_, new_cap)); + AllocationLease charge(budget_, new_cap, AllocationKind::kPayload); + auto *p = static_cast<uint8_t *>(std::malloc(new_cap)); + if (p == nullptr) { + throw std::bad_alloc(); + } + if (size_ != 0) { + std::memcpy(p, data_, size_); + } + std::free(data_); + charge_ = std::move(charge); data_ = p; cap_ = new_cap; } - uint8_t *data_ = nullptr; - size_t size_ = 0; - size_t cap_ = 0; + std::shared_ptr<AllocationBudget> budget_; + AllocationLease charge_; + uint8_t *data_ = nullptr; + size_t size_ = 0; + size_t cap_ = 0; }; } // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/tree.h b/lib/crowdb-tree/include/crowdb-tree/btree/tree.h index 267feb80d..45cef3430 100644 --- a/lib/crowdb-tree/include/crowdb-tree/btree/tree.h +++ b/lib/crowdb-tree/include/crowdb-tree/btree/tree.h @@ -1,9 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// Crowdbtree: one ordered, single-version-per-key store per consensus group. -// Two-level write path: apply() lands in the MemTable -// (L0); flush() merges the contiguous-applied prefix into the COW B+tree (L1). +// A Tree facade delegates every operation to one immutable-publication core. + #pragma once #include "crowdb-common/metrics/metrics.h" @@ -12,25 +11,16 @@ #include "crowdb-tree/btree/read_result.h" #include "crowdb-tree/btree/scan_packed.h" #include "crowdb-tree/config.h" -#include "crowdb-tree/epoch.h" -#include "crowdb-tree/maptable/mapping_table.h" #include "crowdb-tree/maptable/page.h" -#include "crowdb-tree/memtable/memtable.h" -#include "crowdb-tree/memtable/source.h" #include "crowdb-tree/snapshot/native_frame.h" -#include "crowdb-tree/snapshot/prepared.h" #include "crowdb-tree/snapshot/snapshot.h" #include "crowdb-tree/status.h" #include <atomic> #include <cstdint> -#include <deque> #include <functional> #include <limits> #include <memory> -#include <mutex> -#include <set> -#include <shared_mutex> #include <string> #include <vector> @@ -39,11 +29,6 @@ namespace crowdb::tree struct RangeRebuildStats; -// The metrics core moved to crowdb-common::metrics (R12); bridge the moved types -// into `crowdb-tree` with per-type using-declarations so existing -// `Counter*`/`Gauge*`/`LatencySummary*`/`MetricsRegistry`/`Bandwidth` -// references compile unchanged. (Not a `namespace crowdb::tree = -// crowdb::common::metrics;` alias — only the specific types are bridged.) using crowdb::common::metrics::Bandwidth; using crowdb::common::metrics::CallbackGauge; using crowdb::common::metrics::Counter; @@ -52,13 +37,13 @@ using crowdb::common::metrics::LatencySummary; using crowdb::common::metrics::MetricsRegistry; class AsyncPageStore; +class TreeCore; -// One mutation in a batch. All ops in a batch share the batch's slot. struct batch_op { std::string key; OpKind kind; - std::string value; // empty for Delete + std::string value; }; struct Batch @@ -75,320 +60,107 @@ class Crowdbtree Crowdbtree(const Crowdbtree &) = delete; Crowdbtree &operator=(const Crowdbtree &) = delete; - // open a tree, recovering durable state from opt.page_store if a valid - // snapshot exists; otherwise start empty. Requires opt.page_store != null. + // Recover durable state or initialize an empty backend. static Status open(const Config &opt, std::unique_ptr<Crowdbtree> *out); - - // Persist the materialized L1 state durably. Folds delta chains, writes - // dirty base pages plus a fresh image for each dirty mapping-table - // segment and the segment directory, then commits the inactive A/B - // anchor slot. Returns the durable last_applied_slot via out (if - // non-null). Requires opt.page_store != null. + // Host-only migration. Stop and drain every legacy writer first, claim its + // catalog authority, and inject the runtime Chunk factory. The exact old + // generation stays pinned until the new root CAS is confirmed. On failure + // reopen with a fresh catalog incarnation before resolving/retrying. + static Status migrate_legacy_chunk(const Config &opt, uint64_t expected_generation, + std::unique_ptr<Crowdbtree> *out); + + // Finite source handoff, frame writes, durable root commit, and publication reconciliation. Status snapshot(uint64_t *out_last_applied = nullptr, uint64_t *out_snapshot_seq = nullptr); - // Run one bounded immutable-backend ownership cleanup pass under the same - // generation gate as snapshot publication. Local stores complete as a no-op. Status materialize_ownership(uint64_t *bytes_written, bool *complete); - // Async twin of snapshot(). Always genuinely - // async from *this* caller's perspective when Config::async_page_store is - // wired (flush/snapshot are - // *always* slow-path, unlike get/scan): snapshot_async() returns - // immediately after kicking off the walk + first I/O submission, and - // on_done fires later from the Reactor thread with the same result - // snapshot() would have returned. - // - // Lock discipline (this is the one place in the engine where a - // completion legitimately runs on a *different* thread than the one - // that started the operation, so it gets its own note): write_mutex_ - // itself is only ever locked and unlocked on the *same* thread, for the - // brief synchronous prepare_snapshot_locked() walk, exactly like every - // other writer entry point -- std::mutex has no defined behavior for a - // cross-thread unlock, so it is never held across the async write - // phase. What *does* span the whole prepare-through-commit sequence is - // snapshot_inflight_, a plain std::atomic<bool> spin-gate (a mutex's - // "same thread unlocks it" restriction is a pthread_mutex_t property, - // not a general lock property -- an atomic has no such restriction): - // it serializes this generation's SpaceAllocator against a *second* - // overlapping snapshot(_async) call (which would otherwise rebuild an - // allocator from the same last-*committed* anchor and could hand - // out the same "free" byte range to two different pages -- silent - // corruption), without blocking apply()/flush()/evict_clean_leaves(), - // which remain free to run concurrently against write_mutex_ as usual. - // That safety hinges on prepare_snapshot_locked() never eagerly setting - // a dirty page's PageBase::durable_addr -- see its doc comment -- - // because evict_clean_leaves_locked() treats durable_addr != kNoAddr as - // "safe to evict, a durable copy already exists"; commit_prepared_snapshot() - // is what actually sets it, one write_mutex_ critical section per page, - // only once that page's specific byte write has landed. - // - // Falls back to running the existing synchronous snapshot() in the - // caller's stack frame (still correct, just not async) when no async - // backend is wired -- e.g. a MemPageStore-backed tree. + // Completion runs on the shared host runtime; close is a host-thread operation. void snapshot_async(std::function<void(Status, uint64_t last_applied)> on_done); - // Ingest a batch at `slot`. The tree internally tracks received slots and - // computes the contiguous prefix (how far the flusher may flush) itself, so - // callers no longer pass Paxos/learner state. Lands in L0; may trigger a - // size-based flush. For a slot with no data (a NoOp), call force_advance_slot. + // Explicit-slot apply preserves higher versions beyond a contiguous gap. Status apply(uint64_t slot, const Batch &batch); - // One already-encoded op for apply_encoded (plan-tree #5 B2d): `cell` is - // a slot+kind+value cell already packed via encode_cell_buf. Lets a - // caller that owns the raw bytes up front (the C API boundary) allocate - // the key and cell buffers exactly once and move them straight down to - // MemTable::upsert, instead of building a Batch (plain key/kind/value - // strings) that apply_batch would otherwise re-encode into a cell here. struct encoded_op { std::string key; buffer cell; }; - // Same semantics as apply() (oversized-key rejection, slot bookkeeping, - // maybe_swap_active), but for pre-encoded ops -- no encode_cell_buf call - // in here, no intermediate Batch/batch_op. Intra-batch: last occurrence - // (by vector order) wins, same as apply_batch. Status apply_encoded(uint64_t slot, std::vector<encoded_op> ops); - // One zero-copy op for apply_external (R30): `value` is a kExternal buffer - // borrowing bytes from a Rust `bytes::Bytes` (Put) or an empty buffer - // (Delete, `flags = kFlagTombstone`). The 9-byte cell header is NOT in the - // buffer — it is stored as the `flags` field here plus the `slot` argument, - // and materialized into a contiguous cell at MemTable drain/get. Lets the - // consensus apply path skip the value memcpy that encode_cell_buf performs. struct external_op { std::string key; - uint8_t flags = 0; // kPut (0) or kFlagTombstone - buffer value; // kExternal (borrowed) for Put; empty for Delete + uint8_t flags = 0; + buffer value; }; - // Same semantics as apply_encoded (oversized-key rejection, slot - // bookkeeping, maybe_swap_active, intra-batch last-key-wins) but stores - // split cells via MemTable::upsert_external — no encode_cell_buf, no value - // memcpy on the apply critical path. The value copy is deferred to flush - // (off the critical path). Status apply_external(uint64_t slot, std::vector<external_op> ops); - // Advance the contiguous frontier up to `slot`, filling any intervening slots - // as NoOps (e.g. after learner NoOp slots or during restore). Explicit and - // free of learner jargon. void force_advance_slot(uint64_t slot); - // Convenience methods: auto-assign the next slot (max_seen + 1) and apply. - // Intended for single-writer use; do not mix with explicit-slot apply calls. + // Automatic slots advance past explicit apply and NoOp slots. Status put(Slice key, Slice value); Status del(Slice key); Status batch_put(const Batch &batch); [[nodiscard]] Status validate_key(Slice key) const; - // Advisory separator from bounded resident index pages, without I/O or flush. - // Callers validate live keys on both sides before publishing a split. + [[nodiscard]] std::optional<std::string> approximate_split_key() const; - // Logical retention GC watermark: - // stores both slots and computes gc_floor_ = min(snapshot_slot, safe_slot). - // Tombstones with slot <= gc_floor_ may be dropped during snapshot - // preparation (folding) and block compaction. Using the min of the two - // (rather than safe_slot alone) is what makes it safe to call this before - // #20's learner wiring: a tombstone whose deletion isn't yet durable on a - // quorum (snapshot_slot) is never dropped early just because every member - // has locally applied it (safe_slot). Monotonic: gc_floor_ never regresses - // even if a later call passes a smaller min. void set_gc_watermark(uint64_t snapshot_slot, uint64_t safe_slot); - [[nodiscard]] uint64_t gc_watermark() const - { - return gc_floor_.load(); - } + [[nodiscard]] uint64_t gc_watermark() const; - // Cadence-driven block compaction (R129). Selects the sparsest source - // blocks above the configured free-ratio threshold, capped by the - // per-pass byte budget, and runs one snapshot that relocates their live - // extents (resident + unloaded pages and current mapping metadata) to - // destinations outside every selected source block. Eligible tombstones - // are folded during the same snapshot. After the second durability - // barrier and prepared-state commit, the shared finalizer deletes only - // blocks unreachable from any retained anchor. Non-block stores and - // disabled configurations return an empty stats result with no snapshot - // write. Uses the existing snapshot gate; no new mutex or blocking lock. Status compact_sparse_blocks(MergeGcStats *out_stats); - // Close the selected active table, capture a finite contiguous frontier, - // publish its successor, then wait for only the captured writers. Publish - // eligible prefix versions to L1 and detach wholly covered tables. Future - // versions remain in their original sources until a later flush covers - // them. This is in-memory publication; snapshot() establishes durability. Status flush(); - // Moves the currently visible L0 tables into one split-owned shared view - // and installs a fresh active table for post-prepare writes. Normal flush - // and reclamation leave that shared view alone until its split session - // explicitly releases it. Status begin_split_memtable_view(uint64_t *out_generation, uint64_t *out_journal_frontier); - // Makes the source's current L0 generations immediately visible to this - // range-bounded writer at `journal_frontier`. The source outlives the - // destination until clear_split_memtable_overlay() completes. Status install_split_memtable_overlay(Crowdbtree &source, uint64_t journal_frontier); - // Stops consulting the source L0 after its filtered entries have been - // bulk-published into this writer. Status clear_split_memtable_overlay(Crowdbtree &source); - // Releases the split-owned shared view after both derived range trees have - // durably published it. The generation fences stale release attempts. Status release_split_memtable_view(uint64_t generation); - // Bulk-publishes the split-owned shared view into one range-bounded - // destination. Source entries remain owned by the session; callers release - // them only after every destination has durably snapshotted its result. Status publish_split_memtable_view(uint64_t generation, uint64_t journal_frontier, Crowdbtree &destination, const KeyRange &range); - // Run finite flush on a completion worker. Waiting for admitted writers - // never blocks the submitting async caller. Destruction drains workers; - // completion callbacks run after the worker releases tree ownership. void flush_async(std::function<void(Status)> on_done); - // Point read (L0 overlay then L1). Returns true if a live value is found; - // tombstones return false. [[nodiscard]] bool get(Slice key, uint64_t *out_slot, std::string *out_value) const; - // Zero-copy point read (plan-tree #5 B3 remaining): same lookup as - // get(), but the returned GetView borrows an L1 hit's value directly - // from its resident frame instead of copying it out. See GetView's doc. + // The returned payload owns its bytes and remains valid after Tree close. [[nodiscard]] GetView get_view(Slice key) const; + // Preserve transient admission and storage errors for status-aware callers. + [[nodiscard]] Status get_view(Slice key, GetView *out) const; - // Async twin of get(). Fast path (every page needed to resolve `key` is already - // resident, or no async backend is wired -- see - // Config::async_page_store) invokes on_done synchronously, before this call - // returns, exactly like get(). A genuine miss on the L1 descent (some - // base page along the root->leaf path is tagged unloaded) submits - // exactly one page load via the reactor and resumes automatically; on - // completion it either resolves (calls on_done) or, if that page was - // itself only a step deeper into the tree, hits another miss and - // repeats -- on_done fires exactly once regardless, but for a miss it - // runs on the Reactor thread, not the caller's. - // - // Scope boundary (deliberate, matches the miss scenario): only - // the L1 base-page descent is async. A value spilled into an overflow - // chain (large values, PT11) still resolves its chain synchronously via - // the existing assemble_overflow_value()/resident() path -- overflow - // chains are the less common case and the miss - // walkthrough only describes "the leaf page is unloaded", not an - // overflow page. - // - // Zero-copy fast path: `on_done` - // receives the resolved `GetView` itself, not a copied-out std::string. - // For the *first* attempt's synchronous resolution (this call's own - // thread, no I/O), the GetView's epoch guard is still live -- it's safe - // to keep borrowing an L1 hit's frame bytes all the way out to the C - // ABI, since ct_future_free (which finally drops the guard) is - // guaranteed to run on this same thread too. Any resolution that - // crosses to the Reactor thread instead (a genuine miss) materializes - // an owned copy and releases its guard before calling on_done -- see - // get_async_attempt's `same_thread` parameter and `materialize_owned`. void get_async(Slice key, std::function<void(Status, GetView)> on_done) const; - // Batched point read. [[nodiscard]] std::vector<get_result> multi_get(const std::vector<Slice> &keys) const; - // Ordered range scan over keys with `prefix` (empty = whole keyspace), latest - // state (L0 overlaid on L1). When `include_tombstones` is false (default), - // tombstones are skipped. Returns up to `limit` entries in key order; sets - // *truncated if more matched beyond the limit. `start_after` (empty = start - // from the beginning) is an exclusive lower bound: only keys strictly - // greater than `start_after` are returned. When non-empty, the descent - // targets the leaf that would contain `start_after` instead of `prefix`, - // so a deep-pagination scan starts at the cursor rather than walking every - // earlier leaf in the prefix range. `end_key` (empty = unbounded) is an - // exclusive upper bound: only keys strictly less than `end_key` are - // returned, and the merge loop early-stops once the winner key reaches it. - // `byte_budget` (0 = unlimited) caps the total key+value bytes emitted; - // the scan stops with *truncated = true when exceeded, always returning at - // least one entry (so a single oversized entry still makes progress). A - // warning is logged for any single entry whose key+value size alone - // exceeds the budget. `keys_only` skips value materialization (no - // overflow-chain assembly, no value copy): entries are staged with empty - // values and the byte budget accounts for key bytes only, so a page fits - // more entries. Default false. Status scan(Slice prefix, Slice start_after, Slice end_key, size_t limit, size_t byte_budget, bool keys_only, uint64_t deadline_ms, std::vector<scan_entry> *out, bool *truncated, bool include_tombstones = false, ScanPackedBuf *out_packed = nullptr, size_t *out_count = nullptr, bool has_start_bound = false, bool start_inclusive = false, bool strict_byte_budget = false) const; - // Resolve the greatest live key <= `start_key`, or < it when `inclusive` - // is false, while merging L0 and L1 under one epoch guard. `begin_key` is - // an inclusive lower bound; empty means unbounded. Status seek_reverse(Slice start_key, bool inclusive, Slice begin_key, scan_entry *out, bool *found) const; - // Descending counterpart to scan(), with an explicit upper cursor and an - // inclusive lower bound. Work and materialization are bounded by the - // requested count and byte budget. Status scan_reverse(Slice start_key, bool has_start_bound, bool start_inclusive, Slice begin_key, size_t limit, size_t byte_budget, std::vector<scan_entry> *out, bool *truncated) const; - // Async twin of scan(). Unlike get_async, - // which has exactly one possible miss point (the root->leaf descent for - // a single key), scan() walks a whole range of leaves via - // right_sibling and any of them -- or an inner page on the initial - // descent to the first leaf -- can be cold. Rather than a resumable - // cursor, a miss simply retries the *whole* scan from scratch once the - // blocking page resolves (matches get_async_attempt's own "retry, still - // correct, not maximally efficient" trade-off) -- each retry is pure - // in-memory work except for exactly one more page becoming permanently - // resident, so this always terminates and does no redundant I/O. - // on_done fires exactly once, synchronously if the whole scan was - // already resident (matching scan()'s cost exactly), or from the - // Reactor thread after however many page loads were needed. `start_after` - // is the same exclusive lower bound as scan()'s. `end_key` is the same - // exclusive upper bound as scan()'s. `byte_budget` is the same total - // key+value byte cap as scan()'s. `keys_only` is the same value-skip flag - // as scan()'s. void scan_async(Slice prefix, Slice start_after, Slice end_key, size_t limit, size_t byte_budget, bool keys_only, uint64_t deadline_ms, std::function<void(Status, ScanPackedBuf, bool truncated)> on_done) const; - // Directional form used by the ordinary KV API. `reverse` keeps - // `start_after` as an exclusive continuation, but walks toward smaller - // keys. With no continuation it begins below `end_key`, or below the - // prefix successor when `end_key` is empty. void scan_directional_async(Slice prefix, Slice start_after, Slice end_key, size_t limit, size_t byte_budget, bool keys_only, uint64_t deadline_ms, bool reverse, std::function<void(Status, ScanPackedBuf, bool truncated)> on_done) const; - // pin a consistent point-in-time view at `last_applied_slot` (the published L1 - // state). Used for scan-at / compare / iter_all / snapshot export. - // R6: returns a PinnedSnapshot (zero-copy, page refcount pins keep frames - // alive across threads). The return type is shared_ptr<Snapshot> for ABI - // compatibility with existing callers; PinnedSnapshot inherits from Snapshot. [[nodiscard]] std::shared_ptr<Snapshot> snapshot_view(); - // Replace the entire engine state with `sorted_entries` (key-sorted, including - // tombstones) at `at_slot`, used by snapshot import. Clears L0/L1 and rebuilds - // a fresh tree, then sets last_applied_slot = at_slot. Serialized against other - // writers by write_mutex_. Concurrent lock-free readers are **safe** (#13): the - // old tree is epoch-retired, not freed, so a reader mid-walk keeps its pages - // under its guard (it may observe a transient empty/partly-replaced tree — a - // consistent snapshot swap via a pinned RootVersion is a later refinement). Status install_snapshot(std::vector<leaf_entry> sorted_entries, uint64_t at_slot); - // plan-tree #16: native snapshot format. `collect_native_frames` walks - // the reachable tree (root -> inner children -> leaf overflow chains, - // folding any delta chain into a fresh consolidated base first -- same - // side effect `snapshot()` already has, unlike the read-only - // `snapshot_view()`) and returns every base/overflow page's *raw frame - // bytes* verbatim, tagged with its PID -- no cell decode, no tuple - // encoding. `install_snapshot_native` is the inverse: installs each - // frame directly as a resident page under its original PID (via - // `from_frame_copy`, the same reconstruction demand-load already uses), - // no entry-by-entry tree rebuild. Both intended for crowdb-tree-to-crowdb-tree - // transfer (Raft InstallSnapshot); `install_snapshot`/`snapshot_view`'s - // portable tuple format remains available for cross-engine scenarios - // and testing (comparable against a non-crowdb-tree oracle). The optional - // next-PID high-water output lets range rebuilds allocate rewritten pages - // beyond every PID ever issued by the selected source lineage. Status collect_native_frames(std::vector<NativeFrame> *out, uint64_t *out_root_page_id, uint64_t *out_at_slot, uint64_t *out_next_page_id = nullptr, const KeyRange *filter = nullptr, uint64_t *out_subtrees_skipped = nullptr); @@ -396,62 +168,20 @@ class Crowdbtree Status install_snapshot_native(std::vector<NativeFrame> frames, uint64_t root_page_id, uint64_t at_slot, uint64_t next_page_id = 0); - // Wipe every key/value and reset watermarks back to a fresh, empty tree - // (the same wipe `install_snapshot` performs on the live tree before - // loading imported entries, factored out for a caller that wants an - // empty tree with nothing to load afterward -- e.g. resetting a - // diverged/corrupted replica in place before a snapshot import). - // Serialized against other writers by write_mutex_; concurrent - // lock-free readers are safe (#13), same as `install_snapshot`. Not - // durable by itself -- an explicit `snapshot()`/`flush()` afterward is - // required to persist the wipe to a file-backed store. + // Install an empty namespace atomically; a subsequent snapshot makes it durable. Status clear(); - // Reassemble a large value spilled into an overflow chain headed at `head_page_id` - // (PT11). Walks the chain via resident under the caller's read epoch guard. - [[nodiscard]] std::string assemble_overflow_value(uint64_t head_page_id, uint64_t total_len) const; - - [[nodiscard]] uint64_t last_applied_slot() const - { - return last_applied_slot_.load(); - } - - [[nodiscard]] uint64_t contiguous_slot() const - { - return contiguous_slot_.load(); - } - - [[nodiscard]] uint64_t version() const - { - return version_.load(); - } + [[nodiscard]] uint64_t last_applied_slot() const; + [[nodiscard]] uint64_t contiguous_slot() const; + [[nodiscard]] uint64_t version() const; - // Read one immutable base frame through a bounded root/child path. Does not - // flush memtables, fold deltas, or follow overflow values. UINT64_MAX starts - // a new observation; subsequent requests must retain the returned version. Status inspect_page(const std::vector<uint32_t> &path, uint64_t expected_version, NativeFrame *out, uint64_t *out_version, uint64_t *out_root, uint32_t *out_deltas) const; - [[nodiscard]] uint64_t durable_snapshot_seq() const - { - return durable_snapshot_seq_.load(std::memory_order_acquire); - } - - [[nodiscard]] uint64_t durable_snapshot_last_applied_slot() const - { - return durable_snapshot_last_applied_slot_.load(std::memory_order_acquire); - } - - [[nodiscard]] uint64_t root_page_id() const - { - return root_page_id_.load(); - } + [[nodiscard]] uint64_t durable_snapshot_seq() const; + [[nodiscard]] uint64_t durable_snapshot_last_applied_slot() const; + [[nodiscard]] uint64_t root_page_id() const; - // Latched media-fault flag (design follow-up). A demand-load that fails to - // read or validate a durable page (`resident`) cannot return an error through - // the lock-free read path, so it latches this flag (and the page reads as a - // miss). A caller can poll this after reads to detect on-disk corruption / - // I/O faults and fail the node out of the group. `clear_io_error` resets it. [[nodiscard]] bool io_failed() const { return io_failed_.load(); @@ -462,475 +192,62 @@ class Crowdbtree io_failed_.store(false); } - // Diagnostics: total entries across every live MemTable (active_ + any - // not-yet-drained frozen_ buffers), not just active_. [[nodiscard]] size_t memtable_count() const; - // Gap 5 step 2: number of frozen memtables waiting to be drained. - // The maintenance loop polls this to decide whether to flush - // immediately or sleep until the next tick. [[nodiscard]] size_t frozen_table_count() const; - MappingTable &mapping() - { - return mapping_; - } + [[nodiscard]] int height() const; + [[nodiscard]] size_t leaf_count() const; - // Diagnostics/tests: the tree-owned epoch manager (plan-tree #7). - EpochManager &epoch() - { - return epoch_; - } + [[nodiscard]] uint64_t leaf_count_atomic() const; + [[nodiscard]] uint64_t inner_count_atomic() const; - [[nodiscard]] const BufferPool *buffer_pool() const - { - return pool_.get(); - } + // Physical frames written by the latest successful durable checkpoint; excludes reused frames. + [[nodiscard]] uint64_t last_snapshot_pages_written() const; - [[nodiscard]] int height() const; // 1 = single-leaf root - [[nodiscard]] size_t leaf_count() const; // live leaves reachable from the root - - // O(1) atomic snapshot of the leaf/inner page counters (maintained at - // SMO sites, restored from the commit anchor on open()). For tests and - // metrics — prefer these over leaf_count() (which walks the tree). - [[nodiscard]] uint64_t leaf_count_atomic() const - { - return leaf_count_.load(std::memory_order_relaxed); - } - - [[nodiscard]] uint64_t inner_count_atomic() const - { - return inner_count_.load(std::memory_order_relaxed); - } - - // # of base pages physically written by the most recent snapshot (the rest - // were clean and retained their durable addr). For incremental-snapshot tests. - [[nodiscard]] uint64_t last_snapshot_pages_written() const - { - return snapshot_pages_written_.load(); - } - - // # of mapping-table segment images physically written by the most - // recent snapshot (plan-tree #14c/#14d: the rest were !is_dirty() and - // reused their existing image_addr/generation as-is). For - // incremental-snapshot tests -- the segment-level analogue of - // last_snapshot_pages_written(). + // Deprecated dirty-segment observation. Use actual written-frame counts; returns zero. [[nodiscard]] uint64_t last_snapshot_segments_written() const { - return snapshot_segments_written_.load(); + return 0; } - // Batched diagnostics snapshot -- see EngineStats. O(1): every field - // reads an already-tracked atomic counter or BufferPool::stats() (also - // O(1)), so this is safe to poll periodically (e.g. from a metrics - // scrape or console panel refresh). - [[nodiscard]] EngineStats stats() const; - [[nodiscard]] TreeSummary tree_summary() const; + [[nodiscard]] EngineStats stats() const; + [[nodiscard]] TreeStatsSnapshot tree_state_stats() const; + [[nodiscard]] BackingStatsSnapshot backing_stats() const noexcept; + [[nodiscard]] TreeSummary tree_summary() const; - // Destructive read of the per-step scan profile since the last call: flushes - // the scan LatencySummary/Counter handles and returns per-step sum/max/avg. - // Returns an all-zero profile if init_metrics() was never called. + // Deprecated empty compatibility profile; use the work metrics collector. [[nodiscard]] ScanProfile scan_profile() const; - // Create the internal MetricsRegistry and register all handles - // using the provided name prefix (e.g. "s.1.g.0"). Called from open(). void init_metrics(const std::string &prefix, const std::string &backend_label); - // Flush all C++ metrics into a formatted string (for FFI return to - // Rust). Uses open_memstream internally. `width` overrides the - // per-section max name length for column alignment with the Rust - // section (0 = use internal max). std::string flush_metrics_str(double window_secs, const char *timestamp, size_t width = 0, size_t count_w = 0, size_t tps_w = 0); - // Return the current max metric name length (for Rust's shared-width - // computation). size_t max_name_len() const; - // Evict clean, delta-free resident leaf bases down to at most - // `max_resident_leaves`, re-tagging their slots unloaded and epoch-retiring the - // pages; returns the number evicted. Safe against lock-free - // readers (epoch-deferred frame reuse); evicted pages reload on next access. [[nodiscard]] size_t evict_clean_leaves(size_t max_resident_leaves); - // plan-tree #17 D3: same contract as evict_clean_leaves, but for clean, - // delta-free resident *inner* bases, ranked and budgeted entirely - // separately -- see evict_clean_inner_locked's doc comment (crowdb-tree.cpp) - // for why a combined leaf+inner budget is unsafe (breaks the - // just-touched-leaf-survives-eviction guarantee). Never evicts a leaf; - // evict_clean_leaves never evicts an inner base. [[nodiscard]] size_t evict_clean_inner(size_t max_resident_inner); - // Effective key size limit (opt_.max_key_size or frame_bytes/2). Keys larger - // than this are rejected at apply() (plan-tree #15). [[nodiscard]] size_t max_key_size() const { return opt_.max_key_size != 0 ? opt_.max_key_size : opt_.frame_bytes / 2; } private: - friend class NativeFrameIterator; - friend Status rebuild_range(Crowdbtree &source, const KeyRange &range, Config destination_options, - std::unique_ptr<Crowdbtree> *out, RangeRebuildStats *stats); - [[nodiscard]] bool seek_reverse_guarded(Slice start_key, bool has_start_bound, bool inclusive, Slice begin_key, - const std::vector<MemTableSource> &memtables, uint64_t root_page_id, - uint64_t gc_floor, scan_entry *out) const; - - [[nodiscard]] bool try_scan_reverse_no_load(Slice prefix, Slice start_after, Slice end_key, size_t limit, - size_t byte_budget, bool keys_only, uint64_t deadline_ms, - ScanPackedBuf *out_packed, size_t *out_count, bool *truncated, - uint64_t *out_pending_page_id) const; -#ifdef CROWDB_TREE_TEST_UTIL - friend struct MemTableAccess_for_tests; - void (*after_flush_group_for_tests_)() = nullptr; -#endif - struct FlushBoundary - { - std::deque<std::shared_ptr<MemTable>> tables; - uint64_t frontier = 0; - }; - - FlushBoundary capture_flush_locked(); - void publish_flush_locked(FlushBoundary &boundary); - // apply a batch's ops into L0 at `slot` (intra-batch last-op-wins). - void apply_batch(uint64_t slot, const Batch &batch); - MemTableBatch admit_batch(); - void finish_batch(MemTableBatch &batch, uint64_t slot, const std::vector<std::string> &keys); - // Shared apply()/apply_encoded() tail: slot bookkeeping (max_seen_slot_, - // received_slots_, contiguous frontier) then a possible L0 size-based swap. - void note_applied_slot(uint64_t slot); - // Fold newly received slots into the contiguous prefix, then prune the - // tracker below the new frontier. Caller holds slot_mutex_. - void recompute_contiguous_locked(); - - [[nodiscard]] std::shared_ptr<MemTable> current_active() const; - [[nodiscard]] std::vector<MemTableSource> all_memtables() const; - [[nodiscard]] std::vector<MemTableSource> local_memtables(uint64_t covered = UINT64_MAX) const; - bool maybe_freeze_active(bool force); - void maybe_swap_active(); - bool drain_all_frozen_locked(std::deque<std::shared_ptr<MemTable>> &to_drain, uint64_t cs); - bool publish_group_to_leaf_locked(uint64_t page_id, uint64_t cs, std::vector<leaf_entry> group); - - void consolidate_locked(uint64_t page_id); // caller holds write_mutex_ - void maybe_split_or_merge_locked(uint64_t page_id); // dispatch on leaf size - void set_children_parent_locked(uint64_t page_id, uint64_t parent_page_id); - void store_preserving_parent_locked(uint64_t page_id, PageBase *new_page); - void refresh_page_summary_locked(PageBase *page); - void refresh_subtree_summaries_locked(uint64_t page_id); - void sync_page_count_gauges(); - std::vector<uint64_t> path_to_page_id_locked(uint64_t target_page_id) const; - void split_leaf_to_threshold_locked(uint64_t leaf_page_id); - void split_leaf_locked(uint64_t leaf_page_id, std::vector<uint64_t> path); - void propagate_split_locked(std::vector<uint64_t> path, uint64_t child_page_id, std::string sep, - uint64_t right_page_id); - void try_merge_leaf_locked(uint64_t leaf_page_id, const std::vector<uint64_t> &path); - void try_merge_inner_locked(uint64_t inner_page_id, std::vector<uint64_t> path); - - [[nodiscard]] uint32_t inner_merge_keys() const - { - if (opt_.inner_merge_keys != 0) { - return opt_.inner_merge_keys; - } - uint32_t q = opt_.inner_max_keys / 4; - return q != 0 ? q : 1; - } - - void retire_page(PageBase *p); - void preserve_native_page_locked(uint64_t page_id, PageBase *page); - void preserve_native_generation_locked(); - void detach_native_iterators(); - Status install_range_snapshot_native(std::vector<NativeFrame> frames, uint64_t root_page_id, uint64_t at_slot, - uint64_t next_page_id, bool mapping_inherited); - void retire_orphaned_page(uint64_t page_id, PageBase *p); - void free_subtree(uint64_t page_id, bool retire); - void free_all_resident_pages(bool retire); - - [[nodiscard]] size_t max_inline_value() const - { - return opt_.max_inline_value != 0 ? opt_.max_inline_value : opt_.frame_bytes / 4; - } - - [[nodiscard]] static std::vector<leaf_entry> - resolve_leaf_chain_for_rebuild(PageBase *head, uint64_t gc_floor, std::vector<uint64_t> *dead_overflow, - size_t *out_tombstones_dropped = nullptr, size_t *out_bytes_dropped = nullptr); - [[nodiscard]] uint64_t spill_value_to_overflow_chain_locked(const std::string &value); - [[nodiscard]] LeafBase *build_leaf_spilling_locked(std::vector<leaf_entry> entries, uint64_t right_sibling); - void retire_overflow_chain_locked(uint64_t head_page_id); - void evict_overflow_chain_locked(uint64_t head_page_id); - void free_overflow_chain(uint64_t head_page_id); - size_t evict_clean_leaves_locked(size_t max_resident_leaves); // caller holds write_mutex_ - size_t evict_clean_inner_locked(size_t max_resident_inner); // caller holds write_mutex_ - void maybe_evict_locked(); // capacity-driven auto-evict (caller holds write_mutex_) - [[nodiscard]] PageBase *resident(uint64_t page_id) const; - - void capture_overflow_chain(uint64_t head_page_id, std::vector<PageBase *> &out); - - [[nodiscard]] PageBase *install_loaded_page(uint64_t page_id, uint64_t addr, uint32_t plen, - const std::vector<uint8_t> &blob) const; - - [[nodiscard]] bool try_get_view_no_load(Slice key, GetView *result, uint64_t *out_pending_page_id) const; - - void get_async_attempt(std::shared_ptr<std::string> key_owned, std::function<void(Status, GetView)> on_done, - bool same_thread) const; - - static GetView materialize_owned(GetView &&v); - - [[nodiscard]] bool try_scan_no_load(Slice prefix, Slice start_after, Slice end_key, size_t limit, - size_t byte_budget, bool keys_only, uint64_t deadline_ms, - std::vector<scan_entry> *out, bool *truncated, uint64_t *out_pending_page_id, - ScanPackedBuf *out_packed = nullptr, size_t *out_count = nullptr) const; - - void scan_async_attempt(std::shared_ptr<std::string> prefix_owned, - const std::shared_ptr<std::string> &start_after_owned, - const std::shared_ptr<std::string> &end_key_owned, size_t limit, size_t byte_budget, - bool keys_only, uint64_t deadline_ms, std::shared_ptr<ScanPackedBuf> accumulated, - std::shared_ptr<std::string> last_key, size_t accumulated_count, - std::function<void(Status, ScanPackedBuf, bool)> on_done) const; - - void scan_reverse_async_attempt(std::shared_ptr<std::string> prefix_owned, - const std::shared_ptr<std::string> &start_after_owned, - const std::shared_ptr<std::string> &end_key_owned, size_t limit, size_t byte_budget, - bool keys_only, uint64_t deadline_ms, std::shared_ptr<ScanPackedBuf> accumulated, - std::shared_ptr<std::string> last_key, size_t accumulated_count, - std::function<void(Status, ScanPackedBuf, bool)> on_done) const; - - Status prepare_snapshot_locked(PreparedSnapshot *out, std::vector<PrefetchedPage> prefetched = {}, - std::set<uint32_t> relocation_blocks = {}); - struct SnapshotPrepareContext; - Status prepare_snapshot_pages_locked(SnapshotPrepareContext &ctx); - Status fold_snapshot_page_locked(uint64_t page_id, uint64_t gc, PageBase **page); - Status queue_snapshot_page_locked(SnapshotPrepareContext &ctx, uint64_t page_id, PageBase *page, - const uint8_t *frame, uint32_t frame_len, bool relocate, uint64_t *addr, - uint32_t *logical_len) const; - Status prepare_snapshot_resident_page_locked(SnapshotPrepareContext &ctx, uint64_t page_id, uint64_t seg_idx); - Status prepare_snapshot_segments_locked(SnapshotPrepareContext &ctx); - static Status prepare_snapshot_slot_locked(SnapshotPrepareContext &ctx, uint64_t page_id, uint64_t word, - uint64_t *durable_word, uint32_t *live_count); - static Status prepare_snapshot_segment_locked(SnapshotPrepareContext &ctx, uint64_t seg_idx, - MappingSegment *segment); - void prepare_snapshot_metadata_locked(SnapshotPrepareContext &ctx); - Status prefetch_sparse_pages(std::vector<PrefetchedPage> *out, std::set<uint32_t> *selected_blocks); - Status persist_compaction_snapshot(std::vector<PrefetchedPage> prefetched, std::set<uint32_t> selected_blocks, - PreparedSnapshot *prepared); - void record_compaction_metrics(const MergeGcStats &stats, uint64_t elapsed_ns); - - void commit_prepared_snapshot(const PreparedSnapshot &prepared); - void finalize_prepared_snapshot(PreparedSnapshot &prepared); - - void acquire_snapshot_slot(); - void release_snapshot_slot(); - Status retry_snapshot_anchor(); - void snapshot_prepare_async(std::function<void(Status, uint64_t last_applied)> on_done); - - void snapshot_write_next_async(std::shared_ptr<PreparedSnapshot> prepared, size_t idx, - std::function<void(Status, uint64_t last_applied)> on_done); - - Config opt_; - std::string name_; - std::shared_ptr<BufferPool> pool_; - MappingTable mapping_; - - // The catalog lock protects source ownership and its published L1 floor. - // Writers register on the selected table before mutation; close/capture/ - // successor publication is one catalog transition. Frozen tables retain - // their immutable future versions until a later prefix covers the table. - // Borrowers keep source ownership and epoch protection independently. - std::shared_ptr<std::atomic<Crowdbtree *>> reclamation_owner_ = std::make_shared<std::atomic<Crowdbtree *>>(this); - std::shared_ptr<std::atomic<uint64_t>> async_flushes_ = std::make_shared<std::atomic<uint64_t>>(0); - bool publication_incomplete_ = false; // protected by write_mutex_ - MemTableCounters memtable_counters_; - GenerationGate generation_; - mutable std::shared_mutex memtable_mutex_; - std::shared_ptr<MemTable> active_; - std::deque<std::shared_ptr<MemTable>> frozen_; - std::vector<std::shared_ptr<MemTable>> split_shared_memtables_; - std::atomic<Crowdbtree *> split_overlay_source_{nullptr}; - std::atomic<uint64_t> split_overlay_frontier_{0}; - uint64_t split_memtable_generation_ = 0; - uint64_t split_memtable_frontier_ = 0; - std::atomic<uint64_t> memtable_next_id_{1}; // monotonic MemTable id for logging - - // internal_error slot tracker (replaces the caller-supplied contiguous_slot). Holds - // received-but-not-yet-contiguous slots above contiguous_slot_; the contiguous - // prefix is folded forward on each apply/force_advance_slot and pruned below - // the frontier to stay bounded. Guarded by slot_mutex_. - mutable std::mutex slot_mutex_; - std::set<uint64_t> received_slots_; - uint64_t max_seen_slot_ = 0; - std::atomic<uint64_t> auto_slot_{0}; // next auto-assigned slot for put/del/batch_put - - std::atomic<uint64_t> root_page_id_{kInvalidPageId}; - std::atomic<uint64_t> contiguous_slot_{0}; - std::atomic<uint64_t> last_applied_slot_{0}; - std::atomic<uint64_t> version_{0}; - std::atomic<uint64_t> durable_snapshot_seq_{0}; - std::atomic<uint64_t> durable_snapshot_last_applied_slot_{0}; - - struct MappingMaterializationState - { - uint64_t version = 0; - uint64_t next_segment = 0; - std::vector<uint64_t> stack; - std::vector<uint64_t> reachable; - }; - - std::unique_ptr<MappingMaterializationState> mapping_materialization_; - uint64_t mapping_pruned_version_ = std::numeric_limits<uint64_t>::max(); - std::atomic<uint64_t> gc_floor_{0}; - std::atomic<uint64_t> snapshot_pages_written_{0}; // pages written by last snapshot - std::atomic<uint64_t> snapshot_pages_total_{0}; // cumulative pages written across all snapshots - std::atomic<uint64_t> snapshot_segments_written_{0}; // segment images written by last snapshot - // Freshly built and native-imported trees have globally verified routing - // separators. Lazy persistent recovery verifies them on the first bounded - // export before allowing separator-based subtree pruning. - std::atomic<bool> routing_fences_trusted_{true}; - mutable std::atomic<bool> io_failed_{false}; // latched demand-load media fault - - // Cumulative operation counters (monotonic since open, exposed via stats()). - // mutable: get_view() is const but increments these counters. - mutable std::atomic<uint64_t> mt_upsert_total_{0}; // apply() writes into L0 - mutable std::atomic<uint64_t> mt_get_total_{0}; // get() lookups in L0 - mutable std::atomic<uint64_t> mt_get_hit_total_{0}; // L0 lookups that found a cell - mutable std::atomic<uint64_t> flush_drain_total_{0}; // drain_all_frozen_locked calls - mutable std::atomic<uint64_t> flush_entries_total_{0}; // entries drained from L0 to L1 - mutable std::atomic<uint64_t> snapshot_total_{0}; // snapshot() calls (durable checkpoints) - mutable std::atomic<uint64_t> l1_get_total_{0}; // get() lookups that descended to L1 - mutable std::atomic<uint64_t> l1_get_hit_total_{0}; // L1 lookups that found a cell - mutable std::atomic<uint64_t> map_lookup_total_{0}; // mapping table lookups (find_leaf_page_id / resident) - mutable std::atomic<uint64_t> demand_load_total_{0}; // demand-load page faults - - // Live leaf/inner page counts (O(1) gauges, maintained at SMO sites and - // restored from the commit anchor on open()). See leaf_count_atomic() / - // inner_count_atomic(). An empty tree starts at leaf=1 (root leaf), inner=0. - std::atomic<uint64_t> leaf_count_{1}; - std::atomic<uint64_t> inner_count_{0}; - mutable std::atomic<uint64_t> summary_root_version_{0}; - mutable std::atomic<uint64_t> summary_covered_slot_{0}; - mutable std::atomic<uint64_t> summary_live_kv_{0}; - mutable std::atomic<uint64_t> summary_live_key_bytes_{0}; - mutable std::atomic<uint64_t> summary_live_value_bytes_{0}; - mutable std::atomic<uint64_t> summary_reachable_overflow_pages_{0}; - mutable std::atomic<uint64_t> summary_reachable_page_capacity_bytes_{0}; - mutable std::atomic<uint64_t> summary_live_logical_bytes_{0}; - mutable std::atomic<bool> summary_available_{false}; - - // Logical clock for CLOCK-informed eviction ranking (plan-tree #17). - // `resident()`'s hot path bumps this and stamps the touched page's own - // `PageBase::last_touch_tick` on every access (a single relaxed atomic - // fetch_add + store -- no lock, so the existing lock-free read path - // stays lock-free). `evict_clean_leaves_locked` then ranks its - // DFS-gathered evictable set by that stamp (oldest first) instead of - // arbitrary DFS order. This is deliberately *not* `BufferPool::pin`'s - // own mutex-guarded page_id/CLOCK tracking: wiring every `resident()` - // hit through that would mean taking a global pool mutex on every page - // access, regressing the lock-free read path #5 B3/#12/#13 built (see - // "residency/eviction driven by real access recency, not arbitrary - // order" goal without that cost. - mutable std::atomic<uint64_t> touch_tick_{0}; - - mutable std::mutex write_mutex_; // serializes flush / consolidate / split-merge - std::vector<std::weak_ptr<NativeFrameIterator::Impl>> native_frame_iterators_; - mutable std::mutex load_mutex_; // serializes cold-path demand loads - // Serializes snapshot(_async) generations against each other across - // snapshot_async's async write phase, where write_mutex_ itself can't be - // held (see acquire_snapshot_slot's doc comment and snapshot_async's). - std::atomic<bool> snapshot_inflight_{false}; - // Protected by snapshot_inflight_: retry an uncertain anchor barrier before - // preparing or writing another image over the backend's staged snapshot. - bool snapshot_anchor_pending_ = false; - - // Block indices that were empty in the previous snapshot (two-generation - // rule for block deletion). A block is only deleted after it's empty in - // two consecutive snapshots — the crash fallback anchor still references it. - - // Tree-owned epoch-based reclamation (plan-tree #7; formerly on CrowdbtreeEnv). - // Declared last so it is destroyed first: ~Crowdbtree frees the live tree via - // free_subtree(root, /*retire=*/false) (no readers at teardown), then epoch_'s - // destructor reclaims any pages still pending from earlier retire()s (eviction, - // consolidation, install_snapshot) while pool_ / mapping_ are still alive. - // mutable: readers take a guard in const get(). - mutable EpochManager epoch_; - - // ── Metrics handles (registered in init_metrics) ── - struct MetricsHandles - { - LatencySummary *mt_version_copy_l = nullptr; - // Buffer pool (backend I/O) - Counter *buf_evictions = nullptr; - Counter *buf_writebacks = nullptr; - Gauge *buf_resident = nullptr; - Gauge *buf_dirty = nullptr; - // Flush (L0 → L1) - LatencySummary *flush_l = nullptr; - Counter *flush_drain_c = nullptr; - Counter *flush_entries_c = nullptr; - LatencySummary *split_view_begin_l = nullptr; - LatencySummary *split_view_publish_l = nullptr; - LatencySummary *split_view_release_l = nullptr; - // MemTable (L0) operation latency - LatencySummary *mt_apply_l = nullptr; - Counter *mt_get_c = nullptr; - Counter *mt_get_hit_c = nullptr; - LatencySummary *mt_get_l = nullptr; - CallbackGauge *mt_frozen_g = nullptr; - CallbackGauge *mt_records_g = nullptr; - Counter *mt_freeze_c = nullptr; - // L1 (B-tree) query counters + latency - Counter *l1_get_c = nullptr; - Counter *l1_get_hit_c = nullptr; - LatencySummary *l1_get_l = nullptr; - // B+tree page mutation (during drain/split/merge/consolidate) - LatencySummary *page_write_l = nullptr; - Counter *page_split_c = nullptr; - Counter *page_merge_c = nullptr; - Counter *page_consolidate_c = nullptr; - // Tree structure - CallbackGauge *tree_height_g = nullptr; - Gauge *tree_leaf_count_g = nullptr; - Gauge *tree_inner_count_g = nullptr; - CallbackGauge *tree_retired_count_g = nullptr; - // Mapping table - Counter *page_find_c = nullptr; - Counter *page_map_alloc_c = nullptr; - CallbackGauge *page_map_total_pids_g = nullptr; - CallbackGauge *page_map_segments_g = nullptr; - // Demand-load (page fault I/O) latency - LatencySummary *page_load_l = nullptr; - // Page writeback (eviction I/O) latency - LatencySummary *page_writeback_l = nullptr; - // Page writeback bandwidth (eviction I/O, non-snapshot) - Bandwidth *page_writeback_bw = nullptr; - // fsync/barrier latency - LatencySummary *fsync_l = nullptr; - // Snapshot — logical (full snapshot wall time) - LatencySummary *snapshot_l = nullptr; - // Snapshot I/O sub-metrics (backend) - LatencySummary *snapshot_apply_l = nullptr; - LatencySummary *snapshot_page_write_l = nullptr; - Counter *snapshot_page_write_cache_c = nullptr; - Bandwidth *snapshot_page_write_bw = nullptr; - Bandwidth *snapshot_meta_write_bw = nullptr; - Bandwidth *page_read_bw = nullptr; - Counter *snapshot_pages_c = nullptr; - // Scan - Counter *scan_entries_c = nullptr; - LatencySummary *scan_l = nullptr; - LatencySummary *scan_l0_l = nullptr; - LatencySummary *scan_l1_l = nullptr; - LatencySummary *scan_merge_l = nullptr; - Counter *scan_retry_c = nullptr; - // GC (snapshot folding + block compaction) - Counter *gc_tombstones_c = nullptr; // tombstones dropped by snapshot folding - Counter *merge_gc_blocks_c = nullptr; // source blocks selected per compaction pass - Counter *merge_gc_relocated_c = nullptr; // pages relocated per compaction pass - Counter *merge_gc_deleted_c = nullptr; // blocks deleted per compaction pass - LatencySummary *merge_gc_l = nullptr; // compaction pass latency - }; - - MetricsHandles metrics_; - - // Internal metrics registry (owned by the engine). - std::unique_ptr<MetricsRegistry> metrics_registry_; + friend struct TreeCoreAccess; + friend Status rebuild_range(Crowdbtree &, const KeyRange &, Config, std::unique_ptr<Crowdbtree> *, + RangeRebuildStats *); + Crowdbtree(Config opt, std::shared_ptr<TreeCore> core); + void record_compaction_metrics(const MergeGcStats &stats, uint64_t elapsed_ns); + void init_core_metrics(const std::string &prefix); + + Config opt_; + std::string name_; + std::shared_ptr<TreeCore> core_; + AllocationLease core_facade_charge_; + std::shared_ptr<MetricsRegistry> metrics_registry_; + mutable std::atomic<bool> io_failed_{false}; }; - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/work_cause.h b/lib/crowdb-tree/include/crowdb-tree/btree/work_cause.h new file mode 100644 index 000000000..8f30206a6 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/btree/work_cause.h @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include <cstdint> + +namespace crowdb::tree +{ +// Fixed attribution carried across asynchronous I/O. Other is an explicit +// unclassified caller, never inferred from a value or registered dynamically. +enum class TreeWorkCause : uint8_t { + Other, + Flush, + CachePressure, + Checkpoint, + GarbageCollection, + Read, + Snapshot, + Allocation, + Count +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/buffer.h b/lib/crowdb-tree/include/crowdb-tree/buffer.h index 4838b5029..05b4117fe 100644 --- a/lib/crowdb-tree/include/crowdb-tree/buffer.h +++ b/lib/crowdb-tree/include/crowdb-tree/buffer.h @@ -36,10 +36,12 @@ #include <cstdlib> #include <cstring> #include <limits> +#include <memory> #include <new> namespace crowdb::tree { +class ExternalAllocation; class buffer { @@ -118,15 +120,25 @@ class buffer static buffer wrap_external(const uint8_t *data, size_t len, void *owner, void (*drop_fn)(void *)) { buffer b; - b.mode_ = mode::kExternal; - b.heap_ = const_cast<uint8_t *>(data); // borrowed; never freed through heap_ - b.size_ = len; - b.capacity_ = len; - b.ext_.drop_fn_ = drop_fn; - b.ext_.owner_ = owner; + b.mode_ = mode::kExternal; + b.heap_ = const_cast<uint8_t *>(data); // borrowed; never freed through heap_ + b.size_ = len; + b.capacity_ = len; + b.ext_.drop_fn_ = drop_fn; + b.ext_.owner_ = owner; + b.ext_.allocation_ = nullptr; return b; } + // Retain a checked slice of one capacity-accounted allocation. This keeps + // the canonical owner alive without charging its payload again per slice. + static buffer share_external(std::shared_ptr<const ExternalAllocation> allocation, size_t offset, size_t len); + + [[nodiscard]] const ExternalAllocation *external_allocation() const noexcept + { + return mode_ == mode::kExternal ? ext_.allocation_ : nullptr; + } + // Take ownership of a raw heap pointer already allocated by the matching // allocator (the alloc() seam, i.e. std::malloc). Always heap (not inline). static buffer move_from(uint8_t *data, size_t len, size_t cap) @@ -290,9 +302,10 @@ class buffer heap_ = nullptr; } else if (mode_ == mode::kExternal) { - heap_ = o.heap_; - ext_.drop_fn_ = o.ext_.drop_fn_; - ext_.owner_ = o.ext_.owner_; + heap_ = o.heap_; + ext_.drop_fn_ = o.ext_.drop_fn_; + ext_.owner_ = o.ext_.owner_; + ext_.allocation_ = o.ext_.allocation_; } else { heap_ = o.heap_; @@ -338,8 +351,9 @@ class buffer struct { - void (*drop_fn_)(void *); // called on destruction (kExternal) - void *owner_; // opaque Rust handle (e.g. boxed Arc<Bytes>) + void (*drop_fn_)(void *); // called on destruction (kExternal) + void *owner_; // opaque Rust handle (e.g. boxed Arc<Bytes>) + const ExternalAllocation *allocation_; } ext_; }; }; diff --git a/lib/crowdb-tree/include/crowdb-tree/c_api.h b/lib/crowdb-tree/include/crowdb-tree/c_api.h index 35e7c7a10..2d3f539a0 100644 --- a/lib/crowdb-tree/include/crowdb-tree/c_api.h +++ b/lib/crowdb-tree/include/crowdb-tree/c_api.h @@ -13,6 +13,9 @@ #ifndef CROWDB_TREE_C_API_H #define CROWDB_TREE_C_API_H +#include "crowdb-tree/runtime_c_api.h" +#include "crowdb-tree/tuning_c_api.h" + #include <cstddef> #include <cstdint> @@ -32,6 +35,8 @@ using ct_iter = struct ct_iter; using ct_export = struct ct_export; using ct_import = struct ct_import; using ct_write_handle = struct ct_write_handle; +using ct_scan_result = struct ct_scan_result; +using ct_future = struct ct_future; using ct_uring = struct ct_uring; using ct_uring_callback = void (*)(void *context, int32_t result); @@ -44,6 +49,44 @@ using ct_buf = struct void ct_free_buf(ct_buf *buf); +// Versioned bounded scan with an opaque owner. The owner retains actual +// allocation capacity/charge until ct_scan_result_free, including after Tree +// closure. The borrowed data must never be passed to ct_free_buf. +struct ct_scan_options_v2 +{ + size_t struct_size; + const uint8_t *prefix; + size_t prefix_len; + const uint8_t *start; + size_t start_len; + const uint8_t *end; + size_t end_len; + const uint8_t *begin; + size_t begin_len; + size_t limit; + size_t byte_budget; + uint64_t deadline_ms; + int32_t reverse; + int32_t keys_only; + int32_t include_tombstones; + int32_t has_start_bound; + int32_t start_inclusive; + int32_t strict_byte_budget; +}; + +// Requires a runtime-backed Tree; at most one configured result window is +// returned. A truncated result must be resumed using its last key. +ct_status ct_scan_v2(ct_tree *tree, const ct_scan_options_v2 *options, ct_scan_result **out); +// Copies all bounds before returning. has_start_bound distinguishes an empty +// key cursor from an unbounded scan. Free an abandoned future with ct_future_free. +ct_status ct_scan_async_v2(ct_tree *tree, const ct_scan_options_v2 *options, ct_future **out); +// On done=1, consumes the scan future even on error. Ownership transfers to +// out only on success. Pending futures remain owned by the caller. +ct_status ct_future_poll_scan_v2(ct_future *future, int32_t *done, ct_scan_result **out); +ct_status ct_scan_result_data(const ct_scan_result *result, const uint8_t **data, size_t *len, uint64_t *count, + int32_t *truncated); +void ct_scan_result_free(ct_scan_result *result); + // Result of a cadence-driven ct_compact_sparse_blocks pass (R129); mirrors // crowdb::tree::MergeGcStats. using ct_merge_gc_stats = struct @@ -54,7 +97,7 @@ using ct_merge_gc_stats = struct uint64_t blocks_deleted; }; -using ct_range_rebuild_stats = struct +using ct_range_rebuild_stats = struct ct_range_rebuild_stats { uint64_t entries_examined; uint64_t entries_emitted; @@ -64,10 +107,9 @@ using ct_range_rebuild_stats = struct uint64_t subtrees_skipped; }; -// Batched diagnostics snapshot; mirrors crowdb::tree::EngineStats. Every field -// is O(1) (an already-tracked atomic counter or BufferPool::stats()), so -// ct_get_stats is safe to poll periodically (metrics scrape / console -// panel refresh) without touching the tree itself. +// Legacy diagnostics snapshot; mirrors crowdb::tree::EngineStats. Legacy +// buffer-pool sampling scans frame metadata. Runtime publication snapshots +// use ct_get_tree_state_stats from tree_stats_c_api.h. using ct_stats = struct { uint64_t last_applied_slot; @@ -126,33 +168,34 @@ enum ct_sync_mode : uint8_t { // Backend / engine configuration. `path` null or empty selects an in-memory // store. Zero numeric fields take engine defaults. -using ct_options = struct +struct ct_options { // Optional injected backend. The tree retains shared ownership; callers // may release their handle after ct_open returns. When non-null, path and // backend are ignored. - ct_page_store *page_store; - uint8_t range_bounded; // 0 = unbounded; 1 = use endpoints below - const uint8_t *range_start; // null = minimum-unbounded; non-null empty is an empty key - size_t range_start_len; - const uint8_t *range_end; // null = maximum-unbounded; exclusive when present - size_t range_end_len; - const char *path; // durable file path; null/empty => in-memory - uint32_t iu_size; // 0 => default (1 for mem, 4096 for file) - uint32_t frame_bytes; // 0 => default - uint64_t buffer_pool_bytes; // 0 => default - uint8_t compression; // 0 = none, 1 = lz4 - uint64_t max_inline_value; // 0 => default (frame_bytes/4) - enum ct_backend backend; // default CT_BACKEND_BLOCK; ignored for in-memory - uint64_t block_size; // 0 => default 64 MiB; ignored for file/mem-block - uint32_t store_id; // default 0; block file naming - uint32_t group_id; // default 0; maps to PxGroupId in CrowdbKV - enum ct_sync_mode sync_mode; // default CT_SYNC_FULL - const char *log_dir; // null/empty => no C++ file logging - const char *log_level; // null/empty => "info" - const char *log_file_prefix; // null/empty => "crowdb-tree"; filename prefix - size_t log_max_file_mb; // 0 => default 30 - size_t log_max_files; // 0 => default 5 + ct_page_store *page_store; + uint8_t range_bounded; // 0 = unbounded; 1 = use endpoints below + const uint8_t *range_start; // null = minimum-unbounded; non-null empty is an empty key + size_t range_start_len; + const uint8_t *range_end; // null = maximum-unbounded; exclusive when present + size_t range_end_len; + const char *path; // durable file path; null/empty => in-memory + uint32_t iu_size; // 0 => default (1 for mem, 4096 for file) + uint32_t frame_bytes; // 0 => default + uint64_t buffer_pool_bytes; // 0 => default + uint8_t compression; // 0 = none, 1 = lz4 + uint64_t max_inline_value; // 0 => default (frame_bytes/4) + enum ct_backend backend; // default CT_BACKEND_BLOCK; ignored for in-memory + uint64_t block_size; // 0 => default 64 MiB; ignored for file/mem-block + uint32_t store_id; // default 0; block file naming + uint32_t group_id; // default 0; maps to PxGroupId in CrowdbKV + enum ct_sync_mode sync_mode; // default CT_SYNC_FULL + const char *log_dir; // null/empty => no C++ file logging + const char *log_level; // null/empty => "info" + const char *log_file_prefix; // null/empty => "crowdb-tree"; filename prefix + size_t log_max_file_mb; // 0 => default 30 + size_t log_max_files; // 0 => default 5 + const ct_tree_tuning *tuning; // null => native tuning defaults }; // ── Lifecycle + durability ──────────────────────────────────────── @@ -254,6 +297,7 @@ enum ct_root_catalog_object_kind { CT_ROOT_CATALOG_CURRENT_MANIFEST = 1, CT_ROOT_CATALOG_MANIFEST = 2, CT_ROOT_CATALOG_REFERENCE_SEGMENT = 3, + CT_ROOT_CATALOG_LEGACY_MIGRATION = 4, }; struct ct_root_catalog_callbacks @@ -297,6 +341,16 @@ ct_status ct_root_catalog_pin_generation(ct_root_catalog *catalog, uint64_t tree ct_status ct_root_catalog_unpin_generation(ct_root_catalog *catalog, uint64_t tree_id, uint64_t transition_high, uint64_t transition_low); ct_status ct_open(const ct_options *opt, ct_tree **out); +// Uses the budgeted publication core and validates binding for either host. +// A null runtime selects the process-shared host and ct_open backend defaults. +// With an explicit runtime, an empty path without an injected store creates a +// volatile Tree; checkpoint requires a reloadable backend pair. +ct_status ct_open_with_runtime(const ct_options *opt, ct_runtime *runtime, const ct_tree_runtime_options *binding, + ct_tree **out); +// A null runtime selects the process-shared host with the supplied binding. +ct_status ct_rebuild_range_with_runtime(ct_tree *source, const ct_options *opt, ct_runtime *runtime, + const ct_tree_runtime_options *binding, ct_tree **out, + ct_range_rebuild_stats *stats); ct_status ct_rebuild_range(ct_tree *source, const ct_options *destination_options, ct_tree **out, ct_range_rebuild_stats *stats); void ct_close(ct_tree *t); @@ -454,13 +508,11 @@ using ct_kv_ref = struct // instead of a packed buffer — eliminates the Rust-side packing copy. ct_status ct_apply_batch_slices(ct_tree *t, uint64_t slot, const ct_kv_ref *ops, uint64_t count); -// One zero-copy key/value op for ct_apply_batch_external (R30). `value` is a -// non-owning pointer into Rust-owned memory (a `bytes::Bytes` slice); crowdb-tree -// borrows it via a kExternal buffer and calls `drop_fn(bytes_ref)` when the -// buffer is freed (at MemTable drain/overwrite). `bytes_ref` is an opaque -// Rust handle (e.g. a boxed `Arc<Bytes>`); `drop_fn` decrements the Rust -// refcount. kind: 0 = put, 1 = delete (value/value_len/bytes_ref/drop_fn are -// unused for delete). +// Legacy external value input. Its backing allocation identity/capacity is +// unknown: `value_len` describes only the slice. Values are copied into owned +// cells during the call; `drop_fn(bytes_ref)` releases the original owner +// exactly once, including failures. kind: 0 = put, 1 = delete; owner fields +// are unused for delete. Drop callbacks must not throw. using ct_ext_op = struct { const uint8_t *key; @@ -472,12 +524,9 @@ using ct_ext_op = struct void (*drop_fn)(void *); // Rust drop callback (put only); NULL for delete }; -// Zero-copy apply (R30): same semantics as ct_apply_batch_slices (atomic -// multi-key apply, highest-slot-wins, oversized-key rejection) but the value -// bytes are borrowed from Rust-owned memory instead of copied at the FFI -// boundary. The value memcpy is deferred to MemTable drain (flush, off the -// apply critical path). Ownership of every `bytes_ref` transfers to crowdb-tree; -// `drop_fn` is called exactly once per op when the borrowed buffer is freed. +// Same semantics as ct_apply_batch_slices, consuming every put's owner. +// Unknown backing capacity requires a controlled copy; a short slice must +// never keep an unaccounted large allocation alive inside the tree. ct_status ct_apply_batch_external(ct_tree *t, uint64_t slot, const ct_ext_op *ops, uint64_t count); // Convenience: auto-assign the next slot and apply (single-writer only). @@ -503,41 +552,30 @@ ct_status ct_get(ct_tree *t, const uint8_t *key, size_t klen, int32_t *found, ui // with ct_future_poll. For ct_flush_async/ct_snapshot_async, a future that // ct_future_poll reports done (*done=1) is freed by that same call -- do not // also call ct_future_free on it. For ct_get_async specifically, done=1 does -// *not* free it (zero-copy fast path: -// *out_value may borrow bytes from a resident frame, kept alive by an epoch -// guard this ct_future owns) -- the caller must always follow up with +// *not* free it: *out_value borrows bytes from an independent payload owner +// retained by the future; no epoch guard spans the callback. Follow up with // ct_future_free once done reading *out_value, for both a found and a // not-found/errored result. ct_future_free is otherwise only for abandoning // a still-pending future early (e.g. the Rust Future was dropped/cancelled // before completion); calling it twice, or calling it after ct_future_poll // already freed a flush/snapshot future, is undefined behavior. -using ct_future = struct ct_future; - -// Fast path (no demand-load needed): the returned future already reports -// done=1 on the very first ct_future_poll call. Slow path (page must be -// loaded): the future stays pending until the tree's Reactor completes the -// I/O (or, with no Reactor wired -- e.g. an in-memory tree, or a build -// without liburing -- until it falls back to a synchronous load; still -// correct, just not genuinely async). Returns null only if -// `t` is itself null. + +// Reads run through the shared CPU/backend runtime. The first poll may be +// ready or pending for either a resident or a cold value. Completion retains +// the independent payload owner until ct_future_free. Returns null if no +// completion handle can be created, including when t is null. ct_future *ct_get_async(ct_tree *t, const uint8_t *key, size_t klen); -// flush()/snapshot() twins. flush_async's future never has genuine I/O to -// wait on (flush only touches the in-memory L1, never page_store) so it -// always completes synchronously, same as the fast path above; it exists -// so Rust has one uniform ct_future-based shape for all three ops. -// snapshot_async's future, on completion, carries the durable -// last_applied_slot in ct_future_poll's *out_slot (mirrors ct_snapshot's -// own out param); *out_found and *out_value are unused for both. +// flush()/snapshot() twins use the shared runtime and can be pending on the +// first poll. snapshot_async completion carries the durable last_applied_slot +// in ct_future_poll's *out_slot (mirrors ct_snapshot); *out_found and *out_value +// are unused for both operations. ct_future *ct_flush_async(ct_tree *t); ct_future *ct_snapshot_async(ct_tree *t); -// scan() twin. Unlike ct_get_async, a single -// pending scan may need more than one page load (one per cold leaf, plus -// possibly the initial root->leaf descent) -- each is resolved by retrying -// the whole scan from scratch (still correct: every retry either resolves -// or permanently loads one more page, so it always terminates; see -// Crowdbtree::scan_async's doc comment). On completion (ct_future_poll), +// scan() twin. One captured view and cursor continue across bounded work +// windows and cold-page loads; completion does not restart the whole scan. +// The first poll may be ready or pending. On completion (ct_future_poll), // *out_value carries the same packed record format as ct_scan // (`[u32 klen][key][u64 slot][u8 tombstone][u32 vlen][val]*`, always a *malloc'd*, owned // buffer -- pass it to ct_free_buf, no zero-copy borrow attempted here, @@ -657,6 +695,13 @@ uint64_t ct_snapshot_export_offset(const ct_export *e); ct_status ct_snapshot_export_next(ct_export *e, uint64_t offset, ct_buf *chunk, int32_t *done); void ct_snapshot_export_end(ct_export *e); +// Owned chunks retain their engine allocation charge across clones/slices in +// foreign callers. data is borrowed from the handle; never ct_free_buf it. +struct ct_snapshot_chunk; +ct_status ct_snapshot_export_next_v2(ct_export *e, uint64_t offset, struct ct_snapshot_chunk **out, int32_t *done); +ct_status ct_snapshot_chunk_data(const struct ct_snapshot_chunk *chunk, const uint8_t **data, size_t *len); +void ct_snapshot_chunk_free(struct ct_snapshot_chunk *chunk); + ct_status ct_snapshot_import_begin(ct_tree *t, ct_import **out); ct_status ct_snapshot_import_feed(ct_import *im, const uint8_t *chunk, size_t len); ct_status ct_snapshot_import_finish(ct_import *im, uint64_t *out_at_slot); diff --git a/lib/crowdb-tree/include/crowdb-tree/chunk_migration_c_api.h b/lib/crowdb-tree/include/crowdb-tree/chunk_migration_c_api.h new file mode 100644 index 000000000..a96ef3d5c --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/chunk_migration_c_api.h @@ -0,0 +1,18 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/c_api.h" +#include "crowdb-tree/runtime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif +// Blocking host maintenance entry. Stop and drain all old-format writers in +// this catalog namespace first. The supplied backend must use the runtime +// Chunk factory. Unknown commits retain the original root pin and are resolved +// by retrying the same expected_generation with a fresh catalog writer owner. +ct_status ct_migrate_legacy_chunk_with_runtime(const ct_options *, ct_runtime *, const ct_tree_runtime_options *, + uint64_t expected_generation, ct_tree **); +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/chunk_pin_service_c_api.h b/lib/crowdb-tree/include/crowdb-tree/chunk_pin_service_c_api.h new file mode 100644 index 000000000..e5e6b311d --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/chunk_pin_service_c_api.h @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/chunk_protection_c_api.h" +#include "crowdb-tree/runtime_c_api.h" +#ifdef __cplusplus +extern "C" { +#endif +typedef struct ct_chunk_pin_service ct_chunk_pin_service; + +struct ct_chunk_pin_service_step +{ + size_t completed; + size_t failed; + uint8_t more; + uint8_t drained; +}; + +// Context ownership transfers on success. Wake must not block or allocate. +ct_status ct_chunk_pin_service_create(ct_runtime *, size_t capacity, void (*wake)(void *), void (*drop_context)(void *), + void *, ct_chunk_pin_service **); +ct_status ct_root_catalog_bind_pin_service(ct_root_catalog *, ct_chunk_pin_service *); +// Call pump on the host blocking-backend executor, never a Tree CPU worker. +ct_chunk_pin_service_step ct_chunk_pin_service_pump(ct_chunk_pin_service *, size_t limit); +// Non-draining host queue counters: used/allocated count all registered pins, +// waiters count pending final releases, and denied_host counts capacity denials. +ct_status ct_chunk_pin_service_resource_usage(const ct_chunk_pin_service *, ct_resource_usage_v1 *out); +void ct_chunk_pin_service_stop(ct_chunk_pin_service *); +void ct_chunk_pin_service_free(ct_chunk_pin_service *); +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/chunk_protection_c_api.h b/lib/crowdb-tree/include/crowdb-tree/chunk_protection_c_api.h new file mode 100644 index 000000000..2a5824323 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/chunk_protection_c_api.h @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/chunk_stage_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +enum { CT_CHUNK_PROTECTION_SCHEMA_V1 = 1 }; + +enum ct_chunk_candidate_action { + CT_CHUNK_CANDIDATE_COMMIT = 1, + CT_CHUNK_CANDIDATE_ABORT = 2, + CT_CHUNK_CANDIDATE_RECOVER = 3 +}; + +struct ct_chunk_candidate_status +{ + ct_chunk_stage_id id; + uint64_t generation; + uint64_t reference_count; + uint64_t prepared_references; + uint64_t prepared_windows; + uint64_t released_windows; + uint32_t state; +}; + +struct ct_chunk_pin_progress +{ + ct_chunk_stage_id after; + uint32_t examined; + uint32_t released; + uint8_t has_after; + uint8_t more; +}; + +// Calls may block on metadata and run only on the host backend lane. The +// original catalog owns context/free_blob; this table never owns it twice. +struct ct_root_catalog_protection_callbacks +{ + uint32_t version; + uint32_t struct_bytes; + uint64_t capabilities; + ct_status (*check_reader)(void *); + ct_status (*writer_quiesced)(void *, uint64_t, uint64_t); + ct_status (*begin)(void *, uint64_t, ct_chunk_stage_id, uint64_t, uint64_t, uint32_t, const uint8_t *, size_t); + ct_status (*load)(void *, uint64_t, ct_chunk_stage_id, ct_chunk_candidate_status *); + ct_status (*list)(void *, uint64_t, const ct_chunk_stage_id *, uint32_t, ct_chunk_candidate_status *, size_t *); + ct_status (*prepare)(void *, uint64_t, ct_chunk_stage_id, uint64_t, const ct_chunk_stage_id *, size_t); + ct_status (*finish)(void *, uint64_t, ct_chunk_stage_id, uint32_t); + ct_status (*release)(void *, uint64_t, ct_chunk_stage_id, uint64_t); + ct_status (*pin)(void *, uint64_t, ct_chunk_stage_id, ct_chunk_stage_id, uint8_t); + ct_status (*prune)(void *, uint64_t, ct_chunk_stage_id, const ct_chunk_stage_id *, uint32_t, + ct_chunk_pin_progress *); +}; + +ct_status ct_callback_root_catalog_open_v3(const ct_root_catalog_callbacks *, const ct_root_catalog_staging_callbacks *, + const ct_root_catalog_protection_callbacks *, void *, ct_root_catalog **); +uint64_t ct_root_catalog_protection_capabilities(const ct_root_catalog *); +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/chunk_runtime_c_api.h b/lib/crowdb-tree/include/crowdb-tree/chunk_runtime_c_api.h new file mode 100644 index 000000000..354beca6e --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/chunk_runtime_c_api.h @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/c_api.h" +#include "crowdb-tree/runtime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct ct_chunk_runtime_options +{ + uint32_t version; + uint32_t struct_bytes; + uint64_t tree_id; + uint64_t owner_epoch; + uint64_t open_generation; + uint32_t iu_size; + uint32_t mirror_copies; + uint64_t max_chunk_bytes; +}; + +// No storage operation runs here. The paid handle is injected into a Tree +// using the same host runtime. Catalog and transport require staged protection, +// exact fixed-pack I/O and a bound host pin-release service. One catalog owner +// represents one writer incarnation; reopening requires a fresh catalog claim. +ct_status ct_chunk_runtime_page_store_open(ct_runtime *, const ct_chunk_runtime_options *, ct_root_catalog *, + ct_chunk_transport *, ct_page_store **); + +struct ct_chunk_collection_result_v1 +{ + uint32_t struct_size; + uint32_t version; + int32_t terminal_status; + uint32_t more; + uint64_t candidates_examined; + uint64_t reference_windows; + uint64_t stages_examined; + uint64_t pins_examined; + uint64_t pins_released; + uint64_t stages_removed; + uint64_t logical_capacity_bytes; +}; + +// A finite pass does not advance root retention or WAL durability. A failed +// pass retains partial progress and the exact operation identities for retry. +ct_status ct_chunk_collect_async(ct_tree *, ct_future **out); +ct_status ct_future_poll_chunk_collection(ct_future *, int32_t *done, ct_chunk_collection_result_v1 *out); + +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/chunk_stage_c_api.h b/lib/crowdb-tree/include/crowdb-tree/chunk_stage_c_api.h new file mode 100644 index 000000000..34d99aade --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/chunk_stage_c_api.h @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct ct_chunk_stage_id +{ + uint64_t owner_epoch; + uint64_t operation_high; + uint64_t operation_low; +}; + +// Independent staging is enabled only when catalog, transport, and reclamation +// all support it. This bit advertises the persistent record schema alone. +enum { CT_CHUNK_STAGE_SCHEMA_V1 = 1 }; + +struct ct_root_catalog_staging_callbacks +{ + uint32_t version; + uint32_t struct_bytes; + uint64_t capabilities; + ct_status (*load)(void *, uint64_t, ct_chunk_stage_id, const uint8_t **, size_t *); + ct_status (*compare)(void *, uint64_t, uint64_t, const uint8_t *, size_t, const uint8_t *, size_t); + ct_status (*list)(void *, uint64_t, const ct_chunk_stage_id *, uint32_t, const uint8_t **, size_t *); +}; + +ct_status ct_callback_root_catalog_open_v2(const ct_root_catalog_callbacks *, const ct_root_catalog_staging_callbacks *, + void *context, ct_root_catalog **out); +uint64_t ct_root_catalog_stage_capabilities(const ct_root_catalog *); +ct_status ct_root_catalog_stage_load(ct_root_catalog *, uint64_t, ct_chunk_stage_id, ct_buf *); +ct_status ct_root_catalog_stage_compare(ct_root_catalog *, uint64_t, uint64_t, const uint8_t *, size_t, const uint8_t *, + size_t); +ct_status ct_root_catalog_stage_list(ct_root_catalog *, uint64_t, const ct_chunk_stage_id *, uint32_t, ct_buf *); + +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/config.h b/lib/crowdb-tree/include/crowdb-tree/config.h index 2b1892ef5..b603b8efd 100644 --- a/lib/crowdb-tree/include/crowdb-tree/config.h +++ b/lib/crowdb-tree/include/crowdb-tree/config.h @@ -7,6 +7,8 @@ #include "crowdb-tree/btree/key_range.h" #include "crowdb-tree/maptable/compressor.h" +#include "crowdb-tree/tree_execution_options.h" +#include "crowdb-tree/tree_runtime.h" #ifdef CROWDB_HAVE_LIBURING # include "crowdb-common/diskio_uring.h" @@ -24,30 +26,34 @@ class AsyncPageStore; struct Config { - // ── Consolidation (core doc §7) ── - // Fold a leaf's delta chain into a fresh base when either threshold trips. - // O3: max_delta_len raised from 16 to 64 — with flush_notify draining - // small batches (1-2 frozen tables per flush), the chain grows slowly. - // 64 gives headroom for sustained write bursts before consolidation - // fires. max_delta_bytes stays at 512 KiB. The snapshot's - // prepare_snapshot_locked folds chains, so deferring the fold is safe. + // The host owns one shared runtime; new Tree core operations retain their + // binding through all CPU and backend callbacks. Null selects the shared + // process standalone runtime; it does not create a pool per Tree. + std::shared_ptr<TreeRuntime> runtime; + TreeBindingOptions runtime_binding{}; + TreeExecutionOptions execution{}; + + // Deprecated compatibility fields. Immutable publication uses bounded + // overlay runs and maintenance; it does not create pointer delta chains. uint32_t max_delta_len = 64; size_t max_delta_bytes = 512ULL * 1024; // 512 KiB // ── Leaf split / merge (core doc §8) ── - // Split when a consolidated leaf exceeds leaf_split_bytes; merge when it - // drops below leaf_merge_bytes. Hysteresis: merge threshold is well below - // split to avoid oscillation. - // One default durable page fits in a public chunk frame after its 14-byte - // header and 20-byte footer. + // Leaf thresholds count the record region, excluding frame headers/trailer. + // Runtime planning adds that overhead and caps the split target to the + // configured frame and current public frame payload capacity. A default + // split target derives the usable record capacity for smaller frames. + // Merge triggers below leaf_merge_bytes; the merged payload must also fit + // within three quarters of the split target to preserve hysteresis. Zero + // disables merging. Values that overflow frame arithmetic are rejected. size_t leaf_split_bytes = 65'502; - size_t leaf_merge_bytes = 16ULL * 1024; // split/4 + size_t leaf_merge_bytes = 16ULL * 1024; // Inner-page fanout bound (separator count) before an inner split. uint32_t inner_max_keys = 256; // Inner-page underflow threshold: when a non-root inner page drops below this - // many separators (after a child merge), it is merged with its left sibling so - // delete-heavy workloads don't leave a sparse upper tree. 0 = derive + // many separators (after a child merge), it is merged with its left sibling + // so delete-heavy workloads don't leave a sparse upper tree. 0 = derive // max(1, inner_max_keys / 4) (well below the post-split size for hysteresis). uint32_t inner_merge_keys = 0; @@ -66,52 +72,31 @@ struct Config // key comfortably within a single leaf/inner frame alongside its cell. size_t max_key_size = 0; - // ── In-frame delta region (PT12) ── - // When enabled, a small flush appends its mutations as in-frame deltas via a - // cheap frame COW (memcpy + append) instead of building a fresh sorted base or - // a heap delta node; the leaf folds to a fresh base once delta_count reaches - // max_inframe_delta or the deltas no longer fit. default_env off (the heap delta - // overlay already works; this is an optimization measured against COW-rebuild). + // Deprecated compatibility fields. New publication does not append an + // in-frame delta region; historical delta frames remain readable. bool inframe_delta = false; uint32_t max_inframe_delta = 8; - // ── MemTable flush triggers ── - // 16 MiB: large enough that a single flush captures a meaningful batch - // (reducing flush frequency), but small enough to bound active_ memory - // under sustained write load. With max_memtable_count=10, worst-case - // memory is ~160 MiB of un-drained memtables before backpressure kicks in. - size_t memtable_flush_bytes = 16ULL * 1024 * 1024; // 16 MiB + // Soft per-source thresholds. A successful write crossing either bound + // publishes a fresh writable source while the old source remains readable. + // Values of zero request rotation after each nonempty write. Independent + // hard memory/version limits still apply to admission. + size_t memtable_flush_bytes = 16ULL * 1024 * 1024; uint32_t memtable_flush_entries = 100000; - // ── MemTable double buffering (plan-tree #3) ── - // Once active_ crosses memtable_flush_bytes/_entries, it is frozen (no - // longer accepts writes) and a fresh, empty MemTable takes over as - // active_; the frozen table is drained into L1 by flush() (explicit call - // from the upper-layer maintenance loop). This bounds how large any *one* - // buffer can grow and lets new writes proceed without contending with an - // in-progress drain (they land in a different MemTable object entirely). - // This is the count of buffers total: 1 active_ + up to - // (max_memtable_count - 1) queued frozen buffers. Must be >= 2 (2 = the - // common active_/flushing_ case); values are not currently validated by - // the engine. When the frozen queue is already at (max_memtable_count - - // 1), a further threshold-triggered freeze is skipped (active_ is allowed - // to keep growing past its threshold) rather than stalling the writer -- - // an explicit flush() is expected to catch up and free a slot. See - // Crowdbtree's active_/frozen_ member comment for the full design, - // including how non-contiguous (slot > the current contiguous frontier) - // leftovers are handled when a frozen buffer is drained. Default 10 - // (1 active + 9 frozen) gives the maintenance loop enough headroom to - // absorb writes during a multi-second flush/snapshot without active_ - // growing unbounded. With 16 MiB memtables, worst-case un-drained - // memory is ~160 MiB before backpressure kicks in. + // Total soft source count, including one writable source (minimum two). + // Runtime maintenance can trigger earlier to keep inherited tables, a + // capture successor and a child writer within fixed read/flush windows. + // At this limit another threshold crossing requests the existing finite + // handoff, so sustained writes make bounded progress without growing the + // source queue. A gap or unavailable whole-batch credit leaves accepted + // data readable; explicit maintenance and the accepted/gap apply channels + // retain their own progress admission. A failed soft maintenance attempt + // never changes the already committed write result. uint32_t max_memtable_count = 10; - // ── Mapping table redesign (plan-tree #14) ── - // Packed slot words per segment. - // Fixed for the lifetime of a tree. Not yet consumed by MappingTable -- - // it still hardcodes MappingTable::kSegmentSize (#14c follow-up: plumb - // this through as the runtime segment size) -- present now so - // callers/tests can start tuning it ahead of that wiring. + // Deprecated compatibility field. Immutable MappingView radix geometry + // and checkpoint metadata layout are fixed by their format. uint32_t mapping_segment_slots = 1024; // ── Persistence ── @@ -122,22 +107,21 @@ struct Config // ── Async I/O ── // Non-owning; the caller owns and outlives the Crowdbtree. When null, - // async entry points complete synchronous storage work in the caller's - // stack frame. The store must address the same durable image as + // storage work runs on the host runtime's bounded backend executor. + // The store must address the same durable image as // `page_store`. This contract is independent of io_uring so RPC-backed // stores work on builds without liburing. AsyncPageStore *async_page_store = nullptr; + // Optional shared owner of both raw backend fields and their reactor. + // Detached runtime snapshots keep this owner after the Tree closes. + std::shared_ptr<const void> backend_owner; #ifdef CROWDB_HAVE_LIBURING // Optional local reactor used only by BlockAsyncPageStore. ::crowdb::common::DiskIOUring *async_uring = nullptr; #endif - // ── Buffer pool ── - // The arena that holds base-page frames is a flat array of equal-size frames. - // `frame_bytes` is that fixed size; a base page larger than a frame (rare; - // overflow pages are PT11) or built when the pool is full falls back to a - // heap buffer, so correctness is independent of these knobs. `frame_bytes` - // should be >= leaf_split_bytes so normal leaves are pool-resident. + // Fixed output frame geometry and the independent decoded-page cache + // capacity. Overflow values use bounded chains of fixed-size frames. uint32_t frame_bytes = 64ULL * 1024; // fixed arena frame size size_t buffer_pool_bytes = 64ULL * 1024 * 1024; // arena capacity (frames * frame_bytes) @@ -146,9 +130,9 @@ struct Config // ([algo][raw_len][stored_len][crc][stored]); the buffer pool always holds // uncompressed frames. The algo is recorded per page, so a page is decoded // correctly regardless of the option in force when it is read back (mixed - // pages across incremental snapshots decode fine). default_env kNone keeps the - // stored bytes equal to the raw frame; kLz4 compresses when it shrinks the - // page (falling back to stored-raw when LZ4 is unavailable or unhelpful). + // pages across incremental snapshots decode fine). default_env kNone keeps + // the stored bytes equal to the raw frame; kLz4 compresses when it shrinks + // the page (falling back to stored-raw when LZ4 is unavailable or unhelpful). compress_algo compression = compress_algo::kNone; // ── Engine identity (logging context) ── diff --git a/lib/crowdb-tree/include/crowdb-tree/epoch.h b/lib/crowdb-tree/include/crowdb-tree/epoch.h index 5ac5a8140..8d74ab3d1 100644 --- a/lib/crowdb-tree/include/crowdb-tree/epoch.h +++ b/lib/crowdb-tree/include/crowdb-tree/epoch.h @@ -24,9 +24,12 @@ // cannot reference (the mapping slot was already swapped). #pragma once +#include "crowdb-tree/memory_budget.h" #include "crowdb-tree/memtable/allocation.h" +#include <array> #include <atomic> +#include <bit> #include <cstdint> #include <functional> #include <memory> @@ -46,9 +49,52 @@ class EpochManager // Cache-line padded so readers on different threads don't false-share. struct alignas(64) Participant { - std::atomic<uint64_t> local_epoch{0}; // 0 = inactive (not inside a guard) - Participant *next{nullptr}; // intrusive list; published w/ release - uint32_t nest{0}; // reentrancy depth (owner thread only) + std::atomic<uint64_t> local_epoch{0}; // 0 = inactive (not inside a guard) + Participant *next{nullptr}; // intrusive list; published w/ release + uint32_t nest{0}; // reentrancy depth (owner thread only) + uint64_t thread_id{0}; // immutable after publication + std::shared_ptr<AllocationBudget> budget; + }; + + // The closed flag and head share one CAS word: a registration that read + // the old head cannot append after an immutable source seals its list. + class ParticipantList + { + public: + [[nodiscard]] Participant *load(std::memory_order order) const noexcept + { + return pointer(head_.load(order)); + } + + Participant *exchange(Participant *value, std::memory_order order = std::memory_order_seq_cst) noexcept + { + return pointer(head_.exchange(reinterpret_cast<uintptr_t>(value), order)); + } + + [[nodiscard]] bool sealed() const noexcept + { + return (head_.load(std::memory_order_acquire) & 1) != 0; + } + + Participant *seal() noexcept + { + return pointer(head_.fetch_or(1, std::memory_order_acq_rel)); + } + + bool push(Participant *participant, uintptr_t head) noexcept; + + [[nodiscard]] uintptr_t registration_head() const noexcept + { + return head_.load(std::memory_order_acquire); + } + + private: + static Participant *pointer(uintptr_t value) noexcept + { + return std::bit_cast<Participant *>(value & ~uintptr_t{1}); + } + + std::atomic<uintptr_t> head_{0}; }; public: @@ -107,14 +153,32 @@ class EpochManager std::shared_ptr<Lifetime> lifetime_; }; - EpochManager(); + explicit EpochManager(std::shared_ptr<AllocationBudget> budget = {}, bool enabled = true); ~EpochManager(); + [[nodiscard]] bool source_sealed() const noexcept; +#ifdef CROWDB_TREE_TEST_UTIL + void set_before_participant_append_for_tests(void (*hook)(void *), void *context) noexcept + { + append_hook_ = hook; + append_context_ = context; + } +#endif + EpochManager(const EpochManager &) = delete; EpochManager &operator=(const EpochManager &) = delete; // Open a reader guard at the current epoch (lock-free). - Guard enter(); + // A new thread's participant uses the current operation's credit when + // supplied. Existing participants require no new allocation after seal. + Guard enter(std::shared_ptr<AllocationBudget> budget = {}); + + [[nodiscard]] static size_t participant_allocation_bytes() noexcept + { + return sizeof(Participant); + } + + [[nodiscard]] static size_t infrastructure_allocation_bound() noexcept; // Defer deletion of `ptr` until no guard opened at-or-before now remains. void retire(void *ptr, Deleter deleter); @@ -139,7 +203,13 @@ class EpochManager private: friend class Guard; - Participant *participant_for_this_thread(); + friend class MemTable; + void seal_source() noexcept; + friend class BlockDirectory; + // Exclusive backend teardown: no remaining guards or accepted operations. + bool retire_backend_step() noexcept; + Participant *participant_for_this_thread(const std::shared_ptr<AllocationBudget> &budget); + static void destroy_participant(Participant *participant) noexcept; size_t reclaim_locked(); // caller holds reclaim_mu_ uint64_t min_active_epoch(); // oldest epoch any reader might still see size_t reclaim_deferred_locked(); @@ -151,19 +221,19 @@ class EpochManager Deleter deleter; }; - std::shared_ptr<AllocationCounter> memtable_allocation_ = std::make_shared<AllocationCounter>(0); + std::shared_ptr<AllocationBudget> budget_; + std::shared_ptr<AllocationCounter> memtable_allocation_; std::shared_ptr<Lifetime> lifetime_; const uint64_t id_; // stable key for per-thread cache std::atomic<uint64_t> global_epoch_{1}; - std::atomic<Participant *> participants_{nullptr}; // lock-free push list head - - // Recursive: a retired object's deleter can legitimately trigger another - // retire() on this *same* EpochManager before this call's reclaim_locked() - // returns (e.g. retire_orphaned_page's deferred mapping_.clear() can, as - // its very last live slot empties, recycle the owning MappingSegment via - // recycle_segment_if_empty() -> epoch.retire_object(seg) -- crowdb-tree.cpp/ - // mapping_table.cpp share this one EpochManager). A plain mutex would - // deadlock on that same-thread re-entry. + ParticipantList participants_; +#ifdef CROWDB_TREE_TEST_UTIL + void (*append_hook_)(void *) = nullptr; + void *append_context_ = nullptr; +#endif + + // A retired object's deleter can retire another object on this same manager. + // Reclamation permits that same-thread re-entry. std::recursive_mutex reclaim_mu_; // guards retired_ (writer side) std::vector<Retired> retired_; std::atomic<Deferred *> deferred_{nullptr}; diff --git a/lib/crowdb-tree/include/crowdb-tree/external_allocation.h b/lib/crowdb-tree/include/crowdb-tree/external_allocation.h new file mode 100644 index 000000000..b4ed08303 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/external_allocation.h @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" + +#include <cstdint> +#include <stdexcept> + +namespace crowdb::tree +{ +// Construct once at the allocation boundary, before making slices. Ownership +// of the complete allocation, including spare capacity, is transferred here. +// Every retained slice shares this identity and therefore one payload charge. +class ExternalAllocation +{ + public: + using Owner = std::unique_ptr<void, void (*)(void *)>; + + ExternalAllocation(const uint8_t *data, size_t capacity, Owner owner, std::shared_ptr<AllocationBudget> budget) + : budget_(std::move(budget)), + credit_(budget_, capacity, AllocationKind::kPayload), + owner_(std::move(owner)), + data_(data), + capacity_(capacity) + { + if ((capacity_ != 0 && data_ == nullptr) || !owner_ || owner_.get_deleter() == nullptr) { + throw std::invalid_argument("external allocation requires a complete allocation owner"); + } + } + + [[nodiscard]] static std::shared_ptr<const ExternalAllocation> + create(const uint8_t *data, size_t capacity, Owner owner, const std::shared_ptr<AllocationBudget> &budget) + { + return std::allocate_shared<ExternalAllocation>(MetadataAllocator<ExternalAllocation>(budget), data, capacity, + std::move(owner), budget); + } + + [[nodiscard]] const uint8_t *data() const noexcept + { + return data_; + } + + [[nodiscard]] size_t capacity() const noexcept + { + return capacity_; + } + + [[nodiscard]] const std::shared_ptr<AllocationBudget> &budget() const noexcept + { + return budget_; + } + + [[nodiscard]] bool charged_to(const std::shared_ptr<AllocationBudget> &budget) const noexcept + { + return budget_ && budget && budget_->domain() == budget->domain(); + } + + private: + std::shared_ptr<AllocationBudget> budget_; + AllocationLease credit_; + + Owner owner_; + const uint8_t *data_; + size_t capacity_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/immutable_snapshot_c_api.h b/lib/crowdb-tree/include/crowdb-tree/immutable_snapshot_c_api.h new file mode 100644 index 000000000..1acc8d885 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/immutable_snapshot_c_api.h @@ -0,0 +1,26 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/c_api.h" + +extern "C" { +struct ct_immutable_snapshot; + +// Host-thread operations. Capture shares one immutable L1 mapping and exact +// backing owners; it neither opens a writer nor materializes all entries. +// expected_sequence=0 permits an in-memory fixed coverage without a checkpoint. +ct_status ct_immutable_snapshot_capture(ct_tree *tree, uint64_t expected_sequence, uint64_t expected_coverage, + ct_immutable_snapshot **out); +void ct_immutable_snapshot_free(ct_immutable_snapshot *snapshot); + +// Synchronous reads return existing owned result handles. The get future is +// already terminal and follows ct_future_poll/free's kGet ownership contract. +ct_status ct_immutable_snapshot_get(ct_immutable_snapshot *snapshot, const uint8_t *key, size_t size, ct_future **out); +ct_status ct_immutable_snapshot_scan(ct_immutable_snapshot *snapshot, const ct_scan_options_v2 *options, + ct_scan_result **out); + +// Rebuilds the destination's configured range from this fixed view and writes +// its own checkpoint. Destination is a separately opened runtime Tree/account. +ct_status ct_immutable_snapshot_rebuild(ct_immutable_snapshot *snapshot, ct_tree *destination, + ct_range_rebuild_stats *stats); +} diff --git a/lib/crowdb-tree/include/crowdb-tree/maintenance_c_api.h b/lib/crowdb-tree/include/crowdb-tree/maintenance_c_api.h new file mode 100644 index 000000000..3175c6cda --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/maintenance_c_api.h @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <cstdint> + +extern "C" { +struct ct_tree; +struct ct_future; + +struct ct_flush_result_v1 +{ + uint32_t struct_size; + uint32_t version; + int32_t terminal_status; + int32_t handoff_status; + int32_t maintenance_status; + uint32_t phase; + uint64_t batch_id; + uint64_t floor; + uint64_t frontier; + uint64_t published_coverage; + uint64_t deferred_pages; + int32_t handoff_completed; + int32_t handoff_advanced; +}; + +struct ct_checkpoint_result_v1 +{ + uint32_t struct_size; + uint32_t version; + int32_t terminal_status; + int32_t durable_committed; + int32_t writeback_completed; + uint64_t durable_coverage; + uint64_t snapshot_sequence; + uint64_t checkpoint_token; + uint64_t captured_coverage; + uint64_t primary_frames; + uint64_t written_frames; + uint64_t reused_frames; + uint64_t rebased_pages; + uint64_t rebase_deferred; +}; + +struct ct_compaction_options_v1 +{ + uint32_t struct_size; + uint32_t version; + double free_threshold; + uint64_t max_relocation_bytes; +}; + +struct ct_compaction_result_v1 +{ + uint32_t struct_size; + uint32_t version; + int32_t terminal_status; + uint64_t blocks_selected; + uint64_t pages_relocated; + uint64_t bytes_relocated; + uint64_t blocks_deleted; +}; + +// Runtime-only bounded pass. Null options select threshold 0.70 and one +// eligible block; explicit max_relocation_bytes=0 disables relocation. +int32_t ct_compact_sparse_blocks_async(ct_tree *tree, const ct_compaction_options_v1 *options, ct_future **out); +int32_t ct_future_poll_compaction(ct_future *future, int32_t *done, ct_compaction_result_v1 *out); + +// Exact checkpoint frontier; publication/flush alone never advances it. +// Returns zero for a null handle or a Tree with no durable checkpoint. +uint64_t ct_durable_coverage(ct_tree *tree); + +// Runtime Trees only. A successful start returns an owned future, including +// immediate operation failures. Admission failures return no future. +int32_t ct_flush_report_async(ct_tree *tree, ct_future **out); +int32_t ct_writeback_async(ct_tree *tree, ct_future **out); +int32_t ct_checkpoint_report_async(ct_tree *tree, ct_future **out); + +// Initialize struct_size=sizeof(*out), version=1. A poll error leaves the +// future and output unchanged. done=0 retains the future. done=1 copies the +// report and consumes the future, even when report.terminal_status is an error. +// Stage errors never discard an accepted operation's progress report. +// ct_future_free may abandon a pending report; accepted callbacks retain all +// native operation state until terminal. Legacy ct_future_poll rejects it. +int32_t ct_future_poll_flush_report(ct_future *future, int32_t *done, ct_flush_result_v1 *out); +int32_t ct_future_poll_checkpoint_report(ct_future *future, int32_t *done, ct_checkpoint_result_v1 *out); +} diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/compressor.h b/lib/crowdb-tree/include/crowdb-tree/maptable/compressor.h index 0ef504557..4460c64cc 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/compressor.h +++ b/lib/crowdb-tree/include/crowdb-tree/maptable/compressor.h @@ -20,6 +20,7 @@ #include "crowdb-tree/status.h" #include <cstdint> +#include <span> #include <vector> namespace crowdb::tree @@ -37,6 +38,14 @@ inline constexpr size_t kDurableBlobHeader = 1 + 4 + 4 + 4; // when it actually shrinks the page (else stored raw with algo = kNone). void encode_durable_page(const uint8_t *frame, uint32_t page_bytes, compress_algo prefer, std::vector<uint8_t> *out); +// Allocation-free encoding into a caller-owned output window of at least +// kDurableBlobHeader + frame.size() bytes. Input/output must not overlap. +// Scratch includes codec-state alignment slack and may start at any address. +// The returned logical length excludes any backend alignment padding. +[[nodiscard]] size_t durable_encode_scratch_bytes(compress_algo prefer) noexcept; +Status encode_durable_page_into(std::span<const uint8_t> frame, compress_algo prefer, std::span<uint8_t> output, + std::span<uint8_t> scratch, uint32_t *logical_bytes); + // Read the logical (raw, uncompressed) frame length recorded in a durable // blob's header, so a reader can size the decode buffer without knowing the // frame geometry up front. Returns 0 if the blob is shorter than the header. diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/frame_page.h b/lib/crowdb-tree/include/crowdb-tree/maptable/frame_page.h index df37ae9f5..cf78f4ec2 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/frame_page.h +++ b/lib/crowdb-tree/include/crowdb-tree/maptable/frame_page.h @@ -6,8 +6,9 @@ // A page is a fixed-size byte frame whose in-memory layout IS its on-disk // layout: no encode/decode. The B+tree reads, binary-searches, and compares // keys directly on the frame bytes via the non-owning views below; Slices point -// into the frame (zero copy). Builders construct a frame from already key-sorted -// input (the flush/consolidate/split output) and stamp a CRC32C trailer. +// into the frame (zero copy). Builders construct a frame from already +// key-sorted input (the flush/consolidate/split output) and stamp a CRC32C +// trailer. // // Layout (leaf): // [FrameHeader 64B][Slot[slot_count] (sorted, grows fwd)] ... free ... @@ -25,6 +26,7 @@ #include <cstdint> #include <cstring> +#include <span> #include <vector> namespace crowdb::tree @@ -45,43 +47,43 @@ inline constexpr size_t kInnerSlotSize = 8; // rec_off, key_len (u32) struct PageSummary { - uint64_t live_kv = 0; - uint64_t live_key_bytes = 0; - uint64_t live_value_bytes = 0; - uint64_t live_logical_bytes = 0; - uint64_t reachable_leaf_pages = 0; - uint64_t reachable_inner_pages = 0; - uint64_t reachable_overflow_pages = 0; + uint64_t live_kv = 0; + uint64_t live_key_bytes = 0; + uint64_t live_value_bytes = 0; + uint64_t live_logical_bytes = 0; + uint64_t reachable_leaf_pages = 0; + uint64_t reachable_inner_pages = 0; + uint64_t reachable_overflow_pages = 0; uint64_t reachable_page_capacity_bytes = 0; - bool exact = false; + bool exact = false; }; // Header field byte offsets within a frame. namespace fh { -inline constexpr size_t kMagic = 0; // u32 -inline constexpr size_t kType = 4; // u8 (page_type) -inline constexpr size_t kFormatVersion = 5; // u8 -inline constexpr size_t kFlags = 6; // u8 (reserved; compression is on stored bytes) -inline constexpr size_t kSlotCount = 8; // u32 (leaf: entries; inner: separators) -inline constexpr size_t kFreeLo = 12; // u32 (end of main slot dir) -inline constexpr size_t kFreeHi = 16; // u32 (lowest used record offset) -inline constexpr size_t kDeltaCount = 20; // u32 (leaf: in-frame delta count, PT12) -inline constexpr size_t kSelfpage_id = 24; // u64 -inline constexpr size_t kRightSibling = 32; // u64 (leaf only) -inline constexpr size_t kLowerFenceOff = 40; // u32 -inline constexpr size_t kLowerFenceLen = 44; // u32 -inline constexpr size_t kUpperFenceOff = 48; // u32 -inline constexpr size_t kUpperFenceLen = 52; // u32 -inline constexpr size_t kSummaryLiveKv = 64; -inline constexpr size_t kSummaryLiveKeyBytes = 72; -inline constexpr size_t kSummaryLiveValueBytes = 80; -inline constexpr size_t kSummaryLiveLogicalBytes = 88; -inline constexpr size_t kSummaryLeafPages = 96; -inline constexpr size_t kSummaryInnerPages = 104; -inline constexpr size_t kSummaryOverflowPages = 112; +inline constexpr size_t kMagic = 0; // u32 +inline constexpr size_t kType = 4; // u8 (page_type) +inline constexpr size_t kFormatVersion = 5; // u8 +inline constexpr size_t kFlags = 6; // u8 (reserved; compression is on stored bytes) +inline constexpr size_t kSlotCount = 8; // u32 (leaf: entries; inner: separators) +inline constexpr size_t kFreeLo = 12; // u32 (end of main slot dir) +inline constexpr size_t kFreeHi = 16; // u32 (lowest used record offset) +inline constexpr size_t kDeltaCount = 20; // u32 (leaf: in-frame delta count, PT12) +inline constexpr size_t kSelfpage_id = 24; // u64 +inline constexpr size_t kRightSibling = 32; // u64 (leaf only) +inline constexpr size_t kLowerFenceOff = 40; // u32 +inline constexpr size_t kLowerFenceLen = 44; // u32 +inline constexpr size_t kUpperFenceOff = 48; // u32 +inline constexpr size_t kUpperFenceLen = 52; // u32 +inline constexpr size_t kSummaryLiveKv = 64; +inline constexpr size_t kSummaryLiveKeyBytes = 72; +inline constexpr size_t kSummaryLiveValueBytes = 80; +inline constexpr size_t kSummaryLiveLogicalBytes = 88; +inline constexpr size_t kSummaryLeafPages = 96; +inline constexpr size_t kSummaryInnerPages = 104; +inline constexpr size_t kSummaryOverflowPages = 112; inline constexpr size_t kSummaryPageCapacityBytes = 120; -inline constexpr size_t kSummaryFlags = 127; +inline constexpr size_t kSummaryFlags = 127; } // namespace fh // ── little-endian frame accessors ───────────────────────────────── @@ -149,18 +151,18 @@ void frame_restamp_crc(uint8_t *f, uint32_t page_bytes); // record space. Inner frames instead persist the page IDs of the leftmost and // rightmost reachable leaves, whose own fences hold the exact boundary keys. // Empty subtrees carry neither representation. -[[nodiscard]] bool frame_set_fences(uint8_t *f, uint32_t page_bytes, const Slice *lower, const Slice *upper); -[[nodiscard]] bool frame_has_lower_fence(const uint8_t *f); -[[nodiscard]] bool frame_has_upper_fence(const uint8_t *f); -[[nodiscard]] Slice frame_lower_fence(const uint8_t *f); -[[nodiscard]] Slice frame_upper_fence(const uint8_t *f); -[[nodiscard]] bool frame_fences_are_page_ids(const uint8_t *f); -[[nodiscard]] uint64_t frame_lower_fence_page_id(const uint8_t *f); -[[nodiscard]] uint64_t frame_upper_fence_page_id(const uint8_t *f); -void frame_set_inner_fence_pages(uint8_t *f, uint32_t page_bytes, uint64_t lower_leaf_page_id, - uint64_t upper_leaf_page_id); +[[nodiscard]] bool frame_set_fences(uint8_t *f, uint32_t page_bytes, const Slice *lower, const Slice *upper); +[[nodiscard]] bool frame_has_lower_fence(const uint8_t *f); +[[nodiscard]] bool frame_has_upper_fence(const uint8_t *f); +[[nodiscard]] Slice frame_lower_fence(const uint8_t *f); +[[nodiscard]] Slice frame_upper_fence(const uint8_t *f); +[[nodiscard]] bool frame_fences_are_page_ids(const uint8_t *f); +[[nodiscard]] uint64_t frame_lower_fence_page_id(const uint8_t *f); +[[nodiscard]] uint64_t frame_upper_fence_page_id(const uint8_t *f); +void frame_set_inner_fence_pages(uint8_t *f, uint32_t page_bytes, uint64_t lower_leaf_page_id, + uint64_t upper_leaf_page_id); [[nodiscard]] PageSummary frame_summary(const uint8_t *f); -void frame_set_summary(uint8_t *f, uint32_t page_bytes, const PageSummary &summary); +void frame_set_summary(uint8_t *f, uint32_t page_bytes, const PageSummary &summary); // ── Leaf view (zero-copy) ───────────────────────────────────────── class LeafFrameView @@ -406,11 +408,15 @@ class LeafFrameBuilder bool inner_frame_build(uint8_t *f, uint32_t page_bytes, uint64_t self_page_id, const std::vector<uint64_t> &children, const std::vector<Slice> &separators); +bool inner_frame_build_span(uint8_t *f, uint32_t page_bytes, uint64_t self_page_id, std::span<const uint64_t> children, + std::span<const Slice> separators); + // ── Overflow frame (PT11): one chunk of a large value ───────────── -// Layout: [FrameHeader 64B (chunk_len @ kSlotCount, next_page_id @ kRightSibling)] +// Layout: [FrameHeader 64B (chunk_len @ kSlotCount, next_page_id @ +// kRightSibling)] // [payload chunk] ... zero pad ... [Trailer 8B]. -// A large value of N bytes spans ceil(N / overflow_chunk_cap(page_bytes)) frames -// linked by next_page_id (kInvalidPageId on the last). +// A large value of N bytes spans ceil(N / overflow_chunk_cap(page_bytes)) +// frames linked by next_page_id (kInvalidPageId on the last). [[nodiscard]] inline uint32_t overflow_chunk_cap(uint32_t page_bytes) { return page_bytes - static_cast<uint32_t>(kFrameHeaderSize) - static_cast<uint32_t>(kFrameTrailerSize); diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_geometry.h b/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_geometry.h new file mode 100644 index 000000000..e1b5f0b88 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_geometry.h @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <cstdint> + +namespace crowdb::tree +{ +// Durable local segment geometry. These values are part of the existing +// anchor/segment format and do not depend on an in-memory mapping +// implementation. +inline constexpr uint64_t kMappingSegmentSize = 1024; +inline constexpr uint64_t kMaxMappingSegments = 1U << 16; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_persist.h b/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_persist.h index fca89abef..b3c02898b 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_persist.h +++ b/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_persist.h @@ -1,25 +1,18 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// On-disk mapping-table formats: segment image + segment directory -// (plan-tree #14c). +// Durable segment image and directory codecs. Local checkpoints and explicit +// legacy migration use these backend-neutral blobs. Anchor encoding and +// validation are separate snapshot metadata operations. // -// Both formats are backend-neutral byte blobs (no fixed file region) -- -// callers allocate/write/read them through PageStore like any other durable -// blob. The commit anchor (§7.3) that ties a snapshot generation to a -// segment-directory address is encoded/decoded in persist.cpp, reusing (and -// extending) the existing superblock A/B machinery rather than duplicating -// it here. +// Segment images hold packed durable words (mapping_slot.h), never resident +// pointers. Version 1 carries local byte descriptors; version 2 additionally +// permits immutable-store page references for legacy migration. Recovery +// validates the words before constructing typed backing descriptors. // -// Segment image: a dirty MappingSegment's full packed-word array, verbatim -// (crowdb::tree::slot_word encoding, mapping_slot.h) -- recovery installs the -// words into slots[] with zero decode. Version 1 contains local byte -// descriptors. Version 2 permits self-tagged immutable-store page references -// alongside legacy descriptors for lazy migration. -// -// Segment directory: the list of every *live* segment's latest generation + -// image location, rewritten in full whenever any segment's generation -// changes. Small (O(live segments), not O(resident pages)). +// A directory records each live segment's generation, image location and CRC. +// The existing on-disk encoding is independent of the mutable mapping test +// model. #pragma once #include "crowdb-tree/status.h" diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_slot.h b/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_slot.h index b43a6a2ed..edee27581 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_slot.h +++ b/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_slot.h @@ -1,25 +1,20 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// Packed 64-bit mapping-table slot word (plan-tree #14a). +// Packed 64-bit mapping slot words. Segment recovery validates durable words +// before constructing typed backing descriptors. // -// The *same* encoding is used in -// memory and on disk, so a persisted segment image is literally the array of -// packed words and recovery installs them with zero decode: +// Zero is empty. A nonzero word whose low three bits are zero is an aligned +// resident PageBase pointer, valid only in memory. // -// word == 0 -> empty (dead or never-allocated PID) -// (word & 7) == 0 (and != 0) -> resident: PageBase* (8-byte aligned) [memory only] -// (word & 1) == 1 -> legacy/local unloaded descriptor: -// bits [63:24] iu_index (durable PageAddr, in IUs) -// bits [23:1] iu_count (page length, in IUs) -// bit [0] tag = 1 -// (word & 7) == 2 -> chunk page-reference descriptor: -// bits [63:24] reference ordinal -// bits [23:3] IU count -// bits [2:0] tag = 2 +// A local descriptor has tag 1 in bit 0, an IU count in bits [23:1], and an +// address measured in IUs in bits [63:24]. // -// Resident pointers are never persisted; on disk a slot is only `0` or a tagged -// unloaded descriptor. Adopted by the live `MappingTable` (mapping_table.h). +// A Chunk reference has tag 2 in bits [2:0], an IU count in bits [23:3], and a +// reference ordinal in bits [63:24]. +// +// Resident pointers are never persisted; disk slots are empty or tagged +// descriptors. The legacy mutable mapping model survives only in tests. #pragma once #include "crowdb-tree/maptable/page_types.h" // PageBase diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/page.h b/lib/crowdb-tree/include/crowdb-tree/maptable/page.h index 532cf568b..94014ea06 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/page.h +++ b/lib/crowdb-tree/include/crowdb-tree/maptable/page.h @@ -3,17 +3,17 @@ // Pages. // -// The core in-memory engine represents pages as C++ objects (not the byte-packed -// on-disk offset-array layout; that lives in the persistence plan). Semantics -// match the design: leaves hold sorted (key, cell) entries with a right-sibling -// link; inner pages hold separator keys + child PIDs. Delta records (CT8) link -// in front of a LeafBase via the chain fields in PageBase. +// The core in-memory engine represents pages as C++ objects (not the +// byte-packed on-disk offset-array layout; that lives in the persistence plan). +// Semantics match the design: leaves hold sorted (key, cell) entries with a +// right-sibling link; inner pages hold separator keys + child PIDs. Delta +// records (CT8) link in front of a LeafBase via the chain fields in PageBase. #pragma once #include "crowdb-tree/btree/cell.h" -#include "crowdb-tree/maptable/buffer_pool.h" #include "crowdb-tree/maptable/frame_page.h" #include "crowdb-tree/maptable/page_types.h" +#include "crowdb-tree/memory_budget.h" #include "crowdb-tree/slice.h" #include <cstdint> @@ -25,44 +25,27 @@ namespace crowdb::tree { -// Backing bytes for a base page. Either a buffer-pool frame (the -// page co-owns the pool via shared_ptr so the frame is valid even when the page -// is freed late by the env-level epoch manager) or a heap buffer (used by -// standalone/unit construction, recovery, oversized pages, or pool exhaustion). +// Owned backing bytes for a decoded or constructed base page. Reference +// identity follows the allocation; shared cache residency is managed +// separately. struct FrameStore { - std::shared_ptr<BufferPool> pool; // non-null => pool-backed - uint32_t frame_idx = 0; - std::vector<uint8_t> owned; // used iff pool == nullptr - uint8_t *ptr = nullptr; - uint32_t page_bytes = 0; - FrameStore() = default; - FrameStore(const FrameStore &) = delete; - FrameStore &operator=(const FrameStore &) = delete; + std::vector<uint8_t, BudgetAllocator<uint8_t, AllocationBudget, AllocationKind::kFrame>> owned; + uint8_t *ptr = nullptr; + uint32_t page_bytes = 0; - ~FrameStore() + explicit FrameStore(std::shared_ptr<AllocationBudget> budget = {}) + : owned(BudgetAllocator<uint8_t, AllocationBudget, AllocationKind::kFrame>(std::move(budget))) { - if (pool) { - pool->release_frame(frame_idx); - } } - // Allocate writable backing for a `need`-byte page. Uses a fixed pool frame - // when one is available and large enough; otherwise a tight heap buffer. - uint8_t *alloc(size_t need, const std::shared_ptr<BufferPool> &p, uint32_t frame_bytes) + FrameStore(const FrameStore &) = delete; + FrameStore &operator=(const FrameStore &) = delete; + + // Allocate an owned, aligned logical frame for construction. + uint8_t *alloc(size_t need) { - if (p && need <= frame_bytes) { - uint32_t idx = 0; - uint8_t *bytes = nullptr; - if (p->acquire_frame(&idx, &bytes).ok()) { - pool = p; - frame_idx = idx; - ptr = bytes; - page_bytes = frame_bytes; - return ptr; - } - } auto pb = static_cast<uint32_t>((need < 128 ? 128 : need + 7) & ~static_cast<size_t>(7)); owned.assign(pb, 0); ptr = owned.data(); @@ -70,25 +53,9 @@ struct FrameStore return ptr; } - // Wrap a copy of an existing `n`-byte frame image (demand load / recovery). - // Copies into a pool frame when one fits and is available, else a heap buffer. - // page_bytes stays `n` (the durable logical length); a pool frame may be - // larger (the tail past `n` is zero from acquire_frame and unused). - uint8_t *adopt_copy(const uint8_t *buf, uint32_t n, const std::shared_ptr<BufferPool> &p = nullptr, - uint32_t frame_bytes = 0) - { - if (p && n <= frame_bytes) { - uint32_t idx = 0; - uint8_t *bytes = nullptr; - if (p->acquire_frame(&idx, &bytes).ok()) { - pool = p; - frame_idx = idx; - ptr = bytes; - page_bytes = n; - std::memcpy(bytes, buf, n); - return ptr; - } - } + // Copy an existing frame while preserving its durable logical length. + uint8_t *adopt_copy(const uint8_t *buf, uint32_t n) + { owned.assign(buf, buf + n); ptr = owned.data(); page_bytes = n; @@ -102,30 +69,32 @@ struct FrameStore class LeafBase : public PageBase { public: - LeafBase() : PageBase(page_type::kLeafBase) + explicit LeafBase(std::shared_ptr<AllocationBudget> budget = {}) + : PageBase(page_type::kLeafBase), + fs_(std::move(budget)) { } // build from already key-sorted, deduplicated entries. Entries are only READ - // (their bytes are copied into the frame — that copy is page construction), so - // take them by const ref: this both avoids requiring a move at every call site - // and works with the move-only `buffer` cell (leaf_entry is non-copyable). - static LeafBase *build(const std::vector<leaf_entry> &sorted_entries, uint64_t right_sibling = kInvalidPageId, - const std::shared_ptr<BufferPool> &pool = nullptr, uint32_t frame_bytes = 0) + // (their bytes are copied into the frame — that copy is page construction), + // so take them by const ref: this both avoids requiring a move at every call + // site and works with the move-only `buffer` cell (leaf_entry is + // non-copyable). + static LeafBase *build(const std::vector<leaf_entry> &sorted_entries, uint64_t right_sibling = kInvalidPageId) { auto *p = new LeafBase(); size_t need = kFrameHeaderSize + kFrameTrailerSize + (sorted_entries.size() * kLeafSlotSize); for (const auto &e : sorted_entries) { need += e.key.size() + e.cell.size(); } - uint8_t *dst = p->fs_.alloc(need, pool, frame_bytes); + uint8_t *dst = p->fs_.alloc(need); LeafFrameBuilder b(dst, p->fs_.page_bytes); for (const auto &e : sorted_entries) { b.try_append_sorted(Slice(e.key), Slice(e.cell)); } b.finish(p->page_id, right_sibling); PageSummary summary; - summary.reachable_leaf_pages = 1; + summary.reachable_leaf_pages = 1; summary.reachable_page_capacity_bytes = p->fs_.page_bytes; for (const auto &e : sorted_entries) { CellView cell{Slice(e.cell)}; @@ -148,12 +117,19 @@ class LeafBase : public PageBase } // Wrap a copy of an existing frame image (e.g. read from durable storage). - static LeafBase *from_frame_copy(const uint8_t *buf, uint32_t page_bytes, - const std::shared_ptr<BufferPool> &pool = nullptr, uint32_t frame_bytes = 0) + static LeafBase *from_frame_copy(const uint8_t *buf, uint32_t page_bytes) { - auto *p = new LeafBase(); - p->fs_.adopt_copy(buf, page_bytes, pool, frame_bytes); - return p; + auto p = std::make_unique<LeafBase>(); + p->fs_.adopt_copy(buf, page_bytes); + return p.release(); + } + + static std::shared_ptr<const LeafBase> from_frame_copy_shared(const uint8_t *buf, uint32_t page_bytes, + const std::shared_ptr<AllocationBudget> &budget) + { + auto page = std::allocate_shared<LeafBase>(MetadataAllocator<LeafBase>(budget), budget); + page->fs_.adopt_copy(buf, page_bytes); + return page; } [[nodiscard]] LeafFrameView view() const @@ -280,8 +256,7 @@ class InnerBase : public PageBase { } - static InnerBase *build(const std::vector<std::string> &separators, const std::vector<uint64_t> &children, - const std::shared_ptr<BufferPool> &pool = nullptr, uint32_t frame_bytes = 0) + static InnerBase *build(const std::vector<std::string> &separators, const std::vector<uint64_t> &children) { auto *p = new InnerBase(); size_t need = kFrameHeaderSize + kFrameTrailerSize + (children.size() * sizeof(uint64_t)) + @@ -289,7 +264,7 @@ class InnerBase : public PageBase for (const auto &s : separators) { need += s.size(); } - uint8_t *dst = p->fs_.alloc(need, pool, frame_bytes); + uint8_t *dst = p->fs_.alloc(need); std::vector<Slice> sep_slices; sep_slices.reserve(separators.size()); for (const auto &s : separators) { @@ -300,11 +275,10 @@ class InnerBase : public PageBase } // Wrap a copy of an existing frame image (e.g. read from durable storage). - static InnerBase *from_frame_copy(const uint8_t *buf, uint32_t page_bytes, - const std::shared_ptr<BufferPool> &pool = nullptr, uint32_t frame_bytes = 0) + static InnerBase *from_frame_copy(const uint8_t *buf, uint32_t page_bytes) { auto *p = new InnerBase(); - p->fs_.adopt_copy(buf, page_bytes, pool, frame_bytes); + p->fs_.adopt_copy(buf, page_bytes); return p; } @@ -391,8 +365,8 @@ class InnerBase : public PageBase }; // One frame of a large value spilled out of a leaf (PT11). Referenced by an -// overflow pointer cell, not by a child PID; chained via next_page_id. resident as -// an ordinary mapping-table page so it demand-loads + evicts like any base. +// overflow pointer cell, not by a child PID; chained via next_page_id. resident +// as an ordinary mapping-table page so it demand-loads + evicts like any base. class OverflowBase : public PageBase { public: @@ -401,21 +375,19 @@ class OverflowBase : public PageBase } // build a frame carrying `chunk_len` payload bytes (<= overflow_chunk_cap). - static OverflowBase *build(uint64_t next_page_id, const uint8_t *payload, uint32_t chunk_len, - const std::shared_ptr<BufferPool> &pool = nullptr, uint32_t frame_bytes = 0) + static OverflowBase *build(uint64_t next_page_id, const uint8_t *payload, uint32_t chunk_len) { auto *p = new OverflowBase(); size_t need = kFrameHeaderSize + kFrameTrailerSize + chunk_len; - uint8_t *dst = p->fs_.alloc(need, pool, frame_bytes); + uint8_t *dst = p->fs_.alloc(need); overflow_frame_build(dst, p->fs_.page_bytes, p->page_id, next_page_id, payload, chunk_len); return p; } - static OverflowBase *from_frame_copy(const uint8_t *buf, uint32_t page_bytes, - const std::shared_ptr<BufferPool> &pool = nullptr, uint32_t frame_bytes = 0) + static OverflowBase *from_frame_copy(const uint8_t *buf, uint32_t page_bytes) { auto *p = new OverflowBase(); - p->fs_.adopt_copy(buf, page_bytes, pool, frame_bytes); + p->fs_.adopt_copy(buf, page_bytes); return p; } diff --git a/lib/crowdb-tree/include/crowdb-tree/memory_budget.h b/lib/crowdb-tree/include/crowdb-tree/memory_budget.h new file mode 100644 index 000000000..b0c8609c3 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/memory_budget.h @@ -0,0 +1,260 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <memory> +#include <new> +#include <utility> + +namespace crowdb::tree +{ +class TreeWorkMetrics; +class TreeResourceMetrics; +class AllocationBudget; + +namespace detail +{ +class RetirementQueue; +template <class T> struct RetirementTraits; + +struct RetirementCompletion +{ + std::shared_ptr<const void> owner; + void (*finish)(const void *) noexcept = nullptr; +}; + +struct RetirementObservation +{ + void start() noexcept; + std::shared_ptr<TreeWorkMetrics> metrics; + std::shared_ptr<TreeResourceMetrics> resources; + size_t bytes = 0; + std::atomic<uint64_t> started_at{0}; + size_t age_slot = std::numeric_limits<size_t>::max(); +}; + +// A copied immutable descriptor receives a fresh observer from its factory. +// The observer storage is embedded in the charged lifetime node, outside T. +class RetirementObserver +{ + public: + RetirementObserver() = default; + + RetirementObserver(const RetirementObserver & /*other*/) noexcept + { + } + + RetirementObserver &operator=(const RetirementObserver &) = delete; + + void bind(RetirementObservation *observation) noexcept + { + observation_ = observation; + } + + void mark() const noexcept + { + if (observation_ != nullptr) { + observation_->start(); + } + } + + private: + RetirementObservation *observation_ = nullptr; +}; + +struct RetirementNode +{ + std::shared_ptr<AllocationBudget> retirement_budget; + RetirementCompletion retirement_completion; + RetirementObservation observation; + RetirementNode *next = nullptr; + bool (*dispose)(RetirementNode *) noexcept = nullptr; +}; +} // namespace detail + +enum class AllocationKind : uint8_t { kUnknown, kPayload, kFrame, kMetadata, kIo, kScratch, kCount }; + +// Allocation owners can outlive the Tree and every source catalog. Claims +// fail without waiting. Reservations consume admitted credit; stable backend +// accounts admit new capacity against both their current Tree and host quota. +class AllocationBudget +{ + public: + virtual ~AllocationBudget() = default; + [[nodiscard]] virtual bool claim(size_t bytes) noexcept = 0; + virtual void release(size_t bytes) noexcept = 0; + + // Physical categories come from the actual allocation owner, independently + // of the admission partition. Observations never govern admission. + virtual void record_allocation(AllocationKind /*kind*/, size_t /*bytes*/, bool /*acquired*/) noexcept + { + } + + [[nodiscard]] virtual std::shared_ptr<TreeWorkMetrics> retirement_metrics() const noexcept + { + return {}; + } + + [[nodiscard]] virtual std::shared_ptr<TreeResourceMetrics> resource_metrics() const noexcept + { + return {}; + } + + [[nodiscard]] virtual detail::RetirementCompletion retirement_started() noexcept + { + return {}; + } + + [[nodiscard]] virtual std::shared_ptr<detail::RetirementQueue> retirement_queue() const noexcept + { + return {}; + } + + [[nodiscard]] virtual const void *domain() const noexcept + { + return this; + } +}; + +class AllocationLease +{ + public: + AllocationLease() = default; + + AllocationLease(std::shared_ptr<AllocationBudget> budget, size_t bytes, + AllocationKind kind = AllocationKind::kUnknown) + : budget_(std::move(budget)), + bytes_(bytes), + kind_(kind) + { + if (budget_ && !budget_->claim(bytes_)) { + throw std::bad_alloc(); + } + if (budget_ && kind_ != AllocationKind::kUnknown) { + budget_->record_allocation(kind_, bytes_, true); + } + } + + ~AllocationLease() + { + reset(); + } + + AllocationLease(const AllocationLease &) = delete; + AllocationLease &operator=(const AllocationLease &) = delete; + + AllocationLease(AllocationLease &&other) noexcept + : budget_(std::move(other.budget_)), + bytes_(std::exchange(other.bytes_, 0)), + kind_(other.kind_) + { + } + + AllocationLease &operator=(AllocationLease &&other) noexcept + { + if (this != &other) { + reset(); + budget_ = std::move(other.budget_); + bytes_ = std::exchange(other.bytes_, 0); + kind_ = other.kind_; + } + return *this; + } + + void reset() noexcept + { + if (auto budget = std::move(budget_)) { + if (kind_ != AllocationKind::kUnknown) { + budget->record_allocation(kind_, bytes_, false); + } + budget->release(std::exchange(bytes_, 0)); + } + } + + private: + std::shared_ptr<AllocationBudget> budget_; + size_t bytes_ = 0; + AllocationKind kind_ = AllocationKind::kUnknown; +}; + +// The allocator sees the actual control-block/container capacity requested by +// the standard library, including growth and simultaneous old/new buffers. +template <class T, class Budget = AllocationBudget, AllocationKind Kind = AllocationKind::kUnknown> +class BudgetAllocator +{ + public: + using value_type = T; + using is_always_equal = std::false_type; + + template <class U> struct rebind + { + using other = BudgetAllocator<U, Budget, Kind>; + }; + + BudgetAllocator() = default; + + explicit BudgetAllocator(std::shared_ptr<Budget> owner) noexcept : owner_(std::move(owner)) + { + } + + template <class U> BudgetAllocator(const BudgetAllocator<U, Budget, Kind> &other) noexcept : owner_(other.owner()) + { + } + + [[nodiscard]] T *allocate(size_t count) + { + if (count > std::numeric_limits<size_t>::max() / sizeof(T)) { + throw std::bad_array_new_length(); + } + const auto bytes = count * sizeof(T); + if (owner_ && !owner_->claim(bytes)) { + throw std::bad_alloc(); + } + try { + auto *result = static_cast<T *>(::operator new(bytes, std::align_val_t{alignof(T)})); + if constexpr (Kind != AllocationKind::kUnknown) { + if (owner_) { + owner_->record_allocation(Kind, bytes, true); + } + } + return result; + } + catch (...) { + if (owner_) { + owner_->release(bytes); + } + throw; + } + } + + void deallocate(T *pointer, size_t count) noexcept + { + ::operator delete(pointer, std::align_val_t{alignof(T)}); + if (owner_) { + if constexpr (Kind != AllocationKind::kUnknown) { + owner_->record_allocation(Kind, count * sizeof(T), false); + } + owner_->release(count * sizeof(T)); + } + } + + [[nodiscard]] const std::shared_ptr<Budget> &owner() const noexcept + { + return owner_; + } + + template <class U> bool operator==(const BudgetAllocator<U, Budget, Kind> &other) const noexcept + { + return owner_ == other.owner(); + } + + private: + std::shared_ptr<Budget> owner_; +}; +template <class T, class Budget = AllocationBudget> +using MetadataAllocator = BudgetAllocator<T, Budget, AllocationKind::kMetadata>; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/memtable/allocation.h b/lib/crowdb-tree/include/crowdb-tree/memtable/allocation.h index fd97d9e56..1fd909089 100644 --- a/lib/crowdb-tree/include/crowdb-tree/memtable/allocation.h +++ b/lib/crowdb-tree/include/crowdb-tree/memtable/allocation.h @@ -9,7 +9,11 @@ namespace crowdb::tree { // Allocation accounting follows ownership, including descriptors and payloads // awaiting epoch collection. Logical mutation events are aggregated per batch. -using AllocationCounter = std::atomic<size_t>; +struct AllocationCounter : std::atomic<size_t> +{ + using std::atomic<size_t>::atomic; + using std::atomic<size_t>::operator=; +}; class AllocationCharge { diff --git a/lib/crowdb-tree/include/crowdb-tree/memtable/generation.h b/lib/crowdb-tree/include/crowdb-tree/memtable/generation.h deleted file mode 100644 index c327afdb1..000000000 --- a/lib/crowdb-tree/include/crowdb-tree/memtable/generation.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. -#pragma once - -#include <atomic> -#include <cstdint> -#include <utility> - -namespace crowdb::tree -{ -// Replacement fence, independent from per-table rotation. Ordinary admission -// uses atomic RMWs; replacement waits without holding source-selection locks. -class GenerationGate -{ - public: - class Guard - { - public: - explicit Guard(const GenerationGate *gate = nullptr) : gate_(gate) - { - } - - Guard(Guard &&other) noexcept : gate_(std::exchange(other.gate_, nullptr)) - { - } - - Guard(const Guard &) = delete; - Guard &operator=(const Guard &) = delete; - - ~Guard() - { - if (gate_ != nullptr) { - const auto previous = gate_->state_.fetch_sub(1, std::memory_order_acq_rel); - if (previous == kClosed + 1 || previous == kClosed - 1) { - gate_->state_.notify_all(); - } - } - } - - private: - const GenerationGate *gate_; - }; - - [[nodiscard]] Guard enter() const - { - auto state = state_.load(std::memory_order_acquire); - for (;;) { - if ((state & kClosed) != 0 || state == kClosed - 1) { - state_.wait(state, std::memory_order_acquire); - state = state_.load(std::memory_order_acquire); - } - else if (state_.compare_exchange_weak(state, state + 1, std::memory_order_acq_rel)) { - return Guard(this); - } - } - } - - void close() - { - auto state = state_.fetch_or(kClosed, std::memory_order_acq_rel) | kClosed; - while (state != kClosed) { - state_.wait(state, std::memory_order_acquire); - state = state_.load(std::memory_order_acquire); - } - } - - void open() - { - state_.store(0, std::memory_order_release); - state_.notify_all(); - } -#ifdef CROWDB_TREE_TEST_UTIL - [[nodiscard]] bool closed_for_tests() const - { - return (state_.load() & kClosed) != 0; - } -#endif - class Replacement - { - public: - explicit Replacement(GenerationGate &gate) : gate_(gate) - { - gate_.close(); - } - - ~Replacement() - { - gate_.open(); - } - - Replacement(const Replacement &) = delete; - Replacement &operator=(const Replacement &) = delete; - - private: - GenerationGate &gate_; - }; - - private: - static constexpr uint64_t kClosed = uint64_t{1} << 63; - mutable std::atomic<uint64_t> state_{0}; -}; -} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/memtable/memtable.h b/lib/crowdb-tree/include/crowdb-tree/memtable/memtable.h index bf0843901..c68887a27 100644 --- a/lib/crowdb-tree/include/crowdb-tree/memtable/memtable.h +++ b/lib/crowdb-tree/include/crowdb-tree/memtable/memtable.h @@ -32,18 +32,32 @@ class MemTable // `epoch` is the engine's EpochManager — used to retire unlinked nodes // and overwritten cell versions. Must outlive this MemTable (owned by // the Crowdbtree, which outlives all MemTables via shared_ptr). - explicit MemTable(uint64_t id = 0, EpochManager *epoch = nullptr) : list_(epoch), epoch_(&list_.epoch()), id_(id) + explicit MemTable(uint64_t id = 0, EpochManager *epoch = nullptr, std::shared_ptr<AllocationBudget> budget = {}) + : list_(epoch, std::move(budget)), + epoch_(&list_.epoch()), + id_(id) { } + ~MemTable(); + + // Attach only before this table is exposed to writers. Detach only after + // closure and writer drain; an old reader may still retain the table. + void attach_source_stats(std::shared_ptr<SourceMemoryLedger> ledger) noexcept; + void detach_source_stats() noexcept; + [[nodiscard]] SourceMemoryStats source_stats() const noexcept; + [[nodiscard]] VersionMemory memory() const { return list_.memory(); } - [[nodiscard]] EpochManager::Guard read_guard() const + [[nodiscard]] EpochManager::Guard read_guard(std::shared_ptr<AllocationBudget> budget = {}) const { - return epoch_->enter(); + if (list_.owned_epoch_ && closed() && writers() == 0) { + list_.owned_epoch_->seal_source(); + } + return epoch_->enter(std::move(budget)); } [[nodiscard]] uint64_t id() const @@ -56,13 +70,23 @@ class MemTable // slot <= durable_floor (already in L1) unless allow_old_slots is set. bool upsert(Slice key, uint64_t slot, Slice cell_payload); bool upsert(Slice key, uint64_t slot, buffer &&cell_payload, uint64_t bound = UINT64_MAX, - MutationStats *stats = nullptr); + MutationStats *stats = nullptr, std::shared_ptr<AllocationBudget> budget = {}); // Zero-copy apply path (R30): store a split cell — the value is borrowed // from a Rust `bytes::Bytes` via a kExternal buffer (no value memcpy), and // the 9-byte cell header is stored as `slot`/`flags` fields. bool upsert_external(Slice key, uint64_t slot, uint8_t flags, buffer &&value, uint64_t bound = UINT64_MAX, - MutationStats *stats = nullptr); + MutationStats *stats = nullptr, std::shared_ptr<AllocationBudget> budget = {}); + + using PreparedWrite = ConcurrentSkipList::PreparedWrite; + // Inputs are already owned and charged by source admission. Preparation + // stays on this thread until commit or destruction, with the table alive. + [[nodiscard]] PreparedWrite prepare_write(Slice key, buffer cell, uint8_t flags, AllocationLease payload, + uint64_t bound, size_t max_versions, + const std::shared_ptr<AllocationBudget> &budget); + [[nodiscard]] PreparedWrite prepare_reference(Slice key, std::shared_ptr<const CellVersion> cell, uint64_t bound, + size_t max_versions, const std::shared_ptr<AllocationBudget> &budget); + bool commit_write(PreparedWrite write, uint64_t slot) noexcept; void set_durable_floor(uint64_t slot); @@ -125,6 +149,12 @@ class MemTable return list_.prefix_cursor(frontier, floor); } + [[nodiscard]] ConcurrentSkipList::Cursor range_cursor(std::optional<Slice> lower, std::optional<Slice> upper, + uint64_t frontier, uint64_t floor = 0) const + { + return list_.range_cursor(lower, upper, frontier, floor); + } + void prune(Slice key, uint64_t bound, MutationStats *stats = nullptr) { list_.prune(key, bound, stats); @@ -136,17 +166,73 @@ class MemTable } #endif + struct SealedAdmissionBound + { + size_t entries; + size_t key_bytes; + }; + + // Set once by SourceAdmission while ordinary admission is paused, before + // permanent closure. Includes every accepted writer and all gap capacity. + void set_sealed_admission_bound(size_t entries, size_t key_bytes) noexcept + { + sealed_admission_ = {.entries = entries, .key_bytes = key_bytes}; + has_sealed_admission_.store(true, std::memory_order_release); + } + + [[nodiscard]] std::optional<SealedAdmissionBound> sealed_admission_bound() const noexcept + { + return has_sealed_admission_.load(std::memory_order_acquire) + ? std::optional<SealedAdmissionBound>{sealed_admission_} + : std::nullopt; + } + bool try_enter() noexcept; bool validate_open() noexcept; void leave() noexcept; void close() noexcept; void wait_frozen() const noexcept; + // Intrusive one-shot notification. The caller owns the node and retains + // this table until notification returns. A registered node must remain + // alive until its callback runs; cancellation does not unlink the node. + class FrozenWaiter + { + public: + using Complete = void (*)(void *) noexcept; + + FrozenWaiter(Complete complete, void *context) : complete_(complete), context_(context) + { + } + + FrozenWaiter(const FrozenWaiter &) = delete; + FrozenWaiter &operator=(const FrozenWaiter &) = delete; + + private: + friend class MemTable; + FrozenWaiter() = default; + Complete complete_ = nullptr; + void *context_ = nullptr; + FrozenWaiter *next_ = nullptr; + std::atomic_flag registered_ = ATOMIC_FLAG_INIT; + }; + + // Registration is followed by a readiness recheck. Notification may run + // inline, or on the last writer's thread; callbacks must not block. + [[nodiscard]] bool notify_when_frozen(FrozenWaiter &waiter) noexcept; + [[nodiscard]] bool closed() const { return (writers_.load(std::memory_order_acquire) & kClosed) != 0; } +#ifdef CROWDB_TREE_TEST_UTIL + [[nodiscard]] EpochManager &epoch_for_tests() const noexcept + { + return list_.epoch(); + } +#endif + [[nodiscard]] uint64_t writers() const { return writers_.load(std::memory_order_acquire) & ~kClosed; @@ -177,6 +263,16 @@ class MemTable } private: + friend struct detail::RetirementTraits<MemTable>; + static constexpr bool observe_retirement = true; + + bool retire_step() noexcept + { + return list_.retire_step(); + } + + void notify_frozen() noexcept; + void update_slot_range(uint64_t slot) { // Concurrent extrema are monotonic until the whole table is retired. @@ -188,9 +284,15 @@ class MemTable } } - static constexpr uint64_t kClosed = uint64_t{1} << 63; - std::atomic<uint64_t> writers_{0}; - std::atomic<bool> backlog_warned_{false}; + SealedAdmissionBound sealed_admission_{}; + std::atomic<bool> has_sealed_admission_{false}; + static constexpr uint64_t kClosed = uint64_t{1} << 63; + std::atomic<uint64_t> writers_{0}; + static FrozenWaiter frozen_sentinel_; + std::atomic<FrozenWaiter *> frozen_waiters_{nullptr}; + std::atomic<bool> backlog_warned_{false}; + std::atomic_flag frozen_notified_ = ATOMIC_FLAG_INIT; + bool source_attached_ = false; ConcurrentSkipList list_; EpochManager *epoch_; @@ -199,6 +301,8 @@ class MemTable std::atomic<bool> allow_old_slots_{false}; std::atomic<uint64_t> min_slot_{UINT64_MAX}; std::atomic<uint64_t> max_slot_{0}; + + public: }; } // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/memtable/skip_list.h b/lib/crowdb-tree/include/crowdb-tree/memtable/skip_list.h index b0128a5e9..2a30a42d8 100644 --- a/lib/crowdb-tree/include/crowdb-tree/memtable/skip_list.h +++ b/lib/crowdb-tree/include/crowdb-tree/memtable/skip_list.h @@ -9,11 +9,14 @@ #include "crowdb-tree/btree/cell.h" #include "crowdb-tree/buffer.h" #include "crowdb-tree/epoch.h" +#include "crowdb-tree/memory_budget.h" +#include "crowdb-tree/memtable/source_stats.h" #include "crowdb-tree/slice.h" #include <atomic> #include <cstdint> #include <memory> +#include <optional> #include <string> #include <vector> @@ -27,13 +30,19 @@ namespace crowdb::tree // value-only borrowed from a Rust Bytes (drop_fn fires on destruction). struct CellVersion { - buffer cell; - uint64_t slot; - uint8_t flags; - AllocationCharge allocation; - - CellVersion(buffer bytes, uint64_t version, uint8_t kind, std::shared_ptr<AllocationCounter> counter = nullptr) - : cell(std::move(bytes)), + static std::shared_ptr<CellVersion> create(buffer bytes, uint64_t version, uint8_t kind, + std::shared_ptr<AllocationCounter> counter, AllocationLease credit, + const std::shared_ptr<AllocationBudget> &budget); + AllocationLease payload_credit; + buffer cell; + uint64_t slot; + uint8_t flags; + AllocationCharge allocation; + + CellVersion(buffer bytes, uint64_t version, uint8_t kind, std::shared_ptr<AllocationCounter> counter = nullptr, + AllocationLease credit = {}) + : payload_credit(std::move(credit)), + cell(std::move(bytes)), slot(version), flags(kind) { @@ -45,25 +54,27 @@ struct CellVersion // Immutable descriptor. The common case has no history allocation. History // is ordered newest first; payload ownership is shared across descriptors. -struct VersionSet : EpochManager::Deferred +struct VersionSet : EpochManager::Deferred, detail::RetirementNode { - std::shared_ptr<const CellVersion> current; - std::vector<std::shared_ptr<const CellVersion>> history; - uint64_t bound = 0; - size_t bytes = 0; - AllocationCharge allocation; - - VersionSet(); - [[nodiscard]] const CellVersion *at(uint64_t frontier) const; -}; + AllocationLease descriptor_credit; + std::shared_ptr<detail::RetirementQueue> retirement; + std::shared_ptr<const CellVersion> current; + std::vector<std::shared_ptr<const CellVersion>, MetadataAllocator<std::shared_ptr<const CellVersion>>> history; + uint64_t bound = 0; + size_t bytes = 0; + AllocationCharge allocation; -struct VersionMemory -{ - uint64_t history_count = 0; - uint64_t history_bytes = 0; - uint64_t descriptor_bytes = 0; - uint64_t node_bytes = 0; - uint64_t payload_bytes = 0; + explicit VersionSet(AllocationLease credit = {}, std::shared_ptr<AllocationBudget> budget = {}); + + struct Deleter + { + void operator()(VersionSet *versions) const noexcept; + }; + + using Owner = std::unique_ptr<VersionSet, Deleter>; + static Owner create(const std::shared_ptr<AllocationBudget> &budget); + [[nodiscard]] const CellVersion *at(uint64_t frontier) const; + [[nodiscard]] const std::shared_ptr<const CellVersion> *owner_at(uint64_t frontier) const; }; struct MutationStats @@ -85,6 +96,7 @@ struct MutationStats struct Node : EpochManager::Deferred { + AllocationLease allocation_credit; AllocationCharge allocation; std::atomic<VersionSet *> versions_{nullptr}; uint32_t height_{1}; // tower height @@ -134,6 +146,53 @@ class ConcurrentSkipList public: static constexpr uint32_t kMaxHeight = 12; + // Preparation is thread-bound and owns every allocation needed by commit. + // The source admission authority bounds all versions that may race this + // write, including writers that have not reached their CAS yet. + class PreparedWrite + { + public: + PreparedWrite() = default; + PreparedWrite(PreparedWrite &&) noexcept = default; + PreparedWrite &operator=(PreparedWrite &&) noexcept = default; + PreparedWrite(const PreparedWrite &) = delete; + PreparedWrite &operator=(const PreparedWrite &) = delete; + + [[nodiscard]] Slice key() const noexcept + { + return candidate_->key_slice(); + } + + [[nodiscard]] const std::shared_ptr<const CellVersion> &cell_owner() const noexcept + { + return cell_; + } + + private: + friend class ConcurrentSkipList; + + struct NodeDeleter + { + void operator()(Node *node) const noexcept; + }; + + EpochManager::Guard guard_; + std::shared_ptr<const CellVersion> cell_; + bool initialize_slot_ = true; + VersionSet::Owner initial_; + VersionSet::Owner replacement_; + std::unique_ptr<Node, NodeDeleter> candidate_; + uint64_t bound_ = 0; + }; + + [[nodiscard]] PreparedWrite prepare_write(Slice key, std::shared_ptr<CellVersion> cell, uint64_t bound, + size_t max_versions, const std::shared_ptr<AllocationBudget> &budget); + // Consumes preparation once. Concurrent version changes are rebuilt using + // its fixed capacity; this path does not allocate or return an error. + [[nodiscard]] PreparedWrite prepare_reference(Slice key, std::shared_ptr<const CellVersion> cell, uint64_t bound, + size_t max_versions, const std::shared_ptr<AllocationBudget> &budget); + bool commit_write(PreparedWrite write, uint64_t slot) noexcept; + // Ordered cursor with one coherent version candidate per position. The // caller retains source ownership and epoch protection for its lifetime. class Cursor @@ -141,10 +200,12 @@ class ConcurrentSkipList public: Cursor() = default; - explicit Cursor(const Node *n, uint64_t frontier = UINT64_MAX, uint64_t floor = 0) + explicit Cursor(const Node *n, uint64_t frontier = UINT64_MAX, uint64_t floor = 0, + std::optional<Slice> upper = std::nullopt) : cur_(n), frontier_(frontier), - floor_(floor) + floor_(floor), + upper_(upper) { select(); } @@ -164,6 +225,25 @@ class ConcurrentSkipList return candidate_; } + // Copy while the source and this thread's epoch guard are alive. The + // result owns only this version, independently of the node/history. + [[nodiscard]] std::shared_ptr<const CellVersion> cell_owner() const + { + return candidate_owner_ != nullptr ? *candidate_owner_ : nullptr; + } + + // Select this exact key at a captured frontier without advancing to + // another key. The caller keeps the source and its epoch guard alive. + [[nodiscard]] std::shared_ptr<const CellVersion> cell_owner_at(uint64_t frontier) const + { + if (cur_ == nullptr) { + return {}; + } + const auto *versions = cur_->versions_.load(std::memory_order_acquire); + const auto *owner = versions->owner_at(frontier); + return owner != nullptr ? *owner : nullptr; + } + void set_floor(uint64_t floor) { floor_ = floor; @@ -187,21 +267,31 @@ class ConcurrentSkipList } private: - void select(); - const Node *cur_ = nullptr; - const CellVersion *candidate_ = nullptr; - uint64_t frontier_ = UINT64_MAX; - uint64_t floor_ = 0; + void select(); + const Node *cur_ = nullptr; + const CellVersion *candidate_ = nullptr; + uint64_t frontier_ = UINT64_MAX; + uint64_t floor_ = 0; + std::optional<Slice> upper_; + const std::shared_ptr<const CellVersion> *candidate_owner_ = nullptr; }; - explicit ConcurrentSkipList(EpochManager *epoch = nullptr); + explicit ConcurrentSkipList(EpochManager *epoch = nullptr, std::shared_ptr<AllocationBudget> budget = {}); ~ConcurrentSkipList(); ConcurrentSkipList(const ConcurrentSkipList &) = delete; ConcurrentSkipList &operator=(const ConcurrentSkipList &) = delete; #ifdef CROWDB_TREE_TEST_UTIL - enum class PausePoint : uint8_t { kBeforeSearch, kAfterLevelZero, kBeforeVersionCas }; + enum class PausePoint : uint8_t { + kBeforePrepare, + kBeforeSearch, + kAfterLevelZero, + kBeforeVersionCas, + kBeforeInsertAccounting, + kBeforeLeaveAccounting, + kBeforeCloseAccounting + }; using TestHook = void (*)(void *, PausePoint, Slice); void set_hook_for_tests(void *context, TestHook hook) @@ -212,16 +302,32 @@ class ConcurrentSkipList #endif // Consumes the payload on every outcome. Bound is validated by admission. - bool upsert(Slice key, std::shared_ptr<const CellVersion> cv, uint64_t bound, MutationStats *stats = nullptr); + bool upsert(Slice key, std::shared_ptr<const CellVersion> cv, uint64_t bound, MutationStats *stats = nullptr, + std::shared_ptr<AllocationBudget> budget = {}); void prune(Slice key, uint64_t bound, MutationStats *stats = nullptr); - [[nodiscard]] Cursor prefix_cursor(uint64_t frontier, uint64_t floor = 0) const; + [[nodiscard]] Cursor prefix_cursor(uint64_t frontier, uint64_t floor = 0) const; + // Bounds are [lower, upper); an absent bound differs from an empty key. + // Bound storage and source/epoch protection must outlive the cursor. + [[nodiscard]] Cursor range_cursor(std::optional<Slice> lower, std::optional<Slice> upper, uint64_t frontier, + uint64_t floor = 0) const; + // Constant-time independently sampled logical/source capacities. These + // counters describe the current linked versions, not physical allocation + // identity or memory retained by old readers. [[nodiscard]] VersionMemory memory() const; +#ifdef CROWDB_TREE_TEST_UTIL + [[nodiscard]] VersionMemory scan_memory_for_tests() const; +#endif [[nodiscard]] EpochManager &epoch() const { return *epoch_; } + [[nodiscard]] const std::shared_ptr<AllocationBudget> &budget() const noexcept + { + return budget_; + } + // Point lookup: returns the CellVersion* for `key`, or nullptr. The // returned pointer is valid only while the caller's epoch guard is held // (a concurrent overwrite retires the old version via epoch). @@ -258,25 +364,37 @@ class ConcurrentSkipList void add_bytes(size_t n) { bytes_.fetch_add(n, std::memory_order_relaxed); + if (source_ledger_) { + source_ledger_->replace({}, {.logical_bytes = n}); + } } void sub_bytes(size_t n) { bytes_.fetch_sub(n, std::memory_order_relaxed); + if (source_ledger_) { + source_ledger_->replace({.logical_bytes = n}, {}); + } } // Public so MemTable can pass it to epoch_.retire() as the deleter. static void free_node(void *p); private: + friend class MemTable; + bool retire_step() noexcept; + std::shared_ptr<SourceMemoryLedger> source_ledger_; + void account_versions(const VersionSet *previous, const VersionSet &next) noexcept; + void account_node(const Node &node, const VersionSet &versions) noexcept; friend class Cursor; - [[nodiscard]] static Node *alloc_node(uint32_t height, Slice key); + [[nodiscard]] static Node *alloc_node(uint32_t height, Slice key, const std::shared_ptr<AllocationBudget> &budget); static uint32_t random_height(); Node *find_or_insert(Slice key, VersionSet *versions, bool *inserted); void link_upper(Node *node); - bool replace(Node *node, std::shared_ptr<const CellVersion> cv, uint64_t bound, MutationStats *stats); + bool replace(Node *node, std::shared_ptr<const CellVersion> cv, uint64_t bound, MutationStats *stats, + const std::shared_ptr<AllocationBudget> &budget); Node *find_ge(Slice key, Node **prev, Node **successors = nullptr) const; @@ -284,12 +402,19 @@ class ConcurrentSkipList void *test_context_ = nullptr; TestHook test_hook_ = nullptr; #endif - Node *head_; - std::atomic<uint32_t> max_height_{1}; - std::atomic<size_t> count_{0}; - std::atomic<size_t> bytes_{0}; - std::unique_ptr<EpochManager> owned_epoch_; - EpochManager *epoch_; + Node *head_; + std::atomic<uint32_t> max_height_{1}; + std::atomic<size_t> count_{0}; + std::atomic<size_t> bytes_{0}; + AllocationLease epoch_credit_; + std::unique_ptr<EpochManager> owned_epoch_; + EpochManager *epoch_; + std::shared_ptr<AllocationBudget> budget_; + std::atomic<uint64_t> history_count_{0}; + std::atomic<uint64_t> history_bytes_{0}; + std::atomic<uint64_t> descriptor_bytes_{0}; + std::atomic<uint64_t> node_bytes_{0}; + std::atomic<uint64_t> payload_bytes_{0}; }; } // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/memtable/source.h b/lib/crowdb-tree/include/crowdb-tree/memtable/source.h deleted file mode 100644 index 2c654aa5e..000000000 --- a/lib/crowdb-tree/include/crowdb-tree/memtable/source.h +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. -#pragma once - -#include "crowdb-common/metrics/latency_summary.h" -#include "crowdb-tree/memtable/generation.h" -#include "crowdb-tree/memtable/memtable.h" - -namespace crowdb::tree -{ -// Coverage is captured with source membership, before any L1 head is read. -// It is not a scan timestamp: live updates above this floor remain observable. -struct MemTableSource -{ - std::shared_ptr<MemTable> owner; - uint64_t floor = 0; - std::shared_ptr<EpochManager::Guard> guard; - - MemTableSource(std::shared_ptr<MemTable> table, uint64_t covered) - : owner(std::move(table)), - floor(covered), - guard(std::make_shared<EpochManager::Guard>(owner->read_guard())) - { - } - - const MemTableSource *operator->() const - { - return this; - } - - [[nodiscard]] const CellVersion *find(Slice key) const - { - const auto *cv = owner->find(key); - return cv != nullptr && cv->slot > floor ? cv : nullptr; - } - - [[nodiscard]] ConcurrentSkipList::Cursor cursor(Slice key) const - { - return filter(owner->cursor(key)); - } - - [[nodiscard]] ConcurrentSkipList::Cursor cursor_from(Slice key, bool inclusive) const - { - return filter(owner->cursor_from(key, inclusive)); - } - - [[nodiscard]] ConcurrentSkipList::Cursor cursor_reverse(Slice key, bool bounded, bool inclusive) const - { - auto cur = owner->cursor_reverse(key, bounded, inclusive); - while (cur.valid() && cur.cell_version()->slot <= floor) { - cur = owner->cursor_reverse(cur.key(), true, false); - } - return cur; - } - - [[nodiscard]] size_t count() const - { - size_t result = 0; - for (auto cur = owner->prefix_cursor(UINT64_MAX, floor); cur.valid(); cur.advance()) { - ++result; - } - return result; - } - - [[nodiscard]] size_t approx_bytes() const - { - return owner->approx_bytes(); - } - - private: - [[nodiscard]] ConcurrentSkipList::Cursor filter(ConcurrentSkipList::Cursor cur) const - { - cur.set_floor(floor); - return cur; - } -}; - -struct MemTableCounters -{ - std::atomic<uint64_t> cas_retries{0}; - std::atomic<uint64_t> overwrite{0}; - std::atomic<uint64_t> keep{0}; - std::atomic<uint64_t> merged{0}; - std::atomic<uint64_t> failed_batches{0}; - std::atomic<uint64_t> keep_gap{0}; - std::atomic<uint64_t> keep_pending{0}; - std::atomic<uint64_t> keep_boundary{0}; -}; - -struct MemTableBatch -{ - std::shared_ptr<MemTable> table; - uint64_t bound; - MutationStats stats; - GenerationGate::Guard generation; - MemTableCounters *counters; - crowdb::common::metrics::LatencySummary *copy_latency = nullptr; - bool completed = false; - - ~MemTableBatch() - { - if (copy_latency != nullptr && stats.copy_count != 0) { - copy_latency->observe_batch(stats.copy_count, stats.copy_ns, stats.copy_max_ns); - } - if (stats.cas_retries != 0) { - counters->cas_retries.fetch_add(stats.cas_retries); - } - if (stats.overwrite != 0) { - counters->overwrite.fetch_add(stats.overwrite); - } - if (stats.keep != 0) { - counters->keep.fetch_add(stats.keep); - counters->keep_gap.fetch_add(stats.keep_gap); - counters->keep_pending.fetch_add(stats.keep_pending); - counters->keep_boundary.fetch_add(stats.keep_boundary); - } - if (stats.merged != 0) { - counters->merged.fetch_add(stats.merged); - } - if (!completed) { - counters->failed_batches.fetch_add(1); - } - table->leave(); - } -}; -} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/memtable/source_stats.h b/lib/crowdb-tree/include/crowdb-tree/memtable/source_stats.h new file mode 100644 index 000000000..b450209f3 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/memtable/source_stats.h @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" + +#include <atomic> +#include <cstdint> + +namespace crowdb::tree +{ +struct VersionMemory +{ + uint64_t history_count = 0; + uint64_t history_bytes = 0; + uint64_t descriptor_bytes = 0; + uint64_t node_bytes = 0; + uint64_t payload_bytes = 0; + bool operator==(const VersionMemory &) const = default; +}; + +// These are current source membership and linked-version summaries, not +// unique physical allocation bytes or retention by external views. +struct SourceMemoryStats +{ + uint64_t tables = 0; + uint64_t frozen_tables = 0; + uint64_t writers = 0; + uint64_t freezing_writers = 0; + uint64_t frozen_retained_bytes = 0; + uint64_t records = 0; + uint64_t logical_bytes = 0; + VersionMemory memory{}; + bool known = false; + + SourceMemoryStats &operator+=(const SourceMemoryStats &other) noexcept; + bool operator==(const SourceMemoryStats &) const = default; +}; + +// One ledger per source admission generation. Mutation deltas commute and +// independent samples converge after writers drain. No source scan is needed. +class SourceMemoryLedger +{ + public: + [[nodiscard]] SourceMemoryStats snapshot() const noexcept; + void replace(const SourceMemoryStats &before, const SourceMemoryStats &after) noexcept; + void writer_enter() noexcept; + void writer_leave(bool closed) noexcept; + void table_closed(uint64_t writers) noexcept; + void table_frozen(uint64_t bytes) noexcept; + + private: + std::atomic<uint64_t> tables_{0}; + std::atomic<uint64_t> frozen_tables_{0}; + std::atomic<uint64_t> writers_{0}; + std::atomic<uint64_t> freezing_writers_{0}; + std::atomic<uint64_t> frozen_retained_bytes_{0}; + std::atomic<uint64_t> records_{0}; + std::atomic<uint64_t> logical_bytes_{0}; + std::atomic<uint64_t> history_count_{0}; + std::atomic<uint64_t> history_bytes_{0}; + std::atomic<uint64_t> descriptor_bytes_{0}; + std::atomic<uint64_t> node_bytes_{0}; + std::atomic<uint64_t> payload_bytes_{0}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/owned_allocation_c_api.h b/lib/crowdb-tree/include/crowdb-tree/owned_allocation_c_api.h new file mode 100644 index 000000000..b705d360d --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/owned_allocation_c_api.h @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ct_owned_allocation ct_owned_allocation; + +// Transfer a complete immutable allocation. Capacity includes spare bytes; +// length is the initialized prefix. The context is consumed on every return, +// including failure, and drop_context runs exactly once. drop_context must +// be nonnull. control_bytes is the +// foreign owner/control allocation size, independently charged as metadata. +// Requires a runtime Tree. Retained owners do not retain its CPU executor. +ct_status ct_owned_allocation_create(ct_tree *tree, const uint8_t *data, size_t length, size_t capacity, + void (*drop_context)(void *), void *context, size_t control_bytes, + ct_owned_allocation **out); +void ct_owned_allocation_retain(ct_owned_allocation *owner); +void ct_owned_allocation_free(ct_owned_allocation *owner); + +typedef struct ct_owned_kv_ref +{ + const uint8_t *key; + size_t key_len; + ct_owned_allocation *owner; + size_t offset; + size_t length; + uint8_t kind; // 0 put, 1 delete +} ct_owned_kv_ref; + +// Borrow owners for the synchronous call. Same-host values retain the shared +// allocation identity; different hosts copy into target-admitted owned cells. +// Validation/admission failure never publishes a partial batch. +ct_status ct_apply_batch_owned(ct_tree *tree, uint64_t slot, const ct_owned_kv_ref *operations, size_t count); + +// The callback borrows stable immutable descriptors until return. It is called +// synchronously, may be called repeatedly for an index, and must not throw. +// This avoids an unaccounted temporary descriptor array in language adapters. +typedef ct_owned_kv_ref (*ct_owned_read_fn)(const void *context, size_t index); +ct_status ct_apply_batch_owned_reader(ct_tree *tree, uint64_t slot, size_t count, ct_owned_read_fn read, + const void *context); + +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/owned_budget_c_api.h b/lib/crowdb-tree/include/crowdb-tree/owned_budget_c_api.h new file mode 100644 index 000000000..21e24ed9f --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/owned_budget_c_api.h @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#ifdef __cplusplus +# include <cstddef> +# include <cstdint> +#else +# include <stddef.h> +# include <stdint.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif +typedef struct ct_owned_budget ct_owned_budget; +typedef struct ct_future ct_future; +typedef struct ct_scan_result ct_scan_result; +typedef struct ct_snapshot_chunk ct_snapshot_chunk; + +// Retains the paid control-allocation window of an immutable returned owner. +// Legacy owners return success with *out=nullptr (caller-owned allocation). +int32_t ct_future_owned_budget(const ct_future *future, ct_owned_budget **out); +int32_t ct_scan_result_owned_budget(const ct_scan_result *result, ct_owned_budget **out); +int32_t ct_snapshot_chunk_owned_budget(const ct_snapshot_chunk *chunk, ct_owned_budget **out); + +// Claim before allocation, record after successful allocation, then record +// freed and release only after actual deallocation. Admission refusal acquires +// nothing. The owner must pair every successful claim with one release. +int32_t ct_owned_budget_claim(ct_owned_budget *budget, size_t bytes); +void ct_owned_budget_record(ct_owned_budget *budget, size_t bytes, int32_t acquired); +void ct_owned_budget_release(ct_owned_budget *budget, size_t bytes); +void ct_owned_budget_free(ct_owned_budget *budget); +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/resource_usage.h b/lib/crowdb-tree/include/crowdb-tree/resource_usage.h new file mode 100644 index 000000000..813c79274 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/resource_usage.h @@ -0,0 +1,27 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <array> +#include <cstdint> + +namespace crowdb::tree +{ +// Six budget partitions, CPU/backend queue permits, then host CPU/backend +// workers. Tree samples omit the last two shared resources. Queue permits +// include idle prepaid progress registrations, unlike runnable queue length. +struct ResourceUsage +{ + uint64_t limit = 0; + uint64_t used = 0; + uint64_t reserved = 0; + uint64_t allocated = 0; + uint64_t denied_tree = 0; + uint64_t denied_host = 0; + uint64_t waits = 0; + uint64_t wait_ns = 0; + uint64_t waiters = 0; +}; + +using ResourceUsageSnapshot = std::array<ResourceUsage, 10>; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/runtime_c_api.h b/lib/crowdb-tree/include/crowdb-tree/runtime_c_api.h new file mode 100644 index 000000000..716ebdf29 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/runtime_c_api.h @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <cstddef> +#include <cstdint> + +extern "C" { + +using ct_runtime = struct ct_runtime; +using ct_tree = struct ct_tree; + +struct ct_resource_usage_v1 +{ + uint64_t limit, used, reserved, allocated; + uint64_t denied_tree, denied_host, waits, wait_ns, waiters; +}; + +// Non-draining relaxed samples. Set bits identify supported resource indices: +// handoff, routing, reclaim, writer, gap_repair, io, cpu_queue_slots, +// backend_queue_slots, cpu_workers, backend_workers, cache_bytes, cache_pages, +// cache_request_slots, backend_operation_slots, apply_operation_slots, +// replay_operation_slots, source_entry_slots, source_key_bytes, +// gap_source_entry_slots, gap_source_key_bytes. +// Hosts expose 0..9; Trees expose 0..7 and 10..19. Source rows describe the +// active source and include admitted writers; their contention counters are +// not measured. Operation rows include accepted owners through callback tails. +// Partitions 0..4, cache_bytes and source key capacities use bytes; the others +// use counts. Queue and +// cache request slots include prepaid registrations even while idle. +struct ct_resource_usage_snapshot_v1 +{ + size_t struct_size; + uint32_t version; + uint32_t resource_mask; + uint64_t scope_id; + ct_resource_usage_v1 resources[20]; +}; + +int32_t ct_runtime_resource_usage(const ct_runtime *runtime, ct_resource_usage_snapshot_v1 *out); +int32_t ct_tree_resource_usage(ct_tree *tree, ct_resource_usage_snapshot_v1 *out); + +struct ct_resource_limits +{ + size_t handoff_bytes; + size_t routing_bytes; + size_t reclaim_bytes; + size_t writer_bytes; + size_t gap_repair_bytes; + size_t io_operations; +}; + +// Version 1 requires struct_size == sizeof(ct_runtime_options). A null options +// pointer selects defaults; supplied numeric values are used without defaults. +struct ct_runtime_options +{ + size_t struct_size; + size_t workers; + size_t cpu_queue_limit; + size_t max_waiters_per_class; + size_t lifecycle_bytes; + ct_resource_limits limits; + size_t backend_workers; + size_t backend_queue_limit; +}; + +struct ct_tree_runtime_options +{ + size_t struct_size; + size_t cpu_queue_limit; + size_t lifecycle_bytes; + ct_resource_limits limits; + size_t backend_queue_limit; +}; + +struct ct_executor_snapshot_v1 +{ + uint64_t queued; + uint64_t inflight; + uint64_t submitted; + uint64_t completed; + uint64_t rejected; + uint64_t queue_ns; + uint64_t execute_ns; +}; + +struct ct_runtime_snapshot_v1 +{ + size_t struct_size; + uint32_t version; + uint32_t stopped; + ct_resource_limits reserved; + ct_resource_limits allocated; + uint64_t bindings; + ct_executor_snapshot_v1 cpu; + ct_executor_snapshot_v1 backend; + uint64_t budget_waiters; + uint64_t budget_grants; + uint64_t budget_cancelled; + uint64_t budget_wait_ns; +}; + +// Requires exact struct_size and version 1. Does not drain counters, allocate, +// submit work or perform backend I/O; invalid inputs leave output unchanged. +int32_t ct_runtime_snapshot(const ct_runtime *runtime, ct_runtime_snapshot_v1 *out); + +void ct_runtime_options_default(ct_runtime_options *out); +void ct_tree_runtime_options_default(ct_tree_runtime_options *out); +int32_t ct_runtime_create(const ct_runtime_options *options, ct_runtime **out); +void ct_runtime_stop_admission(ct_runtime *runtime); +// Host blocking boundary after all Tree operations and backend tickets end. +// This closes CPU submission and waits for CPU jobs only, not external I/O. +void ct_runtime_drain_cpu(ct_runtime *runtime); +// Release the caller's reference. Live Trees/operations retain their owners; +// this call does not wait for them or stop their admission. +void ct_runtime_free(ct_runtime *runtime); +} diff --git a/lib/crowdb-tree/include/crowdb-tree/snapshot/native_frame.h b/lib/crowdb-tree/include/crowdb-tree/snapshot/native_frame.h index 57777303f..d4be9c40d 100644 --- a/lib/crowdb-tree/include/crowdb-tree/snapshot/native_frame.h +++ b/lib/crowdb-tree/include/crowdb-tree/snapshot/native_frame.h @@ -3,9 +3,12 @@ #pragma once #include "crowdb-tree/maptable/page.h" +#include "crowdb-tree/memory_budget.h" #include "crowdb-tree/status.h" #include <memory> +#include <optional> +#include <span> #include <vector> namespace crowdb::tree @@ -17,18 +20,43 @@ namespace crowdb::tree // overflow page round-trips as one `memcpy`-equivalent copy. struct NativeFrame { - uint64_t page_id = kInvalidPageId; - std::vector<uint8_t> frame; // raw in-memory frame bytes (page_bytes() length) - uint64_t durable_addr = kNoAddr; - uint32_t durable_plen = 0; - bool inherited = false; + uint64_t page_id = kInvalidPageId; + // Runtime compatibility collectors account the original copied payload. + // Declared before frame so its storage is freed before returning credit. + // The outer vector and any caller copies or mutations are caller-owned. + std::shared_ptr<const void> allocation_owner = nullptr; + std::vector<uint8_t> frame; // raw in-memory frame bytes (page_bytes() length) + uint64_t durable_addr = kNoAddr; + uint32_t durable_plen = 0; + bool inherited = false; }; -// Resumable view over one native tree generation. Creation folds pending leaf -// deltas, then pins only the root. Each next() call advances a bounded DFS -// batch; concurrent writers preserve overwritten pre-generation pages under a -// fixed pin budget, and consumed pins are released immediately. Different -// cursors share no mutable traversal state. +// Immutable output window. The owner accounts for and retains the byte +// capacity; copies share bytes, and the next window never overwrites them. +struct NativeFrameView +{ + uint64_t page_id = kInvalidPageId; + std::span<const uint8_t> bytes; + std::shared_ptr<const void> owner; + uint64_t durable_addr = kNoAddr; + uint32_t durable_plen = 0; + bool inherited = false; +}; + +struct NativeStreamInfo +{ + uint64_t root_page_id = kInvalidPageId; + uint64_t at_slot = 0; + uint64_t next_page_id = 0; + uint64_t frame_count = 0; + bool frame_count_known = false; +}; + +// Resumable view over one native tree generation. Runtime cursors capture an +// immutable mapping, reserve their overflow PIDs from the shared authority, +// and emit one independently owned frame window at a time. +class PublishedNativeCursor; + class NativeFrameIterator { public: @@ -39,17 +67,19 @@ class NativeFrameIterator NativeFrameIterator(NativeFrameIterator &&) noexcept; NativeFrameIterator &operator=(NativeFrameIterator &&) noexcept; - Status next(size_t max_frames, std::vector<NativeFrame> *out, bool *complete); - [[nodiscard]] uint64_t root_page_id() const; - [[nodiscard]] uint64_t at_slot() const; - [[nodiscard]] uint64_t next_page_id() const; - [[nodiscard]] uint64_t subtrees_skipped() const; + Status next(size_t max_frames, std::vector<NativeFrame> *out, bool *complete); + Status next_owned(std::optional<NativeFrameView> *out, bool *complete); + [[nodiscard]] NativeStreamInfo stream_info() const; + [[nodiscard]] uint64_t root_page_id() const; + [[nodiscard]] uint64_t at_slot() const; + [[nodiscard]] uint64_t next_page_id() const; + [[nodiscard]] uint64_t subtrees_skipped() const; private: - struct Impl; - explicit NativeFrameIterator(std::shared_ptr<Impl> impl); + explicit NativeFrameIterator(std::shared_ptr<PublishedNativeCursor> runtime); - std::shared_ptr<Impl> impl_; + std::shared_ptr<PublishedNativeCursor> runtime_; + AllocationLease facade_credit_; friend class Crowdbtree; }; diff --git a/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h b/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h deleted file mode 100644 index 64d62ef00..000000000 --- a/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. -#pragma once - -#include "crowdb-tree/maptable/mapping_table.h" - -#include <cstdint> -#include <set> -#include <vector> - -namespace crowdb::tree -{ -// One durable blob to write at a fixed offset, computed ahead of time by -// prepare_snapshot_locked() (persist.cpp) so the actual store->write_at()/ -// submit_write() call is a pure I/O op with no further encoding logic -- -// shared by snapshot()'s synchronous writes and snapshot_async()'s async -// ones. -struct PreparedSnapshotWrite -{ - uint64_t addr = 0; - std::vector<uint8_t> blob; // already IU-padded -}; - -// A page write plus enough identity to safely mark the *live* page durable -// once the write actually lands (see prepare_snapshot_locked's doc comment -// on why this can't happen eagerly at prepare time for the async path). -// `page` is never dereferenced except as an opaque identity check under -// write_mutex_ (mapping_.get_resident(page_id) == page) -- it may have been retired -// and its frame reused by the time the write completes (a concurrent -// consolidate/flush/split replaced this page_id's mapping entry with a -// fresh COW page in the meantime), in which case the identity check simply -// fails and this write's durable-bookkeeping is skipped (harmless: the old -// blob is still correctly on disk and referenced by *this* generation's -// segment image; the fresh page is independently dirty and picked up by -// the next snapshot). -struct PreparedPageWrite -{ - uint64_t page_id = 0; - PageBase *page = nullptr; // opaque identity only - uint64_t prior_addr = kNoAddr; - uint64_t addr = 0; - uint32_t logical_len = 0; // unpadded; mirrors PageBase::durable_plen - std::vector<uint8_t> blob; // padded to IU at submission; released after completion -}; - -// A dirty MappingSegment's fresh image write, plus enough identity to -// safely mark it durable at commit time (mirrors PreparedPageWrite's -// identity-check pattern, extended with `seen_write_seq` -- see -// MappingSegment's doc comment on why a segment needs a seq check, not just -// a pointer identity check: unlike a page, whose whole *pointer* is -// replaced on any change, a segment's pointer stays the same across a -// slot mutation, so identity alone can't detect "written again during the -// prepare-to-commit gap"). -struct PreparedSegmentWrite -{ - uint64_t seg_idx = 0; - MappingSegment *seg = nullptr; // opaque identity only - uint64_t seen_write_seq = 0; - uint64_t new_generation = 0; - uint64_t addr = 0; - uint32_t logical_len = 0; // unpadded - uint32_t image_crc = 0; // body-only CRC, matches the directory entry prepare wrote - std::vector<uint8_t> blob; // already IU-padded -}; - -struct PreparedUnloadedRelocation -{ - uint64_t page_id = 0; - uint64_t old_word = slot_word::kEmpty; - uint64_t new_word = slot_word::kEmpty; -}; - -// A prefetched unloaded page read outside write_mutex_ (R129). The mapping -// word is revalidated under the mutex during prepare; a mismatch discards -// the prefetched blob and the page is skipped for this pass. -struct PrefetchedPage -{ - uint64_t page_id = 0; - uint64_t old_word = slot_word::kEmpty; - std::vector<uint8_t> blob; // IU-padded page content read from the store -}; - -// Output of prepare_snapshot_locked(): encoded page payloads and metadata -// bytes computed synchronously under write_mutex_ (the segment -// scan + delta-fold + page/segment-image/directory encode is CPU/memory-only -// without device padding). Page payloads gain device padding -// only at submission. The caller writes -// `page_writes` and `segment_writes` (any order/concurrency) then -// `directory_write`, then a durability barrier, then `anchor_write` -- -// writing the anchor before that barrier would violate the crash-safety -// invariant persist.cpp's header comment documents (a crash mid-snapshot -// must fall back intact to the last *committed* anchor) -- then -// commit_prepared_snapshot() to mark each page/segment durable and publish -// the new version. -struct PreparedSnapshot -{ - std::vector<PreparedPageWrite> page_writes; - std::vector<PreparedSegmentWrite> segment_writes; - std::vector<PreparedUnloadedRelocation> unloaded_relocations; - PreparedSnapshotWrite directory_write; - PreparedSnapshotWrite anchor_write; - uint64_t last_applied_slot = 0; - // Diagnostics for the "snapshot committed" log line (matches the - // pre-refactor synchronous snapshot()'s log fields exactly). - uint64_t seq = 0; - uint64_t live_page_count = 0; // live slots across every present segment - uint64_t pages_written = 0; - uint64_t segdir_len = 0; - std::set<uint32_t> empty_blocks; // block indices with zero live bytes (block compaction) - // Block compaction stats (R129). blocks_selected is the count of source - // blocks chosen for relocation this pass; pages_relocated and - // bytes_relocated count only pages that were actually moved (not clean - // pages that kept their durable address). blocks_deleted is filled by - // finalize_prepared_snapshot after the finalizer runs. - uint64_t blocks_selected = 0; - uint64_t pages_relocated = 0; - uint64_t bytes_relocated = 0; - uint64_t blocks_deleted = 0; -}; - -} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/snapshot/snapshot.h b/lib/crowdb-tree/include/crowdb-tree/snapshot/snapshot.h index b61d3d735..9e26fb9db 100644 --- a/lib/crowdb-tree/include/crowdb-tree/snapshot/snapshot.h +++ b/lib/crowdb-tree/include/crowdb-tree/snapshot/snapshot.h @@ -1,20 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// Consistent point-in-time view. A Snapshot is an immutable, key-sorted materialization of the -// L1 tree at a given slot, used for scan-at / compare / iter_all / export. -// -// NOTE (deviation): the design specifies a snapshot as a pinned immutable COW -// root (zero-copy). The in-memory core materializes the keyspace into an -// independent immutable copy instead; this is correct and keeps writes lock-free -// for latest reads, at O(N) snapshot cost. Path-copy COW is a later optimization. -// -// R6: PinnedSnapshot (below) is the zero-copy variant for the cross-thread -// handoff paths. It inherits from Snapshot so callers that hold -// shared_ptr<Snapshot> and call entries()/get()/compare() work unchanged; the -// difference is that entries() materializes lazily from pinned frames on first -// call, and the pages stay alive via per-page refcount (pin_state_ on PageBase) -// until the PinnedSnapshot is dropped — on any thread. +// Immutable snapshot interface. Runtime views retain independent mapping and backing owners. + #pragma once #include "crowdb-tree/btree/cell.h" @@ -30,7 +18,6 @@ namespace crowdb::tree { -// A difference between two snapshots (for compare / parity). struct engine_diff { std::string key; @@ -47,8 +34,6 @@ class Snapshot { } - // R6: PinnedSnapshot lazily materializes entries_ from pinned frames on - // first call. Base Snapshot already has entries_ populated in the ctor. virtual ~Snapshot() = default; [[nodiscard]] uint64_t at_slot() const @@ -56,8 +41,6 @@ class Snapshot return at_slot_; } - // All entries including tombstones, key-sorted. Virtual so PinnedSnapshot - // can lazily materialize from pinned frames on first call. [[nodiscard]] virtual const std::vector<leaf_entry> &entries() const { return entries_; @@ -89,7 +72,6 @@ class Snapshot return -1; } - // Live read: false for absent or tombstoned keys. [[nodiscard]] virtual bool get(Slice key, uint64_t *out_slot, std::string *out_value) const { int i = find(key); @@ -109,7 +91,6 @@ class Snapshot return true; } - // Structural comparison including tombstones (used by parity tests). [[nodiscard]] std::vector<engine_diff> compare(const Snapshot &other) const { std::vector<engine_diff> diffs; @@ -150,84 +131,8 @@ class Snapshot } protected: - // R6: PinnedSnapshot populates entries_ lazily; mutable so entries() can - // materialize on first call through a const method. uint64_t at_slot_; mutable std::vector<leaf_entry> entries_; }; -// R6: zero-copy pinned snapshot. Captures PageBase* pointers during the -// snapshot_view() walk (under an epoch guard), pins each via pin_state_, then -// releases the guard. The pages stay alive via refcount until this snapshot is -// dropped — on any thread. entries() materializes lazily from the pinned frames -// on first call (callers that only use get()/find()/size() never pay the O(N) -// materialization cost; callers that need the flat vector pay it once). -// -// The pinned pages include the full leaf chain (head → ... → base for each -// leaf visited — every delta node, not just the head) and any overflow pages -// whose values are referenced by leaf entries. Overflow values are assembled -// from pinned overflow frames during materialization. -class PinnedSnapshot : public Snapshot -{ - public: - // `leaf_chain_heads` is the ordered list of chain heads (one per leaf, - // left-to-right). `all_pinned_pages` is every page to pin (all chain - // nodes + overflow pages). `overflow_pages` are the overflow chain pages - // referenced by leaf entries (subset of all_pinned_pages, used by - // materialize() to assemble overflow values). - PinnedSnapshot(uint64_t at_slot, std::vector<PageBase *> leaf_chain_heads, std::vector<PageBase *> all_pinned_pages, - std::vector<PageBase *> overflow_pages) - : Snapshot(at_slot, {}), - leaf_chain_heads_(std::move(leaf_chain_heads)), - all_pinned_pages_(std::move(all_pinned_pages)), - overflow_pages_(std::move(overflow_pages)) - { - for (PageBase *p : all_pinned_pages_) { - p->pin(); - } - } - - ~PinnedSnapshot() override - { - for (PageBase *p : all_pinned_pages_) { - p->unpin(); - } - } - - PinnedSnapshot(const PinnedSnapshot &) = delete; - PinnedSnapshot &operator=(const PinnedSnapshot &) = delete; - PinnedSnapshot(PinnedSnapshot &&) = delete; - PinnedSnapshot &operator=(PinnedSnapshot &&) = delete; - - // Lazily materialize entries_ from the pinned leaf chain frames on first - // call. Walks the captured leaf chain heads (in order), resolves each - // chain via resolve_chain_sorted, and assembles overflow values from the - // pinned overflow pages. Subsequent calls return the cached vector. - [[nodiscard]] const std::vector<leaf_entry> &entries() const override - { - if (entries_.empty() && !leaf_chain_heads_.empty()) { - materialize(); - } - return entries_; - } - - [[nodiscard]] size_t size() const override - { - return entries().size(); - } - - private: - // The ordered list of leaf chain heads (one per leaf, left-to-right). - // materialize() calls resolve_chain_sorted on each, following head->next - // (the delta chain) which is kept alive by all_pinned_pages_. - std::vector<PageBase *> leaf_chain_heads_; - // Every page to pin/unpin (all chain nodes + overflow pages). - std::vector<PageBase *> all_pinned_pages_; - // Overflow chain pages (subset of all_pinned_pages_), used by materialize() - // to assemble overflow values from pinned frames. - std::vector<PageBase *> overflow_pages_; - - void materialize() const; -}; - -} // namespace crowdb::tree +} // namespace crowdb::tree \ No newline at end of file diff --git a/lib/crowdb-tree/include/crowdb-tree/snapshot/snapshot_io.h b/lib/crowdb-tree/include/crowdb-tree/snapshot/snapshot_io.h index b1f6e9a6a..aedd31b09 100644 --- a/lib/crowdb-tree/include/crowdb-tree/snapshot/snapshot_io.h +++ b/lib/crowdb-tree/include/crowdb-tree/snapshot/snapshot_io.h @@ -1,27 +1,17 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// Snapshot export / import. -// -// The streaming byte form is the primitive: it feeds the network snapshot -// service for new-member install without ever touching local disk. Dumping to a -// `.ctsnap` file is just streaming into a file writer. -// -// v1 ships the **portable** format only: a versioned header, then key-sorted -// `(klen,key,slot,kind,vlen,value)` tuples (including tombstones), then a -// whole-stream CRC32C. It is deterministic and engine-independent, so an export -// re-imports identically on any backend / page size and is comparable against -// the in-mem oracle. The native frame-dump format is deferred. -// -// Key work: portable stream encode/decode, chunked export, CRC-checked import, -// file convenience wrappers. +// Bounded portable tuple and native frame streams over an immutable root. +// Both formats retain owned windows and install through a private generation. #pragma once +#include "crowdb-tree/memory_budget.h" #include "crowdb-tree/slice.h" #include "crowdb-tree/status.h" #include <cstdint> #include <memory> +#include <span> #include <string> namespace crowdb::tree @@ -29,28 +19,43 @@ namespace crowdb::tree class Crowdbtree; class Snapshot; +class RuntimeSnapshotImport; +class PublishedByteExport; enum class snapshot_format : uint8_t { - kPortable = 0, // v1 default: portable tuple stream - kNative = 1, // deferred: raw frame images + remapped manifest + kPortable = 0, // versioned logical tuple stream + kNative = 1, // raw frame images and an immutable root descriptor }; -// default_env export chunk size (fixed ≤1 MiB chunks). +// Default export chunk size (fixed ≤1 MiB chunks). inline constexpr size_t kSnapshotChunkBytes = 1U << 20; +// Per-window credit reserved for one foreign-language owner handle. +inline constexpr size_t kSnapshotChunkHandleBytes = 1024; -// A resumable, chunked export over an immutable point-in-time view. The whole -// stream is materialized once at begin (deterministic), then sliced into chunks -// of at most `chunk_bytes`, so chunk boundaries are stable across exports. +// A retained engine-owned chunk. The capacity stays charged until the last +// copy of owner is released, independently of the stream or Tree lifetime. +struct SnapshotChunk +{ + std::span<const uint8_t> bytes; + std::shared_ptr<const void> owner; + std::shared_ptr<AllocationBudget> allocation_budget = nullptr; +}; + +// A resumable export over an immutable point-in-time view. Runtime sources +// use bounded owned chunks and retain no whole serialized stream. A bounded +// preflight computes exact length and CRC before the first emitted chunk. class SnapshotExport { public: - // Portable export: retain the immutable logical view and encode it + // Compatibility adapter for an already materialized caller-owned view. + // Retain the immutable logical view and encode it // incrementally. The constructor performs a metadata-only pass for the // total length and CRC, but never materializes the serialized stream. SnapshotExport(std::shared_ptr<Snapshot> snapshot, size_t chunk_bytes); + explicit SnapshotExport(std::shared_ptr<PublishedByteExport> runtime); - // Native export still uses an owned encoded stream. Native incremental - // frame export is separate work from the portable install protocol. + // Compatibility adapter for an already encoded caller-owned stream. + // Tree exports use the bounded PublishedByteExport constructor. SnapshotExport(std::string stream, size_t chunk_bytes, uint64_t at_slot) : stream_(std::move(stream)), chunk_bytes_(chunk_bytes == 0 ? kSnapshotChunkBytes : chunk_bytes), @@ -63,6 +68,8 @@ class SnapshotExport // final chunk has been returned (an empty stream returns one empty final // chunk). Status next_chunk(std::string *out, bool *done); + // Independently owned capacity remains accounted across chunk copies. + Status next_owned(SnapshotChunk *out, bool *done); [[nodiscard]] uint64_t at_slot() const { @@ -92,30 +99,29 @@ class SnapshotExport private: Status next_portable_chunk(std::string *out, bool *done); - std::shared_ptr<Snapshot> snapshot_; - std::string stream_; - size_t pos_ = 0; - size_t chunk_bytes_; - uint64_t at_slot_; - size_t total_bytes_ = 0; - uint32_t final_crc32c_ = 0; - size_t piece_index_ = 0; - size_t piece_offset_ = 0; + std::shared_ptr<Snapshot> snapshot_; + std::string stream_; + size_t pos_ = 0; + size_t chunk_bytes_; + uint64_t at_slot_; + size_t total_bytes_ = 0; + uint32_t final_crc32c_ = 0; + size_t piece_index_ = 0; + size_t piece_offset_ = 0; + std::shared_ptr<PublishedByteExport> runtime_; + AllocationLease facade_credit_; }; -// Begin a snapshot export of `tree`: exports the current durable view (at the -// engine's last_applied_slot, recorded in the stream header). Both kPortable -// and kNative (plan-tree #16) are supported. (Historical/arbitrary-slot -// export is deferred until path-copy COW RootVersions exist; there is no -// slot selector.) +// Capture the published materialized frontier. Both formats preserve that +// fixed root while subsequent writes and checkpoints proceed. Status snapshot_export_begin(Crowdbtree &tree, snapshot_format fmt, size_t chunk_bytes, std::unique_ptr<SnapshotExport> *out); // Convenience: dump a whole snapshot to a `.ctsnap` file (loops next_chunk). Status snapshot_dump_to_file(Crowdbtree &tree, snapshot_format fmt, const std::string &path); -// Accumulates a portable stream and atomically installs it into `tree` on -// finish (verifying the whole-stream CRC first). +// Stage bounded native frames or portable tuples and atomically install the +// validated generation on finish, after checking the whole-stream CRC. class SnapshotImport { public: @@ -135,14 +141,8 @@ class SnapshotImport Status finish(uint64_t *out_at_slot); private: - struct PortableState; - - Status finish_native(const uint8_t *p, size_t len, uint64_t *out_at_slot); - Crowdbtree &tree_; - std::unique_ptr<PortableState> portable_; - std::string buf_; - bool format_selected_ = false; - bool native_ = false; + std::shared_ptr<RuntimeSnapshotImport> runtime_; + Crowdbtree &tree_; }; // Convenience: load a `.ctsnap` file into `tree`. diff --git a/lib/crowdb-tree/include/crowdb-tree/split_overlay_c_api.h b/lib/crowdb-tree/include/crowdb-tree/split_overlay_c_api.h new file mode 100644 index 000000000..68e0adaed --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/split_overlay_c_api.h @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ct_prepared_split_overlay ct_prepared_split_overlay; + +// Both v1 structures require version == 1 and struct_size exactly equal to +// sizeof(the v1 structure). Rejected calls leave caller outputs unchanged. +typedef struct ct_split_overlay_limits_v1 +{ + uint32_t version; + uint32_t struct_size; + uint64_t max_records; + uint64_t max_bytes; +} ct_split_overlay_limits_v1; + +typedef struct ct_installed_split_overlay_v1 +{ + uint32_t version; + uint32_t struct_size; + uint64_t generation; + uint64_t frontier; +} ct_installed_split_overlay_v1; + +// Background-only preparation. The destination's durable checkpoint is not +// changed. The returned capability exclusively owns its next installation. +ct_status ct_arm_split_overlay(ct_tree *source, ct_tree *destination, const ct_split_overlay_limits_v1 *limits, + ct_prepared_split_overlay **out, uint64_t *start_frontier); +// A is the pre-registration write high-watermark, including out-of-order +// slots. Replay the private child to a fixed P >= A without +// changing its durable checkpoint, then complete this background preparation. +ct_status ct_prepared_split_overlay_prepare(ct_prepared_split_overlay *prepared, uint64_t replayed_prefix); + +// No allocation, I/O, memtable drain, or executor wait. Repeating the committed +// frontier is idempotent; a different frontier never extends an installed +// token. +ct_status ct_prepared_split_overlay_try_install(ct_prepared_split_overlay *prepared, uint64_t frontier, + ct_installed_split_overlay_v1 *out); + +// A nonzero return means this token has committed; out then receives its fixed +// frontier. This query neither advances the token nor pins additional storage. +int ct_prepared_split_overlay_committed_frontier(const ct_prepared_split_overlay *prepared, uint64_t *out); + +// Host-only destruction releases old controls and unused admission credit. +// Cancels an uninstalled capability. Installed children retain independent +// owners; releasing this handle does not remove their inherited prefix. +void ct_prepared_split_overlay_destroy(ct_prepared_split_overlay *prepared); + +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/status.h b/lib/crowdb-tree/include/crowdb-tree/status.h index 1d2be710e..5299d30eb 100644 --- a/lib/crowdb-tree/include/crowdb-tree/status.h +++ b/lib/crowdb-tree/include/crowdb-tree/status.h @@ -85,6 +85,11 @@ class Status return code_; } + [[nodiscard]] Status without_message() const noexcept + { + return {code_, {}}; + } + [[nodiscard]] const std::string &message() const { return msg_; diff --git a/lib/crowdb-tree/include/crowdb-tree/tree_execution_options.h b/lib/crowdb-tree/include/crowdb-tree/tree_execution_options.h new file mode 100644 index 000000000..8ed641e82 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/tree_execution_options.h @@ -0,0 +1,30 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <cstddef> + +namespace crowdb::tree +{ +// Work and allocation windows, independent of durable frame geometry. +struct TreeExecutionOptions +{ + size_t source_max_entries = 4096; + size_t source_max_key_bytes = 1024 * 1024; + size_t source_max_gap_entries = 64; + size_t source_max_gap_key_bytes = 128 * 1024; + // Zero derives min(CPU workers, per-tree I/O permits). + size_t flush_parallelism = 0; + size_t run_max_entries = 128; + size_t run_max_key_bytes = 256 * 1024; + size_t run_key_block_bytes = 1024; + size_t run_max_bytes = 1024 * 1024; + size_t install_run_window = 64; + size_t maintenance_visits_per_step = 64; + size_t maintenance_max_visits = 4096; + size_t maintenance_max_candidates = 256; + size_t maintenance_max_steps = 1024; + size_t maintenance_metadata_bytes = 1024 * 1024; + size_t cache_max_pages = 256; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/tree_runtime.h b/lib/crowdb-tree/include/crowdb-tree/tree_runtime.h new file mode 100644 index 000000000..48d9ee74d --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/tree_runtime.h @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/resource_usage.h" +#include "crowdb-tree/status.h" + +#include <cstddef> +#include <cstdint> +#include <memory> + +namespace crowdb::tree +{ +struct TreeResourceLimits +{ + size_t handoff_bytes = 256ULL * 1024 * 1024; + size_t routing_bytes = 512ULL * 1024 * 1024; + size_t reclaim_bytes = 128ULL * 1024 * 1024; + size_t writer_bytes = 512ULL * 1024 * 1024; + size_t gap_repair_bytes = 64ULL * 1024 * 1024; + size_t io_operations = 128; +}; + +struct TreeRuntimeOptions +{ + size_t workers = 4; + size_t cpu_queue_limit = 256; + size_t max_waiters_per_class = 256; + size_t backend_workers = 2; + size_t backend_queue_limit = 256; + // Retained routing reserve for the CPU queue and host-owned descriptors. + size_t lifecycle_bytes = 4ULL * 1024 * 1024; + TreeResourceLimits limits{}; +}; + +struct TreeBindingOptions +{ + size_t cpu_queue_limit = 64; + size_t backend_queue_limit = 64; + // Retained per-tree routing reserve for its lane and core metadata. + size_t lifecycle_bytes = 1024ULL * 1024; + TreeResourceLimits limits{.handoff_bytes = 64ULL * 1024 * 1024, + .routing_bytes = 256ULL * 1024 * 1024, + .reclaim_bytes = 64ULL * 1024 * 1024, + .writer_bytes = 128ULL * 1024 * 1024, + .gap_repair_bytes = 16ULL * 1024 * 1024, + .io_operations = 16}; +}; + +struct TreeExecutorSnapshot +{ + uint64_t queued = 0; + uint64_t inflight = 0; + uint64_t submitted = 0; + uint64_t completed = 0; + uint64_t rejected = 0; + uint64_t queue_ns = 0; + uint64_t execute_ns = 0; +}; + +// Cumulative relaxed-atomic samples. CPU and blocking-backend durations are +// separate; concurrent worker durations are not a host wall-clock interval. +struct TreeRuntimeSnapshot +{ + TreeResourceLimits reserved{.handoff_bytes = 0, + .routing_bytes = 0, + .reclaim_bytes = 0, + .writer_bytes = 0, + .gap_repair_bytes = 0, + .io_operations = 0}; + TreeResourceLimits allocated{.handoff_bytes = 0, + .routing_bytes = 0, + .reclaim_bytes = 0, + .writer_bytes = 0, + .gap_repair_bytes = 0, + .io_operations = 0}; + uint64_t bindings = 0; + uint64_t queued = 0; + uint64_t inflight = 0; + bool stopped = false; + TreeExecutorSnapshot cpu{}; + TreeExecutorSnapshot backend{}; + uint64_t budget_waiters = 0; + uint64_t budget_grants = 0; + uint64_t budget_cancelled = 0; + uint64_t budget_wait_ns = 0; +}; + +class TreeRuntimeBinding; +struct TreeRuntimeAccess; + +// A host shares CPU and blocking-backend pools and a resource ledger across its +// Trees. Tree operations retain a binding until their CPU work and backend +// callbacks end. Page caches remain owned by their individual publication +// coordinators. +class TreeRuntime +{ + struct State; + + public: + static Status create(TreeRuntimeOptions options, std::shared_ptr<TreeRuntime> *out); + ~TreeRuntime(); + TreeRuntime(const TreeRuntime &) = delete; + TreeRuntime &operator=(const TreeRuntime &) = delete; + + // Reject new bindings/reservations and stop budget waiters. Existing + // operations retain their allocation credit and can still use the CPU. + void stop_admission() noexcept; + // Host-only blocking boundary. Call after Tree operations and backend + // tickets have drained. This closes CPU submission and waits for CPU jobs; + // it does not wait for or cancel external I/O on the caller's behalf. + void drain_cpu() noexcept; + + [[nodiscard]] bool stopped() const noexcept; + [[nodiscard]] TreeRuntimeSnapshot snapshot() const noexcept; + [[nodiscard]] ResourceUsageSnapshot resource_usage() const noexcept; + [[nodiscard]] const TreeRuntimeOptions &options() const noexcept; + [[nodiscard]] Status validate_binding(const TreeBindingOptions &options) const noexcept; + + private: + friend class TreeRuntimeBinding; + friend struct TreeRuntimeAccess; + + explicit TreeRuntime(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/include/crowdb-tree/tree_stats_c_api.h b/lib/crowdb-tree/include/crowdb-tree/tree_stats_c_api.h new file mode 100644 index 000000000..87d430175 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/tree_stats_c_api.h @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include <cstddef> +#include <cstdint> + +extern "C" { +struct ct_tree; + +// Every logical field belongs to the same immutable publication. Unknown +// page summaries are explicit; estimates must not replace exact subtotals. +struct ct_tree_logical_stats_v1 +{ + uint64_t leaf_pages; + uint64_t inner_pages; + uint64_t height; + uint64_t dirty_pages; + uint64_t overlay_pages; + uint64_t overlay_runs; + uint64_t overlay_entries; + uint64_t page_size_known_bytes; + uint64_t page_size_unknown_pages; + uint64_t page_size_estimate_bytes; + uint64_t page_size_max_observed_bytes; + uint64_t kv_known_count; + uint64_t kv_known_bytes; + uint64_t kv_unknown_pages; + uint64_t consolidate_candidate_pages; + uint64_t split_candidate_pages; + uint64_t split_ready_pages; + uint64_t merge_candidate_pages; + uint64_t merge_candidate_pairs; + uint64_t merge_ready_pairs; + uint64_t writeback_candidate_pages; + uint64_t evict_candidate_pages; + uint64_t checkpoint_pending_pages; + uint64_t checkpoint_mapping_pending; +}; + +// Independent approximate samples. Referenced/pinned subsets overlap. +struct ct_tree_resource_stats_v1 +{ + uint64_t allocated_bytes; + uint64_t reserved_bytes; + uint64_t payload_bytes; + uint64_t frame_bytes; + uint64_t metadata_bytes; + uint64_t io_bytes; + uint64_t scratch_bytes; + uint64_t retired_bytes; + uint64_t resident_frames; + uint64_t cache_bytes; + uint64_t cache_pinned_bytes; + uint64_t proxy_pages; + uint64_t read_inflight_bytes; + uint64_t write_inflight_bytes; + uint64_t materialize_inflight_bytes; + uint64_t allocated_backing_bytes; + uint64_t maintenance_queued; + uint64_t maintenance_inflight; +}; + +struct ct_tree_stats_snapshot_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t instance_generation; + uint64_t commit_seq; + uint64_t sampled_at_ns; + ct_tree_logical_stats_v1 logical; + ct_tree_resource_stats_v1 resources; +}; + +// Set struct_size=sizeof(*out), version=1. Unsupported versions/sizes leave +// output unchanged. Legacy Trees return Unavailable; no mapping scan occurs. +int32_t ct_get_tree_state_stats(ct_tree *tree, ct_tree_stats_snapshot_v1 *out); + +struct ct_tree_progress_snapshot_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t instance_generation; + uint64_t commit_seq; + uint64_t sampled_at_ns; + uint64_t candidate_oldest_age_ns; + uint64_t checkpoint_oldest_pending_age_ns; + uint64_t maintenance_oldest_age_ns; +}; + +// Independent versioned extension; the existing state snapshot ABI is stable. +int32_t ct_get_tree_progress_stats(ct_tree *tree, ct_tree_progress_snapshot_v1 *out); +} diff --git a/lib/crowdb-tree/include/crowdb-tree/tree_wait_metrics_c_api.h b/lib/crowdb-tree/include/crowdb-tree/tree_wait_metrics_c_api.h new file mode 100644 index 000000000..45ded55c7 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/tree_wait_metrics_c_api.h @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#ifdef __cplusplus +# include <cstdint> +extern "C" { +#else +# include <stdint.h> +#endif +struct ct_tree; +struct ct_work_metrics_collector; + +enum { CT_TREE_WAIT_CLASSES_V1 = 7, CT_TREE_WAIT_OUTCOMES_V1 = 3 }; + +// Order: handoff, routing, reclaim, admitted writer, gap repair, I/O, +// shared-cache join. Outcomes: granted/ready, error, cancelled. +struct ct_tree_wait_summary_v1 +{ + uint64_t attempts; + uint64_t inflight; + uint64_t total_count; + uint64_t total_sum_ns; + uint64_t total_max_ns; + uint64_t outcomes[CT_TREE_WAIT_OUTCOMES_V1]; + uint64_t tracked; + uint64_t untracked; + uint64_t observed_oldest_age_ns; + uint64_t complete; +}; + +// Independent cumulative relaxed sample; never drains a collector window. +// Incomplete age coverage never rejects work. Oldest age covers tracked work +// only unless complete=1. Budget durations cover Request submit to terminal; +// host totals include other accounts and direct non-waiting reservations too. +struct ct_tree_wait_snapshot_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t sampled_at_ns; + struct ct_tree_wait_summary_v1 classes[CT_TREE_WAIT_CLASSES_V1]; +}; + +int32_t ct_tree_wait_totals(struct ct_tree *tree, struct ct_tree_wait_snapshot_v1 *out); +int32_t ct_work_metrics_collector_wait_totals(struct ct_work_metrics_collector *collector, + struct ct_tree_wait_snapshot_v1 *out); +#ifdef __cplusplus +} +#endif diff --git a/lib/crowdb-tree/include/crowdb-tree/tree_work_histograms_c_api.h b/lib/crowdb-tree/include/crowdb-tree/tree_work_histograms_c_api.h new file mode 100644 index 000000000..04f26d9ce --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/tree_work_histograms_c_api.h @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/tree_work_metrics_c_api.h" + +extern "C" { +// Nanoseconds. Quantiles use common LatencyHistogram bucket upper bounds; +// overflow quantiles are UINT64_MAX. The exact sum is (sum_high << 64)|sum_low. +struct ct_tree_histogram_v1 +{ + uint64_t count; + uint64_t sum_low; + uint64_t sum_high; + uint64_t min_ns; + uint64_t max_ns; + uint64_t p50_ns; + uint64_t p90_ns; + uint64_t p95_ns; + uint64_t p99_ns; + uint64_t p999_ns; + uint64_t zero_count; + uint64_t overflow_count; + uint64_t total_count; +}; + +struct ct_tree_work_histograms_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t window_sequence; + uint64_t sampled_at_ns; + uint64_t enabled_stages; + ct_tree_histogram_v1 stages[CT_TREE_WORK_STAGES_V1]; +}; + +// Stage follows the fixed work-v1 order. Default is disabled. Enable is +// additive/idempotent, charges real bucket storage to this Tree, and can fail +// without changing the selected set or business admission. No completion +// allocates. +int32_t ct_work_histogram_enable_v1(ct_tree *tree, uint32_t stage); +// Non-draining cumulative sample. Disabled slots are zero and absent in mask. +int32_t ct_work_histograms_totals_v1(ct_tree *tree, ct_tree_work_histograms_v1 *out); +// The same exclusive collector claim drains summaries and selected histograms +// exactly once. Fan out the immutable result; old v1/v2 layouts are unchanged. +// Invalid headers or overlapping calls leave all outputs/windows untouched. +int32_t ct_work_metrics_capture_v3(ct_work_metrics_collector *collector, ct_tree_work_snapshot_v1 *out, + ct_tree_work_ages_v1 *ages, ct_tree_work_counters_v1 *counters, + ct_tree_work_histograms_v1 *histograms); +} diff --git a/lib/crowdb-tree/include/crowdb-tree/tree_work_metrics_c_api.h b/lib/crowdb-tree/include/crowdb-tree/tree_work_metrics_c_api.h new file mode 100644 index 000000000..022546cda --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/tree_work_metrics_c_api.h @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include <cstdint> + +extern "C" { +struct ct_tree; +struct ct_work_metrics_collector; + +enum { CT_TREE_WORK_STAGES_V1 = 21, CT_TREE_WORK_OUTCOMES_V1 = 5 }; + +struct ct_tree_work_stage_v1 +{ + uint64_t attempts; + uint64_t inflight; + uint64_t total_count; + uint64_t total_sum_ns; + uint64_t total_max_ns; + uint64_t outcomes[CT_TREE_WORK_OUTCOMES_V1]; + uint64_t window_count; + uint64_t window_sum_ns; + uint64_t window_max_ns; +}; + +// Stage order is fixed by v1. Totals are non-resetting relaxed samples; +// window fields are populated only by the single collector capture. +struct ct_tree_work_snapshot_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t window_sequence; + uint64_t sampled_at_ns; + ct_tree_work_stage_v1 stages[CT_TREE_WORK_STAGES_V1]; +}; + +// Exactly one collector per runtime Tree. Legacy Trees and a second claim +// return Unavailable. The handle retains the metrics ledger after Tree close. +int32_t ct_work_metrics_collector_create(ct_tree *tree, ct_work_metrics_collector **out); +void ct_work_metrics_collector_free(ct_work_metrics_collector *collector); +// Set size/version=1. Concurrent capture returns Unavailable without draining. +// Distribute one returned value to all consumers; copying never resets samples. +int32_t ct_work_metrics_capture(ct_work_metrics_collector *collector, ct_tree_work_snapshot_v1 *out); +int32_t ct_work_metrics_totals(ct_tree *tree, ct_tree_work_snapshot_v1 *out); + +// Independent, non-draining age schema. Samples are approximate during work. +// A full diagnostic window never rejects an operation: untracked reports lost +// coverage, and observed_oldest_age_ns only covers the tracked subset. Treat it +// as the oldest active age only when complete is 1. An empty stage has age 0. +struct ct_tree_work_stage_age_v1 +{ + uint64_t inflight; + uint64_t tracked; + uint64_t untracked; + uint64_t observed_oldest_age_ns; + uint64_t complete; +}; + +struct ct_tree_work_ages_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t sampled_at_ns; + ct_tree_work_stage_age_v1 stages[CT_TREE_WORK_STAGES_V1]; +}; + +int32_t ct_work_metrics_ages(ct_tree *tree, ct_tree_work_ages_v1 *out); +// Valid after Tree close, while the collector handle remains alive. +int32_t ct_work_metrics_collector_ages(ct_work_metrics_collector *collector, ct_tree_work_ages_v1 *out); + +// Independent versioned schema; sampling never drains the stage collector. +// Counters are cumulative except active_steps, which is a relaxed gauge. +struct ct_tree_flush_dispatch_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t sampled_at_ns; + uint64_t inline_probes; + uint64_t inline_ranges; + uint64_t queued_steps; + uint64_t prepared_runs; + uint64_t prepared_entries; + uint64_t prepared_bytes; + uint64_t cow_bytes; + uint64_t active_steps; + uint64_t peak_steps; +}; + +int32_t ct_flush_dispatch_totals(ct_tree *tree, ct_tree_flush_dispatch_v1 *out); + +// Additive fixed-dimension quantities. Cause order: other, flush, cache +// pressure, checkpoint, GC, read, snapshot, allocation. No field resets. +enum { CT_TREE_WORK_COUNTERS_V1 = 55, CT_TREE_WORK_CAUSES_V1 = 8 }; + +struct ct_tree_work_counters_v1 +{ + uint32_t struct_size; + uint32_t version; + uint64_t sampled_at_ns; + uint64_t values[CT_TREE_WORK_CAUSES_V1][CT_TREE_WORK_COUNTERS_V1]; +}; + +int32_t ct_work_counters_totals(ct_tree *tree, ct_tree_work_counters_v1 *out); +int32_t ct_work_metrics_collector_counters(ct_work_metrics_collector *collector, ct_tree_work_counters_v1 *out); +// One collector claim samples all additive schemas and drains exactly one +// window. Invalid headers or concurrent capture leave every output untouched. +int32_t ct_work_metrics_capture_v2(ct_work_metrics_collector *collector, ct_tree_work_snapshot_v1 *out, + ct_tree_work_ages_v1 *ages, ct_tree_work_counters_v1 *counters); +const char *ct_work_counter_name_v1(uint32_t index); +const char *ct_work_cause_name_v1(uint32_t index); +} diff --git a/lib/crowdb-tree/include/crowdb-tree/tuning_c_api.h b/lib/crowdb-tree/include/crowdb-tree/tuning_c_api.h new file mode 100644 index 000000000..e39d99c73 --- /dev/null +++ b/lib/crowdb-tree/include/crowdb-tree/tuning_c_api.h @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include <cstddef> +#include <cstdint> + +extern "C" { +// Complete tuning values. Initialize with ct_tree_tuning_default before editing. +struct ct_tree_tuning +{ + size_t struct_size; + size_t memtable_flush_bytes; + uint32_t memtable_flush_entries; + uint32_t max_memtable_count; + size_t leaf_split_bytes; + size_t leaf_merge_bytes; + uint32_t inner_max_keys; + uint32_t inner_merge_keys; + size_t max_key_size; + size_t source_max_entries; + size_t source_max_key_bytes; + size_t source_max_gap_entries; + size_t source_max_gap_key_bytes; + size_t flush_parallelism; + size_t run_max_entries; + size_t run_max_key_bytes; + size_t run_key_block_bytes; + size_t run_max_bytes; + size_t install_run_window; + size_t maintenance_visits_per_step; + size_t maintenance_max_visits; + size_t maintenance_max_candidates; + size_t maintenance_max_steps; + size_t maintenance_metadata_bytes; + size_t cache_max_pages; +}; + +void ct_tree_tuning_default(ct_tree_tuning *out); +} diff --git a/lib/crowdb-tree/src/backend/backend_release_c_api.cpp b/lib/crowdb-tree/src/backend/backend_release_c_api.cpp new file mode 100644 index 000000000..6cacb7f17 --- /dev/null +++ b/lib/crowdb-tree/src/backend/backend_release_c_api.cpp @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/backend_release_c_api.h" + +#include "c_api_internal.h" +#include "crowdb-tree/backend/backend_release_service.h" +#include "crowdb-tree/backend/block_page_store.h" + +using namespace crowdb::tree; + +struct ct_backend_release_service +{ + std::shared_ptr<AllocationBudget> budget; + std::shared_ptr<BackendReleaseService> queue; +}; + +namespace +{ +struct WakeContext +{ + void (*wake)(void *) = nullptr; + void (*drop)(void *) = nullptr; + void *value = nullptr; + + ~WakeContext() + { + if (drop != nullptr) { + drop(value); + } + } + + static void invoke(void *context) noexcept + { + auto &self = *static_cast<WakeContext *>(context); + self.wake(self.value); + } +}; +} // namespace + +ct_status ct_tree_backend_release_service(ct_tree *tree, void (*wake)(void *), void (*drop)(void *), void *context, + ct_backend_release_service **out) +try { + if (tree == nullptr || wake == nullptr || drop == nullptr || out == nullptr || *out != nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + auto *store = dynamic_cast<BlockPageStore *>(c_api_page_store(tree)); + if (store == nullptr || !store->release_service()) { + return static_cast<ct_status>(Code::kNotFound); + } + const auto &queue = store->release_service(); + const auto &budget = queue->allocation_budget(); + auto callback = std::allocate_shared<WakeContext>(MetadataAllocator<WakeContext>(budget)); + callback->wake = wake; + callback->value = context; + MetadataAllocator<ct_backend_release_service> allocator(budget); + auto *handle = allocator.allocate(1); + std::construct_at(handle, ct_backend_release_service{.budget = budget, .queue = queue}); + const auto status = queue->bind_wake(callback, &WakeContext::invoke, callback.get()); + if (!status.ok()) { + std::destroy_at(handle); + allocator.deallocate(handle, 1); + return static_cast<ct_status>(status.code()); + } + callback->drop = drop; + *out = handle; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +ct_backend_release_step ct_backend_release_service_pump(ct_backend_release_service *service, size_t limit) +{ + if (service == nullptr || limit == 0 || limit > 64) { + return {}; + } + const auto step = service->queue->pump(limit); + return {.visited = step.visited, + .completed = step.completed, + .more = static_cast<uint8_t>(step.more), + .drained = static_cast<uint8_t>(service->queue->drained())}; +} + +void ct_backend_release_service_stop(ct_backend_release_service *service) +{ + if (service != nullptr) { + service->queue->stop_admission(); + } +} + +void ct_backend_release_service_free(ct_backend_release_service *service) +{ + if (service == nullptr) { + return; + } + auto budget = service->budget; + std::destroy_at(service); + MetadataAllocator<ct_backend_release_service>(budget).deallocate(service, 1); +} diff --git a/lib/crowdb-tree/src/backend/backend_release_service.cpp b/lib/crowdb-tree/src/backend/backend_release_service.cpp new file mode 100644 index 000000000..6160ec7f7 --- /dev/null +++ b/lib/crowdb-tree/src/backend/backend_release_service.cpp @@ -0,0 +1,153 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/backend/backend_release_service.h" + +#include "btree/allocation_bound.h" + +#include <cassert> +#include <utility> + +namespace crowdb::tree +{ +Status BackendReleaseService::create(const std::shared_ptr<AllocationBudget> &budget, size_t capacity, + std::shared_ptr<const void> wake_owner, Wake wake, void *context, + std::shared_ptr<BackendReleaseService> *out) +try { + if (!budget || capacity == 0 || capacity >= kClosed || out == nullptr || *out || (wake != nullptr && !wake_owner)) { + return Status::invalid_argument(); + } + auto service = std::allocate_shared<BackendReleaseService>(MetadataAllocator<BackendReleaseService>(budget)); + service->budget_ = budget; + service->wake_owner_ = std::move(wake_owner); + service->wake_ = wake; + service->context_ = context; + service->capacity_ = capacity; + *out = std::move(service); + return {}; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status BackendReleaseService::bind_wake(std::shared_ptr<const void> owner, Wake wake, void *context) +{ + if (!owner || wake == nullptr || wake_ != nullptr || (state_.load(std::memory_order_acquire) & kClosed) != 0) { + return Status::invalid_argument(); + } + wake_owner_ = std::move(owner); + context_ = context; + wake_ = wake; + wake_(context_); + return {}; +} + +BackendReleaseService::~BackendReleaseService() +{ + assert((state_.load(std::memory_order_relaxed) & ~kClosed) == 0); + assert(incoming_.load(std::memory_order_relaxed) == nullptr && active_ == nullptr && deferred_ == nullptr); +} + +size_t BackendReleaseService::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<BackendReleaseService>(); +} + +Status BackendReleaseService::register_node(Node &node) +{ + if (node.dispose == nullptr || node.service || node.next != nullptr || node.queued) { + return Status::invalid_argument(); + } + auto keep = shared_from_this(); + auto state = state_.load(std::memory_order_acquire); + do { + if ((state & kClosed) != 0) { + return Status::unavailable(); + } + if (state >= capacity_) { + return Status::resource_exhausted(); + } + } while (!state_.compare_exchange_weak(state, state + 1, std::memory_order_acq_rel)); + node.service = std::move(keep); + return {}; +} + +void BackendReleaseService::enqueue(Node &node) noexcept +{ + assert(node.service && !node.queued); + auto keep = node.service; + node.queued = true; + keep->pending_.fetch_add(1, std::memory_order_relaxed); + auto *head = keep->incoming_.load(std::memory_order_relaxed); + do { + node.next = head; + } while (!keep->incoming_.compare_exchange_weak(head, &node, std::memory_order_release, std::memory_order_relaxed)); + keep->wake(); +} + +BackendReleaseService::Step BackendReleaseService::pump(size_t limit) noexcept +{ + if (consumer_.test_and_set(std::memory_order_acquire)) { + return {.more = true}; + } + auto keep_service = weak_from_this().lock(); + assert(keep_service); + Step result; + if (active_ == nullptr) { + if (prefer_deferred_ && deferred_ != nullptr) { + active_ = std::exchange(deferred_, nullptr); + prefer_deferred_ = false; + } + else { + active_ = incoming_.exchange(nullptr, std::memory_order_acq_rel); + prefer_deferred_ = true; + if (active_ == nullptr) { + active_ = std::exchange(deferred_, nullptr); + prefer_deferred_ = false; + } + } + } + for (; result.visited < limit && active_ != nullptr; ++result.visited) { + auto *node = active_; + active_ = node->next; + // The method retains the service through actual free and publication. + if (node->dispose(node)) { + pending_.fetch_sub(1, std::memory_order_relaxed); + state_.fetch_sub(1, std::memory_order_release); + ++result.completed; + } + else { + node->next = deferred_; + deferred_ = node; + } + } + result.more = active_ != nullptr || deferred_ != nullptr || incoming_.load(std::memory_order_acquire) != nullptr; + consumer_.clear(std::memory_order_release); + return result; +} + +void BackendReleaseService::stop_admission() noexcept +{ + state_.fetch_or(kClosed, std::memory_order_acq_rel); + wake(); +} + +BackendReleaseService::Snapshot BackendReleaseService::snapshot() const noexcept +{ + const auto state = state_.load(std::memory_order_acquire); + return {.registered = state & ~kClosed, + .pending = pending_.load(std::memory_order_acquire), + .closed = (state & kClosed) != 0}; +} + +bool BackendReleaseService::drained() const noexcept +{ + return state_.load(std::memory_order_acquire) == kClosed; +} + +void BackendReleaseService::wake() const noexcept +{ + if (wake_ != nullptr) { + wake_(context_); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/backing_io.cpp b/lib/crowdb-tree/src/backend/backing_io.cpp new file mode 100644 index 000000000..4ee00afc8 --- /dev/null +++ b/lib/crowdb-tree/src/backend/backing_io.cpp @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backing_io.h" + +#include "backend/local/scheduled_page_store.h" +#include "chunk/chunk_backing.h" + +namespace crowdb::tree +{ +void submit_backing_read(AsyncPageStore &store, std::shared_ptr<const BackingRef> backing, void *buffer, size_t bytes, + std::shared_ptr<AllocationBudget> metadata, AsyncCompletion complete) +{ + if (!backing || !backing->reloadable() || buffer == nullptr || bytes != backing->length) { + complete.complete(Status::invalid_argument("read requires one exact immutable backing")); + return; + } + if (backing->storage == BackingStorage::kDurableBlob) { + store.submit_read(backing->address, buffer, bytes, std::move(complete)); + return; + } + auto *scheduled = dynamic_cast<ScheduledPageStore *>(&store); + if (scheduled == nullptr || !metadata) { + complete.complete(Status::not_supported("Chunk reads require a paid backend lane")); + return; + } + const std::span<uint8_t> output(static_cast<uint8_t *>(buffer), bytes); + auto status = + scheduled->submit_step([pack = backing->chunk, output, + metadata = std::move(metadata)] { return read_chunk_backing(*pack, output, metadata); }, + complete, bytes, 0); + if (!status.ok()) { + complete.complete(std::move(status)); + } +} + +void submit_chunk_write(AsyncPageStore &store, std::shared_ptr<ChunkPackBacking> backing, + std::span<const uint8_t> input, std::shared_ptr<AllocationBudget> metadata, + AsyncCompletion complete) +{ + auto *scheduled = dynamic_cast<ScheduledPageStore *>(&store); + if (scheduled == nullptr || !backing || !metadata) { + complete.complete(Status::not_supported("Chunk writes require a paid backend lane")); + return; + } + auto status = scheduled->submit_step( + [backing = std::move(backing), input, metadata = std::move(metadata)] { + return write_chunk_backing(*backing, input, metadata); + }, + complete, 0, input.size()); + if (!status.ok()) { + complete.complete(std::move(status)); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/backing_io.h b/lib/crowdb-tree/src/backend/backing_io.h new file mode 100644 index 000000000..b86fc92fd --- /dev/null +++ b/lib/crowdb-tree/src/backend/backing_io.h @@ -0,0 +1,16 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/backing_ref.h" +#include "crowdb-tree/backend/async_page_store.h" + +namespace crowdb::tree +{ +// All paths retain the callback's exact buffer owner through the backend tail. +// Chunk reads require a scheduled backend; they never fall back to byte offsets. +void submit_backing_read(AsyncPageStore &store, std::shared_ptr<const BackingRef> backing, void *buffer, size_t bytes, + std::shared_ptr<AllocationBudget> metadata, AsyncCompletion complete); +void submit_chunk_write(AsyncPageStore &store, std::shared_ptr<ChunkPackBacking> backing, + std::span<const uint8_t> input, std::shared_ptr<AllocationBudget> metadata, + AsyncCompletion complete); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/checkpoint_backend.h b/lib/crowdb-tree/src/backend/checkpoint_backend.h new file mode 100644 index 000000000..2ed3720e6 --- /dev/null +++ b/lib/crowdb-tree/src/backend/checkpoint_backend.h @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/backing_stats_sampler.h" +#include "btree/page_writeback.h" +#include "crowdb-tree/btree/diagnostics.h" + +namespace crowdb::tree +{ +struct PublishedTreeState; +struct CapturedDurablePages; + +class BackendCheckpoint +{ + public: + virtual ~BackendCheckpoint() = default; + virtual detail::CallbackSender sender() noexcept = 0; + virtual void request_stop() noexcept = 0; + // Reconciles durable authority even when the transport returned an error. + virtual void completed() noexcept = 0; + [[nodiscard]] virtual bool committed() const noexcept = 0; + [[nodiscard]] virtual Status status() const = 0; +}; + +struct BackendCheckpointInput +{ + std::shared_ptr<const CapturedDurablePages> pages; + const PublishedTreeState &view; + uint64_t wal_offset; + size_t max_dependency_steps; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + PageLoadBudget budget; + std::shared_ptr<const void> owner; +}; + +// An injected immutable-storage authority. Generic Tree orchestration owns no +// concrete catalog, protocol, or remote transport implementation. +class CheckpointBackend +{ + public: + virtual ~CheckpointBackend() = default; + + std::shared_ptr<AsyncPageStore> store; + uint32_t iu_size = 0; + size_t max_frame_bytes = 0; + size_t max_blob_bytes = 0; + std::atomic<uint64_t> durable_coverage{0}; + std::atomic<uint64_t> snapshot_sequence{0}; + std::atomic<bool> needs_recovery{false}; + + [[nodiscard]] virtual bool supported() const noexcept = 0; + virtual void close_writer() noexcept = 0; + [[nodiscard]] virtual uint64_t backend_identity() const noexcept = 0; + [[nodiscard]] virtual uint64_t incarnation() const noexcept = 0; + [[nodiscard]] virtual uint64_t requested_wal_offset() const noexcept = 0; + [[nodiscard]] virtual std::optional<size_t> checkpoint_allocation_bound(size_t frames, + size_t heads) const noexcept = 0; + [[nodiscard]] virtual size_t reservation_allocation_bound() const noexcept = 0; + [[nodiscard]] virtual PageWriteOperation::ReserveExtent + reserve(const std::shared_ptr<detail::AllocationReservation> &metadata) const = 0; + virtual Status prepare_checkpoint(BackendCheckpointInput input, std::shared_ptr<BackendCheckpoint> *out) = 0; + + [[nodiscard]] virtual BackingStatsSampler backing_sampler() const noexcept + { + return {}; + } +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_backend_checkpoint.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_backend_checkpoint.cpp new file mode 100644 index 000000000..769b1d8ed --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_backend_checkpoint.cpp @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_backend_state.h" +#include "chunk_protocol_operation.h" +#include "snapshot/chunk_root_commit.h" + +namespace crowdb::tree +{ +namespace +{ +class ChunkCheckpoint final : public BackendCheckpoint +{ + public: + ChunkBackendState *backend = nullptr; + std::shared_ptr<ChunkRootCommit> commit; + std::shared_ptr<ChunkProtocolOperation> operation; + + detail::CallbackSender sender() noexcept override + { + return operation->sender(); + } + + void request_stop() noexcept override + { + operation->request_stop(); + } + + void completed() noexcept override + { + backend->completed(*commit); + } + + [[nodiscard]] bool committed() const noexcept override + { + return commit->committed(); + } + + [[nodiscard]] Status status() const override + { + return commit->status(); + } +}; +} // namespace + +bool ChunkBackendState::supported() const noexcept +{ + return domain && domain->supported() && ids && facade != nullptr; +} + +std::optional<size_t> ChunkBackendState::checkpoint_allocation_bound(size_t frames, size_t heads) const noexcept +{ + return detail::AllocationBound() + .add(ChunkRootCommit::allocation_bound(frames)) + .add(ChunkRootCommit::window_allocation_bound(heads)) + .add(ChunkProtocolOperation::allocation_bound()) + .add(detail::shared_allocation_bound<ChunkCheckpoint>()) + .value(); +} + +size_t ChunkBackendState::reservation_allocation_bound() const noexcept +{ + return chunk_backing_allocation_bound(); +} + +Status ChunkBackendState::prepare_checkpoint(BackendCheckpointInput input, std::shared_ptr<BackendCheckpoint> *out) +{ + auto identity = next_identity(input.view); + identity.next_page_id = input.pages->next_page_id; + identity.wal_replay_offset = input.wal_offset; + auto result = detail::reserved_shared<ChunkCheckpoint>(input.budget.bytes); + result->backend = this; + auto status = ChunkRootCommit::create(input.pages, domain, ids, identity, input.budget.bytes, input.budget.bytes, + &result->commit, input.max_dependency_steps); + if (!status.ok()) { + return status; + } + status = ChunkProtocolOperation::create( + input.store, input.runtime, input.lane, input.budget, input.owner, + [operation = result->commit](bool stopped) { return operation->step(stopped); }, + [operation = result->commit] { return operation->done(); }, &result->operation); + if (status.ok()) { + *out = std::move(result); + } + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_backend_state.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_backend_state.cpp new file mode 100644 index 000000000..fc60bae45 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_backend_state.cpp @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_backend_state.h" + +#include "backend/chunk/chunk_frame.h" +#include "backend/chunk/chunk_physical_registry.h" +#include "snapshot/chunk_root_commit.h" + +namespace crowdb::tree +{ +ChunkBackendState::~ChunkBackendState() +{ + close_writer(); + if (owns_writer_domain) { + facade->release_runtime_domain(); + } +} + +BackingStatsSampler ChunkBackendState::backing_sampler() const noexcept +{ + if (!domain || !domain->physical_registry) { + return {}; + } + const auto ®istry = domain->physical_registry; + return {.scope_id = registry->backing_stats().scope_id, .metrics = registry->backing_metrics()}; +} + +void ChunkBackendState::close_writer() noexcept +{ + writer_session.reset(); +} + +void ChunkBackendState::recovered(const ChunkRootIdentity &identity) noexcept +{ + durable_coverage.store(identity.coverage, std::memory_order_release); + snapshot_sequence.store(identity.snapshot_sequence, std::memory_order_release); + root_generation.store(identity.root_generation, std::memory_order_release); + facade->committed(identity.root_generation, identity.wal_replay_offset); +} + +void ChunkBackendState::completed(const ChunkRootCommit &operation) noexcept +{ + if (operation.committed()) { + recovered(operation.identity()); + } + if (operation.needs_recovery() || !operation.done()) { + needs_recovery.store(true, std::memory_order_release); + } +} + +ChunkRootIdentity ChunkBackendState::next_identity(const PublishedTreeState &view) const noexcept +{ + return {.tree = domain->tree, + .root_generation = root_generation.load(std::memory_order_acquire) + 1, + .owner_epoch = domain->epoch, + .snapshot_sequence = snapshot_sequence.load(std::memory_order_acquire) + 1, + .page_generation = view.generation, + .coverage = view.coverage, + .gc_floor = view.gc_floor, + .root_page_id = view.root_page_id, + .next_page_id = view.next_page_id, + .wal_replay_offset = facade->requested_wal_offset()}; +} + +PageWriteOperation::ReserveExtent +ChunkBackendState::reserve(const std::shared_ptr<detail::AllocationReservation> &metadata) const +{ + return [domain = domain, ids = ids, metadata, + capacity = facade->options().max_chunk_bytes](uint64_t bytes, PageWriteOperation::ExtentComplete complete) { + if (bytes == 0 || bytes > UINT32_MAX || detail::chunk_framed_length(bytes) > capacity) { + complete(Status::resource_exhausted(), {}); + return; + } + detail::ChunkStageId stage; + detail::ChunkId chunk; + auto status = ids->next(PagePurpose::kBtreePage, &stage, &chunk); + std::shared_ptr<ChunkPackBacking> backing; + if (status.ok()) { + status = prepare_chunk_backing(domain, stage, chunk, static_cast<uint32_t>(bytes), PagePurpose::kBtreePage, + metadata, &backing); + } + if (!status.ok()) { + complete(std::move(status), {}); + return; + } + complete(Status::Ok(), {.backend_id = domain->backend_id, + .address = 0, + .length = bytes, + .content_id = stage.operation_low, + .generation = domain->epoch, + .owner = backing, + .physical_metrics = {}, + .chunk = backing}); + }; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_backend_state.h b/lib/crowdb-tree/src/backend/chunk/chunk_backend_state.h new file mode 100644 index 000000000..bfa3831b9 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_backend_state.h @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/checkpoint_backend.h" +#include "backend/chunk/chunk_collector.h" +#include "backend/chunk/chunk_pin_release.h" +#include "backend/chunk/chunk_runtime_ids.h" +#include "backend/chunk/chunk_runtime_page_store.h" +#include "snapshot/chunk_root_codec.h" + +namespace crowdb::tree +{ +class ChunkRootCommit; + +// Stable authority owner. Root identity is independent of the live mapping +// generation and its staged pack references. No whole committed map is kept. +struct ChunkBackendState final : CheckpointBackend +{ + ~ChunkBackendState() override; + std::shared_ptr<detail::ChunkPinOwner> writer_session; + bool owns_writer_domain = false; + std::shared_ptr<const ChunkBackingDomain> domain; + std::shared_ptr<detail::ChunkRuntimeIds> ids; + std::shared_ptr<const void> backend_owner; + detail::ChunkRuntimePageStore *facade = nullptr; + // Reserved for the catalog's actual root-retention owner, including old + // committed roots until authority GC retires their reference windows. + std::shared_ptr<const void> checkpoint_roles; + std::atomic<uint64_t> root_generation{0}; + std::atomic<bool> collection_active{false}; + detail::ChunkCollector collector; + + [[nodiscard]] bool supported() const noexcept override; + [[nodiscard]] BackingStatsSampler backing_sampler() const noexcept override; + + [[nodiscard]] uint64_t backend_identity() const noexcept override + { + return domain->backend_id; + } + + [[nodiscard]] uint64_t incarnation() const noexcept override + { + return domain->epoch; + } + + [[nodiscard]] uint64_t requested_wal_offset() const noexcept override + { + return facade->requested_wal_offset(); + } + + [[nodiscard]] std::optional<size_t> checkpoint_allocation_bound(size_t frames, + size_t heads) const noexcept override; + [[nodiscard]] size_t reservation_allocation_bound() const noexcept override; + Status prepare_checkpoint(BackendCheckpointInput input, std::shared_ptr<BackendCheckpoint> *out) override; + void close_writer() noexcept override; + void recovered(const ChunkRootIdentity &identity) noexcept; + void completed(const ChunkRootCommit &operation) noexcept; + [[nodiscard]] ChunkRootIdentity next_identity(const PublishedTreeState &view) const noexcept; + [[nodiscard]] PageWriteOperation::ReserveExtent + reserve(const std::shared_ptr<detail::AllocationReservation> &metadata) const override; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_backing.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_backing.cpp new file mode 100644 index 000000000..071c52fbb --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_backing.cpp @@ -0,0 +1,189 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_backing.h" + +#include "btree/allocation_bound.h" +#include "chunk_frame.h" +#include "chunk_physical_registry.h" + +#include <algorithm> + +namespace crowdb::tree +{ +static Status read_chunk_pack(const ChunkPackBacking &backing, std::span<uint8_t> output, + const std::shared_ptr<AllocationBudget> &budget); +static Status write_chunk_pack(ChunkPackBacking &backing, std::span<const uint8_t> input, + const std::shared_ptr<AllocationBudget> &budget); + +namespace +{ +Status validate_pin_pack(const ChunkBackingDomain &domain, const detail::ChunkFixedPackRef &pack, + const std::shared_ptr<AllocationBudget> &metadata) +{ + std::optional<detail::ChunkStageRecord> record(detail::ChunkStageRecord{ + .tree_id = 0, + .id = {}, + .revision = 0, + .chunk_high = 0, + .chunk_low = 0, + .capacity = 0, + .manifest_generation = 0, + .purpose = 0, + .state = detail::ChunkStageState::Reserved, + .packs = decltype(detail::ChunkStageRecord::packs)(MetadataAllocator<detail::ChunkStagePack>(metadata))}); + auto status = domain.catalog->load_stage(domain.tree, pack.stage, &record); + if (!status.ok()) { + return status; + } + if (!record || record->tree_id != domain.tree || record->id != pack.stage || + record->chunk_high != pack.chunk.high || record->chunk_low != pack.chunk.low || + record->purpose != static_cast<uint32_t>(pack.purpose) || pack.offset > record->capacity || + detail::chunk_framed_length(pack.length) > record->capacity - pack.offset || + std::none_of(record->packs.begin(), record->packs.end(), [&](const auto &stored) { + return stored.offset == pack.offset && stored.length == pack.length && stored.checksum == pack.checksum; + })) { + return Status::corruption("Chunk pack is not an exact member of its protected stage"); + } + return record->state == detail::ChunkStageState::Reloadable || record->state == detail::ChunkStageState::Referenced + ? Status::Ok() + : Status::unavailable("Chunk stage no longer admits readers"); +} +} // namespace + +size_t chunk_pin_validation_bound() noexcept +{ + return 2 * detail::kMaxStagePacks * sizeof(detail::ChunkStagePack); +} + +Status prepare_chunk_backing(const std::shared_ptr<const ChunkBackingDomain> &domain, detail::ChunkStageId stage, + detail::ChunkId chunk, uint32_t length, PagePurpose purpose, + const std::shared_ptr<AllocationBudget> &metadata, std::shared_ptr<ChunkPackBacking> *out) +try { + if (!domain || !domain->supported() || !metadata || out == nullptr || *out || stage.owner_epoch != domain->epoch || + detail::chunk_purpose(chunk) != purpose || detail::chunk_framed_length(length) > domain->max_chunk_bytes || + !detail::ChunkFixedPackStore::write_allocation_bound(length)) { + return Status::invalid_argument(); + } + auto result = std::allocate_shared<ChunkPackBacking>(MetadataAllocator<ChunkPackBacking>(metadata)); + result->read = &read_chunk_pack; + result->write = &write_chunk_pack; + auto status = domain->catalog->pin_release_queue()->prepare(metadata, domain->catalog, domain->tree, stage, stage, + &result->pin); + if (!status.ok()) { + return status; + } + result->physical_budget = metadata; + result->domain = domain; + result->pack = {.stage = stage, .chunk = chunk, .offset = 0, .length = length, .checksum = 0, .purpose = purpose}; + *out = std::move(result); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status pin_chunk_backing(const std::shared_ptr<const ChunkBackingDomain> &domain, detail::ChunkFixedPackRef pack, + detail::ChunkStageId pin, const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<const ChunkPackBacking> *out) +try { + if (!domain || !domain->supported() || !metadata || out == nullptr || *out || pin.owner_epoch != domain->epoch) { + return Status::invalid_argument(); + } + auto result = std::allocate_shared<ChunkPackBacking>(MetadataAllocator<ChunkPackBacking>(metadata)); + result->read = &read_chunk_pack; + result->write = &write_chunk_pack; + result->domain = domain; + result->pack = pack; + auto status = domain->catalog->pin_release_queue()->prepare(metadata, domain->catalog, domain->tree, pack.stage, + pin, &result->pin); + if (!status.ok()) { + return status; + } + if (!chunk_backing_matches(*result, domain->backend_id, pack.length)) { + return Status::corruption("checkpoint pack identity is invalid"); + } + auto *protection = domain->catalog->protection(); + status = protection->check_read_owner(); + if (!status.ok()) { + return status; + } + status = validate_pin_pack(*domain, pack, metadata); + if (!status.ok()) { + return status; + } + result->pin->arm(); + status = protection->change_pin(domain->tree, pack.stage, pin, true); + if (status.ok()) { + status = protection->check_read_owner(); + } + if (status.ok() && domain->physical_registry) { + auto registry = domain->physical_registry; + status = registry->prepare(pack.stage, pack.chunk, metadata, &result->physical); + detail::ChunkAllocationIdentity receipt; + if (status.ok()) { + status = domain->transport->inspect_fixed_allocation(pack.chunk, pack.stage.owner_epoch, &receipt); + } + if (status.ok()) { + status = registry->observe(result->physical, receipt, metadata); + } + if (status.ok()) { + result->physical_metrics = ChunkPhysicalRegistry::extent(result->physical); + } + } + if (status.ok()) { + *out = std::move(result); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +static Status read_chunk_pack(const ChunkPackBacking &backing, std::span<uint8_t> output, + const std::shared_ptr<AllocationBudget> &budget) +{ + if (!backing.domain || !chunk_backing_matches(backing, backing.domain->backend_id, output.size())) { + return Status::corruption("read does not name one owned Chunk pack"); + } + const auto &domain = *backing.domain; + auto *protection = domain.catalog->protection(); + auto status = protection->check_read_owner(); + if (!status.ok()) { + return status; + } + detail::ChunkFixedPackStore store(domain.tree, domain.epoch, domain.mirrors, domain.catalog, domain.transport); + status = store.read_into(backing.pack, budget, output); + if (status.ok()) { + status = protection->check_read_owner(); + } + return status; +} + +static Status write_chunk_pack(ChunkPackBacking &backing, std::span<const uint8_t> input, + const std::shared_ptr<AllocationBudget> &budget) +{ + if (!backing.domain || !chunk_backing_matches(backing, backing.domain->backend_id, input.size()) || + backing.pack.stage.owner_epoch != backing.domain->epoch) { + return Status::invalid_argument("write does not name one reserved Chunk pack"); + } + const auto &domain = *backing.domain; + auto status = domain.catalog->protection()->check_read_owner(); + if (!status.ok()) { + return status; + } + backing.pin->arm(); + detail::ChunkFixedPackStore store(domain.tree, domain.epoch, domain.mirrors, domain.catalog, domain.transport, + domain.physical_registry); + detail::ChunkFixedPackRef result; + status = store.write(backing.pack.stage, backing.pack.chunk, backing.pack.purpose, input, budget, &result, + &backing.physical, backing.physical_budget); + if (status.ok()) { + status = domain.catalog->protection()->check_read_owner(); + } + backing.physical_metrics = ChunkPhysicalRegistry::extent(backing.physical); + if (status.ok()) { + backing.pack = result; + } + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_backing.h b/lib/crowdb-tree/src/backend/chunk/chunk_backing.h new file mode 100644 index 000000000..271af2e28 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_backing.h @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "chunk_fixed_pack.h" +#include "chunk_pin_release.h" + +namespace crowdb::tree +{ +class BackingMetricExtent; +class ChunkPhysicalRegistry; +struct ChunkPhysicalPack; + +// A stable storage domain, independent of a Tree generation or CPU executor. +// Every descriptor owns a paid pin; the host release service outlives them. +struct ChunkBackingDomain +{ + uint64_t backend_id = 0; + uint64_t tree = 0; + uint64_t epoch = 0; + uint32_t mirrors = 0; + uint64_t max_chunk_bytes = 256 * 1024 * 1024; + std::shared_ptr<detail::RootCatalog> catalog; + std::shared_ptr<detail::ChunkTransport> transport; + std::shared_ptr<ChunkPhysicalRegistry> physical_registry; + + [[nodiscard]] bool supported() const noexcept; +}; + +// Filled privately by the writer before its completion publishes a const view. +// Shared page descriptors use this one pack owner, including its physical-byte +// observation; no per-page copy of whole-pack storage accounting is needed. +struct ChunkPackBacking +{ + using Read = Status (*)(const ChunkPackBacking &, std::span<uint8_t>, const std::shared_ptr<AllocationBudget> &); + using Write = Status (*)(ChunkPackBacking &, std::span<const uint8_t>, const std::shared_ptr<AllocationBudget> &); + Read read = nullptr; + Write write = nullptr; + detail::ChunkFixedPackRef pack; + std::shared_ptr<const ChunkBackingDomain> domain; + std::shared_ptr<detail::ChunkPinOwner> pin; + std::shared_ptr<AllocationBudget> physical_budget; + std::shared_ptr<ChunkPhysicalPack> physical; + std::shared_ptr<BackingMetricExtent> physical_metrics; +}; + +[[nodiscard]] bool chunk_backing_matches(const ChunkPackBacking &backing, uint64_t domain, uint64_t length) noexcept; +[[nodiscard]] bool same_chunk_backing(const std::shared_ptr<const ChunkPackBacking> &left, + const std::shared_ptr<const ChunkPackBacking> &right) noexcept; +[[nodiscard]] std::array<uint64_t, 8> chunk_backing_identity(const ChunkPackBacking &backing) noexcept; +[[nodiscard]] size_t chunk_backing_allocation_bound() noexcept; +[[nodiscard]] size_t chunk_pin_validation_bound() noexcept; + +// Preparation spends no metadata I/O. Arm before the first mutation so an +// unknown StageReserved result still receives an exact, durable final release. +Status prepare_chunk_backing(const std::shared_ptr<const ChunkBackingDomain> &domain, detail::ChunkStageId stage, + detail::ChunkId chunk, uint32_t length, PagePurpose purpose, + const std::shared_ptr<AllocationBudget> &metadata, std::shared_ptr<ChunkPackBacking> *out); +Status pin_chunk_backing(const std::shared_ptr<const ChunkBackingDomain> &domain, detail::ChunkFixedPackRef pack, + detail::ChunkStageId pin, const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<const ChunkPackBacking> *out); + +// Finite blocking backend steps; the caller must dispatch them on its bounded +// backend lane. Exact identities are never translated to synthetic file +// offsets. +Status read_chunk_backing(const ChunkPackBacking &backing, std::span<uint8_t> output, + const std::shared_ptr<AllocationBudget> &budget); +Status write_chunk_backing(ChunkPackBacking &backing, std::span<const uint8_t> input, + const std::shared_ptr<AllocationBudget> &budget); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_catalog_scan.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_catalog_scan.cpp new file mode 100644 index 000000000..06e3404f7 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_catalog_scan.cpp @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_protection_callback.h" +#include "crowdb-tree/maptable/frame_page.h" + +namespace crowdb::tree::detail +{ +namespace pc = protection_callback; + +Status CallbackRootCatalog::next_stage(uint64_t tree, std::optional<ChunkStageId> after, + const std::shared_ptr<AllocationBudget> &budget, + std::optional<ChunkStageRecord> *out) const +try { + if (!stage_schema_supported()) { + return Status::not_supported(); + } + if (tree == 0 || !budget || out == nullptr || (after && !pc::valid(*after))) { + return Status::invalid_argument(); + } + const auto cursor = pc::external(after.value_or(ChunkStageId{})); + + struct Blob + { + ct_root_catalog_callbacks callbacks; + void *context; + const uint8_t *data = nullptr; + size_t length = 0; + + ~Blob() + { + if (data != nullptr) { + callbacks.free_blob(context, data, length); + } + } + } blob{.callbacks = callbacks_, .context = context_}; + + auto status = pc::status(staging_.list(context_, tree, after ? &cursor : nullptr, 1, &blob.data, &blob.length)); + if (!status.ok()) { + return status; + } + if (blob.data == nullptr || blob.length < 4 || blob.length > 8 + kMaxStageBytes) { + return Status::corruption(); + } + const auto count = frame_u32(blob.data, 0); + if (count == 0 && blob.length == 4) { + out->reset(); + return Status::Ok(); + } + if (count != 1 || blob.length < 8 || frame_u32(blob.data, 4) != blob.length - 8) { + return Status::corruption(); + } + auto record = + ChunkStageRecord{.tree_id = 0, + .id = {}, + .revision = 0, + .chunk_high = 0, + .chunk_low = 0, + .capacity = 0, + .manifest_generation = 0, + .purpose = 0, + .state = ChunkStageState::Reserved, + .packs = decltype(ChunkStageRecord::packs)(MetadataAllocator<ChunkStagePack>(budget))}; + status = decode_stage_record({blob.data + 8, blob.length - 8}, &record); + if (!status.ok()) { + return status; + } + if (record.tree_id != tree || (after && record.id <= *after)) { + return Status::corruption(); + } + *out = std::move(record); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status CallbackRootCatalog::next_candidate(uint64_t tree, std::optional<ChunkStageId> after, + std::optional<ChunkCandidateStatus> *out) const +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || out == nullptr || (after && !pc::valid(*after))) { + return Status::invalid_argument(); + } + ct_chunk_candidate_status wire{}; + size_t count = 1; + const auto cursor = pc::external(after.value_or(ChunkStageId{})); + auto status = pc::status(protection_.list(context_, tree, after ? &cursor : nullptr, 1, &wire, &count)); + if (!status.ok()) { + return status; + } + if (count == 0) { + out->reset(); + return Status::Ok(); + } + if (count != 1) { + return Status::corruption(); + } + ChunkCandidateStatus value; + status = pc::decode(wire, &value); + if (!status.ok() || (after && value.id <= *after)) { + return Status::corruption(); + } + *out = value; + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.cpp new file mode 100644 index 000000000..e03190c21 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.cpp @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_collection_operation.h" + +#include "btree/allocation_bound.h" +#include "chunk_backend_state.h" + +namespace crowdb::tree +{ +struct ChunkCollectionOperation::State +{ + std::shared_ptr<ChunkBackendState> backend; + PageLoadBudget budget; + detail::ChunkCollectionProgress progress; + size_t steps = 0; + bool owns_gate = false; + + ~State() + { + release_gate(); + if (budget.bytes) { + budget.bytes->seal(); + } + if (budget.io) { + budget.io->seal(); + } + } + + void release_gate() noexcept + { + if (owns_gate) { + owns_gate = false; + backend->collection_active.store(false, std::memory_order_release); + backend.reset(); + } + } + + Status step(bool stopped) + { + if (stopped) { + release_gate(); + return Status::unavailable(); + } + const auto &domain = *backend->domain; + auto status = backend->collector.step(*domain.catalog, *domain.transport, domain.tree, domain.epoch, + budget.bytes, &progress, domain.physical_registry); + ++steps; + if (!status.ok() || done()) { + release_gate(); + } + return status; + } + + [[nodiscard]] bool done() const noexcept + { + return steps == 8 || !progress.more; + } +}; + +size_t ChunkCollectionOperation::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<State>() + detail::shared_allocation_bound<ChunkCollectionOperation>() + + ChunkProtocolOperation::allocation_bound() + detail::ChunkCollector::allocation_bound(); +} + +Status ChunkCollectionOperation::create(std::shared_ptr<ChunkBackendState> backend, + std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageLoadBudget budget, + std::shared_ptr<ChunkCollectionOperation> *out) +try { + if (!backend || !store || !budget.bytes || !budget.io || out == nullptr || *out) { + return Status::invalid_argument(); + } + auto state = detail::reserved_shared<State>(budget.bytes); + state->backend = std::move(backend); + state->budget = budget; + if (state->backend->needs_recovery.load(std::memory_order_acquire) || + state->backend->collection_active.exchange(true, std::memory_order_acq_rel)) { + return Status::unavailable(); + } + state->owns_gate = true; + auto operation = detail::reserved_construct<ChunkCollectionOperation>( + budget.bytes, [&](auto *storage) { ::new (static_cast<void *>(storage)) ChunkCollectionOperation(state); }); + auto status = ChunkProtocolOperation::create( + std::move(store), std::move(runtime), std::move(lane), budget, state, + [state](bool stopped) { return state->step(stopped); }, [state] { return state->done(); }, + &operation->protocol_); + if (status.ok()) { + *out = std::move(operation); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +detail::CallbackSender ChunkCollectionOperation::sender() noexcept +{ + return protocol_->sender(); +} + +const detail::ChunkCollectionProgress &ChunkCollectionOperation::result() const noexcept +{ + return state_->progress; +} + +void ChunkCollectionOperation::request_stop() noexcept +{ + protocol_->request_stop(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.h b/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.h new file mode 100644 index 000000000..83450420c --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.h @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_collector.h" +#include "chunk_protocol_operation.h" + +namespace crowdb::tree +{ +struct ChunkBackendState; + +class ChunkCollectionOperation +{ + struct State; + + public: + static Status create(std::shared_ptr<ChunkBackendState> backend, std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageLoadBudget budget, + std::shared_ptr<ChunkCollectionOperation> *out); + [[nodiscard]] static size_t allocation_bound() noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const detail::ChunkCollectionProgress &result() const noexcept; + void request_stop() noexcept; + + private: + explicit ChunkCollectionOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; + std::shared_ptr<ChunkProtocolOperation> protocol_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_collector.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_collector.cpp new file mode 100644 index 000000000..b4e0ee1d3 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_collector.cpp @@ -0,0 +1,179 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_collector.h" + +#include "chunk_physical_registry.h" + +namespace crowdb::tree::detail +{ +size_t ChunkCollector::allocation_bound() noexcept +{ + // Active, loaded and CAS copy plus encoded old/new records coexist. The + // cursor itself contains identities only and retains no unbounded result. + return (4 * kMaxStagePacks * sizeof(ChunkStagePack)) + (2 * kMaxStageBytes) + + ChunkPhysicalRegistry::pack_allocation_bound(); +} + +Status ChunkCollector::step(RootCatalog &catalog, ChunkTransport &transport, uint64_t tree, uint64_t epoch, + const std::shared_ptr<AllocationBudget> &budget, ChunkCollectionProgress *out, + const std::shared_ptr<ChunkPhysicalRegistry> ®istry) +try { + auto *protection = catalog.protection(); + if (tree == 0 || epoch == 0 || !budget || out == nullptr) { + return Status::invalid_argument(); + } + if (protection == nullptr || !transport.fixed_staging_supported()) { + return Status::not_supported(); + } + auto status = protection->check_read_owner(); + if (!status.ok()) { + return status; + } + if (candidates_end_ && stages_end_) { + candidates_end_ = false; + stages_end_ = false; + candidate_after_.reset(); + stage_after_.reset(); + } + const bool candidate = !candidates_end_ && (next_candidate_ || stages_end_); + status = candidate ? candidate_step(*protection, tree, *out) + : stage_step(catalog, transport, tree, epoch, budget, *out, registry); + next_candidate_ = !candidate; + out->more = !(candidates_end_ && stages_end_); + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status ChunkCollector::candidate_step(ChunkProtectionCatalog &catalog, uint64_t tree, ChunkCollectionProgress &out) +{ + std::optional<ChunkCandidateStatus> record; + auto status = candidate_active_ ? catalog.load_candidate(tree, *candidate_active_, &record) + : catalog.next_candidate(tree, candidate_after_, &record); + if (!status.ok()) { + return status; + } + if (!record) { + if (candidate_active_) { + return Status::corruption("active candidate disappeared before reference release"); + } + candidates_end_ = true; + return Status::Ok(); + } + if (!candidate_active_) { + ++out.candidates_examined; + } + candidate_active_ = record->id; + if (record->state == ChunkCandidateState::Preparing || record->state == ChunkCandidateState::Ready) { + status = catalog.finish_candidate(tree, record->id, CT_CHUNK_CANDIDATE_RECOVER); + if (status.ok()) { + return status; // Query this same identity before releasing anything. + } + if (status.code() != Code::kUnavailable) { + return status; + } + // A live writer or an unavailable proof retains its intents. Revisit + // next sweep; it must not starve later abandoned identities. + } + else if ((record->state == ChunkCandidateState::Aborted || record->state == ChunkCandidateState::Reclaimed) && + record->released_windows < record->prepared_windows) { + status = catalog.release_references(tree, record->id, record->released_windows); + if (status.ok()) { + ++out.reference_windows; + } + return status; // Unknown reply is resolved by load_candidate next time. + } + candidate_after_ = record->id; + candidate_active_.reset(); + return Status::Ok(); +} + +Status ChunkCollector::stage_step(RootCatalog &catalog, ChunkTransport &transport, uint64_t tree, uint64_t epoch, + const std::shared_ptr<AllocationBudget> &budget, ChunkCollectionProgress &out, + const std::shared_ptr<ChunkPhysicalRegistry> ®istry) +{ + std::optional<ChunkStageRecord> record{ + ChunkStageRecord{.tree_id = 0, + .id = {}, + .revision = 0, + .chunk_high = 0, + .chunk_low = 0, + .capacity = 0, + .manifest_generation = 0, + .purpose = 0, + .state = ChunkStageState::Reserved, + .packs = decltype(ChunkStageRecord::packs)(MetadataAllocator<ChunkStagePack>(budget))} + }; + auto status = stage_active_ ? catalog.load_stage(tree, *stage_active_, &record) + : catalog.next_stage(tree, stage_after_, budget, &record); + if (!status.ok()) { + return status; + } + if (!record) { + if (stage_active_) { + stage_after_ = stage_active_; + stage_active_.reset(); + pin_after_.reset(); + } + else { + stages_end_ = true; + } + return Status::Ok(); + } + if (!stage_active_) { + ++out.stages_examined; + stage_active_ = record->id; + } + if (record->id.owner_epoch > epoch) { + stage_after_ = record->id; + stage_active_.reset(); + return Status::Ok(); + } + if (record->state < ChunkStageState::Retired) { + ChunkPinProgress progress; + status = catalog.protection()->prune_pins(tree, record->id, pin_after_, 64, &progress); + if (!status.ok()) { + return status; + } + if (progress.examined > 64 || progress.released > progress.examined || + (progress.more && (!progress.after || (pin_after_ && *progress.after <= *pin_after_)))) { + return Status::corruption("pin sweep did not advance its bounded cursor"); + } + out.pins_examined += progress.examined; + out.pins_released += progress.released; + pin_after_ = progress.after; + if (progress.more) { + return Status::Ok(); + } + if (record->revision == UINT64_MAX) { + return Status::corruption(); + } + auto retired = *record; + ++retired.revision; + retired.state = ChunkStageState::Retired; + status = catalog.compare_stage(tree, epoch, record, retired); + if (!status.ok() && status.code() != Code::kUnavailable) { + return status; + } + if (status.ok()) { + pin_after_.reset(); + return Status::Ok(); // Durable retirement precedes physical deletion. + } + } + else { + const auto result = reclaim_retired_stage(catalog, transport, tree, epoch, record->id, budget, registry); + if (!result.status.ok()) { + return result.status; + } + if (result.stage_removed) { + ++out.stages_removed; + out.logical_capacity_bytes += record->capacity; + } + } + stage_after_ = record->id; + stage_active_.reset(); + pin_after_.reset(); + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_collector.h b/lib/crowdb-tree/src/backend/chunk/chunk_collector.h new file mode 100644 index 000000000..50f0afd52 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_collector.h @@ -0,0 +1,45 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_stage_gc.h" + +namespace crowdb::tree::detail +{ +struct ChunkCollectionProgress +{ + uint64_t candidates_examined = 0; + uint64_t reference_windows = 0; + uint64_t stages_examined = 0; + uint64_t pins_examined = 0; + uint64_t pins_released = 0; + uint64_t stages_removed = 0; + uint64_t logical_capacity_bytes = 0; + bool more = false; +}; + +// One accepted maintenance operation owns this fixed cursor exclusively. No +// mapping traversal or inferred age makes a stage eligible: the catalog CAS +// validates recovery, process exclusion, references and pins at retirement. +class ChunkCollector +{ + public: + [[nodiscard]] static size_t allocation_bound() noexcept; + Status step(RootCatalog &catalog, ChunkTransport &transport, uint64_t tree, uint64_t epoch, + const std::shared_ptr<AllocationBudget> &budget, ChunkCollectionProgress *out, + const std::shared_ptr<ChunkPhysicalRegistry> ®istry = {}); + + private: + Status candidate_step(ChunkProtectionCatalog &catalog, uint64_t tree, ChunkCollectionProgress &out); + Status stage_step(RootCatalog &catalog, ChunkTransport &transport, uint64_t tree, uint64_t epoch, + const std::shared_ptr<AllocationBudget> &budget, ChunkCollectionProgress &out, + const std::shared_ptr<ChunkPhysicalRegistry> ®istry); + std::optional<ChunkStageId> candidate_after_; + std::optional<ChunkStageId> candidate_active_; + std::optional<ChunkStageId> stage_after_; + std::optional<ChunkStageId> stage_active_; + std::optional<ChunkStageId> pin_after_; + bool next_candidate_ = true; + bool candidates_end_ = false; + bool stages_end_ = false; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.cpp new file mode 100644 index 000000000..08f571368 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.cpp @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_fixed_pack.h" + +#include "btree/allocation_bound.h" +#include "chunk_backing.h" +#include "chunk_frame.h" +#include "chunk_physical_registry.h" +#include "crowdb-common/crc32c.h" + +namespace crowdb::tree::detail +{ +ChunkFixedPackStore::ChunkFixedPackStore(uint64_t tree, uint64_t epoch, uint32_t mirrors, + std::shared_ptr<RootCatalog> catalog, + std::shared_ptr<ChunkTransport> transport, + std::shared_ptr<ChunkPhysicalRegistry> registry) + : tree_(tree), + epoch_(epoch), + mirrors_(mirrors), + catalog_(std::move(catalog)), + transport_(std::move(transport)), + registry_(std::move(registry)) +{ +} + +bool ChunkFixedPackStore::supported() const noexcept +{ + return tree_ != 0 && epoch_ != 0 && mirrors_ != 0 && mirrors_ <= kMaxMirrorCopies && catalog_ && + catalog_->stage_schema_supported() && transport_ && transport_->fixed_staging_supported(); +} + +Status ChunkFixedPackStore::write(ChunkStageId operation, ChunkId fixed, PagePurpose purpose, + std::span<const uint8_t> input, const std::shared_ptr<AllocationBudget> &budget, + ChunkFixedPackRef *out, std::shared_ptr<ChunkPhysicalPack> *physical, + const std::shared_ptr<AllocationBudget> &physical_budget) +try { + const auto context = ChunkIoContext::capture(); + if (!supported()) { + return Status::not_supported("fixed staged chunk writes are unavailable"); + } + if (!budget || out == nullptr || operation.owner_epoch != epoch_ || !write_allocation_bound(input.size())) { + return Status::invalid_argument(); + } + auto framed = std::allocate_shared<ChunkPackWindow>(MetadataAllocator<ChunkPackWindow>(budget), budget); + framed->bytes.resize(chunk_framed_length(input.size())); + auto status = encode_fixed_pack(fixed, purpose, input, framed->bytes); + if (!status.ok()) { + return status; + } + ChunkStageRecord reserved{.tree_id = tree_, + .id = operation, + .revision = 1, + .chunk_high = fixed.high, + .chunk_low = fixed.low, + .capacity = framed->bytes.size(), + .manifest_generation = 0, + .purpose = static_cast<uint32_t>(purpose), + .state = ChunkStageState::Reserved, + .packs = decltype(ChunkStageRecord::packs)(MetadataAllocator<ChunkStagePack>(budget))}; + status = validate_stage_record(reserved); + if (!status.ok()) { + return status; + } + std::shared_ptr<ChunkPhysicalPack> observed; + ChunkPhysicalAdmission physical_admission; + if (registry_) { + if (physical == nullptr) { + return Status::invalid_argument(); + } + status = registry_->prepare(operation, fixed, physical_budget ? physical_budget : budget, &observed, + &physical_admission); + if (!status.ok()) { + return status; + } + *physical = observed; + } + // This durable edge must precede even submission of AllocateChunk. + status = catalog_->compare_stage(tree_, epoch_, {}, reserved); + if (!status.ok()) { + return status; + } + physical_admission.accept(); + status = transport_->allocate_fixed_chunk(fixed, framed->bytes.size(), epoch_, purpose); + if (registry_) { + ChunkAllocationIdentity receipt; + auto inspected = transport_->inspect_fixed_allocation(fixed, epoch_, &receipt); + if (inspected.ok()) { + inspected = registry_->observe(observed, receipt, budget); + } + if (status.ok() && !inspected.ok()) { + return inspected; + } + } + if (!status.ok()) { + return status; + } + status = transport_->verify_fixed_read_fence(fixed, mirrors_, context); + if (!status.ok()) { + return status; + } + auto writing = reserved; + writing.revision++; + writing.state = ChunkStageState::Writing; + writing.packs.push_back({.offset = 0, + .length = static_cast<uint32_t>(input.size()), + .checksum = crowdb::common::crc32c(input.data(), input.size())}); + status = catalog_->compare_stage(tree_, epoch_, reserved, writing); + if (!status.ok()) { + return status; + } + for (uint32_t mirror = 0; mirror < mirrors_; ++mirror) { + status = transport_->write_fixed_mirror(fixed, mirror, 0, framed->bytes.data(), framed->bytes.size(), context); + if (!status.ok()) { + return status; + } + } + status = transport_->advance_write(fixed, 0, framed->bytes.size()); + if (!status.ok()) { + return status; + } + status = transport_->seal_chunk(fixed, epoch_, framed->bytes.size()); + if (!status.ok()) { + return status; + } + auto reloadable = writing; + reloadable.revision++; + reloadable.state = ChunkStageState::Reloadable; + status = catalog_->compare_stage(tree_, epoch_, writing, reloadable); + if (!status.ok()) { + return status; + } + *out = {.stage = operation, + .chunk = fixed, + .offset = 0, + .length = writing.packs.front().length, + .checksum = writing.packs.front().checksum, + .purpose = purpose}; + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status ChunkFixedPackStore::read(const ChunkFixedPackRef &ref, const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<const ChunkPackWindow> *out) const +try { + if (!budget || out == nullptr || *out || !read_allocation_bound(ref.length)) { + return Status::invalid_argument(); + } + auto result = std::allocate_shared<ChunkPackWindow>(MetadataAllocator<ChunkPackWindow>(budget), budget); + result->bytes.resize(ref.length); + auto status = read_into(ref, budget, result->bytes); + if (status.ok()) { + *out = std::move(result); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status ChunkFixedPackStore::read_into(const ChunkFixedPackRef &ref, const std::shared_ptr<AllocationBudget> &budget, + std::span<uint8_t> output) const +try { + const auto context = ChunkIoContext::capture(); + if (!transport_ || !budget || output.size() != ref.length || !read_into_allocation_bound(ref.length)) { + return Status::invalid_argument(); + } + auto framed = std::allocate_shared<ChunkPackWindow>(MetadataAllocator<ChunkPackWindow>(budget), budget); + framed->bytes.resize(chunk_framed_length(ref.length)); + if (ref.offset > UINT64_MAX - framed->bytes.size()) { + return Status::corruption(); + } + bool responded = false; + for (uint32_t mirror = 0; mirror < mirrors_; ++mirror) { + auto status = transport_->read_fixed_mirror(ref.chunk, mirror, ref.offset, framed->bytes.data(), + framed->bytes.size(), context); + if (!status.ok()) { + continue; + } + responded = true; + status = decode_fixed_pack(ref, framed->bytes, output); + if (status.ok()) { + return status; + } + } + return responded ? Status::corruption("no mirror supplied the fixed pack") + : Status::unavailable("fixed pack is unavailable on every mirror"); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.h b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.h new file mode 100644 index 000000000..43e414a89 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.h @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/allocation_reservation.h" +#include "chunk_page_store.h" + +#include <span> + +namespace crowdb::tree +{ +class ChunkPhysicalRegistry; +struct ChunkPhysicalPack; +} // namespace crowdb::tree + +namespace crowdb::tree::detail +{ +// Immutable identity carried by each page; readers never consult the active +// manifest. Stage/manifest ownership is held separately from the byte window. +struct ChunkFixedPackRef +{ + ChunkStageId stage; + ChunkId chunk; + uint64_t offset = 0; + uint32_t length = 0; + uint32_t checksum = 0; + PagePurpose purpose = PagePurpose::kInvalid; +}; + +struct ChunkPackWindow +{ + explicit ChunkPackWindow(const std::shared_ptr<AllocationBudget> &budget) + : bytes(std::vector<uint8_t, BudgetAllocator<uint8_t, AllocationBudget, AllocationKind::kIo>>::allocator_type( + budget)) + { + } + + std::vector<uint8_t, BudgetAllocator<uint8_t, AllocationBudget, AllocationKind::kIo>> bytes; +}; + +// Blocking finite backend steps. Call only on the host's bounded backend lane; +// no Tree CPU task may wait for catalog or transport completion. +class ChunkFixedPackStore +{ + public: + ChunkFixedPackStore(uint64_t tree, uint64_t epoch, uint32_t mirrors, std::shared_ptr<RootCatalog> catalog, + std::shared_ptr<ChunkTransport> transport, + std::shared_ptr<ChunkPhysicalRegistry> registry = {}); + [[nodiscard]] bool supported() const noexcept; + [[nodiscard]] static std::optional<size_t> write_allocation_bound(size_t bytes) noexcept; + [[nodiscard]] static std::optional<size_t> read_allocation_bound(size_t bytes) noexcept; + [[nodiscard]] static std::optional<size_t> read_into_allocation_bound(size_t bytes) noexcept; + Status write(ChunkStageId operation, ChunkId fixed, PagePurpose purpose, std::span<const uint8_t> input, + const std::shared_ptr<AllocationBudget> &budget, ChunkFixedPackRef *out, + std::shared_ptr<ChunkPhysicalPack> *physical = nullptr, + const std::shared_ptr<AllocationBudget> &physical_budget = {}); + Status read(const ChunkFixedPackRef &ref, const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<const ChunkPackWindow> *out) const; + [[nodiscard]] Status read_into(const ChunkFixedPackRef &ref, const std::shared_ptr<AllocationBudget> &budget, + std::span<uint8_t> output) const; + + private: + uint64_t tree_; + uint64_t epoch_; + uint32_t mirrors_; + std::shared_ptr<RootCatalog> catalog_; + std::shared_ptr<ChunkTransport> transport_; + std::shared_ptr<ChunkPhysicalRegistry> registry_; +}; + +Status encode_fixed_pack(ChunkId id, PagePurpose purpose, std::span<const uint8_t> input, std::span<uint8_t> framed); +Status decode_fixed_pack(const ChunkFixedPackRef &ref, std::span<const uint8_t> framed, std::span<uint8_t> output); +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack_codec.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack_codec.cpp new file mode 100644 index 000000000..ccf6dfd74 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack_codec.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_fixed_pack.h" +#include "chunk_frame.h" +#include "crowdb-common/crc32c.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +namespace +{ +void little(std::span<uint8_t> out, uint64_t value) +{ + for (size_t index = 0; index < out.size(); ++index) { + out[index] = static_cast<uint8_t>(value >> (index * 8)); + } +} + +void big(std::span<uint8_t> out, uint64_t value) +{ + for (size_t index = 0; index < out.size(); ++index) { + out[index] = static_cast<uint8_t>(value >> ((out.size() - index - 1) * 8)); + } +} +} // namespace + +Status encode_fixed_pack(ChunkId id, PagePurpose purpose, std::span<const uint8_t> input, std::span<uint8_t> framed) +{ + using namespace crowdb::protocol; + if (input.empty() || input.size() > kMaxChunkPackBytes || !valid_page_purpose(purpose) || + chunk_purpose(id) != purpose || chunk_framed_length(input.size()) != framed.size()) { + return Status::invalid_argument("invalid bounded chunk pack"); + } + while (!input.empty()) { + const auto length = std::min(input.size(), kMaxFrameV2PayloadBytes); + const auto physical = kFrameV2HeaderBytes + length + kFrameFooterBytes; + auto frame = framed.first(physical); + little(frame.first(2), static_cast<uint16_t>(chunk_page_frame_magic(purpose))); + little(frame.subspan(2, 2), kFrameV2HeaderBytes); + little(frame.subspan(4, 2), length); + little(frame.subspan(6, 8), 0); + frame[14] = 0; + std::copy_n(input.begin(), length, frame.begin() + kFrameV2HeaderBytes); + const auto footer = kFrameV2HeaderBytes + length; + little(frame.subspan(footer, 4), crowdb::common::crc32c(frame.data(), footer)); + big(frame.subspan(footer + 4, 8), id.high); + big(frame.subspan(footer + 12, 8), id.low); + input = input.subspan(length); + framed = framed.subspan(physical); + } + return Status::Ok(); +} + +Status decode_fixed_pack(const ChunkFixedPackRef &ref, std::span<const uint8_t> framed, std::span<uint8_t> output) +{ + using namespace crowdb::protocol; + if (output.size() != ref.length || chunk_framed_length(ref.length) != framed.size() || + chunk_purpose(ref.chunk) != ref.purpose || !valid_page_purpose(ref.purpose)) { + return Status::corruption("fixed chunk pack identity or geometry mismatch"); + } + const auto whole = output; + while (!framed.empty()) { + ParsedFrame frame; + auto error = parse_frame(framed, {.high = ref.chunk.high, .low = ref.chunk.low}, &frame); + if (error != FrameError::Ok || frame.header.magic != chunk_page_frame_magic(ref.purpose) || + frame.payload.empty() || frame.payload.size() > output.size()) { + return Status::corruption("fixed chunk pack frame validation failed"); + } + std::copy(frame.payload.begin(), frame.payload.end(), output.begin()); + output = output.subspan(frame.payload.size()); + framed = framed.subspan(frame.physical_length); + } + if (!output.empty() || crowdb::common::crc32c(whole.data(), whole.size()) != ref.checksum) { + return Status::corruption("fixed chunk pack payload checksum mismatch"); + } + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_io_context.h b/lib/crowdb-tree/src/backend/chunk/chunk_io_context.h new file mode 100644 index 000000000..7cf445958 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_io_context.h @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include <chrono> +#include <cstdint> + +namespace crowdb::tree::detail +{ +// Captured before authority/admission and reused through route lookup, all +// strips, mirrors and retries. Queueing must never create a new request age. +struct ChunkIoContext +{ + uint64_t created_ms = 0; + + static ChunkIoContext capture() noexcept + { + const auto now = + std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()) + .count(); + return {.created_ms = now > 0 ? static_cast<uint64_t>(now) : 0}; + } +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_legacy_runtime.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_legacy_runtime.cpp new file mode 100644 index 000000000..ad70d5bff --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_legacy_runtime.cpp @@ -0,0 +1,13 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_page_store.h" + +namespace crowdb::tree::detail +{ +PageStore::RuntimeFactory ChunkPageStore::runtime_factory() const noexcept +{ + return [](const crowdb::tree::Config &, std::shared_ptr<TreeCore> *) { + return Status::not_supported("legacy ChunkPageStore requires migrate_legacy_chunk"); + }; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_migration_c_api.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_migration_c_api.cpp new file mode 100644 index 000000000..3c6190085 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_migration_c_api.cpp @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/chunk_migration_c_api.h" + +#include "c_api_internal.h" +#include "crowdb-tree/btree/tree.h" + +ct_status ct_migrate_legacy_chunk_with_runtime(const ct_options *options, ct_runtime *runtime, + const ct_tree_runtime_options *binding, uint64_t generation, + ct_tree **out) +{ + return ct_migrate_configured(options, runtime, binding, generation, &crowdb::tree::Crowdbtree::migrate_legacy_chunk, + out); +} diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp index d07b4925d..6224d1300 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp @@ -160,7 +160,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable : store(owner), expected_generation(generation), cancellation(cancel), - completion(on_complete) + completion(std::move(on_complete)) { } diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp index a3eecd15b..cf84ae167 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp @@ -11,6 +11,7 @@ #include "chunk_catalog_codec.h" #include "chunk_frame.h" #include "chunk_pack_pipeline.h" +#include "chunk_runtime_page_store.h" #include "crowdb-common/crc32c.h" #include "crowdb-common/log.h" #include "crowdb-protocol/frame.h" @@ -432,16 +433,17 @@ Status MemoryRootCatalog::unpin_generation(uint64_t tree_id, uint64_t transition if (state == nullptr) { return Status::Ok(); } - auto next = std::make_shared<CatalogState>(*state); - next->pins.erase(std::remove_if(next->pins.begin(), next->pins.end(), - [tree_id, transition_high, transition_low](const auto &pin) { - return pin.tree_id == tree_id && pin.transition_high == transition_high && - pin.transition_low == transition_low; - }), - next->pins.end()); - if (next->pins.size() == state->pins.size()) { + const auto pinned = std::find_if(state->pins.begin(), state->pins.end(), + [tree_id, transition_high, transition_low](const auto &pin) { + return pin.tree_id == tree_id && pin.transition_high == transition_high && + pin.transition_low == transition_low; + }); + if (pinned == state->pins.end()) { return Status::Ok(); } + const auto index = pinned - state->pins.begin(); + auto next = std::make_shared<CatalogState>(*state); + next->pins.erase(next->pins.begin() + index); if (state_.compare_exchange_weak(state, next, std::memory_order_release, std::memory_order_acquire)) { return Status::Ok(); } @@ -833,7 +835,7 @@ bool ChunkPageStore::has_shared_ownership() const return false; } return std::any_of(manifest->packs.begin(), manifest->packs.end(), - [this, &manifest](const ChunkPagePack &pack) { return pack.owner_tree_id != config_.tree_id; }); + [this](const ChunkPagePack &pack) { return pack.owner_tree_id != config_.tree_id; }); } bool ChunkPageStore::inherited_snapshot_matches(const PageStore &source_store) const @@ -1649,6 +1651,9 @@ ct_status ct_chunk_page_store_estimated_bytes(const ct_page_store *store, uint64 if (store == nullptr || store->bundle == nullptr || out == nullptr) { return static_cast<ct_status>(crowdb::tree::Code::kInvalidArgument); } + if (dynamic_cast<crowdb::tree::detail::ChunkRuntimePageStore *>(store->bundle->store.get()) != nullptr) { + return static_cast<ct_status>(crowdb::tree::Code::kNotSupported); + } auto *chunk = dynamic_cast<crowdb::tree::detail::ChunkPageStore *>(store->bundle->store.get()); if (chunk == nullptr) { return static_cast<ct_status>(crowdb::tree::Code::kInvalidArgument); @@ -1662,6 +1667,9 @@ ct_status ct_chunk_page_store_get_stats(const ct_page_store *store, ct_chunk_pag if (store == nullptr || store->bundle == nullptr || out == nullptr) { return static_cast<ct_status>(crowdb::tree::Code::kInvalidArgument); } + if (dynamic_cast<crowdb::tree::detail::ChunkRuntimePageStore *>(store->bundle->store.get()) != nullptr) { + return static_cast<ct_status>(crowdb::tree::Code::kNotSupported); + } auto *chunk = dynamic_cast<crowdb::tree::detail::ChunkPageStore *>(store->bundle->store.get()); if (chunk == nullptr) { return static_cast<ct_status>(crowdb::tree::Code::kInvalidArgument); @@ -1708,6 +1716,10 @@ ct_status ct_chunk_page_store_set_wal_replay_offset(ct_page_store *store, uint64 if (store == nullptr || store->bundle == nullptr) { return static_cast<ct_status>(crowdb::tree::Code::kInvalidArgument); } + if (auto *runtime = dynamic_cast<crowdb::tree::detail::ChunkRuntimePageStore *>(store->bundle->store.get())) { + runtime->request_wal_offset(offset); + return 0; + } auto *chunk = dynamic_cast<crowdb::tree::detail::ChunkPageStore *>(store->bundle->store.get()); return chunk == nullptr ? static_cast<ct_status>(crowdb::tree::Code::kOk) : static_cast<ct_status>(chunk->set_wal_replay_offset(offset).code()); @@ -1718,6 +1730,11 @@ ct_status ct_chunk_page_store_get_wal_replay_offset(const ct_page_store *store, if (store == nullptr || store->bundle == nullptr || offset == nullptr) { return static_cast<ct_status>(crowdb::tree::Code::kInvalidArgument); } + if (const auto *runtime = + dynamic_cast<const crowdb::tree::detail::ChunkRuntimePageStore *>(store->bundle->store.get())) { + *offset = runtime->durable_wal_offset(); + return 0; + } auto *chunk = dynamic_cast<crowdb::tree::detail::ChunkPageStore *>(store->bundle->store.get()); if (chunk == nullptr) { *offset = 0; @@ -1731,6 +1748,11 @@ ct_status ct_chunk_page_store_get_manifest_generation(const ct_page_store *store if (store == nullptr || store->bundle == nullptr || generation == nullptr) { return static_cast<ct_status>(crowdb::tree::Code::kInvalidArgument); } + if (const auto *runtime = + dynamic_cast<const crowdb::tree::detail::ChunkRuntimePageStore *>(store->bundle->store.get())) { + *generation = runtime->manifest_generation(); + return 0; + } const auto *chunk = dynamic_cast<const crowdb::tree::detail::ChunkPageStore *>(store->bundle->store.get()); return chunk == nullptr ? static_cast<ct_status>(crowdb::tree::Code::kInvalidArgument) : static_cast<ct_status>(chunk->manifest_generation(generation).code()); diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h index 762a94bf5..7a88684d5 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h @@ -7,11 +7,15 @@ #include "chunk_cancellation.h" #include "chunk_page_staging.h" +#include "chunk_protection.h" #include "chunk_purpose.h" +#include "chunk_stage.h" #include "chunk_transport.h" #include "crowdb-tree/backend/async_page_store.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/c_api.h" +#include "crowdb-tree/chunk_protection_c_api.h" +#include "crowdb-tree/chunk_stage_c_api.h" #include <array> #include <atomic> @@ -27,6 +31,7 @@ namespace crowdb::tree::detail inline constexpr uint32_t kChunkManifestFormat = 7; class ChunkAsyncExecutor; +class ChunkPinReleaseQueue; class ChunkPackPipeline; class ChunkPackPipelineImpl; @@ -86,6 +91,61 @@ class RootCatalog public: virtual ~RootCatalog() = default; + // One catalog callback owner is one writer incarnation. A new open after + // close must claim a fresh catalog; delayed quiescence cannot close it. + bool claim_runtime_writer() noexcept + { + bool expected = false; + return runtime_writer_claimed_.compare_exchange_strong(expected, true, std::memory_order_acq_rel); + } + + [[nodiscard]] virtual std::shared_ptr<ChunkPinReleaseQueue> pin_release_queue() const noexcept + { + return {}; + } + + virtual Status bind_pin_release_queue(std::shared_ptr<ChunkPinReleaseQueue> /*unused*/) + { + return Status::not_supported(); + } + + [[nodiscard]] virtual ChunkProtectionCatalog *protection() noexcept + { + return nullptr; + } + + [[nodiscard]] virtual bool stage_schema_supported() const noexcept + { + return false; + } + + virtual Status load_stage(uint64_t /*tree_id*/, ChunkStageId /*id*/, + std::optional<ChunkStageRecord> * /*out*/) const + { + return Status::not_supported(); + } + + virtual Status compare_stage(uint64_t /*tree_id*/, uint64_t /*owner_epoch*/, + const std::optional<ChunkStageRecord> & /*expected*/, + const std::optional<ChunkStageRecord> & /*next*/) + { + return Status::not_supported(); + } + + virtual Status list_stages(uint64_t /*tree_id*/, std::optional<ChunkStageId> /*after*/, uint32_t /*limit*/, + std::vector<ChunkStageRecord> * /*out*/) const + { + return Status::not_supported(); + } + + // Returns one bounded record using the caller's actual metadata allocator. + virtual Status next_stage(uint64_t /*tree_id*/, std::optional<ChunkStageId> /*after*/, + const std::shared_ptr<AllocationBudget> & /*budget*/, + std::optional<ChunkStageRecord> * /*out*/) const + { + return Status::not_supported(); + } + [[nodiscard]] virtual std::shared_ptr<const ChunkManifest> load(uint64_t tree_id) const = 0; [[nodiscard]] virtual std::shared_ptr<const ChunkManifest> load_generation(uint64_t tree_id, uint64_t generation) const = 0; @@ -104,6 +164,27 @@ class RootCatalog [[nodiscard]] virtual uint64_t retained_manifest_count(uint64_t tree_id) const = 0; [[nodiscard]] virtual uint64_t pinned_bytes(uint64_t tree_id) const = 0; [[nodiscard]] virtual uint64_t oldest_pin_age_ms(uint64_t tree_id) const = 0; + + virtual Status load_legacy_reference_owned(uint64_t /*tree*/, uint64_t /*object*/, ChunkRootBytes * /*out*/) const + { + return Status::not_supported(); + } + + // Immutable copy of the original root, persisted before migration commits. + virtual Status prepare_legacy_migration(uint64_t /*tree*/, uint64_t /*generation*/, + std::span<const uint8_t> /*root*/) + { + return Status::not_supported(); + } + + virtual Status load_legacy_migration_owned(uint64_t /*tree*/, uint64_t /*generation*/, + ChunkRootBytes * /*out*/) const + { + return Status::not_supported(); + } + + private: + std::atomic<bool> runtime_writer_claimed_{false}; }; // Test and embeddable catalog implementation. Its read side is one atomic @@ -187,10 +268,60 @@ class MemoryRootCatalog final : public RootCatalog std::atomic<bool> release_publish_{false}; }; -class CallbackRootCatalog final : public RootCatalog +class CallbackRootCatalog final : public RootCatalog, public ChunkProtectionCatalog { public: + [[nodiscard]] std::shared_ptr<ChunkPinReleaseQueue> pin_release_queue() const noexcept override + { + return release_queue_; + } + + // Host initialization only, before this catalog is exposed to a Tree. + Status bind_pin_release_queue(std::shared_ptr<ChunkPinReleaseQueue> queue) override + { + if (!queue || (release_queue_ && release_queue_ != queue)) { + return Status::invalid_argument(); + } + release_queue_ = std::move(queue); + return Status::Ok(); + } + CallbackRootCatalog(ct_root_catalog_callbacks callbacks, void *context); + CallbackRootCatalog(ct_root_catalog_callbacks, ct_root_catalog_staging_callbacks, void *); + CallbackRootCatalog(ct_root_catalog_callbacks, ct_root_catalog_staging_callbacks, + ct_root_catalog_protection_callbacks, void *); + [[nodiscard]] ChunkProtectionCatalog *protection() noexcept override; + [[nodiscard]] Status check_read_owner() const override; + Status writer_quiesced(uint64_t tree, uint64_t epoch) override; + Status load_root_bytes(uint64_t tree, uint64_t generation, std::vector<uint8_t> *out) const override; + Status load_root_owned(uint64_t tree, uint64_t generation, ChunkRootBytes *out) const override; + Status prepare_legacy_migration(uint64_t tree, uint64_t generation, std::span<const uint8_t> manifest) override; + Status load_legacy_migration_owned(uint64_t tree, uint64_t generation, ChunkRootBytes *out) const override; + Status load_legacy_reference_owned(uint64_t tree, uint64_t object, ChunkRootBytes *out) const override; + Status begin_candidate(uint64_t tree, const ChunkManifestCandidate & /*unused*/) override; + Status load_candidate(uint64_t tree, ChunkStageId /*unused*/, + std::optional<ChunkCandidateStatus> *out) const override; + Status list_candidates(uint64_t tree, std::optional<ChunkStageId> after, uint32_t limit, + std::vector<ChunkCandidateStatus> *out) const override; + Status prepare_references(uint64_t tree, ChunkStageId /*unused*/, uint64_t window, + std::span<const ChunkStageId> /*unused*/) override; + Status finish_candidate(uint64_t tree, ChunkStageId /*unused*/, uint32_t action) override; + Status release_references(uint64_t tree, ChunkStageId /*unused*/, uint64_t window) override; + Status change_pin(uint64_t tree, ChunkStageId stage, ChunkStageId pin, bool acquire) override; + Status prune_pins(uint64_t tree, ChunkStageId /*unused*/, std::optional<ChunkStageId> after, uint32_t limit, + ChunkPinProgress *out) override; + [[nodiscard]] bool stage_schema_supported() const noexcept override; + Status load_stage(uint64_t tree_id, ChunkStageId id, std::optional<ChunkStageRecord> *out) const override; + Status compare_stage(uint64_t tree_id, uint64_t owner_epoch, const std::optional<ChunkStageRecord> &expected, + const std::optional<ChunkStageRecord> &next) override; + Status list_stages(uint64_t tree_id, std::optional<ChunkStageId> after, uint32_t limit, + std::vector<ChunkStageRecord> *out) const override; + Status next_stage(uint64_t tree_id, std::optional<ChunkStageId> after, + const std::shared_ptr<AllocationBudget> &budget, + std::optional<ChunkStageRecord> *out) const override; + Status next_candidate(uint64_t tree, std::optional<ChunkStageId> after, + std::optional<ChunkCandidateStatus> *out) const override; + ~CallbackRootCatalog() override; [[nodiscard]] std::shared_ptr<const ChunkManifest> load(uint64_t tree_id) const override; @@ -225,8 +356,11 @@ class CallbackRootCatalog final : public RootCatalog } private: - ct_root_catalog_callbacks callbacks_; - void *context_; + std::shared_ptr<ChunkPinReleaseQueue> release_queue_; + ct_root_catalog_protection_callbacks protection_{}; + ct_root_catalog_staging_callbacks staging_{}; + ct_root_catalog_callbacks callbacks_; + void *context_; }; struct ChunkPageStoreStats @@ -291,6 +425,8 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore ChunkPageStore(Config config, std::shared_ptr<RootCatalog> catalog, std::shared_ptr<ChunkTransport> transport); ~ChunkPageStore() override; + [[nodiscard]] RuntimeFactory runtime_factory() const noexcept override; + [[nodiscard]] const Status &open_status() const { return open_status_; @@ -327,7 +463,8 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore } [[nodiscard]] ChunkPageStoreStats stats() const; - // Approximate retained pack bytes from the already opened layout, with no I/O. + // Approximate retained pack bytes from the already opened layout, with no + // I/O. [[nodiscard]] uint64_t estimated_bytes() const; uint64_t reclaim_orphans(); Status materialize_ownership(uint64_t *bytes_written, bool *complete) override; @@ -353,6 +490,7 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore private: friend class ChunkAsyncExecutor; + class ChunkPinReleaseQueue; friend class ChunkPackPipeline; friend class ChunkPackPipelineImpl; friend class MemoryRootCatalog; diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry.cpp new file mode 100644 index 000000000..ad3f5098c --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry.cpp @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/backing_identity.h" +#include "btree/retired_shared.h" +#include "btree/tree_metrics.h" +#include "btree/tree_work_counters.h" +#include "chunk_frame.h" +#include "chunk_physical_registry_internal.h" + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kMaximumPacks = 1 << 24; +} + +Status ChunkPhysicalRegistry::create(const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkPhysicalRegistry> *out) +try { + if (!metadata || !metadata->retirement_queue() || out == nullptr || *out) { + return Status::invalid_argument("physical registry requires a paid host retirement lane"); + } + auto state = detail::retired_shared<State>(metadata, metadata->retirement_queue(), metadata->retirement_queue()); + state->backing_metrics = + std::allocate_shared<TreeResourceMetrics>(MetadataAllocator<TreeResourceMetrics>(metadata)); + state->backing_scope = detail::allocate_backend_identity(); + auto *value = MetadataAllocator<ChunkPhysicalRegistry>(metadata).allocate(1); + ::new (static_cast<void *>(value)) ChunkPhysicalRegistry(std::move(state)); + *out = {value, + [metadata](ChunkPhysicalRegistry *entry) noexcept { + std::destroy_at(entry); + MetadataAllocator<ChunkPhysicalRegistry>(metadata).deallocate(entry, 1); + }, + MetadataAllocator<ChunkPhysicalRegistry>(metadata)}; + return {}; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("physical registry metadata unavailable"); +} + +bool ChunkPhysicalRegistry::uses_queue(const detail::RetirementQueue *queue) const noexcept +{ + return state_->queue.get() == queue; +} + +size_t ChunkPhysicalRegistry::allocation_bound() noexcept +{ + return detail::retired_shared_allocation_bound<State>() + sizeof(ChunkPhysicalRegistry) + + detail::shared_allocation_bound<ChunkPhysicalRegistry>() + + detail::shared_allocation_bound<TreeResourceMetrics>(); +} + +const std::shared_ptr<TreeResourceMetrics> &ChunkPhysicalRegistry::backing_metrics() const noexcept +{ + return state_->backing_metrics; +} + +BackingStatsSnapshot ChunkPhysicalRegistry::backing_stats() const noexcept +{ + const auto value = state_->backing_metrics->snapshot(); + return {.scope_id = state_->backing_scope, .allocated_bytes = value.allocated_backing_bytes}; +} + +bool ChunkPhysicalRegistry::State::retire_step() noexcept +{ + for (size_t count = 0; count < 64 && !packs.empty(); ++count) { + packs.erase(packs.begin()); + } + return packs.empty(); +} + +Status ChunkPhysicalRegistry::State::execute(const std::shared_ptr<Request> &request) +{ + request->state = shared_from_this(); + request->queued_owner = request; + request->dispose = &State::dispatch; + request->retirement_budget = request->metadata; + queue->retire(request.get()); + while (!request->done.load(std::memory_order_acquire)) { + request->done.wait(false, std::memory_order_acquire); + } + return request->status; +} + +void ChunkPhysicalRegistry::State::abort(const std::shared_ptr<void> &owner) noexcept +{ + auto request = std::static_pointer_cast<Request>(owner); + request->queued_owner = request; + request->dispose = &State::dispatch; + request->retirement_budget = request->metadata; + request->state->queue->retire(request.get()); +} + +bool ChunkPhysicalRegistry::State::dispatch(detail::RetirementNode *node) noexcept +{ + auto request = std::move(static_cast<Request *>(node)->queued_owner); + try { + request->state->apply(*request); + } + catch (const std::bad_alloc &) { + request->status = Status::resource_exhausted("physical registry operation exceeded paid metadata"); + } + catch (...) { + request->status = Status::internal_error("physical registry operation failed"); + } + // Keep the paid command alive across both notification and the callback tail. + // RetirementQueue copied accounting before dispatch and does not read it again. + request->done.store(true, std::memory_order_release); + request->done.notify_all(); + return true; +} + +void ChunkPhysicalRegistry::State::apply(Request &request) +{ + const auto identity = key(request.stage, request.chunk); + auto found = packs.find(identity); + if (request.action == Action::Prepare) { + const bool inserted = found == packs.end(); + if (inserted) { + if (packs.size() == kMaximumPacks) { + request.status = Status::resource_exhausted("physical registry capacity exhausted"); + return; + } + detail::BudgetMemoryResource::Scope scope(memory, request.metadata); + auto owner = std::allocate_shared<ChunkPhysicalPack>(MetadataAllocator<ChunkPhysicalPack>(request.metadata), + request.stage, request.chunk); + found = packs.emplace(identity, std::move(owner)).first; + } + if (found->second->stage != request.stage) { + request.status = Status::corruption("physical Chunk identity already belongs to another stage"); + return; + } + request.pack = found->second; + return; + } + if (found == packs.end()) { + request.status = request.action == Action::Delete + ? Status::Ok() + : Status::unavailable("physical receipt is no longer registered"); + return; + } + auto &pack = *found->second; + if (pack.stage != request.stage || (request.pack && request.pack != found->second)) { + request.status = Status::corruption("physical receipt stage incarnation changed"); + return; + } + if (request.action == Action::Forget) { + if (!pack.receipt) { + packs.erase(found); + } + return; + } + if (request.receipt) { + const auto &receipt = *request.receipt; + if (receipt.chunk != pack.chunk || receipt.owner_epoch != pack.stage.owner_epoch || receipt.created_ms == 0 || + receipt.physical_bytes == 0 || + (pack.receipt && (pack.receipt->created_ms != receipt.created_ms || + pack.receipt->physical_bytes != receipt.physical_bytes))) { + request.status = Status::corruption("physical allocation receipt differs from its registered incarnation"); + return; + } + } + if (request.action == Action::Observe) { + if (!pack.receipt && request.receipt) { + pack.receipt = request.receipt; + pack.physical.activate(backing_metrics, request.receipt->physical_bytes); + } + } + else { + if (!pack.deleted && pack.receipt) { + record_work(request.metadata, TreeWorkCounter::BackingDeletedBytes, pack.receipt->physical_bytes, + TreeWorkCause::GarbageCollection); + } + pack.deleted = true; + pack.physical.mark_deleted(); + packs.erase(found); + } +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry.h b/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry.h new file mode 100644 index 000000000..53cd23510 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry.h @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/backing_metrics.h" +#include "chunk_fixed_pack.h" + +namespace crowdb::tree +{ +struct ChunkPhysicalPack; +struct BackingStatsSnapshot; +class TreeResourceMetrics; + +class ChunkPhysicalAdmission +{ + public: + ChunkPhysicalAdmission() = default; + + ~ChunkPhysicalAdmission() + { + if (rollback_) { + abort_(rollback_); + } + } + + ChunkPhysicalAdmission(const ChunkPhysicalAdmission &) = delete; + ChunkPhysicalAdmission &operator=(const ChunkPhysicalAdmission &) = delete; + + void accept() noexcept + { + rollback_.reset(); + } + + private: + friend class ChunkPhysicalRegistry; + std::shared_ptr<void> rollback_; + void (*abort_)(const std::shared_ptr<void> &) noexcept = nullptr; +}; + +// The stable host transport owns a canonical ledger per logical Tree across +// PageStore facades and writer reopen. +// Commands use the host retirement lane; only blocking backend/host callers may +// wait for them. No command performs I/O or walks the full allocation catalog. +class ChunkPhysicalRegistry +{ + struct State; + + public: + static Status create(const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkPhysicalRegistry> *out); + [[nodiscard]] BackingStatsSnapshot backing_stats() const noexcept; + [[nodiscard]] const std::shared_ptr<TreeResourceMetrics> &backing_metrics() const noexcept; + [[nodiscard]] bool uses_queue(const detail::RetirementQueue *queue) const noexcept; + [[nodiscard]] static size_t allocation_bound() noexcept; + [[nodiscard]] static size_t pack_allocation_bound() noexcept; + // Admission rolls back an unallocated pack if durable staging fails. + Status prepare(detail::ChunkStageId stage, detail::ChunkId chunk, const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkPhysicalPack> *out, ChunkPhysicalAdmission *admission = nullptr); + Status observe(const std::shared_ptr<ChunkPhysicalPack> &pack, detail::ChunkAllocationIdentity receipt, + const std::shared_ptr<AllocationBudget> &metadata); + // receipt is absent only after a durable Deleting record and an idempotent + // physical Delete acknowledgement. The stage identity must still match. + Status deleted(detail::ChunkStageId stage, detail::ChunkId chunk, + const std::optional<detail::ChunkAllocationIdentity> &receipt, + const std::shared_ptr<AllocationBudget> &metadata); + [[nodiscard]] static std::shared_ptr<BackingMetricExtent> + extent(const std::shared_ptr<ChunkPhysicalPack> &pack) noexcept; + + private: + explicit ChunkPhysicalRegistry(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry_api.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry_api.cpp new file mode 100644 index 000000000..1a0c76098 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry_api.cpp @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_frame.h" +#include "chunk_physical_registry_internal.h" + +namespace crowdb::tree +{ +Status ChunkPhysicalRegistry::prepare(detail::ChunkStageId stage, detail::ChunkId chunk, + const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkPhysicalPack> *out, ChunkPhysicalAdmission *admission) +try { + if ((admission != nullptr && admission->rollback_) || stage.owner_epoch == 0 || + (stage.operation_high == 0 && stage.operation_low == 0) || chunk.empty() || !metadata || out == nullptr || + *out || metadata->retirement_queue() != state_->queue) { + return Status::invalid_argument("physical pack requires the same paid host queue"); + } + auto request = std::allocate_shared<State::Request>(MetadataAllocator<State::Request>(metadata)); + request->action = State::Action::Prepare; + request->stage = stage; + request->chunk = chunk; + request->metadata = metadata; + std::shared_ptr<State::Request> rollback; + if (admission != nullptr) { + rollback = std::allocate_shared<State::Request>(MetadataAllocator<State::Request>(metadata)); + } + auto status = state_->execute(request); + if (status.ok()) { + if (admission != nullptr) { + rollback->state = state_; + rollback->action = State::Action::Forget; + rollback->stage = stage; + rollback->chunk = chunk; + rollback->pack = request->pack; + rollback->metadata = metadata; + admission->rollback_ = std::move(rollback); + admission->abort_ = &State::abort; + } + *out = std::move(request->pack); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("physical registry command credit unavailable"); +} + +Status ChunkPhysicalRegistry::observe(const std::shared_ptr<ChunkPhysicalPack> &pack, + detail::ChunkAllocationIdentity receipt, + const std::shared_ptr<AllocationBudget> &metadata) +try { + if (!pack || !metadata || metadata->retirement_queue() != state_->queue) { + return Status::invalid_argument(); + } + auto request = std::allocate_shared<State::Request>(MetadataAllocator<State::Request>(metadata)); + request->action = State::Action::Observe; + request->stage = pack->stage; + request->chunk = pack->chunk; + request->pack = pack; + request->receipt = receipt; + request->metadata = metadata; + return state_->execute(request); +} + +catch (const std::bad_alloc &) { + return Status::resource_exhausted("physical registry command credit unavailable"); +} + +Status ChunkPhysicalRegistry::deleted(detail::ChunkStageId stage, detail::ChunkId chunk, + const std::optional<detail::ChunkAllocationIdentity> &receipt, + const std::shared_ptr<AllocationBudget> &metadata) +try { + if (!metadata || metadata->retirement_queue() != state_->queue) { + return Status::invalid_argument(); + } + auto request = std::allocate_shared<State::Request>(MetadataAllocator<State::Request>(metadata)); + request->action = State::Action::Delete; + request->stage = stage; + request->chunk = chunk; + request->receipt = receipt; + request->metadata = metadata; + return state_->execute(request); +} + +catch (const std::bad_alloc &) { + return Status::resource_exhausted("physical registry command credit unavailable"); +} + +std::shared_ptr<BackingMetricExtent> +ChunkPhysicalRegistry::extent(const std::shared_ptr<ChunkPhysicalPack> &pack) noexcept +{ + return pack ? std::shared_ptr<BackingMetricExtent>(pack, &pack->physical) : nullptr; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry_internal.h b/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry_internal.h new file mode 100644 index 000000000..b1d3c43b8 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_physical_registry_internal.h @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/budget_memory_resource.h" +#include "chunk_physical_registry.h" + +#include <map> + +namespace crowdb::tree +{ +struct ChunkPhysicalPack +{ + ChunkPhysicalPack(detail::ChunkStageId stage, detail::ChunkId chunk) : stage(stage), chunk(chunk) + { + } + + const detail::ChunkStageId stage; + const detail::ChunkId chunk; + BackingMetricExtent physical; + std::optional<detail::ChunkAllocationIdentity> receipt; + bool deleted = false; +}; + +struct ChunkPhysicalRegistry::State : std::enable_shared_from_this<State> +{ + using Key = std::array<uint64_t, 3>; + enum class Action : uint8_t { Prepare, Observe, Delete, Forget }; + + struct Request : detail::RetirementNode + { + std::shared_ptr<State> state; + std::shared_ptr<Request> queued_owner; + Action action = Action::Prepare; + detail::ChunkStageId stage; + detail::ChunkId chunk; + std::shared_ptr<AllocationBudget> metadata; + std::shared_ptr<ChunkPhysicalPack> pack; + std::optional<detail::ChunkAllocationIdentity> receipt; + Status status; + std::atomic<bool> done{false}; + }; + + explicit State(std::shared_ptr<detail::RetirementQueue> queue) : queue(std::move(queue)) + { + } + + bool retire_step() noexcept; + Status execute(const std::shared_ptr<Request> &request); + void apply(Request &request); + static bool dispatch(detail::RetirementNode *node) noexcept; + static void abort(const std::shared_ptr<void> &owner) noexcept; + + static Key key(detail::ChunkStageId stage, detail::ChunkId chunk) noexcept + { + return {chunk.high, chunk.low, stage.owner_epoch}; + } + + std::shared_ptr<TreeResourceMetrics> backing_metrics; + uint64_t backing_scope = 0; + std::shared_ptr<detail::RetirementQueue> queue; + detail::BudgetMemoryResource memory; + std::pmr::map<Key, std::shared_ptr<ChunkPhysicalPack>> packs{&memory}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pin_release.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_pin_release.cpp new file mode 100644 index 000000000..24475a2c6 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pin_release.cpp @@ -0,0 +1,220 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_page_store.h" +#include "chunk_pin_release_internal.h" + +#include <cassert> + +namespace crowdb::tree::detail +{ + +ChunkPinOwner::~ChunkPinOwner() +{ + if (node_ != nullptr) { + auto keep = node_->queue; + keep->release(node_); + } +} + +void ChunkPinOwner::arm() noexcept +{ + assert(node_ != nullptr); + node_->armed = true; +} + +Status ChunkPinReleaseQueue::create(const std::shared_ptr<AllocationBudget> &budget, size_t capacity, + std::shared_ptr<const void> wake_owner, Wake wake, void *context, + std::shared_ptr<ChunkPinReleaseQueue> *out) +try { + if (!budget || capacity == 0 || capacity >= kClosed || !wake_owner || wake == nullptr || out == nullptr || *out) { + return Status::invalid_argument(); + } + auto queue = std::allocate_shared<ChunkPinReleaseQueue>(MetadataAllocator<ChunkPinReleaseQueue>(budget)); + queue->capacity_ = capacity; + queue->wake_owner_ = std::move(wake_owner); + queue->wake_ = wake; + queue->context_ = context; + *out = std::move(queue); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +ChunkPinReleaseQueue::~ChunkPinReleaseQueue() +{ + assert((state_.load(std::memory_order_relaxed) & ~kClosed) == 0); + assert(incoming_.load(std::memory_order_relaxed) == nullptr); + assert(active_ == nullptr && retry_ == nullptr); +} + +size_t ChunkPinReleaseQueue::allocation_bound() noexcept +{ + return shared_allocation_bound<ChunkPinReleaseQueue>(); +} + +Status ChunkPinReleaseQueue::prepare(const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<RootCatalog> catalog, uint64_t tree, ChunkStageId stage, + ChunkStageId pin, std::shared_ptr<ChunkPinOwner> *out) +{ + const auto valid = [](ChunkStageId id) { + return id.owner_epoch != 0 && (id.operation_high != 0 || id.operation_low != 0); + }; + if (!budget || !catalog || catalog->protection() == nullptr || tree == 0 || !valid(stage) || !valid(pin) || + out == nullptr || *out) { + return Status::invalid_argument(); + } + return prepare_node(budget, std::move(catalog), tree, stage, pin, false, out); +} + +Status ChunkPinReleaseQueue::prepare_writer(const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<RootCatalog> catalog, uint64_t tree, uint64_t epoch, + std::shared_ptr<ChunkPinOwner> *out) +{ + if (!budget || !catalog || catalog->protection() == nullptr || tree == 0 || epoch == 0 || out == nullptr || *out) { + return Status::invalid_argument(); + } + return prepare_node(budget, std::move(catalog), tree, {.owner_epoch = epoch}, {}, true, out); +} + +Status ChunkPinReleaseQueue::prepare_node(const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<RootCatalog> catalog, uint64_t tree, ChunkStageId stage, + ChunkStageId pin, bool writer, std::shared_ptr<ChunkPinOwner> *out) +{ + auto state = state_.load(std::memory_order_acquire); + do { + if ((state & kClosed) != 0) { + return Status::unavailable(); + } + if (state >= capacity_) { + denied_.fetch_add(1, std::memory_order_relaxed); + return Status::resource_exhausted(); + } + } while (!state_.compare_exchange_weak(state, state + 1, std::memory_order_acq_rel)); + ChunkPinReleaseNode *node = nullptr; + MetadataAllocator<ChunkPinReleaseNode> allocator(budget); + try { + auto owner = std::allocate_shared<ChunkPinOwner>(MetadataAllocator<ChunkPinOwner>(budget)); + node = allocator.allocate(1); + std::construct_at(node); + node->queue = shared_from_this(); + node->budget = budget; + node->catalog = std::move(catalog); + node->tree = tree; + node->stage = stage; + node->pin = pin; + node->writer = writer; + owner->node_ = node; + *out = std::move(owner); + return Status::Ok(); + } + catch (const std::bad_alloc &) { + if (node != nullptr) { + std::destroy_at(node); + allocator.deallocate(node, 1); + } + state_.fetch_sub(1, std::memory_order_release); + wake_(context_); + return Status::resource_exhausted(); + } +} + +void ChunkPinReleaseQueue::push(ChunkPinReleaseNode *node) noexcept +{ + auto *head = incoming_.load(std::memory_order_relaxed); + do { + node->next = head; + } while (!incoming_.compare_exchange_weak(head, node, std::memory_order_release, std::memory_order_relaxed)); +} + +void ChunkPinReleaseQueue::release(ChunkPinReleaseNode *node) noexcept +{ + if (!node->armed) { + finish(node); + return; + } + pending_.fetch_add(1, std::memory_order_relaxed); + push(node); + wake_(context_); +} + +void ChunkPinReleaseQueue::finish(ChunkPinReleaseNode *node) noexcept +{ + auto budget = node->budget; + std::destroy_at(node); + MetadataAllocator<ChunkPinReleaseNode>(budget).deallocate(node, 1); + state_.fetch_sub(1, std::memory_order_release); + wake_(context_); +} + +ChunkPinReleaseQueue::Step ChunkPinReleaseQueue::pump(size_t limit) noexcept +{ + if (consumer_.test_and_set(std::memory_order_acquire)) { + return {.completed = 0, .failed = 0, .more = true}; + } + Step result{.completed = 0, .failed = 0, .more = false}; + // Keep one finite arrival snapshot across steps. Failed operations wait in + // a separate batch, so they never repeatedly hide untouched older nodes. + if (active_ == nullptr) { + if (prefer_retry_ && retry_ != nullptr) { + active_ = std::exchange(retry_, nullptr); + prefer_retry_ = false; + } + else { + active_ = incoming_.exchange(nullptr, std::memory_order_acq_rel); + prefer_retry_ = true; + if (active_ == nullptr) { + active_ = std::exchange(retry_, nullptr); + prefer_retry_ = false; + } + } + } + for (size_t visited = 0; visited < limit && active_ != nullptr; ++visited) { + auto *node = active_; + active_ = node->next; + auto status = Status::internal_error(); + try { + auto *catalog = node->catalog->protection(); + status = node->writer ? catalog->writer_quiesced(node->tree, node->stage.owner_epoch) + : catalog->change_pin(node->tree, node->stage, node->pin, false); + } + catch (...) { + status = Status::internal_error(); + } + if (status.ok()) { + pending_.fetch_sub(1, std::memory_order_relaxed); + ++result.completed; + finish(node); + } + else { + node->next = retry_; + retry_ = node; + ++result.failed; + } + } + result.more = active_ != nullptr || retry_ != nullptr || incoming_.load(std::memory_order_acquire) != nullptr; + consumer_.clear(std::memory_order_release); + return result; +} + +void ChunkPinReleaseQueue::stop_admission() noexcept +{ + state_.fetch_or(kClosed, std::memory_order_acq_rel); + wake_(context_); +} + +ChunkPinReleaseQueue::Snapshot ChunkPinReleaseQueue::snapshot() const noexcept +{ + const auto state = state_.load(std::memory_order_acquire); + return {.capacity = capacity_, + .registered = state & ~kClosed, + .pending = pending_.load(std::memory_order_acquire), + .denied = denied_.load(std::memory_order_relaxed), + .closed = (state & kClosed) != 0}; +} + +bool ChunkPinReleaseQueue::drained() const noexcept +{ + return state_.load(std::memory_order_acquire) == kClosed; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pin_release.h b/lib/crowdb-tree/src/backend/chunk/chunk_pin_release.h new file mode 100644 index 000000000..11ea81008 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pin_release.h @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/allocation_bound.h" +#include "chunk_stage.h" + +#include <atomic> +#include <memory> + +namespace crowdb::tree::detail +{ +class RootCatalog; +class ChunkPinReleaseQueue; +struct ChunkPinReleaseNode; + +class ChunkPinOwner +{ + public: + ChunkPinOwner() = default; + ~ChunkPinOwner(); + ChunkPinOwner(const ChunkPinOwner &) = delete; + ChunkPinOwner &operator=(const ChunkPinOwner &) = delete; + // Arm before the first metadata call: an unknown reply still needs release. + void arm() noexcept; + + private: + friend class ChunkPinReleaseQueue; + ChunkPinReleaseNode *node_ = nullptr; +}; + +// This paid queue owns neither a Tree nor an executor. The host backend service +// pumps it while its transport and metadata clients remain alive. Dropping a +// pin only transfers its preallocated node; it performs no metadata operation. +class ChunkPinReleaseQueue : public std::enable_shared_from_this<ChunkPinReleaseQueue> +{ + public: + using Wake = void (*)(void *) noexcept; + + struct Snapshot + { + size_t capacity; + size_t registered; + size_t pending; + uint64_t denied; + bool closed; + }; + + struct Step + { + size_t completed; + size_t failed; + bool more; + }; + + static Status create(const std::shared_ptr<AllocationBudget> &budget, size_t capacity, + std::shared_ptr<const void> wake_owner, Wake wake, void *context, + std::shared_ptr<ChunkPinReleaseQueue> *out); + ~ChunkPinReleaseQueue(); + static size_t allocation_bound() noexcept; + static size_t pin_allocation_bound() noexcept; + Status prepare(const std::shared_ptr<AllocationBudget> &budget, std::shared_ptr<RootCatalog> catalog, uint64_t tree, + ChunkStageId stage, ChunkStageId pin, std::shared_ptr<ChunkPinOwner> *out); + // A writer session is irreversible. Call arm only after claiming its exact + // catalog incarnation; release after accepted backend work has drained. + Status prepare_writer(const std::shared_ptr<AllocationBudget> &budget, std::shared_ptr<RootCatalog> catalog, + uint64_t tree, uint64_t epoch, std::shared_ptr<ChunkPinOwner> *out); + // The single host backend consumer runs at most limit metadata operations. + // Failed operations retain their exact identity and credit for a later tick. + Step pump(size_t limit) noexcept; + void stop_admission() noexcept; + Snapshot snapshot() const noexcept; + bool drained() const noexcept; + + private: + friend class ChunkPinOwner; + Status prepare_node(const std::shared_ptr<AllocationBudget> &budget, std::shared_ptr<RootCatalog> catalog, + uint64_t tree, ChunkStageId stage, ChunkStageId pin, bool writer, + std::shared_ptr<ChunkPinOwner> *out); + void release(ChunkPinReleaseNode *node) noexcept; + void finish(ChunkPinReleaseNode *node) noexcept; + void push(ChunkPinReleaseNode *node) noexcept; + static constexpr size_t kClosed = size_t{1} << ((sizeof(size_t) * 8) - 1); + std::shared_ptr<const void> wake_owner_; + Wake wake_ = nullptr; + void *context_ = nullptr; + size_t capacity_ = 0; + std::atomic<size_t> state_{0}; + std::atomic<size_t> pending_{0}; + std::atomic<uint64_t> denied_{0}; + std::atomic<ChunkPinReleaseNode *> incoming_{nullptr}; + ChunkPinReleaseNode *active_ = nullptr; + ChunkPinReleaseNode *retry_ = nullptr; + bool prefer_retry_ = false; + std::atomic_flag consumer_ = ATOMIC_FLAG_INIT; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pin_release_internal.h b/lib/crowdb-tree/src/backend/chunk/chunk_pin_release_internal.h new file mode 100644 index 000000000..6389878ff --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pin_release_internal.h @@ -0,0 +1,20 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_pin_release.h" + +namespace crowdb::tree::detail +{ +struct ChunkPinReleaseNode +{ + std::shared_ptr<ChunkPinReleaseQueue> queue; + std::shared_ptr<AllocationBudget> budget; + std::shared_ptr<RootCatalog> catalog; + uint64_t tree = 0; + ChunkStageId stage{}; + ChunkStageId pin{}; + bool armed = false; + bool writer = false; + ChunkPinReleaseNode *next = nullptr; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pin_service_c_api.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_pin_service_c_api.cpp new file mode 100644 index 000000000..88e1be3e8 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pin_service_c_api.cpp @@ -0,0 +1,129 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/chunk_pin_service_c_api.h" + +#include "chunk_c_api_internal.h" +#include "chunk_pin_release.h" +#include "runtime/tree_runtime_c_api.h" + +using namespace crowdb::tree; +using namespace crowdb::tree::detail; + +struct ct_chunk_pin_service +{ + std::shared_ptr<AllocationBudget> budget; + std::shared_ptr<ChunkPinReleaseQueue> queue; +}; + +namespace +{ +struct WakeContext +{ + void (*wake)(void *) = nullptr; + void (*drop)(void *) = nullptr; + void *value = nullptr; + + ~WakeContext() + { + if (drop != nullptr) { + drop(value); + } + } + + static void invoke(void *context) noexcept + { + auto &self = *static_cast<WakeContext *>(context); + self.wake(self.value); + } +}; +} // namespace +ct_status ct_chunk_pin_service_create(ct_runtime *runtime, size_t capacity, void (*wake)(void *), + void (*drop_context)(void *), void *context, ct_chunk_pin_service **out) +try { + if (wake == nullptr || drop_context == nullptr || out == nullptr || *out != nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + std::shared_ptr<AllocationBudget> budget; + auto status = runtime_host_allocation_budget(runtime, &budget); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + auto callback = std::allocate_shared<WakeContext>(MetadataAllocator<WakeContext>(budget)); + callback->wake = wake; + callback->value = context; + std::shared_ptr<ChunkPinReleaseQueue> queue; + status = ChunkPinReleaseQueue::create(budget, capacity, callback, &WakeContext::invoke, callback.get(), &queue); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + MetadataAllocator<ct_chunk_pin_service> allocator(budget); + auto *handle = allocator.allocate(1); + std::construct_at(handle, ct_chunk_pin_service{.budget = budget, .queue = std::move(queue)}); + callback->drop = drop_context; + *out = handle; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +ct_status ct_root_catalog_bind_pin_service(ct_root_catalog *catalog, ct_chunk_pin_service *service) +{ + if (catalog == nullptr || !catalog->catalog || catalog->catalog->protection() == nullptr || service == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + if (service->queue->snapshot().closed) { + return static_cast<ct_status>(Code::kUnavailable); + } + return static_cast<ct_status>(catalog->catalog->bind_pin_release_queue(service->queue).code()); +} + +ct_chunk_pin_service_step ct_chunk_pin_service_pump(ct_chunk_pin_service *service, size_t limit) +{ + if (service == nullptr || limit == 0 || limit > 64) { + return {.completed = 0, .failed = 0, .more = 0, .drained = 0}; + } + const auto step = service->queue->pump(limit); + return {.completed = step.completed, + .failed = step.failed, + .more = static_cast<uint8_t>(step.more), + .drained = static_cast<uint8_t>(service->queue->drained())}; +} + +void ct_chunk_pin_service_stop(ct_chunk_pin_service *service) +{ + if (service != nullptr) { + service->queue->stop_admission(); + } +} + +ct_status ct_chunk_pin_service_resource_usage(const ct_chunk_pin_service *service, ct_resource_usage_v1 *out) +{ + if (service == nullptr || out == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + const auto value = service->queue->snapshot(); + *out = {.limit = value.capacity, + .used = value.registered, + .reserved = 0, + .allocated = value.registered, + .denied_tree = 0, + .denied_host = value.denied, + .waits = 0, + .wait_ns = 0, + .waiters = value.pending}; + return 0; +} + +void ct_chunk_pin_service_free(ct_chunk_pin_service *service) +{ + if (service == nullptr) { + return; + } + auto budget = service->budget; + std::destroy_at(service); + MetadataAllocator<ct_chunk_pin_service>(budget).deallocate(service, 1); +} diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_protection.h b/lib/crowdb-tree/src/backend/chunk/chunk_protection.h new file mode 100644 index 000000000..715bc5ca2 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_protection.h @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_stage.h" + +namespace crowdb::tree::detail +{ +// Leave four metadata keys for the final window's atomic release records. +inline constexpr size_t kMaxReferenceWindow = 60; +inline constexpr uint64_t kMaxCandidateReferences = 16 * 1024 * 1024; +inline constexpr size_t kMaxCandidateManifestBytes = 16 * 1024 * 1024; +enum class ChunkCandidateState : uint8_t { Preparing = 1, Ready, Committed, Aborted, Reclaimed }; + +struct ChunkCandidateStatus +{ + ChunkStageId id{}; + uint64_t generation = 0; + uint64_t reference_count = 0; + uint64_t prepared_references = 0; + uint64_t prepared_windows = 0; + uint64_t released_windows = 0; + ChunkCandidateState state = ChunkCandidateState::Preparing; +}; + +struct ChunkManifestCandidate +{ + ChunkStageId id{}; + uint64_t expected_generation = 0; + uint64_t reference_count = 0; + uint32_t reference_checksum = 0; + std::span<const uint8_t> manifest; +}; + +struct ChunkPinProgress +{ + std::optional<ChunkStageId> after; + uint32_t examined = 0; + uint32_t released = 0; + bool more = false; +}; + +// Accessed only while the owning RootCatalog is retained. Page backing owners +// hold that catalog independently of the live Tree and its mutable sources. +using ChunkRootBytes = std::vector<uint8_t, MetadataAllocator<uint8_t>>; + +class ChunkProtectionCatalog +{ + public: + virtual ~ChunkProtectionCatalog() = default; + [[nodiscard]] virtual Status check_read_owner() const = 0; + virtual Status writer_quiesced(uint64_t tree, uint64_t epoch) = 0; + virtual Status load_root_bytes(uint64_t tree, uint64_t generation, std::vector<uint8_t> *out) const = 0; + + virtual Status load_root_owned(uint64_t /*tree*/, uint64_t /*generation*/, ChunkRootBytes * /*out*/) const + { + return Status::not_supported(); + } + + virtual Status begin_candidate(uint64_t tree, const ChunkManifestCandidate &) = 0; + virtual Status load_candidate(uint64_t tree, ChunkStageId, std::optional<ChunkCandidateStatus> *out) const = 0; + virtual Status list_candidates(uint64_t tree, std::optional<ChunkStageId> after, uint32_t limit, + std::vector<ChunkCandidateStatus> *out) const = 0; + + virtual Status next_candidate(uint64_t /*tree*/, std::optional<ChunkStageId> /*after*/, + std::optional<ChunkCandidateStatus> * /*out*/) const + { + return Status::not_supported(); + } + + virtual Status prepare_references(uint64_t tree, ChunkStageId, uint64_t window, std::span<const ChunkStageId>) = 0; + virtual Status finish_candidate(uint64_t tree, ChunkStageId, uint32_t action) = 0; + virtual Status release_references(uint64_t tree, ChunkStageId, uint64_t window) = 0; + virtual Status change_pin(uint64_t tree, ChunkStageId stage, ChunkStageId pin, bool acquire) = 0; + virtual Status prune_pins(uint64_t tree, ChunkStageId, std::optional<ChunkStageId> after, uint32_t limit, + ChunkPinProgress *out) = 0; +}; + +uint32_t chunk_reference_checksum(uint32_t initial, std::span<const ChunkStageId> references) noexcept; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_protection_c_api.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_protection_c_api.cpp new file mode 100644 index 000000000..01fc3c9cd --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_protection_c_api.cpp @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_c_api_internal.h" +#include "chunk_protection_callback.h" + +ct_status ct_callback_root_catalog_open_v3(const ct_root_catalog_callbacks *callbacks, + const ct_root_catalog_staging_callbacks *staging, + const ct_root_catalog_protection_callbacks *protection, void *context, + ct_root_catalog **out) +try { + using namespace crowdb::tree; + using namespace crowdb::tree::detail; + if (callbacks == nullptr || callbacks->load == nullptr || callbacks->free_blob == nullptr || + callbacks->store == nullptr || callbacks->publish == nullptr || + callbacks->allocate_reference_segment_id == nullptr || staging == nullptr || staging->version != 1 || + staging->struct_bytes != sizeof(*staging) || staging->capabilities != CT_CHUNK_STAGE_SCHEMA_V1 || + staging->load == nullptr || staging->compare == nullptr || staging->list == nullptr || protection == nullptr || + !protection_callback::supported(*protection) || out == nullptr || *out != nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + auto handle = std::make_unique<ct_root_catalog>(); + handle->transport = std::make_shared<MemoryChunkTransport>(); + handle->catalog = std::make_shared<CallbackRootCatalog>(*callbacks, *staging, *protection, context); + *out = handle.release(); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(crowdb::tree::Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(crowdb::tree::Code::kInternal); +} + +uint64_t ct_root_catalog_protection_capabilities(const ct_root_catalog *catalog) +{ + return (catalog != nullptr) && catalog->catalog && (catalog->catalog->protection() != nullptr) + ? CT_CHUNK_PROTECTION_SCHEMA_V1 + : 0; +} diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_protection_callback.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_protection_callback.cpp new file mode 100644 index 000000000..9780a5a9a --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_protection_callback.cpp @@ -0,0 +1,213 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_protection_callback.h" + +namespace crowdb::tree::detail +{ +namespace pc = protection_callback; + +CallbackRootCatalog::CallbackRootCatalog(ct_root_catalog_callbacks callbacks, ct_root_catalog_staging_callbacks staging, + ct_root_catalog_protection_callbacks protection, void *context) + : protection_(protection), + staging_(staging), + callbacks_(callbacks), + context_(context) +{ +} + +ChunkProtectionCatalog *CallbackRootCatalog::protection() noexcept +{ + return stage_schema_supported() && pc::supported(protection_) ? this : nullptr; +} + +Status CallbackRootCatalog::check_read_owner() const +{ + return pc::supported(protection_) ? pc::status(protection_.check_reader(context_)) : Status::not_supported(); +} + +Status CallbackRootCatalog::writer_quiesced(uint64_t tree, uint64_t epoch) +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || epoch == 0) { + return Status::invalid_argument(); + } + return pc::status(protection_.writer_quiesced(context_, tree, epoch)); +} + +namespace +{ +template <class Bytes> +Status load_root(const ct_root_catalog_callbacks &callbacks, const ct_root_catalog_protection_callbacks &protection, + void *context, uint64_t tree, uint64_t generation, Bytes *out, int kind = 0) +{ + if (!pc::supported(protection)) { + return Status::not_supported(); + } + if (tree == 0 || out == nullptr) { + return Status::invalid_argument(); + } + const uint8_t *data = nullptr; + size_t length = 0; + if (kind == 0) { + kind = generation == 0 ? CT_ROOT_CATALOG_CURRENT_MANIFEST : CT_ROOT_CATALOG_MANIFEST; + } + const auto result = callbacks.load(context, kind, tree, generation, &data, &length); + + struct Blob + { + const ct_root_catalog_callbacks &callbacks; + void *context; + const uint8_t *data; + size_t length; + + ~Blob() + { + if (data != nullptr) { + callbacks.free_blob(context, data, length); + } + } + } blob{.callbacks = callbacks, .context = context, .data = data, .length = length}; + + auto status = pc::status(result); + if (!status.ok()) { + return status; + } + if (data == nullptr || length == 0 || length > kMaxCandidateManifestBytes) { + return Status::corruption(); + } + out->assign(data, data + length); + return Status::Ok(); +} + +} // namespace + +Status CallbackRootCatalog::load_root_bytes(uint64_t tree, uint64_t generation, std::vector<uint8_t> *out) const +{ + return load_root(callbacks_, protection_, context_, tree, generation, out); +} + +Status CallbackRootCatalog::load_root_owned(uint64_t tree, uint64_t generation, ChunkRootBytes *out) const +try { + if (out == nullptr || !out->get_allocator().owner()) { + return Status::invalid_argument(); + } + return load_root(callbacks_, protection_, context_, tree, generation, out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status CallbackRootCatalog::load_legacy_reference_owned(uint64_t tree, uint64_t object, ChunkRootBytes *out) const +try { + if (object == 0 || out == nullptr || !out->get_allocator().owner()) { + return Status::invalid_argument(); + } + return load_root(callbacks_, protection_, context_, tree, object, out, CT_ROOT_CATALOG_REFERENCE_SEGMENT); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status CallbackRootCatalog::prepare_legacy_migration(uint64_t tree, uint64_t generation, + std::span<const uint8_t> manifest) +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || generation == 0 || manifest.empty() || manifest.size() > kMaxCandidateManifestBytes) { + return Status::invalid_argument(); + } + return pc::status(callbacks_.store(context_, CT_ROOT_CATALOG_LEGACY_MIGRATION, tree, generation, manifest.data(), + manifest.size())); +} + +Status CallbackRootCatalog::load_legacy_migration_owned(uint64_t tree, uint64_t generation, ChunkRootBytes *out) const +try { + if (generation == 0 || out == nullptr || !out->get_allocator().owner()) { + return Status::invalid_argument(); + } + return load_root(callbacks_, protection_, context_, tree, generation, out, CT_ROOT_CATALOG_LEGACY_MIGRATION); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status CallbackRootCatalog::begin_candidate(uint64_t tree, const ChunkManifestCandidate &candidate) +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || !pc::valid(candidate.id) || candidate.expected_generation == UINT64_MAX || + candidate.reference_count > kMaxCandidateReferences || candidate.manifest.empty() || + candidate.manifest.size() > kMaxCandidateManifestBytes || + (candidate.reference_count == 0 && candidate.reference_checksum != 0)) { + return Status::invalid_argument(); + } + return pc::status(protection_.begin(context_, tree, pc::external(candidate.id), candidate.expected_generation, + candidate.reference_count, candidate.reference_checksum, + candidate.manifest.data(), candidate.manifest.size())); +} + +Status CallbackRootCatalog::load_candidate(uint64_t tree, ChunkStageId id, + std::optional<ChunkCandidateStatus> *out) const +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || !pc::valid(id) || out == nullptr) { + return Status::invalid_argument(); + } + ct_chunk_candidate_status wire{}; + auto status = pc::status(protection_.load(context_, tree, pc::external(id), &wire)); + if (status.code() == Code::kNotFound) { + out->reset(); + return Status::Ok(); + } + if (!status.ok()) { + return status; + } + ChunkCandidateStatus candidate; + status = pc::decode(wire, &candidate); + if (!status.ok() || candidate.id != id) { + return Status::corruption(); + } + *out = candidate; + return Status::Ok(); +} + +Status CallbackRootCatalog::list_candidates(uint64_t tree, std::optional<ChunkStageId> after, uint32_t limit, + std::vector<ChunkCandidateStatus> *out) const +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || limit == 0 || limit > kMaxStageScan || out == nullptr || (after && !pc::valid(*after))) { + return Status::invalid_argument(); + } + std::array<ct_chunk_candidate_status, kMaxStageScan> wire{}; + size_t count = limit; + const auto cursor = pc::external(after.value_or(ChunkStageId{})); + auto status = pc::status(protection_.list(context_, tree, after ? &cursor : nullptr, limit, wire.data(), &count)); + if (!status.ok()) { + return status; + } + if (count > limit) { + return Status::corruption(); + } + std::vector<ChunkCandidateStatus> result; + result.reserve(count); + for (size_t i = 0; i < count; ++i) { + ChunkCandidateStatus value; + status = pc::decode(wire[i], &value); + if (!status.ok() || (after && value.id <= *after)) { + return Status::corruption(); + } + after = value.id; + result.push_back(value); + } + *out = std::move(result); + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_protection_callback.h b/lib/crowdb-tree/src/backend/chunk/chunk_protection_callback.h new file mode 100644 index 000000000..b814b97d4 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_protection_callback.h @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_page_store.h" + +namespace crowdb::tree::detail::protection_callback +{ +inline bool valid(ChunkStageId id) noexcept +{ + return id.owner_epoch != 0 && (id.operation_high != 0 || id.operation_low != 0); +} + +inline ct_chunk_stage_id external(ChunkStageId id) noexcept +{ + return {.owner_epoch = id.owner_epoch, .operation_high = id.operation_high, .operation_low = id.operation_low}; +} + +inline ChunkStageId internal(ct_chunk_stage_id id) noexcept +{ + return {.owner_epoch = id.owner_epoch, .operation_high = id.operation_high, .operation_low = id.operation_low}; +} + +inline Status status(ct_status code) +{ + switch (code) { + case 0: + return Status::Ok(); + case -1: + return Status::not_found(); + case -2: + return Status::invalid_argument(); + case -3: + return Status::corruption(); + case -4: + return Status::io_error(); + case -5: + return Status::not_supported(); + case -6: + return Status::internal_error(); + case -7: + return Status::resource_exhausted(); + default: + return Status::unavailable(); + } +} + +inline bool supported(const ct_root_catalog_protection_callbacks &c) noexcept +{ + return c.version == 1 && c.struct_bytes == sizeof(c) && c.capabilities == CT_CHUNK_PROTECTION_SCHEMA_V1 && + (c.check_reader != nullptr) && (c.writer_quiesced != nullptr) && (c.begin != nullptr) && + (c.load != nullptr) && (c.list != nullptr) && (c.prepare != nullptr) && (c.finish != nullptr) && + (c.release != nullptr) && (c.pin != nullptr) && (c.prune != nullptr); +} + +inline Status decode(const ct_chunk_candidate_status &wire, ChunkCandidateStatus *out) +{ + auto id = internal(wire.id); + if (!valid(id) || wire.state < 1 || wire.state > 5 || wire.generation == 0 || + wire.reference_count > kMaxCandidateReferences || wire.prepared_references > wire.reference_count || + wire.prepared_windows > wire.prepared_references || wire.released_windows > wire.prepared_windows) { + return Status::corruption(); + } + *out = {.id = id, + .generation = wire.generation, + .reference_count = wire.reference_count, + .prepared_references = wire.prepared_references, + .prepared_windows = wire.prepared_windows, + .released_windows = wire.released_windows, + .state = static_cast<ChunkCandidateState>(wire.state)}; + return Status::Ok(); +} +} // namespace crowdb::tree::detail::protection_callback diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_protection_mutation.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_protection_mutation.cpp new file mode 100644 index 000000000..97a01391c --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_protection_mutation.cpp @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_protection_callback.h" +#include "crowdb-common/crc32c.h" + +#include <array> + +namespace crowdb::tree::detail +{ +namespace pc = protection_callback; + +Status CallbackRootCatalog::prepare_references(uint64_t tree, ChunkStageId id, uint64_t window, + std::span<const ChunkStageId> references) +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || !pc::valid(id) || references.empty() || references.size() > kMaxReferenceWindow) { + return Status::invalid_argument(); + } + std::array<ct_chunk_stage_id, kMaxReferenceWindow> wire{}; + for (size_t i = 0; i < references.size(); ++i) { + if (!pc::valid(references[i]) || ((i != 0U) && references[i - 1] >= references[i])) { + return Status::invalid_argument(); + } + wire[i] = pc::external(references[i]); + } + return pc::status(protection_.prepare(context_, tree, pc::external(id), window, wire.data(), references.size())); +} + +Status CallbackRootCatalog::finish_candidate(uint64_t tree, ChunkStageId id, uint32_t action) +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || !pc::valid(id) || action < CT_CHUNK_CANDIDATE_COMMIT || action > CT_CHUNK_CANDIDATE_RECOVER) { + return Status::invalid_argument(); + } + return pc::status(protection_.finish(context_, tree, pc::external(id), action)); +} + +Status CallbackRootCatalog::release_references(uint64_t tree, ChunkStageId id, uint64_t window) +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || !pc::valid(id)) { + return Status::invalid_argument(); + } + return pc::status(protection_.release(context_, tree, pc::external(id), window)); +} + +Status CallbackRootCatalog::change_pin(uint64_t tree, ChunkStageId stage, ChunkStageId pin, bool acquire) +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || !pc::valid(stage) || !pc::valid(pin)) { + return Status::invalid_argument(); + } + return pc::status(protection_.pin(context_, tree, pc::external(stage), pc::external(pin), acquire ? 1 : 0)); +} + +Status CallbackRootCatalog::prune_pins(uint64_t tree, ChunkStageId stage, std::optional<ChunkStageId> after, + uint32_t limit, ChunkPinProgress *out) +{ + if (!pc::supported(protection_)) { + return Status::not_supported(); + } + if (tree == 0 || !pc::valid(stage) || limit == 0 || limit > kMaxStageScan || out == nullptr || + (after && !pc::valid(*after))) { + return Status::invalid_argument(); + } + const auto cursor = pc::external(after.value_or(ChunkStageId{})); + ct_chunk_pin_progress result{}; + auto status = + pc::status(protection_.prune(context_, tree, pc::external(stage), after ? &cursor : nullptr, limit, &result)); + if (!status.ok()) { + return status; + } + const auto next = pc::internal(result.after); + if (result.examined > limit || result.released > result.examined || result.has_after > 1 || result.more > 1 || + ((result.examined != 0) != static_cast<bool>(result.has_after)) || + ((result.more != 0U) && (result.has_after == 0U)) || + ((result.has_after != 0U) && (!pc::valid(next) || (after && next <= *after)))) { + return Status::corruption(); + } + *out = {.after = (result.has_after != 0U) ? std::optional(next) : std::nullopt, + .examined = result.examined, + .released = result.released, + .more = result.more != 0}; + return Status::Ok(); +} + +uint32_t chunk_reference_checksum(uint32_t initial, std::span<const ChunkStageId> references) noexcept +{ + for (const auto &id : references) { + for (const uint64_t word : {id.owner_epoch, id.operation_high, id.operation_low}) { + std::array<uint8_t, 8> bytes{}; + for (size_t i = 0; i < bytes.size(); ++i) { + bytes[i] = static_cast<uint8_t>(word >> (8 * i)); + } + initial = crowdb::common::crc32c_update(initial, bytes.data(), bytes.size()); + } + } + return initial; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.cpp new file mode 100644 index 000000000..dce7e9848 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.cpp @@ -0,0 +1,181 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_protocol_operation.h" + +#include "btree/allocation_bound.h" +#include "btree/apply_dispatcher.h" + +#include <cassert> + +namespace crowdb::tree +{ +struct ChunkProtocolOperation::State : std::enable_shared_from_this<State> +{ + explicit State(PageLoadBudget credit) : budget(std::move(credit)), io(budget.io, 1) + { + } + + PageLoadBudget budget; + AllocationLease io; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime::Progress> progress; + std::shared_ptr<ApplyDispatcher> dispatcher; + ApplyDispatcher::Node node{&State::run, this}; + std::shared_ptr<const void> owner; + Step step; + Done done; + std::atomic<uint8_t> lifecycle{0}; + std::atomic<bool> stopped{false}; + std::atomic<bool> ready{false}; + bool waiting = false; + Status event_status; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + + void wake() noexcept + { + if (lifecycle.load(std::memory_order_acquire) != 2) { + return; + } + auto self = weak_from_this().lock(); + assert(self); + if (dispatcher) { + dispatcher->request(node, std::move(self)); + } + else { + progress->request(std::move(self)); + } + } + + static void completed(void *context, Status status) + { + auto *self = static_cast<State *>(context); + self->event_status = std::move(status); + self->ready.store(true, std::memory_order_release); + self->wake(); + } + + static void run(void *context) noexcept + { + auto *self = static_cast<State *>(context); + if (self->lifecycle.load(std::memory_order_acquire) != 2) { + return; + } + try { + self->pump(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted()); + } + catch (...) { + self->finish(Status::internal_error()); + } + } + + void pump() + { + if (waiting) { + if (!ready.load(std::memory_order_acquire)) { + return; + } + waiting = false; + if (!event_status.ok()) { + finish(std::move(event_status)); + return; + } + if (done()) { + finish(Status::Ok()); + return; + } + } + auto self = shared_from_this(); + auto *scheduled = static_cast<ScheduledPageStore *>(store.get()); + waiting = true; + ready.store(false, std::memory_order_release); + auto status = scheduled->submit_step( + [self] { return self->step(self->stopped.load(std::memory_order_acquire)); }, + {.context = this, .complete_fn = &State::completed, .owner = self, .allocations = budget.bytes}); + if (!status.ok()) { + waiting = false; + finish(std::move(status)); + } + } + + void finish(Status status) noexcept + { + if (lifecycle.exchange(3, std::memory_order_acq_rel) == 3) { + return; + } + io = {}; + step = {}; + done = {}; + owner.reset(); + store.reset(); + complete(context, status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError, + status.without_message()); + } +}; + +size_t ChunkProtocolOperation::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<State>() + detail::shared_allocation_bound<ChunkProtocolOperation>() + + detail::FlushRuntime::progress_allocation_bound(); +} + +Status ChunkProtocolOperation::create(std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageLoadBudget budget, + std::shared_ptr<const void> owner, Step step, Done done, + std::shared_ptr<ChunkProtocolOperation> *out) +try { + if (!store || dynamic_cast<ScheduledPageStore *>(store.get()) == nullptr || !runtime || !lane || !budget.bytes || + !budget.io || !step || !done || out == nullptr || *out || budget.bytes->account() != budget.io->account() || + budget.io->category() != detail::BudgetClass::kIo) { + return Status::invalid_argument(); + } + auto state = detail::reserved_shared<State>(budget.bytes, budget); + state->store = std::move(store); + state->owner = std::move(owner); + state->step = std::move(step); + state->done = std::move(done); + state->dispatcher = budget.cpu_progress; + if (!state->dispatcher) { + auto status = runtime->reserve_progress(lane, &State::run, state.get(), budget.bytes, &state->progress); + if (!status.ok()) { + return status; + } + } + *out = detail::reserved_construct<ChunkProtocolOperation>(budget.bytes, [&](auto *storage) { + ::new (static_cast<void *>(storage)) ChunkProtocolOperation(std::move(state)); + }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +detail::CallbackSender ChunkProtocolOperation::sender() noexcept +{ + return {this, &ChunkProtocolOperation::submit}; +} + +void ChunkProtocolOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void ChunkProtocolOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto state = static_cast<ChunkProtocolOperation *>(source)->state_; + uint8_t idle = 0; + if (!state->lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + complete(context, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + state->complete = complete; + state->context = context; + state->lifecycle.store(2, std::memory_order_release); + state->wake(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.h b/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.h new file mode 100644 index 000000000..90c4c2f89 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.h @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/local/scheduled_page_store.h" +#include "btree/page_load.h" + +namespace crowdb::tree +{ +// A finite metadata protocol runs one blocking step per paid backend job. +// The protocol resolves cancellation/unknown results itself; the runner never +// drops an accepted mutation or destroys its context ahead of the callback. +class ChunkProtocolOperation +{ + struct State; + + public: + using Step = detail::InlineFunction<Status(bool)>; + using Done = detail::InlineFunction<bool()>; + static Status create(std::shared_ptr<AsyncPageStore> store, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageLoadBudget budget, + std::shared_ptr<const void> owner, Step step, Done done, + std::shared_ptr<ChunkProtocolOperation> *out); + [[nodiscard]] static size_t allocation_bound() noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + void request_stop() noexcept; + + private: + explicit ChunkProtocolOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *, detail::CallbackComplete, void *); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_runtime_c_api.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_c_api.cpp new file mode 100644 index 000000000..edbd33fd0 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_c_api.cpp @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/chunk_runtime_c_api.h" + +#include "c_api_internal.h" +#include "chunk_c_api_internal.h" +#include "chunk_runtime_page_store.h" +#include "runtime/tree_runtime_c_api.h" + +using namespace crowdb::tree; +using namespace crowdb::tree::detail; + +ct_status ct_chunk_runtime_page_store_open(ct_runtime *runtime, const ct_chunk_runtime_options *options, + ct_root_catalog *catalog, ct_chunk_transport *transport, ct_page_store **out) +try { + if (options == nullptr || options->version != 1 || options->struct_bytes != sizeof(*options) || + catalog == nullptr || !catalog->catalog || transport == nullptr || !transport->transport || out == nullptr || + *out != nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + std::shared_ptr<AllocationBudget> budget; + const auto status = runtime_host_allocation_budget(runtime, &budget); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + AllocationLease handle_credit(budget, sizeof(ct_page_store), AllocationKind::kMetadata); + AllocationLease store_credit(budget, sizeof(ChunkRuntimePageStore), AllocationKind::kMetadata); + auto handle = std::make_unique<ct_page_store>(); + handle->allocation = std::move(handle_credit); + handle->bundle = std::allocate_shared<PageStoreBundle>(MetadataAllocator<PageStoreBundle>(budget)); + auto store = std::make_unique<ChunkRuntimePageStore>( + ChunkRuntimeOptions{.tree = options->tree_id, + .owner_epoch = options->owner_epoch, + .open_generation = options->open_generation, + .iu_size = options->iu_size, + .mirrors = options->mirror_copies, + .max_chunk_bytes = options->max_chunk_bytes}, + catalog->catalog, transport->transport, std::move(store_credit), budget.get()); + if (!store->supported()) { + return static_cast<ct_status>(Code::kNotSupported); + } + handle->bundle->store = std::move(store); + handle->bundle->backend_label = "chunk-runtime"; + *out = handle.release(); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_runtime_ids.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_ids.cpp new file mode 100644 index 000000000..d1055a033 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_ids.cpp @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_runtime_ids.h" + +#include "chunk_purpose.h" + +#include <random> + +namespace crowdb::tree::detail +{ +ChunkRuntimeIds::ChunkRuntimeIds(uint64_t epoch, uint64_t high, uint64_t low) noexcept + : epoch_(epoch), + high_(high & 0x00FFFFFFFFFFFFFFULL), + low_(low & 0xFFFFFFFF00000000ULL) +{ +} + +Status ChunkRuntimeIds::create(uint64_t epoch, const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkRuntimeIds> *out) +try { + if (epoch == 0 || !metadata || out == nullptr || *out) { + return Status::invalid_argument(); + } + std::random_device random; + const auto high = (uint64_t{random()} << 32U) | random(); + const auto low = (uint64_t{random()} << 32U) | random(); + *out = std::allocate_shared<ChunkRuntimeIds>(MetadataAllocator<ChunkRuntimeIds>(metadata), epoch, high, low); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (const std::exception &) { + return Status::unavailable("Chunk operation namespace could not be initialized"); +} + +Status ChunkRuntimeIds::next(PagePurpose purpose, ChunkStageId *stage, ChunkId *chunk) noexcept +{ + if (!valid_page_purpose(purpose) || stage == nullptr || chunk == nullptr || epoch_ == 0) { + return Status::invalid_argument(); + } + auto value = next_.load(std::memory_order_relaxed); + do { + if (value > UINT32_MAX) { + return Status::resource_exhausted(); + } + } while (!next_.compare_exchange_weak(value, value + 1, std::memory_order_relaxed)); + *stage = {.owner_epoch = epoch_, .operation_high = high_, .operation_low = low_ | value}; + *chunk = {(uint64_t{static_cast<uint32_t>(purpose)} << 56U) | high_, low_ | value}; + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_runtime_ids.h b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_ids.h new file mode 100644 index 000000000..448e1d32b --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_ids.h @@ -0,0 +1,26 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_stage.h" +#include "chunk_transport.h" + +namespace crowdb::tree::detail +{ +// One random 88-bit namespace is prepared at open. Allocation on a CPU worker +// is a bounded atomic counter; exhaustion never wraps or starts a new +// namespace. +class ChunkRuntimeIds +{ + public: + ChunkRuntimeIds(uint64_t epoch, uint64_t high, uint64_t low) noexcept; + static Status create(uint64_t epoch, const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkRuntimeIds> *out); + Status next(PagePurpose purpose, ChunkStageId *stage, ChunkId *chunk) noexcept; + + private: + uint64_t epoch_; + uint64_t high_; + uint64_t low_; + std::atomic<uint64_t> next_{1}; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_runtime_page_store.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_page_store.cpp new file mode 100644 index 000000000..3866e9bf6 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_page_store.cpp @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_runtime_page_store.h" + +#include "btree/tree_core.h" +#include "chunk_frame.h" +#include "chunk_physical_registry.h" + +#include <bit> + +namespace crowdb::tree::detail +{ +ChunkRuntimePageStore::ChunkRuntimePageStore(ChunkRuntimeOptions options, std::shared_ptr<RootCatalog> catalog, + std::shared_ptr<ChunkTransport> transport, + const std::shared_ptr<AllocationBudget> &budget) + : ChunkRuntimePageStore(options, std::move(catalog), std::move(transport), + AllocationLease(budget, sizeof(ChunkRuntimePageStore), AllocationKind::kMetadata)) +{ +} + +ChunkRuntimePageStore::ChunkRuntimePageStore(ChunkRuntimeOptions options, std::shared_ptr<RootCatalog> catalog, + std::shared_ptr<ChunkTransport> transport, AllocationLease allocation, + const void *host_domain) + : allocation_(std::move(allocation)), + host_domain_(host_domain), + options_(options), + catalog_(std::move(catalog)), + transport_(std::move(transport)) +{ +} + +PageStore::RuntimeFactory ChunkRuntimePageStore::runtime_factory() const noexcept +{ + return &TreeCore::open_chunk; +} + +bool ChunkRuntimePageStore::supported() const noexcept +{ + return options_.tree != 0 && options_.owner_epoch != 0 && std::has_single_bit(options_.iu_size) && + options_.iu_size <= kMaxChunkPackBytes && options_.mirrors != 0 && options_.mirrors <= kMaxMirrorCopies && + options_.max_chunk_bytes <= kMaxChunkPackBytes && options_.max_chunk_bytes >= options_.iu_size && catalog_ && + catalog_->stage_schema_supported() && catalog_->protection() != nullptr && catalog_->pin_release_queue() && + transport_ && transport_->fixed_staging_supported(); +} + +Status ChunkRuntimePageStore::make_domain(uint64_t identity, const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<const ChunkBackingDomain> *out) const +try { + if (!supported()) { + return Status::not_supported("Chunk runtime needs fixed packs, protection " + "authority and a host release service"); + } + if (!budget || identity == 0 || out == nullptr || *out) { + return Status::invalid_argument(); + } + if (!physical_registry_) { + auto status = transport_->physical_registry(options_.tree, budget, &physical_registry_); + if (!status.ok()) { + return status; + } + } + else if (!physical_registry_->uses_queue(budget->retirement_queue().get())) { + return Status::invalid_argument("Chunk PageStore belongs to another host retirement domain"); + } + auto domain = std::allocate_shared<ChunkBackingDomain>(MetadataAllocator<ChunkBackingDomain>(budget)); + domain->backend_id = identity; + domain->tree = options_.tree; + domain->epoch = options_.owner_epoch; + domain->mirrors = options_.mirrors; + domain->max_chunk_bytes = options_.max_chunk_bytes; + domain->catalog = catalog_; + domain->transport = transport_; + domain->physical_registry = physical_registry_; + *out = std::move(domain); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status ChunkRuntimePageStore::write_at(uint64_t /*offset*/, const uint8_t * /*data*/, size_t /*length*/) +{ + return Status::not_supported("typed Chunk runtime has no mutable byte-address namespace"); +} + +Status ChunkRuntimePageStore::read_at(uint64_t /*offset*/, uint8_t * /*data*/, size_t /*length*/) const +{ + return Status::not_supported("typed Chunk runtime reads require exact pack identities"); +} + +Status ChunkRuntimePageStore::sync() +{ + return Status::not_supported("Chunk durability requires an exact root authority commit"); +} + +void ChunkRuntimePageStore::request_wal_offset(uint64_t value) noexcept +{ + auto previous = wal_requested_.load(std::memory_order_relaxed); + while (previous < value && !wal_requested_.compare_exchange_weak(previous, value, std::memory_order_release)) { + } +} + +void ChunkRuntimePageStore::committed(uint64_t generation, uint64_t wal_offset) noexcept +{ + // Serialized root recovery/publication supplies the already verified pair. + wal_committed_.store(wal_offset, std::memory_order_release); + generation_.store(generation, std::memory_order_release); + request_wal_offset(wal_offset); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_runtime_page_store.h b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_page_store.h new file mode 100644 index 000000000..f34b02083 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_runtime_page_store.h @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_backing.h" + +namespace crowdb::tree::detail +{ +struct ChunkRuntimeOptions +{ + uint64_t tree = 0; + uint64_t owner_epoch = 0; + uint64_t open_generation = 0; + uint32_t iu_size = 64 * 1024; + uint32_t mirrors = 2; + uint64_t max_chunk_bytes = 256 * 1024 * 1024; +}; + +// This injected handle only identifies the service and root namespace. It +// never opens the legacy manifest layout or starts a private executor. +class ChunkRuntimePageStore final : public PageStore +{ + public: + ChunkRuntimePageStore(ChunkRuntimeOptions options, std::shared_ptr<RootCatalog> catalog, + std::shared_ptr<ChunkTransport> transport, const std::shared_ptr<AllocationBudget> &budget); + + ChunkRuntimePageStore(ChunkRuntimeOptions options, std::shared_ptr<RootCatalog> catalog, + std::shared_ptr<ChunkTransport> transport, AllocationLease allocation, + const void *host_domain = nullptr); + + [[nodiscard]] const ChunkRuntimeOptions &options() const noexcept + { + return options_; + } + + [[nodiscard]] bool matches_host(const void *host_domain) const noexcept + { + return host_domain_ == nullptr || host_domain_ == host_domain; + } + + [[nodiscard]] const std::shared_ptr<RootCatalog> &catalog() const noexcept + { + return catalog_; + } + + [[nodiscard]] const std::shared_ptr<ChunkTransport> &transport() const noexcept + { + return transport_; + } + + [[nodiscard]] RuntimeFactory runtime_factory() const noexcept override; + [[nodiscard]] bool supported() const noexcept; + Status make_domain(uint64_t identity, const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<const ChunkBackingDomain> *out) const; + Status write_at(uint64_t offset, const uint8_t *data, size_t length) override; + Status read_at(uint64_t offset, uint8_t *data, size_t length) const override; + Status sync() override; + + [[nodiscard]] uint64_t size() const override + { + return 0; + } + + [[nodiscard]] uint32_t iu_size() const override + { + return options_.iu_size; + } + + [[nodiscard]] bool has_shared_ownership() const override + { + return true; + } + + void request_wal_offset(uint64_t value) noexcept; + + [[nodiscard]] uint64_t requested_wal_offset() const noexcept + { + return wal_requested_.load(std::memory_order_acquire); + } + + [[nodiscard]] uint64_t durable_wal_offset() const noexcept + { + return wal_committed_.load(std::memory_order_acquire); + } + + [[nodiscard]] uint64_t manifest_generation() const noexcept + { + return generation_.load(std::memory_order_acquire); + } + + void committed(uint64_t generation, uint64_t wal_offset) noexcept; + + private: + AllocationLease allocation_; + const void *host_domain_ = nullptr; + ChunkRuntimeOptions options_; + std::shared_ptr<RootCatalog> catalog_; + std::shared_ptr<ChunkTransport> transport_; + mutable std::shared_ptr<ChunkPhysicalRegistry> physical_registry_; + std::atomic<uint64_t> wal_requested_{0}; + std::atomic<uint64_t> wal_committed_{0}; + std::atomic<uint64_t> generation_{0}; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_stage.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_stage.cpp new file mode 100644 index 000000000..81440dfcd --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_stage.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_stage.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +Status validate_stage_record(const ChunkStageRecord &record) +{ + using State = ChunkStageState; + if (record.tree_id == 0 || record.id.owner_epoch == 0 || + (record.id.operation_high == 0 && record.id.operation_low == 0) || record.revision == 0 || + record.capacity == 0 || record.capacity > UINT32_MAX || (record.purpose != 2 && record.purpose != 3) || + record.chunk_high >> 56 != record.purpose || record.packs.size() > kMaxStagePacks || + record.state < State::Reserved || record.state > State::Reclaimed || + ((record.state == State::Reloadable || record.state == State::Referenced) && record.packs.empty()) || + (record.state == State::Referenced && record.manifest_generation == 0) || + (record.state <= State::Reloadable && record.manifest_generation != 0)) { + return Status::invalid_argument("invalid staged chunk identity, state, or geometry"); + } + uint64_t end = 0; + for (const auto &pack : record.packs) { + if (pack.length == 0 || pack.length > 256U * 1024U * 1024U || pack.offset < end || + pack.offset > record.capacity || pack.length > record.capacity - pack.offset) { + return Status::invalid_argument("invalid staged pack range"); + } + end = pack.offset + pack.length; + } + return Status::Ok(); +} + +Status validate_stage_transition(const std::optional<ChunkStageRecord> &expected, + const std::optional<ChunkStageRecord> &next) +{ + using State = ChunkStageState; + if ((expected && !validate_stage_record(*expected).ok()) || (next && !validate_stage_record(*next).ok())) { + return Status::invalid_argument("invalid staged transition record"); + } + if (!expected || !next) { + return ((!expected && next && next->revision == 1 && next->state == State::Reserved) || + (expected && !next && expected->state == State::Reclaimed)) + ? Status::Ok() + : Status::invalid_argument("invalid staged creation or cleanup"); + } + const auto &old = *expected; + const auto &value = *next; + if (old.tree_id != value.tree_id || old.id != value.id || old.chunk_high != value.chunk_high || + old.chunk_low != value.chunk_low || old.capacity != value.capacity || old.purpose != value.purpose || + old.revision == UINT64_MAX || value.revision != old.revision + 1 || value.packs.size() < old.packs.size() || + !std::equal(old.packs.begin(), old.packs.end(), value.packs.begin()) || + (value.packs != old.packs && old.state != State::Reserved && old.state != State::Writing) || + value.manifest_generation < old.manifest_generation) { + return Status::invalid_argument("staged transition changes immutable identity or refs"); + } + bool valid = false; + switch (old.state) { + case State::Reserved: + valid = value.state == State::Writing || value.state == State::Retired; + break; + case State::Writing: + valid = value.state == State::Writing || value.state == State::Reloadable || value.state == State::Retired; + break; + case State::Reloadable: + case State::Referenced: + valid = value.state == State::Referenced || value.state == State::Retired; + break; + case State::Retired: + valid = value.state == State::Deleting; + break; + case State::Deleting: + valid = value.state == State::Reclaimed; + break; + case State::Reclaimed: + break; + } + return valid ? Status::Ok() : Status::invalid_argument("invalid staged state transition"); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_stage.h b/lib/crowdb-tree/src/backend/chunk/chunk_stage.h new file mode 100644 index 000000000..e71ac412e --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_stage.h @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/memory_budget.h" +#include "crowdb-tree/status.h" + +#include <compare> +#include <cstdint> +#include <optional> +#include <span> +#include <vector> + +namespace crowdb::tree::detail +{ +inline constexpr size_t kMaxStagePacks = 1024; +inline constexpr uint32_t kMaxStageScan = 64; +inline constexpr size_t kMaxStageBytes = 100 + (16 * kMaxStagePacks); + +struct ChunkStageId +{ + uint64_t owner_epoch = 0; + uint64_t operation_high = 0; + uint64_t operation_low = 0; + auto operator<=>(const ChunkStageId &) const = default; +}; +enum class ChunkStageState : uint8_t { Reserved = 1, Writing, Reloadable, Referenced, Retired, Deleting, Reclaimed }; + +struct ChunkStagePack +{ + uint64_t offset = 0; + uint32_t length = 0; + uint32_t checksum = 0; + bool operator==(const ChunkStagePack &) const = default; +}; + +struct ChunkStageRecord +{ + uint64_t tree_id = 0; + ChunkStageId id; + uint64_t revision = 0; + uint64_t chunk_high = 0; + uint64_t chunk_low = 0; + uint64_t capacity = 0; + uint64_t manifest_generation = 0; + uint32_t purpose = 0; + ChunkStageState state = ChunkStageState::Reserved; + std::vector<ChunkStagePack, MetadataAllocator<ChunkStagePack>> packs; + bool operator==(const ChunkStageRecord &) const = default; +}; + +Status validate_stage_record(const ChunkStageRecord &record); +Status validate_stage_transition(const std::optional<ChunkStageRecord> &expected, + const std::optional<ChunkStageRecord> &next); +using ChunkStageBytes = std::vector<uint8_t, MetadataAllocator<uint8_t>>; +Status encode_stage_record(const ChunkStageRecord &, std::vector<uint8_t> *); +Status encode_stage_record_owned(const ChunkStageRecord &, ChunkStageBytes *); +Status decode_stage_record(std::span<const uint8_t>, ChunkStageRecord *); +Status encode_stage_list(std::span<const ChunkStageRecord>, std::vector<uint8_t> *); +Status decode_stage_list(std::span<const uint8_t>, std::vector<ChunkStageRecord> *); +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_stage_c_api.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_stage_c_api.cpp new file mode 100644 index 000000000..925b9874c --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_stage_c_api.cpp @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/chunk_stage_c_api.h" + +#include "chunk_c_api_internal.h" + +#include <cstdlib> +#include <cstring> + +namespace +{ +using namespace crowdb::tree; +using namespace crowdb::tree::detail; + +ct_status code(Status status) +{ + return static_cast<ct_status>(status.code()); +} + +ChunkStageId internal(ct_chunk_stage_id id) +{ + return {.owner_epoch = id.owner_epoch, .operation_high = id.operation_high, .operation_low = id.operation_low}; +} + +Status decode_optional(const uint8_t *data, size_t length, std::optional<ChunkStageRecord> *out) +{ + if (length == 0) { + out->reset(); + return Status::Ok(); + } + if (data == nullptr) { + return Status::invalid_argument(); + } + ChunkStageRecord record; + auto status = decode_stage_record({data, length}, &record); + if (status.ok()) { + *out = std::move(record); + } + return status; +} + +ct_status output(std::span<const uint8_t> bytes, ct_buf *out) +{ + auto *buffer = static_cast<uint8_t *>(std::malloc(bytes.size())); + if (buffer == nullptr) { + return code(Status::resource_exhausted()); + } + std::memcpy(buffer, bytes.data(), bytes.size()); + *out = {.data = buffer, .len = bytes.size()}; + return 0; +} +} // namespace +ct_status ct_callback_root_catalog_open_v2(const ct_root_catalog_callbacks *callbacks, + const ct_root_catalog_staging_callbacks *staging, void *context, + ct_root_catalog **out) +try { + if (callbacks == nullptr || callbacks->load == nullptr || callbacks->free_blob == nullptr || + callbacks->store == nullptr || callbacks->publish == nullptr || + callbacks->allocate_reference_segment_id == nullptr || staging == nullptr || staging->version != 1 || + staging->struct_bytes != sizeof(*staging) || staging->capabilities != CT_CHUNK_STAGE_SCHEMA_V1 || + staging->load == nullptr || staging->compare == nullptr || staging->list == nullptr || out == nullptr || + *out != nullptr) { + return code(Status::invalid_argument()); + } + auto handle = std::make_unique<ct_root_catalog>(); + handle->transport = std::make_shared<MemoryChunkTransport>(); + handle->catalog = std::make_shared<CallbackRootCatalog>(*callbacks, *staging, context); + *out = handle.release(); + return 0; +} +catch (const std::bad_alloc &) { + return code(Status::resource_exhausted()); +} +catch (...) { + return code(Status::internal_error()); +} + +uint64_t ct_root_catalog_stage_capabilities(const ct_root_catalog *catalog) +{ + return catalog != nullptr && catalog->catalog && catalog->catalog->stage_schema_supported() + ? CT_CHUNK_STAGE_SCHEMA_V1 + : 0; +} +ct_status ct_root_catalog_stage_load(ct_root_catalog *catalog, uint64_t tree_id, ct_chunk_stage_id id, ct_buf *out) +try { + if (catalog == nullptr || !catalog->catalog || out == nullptr || out->data != nullptr || out->len != 0) { + return code(Status::invalid_argument()); + } + std::optional<ChunkStageRecord> record; + auto status = catalog->catalog->load_stage(tree_id, internal(id), &record); + if (!status.ok()) { + return code(status); + } + if (!record) { + return code(Status::not_found()); + } + std::vector<uint8_t> bytes; + status = encode_stage_record(*record, &bytes); + return status.ok() ? output(bytes, out) : code(status); +} +catch (const std::bad_alloc &) { + return code(Status::resource_exhausted()); +} +catch (...) { + return code(Status::internal_error()); +} + +ct_status ct_root_catalog_stage_compare(ct_root_catalog *catalog, uint64_t tree_id, uint64_t owner_epoch, + const uint8_t *expected, size_t expected_length, const uint8_t *next, + size_t next_length) +try { + if (catalog == nullptr || !catalog->catalog) { + return code(Status::invalid_argument()); + } + std::optional<ChunkStageRecord> old_record; + std::optional<ChunkStageRecord> new_record; + auto status = decode_optional(expected, expected_length, &old_record); + if (status.ok()) { + status = decode_optional(next, next_length, &new_record); + } + return code(status.ok() ? catalog->catalog->compare_stage(tree_id, owner_epoch, old_record, new_record) : status); +} +catch (const std::bad_alloc &) { + return code(Status::resource_exhausted()); +} +catch (...) { + return code(Status::internal_error()); +} + +ct_status ct_root_catalog_stage_list(ct_root_catalog *catalog, uint64_t tree_id, const ct_chunk_stage_id *after, + uint32_t limit, ct_buf *out) +try { + if (catalog == nullptr || !catalog->catalog || out == nullptr || out->data != nullptr || out->len != 0) { + return code(Status::invalid_argument()); + } + std::vector<ChunkStageRecord> records; + auto status = catalog->catalog->list_stages( + tree_id, after != nullptr ? std::optional(internal(*after)) : std::nullopt, limit, &records); + if (!status.ok()) { + return code(status); + } + std::vector<uint8_t> bytes; + status = encode_stage_list(records, &bytes); + return status.ok() ? output(bytes, out) : code(status); +} +catch (const std::bad_alloc &) { + return code(Status::resource_exhausted()); +} +catch (...) { + return code(Status::internal_error()); +} diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_stage_callback.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_stage_callback.cpp new file mode 100644 index 000000000..53c50f4e1 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_stage_callback.cpp @@ -0,0 +1,174 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_page_store.h" + +namespace crowdb::tree::detail +{ +namespace +{ +Status status_from_callback(ct_status code) +{ + switch (code) { + case 0: + return Status::Ok(); + case -1: + return Status::not_found(); + case -2: + return Status::invalid_argument(); + case -3: + return Status::corruption(); + case -4: + return Status::io_error(); + case -5: + return Status::not_supported(); + case -6: + return Status::internal_error(); + case -7: + return Status::resource_exhausted(); + default: + return Status::unavailable(); + } +} + +struct CallbackBlob +{ + ct_root_catalog_callbacks callbacks; + void *context; + const uint8_t *data = nullptr; + size_t length = 0; + + ~CallbackBlob() + { + if (data != nullptr) { + callbacks.free_blob(context, data, length); + } + } +}; + +ct_chunk_stage_id external(ChunkStageId id) +{ + return {.owner_epoch = id.owner_epoch, .operation_high = id.operation_high, .operation_low = id.operation_low}; +} +} // namespace + +CallbackRootCatalog::CallbackRootCatalog(ct_root_catalog_callbacks callbacks, ct_root_catalog_staging_callbacks staging, + void *context) + : staging_(staging), + callbacks_(callbacks), + context_(context) +{ +} + +bool CallbackRootCatalog::stage_schema_supported() const noexcept +{ + return staging_.version == 1 && staging_.struct_bytes == sizeof(staging_) && + (staging_.capabilities & CT_CHUNK_STAGE_SCHEMA_V1) != 0 && staging_.load != nullptr && + staging_.compare != nullptr && staging_.list != nullptr; +} + +Status CallbackRootCatalog::load_stage(uint64_t tree_id, ChunkStageId id, std::optional<ChunkStageRecord> *out) const +{ + if (!stage_schema_supported()) { + return Status::not_supported(); + } + if (out == nullptr) { + return Status::invalid_argument(); + } + CallbackBlob blob{.callbacks = callbacks_, .context = context_}; + auto status = status_from_callback(staging_.load(context_, tree_id, external(id), &blob.data, &blob.length)); + if (status.code() == Code::kNotFound) { + out->reset(); + return Status::Ok(); + } + if (!status.ok()) { + return status; + } + if (blob.data == nullptr || blob.length > kMaxStageBytes) { + return Status::corruption(); + } + ChunkStageRecord record = out->value_or(ChunkStageRecord{}); + record.packs.clear(); + status = decode_stage_record({blob.data, blob.length}, &record); + if (!status.ok()) { + return status; + } + if (record.tree_id != tree_id || record.id != id) { + return Status::corruption(); + } + *out = std::move(record); + return Status::Ok(); +} + +Status CallbackRootCatalog::compare_stage(uint64_t tree_id, uint64_t owner_epoch, + const std::optional<ChunkStageRecord> &expected, + const std::optional<ChunkStageRecord> &next) +{ + if (!stage_schema_supported()) { + return Status::not_supported(); + } + auto status = validate_stage_transition(expected, next); + if (!status.ok()) { + return status; + } + if ((expected && expected->tree_id != tree_id) || (next && next->tree_id != tree_id) || owner_epoch == 0) { + return Status::invalid_argument(); + } + std::shared_ptr<AllocationBudget> budget; + if (next) { + budget = next->packs.get_allocator().owner(); + } + else if (expected) { + budget = expected->packs.get_allocator().owner(); + } + ChunkStageBytes old_bytes{MetadataAllocator<uint8_t>(budget)}; + ChunkStageBytes new_bytes{MetadataAllocator<uint8_t>(budget)}; + if (expected) { + status = encode_stage_record_owned(*expected, &old_bytes); + } + if (status.ok() && next) { + status = encode_stage_record_owned(*next, &new_bytes); + } + if (!status.ok()) { + return status; + } + return status_from_callback(staging_.compare(context_, tree_id, owner_epoch, old_bytes.data(), old_bytes.size(), + new_bytes.data(), new_bytes.size())); +} + +Status CallbackRootCatalog::list_stages(uint64_t tree_id, std::optional<ChunkStageId> after, uint32_t limit, + std::vector<ChunkStageRecord> *out) const +{ + if (!stage_schema_supported()) { + return Status::not_supported(); + } + if (out == nullptr || limit == 0 || limit > kMaxStageScan) { + return Status::invalid_argument(); + } + const auto cursor = external(after.value_or(ChunkStageId{})); + CallbackBlob blob{.callbacks = callbacks_, .context = context_}; + auto status = status_from_callback( + staging_.list(context_, tree_id, after ? &cursor : nullptr, limit, &blob.data, &blob.length)); + if (!status.ok()) { + return status; + } + if (blob.data == nullptr || blob.length > 4 + (kMaxStageScan * (4 + kMaxStageBytes))) { + return Status::corruption(); + } + std::vector<ChunkStageRecord> records; + status = decode_stage_list({blob.data, blob.length}, &records); + if (!status.ok()) { + return status; + } + if (records.size() > limit) { + return Status::corruption(); + } + for (const auto &record : records) { + if (record.tree_id != tree_id || (after && record.id <= *after)) { + return Status::corruption(); + } + after = record.id; + } + *out = std::move(records); + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_stage_codec.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_stage_codec.cpp new file mode 100644 index 000000000..7c4931cb3 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_stage_codec.cpp @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_stage.h" +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/maptable/frame_page.h" + +namespace crowdb::tree::detail +{ +namespace +{ +template <class Bytes> Status encode_record(const ChunkStageRecord &record, Bytes *out) +{ + if (out == nullptr) { + return Status::invalid_argument(); + } + auto status = validate_stage_record(record); + if (!status.ok()) { + return status; + } + out->assign(100 + (16 * record.packs.size()), 0); + auto *data = out->data(); + frame_put_u32(data, 0, 0x47535443); + frame_put_u32(data, 4, 1); + const uint64_t values[] = {record.tree_id, record.id.owner_epoch, record.id.operation_high, + record.id.operation_low, record.revision, record.chunk_high, + record.chunk_low, record.capacity, record.manifest_generation}; + for (size_t i = 0; i < 9; ++i) { + frame_put_u64(data, 8 + (i * 8), values[i]); + } + frame_put_u32(data, 80, record.purpose); + frame_put_u32(data, 84, static_cast<uint32_t>(record.state)); + frame_put_u32(data, 88, static_cast<uint32_t>(record.packs.size())); + for (size_t i = 0; i < record.packs.size(); ++i) { + frame_put_u64(data, 96 + (16 * i), record.packs[i].offset); + frame_put_u32(data, 104 + (16 * i), record.packs[i].length); + frame_put_u32(data, 108 + (16 * i), record.packs[i].checksum); + } + frame_put_u32(data, out->size() - 4, common::crc32c(data, out->size() - 4)); + return Status::Ok(); +} + +} // namespace + +Status encode_stage_record(const ChunkStageRecord &record, std::vector<uint8_t> *out) +{ + return encode_record(record, out); +} + +Status encode_stage_record_owned(const ChunkStageRecord &record, ChunkStageBytes *out) +{ + return encode_record(record, out); +} + +Status decode_stage_record(std::span<const uint8_t> bytes, ChunkStageRecord *out) +{ + const auto *data = bytes.data(); + if (out == nullptr || bytes.size() < 100 || bytes.size() > kMaxStageBytes || frame_u32(data, 0) != 0x47535443) { + return Status::corruption("invalid stage record header"); + } + if (frame_u32(data, 4) != 1) { + return Status::not_supported("stage record version"); + } + const auto count = frame_u32(data, 88); + const auto state = frame_u32(data, 84); + if (state < 1 || state > 7 || count > kMaxStagePacks || bytes.size() != 100 + (16 * count) || + frame_u32(data, 92) != 0 || common::crc32c(data, bytes.size() - 4) != frame_u32(data, bytes.size() - 4)) { + return Status::corruption("invalid stage record checksum or length"); + } + ChunkStageRecord record{ + .tree_id = frame_u64(data, 8), + .id = {.owner_epoch = frame_u64(data, 16), + .operation_high = frame_u64(data, 24), + .operation_low = frame_u64(data, 32)}, + .revision = frame_u64(data, 40), + .chunk_high = frame_u64(data, 48), + .chunk_low = frame_u64(data, 56), + .capacity = frame_u64(data, 64), + .manifest_generation = frame_u64(data, 72), + .purpose = frame_u32(data, 80), + .state = static_cast<ChunkStageState>(state), + .packs = decltype(ChunkStageRecord::packs)(out->packs.get_allocator()) + }; + record.packs.reserve(count); + for (size_t i = 0; i < count; ++i) { + record.packs.push_back({.offset = frame_u64(data, 96 + (16 * i)), + .length = frame_u32(data, 104 + (16 * i)), + .checksum = frame_u32(data, 108 + (16 * i))}); + } + if (!validate_stage_record(record).ok()) { + return Status::corruption("invalid stage record fields"); + } + *out = std::move(record); + return Status::Ok(); +} + +Status encode_stage_list(std::span<const ChunkStageRecord> records, std::vector<uint8_t> *out) +{ + if (out == nullptr || records.size() > kMaxStageScan) { + return Status::invalid_argument(); + } + out->assign(4, 0); + frame_put_u32(out->data(), 0, static_cast<uint32_t>(records.size())); + for (const auto &record : records) { + std::vector<uint8_t> encoded; + auto status = encode_stage_record(record, &encoded); + if (!status.ok()) { + return status; + } + const auto at = out->size(); + out->resize(at + 4); + frame_put_u32(out->data(), at, static_cast<uint32_t>(encoded.size())); + out->insert(out->end(), encoded.begin(), encoded.end()); + } + return Status::Ok(); +} + +Status decode_stage_list(std::span<const uint8_t> bytes, std::vector<ChunkStageRecord> *out) +{ + if (out == nullptr || bytes.size() < 4 || bytes.size() > 4 + (kMaxStageScan * (4 + kMaxStageBytes))) { + return Status::corruption(); + } + const auto count = frame_u32(bytes.data(), 0); + if (count > kMaxStageScan) { + return Status::corruption(); + } + std::vector<ChunkStageRecord> records; + records.reserve(count); + size_t at = 4; + for (size_t i = 0; i < count; ++i) { + if (bytes.size() - at < 4) { + return Status::corruption(); + } + const auto length = frame_u32(bytes.data(), at); + at += 4; + if (length > bytes.size() - at) { + return Status::corruption(); + } + ChunkStageRecord record; + auto status = decode_stage_record(bytes.subspan(at, length), &record); + if (!status.ok()) { + return status; + } + records.push_back(std::move(record)); + at += length; + } + if (at != bytes.size()) { + return Status::corruption(); + } + *out = std::move(records); + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_stage_gc.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_stage_gc.cpp new file mode 100644 index 000000000..cef685395 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_stage_gc.cpp @@ -0,0 +1,123 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "chunk_stage_gc.h" + +#include "chunk_physical_registry.h" + +namespace crowdb::tree::detail +{ +ChunkStageGcResult reclaim_retired_stage(RootCatalog &catalog, ChunkTransport &transport, uint64_t tree, uint64_t epoch, + ChunkStageId id, const std::shared_ptr<AllocationBudget> &budget, + const std::shared_ptr<ChunkPhysicalRegistry> ®istry) +try { + ChunkStageGcResult result; + if (!catalog.stage_schema_supported() || !transport.fixed_staging_supported()) { + result.status = Status::not_supported(); + return result; + } + if (epoch < id.owner_epoch) { + result.status = Status::unavailable("stage belongs to a newer writer"); + return result; + } + std::optional<ChunkStageRecord> record{ + ChunkStageRecord{.tree_id = 0, + .id = {}, + .revision = 0, + .chunk_high = 0, + .chunk_low = 0, + .capacity = 0, + .manifest_generation = 0, + .purpose = 0, + .state = ChunkStageState::Reserved, + .packs = decltype(ChunkStageRecord::packs)(MetadataAllocator<ChunkStagePack>(budget))} + }; + result.status = catalog.load_stage(tree, id, &record); + if (!result.status.ok() || !record) { + return result; + } + if (record->state == ChunkStageState::Retired) { + if (record->revision == UINT64_MAX) { + result.status = Status::corruption(); + return result; + } + auto deleting = *record; + ++deleting.revision; + deleting.state = ChunkStageState::Deleting; + result.status = catalog.compare_stage(tree, epoch, record, deleting); + if (!result.status.ok()) { + return result; + } + record = std::move(deleting); + } + if (record->state == ChunkStageState::Deleting) { + ChunkAllocationIdentity physical; + result.status = transport.inspect_fixed_allocation({record->chunk_high, record->chunk_low}, + record->id.owner_epoch, &physical); + if (!result.status.ok() && result.status.code() != Code::kNotFound) { + return result; + } + std::optional<ChunkAllocationIdentity> receipt; + if (result.status.ok()) { + receipt = physical; + if (registry) { + std::shared_ptr<ChunkPhysicalPack> observed; + result.status = + registry->prepare(record->id, {record->chunk_high, record->chunk_low}, budget, &observed); + if (result.status.ok()) { + result.status = registry->observe(observed, physical, budget); + } + if (!result.status.ok()) { + return result; + } + } + } + result.status = transport.drain_fixed_chunk({record->chunk_high, record->chunk_low}); + if (!result.status.ok() && result.status.code() != Code::kNotFound) { + return result; + } + // A completed earlier delete can make the drain lookup absent. The + // durable Deleting authority still requires an idempotent delete ack. + result.status = transport.delete_chunk({record->chunk_high, record->chunk_low}); + if (!result.status.ok()) { + return result; + } + result.deletion_acknowledged = true; + if (registry) { + result.status = registry->deleted(record->id, {record->chunk_high, record->chunk_low}, receipt, budget); + if (!result.status.ok()) { + return result; + } + } + if (record->revision == UINT64_MAX) { + result.status = Status::corruption(); + return result; + } + auto reclaimed = *record; + ++reclaimed.revision; + reclaimed.state = ChunkStageState::Reclaimed; + result.status = catalog.compare_stage(tree, epoch, record, reclaimed); + if (!result.status.ok()) { + return result; + } + record = std::move(reclaimed); + } + if (record->state != ChunkStageState::Reclaimed) { + result.status = Status::unavailable("stage still protects a live or unresolved pack"); + return result; + } + // A persisted Reclaimed record is itself proof of an acknowledged physical + // delete, including recovery after the final metadata erase was interrupted. + if (registry) { + result.status = registry->deleted(record->id, {record->chunk_high, record->chunk_low}, {}, budget); + if (!result.status.ok()) { + return result; + } + } + result.status = catalog.compare_stage(tree, epoch, record, {}); + result.stage_removed = result.status.ok(); + return result; +} +catch (const std::bad_alloc &) { + return {.status = Status::resource_exhausted()}; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_stage_gc.h b/lib/crowdb-tree/src/backend/chunk/chunk_stage_gc.h new file mode 100644 index 000000000..4bbdf25d6 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_stage_gc.h @@ -0,0 +1,26 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_page_store.h" + +namespace crowdb::tree +{ +class ChunkPhysicalRegistry; +} + +namespace crowdb::tree::detail +{ +struct ChunkStageGcResult +{ + Status status; + bool deletion_acknowledged = false; + bool stage_removed = false; +}; + +// Retirement is a separate ownership decision. This finite backend step only +// consumes a durably retired identity; time, absence, or a write error alone +// never authorize reclamation. +ChunkStageGcResult reclaim_retired_stage(RootCatalog &catalog, ChunkTransport &transport, uint64_t tree, uint64_t epoch, + ChunkStageId id, const std::shared_ptr<AllocationBudget> &budget = {}, + const std::shared_ptr<ChunkPhysicalRegistry> ®istry = {}); +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_transport.h b/lib/crowdb-tree/src/backend/chunk/chunk_transport.h index 40448f490..ebb9a00a8 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_transport.h +++ b/lib/crowdb-tree/src/backend/chunk/chunk_transport.h @@ -3,6 +3,7 @@ #pragma once +#include "chunk_io_context.h" #include "crowdb-common/atomic_shared_ptr.h" #include "crowdb-tree/backend/page_purpose.h" #include "crowdb-tree/status.h" @@ -15,6 +16,12 @@ #include <utility> #include <vector> +namespace crowdb::tree +{ +class AllocationBudget; +class ChunkPhysicalRegistry; +} // namespace crowdb::tree + namespace crowdb::tree::detail { @@ -37,7 +44,7 @@ struct ChunkId { } - constexpr bool operator==(const ChunkId &) const = default; + constexpr bool operator==(const ChunkId /*unused*/ &) const = default; [[nodiscard]] constexpr bool empty() const { @@ -45,6 +52,15 @@ struct ChunkId } }; +// One immutable physical allocation, including every rounded mirror segment. +struct ChunkAllocationIdentity +{ + ChunkId chunk; + uint64_t owner_epoch = 0; + uint64_t created_ms = 0; + uint64_t physical_bytes = 0; +}; + struct ChunkLayout { ChunkId chunk_id; @@ -72,16 +88,76 @@ struct ChunkTransportCompletion } }; +struct ChunkRegistryPool; + class ChunkTransport { public: - virtual ~ChunkTransport() = default; + virtual ~ChunkTransport(); + // Host open only. Registry identity is stable across writer incarnations + // for this transport, retirement domain and logical Tree ID. + Status physical_registry(uint64_t tree, const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkPhysicalRegistry> *out); virtual Status allocate_mirror_chunk(uint64_t logical_capacity, uint64_t owner_epoch, ChunkId *chunk_id, PagePurpose purpose) = 0; // The ordered pack planner prepares capacity before submitting mirror I/O. // Allocation capacity is a ceiling; RPC-backed chunks grow in small strips. virtual Status ensure_capacity(ChunkId chunk_id, uint64_t required_bytes); + + // Fixed identity is durably protected before this call. An unknown result + // never authorizes generating another identity or releasing that protection. + [[nodiscard]] virtual bool fixed_staging_supported() const noexcept + { + return false; + } + + virtual Status allocate_fixed_chunk(ChunkId /*unused*/, uint64_t /*unused*/, uint64_t /*unused*/, + PagePurpose /*unused*/) + { + return Status::not_supported(); + } + + [[nodiscard]] virtual Status verify_fixed_read_fence(ChunkId /*unused*/, uint32_t /*unused*/, + ChunkIoContext /*unused*/) const + { + return Status::not_supported(); + } + + virtual Status write_fixed_mirror(ChunkId /*unused*/, uint32_t /*unused*/, uint64_t /*unused*/, + const uint8_t * /*unused*/, size_t /*unused*/, ChunkIoContext /*unused*/) + { + return Status::not_supported(); + } + + virtual Status read_fixed_mirror(ChunkId /*unused*/, uint32_t /*unused*/, uint64_t /*unused*/, uint8_t * /*unused*/, + size_t /*unused*/, ChunkIoContext /*unused*/) const + { + return Status::not_supported(); + } + + // Called only after the creator's fixed request ages are excluded. A + // successful barrier waits for all already accepted ordered disk writes. + [[nodiscard]] virtual Status drain_fixed_chunk(ChunkId /*unused*/) const + { + return Status::not_supported(); + } + + // Always query authority, not a cached route/layout. A different physical + // owner cannot be reclaimed using an old stage's logical reference. + virtual Status inspect_fixed_allocation(ChunkId /*unused*/, uint64_t /*unused*/, + ChunkAllocationIdentity * /*unused*/) const + { + return Status::not_supported(); + } + + // Success means ChunkDB has completed physical deletion. Lost replies retain + // protection and may retry the same immutable identity. + virtual Status delete_chunk(ChunkId /*unused*/) + { + return Status::not_supported(); + } + virtual Status write_mirror(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, const uint8_t *data, size_t length) = 0; virtual void submit_write_mirror(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, const uint8_t *data, @@ -91,6 +167,9 @@ class ChunkTransport size_t length) const = 0; virtual Status query_chunk(ChunkId chunk_id, ChunkLayout *layout) const = 0; virtual Status seal_chunk(ChunkId chunk_id, uint64_t owner_epoch, uint64_t acknowledged_bytes) = 0; + + private: + std::atomic<ChunkRegistryPool *> registry_pool_{nullptr}; }; // Lock-free immutable-snapshot transport for tests and embedded use. It models diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_transport_registry.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_transport_registry.cpp new file mode 100644 index 000000000..2c413ba3a --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_transport_registry.cpp @@ -0,0 +1,167 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/budget_memory_resource.h" +#include "btree/retired_shared.h" +#include "chunk_physical_registry.h" +#include "chunk_transport.h" + +#include <cassert> +#include <map> + +namespace crowdb::tree::detail +{ +namespace +{ +struct RegistryPoolState : std::enable_shared_from_this<RegistryPoolState> +{ + struct Key + { + const RetirementQueue *host = nullptr; + uint64_t tree = 0; + }; + + struct Less + { + bool operator()(Key left, Key right) const noexcept + { + return left.host == right.host ? left.tree < right.tree + : std::less<const RetirementQueue *>{}(left.host, right.host); + } + }; + + struct Entry + { + // The scope's host identity remains owned even after its executor closes. + std::shared_ptr<RetirementQueue> host; + std::shared_ptr<ChunkPhysicalRegistry> registry; + }; + + struct Request : RetirementNode + { + std::shared_ptr<RegistryPoolState> state; + std::shared_ptr<Request> queued_owner; + std::shared_ptr<AllocationBudget> budget; + std::shared_ptr<ChunkPhysicalRegistry> result; + uint64_t tree = 0; + Status status; + std::atomic<bool> done{false}; + }; + + explicit RegistryPoolState(std::shared_ptr<RetirementQueue> executor) : queue(std::move(executor)) + { + } + + bool retire_step() noexcept + { + for (size_t count = 0; count < 32 && !entries.empty(); ++count) { + entries.erase(entries.begin()); + } + return entries.empty(); + } + + static bool dispatch(RetirementNode *node) noexcept + { + auto request = std::move(static_cast<Request *>(node)->queued_owner); + try { + auto &self = *request->state; + auto host = request->budget->retirement_queue(); + const Key key{.host = host.get(), .tree = request->tree}; + auto found = self.entries.find(key); + if (found == self.entries.end()) { + std::shared_ptr<ChunkPhysicalRegistry> registry; + request->status = ChunkPhysicalRegistry::create(request->budget, ®istry); + if (request->status.ok()) { + BudgetMemoryResource::Scope credit(self.memory, request->budget); + found = self.entries.emplace(key, Entry{.host = std::move(host), .registry = std::move(registry)}) + .first; + } + } + if (request->status.ok()) { + request->result = found->second.registry; + } + } + catch (const std::bad_alloc &) { + request->status = Status::resource_exhausted(); + } + catch (...) { + request->status = Status::internal_error(); + } + request->state.reset(); + request->done.store(true, std::memory_order_release); + request->done.notify_all(); + return true; + } + + Status acquire(uint64_t tree, const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkPhysicalRegistry> *out) + { + auto request = std::allocate_shared<Request>(MetadataAllocator<Request>(metadata)); + request->state = shared_from_this(); + request->queued_owner = request; + request->budget = metadata; + request->retirement_budget = metadata; + request->tree = tree; + request->dispose = &dispatch; + queue->retire(request.get()); + while (!request->done.load(std::memory_order_acquire)) { + request->done.wait(false, std::memory_order_acquire); + } + if (request->status.ok()) { + *out = std::move(request->result); + } + return request->status; + } + + std::shared_ptr<RetirementQueue> queue; + BudgetMemoryResource memory; + std::pmr::map<Key, Entry, Less> entries{&memory}; +}; +} // namespace + +struct ChunkRegistryPool +{ + std::shared_ptr<AllocationBudget> metadata; + std::shared_ptr<RegistryPoolState> state; + + static void destroy(ChunkRegistryPool *pool) noexcept + { + if (pool != nullptr) { + MetadataAllocator<ChunkRegistryPool> allocator(pool->metadata); + std::destroy_at(pool); + allocator.deallocate(pool, 1); + } + } +}; + +ChunkTransport::~ChunkTransport() +{ + ChunkRegistryPool::destroy(registry_pool_.load(std::memory_order_relaxed)); +} + +Status ChunkTransport::physical_registry(uint64_t tree, const std::shared_ptr<AllocationBudget> &metadata, + std::shared_ptr<ChunkPhysicalRegistry> *out) +try { + if (tree == 0 || !metadata || !metadata->retirement_queue() || out == nullptr || *out) { + return Status::invalid_argument(); + } + auto *pool = registry_pool_.load(std::memory_order_acquire); + if (pool == nullptr) { + auto state = + retired_shared<RegistryPoolState>(metadata, metadata->retirement_queue(), metadata->retirement_queue()); + MetadataAllocator<ChunkRegistryPool> allocator(metadata); + auto *prepared = allocator.allocate(1); + std::construct_at(prepared, ChunkRegistryPool{.metadata = metadata, .state = std::move(state)}); + if (registry_pool_.compare_exchange_strong(pool, prepared, std::memory_order_release, + std::memory_order_acquire)) { + pool = prepared; + } + else { + ChunkRegistryPool::destroy(prepared); + } + } + return pool->state->acquire(tree, metadata, out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_layout.cpp b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_layout.cpp new file mode 100644 index 000000000..35446eed1 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_layout.cpp @@ -0,0 +1,136 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "legacy_chunk_layout.h" + +#include "chunk_frame.h" +#include "crowdb-tree/maptable/frame_page.h" + +namespace crowdb::tree::detail +{ +namespace +{ +constexpr size_t kHeader = 80; +constexpr size_t kSegment = 33; +constexpr size_t kPack = 53; +constexpr size_t kSegmentRefs = 256; +} // namespace + +Status LegacyChunkLayout::parse(std::span<const uint8_t> bytes, uint64_t tree, uint64_t epoch, LegacyChunkLayout *out) +{ + if (out == nullptr || bytes.size() < kHeader + 8 || frame_u32(bytes.data(), 0) != kChunkManifestFormat || + frame_u64(bytes.data(), 4) != tree || frame_u64(bytes.data(), 12) == 0 || + frame_u64(bytes.data(), 12) == UINT64_MAX || frame_u64(bytes.data(), 20) > epoch) { + return Status::corruption("invalid legacy Chunk manifest identity"); + } + LegacyChunkLayout result; + result.bytes_ = bytes; + const auto segments = frame_u64(bytes.data(), 72); + if (segments > (bytes.size() - kHeader - 8) / kSegment) { + return Status::corruption("invalid legacy Chunk segment count"); + } + result.segments_ = static_cast<size_t>(segments); + result.packs_offset_ = kHeader + (result.segments_ * kSegment) + 8; + const auto packs = frame_u64(bytes.data(), result.packs_offset_ - 8); + if (packs != (bytes.size() - result.packs_offset_) / kPack || (bytes.size() - result.packs_offset_) % kPack != 0 || + segments != (packs + kSegmentRefs - 1) / kSegmentRefs) { + return Status::corruption("invalid legacy Chunk pack directory"); + } + result.packs_ = static_cast<size_t>(packs); + auto crc = common::crc32c(bytes.data(), 68); + crc = common::crc32c_update(crc, bytes.data() + result.packs_offset_, result.packs_ * kPack); + crc = common::crc32c_update(crc, bytes.data() + kHeader, result.segments_ * kSegment); + if (crc != frame_u32(bytes.data(), 68)) { + return Status::corruption("legacy Chunk manifest checksum mismatch"); + } + uint64_t end = 0; + uint64_t reused = 0; + uint64_t reused_bytes = 0; + for (size_t i = 0; i < result.packs_; ++i) { + const auto pack = result.pack(i); + const auto flag = bytes[result.packs_offset_ + (i * kPack) + 52]; + if (pack.ordinal != i || pack.owner_tree_id == 0 || pack.ref.chunk_id.empty() || + !valid_page_purpose(chunk_purpose(pack.ref.chunk_id)) || pack.ref.length == 0 || + pack.ref.length > kMaxChunkPackBytes || pack.logical_offset < end || + pack.logical_offset > result.logical_size() || + pack.ref.length > result.logical_size() - pack.logical_offset || flag > 1 || + chunk_framed_length(pack.ref.length) > UINT64_MAX - pack.ref.offset) { + return Status::corruption("invalid legacy Chunk pack bounds"); + } + end = pack.logical_offset + pack.ref.length; + reused += pack.reused ? 1 : 0; + reused_bytes += pack.reused ? pack.ref.length : 0; + } + if (reused != frame_u64(bytes.data(), 44) || reused_bytes != frame_u64(bytes.data(), 52)) { + return Status::corruption("invalid legacy Chunk reuse counts"); + } + *out = result; + return Status::Ok(); +} + +ChunkPagePack LegacyChunkLayout::pack(size_t index) const noexcept +{ + const auto *p = bytes_.data() + packs_offset_ + (index * kPack); + return { + .owner_tree_id = frame_u64(p, 0), + .ordinal = frame_u64(p, 8), + .logical_offset = frame_u64(p, 16), + .ref = {.chunk_id = {frame_u64(p, 24), frame_u64(p, 32)}, + .offset = frame_u64(p, 40), + .length = frame_u32(p, 48)}, + .reused = p[52] != 0 + }; +} + +ChunkReferenceSegment LegacyChunkLayout::segment(size_t index) const noexcept +{ + const auto *p = bytes_.data() + kHeader + (index * kSegment); + return {.owner_tree_id = frame_u64(p, 0), + .object_id = frame_u64(p, 8), + .first_ordinal = frame_u64(p, 16), + .ref_count = frame_u32(p, 24), + .checksum = frame_u32(p, 28), + .reused = p[32] != 0}; +} + +Status LegacyChunkLayout::validate_segment(size_t index, std::span<const uint8_t> bytes) const +{ + const auto descriptor = segment(index); + const auto first = index * kSegmentRefs; + const auto count = std::min(kSegmentRefs, packs_ - first); + if (descriptor.owner_tree_id == 0 || descriptor.object_id == 0 || descriptor.first_ordinal != first || + descriptor.ref_count != count || bytes_[kHeader + (index * kSegment) + 32] > 1 || + bytes.size() != 24 + (count * 28) || frame_u64(bytes.data(), 0) != descriptor.object_id || + frame_u64(bytes.data(), 8) != first || frame_u64(bytes.data(), 16) != count) { + return Status::corruption("invalid legacy Chunk reference segment"); + } + auto crc = common::crc32c(bytes.data(), 16); + crc = common::crc32c_update(crc, bytes.data() + 24, bytes.size() - 24); + if (crc != descriptor.checksum) { + return Status::corruption("legacy Chunk reference checksum mismatch"); + } + for (size_t i = 0; i < count; ++i) { + const auto ref = pack(first + i).ref; + const auto *p = bytes.data() + 24 + (i * 28); + if (ref.chunk_id.high != frame_u64(p, 0) || ref.chunk_id.low != frame_u64(p, 8) || + ref.offset != frame_u64(p, 16) || ref.length != frame_u32(p, 24)) { + return Status::corruption("legacy Chunk reference differs from its pack"); + } + } + return Status::Ok(); +} + +uint64_t LegacyChunkLayout::generation() const noexcept +{ + return frame_u64(bytes_.data(), 12); +} + +uint64_t LegacyChunkLayout::logical_size() const noexcept +{ + return frame_u64(bytes_.data(), 28); +} + +uint64_t LegacyChunkLayout::wal_offset() const noexcept +{ + return frame_u64(bytes_.data(), 60); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_layout.h b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_layout.h new file mode 100644 index 000000000..6b76e223d --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_layout.h @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "chunk_page_store.h" + +namespace crowdb::tree::detail +{ +// A validated view over an owned format-seven manifest. No mutable legacy +// PageStore or executor is created by the migration reader. +class LegacyChunkLayout +{ + public: + static Status parse(std::span<const uint8_t> bytes, uint64_t tree, uint64_t epoch, LegacyChunkLayout *out); + [[nodiscard]] ChunkPagePack pack(size_t index) const noexcept; + [[nodiscard]] ChunkReferenceSegment segment(size_t index) const noexcept; + [[nodiscard]] Status validate_segment(size_t index, std::span<const uint8_t> bytes) const; + [[nodiscard]] uint64_t generation() const noexcept; + [[nodiscard]] uint64_t logical_size() const noexcept; + [[nodiscard]] uint64_t wal_offset() const noexcept; + + [[nodiscard]] size_t pack_count() const noexcept + { + return packs_; + } + + [[nodiscard]] size_t segment_count() const noexcept + { + return segments_; + } + + private: + std::span<const uint8_t> bytes_; + size_t segments_ = 0; + size_t packs_ = 0; + size_t packs_offset_ = 0; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_migration_receipt.cpp b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_migration_receipt.cpp new file mode 100644 index 000000000..62b2bba1c --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_migration_receipt.cpp @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "legacy_chunk_read_store.h" +#include "snapshot/chunk_root_codec.h" + +namespace crowdb::tree::detail +{ +Status LegacyChunkReadStore::prepare_migration() +{ + if (!pinned_ || !verified_) { + return Status::invalid_argument(); + } + return catalog_->prepare_legacy_migration(tree_, layout_.generation(), manifest_); +} + +Status LegacyChunkReadStore::resolve_migration(uint64_t generation, bool *committed) +{ + if (committed == nullptr || generation == 0 || generation == UINT64_MAX || !catalog_ || + catalog_->protection() == nullptr || !metadata_) { + return Status::invalid_argument(); + } + *committed = false; + auto status = catalog_->protection()->check_read_owner(); + if (status.ok()) { + status = catalog_->protection()->load_root_owned(tree_, 0, &manifest_); + } + if (!status.ok()) { + return status; + } + if (manifest_.size() >= 4 && frame_u32(manifest_.data(), 0) == kChunkManifestFormat) { + return Status::Ok(); + } + { + ChunkRootManifest root{metadata_}; + status = decode_chunk_root(manifest_, tree_, 0, &root); + if (!status.ok()) { + return status; + } + if (root.identity.root_generation <= generation || root.identity.owner_epoch > epoch_) { + return Status::unavailable("current Chunk root predates the requested migration"); + } + } + ChunkRootBytes receipt{MetadataAllocator<uint8_t>(metadata_)}; + status = catalog_->load_legacy_migration_owned(tree_, generation, &receipt); + if (!status.ok()) { + return status; + } + manifest_ = std::move(receipt); + status = LegacyChunkLayout::parse(manifest_, tree_, epoch_, &layout_); + if (!status.ok() || layout_.generation() != generation) { + return Status::corruption("legacy Chunk migration receipt identity differs"); + } + // The immutable receipt predates root CAS. A newer typed root confirms + // the data copy, while the original generation pin is released idempotently. + pinned_ = true; + *committed = true; + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_read_store.cpp b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_read_store.cpp new file mode 100644 index 000000000..15f840ee2 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_read_store.cpp @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "legacy_chunk_read_store.h" + +#include "chunk_frame.h" + +#include <algorithm> +#include <bit> +#include <cstring> + +namespace crowdb::tree::detail +{ +namespace +{ +// A stable migration pin is idempotent across lost replies and restart. It is +// released explicitly only after the migration outcome is known. +constexpr uint64_t kMigrationPin = 0x43544d4947524154ULL; +} // namespace + +LegacyChunkReadStore::LegacyChunkReadStore(std::shared_ptr<RootCatalog> catalog, + std::shared_ptr<ChunkTransport> transport, + std::shared_ptr<AllocationReservation> metadata, uint64_t tree, + uint64_t epoch, uint32_t iu_size, uint32_t mirrors) + : catalog_(std::move(catalog)), + transport_(std::move(transport)), + metadata_(std::move(metadata)), + manifest_(MetadataAllocator<uint8_t>(metadata_)), + tree_(tree), + epoch_(epoch), + iu_size_(iu_size), + mirrors_(mirrors) +{ +} + +Status LegacyChunkReadStore::open(uint64_t expected_generation) +{ + if (tree_ == 0 || epoch_ == 0 || expected_generation == 0 || expected_generation == UINT64_MAX || !catalog_ || + catalog_->protection() == nullptr || !transport_ || !metadata_ || !std::has_single_bit(iu_size_) || + mirrors_ == 0 || mirrors_ > kMaxMirrorCopies || pinned_) { + return Status::invalid_argument(); + } + auto *protection = catalog_->protection(); + auto status = protection->check_read_owner(); + if (status.ok()) { + status = protection->load_root_owned(tree_, 0, &manifest_); + } + if (status.ok()) { + status = LegacyChunkLayout::parse(manifest_, tree_, epoch_, &layout_); + } + if (!status.ok()) { + return status; + } + if (layout_.generation() != expected_generation) { + return Status::unavailable("legacy Chunk generation changed before migration"); + } + status = catalog_->pin_generation(tree_, kMigrationPin, expected_generation, expected_generation); + if (!status.ok()) { + return status; + } + pinned_ = true; + ChunkRootBytes current{MetadataAllocator<uint8_t>(metadata_)}; + status = protection->load_root_owned(tree_, 0, ¤t); + if (!status.ok() || current != manifest_) { + return status.ok() ? Status::unavailable("legacy Chunk root changed while pinning") : status; + } + for (size_t i = 0; i < layout_.segment_count(); ++i) { + const auto descriptor = layout_.segment(i); + current.clear(); + status = catalog_->load_legacy_reference_owned(descriptor.owner_tree_id, descriptor.object_id, ¤t); + if (status.ok()) { + status = layout_.validate_segment(i, current); + } + if (!status.ok()) { + return status; + } + } + status = protection->check_read_owner(); + verified_ = status.ok(); + return status; +} + +Status LegacyChunkReadStore::release_pin() +{ + if (!pinned_) { + return Status::Ok(); + } + auto status = catalog_->unpin_generation(tree_, kMigrationPin, layout_.generation()); + if (status.ok()) { + pinned_ = false; + verified_ = false; + } + return status; +} + +Status LegacyChunkReadStore::read_at(uint64_t offset, uint8_t *data, size_t length) const +try { + if (!pinned_ || !verified_ || (data == nullptr && length != 0)) { + return Status::invalid_argument(); + } + if (offset > size() || length > size() - offset) { + return Status::not_found(); + } + auto status = catalog_->protection()->check_read_owner(); + if (!status.ok() || length == 0) { + return status; + } + ReservedVector<uint8_t, AllocationKind::kIo> scratch(protocol::kMaxFrameBytes, + ReservationAllocator<uint8_t, AllocationKind::kIo>(metadata_)); + size_t lower = 0; + size_t upper = layout_.pack_count(); + while (lower < upper) { + const auto middle = lower + ((upper - lower) / 2); + const auto pack = layout_.pack(middle); + if (pack.logical_offset + pack.ref.length <= offset) { + lower = middle + 1; + } + else { + upper = middle; + } + } + size_t copied = 0; + for (size_t index = lower; copied < length && index < layout_.pack_count(); ++index) { + const auto pack = layout_.pack(index); + const auto position = offset + copied; + if (pack.logical_offset > position) { + return Status::not_found(); + } + const auto begin = position - pack.logical_offset; + const auto bytes = std::min<uint64_t>(length - copied, pack.ref.length - begin); + status = read_pack(pack.ref, begin, {data + copied, static_cast<size_t>(bytes)}, scratch); + if (!status.ok()) { + return status; + } + copied += bytes; + } + return copied == length ? catalog_->protection()->check_read_owner() : Status::not_found(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status LegacyChunkReadStore::read_pack(const ChunkPageRef &ref, uint64_t offset, std::span<uint8_t> output, + std::span<uint8_t> scratch) const +{ + while (!output.empty()) { + const auto frame_begin = (offset / protocol::kMaxFrameV2PayloadBytes) * protocol::kMaxFrameV2PayloadBytes; + const auto frame_bytes = std::min<uint64_t>(protocol::kMaxFrameV2PayloadBytes, ref.length - frame_begin); + std::span<const uint8_t> payload; + auto status = read_frame(ref, frame_begin, frame_bytes, scratch, &payload); + if (!status.ok()) { + return status; + } + const auto begin = offset - frame_begin; + const auto bytes = std::min(output.size(), payload.size() - begin); + std::memcpy(output.data(), payload.data() + begin, bytes); + offset += bytes; + output = output.subspan(bytes); + } + return Status::Ok(); +} + +Status LegacyChunkReadStore::read_frame(const ChunkPageRef &ref, size_t logical_offset, size_t logical_bytes, + std::span<uint8_t> scratch, std::span<const uint8_t> *payload) const +{ + const auto physical_offset = (logical_offset / protocol::kMaxFrameV2PayloadBytes) * protocol::kMaxFrameBytes; + const auto bytes = logical_bytes + protocol::kFrameV2HeaderBytes + protocol::kFrameFooterBytes; + bool responded = false; + for (uint32_t mirror = 0; mirror < mirrors_; ++mirror) { + auto status = + transport_->read_mirror(ref.chunk_id, mirror, ref.offset + physical_offset, scratch.data(), bytes); + if (!status.ok()) { + continue; + } + responded = true; + protocol::ParsedFrame frame; + const auto error = + protocol::parse_frame(scratch.first(bytes), {.high = ref.chunk_id.high, .low = ref.chunk_id.low}, &frame); + if (error == protocol::FrameError::Ok && + frame.header.magic == chunk_page_frame_magic(chunk_purpose(ref.chunk_id)) && + frame.header.payload_offset == protocol::kFrameV2HeaderBytes && frame.payload.size() == logical_bytes && + frame.physical_length == bytes) { + *payload = frame.payload; + return Status::Ok(); + } + } + return responded ? Status::corruption("legacy Chunk frame checksum or identity mismatch") + : Status::unavailable("legacy Chunk frame mirrors unavailable"); +} + +Status LegacyChunkReadStore::write_at(uint64_t /*offset*/, const uint8_t * /*data*/, size_t /*length*/) +{ + return Status::not_supported("legacy Chunk migration source is immutable"); +} + +Status LegacyChunkReadStore::sync() +{ + return Status::not_supported(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_read_store.h b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_read_store.h new file mode 100644 index 000000000..518d0aae7 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_read_store.h @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "legacy_chunk_layout.h" +#include "runtime/tree_runtime_binding.h" + +namespace crowdb::tree::detail +{ +// Startup migration only. The immutable generation is pinned before any page +// I/O. Reads verify one public frame at a time and never instantiate a legacy +// writer, private executor, or mutable logical-address cache. +class LegacyChunkReadStore final : public PageStore +{ + public: + LegacyChunkReadStore(std::shared_ptr<RootCatalog> catalog, std::shared_ptr<ChunkTransport> transport, + std::shared_ptr<AllocationReservation> metadata, uint64_t tree, uint64_t epoch, + uint32_t iu_size, uint32_t mirrors); + Status open(uint64_t expected_generation); + Status resolve_migration(uint64_t expected_generation, bool *committed); + Status prepare_migration(); + Status release_pin(); + + [[nodiscard]] uint64_t generation() const noexcept + { + return layout_.generation(); + } + + [[nodiscard]] uint64_t wal_offset() const noexcept + { + return layout_.wal_offset(); + } + + [[nodiscard]] const std::shared_ptr<RootCatalog> &catalog() const noexcept + { + return catalog_; + } + + [[nodiscard]] const std::shared_ptr<ChunkTransport> &transport() const noexcept + { + return transport_; + } + + [[nodiscard]] uint64_t size() const override + { + return layout_.logical_size(); + } + + [[nodiscard]] uint32_t iu_size() const override + { + return iu_size_; + } + + Status read_at(uint64_t offset, uint8_t *data, size_t length) const override; + Status write_at(uint64_t offset, const uint8_t *data, size_t length) override; + Status sync() override; + + private: + [[nodiscard]] Status read_pack(const ChunkPageRef &ref, uint64_t offset, std::span<uint8_t> output, + std::span<uint8_t> scratch) const; + Status read_frame(const ChunkPageRef &ref, size_t logical_offset, size_t logical_bytes, std::span<uint8_t> scratch, + std::span<const uint8_t> *payload) const; + std::shared_ptr<RootCatalog> catalog_; + std::shared_ptr<ChunkTransport> transport_; + std::shared_ptr<AllocationReservation> metadata_; + ChunkRootBytes manifest_; + LegacyChunkLayout layout_; + uint64_t tree_; + uint64_t epoch_; + uint32_t iu_size_; + uint32_t mirrors_; + bool pinned_ = false; + bool verified_ = false; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.cpp b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.cpp new file mode 100644 index 000000000..e9c31d53a --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.cpp @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "legacy_chunk_recovery.h" + +#include "backend/local/local_backend_state.h" +#include "backend/local/scheduled_page_store.h" +#include "btree/backing_identity.h" +#include "chunk_frame.h" +#include "chunk_runtime_page_store.h" +#include "snapshot/local_recovery.h" +#include "snapshot/operation_wait.h" +#include "snapshot/recovered_state.h" + +namespace crowdb::tree::detail +{ +namespace +{ +struct RecoveryCredits +{ + PageLoadBudget budget; + + ~RecoveryCredits() + { + if (budget.io) { + budget.io->seal(); + } + if (budget.bytes) { + budget.bytes->seal(); + } + } +}; + +Status recover(const Config &config, const std::shared_ptr<TreeRuntimeBinding> &binding, + const std::shared_ptr<LegacyChunkReadStore> &store, size_t reader_bytes, std::shared_ptr<TreeCore> *out) +{ + const auto available = binding->options().limits.routing_bytes; + const auto lifecycle = binding->options().lifecycle_bytes; + if (lifecycle >= available || reader_bytes >= available - lifecycle) { + return Status::resource_exhausted("legacy Chunk metadata leaves no recovery workspace"); + } + RecoveryCredits credits; + auto bytes = + ResourceBudget::try_reserve(binding->account(), BudgetClass::kRouting, available - lifecycle - reader_bytes); + auto io = ResourceBudget::try_reserve(binding->account(), BudgetClass::kIo, 1); + if (!bytes || !io) { + return Status::resource_exhausted(); + } + credits.budget.bytes = AllocationReservation::create(std::move(*bytes)); + credits.budget.io = reserved_shared<AllocationReservation>(credits.budget.bytes, std::move(*io)); + auto local = reserved_shared<LocalBackendState>(binding->lifecycle_budget()); + auto status = LocalBackendDomain::claim(store.get(), store, binding->lifecycle_budget(), &local->domain); + if (status.ok()) { + status = ScheduledPageStore::create(store.get(), nullptr, store, binding, binding->lifecycle_budget(), + binding->backend_lane(), &local->store); + } + if (!status.ok()) { + return status; + } + LocalRecoveryLimits limits; + limits.accept_legacy_chunk_refs = true; + limits.max_frame_bytes = std::max(limits.max_frame_bytes, size_t{config.frame_bytes}); + limits.max_blob_bytes = round_up_to_iu(limits.max_frame_bytes + kDurableBlobHeader, store->iu_size()); + ExtentLedgerLimits domain{.backend_id = allocate_backend_identity(), + .region_begin = snapshot::region_base_for(store->iu_size()), + .file_size = store->size(), + .iu_size = store->iu_size()}; + std::shared_ptr<LocalRecoveryOperation> recovered; + status = LocalRecoveryOperation::create(local->store, binding->cpu(), binding->lane(), domain, limits, + credits.budget, &recovered); + if (status.ok()) { + status = wait_snapshot_operation(recovered); + } + if (!status.ok()) { + return status; + } + local->ledger = recovered->result().ledger; + local->checkpoints = recovered->result().registry; + local->inherited = recovered->result().latest; + if (!local->inherited) { + return Status::corruption("legacy Chunk root has no committed Tree anchor"); + } + PageLoadLimits loads; + loads.max_blob_bytes = limits.max_blob_bytes; + loads.max_frame_bytes = limits.max_frame_bytes; + auto loader = + reserved_shared<AsyncStorePageLoader>(credits.budget.bytes, local->store, binding->cpu(), binding->lane(), + loads, OverflowBackingResolver{}, credits.budget); + std::shared_ptr<RecoveredStateOperation> graph; + status = RecoveredStateOperation::create(local->inherited, loader, binding->cpu(), binding->lane(), {}, + credits.budget.bytes, &graph); + if (status.ok()) { + status = wait_snapshot_operation(graph); + } + if (!status.ok()) { + return status; + } + const auto initial = graph->result().state; + graph.reset(); + loader.reset(); + recovered.reset(); + credits.budget.io->seal(); + credits.budget.bytes->seal(); + binding->drain(); + return TreeCore::create(config, binding, {.initial = initial, .local = std::move(local)}, out); +} +} // namespace + +Status recover_legacy_chunk_source(const Config &destination, uint64_t generation, + std::shared_ptr<LegacyChunkReadStore> *store, std::shared_ptr<TreeCore> *source) +{ + auto *facade = dynamic_cast<ChunkRuntimePageStore *>(destination.page_store); + if (facade == nullptr || !destination.runtime || store == nullptr || *store || source == nullptr || *source) { + return Status::invalid_argument(); + } + std::shared_ptr<TreeRuntimeBinding> binding; + auto status = TreeRuntimeBinding::create(destination.runtime, destination.runtime_binding, &binding); + if (!status.ok()) { + return status; + } + const auto reader_bytes = (2 * kMaxCandidateManifestBytes) + ((256 * 28) + 24) + + (destination.runtime->options().backend_workers * protocol::kMaxFrameBytes) + + shared_allocation_bound<LegacyChunkReadStore>() + 4096; + auto reserved = ResourceBudget::try_reserve(binding->account(), BudgetClass::kRouting, reader_bytes); + if (!reserved) { + return Status::resource_exhausted(); + } + auto metadata = AllocationReservation::create(std::move(*reserved)); + const auto &options = facade->options(); + *store = reserved_shared<LegacyChunkReadStore>(metadata, facade->catalog(), facade->transport(), metadata, + options.tree, options.owner_epoch, options.iu_size, options.mirrors); + bool committed = false; + status = (*store)->resolve_migration(generation, &committed); + if (!status.ok() || committed) { + return status; + } + status = (*store)->open(generation); + if (!status.ok()) { + return status; + } + Config config = destination; + config.page_store = store->get(); + config.backend_owner = *store; + config.async_page_store = nullptr; + config.backend_label = "chunk-migration-source"; + return recover(config, binding, *store, reader_bytes, source); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.h b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.h new file mode 100644 index 000000000..93d8d6dec --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.h @@ -0,0 +1,13 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/tree_core.h" +#include "legacy_chunk_read_store.h" + +namespace crowdb::tree::detail +{ +// Host-only, before the destination is exposed. The caller explicitly resolves +// the generation pin after deciding whether root publication is known. +Status recover_legacy_chunk_source(const Config &destination, uint64_t generation, + std::shared_ptr<LegacyChunkReadStore> *store, std::shared_ptr<TreeCore> *source); +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocate.cpp b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocate.cpp new file mode 100644 index 000000000..fb4f353b9 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocate.cpp @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "rpc_chunk_transport_internal.h" + +namespace crowdb::tree::detail +{ +Status RpcChunkTransport::allocate_mirror_chunk(uint64_t logical_capacity, uint64_t owner_epoch, ChunkId *chunk_id, + PagePurpose purpose) +{ + return allocate_requested_chunk({}, logical_capacity, owner_epoch, chunk_id, purpose); +} + +Status RpcChunkTransport::allocate_fixed_chunk(ChunkId id, uint64_t capacity, uint64_t epoch, PagePurpose purpose) +{ + if (id.empty()) { + return Status::invalid_argument("fixed chunk allocation requires a nonempty identity"); + } + ChunkId allocated; + return allocate_requested_chunk(id, capacity, epoch, &allocated, purpose); +} + +Status RpcChunkTransport::allocate_requested_chunk(ChunkId requested, uint64_t logical_capacity, uint64_t owner_epoch, + ChunkId *chunk_id, PagePurpose purpose) const +{ + if (!valid() || chunk_id == nullptr || logical_capacity == 0 || !valid_page_purpose(purpose) || + logical_capacity > std::numeric_limits<uint32_t>::max() || + (!requested.empty() && chunk_purpose(requested) != purpose)) { + return Status::invalid_argument("tree chunk RPC allocation arguments are invalid"); + } + const uint64_t request_id = impl_->next_request_id(); + const uint64_t initial_capacity = + requested.empty() ? std::min(logical_capacity, uint64_t{4U * 1024U * 1024U}) : logical_capacity; + const auto geometry = + rpc_chunk_allocation(initial_capacity, impl_->options.strip_capacity_bytes, impl_->options.mirror_copies); + const FBInt128 requested_id(requested.high, requested.low); + flatbuffers::FlatBufferBuilder builder; + auto request = crowdb::chunkdb::proto::CreateFBAllocateChunkRequest( + builder, request_id, monotonic_nanos(), requested.empty() ? nullptr : &requested_id, geometry.granularity_kb, + geometry.strip_count, FBStripType_Mirror, 0, 0, + impl_->options.mirror_copies == 0 ? 2 : impl_->options.mirror_copies, + static_cast<crowdb::chunkdb::proto::FBChunkType>(purpose), owner_epoch, impl_->options.writer_lease_ms); + builder.Finish(request); + std::vector<uint8_t> control(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); + RpcResult result; + Status status = impl_->call_chunkdb<crowdb::chunkdb::proto::FBAllocateChunkResponse>( + requested, request_id, crowdb::rpc::proto::FBMsgType_EAllocateChunkRequest, control, &result); + if (!status.ok()) { + return status; + } + const auto *response = verified_response<crowdb::chunkdb::proto::FBAllocateChunkResponse>(result.control); + if (response == nullptr) { + return Status::corruption("ChunkDB allocation response is malformed"); + } + status = chunkdb_status(response->ret_code(), response->error_msg()); + if (!status.ok()) { + return status; + } + Impl::RemoteChunk remote; + status = crowdb::tree::detail::RpcChunkTransport::Impl::parse_chunk(response->chunk(), &remote); + if (!status.ok() || remote.layout.chunk_id.empty() || (!requested.empty() && remote.layout.chunk_id != requested) || + remote.owner_epoch != owner_epoch || remote.layout.logical_capacity < initial_capacity || + chunk_purpose(remote.layout.chunk_id) != purpose) { + return status.ok() ? Status::corruption("ChunkDB allocation metadata mismatch") : status; + } + remote.maximum_capacity = logical_capacity; + remote.valid_until_ms = std::numeric_limits<uint64_t>::max(); + remote.self_fence_deadline_ms = monotonic_millis() + kLivenessSelfFenceMs; + *chunk_id = remote.layout.chunk_id; + impl_->cache(std::move(remote)); + return Status::Ok(); +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocation_identity.cpp b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocation_identity.cpp new file mode 100644 index 000000000..8a19f80b4 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocation_identity.cpp @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "rpc_chunk_transport_internal.h" + +namespace crowdb::tree::detail +{ +Status RpcChunkTransport::inspect_fixed_allocation(ChunkId id, uint64_t epoch, ChunkAllocationIdentity *out) const +{ + using namespace crowdb::chunkdb::proto; + if (!valid() || id.empty() || epoch == 0 || out == nullptr) { + return Status::invalid_argument(); + } + const auto request_id = impl_->next_request_id(); + const FBInt128 wire_id(id.high, id.low); + flatbuffers::FlatBufferBuilder query; + query.Finish(CreateFBQueryChunkRequest(query, request_id, monotonic_nanos(), &wire_id)); + std::vector<uint8_t> control(query.GetBufferPointer(), query.GetBufferPointer() + query.GetSize()); + RpcResult result; + auto status = impl_->call_chunkdb<FBQueryChunkResponse>( + id, request_id, crowdb::rpc::proto::FBMsgType_EQueryChunkRequest, control, &result); + if (!status.ok()) { + return status; + } + const auto *response = verified_response<FBQueryChunkResponse>(result.control); + if (response == nullptr) { + return Status::corruption("Chunk allocation query is malformed"); + } + status = chunkdb_status(response->ret_code(), response->error_msg()); + if (!status.ok()) { + return status; + } + const auto *chunk = response->chunk(); + if (chunk == nullptr || chunk->id() == nullptr || chunk->id()->high() != id.high || chunk->id()->low() != id.low) { + return Status::corruption("Chunk allocation identity differs"); + } + if (chunk->state() == FBChunkState_Deleted && chunk->capacity() == 0 && + (chunk->strips() == nullptr || chunk->strips()->empty())) { + return Status::not_found(); + } + if (chunk->writer_epoch() != epoch) { + return Status::unavailable("retired stage names another physical owner"); + } + if (chunk->strips() == nullptr || chunk->strips()->empty()) { + return Status::corruption("Chunk allocation has no physical segments"); + } + uint64_t bytes = 0; + for (const auto *strip : *chunk->strips()) { + const auto *mirror = strip == nullptr ? nullptr : strip->strip_body_as_FBMirrorStrip(); + if (strip == nullptr || strip->strip_type() != FBStripType_Mirror || strip->unit_kb() == 0 || + mirror == nullptr || mirror->segments() == nullptr || mirror->segments()->empty()) { + return Status::corruption("Chunk allocation is not a complete mirror layout"); + } + const uint64_t unit = static_cast<uint64_t>(strip->unit_kb()) * 1024; + for (const auto *segment : *mirror->segments()) { + if (segment == nullptr || segment->unit_count() == 0 || segment->owner_chunk().high() != id.high || + segment->owner_chunk().low() != id.low || segment->unit_count() > (UINT64_MAX - bytes) / unit) { + return Status::corruption("Chunk physical segment identity or capacity differs"); + } + bytes += segment->unit_count() * unit; + } + } + *out = {.chunk = id, .owner_epoch = epoch, .created_ms = chunk->create_ts_ms(), .physical_bytes = bytes}; + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_delete.cpp b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_delete.cpp new file mode 100644 index 000000000..a1f616dbc --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_delete.cpp @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "rpc_chunk_transport_internal.h" + +namespace crowdb::tree::detail +{ +Status RpcChunkTransport::delete_chunk(ChunkId id) +{ + using namespace crowdb::chunkdb::proto; + if (!valid() || id.empty() || !valid_page_purpose(chunk_purpose(id))) { + return Status::invalid_argument("chunk deletion requires a concrete tree chunk identity"); + } + const uint64_t request_id = impl_->next_request_id(); + const FBInt128 wire_id(id.high, id.low); + flatbuffers::FlatBufferBuilder builder; + builder.Finish(CreateFBDeleteChunkRequest(builder, request_id, monotonic_nanos(), &wire_id, true)); + std::vector<uint8_t> control(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); + RpcResult result; + auto status = impl_->call_chunkdb<FBDeleteChunkResponse>( + id, request_id, crowdb::rpc::proto::FBMsgType_EDeleteChunkRequest, control, &result); + if (!status.ok()) { + return status; + } + const auto *response = verified_response<FBDeleteChunkResponse>(result.control); + if (response == nullptr) { + return Status::corruption("ChunkDB deletion response is malformed"); + } + status = chunkdb_status(response->ret_code(), response->error_msg()); + if (status.code() == Code::kNotFound) { + // The caller has already fenced late allocation and durable Deleting + // forbids new pins. An exact absent identity has no physical strips. + impl_->forget(id); + return Status::Ok(); + } + if (!status.ok()) { + return status; + } + const auto *chunk = response->chunk(); + if (chunk == nullptr || chunk->id() == nullptr || chunk->id()->high() != id.high || chunk->id()->low() != id.low || + chunk->state() != FBChunkState_Deleted || chunk->capacity() != 0 || + (chunk->strips() != nullptr && !chunk->strips()->empty())) { + return Status::corruption("ChunkDB deletion has not released every physical strip"); + } + impl_->forget(id); + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_drain.cpp b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_drain.cpp new file mode 100644 index 000000000..cb7ce689e --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_drain.cpp @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "rpc_chunk_transport_internal.h" + +namespace crowdb::tree::detail +{ +namespace +{ +template <class Segment> +Status drain_segment(const Segment &segment, const RpcChunkRouteLease &lease, uint64_t request_id, uint64_t offset) +{ + const FBInt128 disk(segment.disk_high, segment.disk_low); + flatbuffers::FlatBufferBuilder builder; + builder.Finish(crowdb::diskio::proto::CreateFBDiskWriteRequest( + builder, request_id, monotonic_nanos(), &disk, segment.zone_index, offset, 0, offset, wall_time_ms(), true)); + std::vector<uint8_t> request(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); + RpcResult drained; + auto status = call_rpc(lease.route, request_id, crowdb::rpc::proto::FBMsgType_EDiskWriteRequest, request, nullptr, + 0, &drained); + if (!status.ok()) { + return status; + } + const auto *receipt = verified_response<crowdb::diskio::proto::FBDiskWriteResponse>(drained.control); + if (receipt == nullptr) { + return Status::corruption("DiskIO drain response is malformed"); + } + status = diskio_status(receipt->ret_code()); + if (!status.ok()) { + return status; + } + if (!receipt->sync_completed()) { + return Status::not_supported("DiskIO cannot confirm ordered drain"); + } + return Status::Ok(); +} +} // namespace + +Status RpcChunkTransport::drain_fixed_chunk(ChunkId id) const +{ + using namespace crowdb::chunkdb::proto; + if (!valid() || id.empty() || !valid_page_purpose(chunk_purpose(id))) { + return Status::invalid_argument(); + } + const auto query_id = impl_->next_request_id(); + const FBInt128 wire_id(id.high, id.low); + flatbuffers::FlatBufferBuilder query; + query.Finish(CreateFBQueryChunkRequest(query, query_id, monotonic_nanos(), &wire_id)); + std::vector<uint8_t> control(query.GetBufferPointer(), query.GetBufferPointer() + query.GetSize()); + RpcResult result; + auto status = impl_->call_chunkdb<FBQueryChunkResponse>( + id, query_id, crowdb::rpc::proto::FBMsgType_EQueryChunkRequest, control, &result); + if (!status.ok()) { + return status; + } + const auto *response = verified_response<FBQueryChunkResponse>(result.control); + if (response == nullptr) { + return Status::corruption("ChunkDB drain query is malformed"); + } + status = chunkdb_status(response->ret_code(), response->error_msg()); + if (status.code() == Code::kNotFound) { + return Status::Ok(); + } + if (!status.ok()) { + return status; + } + const auto *chunk = response->chunk(); + if (chunk == nullptr || chunk->id() == nullptr || chunk->id()->high() != id.high || chunk->id()->low() != id.low) { + return Status::corruption("ChunkDB drain identity differs"); + } + if (chunk->strips() == nullptr) { + return chunk->capacity() == 0 ? Status::Ok() : Status::corruption("ChunkDB drain layout is absent"); + } + for (const auto *strip : *chunk->strips()) { + const auto *mirror = strip == nullptr ? nullptr : strip->strip_body_as_FBMirrorStrip(); + if (strip == nullptr || strip->strip_type() != FBStripType_Mirror || strip->unit_kb() == 0 || + mirror == nullptr || mirror->segments() == nullptr) { + return Status::corruption("ChunkDB drain layout is invalid"); + } + for (const auto *segment : *mirror->segments()) { + const uint64_t unit_bytes = static_cast<uint64_t>(strip->unit_kb()) * 1024; + if (unit_bytes == 0 || segment == nullptr || segment->unit_offset() > UINT64_MAX / unit_bytes) { + return Status::corruption(); + } + const uint64_t offset = segment->unit_offset() * unit_bytes; + Impl::Segment route_segment{.disk_high = segment->disk_id().high(), + .disk_low = segment->disk_id().low(), + .unit_offset = segment->unit_offset(), + .zone_index = segment->zone_index()}; + RpcChunkRouteLease lease; + status = impl_->resolve_route(route_segment, &lease); + if (!status.ok()) { + return status; + } + status = drain_segment(route_segment, lease, impl_->next_request_id(), offset); + if (!status.ok()) { + return status; + } + } + } + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_growth.cpp b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_growth.cpp index 5a9c0e488..48a4f93f1 100644 --- a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_growth.cpp +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_growth.cpp @@ -11,52 +11,6 @@ namespace constexpr uint64_t kGrowthBytes = 4U * 1024U * 1024U; } -Status RpcChunkTransport::allocate_mirror_chunk(uint64_t logical_capacity, uint64_t owner_epoch, ChunkId *chunk_id, - PagePurpose purpose) -{ - if (!valid() || chunk_id == nullptr || logical_capacity == 0 || !valid_page_purpose(purpose) || - logical_capacity > std::numeric_limits<uint32_t>::max()) { - return Status::invalid_argument("tree chunk RPC allocation arguments are invalid"); - } - const uint64_t request_id = impl_->next_request_id(); - const uint64_t initial_capacity = std::min(logical_capacity, kGrowthBytes); - const auto geometry = - rpc_chunk_allocation(initial_capacity, impl_->options.strip_capacity_bytes, impl_->options.mirror_copies); - flatbuffers::FlatBufferBuilder builder; - auto request = crowdb::chunkdb::proto::CreateFBAllocateChunkRequest( - builder, request_id, monotonic_nanos(), nullptr, geometry.granularity_kb, geometry.strip_count, - FBStripType_Mirror, 0, 0, impl_->options.mirror_copies == 0 ? 2 : impl_->options.mirror_copies, - static_cast<crowdb::chunkdb::proto::FBChunkType>(purpose), owner_epoch, impl_->options.writer_lease_ms); - builder.Finish(request); - std::vector<uint8_t> control(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); - RpcResult result; - Status status = impl_->call_chunkdb<crowdb::chunkdb::proto::FBAllocateChunkResponse>( - {}, request_id, crowdb::rpc::proto::FBMsgType_EAllocateChunkRequest, control, &result); - if (!status.ok()) { - return status; - } - const auto *response = verified_response<crowdb::chunkdb::proto::FBAllocateChunkResponse>(result.control); - if (response == nullptr) { - return Status::corruption("ChunkDB allocation response is malformed"); - } - status = chunkdb_status(response->ret_code(), response->error_msg()); - if (!status.ok()) { - return status; - } - Impl::RemoteChunk remote; - status = crowdb::tree::detail::RpcChunkTransport::Impl::parse_chunk(response->chunk(), &remote); - if (!status.ok() || remote.layout.chunk_id.empty() || remote.owner_epoch != owner_epoch || - remote.layout.logical_capacity < initial_capacity || chunk_purpose(remote.layout.chunk_id) != purpose) { - return status.ok() ? Status::corruption("ChunkDB allocation metadata mismatch") : status; - } - remote.valid_until_ms = std::numeric_limits<uint64_t>::max(); - remote.maximum_capacity = logical_capacity; - remote.self_fence_deadline_ms = monotonic_millis() + kLivenessSelfFenceMs; - *chunk_id = remote.layout.chunk_id; - impl_->cache(std::move(remote)); - return Status::Ok(); -} - Status RpcChunkTransport::ensure_capacity(ChunkId chunk_id, uint64_t required_bytes) { // Only the ordered planner grows chunks. Mirror submissions never allocate, diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_read_fence.cpp b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_read_fence.cpp new file mode 100644 index 000000000..5a88bfada --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_read_fence.cpp @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "rpc_chunk_transport_internal.h" + +namespace crowdb::tree::detail +{ +Status RpcChunkTransport::verify_fixed_read_fence(ChunkId id, uint32_t mirrors, ChunkIoContext context) const +{ + if (context.created_ms == 0 || mirrors == 0 || mirrors > kMaxMirrorCopies) { + return Status::invalid_argument(); + } + Impl::RemoteChunk chunk; + if (!impl_->cached_valid(id, &chunk)) { + auto status = impl_->query_remote(id, &chunk); + if (!status.ok()) { + return status; + } + } + for (const auto &strip : chunk.strips) { + if (strip.mirrors.size() < mirrors) { + return Status::invalid_argument(); + } + for (uint32_t index = 0; index < mirrors; ++index) { + const auto &segment = strip.mirrors[index]; + RpcChunkRouteLease lease; + auto status = impl_->resolve_route(segment, &lease); + if (!status.ok()) { + return status; + } + const auto request_id = impl_->next_request_id(); + const FBInt128 disk(segment.disk_high, segment.disk_low); + flatbuffers::FlatBufferBuilder builder; + builder.Finish(crowdb::diskio::proto::CreateFBDiskReadRequest( + builder, request_id, monotonic_nanos(), &disk, segment.zone_index, 0, 0, 0, context.created_ms, true)); + std::vector<uint8_t> control(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); + RpcResult result; + status = call_rpc(lease.route, request_id, crowdb::rpc::proto::FBMsgType_EDiskReadRequest, control, nullptr, + 0, &result); + if (!status.ok()) { + return status; + } + const auto *response = verified_response<crowdb::diskio::proto::FBDiskReadResponse>(result.control); + if (response == nullptr) { + return Status::corruption("DiskIO fence response is malformed"); + } + status = diskio_status(response->ret_code()); + if (!status.ok()) { + return status; + } + if (!response->request_age_checked()) { + return Status::not_supported("DiskIO cannot fence fixed-pack reads"); + } + } + } + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_result.h b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_result.h index e84b13c34..58e65bf2e 100644 --- a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_result.h +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_result.h @@ -4,6 +4,7 @@ #pragma once #include "crowdb-rpc/c_api.h" +#include "host_completion.h" #include <atomic> #include <utility> @@ -59,8 +60,8 @@ struct RpcResult struct RpcCallState { - std::atomic<bool> done{false}; - RpcResult result; + HostCompletion done; + RpcResult result; }; inline void rpc_complete(uint64_t /*unused*/, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, @@ -70,8 +71,7 @@ inline void rpc_complete(uint64_t /*unused*/, crowdb_rpc_buffer_t control, crowd state->result.status = status; state->result.control = control; state->result.data = data; - state->done.store(true, std::memory_order_release); - state->done.notify_one(); + state->done.signal(); } } // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.cpp b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.cpp index 51dbc2b7b..60b1b9ccc 100644 --- a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.cpp +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.cpp @@ -147,6 +147,21 @@ bool RpcChunkTransport::valid() const Status RpcChunkTransport::write_mirror(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, const uint8_t *data, size_t length) +{ + return write_mirror_impl(chunk_id, mirror_index, offset, data, length, nullptr); +} + +Status RpcChunkTransport::write_fixed_mirror(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, + const uint8_t *data, size_t length, ChunkIoContext context) +{ + if (context.created_ms == 0) { + return Status::invalid_argument(); + } + return write_mirror_impl(chunk_id, mirror_index, offset, data, length, &context); +} + +Status RpcChunkTransport::write_mirror_impl(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, + const uint8_t *data, size_t length, const ChunkIoContext *context) { if ((data == nullptr && length != 0)) { return Status::invalid_argument("tree chunk RPC mirror write arguments are invalid"); @@ -186,7 +201,8 @@ Status RpcChunkTransport::write_mirror(ChunkId chunk_id, uint32_t mirror_index, flatbuffers::FlatBufferBuilder builder; auto request = crowdb::diskio::proto::CreateFBDiskWriteRequest( builder, request_id, monotonic_nanos(), &disk_id, segment.zone_index, zone_offset, - static_cast<uint32_t>(part), segment_base, wall_time_ms()); + static_cast<uint32_t>(part), segment_base, context != nullptr ? context->created_ms : wall_time_ms(), + context != nullptr); builder.Finish(request); std::vector<uint8_t> control(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); RpcResult result; @@ -200,6 +216,9 @@ Status RpcChunkTransport::write_mirror(ChunkId chunk_id, uint32_t mirror_index, return Status::corruption("DiskIO write response is malformed"); } status = diskio_status(response->ret_code()); + if (status.ok() && context != nullptr && !response->sync_completed()) { + return Status::not_supported("fixed pack write requires synchronized completion"); + } if (!status.ok()) { return status; } @@ -275,6 +294,21 @@ Status RpcChunkTransport::advance_write(ChunkId chunk_id, uint64_t expected_byte Status RpcChunkTransport::read_mirror(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, uint8_t *data, size_t length) const +{ + return read_mirror_impl(chunk_id, mirror_index, offset, data, length, nullptr); +} + +Status RpcChunkTransport::read_fixed_mirror(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, uint8_t *data, + size_t length, ChunkIoContext context) const +{ + if (context.created_ms == 0) { + return Status::invalid_argument(); + } + return read_mirror_impl(chunk_id, mirror_index, offset, data, length, &context); +} + +Status RpcChunkTransport::read_mirror_impl(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, uint8_t *data, + size_t length, const ChunkIoContext *context) const { if ((data == nullptr && length != 0)) { return Status::invalid_argument("tree chunk RPC mirror read arguments are invalid"); @@ -315,9 +349,9 @@ Status RpcChunkTransport::read_mirror(ChunkId chunk_id, uint32_t mirror_index, u const uint64_t request_id = impl_->next_request_id(); const FBInt128 disk_id(segment.disk_high, segment.disk_low); flatbuffers::FlatBufferBuilder builder; - auto request = crowdb::diskio::proto::CreateFBDiskReadRequest(builder, request_id, monotonic_nanos(), &disk_id, - segment.zone_index, zone_offset, - static_cast<uint32_t>(part), 0); + auto request = crowdb::diskio::proto::CreateFBDiskReadRequest( + builder, request_id, monotonic_nanos(), &disk_id, segment.zone_index, zone_offset, + static_cast<uint32_t>(part), 0, context != nullptr ? context->created_ms : 0, false); builder.Finish(request); std::vector<uint8_t> control(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); RpcResult result; @@ -331,6 +365,9 @@ Status RpcChunkTransport::read_mirror(ChunkId chunk_id, uint32_t mirror_index, u return Status::corruption("DiskIO read response is malformed"); } status = diskio_status(response->ret_code()); + if (status.ok() && context != nullptr && !response->request_age_checked()) { + return Status::not_supported("fixed pack read requires request-age fencing"); + } if (!status.ok()) { return status; } diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.h b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.h index 200bdd00f..130e030aa 100644 --- a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.h +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.h @@ -23,6 +23,21 @@ class RpcChunkTransport final : public ChunkTransport Status allocate_mirror_chunk(uint64_t logical_capacity, uint64_t owner_epoch, ChunkId *chunk_id, PagePurpose purpose) override; Status ensure_capacity(ChunkId chunk_id, uint64_t required_bytes) override; + Status allocate_fixed_chunk(ChunkId id, uint64_t capacity, uint64_t epoch, PagePurpose purpose) override; + Status delete_chunk(ChunkId id) override; + Status inspect_fixed_allocation(ChunkId id, uint64_t epoch, ChunkAllocationIdentity *out) const override; + [[nodiscard]] Status drain_fixed_chunk(ChunkId id) const override; + [[nodiscard]] Status verify_fixed_read_fence(ChunkId id, uint32_t mirrors, ChunkIoContext context) const override; + Status write_fixed_mirror(ChunkId id, uint32_t mirror, uint64_t offset, const uint8_t *data, size_t length, + ChunkIoContext context) override; + Status read_fixed_mirror(ChunkId id, uint32_t mirror, uint64_t offset, uint8_t *data, size_t length, + ChunkIoContext context) const override; + + [[nodiscard]] bool fixed_staging_supported() const noexcept override + { + return true; + } + Status write_mirror(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, const uint8_t *data, size_t length) override; void submit_write_mirror(ChunkId chunk_id, uint32_t mirror_index, uint64_t offset, const uint8_t *data, @@ -36,6 +51,12 @@ class RpcChunkTransport final : public ChunkTransport [[nodiscard]] bool valid() const; private: + Status allocate_requested_chunk(ChunkId requested, uint64_t logical_capacity, uint64_t owner_epoch, + ChunkId *chunk_id, PagePurpose purpose) const; + Status write_mirror_impl(ChunkId id, uint32_t mirror, uint64_t offset, const uint8_t *data, size_t length, + const ChunkIoContext *context); + Status read_mirror_impl(ChunkId id, uint32_t mirror, uint64_t offset, uint8_t *data, size_t length, + const ChunkIoContext *context) const; struct Impl; std::unique_ptr<Impl> impl_; }; diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport_internal.h b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport_internal.h index ed1875c19..21fbc2f00 100644 --- a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport_internal.h +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport_internal.h @@ -89,9 +89,7 @@ inline Status call_rpc(const ct_chunk_rpc_route &route, uint64_t request_id, uin return submit == CROWDB_RPC_ERR_SEND_QUEUE ? Status::resource_exhausted("chunk RPC submission queue is full") : Status::unavailable("chunk RPC submission failed"); } - while (!state.done.load(std::memory_order_acquire)) { - state.done.wait(false, std::memory_order_acquire); - } + state.done.wait(); *out = std::move(state.result); return out->status == CROWDB_RPC_OK ? Status::Ok() : Status::unavailable("chunk RPC call failed"); } @@ -114,6 +112,8 @@ inline Status chunkdb_status(FBChunkdbRetCode code, const flatbuffers::String *e switch (code) { case FBChunkdbRetCode_Success: return Status::Ok(); + case crowdb::chunkdb::proto::FBChunkdbRetCode_NotFound: + return Status::not_found(message); case FBChunkdbRetCode_InvalidArgument: case FBChunkdbRetCode_FailedPrecondition: case FBChunkdbRetCode_StripIndexOutOfRange: @@ -267,7 +267,7 @@ struct RpcChunkTransport::Impl return status; } - uint64_t next_request_id() const + [[nodiscard]] uint64_t next_request_id() const { return request_ids.fetch_add(1, std::memory_order_relaxed); } @@ -378,6 +378,18 @@ struct RpcChunkTransport::Impl } } + void forget(ChunkId id) const + { + auto current = chunks.load(std::memory_order_acquire); + while (current != nullptr) { + auto next = std::make_shared<RemoteChunks>(*current); + std::erase_if(*next, [id](const RemoteChunk &entry) { return entry.layout.chunk_id == id; }); + if (chunks.compare_exchange_weak(current, next, std::memory_order_release, std::memory_order_acquire)) { + return; + } + } + } + [[nodiscard]] bool cached(ChunkId chunk_id, RemoteChunk *out) const { auto current = chunks.load(std::memory_order_acquire); diff --git a/lib/crowdb-tree/src/backend/local/backend_io_allocation.cpp b/lib/crowdb-tree/src/backend/local/backend_io_allocation.cpp new file mode 100644 index 000000000..6cd64683e --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/backend_io_allocation.cpp @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend_io_allocation.h" +#ifdef CROWDB_HAVE_LIBURING +namespace crowdb::tree::detail +{ +::crowdb::common::IOAllocation io_allocation(const std::shared_ptr<AllocationBudget> &budget) +{ + return {.owner = budget, + .context = budget.get(), + .allocate = [](void *context, size_t bytes, size_t alignment) -> void * { + auto &budget = *static_cast<AllocationBudget *>(context); + if (!budget.claim(bytes)) { + throw std::bad_alloc(); + } + try { + auto *storage = ::operator new(bytes, std::align_val_t{alignment}); + budget.record_allocation(AllocationKind::kMetadata, bytes, true); + return storage; + } + catch (...) { + budget.release(bytes); + throw; + } + }, + .deallocate = + [](void *context, void *storage, size_t bytes, size_t alignment) noexcept { + auto &budget = *static_cast<AllocationBudget *>(context); + ::operator delete(storage, std::align_val_t{alignment}); + budget.record_allocation(AllocationKind::kMetadata, bytes, false); + budget.release(bytes); + }}; +} +} // namespace crowdb::tree::detail +#endif diff --git a/lib/crowdb-tree/src/backend/local/backend_io_allocation.h b/lib/crowdb-tree/src/backend/local/backend_io_allocation.h new file mode 100644 index 000000000..6de648759 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/backend_io_allocation.h @@ -0,0 +1,10 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/backend/async_page_store.h" +#ifdef CROWDB_HAVE_LIBURING +namespace crowdb::tree::detail +{ +::crowdb::common::IOAllocation io_allocation(const std::shared_ptr<AllocationBudget> &budget); +} +#endif diff --git a/lib/crowdb-tree/src/backend/local/backend_metadata.cpp b/lib/crowdb-tree/src/backend/local/backend_metadata.cpp new file mode 100644 index 000000000..bcefbcea7 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/backend_metadata.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend_metadata.h" + +#include <algorithm> +#include <limits> + +namespace crowdb::tree::detail +{ +namespace +{ +struct Header +{ + std::shared_ptr<AllocationBudget> budget; + size_t bytes; + size_t alignment; +}; +} // namespace + +BackendObjectStorage::BackendObjectStorage(size_t bytes, size_t alignment, std::shared_ptr<AllocationBudget> budget) +{ + alignment = std::max(alignment, alignof(Header)); + const auto prefix = (sizeof(Header) + sizeof(Header *) + alignment - 1) & ~(alignment - 1); + if (bytes > std::numeric_limits<size_t>::max() - prefix) { + throw std::bad_array_new_length(); + } + bytes += prefix; + if (budget && !budget->claim(bytes)) { + throw std::bad_alloc(); + } + try { + auto *base = static_cast<std::byte *>(::operator new(bytes, std::align_val_t{alignment})); + auto *header = + ::new (static_cast<void *>(base)) Header{.budget = budget, .bytes = bytes, .alignment = alignment}; + pointer_ = base + prefix; + ::new (static_cast<void *>(base + prefix - sizeof(Header *))) Header *(header); + if (budget) { + budget->record_allocation(AllocationKind::kMetadata, bytes, true); + } + } + catch (...) { + if (budget) { + budget->release(bytes); + } + throw; + } +} + +BackendObjectStorage::~BackendObjectStorage() +{ + destroy(pointer_); +} + +void BackendObjectStorage::destroy(void *pointer) noexcept +{ + if (pointer == nullptr) { + return; + } + auto *header = *(reinterpret_cast<Header **>(pointer) - 1); + auto budget = std::move(header->budget); + const auto bytes = header->bytes; + const auto alignment = header->alignment; + std::destroy_at(header); + ::operator delete(header, std::align_val_t{alignment}); + if (budget) { + budget->record_allocation(AllocationKind::kMetadata, bytes, false); + budget->release(bytes); + } +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/local/backend_metadata.h b/lib/crowdb-tree/src/backend/local/backend_metadata.h new file mode 100644 index 000000000..e04285863 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/backend_metadata.h @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" + +namespace crowdb::tree::detail +{ +// Ordinary unique_ptr deletion must release the object storage before its +// admission. A small allocation header keeps that owner past the destructor. +class BackendObjectStorage +{ + public: + BackendObjectStorage(size_t bytes, size_t alignment, std::shared_ptr<AllocationBudget> budget); + ~BackendObjectStorage(); + BackendObjectStorage(const BackendObjectStorage &) = delete; + BackendObjectStorage &operator=(const BackendObjectStorage &) = delete; + + [[nodiscard]] void *get() const noexcept + { + return pointer_; + } + + [[nodiscard]] void *release() noexcept + { + return std::exchange(pointer_, nullptr); + } + + static void destroy(void *pointer) noexcept; + + private: + void *pointer_ = nullptr; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/local/block_account.cpp b/lib/crowdb-tree/src/backend/local/block_account.cpp new file mode 100644 index 000000000..abeb16085 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_account.cpp @@ -0,0 +1,192 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "block_account.h" + +#include "block_async_io.h" +#include "block_directory.h" +#include "btree/allocation_account.h" +#include "btree/allocation_account_grant.h" +#include "btree/allocation_bound.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "crowdb-tree/epoch.h" + +#include <climits> + +namespace crowdb::tree::detail +{ +struct BlockAccountAccess +{ + static std::optional<size_t> operation_bound(const BlockPageStore &store, BackendOperationLimits limits) noexcept; + + static Status create_grant(const BlockPageStore &store, ResourceBudget::Credit credit, + std::shared_ptr<AllocationBudget> *out) + { + std::shared_ptr<BackendAllocationGrant> grant; + auto status = BackendAllocationGrant::create(store.allocations_, std::move(credit), &grant); + if (status.ok()) { + *out = std::move(grant); + } + return status; + } + + static std::shared_ptr<AllocationBudget> selected(const BlockPageStore &store, + const std::shared_ptr<AllocationBudget> &budget) + { + if (!valid(store, budget)) { + return nullptr; + } + return budget ? budget : store.allocations_; + } + + static bool valid(const BlockPageStore &store, const std::shared_ptr<AllocationBudget> &budget) noexcept + { + if (!budget || budget == store.allocations_) { + return true; + } + const auto *grant = dynamic_cast<const BackendAllocationGrant *>(budget.get()); + return grant != nullptr && grant->account() == store.allocations_; + } + + static Status bind(BlockPageStore &store, const std::shared_ptr<ResourceBudget::Account> &account, size_t workers, + BackendOpenBudget *out) + { + // Recovery only reads existing extents. Each backend worker and the + // opening host thread may need one previously unseen epoch participant. + const auto growth = AllocationBound(EpochManager::participant_allocation_bytes()) + .add_product(workers, EpochManager::participant_allocation_bytes()) + .value(); + if (!growth) { + return Status::resource_exhausted("backend startup metadata bound overflow"); + } + auto status = store.allocations_->rebind(account); + if (!status.ok()) { + return status; + } + out->owned_bytes = store.allocations_->owned_bytes(); + out->growth_bytes = store.blocks_ ? *growth : 0; + return Status::Ok(); + } +}; + +std::shared_ptr<AllocationBudget> block_allocation_budget(const BlockPageStore &store, + const std::shared_ptr<AllocationBudget> &budget) +{ + return BlockAccountAccess::selected(store, budget); +} + +bool valid_block_budget(const BlockPageStore &store, const std::shared_ptr<AllocationBudget> &budget) noexcept +{ + return BlockAccountAccess::valid(store, budget); +} + +std::optional<size_t> BlockAccountAccess::operation_bound(const BlockPageStore &store, + BackendOperationLimits limits) noexcept +{ + const auto unit = std::max<size_t>(1, store.iu_size_); + const auto max_part = (static_cast<size_t>(INT_MAX) / unit) * unit; + if (max_part == 0 || limits.workers == 0) { + return {}; + } + const auto part_bytes = store.blocks_ ? std::min<uint64_t>(store.block_size_, max_part) : max_part; + const auto parts = AllocationBound(limits.max_io_bytes / part_bytes).add(2).value(); + const auto new_extents = store.blocks_ && parts ? AllocationBound().add_product(limits.write_calls, *parts).value() + : std::optional<size_t>{0}; + if (!parts || !new_extents) { + return {}; + } + const auto tail_count = AllocationBound(limits.workers).add(1).value(); + if (!tail_count) { + return {}; + } + auto bound = AllocationBound(BackendAllocationGrant::allocation_bound()) + .add_product(*tail_count, EpochManager::participant_allocation_bytes()) + .add_product(*new_extents, BlockDirectory::extent_allocation_bound()) + .add_product(2, store.dir_.size() + 65); + auto window = AllocationBound(limits.max_io_bytes).add_product(2, unit); +#ifdef CROWDB_HAVE_LIBURING + if (limits.asynchronous) { + window = AllocationBound(); + window.add(block_io_allocation_bound(limits.max_io_bytes, *parts)); + if (limits.barriers) { + const auto targets = AllocationBound(store.blocks_ ? store.blocks_->count() : 1).add(*new_extents).value(); + window.add(targets ? block_sync_allocation_bound(*targets) : std::nullopt); + } + // Completed asynchronous callbacks can retain paid controls while + // the next CPU worker starts another admitted I/O window. + return window.value() ? bound.add_product(*tail_count, *window.value()).value() : std::nullopt; + } +#endif + // Blocking adapters free their scratch span before invoking completion. + return bound.add(window.value()).value(); +} + +std::optional<size_t> backend_operation_bound(PageStore *backend, BackendOperationLimits limits) noexcept +{ + auto *store = dynamic_cast<BlockPageStore *>(backend); + return store != nullptr ? BlockAccountAccess::operation_bound(*store, limits) : std::optional<size_t>{0}; +} + +Status reserve_backend_operation(PageStore *backend, const std::shared_ptr<ResourceBudget::Account> &account, + BackendOperationLimits limits, std::shared_ptr<AllocationBudget> *out) +{ + if (out == nullptr || *out) { + return Status::invalid_argument(); + } + auto *store = dynamic_cast<BlockPageStore *>(backend); + if (store == nullptr) { + return Status::Ok(); + } + const auto bound = BlockAccountAccess::operation_bound(*store, limits); + if (!bound) { + return Status::resource_exhausted("backend operation bound overflow"); + } + auto credit = ResourceBudget::try_reserve(account, BudgetClass::kRouting, *bound); + if (!credit) { + return Status::resource_exhausted("backend operation routing credit unavailable"); + } + return bind_backend_operation(backend, limits, std::move(*credit), out); +} + +Status bind_backend_operation(PageStore *backend, BackendOperationLimits limits, ResourceBudget::Credit credit, + std::shared_ptr<AllocationBudget> *out) +{ + if (out == nullptr || *out || credit.allocated()) { + return Status::invalid_argument("backend binding requires an unused operation grant"); + } + auto *store = dynamic_cast<BlockPageStore *>(backend); + if (store == nullptr) { + return credit.amount() == 0 ? Status::Ok() + : Status::invalid_argument("external backend has no local operation grant"); + } + const auto bound = BlockAccountAccess::operation_bound(*store, limits); + if (!bound || credit.amount() < *bound) { + return Status::resource_exhausted("backend grant is smaller than the complete operation bound"); + } + return BlockAccountAccess::create_grant(*store, std::move(credit), out); +} + +uint64_t backend_size(PageStore &store, const std::shared_ptr<AllocationBudget> &budget) +{ + auto *block = dynamic_cast<BlockPageStore *>(&store); + return block != nullptr ? block->size(budget) : store.size(); +} + +void seal_backend_budget(const std::shared_ptr<AllocationBudget> &budget) noexcept +{ + auto *grant = dynamic_cast<BackendAllocationGrant *>(budget.get()); + if (grant != nullptr) { + grant->seal(); + } +} + +Status bind_block_account(PageStore *store, const std::shared_ptr<ResourceBudget::Account> &account, + size_t backend_workers, BackendOpenBudget *out) +{ + if (store == nullptr || !account || backend_workers == 0 || out == nullptr) { + return Status::invalid_argument("backend account binding requires an owned startup domain"); + } + *out = {}; + auto *block = dynamic_cast<BlockPageStore *>(store); + return block != nullptr ? BlockAccountAccess::bind(*block, account, backend_workers, out) : Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/local/block_account.h b/lib/crowdb-tree/src/backend/local/block_account.h new file mode 100644 index 000000000..c8cef7947 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_account.h @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/resource_budget.h" + +namespace crowdb::tree +{ +class PageStore; +class BlockPageStore; +class AsyncPageStore; + +namespace detail +{ +struct BackendOperationLimits +{ + size_t max_io_bytes = 0; + size_t write_calls = 0; + size_t workers = 1; + bool barriers = false; + bool asynchronous = false; +}; + +std::optional<size_t> backend_operation_bound(PageStore *backend, BackendOperationLimits limits) noexcept; +Status reserve_backend_operation(PageStore *backend, const std::shared_ptr<ResourceBudget::Account> &account, + BackendOperationLimits limits, std::shared_ptr<AllocationBudget> *out); +std::optional<size_t> backend_operation_bound(AsyncPageStore *backend, BackendOperationLimits limits) noexcept; +Status reserve_backend_operation(AsyncPageStore *backend, const std::shared_ptr<ResourceBudget::Account> &account, + BackendOperationLimits limits, std::shared_ptr<AllocationBudget> *out); +// Consume an already admitted complete operation bound. No second resource +// acquisition occurs between the asynchronous grant and backend submission. +Status bind_backend_operation(PageStore *backend, BackendOperationLimits limits, ResourceBudget::Credit credit, + std::shared_ptr<AllocationBudget> *out); +Status bind_backend_operation(AsyncPageStore *backend, BackendOperationLimits limits, ResourceBudget::Credit credit, + std::shared_ptr<AllocationBudget> *out); +uint64_t backend_size(PageStore &store, const std::shared_ptr<AllocationBudget> &budget); + +void seal_backend_budget(const std::shared_ptr<AllocationBudget> &budget) noexcept; + +struct BackendOpenBudget +{ + size_t owned_bytes = 0; + size_t growth_bytes = 0; +}; + +std::shared_ptr<AllocationBudget> block_allocation_budget(const BlockPageStore &store, + const std::shared_ptr<AllocationBudget> &budget = {}); + +bool valid_block_budget(const BlockPageStore &store, const std::shared_ptr<AllocationBudget> &budget) noexcept; + +// Requires the PageStore runtime-domain lease and no backend growth in flight. +// Existing fd owners may free concurrently. Non-Block stores own their external +// implementation accounting; this adapter only binds the local Block backend. +Status bind_block_account(PageStore *store, const std::shared_ptr<ResourceBudget::Account> &account, + size_t backend_workers, BackendOpenBudget *out); +} // namespace detail +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/block_async_io.cpp b/lib/crowdb-tree/src/backend/local/block_async_io.cpp new file mode 100644 index 000000000..955df5789 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_async_io.cpp @@ -0,0 +1,233 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "block_async_io.h" + +#include "backend_io_allocation.h" +#include "block_account.h" +#include "btree/allocation_bound.h" +#include "btree/tree_work_counters.h" + +#ifdef CROWDB_HAVE_LIBURING +# include <algorithm> +# include <atomic> +# include <climits> +# include <cstdlib> +# include <cstring> + +namespace crowdb::tree::detail +{ +namespace +{ +struct IoJoin +{ + explicit IoJoin(AsyncCompletion callback) : complete(std::move(callback)) + { + } + + AsyncCompletion complete; + // The submission loop owns one count so no inline completion can finish + // the operation while more chunks may still be submitted. + std::atomic<size_t> pending{1}; + std::atomic_flag failed = ATOMIC_FLAG_INIT; + std::atomic<bool> missing{false}; + Status first_error; + Status first_missing; + + void done(Status status) + { + if (status.code() == Code::kNotFound) { + if (!missing.exchange(true, std::memory_order_relaxed)) { + first_missing = std::move(status); + } + } + else if (!status.ok() && !failed.test_and_set(std::memory_order_relaxed)) { + first_error = std::move(status); + } + // The winning error is written before its count is released; the + // last completion acquires that write through this release sequence. + if (pending.fetch_sub(1, std::memory_order_acq_rel) == 1) { + // A short part must not hide a transport error in another part. + if (first_error.ok() && missing.load(std::memory_order_relaxed)) { + first_error = std::move(first_missing); + } + complete.complete(std::move(first_error)); + } + } +}; + +struct IoPart +{ + std::shared_ptr<AllocationBudget> budget; + std::shared_ptr<IoJoin> join; + std::shared_ptr<FileMedium> medium; + void *read_buffer = nullptr; + void *aligned = nullptr; + size_t length = 0; + + ~IoPart() + { + if (aligned != nullptr) { + ::free(aligned); + budget->record_allocation(AllocationKind::kIo, length, false); + budget->release(length); + } + } + + void prepare(const void *write_buffer, uint32_t iu) + { + const void *buffer = read_buffer != nullptr ? read_buffer : write_buffer; + if (iu <= 1 || reinterpret_cast<uintptr_t>(buffer) % 4096 == 0) { + return; + } + if (!budget->claim(length)) { + throw std::bad_alloc(); + } + if (::posix_memalign(&aligned, 4096, length) != 0) { + budget->release(length); + throw std::bad_alloc(); + } + budget->record_allocation(AllocationKind::kIo, length, true); + if (read_buffer == nullptr) { + std::memcpy(aligned, write_buffer, length); + record_work(budget, TreeWorkCounter::BackendBounceCopyBytes, length, join->complete.cause); + } + } + + static void completed(void *context, int result) noexcept + { + auto &part = *static_cast<IoPart *>(context); + if (part.read_buffer == nullptr) { + part.medium->mark_dirty(); + } + const bool success = std::cmp_equal(result, part.length); + if (success && part.read_buffer != nullptr && part.aligned != nullptr) { + std::memcpy(part.read_buffer, part.aligned, part.length); + record_work(part.budget, TreeWorkCounter::BackendBounceCopyBytes, part.length, part.join->complete.cause); + } + auto status = success ? Status::Ok() : Status::io_error(); + if (result == -ENOMEM) { + status = Status::resource_exhausted(); + } + if (part.read_buffer != nullptr && result >= 0 && std::cmp_less(result, part.length)) { + try { + status = Status::not_found("BlockAsyncPageStore: short read (" + std::to_string(result) + " of " + + std::to_string(part.length) + " bytes)"); + } + catch (const std::bad_alloc &) { + // Completion must still release accepted I/O when diagnostics cannot allocate. + status = Status::not_found(); + } + } + part.join->done(std::move(status)); + } +}; + +Status submit_part(::crowdb::common::DiskIOUring &uring, const std::shared_ptr<IoJoin> &join, + std::shared_ptr<FileMedium> medium, uint64_t offset, void *read_buffer, const void *write_buffer, + size_t length, uint32_t iu) +{ + const auto &budget = join->complete.allocations; + auto part = std::allocate_shared<IoPart>(MetadataAllocator<IoPart>(budget)); + part->budget = budget; + part->join = join; + part->medium = std::move(medium); + part->read_buffer = read_buffer; + part->length = length; + part->prepare(write_buffer, iu); + ::crowdb::common::IOCompletion callback{ + .context = part.get(), .callback = &IoPart::completed, .owner = part, .allocation = io_allocation(budget)}; + join->pending.fetch_add(1, std::memory_order_relaxed); + if (read_buffer != nullptr) { + uring.submit_read_owned(part->medium->fd(), part->aligned != nullptr ? part->aligned : read_buffer, length, + static_cast<off_t>(offset), std::move(callback)); + } + else { + uring.submit_write_owned(part->medium->fd(), part->aligned != nullptr ? part->aligned : write_buffer, length, + static_cast<off_t>(offset), std::move(callback)); + } + return Status::Ok(); +} + +Status submit_parts(BlockPageStore &store, ::crowdb::common::DiskIOUring &uring, PageAddr address, void *read_buffer, + const void *write_buffer, size_t length, const std::shared_ptr<IoJoin> &join) +{ + const auto iu = std::max(store.iu_size(), uint32_t{1}); + const auto block_size = store.block_size(); + const auto max_chunk = (static_cast<uint64_t>(INT_MAX) / iu) * iu; + size_t done = 0; + while (done < length) { + uint64_t offset = 0; + auto medium = store.file_for_offset(address + done, &offset, join->complete.allocations); + if (!medium) { + return read_buffer != nullptr && block_size != 0 + ? Status::not_found("BlockAsyncPageStore: no fd for offset") + : Status::io_error("BlockAsyncPageStore: no fd for offset"); + } + const auto available = block_size == 0 ? max_chunk : block_size - offset; + const size_t chunk = std::min({static_cast<uint64_t>(length - done), available, max_chunk}); + if (chunk == 0 || offset % iu != 0 || chunk % iu != 0 || offset > static_cast<uint64_t>(INT64_MAX)) { + return Status::invalid_argument(); + } + auto status = submit_part(uring, join, std::move(medium), offset, + read_buffer != nullptr ? static_cast<uint8_t *>(read_buffer) + done : nullptr, + write_buffer != nullptr ? static_cast<const uint8_t *>(write_buffer) + done : nullptr, + chunk, iu); + if (!status.ok()) { + return status; + } + done += chunk; + } + return Status::Ok(); +} +} // namespace + +std::optional<size_t> block_io_allocation_bound(size_t bytes, size_t parts) noexcept +{ + return AllocationBound(shared_allocation_bound<IoJoin>()) + .add(bytes) + .add_product(parts, shared_allocation_bound<IoPart>() + + ::crowdb::common::DiskIOUring::owned_submission_allocation_bytes()) + .value(); +} + +void submit_block_io(BlockPageStore &store, ::crowdb::common::DiskIOUring &uring, PageAddr address, void *read_buffer, + const void *write_buffer, size_t length, AsyncCompletion complete) +{ + const auto iu = std::max(store.iu_size(), uint32_t{1}); + if (length == 0) { + complete.complete(Status::Ok()); + return; + } + if ((read_buffer == nullptr) == (write_buffer == nullptr) || address > UINT64_MAX - length || address % iu != 0 || + length % iu != 0) { + complete.complete(Status::invalid_argument()); + return; + } + std::shared_ptr<IoJoin> join; + Status status; + try { + complete.allocations = block_allocation_budget(store, complete.allocations); + if (!complete.allocations) { + complete.complete(Status::invalid_argument("backend allocation budget belongs to another store")); + return; + } + join = std::allocate_shared<IoJoin>(MetadataAllocator<IoJoin>(complete.allocations), complete); + if (write_buffer != nullptr) { + status = store.ensure_extents(address, length, complete.allocations); + } + if (status.ok()) { + status = submit_parts(store, uring, address, read_buffer, write_buffer, length, join); + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted(); + } + if (join) { + join->done(std::move(status)); + } + else { + complete.complete(std::move(status)); + } +} +} // namespace crowdb::tree::detail +#endif diff --git a/lib/crowdb-tree/src/backend/local/block_async_io.h b/lib/crowdb-tree/src/backend/local/block_async_io.h new file mode 100644 index 000000000..e1500d493 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_async_io.h @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/backend/block_page_store.h" + +#include <optional> + +#ifdef CROWDB_HAVE_LIBURING +namespace crowdb::tree::detail +{ +std::optional<size_t> block_io_allocation_bound(size_t bytes, size_t parts) noexcept; +std::optional<size_t> block_sync_allocation_bound(size_t extents) noexcept; + +// Splits aligned byte ranges across extents and retains the completion owner +// until every submitted part has completed, including inline failures. +void submit_block_io(BlockPageStore &store, ::crowdb::common::DiskIOUring &uring, PageAddr address, void *read_buffer, + const void *write_buffer, size_t length, AsyncCompletion complete); +} // namespace crowdb::tree::detail +#endif diff --git a/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp b/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp index 502bd7de5..053ff5667 100644 --- a/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp +++ b/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp @@ -3,117 +3,17 @@ // Local asynchronous block-backend implementation. -#include "crowdb-common/diskio_uring.h" -#include "crowdb-tree/backend/async_page_store.h" -#include "crowdb-tree/backend/block_page_store.h" +#include "backend_io_allocation.h" +#include "block_account.h" +#include "block_async_io.h" +#include "btree/allocation_bound.h" -#include <fcntl.h> -#include <unistd.h> - -#include <algorithm> -#include <cerrno> -#include <cstdlib> -#include <cstring> -#include <limits> -#include <memory> #include <utility> #include <vector> namespace crowdb::tree { -namespace -{ -// Maps a DiskIOUring callback's raw CQE `res` (>=0 bytes transferred, <0 -// -errno) to a Status, treating a short read/write (res >= 0 but < the -// requested `len`) as an io_error too. Each extent is submitted once; -// crossing a block boundary does not retry a short read in the first file. -Status result_to_status(int res, size_t len, const char *op) -{ - if (res < 0) { - return Status::io_error(std::string(op) + ": " + std::strerror(-res)); - } - if (std::cmp_less(res, len)) { - return Status::io_error(std::string("short ") + op + " (" + std::to_string(res) + " of " + std::to_string(len) + - " bytes)"); - } - return Status::Ok(); -} - -struct ExtentRead -{ - int fd; - void *buffer; - size_t length; - uint64_t offset; -}; - -struct ExtentReads : std::enable_shared_from_this<ExtentReads> -{ - ::crowdb::common::DiskIOUring *uring = nullptr; - std::vector<ExtentRead> parts; - AsyncCompletion completion; - - void submit(size_t index) - { - const auto &part = parts[index]; - uring->submit_read(part.fd, part.buffer, part.length, static_cast<off_t>(part.offset), - [state = shared_from_this(), index, length = part.length](int result) { - Status status = result_to_status(result, length, "read"); - if (!status.ok() || index + 1 == state->parts.size()) { - if (state->completion) { - state->completion.complete(std::move(status)); - } - return; - } - state->submit(index + 1); - }); - } -}; - -// RAII aligned buffer for O_DIRECT writes via io_uring. std::vector memory -// is not guaranteed IU-aligned; this wraps posix_memalign + memcpy. -class AlignedIoBuf -{ - public: - AlignedIoBuf(size_t len, size_t align) : len_(len) - { - if (::posix_memalign(&ptr_, align, len) != 0) { - ptr_ = nullptr; - } - } - - ~AlignedIoBuf() - { - if (ptr_ != nullptr) { - ::free(ptr_); - } - } - - AlignedIoBuf(const AlignedIoBuf &) = delete; - AlignedIoBuf &operator=(const AlignedIoBuf &) = delete; - - [[nodiscard]] bool ok() const - { - return ptr_ != nullptr; - } - - [[nodiscard]] void *data() const - { - return ptr_; - } - - [[nodiscard]] size_t size() const - { - return len_; - } - - private: - void *ptr_ = nullptr; - size_t len_ = 0; -}; -} // namespace - // ── BlockAsyncPageStore ─────────────────────────────────────────── BlockAsyncPageStore::BlockAsyncPageStore(BlockPageStore *store, ::crowdb::common::DiskIOUring *uring) @@ -124,226 +24,97 @@ BlockAsyncPageStore::BlockAsyncPageStore(BlockPageStore *store, ::crowdb::common uint64_t BlockAsyncPageStore::submit_read(PageAddr addr, void *buf, size_t len, AsyncCompletion on_complete) { - if (len == 0 || len > std::numeric_limits<uint64_t>::max() - addr) { - if (on_complete) { - on_complete.complete(len == 0 ? Status::Ok() : Status::invalid_argument("read range overflow")); - } - return 0; - } - const uint64_t block_size = store_->block_size(); - if (block_size > 0 && len > block_size - (addr % block_size)) { - auto state = std::make_shared<ExtentReads>(); - state->uring = uring_; - state->completion = on_complete; - size_t done = 0; - while (done < len) { - uint64_t local = 0; - int fd = store_->fd_for_offset(addr + done, &local); - if (fd < 0) { - if (on_complete) { - on_complete.complete(Status::io_error("BlockAsyncPageStore: no fd for offset")); - } - return 0; - } - size_t chunk = std::min(static_cast<uint64_t>(len - done), block_size - local); - state->parts.push_back( - {.fd = fd, .buffer = static_cast<uint8_t *>(buf) + done, .length = chunk, .offset = local}); - done += chunk; - } - state->submit(0); - return 0; - } - uint64_t local = 0; - int fd = store_->fd_for_offset(addr, &local); - if (fd < 0) { - if (on_complete) { - on_complete.complete(Status::io_error("BlockAsyncPageStore: no fd for offset")); - } - return 0; - } - uring_->submit_read(fd, buf, len, static_cast<off_t>(local), [len, cb = on_complete](int res) { - if (cb) { - cb.complete(result_to_status(res, len, "read")); - } - }); + detail::submit_block_io(*store_, *uring_, addr, buf, nullptr, len, std::move(on_complete)); return 0; } uint64_t BlockAsyncPageStore::submit_write(PageAddr addr, const void *buf, size_t len, AsyncCompletion on_complete) { - // Ensure block files exist for this address range (mirrors sync - // write_at_extents' allocation loop). - Status es = store_->ensure_extents(addr, len); - if (!es.ok()) { - if (on_complete) { - on_complete.complete(es); - } - return 0; - } - - // O_DIRECT requires the buffer pointer to be IU/4096-aligned. The - // caller's buffer (e.g. a std::vector<uint8_t> blob from persist.cpp) - // is not guaranteed aligned, so when iu_size > 1 we copy into an - // AlignedIoBuf kept alive via shared_ptr until the io_uring completion - // callback fires. - const uint32_t iu = store_->iu_size(); - auto maybe_align = [iu](const void *p, size_t n) -> std::shared_ptr<AlignedIoBuf> { - if (iu <= 1 || (reinterpret_cast<uintptr_t>(p) % 4096 == 0)) { - return nullptr; // already aligned or no alignment requirement - } - auto ab = std::make_shared<AlignedIoBuf>(n, 4096); - if (!ab->ok()) { - return nullptr; // alloc failure — caller will get io_error - } - std::memcpy(ab->data(), p, n); - return ab; - }; - - uint64_t block_size = store_->block_size(); - if (block_size > 0) { - uint64_t first_extent = addr / block_size; - uint64_t last_extent = (addr + len - 1) / block_size; - if (last_extent != first_extent) { - // Cross-extent write: split into per-extent submissions. All - // completions share a WriteState; the last one invokes on_complete. - struct WriteState - { - AsyncCompletion cb; - Status first_error; - int pending; - std::vector<std::shared_ptr<AlignedIoBuf>> align_bufs; // keep alive until all CQEs fire - }; - - auto state = std::make_shared<WriteState>(); - state->cb = on_complete; - state->pending = 0; + detail::submit_block_io(*store_, *uring_, addr, nullptr, buf, len, std::move(on_complete)); + return 0; +} - uint64_t cur = addr; - size_t done = 0; - while (done < len) { - uint64_t local = 0; - int fd = store_->fd_for_offset(cur, &local); - if (fd < 0) { - if (state->cb) { - state->cb.complete(Status::io_error("BlockAsyncPageStore: no fd for offset")); - state->cb = {}; - } - return 0; - } - uint64_t avail = block_size - local; - size_t chunk = std::min(static_cast<uint64_t>(len - done), avail); +namespace +{ +struct FsyncState : std::enable_shared_from_this<FsyncState> +{ + explicit FsyncState(const std::shared_ptr<AllocationBudget> &budget) + : targets(MetadataAllocator<BlockSyncTarget>(budget)) + { + } - const void *write_buf = static_cast<const uint8_t *>(buf) + done; - auto ab = maybe_align(write_buf, chunk); - if (iu > 1 && ab == nullptr && reinterpret_cast<uintptr_t>(write_buf) % 4096 != 0) { - if (state->cb) { - state->cb.complete(Status::io_error("BlockAsyncPageStore: aligned buffer alloc failed")); - state->cb = {}; - } - return 0; - } - if (ab != nullptr) { - write_buf = ab->data(); - state->align_bufs.push_back(ab); - } + BlockPageStore::BlockSyncTargets targets; + size_t idx = 0; + AsyncCompletion callback; + ::crowdb::common::DiskIOUring *uring = nullptr; - state->pending++; - uring_->submit_write(fd, write_buf, chunk, static_cast<off_t>(local), [state, chunk](int res) { - if (res < 0 || std::cmp_less(res, chunk)) { - Status s = result_to_status(res, chunk, "write"); - if (state->first_error.ok()) { - state->first_error = std::move(s); - } - } - state->pending--; - if (state->pending == 0 && state->cb) { - state->cb.complete(state->first_error); - } - }); - cur += chunk; - done += chunk; - } - return 0; - } + static void completed(void *context, int result) noexcept + { + static_cast<FsyncState *>(context)->next(result); } - // Single-extent write - uint64_t local = 0; - int fd = store_->fd_for_offset(addr, &local); - if (fd < 0) { - if (on_complete) { - on_complete.complete(Status::io_error("BlockAsyncPageStore: no fd for offset")); + void next(int result) noexcept + { + if (result >= 0 && idx != 0) { + const auto &target = targets[idx - 1]; + target.medium->acknowledge_sync(target.generation); } - return 0; - } - const void *write_buf = buf; - auto ab = maybe_align(buf, len); - if (iu > 1 && ab == nullptr && reinterpret_cast<uintptr_t>(buf) % 4096 != 0) { - if (on_complete) { - on_complete.complete(Status::io_error("BlockAsyncPageStore: aligned buffer alloc failed")); + if (result < 0 || idx == targets.size()) { + auto status = result < 0 ? Status::io_error() : Status::Ok(); + if (result == -ENOMEM) { + status = Status::resource_exhausted(); + } + callback.complete(std::move(status)); + return; } - return 0; - } - if (ab != nullptr) { - write_buf = ab->data(); - } - // Capture ab in the callback lambda to keep the aligned buffer alive - // until the io_uring completion fires. - uring_->submit_write(fd, write_buf, len, static_cast<off_t>(local), [len, cb = on_complete, ab](int res) { - if (cb) { - cb.complete(result_to_status(res, len, "write")); + auto self = weak_from_this().lock(); + if (!self) { + callback.complete(Status::internal_error("fsync continuation lost its owner")); + return; } - }); - return 0; + const int fd = targets[idx++].medium->fd(); + uring->submit_fsync_owned(fd, false, + {.context = this, + .callback = &completed, + .owner = std::move(self), + .allocation = detail::io_allocation(callback.allocations)}); + } +}; +} // namespace + +namespace detail +{ +std::optional<size_t> block_sync_allocation_bound(size_t extents) noexcept +{ + return AllocationBound(shared_allocation_bound<FsyncState>()) + .add_product(extents, sizeof(BlockSyncTarget)) + .add_product(2, ::crowdb::common::DiskIOUring::owned_submission_allocation_bytes()) + .value(); } +} // namespace detail Status BlockAsyncPageStore::submit_fsync(AsyncCompletion on_complete) -{ - std::vector<int> fds = store_->dirty_fds(); - if (fds.empty()) { - if (on_complete) { - on_complete.complete(Status::Ok()); - } +try { + auto budget = detail::block_allocation_budget(*store_, on_complete.allocations); + if (!budget) { + return Status::invalid_argument("backend allocation budget belongs to another store"); + } + auto targets = store_->sync_targets(budget); + if (targets.empty()) { + on_complete.complete(Status::Ok()); return Status::Ok(); } - - // Chain fsync across all dirty fds: each completion submits the next; - // the last one invokes on_complete. A shared_ptr keeps the state alive - // across the async chain (completions run on the poll thread). - struct FsyncState - { - std::vector<int> fds; - size_t idx = 0; - AsyncCompletion cb; - }; - - auto state = std::make_shared<FsyncState>(); - state->fds = std::move(fds); - state->cb = on_complete; - - auto chain = std::make_shared<std::function<void()>>(); - *chain = [this, state, chain] { - if (state->idx >= state->fds.size()) { - if (state->cb) { - state->cb.complete(Status::Ok()); - } - return; - } - int fd = state->fds[state->idx++]; - uring_->submit_fsync(fd, [this, state, chain](int res) { - if (res < 0) { - if (state->cb) { - state->cb.complete(Status::io_error(std::string("fsync: ") + std::strerror(-res))); - state->cb = {}; - } - return; - } - (*chain)(); - }); - }; - (*chain)(); + auto state = std::allocate_shared<FsyncState>(MetadataAllocator<FsyncState>(budget), budget); + state->targets = std::move(targets); + state->callback = std::move(on_complete); + state->callback.allocations = std::move(budget); + state->uring = uring_; + state->next(0); return Status::Ok(); } +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} void BlockAsyncPageStore::cancel(uint64_t /*op_id*/) { diff --git a/lib/crowdb-tree/src/backend/local/block_directory.cpp b/lib/crowdb-tree/src/backend/local/block_directory.cpp new file mode 100644 index 000000000..db1a29fa4 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_directory.cpp @@ -0,0 +1,293 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "block_directory.h" + +#include "btree/allocation_bound.h" + +#include <unistd.h> + +#include <cerrno> +#include <cstring> + +namespace crowdb::tree +{ +namespace +{ +template <class T> T *create_metadata(const std::shared_ptr<AllocationBudget> &budget) +{ + MetadataAllocator<T> allocator(budget); + auto *result = allocator.allocate(1); + std::construct_at(result, budget); + return result; +} + +template <class T> void destroy_metadata(T *value) noexcept +{ + if (value == nullptr) { + return; + } + MetadataAllocator<T> allocator(value->budget); + std::destroy_at(value); + allocator.deallocate(value, 1); +} +} // namespace + +BlockDirectory::Entry::Entry(std::shared_ptr<AllocationBudget> owner) : budget(std::move(owner)) +{ + destroy = [](EpochManager::Deferred *entry) noexcept { destroy_metadata(static_cast<Entry *>(entry)); }; +} + +BlockDirectory::BlockDirectory(std::shared_ptr<AllocationBudget> budget, std::shared_ptr<BackendReleaseService> release) + : budget_(std::move(budget)), + epoch_(budget_) +{ + if (release) { + retirement_.directory = this; + retirement_.dispose = [](BackendReleaseService::Node *base) noexcept { + auto *directory = static_cast<Retirement *>(base)->directory; + if (!directory->retire_step()) { + return false; + } + MetadataAllocator<BlockDirectory> allocator(directory->budget_); + std::destroy_at(directory); + allocator.deallocate(directory, 1); + return true; + }; + if (!release->register_node(retirement_).ok()) { + throw std::bad_alloc(); + } + } +} + +void BlockDirectoryDeleter::operator()(BlockDirectory *directory) const noexcept +{ + if (directory != nullptr) { + directory->release_owned(); + } +} + +void BlockDirectory::release_owned() noexcept +{ + if (retirement_.service) { + BackendReleaseService::enqueue(retirement_); + } + else { + MetadataAllocator<BlockDirectory> allocator(budget_); + std::destroy_at(this); + allocator.deallocate(this, 1); + } +} + +BlockDirectory::~BlockDirectory() +{ + if (!retiring_) { + (void)epoch_.try_reclaim(); + destroy_node(root_, 24); + } +} + +void BlockDirectory::destroy_node(Node &node, unsigned shift) +{ + for (auto &slot : node.children) { + void *child = slot.load(std::memory_order_relaxed); + if (child == nullptr) { + continue; + } + if (shift == 8) { + auto *segment = static_cast<Segment *>(child); + for (auto &entry : segment->slots) { + destroy_metadata(entry.load(std::memory_order_relaxed)); + } + destroy_metadata(segment); + } + else { + auto *directory = static_cast<Node *>(child); + destroy_node(*directory, shift - 8); + destroy_metadata(directory); + } + } +} + +BlockDirectory::Segment *BlockDirectory::segment(uint32_t index, bool create, + const std::shared_ptr<AllocationBudget> &budget) +{ + auto *node = &root_; + for (unsigned shift = 24; shift >= 8; shift -= 8) { + auto &slot = node->children[(index >> shift) & 255]; + auto *child = slot.load(std::memory_order_acquire); + if (child == nullptr && create) { + auto *candidate = shift == 8 ? static_cast<void *>(create_metadata<Segment>(budget)) + : static_cast<void *>(create_metadata<Node>(budget)); + if (!slot.compare_exchange_strong(child, candidate, std::memory_order_release, std::memory_order_acquire)) { + if (shift == 8) { + destroy_metadata(static_cast<Segment *>(candidate)); + } + else { + destroy_metadata(static_cast<Node *>(candidate)); + } + } + else { + child = candidate; + } + } + if (child == nullptr || shift == 8) { + return static_cast<Segment *>(child); + } + node = static_cast<Node *>(child); + } + return nullptr; +} + +const BlockDirectory::Segment *BlockDirectory::segment(uint32_t index) const +{ + const auto *node = &root_; + for (unsigned shift = 24; shift >= 8; shift -= 8) { + auto *child = node->children[(index >> shift) & 255].load(std::memory_order_acquire); + if (child == nullptr || shift == 8) { + return static_cast<const Segment *>(child); + } + node = static_cast<const Node *>(child); + } + return nullptr; +} + +std::shared_ptr<FileMedium> BlockDirectory::get(uint32_t index, const std::shared_ptr<AllocationBudget> &budget) const +{ + auto guard = epoch_.enter(budget ? budget : budget_); + const auto *leaf = segment(index); + const auto *entry = leaf != nullptr ? leaf->slots[index & 255].load(std::memory_order_acquire) : nullptr; + return entry != nullptr && entry != &busy_ ? entry->medium : std::shared_ptr<FileMedium>{}; +} + +Status BlockDirectory::ensure(uint32_t index, const char *path, bool direct, std::shared_ptr<FileMedium> *out, + const std::shared_ptr<AllocationBudget> &budget) +try { + auto guard = epoch_.enter(budget ? budget : budget_); + auto &slot = segment(index, true, budget ? budget : budget_)->slots[index & 255]; + auto *entry = slot.load(std::memory_order_acquire); + if (entry != nullptr && entry != &busy_) { + *out = entry->medium; + return Status::Ok(); + } + if (entry == &busy_ || !slot.compare_exchange_strong(entry, &busy_, std::memory_order_acq_rel)) { + return Status::unavailable("block metadata operation already in progress"); + } + + struct Restore + { + std::atomic<Entry *> &slot; + bool published = false; + + ~Restore() + { + if (!published) { + slot.store(nullptr, std::memory_order_release); + } + } + } restore{.slot = slot}; + + std::unique_ptr<Entry, void (*)(Entry *)> prepared(create_metadata<Entry>(budget ? budget : budget_), + &destroy_metadata<Entry>); + std::shared_ptr<FileMedium> file; + auto status = FileMedium::open_owned(path, direct, budget ? budget : budget_, &file, retirement_.service); + if (!status.ok()) { + return status; + } + prepared->medium = std::move(file); + *out = prepared->medium; + live_.fetch_add(1, std::memory_order_relaxed); + slot.store(prepared.release(), std::memory_order_release); + restore.published = true; + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status BlockDirectory::erase(uint32_t index, const char *path, const std::shared_ptr<AllocationBudget> &budget) +{ + { + auto guard = epoch_.enter(budget ? budget : budget_); + auto *leaf = segment(index, false, budget ? budget : budget_); + if (leaf == nullptr) { + return Status::Ok(); + } + auto &slot = leaf->slots[index & 255]; + auto *entry = slot.load(std::memory_order_acquire); + if (entry == nullptr) { + return Status::Ok(); + } + if (entry == &busy_ || !slot.compare_exchange_strong(entry, &busy_, std::memory_order_acq_rel)) { + return Status::unavailable("block metadata operation already in progress"); + } + if (::unlink(path) < 0 && errno != ENOENT) { + slot.store(entry, std::memory_order_release); + return Status::io_error(std::string("unlink: ") + std::strerror(errno)); + } + live_.fetch_sub(1, std::memory_order_relaxed); + slot.store(nullptr, std::memory_order_release); + epoch_.defer(entry); + } + (void)epoch_.try_reclaim(); + return Status::Ok(); +} + +bool BlockDirectory::next_node(const Node &node, unsigned shift, uint64_t prefix, uint64_t first, uint32_t *index, + std::shared_ptr<FileMedium> *out) const +{ + for (size_t i = 0; i < 256; ++i) { + const uint64_t begin = prefix | (static_cast<uint64_t>(i) << shift); + if (begin + (uint64_t{1} << shift) <= first) { + continue; + } + const auto *child = node.children[i].load(std::memory_order_acquire); + if (child == nullptr) { + continue; + } + if (shift != 8) { + if (next_node(*static_cast<const Node *>(child), shift - 8, begin, first, index, out)) { + return true; + } + continue; + } + const auto &leaf = *static_cast<const Segment *>(child); + for (auto j = static_cast<size_t>(std::max(first, begin) - begin); j < 256; ++j) { + const auto *entry = leaf.slots[j].load(std::memory_order_acquire); + if (entry != nullptr && entry != &busy_) { + *index = static_cast<uint32_t>(begin + j); + *out = entry->medium; + return true; + } + } + } + return false; +} + +bool BlockDirectory::next(uint64_t *cursor, uint32_t *index, std::shared_ptr<FileMedium> *out, + const std::shared_ptr<AllocationBudget> &budget) const +{ + if (*cursor > UINT32_MAX) { + return false; + } + auto guard = epoch_.enter(budget ? budget : budget_); + if (!next_node(root_, 24, 0, *cursor, index, out)) { + return false; + } + *cursor = static_cast<uint64_t>(*index) + 1; + return true; +} + +size_t BlockDirectory::extent_allocation_bound() noexcept +{ + // A previously absent sparse path owns two nodes, one leaf segment, one + // incarnation and its file object/control. File object header/alignment + // and the supported standard-library control fit in the shared allowance. + return (2 * sizeof(Node)) + sizeof(Segment) + sizeof(Entry) + detail::shared_allocation_bound<FileMedium>() + + FileMedium::close_allocation_bound() + 128; +} + +size_t BlockDirectory::count() const noexcept +{ + return live_.load(std::memory_order_relaxed); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/block_directory.h b/lib/crowdb-tree/src/backend/local/block_directory.h new file mode 100644 index 000000000..28ebfd8fd --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_directory.h @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/backend/backend_release_service.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "crowdb-tree/epoch.h" + +#include <array> + +namespace crowdb::tree +{ +// Sparse stable radix nodes never move while this backend is open. A reader +// copies a file owner under a short epoch guard and releases the guard before +// doing I/O. Only removed file incarnations enter epoch reclamation. +class BlockDirectory +{ + struct Entry : EpochManager::Deferred + { + explicit Entry(std::shared_ptr<AllocationBudget> owner = {}); + std::shared_ptr<AllocationBudget> budget; + std::shared_ptr<FileMedium> medium; + }; + + struct Segment + { + explicit Segment(std::shared_ptr<AllocationBudget> owner) : budget(std::move(owner)) + { + } + + std::shared_ptr<AllocationBudget> budget; + std::array<std::atomic<Entry *>, 256> slots{}; + }; + + struct Node + { + explicit Node(std::shared_ptr<AllocationBudget> owner = {}) : budget(std::move(owner)) + { + } + + std::shared_ptr<AllocationBudget> budget; + std::array<std::atomic<void *>, 256> children{}; + }; + + public: + explicit BlockDirectory(std::shared_ptr<AllocationBudget> budget = {}, + std::shared_ptr<BackendReleaseService> release = {}); + ~BlockDirectory(); + void release_owned() noexcept; + [[nodiscard]] std::shared_ptr<FileMedium> get(uint32_t index, + const std::shared_ptr<AllocationBudget> &budget = {}) const; + Status ensure(uint32_t index, const char *path, bool direct, std::shared_ptr<FileMedium> *out, + const std::shared_ptr<AllocationBudget> &budget = {}); + + Status ensure(uint32_t index, const std::string &path, bool direct, std::shared_ptr<FileMedium> *out) + { + return ensure(index, path.c_str(), direct, out); + } + + // The caller must exclusively own the entire byte extent before unlinking. + // Conflicting metadata operations return Unavailable rather than waiting. + Status erase(uint32_t index, const char *path, const std::shared_ptr<AllocationBudget> &budget = {}); + + Status erase(uint32_t index, const std::string &path) + { + return erase(index, path.c_str()); + } + + // Advance through populated radix branches only; no vector or dense gap scan. + bool next(uint64_t *cursor, uint32_t *index, std::shared_ptr<FileMedium> *out, + const std::shared_ptr<AllocationBudget> &budget = {}) const; + [[nodiscard]] size_t count() const noexcept; + [[nodiscard]] static size_t extent_allocation_bound() noexcept; + + private: + Segment *segment(uint32_t index, bool create, const std::shared_ptr<AllocationBudget> &budget); + const Segment *segment(uint32_t index) const; + bool next_node(const Node &node, unsigned shift, uint64_t prefix, uint64_t first, uint32_t *index, + std::shared_ptr<FileMedium> *out) const; + static void destroy_node(Node &node, unsigned shift); + bool retire_step() noexcept; + + struct Retirement : BackendReleaseService::Node + { + BlockDirectory *directory = nullptr; + }; + + Retirement retirement_; + std::array<void *, 4> retire_path_{}; + std::array<uint16_t, 4> retire_index_{}; + uint8_t retire_depth_ = 0; + bool retiring_ = false; + bool entries_drained_ = false; + + std::shared_ptr<AllocationBudget> budget_; + Node root_; + Entry busy_; + std::atomic<size_t> live_{0}; + mutable EpochManager epoch_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/block_directory_release.cpp b/lib/crowdb-tree/src/backend/local/block_directory_release.cpp new file mode 100644 index 000000000..f31ec1b7d --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_directory_release.cpp @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "block_directory.h" + +namespace crowdb::tree +{ +namespace +{ +template <class T> void destroy_released(T *value) noexcept +{ + if (value != nullptr) { + MetadataAllocator<T> allocator(value->budget); + std::destroy_at(value); + allocator.deallocate(value, 1); + } +} +} // namespace + +bool BlockDirectory::retire_step() noexcept +{ + if (!retiring_) { + retiring_ = true; + retire_path_[0] = &root_; + } + if (entries_drained_) { + return epoch_.retire_backend_step(); + } + + for (size_t work = 0; work < 64; ++work) { + auto &position = retire_index_[retire_depth_]; + if (retire_depth_ == 3) { + auto *leaf = static_cast<Segment *>(retire_path_[3]); + if (position < leaf->slots.size()) { + auto *entry = leaf->slots[position++].exchange(nullptr, std::memory_order_relaxed); + if (entry != &busy_) { + destroy_released(entry); + } + } + else { + destroy_released(leaf); + --retire_depth_; + } + continue; + } + auto *node = static_cast<Node *>(retire_path_[retire_depth_]); + if (position < node->children.size()) { + auto *child = node->children[position++].exchange(nullptr, std::memory_order_relaxed); + if (child != nullptr) { + ++retire_depth_; + retire_path_[retire_depth_] = child; + retire_index_[retire_depth_] = 0; + } + } + else if (retire_depth_ != 0) { + destroy_released(node); + --retire_depth_; + } + else { + entries_drained_ = true; + return false; + } + } + return false; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/block_file_close.cpp b/lib/crowdb-tree/src/backend/local/block_file_close.cpp new file mode 100644 index 000000000..021838484 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_file_close.cpp @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/backend/backend_release_service.h" +#include "crowdb-tree/backend/block_page_store.h" + +#include <unistd.h> + +namespace crowdb::tree +{ +struct FileMedium::CloseNode : BackendReleaseService::Node +{ + std::shared_ptr<AllocationBudget> budget; + int fd = -1; + + explicit CloseNode(std::shared_ptr<AllocationBudget> owner) : budget(std::move(owner)) + { + dispose = [](BackendReleaseService::Node *base) noexcept { + auto *node = static_cast<CloseNode *>(base); + // close consumes this descriptor incarnation even on Linux EINTR. + // Retrying the numeric descriptor could close an unrelated file. + if (node->fd >= 0) { + (void)::close(node->fd); + } + MetadataAllocator<CloseNode> allocator(node->budget); + std::destroy_at(node); + allocator.deallocate(node, 1); + return true; + }; + } +}; + +Status FileMedium::prepare_close(const std::shared_ptr<AllocationBudget> &budget, + const std::shared_ptr<BackendReleaseService> &service) +{ + if (!service) { + return {}; + } + MetadataAllocator<CloseNode> allocator(budget); + auto *node = allocator.allocate(1); + std::construct_at(node, budget); + auto status = service->register_node(*node); + if (!status.ok()) { + std::destroy_at(node); + allocator.deallocate(node, 1); + return status; + } + close_ = node; + return {}; +} + +FileMedium::~FileMedium() +{ + if (close_ != nullptr) { + close_->fd = fd_; + BackendReleaseService::enqueue(*close_); + } + else if (fd_ >= 0) { + // Unique legacy tooling has an explicit synchronous host boundary. + (void)::close(fd_); + } +} + +size_t FileMedium::close_allocation_bound() noexcept +{ + return sizeof(CloseNode); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/block_medium.cpp b/lib/crowdb-tree/src/backend/local/block_medium.cpp new file mode 100644 index 000000000..c43fd9173 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_medium.cpp @@ -0,0 +1,232 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend_metadata.h" +#include "crowdb-tree/backend/block_page_store.h" + +#include <fcntl.h> +#include <unistd.h> + +#include <algorithm> +#include <cerrno> +#include <cstring> + +namespace crowdb::tree +{ +// ── MemoryMedium ─────────────────────────────────────────────────── + +Status MemoryMedium::pwrite_at(uint64_t off, const uint8_t *buf, size_t len) +{ + if (len == 0) { + return Status::Ok(); + } + std::scoped_lock lk(mu_); + if (off + len > data_.size()) { + data_.resize(off + len, 0); + } + std::memcpy(data_.data() + off, buf, len); + return Status::Ok(); +} + +Status MemoryMedium::pread_partial(uint64_t off, uint8_t *buf, size_t len, size_t *out_read) const +{ + std::scoped_lock lk(mu_); + if (off >= data_.size()) { + *out_read = 0; + return Status::Ok(); + } + size_t avail = data_.size() - off; + size_t n = len < avail ? len : avail; + std::memcpy(buf, data_.data() + off, n); + *out_read = n; + return Status::Ok(); +} + +Status MemoryMedium::fsync() +{ + return Status::Ok(); +} + +uint64_t MemoryMedium::size() const +{ + std::scoped_lock lk(mu_); + return data_.size(); +} + +// ── FileMedium ───────────────────────────────────────────────────── + +namespace +{ +Status open_file_descriptor(const char *path, bool o_direct, int *out) +{ +#ifdef __APPLE__ + (void)o_direct; // macOS has no O_DIRECT; use F_NOCACHE instead + int fd = ::open(path, O_RDWR | O_CREAT, 0644); + if (fd < 0) { + return Status::io_error(std::string("open: ") + std::strerror(errno)); + } + if (::fcntl(fd, F_NOCACHE, 1) < 0) { + ::close(fd); + return Status::io_error(std::string("fcntl(F_NOCACHE): ") + std::strerror(errno)); + } +#else + int flags = O_RDWR | O_CREAT; + if (o_direct) { + flags |= O_DIRECT; + } + int fd = ::open(path, flags, 0644); + if (fd < 0) { + return Status::io_error(std::string("open: ") + std::strerror(errno)); + } +#endif + *out = fd; + return Status::Ok(); +} +} // namespace + +void FileMedium::operator delete(void *pointer) noexcept +{ + detail::BackendObjectStorage::destroy(pointer); +} + +Status FileMedium::open(const std::string &path, bool o_direct, std::unique_ptr<FileMedium> *out) +{ + detail::BackendObjectStorage storage(sizeof(FileMedium), alignof(FileMedium), {}); + auto result = std::unique_ptr<FileMedium>(::new (storage.get()) FileMedium(-1)); + (void)storage.release(); + auto status = open_file_descriptor(path.c_str(), o_direct, &result->fd_); + if (status.ok()) { + *out = std::move(result); + } + return status; +} + +Status FileMedium::open_owned(const char *path, bool direct, const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<FileMedium> *out, const std::shared_ptr<BackendReleaseService> &release) +{ + detail::BackendObjectStorage storage(sizeof(FileMedium), alignof(FileMedium), budget); + auto *file = ::new (storage.get()) FileMedium(-1); + (void)storage.release(); + std::shared_ptr<FileMedium> result(file, std::default_delete<FileMedium>{}, MetadataAllocator<FileMedium>(budget)); + auto status = result->prepare_close(budget, release); + if (!status.ok()) { + return status; + } + status = open_file_descriptor(path, direct, &result->fd_); + if (status.ok()) { + *out = std::move(result); + } + return status; +} + +Status FileMedium::pwrite_at(uint64_t off, const uint8_t *buf, size_t len) +{ + if (len == 0) { + return Status::Ok(); + } + mark_dirty(); + + struct CompletionMark + { + FileMedium &medium; + + ~CompletionMark() + { + medium.mark_dirty(); + } + } completion{*this}; + + size_t done = 0; + while (done < len) { + ssize_t n = ::pwrite(fd_, buf + done, len - done, static_cast<off_t>(off + done)); + if (n < 0) { + if (errno == EINTR) { + continue; + } + return Status::io_error(std::string("pwrite: ") + std::strerror(errno)); + } + if (n == 0) { + return Status::io_error("pwrite made no progress"); + } + done += static_cast<size_t>(n); + } + return Status::Ok(); +} + +Status FileMedium::pread_partial(uint64_t off, uint8_t *buf, size_t len, size_t *out_read) const +{ + size_t done = 0; + while (done < len) { + ssize_t n = ::pread(fd_, buf + done, len - done, static_cast<off_t>(off + done)); + if (n < 0) { + if (errno == EINTR) { + continue; + } + *out_read = done; + return Status::io_error(std::string("pread: ") + std::strerror(errno)); + } + if (n == 0) { + break; // EOF: not an error here, caller decides how to treat the + // shortfall + } + done += static_cast<size_t>(n); + } + *out_read = done; + return Status::Ok(); +} + +Status FileMedium::fsync() +{ + const auto generation = dirty_generation(); +#ifdef __APPLE__ + if (::fsync(fd_) < 0) { + return Status::io_error(std::string("fsync: ") + std::strerror(errno)); + } +#else + if (::fdatasync(fd_) < 0) { + return Status::io_error(std::string("fdatasync: ") + std::strerror(errno)); + } +#endif + acknowledge_sync(generation); + return Status::Ok(); +} + +void FileMedium::mark_dirty() noexcept +{ + auto generation = written_.load(std::memory_order_relaxed); + while (generation != UINT64_MAX && + !written_.compare_exchange_weak(generation, generation + 1, std::memory_order_release, + std::memory_order_relaxed)) { + } +} + +uint64_t FileMedium::dirty_generation() const noexcept +{ + return written_.load(std::memory_order_acquire); +} + +bool FileMedium::dirty() const noexcept +{ + const auto synced = synced_.load(std::memory_order_acquire); + const auto written = dirty_generation(); + return written == UINT64_MAX || written != synced; +} + +void FileMedium::acknowledge_sync(uint64_t generation) noexcept +{ + // Saturation remains conservatively dirty instead of wrapping to clean. + if (generation == UINT64_MAX) { + return; + } + auto synced = synced_.load(std::memory_order_relaxed); + while (synced < generation && + !synced_.compare_exchange_weak(synced, generation, std::memory_order_release, std::memory_order_relaxed)) { + } +} + +uint64_t FileMedium::size() const +{ + off_t end = ::lseek(fd_, 0, SEEK_END); + return end < 0 ? 0 : static_cast<uint64_t>(end); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/block_operation_budget.cpp b/lib/crowdb-tree/src/backend/local/block_operation_budget.cpp new file mode 100644 index 000000000..83173f5e8 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_operation_budget.cpp @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "block_account.h" +#include "scheduled_page_store.h" + +namespace crowdb::tree::detail +{ +namespace +{ +PageStore *block_source(AsyncPageStore *backend) noexcept +{ + auto *scheduled = dynamic_cast<ScheduledPageStore *>(backend); + return scheduled != nullptr ? scheduled->synchronous_store() : nullptr; +} +} // namespace + +std::optional<size_t> backend_operation_bound(AsyncPageStore *backend, BackendOperationLimits limits) noexcept +{ + auto *scheduled = dynamic_cast<ScheduledPageStore *>(backend); + if (scheduled != nullptr) { + limits.asynchronous = scheduled->uses_async_transport(); + limits.workers = std::max(limits.workers, scheduled->allocation_workers()); + } + return backend_operation_bound(block_source(backend), limits); +} + +Status reserve_backend_operation(AsyncPageStore *backend, const std::shared_ptr<ResourceBudget::Account> &account, + BackendOperationLimits limits, std::shared_ptr<AllocationBudget> *out) +{ + auto *scheduled = dynamic_cast<ScheduledPageStore *>(backend); + if (scheduled != nullptr) { + limits.asynchronous = scheduled->uses_async_transport(); + limits.workers = std::max(limits.workers, scheduled->allocation_workers()); + } + return reserve_backend_operation(block_source(backend), account, limits, out); +} + +Status bind_backend_operation(AsyncPageStore *backend, BackendOperationLimits limits, ResourceBudget::Credit credit, + std::shared_ptr<AllocationBudget> *out) +{ + auto *scheduled = dynamic_cast<ScheduledPageStore *>(backend); + if (scheduled != nullptr) { + limits.asynchronous = scheduled->uses_async_transport(); + limits.workers = std::max(limits.workers, scheduled->allocation_workers()); + } + return bind_backend_operation(block_source(backend), limits, std::move(credit), out); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/local/block_page_store.cpp b/lib/crowdb-tree/src/backend/local/block_page_store.cpp index 376a7c470..96d131bc3 100644 --- a/lib/crowdb-tree/src/backend/local/block_page_store.cpp +++ b/lib/crowdb-tree/src/backend/local/block_page_store.cpp @@ -1,22 +1,18 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -// Local block-backend implementation. - #include "crowdb-tree/backend/block_page_store.h" -#include <dirent.h> -#include <fcntl.h> +#include "backend_metadata.h" +#include "block_directory.h" +#include "btree/allocation_account.h" +#include "crowdb-tree/backend/backend_release_service.h" + #include <sys/stat.h> -#include <unistd.h> -#include <algorithm> #include <cerrno> -#include <cstdio> #include <cstring> #include <iomanip> #include <sstream> - #ifdef __linux__ # include <linux/fs.h> # include <sys/ioctl.h> @@ -24,204 +20,95 @@ namespace crowdb::tree { - -namespace -{ -// O_DIRECT's buffer-address alignment requirement is filesystem/device -// specific but 4096 (a common page/sector size) satisfies virtually every -// real device; safe to over-align a bit beyond iu_size_ when iu_size_ is -// smaller (e.g. a caller-supplied 512 on a regular file). -constexpr size_t kDirectBufAlign = 4096; - -size_t align_up(size_t v, size_t a) -{ - return (v + a - 1) / a * a; -} - -// RAII aligned scratch buffer (posix_memalign), used for the bounce path -// when a caller's offset/length/buffer isn't already O_DIRECT-aligned. -class AlignedBuffer -{ - public: - AlignedBuffer(size_t len, size_t align) - { - size_t a = align > kDirectBufAlign ? align : kDirectBufAlign; - if (::posix_memalign(reinterpret_cast<void **>(&ptr_), a, len == 0 ? a : len) != 0) { - ptr_ = nullptr; - } - len_ = ptr_ != nullptr ? len : 0; - } - - ~AlignedBuffer() - { - std::free(ptr_); - } - - AlignedBuffer(const AlignedBuffer &) = delete; - AlignedBuffer &operator=(const AlignedBuffer &) = delete; - - [[nodiscard]] uint8_t *data() const - { - return ptr_; - } - - [[nodiscard]] bool ok() const - { - return ptr_ != nullptr; - } - - private: - uint8_t *ptr_ = nullptr; - size_t len_ = 0; -}; -} // namespace - -// ── MemoryMedium ─────────────────────────────────────────────────── - -Status MemoryMedium::pwrite_at(uint64_t off, const uint8_t *buf, size_t len) -{ - if (len == 0) { - return Status::Ok(); +BlockPageStore::BlockPageStore(std::shared_ptr<detail::AllocationAccount> allocations, + std::shared_ptr<BlockPageStoreMedium> medium, uint32_t iu_size, bool is_block_device, + std::shared_ptr<BackendReleaseService> release) + : allocations_(std::move(allocations)), + release_(std::move(release)), + medium_(std::move(medium)), + iu_size_(iu_size), + is_block_device_(is_block_device) +{ +} + +BlockPageStore::BlockPageStore(std::shared_ptr<detail::AllocationAccount> allocations, const std::string &dir, + uint32_t store_id, uint32_t group_id, uint64_t block_size, uint32_t iu_size, + std::shared_ptr<BackendReleaseService> release) + : allocations_(std::move(allocations)), + release_(std::move(release)), + iu_size_(iu_size), + is_block_device_(false), + block_size_(block_size), + store_id_(store_id), + group_id_(group_id), + dir_(dir.data(), dir.size(), MetadataAllocator<char>(allocations_)) +{ + MetadataAllocator<BlockDirectory> allocator(allocations_); + auto *directory = allocator.allocate(1); + try { + std::construct_at(directory, allocations_, release_); } - std::scoped_lock lk(mu_); - if (off + len > data_.size()) { - data_.resize(off + len, 0); + catch (...) { + allocator.deallocate(directory, 1); + throw; } - std::memcpy(data_.data() + off, buf, len); - return Status::Ok(); + blocks_ = {directory, BlockDirectoryDeleter{allocations_}}; } -Status MemoryMedium::pread_partial(uint64_t off, uint8_t *buf, size_t len, size_t *out_read) const +BlockPageStore::~BlockPageStore() { - std::scoped_lock lk(mu_); - if (off >= data_.size()) { - *out_read = 0; - return Status::Ok(); + blocks_.reset(); + medium_.reset(); + if (release_) { + release_->stop_admission(); + if (!release_->externally_driven()) { + // Standalone store destruction is an explicit host boundary. It + // drains only queued work; retained readers keep their paid nodes. + while (release_->snapshot().pending != 0) { + (void)release_->pump(64); + } + } } - size_t avail = data_.size() - off; - size_t n = len < avail ? len : avail; - std::memcpy(buf, data_.data() + off, n); - *out_read = n; - return Status::Ok(); -} - -Status MemoryMedium::fsync() -{ - return Status::Ok(); } -uint64_t MemoryMedium::size() const +void BlockPageStore::operator delete(void *pointer) noexcept { - std::scoped_lock lk(mu_); - return data_.size(); + detail::BackendObjectStorage::destroy(pointer); } -// ── FileMedium ───────────────────────────────────────────────────── - -FileMedium::~FileMedium() +int BlockPageStore::fd() const { - if (fd_ >= 0) { - ::close(fd_); - } + auto *fm = dynamic_cast<FileMedium *>(medium_.get()); + return (fm != nullptr) ? fm->fd() : -1; } -Status FileMedium::open(const std::string &path, bool o_direct, std::unique_ptr<FileMedium> *out) +Status BlockPageStore::open(const std::string &path, uint32_t iu_size, std::unique_ptr<BlockPageStore> *out) { -#ifdef __APPLE__ - (void)o_direct; // macOS has no O_DIRECT; use F_NOCACHE instead - int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0644); - if (fd < 0) { - return Status::io_error(std::string("open: ") + std::strerror(errno)); - } - if (::fcntl(fd, F_NOCACHE, 1) < 0) { - ::close(fd); - return Status::io_error(std::string("fcntl(F_NOCACHE): ") + std::strerror(errno)); - } -#else - int flags = O_RDWR | O_CREAT; - if (o_direct) { - flags |= O_DIRECT; + std::shared_ptr<detail::AllocationAccount> allocations; + auto s = detail::AllocationAccount::create_unbound(&allocations); + if (!s.ok()) { + return s; } - int fd = ::open(path.c_str(), flags, 0644); - if (fd < 0) { - return Status::io_error(std::string("open: ") + std::strerror(errno)); + std::shared_ptr<BackendReleaseService> release; + s = BackendReleaseService::create(allocations, SIZE_MAX / 2, {}, nullptr, nullptr, &release); + if (!s.ok()) { + return s; } -#endif - out->reset(new FileMedium(fd)); - return Status::Ok(); -} -Status FileMedium::pwrite_at(uint64_t off, const uint8_t *buf, size_t len) -{ - size_t done = 0; - while (done < len) { - ssize_t n = ::pwrite(fd_, buf + done, len - done, static_cast<off_t>(off + done)); - if (n < 0) { - if (errno == EINTR) { - continue; - } - return Status::io_error(std::string("pwrite: ") + std::strerror(errno)); - } - done += static_cast<size_t>(n); - } - return Status::Ok(); -} + struct OpenDrain + { + std::shared_ptr<BackendReleaseService> service; -Status FileMedium::pread_partial(uint64_t off, uint8_t *buf, size_t len, size_t *out_read) const -{ - size_t done = 0; - while (done < len) { - ssize_t n = ::pread(fd_, buf + done, len - done, static_cast<off_t>(off + done)); - if (n < 0) { - if (errno == EINTR) { - continue; + ~OpenDrain() + { + while (service->snapshot().pending != 0) { + (void)service->pump(64); } - *out_read = done; - return Status::io_error(std::string("pread: ") + std::strerror(errno)); - } - if (n == 0) { - break; // EOF: not an error here, caller decides how to treat the shortfall } - done += static_cast<size_t>(n); - } - *out_read = done; - return Status::Ok(); -} - -Status FileMedium::fsync() -{ -#ifdef __APPLE__ - if (::fsync(fd_) < 0) { - return Status::io_error(std::string("fsync: ") + std::strerror(errno)); - } -#else - if (::fdatasync(fd_) < 0) { - return Status::io_error(std::string("fdatasync: ") + std::strerror(errno)); - } -#endif - return Status::Ok(); -} - -uint64_t FileMedium::size() const -{ - off_t end = ::lseek(fd_, 0, SEEK_END); - return end < 0 ? 0 : static_cast<uint64_t>(end); -} - -// ── BlockPageStore ───────────────────────────────────────────────── + } open_drain{release}; -BlockPageStore::~BlockPageStore() = default; - -int BlockPageStore::fd() const -{ - auto *fm = dynamic_cast<FileMedium *>(medium_.get()); - return (fm != nullptr) ? fm->fd() : -1; -} - -Status BlockPageStore::open(const std::string &path, uint32_t iu_size, std::unique_ptr<BlockPageStore> *out) -{ - std::unique_ptr<FileMedium> medium; - Status s = FileMedium::open(path, true, &medium); + std::shared_ptr<FileMedium> medium; + s = FileMedium::open_owned(path.c_str(), true, allocations, &medium, release); if (!s.ok()) { return s; } @@ -247,7 +134,10 @@ Status BlockPageStore::open(const std::string &path, uint32_t iu_size, std::uniq } #endif - auto *store = new BlockPageStore(std::move(medium), effective_iu, is_block); + detail::BackendObjectStorage storage(sizeof(BlockPageStore), alignof(BlockPageStore), allocations); + auto *store = ::new (storage.get()) + BlockPageStore(allocations, std::move(medium), effective_iu, is_block, std::move(release)); + (void)storage.release(); store->capacity_ = probed_bytes; out->reset(store); return Status::Ok(); @@ -255,434 +145,20 @@ Status BlockPageStore::open(const std::string &path, uint32_t iu_size, std::uniq Status BlockPageStore::open_mem(uint32_t iu_size, std::unique_ptr<BlockPageStore> *out) { - auto medium = std::make_unique<MemoryMedium>(); - uint32_t effective_iu = iu_size == 0 ? 1 : iu_size; - out->reset(new BlockPageStore(std::move(medium), effective_iu, false)); - return Status::Ok(); -} - -namespace -{ -// Parse a block filename like "1-3.blk-0007" → (store_id=1, group_id=3, idx=7). -// Returns false if the filename doesn't match the pattern. -bool parse_block_filename(const std::string &name, uint32_t &out_store, uint32_t &out_part, uint32_t &out_idx) -{ - auto dot = name.find('.'); - if (dot == std::string::npos) { - return false; - } - if (name.substr(dot + 1, 4) != "blk-") { - return false; - } - auto dash = name.find('-'); - if (dash == std::string::npos || dash >= dot) { - return false; - } - try { - out_store = static_cast<uint32_t>(std::stoul(name.substr(0, dash))); - out_part = static_cast<uint32_t>(std::stoul(name.substr(dash + 1, dot - dash - 1))); - out_idx = static_cast<uint32_t>(std::stoul(name.substr(dot + 5))); - } - catch (...) { - return false; - } - return true; -} -} // namespace - -Status BlockPageStore::open_blocks(const std::string &dir, uint32_t store_id, uint32_t group_id, uint64_t block_size, - uint32_t iu_size, std::unique_ptr<BlockPageStore> *out) -{ - if (block_size == 0) { - return Status::invalid_argument("open_blocks: block_size must be > 0"); - } - - auto *store = new BlockPageStore(dir, store_id, group_id, block_size, iu_size == 0 ? 4096 : iu_size); - - // Scan directory for existing block files matching {store_id}-{group_id}.blk-* - DIR *d = ::opendir(dir.c_str()); - if (d != nullptr) { - struct dirent *ent; - while ((ent = ::readdir(d)) != nullptr) { - uint32_t s_id = 0; - uint32_t p_id = 0; - uint32_t idx = 0; - if (!parse_block_filename(ent->d_name, s_id, p_id, idx)) { - continue; - } - if (s_id != store_id || p_id != group_id) { - continue; - } - // Open the existing block file - std::string path = dir + "/" + ent->d_name; - std::unique_ptr<FileMedium> fm; - Status s = FileMedium::open(path, store->iu_size_ > 1, &fm); - if (!s.ok()) { - ::closedir(d); - delete store; - return s; - } - BlockExtent ext; - ext.medium = std::move(fm); - ext.base_offset = static_cast<uint64_t>(idx) * block_size; - ext.used = ext.medium->size(); - ext.dirty = false; - store->extents_.push_back(std::move(ext)); - } - ::closedir(d); + std::shared_ptr<detail::AllocationAccount> allocations; + auto status = detail::AllocationAccount::create_unbound(&allocations); + if (!status.ok()) { + return status; } - - // Sort extents by base_offset (which is idx * block_size) - std::sort(store->extents_.begin(), store->extents_.end(), - [](const BlockExtent &a, const BlockExtent &b) { return a.base_offset < b.base_offset; }); - - // Insert deleted placeholder extents for missing block indices (deleted - // blocks leave gaps in the index sequence — must preserve offset-to-extent mapping). - if (!store->extents_.empty()) { - std::vector<BlockExtent> dense; - uint32_t expected_idx = 0; - for (auto &ext : store->extents_) { - auto ext_idx = static_cast<uint32_t>(ext.base_offset / block_size); - while (expected_idx < ext_idx) { - BlockExtent gap; - gap.base_offset = static_cast<uint64_t>(expected_idx) * block_size; - gap.deleted = true; - dense.push_back(std::move(gap)); - ++expected_idx; - } - dense.push_back(std::move(ext)); - ++expected_idx; - } - store->extents_ = std::move(dense); - } - - // If no blocks exist, allocate the first one - if (store->extents_.empty()) { - Status s = store->allocate_new_block(); - if (!s.ok()) { - delete store; - return s; - } - } - + auto medium = std::allocate_shared<MemoryMedium>(MetadataAllocator<MemoryMedium>(allocations)); + const uint32_t effective_iu = iu_size == 0 ? 1 : iu_size; + detail::BackendObjectStorage storage(sizeof(BlockPageStore), alignof(BlockPageStore), allocations); + auto *store = ::new (storage.get()) BlockPageStore(allocations, std::move(medium), effective_iu, false); + (void)storage.release(); out->reset(store); return Status::Ok(); } -Status BlockPageStore::allocate_new_block() -{ - auto idx = static_cast<uint32_t>(extents_.size()); - char name[64]; - std::snprintf(name, sizeof(name), "%u-%u.blk-%04u", store_id_, group_id_, idx); - std::string path = dir_ + "/" + name; - - std::unique_ptr<FileMedium> fm; - Status s = FileMedium::open(path, iu_size_ > 1, &fm); - if (!s.ok()) { - return s; - } - - BlockExtent ext; - ext.medium = std::move(fm); - ext.base_offset = static_cast<uint64_t>(idx) * block_size_; - ext.used = 0; - ext.dirty = false; - extents_.push_back(std::move(ext)); - return Status::Ok(); -} - -Status BlockPageStore::delete_block(uint32_t block_idx) -{ - if (block_idx >= extents_.size()) { - return Status::invalid_argument("delete_block: block index out of range"); - } - - auto &ext = extents_[block_idx]; - if (ext.deleted) { - return Status::Ok(); // already deleted - } - - char name[64]; - std::snprintf(name, sizeof(name), "%u-%u.blk-%04u", store_id_, group_id_, block_idx); - std::string path = dir_ + "/" + name; - - if (::unlink(path.c_str()) < 0 && errno != ENOENT) { - return Status::io_error(std::string("unlink: ") + std::strerror(errno)); - } - - ext.medium.reset(); - ext.deleted = true; - while (!extents_.empty() && extents_.back().deleted) { - extents_.pop_back(); - } - return Status::Ok(); -} - -Status BlockPageStore::write_at_extents(uint64_t off, const uint8_t *buf, size_t len) -{ - // Ensure enough extents are allocated - uint64_t end = off + len; - while (extents_.back().base_offset + block_size_ < end) { - Status s = allocate_new_block(); - if (!s.ok()) { - return s; - } - } - - // Split write across extent boundaries - size_t done = 0; - uint64_t cur = off; - while (done < len) { - // Find the extent containing `cur` - uint64_t extent_idx = cur / block_size_; - if (extent_idx >= extents_.size()) { - return Status::io_error("BlockPageStore: extent index out of range"); - } - auto &ext = extents_[extent_idx]; - if (ext.deleted) { - return Status::io_error("BlockPageStore: write to deleted block"); - } - uint64_t local = cur - ext.base_offset; - uint64_t avail = block_size_ - local; - size_t chunk = std::min(static_cast<uint64_t>(len - done), avail); - - // For IU=1, write directly. For IU>1, use the bounce path via medium. - Status s; - if (iu_size_ <= 1) { - s = ext.medium->pwrite_at(local, buf + done, chunk); - } - else { - // Use the same alignment logic as single-medium mode - bool aligned = (local % iu_size_ == 0) && (chunk % iu_size_ == 0) && - (reinterpret_cast<uintptr_t>(buf + done) % kDirectBufAlign == 0); - if (aligned) { - s = ext.medium->pwrite_at(local, buf + done, chunk); - } - else { - uint64_t start = (local / iu_size_) * iu_size_; - uint64_t e = align_up(local + chunk, iu_size_); - size_t span = e - start; - AlignedBuffer scratch(span, iu_size_); - if (!scratch.ok()) { - return Status::io_error("BlockPageStore: aligned scratch allocation failed"); - } - size_t got = 0; - s = ext.medium->pread_partial(start, scratch.data(), span, &got); - if (s.ok()) { - if (got < span) { - std::memset(scratch.data() + got, 0, span - got); - } - std::memcpy(scratch.data() + (local - start), buf + done, chunk); - s = ext.medium->pwrite_at(start, scratch.data(), span); - } - } - } - if (!s.ok()) { - return s; - } - ext.used = std::max(ext.used, local + chunk); - ext.dirty = true; - done += chunk; - cur += chunk; - } - return Status::Ok(); -} - -Status BlockPageStore::read_at_extents(uint64_t off, uint8_t *buf, size_t len) const -{ - size_t done = 0; - uint64_t cur = off; - while (done < len) { - uint64_t extent_idx = cur / block_size_; - if (extent_idx >= extents_.size()) { - return Status::not_found("BlockPageStore: read past end (no extent)"); - } - const auto &ext = extents_[extent_idx]; - if (ext.deleted) { - return Status::io_error("BlockPageStore: read from deleted block"); - } - uint64_t local = cur - ext.base_offset; - uint64_t avail = block_size_ - local; - size_t chunk = std::min(static_cast<uint64_t>(len - done), avail); - - Status s; - if (iu_size_ <= 1) { - s = ext.medium->pread_at(local, buf + done, chunk); - } - else { - bool aligned = (local % iu_size_ == 0) && (chunk % iu_size_ == 0) && - (reinterpret_cast<uintptr_t>(buf + done) % kDirectBufAlign == 0); - if (aligned) { - s = ext.medium->pread_at(local, buf + done, chunk); - } - else { - uint64_t start = (local / iu_size_) * iu_size_; - uint64_t e = align_up(local + chunk, iu_size_); - size_t span = e - start; - AlignedBuffer scratch(span, iu_size_); - if (!scratch.ok()) { - return Status::io_error("BlockPageStore: aligned scratch allocation failed"); - } - size_t got = 0; - s = ext.medium->pread_partial(start, scratch.data(), span, &got); - if (s.ok()) { - if (got < (local - start) + chunk) { - return Status::not_found("BlockPageStore: read past end"); - } - std::memcpy(buf + done, scratch.data() + (local - start), chunk); - } - } - } - if (!s.ok()) { - return s; - } - done += chunk; - cur += chunk; - } - return Status::Ok(); -} - -Status BlockPageStore::write_at(uint64_t off, const uint8_t *buf, size_t len) -{ - if (len == 0) { - return Status::Ok(); - } - - // Array-of-blocks mode - if (!extents_.empty()) { - return write_at_extents(off, buf, len); - } - - // Single-medium mode - // IU=1 (byte-aligned): no alignment checks, no bounce buffer. - if (iu_size_ <= 1) { - return medium_->pwrite_at(off, buf, len); - } - - bool aligned = - (off % iu_size_ == 0) && (len % iu_size_ == 0) && (reinterpret_cast<uintptr_t>(buf) % kDirectBufAlign == 0); - if (aligned) { - return medium_->pwrite_at(off, buf, len); - } - - // Bounce through an IU-aligned scratch span covering [off, off+len): - // read-modify-write, since O_DIRECT can't write a sub-IU or misaligned - // range directly. - uint64_t start = (off / iu_size_) * iu_size_; - uint64_t end = align_up(off + len, iu_size_); - size_t span = end - start; - AlignedBuffer scratch(span, iu_size_); - if (!scratch.ok()) { - return Status::io_error("BlockPageStore: aligned scratch allocation failed"); - } - size_t got = 0; - Status rs = medium_->pread_partial(start, scratch.data(), span, &got); - if (!rs.ok()) { - return rs; - } - if (got < span) { - // Genuinely new territory past the current end (growing the - // store) rather than a read failure -- zero-fill the unread tail. - std::memset(scratch.data() + got, 0, span - got); - } - std::memcpy(scratch.data() + (off - start), buf, len); - return medium_->pwrite_at(start, scratch.data(), span); -} - -Status BlockPageStore::read_at(uint64_t off, uint8_t *buf, size_t len) const -{ - if (len == 0) { - return Status::Ok(); - } - - // Array-of-blocks mode - if (!extents_.empty()) { - return read_at_extents(off, buf, len); - } - - // Single-medium mode - // IU=1 (byte-aligned): no alignment checks, no bounce buffer. - if (iu_size_ <= 1) { - return medium_->pread_at(off, buf, len); - } - - bool aligned = - (off % iu_size_ == 0) && (len % iu_size_ == 0) && (reinterpret_cast<uintptr_t>(buf) % kDirectBufAlign == 0); - if (aligned) { - return medium_->pread_at(off, buf, len); - } - - uint64_t start = (off / iu_size_) * iu_size_; - uint64_t end = align_up(off + len, iu_size_); - size_t span = end - start; - AlignedBuffer scratch(span, iu_size_); - if (!scratch.ok()) { - return Status::io_error("BlockPageStore: aligned scratch allocation failed"); - } - size_t got = 0; - Status rs = medium_->pread_partial(start, scratch.data(), span, &got); - if (!rs.ok()) { - return rs; - } - if (got < (off - start) + len) { - return Status::not_found("BlockPageStore: read past end"); - } - std::memcpy(buf, scratch.data() + (off - start), len); - return Status::Ok(); -} - -Status BlockPageStore::sync() -{ - // CT_SYNC_SKIP: no fsync at all (tests/CI only) - if (sync_mode_ == SyncMode::kSkip) { - return Status::Ok(); - } - // CT_SYNC_FULL and CT_SYNC_BATCH both fsync here. In BATCH mode, - // persist.cpp's snapshot() calls sync() only at the two barrier points - // (before and after the anchor write), so the batching is already - // handled by the caller not calling sync() per-page. - // Array-of-blocks mode: fsync all dirty extents - if (!extents_.empty()) { - for (auto &ext : extents_) { - if (ext.deleted || !ext.dirty) { - continue; - } - Status s = ext.medium->fsync(); - if (!s.ok()) { - return s; - } - ext.dirty = false; - } - return Status::Ok(); - } - - // Single-medium mode - return medium_->fsync(); -} - -uint64_t BlockPageStore::size() const -{ - // Array-of-blocks mode: logical high-water mark (skip deleted blocks) - if (!extents_.empty()) { - uint64_t total = 0; - for (const auto &ext : extents_) { - if (!ext.deleted) { - total = std::max(total, ext.base_offset + ext.used); - } - } - return total; - } - - // Single-medium mode - if (is_block_device_ && capacity_ > 0) { - return capacity_; - } - if (medium_ == nullptr) { - return 0; - } - return medium_->size(); -} - // ── Dump utility ─────────────────────────────────────────────────── Status dump_block_file(const std::string &path, uint32_t iu_size, std::string *out) @@ -721,111 +197,4 @@ Status dump_block_file(const std::string &path, uint32_t iu_size, std::string *o return Status::Ok(); } -int BlockPageStore::fd_for_offset(uint64_t off, uint64_t *out_local) const -{ - if (extents_.empty()) { - // Single-medium mode: only works if the medium is a FileMedium - auto *fm = dynamic_cast<FileMedium *>(medium_.get()); - if (fm == nullptr) { - return -1; - } - *out_local = off; - return fm->fd(); - } - // Array-of-blocks mode - uint64_t extent_idx = off / block_size_; - if (extent_idx >= extents_.size()) { - return -1; - } - const auto &ext = extents_[extent_idx]; - if (ext.deleted) { - return -1; - } - auto *fm = dynamic_cast<FileMedium *>(ext.medium.get()); - if (fm == nullptr) { - return -1; - } - *out_local = off - ext.base_offset; - return fm->fd(); -} - -Status BlockPageStore::ensure_extents(uint64_t off, size_t len) -{ - if (extents_.empty()) { - // Single-medium mode: no block allocation needed - return Status::Ok(); - } - // Allocate blocks until the range [off, off+len) is covered - uint64_t end = off + len; - while (extents_.back().base_offset + block_size_ < end) { - Status s = allocate_new_block(); - if (!s.ok()) { - return s; - } - } - // Mark covered extents as dirty - uint64_t cur = off; - while (cur < end) { - uint64_t extent_idx = cur / block_size_; - if (extent_idx < extents_.size()) { - extents_[extent_idx].dirty = true; - } - uint64_t next_boundary = (extent_idx + 1) * block_size_; - cur = std::min(end, next_boundary); - } - return Status::Ok(); -} - -std::vector<int> BlockPageStore::dirty_fds() -{ - std::vector<int> fds; - if (sync_mode_ == SyncMode::kSkip) { - return fds; - } - if (extents_.empty()) { - // Single-medium mode - auto *fm = dynamic_cast<FileMedium *>(medium_.get()); - if (fm != nullptr && fm->fd() >= 0) { - fds.push_back(fm->fd()); - } - return fds; - } - // Array-of-blocks mode: collect dirty, non-deleted extent fds - for (auto &ext : extents_) { - if (ext.deleted || !ext.dirty) { - continue; - } - auto *fm = dynamic_cast<FileMedium *>(ext.medium.get()); - if (fm != nullptr && fm->fd() >= 0) { - fds.push_back(fm->fd()); - ext.dirty = false; - } - } - return fds; -} - -std::vector<int> BlockPageStore::all_extent_fds() const -{ - std::vector<int> fds; - if (extents_.empty()) { - // Single-medium mode - auto *fm = dynamic_cast<FileMedium *>(medium_.get()); - if (fm != nullptr && fm->fd() >= 0) { - fds.push_back(fm->fd()); - } - return fds; - } - // Array-of-blocks mode: collect all non-deleted extent fds - for (const auto &ext : extents_) { - if (ext.deleted) { - continue; - } - auto *fm = dynamic_cast<FileMedium *>(ext.medium.get()); - if (fm != nullptr && fm->fd() >= 0) { - fds.push_back(fm->fd()); - } - } - return fds; -} - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/block_page_store_directory.cpp b/lib/crowdb-tree/src/backend/local/block_page_store_directory.cpp new file mode 100644 index 000000000..ae457e0ab --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_page_store_directory.cpp @@ -0,0 +1,345 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend_metadata.h" +#include "block_account.h" +#include "block_directory.h" +#include "btree/allocation_account.h" +#include "crowdb-tree/backend/backend_release_service.h" + +#include <dirent.h> + +#include <cerrno> +#include <cstdio> +#include <cstring> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +// Parse a block filename like "1-3.blk-0007" → (store_id=1, group_id=3, idx=7). +// Returns false if the filename doesn't match the pattern. +bool parse_block_filename(const std::string &name, uint32_t &out_store, uint32_t &out_part, uint32_t &out_idx) +{ + auto dot = name.find('.'); + if (dot == std::string::npos) { + return false; + } + if (name.substr(dot + 1, 4) != "blk-") { + return false; + } + auto dash = name.find('-'); + if (dash == std::string::npos || dash >= dot) { + return false; + } + try { + out_store = static_cast<uint32_t>(std::stoul(name.substr(0, dash))); + out_part = static_cast<uint32_t>(std::stoul(name.substr(dash + 1, dot - dash - 1))); + out_idx = static_cast<uint32_t>(std::stoul(name.substr(dot + 5))); + } + catch (...) { + return false; + } + return true; +} +} // namespace + +Status BlockPageStore::open_blocks(const std::string &dir, uint32_t store_id, uint32_t group_id, uint64_t block_size, + uint32_t iu_size, std::unique_ptr<BlockPageStore> *out) +try { + const auto unit = iu_size == 0 ? 4096 : iu_size; + if (out == nullptr || block_size == 0 || block_size % unit != 0) { + return Status::invalid_argument("open_blocks: invalid block geometry"); + } + std::shared_ptr<detail::AllocationAccount> allocations; + auto status = detail::AllocationAccount::create_unbound(&allocations); + if (!status.ok()) { + return status; + } + std::shared_ptr<BackendReleaseService> release; + status = BackendReleaseService::create(allocations, SIZE_MAX / 2, {}, nullptr, nullptr, &release); + if (!status.ok()) { + return status; + } + detail::BackendObjectStorage storage(sizeof(BlockPageStore), alignof(BlockPageStore), allocations); + auto *value = ::new (storage.get()) + BlockPageStore(allocations, dir, store_id, group_id, block_size, unit, std::move(release)); + (void)storage.release(); + auto store = std::unique_ptr<BlockPageStore>(value); + DIR *directory = ::opendir(dir.c_str()); + if (directory == nullptr) { + return Status::io_error(std::string("opendir: ") + std::strerror(errno)); + } + + struct CloseDirectory + { + DIR *value; + + ~CloseDirectory() + { + ::closedir(value); + } + } close{directory}; + + while (auto *entry = ::readdir(directory)) { + uint32_t found_store = 0; + uint32_t found_group = 0; + uint32_t index = 0; + if (!parse_block_filename(entry->d_name, found_store, found_group, index) || found_store != store_id || + found_group != group_id) { + continue; + } + std::shared_ptr<FileMedium> file; + auto status = store->blocks_->ensure(index, dir + "/" + entry->d_name, unit > 1, &file); + if (!status.ok()) { + return status; + } + } + if (store->num_extents() == 0) { + std::shared_ptr<FileMedium> file; + auto status = store->blocks_->ensure(0, store->block_path(0).c_str(), unit > 1, &file); + if (!status.ok()) { + return status; + } + } + *out = std::move(store); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +BlockPageStore::BlockPath BlockPageStore::block_path(uint32_t index, + const std::shared_ptr<AllocationBudget> &budget) const +{ + char name[64]; + std::snprintf(name, sizeof(name), "%u-%u.blk-%04u", store_id_, group_id_, index); + BlockPath path(dir_, MetadataAllocator<char>(budget ? budget : allocations_)); + path += '/'; + path += name; + return path; +} + +Status BlockPageStore::delete_block(uint32_t block_idx) +{ + return delete_block(block_idx, {}); +} + +Status BlockPageStore::delete_block(uint32_t block_idx, const std::shared_ptr<AllocationBudget> &budget) +try { + if (!detail::valid_block_budget(*this, budget)) { + return Status::invalid_argument("backend allocation budget belongs to another store"); + } + if (!blocks_) { + return Status::invalid_argument("delete_block requires an array of blocks"); + } + return blocks_->erase(block_idx, block_path(block_idx, budget).c_str(), budget); +} + +catch (const std::bad_alloc &) { + return Status::resource_exhausted("backend metadata allocation unavailable"); +} + +size_t BlockPageStore::num_extents() const +{ + return blocks_ ? blocks_->count() : 0; +} + +Status BlockPageStore::ensure_extents(uint64_t off, size_t len, const std::shared_ptr<AllocationBudget> &budget) +try { + if (!detail::valid_block_budget(*this, budget)) { + return Status::invalid_argument("backend allocation budget belongs to another store"); + } + if (len == 0) { + return Status::Ok(); + } + if (off > UINT64_MAX - len || (blocks_ && (off + len - 1) / block_size_ > UINT32_MAX)) { + return Status::invalid_argument("BlockPageStore: byte range overflow"); + } + if (!blocks_) { + if (auto *file = dynamic_cast<FileMedium *>(medium_.get())) { + file->mark_dirty(); + } + return Status::Ok(); + } + const auto last = (off + len - 1) / block_size_; + for (auto index = off / block_size_; index <= last; ++index) { + std::shared_ptr<FileMedium> file; + auto status = + blocks_->ensure(static_cast<uint32_t>(index), block_path(static_cast<uint32_t>(index), budget).c_str(), + iu_size_ > 1, &file, budget); + if (!status.ok()) { + return status; + } + file->mark_dirty(); + } + return Status::Ok(); +} + +catch (const std::bad_alloc &) { + return Status::resource_exhausted("backend metadata allocation unavailable"); +} + +int BlockPageStore::fd_for_offset(uint64_t off, uint64_t *out_local) const +{ + auto file = file_for_offset(off, out_local); + return file ? file->fd() : -1; +} + +std::shared_ptr<FileMedium> BlockPageStore::file_for_offset(uint64_t off, uint64_t *out_local, + const std::shared_ptr<AllocationBudget> &budget) const +{ + if (!detail::valid_block_budget(*this, budget)) { + return {}; + } + if (!blocks_) { + *out_local = off; + return std::dynamic_pointer_cast<FileMedium>(medium_); + } + if (off / block_size_ > UINT32_MAX) { + return {}; + } + *out_local = off % block_size_; + return blocks_->get(static_cast<uint32_t>(off / block_size_), budget); +} + +Status BlockPageStore::sync() +{ + return sync({}); +} + +Status BlockPageStore::sync(const std::shared_ptr<AllocationBudget> &budget) +{ + if (!detail::valid_block_budget(*this, budget)) { + return Status::invalid_argument("backend allocation budget belongs to another store"); + } + if (sync_mode_ == SyncMode::kSkip) { + return Status::Ok(); + } + if (!blocks_) { + return medium_->fsync(); + } + uint64_t cursor = 0; + uint32_t index = 0; + std::shared_ptr<FileMedium> file; + while (blocks_->next(&cursor, &index, &file, budget)) { + if (file->dirty()) { + auto status = file->fsync(); + if (!status.ok()) { + return status; + } + } + } + return Status::Ok(); +} + +uint64_t BlockPageStore::size() const +{ + return size({}); +} + +uint64_t BlockPageStore::size(const std::shared_ptr<AllocationBudget> &budget) const +{ + if (!detail::valid_block_budget(*this, budget)) { + throw std::invalid_argument("backend allocation budget belongs to another store"); + } + if (!blocks_) { + if (is_block_device_ && capacity_ > 0) { + return capacity_; + } + return medium_ ? medium_->size() : 0; + } + uint64_t total = 0; + uint64_t cursor = 0; + uint32_t index = 0; + std::shared_ptr<FileMedium> file; + while (blocks_->next(&cursor, &index, &file, budget)) { + total = std::max(total, (static_cast<uint64_t>(index) * block_size_) + file->size()); + } + return total; +} + +std::vector<BlockSyncTarget> BlockPageStore::sync_targets() const +{ + std::vector<BlockSyncTarget> targets; + if (sync_mode_ == SyncMode::kSkip) { + return targets; + } + if (!blocks_) { + auto file = std::dynamic_pointer_cast<FileMedium>(medium_); + if (file && file->dirty()) { + targets.push_back({.medium = file, .generation = file->dirty_generation()}); + } + return targets; + } + uint64_t cursor = 0; + uint32_t index = 0; + std::shared_ptr<FileMedium> file; + while (blocks_->next(&cursor, &index, &file)) { + if (file->dirty()) { + targets.push_back({.medium = file, .generation = file->dirty_generation()}); + } + } + return targets; +} + +BlockPageStore::BlockSyncTargets BlockPageStore::sync_targets(const std::shared_ptr<AllocationBudget> &budget) const +{ + auto allocations = detail::block_allocation_budget(*this, budget); + if (!allocations) { + throw std::invalid_argument("backend allocation budget belongs to another store"); + } + BlockSyncTargets targets{MetadataAllocator<BlockSyncTarget>(allocations)}; + if (sync_mode_ == SyncMode::kSkip) { + return targets; + } + targets.reserve(blocks_ ? blocks_->count() : 1); + if (!blocks_) { + auto file = std::dynamic_pointer_cast<FileMedium>(medium_); + if (file && file->dirty()) { + targets.push_back({.medium = file, .generation = file->dirty_generation()}); + } + return targets; + } + uint64_t cursor = 0; + uint32_t index = 0; + std::shared_ptr<FileMedium> file; + while (blocks_->next(&cursor, &index, &file, allocations)) { + if (file->dirty()) { + if (targets.size() == targets.capacity()) { + throw std::bad_alloc(); + } + targets.push_back({.medium = file, .generation = file->dirty_generation()}); + } + } + return targets; +} + +std::vector<int> BlockPageStore::dirty_fds() const +{ + std::vector<int> fds; + for (const auto &target : sync_targets()) { + fds.push_back(target.medium->fd()); + } + return fds; +} + +std::vector<int> BlockPageStore::all_extent_fds() const +{ + std::vector<int> fds; + if (!blocks_) { + if (auto *file = dynamic_cast<FileMedium *>(medium_.get())) { + fds.push_back(file->fd()); + } + return fds; + } + uint64_t cursor = 0; + uint32_t index = 0; + std::shared_ptr<FileMedium> file; + while (blocks_->next(&cursor, &index, &file)) { + fds.push_back(file->fd()); + } + return fds; +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/block_page_store_io.cpp b/lib/crowdb-tree/src/backend/local/block_page_store_io.cpp new file mode 100644 index 000000000..e7340372f --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/block_page_store_io.cpp @@ -0,0 +1,323 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "block_account.h" +#include "block_directory.h" +#include "btree/allocation_account.h" +#include "btree/tree_work_counters.h" +#include "crowdb-tree/backend/block_page_store.h" + +#include <algorithm> +#include <cstdlib> +#include <cstring> + +namespace crowdb::tree +{ +namespace +{ +// O_DIRECT's buffer-address alignment requirement is filesystem/device +// specific but 4096 (a common page/sector size) satisfies virtually every +// real device; safe to over-align a bit beyond iu_size_ when iu_size_ is +// smaller (e.g. a caller-supplied 512 on a regular file). +constexpr size_t kDirectBufAlign = 4096; + +size_t align_up(size_t v, size_t a) +{ + return (v + a - 1) / a * a; +} + +// RAII aligned scratch buffer (posix_memalign), used for the bounce path +// when a caller's offset/length/buffer isn't already O_DIRECT-aligned. +class AlignedBuffer +{ + public: + AlignedBuffer(size_t len, size_t align, std::shared_ptr<AllocationBudget> budget) : budget_(std::move(budget)) + { + size_t a = align > kDirectBufAlign ? align : kDirectBufAlign; + len_ = len == 0 ? a : len; + if (budget_ && !budget_->claim(len_)) { + throw std::bad_alloc(); + } + if (::posix_memalign(reinterpret_cast<void **>(&ptr_), a, len == 0 ? a : len) != 0) { + ptr_ = nullptr; + if (budget_) { + budget_->release(len_); + } + } + len_ = ptr_ != nullptr ? len_ : 0; + if (ptr_ != nullptr && budget_) { + budget_->record_allocation(AllocationKind::kIo, len_, true); + } + } + + ~AlignedBuffer() + { + std::free(ptr_); + if (ptr_ != nullptr && budget_) { + budget_->record_allocation(AllocationKind::kIo, len_, false); + budget_->release(len_); + } + } + + AlignedBuffer(const AlignedBuffer &) = delete; + AlignedBuffer &operator=(const AlignedBuffer &) = delete; + + [[nodiscard]] uint8_t *data() const + { + return ptr_; + } + + [[nodiscard]] bool ok() const + { + return ptr_ != nullptr; + } + + private: + std::shared_ptr<AllocationBudget> budget_; + uint8_t *ptr_ = nullptr; + size_t len_ = 0; +}; +} // namespace + +Status BlockPageStore::write_at_extents(uint64_t off, const uint8_t *buf, size_t len, + const std::shared_ptr<AllocationBudget> &budget) +{ + auto prepared = ensure_extents(off, len, budget); + if (!prepared.ok()) { + return prepared; + } + + // Split write across extent boundaries + size_t done = 0; + uint64_t cur = off; + while (done < len) { + uint64_t local = 0; + auto file = file_for_offset(cur, &local, budget); + if (!file) { + return Status::io_error("BlockPageStore: write extent unavailable"); + } + uint64_t avail = block_size_ - local; + size_t chunk = std::min(static_cast<uint64_t>(len - done), avail); + + // For IU=1, write directly. For IU>1, use the bounce path via medium. + Status s; + if (iu_size_ <= 1) { + s = file->pwrite_at(local, buf + done, chunk); + } + else { + // Use the same alignment logic as single-medium mode + bool aligned = (local % iu_size_ == 0) && (chunk % iu_size_ == 0) && + (reinterpret_cast<uintptr_t>(buf + done) % kDirectBufAlign == 0); + if (aligned) { + s = file->pwrite_at(local, buf + done, chunk); + } + else { + uint64_t start = (local / iu_size_) * iu_size_; + uint64_t e = align_up(local + chunk, iu_size_); + size_t span = e - start; + AlignedBuffer scratch(span, iu_size_, budget); + if (!scratch.ok()) { + return Status::io_error("BlockPageStore: aligned scratch allocation failed"); + } + size_t got = 0; + s = file->pread_partial(start, scratch.data(), span, &got); + if (s.ok()) { + if (got < span) { + std::memset(scratch.data() + got, 0, span - got); + } + std::memcpy(scratch.data() + (local - start), buf + done, chunk); + record_work(budget, TreeWorkCounter::BackendBounceCopyBytes, chunk); + s = file->pwrite_at(start, scratch.data(), span); + } + } + } + if (!s.ok()) { + return s; + } + done += chunk; + cur += chunk; + } + return Status::Ok(); +} + +Status BlockPageStore::read_at_extents(uint64_t off, uint8_t *buf, size_t len, + const std::shared_ptr<AllocationBudget> &budget) const +{ + size_t done = 0; + uint64_t cur = off; + while (done < len) { + uint64_t local = 0; + auto file = file_for_offset(cur, &local, budget); + if (!file) { + return Status::not_found("BlockPageStore: read extent unavailable"); + } + uint64_t avail = block_size_ - local; + size_t chunk = std::min(static_cast<uint64_t>(len - done), avail); + + Status s; + if (iu_size_ <= 1) { + s = file->pread_at(local, buf + done, chunk); + } + else { + bool aligned = (local % iu_size_ == 0) && (chunk % iu_size_ == 0) && + (reinterpret_cast<uintptr_t>(buf + done) % kDirectBufAlign == 0); + if (aligned) { + s = file->pread_at(local, buf + done, chunk); + } + else { + uint64_t start = (local / iu_size_) * iu_size_; + uint64_t e = align_up(local + chunk, iu_size_); + size_t span = e - start; + AlignedBuffer scratch(span, iu_size_, budget); + if (!scratch.ok()) { + return Status::io_error("BlockPageStore: aligned scratch allocation failed"); + } + size_t got = 0; + s = file->pread_partial(start, scratch.data(), span, &got); + if (s.ok()) { + if (got < (local - start) + chunk) { + return Status::not_found("BlockPageStore: read past end"); + } + std::memcpy(buf + done, scratch.data() + (local - start), chunk); + record_work(budget, TreeWorkCounter::BackendBounceCopyBytes, chunk); + } + } + } + if (!s.ok()) { + return s; + } + done += chunk; + cur += chunk; + } + return Status::Ok(); +} + +Status BlockPageStore::write_at(uint64_t off, const uint8_t *buf, size_t len) +{ + return write_at(off, buf, len, allocations_); +} + +Status BlockPageStore::write_at(uint64_t off, const uint8_t *buf, size_t len, + const std::shared_ptr<AllocationBudget> &budget) +try { + if (!budget) { + return write_at(off, buf, len); + } + if (!detail::valid_block_budget(*this, budget)) { + return Status::invalid_argument("backend allocation budget belongs to another store"); + } + if (len == 0) { + return Status::Ok(); + } + if (off > UINT64_MAX - len) { + return Status::invalid_argument("BlockPageStore: byte range overflow"); + } + + // Array-of-blocks mode + if (blocks_) { + return write_at_extents(off, buf, len, budget); + } + + // Single-medium mode + // IU=1 (byte-aligned): no alignment checks, no bounce buffer. + if (iu_size_ <= 1) { + return medium_->pwrite_at(off, buf, len); + } + + bool aligned = + (off % iu_size_ == 0) && (len % iu_size_ == 0) && (reinterpret_cast<uintptr_t>(buf) % kDirectBufAlign == 0); + if (aligned) { + return medium_->pwrite_at(off, buf, len); + } + + // Bounce through an IU-aligned scratch span covering [off, off+len): + // read-modify-write, since O_DIRECT can't write a sub-IU or misaligned + // range directly. + uint64_t start = (off / iu_size_) * iu_size_; + uint64_t end = align_up(off + len, iu_size_); + size_t span = end - start; + AlignedBuffer scratch(span, iu_size_, budget); + if (!scratch.ok()) { + return Status::io_error("BlockPageStore: aligned scratch allocation failed"); + } + size_t got = 0; + Status rs = medium_->pread_partial(start, scratch.data(), span, &got); + if (!rs.ok()) { + return rs; + } + if (got < span) { + // Genuinely new territory past the current end (growing the + // store) rather than a read failure -- zero-fill the unread tail. + std::memset(scratch.data() + got, 0, span - got); + } + std::memcpy(scratch.data() + (off - start), buf, len); + record_work(budget, TreeWorkCounter::BackendBounceCopyBytes, len); + return medium_->pwrite_at(start, scratch.data(), span); +} + +catch (const std::bad_alloc &) { + return Status::resource_exhausted("backend metadata allocation unavailable"); +} + +Status BlockPageStore::read_at(uint64_t off, uint8_t *buf, size_t len) const +{ + return read_at(off, buf, len, allocations_); +} + +Status BlockPageStore::read_at(uint64_t off, uint8_t *buf, size_t len, + const std::shared_ptr<AllocationBudget> &budget) const +try { + if (!budget) { + return read_at(off, buf, len); + } + if (!detail::valid_block_budget(*this, budget)) { + return Status::invalid_argument("backend allocation budget belongs to another store"); + } + if (len == 0) { + return Status::Ok(); + } + if (off > UINT64_MAX - len) { + return Status::invalid_argument("BlockPageStore: byte range overflow"); + } + + // Array-of-blocks mode + if (blocks_) { + return read_at_extents(off, buf, len, budget); + } + + // Single-medium mode + // IU=1 (byte-aligned): no alignment checks, no bounce buffer. + if (iu_size_ <= 1) { + return medium_->pread_at(off, buf, len); + } + + bool aligned = + (off % iu_size_ == 0) && (len % iu_size_ == 0) && (reinterpret_cast<uintptr_t>(buf) % kDirectBufAlign == 0); + if (aligned) { + return medium_->pread_at(off, buf, len); + } + + uint64_t start = (off / iu_size_) * iu_size_; + uint64_t end = align_up(off + len, iu_size_); + size_t span = end - start; + AlignedBuffer scratch(span, iu_size_, budget); + if (!scratch.ok()) { + return Status::io_error("BlockPageStore: aligned scratch allocation failed"); + } + size_t got = 0; + Status rs = medium_->pread_partial(start, scratch.data(), span, &got); + if (!rs.ok()) { + return rs; + } + if (got < (off - start) + len) { + return Status::not_found("BlockPageStore: read past end"); + } + std::memcpy(buf, scratch.data() + (off - start), len); + record_work(budget, TreeWorkCounter::BackendBounceCopyBytes, len); + return Status::Ok(); +} + +catch (const std::bad_alloc &) { + return Status::resource_exhausted("backend metadata allocation unavailable"); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/extent_ledger.cpp b/lib/crowdb-tree/src/backend/local/extent_ledger.cpp new file mode 100644 index 000000000..a881462ee --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/extent_ledger.cpp @@ -0,0 +1,224 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "crowdb-common/log.h" +#include "extent_ledger_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +size_t ExtentLedger::creation_metadata_bound() noexcept +{ + return detail::shared_allocation_bound<State>() + detail::shared_allocation_bound<ExtentLedger>() + (2 * 256); +} + +size_t ExtentLedger::reservation_metadata_bound() noexcept +{ + // Includes four red/black-tree nodes, their charged headers/alignment and + // separate record/control allocations on the supported standard libraries. + return detail::shared_allocation_bound<State::Request>() + detail::shared_allocation_bound<State::Record>() + + (4 * 256); +} + +ExtentLedger::State::State(ExtentLedgerLimits bounds, const std::shared_ptr<detail::AllocationReservation> &metadata) + : limits(bounds), + append(bounds.file_size), + address_high_water(bounds.file_size) +{ + detail::BudgetMemoryResource::Scope credit(resource, metadata); + if (append > limits.region_begin) { + ranges.emplace(limits.region_begin, Range{.length = append - limits.region_begin, .record = nullptr}); + free.emplace(append - limits.region_begin, limits.region_begin); + } +} + +ExtentLedger::State::~State() +{ + assert(protected_ranges.load() == 0 && incoming.load() == nullptr && excluded == nullptr); +} + +Status ExtentLedger::create(ExtentLedgerLimits limits, std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<ExtentLedger> *out) +{ + if (out == nullptr || limits.backend_id == 0 || limits.generation == 0 || limits.iu_size == 0 || + limits.region_begin % limits.iu_size != 0 || limits.file_size % limits.iu_size != 0 || + limits.file_size < limits.region_begin || limits.file_size > limits.address_limit || limits.max_ranges == 0 || + limits.queue_capacity == 0 || (metadata && metadata->category() == detail::BudgetClass::kIo)) { + return Status::invalid_argument(); + } + try { + auto state = detail::reserved_shared<State>(metadata, limits, metadata); + *out = detail::reserved_construct<ExtentLedger>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) ExtentLedger(std::move(state)); }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +Status ExtentLedger::recover(uint64_t address, uint64_t length, std::shared_ptr<detail::AllocationReservation> metadata, + Complete complete) +{ + return state_->submit(State::Action::Recover, address, length, std::move(metadata), std::move(complete)); +} + +Status ExtentLedger::activate(std::shared_ptr<detail::AllocationReservation> metadata, Complete complete) +{ + return state_->submit(State::Action::Activate, 0, 0, std::move(metadata), std::move(complete)); +} + +Status ExtentLedger::reserve(uint64_t length, std::shared_ptr<detail::AllocationReservation> metadata, + Complete complete) +{ + return state_->submit(State::Action::Reserve, 0, length, std::move(metadata), std::move(complete)); +} + +Status ExtentLedger::retain(uint64_t address, uint64_t length, std::shared_ptr<detail::AllocationReservation> metadata, + Complete complete) +{ + return state_->submit(State::Action::Retain, address, length, std::move(metadata), std::move(complete)); +} + +Status ExtentLedger::claim_free(uint64_t address, uint64_t length, + std::shared_ptr<detail::AllocationReservation> metadata, Complete complete) +{ + return state_->submit(State::Action::ClaimFree, address, length, std::move(metadata), std::move(complete)); +} + +Status ExtentLedger::exclude_allocations(uint64_t address, uint64_t length, + std::shared_ptr<detail::AllocationReservation> metadata, Complete complete) +{ + return state_->submit(State::Action::Exclude, address, length, std::move(metadata), std::move(complete)); +} + +ExtentLedgerStats ExtentLedger::stats() const noexcept +{ + return {.address_high_water = state_->address_high_water.load(std::memory_order_acquire), + .protected_bytes = state_->protected_bytes.load(std::memory_order_relaxed), + .protected_ranges = state_->protected_ranges.load(std::memory_order_relaxed), + .reused_allocations = state_->reused_allocations.load(std::memory_order_relaxed), + .appended_allocations = state_->appended_allocations.load(std::memory_order_relaxed)}; +} + +const ExtentLedgerLimits &ExtentLedger::limits() const noexcept +{ + return state_->limits; +} + +void ExtentLedger::close() noexcept +{ + state_->closed.store(true, std::memory_order_release); +} + +Status ExtentLedger::State::submit(Action action, uint64_t address, uint64_t length, + std::shared_ptr<detail::AllocationReservation> metadata, Complete complete) +{ + if (!complete || (metadata && metadata->category() == detail::BudgetClass::kIo)) { + return Status::invalid_argument(); + } + if (closed.load(std::memory_order_acquire)) { + return Status::unavailable(); + } + std::shared_ptr<Request> request; + try { + request = detail::reserved_shared<Request>(metadata); + request->command = {.action = action, .context = request.get()}; + request->address = address; + request->length = length; + request->metadata = std::move(metadata); + request->complete = std::move(complete); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + auto count = queued.load(std::memory_order_relaxed); + do { + if (count >= limits.queue_capacity) { + return Status::resource_exhausted(); + } + } while (!queued.compare_exchange_weak(count, count + 1, std::memory_order_acq_rel)); + request->ledger = shared_from_this(); + request->queued_owner = request; + enqueue(&request->command); + return Status::Ok(); +} + +void ExtentLedger::State::enqueue(Command *command) noexcept +{ + auto self = weak_from_this().lock(); + assert(self); + auto *head = incoming.load(std::memory_order_relaxed); + do { + command->next = head; + } while (!incoming.compare_exchange_weak(head, command, std::memory_order_release, std::memory_order_relaxed)); + if (!running.exchange(true, std::memory_order_acq_rel)) { + drain(); + } +} + +void ExtentLedger::State::drain() noexcept +{ + for (;;) { + auto *list = incoming.exchange(nullptr, std::memory_order_acquire); + Command *ordered = nullptr; + while (list != nullptr) { + auto *next = list->next; + list->next = ordered; + ordered = list; + list = next; + } + while (ordered != nullptr) { + auto *next = ordered->next; + if (ordered->action == Action::Release) { + release(static_cast<Record *>(ordered->context)); + } + else { + process(static_cast<Request *>(ordered->context)); + } + ordered = next; + } + (void)running.exchange(false, std::memory_order_acq_rel); + if (incoming.load(std::memory_order_acquire) == nullptr || running.exchange(true, std::memory_order_acq_rel)) { + return; + } + } +} + +void ExtentLedger::State::process(Request *request) noexcept +{ + auto owner = std::move(request->queued_owner); + ExtentLedgerResult result; + try { + detail::BudgetMemoryResource::Scope credit(resource, request->metadata); + result = apply(*request); + } + catch (const std::bad_alloc &) { + result.status = Status::resource_exhausted(); + } + catch (...) { + result.status = Status::internal_error(); + } + queued.fetch_sub(1, std::memory_order_release); + try { + request->complete(std::move(result)); + } + catch (...) { + CRB_LOG_ERROR("extent receiver threw; remaining allocation/release commands will continue"); + } +} + +ProtectedExtent ExtentLedger::State::describe(Record &record, std::shared_ptr<const void> owner) +{ + auto metrics = record.exclusive ? nullptr : std::shared_ptr<BackingMetricExtent>(owner, &record.metrics); + return {.backend_id = record.ledger->limits.backend_id, + .generation = record.ledger->limits.generation, + .identity = record.identity, + .address = record.address, + .length = record.length, + .owner = std::move(owner), + .physical_metrics = std::move(metrics)}; +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/extent_ledger.h b/lib/crowdb-tree/src/backend/local/extent_ledger.h new file mode 100644 index 000000000..180f0c383 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/extent_ledger.h @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_reservation.h" +#include "btree/inline_function.h" +#include "crowdb-tree/status.h" + +namespace crowdb::tree +{ +class BackingMetricExtent; + +struct ExtentLedgerLimits +{ + uint64_t backend_id = 0; + uint64_t generation = 1; + uint64_t region_begin = 8192; + uint64_t file_size = 8192; + uint64_t address_limit = UINT64_MAX; + uint32_t iu_size = 4096; + size_t max_ranges = 1024 * 1024; + size_t queue_capacity = 64; +}; + +struct ProtectedExtent +{ + uint64_t backend_id = 0; + uint64_t generation = 0; + uint64_t identity = 0; + uint64_t address = 0; + uint64_t length = 0; + // Share this owner through in-flight I/O, backing, snapshots and readers. + // Its final release, rather than a checkpoint's current live map, permits reuse. + std::shared_ptr<const void> owner; + std::weak_ptr<BackingMetricExtent> physical_metrics; +}; + +struct ExtentLedgerResult +{ + Status status; + ProtectedExtent extent; + uint64_t occupied_bytes = 0; +}; + +struct ExtentLedgerStats +{ + uint64_t address_high_water = 0; + uint64_t protected_bytes = 0; + uint64_t protected_ranges = 0; + uint64_t reused_allocations = 0; + uint64_t appended_allocations = 0; +}; + +// A shared byte allocation domain. Recovery first protects every retained +// anchor reference, then activates allocation. Short metadata commands use +// bounded MPSC admission; callbacks may run inline and must not block. +class ExtentLedger +{ + struct State; + + public: + using Complete = detail::InlineFunction<void(ExtentLedgerResult)>; + static Status create(ExtentLedgerLimits limits, std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<ExtentLedger> *out); + [[nodiscard]] static size_t creation_metadata_bound() noexcept; + [[nodiscard]] static size_t reservation_metadata_bound() noexcept; + // Submission failure does not invoke complete. Accepted commands invoke it once. + Status recover(uint64_t address, uint64_t length, std::shared_ptr<detail::AllocationReservation> metadata, + Complete complete); + Status activate(std::shared_ptr<detail::AllocationReservation> metadata, Complete complete); + Status reserve(uint64_t length, std::shared_ptr<detail::AllocationReservation> metadata, Complete complete); + Status retain(uint64_t address, uint64_t length, std::shared_ptr<detail::AllocationReservation> metadata, + Complete complete); + // Exclusively protect an already free byte range during physical deletion. + // Never borrows a data owner or extends the allocation frontier. Retain + // rejects this temporary lease so it cannot become a readable backing. + Status claim_free(uint64_t address, uint64_t length, std::shared_ptr<detail::AllocationReservation> metadata, + Complete complete); + // Freeze allocation into one existing byte interval while compaction moves + // its live references elsewhere. Existing data owners remain readable. + // The returned owner is an allocation exclusion, never readable backing. + // occupied_bytes counts protected bytes at admission. One exclusion per + // ledger and at most 4096 range visits keep metadata work bounded. + Status exclude_allocations(uint64_t address, uint64_t length, + std::shared_ptr<detail::AllocationReservation> metadata, Complete complete); + [[nodiscard]] ExtentLedgerStats stats() const noexcept; + [[nodiscard]] const ExtentLedgerLimits &limits() const noexcept; + void close() noexcept; + + private: + explicit ExtentLedger(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/extent_ledger_allocate.cpp b/lib/crowdb-tree/src/backend/local/extent_ledger_allocate.cpp new file mode 100644 index 000000000..265630944 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/extent_ledger_allocate.cpp @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "extent_ledger_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +ExtentLedger::State::PreparedNodes ExtentLedger::State::prepare_nodes() +{ + PreparedNodes nodes; + Ranges spare_ranges(&resource); + FreeRanges spare_free(&resource); + for (size_t i = 0; i < nodes.ranges.size(); ++i) { + spare_ranges.emplace(0, Range{.length = 0, .record = nullptr}); + nodes.ranges[i] = spare_ranges.extract(spare_ranges.begin()); + spare_free.emplace(0, 0); + nodes.free[i] = spare_free.extract(spare_free.begin()); + } + return nodes; +} + +ExtentLedgerResult ExtentLedger::State::allocate(Request &request, Ranges::iterator gap, uint64_t address) +{ + const bool reuse = gap != ranges.end(); + const auto old_begin = reuse ? gap->first : address; + const auto old_end = reuse ? old_begin + gap->second.length : address + request.length; + const auto end = address + request.length; + const size_t extra = static_cast<size_t>(old_begin < address) + static_cast<size_t>(end < old_end); + if (next_identity == UINT64_MAX || extra + static_cast<size_t>(!reuse) > limits.max_ranges - ranges.size()) { + return {.status = Status::resource_exhausted(), .extent = {}}; + } + // Every node needed for splitting now or returning this extent later is + // paid before changing either index. Final owner release never allocates. + auto nodes = prepare_nodes(); + auto allocated = detail::reserved_unique<Record>(request.metadata); + allocated->metadata = request.metadata; + allocated->ledger = shared_from_this(); + auto *record = allocated.release(); + auto owner = std::shared_ptr<Record>( + record, + [](Record *released) { + auto ledger = released->ledger; + ledger->enqueue(&released->command); + }, + detail::ReservationAllocator<Record>(request.metadata)); + record->exclusive = request.command.action == Action::ClaimFree; + record->owner = owner; + record->identity = next_identity; + record->address = address; + record->length = request.length; + + Ranges::node_type current; + if (reuse) { + record->free_node = free.extract({gap->second.length, gap->first}); + assert(!record->free_node.empty()); + current = ranges.extract(gap); + size_t index = 0; + auto remainder = [&](uint64_t begin, uint64_t length) { + auto range = std::move(nodes.ranges[index]); + range.key() = begin; + range.mapped() = {.length = length, .record = nullptr}; + ranges.insert(std::move(range)); + auto available = std::move(nodes.free[index++]); + available.value() = {length, begin}; + free.insert(std::move(available)); + }; + if (old_begin < address) { + remainder(old_begin, address - old_begin); + } + if (end < old_end) { + remainder(end, old_end - end); + } + } + else { + current = std::move(nodes.ranges[0]); + record->free_node = std::move(nodes.free[0]); + append = end; + } + current.key() = address; + current.mapped() = {.length = request.length, .record = record}; + ranges.insert(std::move(current)); + record->installed = true; + if (!record->exclusive) { + record->metrics.activate(request.metadata ? request.metadata->resource_metrics() : nullptr, request.length); + } + address_high_water.store(append, std::memory_order_release); + ++next_identity; + protected_bytes.fetch_add(request.length, std::memory_order_relaxed); + protected_ranges.fetch_add(1, std::memory_order_relaxed); + if (request.command.action == Action::Reserve) { + (reuse ? reused_allocations : appended_allocations).fetch_add(1, std::memory_order_relaxed); + } + return {.status = Status::Ok(), .extent = describe(*record, std::move(owner))}; +} + +void ExtentLedger::State::release(Record *record) noexcept +{ + if (record == excluded) { + excluded = nullptr; + } + if (record->installed) { + auto at = ranges.find(record->address); + assert(at != ranges.end() && at->second.record == record); + auto node = ranges.extract(at); + auto begin = record->address; + auto end = begin + record->length; + auto next = ranges.lower_bound(begin); + if (next != ranges.end() && next->first == end && next->second.record == nullptr) { + end += next->second.length; + free.erase({next->second.length, next->first}); + ranges.erase(next); + } + next = ranges.lower_bound(begin); + if (next != ranges.begin()) { + auto previous = std::prev(next); + if (previous->second.record == nullptr && previous->first + previous->second.length == begin) { + begin = previous->first; + free.erase({previous->second.length, previous->first}); + ranges.erase(previous); + } + } + node.key() = begin; + node.mapped() = {.length = end - begin, .record = nullptr}; + ranges.insert(std::move(node)); + record->free_node.value() = {end - begin, begin}; + free.insert(std::move(record->free_node)); + protected_bytes.fetch_sub(record->length, std::memory_order_relaxed); + protected_ranges.fetch_sub(1, std::memory_order_relaxed); + } + detail::ReservationAllocator<Record> allocator(record->metadata); + std::destroy_at(record); + allocator.deallocate(record, 1); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/extent_ledger_apply.cpp b/lib/crowdb-tree/src/backend/local/extent_ledger_apply.cpp new file mode 100644 index 000000000..c572268b7 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/extent_ledger_apply.cpp @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "extent_ledger_internal.h" + +namespace crowdb::tree +{ +ExtentLedgerResult ExtentLedger::State::apply(Request &request) +{ + if (request.command.action == Action::Activate) { + active = true; + return {}; + } + if (request.length == 0 || request.length % limits.iu_size != 0 || request.address % limits.iu_size != 0 || + request.address > UINT64_MAX - request.length) { + return {.status = Status::invalid_argument(), .extent = {}}; + } + if (request.command.action == Action::Exclude) { + return exclude(request); + } + const bool claim = request.command.action == Action::ClaimFree; + if (claim && !active) { + return {.status = Status::unavailable(), .extent = {}}; + } + if (request.command.action == Action::Reserve) { + if (!active) { + return {.status = Status::unavailable(), .extent = {}}; + } + if (excluded != nullptr) { + return reserve_outside_exclusion(request); + } + const auto gap = free.lower_bound({request.length, 0}); + if (gap != free.end()) { + return allocate(request, ranges.find(gap->second), gap->second); + } + if (request.length > limits.address_limit - append) { + return {.status = Status::resource_exhausted(), .extent = {}}; + } + return allocate(request, ranges.end(), append); + } + if (request.address < limits.region_begin || request.address + request.length > append) { + return {.status = claim ? Status::not_found() : Status::invalid_argument(), .extent = {}}; + } + if (request.command.action == Action::Recover && active) { + return {.status = Status::unavailable(), .extent = {}}; + } + auto at = ranges.upper_bound(request.address); + if (at == ranges.begin()) { + return {.status = Status::not_found(), .extent = {}}; + } + --at; + if (request.address + request.length > at->first + at->second.length) { + return {.status = claim ? Status::not_found() : Status::corruption(), .extent = {}}; + } + if (at->second.record != nullptr) { + auto &record = *at->second.record; + if (claim || record.exclusive) { + return {.status = Status::not_found(), .extent = {}}; + } + if (request.command.action == Action::Recover && + (request.address != record.address || request.length != record.length)) { + return {.status = Status::corruption(), .extent = {}}; + } + auto owner = record.owner.lock(); + if (!owner) { + return {.status = Status::unavailable(), .extent = {}}; + } + return {.status = Status::Ok(), .extent = describe(record, std::move(owner))}; + } + if (request.command.action == Action::Retain) { + return {.status = Status::not_found(), .extent = {}}; + } + return allocate(request, at, request.address); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/extent_ledger_exclusion.cpp b/lib/crowdb-tree/src/backend/local/extent_ledger_exclusion.cpp new file mode 100644 index 000000000..5f9df86e9 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/extent_ledger_exclusion.cpp @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "extent_ledger_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +ExtentLedgerResult ExtentLedger::State::exclude(Request &request) +{ + if (!active || excluded != nullptr) { + return {.status = Status::unavailable(), .extent = {}}; + } + if (request.address < limits.region_begin || request.address + request.length > append) { + return {.status = Status::invalid_argument(), .extent = {}}; + } + if (next_identity == UINT64_MAX) { + return {.status = Status::resource_exhausted(), .extent = {}}; + } + const auto end = request.address + request.length; + auto next = ranges.upper_bound(request.address); + if (next != ranges.begin()) { + --next; + } + uint64_t occupied = 0; + size_t visits = 0; + for (; next != ranges.end() && next->first < end; ++next) { + if (++visits > 4096) { + return {.status = Status::resource_exhausted(), .extent = {}}; + } + const auto overlap_begin = std::max(next->first, request.address); + const auto overlap_end = std::min(next->first + next->second.length, end); + if (next->second.record != nullptr && overlap_begin < overlap_end) { + occupied += overlap_end - overlap_begin; + } + } + auto prepared = detail::reserved_unique<Record>(request.metadata); + prepared->metadata = request.metadata; + prepared->ledger = shared_from_this(); + auto *record = prepared.release(); + auto owner = std::shared_ptr<Record>( + record, + [](Record *released) { + auto ledger = released->ledger; + ledger->enqueue(&released->command); + }, + detail::ReservationAllocator<Record>(request.metadata)); + record->identity = next_identity++; + record->address = request.address; + record->length = request.length; + record->exclusive = true; + record->owner = owner; + excluded = record; + return {.status = Status::Ok(), .extent = describe(*record, std::move(owner)), .occupied_bytes = occupied}; +} + +ExtentLedgerResult ExtentLedger::State::reserve_outside_exclusion(Request &request) +{ + const auto avoid_begin = excluded->address; + const auto avoid_end = avoid_begin + excluded->length; + size_t visits = 0; + for (auto gap = free.lower_bound({request.length, 0}); gap != free.end() && visits < 64; ++gap, ++visits) { + const auto begin = gap->second; + const auto end = begin + gap->first; + if (begin >= avoid_end || (begin < avoid_begin && request.length <= avoid_begin - begin)) { + return allocate(request, ranges.find(begin), begin); + } + if (end > avoid_end && request.length <= end - avoid_end) { + return allocate(request, ranges.find(begin), avoid_end); + } + } + // Exclusions cover existing storage only, so append is always outside. + if (request.length > limits.address_limit - append) { + return {.status = Status::resource_exhausted(), .extent = {}}; + } + return allocate(request, ranges.end(), append); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/extent_ledger_internal.h b/lib/crowdb-tree/src/backend/local/extent_ledger_internal.h new file mode 100644 index 000000000..aa55c90e9 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/extent_ledger_internal.h @@ -0,0 +1,114 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/backing_metrics.h" +#include "btree/budget_memory_resource.h" +#include "extent_ledger.h" + +#include <array> +#include <map> +#include <set> + +namespace crowdb::tree +{ +struct ExtentLedger::State : std::enable_shared_from_this<State> +{ + enum class Action : uint8_t { Recover, Activate, Reserve, Retain, ClaimFree, Exclude, Release }; + struct Record; + + struct Range + { + uint64_t length; + Record *record; + }; + + using Ranges = std::pmr::map<uint64_t, Range>; + using FreeRanges = std::pmr::set<std::pair<uint64_t, uint64_t>>; // length, address + + struct Command + { + Action action; + void *context; + Command *next = nullptr; + }; + + struct Request + { + Command command; + uint64_t address; + uint64_t length; + std::shared_ptr<detail::AllocationReservation> metadata; + Complete complete; + std::shared_ptr<Request> queued_owner; + std::shared_ptr<State> ledger; + }; + + struct Record + { + BackingMetricExtent metrics; + Command command{.action = Action::Release, .context = this}; + std::shared_ptr<State> ledger; + std::shared_ptr<detail::AllocationReservation> metadata; + std::weak_ptr<const void> owner; + FreeRanges::node_type free_node; + uint64_t identity = 0; + uint64_t address = 0; + uint64_t length = 0; + bool installed = false; + bool exclusive = false; + }; + + struct PreparedNodes + { + std::array<Ranges::node_type, 2> ranges; + std::array<FreeRanges::node_type, 2> free; + }; + + State(ExtentLedgerLimits bounds, const std::shared_ptr<detail::AllocationReservation> &metadata); + ~State(); + + bool retire_step() noexcept + { + size_t steps = 0; + for (; steps < 64 && !ranges.empty(); ++steps) { + ranges.erase(ranges.begin()); + } + for (; steps < 64 && !free.empty(); ++steps) { + free.erase(free.begin()); + } + return ranges.empty() && free.empty(); + } + + Status submit(Action action, uint64_t address, uint64_t length, + std::shared_ptr<detail::AllocationReservation> metadata, Complete complete); + void enqueue(Command *command) noexcept; + void drain() noexcept; + void process(Request *request) noexcept; + ExtentLedgerResult apply(Request &request); + ExtentLedgerResult allocate(Request &request, Ranges::iterator gap, uint64_t address); + ExtentLedgerResult exclude(Request &request); + ExtentLedgerResult reserve_outside_exclusion(Request &request); + PreparedNodes prepare_nodes(); + void release(Record *record) noexcept; + static ProtectedExtent describe(Record &record, std::shared_ptr<const void> owner); + + const ExtentLedgerLimits limits; + detail::BudgetMemoryResource resource; + Ranges ranges{&resource}; + FreeRanges free{&resource}; + uint64_t append; + uint64_t next_identity = 1; + Record *excluded = nullptr; + bool active = false; + std::atomic<Command *> incoming{nullptr}; + std::atomic<bool> running{false}; + std::atomic<bool> closed{false}; + std::atomic<size_t> queued{0}; + std::atomic<uint64_t> address_high_water; + std::atomic<uint64_t> protected_bytes{0}; + std::atomic<uint64_t> protected_ranges{0}; + std::atomic<uint64_t> reused_allocations{0}; + std::atomic<uint64_t> appended_allocations{0}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/extent_writeback.cpp b/lib/crowdb-tree/src/backend/local/extent_writeback.cpp new file mode 100644 index 000000000..74b7b09b9 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/extent_writeback.cpp @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "extent_writeback.h" + +namespace crowdb::tree +{ +PageWriteOperation::ReserveExtent extent_write_reservation(std::shared_ptr<ExtentLedger> ledger, + std::shared_ptr<detail::AllocationReservation> metadata) +{ + return [ledger = std::move(ledger), metadata = std::move(metadata)](uint64_t length, + PageWriteOperation::ExtentComplete complete) { + auto rejected = complete; + if (!ledger) { + complete(Status::invalid_argument(), {}); + return; + } + auto status = ledger->reserve(length, metadata, [complete = std::move(complete)](ExtentLedgerResult result) { + auto &extent = result.extent; + complete(std::move(result.status), {.backend_id = extent.backend_id, + .address = extent.address, + .length = extent.length, + .content_id = extent.identity, + .generation = extent.generation, + .owner = std::move(extent.owner), + .physical_metrics = std::move(extent.physical_metrics), + .chunk = {}}); + }); + if (!status.ok()) { + rejected(std::move(status), {}); + } + }; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/extent_writeback.h b/lib/crowdb-tree/src/backend/local/extent_writeback.h new file mode 100644 index 000000000..291315ec7 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/extent_writeback.h @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_writeback.h" +#include "extent_ledger.h" + +namespace crowdb::tree +{ +// Allocation is an asynchronous metadata step; a Tree CPU worker never waits +// for another ledger command or reserves an uncompressed worst-case disk span. +PageWriteOperation::ReserveExtent extent_write_reservation(std::shared_ptr<ExtentLedger> ledger, + std::shared_ptr<detail::AllocationReservation> metadata); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/local_backend_state.cpp b/lib/crowdb-tree/src/backend/local/local_backend_state.cpp new file mode 100644 index 000000000..dae4711ea --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/local_backend_state.cpp @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/local_backend_state.h" + +#include "crowdb-tree/backend/page_store.h" + +namespace crowdb::tree +{ +Status LocalBackendDomain::claim(PageStore *store, std::shared_ptr<const void> backend_owner, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<LocalBackendDomain> *out) +try { + if (store == nullptr || out == nullptr || *out) { + return Status::invalid_argument("backend domain requires a store and empty output"); + } + auto domain = detail::reserved_construct<LocalBackendDomain>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) LocalBackendDomain(std::move(backend_owner)); + }); + if (!store->try_claim_runtime_domain()) { + return Status::unavailable("local backend already has a runtime allocation authority"); + } + domain->store_ = store; + *out = std::move(domain); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("backend domain allocation failed"); +} + +LocalBackendDomain::~LocalBackendDomain() +{ + if (store_ != nullptr) { + store_->release_runtime_domain(); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/local_backend_state.h b/lib/crowdb-tree/src/backend/local/local_backend_state.h new file mode 100644 index 000000000..e661e2829 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/local_backend_state.h @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/local_checkpoint.h" + +namespace crowdb::tree +{ +class PageStore; + +// The claim outlives registry/Core owners when a detached snapshot still +// reads protected extents. The backend owner is released after the claim. +class LocalBackendDomain +{ + public: + static Status claim(PageStore *store, std::shared_ptr<const void> backend_owner, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<LocalBackendDomain> *out); + ~LocalBackendDomain(); + LocalBackendDomain(const LocalBackendDomain &) = delete; + LocalBackendDomain &operator=(const LocalBackendDomain &) = delete; + + private: + explicit LocalBackendDomain(std::shared_ptr<const void> owner) : backend_owner_(std::move(owner)) + { + } + + std::shared_ptr<const void> backend_owner_; + PageStore *store_ = nullptr; +}; + +// One allocation authority for a local Tree lifetime. The registry retains +// both anchors and an uncertain candidate; live backing and I/O owners retain +// their own extents independently. Legacy allocation must not use this domain. +struct LocalBackendState +{ + LocalBackendState() = default; + ~LocalBackendState() = default; + LocalBackendState(const LocalBackendState &) = delete; + LocalBackendState &operator=(const LocalBackendState &) = delete; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<ExtentLedger> ledger; + std::shared_ptr<LocalCheckpointRegistry> checkpoints; + // Startup-only recovered descriptors, consumed while sizing Core loaders. + // Live primary backings retain their exact typed overflow dependencies. + std::shared_ptr<const CapturedDurablePages> inherited; + // Set only after claiming the PageStore runtime allocation domain. + std::shared_ptr<LocalBackendDomain> domain; + std::atomic<uint64_t> gc_cursor{0}; + std::atomic<bool> gc_active{false}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/local_block_gc.cpp b/lib/crowdb-tree/src/backend/local/local_block_gc.cpp new file mode 100644 index 000000000..e11250d82 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/local_block_gc.cpp @@ -0,0 +1,106 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_block_gc.h" + +#include "btree/allocation_bound.h" +#include "btree/tree_work_counters.h" +#include "host_completion.h" + +namespace crowdb::tree +{ +namespace +{ +ExtentLedgerResult claim_block(ExtentLedger &ledger, uint64_t begin, uint64_t bytes, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + ExtentLedgerResult result; + detail::HostCompletion done; + auto status = ledger.claim_free(begin, bytes, metadata, [&](ExtentLedgerResult value) { + result = std::move(value); + done.signal(); + }); + if (!status.ok()) { + return {.status = std::move(status), .extent = {}}; + } + done.wait(); + return result; +} +} // namespace + +std::optional<size_t> local_block_gc_metadata_bound(LocalBlockGcLimits limits) noexcept +{ + if (limits.max_blocks == 0 || limits.max_deleted_bytes == 0) { + return std::nullopt; + } + return detail::AllocationBound().add_product(limits.max_blocks, ExtentLedger::reservation_metadata_bound()).value(); +} + +uint64_t local_block_gc_unit(const PageStore &store) noexcept +{ + if (const auto *block = dynamic_cast<const BlockPageStore *>(&store)) { + return block->block_size(); + } + return dynamic_cast<const MemPageStore *>(&store) != nullptr ? MemPageStore::kBackingPageBytes : 0; +} + +LocalBlockGcResult collect_free_local_blocks(PageStore &store, ExtentLedger &ledger, uint64_t first_block, + LocalBlockGcLimits limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, + const std::shared_ptr<AllocationBudget> &backend) +{ + LocalBlockGcResult result; + const auto bytes = local_block_gc_unit(store); + if (!local_block_gc_metadata_bound(limits) || bytes == 0 || bytes % ledger.limits().iu_size != 0) { + result.status = Status::invalid_argument("invalid local GC geometry or limits"); + return result; + } + if (bytes > limits.max_deleted_bytes) { + result.status = Status::resource_exhausted("GC deletion budget cannot hold one block"); + return result; + } + auto *memory = dynamic_cast<MemPageStore *>(&store); + auto *block = dynamic_cast<BlockPageStore *>(&store); + const auto begin = ledger.limits().region_begin; + const auto first = (begin / bytes) + static_cast<uint64_t>(begin % bytes != 0); + const auto end = + std::min(ledger.stats().address_high_water / bytes, memory != nullptr ? UINT64_MAX : uint64_t{UINT32_MAX} + 1); + result.next_block = std::max(first_block, first); + while (result.next_block < end && result.inspected < limits.max_blocks && + bytes <= limits.max_deleted_bytes - result.deleted_bytes) { + const auto index = result.next_block; + auto lease = claim_block(ledger, index * bytes, bytes, metadata); + ++result.inspected; + if (lease.status.code() == Code::kNotFound) { + ++result.protected_blocks; + ++result.next_block; + continue; + } + if (!lease.status.ok()) { + result.status = std::move(lease.status); + return result; + } + bool deleted = false; + if (memory != nullptr) { + deleted = memory->release_backing_page(index); + } + else { + uint64_t offset = 0; + if (block->file_for_offset(index * bytes, &offset, backend)) { + result.status = block->delete_block(static_cast<uint32_t>(index), backend); + deleted = result.status.ok(); + } + if (!result.status.ok()) { + return result; + } + } + if (deleted) { + ++result.deleted_blocks; + result.deleted_bytes += bytes; + record_work(metadata, TreeWorkCounter::BackingDeletedBytes, bytes, TreeWorkCause::GarbageCollection); + } + ++result.next_block; + } + result.complete = result.next_block >= end; + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/local_block_gc.h b/lib/crowdb-tree/src/backend/local/local_block_gc.h new file mode 100644 index 000000000..88d173de1 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/local_block_gc.h @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/backend/block_page_store.h" +#include "extent_ledger.h" + +namespace crowdb::tree +{ +struct LocalBlockGcLimits +{ + size_t max_blocks = 64; + uint64_t max_deleted_bytes = 64 * 1024 * 1024; +}; + +struct LocalBlockGcResult +{ + Status status; + uint64_t next_block = 0; + size_t inspected = 0; + size_t protected_blocks = 0; + size_t deleted_blocks = 0; + uint64_t deleted_bytes = 0; + bool complete = false; +}; + +// One synchronous host-side pass. Every deletion holds an exclusive ledger +// lease, covering retained anchors, readers, staged frames and pending writes. +// The caller pre-admits metadata for the bounded pass. Partial progress remains +// valid on failure; resumption starts at next_block without skipping a failure. +[[nodiscard]] std::optional<size_t> local_block_gc_metadata_bound(LocalBlockGcLimits limits) noexcept; +// Memory backing pages are reclaimable without enabling legacy block compaction. +[[nodiscard]] uint64_t local_block_gc_unit(const PageStore &store) noexcept; +LocalBlockGcResult collect_free_local_blocks(PageStore &store, ExtentLedger &ledger, uint64_t first_block, + LocalBlockGcLimits limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, + const std::shared_ptr<AllocationBudget> &backend = {}); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/local_sparse_gc.cpp b/lib/crowdb-tree/src/backend/local/local_sparse_gc.cpp new file mode 100644 index 000000000..2e8fae4f0 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/local_sparse_gc.cpp @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_sparse_gc.h" + +#include "host_completion.h" + +#include <algorithm> +#include <cmath> + +namespace crowdb::tree +{ +LocalSparseBlock select_sparse_local_block(BlockPageStore &store, ExtentLedger &ledger, uint64_t first_block, + const std::shared_ptr<detail::AllocationReservation> &metadata, + double free_threshold, const std::shared_ptr<AllocationBudget> &backend) +{ + LocalSparseBlock result; + const auto bytes = store.block_size(); + if (free_threshold <= 0.0 || free_threshold >= 1.0 || std::isnan(free_threshold)) { + result.status = Status::invalid_argument(); + return result; + } + if (bytes == 0 || bytes > 64 * 1024 * 1024 || bytes % ledger.limits().iu_size != 0) { + result.status = Status::not_supported(); + return result; + } + const auto begin = ledger.limits().region_begin; + const auto first = (begin / bytes) + static_cast<uint64_t>(begin % bytes != 0); + const auto end = std::min(ledger.stats().address_high_water / bytes, uint64_t{UINT32_MAX} + 1); + auto next = std::max(first_block, first); + for (size_t visits = 0; next < end && visits < 8; ++next, ++visits) { + result.next_block = next + 1 < end ? next + 1 : 0; + uint64_t offset = 0; + if (!store.file_for_offset(next * bytes, &offset, backend)) { + continue; + } + ExtentLedgerResult fence; + detail::HostCompletion done; + result.status = ledger.exclude_allocations(next * bytes, bytes, metadata, [&](ExtentLedgerResult value) { + fence = std::move(value); + done.signal(); + }); + if (!result.status.ok()) { + return result; + } + done.wait(); + if (!fence.status.ok()) { + result.status = std::move(fence.status); + return result; + } + if (fence.occupied_bytes != 0 && + static_cast<long double>(bytes - fence.occupied_bytes) / bytes > free_threshold) { + result.exclusion = std::move(fence.extent); + return result; + } + } + result.status = Status::not_found(); + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/local_sparse_gc.h b/lib/crowdb-tree/src/backend/local/local_sparse_gc.h new file mode 100644 index 000000000..f77d900a9 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/local_sparse_gc.h @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "local_block_gc.h" + +namespace crowdb::tree +{ +struct LocalSparseBlock +{ + Status status; + ProtectedExtent exclusion; + uint64_t next_block = 0; +}; + +// Backend lane only: inspect at most eight existing blocks and fence the first +// whose free fraction exceeds the selected threshold. The exclusion does not +// replace readable leases and must survive both anchor rotations and deletion. +LocalSparseBlock select_sparse_local_block(BlockPageStore &store, ExtentLedger &ledger, uint64_t first_block, + const std::shared_ptr<detail::AllocationReservation> &metadata, + double free_threshold = 0.5, + const std::shared_ptr<AllocationBudget> &backend = {}); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/memory_backing_test.h b/lib/crowdb-tree/src/backend/local/memory_backing_test.h new file mode 100644 index 000000000..deb2b90fe --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/memory_backing_test.h @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#ifdef CROWDB_TREE_TEST_UTIL +# include <cstddef> +# include <optional> + +namespace crowdb::tree +{ +// Synchronous backend allocation fault, scoped to the calling thread. +void set_memory_backing_failure_after_for_tests(std::optional<size_t> pages) noexcept; +} // namespace crowdb::tree +#endif diff --git a/lib/crowdb-tree/src/backend/local/page_store.cpp b/lib/crowdb-tree/src/backend/local/page_store.cpp index 74b6d0f8b..22c51213f 100644 --- a/lib/crowdb-tree/src/backend/local/page_store.cpp +++ b/lib/crowdb-tree/src/backend/local/page_store.cpp @@ -5,45 +5,157 @@ #include "crowdb-tree/backend/page_store.h" -#include <fcntl.h> -#include <unistd.h> +#include "memory_backing_test.h" -#include <cerrno> +#include <algorithm> +#include <array> #include <cstring> - -#ifdef __APPLE__ -# include <sys/stat.h> -#endif +#include <limits> +#include <new> +#include <unordered_map> namespace crowdb::tree { +#ifdef CROWDB_TREE_TEST_UTIL +namespace +{ +thread_local std::optional<size_t> new_pages_before_failure; +} + +void set_memory_backing_failure_after_for_tests(std::optional<size_t> pages) noexcept +{ + new_pages_before_failure = pages; +} +#endif // ── MemPageStore ────────────────────────────────────────────────── -Status MemPageStore::write_at(uint64_t off, const uint8_t *buf, size_t len) +struct MemPageStore::Data { - std::scoped_lock lk(mu_); - if (off + len > data_.size()) { - data_.resize(off + len, 0); + static constexpr size_t kPageBytes = MemPageStore::kBackingPageBytes; + using Page = std::array<uint8_t, kPageBytes>; + std::unordered_map<uint64_t, std::unique_ptr<Page>> pages; + uint64_t bytes = 0; + + void prepare(uint64_t off, size_t len) + { + std::unordered_map<uint64_t, std::unique_ptr<Page>> prepared; + const auto last = (off + len - 1) / kPageBytes; + for (auto index = off / kPageBytes; index <= last; ++index) { + if (!pages.contains(index)) { +#ifdef CROWDB_TREE_TEST_UTIL + if (new_pages_before_failure) { + if (*new_pages_before_failure == 0) { + throw std::bad_alloc(); + } + --*new_pages_before_failure; + } +#endif + prepared.emplace(index, std::make_unique<Page>()); + } + } + if (!prepared.empty()) { + if (prepared.size() > pages.max_size() - pages.size()) { + throw std::bad_alloc(); + } + // Reserve the index before transferring nodes. Failure releases + // every new data page while leaving the stored bytes unchanged. + const auto required = pages.size() + prepared.size(); + if (pages.empty() || required > pages.bucket_count()) { + const auto grown = + pages.bucket_count() <= pages.max_size() / 2 ? 2 * pages.bucket_count() : pages.max_size(); + pages.reserve(std::max(required, grown)); + } + pages.merge(prepared); + } + } + + void write(uint64_t off, const uint8_t *buf, size_t len) + { + while (len != 0) { + const auto offset = static_cast<size_t>(off % kPageBytes); + const auto count = std::min(len, kPageBytes - offset); + std::memcpy(pages.at(off / kPageBytes)->data() + offset, buf, count); + off += count; + buf += count; + len -= count; + } + bytes = std::max(bytes, off); } - std::memcpy(data_.data() + off, buf, len); + + void read(uint64_t off, uint8_t *buf, size_t len) const + { + while (len != 0) { + const auto offset = static_cast<size_t>(off % kPageBytes); + const auto count = std::min(len, kPageBytes - offset); + const auto page = pages.find(off / kPageBytes); + if (page == pages.end()) { + std::memset(buf, 0, count); + } + else { + std::memcpy(buf, page->second->data() + offset, count); + } + off += count; + buf += count; + len -= count; + } + } +}; + +MemPageStore::MemPageStore(uint32_t iu_size) : data_(std::make_unique<Data>()), iu_size_(iu_size) +{ +} + +MemPageStore::~MemPageStore() = default; + +Status MemPageStore::write_at(uint64_t off, const uint8_t *buf, size_t len) +try { + if (len == 0) { + return Status::Ok(); + } + if (buf == nullptr || off > std::numeric_limits<uint64_t>::max() - len) { + return Status::invalid_argument("memory write has a null input or overflowing range"); + } + std::scoped_lock lk(mu_); + // Grow only the written pages. Existing data never moves, and sparse holes + // do not become physical allocations during checkpoint address growth. + data_->prepare(off, len); + data_->write(off, buf, len); return Status::Ok(); } +catch (const std::bad_alloc &) { + return Status::resource_exhausted("memory backing page allocation failed"); +} Status MemPageStore::read_at(uint64_t off, uint8_t *buf, size_t len) const { std::scoped_lock lk(mu_); - if (off + len > data_.size()) { + if (off > data_->bytes || len > data_->bytes - off) { return Status::io_error("MemPageStore: read past end"); } - std::memcpy(buf, data_.data() + off, len); + if (len != 0 && buf == nullptr) { + return Status::invalid_argument("memory read has a null output"); + } + data_->read(off, buf, len); return Status::Ok(); } uint64_t MemPageStore::size() const { std::scoped_lock lk(mu_); - return data_.size(); + return data_->bytes; +} + +size_t MemPageStore::resident_bytes() const +{ + std::scoped_lock lk(mu_); + return data_->pages.size() * Data::kPageBytes; +} + +bool MemPageStore::release_backing_page(uint64_t index) +{ + std::scoped_lock lk(mu_); + return data_->pages.erase(index) != 0; } } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/scheduled_page_store.cpp b/lib/crowdb-tree/src/backend/local/scheduled_page_store.cpp new file mode 100644 index 000000000..d9285d0db --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/scheduled_page_store.cpp @@ -0,0 +1,321 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "scheduled_page_store.h" + +#include "btree/allocation_bound.h" +#include "btree/tree_metrics.h" +#include "crowdb-common/log.h" +#include "crowdb-tree/backend/block_page_store.h" + +namespace crowdb::tree +{ +struct ScheduledPageStore::State +{ + PageStore *sync = nullptr; + AsyncPageStore *async = nullptr; + std::shared_ptr<const void> backend_owner; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<detail::AllocationReservation> jobs; + std::atomic<size_t> active{0}; + + ~State() + { + if (jobs) { + jobs->seal(); + } + } +}; + +struct ScheduledPageStore::Job +{ + std::shared_ptr<State> state; + std::shared_ptr<Job> queued_owner; + AsyncCompletion consumer; + detail::InlineFunction<Status()> step; + PageAddr address = 0; + void *read = nullptr; + const void *write = nullptr; + size_t length = 0; + PagePurpose purpose = PagePurpose::kBtreePage; + size_t read_bytes = 0; + size_t write_bytes = 0; + bool io_started = false; + std::atomic<bool> completed{false}; + bool admitted = false; + ApplyDispatcher::Node node{&Job::execute, this}; + + ~Job() + { + if (admitted) { + finish(Status::unavailable("backend dropped its accepted completion")); + } + } + + void finish(Status status, bool callback = false) noexcept + { + if (completed.exchange(true, std::memory_order_acq_rel)) { + if (callback) { + record_work(state->binding->work_metrics(), TreeWorkCounter::StaleCallbacks, 1, consumer.cause); + } + return; + } + if (io_started) { + const auto &metrics = state->binding->work_metrics(); + if (read_bytes != 0) { + record_work(metrics, status.ok() ? TreeWorkCounter::ReadBytes : TreeWorkCounter::ReadErrors, + status.ok() ? read_bytes : 1, consumer.cause); + } + if (write_bytes != 0) { + record_work(metrics, status.ok() ? TreeWorkCounter::WriteBytes : TreeWorkCounter::WriteErrors, + status.ok() ? write_bytes : 1, consumer.cause); + } + } + state->active.fetch_sub(1, std::memory_order_acq_rel); + try { + consumer.complete(std::move(status)); + } + catch (...) { + CRB_LOG_ERROR("backend completion receiver threw after terminal"); + } + } + + static void completed_io(void *context, Status status) + { + static_cast<Job *>(context)->finish(std::move(status), true); + } + + static void execute(void *context) noexcept + { + auto *job = static_cast<Job *>(context); + auto self = std::move(job->queued_owner); + try { + job->io_started = true; + record_work(job->state->binding->work_metrics(), TreeWorkCounter::ReadOperations, + static_cast<uint64_t>(job->read_bytes != 0), job->consumer.cause); + record_work(job->state->binding->work_metrics(), TreeWorkCounter::WriteOperations, + static_cast<uint64_t>(job->write_bytes != 0), job->consumer.cause); + if (job->step) { + job->finish(job->step()); + return; + } + if (job->state->async != nullptr) { + AsyncCompletion callback{.context = job, + .complete_fn = &completed_io, + .owner = self, + .allocations = job->consumer.allocations, + .cause = job->consumer.cause}; + if (job->read != nullptr) { + job->state->async->submit_read(job->address, job->read, job->length, std::move(callback)); + } + else if (job->write != nullptr) { + job->state->async->submit_typed_write(job->purpose, job->address, job->write, job->length, + std::move(callback)); + } + else { + auto status = job->state->async->submit_fsync(std::move(callback)); + if (!status.ok()) { + job->finish(std::move(status)); + } + } + return; + } + if (job->state->sync == nullptr) { + job->finish(Status::unavailable("backend owner has no synchronous store")); + return; + } + Status status; + auto *block = dynamic_cast<BlockPageStore *>(job->state->sync); + if (block != nullptr) { + const auto &budget = job->consumer.allocations; + if (job->read != nullptr) { + status = block->read_at(job->address, static_cast<uint8_t *>(job->read), job->length, budget); + } + else if (job->write != nullptr) { + status = + block->write_at(job->address, static_cast<const uint8_t *>(job->write), job->length, budget); + } + else { + status = block->sync(budget); + } + job->finish(std::move(status)); + return; + } + if (job->read != nullptr) { + status = job->state->sync->read_at(job->address, static_cast<uint8_t *>(job->read), job->length); + } + else if (job->write != nullptr) { + status = job->state->sync->write_typed_at(job->purpose, job->address, + static_cast<const uint8_t *>(job->write), job->length); + } + else { + status = job->state->sync->sync(); + } + job->finish(std::move(status)); + } + catch (const std::bad_alloc &) { + job->finish(Status::resource_exhausted()); + } + catch (...) { + job->finish(Status::io_error("backend submission threw")); + } + } +}; + +std::optional<size_t> ScheduledPageStore::allocation_bound(size_t operations) noexcept +{ + using namespace detail; + return AllocationBound(shared_allocation_bound<State>() + shared_allocation_bound<ScheduledPageStore>() + + shared_allocation_bound<AllocationReservation>()) + .add_product(operations, shared_allocation_bound<Job>()) + .add(2 * shared_allocation_bound<Job>()) + .value(); +} + +Status ScheduledPageStore::create(PageStore *sync, AsyncPageStore *async, std::shared_ptr<const void> backend_owner, + std::shared_ptr<TreeRuntimeBinding> binding, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + std::shared_ptr<AsyncPageStore> *out) +try { + if ((sync == nullptr && async == nullptr) || !binding || !metadata || !lane || out == nullptr || *out) { + return Status::invalid_argument(); + } + const auto bound = detail::AllocationBound(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add_product(binding->options().limits.io_operations, detail::shared_allocation_bound<Job>()) + .add(2 * detail::shared_allocation_bound<Job>()) + .value(); + if (!bound) { + return Status::resource_exhausted(); + } + auto credit = metadata->take_reserved(*bound); + if (!credit) { + return Status::resource_exhausted(); + } + auto state = detail::reserved_shared<State>(metadata); + state->jobs = detail::AllocationReservation::create(std::move(*credit)); + state->lane = std::move(lane); + state->sync = sync; + state->async = async; + state->backend_owner = std::move(backend_owner); + state->binding = std::move(binding); + *out = detail::reserved_construct<ScheduledPageStore>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) ScheduledPageStore(std::move(state)); }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +PageStore *ScheduledPageStore::synchronous_store() const noexcept +{ + return state_->sync; +} + +bool ScheduledPageStore::uses_async_transport() const noexcept +{ + return state_->async != nullptr; +} + +size_t ScheduledPageStore::allocation_workers() const noexcept +{ + return std::max(state_->binding->cpu()->worker_count(), state_->binding->backend_cpu()->worker_count()); +} + +bool ScheduledPageStore::write_completion_is_reloadable() const noexcept +{ + return state_->async != nullptr ? state_->async->write_completion_is_reloadable() + : !state_->sync->has_shared_ownership(); +} + +Status ScheduledPageStore::submit(PageAddr address, void *read, const void *write, size_t length, PagePurpose purpose, + AsyncCompletion complete, detail::InlineFunction<Status()> step, size_t read_bytes, + size_t write_bytes) +try { + if (!complete || (length != 0 && read == nullptr && write == nullptr) || address > UINT64_MAX - length) { + return Status::invalid_argument(); + } + using namespace detail; + auto job = reserved_shared<Job>(state_->jobs); + job->state = state_; + job->step = std::move(step); + job->consumer = std::move(complete); + job->address = address; + job->read = read; + job->write = write; + job->length = length; + job->read_bytes = read != nullptr ? length : read_bytes; + job->write_bytes = write != nullptr ? length : write_bytes; + job->purpose = purpose; + auto active = state_->active.load(std::memory_order_relaxed); + do { + if (active >= state_->binding->options().limits.io_operations) { + record_work(state_->binding->work_metrics(), TreeWorkCounter::AdmissionRejected, 1, job->consumer.cause); + return Status::resource_exhausted(); + } + } while (!state_->active.compare_exchange_weak(active, active + 1, std::memory_order_acq_rel)); + job->admitted = true; + job->queued_owner = job; + auto status = state_->binding->backend_cpu()->try_submit(state_->lane, &Job::execute, job.get()); + if (!status.ok()) { + state_->binding->backend_progress()->request(job->node, job); + } + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +uint64_t ScheduledPageStore::submit_read(PageAddr address, void *buffer, size_t length, AsyncCompletion complete) +{ + if (length == 0) { + complete.complete(Status::Ok()); + return 0; + } + auto status = submit(address, buffer, nullptr, length, PagePurpose::kBtreePage, complete); + if (!status.ok()) { + complete.complete(std::move(status)); + } + return 0; +} + +uint64_t ScheduledPageStore::submit_write(PageAddr address, const void *buffer, size_t length, AsyncCompletion complete) +{ + return submit_typed_write(PagePurpose::kBtreePage, address, buffer, length, std::move(complete)); +} + +uint64_t ScheduledPageStore::submit_typed_write(PagePurpose purpose, PageAddr address, const void *buffer, + size_t length, AsyncCompletion complete) +{ + if (length == 0) { + complete.complete(Status::Ok()); + return 0; + } + auto status = submit(address, nullptr, buffer, length, purpose, complete); + if (!status.ok()) { + complete.complete(std::move(status)); + } + return 0; +} + +Status ScheduledPageStore::submit_fsync(AsyncCompletion complete) +{ + return submit(0, nullptr, nullptr, 0, PagePurpose::kBtreePage, std::move(complete)); +} + +Status ScheduledPageStore::submit_step(detail::InlineFunction<Status()> step, AsyncCompletion complete, + size_t read_bytes, size_t write_bytes) +{ + if (!step || (read_bytes != 0 && write_bytes != 0)) { + return Status::invalid_argument(); + } + return submit(0, nullptr, nullptr, 0, PagePurpose::kBtreePage, std::move(complete), std::move(step), read_bytes, + write_bytes); +} + +void ScheduledPageStore::cancel(uint64_t /*operation*/) +{ + // Accepted operations keep their owners until real terminal or transport + // disposal. Cancellation of a consumer never releases in-flight I/O early. +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/scheduled_page_store.h b/lib/crowdb-tree/src/backend/local/scheduled_page_store.h new file mode 100644 index 000000000..da4c3eba1 --- /dev/null +++ b/lib/crowdb-tree/src/backend/local/scheduled_page_store.h @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/inline_function.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/backend/page_store.h" +#include "runtime/tree_runtime_binding.h" + +namespace crowdb::tree +{ +// Dispatches blocking storage preparation and synchronous backends away from +// Tree CPU workers. Queue slots, operation controls and outstanding callbacks +// are bounded; an uncertain submission retains the caller's buffer owner. +class ScheduledPageStore final : public AsyncPageStore +{ + struct State; + struct Job; + + public: + static std::optional<size_t> allocation_bound(size_t operations) noexcept; + static Status create(PageStore *sync, AsyncPageStore *async, std::shared_ptr<const void> backend_owner, + std::shared_ptr<TreeRuntimeBinding> binding, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<detail::FlushRuntime::Lane> lane, std::shared_ptr<AsyncPageStore> *out); + [[nodiscard]] bool write_completion_is_reloadable() const noexcept override; + [[nodiscard]] PageStore *synchronous_store() const noexcept; + [[nodiscard]] bool uses_async_transport() const noexcept; + [[nodiscard]] size_t allocation_workers() const noexcept; + uint64_t submit_read(PageAddr address, void *buffer, size_t length, AsyncCompletion complete) override; + uint64_t submit_write(PageAddr address, const void *buffer, size_t length, AsyncCompletion complete) override; + uint64_t submit_typed_write(PagePurpose purpose, PageAddr address, const void *buffer, size_t length, + AsyncCompletion complete) override; + Status submit_fsync(AsyncCompletion complete) override; + void cancel(uint64_t operation) override; + // A finite, paid backend capability step uses the same admission and drain + // boundary as file I/O; it never executes on the submitting CPU worker. + Status submit_step(detail::InlineFunction<Status()> step, AsyncCompletion complete, size_t read_bytes = 0, + size_t write_bytes = 0); + + private: + explicit ScheduledPageStore(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + Status submit(PageAddr address, void *read, const void *write, size_t length, PagePurpose purpose, + AsyncCompletion complete, detail::InlineFunction<Status()> step = {}, size_t read_bytes = 0, + size_t write_bytes = 0); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/pack_allocation_bound.cpp b/lib/crowdb-tree/src/backend/pack_allocation_bound.cpp new file mode 100644 index 000000000..6dce8099d --- /dev/null +++ b/lib/crowdb-tree/src/backend/pack_allocation_bound.cpp @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "chunk/chunk_fixed_pack.h" +#include "chunk/chunk_frame.h" +#include "chunk/chunk_physical_registry.h" +#include "chunk/chunk_pin_release_internal.h" + +namespace crowdb::tree::detail +{ +std::optional<size_t> ChunkFixedPackStore::write_allocation_bound(size_t bytes) noexcept +{ + if (bytes == 0 || bytes > kMaxChunkPackBytes) { + return {}; + } + return AllocationBound(chunk_framed_length(bytes)) + .add(shared_allocation_bound<ChunkPackWindow>() + ChunkPhysicalRegistry::pack_allocation_bound()) + .add(12 * sizeof(ChunkStagePack)) + .add(2 * (100 + 16)) + .value(); +} + +std::optional<size_t> ChunkFixedPackStore::read_allocation_bound(size_t bytes) noexcept +{ + if (bytes == 0 || bytes > kMaxChunkPackBytes) { + return {}; + } + return AllocationBound(chunk_framed_length(bytes)) + .add(bytes) + .add(2 * shared_allocation_bound<ChunkPackWindow>()) + .value(); +} + +std::optional<size_t> ChunkFixedPackStore::read_into_allocation_bound(size_t bytes) noexcept +{ + if (bytes == 0 || bytes > kMaxChunkPackBytes) { + return {}; + } + return AllocationBound(chunk_framed_length(bytes)).add(shared_allocation_bound<ChunkPackWindow>()).value(); +} + +size_t ChunkPinReleaseQueue::pin_allocation_bound() noexcept +{ + return sizeof(ChunkPinReleaseNode) + shared_allocation_bound<ChunkPinOwner>(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/pack_backing.cpp b/lib/crowdb-tree/src/backend/pack_backing.cpp new file mode 100644 index 000000000..e7fd8f18d --- /dev/null +++ b/lib/crowdb-tree/src/backend/pack_backing.cpp @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "chunk/chunk_backing.h" +#include "chunk/chunk_frame.h" +#include "chunk/chunk_physical_registry.h" + +namespace crowdb::tree +{ +bool ChunkBackingDomain::supported() const noexcept +{ + return backend_id != 0 && tree != 0 && epoch != 0 && mirrors != 0 && mirrors <= detail::kMaxMirrorCopies && + max_chunk_bytes != 0 && max_chunk_bytes <= detail::kMaxChunkPackBytes && catalog && + catalog->stage_schema_supported() && catalog->protection() != nullptr && catalog->pin_release_queue() && + transport && transport->fixed_staging_supported(); +} + +bool chunk_backing_matches(const ChunkPackBacking &backing, uint64_t domain, uint64_t length) noexcept +{ + const auto &pack = backing.pack; + return backing.domain && backing.domain->supported() && backing.domain->backend_id == domain && backing.pin && + pack.stage.owner_epoch != 0 && (pack.stage.operation_high != 0 || pack.stage.operation_low != 0) && + !pack.chunk.empty() && length != 0 && length == pack.length && length <= detail::kMaxChunkPackBytes && + detail::valid_page_purpose(pack.purpose) && detail::chunk_purpose(pack.chunk) == pack.purpose && + pack.offset <= UINT64_MAX - detail::chunk_framed_length(pack.length); +} + +std::array<uint64_t, 8> chunk_backing_identity(const ChunkPackBacking &backing) noexcept +{ + const auto &pack = backing.pack; + return {pack.stage.owner_epoch, + pack.stage.operation_high, + pack.stage.operation_low, + pack.chunk.high, + pack.chunk.low, + pack.offset, + pack.length, + (uint64_t{pack.checksum} << 32U) | static_cast<uint32_t>(pack.purpose)}; +} + +bool same_chunk_backing(const std::shared_ptr<const ChunkPackBacking> &left, + const std::shared_ptr<const ChunkPackBacking> &right) noexcept +{ + if (!left || !right) { + return left == right; + } + return chunk_backing_identity(*left) == chunk_backing_identity(*right) && left->pack.length == right->pack.length && + left->pack.checksum == right->pack.checksum && left->pack.purpose == right->pack.purpose && left->domain && + right->domain && left->domain->backend_id == right->domain->backend_id; +} + +size_t chunk_backing_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<ChunkPackBacking>() + detail::ChunkPinReleaseQueue::pin_allocation_bound() + + ChunkPhysicalRegistry::pack_allocation_bound(); +} + +Status read_chunk_backing(const ChunkPackBacking &backing, std::span<uint8_t> output, + const std::shared_ptr<AllocationBudget> &budget) +{ + return backing.read != nullptr ? backing.read(backing, output, budget) + : Status::corruption("immutable pack has no read capability"); +} + +Status write_chunk_backing(ChunkPackBacking &backing, std::span<const uint8_t> input, + const std::shared_ptr<AllocationBudget> &budget) +{ + return backing.write != nullptr ? backing.write(backing, input, budget) + : Status::invalid_argument("immutable pack has no write capability"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/pack_physical_bound.cpp b/lib/crowdb-tree/src/backend/pack_physical_bound.cpp new file mode 100644 index 000000000..d47763db8 --- /dev/null +++ b/lib/crowdb-tree/src/backend/pack_physical_bound.cpp @@ -0,0 +1,20 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/retired_shared.h" +#include "chunk/chunk_physical_registry_internal.h" + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kNodeOverhead = (8 * sizeof(void *)) + sizeof(AllocationLease); +} + +size_t ChunkPhysicalRegistry::pack_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<ChunkPhysicalPack>() + sizeof(State::Key) + + sizeof(std::shared_ptr<ChunkPhysicalPack>) + kNodeOverhead + + (3 * detail::shared_allocation_bound<State::Request>()); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/admitted_batch.cpp b/lib/crowdb-tree/src/btree/admitted_batch.cpp new file mode 100644 index 000000000..27e130e15 --- /dev/null +++ b/lib/crowdb-tree/src/btree/admitted_batch.cpp @@ -0,0 +1,229 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "admitted_batch.h" + +#include "allocation_bound.h" + +#include <algorithm> +#include <cassert> +#include <cstring> +#include <numeric> + +namespace crowdb::tree +{ +namespace +{ +Status validate(const Config &config, const SourceWriteOptions &options, size_t max_value_bytes, size_t count, + const CoreWriteReader &read, size_t *bytes) +{ + if (!options.account || count > options.max_entries) { + return Status::invalid_argument("proposal batch exceeds the configured entry limit"); + } + detail::AllocationBound bound; + for (size_t index = 0; index < count; ++index) { + const auto input = read(index); + if (input.encoded || (input.flags & ~kFlagTombstone) != 0 || !config.key_range.contains(input.key) || + input.key.size() > options.max_key_size || input.value.size() > SIZE_MAX - kCellHeaderSize || + ((input.flags & kFlagTombstone) == 0 && input.value.size() > max_value_bytes)) { + return Status::invalid_argument("proposal contains an invalid key, value or encoding"); + } + if (!primary_write_fits(config, input.key.size(), input.value.size(), (input.flags & kFlagTombstone) != 0)) { + return Status::invalid_argument("proposal key/cell cannot fit a primary leaf or inner separator"); + } + bound.add(input.key.size()); + if ((input.flags & kFlagTombstone) == 0) { + bound.add(input.value.size()); + } + } + const auto size = bound.value(); + if (!size) { + return Status::resource_exhausted("proposal input bound overflow"); + } + *bytes = *size; + return Status::Ok(); +} + +size_t encoded_capacity(size_t value_bytes, uint8_t flags) +{ + const auto size = (flags & kFlagTombstone) != 0 ? kCellHeaderSize : kCellHeaderSize + value_bytes; + return size <= buffer::kInlineCap ? 0 : size; +} +} // namespace + +AdmittedBatch::AdmittedBatch(std::shared_ptr<detail::AllocationReservation> metadata) + : metadata_(std::move(metadata)), + bytes_(decltype(bytes_)::allocator_type(metadata_)), + cells_(detail::ReservationAllocator<Cell>(metadata_)), + prepared_(detail::ReservationAllocator<SourceAdmission::Writer::Prepared>(metadata_)) +{ +} + +Status AdmittedBatch::create(const Config &config, const SourceWriteOptions &options, size_t max_value_bytes, + size_t count, const CoreWriteReader &read, detail::BudgetClass category, + std::shared_ptr<AdmittedBatch> *out) +try { + if (out == nullptr || *out || + (category != detail::BudgetClass::kAdmittedWriter && category != detail::BudgetClass::kGapRepair)) { + return Status::invalid_argument("proposal admission requires an empty output and apply partition"); + } + size_t incoming_bytes = 0; + auto status = validate(config, options, max_value_bytes, count, read, &incoming_bytes); + if (!status.ok()) { + return status; + } + const auto bound = + detail::AllocationBound(detail::shared_allocation_bound<AdmittedBatch>()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add(incoming_bytes) + .add_product(count, sizeof(Cell) + sizeof(size_t) + sizeof(SourceAdmission::Writer::Prepared)) + .value(); + auto reserved = bound ? detail::ResourceBudget::try_reserve(options.account, category, *bound) : std::nullopt; + if (!reserved) { + return Status::unavailable("proposal staging credit unavailable"); + } + auto credit = detail::AllocationReservation::create(std::move(*reserved)); + auto batch = detail::reserved_construct<AdmittedBatch>( + credit, [&](auto *storage) { ::new (static_cast<void *>(storage)) AdmittedBatch(credit); }); + status = batch->initialize(options, count, read, category, incoming_bytes); + if (!status.ok()) { + return status; + } + credit->seal(); + *out = std::move(batch); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("proposal admission allocation failed"); +} + +Status AdmittedBatch::initialize(const SourceWriteOptions &options, size_t count, const CoreWriteReader &read, + detail::BudgetClass category, size_t incoming_bytes, SourceWriteReservation prepaid) +{ + detail::ReservedVector<size_t> order(count, detail::ReservationAllocator<size_t>(metadata_)); + std::iota(order.begin(), order.end(), size_t{0}); + std::sort(order.begin(), order.end(), [&](size_t left, size_t right) { + const auto comparison = read(left).key.compare(read(right).key); + return comparison == 0 ? left < right : comparison < 0; + }); + WriteFootprint footprint; + detail::AllocationBound payload; + size_t selected = 0; + for (size_t index = 0; index < order.size(); ++index) { + if (index + 1 != order.size() && read(order[index]).key == read(order[index + 1]).key) { + continue; + } + const auto input = read(order[index]); + order[selected++] = order[index]; + footprint.key_bytes += input.key.size(); + footprint.max_key_size = std::max(footprint.max_key_size, input.key.size()); + payload.add(encoded_capacity(input.value.size(), input.flags)); + } + order.resize(selected); + footprint.entries = selected; + const auto capacity = payload.value(); + if (!capacity) { + return Status::resource_exhausted("proposal payload bound overflow"); + } + footprint.payload_bytes = *capacity; + if (prepaid.valid()) { + const auto paid = prepaid.footprint(); + if (paid.entries != footprint.entries || paid.key_bytes != footprint.key_bytes || + paid.payload_bytes != footprint.payload_bytes || paid.max_key_size != footprint.max_key_size) { + return Status::invalid_argument("proposal inputs changed after resource planning"); + } + source_ = std::move(prepaid); + } + else if (selected != 0) { + auto status = SourceWriteReservation::create(options, footprint, category, &source_); + if (!status.ok()) { + return status; + } + } + bytes_.resize(incoming_bytes); + cells_.reserve(selected); + prepared_.reserve(selected); + size_t offset = 0; + for (const auto index : order) { + const auto input = read(index); + const auto value_size = (input.flags & kFlagTombstone) != 0 ? 0 : input.value.size(); + cells_.push_back({.key_offset = offset, + .key_size = input.key.size(), + .value_offset = offset + input.key.size(), + .value_size = value_size, + .flags = input.flags}); + if (!input.key.empty()) { + std::memcpy(bytes_.data() + offset, input.key.data(), input.key.size()); + } + offset += input.key.size(); + if (value_size != 0) { + std::memcpy(bytes_.data() + offset, input.value.data(), value_size); + } + offset += value_size; + } + return Status::Ok(); +} + +Status AdmittedBatch::prepare_cells(SourceAdmission::Writer &writer) +{ + for (const auto &cell : cells_) { + AllocationLease charge(writer.allocation_budget(), encoded_capacity(cell.value_size, cell.flags), + AllocationKind::kPayload); + auto value = buffer::alloc(cell.value_size, kCellHeaderSize); + std::memset(value.data(), 0, kCellHeaderSize); + value.data()[sizeof(uint64_t)] = cell.flags; + if (cell.value_size != 0) { + std::memcpy(value.data() + kCellHeaderSize, bytes_.data() + cell.value_offset, cell.value_size); + } + prepared_.emplace_back(); + charge = {}; + const auto *key = cell.key_size == 0 ? nullptr : bytes_.data() + cell.key_offset; + auto status = writer.prepare_write(Slice(key, cell.key_size), cell.flags, std::move(value), &prepared_.back()); + if (!status.ok()) { + return status; + } + } + return Status::Ok(); +} + +Status AdmittedBatch::try_commit(SourceAdmission &sources, ApplyFrontier &frontier, uint64_t slot) +try { + if (consumed_) { + return Status::invalid_argument("admitted batch has already been consumed"); + } + auto status = frontier.check_slot(slot); + if (!status.ok()) { + return status; + } + if (cells_.empty() || slot <= sources.catalog()->floor) { + consumed_ = true; + source_ = {}; + return frontier.complete(slot); + } + const auto completed = frontier.contiguous().load(std::memory_order_acquire); + // Higher missing slots use ordinary admission or wait. Only the next + // contiguous slot may borrow the finite gap allowance; otherwise later + // holes can prevent the frontier from ever advancing. + const bool gap = slot > completed && slot - completed == 1 && slot <= frontier.max_seen(); + const auto partition = gap ? detail::BudgetClass::kGapRepair : detail::BudgetClass::kAdmittedWriter; + SourceAdmission::Writer writer; + status = sources.try_admit_reserved(completed, &source_, &writer, partition); + if (!status.ok()) { + return status; + } + consumed_ = true; + status = prepare_cells(writer); + if (!status.ok()) { + return status; + } + for (auto &write : prepared_) { + writer.commit_write(std::move(write), slot); + } + prepared_.clear(); + status = frontier.complete(slot); + assert(status.ok()); + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("prepaid batch preparation allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/admitted_batch.h b/lib/crowdb-tree/src/btree/admitted_batch.h new file mode 100644 index 000000000..60d328896 --- /dev/null +++ b/lib/crowdb-tree/src/btree/admitted_batch.h @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "tree_core_batch.h" + +namespace crowdb::tree +{ +// Owns one validated, deduplicated request before consensus assigns its slot. +// This owner has no source entrance or epoch guard. Incoming bytes and commit +// descriptors are charged separately from the prepaid source allocation. +class AdmittedBatch +{ + public: + struct Plan + { + size_t incoming_bytes = 0; + size_t staging_bytes = 0; + size_t source_bytes = 0; + WriteFootprint footprint; + }; + + // Derive the whole input/source bound without allocation or source entrance. + static Status plan(const Config &config, const SourceWriteOptions &options, size_t max_value_bytes, size_t count, + const CoreWriteReader &read, Plan *out); + static Status bind_prepaid(const SourceWriteOptions &options, size_t count, const CoreWriteReader &read, + detail::BudgetClass category, const Plan &plan, detail::ResourceBudget::Credit staging, + detail::ResourceBudget::Credit source, std::shared_ptr<AdmittedBatch> *out); + static Status create(const Config &config, const SourceWriteOptions &options, size_t max_value_bytes, size_t count, + const CoreWriteReader &read, detail::BudgetClass category, + std::shared_ptr<AdmittedBatch> *out); + + // A busy source or distant frontier leaves all bytes and credit intact. + // Once a source is bound, this batch is consumed exactly once. The caller + // retains the generation and supplies asynchronous progress on busy. + Status try_commit(SourceAdmission &sources, ApplyFrontier &frontier, uint64_t slot); + + [[nodiscard]] bool consumed() const noexcept + { + return consumed_; + } + + [[nodiscard]] size_t size() const noexcept + { + return cells_.size(); + } + + [[nodiscard]] WriteFootprint footprint() const noexcept + { + return source_.footprint(); + } + + private: + struct Cell + { + size_t key_offset = 0; + size_t key_size = 0; + size_t value_offset = 0; + size_t value_size = 0; + uint8_t flags = 0; + }; + + explicit AdmittedBatch(std::shared_ptr<detail::AllocationReservation> metadata); + Status initialize(const SourceWriteOptions &options, size_t count, const CoreWriteReader &read, + detail::BudgetClass category, size_t incoming_bytes, SourceWriteReservation prepaid = {}); + Status prepare_cells(SourceAdmission::Writer &writer); + + std::shared_ptr<detail::AllocationReservation> metadata_; + SourceWriteReservation source_; + detail::ReservedVector<uint8_t, AllocationKind::kPayload> bytes_; + detail::ReservedVector<Cell> cells_; + detail::ReservedVector<SourceAdmission::Writer::Prepared> prepared_; + bool consumed_ = false; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/admitted_batch_plan.cpp b/lib/crowdb-tree/src/btree/admitted_batch_plan.cpp new file mode 100644 index 000000000..2880614f7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/admitted_batch_plan.cpp @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "admitted_batch.h" +#include "allocation_bound.h" + +#include <algorithm> +#include <array> +#include <numeric> + +namespace crowdb::tree +{ +Status AdmittedBatch::plan(const Config &config, const SourceWriteOptions &options, size_t max_value_bytes, + size_t count, const CoreWriteReader &read, Plan *out) +{ + // Stack scratch is bounded independently of untrusted proposal lengths. + std::array<size_t, 4096> indices{}; + if (out == nullptr || !options.account || count > options.max_entries || count > indices.size()) { + return Status::invalid_argument("proposal input count exceeds its bounded planning window"); + } + detail::AllocationBound incoming; + for (size_t index = 0; index < count; ++index) { + const auto input = read(index); + if (input.encoded || (input.flags & ~kFlagTombstone) != 0 || !config.key_range.contains(input.key) || + input.key.size() > options.max_key_size || input.value.size() > SIZE_MAX - kCellHeaderSize || + ((input.flags & kFlagTombstone) == 0 && input.value.size() > max_value_bytes)) { + return Status::invalid_argument("proposal contains an invalid key, value or encoding"); + } + if (!primary_write_fits(config, input.key.size(), input.value.size(), (input.flags & kFlagTombstone) != 0)) { + return Status::invalid_argument("proposal key/cell cannot fit a primary leaf or inner separator"); + } + incoming.add(input.key.size()); + if ((input.flags & kFlagTombstone) == 0) { + incoming.add(input.value.size()); + } + } + const auto input_bytes = incoming.value(); + if (!input_bytes) { + return Status::invalid_argument("proposal input bound overflow"); + } + auto order = std::span(indices).first(count); + std::iota(order.begin(), order.end(), size_t{0}); + std::sort(order.begin(), order.end(), [&](size_t left, size_t right) { + const auto compared = read(left).key.compare(read(right).key); + return compared == 0 ? left < right : compared < 0; + }); + WriteFootprint footprint; + detail::AllocationBound keys; + detail::AllocationBound payload; + for (size_t index = 0; index < count; ++index) { + const auto input = read(order[index]); + if (index + 1 != count && input.key == read(order[index + 1]).key) { + continue; + } + ++footprint.entries; + keys.add(input.key.size()); + footprint.max_key_size = std::max(footprint.max_key_size, input.key.size()); + const auto bytes = kCellHeaderSize + ((input.flags & kFlagTombstone) == 0 ? input.value.size() : 0); + if (bytes > buffer::kInlineCap) { + payload.add(bytes); + } + } + const auto key_bytes = keys.value(); + const auto payload_bytes = payload.value(); + if (!key_bytes || !payload_bytes) { + return Status::invalid_argument("proposal unique-cell bound overflow"); + } + footprint.key_bytes = *key_bytes; + footprint.payload_bytes = *payload_bytes; + const auto staging = + detail::AllocationBound(detail::shared_allocation_bound<AdmittedBatch>()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add(*input_bytes) + .add_product(count, sizeof(Cell) + sizeof(size_t) + sizeof(SourceAdmission::Writer::Prepared)) + .value(); + const auto source = SourceWriteBudget::allocation_bound(options, footprint); + if (!staging || !source) { + return Status::invalid_argument("proposal complete allocation bound overflow"); + } + *out = {.incoming_bytes = *input_bytes, .staging_bytes = *staging, .source_bytes = *source, .footprint = footprint}; + return Status::Ok(); +} + +Status AdmittedBatch::bind_prepaid(const SourceWriteOptions &options, size_t count, const CoreWriteReader &read, + detail::BudgetClass category, const Plan &plan, + detail::ResourceBudget::Credit staging, detail::ResourceBudget::Credit source, + std::shared_ptr<AdmittedBatch> *out) +{ + if (out == nullptr || *out || !staging.belongs_to(options.account, category) || staging.allocated() || + staging.amount() < plan.staging_bytes) { + return Status::invalid_argument("proposal staging requires its complete unused grant"); + } + SourceWriteReservation reservation; + if (plan.footprint.entries != 0) { + auto status = + SourceWriteReservation::bind_prepaid(options, plan.footprint, category, std::move(source), &reservation); + if (!status.ok()) { + return status; + } + } + auto credit = detail::AllocationReservation::create(std::move(staging)); + auto batch = detail::reserved_construct<AdmittedBatch>( + credit, [&](auto *storage) { ::new (static_cast<void *>(storage)) AdmittedBatch(credit); }); + auto status = batch->initialize(options, count, read, category, plan.incoming_bytes, std::move(reservation)); + if (!status.ok()) { + return status; + } + credit->seal(); + *out = std::move(batch); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/allocation_account.cpp b/lib/crowdb-tree/src/btree/allocation_account.cpp new file mode 100644 index 000000000..be1f440e2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_account.cpp @@ -0,0 +1,188 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account.h" + +#include <algorithm> +#include <cassert> +#include <limits> + +namespace crowdb::tree::detail +{ +AllocationAccount::AllocationAccount(std::shared_ptr<ResourceBudget::Account> account, BudgetClass category) + : retirement_(account ? account->retirement_.lock() : nullptr), + account_(std::move(account)), + category_(category), + domain_(ResourceBudget::allocation_domain(account_)) +{ +} + +void AllocationAccount::leave() noexcept +{ + const auto old = ingress_.fetch_sub(1, std::memory_order_acq_rel); + if ((old & kClosed) != 0) { + ingress_.notify_one(); + } +} + +bool AllocationAccount::claim(size_t bytes) noexcept +{ + const auto entered = ingress_.fetch_add(1, std::memory_order_acq_rel); + if ((entered & kClosed) != 0) { + leave(); + return false; + } + auto current = allocated_.load(std::memory_order_relaxed); + while (bytes <= std::numeric_limits<size_t>::max() - current) { + if (!allocated_.compare_exchange_weak(current, current + bytes, std::memory_order_acq_rel)) { + continue; + } + const bool accepted = !account_ || ResourceBudget::claim_allocation(account_, category_, bytes); + if (!accepted) { + allocated_.fetch_sub(bytes, std::memory_order_relaxed); + } + leave(); + return accepted; + } + leave(); + return false; +} + +void AllocationAccount::apply_release(size_t bytes) noexcept +{ + if (bytes == 0) { + return; + } + [[maybe_unused]] const auto old = allocated_.fetch_sub(bytes, std::memory_order_relaxed); + assert(old >= bytes); + ResourceBudget::release_allocation(account_, category_, bytes); +} + +void AllocationAccount::release(size_t bytes) noexcept +{ + release(bytes, nullptr, nullptr); +} + +void AllocationAccount::release(size_t bytes, Hook hook, void *context) noexcept +{ + const auto entered = ingress_.fetch_add(1, std::memory_order_acq_rel); + if (hook != nullptr) { + hook(context, Step::kReleaseEntered, 0); + } + if ((entered & kClosed) != 0) { + pending_release_.fetch_add(bytes, std::memory_order_relaxed); + ingress_.fetch_or(kDirty, std::memory_order_release); + } + else { + apply_release(bytes); + } + leave(); +} + +void AllocationAccount::apply_record(size_t kind, size_t bytes, bool acquired) noexcept +{ + if (bytes == 0) { + return; + } + if (acquired) { + physical_[kind].fetch_add(bytes, std::memory_order_relaxed); + } + else { + [[maybe_unused]] const auto old = physical_[kind].fetch_sub(bytes, std::memory_order_relaxed); + assert(old >= bytes); + } + ResourceBudget::record_allocation(account_, static_cast<AllocationKind>(kind), bytes, acquired); +} + +void AllocationAccount::record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept +{ + const auto index = static_cast<size_t>(kind); + if (kind == AllocationKind::kUnknown || index >= kKinds) { + return; + } + const auto entered = ingress_.fetch_add(1, std::memory_order_acq_rel); + if ((entered & kClosed) != 0) { + (acquired ? pending_acquired_[index] : pending_freed_[index]).fetch_add(bytes, std::memory_order_relaxed); + ingress_.fetch_or(kDirty, std::memory_order_release); + } + else { + apply_record(index, bytes, acquired); + } + leave(); +} + +void AllocationAccount::reopen(Hook hook, void *context) noexcept +{ + for (;;) { + ingress_.fetch_and(~kDirty, std::memory_order_acq_rel); + for (size_t kind = 1; kind < kKinds; ++kind) { + apply_record(kind, pending_acquired_[kind].exchange(0, std::memory_order_acq_rel), true); + if (hook != nullptr) { + hook(context, Step::kAcquiredDrained, kind); + } + const auto freed = pending_freed_[kind].exchange(0, std::memory_order_acq_rel); + const auto known = physical_[kind].load(std::memory_order_relaxed); + apply_record(kind, std::min(freed, known), false); + if (freed > known) { + // An acquire record can arrive after its exchange but before + // the matching free exchange. Keep the unmatched free until + // the next pass instead of exposing an unsigned underflow. + pending_freed_[kind].fetch_add(freed - known, std::memory_order_relaxed); + ingress_.fetch_or(kDirty, std::memory_order_release); + } + } + apply_release(pending_release_.exchange(0, std::memory_order_acq_rel)); + if (hook != nullptr) { + hook(context, Step::kBeforeReopen, 0); + } + uint64_t expected = kClosed; + if (ingress_.compare_exchange_strong(expected, 0, std::memory_order_release, std::memory_order_acquire)) { + return; + } + if ((expected & kActive) != 0) { + if (hook != nullptr) { + hook(context, Step::kWaitForWriter, 0); + } + ingress_.wait(expected, std::memory_order_acquire); + } + } +} + +Status AllocationAccount::rebind(std::shared_ptr<ResourceBudget::Account> account) +{ + return rebind(std::move(account), nullptr, nullptr); +} + +Status AllocationAccount::rebind(std::shared_ptr<ResourceBudget::Account> account, Hook hook, void *context) +{ + const auto *current_domain = domain(); + if (!account || (current_domain != nullptr && ResourceBudget::allocation_domain(account) != current_domain)) { + return Status::invalid_argument("backend metadata belongs to another host; reopen the backend object"); + } + uint64_t idle = 0; + if (!ingress_.compare_exchange_strong(idle, kClosed, std::memory_order_acq_rel)) { + return Status::unavailable("backend metadata allocation is active"); + } + const auto *closed_domain = domain(); + if (closed_domain != nullptr && ResourceBudget::allocation_domain(account) != closed_domain) { + reopen(nullptr, nullptr); + return Status::invalid_argument("backend metadata belongs to another host; reopen the backend object"); + } + if (hook != nullptr) { + hook(context, Step::kClosed, 0); + } + std::array<size_t, kKinds> physical{}; + for (size_t kind = 0; kind < kKinds; ++kind) { + physical[kind] = physical_[kind].load(std::memory_order_relaxed); + } + const bool moved = ResourceBudget::transfer_allocation(account_, account, category_, allocated_.load(), physical); + if (moved) { + account_ = std::move(account); + domain_.store(ResourceBudget::allocation_domain(account_), std::memory_order_release); + } + if (hook != nullptr) { + hook(context, Step::kTransferred, 0); + } + reopen(hook, context); + return moved ? Status::Ok() : Status::resource_exhausted("new Tree cannot adopt backend metadata capacity"); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_account.h b/lib/crowdb-tree/src/btree/allocation_account.h new file mode 100644 index 000000000..dd35155dd --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_account.h @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "resource_budget.h" + +namespace crowdb::tree::detail +{ +template <class T> class AccountControlAllocator; + +// Stable allocation owner for backend metadata that can outlive a Tree. New +// capacity is admitted against both its current Tree account and host limit. +class AllocationAccount final : public AllocationBudget +{ + public: + static Status create(std::shared_ptr<ResourceBudget::Account> account, BudgetClass category, + std::shared_ptr<AllocationAccount> *out); + // Legacy backend construction records physical capacity before a Tree + // exists. First rebind admits every retained byte before runtime use. + static Status create_unbound(std::shared_ptr<AllocationAccount> *out); + [[nodiscard]] bool claim(size_t bytes) noexcept override; + void release(size_t bytes) noexcept override; + void record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept override; + + [[nodiscard]] const void *domain() const noexcept override + { + return domain_.load(std::memory_order_acquire); + } + + [[nodiscard]] std::shared_ptr<RetirementQueue> retirement_queue() const noexcept override + { + return retirement_; + } + + [[nodiscard]] size_t owned_bytes() const noexcept + { + return allocated_.load(std::memory_order_acquire); + } + + // Transfer already-admitted operation credit into stable backend ownership. + // Both forms preserve host admission; a different Tree must have quota. + bool adopt_allocated(ResourceBudget::Credit *credit) noexcept; + bool claim_from(AllocationReservation &source, size_t bytes) noexcept; + void release_to(AllocationReservation &source, size_t bytes) noexcept; + + // Caller excludes backend growth while reopening a Tree. Existing owners + // may free concurrently. Refusal leaves the original account usable. + Status rebind(std::shared_ptr<ResourceBudget::Account> account); + + private: + template <class T> friend class AccountControlAllocator; +#ifdef CROWDB_TREE_TEST_UTIL + friend struct AllocationAccountTestAccess; +#endif + static constexpr uint64_t kClosed = uint64_t{1} << 63; + static constexpr uint64_t kDirty = uint64_t{1} << 62; + static constexpr uint64_t kActive = kDirty - 1; + static constexpr size_t kKinds = static_cast<size_t>(AllocationKind::kCount); + enum class Step : uint8_t { + kClosed, + kTransferred, + kAcquiredDrained, + kBeforeReopen, + kReleaseEntered, + kWaitForWriter + }; + using Hook = void (*)(void *, Step, size_t) noexcept; + AllocationAccount(std::shared_ptr<ResourceBudget::Account> account, BudgetClass category); + static Status build(std::shared_ptr<ResourceBudget::Account> account, BudgetClass category, + std::shared_ptr<AllocationAccount> *out); + Status rebind(std::shared_ptr<ResourceBudget::Account> account, Hook hook, void *context); + void release(size_t bytes, Hook hook, void *context) noexcept; + void leave() noexcept; + bool adopt_existing(const std::shared_ptr<ResourceBudget::Account> &source, BudgetClass category, + size_t bytes) noexcept; + void apply_release(size_t bytes) noexcept; + void apply_record(size_t kind, size_t bytes, bool acquired) noexcept; + void reopen(Hook hook, void *context) noexcept; + void destroy_control(void *storage, size_t bytes, size_t alignment) noexcept; + + // Bound payload accounts use the immutable host service across same-host + // ownership transfers. Unbound backend accounts do not observe payloads. + const std::shared_ptr<RetirementQueue> retirement_; + std::shared_ptr<ResourceBudget::Account> account_; + BudgetClass category_; + std::atomic<const void *> domain_; + // Closing succeeds only between active account accesses. Closed releases + // publish bounded deltas without waiting. Reopen requires no active writer + // and no undrained delta in the same compare-and-exchange operation. + std::atomic<uint64_t> ingress_{0}; + std::atomic<size_t> allocated_{0}; + std::array<std::atomic<size_t>, kKinds> physical_{}; + std::atomic<size_t> pending_release_{0}; + std::array<std::atomic<size_t>, kKinds> pending_acquired_{}; + std::array<std::atomic<size_t>, kKinds> pending_freed_{}; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_account_control.cpp b/lib/crowdb-tree/src/btree/allocation_account_control.cpp new file mode 100644 index 000000000..ae7d27eed --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_account_control.cpp @@ -0,0 +1,125 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account.h" + +#include <cassert> +#include <limits> +#include <new> + +namespace crowdb::tree::detail +{ +// The stable budget survives until its last weak control reference. Its control +// allocator borrows the budget, so neither object creates a strong owner cycle. +template <class T> class AccountControlAllocator +{ + public: + using value_type = T; + + explicit AccountControlAllocator(AllocationAccount *owner) noexcept : owner_(owner) + { + } + + template <class U> AccountControlAllocator(const AccountControlAllocator<U> &other) noexcept : owner_(other.owner()) + { + } + + [[nodiscard]] T *allocate(size_t count) + { + if (count > std::numeric_limits<size_t>::max() / sizeof(T)) { + throw std::bad_array_new_length(); + } + const auto bytes = count * sizeof(T); + if (!owner_->claim(bytes)) { + throw std::bad_alloc(); + } + try { + auto *storage = static_cast<T *>(::operator new(bytes, std::align_val_t{alignof(T)})); + owner_->record_allocation(AllocationKind::kMetadata, bytes, true); + return storage; + } + catch (...) { + owner_->release(bytes); + throw; + } + } + + void deallocate(T *storage, size_t count) noexcept + { + owner_->destroy_control(storage, count * sizeof(T), alignof(T)); + } + + [[nodiscard]] AllocationAccount *owner() const noexcept + { + return owner_; + } + + template <class U> bool operator==(const AccountControlAllocator<U> &other) const noexcept + { + return owner_ == other.owner(); + } + + private: + AllocationAccount *owner_; +}; + +void AllocationAccount::destroy_control(void *storage, size_t bytes, size_t alignment) noexcept +{ + // Every allocation owner also holds a strong budget reference. Once the + // last control is freed, only the budget object and that control remain. + const auto total = bytes + sizeof(AllocationAccount); + auto account = std::move(account_); + const auto category = category_; + assert(ingress_.load() == 0); + assert(allocated_.load() == total); + assert(physical_[static_cast<size_t>(AllocationKind::kMetadata)].load() == total); + ::operator delete(storage, std::align_val_t{alignment}); + delete this; + ResourceBudget::record_allocation(account, AllocationKind::kMetadata, total, false); + ResourceBudget::release_allocation(account, category, total); +} + +Status AllocationAccount::create(std::shared_ptr<ResourceBudget::Account> account, BudgetClass category, + std::shared_ptr<AllocationAccount> *out) +{ + if (!account || out == nullptr || category == BudgetClass::kIo || category >= BudgetClass::kCount) { + return Status::invalid_argument("backend metadata requires a byte account and output"); + } + return build(std::move(account), category, out); +} + +Status AllocationAccount::create_unbound(std::shared_ptr<AllocationAccount> *out) +{ + if (out == nullptr) { + return Status::invalid_argument("backend allocation account requires output"); + } + return build({}, BudgetClass::kRouting, out); +} + +Status AllocationAccount::build(std::shared_ptr<ResourceBudget::Account> account, BudgetClass category, + std::shared_ptr<AllocationAccount> *out) +{ + constexpr auto bytes = sizeof(AllocationAccount); + if (account && !ResourceBudget::claim_allocation(account, category, bytes)) { + return Status::resource_exhausted("backend allocation account reserve unavailable"); + } + AllocationAccount *owner = nullptr; + try { + owner = new AllocationAccount(account, category); + owner->allocated_.store(bytes, std::memory_order_relaxed); + owner->apply_record(static_cast<size_t>(AllocationKind::kMetadata), bytes, true); + std::shared_ptr<AllocationAccount> result( + owner, [](AllocationAccount *) noexcept {}, AccountControlAllocator<AllocationAccount>(owner)); + *out = std::move(result); + return Status::Ok(); + } + catch (const std::bad_alloc &) { + const bool allocated = owner != nullptr; + delete owner; + if (allocated) { + ResourceBudget::record_allocation(account, AllocationKind::kMetadata, bytes, false); + } + ResourceBudget::release_allocation(account, category, bytes); + return Status::resource_exhausted("backend allocation account allocation failed"); + } +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_account_credit.cpp b/lib/crowdb-tree/src/btree/allocation_account_credit.cpp new file mode 100644 index 000000000..5ba9726db --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_account_credit.cpp @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account.h" +#include "allocation_reservation.h" + +#include <cassert> + +namespace crowdb::tree::detail +{ +bool AllocationAccount::adopt_existing(const std::shared_ptr<ResourceBudget::Account> &source, BudgetClass category, + size_t bytes) noexcept +{ + const auto entered = ingress_.fetch_add(1, std::memory_order_acq_rel); + if ((entered & kClosed) != 0) { + leave(); + return false; + } + const bool accepted = account_ && category == category_ && + ResourceBudget::transfer_allocation(source, account_, category_, bytes, {}); + if (accepted) { + allocated_.fetch_add(bytes, std::memory_order_relaxed); + } + leave(); + return accepted; +} + +bool AllocationAccount::adopt_allocated(ResourceBudget::Credit *credit) noexcept +{ + if (credit == nullptr || !credit->allocated_ || !credit->account_ || credit->kind_ != AllocationKind::kUnknown || + !adopt_existing(credit->account_, static_cast<BudgetClass>(credit->category_), credit->amount_)) { + return false; + } + credit->amount_ = 0; + credit->account_.reset(); + return true; +} + +bool AllocationAccount::claim_from(AllocationReservation &source, size_t bytes) noexcept +{ + if (!source.claim(bytes)) { + return false; + } + if (adopt_existing(source.account(), source.category(), bytes)) { + return true; + } + source.release(bytes); + return false; +} + +void AllocationAccount::release_to(AllocationReservation &source, size_t bytes) noexcept +{ + const auto entered = ingress_.fetch_add(1, std::memory_order_acq_rel); + if ((entered & kClosed) != 0) { + // The operation normally seals before a backend rebind. If a final + // owner races that handoff, return its capacity through the same + // bounded pending-release path instead of reviving a source reserve. + pending_release_.fetch_add(bytes, std::memory_order_relaxed); + ingress_.fetch_or(kDirty, std::memory_order_release); + leave(); + return; + } + [[maybe_unused]] const auto before = allocated_.fetch_sub(bytes, std::memory_order_relaxed); + assert(before >= bytes); + if (ResourceBudget::transfer_allocation(account_, source.account(), category_, bytes, {})) { + source.release(bytes); + } + else { + ResourceBudget::release_allocation(account_, category_, bytes); + } + leave(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_account_grant.cpp b/lib/crowdb-tree/src/btree/allocation_account_grant.cpp new file mode 100644 index 000000000..7d9ac3245 --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_account_grant.cpp @@ -0,0 +1,129 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_grant.h" + +#include <limits> +#include <new> +#include <stdexcept> + +namespace crowdb::tree::detail +{ +namespace +{ +// Only allocate() borrows the factory's stack credit. The stored allocator +// frees through the independent stable owner after the grant is destroyed. +template <class T> class GrantControlAllocator +{ + public: + using value_type = T; + + GrantControlAllocator(std::shared_ptr<AllocationAccount> owner, ResourceBudget::Credit *source) noexcept + : owner_(std::move(owner)), + source_(source) + { + } + + template <class U> + GrantControlAllocator(const GrantControlAllocator<U> &other) noexcept + : owner_(other.owner()), + source_(other.source()) + { + } + + [[nodiscard]] T *allocate(size_t count) + { + if (count > std::numeric_limits<size_t>::max() / sizeof(T)) { + throw std::bad_array_new_length(); + } + const auto bytes = count * sizeof(T); + auto credit = source_->allocate(bytes); + if (!credit || !owner_->adopt_allocated(&*credit)) { + throw std::bad_alloc(); + } + try { + auto *result = static_cast<T *>(::operator new(bytes, std::align_val_t{alignof(T)})); + owner_->record_allocation(AllocationKind::kMetadata, bytes, true); + return result; + } + catch (...) { + owner_->release(bytes); + throw; + } + } + + void deallocate(T *pointer, size_t count) noexcept + { + ::operator delete(pointer, std::align_val_t{alignof(T)}); + owner_->record_allocation(AllocationKind::kMetadata, count * sizeof(T), false); + owner_->release(count * sizeof(T)); + } + + [[nodiscard]] const std::shared_ptr<AllocationAccount> &owner() const noexcept + { + return owner_; + } + + [[nodiscard]] ResourceBudget::Credit *source() const noexcept + { + return source_; + } + + template <class U> bool operator==(const GrantControlAllocator<U> &other) const noexcept + { + return owner_ == other.owner() && source_ == other.source(); + } + + private: + std::shared_ptr<AllocationAccount> owner_; + ResourceBudget::Credit *source_; +}; +} // namespace + +BackendAllocationGrant::BackendAllocationGrant(std::shared_ptr<AllocationAccount> owner, ResourceBudget::Credit credit) + : owner_(std::move(owner)), + source_(std::move(credit)) +{ +} + +Status BackendAllocationGrant::create(std::shared_ptr<AllocationAccount> owner, ResourceBudget::Credit credit, + std::shared_ptr<BackendAllocationGrant> *out) +try { + if (!owner || out == nullptr || *out || credit.allocated() || credit.amount() == 0) { + return Status::invalid_argument("backend grant requires unused operation credit"); + } + *out = std::allocate_shared<BackendAllocationGrant>(GrantControlAllocator<BackendAllocationGrant>(owner, &credit), + owner, std::move(credit)); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("backend operation credit unavailable"); +} +catch (const std::invalid_argument &) { + return Status::invalid_argument("backend grant requires byte credit"); +} + +bool BackendAllocationGrant::claim(size_t bytes) noexcept +{ + return owner_->claim_from(source_, bytes); +} + +void BackendAllocationGrant::release(size_t bytes) noexcept +{ + owner_->release_to(source_, bytes); +} + +void BackendAllocationGrant::record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept +{ + owner_->record_allocation(kind, bytes, acquired); +} + +const void *BackendAllocationGrant::domain() const noexcept +{ + return owner_->domain(); +} + +void BackendAllocationGrant::seal() noexcept +{ + source_.seal(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_account_grant.h b/lib/crowdb-tree/src/btree/allocation_account_grant.h new file mode 100644 index 000000000..602496862 --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_account_grant.h @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_account.h" +#include "allocation_reservation.h" +#include "btree/allocation_bound.h" + +namespace crowdb::tree::detail +{ +// Per-operation backend capacity. Retained backend objects keep only this +// small budget and their stable account, never the Tree/operation/runtime. +// seal returns unused capacity; allocated nodes/fds keep their actual charge. +class BackendAllocationGrant final : public AllocationBudget +{ + public: + BackendAllocationGrant(std::shared_ptr<AllocationAccount> owner, ResourceBudget::Credit credit); + static Status create(std::shared_ptr<AllocationAccount> owner, ResourceBudget::Credit credit, + std::shared_ptr<BackendAllocationGrant> *out); + + [[nodiscard]] static constexpr size_t allocation_bound() noexcept + { + return shared_allocation_bound<BackendAllocationGrant>(); + } + + [[nodiscard]] bool claim(size_t bytes) noexcept override; + void release(size_t bytes) noexcept override; + void record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept override; + [[nodiscard]] const void *domain() const noexcept override; + void seal() noexcept; + + [[nodiscard]] std::shared_ptr<TreeWorkMetrics> retirement_metrics() const noexcept override + { + return source_.account()->work_metrics(); + } + + [[nodiscard]] const std::shared_ptr<AllocationAccount> &account() const noexcept + { + return owner_; + } + + private: + std::shared_ptr<AllocationAccount> owner_; + AllocationReservation source_; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_bound.h b/lib/crowdb-tree/src/btree/allocation_bound.h new file mode 100644 index 000000000..d8e079300 --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_bound.h @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" +#include "retirement_traits.h" + +#include <cstddef> +#include <limits> +#include <optional> + +namespace crowdb::tree::detail +{ +// Admission arithmetic never wraps into a smaller reservation. Once an +// operand overflows, no later operation can turn the result into a valid bound. +class AllocationBound +{ + public: + explicit AllocationBound(size_t bytes = 0) : bytes_(bytes), valid_(bytes <= kLimit) + { + } + + AllocationBound &add(size_t bytes) noexcept + { + if (!valid_ || bytes > kLimit - bytes_) { + valid_ = false; + } + else { + bytes_ += bytes; + } + return *this; + } + + AllocationBound &add_product(size_t count, size_t bytes) noexcept + { + if (bytes != 0 && count > kLimit / bytes) { + valid_ = false; + return *this; + } + return add(count * bytes); + } + + AllocationBound &add(std::optional<size_t> bytes) noexcept + { + if (!bytes) { + valid_ = false; + return *this; + } + return add(*bytes); + } + + [[nodiscard]] std::optional<size_t> value() const noexcept + { + return valid_ ? std::optional(bytes_) : std::nullopt; + } + + private: + // AllocationReservation uses the high bit to mark a sealed owner. + static constexpr size_t kLimit = std::numeric_limits<size_t>::max() / 2; + size_t bytes_; + bool valid_; +}; + +// Control-block/allocator/alignment allowance for the supported standard +// libraries. Payload and container capacities are counted separately. +template <class T> constexpr size_t shared_allocation_bound() noexcept +{ + const auto storage = sizeof(T) + (RetirementTraits<T>::enabled ? 256 : 128); + return storage; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_observation.cpp b/lib/crowdb-tree/src/btree/allocation_observation.cpp new file mode 100644 index 000000000..9e2ba0cf4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_observation.cpp @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_reservation.h" +#include "tree_work_counters.h" + +#include <cassert> + +namespace crowdb::tree::detail +{ +void AllocationReservation::record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept +{ + const auto index = static_cast<size_t>(kind); + if (kind == AllocationKind::kUnknown || index >= account_->physical_.size()) { + return; + } + // The I/O admission partition counts operations, not storage bytes. + assert(category_ != static_cast<size_t>(BudgetClass::kIo)); + auto &value = account_->physical_[index]; + if (acquired) { + value.fetch_add(bytes, std::memory_order_relaxed); + } + else { + [[maybe_unused]] const auto previous = value.fetch_sub(bytes, std::memory_order_relaxed); + assert(previous >= bytes); + record_work(account_->work_metrics(), TreeWorkCounter::MemoryReclaimedBytes, bytes, TreeWorkCause::Allocation); + } +} + +void ResourceBudget::record_allocation(const std::shared_ptr<Account> &account, AllocationKind kind, size_t bytes, + bool acquired) noexcept +{ + const auto index = static_cast<size_t>(kind); + if (!account || kind == AllocationKind::kUnknown || index >= account->physical_.size()) { + return; + } + if (acquired) { + account->physical_[index].fetch_add(bytes, std::memory_order_relaxed); + } + else { + [[maybe_unused]] const auto previous = account->physical_[index].fetch_sub(bytes, std::memory_order_relaxed); + assert(previous >= bytes); + record_work(account->work_metrics(), TreeWorkCounter::MemoryReclaimedBytes, bytes, TreeWorkCause::Allocation); + } +} + +bool ResourceBudget::Credit::mark_allocation(AllocationKind kind) noexcept +{ + const auto index = static_cast<size_t>(kind); + if (!account_ || !allocated_ || category_ == static_cast<size_t>(BudgetClass::kIo) || + kind_ != AllocationKind::kUnknown || kind == AllocationKind::kUnknown || index >= account_->physical_.size()) { + return false; + } + kind_ = kind; + account_->physical_[index].fetch_add(amount_, std::memory_order_relaxed); + return true; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_reservation.cpp b/lib/crowdb-tree/src/btree/allocation_reservation.cpp new file mode 100644 index 000000000..fcb837fda --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_reservation.cpp @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_reservation.h" + +#include <algorithm> +#include <cassert> +#include <stdexcept> + +namespace crowdb::tree::detail +{ +namespace +{ +// allocate_shared first allocates its control/object storage, then moves the +// remaining credit into AllocationReservation. Only allocate() borrows that +// stack credit; deallocate() uses the independent token embedded in storage. +template <class T> class CreditAllocator +{ + struct Header + { + ResourceBudget::Credit allocation; + }; + + static constexpr size_t alignment = std::max(alignof(T), alignof(Header)); + static constexpr size_t offset = (sizeof(Header) + alignment - 1) & ~(alignment - 1); + + public: + using value_type = T; + + explicit CreditAllocator(ResourceBudget::Credit *source) : source_(source) + { + } + + template <class U> CreditAllocator(const CreditAllocator<U> &other) : source_(other.source()) + { + } + + [[nodiscard]] ResourceBudget::Credit *source() const noexcept + { + return source_; + } + + [[nodiscard]] T *allocate(size_t count) + { + if (count > (SIZE_MAX - offset) / sizeof(T)) { + throw std::bad_array_new_length(); + } + const auto bytes = (count * sizeof(T)) + offset; + auto credit = source_->allocate(bytes); + if (!credit) { + throw std::bad_alloc(); + } + auto *base = static_cast<std::byte *>(::operator new(bytes, std::align_val_t{alignment})); + [[maybe_unused]] const bool marked = credit->mark_allocation(AllocationKind::kMetadata); + assert(marked); + std::construct_at(reinterpret_cast<Header *>(base), Header{std::move(*credit)}); + return reinterpret_cast<T *>(base + offset); + } + + void deallocate(T *pointer, size_t /*count*/) noexcept + { + auto *base = reinterpret_cast<std::byte *>(pointer) - offset; + auto *header = std::launder(reinterpret_cast<Header *>(base)); + auto credit = std::move(header->allocation); + std::destroy_at(header); + ::operator delete(base, std::align_val_t{alignment}); + } + + template <class U> bool operator==(const CreditAllocator<U> &other) const noexcept + { + return source_ == other.source(); + } + + private: + ResourceBudget::Credit *source_; +}; +} // namespace + +std::shared_ptr<AllocationReservation> AllocationReservation::create(ResourceBudget::Credit credit) +{ + if (!credit.account_ || credit.allocated_ || credit.category_ == static_cast<size_t>(BudgetClass::kIo)) { + throw std::invalid_argument("reservation bootstrap requires unconsumed byte credit"); + } + return std::allocate_shared<AllocationReservation>(CreditAllocator<AllocationReservation>(&credit), + std::move(credit)); +} + +std::shared_ptr<AllocationReservation> +AllocationReservation::create(const std::shared_ptr<AllocationReservation> &parent, size_t bytes) +{ + auto credit = parent ? parent->take_reserved(bytes) : std::nullopt; + if (!credit) { + throw std::bad_alloc(); + } + try { + return std::allocate_shared<AllocationReservation>(MetadataAllocator<AllocationReservation>(parent), + std::move(*credit), parent); + } + catch (...) { + if (credit->amount_ != 0 && parent->restore_reserved(credit->amount_)) { + credit->amount_ = 0; + } + throw; + } +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/allocation_reservation.h b/lib/crowdb-tree/src/btree/allocation_reservation.h new file mode 100644 index 000000000..caf83b324 --- /dev/null +++ b/lib/crowdb-tree/src/btree/allocation_reservation.h @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" +#include "resource_budget.h" +#include "retired_shared.h" + +#include <limits> +#include <new> +#include <utility> +#include <vector> + +namespace crowdb::tree::detail +{ +// A whole-operation reservation can be consumed by independent workers without +// another budget wait. Allocators retain this owner across publication and old +// views. seal() returns unused credit; subsequent frees return allocated credit +// to the host. Before sealing, freed bytes are reusable by a repair attempt. +class AllocationReservation : public AllocationBudget +{ + public: + explicit AllocationReservation(ResourceBudget::Credit credit, std::shared_ptr<AllocationReservation> parent = {}); + // Bootstrap a byte reservation from its own credit. Its shared control + // allocation remains charged even after destruction while weak refs exist. + [[nodiscard]] static std::shared_ptr<AllocationReservation> create(ResourceBudget::Credit credit); + // Retain independent progress after the parent seals; return unused credit + // and later frees to that parent's window while it remains open. + [[nodiscard]] static std::shared_ptr<AllocationReservation> + create(const std::shared_ptr<AllocationReservation> &parent, size_t bytes); + ~AllocationReservation() override; + AllocationReservation(const AllocationReservation &) = delete; + AllocationReservation &operator=(const AllocationReservation &) = delete; + + [[nodiscard]] bool claim(size_t bytes) noexcept override; + void release(size_t bytes) noexcept override; + void record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept override; + void seal() noexcept; + [[nodiscard]] size_t available() const noexcept; + // Carve out independent progress credit without changing physical usage. + [[nodiscard]] std::optional<ResourceBudget::Credit> take_reserved(size_t bytes) noexcept; + [[nodiscard]] const void *domain() const noexcept override; + + [[nodiscard]] const std::shared_ptr<ResourceBudget::Account> &account() const noexcept + { + return account_; + } + + [[nodiscard]] std::shared_ptr<TreeResourceMetrics> resource_metrics() const noexcept override + { + return account_->resource_metrics(); + } + + [[nodiscard]] std::shared_ptr<TreeWorkMetrics> retirement_metrics() const noexcept override + { + return account_->retirement_metrics_.lock(); + } + + [[nodiscard]] RetirementCompletion retirement_started() noexcept override + { + account_->retirement_pending_.fetch_add(1, std::memory_order_acq_rel); + return {.owner = account_, .finish = [](const void *raw) noexcept { + const auto &account = *static_cast<const ResourceBudget::Account *>(raw); + account.retirement_pending_.fetch_sub(1, std::memory_order_acq_rel); + account.retirement_pending_.notify_all(); + }}; + } + + [[nodiscard]] std::shared_ptr<RetirementQueue> retirement_queue() const noexcept override + { + return account_->retirement_.lock(); + } + + [[nodiscard]] BudgetClass category() const noexcept + { + return static_cast<BudgetClass>(category_); + } + + private: + bool restore_reserved(size_t bytes) noexcept; + static constexpr size_t kSealed = size_t{1} << (std::numeric_limits<size_t>::digits - 1); + std::shared_ptr<ResourceBudget::Account> account_; + size_t category_; + std::atomic<size_t> available_; + std::shared_ptr<AllocationReservation> parent_; +}; + +template <class T, AllocationKind Kind = AllocationKind::kMetadata> +using ReservationAllocator = BudgetAllocator<T, AllocationReservation, Kind>; +template <class T, AllocationKind Kind = AllocationKind::kMetadata> +using ReservedVector = std::vector<T, ReservationAllocator<T, Kind>>; + +template <class T> struct ReservationDeleter +{ + std::shared_ptr<AllocationReservation> owner; + + void operator()(T *value) const noexcept + { + if (value != nullptr) { + ReservationAllocator<T> allocator(owner); + std::destroy_at(value); + allocator.deallocate(value, 1); + } + } +}; + +template <class T> using ReservedUnique = std::unique_ptr<T, ReservationDeleter<T>>; + +template <class T, class... Args> +ReservedUnique<T> reserved_unique(const std::shared_ptr<AllocationReservation> &owner, Args &&...args) +{ + ReservationAllocator<T> allocator(owner); + auto *value = allocator.allocate(1); + try { + std::construct_at(value, std::forward<Args>(args)...); + } + catch (...) { + allocator.deallocate(value, 1); + throw; + } + return ReservedUnique<T>(value, ReservationDeleter<T>{owner}); +} + +template <class T, class... Args> +std::shared_ptr<T> reserved_shared(const std::shared_ptr<AllocationReservation> &owner, Args &&...args) +{ + if constexpr (RetirementTraits<T>::enabled) { + if (owner) { + if (auto queue = owner->retirement_queue()) { + return retired_shared<T>(owner, queue, std::forward<Args>(args)...); + } + } + } + return std::allocate_shared<T>(ReservationAllocator<T>(owner), std::forward<Args>(args)...); +} + +// Allows a class factory to construct a private operation handle while both +// its object and shared control block remain charged to the same reservation. +template <class T, class Construct> +std::shared_ptr<T> reserved_construct(const std::shared_ptr<AllocationReservation> &owner, Construct &&construct) +{ + ReservationAllocator<T> allocator(owner); + auto *value = allocator.allocate(1); + try { + std::forward<Construct>(construct)(value); + } + catch (...) { + allocator.deallocate(value, 1); + throw; + } + // Keep raw storage guarded until shared_ptr owns its failure cleanup. + // shared_ptr invokes the deleter if allocating its control block fails. + return std::shared_ptr<T>(value, ReservationDeleter<T>{owner}, allocator); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/apply_dispatcher.cpp b/lib/crowdb-tree/src/btree/apply_dispatcher.cpp new file mode 100644 index 000000000..fc83f0959 --- /dev/null +++ b/lib/crowdb-tree/src/btree/apply_dispatcher.cpp @@ -0,0 +1,125 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "apply_dispatcher.h" + +#include "allocation_bound.h" + +#include <cassert> + +namespace crowdb::tree +{ +ApplyDispatcher::Node::~Node() +{ + assert(!queued_.load(std::memory_order_acquire)); +} + +ApplyDispatcher::ApplyDispatcher(std::shared_ptr<detail::FlushRuntime> runtime, size_t steps_per_turn) + : runtime_(std::move(runtime)), + steps_per_turn_(steps_per_turn) +{ +} + +size_t ApplyDispatcher::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<ApplyDispatcher>() + detail::FlushRuntime::progress_allocation_bound(); +} + +Status ApplyDispatcher::create(std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, size_t steps_per_turn, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyDispatcher> *out) +try { + if (!runtime || !lane || !metadata || steps_per_turn == 0 || out == nullptr || *out) { + return Status::invalid_argument("apply dispatcher requires a bounded runtime and charged storage"); + } + auto dispatcher = detail::reserved_construct<ApplyDispatcher>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) ApplyDispatcher(std::move(runtime), steps_per_turn); + }); + auto status = dispatcher->runtime_->reserve_progress(std::move(lane), &ApplyDispatcher::run, dispatcher.get(), + metadata, &dispatcher->progress_); + if (!status.ok()) { + return status; + } + *out = std::move(dispatcher); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("apply dispatcher allocation failed"); +} + +void ApplyDispatcher::enqueue(Node *node) noexcept +{ + node->next_.store(nullptr, std::memory_order_relaxed); + auto *previous = tail_.exchange(node, std::memory_order_acq_rel); + previous->next_.store(node, std::memory_order_release); +} + +void ApplyDispatcher::request(Node &node, std::shared_ptr<const void> owner) noexcept +{ + assert(owner && node.task_ != nullptr); + if (node.queued_.exchange(true, std::memory_order_acq_rel)) { + return; + } + node.owner_ = std::move(owner); + enqueue(&node); + progress_->request(weak_from_this().lock()); +} + +ApplyDispatcher::Node *ApplyDispatcher::pop() noexcept +{ + auto *head = head_; + auto *next = head->next_.load(std::memory_order_acquire); + if (head == &stub_) { + if (next == nullptr) { + return nullptr; + } + head_ = next; + head = next; + next = next->next_.load(std::memory_order_acquire); + } + if (next != nullptr) { + head_ = next; + return head; + } + if (head != tail_.load(std::memory_order_acquire)) { + // A producer has exchanged tail but has not linked its predecessor. + // That producer requests another CPU turn after publishing the link. + return nullptr; + } + enqueue(&stub_); + next = head->next_.load(std::memory_order_acquire); + if (next != nullptr) { + head_ = next; + return head; + } + return nullptr; +} + +void ApplyDispatcher::run(void *context) noexcept +{ + auto &dispatcher = *static_cast<ApplyDispatcher *>(context); + for (size_t step = 0; step < dispatcher.steps_per_turn_; ++step) { + auto *node = dispatcher.pop(); + if (node == nullptr) { + return; + } + auto owner = std::move(node->owner_); + const auto task = node->task_; + auto *argument = node->context_; + // Coalesced producers publish operation state before observing this + // claim. Acquire those publications before executing the node. + node->queued_.exchange(false, std::memory_order_acq_rel); + task(argument); + } + dispatcher.progress_->request(dispatcher.weak_from_this().lock()); +} + +void ApplyDispatcher::close_and_drain() noexcept +{ + // Internal continuations may request the next bounded turn during drain. + // Close the registration only after that finite accepted queue empties. + progress_->drain(); + progress_->close_and_drain(); + assert(head_ == &stub_ && tail_.load(std::memory_order_acquire) == &stub_); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/apply_dispatcher.h b/lib/crowdb-tree/src/btree/apply_dispatcher.h new file mode 100644 index 000000000..e3673e42a --- /dev/null +++ b/lib/crowdb-tree/src/btree/apply_dispatcher.h @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "flush_runtime.h" + +namespace crowdb::tree +{ +// One prepaid CPU registration per Tree serves all accepted applies. A ready +// operation carries its queue node, so a wakeup neither allocates nor consumes +// another ordinary runtime queue slot while waiting for handoff progress. +class ApplyDispatcher : public std::enable_shared_from_this<ApplyDispatcher> +{ + public: + using Task = detail::FlushRuntime::Task; + + class Node + { + public: + Node(Task task, void *context) : task_(task), context_(context) + { + } + + ~Node(); + Node(const Node &) = delete; + Node &operator=(const Node &) = delete; + + private: + friend class ApplyDispatcher; + std::atomic<Node *> next_{nullptr}; + std::atomic<bool> queued_{false}; + std::shared_ptr<const void> owner_; + Task task_; + void *context_; + }; + + static Status create(std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, size_t steps_per_turn, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyDispatcher> *out); + [[nodiscard]] static size_t allocation_bound() noexcept; + + void request(Node &node, std::shared_ptr<const void> owner) noexcept; + // Host-only, after accepted apply owners and their wakeup producers join. + void close_and_drain() noexcept; + + private: + ApplyDispatcher(std::shared_ptr<detail::FlushRuntime> runtime, size_t steps_per_turn); + static void run(void *context) noexcept; + void enqueue(Node *node) noexcept; + [[nodiscard]] Node *pop() noexcept; + + std::shared_ptr<detail::FlushRuntime> runtime_; + std::shared_ptr<detail::FlushRuntime::Progress> progress_; + const size_t steps_per_turn_; + Node stub_{nullptr, nullptr}; + std::atomic<Node *> tail_{&stub_}; + Node *head_ = &stub_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/apply_frontier.cpp b/lib/crowdb-tree/src/btree/apply_frontier.cpp new file mode 100644 index 000000000..b0bb09ba3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/apply_frontier.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "apply_frontier.h" + +#include "allocation_bound.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +void publish_max(std::atomic<uint64_t> &target, uint64_t value) noexcept +{ + auto previous = target.load(std::memory_order_acquire); + while (previous < value && + !target.compare_exchange_weak(previous, value, std::memory_order_acq_rel, std::memory_order_acquire)) { + } +} +} // namespace + +ApplyFrontier::ApplyFrontier(uint64_t initial, size_t window, + const std::shared_ptr<detail::AllocationReservation> &metadata) + : completed_(window, detail::ReservationAllocator<std::atomic<uint64_t>>(metadata)), + contiguous_(initial), + max_seen_(initial) +{ +} + +Status ApplyFrontier::create(uint64_t initial, size_t window, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyFrontier> *out) +{ + if (out == nullptr || !allocation_bound(window)) { + return Status::invalid_argument(); + } + try { + *out = detail::reserved_construct<ApplyFrontier>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) ApplyFrontier(initial, window, metadata); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +std::optional<size_t> ApplyFrontier::allocation_bound(size_t window) noexcept +{ + if (window == 0 || window > kMaxWindow) { + return {}; + } + return detail::AllocationBound(detail::shared_allocation_bound<ApplyFrontier>()) + .add_product(window, sizeof(std::atomic<uint64_t>)) + .value(); +} + +Status ApplyFrontier::check_slot(uint64_t slot) const noexcept +{ + const auto frontier = contiguous_.load(std::memory_order_acquire); + return slot > frontier && slot - frontier > completed_.size() ? Status::resource_exhausted() : Status::Ok(); +} + +Status ApplyFrontier::complete(uint64_t slot) noexcept +{ + auto status = check_slot(slot); + if (!status.ok()) { + return status; + } + publish_max(max_seen_, slot); + if (slot > contiguous_.load(std::memory_order_acquire)) { + // Each location holds the latest completed slot in its ring residue. + // A later slot can enter this residue only after the frontier passed + // its predecessor, so an old duplicate must never clear the location. + publish_max(completed_[slot % completed_.size()], slot); + } + advance(); + return Status::Ok(); +} + +void ApplyFrontier::force_advance(uint64_t slot) noexcept +{ + publish_max(max_seen_, slot); + publish_max(contiguous_, slot); + advance(); +} + +void ApplyFrontier::reset_quiescent(uint64_t slot) noexcept +{ + for (auto &completed : completed_) { + completed.store(0, std::memory_order_relaxed); + } + max_seen_.store(slot, std::memory_order_release); + contiguous_.store(slot, std::memory_order_release); +} + +void ApplyFrontier::advance() noexcept +{ + auto frontier = contiguous_.load(std::memory_order_acquire); + // Later producers may keep extending the frontier. Each caller helps at + // most the window it observed, rather than draining an endless stream. + const auto end = frontier + std::min<uint64_t>(completed_.size(), UINT64_MAX - frontier); + while (frontier < end) { + const auto next = frontier + 1; + if (completed_[next % completed_.size()].load(std::memory_order_acquire) != next) { + return; + } + // The acquire of each completed cell and this release carry the + // corresponding batch's source writes to a captured flush frontier. + if (contiguous_.compare_exchange_weak(frontier, next, std::memory_order_acq_rel, std::memory_order_acquire)) { + frontier = next; + } + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/apply_frontier.h b/lib/crowdb-tree/src/btree/apply_frontier.h new file mode 100644 index 000000000..4e4194fb2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/apply_frontier.h @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" + +namespace crowdb::tree +{ +// Bounded completion tracking for one Tree generation. Admission checks the +// slot distance before touching a source; completing an admitted batch needs +// no allocation. A NoOp completes only its own slot. Replays below the current +// frontier are harmless and cannot overwrite a newer ring generation. +class ApplyFrontier +{ + public: + static constexpr size_t kMaxWindow = 65536; + + static Status create(uint64_t initial, size_t window, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyFrontier> *out); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t window) noexcept; + [[nodiscard]] Status check_slot(uint64_t slot) const noexcept; + [[nodiscard]] Status complete(uint64_t slot) noexcept; + + // Recovery or an explicitly authorized skip of old gaps only. Ordinary + // writes and NoOps use complete(). + void force_advance(uint64_t slot) noexcept; + // Only after generation admission and all accepted operations have drained. + void reset_quiescent(uint64_t slot) noexcept; + + [[nodiscard]] const std::atomic<uint64_t> &contiguous() const noexcept + { + return contiguous_; + } + + [[nodiscard]] uint64_t max_seen() const noexcept + { + return max_seen_.load(std::memory_order_acquire); + } + + private: + ApplyFrontier(uint64_t initial, size_t window, const std::shared_ptr<detail::AllocationReservation> &metadata); + void advance() noexcept; + + detail::ReservedVector<std::atomic<uint64_t>> completed_; + std::atomic<uint64_t> contiguous_; + std::atomic<uint64_t> max_seen_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/apply_progress.cpp b/lib/crowdb-tree/src/btree/apply_progress.cpp new file mode 100644 index 000000000..b64fdbadb --- /dev/null +++ b/lib/crowdb-tree/src/btree/apply_progress.cpp @@ -0,0 +1,167 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "apply_progress.h" + +#include "allocation_bound.h" +#include "host_completion.h" + +#include <cassert> + +namespace crowdb::tree +{ +struct ApplyProgress::State +{ + State(size_t capacity, const std::shared_ptr<detail::AllocationReservation> &metadata) + : waiters(capacity, detail::ReservationAllocator<std::atomic<Request *>>(metadata)) + { + } + + void notify() noexcept + { + if (notifications.fetch_add(1, std::memory_order_acq_rel) != 0) { + return; + } + do { + for (auto &slot : waiters) { + auto *request = slot.load(std::memory_order_acquire); + if (request == nullptr) { + continue; + } + const bool stopped = + closed.load(std::memory_order_acquire) || request->stopped_.load(std::memory_order_acquire); + if (!stopped && sequence.load(std::memory_order_acquire) == request->observed_) { + continue; + } + slot.store(nullptr, std::memory_order_release); + auto callback = request->complete_; + auto *context = request->context_; + request->pending_.store(false, std::memory_order_release); + callback(context, stopped ? detail::CallbackSignal::kStopped : detail::CallbackSignal::kValue, + Status::Ok()); + } + } while (notifications.fetch_sub(1, std::memory_order_acq_rel) != 1); + } + + detail::ReservedVector<std::atomic<Request *>> waiters; + std::atomic<uint64_t> sequence{0}; + std::atomic<size_t> notifications{0}; + std::atomic<bool> closed{false}; +}; + +std::optional<size_t> ApplyProgress::allocation_bound(size_t capacity) noexcept +{ + if (capacity == 0) { + return {}; + } + return detail::AllocationBound(detail::shared_allocation_bound<ApplyProgress>() + + detail::shared_allocation_bound<State>()) + .add_product(capacity, sizeof(std::atomic<Request *>)) + .value(); +} + +Status ApplyProgress::create(size_t capacity, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyProgress> *out) +try { + if (out == nullptr || *out || !metadata || !allocation_bound(capacity)) { + return Status::invalid_argument("apply progress requires bounded waiter storage and credit"); + } + auto state = detail::reserved_shared<State>(metadata, capacity, metadata); + *out = detail::reserved_construct<ApplyProgress>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) ApplyProgress(std::move(state)); }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("apply progress waiter allocation failed"); +} + +uint64_t ApplyProgress::sequence() const noexcept +{ + return state_->sequence.load(std::memory_order_acquire); +} + +Status ApplyProgress::wait(uint64_t observed_sequence) const +{ + struct Completion + { + detail::HostCompletion done; + Status status; + } completion; + + Request request(*this, observed_sequence); + auto callback = [](void *context, detail::CallbackSignal signal, Status status) { + auto &result = *static_cast<Completion *>(context); + result.status = signal == detail::CallbackSignal::kStopped ? Status::unavailable("apply progress closed") + : std::move(status); + result.done.signal(); + }; + Request::submit(&request, callback, &completion); + completion.done.wait(); + return completion.status; +} + +ApplyProgress::~ApplyProgress() +{ + close(); +} + +void ApplyProgress::advance() noexcept +{ + auto state = state_; + state->sequence.fetch_add(1, std::memory_order_acq_rel); + state->notify(); +} + +void ApplyProgress::close() noexcept +{ + auto state = state_; + state->closed.store(true, std::memory_order_release); + state->notify(); +} + +ApplyProgress::Request::Request(const ApplyProgress &progress, uint64_t observed_sequence) + : state_(progress.state_), + observed_(observed_sequence) +{ +} + +ApplyProgress::Request::~Request() +{ + assert(!pending_.load(std::memory_order_acquire)); +} + +detail::CallbackSender ApplyProgress::Request::sender() noexcept +{ + return {this, &Request::submit}; +} + +void ApplyProgress::Request::request_stop() noexcept +{ + auto state = state_; + stopped_.store(true, std::memory_order_release); + state->notify(); +} + +void ApplyProgress::Request::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto &request = *static_cast<Request *>(source); + if (request.submitted_.test_and_set(std::memory_order_acq_rel)) { + complete(context, detail::CallbackSignal::kError, Status::invalid_argument()); + return; + } + auto state = request.state_; + request.complete_ = complete; + request.context_ = context; + request.pending_.store(true, std::memory_order_release); + for (auto &slot : state->waiters) { + Request *expected = nullptr; + if (slot.compare_exchange_strong(expected, &request, std::memory_order_release, std::memory_order_relaxed)) { + // Covers an advance racing before or after publication of this + // waiter; the notifier may destroy the request at terminal. + state->notify(); + return; + } + } + request.pending_.store(false, std::memory_order_release); + complete(context, detail::CallbackSignal::kError, Status::unavailable("apply progress waiter capacity exhausted")); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/apply_progress.h b/lib/crowdb-tree/src/btree/apply_progress.h new file mode 100644 index 000000000..c4f055a02 --- /dev/null +++ b/lib/crowdb-tree/src/btree/apply_progress.h @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" + +namespace crowdb::tree +{ +// Bounded wakeups for accepted apply operations waiting for a source, flush +// attempt or frontier transition. A wakeup carries no mutable operation report: +// the receiver rechecks its own predicate on the shared CPU runtime. +class ApplyProgress +{ + struct State; + + public: + ~ApplyProgress(); + ApplyProgress(const ApplyProgress &) = delete; + ApplyProgress &operator=(const ApplyProgress &) = delete; + + class Request + { + public: + Request(const ApplyProgress &progress, uint64_t observed_sequence); + ~Request(); + Request(const Request &) = delete; + Request &operator=(const Request &) = delete; + + // The accepted operation retains this request and its connected state + // through terminal. Cancellation is an event, not request destruction. + [[nodiscard]] detail::CallbackSender sender() noexcept; + void request_stop() noexcept; + + private: + friend struct State; + friend class ApplyProgress; + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; + uint64_t observed_; + std::atomic<bool> pending_{false}; + std::atomic<bool> stopped_{false}; + std::atomic_flag submitted_ = ATOMIC_FLAG_INIT; + detail::CallbackComplete complete_ = nullptr; + void *context_ = nullptr; + }; + + static Status create(size_t capacity, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyProgress> *out); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t capacity) noexcept; + [[nodiscard]] uint64_t sequence() const noexcept; + // Host-only wait; registers first and rechecks the observed sequence. + [[nodiscard]] Status wait(uint64_t observed_sequence) const; + void advance() noexcept; + void close() noexcept; + + private: + explicit ApplyProgress(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/apply_waiters.cpp b/lib/crowdb-tree/src/btree/apply_waiters.cpp new file mode 100644 index 000000000..2c673dcc4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/apply_waiters.cpp @@ -0,0 +1,29 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "apply_waiters.h" + +namespace crowdb::tree +{ +Status ApplyWaiters::create(const std::shared_ptr<detail::ResourceBudget::Account> &account, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyWaiters> *out) +{ + auto result = detail::reserved_shared<ApplyWaiters>(metadata); + for (auto category : {detail::BudgetClass::kHandoff, detail::BudgetClass::kRouting, detail::BudgetClass::kReclaim, + detail::BudgetClass::kGapRepair, detail::BudgetClass::kIo}) { + auto &slot = result->slots_[static_cast<size_t>(category)]; + slot = detail::ResourceBudget::try_register(account, category); + if (!slot) { + return Status::unavailable("host budget waiter capacity must be admitted before choosing"); + } + } + *out = std::move(result); + return Status::Ok(); +} + +const detail::ResourceBudget::Registration *ApplyWaiters::get(detail::BudgetClass category) const noexcept +{ + const auto &slot = slots_[static_cast<size_t>(category)]; + return slot ? &*slot : nullptr; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/apply_waiters.h b/lib/crowdb-tree/src/btree/apply_waiters.h new file mode 100644 index 000000000..0fed4fc65 --- /dev/null +++ b/lib/crowdb-tree/src/btree/apply_waiters.h @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "allocation_reservation.h" + +namespace crowdb::tree +{ +// One complete wait path, reserved before consensus or at Tree startup. Replay +// borrowers serialize only the current gap; high slots await ApplyProgress. +class ApplyWaiters +{ + public: + static Status create(const std::shared_ptr<detail::ResourceBudget::Account> &account, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyWaiters> *out); + [[nodiscard]] const detail::ResourceBudget::Registration *get(detail::BudgetClass category) const noexcept; + + [[nodiscard]] bool try_acquire() noexcept + { + return !busy_.test_and_set(std::memory_order_acq_rel); + } + + void release() noexcept + { + busy_.clear(std::memory_order_release); + } + + private: + std::array<std::optional<detail::ResourceBudget::Registration>, detail::ResourceBudget::kClasses> slots_; + std::atomic_flag busy_ = ATOMIC_FLAG_INIT; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/backing_identity.cpp b/lib/crowdb-tree/src/btree/backing_identity.cpp new file mode 100644 index 000000000..628ba34de --- /dev/null +++ b/lib/crowdb-tree/src/btree/backing_identity.cpp @@ -0,0 +1,20 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/backing_identity.h" + +#include <atomic> + +namespace crowdb::tree::detail +{ +uint64_t allocate_backend_identity() noexcept +{ + static std::atomic<uint64_t> next{1}; + auto value = next.load(std::memory_order_relaxed); + do { + if (value == UINT64_MAX) { + return 0; + } + } while (!next.compare_exchange_weak(value, value + 1, std::memory_order_relaxed)); + return value; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/backing_identity.h b/lib/crowdb-tree/src/btree/backing_identity.h new file mode 100644 index 000000000..cd52c098f --- /dev/null +++ b/lib/crowdb-tree/src/btree/backing_identity.h @@ -0,0 +1,10 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include <cstdint> + +namespace crowdb::tree::detail +{ +// Zero reports permanent exhaustion; identities never wrap into a live domain. +uint64_t allocate_backend_identity() noexcept; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/backing_metrics.cpp b/lib/crowdb-tree/src/btree/backing_metrics.cpp new file mode 100644 index 000000000..3a3e5a3e4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/backing_metrics.cpp @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backing_metrics.h" + +#include "tree_metrics.h" + +#include <cassert> +#include <utility> + +namespace crowdb::tree +{ +void BackingMetricExtent::activate(const std::shared_ptr<TreeResourceMetrics> &metrics, uint64_t capacity) noexcept +{ + assert(capacity_ == 0); + metrics_ = metrics; + capacity_ = capacity; + allocated_.store(true, std::memory_order_relaxed); + if (metrics) { + const bool added = metrics->add(TreeResourceMetric::AllocatedBackingBytes, capacity); + assert(added); + (void)added; + } +} + +void BackingMetricExtent::mark_deleted() noexcept +{ + if (allocated_.exchange(false, std::memory_order_acq_rel)) { + if (const auto metrics = metrics_.lock()) { + const bool removed = metrics->remove(TreeResourceMetric::AllocatedBackingBytes, capacity_); + assert(removed); + (void)removed; + } + } +} + +BackingMetricExtent::~BackingMetricExtent() +{ + mark_deleted(); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/backing_metrics.h b/lib/crowdb-tree/src/btree/backing_metrics.h new file mode 100644 index 000000000..1f704f9ab --- /dev/null +++ b/lib/crowdb-tree/src/btree/backing_metrics.h @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <array> +#include <atomic> +#include <cstdint> +#include <memory> + +namespace crowdb::tree +{ +class TreeResourceMetrics; + +// Exactly one observation per physical extent or pack. The backend embeds it +// in its paid allocation owner; page views share an alias of that same owner. +class BackingMetricExtent +{ + public: + BackingMetricExtent() = default; + ~BackingMetricExtent(); + BackingMetricExtent(const BackingMetricExtent &) = delete; + BackingMetricExtent &operator=(const BackingMetricExtent &) = delete; + // Ends the physical observation after an authoritative Delete acknowledgement. + void mark_deleted() noexcept; + void activate(const std::shared_ptr<TreeResourceMetrics> &metrics, uint64_t capacity) noexcept; + + private: + std::weak_ptr<TreeResourceMetrics> metrics_; + uint64_t capacity_ = 0; + std::atomic<bool> allocated_{false}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/backing_ref.h b/lib/crowdb-tree/src/btree/backing_ref.h new file mode 100644 index 000000000..8484cc7e9 --- /dev/null +++ b/lib/crowdb-tree/src/btree/backing_ref.h @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <array> +#include <cstdint> +#include <memory> +#include <span> + +namespace crowdb::tree +{ +class BackingMetricExtent; +struct OverflowBackingSet; +struct ChunkPackBacking; +[[nodiscard]] bool chunk_backing_matches(const ChunkPackBacking &, uint64_t domain, uint64_t length) noexcept; +[[nodiscard]] bool same_chunk_backing(const std::shared_ptr<const ChunkPackBacking> &, + const std::shared_ptr<const ChunkPackBacking> &) noexcept; +[[nodiscard]] std::array<uint64_t, 8> chunk_backing_identity(const ChunkPackBacking &) noexcept; +enum class BackingStorage : uint8_t { kDurableBlob, kResidentFrame, kChunkPack }; +enum class BackingDurability : uint8_t { kReloadable, kCheckpointCommitted, kResident }; + +struct BackingCapabilities +{ + bool independent_reload = false; + bool staged_lifetime = false; + + [[nodiscard]] bool can_stage() const + { + return independent_reload && staged_lifetime; + } +}; + +// A readable version, not a promise that a write will eventually finish. The +// backend owner protects every extent/overflow object reachable from this page. +// Only a completed checkpoint changes the durable frontier. +struct BackingRef +{ + bool retire_step() noexcept + { + return true; + } + + uint64_t backend_id = 0; + uint64_t address = 0; + uint64_t length = 0; + uint64_t content_id = 0; + uint64_t generation = 0; + uint64_t checksum = 0; + uint32_t encoding = 0; + BackingDurability durability = BackingDurability::kReloadable; + std::shared_ptr<const void> owner; + // Exact allocation protection, without transitive page dependencies. GC + // can rebind a retained frame to a new dependency graph without retaining + // the old graph. It is never sufficient as a complete primary owner. + std::shared_ptr<const void> physical_owner; + std::shared_ptr<BackingMetricExtent> physical_metrics = nullptr; + // Exact immutable physical dependencies of this primary frame. Keeping a + // leaf readable must not pin unrelated pages from its checkpoint. + std::shared_ptr<const OverflowBackingSet> overflow; + + // An imported raw frame has an allocation owner, not a file extent. It + // never qualifies for checkpoint reuse or backend garbage collection. + BackingStorage storage = BackingStorage::kDurableBlob; + std::span<const uint8_t> resident_frame; + std::shared_ptr<const ChunkPackBacking> chunk; + + [[nodiscard]] bool reloadable() const + { + return length != 0 && owner != nullptr && + (storage == BackingStorage::kDurableBlob || + (storage == BackingStorage::kChunkPack && address == 0 && chunk && + chunk_backing_matches(*chunk, backend_id, length))); + } + + [[nodiscard]] bool readable() const + { + return reloadable() || (storage == BackingStorage::kResidentFrame && owner != nullptr && + !resident_frame.empty() && length == 0 && address == 0 && backend_id == 0); + } +}; + +// A nonwrapping physical interval selected for relocation. A zero length +// leaves ordinary clean-primary and overflow reuse unchanged. +struct BackingExclusion +{ + uint64_t address = 0; + uint64_t length = 0; + + [[nodiscard]] bool overlaps(const BackingRef &backing) const noexcept + { + return length != 0 && backing.storage == BackingStorage::kDurableBlob && backing.reloadable() && + (backing.address >= address ? backing.address - address < length + : address - backing.address < backing.length); + } +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/backing_stats_sampler.cpp b/lib/crowdb-tree/src/btree/backing_stats_sampler.cpp new file mode 100644 index 000000000..23a5232b3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/backing_stats_sampler.cpp @@ -0,0 +1,17 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backing_stats_sampler.h" + +#include "tree_metrics.h" + +namespace crowdb::tree +{ +BackingStatsSnapshot BackingStatsSampler::snapshot() const noexcept +{ + if (!metrics || scope_id == 0) { + return {}; + } + const auto value = metrics->snapshot(); + return {.scope_id = scope_id, .allocated_bytes = value.allocated_backing_bytes}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/backing_stats_sampler.h b/lib/crowdb-tree/src/btree/backing_stats_sampler.h new file mode 100644 index 000000000..07583d307 --- /dev/null +++ b/lib/crowdb-tree/src/btree/backing_stats_sampler.h @@ -0,0 +1,20 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/btree/diagnostics.h" + +#include <memory> + +namespace crowdb::tree +{ +class TreeResourceMetrics; + +// A sampling handle never retains pages, a backend, or an executor. Scope IDs +// distinguish independent ledgers; multiple handles of one scope are not summed. +struct BackingStatsSampler +{ + uint64_t scope_id = 0; + std::shared_ptr<TreeResourceMetrics> metrics; + [[nodiscard]] BackingStatsSnapshot snapshot() const noexcept; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/budget_memory_resource.cpp b/lib/crowdb-tree/src/btree/budget_memory_resource.cpp new file mode 100644 index 000000000..fc55fba28 --- /dev/null +++ b/lib/crowdb-tree/src/btree/budget_memory_resource.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "budget_memory_resource.h" + +#include <algorithm> +#include <stdexcept> + +namespace crowdb::tree::detail +{ +BudgetMemoryResource::BudgetMemoryResource(std::pmr::memory_resource *upstream) : upstream_(upstream) +{ + if (upstream_ == nullptr) { + throw std::invalid_argument("budget memory resource requires an upstream allocator"); + } +} + +BudgetMemoryResource::Scope::Scope(BudgetMemoryResource &resource, std::shared_ptr<AllocationBudget> budget) + : resource_(resource), + previous_(std::exchange(resource.budget_, std::move(budget))) +{ +} + +BudgetMemoryResource::Scope::~Scope() +{ + resource_.budget_ = std::move(previous_); +} + +void *BudgetMemoryResource::do_allocate(size_t bytes, size_t alignment) +{ + alignment = std::max(alignment, alignof(Header)); + if (alignment - 1 > SIZE_MAX - sizeof(Header)) { + throw std::bad_alloc(); + } + const auto offset = (sizeof(Header) + alignment - 1) & ~(alignment - 1); + if (bytes > SIZE_MAX - offset) { + throw std::bad_alloc(); + } + const auto total = bytes + offset; + AllocationLease credit(budget_, total, AllocationKind::kMetadata); + auto *base = static_cast<std::byte *>(upstream_->allocate(total, alignment)); + std::construct_at(reinterpret_cast<Header *>(base), Header{std::move(credit)}); + retained_bytes_.fetch_add(total, std::memory_order_relaxed); + return base + offset; +} + +void BudgetMemoryResource::do_deallocate(void *pointer, size_t bytes, size_t alignment) +{ + alignment = std::max(alignment, alignof(Header)); + const auto offset = (sizeof(Header) + alignment - 1) & ~(alignment - 1); + auto *base = static_cast<std::byte *>(pointer) - offset; + auto *header = reinterpret_cast<Header *>(base); + auto credit = std::move(header->credit); + std::destroy_at(header); + upstream_->deallocate(base, bytes + offset, alignment); + retained_bytes_.fetch_sub(bytes + offset, std::memory_order_relaxed); +} + +bool BudgetMemoryResource::do_is_equal(const std::pmr::memory_resource &other) const noexcept +{ + return this == &other; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/budget_memory_resource.h b/lib/crowdb-tree/src/btree/budget_memory_resource.h new file mode 100644 index 000000000..2cef41495 --- /dev/null +++ b/lib/crowdb-tree/src/btree/budget_memory_resource.h @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" + +#include <atomic> +#include <memory_resource> + +namespace crowdb::tree::detail +{ +// Publication owns allocation admission. Each allocation embeds its original +// credit owner, so later transactions and off-thread retirement can free it +// after that reservation has sealed. No allocator binding escapes a commit. +class BudgetMemoryResource final : public std::pmr::memory_resource +{ + public: + explicit BudgetMemoryResource(std::pmr::memory_resource *upstream = std::pmr::get_default_resource()); + + class Scope + { + public: + Scope(BudgetMemoryResource &resource, std::shared_ptr<AllocationBudget> budget); + ~Scope(); + Scope(const Scope &) = delete; + Scope &operator=(const Scope &) = delete; + + private: + BudgetMemoryResource &resource_; + std::shared_ptr<AllocationBudget> previous_; + }; + + [[nodiscard]] size_t retained_bytes() const noexcept + { + return retained_bytes_.load(std::memory_order_relaxed); + } + + private: + struct Header + { + AllocationLease credit; + }; + + void *do_allocate(size_t bytes, size_t alignment) override; + void do_deallocate(void *pointer, size_t bytes, size_t alignment) override; + [[nodiscard]] bool do_is_equal(const std::pmr::memory_resource &other) const noexcept override; + + std::pmr::memory_resource *upstream_; + std::shared_ptr<AllocationBudget> budget_; + std::atomic<size_t> retained_bytes_{0}; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/checkpoint_metrics.cpp b/lib/crowdb-tree/src/btree/checkpoint_metrics.cpp new file mode 100644 index 000000000..a1b3a30d0 --- /dev/null +++ b/lib/crowdb-tree/src/btree/checkpoint_metrics.cpp @@ -0,0 +1,237 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "btree/tree_metrics.h" + +#include <algorithm> +#include <cassert> +#include <new> + +namespace crowdb::tree +{ +struct CheckpointPendingMetrics::CaptureRetirement +{ + explicit CaptureRetirement(std::pmr::memory_resource *resource, std::shared_ptr<const void> owner = {}) + : resource_owner(std::move(owner)), + pages(resource) + { + } + + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !pages.empty(); ++i) { + pages.erase_first(); + } + return pages.empty(); + } + + std::shared_ptr<const void> resource_owner; + PendingPageTimes pages; +}; + +CheckpointPendingMetrics::CheckpointPendingMetrics(std::pmr::memory_resource *resource) + : pending_(resource), + since_capture_(resource) +{ +} + +CheckpointPendingMetrics::PreparedDelta::PreparedDelta(CheckpointPendingMetrics &owner, size_t max_pages) + : owner_(owner), + edits_(owner.pending_.resource()), + changed_at_ns_(tree_metrics_now_ns()) +{ + edits_.reserve(max_pages); +} + +CheckpointPendingMetrics::PreparedCapture::PreparedCapture(CheckpointPendingMetrics &owner, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<const void> resource_owner) + : owner_(owner), + retired_(metadata && resource_owner + ? detail::reserved_shared<CaptureRetirement>(metadata, owner.pending_.resource(), + std::move(resource_owner)) + : std::allocate_shared<CaptureRetirement>( + std::pmr::polymorphic_allocator<CaptureRetirement>{owner.pending_.resource()}, + owner.pending_.resource())) +{ +} + +std::shared_ptr<const void> CheckpointPendingMetrics::PreparedCapture::commit(uint64_t token) noexcept +{ + assert(owner_.can_capture(token)); + retired_->pages.swap(owner_.since_capture_); + owner_.mapping_since_capture_ = false; + owner_.mapping_capture_since_ns_ = 0; + owner_.token_ = token; + owner_.inflight_ = true; + return std::move(retired_); +} + +CheckpointPendingMetrics::PreparedDelta::~PreparedDelta() +{ + if (!committed_) { + for (const auto &edit : edits_) { + if (edit.inserted_pending) { + owner_.pending_.erase(edit.pid); + } + if (edit.inserted_since) { + owner_.since_capture_.erase(edit.pid); + } + } + } +} + +void CheckpointPendingMetrics::PreparedDelta::change_page(uint64_t pid) +{ + if (edits_.size() == edits_.capacity()) { + throw std::bad_alloc(); + } + const bool inserted = owner_.pending_.insert(pid, changed_at_ns_); + bool inserted_since = false; + try { + if (owner_.inflight_) { + inserted_since = owner_.since_capture_.insert(pid, changed_at_ns_); + } + } + catch (...) { + if (inserted) { + owner_.pending_.erase(pid); + } + throw; + } + edits_.push_back({.pid = pid, .inserted_pending = inserted, .inserted_since = inserted_since}); +} + +void CheckpointPendingMetrics::PreparedDelta::remove_page(uint64_t pid) +{ + if (edits_.size() == edits_.capacity()) { + throw std::bad_alloc(); + } + edits_.push_back({.pid = pid, .removed = true}); + change_mapping(); +} + +void CheckpointPendingMetrics::PreparedDelta::change_mapping() noexcept +{ + mapping_changed_ = true; +} + +void CheckpointPendingMetrics::PreparedDelta::copy_to(TreeLogicalStats &stats) const noexcept +{ + owner_.copy_to(stats); + for (const auto &edit : edits_) { + if (edit.removed && owner_.pending_.contains(edit.pid)) { + --stats.checkpoint_pending_pages; + } + } + stats.checkpoint_mapping_pending = stats.checkpoint_mapping_pending || mapping_changed_; +} + +void CheckpointPendingMetrics::PreparedDelta::commit() noexcept +{ + for (const auto &edit : edits_) { + if (edit.removed) { + owner_.pending_.erase(edit.pid); + owner_.since_capture_.erase(edit.pid); + } + } + if (mapping_changed_) { + owner_.change_mapping(); + } + committed_ = true; +} + +void CheckpointPendingMetrics::change_page(uint64_t pid) +{ + PreparedDelta delta(*this, 1); + delta.change_page(pid); + delta.commit(); +} + +void CheckpointPendingMetrics::remove_page(uint64_t pid) +{ + pending_.erase(pid); + since_capture_.erase(pid); + change_mapping(); +} + +void CheckpointPendingMetrics::change_mapping() noexcept +{ + const auto now = tree_metrics_now_ns(); + if (!mapping_pending_) { + mapping_since_ns_ = now; + } + if (inflight_ && !mapping_since_capture_) { + mapping_capture_since_ns_ = now; + } + mapping_pending_ = true; + mapping_since_capture_ = mapping_since_capture_ || inflight_; +} + +bool CheckpointPendingMetrics::can_capture(uint64_t token) const noexcept +{ + return !inflight_ && token > token_; +} + +bool CheckpointPendingMetrics::can_finish(uint64_t token) const noexcept +{ + return inflight_ && token == token_; +} + +bool CheckpointPendingMetrics::capture(uint64_t token) +{ + if (!can_capture(token)) { + return false; + } + PreparedCapture prepared(*this); + (void)prepared.commit(token); + return true; +} + +bool CheckpointPendingMetrics::finish(uint64_t token, bool committed) noexcept +{ + if (!can_finish(token)) { + return false; + } + if (committed) { + pending_.swap(since_capture_); + mapping_pending_ = mapping_since_capture_; + mapping_since_ns_ = mapping_capture_since_ns_; + } + inflight_ = false; + return true; +} + +void CheckpointPendingMetrics::copy_to(TreeLogicalStats &stats) const noexcept +{ + stats.checkpoint_pending_pages = pending_.size(); + stats.checkpoint_mapping_pending = mapping_pending_; +} + +bool CheckpointPendingMetrics::contains(uint64_t pid) const +{ + return pending_.contains(pid); +} + +bool CheckpointPendingMetrics::retire_step() noexcept +{ + size_t steps = 0; + for (; steps < 64 && !pending_.empty(); ++steps) { + pending_.erase_first(); + } + for (; steps < 64 && !since_capture_.empty(); ++steps) { + since_capture_.erase_first(); + } + return pending_.empty() && since_capture_.empty(); +} + +uint64_t CheckpointPendingMetrics::oldest_since_ns() const noexcept +{ + const auto page_since = pending_.oldest_since_ns(); + if (!mapping_pending_) { + return page_since; + } + return page_since == 0 ? mapping_since_ns_ : std::min(page_since, mapping_since_ns_); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/crowdb-tree.cpp b/lib/crowdb-tree/src/btree/crowdb-tree.cpp index bd8c90341..063990996 100644 --- a/lib/crowdb-tree/src/btree/crowdb-tree.cpp +++ b/lib/crowdb-tree/src/btree/crowdb-tree.cpp @@ -1,619 +1,51 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -// B+tree engine implementation. - -#include "crowdb-tree/crowdb-tree.h" - -#include "async_completion_adapter.h" -#include "crowdb-common/log.h" -#include "crowdb-tree/backend/async_page_store.h" -#include "crowdb-tree/btree/delta.h" -#include "crowdb-tree/btree/descent.h" -#include "crowdb-tree/btree/leaf_cursor.h" -#include "crowdb-tree/maptable/compressor.h" -#include "crowdb-tree/maptable/mapping_slot.h" -#include "merge_sources.h" -#include "native_frames.h" - -#include <algorithm> -#include <chrono> -#include <cstring> -#include <functional> -#include <map> -#include <memory> -#include <ranges> -#include <unordered_map> -#include <unordered_set> -#include <vector> +#include "btree/tree_core.h" +#include "btree/tree_options.h" namespace crowdb::tree { -using detail::NativeBounds; -using detail::set_native_frame_fences; -using detail::validate_native_snapshot_graph; - -namespace -{ - -// Copy a byte range into a fresh owned cell buffer (SBO-inline for small cells). -buffer cell_of(Slice s) -{ - buffer b = buffer::alloc(s.size()); - if (!s.empty()) { - std::memcpy(b.data(), s.data(), s.size()); - } - return b; -} - -inline std::vector<leaf_entry> resolve_chain_sorted(PageBase *head, uint64_t gc_floor) -{ - std::vector<leaf_entry> out; - LeafChainCursor cur(head, gc_floor); - out.reserve(cur.remaining_hint()); - for (; cur.valid(); cur.next()) { - out.push_back({.key = cur.key().to_string(), .cell = cell_of(cur.cell())}); - } - return out; -} - -// Collect all live entries in key order by walking the leaf chain via -// right_sibling, starting at the leftmost leaf (found the same way scan()'s -// range walk does: find_leaf_page_id with an empty key, which compares less -// than every real key). This is a full-tree equivalent of scan()'s -// right-sibling walk, so it inherits the same concurrency-safety argument -// (see scan()'s header comment) -- it does NOT do a top-down parent/children -// DFS, so it is safe to run under only an epoch guard (no write_mutex_) -// concurrently with a split or merge: split_leaf_locked publishes the new -// right half and repoints the parent *before* shrinking the original PID, -// and try_merge_leaf_locked gives the merged page the removed leaf's old -// right_sibling, so a leaf read at any point mid-SMO either still holds its -// full pre-SMO entry set (old right_sibling, no gap) or the new content with -// right_sibling already repointed correctly (no gap, no duplicate). -template <class Resolve> -void collect_in_order(Resolve &&resolve, uint64_t root_page_id, uint64_t gc_floor, std::vector<leaf_entry> *out) -{ - if (root_page_id == kInvalidPageId) { - return; - } - uint64_t page_id = find_leaf_page_id(resolve, root_page_id, Slice()); - while (page_id != kInvalidPageId) { - PageBase *head = resolve(page_id); - if (head == nullptr) { - return; - } - for (auto &e : resolve_chain_sorted(head, gc_floor)) { - out->push_back(std::move(e)); - } - LeafBase *base = chain_leaf_base(head); - page_id = base != nullptr ? base->right_sibling() : kInvalidPageId; - } -} - -// R58: uniform view over a merge source (L0 skip-list cursor, L1 leaf cursor, -// or a drained vector) for the loser tree. Wraps the source types behind a -// common key/slot/advance interface so the tree can compare them generically. - -} // namespace - Crowdbtree::Crowdbtree(Config opt) : opt_(std::move(opt)), name_(opt_.name) { - pool_ = std::make_shared<BufferPool>(opt_.buffer_pool_bytes, opt_.frame_bytes, opt_.page_store); - // Segment recycling (#14b) hands emptied segments to the tree-owned epoch - // manager so a lock-free reader that already loaded a segment pointer - // keeps a valid one until its guard drains. - mapping_.set_epoch_manager(&epoch_); - active_ = std::make_shared<MemTable>(memtable_next_id_.fetch_add(1, std::memory_order_relaxed), &epoch_); - // Initialize with a single empty leaf as the root. - uint64_t page_id = mapping_.allocate_page_id(); - mapping_.store(page_id, LeafBase::build({}, kInvalidPageId, pool_, opt_.frame_bytes)); - root_page_id_.store(page_id); -} - -void Crowdbtree::sync_page_count_gauges() -{ - if (metrics_.tree_leaf_count_g != nullptr) { - metrics_.tree_leaf_count_g->set(leaf_count_.load(std::memory_order_relaxed)); - } - if (metrics_.tree_inner_count_g != nullptr) { - metrics_.tree_inner_count_g->set(inner_count_.load(std::memory_order_relaxed)); - } -} - -Crowdbtree::~Crowdbtree() -{ - auto pending = async_flushes_->load(); - while (pending != 0) { - async_flushes_->wait(pending); - pending = async_flushes_->load(); - } - generation_.close(); - reclamation_owner_->store(nullptr, std::memory_order_release); - active_.reset(); - frozen_.clear(); - split_shared_memtables_.clear(); - detach_native_iterators(); - try { - free_all_resident_pages(/*retire=*/true); - } - catch (...) { // NOLINT(bugprone-empty-catch) - // Destructors must not throw. - } - CRB_LOG_INFO("[{}] close: done last_applied={} contiguous={}", name_, last_applied_slot_.load(), - contiguous_slot_.load()); -} - -void Crowdbtree::retire_page(PageBase *p) -{ - // R6: the deleter sets kRetiredBit instead of deleting outright. If a - // cross-thread pin (get_async handoff / PinnedSnapshot) is outstanding, - // the delete defers to the last unpin(). Otherwise it frees immediately - // (same cost as the old delete). - preserve_native_page_locked(p->page_id, p); - epoch_.retire(p, [](void *ptr) { static_cast<PageBase *>(ptr)->retire_with_pins(); }); -} - -void Crowdbtree::retire_orphaned_page(uint64_t page_id, PageBase *p) -{ - preserve_native_page_locked(page_id, p); - epoch_.retire(p, [owner = reclamation_owner_, page_id](void *ptr) { - if (auto *tree = owner->load(std::memory_order_acquire); - tree != nullptr && tree->mapping_.get_resident(page_id) == ptr) { - tree->mapping_.clear(page_id); - static_cast<PageBase *>(ptr)->retire_with_pins(); - } - // Replacement/teardown owns pages in detached mappings. - }); -} - -PageBase *Crowdbtree::resident(uint64_t page_id) const -{ - map_lookup_total_.fetch_add(1, std::memory_order_relaxed); - if (metrics_.page_find_c != nullptr) { - metrics_.page_find_c->inc(); - } - uint64_t w = mapping_.get_word(page_id); - if (slot_word::is_empty(w) || !slot_word::is_unloaded(w)) { - if (slot_word::is_resident(w)) { - PageBase *v = slot_word::resident_ptr(w); - // CLOCK-informed eviction ranking (plan-tree #17): stamp this - // touch. Relaxed/relaxed: this is a recency *hint*, not a - // synchronization point -- ordering across threads doesn't - // matter, only that concurrent touches keep advancing the - // stamp, which fetch_add guarantees without a lock. - v->last_touch_tick.store(touch_tick_.fetch_add(1, std::memory_order_relaxed), std::memory_order_relaxed); - return v; - } - return nullptr; // hot path / unset - } - // Cold path: demand-load this base page. Serialized by - // load_mutex_; double-checked so only one loader installs. The unloaded - // descriptor is inline in the slot word (no heap allocation), so there is - // no descriptor to free -- just re-read and check. - auto dl_t0 = std::chrono::steady_clock::now(); - std::scoped_lock lk(load_mutex_); - w = mapping_.get_word(page_id); - if (slot_word::is_empty(w) || !slot_word::is_unloaded(w)) { - return slot_word::is_resident(w) ? slot_word::resident_ptr(w) : nullptr; // another loader won - } - demand_load_total_.fetch_add(1, std::memory_order_relaxed); - uint64_t addr = 0; - uint32_t phys_len = 0; - Status location = opt_.page_store->decode_mapping_location(w, &addr, &phys_len); - if (!location.ok()) { - io_failed_.store(true); - return nullptr; - } - // phys_len is the IU-padded physical extent (PT9). The blob header records - // the raw frame length so we size the decoded frame without other state. - std::vector<uint8_t> blob(phys_len); - Status s = opt_.page_store->read_at(addr, blob.data(), blob.size()); - // A demand-load failure (I/O error or CRC mismatch) is a hard media fault for - // a committed page; latch it so callers can detect it (the read still degrades - // to a miss, since the lock-free path can't propagate a Status). - if (!s.ok()) { - CRB_LOG_ERROR("[{}] demand-load I/O fault: pid={} addr={} len={} status={}", name_, page_id, addr, phys_len, - s.to_string()); - io_failed_.store(true); - if (metrics_.page_load_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - dl_t0).count(); - metrics_.page_load_l->observe(static_cast<uint64_t>(ns)); - } - if (metrics_.page_read_bw != nullptr) { - metrics_.page_read_bw->observe(blob.size()); - } - return nullptr; - } - if (metrics_.page_load_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - dl_t0).count(); - metrics_.page_load_l->observe(static_cast<uint64_t>(ns)); - } - if (metrics_.page_read_bw != nullptr) { - metrics_.page_read_bw->observe(blob.size()); - } - return install_loaded_page(page_id, addr, phys_len, blob); -} - -PageBase *Crowdbtree::install_loaded_page(uint64_t page_id, uint64_t addr, uint32_t /*plen*/, - const std::vector<uint8_t> &blob) const -{ - uint32_t raw_len = durable_blob_raw_len(blob.data(), blob.size()); - if (raw_len == 0) { - CRB_LOG_ERROR("[{}] demand-load corrupt blob (raw_len=0): pid={} addr={}", name_, page_id, addr); - io_failed_.store(true); - return nullptr; - } - std::vector<uint8_t> frame(raw_len); - if (!decode_durable_page(blob.data(), blob.size(), frame.data(), raw_len).ok()) { - CRB_LOG_ERROR("[{}] demand-load decode failed: pid={} addr={} raw_len={}", name_, page_id, addr, raw_len); - io_failed_.store(true); - return nullptr; - } - // A retained split parent can reopen the original physical root with a - // narrower logical range. Range checks belong to requests; inherited - // frames remain valid until background pruning removes their other keys. - if (!frame_validate(frame.data(), raw_len)) { - CRB_LOG_ERROR("[{}] demand-load frame validation failed: pid={} addr={}", name_, page_id, addr); - io_failed_.store(true); - return nullptr; - } - page_type ft = frame_page_type(frame.data()); - PageBase *page = nullptr; - if (ft == page_type::kLeafBase) { - page = LeafBase::from_frame_copy(frame.data(), raw_len, pool_, opt_.frame_bytes); - } - else if (ft == page_type::kInnerBase) { - page = InnerBase::from_frame_copy(frame.data(), raw_len, pool_, opt_.frame_bytes); - } - else { // kOverflowFrame - page = OverflowBase::from_frame_copy(frame.data(), raw_len, pool_, opt_.frame_bytes); - } - page->page_id = page_id; - page->durable_addr = addr; // loaded from here -> clean - // durable_plen is the logical (unpadded) blob length, recovered from the - // blob header rather than the IU-padded physical extent `plen` (which is - // iu_count * iu from the packed slot word). This is what the manifest - // records and what store_unloaded re-tags. - page->durable_plen = durable_blob_logical_len(blob.data(), blob.size()); - page->last_touch_tick.store(touch_tick_.fetch_add(1, std::memory_order_relaxed), std::memory_order_relaxed); - const_cast<MappingTable &>(mapping_).store(page_id, page); // publish resident - return page; -} - -void Crowdbtree::free_subtree(uint64_t page_id, bool retire) -{ - uint64_t w = mapping_.get_word(page_id); - // Skip unset and *unloaded* slots: an unloaded slot has no heap page to free - // (the descriptor is inline in the word); its subtree was never loaded. - if (slot_word::is_empty(w) || slot_word::is_unloaded(w)) { - return; - } - PageBase *head = slot_word::resident_ptr(w); - // Resolve to the base node to learn the page kind / children. - PageBase *base = head; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - if (base != nullptr && base->type == page_type::kInnerBase) { - auto *inner = static_cast<InnerBase *>(base); - for (uint64_t child : inner->children()) { - free_subtree(child, retire); - } - } - else if (base != nullptr && base->type == page_type::kLeafBase) { - // Free the overflow chains referenced by this leaf's pointer cells (they are - // not reachable via child PIDs). Deltas above carry inline values only. - LeafFrameView v = static_cast<LeafBase *>(base)->view(); - for (uint32_t i = 0; i < v.count(); ++i) { - CellView c{v.cell(i)}; - if (c.is_overflow()) { - if (retire) { - retire_overflow_chain_locked(c.overflow_head()); - } - else { - free_overflow_chain(c.overflow_head()); - } - } - } - } - if (retire) { - // Live tree (install_snapshot): clear the slot first so a new reader sees - // "gone", then epoch-retire each node in the chain. A reader that already - // loaded a node keeps using it under its guard; the frame is freed only once - // that guard drains. - mapping_.clear(page_id); - PageBase *n = head; - while (n != nullptr) { - PageBase *next = n->next; - retire_page(n); - n = next; - } - } - else { - // Teardown / clear: no concurrent readers, delete the chain immediately. - PageBase *n = head; - while (n != nullptr) { - PageBase *next = n->next; - delete n; - n = next; - } - mapping_.clear(page_id); - } -} - -void Crowdbtree::free_all_resident_pages(bool retire) -{ - // Segment-scan, not a root->children walk (see free_subtree's caution - // comment on crowdb-tree.h for why that matters): every present segment's - // slots are inspected directly, so a resident leaf/inner/overflow page is - // found and freed regardless of whether any of its ancestors -- or, for - // an overflow page, the leaf that spilled it -- happen to be unloaded. - // This also means, unlike free_subtree, there is no need to separately - // walk a leaf's cells to find its overflow chains: every overflow page - // has its own mapping slot (spill_value_to_overflow_chain_locked stores - // each one under its own allocated PID), so the scan below visits it - // directly too. - // - // Two passes, like prepare_snapshot_locked's own segment scan: pass 1 - // only *reads* segment_at()/slots[i] to collect (page_id, head) pairs, - // never mutating anything, so it can never race MappingSegment recycling - // (#14b) -- clearing a slot in pass 2 below can bring a segment's - // live_count to 0 and epoch-retire the MappingSegment itself; mutating - // while *this* function's own scan is still walking that same segment's - // `seg->slots[]` would risk exactly the kind of dangling-segment-pointer - // access #14b's own design guards against elsewhere. - struct ResidentEntry - { - uint64_t page_id; - PageBase *head; - }; - - std::vector<ResidentEntry> resident; - for (uint64_t seg_idx = 0; seg_idx < MappingTable::kMaxSegments; ++seg_idx) { - MappingSegment *seg = mapping_.segment_at(seg_idx); - if (seg == nullptr) { - continue; - } - for (uint32_t i = 0; i < seg->slot_count; ++i) { - uint64_t w = seg->slots[i].load(std::memory_order_relaxed); - if (slot_word::is_resident(w)) { - resident.push_back( - {.page_id = (seg_idx * MappingTable::kSegmentSize) + i, .head = slot_word::resident_ptr(w)}); - } - } - } - for (const auto &e : resident) { - if (retire) { - // Live tree (install_snapshot(_native)): clear the slot first so a - // new reader sees "gone", then epoch-retire each node in the chain - // -- same ordering as free_subtree's retire=true path, and for the - // same reason (a reader that already loaded a node keeps using it - // under its guard; the frame is freed only once that guard drains). - mapping_.clear(e.page_id); - for (PageBase *n = e.head; n != nullptr;) { - PageBase *next = n->next; - retire_page(n); - n = next; - } - } - else { - // Teardown: no concurrent readers, but a PinnedSnapshot may still - // hold refcount pins on these pages (R6). Use retire_with_pins() - // instead of delete: if pins are outstanding, the delete defers - // to the last unpin; if no pins, it frees immediately (same cost - // as delete). - for (PageBase *n = e.head; n != nullptr;) { - PageBase *next = n->next; - n->retire_with_pins(); - n = next; - } - mapping_.clear(e.page_id); - } + auto status = configure_tree_options(&opt_); + if (status.ok() && opt_.page_store != nullptr) { + status = TreeCore::open(opt_, &core_); } -} - -size_t Crowdbtree::evict_clean_leaves_locked(size_t max_resident_leaves) -{ - // Collect resident, delta-free, clean leaf pids (the evictable set, §4.6). - // Descend only into already-resident inner children — never demand-load a page - // just to evict it. - // (page_id, last_touch_tick) so the candidate set can be ranked by real - // access recency below (plan-tree #17) instead of arbitrary DFS order. - std::vector<std::pair<uint64_t, uint64_t>> evictable_ranked; - std::function<void(uint64_t)> dfs = [&](uint64_t page_id) { - uint64_t wv = mapping_.get_word(page_id); - if (slot_word::is_empty(wv) || slot_word::is_unloaded(wv)) { - return; - } - PageBase *v = slot_word::resident_ptr(wv); - PageBase *base = v; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; + else if (status.ok()) { + std::shared_ptr<TreeRuntimeBinding> binding; + status = TreeRuntimeBinding::create(opt_.runtime, opt_.runtime_binding, &binding); + if (status.ok()) { + status = TreeCore::create(opt_, binding, {}, &core_); } - if (base == nullptr) { - return; - } - if (base->type == page_type::kLeafBase) { - // Clean (durable bytes match) and no deltas above (v == base) ⇒ evictable. - if (v == base && v->durable_addr != kNoAddr) { - evictable_ranked.emplace_back(page_id, v->last_touch_tick.load(std::memory_order_relaxed)); - } - return; - } - for (uint64_t c : static_cast<InnerBase *>(base)->children()) { - uint64_t cw = mapping_.get_word(c); - if (slot_word::is_resident(cw)) { - dfs(c); - } - } - }; - dfs(root_page_id_.load()); - - if (evictable_ranked.size() <= max_resident_leaves) { - return 0; } - // Oldest-touched first: evict genuinely cold pages ahead of recently - // accessed ones, rather than whichever DFS happened to visit first. - std::ranges::sort(evictable_ranked.begin(), evictable_ranked.end(), - [](const auto &a, const auto &b) { return a.second < b.second; }); - size_t to_evict = evictable_ranked.size() - max_resident_leaves; - size_t evicted = 0; - for (const auto &[page_id, tick] : evictable_ranked) { - if (evicted >= to_evict) { - break; - } - uint64_t wv = mapping_.get_word(page_id); // re-check (belt-and-suspenders; we hold write_mutex_) - if (slot_word::is_empty(wv) || slot_word::is_unloaded(wv)) { - continue; - } - PageBase *v = slot_word::resident_ptr(wv); - if (v->type != page_type::kLeafBase || v->durable_addr == kNoAddr) { - continue; - } - // Evict this leaf's overflow chains too, so their pages don't orphan - // (resident but unreachable from the now-unloaded leaf). - LeafFrameView lv = static_cast<LeafBase *>(v)->view(); - for (uint32_t i = 0; i < lv.count(); ++i) { - CellView c{lv.cell(i)}; - if (c.is_overflow()) { - evict_overflow_chain_locked(c.overflow_head()); - } - } - // Re-tag the slot unloaded, then epoch-retire the resident page. A reader - // that already loaded `v` keeps using it under its guard (frame freed only - // once that guard drains); a later reader sees the tag and demand-loads. - uint64_t unloaded = slot_word::kEmpty; - if (!opt_.page_store->encode_mapping_location(v->durable_addr, v->durable_plen, &unloaded).ok()) { - io_failed_.store(true); - continue; - } - mapping_.store_word(page_id, unloaded); - retire_page(v); - ++evicted; + if (!status.ok()) { + throw std::runtime_error(status.to_string()); } - return evicted; -} - -size_t Crowdbtree::evict_clean_leaves(size_t max_resident_leaves) -{ - std::scoped_lock lk(write_mutex_); - return evict_clean_leaves_locked(max_resident_leaves); + core_facade_charge_ = + AllocationLease(core_->binding()->lifecycle_budget(), sizeof(Crowdbtree), AllocationKind::kMetadata); } -// plan-tree #17 D3: inner bases get their *own* ranked budget/pass, entirely -// separate from evict_clean_leaves_locked's. An earlier attempt shared one -// combined ranked list between leaves and inner bases and broke -// Eviction.RecentlyTouchedLeafSurvivesEvictionOverColderOnes: a get() stamps -// last_touch_tick on every page it walks through, leaf *and* ancestor inner -// nodes alike, all in the same call -- a single combined budget can rank an -// ancestor behind some other, unrelated leaf and evict it, forcing an -// unwanted demand-load on the very next access to a leaf the test expects to -// stay fully resident with zero extra reads. Keeping the two passes disjoint -// means the leaf test's shared-budget contention can never happen: this -// function never evicts a kLeafBase, and evict_clean_leaves_locked never -// evicts a kInnerBase. -size_t Crowdbtree::evict_clean_inner_locked(size_t max_resident_inner) +Crowdbtree::Crowdbtree(Config opt, std::shared_ptr<TreeCore> core) + : opt_(std::move(opt)), + name_(opt_.name), + core_(std::move(core)), + core_facade_charge_(core_->binding()->lifecycle_budget(), sizeof(Crowdbtree), AllocationKind::kMetadata) { - // Same DFS shape as evict_clean_leaves_locked (descend only into already- - // resident children -- never demand-load a page just to evict it), but - // collecting kInnerBase candidates instead of kLeafBase ones. - std::vector<std::pair<uint64_t, uint64_t>> evictable_ranked; - std::function<void(uint64_t)> dfs = [&](uint64_t page_id) { - uint64_t wv = mapping_.get_word(page_id); - if (slot_word::is_empty(wv) || slot_word::is_unloaded(wv)) { - return; - } - PageBase *v = slot_word::resident_ptr(wv); - PageBase *base = v; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - if (base == nullptr || base->type != page_type::kInnerBase) { - return; // a leaf base: nothing to descend into, nothing to collect here - } - // Clean (durable bytes match) and no deltas above (v == base) ⇒ - // evictable. Inner bases are never delta-chained today (split/merge - // always mapping_.store()s a fresh consolidated InnerBase), but this - // mirrors the leaf pass's check rather than assuming that invariant. - if (v == base && v->durable_addr != kNoAddr) { - evictable_ranked.emplace_back(page_id, v->last_touch_tick.load(std::memory_order_relaxed)); - } - for (uint64_t c : static_cast<InnerBase *>(base)->children()) { - uint64_t cw = mapping_.get_word(c); - if (slot_word::is_resident(cw)) { - dfs(c); - } - } - }; - dfs(root_page_id_.load()); - - if (evictable_ranked.size() <= max_resident_inner) { - return 0; - } - // Oldest-touched first, same rationale as the leaf pass. - std::ranges::sort(evictable_ranked.begin(), evictable_ranked.end(), - [](const auto &a, const auto &b) { return a.second < b.second; }); - size_t to_evict = evictable_ranked.size() - max_resident_inner; - size_t evicted = 0; - for (const auto &[page_id, tick] : evictable_ranked) { - if (evicted >= to_evict) { - break; - } - uint64_t wv = mapping_.get_word(page_id); // re-check (belt-and-suspenders; we hold write_mutex_) - if (slot_word::is_empty(wv) || slot_word::is_unloaded(wv)) { - continue; - } - PageBase *v = slot_word::resident_ptr(wv); - if (v->type != page_type::kInnerBase || v->durable_addr == kNoAddr) { - continue; - } - // Re-tag the slot unloaded, then epoch-retire the resident page -- same - // mechanism as the leaf pass; a reader that already loaded `v` keeps - // using it under its guard, a later reader demand-loads. - uint64_t unloaded = slot_word::kEmpty; - if (!opt_.page_store->encode_mapping_location(v->durable_addr, v->durable_plen, &unloaded).ok()) { - io_failed_.store(true); - continue; - } - mapping_.store_word(page_id, unloaded); - retire_page(v); - ++evicted; - } - return evicted; } -size_t Crowdbtree::evict_clean_inner(size_t max_resident_inner) +Crowdbtree::~Crowdbtree() { - std::scoped_lock lk(write_mutex_); - return evict_clean_inner_locked(max_resident_inner); + core_->close_and_drain(); } -void Crowdbtree::maybe_evict_locked() +void Crowdbtree::set_gc_watermark(uint64_t snapshot_slot, uint64_t safe_slot) { - if (!pool_) { - return; - } - BufferPool::Stats st = pool_->stats(); - if (st.num_frames == 0) { - return; - } - // High-water 85%: evict clean leaves down to ~70% of the arena. Best-effort — - // inner pages and dirty/working-set frames are not evictable, so usage may - // remain above target until the next snapshot cleans the working set. - if (static_cast<uint64_t>(st.used) * 100 < static_cast<uint64_t>(st.num_frames) * 85) { - return; - } - evict_clean_leaves_locked((static_cast<size_t>(st.num_frames) * 70) / 100); + core_->request_gc_floor(std::min(snapshot_slot, safe_slot)); } -void Crowdbtree::set_gc_watermark(uint64_t snapshot_slot, uint64_t safe_slot) +uint64_t Crowdbtree::gc_watermark() const { - uint64_t floor = std::min(snapshot_slot, safe_slot); - uint64_t prev = gc_floor_.load(); - while (floor > prev && !gc_floor_.compare_exchange_weak(prev, floor)) { - } + return core_->requested_gc_floor(); } Status Crowdbtree::put(Slice key, Slice value) @@ -624,19 +56,19 @@ Status Crowdbtree::put(Slice key, Slice value) .kind = OpKind::kPut, .value = std::string(value.data(), value.size()), }); - return apply(auto_slot_.fetch_add(1) + 1, b); + return core_->apply_auto(b); } Status Crowdbtree::del(Slice key) { Batch b; b.ops.push_back({.key = std::string(key.data(), key.size()), .kind = OpKind::kDelete, .value = std::string()}); - return apply(auto_slot_.fetch_add(1) + 1, b); + return core_->apply_auto(b); } Status Crowdbtree::batch_put(const Batch &batch) { - return apply(auto_slot_.fetch_add(1) + 1, batch); + return core_->apply_auto(batch); } Status Crowdbtree::validate_key(Slice key) const @@ -647,3742 +79,13 @@ Status Crowdbtree::validate_key(Slice key) const return Status::Ok(); } -void Crowdbtree::consolidate_locked(uint64_t page_id) -{ - auto t0 = std::chrono::steady_clock::now(); - PageBase *head = resident(page_id); - if (head == nullptr) { - return; - } - if (metrics_.page_consolidate_c != nullptr) { - metrics_.page_consolidate_c->inc(); - } - // A bare leaf base with no in-frame deltas (PT12) has nothing to fold; a base - // carrying in-frame deltas DOES (we fold them into a fresh sorted base). - if (head->type == page_type::kLeafBase && static_cast<LeafBase *>(head)->view().delta_count() == 0) { - return; - } - - LeafBase *old_leaf = chain_leaf_base(head); - uint64_t right = old_leaf != nullptr ? old_leaf->right_sibling() : kInvalidPageId; - - // Fold the chain by highest-slot-wins per key (GC drops tombstones <= floor), - // spilling new large values into overflow chains. Overflow chains superseded - // by higher-slot writes are retired so they don't leak. - std::vector<uint64_t> dead_overflow; - std::vector<leaf_entry> entries = resolve_leaf_chain_for_rebuild(head, gc_floor_.load(), &dead_overflow); - LeafBase *fresh = build_leaf_spilling_locked(std::move(entries), right); - store_preserving_parent_locked(page_id, fresh); - - // retire the old chain (deltas + old base). - for (PageBase *node = head; node != nullptr;) { - PageBase *next = node->next; - retire_page(node); - node = next; - } - for (uint64_t h : dead_overflow) { - retire_overflow_chain_locked(h); - } - - maybe_split_or_merge_locked(page_id); - if (metrics_.page_write_l != nullptr) { - auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count(); - metrics_.page_write_l->observe(static_cast<uint64_t>(ns)); - } -} - -void Crowdbtree::set_children_parent_locked(uint64_t page_id, uint64_t parent_page_id) -{ - PageBase *head = resident(page_id); - if (head == nullptr) { - return; - } - PageBase *base = head; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - if (base == nullptr || base->type != page_type::kInnerBase) { - return; - } - for (uint64_t child : static_cast<InnerBase *>(base)->children()) { - PageBase *child_head = resident(child); - if (child_head != nullptr) { - child_head->parent_page_id = parent_page_id; - } - } -} - -void Crowdbtree::store_preserving_parent_locked(uint64_t page_id, PageBase *new_page) -{ - PageBase *old = resident(page_id); - if (old != nullptr) { - new_page->parent_page_id = old->parent_page_id; - preserve_native_page_locked(page_id, old); - } - refresh_page_summary_locked(new_page); - mapping_.store(page_id, new_page); - uint64_t parent_id = new_page->parent_page_id; - while (parent_id != kInvalidPageId) { - PageBase *parent = resident(parent_id); - if (parent == nullptr || parent->type != page_type::kInnerBase) { - break; - } - refresh_page_summary_locked(parent); - const uint64_t next_parent = parent->parent_page_id; - mapping_.store(parent_id, parent); - parent_id = next_parent; - } -} - -void Crowdbtree::refresh_page_summary_locked(PageBase *page) -{ - if (page == nullptr) { - return; - } - if (page->type == page_type::kLeafBase) { - auto *leaf = static_cast<LeafBase *>(page); - PageSummary summary = leaf->summary(); - if (leaf->view().delta_count() != 0) { - summary.exact = false; - } - summary.reachable_leaf_pages = 1; - summary.reachable_inner_pages = 0; - summary.reachable_page_capacity_bytes = leaf->page_bytes(); - leaf->set_summary(summary); - return; - } - if (page->type == page_type::kOverflowFrame) { - auto *overflow = static_cast<OverflowBase *>(page); - PageSummary summary; - summary.reachable_overflow_pages = 1; - summary.reachable_page_capacity_bytes = overflow->page_bytes(); - summary.exact = true; - overflow->set_summary(summary); - return; - } - if (page->type != page_type::kInnerBase) { - return; - } - PageSummary aggregate; - aggregate.reachable_inner_pages = 1; - aggregate.reachable_page_capacity_bytes = static_cast<InnerBase *>(page)->page_bytes(); - aggregate.exact = true; - for (uint64_t child_id : static_cast<InnerBase *>(page)->children()) { - PageBase *child = resident(child_id); - if (child == nullptr) { - aggregate.exact = false; - continue; - } - PageBase *base = child; - while (base->type == page_type::kBatchDelta && base->next != nullptr) { - base = base->next; - } - PageSummary child_summary; - if (base->type == page_type::kLeafBase) { - child_summary = static_cast<LeafBase *>(base)->summary(); - } - else if (base->type == page_type::kInnerBase) { - child_summary = static_cast<InnerBase *>(base)->summary(); - } - else { - aggregate.exact = false; - continue; - } - aggregate.live_kv += child_summary.live_kv; - aggregate.live_key_bytes += child_summary.live_key_bytes; - aggregate.live_value_bytes += child_summary.live_value_bytes; - aggregate.live_logical_bytes += child_summary.live_logical_bytes; - aggregate.reachable_leaf_pages += child_summary.reachable_leaf_pages; - aggregate.reachable_inner_pages += child_summary.reachable_inner_pages; - aggregate.reachable_overflow_pages += child_summary.reachable_overflow_pages; - aggregate.reachable_page_capacity_bytes += child_summary.reachable_page_capacity_bytes; - aggregate.exact = aggregate.exact && child_summary.exact; - } - static_cast<InnerBase *>(page)->set_summary(aggregate); -} - -void Crowdbtree::refresh_subtree_summaries_locked(uint64_t page_id) -{ - PageBase *page = resident(page_id); - if (page == nullptr) { - return; - } - PageBase *base = page; - while (base->type == page_type::kBatchDelta && base->next != nullptr) { - base = base->next; - } - if (base->type == page_type::kInnerBase) { - for (uint64_t child : static_cast<InnerBase *>(base)->children()) { - refresh_subtree_summaries_locked(child); - } - } - PageSummary before; - if (base->type == page_type::kLeafBase) { - before = static_cast<LeafBase *>(base)->summary(); - } - else if (base->type == page_type::kInnerBase) { - before = static_cast<InnerBase *>(base)->summary(); - } - refresh_page_summary_locked(base); - PageSummary after; - if (base->type == page_type::kLeafBase) { - after = static_cast<LeafBase *>(base)->summary(); - } - else if (base->type == page_type::kInnerBase) { - after = static_cast<InnerBase *>(base)->summary(); - } - const bool changed = before.live_kv != after.live_kv || before.live_key_bytes != after.live_key_bytes || - before.live_value_bytes != after.live_value_bytes || - before.live_logical_bytes != after.live_logical_bytes || - before.reachable_leaf_pages != after.reachable_leaf_pages || - before.reachable_inner_pages != after.reachable_inner_pages || - before.reachable_overflow_pages != after.reachable_overflow_pages || - before.reachable_page_capacity_bytes != after.reachable_page_capacity_bytes || - before.exact != after.exact; - if (changed) { - base->durable_addr = kNoAddr; - mapping_.store(base->page_id, base); - } -} - -std::vector<uint64_t> Crowdbtree::path_to_page_id_locked(uint64_t target_page_id) const -{ - // O4: parent-pointer walk — O(depth) instead of O(tree size) DFS. Walk - // from target up to the root via parent_page_id, then reverse. The root - // has parent_page_id == kInvalidPageId. All parent pointers are maintained - // under write_mutex_ on every split/merge/root-change, and this function - // is only called under write_mutex_, so no synchronization is needed. - // Fallback: if a page was demand-loaded (evicted then reloaded from disk), - // its parent_page_id is kInvalidPageId (not persisted). In that case, fall - // back to DFS from the root. This is O(tree size) but only on the cold - // (demand-load) path — the hot path (split/merge) always has valid parent - // pointers. - std::vector<uint64_t> path; - uint64_t root = root_page_id_.load(); - if (target_page_id == root) { - return path; // root has no parent path - } - // Try the fast parent-pointer walk first. Depth limit prevents - // infinite loops from stale/cyclic parent pointers (defensive). - bool fast_ok = true; - const int kMaxDepth = 64; - int depth = 0; - for (uint64_t pid = target_page_id; pid != kInvalidPageId && pid != root && depth < kMaxDepth;) { - PageBase *head = resident(pid); - if (head == nullptr) { - fast_ok = false; - break; - } - uint64_t parent = head->parent_page_id; - if (parent == kInvalidPageId || parent == pid) { - // Broken chain (demand-loaded page) or self-loop: fall back to DFS. - fast_ok = false; - break; - } - path.push_back(parent); - if (parent == root) { - break; - } - pid = parent; - ++depth; - } - if (depth >= kMaxDepth) { - fast_ok = false; // too deep — fall back to DFS - } - if (fast_ok) { - std::reverse(path.begin(), path.end()); - return path; - } - // Fallback: DFS from the root (O(tree size) — cold path only). - path.clear(); - std::function<bool(uint64_t)> dfs = [&](uint64_t page_id) -> bool { - if (page_id == target_page_id) { - return true; - } - PageBase *head = resident(page_id); - if (head == nullptr) { - return false; - } - PageBase *base = head; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - if (base == nullptr || base->type != page_type::kInnerBase) { - return false; - } - path.push_back(page_id); - for (uint64_t child : static_cast<InnerBase *>(base)->children()) { - if (dfs(child)) { - return true; - } - } - path.pop_back(); - return false; - }; - dfs(root); - return path; -} - -void Crowdbtree::maybe_split_or_merge_locked(uint64_t page_id) -{ - PageBase *head = resident(page_id); - if (head == nullptr || head->type != page_type::kLeafBase) { - return; - } - auto *leaf = static_cast<LeafBase *>(head); - if (leaf->count() >= 2 && leaf->data_bytes() > opt_.leaf_split_bytes) { - split_leaf_to_threshold_locked(page_id); - } - else if (leaf->data_bytes() < opt_.leaf_merge_bytes && page_id != root_page_id_.load()) { - // Includes empty leaves (count 0) so fully-deleted leaves merge away. - try_merge_leaf_locked(page_id, path_to_page_id_locked(page_id)); - } -} - -void Crowdbtree::split_leaf_to_threshold_locked(uint64_t leaf_page_id) -{ - // Consolidation can fold a large delta chain into a leaf that is many - // times the split threshold. A single 2-way split leaves both halves - // oversized when the leaf is > 2x the threshold. Iteratively split each - // oversized half (via a worklist) until every leaf fits under the - // threshold. Reuses split_leaf_locked for each halving; the worklist - // ensures both the left half (still at leaf_page_id) and the right half - // (a new sibling) get re-checked. Terminates because each split halves - // the entry count and we stop at count < 2. - std::vector<uint64_t> worklist; - worklist.push_back(leaf_page_id); - while (!worklist.empty()) { - uint64_t pid = worklist.back(); - worklist.pop_back(); - PageBase *head = resident(pid); - if (head == nullptr || head->type != page_type::kLeafBase) { - continue; - } - auto *leaf = static_cast<LeafBase *>(head); - if (leaf->count() < 2 || leaf->data_bytes() <= opt_.leaf_split_bytes) { - continue; - } - split_leaf_locked(pid, path_to_page_id_locked(pid)); - // Both halves may still exceed the threshold; re-check them. - PageBase *fresh = resident(pid); - if (fresh != nullptr && fresh->type == page_type::kLeafBase) { - worklist.push_back(pid); - uint64_t right = static_cast<LeafBase *>(fresh)->right_sibling(); - if (right != kInvalidPageId) { - worklist.push_back(right); - } - } - } -} - -void Crowdbtree::split_leaf_locked(uint64_t leaf_page_id, std::vector<uint64_t> path) -{ - if (metrics_.page_split_c != nullptr) { - metrics_.page_split_c->inc(); - } - auto *leaf = static_cast<LeafBase *>(resident(leaf_page_id)); - std::vector<leaf_entry> e = leaf->entries(); // materialized owned copy - size_t mid = e.size() / 2; - // leaf_entry is move-only (buffer cell): move the halves out, don't copy. - std::vector<leaf_entry> lo(std::make_move_iterator(e.begin()), - std::make_move_iterator(e.begin() + static_cast<std::ptrdiff_t>(mid))); - std::vector<leaf_entry> hi(std::make_move_iterator(e.begin() + static_cast<std::ptrdiff_t>(mid)), - std::make_move_iterator(e.end())); - std::string sep = hi.front().key; - - // Publish the right sibling, then repoint the parent(s) at it — all while - // `leaf_page_id` still holds the FULL entry set. A concurrent reader routed to - // `leaf_page_id` for an upper-half key still finds it (the parent only starts - // routing upper-half keys to right_page_id once it references it). Only after the - // whole path is repointed do we shrink `leaf_page_id` to the lower half. - uint64_t right_page_id = mapping_.allocate_page_id(); - LeafBase *right = LeafBase::build(hi, leaf->right_sibling(), pool_, opt_.frame_bytes); - refresh_page_summary_locked(right); - mapping_.store(right_page_id, right); - propagate_split_locked(std::move(path), leaf_page_id, std::move(sep), right_page_id); - - LeafBase *left = LeafBase::build(lo, right_page_id, pool_, opt_.frame_bytes); - store_preserving_parent_locked(leaf_page_id, left); - retire_page(leaf); - leaf_count_.fetch_add(1, std::memory_order_relaxed); - sync_page_count_gauges(); -} - -void Crowdbtree::propagate_split_locked(std::vector<uint64_t> path, uint64_t child_page_id, std::string sep, - uint64_t right_page_id) +size_t Crowdbtree::evict_clean_leaves(size_t max_resident_leaves) { - if (path.empty()) { - // child was the root: grow a new root one level up. - uint64_t new_root = mapping_.allocate_page_id(); - InnerBase *root = InnerBase::build({std::move(sep)}, {child_page_id, right_page_id}, pool_, opt_.frame_bytes); - refresh_page_summary_locked(root); - mapping_.store(new_root, root); - root_page_id_.store(new_root); - // O4: set parent pointers on the new root's children. - set_children_parent_locked(new_root, new_root); - inner_count_.fetch_add(1, std::memory_order_relaxed); - sync_page_count_gauges(); - return; - } - uint64_t parent_page_id = path.back(); - path.pop_back(); - auto *parent = static_cast<InnerBase *>(resident(parent_page_id)); - - // Locate child_page_id among the parent's children. - const std::vector<uint64_t> &ch = parent->children(); - size_t idx = 0; - while (idx < ch.size() && ch[idx] != child_page_id) { - ++idx; - } - - std::vector<std::string> seps = parent->separators(); - std::vector<uint64_t> children = parent->children(); - seps.insert(seps.begin() + static_cast<std::ptrdiff_t>(idx), std::move(sep)); - children.insert(children.begin() + static_cast<std::ptrdiff_t>(idx + 1), right_page_id); - - if (seps.size() <= opt_.inner_max_keys) { - store_preserving_parent_locked(parent_page_id, InnerBase::build(seps, children, pool_, opt_.frame_bytes)); - retire_page(parent); - // O4: update parent pointers for all children (the new right_page_id - // child and any existing children whose parent was just rebuilt). - set_children_parent_locked(parent_page_id, parent_page_id); - sync_page_count_gauges(); - return; - } - - // Inner overflow: split this inner node, pushing the median separator up. - size_t m = seps.size() / 2; - std::string median = seps[m]; - std::vector<std::string> lseps(seps.begin(), seps.begin() + static_cast<std::ptrdiff_t>(m)); - std::vector<uint64_t> lchildren(children.begin(), children.begin() + static_cast<std::ptrdiff_t>(m + 1)); - std::vector<std::string> rseps(seps.begin() + static_cast<std::ptrdiff_t>(m + 1), seps.end()); - std::vector<uint64_t> rchildren(children.begin() + static_cast<std::ptrdiff_t>(m + 1), children.end()); - - uint64_t rinner_page_id = mapping_.allocate_page_id(); - store_preserving_parent_locked(parent_page_id, InnerBase::build(lseps, lchildren, pool_, opt_.frame_bytes)); - InnerBase *rinner = InnerBase::build(rseps, rchildren, pool_, opt_.frame_bytes); - refresh_page_summary_locked(rinner); - mapping_.store(rinner_page_id, rinner); - retire_page(parent); - // O4: set parent pointers on children of both split inner pages. - set_children_parent_locked(parent_page_id, parent_page_id); - set_children_parent_locked(rinner_page_id, rinner_page_id); - inner_count_.fetch_add(1, std::memory_order_relaxed); - - propagate_split_locked(std::move(path), parent_page_id, std::move(median), rinner_page_id); + return core_->evict_clean_pages(PageCacheKind::Leaf, max_resident_leaves); } -void Crowdbtree::try_merge_leaf_locked(uint64_t leaf_page_id, const std::vector<uint64_t> &path) +size_t Crowdbtree::evict_clean_inner(size_t max_resident_inner) { - if (path.empty()) { - return; // root leaf: nothing to merge with - } - if (metrics_.page_merge_c != nullptr) { - metrics_.page_merge_c->inc(); - } - uint64_t parent_page_id = path.back(); - auto *parent = static_cast<InnerBase *>(resident(parent_page_id)); - const std::vector<uint64_t> &ch = parent->children(); - size_t idx = 0; - while (idx < ch.size() && ch[idx] != leaf_page_id) { - ++idx; - } - if (idx == 0) { - return; // no left sibling under this parent (v1: left-merge only) - } - - uint64_t left_page_id = ch[idx - 1]; - auto *left_head = resident(left_page_id); - if (left_head == nullptr || left_head->type != page_type::kLeafBase) { - return; - } - auto *left = static_cast<LeafBase *>(left_head); - auto *leaf = static_cast<LeafBase *>(resident(leaf_page_id)); - - // 1. Publish the merged left sibling (superset of left+leaf entries). Readers - // routed to left_page_id now find both halves; readers still routed to leaf_page_id - // (via the not-yet-updated parent) also still find leaf's entries. - // GC-drop tombstones <= floor so merged leaves don't accumulate garbage - // (otherwise the leftmost leaf bloats and the root never collapses). - // Resolve each sibling's full entry set (main + in-frame deltas, PT12), - // GC-dropping tombstones <= floor. The two key ranges are disjoint and each - // resolve returns sorted storage cells, so concatenation stays sorted. Collect - // overflow chains that a higher-slot write (e.g. a delete delta) superseded - // within either chain so they are retired, not leaked. - uint64_t gc = gc_floor_.load(); - std::vector<uint64_t> dead_overflow; - std::vector<leaf_entry> merged = resolve_leaf_chain_for_rebuild(left_head, gc, &dead_overflow); - std::vector<leaf_entry> leaf_entries = resolve_leaf_chain_for_rebuild(leaf, gc, &dead_overflow); - for (auto &e : leaf_entries) { - merged.push_back(std::move(e)); - } - LeafBase *fresh = build_leaf_spilling_locked(std::move(merged), leaf->right_sibling()); - store_preserving_parent_locked(left_page_id, fresh); - retire_page(left); - for (uint64_t h : dead_overflow) { - retire_overflow_chain_locked(h); - } - leaf_count_.fetch_sub(1, std::memory_order_relaxed); - - // 2. Repoint the parent: drop separators_[idx-1] and children_[idx]. - std::vector<std::string> seps = parent->separators(); - std::vector<uint64_t> children = parent->children(); - seps.erase(seps.begin() + static_cast<std::ptrdiff_t>(idx - 1)); - children.erase(children.begin() + static_cast<std::ptrdiff_t>(idx)); - - bool parent_underfull = false; - if (children.size() == 1 && parent_page_id == root_page_id_.load()) { - // Root now has a single child: collapse the root one level down. - // `parent`'s own PID gets no replacement store() -- orphaned. - root_page_id_.store(children[0]); - // O4: the new root (single child) has no parent. - PageBase *new_root_head = resident(children[0]); - if (new_root_head != nullptr) { - new_root_head->parent_page_id = kInvalidPageId; - } - retire_orphaned_page(parent_page_id, parent); - inner_count_.fetch_sub(1, std::memory_order_relaxed); - } - else { - size_t parent_seps = seps.size(); - store_preserving_parent_locked(parent_page_id, InnerBase::build(seps, children, pool_, opt_.frame_bytes)); - retire_page(parent); - // O4: update parent pointers for the rebuilt parent's children. - set_children_parent_locked(parent_page_id, parent_page_id); - parent_underfull = parent_page_id != root_page_id_.load() && parent_seps < inner_merge_keys(); - } - - // 3. The leaf is now unreachable by new readers. retire_orphaned_page - // epoch-retires it (stragglers holding an old parent are protected by - // their epoch guard) and clears its mapping slot once that's safe - // -- deferred, not - // immediate, so it can never race a straggler still walking in via a - // stale parent from before this retirement (see retire_orphaned_ - // page's doc comment). The PID itself is never recycled (D1). - retire_orphaned_page(leaf_page_id, leaf); - - // 4. Inner-node underflow: if the parent dropped below the merge threshold, - // merge it with its left sibling (recurses up, may collapse the root). - if (parent_underfull) { - std::vector<uint64_t> ppath = path; // root..parent - ppath.pop_back(); // -> root..grandparent (parent's path) - try_merge_inner_locked(parent_page_id, std::move(ppath)); - } - sync_page_count_gauges(); + return core_->evict_clean_pages(PageCacheKind::Inner, max_resident_inner); } - -void Crowdbtree::try_merge_inner_locked(uint64_t inner_page_id, std::vector<uint64_t> path) -{ - if (path.empty()) { - return; // inner is the root: nothing to merge with - } - uint64_t gp_page_id = path.back(); - auto *gp_head = resident(gp_page_id); - if (gp_head == nullptr || gp_head->type != page_type::kInnerBase) { - return; - } - auto *gp = static_cast<InnerBase *>(gp_head); - - const std::vector<uint64_t> &gch = gp->children(); - size_t idx = 0; - while (idx < gch.size() && gch[idx] != inner_page_id) { - ++idx; - } - if (idx == 0 || idx >= gch.size()) { - return; // no left sibling (v1: left-merge only) - } - - uint64_t left_page_id = gch[idx - 1]; - auto *left_head = resident(left_page_id); - if (left_head == nullptr || left_head->type != page_type::kInnerBase) { - return; - } - auto *left = static_cast<InnerBase *>(left_head); - auto *inner_head = resident(inner_page_id); - if (inner_head == nullptr || inner_head->type != page_type::kInnerBase) { - return; - } - auto *inner = static_cast<InnerBase *>(inner_head); - - // Only merge if the combined node still fits the fanout bound; otherwise leave - // the page underfull (correct, just less compact) rather than build an - // immediately-oversized inner. - size_t combined_seps = left->num_separators() + 1 + inner->num_separators(); - if (combined_seps > opt_.inner_max_keys) { - return; - } - - // 1. Publish the merged left sibling = left.children + inner.children, with the - // grandparent's separator-between spliced in. Readers via the old - // grandparent still reach `inner` (retired, epoch-safe) with its children; - // readers via the new grandparent reach merged-left with both subtrees. - std::vector<std::string> mseps = left->separators(); - mseps.push_back(gp->separator_at(idx - 1)); - for (auto &s : inner->separators()) { - mseps.push_back(std::move(s)); - } - std::vector<uint64_t> mchildren = left->children(); - for (uint64_t c : inner->children()) { - mchildren.push_back(c); - } - store_preserving_parent_locked(left_page_id, InnerBase::build(mseps, mchildren, pool_, opt_.frame_bytes)); - retire_page(left); - // O4: update parent pointers for the merged left sibling's children. - set_children_parent_locked(left_page_id, left_page_id); - inner_count_.fetch_sub(1, std::memory_order_relaxed); // two inners → one - - // 2. Repoint the grandparent: drop separators[idx-1] and children[idx]. - std::vector<std::string> gseps = gp->separators(); - std::vector<uint64_t> gchildren = gp->children(); - gseps.erase(gseps.begin() + static_cast<std::ptrdiff_t>(idx - 1)); - gchildren.erase(gchildren.begin() + static_cast<std::ptrdiff_t>(idx)); - - bool gp_underfull = false; - if (gchildren.size() == 1 && gp_page_id == root_page_id_.load()) { - // Root now has a single child: collapse one level down. `gp`'s own - // PID gets no replacement store() -- orphaned. - root_page_id_.store(gchildren[0]); - // O4: the new root (single child) has no parent. - PageBase *new_root_head = resident(gchildren[0]); - if (new_root_head != nullptr) { - new_root_head->parent_page_id = kInvalidPageId; - } - retire_orphaned_page(gp_page_id, gp); - inner_count_.fetch_sub(1, std::memory_order_relaxed); // root inner retired - } - else { - size_t gp_seps = gseps.size(); - store_preserving_parent_locked(gp_page_id, InnerBase::build(gseps, gchildren, pool_, opt_.frame_bytes)); - retire_page(gp); - // O4: update parent pointers for the rebuilt grandparent's children. - set_children_parent_locked(gp_page_id, gp_page_id); - gp_underfull = gp_page_id != root_page_id_.load() && gp_seps < inner_merge_keys(); - } - - // 3. The merged-away inner is unreachable by new readers; retire_orphaned_page - // epoch-retires it (safe for stragglers) and clears its mapping slot once - // that's safe (deferred, not immediate -- see that method's doc comment). - // Its children are now owned by merged-left, so retiring this single page - // does not free them. PID itself never recycled (D1). - retire_orphaned_page(inner_page_id, inner); - - // 4. Recurse: the grandparent may now be underfull. - if (gp_underfull) { - path.pop_back(); // -> root..great-grandparent (grandparent's path) - try_merge_inner_locked(gp_page_id, std::move(path)); - } - sync_page_count_gauges(); -} - -GetView Crowdbtree::get_view(Slice key) const -{ - auto generation = generation_.enter(); - GetView result; - if (!opt_.key_range.contains(key)) { - return result; - } - result.guard_ = epoch_.enter(); - - // L0: check every live MemTable (active_ + any not-yet-drained frozen_ - // buffers) and keep the highest-slot hit. Unlike the single-buffer - // design, a key can legitimately be present in more than one live - // MemTable at once with *different* slots (out-of-order slot delivery - // can straddle a freeze boundary) -- see the active_/frozen_ member - // comment (plan-tree #3) for the full argument. Any key present in ANY - // live MemTable is still guaranteed strictly newer than L1, so a hit - // here never needs to fall through to L1. - // - // R50: an L0 hit borrows the value directly from the CellVersion's - // buffer — the epoch guard keeps the skip-list node (and its cell - // version) alive past any concurrent overwrite/drain, exactly as it - // keeps an L1 frame resident. No copy, no std::string staging. - auto l0_t0 = std::chrono::steady_clock::now(); - auto tables = all_memtables(); - const CellVersion *best = nullptr; - for (auto &mt : tables) { - const CellVersion *cv = mt->find(key); - if (cv == nullptr) { - continue; - } - mt_get_total_.fetch_add(1, std::memory_order_relaxed); - if (metrics_.mt_get_c != nullptr) { - metrics_.mt_get_c->inc(); - } - if (best == nullptr || cv->slot >= best->slot) { - best = cv; - result.source_guard_ = mt.guard; - } - } - if (best != nullptr) { - mt_get_hit_total_.fetch_add(1, std::memory_order_relaxed); - if (metrics_.mt_get_hit_c != nullptr) { - metrics_.mt_get_hit_c->inc(); - } - if (metrics_.mt_get_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - l0_t0).count(); - metrics_.mt_get_l->observe(static_cast<uint64_t>(ns)); - } - if ((best->flags & kFlagTombstone) != 0) { - return result; // not found - } - result.found_ = true; - result.slot_ = best->slot; - // Borrow the value: contiguous cell -> value after the 9-byte header; - // split (kExternal) cell -> the buffer itself is the value. - if (best->cell.ownership() != buffer::mode::kExternal) { - result.value_ = {best->cell.data() + kCellHeaderSize, best->cell.size() - kCellHeaderSize}; - } - else { - result.value_ = best->cell.slice(); - } - return result; - } - if (metrics_.mt_get_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - l0_t0).count(); - metrics_.mt_get_l->observe(static_cast<uint64_t>(ns)); - } - - // L1: descend to the leaf and resolve its chain. A non-overflow cell's - // value lives directly in head's frame, which result.guard_ keeps - // resident for result's lifetime -- borrow it, no copy. - auto l1_t0 = std::chrono::steady_clock::now(); - l1_get_total_.fetch_add(1, std::memory_order_relaxed); - if (metrics_.l1_get_c != nullptr) { - metrics_.l1_get_c->inc(); - } - uint64_t page_id = find_leaf_page_id([this](uint64_t p) { return resident(p); }, root_page_id_.load(), key); - if (page_id == kInvalidPageId) { - if (metrics_.l1_get_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - l1_t0).count(); - metrics_.l1_get_l->observe(static_cast<uint64_t>(ns)); - } - return result; - } - PageBase *head = resident(page_id); - CellView v; - if (!resolve_chain(head, key, &v)) { - if (metrics_.l1_get_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - l1_t0).count(); - metrics_.l1_get_l->observe(static_cast<uint64_t>(ns)); - } - return result; - } - if (v.is_tombstone()) { - if (metrics_.l1_get_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - l1_t0).count(); - metrics_.l1_get_l->observe(static_cast<uint64_t>(ns)); - } - return result; - } - l1_get_hit_total_.fetch_add(1, std::memory_order_relaxed); - if (metrics_.l1_get_hit_c != nullptr) { - metrics_.l1_get_hit_c->inc(); - } - if (metrics_.l1_get_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - l1_t0).count(); - metrics_.l1_get_l->observe(static_cast<uint64_t>(ns)); - } - result.found_ = true; - result.slot_ = v.slot(); - if (v.is_overflow()) { - // Assembled from multiple overflow pages -- no single frame to - // borrow from, so materialize it like an L0 hit. - result.owned_ = buffer::copy_of(assemble_overflow_value(v.overflow_head(), v.overflow_len())); - result.value_ = result.owned_.slice(); - } - else { - result.value_ = v.value(); // borrowed: lives in head's frame - } - return result; -} - -bool Crowdbtree::try_get_view_no_load(Slice key, GetView *result, uint64_t *out_pending_page_id) const -{ - auto generation = generation_.enter(); - result->guard_ = epoch_.enter(); - - // L0: identical to get_view() -- never touches the page store, so there - // is no I/O to avoid here. R50: borrows the value directly (no copy). - auto tables = all_memtables(); - const CellVersion *best = nullptr; - for (auto &mt : tables) { - const CellVersion *cv = mt->find(key); - if (cv == nullptr) { - continue; - } - if (best == nullptr || cv->slot >= best->slot) { - best = cv; - result->source_guard_ = mt.guard; - } - } - if (best != nullptr) { - if ((best->flags & kFlagTombstone) != 0) { - return true; // resolved: not found - } - result->found_ = true; - result->slot_ = best->slot; - if (best->cell.ownership() != buffer::mode::kExternal) { - result->value_ = {best->cell.data() + kCellHeaderSize, best->cell.size() - kCellHeaderSize}; - } - else { - result->value_ = best->cell.slice(); - } - return true; - } - - // L1: same descent as get_view(), but `probe` bails out (returning - // nullptr, as if the slot were unset) the moment it sees an unloaded - // slot, instead of demand-loading it -- recording *which* page_id via - // `blocked_page_id`. Never unpacks the unloaded descriptor (see this - // method's doc comment on crowdb-tree.h): only slot_word::is_unloaded(), a - // plain tag-bit check on the packed word, which needs no lock. - uint64_t blocked_page_id = kInvalidPageId; - auto probe = [this, &blocked_page_id](uint64_t p) -> PageBase *{ - uint64_t w = mapping_.get_word(p); - if (slot_word::is_empty(w)) { - return nullptr; - } - if (slot_word::is_unloaded(w)) { - blocked_page_id = p; - return nullptr; - } - PageBase *v = slot_word::resident_ptr(w); - v->last_touch_tick.store(touch_tick_.fetch_add(1, std::memory_order_relaxed), std::memory_order_relaxed); - return v; - }; - - uint64_t page_id = find_leaf_page_id(probe, root_page_id_.load(), key); - if (blocked_page_id != kInvalidPageId) { - *out_pending_page_id = blocked_page_id; - return false; // genuine miss - } - if (page_id == kInvalidPageId) { - return true; // resolved: not found (empty/malformed tree) - } - // Re-probe the leaf head, mirroring get_view()'s separate resident() - // call after find_leaf_page_id -- find_leaf_page_id doesn't return the - // resolved pointer, only the page_id, and a fresh probe is cheap - // (lock-free) and tolerates a concurrent mutation the same way - // get_view() already does. - PageBase *head = probe(page_id); - if (blocked_page_id != kInvalidPageId) { - *out_pending_page_id = blocked_page_id; - return false; // genuine miss (raced with a concurrent eviction) - } - if (head == nullptr) { - return true; // resolved: not found - } - CellView v; - if (!resolve_chain(head, key, &v)) { - return true; // resolved: not found - } - if (v.is_tombstone()) { - return true; // resolved: not found - } - result->found_ = true; - result->slot_ = v.slot(); - if (v.is_overflow()) { - // Scope boundary (see get_async's doc comment on crowdb-tree.h): - // overflow-chain misses stay synchronous. - result->owned_ = buffer::copy_of(assemble_overflow_value(v.overflow_head(), v.overflow_len())); - result->value_ = result->owned_.slice(); - } - else { - result->value_ = v.value(); // borrowed: lives in head's frame - result->borrowed_chain_head_ = head; // R6: pin target for slow path - } - return true; // resolved: found -} - -GetView Crowdbtree::materialize_owned(GetView &&v) -{ - if (v.found_ && v.owned_.empty() && v.borrowed_chain_head_ != nullptr) { - // R6: frame-borrowed value on the slow path. Pin the chain (head → - // base) so the borrowed Slice survives the thread boundary, then - // release the epoch guard on this (the entering) thread. The last - // unpin (from ct_future_free on any thread) frees if the page was - // retired in the meantime. - for (PageBase *n = v.borrowed_chain_head_; n != nullptr; n = n->next) { - n->pin(); - v.pins_.push_back(n); - } - v.borrowed_chain_head_ = nullptr; - } - else if (v.found_ && v.owned_.empty()) { - // Overflow-chain value (assembled, no single frame to borrow): copy - // as before. R6 doesn't change this path. - v.owned_ = buffer::copy_of(v.value_); - v.value_ = v.owned_.slice(); - } - // Release on this (the entering) thread before on_done can hand this - // GetView off across the FFI boundary to a ct_future_free that might - // run on a different one. For the pin path, the pages stay alive via - // refcount; for the copy path, the owned buffer is independent. - v.guard_ = EpochManager::Guard(); - v.source_guard_.reset(); - return std::move(v); -} - -void Crowdbtree::get_async(Slice key, std::function<void(Status, GetView)> on_done) const -{ - // Copy the key upfront: unlike get_view()'s Slice (borrowed, valid only - // for this one synchronous call), get_async's key must survive across - // an arbitrary number of async round trips, each on a different call - // stack than this one. - get_async_attempt(std::make_shared<std::string>(key.to_string()), std::move(on_done), /*same_thread=*/true); -} - -void Crowdbtree::get_async_attempt(std::shared_ptr<std::string> key_owned, std::function<void(Status, GetView)> on_done, - bool same_thread) const -{ - GetView result; - uint64_t pending_page_id = kInvalidPageId; - if (try_get_view_no_load(Slice(*key_owned), &result, &pending_page_id)) { - // same_thread: zero-copy fast path -- hand the GetView straight through, guard and all. - // Otherwise this resolved on (or after being handed off from) the - // Reactor thread, so materialize_owned() releases the guard here, - // on the thread that entered it, before on_done can cross back out. - on_done(Status::Ok(), same_thread ? std::move(result) : materialize_owned(std::move(result))); - return; - } - - if (opt_.async_page_store != nullptr) { - // Re-verify under load_mutex_ before unpacking the unloaded - // descriptor from the slot word (see try_get_view_no_load's doc - // comment on crowdb-tree.h): the word may be concurrently replaced by - // a loader installing the resident replacement, so re-read under - // the lock -- mirrors resident()'s own double-checked locking - // exactly, just split across the async submission below. - uint64_t addr = 0; - uint32_t plen = 0; - bool still_unloaded = false; - uint64_t requested_word = 0; - Status location_status; - { - auto generation = generation_.enter(); - std::scoped_lock lk(load_mutex_); - uint64_t w = mapping_.get_word(pending_page_id); - if (slot_word::is_unloaded(w)) { - requested_word = w; - location_status = opt_.page_store->decode_mapping_location(w, &addr, &plen); - still_unloaded = location_status.ok(); - } - } - if (!location_status.ok()) { - io_failed_.store(true); - on_done(location_status, {}); - return; - } - if (!still_unloaded) { - // Another loader (sync resident() or a concurrent get_async) - // already resolved this page_id between the lock-free probe - // above and this re-check -- just retry, still on this thread. - get_async_attempt(std::move(key_owned), std::move(on_done), same_thread); - return; - } - uint32_t iu = opt_.page_store->iu_size(); - auto blob = std::make_shared<std::vector<uint8_t>>(round_up_to_iu(plen, iu)); - demand_load_total_.fetch_add(1, std::memory_order_relaxed); - opt_.async_page_store->submit_read( - addr, blob->data(), blob->size(), - detail::own_async_completion([this, page_id = pending_page_id, requested_word, addr, plen, blob, key_owned, - on_done](Status st) mutable { - if (!st.ok()) { - CRB_LOG_ERROR("[{}] get_async: demand-load I/O fault: pid={} addr={} len={} status={}", name_, - page_id, addr, plen, st.to_string()); - if (st.code() != Code::kUnavailable && st.code() != Code::kResourceExhausted) { - io_failed_.store(true); - } - on_done(std::move(st), GetView()); - return; - } - bool installed_ok = true; - { - auto generation = generation_.enter(); - std::scoped_lock lk(load_mutex_); - uint64_t w = mapping_.get_word(page_id); - if (w == requested_word) { - installed_ok = install_loaded_page(page_id, addr, plen, *blob) != nullptr; - } - // else: another loader already installed it -- retry below. - } - if (!installed_ok) { - // Decode/CRC/validation failure -- io_failed_ already - // latched by install_loaded_page; matches resident()'s - // own "degrades to a miss" contract. - on_done(Status::corruption("get_async: demand-load decode, structure, or range failure"), - GetView()); - return; - } - // This callback runs on the Reactor's own thread (design's - // thread model table) -- everything from here on is *not* - // same_thread relative to the original caller. - get_async_attempt(std::move(key_owned), std::move(on_done), /*same_thread=*/false); - })); - return; - } - // No async backend wired (e.g. a MemPageStore-backed tree -- design - // §6.3: no MemAsyncPageStore, nothing is genuinely pending there) -- - // fall back to the existing synchronous demand-load and retry, still - // on this same thread. - { - auto generation = generation_.enter(); - (void)resident(pending_page_id); - } - get_async_attempt(std::move(key_owned), std::move(on_done), same_thread); -} - -bool Crowdbtree::get(Slice key, uint64_t *out_slot, std::string *out_value) const -{ - GetView v = get_view(key); - if (!v.found()) { - return false; - } - if (out_slot != nullptr) { - *out_slot = v.slot(); - } - if (out_value != nullptr) { - *out_value = v.value().to_string(); // clone; v.guard_ releases at end of this function - } - return true; -} - -std::vector<get_result> Crowdbtree::multi_get(const std::vector<Slice> &keys) const -{ - std::vector<get_result> results; - results.reserve(keys.size()); - for (const Slice &k : keys) { - get_result g; - g.found = get(k, &g.slot, &g.value); - results.push_back(std::move(g)); - } - return results; -} - -Status -Crowdbtree::scan(Slice prefix, Slice start_after, Slice end_key, size_t limit, size_t byte_budget, bool keys_only, - uint64_t deadline_ms, - std::vector<scan_entry> *out, // NOLINT(readability-non-const-parameter) written to via push_back - bool *truncated, bool include_tombstones, ScanPackedBuf *out_packed, - size_t *out_count, // NOLINT(readability-non-const-parameter) written to via *out_count - bool has_start_bound, bool start_inclusive, bool strict_byte_budget) const -{ - auto generation = generation_.enter(); - // Preserve scan()'s original `start_after` contract for direct C++ - // callers. The explicit flag additionally makes an empty key usable as - // a real lower bound through scan-from APIs. - has_start_bound = has_start_bound || !start_after.empty(); - if (out == nullptr && out_packed == nullptr) { - return Status::invalid_argument("scan requires an output buffer"); - } - if (out != nullptr) { - out->clear(); - } - if (out_packed != nullptr) { - *out_packed = ScanPackedBuf{}; - } - size_t packed_count = 0; - if (truncated != nullptr) { - *truncated = false; - } - // plan-tree #5 B3: lock-free scan. An epoch guard (not write_mutex_) is - // sufficient: L1 is walked leaf-by-leaf via right_sibling starting at the - // leaf that would contain `prefix`, one leaf resolved at a time, instead of - // materializing the whole reachable tree up front under a lock. This is - // safe against a concurrent split/merge because both always keep a leaf's - // right_sibling link consistent with the content it's attached to via a - // single atomic mapping_.store: split_leaf_locked publishes the new right - // half and repoints the parent *before* shrinking the original PID, so a - // leaf read mid-split either still holds its full pre-split entry set (old - // right_sibling, no gap) or the shrunk half with right_sibling already - // pointing at the new right half (no gap, no duplicate); a merge folds the - // removed leaf's entries into its left sibling and gives the merged page - // the removed leaf's old right_sibling, so a stale reader still positioned - // at the removed (retired, epoch-alive) PID reaches the correct successor - // via its own unchanged right_sibling. See split_leaf_locked / - // try_merge_leaf_locked for the exact ordering this relies on. - EpochManager::Guard guard = epoch_.enter(); - - auto dur_ns = [](std::chrono::steady_clock::time_point from) { - return static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - from).count()); - }; - auto t_total = std::chrono::steady_clock::now(); - - // L0: one lock-free cursor per live MemTable (active_ + any not-yet- - // drained frozen_ buffers). R50: the cursor borrows key/cell Slices - // directly off the skip-list node — no snapshot copy. The epoch guard - // (taken above) keeps the node alive past any concurrent drain/overwrite. - // Unlike the single-buffer design, more than one of these can hold the - // same key with a *different* slot (out-of-order slot delivery can - // straddle a freeze boundary -- see the active_/frozen_ member comment, - // plan-tree #3), so the merge below picks the highest-slot cell among - // whichever sources tie on a key, instead of unconditionally preferring - // "the" L0 stream. - struct L0Cursor - { - ConcurrentSkipList::Cursor cur; - }; - - auto t0 = std::chrono::steady_clock::now(); - std::vector<L0Cursor> l0; - auto sources_owner = all_memtables(); - l0.reserve(sources_owner.size()); - for (auto &mt : sources_owner) { - l0.push_back( - {.cur = has_start_bound ? mt->cursor_from(start_after, start_inclusive) : mt->cursor(start_after)}); - } - uint64_t l0_ns = dur_ns(t0); - - uint64_t l1_ns = 0; - uint64_t gc = gc_floor_.load(); - uint64_t page_id = root_page_id_.load(); - // Descend at the cursor when present, else at the prefix start -- a - // non-empty start_after lands directly on the leaf containing it, - // skipping every earlier leaf in the prefix range. - Slice descend_key = !start_after.empty() ? start_after : prefix; - if (page_id != kInvalidPageId) { - t0 = std::chrono::steady_clock::now(); - page_id = find_leaf_page_id([this](uint64_t p) { return resident(p); }, page_id, descend_key); - l1_ns += page_id != kInvalidPageId ? dur_ns(t0) : 0; - } - // A lazy cursor over the current leaf's chain, not a materialized - // vector -- it yields borrowed key/cell Slices in key order and only - // resolves as far as the merge loop pulls, so a limit-bounded scan never - // pays for the rest of the leaf. The Slices stay valid for this whole - // synchronous call under the epoch guard entered above. - LeafChainCursor l1; - bool first_leaf = true; - - // Pull the next non-exhausted leaf (an all-tombstone/GC'd leaf yields - // nothing; keep walking right past it) until the cursor has an entry or the - // chain is exhausted. Idempotent when the cursor is already positioned. - auto refill_l1 = [&]() -> bool { - while (!l1.valid() && page_id != kInvalidPageId) { - PageBase *head = resident(page_id); - if (head == nullptr) { - page_id = kInvalidPageId; - break; - } - auto rt = std::chrono::steady_clock::now(); - l1.reset(head, gc); - // Only the first leaf can hold entries at or before the cursor: - // the descent landed on it. Seek past them by binary search - // instead of letting the merge loop step over them one by one. - if (first_leaf && !descend_key.empty()) { - l1.seek(descend_key, /*exclusive=*/has_start_bound && !start_inclusive); - } - first_leaf = false; - l1_ns += dur_ns(rt); - LeafBase *base = chain_leaf_base(head); - page_id = base != nullptr ? base->right_sibling() : kInvalidPageId; - // R58: prefetch the right-sibling leaf's memory while the merge - // loop works on the current leaf — overlaps the cache fill with - // merge work. The page is already resident (sync scan path); this - // targets CPU cache, not disk. Uses mapping_ directly to avoid - // the touch_tick overhead of resident(). - if (page_id != kInvalidPageId) { - uint64_t w = mapping_.get_word(page_id); - if (slot_word::is_resident(w)) { - __builtin_prefetch(slot_word::resident_ptr(w), 0, 2); - } - } - } - return l1.valid(); - }; - - size_t accumulated_bytes = 0; - auto consider = [&](Slice key, Slice cell) -> bool { - if (opt_.key_range.before(key)) { - return true; - } - if (opt_.key_range.at_or_after_end(key)) { - return false; - } - if (has_start_bound && (start_inclusive ? key.compare(start_after) < 0 : key.compare(start_after) <= 0)) { - return true; - } - if (!key.starts_with(prefix)) { - return true; - } - CellView v{cell}; - if (v.is_tombstone() && !include_tombstones) { - return true; - } - assert(out_packed != nullptr || out != nullptr); - size_t cur_count = out_packed != nullptr ? packed_count : out->size(); - if (limit != 0 && cur_count >= limit) { - if (truncated != nullptr) { - *truncated = true; - } - return false; // stop: a matching entry didn't fit - } - if (v.is_tombstone()) { - size_t entry_bytes = key.size(); - if (byte_budget != 0 && (strict_byte_budget || cur_count > 0) && - (accumulated_bytes > byte_budget || entry_bytes > byte_budget - accumulated_bytes)) { - if (truncated != nullptr) { - *truncated = true; - } - return false; // byte budget would be exceeded; keep what we have - } - if (out_packed != nullptr) { - out_packed->pack_u32(static_cast<uint32_t>(key.size())); - out_packed->append(key); - out_packed->pack_u64(v.slot()); - out_packed->push_back(1); - out_packed->pack_u32(0); - } - else { - out->push_back({.key = key.to_string(), .slot = v.slot(), .value = "", .tombstone = true}); - } - ++packed_count; - accumulated_bytes += entry_bytes; - return true; - } - size_t key_size = key.size(); - size_t value_size = 0; - if (!keys_only) { - value_size = v.is_overflow() ? v.overflow_len() : v.value().size(); - } - size_t entry_bytes = key_size + value_size; - if (byte_budget != 0 && (strict_byte_budget || cur_count > 0) && - (accumulated_bytes > byte_budget || entry_bytes > byte_budget - accumulated_bytes)) { - if (truncated != nullptr) { - *truncated = true; - } - return false; // byte budget would be exceeded; keep what we have - } - std::string val; - if (!keys_only) { - val = - v.is_overflow() ? assemble_overflow_value(v.overflow_head(), v.overflow_len()) : v.value().to_string(); - } - if (out_packed != nullptr) { - out_packed->pack_u32(static_cast<uint32_t>(key_size)); - out_packed->append(key); - out_packed->pack_u64(v.slot()); - out_packed->push_back(0); - out_packed->pack_u32(static_cast<uint32_t>(value_size)); - out_packed->append(val); - } - else { - out->push_back({.key = key.to_string(), .slot = v.slot(), .value = std::move(val), .tombstone = false}); - } - ++packed_count; - accumulated_bytes += entry_bytes; - if (byte_budget != 0 && entry_bytes > byte_budget) { - CRB_LOG_WARN("[{}] scan: oversized entry key_size={} value_size={} exceeds byte_budget={}", name_, key_size, - value_size, byte_budget); - } - return true; - }; - - // R58: merge loop with 2-source fast path + loser tree. On a key collision - // across multiple sources (possible across L0 streams -- see the L0Cursor - // comment above; L1 only ever collides with L0, never with itself), the - // highest-slot cell wins and every cursor sitting on that key is advanced, - // so a key present in more than one source still yields exactly one output - // entry. The match function: lower key wins; tie → higher slot; tie → - // lower source index (deterministic, matching the original iteration order). - // - // R50: L0 cursors borrow key/cell off the skip-list node. The winner is - // tracked as a CellVersion* (L0) or a cell Slice (L1); slot comparison - // uses cv->slot / CellView::slot respectively. The winning L0 cell is - // materialized into a contiguous buffer only when it reaches the output - // (O(limit), not O(N_l0)). - size_t n_valid_l0 = 0; - for (const auto &c : l0) { - if (c.cur.valid()) { - ++n_valid_l0; - } - } - - LoserTree lt; - std::vector<MergeSource> lt_sources; - bool lt_built = false; - - auto t_loop = std::chrono::steady_clock::now(); - size_t deadline_counter = 0; // check deadline every kDeadlineCheckInterval entries - while (true) { - // Periodic deadline check: amortize the clock read over 1024 entries. - // When exceeded, break with truncated = true and return the partial - // result accumulated so far. - if (deadline_ms != 0 && ++deadline_counter >= 1024) { - deadline_counter = 0; - auto now_ms = static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::system_clock::now().time_since_epoch()) - .count()); - if (now_ms >= deadline_ms) { - if (truncated != nullptr) { - *truncated = true; - } - break; - } - } - bool l1_was_valid_before = l1.valid(); - bool have_l1 = refill_l1(); - bool l1_refilled = have_l1 && !l1_was_valid_before; - - size_t n_sources = n_valid_l0 + (have_l1 ? 1 : 0); - if (n_sources == 0) { - break; - } - - Slice winner_key; - const CellVersion *l0_winner = nullptr; - Slice l1_winner_cell; - buffer l0_materialized; - - if (n_sources == 1) { - // Single source: no merge compare needed. - if (have_l1) { - winner_key = l1.key(); - l1_winner_cell = l1.cell(); - l1.next(); - } - else { - for (auto &c : l0) { - if (!c.cur.valid()) { - continue; - } - winner_key = c.cur.key(); - l0_winner = c.cur.cell_version(); - c.cur.prefetch_next(); - c.cur.advance(); - if (!c.cur.valid()) { - --n_valid_l0; - } - break; - } - } - } - else if (n_sources == 2) { - // 2-source fast path: 1 compare instead of 2×2. The common - // steady-state case (1 active L0 + L1, no frozen memtables). - ConcurrentSkipList::Cursor *c0 = nullptr; - ConcurrentSkipList::Cursor *c1 = nullptr; - for (auto &c : l0) { - if (!c.cur.valid()) { - continue; - } - if (c0 == nullptr) { - c0 = &c.cur; - } - else { - c1 = &c.cur; - break; - } - } - if (n_valid_l0 == 2) { - int cmp = c0->key().compare(c1->key()); - if (cmp < 0) { - winner_key = c0->key(); - l0_winner = c0->cell_version(); - c0->prefetch_next(); - c0->advance(); - if (!c0->valid()) { - --n_valid_l0; - } - } - else if (cmp > 0) { - winner_key = c1->key(); - l0_winner = c1->cell_version(); - c1->prefetch_next(); - c1->advance(); - if (!c1->valid()) { - --n_valid_l0; - } - } - else { - // Tie: higher slot wins, advance both. - const CellVersion *cv0 = c0->cell_version(); - const CellVersion *cv1 = c1->cell_version(); - uint64_t s0 = cv0 != nullptr ? cv0->slot : 0; - uint64_t s1 = cv1 != nullptr ? cv1->slot : 0; - if (s0 >= s1) { - winner_key = c0->key(); - l0_winner = cv0; - } - else { - winner_key = c1->key(); - l0_winner = cv1; - } - c0->prefetch_next(); - c0->advance(); - if (!c0->valid()) { - --n_valid_l0; - } - c1->prefetch_next(); - c1->advance(); - if (!c1->valid()) { - --n_valid_l0; - } - } - } - else { - // 1 valid L0 + L1. - if (c0 == nullptr) { - return Status::internal_error("scan cursor count disagrees with live sources"); - } - int cmp = c0->key().compare(l1.key()); - if (cmp < 0) { - winner_key = c0->key(); - l0_winner = c0->cell_version(); - c0->prefetch_next(); - c0->advance(); - if (!c0->valid()) { - --n_valid_l0; - } - } - else if (cmp > 0) { - winner_key = l1.key(); - l1_winner_cell = l1.cell(); - l1.next(); - } - else { - // Tie: higher slot wins, advance both. - const CellVersion *cv = c0->cell_version(); - uint64_t s0 = cv != nullptr ? cv->slot : 0; - uint64_t s1 = CellView{l1.cell()}.slot(); - if (s0 >= s1) { - winner_key = c0->key(); - l0_winner = cv; - } - else { - winner_key = l1.key(); - l1_winner_cell = l1.cell(); - } - c0->prefetch_next(); - c0->advance(); - if (!c0->valid()) { - --n_valid_l0; - } - l1.next(); - } - } - } - else { - // Loser tree (k > 2): O(log k) per merge step. - if (!lt_built) { - lt_sources.clear(); - lt_sources.reserve(l0.size() + 1); - for (auto &c : l0) { - lt_sources.push_back({.kind = MergeSource::kL0, .l0 = &c.cur, .l1 = nullptr}); - } - lt_sources.push_back({.kind = MergeSource::kL1, .l0 = nullptr, .l1 = &l1}); - lt.init(lt_sources.data(), static_cast<int>(lt_sources.size())); - lt_built = true; - } - else if (l1_refilled) { - // L1 refilled (new leaf): replay L1 in the tree. - lt.replay_source(static_cast<int>(lt_sources.size() - 1)); - } - if (!lt.winner_valid()) { - break; - } - int w = lt.winner(); - winner_key = lt_sources[w].key(); - // Capture winner's cell BEFORE advancing. - if (lt_sources[w].kind == MergeSource::kL0) { - l0_winner = lt_sources[w].l0->cell_version(); - } - else { - l1_winner_cell = lt_sources[w].l1->cell(); - } - // Prefetch + advance the winner. - lt_sources[w].prefetch_next(); - lt.advance_winner(); - if (lt_sources[w].kind == MergeSource::kL0 && !lt_sources[w].valid()) { - --n_valid_l0; - } - // Collision drain: advance all other sources on the same key - // (duplicate — already emitted). After the winner advances, any - // other cursor on the same key naturally bubbles to the root. - while (lt.winner_valid() && lt_sources[lt.winner()].key().compare(winner_key) == 0) { - int cw = lt.winner(); - lt_sources[cw].prefetch_next(); - lt.drain_winner(); - if (lt_sources[cw].kind == MergeSource::kL0 && !lt_sources[cw].valid()) { - --n_valid_l0; - } - } - } - - if (strict_byte_budget) { - if (opt_.key_range.at_or_after_end(winner_key) || (!end_key.empty() && winner_key.compare(end_key) >= 0) || - (!prefix.empty() && !winner_key.starts_with(prefix) && winner_key.compare(prefix) > 0)) { - break; - } - if (opt_.key_range.before(winner_key) || !winner_key.starts_with(prefix) || - (has_start_bound && - (start_inclusive ? winner_key.compare(start_after) < 0 : winner_key.compare(start_after) <= 0))) { - continue; - } - if (l0_winner != nullptr && l0_winner->cell.ownership() == buffer::mode::kExternal && - (l0_winner->flags & kFlagTombstone) == 0) { - const size_t value_size = keys_only ? 0 : l0_winner->cell.size(); - if (accumulated_bytes > byte_budget || winner_key.size() > byte_budget - accumulated_bytes || - value_size > byte_budget - accumulated_bytes - winner_key.size()) { - if (truncated != nullptr) { - *truncated = true; - } - break; - } - } - } - - // Materialize the winning cell for the consider lambda. L1: borrow - // directly. L0: materialize a contiguous cell (only for the winner). - Slice winner_cell; - if (l0_winner != nullptr) { - if (l0_winner->cell.ownership() != buffer::mode::kExternal) { - winner_cell = l0_winner->cell.slice(); - } - else { - // Split cell (R30): build the contiguous [header][value]. - size_t vlen = l0_winner->cell.size(); - l0_materialized = buffer::alloc(vlen, kCellHeaderSize); - uint8_t *p = l0_materialized.data(); - for (int i = 0; i < 8; ++i) { - p[i] = static_cast<uint8_t>((l0_winner->slot >> (8 * i)) & 0xff); - } - p[8] = l0_winner->flags; - if (vlen > 0) { - std::memcpy(p + kCellHeaderSize, l0_winner->cell.data(), vlen); - } - winner_cell = l0_materialized.slice(); - } - } - else { - winner_cell = l1_winner_cell; - } - - // Early stop: every stream is non-decreasing, so once a key has moved - // past the prefix range (not merely before it), no later key can match. - if (!prefix.empty() && !winner_key.starts_with(prefix) && winner_key.compare(prefix) > 0) { - break; - } - // Exclusive upper bound: once the winner reaches end_key, no later key - // can be < end_key (streams are non-decreasing), so stop. - if (!end_key.empty() && winner_key.compare(end_key) >= 0) { - break; - } - if (!consider(winner_key, winner_cell)) { - break; - } - } - auto loop_ns = dur_ns(t_loop); - // merge = loop overhead excluding L1 work (refill bookkeeping + min-key - // select + winner + decode). - uint64_t merge_ns = (loop_ns > l1_ns) ? loop_ns - l1_ns : 0; - uint64_t total_ns = dur_ns(t_total); - if (metrics_.scan_l != nullptr) { - metrics_.scan_entries_c->inc_by(packed_count); - metrics_.scan_l->observe(total_ns); - metrics_.scan_l0_l->observe(l0_ns); - metrics_.scan_l1_l->observe(l1_ns); - metrics_.scan_merge_l->observe(merge_ns); - } - if (out_count != nullptr) { - *out_count = packed_count; - } - return Status::Ok(); -} - -Status Crowdbtree::seek_reverse(Slice start_key, bool inclusive, Slice begin_key, scan_entry *out, bool *found) const -{ - auto generation = generation_.enter(); - if (out == nullptr || found == nullptr) { - return Status::invalid_argument("seek_reverse output is null"); - } - *out = {}; - EpochManager::Guard guard = epoch_.enter(); - auto memtables = all_memtables(); - const uint64_t root_page_id = root_page_id_.load(); - const uint64_t gc_floor = gc_floor_.load(); - *found = seek_reverse_guarded(start_key, true, inclusive, begin_key, memtables, root_page_id, gc_floor, out); - return Status::Ok(); -} - -bool Crowdbtree::seek_reverse_guarded(Slice start_key, bool has_start_bound, bool inclusive, Slice begin_key, - const std::vector<MemTableSource> &memtables, uint64_t root_page_id, - uint64_t gc_floor, scan_entry *out) const -{ - - struct ParentStep - { - InnerBase *page; - size_t child_index; - }; - - auto resolve_base = [](PageBase *head) { - while (head != nullptr && head->type == page_type::kBatchDelta) { - head = head->next; - } - return head; - }; - auto l1_predecessor = [&](Slice bound, bool has_bound, bool include, Slice *key, Slice *cell) -> bool { - std::vector<ParentStep> path; - uint64_t page_id = root_page_id; - while (page_id != kInvalidPageId) { - PageBase *head = resident(page_id); - PageBase *base = resolve_base(head); - if (base == nullptr) { - return false; - } - if (base->type == page_type::kLeafBase) { - LeafChainCursor cursor(head, gc_floor); - if (has_bound) { - cursor.seek_reverse(bound, include); - } - else { - cursor.seek_last(); - } - if (cursor.valid()) { - *key = cursor.key(); - *cell = cursor.cell(); - return true; - } - break; - } - auto *inner = static_cast<InnerBase *>(base); - size_t index = has_bound ? inner->child_index_for(bound) : inner->num_children() - 1; - path.push_back({.page = inner, .child_index = index}); - page_id = inner->child_at(index); - } - - while (!path.empty()) { - ParentStep step = path.back(); - path.pop_back(); - if (step.child_index == 0) { - continue; - } - page_id = step.page->child_at(step.child_index - 1); - while (page_id != kInvalidPageId) { - PageBase *head = resident(page_id); - PageBase *base = resolve_base(head); - if (base == nullptr) { - return false; - } - if (base->type == page_type::kLeafBase) { - LeafChainCursor cursor(head, gc_floor); - cursor.seek_last(); - if (cursor.valid()) { - *key = cursor.key(); - *cell = cursor.cell(); - return true; - } - break; - } - auto *inner = static_cast<InnerBase *>(base); - size_t index = inner->num_children() - 1; - path.push_back({.page = inner, .child_index = index}); - page_id = inner->child_at(index); - } - } - return false; - }; - - std::string bound = start_key.to_string(); - bool has_bound = has_start_bound; - bool include_bound = inclusive; - while (true) { - Slice winner_key; - const CellVersion *winner_l0 = nullptr; - Slice winner_l1; - bool have_winner = false; - for (const auto &memtable : memtables) { - auto cursor = memtable->cursor_reverse(Slice(bound), has_bound, include_bound); - if (!cursor.valid()) { - continue; - } - uint64_t slot = cursor.cell_version() != nullptr ? cursor.cell_version()->slot : 0; - uint64_t winner_slot = 0; - if (winner_l0 != nullptr) { - winner_slot = winner_l0->slot; - } - else if (!winner_l1.empty()) { - winner_slot = CellView{winner_l1}.slot(); - } - int comparison = have_winner ? cursor.key().compare(winner_key) : 1; - if (comparison > 0 || (comparison == 0 && slot > winner_slot)) { - winner_key = cursor.key(); - winner_l0 = cursor.cell_version(); - winner_l1 = {}; - have_winner = true; - } - } - - Slice l1_key; - Slice l1_cell; - if (l1_predecessor(Slice(bound), has_bound, include_bound, &l1_key, &l1_cell)) { - uint64_t l1_slot = CellView{l1_cell}.slot(); - uint64_t winner_slot = 0; - if (winner_l0 != nullptr) { - winner_slot = winner_l0->slot; - } - else if (!winner_l1.empty()) { - winner_slot = CellView{winner_l1}.slot(); - } - int comparison = have_winner ? l1_key.compare(winner_key) : 1; - if (comparison > 0 || (comparison == 0 && l1_slot > winner_slot)) { - winner_key = l1_key; - winner_l0 = nullptr; - winner_l1 = l1_cell; - have_winner = true; - } - } - if (!have_winner || (!begin_key.empty() && winner_key.compare(begin_key) < 0) || - opt_.key_range.before(winner_key)) { - return false; - } - if (opt_.key_range.at_or_after_end(winner_key)) { - bound = winner_key.to_string(); - has_bound = true; - include_bound = false; - continue; - } - - buffer materialized; - Slice winner_cell = winner_l1; - if (winner_l0 != nullptr) { - if (winner_l0->cell.ownership() != buffer::mode::kExternal) { - winner_cell = winner_l0->cell.slice(); - } - else { - size_t value_len = winner_l0->cell.size(); - materialized = buffer::alloc(value_len, kCellHeaderSize); - uint8_t *data = materialized.data(); - for (int i = 0; i < 8; ++i) { - data[i] = static_cast<uint8_t>((winner_l0->slot >> (8 * i)) & 0xff); - } - data[8] = winner_l0->flags; - if (value_len > 0) { - std::memcpy(data + kCellHeaderSize, winner_l0->cell.data(), value_len); - } - winner_cell = materialized.slice(); - } - } - CellView value{winner_cell}; - if (value.is_tombstone()) { - bound = winner_key.to_string(); - has_bound = true; - include_bound = false; - continue; - } - out->key = winner_key.to_string(); - out->slot = value.slot(); - out->value = value.is_overflow() ? assemble_overflow_value(value.overflow_head(), value.overflow_len()) - : value.value().to_string(); - out->tombstone = false; - return true; - } -} - -Status Crowdbtree::scan_reverse(Slice start_key, bool has_start_bound, bool start_inclusive, Slice begin_key, - size_t limit, size_t byte_budget, std::vector<scan_entry> *out, bool *truncated) const -{ - auto generation = generation_.enter(); - if (out == nullptr || truncated == nullptr) { - return Status::invalid_argument("scan_reverse output is null"); - } - out->clear(); - *truncated = false; - EpochManager::Guard guard = epoch_.enter(); - auto memtables = all_memtables(); - const uint64_t root_page_id = root_page_id_.load(); - const uint64_t gc_floor = gc_floor_.load(); - std::string bound = start_key.to_string(); - bool has_bound = has_start_bound; - bool inclusive = start_inclusive; - size_t accumulated_bytes = 0; - while (true) { - scan_entry entry; - if (!seek_reverse_guarded(Slice(bound), has_bound, inclusive, begin_key, memtables, root_page_id, gc_floor, - &entry)) { - break; - } - size_t entry_bytes = entry.key.size() + entry.value.size(); - if ((limit != 0 && out->size() >= limit) || - (byte_budget != 0 && !out->empty() && accumulated_bytes + entry_bytes > byte_budget)) { - *truncated = true; - break; - } - bound = entry.key; - has_bound = true; - inclusive = false; - accumulated_bytes += entry_bytes; - out->push_back(std::move(entry)); - } - return Status::Ok(); -} - -bool Crowdbtree::try_scan_reverse_no_load(Slice prefix, Slice start_after, Slice end_key, size_t limit, - size_t byte_budget, bool keys_only, uint64_t deadline_ms, - ScanPackedBuf *out_packed, size_t *out_count, bool *truncated, - uint64_t *out_pending_page_id) const -{ - auto generation = generation_.enter(); - *out_packed = ScanPackedBuf{}; - *out_count = 0; - *truncated = false; - *out_pending_page_id = kInvalidPageId; - - EpochManager::Guard guard = epoch_.enter(); - auto memtables = all_memtables(); - const uint64_t root_page_id = root_page_id_.load(); - const uint64_t gc_floor = gc_floor_.load(); - uint64_t blocked = kInvalidPageId; - - auto probe = [this, &blocked](uint64_t page_id) -> PageBase * { - uint64_t word = mapping_.get_word(page_id); - if (slot_word::is_empty(word)) { - return nullptr; - } - if (slot_word::is_unloaded(word)) { - blocked = page_id; - return nullptr; - } - PageBase *page = slot_word::resident_ptr(word); - page->last_touch_tick.store(touch_tick_.fetch_add(1, std::memory_order_relaxed), std::memory_order_relaxed); - return page; - }; - auto resolve_base = [](PageBase *head) { - while (head != nullptr && head->type == page_type::kBatchDelta) { - head = head->next; - } - return head; - }; - - struct ParentStep - { - InnerBase *page; - size_t child_index; - }; - - auto l1_predecessor = [&](Slice bound, bool has_bound, Slice *key, Slice *cell) -> bool { - std::vector<ParentStep> path; - uint64_t page_id = root_page_id; - while (page_id != kInvalidPageId) { - PageBase *head = probe(page_id); - if (blocked != kInvalidPageId) { - return false; - } - PageBase *base = resolve_base(head); - if (base == nullptr) { - return false; - } - if (base->type == page_type::kLeafBase) { - LeafChainCursor cursor(head, gc_floor); - if (has_bound) { - cursor.seek_reverse(bound, false); - } - else { - cursor.seek_last(); - } - if (cursor.valid()) { - *key = cursor.key(); - *cell = cursor.cell(); - return true; - } - break; - } - auto *inner = static_cast<InnerBase *>(base); - size_t index = has_bound ? inner->child_index_for(bound) : inner->num_children() - 1; - path.push_back({.page = inner, .child_index = index}); - page_id = inner->child_at(index); - } - while (!path.empty()) { - ParentStep step = path.back(); - path.pop_back(); - if (step.child_index == 0) { - continue; - } - page_id = step.page->child_at(step.child_index - 1); - while (page_id != kInvalidPageId) { - PageBase *head = probe(page_id); - if (blocked != kInvalidPageId) { - return false; - } - PageBase *base = resolve_base(head); - if (base == nullptr) { - return false; - } - if (base->type == page_type::kLeafBase) { - LeafChainCursor cursor(head, gc_floor); - cursor.seek_last(); - if (cursor.valid()) { - *key = cursor.key(); - *cell = cursor.cell(); - return true; - } - break; - } - auto *inner = static_cast<InnerBase *>(base); - size_t index = inner->num_children() - 1; - path.push_back({.page = inner, .child_index = index}); - page_id = inner->child_at(index); - } - } - return false; - }; - - std::string bound; - bool has_bound = false; - if (!start_after.empty()) { - bound = start_after.to_string(); - has_bound = true; - } - else if (!end_key.empty()) { - bound = end_key.to_string(); - has_bound = true; - } - else if (!prefix.empty()) { - bound = prefix.to_string(); - for (size_t i = bound.size(); i > 0; --i) { - auto byte = static_cast<uint8_t>(bound[i - 1]); - if (byte != 0xff) { - bound[i - 1] = static_cast<char>(byte + 1); - bound.resize(i); - has_bound = true; - break; - } - } - } - - size_t accumulated_bytes = 0; - size_t deadline_counter = 0; - while (true) { - if (deadline_ms != 0 && ++deadline_counter >= 1024) { - deadline_counter = 0; - auto now_ms = static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::system_clock::now().time_since_epoch()) - .count()); - if (now_ms >= deadline_ms) { - *truncated = true; - return true; - } - } - - Slice winner_key; - const CellVersion *winner_l0 = nullptr; - Slice winner_l1; - bool have_winner = false; - for (const auto &memtable : memtables) { - auto cursor = memtable->cursor_reverse(Slice(bound), has_bound, false); - if (!cursor.valid()) { - continue; - } - uint64_t slot = cursor.cell_version() != nullptr ? cursor.cell_version()->slot : 0; - uint64_t winner_slot = 0; - if (winner_l0 != nullptr) { - winner_slot = winner_l0->slot; - } - else if (!winner_l1.empty()) { - winner_slot = CellView{winner_l1}.slot(); - } - int comparison = have_winner ? cursor.key().compare(winner_key) : 1; - if (comparison > 0 || (comparison == 0 && slot > winner_slot)) { - winner_key = cursor.key(); - winner_l0 = cursor.cell_version(); - winner_l1 = {}; - have_winner = true; - } - } - Slice l1_key; - Slice l1_cell; - if (l1_predecessor(Slice(bound), has_bound, &l1_key, &l1_cell)) { - uint64_t l1_slot = CellView{l1_cell}.slot(); - uint64_t winner_slot = 0; - if (winner_l0 != nullptr) { - winner_slot = winner_l0->slot; - } - else if (!winner_l1.empty()) { - winner_slot = CellView{winner_l1}.slot(); - } - int comparison = have_winner ? l1_key.compare(winner_key) : 1; - if (comparison > 0 || (comparison == 0 && l1_slot > winner_slot)) { - winner_key = l1_key; - winner_l0 = nullptr; - winner_l1 = l1_cell; - have_winner = true; - } - } - if (blocked != kInvalidPageId) { - *out_pending_page_id = blocked; - return false; - } - if (!have_winner || opt_.key_range.before(winner_key)) { - return true; - } - bound = winner_key.to_string(); - has_bound = true; - - if ((!end_key.empty() && winner_key.compare(end_key) >= 0) || opt_.key_range.at_or_after_end(winner_key)) { - continue; - } - if (!prefix.empty() && !winner_key.starts_with(prefix)) { - if (winner_key.compare(prefix) < 0) { - return true; - } - continue; - } - - buffer materialized; - Slice winner_cell = winner_l1; - if (winner_l0 != nullptr) { - if (winner_l0->cell.ownership() != buffer::mode::kExternal) { - winner_cell = winner_l0->cell.slice(); - } - else { - size_t value_len = winner_l0->cell.size(); - materialized = buffer::alloc(value_len, kCellHeaderSize); - uint8_t *data = materialized.data(); - for (int i = 0; i < 8; ++i) { - data[i] = static_cast<uint8_t>((winner_l0->slot >> (8 * i)) & 0xff); - } - data[8] = winner_l0->flags; - if (value_len > 0) { - std::memcpy(data + kCellHeaderSize, winner_l0->cell.data(), value_len); - } - winner_cell = materialized.slice(); - } - } - CellView value{winner_cell}; - if (value.is_tombstone()) { - continue; - } - if (limit != 0 && *out_count >= limit) { - *truncated = true; - return true; - } - std::string val; - if (!keys_only) { - val = value.is_overflow() ? assemble_overflow_value(value.overflow_head(), value.overflow_len()) - : value.value().to_string(); - } - size_t entry_bytes = winner_key.size() + val.size(); - if (byte_budget != 0 && *out_count > 0 && accumulated_bytes + entry_bytes > byte_budget) { - *truncated = true; - return true; - } - out_packed->pack_u32(static_cast<uint32_t>(winner_key.size())); - out_packed->append(winner_key); - out_packed->pack_u64(value.slot()); - out_packed->push_back(0); - out_packed->pack_u32(static_cast<uint32_t>(val.size())); - out_packed->append(val); - ++*out_count; - accumulated_bytes += entry_bytes; - } -} - -bool Crowdbtree::try_scan_no_load( - Slice prefix, Slice start_after, Slice end_key, size_t limit, size_t byte_budget, bool keys_only, - uint64_t deadline_ms, - std::vector<scan_entry> *out, // NOLINT(readability-non-const-parameter) written to via push_back - bool *truncated, uint64_t *out_pending_page_id, ScanPackedBuf *out_packed, - size_t *out_count) // NOLINT(readability-non-const-parameter) written to via *out_count - const -{ - auto generation = generation_.enter(); - if (out == nullptr && out_packed == nullptr) { - return false; - } - if (out != nullptr) { - out->clear(); - } - if (out_packed != nullptr) { - *out_packed = ScanPackedBuf{}; - } - size_t packed_count = 0; - if (truncated != nullptr) { - *truncated = false; - } - EpochManager::Guard guard = epoch_.enter(); - - auto dur_ns = [](std::chrono::steady_clock::time_point from) { - return static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - from).count()); - }; - auto t_total = std::chrono::steady_clock::now(); - - // L0: identical to scan() -- never touches the page store. R50: lock-free - // cursor, no snapshot copy. - struct L0Cursor - { - ConcurrentSkipList::Cursor cur; - }; - - auto t0 = std::chrono::steady_clock::now(); - std::vector<L0Cursor> l0; - auto sources_owner = all_memtables(); - l0.reserve(sources_owner.size()); - for (auto &mt : sources_owner) { - l0.push_back({.cur = mt->cursor(start_after)}); - } - uint64_t l0_ns = dur_ns(t0); - - // Non-blocking probe (mirrors try_get_view_no_load's `probe`): bails out - // on an unloaded slot instead of demand-loading it, recording which - // page_id via `blocked_page_id`. - uint64_t blocked_page_id = kInvalidPageId; - auto probe = [this, &blocked_page_id](uint64_t p) -> PageBase *{ - uint64_t w = mapping_.get_word(p); - if (slot_word::is_empty(w)) { - return nullptr; - } - if (slot_word::is_unloaded(w)) { - blocked_page_id = p; - return nullptr; - } - PageBase *v = slot_word::resident_ptr(w); - v->last_touch_tick.store(touch_tick_.fetch_add(1, std::memory_order_relaxed), std::memory_order_relaxed); - return v; - }; - - uint64_t l1_ns = 0; - uint64_t gc = gc_floor_.load(); - uint64_t page_id = root_page_id_.load(); - // Descend at the cursor when present, else at the prefix start -- see - // scan()'s own comment. - Slice descend_key = !start_after.empty() ? start_after : prefix; - if (page_id != kInvalidPageId) { - t0 = std::chrono::steady_clock::now(); - page_id = find_leaf_page_id(probe, page_id, descend_key); - if (blocked_page_id != kInvalidPageId) { - *out_pending_page_id = blocked_page_id; - return false; // genuine miss on the initial descent - } - l1_ns += page_id != kInvalidPageId ? dur_ns(t0) : 0; - } - // Lazy leaf cursor -- see scan()'s own comment. - LeafChainCursor l1; - bool first_leaf = true; - - auto refill_l1 = [&]() -> bool { - while (!l1.valid() && page_id != kInvalidPageId) { - PageBase *head = probe(page_id); - if (blocked_page_id != kInvalidPageId) { - return false; // caller checks blocked_page_id, distinct from "chain exhausted" - } - if (head == nullptr) { - page_id = kInvalidPageId; - break; - } - auto rt = std::chrono::steady_clock::now(); - l1.reset(head, gc); - if (first_leaf && !descend_key.empty()) { - l1.seek(descend_key, /*exclusive=*/!start_after.empty()); - } - first_leaf = false; - l1_ns += dur_ns(rt); - LeafBase *base = chain_leaf_base(head); - page_id = base != nullptr ? base->right_sibling() : kInvalidPageId; - // R58: prefetch the right-sibling leaf (see scan()'s refill_l1). - if (page_id != kInvalidPageId) { - uint64_t w = mapping_.get_word(page_id); - if (slot_word::is_resident(w)) { - __builtin_prefetch(slot_word::resident_ptr(w), 0, 2); - } - } - } - return l1.valid(); - }; - - size_t accumulated_bytes = 0; - auto consider = [&](Slice key, Slice cell) -> bool { - if (opt_.key_range.before(key)) { - return true; - } - if (opt_.key_range.at_or_after_end(key)) { - return false; - } - if (!start_after.empty() && key.compare(start_after) <= 0) { - return true; // cursor: skip keys <= start_after (exclusive lower bound) - } - if (!key.starts_with(prefix)) { - return true; - } - CellView v{cell}; - if (v.is_tombstone()) { - return true; - } - assert(out_packed != nullptr || out != nullptr); - size_t cur_count = out_packed != nullptr ? packed_count : out->size(); - if (limit != 0 && cur_count >= limit) { - if (truncated != nullptr) { - *truncated = true; - } - return false; - } - std::string val; - if (!keys_only) { - val = - v.is_overflow() ? assemble_overflow_value(v.overflow_head(), v.overflow_len()) : v.value().to_string(); - } - size_t key_size = key.size(); - size_t value_size = val.size(); - size_t entry_bytes = key_size + value_size; - if (byte_budget != 0 && cur_count > 0 && accumulated_bytes + entry_bytes > byte_budget) { - if (truncated != nullptr) { - *truncated = true; - } - return false; // byte budget would be exceeded; keep what we have - } - if (out_packed != nullptr) { - out_packed->pack_u32(static_cast<uint32_t>(key_size)); - out_packed->append(key); - out_packed->pack_u64(v.slot()); - out_packed->push_back(0); - out_packed->pack_u32(static_cast<uint32_t>(value_size)); - out_packed->append(val); - } - else { - out->push_back({.key = key.to_string(), .slot = v.slot(), .value = std::move(val)}); - } - ++packed_count; - accumulated_bytes += entry_bytes; - if (byte_budget != 0 && entry_bytes > byte_budget) { - CRB_LOG_WARN("[{}] scan: oversized entry key_size={} value_size={} exceeds byte_budget={}", name_, key_size, - value_size, byte_budget); - } - return true; - }; - - // R58: merge loop with 2-source fast path + loser tree (same structure as - // scan()'s — see the comment there). The only difference is the - // blocked_page_id early-return on a cold leaf. - size_t n_valid_l0 = 0; - for (const auto &c : l0) { - if (c.cur.valid()) { - ++n_valid_l0; - } - } - - LoserTree lt; - std::vector<MergeSource> lt_sources; - bool lt_built = false; - - auto t_loop = std::chrono::steady_clock::now(); - size_t deadline_counter = 0; // check deadline every kDeadlineCheckInterval entries - while (true) { - // Periodic deadline check (same as scan()'s). - if (deadline_ms != 0 && ++deadline_counter >= 1024) { - deadline_counter = 0; - auto now_ms = static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::system_clock::now().time_since_epoch()) - .count()); - if (now_ms >= deadline_ms) { - if (truncated != nullptr) { - *truncated = true; - } - return true; // resolved with partial result - } - } - bool l1_was_valid_before = l1.valid(); - bool have_l1 = refill_l1(); - if (blocked_page_id != kInvalidPageId) { - *out_pending_page_id = blocked_page_id; - return false; // genuine miss mid-walk - } - bool l1_refilled = have_l1 && !l1_was_valid_before; - - size_t n_sources = n_valid_l0 + (have_l1 ? 1 : 0); - if (n_sources == 0) { - break; - } - - Slice winner_key; - const CellVersion *l0_winner = nullptr; - Slice l1_winner_cell; - buffer l0_materialized; - - if (n_sources == 1) { - // Single source: no merge compare needed. - if (have_l1) { - winner_key = l1.key(); - l1_winner_cell = l1.cell(); - l1.next(); - } - else { - for (auto &c : l0) { - if (!c.cur.valid()) { - continue; - } - winner_key = c.cur.key(); - l0_winner = c.cur.cell_version(); - c.cur.prefetch_next(); - c.cur.advance(); - if (!c.cur.valid()) { - --n_valid_l0; - } - break; - } - } - } - else if (n_sources == 2) { - // 2-source fast path: 1 compare instead of 2×2. - ConcurrentSkipList::Cursor *c0 = nullptr; - ConcurrentSkipList::Cursor *c1 = nullptr; - for (auto &c : l0) { - if (!c.cur.valid()) { - continue; - } - if (c0 == nullptr) { - c0 = &c.cur; - } - else { - c1 = &c.cur; - break; - } - } - if (n_valid_l0 == 2) { - int cmp = c0->key().compare(c1->key()); - if (cmp < 0) { - winner_key = c0->key(); - l0_winner = c0->cell_version(); - c0->prefetch_next(); - c0->advance(); - if (!c0->valid()) { - --n_valid_l0; - } - } - else if (cmp > 0) { - winner_key = c1->key(); - l0_winner = c1->cell_version(); - c1->prefetch_next(); - c1->advance(); - if (!c1->valid()) { - --n_valid_l0; - } - } - else { - const CellVersion *cv0 = c0->cell_version(); - const CellVersion *cv1 = c1->cell_version(); - uint64_t s0 = cv0 != nullptr ? cv0->slot : 0; - uint64_t s1 = cv1 != nullptr ? cv1->slot : 0; - if (s0 >= s1) { - winner_key = c0->key(); - l0_winner = cv0; - } - else { - winner_key = c1->key(); - l0_winner = cv1; - } - c0->prefetch_next(); - c0->advance(); - if (!c0->valid()) { - --n_valid_l0; - } - c1->prefetch_next(); - c1->advance(); - if (!c1->valid()) { - --n_valid_l0; - } - } - } - else { - int cmp = c0->key().compare(l1.key()); - if (cmp < 0) { - winner_key = c0->key(); - l0_winner = c0->cell_version(); - c0->prefetch_next(); - c0->advance(); - if (!c0->valid()) { - --n_valid_l0; - } - } - else if (cmp > 0) { - winner_key = l1.key(); - l1_winner_cell = l1.cell(); - l1.next(); - } - else { - const CellVersion *cv = c0->cell_version(); - uint64_t s0 = cv != nullptr ? cv->slot : 0; - uint64_t s1 = CellView{l1.cell()}.slot(); - if (s0 >= s1) { - winner_key = c0->key(); - l0_winner = cv; - } - else { - winner_key = l1.key(); - l1_winner_cell = l1.cell(); - } - c0->prefetch_next(); - c0->advance(); - if (!c0->valid()) { - --n_valid_l0; - } - l1.next(); - } - } - } - else { - // Loser tree (k > 2): O(log k) per merge step. - if (!lt_built) { - lt_sources.clear(); - lt_sources.reserve(l0.size() + 1); - for (auto &c : l0) { - lt_sources.push_back({.kind = MergeSource::kL0, .l0 = &c.cur, .l1 = nullptr}); - } - lt_sources.push_back({.kind = MergeSource::kL1, .l0 = nullptr, .l1 = &l1}); - lt.init(lt_sources.data(), static_cast<int>(lt_sources.size())); - lt_built = true; - } - else if (l1_refilled) { - lt.replay_source(static_cast<int>(lt_sources.size() - 1)); - } - if (!lt.winner_valid()) { - break; - } - int w = lt.winner(); - winner_key = lt_sources[w].key(); - if (lt_sources[w].kind == MergeSource::kL0) { - l0_winner = lt_sources[w].l0->cell_version(); - } - else { - l1_winner_cell = lt_sources[w].l1->cell(); - } - lt_sources[w].prefetch_next(); - lt.advance_winner(); - if (lt_sources[w].kind == MergeSource::kL0 && !lt_sources[w].valid()) { - --n_valid_l0; - } - while (lt.winner_valid() && lt_sources[lt.winner()].key().compare(winner_key) == 0) { - int cw = lt.winner(); - lt_sources[cw].prefetch_next(); - lt.drain_winner(); - if (lt_sources[cw].kind == MergeSource::kL0 && !lt_sources[cw].valid()) { - --n_valid_l0; - } - } - } - - Slice winner_cell; - if (l0_winner != nullptr) { - if (l0_winner->cell.ownership() != buffer::mode::kExternal) { - winner_cell = l0_winner->cell.slice(); - } - else { - size_t vlen = l0_winner->cell.size(); - l0_materialized = buffer::alloc(vlen, kCellHeaderSize); - uint8_t *p = l0_materialized.data(); - for (int i = 0; i < 8; ++i) { - p[i] = static_cast<uint8_t>((l0_winner->slot >> (8 * i)) & 0xff); - } - p[8] = l0_winner->flags; - if (vlen > 0) { - std::memcpy(p + kCellHeaderSize, l0_winner->cell.data(), vlen); - } - winner_cell = l0_materialized.slice(); - } - } - else { - winner_cell = l1_winner_cell; - } - - if (!prefix.empty() && !winner_key.starts_with(prefix) && winner_key.compare(prefix) > 0) { - break; - } - if (!end_key.empty() && winner_key.compare(end_key) >= 0) { - break; - } - if (!consider(winner_key, winner_cell)) { - break; - } - } - auto loop_ns = dur_ns(t_loop); - uint64_t merge_ns = (loop_ns > l1_ns) ? loop_ns - l1_ns : 0; - uint64_t total_ns = dur_ns(t_total); - if (metrics_.scan_l != nullptr) { - metrics_.scan_entries_c->inc_by(packed_count); - metrics_.scan_l->observe(total_ns); - metrics_.scan_l0_l->observe(l0_ns); - metrics_.scan_l1_l->observe(l1_ns); - metrics_.scan_merge_l->observe(merge_ns); - } - if (out_count != nullptr) { - *out_count = packed_count; - } - return true; // fully resolved -} - -// Build the resume `start_after` for scan_async_attempt: if entries have -// been accumulated across prior cold-leaf retries, resume from the last -// resolved key; otherwise use the original start_after. This avoids -// re-traversing already-resolved leaves after a demand-load completes. -static std::shared_ptr<std::string> make_resume_after(const std::shared_ptr<std::string> &start_after_owned, - const std::shared_ptr<std::string> &last_key) -{ - if (last_key != nullptr && !last_key->empty()) { - return last_key; - } - return start_after_owned; -} - -void Crowdbtree::scan_async(Slice prefix, Slice start_after, Slice end_key, size_t limit, size_t byte_budget, - bool keys_only, uint64_t deadline_ms, - std::function<void(Status, ScanPackedBuf, bool)> on_done) const -{ - // Copy the keys upfront: unlike scan()'s Slice (borrowed, valid only - // for this one synchronous call), scan_async's keys must survive across - // an arbitrary number of async round trips. `accumulated` collects - // packed entries resolved before each cold leaf so retries resume from - // the last resolved key instead of re-traversing already-resolved leaves. - scan_async_attempt(std::make_shared<std::string>(prefix.to_string()), - std::make_shared<std::string>(start_after.to_string()), - std::make_shared<std::string>(end_key.to_string()), limit, byte_budget, keys_only, deadline_ms, - std::make_shared<ScanPackedBuf>(), nullptr, 0, std::move(on_done)); -} - -void Crowdbtree::scan_directional_async(Slice prefix, Slice start_after, Slice end_key, size_t limit, - size_t byte_budget, bool keys_only, uint64_t deadline_ms, bool reverse, - std::function<void(Status, ScanPackedBuf, bool)> on_done) const -{ - if (!reverse) { - scan_async(prefix, start_after, end_key, limit, byte_budget, keys_only, deadline_ms, std::move(on_done)); - return; - } - scan_reverse_async_attempt(std::make_shared<std::string>(prefix.to_string()), - std::make_shared<std::string>(start_after.to_string()), - std::make_shared<std::string>(end_key.to_string()), limit, byte_budget, keys_only, - deadline_ms, std::make_shared<ScanPackedBuf>(), nullptr, 0, std::move(on_done)); -} - -// Extract the last key from a packed scan buffer (wire format: -// [u32 klen][key][u64 slot][u8 tombstone][u32 vlen][value] per entry). -// Used by scan_async_attempt to resume from the last resolved key. -static std::string last_key_from_packed(const uint8_t *data, size_t len) -{ - size_t pos = 0; - std::string last; - while (pos + 4 <= len) { - uint32_t klen = 0; - for (int i = 0; i < 4; ++i) { - klen |= static_cast<uint32_t>(data[pos + i]) << (8 * i); - } - pos += 4; - if (pos + klen > len) { - break; - } - last.assign(reinterpret_cast<const char *>(data + pos), klen); - pos += klen; - // skip slot (8) + tombstone (1) - if (pos + 9 > len) { - break; - } - pos += 9; - if (pos + 4 > len) { - break; - } - uint32_t vlen = 0; - for (int i = 0; i < 4; ++i) { - vlen |= static_cast<uint32_t>(data[pos + i]) << (8 * i); - } - pos += 4; - pos += vlen; - } - return last; -} - -static size_t payload_bytes_from_packed(const uint8_t *data, size_t len) -{ - size_t pos = 0; - size_t total = 0; - while (pos + 4 <= len) { - uint32_t key_len = 0; - for (int i = 0; i < 4; ++i) { - key_len |= static_cast<uint32_t>(data[pos + i]) << (8 * i); - } - pos += 4; - if (pos + key_len + 13 > len) { - break; - } - pos += key_len + 9; - uint32_t value_len = 0; - for (int i = 0; i < 4; ++i) { - value_len |= static_cast<uint32_t>(data[pos + i]) << (8 * i); - } - pos += 4; - if (pos + value_len > len) { - break; - } - pos += value_len; - total += key_len + value_len; - } - return total; -} - -void Crowdbtree::scan_async_attempt(std::shared_ptr<std::string> prefix_owned, - const std::shared_ptr<std::string> &start_after_owned, - const std::shared_ptr<std::string> &end_key_owned, size_t limit, size_t byte_budget, - bool keys_only, uint64_t deadline_ms, std::shared_ptr<ScanPackedBuf> accumulated, - std::shared_ptr<std::string> last_key, size_t accumulated_count, - std::function<void(Status, ScanPackedBuf, bool)> on_done) const -{ - // Adjust the byte budget by entries already accumulated across prior - // cold-leaf retries, mirroring the remaining_limit adjustment below. - size_t accumulated_bytes = payload_bytes_from_packed(accumulated->data(), accumulated->size()); - if (byte_budget != 0 && accumulated_bytes >= byte_budget && accumulated_count > 0) { - on_done(Status::Ok(), std::move(*accumulated), true); - return; - } - size_t remaining_byte_budget = (byte_budget != 0) ? byte_budget - accumulated_bytes : 0; - - // Deadline check before each retry: if exceeded, deliver the accumulated - // partial result with truncated = true instead of starting another attempt. - if (deadline_ms != 0) { - auto now_ms = static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()) - .count()); - if (now_ms >= deadline_ms) { - on_done(Status::Ok(), std::move(*accumulated), true); - return; - } - } - - ScanPackedBuf out_packed; - size_t out_count = 0; - bool truncated = false; - uint64_t pending_page_id = kInvalidPageId; - if (try_scan_no_load(Slice(*prefix_owned), Slice(*start_after_owned), Slice(*end_key_owned), limit, - remaining_byte_budget, keys_only, deadline_ms, nullptr, &truncated, &pending_page_id, - &out_packed, &out_count)) { - // Append this attempt's packed entries to the accumulated buffer. - if (out_count > 0) { - accumulated->append(out_packed.data(), out_packed.size()); - } - on_done(Status::Ok(), std::move(*accumulated), truncated); - return; - } - - // Cold leaf hit: append the entries resolved so far (those before the - // cold leaf) to `accumulated`, then resume from the last resolved key - // after the demand-load completes — no re-traversal of already-resolved - // leaves. `out_packed` contains only entries before the cold page because - // try_scan_no_load bails immediately on the first cold page. - if (out_count > 0) { - accumulated->append(out_packed.data(), out_packed.size()); - accumulated_count += out_count; - last_key = std::make_shared<std::string>(last_key_from_packed(out_packed.data(), out_packed.size())); - } - // Adjust limit by the number of entries already accumulated so the - // final result respects the caller's limit. - size_t remaining_limit = (limit > accumulated_count) ? (limit - accumulated_count) : 0; - - if (opt_.async_page_store != nullptr) { - uint64_t addr = 0; - uint32_t plen = 0; - bool still_unloaded = false; - uint64_t requested_word = 0; - Status location_status; - { - auto generation = generation_.enter(); - std::scoped_lock lk(load_mutex_); - uint64_t w = mapping_.get_word(pending_page_id); - if (slot_word::is_unloaded(w)) { - requested_word = w; - location_status = opt_.page_store->decode_mapping_location(w, &addr, &plen); - still_unloaded = location_status.ok(); - } - } - if (!location_status.ok()) { - io_failed_.store(true); - on_done(location_status, ScanPackedBuf{}, false); - return; - } - if (!still_unloaded) { - // Another loader already resolved this page_id between the - // lock-free probe above and this re-check -- retry, still here. - // Resume from the last accumulated key (if any) to avoid - // re-traversing already-resolved leaves. - auto resume_after = make_resume_after(start_after_owned, last_key); - if (metrics_.scan_retry_c != nullptr) { - metrics_.scan_retry_c->inc(); - } - scan_async_attempt(std::move(prefix_owned), resume_after, end_key_owned, remaining_limit, byte_budget, - keys_only, deadline_ms, std::move(accumulated), std::move(last_key), accumulated_count, - std::move(on_done)); - return; - } - uint32_t iu = opt_.page_store->iu_size(); - auto blob = std::make_shared<std::vector<uint8_t>>(round_up_to_iu(plen, iu)); - demand_load_total_.fetch_add(1, std::memory_order_relaxed); - opt_.async_page_store->submit_read( - addr, blob->data(), blob->size(), - detail::own_async_completion([this, page_id = pending_page_id, requested_word, addr, plen, blob, - prefix_owned, start_after_owned, end_key_owned, remaining_limit, byte_budget, - keys_only, deadline_ms, accumulated, last_key, accumulated_count, - on_done](Status st) mutable { - if (!st.ok()) { - CRB_LOG_ERROR("[{}] scan_async: demand-load I/O fault: pid={} addr={} len={} status={}", name_, - page_id, addr, plen, st.to_string()); - io_failed_.store(true); - on_done(st, ScanPackedBuf{}, false); - return; - } - bool installed_ok = true; - { - auto generation = generation_.enter(); - std::scoped_lock lk(load_mutex_); - uint64_t w = mapping_.get_word(page_id); - if (w == requested_word) { - installed_ok = install_loaded_page(page_id, addr, plen, *blob) != nullptr; - } - } - if (!installed_ok) { - on_done(Status::io_error("scan_async: demand-load decode/CRC failure"), ScanPackedBuf{}, false); - return; - } - // Resume from the last accumulated key to avoid - // re-traversing already-resolved leaves. - auto resume_after = make_resume_after(start_after_owned, last_key); - if (metrics_.scan_retry_c != nullptr) { - metrics_.scan_retry_c->inc(); - } - scan_async_attempt(std::move(prefix_owned), resume_after, end_key_owned, remaining_limit, byte_budget, - keys_only, deadline_ms, std::move(accumulated), std::move(last_key), - accumulated_count, std::move(on_done)); - })); - return; - } - // No async backend wired -- fall back to the existing synchronous - // demand-load and retry, still on this same thread. - if (metrics_.scan_retry_c != nullptr) { - metrics_.scan_retry_c->inc(); - } - { - auto generation = generation_.enter(); - (void)resident(pending_page_id); - } - auto resume_after = make_resume_after(start_after_owned, last_key); - scan_async_attempt(std::move(prefix_owned), resume_after, end_key_owned, remaining_limit, byte_budget, keys_only, - deadline_ms, std::move(accumulated), std::move(last_key), accumulated_count, std::move(on_done)); -} - -void Crowdbtree::scan_reverse_async_attempt(std::shared_ptr<std::string> prefix_owned, - const std::shared_ptr<std::string> &start_after_owned, - const std::shared_ptr<std::string> &end_key_owned, size_t limit, - size_t byte_budget, bool keys_only, uint64_t deadline_ms, - std::shared_ptr<ScanPackedBuf> accumulated, - std::shared_ptr<std::string> last_key, size_t accumulated_count, - std::function<void(Status, ScanPackedBuf, bool)> on_done) const -{ - size_t accumulated_bytes = payload_bytes_from_packed(accumulated->data(), accumulated->size()); - if (byte_budget != 0 && accumulated_bytes >= byte_budget && accumulated_count > 0) { - on_done(Status::Ok(), std::move(*accumulated), true); - return; - } - if (limit != 0 && accumulated_count >= limit) { - on_done(Status::Ok(), std::move(*accumulated), true); - return; - } - size_t remaining_byte_budget = byte_budget != 0 ? byte_budget - accumulated_bytes : 0; - if (deadline_ms != 0) { - auto now_ms = static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()) - .count()); - if (now_ms >= deadline_ms) { - on_done(Status::Ok(), std::move(*accumulated), true); - return; - } - } - - auto continuation = make_resume_after(start_after_owned, last_key); - ScanPackedBuf out_packed; - size_t out_count = 0; - bool truncated = false; - uint64_t pending_page_id = kInvalidPageId; - size_t attempt_limit = limit > accumulated_count ? limit - accumulated_count : 0; - if (try_scan_reverse_no_load(Slice(*prefix_owned), Slice(*continuation), Slice(*end_key_owned), attempt_limit, - remaining_byte_budget, keys_only, deadline_ms, &out_packed, &out_count, &truncated, - &pending_page_id)) { - if (out_count > 0) { - accumulated->append(out_packed.data(), out_packed.size()); - } - on_done(Status::Ok(), std::move(*accumulated), truncated); - return; - } - - if (out_count > 0) { - accumulated->append(out_packed.data(), out_packed.size()); - accumulated_count += out_count; - last_key = std::make_shared<std::string>(last_key_from_packed(out_packed.data(), out_packed.size())); - } - if (opt_.async_page_store != nullptr) { - uint64_t addr = 0; - uint32_t plen = 0; - bool still_unloaded = false; - uint64_t requested_word = 0; - Status location_status; - { - auto generation = generation_.enter(); - std::scoped_lock lk(load_mutex_); - uint64_t word = mapping_.get_word(pending_page_id); - if (slot_word::is_unloaded(word)) { - requested_word = word; - location_status = opt_.page_store->decode_mapping_location(word, &addr, &plen); - still_unloaded = location_status.ok(); - } - } - if (!location_status.ok()) { - io_failed_.store(true); - on_done(location_status, ScanPackedBuf{}, false); - return; - } - if (!still_unloaded) { - if (metrics_.scan_retry_c != nullptr) { - metrics_.scan_retry_c->inc(); - } - scan_reverse_async_attempt(std::move(prefix_owned), start_after_owned, end_key_owned, limit, byte_budget, - keys_only, deadline_ms, std::move(accumulated), std::move(last_key), - accumulated_count, std::move(on_done)); - return; - } - uint32_t iu = opt_.page_store->iu_size(); - auto blob = std::make_shared<std::vector<uint8_t>>(round_up_to_iu(plen, iu)); - demand_load_total_.fetch_add(1, std::memory_order_relaxed); - opt_.async_page_store->submit_read( - addr, blob->data(), blob->size(), - detail::own_async_completion([this, page_id = pending_page_id, requested_word, addr, plen, blob, - prefix_owned = std::move(prefix_owned), start_after_owned, end_key_owned, - limit, byte_budget, keys_only, deadline_ms, - accumulated = std::move(accumulated), last_key = std::move(last_key), - accumulated_count, on_done = std::move(on_done)](Status status) mutable { - if (!status.ok()) { - CRB_LOG_ERROR("[{}] scan_reverse_async: demand-load I/O fault: pid={} addr={} len={} status={}", - name_, page_id, addr, plen, status.to_string()); - io_failed_.store(true); - on_done(status, ScanPackedBuf{}, false); - return; - } - bool installed_ok = true; - { - auto generation = generation_.enter(); - std::scoped_lock lk(load_mutex_); - uint64_t word = mapping_.get_word(page_id); - if (word == requested_word) { - installed_ok = install_loaded_page(page_id, addr, plen, *blob) != nullptr; - } - } - if (!installed_ok) { - io_failed_.store(true); - on_done(Status::io_error("scan_reverse_async: demand-load decode/CRC failure"), ScanPackedBuf{}, - false); - return; - } - if (metrics_.scan_retry_c != nullptr) { - metrics_.scan_retry_c->inc(); - } - scan_reverse_async_attempt(std::move(prefix_owned), start_after_owned, end_key_owned, limit, - byte_budget, keys_only, deadline_ms, std::move(accumulated), - std::move(last_key), accumulated_count, std::move(on_done)); - })); - return; - } - { - auto generation = generation_.enter(); - (void)resident(pending_page_id); - } - if (metrics_.scan_retry_c != nullptr) { - metrics_.scan_retry_c->inc(); - } - scan_reverse_async_attempt(std::move(prefix_owned), start_after_owned, end_key_owned, limit, byte_budget, keys_only, - deadline_ms, std::move(accumulated), std::move(last_key), accumulated_count, - std::move(on_done)); -} - -int Crowdbtree::height() const -{ - auto generation = generation_.enter(); - auto guard = epoch_.enter(); - int h = 0; - uint64_t page_id = root_page_id_.load(); - for (int d = 0; d < 64; ++d) { - PageBase *head = resident(page_id); - if (head == nullptr) { - break; - } - PageBase *base = head; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - ++h; - if (base == nullptr || base->type == page_type::kLeafBase) { - break; - } - page_id = static_cast<InnerBase *>(base)->child_at(0); - } - return h; -} - -size_t Crowdbtree::leaf_count() const -{ - auto generation = generation_.enter(); - auto guard = epoch_.enter(); - std::function<size_t(uint64_t)> rec = [&](uint64_t page_id) -> size_t { - PageBase *head = resident(page_id); - if (head == nullptr) { - return 0; - } - PageBase *base = head; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - if (base == nullptr) { - return 0; - } - if (base->type == page_type::kLeafBase) { - return 1; - } - size_t n = 0; - for (uint64_t c : static_cast<InnerBase *>(base)->children()) { - n += rec(c); - } - return n; - }; - return rec(root_page_id_.load()); -} - -Status Crowdbtree::install_snapshot(std::vector<leaf_entry> sorted_entries, uint64_t at_slot) -try { - // Construct everything privately. Failure cannot expose an empty or - // partially imported generation to callers of this tree. - Crowdbtree staged(opt_); - auto active = staged.current_active(); - active->set_allow_old_slots(true); - for (auto &entry : sorted_entries) { - const CellView cell{entry.cell.slice()}; - if (!cell.valid()) { - return Status::invalid_argument("snapshot contains an invalid cell"); - } - const auto slot = cell.slot(); - if (slot > at_slot || !opt_.key_range.contains(Slice(entry.key))) { - return Status::invalid_argument("snapshot entry is outside its prefix or key range"); - } - active->upsert(Slice(entry.key), slot, std::move(entry.cell)); - } - staged.force_advance_slot(at_slot); - auto status = staged.flush(); - if (!status.ok()) { - return status; - } - std::vector<NativeFrame> frames; - uint64_t root = 0; - uint64_t next = 0; - status = staged.collect_native_frames(&frames, &root, nullptr, &next); - if (!status.ok()) { - return status; - } - return install_snapshot_native(std::move(frames), root, at_slot, next); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("operation allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); -} - -Status Crowdbtree::collect_native_frames(std::vector<NativeFrame> *out, uint64_t *out_root_page_id, - uint64_t *out_at_slot, uint64_t *out_next_page_id, const KeyRange *filter, - uint64_t *out_subtrees_skipped) -{ - std::scoped_lock lk(write_mutex_); - if (publication_incomplete_) { - return Status::unavailable("flush publication requires repair"); - } - uint64_t gc = gc_floor_.load(); - const bool can_prune = filter != nullptr && routing_fences_trusted_.load(std::memory_order_acquire); - const KeyRange *effective_filter = can_prune ? filter : nullptr; - - // Same DFS shape as the pre-#14c manifest walk: fold any delta chain - // into a fresh consolidated base first (a real side effect on the live - // tree, same as snapshot()'s prepare phase -- unlike the read-only - // snapshot_view()), then dump the resolved base's frame bytes verbatim, - // recursing into inner children / leaf overflow chains. - auto intersects = [effective_filter](const std::optional<std::string> &lower, - const std::optional<std::string> &upper) { - if (effective_filter == nullptr || !effective_filter->is_bounded()) { - return true; - } - if (effective_filter->start().has_value() && effective_filter->end().has_value() && - Slice(*effective_filter->start()).compare(Slice(*effective_filter->end())) == 0) { - return false; - } - const bool below_end = !effective_filter->end().has_value() || !lower.has_value() || - Slice(*lower).compare(Slice(*effective_filter->end())) < 0; - const bool above_start = !effective_filter->start().has_value() || !upper.has_value() || - Slice(*upper).compare(Slice(*effective_filter->start())) > 0; - return below_end && above_start; - }; - - uint64_t skipped = 0; - std::function<Status(uint64_t, std::optional<std::string>, std::optional<std::string>, bool, NativeBounds *)> walk = - [&](uint64_t page_id, std::optional<std::string> lower, std::optional<std::string> upper, bool is_root, - NativeBounds *bounds) -> Status { - if (!is_root && !intersects(lower, upper)) { - ++skipped; - return Status::Ok(); - } - PageBase *head = resident(page_id); - if (head == nullptr) { - return Status::internal_error("native snapshot: null page in walk"); - } - const bool has_inframe_deltas = - head->type == page_type::kLeafBase && static_cast<LeafBase *>(head)->view().delta_count() != 0; - if (head->type == page_type::kBatchDelta || has_inframe_deltas) { - PageBase *b = head; - while (b != nullptr && b->type == page_type::kBatchDelta) { - b = b->next; - } - if (b == nullptr || b->type != page_type::kLeafBase) { - return Status::internal_error("native snapshot: delta chain without leaf base"); - } - uint64_t right = static_cast<LeafBase *>(b)->right_sibling(); - std::vector<uint64_t> dead_overflow; - LeafBase *fresh = - build_leaf_spilling_locked(resolve_leaf_chain_for_rebuild(head, gc, &dead_overflow), right); - store_preserving_parent_locked(page_id, fresh); - for (PageBase *n = head; n != nullptr;) { - PageBase *nx = n->next; - retire_page(n); - n = nx; - } - for (uint64_t h : dead_overflow) { - retire_overflow_chain_locked(h); - } - head = fresh; - } - PageBase *base = head; // now a single base (no deltas above it) - - const uint8_t *frame = nullptr; - uint32_t plen = 0; - if (base->type == page_type::kLeafBase) { - frame = static_cast<LeafBase *>(base)->frame(); - plen = static_cast<LeafBase *>(base)->page_bytes(); - } - else if (base->type == page_type::kInnerBase) { - frame = static_cast<InnerBase *>(base)->frame(); - plen = static_cast<InnerBase *>(base)->page_bytes(); - } - else { - return Status::internal_error("native snapshot: unexpected base type"); - } - const size_t output_index = out->size(); - out->push_back(NativeFrame{.page_id = page_id, .frame = std::vector<uint8_t>(frame, frame + plen)}); - - if (base->type == page_type::kInnerBase) { - InnerFrameView inner = static_cast<InnerBase *>(base)->view(); - for (uint32_t index = 0; index < inner.num_children(); ++index) { - std::optional<std::string> child_lower = - index == 0 ? lower : std::optional<std::string>(inner.separator_at(index - 1).to_string()); - std::optional<std::string> child_upper = - index == inner.num_separators() ? upper - : std::optional<std::string>(inner.separator_at(index).to_string()); - NativeBounds child_bounds; - Status cs = - walk(inner.child_at(index), std::move(child_lower), std::move(child_upper), false, &child_bounds); - if (!cs.ok()) { - return cs; - } - if (!bounds->lower.has_value() && child_bounds.lower.has_value()) { - bounds->lower = child_bounds.lower; - bounds->lower_leaf_page_id = child_bounds.lower_leaf_page_id; - } - if (child_bounds.upper.has_value()) { - bounds->upper = child_bounds.upper; - bounds->upper_leaf_page_id = child_bounds.upper_leaf_page_id; - } - } - NativeFrame &output_frame = (*out)[output_index]; - if (!set_native_frame_fences(&output_frame, *bounds)) { - return Status::resource_exhausted("native snapshot: cannot persist inner fences"); - } - } - else { // leaf: dump its overflow chains too (reachable via cells, PT11) - LeafFrameView v = static_cast<LeafBase *>(base)->view(); - if (!v.empty()) { - bounds->lower = v.key(0).to_string(); - bounds->upper = v.key(v.count() - 1).to_string(); - bounds->lower_leaf_page_id = page_id; - bounds->upper_leaf_page_id = page_id; - } - NativeFrame &output_frame = (*out)[output_index]; - if (!set_native_frame_fences(&output_frame, *bounds)) { - return Status::resource_exhausted("native snapshot: leaf fence keys do not fit frame"); - } - for (uint32_t i = 0; i < v.count(); ++i) { - CellView c{v.cell(i)}; - if (!c.is_overflow()) { - continue; - } - uint64_t opid = c.overflow_head(); - while (opid != kInvalidPageId) { - PageBase *op = resident(opid); - if (op == nullptr || op->type != page_type::kOverflowFrame) { - return Status::internal_error("native snapshot: bad overflow page"); - } - auto *ov = static_cast<OverflowBase *>(op); - out->push_back(NativeFrame{ - .page_id = opid, - .frame = std::vector<uint8_t>(ov->frame(), ov->frame() + ov->page_bytes()), - }); - opid = ov->next_page_id(); - } - } - } - return Status::Ok(); - }; - - out->clear(); - NativeBounds root_bounds; - Status ws = walk(root_page_id_.load(), std::nullopt, std::nullopt, true, &root_bounds); - if (!ws.ok()) { - return ws; - } - if (filter != nullptr && !can_prune) { - Status graph_status = validate_native_snapshot_graph(out, root_page_id_.load()); - if (!graph_status.ok()) { - return graph_status; - } - routing_fences_trusted_.store(true, std::memory_order_release); - } - if (out_root_page_id != nullptr) { - *out_root_page_id = root_page_id_.load(); - } - if (out_at_slot != nullptr) { - *out_at_slot = last_applied_slot_.load(); - } - if (out_next_page_id != nullptr) { - *out_next_page_id = mapping_.next_page_id(); - } - if (out_subtrees_skipped != nullptr) { - *out_subtrees_skipped = skipped; - } - return Status::Ok(); -} - -EngineStats Crowdbtree::stats() const -{ - EngineStats s; - s.last_applied_slot = last_applied_slot_.load(); - s.contiguous_slot = contiguous_slot_.load(); - s.gc_watermark = gc_floor_.load(); - s.io_failed = io_failed_.load(); - s.snapshot_pages_written = snapshot_pages_written_.load(); - s.snapshot_pages_total = snapshot_pages_total_.load(); - s.snapshot_segments_written = snapshot_segments_written_.load(); - - BufferPool::Stats bp = pool_->stats(); - s.buffer_pool_hits = bp.hits; - s.buffer_pool_misses = bp.misses; - s.buffer_pool_evictions = bp.evictions; - s.buffer_pool_writebacks = bp.writebacks; - s.buffer_pool_resident = bp.resident; - s.buffer_pool_dirty = bp.dirty; - s.buffer_pool_used = bp.used; - s.buffer_pool_num_frames = bp.num_frames; - - s.mt_upsert_total = mt_upsert_total_.load(std::memory_order_relaxed); - s.mt_overwrite_total = memtable_counters_.overwrite.load(); - s.mt_history_keep_total = memtable_counters_.keep.load(); - s.mt_history_merge_total = memtable_counters_.merged.load(); - s.mt_version_cas_retries = memtable_counters_.cas_retries.load(); - s.mt_resident_bytes = epoch_.memtable_allocation()->load(); - s.mt_get_total = mt_get_total_.load(std::memory_order_relaxed); - s.mt_get_hit_total = mt_get_hit_total_.load(std::memory_order_relaxed); - s.flush_drain_total = flush_drain_total_.load(std::memory_order_relaxed); - s.flush_entries_total = flush_entries_total_.load(std::memory_order_relaxed); - s.snapshot_total = snapshot_total_.load(std::memory_order_relaxed); - s.l1_get_total = l1_get_total_.load(std::memory_order_relaxed); - s.l1_get_hit_total = l1_get_hit_total_.load(std::memory_order_relaxed); - s.map_lookup_total = map_lookup_total_.load(std::memory_order_relaxed); - s.demand_load_total = demand_load_total_.load(std::memory_order_relaxed); - s.leaf_count = leaf_count_.load(std::memory_order_relaxed); - s.inner_count = inner_count_.load(std::memory_order_relaxed); - return s; -} - -TreeSummary Crowdbtree::tree_summary() const -{ - TreeSummary s; - s.root_version = summary_root_version_.load(std::memory_order_acquire); - s.covered_slot = summary_covered_slot_.load(std::memory_order_acquire); - s.live_kv = summary_live_kv_.load(std::memory_order_acquire); - s.live_key_bytes = summary_live_key_bytes_.load(std::memory_order_acquire); - s.live_value_bytes = summary_live_value_bytes_.load(std::memory_order_acquire); - s.reachable_leaf_pages = leaf_count_.load(std::memory_order_relaxed); - s.reachable_inner_pages = inner_count_.load(std::memory_order_relaxed); - s.reachable_overflow_pages = summary_reachable_overflow_pages_.load(std::memory_order_acquire); - s.reachable_page_capacity_bytes = summary_reachable_page_capacity_bytes_.load(std::memory_order_acquire); - s.live_logical_bytes = summary_live_logical_bytes_.load(std::memory_order_acquire); - s.exact = summary_available_.load(std::memory_order_acquire); - return s; -} - -ScanProfile Crowdbtree::scan_profile() const -{ - ScanProfile p; - if (metrics_registry_ == nullptr) { - return p; - } - auto fill = [](LatencySummary *h, ScanProfile::Step &s, uint64_t count) { - if (h == nullptr || count == 0) { - return; - } - auto snap = h->flush(); - s.sum_ns = snap.sum; - s.max_ns = snap.max; - s.avg_ns = snap.sum / count; - }; - // Count from scan_l (scan_c removed — count derivable from latency). - uint64_t scan_count = 0; - if (metrics_.scan_l != nullptr) { - auto snap = metrics_.scan_l->flush(); - scan_count = snap.count; - p.total.sum_ns = snap.sum; - p.total.max_ns = snap.max; - if (snap.count > 0) { - p.total.avg_ns = snap.sum / snap.count; - } - } - p.count = scan_count; - p.entries = metrics_.scan_entries_c != nullptr ? metrics_.scan_entries_c->flush().count : 0; - fill(metrics_.scan_l0_l, p.l0, p.count); - fill(metrics_.scan_l1_l, p.l1, p.count); - fill(metrics_.scan_merge_l, p.merge, p.count); - return p; -} - -void Crowdbtree::init_metrics(const std::string &prefix, const std::string &backend_label) -{ - metrics_registry_ = std::make_unique<MetricsRegistry>(); - auto *r = metrics_registry_.get(); - // Backend-specific I/O prefix (empty for mem backend → no suffix). - std::string io = backend_label.empty() ? prefix : prefix + "." + backend_label; - - // ── Logical metrics (backend-independent) ── - metrics_.flush_l = r->register_summary(prefix + ".flush.l"); - metrics_.flush_drain_c = r->register_counter(prefix + ".flush.drain.c"); - metrics_.flush_entries_c = r->register_counter(prefix + ".flush.entries.c"); - metrics_.split_view_begin_l = r->register_summary(prefix + ".split.view.begin.l"); - metrics_.split_view_publish_l = r->register_summary(prefix + ".split.view.publish.l"); - metrics_.split_view_release_l = r->register_summary(prefix + ".split.view.release.l"); - metrics_.mt_apply_l = r->register_summary(prefix + ".mt.apply.l"); - metrics_.mt_get_c = r->register_counter(prefix + ".mt.get.c"); - metrics_.mt_get_hit_c = r->register_counter(prefix + ".mt.get.hit.c"); - metrics_.mt_get_l = r->register_summary(prefix + ".mt.get.l"); - metrics_.mt_frozen_g = r->register_callback_gauge(prefix + ".mt.frozen.g", - [this] { return static_cast<uint64_t>(frozen_table_count()); }); - metrics_.mt_records_g = r->register_callback_gauge(prefix + ".mt.records.g", [this] { - size_t total = 0; - for (const auto &mt : all_memtables()) { - total += mt->count(); - } - return static_cast<uint64_t>(total); - }); - r->register_callback_gauge(prefix + ".mt.overwrite.total", [this] { return memtable_counters_.overwrite.load(); }); - metrics_.mt_version_copy_l = r->register_summary(prefix + ".mt.version.copy.l"); - r->register_callback_gauge(prefix + ".mt.history.keep.total", [this] { return memtable_counters_.keep.load(); }); - r->register_callback_gauge(prefix + ".mt.history.merge.total", [this] { return memtable_counters_.merged.load(); }); - r->register_callback_gauge(prefix + ".mt.history.keep.gap.total", - [this] { return memtable_counters_.keep_gap.load(); }); - r->register_callback_gauge(prefix + ".mt.history.keep.pending.total", - [this] { return memtable_counters_.keep_pending.load(); }); - r->register_callback_gauge(prefix + ".mt.history.keep.boundary.total", - [this] { return memtable_counters_.keep_boundary.load(); }); - r->register_callback_gauge(prefix + ".mt.batch.failed.total", - [this] { return memtable_counters_.failed_batches.load(); }); - auto memory_gauge = [this, r, &prefix](const char *suffix, uint64_t VersionMemory::*member) { - r->register_callback_gauge(prefix + suffix, [this, member] { - uint64_t total = 0; - for (const auto &source : all_memtables()) { - total += source.owner->memory().*member; - } - return total; - }); - }; - memory_gauge(".mt.history.live.g", &VersionMemory::history_count); - memory_gauge(".mt.history.bytes.g", &VersionMemory::history_bytes); - memory_gauge(".mt.descriptor.bytes.g", &VersionMemory::descriptor_bytes); - memory_gauge(".mt.node.bytes.g", &VersionMemory::node_bytes); - memory_gauge(".mt.payload.bytes.g", &VersionMemory::payload_bytes); - r->register_callback_gauge(prefix + ".mt.freezing.writers.g", [this] { - uint64_t writers = 0; - for (const auto &source : local_memtables()) { - if (source.owner->closed()) { - writers += source.owner->writers(); - } - } - return writers; - }); - r->register_callback_gauge(prefix + ".mt.frozen.retained.bytes.g", [this] { - uint64_t bytes = 0; - for (const auto &source : local_memtables()) { - if (source.owner->closed() && source.owner->writers() == 0) { - bytes += source.owner->approx_bytes(); - } - } - return bytes; - }); - r->register_callback_gauge(prefix + ".mt.resident.bytes.g", - [this] { return epoch_.memtable_allocation()->load(); }); - r->register_callback_gauge(prefix + ".mt.retired.bytes.g", [this] { - uint64_t live = 0; - for (const auto &source : local_memtables()) { - const auto memory = source.owner->memory(); - live += memory.node_bytes + memory.descriptor_bytes + memory.payload_bytes; - } - const auto resident = epoch_.memtable_allocation()->load(); - return resident > live ? resident - live : 0; - }); - metrics_.mt_freeze_c = r->register_counter(prefix + ".mt.freeze.c"); - metrics_.l1_get_c = r->register_counter(prefix + ".l1.get.c"); - metrics_.l1_get_hit_c = r->register_counter(prefix + ".l1.get.hit.c"); - metrics_.l1_get_l = r->register_summary(prefix + ".l1.get.l"); - metrics_.page_write_l = r->register_summary(prefix + ".page.write.l"); - metrics_.page_split_c = r->register_counter(prefix + ".page.split.c"); - metrics_.page_merge_c = r->register_counter(prefix + ".page.merge.c"); - metrics_.page_consolidate_c = r->register_counter(prefix + ".page.consolidate.c"); - metrics_.tree_height_g = - r->register_callback_gauge(prefix + ".tree.height.g", [this] { return static_cast<uint64_t>(height()); }); - metrics_.tree_leaf_count_g = r->register_gauge(prefix + ".tree.leaf.count.g"); - metrics_.tree_inner_count_g = r->register_gauge(prefix + ".tree.inner.count.g"); - metrics_.tree_retired_count_g = - r->register_callback_gauge(prefix + ".tree.retired.count.g", [this] { return epoch_.pending_retired(); }); - // Seed the leaf/inner gauges with the current counter values (so the first - // metrics flush before any SMO shows the right counts). - metrics_.tree_leaf_count_g->set(leaf_count_.load(std::memory_order_relaxed)); - metrics_.tree_inner_count_g->set(inner_count_.load(std::memory_order_relaxed)); - metrics_.page_map_alloc_c = r->register_counter(prefix + ".page.map.alloc.c"); - metrics_.page_map_total_pids_g = r->register_callback_gauge(prefix + ".page.map.total_pids.g", [this] { - auto generation = generation_.enter(); - return mapping_.next_page_id(); - }); - metrics_.page_map_segments_g = r->register_callback_gauge(prefix + ".page.map.segments.g", [this] { - auto generation = generation_.enter(); - return static_cast<uint64_t>(mapping_.segments_allocated()); - }); - metrics_.snapshot_l = r->register_summary(prefix + ".snapshot.l"); - metrics_.snapshot_pages_c = r->register_counter(prefix + ".snapshot.pages.c"); - metrics_.scan_entries_c = r->register_counter(prefix + ".scan.entries.c"); - metrics_.scan_l = r->register_summary(prefix + ".scan.l"); - metrics_.scan_l0_l = r->register_summary(prefix + ".scan.l0.l"); - metrics_.scan_l1_l = r->register_summary(prefix + ".scan.l1.l"); - metrics_.scan_merge_l = r->register_summary(prefix + ".scan.merge.l"); - metrics_.scan_retry_c = r->register_counter(prefix + ".scan.retry.c"); - metrics_.gc_tombstones_c = r->register_counter(prefix + ".gc.tombstones.c"); - metrics_.merge_gc_blocks_c = r->register_counter(prefix + ".merge_gc.blocks.c"); - metrics_.merge_gc_relocated_c = r->register_counter(prefix + ".merge_gc.relocated.c"); - metrics_.merge_gc_deleted_c = r->register_counter(prefix + ".merge_gc.deleted.c"); - metrics_.merge_gc_l = r->register_summary(prefix + ".merge_gc.l"); - - // ── Backend I/O metrics ── - metrics_.page_find_c = r->register_counter(io + ".page.find.c"); - metrics_.buf_evictions = r->register_counter(io + ".buf.evictions.c"); - metrics_.buf_writebacks = r->register_counter(io + ".buf.writebacks.c"); - metrics_.buf_resident = r->register_gauge(io + ".buf.resident.g"); - metrics_.buf_dirty = r->register_gauge(io + ".buf.dirty.g"); - metrics_.page_load_l = r->register_summary(io + ".page.load.l"); - metrics_.page_writeback_l = r->register_summary(io + ".page.writeback.l"); - metrics_.page_writeback_bw = r->register_bandwidth(io + ".page.writeback.bw"); - metrics_.fsync_l = r->register_summary(io + ".fsync.l"); - metrics_.snapshot_apply_l = r->register_summary(io + ".snapshot.apply.l"); - metrics_.snapshot_page_write_l = r->register_summary(io + ".snapshot.page.write.io.l"); - metrics_.snapshot_page_write_cache_c = r->register_counter(io + ".snapshot.page.write.cache.c"); - metrics_.snapshot_page_write_bw = r->register_bandwidth(io + ".snapshot.page.write.bw"); - metrics_.snapshot_meta_write_bw = r->register_bandwidth(io + ".snapshot.meta.write.bw"); - metrics_.page_read_bw = r->register_bandwidth(io + ".page.read.bw"); - - pool_->set_metrics(metrics_.buf_evictions, metrics_.buf_writebacks, metrics_.buf_resident, metrics_.buf_dirty, - metrics_.page_writeback_l, metrics_.page_writeback_bw); - mapping_.set_alloc_counter(metrics_.page_map_alloc_c); -} - -std::string Crowdbtree::flush_metrics_str(double window_secs, const char *timestamp, size_t width, size_t count_w, - size_t tps_w) -{ - if (metrics_registry_ == nullptr) { - return {}; - } - char *buf = nullptr; - size_t len = 0; - FILE *fp = open_memstream(&buf, &len); - if (fp == nullptr) { - return {}; - } - metrics_registry_->flush_to(fp, window_secs, timestamp, "cpp-tree", width, count_w, tps_w); - std::fflush(fp); - std::fclose(fp); - std::string result(buf, len); - free(buf); - return result; -} - -size_t Crowdbtree::max_name_len() const -{ - if (metrics_registry_ == nullptr) { - return 0; - } - return metrics_registry_->max_name_len(); -} - -std::shared_ptr<Snapshot> Crowdbtree::snapshot_view() -{ - std::scoped_lock writer(write_mutex_); - if (publication_incomplete_) { - throw std::runtime_error("flush publication requires repair"); - } - - // R6: zero-copy pinned snapshot. Walks the leaf chain under an epoch guard - // (same safety argument as the old materialized version — see the comment - // below on the walk's concurrency properties), captures every PageBase* - // touched (leaf chain heads + overflow pages), pins each via pin_state_, - // then releases the guard. Returns a PinnedSnapshot that holds the pins - // and materializes entries lazily from the pinned frames on first call. - // The pages stay alive via refcount until the PinnedSnapshot is dropped — - // on any thread. - // - // Concurrency: the walk uses the same collect_in_order right_sibling - // technique as scan() and the old snapshot_view(), under a single epoch - // guard. A concurrent split/merge/flush is never blocked. The guard is - // entered and released on this same thread (respecting Guard's - // thread-bound contract); the PinnedSnapshot's pins are thread-independent. - // - // at_slot is captured *before* the walk (same rationale as before: flush - // only bumps last_applied_slot_ after publishing, so a racing flush can - // only make the walk see more, never less). - EpochManager::Guard guard = epoch_.enter(); - uint64_t at_slot = last_applied_slot_.load(); - - // Walk the leaf chain, capturing page pointers. We inline the - // collect_in_order walk so we can capture both leaf chain pages (head → - // ... → base for each leaf — the full delta chain, not just the head) and - // overflow pages in a single pass. materialize() follows head->next, so - // every node in every chain must be pinned. - std::vector<PageBase *> leaf_chain_heads; - std::vector<PageBase *> all_pinned_pages; - std::vector<PageBase *> overflow_pages; - uint64_t root_pid = root_page_id_.load(); - if (root_pid != kInvalidPageId) { - uint64_t page_id = find_leaf_page_id([this](uint64_t p) { return resident(p); }, root_pid, Slice()); - while (page_id != kInvalidPageId) { - PageBase *head = resident(page_id); - if (head == nullptr) { - break; - } - leaf_chain_heads.push_back(head); - // Capture the entire chain (head → ... → base), not just the head. - // materialize() calls resolve_chain_sorted(head) which follows - // head->next; every delta node must be pinned to survive the - // epoch guard release. - for (PageBase *node = head; node != nullptr; node = node->next) { - all_pinned_pages.push_back(node); - if (node->type == page_type::kLeafBase) { - LeafFrameView v = static_cast<LeafBase *>(node)->view(); - for (uint32_t i = 0; i < v.count(); ++i) { - CellView c{v.cell(i)}; - if (c.is_overflow()) { - capture_overflow_chain(c.overflow_head(), overflow_pages); - } - } - for (uint32_t i = 0; i < v.delta_count(); ++i) { - CellView c{v.delta_cell(i)}; - if (c.is_overflow()) { - capture_overflow_chain(c.overflow_head(), overflow_pages); - } - } - } - else if (node->type == page_type::kBatchDelta) { - for (const leaf_entry &e : static_cast<BatchDelta *>(node)->entries()) { - CellView c{Slice(e.cell)}; - if (c.is_overflow()) { - capture_overflow_chain(c.overflow_head(), overflow_pages); - } - } - } - } - LeafBase *base = chain_leaf_base(head); - page_id = base != nullptr ? base->right_sibling() : kInvalidPageId; - } - } - - // Dedup overflow pages (a single overflow chain may be referenced by - // multiple keys in the same leaf). Also add them to all_pinned_pages. - std::ranges::sort(overflow_pages); - auto [first_dup, last_dup] = std::ranges::unique(overflow_pages); - overflow_pages.erase(first_dup, last_dup); - for (PageBase *p : overflow_pages) { - all_pinned_pages.push_back(p); - } - - auto snap = std::make_shared<PinnedSnapshot>(at_slot, std::move(leaf_chain_heads), std::move(all_pinned_pages), - std::move(overflow_pages)); - // The PinnedSnapshot ctor pins the captured pages; release the epoch guard - // now (the pins keep the pages resident across threads). - guard = EpochManager::Guard(); - return snap; -} - -// Capture all pages in an overflow chain starting at head_page_id. Used by -// snapshot_view() to pin overflow pages so PinnedSnapshot::materialize() can -// assemble overflow values from pinned frames without re-entering the mapping -// table. -void Crowdbtree::capture_overflow_chain(uint64_t head_page_id, std::vector<PageBase *> &out) -{ - uint64_t page_id = head_page_id; - for (int guard = 0; page_id != kInvalidPageId && guard < (1 << 24); ++guard) { - PageBase *p = resident(page_id); - if (p == nullptr || p->type != page_type::kOverflowFrame) { - break; - } - out.push_back(p); - page_id = static_cast<OverflowBase *>(p)->next_page_id(); - } -} - -// R6: PinnedSnapshot::materialize() — walk the captured leaf chain heads (in -// order), resolve each chain via resolve_chain_sorted, and assemble overflow -// values from the captured overflow pages. Called lazily on first entries() -// access. Defined here (not in snapshot.h) because it needs resolve_chain_sorted -// and the OverflowBase/LeafBase page types from crowdb-tree.cpp's internal helpers. -void PinnedSnapshot::materialize() const -{ - // Build a lookup from page_id → PageBase* for the captured overflow pages - // so we can assemble overflow values without re-entering the mapping table. - std::unordered_map<uint64_t, PageBase *> overflow_by_id; - for (PageBase *p : overflow_pages_) { - overflow_by_id[p->page_id] = p; - } - - for (PageBase *head : leaf_chain_heads_) { - auto entries = resolve_chain_sorted(head, 0); // gc_floor=0: keep all (snapshot is point-in-time) - for (auto &e : entries) { - CellView v{Slice(e.cell)}; - if (v.is_overflow()) { - // Assemble from pinned overflow pages. - std::string assembled; - assembled.reserve(v.overflow_len()); - uint64_t page_id = v.overflow_head(); - for (int guard = 0; page_id != kInvalidPageId && guard < (1 << 24); ++guard) { - auto it = overflow_by_id.find(page_id); - if (it == overflow_by_id.end()) { - break; - } - auto *ov = static_cast<OverflowBase *>(it->second); - Slice chunk = ov->payload(); - assembled.append(chunk.data(), chunk.size()); - page_id = ov->next_page_id(); - } - if (assembled.size() > v.overflow_len()) { - assembled.resize(v.overflow_len()); - } - e.cell = encode_cell_buf(v.slot(), OpKind::kPut, Slice(assembled)); - } - entries_.push_back(std::move(e)); - } - } -} - -std::string Crowdbtree::assemble_overflow_value(uint64_t head_page_id, uint64_t total_len) const -{ - std::string out; - out.reserve(total_len); - uint64_t page_id = head_page_id; - for (int guard = 0; page_id != kInvalidPageId && guard < (1 << 24); ++guard) { - PageBase *p = resident(page_id); - if (p == nullptr || p->type != page_type::kOverflowFrame) { - break; // corruption -> short value - } - auto *ov = static_cast<OverflowBase *>(p); - Slice chunk = ov->payload(); - out.append(chunk.data(), chunk.size()); - page_id = ov->next_page_id(); - } - if (out.size() > total_len) { - out.resize(total_len); - } - return out; -} - -std::vector<leaf_entry> Crowdbtree::resolve_leaf_chain_for_rebuild(PageBase *head, uint64_t gc_floor, - std::vector<uint64_t> *dead_overflow, - size_t *out_tombstones_dropped, - size_t *out_bytes_dropped) -{ - std::map<std::string, std::string> resolved; // key -> encoded storage cell - auto consider = [&](Slice key, Slice cell) { - CellView incoming{cell}; - uint64_t s = incoming.slot(); - std::string k = key.to_string(); - auto it = resolved.find(k); - if (it == resolved.end()) { - resolved[k] = cell.to_string(); - return; - } - CellView current{Slice(it->second)}; - if (s > current.slot()) { - if (dead_overflow && current.is_overflow()) { - dead_overflow->push_back(current.overflow_head()); - } - it->second = cell.to_string(); - } - else if (dead_overflow && incoming.is_overflow()) { - dead_overflow->push_back(incoming.overflow_head()); // incoming loses - } - }; - for (PageBase *node = head; node != nullptr; node = node->next) { - if (node->type == page_type::kBatchDelta) { - for (const leaf_entry &e : static_cast<BatchDelta *>(node)->entries()) { - consider(Slice(e.key), Slice(e.cell)); - } - } - else if (node->type == page_type::kLeafBase) { - LeafFrameView v = static_cast<LeafBase *>(node)->view(); - for (uint32_t i = 0; i < v.count(); ++i) { - consider(v.key(i), v.cell(i)); - } - for (uint32_t i = 0; i < v.delta_count(); ++i) { - consider(v.delta_key(i), v.delta_cell(i)); - } - } - } - std::vector<leaf_entry> out; - out.reserve(resolved.size()); - size_t dropped = 0; - size_t dropped_bytes = 0; - for (auto &kv : resolved) { - CellView v{Slice(kv.second)}; - if (v.is_tombstone() && v.slot() <= gc_floor) { - ++dropped; - dropped_bytes += kv.first.size() + kv.second.size(); - continue; // GC drop - } - out.push_back({.key = kv.first, .cell = cell_of(kv.second)}); - } - if (out_tombstones_dropped != nullptr) { - *out_tombstones_dropped = dropped; - } - if (out_bytes_dropped != nullptr) { - *out_bytes_dropped = dropped_bytes; - } - return out; -} - -uint64_t Crowdbtree::spill_value_to_overflow_chain_locked(const std::string &value) -{ - const uint32_t cap = overflow_chunk_cap(opt_.frame_bytes); - // Split into chunks; build the chain tail-first so each frame knows its next. - size_t n = value.size(); - size_t nchunks = n == 0 ? 1 : (n + cap - 1) / cap; - std::vector<uint64_t> pids(nchunks); - for (size_t i = 0; i < nchunks; ++i) { - pids[i] = mapping_.allocate_page_id(); - } - uint64_t next = kInvalidPageId; - for (size_t i = nchunks; i-- > 0;) { - size_t off = i * cap; - uint32_t len = static_cast<uint32_t>(std::min<size_t>(cap, n - off)); - OverflowBase *page = OverflowBase::build(next, reinterpret_cast<const uint8_t *>(value.data() + off), len, - pool_, opt_.frame_bytes); - refresh_page_summary_locked(page); - mapping_.store(pids[i], page); - next = pids[i]; - } - return pids[0]; -} - -LeafBase *Crowdbtree::build_leaf_spilling_locked(std::vector<leaf_entry> entries, uint64_t right_sibling) -{ - const size_t threshold = max_inline_value(); - for (leaf_entry &e : entries) { - CellView v{Slice(e.cell)}; - if (v.is_overflow() || v.is_tombstone()) { - continue; // pointer / tombstone: keep - } - Slice val = v.value(); - if (val.size() > threshold) { - std::string value = val.to_string(); - uint64_t head = spill_value_to_overflow_chain_locked(value); - e.cell = encode_overflow_cell_buf(v.slot(), head, value.size()); - } - } - return LeafBase::build(entries, right_sibling, pool_, opt_.frame_bytes); -} - -void Crowdbtree::retire_overflow_chain_locked(uint64_t head_page_id) -{ - uint64_t page_id = head_page_id; - for (int guard = 0; page_id != kInvalidPageId && guard < (1 << 24); ++guard) { - // Demand-load unloaded links so we can read their next_page_id and retire the - // whole chain (no descriptor/extent leak when a tail link was evicted). Lock - // order write_mutex_ -> load_mutex_ holds (caller holds write_mutex_). - PageBase *p = resident(page_id); - if (p == nullptr || p->type != page_type::kOverflowFrame) { - mapping_.clear(page_id); // clear a stray slot if any - break; - } - uint64_t next = static_cast<OverflowBase *>(p)->next_page_id(); - mapping_.clear(page_id); // unlink before retiring - retire_page(p); - page_id = next; - } -} - -void Crowdbtree::evict_overflow_chain_locked(uint64_t head_page_id) -{ - uint64_t page_id = head_page_id; - for (int guard = 0; page_id != kInvalidPageId && guard < (1 << 24); ++guard) { - uint64_t w = mapping_.get_word(page_id); - // Stop at an already-unloaded link: chains evict whole, so the tail is - // already unloaded (and not leaking). A dirty page (no durable addr) can't - // be evicted; leave it resident. - if (slot_word::is_empty(w) || slot_word::is_unloaded(w)) { - break; - } - PageBase *p = slot_word::resident_ptr(w); - if (p->type != page_type::kOverflowFrame || p->durable_addr == kNoAddr) { - break; - } - uint64_t next = static_cast<OverflowBase *>(p)->next_page_id(); - uint64_t unloaded = slot_word::kEmpty; - if (!opt_.page_store->encode_mapping_location(p->durable_addr, p->durable_plen, &unloaded).ok()) { - io_failed_.store(true); - break; - } - mapping_.store_word(page_id, unloaded); - retire_page(p); - page_id = next; - } -} - -void Crowdbtree::free_overflow_chain(uint64_t head_page_id) -{ - uint64_t page_id = head_page_id; - for (int guard = 0; page_id != kInvalidPageId && guard < (1 << 24); ++guard) { - uint64_t w = mapping_.get_word(page_id); - if (slot_word::is_empty(w) || slot_word::is_unloaded(w)) { - mapping_.clear(page_id); // clear any unloaded descriptor - break; - } - PageBase *p = slot_word::resident_ptr(w); - if (p->type != page_type::kOverflowFrame) { - break; - } - uint64_t next = static_cast<OverflowBase *>(p)->next_page_id(); - mapping_.clear(page_id); - delete p; // teardown / clear: no concurrent readers - page_id = next; - } -} - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/entry_key_arena.h b/lib/crowdb-tree/src/btree/entry_key_arena.h new file mode 100644 index 000000000..bf10c1d65 --- /dev/null +++ b/lib/crowdb-tree/src/btree/entry_key_arena.h @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" + +#include <vector> + +namespace crowdb::tree +{ +// Shared immutable key block after its builder has exposed the first entry. +class EntryKeyArena +{ + using Allocator = detail::ReservationAllocator<char>; + + public: + EntryKeyArena(size_t bytes, const std::shared_ptr<detail::AllocationReservation> &metadata) + : bytes_(bytes, '\0', Allocator(metadata)) + { + } + + [[nodiscard]] size_t size() const noexcept + { + return bytes_.size(); + } + + [[nodiscard]] const char *data() const noexcept + { + return bytes_.data(); + } + + [[nodiscard]] char *data() noexcept + { + return bytes_.data(); + } + + private: + std::vector<char, Allocator> bytes_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_batch.cpp b/lib/crowdb-tree/src/btree/flush_batch.cpp new file mode 100644 index 000000000..e0479341d --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_batch.cpp @@ -0,0 +1,238 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_batch_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +Status FlushBatch::prepare(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, uint64_t batch_id, uint64_t frontier, + FlushBatchOptions options, Maintenance maintenance, std::shared_ptr<FlushBatch> *out, + std::shared_ptr<const PublishedSources> inputs, + std::shared_ptr<detail::AllocationReservation> metadata, uint64_t flush_admission_owner) +try { + if (out == nullptr || !lane || options.parallelism == 0 || options.install_run_window == 0 || + options.inline_entries == 0 || options.inline_bytes == 0 || options.max_run_bytes == 0 || + options.run_limits.max_entries == 0 || options.run_limits.max_key_bytes == 0 || + options.run_limits.key_block_bytes == 0 || options.routing_leaves_per_step == 0 || + options.max_route_depth == 0 || options.max_route_depth > 64) { + return Status::invalid_argument("flush needs bounded ranges, runs and a runtime lane"); + } + auto state = detail::reserved_shared<State>(metadata, publisher, runtime, metadata); + state->captured = publisher.capture(); + state->inputs = inputs ? std::move(inputs) : state->captured->sources; + state->metadata = std::move(metadata); + state->admission_owner = flush_admission_owner; + if (state->captured->flush_admission_owner != flush_admission_owner) { + return Status::unavailable("flush does not own the captured publication admission"); + } + if (frontier < state->captured->coverage || state->inputs->floor != state->captured->coverage || + state->inputs->tables.size() > options.max_sources) { + return Status::invalid_argument("invalid flush frontier or source count"); + } + if (state->captured->sources->inheritance) { + const auto &inherited = *state->captured->sources->inheritance; + const auto &published = state->captured->sources->tables; + if (inherited.local_count > published.size() || frontier != inherited.frontier || + state->inputs->tables.size() != published.size() - inherited.local_count || + !std::equal(state->inputs->tables.begin(), state->inputs->tables.end(), + published.begin() + static_cast<std::ptrdiff_t>(inherited.local_count))) { + return Status::invalid_argument("split handoff must consume exactly its inherited source suffix"); + } + } + for (const auto &table : state->inputs->tables) { + if (!table || !table->closed() || table->writers() != 0) { + return Status::unavailable("captured writers must drain asynchronously before handoff"); + } + if (std::find(state->captured->sources->tables.begin(), state->captured->sources->tables.end(), table) == + state->captured->sources->tables.end()) { + return Status::invalid_argument("flush input is absent from the published catalog"); + } + } + for (const auto &table : state->captured->sources->tables) { + if (!table || (table->closed() && std::find(state->inputs->tables.begin(), state->inputs->tables.end(), + table) == state->inputs->tables.end())) { + return Status::invalid_argument("every closed source must participate in the captured batch"); + } + } + state->lane = std::move(lane); + state->options = std::move(options); + state->maintenance = std::move(maintenance); + state->report.batch_id = batch_id; + state->report.floor = state->captured->coverage; + state->report.frontier = frontier; + state->report.published_coverage = state->captured->coverage; + state->coverage_record = detail::reserved_shared<InstallRecord>( + state->metadata, + InstallToken{.generation = state->captured->generation, .batch = batch_id, .range = UINT64_MAX, .sequence = 0}); + state->routing_record = detail::reserved_shared<InstallRecord>( + state->metadata, + InstallToken{ + .generation = state->captured->generation, .batch = batch_id, .range = UINT64_MAX - 2, .sequence = 0}); + const auto root = state->captured->mapping.get(state->captured->root_page_id); + if (!root || root->level > 64) { + return Status::unavailable("flush needs an initialized bounded routing tree"); + } + *out = detail::reserved_construct<FlushBatch>( + state->metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) FlushBatch(std::move(state)); }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("flush preparation allocation failed"); +} + +FlushBatch::~FlushBatch() = default; + +detail::CallbackSender FlushBatch::sender() noexcept +{ + return {this, &FlushBatch::submit}; +} + +void FlushBatch::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto state = static_cast<FlushBatch *>(source)->state_; + state->start(complete, context); +} + +void FlushBatch::request_stop() noexcept +{ + state_->lifecycle.fetch_or(State::kStopped, std::memory_order_acq_rel); + // The join owns late completions. Each queued CPU step observes this flag; + // it never frees another step's receiver or payload on cancellation. +} + +const FlushResult &FlushBatch::result() const noexcept +{ + return state_->report; +} + +void FlushBatch::State::start(detail::CallbackComplete callback, void *argument) +{ + auto previous = lifecycle.load(std::memory_order_acquire); + uint8_t accepted; + do { + if ((previous & kRunning) != 0) { + callback(argument, detail::CallbackSignal::kError, Status::unavailable("flush batch already running")); + return; + } + accepted = kRunning | kStarted | ((previous & kStarted) == 0 ? previous & kStopped : 0); + } while (!lifecycle.compare_exchange_weak(previous, accepted, std::memory_order_acq_rel)); + complete = callback; + context = argument; + terminal.store(false); + batch_metrics.reset(); + if (options.measure_batch) { + batch_metrics.emplace(publisher.work_metrics(), TreeStage::FlushBatch, tree_metrics_now_ns()); + } + handoff_metrics.reset(); + maintenance_metrics.reset(); + try { + if (coverage_record->committed.load(std::memory_order_acquire)) { + maintain(); + return; + } + record_work(publisher.work_metrics(), TreeWorkCounter::FlushInputTables, inputs->tables.size(), + TreeWorkCause::Flush); + report.phase = FlushPhase::kHandoff; + handoff_metrics.emplace(publisher.work_metrics(), TreeStage::FlushHandoff, tree_metrics_now_ns()); + if (!planned) { + // Planning can fail before any range is installed. Retry starts + // from the same frozen sources and immutable routing view. + ranges.clear(); + route.reset(); + plan_key.reset(); + schedule_plan(); + } + else { + start_handoff(); + } + } + catch (const std::bad_alloc &) { + joined(Status::resource_exhausted()); + } + catch (const std::exception &) { + joined(Status::internal_error()); + } +} + +void FlushBatch::State::start_handoff() +{ + auto self = shared_from_this(); + join = detail::reserved_unique<detail::BoundedRangeJoin>( + metadata, ranges.size(), options.parallelism, this, + +[](void *context, size_t index, detail::BoundedRangeJoin::Completion done) { + static_cast<State *>(context)->start_range(index, std::move(done)); + }, + metadata); + operation = detail::reserved_unique<Attempt>(metadata, join->sender(), self); + stdexec::start(operation->connected); +} + +void FlushBatch::State::Receiver::set_value() noexcept +{ + auto self = std::move(owner); + self->joined(Status::Ok()); +} + +void FlushBatch::State::Receiver::set_error(Status error) noexcept +{ + auto self = std::move(owner); + self->joined(std::move(error)); +} + +void FlushBatch::State::Receiver::set_stopped() noexcept +{ + auto self = std::move(owner); + self->joined(Status::unavailable("flush stopped")); +} + +void FlushBatch::State::joined(Status status) noexcept +{ + report.handoff_status = std::move(status); + if (!report.handoff_status.ok()) { + report.phase = FlushPhase::kRepairRequired; + finish(); + return; + } + try { + cover(); + } + catch (const std::bad_alloc &) { + report.handoff_status = Status::resource_exhausted(); + report.phase = FlushPhase::kRepairRequired; + finish(); + } + catch (const std::exception &) { + report.handoff_status = Status::internal_error(); + report.phase = FlushPhase::kRepairRequired; + finish(); + } +} + +void FlushBatch::State::finish() noexcept +{ + if (terminal.exchange(true, std::memory_order_acq_rel)) { + return; + } + auto self = weak_from_this().lock(); + auto callback = complete; + auto *argument = context; + const auto now = tree_metrics_now_ns(); + auto outcome = TreeOutcome::Success; + if (!report.handoff_status.ok() || !report.maintenance_status.ok()) { + outcome = (lifecycle.load() & kStopped) != 0 ? TreeOutcome::Cancelled : TreeOutcome::Error; + } + if (batch_metrics) { + (void)batch_metrics->finish(outcome, now); + } + if (handoff_metrics) { + (void)handoff_metrics->finish(report.handoff_status.ok() ? TreeOutcome::Success : outcome, now); + } + if (maintenance_metrics) { + (void)maintenance_metrics->finish(report.maintenance_status.ok() ? TreeOutcome::Success : outcome, now); + } + lifecycle.fetch_and(static_cast<uint8_t>(~kRunning), std::memory_order_release); + callback(argument, detail::CallbackSignal::kValue, Status::Ok()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_batch.h b/lib/crowdb-tree/src/btree/flush_batch.h new file mode 100644 index 000000000..925465298 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_batch.h @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "flush_runtime.h" +#include "logical_page.h" +#include "published_state.h" + +namespace crowdb::tree +{ +class PageLoader; +class ApplyDispatcher; +class PublicationLane; +enum class FlushPhase : uint8_t { kCaptured, kHandoff, kRepairRequired, kCovered, kMaintenance, kComplete }; + +struct FlushResult +{ + uint64_t batch_id = 0; + uint64_t floor = 0; + uint64_t frontier = 0; + uint64_t published_coverage = 0; + Status handoff_status; + Status maintenance_status; + uint64_t deferred_pages = 0; + FlushPhase phase = FlushPhase::kCaptured; + // True only for the attempt that first advances the covered frontier. + bool handoff_advanced = false; + // A newer publication invalidated preparation before admission/capture. + // Only this failure permits automatic replanning of the soft request. + bool admission_stale = false; +}; + +struct FlushBatchOptions +{ + RunLimits run_limits; + size_t max_sources = 64; + size_t parallelism = 4; + size_t inline_entries = 32; + size_t inline_bytes = 64 * 1024; + size_t max_run_bytes = 1024 * 1024; + size_t inline_ranges_per_step = 8; + // Install the first bounded run window early; later windows double so + // immutable prefix-array copies remain linear in the batch input. + size_t install_run_window = 64; + size_t routing_leaves_per_step = 64; + size_t max_route_depth = 64; + std::shared_ptr<PageLoader> loader = nullptr; + PageKeyRange selection; + std::shared_ptr<ApplyDispatcher> cpu_progress = nullptr; + std::shared_ptr<PublicationLane> publication = nullptr; + // A controller measures the complete attempt including source capture. + bool measure_batch = true; +#ifdef CROWDB_TREE_TEST_UTIL + // Runs after publication admission and before source closure. + std::function<void()> before_capture_for_tests = nullptr; + // Called after a bounded run is owned and all source guards are released. + std::function<void(size_t, size_t)> after_run_for_tests = nullptr; +#endif +}; + +// The source admission layer supplies an already closed/drained catalog and +// a fixed frontier. It retains this batch after any partial failure and retries +// it before accepting another batch or changing routing. No Tree worker waits. +// This core has its own mapping owner; legacy mutable mapping cannot be mixed +// with it. Backend/capture adapters must finish migration before default use. +// Host runtime/publisher ownership must outlive all CPU steps, including the +// return from terminal callbacks. Runtime destruction is a host-side operation. +class FlushBatch +{ + struct State; + + public: + using MaintenanceComplete = detail::InlineFunction<void(Status, uint64_t)>; + using Maintenance = detail::InlineFunction<void(std::shared_ptr<const PublishedTreeState>, MaintenanceComplete)>; + // Bound the batch's metadata before closing sources. Counts include + // admitted writers, not only currently materialized table entries. Source + // capture/controller and cold routing byte/I/O credit are added separately. + [[nodiscard]] static std::optional<size_t> metadata_reserve_bound(const MappingView &mapping, size_t sources, + size_t entries, size_t key_bytes, size_t workers, + const FlushBatchOptions &options, + bool track_residency = false); + static Status prepare(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, uint64_t batch_id, uint64_t frontier, + FlushBatchOptions options, Maintenance maintenance, std::shared_ptr<FlushBatch> *out, + std::shared_ptr<const PublishedSources> inputs = {}, + std::shared_ptr<detail::AllocationReservation> metadata = {}, + uint64_t flush_admission_owner = 0); + ~FlushBatch(); + FlushBatch(const FlushBatch &) = delete; + FlushBatch &operator=(const FlushBatch &) = delete; + + // One connected operation at a time. Reconnecting after terminal retries + // only missing records. Once covered, retries never repeat the handoff. + [[nodiscard]] detail::CallbackSender sender() noexcept; + void request_stop() noexcept; + // Read only after sender terminal (which returns a progress-bearing value). + [[nodiscard]] const FlushResult &result() const noexcept; + + private: + explicit FlushBatch(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_batch_budget.cpp b/lib/crowdb-tree/src/btree/flush_batch_budget.cpp new file mode 100644 index 000000000..fc40c83c3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_batch_budget.cpp @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "flush_batch_internal.h" +#include "page_cache_residency.h" + +#include <algorithm> + +namespace crowdb::tree +{ +std::optional<size_t> FlushBatch::metadata_reserve_bound(const MappingView &mapping, size_t sources, size_t entries, + size_t key_bytes, size_t workers, + const FlushBatchOptions &options, bool track_residency) +{ + if (sources == 0 || sources > options.max_sources || workers == 0 || options.parallelism == 0 || + options.run_limits.max_entries == 0 || options.run_limits.max_key_bytes == 0 || + options.run_limits.key_block_bytes == 0 || options.routing_leaves_per_step == 0 || + options.install_run_window == 0 || options.inline_entries == 0 || options.inline_bytes == 0 || + options.max_run_bytes == 0) { + return {}; + } + const auto mapping_bounds = mapping.copy_bounds(); + // A finite source prefix can span any number of existing leaves. Its + // entry allowance bounds the complete range metadata; parallelism and + // routing steps bound active work independently of the Tree's size. + const auto ranges = std::min(entries, mapping_bounds.leaf_pages); + if (mapping_bounds.leaf_pages == 0) { + return {}; + } + using detail::AllocationBound; + using detail::shared_allocation_bound; + const auto installs = AllocationBound(ranges).add(entries / options.install_run_window).value(); + if (!installs) { + return {}; + } + const auto parallelism = std::min(ranges, options.parallelism); + AllocationBound bytes(shared_allocation_bound<State>() + shared_allocation_bound<FlushBatch>() + + sizeof(State::Attempt)); + // Standalone batches can publish a first partial handoff without a + // controller admission. Retain its complete source-free base once. + bytes.add(PublicationCoordinator::complete_view_reserve_bytes()); + bytes.add(PublishedRouteCursor::allocation_bound(options.max_route_depth)); + bytes.add(detail::BoundedRangeJoin::allocation_bound(options.parallelism)); + + // Every accepted range can leave a retired COW root alive. Count all paths, + // request/record storage, checkpoint membership nodes and the final join + // proof even when no explicit reclamation runs during this batch. + bytes.add_product(*installs, MappingView::path_copy_bound()); + bytes.add_product(*installs, PublicationCoordinator::snapshot_change_reserve_bytes()); + bytes.add_product(*installs, PublicationCoordinator::empty_commit_reserve_bytes()); + bytes.add_product(*installs, MergePairMetrics::invalidation_bytes(2)); + // The admitted handoff only removes existing sibling pairs. Each endpoint + // belongs to at most two pairs, so its membership/proof can change at most + // twice across the whole batch, regardless of the number of install windows. + const auto memberships = AllocationBound().add_product(2, mapping_bounds.merge_candidate_pages).value(); + const auto per_install = AllocationBound().add_product(4, *installs).value(); + if (!memberships || !per_install) { + return {}; + } + bytes.add(PublicationCoordinator::merge_membership_reserve_bytes(std::min(*memberships, *per_install))); + bytes.add_product(ranges, (3 * sizeof(State::Range)) + sizeof(std::shared_ptr<InstallRecord>) + + sizeof(PageReplacement) + shared_allocation_bound<TreeMetricsOperation>() + 1024); + bytes.add_product(*installs, detail::shared_allocation_bound<InstallRecord>() + + (3 * sizeof(std::shared_ptr<InstallRecord>)) + sizeof(PageReplacement)); + bytes.add_product(2, PublicationCoordinator::empty_commit_reserve_bytes()); + bytes.add_product(*installs, mapping_bounds.max_leaf_append_bytes); + if (track_residency) { + // Handoff adds overlays but preserves each leaf's base array. Overlay + // descriptor bytes cannot be treated as additional cache base slots. + const auto touched_bases = AllocationBound().add_product(*installs, mapping_bounds.max_leaf_bases).value(); + if (!touched_bases) { + return {}; + } + bytes.add_product(*installs, PageCacheResidency::page_allocation_bound(0)); + bytes.add_product(*touched_bases, + PageCacheResidency::page_allocation_bound(1) - PageCacheResidency::page_allocation_bound(0)); + } + + // At worst every input entry owns one arena block and one run. Vector + // growth includes old/new arrays; run references also coexist in the + // prepared range and the new PageState. Each append can allocate at most + // one entry array, including all retired growth arrays, so the total number + // of arrays is bounded by entries. Values retain existing owners. + const auto block = std::min(options.run_limits.key_block_bytes, options.run_limits.max_key_bytes); + bytes.add_product(entries, block); + + bytes.add_product(entries, shared_allocation_bound<EntryKeyArena>() + shared_allocation_bound<OverlayBatch>() + + (3 * sizeof(EntryRef)) + (9 * sizeof(std::shared_ptr<const OverlayBatch>))); + bytes.add_product(4, key_bytes); + bytes.add_product(3, std::min(key_bytes, options.run_limits.max_key_bytes)); + bytes.add_product(ranges, 2 * OwnedKey::allocation_overhead()); + bytes.add_product(3, OwnedKey::allocation_overhead()); + + const auto merge_source = sizeof(std::shared_ptr<MemTable>) + sizeof(EpochManager::Guard) + + sizeof(ConcurrentSkipList::Cursor) + sizeof(MergeSource) + sizeof(int); + const auto source_work = AllocationBound() + .add_product(parallelism, merge_source) + .add_product(workers, EpochManager::participant_allocation_bytes()) + .value(); + if (!source_work) { + return {}; + } + bytes.add_product(sources, *source_work); + bytes.add_product(workers, 2 * (sizeof(State::Work) + sizeof(State::PlanWork))); + bytes.add_product(parallelism, 2 * sizeof(State::Work)); + bytes.add(2 * sizeof(State::PlanWork)); + bytes.add(PublishedSources::allocation_bound() + (3 * sizeof(std::shared_ptr<MemTable>))); + bytes.add_product(sources, 3 * sizeof(std::shared_ptr<MemTable>)); + return bytes.value(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_batch_dispatch.cpp b/lib/crowdb-tree/src/btree/flush_batch_dispatch.cpp new file mode 100644 index 000000000..bae7342c2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_batch_dispatch.cpp @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_batch_internal.h" + +namespace crowdb::tree +{ +thread_local FlushBatch::State::DispatchScope *FlushBatch::State::DispatchScope::active = nullptr; + +FlushBatch::State::DispatchScope::DispatchScope(State &state) + : owner(state), + remaining(state.options.inline_ranges_per_step), + previous(active) +{ + active = this; + owner.publisher.work_metrics()->dispatch.begin(); +} + +FlushBatch::State::DispatchScope::~DispatchScope() +{ + owner.publisher.work_metrics()->dispatch.finish(); + active = previous; +} + +bool FlushBatch::State::DispatchScope::take(State &state) noexcept +{ + if (active == nullptr || &active->owner != &state || active->remaining == 0) { + return false; + } + --active->remaining; + return true; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_batch_handoff.cpp b/lib/crowdb-tree/src/btree/flush_batch_handoff.cpp new file mode 100644 index 000000000..5ecef6121 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_batch_handoff.cpp @@ -0,0 +1,324 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_batch_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<const PublishedPage> append_runs(const PublishedPage &old, + std::span<const std::shared_ptr<const OverlayBatch>> runs, + size_t installed, uint64_t entries, uint64_t bytes, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + auto logical = detail::reserved_shared<PageState>(metadata, *old.leaf, metadata); + logical->overlays.insert(logical->overlays.begin() + static_cast<std::ptrdiff_t>(installed), runs.begin(), + runs.end()); + ++logical->content_id; + ++logical->representation_id; + auto page = detail::reserved_shared<PublishedPage>(metadata, old); + page->leaf = std::move(logical); + page->merge_outgoing.reset(); + ++page->revision; + auto &metrics = page->metrics; + metrics.dirty = true; + metrics.kv_known = false; + metrics.page_size_known = false; + metrics.consolidate_candidate = true; + metrics.merge_candidate = false; + metrics.writeback_candidate = true; + metrics.overlay_runs += runs.size(); + metrics.overlay_entries += entries; + metrics.page_size_estimate_bytes += bytes; + return page; +} +} // namespace + +void FlushBatch::State::start_range(size_t index, detail::BoundedRangeJoin::Completion done) +{ + if (ranges[index].record->committed.load(std::memory_order_acquire)) { + done(); + return; + } + if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { + finish_range(index, done, Status::unavailable("flush stopped before range dispatch")); + return; + } + if (!ranges[index].probed && DispatchScope::take(*this)) { + ranges[index].probed = true; + publisher.work_metrics()->dispatch.inline_probe(); + execute_range(index, done, true); + return; + } + ranges[index].probed = true; + auto work = detail::reserved_unique<Work>(metadata, shared_from_this(), index, done); + auto status = runtime.try_submit(lane, &Work::execute, work.get()); + if (status.ok()) { + publisher.work_metrics()->dispatch.queued_step(); + [[maybe_unused]] auto *scheduled = work.release(); + } + else if (options.cpu_progress) { + publisher.work_metrics()->dispatch.queued_step(); + auto *ready = work.release(); + options.cpu_progress->request(ready->node, ready->owner); + } + else { + finish_range(index, done, std::move(status)); + } +} + +void FlushBatch::State::Work::execute(void *context) noexcept +{ + auto *raw = static_cast<Work *>(context); + detail::ReservedUnique<Work> work(raw, detail::ReservationDeleter<Work>{raw->owner->metadata}); + DispatchScope scope(*work->owner); + work->owner->execute_range(work->index, work->done, false); +} + +void FlushBatch::State::execute_range(size_t index, detail::BoundedRangeJoin::Completion done, + bool inline_probe) noexcept +{ + try { + auto &metrics = ranges[index].metrics; + if (!metrics) { + record_work(publisher.work_metrics(), TreeWorkCounter::FlushRangeJobs, 1, TreeWorkCause::Flush); + metrics = detail::reserved_shared<TreeMetricsOperation>(metadata, publisher.work_metrics(), + TreeStage::FlushRange, tree_metrics_now_ns()); + } + install(index, done, inline_probe); + } + catch (const std::bad_alloc &) { + finish_range(index, done, Status::resource_exhausted()); + } + catch (const std::exception &) { + finish_range(index, done, Status::internal_error()); + } +} + +void FlushBatch::State::install(size_t index, detail::BoundedRangeJoin::Completion done, bool inline_probe) +{ + if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { + finish_range(index, done, Status::unavailable("flush stopped before range execution")); + return; + } + auto &range = ranges[index]; + if (!range.ready && range.runs.size() < range.next_install_at) { + auto status = prepare_run(range, inline_probe); + if (!status.ok()) { + finish_range(index, done, std::move(status)); + return; + } +#ifdef CROWDB_TREE_TEST_UTIL + if (options.after_run_for_tests) { + options.after_run_for_tests(index, range.runs.size()); + } +#endif + if (inline_probe && range.ready) { + publisher.work_metrics()->dispatch.inline_range(); + } + if (!range.ready && range.runs.size() < range.next_install_at) { + start_range(index, std::move(done)); + return; + } + } + if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { + finish_range(index, done, Status::unavailable("flush stopped before range publication")); + return; + } + if (!range.window_prepared) { + std::shared_ptr<InstallRecord> following; + if (!range.ready) { + auto token = range.record->token; + ++token.sequence; + following = detail::reserved_shared<InstallRecord>(metadata, token); + } + std::shared_ptr<const PublishedPage> replacement; + if (range.runs.size() != range.installed_runs) { + const auto runs = + std::span<const std::shared_ptr<const OverlayBatch>>(range.runs).subspan(range.installed_runs); + replacement = + append_runs(*range.expected, runs, range.installed_runs, range.entries - range.installed_entries, + range.bytes - range.installed_bytes, metadata); + } + // Every post-commit owner is ready before the root exchange. A queue + // rejection retries this immutable window without preparing it again. + range.installs.push_back(range.record); + range.prepared_record = std::move(following); + range.prepared_page = std::move(replacement); + range.window_prepared = true; + } + auto request = detail::reserved_shared<PublicationRequest>(metadata, metadata); + request->transaction.record = range.record; + request->transaction.metadata = metadata; + request->transaction.flush_admission_owner = admission_owner; + request->transaction.expected_coverage = report.floor; + request->transaction.handoff = {.batch = report.batch_id, .frontier = report.frontier}; + request->transaction.snapshot_effect = SnapshotChangeEffect::AcceptedPrefix; + if (range.prepared_page) { + request->transaction.pages.push_back( + {.page_id = range.page_id, .expected = range.expected, .replacement = range.prepared_page}); + } + request->on_done = [self = shared_from_this(), index, done](PublicationResult result) noexcept { + if (!result.status.ok()) { + self->finish_range(index, done, std::move(result.status)); + return; + } + self->publisher.work_metrics()->dispatch.installed(result.copied.bytes); + auto &range = self->ranges[index]; + record_work(self->publisher.work_metrics(), TreeWorkCounter::FlushHandoffBytes, + range.bytes - range.installed_bytes, TreeWorkCause::Flush); + range.installed_runs = range.runs.size(); + range.installed_entries = range.entries; + range.installed_bytes = range.bytes; + if (range.prepared_page) { + range.expected = std::move(range.prepared_page); + } + range.window_prepared = false; + if (!range.prepared_record) { + self->finish_range(index, done); + return; + } + range.record = std::move(range.prepared_record); + range.next_install_at = range.next_install_at > SIZE_MAX / 2 ? SIZE_MAX : range.next_install_at * 2; + try { + self->start_range(index, done); + } + catch (const std::bad_alloc &) { + self->finish_range(index, done, Status::resource_exhausted()); + } + catch (const std::exception &) { + self->finish_range(index, done, Status::internal_error()); + } + }; + if (!publisher.try_submit(request, options.publication)) { + finish_range(index, done, Status::resource_exhausted("publication queue is full or closed")); + } +} + +void FlushBatch::State::finish_range(size_t index, detail::BoundedRangeJoin::Completion done, Status status) noexcept +{ + auto metrics = std::move(ranges[index].metrics); + if (metrics) { + auto outcome = status.ok() ? TreeOutcome::Success : TreeOutcome::Error; + if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { + outcome = TreeOutcome::Cancelled; + } + (void)metrics->finish(outcome, tree_metrics_now_ns()); + } + done(std::move(status)); +} + +void FlushBatch::State::cover() +{ + auto request = detail::reserved_shared<PublicationRequest>(metadata, metadata); + auto &transaction = request->transaction; + transaction.record = coverage_record; + transaction.metadata = metadata; + transaction.flush_admission_owner = admission_owner; + transaction.expected_coverage = report.floor; + transaction.coverage = report.frontier; + if (report.frontier > report.floor) { + transaction.handoff = {.batch = report.batch_id, .frontier = report.frontier}; + } + transaction.expected_sources = captured->sources; + auto sources = detail::reserved_shared<PublishedSources>(metadata, metadata); + sources->floor = report.frontier; + sources->memory = captured->sources->memory; + const auto retained_count = + captured->sources->inheritance ? captured->sources->inheritance->local_count : captured->sources->tables.size(); + uint64_t detached = 0; + for (size_t index = 0; index < retained_count; ++index) { + const auto &table = captured->sources->tables[index]; + const bool captured_input = + std::find(inputs->tables.begin(), inputs->tables.end(), table) != inputs->tables.end(); + if (!captured_input || table->slot_range().max > report.frontier) { + sources->tables.push_back(table); + } + else { + ++detached; + } + } + transaction.sources = std::move(sources); + transaction.required_installs.reserve(ranges.size()); + for (const auto &range : ranges) { + transaction.required_installs.push_back(range.record); + } + auto self = shared_from_this(); + request->on_done = [self, detached](PublicationResult result) { + self->report.handoff_status = std::move(result.status); + if (!self->report.handoff_status.ok()) { + self->report.phase = FlushPhase::kRepairRequired; + self->finish(); + return; + } + record_work(self->publisher.work_metrics(), TreeWorkCounter::SourceDetached, detached, TreeWorkCause::Flush); + self->report.published_coverage = self->report.frontier; + self->report.phase = FlushPhase::kCovered; + (void)self->handoff_metrics->finish(TreeOutcome::Success, tree_metrics_now_ns()); + self->captured.reset(); + self->inputs.reset(); + self->options.loader.reset(); + self->ranges.clear(); + self->maintain(); + }; + if (!publisher.try_submit(request, options.publication)) { + report.handoff_status = Status::resource_exhausted("coverage publication not accepted"); + report.phase = FlushPhase::kRepairRequired; + finish(); + } +} + +void FlushBatch::State::maintain() noexcept +{ + try { + do_maintain(); + } + catch (const std::bad_alloc &) { + report.maintenance_status = Status::resource_exhausted(); + report.phase = FlushPhase::kComplete; + finish(); + } + catch (const std::exception &) { + report.maintenance_status = Status::internal_error(); + report.phase = FlushPhase::kComplete; + finish(); + } +} + +void FlushBatch::State::do_maintain() +{ + maintenance_metrics.emplace(publisher.work_metrics(), TreeStage::MaintenancePlan, tree_metrics_now_ns()); + report.phase = FlushPhase::kMaintenance; + if (!maintenance) { + report.deferred_pages = publisher.capture()->stats.dirty_pages; + report.maintenance_status = Status::Ok(); + report.phase = FlushPhase::kComplete; + finish(); + return; + } + const auto generation = maintenance_generation.fetch_add(2, std::memory_order_acq_rel) + 2; + auto self = shared_from_this(); + auto done = [self, generation](Status status, uint64_t deferred) { + auto expected = generation; + if (!self->maintenance_generation.compare_exchange_strong(expected, generation | 1, + std::memory_order_acq_rel)) { + return; + } + self->report.maintenance_status = std::move(status); + self->report.deferred_pages = deferred; + self->report.phase = FlushPhase::kComplete; + self->finish(); + }; + try { + maintenance(publisher.capture(), done); + } + catch (const std::bad_alloc &) { + done(Status::resource_exhausted(), publisher.capture()->stats.dirty_pages); + } + catch (const std::exception &) { + done(Status::internal_error(), publisher.capture()->stats.dirty_pages); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_batch_internal.h b/lib/crowdb-tree/src/btree/flush_batch_internal.h new file mode 100644 index 000000000..687f3f2ed --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_batch_internal.h @@ -0,0 +1,177 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "apply_dispatcher.h" +#include "flush_batch.h" +#include "published_route.h" + +namespace crowdb::tree +{ +struct FlushBatch::State : std::enable_shared_from_this<State> +{ + struct Range + { + uint64_t page_id; + PageKeyRange bounds; + std::shared_ptr<const PublishedPage> expected; + std::shared_ptr<InstallRecord> record; + OverlayStorage runs; + detail::ReservedVector<std::shared_ptr<InstallRecord>> installs; + std::optional<OwnedKey> next_key; + std::shared_ptr<InstallRecord> prepared_record = nullptr; + std::shared_ptr<const PublishedPage> prepared_page = nullptr; + bool window_prepared = false; + bool probed = false; + size_t installed_runs = 0; + size_t next_install_at = 0; + uint64_t installed_entries = 0; + uint64_t installed_bytes = 0; + uint64_t entries = 0; + uint64_t bytes = 0; + bool ready = false; + std::shared_ptr<TreeMetricsOperation> metrics = nullptr; + }; + + struct DispatchScope + { + explicit DispatchScope(State &state); + ~DispatchScope(); + static bool take(State &state) noexcept; + State &owner; + size_t remaining; + DispatchScope *previous; + static thread_local DispatchScope *active; + }; + + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + std::shared_ptr<State> owner; + void set_value() noexcept; + void set_error(Status error) noexcept; + void set_stopped() noexcept; + }; + + using JoinOperation = stdexec::connect_result_t<detail::CallbackSender, Receiver>; + + struct Attempt + { + Attempt(detail::CallbackSender sender, std::shared_ptr<State> owner) + : connected(stdexec::connect(sender, Receiver{std::move(owner)})) + { + } + + JoinOperation connected; + }; + + struct Work + { + Work(std::shared_ptr<State> state, size_t range, detail::BoundedRangeJoin::Completion completion) + : owner(std::move(state)), + index(range), + done(std::move(completion)), + node(&Work::execute, this) + { + } + + std::shared_ptr<State> owner; + size_t index; + detail::BoundedRangeJoin::Completion done; + ApplyDispatcher::Node node; + static void execute(void *context) noexcept; + }; + + struct PlanWork + { + PlanWork(std::shared_ptr<State> state, Status result, std::shared_ptr<const InnerPageState> inner, bool resume) + : owner(std::move(state)), + status(std::move(result)), + loaded(std::move(inner)), + resume(resume), + node(&PlanWork::execute, this) + { + } + + std::shared_ptr<State> owner; + Status status; + std::shared_ptr<const InnerPageState> loaded; + bool resume = false; + ApplyDispatcher::Node node; + static void execute(void *context) noexcept; + }; + + State(PublicationCoordinator &publish, detail::FlushRuntime &cpu, + std::shared_ptr<detail::AllocationReservation> credit) + : publisher(publish), + runtime(cpu), + metadata(std::move(credit)), + ranges(detail::ReservationAllocator<Range>(metadata)) + { + } + + bool retire_step() noexcept + { + for (size_t step = 0; step < 64 && !ranges.empty(); ++step) { + auto &range = ranges.back(); + if (!range.runs.empty()) { + range.runs.pop_back(); + } + else if (!range.installs.empty()) { + range.installs.pop_back(); + } + else { + ranges.pop_back(); + } + } + return ranges.empty(); + } + + void schedule_plan(Status status = Status::Ok(), std::shared_ptr<const InnerPageState> loaded = {}, + bool resume = false); + void plan_step(std::shared_ptr<const InnerPageState> loaded, bool resume); + std::optional<OwnedKey> first_input(std::optional<Slice> lower); + void start_handoff(); + void acquire_route(); + void start(detail::CallbackComplete callback, void *argument); + void start_range(size_t index, detail::BoundedRangeJoin::Completion done); + Status prepare_run(Range &range, bool inline_probe); + void finish_range(size_t index, detail::BoundedRangeJoin::Completion done, Status status = Status::Ok()) noexcept; + void execute_range(size_t index, detail::BoundedRangeJoin::Completion done, bool inline_probe) noexcept; + void install(size_t index, detail::BoundedRangeJoin::Completion done, bool inline_probe); + void joined(Status status) noexcept; + void cover(); + void maintain() noexcept; + void do_maintain(); + void finish() noexcept; + + PublicationCoordinator &publisher; + detail::FlushRuntime &runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<const PublishedTreeState> captured; + std::shared_ptr<const PublishedSources> inputs; + std::shared_ptr<detail::AllocationReservation> metadata; + uint64_t admission_owner = 0; + std::vector<Range, detail::ReservationAllocator<Range>> ranges; + detail::ReservedUnique<PublishedRouteCursor> route; + std::optional<OwnedKey> plan_key; + bool planned = false; + std::shared_ptr<InstallRecord> coverage_record; + std::shared_ptr<InstallRecord> routing_record; + FlushBatchOptions options; + Maintenance maintenance; + FlushResult report; + static constexpr uint8_t kRunning = 1; + static constexpr uint8_t kStopped = 2; + static constexpr uint8_t kStarted = 4; + std::atomic<uint8_t> lifecycle{0}; + std::atomic<bool> terminal{false}; + std::atomic<uint64_t> maintenance_generation{0}; + std::optional<TreeMetricsOperation> batch_metrics; + std::optional<TreeMetricsOperation> handoff_metrics; + std::optional<TreeMetricsOperation> maintenance_metrics; + detail::ReservedUnique<detail::BoundedRangeJoin> join; + detail::ReservedUnique<Attempt> operation; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_batch_route.cpp b/lib/crowdb-tree/src/btree/flush_batch_route.cpp new file mode 100644 index 000000000..a5f48d037 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_batch_route.cpp @@ -0,0 +1,176 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_batch_internal.h" +#include "page_load.h" + +namespace crowdb::tree +{ +void FlushBatch::State::acquire_route() +{ + const auto current = publisher.capture(); + if (current->coverage != report.floor || current->sources != captured->sources || + current->generation != captured->generation) { + joined(Status::unavailable("flush sources or coverage changed before routing")); + return; + } + captured = current; + auto request = detail::reserved_shared<PublicationRequest>(metadata, metadata); + request->transaction = { + .record = routing_record, + .expected_coverage = report.floor, + .metadata = metadata, + .handoff = PendingHandoff{.batch = report.batch_id, .frontier = report.frontier}, + .expected_commit_seq = current->commit_seq, + .flush_admission_owner = admission_owner + }; + auto self = shared_from_this(); + request->on_done = [self](PublicationResult result) noexcept { + try { + self->schedule_plan(std::move(result.status)); + } + catch (const std::bad_alloc &) { + self->joined(Status::resource_exhausted()); + } + catch (const std::exception &) { + self->joined(Status::internal_error()); + } + }; + if (!publisher.try_submit(request, options.publication)) { + joined(Status::resource_exhausted("flush routing admission queue full")); + } +} + +std::optional<OwnedKey> FlushBatch::State::first_input(std::optional<Slice> lower) +{ + if (options.selection.lower && (!lower || *lower < Slice(*options.selection.lower))) { + lower = Slice(*options.selection.lower); + } + std::optional<OwnedKey> first; + for (const auto &table : inputs->tables) { + auto guard = table->read_guard(metadata); + auto cursor = table->range_cursor(lower, options.selection.upper_slice(), report.frontier, report.floor); + if (cursor.valid() && (!first || cursor.key().compare(Slice(*first)) < 0)) { + first.emplace(cursor.key(), metadata); + } + } + return first; +} + +void FlushBatch::State::schedule_plan(Status status, std::shared_ptr<const InnerPageState> loaded, bool resume) +{ + auto work = + detail::reserved_unique<PlanWork>(metadata, shared_from_this(), std::move(status), std::move(loaded), resume); + auto submitted = runtime.try_submit(lane, &PlanWork::execute, work.get()); + if (submitted.ok()) { + [[maybe_unused]] auto *scheduled = work.release(); + } + else if (options.cpu_progress) { + auto *ready = work.release(); + options.cpu_progress->request(ready->node, ready->owner); + } + else { + joined(std::move(submitted)); + } +} + +void FlushBatch::State::PlanWork::execute(void *context) noexcept +{ + auto *raw = static_cast<PlanWork *>(context); + detail::ReservedUnique<PlanWork> work(raw, detail::ReservationDeleter<PlanWork>{raw->owner->metadata}); + DispatchScope scope(*work->owner); + try { + if (!work->status.ok()) { + work->owner->joined(std::move(work->status)); + return; + } + work->owner->plan_step(std::move(work->loaded), work->resume); + } + catch (const std::bad_alloc &) { + work->owner->joined(Status::resource_exhausted("flush routing allocation failed")); + } + catch (const std::exception &) { + work->owner->joined(Status::internal_error("flush routing failed")); + } +} + +void FlushBatch::State::plan_step(std::shared_ptr<const InnerPageState> loaded, bool resume) +{ + if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { + joined(Status::unavailable("flush stopped while routing")); + return; + } + if (report.frontier > report.floor && !routing_record->committed.load(std::memory_order_acquire)) { + acquire_route(); + return; + } + if (!route) { + route = detail::reserved_unique<PublishedRouteCursor>(metadata, captured->root_page_id, captured->mapping, + options.max_route_depth, false, metadata); + plan_key = first_input({}); + } + for (size_t visited = 0; visited < options.routing_leaves_per_step && plan_key; ++visited) { + auto status = resume ? route->resume_inner(std::move(loaded)) : route->seek(Slice(*plan_key), false); + resume = false; + if (!status.ok()) { + joined(std::move(status)); + return; + } + if (auto backing = route->needs_inner()) { + if (!options.loader) { + joined(Status::unavailable("cold flush index requires an asynchronous loader")); + return; + } + auto self = shared_from_this(); + options.loader->load_inner(std::move(backing), [self](Status loaded_status, auto inner) noexcept { + try { + self->schedule_plan(std::move(loaded_status), std::move(inner), true); + } + catch (const std::bad_alloc &) { + self->joined(Status::resource_exhausted()); + } + catch (const std::exception &) { + self->joined(Status::internal_error()); + } + }); + return; + } + const auto &leaf = route->leaf(); + if (!leaf || !leaf->range.contains(Slice(*plan_key))) { + joined(Status::corruption("flush input has no matching captured leaf range")); + return; + } + const auto pid = route->leaf_page_id(); + auto progress = detail::reserved_shared<InstallRecord>( + metadata, + InstallToken{ + .generation = captured->generation, .batch = report.batch_id, .range = ranges.size(), .sequence = 0}); + auto selected = leaf->range; + if (options.selection.lower && (!selected.lower || *selected.lower < *options.selection.lower)) { + selected.lower = options.selection.lower; + } + if (options.selection.upper && (!selected.upper || *selected.upper > *options.selection.upper)) { + selected.upper = options.selection.upper; + } + ranges.push_back({.page_id = pid, + .bounds = std::move(selected), + .expected = captured->mapping.get(pid), + .record = std::move(progress), + .runs = OverlayStorage{OverlayAllocator(metadata)}, + .installs = + detail::ReservedVector<std::shared_ptr<InstallRecord>>{ + detail::ReservationAllocator<std::shared_ptr<InstallRecord>>(metadata)}, + .next_key = {}, + .next_install_at = options.install_run_window}); + // The next source seek jumps past this entire leaf, including keys + // belonging to other source tables. Untouched subtrees are never read. + plan_key = leaf->range.upper ? first_input(Slice(*leaf->range.upper)) : std::nullopt; + } + if (plan_key) { + schedule_plan(); + return; + } + route.reset(); + planned = true; + start_handoff(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_batch_run.cpp b/lib/crowdb-tree/src/btree/flush_batch_run.cpp new file mode 100644 index 000000000..4595a84d7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_batch_run.cpp @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_batch_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +std::atomic<uint64_t> next_run_identity{1}; +} + +Status FlushBatch::State::prepare_run(Range &range, bool inline_probe) +{ + auto bounds = range.bounds; + if (range.next_key) { + bounds.lower = range.next_key; + } + auto limits = options.run_limits; + if (inline_probe) { + limits.max_entries = std::min(limits.max_entries, options.inline_entries); + } + const auto byte_limit = + inline_probe ? std::min(options.inline_bytes, options.max_run_bytes) : options.max_run_bytes; + MemTableRangeMerge merge(std::span<const std::shared_ptr<MemTable>>(inputs->tables), std::move(bounds), + report.frontier, report.floor, limits, options.max_sources, metadata); + if (!merge.status().ok()) { + return merge.status(); + } + OverlayBuilder builder(limits, metadata); + uint64_t entries = 0; + uint64_t bytes = 0; + uint64_t key_bytes = 0; + uint64_t value_bytes = 0; + while (merge.valid() && (entries == 0 || bytes < byte_limit)) { + EntryRef entry; + auto status = merge.next(&entry); + // The next key stays in the merge when its arena window is full. + // A key that cannot fit even an empty window is a real error. + if (status.code() == Code::kResourceExhausted && entries != 0) { + break; + } + if (!status.ok()) { + return status; + } + key_bytes += entry.key_length; + value_bytes += entry.value().size(); + bytes += entry.key_length + entry.value().size() + kCellHeaderSize; + status = builder.append(std::move(entry)); + if (!status.ok()) { + return status; + } + ++entries; + } + // Re-seek under fresh thread-local guards on the next worker. Frozen + // sources and the fixed P/F make the first unconsumed key a stable bookmark, + // including empty/binary keys and duplicate keys in different sources. + if (merge.valid() && merge.key().size() > options.run_limits.max_key_bytes) { + return Status::resource_exhausted("flush continuation key exceeds its arena window"); + } + auto next_key = merge.valid() ? std::make_optional<OwnedKey>(merge.key(), metadata) : std::nullopt; + if (entries != 0) { + std::shared_ptr<const OverlayBatch> run; + auto status = builder.finish(next_run_identity.fetch_add(1, std::memory_order_relaxed), &run); + if (!status.ok()) { + return status; + } + range.runs.push_back(std::move(run)); + } + // Advance only after every potentially failing allocation. A rejected + // continuation retains the prepared prefix and resumes it on retry. + range.next_key = std::move(next_key); + range.entries += entries; + range.bytes += bytes; + range.ready = !range.next_key; + if (entries != 0) { + const auto &work = publisher.work_metrics(); + work->dispatch.run(entries, bytes); + record_work(work, TreeWorkCounter::FlushCandidateVersions, merge.candidates(), TreeWorkCause::Flush); + record_work(work, TreeWorkCounter::FlushWinningVersions, entries, TreeWorkCause::Flush); + record_work(work, TreeWorkCounter::FlushWinningKeyBytes, key_bytes, TreeWorkCause::Flush); + record_work(work, TreeWorkCounter::FlushWinningValueBytes, value_bytes, TreeWorkCause::Flush); + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_controller.cpp b/lib/crowdb-tree/src/btree/flush_controller.cpp new file mode 100644 index 000000000..197a0d6a8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_controller.cpp @@ -0,0 +1,291 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "apply_progress.h" +#include "flush_controller_internal.h" +#include "flush_controller_test.h" +#include "flush_operation_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +std::atomic<uint64_t> next_admission_owner{1}; +#ifdef CROWDB_TREE_TEST_UTIL +thread_local FlushPrepareHook prepare_hook_for_tests = nullptr; +thread_local void *prepare_context_for_tests = nullptr; +#endif +} // namespace + +#ifdef CROWDB_TREE_TEST_UTIL +void set_flush_prepare_hook_for_tests(FlushPrepareHook hook, void *context) noexcept +{ + prepare_hook_for_tests = hook; + prepare_context_for_tests = context; +} +#endif + +size_t FlushController::source_window() const noexcept +{ + return state_->options.max_sources; +} + +bool FlushController::State::try_enter() noexcept +{ + if (closed.load(std::memory_order_acquire)) { + return false; + } + if (!preparing.test_and_set(std::memory_order_acquire)) { +#ifdef CROWDB_TREE_TEST_UTIL + if (prepare_hook_for_tests != nullptr) { + prepare_hook_for_tests(prepare_context_for_tests, true); + } +#endif + return true; + } + preparing_waiter.store(true, std::memory_order_release); +#ifdef CROWDB_TREE_TEST_UTIL + if (prepare_hook_for_tests != nullptr) { + prepare_hook_for_tests(prepare_context_for_tests, false); + } +#endif + // Covers the holder releasing just before this failed entrant registered. + // Only an observed release is progress; uncontended planning is not. + if (!preparing.test(std::memory_order_acquire) && progress) { + progress->advance(); + } + return false; +} + +void FlushController::State::leave() noexcept +{ + preparing.clear(std::memory_order_release); + if (preparing_waiter.exchange(false, std::memory_order_acq_rel) && progress) { + progress->advance(); + } +} + +bool FlushController::State::has_pending() const noexcept +{ + return pending && !pending->state_->complete_success.load(std::memory_order_acquire) && + !pending->state_->replaceable.load(std::memory_order_acquire); +} + +size_t FlushController::State::control_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<FlushOperation::State>() + + detail::shared_allocation_bound<FlushOperation>() + + (4 * PublicationCoordinator::empty_commit_reserve_bytes()) + + PublicationCoordinator::complete_view_reserve_bytes(); +} + +FlushController::FlushController(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, + SourceAdmission &sources, const std::atomic<uint64_t> &contiguous, + std::shared_ptr<detail::FlushRuntime::Lane> lane, FlushBatchOptions options, + FlushBatch::Maintenance maintenance, + std::shared_ptr<detail::AllocationReservation> lifecycle, + std::shared_ptr<ApplyProgress> progress) + : state_(detail::reserved_unique<State>(lifecycle, publisher, runtime, sources, contiguous, std::move(lane), + std::move(options), std::move(maintenance))) +{ + state_->progress = std::move(progress); + state_->lifecycle = std::move(lifecycle); +} + +size_t FlushController::allocation_bound() noexcept +{ + return sizeof(FlushController) + sizeof(State) + detail::shared_allocation_bound<SourcePause>(); +} + +FlushController::~FlushController() +{ + close(); +} + +Status FlushController::prepare(std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<FlushOperation> *out, std::optional<uint64_t> expected_commit_seq) +{ + return prepare_impl(std::move(metadata), out, expected_commit_seq, nullptr); +} + +Status FlushController::prepare(const FlushPreparationOptions &preparation, std::shared_ptr<FlushOperation> *out) +{ + return prepare_impl({}, out, {}, &preparation); +} + +Status FlushController::prepare(const FlushPreparationOptions &preparation, std::shared_ptr<FlushOperation> *out, + bool *blocked) +{ + return prepare_impl({}, out, {}, &preparation, nullptr, nullptr, blocked); +} + +Status FlushController::prepare_impl(std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<FlushOperation> *out, std::optional<uint64_t> expected_commit_seq, + const FlushPreparationOptions *preparation, const FlushReservePlan *plan, + detail::FlushCredits *credits, bool *blocked) +{ + if (blocked != nullptr) { + *blocked = false; + } + if (out == nullptr || (!metadata && preparation == nullptr) || !state_->lane || + (preparation != nullptr && preparation->runtime.get() != &state_->runtime)) { + return Status::invalid_argument("flush needs output, credit and lane"); + } + if (!state_->try_enter()) { + if (blocked != nullptr) { + *blocked = !state_->closed.load(std::memory_order_acquire); + } + return Status::unavailable("flush controller closed or preparing"); + } + + struct Release + { + State &state; + + ~Release() + { + state.leave(); + } + } release{*state_}; + + if (state_->has_pending()) { + *out = state_->pending; + return Status::Ok(); + } + state_->pending.reset(); + try { + const auto source_view = state_->sources.catalog(); + if (source_view->active_budget) { + const auto &limits = source_view->active_budget->options(); + const auto largest_key = + std::min(limits.max_key_size, std::max(limits.max_key_bytes, limits.max_gap_key_bytes)); + if (largest_key > state_->options.run_limits.max_key_bytes) { + return Status::invalid_argument("flush key window is smaller than the admitted source key limit"); + } + } + const auto current = state_->publisher.capture(); + if (plan != nullptr && (credits == nullptr || !plan->view || !plan->sources || plan->view != current || + plan->sources != source_view)) { + return Status::unavailable("flush reserve plan changed before binding"); + } + if (preparation != nullptr && plan == nullptr) { + std::shared_ptr<FlushOperation> covered; + auto status = state_->prepare_covered(*preparation, current, source_view, &covered); + if (!status.ok() || covered) { + if (covered) { + *out = std::move(covered); + } + return status; + } + } + if (preparation != nullptr) { + if (plan != nullptr) { + if (!plan->pause || !plan->pause->ready() || !plan->pause->matches(*source_view)) { + return Status::unavailable("flush plan lost its stable admission bound"); + } + } + else { + auto status = state_->pause_source(*source_view); + if (!status.ok()) { + if (blocked != nullptr) { + *blocked = status.code() == Code::kUnavailable; + } + return status; + } + } + } + + struct ReleasePause + { + State &state; + + ~ReleasePause() + { + state.pause.reset(); + } + } release_pause{*state_}; + + auto options = state_->options; + auto maintenance = state_->maintenance; + detail::FlushReservations reservations; + if (preparation != nullptr) { + auto status = + plan != nullptr + ? detail::bind_flush_reservations( + state_->publisher, *preparation, source_view->active_budget->options().account, state_->lane, + plan->bounds, std::move(*credits), &options, &maintenance, &reservations) + : detail::prepare_flush_reservations(state_->publisher, *current, *source_view, *preparation, + State::control_allocation_bound(), state_->lane, &options, + &maintenance, &reservations, state_->pause->bound()); + if (!status.ok()) { + return status; + } + metadata = reservations.handoff; + expected_commit_seq = current->commit_seq; + } + auto operation = detail::reserved_shared<FlushOperation::State>( + metadata, state_->publisher, state_->runtime, state_->sources, state_->contiguous, state_->lane, + std::move(options), std::move(maintenance)); + operation->reservations = std::move(reservations); + operation->progress = state_->progress; + operation->expected_catalog = preparation != nullptr ? source_view : nullptr; + operation->source_pause = plan != nullptr ? plan->pause : state_->pause; + operation->report.floor = current->coverage; + operation->report.frontier = current->coverage; + operation->report.published_coverage = current->coverage; + operation->metadata = std::move(metadata); + operation->expected_commit_seq = expected_commit_seq.value_or(current->commit_seq); + operation->admission_owner = next_admission_owner.fetch_add(1, std::memory_order_relaxed); + operation->admission_record = detail::reserved_shared<InstallRecord>( + operation->metadata, InstallToken{.generation = current->generation, + .batch = operation->admission_owner, + .range = UINT64_MAX - 3, + .sequence = 0}); + auto release_credit = operation->metadata->take_reserved(PublicationCoordinator::empty_commit_reserve_bytes()); + if (!release_credit) { + return Status::resource_exhausted("flush needs independent admission release credit"); + } + operation->release_metadata = + detail::reserved_shared<detail::AllocationReservation>(operation->metadata, std::move(*release_credit)); + operation->release_request = + detail::reserved_shared<PublicationRequest>(operation->metadata, operation->metadata); + operation->release_request->transaction = { + .record = detail::reserved_shared<InstallRecord>(operation->metadata, + InstallToken{.generation = current->generation, + .batch = operation->admission_owner, + .range = UINT64_MAX - 3, + .sequence = 1} + ), + .metadata = operation->release_metadata, + .flush_admission = + FlushAdmissionPublication{.action = FlushAdmissionAction::Release, .owner = operation->admission_owner}, + .flush_admission_owner = operation->admission_owner + }; + state_->pending = detail::reserved_construct<FlushOperation>(operation->metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) FlushOperation(std::move(operation)); + }); + *out = state_->pending; + return Status::Ok(); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted("flush operation allocation failed"); + } +} + +void FlushController::close() noexcept +{ + state_->closed.store(true, std::memory_order_release); + if (state_->progress) { + state_->progress->advance(); + } +} + +void FlushController::release_pending() noexcept +{ + state_->pending.reset(); + state_->pause.reset(); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_controller_covered.cpp b/lib/crowdb-tree/src/btree/flush_controller_covered.cpp new file mode 100644 index 000000000..e83811e54 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_controller_covered.cpp @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "flush_controller_internal.h" +#include "flush_operation_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +Status FlushController::State::prepare_covered(const FlushPreparationOptions &preparation, + const std::shared_ptr<const PublishedTreeState> &view, + const std::shared_ptr<const SourceCatalog> &catalog, + std::shared_ptr<FlushOperation> *out) +try { + if (!preparation.store || !catalog->active_budget || options.loader || preparation.maintenance.prepaid || + preparation.maintenance.acquire || preparation.maintenance.publication.metadata || !view->sources || + view->sources->inheritance || view->pending_handoff || view->flush_admission_owner != 0 || + catalog->floor != view->coverage || view->sources->floor != catalog->floor || + view->sources->tables.size() != catalog->tables.size() || + !std::equal(view->sources->tables.begin(), view->sources->tables.end(), catalog->tables.begin()) || + contiguous.load(std::memory_order_acquire) > catalog->floor) { + return Status::Ok(); + } + // This load fixes an already covered prefix. Writers completing after it + // remain in the current source and belong to a subsequent flush. + constexpr auto bytes = detail::shared_allocation_bound<FlushOperation::State>() + + detail::shared_allocation_bound<FlushOperation>() + + detail::shared_allocation_bound<detail::AllocationReservation>(); + auto paid = detail::ResourceBudget::try_reserve(catalog->active_budget->options().account, + detail::BudgetClass::kHandoff, bytes); + if (!paid) { + return Status::unavailable("covered flush control reserve unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*paid)); + auto bounds = options; + bounds.cpu_progress = preparation.cpu_progress; + bounds.publication = preparation.publication; + auto operation = detail::reserved_shared<FlushOperation::State>(metadata, publisher, runtime, sources, contiguous, + lane, std::move(bounds), FlushBatch::Maintenance{}); + operation->progress = progress; + operation->metadata = metadata; + operation->frozen.base = view; + operation->frozen.frontier = catalog->floor; + operation->report.floor = catalog->floor; + operation->report.frontier = catalog->floor; + operation->report.published_coverage = catalog->floor; + operation->report.handoff_status = Status::Ok(); + operation->report.maintenance_status = Status::Ok(); + operation->report.deferred_pages = view->stats.dirty_pages; + operation->report.phase = FlushPhase::kComplete; + operation->complete_success.store(true, std::memory_order_release); + auto result = detail::reserved_construct<FlushOperation>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) FlushOperation(std::move(operation)); }); + metadata->seal(); + pending = std::move(result); + *out = pending; + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("covered flush control allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_controller_internal.h b/lib/crowdb-tree/src/btree/flush_controller_internal.h new file mode 100644 index 000000000..87c294f56 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_controller_internal.h @@ -0,0 +1,50 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "flush_operation.h" +#include "source_pause.h" + +namespace crowdb::tree +{ +struct FlushController::State +{ + State(PublicationCoordinator &publish, detail::FlushRuntime &cpu, SourceAdmission &admission, + const std::atomic<uint64_t> &frontier, std::shared_ptr<detail::FlushRuntime::Lane> tree_lane, + FlushBatchOptions bounds, FlushBatch::Maintenance maintain) + : publisher(publish), + runtime(cpu), + sources(admission), + contiguous(frontier), + lane(std::move(tree_lane)), + options(std::move(bounds)), + maintenance(std::move(maintain)) + { + } + + Status pause_source(const SourceCatalog &catalog); + [[nodiscard]] bool try_enter() noexcept; + void leave() noexcept; + [[nodiscard]] bool has_pending() const noexcept; + [[nodiscard]] static size_t control_allocation_bound() noexcept; + + Status prepare_covered(const FlushPreparationOptions &preparation, + const std::shared_ptr<const PublishedTreeState> &view, + const std::shared_ptr<const SourceCatalog> &catalog, std::shared_ptr<FlushOperation> *out); + + PublicationCoordinator &publisher; + detail::FlushRuntime &runtime; + SourceAdmission &sources; + const std::atomic<uint64_t> &contiguous; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<ApplyProgress> progress; + FlushBatchOptions options; + FlushBatch::Maintenance maintenance; + std::atomic<bool> closed{false}; + std::atomic_flag preparing = ATOMIC_FLAG_INIT; + std::atomic<bool> preparing_waiter{false}; + std::shared_ptr<FlushOperation> pending; + std::shared_ptr<detail::AllocationReservation> lifecycle; + std::shared_ptr<SourcePause> pause; +}; + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_controller_pause.cpp b/lib/crowdb-tree/src/btree/flush_controller_pause.cpp new file mode 100644 index 000000000..cc981f5b3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_controller_pause.cpp @@ -0,0 +1,28 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_controller_internal.h" + +namespace crowdb::tree +{ +Status FlushController::State::pause_source(const SourceCatalog &catalog) +{ + if (pause && !pause->matches(catalog)) { + pause.reset(); + } + if (!pause) { + auto status = SourcePause::create(catalog, lifecycle, progress, &pause); + if (!status.ok()) { + return status; + } + } + if (!pause->ready()) { + // No caller owns an accepted plan yet. The pause retains itself until + // the last short bind exits, then resumes ordinary admission and wakes + // existing progress waiters. A synchronous rejection cannot strand it. + pause->wake_on_release(); + pause.reset(); + return Status::unavailable("source binds are completing before flush planning"); + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_controller_plan.cpp b/lib/crowdb-tree/src/btree/flush_controller_plan.cpp new file mode 100644 index 000000000..cad1e30a6 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_controller_plan.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "apply_progress.h" +#include "flush_controller_internal.h" + +namespace crowdb::tree +{ +Status FlushController::prepare_or_plan(const FlushPreparationOptions &preparation, + std::shared_ptr<FlushOperation> *existing, FlushReservePlan *plan) +{ + if (existing == nullptr || *existing || plan == nullptr || plan->view || + preparation.runtime.get() != &state_->runtime || !state_->lane) { + return Status::invalid_argument("flush planning requires empty outputs and the bound runtime"); + } + if (!state_->try_enter()) { + return Status::unavailable("flush controller closed or preparing"); + } + + struct Release + { + State &state; + + ~Release() + { + state.leave(); + } + } release{*state_}; + + if (state_->has_pending()) { + *existing = state_->pending; + return Status::Ok(); + } + const auto sources = state_->sources.catalog(); + const auto view = state_->publisher.capture(); + auto status = state_->prepare_covered(preparation, view, sources, existing); + if (!status.ok() || *existing) { + return status; + } + status = state_->pause_source(*sources); + if (!status.ok()) { + return status; + } + auto pause = std::move(state_->pause); + detail::FlushReserveBounds bounds; + status = detail::flush_reserve_bounds(state_->publisher, *view, *sources, preparation, + State::control_allocation_bound(), state_->options, &bounds, pause->bound()); + if (!status.ok()) { + return status; + } + *plan = {.view = view, .sources = sources, .bounds = bounds, .pause = std::move(pause)}; + return Status::Ok(); +} + +Status FlushController::prepare_prepaid(const FlushPreparationOptions &preparation, const FlushReservePlan &plan, + detail::FlushCredits credits, std::shared_ptr<FlushOperation> *out) +{ + return prepare_impl({}, out, {}, &preparation, &plan, &credits); +} + +Status FlushController::check_rotation(const FlushPreparationOptions &preparation, const PublishedTreeState &view, + const SourceCatalog &catalog, WriteFootprint sealed) const +{ + detail::FlushReserveBounds bounds; + auto status = + detail::flush_reserve_bounds(state_->publisher, view, catalog, preparation, State::control_allocation_bound(), + state_->options, &bounds, sealed, true); + if (!status.ok()) { + return status; + } + for (size_t i = 0; i < bounds.amounts.size(); ++i) { + if (bounds.amounts[i] > detail::ResourceBudget::capacity(catalog.active_budget->options().account, + detail::FlushReserveBounds::categories[i])) { + return Status::resource_exhausted("source rotation would exceed a complete flush partition"); + } + } + return Status::Ok(); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_controller_test.h b/lib/crowdb-tree/src/btree/flush_controller_test.h new file mode 100644 index 000000000..e3aa42c00 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_controller_test.h @@ -0,0 +1,12 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#ifdef CROWDB_TREE_TEST_UTIL +namespace crowdb::tree +{ +// The calling thread observes either acquiring or waiting for the prepare gate. +using FlushPrepareHook = void (*)(void *context, bool acquired) noexcept; +void set_flush_prepare_hook_for_tests(FlushPrepareHook hook, void *context) noexcept; +} // namespace crowdb::tree +#endif diff --git a/lib/crowdb-tree/src/btree/flush_dispatch_metrics.h b/lib/crowdb-tree/src/btree/flush_dispatch_metrics.h new file mode 100644 index 000000000..f9c00c4d4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_dispatch_metrics.h @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <atomic> +#include <cstdint> + +namespace crowdb::tree +{ +struct FlushDispatchSnapshot +{ + uint64_t inline_probes = 0; + uint64_t inline_ranges = 0; + uint64_t queued_steps = 0; + uint64_t prepared_runs = 0; + uint64_t prepared_entries = 0; + uint64_t prepared_bytes = 0; + uint64_t cow_bytes = 0; + uint64_t active_steps = 0; + uint64_t peak_steps = 0; +}; + +// Fixed dimensions, updated once per work step, run, or committed window. +class FlushDispatchMetrics +{ + public: + void inline_probe() noexcept + { + inline_probes_.fetch_add(1, std::memory_order_relaxed); + } + + void inline_range() noexcept + { + inline_ranges_.fetch_add(1, std::memory_order_relaxed); + } + + void queued_step() noexcept + { + queued_steps_.fetch_add(1, std::memory_order_relaxed); + } + + void run(uint64_t entries, uint64_t bytes) noexcept + { + prepared_runs_.fetch_add(1, std::memory_order_relaxed); + prepared_entries_.fetch_add(entries, std::memory_order_relaxed); + prepared_bytes_.fetch_add(bytes, std::memory_order_relaxed); + } + + void installed(uint64_t bytes) noexcept + { + cow_bytes_.fetch_add(bytes, std::memory_order_relaxed); + } + + void begin() noexcept + { + const auto active = active_steps_.fetch_add(1, std::memory_order_relaxed) + 1; + auto peak = peak_steps_.load(std::memory_order_relaxed); + while (active > peak && !peak_steps_.compare_exchange_weak(peak, active, std::memory_order_relaxed)) { + } + } + + void finish() noexcept + { + active_steps_.fetch_sub(1, std::memory_order_relaxed); + } + + [[nodiscard]] FlushDispatchSnapshot snapshot() const noexcept + { + return {.inline_probes = inline_probes_.load(std::memory_order_relaxed), + .inline_ranges = inline_ranges_.load(std::memory_order_relaxed), + .queued_steps = queued_steps_.load(std::memory_order_relaxed), + .prepared_runs = prepared_runs_.load(std::memory_order_relaxed), + .prepared_entries = prepared_entries_.load(std::memory_order_relaxed), + .prepared_bytes = prepared_bytes_.load(std::memory_order_relaxed), + .cow_bytes = cow_bytes_.load(std::memory_order_relaxed), + .active_steps = active_steps_.load(std::memory_order_relaxed), + .peak_steps = peak_steps_.load(std::memory_order_relaxed)}; + } + + private: + std::atomic<uint64_t> inline_probes_{0}, inline_ranges_{0}, queued_steps_{0}; + std::atomic<uint64_t> prepared_runs_{0}, prepared_entries_{0}, prepared_bytes_{0}, cow_bytes_{0}; + std::atomic<uint64_t> active_steps_{0}, peak_steps_{0}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_operation.cpp b/lib/crowdb-tree/src/btree/flush_operation.cpp new file mode 100644 index 000000000..167c306c8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_operation.cpp @@ -0,0 +1,384 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "apply_progress.h" +#include "flush_operation_internal.h" +#include "source_pause.h" + +#include <algorithm> + +namespace crowdb::tree +{ +FlushOperation::~FlushOperation() = default; + +detail::CallbackSender FlushOperation::sender() noexcept +{ + return {this, &FlushOperation::submit}; +} + +FlushReportSender FlushOperation::report_sender() noexcept +{ + return {this, &FlushOperation::submit_report}; +} + +FlushReportSender FlushOperation::capture_sender() noexcept +{ + return {this, &FlushOperation::submit_capture}; +} + +FrozenSourceView FlushOperation::frozen_sources() const noexcept +{ + return state_->frozen; +} + +void FlushOperation::submit_capture(void *source, FlushReportComplete complete, void *context) +{ + auto state = static_cast<FlushOperation *>(source)->state_; + state->start(nullptr, context, complete, true); +} + +void FlushOperation::submit_report(void *source, FlushReportComplete complete, void *context) +{ + auto state = static_cast<FlushOperation *>(source)->state_; + state->start(nullptr, context, complete); +} + +const FlushResult &FlushOperation::result() const noexcept +{ + return state_->report; +} + +void FlushOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto state = static_cast<FlushOperation *>(source)->state_; + state->start(complete, context); +} + +void FlushOperation::request_stop() noexcept +{ + auto state = state_; + state->lifecycle.fetch_or(State::kStopped, std::memory_order_acq_rel); + if (auto *batch = state->cancel_batch.load(std::memory_order_acquire)) { + batch->request_stop(); + } +} + +bool FlushOperation::State::stopped() const noexcept +{ + return (lifecycle.load(std::memory_order_acquire) & kStopped) != 0; +} + +bool FlushOperation::State::start(detail::CallbackComplete callback, void *argument, + FlushReportComplete report_callback, bool only_capture, bool reject_running) noexcept +{ + auto previous = lifecycle.load(std::memory_order_acquire); + do { + if ((previous & kRunning) != 0) { + if (!reject_running) { + return false; + } + if (report_callback != nullptr) { + report_callback(argument, detail::CallbackSignal::kError, Status::unavailable(), {}); + } + else { + callback(argument, detail::CallbackSignal::kError, Status::unavailable()); + } + return false; + } + } while (!lifecycle.compare_exchange_weak(previous, previous | kRunning, std::memory_order_acq_rel)); + batch_metrics.reset(); + if (!only_capture) { + batch_metrics.emplace(publisher.work_metrics(), TreeStage::FlushBatch, tree_metrics_now_ns()); + } + capture_metrics.reset(); + drain_metrics.reset(); + complete = callback; + report_complete = report_callback; + context = argument; + capture_only = only_capture; + report.admission_stale = false; + covered_at_start = complete_success.load(std::memory_order_acquire) || + (report.handoff_status.ok() && + (report.phase == FlushPhase::kCovered || report.phase == FlushPhase::kMaintenance || + report.phase == FlushPhase::kComplete)); + replaceable.store(false, std::memory_order_release); + dispatch(CpuStage::kBegin); + return true; +} + +void FlushOperation::State::dispatch(CpuStage stage) noexcept +{ + queued_stage = stage; + queued_owner = weak_from_this().lock(); + try { + auto status = runtime.try_submit(lane, &State::run_cpu, this); + if (status.ok()) { + return; + } + if (options.cpu_progress) { + options.cpu_progress->request(cpu_node, queued_owner); + return; + } + auto self = std::move(queued_owner); + fail(std::move(status)); + } + catch (const std::bad_alloc &) { + if (options.cpu_progress) { + options.cpu_progress->request(cpu_node, queued_owner); + return; + } + auto self = std::move(queued_owner); + fail(Status::resource_exhausted()); + } +} + +void FlushOperation::State::run_cpu(void *context) noexcept +{ + auto *raw = static_cast<State *>(context); + auto self = std::move(raw->queued_owner); + const auto stage = self->queued_stage; + try { + if (stage == CpuStage::kBegin) { + self->begin(); + } + else if (stage == CpuStage::kDrain) { + self->drain_sources(); + } + else { + self->start_batch(); + } + } + catch (const std::bad_alloc &) { + self->fail(Status::resource_exhausted()); + } + catch (const std::exception &) { + self->fail(Status::internal_error()); + } +} + +void FlushOperation::State::begin() +{ + if (capture_only && !frozen.base && covered_at_start) { + // A normal connection already handed off this source set. Pair the + // newly requested capture with that complete source-free L1, not its + // former catalog floor. Existing capture pairs never change. + frozen.base = publisher.capture_complete(); + frozen.frontier = frozen.base->coverage; + report.floor = frozen.frontier; + report.frontier = frozen.frontier; + report.published_coverage = frozen.frontier; + report.handoff_status = Status::Ok(); + report.phase = FlushPhase::kCaptured; + finish(); + return; + } + if (capture_only && (frozen.base || (capture && capture->drained()))) { + if (!frozen.base) { + frozen = {.catalog = capture->expected_catalog(), + .base = publisher.capture_complete(), + .frontier = capture->frontier(), + .batch_id = capture->batch_id()}; + } + report.handoff_status = Status::Ok(); + report.phase = FlushPhase::kCaptured; + finish(); + return; + } + if (complete_success.load(std::memory_order_acquire)) { + report.phase = FlushPhase::kComplete; + finish(); + return; + } + if (admission_released) { + fail(Status::unavailable("released admission requires a newly prepared flush")); + return; + } + if (!capture && !batch) { + const auto view = publisher.capture(); + report.floor = view->coverage; + report.frontier = view->coverage; + report.published_coverage = view->coverage; + } + if (stopped()) { + fail(Status::unavailable("flush stopped before dispatch")); + return; + } + if (batch) { + start_batch(); + return; + } + if (!capture) { + auto source_view = sources.catalog(); + const auto tree_view = publisher.capture(); + if (tree_view->sources->inheritance) { + fail(Status::unavailable("split inheritance must publish its fixed " + "prefix before ordinary flush")); + return; + } + if (expected_catalog && source_view != expected_catalog) { + fail(Status::unavailable("source catalog changed after reserve preparation")); + return; + } + if (tree_view->coverage != source_view->floor || tree_view->sources->floor != source_view->floor || + tree_view->sources->tables.size() != source_view->tables.size() || + !std::equal(tree_view->sources->tables.begin(), tree_view->sources->tables.end(), + source_view->tables.begin())) { + fail(Status::unavailable("Tree and write source catalogs differ")); + return; + } + report.floor = source_view->floor; + report.frontier = source_view->floor; + report.published_coverage = source_view->floor; + if (!admitted && contiguous.load(std::memory_order_acquire) <= source_view->floor) { + if (capture_only) { + frozen.base = tree_view; + frozen.frontier = source_view->floor; + } + report.handoff_status = Status::Ok(); + report.maintenance_status = Status::Ok(); + report.deferred_pages = tree_view->stats.dirty_pages; + report.phase = FlushPhase::kComplete; + metadata->seal(); + release_metadata->seal(); + complete_success.store(true, std::memory_order_release); + finish(); + return; + } + if (!admitted) { + acquire_admission(); + return; + } +#ifdef CROWDB_TREE_TEST_UTIL + if (options.before_capture_for_tests) { + options.before_capture_for_tests(); + } +#endif + capture_metrics.emplace(publisher.work_metrics(), TreeStage::FlushCapture, tree_metrics_now_ns()); + auto status = sources.try_capture(contiguous, metadata, &capture, expected_catalog); + if (!status.ok()) { + fail(std::move(status)); + return; + } + source_pause.reset(); + captured_at_ns = tree_metrics_now_ns(); + release_metadata->seal(); + expected_catalog.reset(); + release_metadata.reset(); + release_request.reset(); + report.batch_id = capture->batch_id(); + report.floor = capture->floor(); + report.frontier = capture->frontier(); + expected_sources = tree_view->sources; + } + if (!capture->successor_active()) { + if (!capture_metrics) { + capture_metrics.emplace(publisher.work_metrics(), TreeStage::FlushCapture, tree_metrics_now_ns()); + } + publish_successor(); + return; + } + drain_sources(); +} + +void FlushOperation::State::fail(Status status) noexcept +{ + if (admitted && !capture && !batch) { + try { + release_admission(std::move(status)); + } + catch (const std::bad_alloc &) { + fail_terminal(Status::resource_exhausted("flush admission release allocation failed")); + } + catch (const std::exception &) { + fail_terminal(Status::internal_error("flush admission release failed")); + } + return; + } + fail_terminal(std::move(status)); +} + +void FlushOperation::State::fail_terminal(Status status) noexcept +{ + report.admission_stale = status.code() == Code::kUnavailable && !admitted && !admission_released && !capture && + !batch && !stopped() && expected_commit_seq != publisher.capture()->commit_seq; + if (report.published_coverage == report.frontier && batch) { + report.maintenance_status = std::move(status); + report.phase = FlushPhase::kComplete; + } + else { + report.handoff_status = std::move(status); + report.phase = capture ? FlushPhase::kRepairRequired : FlushPhase::kCaptured; + if (capture) { + (void)sources.mark_repair(capture, capture->batch_id(), report.floor); + } + } + finish(); +} + +void FlushOperation::State::finish() noexcept +{ + auto self = weak_from_this().lock(); + auto callback = complete; + auto *argument = context; + auto report_callback = report_complete; + if (!admitted && !capture && !batch) { + source_pause.reset(); + options.loader.reset(); + metadata->seal(); + if (release_metadata) { + release_metadata->seal(); + } + reservations.seal(); + } + else if (report.phase == FlushPhase::kComplete) { + options.loader.reset(); + reservations.seal(); + } + auto outcome = report.handoff_status.ok() ? TreeOutcome::Success : TreeOutcome::Error; + if (stopped()) { + outcome = TreeOutcome::Cancelled; + } + const auto now = tree_metrics_now_ns(); + if (batch_metrics) { + const auto batch_outcome = + outcome == TreeOutcome::Success && !report.maintenance_status.ok() ? TreeOutcome::Error : outcome; + (void)batch_metrics->finish(batch_outcome, now); + } + if (capture_metrics) { + (void)capture_metrics->finish(outcome, now); + } + if (drain_metrics) { + (void)drain_metrics->finish(outcome, now); + } + if (capture && report.phase == FlushPhase::kRepairRequired && !repair_metrics) { + repair_metrics.emplace(publisher.work_metrics(), TreeStage::FlushRepair, captured_at_ns); + } + const bool covered = report.handoff_status.ok() && + (report.phase == FlushPhase::kCovered || report.phase == FlushPhase::kMaintenance || + report.phase == FlushPhase::kComplete); + report.handoff_advanced = !covered_at_start && covered && report.frontier > report.floor; + // Freeze before reopening the lifecycle. A concurrent reconnect may now + // mutate the operation report without changing this terminal value. + FlushResult completed{.batch_id = report.batch_id, + .floor = report.floor, + .frontier = report.frontier, + .published_coverage = report.published_coverage, + .handoff_status = report.handoff_status.without_message(), + .maintenance_status = report.maintenance_status.without_message(), + .deferred_pages = report.deferred_pages, + .phase = report.phase, + .handoff_advanced = report.handoff_advanced, + .admission_stale = report.admission_stale}; + replaceable.store(!admitted && !capture && !batch, std::memory_order_release); + lifecycle.exchange(0, std::memory_order_acq_rel); + lifecycle.notify_all(); + if (progress) { + progress->advance(); + } + if (report_callback != nullptr) { + report_callback(argument, detail::CallbackSignal::kValue, Status::Ok(), std::move(completed)); + } + else { + callback(argument, detail::CallbackSignal::kValue, Status::Ok()); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_operation.h b/lib/crowdb-tree/src/btree/flush_operation.h new file mode 100644 index 000000000..ccb2083a3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_operation.h @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "flush_report_sender.h" +#include "flush_reservation.h" +#include "source_admission.h" + +namespace crowdb::tree +{ +class ApplyProgress; +class SourcePause; + +struct FrozenSourceView +{ + std::shared_ptr<const SourceCatalog> catalog; + // Complete L1 paired with catalog.floor, fixed before capture terminal. + std::shared_ptr<const PublishedTreeState> base = nullptr; + uint64_t frontier = 0; + uint64_t batch_id = 0; +}; + +struct FlushReservePlan +{ + std::shared_ptr<const PublishedTreeState> view = nullptr; + std::shared_ptr<const SourceCatalog> sources = nullptr; + detail::FlushReserveBounds bounds{}; + std::shared_ptr<SourcePause> pause = nullptr; +}; + +class FlushOperation +{ + struct State; + + public: + ~FlushOperation(); + FlushOperation(const FlushOperation &) = delete; + FlushOperation &operator=(const FlushOperation &) = delete; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] FlushReportSender report_sender() noexcept; + // Close, publish the successor, and drain this exact finite source set. + // The normal sender resumes its handoff; capture does not credit coverage. + // Like the normal sender, only one connection may run at a time. + [[nodiscard]] FlushReportSender capture_sender() noexcept; + // Read after a successful capture terminal. These immutable owners remain + // valid while the operation resumes and after the source Tree closes. + [[nodiscard]] FrozenSourceView frozen_sources() const noexcept; + // Host-only: wait for an existing connection, then run this finite operation. + [[nodiscard]] Status wait(); + // Host capture joins a current connection and preserves its finite source + // identity; asynchronous sender connections still reject concurrent use. + [[nodiscard]] Status wait_capture(FlushResult *out); + void request_stop() noexcept; + // Read only after terminal. A retry preserves batch/capture identities. + [[nodiscard]] const FlushResult &result() const noexcept; + + private: + friend class FlushController; + + explicit FlushOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + [[nodiscard]] Status wait_report(bool capture_only, FlushResult *out); + static void submit(void *source, detail::CallbackComplete complete, void *context); + static void submit_report(void *source, FlushReportComplete complete, void *context); + static void submit_capture(void *source, FlushReportComplete complete, void *context); + std::shared_ptr<State> state_; +}; + +// One controller per Tree owns repair state even if a caller drops its handle. +// Before enabling writes, the host must publish SourceAdmission::catalog()'s +// complete initial membership with the initial Tree mapping. Runtime, source +// admission, publisher and contiguous tracker outlive all operation callbacks. +class FlushController +{ + public: + FlushController(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, SourceAdmission &sources, + const std::atomic<uint64_t> &contiguous, std::shared_ptr<detail::FlushRuntime::Lane> lane, + FlushBatchOptions options = {}, FlushBatch::Maintenance maintenance = {}, + std::shared_ptr<detail::AllocationReservation> lifecycle = {}, + std::shared_ptr<ApplyProgress> progress = {}); + [[nodiscard]] static size_t allocation_bound() noexcept; + [[nodiscard]] size_t source_window() const noexcept; + [[nodiscard]] Status check_rotation(const FlushPreparationOptions &preparation, const PublishedTreeState &view, + const SourceCatalog &catalog, WriteFootprint sealed) const; + ~FlushController(); + FlushController(const FlushController &) = delete; + FlushController &operator=(const FlushController &) = delete; + + // The caller has already acquired whole-batch metadata credit against the + // named immutable state. A stale sequence fails before source closure. + // Omitting the sequence captures it here for standalone callers. Existing + // repair work is returned unchanged; only a completed operation permits a + // new capture. The supplied maintenance callback owns its reclaim admission. + [[nodiscard]] Status prepare(std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<FlushOperation> *out, + std::optional<uint64_t> expected_commit_seq = {}); + // Derive the complete handoff bound from source admission caps and the + // immutable mapping, acquire all progress partitions, then prepare their + // loader/maintenance consumers. Rejection leaves sources open. + [[nodiscard]] Status prepare(const FlushPreparationOptions &preparation, std::shared_ptr<FlushOperation> *out); + [[nodiscard]] Status prepare(const FlushPreparationOptions &preparation, std::shared_ptr<FlushOperation> *out, + bool *blocked); + // Return existing repair work, or a complete immutable-state reserve plan. + // The caller may await each named partition without holding this gate. + [[nodiscard]] Status prepare_or_plan(const FlushPreparationOptions &preparation, + std::shared_ptr<FlushOperation> *existing, FlushReservePlan *plan); + [[nodiscard]] Status prepare_prepaid(const FlushPreparationOptions &preparation, const FlushReservePlan &plan, + detail::FlushCredits credits, std::shared_ptr<FlushOperation> *out); + // Reject new prepare calls without cancelling accepted work. The host + // awaits operation terminals and releases writers before draining and + // destroying the shared runtime, source admission or publisher. + void close() noexcept; + // Host teardown after prepare callers have joined. Running/late callbacks + // retain their own state; the controller must not retain backend tickets + // while the host waits for those tickets to drain. + void release_pending() noexcept; + + private: + struct State; + Status prepare_impl(std::shared_ptr<detail::AllocationReservation> metadata, std::shared_ptr<FlushOperation> *out, + std::optional<uint64_t> expected_commit_seq, const FlushPreparationOptions *preparation, + const FlushReservePlan *plan = nullptr, detail::FlushCredits *credits = nullptr, + bool *blocked = nullptr); + detail::ReservedUnique<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_operation_admission.cpp b/lib/crowdb-tree/src/btree/flush_operation_admission.cpp new file mode 100644 index 000000000..89dbaefd7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_operation_admission.cpp @@ -0,0 +1,69 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_operation_internal.h" + +namespace crowdb::tree +{ +void FlushOperation::State::acquire_admission() +{ + auto request = detail::reserved_shared<PublicationRequest>(metadata, metadata); + request->transaction = { + .record = admission_record, + .metadata = metadata, + .expected_commit_seq = expected_commit_seq, + .flush_admission = FlushAdmissionPublication{.action = FlushAdmissionAction::Acquire, .owner = admission_owner}, + .flush_admission_owner = admission_owner + }; + auto self = shared_from_this(); + request->on_done = [self](PublicationResult result) noexcept { + if (!result.status.ok()) { + self->fail(std::move(result.status)); + return; + } + self->admitted = true; + self->dispatch(CpuStage::kBegin); + }; + if (!publisher.try_submit(request, options.publication)) { + fail(Status::resource_exhausted("flush admission queue full or closed")); + } +} + +void FlushOperation::State::release_admission(Status status) +{ + // This descriptor and root credit were retained before admission. Source + // preparation cannot consume the capacity needed to release the gate. + if (!release_request) { + release_request = detail::reserved_shared<PublicationRequest>(release_metadata, release_metadata); + release_request->transaction = { + .record = detail::reserved_shared<InstallRecord>( + release_metadata, InstallToken{.generation = admission_record->token.generation, + .batch = admission_owner, + .range = UINT64_MAX - 3, + .sequence = 1} + ), + .metadata = release_metadata, + .flush_admission = + FlushAdmissionPublication{.action = FlushAdmissionAction::Release, .owner = admission_owner}, + .flush_admission_owner = admission_owner + }; + } + report.handoff_status = std::move(status); + auto request = std::move(release_request); + auto self = shared_from_this(); + request->on_done = [self](PublicationResult result) noexcept { + if (result.status.ok()) { + self->admitted = false; + self->admission_released = true; + self->fail_terminal(std::move(self->report.handoff_status)); + } + else { + self->fail_terminal(std::move(result.status)); + } + }; + if (!publisher.try_submit(request, options.publication)) { + release_request = std::move(request); + release_request->on_done = {}; + fail_terminal(Status::resource_exhausted("flush admission release queue full or closed")); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_operation_internal.h b/lib/crowdb-tree/src/btree/flush_operation_internal.h new file mode 100644 index 000000000..e41073963 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_operation_internal.h @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "apply_dispatcher.h" +#include "flush_operation.h" + +namespace crowdb::tree +{ +struct FlushOperation::State : std::enable_shared_from_this<State> +{ + enum class Stage : uint8_t { kDrain, kBatch }; + enum class CpuStage : uint8_t { kBegin, kDrain, kBatch }; + + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + std::shared_ptr<State> owner; + Stage stage; + void set_value() noexcept; + void set_error(Status status) noexcept; + void set_stopped() noexcept; + }; + + using Connected = stdexec::connect_result_t<detail::CallbackSender, Receiver>; + + struct Attempt + { + Attempt(detail::CallbackSender sender, std::shared_ptr<State> owner, Stage stage) + : connected(stdexec::connect(sender, Receiver{.owner = std::move(owner), .stage = stage})) + { + } + + Connected connected; + }; + + State(PublicationCoordinator &publish, detail::FlushRuntime &cpu, SourceAdmission &admission, + const std::atomic<uint64_t> &frontier, std::shared_ptr<detail::FlushRuntime::Lane> tree_lane, + FlushBatchOptions bounds, FlushBatch::Maintenance maintain) + : publisher(publish), + runtime(cpu), + sources(admission), + contiguous(frontier), + lane(std::move(tree_lane)), + options(std::move(bounds)), + maintenance(std::move(maintain)) + { + } + + bool start(detail::CallbackComplete callback, void *argument, FlushReportComplete report_callback = nullptr, + bool only_capture = false, bool reject_running = true) noexcept; + void dispatch(CpuStage stage) noexcept; + static void run_cpu(void *context) noexcept; + void begin(); + void acquire_admission(); + void release_admission(Status status); + void fail_terminal(Status status) noexcept; + void publish_successor(); + void drain_sources(); + void start_batch(); + void stage_done(Stage stage, Status status) noexcept; + void maintain(std::shared_ptr<const PublishedTreeState> view, FlushBatch::MaintenanceComplete done); + Status acknowledge(const PublishedTreeState &view); + void fail(Status status) noexcept; + void finish() noexcept; + [[nodiscard]] bool stopped() const noexcept; + + PublicationCoordinator &publisher; + detail::FlushRuntime &runtime; + SourceAdmission &sources; + const std::atomic<uint64_t> &contiguous; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<ApplyProgress> progress; + FlushBatchOptions options; + FlushBatch::Maintenance maintenance; + std::shared_ptr<detail::AllocationReservation> metadata; + detail::FlushReservations reservations; + std::shared_ptr<const SourceCatalog> expected_catalog; + std::shared_ptr<SourcePause> source_pause; + uint64_t admission_owner = 0; + uint64_t expected_commit_seq = 0; + bool admitted = false; + std::shared_ptr<detail::AllocationReservation> release_metadata; + bool admission_released = false; + std::shared_ptr<InstallRecord> admission_record; + std::shared_ptr<PublicationRequest> release_request; + std::atomic<bool> replaceable{false}; + std::shared_ptr<SourceCapture> capture; + FrozenSourceView frozen; + bool capture_only = false; + std::shared_ptr<FlushBatch> batch; + std::shared_ptr<InstallRecord> source_record; + std::shared_ptr<const PublishedSources> expected_sources; + std::shared_ptr<const PublishedSources> published_sources; + std::optional<Attempt> drain_attempt; + std::optional<Attempt> batch_attempt; + std::shared_ptr<State> queued_owner; + ApplyDispatcher::Node cpu_node{&State::run_cpu, this}; + CpuStage queued_stage = CpuStage::kBegin; + std::atomic<FlushBatch *> cancel_batch{nullptr}; + std::atomic<bool> complete_success{false}; + bool source_acknowledged = false; + static constexpr uint8_t kRunning = 1; + static constexpr uint8_t kStopped = 2; + std::atomic<uint8_t> lifecycle{0}; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + FlushReportComplete report_complete = nullptr; + bool covered_at_start = false; + FlushResult report; + std::optional<TreeMetricsOperation> batch_metrics; + std::optional<TreeMetricsOperation> capture_metrics; + std::optional<TreeMetricsOperation> drain_metrics; + std::optional<TreeMetricsOperation> repair_metrics; + uint64_t captured_at_ns = 0; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_operation_stages.cpp b/lib/crowdb-tree/src/btree/flush_operation_stages.cpp new file mode 100644 index 000000000..e7517c122 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_operation_stages.cpp @@ -0,0 +1,230 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "apply_progress.h" +#include "flush_operation_internal.h" + +namespace crowdb::tree +{ +void FlushOperation::State::publish_successor() +{ + if (!published_sources) { + auto next = detail::reserved_shared<PublishedSources>(metadata, metadata); + next->floor = capture->floor(); + next->memory = capture->next_catalog()->memory; + next->tables.assign(capture->next_catalog()->tables.begin(), capture->next_catalog()->tables.end()); + published_sources = std::move(next); + } + if (!source_record) { + source_record = detail::reserved_shared<InstallRecord>( + metadata, InstallToken{.generation = capture->next_catalog()->generation, + .batch = capture->batch_id(), + .range = UINT64_MAX - 1, + .sequence = 0}); + } + auto request = detail::reserved_shared<PublicationRequest>(metadata, metadata); + request->transaction = {.record = source_record, + .expected_coverage = report.floor, + .expected_sources = expected_sources, + .sources = published_sources, + .metadata = metadata, + .flush_admission_owner = admission_owner}; + auto self = shared_from_this(); + request->on_done = [self](PublicationResult result) { + try { + if (!result.status.ok()) { + self->fail(std::move(result.status)); + return; + } + auto status = self->sources.activate_successor(self->capture, self->capture->batch_id()); + if (!status.ok()) { + self->fail(std::move(status)); + return; + } + if (self->capture_metrics) { + (void)self->capture_metrics->finish(TreeOutcome::Success, tree_metrics_now_ns()); + } + if (self->progress) { + self->progress->advance(); + } + self->dispatch(CpuStage::kDrain); + } + catch (const std::bad_alloc &) { + self->fail(Status::resource_exhausted()); + } + catch (const std::exception &) { + self->fail(Status::internal_error()); + } + }; + if (!publisher.try_submit(request, options.publication)) { + fail(Status::resource_exhausted("source publication queue full")); + } +} + +void FlushOperation::State::drain_sources() +{ + if (!capture->drained()) { + drain_metrics.emplace(publisher.work_metrics(), TreeStage::FlushWriterDrain, tree_metrics_now_ns()); + } + drain_attempt.emplace(capture->drain_sender(), shared_from_this(), Stage::kDrain); + stdexec::start(drain_attempt->connected); +} + +void FlushOperation::State::start_batch() +{ + if (stopped()) { + fail(Status::unavailable("flush stopped before handoff")); + return; + } + if (!batch) { + auto inputs = detail::reserved_shared<PublishedSources>(metadata, metadata); + inputs->floor = capture->floor(); + inputs->memory = capture->expected_catalog()->memory; + inputs->tables.assign(capture->expected_catalog()->tables.begin(), capture->expected_catalog()->tables.end()); + auto weak = weak_from_this(); + auto maintenance_step = [weak](auto view, auto done) { + if (auto self = weak.lock()) { + self->maintain(std::move(view), std::move(done)); + } + else { + done(Status::unavailable(), 0); + } + }; + options.measure_batch = false; + auto status = + FlushBatch::prepare(publisher, runtime, lane, capture->batch_id(), capture->frontier(), options, + std::move(maintenance_step), &batch, std::move(inputs), metadata, admission_owner); + if (!status.ok()) { + fail(std::move(status)); + return; + } + cancel_batch.store(batch.get(), std::memory_order_release); + } + batch_attempt.emplace(batch->sender(), shared_from_this(), Stage::kBatch); + if (stopped()) { + batch->request_stop(); + } + stdexec::start(batch_attempt->connected); +} + +void FlushOperation::State::Receiver::set_value() noexcept +{ + auto self = std::move(owner); + self->stage_done(stage, Status::Ok()); +} + +void FlushOperation::State::Receiver::set_error(Status status) noexcept +{ + auto self = std::move(owner); + self->stage_done(stage, std::move(status)); +} + +void FlushOperation::State::Receiver::set_stopped() noexcept +{ + auto self = std::move(owner); + self->stage_done(stage, Status::unavailable()); +} + +void FlushOperation::State::stage_done(Stage stage, Status status) noexcept +{ + try { + if (stage == Stage::kDrain && drain_metrics) { + auto outcome = status.ok() ? TreeOutcome::Success : TreeOutcome::Error; + if (stopped()) { + outcome = TreeOutcome::Cancelled; + } + (void)drain_metrics->finish(outcome, tree_metrics_now_ns()); + } + if (!status.ok()) { + fail(std::move(status)); + return; + } + if (stage == Stage::kDrain) { + if (capture_only && !frozen.base) { + frozen = {.catalog = capture->expected_catalog(), + .base = publisher.capture_complete(), + .frontier = capture->frontier(), + .batch_id = capture->batch_id()}; + } + if (capture_only) { + report.handoff_status = Status::Ok(); + report.phase = FlushPhase::kCaptured; + finish(); + return; + } + dispatch(CpuStage::kBatch); + return; + } + report = batch->result(); + if (!report.handoff_status.ok() && capture) { + (void)sources.mark_repair(capture, capture->batch_id(), report.published_coverage); + } + if (report.handoff_status.ok() && report.maintenance_status.ok() && source_acknowledged) { + complete_success.store(true, std::memory_order_release); + } + finish(); + } + catch (const std::bad_alloc &) { + fail(Status::resource_exhausted()); + } + catch (const std::exception &) { + fail(Status::internal_error()); + } +} + +Status FlushOperation::State::acknowledge(const PublishedTreeState &view) +{ + if (source_acknowledged) { + return Status::Ok(); + } + if (!capture || view.coverage != capture->frontier() || view.sources->floor != capture->frontier()) { + return Status::unavailable("published coverage does not match source capture"); + } + size_t index = 0; + for (const auto &table : capture->next_catalog()->tables) { + if (table != capture->successor() && table->slot_range().max <= capture->frontier()) { + continue; + } + if (index == view.sources->tables.size() || view.sources->tables[index++] != table) { + return Status::unavailable("published source membership does not match covered capture"); + } + } + if (index != view.sources->tables.size()) { + return Status::unavailable("unexpected covered source membership"); + } + auto status = sources.finish_covered(capture, capture->batch_id(), view.coverage); + if (!status.ok()) { + return status; + } + if (repair_metrics) { + (void)repair_metrics->finish(TreeOutcome::Success, tree_metrics_now_ns()); + } + report.published_coverage = view.coverage; + source_acknowledged = true; + admitted = false; + capture.reset(); + source_record.reset(); + expected_sources.reset(); + published_sources.reset(); + options.loader.reset(); + metadata->seal(); + // Automatic maintenance still owns the prepaid cold-page load window. + // The terminal flush path seals Routing and I/O after its last callback. + return Status::Ok(); +} + +void FlushOperation::State::maintain(std::shared_ptr<const PublishedTreeState> view, + FlushBatch::MaintenanceComplete done) +{ + auto status = acknowledge(*view); + if (!status.ok()) { + done(std::move(status), view->stats.dirty_pages); + return; + } + if (maintenance) { + maintenance(std::move(view), std::move(done)); + } + else { + done(Status::Ok(), view->stats.dirty_pages); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_operation_wait.cpp b/lib/crowdb-tree/src/btree/flush_operation_wait.cpp new file mode 100644 index 000000000..4fc6884f9 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_operation_wait.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_operation_internal.h" +#include "host_completion.h" + +namespace crowdb::tree +{ +Status FlushOperation::wait() +{ + FlushResult report; + auto status = wait_report(false, &report); + if (!status.ok()) { + return status; + } + return report.handoff_status.ok() ? std::move(report.maintenance_status) : std::move(report.handoff_status); +} + +Status FlushOperation::wait_capture(FlushResult *out) +{ + return wait_report(true, out); +} + +Status FlushOperation::wait_report(bool capture_only, FlushResult *out) +{ + if (out == nullptr) { + return Status::invalid_argument("flush host wait requires a report output"); + } + if (detail::FlushRuntime::on_worker_thread()) { + return Status::unavailable("a runtime worker cannot wait for a flush"); + } + + struct Completion + { + detail::HostCompletion done; + Status status; + FlushResult report; + } completion; + + auto callback = [](void *context, detail::CallbackSignal signal, Status status, FlushResult report) { + auto &result = *static_cast<Completion *>(context); + if (signal == detail::CallbackSignal::kValue) { + result.report = std::move(report); + } + else { + result.status = + signal == detail::CallbackSignal::kStopped ? Status::unavailable("flush stopped") : std::move(status); + } + result.done.signal(); + }; + auto state = state_; + while (!state->start(nullptr, &completion, callback, capture_only, false)) { + auto observed = state->lifecycle.load(std::memory_order_acquire); + while ((observed & State::kRunning) != 0) { + state->lifecycle.wait(observed, std::memory_order_acquire); + observed = state->lifecycle.load(std::memory_order_acquire); + } + } + completion.done.wait(); + *out = std::move(completion.report); + return completion.status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_range_join.cpp b/lib/crowdb-tree/src/btree/flush_range_join.cpp new file mode 100644 index 000000000..15048b9d4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_range_join.cpp @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "flush_runtime.h" + +#include <cassert> +#include <stdexcept> +#include <vector> + +namespace crowdb::tree::detail +{ +struct BoundedRangeJoin::State : std::enable_shared_from_this<State> +{ + struct Slot + { + // Low bits: idle=0, active=1, completing=2, complete=3. + std::atomic<uint64_t> phase{0}; + Status status; + }; + + State(size_t count, size_t width, StartChild work, const std::shared_ptr<AllocationBudget> &budget) + : ranges(count), + slots(width, MetadataAllocator<Slot>(budget)), + start_child(std::move(work)) + { + } + + void drive() noexcept + { + if (notifications.fetch_add(1, std::memory_order_acq_rel) != 0) { + return; + } + auto keep_alive = weak_from_this().lock(); + assert(keep_alive); + do { + advance(); + } while (notifications.fetch_sub(1, std::memory_order_acq_rel) != 1); + } + + void advance() noexcept + { + if (!started.load(std::memory_order_acquire) || terminal) { + return; + } + for (size_t i = 0; i < slots.size(); ++i) { + auto &slot = slots[i]; + auto phase = slot.phase.load(std::memory_order_acquire); + if ((phase & 3) == 3) { + --pending; + if (first_failure.load(std::memory_order_acquire) == i + 1) { + error = std::move(slot.status); + } + phase = (phase & ~uint64_t{3}) + 4; + slot.phase.store(phase, std::memory_order_release); + } + if ((phase & 3) != 0 || next == ranges || first_failure.load(std::memory_order_acquire) != 0 || + stopped.load(std::memory_order_acquire)) { + continue; + } + ++pending; + slot.phase.store(phase | 1, std::memory_order_release); + Completion completion(weak_from_this().lock(), i, phase); + try { + const auto index = next++; + if (start_function != nullptr) { + start_function(child_context, index, completion); + } + else { + start_child(index, completion); + } + } + catch (...) { + completion(Status::internal_error()); + } + } + bool sealed = next == ranges || first_failure.load(std::memory_order_acquire) != 0 || + stopped.load(std::memory_order_acquire); + if (sealed && pending == 0) { + terminal = true; + auto signal = CallbackSignal::kValue; + if (!error.ok()) { + signal = CallbackSignal::kError; + } + else if (stopped.load(std::memory_order_acquire)) { + signal = CallbackSignal::kStopped; + } + complete(context, signal, std::move(error)); + } + } + + size_t ranges; + std::vector<Slot, MetadataAllocator<Slot>> slots; + StartChild start_child; + StartFunction start_function = nullptr; + void *child_context = nullptr; + std::atomic<size_t> notifications{0}; + std::atomic<bool> stopped{false}; + std::atomic<size_t> first_failure{0}; + std::atomic<bool> started{false}; + std::atomic_flag submitted = ATOMIC_FLAG_INIT; + CallbackComplete complete = nullptr; + void *context = nullptr; + size_t next = 0; + size_t pending = 0; + bool terminal = false; + Status error; +}; + +void BoundedRangeJoin::Completion::operator()(Status status) const noexcept +{ + auto &slot = state_->slots[slot_]; + auto expected = generation_ | 1; + if (!slot.phase.compare_exchange_strong(expected, generation_ | 2, std::memory_order_acq_rel)) { + return; + } + slot.status = std::move(status); + if (!slot.status.ok()) { + size_t first = 0; + state_->first_failure.compare_exchange_strong(first, slot_ + 1, std::memory_order_acq_rel); + } + slot.phase.store(generation_ | 3, std::memory_order_release); + state_->drive(); +} + +std::optional<size_t> BoundedRangeJoin::allocation_bound(size_t parallelism) noexcept +{ + return AllocationBound(sizeof(BoundedRangeJoin)) + .add(shared_allocation_bound<State>()) + .add_product(parallelism, sizeof(State::Slot)) + .value(); +} + +BoundedRangeJoin::BoundedRangeJoin(size_t ranges, size_t parallelism, StartChild start_child, + std::shared_ptr<AllocationBudget> budget) +{ + if (parallelism == 0 || !start_child) { + throw std::invalid_argument("range join requires operation slots and child starter"); + } + state_ = std::allocate_shared<State>(MetadataAllocator<State>(budget), ranges, parallelism, std::move(start_child), + budget); +} + +BoundedRangeJoin::BoundedRangeJoin(size_t ranges, size_t parallelism, void *context, StartFunction start_child, + std::shared_ptr<AllocationBudget> budget) +{ + if (parallelism == 0 || start_child == nullptr) { + throw std::invalid_argument("range join requires operation slots and child starter"); + } + state_ = std::allocate_shared<State>(MetadataAllocator<State>(budget), ranges, parallelism, StartChild{}, budget); + state_->child_context = context; + state_->start_function = start_child; +} + +BoundedRangeJoin::~BoundedRangeJoin() = default; + +CallbackSender BoundedRangeJoin::sender() noexcept +{ + return {this, &BoundedRangeJoin::submit}; +} + +void BoundedRangeJoin::submit(void *source, CallbackComplete complete, void *context) +{ + auto state = static_cast<BoundedRangeJoin *>(source)->state_; + if (state->submitted.test_and_set(std::memory_order_acq_rel)) { + complete(context, CallbackSignal::kError, Status::invalid_argument("range join already started")); + return; + } + state->complete = complete; + state->context = context; + state->started.store(true, std::memory_order_release); + state->drive(); +} + +void BoundedRangeJoin::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->drive(); +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/flush_report_sender.h b/lib/crowdb-tree/src/btree/flush_report_sender.h new file mode 100644 index 000000000..97d6ae747 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_report_sender.h @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "flush_batch.h" + +namespace crowdb::tree +{ +using FlushReportComplete = void (*)(void *, detail::CallbackSignal, Status, FlushResult); +using FlushReportSubmit = void (*)(void *, FlushReportComplete, void *); + +// The completed attempt owns its report independently of subsequent retries. +// Connecting adds no heap allocation; the caller owns the connected state. +class FlushReportSender +{ + public: + using sender_concept = stdexec::sender_tag; + using completion_signatures = + stdexec::completion_signatures<stdexec::set_value_t(FlushResult), stdexec::set_error_t(Status), + stdexec::set_stopped_t()>; + + FlushReportSender(void *source, FlushReportSubmit submit) : source_(source), submit_(submit) + { + } + + template <class Receiver> struct Operation + { + Operation(Receiver receiver, void *source, FlushReportSubmit submit) + : receiver_(std::move(receiver)), + source_(source), + submit_(submit) + { + } + + Operation(const Operation &) = delete; + Operation &operator=(const Operation &) = delete; + Operation(Operation &&) = delete; + Operation &operator=(Operation &&) = delete; + + void start() & noexcept + { + submit_(source_, &Operation::complete, this); + } + + private: + static void complete(void *context, detail::CallbackSignal signal, Status status, FlushResult report) noexcept + { + auto *self = static_cast<Operation *>(context); + if (self->terminal_.test_and_set(std::memory_order_acq_rel)) { + return; + } + switch (signal) { + case detail::CallbackSignal::kValue: + stdexec::set_value(std::move(self->receiver_), std::move(report)); + break; + case detail::CallbackSignal::kError: + stdexec::set_error(std::move(self->receiver_), std::move(status)); + break; + case detail::CallbackSignal::kStopped: + stdexec::set_stopped(std::move(self->receiver_)); + break; + } + } + + Receiver receiver_; + void *source_; + FlushReportSubmit submit_; + std::atomic_flag terminal_ = ATOMIC_FLAG_INIT; + }; + + template <class Receiver> [[nodiscard]] auto connect(Receiver receiver) const -> Operation<Receiver> + { + return {std::move(receiver), source_, submit_}; + } + + private: + void *source_; + FlushReportSubmit submit_; +}; + +static_assert(stdexec::sender<FlushReportSender>); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_reservation.cpp b/lib/crowdb-tree/src/btree/flush_reservation.cpp new file mode 100644 index 000000000..b68d0cb25 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_reservation.cpp @@ -0,0 +1,163 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "flush_reservation_internal.h" +#include "page_cache.h" +#include "publication_lane.h" + +namespace crowdb::tree::detail +{ +Status ReclaimAdmission::acquire_window(const MappingView &mapping, MaintenancePublicationOptions *publication, + std::shared_ptr<AllocationReservation> *out) const +{ + if (publication == nullptr || out == nullptr || *out) { + return Status::invalid_argument("maintenance window requires fresh outputs"); + } + auto selected = options; + limit_page_maintenance_window(mapping, 0, ResourceBudget::available(account, BudgetClass::kReclaim), + ResourceBudget::available(account, BudgetClass::kRouting), + publisher->tracks_cache_residency(), &selected); + const auto snapshot = page_maintenance_snapshot_reserve_bound(mapping, selected); + const auto reclaim = + page_maintenance_reserve_bound(mapping, 0, selected, publisher->tracks_cache_residency(), true); + if (!snapshot || !reclaim) { + return Status::resource_exhausted("maintenance window reserve bound overflow"); + } + auto snapshot_credit = ResourceBudget::try_reserve(account, BudgetClass::kRouting, *snapshot); + if (!snapshot_credit) { + return Status::unavailable("maintenance snapshot reserve unavailable"); + } + auto reclaim_credit = ResourceBudget::try_reserve(account, BudgetClass::kReclaim, *reclaim); + if (!reclaim_credit) { + return Status::unavailable("maintenance reclaim reserve unavailable"); + } + auto snapshot_owner = AllocationReservation::create(std::move(*snapshot_credit)); + auto reclaim_owner = AllocationReservation::create(std::move(*reclaim_credit)); + publication->pages.max_output_pages = selected.publication.pages.max_output_pages; + publication->snapshot_metadata = std::move(snapshot_owner); + *out = std::move(reclaim_owner); + return Status::Ok(); +} + +Status bind_flush_reservations(PublicationCoordinator &publisher, const FlushPreparationOptions &preparation, + const std::shared_ptr<ResourceBudget::Account> &account, + std::shared_ptr<FlushRuntime::Lane> lane, const FlushReserveBounds &bounds, + FlushCredits credits, FlushBatchOptions *batch, FlushBatch::Maintenance *maintenance, + FlushReservations *out) +{ + if (!preparation.runtime || !preparation.store || !lane || out == nullptr || !account || batch == nullptr || + batch->loader || maintenance == nullptr || *maintenance || bounds.snapshot_bytes == 0 || + bounds.maintenance_output_pages < 2 || + bounds.maintenance_output_pages > preparation.maintenance.publication.pages.max_output_pages) { + return Status::invalid_argument("flush binding requires fresh consumers and owned progress grants"); + } + if (preparation.publication && preparation.publication->capacity() <= batch->parallelism) { + return Status::invalid_argument("flush publication window is smaller than the parallel handoff"); + } + for (size_t index = 0; index < credits.size(); ++index) { + if (!credits[index].belongs_to(account, FlushReserveBounds::categories[index]) || credits[index].allocated() || + credits[index].amount() < bounds.amounts[index]) { + return Status::invalid_argument("flush binding requires each complete unused partition grant"); + } + } + auto backend_credit = credits[1].split(bounds.backend_bytes); + if (!backend_credit) { + return Status::invalid_argument("flush routing grant lacks backend capacity"); + } + auto snapshot_credit = credits[1].split(bounds.snapshot_bytes); + if (!snapshot_credit) { + return Status::invalid_argument("flush routing grant lacks snapshot selection capacity"); + } + FlushReservations reserved; + reserved.cache = preparation.cache; + reserved.handoff = AllocationReservation::create(std::move(credits[0])); + reserved.routing = reserved_shared<AllocationReservation>(reserved.handoff, std::move(credits[1])); + reserved.reclaim = reserved_shared<AllocationReservation>(reserved.handoff, std::move(credits[2])); + reserved.io = reserved_shared<AllocationReservation>(reserved.handoff, std::move(credits[3])); + reserved.snapshot = AllocationReservation::create(std::move(*snapshot_credit)); + auto backend_status = bind_backend_operation(preparation.store.get(), + {.max_io_bytes = preparation.loads.max_blob_bytes, + .write_calls = 0, + .workers = preparation.runtime->worker_count(), + .barriers = false}, + std::move(*backend_credit), &reserved.backend); + if (!backend_status.ok()) { + return backend_status; + } + batch->cpu_progress = preparation.cpu_progress; + batch->publication = preparation.publication; + auto loads = preparation.loads; + loads.max_inflight = 1; + batch->loader = reserved_shared<AsyncStorePageLoader>(reserved.routing, preparation.store, preparation.runtime, + lane, loads, OverflowBackingResolver{}, + PageLoadBudget{.bytes = reserved.routing, + .io = reserved.io, + .work_metrics = publisher.work_metrics(), + .backend = reserved.backend, + .cpu_progress = preparation.cpu_progress, + .cause = TreeWorkCause::Flush}); + // Maintenance runs after handoff metadata is sealed. Its bounded decoded + // bases use Routing directly instead of allocating new cache waiters there. + auto maintenance_loader = batch->loader; + if (preparation.cache) { + std::shared_ptr<PageLoader> binding; + auto status = + preparation.cache->bind_reserved(batch->loader, reserved.handoff, &binding, PageLoadPriority::Maintenance); + if (!status.ok()) { + return status; + } + batch->loader = std::move(binding); + } + auto maintenance_options = preparation.maintenance; + maintenance_options.publication.pages.max_output_pages = bounds.maintenance_output_pages; + auto fallback = reserved_shared<ReclaimAdmission>( + reserved.handoff, + ReclaimAdmission{.publisher = &publisher, .account = account, .options = preparation.maintenance}); + maintenance_options.prepaid = reserved.reclaim; + maintenance_options.publication.snapshot_metadata = reserved.snapshot; + maintenance_options.acquire_window = [fallback](const auto &mapping, auto *publication, auto *credit) { + return fallback->acquire_window(mapping, publication, credit); + }; + *maintenance = make_page_maintenance(publisher, *preparation.runtime, lane, std::move(maintenance_options), + std::move(maintenance_loader), preparation.cpu_progress); + *out = std::move(reserved); + return Status::Ok(); +} + +Status prepare_flush_reservations(PublicationCoordinator &publisher, const PublishedTreeState &view, + const SourceCatalog &sources, const FlushPreparationOptions &preparation, + size_t control_bytes, std::shared_ptr<FlushRuntime::Lane> lane, + FlushBatchOptions *batch, FlushBatch::Maintenance *maintenance, + FlushReservations *out, std::optional<WriteFootprint> stable_active) +{ + if (!lane || batch == nullptr || maintenance == nullptr || *maintenance || out == nullptr) { + return Status::invalid_argument("flush preparation requires fresh owned progress consumers"); + } + FlushReserveBounds bounds; + auto status = + flush_reserve_bounds(publisher, view, sources, preparation, control_bytes, *batch, &bounds, stable_active); + if (!status.ok()) { + return status; + } + FlushCredits credits; + constexpr std::array names{"handoff", "routing", "reclaim", "I/O"}; + for (size_t index = 0; index < credits.size(); ++index) { + const auto capacity = + ResourceBudget::capacity(sources.active_budget->options().account, FlushReserveBounds::categories[index]); + if (bounds.amounts[index] > capacity) { + return Status::resource_exhausted( + "flush " + std::string(names[index]) + " bound exceeds configured capacity: required " + + std::to_string(bounds.amounts[index]) + " limit " + std::to_string(capacity)); + } + auto credit = ResourceBudget::try_reserve(sources.active_budget->options().account, + FlushReserveBounds::categories[index], bounds.amounts[index]); + if (!credit) { + return Status::unavailable("flush " + std::string(names[index]) + " reserve unavailable: required " + + std::to_string(bounds.amounts[index])); + } + credits[index] = std::move(*credit); + } + return bind_flush_reservations(publisher, preparation, sources.active_budget->options().account, std::move(lane), + bounds, std::move(credits), batch, maintenance, out); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/flush_reservation.h b/lib/crowdb-tree/src/btree/flush_reservation.h new file mode 100644 index 000000000..bb77540c4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_reservation.h @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/local/block_account.h" +#include "maintenance_operation.h" +#include "page_load.h" +#include "source_admission.h" + +namespace crowdb::tree +{ +class SharedPageCache; + +struct FlushPreparationOptions +{ + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<AsyncPageStore> store; + PageLoadLimits loads; + PageMaintenanceOperationOptions maintenance; + std::shared_ptr<SharedPageCache> cache = nullptr; + std::shared_ptr<ApplyDispatcher> cpu_progress = nullptr; + std::shared_ptr<PublicationLane> publication = nullptr; +}; + +namespace detail +{ +struct FlushReserveBounds +{ + static constexpr std::array categories{BudgetClass::kHandoff, BudgetClass::kRouting, BudgetClass::kReclaim, + BudgetClass::kIo}; + std::array<size_t, categories.size()> amounts{}; + size_t backend_bytes = 0; + size_t snapshot_bytes = 0; + size_t maintenance_output_pages = 0; +}; + +using FlushCredits = std::array<ResourceBudget::Credit, FlushReserveBounds::categories.size()>; + +struct FlushReservations +{ + std::shared_ptr<AllocationBudget> backend; + std::shared_ptr<SharedPageCache> cache; + std::shared_ptr<AllocationReservation> handoff; + std::shared_ptr<AllocationReservation> routing; + std::shared_ptr<AllocationReservation> reclaim; + std::shared_ptr<AllocationReservation> io; + std::shared_ptr<AllocationReservation> snapshot; + + void seal() const noexcept + { + seal_backend_budget(backend); + for (const auto &credit : {handoff, routing, reclaim, io, snapshot}) { + if (credit) { + credit->seal(); + } + } + } +}; + +// Derive the complete fixed-state bound without acquiring any resources. +Status flush_reserve_bounds(PublicationCoordinator &publisher, const PublishedTreeState &view, + const SourceCatalog &sources, const FlushPreparationOptions &preparation, + size_t control_bytes, const FlushBatchOptions &batch, FlushReserveBounds *out, + std::optional<WriteFootprint> stable_active = {}, bool add_successor = false); +// The caller transfers complete grants directly from asynchronous admission; +// loader, backend and maintenance construction never acquire the same credit again. +Status bind_flush_reservations(PublicationCoordinator &publisher, const FlushPreparationOptions &preparation, + const std::shared_ptr<ResourceBudget::Account> &account, + std::shared_ptr<FlushRuntime::Lane> lane, const FlushReserveBounds &bounds, + FlushCredits credits, FlushBatchOptions *batch, FlushBatch::Maintenance *maintenance, + FlushReservations *out); + +// All-or-rollback admission: no source is closed and no publication gate is +// taken while these independent partitions are being acquired/prepared. +Status prepare_flush_reservations(PublicationCoordinator &publisher, const PublishedTreeState &view, + const SourceCatalog &sources, const FlushPreparationOptions &preparation, + size_t control_bytes, std::shared_ptr<FlushRuntime::Lane> lane, + FlushBatchOptions *batch, FlushBatch::Maintenance *maintenance, + FlushReservations *out, std::optional<WriteFootprint> stable_active = {}); +} // namespace detail +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_reservation_internal.h b/lib/crowdb-tree/src/btree/flush_reservation_internal.h new file mode 100644 index 000000000..6d80938ea --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_reservation_internal.h @@ -0,0 +1,16 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "flush_reservation.h" + +namespace crowdb::tree::detail +{ +struct ReclaimAdmission +{ + PublicationCoordinator *publisher; + std::shared_ptr<ResourceBudget::Account> account; + PageMaintenanceOperationOptions options; + Status acquire_window(const MappingView &mapping, MaintenancePublicationOptions *publication, + std::shared_ptr<AllocationReservation> *out) const; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp b/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp new file mode 100644 index 000000000..1300defb2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp @@ -0,0 +1,148 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "flush_reservation_internal.h" +#include "page_cache.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +Status flush_reserve_bounds(PublicationCoordinator &publisher, const PublishedTreeState &view, + const SourceCatalog &sources, const FlushPreparationOptions &preparation, + size_t control_bytes, const FlushBatchOptions &batch, FlushReserveBounds *out, + std::optional<WriteFootprint> stable_active, bool add_successor) +{ + if (!preparation.runtime || !preparation.store || !sources.active_budget || out == nullptr || batch.loader || + preparation.maintenance.prepaid || preparation.maintenance.acquire || + preparation.maintenance.acquire_snapshot || preparation.maintenance.acquire_window || + preparation.maintenance.publication.metadata) { + return Status::invalid_argument("automatic flush requires owned runtime/store and unbound progress " + "consumers"); + } + if (sources.floor != view.coverage || !view.sources || view.sources->floor != sources.floor || + sources.tables.size() != view.sources->tables.size() || + !std::equal(sources.tables.begin(), sources.tables.end(), view.sources->tables.begin())) { + return Status::unavailable("source catalog differs from the reserved mapping view"); + } + const auto root = view.mapping.get(view.root_page_id); + if (!root || root->level > batch.max_route_depth) { + return Status::invalid_argument("automatic flush needs an initialized bounded routing tree"); + } + const auto &source_limits = sources.active_budget->options(); + const auto per_source_entries = + AllocationBound(source_limits.max_entries).add(source_limits.max_gap_entries).value(); + const auto per_source_keys = + AllocationBound(source_limits.max_key_bytes).add(source_limits.max_gap_key_bytes).value(); + if (!per_source_entries || !per_source_keys) { + return Status::resource_exhausted("source admission bound overflow"); + } + AllocationBound entry_bound; + AllocationBound key_bound; + for (const auto &table : sources.tables) { + if (!table) { + return Status::invalid_argument("source catalog contains an absent table"); + } + // Closed, drained sources cannot grow. Each linked key contributes at + // most one frontier-selected version; node storage includes its key. + // An accepted writer still requires the full configured allowance. + const auto summary = table->source_stats(); + const auto sealed = table->sealed_admission_bound(); + if (stable_active && table == sources.active) { + entry_bound.add(stable_active->entries); + key_bound.add(stable_active->key_bytes); + } + else if (summary.known && summary.frozen_tables != 0 && summary.writers == 0) { + entry_bound.add(summary.records); + key_bound.add(summary.memory.node_bytes); + } + else if (sealed) { + entry_bound.add(sealed->entries); + key_bound.add(sealed->key_bytes); + } + else { + entry_bound.add(*per_source_entries); + key_bound.add(*per_source_keys); + } + } + // Rotation seals the current source and installs one still-open successor. + // The latter keeps its complete future ordinary and gap allowance. + const auto source_count = sources.tables.size() + (add_successor ? 1 : 0); + if (add_successor) { + entry_bound.add(*per_source_entries); + key_bound.add(*per_source_keys); + } + const auto entries = entry_bound.value(); + const auto keys = key_bound.value(); + if (!entries || !keys) { + return Status::resource_exhausted("captured source bound overflow"); + } + const auto workers = preparation.runtime->worker_count(); + auto handoff = AllocationBound(control_bytes) + .add(FlushBatch::metadata_reserve_bound(view.mapping, source_count, *entries, *keys, workers, + batch, publisher.tracks_cache_residency())) + .add(SourceAdmission::capture_allocation_bound(source_count)) + .add_product(2, PublishedSources::allocation_bound()) + .add_product(source_count, 2 * sizeof(std::shared_ptr<MemTable>)) + .add(2 * sizeof(std::shared_ptr<MemTable>)) + .add_product(4, shared_allocation_bound<AllocationReservation>()) + .add(shared_allocation_bound<ReclaimAdmission>()) + .add(preparation.cache ? SharedPageCache::binding_allocation_bound() : 0) + .add(preparation.cache ? preparation.cache->load_reserve_bound(root->level, workers) + : std::optional<size_t>{0}) + .value(); + // Handoff adds overlays without growing the captured base arrays. The + // Analyze may split bases and add resident inners. Inner underflow repair + // retains one captured route and both adjacent siblings at each level. + // Maintenance-created inners already own resident routes. + // Concurrent checkpoint completion can replace a leaf with one + // cold base, so an initially empty array still needs that single allowance. + const auto cold_bases = std::min(preparation.maintenance.publication.pages.max_sources, + std::max(size_t{1}, view.mapping.copy_bounds().max_leaf_bases)); + const auto path_depth = + std::min(static_cast<size_t>(root->level), preparation.maintenance.publication.max_path_pages); + const auto cold_depth = + path_depth + + (root->level < 2 ? 0 : static_cast<size_t>(std::min<uint64_t>(view.stats.inner_pages, 3 * path_depth))); + // Only captured inner pages can become cold cache entries during this + // operation. Maintenance-created inner pages already own resident routes. + const auto routing = + AllocationBound() + .add(AsyncStorePageLoader::routing_reserve_bound( + preparation.loads, root->level, workers, + preparation.cache ? std::min<uint64_t>(preparation.cache->retained_page_limit(), view.stats.inner_pages) + : 0)) + .add(AsyncStorePageLoader::primary_reserve_bound(preparation.loads, cold_depth, 1, cold_bases)) + .value(); + const auto backend = backend_operation_bound( + preparation.store.get(), + {.max_io_bytes = preparation.loads.max_blob_bytes, .write_calls = 0, .workers = workers, .barriers = false}); + auto maintenance = preparation.maintenance; + if (routing && backend) { + const auto account = sources.active_budget->options().account; + const auto reclaim_limit = add_successor ? ResourceBudget::capacity(account, BudgetClass::kReclaim) + : ResourceBudget::available(account, BudgetClass::kReclaim); + const auto routing_limit = add_successor ? ResourceBudget::capacity(account, BudgetClass::kRouting) + : ResourceBudget::available(account, BudgetClass::kRouting); + const auto fixed = AllocationBound(*routing).add(*backend).value(); + limit_page_maintenance_window(view.mapping, *entries, reclaim_limit, + fixed && *fixed < routing_limit ? routing_limit - *fixed : 0, + publisher.tracks_cache_residency(), &maintenance); + } + const auto reclaim = + page_maintenance_reserve_bound(view.mapping, *entries, maintenance, publisher.tracks_cache_residency(), true); + const auto snapshot = page_maintenance_snapshot_reserve_bound(view.mapping, maintenance); + const auto total_routing = + routing && backend && snapshot ? AllocationBound(*routing).add(*backend).add(*snapshot).value() : std::nullopt; + if (!handoff || !total_routing || !reclaim || !snapshot) { + return Status::resource_exhausted("flush reserve bounds exceed the configured work limits"); + } + *out = { + .amounts = {*handoff, *total_routing, *reclaim, size_t{1}}, + .backend_bytes = *backend, + .snapshot_bytes = *snapshot, + .maintenance_output_pages = maintenance.publication.pages.max_output_pages + }; + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/flush_runtime.cpp b/lib/crowdb-tree/src/btree/flush_runtime.cpp new file mode 100644 index 000000000..b9ca58181 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_runtime.cpp @@ -0,0 +1,185 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "allocation_bound.h" +#include "flush_runtime_internal.h" + +#include <cassert> +#include <stdexcept> + +namespace crowdb::tree::detail +{ +namespace +{ +thread_local bool executing_runtime_job = false; + +struct RuntimeWorkerScope +{ + RuntimeWorkerScope() : previous(executing_runtime_job) + { + executing_runtime_job = true; + } + + ~RuntimeWorkerScope() + { + executing_runtime_job = previous; + } + + bool previous; +}; +} // namespace + +bool FlushRuntime::on_worker_thread() noexcept +{ + return executing_runtime_job; +} + +void FlushRuntime::State::Receiver::set_value() const noexcept +{ + RuntimeWorkerScope worker_scope; + std::unique_ptr<Job, decltype(&State::release_job)> owned(job, &State::release_job); + auto *state = owned->state; + auto started = flush_runtime_now_ns(); + state->inflight.fetch_add(1, std::memory_order_relaxed); + state->queued.fetch_sub(1, std::memory_order_relaxed); + if (!owned->progress) { + owned->lane->queued_.fetch_sub(1, std::memory_order_relaxed); + state->queue_permits.fetch_sub(1, std::memory_order_relaxed); + } + else { + owned->progress->requested.exchange(false, std::memory_order_acq_rel); + } + state->queue_ns.fetch_add(started - owned->queued_at, std::memory_order_relaxed); + owned->task(owned->context); + auto progress = owned->progress; + auto owner = std::move(owned->context_owner); + auto lane = owned->lane; + owned.reset(); + if (progress) { + progress->complete(owner); + } + owner.reset(); + progress.reset(); + // Ownership cleanup still occupies this worker. Count it before publishing + // completion, otherwise resource samples hide expensive retirement tails. + state->execute_ns.fetch_add(flush_runtime_now_ns() - started, std::memory_order_relaxed); + state->completed.fetch_add(1, std::memory_order_relaxed); + state->inflight.fetch_sub(1, std::memory_order_relaxed); + state->finish(); + lane->jobs_.fetch_sub(1, std::memory_order_acq_rel); + lane->jobs_.notify_all(); +} + +void FlushRuntime::Lane::drain() noexcept +{ + auto count = jobs_.load(std::memory_order_acquire); + while (count != 0) { + jobs_.wait(count, std::memory_order_acquire); + count = jobs_.load(std::memory_order_acquire); + } +} + +std::optional<size_t> FlushRuntime::allocation_bound(size_t workers, size_t queue_limit) noexcept +{ + if (workers == 0 || workers > UINT16_MAX || queue_limit == 0 || workers > SIZE_MAX - queue_limit) { + return {}; + } + return AllocationBound(sizeof(State)) + .add(exec::static_thread_pool::allocation_bound(static_cast<uint32_t>(workers))) + .add_product(workers, PoolStorage::worker_bytes()) + .add_product(workers + queue_limit, sizeof(State::Slot)) + .add_product(workers, sizeof(exec::static_thread_pool::scheduler)) + .add(shared_allocation_bound<std::atomic<uint64_t>>()) + .value(); +} + +FlushRuntime::FlushRuntime(size_t workers, size_t queue_limit, std::shared_ptr<AllocationBudget> budget) +{ + if (!allocation_bound(workers, queue_limit)) { + throw std::invalid_argument("flush runtime requires workers and queue capacity"); + } + state_credit_ = AllocationLease(budget, sizeof(State), AllocationKind::kMetadata); + state_ = std::make_unique<State>(workers, queue_limit, budget); +} + +FlushRuntime::~FlushRuntime() +{ + close(); + drain(); + assert(state_->queue_permits.load(std::memory_order_acquire) == 0 && + "host must release progress registrations before runtime destruction"); +} + +Status FlushRuntime::try_submit(std::shared_ptr<Lane> lane, Task task, void *context) +{ + if (!lane || task == nullptr) { + return Status::invalid_argument("flush runtime requires a lane and task"); + } + auto value = state_->lifecycle->load(std::memory_order_acquire); + do { + if ((value & State::kClosed) != 0) { + state_->rejected.fetch_add(1, std::memory_order_relaxed); + return Status::unavailable("flush runtime closed"); + } + } while (!state_->lifecycle->compare_exchange_weak(value, value + 1, std::memory_order_acq_rel)); + + bool lane_acquired = flush_runtime_acquire(lane->queued_, lane->limit_); + if (!lane_acquired || !flush_runtime_acquire(state_->queue_permits, state_->queue_limit)) { + if (lane_acquired) { + lane->queued_.fetch_sub(1, std::memory_order_relaxed); + } + state_->rejected.fetch_add(1, std::memory_order_relaxed); + state_->finish(); + return Status::resource_exhausted("flush runtime queue full"); + } + auto *job = state_->prepare_job(lane, task, context); + if (job == nullptr) { + state_->queue_permits.fetch_sub(1, std::memory_order_relaxed); + lane->queued_.fetch_sub(1, std::memory_order_relaxed); + state_->rejected.fetch_add(1, std::memory_order_relaxed); + state_->finish(); + return Status::resource_exhausted("flush runtime job slots unavailable"); + } + state_->queued.fetch_add(1, std::memory_order_relaxed); + state_->submitted.fetch_add(1, std::memory_order_relaxed); + lane->jobs_.fetch_add(1, std::memory_order_acq_rel); + stdexec::start(job->operation); + return Status::Ok(); +} + +void FlushRuntime::close() noexcept +{ + state_->lifecycle->fetch_or(State::kClosed, std::memory_order_acq_rel); +} + +void FlushRuntime::drain() noexcept +{ + auto value = state_->lifecycle->load(std::memory_order_acquire); + while ((value & ~State::kClosed) != 0) { + state_->lifecycle->wait(value, std::memory_order_acquire); + value = state_->lifecycle->load(std::memory_order_acquire); + } +} + +FlushRuntime::Snapshot FlushRuntime::snapshot() const noexcept +{ + return {.queued = state_->queued.load(std::memory_order_relaxed), + .inflight = state_->inflight.load(std::memory_order_relaxed), + .submitted = state_->submitted.load(std::memory_order_relaxed), + .completed = state_->completed.load(std::memory_order_relaxed), + .rejected = state_->rejected.load(std::memory_order_relaxed), + .queue_ns = state_->queue_ns.load(std::memory_order_relaxed), + .execute_ns = state_->execute_ns.load(std::memory_order_relaxed)}; +} + +size_t FlushRuntime::worker_count() const noexcept +{ + return state_->worker_count; +} + +size_t FlushRuntime::queue_permits() const noexcept +{ + return state_->queue_permits.load(std::memory_order_relaxed); +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/flush_runtime.h b/lib/crowdb-tree/src/btree/flush_runtime.h new file mode 100644 index 000000000..f4df20509 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_runtime.h @@ -0,0 +1,165 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-tree/memory_budget.h" +#include "stdexec_adapter.h" + +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <memory> +#include <optional> + +namespace crowdb::tree::detail +{ + +// CPU steps must return instead of waiting for children or I/O. Closing and +// destroying the runtime belongs to the host, never one of its workers. +class FlushRuntime +{ + struct ProgressState; + + public: + struct Snapshot + { + uint64_t queued; + uint64_t inflight; + uint64_t submitted; + uint64_t completed; + uint64_t rejected; + uint64_t queue_ns; + uint64_t execute_ns; + }; + + class Lane + { + public: + explicit Lane(size_t queue_limit) : limit_(queue_limit) + { + } + + // Host-thread boundary after producers have joined. Includes the + // scheduler tail after user completion and context-owner release. + void drain() noexcept; + + [[nodiscard]] size_t queue_permits() const noexcept + { + return queued_.load(std::memory_order_relaxed); + } + + private: + friend class FlushRuntime; + size_t limit_; + std::atomic<size_t> queued_{0}; + std::atomic<size_t> jobs_{0}; + }; + + using Task = void (*)(void *) noexcept; + + class Progress + { + public: + explicit Progress(std::shared_ptr<ProgressState> state); + // Every request for this registration uses the same context owner. + // Coalesced requests retain the currently queued/running owner. + void request(std::shared_ptr<const void> owner) noexcept; + // Host-thread only. Drain this registration without waiting for other + // Trees. Producers must stop before drain can be used as a boundary. + void drain() noexcept; + // Reject late requests, drain accepted work and return the fixed slot. + // The closed handle may outlive the runtime. Do not call from its task. + void close_and_drain() noexcept; + + private: + std::shared_ptr<ProgressState> state_; + }; + + FlushRuntime(size_t workers, size_t queue_limit, std::shared_ptr<AllocationBudget> budget = {}); + ~FlushRuntime(); + FlushRuntime(const FlushRuntime &) = delete; + FlushRuntime &operator=(const FlushRuntime &) = delete; + + // No callback on rejection. The context must live through task return. + // A lane caps queued work from one Tree; executing steps release that + // credit before starting, so a single-worker nested submission progresses. + [[nodiscard]] Status try_submit(std::shared_ptr<Lane> lane, Task task, void *context); + // Registration reserves one real job slot and one lane/global queue + // credit. Requests never allocate or fail because ordinary queues fill. + // Registered work remains usable after close to finish admitted work. + // The host joins producers and releases registrations before destroying + // this runtime; drain includes all queued/running progress continuations. + [[nodiscard]] Status reserve_progress(std::shared_ptr<Lane> lane, Task task, void *context, + std::shared_ptr<AllocationBudget> budget, std::shared_ptr<Progress> *out); + [[nodiscard]] static size_t progress_allocation_bound() noexcept; + // Owned state, fixed job cells and lifecycle control. The host separately + // provisions the shared executor's thread/bootstrap storage. + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t workers, size_t queue_limit) noexcept; + void close() noexcept; + void drain() noexcept; + [[nodiscard]] Snapshot snapshot() const noexcept; + [[nodiscard]] size_t worker_count() const noexcept; + [[nodiscard]] size_t queue_permits() const noexcept; + [[nodiscard]] static bool on_worker_thread() noexcept; + + private: + struct State; + AllocationLease state_credit_; + std::unique_ptr<State> state_; +}; + +// A dynamically sized range has a fixed number of reusable operation slots. +// Completion is copyable and retains the join until late transports release +// it; duplicates and stale slot generations cannot finish another child. +class BoundedRangeJoin +{ + struct State; + + public: + class Completion + { + public: + void operator()(Status status = Status::Ok()) const noexcept; + + private: + friend struct State; + + Completion(std::shared_ptr<State> state, size_t slot, uint64_t generation) + : state_(std::move(state)), + slot_(slot), + generation_(generation) + { + } + + std::shared_ptr<State> state_; + size_t slot_; + uint64_t generation_; + }; + + using StartChild = std::function<void(size_t, Completion)>; + using StartFunction = void (*)(void *, size_t, Completion); + + // start_child initiates asynchronous work or a bounded CPU step. It may + // complete inline; it must not wait. Throwing is converted to an error. + BoundedRangeJoin(size_t ranges, size_t parallelism, StartChild start_child, + std::shared_ptr<AllocationBudget> budget = {}); + // The connected receiver retains context through terminal; late duplicate + // completions never start another child or dereference this context. + BoundedRangeJoin(size_t ranges, size_t parallelism, void *context, StartFunction start_child, + std::shared_ptr<AllocationBudget> budget = {}); + ~BoundedRangeJoin(); + BoundedRangeJoin(const BoundedRangeJoin &) = delete; + BoundedRangeJoin &operator=(const BoundedRangeJoin &) = delete; + + [[nodiscard]] CallbackSender sender() noexcept; + void request_stop() noexcept; + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t parallelism) noexcept; + + private: + static void submit(void *source, CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/flush_runtime_internal.h b/lib/crowdb-tree/src/btree/flush_runtime_internal.h new file mode 100644 index 000000000..4a9e0c0d0 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_runtime_internal.h @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "flush_runtime.h" +#include "runtime/pool_storage.h" + +#include <array> +#include <chrono> +#include <exec/static_thread_pool.hpp> +#include <vector> + +namespace crowdb::tree::detail +{ +inline bool flush_runtime_acquire(std::atomic<size_t> &counter, size_t limit) noexcept +{ + auto value = counter.load(std::memory_order_relaxed); + while (value < limit) { + if (counter.compare_exchange_weak(value, value + 1, std::memory_order_acq_rel)) { + return true; + } + } + return false; +} + +inline uint64_t flush_runtime_now_ns() noexcept +{ + return static_cast<uint64_t>( + std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +struct FlushRuntime::State +{ + static constexpr uint64_t kClosed = uint64_t{1} << 63; + + struct Job; + + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + Job *job; + void set_value() const noexcept; + }; + + struct Job + { + using Operation = stdexec::connect_result_t< + decltype(stdexec::schedule(std::declval<exec::static_thread_pool &>().get_scheduler())), Receiver>; + + Job(State *owner, size_t slot_index, std::shared_ptr<Lane> tree_lane, Task work, void *argument, + std::shared_ptr<ProgressState> progress_owner = {}, + std::shared_ptr<const void> argument_owner = {}) noexcept + : state(owner), + slot(slot_index), + lane(std::move(tree_lane)), + task(work), + context(argument), + progress(std::move(progress_owner)), + context_owner(std::move(argument_owner)), + queued_at(flush_runtime_now_ns()), + operation(stdexec::connect(stdexec::schedule(state->schedulers[slot_index % state->schedulers.size()]), + Receiver{this})) + { + } + + State *state; + size_t slot; + std::shared_ptr<Lane> lane; + Task task; + void *context; + std::shared_ptr<ProgressState> progress; + std::shared_ptr<const void> context_owner; + uint64_t queued_at; + Operation operation; + }; + + struct Slot + { + std::atomic<bool> occupied{false}; + alignas(Job) std::array<std::byte, sizeof(Job)> storage; + }; + + State(size_t workers, size_t limit, const std::shared_ptr<AllocationBudget> &budget) + : pool_storage(budget), + pool(static_cast<uint32_t>(workers), {}, exec::no_numa_policy{}, &pool_storage, + pool_storage.thread_factory()), + schedulers(MetadataAllocator<exec::static_thread_pool::scheduler>(budget)), + worker_count(workers), + queue_limit(limit), + slots(workers + limit, MetadataAllocator<Slot>(budget)), + lifecycle(std::allocate_shared<std::atomic<uint64_t>>(MetadataAllocator<std::atomic<uint64_t>>(budget), 0)) + { + // Register on the bootstrap thread once. A cached target scheduler + // submits through its existing shared MPSC remote queue, without + // creating a queue for each new host or callback thread. + schedulers.reserve(workers); + for (size_t index = 0; index < workers; ++index) { + schedulers.push_back(pool.get_scheduler_on_thread(index)); + } + } + + [[nodiscard]] size_t claim_slot() noexcept + { + const auto first = next_slot.fetch_add(1, std::memory_order_relaxed) % slots.size(); + // Reserved progress and ordinary jobs share this fixed slot array. + for (size_t offset = 0; offset < slots.size(); ++offset) { + const auto index = (first + offset) % slots.size(); + bool expected = false; + if (slots[index].occupied.compare_exchange_strong(expected, true, std::memory_order_acquire)) { + return index; + } + } + return SIZE_MAX; + } + + [[nodiscard]] Job *prepare_job(const std::shared_ptr<Lane> &lane, Task task, void *context) noexcept + { + const auto index = claim_slot(); + return index == SIZE_MAX ? nullptr + : ::new (static_cast<void *>(slots[index].storage.data())) + Job(this, index, lane, task, context); + } + + static void release_job(Job *job) noexcept + { + auto *state = job->state; + const auto index = job->slot; + const bool reserved = job->progress != nullptr; + std::destroy_at(job); + if (!reserved) { + state->slots[index].occupied.store(false, std::memory_order_release); + } + } + + void finish() const noexcept + { + // Zero lets the host start teardown. Retain the notification word + // until notification returns; pool teardown joins this receiver. + auto completion = lifecycle; + completion->fetch_sub(1, std::memory_order_acq_rel); + completion->notify_all(); + } + + PoolStorage pool_storage; + exec::static_thread_pool pool; + std::vector<exec::static_thread_pool::scheduler, MetadataAllocator<exec::static_thread_pool::scheduler>> schedulers; + const size_t worker_count; + size_t queue_limit; + std::vector<Slot, MetadataAllocator<Slot>> slots; + std::atomic<size_t> next_slot{0}; + std::shared_ptr<std::atomic<uint64_t>> lifecycle; + std::atomic<size_t> queue_permits{0}; + std::atomic<size_t> queued{0}; + std::atomic<uint64_t> inflight{0}; + std::atomic<uint64_t> submitted{0}; + std::atomic<uint64_t> completed{0}; + std::atomic<uint64_t> rejected{0}; + std::atomic<uint64_t> queue_ns{0}; + std::atomic<uint64_t> execute_ns{0}; +}; + +struct FlushRuntime::ProgressState : std::enable_shared_from_this<ProgressState> +{ + ProgressState(State *runtime, size_t index, std::shared_ptr<Lane> tree_lane, Task work, void *argument) + : runtime(runtime), + slot(index), + lane(std::move(tree_lane)), + task(work), + context(argument) + { + } + + ~ProgressState(); + void request(const std::shared_ptr<const void> &owner) noexcept; + void schedule(const std::shared_ptr<const void> &owner) noexcept; + void complete(const std::shared_ptr<const void> &owner) noexcept; + void drain() noexcept; + void close_and_drain() noexcept; + void detach() noexcept; + + State *runtime; + size_t slot; + std::shared_ptr<Lane> lane; + Task task; + void *context; + std::atomic<bool> active{false}, requested{false}; + std::atomic<uint64_t> ingress{0}; + std::atomic<size_t> jobs{0}; + std::atomic<bool> detached{false}; +}; + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/flush_runtime_progress.cpp b/lib/crowdb-tree/src/btree/flush_runtime_progress.cpp new file mode 100644 index 000000000..b51fbd31e --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_runtime_progress.cpp @@ -0,0 +1,159 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "flush_runtime_internal.h" + +#include <cassert> + +namespace crowdb::tree::detail +{ +FlushRuntime::ProgressState::~ProgressState() +{ + assert(!active.load(std::memory_order_acquire)); + detach(); +} + +void FlushRuntime::ProgressState::detach() noexcept +{ + if (detached.exchange(true, std::memory_order_acq_rel)) { + return; + } + runtime->slots[slot].occupied.store(false, std::memory_order_release); + lane->queued_.fetch_sub(1, std::memory_order_relaxed); + runtime->queue_permits.fetch_sub(1, std::memory_order_relaxed); +} + +void FlushRuntime::ProgressState::schedule(const std::shared_ptr<const void> &owner) noexcept +{ + // The registration owns the cell and its queue permits for its lifetime. + // Connecting a static-pool scheduler constructs only the inline operation. + runtime->lifecycle->fetch_add(1, std::memory_order_acq_rel); + jobs.fetch_add(1, std::memory_order_acq_rel); + lane->jobs_.fetch_add(1, std::memory_order_acq_rel); + auto *job = ::new (static_cast<void *>(runtime->slots[slot].storage.data())) + State::Job(runtime, slot, lane, task, context, weak_from_this().lock(), owner); + runtime->queued.fetch_add(1, std::memory_order_relaxed); + runtime->submitted.fetch_add(1, std::memory_order_relaxed); + stdexec::start(job->operation); +} + +void FlushRuntime::ProgressState::request(const std::shared_ptr<const void> &owner) noexcept +{ + assert(owner); + auto value = ingress.load(std::memory_order_acquire); + do { + if ((value & State::kClosed) != 0) { + return; + } + } while (!ingress.compare_exchange_weak(value, value + 1, std::memory_order_acq_rel)); + requested.store(true, std::memory_order_release); + if (!active.exchange(true, std::memory_order_acq_rel)) { + schedule(owner); + } + ingress.fetch_sub(1, std::memory_order_acq_rel); + ingress.notify_all(); +} + +void FlushRuntime::ProgressState::complete(const std::shared_ptr<const void> &owner) noexcept +{ + // Pair with request's exchange: either this RMW acquires the preceding + // requested publication, or the producer observes idle and schedules it. + // A release store followed by an independent load can lose both wakes. + active.exchange(false, std::memory_order_acq_rel); + if (requested.load(std::memory_order_acquire) && !active.exchange(true, std::memory_order_acq_rel)) { + schedule(owner); + } + jobs.fetch_sub(1, std::memory_order_acq_rel); + jobs.notify_all(); +} + +void FlushRuntime::ProgressState::drain() noexcept +{ + auto value = jobs.load(std::memory_order_acquire); + while (value != 0) { + jobs.wait(value, std::memory_order_acquire); + value = jobs.load(std::memory_order_acquire); + } +} + +void FlushRuntime::ProgressState::close_and_drain() noexcept +{ + auto value = ingress.fetch_or(State::kClosed, std::memory_order_acq_rel) | State::kClosed; + while ((value & ~State::kClosed) != 0) { + ingress.wait(value, std::memory_order_acquire); + value = ingress.load(std::memory_order_acquire); + } + drain(); + detach(); +} + +FlushRuntime::Progress::Progress(std::shared_ptr<ProgressState> state) : state_(std::move(state)) +{ +} + +void FlushRuntime::Progress::request(std::shared_ptr<const void> owner) noexcept +{ + state_->request(owner); +} + +void FlushRuntime::Progress::drain() noexcept +{ + state_->drain(); +} + +void FlushRuntime::Progress::close_and_drain() noexcept +{ + state_->close_and_drain(); +} + +size_t FlushRuntime::progress_allocation_bound() noexcept +{ + return shared_allocation_bound<ProgressState>() + shared_allocation_bound<Progress>(); +} + +Status FlushRuntime::reserve_progress(std::shared_ptr<Lane> lane, Task task, void *context, + std::shared_ptr<AllocationBudget> budget, std::shared_ptr<Progress> *out) +{ + if (!lane || task == nullptr || out == nullptr) { + return Status::invalid_argument("runtime progress requires lane, task and output"); + } + auto value = state_->lifecycle->load(std::memory_order_acquire); + do { + if ((value & State::kClosed) != 0) { + return Status::unavailable("flush runtime closed"); + } + } while (!state_->lifecycle->compare_exchange_weak(value, value + 1, std::memory_order_acq_rel)); + const bool lane_acquired = flush_runtime_acquire(lane->queued_, lane->limit_); + if (!lane_acquired || !flush_runtime_acquire(state_->queue_permits, state_->queue_limit)) { + if (lane_acquired) { + lane->queued_.fetch_sub(1, std::memory_order_relaxed); + } + state_->finish(); + return Status::resource_exhausted("runtime progress queue credit unavailable"); + } + const auto slot = state_->claim_slot(); + std::shared_ptr<ProgressState> progress; + Status status = Status::Ok(); + try { + if (slot == SIZE_MAX) { + throw std::bad_alloc(); + } + progress = std::allocate_shared<ProgressState>(MetadataAllocator<ProgressState>(budget), state_.get(), slot, + lane, task, context); + *out = std::allocate_shared<Progress>(MetadataAllocator<Progress>(budget), progress); + } + catch (const std::bad_alloc &) { + if (!progress) { + if (slot != SIZE_MAX) { + state_->slots[slot].occupied.store(false, std::memory_order_release); + } + lane->queued_.fetch_sub(1, std::memory_order_relaxed); + state_->queue_permits.fetch_sub(1, std::memory_order_relaxed); + } + status = Status::resource_exhausted("runtime progress allocation rejected"); + } + progress.reset(); + state_->finish(); + return status; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/generation_replace.cpp b/lib/crowdb-tree/src/btree/generation_replace.cpp index d507ab455..20c7ba2c6 100644 --- a/lib/crowdb-tree/src/btree/generation_replace.cpp +++ b/lib/crowdb-tree/src/btree/generation_replace.cpp @@ -1,223 +1,18 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -#include "crowdb-tree/btree/delta.h" -#include "crowdb-tree/crowdb-tree.h" -#include "crowdb-tree/maptable/mapping_slot.h" -#include "native_frames.h" - -#include <algorithm> +#include "btree/tree_core.h" namespace crowdb::tree { -namespace -{ -// Owns either a private candidate or the detached previous mapping. All -// retirement bookkeeping exists before the mapping publication can succeed. -struct MappingGeneration : EpochManager::Deferred -{ - MappingTable mapping; - - MappingGeneration() - : EpochManager::Deferred{ - .destroy = [](EpochManager::Deferred *entry) noexcept { delete static_cast<MappingGeneration *>(entry); }} - { - } - - ~MappingGeneration() - { - for (uint64_t index = 0; index < MappingTable::kMaxSegments; ++index) { - auto *segment = mapping.segment_at(index); - if (segment == nullptr) { - continue; - } - for (uint64_t slot = 0; slot < MappingTable::kSegmentSize; ++slot) { - const auto word = segment->slots[slot].load(); - if (!slot_word::is_resident(word)) { - continue; - } - for (auto *page = slot_word::resident_ptr(word); page != nullptr;) { - auto *next = page->next; - page->retire_with_pins(); - page = next; - } - } - } - } -}; - -PageBase *copy_native(const NativeFrame &frame, const std::shared_ptr<BufferPool> &pool, uint32_t frame_bytes) -{ - const auto *data = frame.frame.data(); - const auto size = static_cast<uint32_t>(frame.frame.size()); - switch (frame_page_type(data)) { - case page_type::kLeafBase: - return LeafBase::from_frame_copy(data, size, pool, frame_bytes); - case page_type::kInnerBase: - return InnerBase::from_frame_copy(data, size, pool, frame_bytes); - case page_type::kOverflowFrame: - return OverflowBase::from_frame_copy(data, size, pool, frame_bytes); - default: - return nullptr; - } -} - -Status inherit_mapping(const MappingTable &source, MappingTable &destination, PageStore &store) -{ - for (uint64_t index = 0; index < MappingTable::kMaxSegments; ++index) { - auto *segment = source.segment_at(index); - if (segment == nullptr) { - continue; - } - std::vector<uint64_t> words(MappingTable::kSegmentSize); - for (uint64_t slot = 0; slot < words.size(); ++slot) { - auto word = segment->slots[slot].load(); - if (slot_word::is_resident(word)) { - auto *page = slot_word::resident_ptr(word); - if (page->durable_addr == kNoAddr || page->durable_plen == 0) { - return Status::invalid_argument("inherited mapping must refer to persisted pages"); - } - auto status = store.encode_mapping_location(page->durable_addr, page->durable_plen, &word); - if (!status.ok()) { - return status; - } - } - words[slot] = word; - } - destination.install_recovered_segment(index, segment->generation.load(), segment->live_count.load(), words, - segment->image_addr, segment->image_len, segment->image_crc); - auto *copy = destination.segment_at(index); - copy->write_seq.store(segment->write_seq.load()); - copy->persisted_seq.store(segment->persisted_seq.load()); - } - return Status::Ok(); -} - -void assign_parents(MappingTable &mapping, const std::vector<NativeFrame> &frames) -{ - for (const auto &frame : frames) { - if (frame_page_type(frame.frame.data()) != page_type::kInnerBase) { - continue; - } - InnerFrameView view(frame.frame.data(), static_cast<uint32_t>(frame.frame.size())); - for (uint32_t i = 0; i < view.num_children(); ++i) { - if (auto *child = mapping.get_resident(view.child_at(i)); child != nullptr) { - child->parent_page_id = frame.page_id; - } - } - } -} -} // namespace - -Status Crowdbtree::install_snapshot_native(std::vector<NativeFrame> frames, uint64_t root_page_id, uint64_t at_slot, - uint64_t next_page_id) +Status Crowdbtree::install_snapshot_native(std::vector<NativeFrame> frames, uint64_t root, uint64_t slot, uint64_t next) { - return install_range_snapshot_native(std::move(frames), root_page_id, at_slot, next_page_id, false); -} - -Status Crowdbtree::install_range_snapshot_native(std::vector<NativeFrame> frames, uint64_t root_page_id, - uint64_t at_slot, uint64_t next_page_id, bool mapping_inherited) -try { - acquire_snapshot_slot(); - auto release_snapshot = [this](Crowdbtree *) { release_snapshot_slot(); }; - std::unique_ptr<Crowdbtree, decltype(release_snapshot)> snapshot_slot(this, release_snapshot); - std::scoped_lock writer(write_mutex_); - for (const auto &frame : frames) { - if (frame.page_id == kInvalidPageId || frame.frame.empty() || - !frame_validate_key_range(frame.frame.data(), static_cast<uint32_t>(frame.frame.size()), opt_.key_range)) { - return Status::corruption("native snapshot: frame CRC, structure, or range invalid"); - } - } - auto status = detail::validate_native_snapshot_graph(&frames, root_page_id); - if (!status.ok()) { - return status; - } - auto staged = std::make_unique<MappingGeneration>(); - if (mapping_inherited) { - status = inherit_mapping(mapping_, staged->mapping, *opt_.page_store); - if (!status.ok()) { - return status; - } - } - uint64_t leaves = 0; - uint64_t inners = 0; - uint64_t next = next_page_id; - bool all_frames_inherited = mapping_inherited; - for (const auto &frame : frames) { - all_frames_inherited = all_frames_inherited && frame.inherited; - next = std::max(next, frame.page_id + 1); - const auto type = frame_page_type(frame.frame.data()); - leaves += static_cast<uint64_t>(type == page_type::kLeafBase); - inners += static_cast<uint64_t>(type == page_type::kInnerBase); - uint64_t word = slot_word::kEmpty; - if (mapping_inherited && frame.inherited && frame.durable_addr != kNoAddr && frame.durable_plen != 0 && - opt_.page_store->encode_mapping_location(frame.durable_addr, frame.durable_plen, &word).ok() && - staged->mapping.get_word(frame.page_id) == word) { - continue; - } - std::unique_ptr<PageBase> page(copy_native(frame, pool_, opt_.frame_bytes)); - if (page == nullptr) { - return Status::corruption("native snapshot: unknown frame type"); - } - staged->mapping.store(frame.page_id, page.get()); - [[maybe_unused]] auto *published = page.release(); - } - staged->mapping.set_next_page_id(next); - assign_parents(staged->mapping, frames); - auto successor = std::make_shared<MemTable>(memtable_next_id_.fetch_add(1), &epoch_); - successor->set_durable_floor(at_slot); - preserve_native_generation_locked(); - GenerationGate::Replacement replacement(generation_); - std::unique_lock catalog(memtable_mutex_); - active_->close(); - for (const auto &table : frozen_) { - table->close(); - } - for (const auto &table : split_shared_memtables_) { - table->close(); - } - // Generation admission has drained all old batches before slots/mappings - // change. The remaining publication consists only of non-allocating swaps. - mapping_.swap_quiescent(staged->mapping); - if (all_frames_inherited) { - for (uint64_t segment_index = 0; segment_index < MappingTable::kMaxSegments; ++segment_index) { - auto *segment = mapping_.segment_at(segment_index); - if (segment != nullptr) { - segment->persisted_seq.store(segment->write_seq.load(std::memory_order_relaxed), - std::memory_order_relaxed); - } - } - } - root_page_id_.store(root_page_id); - active_ = std::move(successor); - frozen_.clear(); - split_shared_memtables_.clear(); - split_overlay_source_.store(nullptr); - split_overlay_frontier_.store(0); - leaf_count_.store(leaves); - inner_count_.store(inners); - last_applied_slot_.store(at_slot); - contiguous_slot_.store(at_slot); - gc_floor_.store(0); - auto_slot_.store(at_slot); - received_slots_.clear(); - max_seen_slot_ = at_slot; - version_.fetch_add(1); - routing_fences_trusted_.store(true); - publication_incomplete_ = false; - epoch_.defer(staged.release()); - epoch_.try_reclaim(); - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("operation allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); + const auto count = frames.size(); + return core_->install_native(std::move(frames), + {.root_page_id = root, .at_slot = slot, .next_page_id = next, .frame_count = count}); } Status Crowdbtree::clear() { - return install_snapshot({}, 0); + return core_->clear(); } } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/ingest.cpp b/lib/crowdb-tree/src/btree/ingest.cpp index 9c257ac43..66f8cd0f7 100644 --- a/lib/crowdb-tree/src/btree/ingest.cpp +++ b/lib/crowdb-tree/src/btree/ingest.cpp @@ -1,240 +1,26 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -#include "crowdb-tree/crowdb-tree.h" - -#include <chrono> -#include <map> +#include "btree/tree_core.h" namespace crowdb::tree { -MemTableBatch Crowdbtree::admit_batch() -{ - auto generation = generation_.enter(); - for (;;) { - auto table = current_active(); - const auto bound = contiguous_slot_.load(std::memory_order_acquire); - if (table->try_enter()) { - return {.table = std::move(table), - .bound = bound, - .stats = {.measure_copy = metrics_.mt_version_copy_l != nullptr}, - .generation = std::move(generation), - .counters = &memtable_counters_, - .copy_latency = metrics_.mt_version_copy_l}; - } - } -} - -void Crowdbtree::finish_batch(MemTableBatch &batch, uint64_t slot, const std::vector<std::string> &keys) -{ - note_applied_slot(slot); - batch.completed = true; - const auto bound = contiguous_slot_.load(std::memory_order_acquire); - if (!batch.table->validate_open()) { - return; - } - try { - for (const auto &key : keys) { - batch.table->prune(Slice(key), bound, &batch.stats); - } - } - catch (const std::bad_alloc &) { - // Publication already succeeded. Optional compaction may be retried by - // a later writer; allocation failure cannot make this slot incomplete. - return; - } -} - -void Crowdbtree::apply_batch(uint64_t slot, const Batch &batch) -{ - // Intra-batch: last occurrence wins (all ops share `slot`). - auto apply_t0 = std::chrono::steady_clock::now(); - std::map<std::string, buffer> latest; // key -> single-alloc encoded cell buffer - for (const auto &op : batch.ops) { - latest[op.key] = encode_cell_buf(slot, op.kind, Slice(op.value)); - } - auto admission = admit_batch(); - admission.stats.gap = slot > admission.bound && slot - admission.bound > 1; - std::vector<std::string> keys; - keys.reserve(latest.size()); - while (!latest.empty()) { - auto node = latest.extract(latest.begin()); - keys.push_back(node.key()); - admission.table->upsert(Slice(node.key()), slot, std::move(node.mapped()), admission.bound, &admission.stats); - mt_upsert_total_.fetch_add(1, std::memory_order_relaxed); - } - finish_batch(admission, slot, keys); - if (metrics_.mt_apply_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - apply_t0).count(); - metrics_.mt_apply_l->observe(static_cast<uint64_t>(ns)); - } -} - -void Crowdbtree::recompute_contiguous_locked() -{ - // Fold received slots that extend the frontier one-by-one, then prune the - // tracker below the (possibly advanced) frontier so it stays bounded. - uint64_t cur = contiguous_slot_.load(); - auto it = received_slots_.upper_bound(cur); - while (it != received_slots_.end() && *it == cur + 1) { - cur = *it; - ++it; - } - contiguous_slot_.store(cur); - received_slots_.erase(received_slots_.begin(), received_slots_.upper_bound(cur)); -} - -void Crowdbtree::note_applied_slot(uint64_t slot) -{ - { - std::scoped_lock lk(slot_mutex_); - max_seen_slot_ = std::max(max_seen_slot_, slot); - received_slots_.insert(slot); - recompute_contiguous_locked(); - } -} - Status Crowdbtree::apply(uint64_t slot, const Batch &batch) -try { - // Reject oversized keys before any state is mutated (plan-tree #15). A key - // this large is assumed to be a caller bug; validating up front keeps apply - // all-or-nothing. - const size_t key_limit = max_key_size(); - for (const auto &op : batch.ops) { - Status range_status = validate_key(Slice(op.key)); - if (!range_status.ok()) { - return range_status; - } - if (op.key.size() > key_limit) { - return Status::invalid_argument("key exceeds max_key_size (" + std::to_string(op.key.size()) + " > " + - std::to_string(key_limit) + ")"); - } - } - apply_batch(slot, batch); - maybe_swap_active(); - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("apply allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); +{ + return core_->apply(slot, batch); } Status Crowdbtree::apply_encoded(uint64_t slot, std::vector<encoded_op> ops) -try { - // Same guard as apply() (plan-tree #15): validate every key before any - // state is mutated. - const size_t key_limit = max_key_size(); - for (const encoded_op &op : ops) { - const CellView cell(op.cell.slice()); - if (!cell.valid() || cell.slot() != slot) { - return Status::invalid_argument("encoded cell slot must match its batch"); - } - - Status range_status = validate_key(Slice(op.key)); - if (!range_status.ok()) { - return range_status; - } - if (op.key.size() > key_limit) { - return Status::invalid_argument("key exceeds max_key_size (" + std::to_string(op.key.size()) + " > " + - std::to_string(key_limit) + ")"); - } - } - { - auto admission = admit_batch(); - admission.stats.gap = slot > admission.bound && slot - admission.bound > 1; - std::vector<std::string> keys; - keys.reserve(ops.size()); - // Intra-batch: last occurrence (vector order) wins, same as - // apply_batch. Cells already come in pre-encoded (single alloc at - // the caller's boundary, e.g. the C API -- plan-tree #5 B2d) -- - // move key+cell straight down, no encode_cell_buf call here. - std::map<std::string, buffer> latest; - for (encoded_op &op : ops) { - latest[std::move(op.key)] = std::move(op.cell); - } - while (!latest.empty()) { - auto node = latest.extract(latest.begin()); - keys.push_back(node.key()); - admission.table->upsert(Slice(node.key()), slot, std::move(node.mapped()), admission.bound, - &admission.stats); - mt_upsert_total_.fetch_add(1, std::memory_order_relaxed); - } - finish_batch(admission, slot, keys); - } - maybe_swap_active(); - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("apply allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); +{ + return core_->apply_encoded(slot, std::move(ops)); } Status Crowdbtree::apply_external(uint64_t slot, std::vector<external_op> ops) -try { - // Same guard as apply_encoded: validate every key before any state mutation. - const size_t key_limit = max_key_size(); - for (const external_op &op : ops) { - Status range_status = validate_key(Slice(op.key)); - if (!range_status.ok()) { - return range_status; - } - if (op.key.size() > key_limit) { - return Status::invalid_argument("key exceeds max_key_size (" + std::to_string(op.key.size()) + " > " + - std::to_string(key_limit) + ")"); - } - } - { - auto admission = admit_batch(); - admission.stats.gap = slot > admission.bound && slot - admission.bound > 1; - std::vector<std::string> keys; - keys.reserve(ops.size()); - // Intra-batch: last occurrence (vector order) wins, same as apply_encoded. - // Track {flags, value} per key; the value buffer is moved straight down - // (no encode_cell_buf, no value memcpy). - std::map<std::string, std::pair<uint8_t, buffer>> latest; - for (external_op &op : ops) { - latest[std::move(op.key)] = {op.flags, std::move(op.value)}; - } - while (!latest.empty()) { - auto node = latest.extract(latest.begin()); - keys.push_back(node.key()); - admission.table->upsert_external(Slice(node.key()), slot, node.mapped().first, - std::move(node.mapped().second), admission.bound, &admission.stats); - mt_upsert_total_.fetch_add(1, std::memory_order_relaxed); - } - finish_batch(admission, slot, keys); - } - maybe_swap_active(); - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("apply allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); +{ + return core_->apply_external(slot, std::move(ops)); } void Crowdbtree::force_advance_slot(uint64_t slot) { - auto admission = admit_batch(); - admission.stats.gap = slot > admission.bound && slot - admission.bound > 1; - { - std::scoped_lock lk(slot_mutex_); - max_seen_slot_ = std::max(max_seen_slot_, slot); - // Treat any gap up to `slot` as NoOps: jump the frontier, then fold in any - // already-received slots that are now contiguous with it. - if (slot > contiguous_slot_.load()) { - contiguous_slot_.store(slot); - } - recompute_contiguous_locked(); - } - admission.completed = true; - maybe_swap_active(); + core_->force_advance_slot(slot); } - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/inline_function.h b/lib/crowdb-tree/src/btree/inline_function.h new file mode 100644 index 000000000..2aa675eae --- /dev/null +++ b/lib/crowdb-tree/src/btree/inline_function.h @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <array> +#include <cstddef> +#include <functional> +#include <memory> +#include <type_traits> +#include <utility> + +namespace crowdb::tree::detail +{ +template <class Signature, size_t Capacity = 64> class InlineFunction; + +// Publication continuations live inside their already budgeted request. There +// is no heap fallback; oversized or throwing-move closures fail to compile. +template <class R, class... Args, size_t Capacity> class InlineFunction<R(Args...), Capacity> +{ + struct Operations + { + R (*invoke)(void *, Args &&...); + void (*copy)(void *, const void *); + void (*move)(void *, void *) noexcept; + void (*destroy)(void *) noexcept; + }; + + template <class F> + static constexpr bool accepts = + sizeof(std::decay_t<F>) <= Capacity && alignof(std::decay_t<F>) <= alignof(std::max_align_t) && + std::is_copy_constructible_v<std::decay_t<F>> && std::is_nothrow_move_constructible_v<std::decay_t<F>> && + std::is_invocable_r_v<R, std::decay_t<F> &, Args...>; + + public: + InlineFunction() = default; + + template <class F> + requires(!std::is_same_v<std::decay_t<F>, InlineFunction>) && accepts<F> + InlineFunction(F &&function) + { + using Callable = std::decay_t<F>; + if constexpr (std::is_pointer_v<Callable> || std::is_member_pointer_v<Callable>) { + if (function == nullptr) { + return; + } + } + static const Operations operations{ + .invoke = [](void *value, Args &&...args) -> R { + return std::invoke(*std::launder(reinterpret_cast<Callable *>(value)), std::forward<Args>(args)...); + }, + .copy = + [](void *destination, const void *source) { + std::construct_at(reinterpret_cast<Callable *>(destination), + *std::launder(reinterpret_cast<const Callable *>(source))); + }, + .move = + [](void *destination, void *source) noexcept { + auto *value = std::launder(reinterpret_cast<Callable *>(source)); + std::construct_at(reinterpret_cast<Callable *>(destination), std::move(*value)); + std::destroy_at(value); + }, + .destroy = + [](void *value) noexcept { std::destroy_at(std::launder(reinterpret_cast<Callable *>(value))); }}; + std::construct_at(reinterpret_cast<Callable *>(storage_.data()), std::forward<F>(function)); + operations_ = &operations; + } + + InlineFunction(const InlineFunction &other) + { + if (other.operations_ != nullptr) { + other.operations_->copy(storage_.data(), other.storage_.data()); + operations_ = other.operations_; + } + } + + InlineFunction(InlineFunction &&other) noexcept + { + if (other.operations_ != nullptr) { + other.operations_->move(storage_.data(), other.storage_.data()); + operations_ = std::exchange(other.operations_, nullptr); + } + } + + InlineFunction &operator=(const InlineFunction &other) + { + if (this != &other) { + InlineFunction copy(other); + *this = std::move(copy); + } + return *this; + } + + InlineFunction &operator=(InlineFunction &&other) noexcept + { + if (this != &other) { + reset(); + if (other.operations_ != nullptr) { + other.operations_->move(storage_.data(), other.storage_.data()); + operations_ = std::exchange(other.operations_, nullptr); + } + } + return *this; + } + + ~InlineFunction() + { + reset(); + } + + void reset() noexcept + { + if (operations_ != nullptr) { + operations_->destroy(storage_.data()); + operations_ = nullptr; + } + } + + [[nodiscard]] explicit operator bool() const noexcept + { + return operations_ != nullptr; + } + + R operator()(Args... args) const + { + if (operations_ == nullptr) { + throw std::bad_function_call(); + } + return operations_->invoke(storage_.data(), std::forward<Args>(args)...); + } + + private: + alignas(std::max_align_t) mutable std::array<std::byte, Capacity> storage_; + const Operations *operations_ = nullptr; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/inspect.cpp b/lib/crowdb-tree/src/btree/inspect.cpp index 7d16e3155..8d4fbee57 100644 --- a/lib/crowdb-tree/src/btree/inspect.cpp +++ b/lib/crowdb-tree/src/btree/inspect.cpp @@ -1,82 +1,13 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -#include "crowdb-tree/btree/tree.h" +#include "btree/published_inspection.h" +#include "btree/tree_core.h" namespace crowdb::tree { Status Crowdbtree::inspect_page(const std::vector<uint32_t> &path, uint64_t expected_version, NativeFrame *out, uint64_t *out_version, uint64_t *out_root, uint32_t *out_deltas) const { - if (path.size() > 32 || out == nullptr || out_version == nullptr || out_root == nullptr || out_deltas == nullptr) { - return Status::invalid_argument("page inspection path exceeds 32 levels"); - } - auto guard = epoch_.enter(); - const auto rev = version(); - const auto root = root_page_id(); - auto id = root; - if (expected_version != kInvalidPageId && expected_version != rev) { - return Status::unavailable("tree changed; restart page inspection"); - } - std::vector<std::pair<uint64_t, PageBase *>> observed; - for (size_t depth = 0; depth <= path.size(); ++depth) { - PageBase *head = resident(id); - if (head == nullptr) { - return io_failed() ? Status::io_error("page load failed") : Status::not_found("page unavailable"); - } - observed.emplace_back(id, head); - PageBase *base = head; - uint32_t deltas = 0; - while (base->next != nullptr && deltas < 256) { - base = base->next; - ++deltas; - } - if (base->next != nullptr) { - return Status::resource_exhausted("page delta chain exceeds inspection budget"); - } - if (depth < path.size()) { - if (base->type != page_type::kInnerBase) { - return Status::unavailable("page path changed; restart inspection"); - } - const auto view = static_cast<InnerBase *>(base)->view(); - if (path[depth] >= view.num_children()) { - return Status::invalid_argument("child outside page"); - } - id = view.child_at(path[depth]); - continue; - } - const uint8_t *frame = nullptr; - uint32_t bytes = 0; - if (base->type == page_type::kInnerBase) { - const auto *inner = static_cast<InnerBase *>(base); - frame = inner->frame(); - bytes = inner->page_bytes(); - } - else if (base->type == page_type::kLeafBase) { - const auto *leaf = static_cast<LeafBase *>(base); - frame = leaf->frame(); - bytes = leaf->page_bytes(); - } - else { - return Status::not_supported("not a tree base page"); - } - if (bytes > 1024 * 1024) { - return Status::resource_exhausted("page exceeds 1 MiB inspection budget"); - } - out->page_id = id; - out->frame.assign(frame, frame + bytes); - *out_deltas = deltas; - } - for (const auto &[page_id, head] : observed) { - if (mapping_.get_resident(page_id) != head) { - return Status::unavailable("page changed during inspection"); - } - } - if (version() != rev || root_page_id() != root) { - return Status::unavailable("tree changed during inspection"); - } - *out_version = rev; - *out_root = root; - return Status::Ok(); + return inspect_published_page(*core_, path, expected_version, out, out_version, out_root, out_deltas); } } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/local_compaction_internal.h b/lib/crowdb-tree/src/btree/local_compaction_internal.h new file mode 100644 index 000000000..7ca58c588 --- /dev/null +++ b/lib/crowdb-tree/src/btree/local_compaction_internal.h @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/local/local_sparse_gc.h" +#include "btree/local_compaction_operation.h" +#include "btree/tree_core_checkpoint_internal.h" + +namespace crowdb::tree +{ +struct LocalCompactionOperation::State : std::enable_shared_from_this<State> +{ + enum class Phase : uint8_t { kBegin, kSelected, kCheckpoint, kCleaned, kDone }; + + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + std::shared_ptr<State> owner; + void set_value() noexcept; + void set_error(Status status) noexcept; + void set_stopped() noexcept; + }; + + struct Attempt + { + Attempt(detail::CallbackSender sender, std::shared_ptr<State> owner) + : connected(stdexec::connect(sender, Receiver{std::move(owner)})) + { + } + + stdexec::connect_result_t<detail::CallbackSender, Receiver> connected; + }; + + ~State(); + void start(detail::CallbackComplete callback, void *argument) noexcept; + static void cpu_run(void *context) noexcept; + static void backend_run(void *context) noexcept; + void pump(); + void backend_step(); + void checkpoint(); + void child_done(Status status) noexcept; + void finish(Status status) noexcept; + void wake() noexcept; + + std::shared_ptr<detail::AllocationReservation> credit; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<AllocationBudget> backend; + TreeCore *core = nullptr; + BlockPageStore *store = nullptr; + std::shared_ptr<LocalBackendState> local; + ApplyDispatcher::Node cpu_node{&State::cpu_run, this}; + ApplyDispatcher::Node backend_node{&State::backend_run, this}; + std::shared_ptr<ApplyDispatcher> cpu_progress; + std::shared_ptr<ApplyDispatcher> backend_progress; + std::shared_ptr<CheckpointOperation> child; + std::optional<Attempt> attempt; + LocalSparseBlock candidate; + MergeGcStats report; + detail::InlineFunction<void(const MergeGcStats &), 32> observer; + LocalCompactionOptions options; + Status child_status; + std::atomic<bool> child_ready{false}; + std::atomic<bool> stopped{false}; + std::atomic<bool> started{false}; + std::atomic<bool> callback_ready{false}; + Phase phase = Phase::kBegin; + size_t rotations = 0; + bool gate_held = false; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/local_compaction_operation.cpp b/lib/crowdb-tree/src/btree/local_compaction_operation.cpp new file mode 100644 index 000000000..88476ab9e --- /dev/null +++ b/lib/crowdb-tree/src/btree/local_compaction_operation.cpp @@ -0,0 +1,181 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/block_account.h" +#include "btree/allocation_bound.h" +#include "btree/local_compaction_internal.h" + +#include <cmath> + +namespace crowdb::tree +{ +std::optional<size_t> LocalCompactionOperation::allocation_bound() noexcept +{ + using namespace detail; + return AllocationBound(local_block_gc_metadata_bound({}).value_or(SIZE_MAX)) + .add(shared_allocation_bound<State>()) + .add(shared_allocation_bound<LocalCompactionOperation>()) + .add(shared_allocation_bound<AllocationReservation>()) + .add(TreeOperationLifetime::ticket_allocation_bound()) + .value(); +} +Status LocalCompactionOperation::create(TreeCore &core, LocalCompactionOptions options, + std::shared_ptr<ApplyDispatcher> cpu, + std::shared_ptr<detail::AllocationReservation> credit, + std::shared_ptr<TreeOperationLifetime::Ticket> ticket, + std::shared_ptr<LocalCompactionOperation> *out) +try { + if (out == nullptr || *out || !credit || !ticket || !cpu) { + return Status::invalid_argument(); + } + auto state = detail::reserved_shared<State>(credit); + state->credit = credit; + state->ticket = std::move(ticket); + state->core = &core; + state->options = options; + state->local = core.local_backend(); + state->store = dynamic_cast<BlockPageStore *>(core.config().page_store); + state->cpu_progress = std::move(cpu); + state->backend_progress = core.binding()->backend_progress(); + Status status; + if (state->store != nullptr && state->store->block_size() != 0 && options.max_relocation_bytes != 0) { + if (!state->local || !state->local->ledger || !std::isfinite(options.free_threshold) || + options.free_threshold <= 0.0 || options.free_threshold >= 1.0) { + return Status::invalid_argument(); + } + status = detail::reserve_backend_operation(state->store, core.binding()->account(), + {.workers = core.binding()->backend_cpu()->worker_count()}, + &state->backend); + if (!status.ok()) { + return status; + } + if (state->local->gc_active.exchange(true, std::memory_order_acq_rel)) { + return Status::unavailable("another block cleanup is active"); + } + state->gate_held = true; + } + else { + state->store = nullptr; + } + *out = + detail::reserved_construct<LocalCompactionOperation>(credit, [state = std::move(state)](void *where) mutable { + return ::new (where) LocalCompactionOperation(std::move(state)); + }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +LocalCompactionOperation::State::~State() +{ + if (gate_held) { + local->gc_active.store(false, std::memory_order_release); + } + detail::seal_backend_budget(backend); + if (credit) { + credit->seal(); + } +} + +detail::CallbackSender LocalCompactionOperation::sender() noexcept +{ + return {state_.get(), [](void *source, detail::CallbackComplete done, void *context) { + static_cast<State *>(source)->start(done, context); + }}; +} + +const MergeGcStats &LocalCompactionOperation::result() const noexcept +{ + return state_->report; +} + +void LocalCompactionOperation::observe_completion( + detail::InlineFunction<void(const MergeGcStats &), 32> observer) noexcept +{ + state_->observer = std::move(observer); +} + +void LocalCompactionOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + if (state_->callback_ready.load(std::memory_order_acquire)) { + state_->wake(); + } +} + +void LocalCompactionOperation::State::start(detail::CallbackComplete callback, void *argument) noexcept +{ + if (started.exchange(true, std::memory_order_acq_rel)) { + callback(argument, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + complete = callback; + context = argument; + callback_ready.store(true, std::memory_order_release); + wake(); +} + +void LocalCompactionOperation::State::wake() noexcept +{ + cpu_progress->request(cpu_node, weak_from_this().lock()); +} + +void LocalCompactionOperation::State::cpu_run(void *context) noexcept +{ + auto &state = *static_cast<State *>(context); + try { + state.pump(); + } + catch (const std::bad_alloc &) { + state.finish(Status::resource_exhausted()); + } + catch (...) { + state.finish(Status::io_error()); + } +} + +void LocalCompactionOperation::State::backend_run(void *context) noexcept +{ + auto &state = *static_cast<State *>(context); + try { + state.backend_step(); + } + catch (const std::bad_alloc &) { + state.child_status = Status::resource_exhausted(); + } + catch (...) { + state.child_status = Status::io_error(); + } + state.child_ready.store(true, std::memory_order_release); + state.wake(); +} + +void LocalCompactionOperation::State::finish(Status status) noexcept +{ + if (phase == Phase::kDone) { + return; + } + phase = Phase::kDone; + attempt.reset(); + child.reset(); + candidate.exclusion = {}; + if (gate_held) { + local->gc_active.store(false, std::memory_order_release); + gate_held = false; + } + detail::seal_backend_budget(backend); + credit->seal(); + if (observer) { + try { + observer(report); + } + catch (...) { + // Reporting cannot change the already completed storage outcome. + observer.reset(); + } + observer.reset(); + } + const auto signal = status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + complete(context, signal, std::move(status)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/local_compaction_operation.h b/lib/crowdb-tree/src/btree/local_compaction_operation.h new file mode 100644 index 000000000..04d86c715 --- /dev/null +++ b/lib/crowdb-tree/src/btree/local_compaction_operation.h @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/allocation_reservation.h" +#include "btree/inline_function.h" +#include "crowdb-tree/btree/diagnostics.h" +#include "runtime/tree_operation_lifetime.h" +#include "stdexec_adapter.h" + +namespace crowdb::tree +{ +class TreeCore; +class ApplyDispatcher; + +struct LocalCompactionOptions +{ + double free_threshold = 0.70; + // At most one eligible source block is selected per pass. Zero disables + // relocation; the first source block may exceed a nonzero byte budget. + uint64_t max_relocation_bytes = 1; +}; + +class LocalCompactionOperation +{ + struct State; + + public: + static Status create(TreeCore &core, LocalCompactionOptions options, std::shared_ptr<ApplyDispatcher> cpu, + std::shared_ptr<detail::AllocationReservation> credit, + std::shared_ptr<TreeOperationLifetime::Ticket> ticket, + std::shared_ptr<LocalCompactionOperation> *out); + [[nodiscard]] static std::optional<size_t> allocation_bound() noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const MergeGcStats &result() const noexcept; + void request_stop() noexcept; + // Install before connecting the one-shot sender. Storage is prepaid in State. + void observe_completion(detail::InlineFunction<void(const MergeGcStats &), 32> observer) noexcept; + + private: + explicit LocalCompactionOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/local_compaction_steps.cpp b/lib/crowdb-tree/src/btree/local_compaction_steps.cpp new file mode 100644 index 000000000..83117fd40 --- /dev/null +++ b/lib/crowdb-tree/src/btree/local_compaction_steps.cpp @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/block_account.h" +#include "btree/local_compaction_internal.h" + +namespace crowdb::tree +{ +void LocalCompactionOperation::State::Receiver::set_value() noexcept +{ + set_error(Status::Ok()); +} + +void LocalCompactionOperation::State::Receiver::set_error(Status status) noexcept +{ + auto retained = std::move(owner); + retained->child_done(std::move(status)); +} + +void LocalCompactionOperation::State::Receiver::set_stopped() noexcept +{ + set_error(Status::unavailable()); +} + +void LocalCompactionOperation::State::child_done(Status status) noexcept +{ + child_status = std::move(status); + child_ready.store(true, std::memory_order_release); + wake(); +} + +void LocalCompactionOperation::State::checkpoint() +{ + auto options = checkpoint_detail::defaults(*core); + options.cause = TreeWorkCause::GarbageCollection; + options.write.frames.exclusion = {.address = candidate.exclusion.address, .length = candidate.exclusion.length}; + const auto status = core->prepare_checkpoint(options, &child, ticket); + if (!status.ok()) { + finish(status.without_message()); + return; + } + phase = Phase::kCheckpoint; + child_ready.store(false, std::memory_order_release); + attempt.emplace(child->sender(), weak_from_this().lock()); + stdexec::start(attempt->connected); +} + +void LocalCompactionOperation::State::pump() +{ + if (phase == Phase::kDone) { + return; + } + if (phase == Phase::kBegin) { + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + if (store == nullptr) { + finish(Status::Ok()); + return; + } + phase = Phase::kSelected; + backend_progress->request(backend_node, weak_from_this().lock()); + return; + } + if (phase == Phase::kCheckpoint && stopped.load(std::memory_order_acquire)) { + child->request_stop(); + } + if (!child_ready.load(std::memory_order_acquire)) { + return; + } + if (phase == Phase::kCheckpoint) { + report.pages_relocated += child->result().written_frames; + report.bytes_relocated += child->result().written_bytes; + attempt.reset(); + child.reset(); + ++rotations; + } + if (!child_status.ok()) { + finish(child_status.without_message()); + return; + } + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + if (phase == Phase::kCleaned || !candidate.exclusion.owner) { + finish(Status::Ok()); + return; + } + if (rotations < 2) { + checkpoint(); + return; + } + phase = Phase::kCleaned; + child_ready.store(false, std::memory_order_release); + backend_progress->request(backend_node, weak_from_this().lock()); +} + +void LocalCompactionOperation::State::backend_step() +{ + // Retirement drains are host/backend waits. The CPU lane remains free to + // execute the paid destructor continuations they depend on. + core->publisher().reclaim(); + core->binding()->account()->drain_retirements(); + if (phase == Phase::kSelected) { + const LocalBlockGcLimits limits; + auto freed = + collect_free_local_blocks(*store, *local->ledger, local->gc_cursor.load(), limits, credit, backend); + report.blocks_deleted += freed.deleted_blocks; + child_status = freed.status; + if (!child_status.ok()) { + return; + } + candidate = select_sparse_local_block(*store, *local->ledger, local->gc_cursor.load(), credit, + options.free_threshold, backend); + local->gc_cursor.store(candidate.next_block); + child_status = candidate.status.code() == Code::kNotFound ? Status::Ok() : candidate.status.without_message(); + if (candidate.exclusion.owner) { + report.blocks_selected = 1; + } + } + else { + core->cache()->evict(); + core->publisher().reclaim(); + core->binding()->account()->drain_retirements(); + const LocalBlockGcLimits limits{.max_blocks = 1, .max_deleted_bytes = store->block_size()}; + auto freed = collect_free_local_blocks( + *store, *local->ledger, candidate.exclusion.address / store->block_size(), limits, credit, backend); + report.blocks_deleted += freed.deleted_blocks; + child_status = freed.status; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/logical_page.cpp b/lib/crowdb-tree/src/btree/logical_page.cpp new file mode 100644 index 000000000..5ded8122e --- /dev/null +++ b/lib/crowdb-tree/src/btree/logical_page.cpp @@ -0,0 +1,349 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" + +#include "btree/tree_work_counters.h" + +#include <algorithm> +#include <cstring> + +namespace crowdb::tree +{ +bool PageKeyRange::contains(Slice key) const +{ + return (!lower.has_value() || key >= Slice(*lower)) && (!upper.has_value() || key < Slice(*upper)); +} + +bool PageKeyRange::empty() const +{ + return lower.has_value() && upper.has_value() && Slice(*lower) >= Slice(*upper); +} + +bool PageKeyRange::overlaps(const PageKeyRange &other) const +{ + return !empty() && !other.empty() && + (!upper.has_value() || !other.lower.has_value() || Slice(*upper) > Slice(*other.lower)) && + (!lower.has_value() || !other.upper.has_value() || Slice(*lower) < Slice(*other.upper)); +} + +std::optional<Slice> PageKeyRange::lower_slice() const +{ + return lower.has_value() ? std::optional<Slice>(Slice(*lower)) : std::nullopt; +} + +std::optional<Slice> PageKeyRange::upper_slice() const +{ + return upper.has_value() ? std::optional<Slice>(Slice(*upper)) : std::nullopt; +} + +Slice EntryRef::key() const +{ + return key_arena != nullptr ? Slice(key_arena->data() + key_offset, key_length) : Slice(); +} + +Slice EntryRef::value() const +{ + if (owner == nullptr || (owner->flags & (kFlagTombstone | kFlagOverflow)) != 0) { + return {}; + } + return owner->cell.ownership() == buffer::mode::kExternal ? owner->cell.slice() + : CellView(owner->cell.slice()).value(); +} + +uint64_t EntryRef::slot() const +{ + return owner != nullptr ? owner->slot : 0; +} + +uint8_t EntryRef::flags() const +{ + return owner != nullptr ? owner->flags : 0; +} + +Status KeyArenaBuilder::append(Slice key, std::shared_ptr<const CellVersion> owner, EntryRef *out) +{ + if (out == nullptr || owner == nullptr) { + return Status::invalid_argument("entry needs a cell owner and output"); + } + if (entries_ >= limits_.max_entries || key.size() > limits_.max_key_bytes - bytes_ || + limits_.key_block_bytes == 0) { + return Status::resource_exhausted("overlay key/entry budget exhausted"); + } + if (block_ == nullptr || key.size() > block_->size() - used_) { + if (key.size() > limits_.max_key_bytes - allocated_) { + return Status::resource_exhausted("overlay arena capacity budget exhausted"); + } + const size_t capacity = + std::max(key.size(), std::min(limits_.key_block_bytes, limits_.max_key_bytes - allocated_)); + block_ = detail::reserved_shared<EntryKeyArena>(metadata_, capacity, metadata_); + allocated_ += capacity; + used_ = 0; + } + if (!key.empty()) { + std::memcpy(block_->data() + used_, key.data(), key.size()); + record_work(metadata_, TreeWorkCounter::KeyCopyBytes, key.size()); + } + *out = {.key_arena = block_, .key_offset = used_, .key_length = key.size(), .owner = std::move(owner)}; + used_ += key.size(); + bytes_ += key.size(); + ++entries_; + return Status::Ok(); +} + +Status OverlayBatch::build(uint64_t id, std::vector<EntryRef> entries, const RunLimits &limits, + std::shared_ptr<const OverlayBatch> *out, + std::shared_ptr<detail::AllocationReservation> metadata) +{ + EntryStorage owned(std::make_move_iterator(entries.begin()), std::make_move_iterator(entries.end()), + EntryAllocator(std::move(metadata))); + return build_owned(id, std::move(owned), limits, out); +} + +Status OverlayBatch::build_owned(uint64_t id, EntryStorage entries, const RunLimits &limits, + std::shared_ptr<const OverlayBatch> *out) +{ + if (out == nullptr) { + return Status::invalid_argument("overlay needs an output"); + } + if (entries.size() > limits.max_entries) { + return Status::resource_exhausted("overlay entry budget exhausted"); + } + auto metadata = entries.get_allocator().owner(); + auto run = detail::reserved_shared<OverlayBatch>(metadata, metadata); + for (size_t i = 0; i < entries.size(); ++i) { + const auto &entry = entries[i]; + if (entry.owner == nullptr || entry.key_arena == nullptr || entry.key_offset > entry.key_arena->size() || + entry.key_length > entry.key_arena->size() - entry.key_offset) { + return Status::invalid_argument("overlay entry has no valid independent owner"); + } + if (i != 0 && entries[i - 1].key() >= entry.key()) { + return Status::invalid_argument("overlay entries must be sorted and unique"); + } + if (entry.key_length > limits.max_key_bytes - run->key_bytes_) { + return Status::resource_exhausted("overlay key budget exhausted"); + } + run->key_bytes_ += entry.key_length; + run->min_slot_ = std::min(run->min_slot_, entry.slot()); + run->max_slot_ = std::max(run->max_slot_, entry.slot()); + run->has_overflow_values_ = run->has_overflow_values_ || (entry.flags() & kFlagOverflow) != 0; + } + run->id_ = id; + run->entries_ = std::move(entries); + *out = std::move(run); + return Status::Ok(); +} + +size_t OverlayBatch::lower_bound(Slice key) const +{ + return static_cast<size_t>( + std::lower_bound(entries_.begin(), entries_.end(), key, + [](const EntryRef &entry, Slice wanted) { return entry.key() < wanted; }) - + entries_.begin()); +} + +const EntryRef *OverlayBatch::find(Slice key) const +{ + const size_t index = lower_bound(key); + return index < entries_.size() && entries_[index].key() == key ? &entries_[index] : nullptr; +} + +Status OverlayBuilder::append(Slice key, std::shared_ptr<const CellVersion> owner) +{ + if (!entries_.empty() && entries_.back().key() >= key) { + return Status::invalid_argument("overlay entries must be sorted and unique"); + } + EntryRef entry; + auto status = arena_.append(key, std::move(owner), &entry); + return status.ok() ? append(std::move(entry)) : status; +} + +Status OverlayBuilder::append(EntryRef entry) +{ + if (entry.owner == nullptr || entry.key_arena == nullptr || entry.key_offset > entry.key_arena->size() || + entry.key_length > entry.key_arena->size() - entry.key_offset || + (!entries_.empty() && entries_.back().key() >= entry.key())) { + return Status::invalid_argument("overlay entry must own a valid key and be strictly ordered"); + } + if (entries_.size() >= limits_.max_entries || entry.key_length > limits_.max_key_bytes - key_bytes_) { + return Status::resource_exhausted("overlay key/entry budget exhausted"); + } + key_bytes_ += entry.key_length; + entries_.push_back(std::move(entry)); + return Status::Ok(); +} + +Status OverlayBuilder::finish(uint64_t id, std::shared_ptr<const OverlayBatch> *out) +{ + auto status = OverlayBatch::build_owned(id, std::move(entries_), limits_, out); + if (status.ok()) { + key_bytes_ = 0; + arena_ = KeyArenaBuilder(limits_, metadata_); + } + return status; +} + +bool BaseRef::valid() const +{ + return resident != nullptr || (backing != nullptr && backing->readable()); +} + +LogicalCell LogicalCell::from_entry(const EntryRef &entry) +{ + return {.owner = entry.owner, + .slot = entry.slot(), + .flags = entry.flags(), + .value = entry.value(), + .encoded = entry.owner->cell.ownership() == buffer::mode::kExternal ? Slice() : entry.owner->cell.slice()}; +} + +LogicalCell LogicalCell::from_base(const std::shared_ptr<const LeafBase> &base, size_t index, + std::shared_ptr<const BackingRef> backing) +{ + const CellView view(base->cell(index)); + return {.owner = base, + .slot = view.slot(), + .flags = view.flags(), + .value = view.value(), + .encoded = view.raw(), + .backing = std::move(backing)}; +} + +LogicalLookup PageState::lookup(Slice key, uint64_t frontier) const +{ + LogicalLookup result; + if (!range.contains(key)) { + return result; + } + size_t winner_source = 0; + auto consider = [&](LogicalCell candidate, size_t source) { + if (candidate.slot <= frontier && (result.cell.owner == nullptr || candidate.slot > result.cell.slot)) { + result.cell = std::move(candidate); + winner_source = source; + } + }; + for (size_t i = 0; i < overlays.size(); ++i) { + const auto &run = overlays[i]; + if (run == nullptr) { + result.state = LogicalLookupState::kInvalid; + result.status = Status::corruption("logical overlay has no owner"); + return result; + } + if (const auto *entry = run->find(key); entry != nullptr) { + consider(LogicalCell::from_entry(*entry), i); + } + } + for (size_t i = 0; i < bases.size(); ++i) { + const auto &base = bases[i]; + if (!base.range.contains(key)) { + continue; + } + if (!base.valid()) { + result.state = LogicalLookupState::kInvalid; + result.status = Status::corruption("logical base has neither resident nor reloadable owner"); + result.base_index = i; + return result; + } + if (base.resident != nullptr) { + const int index = base.resident->find(key); + if (index >= 0) { + consider(LogicalCell::from_base(base.resident, static_cast<size_t>(index), base.backing), + overlays.size() + i); + } + } + } + for (size_t i = 0; i < bases.size(); ++i) { + const auto &base = bases[i]; + if (base.range.contains(key) && !base.resident && + (result.cell.owner == nullptr || !base.max_slot.has_value() || result.cell.slot < *base.max_slot || + (result.cell.slot == *base.max_slot && winner_source > overlays.size() + i))) { + result.state = LogicalLookupState::kNeedsBase; + result.base_index = i; + return result; + } + } + if (result.cell.owner != nullptr) { + result.state = result.cell.tombstone() ? LogicalLookupState::kTombstone : LogicalLookupState::kValue; + } + return result; +} + +MemTableRangeMerge::MemTableRangeMerge(std::initializer_list<std::shared_ptr<MemTable>> sources, PageKeyRange range, + uint64_t frontier, uint64_t floor, RunLimits limits, size_t max_sources, + std::shared_ptr<detail::AllocationReservation> metadata) + : MemTableRangeMerge(std::span<const std::shared_ptr<MemTable>>(sources), std::move(range), frontier, floor, limits, + max_sources, std::move(metadata)) +{ +} + +MemTableRangeMerge::MemTableRangeMerge(std::span<const std::shared_ptr<MemTable>> sources, PageKeyRange range, + uint64_t frontier, uint64_t floor, RunLimits limits, size_t max_sources, + std::shared_ptr<detail::AllocationReservation> metadata) + : owners_(detail::ReservationAllocator<std::shared_ptr<MemTable>>(metadata)), + guards_(detail::ReservationAllocator<EpochManager::Guard>(metadata)), + range_(std::move(range)), + limits_(limits), + metadata_(std::move(metadata)), + arena_(limits, metadata_), + cursors_(detail::ReservationAllocator<ConcurrentSkipList::Cursor>(metadata_)), + sources_(detail::ReservationAllocator<MergeSource>(metadata_)), + merge_(metadata_) +{ + if (sources.size() > max_sources || sources.size() > static_cast<size_t>(std::numeric_limits<int>::max())) { + status_ = Status::resource_exhausted("MemTable merge source budget exhausted"); + return; + } + owners_.assign(sources.begin(), sources.end()); + guards_.reserve(owners_.size()); + cursors_.reserve(owners_.size()); + sources_.reserve(owners_.size()); + for (const auto &owner : owners_) { + if (owner == nullptr) { + status_ = Status::invalid_argument("MemTable merge source has no owner"); + return; + } + guards_.push_back(owner->read_guard(metadata_)); + cursors_.push_back(owner->range_cursor(range_.lower_slice(), range_.upper_slice(), frontier, floor)); + sources_.push_back({.kind = MergeSource::kL0, .l0 = &cursors_.back(), .l1 = nullptr}); + } + if (!sources_.empty()) { + merge_.init(sources_.data(), static_cast<int>(sources_.size())); + } +} + +bool MemTableRangeMerge::valid() const +{ + return status_.ok() && !sources_.empty() && merge_.winner_valid(); +} + +Slice MemTableRangeMerge::key() const +{ + return valid() ? sources_[static_cast<size_t>(merge_.winner())].key() : Slice(); +} + +Status MemTableRangeMerge::next(EntryRef *out) +{ + if (!status_.ok()) { + return status_; + } + if (!valid()) { + return Status::not_found("MemTable range exhausted"); + } + auto &cursor = cursors_[static_cast<size_t>(merge_.winner())]; + // Acquire the independent owner before any source cursor advances. + auto status = arena_.append(cursor.key(), cursor.cell_owner(), out); + if (!status.ok()) { + return status; + } + const Slice key = out->key(); + do { + ++candidates_; + merge_.advance_winner(); + } while (merge_.winner_valid() && sources_[static_cast<size_t>(merge_.winner())].key() == key); + return Status::Ok(); +} + +void MemTableRangeMerge::rotate_arena() +{ + arena_ = KeyArenaBuilder(limits_, metadata_); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/logical_page.h b/lib/crowdb-tree/src/btree/logical_page.h new file mode 100644 index 000000000..2f359f4dc --- /dev/null +++ b/lib/crowdb-tree/src/btree/logical_page.h @@ -0,0 +1,374 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_reservation.h" +#include "btree/backing_ref.h" +#include "btree/entry_key_arena.h" +#include "btree/merge_sources.h" +#include "btree/owned_key.h" +#include "btree/page_key_range.h" +#include "crowdb-tree/maptable/page.h" +#include "crowdb-tree/memtable/memtable.h" +#include "crowdb-tree/status.h" + +#include <limits> +#include <optional> +#include <span> + +namespace crowdb::tree +{ +struct EntryRef +{ + std::shared_ptr<const EntryKeyArena> key_arena; + size_t key_offset = 0; + size_t key_length = 0; + std::shared_ptr<const CellVersion> owner; + + [[nodiscard]] Slice key() const; + [[nodiscard]] Slice value() const; + [[nodiscard]] uint64_t slot() const; + [[nodiscard]] uint8_t flags() const; +}; + +using EntryAllocator = detail::ReservationAllocator<EntryRef>; +using EntryStorage = std::vector<EntryRef, EntryAllocator>; + +struct RunLimits +{ + size_t max_entries = 4096; + size_t max_key_bytes = 256 * 1024; + size_t key_block_bytes = 16 * 1024; +}; + +// The builder owns writable arena blocks; published entries expose only const +// blocks. Blocks never grow after the first view has been handed out. +class KeyArenaBuilder +{ + public: + explicit KeyArenaBuilder(RunLimits limits, std::shared_ptr<detail::AllocationReservation> metadata = {}) + : limits_(limits), + metadata_(std::move(metadata)) + { + } + + Status append(Slice key, std::shared_ptr<const CellVersion> owner, EntryRef *out); + + private: + RunLimits limits_; + std::shared_ptr<detail::AllocationReservation> metadata_; + size_t entries_ = 0; + size_t bytes_ = 0; + size_t allocated_ = 0; + size_t used_ = 0; + std::shared_ptr<EntryKeyArena> block_; +}; + +class OverlayBatch +{ + public: + static constexpr bool observe_retirement = true; + + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !entries_.empty(); ++i) { + entries_.pop_back(); + } + return entries_.empty(); + } + + static Status build(uint64_t id, std::vector<EntryRef> entries, const RunLimits &limits, + std::shared_ptr<const OverlayBatch> *out, + std::shared_ptr<detail::AllocationReservation> metadata = {}); + static Status build_owned(uint64_t id, EntryStorage entries, const RunLimits &limits, + std::shared_ptr<const OverlayBatch> *out); + + explicit OverlayBatch(std::shared_ptr<detail::AllocationReservation> metadata = {}) + : entries_(EntryAllocator(std::move(metadata))) + { + } + + [[nodiscard]] uint64_t id() const + { + return id_; + } + + [[nodiscard]] std::span<const EntryRef> entries() const + { + return entries_; + } + + [[nodiscard]] const EntryRef *find(Slice key) const; + [[nodiscard]] size_t lower_bound(Slice key) const; + + [[nodiscard]] uint64_t min_slot() const + { + return min_slot_; + } + + [[nodiscard]] uint64_t max_slot() const + { + return max_slot_; + } + + [[nodiscard]] size_t key_bytes() const + { + return key_bytes_; + } + + [[nodiscard]] bool has_overflow_values() const noexcept + { + return has_overflow_values_; + } + + private: + uint64_t id_ = 0; + uint64_t min_slot_ = UINT64_MAX; + uint64_t max_slot_ = 0; + size_t key_bytes_ = 0; + bool has_overflow_values_ = false; + EntryStorage entries_; +}; + +class OverlayBuilder +{ + public: + explicit OverlayBuilder(RunLimits limits, std::shared_ptr<detail::AllocationReservation> metadata = {}) + : limits_(limits), + metadata_(std::move(metadata)), + arena_(limits, metadata_), + entries_(EntryAllocator(metadata_)) + { + } + + Status append(Slice key, std::shared_ptr<const CellVersion> owner); + Status append(EntryRef entry); + Status finish(uint64_t id, std::shared_ptr<const OverlayBatch> *out); + + private: + RunLimits limits_; + std::shared_ptr<detail::AllocationReservation> metadata_; + KeyArenaBuilder arena_; + size_t key_bytes_ = 0; + EntryStorage entries_; +}; + +struct BaseRef +{ + uint64_t identity = 0; + uint64_t generation = 0; + PageKeyRange range{}; + std::optional<uint64_t> max_slot = std::nullopt; + // A resident owner can be an aliasing shared_ptr backed by a page pin. + std::shared_ptr<const LeafBase> resident = nullptr; + std::shared_ptr<const BackingRef> backing = nullptr; + [[nodiscard]] bool valid() const; +}; + +struct LogicalCell +{ + std::shared_ptr<const void> owner; + uint64_t slot = 0; + uint8_t flags = 0; + Slice value; + Slice encoded; + std::shared_ptr<const BackingRef> backing = nullptr; + + [[nodiscard]] bool tombstone() const + { + return (flags & kFlagTombstone) != 0; + } + + [[nodiscard]] bool overflow() const + { + return (flags & kFlagOverflow) != 0; + } + + static LogicalCell from_entry(const EntryRef &entry); + static LogicalCell from_base(const std::shared_ptr<const LeafBase> &base, size_t index, + std::shared_ptr<const BackingRef> backing = {}); +}; + +enum class LogicalLookupState : uint8_t { kMissing, kValue, kTombstone, kNeedsBase, kInvalid }; + +struct LogicalLookup +{ + LogicalLookupState state = LogicalLookupState::kMissing; + LogicalCell cell; + size_t base_index = 0; + Status status; +}; + +// Construct off to the side and publish as shared_ptr<const PageState>. No +// published container or frame may be mutated. Residency changes create a new +// descriptor while preserving base/content identity. +using OverlayAllocator = detail::ReservationAllocator<std::shared_ptr<const OverlayBatch>>; +using OverlayStorage = std::vector<std::shared_ptr<const OverlayBatch>, OverlayAllocator>; + +struct PageState +{ + + using BaseAllocator = detail::ReservationAllocator<BaseRef>; + using Bases = std::vector<BaseRef, BaseAllocator>; + + [[nodiscard]] static size_t allocation_bound() noexcept + { + return detail::shared_allocation_bound<PageState>(); + } + + bool retire_step() noexcept + { + size_t steps = 0; + for (; steps < 64 && !overlays.empty(); ++steps) { + overlays.pop_back(); + } + for (; steps < 64 && !bases.empty(); ++steps) { + bases.pop_back(); + } + return overlays.empty() && bases.empty(); + } + + explicit PageState(std::shared_ptr<detail::AllocationReservation> metadata = {}) + : bases(BaseAllocator(metadata)), + overlays(OverlayAllocator(std::move(metadata))) + { + } + + PageState(const PageState &source, std::shared_ptr<detail::AllocationReservation> metadata = {}) + : range(source.range), + route_generation(source.route_generation), + content_id(source.content_id), + representation_id(source.representation_id), + right_sibling(source.right_sibling), + bases(source.bases.begin(), source.bases.end(), BaseAllocator(metadata)), + overlays(source.overlays.begin(), source.overlays.end(), OverlayAllocator(std::move(metadata))) + { + } + + PageKeyRange range; + uint64_t route_generation = 0; + uint64_t content_id = 0; + uint64_t representation_id = 0; + uint64_t right_sibling = kInvalidPageId; + Bases bases; + // Stable source order breaks equal-slot ties; callers place overlays first. + OverlayStorage overlays; + + [[nodiscard]] LogicalLookup lookup(Slice key, uint64_t frontier = UINT64_MAX) const; +}; + +// Thread-bound merge operation. Its guards never cross an asynchronous suspend. +// next() returns independent key/cell owners that can cross threads freely. +class MemTableRangeMerge +{ + public: + MemTableRangeMerge(std::initializer_list<std::shared_ptr<MemTable>> sources, PageKeyRange range, uint64_t frontier, + uint64_t floor, RunLimits limits, size_t max_sources, + std::shared_ptr<detail::AllocationReservation> metadata = {}); + MemTableRangeMerge(std::span<const std::shared_ptr<MemTable>> sources, PageKeyRange range, uint64_t frontier, + uint64_t floor, RunLimits limits, size_t max_sources, + std::shared_ptr<detail::AllocationReservation> metadata = {}); + MemTableRangeMerge(const MemTableRangeMerge &) = delete; + MemTableRangeMerge &operator=(const MemTableRangeMerge &) = delete; + + [[nodiscard]] const Status &status() const + { + return status_; + } + + [[nodiscard]] bool valid() const; + // Borrowed from the current guarded source; copy before leaving this step. + [[nodiscard]] Slice key() const; + Status next(EntryRef *out); + // After publishing/finishing a bounded run, start another arena window + // without changing the source positions. Existing entries keep their blocks. + void rotate_arena(); + + [[nodiscard]] uint64_t candidates() const noexcept + { + return candidates_; + } + + private: + uint64_t candidates_ = 0; + std::vector<std::shared_ptr<MemTable>, detail::ReservationAllocator<std::shared_ptr<MemTable>>> owners_; + std::vector<EpochManager::Guard, detail::ReservationAllocator<EpochManager::Guard>> guards_; + PageKeyRange range_; + RunLimits limits_; + std::shared_ptr<detail::AllocationReservation> metadata_; + KeyArenaBuilder arena_; + std::vector<ConcurrentSkipList::Cursor, detail::ReservationAllocator<ConcurrentSkipList::Cursor>> cursors_; + std::vector<MergeSource, detail::ReservationAllocator<MergeSource>> sources_; + LoserTree merge_; + Status status_; +}; + +struct LogicalCursorStart +{ + OwnedKey key; + bool inclusive = true; +}; + +// Bounded merge of immutable runs and base ranges. Ordinary cold bases need +// a new page descriptor; range frames resume the same cursor after each load. +// max_sources controls the merge window rather than legal leaf contents. +class LogicalPageCursor +{ + public: + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t max_sources) noexcept; + LogicalPageCursor(std::shared_ptr<const PageState> page, PageKeyRange range, uint64_t frontier, size_t max_sources, + bool reverse = false, std::shared_ptr<detail::AllocationReservation> metadata = {}, + std::optional<LogicalCursorStart> start = {}); + ~LogicalPageCursor(); + + [[nodiscard]] const Status &status() const + { + return status_; + } + + [[nodiscard]] std::optional<size_t> needs_base() const + { + return needs_base_; + } + + [[nodiscard]] bool valid() const + { + return winner_.has_value(); + } + + [[nodiscard]] Slice key() const; + [[nodiscard]] LogicalCell cell() const; + void next(); + + private: + struct Source + { + const OverlayBatch *run = nullptr; + const BaseRef *base = nullptr; + size_t index = 0; + bool exhausted = false; + }; + + [[nodiscard]] Slice source_key(const Source &source) const; + [[nodiscard]] LogicalCell source_cell(const Source &source) const; + [[nodiscard]] uint64_t source_slot(const Source &source) const; + [[nodiscard]] size_t source_size(const Source &source) const; + void position(Source *source); + void step(Source *source); + void select(); + void seek(Source *source, std::optional<Slice> bound, bool exclusive); + void select_windowed(std::optional<Slice> after); + [[nodiscard]] std::pair<std::optional<Slice>, bool> initial_bound() const noexcept; + + std::shared_ptr<const PageState> page_; + PageKeyRange range_; + uint64_t frontier_; + bool reverse_; + bool windowed_ = false; + detail::ReservedVector<Source> sources_; + std::optional<LogicalCursorStart> start_; + std::optional<size_t> winner_; + std::optional<size_t> needs_base_; + Status status_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/logical_page_cursor.cpp b/lib/crowdb-tree/src/btree/logical_page_cursor.cpp new file mode 100644 index 000000000..1fca6efb2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/logical_page_cursor.cpp @@ -0,0 +1,188 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/logical_page.h" + +#include <algorithm> + +namespace crowdb::tree +{ +std::optional<size_t> LogicalPageCursor::allocation_bound(size_t max_sources) noexcept +{ + return detail::AllocationBound(sizeof(LogicalPageCursor)).add_product(max_sources, sizeof(Source)).value(); +} + +LogicalPageCursor::~LogicalPageCursor() = default; + +LogicalPageCursor::LogicalPageCursor(std::shared_ptr<const PageState> page, PageKeyRange range, uint64_t frontier, + size_t max_sources, bool reverse, + std::shared_ptr<detail::AllocationReservation> metadata, + std::optional<LogicalCursorStart> start) + : page_(std::move(page)), + range_(std::move(range)), + frontier_(frontier), + reverse_(reverse), + sources_(detail::ReservationAllocator<Source>(metadata)), + start_(std::move(start)) +{ + if (page_ == nullptr) { + status_ = Status::invalid_argument("logical cursor needs a page owner"); + return; + } + if (max_sources == 0) { + status_ = Status::invalid_argument("logical cursor needs a nonzero source window"); + return; + } + windowed_ = page_->overlays.size() > max_sources || + page_->bases.size() > max_sources - std::min(max_sources, page_->overlays.size()); + if (!range_.overlaps(page_->range)) { + return; + } + if (page_->range.lower.has_value() && + (!range_.lower.has_value() || Slice(*range_.lower) < Slice(*page_->range.lower))) { + range_.lower = page_->range.lower; + } + if (page_->range.upper.has_value() && + (!range_.upper.has_value() || Slice(*range_.upper) > Slice(*page_->range.upper))) { + range_.upper = page_->range.upper; + } + sources_.reserve(windowed_ ? 1 : page_->overlays.size() + page_->bases.size()); + for (const auto &run : page_->overlays) { + if (run == nullptr) { + status_ = Status::invalid_argument("logical cursor overlay has no owner"); + return; + } + if (!windowed_) { + sources_.push_back({.run = run.get(), .base = nullptr, .index = 0, .exhausted = false}); + } + } + for (size_t i = 0; i < page_->bases.size(); ++i) { + const auto &base = page_->bases[i]; + if (!base.range.overlaps(range_)) { + continue; + } + if (!base.valid()) { + status_ = Status::corruption("logical base has neither resident nor reloadable owner"); + return; + } + if (base.resident == nullptr) { + needs_base_ = i; + return; + } + if (!windowed_) { + sources_.push_back({.run = nullptr, .base = &base, .index = 0, .exhausted = false}); + } + } + if (windowed_) { + select_windowed({}); + return; + } + for (auto &source : sources_) { + const auto [bound, exclusive] = initial_bound(); + seek(&source, bound, exclusive); + } + select(); +} + +size_t LogicalPageCursor::source_size(const Source &source) const +{ + return source.run != nullptr ? source.run->entries().size() : source.base->resident->count(); +} + +Slice LogicalPageCursor::source_key(const Source &source) const +{ + return source.run != nullptr ? source.run->entries()[source.index].key() : source.base->resident->key(source.index); +} + +LogicalCell LogicalPageCursor::source_cell(const Source &source) const +{ + return source.run != nullptr ? LogicalCell::from_entry(source.run->entries()[source.index]) + : LogicalCell::from_base(source.base->resident, source.index, source.base->backing); +} + +uint64_t LogicalPageCursor::source_slot(const Source &source) const +{ + return source.run != nullptr ? source.run->entries()[source.index].slot() + : CellView(source.base->resident->cell(source.index)).slot(); +} + +void LogicalPageCursor::step(Source *source) +{ + if (reverse_) { + source->exhausted = source->index == 0; + if (!source->exhausted) { + --source->index; + } + } + else { + ++source->index; + source->exhausted = source->index == source_size(*source); + } +} + +void LogicalPageCursor::position(Source *source) +{ + while (!source->exhausted) { + const Slice key = source_key(*source); + if ((!reverse_ && range_.upper.has_value() && key >= Slice(*range_.upper)) || + (reverse_ && range_.lower.has_value() && key < Slice(*range_.lower))) { + source->exhausted = true; + return; + } + if (range_.contains(key) && (source->base == nullptr || source->base->range.contains(key)) && + source_slot(*source) <= frontier_) { + return; + } + step(source); + } +} + +void LogicalPageCursor::select() +{ + winner_.reset(); + for (size_t i = 0; i < sources_.size(); ++i) { + if (sources_[i].exhausted) { + continue; + } + if (!winner_.has_value()) { + winner_ = i; + continue; + } + const auto &candidate = sources_[i]; + const auto &winner = sources_[*winner_]; + const int order = source_key(candidate).compare(source_key(winner)); + if ((reverse_ ? order > 0 : order < 0) || (order == 0 && source_slot(candidate) > source_slot(winner))) { + winner_ = i; + } + } +} + +Slice LogicalPageCursor::key() const +{ + return winner_.has_value() ? source_key(sources_[*winner_]) : Slice(); +} + +LogicalCell LogicalPageCursor::cell() const +{ + return winner_.has_value() ? source_cell(sources_[*winner_]) : LogicalCell(); +} + +void LogicalPageCursor::next() +{ + if (!winner_.has_value()) { + return; + } + const Slice emitted = key(); + if (windowed_) { + select_windowed(emitted); + return; + } + for (auto &source : sources_) { + if (!source.exhausted && source_key(source) == emitted) { + step(&source); + position(&source); + } + } + select(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/logical_page_cursor_window.cpp b/lib/crowdb-tree/src/btree/logical_page_cursor_window.cpp new file mode 100644 index 000000000..ab4cffe00 --- /dev/null +++ b/lib/crowdb-tree/src/btree/logical_page_cursor_window.cpp @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" + +namespace crowdb::tree +{ +std::pair<std::optional<Slice>, bool> LogicalPageCursor::initial_bound() const noexcept +{ + auto bound = reverse_ ? range_.upper_slice() : range_.lower_slice(); + if (start_ && (!bound || (reverse_ ? Slice(start_->key) < *bound : Slice(start_->key) >= *bound))) { + return {Slice(start_->key), !start_->inclusive}; + } + return {bound, reverse_}; +} + +void LogicalPageCursor::seek(Source *source, std::optional<Slice> bound, bool exclusive) +{ + if (source->base != nullptr) { + const auto base_bound = reverse_ ? source->base->range.upper_slice() : source->base->range.lower_slice(); + if (base_bound && (!bound || (reverse_ ? *base_bound < *bound : *base_bound > *bound))) { + bound = base_bound; + exclusive = reverse_; + } + } + size_t lower = 0; + size_t upper = source_size(*source); + if (bound) { + while (lower < upper) { + source->index = lower + ((upper - lower) / 2); + const auto candidate = source_key(*source); + if (candidate < *bound || (candidate == *bound && (reverse_ ? !exclusive : exclusive))) { + lower = source->index + 1; + } + else { + upper = source->index; + } + } + } + else { + lower = reverse_ ? upper : 0; + } + source->exhausted = reverse_ ? lower == 0 : lower == source_size(*source); + source->index = reverse_ && lower != 0 ? lower - 1 : lower; + position(source); +} + +void LogicalPageCursor::select_windowed(std::optional<Slice> after) +{ + // Keep only the winning position. Immutable run/base indices can be sought + // again without materializing values or allocating a cursor per source. + auto [bound, exclusive] = initial_bound(); + if (after) { + bound = after; + exclusive = true; + } + Source best; + bool found = false; + auto consider = [&](Source candidate) { + seek(&candidate, bound, exclusive); + if (candidate.exhausted) { + return; + } + const auto order = found ? source_key(candidate).compare(source_key(best)) : 0; + if (!found || (reverse_ ? order > 0 : order < 0) || + (order == 0 && source_slot(candidate) > source_slot(best))) { + best = candidate; + found = true; + } + }; + for (const auto &run : page_->overlays) { + consider({.run = run.get()}); + } + for (const auto &base : page_->bases) { + if (base.range.overlaps(range_)) { + consider({.base = &base}); + } + } + sources_.clear(); + winner_.reset(); + if (found) { + sources_.push_back(best); + winner_ = 0; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/maintenance_cold.cpp b/lib/crowdb-tree/src/btree/maintenance_cold.cpp new file mode 100644 index 000000000..243e14ebe --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_cold.cpp @@ -0,0 +1,212 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "maintenance_operation_internal.h" +#include "maintenance_split_layout.h" +#include "page_load.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +namespace +{ +size_t route_child(const InnerPageState &inner, const PageState &leaf) +{ + return leaf.range.lower.has_value() + ? static_cast<size_t>( + std::upper_bound(inner.separators.begin(), inner.separators.end(), *leaf.range.lower) - + inner.separators.begin()) + : 0; +} + +bool needs_load(const PublishedTreeState &view, uint64_t target, const PageState &leaf, size_t max_path, + bool inner_repair) +{ + if (!inner_repair && std::ranges::any_of(leaf.bases, [](const auto &base) { return !base.resident; })) { + return true; + } + auto pid = view.root_page_id; + for (size_t depth = 0; pid != target && depth < max_path; ++depth) { + const auto page = view.mapping.get(pid); + if (!page || !page->inner) { + return page && page->inner_backing && page->inner_backing->readable(); + } + const auto child = route_child(*page->inner, leaf); + if (child >= page->inner->children.size()) { + return false; + } + if (inner_repair && page->level > 1) { + for (const auto at : + {child == 0 ? child : child - 1, std::min(child + 1, page->inner->children.size() - 1)}) { + const auto sibling = view.mapping.get(page->inner->children[at]); + if (sibling && !sibling->inner && sibling->inner_backing) { + return true; + } + } + } + pid = page->inner->children[child]; + } + return false; +} +} // namespace + +bool MaintenanceOperation::begin_cold(std::shared_ptr<const PublishedTreeState> view, + std::shared_ptr<const PublishedPage> page) +{ + auto &started = phase == Phase::InnerMerge ? cold_inner_started : cold_started; + if (started || !loader || + !needs_load(*view, (*candidates)[index].pid, *page->leaf, publication.max_path_pages, + phase == Phase::InnerMerge)) { + return false; + } + auto pending = reserved_shared<Cold>(publication.metadata, publication.metadata); + pending->replacements.reserve((3 * publication.max_path_pages) + 1); + pending->siblings.reserve(2 * publication.max_path_pages); + pending->leaf = reserved_shared<PageState>(publication.metadata, *page->leaf, publication.metadata); + pending->route_pid = view->root_page_id; + pending->view = std::move(view); + pending->expected = std::move(page); + started = true; + cold = std::move(pending); + schedule(); + return true; +} + +bool MaintenanceOperation::cold_route() +{ + auto &pending = *cold; + const auto target = (*candidates)[index].pid; + while (pending.route_pid != target && pending.route_steps < publication.max_path_pages) { + const auto original = cold_page(pending.route_pid); + if (!original || original->level == 0) { + finish(Status::corruption("maintenance cold route has no inner page")); + return false; + } + auto inner = original->inner; + if (!inner) { + return load_cold_inner(pending.route_pid); + } + const auto child = route_child(*inner, *pending.leaf); + if (inner->children.size() != inner->separators.size() + 1 || child >= inner->children.size()) { + finish(Status::corruption("maintenance cold route has invalid child bounds")); + return false; + } + if (phase == Phase::InnerMerge && original->level > 1) { + if (child != 0) { + pending.siblings.push_back(inner->children[child - 1]); + } + if (child + 1 < inner->children.size()) { + pending.siblings.push_back(inner->children[child + 1]); + } + } + pending.route_pid = inner->children[child]; + ++pending.route_steps; + } + if (pending.route_pid != target) { + finish(Status::resource_exhausted("maintenance cold route exceeds its depth bound")); + return false; + } + return true; +} + +void MaintenanceOperation::cold_step() +{ + auto &pending = *cold; + if (pending.waiting) { + if (!pending.ready.load(std::memory_order_acquire)) { + return; + } + if (!pending.status.ok() || (pending.loading_inner ? !pending.loaded_inner : !pending.loaded)) { + finish(pending.status.ok() ? Status::corruption("maintenance loader returned no page") + : std::move(pending.status)); + return; + } + if (pending.loading_inner) { + const auto original = pending.view->mapping.get(pending.loading_pid); + auto page = reserved_shared<PublishedPage>(publication.metadata, *original); + page->inner = std::move(pending.loaded_inner); + pending.replacements.push_back( + {.page_id = pending.loading_pid, .expected = original, .replacement = std::move(page)}); + } + else { + pending.leaf->bases[pending.pending_base].resident = std::move(pending.loaded); + } + pending.loading_inner = false; + pending.waiting = false; + } + if (!cold_route()) { + return; + } + if (phase == Phase::InnerMerge) { + if (cold_siblings()) { + cold_plan(); + } + return; + } + while (pending.next_base < pending.leaf->bases.size()) { + const auto index = pending.next_base++; + const auto &base = pending.leaf->bases[index]; + if (base.resident) { + continue; + } + if (!base.backing || !base.backing->readable()) { + finish(Status::corruption("maintenance cold base has no readable owner")); + return; + } + pending.pending_base = index; + pending.waiting = true; + pending.ready.store(false, std::memory_order_release); + auto self = shared_from_this(); + loader->load_leaf(base, [self, pending = cold](Status status, auto leaf) { + pending->status = std::move(status); + pending->loaded = std::move(leaf); + pending->ready.store(true, std::memory_order_release); + self->schedule(); + }); + return; + } + cold_plan(); +} + +void MaintenanceOperation::cold_plan() +{ + const auto pid = (*candidates)[index].pid; + auto page = reserved_shared<PublishedPage>(publication.metadata, *cold->expected); + if (phase != Phase::InnerMerge) { + page->leaf = std::move(cold->leaf); + cold->replacements.push_back({.page_id = pid, .expected = cold->expected, .replacement = page}); + } + auto resolved = reserved_shared<PublishedTreeState>(publication.metadata, *cold->view); + MappingView::CopyStats copied; + std::sort(cold->replacements.begin(), cold->replacements.end(), + [](const auto &left, const auto &right) { return left.page_id < right.page_id; }); + resolved->mapping = cold->view->mapping.replace(cold->replacements, copied, publication.metadata); + auto result = plan(resolved, page); + // Loaded descriptors are private scratch. Compare every original page so a + // concurrent route or writer change invalidates the complete transaction. + for (auto &change : result.transaction.pages) { + for (const auto &loaded : cold->replacements) { + if (change.page_id == loaded.page_id) { + change.expected = loaded.expected; + if (change.replacement == loaded.replacement) { + change.replacement = loaded.expected; + } + break; + } + } + } + cold.reset(); + if (!result.status.ok()) { + finish(std::move(result.status)); + return; + } + if (result.readiness != MaintenanceReadiness::kReady || !result.changed()) { + advance(); + return; + } + if (phase == Phase::InnerMerge) { + ++steps; + } + submit(std::move(result)); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_cold_siblings.cpp b/lib/crowdb-tree/src/btree/maintenance_cold_siblings.cpp new file mode 100644 index 000000000..3d7cdea28 --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_cold_siblings.cpp @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "maintenance_operation_internal.h" +#include "page_load.h" + +namespace crowdb::tree::detail +{ +std::shared_ptr<const PublishedPage> MaintenanceOperation::cold_page(uint64_t pid) const +{ + for (const auto &page : cold->replacements) { + if (page.page_id == pid) { + return page.replacement; + } + } + return cold->view->mapping.get(pid); +} + +bool MaintenanceOperation::load_cold_inner(uint64_t pid) +{ + const auto page = cold_page(pid); + if (!page || !page->inner_backing || !page->inner_backing->readable() || + cold->replacements.size() >= 3 * publication.max_path_pages) { + finish(Status::corruption("maintenance cold inner exceeds its captured backing bound")); + return false; + } + cold->loading_inner = true; + cold->loading_pid = pid; + cold->waiting = true; + cold->ready.store(false, std::memory_order_release); + auto self = shared_from_this(); + loader->load_inner(page->inner_backing, [self, pending = cold](Status status, auto inner) { + pending->status = std::move(status); + pending->loaded_inner = std::move(inner); + pending->ready.store(true, std::memory_order_release); + self->schedule(); + }); + return false; +} + +bool MaintenanceOperation::cold_siblings() +{ + while (cold->next_sibling < cold->siblings.size()) { + const auto pid = cold->siblings[cold->next_sibling++]; + const auto page = cold_page(pid); + if (!page || page->level == 0) { + finish(Status::corruption("inner maintenance sibling is not an inner page")); + return false; + } + if (!page->inner) { + return load_cold_inner(pid); + } + } + return true; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_inner_merge.cpp b/lib/crowdb-tree/src/btree/maintenance_inner_merge.cpp new file mode 100644 index 000000000..810088e58 --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_inner_merge.cpp @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "btree/maintenance_publication_internal.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +namespace +{ +bool fits_merged(const InnerPageState &left, const InnerPageState &right, const OwnedKey &separator, + const MaintenancePublicationOptions &options) +{ + const auto count = AllocationBound(left.children.size()).add(right.children.size()).value(); + if (!count || *count > options.max_inner_children) { + return false; + } + AllocationBound bytes(kFrameHeaderSize + kFrameTrailerSize); + bytes.add_product(*count, sizeof(uint64_t)).add_product(*count - 1, kInnerSlotSize).add(separator.size()); + for (const auto &key : left.separators) { + bytes.add(key.size()); + } + for (const auto &key : right.separators) { + bytes.add(key.size()); + } + return bytes.value().has_value() && *bytes.value() <= options.max_inner_bytes; +} +} // namespace + +Status MaintenanceBuilder::merge_inner_pair(const MaintenancePathEntry &ancestor, uint64_t target, bool *changed) +{ + const auto parent = lookup(ancestor.pid); + const auto node = lookup(target); + if (!parent || !parent->inner || !node || !node->inner || parent->level != node->level + 1) { + return Status::unavailable("inner merge needs a resident guarded route"); + } + const auto &route = *parent->inner; + const auto found = std::find(route.children.begin(), route.children.end(), target); + if (found == route.children.end()) { + return Status::corruption("inner merge target is outside its parent"); + } + if (route.children.size() < 2 || node->inner->separators.size() >= options.inner_merge_keys) { + return Status::Ok(); + } + const size_t index = static_cast<size_t>(found - route.children.begin()); + const size_t left_index = index == 0 ? 0 : index - 1; + const auto left_pid = route.children[left_index]; + const auto right_pid = route.children[left_index + 1]; + const auto left = lookup(left_pid); + const auto right = lookup(right_pid); + if (!left || !right || left->level != node->level || right->level != node->level) { + return Status::corruption("inner merge sibling has an invalid level"); + } + if (!left->inner || !right->inner) { + // A later maintenance window loads the bounded route and its siblings. + return Status::Ok(); + } + if (!fits_merged(*left->inner, *right->inner, route.separators[left_index], options)) { + return Status::Ok(); + } + InnerPageState merged(*left->inner, options.metadata); + merged.children.insert(merged.children.end(), right->inner->children.begin(), right->inner->children.end()); + merged.separators.push_back(route.separators[left_index]); + merged.separators.insert(merged.separators.end(), right->inner->separators.begin(), right->inner->separators.end()); + MaintenanceChildren output(options.metadata); + auto status = partition_inner(left_pid, left->level, merged, &output); + if (!status.ok()) { + return status; + } + status = replace(right_pid, {}); + if (!status.ok()) { + return status; + } + InnerPageState reduced(route, options.metadata); + reduced.children.erase(reduced.children.begin() + static_cast<ptrdiff_t>(left_index + 1)); + reduced.separators.erase(reduced.separators.begin() + static_cast<ptrdiff_t>(left_index)); + MaintenanceChildren parents(options.metadata); + status = partition_inner(ancestor.pid, parent->level, reduced, &parents); + if (status.ok()) { + ++inner_merges_; + } + *changed = *changed || status.ok(); + return status; +} + +Status MaintenanceBuilder::merge_underfull(const MaintenancePath &path, bool *changed) +{ + for (size_t depth = path.size(); depth > 1; --depth) { + auto status = merge_inner_pair(path[depth - 2], path[depth - 1].pid, changed); + if (!status.ok()) { + return status; + } + } + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_leaf_merge.cpp b/lib/crowdb-tree/src/btree/maintenance_leaf_merge.cpp new file mode 100644 index 000000000..1e23e5b2c --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_leaf_merge.cpp @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "maintenance_publication_internal.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +MaintenanceLeafMerge plan_published_leaf_merge(const PublishedPage &left, const PublishedPage &right, + const LeafMergeInput &input, const PageMaintenanceLimits &limits, + PageVersion version, uint64_t first_run_id) +{ + MaintenanceLeafMerge result; + const auto &a = left.metrics; + const auto &b = right.metrics; + if (left.leaf != input.left || right.leaf != input.right || limits.merge_target_bytes >= limits.target_page_bytes || + limits.target_page_bytes > limits.primary_page_bytes) { + result.logical.status = Status::invalid_argument("merge metrics must belong to the exact bounded source pages"); + return result; + } + if (a.page_size_known && a.kv_known && b.page_size_known && b.kv_known) { + constexpr size_t overhead = kFrameHeaderSize + kFrameTrailerSize; + if (a.page_size_bytes < overhead || b.page_size_bytes < overhead) { + result.logical.status = Status::corruption("exact leaf size is smaller than its frame overhead"); + return result; + } + if (std::min(a.page_size_bytes, b.page_size_bytes) >= limits.merge_trigger_bytes || + a.page_size_bytes > limits.merge_target_bytes || + b.page_size_bytes - overhead > limits.merge_target_bytes - a.page_size_bytes) { + return result; + } + if (b.kv_count > UINT64_MAX - a.kv_count || b.kv_bytes > UINT64_MAX - a.kv_bytes) { + result.logical.status = Status::resource_exhausted("merged leaf statistics overflow"); + return result; + } + result.logical = join_leaf_ranges(input, limits, version); + result.primary_bytes = a.page_size_bytes + b.page_size_bytes - overhead; + result.statistics = LeafLayoutStats{.kv_count = a.kv_count + b.kv_count, .kv_bytes = a.kv_bytes + b.kv_bytes}; + return result; + } + result.logical = plan_leaf_merge(input, limits, version, first_run_id); + if (!result.logical.status.ok() || !result.logical.page) { + return result; + } + auto layout = plan_leaf_layout(result.logical.page, limits); + result.logical.status = std::move(layout.status); + result.logical.readiness = layout.readiness; + result.logical.loads = std::move(layout.loads); + if (result.logical.status.ok() && layout.readiness == MaintenanceReadiness::kReady) { + if (layout.primary_bytes.size() != 1) { + result.logical.status = Status::corruption("accepted merge must have a complete single-page layout"); + } + else { + result.primary_bytes = layout.primary_bytes.front(); + } + } + return result; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_operation.cpp b/lib/crowdb-tree/src/btree/maintenance_operation.cpp new file mode 100644 index 000000000..cbab7843f --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_operation.cpp @@ -0,0 +1,393 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "crowdb-common/log.h" +#include "maintenance_operation_internal.h" +#include "maintenance_split_layout.h" +#include "page_cache_residency.h" + +#include <algorithm> +#include <cassert> +#include <exception> +#include <stdexcept> + +namespace crowdb::tree::detail +{ +namespace +{ +std::atomic<uint64_t> next_maintenance_invocation{1}; +} + +MaintenanceOperation::MaintenanceOperation(std::shared_ptr<MaintenanceFactory> owner, uint64_t tree_generation, + uint64_t dirty_pages, FlushBatch::MaintenanceComplete completion, + std::shared_ptr<AllocationReservation> reservation) + : factory(std::move(owner)), + loader(std::exchange(factory->loader, {})), + done(std::move(completion)), + publication(factory->options.publication), + generation(tree_generation), + invocation(next_maintenance_invocation.fetch_add(1, std::memory_order_relaxed)), + dirty(dirty_pages), + queued_resources(reservation, TreeResourceMetric::MaintenanceQueued), + active_resources(reservation, TreeResourceMetric::MaintenanceInflight), + outputs(ReservationAllocator<uint64_t>(reservation)) +{ + if (!loader) { + loader = factory->retry_loader.lock(); + } + publication.metadata = std::move(reservation); + publication.pages.metadata = publication.metadata; + activity_slot = factory->publisher.start_maintenance(tree_metrics_now_ns()); +} + +MaintenanceOperation::~MaintenanceOperation() +{ + if (activity_slot != PublicationCoordinator::kMaintenanceSlots) { + factory->publisher.finish_maintenance(activity_slot); + } +} + +void MaintenanceOperation::schedule() noexcept +{ + wake_requested.store(true, std::memory_order_release); + if (finished.load(std::memory_order_acquire) || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + queued_resources.begin(1); + try { + if (factory->cpu_progress) { + factory->cpu_progress->request(cpu_node, lifetime); + return; + } + if (factory->runtime.try_submit(factory->lane, &MaintenanceOperation::run, this).ok()) { + return; + } + } + catch (const std::bad_alloc &) { + scheduled.exchange(false, std::memory_order_acq_rel); + finish(target_view ? Status::resource_exhausted() : Status::Ok()); + return; + } + scheduled.exchange(false, std::memory_order_acq_rel); + finish(target_view ? Status::unavailable("split continuation unavailable") : Status::Ok()); +} + +void MaintenanceOperation::run(void *context) noexcept +{ + auto self = static_cast<MaintenanceOperation *>(context)->weak_from_this().lock(); + if (!self) { + std::terminate(); + } + self->wake_requested.exchange(false, std::memory_order_acq_rel); + self->queued_resources.reset(); + if (!self->metrics_started) { + self->metrics_started = true; + self->active_resources.begin(1); + } + try { + self->step(); + } + catch (const std::bad_alloc &) { + self->finish(self->target_view ? Status::resource_exhausted() : Status::Ok()); + } + catch (const std::exception &) { + self->finish(Status::internal_error()); + } + self->scheduled.exchange(false, std::memory_order_acq_rel); + if (self->wake_requested.load(std::memory_order_acquire) && !self->finished.load(std::memory_order_acquire)) { + self->schedule(); + } +} + +void MaintenanceOperation::step() +{ + if (target_stopped.load(std::memory_order_acquire)) { + if (!cold || !cold->waiting || cold->ready.load(std::memory_order_acquire)) { + finish(Status::unavailable("checkpoint split cancelled")); + } + return; + } + if (cold) { + cold_step(); + return; + } + if (!initialized) { + initialize(); + return; + } + if (phase == Phase::Collect) { + collect(); + return; + } + if (steps == factory->options.max_steps || candidates->empty()) { + finish(Status::Ok()); + return; + } + if (index == candidates->size()) { + if (phase == Phase::Merge) { + finish(Status::Ok()); + return; + } + if (phase == Phase::InnerMerge) { + phase = Phase::Analyze; + } + else if (phase == Phase::Analyze) { + phase = Phase::Coalesce; + } + else if (phase == Phase::Coalesce) { + phase = Phase::ClassifyMerge; + } + else { + phase = Phase::Merge; + } + index = 0; + } + process(); +} + +void MaintenanceOperation::initialize() +{ + const auto &options = factory->options; + if (!factory->lane || options.visits_per_step == 0 || options.max_visits == 0 || options.max_candidates == 0 || + options.max_steps == 0 || options.publication.metadata) { + finish(Status::invalid_argument("maintenance needs bounded work and fresh reclaim credit")); + return; + } + candidates = + reserved_shared<Candidates>(publication.metadata, ReservationAllocator<Candidate>(publication.metadata)); + candidates->reserve(options.max_candidates); + const auto view = capture_l1(); + if (view->generation != generation) { + finish(Status::unavailable("maintenance generation changed")); + return; + } + if (view->root_page_id == kInvalidPageId || view->mapping.empty()) { + finish(Status::invalid_argument("maintenance requires an initialized tree")); + return; + } + if (!view->mapping.get(view->root_page_id) || view->next_page_id == 0 || + view->next_page_id > MappingView::kPageLimit) { + finish(Status::corruption("maintenance root is missing")); + return; + } + if (target_view) { + const auto current = factory->publisher.capture(); + if (current->generation != generation || current->l1_seq != target_view->l1_seq) { + finish(Status::unavailable("checkpoint split view changed")); + return; + } + candidates->push_back({.pid = target_pid}); + initialized = true; + phase = Phase::Analyze; + schedule(); + return; + } + scan_limit = view->next_page_id; + // Each flush creates a fresh factory. Keep fairness on the Tree owner so + // bounded collection does not repeatedly start at the leftmost pages. + scan_pid = factory->publisher.maintenance_scan_pid() % scan_limit; + initialized = true; + schedule(); +} + +std::shared_ptr<const PublishedTreeState> MaintenanceOperation::capture_l1() +{ + if (target_view) { + return target_view; + } + auto current = factory->publisher.capture(); + dirty = current->stats.dirty_pages; + auto view = reserved_shared<PublishedTreeState>(publication.metadata, *current); + view->sources.reset(); + return view; +} + +void MaintenanceOperation::collect() +{ + const auto view = capture_l1(); + if (view->generation != generation) { + finish(Status::unavailable("maintenance generation changed")); + return; + } + const auto &options = factory->options; + const auto limit = std::min<uint64_t>(options.max_visits, scan_limit); + size_t visited = 0; + while (visits < limit && visited < options.visits_per_step && candidates->size() < options.max_candidates) { + const auto pid = scan_pid; + scan_pid = (scan_pid + 1) % scan_limit; + ++visits; + ++visited; + const auto page = view->mapping.get(pid); + if (!page) { + continue; + } + if ((page->level == 0 && (!page->leaf || page->inner)) || + (page->level != 0 && + (page->leaf || (page->inner ? page->inner->children.empty() || + page->inner->separators.size() + 1 != page->inner->children.size() + : !page->inner_backing || !page->inner_backing->readable())))) { + finish(Status::corruption("maintenance found an invalid page shape")); + return; + } + const auto right = page->leaf ? view->mapping.get(page->leaf->right_sibling) : nullptr; + // The right leaf can become small during this operation's later + // analyze/coalesce phases. Retain its left pair before that happens. + const bool shrinking_right = + right && right->leaf && + (!right->metrics.page_size_known || !right->metrics.kv_known || right->metrics.consolidate_candidate || + maintenance_needs_coalesce(*right->leaf, publication.pages.max_sources) || + right->metrics.page_size_bytes < publication.pages.merge_trigger_bytes); + if (page->level == 0 && + ((publication.inner_merge_keys != 0 && view->mapping.get(view->root_page_id)->level > 1) || + shrinking_right || !page->metrics.page_size_known || !page->metrics.kv_known || + page->metrics.split_candidate || page->metrics.page_size_bytes > publication.pages.primary_page_bytes || + page->metrics.consolidate_candidate || page->metrics.merge_candidate || + page->metrics.page_size_bytes < publication.pages.merge_trigger_bytes || + maintenance_needs_coalesce(*page->leaf, publication.pages.max_sources))) { + candidates->push_back({.pid = pid}); + } + } + factory->publisher.set_maintenance_scan_pid(scan_pid); + if (visits == limit || candidates->size() == options.max_candidates) { + phase = Phase::InnerMerge; + } + schedule(); +} + +void MaintenanceOperation::advance() +{ + if (target_view) { + finish(Status::resource_exhausted("checkpoint split made no structural progress")); + return; + } + ++index; + schedule(); +} + +void MaintenanceOperation::finish(Status status) noexcept +{ + if (finished.exchange(true, std::memory_order_acq_rel)) { + return; + } + if (activity_slot != PublicationCoordinator::kMaintenanceSlots) { + factory->publisher.finish_maintenance(activity_slot); + activity_slot = PublicationCoordinator::kMaintenanceSlots; + } + cold.reset(); + loader.reset(); + queued_resources.reset(); + active_resources.reset(); + if (action_metrics) { + (void)action_metrics->finish(status.ok() ? TreeOutcome::Retry : TreeOutcome::Error, tree_metrics_now_ns()); + } + try { + dirty = factory->publisher.capture()->stats.dirty_pages; + } + catch (const std::bad_alloc &) { + if (status.ok()) { + status = Status::resource_exhausted(); + } + } + if (publication.metadata) { + publication.metadata->seal(); + } + if (publication.snapshot_metadata) { + publication.snapshot_metadata->seal(); + } + auto self = lifetime; + lifetime.reset(); + factory->active.clear(std::memory_order_release); + try { + done(std::move(status), dirty); + } + catch (...) { + CRB_LOG_ERROR("maintenance receiver threw after terminal; operation " + "ownership was released"); + } +} +} // namespace crowdb::tree::detail + +namespace crowdb::tree +{ +FlushBatch::Maintenance make_page_maintenance(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + PageMaintenanceOperationOptions options) +{ + return make_page_maintenance(publisher, runtime, std::move(lane), std::move(options), {}, {}); +} + +FlushBatch::Maintenance make_page_maintenance(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + PageMaintenanceOperationOptions options, + std::shared_ptr<PageLoader> loader, + std::shared_ptr<ApplyDispatcher> progress) +{ + auto prepaid = options.prepaid; + if (prepaid && prepaid->category() != detail::BudgetClass::kReclaim) { + throw std::invalid_argument("maintenance prepaid credit must use the reclaim partition"); + } + auto factory = detail::reserved_shared<detail::MaintenanceFactory>( + prepaid, publisher, runtime, std::move(lane), std::move(options), std::move(loader), std::move(progress)); + return [factory](auto view, auto done) { + if (!view) { + done(Status::invalid_argument("maintenance needs a captured view"), 0); + return; + } + if (factory->active.test_and_set(std::memory_order_acquire)) { + done(Status::unavailable("maintenance already active"), view->stats.dirty_pages); + return; + } + auto credit = std::exchange(factory->options.prepaid, {}); + Status status; + std::shared_ptr<detail::MaintenanceOperation> operation; + try { + if (!credit && factory->options.acquire_window) { + status = factory->options.acquire_window(view->mapping, &factory->options.publication, &credit); + } + else if (!credit) { + if (factory->options.acquire_snapshot) { + status = factory->options.acquire_snapshot(&factory->options.publication.snapshot_metadata); + } + if (status.ok()) { + status = factory->options.acquire ? factory->options.acquire(&credit) + : Status::invalid_argument("maintenance needs reclaim credit"); + } + } + if (status.ok() && credit && credit->category() != detail::BudgetClass::kReclaim) { + status = Status::invalid_argument("maintenance credit must use the reclaim partition"); + } + if (status.ok() && credit && credit->available() != 0) { + operation = detail::reserved_shared<detail::MaintenanceOperation>( + credit, factory, view->generation, view->stats.dirty_pages, done, credit); + if (operation->activity_slot == PublicationCoordinator::kMaintenanceSlots) { + operation.reset(); + } + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted(); + } + catch (const std::exception &) { + status = Status::internal_error("maintenance credit acquisition failed"); + } + if (!operation) { + factory->loader.reset(); + if (factory->options.publication.snapshot_metadata) { + factory->options.publication.snapshot_metadata->seal(); + } + if (credit && credit->category() == detail::BudgetClass::kReclaim) { + credit->seal(); + } + factory->active.clear(std::memory_order_release); + done(status.ok() || status.code() == Code::kResourceExhausted || status.code() == Code::kUnavailable + ? Status::Ok() + : std::move(status), + view->stats.dirty_pages); + return; + } + operation->lifetime = operation; + view.reset(); + operation->schedule(); + }; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/maintenance_operation.h b/lib/crowdb-tree/src/btree/maintenance_operation.h new file mode 100644 index 000000000..e429af93e --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_operation.h @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "flush_batch.h" +#include "maintenance_publication.h" + +namespace crowdb::tree +{ +struct PageMaintenanceOperationOptions +{ + using AcquireReservation = detail::InlineFunction<Status(std::shared_ptr<detail::AllocationReservation> *)>; + using AcquireWindow = detail::InlineFunction<Status(const MappingView &, MaintenancePublicationOptions *, + std::shared_ptr<detail::AllocationReservation> *)>; + + MaintenancePublicationOptions publication; + size_t visits_per_step = 64; + size_t max_visits = 4096; + size_t max_candidates = 256; + size_t max_steps = 1024; + size_t max_stale_replans = 2; + // Optional credit acquired before source closure, consumed by the first + // invocation. Later idle/retry calls use acquire for a fresh reservation. + std::shared_ptr<detail::AllocationReservation> prepaid; + // Nonblocking acquisition of a fresh reclaim reservation when needed. + // publication.metadata must be empty. Handoff credit is never reused. + AcquireReservation acquire; + // Acquired before Reclaim. The persistent journal must not retain the + // working partition needed to release another accepted replay payload. + AcquireReservation acquire_snapshot; + // Later invocations may select a larger or smaller window as pressure + // changes. Acquire both partitions for the supplied fixed mapping before + // changing its output count; publication metadata remains independently owned. + AcquireWindow acquire_window; +}; + +// Control, traversal, one complete logical maintenance transaction and its +// publication. Later bounded steps may stop when this reservation is spent. +// New handoff runs are included before they appear in the current mapping. +[[nodiscard]] std::optional<size_t> page_maintenance_reserve_bound(const MappingView &mapping, size_t new_runs, + const PageMaintenanceOperationOptions &options, + bool track_residency = false, + bool separate_snapshot_credit = false) noexcept; +[[nodiscard]] std::optional<size_t> +page_maintenance_snapshot_reserve_bound(const MappingView &mapping, + const PageMaintenanceOperationOptions &options) noexcept; + +// Choose the largest output window whose complete prepaid bounds fit. Keep +// the minimum two-range window when pressure requires admission to wait. +void limit_page_maintenance_window(const MappingView &mapping, size_t new_runs, size_t reclaim_bytes, + size_t snapshot_bytes, bool track_residency, + PageMaintenanceOperationOptions *options) noexcept; + +// One invocation per factory may run at a time. Busy is reported explicitly. +// The callback also supports idle maintenance without a new flush. Runtime and +// publisher outlive all callbacks. This stage does no I/O or writeback: its +// deferred count is the live dirty-page count, even after structural progress. +[[nodiscard]] FlushBatch::Maintenance make_page_maintenance(PublicationCoordinator &publisher, + detail::FlushRuntime &runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + PageMaintenanceOperationOptions options); +// Automatic handoff can lend its prepaid asynchronous load window until this +// maintenance callback completes. At most one cold leaf is resolved per call. +[[nodiscard]] FlushBatch::Maintenance +make_page_maintenance(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageMaintenanceOperationOptions options, + std::shared_ptr<PageLoader> loader, std::shared_ptr<ApplyDispatcher> progress); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/maintenance_operation_internal.h b/lib/crowdb-tree/src/btree/maintenance_operation_internal.h new file mode 100644 index 000000000..0b35417f2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_operation_internal.h @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "apply_dispatcher.h" +#include "maintenance_operation.h" +#include "resource_observation.h" + +namespace crowdb::tree::detail +{ + +struct MaintenanceFactory +{ + MaintenanceFactory(PublicationCoordinator &publish, FlushRuntime &cpu, std::shared_ptr<FlushRuntime::Lane> queue, + PageMaintenanceOperationOptions limits, std::shared_ptr<PageLoader> loads = {}, + std::shared_ptr<ApplyDispatcher> progress = {}) + : publisher(publish), + runtime(cpu), + lane(std::move(queue)), + options(std::move(limits)), + loader(std::move(loads)), + retry_loader(loader), + cpu_progress(std::move(progress)) + { + } + + PublicationCoordinator &publisher; + FlushRuntime &runtime; + std::shared_ptr<FlushRuntime::Lane> lane; + PageMaintenanceOperationOptions options; + std::shared_ptr<PageLoader> loader; + std::weak_ptr<PageLoader> retry_loader; + std::shared_ptr<ApplyDispatcher> cpu_progress; + std::atomic_flag active = ATOMIC_FLAG_INIT; +}; + +struct MaintenanceOperation : std::enable_shared_from_this<MaintenanceOperation> +{ + enum class Phase : uint8_t { Collect, InnerMerge, Analyze, Coalesce, ClassifyMerge, Merge }; + + struct Candidate + { + uint64_t pid; + size_t stale_replans = 0; + }; + + MaintenanceOperation(std::shared_ptr<MaintenanceFactory> owner, uint64_t generation, uint64_t dirty, + FlushBatch::MaintenanceComplete completion, + std::shared_ptr<AllocationReservation> reservation); + ~MaintenanceOperation(); + void schedule() noexcept; + static void run(void *context) noexcept; + void step(); + void initialize(); + void collect(); + void process(); + bool begin_cold(std::shared_ptr<const PublishedTreeState> view, std::shared_ptr<const PublishedPage> page); + void cold_step(); + bool cold_route(); + bool cold_siblings(); + bool load_cold_inner(uint64_t pid); + [[nodiscard]] std::shared_ptr<const PublishedPage> cold_page(uint64_t pid) const; + void cold_plan(); + void advance(); + void submit(MaintenancePublicationResult result); + void published(PublicationResult result); + void finish(Status status) noexcept; + [[nodiscard]] std::shared_ptr<const PublishedTreeState> capture_l1(); + [[nodiscard]] MaintenancePublicationResult plan(const std::shared_ptr<const PublishedTreeState> &view, + const std::shared_ptr<const PublishedPage> &page); + [[nodiscard]] Status merge_pair(const std::shared_ptr<const PublishedTreeState> &view, + const std::shared_ptr<const PublishedPage> &left, bool *eligible) const; + [[nodiscard]] Status classify_merge(const std::shared_ptr<const PublishedTreeState> &view, + const std::shared_ptr<const PublishedPage> &left, + MergePairPublication *classification) const; + + struct Cold + { + explicit Cold(const std::shared_ptr<AllocationReservation> &metadata) + : replacements(ReservationAllocator<PageReplacement>(metadata)), + siblings(ReservationAllocator<uint64_t>(metadata)) + { + } + + ReservedVector<PageReplacement> replacements; + ReservedVector<uint64_t> siblings; + size_t next_sibling = 0; + uint64_t loading_pid = kInvalidPageId; + uint64_t route_pid = kInvalidPageId; + size_t route_steps = 0; + std::shared_ptr<const InnerPageState> loaded_inner; + bool loading_inner = false; + std::shared_ptr<const PublishedTreeState> view; + std::shared_ptr<const PublishedPage> expected; + std::shared_ptr<PageState> leaf; + std::shared_ptr<const LeafBase> loaded; + Status status; + size_t next_base = 0; + size_t pending_base = 0; + bool waiting = false; + std::atomic<bool> ready{false}; + }; + + // Fixed-target split mode never scans or merges unrelated pages. + std::shared_ptr<const PublishedTreeState> target_view; + uint64_t target_pid = kInvalidPageId; + uint64_t target_commit_seq = 0; + uint64_t target_l1_seq = 0; + TreeWorkCause cause = TreeWorkCause::Flush; + std::atomic<bool> target_stopped{false}; + std::atomic<bool> target_started{false}; + + ApplyDispatcher::Node cpu_node{&MaintenanceOperation::run, this}; + std::atomic<bool> scheduled{false}; + std::atomic<bool> wake_requested{false}; + std::shared_ptr<Cold> cold; + bool cold_started = false; + bool cold_inner_started = false; + size_t inner_merge_pairs = 0; + std::shared_ptr<MaintenanceFactory> factory; + std::shared_ptr<PageLoader> loader; + std::shared_ptr<MaintenanceOperation> lifetime; + FlushBatch::MaintenanceComplete done; + MaintenancePublicationOptions publication; + using Candidates = std::vector<Candidate, ReservationAllocator<Candidate>>; + std::shared_ptr<Candidates> candidates; + uint64_t generation; + uint64_t invocation; + uint64_t sequence = 0; + uint64_t dirty; + uint64_t scan_pid = 0; + uint64_t scan_limit = 0; + size_t visits = 0; + size_t steps = 0; + size_t index = 0; + Phase phase = Phase::Collect; + bool initialized = false; + std::atomic<bool> finished{false}; + ResourceObservation queued_resources; + ResourceObservation active_resources; + bool metrics_started = false; + size_t activity_slot = PublicationCoordinator::kMaintenanceSlots; + uint64_t coalesce_input_runs = 0; + uint64_t coalesce_output_runs = 0; + uint64_t coalesce_input_entries = 0; + uint64_t coalesce_output_entries = 0; + ReservedVector<uint64_t> outputs; + std::optional<TreeMetricsOperation> action_metrics; +}; + +[[nodiscard]] bool maintenance_needs_coalesce(const PageState &page, size_t max_sources); +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_operation_step.cpp b/lib/crowdb-tree/src/btree/maintenance_operation_step.cpp new file mode 100644 index 000000000..7f91b457d --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_operation_step.cpp @@ -0,0 +1,368 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "maintenance_operation_internal.h" +#include "maintenance_publication_internal.h" +#include "maintenance_split_layout.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +bool maintenance_needs_coalesce(const PageState &page, size_t max_sources) +{ + if (page.overlays.size() > 1) { + return true; + } + if (page.overlays.size() > max_sources) { + return false; + } + return std::ranges::any_of(page.overlays, [&](const auto &run) { + return !run || (!run->entries().empty() && (!page.range.contains(run->entries().front().key()) || + !page.range.contains(run->entries().back().key()))); + }); +} + +namespace +{ +Status next_run(const PageState &page, size_t max_sources, uint64_t *first) +{ + for (const auto &run : page.overlays) { + if (!run) { + return Status::corruption("maintenance overlay has no owner"); + } + if (run->id() >= UINT64_MAX - max_sources) { + return Status::resource_exhausted("maintenance run identity exhausted"); + } + *first = std::max(*first, run->id() + 1); + } + return Status::Ok(); +} + +bool deferred(const Status &status) +{ + return status.code() == Code::kResourceExhausted || status.code() == Code::kUnavailable; +} + +TreeOutcome outcome(const Status &status, bool ready = true, bool publication = false) +{ + if (status.ok()) { + return ready ? TreeOutcome::Success : TreeOutcome::Retry; + } + if (publication && status.code() == Code::kUnavailable) { + return TreeOutcome::Stale; + } + return deferred(status) ? TreeOutcome::Retry : TreeOutcome::Error; +} +} // namespace + +Status MaintenanceOperation::merge_pair(const std::shared_ptr<const PublishedTreeState> &view, + const std::shared_ptr<const PublishedPage> &left, bool *eligible) const +{ + MergePairPublication classification; + auto status = classify_merge(view, left, &classification); + *eligible = status.ok() && classification.candidate && classification.candidate->ready; + return status; +} + +Status MaintenanceOperation::classify_merge(const std::shared_ptr<const PublishedTreeState> &view, + const std::shared_ptr<const PublishedPage> &left, + MergePairPublication *classification) const +{ + classification->left = (*candidates)[index].pid; + classification->expected_left = left; + if (left->metrics.split_candidate || left->leaf->right_sibling == kInvalidPageId) { + return Status::Ok(); + } + const auto right = view->mapping.get(left->leaf->right_sibling); + if (!right || !right->leaf || right->metrics.split_candidate) { + return Status::Ok(); + } + const auto &limits = publication.pages; + constexpr size_t overhead = kFrameHeaderSize + kFrameTrailerSize; + const bool left_small = left->metrics.page_size_known && left->metrics.page_size_bytes < limits.merge_trigger_bytes; + const bool right_small = + right->metrics.page_size_known && right->metrics.page_size_bytes < limits.merge_trigger_bytes; + if (!left_small && !right_small) { + return Status::Ok(); + } + MaintenanceBuilder builder(view, {}, publication); + MaintenancePath path{ReservationAllocator<MaintenancePathEntry>(publication.metadata)}; + auto status = builder.path_to((*candidates)[index].pid, &path); + if (!status.ok() || path.empty()) { + return status; + } + const auto &parent = path.back(); + if (parent.child + 1 == parent.page->inner->children.size() || + parent.page->inner->children[parent.child + 1] != left->leaf->right_sibling) { + return Status::Ok(); + } + classification->expected_right = right; + classification->expected_parent = parent.page; + const bool ready = + left->metrics.page_size_known && left->metrics.kv_known && right->metrics.page_size_known && + right->metrics.kv_known && left->metrics.page_size_bytes <= limits.merge_target_bytes && + right->metrics.page_size_bytes >= overhead && + right->metrics.page_size_bytes - overhead <= limits.merge_target_bytes - left->metrics.page_size_bytes; + classification->candidate = MergePairProof{.left = classification->left, + .right = left->leaf->right_sibling, + .parent = parent.pid, + .left_content = left->leaf->content_id, + .right_content = right->leaf->content_id, + .left_route = left->leaf->route_generation, + .right_route = right->leaf->route_generation, + .ready = ready}; + return Status::Ok(); +} + +MaintenancePublicationResult MaintenanceOperation::plan(const std::shared_ptr<const PublishedTreeState> &view, + const std::shared_ptr<const PublishedPage> &page) +{ + const auto pid = (*candidates)[index].pid; + auto record = reserved_shared<InstallRecord>( + publication.metadata, + InstallToken{.generation = generation, .batch = invocation, .range = UINT64_MAX - 2, .sequence = ++sequence}); + if (phase == Phase::InnerMerge) { + auto result = build_inner_merge_publication(view, pid, std::move(record), publication); + if (result.changed()) { + action_metrics.emplace(factory->publisher.work_metrics(), TreeStage::PageMerge, tree_metrics_now_ns()); + } + return result; + } + if (phase == Phase::ClassifyMerge) { + MaintenancePublicationResult result(publication.metadata); + MergePairPublication classification; + result.status = classify_merge(view, page, &classification); + if (result.status.ok() && classification.candidate != page->merge_outgoing) { + result.transaction.record = std::move(record); + result.transaction.merge_pair = std::move(classification); + } + return result; + } + if (phase == Phase::Analyze) { + const auto started = tree_metrics_now_ns(); + TreeMetricsOperation layout(factory->publisher.work_metrics(), TreeStage::PageLayout, started); + auto result = build_split_step_publication(view, pid, std::move(record), publication); + (void)layout.finish(outcome(result.status, result.readiness == MaintenanceReadiness::kReady), + tree_metrics_now_ns()); + if (result.output_leaf_pids.size() > 1) { + action_metrics.emplace(factory->publisher.work_metrics(), TreeStage::PageSplit, started); + } + return result; + } + if (phase == Phase::Coalesce && !maintenance_needs_coalesce(*page->leaf, publication.pages.max_sources)) { + return {}; + } + if (phase == Phase::Merge) { + bool eligible = false; + auto status = merge_pair(view, page, &eligible); + if (!status.ok() || !eligible) { + MaintenancePublicationResult result; + result.status = std::move(status); + return result; + } + } + uint64_t first_run = 1; + auto status = next_run(*page->leaf, publication.pages.max_sources, &first_run); + if (status.ok() && phase == Phase::Merge) { + status = + next_run(*view->mapping.get(page->leaf->right_sibling)->leaf, publication.pages.max_sources, &first_run); + } + if (!status.ok()) { + MaintenancePublicationResult result; + result.status = std::move(status); + return result; + } + action_metrics.emplace(factory->publisher.work_metrics(), + phase == Phase::Coalesce ? TreeStage::OverlayCoalesce : TreeStage::PageMerge, + tree_metrics_now_ns()); + return phase == Phase::Coalesce + ? build_coalesce_publication(view, pid, std::move(record), first_run, publication) + : build_merge_publication(view, pid, page->leaf->right_sibling, std::move(record), first_run, publication); +} + +void MaintenanceOperation::process() +{ + auto view = capture_l1(); + if (view->generation != generation) { + finish(Status::unavailable("maintenance generation changed")); + return; + } + record_work(factory->publisher.work_metrics(), TreeWorkCounter::CandidateChecks, 1, cause); + const auto page = view->mapping.get((*candidates)[index].pid); + if (!page || !page->leaf || page->level != 0) { + record_work(factory->publisher.work_metrics(), TreeWorkCounter::CandidateSkips, 1, cause); + advance(); + return; + } + if (phase == Phase::InnerMerge && + (publication.inner_merge_keys == 0 || view->mapping.get(view->root_page_id)->level < 2)) { + advance(); + return; + } + if ((!target_view && phase == Phase::Analyze && page->metrics.page_size_known && page->metrics.kv_known && + !page->metrics.split_candidate && page->metrics.page_size_bytes <= publication.pages.primary_page_bytes) || + (phase == Phase::Coalesce && !maintenance_needs_coalesce(*page->leaf, publication.pages.max_sources))) { + record_work(factory->publisher.work_metrics(), TreeWorkCounter::CandidateSkips, 1, cause); + advance(); + return; + } + if (phase == Phase::Merge) { + bool eligible = false; + auto status = merge_pair(view, page, &eligible); + if (!status.ok() && !deferred(status)) { + finish(std::move(status)); + return; + } + if (!eligible) { + record_work(factory->publisher.work_metrics(), TreeWorkCounter::CandidateSkips, 1, cause); + advance(); + return; + } + } + if (phase != Phase::ClassifyMerge && phase != Phase::InnerMerge) { + ++steps; + } + factory->publisher.set_maintenance_scan_pid((*candidates)[index].pid + 1); + action_metrics.reset(); + if ((phase == Phase::Analyze || phase == Phase::InnerMerge) && begin_cold(view, page)) { + return; + } + auto result = plan(view, page); + if (result.status.ok() && result.readiness == MaintenanceReadiness::kNeedsBase && phase == Phase::Analyze && + begin_cold(std::move(view), page)) { + return; + } + if (!result.status.ok() || result.readiness != MaintenanceReadiness::kReady || !result.changed()) { + if (action_metrics) { + (void)action_metrics->finish(outcome(result.status, result.readiness == MaintenanceReadiness::kReady), + tree_metrics_now_ns()); + } + if (!result.status.ok() && (target_view || !deferred(result.status))) { + finish(std::move(result.status)); + return; + } + record_work(factory->publisher.work_metrics(), TreeWorkCounter::CandidateSkips, 1, cause); + advance(); + return; + } + if (phase == Phase::ClassifyMerge || phase == Phase::InnerMerge) { + ++steps; + } + submit(std::move(result)); +} + +void MaintenanceOperation::submit(MaintenancePublicationResult result) +{ + if (target_view) { + const bool classified = + result.output_leaf_pids.size() == 1 && + std::ranges::any_of(result.transaction.pages, [&](const PageReplacement &change) { + return change.page_id == target_pid && change.expected && change.replacement && + change.expected->metrics.split_candidate && !change.replacement->metrics.split_candidate && + change.replacement->metrics.page_size_known && change.replacement->metrics.kv_known; + }); + if (result.output_leaf_pids.size() < 2 && !classified) { + finish(Status::resource_exhausted("checkpoint split made no structural progress")); + return; + } + result.transaction.expected_l1_seq = target_view->l1_seq; + result.transaction.expected_coverage = target_view->coverage; + } + inner_merge_pairs = result.merged_inner_pairs; + coalesce_input_runs = coalesce_output_runs = coalesce_input_entries = coalesce_output_entries = 0; + if (phase == Phase::Coalesce) { + for (const auto &change : result.transaction.pages) { + if (change.page_id != (*candidates)[index].pid || !change.expected || !change.replacement) { + continue; + } + const auto &before = change.expected->leaf; + const auto &after = change.replacement->leaf; + if (!before || !after) { + continue; + } + coalesce_input_runs = before->overlays.size(); + coalesce_output_runs = after->overlays.size(); + for (const auto &run : before->overlays) { + size_t begin = 0; + size_t end = run->entries().size(); + const auto &lower = before->range.lower; + const auto &upper = before->range.upper; + if (lower.has_value()) { + begin = run->lower_bound(Slice(lower.value())); + } + if (upper.has_value()) { + end = run->lower_bound(Slice(upper.value())); + } + coalesce_input_entries += end - begin; + } + for (const auto &run : after->overlays) { + coalesce_output_entries += run->entries().size(); + } + } + } + outputs = std::move(result.output_leaf_pids); + auto request = reserved_shared<PublicationRequest>(publication.metadata, publication.metadata); + request->transaction = std::move(result.transaction); + auto self = shared_from_this(); + request->on_done = [self](PublicationResult published) { self->published(std::move(published)); }; + if (!factory->publisher.try_submit(request)) { + finish(target_view ? Status::unavailable("checkpoint split publication unavailable") : Status::Ok()); + } +} + +void MaintenanceOperation::published(PublicationResult result) +{ + if (action_metrics) { + (void)action_metrics->finish(outcome(result.status, true, true), tree_metrics_now_ns()); + } + const auto &work = factory->publisher.work_metrics(); + if (result.status.code() == Code::kUnavailable) { + record_work(work, TreeWorkCounter::PublicationMismatches, 1, cause); + } + if (result.status.ok()) { + if (phase == Phase::Analyze && outputs.size() > 1) { + record_work(work, TreeWorkCounter::SplitOperations, 1, cause); + record_work(work, TreeWorkCounter::SplitOutputPages, outputs.size(), cause); + } + else if (phase == Phase::Merge || phase == Phase::InnerMerge) { + const size_t pairs = inner_merge_pairs + (phase == Phase::Merge ? 1 : 0); + record_work(work, TreeWorkCounter::MergeOperations, pairs, cause); + record_work(work, TreeWorkCounter::MergeInputPages, 2 * pairs, cause); + record_work(work, TreeWorkCounter::MergeOutputPages, pairs, cause); + } + else if (phase == Phase::Coalesce) { + record_work(work, TreeWorkCounter::CoalesceOperations, 1, cause); + record_work(work, TreeWorkCounter::CoalesceInputRuns, coalesce_input_runs, cause); + record_work(work, TreeWorkCounter::CoalesceOutputRuns, coalesce_output_runs, cause); + record_work(work, TreeWorkCounter::CoalesceInputEntries, coalesce_input_entries, cause); + record_work(work, TreeWorkCounter::CoalesceOutputEntries, coalesce_output_entries, cause); + if (coalesce_input_entries > coalesce_output_entries) { + record_work(work, TreeWorkCounter::PayloadReferencesDropped, + coalesce_input_entries - coalesce_output_entries, cause); + } + } + } + if (target_view) { + target_commit_seq = result.commit_seq; + target_l1_seq = result.l1_seq; + finish(std::move(result.status)); + return; + } + auto &candidate = (*candidates)[index]; + if (result.status.code() == Code::kUnavailable && candidate.stale_replans++ < factory->options.max_stale_replans) { + record_work(factory->publisher.work_metrics(), TreeWorkCounter::Retries, 1, cause); + schedule(); + return; + } + if (!result.status.ok() && !deferred(result.status)) { + finish(std::move(result.status)); + return; + } + if (result.status.ok() && phase == Phase::Analyze) { + for (size_t i = 1; i < outputs.size() && candidates->size() < factory->options.max_candidates; ++i) { + candidates->push_back({.pid = outputs[i]}); + } + } + advance(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_publication.cpp b/lib/crowdb-tree/src/btree/maintenance_publication.cpp new file mode 100644 index 000000000..0efe7ae4d --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_publication.cpp @@ -0,0 +1,256 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_publication_internal.h" + +#include <new> + +namespace crowdb::tree +{ +namespace +{ +using detail::MaintenanceBuilder; +using detail::MaintenanceChildren; +using detail::MaintenancePath; +using detail::MaintenancePathEntry; + +template <class Prepare> +MaintenancePublicationResult prepare(const MaintenancePublicationOptions &options, Prepare &&operation) +{ + MaintenancePublicationResult result(options.metadata); + try { + result.status = operation(&result); + } + catch (const std::bad_alloc &) { + result.status = Status::resource_exhausted(); + result.readiness = MaintenanceReadiness::kDeferred; + } + if (!result.status.ok()) { + result.transaction = {}; + result.output_leaf_pids.clear(); + if (result.status.code() == Code::kResourceExhausted) { + result.readiness = MaintenanceReadiness::kDeferred; + } + } + return result; +} + +Status coalesce(MaintenanceBuilder &builder, uint64_t pid, uint64_t first_run_id, MaintenancePublicationResult *result) +{ + MaintenancePath path{detail::ReservationAllocator<MaintenancePathEntry>(builder.options.metadata)}; + auto status = builder.path_to(pid, &path); + if (!status.ok()) { + return status; + } + const auto old = builder.view->mapping.get(pid); + if (old->leaf->representation_id == UINT64_MAX) { + return Status::resource_exhausted("coalesce representation identity exhausted"); + } + auto limits = builder.options.pages; + limits.gc_floor = builder.view->gc_floor; + auto compacted = coalesce_overlays(old->leaf, limits, old->leaf->representation_id + 1, first_run_id); + result->readiness = compacted.readiness; + result->loads = std::move(compacted.loads); + if (!compacted.status.ok() || !compacted.page) { + return compacted.status; + } + status = builder.make_leaf(pid, old, std::move(compacted.page), std::nullopt, {}, nullptr, + compacted.removed_primary_bytes); + if (status.ok()) { + status = builder.guard_path(path); + } + if (status.ok()) { + result->output_leaf_pids = {pid}; + status = builder.finish(result); + } + return status; +} + +Status split(MaintenanceBuilder &builder, uint64_t pid, MaintenancePublicationResult *result) +{ + MaintenancePath path{detail::ReservationAllocator<MaintenancePathEntry>(builder.options.metadata)}; + auto status = builder.path_to(pid, &path); + if (!status.ok()) { + return status; + } + const auto old = builder.view->mapping.get(pid); + auto layout = plan_leaf_layout(old->leaf, builder.options.pages); + result->readiness = layout.readiness; + result->loads = layout.loads; + if (!layout.status.ok() || layout.readiness != MaintenanceReadiness::kReady) { + return layout.status; + } + if (layout.primary_bytes.size() == 1) { + return Status::Ok(); + } + PageVersion version; + status = detail::next_page_version(*old->leaf, nullptr, &version); + if (!status.ok()) { + return status; + } + detail::ReservedVector<SplitTarget> targets{detail::ReservationAllocator<SplitTarget>(builder.options.metadata)}; + targets.reserve(layout.primary_bytes.size()); + for (size_t i = 0; i < layout.primary_bytes.size(); ++i) { + uint64_t output = pid; + if (i != 0) { + status = builder.allocate_pid(&output); + if (!status.ok()) { + return status; + } + } + targets.push_back({.page_id = output, .version = version}); + } + auto leaves = split_leaf_ranges(layout, targets, builder.options.pages); + result->readiness = leaves.readiness; + if (!leaves.status.ok()) { + return leaves.status; + } + MaintenanceChildren children(builder.options.metadata); + children.separators.assign(layout.separators.begin(), layout.separators.end()); + for (size_t i = 0; i < targets.size(); ++i) { + status = builder.make_leaf(targets[i].page_id, old, std::move(leaves.pages[i]), layout.primary_bytes[i]); + if (!status.ok()) { + return status; + } + children.pids.push_back(targets[i].page_id); + } + result->output_leaf_pids = children.pids; + status = builder.propagate(path, 1, std::move(children), 0); + if (status.ok()) { + status = builder.guard_path(path); + } + return status.ok() ? builder.finish(result) : status; +} + +Status merge(MaintenanceBuilder &builder, uint64_t left_pid, uint64_t right_pid, uint64_t first_run_id, + MaintenancePublicationResult *result) +{ + MaintenancePath path{detail::ReservationAllocator<MaintenancePathEntry>(builder.options.metadata)}; + auto status = builder.path_to(left_pid, &path); + if (!status.ok()) { + return status; + } + const auto left = builder.view->mapping.get(left_pid); + const auto right = builder.view->mapping.get(right_pid); + if (path.empty() || !right || !right->leaf) { + return Status::invalid_argument("merge requires two leaves with a common parent"); + } + const auto &parent = path.back(); + if (parent.child + 1 >= parent.page->inner->children.size() || + parent.page->inner->children[parent.child + 1] != right_pid) { + return Status::invalid_argument("merge leaves must be adjacent children of one parent"); + } + MaintenancePath right_path{detail::ReservationAllocator<MaintenancePathEntry>(builder.options.metadata)}; + status = builder.path_to(right_pid, &right_path); + if (!status.ok()) { + return status; + } + if (right_path.empty() || right_path.back().pid != parent.pid || right_path.back().child != parent.child + 1) { + return Status::invalid_argument("merge right leaf has a different route"); + } + PageVersion version; + status = detail::next_page_version(*left->leaf, right->leaf.get(), &version); + if (!status.ok()) { + return status; + } + auto merged = detail::plan_published_leaf_merge(*left, *right, + {.left = left->leaf, + .right = right->leaf, + .left_page_id = left_pid, + .right_page_id = right_pid, + .left_parent = parent.pid, + .right_parent = parent.pid}, + builder.options.pages, version, first_run_id); + result->readiness = merged.logical.readiness; + result->loads = std::move(merged.logical.loads); + if (!merged.logical.status.ok() || !merged.logical.page || result->readiness != MaintenanceReadiness::kReady) { + return merged.logical.status; + } + status = builder.make_leaf(left_pid, left, std::move(merged.logical.page), merged.primary_bytes, + right->allocation_owner, merged.statistics ? &*merged.statistics : nullptr); + if (status.ok()) { + status = builder.replace(right_pid, {}); + } + if (status.ok()) { + MaintenanceChildren children(builder.options.metadata); + children.pids.push_back(left_pid); + status = builder.propagate(path, 2, std::move(children), 0); + } + if (status.ok()) { + bool changed = false; + status = builder.merge_underfull(path, &changed); + } + if (status.ok()) { + status = builder.shrink_root(); + } + if (status.ok()) { + status = builder.guard_path(path); + } + if (status.ok()) { + result->output_leaf_pids = {left_pid}; + status = builder.finish(result); + } + return status; +} +} // namespace + +MaintenancePublicationResult build_coalesce_publication(std::shared_ptr<const PublishedTreeState> view, + uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, + uint64_t first_run_id, + const MaintenancePublicationOptions &options) +{ + return prepare(options, [&](auto *result) { + MaintenanceBuilder builder(std::move(view), std::move(record), options); + auto status = builder.validate(); + return status.ok() ? coalesce(builder, leaf_pid, first_run_id, result) : status; + }); +} + +MaintenancePublicationResult build_split_publication(std::shared_ptr<const PublishedTreeState> view, uint64_t leaf_pid, + std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &options) +{ + return prepare(options, [&](auto *result) { + MaintenanceBuilder builder(std::move(view), std::move(record), options); + auto status = builder.validate(); + return status.ok() ? split(builder, leaf_pid, result) : status; + }); +} + +MaintenancePublicationResult build_merge_publication(std::shared_ptr<const PublishedTreeState> view, uint64_t left_pid, + uint64_t right_pid, std::shared_ptr<InstallRecord> record, + uint64_t first_run_id, + const MaintenancePublicationOptions &options) +{ + return prepare(options, [&](auto *result) { + MaintenanceBuilder builder(std::move(view), std::move(record), options); + auto status = builder.validate(); + return status.ok() ? merge(builder, left_pid, right_pid, first_run_id, result) : status; + }); +} + +MaintenancePublicationResult build_inner_merge_publication(std::shared_ptr<const PublishedTreeState> view, + uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &options) +{ + return prepare(options, [&](auto *result) { + MaintenanceBuilder builder(std::move(view), std::move(record), options); + auto status = builder.validate(); + MaintenancePath path{detail::ReservationAllocator<MaintenancePathEntry>(options.metadata)}; + if (status.ok()) { + status = builder.path_to(leaf_pid, &path); + } + bool changed = false; + if (status.ok()) { + status = builder.merge_underfull(path, &changed); + } + if (!status.ok() || !changed) { + return status; + } + status = builder.shrink_root(); + if (status.ok()) { + status = builder.guard_path(path); + } + return status.ok() ? builder.finish(result) : status; + }); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/maintenance_publication.h b/lib/crowdb-tree/src/btree/maintenance_publication.h new file mode 100644 index 000000000..778b024c6 --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_publication.h @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_maintenance.h" +#include "btree/published_state.h" + +namespace crowdb::tree +{ +struct MaintenancePublicationOptions +{ + PageMaintenanceLimits pages; + size_t max_inner_children = 257; + size_t max_inner_bytes = 64 * 1024; + size_t inner_merge_keys = 64; + size_t max_path_pages = 64; + size_t max_changed_pages = 256; + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<detail::AllocationReservation> snapshot_metadata = nullptr; +}; + +struct MaintenancePublicationResult +{ + MaintenancePublicationResult() = default; + + explicit MaintenancePublicationResult(const std::shared_ptr<detail::AllocationReservation> &metadata) + : loads(detail::ReservationAllocator<BaseLoadRequest>(metadata)), + transaction{.pages = PublicationTransaction::Pages(detail::ReservationAllocator<PageReplacement>(metadata)), + .metadata = metadata}, + output_leaf_pids(detail::ReservationAllocator<uint64_t>(metadata)) + { + } + + Status status; + MaintenanceReadiness readiness = MaintenanceReadiness::kReady; + detail::ReservedVector<BaseLoadRequest> loads; + PublicationTransaction transaction; + detail::ReservedVector<uint64_t> output_leaf_pids; + size_t merged_inner_pairs = 0; + + // A ready result may be a no-op (one page already fits or merge hysteresis). + [[nodiscard]] bool changed() const noexcept + { + return transaction.record != nullptr; + } +}; + +// Preparation only. All expected pages, ancestor route guards and the root +// precondition publish as one transaction. A stale result must be replanned; +// it cannot be split into child transactions. Cold data is returned as loads. +MaintenancePublicationResult build_coalesce_publication(std::shared_ptr<const PublishedTreeState> view, + uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, + uint64_t first_run_id, + const MaintenancePublicationOptions &options = {}); +MaintenancePublicationResult build_split_publication(std::shared_ptr<const PublishedTreeState> view, uint64_t leaf_pid, + std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &options = {}); + +// A bounded window becomes legal primary-page ranges in one transaction. An +// unvisited suffix keeps unknown size and remains a split candidate. Planning +// shares payloads and does not encode a frame or collect the complete large leaf. +MaintenancePublicationResult build_split_step_publication(std::shared_ptr<const PublishedTreeState> view, + uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &options = {}); +MaintenancePublicationResult build_merge_publication(std::shared_ptr<const PublishedTreeState> view, uint64_t left_pid, + uint64_t right_pid, std::shared_ptr<InstallRecord> record, + uint64_t first_run_id, + const MaintenancePublicationOptions &options = {}); + +// Repair underfull routing pages on one bounded leaf route. Adjacent siblings, +// ancestor updates and root contraction are guarded by the same publication. +MaintenancePublicationResult build_inner_merge_publication(std::shared_ptr<const PublishedTreeState> view, + uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &options = {}); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/maintenance_publication_internal.h b/lib/crowdb-tree/src/btree/maintenance_publication_internal.h new file mode 100644 index 000000000..cf93195fc --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_publication_internal.h @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_reservation.h" +#include "btree/maintenance_publication.h" + +#include <map> + +namespace crowdb::tree::detail +{ +struct MaintenancePathEntry +{ + uint64_t pid = 0; + size_t child = 0; + std::shared_ptr<const PublishedPage> page; +}; + +using MaintenancePath = ReservedVector<MaintenancePathEntry>; + +struct MaintenanceChildren +{ + explicit MaintenanceChildren(const std::shared_ptr<AllocationReservation> &metadata) + : pids(ReservationAllocator<uint64_t>(metadata)), + separators(ReservationAllocator<OwnedKey>(metadata)) + { + } + + ReservedVector<uint64_t> pids; + ReservedVector<OwnedKey> separators; +}; + +class MaintenanceBuilder +{ + public: + MaintenanceBuilder(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &limits); + [[nodiscard]] Status validate() const; + Status path_to(uint64_t pid, MaintenancePath *path) const; + Status guard_path(const MaintenancePath &path); + Status allocate_pid(uint64_t *pid); + Status replace(uint64_t pid, std::shared_ptr<const PublishedPage> replacement); + Status propagate(const MaintenancePath &path, size_t removed_children, MaintenanceChildren children, + uint32_t child_level); + Status shrink_root(); + Status merge_underfull(const MaintenancePath &path, bool *changed); + Status make_leaf(uint64_t pid, const std::shared_ptr<const PublishedPage> &old, + std::shared_ptr<const PageState> logical, std::optional<size_t> exact_bytes, + std::shared_ptr<const void> second_owner = {}, const LeafLayoutStats *stats = nullptr, + size_t removed_primary_bytes = 0); + Status finish(MaintenancePublicationResult *result); + [[nodiscard]] std::shared_ptr<const PublishedPage> lookup(uint64_t pid) const; + + const std::shared_ptr<const PublishedTreeState> view; + MaintenancePublicationOptions options; + + private: + Status merge_inner_pair(const MaintenancePathEntry &ancestor, uint64_t target, bool *changed); + Status partition_inner(uint64_t pid, uint32_t level, const InnerPageState &inner, MaintenanceChildren *output); + Status grow_root(MaintenanceChildren children, uint32_t child_level); + std::shared_ptr<InstallRecord> record_; + using Change = std::pair<const uint64_t, PageReplacement>; + std::map<uint64_t, PageReplacement, std::less<>, ReservationAllocator<Change>> changes_; + uint64_t next_pid_; + uint64_t root_pid_; + size_t inner_merges_ = 0; +}; + +Status next_page_version(const PageState &left, const PageState *right, PageVersion *version); +Status summarize_leaf(const std::shared_ptr<const PageState> &page, const PageMaintenanceLimits &limits, + PageMetricsContribution *metrics); + +struct MaintenanceLeafMerge +{ + PageMaintenanceResult logical; + size_t primary_bytes = 0; + std::optional<LeafLayoutStats> statistics; +}; + +MaintenanceLeafMerge plan_published_leaf_merge(const PublishedPage &left, const PublishedPage &right, + const LeafMergeInput &input, const PageMaintenanceLimits &limits, + PageVersion version, uint64_t first_run_id); +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_publication_state.cpp b/lib/crowdb-tree/src/btree/maintenance_publication_state.cpp new file mode 100644 index 000000000..2ae68beb0 --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_publication_state.cpp @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_publication_internal.h" + +#include <algorithm> +#include <array> + +namespace crowdb::tree::detail +{ +MaintenanceBuilder::MaintenanceBuilder(std::shared_ptr<const PublishedTreeState> captured, + std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &limits) + : view(std::move(captured)), + options(limits), + record_(std::move(record)), + changes_(ReservationAllocator<Change>(limits.metadata)), + next_pid_(view ? view->next_page_id : 0), + root_pid_(view ? view->root_page_id : kInvalidPageId) +{ + options.pages.metadata = options.metadata; +} + +Status MaintenanceBuilder::validate() const +{ + constexpr size_t minimum_inner = kFrameHeaderSize + kFrameTrailerSize + (2 * sizeof(uint64_t)) + kInnerSlotSize; + if (!view || !record_ || record_->token.generation != view->generation || options.max_inner_children < 2 || + options.max_inner_bytes < minimum_inner || options.max_inner_bytes > UINT32_MAX || + options.max_path_pages == 0 || options.max_changed_pages == 0 || + options.pages.max_metadata_bytes < minimum_inner) { + return Status::invalid_argument("invalid maintenance view, token or bounds"); + } + return Status::Ok(); +} + +std::shared_ptr<const PublishedPage> MaintenanceBuilder::lookup(uint64_t pid) const +{ + const auto changed = changes_.find(pid); + return changed == changes_.end() ? view->mapping.get(pid) : changed->second.replacement; +} + +Status MaintenanceBuilder::allocate_pid(uint64_t *pid) +{ + if (next_pid_ >= MappingView::kPageLimit) { + return Status::resource_exhausted("maintenance PID space exhausted"); + } + *pid = next_pid_++; + return Status::Ok(); +} + +Status MaintenanceBuilder::replace(uint64_t pid, std::shared_ptr<const PublishedPage> replacement) +{ + const auto expected = view->mapping.get(pid); + if (!expected && !replacement) { + changes_.erase(pid); + return Status::Ok(); + } + if (!changes_.contains(pid) && changes_.size() >= options.max_changed_pages) { + return Status::resource_exhausted("maintenance transaction page bound exhausted"); + } + changes_.insert_or_assign( + pid, PageReplacement{.page_id = pid, .expected = expected, .replacement = std::move(replacement)}); + return Status::Ok(); +} + +Status next_page_version(const PageState &left, const PageState *right, PageVersion *version) +{ + const uint64_t route = std::max(left.route_generation, right != nullptr ? right->route_generation : 0); + const uint64_t content = std::max(left.content_id, right != nullptr ? right->content_id : 0); + const uint64_t representation = std::max(left.representation_id, right != nullptr ? right->representation_id : 0); + if (route == UINT64_MAX || content == UINT64_MAX || representation == UINT64_MAX) { + return Status::resource_exhausted("logical page identity exhausted"); + } + *version = {.route_generation = route + 1, .content_id = content + 1, .representation_id = representation + 1}; + return Status::Ok(); +} + +Status summarize_leaf(const std::shared_ptr<const PageState> &page, const PageMaintenanceLimits &limits, + PageMetricsContribution *metrics) +{ + LogicalPageCursor cursor(page, page->range, UINT64_MAX, limits.max_sources, false, limits.metadata); + if (!cursor.status().ok()) { + return cursor.status(); + } + if (cursor.needs_base().has_value()) { + return Status::unavailable("exact maintenance summary still needs a base"); + } + metrics->kv_known = true; + metrics->kv_count = 0; + metrics->kv_bytes = 0; + size_t visited = 0; + while (cursor.valid()) { + if (++visited > limits.max_work_entries) { + return Status::resource_exhausted("maintenance summary work bound exhausted"); + } + const auto cell = cursor.cell(); + if (!cell.tombstone()) { + if (cell.overflow() && cell.encoded.size() < kOverflowCellSize) { + return Status::corruption("maintenance found an invalid overflow cell"); + } + const uint64_t value_bytes = cell.overflow() ? CellView(cell.encoded).overflow_len() : cell.value.size(); + if (value_bytes > UINT64_MAX - cursor.key().size() || + value_bytes + cursor.key().size() > UINT64_MAX - metrics->kv_bytes) { + return Status::resource_exhausted("maintenance summary size overflow"); + } + ++metrics->kv_count; + metrics->kv_bytes += value_bytes + cursor.key().size(); + } + cursor.next(); + } + if (!cursor.status().ok()) { + return cursor.status(); + } + if (cursor.needs_base()) { + return Status::unavailable("exact maintenance summary requires another physical frame"); + } + return Status::Ok(); +} + +Status MaintenanceBuilder::make_leaf(uint64_t pid, const std::shared_ptr<const PublishedPage> &old, + std::shared_ptr<const PageState> logical, std::optional<size_t> exact_bytes, + std::shared_ptr<const void> second_owner, const LeafLayoutStats *stats, + size_t removed_primary_bytes) +{ + const auto previous = view->mapping.get(pid); + if (previous && previous->revision == UINT64_MAX) { + return Status::resource_exhausted("leaf revision exhausted"); + } + auto page = reserved_shared<PublishedPage>(options.metadata); + page->revision = previous ? previous->revision + 1 : 1; + page->leaf = std::move(logical); + page->allocation_owner = old->allocation_owner; + if (second_owner && second_owner != old->allocation_owner) { + page->allocation_owner = reserved_shared<std::array<std::shared_ptr<const void>, 2>>( + options.metadata, + std::array<std::shared_ptr<const void>, 2>{old->allocation_owner, std::move(second_owner)}); + } + auto &metrics = page->metrics; + if (exact_bytes.has_value()) { + metrics.dirty = true; + metrics.writeback_candidate = true; + metrics.page_size_known = true; + metrics.page_size_bytes = *exact_bytes; + metrics.page_size_estimate_bytes = *exact_bytes; + if (stats != nullptr) { + metrics.kv_known = true; + metrics.kv_count = stats->kv_count; + metrics.kv_bytes = stats->kv_bytes; + } + else { + auto status = summarize_leaf(page->leaf, options.pages, &metrics); + if (!status.ok()) { + return status; + } + } + } + else { + metrics = old->metrics; + metrics.consolidate_candidate = false; + if (metrics.page_size_known && removed_primary_bytes != 0) { + constexpr size_t overhead = kFrameHeaderSize + kFrameTrailerSize; + if (metrics.page_size_bytes < overhead || removed_primary_bytes > metrics.page_size_bytes - overhead) { + return Status::corruption("coalesced tombstones exceed the captured primary size"); + } + metrics.page_size_bytes -= removed_primary_bytes; + } + metrics.page_size_estimate_bytes -= std::min(metrics.page_size_estimate_bytes, removed_primary_bytes); + } + metrics.overlay_runs = page->leaf->overlays.size(); + metrics.overlay_entries = 0; + for (const auto &run : page->leaf->overlays) { + const auto &range = page->leaf->range; + const size_t first = range.lower ? run->lower_bound(*range.lower) : 0; + const size_t last = range.upper ? run->lower_bound(*range.upper) : run->entries().size(); + metrics.overlay_entries += last - first; + } + return replace(pid, std::move(page)); +} + +Status MaintenanceBuilder::finish(MaintenancePublicationResult *result) +{ + PublicationTransaction transaction{ + .pages = PublicationTransaction::Pages(ReservationAllocator<PageReplacement>(options.metadata))}; + transaction.record = record_; + transaction.root_page_id = root_pid_; + transaction.expected_root_page_id = view->root_page_id; + transaction.expected_next_page_id = view->next_page_id; + transaction.metadata = options.metadata; + transaction.snapshot_metadata = options.snapshot_metadata; + transaction.snapshot_effect = SnapshotChangeEffect::SourceStructure; + transaction.pages.reserve(changes_.size()); + for (auto &[pid, replacement] : changes_) { + (void)pid; + transaction.pages.push_back(std::move(replacement)); + } + result->transaction = std::move(transaction); + result->merged_inner_pairs = inner_merges_; + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_reserve.cpp b/lib/crowdb-tree/src/btree/maintenance_reserve.cpp new file mode 100644 index 000000000..c1fb9d861 --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_reserve.cpp @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "maintenance_operation_internal.h" +#include "maintenance_split_layout.h" +#include "page_cache_residency.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +struct MaintenanceWindow +{ + size_t changed; + size_t leaves; +}; + +std::optional<MaintenanceWindow> maintenance_window(const MappingView::CopyBounds &shape, + const MaintenancePublicationOptions &options) noexcept +{ + if (options.pages.max_output_pages < 2 || options.max_changed_pages == 0 || options.pages.max_work_entries == 0) { + return {}; + } + const auto outputs = std::min(options.pages.max_output_pages - 1, options.pages.max_work_entries) + 1; + const auto root_pages = detail::AllocationBound().add_product(2, outputs).value(); + const auto leaf_pages = detail::AllocationBound(outputs).add(2).value(); + if (!root_pages || !leaf_pages) { + return {}; + } + const auto changed = + shape.max_level == 0 ? std::min(options.max_changed_pages, *root_pages) : options.max_changed_pages; + return MaintenanceWindow{.changed = changed, .leaves = std::min(changed, *leaf_pages)}; +} +} // namespace + +void limit_page_maintenance_window(const MappingView &mapping, size_t new_runs, size_t reclaim_bytes, + size_t snapshot_bytes, bool track_residency, + PageMaintenanceOperationOptions *options) noexcept +{ + auto &pages = options->publication.pages; + if (pages.max_output_pages < 2 || pages.max_work_entries == 0) { + return; + } + size_t low = 2; + size_t high = std::min(pages.max_output_pages - 1, pages.max_work_entries) + 1; + const auto fits = [&](size_t count) { + pages.max_output_pages = count; + const auto reclaim = page_maintenance_reserve_bound(mapping, new_runs, *options, track_residency, true); + const auto snapshot = page_maintenance_snapshot_reserve_bound(mapping, *options); + return reclaim && snapshot && *reclaim <= reclaim_bytes && *snapshot <= snapshot_bytes; + }; + if (fits(high)) { + return; + } + while (low < high) { + const auto middle = low + ((high - low + 1) / 2); + if (fits(middle)) { + low = middle; + } + else { + high = middle - 1; + } + } + pages.max_output_pages = low; +} + +std::optional<size_t> page_maintenance_snapshot_reserve_bound(const MappingView &mapping, + const PageMaintenanceOperationOptions &options) noexcept +{ + const auto window = maintenance_window(mapping.copy_bounds(), options.publication); + if (!window) { + return {}; + } + const auto changed = window->changed; + detail::AllocationBound bytes(detail::shared_allocation_bound<detail::AllocationReservation>()); + bytes.add_product(changed, PublicationCoordinator::snapshot_change_reserve_bytes()); + return bytes.value(); +} + +std::optional<size_t> page_maintenance_reserve_bound(const MappingView &mapping, size_t new_runs, + const PageMaintenanceOperationOptions &options, + bool track_residency, bool separate_snapshot_credit) noexcept +{ + using detail::AllocationBound; + using detail::shared_allocation_bound; + const auto &publication = options.publication; + const auto &pages = publication.pages; + if (options.max_candidates == 0 || options.max_steps == 0 || options.max_visits == 0 || + options.visits_per_step == 0 || publication.max_changed_pages == 0 || publication.max_path_pages == 0 || + publication.max_path_pages > 64 || pages.max_work_entries == 0) { + return {}; + } + const auto mapping_bounds = mapping.copy_bounds(); + // A leaf root has no existing routing path. Its first bounded split can + // create at most N leaves and N - 1 inner pages, even at minimum fanout. + // Later steps may defer when this invocation's prepaid credit is spent. + const auto window = maintenance_window(mapping_bounds, publication); + if (!window) { + return {}; + } + const auto changed = window->changed; + const auto leaf = AllocationBound(mapping_bounds.max_leaf_bytes) + .add_product(new_runs, 3 * sizeof(std::shared_ptr<const OverlayBatch>)) + .value(); + const auto inner = AllocationBound(mapping_bounds.max_inner_bytes).add(InnerPageState::allocation_bound()).value(); + if (!leaf || !inner) { + return {}; + } + AllocationBound bytes(shared_allocation_bound<detail::MaintenanceFactory>() + + shared_allocation_bound<detail::MaintenanceOperation>() + + shared_allocation_bound<detail::MaintenanceOperation::Candidates>()); + bytes.add_product(options.max_candidates, sizeof(detail::MaintenanceOperation::Candidate)); + bytes.add(shared_allocation_bound<detail::MaintenanceOperation::Cold>()); + bytes.add(shared_allocation_bound<PublishedTreeState>()); + bytes.add(shared_allocation_bound<PublishedPage>()); + // Only captured routing pages require cold scratch. Inner pages created + // by this maintenance window already own resident routes. Retain the full + // bounded replacement arrays, but charge copied radix paths by that depth. + const auto cold_depth = std::min<size_t>(publication.max_path_pages, mapping_bounds.max_level); + bytes.add_product((3 * cold_depth) + 1, MappingView::path_copy_bound()); + bytes.add_product((3 * publication.max_path_pages) + 1, sizeof(PageReplacement)); + bytes.add_product(2 * publication.max_path_pages, sizeof(uint64_t)); + bytes.add(*leaf); + bytes.add_product(window->leaves, *leaf); + bytes.add_product(changed, *inner); + // Each new child belongs to one replacement parent. Existing child arrays + // are covered above, including both inputs of an inner merge. Charge new + // route entries once across the transaction, not once per changed page. + bytes.add_product(changed, 3 * (sizeof(uint64_t) + sizeof(OwnedKey))); + bytes.add_product(changed, MappingView::path_copy_bound()); + if (!separate_snapshot_credit) { + bytes.add_product(changed, PublicationCoordinator::snapshot_change_reserve_bytes()); + } + bytes.add_product(changed, sizeof(PageReplacement) + 1024); + bytes.add_product(changed, OwnedKey::allocation_overhead()); + // Inner sibling repair changes two leaf-parent routes at the bottom + // level. Their direct pairs and incident changed-leaf pairs retire together. + const auto invalidated_pairs = + AllocationBound().add_product(2, publication.max_inner_children).add_product(2, changed).value(); + if (!invalidated_pairs) { + return {}; + } + bytes.add(MergePairMetrics::invalidation_bytes(*invalidated_pairs)); + bytes.add_product(options.max_candidates, MergePairMetrics::insertion_bytes()); + const auto changed_memberships = + detail::AllocationBound(mapping_bounds.merge_candidate_pages).add_product(options.max_candidates, 2).value(); + if (!changed_memberships) { + return {}; + } + bytes.add(PublicationCoordinator::merge_membership_reserve_bytes(*changed_memberships)); + if (track_residency) { + // A split step replaces one leaf with a bounded prefix and suffix; + // coalesce/merge touch at most two leaves. The remaining replacements + // are routing pages and cannot acquire a leaf-sized base array. + const auto leaves = window->leaves; + bytes.add_product(changed, PageCacheResidency::page_allocation_bound(1)); + bytes.add_product(leaves, PageCacheResidency::page_allocation_bound(std::max(size_t{1}, pages.max_base_views)) - + PageCacheResidency::page_allocation_bound(1)); + } + bytes.add_product(4, PublicationCoordinator::empty_commit_reserve_bytes()); + // Bounded merge arrays, coalesced entries/runs and separator/layout scratch. + bytes.add_product(8, pages.max_metadata_bytes); + bytes.add_product(pages.max_work_entries, (4 * sizeof(EntryRef)) + shared_allocation_bound<OverlayBatch>()); + bytes.add_product(pages.max_sources, 1024); + bytes.add_product(pages.max_base_views, 1024); + bytes.add_product(pages.max_output_pages, pages.primary_page_bytes); + return bytes.value(); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/maintenance_routing.cpp b/lib/crowdb-tree/src/btree/maintenance_routing.cpp new file mode 100644 index 000000000..ef62553aa --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_routing.cpp @@ -0,0 +1,226 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_publication_internal.h" + +#include <algorithm> +#include <limits> +#include <ranges> + +namespace crowdb::tree::detail +{ +namespace +{ +constexpr size_t kInnerOverhead = kFrameHeaderSize + kFrameTrailerSize + sizeof(uint64_t); + +Status validate_inner(const InnerPageState &inner, size_t metadata_limit) +{ + if (inner.children.empty() || inner.separators.size() != inner.children.size() - 1 || + inner.children.size() > metadata_limit / sizeof(uint64_t)) { + return Status::corruption("invalid inner routing shape or metadata bound"); + } + size_t bytes = inner.children.size() * sizeof(uint64_t); + for (size_t i = 0; i < inner.separators.size(); ++i) { + if (inner.separators[i].size() > metadata_limit - bytes || + (i != 0 && inner.separators[i - 1] >= inner.separators[i])) { + return Status::corruption("inner separators exceed the bound or are not ordered"); + } + bytes += inner.separators[i].size(); + } + return Status::Ok(); +} +} // namespace + +Status MaintenanceBuilder::path_to(uint64_t pid, MaintenancePath *path) const +{ + const auto leaf = view->mapping.get(pid); + if (!leaf || leaf->level != 0 || !leaf->leaf) { + return Status::invalid_argument("maintenance target must be a live leaf"); + } + uint64_t current = view->root_page_id; + PageKeyRange bounds; + while (current != pid) { + if (path->size() >= options.max_path_pages) { + return Status::resource_exhausted("maintenance route depth exceeded"); + } + const auto parent = view->mapping.get(current); + if (!parent || !parent->inner || parent->level == 0) { + return Status::unavailable("target leaf is not reachable through its range"); + } + auto status = validate_inner(*parent->inner, options.pages.max_metadata_bytes); + if (!status.ok()) { + return status; + } + const auto &inner = *parent->inner; + const size_t child = leaf->leaf->range.lower.has_value() + ? static_cast<size_t>(std::upper_bound(inner.separators.begin(), inner.separators.end(), + *leaf->leaf->range.lower) - + inner.separators.begin()) + : 0; + if (child != 0) { + bounds.lower = inner.separators[child - 1]; + } + if (child < inner.separators.size()) { + bounds.upper = inner.separators[child]; + } + const auto next = view->mapping.get(inner.children[child]); + if (!next || parent->level - 1 != next->level) { + return Status::corruption("inner child level does not match its parent"); + } + path->push_back({.pid = current, .child = child, .page = parent}); + current = inner.children[child]; + } + if ((bounds.lower.has_value() && bounds.lower != leaf->leaf->range.lower) || + (bounds.upper.has_value() && bounds.upper != leaf->leaf->range.upper)) { + return Status::corruption("leaf range disagrees with its parent route"); + } + return Status::Ok(); +} + +Status MaintenanceBuilder::guard_path(const MaintenancePath &path) +{ + for (const auto &entry : path) { + if (!changes_.contains(entry.pid)) { + auto status = replace(entry.pid, entry.page); + if (!status.ok()) { + return status; + } + } + } + return Status::Ok(); +} + +Status MaintenanceBuilder::partition_inner(uint64_t pid, uint32_t level, const InnerPageState &inner, + MaintenanceChildren *output) +{ + auto status = validate_inner(inner, options.pages.max_metadata_bytes); + if (!status.ok()) { + return status; + } + for (const auto &separator : inner.separators) { + if (separator.size() > options.max_inner_bytes - kInnerOverhead - sizeof(uint64_t) - kInnerSlotSize) { + return Status::invalid_argument("one separator cannot fit an inner page with two children"); + } + } + size_t begin = 0; + while (begin < inner.children.size()) { + size_t end = begin + 1; + size_t bytes = kInnerOverhead; + while (end < inner.children.size() && end - begin < options.max_inner_children) { + const size_t added = sizeof(uint64_t) + kInnerSlotSize + inner.separators[end - 1].size(); + if (added > options.max_inner_bytes - bytes) { + break; + } + bytes += added; + ++end; + } + uint64_t output_pid = pid; + if (begin != 0 || output_pid == kInvalidPageId) { + status = allocate_pid(&output_pid); + if (!status.ok()) { + return status; + } + } + auto content = reserved_shared<InnerPageState>(options.metadata, options.metadata); + content->children.assign(inner.children.begin() + static_cast<ptrdiff_t>(begin), + inner.children.begin() + static_cast<ptrdiff_t>(end)); + content->separators.assign(inner.separators.begin() + static_cast<ptrdiff_t>(begin), + inner.separators.begin() + static_cast<ptrdiff_t>(end - 1)); + auto page = reserved_shared<PublishedPage>(options.metadata); + const auto old = view->mapping.get(output_pid); + if (old && old->revision == UINT64_MAX) { + return Status::resource_exhausted("inner revision exhausted"); + } + page->revision = old ? old->revision + 1 : 1; + page->level = level; + page->inner = std::move(content); + page->metrics.leaf = false; + page->metrics.dirty = true; + page->metrics.writeback_candidate = true; + page->metrics.page_size_known = true; + page->metrics.page_size_bytes = bytes; + page->metrics.page_size_estimate_bytes = bytes; + status = replace(output_pid, std::move(page)); + if (!status.ok()) { + return status; + } + output->pids.push_back(output_pid); + if (end < inner.children.size()) { + output->separators.push_back(inner.separators[end - 1]); + } + begin = end; + } + return Status::Ok(); +} + +Status MaintenanceBuilder::grow_root(MaintenanceChildren children, uint32_t child_level) +{ + while (children.pids.size() > 1) { + if (child_level >= options.max_path_pages || child_level == std::numeric_limits<uint32_t>::max()) { + return Status::resource_exhausted("new root exceeds the route depth limit"); + } + InnerPageState inner(options.metadata); + inner.separators.assign(children.separators.begin(), children.separators.end()); + inner.children.assign(children.pids.begin(), children.pids.end()); + MaintenanceChildren parents(options.metadata); + auto status = partition_inner(kInvalidPageId, child_level + 1, inner, &parents); + if (!status.ok()) { + return status; + } + children = std::move(parents); + ++child_level; + } + root_pid_ = children.pids.front(); + return Status::Ok(); +} + +Status MaintenanceBuilder::propagate(const MaintenancePath &path, size_t removed_children, MaintenanceChildren children, + uint32_t child_level) +{ + for (const auto &parent : std::views::reverse(path)) { + InnerPageState inner(*parent.page->inner, options.metadata); + const auto at = static_cast<ptrdiff_t>(parent.child); + if (removed_children > inner.children.size() - parent.child) { + return Status::corruption("maintenance child replacement is outside its parent"); + } + inner.children.erase(inner.children.begin() + at, + inner.children.begin() + at + static_cast<ptrdiff_t>(removed_children)); + inner.children.insert(inner.children.begin() + at, children.pids.begin(), children.pids.end()); + inner.separators.erase(inner.separators.begin() + at, + inner.separators.begin() + at + static_cast<ptrdiff_t>(removed_children - 1)); + inner.separators.insert(inner.separators.begin() + at, children.separators.begin(), children.separators.end()); + MaintenanceChildren outputs(options.metadata); + auto status = partition_inner(parent.pid, parent.page->level, inner, &outputs); + if (!status.ok()) { + return status; + } + if (outputs.pids.size() == 1) { + return Status::Ok(); + } + children = std::move(outputs); + child_level = parent.page->level; + removed_children = 1; + } + return grow_root(std::move(children), child_level); +} + +Status MaintenanceBuilder::shrink_root() +{ + for (size_t depth = 0; depth < options.max_path_pages; ++depth) { + const auto root = lookup(root_pid_); + if (!root || !root->inner || root->inner->children.size() != 1) { + return root ? Status::Ok() : Status::corruption("maintenance root disappeared"); + } + const auto child_pid = root->inner->children.front(); + const auto child = lookup(child_pid); + if (!child || root->level == 0 || child->level != root->level - 1) { + return Status::corruption("root contraction has an invalid child"); + } + auto status = replace(root_pid_, {}); + if (!status.ok()) { + return status; + } + root_pid_ = child_pid; + } + return Status::resource_exhausted("root contraction depth exceeded"); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_split_layout.cpp b/lib/crowdb-tree/src/btree/maintenance_split_layout.cpp new file mode 100644 index 000000000..a0c65edd4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_split_layout.cpp @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "maintenance_split_layout.h" + +#include "allocation_bound.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +namespace +{ +constexpr size_t kPageOverhead = kFrameHeaderSize + kFrameTrailerSize; + +Status primary_bytes(Slice key, const LogicalCell &cell, const PageMaintenanceLimits &limits, size_t *bytes) +{ + const bool overflow = !cell.tombstone() && (cell.overflow() || cell.value.size() > limits.inline_value_bytes); + const size_t value = overflow ? kOverflowCellSize : kCellHeaderSize + cell.value.size(); + const auto available = limits.primary_page_bytes - kPageOverhead - kLeafSlotSize; + if (key.size() > available || value > available - key.size()) { + return Status::invalid_argument("one key/cell cannot fit an uncompressed primary leaf frame"); + } + *bytes = key.size() + value + kLeafSlotSize; + return Status::Ok(); +} +} // namespace + +SplitWindowCursor::SplitWindowCursor(std::shared_ptr<const PageState> page, const PageMaintenanceLimits &limits) + : page_(std::move(page)), + limits_(limits), + layout_(limits.metadata), + cursor_(page_, page_ ? page_->range : PageKeyRange{}, UINT64_MAX, limits.max_sources, false, limits.metadata) +{ + layout_.source = page_; +} + +std::optional<BaseLoadRequest> SplitWindowCursor::needs_base() const +{ + const auto index = cursor_.needs_base(); + if (!index) { + return std::nullopt; + } + return BaseLoadRequest{.page = page_, .base_index = *index}; +} + +Status SplitWindowCursor::advance() +{ + const auto &limits = limits_; + auto *layout = &layout_; + if (done_) { + return Status::Ok(); + } + if (limits.primary_page_bytes <= kPageOverhead + kLeafSlotSize || limits.primary_page_bytes > UINT32_MAX || + limits.target_page_bytes <= kPageOverhead || limits.target_page_bytes > limits.primary_page_bytes || + limits.max_work_entries == 0 || limits.max_output_pages < 2) { + return Status::invalid_argument("split window requires bounded work and at least two output ranges"); + } + if (!cursor_.status().ok()) { + return cursor_.status(); + } + if (cursor_.needs_base()) { + return Status::Ok(); + } + const auto outputs = limits.max_output_pages; + while (cursor_.valid()) { + const bool work_limit = entries_ == limits.max_work_entries; + size_t bytes = 0; + if (!work_limit) { + const auto status = primary_bytes(cursor_.key(), cursor_.cell(), limits, &bytes); + if (!status.ok()) { + return status; + } + } + if (work_limit || (current_bytes_ != kPageOverhead && + bytes > limits.target_page_bytes - std::min(current_bytes_, limits.target_page_bytes))) { + if (cursor_.key().size() > limits.max_metadata_bytes - separator_bytes_) { + return Status::resource_exhausted("split separators exceed metadata budget"); + } + layout->primary_bytes.push_back(current_bytes_); + layout->statistics.push_back(current_stats_); + layout->separators.emplace_back(cursor_.key(), limits.metadata); + separator_bytes_ += cursor_.key().size(); + if (work_limit || layout->separators.size() + 1 == outputs) { + done_ = true; + return Status::Ok(); + } + current_bytes_ = kPageOverhead; + current_stats_ = {}; + } + auto status = accumulate_leaf_stats(cursor_.key(), cursor_.cell(), ¤t_stats_); + if (!status.ok()) { + return status; + } + current_bytes_ += bytes; + ++entries_; + cursor_.next(); + if (!cursor_.status().ok()) { + return cursor_.status(); + } + if (cursor_.needs_base()) { + return Status::Ok(); + } + } + layout->primary_bytes.push_back(current_bytes_); + layout->statistics.push_back(current_stats_); + done_ = true; + return Status::Ok(); +} + +Status plan_split_window(const std::shared_ptr<const PageState> &page, const PageMaintenanceLimits &limits, + SplitWindowLayout *layout, MaintenancePublicationResult *result) +{ + SplitWindowCursor cursor(page, limits); + auto status = cursor.advance(); + if (status.ok()) { + if (auto request = cursor.needs_base()) { + result->readiness = MaintenanceReadiness::kNeedsBase; + result->loads.push_back(std::move(*request)); + } + else { + *layout = cursor.take_layout(); + } + } + return status; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_split_layout.h b/lib/crowdb-tree/src/btree/maintenance_split_layout.h new file mode 100644 index 000000000..f4aee5e86 --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_split_layout.h @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "maintenance_publication.h" + +namespace crowdb::tree::detail +{ +struct SplitWindowLayout +{ + explicit SplitWindowLayout(const std::shared_ptr<AllocationReservation> &metadata) + : separators(ReservationAllocator<OwnedKey>(metadata)), + primary_bytes(ReservationAllocator<size_t>(metadata)), + statistics(ReservationAllocator<LeafLayoutStats>(metadata)) + { + } + + ReservedVector<OwnedKey> separators; + // An unvisited suffix has no size entry; every other output is exact. + ReservedVector<size_t> primary_bytes; + ReservedVector<LeafLayoutStats> statistics; + std::shared_ptr<const PageState> source; +}; + +// Bounded split planning preserves exact output metrics for publication. +class SplitWindowCursor +{ + public: + SplitWindowCursor(std::shared_ptr<const PageState> page, const PageMaintenanceLimits &limits); + Status advance(); + + [[nodiscard]] bool done() const noexcept + { + return done_; + } + + [[nodiscard]] std::optional<BaseLoadRequest> needs_base() const; + + [[nodiscard]] const SplitWindowLayout &layout() const noexcept + { + return layout_; + } + + SplitWindowLayout take_layout() + { + return std::move(layout_); + } + + private: + std::shared_ptr<const PageState> page_; + PageMaintenanceLimits limits_; + SplitWindowLayout layout_; + LogicalPageCursor cursor_; + LeafLayoutStats current_stats_; + size_t current_bytes_ = kFrameHeaderSize + kFrameTrailerSize; + size_t separator_bytes_ = 0; + size_t entries_ = 0; + bool done_ = false; +}; + +Status plan_split_window(const std::shared_ptr<const PageState> &page, const PageMaintenanceLimits &limits, + SplitWindowLayout *layout, MaintenancePublicationResult *result); +MaintenancePublicationResult build_split_layout_publication(std::shared_ptr<const PublishedTreeState> view, + uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &options, + const SplitWindowLayout &layout); +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/maintenance_split_step.cpp b/lib/crowdb-tree/src/btree/maintenance_split_step.cpp new file mode 100644 index 000000000..0703b0bdb --- /dev/null +++ b/lib/crowdb-tree/src/btree/maintenance_split_step.cpp @@ -0,0 +1,207 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_publication_internal.h" +#include "maintenance_split_layout.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +void clip_bases(PageState &page) +{ + for (auto &base : page.bases) { + if (page.range.lower && (!base.range.lower || *page.range.lower > *base.range.lower)) { + base.range.lower = page.range.lower; + } + if (page.range.upper && (!base.range.upper || *page.range.upper < *base.range.upper)) { + base.range.upper = page.range.upper; + } + } + std::erase_if(page.bases, [](const auto &base) { return base.range.empty(); }); + std::erase_if(page.overlays, [&](const auto &run) { + const size_t first = page.range.lower ? run->lower_bound(*page.range.lower) : 0; + const size_t last = page.range.upper ? run->lower_bound(*page.range.upper) : run->entries().size(); + return first == last; + }); +} + +void count_runs(PublishedPage &page) +{ + page.metrics.overlay_runs = page.leaf->overlays.size(); + page.metrics.overlay_entries = 0; + for (const auto &run : page.leaf->overlays) { + const auto &range = page.leaf->range; + const size_t first = range.lower ? run->lower_bound(*range.lower) : 0; + const size_t last = range.upper ? run->lower_bound(*range.upper) : run->entries().size(); + page.metrics.overlay_entries += last - first; + } +} + +Status prepare_outputs(detail::MaintenanceBuilder &builder, const std::shared_ptr<const PublishedPage> &old, + const detail::MaintenanceChildren &children, const detail::SplitWindowLayout &layout, + PageVersion version) +{ + size_t prefix_bytes = 0; + Status status; + for (size_t i = 0; i < children.pids.size(); ++i) { + auto logical = + detail::reserved_shared<PageState>(builder.options.metadata, *old->leaf, builder.options.metadata); + logical->range.lower = i == 0 ? old->leaf->range.lower : std::optional(children.separators[i - 1]); + logical->range.upper = + i == children.separators.size() ? old->leaf->range.upper : std::optional(children.separators[i]); + logical->right_sibling = i + 1 == children.pids.size() ? old->leaf->right_sibling : children.pids[i + 1]; + logical->route_generation = version.route_generation; + logical->content_id = version.content_id; + logical->representation_id = version.representation_id; + clip_bases(*logical); + if (i < layout.primary_bytes.size()) { + status = builder.make_leaf(children.pids[i], old, std::move(logical), layout.primary_bytes[i], {}, + &layout.statistics[i]); + prefix_bytes += layout.primary_bytes[i]; + } + else { + // This suffix was not traversed; its size/count remain unknown. + auto tail = detail::reserved_shared<PublishedPage>(builder.options.metadata, *old); + tail->revision = 1; + tail->leaf = std::move(logical); + tail->primary.reset(); + + tail->metrics.page_size_known = false; + tail->metrics.kv_known = false; + tail->metrics.split_candidate = true; + tail->metrics.split_ready = false; + tail->metrics.dirty = true; + tail->metrics.writeback_candidate = true; + tail->metrics.page_size_estimate_bytes = old->metrics.page_size_estimate_bytes > prefix_bytes + ? old->metrics.page_size_estimate_bytes - prefix_bytes + : 0; + count_runs(*tail); + status = builder.replace(children.pids[i], std::move(tail)); + } + if (!status.ok()) { + return status; + } + } + return Status::Ok(); +} + +Status prepare_step(detail::MaintenanceBuilder &builder, uint64_t pid, MaintenancePublicationResult *result, + const detail::SplitWindowLayout *prepared = nullptr) +{ + auto status = builder.validate(); + detail::MaintenancePath path{detail::ReservationAllocator<detail::MaintenancePathEntry>(builder.options.metadata)}; + if (status.ok()) { + status = builder.path_to(pid, &path); + } + if (!status.ok()) { + return status; + } + const auto old = builder.view->mapping.get(pid); + detail::SplitWindowLayout layout(builder.options.metadata); + if (prepared != nullptr) { + if (prepared->source != old->leaf || prepared->primary_bytes.empty() || + prepared->statistics.size() != prepared->primary_bytes.size() || + prepared->separators.size() > prepared->primary_bytes.size() || + prepared->primary_bytes.size() > prepared->separators.size() + 1) { + return Status::invalid_argument("prepared split layout does not match its exact source"); + } + layout = *prepared; + } + else { + status = detail::plan_split_window(old->leaf, builder.options.pages, &layout, result); + } + if (!status.ok() || result->readiness != MaintenanceReadiness::kReady) { + return status; + } + if (layout.separators.empty()) { + if (old->metrics.page_size_known && old->metrics.kv_known && !old->metrics.split_candidate) { + return Status::Ok(); + } + status = builder.make_leaf(pid, old, old->leaf, layout.primary_bytes.front(), {}, &layout.statistics.front()); + if (status.ok()) { + status = builder.guard_path(path); + } + if (status.ok()) { + result->output_leaf_pids = {pid}; + status = builder.finish(result); + } + return status; + } + PageVersion version; + status = detail::next_page_version(*old->leaf, nullptr, &version); + if (!status.ok()) { + return status; + } + detail::MaintenanceChildren children(builder.options.metadata); + children.pids.push_back(pid); + children.separators = std::move(layout.separators); + for (size_t i = 0; i < children.separators.size(); ++i) { + uint64_t output = 0; + status = builder.allocate_pid(&output); + if (!status.ok()) { + return status; + } + children.pids.push_back(output); + } + status = prepare_outputs(builder, old, children, layout, version); + if (!status.ok()) { + return status; + } + result->output_leaf_pids = children.pids; + status = builder.propagate(path, 1, std::move(children), 0); + if (status.ok()) { + status = builder.guard_path(path); + } + if (status.ok()) { + status = builder.finish(result); + } + return status; +} +} // namespace + +MaintenancePublicationResult build_split_step_publication(std::shared_ptr<const PublishedTreeState> view, + uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &options) +{ + MaintenancePublicationResult result(options.metadata); + try { + detail::MaintenanceBuilder builder(std::move(view), std::move(record), options); + result.status = prepare_step(builder, leaf_pid, &result); + } + catch (const std::bad_alloc &) { + result.status = Status::resource_exhausted(); + } + if (!result.status.ok()) { + result.transaction = {}; + result.output_leaf_pids.clear(); + result.readiness = MaintenanceReadiness::kDeferred; + } + return result; +} +} // namespace crowdb::tree + +namespace crowdb::tree::detail +{ +MaintenancePublicationResult build_split_layout_publication(std::shared_ptr<const PublishedTreeState> view, + uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, + const MaintenancePublicationOptions &options, + const SplitWindowLayout &layout) +{ + MaintenancePublicationResult result(options.metadata); + try { + MaintenanceBuilder builder(std::move(view), std::move(record), options); + result.status = prepare_step(builder, leaf_pid, &result, &layout); + } + catch (const std::bad_alloc &) { + result.status = Status::resource_exhausted("prepared split publication credit unavailable"); + } + if (!result.status.ok()) { + result.transaction = {}; + result.output_leaf_pids.clear(); + result.readiness = MaintenanceReadiness::kDeferred; + } + return result; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/memtable_flush.cpp b/lib/crowdb-tree/src/btree/memtable_flush.cpp index dde82564f..7d2544894 100644 --- a/lib/crowdb-tree/src/btree/memtable_flush.cpp +++ b/lib/crowdb-tree/src/btree/memtable_flush.cpp @@ -1,276 +1,16 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -#include "crowdb-tree/btree/delta.h" -#include "crowdb-tree/btree/descent.h" -#include "crowdb-tree/crowdb-tree.h" -#include "merge_sources.h" - -#include <algorithm> -#include <chrono> -#include <cstring> -#include <thread> +#include "btree/tree_core.h" namespace crowdb::tree { -bool Crowdbtree::publish_group_to_leaf_locked(uint64_t page_id, uint64_t cs, std::vector<leaf_entry> group) -{ - PageBase *head = resident(page_id); - // In-frame delta fast path (PT12, opt-in): if the leaf is a bare base, try a - // cheap COW-append of this group as in-frame deltas instead of a heap delta - // node. Falls back to the heap path on no-room; folds at the delta cap. - if (opt_.inframe_delta && head != nullptr && head->type == page_type::kLeafBase) { - auto *leaf = static_cast<LeafBase *>(head); - uint32_t cur = leaf->view().delta_count(); - uint32_t after = cur + static_cast<uint32_t>(group.size()); - std::vector<uint8_t> out(leaf->page_bytes()); - if (after <= opt_.max_inframe_delta && - leaf_frame_append_deltas(leaf->frame(), leaf->page_bytes(), group, out.data())) { - LeafBase *fresh = LeafBase::from_frame_copy(out.data(), leaf->page_bytes(), pool_, opt_.frame_bytes); - store_preserving_parent_locked(page_id, fresh); - retire_page(leaf); - if (after >= opt_.max_inframe_delta || fresh->data_bytes() > opt_.leaf_split_bytes) { - consolidate_locked(page_id); - return true; - } - return false; - } - } - BatchDelta *delta = BatchDelta::build(cs, std::move(group), head); - store_preserving_parent_locked(page_id, delta); -#ifdef CROWDB_TREE_TEST_UTIL - if (after_flush_group_for_tests_ != nullptr) { - after_flush_group_for_tests_(); - } -#endif - if (delta->delta_len > opt_.max_delta_len || delta->chain_bytes > opt_.max_delta_bytes) { - consolidate_locked(page_id); - return true; - } - return false; -} - -bool Crowdbtree::drain_all_frozen_locked(std::deque<std::shared_ptr<MemTable>> &to_drain, uint64_t cs) -{ - // Phase 1: Open cursors on all frozen memtables for a non-destructive - // k-way merge read. Entries stay in L0 until covered source detachment, so - // concurrent scans always see them in L0 or L1, never in neither. - std::vector<ConcurrentSkipList::Cursor> cursors; - cursors.reserve(to_drain.size()); - bool has_any = false; - for (auto &mt : to_drain) { - cursors.push_back(mt->prefix_cursor(cs, last_applied_slot_.load())); - if (cursors.back().valid()) { - has_any = true; - } - } - if (!has_any) { - return false; - } - - // Phase 2: K-way merge the cursors with highest-slot-wins dedup, - // feeding the merged stream to a sort-aware descent + publish loop (O1+O5). - // Only entries with slot <= cs are emitted; entries with slot > cs are - // skipped (they remain in L0 for a future flush). - std::vector<MergeSource> sources; - sources.reserve(cursors.size()); - for (auto &c : cursors) { - sources.push_back({.kind = MergeSource::kL0, .l0 = &c, .l1 = nullptr}); - } - LoserTree lt; - lt.init(sources.data(), static_cast<int>(sources.size())); - - flush_drain_total_.fetch_add(1, std::memory_order_relaxed); - if (metrics_.flush_drain_c != nullptr) { - metrics_.flush_drain_c->inc(); - } - - auto resolve = [this](uint64_t p) { return resident(p); }; - uint64_t page_id = kInvalidPageId; - Slice high_key; - bool have_leaf = false; - uint64_t entries_published = 0; - std::vector<leaf_entry> group; - buffer materialized_cell; - - while (lt.winner_valid()) { - int w = lt.winner(); - Slice key = sources[w].key(); - uint64_t slot = sources[w].slot(); - - if (slot > cs) { - // Not yet contiguous — skip (remains in L0 for a future flush). - lt.advance_winner(); - continue; - } - - // Sort-aware descent (O1): reuse the cached leaf when the key is - // within its range (key <= high_key). Re-descend when crossing a - // leaf boundary. After a publish that triggers a consolidate (which - // may split), the cached high_key is stale — the next key > high_key - // check naturally re-descends. - if (!have_leaf || key.compare(high_key) > 0) { - if (!group.empty()) { - publish_group_to_leaf_locked(page_id, cs, std::move(group)); - group.clear(); - } - page_id = find_leaf_page_id(resolve, root_page_id_.load(), key); - PageBase *head = resident(page_id); - LeafBase *leaf = chain_leaf_base(head); - high_key = leaf != nullptr ? leaf->high_key() : Slice(); - have_leaf = true; - } - - // Materialize the winning cell from the cursor's CellVersion. - const CellVersion *cv = sources[w].l0->cell_version(); - if (cv != nullptr && cv->cell.ownership() != buffer::mode::kExternal) { - materialized_cell = cv->cell.clone(); - } - else if (cv != nullptr) { - size_t vlen = cv->cell.size(); - materialized_cell = buffer::alloc(vlen, kCellHeaderSize); - uint8_t *p = materialized_cell.data(); - for (int i = 0; i < 8; ++i) { - p[i] = static_cast<uint8_t>((cv->slot >> (8 * i)) & 0xff); - } - p[8] = cv->flags; - if (vlen > 0) { - std::memcpy(p + kCellHeaderSize, cv->cell.data(), vlen); - } - } - else { - materialized_cell = buffer::alloc(0, kCellHeaderSize); - } - group.push_back({.key = key.to_string(), .cell = std::move(materialized_cell)}); - ++entries_published; - lt.advance_winner(); - - // Collision drain: advance all other sources on the same key - // (duplicate — highest-slot-wins among <= cs already picked the winner). - while (lt.winner_valid() && sources[lt.winner()].key().compare(key) == 0) { - lt.drain_winner(); - } - } - - // Publish the last pending group. - if (!group.empty()) { - publish_group_to_leaf_locked(page_id, cs, std::move(group)); - } - - flush_entries_total_.fetch_add(entries_published, std::memory_order_relaxed); - if (metrics_.flush_entries_c != nullptr) { - metrics_.flush_entries_c->inc_by(entries_published); - } - - return entries_published > 0; -} - -Crowdbtree::FlushBoundary Crowdbtree::capture_flush_locked() -{ - FlushBoundary boundary; - auto &captured = boundary.tables; - auto &frontier = boundary.frontier; - { - std::unique_lock catalog(memtable_mutex_); - // Every allocation precedes closure. Publishing the successor after - // the bound capture prevents new batches from widening this flush. - auto successor = std::make_shared<MemTable>(memtable_next_id_.fetch_add(1), &epoch_); - successor->set_durable_floor(last_applied_slot_.load()); - captured = frozen_; - captured.insert(captured.end(), split_shared_memtables_.begin(), split_shared_memtables_.end()); - captured.push_back(active_); - frozen_.push_back(active_); - active_->close(); - frontier = contiguous_slot_.load(std::memory_order_acquire); - active_ = std::move(successor); - } - return boundary; -} - -void Crowdbtree::publish_flush_locked(FlushBoundary &boundary) -{ - auto &captured = boundary.tables; - const auto frontier = boundary.frontier; - for (const auto &table : captured) { - table->wait_frozen(); - } - publication_incomplete_ = true; - drain_all_frozen_locked(captured, frontier); - { - std::unique_lock catalog(memtable_mutex_); - // All prefix publications precede the new catalog floor. Old readers - // retain immutable sources; future records never move between tables. - last_applied_slot_.store(frontier, std::memory_order_release); - active_->set_durable_floor(frontier); - publication_incomplete_ = false; - for (auto it = frozen_.begin(); it != frozen_.end();) { - const auto &table = *it; - if (std::find(captured.begin(), captured.end(), table) != captured.end() && - table->slot_range().max <= frontier) { - it = frozen_.erase(it); - } - else { - ++it; - } - } - } -} - Status Crowdbtree::flush() -try { - auto started = std::chrono::steady_clock::now(); - std::scoped_lock writer(write_mutex_); - if (split_overlay_source_.load() != nullptr && last_applied_slot_.load() < split_overlay_frontier_.load()) { - return Status::unavailable("split overlay does not establish published coverage"); - } - auto guard = epoch_.enter(); - auto boundary = capture_flush_locked(); - publish_flush_locked(boundary); - version_.fetch_add(1); - maybe_evict_locked(); - if (metrics_.flush_l != nullptr) { - metrics_.flush_l->observe(static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count())); - } - guard = {}; - boundary.tables.clear(); - epoch_.try_reclaim(); - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("flush allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); +{ + return core_->flush(); } void Crowdbtree::flush_async(std::function<void(Status)> on_done) { - auto pending = async_flushes_; - auto completion = std::make_shared<std::function<void(Status)>>(std::move(on_done)); - pending->fetch_add(1); - try { - std::thread([this, pending, completion] { - Status status; - try { - status = flush(); - } - catch (const std::bad_alloc &) { - status = Status::resource_exhausted("flush allocation failed"); - } - catch (const std::exception &error) { - status = Status::internal_error(error.what()); - } - pending->fetch_sub(1); - pending->notify_all(); - (*completion)(status); - }).detach(); - } - catch (const std::exception &error) { - pending->fetch_sub(1); - pending->notify_all(); - (*completion)(Status::resource_exhausted(error.what())); - } + core_->flush_async(std::move(on_done)); } - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/memtable_sources.cpp b/lib/crowdb-tree/src/btree/memtable_sources.cpp index 0b9641ccb..2526dbc65 100644 --- a/lib/crowdb-tree/src/btree/memtable_sources.cpp +++ b/lib/crowdb-tree/src/btree/memtable_sources.cpp @@ -1,113 +1,16 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -#include "crowdb-common/log.h" -#include "crowdb-tree/crowdb-tree.h" +#include "btree/tree_core.h" namespace crowdb::tree { -std::shared_ptr<MemTable> Crowdbtree::current_active() const -{ - std::shared_lock<std::shared_mutex> lk(memtable_mutex_); - return active_; -} - -std::vector<MemTableSource> Crowdbtree::all_memtables() const -{ - auto out = local_memtables(); - if (Crowdbtree *source = split_overlay_source_.load(std::memory_order_acquire); source != nullptr) { - // Parent flush coverage does not establish coverage in the child. - auto inherited = source->local_memtables(last_applied_slot_.load()); - out.insert(out.end(), inherited.begin(), inherited.end()); - } - return out; -} - -std::vector<MemTableSource> Crowdbtree::local_memtables(uint64_t covered) const -{ - std::shared_lock<std::shared_mutex> lk(memtable_mutex_); - std::vector<MemTableSource> out; - const auto floor = covered == UINT64_MAX ? last_applied_slot_.load(std::memory_order_acquire) : covered; - out.reserve(split_shared_memtables_.size() + frozen_.size() + 1); - for (const auto &mt : split_shared_memtables_) { - out.emplace_back(mt, floor); - } - for (const auto &mt : frozen_) { - out.emplace_back(mt, floor); - } - out.emplace_back(active_, floor); - return out; -} - -bool Crowdbtree::maybe_freeze_active(bool force) -{ - std::shared_ptr<MemTable> active = current_active(); - if (!force && active->approx_bytes() < opt_.memtable_flush_bytes && active->count() < opt_.memtable_flush_entries) { - return false; - } - std::unique_lock<std::shared_mutex> lk(memtable_mutex_); - // Re-check under the exclusive lock: another thread may have already - // frozen this exact active_ (or installed a fresh, still-small one) - // between the check above and taking the lock. - if (active_ != active || (active_->empty() && active_->writers() == 0)) { - return false; - } - if (!force) { - size_t max_frozen = opt_.max_memtable_count > 1 ? static_cast<size_t>(opt_.max_memtable_count) - 1 : 1; - if (frozen_.size() >= max_frozen) { - if (active_->mark_backlog_warning()) { - CRB_LOG_WARN("[{}] memtable backlog: frozen={} active_bytes={} contiguous={} published={}; " - "active remains writable, inspect stalled writers/readers or slot gaps", - name_, frozen_.size(), active_->approx_bytes(), contiguous_slot_.load(), - last_applied_slot_.load()); - } - // At capacity: no free buffer slot. Let active_ keep growing past - // its threshold rather than stall the writer -- an explicit - // flush()/the background thread is expected to drain a slot free - // (documented in Config::max_memtable_count). - return false; - } - } - auto successor = std::make_shared<MemTable>(memtable_next_id_.fetch_add(1, std::memory_order_relaxed), &epoch_); - successor->set_durable_floor(last_applied_slot_.load()); - frozen_.push_back(active_); - active_->close(); - active_ = std::move(successor); - if (metrics_.mt_freeze_c != nullptr) { - metrics_.mt_freeze_c->inc(); - } - // Propagate the known-durable floor to the fresh table immediately (not - // just on its first flush()) so a stale re-apply landing in it before - // its own first drain is still correctly rejected. - active_->set_durable_floor(last_applied_slot_.load()); - return true; -} - -void Crowdbtree::maybe_swap_active() -{ - try { - maybe_freeze_active(/*force=*/false); - } - catch (const std::bad_alloc &) { - // Soft rotation is optional; the batch is already applied. All - // allocations precede closure, so the active table remains usable. - return; - } -} - size_t Crowdbtree::memtable_count() const { - size_t n = 0; - for (auto &mt : all_memtables()) { - n += mt->count(); - } - return n; + return core_->memtable_count(); } size_t Crowdbtree::frozen_table_count() const { - std::shared_lock<std::shared_mutex> lk(memtable_mutex_); - return frozen_.size(); + return core_->frozen_table_count(); } - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/memtable_split.cpp b/lib/crowdb-tree/src/btree/memtable_split.cpp index bab308011..6e4c10b7b 100644 --- a/lib/crowdb-tree/src/btree/memtable_split.cpp +++ b/lib/crowdb-tree/src/btree/memtable_split.cpp @@ -1,214 +1,32 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -#include "crowdb-tree/btree/delta.h" -#include "crowdb-tree/btree/descent.h" -#include "crowdb-tree/crowdb-tree.h" - -#include <algorithm> -#include <chrono> +#include "btree/tree_core.h" namespace crowdb::tree { -Status Crowdbtree::install_split_memtable_overlay(Crowdbtree &source, uint64_t journal_frontier) -try { - if (&source == this) { - return Status::invalid_argument("split memtable overlay source must differ from destination"); - } - Crowdbtree *expected = nullptr; - if (!split_overlay_source_.compare_exchange_strong(expected, &source, std::memory_order_acq_rel) && - expected != &source) { - return Status::invalid_argument("another split memtable overlay is already installed"); - } - split_overlay_frontier_.store(journal_frontier); - force_advance_slot(journal_frontier); - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("operation allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); +Status Crowdbtree::install_split_memtable_overlay(Crowdbtree &source, uint64_t frontier) +{ + return core_->install_split_overlay(*source.core_, frontier); } Status Crowdbtree::clear_split_memtable_overlay(Crowdbtree &source) { - if (last_applied_slot_.load() < split_overlay_frontier_.load()) { - return Status::unavailable("split prefix publication is incomplete"); - } - Crowdbtree *expected = &source; - if (!split_overlay_source_.compare_exchange_strong(expected, nullptr, std::memory_order_acq_rel)) { - if (expected == nullptr) { - return Status::Ok(); - } - return Status::invalid_argument("split memtable overlay source does not match"); - } - return Status::Ok(); + return core_->clear_split_overlay(*source.core_); } -Status Crowdbtree::begin_split_memtable_view(uint64_t *out_generation, uint64_t *out_journal_frontier) -try { - const auto started = std::chrono::steady_clock::now(); - if (out_generation == nullptr || out_journal_frontier == nullptr) { - return Status::invalid_argument("split memtable view requires output parameters"); - } - std::scoped_lock write_lk(write_mutex_); - std::unique_lock memtable_lk(memtable_mutex_); - if (!split_shared_memtables_.empty()) { - return Status::invalid_argument("a split memtable view is already active"); - } - auto successor = std::make_shared<MemTable>(memtable_next_id_.fetch_add(1), &epoch_); - successor->set_durable_floor(last_applied_slot_.load()); - split_shared_memtables_.reserve(frozen_.size() + 1); - split_shared_memtables_.insert(split_shared_memtables_.end(), frozen_.begin(), frozen_.end()); - frozen_.clear(); - split_shared_memtables_.push_back(active_); - active_->close(); - const auto frontier = contiguous_slot_.load(std::memory_order_acquire); - split_memtable_frontier_ = frontier; - active_ = std::move(successor); - ++split_memtable_generation_; - *out_generation = split_memtable_generation_; - *out_journal_frontier = frontier; - memtable_lk.unlock(); - for (const auto &table : split_shared_memtables_) { - table->wait_frozen(); - } - if (metrics_.split_view_begin_l != nullptr) { - metrics_.split_view_begin_l->observe(static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count())); - } - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("operation allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); +Status Crowdbtree::begin_split_memtable_view(uint64_t *generation, uint64_t *frontier) +{ + return core_->begin_split_view(generation, frontier); } Status Crowdbtree::release_split_memtable_view(uint64_t generation) -try { - const auto started = std::chrono::steady_clock::now(); - std::scoped_lock write_lk(write_mutex_); - std::unique_lock memtable_lk(memtable_mutex_); - if (generation == 0 || generation != split_memtable_generation_ || split_shared_memtables_.empty()) { - return Status::invalid_argument("split memtable view generation is not active"); - } - auto pending = frozen_; - for (const auto &table : split_shared_memtables_) { - if (table->slot_range().max > last_applied_slot_.load()) { - pending.push_back(table); - } - } - frozen_.swap(pending); - split_shared_memtables_.clear(); - if (metrics_.split_view_release_l != nullptr) { - metrics_.split_view_release_l->observe(static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count())); - } - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("operation allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); +{ + return core_->release_split_view(generation); } -Status Crowdbtree::publish_split_memtable_view(uint64_t generation, uint64_t journal_frontier, Crowdbtree &destination, +Status Crowdbtree::publish_split_memtable_view(uint64_t generation, uint64_t frontier, Crowdbtree &destination, const KeyRange &range) -try { - const auto started = std::chrono::steady_clock::now(); - if (&destination == this) { - return Status::invalid_argument("split memtable destination must differ from its source"); - } - Status range_status = range.validate(); - if (!range_status.ok()) { - return range_status; - } - std::scoped_lock write_lk(write_mutex_, destination.write_mutex_); - std::vector<mem_entry> entries; - { - std::shared_lock memtable_lk(memtable_mutex_); - if (generation == 0 || generation != split_memtable_generation_ || split_shared_memtables_.empty()) { - return Status::invalid_argument("split memtable view generation is not active"); - } - if (journal_frontier > split_memtable_frontier_) { - return Status::invalid_argument("split publication exceeds its captured frontier"); - } - for (const auto &table : split_shared_memtables_) { - auto snapshot = table->snapshot(journal_frontier); - entries.insert(entries.end(), std::make_move_iterator(snapshot.begin()), - std::make_move_iterator(snapshot.end())); - } - } - std::sort(entries.begin(), entries.end(), [](const mem_entry &left, const mem_entry &right) { - return left.key == right.key ? left.slot > right.slot : left.key < right.key; - }); - - // The retained parent keeps appending while the view is captured in the - // background. Publish only the inherited prefix, without crediting later - // parent records to the child's independent sequence namespace. - const uint64_t cs = journal_frontier; - uint64_t page_id = kInvalidPageId; - Slice high_key; - bool have_leaf = false; - std::vector<leaf_entry> group; - if (destination.last_applied_slot_.load() > cs) { - return Status::invalid_argument("split prefix precedes destination publication"); - } - destination.publication_incomplete_ = true; - for (size_t index = 0; index < entries.size();) { - mem_entry &entry = entries[index]; - const std::string key = entry.key; - ++index; - while (index < entries.size() && entries[index].key == key) { - ++index; - } - if (entry.slot > cs || !range.contains(Slice(key))) { - continue; - } - Slice key_slice(key); - // An empty high key is the rightmost leaf's +infinity sentinel. It - // must retain the rest of this bulk publish as one leaf group; treating - // it as an ordinary empty key would re-find and publish every entry - // separately. - if (!have_leaf || (!high_key.empty() && key_slice.compare(high_key) > 0)) { - if (!group.empty()) { - destination.publish_group_to_leaf_locked(page_id, cs, std::move(group)); - group.clear(); - } - page_id = find_leaf_page_id([&destination](uint64_t page) { return destination.resident(page); }, - destination.root_page_id_.load(), key_slice); - auto *head = destination.resident(page_id); - auto *leaf = chain_leaf_base(head); - high_key = leaf != nullptr ? leaf->high_key() : Slice(); - have_leaf = true; - } - group.push_back({.key = key, .cell = std::move(entry.cell)}); - } - if (!group.empty()) { - destination.publish_group_to_leaf_locked(page_id, cs, std::move(group)); - } - destination.force_advance_slot(cs); - { - std::unique_lock catalog(destination.memtable_mutex_); - destination.last_applied_slot_.store(cs); - destination.active_->set_durable_floor(cs); - destination.publication_incomplete_ = false; - } - destination.version_.fetch_add(1); - if (metrics_.split_view_publish_l != nullptr) { - metrics_.split_view_publish_l->observe(static_cast<uint64_t>( - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started).count())); - } - return Status::Ok(); -} -catch (const std::bad_alloc &) { - return Status::resource_exhausted("operation allocation failed"); -} -catch (const std::exception &error) { - return Status::internal_error(error.what()); +{ + return core_->publish_split_view(generation, frontier, *destination.core_, range); } - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/merge_pair_metrics.cpp b/lib/crowdb-tree/src/btree/merge_pair_metrics.cpp new file mode 100644 index 000000000..651221595 --- /dev/null +++ b/lib/crowdb-tree/src/btree/merge_pair_metrics.cpp @@ -0,0 +1,310 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "merge_pair_metrics.h" + +#include "allocation_bound.h" + +#include <algorithm> +#include <cassert> +#include <stdexcept> + +namespace crowdb::tree +{ +struct MergePairMetrics::Retired +{ + struct Nodes + { + Pairs::node_type pair; + std::array<Index::node_type, 4> index; + }; + + Retired(const std::shared_ptr<detail::AllocationReservation> &metadata, std::shared_ptr<const void> resource) + : resource_owner(std::move(resource)), + nodes(decltype(nodes)::allocator_type(metadata)) + { + } + + bool retire_step() noexcept + { + for (size_t count = 0; count < 16 && !nodes.empty(); ++count) { + nodes.pop_back(); + } + return nodes.empty(); + } + + std::shared_ptr<const void> resource_owner; + detail::ReservedVector<Nodes> nodes; +}; + +MergePairMetrics::MergePairMetrics(std::pmr::memory_resource *resource) : pairs_(resource), index_(resource) +{ +} + +MergePairMetrics::Prepared::Prepared(MergePairMetrics &owner, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const void> resource_owner) + : owner_(owner), + removed_(decltype(removed_)::allocator_type(metadata)), + endpoints_(decltype(endpoints_)::allocator_type(metadata)), + membership_(decltype(membership_)::allocator_type(metadata)), + added_pairs_(owner.pairs_.get_allocator().resource()), + added_index_(owner.index_.get_allocator().resource()), + retired_(detail::reserved_shared<Retired>(metadata, metadata, std::move(resource_owner))), + next_(owner.totals_) +{ +} + +void MergePairMetrics::Prepared::invalidate(Kind kind, uint64_t value) +{ + assert(!finished_); + auto it = owner_.index_.lower_bound({.kind = kind, .value = value, .left = 0}); + while (it != owner_.index_.end() && it->first.kind == kind && it->first.value == value) { + removed_.push_back(it->first.left); + ++it; + } +} + +void MergePairMetrics::Prepared::invalidate_page(uint64_t pid) +{ + invalidate(Kind::Endpoint, pid); +} + +void MergePairMetrics::Prepared::invalidate_parent(uint64_t pid) +{ + invalidate(Kind::Parent, pid); +} + +void MergePairMetrics::Prepared::clear_pair(uint64_t left) +{ + assert(!finished_); + if (owner_.pairs_.contains(left)) { + removed_.push_back(left); + } +} + +void MergePairMetrics::Prepared::set_pair(MergePairProof proof, uint64_t now_ns) +{ + assert(!finished_ && added_pairs_.empty()); + if (proof.left == proof.right || proof.parent == proof.left || proof.parent == proof.right || now_ns == 0) { + throw std::invalid_argument("invalid canonical merge pair"); + } + // Each right endpoint has at most one predecessor in the live sibling + // chain. Replacing an outgoing pair retires both old endpoint references. + const auto before = owner_.pairs_.find(proof.left); + if (before != owner_.pairs_.end()) { + auto old = before->second.proof; + old.ready = proof.ready; + if (old == proof) { + now_ns = before->second.since_ns; + } + removed_.push_back(proof.left); + } + added_pairs_.emplace(proof.left, Pair{.proof = proof, .since_ns = now_ns}); + added_index_.emplace(Key{.kind = Kind::Endpoint, .value = proof.left, .left = proof.left}, proof.right); + added_index_.emplace(Key{.kind = Kind::Endpoint, .value = proof.right, .left = proof.left}, proof.right); + added_index_.emplace(Key{.kind = Kind::Parent, .value = proof.parent, .left = proof.left}, proof.right); + added_index_.emplace(Key{.kind = Kind::Time, .value = now_ns, .left = proof.left}, proof.right); +} + +bool MergePairMetrics::Prepared::removes(uint64_t left) const noexcept +{ + return std::binary_search(removed_.begin(), removed_.end(), left); +} + +void MergePairMetrics::Prepared::finish() +{ + assert(!finished_); + std::sort(removed_.begin(), removed_.end()); + removed_.erase(std::unique(removed_.begin(), removed_.end()), removed_.end()); + retired_->nodes.resize(removed_.size()); + endpoints_.reserve((removed_.size() + added_pairs_.size()) * 2); + for (const auto left : removed_) { + const auto &pair = owner_.pairs_.at(left); + --next_.pairs; + next_.ready -= static_cast<uint64_t>(pair.proof.ready); + endpoints_.push_back(pair.proof.left); + endpoints_.push_back(pair.proof.right); + } + for (const auto &[left, pair] : added_pairs_) { + (void)left; + ++next_.pairs; + next_.ready += static_cast<uint64_t>(pair.proof.ready); + endpoints_.push_back(pair.proof.left); + endpoints_.push_back(pair.proof.right); + } + std::sort(endpoints_.begin(), endpoints_.end()); + endpoints_.erase(std::unique(endpoints_.begin(), endpoints_.end()), endpoints_.end()); + membership_.reserve(endpoints_.size()); + for (const auto pid : endpoints_) { + bool after = false; + bool before = false; + auto it = owner_.index_.lower_bound({.kind = Kind::Endpoint, .value = pid, .left = 0}); + while (it != owner_.index_.end() && it->first.kind == Kind::Endpoint && it->first.value == pid) { + before = true; + after = after || !removes(it->first.left); + ++it; + } + auto added = added_index_.lower_bound({.kind = Kind::Endpoint, .value = pid, .left = 0}); + after = + after || (added != added_index_.end() && added->first.kind == Kind::Endpoint && added->first.value == pid); + if (before != after) { + membership_.push_back({.pid = pid, .present = after}); + } + next_.pages -= static_cast<uint64_t>(before); + next_.pages += static_cast<uint64_t>(after); + } + next_.oldest_since_ns = 0; + auto time = owner_.index_.lower_bound({.kind = Kind::Time, .value = 0, .left = 0}); + while (time != owner_.index_.end() && removes(time->first.left)) { + ++time; + } + if (time != owner_.index_.end()) { + next_.oldest_since_ns = time->first.value; + } + for (const auto &[left, pair] : added_pairs_) { + (void)left; + if (next_.oldest_since_ns == 0 || pair.since_ns < next_.oldest_since_ns) { + next_.oldest_since_ns = pair.since_ns; + } + } + finished_ = true; +} + +std::span<const MergePairMetrics::Membership> MergePairMetrics::Prepared::membership_changes() const noexcept +{ + assert(finished_); + return membership_; +} + +std::span<const uint64_t> MergePairMetrics::Prepared::affected_pages() const noexcept +{ + assert(finished_); + return endpoints_; +} + +std::optional<MergePairProof> MergePairMetrics::Prepared::pair(uint64_t left) const noexcept +{ + assert(finished_); + const auto found = added_pairs_.find(left); + if (found != added_pairs_.end()) { + return found->second.proof; + } + return removes(left) ? std::nullopt : owner_.pair(left); +} + +bool MergePairMetrics::Prepared::contains_page(uint64_t pid) const noexcept +{ + assert(finished_); + const auto found = std::lower_bound(membership_.begin(), membership_.end(), pid, + [](const Membership &entry, uint64_t value) { return entry.pid < value; }); + return found != membership_.end() && found->pid == pid ? found->present : owner_.contains_page(pid); +} + +MergePairMetrics::Snapshot MergePairMetrics::Prepared::snapshot() const noexcept +{ + assert(finished_); + return next_; +} + +std::shared_ptr<const void> MergePairMetrics::Prepared::commit() noexcept +{ + assert(finished_ && !committed_); + committed_ = true; + size_t position = 0; + for (const auto left : removed_) { + const auto found = owner_.pairs_.find(left); + assert(found != owner_.pairs_.end()); + const auto pair = found->second; + auto &nodes = retired_->nodes[position++]; + nodes.pair = owner_.pairs_.extract(found); + nodes.index = {owner_.index_.extract({.kind = Kind::Endpoint, .value = pair.proof.left, .left = left}), + owner_.index_.extract({.kind = Kind::Endpoint, .value = pair.proof.right, .left = left}), + owner_.index_.extract({.kind = Kind::Parent, .value = pair.proof.parent, .left = left}), + owner_.index_.extract({.kind = Kind::Time, .value = pair.since_ns, .left = left})}; + } + while (!added_pairs_.empty()) { + owner_.pairs_.insert(added_pairs_.extract(added_pairs_.begin())); + } + while (!added_index_.empty()) { + owner_.index_.insert(added_index_.extract(added_index_.begin())); + } + owner_.totals_ = next_; + return std::move(retired_); +} + +MergePairMetrics::Snapshot MergePairMetrics::snapshot() const noexcept +{ + return totals_; +} + +std::optional<MergePairProof> MergePairMetrics::pair(uint64_t left) const noexcept +{ + const auto found = pairs_.find(left); + return found == pairs_.end() ? std::nullopt : std::optional(found->second.proof); +} + +bool MergePairMetrics::contains_page(uint64_t pid) const noexcept +{ + const auto found = index_.lower_bound({.kind = Kind::Endpoint, .value = pid, .left = 0}); + return found != index_.end() && found->first.kind == Kind::Endpoint && found->first.value == pid; +} + +bool MergePairMetrics::consistent() const noexcept +{ + Snapshot actual; + uint64_t last = UINT64_MAX; + for (const auto &[key, value] : index_) { + const auto found = pairs_.find(key.left); + if (found == pairs_.end() || found->second.proof.right != value) { + return false; + } + if (key.kind == Kind::Endpoint && key.value != last) { + ++actual.pages; + last = key.value; + } + } + for (const auto &[left, pair] : pairs_) { + ++actual.pairs; + actual.ready += static_cast<uint64_t>(pair.proof.ready); + if (actual.oldest_since_ns == 0 || pair.since_ns < actual.oldest_since_ns) { + actual.oldest_since_ns = pair.since_ns; + } + for (const auto key : { + Key{.kind = Kind::Endpoint, .value = pair.proof.left, .left = left}, + Key{.kind = Kind::Endpoint, .value = pair.proof.right, .left = left}, + Key{.kind = Kind::Parent, .value = pair.proof.parent, .left = left}, + Key{.kind = Kind::Time, .value = pair.since_ns, .left = left} + }) { + if (!index_.contains(key)) { + return false; + } + } + } + return actual == totals_ && index_.size() == pairs_.size() * 4; +} + +bool MergePairMetrics::retire_step() noexcept +{ + for (size_t n = 0; n < 64 && !index_.empty(); ++n) { + index_.erase(index_.begin()); + } + for (size_t n = 0; n < 16 && !pairs_.empty(); ++n) { + pairs_.erase(pairs_.begin()); + } + return index_.empty() && pairs_.empty(); +} + +size_t MergePairMetrics::invalidation_bytes(size_t pairs) noexcept +{ + return detail::AllocationBound(detail::shared_allocation_bound<Retired>()) + .add_product(pairs, sizeof(Retired::Nodes) + 192) + .value() + .value_or(SIZE_MAX); +} + +size_t MergePairMetrics::insertion_bytes() noexcept +{ + // Five red-black nodes, their allocation headers and staging arrays. + return invalidation_bytes(1) + 1536; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/merge_pair_metrics.h b/lib/crowdb-tree/src/btree/merge_pair_metrics.h new file mode 100644 index 000000000..d1b635044 --- /dev/null +++ b/lib/crowdb-tree/src/btree/merge_pair_metrics.h @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "crowdb-tree/btree/diagnostics.h" + +#include <array> +#include <compare> +#include <map> +#include <memory_resource> +#include <optional> +#include <span> + +namespace crowdb::tree +{ +// The left-to-right sibling relation gives every adjacent pair one identity. +// The proof contains logical versions only; a representation rebase preserves it. +struct MergePairProof +{ + uint64_t left = 0; + uint64_t right = 0; + uint64_t parent = 0; + uint64_t left_content = 0; + uint64_t right_content = 0; + uint64_t left_route = 0; + uint64_t right_route = 0; + bool ready = false; + + bool operator==(const MergePairProof &) const = default; +}; + +class MergePairMetrics +{ + struct Pair + { + MergePairProof proof; + uint64_t since_ns; + }; + enum class Kind : uint8_t { Endpoint, Parent, Time }; + + struct Key + { + Kind kind; + uint64_t value; + uint64_t left; + auto operator<=>(const Key &) const = default; + }; + + using Pairs = std::pmr::map<uint64_t, Pair>; + using Index = std::pmr::map<Key, uint64_t>; + struct Retired; + + public: + struct Snapshot + { + uint64_t pages = 0; + uint64_t pairs = 0; + uint64_t ready = 0; + uint64_t oldest_since_ns = 0; + bool operator==(const Snapshot &) const = default; + }; + + struct Membership + { + uint64_t pid; + bool present; + }; + + explicit MergePairMetrics(std::pmr::memory_resource *resource = std::pmr::get_default_resource()); + MergePairMetrics(const MergePairMetrics &) = delete; + MergePairMetrics &operator=(const MergePairMetrics &) = delete; + + // Only the publication owner accesses this ledger. Preparation leaves the + // live maps untouched. Commit moves already allocated map nodes and hands + // old nodes to a bounded retirement owner; it cannot allocate or fail. + class Prepared + { + public: + Prepared(MergePairMetrics &owner, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const void> resource_owner); + Prepared(const Prepared &) = delete; + Prepared &operator=(const Prepared &) = delete; + [[nodiscard]] std::span<const Membership> membership_changes() const noexcept; + [[nodiscard]] std::span<const uint64_t> affected_pages() const noexcept; + [[nodiscard]] std::optional<MergePairProof> pair(uint64_t left) const noexcept; + [[nodiscard]] bool contains_page(uint64_t pid) const noexcept; + void invalidate_page(uint64_t pid); + void invalidate_parent(uint64_t pid); + void clear_pair(uint64_t left); + void set_pair(MergePairProof proof, uint64_t now_ns); + void finish(); + [[nodiscard]] Snapshot snapshot() const noexcept; + [[nodiscard]] std::shared_ptr<const void> commit() noexcept; + + private: + void invalidate(Kind kind, uint64_t value); + [[nodiscard]] bool removes(uint64_t left) const noexcept; + MergePairMetrics &owner_; + detail::ReservedVector<uint64_t> removed_; + detail::ReservedVector<uint64_t> endpoints_; + detail::ReservedVector<Membership> membership_; + Pairs added_pairs_; + Index added_index_; + std::shared_ptr<Retired> retired_; + Snapshot next_; + bool finished_ = false; + bool committed_ = false; + }; + + [[nodiscard]] Snapshot snapshot() const noexcept; + [[nodiscard]] std::optional<MergePairProof> pair(uint64_t left) const noexcept; + [[nodiscard]] bool contains_page(uint64_t pid) const noexcept; + [[nodiscard]] bool consistent() const noexcept; + bool retire_step() noexcept; + // A leaf change invalidates at most its incoming and outgoing pair. An + // index change additionally invalidates its own direct child pairs. + [[nodiscard]] static size_t invalidation_bytes(size_t pairs) noexcept; + [[nodiscard]] static size_t insertion_bytes() noexcept; + + private: + Pairs pairs_; + Index index_; + Snapshot totals_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/merge_sources.h b/lib/crowdb-tree/src/btree/merge_sources.h index 35d61b7fe..362736a1a 100644 --- a/lib/crowdb-tree/src/btree/merge_sources.h +++ b/lib/crowdb-tree/src/btree/merge_sources.h @@ -66,6 +66,11 @@ struct MergeSource class LoserTree { public: + explicit LoserTree(std::shared_ptr<AllocationBudget> budget = {}) + : losers_(BudgetAllocator<int, AllocationBudget, AllocationKind::kScratch>(std::move(budget))) + { + } + void init(MergeSource *sources, int k) { sources_ = sources; @@ -119,9 +124,10 @@ class LoserTree } private: - MergeSource *sources_ = nullptr; - int k_ = 0; - std::vector<int> losers_; // [0] = winner, [1..k-1] = losers + MergeSource *sources_ = nullptr; + int k_ = 0; + std::vector<int, BudgetAllocator<int, AllocationBudget, AllocationKind::kScratch>> + losers_; // [0] = winner, [1..k-1] = losers // a wins over b: lower key; tie → higher slot; tie → lower index. [[nodiscard]] bool less(int a, int b) const diff --git a/lib/crowdb-tree/src/btree/native_frames.h b/lib/crowdb-tree/src/btree/native_frames.h deleted file mode 100644 index 3c237dc84..000000000 --- a/lib/crowdb-tree/src/btree/native_frames.h +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. -#pragma once -#include "crowdb-tree/btree/delta.h" -#include "crowdb-tree/btree/descent.h" -#include "crowdb-tree/crowdb-tree.h" - -#include <unordered_map> -#include <unordered_set> - -namespace crowdb::tree::detail -{ -struct NativeBounds -{ - std::optional<std::string> lower; - std::optional<std::string> upper; - uint64_t lower_leaf_page_id = kInvalidPageId; - uint64_t upper_leaf_page_id = kInvalidPageId; -}; - -inline bool set_native_frame_fences(NativeFrame *frame, const NativeBounds &bounds) -{ - uint8_t *bytes = frame->frame.data(); - const auto page_bytes = static_cast<uint32_t>(frame->frame.size()); - if (!bounds.lower.has_value()) { - return frame_set_fences(bytes, page_bytes, nullptr, nullptr); - } - if (!bounds.upper.has_value()) { - return false; - } - Slice lower(*bounds.lower); - Slice upper(*bounds.upper); - if (frame_page_type(bytes) == page_type::kLeafBase) { - LeafFrameView leaf(bytes, page_bytes); - for (uint32_t index = 0; index < leaf.count(); ++index) { - if (leaf.key(index).compare(lower) == 0) { - lower = leaf.key(index); - } - if (leaf.key(index).compare(upper) == 0) { - upper = leaf.key(index); - } - } - for (uint32_t index = 0; index < leaf.delta_count(); ++index) { - if (leaf.delta_key(index).compare(lower) == 0) { - lower = leaf.delta_key(index); - } - if (leaf.delta_key(index).compare(upper) == 0) { - upper = leaf.delta_key(index); - } - } - return frame_set_fences(bytes, page_bytes, &lower, &upper); - } - frame_set_inner_fence_pages(bytes, page_bytes, bounds.lower_leaf_page_id, bounds.upper_leaf_page_id); - return true; -} - -inline Status validate_native_snapshot_graph(std::vector<NativeFrame> *frames, uint64_t root_page_id) -{ - std::unordered_map<uint64_t, NativeFrame *> by_id; - by_id.reserve(frames->size()); - for (NativeFrame &frame : *frames) { - const uint64_t stored_page_id = - frame.frame.empty() ? kInvalidPageId : frame_u64(frame.frame.data(), fh::kSelfpage_id); - if (frame.page_id == kInvalidPageId || frame.frame.empty() || - (stored_page_id != kInvalidPageId && stored_page_id != frame.page_id) || - !by_id.emplace(frame.page_id, &frame).second) { - return Status::corruption("native snapshot: duplicate, invalid, or mismatched page ID"); - } - } - const auto root = by_id.find(root_page_id); - if (root == by_id.end() || frame_page_type(root->second->frame.data()) == page_type::kOverflowFrame) { - return Status::corruption("native snapshot: root page is missing or has an invalid type"); - } - - auto validate_fences = [](NativeFrame *frame, const NativeBounds &bounds) { - const uint8_t *bytes = frame->frame.data(); - if (!frame_has_lower_fence(bytes) && bounds.lower.has_value() && !set_native_frame_fences(frame, bounds)) { - return false; - } - bytes = frame->frame.data(); - if (frame_has_lower_fence(bytes) != bounds.lower.has_value() || - frame_has_upper_fence(bytes) != bounds.upper.has_value()) { - return false; - } - if (!bounds.lower.has_value()) { - return true; - } - if (frame_fences_are_page_ids(bytes)) { - return frame_lower_fence_page_id(bytes) == bounds.lower_leaf_page_id && - frame_upper_fence_page_id(bytes) == bounds.upper_leaf_page_id; - } - return frame_lower_fence(bytes).compare(Slice(*bounds.lower)) == 0 && - frame_upper_fence(bytes).compare(Slice(*bounds.upper)) == 0; - }; - - std::unordered_set<uint64_t> reached; - std::unordered_set<uint64_t> active; - std::unordered_set<uint64_t> overflow_reached; - std::vector<const NativeFrame *> leaves; - std::function<Status(uint64_t, const std::optional<std::string> &, const std::optional<std::string> &, - NativeBounds *)> - walk = [&](uint64_t page_id, const std::optional<std::string> &expected_lower, - const std::optional<std::string> &expected_upper, NativeBounds *bounds) -> Status { - if (active.contains(page_id) || reached.contains(page_id)) { - return Status::corruption("native snapshot: cyclic or multiply referenced tree page"); - } - const auto found = by_id.find(page_id); - if (found == by_id.end()) { - return Status::corruption("native snapshot: missing child page"); - } - NativeFrame &frame = *found->second; - const page_type type = frame_page_type(frame.frame.data()); - if (type == page_type::kOverflowFrame) { - return Status::corruption("native snapshot: overflow page used as a tree child"); - } - active.insert(page_id); - reached.insert(page_id); - - if (type == page_type::kLeafBase) { - LeafFrameView leaf(frame.frame.data(), static_cast<uint32_t>(frame.frame.size())); - leaves.push_back(&frame); - bounds->lower_leaf_page_id = page_id; - bounds->upper_leaf_page_id = page_id; - auto remember_key = [bounds](Slice key) { - if (!bounds->lower.has_value() || key.compare(Slice(*bounds->lower)) < 0) { - bounds->lower = key.to_string(); - } - if (!bounds->upper.has_value() || key.compare(Slice(*bounds->upper)) > 0) { - bounds->upper = key.to_string(); - } - }; - for (uint32_t index = 0; index < leaf.count(); ++index) { - remember_key(leaf.key(index)); - } - for (uint32_t index = 0; index < leaf.delta_count(); ++index) { - remember_key(leaf.delta_key(index)); - } - if ((expected_lower.has_value() && bounds->lower.has_value() && - Slice(*bounds->lower).compare(Slice(*expected_lower)) < 0) || - (expected_upper.has_value() && bounds->upper.has_value() && - Slice(*bounds->upper).compare(Slice(*expected_upper)) >= 0)) { - return Status::corruption("native snapshot: leaf escapes its routed bounds"); - } - auto validate_cell = [&](Slice raw_cell) -> Status { - CellView cell{raw_cell}; - if (!cell.valid() || (cell.is_overflow() && raw_cell.size() != kOverflowCellSize)) { - return Status::corruption("native snapshot: invalid leaf cell"); - } - if (!cell.is_overflow()) { - return Status::Ok(); - } - std::unordered_set<uint64_t> chain; - uint64_t overflow_id = cell.overflow_head(); - while (overflow_id != kInvalidPageId) { - if (!chain.insert(overflow_id).second) { - return Status::corruption("native snapshot: cyclic overflow chain"); - } - const auto overflow = by_id.find(overflow_id); - if (overflow == by_id.end() || - frame_page_type(overflow->second->frame.data()) != page_type::kOverflowFrame) { - return Status::corruption("native snapshot: missing overflow page"); - } - if (!overflow_reached.insert(overflow_id).second) { - break; - } - OverflowFrameView view(overflow->second->frame.data(), - static_cast<uint32_t>(overflow->second->frame.size())); - overflow_id = view.next_page_id(); - } - return Status::Ok(); - }; - for (uint32_t index = 0; index < leaf.count(); ++index) { - Status status = validate_cell(leaf.cell(index)); - if (!status.ok()) { - return status; - } - } - for (uint32_t index = 0; index < leaf.delta_count(); ++index) { - Status status = validate_cell(leaf.delta_cell(index)); - if (!status.ok()) { - return status; - } - } - if (!validate_fences(&frame, *bounds)) { - return Status::corruption("native snapshot: leaf fences are missing or inconsistent"); - } - active.erase(page_id); - return Status::Ok(); - } - - InnerFrameView inner(frame.frame.data(), static_cast<uint32_t>(frame.frame.size())); - for (uint32_t index = 0; index < inner.num_children(); ++index) { - std::optional<std::string> child_lower = - index == 0 ? expected_lower : std::optional<std::string>(inner.separator_at(index - 1).to_string()); - std::optional<std::string> child_upper = - index == inner.num_separators() ? expected_upper - : std::optional<std::string>(inner.separator_at(index).to_string()); - if ((expected_lower.has_value() && child_lower.has_value() && - Slice(*child_lower).compare(Slice(*expected_lower)) < 0) || - (expected_upper.has_value() && child_upper.has_value() && - Slice(*child_upper).compare(Slice(*expected_upper)) > 0)) { - return Status::corruption("native snapshot: inner separator escapes its routed bounds"); - } - NativeBounds child; - Status child_status = walk(inner.child_at(index), child_lower, child_upper, &child); - if (!child_status.ok()) { - return child_status; - } - if (!bounds->lower.has_value() && child.lower.has_value()) { - bounds->lower = child.lower; - bounds->lower_leaf_page_id = child.lower_leaf_page_id; - } - if (child.upper.has_value()) { - bounds->upper = child.upper; - bounds->upper_leaf_page_id = child.upper_leaf_page_id; - } - } - if (!validate_fences(&frame, *bounds)) { - return Status::corruption("native snapshot: inner fences are missing or inconsistent"); - } - active.erase(page_id); - return Status::Ok(); - }; - - NativeBounds root_bounds; - Status graph_status = walk(root_page_id, std::nullopt, std::nullopt, &root_bounds); - if (!graph_status.ok()) { - return graph_status; - } - for (size_t index = 0; index < leaves.size(); ++index) { - LeafFrameView leaf(leaves[index]->frame.data(), static_cast<uint32_t>(leaves[index]->frame.size())); - const uint64_t expected = index + 1 < leaves.size() ? leaves[index + 1]->page_id : kInvalidPageId; - if (leaf.right_sibling() != expected) { - return Status::corruption("native snapshot: leaf sibling escapes tree order"); - } - } - if (reached.size() + overflow_reached.size() != frames->size()) { - return Status::corruption("native snapshot: unreachable page frame"); - } - return Status::Ok(); -} - -// Resolve a leaf chain (head -> ... -> LeafBase) to key-sorted entries by -// highest-slot-wins. Tombstones whose slot <= gc_floor are dropped (logical -// retention GC); all other tombstones are kept. -// -// This is the whole-page form, for the callers that genuinely need every live -// entry (collect_in_order for iter_all/compare, PinnedSnapshot::materialize, -// GC's live walk) -- O(N) is the right cost there. It is a thin loop over -// LeafChainCursor, which merges the chain's already-sorted streams lazily; the -// scan paths drive that cursor directly instead, so a limit-bounded scan pays -// O(limit) rather than O(entries-per-leaf). Each key/cell the cursor -// yields is borrowed from the chain's own resident storage and is copied into -// an owned leaf_entry exactly once, here. -} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/native_import.h b/lib/crowdb-tree/src/btree/native_import.h new file mode 100644 index 000000000..e079ed6f9 --- /dev/null +++ b/lib/crowdb-tree/src/btree/native_import.h @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/config.h" +#include "crowdb-tree/snapshot/native_frame.h" +#include "page_load.h" +#include "snapshot/readable_page_catalog.h" + +namespace crowdb::tree +{ +struct NativeImportLimits +{ + size_t max_frames = 1024 * 1024; + size_t max_dependency_steps = 1024 * 1024; +}; + +// Upper bounds for one complete immutable import, derived before staging. +struct NativeImportShape +{ + size_t frames = 0; + size_t primary_pages = 0; + size_t leaf_pages = 0; + size_t stored_references = 0; + size_t max_leaf_entries = 0; + size_t dependency_steps = 0; + size_t max_frame_bytes = 0; + size_t max_key_bytes = 0; + size_t max_value_bytes = 0; +}; + +[[nodiscard]] std::optional<size_t> native_import_allocation_bound(const NativeImportShape &shape) noexcept; +[[nodiscard]] PageLoadLimits resident_import_load_limits(PageLoadLimits limits, size_t frame_bytes) noexcept; + +Status append_native_frame(ReadablePageCatalog &catalog, uint64_t pid, std::span<const uint8_t> frame, + const Config &config, const PageLoadLimits &loads, NativeImportLimits limits, + const std::shared_ptr<detail::AllocationReservation> &metadata); +Status build_native_catalog(std::span<const NativeFrame> frames, NativeStreamInfo info, const Config &config, + const PageLoadLimits &loads, NativeImportLimits limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ReadablePageCatalog> *out); +Status attach_native_dependencies(ReadablePageCatalog &catalog, NativeImportLimits limits, + const std::shared_ptr<detail::AllocationReservation> &metadata); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/native_import_dependencies.cpp b/lib/crowdb-tree/src/btree/native_import_dependencies.cpp new file mode 100644 index 000000000..7661edef7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/native_import_dependencies.cpp @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "native_import.h" +#include "overflow_backing.h" + +namespace crowdb::tree +{ +Status attach_native_dependencies(ReadablePageCatalog &catalog, NativeImportLimits limits, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + size_t steps = 0; + for (auto &[pid, primary] : catalog.frames) { + (void)pid; + if (primary.type != page_type::kLeafBase || !primary.stored_overflow || primary.stored_overflow->empty()) { + continue; + } + auto backing = detail::reserved_shared<BackingRef>(metadata, *primary.backing); + auto dependencies = detail::reserved_shared<OverflowBackingSet>(metadata, metadata); + for (const auto &reference : *primary.stored_overflow) { + auto next = reference.head; + auto remaining = reference.bytes; + while (remaining != 0) { + if (steps++ == limits.max_dependency_steps) { + return Status::resource_exhausted("native import dependency work exceeds configured limit"); + } + const auto found = catalog.frames.find(next); + if (found == catalog.frames.end() || found->second.type != page_type::kOverflowFrame) { + return Status::corruption("native import overflow chain names a missing or invalid frame"); + } + const auto &frame = found->second; + if (frame.overflow_bytes == 0 || frame.overflow_bytes > remaining || + (frame.next_overflow == kInvalidPageId) != (frame.overflow_bytes == remaining)) { + return Status::corruption("native import overflow topology differs from the stored length"); + } + dependencies->pages.emplace(next, OverflowBackingPage{.backing = frame.backing, + .next = frame.next_overflow, + .bytes = frame.overflow_bytes, + .frame_bytes = frame.frame_bytes, + .logical_bytes = frame.logical_bytes}); + remaining -= frame.overflow_bytes; + next = frame.next_overflow; + } + } + backing->overflow = std::move(dependencies); + primary.backing = std::move(backing); + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/native_import_frames.cpp b/lib/crowdb-tree/src/btree/native_import_frames.cpp new file mode 100644 index 000000000..f03e410f1 --- /dev/null +++ b/lib/crowdb-tree/src/btree/native_import_frames.cpp @@ -0,0 +1,183 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "native_import.h" +#include "overflow_backing.h" +#include "snapshot/stored_overflow.h" + +namespace crowdb::tree +{ +namespace +{ +struct ResidentImportFrame +{ + ResidentImportFrame(std::span<const uint8_t> input, const std::shared_ptr<detail::AllocationReservation> &metadata) + : bytes(input.begin(), input.end(), detail::ReservationAllocator<uint8_t, AllocationKind::kFrame>(metadata)) + { + } + + detail::ReservedVector<uint8_t, AllocationKind::kFrame> bytes; +}; + +Status validate_import_cell(Slice raw, uint64_t coverage, const PageLoadLimits &limits) +{ + const CellView cell(raw); + if (!cell.valid() || cell.slot() > coverage || + (cell.flags() != 0 && cell.flags() != kFlagTombstone && cell.flags() != kFlagOverflow) || + (cell.is_tombstone() && raw.size() != kCellHeaderSize) || + (cell.is_overflow() && + (raw.size() != kOverflowCellSize || cell.overflow_head() == kInvalidPageId || cell.overflow_len() == 0))) { + return Status::corruption("native import has an invalid cell or coverage"); + } + const auto length = cell.is_overflow() ? cell.overflow_len() : cell.value().size(); + return length <= limits.max_value_bytes ? Status::Ok() + : Status::resource_exhausted("native import value exceeds read limit"); +} + +Status inspect_import_leaf(const ResidentImportFrame &owned, uint64_t coverage, const PageLoadLimits &limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, + ReadablePageFrame *descriptor) +{ + const LeafFrameView leaf(owned.bytes.data(), static_cast<uint32_t>(owned.bytes.size())); + if (leaf.count() > limits.max_leaf_entries || leaf.delta_count() > limits.max_leaf_entries - leaf.count()) { + return Status::resource_exhausted("native import leaf exceeds entry limit"); + } + for (uint32_t i = 0; i < leaf.count() + leaf.delta_count(); ++i) { + const auto cell = i < leaf.count() ? leaf.cell(i) : leaf.delta_cell(i - leaf.count()); + auto status = validate_import_cell(cell, coverage, limits); + if (!status.ok()) { + return status; + } + } + return capture_stored_overflow(leaf, limits.max_leaf_entries, metadata, &descriptor->stored_overflow); +} +} // namespace + +std::optional<size_t> native_import_allocation_bound(const NativeImportShape &shape) noexcept +{ + using detail::shared_allocation_bound; + if (shape.frames > NativeImportLimits{}.max_frames || shape.primary_pages > shape.frames || + shape.leaf_pages > shape.primary_pages || shape.dependency_steps > NativeImportLimits{}.max_dependency_steps) { + return {}; + } + const auto frame = detail::AllocationBound(shared_allocation_bound<ResidentImportFrame>()) + .add(shape.max_frame_bytes) + .add(shared_allocation_bound<BackingRef>()) + .add(shared_allocation_bound<ReadablePageCatalog::Entry>()) + .value(); + if (!frame) { + return {}; + } + return detail::AllocationBound(shared_allocation_bound<ReadablePageCatalog>()) + .add_product(shape.frames, *frame) + .add_product(shape.leaf_pages, shared_allocation_bound<StoredOverflowReferences>() + + shared_allocation_bound<BackingRef>() + + shared_allocation_bound<OverflowBackingSet>()) + .add_product(shape.max_leaf_entries, sizeof(Slice)) + .add_product(shape.stored_references, sizeof(StoredOverflowReference)) + .add_product(shape.dependency_steps, shared_allocation_bound<OverflowBackingSet::Entry>()) + .value(); +} + +PageLoadLimits resident_import_load_limits(PageLoadLimits limits, size_t frame_bytes) noexcept +{ + limits.max_frame_bytes = std::min(limits.max_frame_bytes, frame_bytes); + limits.max_blob_bytes = limits.max_frame_bytes + kDurableBlobHeader; + limits.max_chunk_blob_bytes = 0; + limits.max_inner_children = std::min(limits.max_inner_children, frame_bytes / sizeof(uint64_t)); + limits.max_inner_key_bytes = std::min(limits.max_inner_key_bytes, frame_bytes); + limits.max_leaf_entries = std::min(limits.max_leaf_entries, frame_bytes / kLeafSlotSize); + limits.max_leaf_key_bytes = std::min(limits.max_leaf_key_bytes, frame_bytes); + return limits; +} + +Status append_native_frame(ReadablePageCatalog &catalog, uint64_t pid, std::span<const uint8_t> frame, + const Config &config, const PageLoadLimits &loads, NativeImportLimits limits, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + if (catalog.frames.size() == limits.max_frames) { + return Status::resource_exhausted("native import frame limit exceeded"); + } + if (pid >= MappingView::kPageLimit || frame.size() > loads.max_frame_bytes || frame.size() > UINT32_MAX) { + return Status::resource_exhausted("native import PID or frame exceeds configured limits"); + } + const auto bytes = static_cast<uint32_t>(frame.size()); + if (!frame_validate_key_range(frame.data(), bytes, config.key_range)) { + return Status::corruption("native import frame CRC, structure, or key range is invalid"); + } + const auto stored_pid = frame_u64(frame.data(), fh::kSelfpage_id); + if (stored_pid != kInvalidPageId && stored_pid != pid) { + return Status::corruption("native import frame identity differs from its descriptor"); + } + auto owned = detail::reserved_shared<ResidentImportFrame>(metadata, frame, metadata); + if (stored_pid == kInvalidPageId) { + frame_put_u64(owned->bytes.data(), fh::kSelfpage_id, pid); + frame_restamp_crc(owned->bytes.data(), bytes); + } + auto backing = detail::reserved_shared<BackingRef>(metadata); + backing->content_id = pid + 1; + backing->generation = catalog.generation; + backing->storage = BackingStorage::kResidentFrame; + backing->durability = BackingDurability::kResident; + backing->resident_frame = owned->bytes; + backing->owner = owned; + ReadablePageFrame descriptor{.page_id = pid, + .type = frame_page_type(owned->bytes.data()), + .frame_bytes = bytes, + .logical_bytes = bytes, + .backing = backing}; + if (descriptor.type == page_type::kLeafBase) { + auto status = inspect_import_leaf(*owned, catalog.coverage, loads, metadata, &descriptor); + if (!status.ok()) { + return status; + } + } + else if (descriptor.type == page_type::kOverflowFrame) { + const OverflowFrameView overflow(owned->bytes.data(), bytes); + descriptor.next_overflow = overflow.next_page_id(); + descriptor.overflow_bytes = overflow.chunk_len(); + if (descriptor.overflow_bytes == 0) { + return Status::corruption("native import contains an empty overflow frame"); + } + } + else if (descriptor.type != page_type::kInnerBase) { + return Status::corruption("native import contains an unsupported frame"); + } + if (!catalog.frames.emplace(pid, std::move(descriptor)).second) { + return Status::corruption("native import contains duplicate PIDs"); + } + catalog.next_page_id = std::max(catalog.next_page_id, pid + 1); + return Status::Ok(); +} + +Status build_native_catalog(std::span<const NativeFrame> frames, NativeStreamInfo info, const Config &config, + const PageLoadLimits &loads, NativeImportLimits limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ReadablePageCatalog> *out) +{ + if (out == nullptr || *out || frames.size() != info.frame_count || info.next_page_id > MappingView::kPageLimit) { + return Status::invalid_argument("native import requires consistent bounded stream metadata"); + } + if (frames.size() > limits.max_frames) { + return Status::resource_exhausted("native import frame limit exceeded"); + } + auto catalog = detail::reserved_shared<ReadablePageCatalog>(metadata, metadata); + catalog->root_page_id = info.root_page_id; + catalog->next_page_id = info.next_page_id; + catalog->coverage = info.at_slot; + catalog->generation = 1; + // Resident identities never claim a backend domain. Each page's owner + // distinguishes separate imports, including equal PIDs in old captures. + catalog->backend_id = 0; + for (const auto &input : frames) { + const auto status = append_native_frame(*catalog, input.page_id, input.frame, config, loads, limits, metadata); + if (!status.ok()) { + return status; + } + } + auto status = attach_native_dependencies(*catalog, limits, metadata); + if (status.ok()) { + *out = std::move(catalog); + } + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/native_iterator.cpp b/lib/crowdb-tree/src/btree/native_iterator.cpp index 47d52d672..8bb1eee78 100644 --- a/lib/crowdb-tree/src/btree/native_iterator.cpp +++ b/lib/crowdb-tree/src/btree/native_iterator.cpp @@ -1,139 +1,13 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -#include "crowdb-tree/btree/delta.h" -#include "crowdb-tree/crowdb-tree.h" -#include "crowdb-tree/maptable/mapping_slot.h" -#include "native_frames.h" - -#include <algorithm> -#include <ranges> -#include <unordered_map> +#include "btree/tree_core.h" +#include "snapshot/published_native.h" namespace crowdb::tree { -using detail::NativeBounds; -using detail::set_native_frame_fences; - -struct NativeFrameIterator::Impl -{ - struct SavedPage - { - PageBase *head = nullptr; - std::vector<PageBase *> pins; - }; - - struct Task - { - uint64_t page_id = kInvalidPageId; - std::optional<std::string> lower; - std::optional<std::string> upper; - bool is_root = false; - bool overflow = false; - }; - - ~Impl() - { - for (const auto &[page_id, saved] : saved_pages) { - (void)page_id; - for (PageBase *page : saved.pins) { - page->unpin(); - } - } - } - - [[nodiscard]] bool consumed(uint64_t page_id) const - { - return page_id < next_page_id && (consumed_pages[page_id / 64] & (uint64_t{1} << (page_id % 64))) != 0; - } - - [[nodiscard]] bool intersects(const Task &task) const - { - if (!filter.has_value() || !filter->is_bounded() || task.is_root || task.overflow) { - return true; - } - if (filter->start().has_value() && filter->end().has_value() && - Slice(*filter->start()).compare(Slice(*filter->end())) == 0) { - return false; - } - const bool below_end = !filter->end().has_value() || !task.lower.has_value() || - Slice(*task.lower).compare(Slice(*filter->end())) < 0; - const bool above_start = !filter->start().has_value() || !task.upper.has_value() || - Slice(*task.upper).compare(Slice(*filter->start())) > 0; - return below_end && above_start; - } - - void schedule(Task task) - { - pending_tasks.insert_or_assign(task.page_id, task); - tasks.push_back(std::move(task)); - } - - void preserve(Crowdbtree &source, uint64_t page_id, PageBase *head) - { - (void)source; - const auto pending = pending_tasks.find(page_id); - if (!active || !terminal_status.ok() || head == nullptr || page_id >= next_page_id || consumed(page_id) || - saved_pages.contains(page_id) || (pending != pending_tasks.end() && !intersects(pending->second))) { - return; - } - size_t chain_pages = 0; - for (PageBase *page = head; page != nullptr; page = page->next) { - ++chain_pages; - } - if (saved_pin_count + chain_pages > max_saved_pages) { - terminal_status = Status::resource_exhausted("native frame iterator preservation budget exceeded"); - return; - } - SavedPage saved{.head = head, .pins = {}}; - for (PageBase *page = head; page != nullptr; page = page->next) { - page->pin(); - saved.pins.push_back(page); - } - saved_pin_count += saved.pins.size(); - saved_pages.emplace(page_id, std::move(saved)); - } - - PageBase *resolve(Crowdbtree &source, uint64_t page_id) const - { - const auto saved = saved_pages.find(page_id); - return saved == saved_pages.end() ? source.resident(page_id) : saved->second.head; - } - - void consume(uint64_t page_id) - { - if (page_id < next_page_id) { - consumed_pages[page_id / 64] |= uint64_t{1} << (page_id % 64); - } - const auto saved = saved_pages.find(page_id); - if (saved == saved_pages.end()) { - return; - } - for (PageBase *page : saved->second.pins) { - page->unpin(); - } - saved_pin_count -= saved->second.pins.size(); - saved_pages.erase(saved); - } - - Crowdbtree *source = nullptr; - std::optional<KeyRange> filter; - std::vector<Task> tasks; - std::unordered_map<uint64_t, Task> pending_tasks; - std::unordered_map<uint64_t, SavedPage> saved_pages; - std::vector<NativeFrame> owned_frames; - bool active = true; - size_t cursor = 0; - size_t saved_pin_count = 0; - size_t max_saved_pages = 1; - uint64_t root_page_id = kInvalidPageId; - uint64_t at_slot = 0; - uint64_t next_page_id = 0; - uint64_t subtrees_skipped = 0; - std::vector<uint64_t> consumed_pages; - Status terminal_status; -}; - -NativeFrameIterator::NativeFrameIterator(std::shared_ptr<Impl> impl) : impl_(std::move(impl)) +NativeFrameIterator::NativeFrameIterator(std::shared_ptr<PublishedNativeCursor> runtime) + : runtime_(std::move(runtime)), + facade_credit_(runtime_->metadata(), sizeof(NativeFrameIterator), AllocationKind::kMetadata) { } @@ -143,278 +17,67 @@ NativeFrameIterator &NativeFrameIterator::operator=(NativeFrameIterator &&) noex Status NativeFrameIterator::next(size_t max_frames, std::vector<NativeFrame> *out, bool *complete) { - if (impl_ == nullptr || out == nullptr || complete == nullptr || max_frames == 0) { - return Status::invalid_argument("native frame iterator requires a non-empty output batch"); + if (!runtime_ || out == nullptr || complete == nullptr || max_frames == 0) { + return Status::invalid_argument("native iterator requires a non-empty output batch"); } + // The compatibility vector is owned by the caller; engine consumers retain next_owned windows. out->clear(); - if (!impl_->owned_frames.empty()) { - const size_t total = impl_->owned_frames.size(); - const size_t end = std::min(total, impl_->cursor + std::min(max_frames, total - impl_->cursor)); - out->reserve(end - impl_->cursor); - while (impl_->cursor < end) { - out->push_back(std::move(impl_->owned_frames[impl_->cursor++])); - } - *complete = impl_->cursor == total; - return Status::Ok(); - } - if (impl_->source == nullptr) { - if (impl_->terminal_status.ok()) { - impl_->terminal_status = Status::internal_error("native frame iterator source no longer exists"); - } - return impl_->terminal_status; - } - - Crowdbtree &source = *impl_->source; - std::scoped_lock lock(source.write_mutex_); - if (!impl_->terminal_status.ok()) { - return impl_->terminal_status; - } - out->reserve(max_frames); - while (out->size() < max_frames && !impl_->tasks.empty()) { - Impl::Task task = std::move(impl_->tasks.back()); - impl_->tasks.pop_back(); - impl_->pending_tasks.erase(task.page_id); - if (!impl_->intersects(task)) { - ++impl_->subtrees_skipped; - impl_->consume(task.page_id); - continue; - } - PageBase *page = impl_->resolve(source, task.page_id); - if (page == nullptr) { - impl_->terminal_status = Status::internal_error("native frame iterator encountered a missing page"); - return impl_->terminal_status; - } - if (page->type == page_type::kBatchDelta || - (page->type == page_type::kLeafBase && static_cast<LeafBase *>(page)->view().delta_count() != 0)) { - impl_->terminal_status = - Status::internal_error("native frame iterator requires a flushed immutable source generation"); - return impl_->terminal_status; - } - - const uint8_t *bytes = nullptr; - uint32_t length = 0; - if (page->type == page_type::kLeafBase) { - auto *leaf = static_cast<LeafBase *>(page); - bytes = leaf->frame(); - length = leaf->page_bytes(); - std::vector<uint64_t> overflow_heads; - LeafFrameView view = leaf->view(); - for (uint32_t index = 0; index < view.count(); ++index) { - CellView cell{view.cell(index)}; - if (cell.is_overflow()) { - overflow_heads.push_back(cell.overflow_head()); - } - } - for (auto &overflow_head : std::views::reverse(overflow_heads)) { - impl_->schedule( - {.page_id = overflow_head, .lower = std::nullopt, .upper = std::nullopt, .overflow = true}); - } - } - else if (page->type == page_type::kInnerBase) { - auto *inner = static_cast<InnerBase *>(page); - bytes = inner->frame(); - length = inner->page_bytes(); - InnerFrameView view = inner->view(); - for (uint32_t reverse = view.num_children(); reverse != 0; --reverse) { - const uint32_t index = reverse - 1; - impl_->schedule({ - .page_id = view.child_at(index), - .lower = - index == 0 ? task.lower : std::optional<std::string>(view.separator_at(index - 1).to_string()), - .upper = index == view.num_separators() - ? task.upper - : std::optional<std::string>(view.separator_at(index).to_string()), - }); - } - } - else if (page->type == page_type::kOverflowFrame) { - auto *overflow = static_cast<OverflowBase *>(page); - bytes = overflow->frame(); - length = overflow->page_bytes(); - if (overflow->next_page_id() != kInvalidPageId) { - impl_->schedule({ - .page_id = overflow->next_page_id(), - .lower = std::nullopt, - .upper = std::nullopt, - .overflow = true, - }); - } + *complete = false; + constexpr size_t kMaxBatch = 64; + while (out->size() < std::min(max_frames, kMaxBatch) && !*complete) { + std::optional<NativeFrameView> frame; + auto status = runtime_->next(&frame, complete); + if (!status.ok()) { + return status; } - else { - impl_->terminal_status = - Status::internal_error("native frame iterator encountered an unexpected page type"); - return impl_->terminal_status; + if (frame) { + out->push_back({.page_id = frame->page_id, + .frame = std::vector<uint8_t>(frame->bytes.begin(), frame->bytes.end()), + .durable_addr = kNoAddr, + .durable_plen = 0, + .inherited = false}); } - out->push_back({ - .page_id = task.page_id, - .frame = std::vector<uint8_t>(bytes, bytes + length), - .durable_addr = page->durable_addr, - .durable_plen = page->durable_plen, - }); - impl_->consume(task.page_id); - } - *complete = impl_->tasks.empty(); - if (*complete) { - impl_->active = false; } return Status::Ok(); } uint64_t NativeFrameIterator::root_page_id() const { - return impl_ == nullptr ? kInvalidPageId : impl_->root_page_id; + return runtime_ ? runtime_->info().root_page_id : kInvalidPageId; } uint64_t NativeFrameIterator::at_slot() const { - return impl_ == nullptr ? 0 : impl_->at_slot; + return runtime_ ? runtime_->info().at_slot : 0; } uint64_t NativeFrameIterator::next_page_id() const { - return impl_ == nullptr ? 0 : impl_->next_page_id; + return runtime_ ? runtime_->info().next_page_id : 0; } uint64_t NativeFrameIterator::subtrees_skipped() const { - return impl_ == nullptr ? 0 : impl_->subtrees_skipped; -} - -void Crowdbtree::preserve_native_page_locked(uint64_t page_id, PageBase *page) -{ - std::erase_if(native_frame_iterators_, [](const auto &iterator) { return iterator.expired(); }); - for (const std::weak_ptr<NativeFrameIterator::Impl> &weak : native_frame_iterators_) { - if (auto iterator = weak.lock()) { - iterator->preserve(*this, page_id, page); - } - } + return 0; } Status Crowdbtree::open_native_frame_iterator(const KeyRange *filter, std::unique_ptr<NativeFrameIterator> *out) { if (out == nullptr) { - return Status::invalid_argument("native frame iterator requires an output"); + return Status::invalid_argument("native iterator requires output"); } - out->reset(); if (filter != nullptr) { - Status range_status = filter->validate(); - if (!range_status.ok()) { - return range_status; - } - } - if (filter != nullptr && !routing_fences_trusted_.load(std::memory_order_acquire)) { - auto impl = std::make_shared<NativeFrameIterator::Impl>(); - uint64_t skipped = 0; - Status status = collect_native_frames(&impl->owned_frames, &impl->root_page_id, &impl->at_slot, - &impl->next_page_id, filter, &skipped); + auto status = filter->validate(); if (!status.ok()) { return status; } - impl->subtrees_skipped = skipped; - *out = std::unique_ptr<NativeFrameIterator>(new NativeFrameIterator(std::move(impl))); - return Status::Ok(); } - - std::scoped_lock lk(write_mutex_); - if (publication_incomplete_) { - return Status::unavailable("flush publication requires repair"); - } - const uint64_t gc = gc_floor_.load(); - std::function<Status(uint64_t)> prepare = [&](uint64_t page_id) -> Status { - PageBase *head = resident(page_id); - if (head == nullptr) { - return Status::internal_error("native frame iterator: missing page during preparation"); - } - const bool has_inframe_deltas = - head->type == page_type::kLeafBase && static_cast<LeafBase *>(head)->view().delta_count() != 0; - if (head->type == page_type::kBatchDelta || has_inframe_deltas) { - PageBase *base = head; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - if (base == nullptr || base->type != page_type::kLeafBase) { - return Status::internal_error("native frame iterator: delta chain without leaf base"); - } - const uint64_t right = static_cast<LeafBase *>(base)->right_sibling(); - std::vector<uint64_t> dead_overflow; - LeafBase *fresh = - build_leaf_spilling_locked(resolve_leaf_chain_for_rebuild(head, gc, &dead_overflow), right); - store_preserving_parent_locked(page_id, fresh); - for (PageBase *node = head; node != nullptr;) { - PageBase *next = node->next; - retire_page(node); - node = next; - } - for (uint64_t overflow_head : dead_overflow) { - retire_overflow_chain_locked(overflow_head); - } - head = fresh; - } - if (head->type != page_type::kInnerBase) { - return head->type == page_type::kLeafBase ? Status::Ok() - : Status::internal_error("native frame iterator: bad base page"); - } - for (uint64_t child : static_cast<InnerBase *>(head)->children()) { - Status child_status = prepare(child); - if (!child_status.ok()) { - return child_status; - } - } - return Status::Ok(); - }; - const uint64_t prepared_root = root_page_id_.load(); - Status prepare_status = prepare(prepared_root); - if (!prepare_status.ok()) { - return prepare_status; - } - auto state = std::make_shared<NativeFrameIterator::Impl>(); - state->source = this; - if (filter != nullptr) { - state->filter = *filter; - } - state->root_page_id = prepared_root; - state->next_page_id = mapping_.next_page_id(); - state->max_saved_pages = std::max<size_t>(1, (4U * 1024U * 1024U) / opt_.frame_bytes); - state->consumed_pages.resize((state->next_page_id + 63) / 64); - PageBase *root = resident(state->root_page_id); - if (root == nullptr) { - return Status::internal_error("native frame iterator: missing root page"); - } - state->preserve(*this, state->root_page_id, root); - state->schedule({.page_id = state->root_page_id, .lower = std::nullopt, .upper = std::nullopt, .is_root = true}); - native_frame_iterators_.push_back(state); - state->at_slot = last_applied_slot_.load(); - *out = std::unique_ptr<NativeFrameIterator>(new NativeFrameIterator(std::move(state))); - return Status::Ok(); -} - -void Crowdbtree::detach_native_iterators() -{ - for (const std::weak_ptr<NativeFrameIterator::Impl> &weak : native_frame_iterators_) { - if (auto iterator = weak.lock()) { - iterator->source = nullptr; - } - } -} - -void Crowdbtree::preserve_native_generation_locked() -{ - std::erase_if(native_frame_iterators_, [](const auto &iterator) { return iterator.expired(); }); - if (native_frame_iterators_.empty()) { - return; - } - for (uint64_t segment_index = 0; segment_index < MappingTable::kMaxSegments; ++segment_index) { - auto *segment = mapping_.segment_at(segment_index); - if (segment == nullptr) { - continue; - } - for (uint64_t index = 0; index < MappingTable::kSegmentSize; ++index) { - if (slot_word::is_empty(segment->slots[index].load())) { - continue; - } - const auto page_id = (segment_index * MappingTable::kSegmentSize) + index; - preserve_native_page_locked(page_id, resident(page_id)); - } + // Native output preserves a complete recoverable graph. Range consumers filter logical entries. + std::shared_ptr<PublishedNativeCursor> cursor; + auto status = PublishedNativeCursor::create(core_, {}, &cursor); + if (status.ok()) { + *out = std::unique_ptr<NativeFrameIterator>(new NativeFrameIterator(std::move(cursor))); } + return status; } } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/native_iterator_owned.cpp b/lib/crowdb-tree/src/btree/native_iterator_owned.cpp new file mode 100644 index 000000000..da18c3911 --- /dev/null +++ b/lib/crowdb-tree/src/btree/native_iterator_owned.cpp @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/snapshot/native_frame.h" +#include "snapshot/published_native.h" + +namespace crowdb::tree +{ +Status NativeFrameIterator::next_owned(std::optional<NativeFrameView> *out, bool *complete) +{ + if (!runtime_ || out == nullptr || complete == nullptr) { + return Status::invalid_argument("native owned iteration requires output"); + } + out->reset(); + return runtime_->next(out, complete); +} + +NativeStreamInfo NativeFrameIterator::stream_info() const +{ + return runtime_ ? runtime_->info() : NativeStreamInfo{}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/overflow_backing.h b/lib/crowdb-tree/src/btree/overflow_backing.h new file mode 100644 index 000000000..e828f53a8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/overflow_backing.h @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "backing_ref.h" + +#include <map> + +namespace crowdb::tree +{ +struct OverflowBackingPage +{ + std::shared_ptr<const BackingRef> backing; + uint64_t next = UINT64_MAX; + uint32_t bytes = 0; + uint32_t frame_bytes = 0; + uint32_t logical_bytes = 0; +}; + +struct OverflowBackingSet +{ + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !pages.empty(); ++i) { + pages.erase(pages.begin()); + } + return pages.empty(); + } + + using Entry = std::pair<const uint64_t, OverflowBackingPage>; + using Pages = std::map<uint64_t, OverflowBackingPage, std::less<>, detail::ReservationAllocator<Entry>>; + + explicit OverflowBackingSet(const std::shared_ptr<detail::AllocationReservation> &metadata) + : pages(std::less<>{}, detail::ReservationAllocator<Entry>(metadata)) + { + } + + Pages pages; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/owned_key.cpp b/lib/crowdb-tree/src/btree/owned_key.cpp new file mode 100644 index 000000000..f6201f0c7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/owned_key.cpp @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "owned_key.h" + +#include "tree_work_counters.h" + +#include <cstring> +#include <vector> + +namespace crowdb::tree +{ +struct OwnedKey::Storage +{ + using Allocator = detail::ReservationAllocator<char>; + + std::vector<char, Allocator> bytes; + + Storage(Slice key, const std::shared_ptr<detail::AllocationReservation> &metadata) : bytes(Allocator(metadata)) + { + bytes.assign(key.data(), key.data() + key.size()); + record_work(metadata, TreeWorkCounter::KeyCopyBytes, key.size()); + } + + [[nodiscard]] const char *data() const noexcept + { + return bytes.data(); + } +}; + +OwnedKey::OwnedKey(Slice key, const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + if (key.size() <= inline_capacity) { + if (!key.empty()) { + std::memcpy(contents_.inline_bytes.data(), key.data(), key.size()); + } + record_work(metadata, TreeWorkCounter::KeyCopyBytes, key.size()); + } + else { + auto owner = detail::reserved_shared<Storage>(metadata, key, metadata); + const auto *data = owner->data(); + std::construct_at(&contents_.heap, Heap{.owner = std::move(owner), .data = data}); + } + size_ = key.size(); +} + +OwnedKey::OwnedKey(const OwnedKey &other) noexcept : size_(other.size_) +{ + if (is_heap()) { + std::construct_at(&contents_.heap, other.contents_.heap); + } + else { + contents_.inline_bytes = other.contents_.inline_bytes; + } +} + +OwnedKey::OwnedKey(OwnedKey &&other) noexcept : size_(other.size_) +{ + if (is_heap()) { + std::construct_at(&contents_.heap, std::move(other.contents_.heap)); + std::destroy_at(&other.contents_.heap); + std::construct_at(&other.contents_.inline_bytes); + } + else { + contents_.inline_bytes = other.contents_.inline_bytes; + } + other.size_ = 0; +} + +OwnedKey &OwnedKey::operator=(const OwnedKey &other) noexcept +{ + if (this != &other) { + OwnedKey copy(other); + *this = std::move(copy); + } + return *this; +} + +OwnedKey &OwnedKey::operator=(OwnedKey &&other) noexcept +{ + if (this != &other) { + this->~OwnedKey(); + std::construct_at(this, std::move(other)); + } + return *this; +} + +OwnedKey::~OwnedKey() +{ + if (is_heap()) { + std::destroy_at(&contents_.heap); + } +} + +size_t OwnedKey::allocation_overhead() noexcept +{ + return detail::shared_allocation_bound<Storage>() + 1; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/owned_key.h b/lib/crowdb-tree/src/btree/owned_key.h new file mode 100644 index 000000000..c759dc95f --- /dev/null +++ b/lib/crowdb-tree/src/btree/owned_key.h @@ -0,0 +1,121 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "crowdb-tree/slice.h" + +#include <array> +#include <compare> +#include <string> + +namespace crowdb::tree +{ +using KeyArenaStorage = std::basic_string<char, std::char_traits<char>, detail::ReservationAllocator<char>>; + +// Immutable boundaries and continuation keys can cross workers and old views. +// Short keys live inside their containing object; long copies share storage. +// Borrowed slices remain valid until their owning key is moved or destroyed. +class OwnedKey +{ + struct Storage; + + public: + OwnedKey() = default; + OwnedKey(Slice key, const std::shared_ptr<detail::AllocationReservation> &metadata = {}); + + OwnedKey(const std::string &key) : OwnedKey(Slice(key)) + { + } + + OwnedKey(const char *key) : OwnedKey(Slice(key)) + { + } + + OwnedKey(const OwnedKey &other) noexcept; + OwnedKey &operator=(const OwnedKey &other) noexcept; + OwnedKey(OwnedKey &&other) noexcept; + OwnedKey &operator=(OwnedKey &&other) noexcept; + ~OwnedKey(); + + // Per-key fixed allocation overhead; add the key bytes separately. + [[nodiscard]] static size_t allocation_overhead() noexcept; + + [[nodiscard]] operator Slice() const noexcept + { + return {data(), size_}; + } + + [[nodiscard]] size_t size() const noexcept + { + return size_; + } + + [[nodiscard]] const char *data() const noexcept + { + if (size_ == 0) { + return nullptr; + } + return is_heap() ? contents_.heap.data : contents_.inline_bytes.data(); + } + + [[nodiscard]] bool operator==(const OwnedKey &other) const noexcept + { + return Slice(*this) == Slice(other); + } + + [[nodiscard]] std::strong_ordering operator<=>(const OwnedKey &other) const noexcept + { + return Slice(*this).compare(Slice(other)) <=> 0; + } + + [[nodiscard]] bool operator==(const std::string &other) const noexcept + { + return Slice(*this) == Slice(other); + } + + [[nodiscard]] std::strong_ordering operator<=>(const std::string &other) const noexcept + { + return Slice(*this).compare(Slice(other)) <=> 0; + } + + [[nodiscard]] bool operator==(const char *other) const noexcept + { + return Slice(*this) == Slice(other); + } + + [[nodiscard]] std::strong_ordering operator<=>(const char *other) const noexcept + { + return Slice(*this).compare(Slice(other)) <=> 0; + } + + private: + static constexpr size_t inline_capacity = 24; + + struct Heap + { + std::shared_ptr<const Storage> owner; + const char *data = nullptr; + }; + + union Contents { + std::array<char, inline_capacity> inline_bytes; + Heap heap; + + Contents() : inline_bytes{} + { + } + + ~Contents() + { + } + } contents_; + + size_t size_ = 0; + + [[nodiscard]] bool is_heap() const noexcept + { + return size_ > inline_capacity; + } +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache.cpp b/lib/crowdb-tree/src/btree/page_cache.cpp new file mode 100644 index 000000000..cbeb25284 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache.cpp @@ -0,0 +1,298 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "page_cache_internal.h" +#include "page_load_internal.h" + +#include <cassert> +#include <stdexcept> + +namespace crowdb::tree +{ +class SharedPageCache::Binding final : public PageLoader +{ + public: + Binding(std::shared_ptr<State> state, std::shared_ptr<PageLoader> upstream, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<State::Reservation> reservation = {}, PageLoadPriority priority = PageLoadPriority::Read) + : state_(std::move(state)), + upstream_(std::move(upstream)), + metadata_(std::move(metadata)), + reservation_(std::move(reservation)), + priority_(priority) + { + } + + void load_leaf(BaseRef base, LeafComplete complete) override + { + if (base.resident) { + complete_page_load(complete, Status::Ok(), std::move(base.resident)); + return; + } + if (!base.backing || !base.backing->readable()) { + complete_page_load(complete, Status::invalid_argument(), std::shared_ptr<const LeafBase>{}); + return; + } + auto *request = claim(); + if (request == nullptr) { + complete_page_load(complete, Status::unavailable(), std::shared_ptr<const LeafBase>{}); + return; + } + request->priority = priority_; + request->base = std::move(base); + request->key = State::make_key(State::Kind::Leaf, request->base); + request->leaf_done = std::move(complete); + request->upstream = upstream_; + request->metadata = metadata_; + state_->submit(request); + } + + void load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete) override + { + if (!backing || !backing->readable()) { + complete_page_load(complete, Status::invalid_argument(), std::shared_ptr<const InnerPageState>{}); + return; + } + auto *request = claim(); + if (request == nullptr) { + complete_page_load(complete, Status::unavailable(), std::shared_ptr<const InnerPageState>{}); + return; + } + request->priority = priority_; + request->base.backing = std::move(backing); + request->key = State::make_key(State::Kind::Inner, request->base); + request->inner_done = std::move(complete); + request->upstream = upstream_; + request->metadata = metadata_; + state_->submit(request); + } + + void load_overflow(uint64_t head, uint64_t length, ValueComplete complete) override + { + if (state_->closed.load(std::memory_order_acquire)) { + complete_page_load(complete, Status::unavailable(), std::shared_ptr<const buffer>{}); + return; + } + // Overflow IDs are meaningful only in this binding's captured resolver. + // They are streamed through its budget and never keyed by bare PID. + upstream_->load_overflow(head, length, std::move(complete)); + } + + void load_overflow(std::shared_ptr<const BackingRef> primary, uint64_t head, uint64_t length, + ValueComplete complete) override + { + if (state_->closed.load(std::memory_order_acquire)) { + complete_page_load(complete, Status::unavailable(), std::shared_ptr<const buffer>{}); + return; + } + upstream_->load_overflow(std::move(primary), head, length, std::move(complete)); + } + + private: + State::Request *claim() noexcept + { + if (!reservation_) { + return state_->claim(); + } + bool idle = false; + if (state_->closed.load(std::memory_order_acquire) || + !reservation_->active.compare_exchange_strong(idle, true, std::memory_order_acq_rel)) { + state_->rejected.fetch_add(1, std::memory_order_relaxed); + return nullptr; + } + auto *request = reservation_->slot; + request->owner = state_; + request->reservation = reservation_; + state_->waiters.fetch_add(1, std::memory_order_relaxed); + return request; + } + + std::shared_ptr<State> state_; + std::shared_ptr<PageLoader> upstream_; + std::shared_ptr<detail::AllocationReservation> metadata_; + std::shared_ptr<State::Reservation> reservation_; + PageLoadPriority priority_; +}; + +SharedPageCache::SharedPageCache(PageCacheLimits limits, std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<TreeResourceMetrics> resources, detail::FlushRuntime *runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane) +{ + if (!allocation_bound(limits)) { + throw std::invalid_argument("page cache requires finite nonzero page and waiter capacities"); + } + state_ = detail::reserved_shared<State>(metadata, limits, metadata, std::move(resources), runtime, std::move(lane)); +} + +SharedPageCache::~SharedPageCache() +{ + close(); +} + +std::optional<size_t> SharedPageCache::allocation_bound(PageCacheLimits limits) noexcept +{ + if (limits.max_pages == 0 || limits.max_waiters == 0 || limits.max_read_burst == 0 || + limits.low_resident_bytes > limits.max_resident_bytes) { + return {}; + } + return detail::AllocationBound(detail::shared_allocation_bound<State>() + + detail::shared_allocation_bound<SharedPageCache>() + + detail::shared_allocation_bound<Binding>() + PageCacheResidency::allocation_bound()) + .add_product(limits.max_pages, sizeof(State::Entry)) + .add_product(limits.max_waiters, sizeof(State::Request)) + .value(); +} + +std::shared_ptr<PageLoader> SharedPageCache::bind(std::shared_ptr<PageLoader> upstream, + std::shared_ptr<detail::AllocationReservation> metadata, + PageLoadPriority priority) const +{ + if (!upstream) { + throw std::invalid_argument("page cache binding requires an upstream loader"); + } + return detail::reserved_shared<Binding>(metadata, state_, std::move(upstream), metadata, nullptr, priority); +} + +size_t SharedPageCache::binding_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<Binding>() + detail::shared_allocation_bound<State::Reservation>(); +} + +size_t SharedPageCache::load_allocation_bound() noexcept +{ + return PageCacheResidency::ready_allocation_bound() + PageCacheFrame::allocation_bound() + pin_allocation_bound(); +} + +size_t SharedPageCache::pin_allocation_bound() noexcept +{ + return PageCacheFrame::pin_allocation_bound(); +} + +std::optional<size_t> SharedPageCache::load_reserve_bound(size_t pinned_frames, size_t workers) const noexcept +{ + if (!state_->residency) { + return 0; + } + const auto frames = + detail::AllocationBound(state_->limits.max_pages).add(pinned_frames).add(workers).add(1).value(); + if (!frames) { + return {}; + } + return detail::AllocationBound().add_product(*frames, load_allocation_bound()).value(); +} + +size_t SharedPageCache::retained_page_limit() const noexcept +{ + return state_->limits.max_pages; +} + +Status SharedPageCache::bind_reserved(std::shared_ptr<PageLoader> upstream, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<PageLoader> *out, PageLoadPriority priority) const +{ + if (!upstream || out == nullptr) { + return Status::invalid_argument("reserved cache binding requires upstream and output"); + } + auto *slot = state_->claim(false); + if (slot == nullptr) { + return Status::unavailable("cache waiter reservation unavailable"); + } + std::shared_ptr<State::Reservation> reservation; + try { + reservation = detail::reserved_shared<State::Reservation>(metadata, state_, slot); + *out = detail::reserved_shared<Binding>(metadata, state_, std::move(upstream), metadata, reservation, priority); + } + catch (const std::bad_alloc &) { + if (!reservation) { + slot->occupied.store(false, std::memory_order_release); + } + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +PageCacheSnapshot SharedPageCache::snapshot() const noexcept +{ + return {.hits = state_->hits.load(std::memory_order_relaxed), + .misses = state_->misses.load(std::memory_order_relaxed), + .joined = state_->joined.load(std::memory_order_relaxed), + .evictions = state_->evictions.load(std::memory_order_relaxed), + .rejected = state_->rejected.load(std::memory_order_relaxed), + .inflight = state_->inflight.load(std::memory_order_relaxed), + .waiters = state_->waiters.load(std::memory_order_relaxed), + .reserved_waiters = state_->reserved_waiters.load(std::memory_order_relaxed), + .resident_pages = state_->resident_pages.load(std::memory_order_relaxed), + .resident_bytes = state_->resident_bytes.load(std::memory_order_relaxed), + .proxy_pages = state_->residency ? state_->residency->proxy_pages() : 0, + .request_slots = state_->reserved_waiters.load(std::memory_order_relaxed) + + state_->ordinary_waiters.load(std::memory_order_relaxed)}; +} + +const std::shared_ptr<PageCacheResidency> &SharedPageCache::residency() const noexcept +{ + return state_->residency; +} + +bool SharedPageCache::claim_publication() noexcept +{ + return state_->residency && !state_->publication_claimed.exchange(true, std::memory_order_acq_rel); +} + +void SharedPageCache::evict() noexcept +{ + state_->evict_requested.store(true, std::memory_order_release); + state_->wake(); +} + +void SharedPageCache::close() noexcept +{ + state_->closed.store(true, std::memory_order_release); + evict(); +} + +void SharedPageCache::close_and_drain() noexcept +{ + close(); + if (state_->residency) { + state_->residency->close_and_drain(); + } +} + +SharedPageCache::State::State(PageCacheLimits bounds, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<TreeResourceMetrics> resources, detail::FlushRuntime *runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane) + : limits(bounds), + residency(resources ? detail::reserved_shared<PageCacheResidency>(metadata, std::move(resources), metadata, + runtime, std::move(lane)) + : nullptr), + requests(bounds.max_waiters, detail::ReservationAllocator<Request>(metadata)), + entries(bounds.max_pages, detail::ReservationAllocator<Entry>(metadata)) +{ + for (auto &request : requests) { + request.command.context = &request; + } + for (auto &entry : entries) { + entry.command.context = &entry; + } +} + +SharedPageCache::State::Key SharedPageCache::State::make_key(Kind kind, const BaseRef &base) +{ + return page_cache_key(kind, base); +} + +SharedPageCache::State::Reservation::Reservation(std::shared_ptr<State> owner, Request *slot) + : owner(std::move(owner)), + slot(slot) +{ + this->owner->reserved_waiters.fetch_add(1, std::memory_order_relaxed); +} + +SharedPageCache::State::Reservation::~Reservation() +{ + assert(!active.load(std::memory_order_relaxed)); + owner->reserved_waiters.fetch_sub(1, std::memory_order_relaxed); + slot->occupied.store(false, std::memory_order_release); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache.h b/lib/crowdb-tree/src/btree/page_cache.h new file mode 100644 index 000000000..762cf8915 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache.h @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "page_cache_residency.h" +#include "page_load.h" + +namespace crowdb::tree +{ +enum class PageLoadPriority : uint8_t { Read, Maintenance }; + +struct PageCacheLimits +{ + size_t max_pages = 256; + size_t max_waiters = 1024; + // Decoded frame/array/key bytes referenced by the cache. The allocation + // ledger separately accounts actual capacity, control objects and pins. + size_t max_resident_bytes = 64 * 1024 * 1024; + // Zero selects 75% of the high watermark. A frame larger than the low + // watermark may remain alone if it fits the high watermark. + size_t low_resident_bytes = 0; + size_t max_read_burst = 8; +}; + +struct PageCacheSnapshot +{ + uint64_t hits = 0; + uint64_t misses = 0; + uint64_t joined = 0; + uint64_t evictions = 0; + uint64_t rejected = 0; + uint64_t inflight = 0; + uint64_t waiters = 0; + uint64_t reserved_waiters = 0; + uint64_t resident_pages = 0; + uint64_t resident_bytes = 0; + uint64_t proxy_pages = 0; + uint64_t request_slots = 0; +}; + +// Intrusive one-shot command owned by its admitted caller through completion. +// It only drops cache references; independent reader pins stay valid and charged. +struct PageCacheTrim +{ + PageCacheKind kind = PageCacheKind::Leaf; + size_t max_pages = 0; + void (*complete)(void *, size_t) noexcept = nullptr; + void *context = nullptr; + PageCacheTrim *next = nullptr; +}; + +// One cache may serve independently budgeted loaders for the same backend. +// Nonzero backend_id must uniquely name that backend. Legacy zero IDs also +// compare backing owner addresses, preventing unrelated stores from aliasing. +// It never changes live mapping/overlays: returned owners serve captured views. +class SharedPageCache +{ + struct State; + class Binding; + + public: + // Residency metrics require a host-owned runtime which outlives cache + // owners and is drained after their release. No runtime ownership is held. + explicit SharedPageCache(PageCacheLimits limits = {}, std::shared_ptr<detail::AllocationReservation> metadata = {}, + std::shared_ptr<TreeResourceMetrics> resources = {}, + detail::FlushRuntime *runtime = nullptr, + std::shared_ptr<detail::FlushRuntime::Lane> lane = {}); + ~SharedPageCache(); + SharedPageCache(const SharedPageCache &) = delete; + SharedPageCache &operator=(const SharedPageCache &) = delete; + + // The miss uses the first caller's upstream, including its held load/I/O + // credit. Joined callers never acquire a second I/O permit for that base. + [[nodiscard]] std::shared_ptr<PageLoader> bind(std::shared_ptr<PageLoader> upstream, + std::shared_ptr<detail::AllocationReservation> metadata = {}, + PageLoadPriority priority = PageLoadPriority::Read) const; + // Reserve one request slot before closing flush sources. This binding has + // one serial outstanding load and reuses its slot across routing steps. + [[nodiscard]] Status bind_reserved(std::shared_ptr<PageLoader> upstream, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<PageLoader> *out, + PageLoadPriority priority = PageLoadPriority::Read) const; + [[nodiscard]] static size_t binding_allocation_bound() noexcept; + // Additional per-load progress credit when residency metrics are enabled. + // Each subscriber needs pin_allocation_bound(), including cache hits. + [[nodiscard]] static size_t load_allocation_bound() noexcept; + [[nodiscard]] static size_t pin_allocation_bound() noexcept; + [[nodiscard]] std::optional<size_t> load_reserve_bound(size_t pinned_frames, size_t workers) const noexcept; + [[nodiscard]] size_t retained_page_limit() const noexcept; + [[nodiscard]] static std::optional<size_t> allocation_bound(PageCacheLimits limits) noexcept; + [[nodiscard]] PageCacheSnapshot snapshot() const noexcept; + [[nodiscard]] const std::shared_ptr<PageCacheResidency> &residency() const noexcept; + // One live PID namespace per cache; independently budgeted load bindings + // can share it throughout that publication lifetime. + [[nodiscard]] bool claim_publication() noexcept; + // Nonblocking metadata commands. Accepted loads still complete; their + // ready results are not retained if eviction/close preceded completion. + void evict() noexcept; + void trim(PageCacheTrim &request) noexcept; + void close() noexcept; + // Close admission and detach residency progress; the caller separately + // drains accepted load/I/O owners before destroying its backend. + void close_and_drain() noexcept; + + private: + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_frame.cpp b/lib/crowdb-tree/src/btree/page_cache_frame.cpp new file mode 100644 index 000000000..f1a1f289e --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_frame.cpp @@ -0,0 +1,69 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "page_cache_frame.h" + +#include "allocation_bound.h" + +#include <cassert> + +namespace crowdb::tree +{ +namespace +{ +void change(TreeResourceMetrics &metrics, TreeResourceMetric metric, size_t amount, bool add) +{ + const bool valid = add ? metrics.add(metric, amount) : metrics.remove(metric, amount); + assert(valid); + (void)valid; +} +} // namespace + +PageCacheFrame::PageCacheFrame(std::shared_ptr<TreeResourceMetrics> metrics) : metrics_(std::move(metrics)) +{ +} + +PageCacheFrame::~PageCacheFrame() +{ + leaf.reset(); + inner.reset(); + if (active_) { + change(*metrics_, TreeResourceMetric::ResidentFrames, 1, false); + change(*metrics_, TreeResourceMetric::CacheBytes, bytes_, false); + } +} + +void PageCacheFrame::activate(size_t bytes, std::shared_ptr<const LeafBase> leaf, + std::shared_ptr<const InnerPageState> inner) noexcept +{ + this->leaf = std::move(leaf); + this->inner = std::move(inner); + bytes_ = bytes; + active_ = true; + change(*metrics_, TreeResourceMetric::ResidentFrames, 1, true); + change(*metrics_, TreeResourceMetric::CacheBytes, bytes, true); +} + +PageCacheFrame::Pin::Pin(std::shared_ptr<PageCacheFrame> frame) : frame_(std::move(frame)) +{ + if (frame_->pins_.fetch_add(1, std::memory_order_acq_rel) == 0) { + change(*frame_->metrics_, TreeResourceMetric::CachePinnedBytes, frame_->bytes_, true); + } +} + +PageCacheFrame::Pin::~Pin() +{ + if (frame_->pins_.fetch_sub(1, std::memory_order_acq_rel) == 1) { + change(*frame_->metrics_, TreeResourceMetric::CachePinnedBytes, frame_->bytes_, false); + } +} + +size_t PageCacheFrame::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<PageCacheFrame>(); +} + +size_t PageCacheFrame::pin_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<Pin>(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_frame.h b/lib/crowdb-tree/src/btree/page_cache_frame.h new file mode 100644 index 000000000..bb3d95fb3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_frame.h @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "logical_page.h" +#include "published_state.h" + +namespace crowdb::tree +{ +// The cache and every delivered pin share one frame account. Eviction drops +// the cache reference; physical frame gauges end with the final frame owner. +class PageCacheFrame +{ + public: + explicit PageCacheFrame(std::shared_ptr<TreeResourceMetrics> metrics); + ~PageCacheFrame(); + void activate(size_t bytes, std::shared_ptr<const LeafBase> leaf, + std::shared_ptr<const InnerPageState> inner) noexcept; + + class Pin + { + public: + explicit Pin(std::shared_ptr<PageCacheFrame> frame); + ~Pin(); + Pin(const Pin &) = delete; + Pin &operator=(const Pin &) = delete; + + private: + std::shared_ptr<PageCacheFrame> frame_; + }; + + [[nodiscard]] static size_t allocation_bound() noexcept; + [[nodiscard]] static size_t pin_allocation_bound() noexcept; + std::shared_ptr<const LeafBase> leaf; + std::shared_ptr<const InnerPageState> inner; + + private: + std::shared_ptr<TreeResourceMetrics> metrics_; + size_t bytes_ = 0; + std::atomic<size_t> pins_{0}; + bool active_ = false; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_identity.h b/lib/crowdb-tree/src/btree/page_cache_identity.h new file mode 100644 index 000000000..c1cee24b4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_identity.h @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "logical_page.h" + +#include <compare> + +namespace crowdb::tree +{ +enum class PageCacheKind : uint8_t { Leaf, Inner }; + +struct PageCacheKey +{ + PageCacheKind kind = PageCacheKind::Leaf; + uint64_t backend = 0; + uint64_t address = 0; + uint64_t length = 0; + uint64_t content = 0; + uint64_t generation = 0; + uint64_t checksum = 0; + uint32_t encoding = 0; + uint64_t base = 0; + uint64_t base_generation = 0; + uintptr_t legacy_owner = 0; + BackingStorage storage = BackingStorage::kDurableBlob; + std::array<uint64_t, 8> chunk{}; + auto operator<=>(const PageCacheKey &) const = default; +}; + +inline PageCacheKey page_cache_key(PageCacheKind kind, const BaseRef &base) +{ + const auto &backing = *base.backing; + return {.kind = kind, + .backend = backing.backend_id, + .address = backing.address, + .length = backing.length, + .content = backing.content_id, + .generation = backing.generation, + .checksum = backing.checksum, + .encoding = backing.encoding, + .base = kind == PageCacheKind::Leaf ? base.identity : 0, + .base_generation = kind == PageCacheKind::Leaf ? base.generation : 0, + .legacy_owner = backing.backend_id == 0 ? reinterpret_cast<uintptr_t>(backing.owner.get()) : 0, + .storage = backing.storage, + .chunk = backing.chunk ? chunk_backing_identity(*backing.chunk) : std::array<uint64_t, 8>{}}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_internal.h b/lib/crowdb-tree/src/btree/page_cache_internal.h new file mode 100644 index 000000000..d783002b0 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_internal.h @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "page_cache.h" +#include "page_cache_frame.h" +#include "published_state.h" + +namespace crowdb::tree +{ +struct SharedPageCache::State : std::enable_shared_from_this<SharedPageCache::State> +{ + using Kind = PageCacheKind; + using Key = PageCacheKey; + enum class Phase : uint8_t { Empty, Loading, Ready }; + enum class CommandKind : uint8_t { Request, Loaded }; + + struct Command + { + CommandKind kind = CommandKind::Request; + Command *next = nullptr; + void *context = nullptr; + }; + + struct Reservation; + + struct Request + { + std::atomic<bool> occupied{false}; + Command command; + Request *next = nullptr; + PageLoadPriority priority = PageLoadPriority::Read; + std::shared_ptr<State> owner; + std::shared_ptr<Reservation> reservation; + std::shared_ptr<PageLoader> upstream; + std::shared_ptr<detail::AllocationReservation> metadata; + BaseRef base; + Key key; + PageLoader::LeafComplete leaf_done; + PageLoader::InnerComplete inner_done; + uint64_t registered_at_ns = 0; + TreeWaitSpan shared_wait; + }; + + struct Reservation + { + Reservation(std::shared_ptr<State> owner, Request *slot); + ~Reservation(); + std::shared_ptr<State> owner; + Request *slot; + std::atomic<bool> active{false}; + }; + + struct Entry + { + Command command{.kind = CommandKind::Loaded}; + Phase phase = Phase::Empty; + Key key; + uint64_t touched = 0; + size_t bytes = 0; + uint64_t max_slot = 0; + bool drop_when_ready = false; + PageLoadPriority priority = PageLoadPriority::Maintenance; + Request *waiters = nullptr; + std::shared_ptr<const BackingRef> backing; + std::shared_ptr<PageLoader> upstream; + // One atomic identity/terminal word rejects duplicate/late callbacks, + // including after an entry has been recycled for another generation. + std::atomic<uint64_t> completion{0}; + Status status; + std::shared_ptr<const LeafBase> leaf; + std::shared_ptr<const InnerPageState> inner; + std::shared_ptr<PageCacheResidency::Ready> residency; + std::shared_ptr<PageCacheFrame> frame; + }; + + State(PageCacheLimits bounds, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<TreeResourceMetrics> resources, detail::FlushRuntime *runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane); + static Key make_key(Kind kind, const BaseRef &base); + Request *claim(bool accept = true) noexcept; + void submit(Request *request) noexcept; + void enqueue(Command *command) noexcept; + void wake() noexcept; + void drain() noexcept; + void process(Request *request, bool queued = false); + void pend(Request *request) noexcept; + void loaded(Entry *entry); + void start(Entry *entry, Request *request); + void complete(Entry *entry, uint64_t identity, Status status, std::shared_ptr<const LeafBase> leaf, + std::shared_ptr<const InnerPageState> inner) noexcept; + void deliver(Request *request, const Status &status, std::shared_ptr<const LeafBase> leaf = {}, + std::shared_ptr<const InnerPageState> inner = {}, uint64_t max_slot = 0, + const std::shared_ptr<PageCacheFrame> &frame = {}) noexcept; + void discard(Entry &entry) noexcept; + Entry *victim() noexcept; + Entry *ready_victim() noexcept; + void evict_ready() noexcept; + size_t trim_ready(PageCacheKind kind, size_t max_pages) noexcept; + void drain_trims() noexcept; + void resume_pending(); + + const PageCacheLimits limits; + const std::shared_ptr<PageCacheResidency> residency; + detail::ReservedVector<Request> requests; + detail::ReservedVector<Entry> entries; + std::atomic<size_t> next_request{0}; + std::atomic<Command *> incoming{nullptr}; + std::atomic<PageCacheTrim *> trims{nullptr}; + std::atomic<bool> running{false}; + std::atomic<bool> closed{false}; + std::atomic<bool> evict_requested{false}; + std::atomic<bool> publication_claimed{false}; + uint64_t clock = 0; + uint64_t next_identity = 1; + std::array<Request *, 2> pending{}; + std::array<Request *, 2> pending_tail{}; + size_t read_burst = 0; + bool resume_requested = false; + std::atomic<uint64_t> hits{0}, misses{0}, joined{0}, evictions{0}, rejected{0}; + std::atomic<uint64_t> ordinary_waiters{0}; + std::atomic<uint64_t> inflight{0}, waiters{0}, reserved_waiters{0}, resident_pages{0}, resident_bytes{0}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_load.cpp b/lib/crowdb-tree/src/btree/page_cache_load.cpp new file mode 100644 index 000000000..94283b272 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_load.cpp @@ -0,0 +1,342 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "page_cache_internal.h" +#include "page_load_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +void SharedPageCache::State::process(Request *request, bool queued) +{ + for (auto &entry : entries) { + if (entry.phase == Phase::Empty || entry.key != request->key) { + continue; + } + entry.touched = ++clock; + if (request->priority == PageLoadPriority::Read) { + entry.priority = PageLoadPriority::Read; + } + if (entry.phase == Phase::Ready) { + hits.fetch_add(1, std::memory_order_relaxed); + record_work(request->metadata, TreeWorkCounter::CacheHits, 1, + request->priority == PageLoadPriority::Read ? TreeWorkCause::Read : TreeWorkCause::Flush); + deliver(request, Status::Ok(), entry.leaf, entry.inner, entry.max_slot, entry.frame); + } + else { + request->shared_wait.begin(request->metadata ? request->metadata->account()->work_metrics() : nullptr, + TreeWaitClass::SharedCache, request->registered_at_ns); + request->next = entry.waiters; + entry.waiters = request; + joined.fetch_add(1, std::memory_order_relaxed); + record_work(request->metadata, TreeWorkCounter::CacheJoinedLoads, 1, + request->priority == PageLoadPriority::Read ? TreeWorkCause::Read : TreeWorkCause::Flush); + } + return; + } + if (!queued && (pending[0] != nullptr || pending[1] != nullptr)) { + pend(request); + resume_requested = true; + return; + } + auto *entry = victim(); + if (entry == nullptr) { + pend(request); + return; + } + if (next_identity >= UINT64_MAX / 2) { + rejected.fetch_add(1, std::memory_order_relaxed); + deliver(request, Status::unavailable()); + return; + } + discard(*entry); + start(entry, request); +} + +void SharedPageCache::State::pend(Request *request) noexcept +{ + const auto priority = static_cast<size_t>(request->priority); + request->next = nullptr; + if (pending_tail[priority] != nullptr) { + pending_tail[priority]->next = request; + } + else { + pending[priority] = request; + } + pending_tail[priority] = request; +} + +SharedPageCache::State::Entry *SharedPageCache::State::victim() noexcept +{ + for (auto &entry : entries) { + if (entry.phase == Phase::Empty) { + return &entry; + } + } + return ready_victim(); +} + +SharedPageCache::State::Entry *SharedPageCache::State::ready_victim() noexcept +{ + Entry *oldest = nullptr; + for (auto &entry : entries) { + if (entry.phase == Phase::Ready && (oldest == nullptr || entry.priority > oldest->priority || + (entry.priority == oldest->priority && entry.touched < oldest->touched))) { + oldest = &entry; + } + } + return oldest; +} + +void SharedPageCache::State::start(Entry *entry, Request *request) +{ + if (residency) { + auto ready = residency->prepare_ready(request->key, request->metadata); + auto frame = detail::reserved_shared<PageCacheFrame>(request->metadata, residency->resources()); + entry->residency = std::move(ready); + entry->frame = std::move(frame); + } + entry->phase = Phase::Loading; + entry->key = request->key; + entry->priority = request->priority; + entry->touched = ++clock; + entry->backing = request->base.backing; + entry->upstream = request->upstream; + entry->waiters = request; + request->next = nullptr; + entry->drop_when_ready = closed.load(std::memory_order_acquire); + const auto identity = next_identity++; + entry->completion.store(identity << 1, std::memory_order_release); + misses.fetch_add(1, std::memory_order_relaxed); + record_work(request->metadata, TreeWorkCounter::CacheMisses, 1, + request->priority == PageLoadPriority::Read ? TreeWorkCause::Read : TreeWorkCause::Flush); + inflight.fetch_add(1, std::memory_order_relaxed); + auto self = shared_from_this(); + try { + if (entry->key.kind == Kind::Leaf) { + auto input = request->base; + // Validate each waiter's proof against the common decoded frame, + // so an invalid proof cannot poison other subscribers to this base. + input.max_slot.reset(); + entry->upstream->load_leaf(std::move(input), [self, entry, identity](Status status, auto leaf) { + self->complete(entry, identity, std::move(status), std::move(leaf), {}); + }); + } + else { + entry->upstream->load_inner(entry->backing, [self, entry, identity](Status status, auto inner) { + self->complete(entry, identity, std::move(status), {}, std::move(inner)); + }); + } + } + catch (const std::bad_alloc &) { + complete(entry, identity, Status::resource_exhausted(), {}, {}); + } + catch (...) { + complete(entry, identity, Status::io_error(), {}, {}); + } +} + +void SharedPageCache::State::complete(Entry *entry, uint64_t identity, Status status, + std::shared_ptr<const LeafBase> leaf, + std::shared_ptr<const InnerPageState> inner) noexcept +{ + auto expected = identity << 1; + if (!entry->completion.compare_exchange_strong(expected, expected | 1, std::memory_order_acq_rel)) { + record_work(entries.get_allocator().owner(), TreeWorkCounter::StaleCallbacks); + return; + } + entry->status = std::move(status); + entry->leaf = std::move(leaf); + entry->inner = std::move(inner); + enqueue(&entry->command); +} + +void SharedPageCache::State::loaded(Entry *entry) +{ + resume_requested = true; + inflight.fetch_sub(1, std::memory_order_relaxed); + entry->upstream.reset(); + detail::AllocationBound bytes; + if (entry->status.ok()) { + if (entry->key.kind == Kind::Leaf && entry->leaf && !entry->inner) { + bytes.add(entry->leaf->page_bytes()); + for (size_t i = 0; i < entry->leaf->count(); ++i) { + entry->max_slot = std::max(entry->max_slot, CellView(entry->leaf->cell(i)).slot()); + } + } + else if (entry->key.kind == Kind::Inner && entry->inner && !entry->leaf) { + bytes.add_product(entry->inner->children.capacity(), sizeof(uint64_t)); + bytes.add_product(entry->inner->separators.capacity(), sizeof(OwnedKey)); + for (const auto &key : entry->inner->separators) { + bytes.add(key.size()); + } + } + else { + entry->status = Status::corruption(); + } + } + const auto size = bytes.value(); + if (entry->status.ok() && !size) { + entry->status = Status::resource_exhausted(); + } + if (entry->status.ok() && entry->frame && size) { + entry->frame->activate(*size, entry->leaf, entry->inner); + } + const bool keep = entry->status.ok() && size && *size <= limits.max_resident_bytes && !entry->drop_when_ready && + !closed.load(std::memory_order_acquire); + if (keep) { + if (resident_bytes.load(std::memory_order_relaxed) > limits.max_resident_bytes - *size) { + const auto low = limits.low_resident_bytes != 0 + ? limits.low_resident_bytes + : limits.max_resident_bytes - (limits.max_resident_bytes / 4); + const auto target = *size < low ? low - *size : 0; + while (resident_bytes.load(std::memory_order_relaxed) > target) { + auto *oldest = ready_victim(); + if (oldest == nullptr) { + break; + } + discard(*oldest); + } + } + entry->phase = Phase::Ready; + entry->bytes = *size; + resident_pages.fetch_add(1, std::memory_order_relaxed); + resident_bytes.fetch_add(*size, std::memory_order_relaxed); + if (entry->residency) { + PageCacheResidency::ready(entry->residency); + } + } + while (entry->waiters != nullptr) { + auto *request = entry->waiters; + entry->waiters = request->next; + deliver(request, entry->status, entry->leaf, entry->inner, entry->max_slot, entry->frame); + } + if (!keep) { + discard(*entry); + } +} + +void SharedPageCache::State::deliver(Request *request, const Status &status, std::shared_ptr<const LeafBase> leaf, + std::shared_ptr<const InnerPageState> inner, uint64_t max_slot, + const std::shared_ptr<PageCacheFrame> &frame) noexcept +{ + Status outcome; + try { + outcome = status; + } + catch (const std::bad_alloc &) { + outcome = Status::resource_exhausted(); + } + if (outcome.ok() && request->key.kind == Kind::Leaf && request->base.max_slot && + max_slot > *request->base.max_slot) { + outcome = Status::corruption(); + } + if (outcome.ok() && frame) { + try { + auto pin = detail::reserved_shared<PageCacheFrame::Pin>(request->metadata, frame); + if (leaf) { + leaf = std::shared_ptr<const LeafBase>(pin, leaf.get()); + } + if (inner) { + inner = std::shared_ptr<const InnerPageState>(pin, inner.get()); + } + } + catch (const std::bad_alloc &) { + outcome = Status::resource_exhausted(); + } + } + if (!outcome.ok()) { + leaf.reset(); + inner.reset(); + } + request->shared_wait.finish(outcome.ok() ? TreeWaitOutcome::Granted : TreeWaitOutcome::Error, + tree_metrics_now_ns()); + auto owner = std::move(request->owner); + auto reservation = std::move(request->reservation); + auto leaf_done = std::move(request->leaf_done); + auto inner_done = std::move(request->inner_done); + request->base = {}; + request->upstream.reset(); + request->metadata.reset(); + request->next = nullptr; + waiters.fetch_sub(1, std::memory_order_relaxed); + if (reservation) { + reservation->active.store(false, std::memory_order_release); + } + else { + ordinary_waiters.fetch_sub(1, std::memory_order_relaxed); + request->occupied.store(false, std::memory_order_release); + } + if (leaf_done) { + complete_page_load(leaf_done, std::move(outcome), std::move(leaf)); + } + else { + complete_page_load(inner_done, std::move(outcome), std::move(inner)); + } +} + +void SharedPageCache::State::discard(Entry &entry) noexcept +{ + if (entry.phase == Phase::Ready) { + if (entry.residency) { + PageCacheResidency::evict(entry.residency); + } + resident_pages.fetch_sub(1, std::memory_order_relaxed); + resident_bytes.fetch_sub(entry.bytes, std::memory_order_relaxed); + evictions.fetch_add(1, std::memory_order_relaxed); + record_work(entries.get_allocator().owner(), TreeWorkCounter::CacheEvictions, 1, TreeWorkCause::CachePressure); + } + entry.phase = Phase::Empty; + entry.leaf.reset(); + entry.inner.reset(); + entry.frame.reset(); + entry.residency.reset(); + entry.backing.reset(); + entry.upstream.reset(); + entry.status = {}; + entry.bytes = 0; + entry.max_slot = 0; +} + +void SharedPageCache::State::evict_ready() noexcept +{ + resume_requested = true; + for (auto &entry : entries) { + if (entry.phase == Phase::Ready) { + discard(entry); + } + else if (entry.phase == Phase::Loading) { + entry.drop_when_ready = true; + } + } +} + +void SharedPageCache::State::resume_pending() +{ + // The burst bound gives foreground reads priority without starving the + // prepaid maintenance request which makes memory pressure progress. + for (size_t attempts = 0; attempts < requests.size(); ++attempts) { + if ((pending[0] == nullptr && pending[1] == nullptr) || victim() == nullptr) { + return; + } + const size_t priority = + pending[1] != nullptr && (pending[0] == nullptr || read_burst == limits.max_read_burst) ? 1 : 0; + auto *request = pending[priority]; + pending[priority] = request->next; + if (pending[priority] == nullptr) { + pending_tail[priority] = nullptr; + } + read_burst = priority == 1 ? 0 : std::min(read_burst + 1, limits.max_read_burst); + try { + process(request, true); + } + catch (const std::bad_alloc &) { + deliver(request, Status::resource_exhausted()); + } + catch (...) { + deliver(request, Status::internal_error()); + } + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_queue.cpp b/lib/crowdb-tree/src/btree/page_cache_queue.cpp new file mode 100644 index 000000000..fd32e154d --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_queue.cpp @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "page_cache_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +SharedPageCache::State::Request *SharedPageCache::State::claim(bool accept) noexcept +{ + if (!closed.load(std::memory_order_acquire)) { + const auto first = next_request.fetch_add(1, std::memory_order_relaxed) % requests.size(); + for (size_t offset = 0; offset < requests.size(); ++offset) { + auto &request = requests[(first + offset) % requests.size()]; + bool free = false; + if (request.occupied.compare_exchange_strong(free, true, std::memory_order_acquire)) { + if (accept) { + request.owner = weak_from_this().lock(); + assert(request.owner); + waiters.fetch_add(1, std::memory_order_relaxed); + ordinary_waiters.fetch_add(1, std::memory_order_relaxed); + } + return &request; + } + } + } + rejected.fetch_add(1, std::memory_order_relaxed); + return nullptr; +} + +void SharedPageCache::State::submit(Request *request) noexcept +{ + request->registered_at_ns = tree_metrics_now_ns(); + enqueue(&request->command); +} + +void SharedPageCache::State::enqueue(Command *command) noexcept +{ + auto *head = incoming.load(std::memory_order_relaxed); + do { + command->next = head; + } while (!incoming.compare_exchange_weak(head, command, std::memory_order_release, std::memory_order_relaxed)); + wake(); +} + +void SharedPageCache::State::wake() noexcept +{ + auto owner = weak_from_this().lock(); + assert(owner); + if (!running.exchange(true, std::memory_order_acq_rel)) { + drain(); + } +} + +void SharedPageCache::State::drain() noexcept +{ + for (;;) { + auto *list = incoming.exchange(nullptr, std::memory_order_acquire); + Command *ordered = nullptr; + while (list != nullptr) { + auto *next = list->next; + list->next = ordered; + ordered = list; + list = next; + } + while (ordered != nullptr) { + auto *next = ordered->next; + if (ordered->kind == CommandKind::Request) { + auto *request = static_cast<Request *>(ordered->context); + try { + process(request); + } + catch (const std::bad_alloc &) { + deliver(request, Status::resource_exhausted()); + } + catch (...) { + deliver(request, Status::internal_error()); + } + } + else { + loaded(static_cast<Entry *>(ordered->context)); + } + ordered = next; + } + drain_trims(); + if (evict_requested.exchange(false, std::memory_order_acq_rel)) { + evict_ready(); + } + if (resume_requested) { + resume_requested = false; + resume_pending(); + } + // A producer that observed running publishes its command before its + // exchange. Acquire that handoff before rechecking the incoming lists. + running.exchange(false, std::memory_order_acq_rel); + if ((incoming.load(std::memory_order_acquire) == nullptr && trims.load(std::memory_order_acquire) == nullptr && + !evict_requested.load(std::memory_order_acquire)) || + running.exchange(true, std::memory_order_acq_rel)) { + return; + } + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_residency.cpp b/lib/crowdb-tree/src/btree/page_cache_residency.cpp new file mode 100644 index 000000000..4ddcbde5e --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_residency.cpp @@ -0,0 +1,195 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "page_cache_residency_internal.h" +#include "published_state.h" + +#include <algorithm> +#include <cassert> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +thread_local std::shared_ptr<detail::AllocationReservation> node_budget; + +struct NodeHeader +{ + AllocationLease credit; +}; + +size_t header_bytes(size_t alignment) +{ + return (sizeof(NodeHeader) + alignment - 1) & ~(alignment - 1); +} +} // namespace + +PageCacheNodeResource::Scope::Scope(std::shared_ptr<detail::AllocationReservation> budget) + : previous_(std::exchange(node_budget, std::move(budget))) +{ +} + +PageCacheNodeResource::Scope::~Scope() +{ + node_budget = std::move(previous_); +} + +void *PageCacheNodeResource::do_allocate(size_t bytes, size_t alignment) +{ + alignment = std::max(alignment, alignof(NodeHeader)); + const auto offset = header_bytes(alignment); + if (bytes > SIZE_MAX - offset) { + throw std::bad_alloc(); + } + AllocationLease credit(node_budget, bytes + offset, AllocationKind::kMetadata); + auto *allocation = static_cast<std::byte *>(std::pmr::get_default_resource()->allocate(bytes + offset, alignment)); + std::construct_at(reinterpret_cast<NodeHeader *>(allocation), NodeHeader{std::move(credit)}); + return allocation + offset; +} + +void PageCacheNodeResource::do_deallocate(void *pointer, size_t bytes, size_t alignment) +{ + alignment = std::max(alignment, alignof(NodeHeader)); + const auto offset = header_bytes(alignment); + auto *allocation = static_cast<std::byte *>(pointer) - offset; + auto *header = reinterpret_cast<NodeHeader *>(allocation); + auto credit = std::move(header->credit); + std::destroy_at(header); + std::pmr::get_default_resource()->deallocate(allocation, bytes + offset, alignment); +} + +bool PageCacheNodeResource::do_is_equal(const std::pmr::memory_resource &other) const noexcept +{ + return this == &other; +} + +PageCacheResidency::State::State(std::shared_ptr<TreeResourceMetrics> resources) : metrics(std::move(resources)) +{ +} + +PageCacheResidency::PageCacheResidency(std::shared_ptr<TreeResourceMetrics> resources, + const std::shared_ptr<detail::AllocationReservation> &metadata, + detail::FlushRuntime *runtime, std::shared_ptr<detail::FlushRuntime::Lane> lane) + : state_(detail::reserved_shared<State>(metadata, std::move(resources))) +{ + if (runtime == nullptr || !lane) { + throw std::invalid_argument("page residency requires a host runtime and lane"); + } + const auto status = + runtime->reserve_progress(std::move(lane), &State::step, state_.get(), metadata, &state_->progress); + if (!status.ok()) { + throw std::bad_alloc(); + } +} + +PageCacheResidency::~PageCacheResidency() +{ + clear(); +} + +std::shared_ptr<PageCacheResidency::Update> +PageCacheResidency::prepare(std::span<const PageReplacement> pages, + const std::shared_ptr<detail::AllocationReservation> &metadata) const +{ + auto update = detail::reserved_shared<Update>(metadata, state_); + PageCacheNodeResource::Scope scope(metadata); + for (const auto &edit : pages) { + if (edit.expected == edit.replacement) { + continue; + } + State::prepare_page(*update, edit.page_id, edit.replacement, metadata); + } + return update; +} + +void PageCacheResidency::publish(std::shared_ptr<Update> update) noexcept +{ + while (update) { + auto following = std::move(update->following); + auto *command = &update->command; + auto state = update->state; + update->queued_owner = std::move(update); + state->enqueue(command); + update = std::move(following); + } +} + +std::shared_ptr<PageCacheResidency::Ready> +PageCacheResidency::prepare_ready(const PageCacheKey &key, + const std::shared_ptr<detail::AllocationReservation> &metadata) const +{ + auto result = detail::reserved_shared<Ready>(metadata, state_); + PageCacheNodeResource::Scope scope(metadata); + result->key = key; + result->bases.emplace(key, detail::reserved_shared<State::Base>(metadata)); + return result; +} + +void PageCacheResidency::ready(const std::shared_ptr<Ready> &ready) noexcept +{ + ready->added_owner = ready; + ready->state->enqueue(&ready->added); +} + +void PageCacheResidency::evict(const std::shared_ptr<Ready> &ready) noexcept +{ + ready->removed_owner = ready; + ready->state->enqueue(&ready->removed); +} + +void PageCacheResidency::clear() noexcept +{ + if (!state_->enter()) { + return; + } + state_->clear_requested.store(true, std::memory_order_release); + state_->wake(); + state_->leave(); +} + +void PageCacheResidency::close_and_drain() noexcept +{ + auto value = state_->producers.fetch_or(State::kClosed, std::memory_order_acq_rel) | State::kClosed; + while ((value & ~State::kClosed) != 0) { + state_->producers.wait(value, std::memory_order_acquire); + value = state_->producers.load(std::memory_order_acquire); + } + state_->clear_requested.store(true, std::memory_order_release); + state_->wake(); + // Internal reruns remain enabled until every bounded clear step finishes. + state_->progress->drain(); + state_->progress->close_and_drain(); + state_->bases.clear(); +} + +uint64_t PageCacheResidency::proxy_pages() const noexcept +{ + return state_->proxies.load(std::memory_order_relaxed); +} + +const std::shared_ptr<TreeResourceMetrics> &PageCacheResidency::resources() const noexcept +{ + return state_->metrics; +} + +size_t PageCacheResidency::page_allocation_bound(size_t refs) noexcept +{ + return detail::AllocationBound(detail::shared_allocation_bound<Update>() + + detail::shared_allocation_bound<State::Page>() + 256) + .add_product(refs, sizeof(State::Link) + detail::shared_allocation_bound<State::Base>() + 256) + .value() + .value_or(SIZE_MAX); +} + +size_t PageCacheResidency::ready_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<Ready>() + detail::shared_allocation_bound<State::Base>() + 256; +} + +size_t PageCacheResidency::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<PageCacheResidency>() + detail::shared_allocation_bound<State>() + + detail::FlushRuntime::progress_allocation_bound(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_residency.h b/lib/crowdb-tree/src/btree/page_cache_residency.h new file mode 100644 index 000000000..351bb976b --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_residency.h @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "flush_runtime.h" +#include "page_cache_identity.h" +#include "tree_metrics.h" + +namespace crowdb::tree +{ +struct PageReplacement; +struct PublishedPage; +class MappingView; + +// A single live mapping owns this index. Old captures retain data owners only. +// All updates enter a reserved runtime continuation after preparing their nodes. +// The host owns the runtime through destruction and draining of this index. +class PageCacheResidency +{ + struct State; + + public: + class Update; + class Ready; + explicit PageCacheResidency(std::shared_ptr<TreeResourceMetrics> resources, + const std::shared_ptr<detail::AllocationReservation> &metadata = {}, + detail::FlushRuntime *runtime = nullptr, + std::shared_ptr<detail::FlushRuntime::Lane> lane = {}); + ~PageCacheResidency(); + [[nodiscard]] std::shared_ptr<Update> prepare(std::span<const PageReplacement> pages, + const std::shared_ptr<detail::AllocationReservation> &metadata) const; + [[nodiscard]] std::shared_ptr<Update> + prepare_initial(const MappingView &mapping, const std::shared_ptr<detail::AllocationReservation> &metadata) const; + [[nodiscard]] std::shared_ptr<Update> + prepare_replacement(const MappingView &mapping, + const std::shared_ptr<detail::AllocationReservation> &metadata) const; + [[nodiscard]] static size_t initial_allocation_bound(const MappingView &mapping) noexcept; + static void publish(std::shared_ptr<Update> update) noexcept; + [[nodiscard]] std::shared_ptr<Ready> + prepare_ready(const PageCacheKey &key, const std::shared_ptr<detail::AllocationReservation> &metadata) const; + static void ready(const std::shared_ptr<Ready> &ready) noexcept; + static void evict(const std::shared_ptr<Ready> &ready) noexcept; + void clear() noexcept; + // Host-thread boundary: stop index producers, drain only this progress + // registration and detach from the runtime. Late pin releases are safe. + void close_and_drain() noexcept; + [[nodiscard]] uint64_t proxy_pages() const noexcept; + [[nodiscard]] const std::shared_ptr<TreeResourceMetrics> &resources() const noexcept; + [[nodiscard]] static size_t page_allocation_bound(size_t refs) noexcept; + [[nodiscard]] static size_t allocation_bound() noexcept; + [[nodiscard]] static size_t ready_allocation_bound() noexcept; + + private: + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_residency_apply.cpp b/lib/crowdb-tree/src/btree/page_cache_residency_apply.cpp new file mode 100644 index 000000000..dce162ec7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_residency_apply.cpp @@ -0,0 +1,243 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "page_cache_residency_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +void PageCacheResidency::State::proxy(bool added) noexcept +{ + if (added) { + proxies.fetch_add(1, std::memory_order_relaxed); + } + else { + proxies.fetch_sub(1, std::memory_order_relaxed); + } + if (metrics) { + const bool valid = added ? metrics->add(TreeResourceMetric::ProxyPages, 1) + : metrics->remove(TreeResourceMetric::ProxyPages, 1); + assert(valid); + (void)valid; + } +} + +void PageCacheResidency::State::erase_unused(const PageCacheKey &key) noexcept +{ + auto found = bases.find(key); + if (found != bases.end() && found->second->views == nullptr && found->second->ready == 0) { + bases.erase(found); + } +} + +void PageCacheResidency::State::unlink(Page &page) noexcept +{ + if (page.cold != 0) { + proxy(false); + } + for (auto &link : page.links) { + if (link.previous != nullptr) { + link.previous->next = link.next; + } + else { + link.base->views = link.next; + } + if (link.next != nullptr) { + link.next->previous = link.previous; + } + erase_unused(link.key); + } +} + +void PageCacheResidency::State::attach(Page &page) noexcept +{ + for (auto &link : page.links) { + link.base = bases.find(link.key)->second.get(); + link.page = &page; + link.next = link.base->views; + if (link.next != nullptr) { + link.next->previous = &link; + } + link.base->views = &link; + page.cold += link.base->ready == 0 ? 1 : 0; + } + if (page.cold != 0) { + proxy(true); + } +} + +void PageCacheResidency::State::apply(Update &update) noexcept +{ + // Detach all old links before adding new pages; the prepared base nodes + // also cover bases whose last old view disappears in this transaction. + for (const auto &[pid, replacement] : update.pages) { + auto found = pages.find(pid); + if (found != pages.end()) { + unlink(*found->second); + pages.erase(found); + } + } + while (!update.bases.empty()) { + bases.insert(update.bases.extract(update.bases.begin())); + } + while (!update.pages.empty()) { + auto node = update.pages.extract(update.pages.begin()); + if (node.mapped()) { + attach(*node.mapped()); + pages.insert(std::move(node)); + } + } +} + +void PageCacheResidency::State::change(Base &base, bool present) noexcept +{ + for (auto *link = base.views; link != nullptr; link = link->next) { + auto &page = *link->page; + if (present) { + assert(page.cold != 0); + if (--page.cold == 0) { + proxy(false); + } + } + else if (page.cold++ == 0) { + proxy(true); + } + } +} + +void PageCacheResidency::State::apply(Ready &ready, bool present) noexcept +{ + if (present) { + bases.insert(ready.bases.extract(ready.bases.begin())); + auto &base = *bases.find(ready.key)->second; + if (base.ready++ == 0) { + change(base, true); + } + } + else { + auto &base = *bases.find(ready.key)->second; + assert(base.ready != 0); + if (--base.ready == 0) { + change(base, false); + } + erase_unused(ready.key); + } +} + +void PageCacheResidency::State::enqueue(Command *command) noexcept +{ + if (!enter()) { + discard(command); + return; + } + auto *head = incoming.load(std::memory_order_relaxed); + do { + command->next = head; + } while (!incoming.compare_exchange_weak(head, command, std::memory_order_release, std::memory_order_relaxed)); + wake(); + leave(); +} + +bool PageCacheResidency::State::enter() noexcept +{ + auto value = producers.load(std::memory_order_acquire); + do { + if ((value & kClosed) != 0) { + return false; + } + } while (!producers.compare_exchange_weak(value, value + 1, std::memory_order_acq_rel)); + return true; +} + +void PageCacheResidency::State::leave() noexcept +{ + producers.fetch_sub(1, std::memory_order_acq_rel); + producers.notify_all(); +} + +void PageCacheResidency::State::discard(Command *command) noexcept +{ + if (command->kind == Kind::Mapping) { + auto owner = std::move(static_cast<Update *>(command->context)->queued_owner); + } + else { + auto *ready = static_cast<Ready *>(command->context); + auto owner = command->kind == Kind::Ready ? std::move(ready->added_owner) : std::move(ready->removed_owner); + } +} + +void PageCacheResidency::State::wake() noexcept +{ + auto owner = weak_from_this().lock(); + assert(owner); + progress->request(std::move(owner)); +} + +void PageCacheResidency::State::step(void *context) noexcept +{ + static_cast<State *>(context)->drain(); +} + +void PageCacheResidency::State::apply_command(Command *command) noexcept +{ + if (command->kind == Kind::Mapping) { + auto *update = static_cast<Update *>(command->context); + auto owner = std::move(update->queued_owner); + if (!clear_requested.load(std::memory_order_acquire)) { + apply(*update); + } + } + else { + auto *ready = static_cast<Ready *>(command->context); + const bool present = command->kind == Kind::Ready; + auto owner = present ? std::move(ready->added_owner) : std::move(ready->removed_owner); + apply(*ready, present); + } +} + +void PageCacheResidency::State::drain() noexcept +{ + // Reverse at most one detached producer batch, over bounded steps. New + // arrivals cannot extend that batch or keep this worker in one task. + constexpr size_t kStepCommands = 32; + if (reversing == nullptr && ordered == nullptr) { + reversing = incoming.exchange(nullptr, std::memory_order_acquire); + } + for (size_t i = 0; i < kStepCommands && reversing != nullptr; ++i) { + auto *next = reversing->next; + reversing->next = ordered; + ordered = reversing; + reversing = next; + } + for (size_t i = 0; i < kStepCommands && reversing == nullptr && ordered != nullptr; ++i) { + auto *command = ordered; + if (command->kind == Kind::Mapping) { + auto *update = static_cast<Update *>(command->context); + if (update->reset && !pages.empty()) { + for (size_t removed = 0; removed < kStepCommands && !pages.empty(); ++removed) { + auto page = pages.begin(); + unlink(*page->second); + pages.erase(page); + } + // Keep this command at the head until old live membership is + // gone. Ready owners remain physical cache residents, and + // later generation updates cannot overtake the reset. + break; + } + } + ordered = ordered->next; + apply_command(command); + } + if (clear_requested.load(std::memory_order_acquire)) { + for (size_t i = 0; i < kStepCommands && !pages.empty(); ++i) { + auto page = pages.begin(); + unlink(*page->second); + pages.erase(page); + } + } + if (reversing != nullptr || ordered != nullptr || incoming.load(std::memory_order_acquire) != nullptr || + (clear_requested.load(std::memory_order_acquire) && !pages.empty())) { + wake(); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_residency_internal.h b/lib/crowdb-tree/src/btree/page_cache_residency_internal.h new file mode 100644 index 000000000..337feb07a --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_residency_internal.h @@ -0,0 +1,157 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "page_cache_residency.h" + +#include <map> + +namespace crowdb::tree +{ +// Preparation may run concurrently with cache completion. Allocation binds to +// the calling thread; every node retains its own credit for later reclamation. +class PageCacheNodeResource final : public std::pmr::memory_resource +{ + public: + class Scope + { + public: + explicit Scope(std::shared_ptr<detail::AllocationReservation> budget); + ~Scope(); + Scope(const Scope &) = delete; + Scope &operator=(const Scope &) = delete; + + private: + std::shared_ptr<detail::AllocationReservation> previous_; + }; + + private: + void *do_allocate(size_t bytes, size_t alignment) override; + void do_deallocate(void *pointer, size_t bytes, size_t alignment) override; + [[nodiscard]] bool do_is_equal(const std::pmr::memory_resource &other) const noexcept override; +}; + +struct PageCacheResidency::State : std::enable_shared_from_this<State> +{ + struct Page; + struct Base; + + struct Link + { + PageCacheKey key; + Page *page = nullptr; + Base *base = nullptr; + Link *previous = nullptr; + Link *next = nullptr; + }; + + struct Page + { + explicit Page(const std::shared_ptr<detail::AllocationReservation> &metadata) + : links(detail::ReservationAllocator<Link>(metadata)) + { + } + + detail::ReservedVector<Link> links; + size_t cold = 0; + }; + + struct Base + { + Link *views = nullptr; + uint64_t ready = 0; + }; + + using Pages = std::pmr::map<uint64_t, std::shared_ptr<Page>>; + using Bases = std::pmr::map<PageCacheKey, std::shared_ptr<Base>>; + enum class Kind : uint8_t { Mapping, Ready, Evict }; + + struct Command + { + Kind kind; + Command *next = nullptr; + void *context = nullptr; + }; + + explicit State(std::shared_ptr<TreeResourceMetrics> resources); + static void prepare_page(Update &update, uint64_t pid, const std::shared_ptr<const PublishedPage> &replacement, + const std::shared_ptr<detail::AllocationReservation> &metadata); + void enqueue(Command *command) noexcept; + bool enter() noexcept; + void leave() noexcept; + static void discard(Command *command) noexcept; + void wake() noexcept; + void drain() noexcept; + void apply_command(Command *command) noexcept; + static void step(void *context) noexcept; + void apply(Update &update) noexcept; + void apply(Ready &ready, bool present) noexcept; + void unlink(Page &page) noexcept; + void attach(Page &page) noexcept; + void change(Base &base, bool present) noexcept; + void proxy(bool added) noexcept; + void erase_unused(const PageCacheKey &key) noexcept; + + PageCacheNodeResource allocator; + Pages pages{&allocator}; + Bases bases{&allocator}; + const std::shared_ptr<TreeResourceMetrics> metrics; + std::atomic<Command *> incoming{nullptr}; + static constexpr uint64_t kClosed = uint64_t{1} << 63; + std::atomic<uint64_t> producers{0}; + std::atomic<bool> clear_requested{false}; + std::shared_ptr<detail::FlushRuntime::Progress> progress; + Command *reversing = nullptr; + Command *ordered = nullptr; + std::atomic<uint64_t> proxies{0}; +}; + +class PageCacheResidency::Update +{ + public: + bool retire_step() noexcept + { + size_t steps = 0; + for (; steps < 64 && !pages.empty(); ++steps) { + pages.erase(pages.begin()); + } + for (; steps < 64 && !bases.empty(); ++steps) { + bases.erase(bases.begin()); + } + return pages.empty() && bases.empty(); + } + + explicit Update(std::shared_ptr<State> state) + : state(std::move(state)), + pages(&this->state->allocator), + bases(&this->state->allocator) + { + command.context = this; + } + + std::shared_ptr<State> state; + State::Pages pages; + State::Bases bases; + State::Command command{.kind = State::Kind::Mapping}; + std::shared_ptr<Update> queued_owner; + std::shared_ptr<Update> following; + bool reset = false; +}; + +class PageCacheResidency::Ready +{ + public: + explicit Ready(std::shared_ptr<State> state) : state(std::move(state)), bases(&this->state->allocator) + { + added.context = removed.context = this; + } + + std::shared_ptr<State> state; + State::Bases bases; + PageCacheKey key; + State::Command added{.kind = State::Kind::Ready}; + State::Command removed{.kind = State::Kind::Evict}; + std::shared_ptr<Ready> added_owner; + std::shared_ptr<Ready> removed_owner; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_residency_prepare.cpp b/lib/crowdb-tree/src/btree/page_cache_residency_prepare.cpp new file mode 100644 index 000000000..15050fc20 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_residency_prepare.cpp @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "page_cache_residency_internal.h" +#include "published_state.h" + +namespace crowdb::tree +{ +void PageCacheResidency::State::prepare_page(Update &update, uint64_t pid, + const std::shared_ptr<const PublishedPage> &replacement, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + std::shared_ptr<Page> page; + if (replacement) { + page = detail::reserved_shared<Page>(metadata, metadata); + const auto add = [&](const BaseRef &base, PageCacheKind kind) { + if (!base.backing || base.resident) { + return; + } + const auto key = page_cache_key(kind, base); + page->links.push_back({.key = key}); + if (!update.bases.contains(key)) { + update.bases.emplace(key, detail::reserved_shared<Base>(metadata)); + } + }; + if (replacement->leaf) { + page->links.reserve(replacement->leaf->bases.size()); + for (const auto &base : replacement->leaf->bases) { + add(base, PageCacheKind::Leaf); + } + } + else if (!replacement->inner && replacement->inner_backing) { + page->links.reserve(1); + add(BaseRef{.backing = replacement->inner_backing}, PageCacheKind::Inner); + } + } + update.pages.emplace(pid, std::move(page)); +} + +std::shared_ptr<PageCacheResidency::Update> +PageCacheResidency::prepare_initial(const MappingView &mapping, + const std::shared_ptr<detail::AllocationReservation> &metadata) const +{ + auto update = detail::reserved_shared<Update>(metadata, state_); + PageCacheNodeResource::Scope scope(metadata); + + struct Context + { + std::shared_ptr<Update> update; + const std::shared_ptr<detail::AllocationReservation> &metadata; + } context{.update = update, .metadata = metadata}; + + mapping.visit(&context, [](void *argument, uint64_t pid, const auto &page) { + auto &self = *static_cast<Context *>(argument); + if (self.update->pages.size() == 32) { + self.update->following = detail::reserved_shared<Update>(self.metadata, self.update->state); + self.update = self.update->following; + } + State::prepare_page(*self.update, pid, page, self.metadata); + }); + return update; +} + +std::shared_ptr<PageCacheResidency::Update> +PageCacheResidency::prepare_replacement(const MappingView &mapping, + const std::shared_ptr<detail::AllocationReservation> &metadata) const +{ + auto update = prepare_initial(mapping, metadata); + update->reset = true; + return update; +} + +size_t PageCacheResidency::initial_allocation_bound(const MappingView &mapping) noexcept +{ + detail::AllocationBound bound(detail::shared_allocation_bound<Update>()); + mapping.visit(&bound, [](void *argument, uint64_t, const auto &page) { + auto &total = *static_cast<detail::AllocationBound *>(argument); + const auto refs = page->leaf ? page->leaf->bases.size() : size_t{1}; + total.add(page_allocation_bound(refs)); + }); + return bound.value().value_or(SIZE_MAX); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_cache_trim.cpp b/lib/crowdb-tree/src/btree/page_cache_trim.cpp new file mode 100644 index 000000000..4fa2ed700 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_cache_trim.cpp @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_cache_internal.h" + +namespace crowdb::tree +{ +void SharedPageCache::trim(PageCacheTrim &request) noexcept +{ + auto *head = state_->trims.load(std::memory_order_relaxed); + do { + request.next = head; + } while ( + !state_->trims.compare_exchange_weak(head, &request, std::memory_order_release, std::memory_order_relaxed)); + state_->wake(); +} + +size_t SharedPageCache::State::trim_ready(PageCacheKind kind, size_t max_pages) noexcept +{ + size_t retained = 0; + for (const auto &entry : entries) { + retained += static_cast<size_t>(entry.phase == Phase::Ready && entry.key.kind == kind); + } + size_t evicted = 0; + while (retained > max_pages) { + Entry *oldest = nullptr; + for (auto &entry : entries) { + if (entry.phase == Phase::Ready && entry.key.kind == kind && + (oldest == nullptr || entry.priority > oldest->priority || + (entry.priority == oldest->priority && entry.touched < oldest->touched))) { + oldest = &entry; + } + } + if (oldest == nullptr) { + break; + } + discard(*oldest); + --retained; + ++evicted; + } + resume_requested = evicted != 0 || resume_requested; + return evicted; +} + +void SharedPageCache::State::drain_trims() noexcept +{ + auto *list = trims.exchange(nullptr, std::memory_order_acquire); + PageCacheTrim *ordered = nullptr; + while (list != nullptr) { + auto *next = list->next; + list->next = ordered; + ordered = list; + list = next; + } + while (ordered != nullptr) { + auto *request = ordered; + ordered = request->next; + const auto evicted = trim_ready(request->kind, request->max_pages); + if (request->complete != nullptr) { + request->complete(request->context, evicted); + } + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_encode.cpp b/lib/crowdb-tree/src/btree/page_encode.cpp new file mode 100644 index 000000000..7a983569a --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_encode.cpp @@ -0,0 +1,178 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_encode_internal.h" +#include "btree/tree_metrics.h" + +#include <algorithm> +#include <new> + +namespace crowdb::tree +{ +FinalPageEncoder::FinalPageEncoder(std::shared_ptr<detail::FinalPageEncodeState> state) noexcept + : state_(std::move(state)) +{ +} + +FinalPageEncoder::~FinalPageEncoder() = default; + +const FinalPageSummary &FinalPageEncoder::summary() const noexcept +{ + return state_->summary; +} + +const PageEncodeCounters &FinalPageEncoder::counters() const noexcept +{ + return state_->counters; +} + +std::span<const ReusedOverflowDependency> FinalPageEncoder::reused_overflow() const noexcept +{ + return state_->reused; +} + +std::shared_ptr<const void> FinalPageEncoder::dependency_owner() const noexcept +{ + return state_; +} + +Status FinalPageEncoder::bind_overflow(ReservedOverflowIds ids) +{ + const auto needed = state_->summary.new_overflow_frames; + if (!state_->prepared) { + return Status::unavailable("overflow identities require a complete leaf preparation"); + } + if (state_->counters.primary_frames != 0 || state_->counters.overflow_frames != 0) { + return Status::invalid_argument("overflow identities cannot change after encoding starts"); + } + if (ids.count != needed || + (needed != 0 && (!ids.owner || ids.first < state_->next_page_id || ids.first >= MappingView::kPageLimit || + needed > MappingView::kPageLimit - ids.first))) { + return Status::invalid_argument("overflow interval must be reserved, owned and outside captured PIDs"); + } + state_->ids = std::move(ids); + return Status::Ok(); +} + +PageEncodeStep FinalPageEncoder::next(std::span<uint8_t> output) +{ + if (state_->complete) { + return {.status = Status::Ok(), .complete = true}; + } + TreeMetricsOperation metrics(state_->limits.work_metrics, TreeStage::PageEncode, tree_metrics_now_ns()); + try { + const auto before = state_->counters; + auto result = next_frame(output); + const auto &after = state_->counters; + const auto &work = state_->limits.work_metrics; + const auto cause = state_->limits.cause; + record_work(work, TreeWorkCounter::EncodedPrimaryFrames, after.primary_frames - before.primary_frames, cause); + record_work(work, TreeWorkCounter::EncodedOverflowFrames, after.overflow_frames - before.overflow_frames, + cause); + record_work(work, TreeWorkCounter::EncodingKeyCopyBytes, after.key_copy_bytes - before.key_copy_bytes, cause); + record_work(work, TreeWorkCounter::EncodingValueCopyBytes, + after.inline_value_copy_bytes - before.inline_value_copy_bytes + after.overflow_value_copy_bytes - + before.overflow_value_copy_bytes, + cause); + if (result.frame) { + record_work(work, TreeWorkCounter::EncodedFrameBytes, result.frame->bytes, cause); + } + (void)metrics.finish(result.status.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + return result; + } + catch (...) { + (void)metrics.finish(TreeOutcome::Error, tree_metrics_now_ns()); + throw; + } +} + +PageEncodeStep FinalPageEncoder::next_frame(std::span<uint8_t> output) +{ + auto &state = *state_; + if (!state.prepared) { + return {.status = Status::unavailable("final leaf preparation is waiting for a physical frame")}; + } + if (output.size() < state.limits.frame_bytes) { + return {.status = Status::invalid_argument("final frame output window is too small")}; + } + if (state.summary.new_overflow_frames != 0 && !state.ids.owner) { + return {.status = Status::unavailable("final page needs a reserved overflow identity interval")}; + } + while (state.overflow_entry < state.entries.size() && state.entries[state.overflow_entry].overflow_frames == 0) { + ++state.overflow_entry; + } + if (state.overflow_entry < state.entries.size()) { + const auto &entry = state.entries[state.overflow_entry]; + const uint64_t cap = overflow_chunk_cap(state.limits.frame_bytes); + const uint64_t offset = state.overflow_index * cap; + const auto bytes = static_cast<uint32_t>(std::min<uint64_t>(cap, entry.cell.value.size() - offset)); + const uint64_t pid = state.ids.first + entry.overflow_offset + state.overflow_index; + const uint64_t next = state.overflow_index + 1 == entry.overflow_frames ? kInvalidPageId : pid + 1; + overflow_frame_build(output.data(), state.limits.frame_bytes, pid, next, entry.cell.value.bytes() + offset, + bytes); + ++state.counters.overflow_frames; + state.counters.overflow_value_copy_bytes += bytes; + if (++state.overflow_index == entry.overflow_frames) { + ++state.overflow_entry; + state.overflow_index = 0; + } + return { + .status = Status::Ok(), + .frame = EncodedFrameInfo{.page_id = pid, + .type = page_type::kOverflowFrame, + .bytes = state.limits.frame_bytes, + .dependencies = state.ids.owner} + }; + } + detail::encode_primary(state, output.data()); + ++state.counters.primary_frames; + state.complete = true; + return { + .status = Status::Ok(), + .frame = EncodedFrameInfo{.page_id = state.pid, + .type = state.page->level == 0 ? page_type::kLeafBase : page_type::kInnerBase, + .bytes = state.limits.frame_bytes, + .dependencies = state_}, + .complete = true + }; +} + +PageEncodePreparation prepare_final_page(const std::shared_ptr<const PublishedTreeState> &captured, uint64_t pid, + const CapturedEncodeContext &context, const PageEncodeLimits &limits) +{ + PageEncodePreparation result; + const size_t lower_bytes = context.range.lower.has_value() ? context.range.lower->size() : 0; + const size_t upper_bytes = context.range.upper.has_value() ? context.range.upper->size() : 0; + if (!captured || pid >= MappingView::kPageLimit || + (context.range.lower && context.range.upper && *context.range.lower > *context.range.upper) || + limits.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || limits.max_sources == 0 || + limits.max_entries == 0 || limits.max_work_entries == 0 || captured->gc_floor > captured->coverage || + lower_bytes > limits.max_metadata_bytes || upper_bytes > limits.max_metadata_bytes - lower_bytes) { + result.status = Status::invalid_argument("invalid captured page encoding bounds"); + return result; + } + try { + auto state = detail::reserved_shared<detail::FinalPageEncodeState>(limits.metadata, limits); + state->page = captured->mapping.get(pid); + state->pid = pid; + state->next_page_id = captured->next_page_id; + if (!state->page) { + result.status = Status::invalid_argument("encoded PID is absent from the captured mapping"); + return result; + } + result.status = state->page->level == 0 ? detail::prepare_leaf(*state, *captured, context, &result) + : detail::prepare_inner(*state, *captured, context, &result); + if (result.status.ok() && !result.needs_base && !result.needs_inner) { + detail::ReservationAllocator<FinalPageEncoder> allocator(limits.metadata); + auto *storage = allocator.allocate(1); + ::new (static_cast<void *>(storage)) FinalPageEncoder(std::move(state)); + result.encoder = detail::ReservedUnique<FinalPageEncoder>( + storage, detail::ReservationDeleter<FinalPageEncoder>{limits.metadata}); + } + } + catch (const std::bad_alloc &) { + result.status = Status::resource_exhausted(); + result.encoder.reset(); + } + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_encode.h b/lib/crowdb-tree/src/btree/page_encode.h new file mode 100644 index 000000000..1148ca967 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_encode.h @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_maintenance.h" +#include "btree/published_state.h" + +#include <span> + +namespace crowdb::tree +{ +struct PageEncodeLimits +{ + uint32_t frame_bytes = 64 * 1024; + size_t inline_value_bytes = 8 * 1024; + size_t max_sources = 64; + size_t max_entries = 4096; + size_t max_work_entries = 65536; + size_t max_metadata_bytes = 1024 * 1024; + std::shared_ptr<detail::AllocationReservation> metadata = nullptr; + std::shared_ptr<TreeWorkMetrics> work_metrics = nullptr; + TreeWorkCause cause = TreeWorkCause::Other; +}; + +struct InnerEncodeFences +{ + uint64_t lower_leaf = kInvalidPageId; + uint64_t upper_leaf = kInvalidPageId; +}; + +struct CapturedEncodeContext +{ + // Half-open routing bounds supplied by the captured traversal. Durable + // leaf fences instead contain the actual inclusive first/last output key. + PageKeyRange range{}; + // Required for an inner page. Both invalid explicitly denotes an empty + // subtree; otherwise these are captured descendants from the traversal. + std::optional<InnerEncodeFences> inner_fences = std::nullopt; + // Owns the captured backend mapping/extents for pre-existing overflow + // references. A resident leaf frame alone does not protect its chain. + std::shared_ptr<const void> overflow_dependencies = nullptr; +}; + +struct ReservedOverflowIds +{ + uint64_t first = kInvalidPageId; + uint64_t count = 0; + // The host reserves this interval independently of live page allocation + // and retains its lifetime/protection through writes and checkpoint commit. + std::shared_ptr<const void> owner = nullptr; +}; + +struct ReusedOverflowDependency +{ + uint64_t head = kInvalidPageId; + uint64_t bytes = 0; + std::shared_ptr<const void> owner; + std::shared_ptr<const OverflowBackingSet> backing = nullptr; +}; + +struct FinalPageSummary +{ + // Inner subtree totals require the caller's captured child summaries. + // Their zero-initialized live fields are unknown, not an empty subtree. + std::optional<InnerEncodeFences> inner_fences = std::nullopt; + bool live_counts_known = false; + uint64_t encoded_entries = 0; + uint64_t live_keys = 0; + uint64_t live_key_bytes = 0; + uint64_t live_value_bytes = 0; + uint64_t dropped_tombstones = 0; + uint64_t primary_used_bytes = 0; + uint64_t new_overflow_frames = 0; +}; + +struct PageEncodeCounters +{ + uint64_t primary_frames = 0; + uint64_t overflow_frames = 0; + uint64_t key_copy_bytes = 0; + uint64_t inline_value_copy_bytes = 0; + uint64_t overflow_value_copy_bytes = 0; +}; + +struct EncodedFrameInfo +{ + uint64_t page_id = kInvalidPageId; + page_type type = page_type::kLeafBase; + uint32_t bytes = 0; + // Carry into the asynchronous write and staged backing: bytes alone do + // not protect the overflow references embedded in an output frame. + std::shared_ptr<const void> dependencies = nullptr; +}; + +struct PageEncodeStep +{ + Status status; + std::optional<EncodedFrameInfo> frame = std::nullopt; + bool complete = false; +}; + +struct InnerEncodeLoad +{ + uint64_t page_id = kInvalidPageId; + PageKeyRange range; + std::shared_ptr<const BackingRef> backing; +}; + +namespace detail +{ +struct FinalPageEncodeState; +} +class FinalPageEncoder; + +struct PageEncodePreparation +{ + Status status; + std::optional<BaseLoadRequest> needs_base; + std::optional<InnerEncodeLoad> needs_inner; + detail::ReservedUnique<FinalPageEncoder> encoder; + // A layout capacity failure, distinct from metadata admission failure. + bool needs_split = false; +}; + +// One encoder owns one captured page. It has one consumer and never performs +// I/O, consults a live mapping, or allocates output buffers. Each next() fills +// at most one caller-owned final frame; the caller controls in-flight windows. +class FinalPageEncoder +{ + public: + ~FinalPageEncoder(); + FinalPageEncoder(const FinalPageEncoder &) = delete; + FinalPageEncoder &operator=(const FinalPageEncoder &) = delete; + [[nodiscard]] const FinalPageSummary &summary() const noexcept; + [[nodiscard]] const PageEncodeCounters &counters() const noexcept; + [[nodiscard]] std::span<const ReusedOverflowDependency> reused_overflow() const noexcept; + [[nodiscard]] std::shared_ptr<const void> dependency_owner() const noexcept; + [[nodiscard]] Status bind_overflow(ReservedOverflowIds ids); + [[nodiscard]] PageEncodeStep next(std::span<uint8_t> output); + + private: + friend PageEncodePreparation prepare_final_page(const std::shared_ptr<const PublishedTreeState> &, uint64_t, + const CapturedEncodeContext &, const PageEncodeLimits &); + explicit FinalPageEncoder(std::shared_ptr<detail::FinalPageEncodeState> state) noexcept; + [[nodiscard]] PageEncodeStep next_frame(std::span<uint8_t> output); + std::shared_ptr<detail::FinalPageEncodeState> state_; +}; + +// Preparation borrows values and bounds descriptor work. A cold input is +// returned before any output. A physical range request includes an encoder +// continuation; resume it instead of restarting and rescanning the prefix. +// Ordinary cold bases still use a privately loaded descriptor and retry. +// Oversized primary layouts require structural maintenance first. +PageEncodePreparation prepare_final_page(const std::shared_ptr<const PublishedTreeState> &captured, uint64_t pid, + const CapturedEncodeContext &context, const PageEncodeLimits &limits = {}); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_encode_frame.cpp b/lib/crowdb-tree/src/btree/page_encode_frame.cpp new file mode 100644 index 000000000..3887a2de9 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_encode_frame.cpp @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_encode_internal.h" + +#include <algorithm> +#include <cstring> + +namespace crowdb::tree::detail +{ +namespace +{ +void encode_leaf(FinalPageEncodeState &state, uint8_t *output) +{ + const auto bytes = state.limits.frame_bytes; + frame_put_u32(output, fh::kMagic, kFrameMagicLeaf); + output[fh::kType] = static_cast<uint8_t>(page_type::kLeafBase); + frame_put_u64(output, fh::kRightSibling, state.page->leaf->right_sibling); + uint32_t hi = bytes - static_cast<uint32_t>(kFrameTrailerSize); + for (size_t index = 0; index < state.entries.size(); ++index) { + const auto &entry = state.entries[index]; + const auto key_bytes = static_cast<uint32_t>(entry.key.size()); + hi -= key_bytes + entry.encoded_cell_bytes; + if (key_bytes != 0) { + std::memcpy(output + hi, entry.key.data(), key_bytes); + } + state.counters.key_copy_bytes += key_bytes; + auto *cell = output + hi + key_bytes; + frame_put_u64(cell, 0, entry.cell.slot); + cell[sizeof(uint64_t)] = entry.cell.flags; + if (entry.overflow_frames != 0) { + cell[sizeof(uint64_t)] = kFlagOverflow; + frame_put_u64(cell, kCellHeaderSize, state.ids.first + entry.overflow_offset); + frame_put_u64(cell, kCellHeaderSize + sizeof(uint64_t), entry.cell.value.size()); + } + else if (entry.cell.overflow()) { + std::memcpy(cell + kCellHeaderSize, entry.cell.encoded.data() + kCellHeaderSize, + kOverflowCellSize - kCellHeaderSize); + } + else if (!entry.cell.value.empty()) { + std::memcpy(cell + kCellHeaderSize, entry.cell.value.data(), entry.cell.value.size()); + state.counters.inline_value_copy_bytes += entry.cell.value.size(); + } + auto *slot = output + kFrameHeaderSize + (index * kLeafSlotSize); + frame_put_u32(slot, 0, hi); + frame_put_u32(slot, sizeof(uint32_t), key_bytes); + frame_put_u32(slot, 2 * sizeof(uint32_t), entry.encoded_cell_bytes); + } + frame_put_u32(output, fh::kSlotCount, static_cast<uint32_t>(state.entries.size())); + frame_put_u32(output, fh::kFreeLo, + static_cast<uint32_t>(kFrameHeaderSize + (state.entries.size() * kLeafSlotSize))); + frame_put_u32(output, fh::kFreeHi, hi); + if (!state.entries.empty()) { + LeafFrameView view(output, bytes); + const auto lower = view.key(0); + const auto upper = view.key(view.count() - 1); + (void)frame_set_fences(output, bytes, &lower, &upper); + } + else { + frame_restamp_crc(output, bytes); + } +} + +void encode_inner(FinalPageEncodeState &state, uint8_t *output) +{ + const auto &inner = *state.page->inner; + frame_put_u32(output, fh::kMagic, kFrameMagicInner); + output[fh::kType] = static_cast<uint8_t>(page_type::kInnerBase); + auto lo = static_cast<uint32_t>(kFrameHeaderSize); + for (const auto child : inner.children) { + frame_put_u64(output, lo, child); + lo += sizeof(uint64_t); + } + uint32_t hi = state.limits.frame_bytes - static_cast<uint32_t>(kFrameTrailerSize); + for (const auto &separator : inner.separators) { + hi -= static_cast<uint32_t>(separator.size()); + if (separator.size() != 0) { + std::copy_n(separator.data(), separator.size(), output + hi); + } + state.counters.key_copy_bytes += separator.size(); + frame_put_u32(output, lo, hi); + frame_put_u32(output, lo + sizeof(uint32_t), static_cast<uint32_t>(separator.size())); + lo += kInnerSlotSize; + } + frame_put_u32(output, fh::kSlotCount, static_cast<uint32_t>(inner.separators.size())); + frame_put_u32(output, fh::kFreeLo, lo); + frame_put_u32(output, fh::kFreeHi, hi); + frame_set_inner_fence_pages(output, state.limits.frame_bytes, state.fences.lower_leaf, state.fences.upper_leaf); +} +} // namespace + +void encode_primary(FinalPageEncodeState &state, uint8_t *output) +{ + std::memset(output, 0, state.limits.frame_bytes); + output[fh::kFormatVersion] = static_cast<uint8_t>(kFrameVersion); + frame_put_u64(output, fh::kSelfpage_id, state.pid); + if (state.page->level == 0) { + encode_leaf(state, output); + } + else { + encode_inner(state, output); + } +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/page_encode_internal.h b/lib/crowdb-tree/src/btree/page_encode_internal.h new file mode 100644 index 000000000..15c12abe1 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_encode_internal.h @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_encode.h" + +namespace crowdb::tree::detail +{ +struct FinalLeafEntry +{ + Slice key; + LogicalCell cell; + uint64_t overflow_offset = 0; + uint64_t overflow_frames = 0; + uint32_t encoded_cell_bytes = 0; +}; + +struct FinalPageEncodeState +{ + explicit FinalPageEncodeState(const PageEncodeLimits &limits) + : entries(ReservationAllocator<FinalLeafEntry>(limits.metadata)), + reused(ReservationAllocator<ReusedOverflowDependency>(limits.metadata)), + limits(limits) + { + } + + std::shared_ptr<const PublishedPage> page; + std::vector<FinalLeafEntry, ReservationAllocator<FinalLeafEntry>> entries; + std::vector<ReusedOverflowDependency, ReservationAllocator<ReusedOverflowDependency>> reused; + PageEncodeLimits limits; + InnerEncodeFences fences; + FinalPageSummary summary; + PageEncodeCounters counters; + ReservedOverflowIds ids; + std::optional<LogicalPageCursor> cursor; + CapturedEncodeContext context; + uint64_t gc_floor = 0; + size_t visited = 0; + uint64_t pid = kInvalidPageId; + uint64_t next_page_id = 0; + size_t overflow_entry = 0; + uint64_t overflow_index = 0; + bool complete = false; + bool prepared = false; +}; + +Status prepare_leaf(FinalPageEncodeState &state, const PublishedTreeState &captured, + const CapturedEncodeContext &context, PageEncodePreparation *result); +Status advance_leaf(FinalPageEncodeState &state, PageEncodePreparation *result); +Status prepare_inner(FinalPageEncodeState &state, const PublishedTreeState &captured, + const CapturedEncodeContext &context, PageEncodePreparation *result); +void encode_primary(FinalPageEncodeState &state, uint8_t *output); +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/page_encode_leaf.cpp b/lib/crowdb-tree/src/btree/page_encode_leaf.cpp new file mode 100644 index 000000000..18c4346e2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_encode_leaf.cpp @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_encode_internal.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +namespace +{ +constexpr uint64_t kPageOverhead = kFrameHeaderSize + kFrameTrailerSize; + +Status append_leaf_entry(FinalPageEncodeState &state, Slice key, LogicalCell cell, const CapturedEncodeContext &context, + PageEncodePreparation *result) +{ + if (!cell.owner || (cell.flags & ~(kFlagTombstone | kFlagOverflow)) != 0 || + (cell.tombstone() && (cell.overflow() || !cell.value.empty()))) { + return Status::corruption("final leaf has an invalid or unowned logical cell"); + } + const bool spill = !cell.tombstone() && (cell.overflow() || cell.value.size() > state.limits.inline_value_bytes); + if (!spill && cell.value.size() > state.limits.frame_bytes) { + return Status::resource_exhausted("inline value cannot fit the final primary frame"); + } + const uint64_t cell_bytes = spill ? kOverflowCellSize : kCellHeaderSize + cell.value.size(); + const uint64_t available = state.limits.frame_bytes - state.summary.primary_used_bytes; + if (key.size() > available || cell_bytes > available - key.size() || + kLeafSlotSize > available - key.size() - cell_bytes) { + result->needs_split = true; + return Status::resource_exhausted("captured primary page needs structural split before encoding"); + } + if (state.entries.size() == state.entries.capacity()) { + return Status::resource_exhausted("final leaf descriptor budget exhausted"); + } + FinalLeafEntry entry{.key = key, + .cell = std::move(cell), + .overflow_offset = state.summary.new_overflow_frames, + .encoded_cell_bytes = static_cast<uint32_t>(cell_bytes)}; + uint64_t value_bytes = entry.cell.value.size(); + if (entry.cell.overflow()) { + const auto captured_backing = entry.cell.backing ? entry.cell.backing->overflow : nullptr; + if (entry.cell.encoded.size() != kOverflowCellSize || (!captured_backing && !context.overflow_dependencies)) { + return Status::invalid_argument("reused overflow needs its captured backend dependency owner"); + } + const CellView overflow(entry.cell.encoded); + value_bytes = overflow.overflow_len(); + if (overflow.overflow_head() >= MappingView::kPageLimit || value_bytes == 0) { + return Status::corruption("invalid captured overflow pointer"); + } + state.reused.push_back({.head = overflow.overflow_head(), + .bytes = value_bytes, + .owner = captured_backing ? nullptr : context.overflow_dependencies, + .backing = captured_backing}); + } + else if (spill) { + const uint64_t cap = overflow_chunk_cap(state.limits.frame_bytes); + entry.overflow_frames = (value_bytes / cap) + static_cast<uint64_t>(value_bytes % cap != 0); + if (entry.overflow_frames > MappingView::kPageLimit - state.summary.new_overflow_frames) { + return Status::resource_exhausted("encoded overflow identity count exceeds the namespace"); + } + state.summary.new_overflow_frames += entry.overflow_frames; + } + if (!entry.cell.tombstone()) { + if (value_bytes > UINT64_MAX - state.summary.live_value_bytes || + key.size() > UINT64_MAX - state.summary.live_key_bytes) { + return Status::resource_exhausted("final page logical summary overflow"); + } + ++state.summary.live_keys; + state.summary.live_key_bytes += key.size(); + state.summary.live_value_bytes += value_bytes; + } + state.summary.primary_used_bytes += key.size() + cell_bytes + kLeafSlotSize; + ++state.summary.encoded_entries; + state.entries.push_back(std::move(entry)); + return Status::Ok(); +} +} // namespace + +Status prepare_leaf(FinalPageEncodeState &state, const PublishedTreeState &captured, + const CapturedEncodeContext &context, PageEncodePreparation *result) +{ + const auto &page = state.page->leaf; + if (!page || state.page->inner || page->range.lower != context.range.lower || + page->range.upper != context.range.upper) { + return Status::invalid_argument("final leaf must match its captured routing range"); + } + state.context = context; + state.gc_floor = captured.gc_floor; + state.cursor.emplace(page, context.range, captured.coverage, state.limits.max_sources, false, + state.limits.metadata); + const size_t capacity = + std::min({state.limits.max_entries, + static_cast<size_t>((state.limits.frame_bytes - kPageOverhead) / (kLeafSlotSize + kCellHeaderSize)), + state.limits.max_metadata_bytes / (sizeof(FinalLeafEntry) + sizeof(ReusedOverflowDependency))}); + state.entries.reserve(capacity); + state.reused.reserve(capacity); + state.summary.primary_used_bytes = kPageOverhead; + return advance_leaf(state, result); +} + +Status advance_leaf(FinalPageEncodeState &state, PageEncodePreparation *result) +{ + if (!state.cursor) { + return Status::invalid_argument("leaf encoder has no preparation cursor"); + } + auto &cursor = *state.cursor; + if (!cursor.status().ok()) { + return cursor.status(); + } + while (cursor.valid()) { + if (++state.visited > state.limits.max_work_entries) { + return Status::resource_exhausted("final leaf winner work budget exhausted"); + } + auto cell = cursor.cell(); + if (cell.tombstone() && cell.slot <= state.gc_floor) { + ++state.summary.dropped_tombstones; + } + else { + auto status = append_leaf_entry(state, cursor.key(), std::move(cell), state.context, result); + if (!status.ok()) { + return status; + } + } + cursor.next(); + } + if (!cursor.status().ok()) { + return cursor.status(); + } + if (const auto index = cursor.needs_base()) { + result->needs_base = {.page = state.page->leaf, .base_index = *index}; + return Status::Ok(); + } + state.prepared = true; + state.cursor.reset(); + state.context = {}; + state.summary.live_counts_known = true; + return Status::Ok(); +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/page_encode_plan.cpp b/lib/crowdb-tree/src/btree/page_encode_plan.cpp new file mode 100644 index 000000000..3fe4e8113 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_encode_plan.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_encode_internal.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +Status prepare_inner(FinalPageEncodeState &state, const PublishedTreeState &captured, + const CapturedEncodeContext &context, PageEncodePreparation *result) +{ + constexpr uint64_t kPageOverhead = kFrameHeaderSize + kFrameTrailerSize; + if (state.page->leaf) { + return Status::corruption("captured inner page has a leaf representation"); + } + if (!state.page->inner) { + if (!state.page->inner_backing || !state.page->inner_backing->readable()) { + return Status::corruption("cold inner has no captured reloadable backing"); + } + result->needs_inner = {.page_id = state.pid, .range = context.range, .backing = state.page->inner_backing}; + return Status::Ok(); + } + const auto &inner = *state.page->inner; + if (state.page->leaf || inner.children.empty() || inner.children.size() != inner.separators.size() + 1 || + inner.children.size() > state.limits.max_entries || inner.children.size() > state.limits.max_work_entries || + !context.inner_fences.has_value()) { + return Status::invalid_argument("final inner shape, child bound or captured fences are invalid"); + } + if (inner.children.size() > (state.limits.frame_bytes - kPageOverhead) / sizeof(uint64_t)) { + return Status::resource_exhausted("captured inner page needs structural split before encoding"); + } + uint64_t used = kPageOverhead + (inner.children.size() * sizeof(uint64_t)); + for (size_t index = 0; index < inner.separators.size(); ++index) { + const auto &key = inner.separators[index]; + if ((context.range.lower && key < *context.range.lower) || + (context.range.upper && key > *context.range.upper) || (index != 0 && inner.separators[index - 1] >= key)) { + return Status::corruption("inner separators escape captured routing bounds or order"); + } + if (key.size() > state.limits.frame_bytes - used || + kInnerSlotSize > state.limits.frame_bytes - used - key.size()) { + return Status::resource_exhausted("captured inner page needs structural split before encoding"); + } + used += key.size() + kInnerSlotSize; + } + for (const auto pid : inner.children) { + const auto child = captured.mapping.get(pid); + if (!child || child->level + uint64_t{1} != state.page->level) { + return Status::corruption("inner child is absent or has the wrong level in captured mapping"); + } + } + const auto fences = context.inner_fences.value(); + if ((fences.lower_leaf == kInvalidPageId) != (fences.upper_leaf == kInvalidPageId)) { + return Status::invalid_argument("inner encoding fences must be a complete pair"); + } + if (fences.lower_leaf != kInvalidPageId) { + for (const auto pid : {fences.lower_leaf, fences.upper_leaf}) { + const auto leaf = captured.mapping.get(pid); + if (!leaf || !leaf->leaf || leaf->level != 0 || !leaf->leaf->range.overlaps(context.range)) { + return Status::invalid_argument("inner fence leaf is absent from captured range"); + } + } + } + state.prepared = true; + state.summary.inner_fences = fences; + state.fences = fences; + state.summary.primary_used_bytes = used; + state.summary.encoded_entries = inner.separators.size(); + return Status::Ok(); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/page_key_range.h b/lib/crowdb-tree/src/btree/page_key_range.h new file mode 100644 index 000000000..41d4d639c --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_key_range.h @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "owned_key.h" + +#include <optional> + +namespace crowdb::tree +{ +// An absent bound is unbounded; an empty string is a real key boundary. +struct PageKeyRange +{ + std::optional<OwnedKey> lower = std::nullopt; + std::optional<OwnedKey> upper = std::nullopt; + + [[nodiscard]] bool contains(Slice key) const; + [[nodiscard]] bool empty() const; + [[nodiscard]] bool overlaps(const PageKeyRange &other) const; + [[nodiscard]] std::optional<Slice> lower_slice() const; + [[nodiscard]] std::optional<Slice> upper_slice() const; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_layout.cpp b/lib/crowdb-tree/src/btree/page_layout.cpp new file mode 100644 index 000000000..6b742b86d --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_layout.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_maintenance.h" + +#include <limits> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kPageOverhead = kFrameHeaderSize + kFrameTrailerSize; + +Status entry_bytes(Slice key, const LogicalCell &cell, const PageMaintenanceLimits &limits, size_t *bytes, + bool *overflow) +{ + *overflow = !cell.tombstone() && (cell.overflow() || cell.value.size() > limits.inline_value_bytes); + const size_t cell_bytes = *overflow ? kOverflowCellSize : kCellHeaderSize + cell.value.size(); + if (key.size() > limits.primary_page_bytes || cell_bytes > limits.primary_page_bytes || + key.size() > limits.primary_page_bytes - cell_bytes || + key.size() + cell_bytes > limits.primary_page_bytes - kPageOverhead - kLeafSlotSize) { + return Status::invalid_argument("one key/cell cannot fit an uncompressed primary leaf frame"); + } + *bytes = key.size() + cell_bytes + kLeafSlotSize; + return Status::Ok(); +} +} // namespace + +LeafLayoutPlan plan_leaf_layout(std::shared_ptr<const PageState> page, const PageMaintenanceLimits &limits) +{ + LeafLayoutPlan result(limits.metadata); + result.source = std::move(page); + if (result.source == nullptr || limits.primary_page_bytes <= kPageOverhead + kLeafSlotSize || + limits.primary_page_bytes > UINT32_MAX || limits.target_page_bytes <= kPageOverhead || + limits.target_page_bytes > limits.primary_page_bytes || limits.max_output_pages == 0) { + result.status = Status::invalid_argument("invalid leaf layout input or capacity"); + return result; + } + LogicalPageCursor cursor(result.source, result.source->range, UINT64_MAX, limits.max_sources, false, + limits.metadata); + if (!cursor.status().ok()) { + result.status = cursor.status(); + result.readiness = MaintenanceReadiness::kDeferred; + return result; + } + if (cursor.needs_base().has_value()) { + result.readiness = MaintenanceReadiness::kNeedsBase; + result.loads.push_back({.page = result.source, .base_index = *cursor.needs_base()}); + return result; + } + size_t current_bytes = kPageOverhead; + size_t separator_bytes = 0; + bool split_budget_exhausted = false; + result.total_primary_bytes = kPageOverhead; + while (cursor.valid()) { + if (result.entry_count >= limits.max_work_entries) { + result.status = Status::resource_exhausted("leaf layout work budget exhausted"); + break; + } + size_t bytes = 0; + bool overflow = false; + result.status = entry_bytes(cursor.key(), cursor.cell(), limits, &bytes, &overflow); + if (!result.status.ok()) { + break; + } + if (current_bytes != kPageOverhead && + bytes > limits.target_page_bytes - std::min(current_bytes, limits.target_page_bytes)) { + if (result.primary_bytes.size() + 1 >= limits.max_output_pages || + cursor.key().size() > limits.max_metadata_bytes - separator_bytes) { + split_budget_exhausted = true; + } + if (!split_budget_exhausted) { + result.primary_bytes.push_back(current_bytes); + result.separators.emplace_back(cursor.key(), limits.metadata); + separator_bytes += cursor.key().size(); + } + current_bytes = kPageOverhead; + } + if (bytes > std::numeric_limits<size_t>::max() - result.total_primary_bytes) { + result.status = Status::resource_exhausted("leaf layout size overflow"); + break; + } + current_bytes += bytes; + result.total_primary_bytes += bytes; + ++result.entry_count; + result.overflow_values += static_cast<size_t>(overflow); + cursor.next(); + } + if (result.status.ok() && !cursor.status().ok()) { + result.status = cursor.status(); + } + const auto pending_base = cursor.needs_base(); + if (result.status.ok() && pending_base) { + result.readiness = MaintenanceReadiness::kNeedsBase; + result.loads.push_back({.page = result.source, .base_index = *pending_base}); + result.primary_bytes.clear(); + result.separators.clear(); + return result; + } + if (result.status.ok() && split_budget_exhausted && result.total_primary_bytes > limits.primary_page_bytes) { + result.status = Status::resource_exhausted("leaf split output/metadata budget exhausted"); + } + if (!result.status.ok()) { + result.readiness = MaintenanceReadiness::kDeferred; + result.primary_bytes.clear(); + result.separators.clear(); + return result; + } + result.primary_bytes.push_back(current_bytes); + if (result.total_primary_bytes <= limits.primary_page_bytes) { + result.separators.clear(); + result.primary_bytes = {result.total_primary_bytes}; + } + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_layout_stats.cpp b/lib/crowdb-tree/src/btree/page_layout_stats.cpp new file mode 100644 index 000000000..a64a7172f --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_layout_stats.cpp @@ -0,0 +1,24 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "page_maintenance.h" + +namespace crowdb::tree +{ +Status accumulate_leaf_stats(Slice key, const LogicalCell &cell, LeafLayoutStats *stats) +{ + if (cell.tombstone()) { + return Status::Ok(); + } + if (cell.overflow() && cell.encoded.size() < kOverflowCellSize) { + return Status::corruption("leaf layout found an invalid overflow cell"); + } + const uint64_t bytes = cell.overflow() ? CellView(cell.encoded).overflow_len() : cell.value.size(); + if (bytes > UINT64_MAX - key.size() || bytes + key.size() > UINT64_MAX - stats->kv_bytes || + stats->kv_count == UINT64_MAX) { + return Status::resource_exhausted("leaf layout statistics overflow"); + } + ++stats->kv_count; + stats->kv_bytes += bytes + key.size(); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load.cpp b/lib/crowdb-tree/src/btree/page_load.cpp new file mode 100644 index 000000000..6340c7ca3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load.cpp @@ -0,0 +1,197 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/page_load_internal.h" +#include "btree/page_load_leaf.h" +#include "btree/page_load_lifetime.h" + +#include <atomic> +#include <stdexcept> + +namespace crowdb::tree +{ +void PageLoader::load_overflow(std::shared_ptr<const BackingRef> /*primary*/, uint64_t head, uint64_t length, + ValueComplete complete) +{ + load_overflow(head, length, std::move(complete)); +} + +void PageLoader::load_overflow(uint64_t /*head*/, uint64_t /*length*/, ValueComplete complete) +{ + complete(Status::not_supported("captured overflow backing resolver is unavailable"), {}); +} + +namespace +{ +struct LeafLoad : PageLoadProgress, std::enable_shared_from_this<LeafLoad> +{ + LeafLoad(const PageLoadBudget &budget, uint64_t started_at_ns) + : PageLoadProgress(budget, started_at_ns), + metadata(budget.bytes), + blob(detail::ReservationAllocator<uint8_t>(metadata)) + { + } + + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + PageLoadLimits limits; + BaseRef base; + AlignedLoadBytes blob; + PageLoader::LeafComplete complete; + std::shared_ptr<LeafLoad> queued_owner; + std::atomic<bool> terminal{false}; + + void finish(Status status) noexcept + { + if (terminal.exchange(true)) { + return; + } + std::shared_ptr<const LeafBase> leaf; + try { + if (status.ok()) { + status = decode_loaded_leaf({blob.data(), blob.size()}, base, limits, metadata, &leaf); + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted("leaf decode allocation failed"); + } + release_if_idle(); + (void)metrics.finish(status.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + auto callback = std::move(complete); + complete_page_load(callback, std::move(status), std::move(leaf)); + } + + static void decode(void *context) noexcept + { + auto self = std::move(static_cast<LeafLoad *>(context)->queued_owner); + self->finish(Status::Ok()); + } +}; +} // namespace + +std::optional<size_t> AsyncStorePageLoader::primary_reserve_bound(const PageLoadLimits &limits, size_t depth, + size_t workers, size_t retained_leaf_pages, + size_t cached_pages, + std::optional<size_t> cached_inner_pages) noexcept +{ + using detail::AllocationBound; + using detail::shared_allocation_bound; + if (limits.max_inflight == 0 || limits.max_leaf_entries == 0) { + return {}; + } + const auto chunk_window = chunk_load_window_bound(limits); + const auto leaf = decoded_leaf_allocation_bound(limits); + const auto retained = AllocationBound(retained_leaf_pages).add(cached_pages).add(workers).add(2).value(); + const auto operations = AllocationBound(workers).add(1).value(); + const auto decode = AllocationBound() + .add_product(2, limits.max_frame_bytes) + .add_product(limits.max_leaf_entries, sizeof(uint32_t)) + .value(); + if (!leaf || !retained || !operations || !decode || !chunk_window) { + return {}; + } + return AllocationBound() + .add(routing_reserve_bound(limits, depth, workers, cached_inner_pages.value_or(cached_pages))) + .add_product(*retained, *leaf) + .add_product(workers, *decode) + .add_product(*operations, shared_allocation_bound<LeafLoad>() + shared_allocation_bound<PageReadTicket>()) + .add_product(*operations, limits.max_blob_bytes) + .add_product(*operations, *chunk_window) + .add_product(*operations, AlignedLoadBytes::kAlignment - 1) + .value(); +} + +std::optional<size_t> AsyncStorePageLoader::read_reserve_bound(const PageLoadLimits &limits, size_t depth, + size_t workers, size_t retained_leaf_pages, + size_t retained_values, size_t value_bytes, + size_t cached_pages, + std::optional<size_t> cached_inner_pages) noexcept +{ + return detail::AllocationBound() + .add(primary_reserve_bound(limits, depth, workers, retained_leaf_pages, cached_pages, cached_inner_pages)) + .add(overflow_read_allocation_bound(limits, workers, retained_values, value_bytes)) + .value(); +} + +AsyncStorePageLoader::AsyncStorePageLoader(std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageLoadLimits limits, + OverflowBackingResolver overflow, PageLoadBudget budget) +{ + if (static_cast<bool>(budget.bytes) != static_cast<bool>(budget.io) || + (budget.bytes && + ((budget.bytes->category() != detail::BudgetClass::kRouting && + budget.bytes->category() != detail::BudgetClass::kReclaim) || + budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + throw std::invalid_argument("page load needs routing/reclaim and I/O credit from the same account"); + } + state_ = detail::reserved_shared<State>(budget.bytes); + state_->store = std::move(store); + state_->runtime = std::move(runtime); + state_->lane = std::move(lane); + state_->limits = limits; + state_->overflow = std::move(overflow); + state_->budget = std::move(budget); +} + +void AsyncStorePageLoader::load_leaf(BaseRef base, LeafComplete complete) +{ + const auto started_at = tree_metrics_now_ns(); + const auto reject = [&](Status status) { + record_page_load_terminal(state_->budget, started_at, status); + complete(std::move(status), {}); + }; + if (base.resident) { + record_page_load_terminal(state_->budget, started_at, Status::Ok()); + complete(Status::Ok(), std::move(base.resident)); + return; + } + if (!state_->runtime || !state_->lane || !base.valid() || !base.backing || base.backing->encoding != 0 || + (!state_->store && base.backing->storage != BackingStorage::kResidentFrame)) { + reject(Status::invalid_argument("leaf load requires a supported independently owned backing")); + return; + } + if (base.backing->length > state_->limits.max_blob_bytes || !chunk_load_allowed(*base.backing, state_->limits)) { + reject(Status::resource_exhausted("leaf blob exceeds load budget")); + return; + } + if (!acquire_page_load(state_->inflight, state_->limits.max_inflight)) { + reject(Status::resource_exhausted("leaf load concurrency budget exhausted")); + return; + } + std::shared_ptr<LeafLoad> operation; + try { + operation = detail::reserved_shared<LeafLoad>(state_->budget.bytes, state_->budget, started_at); + operation->store = state_->store; + operation->runtime = state_->runtime; + operation->lane = state_->lane; + operation->loader_owner = state_; + operation->inflight = &state_->inflight; + operation->limits = state_->limits; + operation->base = std::move(base); + operation->complete = std::move(complete); + if (operation->base.backing->storage == BackingStorage::kResidentFrame) { + queue_page_load(operation, &LeafLoad::decode); + } + else { + operation->blob.resize(operation->base.backing->length); + submit_page_read(operation, operation->base.backing, operation->blob.data(), operation->blob.size()); + } + } + catch (const std::bad_alloc &) { + if (operation && operation->complete) { + operation->finish(Status::resource_exhausted("leaf load allocation failed")); + } + else { + state_->inflight.fetch_sub(1, std::memory_order_release); + reject(Status::resource_exhausted("leaf load allocation failed")); + } + } + catch (...) { + operation->finish(Status::io_error("leaf read submission threw")); + } +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load.h b/lib/crowdb-tree/src/btree/page_load.h new file mode 100644 index 000000000..55fdbf445 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load.h @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/flush_runtime.h" +#include "btree/inline_function.h" +#include "btree/logical_page.h" +#include "btree/tree_work_counters.h" +#include "crowdb-tree/backend/async_page_store.h" + +#include <functional> + +namespace crowdb::tree +{ +struct InnerPageState; +class TreeWorkMetrics; +class ApplyDispatcher; + +class PageLoader +{ + public: + using InnerComplete = detail::InlineFunction<void(Status, std::shared_ptr<const InnerPageState>)>; + using LeafComplete = detail::InlineFunction<void(Status, std::shared_ptr<const LeafBase>)>; + using ValueComplete = detail::InlineFunction<void(Status, std::shared_ptr<const buffer>)>; + virtual ~PageLoader() = default; + // Exactly one nonthrowing callback, possibly inline. No blocking fallback. + virtual void load_leaf(BaseRef base, LeafComplete complete) = 0; + virtual void load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete); + virtual void load_overflow(uint64_t head, uint64_t length, ValueComplete complete); + virtual void load_overflow(std::shared_ptr<const BackingRef> primary, uint64_t head, uint64_t length, + ValueComplete complete); +}; + +struct PageLoadLimits +{ + size_t max_inflight = 32; + size_t max_blob_bytes = 1024 * 1024; + size_t max_frame_bytes = 1024 * 1024; + size_t max_value_bytes = 64 * 1024 * 1024; + size_t max_overflow_pages = 65536; + size_t max_inner_children = 4096; + size_t max_inner_key_bytes = 256 * 1024; + size_t max_leaf_entries = 4096; + size_t max_leaf_key_bytes = 256 * 1024; + // Zero disables typed Chunk I/O. The extra framing window is included in + // every admitted read bound when a Chunk domain is selected. + size_t max_chunk_blob_bytes = 0; +}; + +// Supplied before a flush closes sources. The loader spends this credit +// directly; it never waits for a second host reservation while holding it. +struct PageLoadBudget +{ + std::shared_ptr<detail::AllocationReservation> bytes; + std::shared_ptr<detail::AllocationReservation> io; + std::shared_ptr<TreeWorkMetrics> work_metrics = nullptr; + std::shared_ptr<AllocationBudget> backend = nullptr; + std::shared_ptr<ApplyDispatcher> cpu_progress = nullptr; + TreeWorkCause cause = TreeWorkCause::Other; +}; + +// The resolver must be bound to the caller's captured backing/mapping view and +// return independently owned extents. The loader never consults a mutable root. +using OverflowBackingResolver = detail::InlineFunction<Status(uint64_t, std::shared_ptr<const BackingRef> *)>; + +class AsyncStorePageLoader final : public PageLoader +{ + struct State; + + public: + // The host retains and drains the runtime off its workers. I/O completion + // only schedules decoding; CPU work never runs on the transport thread. + AsyncStorePageLoader(std::shared_ptr<AsyncPageStore> store, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageLoadLimits limits = {}, + OverflowBackingResolver overflow = {}, PageLoadBudget budget = {}); + void load_leaf(BaseRef base, LeafComplete complete) override; + void load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete) override; + void load_overflow(uint64_t head, uint64_t length, ValueComplete complete) override; + void load_overflow(std::shared_ptr<const BackingRef> primary, uint64_t head, uint64_t length, + ValueComplete complete) override; + // Serial routing can retain one path plus completed decoder steps that + // have handed their result to a continuation but not yet returned. + [[nodiscard]] static std::optional<size_t> routing_reserve_bound(const PageLoadLimits &limits, size_t depth, + size_t workers, size_t cached_pages = 0) noexcept; + // Primary-page maintenance preserves overflow references without loading + // their payloads. Includes inner paths, leaf normalization, and decoder + // callback tails, but no overflow chain or materialized value storage. + [[nodiscard]] static std::optional<size_t> + primary_reserve_bound(const PageLoadLimits &limits, size_t depth, size_t workers, size_t retained_leaf_pages, + size_t cached_pages = 0, std::optional<size_t> cached_inner_pages = {}) noexcept; + // Serial captured reads retain their current leaf sources and previously + // emitted frame owners. value_bytes bounds all simultaneous materialized + // overflow payloads, including results retained across decoder callbacks. + [[nodiscard]] static std::optional<size_t> + read_reserve_bound(const PageLoadLimits &limits, size_t depth, size_t workers, size_t retained_leaf_pages, + size_t retained_values, size_t value_bytes, size_t cached_pages = 0, + std::optional<size_t> cached_inner_pages = {}) noexcept; + + private: + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load_frame.cpp b/lib/crowdb-tree/src/btree/page_load_frame.cpp new file mode 100644 index 000000000..e546ba19a --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load_frame.cpp @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/maptable/compressor.h" +#include "page_load_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +Status decode_backing_frame(std::span<const uint8_t> blob, const BackingRef &backing, const PageLoadLimits &limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, LoadBytes *out) +{ + if (!backing.readable()) { + return Status::corruption("backing has neither an extent nor an owned resident frame"); + } + const bool resident = backing.storage == BackingStorage::kResidentFrame; + const size_t length = resident ? backing.resident_frame.size() : durable_blob_raw_len(blob.data(), blob.size()); + if (length <= kFrameHeaderSize + kFrameTrailerSize || length > limits.max_frame_bytes || length > UINT32_MAX) { + return Status::corruption("backing frame exceeds declared bounds"); + } + LoadBytes frame(length, LoadBytes::allocator_type(metadata)); + if (resident) { + std::copy(backing.resident_frame.begin(), backing.resident_frame.end(), frame.begin()); + } + else { + auto status = decode_durable_page(blob.data(), blob.size(), frame.data(), static_cast<uint32_t>(length)); + if (!status.ok()) { + return status; + } + } + record_work(metadata, TreeWorkCounter::MaterializeCopyBytes, length); + out->swap(frame); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load_inner.cpp b/lib/crowdb-tree/src/btree/page_load_inner.cpp new file mode 100644 index 000000000..3bc7d7d09 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load_inner.cpp @@ -0,0 +1,224 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/page_load_internal.h" +#include "btree/page_load_lifetime.h" +#include "btree/published_state.h" +#include "crowdb-tree/maptable/compressor.h" + +namespace crowdb::tree +{ +void PageLoader::load_inner(std::shared_ptr<const BackingRef> /*backing*/, InnerComplete complete) +{ + complete(Status::not_supported("asynchronous index loader is unavailable"), {}); +} + +namespace +{ +Status read_inner_proof(const uint8_t *frame, uint32_t bytes, const PageLoadLimits &limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, size_t key_bytes, + InnerFrameProof *proof) +{ + proof->self_page_id = frame_u64(frame, fh::kSelfpage_id); + proof->frame_bytes = bytes; + proof->has_fences = frame_has_lower_fence(frame); + proof->page_id_fences = frame_fences_are_page_ids(frame); + if (proof->page_id_fences) { + proof->lower_leaf = frame_lower_fence_page_id(frame); + proof->upper_leaf = frame_upper_fence_page_id(frame); + } + else if (proof->has_fences) { + for (const auto key : {frame_lower_fence(frame), frame_upper_fence(frame)}) { + if (key.size() > limits.max_inner_key_bytes - key_bytes) { + return Status::resource_exhausted("loaded index exceeds fence byte budget"); + } + key_bytes += key.size(); + } + proof->lower = OwnedKey(frame_lower_fence(frame), metadata); + proof->upper = OwnedKey(frame_upper_fence(frame), metadata); + } + return Status::Ok(); +} + +Status decode_inner(const AlignedLoadBytes &blob, const BackingRef &backing, const PageLoadLimits &limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const InnerPageState> *out) +{ + LoadBytes frame{LoadBytes::allocator_type(metadata)}; + auto status = decode_backing_frame({blob.data(), blob.size()}, backing, limits, metadata, &frame); + const auto bytes = static_cast<uint32_t>(frame.size()); + if (!status.ok()) { + return status; + } + if (!frame_validate(frame.data(), bytes) || frame_page_type(frame.data()) != page_type::kInnerBase) { + return Status::corruption("loaded backing is not a valid index frame"); + } + const InnerFrameView index(frame.data(), bytes); + if (index.num_children() > limits.max_inner_children) { + return Status::resource_exhausted("loaded index exceeds child budget"); + } + size_t key_bytes = 0; + for (uint32_t i = 0; i < index.num_separators(); ++i) { + const size_t size = index.separator_at(i).size(); + if (size > limits.max_inner_key_bytes - key_bytes) { + return Status::resource_exhausted("loaded index exceeds separator byte budget"); + } + key_bytes += size; + } + for (uint32_t i = 0; i < index.num_children(); ++i) { + if (index.child_at(i) >= MappingView::kPageLimit) { + return Status::corruption("loaded index contains an invalid child PID"); + } + } + auto inner = detail::reserved_shared<InnerPageState>(metadata, metadata); + inner->children.reserve(index.num_children()); + inner->separators.reserve(index.num_separators()); + for (uint32_t i = 0; i < index.num_children(); ++i) { + inner->children.push_back(index.child_at(i)); + } + for (uint32_t i = 0; i < index.num_separators(); ++i) { + inner->separators.emplace_back(index.separator_at(i), metadata); + } + status = read_inner_proof(frame.data(), bytes, limits, metadata, key_bytes, &inner->frame_proof.emplace()); + if (!status.ok()) { + return status; + } + *out = std::move(inner); + return Status::Ok(); +} + +struct InnerLoad : PageLoadProgress, std::enable_shared_from_this<InnerLoad> +{ + InnerLoad(const PageLoadBudget &budget, uint64_t started_at_ns) + : PageLoadProgress(budget, started_at_ns), + metadata(budget.bytes), + blob(detail::ReservationAllocator<uint8_t>(metadata)) + { + } + + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + PageLoadLimits limits; + std::shared_ptr<const BackingRef> backing; + AlignedLoadBytes blob; + PageLoader::InnerComplete complete; + std::shared_ptr<InnerLoad> queued_owner; + std::atomic<bool> terminal{false}; + + void finish(Status status) noexcept + { + if (terminal.exchange(true)) { + return; + } + std::shared_ptr<const InnerPageState> inner; + try { + if (status.ok()) { + status = decode_inner(blob, *backing, limits, metadata, &inner); + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted("index decode allocation failed"); + } + catch (...) { + status = Status::internal_error("index decode failed"); + } + release_if_idle(); + (void)metrics.finish(status.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + auto callback = std::move(complete); + complete_page_load(callback, std::move(status), std::move(inner)); + } + + static void decode(void *context) noexcept + { + auto self = std::move(static_cast<InnerLoad *>(context)->queued_owner); + self->finish(Status::Ok()); + } +}; +} // namespace + +std::optional<size_t> AsyncStorePageLoader::routing_reserve_bound(const PageLoadLimits &limits, size_t depth, + size_t workers, size_t cached_pages) noexcept +{ + if (depth > 64 || workers == 0 || limits.max_inner_children == 0 || limits.max_inner_children > 4096 || + limits.max_blob_bytes == 0 || limits.max_frame_bytes == 0) { + return {}; + } + using detail::AllocationBound; + using detail::shared_allocation_bound; + const auto chunk_window = chunk_load_window_bound(limits); + const auto inner = AllocationBound(InnerPageState::allocation_bound()) + .add_product(limits.max_inner_children, + sizeof(OwnedKey) + sizeof(uint64_t) + OwnedKey::allocation_overhead()) + .add(limits.max_inner_key_bytes) + .add_product(2, OwnedKey::allocation_overhead()) + .value(); + const auto retained = AllocationBound(depth).add(workers).add(cached_pages).add(2).value(); + const auto operations = AllocationBound(workers).add(1).value(); + if (!inner || !retained || !operations || !chunk_window) { + return {}; + } + return AllocationBound(shared_allocation_bound<AsyncStorePageLoader>() + shared_allocation_bound<State>()) + .add_product(*retained, *inner) + .add_product(*operations, shared_allocation_bound<InnerLoad>() + shared_allocation_bound<PageReadTicket>()) + .add_product(*operations, limits.max_blob_bytes) + .add_product(*operations, *chunk_window) + .add_product(*operations, AlignedLoadBytes::kAlignment - 1) + .add_product(workers, limits.max_frame_bytes) + .value(); +} + +void AsyncStorePageLoader::load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete) +{ + const auto started_at = tree_metrics_now_ns(); + const auto reject = [&](Status status) { + record_page_load_terminal(state_->budget, started_at, status); + complete(std::move(status), {}); + }; + if (!state_->runtime || !state_->lane || !backing || !backing->readable() || backing->encoding != 0 || + (!state_->store && backing->storage != BackingStorage::kResidentFrame)) { + reject(Status::invalid_argument("index load requires a supported independently owned backing and runtime")); + return; + } + if (backing->length > state_->limits.max_blob_bytes || !chunk_load_allowed(*backing, state_->limits)) { + reject(Status::resource_exhausted("index blob exceeds load budget")); + return; + } + if (!acquire_page_load(state_->inflight, state_->limits.max_inflight)) { + reject(Status::resource_exhausted("index load concurrency budget exhausted")); + return; + } + std::shared_ptr<InnerLoad> operation; + try { + operation = detail::reserved_shared<InnerLoad>(state_->budget.bytes, state_->budget, started_at); + operation->store = state_->store; + operation->runtime = state_->runtime; + operation->lane = state_->lane; + operation->loader_owner = state_; + operation->inflight = &state_->inflight; + operation->limits = state_->limits; + operation->backing = std::move(backing); + operation->complete = std::move(complete); + if (operation->backing->storage == BackingStorage::kResidentFrame) { + queue_page_load(operation, &InnerLoad::decode); + } + else { + operation->blob.resize(operation->backing->length); + submit_page_read(operation, operation->backing, operation->blob.data(), operation->blob.size()); + } + } + catch (const std::bad_alloc &) { + if (operation && operation->complete) { + operation->finish(Status::resource_exhausted("index load allocation failed")); + } + else { + state_->inflight.fetch_sub(1, std::memory_order_release); + reject(Status::resource_exhausted("index load allocation failed")); + } + } + catch (...) { + operation->finish(Status::io_error("index read submission threw")); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load_internal.h b/lib/crowdb-tree/src/btree/page_load_internal.h new file mode 100644 index 000000000..239257fb0 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load_internal.h @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/chunk/chunk_fixed_pack.h" +#include "btree/page_load.h" +#include "crowdb-common/log.h" + +namespace crowdb::tree +{ +struct AsyncStorePageLoader::State +{ + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + PageLoadLimits limits; + OverflowBackingResolver overflow; + PageLoadBudget budget; + std::atomic<size_t> inflight{0}; +}; + +inline std::optional<size_t> chunk_load_window_bound(const PageLoadLimits &limits) noexcept +{ + return limits.max_chunk_blob_bytes == 0 + ? std::optional<size_t>{0} + : detail::ChunkFixedPackStore::read_into_allocation_bound(limits.max_chunk_blob_bytes); +} + +inline bool chunk_load_allowed(const BackingRef &backing, const PageLoadLimits &limits) noexcept +{ + return backing.storage != BackingStorage::kChunkPack || backing.length <= limits.max_chunk_blob_bytes; +} + +using LoadBytes = detail::ReservedVector<uint8_t, AllocationKind::kScratch>; + +Status decode_backing_frame(std::span<const uint8_t> blob, const BackingRef &backing, const PageLoadLimits &limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, LoadBytes *out); + +[[nodiscard]] std::optional<size_t> decoded_leaf_allocation_bound(const PageLoadLimits &limits) noexcept; +[[nodiscard]] std::optional<size_t> overflow_read_allocation_bound(const PageLoadLimits &limits, size_t workers, + size_t retained_values, size_t value_bytes) noexcept; + +// The charged storage supplies direct-I/O alignment itself, so a Block read +// does not allocate an unaccounted second copy in the backend adapter. +class AlignedLoadBytes +{ + public: + static constexpr size_t kAlignment = 4096; + + explicit AlignedLoadBytes(detail::ReservationAllocator<uint8_t> allocator) + : storage_(detail::ReservationAllocator<uint8_t, AllocationKind::kIo>(allocator.owner())) + { + } + + void resize(size_t length) + { + if (length > SIZE_MAX - (kAlignment - 1)) { + throw std::bad_alloc(); + } + const auto required = length + kAlignment - 1; + if (required > storage_.size()) { + // Release an old overflow window before growing; vector growth + // must not retain two windows or geometrically exceed the bound. + storage_ = decltype(storage_)(storage_.get_allocator()); + storage_.resize(required); + } + void *data = storage_.data(); + auto available = storage_.size(); + auto *aligned = std::align(kAlignment, length, data, available); + bytes_ = {static_cast<uint8_t *>(aligned), length}; + } + + [[nodiscard]] uint8_t *data() noexcept + { + return bytes_.data(); + } + + [[nodiscard]] const uint8_t *data() const noexcept + { + return bytes_.data(); + } + + [[nodiscard]] size_t size() const noexcept + { + return bytes_.size(); + } + + [[nodiscard]] size_t capacity() const noexcept + { + return storage_.capacity(); + } + + private: + detail::ReservedVector<uint8_t, AllocationKind::kIo> storage_; + std::span<uint8_t> bytes_; +}; + +template <class Callback, class Result> +void complete_page_load(Callback &callback, Status status, Result result) noexcept +{ + try { + callback(std::move(status), std::move(result)); + } + catch (...) { + CRB_LOG_ERROR("page load receiver threw after terminal"); + } +} + +inline bool acquire_page_load(std::atomic<size_t> &inflight, size_t maximum) +{ + size_t current = inflight.load(std::memory_order_relaxed); + while (current < maximum) { + if (inflight.compare_exchange_weak(current, current + 1, std::memory_order_acq_rel)) { + return true; + } + } + return false; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load_leaf.cpp b/lib/crowdb-tree/src/btree/page_load_leaf.cpp new file mode 100644 index 000000000..66ded7faf --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load_leaf.cpp @@ -0,0 +1,170 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_load_leaf.h" + +#include "btree/allocation_bound.h" +#include "btree/page_load_internal.h" +#include "crowdb-tree/maptable/compressor.h" + +#include <algorithm> +#include <numeric> + +namespace crowdb::tree +{ +namespace +{ +struct LoadedLeafOwner +{ + std::shared_ptr<const LeafBase> leaf; + std::shared_ptr<const BackingRef> backing; +}; + +Status check_leaf_window(std::span<const uint8_t> frame, const PageLoadLimits &limits) +{ + const auto *data = frame.data(); + const uint32_t count = frame_u32(data, fh::kSlotCount); + const uint32_t deltas = frame_u32(data, fh::kDeltaCount); + const uint32_t delta_begin = frame_u32(data, fh::kFreeLo); + const size_t body = frame.size() - kFrameTrailerSize; + if (count > (body - kFrameHeaderSize) / kLeafSlotSize || delta_begin > body || + deltas > (body - delta_begin) / kLeafSlotSize) { + return Status::corruption("loaded leaf slot directories exceed frame bounds"); + } + if (count > limits.max_leaf_entries || deltas > limits.max_leaf_entries - count) { + return Status::resource_exhausted("loaded leaf exceeds entry budget"); + } + size_t key_bytes = 0; + for (size_t i = 0; i < static_cast<size_t>(count) + deltas; ++i) { + const auto offset = + i < count ? kFrameHeaderSize + (i * kLeafSlotSize) : delta_begin + ((i - count) * kLeafSlotSize); + const uint32_t size = frame_u32(data + offset, 4); + if (size > limits.max_leaf_key_bytes - key_bytes) { + return Status::resource_exhausted("loaded leaf exceeds key byte budget"); + } + key_bytes += size; + } + return Status::Ok(); +} + +Status check_cells(const LeafFrameView &view, const std::optional<uint64_t> &max_slot) +{ + const size_t count = view.count(); + for (size_t i = 0; i < count + view.delta_count(); ++i) { + const Slice raw = + i < count ? view.cell(static_cast<uint32_t>(i)) : view.delta_cell(static_cast<uint32_t>(i - count)); + const CellView cell(raw); + if (!cell.valid() || (cell.flags() != 0 && cell.flags() != kFlagTombstone && cell.flags() != kFlagOverflow) || + (cell.is_tombstone() && raw.size() != kCellHeaderSize) || + (cell.is_overflow() && + (raw.size() != kOverflowCellSize || cell.overflow_head() == kInvalidPageId || cell.overflow_len() == 0))) { + return Status::corruption("loaded leaf contains an invalid cell"); + } + if (max_slot && cell.slot() > *max_slot) { + return Status::corruption("loaded leaf exceeds its proven slot bound"); + } + } + return Status::Ok(); +} + +Status fold_leaf(LoadBytes &frame, const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + const LeafFrameView view(frame.data(), static_cast<uint32_t>(frame.size())); + if (view.delta_count() == 0) { + return Status::Ok(); + } + std::vector<uint32_t, detail::ReservationAllocator<uint32_t>> order( + view.delta_count(), detail::ReservationAllocator<uint32_t>(metadata)); + std::iota(order.begin(), order.end(), 0U); + // LeafChainCursor uses append order for equal-slot delta ties. Including + // the index in this ordering avoids stable_sort's uncharged scratch space. + std::sort(order.begin(), order.end(), [&view](uint32_t a, uint32_t b) { + const int compared = view.delta_key(a).compare(view.delta_key(b)); + return compared == 0 ? a < b : compared < 0; + }); + size_t unique = 0; + for (const auto index : order) { + if (unique != 0 && view.delta_key(order[unique - 1]) == view.delta_key(index)) { + if (CellView(view.delta_cell(index)).slot() > CellView(view.delta_cell(order[unique - 1])).slot()) { + order[unique - 1] = index; + } + } + else { + order[unique++] = index; + } + } + order.resize(unique); + LoadBytes normalized(frame.size(), LoadBytes::allocator_type(metadata)); + LeafFrameBuilder builder(normalized.data(), static_cast<uint32_t>(normalized.size())); + uint32_t main = 0; + size_t delta = 0; + while (main < view.count() || delta < order.size()) { + const bool have_main = main < view.count(); + const bool have_delta = delta < order.size(); + int compared = 0; + if (have_main && have_delta) { + compared = view.key(main).compare(view.delta_key(order[delta])); + } + const bool take_main = have_main && (!have_delta || compared <= 0); + const bool take_delta = have_delta && (!have_main || compared >= 0); + Slice key = take_main ? view.key(main) : view.delta_key(order[delta]); + Slice cell = take_main ? view.cell(main) : view.delta_cell(order[delta]); + // The main stream wins equal-slot ties, matching LeafChainCursor. + if (take_main && take_delta && CellView(view.delta_cell(order[delta])).slot() > CellView(cell).slot()) { + cell = view.delta_cell(order[delta]); + } + if (!builder.try_append_sorted(key, cell)) { + return Status::corruption("loaded leaf records exceed frame capacity"); + } + main += static_cast<uint32_t>(take_main); + delta += static_cast<size_t>(take_delta); + } + // Keep even slot-zero tombstones: GC belongs to the captured logical view. + builder.finish(view.self_page_id(), view.right_sibling()); + frame.swap(normalized); + return Status::Ok(); +} +} // namespace + +std::optional<size_t> decoded_leaf_allocation_bound(const PageLoadLimits &limits) noexcept +{ + return detail::AllocationBound(detail::shared_allocation_bound<LoadedLeafOwner>()) + .add(detail::shared_allocation_bound<LeafBase>()) + .add(limits.max_frame_bytes) + .value(); +} + +Status decode_loaded_leaf(std::span<const uint8_t> blob, const BaseRef &base, const PageLoadLimits &limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const LeafBase> *out) +{ + LoadBytes frame{LoadBytes::allocator_type(metadata)}; + auto status = decode_backing_frame(blob, *base.backing, limits, metadata, &frame); + const auto length = static_cast<uint32_t>(frame.size()); + if (!status.ok()) { + return status; + } + if (frame_page_type(frame.data()) != page_type::kLeafBase) { + return Status::corruption("loaded backing is not a leaf frame"); + } + status = check_leaf_window(frame, limits); + if (!status.ok()) { + return status; + } + if (!frame_validate(frame.data(), length) || frame_page_type(frame.data()) != page_type::kLeafBase) { + return Status::corruption("loaded backing is not a valid leaf frame"); + } + status = check_cells(LeafFrameView(frame.data(), length), base.max_slot); + if (!status.ok()) { + return status; + } + status = fold_leaf(frame, metadata); + if (!status.ok()) { + return status; + } + auto owner = detail::reserved_shared<LoadedLeafOwner>(metadata); + owner->leaf = LeafBase::from_frame_copy_shared(frame.data(), length, metadata); + owner->backing = base.backing; + *out = std::shared_ptr<const LeafBase>(owner, owner->leaf.get()); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load_leaf.h b/lib/crowdb-tree/src/btree/page_load_leaf.h new file mode 100644 index 000000000..b51407061 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load_leaf.h @@ -0,0 +1,16 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_load.h" + +#include <span> + +namespace crowdb::tree +{ +// Fold the captured frame at its proven slot bound, preserving tombstones and +// the backing's overflow protection. This does not provide historical reads. +Status decode_loaded_leaf(std::span<const uint8_t> blob, const BaseRef &base, const PageLoadLimits &limits, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const LeafBase> *out); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load_lifetime.h b/lib/crowdb-tree/src/btree/page_load_lifetime.h new file mode 100644 index 000000000..2fff02712 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load_lifetime.h @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "apply_dispatcher.h" +#include "backend/backing_io.h" +#include "page_load.h" +#include "resource_observation.h" +#include "tree_metrics.h" + +namespace crowdb::tree +{ +struct PageLoadProgress +{ + PageLoadProgress(const PageLoadBudget &budget, uint64_t started_at_ns) + : io_credit(budget.io, 1), + metrics(budget.work_metrics, TreeStage::PageLoadPipeline, started_at_ns), + backend_budget(budget.backend), + cpu_progress(budget.cpu_progress), + cause(budget.cause), + cpu_node(&PageLoadProgress::run_ready, this) + { + } + + ~PageLoadProgress() + { + release_progress(); + } + + PageLoadProgress(const PageLoadProgress &) = delete; + PageLoadProgress &operator=(const PageLoadProgress &) = delete; + + AllocationLease io_credit; + TreeMetricsOperation metrics; + std::shared_ptr<AllocationBudget> backend_budget; + std::shared_ptr<void> loader_owner; + std::shared_ptr<ApplyDispatcher> cpu_progress; + TreeWorkCause cause; + ApplyDispatcher::Node cpu_node; + detail::FlushRuntime::Task cpu_task = nullptr; + void *cpu_context = nullptr; + + static void run_ready(void *context) noexcept + { + auto &self = *static_cast<PageLoadProgress *>(context); + self.cpu_task(self.cpu_context); + } + + std::atomic<size_t> *inflight = nullptr; + std::atomic<bool> io_pending{false}; + std::atomic<bool> progress_released{false}; + + void release_progress() noexcept + { + if (!progress_released.exchange(true, std::memory_order_acq_rel)) { + io_credit.reset(); + if (inflight != nullptr) { + inflight->fetch_sub(1, std::memory_order_release); + } + } + } + + void release_if_idle() noexcept + { + if (!io_pending.load(std::memory_order_acquire)) { + release_progress(); + } + } +}; + +inline void record_page_load_terminal(const PageLoadBudget &budget, uint64_t started_at_ns, + const Status &status) noexcept +{ + TreeMetricsOperation metrics(budget.work_metrics, TreeStage::PageLoadPipeline, started_at_ns); + (void)metrics.finish(status.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); +} + +// One immutable callback identity per read, including every overflow step. +// Decoding waits for submit_read to return, so inline success followed by an +// exception cannot publish a result or start another read before rejection. +struct PageReadTicket +{ + PageReadTicket(std::shared_ptr<const void> lifetime, void *operation, AsyncCompleteFn callback, + const std::shared_ptr<detail::AllocationReservation> &metadata, size_t bytes) + : resources(metadata, TreeResourceMetric::ReadInflightBytes, bytes), + owner(std::move(lifetime)), + context(operation), + complete(callback) + { + } + + ResourceObservation resources; + std::shared_ptr<const void> owner; + void *context; + AsyncCompleteFn complete; + Status status; + std::atomic<bool> callback_claimed{false}; + std::atomic<unsigned> events{0}; + + static void completed(void *context, Status result) + { + auto *ticket = static_cast<PageReadTicket *>(context); + if (ticket->callback_claimed.exchange(true, std::memory_order_acq_rel)) { + return; + } + ticket->resources.reset(); + ticket->status = std::move(result); + if ((ticket->events.fetch_or(1, std::memory_order_acq_rel) & 2U) != 0) { + ticket->complete(ticket->context, std::move(ticket->status)); + } + } + + void submitted() + { + if ((events.fetch_or(2, std::memory_order_acq_rel) & 1U) != 0) { + complete(context, std::move(status)); + } + } +}; + +template <class Load> +void queue_page_load(const std::shared_ptr<Load> &operation, detail::FlushRuntime::Task task) noexcept +{ + operation->queued_owner = operation; + Status status; + try { + status = operation->runtime->try_submit(operation->lane, task, operation.get()); + } + catch (...) { + status = Status::resource_exhausted(); + } + if (!status.ok() && operation->cpu_progress) { + operation->cpu_task = task; + operation->cpu_context = operation.get(); + operation->cpu_progress->request(operation->cpu_node, operation); + return; + } + if (!status.ok()) { + operation->queued_owner.reset(); + operation->finish(std::move(status)); + } +} + +template <class Load> void page_read_completed(void *context, Status status) noexcept +{ + auto operation = static_cast<Load *>(context)->shared_from_this(); + operation->io_pending.store(false, std::memory_order_release); + if (operation->terminal.load(std::memory_order_acquire)) { + operation->release_progress(); + } + else if (!status.ok()) { + operation->finish(std::move(status)); + } + else { + queue_page_load(operation, &Load::decode); + } +} + +template <class Load> +void submit_page_read(const std::shared_ptr<Load> &operation, std::shared_ptr<const BackingRef> backing, void *buffer, + size_t bytes) +{ + auto ticket = detail::reserved_shared<PageReadTicket>(operation->metadata, operation, operation.get(), + &page_read_completed<Load>, operation->metadata, + operation->blob.capacity()); + operation->io_pending.store(true, std::memory_order_release); + try { + submit_backing_read(*operation->store, std::move(backing), buffer, bytes, operation->metadata, + {.context = ticket.get(), + .complete_fn = &PageReadTicket::completed, + .owner = ticket, + .allocations = operation->backend_budget, + .cause = operation->cause}); + } + catch (const std::bad_alloc &) { + operation->finish(Status::resource_exhausted()); + } + catch (...) { + operation->finish(Status::io_error()); + } + // On rejection, finish reported the error but retained the operation's + // progress credit. A late callback or final ticket destruction releases + // it; neither path may decode or report a second terminal. + ticket->submitted(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load_overflow.cpp b/lib/crowdb-tree/src/btree/page_load_overflow.cpp new file mode 100644 index 000000000..898bea0ea --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_load_overflow.cpp @@ -0,0 +1,273 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/overflow_backing.h" +#include "btree/page_load_internal.h" +#include "btree/page_load_lifetime.h" +#include "btree/recovered_frame_identity.h" +#include "crowdb-tree/maptable/compressor.h" + +#include <limits> +#include <set> + +namespace crowdb::tree +{ +namespace +{ +struct LoadedValue +{ + LoadedValue(const std::shared_ptr<detail::AllocationReservation> &metadata, size_t bytes) + : payload(metadata, bytes > buffer::kInlineCap ? bytes : 0, AllocationKind::kPayload), + value(buffer::alloc(bytes)) + { + } + + AllocationLease payload; + buffer value; +}; + +struct OverflowLoad : PageLoadProgress, std::enable_shared_from_this<OverflowLoad> +{ + OverflowLoad(const PageLoadBudget &budget, uint64_t started_at_ns) + : PageLoadProgress(budget, started_at_ns), + metadata(budget.bytes), + blob(detail::ReservationAllocator<uint8_t>(metadata)), + visited(std::less<>{}, detail::ReservationAllocator<uint64_t>(metadata)) + { + } + + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + PageLoadLimits limits; + OverflowBackingResolver resolve; + PageLoader::ValueComplete complete; + std::shared_ptr<OverflowLoad> queued_owner; + std::shared_ptr<const BackingRef> backing; + std::shared_ptr<buffer> value; + AlignedLoadBytes blob; + std::set<uint64_t, std::less<>, detail::ReservationAllocator<uint64_t>> visited; + uint64_t next = kInvalidPageId; + size_t length = 0; + size_t written = 0; + std::atomic<bool> terminal{false}; + + void finish(Status status) noexcept + { + if (terminal.exchange(true)) { + return; + } + release_if_idle(); + (void)metrics.finish(status.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + auto callback = std::move(complete); + auto result = std::move(value); + if (!status.ok()) { + result.reset(); + } + complete_page_load(callback, std::move(status), std::move(result)); + } + + void read_next() + { + if (visited.contains(next)) { + finish(Status::corruption("overflow backing chain contains a cycle")); + return; + } + if (visited.size() >= limits.max_overflow_pages) { + finish(Status::resource_exhausted("overflow chain exceeds page budget")); + return; + } + visited.insert(next); + backing.reset(); + auto status = resolve(next, &backing); + if (!status.ok()) { + finish(std::move(status)); + return; + } + if (!backing || !backing->readable() || backing->encoding != 0) { + finish(Status::corruption("overflow backing has no independent supported extent")); + return; + } + if (backing->length > limits.max_blob_bytes || !chunk_load_allowed(*backing, limits)) { + finish(Status::resource_exhausted("overflow blob exceeds load budget")); + return; + } + if (backing->storage == BackingStorage::kResidentFrame) { + queue_page_load(shared_from_this(), &OverflowLoad::decode); + } + else { + if (!store) { + finish(Status::invalid_argument("durable overflow read requires a backend")); + return; + } + blob.resize(backing->length); + submit_page_read(shared_from_this(), backing, blob.data(), blob.size()); + } + } + + void decode_frame() + { + LoadBytes frame{LoadBytes::allocator_type(metadata)}; + auto status = decode_backing_frame({blob.data(), blob.size()}, *backing, limits, metadata, &frame); + const auto frame_bytes = static_cast<uint32_t>(frame.size()); + if (!status.ok()) { + finish(std::move(status)); + return; + } + if (!frame_validate(frame.data(), frame_bytes) || frame_page_type(frame.data()) != page_type::kOverflowFrame || + !recovered_frame_identity_matches(*backing, page_type::kOverflowFrame, + frame_u64(frame.data(), fh::kSelfpage_id), next)) { + finish(Status::corruption("loaded backing is not a valid overflow frame")); + return; + } + const OverflowFrameView decoded(frame.data(), frame_bytes); + const size_t chunk = decoded.chunk_len(); + if (chunk == 0 || chunk > overflow_chunk_cap(frame_bytes) || chunk > length - written) { + finish(Status::corruption("overflow chunk disagrees with declared value length")); + return; + } + std::memcpy(value->data() + written, decoded.payload().data(), chunk); + record_work(metadata, TreeWorkCounter::MaterializeCopyBytes, chunk, cause); + written += chunk; + next = decoded.next_page_id(); + if (written == length) { + finish(next == kInvalidPageId ? Status::Ok() : Status::corruption("overflow chain exceeds value length")); + return; + } + if (next == kInvalidPageId) { + finish(Status::corruption("overflow chain ends before declared value length")); + return; + } + read_next(); + } + + static void start(void *context) noexcept + { + auto self = std::move(static_cast<OverflowLoad *>(context)->queued_owner); + try { + auto value = detail::reserved_shared<LoadedValue>(self->metadata, self->metadata, self->length); + self->value = std::shared_ptr<buffer>(value, &value->value); + self->read_next(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted("overflow read allocation failed")); + } + catch (...) { + self->finish(Status::io_error("overflow resolution or submission threw")); + } + } + + static void decode(void *context) noexcept + { + auto self = std::move(static_cast<OverflowLoad *>(context)->queued_owner); + try { + self->decode_frame(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted("overflow decode allocation failed")); + } + catch (...) { + self->finish(Status::io_error("overflow resolution or submission threw")); + } + } +}; +} // namespace + +std::optional<size_t> overflow_read_allocation_bound(const PageLoadLimits &limits, size_t workers, + size_t retained_values, size_t value_bytes) noexcept +{ + using detail::AllocationBound; + using detail::shared_allocation_bound; + if (workers == 0 || limits.max_overflow_pages == 0) { + return {}; + } + const auto operations = AllocationBound(workers).add(1).value(); + const auto values = AllocationBound(retained_values).add(workers).add(1).value(); + // The node allowance covers red/black tree links, color and alignment on + // supported standard libraries, independently of the uint64 payload. + const auto metadata = AllocationBound(shared_allocation_bound<OverflowLoad>()) + .add(shared_allocation_bound<PageReadTicket>()) + .add_product(limits.max_overflow_pages, shared_allocation_bound<uint64_t>()) + .add(limits.max_blob_bytes) + .add(chunk_load_window_bound(limits)) + .add(AlignedLoadBytes::kAlignment - 1) + .value(); + if (!operations || !metadata || !values) { + return {}; + } + return AllocationBound(value_bytes) + .add_product(*operations, *metadata) + .add_product(*values, shared_allocation_bound<LoadedValue>()) + .add_product(workers, limits.max_frame_bytes) + .value(); +} + +void AsyncStorePageLoader::load_overflow(uint64_t head, uint64_t length, ValueComplete complete) +{ + load_overflow({}, head, length, std::move(complete)); +} + +void AsyncStorePageLoader::load_overflow(std::shared_ptr<const BackingRef> primary, uint64_t head, uint64_t length, + ValueComplete complete) +{ + const auto started_at = tree_metrics_now_ns(); + const auto reject = [&](Status status) { + record_page_load_terminal(state_->budget, started_at, status); + complete(std::move(status), {}); + }; + OverflowBackingResolver resolver = state_->overflow; + if (primary && primary->overflow) { + resolver = [primary](uint64_t pid, std::shared_ptr<const BackingRef> *out) { + const auto found = primary->overflow->pages.find(pid); + if (found == primary->overflow->pages.end() || !found->second.backing || + found->second.backing->backend_id != primary->backend_id || + found->second.backing->generation != primary->generation) { + return Status::corruption("overflow is outside the captured primary backing"); + } + *out = found->second.backing; + return Status::Ok(); + }; + } + if (!resolver) { + reject(Status::not_supported("captured overflow backing resolver is unavailable")); + return; + } + if (!state_->runtime || !state_->lane || head == kInvalidPageId || length == 0) { + reject(Status::invalid_argument("overflow read requires a runtime and a nonempty valid chain")); + return; + } + if (length > state_->limits.max_value_bytes || length > std::numeric_limits<size_t>::max()) { + reject(Status::resource_exhausted("overflow value exceeds read budget")); + return; + } + if (!acquire_page_load(state_->inflight, state_->limits.max_inflight)) { + reject(Status::resource_exhausted("overflow read concurrency budget exhausted")); + return; + } + std::shared_ptr<OverflowLoad> operation; + try { + operation = detail::reserved_shared<OverflowLoad>(state_->budget.bytes, state_->budget, started_at); + operation->store = state_->store; + operation->runtime = state_->runtime; + operation->lane = state_->lane; + operation->loader_owner = state_; + operation->inflight = &state_->inflight; + operation->limits = state_->limits; + operation->complete = std::move(complete); + operation->resolve = std::move(resolver); + operation->next = head; + operation->length = static_cast<size_t>(length); + queue_page_load(operation, &OverflowLoad::start); + } + catch (const std::bad_alloc &) { + if (operation && operation->complete) { + operation->finish(Status::resource_exhausted("overflow operation allocation failed")); + } + else { + state_->inflight.fetch_sub(1, std::memory_order_release); + reject(Status::resource_exhausted("overflow operation allocation failed")); + } + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_maintenance.cpp b/lib/crowdb-tree/src/btree/page_maintenance.cpp new file mode 100644 index 000000000..e5a9a1920 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_maintenance.cpp @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_maintenance.h" + +#include <queue> + +namespace crowdb::tree +{ +namespace +{ +class OverlayCoalescer +{ + public: + [[nodiscard]] size_t removed_primary_bytes() const noexcept + { + return removed_primary_bytes_; + } + + OverlayCoalescer(const PageState &page, const PageMaintenanceLimits &limits, uint64_t first_run_id, + std::span<const std::shared_ptr<const OverlayBatch>> inputs) + : page_(page), + inputs_(inputs), + limits_(limits), + next_id_(first_run_id), + positions_(inputs.size(), 0, detail::ReservationAllocator<size_t>(limits.metadata)), + window_(EntryAllocator(limits.metadata)) + { + } + + Status run(OverlayStorage *output) + { + auto later = [&](size_t a, size_t b) { + const auto &left = entry(a); + const auto &right = entry(b); + const int order = left.key().compare(right.key()); + if (order != 0) { + return order > 0; + } + return left.slot() != right.slot() ? left.slot() < right.slot() : a > b; + }; + detail::ReservedVector<size_t> heap{detail::ReservationAllocator<size_t>(limits_.metadata)}; + heap.reserve(inputs_.size()); + std::priority_queue<size_t, detail::ReservedVector<size_t>, decltype(later)> queue(later, std::move(heap)); + for (size_t i = 0; i < inputs_.size(); ++i) { + const auto &run = inputs_[i]; + if (run == nullptr) { + return Status::invalid_argument("coalesce overlay has no owner"); + } + positions_[i] = page_.range.lower.has_value() ? run->lower_bound(Slice(*page_.range.lower)) : 0; + if (valid(i)) { + queue.push(i); + } + } + while (!queue.empty()) { + const EntryRef winner = entry(queue.top()); + auto status = append(winner, output); + if (!status.ok()) { + return status; + } + do { + if (work_ >= limits_.max_work_entries) { + return Status::resource_exhausted("overlay coalesce work budget exhausted"); + } + ++work_; + const size_t source = queue.top(); + queue.pop(); + ++positions_[source]; + if (valid(source)) { + queue.push(source); + } + } while (!queue.empty() && entry(queue.top()).key() == winner.key()); + } + return finish(output); + } + + private: + [[nodiscard]] const EntryRef &entry(size_t source) const + { + return inputs_[source]->entries()[positions_[source]]; + } + + [[nodiscard]] bool valid(size_t source) const + { + return positions_[source] < inputs_[source]->entries().size() && page_.range.contains(entry(source).key()); + } + + Status finish(OverlayStorage *output) + { + if (window_.empty()) { + return Status::Ok(); + } + if (output->size() >= limits_.max_sources || next_id_ == UINT64_MAX) { + return Status::resource_exhausted("coalesce output run/identity budget exhausted"); + } + std::shared_ptr<const OverlayBatch> run; + EntryStorage entries(window_.get_allocator()); + entries.swap(window_); + auto status = OverlayBatch::build_owned(next_id_, std::move(entries), limits_.run, &run); + if (!status.ok()) { + return status; + } + output->push_back(std::move(run)); + ++next_id_; + window_.clear(); + key_bytes_ = 0; + return Status::Ok(); + } + + Status append(const EntryRef &entry, OverlayStorage *output) + { + // Without a base, the merged overlay includes every older version in + // this leaf. A captured GC floor then permits removing the winner. + // Cold or resident bases still need the tombstone to mask their value. + if (page_.bases.empty() && inputs_.size() == page_.overlays.size() && (entry.flags() & kFlagTombstone) != 0 && + entry.slot() <= limits_.gc_floor) { + const size_t bytes = entry.key_length + kCellHeaderSize + kLeafSlotSize; + if (bytes > SIZE_MAX - removed_primary_bytes_) { + return Status::resource_exhausted("coalesced tombstone size overflow"); + } + removed_primary_bytes_ += bytes; + return Status::Ok(); + } + if (limits_.run.max_entries == 0 || entry.key_length > limits_.run.max_key_bytes || + emitted_ >= limits_.max_metadata_bytes / sizeof(EntryRef)) { + return Status::resource_exhausted("coalesce entry metadata budget exhausted"); + } + if (window_.size() >= limits_.run.max_entries || entry.key_length > limits_.run.max_key_bytes - key_bytes_) { + auto status = finish(output); + if (!status.ok()) { + return status; + } + } + window_.push_back(entry); + key_bytes_ += entry.key_length; + ++emitted_; + return Status::Ok(); + } + + const PageState &page_; + std::span<const std::shared_ptr<const OverlayBatch>> inputs_; + const PageMaintenanceLimits &limits_; + uint64_t next_id_; + detail::ReservedVector<size_t> positions_; + EntryStorage window_; + size_t key_bytes_ = 0; + size_t emitted_ = 0; + size_t work_ = 0; + size_t removed_primary_bytes_ = 0; +}; +} // namespace + +PageMaintenanceResult coalesce_overlays(std::shared_ptr<const PageState> page, const PageMaintenanceLimits &limits, + uint64_t representation_id, uint64_t first_run_id) +{ + PageMaintenanceResult result(limits.metadata); + if (page == nullptr) { + result.status = Status::invalid_argument("coalesce needs a page owner"); + return result; + } + size_t first = page->overlays.size(); + size_t work = 0; + while (first != 0 && page->overlays.size() - first < limits.max_sources) { + const auto &run = page->overlays[first - 1]; + if (!run) { + result.status = Status::corruption("coalesce overlay has no owner"); + return result; + } + const auto begin = page->range.lower ? run->lower_bound(*page->range.lower) : 0; + const auto end = page->range.upper ? run->lower_bound(*page->range.upper) : run->entries().size(); + if (end - begin > limits.max_work_entries - work || + end - begin > (limits.max_metadata_bytes / sizeof(EntryRef)) - work) { + break; + } + work += end - begin; + --first; + } + if ((first != 0 && first == page->overlays.size()) || (first != 0 && page->overlays.size() - first < 2)) { + result.status = Status::resource_exhausted("coalesce needs a complete bounded input window"); + result.readiness = MaintenanceReadiness::kDeferred; + return result; + } + OverlayStorage merged(OverlayAllocator(limits.metadata)); + OverlayCoalescer merge(*page, limits, first_run_id, + std::span<const std::shared_ptr<const OverlayBatch>>(page->overlays).subspan(first)); + result.status = merge.run(&merged); + if (!result.status.ok()) { + result.readiness = MaintenanceReadiness::kDeferred; + return result; + } + auto replacement = detail::reserved_shared<PageState>(limits.metadata, *page, limits.metadata); + replacement->overlays.resize(first); + replacement->overlays.insert(replacement->overlays.end(), merged.begin(), merged.end()); + // A partial merge also keeps tombstones masking untouched runs. + replacement->representation_id = representation_id; + result.page = std::move(replacement); + result.removed_primary_bytes = merge.removed_primary_bytes(); + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_maintenance.h b/lib/crowdb-tree/src/btree/page_maintenance.h new file mode 100644 index 000000000..8819e88ef --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_maintenance.h @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/logical_page.h" + +namespace crowdb::tree +{ +struct PageMaintenanceLimits +{ + RunLimits run; + size_t max_sources = 64; + size_t max_base_views = 64; + size_t max_output_pages = 64; + size_t max_work_entries = 65536; + size_t max_metadata_bytes = 16 * 1024 * 1024; + size_t primary_page_bytes = 64 * 1024; + size_t target_page_bytes = 48 * 1024; + size_t merge_trigger_bytes = 16 * 1024; + size_t merge_target_bytes = 40 * 1024; + size_t inline_value_bytes = 8 * 1024; + uint64_t gc_floor = 0; + std::shared_ptr<detail::AllocationReservation> metadata = nullptr; + // Persistent range slices use Routing credit, independently of scratch. +}; + +struct LeafLayoutStats +{ + uint64_t kv_count = 0; + uint64_t kv_bytes = 0; +}; + +enum class MaintenanceReadiness : uint8_t { kReady, kNeedsBase, kDeferred }; + +struct BaseLoadRequest +{ + std::shared_ptr<const PageState> page; + size_t base_index = 0; +}; + +struct PageMaintenanceResult +{ + PageMaintenanceResult() = default; + + explicit PageMaintenanceResult(const std::shared_ptr<detail::AllocationReservation> &metadata) + : loads(detail::ReservationAllocator<BaseLoadRequest>(metadata)) + { + } + + Status status; + MaintenanceReadiness readiness = MaintenanceReadiness::kReady; + detail::ReservedVector<BaseLoadRequest> loads; + std::shared_ptr<const PageState> page; + // Winning tombstones removed by a complete, base-free coalesce. + size_t removed_primary_bytes = 0; +}; + +// This is an uncompressed primary-frame size plan. It does not allocate +// overflow chains, encode frames, persist data, or assert checkpoint readiness. +struct LeafLayoutPlan +{ + LeafLayoutPlan() = default; + + explicit LeafLayoutPlan(const std::shared_ptr<detail::AllocationReservation> &metadata) + : loads(detail::ReservationAllocator<BaseLoadRequest>(metadata)), + separators(detail::ReservationAllocator<OwnedKey>(metadata)), + primary_bytes(detail::ReservationAllocator<size_t>(metadata)) + { + } + + Status status; + MaintenanceReadiness readiness = MaintenanceReadiness::kReady; + std::shared_ptr<const PageState> source; + detail::ReservedVector<BaseLoadRequest> loads; + detail::ReservedVector<OwnedKey> separators; + detail::ReservedVector<size_t> primary_bytes; + size_t entry_count = 0; + size_t total_primary_bytes = 0; + size_t overflow_values = 0; +}; + +struct PageVersion +{ + uint64_t route_generation = 0; + uint64_t content_id = 0; + uint64_t representation_id = 0; +}; + +struct SplitTarget +{ + uint64_t page_id = kInvalidPageId; + PageVersion version; +}; + +struct LeafSplitResult +{ + LeafSplitResult() = default; + + explicit LeafSplitResult(const std::shared_ptr<detail::AllocationReservation> &metadata) + : pages(detail::ReservationAllocator<std::shared_ptr<const PageState>>(metadata)) + { + } + + Status status; + MaintenanceReadiness readiness = MaintenanceReadiness::kReady; + detail::ReservedVector<std::shared_ptr<const PageState>> pages; +}; + +struct LeafMergeInput +{ + std::shared_ptr<const PageState> left; + std::shared_ptr<const PageState> right; + uint64_t left_page_id = kInvalidPageId; + uint64_t right_page_id = kInvalidPageId; + uint64_t left_parent = kInvalidPageId; + uint64_t right_parent = kInvalidPageId; +}; + +// All operations prepare immutable replacements. Publication and the complete +// parent/sibling/root transaction remain the caller's responsibility. +// Coalesce/merge consume at most max_sources consecutive run identities from +// first_run_id; the caller reserves that interval before dispatching workers. +PageMaintenanceResult coalesce_overlays(std::shared_ptr<const PageState> page, const PageMaintenanceLimits &limits, + uint64_t representation_id, uint64_t first_run_id); +LeafLayoutPlan plan_leaf_layout(std::shared_ptr<const PageState> page, const PageMaintenanceLimits &limits); +LeafSplitResult split_leaf_ranges(const LeafLayoutPlan &layout, std::span<const SplitTarget> targets, + const PageMaintenanceLimits &limits); +PageMaintenanceResult plan_leaf_merge(const LeafMergeInput &input, const PageMaintenanceLimits &limits, + PageVersion version, uint64_t first_run_id); +// Joins already validated disjoint ranges without loading or materializing. +// The caller proves merge size/hysteresis from exact captured page metrics. +PageMaintenanceResult join_leaf_ranges(const LeafMergeInput &input, const PageMaintenanceLimits &limits, + PageVersion version); +Status accumulate_leaf_stats(Slice key, const LogicalCell &cell, LeafLayoutStats *stats); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_maintenance_structure.cpp b/lib/crowdb-tree/src/btree/page_maintenance_structure.cpp new file mode 100644 index 000000000..76e1e0489 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_maintenance_structure.cpp @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_maintenance.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +PageKeyRange intersection(PageKeyRange left, const PageKeyRange &right) +{ + if (right.lower.has_value() && (!left.lower.has_value() || Slice(*left.lower) < Slice(*right.lower))) { + left.lower = right.lower; + } + if (right.upper.has_value() && (!left.upper.has_value() || Slice(*left.upper) > Slice(*right.upper))) { + left.upper = right.upper; + } + return left; +} + +void set_version(PageState *page, PageVersion version) +{ + page->route_generation = version.route_generation; + page->content_id = version.content_id; + page->representation_id = version.representation_id; +} + +Status append_range(const PageState &source, const PageKeyRange &range, const PageMaintenanceLimits &limits, + PageState *output) +{ + for (const auto &base : source.bases) { + if (!base.range.overlaps(range)) { + continue; + } + if (output->bases.size() >= limits.max_base_views) { + return Status::resource_exhausted("logical base view budget exhausted"); + } + auto view = base; + view.range = intersection(base.range, range); + output->bases.push_back(std::move(view)); + } + for (const auto &run : source.overlays) { + if (run == nullptr) { + return Status::invalid_argument("logical range has an unowned overlay"); + } + const auto entries = run->entries(); + const size_t first = range.lower.has_value() ? run->lower_bound(Slice(*range.lower)) : 0; + const size_t end = range.upper.has_value() ? run->lower_bound(Slice(*range.upper)) : entries.size(); + if (first == end) { + continue; + } + if (end - first > limits.run.max_entries || end - first > limits.max_metadata_bytes / sizeof(EntryRef)) { + return Status::resource_exhausted("logical range overlay budget exhausted"); + } + if (first == 0 && end == entries.size()) { + output->overlays.push_back(run); + continue; + } + EntryStorage selected(entries.begin() + static_cast<ptrdiff_t>(first), + entries.begin() + static_cast<ptrdiff_t>(end), EntryAllocator(limits.metadata)); + std::shared_ptr<const OverlayBatch> view; + auto status = OverlayBatch::build_owned(run->id(), std::move(selected), limits.run, &view); + if (!status.ok()) { + return status; + } + output->overlays.push_back(std::move(view)); + } + return Status::Ok(); +} +} // namespace + +LeafSplitResult split_leaf_ranges(const LeafLayoutPlan &layout, std::span<const SplitTarget> targets, + const PageMaintenanceLimits &limits) +{ + LeafSplitResult result(limits.metadata); + if (!layout.status.ok() || layout.readiness != MaintenanceReadiness::kReady) { + result.status = layout.status; + result.readiness = layout.readiness; + return result; + } + if (layout.source == nullptr || targets.empty() || targets.size() != layout.separators.size() + 1 || + targets.size() != layout.primary_bytes.size() || targets.size() > limits.max_output_pages) { + result.status = Status::invalid_argument("split targets must match a complete captured layout"); + return result; + } + for (size_t i = 0; i < targets.size(); ++i) { + if (targets[i].page_id == kInvalidPageId || layout.primary_bytes[i] > limits.primary_page_bytes || + targets[i].version.route_generation <= layout.source->route_generation) { + result.status = Status::invalid_argument("split target needs a new route generation and valid layout"); + return result; + } + for (size_t j = 0; j < i; ++j) { + if (targets[i].page_id == targets[j].page_id) { + result.status = Status::invalid_argument("split target page IDs must be unique"); + return result; + } + } + if (i != 0 && (!layout.source->range.contains(Slice(layout.separators[i - 1])) || + (i > 1 && layout.separators[i - 2] >= layout.separators[i - 1]))) { + result.status = Status::invalid_argument("split separators must be ordered within the source range"); + return result; + } + } + result.pages.reserve(targets.size()); + for (size_t i = 0; i < targets.size(); ++i) { + auto page = detail::reserved_shared<PageState>(limits.metadata, limits.metadata); + page->range.lower = i == 0 ? layout.source->range.lower : std::optional(layout.separators[i - 1]); + page->range.upper = i + 1 == targets.size() ? layout.source->range.upper : std::optional(layout.separators[i]); + page->right_sibling = i + 1 == targets.size() ? layout.source->right_sibling : targets[i + 1].page_id; + set_version(page.get(), targets[i].version); + result.status = append_range(*layout.source, page->range, limits, page.get()); + if (!result.status.ok()) { + result.readiness = MaintenanceReadiness::kDeferred; + result.pages.clear(); + return result; + } + result.pages.push_back(std::move(page)); + } + return result; +} + +PageMaintenanceResult plan_leaf_merge(const LeafMergeInput &input, const PageMaintenanceLimits &limits, + PageVersion version, uint64_t first_run_id) +{ + PageMaintenanceResult result(limits.metadata); + if (input.left == nullptr || input.right == nullptr || input.left_page_id == input.right_page_id || + input.left_page_id == kInvalidPageId || input.right_page_id == kInvalidPageId || + input.left_parent == kInvalidPageId || input.left_parent != input.right_parent || + input.left->right_sibling != input.right_page_id || !input.left->range.upper.has_value() || + input.left->range.upper != input.right->range.lower || + version.route_generation <= std::max(input.left->route_generation, input.right->route_generation) || + limits.merge_target_bytes >= limits.target_page_bytes) { + result.status = Status::invalid_argument("merge requires adjacent leaves of one parent and hysteresis"); + return result; + } + const auto left = plan_leaf_layout(input.left, limits); + const auto right = plan_leaf_layout(input.right, limits); + for (const auto *layout : {&left, &right}) { + if (!layout->status.ok()) { + result.status = layout->status; + result.readiness = layout->readiness; + return result; + } + if (layout->readiness == MaintenanceReadiness::kNeedsBase) { + result.readiness = MaintenanceReadiness::kNeedsBase; + result.loads.insert(result.loads.end(), layout->loads.begin(), layout->loads.end()); + } + } + if (result.readiness == MaintenanceReadiness::kNeedsBase) { + return result; + } + constexpr size_t overhead = kFrameHeaderSize + kFrameTrailerSize; + if (std::min(left.total_primary_bytes, right.total_primary_bytes) >= limits.merge_trigger_bytes || + left.total_primary_bytes > limits.merge_target_bytes || + right.total_primary_bytes - overhead > limits.merge_target_bytes - left.total_primary_bytes) { + return result; + } + auto joined = join_leaf_ranges(input, limits, version); + if (!joined.status.ok()) { + return joined; + } + return coalesce_overlays(std::move(joined.page), limits, version.representation_id, first_run_id); +} + +PageMaintenanceResult join_leaf_ranges(const LeafMergeInput &input, const PageMaintenanceLimits &limits, + PageVersion version) +{ + PageMaintenanceResult result(limits.metadata); + if (!input.left || !input.right || input.left_page_id == input.right_page_id || + input.left_page_id == kInvalidPageId || input.right_page_id == kInvalidPageId || + input.left_parent == kInvalidPageId || input.left_parent != input.right_parent || + input.left->right_sibling != input.right_page_id || !input.left->range.upper || + input.left->range.upper != input.right->range.lower || + version.route_generation <= std::max(input.left->route_generation, input.right->route_generation)) { + result.status = Status::invalid_argument("range join requires adjacent leaves of one parent and a new version"); + return result; + } + auto joined = detail::reserved_shared<PageState>(limits.metadata, limits.metadata); + joined->range = {.lower = input.left->range.lower, .upper = input.right->range.upper}; + joined->right_sibling = input.right->right_sibling; + set_version(joined.get(), version); + for (const auto &source : {input.left, input.right}) { + result.status = append_range(*source, source->range, limits, joined.get()); + if (!result.status.ok()) { + result.readiness = MaintenanceReadiness::kDeferred; + return result; + } + } + result.page = std::move(joined); + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_split_operation.cpp b/lib/crowdb-tree/src/btree/page_split_operation.cpp new file mode 100644 index 000000000..0036c1bf8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_split_operation.cpp @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_split_operation.h" + +#include "btree/allocation_bound.h" +#include "btree/maintenance_operation_internal.h" + +namespace crowdb::tree +{ +Status PageSplitOperation::create(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + std::shared_ptr<const PublishedTreeState> view, uint64_t pid, + PageMaintenanceOperationOptions options, std::shared_ptr<PageLoader> loader, + std::shared_ptr<ApplyDispatcher> progress, std::shared_ptr<const void> resources, + TreeWorkCause cause, std::shared_ptr<PageSplitOperation> *out) +try { + const auto page = view ? view->mapping.get(pid) : nullptr; + if (!page || !page->leaf || page->level != 0 || !lane || !progress || !options.prepaid || + options.prepaid->category() != detail::BudgetClass::kReclaim || options.publication.metadata || + options.acquire || out == nullptr || *out) { + return Status::invalid_argument("fixed split requires a leaf and complete owned progress credit"); + } + auto credit = std::exchange(options.prepaid, {}); + if (view->sources && !view->sources->tables.empty()) { + auto fixed = detail::reserved_shared<PublishedTreeState>(credit, *view); + auto sources = detail::reserved_shared<PublishedSources>(credit, credit); + sources->floor = fixed->coverage; + fixed->sources = std::move(sources); + view = std::move(fixed); + } + options.max_candidates = options.max_steps = options.max_visits = options.visits_per_step = 1; + options.max_stale_replans = 0; + auto factory = detail::reserved_shared<detail::MaintenanceFactory>( + credit, publisher, runtime, std::move(lane), std::move(options), std::move(loader), std::move(progress)); + auto state = detail::reserved_shared<detail::MaintenanceOperation>(credit, std::move(factory), view->generation, + view->stats.dirty_pages, + FlushBatch::MaintenanceComplete{}, credit); + if (state->activity_slot == PublicationCoordinator::kMaintenanceSlots) { + return Status::unavailable("checkpoint split activity admission unavailable"); + } + state->target_view = std::move(view); + state->target_pid = pid; + state->cause = cause; + *out = detail::reserved_construct<PageSplitOperation>(credit, [&](auto *storage) { + ::new (static_cast<void *>(storage)) PageSplitOperation(std::move(state), std::move(resources)); + }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +size_t PageSplitOperation::control_allocation_bound() noexcept +{ + // The maintenance reserve already includes its factory and operation. + return detail::shared_allocation_bound<PageSplitOperation>() + + detail::shared_allocation_bound<PublishedTreeState>() + detail::shared_allocation_bound<PublishedSources>(); +} + +detail::CallbackSender PageSplitOperation::sender() noexcept +{ + return {this, &PageSplitOperation::submit}; +} + +void PageSplitOperation::request_stop() noexcept +{ + state_->target_stopped.store(true, std::memory_order_release); +} + +uint64_t PageSplitOperation::commit_sequence() const noexcept +{ + return state_->target_commit_seq; +} + +uint64_t PageSplitOperation::l1_sequence() const noexcept +{ + return state_->target_l1_seq; +} + +std::span<const uint64_t> PageSplitOperation::output_leaf_pids() const noexcept +{ + return state_->outputs; +} + +const std::shared_ptr<const PublishedTreeState> &PageSplitOperation::captured_view() const noexcept +{ + return state_->target_view; +} + +void PageSplitOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto &self = *static_cast<PageSplitOperation *>(source); + auto state = self.state_; + if (state->target_started.exchange(true, std::memory_order_acq_rel)) { + complete(context, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + state->done = [complete, context](Status status, uint64_t) { + complete(context, status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError, + std::move(status)); + }; + state->lifetime = state; + state->schedule(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_split_operation.h b/lib/crowdb-tree/src/btree/page_split_operation.h new file mode 100644 index 000000000..f1f2b4ece --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_split_operation.h @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/maintenance_operation.h" + +#include <span> + +namespace crowdb::tree +{ +namespace detail +{ +struct MaintenanceOperation; +} + +// One exact captured leaf and at most one bounded structural transaction. +// No source handoff, candidate scan or unrelated merge is performed. +class PageSplitOperation +{ + public: + static Status create(PublicationCoordinator &publisher, detail::FlushRuntime &runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + std::shared_ptr<const PublishedTreeState> view, uint64_t pid, + PageMaintenanceOperationOptions options, std::shared_ptr<PageLoader> loader, + std::shared_ptr<ApplyDispatcher> progress, std::shared_ptr<const void> resources, + TreeWorkCause cause, std::shared_ptr<PageSplitOperation> *out); + [[nodiscard]] static size_t control_allocation_bound() noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + void request_stop() noexcept; + [[nodiscard]] uint64_t commit_sequence() const noexcept; + [[nodiscard]] uint64_t l1_sequence() const noexcept; + [[nodiscard]] std::span<const uint64_t> output_leaf_pids() const noexcept; + [[nodiscard]] const std::shared_ptr<const PublishedTreeState> &captured_view() const noexcept; + + private: + explicit PageSplitOperation(std::shared_ptr<detail::MaintenanceOperation> state, + std::shared_ptr<const void> resources) + : state_(std::move(state)), + resources_(std::move(resources)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<detail::MaintenanceOperation> state_; + std::shared_ptr<const void> resources_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_writeback.cpp b/lib/crowdb-tree/src/btree/page_writeback.cpp new file mode 100644 index 000000000..c400de729 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_writeback.cpp @@ -0,0 +1,219 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "backend/chunk/chunk_backing.h" +#include "backend/local/scheduled_page_store.h" +#include "page_writeback_internal.h" + +#include <bit> +#include <cassert> + +namespace crowdb::tree +{ +namespace +{ +size_t padded_blob_bound(PageWriteLimits limits) noexcept +{ + const auto raw = limits.max_frame_bytes + kDurableBlobHeader; + return ((raw + limits.iu_size - 1) / limits.iu_size) * limits.iu_size; +} + +} // namespace + +std::optional<size_t> PageWriteOperation::allocation_bound(PageWriteLimits limits) noexcept +{ + if (limits.max_frame_bytes < kFrameHeaderSize + kFrameTrailerSize || + limits.max_frame_bytes > UINT32_MAX - kDurableBlobHeader || limits.iu_size == 0 || + !std::has_single_bit(limits.buffer_alignment) || + (limits.iu_size > 1 && (limits.buffer_alignment < limits.iu_size || limits.buffer_alignment < 4096)) || + (limits.compression != compress_algo::kNone && limits.compression != compress_algo::kLz4)) { + return {}; + } + return detail::AllocationBound( + detail::shared_allocation_bound<State>() + detail::shared_allocation_bound<PageWriteOperation>() + + detail::shared_allocation_bound<BackingRef>() + detail::shared_allocation_bound<State::Protection>()) + .add(padded_blob_bound(limits)) + .add(limits.buffer_alignment - 1) + .add(durable_encode_scratch_bytes(limits.compression)) + .add(limits.chunk_backed ? detail::ChunkFixedPackStore::write_allocation_bound(padded_blob_bound(limits)) + : std::optional<size_t>{0}) + .add(limits.chunk_backed ? detail::shared_allocation_bound<detail::AllocationReservation>() : 0) + .add(limits.chunk_backed ? chunk_backing_allocation_bound() : 0) + .value(); +} + +size_t PageWriteOperation::retained_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<BackingRef>() + detail::shared_allocation_bound<State::Protection>(); +} + +PageWriteOperation::State::State(PageWriteInput source, PageWriteLimits bounds, PageLoadBudget credits) + : input(std::move(source)), + limits(bounds), + budget(std::move(credits)), + io_credit(budget.io, 1), + io_resources(budget.bytes, TreeResourceMetric::WriteInflightBytes), + blob(padded_blob_bound(limits) + limits.buffer_alignment - 1, + detail::ReservationAllocator<uint8_t, AllocationKind::kIo>(budget.bytes)), + scratch(durable_encode_scratch_bytes(limits.compression), + detail::ReservationAllocator<uint8_t, AllocationKind::kScratch>(budget.bytes)), + backing(detail::reserved_shared<BackingRef>(budget.bytes)), + protection(detail::reserved_shared<Protection>(budget.bytes)) +{ + void *data = blob.data(); + auto capacity = blob.size(); + const auto bytes = padded_blob_bound(limits); + auto *address = std::align(limits.buffer_alignment, bytes, data, capacity); + assert(address != nullptr); + aligned = {static_cast<uint8_t *>(address), bytes}; + protection->dependencies = input.reload_dependencies; + backing->owner = protection; + if (limits.chunk_backed) { + const auto bound = detail::ChunkFixedPackStore::write_allocation_bound(padded_blob_bound(limits)); + if (!bound || !budget.bytes) { + throw std::bad_alloc(); + } + chunk_credit = detail::AllocationReservation::create(budget.bytes, *bound); + } +} + +Status PageWriteOperation::create(std::shared_ptr<AsyncPageStore> store, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageWriteInput input, + PageWriteLimits limits, PageLoadBudget budget, ReserveExtent reserve, + std::shared_ptr<PageWriteOperation> *out) +{ + if (!store || !runtime || !lane || !reserve || out == nullptr || !allocation_bound(limits) || !input.owner || + input.frame.size() != input.info.bytes || input.frame.size() > limits.max_frame_bytes || + input.frame.size() < kFrameHeaderSize + kFrameTrailerSize || + static_cast<bool>(budget.bytes) != static_cast<bool>(budget.io) || + (budget.bytes && + ((budget.bytes->category() != detail::BudgetClass::kRouting && + budget.bytes->category() != detail::BudgetClass::kReclaim) || + budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + return Status::invalid_argument("page write requires owned input and matching byte/I/O credit"); + } + if (limits.chunk_backed ? dynamic_cast<ScheduledPageStore *>(store.get()) == nullptr + : !store->write_completion_is_reloadable()) { + return Status::not_supported("backend write needs checkpoint/seal before it is independently reloadable"); + } + try { + auto state = detail::reserved_shared<State>(budget.bytes, std::move(input), limits, budget); + state->store = std::move(store); + state->runtime = std::move(runtime); + state->lane = std::move(lane); + state->reserve = std::move(reserve); + *out = detail::reserved_construct<PageWriteOperation>(budget.bytes, [&](auto *storage) { + ::new (static_cast<void *>(storage)) PageWriteOperation(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +detail::CallbackSender PageWriteOperation::sender() noexcept +{ + return {this, &PageWriteOperation::submit}; +} + +const PageWriteResult &PageWriteOperation::result() const noexcept +{ + return state_->report; +} + +void PageWriteOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); +} + +void PageWriteOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<PageWriteOperation *>(source)->state_->start(complete, context); +} + +void PageWriteOperation::State::start(detail::CallbackComplete callback, void *context) noexcept +{ + if (started.exchange(true, std::memory_order_acq_rel)) { + callback(context, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + pending = weak_from_this().lock(); + complete = callback; + receiver = context; + if (budget.cpu_progress) { + budget.cpu_progress->request(encode_node, pending); + return; + } + Status status; + try { + status = runtime->try_submit(lane, &State::encode, this); + } + catch (...) { + status = Status::resource_exhausted(); + } + if (!status.ok()) { + finish(std::move(status)); + } +} + +void PageWriteOperation::State::encode(void *context) noexcept +{ + auto self = static_cast<State *>(context)->pending; + assert(self); + if (self->stopped.load(std::memory_order_acquire)) { + self->finish(Status::unavailable()); + return; + } + try { + self->prepare(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted()); + } + catch (...) { + self->finish(Status::io_error()); + } +} + +void PageWriteOperation::State::written(void *context, Status status) noexcept +{ + auto *state = static_cast<State *>(context); + if (state->io_complete.exchange(true, std::memory_order_acq_rel)) { + return; + } + if (status.ok() && state->backing->chunk) { + state->backing->physical_metrics = state->backing->chunk->physical_metrics; + } + state->io_resources.reset(); + state->io_credit.reset(); + state->finish(std::move(status)); +} + +void PageWriteOperation::State::finish(Status status) noexcept +{ + if (finished.exchange(true, std::memory_order_acq_rel)) { + return; + } + if (chunk_credit) { + chunk_credit->seal(); + } + const bool cancelled = stopped.load(std::memory_order_acquire); + report.status = cancelled ? Status::unavailable() : std::move(status); + if (report.status.ok()) { + report.backing = backing; + } + if (!io_submitted) { + io_credit.reset(); + } + input.owner.reset(); + input.info.dependencies.reset(); + input.reload_dependencies.reset(); + auto self = std::move(pending); + auto signal = report.status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + if (cancelled) { + signal = detail::CallbackSignal::kStopped; + } + complete(receiver, signal, report.status.without_message()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_writeback.h b/lib/crowdb-tree/src/btree/page_writeback.h new file mode 100644 index 000000000..f078bf6c9 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_writeback.h @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/maptable/compressor.h" +#include "page_encode.h" +#include "page_load.h" + +namespace crowdb::tree +{ +struct PageWriteLimits +{ + size_t max_frame_bytes = 64 * 1024; + uint32_t iu_size = 4096; + size_t buffer_alignment = 4096; + compress_algo compression = compress_algo::kLz4; + bool chunk_backed = false; +}; + +struct PageWriteInput +{ + EncodedFrameInfo info; + std::span<const uint8_t> frame; + std::shared_ptr<const void> owner; + // Protects actual referenced overflow extents and their captured resolver. + // The encoder's PID reservation alone does not protect those byte ranges. + // This is retained by the resulting backing, independently of raw input. + std::shared_ptr<const void> reload_dependencies; +}; + +struct ReservedWriteExtent +{ + uint64_t backend_id = 0; + uint64_t address = 0; + uint64_t length = 0; + uint64_t content_id = 0; + uint64_t generation = 0; + // Held before submission, including on error/unknown write outcome, until + // the final transport callback and every published backing release it. + std::shared_ptr<const void> owner; + std::weak_ptr<BackingMetricExtent> physical_metrics; + std::weak_ptr<ChunkPackBacking> chunk; +}; + +struct PageWriteResult +{ + Status status; + std::shared_ptr<const BackingRef> backing; + uint32_t logical_bytes = 0; + uint64_t physical_bytes = 0; +}; + +class PageWriteOperation +{ + struct State; + + public: + using ExtentComplete = detail::InlineFunction<void(Status, ReservedWriteExtent), 32>; + using ReserveExtent = detail::InlineFunction<void(uint64_t, ExtentComplete)>; + static Status create(std::shared_ptr<AsyncPageStore> store, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageWriteInput input, PageWriteLimits limits, + PageLoadBudget budget, ReserveExtent reserve, std::shared_ptr<PageWriteOperation> *out); + [[nodiscard]] static std::optional<size_t> allocation_bound(PageWriteLimits limits) noexcept; + [[nodiscard]] static size_t retained_allocation_bound() noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const PageWriteResult &result() const noexcept; + // In-flight I/O is drained; stop never releases/reuses its buffer early. + void request_stop() noexcept; + + private: + explicit PageWriteOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_writeback_encode.cpp b/lib/crowdb-tree/src/btree/page_writeback_encode.cpp new file mode 100644 index 000000000..3cf1f1bd9 --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_writeback_encode.cpp @@ -0,0 +1,163 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/backing_io.h" +#include "backend/chunk/chunk_backing.h" +#include "page_writeback_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +void PageWriteOperation::State::prepare() +{ + const auto *frame = input.frame.data(); + if (!frame_validate(frame, input.info.bytes) || frame_page_type(frame) != input.info.type || + frame_u64(frame, fh::kSelfpage_id) != input.info.page_id) { + finish(Status::corruption("write input does not match its immutable frame identity")); + return; + } + bool needs_dependencies = false; + if (input.info.type == page_type::kLeafBase) { + LeafFrameView leaf(frame, input.info.bytes); + for (size_t i = 0; i < leaf.count(); ++i) { + needs_dependencies = needs_dependencies || CellView(leaf.cell(i)).is_overflow(); + } + } + else if (input.info.type == page_type::kOverflowFrame) { + needs_dependencies = OverflowFrameView(frame, input.info.bytes).next_page_id() != kInvalidPageId; + } + else if (input.info.type != page_type::kInnerBase) { + finish(Status::invalid_argument("only final frame kinds may be written")); + return; + } + if (needs_dependencies && !input.reload_dependencies) { + finish(Status::invalid_argument("overflow write requires captured extent dependencies")); + return; + } + auto status = encode_durable_page_into(input.frame, limits.compression, aligned, scratch, &report.logical_bytes); + if (!status.ok()) { + finish(std::move(status)); + return; + } + report.physical_bytes = ((uint64_t{report.logical_bytes} + limits.iu_size - 1) / limits.iu_size) * limits.iu_size; + auto padding = aligned.subspan(report.logical_bytes, report.physical_bytes - report.logical_bytes); + std::fill(padding.begin(), padding.end(), uint8_t{0}); + try { + reserve(report.physical_bytes, [self = shared_from_this()](Status status, ReservedWriteExtent extent) { + self->reserved(std::move(status), std::move(extent)); + }); + } + catch (const std::bad_alloc &) { + finish(Status::resource_exhausted()); + } + catch (...) { + finish(Status::io_error()); + } + if ((reservation_events.fetch_or(2, std::memory_order_acq_rel) & 1U) != 0) { + queue_write(); + } +} + +void PageWriteOperation::State::reserved(Status status, ReservedWriteExtent extent) noexcept +{ + if (reservation_claimed.exchange(true, std::memory_order_acq_rel) || finished.load(std::memory_order_acquire)) { + return; + } + reservation_status = std::move(status); + reserved_extent = std::move(extent); + if ((reservation_events.fetch_or(1, std::memory_order_acq_rel) & 2U) != 0) { + queue_write(); + } +} + +void PageWriteOperation::State::queue_write() noexcept +{ + if (finished.load(std::memory_order_acquire)) { + reserved_extent.owner.reset(); + return; + } + if (budget.cpu_progress) { + budget.cpu_progress->request(write_node, pending); + return; + } + Status status; + try { + status = runtime->try_submit(lane, &State::write, this); + } + catch (...) { + status = Status::resource_exhausted(); + } + if (!status.ok()) { + reserved_extent.owner.reset(); + finish(std::move(status)); + } +} + +void PageWriteOperation::State::write(void *context) noexcept +{ + auto self = static_cast<State *>(context)->pending; + try { + self->submit_write(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted()); + } + catch (...) { + self->finish(Status::io_error()); + } +} + +void PageWriteOperation::State::submit_write() +{ + auto extent = std::move(reserved_extent); + if (!reservation_status.ok()) { + finish(std::move(reservation_status)); + return; + } + if (!extent.owner || extent.length != report.physical_bytes || extent.address % limits.iu_size != 0 || + extent.address > UINT64_MAX - extent.length) { + finish(Status::invalid_argument("backend allocation must own an aligned nonwrapping write extent")); + return; + } + auto chunk = extent.chunk.lock(); + if (static_cast<bool>(chunk) != limits.chunk_backed || + (chunk && (extent.address != 0 || !chunk_backing_matches(*chunk, extent.backend_id, extent.length)))) { + finish(Status::invalid_argument("backend reservation differs from its admitted storage capability")); + return; + } + protection->extent = std::move(extent.owner); + backing->physical_owner = protection->extent; + backing->physical_metrics = extent.physical_metrics.lock(); + backing->backend_id = extent.backend_id; + backing->address = extent.address; + backing->length = extent.length; + backing->content_id = extent.content_id; + backing->generation = extent.generation; + if (chunk) { + backing->storage = BackingStorage::kChunkPack; + backing->chunk = chunk; + } + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + auto self = shared_from_this(); + io_resources.begin(blob.capacity()); + io_submitted = true; + if (chunk) { + submit_chunk_write(*store, std::move(chunk), aligned.first(report.physical_bytes), chunk_credit, + {.context = this, + .complete_fn = &State::written, + .owner = std::move(self), + .allocations = budget.backend, + .cause = budget.cause}); + return; + } + store->submit_typed_write(PagePurpose::kBtreePage, backing->address, aligned.data(), report.physical_bytes, + {.context = this, + .complete_fn = &State::written, + .owner = std::move(self), + .allocations = budget.backend, + .cause = budget.cause}); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_writeback_internal.h b/lib/crowdb-tree/src/btree/page_writeback_internal.h new file mode 100644 index 000000000..75aa9d01d --- /dev/null +++ b/lib/crowdb-tree/src/btree/page_writeback_internal.h @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/apply_dispatcher.h" +#include "page_writeback.h" +#include "resource_observation.h" + +namespace crowdb::tree +{ +struct PageWriteOperation::State : std::enable_shared_from_this<State> +{ + struct Protection + { + std::shared_ptr<const void> extent; + std::shared_ptr<const void> dependencies; + }; + + State(PageWriteInput source, PageWriteLimits bounds, PageLoadBudget credits); + void start(detail::CallbackComplete callback, void *context) noexcept; + static void encode(void *context) noexcept; + static void write(void *context) noexcept; + static void written(void *context, Status status) noexcept; + void prepare(); + void reserved(Status status, ReservedWriteExtent extent) noexcept; + void queue_write() noexcept; + void submit_write(); + void finish(Status status) noexcept; + + ApplyDispatcher::Node encode_node{&State::encode, this}; + ApplyDispatcher::Node write_node{&State::write, this}; + PageWriteInput input; + PageWriteLimits limits; + PageLoadBudget budget; + std::shared_ptr<detail::AllocationReservation> chunk_credit; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + ReserveExtent reserve; + AllocationLease io_credit; + ResourceObservation io_resources; + detail::ReservedVector<uint8_t, AllocationKind::kIo> blob; + detail::ReservedVector<uint8_t, AllocationKind::kScratch> scratch; + std::span<uint8_t> aligned; + std::shared_ptr<BackingRef> backing; + std::shared_ptr<Protection> protection; + PageWriteResult report; + ReservedWriteExtent reserved_extent; + Status reservation_status; + std::atomic<bool> reservation_claimed{false}; + std::atomic<unsigned> reservation_events{0}; + std::shared_ptr<State> pending; + detail::CallbackComplete complete = nullptr; + void *receiver = nullptr; + std::atomic<bool> started{false}; + std::atomic<bool> stopped{false}; + std::atomic<bool> finished{false}; + bool io_submitted = false; + std::atomic<bool> io_complete{false}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/pending_page_times.cpp b/lib/crowdb-tree/src/btree/pending_page_times.cpp new file mode 100644 index 000000000..7dbc5c3f3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/pending_page_times.cpp @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "pending_page_times.h" + +#include <cassert> +#include <utility> + +namespace crowdb::tree +{ +PendingPageTimes::PendingPageTimes(std::pmr::memory_resource *resource) : pages_(resource) +{ +} + +bool PendingPageTimes::insert(uint64_t pid, uint64_t since_ns) +{ + const auto [position, inserted] = pages_.try_emplace(pid); + if (!inserted) { + return false; + } + auto &entry = position->second; + assert(tail_ == nullptr || tail_->since_ns <= since_ns); + entry.since_ns = since_ns; + entry.previous = tail_; + if (tail_ != nullptr) { + tail_->next = &entry; + } + else { + head_ = &entry; + } + tail_ = &entry; + return true; +} + +void PendingPageTimes::unlink(Entry &entry) noexcept +{ + if (entry.previous != nullptr) { + entry.previous->next = entry.next; + } + else { + head_ = entry.next; + } + if (entry.next != nullptr) { + entry.next->previous = entry.previous; + } + else { + tail_ = entry.previous; + } +} + +void PendingPageTimes::erase(uint64_t pid) noexcept +{ + const auto position = pages_.find(pid); + if (position != pages_.end()) { + unlink(position->second); + pages_.erase(position); + } +} + +void PendingPageTimes::erase_first() noexcept +{ + if (!pages_.empty()) { + unlink(pages_.begin()->second); + pages_.erase(pages_.begin()); + } +} + +void PendingPageTimes::swap(PendingPageTimes &other) noexcept +{ + pages_.swap(other.pages_); + std::swap(head_, other.head_); + std::swap(tail_, other.tail_); +} + +bool PendingPageTimes::contains(uint64_t pid) const +{ + return pages_.contains(pid); +} + +bool PendingPageTimes::empty() const noexcept +{ + return pages_.empty(); +} + +size_t PendingPageTimes::size() const noexcept +{ + return pages_.size(); +} + +uint64_t PendingPageTimes::oldest_since_ns() const noexcept +{ + return head_ == nullptr ? 0 : head_->since_ns; +} + +std::pmr::memory_resource *PendingPageTimes::resource() const noexcept +{ + return pages_.get_allocator().resource(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/pending_page_times.h b/lib/crowdb-tree/src/btree/pending_page_times.h new file mode 100644 index 000000000..238b16c5b --- /dev/null +++ b/lib/crowdb-tree/src/btree/pending_page_times.h @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <cstdint> +#include <map> +#include <memory_resource> + +namespace crowdb::tree +{ +// Coordinator-owned membership. Map nodes also form a chronological list, so +// the oldest pending timestamp needs no scan or additional index allocation. +class PendingPageTimes +{ + struct Entry + { + uint64_t since_ns = 0; + Entry *previous = nullptr; + Entry *next = nullptr; + }; + + public: + explicit PendingPageTimes(std::pmr::memory_resource *resource); + PendingPageTimes(const PendingPageTimes &) = delete; + PendingPageTimes &operator=(const PendingPageTimes &) = delete; + bool insert(uint64_t pid, uint64_t since_ns); + void erase(uint64_t pid) noexcept; + void erase_first() noexcept; + void swap(PendingPageTimes &other) noexcept; + [[nodiscard]] bool contains(uint64_t pid) const; + [[nodiscard]] bool empty() const noexcept; + [[nodiscard]] size_t size() const noexcept; + [[nodiscard]] uint64_t oldest_since_ns() const noexcept; + [[nodiscard]] std::pmr::memory_resource *resource() const noexcept; + + private: + void unlink(Entry &entry) noexcept; + std::pmr::map<uint64_t, Entry> pages_; + Entry *head_ = nullptr; + Entry *tail_ = nullptr; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/portable_import.cpp b/lib/crowdb-tree/src/btree/portable_import.cpp new file mode 100644 index 000000000..3876d46c4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/portable_import.cpp @@ -0,0 +1,195 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "portable_import.h" + +namespace crowdb::tree +{ +PortableImportBuilder::PortableImportBuilder(const Config &config, PageLoadLimits loads, uint64_t coverage, + std::shared_ptr<detail::AllocationReservation> metadata) + : loads_(loads), + metadata_(std::move(metadata)), + catalog_(detail::reserved_shared<ReadablePageCatalog>(metadata_, metadata_)), + leaf_frame_(config.frame_bytes, detail::ReservationAllocator<uint8_t, AllocationKind::kScratch>(metadata_)), + scratch_(config.frame_bytes, detail::ReservationAllocator<uint8_t, AllocationKind::kScratch>(metadata_)), + level_(detail::ReservationAllocator<Node>(metadata_)) +{ + config_.frame_bytes = config.frame_bytes; + config_.max_key_size = config.max_key_size; + config_.max_inline_value = config.max_inline_value; + config_.inner_max_keys = config.inner_max_keys; + if (config.key_range.start()) { + range_.lower = OwnedKey(Slice(*config.key_range.start()), metadata_); + } + if (config.key_range.end()) { + range_.upper = OwnedKey(Slice(*config.key_range.end()), metadata_); + } + catalog_->coverage = coverage; + catalog_->generation = 1; +} + +std::optional<size_t> PortableImportBuilder::allocation_bound(const NativeImportShape &shape) noexcept +{ + // Each level's vector may coexist with its geometric-growth replacement + // and the next parent level. Boundary keys share storage between levels. + const auto key = detail::AllocationBound(OwnedKey::allocation_overhead()).add(shape.max_key_bytes).add(1).value(); + if (!key) { + return {}; + } + return detail::AllocationBound() + .add_product(2, shape.max_frame_bytes) + .add_product(shape.leaf_pages, 4 * sizeof(Node)) + .add_product(shape.leaf_pages, 2 * *key) + .add_product(4, *key) + .add_product(shape.primary_pages, sizeof(uint64_t) + sizeof(Slice)) + .add_product(3, detail::AllocationBound(kCellHeaderSize).add(shape.max_value_bytes).value().value_or(SIZE_MAX)) + .value(); +} + +Status PortableImportBuilder::reserve_pid(uint64_t *out) +{ + if (next_pid_ >= MappingView::kPageLimit || next_pid_ == NativeImportLimits{}.max_frames) { + return Status::resource_exhausted("portable import frame namespace is full"); + } + *out = next_pid_++; + return Status::Ok(); +} + +void PortableImportBuilder::begin_leaf(uint64_t pid) +{ + leaf_pid_ = pid; + leaf_.emplace(leaf_frame_.data(), leaf_frame_.size()); + first_ = {}; + used_ = kFrameHeaderSize + kFrameTrailerSize; + key_bytes_ = 0; +} + +Status PortableImportBuilder::validate(Slice key, Slice encoded) const +{ + const CellView cell(encoded); + if (!cell.valid() || cell.slot() > catalog_->coverage || cell.is_overflow() || + (cell.flags() != 0 && cell.flags() != kFlagTombstone) || + (cell.is_tombstone() && encoded.size() != kCellHeaderSize) || !range_.contains(key) || + (have_previous_ && Slice(previous_).compare(key) >= 0)) { + return Status::invalid_argument("portable import requires sorted cells within its range and coverage"); + } + const auto key_limit = config_.max_key_size != 0 ? config_.max_key_size : config_.frame_bytes / 2; + if (key.size() > key_limit || key.size() > loads_.max_leaf_key_bytes || + cell.value().size() > loads_.max_value_bytes) { + return Status::resource_exhausted("portable import key or value exceeds the Tree read/write limit"); + } + return Status::Ok(); +} + +Status PortableImportBuilder::save_leaf(uint64_t right) +{ + if (!leaf_) { + return Status::internal_error("portable import leaf is absent"); + } + leaf_->finish(leaf_pid_, right); + auto status = append_native_frame(*catalog_, leaf_pid_, leaf_frame_, config_, loads_, {}, metadata_); + if (!status.ok()) { + return status; + } + level_.push_back( + {.pid = leaf_pid_, .first = first_, .last = previous_, .first_leaf = leaf_pid_, .last_leaf = leaf_pid_}); + return Status::Ok(); +} + +Status PortableImportBuilder::append_impl(Slice key, Slice encoded) +{ + auto status = validate(key, encoded); + if (!status.ok()) { + return status; + } + const CellView cell(encoded); + const auto inline_limit = config_.max_inline_value != 0 ? config_.max_inline_value : config_.frame_bytes / 4; + const bool spill = !cell.is_tombstone() && (cell.value().size() > inline_limit || + key.size() + encoded.size() + kLeafSlotSize > + config_.frame_bytes - kFrameHeaderSize - kFrameTrailerSize); + const size_t needed = kLeafSlotSize + key.size() + (spill ? kOverflowCellSize : encoded.size()); + if (needed > config_.frame_bytes - kFrameHeaderSize - kFrameTrailerSize) { + return Status::resource_exhausted("portable import entry cannot fit a primary frame"); + } + if (!leaf_ || leaf_->count() == loads_.max_leaf_entries || key.size() > loads_.max_leaf_key_bytes - key_bytes_ || + needed > config_.frame_bytes - used_) { + uint64_t next = kInvalidPageId; + status = reserve_pid(&next); + if (!status.ok()) { + return status; + } + if (leaf_) { + status = save_leaf(next); + if (!status.ok()) { + return status; + } + } + begin_leaf(next); + } + std::array<uint8_t, kOverflowCellSize> overflow{}; + if (spill) { + status = spill_value(cell, &overflow); + if (!status.ok()) { + return status; + } + encoded = Slice(overflow.data(), overflow.size()); + } + auto owned_key = OwnedKey(key, metadata_); + if (!leaf_) { + return Status::internal_error("portable import leaf was not initialized"); + } + if (!leaf_->try_append_sorted(key, encoded)) { + return Status::internal_error("portable import frame preflight disagreed with its builder"); + } + if (leaf_->count() == 1) { + first_ = owned_key; + } + previous_ = std::move(owned_key); + have_previous_ = true; + used_ += needed; + key_bytes_ += key.size(); + return Status::Ok(); +} + +Status PortableImportBuilder::append(Slice key, Slice encoded) +try { + if (finished_) { + return Status::unavailable("portable import is already finished"); + } + if (status_.ok()) { + status_ = append_impl(key, encoded); + } + return status_; +} +catch (const std::bad_alloc &) { + status_ = Status::resource_exhausted(); + return status_; +} + +Status PortableImportBuilder::finish(std::shared_ptr<ReadablePageCatalog> *out) +try { + if (out == nullptr || *out || finished_) { + return Status::invalid_argument(); + } + finished_ = true; + if (!status_.ok()) { + return status_; + } + if (leaf_) { + status_ = save_leaf(kInvalidPageId); + if (status_.ok()) { + status_ = build_index(); + } + } + if (status_.ok()) { + status_ = attach_native_dependencies(*catalog_, {}, metadata_); + } + if (status_.ok()) { + *out = std::move(catalog_); + } + return status_; +} +catch (const std::bad_alloc &) { + status_ = Status::resource_exhausted(); + return status_; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/portable_import.h b/lib/crowdb-tree/src/btree/portable_import.h new file mode 100644 index 000000000..2a045db2f --- /dev/null +++ b/lib/crowdb-tree/src/btree/portable_import.h @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "native_import.h" +#include "owned_key.h" + +namespace crowdb::tree +{ +// Private import staging owns charged immutable frames and at most two frame +// windows. Input cells are borrowed only for append(); publication happens +// separately after the complete stream and its graph have been validated. +class PortableImportBuilder +{ + public: + PortableImportBuilder(const Config &config, PageLoadLimits loads, uint64_t coverage, + std::shared_ptr<detail::AllocationReservation> metadata); + Status append(Slice key, Slice encoded); + [[nodiscard]] static std::optional<size_t> allocation_bound(const NativeImportShape &shape) noexcept; + Status finish(std::shared_ptr<ReadablePageCatalog> *out); + + private: + struct Node + { + uint64_t pid = kInvalidPageId; + OwnedKey first; + OwnedKey last; + uint64_t first_leaf = kInvalidPageId; + uint64_t last_leaf = kInvalidPageId; + }; + + Status append_impl(Slice key, Slice encoded); + [[nodiscard]] Status validate(Slice key, Slice encoded) const; + Status save_leaf(uint64_t right); + Status spill_value(const CellView &cell, std::array<uint8_t, kOverflowCellSize> *encoded); + Status build_index(); + Status build_parent(std::span<const Node> nodes, Node *out); + Status reserve_pid(uint64_t *out); + void begin_leaf(uint64_t pid); + + Config config_; + PageLoadLimits loads_; + PageKeyRange range_; + std::shared_ptr<detail::AllocationReservation> metadata_; + std::shared_ptr<ReadablePageCatalog> catalog_; + detail::ReservedVector<uint8_t, AllocationKind::kScratch> leaf_frame_; + detail::ReservedVector<uint8_t, AllocationKind::kScratch> scratch_; + detail::ReservedVector<Node> level_; + std::optional<LeafFrameBuilder> leaf_; + OwnedKey first_; + OwnedKey previous_; + uint64_t next_pid_ = 0; + uint64_t leaf_pid_ = kInvalidPageId; + size_t used_ = kFrameHeaderSize + kFrameTrailerSize; + size_t key_bytes_ = 0; + bool have_previous_ = false; + bool finished_ = false; + Status status_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/portable_import_frames.cpp b/lib/crowdb-tree/src/btree/portable_import_frames.cpp new file mode 100644 index 000000000..cb6fc9268 --- /dev/null +++ b/lib/crowdb-tree/src/btree/portable_import_frames.cpp @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "portable_import.h" + +#include <algorithm> + +namespace crowdb::tree +{ +Status PortableImportBuilder::spill_value(const CellView &cell, std::array<uint8_t, kOverflowCellSize> *encoded) +{ + const auto value = cell.value(); + const auto head = next_pid_; + size_t offset = 0; + while (offset != value.size()) { + uint64_t pid = kInvalidPageId; + auto status = reserve_pid(&pid); + if (!status.ok()) { + return status; + } + const auto count = std::min<size_t>(value.size() - offset, overflow_chunk_cap(config_.frame_bytes)); + const auto next = offset + count == value.size() ? kInvalidPageId : next_pid_; + overflow_frame_build(scratch_.data(), scratch_.size(), pid, next, + reinterpret_cast<const uint8_t *>(value.data() + offset), count); + status = append_native_frame(*catalog_, pid, scratch_, config_, loads_, {}, metadata_); + if (!status.ok()) { + return status; + } + offset += count; + } + frame_put_u64(encoded->data(), 0, cell.slot()); + (*encoded)[sizeof(uint64_t)] = kFlagOverflow; + frame_put_u64(encoded->data(), kCellHeaderSize, head); + frame_put_u64(encoded->data(), kCellHeaderSize + sizeof(uint64_t), value.size()); + return Status::Ok(); +} + +Status PortableImportBuilder::build_parent(std::span<const Node> nodes, Node *out) +{ + detail::ReservedVector<uint64_t> children{detail::ReservationAllocator<uint64_t>(metadata_)}; + detail::ReservedVector<Slice> separators{detail::ReservationAllocator<Slice>(metadata_)}; + children.reserve(nodes.size()); + separators.reserve(nodes.size() - 1); + for (size_t i = 0; i < nodes.size(); ++i) { + children.push_back(nodes[i].pid); + if (i != 0) { + separators.push_back(Slice(nodes[i].first)); + } + } + uint64_t pid = kInvalidPageId; + auto status = reserve_pid(&pid); + if (!status.ok()) { + return status; + } + if (!inner_frame_build_span(scratch_.data(), scratch_.size(), pid, children, separators)) { + return Status::internal_error("portable import index preflight disagreed with its builder"); + } + frame_set_inner_fence_pages(scratch_.data(), scratch_.size(), nodes.front().first_leaf, nodes.back().last_leaf); + status = append_native_frame(*catalog_, pid, scratch_, config_, loads_, {}, metadata_); + if (status.ok()) { + *out = {.pid = pid, + .first = nodes.front().first, + .last = nodes.back().last, + .first_leaf = nodes.front().first_leaf, + .last_leaf = nodes.back().last_leaf}; + } + return status; +} + +Status PortableImportBuilder::build_index() +{ + const auto max_children = + std::min<size_t>(static_cast<size_t>(config_.inner_max_keys) + 1, loads_.max_inner_children); + while (level_.size() > 1) { + detail::ReservedVector<Node> parents{detail::ReservationAllocator<Node>(metadata_)}; + size_t begin = 0; + while (begin != level_.size()) { + size_t count = 1; + size_t keys = 0; + size_t used = kFrameHeaderSize + kFrameTrailerSize + sizeof(uint64_t); + while (count < max_children && count != level_.size() - begin) { + const auto bytes = sizeof(uint64_t) + kInnerSlotSize + level_[begin + count].first.size(); + if (bytes > config_.frame_bytes - used || + level_[begin + count].first.size() > loads_.max_inner_key_bytes - keys) { + break; + } + used += bytes; + keys += level_[begin + count].first.size(); + ++count; + } + if (count == 1 && level_.size() - begin > 1) { + return Status::resource_exhausted("portable import separator cannot fit a parent frame"); + } + Node parent; + auto status = build_parent(std::span<const Node>(level_).subspan(begin, count), &parent); + if (!status.ok()) { + return status; + } + parents.push_back(std::move(parent)); + begin += count; + } + level_ = std::move(parents); + } + catalog_->root_page_id = level_.front().pid; + catalog_->next_page_id = next_pid_; + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/portable_import_plan.cpp b/lib/crowdb-tree/src/btree/portable_import_plan.cpp new file mode 100644 index 000000000..f9aaae117 --- /dev/null +++ b/lib/crowdb-tree/src/btree/portable_import_plan.cpp @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "portable_import_plan.h" + +namespace crowdb::tree +{ +Status PortableImportPlan::append(size_t key_bytes, size_t value_bytes, bool tombstone) +{ + if (config_.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || key_bytes > loads_.max_leaf_key_bytes || + value_bytes > loads_.max_value_bytes) { + return Status::resource_exhausted("range input exceeds import geometry"); + } + const auto cap = config_.frame_bytes - kFrameHeaderSize - kFrameTrailerSize; + const auto inline_limit = config_.max_inline_value != 0 ? config_.max_inline_value : config_.frame_bytes / 4; + auto inline_bytes = + detail::AllocationBound(kLeafSlotSize + kCellHeaderSize).add(key_bytes).add(value_bytes).value(); + if (!inline_bytes) { + return Status::resource_exhausted("range entry bound overflow"); + } + const bool spill = !tombstone && (value_bytes > inline_limit || *inline_bytes > cap); + const auto needed = + spill ? detail::AllocationBound(kLeafSlotSize + kOverflowCellSize).add(key_bytes).value() : inline_bytes; + if (!needed || *needed > cap) { + return Status::resource_exhausted("range entry cannot fit an import primary"); + } + if (leaf_count_ == 0 || leaf_count_ == loads_.max_leaf_entries || + key_bytes > loads_.max_leaf_key_bytes - leaf_keys_ || *needed > cap - leaf_used_) { + if (shape_.leaf_pages == NativeImportLimits{}.max_frames) { + return Status::resource_exhausted("range import leaf count limit"); + } + ++shape_.leaf_pages; + leaf_used_ = leaf_keys_ = leaf_count_ = 0; + } + leaf_used_ += *needed; + leaf_keys_ += key_bytes; + ++leaf_count_; + shape_.max_key_bytes = std::max(shape_.max_key_bytes, key_bytes); + shape_.max_value_bytes = std::max(shape_.max_value_bytes, value_bytes); + if (spill) { + const auto capacity = overflow_chunk_cap(config_.frame_bytes); + if (capacity == 0) { + return Status::resource_exhausted("range import has no overflow capacity"); + } + const auto frames = (value_bytes / capacity) + static_cast<size_t>(value_bytes % capacity != 0); + const auto total = detail::AllocationBound(shape_.dependency_steps).add(frames).value(); + if (!total || *total > NativeImportLimits{}.max_dependency_steps) { + return Status::resource_exhausted("range overflow dependency limit"); + } + shape_.dependency_steps = *total; + ++shape_.stored_references; + } + return Status::Ok(); +} + +std::optional<NativeImportShape> PortableImportPlan::finish() const noexcept +{ + auto result = shape_; + // Every parent consumes at least two children except a final unary tail. + // Repeated ceil(n/2) bounds the builder without retaining separator keys. + size_t level = result.leaf_pages; + result.primary_pages = level; + while (level > 1) { + level = (level / 2) + (level % 2); + const auto total = detail::AllocationBound(result.primary_pages).add(level).value(); + if (!total) { + return {}; + } + result.primary_pages = *total; + } + const auto total = detail::AllocationBound(result.primary_pages).add(result.dependency_steps).value(); + if (!total || *total > NativeImportLimits{}.max_frames) { + return {}; + } + result.frames = *total; + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/portable_import_plan.h b/lib/crowdb-tree/src/btree/portable_import_plan.h new file mode 100644 index 000000000..dce179079 --- /dev/null +++ b/lib/crowdb-tree/src/btree/portable_import_plan.h @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "native_import.h" + +namespace crowdb::tree +{ +// Allocation-free first pass over the exact fixed selection. Packing follows +// the builder's byte, entry-count, and key-byte limits; no payload is retained. +class PortableImportPlan +{ + public: + PortableImportPlan(const Config &config, PageLoadLimits loads) : config_(config), loads_(loads) + { + shape_.max_frame_bytes = config.frame_bytes; + for (const auto *key : {&config.key_range.start(), &config.key_range.end()}) { + if (*key) { + shape_.max_key_bytes = std::max(shape_.max_key_bytes, (*key)->size()); + } + } + } + + Status append(size_t key_bytes, size_t value_bytes, bool tombstone); + [[nodiscard]] std::optional<NativeImportShape> finish() const noexcept; + + private: + const Config &config_; + PageLoadLimits loads_; + NativeImportShape shape_; + size_t leaf_used_ = 0; + size_t leaf_keys_ = 0; + size_t leaf_count_ = 0; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/prepared_split_arm.cpp b/lib/crowdb-tree/src/btree/prepared_split_arm.cpp new file mode 100644 index 000000000..574c0e884 --- /dev/null +++ b/lib/crowdb-tree/src/btree/prepared_split_arm.cpp @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "prepared_split_internal.h" + +namespace crowdb::tree +{ +PreparedSplitOverlay::PreparedSplitOverlay(std::shared_ptr<State> state) : state_(std::move(state)) +{ +} + +PreparedSplitOverlay::~PreparedSplitOverlay() +{ + auto &self = *state_; + if (self.tail) { + self.tail->expire(); + } + if (self.domain) { + self.domain->detach(self.subscription); + } + if (self.metadata) { + self.metadata->seal(); + } + self.lease.reset(); + if (self.destination_gate) { + self.destination->state_->split_busy.clear(std::memory_order_release); + } + if (self.source_gate) { + self.source->state_->split_busy.clear(std::memory_order_release); + } +} + +Status PreparedSplitOverlay::arm(TreeCore &source, TreeCore &destination, SplitOverlayLimits limits, + std::shared_ptr<PreparedSplitOverlay> *out, uint64_t *start_frontier) +try { + if (&source == &destination || source.config().runtime != destination.config().runtime || limits.max_records == 0 || + limits.max_bytes == 0 || out == nullptr || *out || start_frontier == nullptr || + detail::FlushRuntime::on_worker_thread()) { + return Status::invalid_argument("split arm requires distinct same-host trees and a finite tail"); + } + const auto bound = detail::AllocationBound() + .add(SourceTail::allocation_bound(limits.max_records, limits.max_bytes)) + .add(SourceTailDomain::registration_bound()) + .add(detail::shared_allocation_bound<State>()) + .add(detail::shared_allocation_bound<PreparedSplitOverlay>()) + .add(detail::shared_allocation_bound<SplitSourceSession>()) + .add(PublishedSources::allocation_bound()) + .add(PublicationCoordinator::complete_view_reserve_bytes()) + .add(4 * sizeof(std::shared_ptr<MemTable>)) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .value(); + if (!bound) { + return Status::resource_exhausted("split tail bound overflow"); + } + auto reserved = + detail::ResourceBudget::try_reserve(destination.binding()->account(), detail::BudgetClass::kRouting, *bound); + if (!reserved) { + return Status::resource_exhausted("split tail credit unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*reserved)); + auto state = detail::reserved_shared<State>(metadata); + state->source = source.shared_from_this(); + state->destination = destination.shared_from_this(); + state->metadata = metadata; + state->tail = detail::reserved_shared<SourceTail>(metadata, metadata, limits.max_records, limits.max_bytes); + auto token = detail::reserved_construct<PreparedSplitOverlay>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) PreparedSplitOverlay(state); }); + { + TreeCore::State::Ingress parent(*source.state_); + TreeCore::State::Ingress child(*destination.state_); + if (parent.owner == nullptr || child.owner == nullptr) { + return Status::unavailable("split ingress closed"); + } + state->domain = source.state_->sources->tail_domain(); + state->source_identity = source.state_->sources->identity(); + state->source_gate = !source.state_->split_busy.test_and_set(std::memory_order_acquire); + if (!state->source_gate) { + return Status::unavailable("source split administration busy"); + } + state->destination_gate = !destination.state_->split_busy.test_and_set(std::memory_order_acquire); + if (!state->destination_gate) { + return Status::unavailable("destination split administration busy"); + } + if ((source.state_->split_sources && source.state_->split_sources->active) || + source.state_->split_sequence == UINT64_MAX) { + return Status::unavailable("source already has a fixed split prefix"); + } + state->session = detail::reserved_shared<SplitSourceSession>(metadata); + state->session->generation = source.capture()->generation; + state->session->session = ++source.state_->split_sequence; + state->session->limits = source.state_->sources->catalog()->active_budget->options(); + state->session->limits.max_entries = limits.max_records; + state->session->limits.max_gap_entries = 0; + state->session->limits.max_key_bytes = limits.max_bytes; + state->session->limits.max_gap_key_bytes = 0; + auto status = state->domain->arm(state->tail, metadata, &state->subscription); + if (!status.ok()) { + return status; + } + } + // No admission pause: only pre-registration writers belong to this epoch + // barrier. They release their observation guard after frontier.complete. + state->subscription->wait(); + TreeCore::State::Ingress parent(*source.state_); + if (parent.owner == nullptr || source.state_->sources->tail_domain() != state->domain || state->tail->expired()) { + return Status::unavailable("split tail expired before its starting frontier"); + } + // Pre-registration out-of-order writes may lie above the contiguous + // frontier. The private child must replay this complete high-watermark. + state->start = source.max_seen_slot(); + *start_frontier = state->start; + *out = std::move(token); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("split arm allocation exhausted its reservation"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/prepared_split_install.cpp b/lib/crowdb-tree/src/btree/prepared_split_install.cpp new file mode 100644 index 000000000..68238f0d3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/prepared_split_install.cpp @@ -0,0 +1,146 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "prepared_split_internal.h" + +namespace crowdb::tree +{ +Status PreparedSplitOverlay::prepare(uint64_t replayed_prefix) +try { + auto &self = *state_; + uint8_t expected = State::Armed; + if (detail::FlushRuntime::on_worker_thread() || + !self.phase.compare_exchange_strong(expected, State::Preparing, std::memory_order_acq_rel)) { + return Status::unavailable("split capability is not awaiting background preparation"); + } + const auto fail = [&](Status status) { + self.tail->expire(); + self.domain->detach(self.subscription); + if (self.lease) { + self.lease->cancel(); + } + self.phase.store(State::Failed, std::memory_order_release); + return status; + }; + if (replayed_prefix < self.start || replayed_prefix == UINT64_MAX || self.tail->expired() || + self.destination->contiguous_slot() != replayed_prefix) { + return fail(Status::invalid_argument("child replay must cover the armed observer start")); + } + auto status = self.destination->flush(); + if (!status.ok()) { + return fail(status); + } + const auto expected_child = self.destination->capture(); + const auto base = self.destination->publisher().capture_complete(); + if (!base || base->coverage != replayed_prefix || expected_child->coverage != replayed_prefix || + self.destination->max_seen_slot() != replayed_prefix || base->generation != expected_child->generation || + base->pending_handoff || base->flush_admission_owner != 0 || self.tail->expired()) { + return fail(Status::unavailable("child prefix changed before split preparation")); + } + // Complete L1 may be the current root with an empty active source. Only + // after flush and the exact P checks may its immutable mapping be copied + // without L0. Lease acquisition rechecks the expected root and drains any + // admitted writer before publication, rejecting a child that advanced. + auto candidate = detail::reserved_shared<PublishedTreeState>(self.metadata, *base); + auto empty_sources = detail::reserved_shared<PublishedSources>(self.metadata, self.metadata); + empty_sources->floor = replayed_prefix; + candidate->sources = std::move(empty_sources); + auto inherited = detail::reserved_shared<PublishedSources>(self.metadata, self.metadata); + inherited->floor = replayed_prefix; + inherited->tables.push_back(self.tail->table()); + // Private placeholder only. The preallocated replacement is unreachable + // by readers until try_install fills its exact F after freezing the delta. + inherited->inheritance = PublishedSources::Inheritance{.owner_identity = self.source_identity, + .session = self.session->session, + .frontier = replayed_prefix + 1, + .local_count = 0}; + SplitGeneration split{.expected = expected_child, .inherited = std::move(inherited)}; + status = TreeGenerationLease::prepare(*self.destination, std::move(candidate), &split, &self.lease); + if (!status.ok()) { + return fail(status); + } + status = self.lease->acquire(); + if (!status.ok()) { + return fail(status); + } + self.replayed = replayed_prefix; + self.phase.store(State::Ready, std::memory_order_release); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + state_->tail->expire(); + state_->domain->detach(state_->subscription); + if (state_->lease) { + state_->lease->cancel(); + } + state_->phase.store(State::Failed, std::memory_order_release); + return Status::resource_exhausted("split preparation exhausted prepaid storage"); +} +catch (...) { + state_->tail->expire(); + state_->domain->detach(state_->subscription); + if (state_->lease) { + state_->lease->cancel(); + } + state_->phase.store(State::Failed, std::memory_order_release); + return Status::internal_error("split preparation failed before installation"); +} + +Status PreparedSplitOverlay::try_install(uint64_t frontier, InstalledSplitOverlay *out) noexcept +{ + auto &self = *state_; + if (out == nullptr) { + return Status::invalid_argument(); + } + uint8_t expected = State::Ready; + if (!self.phase.compare_exchange_strong(expected, State::Installing, std::memory_order_acq_rel)) { + if (expected == State::Installed && frontier == self.installed) { + *out = {.generation = self.session->session, .frontier = self.installed}; + return Status::Ok(); + } + return Status::unavailable(); + } + TreeCore::State::Ingress parent(*self.source->state_); + if (parent.owner == nullptr || self.source->state_->sources->tail_domain() != self.domain || + frontier < self.replayed || frontier > self.source->contiguous_slot() || self.tail->expired()) { + self.phase.store(State::Failed, std::memory_order_release); + return Status::unavailable(); + } + if (!self.tail->freeze()) { + self.phase.store(self.tail->expired() ? State::Failed : State::Ready, std::memory_order_release); + return Status::unavailable(); + } + self.domain->detach(self.subscription); + auto status = self.lease->set_split_frontier(frontier, self.tail->table()->source_stats()); + if (status.ok()) { + status = self.lease->commit(); + } + if (!status.ok()) { + self.phase.store(State::Failed, std::memory_order_release); + return status; + } + self.session->tables[0] = self.tail->table(); + self.session->table_count = 1; + self.session->intervals[0] = {.floor = self.replayed, .frontier = frontier}; + self.session->interval_count = 1; + self.session->frontier = frontier; + self.session->active = true; + self.previous_session = std::move(self.source->state_->split_sources); + self.source->state_->split_sources = self.session; + self.installed = frontier; + self.destination->state_->split_busy.clear(std::memory_order_release); + self.source->state_->split_busy.clear(std::memory_order_release); + self.destination_gate = false; + self.source_gate = false; + self.phase.store(State::Installed, std::memory_order_release); + *out = {.generation = self.session->session, .frontier = frontier}; + return Status::Ok(); +} + +std::optional<uint64_t> PreparedSplitOverlay::committed_frontier() const noexcept +{ + if (state_->phase.load(std::memory_order_acquire) == State::Installed) { + return state_->installed; + } + return {}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/prepared_split_internal.h b/lib/crowdb-tree/src/btree/prepared_split_internal.h new file mode 100644 index 000000000..a6d686426 --- /dev/null +++ b/lib/crowdb-tree/src/btree/prepared_split_internal.h @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "prepared_split_overlay.h" +#include "tree_core_internal.h" +#include "tree_generation_lease.h" + +namespace crowdb::tree +{ +struct PreparedSplitOverlay::State +{ + enum Phase : uint8_t { Armed, Preparing, Ready, Installing, Installed, Failed }; + + // C callers may release either facade while retaining this capability. + // Core owners outlive the lease, its old controls, and source registration. + std::shared_ptr<TreeCore> source; + std::shared_ptr<TreeCore> destination; + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<SourceTailDomain> domain; + std::shared_ptr<SourceTail> tail; + std::shared_ptr<SourceTailDomain::Barrier> subscription; + std::shared_ptr<SplitSourceSession> session; + std::shared_ptr<SplitSourceSession> previous_session; + std::shared_ptr<TreeGenerationLease> lease; + std::atomic<uint8_t> phase{Armed}; + uint64_t source_identity = 0; + uint64_t start = 0; + uint64_t replayed = 0; + uint64_t installed = 0; + bool source_gate = false; + bool destination_gate = false; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/prepared_split_overlay.h b/lib/crowdb-tree/src/btree/prepared_split_overlay.h new file mode 100644 index 000000000..4009fbda2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/prepared_split_overlay.h @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/status.h" + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <optional> + +namespace crowdb::tree +{ +class TreeCore; + +struct SplitOverlayLimits +{ + size_t max_records = 0; + size_t max_bytes = 0; +}; + +struct InstalledSplitOverlay +{ + uint64_t generation = 0; + uint64_t frontier = 0; +}; + +// An independent bounded delta and the child's already prepared publication. +// Preparation is a host operation; installation never waits for maintenance. +class PreparedSplitOverlay +{ + struct State; + + public: + static Status arm(TreeCore &source, TreeCore &destination, SplitOverlayLimits limits, + std::shared_ptr<PreparedSplitOverlay> *out, uint64_t *start_frontier); + Status prepare(uint64_t replayed_prefix); + ~PreparedSplitOverlay(); + PreparedSplitOverlay(const PreparedSplitOverlay &) = delete; + PreparedSplitOverlay &operator=(const PreparedSplitOverlay &) = delete; + + Status try_install(uint64_t frontier, InstalledSplitOverlay *out) noexcept; + [[nodiscard]] std::optional<uint64_t> committed_frontier() const noexcept; + + private: + explicit PreparedSplitOverlay(std::shared_ptr<State> state); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_admission.cpp b/lib/crowdb-tree/src/btree/publication_admission.cpp new file mode 100644 index 000000000..9a721c07a --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_admission.cpp @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "publication_admission.h" + +namespace crowdb::tree +{ +Status validate_flush_admission(const PublishedTreeState ¤t, const PublicationTransaction &transaction) +{ + if (transaction.flush_admission) { + const auto &admission = *transaction.flush_admission; + if (admission.owner == 0 || admission.owner != transaction.flush_admission_owner || + !transaction.pages.empty() || transaction.sources || transaction.coverage || transaction.root_page_id || + transaction.handoff || transaction.checkpoint || transaction.next_page_id) { + return Status::invalid_argument("flush admission must be a standalone ownership transaction"); + } + if (admission.action == FlushAdmissionAction::Acquire) { + if (!transaction.expected_commit_seq) { + return Status::invalid_argument("flush reserve must name its captured publication sequence"); + } + if (current.flush_admission_owner != 0 || current.pending_handoff) { + return Status::unavailable("another flush owns publication admission"); + } + } + else if (admission.action == FlushAdmissionAction::Release) { + if (current.flush_admission_owner != admission.owner || current.pending_handoff) { + return Status::unavailable("flush admission cannot release another owner or a partial handoff"); + } + } + else { + return Status::invalid_argument("invalid flush admission action"); + } + return Status::Ok(); + } + if (current.flush_admission_owner != 0) { + if (transaction.coverage && (!transaction.handoff || *transaction.coverage <= current.coverage)) { + return Status::invalid_argument("admitted coverage must complete its forward handoff"); + } + const bool checkpoint_completion = + transaction.checkpoint && transaction.checkpoint->action != CheckpointAction::Capture; + if (transaction.flush_admission_owner != current.flush_admission_owner && !checkpoint_completion) { + return Status::unavailable("publication waits for the admitted flush boundary"); + } + } + else if (transaction.flush_admission_owner != 0) { + return Status::unavailable("flush no longer owns publication admission"); + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_admission.h b/lib/crowdb-tree/src/btree/publication_admission.h new file mode 100644 index 000000000..0d9838ee8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_admission.h @@ -0,0 +1,11 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "published_state.h" + +namespace crowdb::tree +{ +[[nodiscard]] Status validate_flush_admission(const PublishedTreeState ¤t, + const PublicationTransaction &transaction); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_bootstrap.cpp b/lib/crowdb-tree/src/btree/publication_bootstrap.cpp new file mode 100644 index 000000000..cb9240aeb --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_bootstrap.cpp @@ -0,0 +1,121 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "page_cache.h" +#include "publication_generation_internal.h" +#include "publication_snapshot.h" +#include "source_admission.h" + +#include <algorithm> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +std::atomic<uint64_t> next_publisher_identity{1}; + +std::shared_ptr<const PublishedSources> initial_sources(const std::shared_ptr<detail::AllocationReservation> &metadata, + uint64_t floor) +{ + auto sources = detail::reserved_shared<PublishedSources>(metadata, metadata); + sources->floor = floor; + return sources; +} + +std::shared_ptr<const PublishedTreeState> initial_state(const PublicationBootstrap &bootstrap, uint64_t generation) +{ + auto state = bootstrap.initial + ? detail::reserved_shared<PublishedTreeState>(bootstrap.metadata, *bootstrap.initial) + : detail::reserved_shared<PublishedTreeState>(bootstrap.metadata, bootstrap.metadata); + if (bootstrap.initial && (state->generation != generation || state->pending_handoff || + state->flush_admission_owner != 0 || state->checkpoint_inflight)) { + throw std::invalid_argument("initial publication must be a completed state in this generation"); + } + state->generation = generation; + state->clear_merge_candidates(bootstrap.metadata); + if (bootstrap.source_catalog) { + const auto &catalog = *bootstrap.source_catalog; + if (catalog.floor != state->coverage || catalog.generation != generation || !catalog.active || + catalog.tables.empty() || + std::find(catalog.tables.begin(), catalog.tables.end(), catalog.active) == catalog.tables.end()) { + throw std::invalid_argument("initial source catalog differs from publication floor or generation"); + } + if (state->sources && !state->sources->tables.empty() && + (state->sources->floor != catalog.floor || state->sources->tables.size() != catalog.tables.size() || + !std::equal(state->sources->tables.begin(), state->sources->tables.end(), catalog.tables.begin()))) { + throw std::invalid_argument("initial publication contains a different source catalog"); + } + auto sources = detail::reserved_shared<PublishedSources>(bootstrap.metadata, bootstrap.metadata); + sources->memory = catalog.memory; + sources->tables.assign(catalog.tables.begin(), catalog.tables.end()); + sources->floor = catalog.floor; + state->sources = std::move(sources); + } + if (!state->sources || state->sources->floor != state->coverage) { + throw std::invalid_argument("initial publication source floor differs from coverage"); + } + initialize_snapshot_changes(*state, bootstrap.metadata); + return state; +} +} // namespace + +PublishedTreeState::PublishedTreeState(std::shared_ptr<detail::AllocationReservation> metadata, uint64_t source_floor) + : sources(initial_sources(metadata, source_floor)) +{ +} + +std::optional<size_t> PublicationCoordinator::allocation_bound(size_t source_count, size_t max_threads, + const MappingView *mapping, + bool track_residency) noexcept +{ + using detail::shared_allocation_bound; + auto bound = + detail::AllocationBound( + sizeof(PublicationCoordinator) + sizeof(Root) + shared_allocation_bound<CheckpointLedger>() + + shared_allocation_bound<PublishedTreeState>() + (2 * PublishedSources::allocation_bound()) + + shared_allocation_bound<TreeWorkMetrics>() + EpochManager::infrastructure_allocation_bound()) + .add(mapping != nullptr ? merge_membership_reserve_bytes(mapping->copy_bounds().merge_candidate_pages) : 0) + .add_product(source_count, sizeof(std::shared_ptr<MemTable>)) + .add_product(max_threads, EpochManager::participant_allocation_bytes()) + .add(track_residency && mapping != nullptr ? PageCacheResidency::initial_allocation_bound(*mapping) : 0); + if (mapping != nullptr) { + size_t count = 0; + mapping->visit(&count, [](void *raw, uint64_t, const auto &) { ++*static_cast<size_t *>(raw); }); + bound.add(SnapshotChanges::initial_allocation_bound(count)); + } + return bound.value(); +} + +PublicationCoordinator::PublicationCoordinator(size_t queue_capacity, uint64_t generation, + std::shared_ptr<TreeWorkMetrics> metrics, + std::pmr::memory_resource *checkpoint_resource, + std::shared_ptr<SharedPageCache> page_cache, + PublicationBootstrap bootstrap) + : capacity_(queue_capacity), + identity_(next_publisher_identity.fetch_add(1, std::memory_order_relaxed)), + checkpoint_upstream_(checkpoint_resource), + checkpoint_(detail::reserved_shared<CheckpointLedger>(bootstrap.metadata, checkpoint_resource)), + epoch_(bootstrap.metadata), + work_metrics_(metrics ? std::move(metrics) : detail::reserved_shared<TreeWorkMetrics>(bootstrap.metadata)), + page_cache_(std::move(page_cache)) +{ + if (queue_capacity == 0 || (page_cache_ && !page_cache_->residency())) { + throw std::invalid_argument("publication needs queue capacity and a cache with residency metrics"); + } + auto initial = initial_state(bootstrap, generation); + auto residency = page_cache_ && !initial->mapping.empty() + ? page_cache_->residency()->prepare_initial(initial->mapping, bootstrap.metadata) + : nullptr; + auto *root = Root::create(std::move(initial), bootstrap.metadata); + if (page_cache_ && !page_cache_->claim_publication()) { + Root::dispose(root); + throw std::invalid_argument("cache residency already belongs to a publication lifetime"); + } + root->checkpoint_allocator = checkpoint_; + root_.store(root, std::memory_order_release); + if (residency) { + PageCacheResidency::publish(std::move(residency)); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_checkpoint.cpp b/lib/crowdb-tree/src/btree/publication_checkpoint.cpp new file mode 100644 index 000000000..8bba65cc2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_checkpoint.cpp @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "publication_checkpoint.h" + +#include "tree_metrics.h" + +namespace crowdb::tree +{ +Status validate_checkpoint(const PublicationTransaction &transaction, const PublishedTreeState ¤t, + const CheckpointPendingMetrics &pending) +{ + if (!transaction.checkpoint) { + return Status::Ok(); + } + if (!transaction.pages.empty() || transaction.root_page_id || transaction.sources || transaction.coverage || + transaction.handoff || transaction.merge_pair) { + return Status::invalid_argument("checkpoint action cannot include logical mutations"); + } + const auto &checkpoint = *transaction.checkpoint; + const auto action = checkpoint.action; + const auto token = checkpoint.token; + if (action == CheckpointAction::Capture) { + if (current.pending_handoff || current.flush_admission_owner != 0 || current.sources->inheritance) { + return Status::unavailable("checkpoint capture waits for complete handoff coverage"); + } + if (!transaction.expected_coverage) { + return Status::invalid_argument("checkpoint capture needs an explicit handoff frontier"); + } + return pending.can_capture(token) ? Status::Ok() + : Status::unavailable("checkpoint capture token is stale or busy"); + } + if (action != CheckpointAction::Commit && action != CheckpointAction::Abort) { + return Status::invalid_argument("invalid checkpoint action"); + } + if (transaction.next_page_id) { + return Status::invalid_argument("checkpoint completion cannot reserve page identifiers"); + } + return pending.can_finish(token) ? Status::Ok() : Status::unavailable("checkpoint completion token is stale"); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_checkpoint.h b/lib/crowdb-tree/src/btree/publication_checkpoint.h new file mode 100644 index 000000000..f80074baa --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_checkpoint.h @@ -0,0 +1,12 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "published_state.h" + +namespace crowdb::tree +{ +class CheckpointPendingMetrics; +Status validate_checkpoint(const PublicationTransaction &transaction, const PublishedTreeState ¤t, + const CheckpointPendingMetrics &pending); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_checkpoint_completion.cpp b/lib/crowdb-tree/src/btree/publication_checkpoint_completion.cpp new file mode 100644 index 000000000..d877cc18a --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_checkpoint_completion.cpp @@ -0,0 +1,27 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "published_state.h" + +namespace crowdb::tree +{ +bool PublicationCoordinator::try_submit_checkpoint_completion(const std::shared_ptr<PublicationRequest> &request) +{ + if (!request || !request->transaction.checkpoint || + (request->transaction.checkpoint->action != CheckpointAction::Commit && + request->transaction.checkpoint->action != CheckpointAction::Abort)) { + return false; + } + const auto view = capture(); + if (!view->checkpoint_inflight || view->checkpoint_token != request->transaction.checkpoint->token || + request->submitted_.exchange(true, std::memory_order_acq_rel)) { + return false; + } + if (checkpoint_completion_.exchange(true, std::memory_order_acq_rel)) { + request->submitted_.store(false, std::memory_order_release); + return false; + } + request->completion_slot_ = true; + enqueue(request); + return true; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_commit.cpp b/lib/crowdb-tree/src/btree/publication_commit.cpp new file mode 100644 index 000000000..1967691c5 --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_commit.cpp @@ -0,0 +1,479 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "btree/backing_ref.h" +#include "btree/logical_page.h" +#include "btree/publication_generation_internal.h" +#include "page_cache.h" +#include "publication_admission.h" +#include "publication_checkpoint.h" +#include "publication_snapshot.h" + +#include <algorithm> +#include <cassert> + +namespace crowdb::tree +{ + +namespace +{ +bool invalidates_merge(const PageReplacement &page) +{ + return logical_page_change(page) || (page.replacement && page.replacement->metrics.split_candidate); +} + +bool matches_expected_page(const std::shared_ptr<const PublishedPage> ¤t, + const std::shared_ptr<const PublishedPage> &expected) +{ + if (current == expected) { + return true; + } + if (!current || !expected || current->revision != expected->revision || current->level != expected->level || + current->leaf != expected->leaf || current->inner != expected->inner || + current->inner_backing != expected->inner_backing || current->primary != expected->primary || + current->allocation_owner != expected->allocation_owner) { + return false; + } + // Classification can copy a sibling without changing its representation. + // Its derived membership is recomputed from the current ledger below. + auto current_metrics = current->metrics; + auto expected_metrics = expected->metrics; + current_metrics.merge_candidate = false; + expected_metrics.merge_candidate = false; + return current_metrics == expected_metrics; +} + +void prepare_pending(const PublicationTransaction &transaction, const PublishedTreeState ¤t, + CheckpointPendingMetrics::PreparedDelta &delta) +{ + for (const auto &page : transaction.pages) { + if (!logical_page_change(page)) { + continue; + } + if (page.replacement) { + delta.change_page(page.page_id); + } + else { + delta.remove_page(page.page_id); + } + delta.change_mapping(); + } + if (transaction.root_page_id.value_or(current.root_page_id) != current.root_page_id || + transaction.coverage.value_or(current.coverage) != current.coverage) { + delta.change_mapping(); + } +} + +std::shared_ptr<PublishedTreeState> checkpoint_artifact(const std::shared_ptr<PublishedTreeState> &next, + const PublicationTransaction &transaction) +{ + if (!transaction.checkpoint || transaction.checkpoint->action != CheckpointAction::Capture) { + return {}; + } + auto snapshot = detail::reserved_shared<PublishedTreeState>(transaction.metadata, *next); + auto sources = detail::reserved_shared<PublishedSources>(transaction.metadata, transaction.metadata); + sources->floor = next->coverage; + snapshot->sources = std::move(sources); + return snapshot; +} + +Status validate_merge_pair(const PublishedTreeState ¤t, const PublicationTransaction &transaction) +{ + if (!transaction.merge_pair) { + return Status::Ok(); + } + if (!transaction.pages.empty() || transaction.root_page_id || transaction.sources || transaction.coverage || + transaction.checkpoint || transaction.handoff || transaction.next_page_id || transaction.flush_admission) { + return Status::invalid_argument("merge classification must be a standalone transaction"); + } + const auto &update = *transaction.merge_pair; + if (!update.expected_left || !update.expected_left->leaf || + current.mapping.get(update.left) != update.expected_left) { + return Status::unavailable("merge classification left page changed"); + } + if (!update.candidate) { + return Status::Ok(); + } + const auto &pair = *update.candidate; + const auto &left = update.expected_left; + const auto &right = update.expected_right; + const auto &parent = update.expected_parent; + if (pair.left != update.left || pair.left == pair.right || !right || !right->leaf || !parent || !parent->inner || + current.mapping.get(pair.right) != right || current.mapping.get(pair.parent) != parent || + left->leaf->right_sibling != pair.right || left->leaf->content_id != pair.left_content || + right->leaf->content_id != pair.right_content || left->leaf->route_generation != pair.left_route || + right->leaf->route_generation != pair.right_route || left->metrics.split_candidate || + right->metrics.split_candidate) { + return Status::unavailable("merge classification route or content changed"); + } + const auto child = std::find(parent->inner->children.begin(), parent->inner->children.end(), pair.left); + if (child == parent->inner->children.end() || child + 1 == parent->inner->children.end() || + *(child + 1) != pair.right) { + return Status::unavailable("merge pair no longer shares one parent"); + } + if (pair.ready && (!left->metrics.page_size_known || !left->metrics.kv_known || !right->metrics.page_size_known || + !right->metrics.kv_known)) { + return Status::invalid_argument("ready merge pair requires exact page summaries"); + } + return Status::Ok(); +} + +Status validate_transaction(const PublishedTreeState ¤t, const PublicationTransaction &transaction, + uint64_t publisher_id) +{ + if (!transaction.record || transaction.record->token.generation != current.generation) { + return Status::unavailable("publication belongs to a different tree generation"); + } + if (transaction.summary && + (!transaction.expected_commit_seq || !transaction.pages.empty() || transaction.root_page_id || + transaction.sources || transaction.coverage || transaction.checkpoint || transaction.handoff || + transaction.merge_pair || transaction.flush_admission || transaction.next_page_id || + transaction.summary->covered_slot != current.coverage || !transaction.summary->exact)) { + return Status::invalid_argument("summary installation requires a fixed unmodified graph"); + } + if (transaction.expected_commit_seq && *transaction.expected_commit_seq != current.commit_seq) { + return Status::unavailable("publication state changed before admission"); + } + if (transaction.expected_l1_seq && *transaction.expected_l1_seq != current.l1_seq) { + return Status::unavailable("publication L1 view changed"); + } + if (transaction.expected_next_page_id && *transaction.expected_next_page_id != current.next_page_id) { + return Status::unavailable("page identifier authority changed before publication"); + } + if (transaction.next_page_id && + (!transaction.expected_next_page_id || *transaction.next_page_id < current.next_page_id || + *transaction.next_page_id > MappingView::kPageLimit)) { + return Status::invalid_argument("page identifier reservation must monotonically advance the authority"); + } + if (auto admission = validate_flush_admission(current, transaction); !admission.ok()) { + return admission; + } + if (auto merge = validate_merge_pair(current, transaction); !merge.ok()) { + return merge; + } + if (transaction.handoff) { + const auto &handoff = *transaction.handoff; + if (handoff.batch != transaction.record->token.batch || handoff.frontier <= current.coverage || + (transaction.coverage && *transaction.coverage != handoff.frontier) || transaction.root_page_id) { + return Status::invalid_argument("invalid handoff identity or frontier"); + } + if (current.pending_handoff && current.pending_handoff != transaction.handoff) { + return Status::unavailable("another handoff needs repair before publication"); + } + } + else if (current.pending_handoff && + (!transaction.pages.empty() || transaction.root_page_id || transaction.sources || transaction.coverage)) { + return Status::unavailable("logical mutation waits for complete handoff coverage"); + } + if (transaction.expected_coverage && *transaction.expected_coverage != current.coverage) { + return Status::unavailable("publication coverage changed"); + } + if (transaction.expected_root_page_id && *transaction.expected_root_page_id != current.root_page_id) { + return Status::unavailable("publication root changed"); + } + if (transaction.sources && transaction.expected_sources != current.sources) { + return Status::unavailable("publication source catalog changed"); + } + const auto coverage = transaction.coverage.value_or(current.coverage); + if (transaction.sources && current.sources->inheritance && !transaction.sources->inheritance && + coverage < current.sources->inheritance->frontier) { + return Status::unavailable("split prefix must be covered before inherited sources detach"); + } + if (transaction.sources && transaction.sources->inheritance) { + const auto &inherited = *transaction.sources->inheritance; + if (inherited.owner_identity == 0 || inherited.session == 0 || inherited.frontier < coverage || + inherited.local_count > transaction.sources->tables.size()) { + return Status::invalid_argument("invalid split inheritance identity or frontier"); + } + for (size_t i = inherited.local_count; i < transaction.sources->tables.size(); ++i) { + const auto &table = transaction.sources->tables[i]; + if (!table || !table->closed() || table->writers() != 0) { + return Status::unavailable("inherited sources must be closed and drained before publication"); + } + } + } + if (coverage < current.coverage || (transaction.coverage && !transaction.sources) || + (transaction.sources && transaction.sources->floor != coverage)) { + return Status::invalid_argument("coverage and source floor must publish together"); + } + for (const auto &record : transaction.required_installs) { + if (!record || record->publisher_id() != publisher_id || record->token.generation != current.generation || + record->token.batch != transaction.record->token.batch || + !record->committed.load(std::memory_order_acquire)) { + return Status::unavailable("handoff has unfinished range installations"); + } + } + uint64_t previous = 0; + bool first = true; + for (const auto &page : transaction.pages) { + if (page.page_id >= MappingView::kPageLimit || (!first && page.page_id <= previous)) { + return Status::invalid_argument("replacement PIDs must be unique and ordered"); + } + if (!matches_expected_page(current.mapping.get(page.page_id), page.expected)) { + return Status::unavailable("page changed; publication needs replanning"); + } + if (!page.expected && !page.replacement) { + return Status::invalid_argument("an absent page cannot be removed"); + } + if (!page.expected && page.replacement && page.page_id < current.next_page_id) { + return Status::invalid_argument("a retired PID cannot be reused"); + } + if (page.replacement && ((page.replacement->level == 0) != page.replacement->metrics.leaf)) { + return Status::invalid_argument("page metrics kind disagrees with routing level"); + } + if (page.replacement && + (page.replacement->level == 0 + ? (!page.replacement->leaf || page.replacement->inner || page.replacement->inner_backing) + : (page.replacement->leaf || + (!page.replacement->inner && + (!page.replacement->inner_backing || !page.replacement->inner_backing->readable()))))) { + return Status::invalid_argument("published page must have the representation for its routing level"); + } + previous = page.page_id; + first = false; + } + return Status::Ok(); +} +} // namespace + +size_t PublicationCoordinator::empty_commit_reserve_bytes() noexcept +{ + // Includes a replacement request after a rejected attempt and allocator + // control blocks. No mapping or checkpoint collection grows here. + return detail::shared_allocation_bound<PublishedTreeState>() + sizeof(Root) + sizeof(PublicationRequest) + + sizeof(InstallRecord) + 1024; +} + +size_t PublicationCoordinator::merge_membership_reserve_bytes(size_t pages) noexcept +{ + return detail::AllocationBound() + .add_product(pages, MappingView::path_copy_bound() + detail::shared_allocation_bound<PublishedPage>() + + (3 * sizeof(PageReplacement)) + 64) + .value() + .value_or(SIZE_MAX); +} + +PublicationResult PublicationCoordinator::commit(const PublicationTransaction &transaction) +{ + const auto *current = root_.load(std::memory_order_relaxed); + PublicationResult result; + if (!transaction.record || transaction.record->token.generation != current->state->generation) { + result.status = Status::unavailable("publication belongs to a different tree generation"); + return result; + } + uint64_t expected_owner = 0; + if (!transaction.record->publisher_id_.compare_exchange_strong(expected_owner, identity_, + std::memory_order_acq_rel) && + expected_owner != identity_) { + result.status = Status::unavailable("install record belongs to a different publisher"); + return result; + } + if (transaction.record->committed.load(std::memory_order_acquire)) { + result.commit_seq = transaction.record->commit_seq; + result.l1_seq = transaction.record->l1_seq; + result.already_committed = true; + result.checkpoint_view = transaction.record->checkpoint_view; + return result; + } + result.status = validate_transaction(*current->state, transaction, identity_); + if (!result.status.ok()) { + return result; + } + result.status = validate_checkpoint(transaction, *current->state, checkpoint_->pending); + if (!result.status.ok()) { + return result; + } + result.status = validate_snapshot_publication(transaction, *current->state); + if (!result.status.ok()) { + return result; + } + detail::BudgetMemoryResource::Scope checkpoint_credit(checkpoint_->resource, transaction.metadata); + auto next = detail::reserved_shared<PublishedTreeState>(transaction.metadata, *current->state); + next->next_page_id = transaction.next_page_id.value_or(next->next_page_id); + next->mapping = current->state->mapping.replace(transaction.pages, result.copied, transaction.metadata); + for (const auto &page : transaction.pages) { + const auto previous = current->state->mapping.get(page.page_id); + if (!apply_page_metrics(next->stats, previous ? &previous->metrics : nullptr, + page.replacement ? &page.replacement->metrics : nullptr)) { + result.status = Status::invalid_argument("invalid logical metrics delta"); + return result; + } + next->next_page_id = std::max(next->next_page_id, page.page_id + 1); + } + next->root_page_id = transaction.root_page_id.value_or(next->root_page_id); + const auto root_page = next->mapping.get(next->root_page_id); + if (next->root_page_id != UINT64_MAX && !root_page) { + result.status = Status::invalid_argument("published root must exist in the same mapping"); + return result; + } + next->stats.height = root_page ? uint64_t{root_page->level} + 1 : 0; + next->coverage = transaction.coverage.value_or(next->coverage); + next->gc_floor = + std::max(next->gc_floor, std::min(next->coverage, requested_gc_floor_.load(std::memory_order_acquire))); + if (transaction.flush_admission) { + next->flush_admission_owner = transaction.flush_admission->action == FlushAdmissionAction::Acquire + ? transaction.flush_admission->owner + : 0; + } + else if (transaction.coverage) { + next->flush_admission_owner = 0; + } + if (transaction.handoff) { + next->pending_handoff = transaction.coverage ? std::nullopt : transaction.handoff; + } + if (transaction.sources) { + next->sources = transaction.sources; + } + next->commit_seq++; + const auto pending_changes = + static_cast<size_t>(std::count_if(transaction.pages.begin(), transaction.pages.end(), logical_page_change)); + if (pending_changes != 0 || next->root_page_id != current->state->root_page_id || + next->coverage != current->state->coverage) { + next->summary.reset(); + } + if (transaction.summary) { + next->summary = transaction.summary; + } + CheckpointPendingMetrics::PreparedDelta pending(checkpoint_->pending, pending_changes); + prepare_pending(transaction, *current->state, pending); + pending.copy_to(next->stats); + std::optional<MergePairMetrics::Prepared> merge; + if (transaction.merge_pair || + (checkpoint_->merge.snapshot().pairs != 0 && std::ranges::any_of(transaction.pages, invalidates_merge))) { + merge.emplace(checkpoint_->merge, transaction.metadata, checkpoint_); + for (const auto &page : transaction.pages) { + if (invalidates_merge(page)) { + merge->invalidate_page(page.page_id); + merge->invalidate_parent(page.page_id); + } + } + if (transaction.merge_pair) { + const auto &update = *transaction.merge_pair; + if (update.candidate) { + merge->set_pair(*update.candidate, tree_metrics_now_ns()); + } + else { + merge->clear_pair(update.left); + } + } + merge->finish(); + const auto totals = merge->snapshot(); + // Unique endpoint page flags are published below with the same root. + // The canonical pair ledger owns pair counts and candidate age. + next->stats.merge_candidate_pairs = totals.pairs; + next->stats.merge_ready_pairs = totals.ready; + next->merge_candidate_since_ns = totals.oldest_since_ns; + } + detail::ReservedVector<uint64_t> membership_pids{detail::ReservationAllocator<uint64_t>(transaction.metadata)}; + if (merge) { + for (const auto pid : merge->affected_pages()) { + membership_pids.push_back(pid); + } + } + for (const auto &page : transaction.pages) { + if (page.replacement && page.replacement->leaf) { + const bool present = + merge ? merge->contains_page(page.page_id) : checkpoint_->merge.contains_page(page.page_id); + const auto proof = merge ? merge->pair(page.page_id) : checkpoint_->merge.pair(page.page_id); + if (page.replacement->metrics.merge_candidate != present || page.replacement->merge_outgoing != proof) { + membership_pids.push_back(page.page_id); + } + } + } + std::sort(membership_pids.begin(), membership_pids.end()); + membership_pids.erase(std::unique(membership_pids.begin(), membership_pids.end()), membership_pids.end()); + detail::ReservedVector<PageReplacement> memberships{ + detail::ReservationAllocator<PageReplacement>(transaction.metadata)}; + memberships.reserve(membership_pids.size()); + for (const auto pid : membership_pids) { + const auto before = next->mapping.get(pid); + const bool present = merge ? merge->contains_page(pid) : checkpoint_->merge.contains_page(pid); + const auto proof = merge ? merge->pair(pid) : checkpoint_->merge.pair(pid); + if (!before || (before->metrics.merge_candidate == present && before->merge_outgoing == proof)) { + continue; + } + auto after = detail::reserved_shared<PublishedPage>(transaction.metadata, *before); + after->metrics.merge_candidate = present; + after->merge_outgoing = proof; + if (!apply_page_metrics(next->stats, &before->metrics, &after->metrics)) { + result.status = Status::invalid_argument("invalid merge membership delta"); + return result; + } + memberships.push_back({.page_id = pid, .expected = before, .replacement = std::move(after)}); + } + next->mapping = next->mapping.replace(memberships, result.copied, transaction.metadata); + const auto &previous = *current->state; + if (!next->mapping.same_root(previous.mapping) || next->root_page_id != previous.root_page_id || + next->next_page_id != previous.next_page_id || next->coverage != previous.coverage || + next->gc_floor != previous.gc_floor || next->pending_handoff != previous.pending_handoff || + next->flush_admission_owner != previous.flush_admission_owner) { + if (previous.l1_seq == UINT64_MAX) { + result.status = Status::resource_exhausted("publication L1 sequence exhausted"); + return result; + } + ++next->l1_seq; + } + prepare_snapshot_changes(transaction, *current->state, *next); + auto snapshot = checkpoint_artifact(next, transaction); + std::optional<CheckpointPendingMetrics::PreparedCapture> prepared_capture; + if (snapshot) { + prepared_capture.emplace(checkpoint_->pending, transaction.metadata, checkpoint_); + } + std::shared_ptr<const void> checkpoint_retired; + result.commit_seq = next->commit_seq; + result.l1_seq = next->l1_seq; + auto residency = page_cache_ && !transaction.pages.empty() + ? page_cache_->residency()->prepare(transaction.pages, transaction.metadata) + : std::shared_ptr<PageCacheResidency::Update>{}; +#ifdef CROWDB_TREE_TEST_UTIL + if (before_root_for_tests_) { + before_root_for_tests_(); + } +#endif + auto complete_l1 = prepare_complete_view(*current, *next, transaction.metadata); + auto *replacement = Root::create(next, transaction.metadata); + replacement->complete_l1 = std::move(complete_l1); + replacement->checkpoint_allocator = checkpoint_; + // Every allocation has succeeded. Ledger mutation and the root exchange + // below are no-throw and contain no user callbacks or suspended work. + pending.commit(); + auto merge_retired = merge ? merge->commit() : std::shared_ptr<const void>{}; + if (transaction.checkpoint) { + const auto action = transaction.checkpoint->action; + const auto token = transaction.checkpoint->token; + if (prepared_capture) { + checkpoint_retired = prepared_capture->commit(token); + } + else { + const bool applied = checkpoint_->pending.finish(token, action == CheckpointAction::Commit); + assert(applied); + (void)applied; + } + next->checkpoint_token = token; + next->checkpoint_inflight = action == CheckpointAction::Capture; + checkpoint_->pending.copy_to(next->stats); + if (snapshot) { + snapshot->checkpoint_token = token; + snapshot->checkpoint_inflight = true; + snapshot->stats = next->stats; + snapshot->checkpoint_pending_since_ns = checkpoint_->pending.oldest_since_ns(); + result.checkpoint_view = snapshot; + transaction.record->checkpoint_view = std::move(snapshot); + } + } + next->checkpoint_pending_since_ns = checkpoint_->pending.oldest_since_ns(); + auto *old = root_.exchange(replacement, std::memory_order_acq_rel); + record_work(work_metrics_, TreeWorkCounter::CowMetadataCopyBytes, result.copied.bytes); + if (residency) { + PageCacheResidency::publish(std::move(residency)); + } + old->checkpoint_retired = std::move(checkpoint_retired); + old->merge_retired = std::move(merge_retired); + transaction.record->commit_seq = result.commit_seq; + transaction.record->l1_seq = result.l1_seq; + transaction.record->committed.store(true, std::memory_order_release); + old->state->retirement.mark(); + epoch_.defer(old); + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_complete_view.cpp b/lib/crowdb-tree/src/btree/publication_complete_view.cpp new file mode 100644 index 000000000..24e81faef --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_complete_view.cpp @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/published_state.h" + +namespace crowdb::tree +{ +size_t PublicationCoordinator::complete_view_reserve_bytes() noexcept +{ + return detail::shared_allocation_bound<PublishedTreeState>() + PublishedSources::allocation_bound(); +} + +std::shared_ptr<const PublishedTreeState> PublicationCoordinator::capture_complete() const +{ + auto guard = epoch_.enter(); + const auto *root = root_.load(std::memory_order_acquire); + return root->complete_l1 ? root->complete_l1 : root->state; +} + +std::shared_ptr<const PublishedTreeState> +PublicationCoordinator::prepare_complete_view(const Root ¤t, const PublishedTreeState &next, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + if (!next.pending_handoff && next.flush_admission_owner == 0) { + return {}; + } + if (current.complete_l1) { + return current.complete_l1; + } + // Publication serializes the transition from complete L1 to ownership of + // a pending handoff. Share its immutable mapping without retaining live L0 + // tables or walking pages on the publication owner. + auto complete = detail::reserved_shared<PublishedTreeState>(metadata, *current.state); + auto sources = detail::reserved_shared<PublishedSources>(metadata, metadata); + sources->floor = complete->coverage; + complete->sources = std::move(sources); + return complete; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_generation.cpp b/lib/crowdb-tree/src/btree/publication_generation.cpp new file mode 100644 index 000000000..e59d2a199 --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_generation.cpp @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "page_cache.h" +#include "publication_generation_internal.h" +#include "publication_snapshot.h" +#include "source_admission.h" + +#include <cassert> + +namespace crowdb::tree +{ +std::optional<size_t> PublicationCoordinator::replacement_allocation_bound(const MappingView &mapping, + size_t sources) noexcept +{ + using detail::shared_allocation_bound; + detail::AllocationBound bound(sizeof(Root) + shared_allocation_bound<Replacement>() + + shared_allocation_bound<CheckpointLedger>() + + shared_allocation_bound<PublishedTreeState>() + PublishedSources::allocation_bound()); + bound.add_product(sources, sizeof(std::shared_ptr<MemTable>)); + // Each pending member has a red-black node plus the resource charge header. + size_t pages = 0; + mapping.visit(&pages, [](void *raw, uint64_t, const auto &) { ++*static_cast<size_t *>(raw); }); + bound.add_product(pages, 512); + bound.add(SnapshotChanges::initial_allocation_bound(pages)); + bound.add(PageCacheResidency::initial_allocation_bound(mapping)); + bound.add(merge_membership_reserve_bytes(mapping.copy_bounds().merge_candidate_pages)); + return bound.value(); +} + +Status PublicationCoordinator::prepare_replacement(PublicationBootstrap bootstrap, std::shared_ptr<Replacement> *out, + std::shared_ptr<const PublishedSources> inherited) const +try { + if (out == nullptr || *out || !bootstrap.initial || !bootstrap.source_catalog || !bootstrap.initial->sources || + !bootstrap.initial->sources->tables.empty()) { + return Status::invalid_argument("replacement requires a private mapping and empty source catalog"); + } + const auto &catalog = *bootstrap.source_catalog; + if (catalog.floor != bootstrap.initial->coverage || catalog.generation != bootstrap.initial->generation || + !catalog.active || catalog.tables.size() != 1 || catalog.tables.front() != catalog.active) { + return Status::invalid_argument("replacement source catalog differs from the candidate generation"); + } + if (inherited && (!inherited->inheritance || inherited->floor != catalog.floor || inherited->tables.empty() || + inherited->tables.front() != catalog.active || inherited->inheritance->local_count != 1 || + inherited->inheritance->frontier <= catalog.floor)) { + return Status::invalid_argument("replacement inheritance differs from its local source catalog"); + } + auto prepared = detail::reserved_shared<Replacement>(bootstrap.metadata); + prepared->state = detail::reserved_shared<PublishedTreeState>(bootstrap.metadata, *bootstrap.initial); + auto &state = *prepared->state; + state.gc_floor = 0; + state.checkpoint_token = 0; + state.checkpoint_inflight = false; + state.clear_merge_candidates(bootstrap.metadata); + initialize_snapshot_changes(state, bootstrap.metadata); + state.pending_handoff.reset(); + state.flush_admission_owner = 0; + if (inherited) { + state.sources = std::move(inherited); + } + else { + auto sources = detail::reserved_shared<PublishedSources>(bootstrap.metadata, bootstrap.metadata); + sources->memory = catalog.memory; + sources->tables.assign(catalog.tables.begin(), catalog.tables.end()); + sources->floor = catalog.floor; + state.sources = std::move(sources); + } + prepared->checkpoint = detail::reserved_shared<CheckpointLedger>(bootstrap.metadata, checkpoint_upstream_); + { + detail::BudgetMemoryResource::Scope scope(prepared->checkpoint->resource, bootstrap.metadata); + state.mapping.visit(&prepared->checkpoint->pending, [](void *context, uint64_t pid, const auto &) { + static_cast<CheckpointPendingMetrics *>(context)->change_page(pid); + }); + prepared->checkpoint->pending.change_mapping(); + prepared->checkpoint->pending.copy_to(state.stats); + state.checkpoint_pending_since_ns = prepared->checkpoint->pending.oldest_since_ns(); + } + if (page_cache_) { + prepared->residency = page_cache_->residency()->prepare_replacement(state.mapping, bootstrap.metadata); + } + prepared->root = Root::create(prepared->state, bootstrap.metadata); + prepared->root->checkpoint_allocator = prepared->checkpoint; + *out = std::move(prepared); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("generation publication preparation exhausted credit"); +} + +void PublicationCoordinator::install_replacement(std::shared_ptr<Replacement> replacement) noexcept +{ + assert(replacement && replacement->root != nullptr && idle()); + const auto *current = root_.load(std::memory_order_acquire); + assert(replacement->state->generation == current->state->generation + 1); + replacement->state->commit_seq = current->state->commit_seq + 1; + requested_gc_floor_.store(0, std::memory_order_release); + checkpoint_ = std::move(replacement->checkpoint); + auto *next = std::exchange(replacement->root, nullptr); + auto *old = root_.exchange(next, std::memory_order_acq_rel); + PageCacheResidency::publish(std::move(replacement->residency)); + + old->state->retirement.mark(); + epoch_.defer(old); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_generation_internal.h b/lib/crowdb-tree/src/btree/publication_generation_internal.h new file mode 100644 index 000000000..e59547d8c --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_generation_internal.h @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "page_cache_residency.h" +#include "published_state.h" + +namespace crowdb::tree +{ +struct PublicationCoordinator::CheckpointLedger +{ + bool retire_step() noexcept + { + return pending.retire_step() && merge.retire_step(); + } + + explicit CheckpointLedger(std::pmr::memory_resource *upstream) + : resource(upstream), + pending(&resource), + merge(&resource) + { + } + + detail::BudgetMemoryResource resource; + CheckpointPendingMetrics pending; + MergePairMetrics merge; +}; + +class PublicationCoordinator::Replacement +{ + public: + ~Replacement() + { + if (root != nullptr) { + Root::dispose(root); + } + } + + private: + friend class PublicationCoordinator; + std::shared_ptr<PublishedTreeState> state; + std::shared_ptr<CheckpointLedger> checkpoint; + std::shared_ptr<PageCacheResidency::Update> residency; + Root *root = nullptr; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_lane.cpp b/lib/crowdb-tree/src/btree/publication_lane.cpp new file mode 100644 index 000000000..d232f0cc5 --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_lane.cpp @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "publication_lane.h" + +#include "published_state.h" + +namespace crowdb::tree +{ +Status PublicationCoordinator::prepare_lane(size_t capacity, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<PublicationLane> *out) +{ + if (capacity == 0 || capacity > capacity_ || !metadata || out == nullptr || *out) { + return Status::invalid_argument("publication progress needs a bounded prepaid window"); + } + if (closed_.load(std::memory_order_acquire) || lane_prepared_.exchange(true, std::memory_order_acq_rel)) { + return Status::unavailable("publication progress window already prepared or closed"); + } + try { + *out = detail::reserved_construct<PublicationLane>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) PublicationLane(identity_, capacity); + }); + } + catch (const std::bad_alloc &) { + lane_prepared_.store(false, std::memory_order_release); + return Status::resource_exhausted("publication progress control credit unavailable"); + } + return Status::Ok(); +} + +bool PublicationCoordinator::try_submit(const std::shared_ptr<PublicationRequest> &request, + const std::shared_ptr<PublicationLane> &lane) +{ + if (!lane) { + return try_submit(request); + } + if (!request || lane->publisher_ != identity_ || closed_.load(std::memory_order_acquire) || + request->submitted_.exchange(true, std::memory_order_acq_rel)) { + return false; + } + auto queued = lane->queued_.load(std::memory_order_relaxed); + do { + if (queued == lane->capacity_) { + request->submitted_.store(false, std::memory_order_release); + return false; + } + } while (!lane->queued_.compare_exchange_weak(queued, queued + 1, std::memory_order_acq_rel)); + request->lane_ = lane; + enqueue(request); + return true; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_lane.h b/lib/crowdb-tree/src/btree/publication_lane.h new file mode 100644 index 000000000..01444d5bb --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_lane.h @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include <atomic> +#include <cstddef> +#include <cstdint> + +namespace crowdb::tree +{ +// Fixed publication window reserved before a Tree accepts proposals. Requests +// already own their metadata; this lane only bounds their simultaneous ingress. +class PublicationLane +{ + public: + [[nodiscard]] size_t capacity() const noexcept + { + return capacity_; + } + + private: + friend class PublicationCoordinator; + + PublicationLane(uint64_t publisher, size_t capacity) : publisher_(publisher), capacity_(capacity) + { + } + + const uint64_t publisher_; + const size_t capacity_; + std::atomic<size_t> queued_{0}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_maintenance.cpp b/lib/crowdb-tree/src/btree/publication_maintenance.cpp new file mode 100644 index 000000000..12ee16136 --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_maintenance.cpp @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "published_state.h" + +#include <algorithm> +#include <cassert> + +namespace crowdb::tree +{ +size_t PublicationCoordinator::start_maintenance(uint64_t started_at_ns) noexcept +{ + const auto started = std::max(uint64_t{1}, started_at_ns); + for (size_t i = 0; i < maintenance_times_.size(); ++i) { + uint64_t empty = 0; + if (maintenance_times_[i].compare_exchange_strong(empty, started, std::memory_order_release, + std::memory_order_relaxed)) { + return i; + } + } + return kMaintenanceSlots; +} + +void PublicationCoordinator::finish_maintenance(size_t slot) noexcept +{ + assert(slot < kMaintenanceSlots); + maintenance_times_[slot].store(0, std::memory_order_release); +} + +uint64_t PublicationCoordinator::oldest_maintenance_since_ns() const noexcept +{ + uint64_t oldest = 0; + for (const auto &slot : maintenance_times_) { + const auto started = slot.load(std::memory_order_acquire); + if (started != 0 && (oldest == 0 || started < oldest)) { + oldest = started; + } + } + return oldest; +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_reconcile.cpp b/lib/crowdb-tree/src/btree/publication_reconcile.cpp new file mode 100644 index 000000000..17c7081da --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_reconcile.cpp @@ -0,0 +1,22 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "publication_generation_internal.h" +#include "tree_state_reconcile.h" + +namespace crowdb::tree +{ +PublicationResult PublicationCoordinator::reconcile(const PublicationRequest &request) +{ + const auto &transaction = request.transaction; + if (transaction.record || !transaction.pages.empty() || transaction.root_page_id || transaction.sources || + transaction.coverage || !transaction.required_installs.empty() || transaction.checkpoint || + transaction.handoff || transaction.flush_admission || transaction.next_page_id || transaction.merge_pair || + transaction.summary || request.reconciliation_slots == 0 || request.reconciliation_slots > 64) { + return {.status = Status::invalid_argument("reconciliation requires a standalone bounded request")}; + } + const auto *current = root_.load(std::memory_order_acquire); + return {.status = request.reconciliation->step_ledger(request.reconciliation_slots, *current->state, + checkpoint_->pending, checkpoint_->merge), + .commit_seq = current->state->commit_seq}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_snapshot.cpp b/lib/crowdb-tree/src/btree/publication_snapshot.cpp new file mode 100644 index 000000000..5fc462f08 --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_snapshot.cpp @@ -0,0 +1,157 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "publication_snapshot.h" + +#include "logical_page.h" +#include "snapshot/captured_write.h" +#include "snapshot/snapshot_baseline.h" + +namespace crowdb::tree +{ +namespace +{ +SnapshotPageIdentity page_identity(const std::shared_ptr<const PublishedPage> &page) +{ + if (!page) { + return {}; + } + SnapshotPageIdentity identity{.revision = page->revision, .level = page->level, .present = true}; + if (page->leaf) { + identity.range = page->leaf->range; + identity.content = page->leaf->content_id; + identity.route = page->leaf->route_generation; + identity.right_sibling = page->leaf->right_sibling; + } + return identity; +} +} // namespace + +Status validate_snapshot_publication(const PublicationTransaction &transaction, const PublishedTreeState ¤t) +{ + switch (transaction.snapshot_effect) { + case SnapshotChangeEffect::FullContent: + break; + case SnapshotChangeEffect::AcceptedPrefix: + if (!transaction.handoff) { + return Status::invalid_argument("accepted snapshot delta requires a handoff frontier"); + } + break; + case SnapshotChangeEffect::SourceStructure: + if (transaction.handoff || transaction.coverage || transaction.sources) { + return Status::invalid_argument("source structure cannot publish new accepted contents"); + } + break; + default: + return Status::invalid_argument("snapshot publication requires one typed logical effect"); + } + if (transaction.snapshot_metadata && transaction.snapshot_metadata->category() != detail::BudgetClass::kRouting) { + return Status::invalid_argument("snapshot selection requires Routing metadata credit"); + } + if (!transaction.checkpoint || !transaction.checkpoint->baseline) { + return Status::Ok(); + } + const auto &action = *transaction.checkpoint; + const auto &base = *action.baseline; + if (action.action != CheckpointAction::Commit || base.capture_token != action.token || + base.generation != current.generation || !base.pages || base.pages->generation != base.generation || + base.pages->coverage != base.coverage || base.coverage != current.checkpoint_capture_coverage || + base.gc_floor != current.checkpoint_capture_floor || base.gc_floor > base.coverage || + base.snapshot_sequence == 0 || base.backend_id == 0 || + (!base.pages->frames.empty() && base.pages->backend_id != base.backend_id) || + (current.snapshot_baseline && (base.coverage < current.snapshot_baseline->coverage || + base.snapshot_sequence <= current.snapshot_baseline->snapshot_sequence || + base.backend_id != current.snapshot_baseline->backend_id))) { + return Status::invalid_argument("snapshot baseline differs from its committed capture"); + } + return Status::Ok(); +} + +void initialize_snapshot_changes(PublishedTreeState &state, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + state.snapshot_changes = {}; + state.snapshot_changes_since_capture = {}; + state.snapshot_baseline.reset(); + state.checkpoint_capture_coverage = 0; + state.checkpoint_capture_floor = 0; + + detail::ReservedVector<SnapshotPageChange> changes{detail::ReservationAllocator<SnapshotPageChange>(metadata)}; + size_t count = 0; + state.mapping.visit(&count, [](void *raw, uint64_t, const auto &) { ++*static_cast<size_t *>(raw); }); + changes.reserve(count); + state.mapping.visit(&changes, [](void *raw, uint64_t pid, const auto &page) { + static_cast<detail::ReservedVector<SnapshotPageChange> *>(raw)->push_back( + {.page_id = pid, .before = {}, .after = page_identity(page)}); + }); + state.snapshot_changes = SnapshotChanges::from_sorted(changes, metadata); +} + +bool logical_page_change(const PageReplacement &page) +{ + if (page.expected == page.replacement) { + return false; + } + if (!page.expected || !page.replacement || page.expected->level != page.replacement->level) { + return true; + } + if (page.replacement->level == 0) { + const auto &before = *page.expected->leaf; + const auto &after = *page.replacement->leaf; + return before.content_id != after.content_id || before.route_generation != after.route_generation || + before.range.lower != after.range.lower || before.range.upper != after.range.upper || + before.right_sibling != after.right_sibling; + } + if (!page.expected->inner || !page.replacement->inner) { + return !page.expected->inner_backing || page.expected->inner_backing != page.replacement->inner_backing; + } + return page.expected->inner->children != page.replacement->inner->children || + page.expected->inner->separators != page.replacement->inner->separators; +} + +void prepare_snapshot_changes(const PublicationTransaction &transaction, const PublishedTreeState ¤t, + PublishedTreeState &next) +{ + static_assert(SnapshotChanges::kPageLimit == MappingView::kPageLimit); + const auto &metadata = transaction.snapshot_metadata ? transaction.snapshot_metadata : transaction.metadata; + for (const auto &page : transaction.pages) { + if (!logical_page_change(page)) { + continue; + } + SnapshotPageChange change{.page_id = page.page_id, + .publication_seq = next.commit_seq, + .before = page_identity(current.mapping.get(page.page_id)), + .after = page_identity(next.mapping.get(page.page_id)), + .effect = transaction.snapshot_effect}; + next.snapshot_changes = next.snapshot_changes.replace(change, metadata); + if (current.checkpoint_inflight) { + next.snapshot_changes_since_capture = next.snapshot_changes_since_capture.replace(change, metadata); + } + } + if (!transaction.checkpoint) { + return; + } + switch (transaction.checkpoint->action) { + case CheckpointAction::Capture: + next.snapshot_changes_since_capture = {}; + next.checkpoint_capture_coverage = next.coverage; + next.checkpoint_capture_floor = next.gc_floor; + break; + case CheckpointAction::Commit: + next.snapshot_changes = next.snapshot_changes_since_capture; + next.snapshot_changes_since_capture = {}; + if (transaction.checkpoint->baseline) { + next.snapshot_baseline = transaction.checkpoint->baseline; + } + break; + case CheckpointAction::Abort: + next.snapshot_changes_since_capture = {}; + break; + } +} + +size_t PublicationCoordinator::snapshot_change_reserve_bytes() noexcept +{ + // Both the complete pending set and its post-capture successor can change. + return 2 * SnapshotChanges::change_allocation_bound(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/publication_snapshot.h b/lib/crowdb-tree/src/btree/publication_snapshot.h new file mode 100644 index 000000000..d8267709e --- /dev/null +++ b/lib/crowdb-tree/src/btree/publication_snapshot.h @@ -0,0 +1,16 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "published_state.h" + +namespace crowdb::tree +{ +[[nodiscard]] bool logical_page_change(const PageReplacement &page); +[[nodiscard]] Status validate_snapshot_publication(const PublicationTransaction &transaction, + const PublishedTreeState ¤t); +void initialize_snapshot_changes(PublishedTreeState &state, + const std::shared_ptr<detail::AllocationReservation> &metadata); +void prepare_snapshot_changes(const PublicationTransaction &transaction, const PublishedTreeState ¤t, + PublishedTreeState &next); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_inspection.cpp b/lib/crowdb-tree/src/btree/published_inspection.cpp new file mode 100644 index 000000000..f23a462c3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_inspection.cpp @@ -0,0 +1,292 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_inspection.h" + +#include "btree/allocation_bound.h" +#include "btree/tree_core_read_internal.h" +#include "snapshot/captured_frames.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +constexpr uint32_t kInspectionBytes = 1024 * 1024; + +struct Loaded +{ + Status status; + std::shared_ptr<const LeafBase> leaf; + std::shared_ptr<const InnerPageState> inner; + std::atomic<bool> ready{false}; + + void finish(Status result) noexcept + { + status = std::move(result); + ready.store(true, std::memory_order_release); + ready.notify_one(); + } + + [[nodiscard]] Status wait() const + { + while (!ready.load(std::memory_order_acquire)) { + ready.wait(false, std::memory_order_acquire); + } + return status; + } +}; + +struct Inspection +{ + core_read_detail::ReadPreparation prepared; + std::shared_ptr<const PublishedTreeState> view; + + ~Inspection() + { + if (prepared.credit) { + prepared.credit->seal(); + } + if (prepared.io) { + prepared.io->seal(); + } + } + + Status load(uint64_t pid, std::optional<size_t> base = {}) + { + const auto old = view->mapping.get(pid); + if (!old || (base && (!old->leaf || *base >= old->leaf->bases.size()))) { + return Status::corruption("inspection route or base is absent"); + } + if (!base && old->inner) { + return Status::Ok(); + } + auto event = detail::reserved_shared<Loaded>(prepared.credit); + if (base) { + prepared.loader->load_leaf(old->leaf->bases[*base], [event](Status status, auto value) { + event->leaf = std::move(value); + event->finish(std::move(status)); + }); + } + else { + prepared.loader->load_inner(old->inner_backing, [event](Status status, auto value) { + event->inner = std::move(value); + event->finish(std::move(status)); + }); + } + auto status = event->wait(); + if (!status.ok()) { + return status; + } + auto page = detail::reserved_shared<PublishedPage>(prepared.credit, *old); + if (base) { + if (!event->leaf) { + return Status::corruption("inspection loader returned no leaf"); + } + auto leaf = detail::reserved_shared<PageState>(prepared.credit, *old->leaf, prepared.credit); + const auto maximum = leaf->bases[*base].max_slot; + if (maximum) { + for (size_t i = 0; i < event->leaf->count(); ++i) { + if (CellView(event->leaf->cell(i)).slot() > *maximum) { + return Status::corruption("inspection base exceeds captured coverage"); + } + } + } + leaf->bases[*base].resident = std::move(event->leaf); + page->leaf = std::move(leaf); + } + else { + if (!event->inner) { + return Status::corruption("inspection loader returned no index"); + } + page->inner = std::move(event->inner); + } + auto next = detail::reserved_shared<PublishedTreeState>(prepared.credit, *view); + const PageReplacement change{.page_id = pid, .expected = old, .replacement = std::move(page)}; + MappingView::CopyStats copied; + next->mapping = next->mapping.replace({&change, 1}, copied, prepared.credit); + view = std::move(next); + return Status::Ok(); + } + + Status edge(uint64_t pid, bool right, uint64_t *out) + { + for (size_t depth = 0; depth < prepared.limits.max_route_depth; ++depth) { + auto page = view->mapping.get(pid); + if (!page) { + return Status::corruption("inspection fence route is absent"); + } + if (page->level == 0) { + *out = pid; + return Status::Ok(); + } + auto status = load(pid); + if (!status.ok()) { + return status; + } + page = view->mapping.get(pid); + if (page->inner->children.empty()) { + return Status::corruption("inspection index has no children"); + } + const auto child = right ? page->inner->children.back() : page->inner->children.front(); + auto next = view->mapping.get(child); + if (!next || next->level + 1 != page->level) { + return Status::corruption("inspection fence route level mismatch"); + } + pid = child; + } + return Status::resource_exhausted("inspection fence route exceeds depth bound"); + } +}; + +struct CopiedFrame +{ + CopiedFrame(const std::shared_ptr<detail::AllocationReservation> &credit, size_t capacity) + : bytes(credit, capacity, AllocationKind::kFrame) + { + } + + AllocationLease bytes; + std::shared_ptr<const void> dependencies; +}; +} // namespace + +Status inspect_published_page(TreeCore &core, const std::vector<uint32_t> &path, uint64_t expected_version, + NativeFrame *out, uint64_t *version, uint64_t *root, uint32_t *deltas) +try { + if (path.size() > 32 || out == nullptr || version == nullptr || root == nullptr || deltas == nullptr) { + return Status::invalid_argument("page inspection requires outputs and at most 32 levels"); + } + CapturedFrameLimits frames; + frames.max_depth = 33; + frames.encode.frame_bytes = kInspectionBytes; + frames.encode.inline_value_bytes = kInspectionBytes; + frames.encode.max_sources = core.read_limits().max_page_sources; + frames.encode.work_metrics = core.publisher().work_metrics(); + const auto input = core.capture(); + const auto shape = input->mapping.copy_bounds(); + frames.page_metadata_bytes = shape.max_leaf_bytes; + const auto copies = (3 * frames.max_depth) + frames.encode.max_sources; + const auto bound = + detail::AllocationBound(CapturedFrameStream::allocation_bound(frames).value_or(SIZE_MAX)) + .add_product(copies, MappingView::path_copy_bound() + detail::shared_allocation_bound<PublishedPage>() + + detail::shared_allocation_bound<PublishedTreeState>()) + .add_product(core.binding()->cpu()->worker_count() + 1, detail::shared_allocation_bound<Loaded>()) + .add(detail::shared_allocation_bound<CopiedFrame>()) + .value(); + if (!bound) { + return Status::resource_exhausted(); + } + Inspection inspection; + auto status = core_read_detail::prepare(core, input, core.read_limits(), false, *bound, kInspectionBytes, + &inspection.prepared); + if (!status.ok()) { + return status; + } + if (inspection.prepared.limits.max_route_depth > frames.max_depth) { + return Status::resource_exhausted("inspection depth exceeds its bound"); + } + inspection.view = inspection.prepared.view; + const auto captured = inspection.view; + if (expected_version != kInvalidPageId && expected_version != captured->commit_seq) { + return Status::unavailable("tree changed; restart page inspection"); + } + uint64_t pid = captured->root_page_id; + PageKeyRange range; + for (const auto child : path) { + auto page = inspection.view->mapping.get(pid); + if (!page || page->level == 0) { + return Status::unavailable("page path changed; restart inspection"); + } + status = inspection.load(pid); + if (!status.ok()) { + return status; + } + page = inspection.view->mapping.get(pid); + const auto &inner = *page->inner; + if (inner.children.empty() || inner.children.size() != inner.separators.size() + 1 || + child >= inner.children.size()) { + return Status::invalid_argument("child outside inspected page"); + } + if (child != 0) { + range.lower = inner.separators[child - 1]; + } + if (child < inner.separators.size()) { + range.upper = inner.separators[child]; + } + const auto next = inspection.view->mapping.get(inner.children[child]); + if (!next || next->level + 1 != page->level) { + return Status::corruption("inspection route level mismatch"); + } + pid = inner.children[child]; + } + const auto page = inspection.view->mapping.get(pid); + if (!page) { + return Status::not_found(); + } + CapturedEncodeContext context{.range = range}; + if (page->level != 0) { + InnerEncodeFences fences; + status = inspection.edge(pid, false, &fences.lower_leaf); + if (status.ok()) { + status = inspection.edge(pid, true, &fences.upper_leaf); + } + if (!status.ok()) { + return status; + } + context.inner_fences = fences; + } + else { + context.range = page->leaf->range; + } + frames.encode.metadata = inspection.prepared.credit; + for (size_t attempts = 0; attempts <= frames.encode.max_sources; ++attempts) { + auto encoded = prepare_final_page(inspection.view, pid, context, frames.encode); + if (!encoded.status.ok()) { + return encoded.status; + } + if (encoded.needs_base) { + status = inspection.load(pid, encoded.needs_base->base_index); + } + else if (encoded.needs_inner) { + status = inspection.load(pid); + } + else { + if (encoded.encoder->summary().new_overflow_frames != 0) { + return Status::resource_exhausted("inspection frame exceeds its 1 MiB output bound"); + } + NativeFrame result; + result.page_id = pid; + result.frame.resize(kInspectionBytes); + auto owner = detail::reserved_shared<CopiedFrame>(inspection.prepared.credit, inspection.prepared.credit, + result.frame.capacity()); + // Copied inline bytes are self-contained. Only reused overflow + // addresses still need the captured backing after encoding ends. + if (!encoded.encoder->reused_overflow().empty()) { + owner->dependencies = encoded.encoder->dependency_owner(); + } + result.allocation_owner = std::move(owner); + auto step = encoded.encoder->next(result.frame); + if (!step.status.ok() || !step.frame || !step.complete) { + return step.status.ok() ? Status::corruption("inspection did not produce one primary") : step.status; + } + const auto current = core.capture(); + if (current->generation != captured->generation || current->commit_seq != captured->commit_seq) { + return Status::unavailable("tree changed during inspection"); + } + *version = captured->commit_seq; + *root = captured->root_page_id; + *deltas = page->leaf ? static_cast<uint32_t>(page->leaf->overlays.size()) : 0; + *out = std::move(result); + return Status::Ok(); + } + if (!status.ok()) { + return status; + } + } + return Status::resource_exhausted("inspection source bound exceeded"); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_inspection.h b/lib/crowdb-tree/src/btree/published_inspection.h new file mode 100644 index 000000000..5beec828a --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_inspection.h @@ -0,0 +1,11 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/tree_core.h" + +namespace crowdb::tree +{ +Status inspect_published_page(TreeCore &core, const std::vector<uint32_t> &path, uint64_t expected_version, + NativeFrame *out, uint64_t *version, uint64_t *root, uint32_t *deltas); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_mapping.cpp b/lib/crowdb-tree/src/btree/published_mapping.cpp new file mode 100644 index 000000000..c32df67c1 --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_mapping.cpp @@ -0,0 +1,262 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "logical_page.h" +#include "overflow_backing.h" +#include "published_state.h" +#include "snapshot/primary_backing.h" + +#include <algorithm> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +size_t add_bound(size_t left, size_t right) noexcept +{ + return left > SIZE_MAX - right ? SIZE_MAX : left + right; +} + +size_t multiply_bound(size_t count, size_t bytes) noexcept +{ + return bytes != 0 && count > SIZE_MAX / bytes ? SIZE_MAX : count * bytes; +} + +uint64_t oldest_time(uint64_t left, uint64_t right) noexcept +{ + if (left == 0) { + return right; + } + return right == 0 ? left : std::min(left, right); +} + +bool has_candidate(const std::shared_ptr<const PublishedPage> &page) noexcept +{ + if (!page) { + return false; + } + const auto &metrics = page->metrics; + return metrics.consolidate_candidate || metrics.split_candidate || metrics.merge_candidate || + metrics.writeback_candidate || metrics.evict_candidate; +} + +void include_bounds(MappingView::CopyBounds &total, const MappingView::CopyBounds &part) noexcept +{ + total.merge_candidate_pages = add_bound(total.merge_candidate_pages, part.merge_candidate_pages); + total.leaf_pages = add_bound(total.leaf_pages, part.leaf_pages); + total.leaf_bytes = add_bound(total.leaf_bytes, part.leaf_bytes); + total.max_leaf_bytes = std::max(total.max_leaf_bytes, part.max_leaf_bytes); + total.max_leaf_append_bytes = std::max(total.max_leaf_append_bytes, part.max_leaf_append_bytes); + total.max_leaf_bases = std::max(total.max_leaf_bases, part.max_leaf_bases); + total.may_have_overflow_values = total.may_have_overflow_values || part.may_have_overflow_values; + total.max_inner_bytes = std::max(total.max_inner_bytes, part.max_inner_bytes); + total.max_level = std::max(total.max_level, part.max_level); + total.candidate_since_ns = oldest_time(total.candidate_since_ns, part.candidate_since_ns); + if (part.writeback_page_since_ns != 0 && + (total.writeback_page_since_ns == 0 || part.writeback_page_since_ns < total.writeback_page_since_ns || + (part.writeback_page_since_ns == total.writeback_page_since_ns && + part.oldest_writeback_page_pid < total.oldest_writeback_page_pid))) { + total.writeback_page_since_ns = part.writeback_page_since_ns; + total.oldest_writeback_page_pid = part.oldest_writeback_page_pid; + } + if (part.writeback_since_ns != 0 && + (total.writeback_since_ns == 0 || part.writeback_since_ns < total.writeback_since_ns || + (part.writeback_since_ns == total.writeback_since_ns && + part.oldest_writeback_pid < total.oldest_writeback_pid))) { + total.writeback_since_ns = part.writeback_since_ns; + total.oldest_writeback_pid = part.oldest_writeback_pid; + } +} + +MappingView::CopyBounds page_bounds(const PublishedPage &page) noexcept +{ + if (page.level != 0 && page.inner) { + auto bytes = detail::shared_allocation_bound<PublishedPage>() + InnerPageState::allocation_bound(); + bytes = add_bound(bytes, multiply_bound(page.inner->children.size(), 3 * sizeof(uint64_t))); + bytes = add_bound(bytes, multiply_bound(page.inner->separators.size(), 3 * sizeof(OwnedKey))); + return {.max_inner_bytes = bytes, .max_level = page.level}; + } + if (page.level != 0 || !page.leaf) { + return {.max_level = page.level}; + } + // The old overlay array and its replacement can coexist during insert. + // New run references are counted separately by the incoming batch bound. + auto bytes = detail::shared_allocation_bound<PublishedPage>() + PageState::allocation_bound(); + bytes = add_bound(bytes, multiply_bound(page.leaf->bases.size(), sizeof(BaseRef))); + bytes = + add_bound(bytes, multiply_bound(page.leaf->overlays.size(), 3 * sizeof(std::shared_ptr<const OverlayBatch>))); + const bool overflow = + std::any_of(page.leaf->bases.begin(), page.leaf->bases.end(), + [&](const auto &base) { + // Completed writers attach the exact immutable dependency set. + // Missing metadata remains unknown; a null set does not prove an + // inline-only frame. Recovery can supply the same proof through + // its validated primary receipt without a resident decoded page. + if (base.backing && base.backing->overflow && base.backing->overflow->pages.empty()) { + return false; + } + const auto *frame = page.primary ? &page.primary->frame : nullptr; + return !frame || !base.backing || frame->backing != base.backing || !frame->stored_overflow || + !frame->stored_overflow->empty(); + }) || + std::any_of(page.leaf->overlays.begin(), page.leaf->overlays.end(), + [](const auto &run) { return !run || run->has_overflow_values(); }); + return {.leaf_pages = 1, + .merge_candidate_pages = static_cast<size_t>(page.metrics.merge_candidate), + .leaf_bytes = bytes, + .max_leaf_bytes = bytes, + .max_leaf_append_bytes = bytes, + .max_leaf_bases = page.leaf->bases.size(), + .may_have_overflow_values = overflow}; +} +} // namespace + +MappingView::CopyBounds MappingView::copy_bounds() const noexcept +{ + return root_ ? root_->bounds : CopyBounds{}; +} + +std::optional<uint64_t> MappingView::oldest_writeback_leaf() const noexcept +{ + const auto bounds = copy_bounds(); + return bounds.writeback_since_ns != 0 ? std::optional<uint64_t>(bounds.oldest_writeback_pid) : std::nullopt; +} + +std::optional<uint64_t> MappingView::oldest_writeback_page() const noexcept +{ + const auto bounds = copy_bounds(); + return bounds.writeback_page_since_ns != 0 ? std::optional<uint64_t>(bounds.oldest_writeback_page_pid) + : std::nullopt; +} + +size_t MappingView::path_copy_bound() noexcept +{ + return (4 * detail::shared_allocation_bound<Directory>()) + detail::shared_allocation_bound<Segment>(); +} + +std::shared_ptr<const PublishedPage> MappingView::get(uint64_t page_id) const +{ + if (page_id >= kPageLimit) { + return {}; + } + auto directory = root_; + for (unsigned shift : {24U, 18U, 12U}) { + if (!directory) { + return {}; + } + directory = directory->children[(page_id >> shift) & 63]; + } + if (!directory) { + return {}; + } + const auto &segment = directory->segments[(page_id >> 6) & 63]; + return segment ? segment->pages[page_id & 63] : nullptr; +} + +std::shared_ptr<const MappingView::Directory> +MappingView::replace_directory(const std::shared_ptr<const Directory> &old, unsigned shift, + std::span<const PageReplacement> replacements, CopyStats &copied, + const std::shared_ptr<detail::AllocationReservation> &metadata, uint64_t now_ns) +{ + auto next = old ? detail::reserved_shared<Directory>(metadata, *old) : detail::reserved_shared<Directory>(metadata); + ++copied.directories; + copied.bytes += sizeof(Directory); + while (!replacements.empty()) { + const auto index = (replacements.front().page_id >> shift) & 63; + size_t count = 1; + while (count < replacements.size() && ((replacements[count].page_id >> shift) & 63) == index) { + ++count; + } + auto group = replacements.first(count); + if (shift > 6) { + next->children[index] = + replace_directory(next->children[index], shift - 6, group, copied, metadata, now_ns); + } + else { + auto segment = next->segments[index] ? detail::reserved_shared<Segment>(metadata, *next->segments[index]) + : detail::reserved_shared<Segment>(metadata); + ++copied.segments; + copied.bytes += sizeof(Segment); + for (const auto &replacement : group) { + const auto slot = replacement.page_id & 63; + auto &since = segment->candidate_since_ns[slot]; + if (!has_candidate(replacement.replacement)) { + since = 0; + } + else if (since == 0) { + since = now_ns; + } + auto &writeback_since = segment->writeback_since_ns[slot]; + const auto &page = replacement.replacement; + if (!page || !page->metrics.dirty || !page->metrics.writeback_candidate) { + writeback_since = 0; + } + else if (writeback_since == 0) { + writeback_since = now_ns; + } + segment->pages[slot] = replacement.replacement; + } + segment->bounds = {}; + for (const auto &page : segment->pages) { + if (page) { + include_bounds(segment->bounds, page_bounds(*page)); + } + } + const auto prefix = group.front().page_id & ~(kFanout - 1); + for (size_t slot = 0; slot < kFanout; ++slot) { + if (segment->writeback_since_ns[slot] != 0) { + CopyBounds candidate{.writeback_page_since_ns = segment->writeback_since_ns[slot], + .oldest_writeback_page_pid = prefix + slot}; + if (segment->pages[slot]->level == 0) { + candidate.writeback_since_ns = segment->writeback_since_ns[slot]; + candidate.oldest_writeback_pid = prefix + slot; + } + include_bounds(segment->bounds, candidate); + } + } + for (const auto since : segment->candidate_since_ns) { + segment->bounds.candidate_since_ns = oldest_time(segment->bounds.candidate_since_ns, since); + } + const bool empty = std::none_of(segment->pages.begin(), segment->pages.end(), + [](const auto &page) { return static_cast<bool>(page); }); + next->segments[index] = empty ? nullptr : std::move(segment); + } + replacements = replacements.subspan(count); + } + next->bounds = {}; + for (const auto &child : next->children) { + if (child) { + include_bounds(next->bounds, child->bounds); + } + } + for (const auto &segment : next->segments) { + if (segment) { + include_bounds(next->bounds, segment->bounds); + } + } + const bool has_children = std::any_of(next->children.begin(), next->children.end(), + [](const auto &child) { return static_cast<bool>(child); }); + const bool has_segments = std::any_of(next->segments.begin(), next->segments.end(), + [](const auto &segment) { return static_cast<bool>(segment); }); + return has_children || has_segments ? std::move(next) : nullptr; +} + +MappingView MappingView::replace(std::span<const PageReplacement> replacements, CopyStats &copied, + const std::shared_ptr<detail::AllocationReservation> &metadata) const +{ + uint64_t previous = 0; + bool first = true; + for (const auto &replacement : replacements) { + if (replacement.page_id >= kPageLimit || (!first && replacement.page_id <= previous)) { + throw std::invalid_argument("mapping replacements must have unique ordered PIDs"); + } + first = false; + previous = replacement.page_id; + } + auto result = *this; + if (!replacements.empty()) { + result.root_ = replace_directory(root_, 24, replacements, copied, metadata, tree_metrics_now_ns()); + } + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_mapping_visit.cpp b/lib/crowdb-tree/src/btree/published_mapping_visit.cpp new file mode 100644 index 000000000..dc302a22a --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_mapping_visit.cpp @@ -0,0 +1,69 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "published_state.h" + +namespace crowdb::tree +{ +void MappingView::visit_directory(const Directory &directory, unsigned shift, uint64_t prefix, void *context, + Visitor visitor) +{ + for (size_t i = 0; i < kFanout; ++i) { + const auto page_prefix = prefix | (i << shift); + if (shift > 6) { + if (directory.children[i]) { + visit_directory(*directory.children[i], shift - 6, page_prefix, context, visitor); + } + } + else if (directory.segments[i]) { + const auto &segment = *directory.segments[i]; + for (size_t j = 0; j < kFanout; ++j) { + if (segment.pages[j]) { + visitor(context, page_prefix | j, segment.pages[j]); + } + } + } + } +} + +void MappingView::visit(void *context, Visitor visitor) const +{ + if (root_) { + visit_directory(*root_, 24, 0, context, visitor); + } +} + +bool MappingView::visit_from(uint64_t &next_pid, size_t max_slots, void *context, Visitor visitor) const +{ + if (!root_) { + next_pid = kPageLimit; + return true; + } + for (size_t visited = 0; visited < max_slots && next_pid < kPageLimit; ++visited) { + const auto *directory = root_.get(); + unsigned shift = 24; + while (shift > 6) { + const auto &child = directory->children[(next_pid >> shift) & (kFanout - 1)]; + if (!child) { + break; + } + directory = child.get(); + shift -= 6; + } + if (shift > 6) { + next_pid = ((next_pid >> shift) + 1) << shift; + continue; + } + const auto &segment = directory->segments[(next_pid >> 6) & (kFanout - 1)]; + if (!segment) { + next_pid = ((next_pid >> 6) + 1) << 6; + continue; + } + const auto pid = next_pid++; + const auto &page = segment->pages[pid & (kFanout - 1)]; + if (page) { + visitor(context, pid, page); + } + } + return next_pid >= kPageLimit; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_read.cpp b/lib/crowdb-tree/src/btree/published_read.cpp new file mode 100644 index 000000000..561020aab --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_read.cpp @@ -0,0 +1,194 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/published_read_internal.h" + +namespace crowdb::tree +{ +using namespace published_read_detail; + +struct PublishedGet::State : ReadPump +{ + State(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<PageLoader> transport, Slice wanted, + PublishedReadLimits bounds) + : ReadPump(std::move(captured), std::move(transport), bounds), + key(wanted, bounds.metadata), + walker(view->root_page_id, view->mapping, bounds.max_route_depth, false, bounds.metadata) + { + } + + OwnedKey key; + PublishedRouteCursor walker; + PublishedGetResult report; + LogicalCell memtable; + std::shared_ptr<const PageState> leaf; + size_t loading_base = 0; + bool started = false; + bool loading_value = false; + bool loading_index = false; + + Status &status() override + { + return report.status; + } + + void begin() override + { + if (started) { + report.status = Status::unavailable("point read is a single operation"); + return; + } + started = true; + for (size_t index = 0; index < view->sources->tables.size(); ++index) { + const auto &table = view->sources->tables[index]; + auto guard = table->read_guard(limits.metadata); + auto cursor = table->cursor_from(Slice(key), true); + if (cursor.valid() && cursor.key() == Slice(key)) { + auto owner = cursor.cell_owner_at(view->sources->frontier(index)); + if (owner && owner->slot > view->sources->floor && (!memtable.owner || owner->slot > memtable.slot)) { + memtable = owned_cell(std::move(owner)); + } + } + } + report.status = walker.seek(Slice(key), false); + leaf = walker.leaf(); + } + + bool advance() override + { + if (auto backing = walker.needs_inner()) { + loading_index = true; + load_inner(std::move(backing)); + return false; + } + if (loading_value) { + return true; + } + LogicalLookup lookup; + if (leaf) { + lookup = leaf->lookup(Slice(key)); + } + if (!lookup.status.ok()) { + report.status = lookup.status; + return true; + } + const bool use_memtable = memtable.owner && (!lookup.cell.owner || memtable.slot >= lookup.cell.slot); + if (lookup.state == LogicalLookupState::kNeedsBase) { + std::optional<size_t> needed; + if (use_memtable) { + for (size_t i = 0; i < leaf->bases.size(); ++i) { + const auto &base = leaf->bases[i]; + if (base.range.contains(Slice(key)) && !base.resident && + (!base.max_slot || memtable.slot < *base.max_slot)) { + needed = i; + break; + } + } + } + else { + needed = lookup.base_index; + } + if (needed) { + loading_base = *needed; + load_base(leaf->bases[*needed]); + return false; + } + } + if (leaf) { + uint64_t avoided = 0; + for (const auto &base : leaf->bases) { + avoided += static_cast<uint64_t>(!base.resident && base.range.contains(Slice(key))); + } + record_work(limits.metadata, TreeWorkCounter::ProxyAvoidedLoads, avoided, TreeWorkCause::Read); + } + report.cell = use_memtable ? memtable : lookup.cell; + report.found = report.cell.owner && !report.cell.tombstone(); + if (report.found && report.cell.overflow()) { + loading_value = true; + load_value(report.cell); + return false; + } + return true; + } + + void loaded(const LoadEvent &event) override + { + if (loading_index) { + loading_index = false; + report.status = walker.resume_inner(event.inner); + leaf = walker.leaf(); + return; + } + if (loading_value) { + if (!event.value || report.cell.encoded.size() < kOverflowCellSize || + event.value->size() != CellView(report.cell.encoded).overflow_len()) { + report.status = Status::corruption("overflow loader returned no value owner"); + return; + } + materialize_value(&report.cell, event.value); + return; + } + if (!event.leaf) { + report.status = Status::corruption("leaf loader returned no frame owner"); + return; + } + auto resident = detail::reserved_shared<PageState>(limits.metadata, *leaf, limits.metadata); + auto &base = resident->bases[loading_base]; + base.resident = event.leaf; + leaf = std::move(resident); + } +}; + +std::optional<size_t> PublishedGet::allocation_bound(const PublishedReadLimits &limits, size_t workers) noexcept +{ + return detail::AllocationBound(detail::shared_allocation_bound<State>() + + detail::shared_allocation_bound<PublishedGet>()) + .add(read_metadata_allocation_bound(limits, workers)) + .add(OwnedKey::allocation_overhead()) + .add(limits.max_key_bytes) + .add(1) + .value(); +} + +Status PublishedGet::create(std::shared_ptr<const PublishedTreeState> view, std::shared_ptr<PageLoader> loader, + Slice key, PublishedReadLimits limits, std::shared_ptr<PublishedGet> *out) +{ + auto status = validate_read(view, limits); + if (!status.ok()) { + return status; + } + if (out == nullptr || key.size() > limits.max_key_bytes) { + return Status::invalid_argument("point read key exceeds bounds or output is absent"); + } + try { + auto state = detail::reserved_shared<State>(limits.metadata, std::move(view), std::move(loader), key, limits); + *out = detail::reserved_construct<PublishedGet>(limits.metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) PublishedGet(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted("point read allocation failed"); + } + return Status::Ok(); +} + +detail::CallbackSender PublishedGet::sender() noexcept +{ + return {this, &PublishedGet::submit}; +} + +void PublishedGet::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<PublishedGet *>(source)->state_->start(complete, context); +} + +const PublishedGetResult &PublishedGet::result() const noexcept +{ + return state_->report; +} + +void PublishedGet::request_stop() noexcept +{ + state_->stop(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_read.h b/lib/crowdb-tree/src/btree/published_read.h new file mode 100644 index 000000000..b521e1a1d --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_read.h @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_load.h" +#include "btree/published_state.h" +#include "stdexec_adapter.h" + +namespace crowdb::tree +{ +struct PublishedReadLimits +{ + size_t max_memtables = 64; + size_t max_page_sources = 64; + size_t max_key_bytes = 256 * 1024; + size_t max_route_depth = 64; + size_t max_work = 4096; + size_t max_entries = 128; + size_t max_result_bytes = 1024 * 1024; + // Captured mapping shape, independent of the cursor's source work window. + size_t page_metadata_bytes = 0; + std::shared_ptr<detail::AllocationReservation> metadata = nullptr; +}; + +struct PublishedGetResult +{ + Status status; + bool found = false; + LogicalCell cell; +}; + +struct PublishedScanEntry +{ + OwnedKey key; + LogicalCell cell; +}; + +struct PublishedScanResult +{ + Status status; + detail::ReservedVector<PublishedScanEntry> entries; + bool done = false; + // An eligible candidate is retained because it cannot fit the byte budget. + bool byte_limit_reached = false; + size_t work = 0; +}; + +struct PublishedScanOptions +{ + std::optional<OwnedKey> start = std::nullopt; + bool start_inclusive = false; + bool keys_only = false; + bool include_tombstones = false; + bool allow_oversized_first = false; + // When absent, each connection applies max_result_bytes independently. + // A supplied budget spans every work window; SIZE_MAX means no public cap. + std::optional<size_t> byte_budget = std::nullopt; +}; + +class PublishedGet +{ + struct State; + + public: + static Status create(std::shared_ptr<const PublishedTreeState> view, std::shared_ptr<PageLoader> loader, Slice key, + PublishedReadLimits limits, std::shared_ptr<PublishedGet> *out); + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] static std::optional<size_t> allocation_bound(const PublishedReadLimits &limits, + size_t workers) noexcept; + [[nodiscard]] const PublishedGetResult &result() const noexcept; + void request_stop() noexcept; + + private: + explicit PublishedGet(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; + +// This object is the continuation: mapping/catalog/floor and source positions +// stay fixed across windows. Each next_sender() connection produces one bounded +// window; an empty nonterminal window is valid after skipped/covered entries. +// Values own their bytes independently; no epoch guard crosses a callback. +class PublishedScan +{ + struct State; + + public: + static Status create(std::shared_ptr<const PublishedTreeState> view, std::shared_ptr<PageLoader> loader, + PageKeyRange range, bool reverse, PublishedReadLimits limits, + std::shared_ptr<PublishedScan> *out, PublishedScanOptions options = {}); + [[nodiscard]] detail::CallbackSender next_sender() noexcept; + [[nodiscard]] static std::optional<size_t> allocation_bound(const PublishedReadLimits &limits, + size_t workers) noexcept; + [[nodiscard]] const PublishedScanResult &result() const noexcept; + [[nodiscard]] uint64_t skipped_subtrees() const noexcept; + void request_stop() noexcept; + + private: + explicit PublishedScan(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_read_budget.cpp b/lib/crowdb-tree/src/btree/published_read_budget.cpp new file mode 100644 index 000000000..11e45b2c3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_read_budget.cpp @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/published_read_internal.h" + +#include <algorithm> + +namespace crowdb::tree::published_read_detail +{ +std::optional<size_t> read_metadata_allocation_bound(const PublishedReadLimits &limits, size_t workers) noexcept +{ + using detail::AllocationBound; + using detail::shared_allocation_bound; + if (workers == 0 || limits.max_page_sources == 0 || limits.max_work == 0 || limits.max_entries == 0 || + limits.max_result_bytes == 0) { + return {}; + } + const auto callbacks = AllocationBound(workers).add(2).value(); + const auto page = + AllocationBound(PageState::allocation_bound()) + .add_product(limits.max_page_sources, sizeof(BaseRef) + sizeof(std::shared_ptr<const OverlayBatch>)) + .value(); + const auto key = AllocationBound(OwnedKey::allocation_overhead()).add(limits.max_key_bytes).add(1).value(); + const auto participants = AllocationBound().add_product(workers, limits.max_memtables).value(); + if (!callbacks || !page || !key || !participants) { + return {}; + } + const auto callback = + AllocationBound(shared_allocation_bound<LoadEvent>()).add(std::max(*page, limits.page_metadata_bytes)).value(); + if (!callback) { + return {}; + } + // Route pending keys can coexist with the key used to resume the prior + // decoder. PageState replacement keeps both source arrays alive briefly. + return AllocationBound() + .add(PublishedRouteCursor::allocation_bound(limits.max_route_depth)) + .add_product(*callbacks, *callback) + .add_product(3, *key) + .add_product(*participants, EpochManager::participant_allocation_bytes()) + .value(); +} +} // namespace crowdb::tree::published_read_detail diff --git a/lib/crowdb-tree/src/btree/published_read_internal.h b/lib/crowdb-tree/src/btree/published_read_internal.h new file mode 100644 index 000000000..f0ee2415c --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_read_internal.h @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/published_read.h" +#include "btree/published_route.h" + +namespace crowdb::tree::published_read_detail +{ +struct LoadEvent +{ + std::atomic<bool> ready{false}; + Status status; + std::shared_ptr<const LeafBase> leaf; + std::shared_ptr<const InnerPageState> inner; + std::shared_ptr<const buffer> value; +}; + +class ReadPump : public std::enable_shared_from_this<ReadPump> +{ + public: + ReadPump(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<PageLoader> transport, + PublishedReadLimits bounds) + : view(std::move(captured)), + loader(std::move(transport)), + limits(std::move(bounds)) + { + } + + virtual ~ReadPump() = default; + void start(detail::CallbackComplete complete, void *context) noexcept; + void stop() noexcept; + + protected: + virtual void begin() = 0; + virtual bool advance() = 0; + virtual void loaded(const LoadEvent &event) = 0; + virtual Status &status() = 0; + void load_base(BaseRef base); + void load_inner(std::shared_ptr<const BackingRef> backing); + void load_value(const LogicalCell &cell); + std::shared_ptr<const PublishedTreeState> view; + std::shared_ptr<PageLoader> loader; + PublishedReadLimits limits; + + private: + void wake() noexcept; + void drain() noexcept; + void finish(detail::CallbackSignal signal) noexcept; + std::atomic<bool> gate_{false}; + std::atomic<bool> wake_{false}; + std::atomic<uint8_t> lifecycle_{0}; + std::atomic<bool> stopped_{false}; + bool begin_pending_ = false; + detail::CallbackComplete complete_ = nullptr; + void *context_ = nullptr; + std::shared_ptr<LoadEvent> event_; +}; + +Status validate_read(const std::shared_ptr<const PublishedTreeState> &view, const PublishedReadLimits &limits); +[[nodiscard]] std::optional<size_t> read_metadata_allocation_bound(const PublishedReadLimits &limits, + size_t workers) noexcept; +LogicalCell owned_cell(std::shared_ptr<const CellVersion> owner); +void materialize_value(LogicalCell *cell, const std::shared_ptr<const buffer> &value); +} // namespace crowdb::tree::published_read_detail diff --git a/lib/crowdb-tree/src/btree/published_read_pump.cpp b/lib/crowdb-tree/src/btree/published_read_pump.cpp new file mode 100644 index 000000000..3e30413b1 --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_read_pump.cpp @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_read_internal.h" + +#include <cassert> + +namespace crowdb::tree::published_read_detail +{ +void ReadPump::start(detail::CallbackComplete complete, void *context) noexcept +{ + uint8_t expected = 0; + if (!lifecycle_.compare_exchange_strong(expected, 1, std::memory_order_acq_rel)) { + complete(context, detail::CallbackSignal::kError, Status::unavailable("read operation already running")); + return; + } + complete_ = complete; + context_ = context; + begin_pending_ = true; + lifecycle_.store(2, std::memory_order_release); + wake(); +} + +void ReadPump::stop() noexcept +{ + stopped_.store(true, std::memory_order_release); + wake(); +} + +void ReadPump::wake() noexcept +{ + auto owner = weak_from_this().lock(); + assert(owner); + wake_.store(true, std::memory_order_release); + if (gate_.exchange(true, std::memory_order_acq_rel)) { + return; + } + for (;;) { + while (wake_.exchange(false, std::memory_order_acq_rel)) { + drain(); + } + // Synchronize with a waker that found the pump running. A release + // store followed by a load can miss both sides of that notification. + gate_.exchange(false, std::memory_order_acq_rel); + if (!wake_.load(std::memory_order_acquire) || gate_.exchange(true, std::memory_order_acq_rel)) { + return; + } + } +} + +void ReadPump::drain() noexcept +{ + if (lifecycle_.load(std::memory_order_acquire) != 2) { + return; + } + try { + if (begin_pending_) { + begin_pending_ = false; + begin(); + } + if (stopped_.load(std::memory_order_acquire)) { + status() = Status::unavailable("read stopped"); + finish(detail::CallbackSignal::kStopped); + return; + } + if (event_) { + if (!event_->ready.load(std::memory_order_acquire)) { + return; + } + auto event = std::move(event_); + if (!event->status.ok()) { + status() = std::move(event->status); + } + else { + loaded(*event); + } + } + if (!status().ok()) { + finish(detail::CallbackSignal::kError); + } + else if (advance()) { + finish(status().ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError); + } + } + catch (const std::bad_alloc &) { + status() = Status::resource_exhausted("read operation allocation failed"); + finish(detail::CallbackSignal::kError); + } + catch (...) { + status() = Status::internal_error("read operation threw"); + finish(detail::CallbackSignal::kError); + } +} + +void ReadPump::finish(detail::CallbackSignal signal) noexcept +{ + auto callback = complete_; + auto *context = context_; + Status outcome = status(); + lifecycle_.store(0, std::memory_order_release); + callback(context, signal, std::move(outcome)); +} + +void ReadPump::load_base(BaseRef base) +{ + auto event = detail::reserved_shared<LoadEvent>(limits.metadata); + event_ = event; + if (!loader) { + event->status = Status::unavailable("cold leaf requires an asynchronous loader"); + event->ready.store(true, std::memory_order_release); + wake(); + return; + } + auto self = shared_from_this(); + loader->load_leaf(std::move(base), [self, event](Status status, std::shared_ptr<const LeafBase> leaf) { + event->status = std::move(status); + event->leaf = std::move(leaf); + event->ready.store(true, std::memory_order_release); + self->wake(); + }); +} + +void ReadPump::load_value(const LogicalCell &cell) +{ + auto event = detail::reserved_shared<LoadEvent>(limits.metadata); + event_ = event; + if (!loader || cell.encoded.size() < kOverflowCellSize) { + event->status = Status::not_supported("overflow read requires a captured backing resolver"); + event->ready.store(true, std::memory_order_release); + wake(); + return; + } + const CellView encoded(cell.encoded); + auto self = shared_from_this(); + loader->load_overflow(cell.backing, encoded.overflow_head(), encoded.overflow_len(), + [self, event](Status status, std::shared_ptr<const buffer> value) { + event->status = std::move(status); + event->value = std::move(value); + event->ready.store(true, std::memory_order_release); + self->wake(); + }); +} + +void ReadPump::load_inner(std::shared_ptr<const BackingRef> backing) +{ + auto event = detail::reserved_shared<LoadEvent>(limits.metadata); + event_ = event; + if (!loader) { + event->status = Status::unavailable("cold index requires an asynchronous loader"); + event->ready.store(true, std::memory_order_release); + wake(); + return; + } + auto self = shared_from_this(); + loader->load_inner(std::move(backing), [self, event](Status status, std::shared_ptr<const InnerPageState> inner) { + event->status = std::move(status); + event->inner = std::move(inner); + event->ready.store(true, std::memory_order_release); + self->wake(); + }); +} + +Status validate_read(const std::shared_ptr<const PublishedTreeState> &view, const PublishedReadLimits &limits) +{ + if (!view || !view->sources || limits.max_route_depth == 0 || limits.max_page_sources == 0 || + limits.max_work == 0 || limits.max_entries == 0 || limits.max_result_bytes == 0) { + return Status::invalid_argument("read requires a captured view and finite positive limits"); + } + if (view->sources->tables.size() > limits.max_memtables) { + return Status::resource_exhausted("read MemTable source budget exhausted"); + } + for (const auto &source : view->sources->tables) { + if (!source) { + return Status::corruption("captured MemTable source has no owner"); + } + } + return Status::Ok(); +} + +LogicalCell owned_cell(std::shared_ptr<const CellVersion> owner) +{ + EntryRef entry; + entry.owner = std::move(owner); + return LogicalCell::from_entry(entry); +} + +void materialize_value(LogicalCell *cell, const std::shared_ptr<const buffer> &value) +{ + cell->owner = value; + cell->value = value->slice(); + cell->encoded = {}; + cell->flags &= static_cast<uint8_t>(~kFlagOverflow); +} +} // namespace crowdb::tree::published_read_detail diff --git a/lib/crowdb-tree/src/btree/published_read_route.cpp b/lib/crowdb-tree/src/btree/published_read_route.cpp new file mode 100644 index 000000000..203b6f18e --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_read_route.cpp @@ -0,0 +1,154 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_route.h" + +#include <algorithm> +#include <utility> + +namespace crowdb::tree +{ +PageKeyRange PublishedRouteCursor::child_range(const Frame &frame) const +{ + const auto &separators = frame.inner->separators; + return {.lower = frame.child == 0 ? frame.bounds.lower : std::optional(separators[frame.child - 1]), + .upper = frame.child == separators.size() ? frame.bounds.upper : std::optional(separators[frame.child])}; +} + +Status PublishedRouteCursor::seek(std::optional<Slice> key, bool exclusive) +{ + skipped_ = 0; + stack_.clear(); + leaf_.reset(); + leaf_id_ = kInvalidPageId; + pending_.reset(); + if (root_ == kInvalidPageId && mapping_.empty()) { + return Status::Ok(); + } + if (selection_.empty()) { + skipped_ = 1; + return Status::Ok(); + } + return descend(root_, {}, key, exclusive, std::nullopt); +} + +Status PublishedRouteCursor::validate_index(const InnerPageState &index, const PageKeyRange &bounds) const +{ + if (index.children.empty() || index.children.size() != index.separators.size() + 1 || + index.children.size() > 4096 || stack_.size() >= max_depth_) { + return Status::corruption("captured index exceeds bounded routing layout"); + } + for (const auto id : index.children) { + if (id >= MappingView::kPageLimit) { + return Status::corruption("captured index has an invalid child PID"); + } + } + for (size_t i = 0; i < index.separators.size(); ++i) { + const Slice separator(index.separators[i]); + if ((i != 0 && index.separators[i - 1] >= index.separators[i]) || + (bounds.lower && separator < Slice(*bounds.lower)) || (bounds.upper && separator > Slice(*bounds.upper))) { + return Status::corruption("captured routing separators are not strictly ordered"); + } + } + return Status::Ok(); +} + +Status PublishedRouteCursor::descend(uint64_t id, PageKeyRange bounds, std::optional<Slice> key, bool exclusive, + std::optional<uint32_t> expected_level, + std::shared_ptr<const InnerPageState> loaded) +{ + for (;;) { + const auto page = mapping_.get(id); + if (!page || (expected_level && page->level != *expected_level) || page->level > max_depth_) { + return Status::corruption("captured routing page is absent or has an invalid level"); + } + if (page->level == 0) { + if (!page->leaf || (!stack_.empty() && + (page->leaf->range.lower != bounds.lower || page->leaf->range.upper != bounds.upper))) { + return Status::corruption("captured leaf bounds disagree with routing"); + } + leaf_ = page->leaf; + leaf_id_ = id; + return Status::Ok(); + } + // A resumed index belongs only to this level. A conditional with the + // const resident shared_ptr can copy loaded instead of consuming it. + auto inner = std::exchange(loaded, {}); + if (!inner) { + inner = page->inner; + } + if (!inner) { + if (!page->inner_backing || !page->inner_backing->readable()) { + return Status::corruption("captured index has neither resident state nor owned backing"); + } + pending_ = + Pending{.id = id, + .page = page, + .bounds = std::move(bounds), + .key = key ? std::make_optional<OwnedKey>(*key, stack_.get_allocator().owner()) : std::nullopt, + .exclusive = exclusive}; + return Status::Ok(); + } + auto status = validate_index(*inner, bounds); + if (!status.ok()) { + return status; + } + size_t child = reverse_ ? inner->children.size() - 1 : 0; + if (key) { + auto at = + reverse_ && exclusive + ? std::lower_bound(inner->separators.begin(), inner->separators.end(), *key, + [](const OwnedKey &separator, Slice value) { return Slice(separator) < value; }) + : std::upper_bound(inner->separators.begin(), inner->separators.end(), *key, + [](Slice value, const OwnedKey &separator) { return value < Slice(separator); }); + child = static_cast<size_t>(at - inner->separators.begin()); + } + skipped_ += reverse_ ? inner->children.size() - child - 1 : child; + stack_.push_back({.page = page, .inner = std::move(inner), .bounds = std::move(bounds), .child = child}); + bounds = child_range(stack_.back()); + id = stack_.back().inner->children[child]; + expected_level = page->level - 1; + } +} + +Status PublishedRouteCursor::resume_inner(std::shared_ptr<const InnerPageState> inner) +{ + if (!pending_) { + return Status::invalid_argument("routing cursor has no pending index load"); + } + if (!inner) { + return Status::corruption("index loader returned no immutable state"); + } + auto pending = std::move(*pending_); + pending_.reset(); + const auto key = pending.key ? std::optional<Slice>(*pending.key) : std::nullopt; + return descend(pending.id, std::move(pending.bounds), key, pending.exclusive, pending.page->level, + std::move(inner)); +} + +Status PublishedRouteCursor::next() +{ + if (pending_) { + return Status::unavailable("routing cursor is waiting for an index load"); + } + leaf_.reset(); + leaf_id_ = kInvalidPageId; + while (!stack_.empty()) { + auto &frame = stack_.back(); + const bool end = reverse_ ? frame.child == 0 : frame.child + 1 == frame.inner->children.size(); + if (end) { + stack_.pop_back(); + continue; + } + frame.child = reverse_ ? frame.child - 1 : frame.child + 1; + const uint64_t id = frame.inner->children[frame.child]; + const uint32_t level = frame.page->level - 1; + auto range = child_range(frame); + if (!selection_.overlaps(range)) { + ++skipped_; + continue; + } + return descend(id, std::move(range), std::nullopt, false, level); + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_route.h b/lib/crowdb-tree/src/btree/published_route.h new file mode 100644 index 000000000..94e0cea03 --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_route.h @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/logical_page.h" +#include "btree/published_state.h" + +namespace crowdb::tree +{ +// Owns one immutable mapping and the current root-to-leaf path. A successful +// step yields a leaf, end, or needs_inner(). The caller loads that backing and +// resumes; no thread-bound guard or worker wait is carried between steps. +class PublishedRouteCursor +{ + public: + PublishedRouteCursor(uint64_t root_page_id, MappingView mapping, size_t max_depth, bool reverse, + std::shared_ptr<detail::AllocationReservation> metadata = {}, PageKeyRange selection = {}) + : root_(root_page_id), + mapping_(std::move(mapping)), + max_depth_(max_depth), + reverse_(reverse), + selection_(std::move(selection)), + stack_(detail::ReservationAllocator<Frame>(std::move(metadata))) + { + } + + Status seek(std::optional<Slice> key, bool exclusive); + Status next(); + Status resume_inner(std::shared_ptr<const InnerPageState> inner); + + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t max_depth) noexcept + { + if (max_depth == 0 || max_depth > 64) { + return {}; + } + // Vector growth can temporarily retain both old and new arrays. + return sizeof(PublishedRouteCursor) + (3 * max_depth * sizeof(Frame)); + } + + [[nodiscard]] uint64_t skipped_subtrees() const noexcept + { + return selection_.empty() && root_ != kInvalidPageId ? 1 : skipped_; + } + + [[nodiscard]] const std::shared_ptr<const PageState> &leaf() const + { + return leaf_; + } + + [[nodiscard]] uint64_t leaf_page_id() const + { + return leaf_id_; + } + + [[nodiscard]] std::shared_ptr<const BackingRef> needs_inner() const + { + return pending_ ? pending_->page->inner_backing : nullptr; + } + + private: + struct Frame + { + std::shared_ptr<const PublishedPage> page; + std::shared_ptr<const InnerPageState> inner; + PageKeyRange bounds; + size_t child = 0; + }; + + struct Pending + { + uint64_t id; + std::shared_ptr<const PublishedPage> page; + PageKeyRange bounds; + std::optional<OwnedKey> key; + bool exclusive; + }; + + Status descend(uint64_t id, PageKeyRange bounds, std::optional<Slice> key, bool exclusive, + std::optional<uint32_t> expected_level, std::shared_ptr<const InnerPageState> loaded = {}); + [[nodiscard]] Status validate_index(const InnerPageState &index, const PageKeyRange &bounds) const; + [[nodiscard]] PageKeyRange child_range(const Frame &frame) const; + uint64_t root_; + MappingView mapping_; + size_t max_depth_; + bool reverse_; + PageKeyRange selection_; + uint64_t skipped_ = 0; + std::vector<Frame, detail::ReservationAllocator<Frame>> stack_; + std::shared_ptr<const PageState> leaf_; + uint64_t leaf_id_ = kInvalidPageId; + std::optional<Pending> pending_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_scan.cpp b/lib/crowdb-tree/src/btree/published_scan.cpp new file mode 100644 index 000000000..13e3980ab --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_scan.cpp @@ -0,0 +1,275 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/published_scan_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +using namespace published_read_detail; + +PublishedScan::State::State(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<PageLoader> transport, + PageKeyRange selection, bool descending, PublishedReadLimits bounds, + PublishedScanOptions selection_options) + : ReadPump(std::move(captured), std::move(transport), bounds), + range(std::move(selection)), + reverse(descending), + options(std::move(selection_options)), + report{.status = Status::Ok(), + .entries = detail::ReservedVector<PublishedScanEntry>( + detail::ReservationAllocator<PublishedScanEntry>(bounds.metadata))}, + sources(view->sources->tables.size(), Source{}, detail::ReservationAllocator<Source>(bounds.metadata)), + walker(view->root_page_id, view->mapping, bounds.max_route_depth, descending, bounds.metadata, range) +{ + remaining_bytes = options.byte_budget.value_or(limits.max_result_bytes); +} + +Status &PublishedScan::State::status() +{ + return report.status; +} + +void PublishedScan::State::begin() +{ + report.entries.clear(); + report.work = 0; + report.done = finished || range.empty(); + report.byte_limit_reached = false; + result_bytes = 0; + if (!options.byte_budget) { + remaining_bytes = limits.max_result_bytes; + } +} + +bool PublishedScan::State::prepare_sources() +{ + for (size_t i = 0; i < sources.size(); ++i) { + if (!fill_memtable(i)) { + return false; + } + } + return fill_leaf(); +} + +const PublishedScanEntry *PublishedScan::State::winner() const +{ + const PublishedScanEntry *best = nullptr; + auto consider = [&](const std::optional<PublishedScanEntry> &entry) { + if (!entry) { + return; + } + if (!best) { + best = &*entry; + return; + } + const int order = Slice(entry->key).compare(best->key); + if ((reverse ? order > 0 : order < 0) || (order == 0 && entry->cell.slot > best->cell.slot)) { + best = &*entry; + } + }; + for (const auto &source : sources) { + consider(source.candidate); + } + consider(leaf_candidate); + return best; +} + +void PublishedScan::State::consume(Slice key) +{ + for (auto &source : sources) { + if (source.candidate && Slice(source.candidate->key) == key) { + source.candidate.reset(); + } + } + if (leaf_candidate && Slice(leaf_candidate->key) == key) { + leaf_candidate.reset(); + } +} + +void PublishedScan::State::emit(PublishedScanEntry entry) +{ + if (options.keys_only || entry.cell.tombstone()) { + entry.cell.value = {}; + } + const size_t value_bytes = entry.cell.value.size(); + const auto bytes = entry.key.size() + value_bytes; + result_bytes += bytes; + remaining_bytes -= std::min(remaining_bytes, bytes); + emitted = true; + consume(Slice(entry.key)); + report.entries.push_back(std::move(entry)); + ++report.work; +} + +bool PublishedScan::State::advance() +{ + if (report.done) { + return true; + } + if (pending_entry && !loading_value) { + auto entry = std::move(*pending_entry); + pending_entry.reset(); + emit(std::move(entry)); + } + while (report.work < limits.max_work) { + if (!prepare_sources()) { + return !report.status.ok() || report.work >= limits.max_work; + } + if (report.work >= limits.max_work) { + return true; + } + const auto *selected = winner(); + if (selected == nullptr) { + finished = true; + report.done = true; + return true; + } + const int start_order = options.start ? Slice(selected->key).compare(*options.start) : 0; + const bool before_start = options.start && ((reverse ? start_order > 0 : start_order < 0) || + (start_order == 0 && !options.start_inclusive)); + if (before_start || (selected->cell.tombstone() && !options.include_tombstones)) { + const OwnedKey key = selected->key; + consume(Slice(key)); + ++report.work; + continue; + } + uint64_t value_bytes = 0; + if (!options.keys_only && !selected->cell.tombstone()) { + value_bytes = selected->cell.overflow() && selected->cell.encoded.size() >= kOverflowCellSize + ? CellView(selected->cell.encoded).overflow_len() + : selected->cell.value.size(); + } + // Test the public budget before loading overflow. A work window may + // end earlier without exhausting that budget or consuming this key. + const bool first = options.byte_budget ? !emitted : report.entries.empty(); + if ((selected->key.size() > remaining_bytes || value_bytes > remaining_bytes - selected->key.size()) && + !(first && options.allow_oversized_first)) { + report.byte_limit_reached = true; + return true; + } + if (report.entries.size() >= limits.max_entries) { + return true; + } + const auto window_left = limits.max_result_bytes - std::min(result_bytes, limits.max_result_bytes); + if (!report.entries.empty() && + (selected->key.size() > window_left || value_bytes > window_left - selected->key.size())) { + return true; + } + if (selected->cell.overflow() && !options.keys_only && !selected->cell.tombstone()) { + pending_entry = *selected; + loading_value = true; + load_value(selected->cell); + return false; + } + emit(*selected); + } + return true; +} + +void PublishedScan::State::loaded(const LoadEvent &event) +{ + if (loading_index) { + loading_index = false; + report.status = walker.resume_inner(event.inner); + leaf = walker.leaf(); + return; + } + if (loading_value) { + if (!event.value || !pending_entry || pending_entry->cell.encoded.size() < kOverflowCellSize || + event.value->size() != CellView(pending_entry->cell.encoded).overflow_len()) { + report.status = Status::corruption("overflow loader returned no value owner"); + return; + } + materialize_value(&pending_entry->cell, event.value); + loading_value = false; + return; + } + if (!event.leaf) { + report.status = Status::corruption("leaf loader returned no frame owner"); + return; + } + auto resident = detail::reserved_shared<PageState>(limits.metadata, *leaf, limits.metadata); + resident->bases[loading_base].resident = event.leaf; + leaf = std::move(resident); +} + +std::optional<size_t> PublishedScan::allocation_bound(const PublishedReadLimits &limits, size_t workers) noexcept +{ + using detail::AllocationBound; + using detail::shared_allocation_bound; + const auto keys = AllocationBound().add_product(2, limits.max_memtables).add(limits.max_entries).add(6).value(); + const auto key = AllocationBound(OwnedKey::allocation_overhead()).add(limits.max_key_bytes).add(1).value(); + if (!keys || !key) { + return {}; + } + return AllocationBound(shared_allocation_bound<State>() + shared_allocation_bound<PublishedScan>()) + .add(read_metadata_allocation_bound(limits, workers)) + .add(LogicalPageCursor::allocation_bound(limits.max_page_sources)) + .add_product(limits.max_memtables, sizeof(State::Source)) + .add_product(limits.max_entries, 3 * sizeof(PublishedScanEntry)) + .add_product(*keys, *key) + .value(); +} + +Status PublishedScan::create(std::shared_ptr<const PublishedTreeState> view, std::shared_ptr<PageLoader> loader, + PageKeyRange range, bool reverse, PublishedReadLimits limits, + std::shared_ptr<PublishedScan> *out, PublishedScanOptions options) +{ + auto status = validate_read(view, limits); + if (!status.ok()) { + return status; + } + if (out == nullptr || (range.lower && range.lower->size() > limits.max_key_bytes) || + (range.upper && range.upper->size() > limits.max_key_bytes) || + (options.start && options.start->size() > limits.max_key_bytes)) { + return Status::invalid_argument("scan bounds exceed key budget or output is absent"); + } + try { + if (limits.metadata) { + for (auto *bound : {&range.lower, &range.upper}) { + if (*bound) { + *bound = OwnedKey(Slice(**bound), limits.metadata); + } + } + if (options.start) { + options.start = OwnedKey(Slice(*options.start), limits.metadata); + } + } + auto state = detail::reserved_shared<State>(limits.metadata, std::move(view), std::move(loader), + std::move(range), reverse, limits, std::move(options)); + *out = detail::reserved_construct<PublishedScan>(limits.metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) PublishedScan(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted("scan allocation failed"); + } + return Status::Ok(); +} + +detail::CallbackSender PublishedScan::next_sender() noexcept +{ + return {this, &PublishedScan::submit}; +} + +void PublishedScan::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<PublishedScan *>(source)->state_->start(complete, context); +} + +const PublishedScanResult &PublishedScan::result() const noexcept +{ + return state_->report; +} + +uint64_t PublishedScan::skipped_subtrees() const noexcept +{ + return state_->walker.skipped_subtrees(); +} + +void PublishedScan::request_stop() noexcept +{ + state_->stop(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_scan_internal.h b/lib/crowdb-tree/src/btree/published_scan_internal.h new file mode 100644 index 000000000..38f24627d --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_scan_internal.h @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/published_read_internal.h" + +namespace crowdb::tree +{ +struct PublishedScan::State : published_read_detail::ReadPump +{ + struct Source + { + std::optional<OwnedKey> after; + std::optional<PublishedScanEntry> candidate; + bool done = false; + }; + + State(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<PageLoader> transport, + PageKeyRange selection, bool descending, PublishedReadLimits bounds, PublishedScanOptions options); + Status &status() override; + void begin() override; + bool advance() override; + void loaded(const published_read_detail::LoadEvent &event) override; + bool fill_memtable(size_t index); + bool fill_leaf(); + bool prepare_sources(); + [[nodiscard]] const PublishedScanEntry *winner() const; + void consume(Slice key); + void emit(PublishedScanEntry entry); + + PageKeyRange range; + bool reverse; + PublishedScanOptions options; + PublishedScanResult report; + detail::ReservedVector<Source> sources; + PublishedRouteCursor walker; + bool walked = false; + bool leaf_done = false; + bool finished = false; + size_t result_bytes = 0; + size_t remaining_bytes = 0; + bool emitted = false; + std::shared_ptr<const PageState> leaf; + detail::ReservedUnique<LogicalPageCursor> cursor; + std::optional<PublishedScanEntry> leaf_candidate; + std::optional<PublishedScanEntry> pending_entry; + size_t loading_base = 0; + bool loading_value = false; + bool loading_index = false; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_scan_sources.cpp b/lib/crowdb-tree/src/btree/published_scan_sources.cpp new file mode 100644 index 000000000..3c704e10e --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_scan_sources.cpp @@ -0,0 +1,149 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_scan_internal.h" + +namespace crowdb::tree +{ +using namespace published_read_detail; + +bool PublishedScan::State::fill_memtable(size_t index) +{ + auto &source = sources[index]; + const auto &table = view->sources->tables[index]; + while (!source.done && !source.candidate) { + if (report.work >= limits.max_work) { + return false; + } + ++report.work; + OwnedKey key; + std::shared_ptr<const CellVersion> owner; + { + auto guard = table->read_guard(limits.metadata); + ConcurrentSkipList::Cursor position; + if (reverse) { + auto bound = source.after ? std::optional<Slice>(Slice(*source.after)) : range.upper_slice(); + bool inclusive = false; + if (!source.after && options.start && (!bound || Slice(*options.start) < *bound)) { + bound = Slice(*options.start); + inclusive = options.start_inclusive; + } + position = table->cursor_reverse(bound.value_or(Slice()), bound.has_value(), inclusive); + } + else if (source.after) { + position = table->cursor_from(Slice(*source.after), false); + } + else { + auto bound = range.lower_slice(); + bool inclusive = true; + if (options.start && (!bound || Slice(*options.start) >= *bound)) { + bound = Slice(*options.start); + inclusive = options.start_inclusive; + } + position = bound ? table->cursor_from(*bound, inclusive) : table->cursor({}); + } + if (!position.valid() || !range.contains(position.key())) { + source.done = true; + break; + } + if (position.key().size() > limits.max_key_bytes) { + report.status = Status::resource_exhausted("scan MemTable key exceeds the owned-key budget"); + return false; + } + key = OwnedKey(position.key(), limits.metadata); + owner = position.cell_owner_at(view->sources->frontier(index)); + } + source.after = key; + if (owner && owner->slot > view->sources->floor) { + source.candidate = PublishedScanEntry{.key = std::move(key), .cell = owned_cell(std::move(owner))}; + } + } + return true; +} + +bool PublishedScan::State::fill_leaf() +{ + if (leaf_candidate || leaf_done) { + return true; + } + if (!walked) { + if (report.work >= limits.max_work) { + return false; + } + ++report.work; + auto bound = reverse ? range.upper_slice() : range.lower_slice(); + bool exclusive = reverse; + if (options.start && (!bound || (reverse ? Slice(*options.start) < *bound : Slice(*options.start) >= *bound))) { + bound = Slice(*options.start); + exclusive = !options.start_inclusive; + } + report.status = walker.seek(bound, exclusive); + walked = true; + if (!report.status.ok()) { + return false; + } + leaf = walker.leaf(); + } + while (report.work < limits.max_work) { + if (auto backing = walker.needs_inner()) { + loading_index = true; + load_inner(std::move(backing)); + return false; + } + if (!leaf) { + leaf_done = true; + return true; + } + if (!range.overlaps(leaf->range)) { + const bool past = reverse ? range.lower && leaf->range.upper && *leaf->range.upper <= *range.lower + : range.upper && leaf->range.lower && *leaf->range.lower >= *range.upper; + if (past) { + leaf_done = true; + return true; + } + } + else if (!cursor) { + std::optional<LogicalCursorStart> start; + if (options.start) { + start = LogicalCursorStart{.key = *options.start, .inclusive = options.start_inclusive}; + } + cursor = detail::reserved_unique<LogicalPageCursor>(limits.metadata, leaf, range, UINT64_MAX, + limits.max_page_sources, reverse, limits.metadata, + std::move(start)); + if (!cursor->status().ok()) { + report.status = cursor->status(); + return false; + } + } + if (cursor && !cursor->status().ok()) { + report.status = cursor->status(); + return false; + } + const auto needed = cursor ? cursor->needs_base() : std::nullopt; + if (needed) { + loading_base = *needed; + cursor.reset(); + load_base(leaf->bases[loading_base]); + return false; + } + if (cursor && cursor->valid()) { + if (cursor->key().size() > limits.max_key_bytes) { + report.status = Status::resource_exhausted("scan leaf key exceeds the owned-key budget"); + return false; + } + leaf_candidate = + PublishedScanEntry{.key = OwnedKey(cursor->key(), limits.metadata), .cell = cursor->cell()}; + cursor->next(); + ++report.work; + return true; + } + cursor.reset(); + report.status = walker.next(); + ++report.work; + if (!report.status.ok()) { + return false; + } + leaf = walker.leaf(); + } + return false; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_state.cpp b/lib/crowdb-tree/src/btree/published_state.cpp new file mode 100644 index 000000000..1691bef41 --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_state.cpp @@ -0,0 +1,259 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/log.h" +#include "page_cache.h" +#include "publication_generation_internal.h" +#include "publication_lane.h" + +#include <algorithm> +#include <cassert> +#include <stdexcept> + +namespace crowdb::tree +{ +void PublishedTreeState::clear_merge_candidates(const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + detail::ReservedVector<PageReplacement> changes{detail::ReservationAllocator<PageReplacement>(metadata)}; + changes.reserve(mapping.copy_bounds().merge_candidate_pages); + + struct Context + { + detail::ReservedVector<PageReplacement> &changes; + const std::shared_ptr<detail::AllocationReservation> &metadata; + } context{.changes = changes, .metadata = metadata}; + + mapping.visit(&context, [](void *argument, uint64_t pid, const auto &page) { + if (!page->metrics.merge_candidate) { + return; + } + auto &self = *static_cast<Context *>(argument); + auto replacement = detail::reserved_shared<PublishedPage>(self.metadata, *page); + replacement->metrics.merge_candidate = false; + replacement->merge_outgoing.reset(); + self.changes.push_back({.page_id = pid, .expected = page, .replacement = std::move(replacement)}); + }); + MappingView::CopyStats copied; + mapping = mapping.replace(changes, copied, metadata); + stats.merge_candidate_pages = 0; + stats.merge_candidate_pairs = 0; + stats.merge_ready_pairs = 0; + merge_candidate_since_ns = 0; +} + +namespace +{ +TreeOutcome publication_outcome(const PublicationResult &result) noexcept +{ + if (result.already_committed) { + return TreeOutcome::Retry; + } + if (result.status.ok()) { + return TreeOutcome::Success; + } + if (result.status.code() == Code::kUnavailable) { + return TreeOutcome::Stale; + } + return TreeOutcome::Error; +} +} // namespace + +PublicationCoordinator::Root::Root(std::shared_ptr<const PublishedTreeState> value, + std::shared_ptr<detail::AllocationReservation> reservation) + : state(std::move(value)), + metadata(std::move(reservation)) +{ + destroy = [](EpochManager::Deferred *entry) noexcept { dispose(static_cast<Root *>(entry)); }; +} + +PublicationCoordinator::Root * +PublicationCoordinator::Root::create(std::shared_ptr<const PublishedTreeState> value, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + detail::ReservationAllocator<Root> allocator(metadata); + auto *root = allocator.allocate(1); + try { + std::construct_at(root, std::move(value), metadata); + } + catch (...) { + allocator.deallocate(root, 1); + throw; + } + return root; +} + +void PublicationCoordinator::Root::dispose(Root *root) noexcept +{ + detail::ReservationAllocator<Root> allocator(root->metadata); + std::destroy_at(root); + allocator.deallocate(root, 1); +} + +PublicationCoordinator::~PublicationCoordinator() +{ + close(); + assert(idle()); + if (page_cache_) { + page_cache_->residency()->clear(); + } + auto *root = root_.load(); + root->state->retirement.mark(); + Root::dispose(root); +} + +std::shared_ptr<const PublishedTreeState> PublicationCoordinator::capture() const +{ + auto guard = epoch_.enter(); + return root_.load(std::memory_order_acquire)->state; +} + +TreeStatsSnapshot PublicationCoordinator::stats(const TreeResourceMetrics &resources) const +{ + const auto state = capture(); + const auto now = tree_metrics_now_ns(); + const auto age = [now](uint64_t since) { return since != 0 && now >= since ? now - since : 0; }; + auto candidate_since = state->mapping.copy_bounds().candidate_since_ns; + if (state->merge_candidate_since_ns != 0 && + (candidate_since == 0 || state->merge_candidate_since_ns < candidate_since)) { + candidate_since = state->merge_candidate_since_ns; + } + return { + .instance_generation = state->generation, + .commit_seq = state->commit_seq, + .sampled_at_ns = now, + .logical = state->stats, + .resources = resources.snapshot(), + .progress = {.maintenance_oldest_age_ns = age(oldest_maintenance_since_ns()), + .candidate_oldest_age_ns = age(candidate_since), + .checkpoint_oldest_pending_age_ns = age(state->checkpoint_pending_since_ns)} + }; +} + +void PublicationCoordinator::request_gc_floor(uint64_t floor) noexcept +{ + auto current = requested_gc_floor_.load(std::memory_order_relaxed); + while (floor > current && !requested_gc_floor_.compare_exchange_weak(current, floor, std::memory_order_release, + std::memory_order_relaxed)) { + } +} + +uint64_t PublicationCoordinator::requested_gc_floor() const noexcept +{ + return requested_gc_floor_.load(std::memory_order_acquire); +} + +void PublicationCoordinator::close() noexcept +{ + closed_.store(true, std::memory_order_release); +} + +bool PublicationCoordinator::idle() const noexcept +{ + return queued_.load(std::memory_order_acquire) == 0 && !running_.load(std::memory_order_acquire); +} + +size_t PublicationCoordinator::reclaim() +{ + return epoch_.try_reclaim(); +} + +bool PublicationCoordinator::try_submit(const std::shared_ptr<PublicationRequest> &request) +{ + if (!request || closed_.load(std::memory_order_acquire) || request->submitted_.exchange(true)) { + return false; + } + auto queued = ordinary_queued_.load(std::memory_order_relaxed); + do { + if (queued >= capacity_) { + request->submitted_.store(false, std::memory_order_release); + return false; + } + } while (!ordinary_queued_.compare_exchange_weak(queued, queued + 1, std::memory_order_acq_rel)); + enqueue(request); + return true; +} + +void PublicationCoordinator::enqueue(const std::shared_ptr<PublicationRequest> &request) +{ + queued_.fetch_add(1, std::memory_order_acq_rel); + request->queued_owner_ = request; + request->queue_metrics_.emplace(work_metrics_, TreeStage::PublishQueueWait, tree_metrics_now_ns()); + auto *head = incoming_.load(std::memory_order_relaxed); + do { + request->next_ = head; + } while ( + !incoming_.compare_exchange_weak(head, request.get(), std::memory_order_release, std::memory_order_relaxed)); + if (!running_.exchange(true, std::memory_order_acq_rel)) { + drain(); + } +} + +void PublicationCoordinator::complete(PublicationRequest *request) noexcept +{ + auto owner = std::move(request->queued_owner_); + const auto started_at = tree_metrics_now_ns(); + if (request->queue_metrics_) { + (void)request->queue_metrics_->finish(TreeOutcome::Success, started_at); + } + TreeMetricsOperation commit_metrics(work_metrics_, TreeStage::PublishCommit, started_at); + PublicationResult result; + try { + result = request->reconciliation ? reconcile(*request) : commit(request->transaction); + } + catch (const std::bad_alloc &) { + result.status = Status::resource_exhausted(); + } + catch (const std::exception &) { + result.status = Status::internal_error(); + } + (void)commit_metrics.finish(publication_outcome(result), tree_metrics_now_ns()); + queued_.fetch_sub(1, std::memory_order_release); + if (request->completion_slot_) { + checkpoint_completion_.store(false, std::memory_order_release); + } + else if (request->lane_) { + request->lane_->queued_.fetch_sub(1, std::memory_order_release); + } + else { + ordinary_queued_.fetch_sub(1, std::memory_order_release); + } + // One request is submitted once. A retry uses a new request, the same + // preallocated install record, and the same immutable replacements. + if (request->on_done) { + try { + request->on_done(std::move(result)); + } + catch (...) { + CRB_LOG_ERROR("publication receiver threw; remaining accepted requests " + "will continue draining"); + } + } +} + +void PublicationCoordinator::drain() noexcept +{ + for (;;) { + auto *list = incoming_.exchange(nullptr, std::memory_order_acquire); + PublicationRequest *ordered = nullptr; + while (list != nullptr) { + auto *next = list->next_; + list->next_ = ordered; + ordered = list; + list = next; + } + while (ordered != nullptr) { + auto *next = ordered->next_; + complete(ordered); + ordered = next; + } + // The acquire half observes producers that saw the busy gate before + // release. A plain store permits a StoreLoad reorder that can miss an + // already queued request while its producer still sees running=true. + (void)running_.exchange(false, std::memory_order_acq_rel); + if (incoming_.load(std::memory_order_acquire) == nullptr || + running_.exchange(true, std::memory_order_acq_rel)) { + return; + } + } +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/published_state.h b/lib/crowdb-tree/src/btree/published_state.h new file mode 100644 index 000000000..b79489b7c --- /dev/null +++ b/lib/crowdb-tree/src/btree/published_state.h @@ -0,0 +1,635 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "budget_memory_resource.h" +#include "crowdb-tree/epoch.h" +#include "crowdb-tree/memtable/memtable.h" +#include "crowdb-tree/status.h" +#include "inline_function.h" +#include "merge_pair_metrics.h" +#include "owned_key.h" +#include "snapshot_changes.h" +#include "tree_metrics.h" + +#include <array> +#include <atomic> +#include <functional> +#include <memory> +#include <optional> +#include <span> +#include <vector> + +namespace crowdb::tree +{ +struct PageState; +struct BackingRef; +struct PrimaryBackingMetadata; +struct SnapshotBaseline; +class SharedPageCache; +class TreeStateReconciliation; + +struct InnerFrameProof +{ + uint64_t self_page_id = UINT64_MAX; + uint32_t frame_bytes = 0; + bool has_fences = false; + bool page_id_fences = false; + OwnedKey lower; + OwnedKey upper; + uint64_t lower_leaf = UINT64_MAX; + uint64_t upper_leaf = UINT64_MAX; +}; + +struct InnerPageState +{ + + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !separators.empty(); ++i) { + separators.pop_back(); + } + return separators.empty(); + } + + using KeyAllocator = detail::ReservationAllocator<OwnedKey>; + using ChildAllocator = detail::ReservationAllocator<uint64_t>; + using Keys = std::vector<OwnedKey, KeyAllocator>; + using Children = std::vector<uint64_t, ChildAllocator>; + + // Element bytes are counted separately; copies/inserts can overlap two + // capacities for each owned array. + [[nodiscard]] static size_t allocation_bound() noexcept + { + return detail::shared_allocation_bound<InnerPageState>(); + } + + explicit InnerPageState(const std::shared_ptr<detail::AllocationReservation> &metadata = {}) + : separators(KeyAllocator(metadata)), + children(ChildAllocator(metadata)) + { + } + + InnerPageState(const InnerPageState &source, const std::shared_ptr<detail::AllocationReservation> &metadata = {}) + : separators(source.separators.begin(), source.separators.end(), KeyAllocator(metadata)), + children(source.children.begin(), source.children.end(), ChildAllocator(metadata)) + { + } + + // separators[i] is the inclusive lower bound of children[i + 1]. + Keys separators; + Children children; + // Only a decoded immutable frame carries this proof. Copying routing + // descriptors for a new publication deliberately drops its old fences. + std::optional<InnerFrameProof> frame_proof = std::nullopt; +}; + +struct PublishedPage +{ + bool retire_step() noexcept + { + return true; + } + + uint64_t revision = 0; + uint32_t level = 0; + std::shared_ptr<const PageState> leaf = nullptr; + std::shared_ptr<const InnerPageState> inner = nullptr; + PageMetricsContribution metrics; + // Keeps the admitted metadata charge until the last page/view releases it. + std::shared_ptr<const void> allocation_owner = nullptr; + // A cold index retains a protected version; loading it does not mutate + // the fixed mapping or replace a newer route. + std::shared_ptr<const BackingRef> inner_backing = nullptr; + std::shared_ptr<const PrimaryBackingMetadata> primary = nullptr; + // Immutable classification lets workers skip unchanged pairs without + // accessing the publication owner's private ledger. + std::optional<MergePairProof> merge_outgoing = std::nullopt; +}; + +struct PageReplacement +{ + uint64_t page_id = 0; + std::shared_ptr<const PublishedPage> expected = nullptr; + std::shared_ptr<const PublishedPage> replacement = nullptr; +}; + +// Persistent radix directory: a transaction copies each touched path/segment +// once. Payloads and untouched directory branches are shared with old readers. +class MappingView +{ + struct Segment; + struct Directory; + + public: + static constexpr uint64_t kFanout = 64; + static constexpr uint64_t kPageLimit = uint64_t{1} << 30; + + struct CopyStats + { + uint64_t directories = 0; + uint64_t segments = 0; + uint64_t bytes = 0; + }; + + struct CopyBounds + { + size_t leaf_pages = 0; + size_t merge_candidate_pages = 0; + size_t leaf_bytes = 0; + size_t max_leaf_bytes = 0; + size_t max_leaf_append_bytes = 0; + size_t max_leaf_bases = 0; + bool may_have_overflow_values = false; + size_t max_inner_bytes = 0; + uint32_t max_level = 0; + uint64_t candidate_since_ns = 0; + uint64_t writeback_since_ns = 0; + uint64_t oldest_writeback_pid = kPageLimit; + uint64_t writeback_page_since_ns = 0; + uint64_t oldest_writeback_page_pid = kPageLimit; + }; + + // Maintained on the same copied radix paths, without scanning cold pages. + // Includes descriptor arrays and temporary overlay growth, not payloads. + [[nodiscard]] CopyBounds copy_bounds() const noexcept; + [[nodiscard]] static size_t path_copy_bound() noexcept; + + // Incremental minimum over dirty leaf candidates; no traversal or I/O. + [[nodiscard]] std::optional<uint64_t> oldest_writeback_leaf() const noexcept; + [[nodiscard]] std::optional<uint64_t> oldest_writeback_page() const noexcept; + [[nodiscard]] std::shared_ptr<const PublishedPage> get(uint64_t page_id) const; + // Startup/snapshot traversal visits allocated radix nodes only. Runtime + // updates and collectors use incremental summaries instead. + using Visitor = void (*)(void *, uint64_t, const std::shared_ptr<const PublishedPage> &); + void visit(void *context, Visitor visitor) const; + // Explicit diagnostic traversal. Each unit inspects one radix slot and + // at most five directory levels; missing subtrees are skipped as a unit. + // Resume with the same immutable mapping and returned PID. Returns true + // at the end, including empty mappings. Does not load cold pages. + [[nodiscard]] bool visit_from(uint64_t &next_pid, size_t max_slots, void *context, Visitor visitor) const; + + [[nodiscard]] bool empty() const noexcept + { + return !root_; + } + + [[nodiscard]] bool same_root(const MappingView &other) const noexcept + { + return root_ == other.root_; + } + + // Input must be strictly increasing by PID and within kPageLimit. + [[nodiscard]] MappingView replace(std::span<const PageReplacement> replacements, CopyStats &copied, + const std::shared_ptr<detail::AllocationReservation> &metadata = {}) const; + + private: + struct Segment + { + + bool retire_step() noexcept + { + return true; + } + + std::array<std::shared_ptr<const PublishedPage>, kFanout> pages; + std::array<uint64_t, kFanout> candidate_since_ns{}; + std::array<uint64_t, kFanout> writeback_since_ns{}; + CopyBounds bounds; + }; + + struct Directory + { + + bool retire_step() noexcept + { + return true; + } + + std::array<std::shared_ptr<const Directory>, kFanout> children; + std::array<std::shared_ptr<const Segment>, kFanout> segments; + CopyBounds bounds; + }; + + static void visit_directory(const Directory &directory, unsigned shift, uint64_t prefix, void *context, + Visitor visitor); + static std::shared_ptr<const Directory> + replace_directory(const std::shared_ptr<const Directory> &old, unsigned shift, + std::span<const PageReplacement> replacements, CopyStats &copied, + const std::shared_ptr<detail::AllocationReservation> &metadata, uint64_t now_ns); + std::shared_ptr<const Directory> root_ = nullptr; +}; + +struct PublishedSources +{ + + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !tables.empty(); ++i) { + tables.pop_back(); + } + return tables.empty(); + } + + struct Inheritance + { + uint64_t owner_identity = 0; + uint64_t session = 0; + uint64_t frontier = 0; + size_t local_count = 0; + SourceMemoryStats memory{}; + bool operator==(const Inheritance &) const = default; + }; + + using TableAllocator = detail::ReservationAllocator<std::shared_ptr<MemTable>>; + using Tables = std::vector<std::shared_ptr<MemTable>, TableAllocator>; + + [[nodiscard]] static size_t allocation_bound() noexcept + { + return detail::shared_allocation_bound<PublishedSources>(); + } + + explicit PublishedSources(std::shared_ptr<detail::AllocationReservation> metadata = {}) + : tables(TableAllocator(std::move(metadata))) + { + } + + std::shared_ptr<const SourceMemoryLedger> memory = nullptr; + Tables tables; + uint64_t floor = 0; + // The suffix consists only of independently retained, closed parent + // tables. Parent and child slots above the cutover are separate namespaces. + std::optional<Inheritance> inheritance = std::nullopt; + + [[nodiscard]] SourceMemoryStats source_stats() const noexcept + { + auto result = memory ? memory->snapshot() : SourceMemoryStats{}; + if (inheritance) { + result += inheritance->memory; + } + return result; + } + + [[nodiscard]] uint64_t frontier(size_t index) const noexcept + { + return inheritance && index >= inheritance->local_count ? inheritance->frontier : UINT64_MAX; + } +}; + +struct PendingHandoff +{ + uint64_t batch = 0; + uint64_t frontier = 0; + bool operator==(const PendingHandoff &) const = default; +}; + +struct PublishedTreeState +{ + + void clear_merge_candidates(const std::shared_ptr<detail::AllocationReservation> &metadata); + detail::RetirementObserver retirement; + static constexpr bool observe_retirement = true; + + bool retire_step() noexcept + { + return true; + } + + explicit PublishedTreeState(std::shared_ptr<detail::AllocationReservation> metadata = {}, + uint64_t source_floor = 0); + + MappingView mapping; + SnapshotChanges snapshot_changes; + SnapshotChanges snapshot_changes_since_capture; + std::shared_ptr<const SnapshotBaseline> snapshot_baseline; + std::shared_ptr<const PublishedSources> sources = nullptr; + uint64_t root_page_id = UINT64_MAX; + uint64_t next_page_id = 0; + uint64_t generation = 1; + uint64_t commit_seq = 0; + // Per-generation L1 layout and handoff ownership fence. Publishing only + // a successor L0 catalog leaves a checkpoint's layout proof valid. + uint64_t l1_seq = 0; + uint64_t coverage = 0; + uint64_t gc_floor = 0; + uint64_t checkpoint_pending_since_ns = 0; + uint64_t merge_candidate_since_ns = 0; + uint64_t checkpoint_token = 0; + uint64_t checkpoint_capture_coverage = 0; + uint64_t checkpoint_capture_floor = 0; + bool checkpoint_inflight = false; + // L1 contains part of this batch above coverage. Checkpoint capture and + // unrelated structure changes wait until its coverage commit succeeds. + std::optional<PendingHandoff> pending_handoff = std::nullopt; + // Held before source closure while the batch owns its reserved bounds. + // Readers continue using the immutable state; unrelated writers wait. + uint64_t flush_admission_owner = 0; + TreeLogicalStats stats; + // Exact encoded graph summary, installed only after a bounded validation pass. + std::optional<TreeSummary> summary = std::nullopt; +}; + +struct InstallToken +{ + uint64_t generation = 1; + uint64_t batch = 0; + uint64_t range = 0; + uint64_t sequence = 0; + bool operator==(const InstallToken &) const = default; +}; + +// The batch creates one record per token before dispatch and reuses it on +// every retry. It retains the record until all children and late callbacks end. +struct InstallRecord +{ + explicit InstallRecord(InstallToken value) : token(value) + { + } + + const InstallToken token; + std::atomic<bool> committed{false}; + uint64_t commit_seq = 0; + uint64_t l1_seq = 0; + // Only capture records retain this artifact. Its sources are empty so + // checkpoint lifetime does not retain a detached L0 catalog. + std::shared_ptr<const PublishedTreeState> checkpoint_view = nullptr; + + [[nodiscard]] uint64_t publisher_id() const noexcept + { + return publisher_id_.load(std::memory_order_acquire); + } + + private: + friend class PublicationCoordinator; + std::atomic<uint64_t> publisher_id_{0}; +}; + +enum class CheckpointAction : uint8_t { Capture, Commit, Abort }; + +struct CheckpointPublication +{ + CheckpointAction action = CheckpointAction::Capture; + uint64_t token = 0; + std::shared_ptr<const SnapshotBaseline> baseline = nullptr; +}; + +enum class FlushAdmissionAction : uint8_t { Acquire, Release }; + +struct FlushAdmissionPublication +{ + FlushAdmissionAction action = FlushAdmissionAction::Acquire; + uint64_t owner = 0; +}; + +struct MergePairPublication +{ + uint64_t left = 0; + std::shared_ptr<const PublishedPage> expected_left = nullptr; + std::shared_ptr<const PublishedPage> expected_right = nullptr; + std::shared_ptr<const PublishedPage> expected_parent = nullptr; + std::optional<MergePairProof> candidate = std::nullopt; +}; + +struct PublicationTransaction +{ + using Pages = std::vector<PageReplacement, detail::ReservationAllocator<PageReplacement>>; + using Records = + std::vector<std::shared_ptr<InstallRecord>, detail::ReservationAllocator<std::shared_ptr<InstallRecord>>>; + std::shared_ptr<InstallRecord> record = nullptr; + Pages pages{detail::ReservationAllocator<PageReplacement>{}}; + std::optional<uint64_t> root_page_id = std::nullopt; + std::optional<uint64_t> expected_root_page_id = std::nullopt; + std::optional<uint64_t> expected_coverage = std::nullopt; + std::optional<uint64_t> coverage = std::nullopt; + std::shared_ptr<const PublishedSources> expected_sources = nullptr; + std::shared_ptr<const PublishedSources> sources = nullptr; + // Coverage publication waits for every range, including empty ranges. + Records required_installs{detail::ReservationAllocator<std::shared_ptr<InstallRecord>>{}}; + std::shared_ptr<detail::AllocationReservation> metadata = nullptr; + // Persistent maintenance selection uses independent prepaid Routing credit. + std::shared_ptr<detail::AllocationReservation> snapshot_metadata = nullptr; + SnapshotChangeEffect snapshot_effect = SnapshotChangeEffect::FullContent; + // Capture names an exact frontier or an asynchronous minimum frontier. + // Commit is submitted only after durable checkpoint commit succeeds. + // Checkpoint actions cannot be combined with page/catalog mutations. + std::optional<CheckpointPublication> checkpoint = std::nullopt; + // Range installs set this atomically with their pages. A coverage + // transaction with the same identity clears it after all required installs. + std::optional<PendingHandoff> handoff = std::nullopt; + std::optional<uint64_t> expected_commit_seq = std::nullopt; + std::optional<uint64_t> expected_l1_seq = std::nullopt; + std::optional<FlushAdmissionPublication> flush_admission = std::nullopt; + uint64_t flush_admission_owner = 0; + // Advances the shared PID authority without creating logical mapping + // entries. Capture may reserve an overflow range in the same transaction. + std::optional<uint64_t> expected_next_page_id = std::nullopt; + std::optional<uint64_t> next_page_id = std::nullopt; + // Classification changes only the candidate ledger. The logical mapping + // and checkpoint pending membership remain unchanged. + std::optional<MergePairPublication> merge_pair = std::nullopt; + // Standalone observation installation, fenced by expected_commit_seq. + std::optional<TreeSummary> summary = std::nullopt; +}; + +struct PublicationResult +{ + Status status; + uint64_t commit_seq = 0; + uint64_t l1_seq = 0; + bool already_committed = false; + MappingView::CopyStats copied{}; + std::shared_ptr<const PublishedTreeState> checkpoint_view = nullptr; +}; + +class PublicationLane; + +struct PublicationRequest +{ + explicit PublicationRequest(const std::shared_ptr<detail::AllocationReservation> &metadata = {}) + : transaction{.pages = PublicationTransaction::Pages(detail::ReservationAllocator<PageReplacement>(metadata)), + .required_installs = PublicationTransaction::Records( + detail::ReservationAllocator<std::shared_ptr<InstallRecord>>(metadata)), + .metadata = metadata} + { + } + + // Explicit bounded diagnostics use the same serial owner as publication. + // The fixed view must still be current; diagnostics never publish a root. + std::shared_ptr<TreeStateReconciliation> reconciliation = nullptr; + size_t reconciliation_slots = 64; + + PublicationTransaction transaction; + detail::InlineFunction<void(PublicationResult)> on_done; + + private: + friend class PublicationCoordinator; + std::atomic<bool> submitted_{false}; + PublicationRequest *next_ = nullptr; + std::shared_ptr<PublicationRequest> queued_owner_ = nullptr; + std::optional<TreeMetricsOperation> queue_metrics_ = std::nullopt; + bool completion_slot_ = false; + std::shared_ptr<PublicationLane> lane_ = nullptr; +}; + +struct SourceCatalog; + +struct PublicationBootstrap +{ + std::shared_ptr<detail::AllocationReservation> metadata = nullptr; + std::shared_ptr<const PublishedTreeState> initial = nullptr; + std::shared_ptr<const SourceCatalog> source_catalog = nullptr; +}; + +// Bounded MPSC ingress. The thread winning running_ drains short metadata +// commits; callers dispatch from the shared CPU runtime. Terminal callbacks run +// inline and must not block or wait for another publication. They may enqueue +// continuations; a callback exception cannot strand remaining requests. +class PublicationCoordinator +{ + struct CheckpointLedger; + PublicationResult reconcile(const PublicationRequest &request); + + public: + class Replacement; + explicit PublicationCoordinator(size_t queue_capacity = 64, uint64_t generation = 1, + std::shared_ptr<TreeWorkMetrics> metrics = {}, + std::pmr::memory_resource *checkpoint_resource = std::pmr::get_default_resource(), + std::shared_ptr<SharedPageCache> page_cache = {}, + PublicationBootstrap bootstrap = {}); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t source_count = 0, size_t max_threads = 64, + const MappingView *mapping = nullptr, + bool track_residency = false) noexcept; + ~PublicationCoordinator(); + PublicationCoordinator(const PublicationCoordinator &) = delete; + PublicationCoordinator &operator=(const PublicationCoordinator &) = delete; + + [[nodiscard]] std::shared_ptr<const PublishedTreeState> capture() const; + // Capture the last complete L1 mapping and coverage during a partial + // handoff. Snapshot consumers must ignore this header's live sources. + [[nodiscard]] std::shared_ptr<const PublishedTreeState> capture_complete() const; + [[nodiscard]] static size_t complete_view_reserve_bytes() noexcept; + // Logical values come from one captured root. Resources are a separate, + // approximate atomic sample; this path never loads or traverses pages. + [[nodiscard]] TreeStatsSnapshot stats(const TreeResourceMetrics &resources) const; + // Per-Tree bounded admission for independent maintenance factories. The + // slots remain active across CPU turns and publication waits, even when + // metrics collection is disabled. A full window defers unstarted work. + static constexpr size_t kMaintenanceSlots = 64; + [[nodiscard]] size_t start_maintenance(uint64_t started_at_ns) noexcept; + void finish_maintenance(size_t slot) noexcept; + [[nodiscard]] uint64_t oldest_maintenance_since_ns() const noexcept; + + [[nodiscard]] uint64_t maintenance_scan_pid() const noexcept + { + return maintenance_scan_pid_.load(std::memory_order_relaxed); + } + + void set_maintenance_scan_pid(uint64_t pid) noexcept + { + maintenance_scan_pid_.store(pid, std::memory_order_relaxed); + } + + [[nodiscard]] bool tracks_cache_residency() const noexcept + { + return page_cache_ != nullptr; + } + + [[nodiscard]] const std::shared_ptr<TreeWorkMetrics> &work_metrics() const noexcept + { + return work_metrics_; + } + + // false means no ownership was taken (full/closed/already submitted). + // The coordinator owner must outlive producers and terminal callbacks. + // close() rejects subsequent entry; producers already in try_submit() may + // finish accepting their request and must be joined before destruction. + bool try_submit(const std::shared_ptr<PublicationRequest> &request); + // A Tree owns one prepaid window for its single active handoff. Prepare + // before accepting proposals; the fixed lane cannot be enlarged later. + Status prepare_lane(size_t capacity, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<PublicationLane> *out); + bool try_submit(const std::shared_ptr<PublicationRequest> &request, const std::shared_ptr<PublicationLane> &lane); + // One dedicated slot finishes the already captured checkpoint even after + // ordinary ingress closes. It never consumes ordinary queue capacity. + bool try_submit_checkpoint_completion(const std::shared_ptr<PublicationRequest> &request); + // Apply the requested floor only at a later atomic publication, clamped + // to its completed coverage. Existing captured views remain unchanged. + void request_gc_floor(uint64_t floor) noexcept; + [[nodiscard]] uint64_t requested_gc_floor() const noexcept; + // Build all new owners while ordinary publications still run. Installation + // requires externally fenced ingress and drained accepted operations. + [[nodiscard]] static std::optional<size_t> replacement_allocation_bound(const MappingView &mapping, + size_t sources) noexcept; + Status prepare_replacement(PublicationBootstrap bootstrap, std::shared_ptr<Replacement> *out, + std::shared_ptr<const PublishedSources> inherited = {}) const; + void install_replacement(std::shared_ptr<Replacement> replacement) noexcept; + void close() noexcept; + [[nodiscard]] bool idle() const noexcept; + // Reclamation is an explicit maintenance operation, never part of capture. + size_t reclaim(); + // Admission cancellation reserves this separately from source/worker + // allocations so a failed capture can still release publication ownership. + [[nodiscard]] static size_t empty_commit_reserve_bytes() noexcept; + [[nodiscard]] static size_t snapshot_change_reserve_bytes() noexcept; + [[nodiscard]] static size_t merge_membership_reserve_bytes(size_t pages) noexcept; +#ifdef CROWDB_TREE_TEST_UTIL + void set_before_root_for_tests(std::function<void()> callback) + { + before_root_for_tests_ = std::move(callback); + } +#endif + + private: + struct Root : EpochManager::Deferred + { + std::shared_ptr<const PublishedTreeState> state = nullptr; + std::shared_ptr<detail::AllocationReservation> metadata = nullptr; + // Retired checkpoint sets use this ledger's allocator after a later + // generation has installed a different ledger. + std::shared_ptr<CheckpointLedger> checkpoint_allocator = nullptr; + + std::shared_ptr<const void> checkpoint_retired = nullptr; + std::shared_ptr<const void> merge_retired = nullptr; + // Source-free fallback only while this root owns an incomplete handoff. + std::shared_ptr<const PublishedTreeState> complete_l1 = nullptr; + Root(std::shared_ptr<const PublishedTreeState> value, + std::shared_ptr<detail::AllocationReservation> reservation); + static Root *create(std::shared_ptr<const PublishedTreeState> value, + const std::shared_ptr<detail::AllocationReservation> &metadata = {}); + static void dispose(Root *root) noexcept; + }; + + static std::shared_ptr<const PublishedTreeState> + prepare_complete_view(const Root ¤t, const PublishedTreeState &next, + const std::shared_ptr<detail::AllocationReservation> &metadata); + PublicationResult commit(const PublicationTransaction &transaction); + void drain() noexcept; + void complete(PublicationRequest *request) noexcept; + void enqueue(const std::shared_ptr<PublicationRequest> &request); + + std::array<std::atomic<uint64_t>, kMaintenanceSlots> maintenance_times_{}; + std::atomic<uint64_t> maintenance_scan_pid_{0}; + const size_t capacity_; + const uint64_t identity_; + std::atomic<size_t> queued_{0}; + std::atomic<size_t> ordinary_queued_{0}; + std::atomic<bool> checkpoint_completion_{false}; + std::atomic<bool> lane_prepared_{false}; + std::atomic<bool> closed_{false}; + std::atomic<uint64_t> requested_gc_floor_{0}; + std::atomic<bool> running_{false}; + std::atomic<PublicationRequest *> incoming_{nullptr}; + std::atomic<Root *> root_{nullptr}; + // Retired roots may still contain allocator-backed checkpoint sets when + // epoch destruction drains them, so this resource must outlive the epoch. + std::pmr::memory_resource *const checkpoint_upstream_; + std::shared_ptr<CheckpointLedger> checkpoint_ = nullptr; + mutable EpochManager epoch_; + const std::shared_ptr<TreeWorkMetrics> work_metrics_ = nullptr; + const std::shared_ptr<SharedPageCache> page_cache_ = nullptr; +#ifdef CROWDB_TREE_TEST_UTIL + std::function<void()> before_root_for_tests_; +#endif +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/range_rebuild.cpp b/lib/crowdb-tree/src/btree/range_rebuild.cpp index e8f0fb573..be389e65d 100644 --- a/lib/crowdb-tree/src/btree/range_rebuild.cpp +++ b/lib/crowdb-tree/src/btree/range_rebuild.cpp @@ -1,441 +1,12 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -// B+tree range-rebuild implementation. - -#include "crowdb-tree/btree/range_rebuild.h" - -#include "crowdb-tree/btree/cell.h" -#include "crowdb-tree/maptable/frame_page.h" - -#include <algorithm> -#include <string> -#include <unordered_map> -#include <unordered_set> -#include <utility> -#include <vector> +#include "runtime_range_rebuild.h" namespace crowdb::tree { -namespace -{ - -struct RebuildEntry -{ - std::string key; - std::string cell; -}; - -struct RebuildLeaf -{ - uint64_t source_page_id = kInvalidPageId; - uint64_t page_id = kInvalidPageId; - uint64_t source_right = kInvalidPageId; - bool wholly_contained{false}; - std::vector<RebuildEntry> entries; -}; - -struct RebuildNode -{ - uint64_t page_id = kInvalidPageId; - std::string first_key; - std::string last_key; - uint64_t first_leaf_page_id = kInvalidPageId; - uint64_t last_leaf_page_id = kInvalidPageId; -}; - -using FrameMap = std::unordered_map<uint64_t, const NativeFrame *>; - -Status copy_overflow_chain(uint64_t head, const FrameMap &source_by_id, std::unordered_set<uint64_t> *copied, - std::vector<NativeFrame> *output, RangeRebuildStats *stats) -{ - std::unordered_set<uint64_t> chain; - uint64_t page_id = head; - while (page_id != kInvalidPageId) { - if (!chain.insert(page_id).second) { - return Status::corruption("range rebuild: cyclic overflow chain"); - } - if (copied->contains(page_id)) { - return Status::Ok(); - } - const auto found = source_by_id.find(page_id); - if (found == source_by_id.end() || frame_page_type(found->second->frame.data()) != page_type::kOverflowFrame) { - return Status::corruption("range rebuild: missing overflow frame"); - } - const NativeFrame &frame = *found->second; - output->push_back(frame); - copied->insert(page_id); - ++stats->pages_reused; - OverflowFrameView overflow(frame.frame.data(), static_cast<uint32_t>(frame.frame.size())); - page_id = overflow.next_page_id(); - } - return Status::Ok(); -} - -void collect_rebuild_leaves(const std::vector<NativeFrame> &source_frames, const KeyRange &range, - uint64_t *next_page_id, std::vector<RebuildLeaf> *leaves, RangeRebuildStats *stats) -{ - for (const NativeFrame &frame : source_frames) { - *next_page_id = std::max(*next_page_id, frame.page_id + 1); - if (frame_page_type(frame.frame.data()) != page_type::kLeafBase) { - continue; - } - LeafFrameView leaf(frame.frame.data(), static_cast<uint32_t>(frame.frame.size())); - RebuildLeaf rebuild_leaf; - rebuild_leaf.source_page_id = frame.page_id; - rebuild_leaf.source_right = leaf.right_sibling(); - rebuild_leaf.wholly_contained = !leaf.empty(); - for (uint32_t index = 0; index < leaf.count(); ++index) { - Slice key = leaf.key(index); - ++stats->entries_examined; - if (!range.contains(key)) { - ++stats->entries_filtered; - rebuild_leaf.wholly_contained = false; - continue; - } - Slice cell = leaf.cell(index); - rebuild_leaf.entries.push_back({.key = key.to_string(), .cell = std::string(cell.data(), cell.size())}); - ++stats->entries_emitted; - } - if (!rebuild_leaf.entries.empty()) { - leaves->push_back(std::move(rebuild_leaf)); - } - } - std::sort(leaves->begin(), leaves->end(), [](const RebuildLeaf &left, const RebuildLeaf &right) { - return left.entries.front().key < right.entries.front().key; - }); -} - -Status fit_rebuild_leaves(uint32_t frame_bytes, uint64_t *next_page_id, std::vector<RebuildLeaf> *leaves) -{ - const size_t capacity = frame_bytes - kFrameHeaderSize - kFrameTrailerSize; - std::vector<RebuildLeaf> fitted; - for (RebuildLeaf &source : *leaves) { - size_t source_bytes = 0; - for (const RebuildEntry &entry : source.entries) { - const size_t entry_bytes = kLeafSlotSize + entry.key.size() + entry.cell.size(); - if (entry_bytes > capacity) { - return Status::resource_exhausted("range rebuild: leaf entry does not fit destination frame"); - } - source_bytes += entry_bytes; - } - if (source_bytes <= capacity) { - source.page_id = source.wholly_contained ? source.source_page_id : (*next_page_id)++; - fitted.push_back(std::move(source)); - continue; - } - - RebuildLeaf part; - part.source_page_id = source.source_page_id; - size_t part_bytes = 0; - for (RebuildEntry &entry : source.entries) { - const size_t entry_bytes = kLeafSlotSize + entry.key.size() + entry.cell.size(); - if (part_bytes + entry_bytes > capacity) { - part.page_id = (*next_page_id)++; - fitted.push_back(std::move(part)); - part = RebuildLeaf{}; - part.source_page_id = source.source_page_id; - part_bytes = 0; - } - part.entries.push_back(std::move(entry)); - part_bytes += entry_bytes; - } - part.page_id = (*next_page_id)++; - fitted.push_back(std::move(part)); - } - *leaves = std::move(fitted); - return Status::Ok(); -} - -Status build_leaf_frames(const std::vector<NativeFrame> &source_frames, uint32_t frame_bytes, - std::vector<RebuildLeaf> *leaves, uint64_t *next_page_id, std::vector<NativeFrame> *output, - std::vector<RebuildNode> *level, RangeRebuildStats *stats) -{ - FrameMap source_by_id; - for (const NativeFrame &frame : source_frames) { - source_by_id.emplace(frame.page_id, &frame); - } - std::unordered_set<uint64_t> copied_overflow; - - if (leaves->empty()) { - const uint64_t page_id = (*next_page_id)++; - NativeFrame frame{.page_id = page_id, .frame = std::vector<uint8_t>(frame_bytes)}; - LeafFrameBuilder builder(frame.frame.data(), frame_bytes); - builder.finish(page_id, kInvalidPageId); - output->push_back(std::move(frame)); - level->push_back({.page_id = page_id, .first_key = {}, .last_key = {}}); - ++stats->pages_rebuilt; - return Status::Ok(); - } - - for (size_t leaf_index = 0; leaf_index < leaves->size(); ++leaf_index) { - RebuildLeaf &leaf = (*leaves)[leaf_index]; - uint64_t right_sibling = leaf_index + 1 < leaves->size() ? (*leaves)[leaf_index + 1].page_id : kInvalidPageId; - const auto source = source_by_id.find(leaf.source_page_id); - bool can_reuse = leaf.wholly_contained && source != source_by_id.end() && - source->second->frame.size() == frame_bytes && leaf.source_right == right_sibling; - if (can_reuse) { - output->push_back(*source->second); - ++stats->pages_reused; - } - else { - NativeFrame frame{.page_id = leaf.page_id, .frame = std::vector<uint8_t>(frame_bytes)}; - LeafFrameBuilder builder(frame.frame.data(), frame_bytes); - for (const RebuildEntry &entry : leaf.entries) { - if (!builder.try_append_sorted(Slice(entry.key), Slice(entry.cell))) { - return Status::resource_exhausted("range rebuild: filtered leaf does not fit destination frame"); - } - } - builder.finish(leaf.page_id, right_sibling); - output->push_back(std::move(frame)); - ++stats->pages_rebuilt; - } - - for (const RebuildEntry &entry : leaf.entries) { - CellView cell{Slice(entry.cell)}; - if (!cell.valid()) { - return Status::corruption("range rebuild: invalid leaf cell"); - } - if (!cell.is_overflow()) { - continue; - } - if (entry.cell.size() != kOverflowCellSize) { - return Status::corruption("range rebuild: invalid overflow cell"); - } - Status copy_status = - copy_overflow_chain(cell.overflow_head(), source_by_id, &copied_overflow, output, stats); - if (!copy_status.ok()) { - return copy_status; - } - } - level->push_back({ - .page_id = leaf.page_id, - .first_key = leaf.entries.front().key, - .last_key = leaf.entries.back().key, - .first_leaf_page_id = leaf.page_id, - .last_leaf_page_id = leaf.page_id, - }); - } - return Status::Ok(); -} - -Status build_inner_frames(uint32_t frame_bytes, uint32_t inner_max_keys, uint64_t *next_page_id, - std::vector<RebuildNode> *level, std::vector<NativeFrame> *output, RangeRebuildStats *stats) -{ - const size_t max_children = std::max<size_t>(2, static_cast<size_t>(inner_max_keys) + 1); - while (level->size() > 1) { - std::vector<RebuildNode> parents; - size_t begin = 0; - while (begin < level->size()) { - size_t count = std::min(max_children, level->size() - begin); - if (level->size() - (begin + count) == 1 && count > 2) { - --count; - } - - bool built = false; - NativeFrame frame; - while (count != 0) { - frame.page_id = (*next_page_id)++; - frame.frame.assign(frame_bytes, 0); - std::vector<uint64_t> children; - std::vector<Slice> separators; - children.reserve(count); - separators.reserve(count - 1); - for (size_t index = 0; index < count; ++index) { - const RebuildNode &child = (*level)[begin + index]; - children.push_back(child.page_id); - if (index != 0) { - separators.emplace_back(child.first_key); - } - } - if (inner_frame_build(frame.frame.data(), frame_bytes, frame.page_id, children, separators)) { - frame_set_inner_fence_pages(frame.frame.data(), frame_bytes, (*level)[begin].first_leaf_page_id, - (*level)[begin + count - 1].last_leaf_page_id); - built = true; - break; - } - --count; - } - if (!built) { - return Status::resource_exhausted("range rebuild: inner separator does not fit destination frame"); - } - parents.push_back({ - .page_id = frame.page_id, - .first_key = (*level)[begin].first_key, - .last_key = (*level)[begin + count - 1].last_key, - .first_leaf_page_id = (*level)[begin].first_leaf_page_id, - .last_leaf_page_id = (*level)[begin + count - 1].last_leaf_page_id, - }); - output->push_back(std::move(frame)); - ++stats->pages_rebuilt; - begin += count; - } - *level = std::move(parents); - } - return Status::Ok(); -} - -Status build_filtered_frames(const std::vector<NativeFrame> &source_frames, const KeyRange &range, - const Config &destination_options, uint64_t source_next_page_id, - std::vector<NativeFrame> *output, uint64_t *root_page_id, RangeRebuildStats *stats) -{ - uint64_t next_page_id = source_next_page_id; - std::vector<RebuildLeaf> leaves; - collect_rebuild_leaves(source_frames, range, &next_page_id, &leaves, stats); - Status fit_status = fit_rebuild_leaves(destination_options.frame_bytes, &next_page_id, &leaves); - if (!fit_status.ok()) { - return fit_status; - } - - std::vector<RebuildNode> level; - Status leaf_status = build_leaf_frames(source_frames, destination_options.frame_bytes, &leaves, &next_page_id, - output, &level, stats); - if (!leaf_status.ok()) { - return leaf_status; - } - Status inner_status = build_inner_frames(destination_options.frame_bytes, destination_options.inner_max_keys, - &next_page_id, &level, output, stats); - if (!inner_status.ok()) { - return inner_status; - } - *root_page_id = level.front().page_id; - return Status::Ok(); -} - -} // namespace - -Status rebuild_range(Crowdbtree &source, const KeyRange &range, Config destination_options, - std::unique_ptr<Crowdbtree> *out, RangeRebuildStats *stats) +Status rebuild_range(Crowdbtree &source, const KeyRange &range, Config options, std::unique_ptr<Crowdbtree> *out, + RangeRebuildStats *stats) { - if (out == nullptr || destination_options.page_store == nullptr) { - return Status::invalid_argument("range rebuild requires an output and durable destination store"); - } - Status range_status = range.validate(); - if (!range_status.ok()) { - return range_status; - } - - bool mapping_inherited = false; - if (source.opt_.page_store != nullptr) { - Status inherit_status = destination_options.page_store->inherit_snapshot_from(*source.opt_.page_store); - if (!inherit_status.ok()) { - return inherit_status; - } - mapping_inherited = destination_options.page_store->has_inherited_snapshot(); - } - Config open_options = destination_options; - open_options.key_range = mapping_inherited ? KeyRange::unbounded() : range; - std::unique_ptr<Crowdbtree> destination; - Status open_status = Crowdbtree::open(open_options, &destination); - if (!open_status.ok()) { - return open_status; - } - - RangeRebuildStats local; - std::unique_ptr<NativeFrameIterator> iterator; - Status native_status = source.open_native_frame_iterator(range.is_bounded() ? &range : nullptr, &iterator); - if (!native_status.ok()) { - return native_status; - } - const uint64_t source_root = iterator->root_page_id(); - const uint64_t at_slot = iterator->at_slot(); - const uint64_t next_page_id = iterator->next_page_id(); - - constexpr size_t kNativeBatchBytes = 4U * 1024U * 1024U; - const size_t batch_frames = std::max<size_t>(1, kNativeBatchBytes / source.opt_.frame_bytes); - std::vector<NativeFrame> source_frames; - bool iteration_complete = false; - bool all_contained = local.subtrees_skipped == 0; - bool saw_source_frame = false; - if (source.opt_.frame_bytes != destination_options.frame_bytes) { - return Status::invalid_argument("range rebuild requires matching fixed frame sizes"); - } - while (!iteration_complete) { - std::vector<NativeFrame> batch; - native_status = iterator->next(batch_frames, &batch, &iteration_complete); - if (!native_status.ok()) { - return native_status; - } - for (NativeFrame &frame : batch) { - saw_source_frame = true; - const page_type type = frame_page_type(frame.frame.data()); - if (type == page_type::kLeafBase) { - LeafFrameView leaf(frame.frame.data(), static_cast<uint32_t>(frame.frame.size())); - for (uint32_t index = 0; index < leaf.count(); ++index) { - ++local.entries_examined; - if (!range.contains(leaf.key(index))) { - ++local.entries_filtered; - if (all_contained) { - all_contained = false; - std::erase_if(source_frames, [](const NativeFrame &prior) { - const page_type prior_type = frame_page_type(prior.frame.data()); - return prior_type != page_type::kLeafBase && prior_type != page_type::kOverflowFrame; - }); - } - } - } - } - if (all_contained || type == page_type::kLeafBase || type == page_type::kOverflowFrame) { - source_frames.push_back(std::move(frame)); - } - } - } - local.subtrees_skipped = iterator->subtrees_skipped(); - if (local.subtrees_skipped != 0 && all_contained) { - all_contained = false; - std::erase_if(source_frames, [](const NativeFrame &frame) { - const page_type type = frame_page_type(frame.frame.data()); - return type != page_type::kLeafBase && type != page_type::kOverflowFrame; - }); - } - if (!saw_source_frame) { - return Status::corruption("range rebuild: source snapshot has no root"); - } - bool source_mapping_clean = true; - for (uint64_t seg_idx = 0; seg_idx < MappingTable::kMaxSegments && source_mapping_clean; ++seg_idx) { - auto *segment = source.mapping_.segment_at(seg_idx); - source_mapping_clean = segment == nullptr || !segment->is_dirty(); - } - const bool reuse_inherited_frames = - mapping_inherited && source_mapping_clean && - destination_options.page_store->inherited_snapshot_matches(*source.opt_.page_store); - for (NativeFrame &frame : source_frames) { - frame.inherited = reuse_inherited_frames; - } - - std::vector<NativeFrame> output_frames; - uint64_t output_root = source_root; - if (all_contained) { - local.entries_emitted = local.entries_examined; - local.pages_reused = source_frames.size(); - output_frames = std::move(source_frames); - } - else { - const uint64_t subtrees_skipped = local.subtrees_skipped; - local = {}; - local.subtrees_skipped = subtrees_skipped; - native_status = build_filtered_frames(source_frames, range, destination_options, next_page_id, &output_frames, - &output_root, &local); - if (!native_status.ok()) { - return native_status; - } - } - - native_status = destination->install_range_snapshot_native(std::move(output_frames), output_root, at_slot, - next_page_id, mapping_inherited); - if (!native_status.ok()) { - return native_status; - } - destination->opt_.key_range = range; - Status snapshot_status = destination->snapshot(); - if (!snapshot_status.ok()) { - return snapshot_status; - } - if (stats != nullptr) { - *stats = local; - } - *out = std::move(destination); - return Status::Ok(); + return rebuild_range_runtime(source, range, std::move(options), out, stats); } - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/read_value_bound.h b/lib/crowdb-tree/src/btree/read_value_bound.h new file mode 100644 index 000000000..04cf0381e --- /dev/null +++ b/lib/crowdb-tree/src/btree/read_value_bound.h @@ -0,0 +1,22 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/published_state.h" + +#include <algorithm> + +namespace crowdb::tree +{ +// Inline overlay cells lend their existing owners, even for large values or +// unknown live-key counts. Only encoded overflow cells need an assembly window. +// Bases keep that window unless their exact stored dependencies prove that +// they contain only inline cells, even when their decoded frames are absent. +inline size_t overflow_value_bytes_bound(const PublishedTreeState &view, size_t maximum) noexcept +{ + if (!view.mapping.copy_bounds().may_have_overflow_values) { + return 0; + } + return view.stats.kv_unknown_pages == 0 ? std::min<uint64_t>(maximum, view.stats.kv_known_bytes) : maximum; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/recovered_frame_identity.h b/lib/crowdb-tree/src/btree/recovered_frame_identity.h new file mode 100644 index 000000000..41c3e4ddc --- /dev/null +++ b/lib/crowdb-tree/src/btree/recovered_frame_identity.h @@ -0,0 +1,22 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/backing_ref.h" +#include "crowdb-tree/maptable/frame_page.h" + +namespace crowdb::tree +{ +// Legacy local page builders stamped the not-yet-assigned page id into their +// frames. Recovery's validated anchor/segment or captured overflow descriptor +// supplies the real identity. Chunk packs and imported resident frames always +// require a concrete id; an unrelated concrete id is never a legacy marker. +[[nodiscard]] inline bool recovered_frame_identity_matches(const BackingRef &backing, page_type type, uint64_t stored, + uint64_t expected) noexcept +{ + const bool legacy_type = + type == page_type::kLeafBase || type == page_type::kInnerBase || type == page_type::kOverflowFrame; + return expected != kInvalidPageId && (stored == expected || (backing.storage == BackingStorage::kDurableBlob && + legacy_type && stored == kInvalidPageId)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/replay_controls.cpp b/lib/crowdb-tree/src/btree/replay_controls.cpp new file mode 100644 index 000000000..fc1a3e67c --- /dev/null +++ b/lib/crowdb-tree/src/btree/replay_controls.cpp @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "replay_controls.h" + +#include "allocation_bound.h" + +#include <cassert> + +namespace crowdb::tree +{ +struct ReplayControls::Slot +{ + explicit Slot(std::shared_ptr<detail::AllocationReservation> value) + : metadata(std::move(value)), + available(metadata->available()) + { + } + + ~Slot() + { + metadata->seal(); + } + + std::shared_ptr<detail::AllocationReservation> metadata; + const size_t available; + std::atomic<bool> busy{false}; +}; + +ReplayControls::ReplayControls(std::shared_ptr<detail::AllocationReservation> metadata) + : metadata_(std::move(metadata)), + slots_(decltype(slots_)::allocator_type(metadata_)) +{ +} + +Status ReplayControls::create(const std::shared_ptr<detail::ResourceBudget::Account> &account, size_t ordinary_slots, + size_t control_bytes, std::shared_ptr<ReplayControls> *out) +{ + if (out == nullptr || *out || ordinary_slots == 0 || ordinary_slots > SIZE_MAX - 2) { + return Status::invalid_argument("replay controls require a finite ordinary window"); + } + const auto count = ordinary_slots + 2; + const auto bound = detail::AllocationBound(detail::shared_allocation_bound<ReplayControls>()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add_product(count, detail::shared_allocation_bound<Slot>() + sizeof(std::shared_ptr<Slot>)) + .add_product(count, control_bytes) + .value(); + auto paid = + bound ? detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, *bound) : std::nullopt; + if (!paid) { + return Status::resource_exhausted("replay control window cannot be prepaid"); + } + auto metadata = detail::AllocationReservation::create(std::move(*paid)); + auto controls = detail::reserved_construct<ReplayControls>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) ReplayControls(metadata); }); + controls->slots_.reserve(count); + for (size_t index = 0; index < count; ++index) { + auto credit = metadata->take_reserved(control_bytes); + if (!credit) { + return Status::internal_error("prepaid replay control split failed"); + } + auto slot_budget = detail::AllocationReservation::create(std::move(*credit)); + controls->slots_.push_back(detail::reserved_shared<Slot>(metadata, std::move(slot_budget))); + } + metadata->seal(); + *out = std::move(controls); + return Status::Ok(); +} + +ReplayControls::Lease ReplayControls::acquire(bool current_gap) noexcept +{ + const auto begin = current_gap ? size_t{0} : size_t{2}; + const auto end = slots_.size(); + for (size_t index = begin; index < end; ++index) { + const auto &slot = slots_[index]; + bool expected = false; + if (!slot->busy.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + continue; + } + // Completed foreign result handles may still own control allocations. + // Do not reuse this slot until their actual storage has been freed. + if (slot->metadata->available() != slot->available) { + slot->busy.store(false, std::memory_order_release); + continue; + } + return Lease(slot); + } + return {}; +} + +ReplayControls::Lease::~Lease() +{ + if (slot_) { + slot_->busy.store(false, std::memory_order_release); + } +} + +ReplayControls::Lease::Lease(Lease &&other) noexcept : slot_(std::move(other.slot_)) +{ +} + +ReplayControls::Lease &ReplayControls::Lease::operator=(Lease &&other) noexcept +{ + if (this != &other) { + if (slot_) { + slot_->busy.store(false, std::memory_order_release); + } + slot_ = std::move(other.slot_); + } + return *this; +} + +const std::shared_ptr<detail::AllocationReservation> &ReplayControls::Lease::budget() const noexcept +{ + assert(slot_); + return slot_->metadata; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/replay_controls.h b/lib/crowdb-tree/src/btree/replay_controls.h new file mode 100644 index 000000000..aecf45daa --- /dev/null +++ b/lib/crowdb-tree/src/btree/replay_controls.h @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" + +namespace crowdb::tree +{ +// Fixed control storage, prepaid independently of the payload partition. Two +// slots are reserved for the current gap so distant replay cannot occupy them. +class ReplayControls +{ + struct Slot; + + public: + class Lease + { + public: + Lease() = default; + ~Lease(); + Lease(Lease &&other) noexcept; + Lease &operator=(Lease &&other) noexcept; + Lease(const Lease &) = delete; + Lease &operator=(const Lease &) = delete; + [[nodiscard]] const std::shared_ptr<detail::AllocationReservation> &budget() const noexcept; + + [[nodiscard]] explicit operator bool() const noexcept + { + return static_cast<bool>(slot_); + } + + private: + friend class ReplayControls; + + explicit Lease(std::shared_ptr<Slot> slot) : slot_(std::move(slot)) + { + } + + std::shared_ptr<Slot> slot_; + }; + + static Status create(const std::shared_ptr<detail::ResourceBudget::Account> &account, size_t ordinary_slots, + size_t control_bytes, std::shared_ptr<ReplayControls> *out); + [[nodiscard]] Lease acquire(bool current_gap) noexcept; + + private: + explicit ReplayControls(std::shared_ptr<detail::AllocationReservation> metadata); + std::shared_ptr<detail::AllocationReservation> metadata_; + detail::ReservedVector<std::shared_ptr<Slot>> slots_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/resource_budget.cpp b/lib/crowdb-tree/src/btree/resource_budget.cpp new file mode 100644 index 000000000..5a7ca0756 --- /dev/null +++ b/lib/crowdb-tree/src/btree/resource_budget.cpp @@ -0,0 +1,824 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "btree/resource_budget.h" + +#include "btree/allocation_reservation.h" +#include "tree_work_counters.h" + +#include <algorithm> +#include <cassert> +#include <chrono> +#include <limits> +#include <stdexcept> +#include <utility> +#include <vector> + +namespace crowdb::tree::detail +{ +namespace +{ +std::atomic<uint64_t> next_scope_id{1}; + +uint64_t budget_now_ns() noexcept +{ + return static_cast<uint64_t>( + std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +bool reserve_amount(std::atomic<size_t> &used, size_t limit, size_t amount) noexcept +{ + auto current = used.load(std::memory_order_relaxed); + while (current <= limit && amount <= limit - current) { + if (used.compare_exchange_weak(current, current + amount, std::memory_order_acq_rel)) { + return true; + } + } + return false; +} +} // namespace + +struct ResourceBudget::State +{ + State(Limits capacity, size_t max_waiters, const std::shared_ptr<AllocationBudget> &bootstrap) + : limits(capacity), + waiter_capacity(max_waiters), + waiters(max_waiters * kClasses, MetadataAllocator<std::atomic<Request *>>(bootstrap)) + { + } + + std::optional<Credit> reserve(const std::shared_ptr<Account> &account, size_t category, size_t amount) noexcept + { + if (category >= kClasses || closed.load(std::memory_order_acquire)) { + return {}; + } + if (!reserve_amount(account->used_[category], account->limits_[category], amount)) { + denied(*account, category, false); + return {}; + } + if (!reserve_amount(used[category], limits[category], amount)) { + account->used_[category].fetch_sub(amount, std::memory_order_acq_rel); + denied(*account, category, true); + return {}; + } + reserved[category].fetch_add(amount, std::memory_order_relaxed); + account->reserved_[category].fetch_add(amount, std::memory_order_relaxed); + grants.fetch_add(1, std::memory_order_relaxed); + return Credit(account, category, amount, false); + } + + void denied(Account &account, size_t category, bool host) noexcept + { + auto &tree_counter = + host ? account.observation_[category].denied_host : account.observation_[category].denied_tree; + auto &host_counter = host ? observation[category].denied_host : observation[category].denied_tree; + tree_counter.fetch_add(1, std::memory_order_relaxed); + host_counter.fetch_add(1, std::memory_order_relaxed); + } + + void wait_started(const Request &request) noexcept + { + auto &tree = request.account_->observation_[request.category_]; + auto &host = observation[request.category_]; + tree.waits.fetch_add(1, std::memory_order_relaxed); + host.waits.fetch_add(1, std::memory_order_relaxed); + tree.waiters.fetch_add(1, std::memory_order_relaxed); + host.waiters.fetch_add(1, std::memory_order_relaxed); + } + + void wait_finished(const Request &request) noexcept + { + const auto elapsed = budget_now_ns() - request.queued_at_; + auto &tree = request.account_->observation_[request.category_]; + auto &host = observation[request.category_]; + tree.wait_ns.fetch_add(elapsed, std::memory_order_relaxed); + host.wait_ns.fetch_add(elapsed, std::memory_order_relaxed); + tree.waiters.fetch_sub(1, std::memory_order_relaxed); + host.waiters.fetch_sub(1, std::memory_order_relaxed); + } + + ResourceUsageSnapshot usage(const Account *account) const noexcept + { + ResourceUsageSnapshot result; + for (size_t index = 0; index < kClasses; ++index) { + const auto &counters = account != nullptr ? account->observation_[index] : observation[index]; + auto &out = result[index]; + out.limit = account != nullptr ? account->limits_[index] : limits[index]; + out.used = (account != nullptr ? account->used_[index] : used[index]).load(std::memory_order_relaxed); + out.reserved = std::min( + out.used, static_cast<uint64_t>((account != nullptr ? account->reserved_[index] : reserved[index]) + .load(std::memory_order_relaxed))); + out.allocated = std::min( + out.used, static_cast<uint64_t>((account != nullptr ? account->allocated_[index] : allocated[index]) + .load(std::memory_order_relaxed))); + out.denied_tree = counters.denied_tree.load(std::memory_order_relaxed); + out.denied_host = counters.denied_host.load(std::memory_order_relaxed); + out.waits = counters.waits.load(std::memory_order_relaxed); + out.wait_ns = counters.wait_ns.load(std::memory_order_relaxed); + out.waiters = counters.waiters.load(std::memory_order_relaxed); + } + return result; + } + + enum class LimitFailure : uint8_t { None, Tree, Host }; + + [[nodiscard]] LimitFailure allocation_failure(const Request &request) const noexcept + { + const auto category = request.category_; + const auto tree = request.account_->allocated_[category].load(std::memory_order_acquire); + const auto host = allocated[category].load(std::memory_order_acquire); + if (request.amount_ > + request.account_->limits_[category] - std::min(tree, request.account_->limits_[category])) { + return LimitFailure::Tree; + } + return request.amount_ > limits[category] - std::min(host, limits[category]) ? LimitFailure::Host + : LimitFailure::None; + } + + void allocation_changed() noexcept + { + if (reservation_waiters.load(std::memory_order_acquire) != 0) { + notify(); + } + } + + void notify() noexcept + { + if (notifications.fetch_add(1, std::memory_order_acq_rel) != 0) { + return; + } + size_t observed = 1; + do { + const auto start = next_waiter; + // Keep every release in the notification handshake, even when + // empty: a concurrent registration must force another scan. Most + // releases have no waiters and need not inspect every host slot. + const auto count = waiting.load(std::memory_order_acquire) == 0 ? 0 : waiters.size(); + for (size_t offset = 0; offset < count; ++offset) { + auto index = (start + offset) % waiters.size(); + auto &slot = waiters[index]; + auto *request = slot.load(std::memory_order_acquire); + if (request == nullptr || request == &idle_request) { + continue; + } + bool stopped = + closed.load(std::memory_order_acquire) || request->stopped_.load(std::memory_order_acquire); + const auto failure = !stopped && request->policy_ == Request::WaitPolicy::ReservationsOnly + ? allocation_failure(*request) + : LimitFailure::None; + const bool exhausted = failure != LimitFailure::None; + if (exhausted) { + denied(*request->account_, request->category_, failure == LimitFailure::Host); + } + if (!stopped && !exhausted) { + request->credit_ = reserve(request->account_, request->category_, request->amount_); + if (!request->credit_) { + continue; + } + } + slot.store(request->registration_ != nullptr ? &idle_request : nullptr, std::memory_order_release); + next_waiter = (index + 1) % waiters.size(); + waiting.fetch_sub(1, std::memory_order_relaxed); + if (request->policy_ == Request::WaitPolicy::ReservationsOnly) { + reservation_waiters.fetch_sub(1, std::memory_order_acq_rel); + } + wait_ns.fetch_add(budget_now_ns() - request->queued_at_, std::memory_order_relaxed); + wait_finished(*request); + if (stopped) { + cancelled.fetch_add(1, std::memory_order_relaxed); + } + auto complete = request->complete_; + auto *context = request->context_; + auto outcome = TreeWaitOutcome::Granted; + auto signal = CallbackSignal::kValue; + if (stopped) { + outcome = TreeWaitOutcome::Cancelled; + signal = CallbackSignal::kStopped; + } + else if (exhausted) { + outcome = TreeWaitOutcome::Error; + signal = CallbackSignal::kError; + } + request->wait_span_.finish(outcome, budget_now_ns()); + request->pending_.store(false, std::memory_order_release); + complete(context, signal, exhausted ? Status::resource_exhausted() : Status::Ok()); + } + // One scan observes all releases that preceded it. Coalesce that + // batch so sustained allocation/free traffic cannot leave one + // notifier rescanning every waiter once per historical release. + observed = notifications.fetch_sub(observed, std::memory_order_acq_rel) - observed; + } while (observed != 0); + } + + std::weak_ptr<RetirementQueue> retirement; + Limits limits; + size_t waiter_capacity; + Request idle_request; + std::array<std::atomic<size_t>, kClasses> used{}; + std::array<std::atomic<size_t>, kClasses> reserved{}; + std::array<std::atomic<size_t>, kClasses> allocated{}; + std::array<Account::Observation, kClasses> observation{}; + std::vector<std::atomic<Request *>, MetadataAllocator<std::atomic<Request *>>> waiters; + size_t next_waiter = 0; + std::atomic<size_t> notifications{0}; + std::atomic<bool> closed{false}; + std::atomic<uint64_t> waiting{0}; + std::atomic<uint64_t> reservation_waiters{0}; + std::atomic<uint64_t> grants{0}; + std::atomic<uint64_t> cancelled{0}; + std::atomic<uint64_t> wait_ns{0}; +}; + +size_t ResourceBudget::capacity(const std::shared_ptr<Account> &account, BudgetClass category) noexcept +{ + const auto index = static_cast<size_t>(category); + return account && index < kClasses ? std::min(account->limits_[index], account->state_->limits[index]) : 0; +} + +size_t ResourceBudget::available(const std::shared_ptr<Account> &account, BudgetClass category) noexcept +{ + const auto index = static_cast<size_t>(category); + if (!account || index >= kClasses || account->state_->closed.load(std::memory_order_acquire)) { + return 0; + } + const auto tree = account->used_[index].load(std::memory_order_acquire); + const auto host = account->state_->used[index].load(std::memory_order_acquire); + return std::min(account->limits_[index] - std::min(tree, account->limits_[index]), + account->state_->limits[index] - std::min(host, account->state_->limits[index])); +} + +ResourceBudget::Account::Account(std::shared_ptr<State> state, Limits limits) + : state_(std::move(state)), + retirement_(state_->retirement), + limits_(limits), + scope_id_(next_scope_id.fetch_add(1, std::memory_order_relaxed)) +{ +} + +void ResourceBudget::Account::set_resource_metrics(const std::shared_ptr<TreeResourceMetrics> &metrics) noexcept +{ + assert(resource_metrics_.expired()); + resource_metrics_ = metrics; +} + +std::shared_ptr<TreeResourceMetrics> ResourceBudget::Account::resource_metrics() const noexcept +{ + return resource_metrics_.lock(); +} + +void ResourceBudget::Account::set_work_metrics(const std::shared_ptr<TreeWorkMetrics> &metrics) noexcept +{ + assert(retirement_metrics_.expired()); + retirement_metrics_ = metrics; +} + +void ResourceBudget::Account::drain_retirements() const noexcept +{ + auto pending = retirement_pending_.load(std::memory_order_acquire); + while (pending != 0) { + retirement_pending_.wait(pending, std::memory_order_acquire); + pending = retirement_pending_.load(std::memory_order_acquire); + } +} + +ResourceUsageSnapshot ResourceBudget::Account::resource_usage() const noexcept +{ + return state_->usage(this); +} + +ResourceUsageSnapshot ResourceBudget::resource_usage() const noexcept +{ + return state_->usage(nullptr); +} + +ResourceBudget::AccountSnapshot ResourceBudget::Account::snapshot() const noexcept +{ + AccountSnapshot result; + for (size_t index = 0; index < kClasses; ++index) { + // A release can publish reusable capacity before its mirror updates. + // Clamp each relaxed gauge to an independently sampled admission total. + const auto used = used_[index].load(std::memory_order_relaxed); + result.reserved[index] = std::min(used, reserved_[index].load(std::memory_order_relaxed)); + result.allocated[index] = std::min(used, allocated_[index].load(std::memory_order_relaxed)); + } + for (size_t index = 1; index < result.physical.size(); ++index) { + result.physical[index] = physical_[index].load(std::memory_order_relaxed); + } + return result; +} + +AllocationReservation::AllocationReservation(ResourceBudget::Credit credit, + std::shared_ptr<AllocationReservation> parent) + : category_(0), + available_(0), + parent_(std::move(parent)) +{ + if (!credit.account_ || credit.allocated_ || credit.amount_ >= kSealed) { + throw std::invalid_argument("allocation reservation requires unconsumed metadata credit"); + } + assert(!parent_ || (parent_->account_ == credit.account_ && parent_->category_ == credit.category_)); + account_ = std::move(credit.account_); + category_ = credit.category_; + available_.store(std::exchange(credit.amount_, 0), std::memory_order_relaxed); +} + +AllocationReservation::~AllocationReservation() +{ + seal(); +} + +bool AllocationReservation::claim(size_t bytes) noexcept +{ + auto remaining = available_.load(std::memory_order_relaxed); + while ((remaining & kSealed) == 0 && bytes <= remaining) { + if (available_.compare_exchange_weak(remaining, remaining - bytes, std::memory_order_acq_rel)) { + auto &state = *account_->state_; + account_->allocated_[category_].fetch_add(bytes, std::memory_order_relaxed); + account_->reserved_[category_].fetch_sub(bytes, std::memory_order_relaxed); + state.allocated[category_].fetch_add(bytes, std::memory_order_relaxed); + state.reserved[category_].fetch_sub(bytes, std::memory_order_relaxed); + state.allocation_changed(); + return true; + } + } + return false; +} + +void AllocationReservation::release(size_t bytes) noexcept +{ + auto state = account_->state_; + auto remaining = available_.load(std::memory_order_acquire); + while ((remaining & kSealed) == 0) { + if (available_.compare_exchange_weak(remaining, remaining + bytes, std::memory_order_acq_rel)) { + account_->allocated_[category_].fetch_sub(bytes, std::memory_order_relaxed); + account_->reserved_[category_].fetch_add(bytes, std::memory_order_relaxed); + state->allocated[category_].fetch_sub(bytes, std::memory_order_relaxed); + state->reserved[category_].fetch_add(bytes, std::memory_order_relaxed); + return; + } + } + if (parent_) { + parent_->release(bytes); + return; + } + account_->allocated_[category_].fetch_sub(bytes, std::memory_order_relaxed); + state->allocated[category_].fetch_sub(bytes, std::memory_order_relaxed); + account_->used_[category_].fetch_sub(bytes, std::memory_order_acq_rel); + state->used[category_].fetch_sub(bytes, std::memory_order_acq_rel); + state->notify(); +} + +void AllocationReservation::seal() noexcept +{ + const auto remaining = available_.exchange(kSealed, std::memory_order_acq_rel); + if ((remaining & kSealed) != 0) { + return; + } + if (parent_ && parent_->restore_reserved(remaining)) { + return; + } + auto state = account_->state_; + account_->reserved_[category_].fetch_sub(remaining, std::memory_order_relaxed); + state->reserved[category_].fetch_sub(remaining, std::memory_order_relaxed); + account_->used_[category_].fetch_sub(remaining, std::memory_order_acq_rel); + state->used[category_].fetch_sub(remaining, std::memory_order_acq_rel); + state->notify(); +} + +bool AllocationReservation::restore_reserved(size_t bytes) noexcept +{ + auto remaining = available_.load(std::memory_order_acquire); + while ((remaining & kSealed) == 0) { + assert(bytes < kSealed - remaining); + if (available_.compare_exchange_weak(remaining, remaining + bytes, std::memory_order_acq_rel)) { + return true; + } + } + return false; +} + +size_t AllocationReservation::available() const noexcept +{ + return available_.load(std::memory_order_relaxed) & ~kSealed; +} + +std::optional<ResourceBudget::Credit> AllocationReservation::take_reserved(size_t bytes) noexcept +{ + auto remaining = available_.load(std::memory_order_acquire); + while ((remaining & kSealed) == 0 && bytes <= remaining) { + if (available_.compare_exchange_weak(remaining, remaining - bytes, std::memory_order_acq_rel)) { + return ResourceBudget::Credit(account_, category_, bytes, false); + } + } + return {}; +} + +const void *AllocationReservation::domain() const noexcept +{ + return account_->state_.get(); +} + +ResourceBudget::Credit::Credit(std::shared_ptr<Account> account, size_t category, size_t amount, bool allocated) + : account_(std::move(account)), + category_(category), + amount_(amount), + allocated_(allocated) +{ +} + +ResourceBudget::Credit::~Credit() +{ + release(); +} + +ResourceBudget::Credit::Credit(Credit &&other) noexcept + : account_(std::move(other.account_)), + category_(other.category_), + amount_(std::exchange(other.amount_, 0)), + allocated_(other.allocated_), + kind_(other.kind_) +{ +} + +ResourceBudget::Credit &ResourceBudget::Credit::operator=(Credit &&other) noexcept +{ + if (this != &other) { + release(); + account_ = std::move(other.account_); + category_ = other.category_; + amount_ = std::exchange(other.amount_, 0); + allocated_ = other.allocated_; + kind_ = other.kind_; + } + return *this; +} + +std::optional<ResourceBudget::Credit> ResourceBudget::Credit::allocate(size_t amount) noexcept +{ + if (!account_ || allocated_ || amount > amount_) { + return {}; + } + auto &state = *account_->state_; + account_->allocated_[category_].fetch_add(amount, std::memory_order_relaxed); + account_->reserved_[category_].fetch_sub(amount, std::memory_order_relaxed); + state.allocated[category_].fetch_add(amount, std::memory_order_relaxed); + state.reserved[category_].fetch_sub(amount, std::memory_order_relaxed); + amount_ -= amount; + state.allocation_changed(); + return Credit(account_, category_, amount, true); +} + +std::optional<ResourceBudget::Credit> ResourceBudget::Credit::split(size_t amount) noexcept +{ + if (!account_ || amount > amount_) { + return {}; + } + amount_ -= amount; + Credit part(account_, category_, amount, allocated_); + part.kind_ = kind_; + return part; +} + +void ResourceBudget::Credit::release() noexcept +{ + if (!account_) { + return; + } + auto account = std::move(account_); + if (kind_ != AllocationKind::kUnknown) { + account->physical_[static_cast<size_t>(kind_)].fetch_sub(amount_, std::memory_order_relaxed); + record_work(account->work_metrics(), TreeWorkCounter::MemoryReclaimedBytes, amount_, TreeWorkCause::Allocation); + } + auto state = account->state_; + auto amount = std::exchange(amount_, 0); + auto &counter = allocated_ ? state->allocated[category_] : state->reserved[category_]; + auto &account_counter = allocated_ ? account->allocated_[category_] : account->reserved_[category_]; + account_counter.fetch_sub(amount, std::memory_order_relaxed); + counter.fetch_sub(amount, std::memory_order_relaxed); + account->used_[category_].fetch_sub(amount, std::memory_order_acq_rel); + state->used[category_].fetch_sub(amount, std::memory_order_acq_rel); + state->notify(); +} + +ResourceBudget::Registration::~Registration() +{ + reset(); +} + +ResourceBudget::Registration::Registration(Registration &&other) noexcept + : state_(std::move(other.state_)), + index_(other.index_) +{ +} + +ResourceBudget::Registration &ResourceBudget::Registration::operator=(Registration &&other) noexcept +{ + if (this != &other) { + reset(); + state_ = std::move(other.state_); + index_ = other.index_; + } + return *this; +} + +void ResourceBudget::Registration::reset() noexcept +{ + if (auto state = std::move(state_)) { + auto *expected = &state->idle_request; + [[maybe_unused]] const bool released = + state->waiters[index_].compare_exchange_strong(expected, nullptr, std::memory_order_acq_rel); + assert(released); + } +} + +std::optional<ResourceBudget::Registration> ResourceBudget::try_register(const std::shared_ptr<Account> &account, + BudgetClass category) noexcept +{ + const auto value = static_cast<size_t>(category); + if (!account || value >= kClasses) { + return std::nullopt; + } + auto state = account->state_; + if (state->closed.load(std::memory_order_acquire)) { + return std::nullopt; + } + const auto begin = value * state->waiter_capacity; + for (size_t index = begin; index < begin + state->waiter_capacity; ++index) { + Request *expected = nullptr; + if (state->waiters[index].compare_exchange_strong(expected, &state->idle_request, std::memory_order_acq_rel)) { + return Registration(std::move(state), index); + } + } + return std::nullopt; +} + +ResourceBudget::Request::Request(std::shared_ptr<Account> account, BudgetClass category, size_t amount, + const Registration *registration, WaitPolicy policy) + : account_(std::move(account)), + category_(static_cast<size_t>(category)), + amount_(amount), + registration_(registration), + policy_(policy) +{ + if (!account_ || category_ >= kClasses || + (registration_ != nullptr && (registration_->state_ != account_->state_ || + registration_->index_ / account_->state_->waiter_capacity != category_))) { + throw std::invalid_argument("budget request requires account and valid class"); + } +} + +ResourceBudget::Request::~Request() +{ + assert(!pending_.load(std::memory_order_acquire)); +} + +CallbackSender ResourceBudget::Request::sender() noexcept +{ + return {this, &Request::submit}; +} + +void ResourceBudget::Request::submit(void *source, CallbackComplete complete, void *context) +{ + auto &request = *static_cast<Request *>(source); + if (request.submitted_.test_and_set(std::memory_order_acq_rel)) { + complete(context, CallbackSignal::kError, Status::invalid_argument("budget request already started")); + return; + } + auto state = request.account_->state_; + static_assert(kClasses == static_cast<size_t>(TreeWaitClass::SharedCache)); + request.queued_at_ = budget_now_ns(); + request.wait_span_.begin(request.account_->work_metrics(), static_cast<TreeWaitClass>(request.category_), + request.queued_at_); + if (request.amount_ > request.account_->limits_[request.category_] || + request.amount_ > state->limits[request.category_]) { + state->denied(*request.account_, request.category_, + request.amount_ <= request.account_->limits_[request.category_]); + request.wait_span_.finish(TreeWaitOutcome::Error, budget_now_ns()); + complete(context, CallbackSignal::kError, Status::resource_exhausted("request exceeds budget partition")); + return; + } + request.complete_ = complete; + request.context_ = context; + request.pending_.store(true, std::memory_order_release); + state->wait_started(request); + state->waiting.fetch_add(1, std::memory_order_relaxed); + if (request.policy_ == WaitPolicy::ReservationsOnly) { + state->reservation_waiters.fetch_add(1, std::memory_order_acq_rel); + } + if (request.registration_ != nullptr) { + auto *expected = &state->idle_request; + if (state->waiters[request.registration_->index_].compare_exchange_strong(expected, &request, + std::memory_order_release)) { + record_work(request.account_->work_metrics(), TreeWorkCounter::AdmissionWaits, 1, + TreeWorkCause::Allocation); + state->notify(); + return; + } + state->waiting.fetch_sub(1, std::memory_order_relaxed); + state->wait_finished(request); + if (request.policy_ == WaitPolicy::ReservationsOnly) { + state->reservation_waiters.fetch_sub(1, std::memory_order_acq_rel); + } + request.pending_.store(false, std::memory_order_release); + request.wait_span_.finish(TreeWaitOutcome::Error, budget_now_ns()); + complete(context, CallbackSignal::kError, Status::internal_error("prepaid budget waiter already in use")); + return; + } + auto start = request.category_ * state->waiter_capacity; + for (size_t index = start; index < start + state->waiter_capacity; ++index) { + auto &slot = state->waiters[index]; + Request *expected = nullptr; + if (slot.compare_exchange_strong(expected, &request, std::memory_order_release)) { + record_work(request.account_->work_metrics(), TreeWorkCounter::AdmissionWaits, 1, + TreeWorkCause::Allocation); + // Registration precedes the retry; a concurrent release cannot + // slip between a failed check and going to sleep. + state->notify(); + return; + } + } + state->waiting.fetch_sub(1, std::memory_order_relaxed); + state->wait_finished(request); + if (request.policy_ == WaitPolicy::ReservationsOnly) { + state->reservation_waiters.fetch_sub(1, std::memory_order_acq_rel); + } + request.pending_.store(false, std::memory_order_release); + request.wait_span_.finish(TreeWaitOutcome::Error, budget_now_ns()); + complete(context, CallbackSignal::kError, Status::resource_exhausted("budget waiter slots full")); +} + +void ResourceBudget::Request::request_stop() noexcept +{ + auto state = account_->state_; + stopped_.store(true, std::memory_order_release); + state->notify(); +} + +std::optional<ResourceBudget::Credit> ResourceBudget::Request::take_credit() noexcept +{ + return std::exchange(credit_, std::nullopt); +} + +ResourceBudget::ResourceBudget(Limits limits, size_t max_waiters_per_class, + const std::shared_ptr<AllocationBudget> &bootstrap) +{ + if (max_waiters_per_class == 0 || max_waiters_per_class > std::numeric_limits<size_t>::max() / kClasses) { + throw std::invalid_argument("budget requires bounded waiter slots"); + } + state_ = std::allocate_shared<State>(MetadataAllocator<State>(bootstrap), limits, max_waiters_per_class, bootstrap); +} + +bool ResourceBudget::claim_allocation(const std::shared_ptr<Account> &account, BudgetClass category, + size_t bytes) noexcept +{ + if (category == BudgetClass::kIo) { + return false; + } + auto credit = try_reserve(account, category, bytes); + if (!credit) { + return false; + } + auto allocation = credit->allocate(bytes); + if (!allocation) { + return false; + } + credit->account_.reset(); + allocation->amount_ = 0; + allocation->account_.reset(); + return true; +} + +void ResourceBudget::release_allocation(const std::shared_ptr<Account> &account, BudgetClass category, + size_t bytes) noexcept +{ + if (account) { + Credit allocation(account, static_cast<size_t>(category), bytes, true); + } +} + +const void *ResourceBudget::allocation_domain(const std::shared_ptr<Account> &account) noexcept +{ + return account != nullptr ? account->state_.get() : nullptr; +} + +bool ResourceBudget::transfer_allocation( + const std::shared_ptr<Account> &from, const std::shared_ptr<Account> &to, BudgetClass category, size_t bytes, + const std::array<size_t, static_cast<size_t>(AllocationKind::kCount)> &physical) noexcept +{ + const auto index = static_cast<size_t>(category); + if (!to || (from && from->state_ != to->state_) || index >= kClasses || category == BudgetClass::kIo || + to->state_->closed.load(std::memory_order_acquire)) { + return false; + } + if (!from) { + if (!claim_allocation(to, category, bytes)) { + return false; + } + for (size_t kind = 1; kind < physical.size(); ++kind) { + record_allocation(to, static_cast<AllocationKind>(kind), physical[kind], true); + } + return true; + } + if (from == to) { + return true; + } + if (!reserve_amount(to->used_[index], to->limits_[index], bytes)) { + to->state_->denied(*to, index, false); + return false; + } + to->allocated_[index].fetch_add(bytes, std::memory_order_relaxed); + [[maybe_unused]] const auto previous = from->allocated_[index].fetch_sub(bytes, std::memory_order_relaxed); + assert(previous >= bytes); + from->used_[index].fetch_sub(bytes, std::memory_order_acq_rel); + for (size_t kind = 0; kind < physical.size(); ++kind) { + to->physical_[kind].fetch_add(physical[kind], std::memory_order_relaxed); + [[maybe_unused]] const auto old = from->physical_[kind].fetch_sub(physical[kind], std::memory_order_relaxed); + assert(old >= physical[kind]); + } + // Host admission is unchanged: these are the same live allocations. + from->state_->notify(); + return true; +} + +ResourceBudget::~ResourceBudget() +{ + close(); + // The facade is no longer callable. Existing accounts keep their own weak + // queue association, while the host ledger must not retain a queue control + // block whose allocator points back to that same ledger. + state_->retirement.reset(); +} + +std::shared_ptr<ResourceBudget::Account> ResourceBudget::make_account(Limits limits) +{ + return std::shared_ptr<Account>(new Account(state_, limits)); +} + +std::shared_ptr<ResourceBudget::Account> ResourceBudget::make_account(Limits limits, + const std::shared_ptr<AllocationBudget> &budget) +{ + if (!budget) { + throw std::invalid_argument("charged account requires host metadata credit"); + } + MetadataAllocator<Account> allocator(budget); + auto *account = allocator.allocate(1); + try { + ::new (static_cast<void *>(account)) Account(state_, limits); + } + catch (...) { + allocator.deallocate(account, 1); + throw; + } + return {account, + [allocator](Account *value) mutable { + std::destroy_at(value); + allocator.deallocate(value, 1); + }, + allocator}; +} + +std::optional<ResourceBudget::Credit> ResourceBudget::try_reserve(const std::shared_ptr<Account> &account, + BudgetClass category, size_t amount) noexcept +{ + if (!account) { + return {}; + } + auto state = account->state_; + auto credit = state->reserve(account, static_cast<size_t>(category), amount); + if (!credit) { + record_work(account->work_metrics(), TreeWorkCounter::AdmissionRejected, 1, TreeWorkCause::Allocation); + // A failed host acquisition rolls back a temporary Tree charge. A + // concurrent waiter may have observed that charge; retry after rollback. + state->notify(); + } + return credit; +} + +void ResourceBudget::set_retirement_queue(const std::shared_ptr<RetirementQueue> &queue) +{ + state_->retirement = queue; +} + +void ResourceBudget::close() noexcept +{ + auto state = state_; + state->closed.store(true, std::memory_order_release); + state->notify(); +} + +ResourceBudget::Snapshot ResourceBudget::snapshot() const noexcept +{ + Snapshot result; + for (size_t i = 0; i < kClasses; ++i) { + result.reserved[i] = state_->reserved[i].load(std::memory_order_relaxed); + result.allocated[i] = state_->allocated[i].load(std::memory_order_relaxed); + } + result.waiters = state_->waiting.load(std::memory_order_relaxed); + result.grants = state_->grants.load(std::memory_order_relaxed); + result.cancelled = state_->cancelled.load(std::memory_order_relaxed); + result.wait_ns = state_->wait_ns.load(std::memory_order_relaxed); + return result; +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/resource_budget.h b/lib/crowdb-tree/src/btree/resource_budget.h new file mode 100644 index 000000000..dbd761354 --- /dev/null +++ b/lib/crowdb-tree/src/btree/resource_budget.h @@ -0,0 +1,266 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-tree/memory_budget.h" +#include "crowdb-tree/resource_usage.h" +#include "stdexec_adapter.h" +#include "tree_wait_metrics.h" + +#include <array> +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <optional> + +namespace crowdb::tree::detail +{ +class AllocationReservation; +class AllocationAccount; +class RetirementQueue; + +enum class BudgetClass : uint8_t { kHandoff, kRouting, kReclaim, kAdmittedWriter, kGapRepair, kIo, kCount }; + +// Partitions do not borrow from one another: ordinary handoff cannot consume +// the reserve needed to route, reclaim, or apply already admitted writes. +// Values are bytes except kIo, whose unit is simultaneous I/O operations. +class ResourceBudget +{ + struct State; + + public: + static constexpr size_t kClasses = static_cast<size_t>(BudgetClass::kCount); + using Limits = std::array<size_t, kClasses>; + + struct Snapshot + { + Limits reserved{}; + Limits allocated{}; + uint64_t waiters = 0; + uint64_t grants = 0; + uint64_t cancelled = 0; + uint64_t wait_ns = 0; + }; + + struct AccountSnapshot + { + Limits reserved{}; + Limits allocated{}; + std::array<size_t, static_cast<size_t>(AllocationKind::kCount)> physical{}; + }; + + class Account + { + public: + // Independent relaxed samples; these counters never govern admission. + [[nodiscard]] AccountSnapshot snapshot() const noexcept; + [[nodiscard]] ResourceUsageSnapshot resource_usage() const noexcept; + + [[nodiscard]] uint64_t scope_id() const noexcept + { + return scope_id_; + } + + void drain_retirements() const noexcept; + // Bootstrap only, before this account or its binding is published. + void set_work_metrics(const std::shared_ptr<TreeWorkMetrics> &metrics) noexcept; + + [[nodiscard]] std::shared_ptr<TreeWorkMetrics> work_metrics() const noexcept + { + return retirement_metrics_.lock(); + } + + void set_resource_metrics(const std::shared_ptr<TreeResourceMetrics> &metrics) noexcept; + [[nodiscard]] std::shared_ptr<TreeResourceMetrics> resource_metrics() const noexcept; +#ifdef CROWDB_TREE_TEST_UTIL + [[nodiscard]] size_t retirements_pending_for_tests() const noexcept + { + return retirement_pending_.load(std::memory_order_acquire); + } +#endif + + private: + friend class AllocationReservation; + friend class AllocationAccount; + friend class ResourceBudget; + friend struct State; + explicit Account(std::shared_ptr<State> state, Limits limits); + std::shared_ptr<State> state_; + std::weak_ptr<RetirementQueue> retirement_; + std::weak_ptr<TreeWorkMetrics> retirement_metrics_; + std::weak_ptr<TreeResourceMetrics> resource_metrics_; + mutable std::atomic<size_t> retirement_pending_{0}; + Limits limits_; + uint64_t scope_id_; + std::array<std::atomic<size_t>, kClasses> used_{}; + std::array<std::atomic<size_t>, kClasses> reserved_{}; + std::array<std::atomic<size_t>, kClasses> allocated_{}; + + struct Observation + { + std::atomic<uint64_t> denied_tree{0}, denied_host{0}; + std::atomic<uint64_t> waits{0}, wait_ns{0}, waiters{0}; + }; + + std::array<Observation, kClasses> observation_{}; + std::array<std::atomic<size_t>, static_cast<size_t>(AllocationKind::kCount)> physical_{}; + }; + + class Credit + { + public: + Credit() = default; + ~Credit(); + Credit(Credit &&other) noexcept; + Credit &operator=(Credit &&other) noexcept; + Credit(const Credit &) = delete; + Credit &operator=(const Credit &) = delete; + + [[nodiscard]] size_t amount() const noexcept + { + return amount_; + } + + [[nodiscard]] bool allocated() const noexcept + { + return allocated_; + } + + [[nodiscard]] bool belongs_to(const std::shared_ptr<Account> &account, BudgetClass category) const noexcept + { + return account_ == account && category_ == static_cast<size_t>(category); + } + + // Transfer part of a reservation into allocation ownership, without + // changing admission usage or charging the same bytes a second time. + [[nodiscard]] std::optional<Credit> allocate(size_t amount) noexcept; + [[nodiscard]] std::optional<Credit> split(size_t amount) noexcept; + void release() noexcept; + // Call only after a raw allocation succeeds. The classified token + // follows moves/splits and subtracts its bytes on final release. + [[nodiscard]] bool mark_allocation(AllocationKind kind) noexcept; + + private: + friend class AllocationReservation; + friend class AllocationAccount; + friend class ResourceBudget; + friend struct State; + Credit(std::shared_ptr<Account> account, size_t category, size_t amount, bool allocated); + std::shared_ptr<Account> account_; + size_t category_ = 0; + size_t amount_ = 0; + bool allocated_ = false; + AllocationKind kind_ = AllocationKind::kUnknown; + }; + + class Request; + + // A fixed host waiter slot acquired before an operation can be chosen. + // One request at a time may borrow it; ownership outlives every borrower. + class Registration + { + public: + Registration() = default; + ~Registration(); + Registration(Registration &&other) noexcept; + Registration &operator=(Registration &&other) noexcept; + Registration(const Registration &) = delete; + Registration &operator=(const Registration &) = delete; + + private: + friend class ResourceBudget; + friend class Request; + + Registration(std::shared_ptr<State> state, size_t index) : state_(std::move(state)), index_(index) + { + } + + void reset() noexcept; + std::shared_ptr<State> state_; + size_t index_ = 0; + }; + + [[nodiscard]] static std::optional<Registration> try_register(const std::shared_ptr<Account> &account, + BudgetClass category) noexcept; + + class Request + { + public: + // Unaccepted soft work may wait for reservations to finish, but cannot + // require live allocations that only this same work could retire. + enum class WaitPolicy : uint8_t { AnyRelease, ReservationsOnly }; + + Request(std::shared_ptr<Account> account, BudgetClass category, size_t amount, + const Registration *registration = nullptr, WaitPolicy policy = WaitPolicy::AnyRelease); + ~Request(); + Request(const Request &) = delete; + Request &operator=(const Request &) = delete; + + // Request and connected operation must stay alive until terminal. + [[nodiscard]] CallbackSender sender() noexcept; + void request_stop() noexcept; + [[nodiscard]] std::optional<Credit> take_credit() noexcept; + + private: + friend struct State; + Request() = default; + static void submit(void *source, CallbackComplete complete, void *context); + std::shared_ptr<Account> account_; + size_t category_ = 0; + size_t amount_ = 0; + const Registration *registration_ = nullptr; + WaitPolicy policy_ = WaitPolicy::AnyRelease; + std::optional<Credit> credit_; + std::atomic<bool> stopped_{false}; + std::atomic<bool> pending_{false}; + std::atomic_flag submitted_ = ATOMIC_FLAG_INIT; + CallbackComplete complete_ = nullptr; + void *context_ = nullptr; + uint64_t queued_at_ = 0; + TreeWaitSpan wait_span_; + }; + + // Waiter slots are partitioned too, so handoff waiters cannot exhaust the + // registrations required by routing/reclaim progress. + ResourceBudget(Limits limits, size_t max_waiters_per_class, + const std::shared_ptr<AllocationBudget> &bootstrap = {}); + ~ResourceBudget(); + ResourceBudget(const ResourceBudget &) = delete; + ResourceBudget &operator=(const ResourceBudget &) = delete; + + [[nodiscard]] std::shared_ptr<Account> make_account(Limits limits); + // Charge both the account and its shared control block to host metadata. + [[nodiscard]] std::shared_ptr<Account> make_account(Limits limits, const std::shared_ptr<AllocationBudget> &budget); + [[nodiscard]] static std::optional<Credit> try_reserve(const std::shared_ptr<Account> &account, + BudgetClass category, size_t amount) noexcept; + [[nodiscard]] static size_t capacity(const std::shared_ptr<Account> &account, BudgetClass category) noexcept; + // Planning hint from admission usage, including both host and Tree limits. + // A subsequent reservation must still claim the capacity atomically. + [[nodiscard]] static size_t available(const std::shared_ptr<Account> &account, BudgetClass category) noexcept; + // Stop new admission and cancel waiting requests. Outstanding credit + // remains charged until its actual owner releases it. + // Bootstrap-only before publishing the host runtime. New accounts copy + // this weak association; the queue never retains its executor. + void set_retirement_queue(const std::shared_ptr<RetirementQueue> &queue); + void close() noexcept; + [[nodiscard]] Snapshot snapshot() const noexcept; + [[nodiscard]] ResourceUsageSnapshot resource_usage() const noexcept; + + private: + friend class AllocationAccount; + static bool claim_allocation(const std::shared_ptr<Account> &account, BudgetClass category, size_t bytes) noexcept; + static void release_allocation(const std::shared_ptr<Account> &account, BudgetClass category, + size_t bytes) noexcept; + static const void *allocation_domain(const std::shared_ptr<Account> &account) noexcept; + static bool + transfer_allocation(const std::shared_ptr<Account> &from, const std::shared_ptr<Account> &to, BudgetClass category, + size_t bytes, + const std::array<size_t, static_cast<size_t>(AllocationKind::kCount)> &physical) noexcept; + static void record_allocation(const std::shared_ptr<Account> &account, AllocationKind kind, size_t bytes, + bool acquired) noexcept; + std::shared_ptr<State> state_; +}; + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/resource_observation.cpp b/lib/crowdb-tree/src/btree/resource_observation.cpp new file mode 100644 index 000000000..fb8dd2e20 --- /dev/null +++ b/lib/crowdb-tree/src/btree/resource_observation.cpp @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "resource_observation.h" + +#include <cassert> + +namespace crowdb::tree +{ +ResourceObservation::ResourceObservation(const std::shared_ptr<detail::AllocationReservation> &budget, + TreeResourceMetric metric, uint64_t bytes) noexcept + : metrics_(budget ? budget->resource_metrics() : nullptr), + metric_(metric) +{ + begin(bytes); +} + +ResourceObservation::~ResourceObservation() +{ + reset(); +} + +void ResourceObservation::begin(uint64_t bytes) noexcept +{ + assert(bytes_.load(std::memory_order_relaxed) == 0); + if (metrics_ && bytes != 0) { + const bool valid = metrics_->add(metric_, bytes); + assert(valid); + if (valid) { + bytes_.store(bytes, std::memory_order_release); + } + } +} + +void ResourceObservation::reset() noexcept +{ + const auto bytes = bytes_.exchange(0, std::memory_order_acq_rel); + if (bytes != 0) { + const bool valid = metrics_->remove(metric_, bytes); + assert(valid); + (void)valid; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/resource_observation.h b/lib/crowdb-tree/src/btree/resource_observation.h new file mode 100644 index 000000000..c69274575 --- /dev/null +++ b/lib/crowdb-tree/src/btree/resource_observation.h @@ -0,0 +1,29 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "tree_metrics.h" + +namespace crowdb::tree +{ +// Embedded in the actual buffer/operation owner. Independent of the Tree, +// and safe to finish once from a callback or final owner destruction. +class ResourceObservation +{ + public: + ResourceObservation(const std::shared_ptr<detail::AllocationReservation> &budget, TreeResourceMetric metric, + uint64_t bytes = 0) noexcept; + ~ResourceObservation(); + ResourceObservation(const ResourceObservation &) = delete; + ResourceObservation &operator=(const ResourceObservation &) = delete; + // Called before making the owner visible to a completion callback. + void begin(uint64_t bytes) noexcept; + void reset() noexcept; + + private: + std::shared_ptr<TreeResourceMetrics> metrics_; + TreeResourceMetric metric_; + std::atomic<uint64_t> bytes_{0}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/retired_shared.h b/lib/crowdb-tree/src/btree/retired_shared.h new file mode 100644 index 000000000..0478b4f9b --- /dev/null +++ b/lib/crowdb-tree/src/btree/retired_shared.h @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "retirement_queue.h" + +#include <stdexcept> + +namespace crowdb::tree::detail +{ +template <class T> struct RetiredObject : RetirementNode +{ + std::shared_ptr<RetirementQueue> queue; + std::shared_ptr<AllocationBudget> budget; + alignas(T) std::byte storage[sizeof(T)]; + + T *value() noexcept + { + return std::launder(reinterpret_cast<T *>(storage)); + } + + static bool destroy(RetirementNode *entry) noexcept + { + auto *self = static_cast<RetiredObject *>(entry); + if (!RetirementTraits<T>::release(*self->value())) { + return false; + } + std::destroy_at(self->value()); + MetadataAllocator<RetiredObject> allocator(self->budget); + std::destroy_at(self); + allocator.deallocate(self, 1); + return true; + } +}; + +template <class T> size_t retired_shared_allocation_bound() noexcept +{ + return sizeof(RetiredObject<T>) + shared_allocation_bound<T>(); +} + +template <class T, class... Args> +std::shared_ptr<T> retired_shared(const std::shared_ptr<AllocationBudget> &budget, + const std::shared_ptr<RetirementQueue> &queue, Args &&...args) +{ + if (!budget || !queue) { + throw std::invalid_argument("retired object requires charged storage and a retirement queue"); + } + using Node = RetiredObject<T>; + MetadataAllocator<Node> allocator(budget); + auto *node = allocator.allocate(1); + std::construct_at(node); + node->budget = budget; + node->retirement_budget = budget; + if constexpr (RetirementTraits<T>::observed) { + node->observation.metrics = budget->retirement_metrics(); + } + node->observation.resources = budget->resource_metrics(); + node->observation.bytes = sizeof(Node); + node->queue = queue; + node->dispose = &Node::destroy; + try { + std::construct_at(node->value(), std::forward<Args>(args)...); + if constexpr (requires(T &value) { value.retirement.bind(&node->observation); }) { + node->value()->retirement.bind(&node->observation); + } + } + catch (...) { + std::destroy_at(node); + allocator.deallocate(node, 1); + throw; + } + // If the independent control allocation fails, shared_ptr invokes this + // deleter too. The constructed object is still retired without allocating. + auto dispose = [node](T *) noexcept { node->queue->retire(node); }; + return {node->value(), dispose, MetadataAllocator<T>(budget)}; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/retirement_queue.cpp b/lib/crowdb-tree/src/btree/retirement_queue.cpp new file mode 100644 index 000000000..253390afe --- /dev/null +++ b/lib/crowdb-tree/src/btree/retirement_queue.cpp @@ -0,0 +1,211 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "retirement_queue.h" + +#include "tree_metrics.h" + +#include <cassert> + +namespace crowdb::tree::detail +{ +void RetirementObservation::start() noexcept +{ + if ((!metrics && !resources) || started_at.load(std::memory_order_relaxed) != 0) { + return; + } + const auto now = tree_metrics_now_ns(); + uint64_t before = 0; + if (started_at.compare_exchange_strong(before, now, std::memory_order_release, std::memory_order_relaxed)) { + // Every caller still owns T, so its last-owner disposal cannot race + // this begin. Competing logical retirement markers collapse to one. + if (metrics) { + age_slot = metrics->begin(TreeStage::MemoryRetireWait, now); + record_work(metrics, TreeWorkCounter::LogicalRetirementBytes, bytes, TreeWorkCause::Allocation); + } + if (resources) { + const auto added = resources->add(TreeResourceMetric::RetiredBytes, bytes); + assert(added); + (void)added; + } + } +} + +Status RetirementQueue::create(FlushRuntime &runtime, std::shared_ptr<FlushRuntime::Lane> lane, + const std::shared_ptr<AllocationBudget> &budget, std::shared_ptr<RetirementQueue> *out) +try { + if (!lane || !budget || out == nullptr || *out) { + return Status::invalid_argument(); + } + auto queue = std::allocate_shared<RetirementQueue>(MetadataAllocator<RetirementQueue>(budget)); + auto status = runtime.reserve_progress(std::move(lane), &run, queue.get(), budget, &queue->progress_); + if (!status.ok()) { + return status; + } + *out = std::move(queue); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +size_t RetirementQueue::allocation_bound() noexcept +{ + return shared_allocation_bound<RetirementQueue>() + FlushRuntime::progress_allocation_bound(); +} + +RetirementQueue::~RetirementQueue() +{ + // Binding teardown closes the registration while its executor is alive. + assert(!progress_ || state_.load(std::memory_order_acquire) == 2); + assert(incoming_.load(std::memory_order_relaxed) == nullptr); +} + +void RetirementQueue::retire(RetirementNode *node) noexcept +{ + assert(node != nullptr && node->dispose != nullptr); + auto keep = weak_from_this().lock(); + assert(keep); + if (node->retirement_budget) { + node->retirement_completion = node->retirement_budget->retirement_started(); + } + node->observation.start(); + enqueued_.fetch_add(1, std::memory_order_relaxed); + auto *head = incoming_.load(std::memory_order_relaxed); + do { + node->next = head; + } while (!incoming_.compare_exchange_weak(head, node, std::memory_order_release, std::memory_order_relaxed)); + const auto state = state_.load(std::memory_order_acquire); + if (state == 0) { + progress_->request(keep); + } + else if (state == 2) { + drain_host(); + } +} + +bool RetirementQueue::pending() const noexcept +{ + return incoming_.load(std::memory_order_acquire) != nullptr; +} + +RetirementQueue::Step RetirementQueue::step(size_t limit) noexcept +{ + if (consumer_.test_and_set(std::memory_order_acquire)) { + return Step::kBusy; + } + size_t visited = 0; + while (visited < limit) { + auto *node = incoming_.load(std::memory_order_acquire); + while (node != nullptr && !incoming_.compare_exchange_weak(node, node->next, std::memory_order_acq_rel, + std::memory_order_acquire)) { + } + if (node == nullptr) { + break; + } + ++visited; + auto accounting = node->retirement_completion; + auto metrics = node->observation.metrics; + auto resources = node->observation.resources; + const auto bytes = node->observation.bytes; + const auto started = node->observation.started_at.load(std::memory_order_acquire); + const auto age_slot = node->observation.age_slot; + if (node->dispose(node)) { + if (metrics) { + metrics->complete(TreeStage::MemoryRetireWait, TreeOutcome::Success, tree_metrics_now_ns() - started, + age_slot); + } + metrics.reset(); + if (resources) { + const auto removed = resources->remove(TreeResourceMetric::RetiredBytes, bytes); + assert(removed); + (void)removed; + } + resources.reset(); + freed_.fetch_add(1, std::memory_order_relaxed); + if (accounting.finish != nullptr) { + accounting.finish(accounting.owner.get()); + } + } + else { + auto *head = incoming_.load(std::memory_order_relaxed); + do { + node->next = head; + } while ( + !incoming_.compare_exchange_weak(head, node, std::memory_order_release, std::memory_order_relaxed)); + } + } + auto maximum = max_step_nodes_.load(std::memory_order_relaxed); + while (maximum < visited && !max_step_nodes_.compare_exchange_weak(maximum, visited, std::memory_order_relaxed)) { + } + consumer_.clear(std::memory_order_release); + return visited != 0 ? Step::kWork : Step::kEmpty; +} + +void RetirementQueue::run(void *context) noexcept +{ + auto &queue = *static_cast<RetirementQueue *>(context); + queue.cpu_steps_.fetch_add(1, std::memory_order_relaxed); + (void)queue.step(kNodesPerStep); + if (queue.pending() && queue.state_.load(std::memory_order_acquire) == 0) { + queue.progress_->request(queue.weak_from_this().lock()); + } +} + +void RetirementQueue::drain_host() noexcept +{ + struct Trampoline + { + RetirementQueue *head = nullptr; + bool active = false; + }; + + static thread_local Trampoline local; + if (fallback_queued_.exchange(true, std::memory_order_acq_rel)) { + return; + } + fallback_owner_ = weak_from_this().lock(); + assert(fallback_owner_); + fallback_next_ = local.head; + local.head = this; + if (local.active) { + return; + } + local.active = true; + while (local.head != nullptr) { + auto *queue = local.head; + local.head = queue->fallback_next_; + auto keep = std::move(queue->fallback_owner_); + assert(keep); + while (queue->step(kNodesPerStep) == Step::kWork) { + } + queue->fallback_queued_.exchange(false, std::memory_order_acq_rel); + // An arrival that saw the preceding queued flag relies on this + // recheck. A later arrival owns another trampoline registration. + if (queue->pending()) { + queue->drain_host(); + } + } + local.active = false; +} + +void RetirementQueue::close_and_drain() noexcept +{ + unsigned expected = 0; + if (!state_.compare_exchange_strong(expected, 1, std::memory_order_acq_rel)) { + assert(expected == 2 && "retirement close belongs to one host boundary"); + return; + } + progress_->close_and_drain(); + drain_host(); + state_.store(2, std::memory_order_release); + drain_host(); +} + +RetirementQueue::Snapshot RetirementQueue::snapshot() const noexcept +{ + return {.enqueued = enqueued_.load(std::memory_order_relaxed), + .freed = freed_.load(std::memory_order_relaxed), + .cpu_steps = cpu_steps_.load(std::memory_order_relaxed), + .max_step_nodes = max_step_nodes_.load(std::memory_order_relaxed)}; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/retirement_queue.h b/lib/crowdb-tree/src/btree/retirement_queue.h new file mode 100644 index 000000000..356b55e04 --- /dev/null +++ b/lib/crowdb-tree/src/btree/retirement_queue.h @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_bound.h" +#include "btree/flush_runtime.h" + +namespace crowdb::tree::detail +{ +class RetirementQueue; + +class RetirementQueue : public std::enable_shared_from_this<RetirementQueue> +{ + public: + static constexpr size_t kNodesPerStep = 64; + + struct Snapshot + { + size_t enqueued; + size_t freed; + size_t cpu_steps; + size_t max_step_nodes; + }; + + static Status create(FlushRuntime &runtime, std::shared_ptr<FlushRuntime::Lane> lane, + const std::shared_ptr<AllocationBudget> &budget, std::shared_ptr<RetirementQueue> *out); + [[nodiscard]] static size_t allocation_bound() noexcept; + ~RetirementQueue(); + void retire(RetirementNode *node) noexcept; + // Host-only, before the runtime is destroyed. This joins the reserved + // registration. Late owners keep only this closed queue and allocation + // accounts, and free iteratively without touching an executor. + void close_and_drain() noexcept; + [[nodiscard]] Snapshot snapshot() const noexcept; + + private: + static void run(void *context) noexcept; + enum class Step : uint8_t { kBusy, kEmpty, kWork }; + Step step(size_t limit) noexcept; + void drain_host() noexcept; + [[nodiscard]] bool pending() const noexcept; + + std::shared_ptr<FlushRuntime::Progress> progress_; + std::atomic<RetirementNode *> incoming_{nullptr}; + std::atomic_flag consumer_ = ATOMIC_FLAG_INIT; + std::atomic<bool> fallback_queued_{false}; + RetirementQueue *fallback_next_ = nullptr; + std::shared_ptr<RetirementQueue> fallback_owner_; + // 0 accepts CPU requests; 1 routes arrivals to the closing host; 2 uses + // the caller's iterative fallback after the executor is detached. + std::atomic<unsigned> state_{0}; + std::atomic<size_t> enqueued_{0}, freed_{0}, cpu_steps_{0}, max_step_nodes_{0}; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/retirement_traits.h b/lib/crowdb-tree/src/btree/retirement_traits.h new file mode 100644 index 000000000..ce963ae33 --- /dev/null +++ b/lib/crowdb-tree/src/btree/retirement_traits.h @@ -0,0 +1,24 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <concepts> + +namespace crowdb::tree::detail +{ +template <class T> struct RetirementTraits +{ + static constexpr bool enabled = requires(T &value) { + { value.retire_step() } noexcept -> std::same_as<bool>; + }; + static constexpr bool observed = requires { T::observe_retirement; }; + + static bool release(T &value) noexcept + { + if constexpr (enabled) { + return value.retire_step(); + } + return true; + } +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/runtime_range_native.cpp b/lib/crowdb-tree/src/btree/runtime_range_native.cpp new file mode 100644 index 000000000..9e7f4c8a2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/runtime_range_native.cpp @@ -0,0 +1,129 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "native_import.h" +#include "runtime_range_rebuild.h" +#include "snapshot/published_native.h" +#include "tree_core_access.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +Status measure_native(PublishedNativeCursor &cursor, NativeImportShape *out) +{ + NativeImportShape shape; + size_t overflow_frames = 0; + bool complete = false; + while (!complete) { + std::optional<NativeFrameView> frame; + auto status = cursor.next(&frame, &complete); + if (!status.ok()) { + return status; + } + if (!frame) { + continue; + } + if (shape.frames++ == NativeImportLimits{}.max_frames) { + return Status::resource_exhausted("range native frame limit"); + } + shape.max_frame_bytes = std::max(shape.max_frame_bytes, frame->bytes.size()); + const auto type = frame_page_type(frame->bytes.data()); + if (type == page_type::kOverflowFrame) { + ++overflow_frames; + continue; + } + ++shape.primary_pages; + shape.max_key_bytes = std::max(shape.max_key_bytes, frame->bytes.size()); + if (type == page_type::kLeafBase) { + ++shape.leaf_pages; + const LeafFrameView leaf(frame->bytes.data(), frame->bytes.size()); + const size_t count = static_cast<size_t>(leaf.count()) + leaf.delta_count(); + shape.max_leaf_entries = std::max(shape.max_leaf_entries, count); + for (size_t i = 0; i < count; ++i) { + const CellView cell(i < leaf.count() ? leaf.cell(static_cast<uint32_t>(i)) + : leaf.delta_cell(static_cast<uint32_t>(i - leaf.count()))); + shape.stored_references += static_cast<size_t>(cell.is_overflow()); + } + } + } + // Each stored reference can visit at most all emitted overflow frames. + // Streams beyond the import's configured work limit are rejected later by + // its graph validator; no extra dependency allocation can occur past it. + const auto dependencies = detail::AllocationBound().add_product(shape.stored_references, overflow_frames).value(); + if (!dependencies) { + return Status::resource_exhausted("range dependency bound overflow"); + } + shape.dependency_steps = std::min(*dependencies, NativeImportLimits{}.max_dependency_steps); + *out = shape; + return cursor.rewind(); +} + +uint64_t native_leaf_keys(const NativeFrameView &frame, const std::shared_ptr<detail::AllocationReservation> &credit) +{ + if (frame_page_type(frame.bytes.data()) != page_type::kLeafBase) { + return 0; + } + const LeafFrameView leaf(frame.bytes.data(), frame.bytes.size()); + detail::ReservedVector<Slice> keys{detail::ReservationAllocator<Slice>(credit)}; + keys.reserve(static_cast<size_t>(leaf.count()) + leaf.delta_count()); + for (uint32_t i = 0; i < leaf.count(); ++i) { + keys.push_back(leaf.key(i)); + } + for (uint32_t i = 0; i < leaf.delta_count(); ++i) { + keys.push_back(leaf.delta_key(i)); + } + std::sort(keys.begin(), keys.end()); + return std::unique(keys.begin(), keys.end()) - keys.begin(); +} +} // namespace + +Status copy_native_range(Crowdbtree &source, TreeCore &destination, RuntimeRangeImport &staging) +{ + std::shared_ptr<PublishedNativeCursor> cursor; + auto status = PublishedNativeCursor::create(TreeCoreAccess::get(source)->shared_from_this(), {}, &cursor); + if (!status.ok()) { + return status; + } + NativeImportShape shape; + status = measure_native(*cursor, &shape); + if (!status.ok()) { + return status; + } + shape.max_frame_bytes = std::max<size_t>(shape.max_frame_bytes, destination.config().frame_bytes); + status = reserve_range_shape(destination, shape, false, staging); + if (!status.ok()) { + return status; + } + const auto info = cursor->info(); + staging.catalog = detail::reserved_shared<ReadablePageCatalog>(staging.credit, staging.credit); + staging.catalog->root_page_id = info.root_page_id; + staging.catalog->coverage = info.at_slot; + staging.catalog->generation = 1; + staging.catalog->next_page_id = info.next_page_id; + bool complete = false; + while (!complete) { + std::optional<NativeFrameView> frame; + status = cursor->next(&frame, &complete); + if (!status.ok()) { + return status; + } + if (!frame) { + continue; + } + status = append_native_frame(*staging.catalog, frame->page_id, frame->bytes, destination.config(), + destination.load_limits(), {}, staging.credit); + if (!status.ok()) { + return status; + } + staging.stats.entries_examined += native_leaf_keys(*frame, staging.credit); + ++staging.stats.pages_rebuilt; + } + if (staging.catalog->frames.size() != info.frame_count) { + return Status::corruption("range rebuild native stream count mismatch"); + } + staging.stats.entries_emitted = staging.stats.entries_examined; + return attach_native_dependencies(*staging.catalog, {}, staging.credit); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/runtime_range_portable.cpp b/lib/crowdb-tree/src/btree/runtime_range_portable.cpp new file mode 100644 index 000000000..4ad384295 --- /dev/null +++ b/lib/crowdb-tree/src/btree/runtime_range_portable.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "portable_import.h" +#include "portable_import_plan.h" +#include "runtime_range_rebuild.h" +#include "snapshot/published_snapshot_entries.h" +#include "snapshot/recovered_state.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +Status reserve_range(const std::shared_ptr<PublishedSnapshotContext> &snapshot, TreeCore &destination, + const KeyRange &range, RuntimeRangeImport &staging) +{ + std::shared_ptr<PublishedSnapshotEntries> cursor; + auto status = PublishedSnapshotEntries::create(snapshot, &cursor, range); + if (!status.ok()) { + return status; + } + PortableImportPlan plan(destination.config(), destination.load_limits()); + bool complete = false; + while (!complete) { + std::optional<PublishedScanEntry> entry; + status = cursor->next(&entry, &complete); + if (!status.ok()) { + return status; + } + if (entry) { + status = plan.append(entry->key.size(), entry->cell.value.size(), entry->cell.tombstone()); + if (!status.ok()) { + return status; + } + } + } + const auto shape = plan.finish(); + if (!shape) { + return Status::resource_exhausted("range import shape overflow"); + } + return reserve_range_shape(destination, *shape, true, staging); +} +} // namespace + +Status copy_portable_range(TreeCore &source, TreeCore &destination, const KeyRange &range, RuntimeRangeImport &staging) +{ + const auto frontier = source.contiguous_slot(); + std::shared_ptr<PublishedSnapshotContext> snapshot; + auto status = source.capture_snapshot(&snapshot); + if (!status.ok()) { + return status; + } + if (snapshot->view()->coverage < frontier) { + snapshot.reset(); + status = source.flush(); + if (!status.ok()) { + return status; + } + status = source.capture_snapshot(&snapshot); + if (!status.ok()) { + return status; + } + } + return copy_snapshot_range(snapshot, destination, range, staging); +} + +Status copy_snapshot_range(const std::shared_ptr<PublishedSnapshotContext> &snapshot, TreeCore &destination, + const KeyRange &range, RuntimeRangeImport &staging) +{ + auto status = reserve_range(snapshot, destination, range, staging); + if (!status.ok()) { + return status; + } + std::shared_ptr<PublishedSnapshotEntries> cursor; + status = PublishedSnapshotEntries::create(snapshot, &cursor, range); + if (!status.ok()) { + return status; + } + PortableImportBuilder builder(destination.config(), destination.load_limits(), snapshot->view()->coverage, + staging.credit); + detail::ReservedVector<uint8_t, AllocationKind::kScratch> cell{ + detail::ReservationAllocator<uint8_t, AllocationKind::kScratch>(staging.credit)}; + bool complete = false; + while (!complete) { + std::optional<PublishedScanEntry> entry; + status = cursor->next(&entry, &complete); + if (!status.ok()) { + return status; + } + if (!entry) { + continue; + } + cell.resize(kCellHeaderSize + entry->cell.value.size()); + frame_put_u64(cell.data(), 0, entry->cell.slot); + cell[sizeof(uint64_t)] = entry->cell.tombstone() ? kFlagTombstone : 0; + if (!entry->cell.value.empty()) { + std::copy_n(reinterpret_cast<const uint8_t *>(entry->cell.value.data()), entry->cell.value.size(), + cell.data() + kCellHeaderSize); + } + status = builder.append(Slice(entry->key), Slice(cell.data(), cell.size())); + if (!status.ok()) { + return status; + } + ++staging.stats.entries_examined; + ++staging.stats.entries_emitted; + } + staging.stats.subtrees_skipped = cursor->skipped_subtrees(); + status = builder.finish(&staging.catalog); + if (status.ok()) { + staging.stats.pages_rebuilt = staging.catalog->frames.size(); + } + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/runtime_range_rebuild.cpp b/lib/crowdb-tree/src/btree/runtime_range_rebuild.cpp new file mode 100644 index 000000000..9a9d778f7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/runtime_range_rebuild.cpp @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "runtime_range_rebuild.h" + +#include "portable_import.h" +#include "snapshot/recovered_state.h" +#include "tree_core_access.h" + +namespace crowdb::tree +{ +Status reserve_range_shape(TreeCore &destination, const NativeImportShape &shape, bool portable, + RuntimeRangeImport &staging) +{ + const auto loads = resident_import_load_limits(destination.load_limits(), shape.max_frame_bytes); + const auto depth = std::min<size_t>(64, shape.primary_pages); + auto bound = + detail::AllocationBound() + .add(native_import_allocation_bound(shape)) + .add(RecoveredStateOperation::allocation_bound(shape.frames, shape.primary_pages, shape.max_key_bytes)) + .add(AsyncStorePageLoader::primary_reserve_bound(loads, depth, destination.binding()->cpu()->worker_count(), + 1)) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add(TreeOperationLifetime::ticket_allocation_bound()); + if (portable) { + bound.add(PortableImportBuilder::allocation_bound(shape)); + } + if (!bound.value()) { + return Status::resource_exhausted("range import allocation bound overflow"); + } + return destination.prepare_import_credit(*bound.value(), &staging.credit); +} + +Status rebuild_range_runtime(Crowdbtree &source, const KeyRange &range, Config destination_options, + std::unique_ptr<Crowdbtree> *out, RangeRebuildStats *stats) +try { + auto *source_core = TreeCoreAccess::get(source); + if (source_core == nullptr) { + return Status::not_supported("runtime range rebuild requires a runtime source"); + } + if (out == nullptr) { + return Status::invalid_argument("range rebuild needs an output"); + } + // A new binding supplies the destination's independent account. Neither + // its page directory nor any mutable source catalog is inherited. + destination_options.key_range = range; + std::unique_ptr<Crowdbtree> destination; + auto status = Crowdbtree::open(destination_options, &destination); + if (!status.ok()) { + return status; + } + auto *core = TreeCoreAccess::get(*destination); + RuntimeRangeImport staging; + if (!range.is_bounded() && !source_core->config().key_range.is_bounded() && + source_core->config().frame_bytes == destination_options.frame_bytes) { + status = copy_native_range(source, *core, staging); + } + else { + status = copy_portable_range(*source_core, *core, range, staging); + } + if (!status.ok()) { + return status; + } + status = core->install_readable(std::move(staging.catalog), staging.credit); + if (!status.ok()) { + return status; + } + status = destination->snapshot(); + if (!status.ok()) { + return status; + } + if (stats != nullptr) { + *stats = staging.stats; + } + *out = std::move(destination); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("range rebuild staging allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/runtime_range_rebuild.h b/lib/crowdb-tree/src/btree/runtime_range_rebuild.h new file mode 100644 index 000000000..d1b7e38eb --- /dev/null +++ b/lib/crowdb-tree/src/btree/runtime_range_rebuild.h @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/btree/range_rebuild.h" +#include "tree_core.h" + +namespace crowdb::tree +{ +struct RuntimeRangeImport +{ + ~RuntimeRangeImport() + { + if (credit) { + credit->seal(); + } + } + + std::shared_ptr<detail::AllocationReservation> credit; + std::shared_ptr<ReadablePageCatalog> catalog; + RangeRebuildStats stats; +}; + +struct NativeImportShape; +Status reserve_range_shape(TreeCore &destination, const NativeImportShape &shape, bool portable, + RuntimeRangeImport &staging); +Status copy_snapshot_range(const std::shared_ptr<PublishedSnapshotContext> &snapshot, TreeCore &destination, + const KeyRange &range, RuntimeRangeImport &staging); +Status copy_native_range(Crowdbtree &source, TreeCore &destination, RuntimeRangeImport &staging); +Status copy_portable_range(TreeCore &source, TreeCore &destination, const KeyRange &range, RuntimeRangeImport &staging); +Status rebuild_range_runtime(Crowdbtree &source, const KeyRange &range, Config destination_options, + std::unique_ptr<Crowdbtree> *out, RangeRebuildStats *stats); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/scan_output.h b/lib/crowdb-tree/src/btree/scan_output.h new file mode 100644 index 000000000..11be24f7b --- /dev/null +++ b/lib/crowdb-tree/src/btree/scan_output.h @@ -0,0 +1,24 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_bound.h" +#include "crowdb-tree/btree/scan_packed.h" + +#include <algorithm> + +namespace crowdb::tree::detail +{ +inline void reserve_scan_output(ScanPackedBuf &packed, size_t entry_bytes, size_t initial_capacity) +{ + const auto required = AllocationBound(packed.size()).add(entry_bytes).add(17).value(); + if (!required) { + throw std::bad_alloc(); + } + if (*required <= packed.capacity()) { + return; + } + const auto doubled = AllocationBound().add_product(packed.capacity(), 2).value(); + packed.reserve(std::max({*required, initial_capacity, doubled.value_or(*required)})); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/snapshot_changes.cpp b/lib/crowdb-tree/src/btree/snapshot_changes.cpp new file mode 100644 index 000000000..9c89131bc --- /dev/null +++ b/lib/crowdb-tree/src/btree/snapshot_changes.cpp @@ -0,0 +1,181 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot_changes.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <stdexcept> + +namespace crowdb::tree +{ +struct SnapshotChanges::Node +{ + uint64_t first = 0; + uint64_t last = 0; + uint64_t bit = 0; + size_t count = 1; + + bool retire_step() noexcept + { + return true; + } +}; + +struct SnapshotChanges::Leaf : Node +{ + explicit Leaf(SnapshotPageChange value) : change(std::move(value)) + { + first = last = change.page_id; + } + + SnapshotPageChange change; +}; + +struct SnapshotChanges::Branch : Node +{ + Branch(uint64_t split, Owner left, Owner right) : children{std::move(left), std::move(right)} + { + bit = split; + first = children[0]->first; + last = children[1]->last; + count = children[0]->count + children[1]->count; + } + + std::array<Owner, 2> children; +}; + +SnapshotChanges::Owner SnapshotChanges::insert(const Owner &node, const std::shared_ptr<const Leaf> &leaf, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + if (!node) { + return leaf; + } + const auto differing = std::bit_floor(node->first ^ leaf->first); + if (differing > node->bit) { + const bool right = (leaf->first & differing) != 0; + return detail::reserved_shared<Branch>(metadata, differing, right ? node : leaf, right ? leaf : node); + } + if (node->bit == 0) { + return leaf; + } + const auto &branch = static_cast<const Branch &>(*node); + const bool right = (leaf->first & node->bit) != 0; + return detail::reserved_shared<Branch>(metadata, node->bit, + right ? branch.children[0] : insert(branch.children[0], leaf, metadata), + right ? insert(branch.children[1], leaf, metadata) : branch.children[1]); +} + +SnapshotChanges SnapshotChanges::replace(SnapshotPageChange change, + const std::shared_ptr<detail::AllocationReservation> &metadata) const +{ + if (change.page_id >= kPageLimit) { + throw std::invalid_argument("snapshot change exceeds the PID namespace"); + } + if (const auto *previous = get(change.page_id)) { + change.before = previous->before; + change.effect = change.effect | previous->effect; + } + auto leaf = detail::reserved_shared<Leaf>(metadata, std::move(change)); + SnapshotChanges result; + result.root_ = insert(root_, leaf, metadata); + return result; +} + +const SnapshotPageChange *SnapshotChanges::get(uint64_t pid) const noexcept +{ + const auto *node = root_.get(); + while (node != nullptr && node->bit != 0) { + node = static_cast<const Branch &>(*node).children[(pid & node->bit) != 0 ? 1U : 0U].get(); + } + return node != nullptr && node->first == pid ? &static_cast<const Leaf &>(*node).change : nullptr; +} + +const SnapshotChanges::Leaf *SnapshotChanges::lower_bound(const Owner &node, uint64_t pid) noexcept +{ + if (!node || node->last < pid) { + return nullptr; + } + if (node->bit == 0) { + return &static_cast<const Leaf &>(*node); + } + const auto &branch = static_cast<const Branch &>(*node); + if (const auto *left = lower_bound(branch.children[0], pid)) { + return left; + } + return lower_bound(branch.children[1], pid); +} + +bool SnapshotChanges::visit_from(uint64_t &next_pid, size_t max_pages, void *context, Visitor visitor) const +{ + for (size_t i = 0; i < max_pages; ++i) { + const auto *leaf = lower_bound(root_, next_pid); + if (leaf == nullptr) { + next_pid = kPageLimit; + return true; + } + visitor(context, leaf->change); + next_pid = leaf->first + 1; + } + return !root_ || next_pid > root_->last; +} + +size_t SnapshotChanges::size() const noexcept +{ + return root_ ? root_->count : 0; +} + +bool SnapshotChanges::empty() const noexcept +{ + return !root_; +} + +bool SnapshotChanges::same_root(const SnapshotChanges &other) const noexcept +{ + return root_ == other.root_; +} + +size_t SnapshotChanges::change_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<Leaf>() + (30 * detail::shared_allocation_bound<Branch>()); +} + +std::optional<size_t> SnapshotChanges::initial_allocation_bound(size_t pages) noexcept +{ + return detail::AllocationBound() + .add_product(pages, detail::shared_allocation_bound<Leaf>() + detail::shared_allocation_bound<Branch>() + + sizeof(SnapshotPageChange)) + .value(); +} + +SnapshotChanges::Owner SnapshotChanges::build(std::span<const SnapshotPageChange> changes, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + if (changes.size() == 1) { + return detail::reserved_shared<Leaf>(metadata, changes.front()); + } + const auto bit = std::bit_floor(changes.front().page_id ^ changes.back().page_id); + const auto middle = std::partition_point(changes.begin(), changes.end(), + [bit](const auto &change) { return (change.page_id & bit) == 0; }); + const auto count = static_cast<size_t>(middle - changes.begin()); + return detail::reserved_shared<Branch>(metadata, bit, build(changes.first(count), metadata), + build(changes.subspan(count), metadata)); +} + +SnapshotChanges SnapshotChanges::from_sorted(std::span<const SnapshotPageChange> changes, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + uint64_t next_pid = 0; + for (const auto &change : changes) { + if (change.page_id < next_pid || change.page_id >= kPageLimit) { + throw std::invalid_argument("initial snapshot changes must have unique increasing PIDs"); + } + next_pid = change.page_id + 1; + } + SnapshotChanges result; + if (!changes.empty()) { + result.root_ = build(changes, metadata); + } + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/snapshot_changes.h b/lib/crowdb-tree/src/btree/snapshot_changes.h new file mode 100644 index 000000000..c0e1cecc8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/snapshot_changes.h @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "page_key_range.h" + +namespace crowdb::tree +{ +// Arbitrary replacements require a complete comparison. Admitted handoff +// adds cells above its old frontier; maintenance changes source topology +// without changing the logical key/cell view. A pending PID accumulates bits. +enum class SnapshotChangeEffect : uint8_t { FullContent = 1, AcceptedPrefix = 2, SourceStructure = 4 }; + +class SnapshotChangeEffects +{ + public: + constexpr SnapshotChangeEffects(SnapshotChangeEffect effect = SnapshotChangeEffect::FullContent) noexcept + : bits_(static_cast<uint8_t>(effect)) + { + } + + [[nodiscard]] constexpr bool contains(SnapshotChangeEffect effect) const noexcept + { + return (bits_ & static_cast<uint8_t>(effect)) != 0; + } + + constexpr bool operator==(const SnapshotChangeEffects &) const noexcept = default; + + [[nodiscard]] friend constexpr SnapshotChangeEffects operator|(SnapshotChangeEffects left, + SnapshotChangeEffects right) noexcept + { + left.bits_ |= right.bits_; + return left; + } + + private: + uint8_t bits_; +}; + +[[nodiscard]] constexpr SnapshotChangeEffects operator|(SnapshotChangeEffect left, SnapshotChangeEffect right) noexcept +{ + return SnapshotChangeEffects(left) | SnapshotChangeEffects(right); +} + +// Compact logical identities never own a PublishedPage, source or value buffer. +struct SnapshotPageIdentity +{ + PageKeyRange range{}; + uint64_t content = 0; + uint64_t route = 0; + uint64_t revision = 0; + uint64_t right_sibling = UINT64_MAX; + uint32_t level = 0; + bool present = false; +}; + +struct SnapshotPageChange +{ + uint64_t page_id = 0; + uint64_t publication_seq = 0; + SnapshotPageIdentity before{}; + SnapshotPageIdentity after{}; + SnapshotChangeEffects effect = SnapshotChangeEffect::FullContent; +}; + +// Persistent compressed binary radix tree. Capture shares one root, and an +// update copies at most one branch per PID bit. Deleted pages remain explicit +// until the captured changes commit; no payload or old mapping is retained. +class SnapshotChanges +{ + struct Node; + struct Leaf; + struct Branch; + using Owner = std::shared_ptr<const Node>; + + public: + static constexpr uint64_t kPageLimit = uint64_t{1} << 30; + [[nodiscard]] SnapshotChanges replace(SnapshotPageChange change, + const std::shared_ptr<detail::AllocationReservation> &metadata = {}) const; + [[nodiscard]] const SnapshotPageChange *get(uint64_t pid) const noexcept; + [[nodiscard]] size_t size() const noexcept; + [[nodiscard]] bool empty() const noexcept; + [[nodiscard]] bool same_root(const SnapshotChanges &other) const noexcept; + [[nodiscard]] static size_t change_allocation_bound() noexcept; + [[nodiscard]] static std::optional<size_t> initial_allocation_bound(size_t pages) noexcept; + [[nodiscard]] static SnapshotChanges + from_sorted(std::span<const SnapshotPageChange> changes, + const std::shared_ptr<detail::AllocationReservation> &metadata = {}); + + using Visitor = void (*)(void *, const SnapshotPageChange &); + // Resume at next_pid; each step returns one selected PID, never scans gaps. + [[nodiscard]] bool visit_from(uint64_t &next_pid, size_t max_pages, void *context, Visitor visitor) const; + + private: + static Owner insert(const Owner &node, const std::shared_ptr<const Leaf> &leaf, + const std::shared_ptr<detail::AllocationReservation> &metadata); + static const Leaf *lower_bound(const Owner &node, uint64_t pid) noexcept; + static Owner build(std::span<const SnapshotPageChange> changes, + const std::shared_ptr<detail::AllocationReservation> &metadata); + Owner root_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_admission.cpp b/lib/crowdb-tree/src/btree/source_admission.cpp new file mode 100644 index 000000000..830403d71 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_admission.cpp @@ -0,0 +1,687 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "published_state.h" +#include "source_admission_internal.h" + +#include <algorithm> +#include <utility> + +namespace crowdb::tree +{ +namespace +{ +std::atomic<uint64_t> next_source_admission_identity{1}; + +class AdminGuard +{ + public: + explicit AdminGuard(std::atomic_flag &gate) : gate_(gate), held_(!gate.test_and_set(std::memory_order_acquire)) + { + } + + ~AdminGuard() + { + if (held_) { + gate_.clear(std::memory_order_release); + } + } + + [[nodiscard]] bool held() const noexcept + { + return held_; + } + + private: + std::atomic_flag &gate_; + bool held_; +}; +} // namespace + +struct SourceAdmission::State +{ + explicit State(const std::shared_ptr<detail::AllocationReservation> &metadata) : epoch(metadata) + { + } + + struct Root : EpochManager::Deferred + { + Root(std::shared_ptr<const SourceCatalog> value, AllocationLease credit) + : allocation(std::move(credit)), + catalog(std::move(value)) + { + destroy = [](EpochManager::Deferred *entry) noexcept { Deleter{}(static_cast<Root *>(entry)); }; + } + + struct Deleter + { + void operator()(Root *value) const noexcept + { + if (value != nullptr) { + auto credit = std::move(value->allocation); + delete value; + } + } + }; + + using Owner = std::unique_ptr<Root, Deleter>; + + static Owner create(std::shared_ptr<const SourceCatalog> value, + const std::shared_ptr<detail::AllocationReservation> &budget) + { + AllocationLease credit(budget, sizeof(Root), AllocationKind::kMetadata); + Owner result(new Root(std::move(value), std::move(credit))); + return result; + } + + [[nodiscard]] static size_t allocation_bound() noexcept + { + return sizeof(Root); + } + + AllocationLease allocation; + std::shared_ptr<const SourceCatalog> catalog; + }; + + void publish(Root::Owner next) noexcept + { + auto *previous = root.load(std::memory_order_acquire); + const auto &before = *previous->catalog; + const auto &after = *next->catalog; + if (after.active != before.active) { + after.active->attach_source_stats(after.memory); + } + for (const auto &table : before.tables) { + if (std::find(after.tables.begin(), after.tables.end(), table) == after.tables.end()) { + table->detach_source_stats(); + } + } + previous = root.exchange(next.release(), std::memory_order_acq_rel); + previous->catalog->retirement.mark(); + epoch.defer(previous); + } + + bool matches(const std::shared_ptr<SourceCapture> &capture, uint64_t batch) const noexcept + { + return capture && capture == pending && capture->state_->owner_identity == identity && + capture->batch_id() == batch; + } + + std::shared_ptr<SourceTailDomain> tail_domain; + + const uint64_t identity = next_source_admission_identity.fetch_add(1, std::memory_order_relaxed); + mutable EpochManager epoch; + std::atomic<Root *> root{nullptr}; + std::atomic<bool> closed{false}; + std::atomic_flag admin = ATOMIC_FLAG_INIT; + uint64_t next_table = 2; + uint64_t next_batch = 1; + std::shared_ptr<SourceCapture> pending; + std::shared_ptr<Rotation> rotation; + Root::Owner pending_root; + Root::Owner covered_root; + SourceWriteOptions write_options; +}; + +struct SourceAdmission::Rotation::Data +{ + std::shared_ptr<const SourceCatalog> expected; + std::shared_ptr<SourceCatalog> next; + State::Root::Owner root; + uint64_t owner_identity = 0; + bool committed = false; +}; + +SourceAdmission::Rotation::Rotation(detail::ReservedUnique<Data> data) : data_(std::move(data)) +{ +} + +SourceAdmission::Rotation::~Rotation() = default; + +const std::shared_ptr<const SourceCatalog> &SourceAdmission::Rotation::expected() const noexcept +{ + return data_->expected; +} + +const std::shared_ptr<SourceCatalog> &SourceAdmission::Rotation::next() const noexcept +{ + return data_->next; +} + +std::optional<size_t> SourceAdmission::rotation_allocation_bound(size_t sources) noexcept +{ + using detail::shared_allocation_bound; + return detail::AllocationBound(shared_allocation_bound<Rotation>() + sizeof(Rotation::Data) + + State::Root::allocation_bound() + SourceCatalog::allocation_bound() + + shared_allocation_bound<MemTable>() + shared_allocation_bound<SourceWriteBudget>() + + EpochManager::infrastructure_allocation_bound() + + Node::alloc_size(ConcurrentSkipList::kMaxHeight, 0)) + .add_product(sources + 1, sizeof(std::shared_ptr<MemTable>)) + .value(); +} + +Status SourceAdmission::prepare_rotation(const std::shared_ptr<detail::AllocationReservation> &metadata, + const std::shared_ptr<const SourceCatalog> &expected, + std::shared_ptr<Rotation> *out) +try { + if (!metadata || !expected || out == nullptr || *out || + (state_->write_options.account && metadata->account() != state_->write_options.account)) { + return Status::invalid_argument("source rotation requires its owned metadata and expected catalog"); + } + AdminGuard admin(state_->admin); + if (!admin.held() || state_->closed.load(std::memory_order_acquire) || state_->pending || state_->rotation || + catalog() != expected) { + return Status::unavailable("source capture or administration prevents rotation"); + } + auto data = detail::reserved_unique<Rotation::Data>(metadata); + data->owner_identity = state_->identity; + data->expected = expected; + data->next = detail::reserved_shared<SourceCatalog>(metadata, metadata); + auto &next = *data->next; + next.memory = expected->memory; + next.tables.reserve(expected->tables.size() + 1); + next.tables.assign(expected->tables.begin(), expected->tables.end()); + next.floor = expected->floor; + next.generation = expected->generation; + next.serial = expected->serial + 1; + next.active = detail::reserved_shared<MemTable>(metadata, state_->next_table, nullptr, + expected->active_budget ? metadata : nullptr); + if (expected->active_budget) { + next.active_budget = detail::reserved_shared<SourceWriteBudget>(metadata, state_->write_options); + } + next.active->set_durable_floor(next.floor); + next.tables.push_back(next.active); + data->root = State::Root::create(data->next, metadata); + auto rotation = detail::reserved_construct<Rotation>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) Rotation(std::move(data)); }); + state_->rotation = rotation; + *out = std::move(rotation); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("source rotation allocation failed"); +} + +Status SourceAdmission::activate_rotation(const std::shared_ptr<Rotation> &rotation) +{ + AdminGuard admin(state_->admin); + if (!admin.held()) { + return Status::unavailable("source administration busy"); + } + if (!rotation || rotation->data_->owner_identity != state_->identity) { + return Status::invalid_argument("source rotation belongs to another owner"); + } + auto &data = *rotation->data_; + if (data.committed) { + return Status::Ok(); + } + if (state_->rotation != rotation || catalog() != data.expected) { + return Status::invalid_argument("source rotation lost its expected catalog"); + } + if (data.expected->active_budget && data.expected->active_budget->paused()) { + const auto bound = data.expected->active_budget->paused_bound(); + data.expected->active->set_sealed_admission_bound(bound.entries, bound.key_bytes); + } + data.expected->active->close(); + state_->publish(std::move(data.root)); + ++state_->next_table; + data.committed = true; + state_->rotation.reset(); + return Status::Ok(); +} + +Status SourceAdmission::abort_rotation(const std::shared_ptr<Rotation> &rotation) +{ + AdminGuard admin(state_->admin); + if (!admin.held()) { + return Status::unavailable("source administration busy"); + } + if (state_->rotation != rotation) { + return Status::invalid_argument("source rotation no longer pending"); + } + state_->rotation.reset(); + return Status::Ok(); +} + +struct SourceAdmission::Coverage::Data +{ + std::shared_ptr<const SourceCatalog> expected; + std::shared_ptr<SourceCatalog> next; + State::Root::Owner root; + uint64_t owner_identity = 0; + bool committed = false; +}; + +SourceAdmission::Coverage::Coverage(detail::ReservedUnique<Data> data) : data_(std::move(data)) +{ +} + +SourceAdmission::Coverage::~Coverage() = default; + +uint64_t SourceAdmission::identity() const noexcept +{ + return state_->identity; +} + +std::optional<size_t> SourceAdmission::coverage_allocation_bound(size_t sources) noexcept +{ + return detail::AllocationBound(detail::shared_allocation_bound<Coverage>() + sizeof(Coverage::Data) + + State::Root::allocation_bound() + SourceCatalog::allocation_bound()) + .add_product(sources, sizeof(std::shared_ptr<MemTable>)) + .value(); +} + +Status SourceAdmission::prepare_coverage(uint64_t frontier, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Coverage> *out) +try { + if (!metadata || out == nullptr || *out || + (state_->write_options.account && metadata->account() != state_->write_options.account)) { + return Status::invalid_argument("split coverage requires owned source credit and empty output"); + } + AdminGuard admin(state_->admin); + if (!admin.held() || state_->closed.load(std::memory_order_acquire) || state_->pending || state_->rotation) { + return Status::unavailable("source capture or administration prevents split coverage"); + } + const auto previous = catalog(); + if (frontier < previous->floor) { + return Status::invalid_argument("split coverage cannot move the source floor backwards"); + } + auto data = detail::reserved_unique<Coverage::Data>(metadata); + data->expected = previous; + data->owner_identity = state_->identity; + data->next = detail::reserved_shared<SourceCatalog>(metadata, metadata); + data->next->memory = previous->memory; + data->next->tables = previous->tables; + data->next->active = previous->active; + data->next->active_budget = previous->active_budget; + data->next->floor = frontier; + data->next->generation = previous->generation; + data->next->serial = previous->serial + 1; + data->root = State::Root::create(data->next, metadata); + *out = detail::reserved_construct<Coverage>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) Coverage(std::move(data)); }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("split coverage preparation failed"); +} + +Status SourceAdmission::install_coverage(const std::shared_ptr<Coverage> &prepared, const PublishedTreeState &published) +{ + AdminGuard admin(state_->admin); + if (!admin.held()) { + return Status::unavailable("source administration is busy"); + } + if (!prepared || prepared->data_->owner_identity != state_->identity) { + return Status::invalid_argument("split coverage belongs to another source owner"); + } + auto &data = *prepared->data_; + if (published.generation != data.next->generation || published.coverage != data.next->floor || !published.sources || + published.sources->inheritance || published.sources->floor != data.next->floor || + published.sources->tables.size() != data.next->tables.size() || + !std::equal(published.sources->tables.begin(), published.sources->tables.end(), data.next->tables.begin())) { + return Status::unavailable("split coverage has not atomically published " + "its local source membership"); + } + if (data.committed) { + return Status::Ok(); + } + if (state_->pending || state_->rotation || catalog() != data.expected) { + return Status::unavailable("source catalog changed before split coverage acknowledgement"); + } + data.next->active->set_durable_floor(data.next->floor); + state_->publish(std::move(data.root)); + data.committed = true; + data.expected.reset(); + return Status::Ok(); +} + +std::optional<size_t> SourceAdmission::allocation_bound(size_t max_threads) noexcept +{ + using detail::shared_allocation_bound; + return detail::AllocationBound( + sizeof(SourceAdmission) + sizeof(State) + State::Root::allocation_bound() + + SourceCatalog::allocation_bound() + shared_allocation_bound<MemTable>() + + shared_allocation_bound<SourceWriteBudget>() + shared_allocation_bound<SourceMemoryLedger>() + + shared_allocation_bound<detail::AllocationReservation>() + + (2 * EpochManager::infrastructure_allocation_bound()) + + Node::alloc_size(ConcurrentSkipList::kMaxHeight, 0) + sizeof(std::shared_ptr<MemTable>)) + .add(SourceTailDomain::allocation_bound(max_threads)) + .add_product(max_threads, 2 * EpochManager::participant_allocation_bytes()) + .value(); +} + +SourceAdmission::SourceAdmission(uint64_t floor, uint64_t generation, SourceWriteOptions options, + std::shared_ptr<detail::AllocationReservation> lifecycle) +{ + if (!lifecycle && options.account) { + const auto bytes = allocation_bound(); + if (!bytes) { + throw std::bad_alloc(); + } + auto credit = + detail::ResourceBudget::try_reserve(options.account, detail::BudgetClass::kAdmittedWriter, *bytes); + if (!credit) { + throw std::bad_alloc(); + } + lifecycle = detail::AllocationReservation::create(std::move(*credit)); + } + state_ = detail::reserved_unique<State>(lifecycle, lifecycle); + state_->write_options = std::move(options); + state_->tail_domain = detail::reserved_shared<SourceTailDomain>(lifecycle, lifecycle); + auto initial = detail::reserved_shared<SourceCatalog>(lifecycle, lifecycle); + initial->memory = detail::reserved_shared<SourceMemoryLedger>(lifecycle); + initial->floor = floor; + initial->generation = generation; + initial->active = detail::reserved_shared<MemTable>(lifecycle, 1, nullptr, lifecycle); + if (state_->write_options.account) { + initial->active_budget = detail::reserved_shared<SourceWriteBudget>(lifecycle, state_->write_options); + } + initial->active->set_durable_floor(floor); + initial->tables.push_back(initial->active); + auto initial_root = State::Root::create(initial, lifecycle); + initial->active->attach_source_stats(initial->memory); + state_->root.store(initial_root.release(), std::memory_order_release); +} + +SourceAdmission::~SourceAdmission() +{ + close(); + state_->tail_domain->close(); + auto *root = state_->root.load(std::memory_order_relaxed); + root->catalog->retirement.mark(); + State::Root::Deleter{}(root); +} + +std::optional<size_t> SourceAdmission::capture_allocation_bound(size_t sources) noexcept +{ + using detail::shared_allocation_bound; + // Includes successor infrastructure, two catalogs and pending/covered + // roots. Table vector growth retains old/new buffers before closure. + return detail::AllocationBound( + (2 * SourceCatalog::allocation_bound()) + shared_allocation_bound<MemTable>() + + shared_allocation_bound<SourceWriteBudget>() + shared_allocation_bound<SourceCapture::State>() + + shared_allocation_bound<SourceCapture>() + (2 * State::Root::allocation_bound()) + + EpochManager::infrastructure_allocation_bound() + Node::alloc_size(ConcurrentSkipList::kMaxHeight, 0) + + (6 * sizeof(std::shared_ptr<MemTable>))) + .add_product(sources, sizeof(SourceCapture::State::Node) + (6 * sizeof(std::shared_ptr<MemTable>))) + .value(); +} + +std::shared_ptr<const SourceCatalog> SourceAdmission::catalog() const +{ + auto guard = state_->epoch.enter(); + return state_->root.load(std::memory_order_acquire)->catalog; +} + +Status SourceAdmission::try_admit(uint64_t bound, Writer *out) +try { + if (out == nullptr || out->table()) { + return Status::invalid_argument("writer output must be empty"); + } + if (state_->closed.load(std::memory_order_acquire)) { + return Status::unavailable("source admission closed"); + } + auto observation = state_->tail_domain->observe(); + auto view = catalog(); + if (view->active_budget) { + return Status::invalid_argument("budgeted source requires a writer footprint"); + } + if (!view->active->try_enter()) { + return Status::unavailable("source publication in progress"); + } + *out = Writer(view->active, bound); + out->observation_ = std::move(observation); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("source lifecycle participant credit exhausted"); +} + +Status SourceAdmission::try_admit(uint64_t bound, WriteFootprint footprint, Writer *out, detail::BudgetClass category) +{ + return try_admit(bound, footprint, out, category, nullptr); +} + +Status SourceAdmission::try_admit(uint64_t bound, WriteFootprint footprint, Writer *out, detail::BudgetClass category, + bool *paused) +try { + if (paused != nullptr) { + *paused = false; + } + if (out == nullptr || out->table()) { + return Status::invalid_argument("writer output must be empty"); + } + if (state_->closed.load(std::memory_order_acquire)) { + return Status::unavailable("source admission closed"); + } + auto observation = state_->tail_domain->observe(); + auto view = catalog(); + if (!view->active_budget) { + return Status::invalid_argument("source has no writer budget account"); + } + if (!view->active->try_enter()) { + if (paused != nullptr) { + *paused = true; + } + return Status::unavailable("source publication in progress"); + } + std::shared_ptr<detail::AllocationReservation> credit; + auto status = view->active_budget->admit(footprint, category, &credit, paused); + if (!status.ok()) { + view->active->leave(); + return status; + } + + *out = Writer(view->active, bound); + out->observation_ = std::move(observation); + out->source_budget_ = view->active_budget; + out->credit_ = std::move(credit); + out->remaining_ = footprint; + out->category_ = category; + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("source lifecycle participant credit exhausted"); +} + +Status SourceAdmission::try_admit_reserved(uint64_t bound, SourceWriteReservation *reservation, Writer *out, + std::optional<detail::BudgetClass> source_category) +try { + if (out == nullptr || out->table() || reservation == nullptr || !reservation->valid()) { + return Status::invalid_argument("writer requires an unused reservation and empty output"); + } + if (state_->closed.load(std::memory_order_acquire)) { + return Status::unavailable("source admission closed"); + } + auto observation = state_->tail_domain->observe(); + auto view = catalog(); + if (!view->active_budget) { + return Status::invalid_argument("source has no writer budget account"); + } + if (!view->active->try_enter()) { + return Status::unavailable("source publication in progress"); + } + const auto footprint = reservation->footprint(); + const auto category = source_category.value_or(reservation->category()); + std::shared_ptr<detail::AllocationReservation> credit; + auto status = view->active_budget->bind(reservation, &credit, category); + if (!status.ok()) { + view->active->leave(); + return status; + } + + *out = Writer(view->active, bound); + out->observation_ = std::move(observation); + out->source_budget_ = view->active_budget; + out->credit_ = std::move(credit); + out->remaining_ = footprint; + out->category_ = category; + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("source lifecycle participant credit exhausted"); +} + +Status SourceAdmission::try_capture(const std::atomic<uint64_t> &contiguous, + std::shared_ptr<detail::AllocationReservation> admission_owner, + std::shared_ptr<SourceCapture> *out, + std::shared_ptr<const SourceCatalog> expected_catalog) +try { + if (out == nullptr || !admission_owner) { + return Status::invalid_argument("capture requires admitted metadata credit"); + } + if (state_->write_options.account && admission_owner->account() != state_->write_options.account) { + return Status::invalid_argument("capture credit belongs to another source account"); + } + AdminGuard admin(state_->admin); + if (!admin.held() || state_->closed.load(std::memory_order_acquire)) { + return Status::unavailable("source administration busy or closed"); + } + if (state_->rotation) { + return Status::unavailable("source rotation publication is pending"); + } + if (state_->pending) { + if (expected_catalog && (state_->pending->expected_catalog() != expected_catalog || + state_->pending->admission_owner() != admission_owner)) { + return Status::unavailable("source capture belongs to another prepared reservation"); + } + *out = state_->pending; + return Status::Ok(); + } + const auto previous = catalog(); + if (expected_catalog && previous != expected_catalog) { + return Status::unavailable("source catalog changed after reserve preparation"); + } + if (contiguous.load(std::memory_order_acquire) <= previous->floor) { + return Status::unavailable("no completed prefix beyond source floor"); + } + if (admission_owner->available() == 0) { + return Status::resource_exhausted("capture metadata credit exhausted"); + } + auto next = detail::reserved_shared<SourceCatalog>(admission_owner, admission_owner); + next->memory = previous->memory; + next->tables = previous->tables; + next->floor = previous->floor; + next->generation = previous->generation; + next->serial = previous->serial + 1; + next->active = detail::reserved_shared<MemTable>(admission_owner, state_->next_table, nullptr, + previous->active_budget ? admission_owner : nullptr); + if (previous->active_budget) { + next->active_budget = detail::reserved_shared<SourceWriteBudget>(admission_owner, state_->write_options); + } + next->active->set_durable_floor(previous->floor); + next->tables.push_back(next->active); + auto captured = detail::reserved_shared<SourceCapture::State>(admission_owner, previous, next, admission_owner); + auto covered = detail::reserved_shared<SourceCatalog>(admission_owner, admission_owner); + covered->memory = next->memory; + covered->tables.reserve(next->tables.size()); + covered->generation = next->generation; + covered->serial = next->serial + 1; + covered->active = next->active; + covered->active_budget = next->active_budget; + captured->covered_catalog = covered; + captured->batch = state_->next_batch; + captured->owner_identity = state_->identity; + auto handle = detail::reserved_construct<SourceCapture>( + admission_owner, [&](auto *storage) { ::new (static_cast<void *>(storage)) SourceCapture(captured); }); + auto root = State::Root::create(next, admission_owner); + auto covered_root = State::Root::create(covered, admission_owner); + if (contiguous.load(std::memory_order_acquire) <= previous->floor) { + return Status::unavailable("completed prefix changed before source closure"); + } + previous->active->close(); + captured->frontier = contiguous.load(std::memory_order_acquire); + state_->pending = handle; + state_->pending_root = std::move(root); + state_->covered_root = std::move(covered_root); + ++state_->next_table; + ++state_->next_batch; + *out = std::move(handle); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("source capture allocation failed"); +} + +Status SourceAdmission::activate_successor(const std::shared_ptr<SourceCapture> &capture, uint64_t batch_id) +{ + AdminGuard admin(state_->admin); + if (!admin.held()) { + return Status::unavailable("source administration busy"); + } + if (!state_->matches(capture, batch_id)) { + return Status::invalid_argument("stale source capture"); + } + if (capture->successor_active()) { + return Status::Ok(); + } + state_->publish(std::move(state_->pending_root)); + capture->state_->activated.store(true, std::memory_order_release); + return Status::Ok(); +} + +Status SourceAdmission::mark_repair(const std::shared_ptr<SourceCapture> &capture, uint64_t batch_id, + uint64_t actual_coverage) +{ + AdminGuard admin(state_->admin); + if (!admin.held()) { + return Status::unavailable("source administration busy"); + } + if (!state_->matches(capture, batch_id) || actual_coverage != capture->floor()) { + return Status::invalid_argument("repair must retain this capture at its original floor"); + } + capture->state_->phase.store(SourceCapturePhase::kRepairRequired, std::memory_order_release); + return Status::Ok(); +} + +Status SourceAdmission::finish_covered(const std::shared_ptr<SourceCapture> &capture, uint64_t batch_id, + uint64_t actual_coverage) +try { + AdminGuard admin(state_->admin); + if (!admin.held()) { + return Status::unavailable("source administration busy"); + } + if (!state_->matches(capture, batch_id) || actual_coverage != capture->frontier() || !capture->drained() || + !capture->successor_active()) { + return Status::invalid_argument("coverage acknowledgement must match the drained active capture"); + } + const auto previous = catalog(); + const auto &next = capture->state_->covered_catalog; + next->floor = actual_coverage; + for (const auto &table : previous->tables) { + if (table == next->active || table->slot_range().max > actual_coverage) { + next->tables.push_back(table); + } + } + next->active->set_durable_floor(actual_coverage); + state_->publish(std::move(state_->covered_root)); + capture->state_->phase.store(SourceCapturePhase::kCovered, std::memory_order_release); + state_->pending.reset(); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("source coverage acknowledgement allocation failed"); +} + +void SourceAdmission::close() noexcept +{ + state_->closed.store(true, std::memory_order_release); + state_->tail_domain->close(); +} + +size_t SourceAdmission::reclaim() +{ + return state_->epoch.try_reclaim(); +} + +} // namespace crowdb::tree + +namespace crowdb::tree +{ +const std::shared_ptr<SourceTailDomain> &SourceAdmission::tail_domain() const noexcept +{ + return state_->tail_domain; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_admission.h b/lib/crowdb-tree/src/btree/source_admission.h new file mode 100644 index 000000000..e31507c03 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_admission.h @@ -0,0 +1,276 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "crowdb-tree/memtable/memtable.h" +#include "source_tail.h" +#include "source_write_budget.h" +#include "stdexec_adapter.h" + +#include <memory> + +namespace crowdb::tree +{ +struct PublishedTreeState; + +struct SourceCatalog +{ + + detail::RetirementObserver retirement; + static constexpr bool observe_retirement = true; + + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !tables.empty(); ++i) { + tables.pop_back(); + } + return tables.empty(); + } + + using TableAllocator = detail::ReservationAllocator<std::shared_ptr<MemTable>>; + using Tables = std::vector<std::shared_ptr<MemTable>, TableAllocator>; + + [[nodiscard]] static size_t allocation_bound() noexcept + { + return detail::shared_allocation_bound<SourceCatalog>(); + } + + explicit SourceCatalog(std::shared_ptr<detail::AllocationReservation> credit = {}) + : tables(TableAllocator(std::move(credit))) + { + } + + std::shared_ptr<SourceMemoryLedger> memory; + Tables tables; + std::shared_ptr<MemTable> active; + std::shared_ptr<SourceWriteBudget> active_budget; + uint64_t floor = 0; + uint64_t generation = 1; + uint64_t serial = 0; +}; + +enum class SourceCapturePhase : uint8_t { kCaptured, kDraining, kReady, kRepairRequired, kCovered }; + +class SourceCapture +{ + struct State; + + public: + ~SourceCapture(); + SourceCapture(const SourceCapture &) = delete; + SourceCapture &operator=(const SourceCapture &) = delete; + + [[nodiscard]] uint64_t batch_id() const noexcept; + [[nodiscard]] uint64_t floor() const noexcept; + [[nodiscard]] uint64_t frontier() const noexcept; + [[nodiscard]] SourceCapturePhase phase() const noexcept; + [[nodiscard]] bool drained() const noexcept; + [[nodiscard]] bool successor_active() const noexcept; + [[nodiscard]] const std::shared_ptr<const SourceCatalog> &expected_catalog() const noexcept; + [[nodiscard]] const std::shared_ptr<const SourceCatalog> &next_catalog() const noexcept; + [[nodiscard]] const std::shared_ptr<MemTable> &successor() const noexcept; + [[nodiscard]] const std::shared_ptr<detail::AllocationReservation> &admission_owner() const noexcept; + + // A stopped attempt still drains every admitted writer before terminal. + // Reconnecting after terminal retries against the same fixed P/F/sources. + [[nodiscard]] detail::CallbackSender drain_sender() noexcept; + void request_stop() noexcept; + + private: + friend class SourceAdmission; + + explicit SourceCapture(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; + +// Host-side write source selection. It deliberately does not publish Tree +// mapping/coverage: those belong to the common publication transaction. +class SourceAdmission +{ + struct State; + + public: + class Rotation + { + struct Data; + + public: + ~Rotation(); + Rotation(const Rotation &) = delete; + Rotation &operator=(const Rotation &) = delete; + [[nodiscard]] const std::shared_ptr<const SourceCatalog> &expected() const noexcept; + [[nodiscard]] const std::shared_ptr<SourceCatalog> &next() const noexcept; + + private: + friend class SourceAdmission; + explicit Rotation(detail::ReservedUnique<Data> data); + detail::ReservedUnique<Data> data_; + }; + + class Coverage + { + struct Data; + + public: + ~Coverage(); + Coverage(const Coverage &) = delete; + Coverage &operator=(const Coverage &) = delete; + + private: + friend class SourceAdmission; + explicit Coverage(detail::ReservedUnique<Data> data); + detail::ReservedUnique<Data> data_; + }; + + // Binding a reservation enters thread-bound epoch guards. Keep the Writer + // on that apply thread; transfer SourceWriteReservation between threads. + class Writer + { + public: + class Prepared + { + public: + Prepared() = default; + Prepared(Prepared &&) noexcept = default; + Prepared &operator=(Prepared &&) noexcept = default; + Prepared(const Prepared &) = delete; + Prepared &operator=(const Prepared &) = delete; + + private: + friend class Writer; + std::shared_ptr<MemTable> table_; + MemTable::PreparedWrite write_; + WriteFootprint footprint_; + }; + + Writer() = default; + ~Writer(); + Writer(Writer &&other) noexcept; + Writer &operator=(Writer &&other) noexcept; + Writer(const Writer &) = delete; + Writer &operator=(const Writer &) = delete; + + [[nodiscard]] const std::shared_ptr<MemTable> &table() const noexcept + { + return table_; + } + + [[nodiscard]] uint64_t bound() const noexcept + { + return bound_; + } + + // Whole-batch payload staging consumes the already admitted credit. + // Return its temporary lease before prepare_write adopts those bytes. + [[nodiscard]] const std::shared_ptr<detail::AllocationReservation> &allocation_budget() const noexcept + { + return credit_; + } + + void release() noexcept; + + // A permit is used by one apply caller. It validates the admitted + // footprint before moving an input or changing the source. + Status upsert(Slice key, uint64_t slot, buffer value, bool *changed = nullptr); + Status upsert_external(Slice key, uint64_t slot, uint8_t flags, buffer value, bool *changed = nullptr); + + // Prepare the complete batch before assigning an automatic slot. A + // failure abandons the batch and its permit, returning unused source + // capacity. The buffer is an encoded owned cell or a charged split + // external value; all preparation is thread-bound. + Status prepare_write(Slice key, uint8_t flags, buffer value, Prepared *out); + bool commit_write(Prepared write, uint64_t slot) noexcept; + + private: + friend class SourceAdmission; + Status apply(Slice key, uint64_t slot, uint8_t flags, buffer value, bool split, bool *changed); + + Writer(std::shared_ptr<MemTable> table, uint64_t bound) : table_(std::move(table)), bound_(bound) + { + } + + SourceTailDomain::Observation observation_; + std::shared_ptr<MemTable> table_; + uint64_t bound_ = 0; + std::shared_ptr<SourceWriteBudget> source_budget_; + std::shared_ptr<detail::AllocationReservation> credit_; + WriteFootprint remaining_; + WriteFootprint planned_; + detail::BudgetClass category_ = detail::BudgetClass::kAdmittedWriter; + }; + + // Keep lifecycle credit unsealed for bounded participant registration on + // host threads. Exhaustion rejects ingress before a source is modified. + explicit SourceAdmission(uint64_t floor = 0, uint64_t generation = 1, SourceWriteOptions options = {}, + std::shared_ptr<detail::AllocationReservation> lifecycle = {}); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t max_threads = 64) noexcept; + ~SourceAdmission(); + SourceAdmission(const SourceAdmission &) = delete; + SourceAdmission &operator=(const SourceAdmission &) = delete; + + [[nodiscard]] std::shared_ptr<const SourceCatalog> catalog() const; + [[nodiscard]] uint64_t identity() const noexcept; + [[nodiscard]] const std::shared_ptr<SourceTailDomain> &tail_domain() const noexcept; + [[nodiscard]] static std::optional<size_t> capture_allocation_bound(size_t sources) noexcept; + // bound is the host's completed contiguous frontier. Completion tracking + // and generation fencing remain host responsibilities. An unavailable + // result needs async admission/retry, never a worker spin loop. + [[nodiscard]] Status try_admit(uint64_t bound, Writer *out); + // A client can become a gap by the time consensus chooses it. Select the + // current source counters independently of its prepaid memory partition; + // moving the permit never reserves or charges its allocation bytes again. + [[nodiscard]] Status try_admit_reserved(uint64_t bound, SourceWriteReservation *reservation, Writer *out, + std::optional<detail::BudgetClass> source_category = {}); + [[nodiscard]] Status try_admit(uint64_t bound, WriteFootprint footprint, Writer *out, detail::BudgetClass category, + bool *paused); + [[nodiscard]] Status try_admit(uint64_t bound, WriteFootprint footprint, Writer *out, + detail::BudgetClass category = detail::BudgetClass::kAdmittedWriter); + + // Requires the already-authorized whole-batch metadata credit owner. + // All allocations precede closure. Existing repair captures are returned + // unchanged, rather than closing another active table or widening F. + [[nodiscard]] Status try_capture(const std::atomic<uint64_t> &contiguous, + std::shared_ptr<detail::AllocationReservation> admission_owner, + std::shared_ptr<SourceCapture> *out, + std::shared_ptr<const SourceCatalog> expected_catalog = {}); + // Call only after the Tree publication makes next_catalog's complete + // source membership visible to readers. Before this acknowledgement, + // successor writes are rejected, so no read can miss a new source. + // An accepted capture can still activate after close(); new writers remain + // rejected, while the host can finish draining its accepted flush. + [[nodiscard]] Status activate_successor(const std::shared_ptr<SourceCapture> &capture, uint64_t batch_id); + [[nodiscard]] Status mark_repair(const std::shared_ptr<SourceCapture> &capture, uint64_t batch_id, + uint64_t actual_coverage); + // Call only after the atomic mapping/source coverage commit. It accepts + // exactly this capture's F, never a caller-supplied larger frontier. + [[nodiscard]] Status finish_covered(const std::shared_ptr<SourceCapture> &capture, uint64_t batch_id, + uint64_t actual_coverage); + // Split handoff covers inherited sources while the child's own active + // table keeps later slots. Allocate its new catalog before publication; + // installation only acknowledges the matching immutable covered state. + [[nodiscard]] static std::optional<size_t> coverage_allocation_bound(size_t sources) noexcept; + Status prepare_coverage(uint64_t frontier, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Coverage> *out); + Status install_coverage(const std::shared_ptr<Coverage> &prepared, const PublishedTreeState &published); + [[nodiscard]] static std::optional<size_t> rotation_allocation_bound(size_t sources) noexcept; + // Prepares all successor owners without closing the active source. The + // host publishes next() membership before acknowledging installation. + Status prepare_rotation(const std::shared_ptr<detail::AllocationReservation> &metadata, + const std::shared_ptr<const SourceCatalog> &expected, std::shared_ptr<Rotation> *out); + Status activate_rotation(const std::shared_ptr<Rotation> &rotation); + Status abort_rotation(const std::shared_ptr<Rotation> &rotation); + // Close ingress. Accepted Writer/Capture owners remain valid and must be + // drained by the host before it tears down external epoch/runtime state. + void close() noexcept; + size_t reclaim(); + + private: + detail::ReservedUnique<State> state_; +}; + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_admission_internal.h b/lib/crowdb-tree/src/btree/source_admission_internal.h new file mode 100644 index 000000000..32f5bc3bb --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_admission_internal.h @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "source_admission.h" + +namespace crowdb::tree +{ +struct SourceCapture::State : std::enable_shared_from_this<State> +{ + struct Node + { + State *owner = nullptr; + MemTable::FrozenWaiter waiter{&Node::complete, this}; + static void complete(void *context) noexcept; + }; + + State(std::shared_ptr<const SourceCatalog> previous, std::shared_ptr<const SourceCatalog> next, + std::shared_ptr<detail::AllocationReservation> credit) + : expected(std::move(previous)), + successor_catalog(std::move(next)), + admission(std::move(credit)), + nodes(expected->tables.size(), detail::ReservationAllocator<Node>(admission)) + { + for (auto &node : nodes) { + node.owner = this; + } + } + + void start(detail::CallbackComplete callback, void *argument) noexcept; + void one_drained() noexcept; + void finish() noexcept; + + static constexpr uint8_t kRunning = 1; + static constexpr uint8_t kStopped = 2; + std::shared_ptr<const SourceCatalog> expected; + std::shared_ptr<const SourceCatalog> successor_catalog; + std::shared_ptr<SourceCatalog> covered_catalog; + std::shared_ptr<detail::AllocationReservation> admission; + std::vector<Node, detail::ReservationAllocator<Node>> nodes; + uint64_t batch = 0; + uint64_t frontier = 0; + uint64_t owner_identity = 0; + std::atomic<SourceCapturePhase> phase{SourceCapturePhase::kCaptured}; + std::atomic<bool> activated{false}; + std::atomic<bool> ready{false}; + std::atomic<uint8_t> lifecycle{0}; + std::atomic<size_t> remaining{0}; + bool registered = false; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + std::shared_ptr<State> draining_owner; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_capture.cpp b/lib/crowdb-tree/src/btree/source_capture.cpp new file mode 100644 index 000000000..20e0a26a8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_capture.cpp @@ -0,0 +1,137 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "source_admission_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +void SourceCapture::State::Node::complete(void *context) noexcept +{ + static_cast<Node *>(context)->owner->one_drained(); +} + +void SourceCapture::State::start(detail::CallbackComplete callback, void *argument) noexcept +{ + auto previous = lifecycle.load(std::memory_order_acquire); + do { + if ((previous & kRunning) != 0) { + callback(argument, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + } while (!lifecycle.compare_exchange_weak(previous, previous | kRunning, std::memory_order_acq_rel)); + complete = callback; + context = argument; + if (ready.load(std::memory_order_acquire)) { + finish(); + return; + } + assert(!registered); + registered = true; + auto self = weak_from_this().lock(); + assert(self); + draining_owner = self; + auto captured = SourceCapturePhase::kCaptured; + phase.compare_exchange_strong(captured, SourceCapturePhase::kDraining, std::memory_order_acq_rel); + remaining.store(nodes.size(), std::memory_order_release); + for (size_t i = 0; i < nodes.size(); ++i) { + // Every node is preallocated and registered once. Close precedes start, + // so a last writer cannot be followed by a newly admitted writer. + bool accepted = expected->tables[i]->notify_when_frozen(nodes[i].waiter); + assert(accepted); + (void)accepted; + } +} + +void SourceCapture::State::one_drained() noexcept +{ + auto self = weak_from_this().lock(); + assert(self); + if (remaining.fetch_sub(1, std::memory_order_acq_rel) != 1) { + return; + } + ready.store(true, std::memory_order_release); + auto draining = SourceCapturePhase::kDraining; + phase.compare_exchange_strong(draining, SourceCapturePhase::kReady, std::memory_order_acq_rel); + draining_owner.reset(); + finish(); +} + +void SourceCapture::State::finish() noexcept +{ + auto callback = complete; + auto *argument = context; + const auto previous = lifecycle.exchange(0, std::memory_order_acq_rel); + assert((previous & kRunning) != 0); + const auto signal = (previous & kStopped) != 0 ? detail::CallbackSignal::kStopped : detail::CallbackSignal::kValue; + callback(argument, signal, Status::Ok()); +} + +SourceCapture::~SourceCapture() = default; + +uint64_t SourceCapture::batch_id() const noexcept +{ + return state_->batch; +} + +uint64_t SourceCapture::floor() const noexcept +{ + return state_->expected->floor; +} + +uint64_t SourceCapture::frontier() const noexcept +{ + return state_->frontier; +} + +SourceCapturePhase SourceCapture::phase() const noexcept +{ + return state_->phase.load(std::memory_order_acquire); +} + +bool SourceCapture::drained() const noexcept +{ + return state_->ready.load(std::memory_order_acquire); +} + +bool SourceCapture::successor_active() const noexcept +{ + return state_->activated.load(std::memory_order_acquire); +} + +const std::shared_ptr<const SourceCatalog> &SourceCapture::expected_catalog() const noexcept +{ + return state_->expected; +} + +const std::shared_ptr<const SourceCatalog> &SourceCapture::next_catalog() const noexcept +{ + return state_->successor_catalog; +} + +const std::shared_ptr<MemTable> &SourceCapture::successor() const noexcept +{ + return state_->successor_catalog->active; +} + +const std::shared_ptr<detail::AllocationReservation> &SourceCapture::admission_owner() const noexcept +{ + return state_->admission; +} + +detail::CallbackSender SourceCapture::drain_sender() noexcept +{ + return {this, &SourceCapture::submit}; +} + +void SourceCapture::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto state = static_cast<SourceCapture *>(source)->state_; + state->start(complete, context); +} + +void SourceCapture::request_stop() noexcept +{ + state_->lifecycle.fetch_or(State::kStopped, std::memory_order_acq_rel); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_pause.cpp b/lib/crowdb-tree/src/btree/source_pause.cpp new file mode 100644 index 000000000..e8f13cf3d --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_pause.cpp @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "source_pause.h" + +#include "source_admission.h" + +namespace crowdb::tree +{ +SourcePause::~SourcePause() +{ + if (owns_) { + const bool changed = table_->closed() || budget_->resume(); + if ((changed || notify_) && progress_) { + progress_->advance(); + } + } +} + +Status SourcePause::create(const SourceCatalog &catalog, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyProgress> progress, std::shared_ptr<SourcePause> *out) +try { + if (!catalog.active_budget || out == nullptr || *out) { + return Status::invalid_argument("source pause requires a budgeted active source and empty output"); + } + auto owner = + detail::reserved_shared<SourcePause>(metadata, catalog.active, catalog.active_budget, std::move(progress)); + owner->self_ = owner; + owner->owns_ = true; + if (!catalog.active_budget->pause(&SourcePause::complete, owner.get())) { + owner->owns_ = false; + owner->self_.reset(); + return Status::unavailable("source already has an admission pause owner"); + } + *out = std::move(owner); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("source pause control credit exhausted"); +} + +void SourcePause::complete(void *context) noexcept +{ + auto &owner = *static_cast<SourcePause *>(context); + auto retained = std::move(owner.self_); + owner.ready_.store(true, std::memory_order_release); +} + +bool SourcePause::matches(const SourceCatalog &catalog) const noexcept +{ + return table_ == catalog.active && budget_ == catalog.active_budget; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_pause.h b/lib/crowdb-tree/src/btree/source_pause.h new file mode 100644 index 000000000..7c3d74201 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_pause.h @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "apply_progress.h" +#include "source_write_budget.h" + +namespace crowdb::tree +{ +struct SourceCatalog; + +// Paid immutable-plan ownership. Destruction reopens admission only if the +// captured table has not permanently closed. No executor is retained. +class SourcePause : public std::enable_shared_from_this<SourcePause> +{ + public: + SourcePause(std::shared_ptr<MemTable> table, std::shared_ptr<SourceWriteBudget> budget, + std::shared_ptr<ApplyProgress> progress) + : table_(std::move(table)), + budget_(std::move(budget)), + progress_(std::move(progress)) + { + } + + ~SourcePause(); + static Status create(const SourceCatalog &catalog, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<ApplyProgress> progress, std::shared_ptr<SourcePause> *out); + + [[nodiscard]] bool ready() const noexcept + { + return ready_.load(std::memory_order_acquire); + } + + void wake_on_release() noexcept + { + notify_ = true; + } + + [[nodiscard]] bool matches(const SourceCatalog &catalog) const noexcept; + + [[nodiscard]] WriteFootprint bound() const noexcept + { + return budget_->paused_bound(); + } + + private: + static void complete(void *context) noexcept; + std::shared_ptr<MemTable> table_; + std::shared_ptr<SourceWriteBudget> budget_; + std::shared_ptr<ApplyProgress> progress_; + std::shared_ptr<SourcePause> self_; + std::atomic<bool> ready_{false}; + bool owns_ = false; + bool notify_ = false; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_rotation.cpp b/lib/crowdb-tree/src/btree/source_rotation.cpp new file mode 100644 index 000000000..bfb056fc6 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_rotation.cpp @@ -0,0 +1,318 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "source_rotation.h" + +#include <algorithm> +#include <cassert> + +namespace crowdb::tree +{ +SourceRotation::SourceRotation(SourceAdmission &sources, PublicationCoordinator &publisher, + std::shared_ptr<ApplyDispatcher> dispatcher, std::shared_ptr<ApplyProgress> progress, + std::shared_ptr<detail::ResourceBudget::Account> account, SourceRotationOptions options) + : sources_(&sources), + publisher_(publisher), + dispatcher_(std::move(dispatcher)), + progress_(std::move(progress)), + account_(std::move(account)), + options_(options), + node_(&SourceRotation::run, this) +{ + for (size_t index = 0; index < registrations_.size(); ++index) { + auto registration = + detail::ResourceBudget::try_register(account_, detail::FlushReserveBounds::categories[index]); + if (!registration) { + throw std::bad_alloc(); + } + registrations_[index] = std::move(*registration); + } +} + +void SourceRotation::set_flush(PrepareFlush prepare, CheckRotation check) +{ + prepare_flush_ = std::move(prepare); + check_rotation_ = std::move(check); +} + +void SourceRotation::paused(void *context) noexcept +{ + auto &self = *static_cast<SourceRotation *>(context); + self.phase_.store(Phase::Paused, std::memory_order_release); + self.schedule(); +} + +void SourceRotation::resume() noexcept +{ + if (paused_catalog_) { + // Permanently closed sources keep their exact admitted bound. A failed + // rotation reopens ordinary admission without changing membership. + if (!paused_catalog_->active->closed()) { + paused_catalog_->active_budget->resume(); + } + paused_catalog_.reset(); + paused_owner_.reset(); + progress_->advance(); + } +} + +bool SourceRotation::eligible(const SourceCatalog &catalog) const noexcept +{ + return catalog.active->count() != 0 && + (catalog.active->count() >= options_.entries || catalog.active->approx_bytes() >= options_.bytes || + (catalog.active_budget && catalog.active_budget->full())); +} + +uint64_t SourceRotation::request() noexcept +try { + if (!eligible(*sources_->catalog())) { + return 0; + } + const auto sequence = requested_.fetch_add(1, std::memory_order_acq_rel) + 1; + schedule(); + return sequence; +} +catch (...) { + return 0; +} + +void SourceRotation::wait(uint64_t sequence) noexcept +{ + if (detail::FlushRuntime::on_worker_thread()) { + return; + } + auto value = completed_.load(std::memory_order_acquire); + while (value < sequence) { + completed_.wait(value, std::memory_order_acquire); + value = completed_.load(std::memory_order_acquire); + } +} + +void SourceRotation::stop() noexcept +{ + stopping_.store(true, std::memory_order_release); + schedule(); +} + +void SourceRotation::restart() noexcept +{ + stopping_.store(false, std::memory_order_release); +} + +void SourceRotation::drain() noexcept +{ + stop(); + wait(requested_.load(std::memory_order_acquire)); +} + +void SourceRotation::rebind(SourceAdmission &sources, FlushController &controller) noexcept +{ + assert(completed_.load() == requested_.load()); + sources_ = &sources; + flush_controller_ = &controller; + stopping_.store(false, std::memory_order_release); +} + +void SourceRotation::schedule() noexcept +{ + auto self = weak_from_this().lock(); + if (!self) { + std::terminate(); + } + dispatcher_->request(node_, std::move(self)); +} + +void SourceRotation::run(void *context) noexcept +{ + auto &self = *static_cast<SourceRotation *>(context); + try { + self.step(); + } + catch (...) { + self.committed_ = false; + self.phase_.store(Phase::Acknowledge, std::memory_order_release); + self.schedule(); + } +} + +void SourceRotation::step() +{ + const auto phase = phase_.load(std::memory_order_acquire); + if (wait_step(phase)) { + return; + } + if (phase == Phase::Publishing || phase == Phase::Flushing || phase == Phase::Pausing) { + return; + } + if (phase == Phase::Paused) { + const auto catalog = paused_catalog_; + const auto view = publisher_.capture(); + if (sources_->catalog() != catalog || !view->sources || view->sources->inheritance) { + finish(); + return; + } + auto status = + check_rotation_ ? check_rotation_(*view, *catalog, catalog->active_budget->paused_bound()) : Status::Ok(); + if (status.ok()) { + prepare(catalog); + } + else { + resume(); + if (status.code() == Code::kResourceExhausted) { + flush(); + } + else { + finish(); + } + } + return; + } + if (phase == Phase::FlushReady) { + connected_.reset(); + flush_.reset(); + if (flush_retry_ != FlushRetry::None && flush_controller_ != nullptr && + !stopping_.load(std::memory_order_acquire)) { + if (flush_retry_ == FlushRetry::Replan) { + phase_.store(Phase::FlushPlanning, std::memory_order_release); + schedule(); + } + else { + await_progress(observed_); + } + } + else { + finish(); + } + return; + } + if (phase == Phase::Acknowledge) { + auto status = Status::Ok(); + if (rotation_) { + status = committed_ ? sources_->activate_rotation(rotation_) : sources_->abort_rotation(rotation_); + } + if (status.code() == Code::kUnavailable) { + schedule(); + return; + } + assert(status.ok()); + finish(); + return; + } + target_ = requested_.load(std::memory_order_acquire); + if (target_ == completed_.load(std::memory_order_acquire)) { + return; + } + if (stopping_.load(std::memory_order_acquire)) { + finish(); + return; + } + const auto catalog = sources_->catalog(); + const auto view = publisher_.capture(); + // A captured prefix installs one successor and may be inherited by a + // child with its own active table. Keep both slots before rotating. + const auto source_limit = options_.source_window > 2 ? options_.source_window - 2 : 0; + const bool window_full = view->sources && view->sources->tables.size() >= source_limit; + if (!eligible(*catalog) || (view->sources && view->sources->inheritance)) { + // Inherited catalogs have their own fixed-frontier handoff. They must + // publish coverage before ordinary source maintenance can run. + finish(); + } + else if (window_full || catalog->tables.size() >= std::max<size_t>(2, options_.tables) || + (catalog->active_budget && catalog->active_budget->full())) { + flush(); + } + else if (catalog->active_budget && check_rotation_) { + paused_catalog_ = catalog; + paused_owner_ = shared_from_this(); + phase_.store(Phase::Pausing, std::memory_order_release); + if (!catalog->active_budget->pause(&SourceRotation::paused, this)) { + paused_catalog_.reset(); + paused_owner_.reset(); + finish(); + } + } + else { + prepare(catalog); + } +} + +void SourceRotation::flush() +{ + if (flush_controller_ != nullptr) { + plan_flush(); + return; + } + if (!prepare_flush_ || !prepare_flush_(&flush_).ok()) { + finish(); + return; + } + start_flush(); +} + +void SourceRotation::start_flush() +{ + observed_ = progress_->sequence(); + resumed_ = Status::Ok(); + flush_retry_ = FlushRetry::None; + connected_.emplace(flush_->report_sender(), Receiver{this}); + phase_.store(Phase::Flushing, std::memory_order_release); + stdexec::start(connected_->operation); +} + +void SourceRotation::Receiver::set_value(FlushResult report) const noexcept +{ + owner->resumed_ = + report.handoff_status.ok() ? std::move(report.maintenance_status) : std::move(report.handoff_status); + // Repair and maintenance failures retain their controller-owned batch for + // a later request. Its own terminal progress notification is not a retry. + owner->flush_retry_ = report.admission_stale ? FlushRetry::Replan : FlushRetry::None; + owner->flush_done(); +} + +void SourceRotation::Receiver::set_error(Status status) const noexcept +{ + owner->resumed_ = std::move(status); + owner->flush_retry_ = owner->resumed_.code() == Code::kUnavailable ? FlushRetry::Progress : FlushRetry::None; + owner->flush_done(); +} + +void SourceRotation::Receiver::set_stopped() const noexcept +{ + owner->resumed_ = Status::unavailable(); + owner->flush_retry_ = FlushRetry::None; + owner->flush_done(); +} + +void SourceRotation::flush_done() noexcept +{ + phase_.store(Phase::FlushReady, std::memory_order_release); + schedule(); +} + +void SourceRotation::finish() noexcept +{ + if (admission_metrics_) { + const auto outcome = stopping_.load(std::memory_order_acquire) ? TreeOutcome::Cancelled : TreeOutcome::Error; + (void)admission_metrics_->finish(outcome, tree_metrics_now_ns()); + admission_metrics_.reset(); + } + resume(); + credit_wait_.reset(); + progress_wait_.reset(); + credits_ = {}; + plan_ = {}; + preparation_ = {}; + resumed_ = Status::Ok(); + rotation_.reset(); + if (metadata_) { + metadata_->seal(); + } + metadata_.reset(); + phase_.store(Phase::Idle, std::memory_order_release); + completed_.store(target_, std::memory_order_release); + completed_.notify_all(); + progress_->advance(); + if (requested_.load(std::memory_order_acquire) != target_) { + schedule(); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_rotation.h b/lib/crowdb-tree/src/btree/source_rotation.h new file mode 100644 index 000000000..9b26c7fe2 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_rotation.h @@ -0,0 +1,173 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "apply_dispatcher.h" +#include "apply_progress.h" +#include "flush_operation.h" + +namespace crowdb::tree +{ +struct SourceRotationOptions +{ + size_t bytes = 0; + size_t entries = 0; + size_t tables = 2; + // Actual reader and flush source capacity, including inherited tables. + size_t source_window = 64; +}; + +// A Tree owns this controller through the last accepted rotation and finite +// handoff. Publication and writer catalogs advance together; no new source is +// writable before readers can find it. One dispatcher node serves all writers. +class SourceRotation : public std::enable_shared_from_this<SourceRotation> +{ + public: + using PrepareFlush = detail::InlineFunction<Status(std::shared_ptr<FlushOperation> *)>; + SourceRotation(SourceAdmission &sources, PublicationCoordinator &publisher, + std::shared_ptr<ApplyDispatcher> dispatcher, std::shared_ptr<ApplyProgress> progress, + std::shared_ptr<detail::ResourceBudget::Account> account, SourceRotationOptions options); + using CheckRotation = + detail::InlineFunction<Status(const PublishedTreeState &, const SourceCatalog &, WriteFootprint)>; + void set_flush(PrepareFlush prepare, CheckRotation check = {}); + using PrepareOptions = detail::InlineFunction<Status(FlushPreparationOptions *)>; + void set_async_flush(FlushController &controller, PrepareOptions prepare); + // Zero means no threshold was crossed. Success of the write is independent + // of this best-effort soft maintenance admission. + [[nodiscard]] uint64_t request() noexcept; + void wait(uint64_t sequence) noexcept; + void drain() noexcept; + // Fence only unaccepted soft maintenance; accepted flushes still finish. + void stop() noexcept; + void restart() noexcept; + // Host-only, after old generation ingress, operations and this controller drain. + void rebind(SourceAdmission &sources, FlushController &controller) noexcept; + + private: + enum class Phase : uint8_t { + Idle, + Pausing, + Paused, + Publishing, + Acknowledge, + Flushing, + FlushReady, + FlushPlanning, + CreditWaiting, + CreditReady, + ProgressWaiting, + ProgressReady + }; + + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + SourceRotation *owner; + void set_value(FlushResult report) const noexcept; + void set_error(Status status) const noexcept; + void set_stopped() const noexcept; + }; + + enum class FlushRetry : uint8_t { None, Progress, Replan }; + using Connected = stdexec::connect_result_t<FlushReportSender, Receiver>; + bool eligible(const SourceCatalog &catalog) const noexcept; + static void run(void *context) noexcept; + void schedule() noexcept; + void step(); + static void paused(void *context) noexcept; + void resume() noexcept; + void prepare(const std::shared_ptr<const SourceCatalog> &catalog); + void flush(); + void flush_done() noexcept; + void plan_flush(); + void await_credit(); + void credit_ready(); + void await_progress(uint64_t observed); + bool wait_step(Phase phase); + void start_flush(); + void finish() noexcept; + + SourceAdmission *sources_; + PublicationCoordinator &publisher_; + std::shared_ptr<ApplyDispatcher> dispatcher_; + std::shared_ptr<ApplyProgress> progress_; + std::shared_ptr<detail::ResourceBudget::Account> account_; + const SourceRotationOptions options_; + PrepareFlush prepare_flush_; + CheckRotation check_rotation_; + PrepareOptions prepare_options_; + FlushController *flush_controller_ = nullptr; + FlushPreparationOptions preparation_; + FlushReservePlan plan_; + detail::FlushCredits credits_; + size_t next_credit_ = 0; + std::array<detail::ResourceBudget::Registration, detail::FlushReserveBounds::categories.size()> registrations_; + Status resumed_; + FlushRetry flush_retry_ = FlushRetry::None; + std::atomic<bool> stopping_{false}; + uint64_t observed_ = 0; + std::shared_ptr<const SourceCatalog> paused_catalog_; + std::shared_ptr<SourceRotation> paused_owner_; + ApplyDispatcher::Node node_; + std::atomic<uint64_t> requested_{0}; + std::atomic<uint64_t> completed_{0}; + uint64_t target_ = 0; + std::atomic<Phase> phase_{Phase::Idle}; + bool committed_ = false; + std::shared_ptr<SourceAdmission::Rotation> rotation_; + std::shared_ptr<detail::AllocationReservation> metadata_; + std::shared_ptr<FlushOperation> flush_; + + struct Connection + { + Connection(FlushReportSender sender, Receiver receiver) : operation(stdexec::connect(sender, receiver)) + { + } + + Connected operation; + }; + + std::optional<Connection> connected_; + + struct WaitReceiver + { + using receiver_concept = stdexec::receiver_tag; + SourceRotation *owner; + void set_value() const noexcept; + void set_error(Status status) const noexcept; + void set_stopped() const noexcept; + }; + + using WaitConnected = stdexec::connect_result_t<detail::CallbackSender, WaitReceiver>; + + struct CreditWait + { + CreditWait(SourceRotation &owner, size_t index) + : request(owner.account_, detail::FlushReserveBounds::categories[index], owner.plan_.bounds.amounts[index], + &owner.registrations_[index], detail::ResourceBudget::Request::WaitPolicy::ReservationsOnly), + metrics(owner.publisher_.work_metrics(), TreeStage::BudgetWait, tree_metrics_now_ns()), + connected(stdexec::connect(request.sender(), WaitReceiver{&owner})) + { + } + + detail::ResourceBudget::Request request; + TreeMetricsOperation metrics; + WaitConnected connected; + }; + + struct ProgressWait + { + ProgressWait(SourceRotation &owner, uint64_t observed) + : request(*owner.progress_, observed), + connected(stdexec::connect(request.sender(), WaitReceiver{&owner})) + { + } + + ApplyProgress::Request request; + WaitConnected connected; + }; + + std::optional<CreditWait> credit_wait_; + std::optional<ProgressWait> progress_wait_; + std::optional<TreeMetricsOperation> admission_metrics_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_rotation_prepare.cpp b/lib/crowdb-tree/src/btree/source_rotation_prepare.cpp new file mode 100644 index 000000000..0189aa52a --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_rotation_prepare.cpp @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "source_rotation.h" + +#include <algorithm> + +namespace crowdb::tree +{ +void SourceRotation::prepare(const std::shared_ptr<const SourceCatalog> &catalog) +{ + const auto view = publisher_.capture(); + if (!view->sources) { + finish(); + return; + } + const auto local_count = + view->sources->inheritance ? view->sources->inheritance->local_count : view->sources->tables.size(); + if (local_count != catalog->tables.size() || view->sources->floor != catalog->floor || + view->flush_admission_owner != 0 || local_count > view->sources->tables.size() || + !std::equal(catalog->tables.begin(), catalog->tables.end(), view->sources->tables.begin())) { + finish(); + return; + } + const auto bound = + detail::AllocationBound(SourceAdmission::rotation_allocation_bound(catalog->tables.size()).value_or(SIZE_MAX)) + .add(PublicationCoordinator::empty_commit_reserve_bytes()) + .add(PublishedSources::allocation_bound()) + .add(detail::shared_allocation_bound<PublicationRequest>()) + .add(detail::shared_allocation_bound<InstallRecord>()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add_product(view->sources->tables.size() + 1, sizeof(std::shared_ptr<MemTable>)) + .value(); + auto credit = bound ? detail::ResourceBudget::try_reserve(account_, detail::BudgetClass::kAdmittedWriter, *bound) + : std::nullopt; + if (!credit) { + finish(); + return; + } + metadata_ = detail::AllocationReservation::create(std::move(*credit)); + auto next = detail::reserved_shared<PublishedSources>(metadata_, metadata_); + next->tables.reserve(view->sources->tables.size() + 1); + next->floor = view->sources->floor; + next->memory = catalog->memory; + next->inheritance = view->sources->inheritance; + if (next->inheritance) { + ++next->inheritance->local_count; + } + auto request = detail::reserved_shared<PublicationRequest>(metadata_, metadata_); + request->transaction.record = detail::reserved_shared<InstallRecord>( + metadata_, InstallToken{.generation = view->generation, .batch = target_, .range = 0, .sequence = 0}); + auto status = sources_->prepare_rotation(metadata_, catalog, &rotation_); + if (!status.ok()) { + finish(); + return; + } + next->tables.insert(next->tables.end(), rotation_->next()->tables.begin(), rotation_->next()->tables.end()); + next->tables.insert(next->tables.end(), view->sources->tables.begin() + static_cast<ptrdiff_t>(local_count), + view->sources->tables.end()); + request->transaction.expected_sources = view->sources; + request->transaction.sources = std::move(next); + request->transaction.expected_coverage = catalog->floor; + request->on_done = [self = shared_from_this()](PublicationResult result) noexcept { + self->committed_ = result.status.ok(); + self->phase_.store(Phase::Acknowledge, std::memory_order_release); + self->schedule(); + }; + phase_.store(Phase::Publishing, std::memory_order_release); + if (!publisher_.try_submit(request)) { + committed_ = false; + phase_.store(Phase::Acknowledge, std::memory_order_release); + schedule(); + } +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_rotation_wait.cpp b/lib/crowdb-tree/src/btree/source_rotation_wait.cpp new file mode 100644 index 000000000..b396dc17c --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_rotation_wait.cpp @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "source_rotation.h" + +namespace crowdb::tree +{ +void SourceRotation::set_async_flush(FlushController &controller, PrepareOptions prepare) +{ + flush_controller_ = &controller; + prepare_options_ = std::move(prepare); +} + +void SourceRotation::plan_flush() +{ + if (stopping_.load(std::memory_order_acquire)) { + finish(); + return; + } + plan_ = {}; + credits_ = {}; + if (!preparation_.store) { + auto status = prepare_options_(&preparation_); + if (!status.ok()) { + finish(); + return; + } + } + observed_ = progress_->sequence(); + auto status = flush_controller_->prepare_or_plan(preparation_, &flush_, &plan_); + if (!status.ok()) { + if (status.code() == Code::kUnavailable) { + await_progress(observed_); + } + else { + finish(); + } + return; + } + if (flush_) { + start_flush(); + return; + } + admission_metrics_.emplace(publisher_.work_metrics(), TreeStage::FlushAdmissionWait, tree_metrics_now_ns()); + next_credit_ = 0; + await_credit(); +} + +void SourceRotation::await_credit() +{ + phase_.store(Phase::CreditWaiting, std::memory_order_release); + credit_wait_.emplace(*this, next_credit_); + stdexec::start(credit_wait_->connected); +} + +void SourceRotation::credit_ready() +{ + if (!resumed_.ok() || stopping_.load(std::memory_order_acquire)) { + finish(); + return; + } + if (!credit_wait_) { + std::terminate(); + } + auto credit = credit_wait_->request.take_credit(); + credit_wait_.reset(); + if (!credit) { + finish(); + return; + } + credits_[next_credit_++] = std::move(*credit); + if (next_credit_ != credits_.size()) { + await_credit(); + return; + } + auto status = flush_controller_->prepare_prepaid(preparation_, plan_, std::move(credits_), &flush_); + if (admission_metrics_) { + auto outcome = TreeOutcome::Error; + if (status.ok()) { + outcome = TreeOutcome::Success; + } + else if (status.code() == Code::kUnavailable) { + outcome = TreeOutcome::Stale; + } + (void)admission_metrics_->finish(outcome, tree_metrics_now_ns()); + admission_metrics_.reset(); + } + plan_ = {}; + if (!status.ok()) { + if (status.code() == Code::kUnavailable) { + // The captured immutable plan changed. Replan once through the + // dispatcher; no source was closed by a rejected binding. + phase_.store(Phase::FlushPlanning, std::memory_order_release); + schedule(); + } + else { + finish(); + } + return; + } + start_flush(); +} + +void SourceRotation::await_progress(uint64_t observed) +{ + phase_.store(Phase::ProgressWaiting, std::memory_order_release); + progress_wait_.emplace(*this, observed); + stdexec::start(progress_wait_->connected); +} + +bool SourceRotation::wait_step(Phase phase) +{ + if (phase == Phase::CreditWaiting || phase == Phase::ProgressWaiting) { + if (stopping_.load(std::memory_order_acquire)) { + if (phase == Phase::CreditWaiting) { + if (!credit_wait_) { + std::terminate(); + } + credit_wait_->request.request_stop(); + } + else { + if (!progress_wait_) { + std::terminate(); + } + progress_wait_->request.request_stop(); + } + } + return true; + } + if (phase == Phase::CreditReady) { + credit_ready(); + return true; + } + if (phase == Phase::ProgressReady) { + progress_wait_.reset(); + if (!resumed_.ok()) { + finish(); + } + else { + plan_flush(); + } + return true; + } + if (phase == Phase::FlushPlanning) { + plan_flush(); + return true; + } + return false; +} + +void SourceRotation::WaitReceiver::set_value() const noexcept +{ + if (owner->phase_.load(std::memory_order_relaxed) == Phase::CreditWaiting) { + if (!owner->credit_wait_) { + std::terminate(); + } + (void)owner->credit_wait_->metrics.finish(TreeOutcome::Success, tree_metrics_now_ns()); + } + owner->resumed_ = Status::Ok(); + const auto ready = owner->phase_.load(std::memory_order_relaxed) == Phase::CreditWaiting ? Phase::CreditReady + : Phase::ProgressReady; + owner->phase_.store(ready, std::memory_order_release); + owner->schedule(); +} + +void SourceRotation::WaitReceiver::set_error(Status status) const noexcept +{ + if (owner->phase_.load(std::memory_order_relaxed) == Phase::CreditWaiting) { + if (!owner->credit_wait_) { + std::terminate(); + } + const auto outcome = + owner->stopping_.load(std::memory_order_acquire) ? TreeOutcome::Cancelled : TreeOutcome::Error; + (void)owner->credit_wait_->metrics.finish(outcome, tree_metrics_now_ns()); + } + owner->resumed_ = std::move(status); + const auto ready = owner->phase_.load(std::memory_order_relaxed) == Phase::CreditWaiting ? Phase::CreditReady + : Phase::ProgressReady; + owner->phase_.store(ready, std::memory_order_release); + owner->schedule(); +} + +void SourceRotation::WaitReceiver::set_stopped() const noexcept +{ + set_error(Status::unavailable("soft source maintenance stopped")); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_tail.cpp b/lib/crowdb-tree/src/btree/source_tail.cpp new file mode 100644 index 000000000..fe222b2e6 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_tail.cpp @@ -0,0 +1,146 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "source_tail.h" + +#include "allocation_bound.h" +#include "source_write_budget.h" + +#include <cassert> + +namespace crowdb::tree +{ +namespace +{ +bool claim(std::atomic<size_t> &used, size_t value, size_t maximum) noexcept +{ + auto old = used.load(std::memory_order_relaxed); + while (old <= maximum && value <= maximum - old) { + if (used.compare_exchange_weak(old, old + value, std::memory_order_acq_rel)) { + return true; + } + } + return false; +} +} // namespace + +SourceTail::SourceTail(std::shared_ptr<detail::AllocationReservation> metadata, size_t records, size_t bytes) + : metadata_(std::move(metadata)), + table_(detail::reserved_shared<MemTable>(metadata_, 0, nullptr, metadata_)), + max_records_(records), + max_bytes_(bytes) +{ +} + +std::optional<size_t> SourceTail::allocation_bound(size_t records, size_t bytes) noexcept +{ + SourceWriteOptions options; + options.max_entries = records; + options.max_gap_entries = 0; + // No payload allocation occurs here. The byte cap includes copied keys + // and retained values, conservatively treating all of it as possible keys. + auto writes = SourceWriteBudget::allocation_bound(options, {.entries = records, .key_bytes = bytes}); + return detail::AllocationBound() + .add(writes) + .add(detail::shared_allocation_bound<SourceTail>()) + .add(detail::shared_allocation_bound<MemTable>()) + .add(EpochManager::infrastructure_allocation_bound()) + .add(Node::alloc_size(ConcurrentSkipList::kMaxHeight, 0)) + .add_product(64, EpochManager::participant_allocation_bytes()) + .value(); +} + +SourceTail::Write SourceTail::prepare(const std::shared_ptr<SourceTail> &owner, + const MemTable::PreparedWrite &parent) noexcept +{ + Write result; + if (!owner) { + return result; + } + auto state = owner->state_.load(std::memory_order_acquire); + do { + if ((state & (kStopped | kExpired)) != 0) { + return result; + } + } while (!owner->state_.compare_exchange_weak(state, state + 1, std::memory_order_acq_rel)); + result.owner = owner; + result.entered = true; + const auto bytes = detail::AllocationBound(parent.key().size()).add(parent.cell_owner()->cell.size()).value(); + if (!bytes || !claim(owner->records_, 1, owner->max_records_) || !claim(owner->bytes_, *bytes, owner->max_bytes_)) { + owner->expire(); + return result; + } + try { + // All allocations precede the parent CAS. + // A failed observation expires only the split, never the parent write. + result.prepared = owner->table_->prepare_reference(parent.key(), parent.cell_owner(), 0, owner->max_records_, + owner->metadata_); + } + catch (...) { + owner->expire(); + } + return result; +} + +SourceTail::Write::Write(Write &&other) noexcept + : owner(std::move(other.owner)), + prepared(std::move(other.prepared)), + entered(std::exchange(other.entered, false)) +{ +} + +SourceTail::Write &SourceTail::Write::operator=(Write &&other) noexcept +{ + if (this != &other) { + if (entered) { + owner->leave(); + } + owner = std::move(other.owner); + prepared = std::move(other.prepared); + entered = std::exchange(other.entered, false); + } + return *this; +} + +SourceTail::Write::~Write() +{ + if (entered) { + owner->leave(); + } +} + +void SourceTail::Write::complete(bool changed, uint64_t slot) noexcept +{ + if (entered && !owner->expired() && changed) { + owner->table_->commit_write(std::move(prepared), slot); + } + if (entered) { + entered = false; + owner->leave(); + } +} + +void SourceTail::leave() noexcept +{ + state_.fetch_sub(1, std::memory_order_acq_rel); +} + +bool SourceTail::freeze() noexcept +{ + size_t expected = 0; + if (!state_.compare_exchange_strong(expected, kStopped, std::memory_order_acq_rel)) { + return false; + } + table_->close(); + return true; +} + +void SourceTail::expire() noexcept +{ + state_.fetch_or(kExpired, std::memory_order_acq_rel); +} + +bool SourceTail::expired() const noexcept +{ + return (state_.load(std::memory_order_acquire) & kExpired) != 0; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_tail.h b/lib/crowdb-tree/src/btree/source_tail.h new file mode 100644 index 000000000..fce7eadc0 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_tail.h @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "crowdb-tree/memtable/memtable.h" + +namespace crowdb::tree +{ +// A finite, independent source for a prepared split. Payloads retain their +// original allocation owner; all new nodes and cross-Tree reasons are paid by +// the destination before observation starts. +class SourceTail +{ + public: + struct Write + { + std::shared_ptr<SourceTail> owner; + MemTable::PreparedWrite prepared; + bool entered = false; + Write() = default; + Write(Write &&) noexcept; + Write &operator=(Write &&) noexcept; + ~Write(); + void complete(bool changed, uint64_t slot) noexcept; + }; + + SourceTail(std::shared_ptr<detail::AllocationReservation> metadata, size_t records, size_t bytes); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t records, size_t bytes) noexcept; + static Write prepare(const std::shared_ptr<SourceTail> &owner, const MemTable::PreparedWrite &parent) noexcept; + // Only zero active writers may freeze the tail. Failed attempts never + // change source admission or the destination's published generation. + bool freeze() noexcept; + void expire() noexcept; + [[nodiscard]] bool expired() const noexcept; + + [[nodiscard]] const std::shared_ptr<MemTable> &table() const noexcept + { + return table_; + } + + private: + void leave() noexcept; + static constexpr size_t kStopped = size_t{1} << ((sizeof(size_t) * 8) - 1); + static constexpr size_t kExpired = kStopped >> 1; + std::atomic<size_t> state_{0}; + std::atomic<size_t> records_{0}; + std::atomic<size_t> bytes_{0}; + std::shared_ptr<detail::AllocationReservation> metadata_; + std::shared_ptr<MemTable> table_; + size_t max_records_; + size_t max_bytes_; +}; + +// The source lifetime owns this domain across active-table rotations. Guards +// span a complete admitted Writer, including its frontier acknowledgement. +// Registering a tail then retiring a barrier covers pre-registration writers: +// all pre-registration, unobserved writers have acknowledged before their +// max-seen high-watermark A is read. Child replay must cover A, including gaps. +class SourceTailDomain : public std::enable_shared_from_this<SourceTailDomain> +{ + struct Registration; + + public: + struct Observation + { + std::shared_ptr<SourceTailDomain> domain; + EpochManager::Guard guard; + std::shared_ptr<SourceTail> tail; + Observation() = default; + Observation(Observation &&) noexcept = default; + + Observation &operator=(Observation &&other) noexcept + { + if (this != &other) { + tail.reset(); + guard = {}; + domain.reset(); + domain = std::move(other.domain); + guard = std::move(other.guard); + tail = std::move(other.tail); + } + return *this; + } + }; + + struct Barrier : EpochManager::Deferred + { + std::shared_ptr<Barrier> owner; + std::shared_ptr<Registration> registration; + std::atomic<bool> ready{false}; + void wait() const noexcept; + }; + + explicit SourceTailDomain(std::shared_ptr<detail::AllocationReservation> metadata); + ~SourceTailDomain(); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t threads) noexcept; + [[nodiscard]] static size_t registration_bound() noexcept; + Observation observe(); + Status arm(const std::shared_ptr<SourceTail> &tail, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Barrier> *out); + bool detach(const std::shared_ptr<Barrier> &subscription) noexcept; + void close() noexcept; + + private: + void retire(Registration *entry) noexcept; + std::shared_ptr<detail::AllocationReservation> metadata_; + EpochManager epoch_; + std::atomic<Registration *> registration_{nullptr}; + std::atomic<bool> closed_{false}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_tail_domain.cpp b/lib/crowdb-tree/src/btree/source_tail_domain.cpp new file mode 100644 index 000000000..7c5eaba6a --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_tail_domain.cpp @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "source_tail.h" + +namespace crowdb::tree +{ +struct SourceTailDomain::Registration : EpochManager::Deferred +{ + std::shared_ptr<Registration> owner; + std::shared_ptr<SourceTail> tail; +}; + +SourceTailDomain::SourceTailDomain(std::shared_ptr<detail::AllocationReservation> metadata) + : metadata_(std::move(metadata)), + epoch_(metadata_) +{ +} + +SourceTailDomain::~SourceTailDomain() +{ + close(); +} + +std::optional<size_t> SourceTailDomain::allocation_bound(size_t threads) noexcept +{ + return detail::AllocationBound(detail::shared_allocation_bound<SourceTailDomain>()) + .add(EpochManager::infrastructure_allocation_bound()) + .add_product(threads, EpochManager::participant_allocation_bytes()) + .value(); +} + +size_t SourceTailDomain::registration_bound() noexcept +{ + return detail::shared_allocation_bound<Registration>() + detail::shared_allocation_bound<Barrier>(); +} + +SourceTailDomain::Observation SourceTailDomain::observe() +{ + Observation result; + result.domain = shared_from_this(); + result.guard = epoch_.enter(); + if (auto *entry = registration_.load(std::memory_order_acquire)) { + result.tail = entry->tail; + } + return result; +} +Status SourceTailDomain::arm(const std::shared_ptr<SourceTail> &tail, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Barrier> *out) +try { + if (!tail || !metadata || !metadata_ || !metadata_->retirement_queue() || out == nullptr || *out || + closed_.load(std::memory_order_acquire)) { + return Status::invalid_argument("tail registration requires a live source and owned output"); + } + auto entry = detail::reserved_shared<Registration>(metadata); + auto barrier = detail::reserved_shared<Barrier>(metadata); + entry->tail = tail; + entry->destroy = [](EpochManager::Deferred *node) noexcept { + auto owner = std::move(static_cast<Registration *>(node)->owner); + }; + barrier->destroy = [](EpochManager::Deferred *node) noexcept { + auto &signal = *static_cast<Barrier *>(node); + auto owner = std::move(signal.owner); + signal.ready.store(true, std::memory_order_release); + signal.ready.notify_all(); + }; + Registration *expected = nullptr; + entry->owner = entry; + if (!registration_.compare_exchange_strong(expected, entry.get(), std::memory_order_acq_rel)) { + entry->owner.reset(); + return Status::unavailable("source already has a prepared tail"); + } + barrier->registration = std::move(entry); + barrier->owner = barrier; + epoch_.defer(barrier.get()); + *out = std::move(barrier); + if (closed_.load(std::memory_order_acquire)) { + tail->expire(); + detach(*out); + return Status::unavailable("source closed during tail registration"); + } + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("tail registration credit exhausted"); +} + +void SourceTailDomain::retire(Registration *entry) noexcept +{ + if (entry != nullptr) { + epoch_.defer(entry); + } +} + +bool SourceTailDomain::detach(const std::shared_ptr<Barrier> &subscription) noexcept +{ + if (!subscription || !subscription->registration) { + return false; + } + auto *entry = subscription->registration.get(); + if (registration_.compare_exchange_strong(entry, nullptr, std::memory_order_acq_rel)) { + retire(entry); + return true; + } + return false; +} + +void SourceTailDomain::close() noexcept +{ + closed_.store(true, std::memory_order_release); + if (auto *entry = registration_.exchange(nullptr, std::memory_order_acq_rel)) { + entry->tail->expire(); + retire(entry); + } +} + +void SourceTailDomain::Barrier::wait() const noexcept +{ + while (!ready.load(std::memory_order_acquire)) { + ready.wait(false, std::memory_order_acquire); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_write_budget.cpp b/lib/crowdb-tree/src/btree/source_write_budget.cpp new file mode 100644 index 000000000..4ddc7742f --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_write_budget.cpp @@ -0,0 +1,200 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "source_write_budget.h" + +#include "crowdb-tree/memtable/skip_list.h" + +namespace crowdb::tree +{ +std::array<ResourceUsage, 4> SourceWriteBudget::resource_usage() const noexcept +{ + return { + {{.limit = options_.max_entries, .used = entries_.load(std::memory_order_relaxed)}, + {.limit = options_.max_key_bytes, .used = key_bytes_.load(std::memory_order_relaxed)}, + {.limit = options_.max_gap_entries, .used = gap_entries_.load(std::memory_order_relaxed)}, + {.limit = options_.max_gap_key_bytes, .used = gap_key_bytes_.load(std::memory_order_relaxed)}} + }; +} + +namespace +{ +bool reserve_count(std::atomic<size_t> &counter, size_t amount, size_t maximum) +{ + auto current = counter.load(std::memory_order_relaxed); + while (current <= maximum && amount <= maximum - current) { + if (counter.compare_exchange_weak(current, current + amount, std::memory_order_acq_rel)) { + return true; + } + } + return false; +} + +std::optional<size_t> write_allocation_bound(WriteFootprint footprint, size_t max_versions) +{ + // Includes the old/new buffers during vector growth, an unpublished + // insert descriptor losing to another writer, cell control block, node + // tower and descriptor owners. Payload capacity is supplied separately. + constexpr size_t history_entry = 3 * sizeof(std::shared_ptr<const CellVersion>); + constexpr size_t descriptors = 4096; + if (max_versions > (SIZE_MAX - descriptors) / history_entry) { + return {}; + } + const auto per_entry = descriptors + (max_versions * history_entry); + if (footprint.entries > SIZE_MAX / per_entry) { + return {}; + } + auto bytes = footprint.entries * per_entry; + if (footprint.key_bytes > SIZE_MAX - bytes) { + return {}; + } + bytes += footprint.key_bytes; + if (footprint.payload_bytes > SIZE_MAX - bytes) { + return {}; + } + return bytes + footprint.payload_bytes; +} +} // namespace + +std::optional<size_t> SourceWriteBudget::allocation_bound(const SourceWriteOptions &options, + WriteFootprint footprint) noexcept +{ + if (options.max_gap_entries > SIZE_MAX - options.max_entries) { + return {}; + } + return write_allocation_bound(footprint, options.max_entries + options.max_gap_entries); +} + +Status SourceWriteReservation::create(const SourceWriteOptions &options, WriteFootprint footprint, + detail::BudgetClass category, SourceWriteReservation *out) +{ + if (out == nullptr || out->valid() || !options.account || footprint.entries == 0 || + (category != detail::BudgetClass::kAdmittedWriter && category != detail::BudgetClass::kGapRepair)) { + return Status::invalid_argument("writer requires a nonempty footprint and dedicated apply budget"); + } + const auto largest_key = footprint.max_key_size == 0 ? footprint.key_bytes : footprint.max_key_size; + if (largest_key > footprint.key_bytes || largest_key > options.max_key_size) { + return Status::invalid_argument("writer key footprint exceeds the source key limit"); + } + const bool gap = category == detail::BudgetClass::kGapRepair; + if (footprint.entries > (gap ? options.max_gap_entries : options.max_entries) || + footprint.key_bytes > (gap ? options.max_gap_key_bytes : options.max_key_bytes)) { + return Status::resource_exhausted("writer footprint exceeds one source progress window"); + } + const auto bytes = SourceWriteBudget::allocation_bound(options, footprint); + if (!bytes) { + return Status::resource_exhausted("writer allocation bound overflow"); + } + auto credit = detail::ResourceBudget::try_reserve(options.account, category, *bytes); + if (!credit) { + return Status::unavailable("writer apply credit unavailable"); + } + try { + auto owned = detail::AllocationReservation::create(std::move(*credit)); + out->options_ = options; + out->footprint_ = footprint; + out->category_ = category; + out->credit_ = std::move(owned); + return Status::Ok(); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted("writer reservation allocation failed"); + } +} + +Status SourceWriteBudget::bind(SourceWriteReservation *reservation, std::shared_ptr<detail::AllocationReservation> *out, + std::optional<detail::BudgetClass> source_category) +{ + return bind(reservation, out, source_category, nullptr); +} + +Status SourceWriteBudget::bind(SourceWriteReservation *reservation, std::shared_ptr<detail::AllocationReservation> *out, + std::optional<detail::BudgetClass> source_category, bool *paused) +{ + if (paused != nullptr) { + *paused = false; + } + if (reservation == nullptr || !reservation->valid() || out == nullptr || *out) { + return Status::invalid_argument("writer binding requires an unused reservation"); + } + const auto &options = reservation->options_; + if (options.account != options_.account || options.max_entries != options_.max_entries || + options.max_key_bytes != options_.max_key_bytes || options.max_gap_entries != options_.max_gap_entries || + options.max_gap_key_bytes != options_.max_gap_key_bytes || options.max_key_size != options_.max_key_size) { + return Status::invalid_argument("writer reservation belongs to different source limits"); + } + const auto category = source_category.value_or(reservation->category_); + if (category != detail::BudgetClass::kAdmittedWriter && category != detail::BudgetClass::kGapRepair) { + return Status::invalid_argument("source binding requires ordinary or gap capacity"); + } + const auto footprint = reservation->footprint_; + const bool gap = category == detail::BudgetClass::kGapRepair; + if (footprint.entries > (gap ? options_.max_gap_entries : options_.max_entries) || + footprint.key_bytes > (gap ? options_.max_gap_key_bytes : options_.max_key_bytes)) { + return Status::resource_exhausted("writer footprint cannot fit the selected source partition"); + } + if (!gap && !enter_bind()) { + if (paused != nullptr) { + *paused = true; + } + return Status::unavailable("ordinary source admission is paused for rotation"); + } + + struct Leave + { + SourceWriteBudget &owner; + bool ordinary; + + ~Leave() + { + if (ordinary) { + owner.leave_bind(); + } + } + } leave{.owner = *this, .ordinary = !gap}; + + auto &entries = gap ? gap_entries_ : entries_; + auto &keys = gap ? gap_key_bytes_ : key_bytes_; + if (!reserve_count(entries, footprint.entries, gap ? options_.max_gap_entries : options_.max_entries)) { + if (!gap) { + capacity_rejected_.store(true, std::memory_order_release); + } + return Status::unavailable("source version admission limit reached"); + } + if (!reserve_count(keys, footprint.key_bytes, gap ? options_.max_gap_key_bytes : options_.max_key_bytes)) { + entries.fetch_sub(footprint.entries, std::memory_order_acq_rel); + if (!gap) { + capacity_rejected_.store(true, std::memory_order_release); + } + return Status::unavailable("source key admission limit reached"); + } + *out = std::move(reservation->credit_); + return Status::Ok(); +} + +Status SourceWriteBudget::admit(WriteFootprint footprint, detail::BudgetClass category, + std::shared_ptr<detail::AllocationReservation> *out) +{ + return admit(footprint, category, out, nullptr); +} + +Status SourceWriteBudget::admit(WriteFootprint footprint, detail::BudgetClass category, + std::shared_ptr<detail::AllocationReservation> *out, bool *paused) +{ + if (paused != nullptr) { + *paused = false; + } + if (out == nullptr || *out) { + return Status::invalid_argument("writer output must be empty"); + } + SourceWriteReservation reservation; + auto status = SourceWriteReservation::create(options_, footprint, category, &reservation); + return status.ok() ? bind(&reservation, out, {}, paused) : status; +} + +void SourceWriteBudget::return_unused(WriteFootprint footprint, detail::BudgetClass category) noexcept +{ + const bool gap = category == detail::BudgetClass::kGapRepair; + (gap ? gap_entries_ : entries_).fetch_sub(footprint.entries, std::memory_order_acq_rel); + (gap ? gap_key_bytes_ : key_bytes_).fetch_sub(footprint.key_bytes, std::memory_order_acq_rel); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_write_budget.h b/lib/crowdb-tree/src/btree/source_write_budget.h new file mode 100644 index 000000000..9f5038580 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_write_budget.h @@ -0,0 +1,137 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "crowdb-tree/memtable/memtable.h" + +namespace crowdb::tree +{ +struct WriteFootprint +{ + size_t entries = 0; + size_t key_bytes = 0; + size_t payload_bytes = 0; + // Zero uses key_bytes as a conservative largest-key bound. Batch callers + // supply this explicitly when total key bytes exceed one key's limit. + size_t max_key_size = 0; +}; + +struct SourceWriteOptions +{ + std::shared_ptr<detail::ResourceBudget::Account> account; + size_t max_entries = 4096; + size_t max_key_bytes = 1024 * 1024; + size_t max_gap_entries = 64; + size_t max_gap_key_bytes = 128 * 1024; + size_t max_key_size = 256 * 1024; +}; + +// Transferable resource admission. It owns no source, writer entrance or epoch +// guard, so consensus may retain it while a source is captured or replaced. +// A successful bind consumes the permit exactly once; an unavailable source +// leaves it intact for the operation's bounded asynchronous continuation. +class SourceWriteReservation +{ + public: + SourceWriteReservation() = default; + SourceWriteReservation(SourceWriteReservation &&) noexcept = default; + SourceWriteReservation &operator=(SourceWriteReservation &&) noexcept = default; + SourceWriteReservation(const SourceWriteReservation &) = delete; + SourceWriteReservation &operator=(const SourceWriteReservation &) = delete; + + static Status create(const SourceWriteOptions &options, WriteFootprint footprint, detail::BudgetClass category, + SourceWriteReservation *out); + + static Status bind_prepaid(const SourceWriteOptions &options, WriteFootprint footprint, + detail::BudgetClass category, detail::ResourceBudget::Credit credit, + SourceWriteReservation *out); + + [[nodiscard]] bool valid() const noexcept + { + return credit_ != nullptr; + } + + [[nodiscard]] WriteFootprint footprint() const noexcept + { + return footprint_; + } + + [[nodiscard]] detail::BudgetClass category() const noexcept + { + return category_; + } + + private: + friend class SourceWriteBudget; + friend class SourceAdmission; + SourceWriteOptions options_; + WriteFootprint footprint_; + detail::BudgetClass category_ = detail::BudgetClass::kAdmittedWriter; + std::shared_ptr<detail::AllocationReservation> credit_; +}; + +// Each source caps the number of admitted versions, including writers that +// have not applied yet. Every writer reserves its own worst-case COW history +// before acceptance, so another writer cannot consume its apply allowance. +class SourceWriteBudget +{ + public: + explicit SourceWriteBudget(SourceWriteOptions options) : options_(std::move(options)) + { + } + + Status admit(WriteFootprint footprint, detail::BudgetClass category, + std::shared_ptr<detail::AllocationReservation> *out); + Status admit(WriteFootprint footprint, detail::BudgetClass category, + std::shared_ptr<detail::AllocationReservation> *out, bool *paused); + Status bind(SourceWriteReservation *reservation, std::shared_ptr<detail::AllocationReservation> *out, + std::optional<detail::BudgetClass> source_category, bool *paused); + // Consumes only source counters; memory was reserved before proposal. + Status bind(SourceWriteReservation *reservation, std::shared_ptr<detail::AllocationReservation> *out, + std::optional<detail::BudgetClass> source_category = {}); + [[nodiscard]] static std::optional<size_t> allocation_bound(const SourceWriteOptions &options, + WriteFootprint footprint) noexcept; + void return_unused(WriteFootprint footprint, detail::BudgetClass category) noexcept; + // Active-source admission counts include accepted writers, not just + // materialized records. Source contention counters are not sampled here. + [[nodiscard]] std::array<ResourceUsage, 4> resource_usage() const noexcept; + + // Stops new ordinary binds; previously admitted writers are already included + // in the counters. Gap repair keeps its independent fixed allowance. + using PauseComplete = void (*)(void *) noexcept; + bool pause(PauseComplete complete, void *context) noexcept; + bool resume() noexcept; + [[nodiscard]] bool paused() const noexcept; + [[nodiscard]] WriteFootprint paused_bound() const noexcept; + + [[nodiscard]] bool full() const noexcept + { + return capacity_rejected_.load(std::memory_order_acquire) || + entries_.load(std::memory_order_acquire) >= options_.max_entries || + key_bytes_.load(std::memory_order_acquire) >= options_.max_key_bytes; + } + + [[nodiscard]] const SourceWriteOptions &options() const noexcept + { + return options_; + } + + private: + bool enter_bind() noexcept; + void leave_bind() noexcept; + static constexpr size_t kPaused = size_t{1} << ((sizeof(size_t) * 8) - 1); + std::atomic<size_t> binders_{0}; + std::atomic<PauseComplete> pause_complete_{nullptr}; + void *pause_context_ = nullptr; + + SourceWriteOptions options_; + + std::atomic<size_t> entries_{0}; + std::atomic<size_t> key_bytes_{0}; + std::atomic<size_t> gap_entries_{0}; + std::atomic<size_t> gap_key_bytes_{0}; + std::atomic<bool> pause_waiter_{false}; + std::atomic<bool> capacity_rejected_{false}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_write_pause.cpp b/lib/crowdb-tree/src/btree/source_write_pause.cpp new file mode 100644 index 000000000..64ad4641c --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_write_pause.cpp @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "source_write_budget.h" + +#include <cassert> + +namespace crowdb::tree +{ +bool SourceWriteBudget::enter_bind() noexcept +{ + auto value = binders_.load(std::memory_order_acquire); + for (;;) { + while ((value & kPaused) == 0 && value != kPaused - 1) { + if (binders_.compare_exchange_weak(value, value + 1, std::memory_order_acq_rel)) { + return true; + } + } + pause_waiter_.store(true, std::memory_order_release); + value = binders_.load(std::memory_order_acquire); + if ((value & kPaused) != 0 || value == kPaused - 1) { + return false; + } + // Resume won before waiter publication. Retry the bind instead of + // waiting for an event which has already passed. + } +} + +void SourceWriteBudget::leave_bind() noexcept +{ + if (binders_.fetch_sub(1, std::memory_order_acq_rel) == kPaused + 1) { + const auto complete = pause_complete_.load(std::memory_order_acquire); + auto *context = pause_context_; + complete(context); + } +} + +bool SourceWriteBudget::pause(PauseComplete complete, void *context) noexcept +{ + assert(complete != nullptr); + // The controller and soft rotation compete for one reversible pause. + // Publish callback storage before the last binder may deliver it. + PauseComplete expected = nullptr; + while (!pause_complete_.compare_exchange_strong(expected, complete, std::memory_order_acq_rel)) { + pause_waiter_.store(true, std::memory_order_release); + if (pause_complete_.load(std::memory_order_acquire) != nullptr) { + return false; + } + expected = nullptr; + } + pause_context_ = context; + const auto previous = binders_.fetch_or(kPaused, std::memory_order_acq_rel); + if (previous == 0) { + complete(context); + } + return true; +} + +bool SourceWriteBudget::resume() noexcept +{ + assert(binders_.load(std::memory_order_acquire) == kPaused); + binders_.store(0, std::memory_order_release); + pause_complete_.store(nullptr, std::memory_order_release); + return pause_waiter_.exchange(false, std::memory_order_acq_rel); +} + +bool SourceWriteBudget::paused() const noexcept +{ + return (binders_.load(std::memory_order_acquire) & kPaused) != 0; +} + +WriteFootprint SourceWriteBudget::paused_bound() const noexcept +{ + assert(binders_.load(std::memory_order_acquire) == kPaused); + // Returned unused permits can only reduce these ordinary totals. Gap + // admission remains open, so its complete maximum is always included. + const auto add = [](size_t value, size_t allowance) { + return allowance > SIZE_MAX - value ? SIZE_MAX : value + allowance; + }; + return {.entries = add(entries_.load(std::memory_order_acquire), options_.max_gap_entries), + .key_bytes = add(key_bytes_.load(std::memory_order_acquire), options_.max_gap_key_bytes)}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_write_prepaid.cpp b/lib/crowdb-tree/src/btree/source_write_prepaid.cpp new file mode 100644 index 000000000..c3b99e1f4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_write_prepaid.cpp @@ -0,0 +1,28 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "source_write_budget.h" + +namespace crowdb::tree +{ +Status SourceWriteReservation::bind_prepaid(const SourceWriteOptions &options, WriteFootprint footprint, + detail::BudgetClass category, detail::ResourceBudget::Credit credit, + SourceWriteReservation *out) +{ + const auto bound = SourceWriteBudget::allocation_bound(options, footprint); + const bool gap = category == detail::BudgetClass::kGapRepair; + const auto largest = footprint.max_key_size == 0 ? footprint.key_bytes : footprint.max_key_size; + if (out == nullptr || out->valid() || footprint.entries == 0 || !bound || + (category != detail::BudgetClass::kAdmittedWriter && !gap) || + footprint.entries > (gap ? options.max_gap_entries : options.max_entries) || + footprint.key_bytes > (gap ? options.max_gap_key_bytes : options.max_key_bytes) || + largest > footprint.key_bytes || largest > options.max_key_size || + !credit.belongs_to(options.account, category) || credit.allocated() || credit.amount() < *bound) { + return Status::invalid_argument("writer binding requires its complete unused partition grant"); + } + out->credit_ = detail::AllocationReservation::create(std::move(credit)); + out->options_ = options; + out->footprint_ = footprint; + out->category_ = category; + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_writer.cpp b/lib/crowdb-tree/src/btree/source_writer.cpp new file mode 100644 index 000000000..d2c986576 --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_writer.cpp @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/external_allocation.h" +#include "source_admission.h" + +namespace crowdb::tree +{ +SourceAdmission::Writer::~Writer() +{ + release(); +} + +SourceAdmission::Writer::Writer(Writer &&other) noexcept +{ + *this = std::move(other); +} + +SourceAdmission::Writer &SourceAdmission::Writer::operator=(Writer &&other) noexcept +{ + if (this != &other) { + release(); + observation_ = std::move(other.observation_); + table_ = std::move(other.table_); + bound_ = other.bound_; + source_budget_ = std::move(other.source_budget_); + credit_ = std::move(other.credit_); + remaining_ = std::exchange(other.remaining_, {}); + planned_ = std::exchange(other.planned_, {}); + category_ = other.category_; + } + return *this; +} + +void SourceAdmission::Writer::release() noexcept +{ + planned_ = {}; + if (source_budget_) { + source_budget_->return_unused(std::exchange(remaining_, {}), category_); + source_budget_.reset(); + } + if (credit_) { + credit_->seal(); + credit_.reset(); + } + if (auto table = std::move(table_)) { + table->leave(); + } + observation_ = {}; +} + +Status SourceAdmission::Writer::upsert(Slice key, uint64_t slot, buffer value, bool *changed) +{ + return apply(key, slot, 0, std::move(value), false, changed); +} + +Status SourceAdmission::Writer::upsert_external(Slice key, uint64_t slot, uint8_t flags, buffer value, bool *changed) +{ + return apply(key, slot, flags, std::move(value), true, changed); +} + +Status SourceAdmission::Writer::apply(Slice key, uint64_t slot, uint8_t flags, buffer value, bool split, bool *changed) +try { + if (!table_ || planned_.entries != 0) { + return Status::invalid_argument("writer permit is no longer active"); + } + size_t capacity = 0; + if (credit_) { + if (split) { + const auto *allocation = value.external_allocation(); + if (allocation == nullptr || !allocation->charged_to(credit_)) { + if (value.size() > SIZE_MAX - kCellHeaderSize) { + return Status::invalid_argument("external value capacity overflow"); + } + capacity = value.size() + kCellHeaderSize; + capacity = capacity <= buffer::kInlineCap ? 0 : capacity; + } + } + else { + capacity = value.owned() ? value.capacity() : value.size(); + if (value.inlined() || (!value.owned() && capacity <= buffer::kInlineCap)) { + capacity = 0; + } + } + if (remaining_.entries == 0 || key.size() > remaining_.key_bytes || capacity > remaining_.payload_bytes || + key.size() > source_budget_->options().max_key_size || + (remaining_.max_key_size != 0 && key.size() > remaining_.max_key_size)) { + return Status::invalid_argument("write exceeds its admitted footprint"); + } + } + const bool applied = split ? table_->upsert_external(key, slot, flags, std::move(value), bound_, nullptr, credit_) + : table_->upsert(key, slot, std::move(value), bound_, nullptr, credit_); + if (credit_) { + --remaining_.entries; + remaining_.key_bytes -= key.size(); + remaining_.payload_bytes -= capacity; + } + if (changed != nullptr) { + *changed = applied; + } + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("admitted writer allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/source_writer_prepared.cpp b/lib/crowdb-tree/src/btree/source_writer_prepared.cpp new file mode 100644 index 000000000..47e983faa --- /dev/null +++ b/lib/crowdb-tree/src/btree/source_writer_prepared.cpp @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/external_allocation.h" +#include "source_admission.h" + +#include <cassert> + +namespace crowdb::tree +{ +Status SourceAdmission::Writer::prepare_write(Slice key, uint8_t flags, buffer value, Prepared *out) +try { + if (!table_ || !credit_ || out == nullptr || out->table_) { + return Status::invalid_argument("prepared write needs an admitted permit and empty output"); + } + const auto *external = value.external_allocation(); + const bool split = external != nullptr && external->charged_to(credit_); + const CellView cell(value.slice()); + if (!split && (!value.owned() || !cell.valid())) { + return Status::invalid_argument("prepared write needs owned encoded bytes or a charged external value"); + } + const size_t capacity = split || value.inlined() ? 0 : value.capacity(); + if (planned_.entries >= remaining_.entries || key.size() > remaining_.key_bytes - planned_.key_bytes || + capacity > remaining_.payload_bytes - planned_.payload_bytes || + key.size() > source_budget_->options().max_key_size || + (remaining_.max_key_size != 0 && key.size() > remaining_.max_key_size)) { + return Status::invalid_argument("prepared write exceeds its admitted footprint"); + } + const auto &options = source_budget_->options(); + const auto cell_flags = split ? flags : cell.flags(); + auto prepared = table_->prepare_write(key, std::move(value), cell_flags, + AllocationLease(credit_, capacity, AllocationKind::kPayload), bound_, + options.max_entries + options.max_gap_entries, credit_); + out->table_ = table_; + out->write_ = std::move(prepared); + out->footprint_ = {.entries = 1, .key_bytes = key.size(), .payload_bytes = capacity}; + ++planned_.entries; + planned_.key_bytes += key.size(); + planned_.payload_bytes += capacity; + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("admitted batch preparation allocation failed"); +} + +bool SourceAdmission::Writer::commit_write(Prepared write, uint64_t slot) noexcept +{ + assert(table_ && write.table_ == table_ && write.footprint_.entries == 1 && planned_.entries != 0); + auto observed = SourceTail::prepare(observation_.tail, write.write_); + const bool changed = table_->commit_write(std::move(write.write_), slot); + observed.complete(changed, slot); + --remaining_.entries; + remaining_.key_bytes -= write.footprint_.key_bytes; + remaining_.payload_bytes -= write.footprint_.payload_bytes; + --planned_.entries; + planned_.key_bytes -= write.footprint_.key_bytes; + planned_.payload_bytes -= write.footprint_.payload_bytes; + return changed; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/split_hint.cpp b/lib/crowdb-tree/src/btree/split_hint.cpp index 9e346e82b..20d1993ec 100644 --- a/lib/crowdb-tree/src/btree/split_hint.cpp +++ b/lib/crowdb-tree/src/btree/split_hint.cpp @@ -1,60 +1,131 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/tree_core.h" #include "crowdb-tree/btree/tree.h" +#include <algorithm> + namespace crowdb::tree { +namespace +{ +struct HintRange +{ + const KeyRange &tree; + const PageKeyRange *page = nullptr; + const PageKeyRange *base = nullptr; -std::optional<std::string> Crowdbtree::approximate_split_key() const + [[nodiscard]] bool before(Slice key) const + { + return (tree.start() && key.compare(Slice(*tree.start())) <= 0) || + (page != nullptr && page->lower && key.compare(Slice(*page->lower)) < 0) || + (base != nullptr && base->lower && key.compare(Slice(*base->lower)) < 0); + } + + [[nodiscard]] bool after(Slice key) const + { + return tree.at_or_after_end(key) || (page != nullptr && page->upper && key.compare(Slice(*page->upper)) >= 0) || + (base != nullptr && base->upper && key.compare(Slice(*base->upper)) >= 0); + } +}; + +template <class Key> std::optional<std::string> resident_hint(size_t count, Key key, const HintRange &range) { - auto guard = epoch_.enter(); - auto page_id = root_page_id_.load(std::memory_order_acquire); - const auto valid = [this](Slice key) { - return opt_.key_range.contains(key) && - (!opt_.key_range.start() || key.compare(Slice(*opt_.key_range.start())) > 0); - }; - for (size_t depth = 0; depth < 8 && page_id != kInvalidPageId; ++depth) { - PageBase *page = mapping_.get_resident(page_id); - for (size_t delta = 0; page != nullptr && page->next != nullptr && delta < 16; ++delta) { - page = page->next; - } - if (page == nullptr || page->next != nullptr) { - return std::nullopt; - } - if (page->type == page_type::kInnerBase) { - auto *inner = static_cast<InnerBase *>(page); - const auto view = inner->view(); - size_t first = 0; - size_t end = view.num_separators(); - while (first < end && !valid(view.separator_at(static_cast<uint32_t>(first)))) { - ++first; - } - while (first < end && !valid(view.separator_at(static_cast<uint32_t>(end - 1)))) { - --end; - } - if (first < end) { - return view.separator_at(static_cast<uint32_t>(first + ((end - first) / 2))).to_string(); - } - page_id = inner->child_for(opt_.key_range.start() ? Slice(*opt_.key_range.start()) : Slice()); - } - else if (page->type == page_type::kLeafBase) { - auto *leaf = static_cast<LeafBase *>(page); - size_t first = 0; - size_t end = leaf->count(); - while (first < end && !valid(leaf->key(first))) { - ++first; - } - while (first < end && !valid(leaf->key(end - 1))) { - --end; - } - return first < end ? std::optional(leaf->key(first + ((end - first) / 2)).to_string()) : std::nullopt; + size_t first = 0; + size_t end = count; + while (first < end) { + const auto mid = first + ((end - first) / 2); + if (range.before(key(mid))) { + first = mid + 1; } else { - return std::nullopt; + end = mid; + } + } + const auto begin = first; + end = count; + while (first < end) { + const auto mid = first + ((end - first) / 2); + if (range.after(key(mid))) { + end = mid; + } + else { + first = mid + 1; + } + } + return begin < end ? std::optional(key(begin + ((end - begin) / 2)).to_string()) : std::nullopt; +} + +std::optional<std::string> published_hint(const PublishedTreeState &view, const KeyRange &range) +{ + auto pid = view.root_page_id; + for (size_t depth = 0; depth < 8; ++depth) { + const auto page = view.mapping.get(pid); + if (!page) { + return {}; + } + if (page->inner) { + const auto &inner = *page->inner; + if (auto hint = resident_hint( + inner.separators.size(), [&](size_t i) { return Slice(inner.separators[i]); }, + HintRange{.tree = range})) { + return hint; + } + size_t child = 0; + if (range.start()) { + const auto start = Slice(*range.start()); + child = static_cast<size_t>(std::upper_bound(inner.separators.begin(), inner.separators.end(), start, + [](Slice key, const OwnedKey &separator) { + return key.compare(Slice(separator)) < 0; + }) - + inner.separators.begin()); + } + if (child >= inner.children.size()) { + return {}; + } + pid = inner.children[child]; + continue; + } + if (!page->leaf) { + return {}; + } + const auto &leaf = *page->leaf; + size_t visited = 0; + for (auto it = leaf.overlays.begin(); it != leaf.overlays.end() && visited < 16; ++it, ++visited) { + if (!*it) { + continue; + } + const auto entries = (*it)->entries(); + if (auto hint = resident_hint( + entries.size(), [&](size_t i) { return entries[i].key(); }, + HintRange{.tree = range, .page = &leaf.range})) { + return hint; + } + } + visited = 0; + for (const auto &base : leaf.bases) { + if (++visited > 16) { + break; + } + if (!base.resident) { + continue; + } + if (auto hint = resident_hint( + base.resident->count(), [&](size_t i) { return base.resident->key(i); }, + HintRange{.tree = range, .page = &leaf.range, .base = &base.range})) { + return hint; + } } + return {}; } - return std::nullopt; + return {}; } +} // namespace +std::optional<std::string> Crowdbtree::approximate_split_key() const +{ + return published_hint(*core_->capture(), opt_.key_range); +} } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_apply_operation.cpp b/lib/crowdb-tree/src/btree/tree_apply_operation.cpp new file mode 100644 index 000000000..32f39557c --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_apply_operation.cpp @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "tree_apply_operation_internal.h" +#include "tree_core_internal.h" + +namespace crowdb::tree +{ +TreeApplyOperation::State::State(std::weak_ptr<TreeCore> core, uint64_t generation, + std::shared_ptr<AdmittedBatch> batch, + std::shared_ptr<detail::AllocationReservation> metadata) + : weak_core(std::move(core)), + generation(generation), + batch(std::move(batch)), + metadata(std::move(metadata)), + node(&State::run, this) +{ +} + +std::optional<size_t> TreeApplyOperation::allocation_bound(size_t retained_metadata_bytes) noexcept +{ + return detail::AllocationBound(detail::shared_allocation_bound<State>()) + .add(detail::shared_allocation_bound<TreeApplyOperation>()) + .add(detail::shared_allocation_bound<ApplyWaiters>()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add_product(2, TreeOperationLifetime::ticket_allocation_bound()) + .add(retained_metadata_bytes) + .value(); +} + +Status TreeApplyOperation::bind_prepaid(std::shared_ptr<TreeCore> core, std::shared_ptr<AdmittedBatch> batch, + detail::ResourceBudget::Credit credit, std::shared_ptr<TreeApplyOperation> *out) +{ + auto metadata = detail::AllocationReservation::create(std::move(credit)); + auto state = + detail::reserved_shared<State>(metadata, core, core->capture()->generation, std::move(batch), metadata); + auto status = ApplyWaiters::create(core->binding()->account(), metadata, &state->waiters); + if (!status.ok()) { + return status; + } + *out = detail::reserved_construct<TreeApplyOperation>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) TreeApplyOperation(std::move(state)); }); + return Status::Ok(); +} + +Status TreeApplyOperation::State::arm() +{ + Phase expected = Phase::kPrepared; + if (!phase.compare_exchange_strong(expected, Phase::kArming, std::memory_order_acq_rel)) { + return Status::invalid_argument("apply permit must be armed exactly once"); + } + + struct Rollback + { + State &state; + + ~Rollback() + { + if (state.phase.load(std::memory_order_acquire) == Phase::kArming) { + state.apply_ticket.reset(); + state.backend_ticket.reset(); + state.phase.store(Phase::kPrepared, std::memory_order_release); + } + } + } rollback{*this}; + + auto core = weak_core.lock(); + if (!core) { + return Status::unavailable("apply Tree already closed"); + } + TreeCore::State::Ingress ingress(*core->state_); + if (ingress.owner == nullptr || core->capture()->generation != generation) { + return Status::unavailable("apply permit generation closed or replaced"); + } + auto &state = *core->state_; + auto status = Status::Ok(); + if (replay_control) { + status = state.replay_lifetime->acquire(metadata, &apply_ticket); + backend_ticket = apply_ticket; + } + else { + status = state.apply_lifetime->try_acquire(metadata, &apply_ticket); + if (status.ok()) { + status = state.lifetime->try_acquire(metadata, &backend_ticket); + } + } + if (!status.ok()) { + return status; + } + sources = state.sources.get(); + frontier = state.frontier.get(); + flush = state.flush.get(); + dispatcher = state.apply_dispatcher; + progress = state.apply_progress; + source_rotation = state.source_rotation; + work = state.work; + preparation = state.preparation; + preparation.store = backend_ticket->backend(core->async_backend()); + if (!replay_control) { + metadata->seal(); + } + phase.store(Phase::kArmed, std::memory_order_release); + return Status::Ok(); +} + +TreeApplyOperation::~TreeApplyOperation() = default; + +Status TreeApplyOperation::arm() +try { + return state_->arm(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("apply arm allocation failed"); +} + +void TreeApplyOperation::start(uint64_t slot, detail::CallbackComplete complete, void *context) noexcept +{ + auto state = state_; + state->start(slot, complete, context); +} + +const std::shared_ptr<detail::AllocationReservation> &TreeApplyOperation::allocation_budget() const noexcept +{ + return state_->metadata; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_apply_operation.h b/lib/crowdb-tree/src/btree/tree_apply_operation.h new file mode 100644 index 000000000..16839c0df --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_apply_operation.h @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "allocation_reservation.h" +#include "replay_controls.h" +#include "stdexec_adapter.h" +#include "tree_core_batch.h" + +namespace crowdb::tree +{ +class TreeCore; +class AdmittedBatch; + +// Preparation retains paid bytes without preventing Tree close. arm() is the +// last fallible boundary before proposal and pins exactly that generation. +// After start(), dropping the caller handle never cancels an accepted apply. +class TreeApplyOperation +{ + struct State; + + public: + ~TreeApplyOperation(); + TreeApplyOperation(const TreeApplyOperation &) = delete; + TreeApplyOperation &operator=(const TreeApplyOperation &) = delete; + [[nodiscard]] Status arm(); + void start(uint64_t slot, detail::CallbackComplete complete, void *context) noexcept; + [[nodiscard]] const std::shared_ptr<detail::AllocationReservation> &allocation_budget() const noexcept; + + private: + friend class TreeCore; + static constexpr size_t kReplayCallerControlBytes = 4096; + static Status bind_replay( + std::shared_ptr<TreeCore> core, uint64_t slot, size_t count, CoreWriteReader read, + detail::InlineFunction<std::shared_ptr<const void>(const std::shared_ptr<detail::AllocationReservation> &)> + retain, + size_t retained_metadata_bytes, std::shared_ptr<TreeApplyOperation> *out); + static std::optional<size_t> allocation_bound(size_t retained_metadata_bytes) noexcept; + static Status bind_prepaid(std::shared_ptr<TreeCore> core, std::shared_ptr<AdmittedBatch> batch, + detail::ResourceBudget::Credit credit, std::shared_ptr<TreeApplyOperation> *out); + + explicit TreeApplyOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_apply_operation_internal.h b/lib/crowdb-tree/src/btree/tree_apply_operation_internal.h new file mode 100644 index 000000000..2a7b240f1 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_apply_operation_internal.h @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "admitted_batch.h" +#include "apply_dispatcher.h" +#include "apply_progress.h" +#include "apply_waiters.h" +#include "flush_operation.h" +#include "replay_controls.h" +#include "snapshot/checkpoint_operation.h" +#include "source_rotation.h" +#include "tree_apply_operation.h" + +namespace crowdb::tree +{ +struct TreeApplyOperation::State : std::enable_shared_from_this<State> +{ + enum class Phase : uint8_t { kPrepared, kArming, kArmed, kStarted, kDone }; + enum class Wait : uint8_t { kNone, kProgress, kCredit, kApplyCredit, kPressureCredit, kWriteback, kFlush }; + + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + State *owner; + void set_value() const noexcept; + void set_value(FlushResult result) const noexcept; + void set_error(Status status) const noexcept; + void set_stopped() const noexcept; + }; + + using Connected = stdexec::connect_result_t<detail::CallbackSender, Receiver>; + + struct ProgressAttempt + { + ProgressAttempt(State *state, const ApplyProgress &progress, uint64_t sequence) + : request(progress, sequence), + connected(stdexec::connect(request.sender(), Receiver{state})) + { + } + + ApplyProgress::Request request; + Connected connected; + }; + + struct CreditAttempt + { + CreditAttempt(State *state, std::shared_ptr<detail::ResourceBudget::Account> account, + detail::BudgetClass category, size_t amount, std::shared_ptr<TreeWorkMetrics> work) + : request(std::move(account), category, amount, state->waiters->get(category)), + metrics(std::move(work), TreeStage::BudgetWait, tree_metrics_now_ns()), + connected(stdexec::connect(request.sender(), Receiver{state})) + { + } + + detail::ResourceBudget::Request request; + TreeMetricsOperation metrics; + Connected connected; + }; + + struct FlushAttempt + { + using Connected = stdexec::connect_result_t<FlushReportSender, Receiver>; + + FlushAttempt(State *state, std::shared_ptr<FlushOperation> flush) + : operation(std::move(flush)), + connected(stdexec::connect(operation->report_sender(), Receiver{state})) + { + } + + std::shared_ptr<FlushOperation> operation; + Connected connected; + }; + + struct ReplayInput + { + size_t count = 0; + CoreWriteReader read; + std::shared_ptr<const void> caller_owner; + SourceWriteOptions options; + AdmittedBatch::Plan bound; + }; + + struct WritebackAttempt + { + WritebackAttempt(State *state, std::shared_ptr<CheckpointOperation> value) + : operation(std::move(value)), + connected(stdexec::connect(operation->sender(), Receiver{state})) + { + } + + std::shared_ptr<CheckpointOperation> operation; + Connected connected; + }; + + State(std::weak_ptr<TreeCore> core, uint64_t generation, std::shared_ptr<AdmittedBatch> batch, + std::shared_ptr<detail::AllocationReservation> metadata); + Status arm(); + void start(uint64_t chosen_slot, detail::CallbackComplete callback, void *argument) noexcept; + static void run(void *context) noexcept; + void step(); + void start_flush(); + void await_progress(uint64_t sequence); + void await_credit(); + void credit_ready(); + bool prepare_replay_batch(); + void apply_credit_ready(); + void start_pressure(); + void pressure_credit_ready(); + void writeback_ready(); + void bind_replay_batch(detail::ResourceBudget::Credit grant); + void connect_flush(std::shared_ptr<FlushOperation> operation); + void wake(Status status) noexcept; + void finish(Status status) noexcept; + + std::shared_ptr<ApplyWaiters> waiters; + bool owns_replay_waiters = false; + ReplayControls::Lease replay_control; + std::optional<ReplayInput> replay_input; + uint64_t replay_slot = 0; + uint64_t pressure_pid = kInvalidPageId; + uint64_t replay_flushed = UINT64_MAX; + std::weak_ptr<TreeCore> weak_core; + const uint64_t generation; + std::shared_ptr<AdmittedBatch> batch; + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<TreeOperationLifetime::Ticket> apply_ticket; + std::shared_ptr<TreeOperationLifetime::Ticket> backend_ticket; + std::shared_ptr<SourceRotation> source_rotation; + std::shared_ptr<ApplyProgress> progress; + std::shared_ptr<ApplyDispatcher> dispatcher; + std::shared_ptr<TreeWorkMetrics> work; + SourceAdmission *sources = nullptr; + ApplyFrontier *frontier = nullptr; + FlushController *flush = nullptr; + FlushPreparationOptions preparation{}; + ApplyDispatcher::Node node; + std::atomic<Phase> phase{Phase::kPrepared}; + std::shared_ptr<State> self; + uint64_t slot = 0; + uint64_t observed = 0; + Wait waiting = Wait::kNone; + Status resumed; + bool flush_rejected = false; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + FlushReservePlan plan{}; + detail::FlushCredits credits; + size_t next_credit = 0; + std::optional<TreeMetricsOperation> admission_metrics; + std::optional<ProgressAttempt> progress_attempt; + std::optional<CreditAttempt> credit_attempt; + std::optional<WritebackAttempt> writeback_attempt; + std::optional<FlushAttempt> flush_attempt; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_apply_operation_steps.cpp b/lib/crowdb-tree/src/btree/tree_apply_operation_steps.cpp new file mode 100644 index 000000000..dcffc8897 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_apply_operation_steps.cpp @@ -0,0 +1,187 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/log.h" +#include "tree_apply_operation_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +void TreeApplyOperation::State::start(uint64_t chosen_slot, detail::CallbackComplete callback, void *argument) noexcept +{ + assert(callback != nullptr); + Phase expected = Phase::kArmed; + if (!phase.compare_exchange_strong(expected, Phase::kStarted, std::memory_order_acq_rel)) { + callback(argument, detail::CallbackSignal::kError, Status::invalid_argument()); + return; + } + if (replay_input && chosen_slot != replay_slot) { + phase.store(Phase::kArmed, std::memory_order_release); + callback(argument, detail::CallbackSignal::kError, + Status::invalid_argument("replay slot differs from its accepted input")); + return; + } + slot = chosen_slot; + complete = callback; + context = argument; + self = weak_from_this().lock(); + dispatcher->request(node, self); +} + +void TreeApplyOperation::State::run(void *context) noexcept +{ + auto &state = *static_cast<State *>(context); + try { + state.step(); + } + catch (const std::bad_alloc &) { + state.finish(Status::resource_exhausted()); + } + catch (...) { + state.finish(Status::internal_error()); + } +} + +void TreeApplyOperation::State::wake(Status status) noexcept +{ + resumed = std::move(status); + dispatcher->request(node, self); +} + +void TreeApplyOperation::State::step() +{ + if (waiting == Wait::kPressureCredit) { + pressure_credit_ready(); + return; + } + if (waiting == Wait::kWriteback) { + writeback_ready(); + return; + } + if (waiting == Wait::kApplyCredit) { + apply_credit_ready(); + return; + } + if (waiting == Wait::kCredit) { + credit_ready(); + return; + } + if (waiting == Wait::kFlush) { + flush_attempt.reset(); + waiting = Wait::kNone; + if (flush_rejected && resumed.code() == Code::kUnavailable) { + await_progress(observed); + return; + } + if (!resumed.ok()) { + finish(std::move(resumed)); + return; + } + } + else if (waiting == Wait::kProgress) { + progress_attempt.reset(); + waiting = Wait::kNone; + if (!resumed.ok()) { + finish(std::move(resumed)); + return; + } + } + if (sources->catalog()->generation != generation) { + finish(Status::internal_error("armed apply lost its pinned generation")); + return; + } + observed = progress->sequence(); + if (!frontier->check_slot(slot).ok()) { + await_progress(observed); + return; + } + if (replay_input && !prepare_replay_batch()) { + return; + } + auto status = batch->try_commit(*sources, *frontier, slot); + if (status.ok()) { + progress->advance(); + (void)source_rotation->request(); + finish(Status::Ok()); + return; + } + if (status.code() != Code::kUnavailable || batch->consumed()) { + finish(std::move(status)); + return; + } + if (frontier->contiguous().load(std::memory_order_acquire) <= sources->catalog()->floor) { + await_progress(observed); + return; + } + start_flush(); +} + +void TreeApplyOperation::State::start_flush() +{ + plan = {}; + std::shared_ptr<FlushOperation> operation; + auto status = flush->prepare_or_plan(preparation, &operation, &plan); + if (!status.ok()) { + if (status.code() == Code::kUnavailable) { + await_progress(observed); + } + else { + finish(std::move(status)); + } + return; + } + if (operation) { + connect_flush(std::move(operation)); + return; + } + next_credit = 0; + admission_metrics.emplace(work, TreeStage::FlushAdmissionWait, tree_metrics_now_ns()); + await_credit(); +} + +void TreeApplyOperation::State::connect_flush(std::shared_ptr<FlushOperation> operation) +{ + // A concurrent connection may own this repair. Observe after the planning + // gate released so our own planning wakeup cannot become a retry loop. + observed = progress->sequence(); + flush_rejected = false; + waiting = Wait::kFlush; + flush_attempt.emplace(this, std::move(operation)); + stdexec::start(flush_attempt->connected); +} + +void TreeApplyOperation::State::finish(Status status) noexcept +{ + if (phase.exchange(Phase::kDone, std::memory_order_acq_rel) == Phase::kDone) { + return; + } + auto retained = std::move(self); + if (admission_metrics) { + (void)admission_metrics->finish(status.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + } + progress_attempt.reset(); + credit_attempt.reset(); + flush_attempt.reset(); + writeback_attempt.reset(); + credits = {}; + plan = {}; + batch.reset(); + replay_input.reset(); + if (owns_replay_waiters) { + owns_replay_waiters = false; + waiters->release(); + progress->advance(); + } + waiters.reset(); + preparation.store.reset(); + backend_ticket.reset(); + apply_ticket.reset(); + try { + complete(context, status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError, + std::move(status)); + } + catch (...) { + CRB_LOG_ERROR("Tree accepted apply receiver threw after terminal"); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_apply_operation_wait.cpp b/lib/crowdb-tree/src/btree/tree_apply_operation_wait.cpp new file mode 100644 index 000000000..0c9e4050b --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_apply_operation_wait.cpp @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_apply_operation_internal.h" + +namespace crowdb::tree +{ +void TreeApplyOperation::State::Receiver::set_value() const noexcept +{ + if ((owner->waiting == Wait::kCredit || owner->waiting == Wait::kApplyCredit || + owner->waiting == Wait::kPressureCredit) && + owner->credit_attempt) { + (void)owner->credit_attempt->metrics.finish(TreeOutcome::Success, tree_metrics_now_ns()); + } + owner->wake(Status::Ok()); +} + +void TreeApplyOperation::State::Receiver::set_value(FlushResult result) const noexcept +{ + owner->wake(std::move(result.handoff_status)); +} + +void TreeApplyOperation::State::Receiver::set_error(Status status) const noexcept +{ + if ((owner->waiting == Wait::kCredit || owner->waiting == Wait::kApplyCredit || + owner->waiting == Wait::kPressureCredit) && + owner->credit_attempt) { + (void)owner->credit_attempt->metrics.finish(TreeOutcome::Error, tree_metrics_now_ns()); + } + else if (owner->waiting == Wait::kFlush) { + owner->flush_rejected = true; + } + owner->wake(std::move(status)); +} + +void TreeApplyOperation::State::Receiver::set_stopped() const noexcept +{ + if ((owner->waiting == Wait::kCredit || owner->waiting == Wait::kApplyCredit || + owner->waiting == Wait::kPressureCredit) && + owner->credit_attempt) { + (void)owner->credit_attempt->metrics.finish(TreeOutcome::Cancelled, tree_metrics_now_ns()); + } + owner->wake(Status::unavailable("accepted apply progress stopped")); +} + +void TreeApplyOperation::State::await_progress(uint64_t sequence) +{ + waiting = Wait::kProgress; + progress_attempt.emplace(this, *progress, sequence); + stdexec::start(progress_attempt->connected); +} + +void TreeApplyOperation::State::await_credit() +{ + waiting = Wait::kCredit; + credit_attempt.emplace(this, metadata->account(), detail::FlushReserveBounds::categories[next_credit], + plan.bounds.amounts[next_credit], work); + stdexec::start(credit_attempt->connected); +} + +void TreeApplyOperation::State::credit_ready() +{ + if (!resumed.ok()) { + finish(std::move(resumed)); + return; + } + if (!credit_attempt) { + finish(Status::internal_error("apply budget terminal has no registered request")); + return; + } + auto paid = credit_attempt->request.take_credit(); + credit_attempt.reset(); + waiting = Wait::kNone; + if (!paid) { + finish(Status::internal_error("completed apply budget request has no grant")); + return; + } + credits[next_credit++] = std::move(*paid); + if (next_credit != credits.size()) { + await_credit(); + return; + } + std::shared_ptr<FlushOperation> operation; + auto status = flush->prepare_prepaid(preparation, plan, std::move(credits), &operation); + if (admission_metrics) { + auto outcome = TreeOutcome::Error; + if (status.ok()) { + outcome = TreeOutcome::Success; + } + else if (status.code() == Code::kUnavailable) { + outcome = TreeOutcome::Stale; + } + (void)admission_metrics->finish(outcome, tree_metrics_now_ns()); + admission_metrics.reset(); + } + plan = {}; + if (!status.ok()) { + if (status.code() == Code::kUnavailable) { + // A changed immutable plan is a concrete publication transition. + // Re-enter once through the bounded dispatcher to capture its new bound. + dispatcher->request(node, self); + } + else { + finish(std::move(status)); + } + return; + } + connect_flush(std::move(operation)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_apply_pressure.cpp b/lib/crowdb-tree/src/btree/tree_apply_pressure.cpp new file mode 100644 index 000000000..cb8cecf17 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_apply_pressure.cpp @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_apply_operation_internal.h" +#include "tree_core_checkpoint_internal.h" + +namespace crowdb::tree +{ +void TreeApplyOperation::State::start_pressure() +{ + auto core = weak_core.lock(); + if (!core || (!core->local_backend() && !core->chunk_backend()) || !replay_input) { + finish(Status::invalid_argument("chosen replay pressure requires an owned local backend")); + return; + } + const auto view = core->publisher().capture(); + const auto selected = view->mapping.oldest_writeback_leaf(); + if (!selected) { + // The maintained candidate index is empty. Retained old readers or + // completed retirement tails may still own the paid gap payload. + waiting = Wait::kApplyCredit; + credit_attempt.emplace(this, metadata->account(), detail::BudgetClass::kGapRepair, + replay_input->bound.staging_bytes + replay_input->bound.source_bytes, work); + stdexec::start(credit_attempt->connected); + return; + } + pressure_pid = *selected; + const auto bytes = checkpoint_detail::pressure_control_bound(); + if (!bytes) { + finish(Status::resource_exhausted("pressure control bound overflow")); + return; + } + waiting = Wait::kPressureCredit; + credit_attempt.emplace(this, metadata->account(), detail::BudgetClass::kReclaim, *bytes, work); + stdexec::start(credit_attempt->connected); +} + +void TreeApplyOperation::State::pressure_credit_ready() +{ + if (!resumed.ok()) { + finish(std::move(resumed)); + return; + } + if (!credit_attempt) { + finish(Status::internal_error("pressure control has no grant waiter")); + return; + } + auto credit = credit_attempt->request.take_credit(); + credit_attempt.reset(); + waiting = Wait::kNone; + auto core = weak_core.lock(); + if (!credit || !core) { + finish(Status::internal_error("pressure control lost its accepted owner")); + return; + } + std::shared_ptr<CheckpointOperation> operation; + auto status = checkpoint_detail::prepare_pressure(*core, backend_ticket, pressure_pid, std::move(*credit), + dispatcher, preparation.publication, progress, waiters, + preparation.maintenance, &operation); + if (!status.ok()) { + finish(std::move(status)); + return; + } + observed = progress->sequence(); + waiting = Wait::kWriteback; + writeback_attempt.emplace(this, std::move(operation)); + stdexec::start(writeback_attempt->connected); +} + +void TreeApplyOperation::State::writeback_ready() +{ + writeback_attempt.reset(); + waiting = Wait::kNone; + if (!resumed.ok()) { + if (resumed.code() == Code::kUnavailable) { + await_progress(observed); + } + else { + finish(std::move(resumed)); + } + return; + } + if (auto core = weak_core.lock()) { + core->publisher().reclaim(); + } + dispatcher->request(node, self); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_apply_replay.cpp b/lib/crowdb-tree/src/btree/tree_apply_replay.cpp new file mode 100644 index 000000000..c0be579dc --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_apply_replay.cpp @@ -0,0 +1,167 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "tree_apply_operation_internal.h" +#include "tree_core_internal.h" + +namespace crowdb::tree +{ +Status TreeApplyOperation::bind_replay(std::shared_ptr<TreeCore> core, uint64_t slot, size_t count, + CoreWriteReader read, TreeCore::ReplayOwnerFactory retain, + size_t retained_metadata_bytes, std::shared_ptr<TreeApplyOperation> *out) +{ + if (out == nullptr || *out || (count != 0 && !retain) || retained_metadata_bytes > kReplayCallerControlBytes) { + return Status::invalid_argument("replay needs an immutable caller owner and bounded controls"); + } + const auto sources = core->state_->sources->catalog(); + const auto limits = sources->active_budget->options(); + AdmittedBatch::Plan plan; + auto status = AdmittedBatch::plan(core->config(), limits, core->load_limits().max_value_bytes, count, read, &plan); + if (!status.ok()) { + return status; + } + const auto complete = detail::AllocationBound(plan.staging_bytes) + .add(plan.source_bytes) + .add(allocation_bound(retained_metadata_bytes)) + .value(); + if (!complete || *complete > core->binding()->options().limits.gap_repair_bytes || + plan.footprint.entries > limits.max_gap_entries || plan.footprint.key_bytes > limits.max_gap_key_bytes) { + return Status::invalid_argument("chosen batch exceeds the configured replay progress bound"); + } + const auto completed = core->contiguous_slot(); + const bool current_gap = slot <= completed || slot - completed == 1; + auto controls = core->state_->replay_controls->acquire(current_gap); + if (!controls) { + return Status::unavailable("caller exceeded the bounded replay registration window"); + } + auto metadata = controls.budget(); + auto caller_owner = retain ? retain(metadata) : std::shared_ptr<const void>{}; + if (count != 0 && !caller_owner) { + return Status::invalid_argument("replay input owner is absent"); + } + auto state = + detail::reserved_shared<State>(metadata, core, sources->generation, std::shared_ptr<AdmittedBatch>{}, metadata); + state->waiters = core->state_->replay_waiters; + state->replay_control = std::move(controls); + state->replay_slot = slot; + state->replay_input.emplace(State::ReplayInput{.count = count, + .read = std::move(read), + .caller_owner = std::move(caller_owner), + .options = limits, + .bound = plan}); + auto result = detail::reserved_construct<TreeApplyOperation>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) TreeApplyOperation(state); }); + status = result->arm(); + if (status.ok()) { + *out = std::move(result); + } + return status; +} + +Status TreeCore::prepare_replay(uint64_t slot, size_t count, detail::InlineFunction<CoreWriteInput(size_t)> read, + std::shared_ptr<const void> caller_owner, size_t retained_metadata_bytes, + std::shared_ptr<TreeApplyOperation> *out) +{ + return prepare_replay_adopt( + slot, count, std::move(read), + [owner = std::move(caller_owner)](const std::shared_ptr<detail::AllocationReservation> &) { return owner; }, + retained_metadata_bytes, out); +} + +Status TreeCore::prepare_replay_adopt(uint64_t slot, size_t count, CoreWriteReader read, ReplayOwnerFactory retain, + size_t retained_metadata_bytes, std::shared_ptr<TreeApplyOperation> *out) +try { + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("replay ingress closed or replacing"); + } + return TreeApplyOperation::bind_replay(shared_from_this(), slot, count, std::move(read), std::move(retain), + retained_metadata_bytes, out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("prepaid replay control allocation failed"); +} + +bool TreeApplyOperation::State::prepare_replay_batch() +{ + const auto completed = frontier->contiguous().load(std::memory_order_acquire); + if (slot <= completed) { + finish(Status::Ok()); + return false; + } + if (slot - completed != 1) { + await_progress(observed); + return false; + } + if (!owns_replay_waiters) { + if (!waiters->try_acquire()) { + await_progress(observed); + return false; + } + owns_replay_waiters = true; + } + if (!replay_input) { + finish(Status::internal_error("replay continuation lost its caller input")); + return false; + } + const auto amount = replay_input->bound.staging_bytes + replay_input->bound.source_bytes; + auto grant = detail::ResourceBudget::try_reserve(metadata->account(), detail::BudgetClass::kGapRepair, amount); + if (grant) { + bind_replay_batch(std::move(*grant)); + return static_cast<bool>(batch); + } + if (completed > sources->catalog()->floor && replay_flushed != completed) { + replay_flushed = completed; + start_flush(); + return false; + } + start_pressure(); + return false; +} + +void TreeApplyOperation::State::apply_credit_ready() +{ + if (!resumed.ok()) { + finish(std::move(resumed)); + return; + } + if (!credit_attempt) { + finish(Status::internal_error("replay grant has no waiter")); + return; + } + auto grant = credit_attempt->request.take_credit(); + credit_attempt.reset(); + waiting = Wait::kNone; + if (!grant) { + finish(Status::internal_error("replay waiter completed without its grant")); + return; + } + bind_replay_batch(std::move(*grant)); + if (batch) { + dispatcher->request(node, self); + } +} + +void TreeApplyOperation::State::bind_replay_batch(detail::ResourceBudget::Credit grant) +{ + if (!replay_input) { + finish(Status::internal_error("replay grant has no input plan")); + return; + } + const auto &input = *replay_input; + auto staging = grant.split(input.bound.staging_bytes); + auto source = input.bound.source_bytes != 0 ? grant.split(input.bound.source_bytes) : std::nullopt; + if (!staging || (input.bound.source_bytes != 0 && !source)) { + finish(Status::internal_error("replay complete grant split failed")); + return; + } + auto status = AdmittedBatch::bind_prepaid(input.options, input.count, input.read, detail::BudgetClass::kGapRepair, + input.bound, std::move(*staging), + source ? std::move(*source) : detail::ResourceBudget::Credit{}, &batch); + if (!status.ok()) { + finish(std::move(status)); + return; + } + replay_input.reset(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_checkpoint_diagnostics.cpp b/lib/crowdb-tree/src/btree/tree_checkpoint_diagnostics.cpp new file mode 100644 index 000000000..a60a36302 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_checkpoint_diagnostics.cpp @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_internal.h" + +namespace crowdb::tree +{ +uint64_t TreeCore::last_snapshot_pages_written() const noexcept +{ + return state_->checkpoint_diagnostics.written_frames.load(std::memory_order_acquire); +} + +uint64_t TreeCore::snapshot_pages_total() const noexcept +{ + return state_->checkpoint_diagnostics.total_written_frames.load(std::memory_order_acquire); +} + +uint64_t Crowdbtree::last_snapshot_pages_written() const +{ + return core_->last_snapshot_pages_written(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core.cpp b/lib/crowdb-tree/src/btree/tree_core.cpp new file mode 100644 index 000000000..1be2ee8ee --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core.cpp @@ -0,0 +1,369 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/checkpoint_backend.h" +#include "backend/local/local_backend_state.h" +#include "backend/local/scheduled_page_store.h" +#include "btree/allocation_bound.h" +#include "runtime/tree_tuning_c_api.h" +#include "tree_apply_operation.h" +#include "tree_core_geometry.h" +#include "tree_core_internal.h" + +#include <algorithm> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +class EmptyBackend final : public AsyncPageStore +{ + public: + uint64_t submit_read(PageAddr /*addr*/, void * /*buf*/, size_t /*len*/, AsyncCompletion done) override + { + done.complete(Status::unavailable("in-memory Tree has no durable backing")); + return 0; + } + + uint64_t submit_write(PageAddr /*addr*/, const void * /*buf*/, size_t /*len*/, AsyncCompletion done) override + { + done.complete(Status::unavailable("in-memory Tree has no durable backing")); + return 0; + } + + Status submit_fsync(AsyncCompletion done) override + { + done.complete(Status::unavailable("in-memory Tree has no durable backing")); + return Status::Ok(); + } + + void cancel(uint64_t /*op_id*/) override + { + } +}; + +FlushBatchOptions flush_options(const Config &config, const TreeRuntimeBinding &binding, size_t max_key_size) +{ + FlushBatchOptions batch; + batch.parallelism = + std::max<size_t>(1, std::min(binding.cpu()->worker_count(), binding.options().limits.io_operations)); + if (config.execution.flush_parallelism != 0) { + batch.parallelism = config.execution.flush_parallelism; + } + batch.run_limits = {.max_entries = config.execution.run_max_entries, + .max_key_bytes = std::max(config.execution.run_max_key_bytes, max_key_size), + .key_block_bytes = config.execution.run_key_block_bytes}; + batch.max_run_bytes = config.execution.run_max_bytes; + batch.install_run_window = config.execution.install_run_window; + return batch; +} + +Status configure_maintenance(const Config &config, PageMaintenanceOperationOptions *maintenance) +{ + maintenance->visits_per_step = config.execution.maintenance_visits_per_step; + maintenance->max_visits = config.execution.maintenance_max_visits; + maintenance->max_candidates = config.execution.maintenance_max_candidates; + maintenance->max_steps = config.execution.maintenance_max_steps; + auto &pages = maintenance->publication.pages; + auto status = configure_leaf_geometry(config, &pages); + if (!status.ok()) { + return status; + } + pages.inline_value_bytes = config.max_inline_value != 0 ? config.max_inline_value : config.frame_bytes / 4; + pages.max_metadata_bytes = config.execution.maintenance_metadata_bytes; + // A split window scans at most one primary frame of minimum-size records. + // Its byte-aware outputs may span several ranges. Even an empty-key, + // empty-value record needs a slot and cell header; larger logical inputs + // continue in later steps without reserving a tree-sized merge window. + pages.max_work_entries = + std::min(pages.max_work_entries, + (config.frame_bytes - kFrameHeaderSize - kFrameTrailerSize) / (kLeafSlotSize + kCellHeaderSize)); + maintenance->publication.max_inner_children = config.inner_max_keys + 1ULL; + maintenance->publication.max_inner_bytes = pages.primary_page_bytes; + maintenance->publication.inner_merge_keys = + config.inner_merge_keys != 0 ? config.inner_merge_keys : std::max<uint32_t>(1, config.inner_max_keys / 4); + return Status::Ok(); +} + +} // namespace + +std::shared_ptr<PublishedTreeState> +TreeCore::State::empty_state(const std::shared_ptr<detail::AllocationReservation> &credit) +{ + auto leaf = detail::reserved_shared<PageState>(credit, credit); + auto page = detail::reserved_shared<PublishedPage>(credit); + page->leaf = std::move(leaf); + page->metrics.page_size_known = true; + page->metrics.kv_known = true; + auto initial = detail::reserved_shared<PublishedTreeState>(credit, credit); + const PageReplacement replacement{.page_id = 0, .expected = {}, .replacement = page}; + MappingView::CopyStats copied; + initial->mapping = initial->mapping.replace(std::span(&replacement, 1), copied, credit); + initial->root_page_id = 0; + initial->next_page_id = 1; + (void)apply_page_metrics(initial->stats, nullptr, &page->metrics); + initial->stats.height = 1; + return initial; +} + +TreeCore::State::State(Config options, std::shared_ptr<TreeRuntimeBinding> host) + : config(std::move(options)), + binding(std::move(host)) +{ +} + +TreeCore::State::~State() +{ + if (cache) { + cache->close_and_drain(); + } + // State teardown is a host boundary. Drop roots before waiting for this + // account's accepted retirement work; immutable external views may remain. + source_rotation.reset(); + flush.reset(); + publisher.reset(); + sources.reset(); + cache.reset(); + // Lifetime state also owns the backend. Release it before the final drain + // so registry and ledger roots cannot enqueue retirement after that drain. + replay_lifetime.reset(); + apply_lifetime.reset(); + lifetime.reset(); + replay_waiters.reset(); + replay_controls.reset(); + local.reset(); + chunk.reset(); + backend.reset(); + binding->drain(); +} + +Status TreeCore::State::initialize(TreeCoreBootstrap bootstrap) +{ + const auto credit = binding->lifecycle_budget(); + const auto source_bytes = SourceAdmission::allocation_bound(); + auto source_credit = source_bytes ? detail::ResourceBudget::try_reserve( + binding->account(), detail::BudgetClass::kAdmittedWriter, *source_bytes) + : std::nullopt; + if (!source_credit) { + return Status::resource_exhausted("Tree source lifecycle reserve unavailable"); + } + source_lifecycle = detail::AllocationReservation::create(std::move(*source_credit)); + auto initial = bootstrap.initial ? std::move(bootstrap.initial) : empty_state(credit); + if (initial->mapping.empty() && initial->root_page_id == kInvalidPageId) { + auto empty = detail::reserved_shared<PublishedTreeState>(credit, *empty_state(credit)); + empty->coverage = initial->coverage; + empty->generation = initial->generation; + auto source = detail::reserved_shared<PublishedSources>(credit, credit); + source->floor = initial->coverage; + empty->sources = std::move(source); + initial = std::move(empty); + } + local = std::move(bootstrap.local); + chunk = std::move(bootstrap.chunk); + if (local) { + backend = local->store; + } + else if (chunk) { + backend = chunk->store; + } + else if (config.async_page_store != nullptr || config.page_store != nullptr) { + auto status = ScheduledPageStore::create(config.page_store, config.async_page_store, config.backend_owner, + binding, credit, binding->backend_lane(), &backend); + if (!status.ok()) { + return status; + } + } + else { + backend = detail::reserved_shared<EmptyBackend>(credit); + } + auto status = ApplyFrontier::create(initial->coverage, ApplyFrontier::kMaxWindow, credit, &frontier); + if (!status.ok()) { + return status; + } + std::shared_ptr<const void> lifetime_owner = binding; + if (local) { + lifetime_owner = local; + } + else if (chunk) { + lifetime_owner = chunk; + } + lifetime = detail::reserved_unique<TreeOperationLifetime>(credit, binding->options().cpu_queue_limit, credit, + lifetime_owner); + apply_lifetime = detail::reserved_unique<TreeOperationLifetime>(credit, binding->options().cpu_queue_limit, credit, + lifetime_owner); + replay_lifetime = detail::reserved_unique<TreeOperationLifetime>(credit, binding->options().cpu_queue_limit + 2, + credit, lifetime_owner); + const auto replay_bytes = TreeApplyOperation::allocation_bound(TreeApplyOperation::kReplayCallerControlBytes); + if (!replay_bytes) { + return Status::resource_exhausted("replay control bound overflow"); + } + status = + ReplayControls::create(binding->account(), binding->options().cpu_queue_limit, *replay_bytes, &replay_controls); + if (!status.ok()) { + return status; + } + status = ApplyWaiters::create(binding->account(), credit, &replay_waiters); + if (!status.ok()) { + return status; + } + status = ApplyProgress::create((2 * binding->options().cpu_queue_limit) + 2, credit, &apply_progress); + if (!status.ok()) { + return status; + } + status = ApplyDispatcher::create(binding->cpu(), binding->lane(), 16, credit, &apply_dispatcher); + if (!status.ok()) { + return status; + } + SourceWriteOptions writers{.account = binding->account()}; + writers.max_entries = config.execution.source_max_entries; + writers.max_key_bytes = config.execution.source_max_key_bytes; + writers.max_gap_entries = config.execution.source_max_gap_entries; + writers.max_gap_key_bytes = config.execution.source_max_gap_key_bytes; + writers.max_key_size = config.max_key_size != 0 ? config.max_key_size : config.frame_bytes / 2; + sources = detail::reserved_unique<SourceAdmission>(credit, initial->coverage, initial->generation, writers, + source_lifecycle); + resources = binding->resource_metrics(); + work = binding->work_metrics(); + const PageCacheLimits cache_limits{.max_pages = + std::max<size_t>(1, std::min(config.execution.cache_max_pages, + config.buffer_pool_bytes / config.frame_bytes)), + .max_waiters = binding->options().cpu_queue_limit, + .max_resident_bytes = config.buffer_pool_bytes}; + cache = detail::reserved_shared<SharedPageCache>(credit, cache_limits, credit, resources, binding->cpu().get(), + binding->lane()); + const auto publication_bytes = detail::AllocationBound() + .add(PublicationCoordinator::allocation_bound(sources->catalog()->tables.size(), + 64, &initial->mapping, true)) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .value(); + if (!publication_bytes) { + return Status::resource_exhausted("initial publication metadata bound overflow"); + } + auto publication_credit = + detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kRouting, *publication_bytes); + if (!publication_credit) { + return Status::resource_exhausted("initial publication metadata reserve unavailable"); + } + // Recovered page roles scale with the mapping, independently of the fixed + // core lifecycle. Keep the prepaid participant headroom for later readers. + auto publication_metadata = detail::AllocationReservation::create(std::move(*publication_credit)); + publisher = detail::reserved_unique<PublicationCoordinator>( + publication_metadata, binding->options().cpu_queue_limit, initial->generation, work, + std::pmr::get_default_resource(), cache, + PublicationBootstrap{ + .metadata = publication_metadata, .initial = initial, .source_catalog = sources->catalog()}); + loads.max_frame_bytes = config.frame_bytes; + loads.max_blob_bytes = static_cast<size_t>(config.frame_bytes) + 64; + loads.max_inner_children = config.inner_max_keys + 1ULL; + loads.max_inner_key_bytes = config.frame_bytes; + loads.max_leaf_key_bytes = config.frame_bytes; + if (local && local->inherited) { + for (const auto &[pid, frame] : local->inherited->frames) { + (void)pid; + loads.max_frame_bytes = std::max<size_t>(loads.max_frame_bytes, frame.frame_bytes); + loads.max_blob_bytes = std::max<size_t>(loads.max_blob_bytes, frame.backing->length); + } + loads.max_inner_key_bytes = loads.max_frame_bytes; + loads.max_leaf_key_bytes = loads.max_frame_bytes; + local->inherited.reset(); + } + if (local) { + const auto unit = local->ledger->limits().iu_size; + loads.max_blob_bytes = ((loads.max_blob_bytes + unit - 1) / unit) * unit; + } + if (chunk) { + loads.max_frame_bytes = std::max(loads.max_frame_bytes, chunk->max_frame_bytes); + loads.max_blob_bytes = std::max(loads.max_blob_bytes, chunk->max_blob_bytes); + loads.max_blob_bytes = round_up_to_iu(loads.max_blob_bytes, chunk->iu_size); + loads.max_chunk_blob_bytes = loads.max_blob_bytes; + loads.max_inner_key_bytes = loads.max_frame_bytes; + loads.max_leaf_key_bytes = loads.max_frame_bytes; + } + reads.max_key_bytes = writers.max_key_size; + auto batch = flush_options(config, *binding, writers.max_key_size); + preparation = {.runtime = binding->cpu(), .store = backend, .loads = loads, .maintenance = {}, .cache = cache}; + preparation.cpu_progress = apply_dispatcher; + status = publisher->prepare_lane(batch.parallelism + 1, credit, &preparation.publication); + if (!status.ok()) { + return status; + } + status = configure_maintenance(config, &preparation.maintenance); + if (!status.ok()) { + return status; + } + flush = detail::reserved_unique<FlushController>(credit, *publisher, *binding->cpu(), *sources, + frontier->contiguous(), binding->lane(), batch, + FlushBatch::Maintenance{}, credit, apply_progress); + return Status::Ok(); +} + +TreeCore::TreeCore(detail::ReservedUnique<State> state) : state_(std::move(state)) +{ + state_->source_rotation = detail::reserved_shared<SourceRotation>( + state_->binding->lifecycle_budget(), *state_->sources, *state_->publisher, state_->apply_dispatcher, + state_->apply_progress, state_->binding->account(), + SourceRotationOptions{.bytes = state_->config.memtable_flush_bytes, + .entries = state_->config.memtable_flush_entries, + .tables = state_->config.max_memtable_count, + .source_window = std::min(state_->reads.max_memtables, state_->flush->source_window())}); + state_->source_rotation->set_flush( + [this](std::shared_ptr<FlushOperation> *out) { return prepare_flush(out); }, + [this](const PublishedTreeState &view, const SourceCatalog &catalog, WriteFootprint sealed) { + return state_->flush->check_rotation(state_->preparation, view, catalog, sealed); + }); + state_->source_rotation->set_async_flush(*state_->flush, [this](FlushPreparationOptions *out) { + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return Status::unavailable("Tree ingress closed"); + } + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto status = state_->lifetime->acquire(binding()->lifecycle_budget(), &ticket); + if (!status.ok()) { + return status; + } + *out = state_->preparation; + out->store = ticket->backend(async_backend()); + return Status::Ok(); + }); +} + +Status TreeCore::create(Config config, std::shared_ptr<TreeRuntimeBinding> binding, TreeCoreBootstrap bootstrap, + std::shared_ptr<TreeCore> *out) +try { + if (!binding || out == nullptr || *out || !config.runtime || config.frame_bytes < 512 || + config.buffer_pool_bytes < config.frame_bytes || config.inner_max_keys == 0) { + return Status::invalid_argument("Tree core requires runtime, bounded frames and empty output"); + } + if (config.page_store != nullptr && !bootstrap.local && !bootstrap.chunk) { + return Status::invalid_argument("runtime local PageStore requires recovery and an allocation domain"); + } + auto execution_status = validate_tree_execution(config.execution); + if (!execution_status.ok()) { + return execution_status; + } + if (config.execution.flush_parallelism > binding->cpu()->worker_count() || + config.execution.flush_parallelism > binding->options().limits.io_operations) { + return Status::invalid_argument("flush_parallelism exceeds CPU workers or Tree I/O permits"); + } + const auto credit = binding->lifecycle_budget(); + auto state = detail::reserved_unique<State>(credit, std::move(config), std::move(binding)); + auto status = state->initialize(std::move(bootstrap)); + if (!status.ok()) { + return status; + } + *out = detail::reserved_construct<TreeCore>( + credit, [&](auto *storage) { ::new (static_cast<void *>(storage)) TreeCore(std::move(state)); }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("Tree core lifecycle credit exhausted"); +} +catch (const std::invalid_argument &error) { + return Status::invalid_argument(error.what()); +} + +TreeCore::~TreeCore() +{ + close_and_drain(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core.h b/lib/crowdb-tree/src/btree/tree_core.h new file mode 100644 index 000000000..7375025bc --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core.h @@ -0,0 +1,200 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/flush_operation.h" +#include "btree/page_cache.h" +#include "btree/published_read.h" +#include "crowdb-tree/btree/tree.h" +#include "runtime/tree_operation_lifetime.h" +#include "runtime/tree_runtime_binding.h" + +namespace crowdb::tree +{ +struct BackingStatsSampler; + +namespace detail +{ +class ChunkRuntimePageStore; +class LegacyChunkReadStore; +} // namespace detail +struct LocalBackendState; +class CheckpointBackend; +class CheckpointOperation; +class ChunkCollectionOperation; +class LocalCompactionOperation; +struct LocalCompactionOptions; +class TreeApplyOperation; +struct CoreWriteInput; +struct ReadablePageCatalog; +struct CheckpointOptions; +struct SplitGeneration; +struct SplitSourceSession; + +struct TreeCoreBootstrap +{ + std::shared_ptr<const PublishedTreeState> initial; + std::shared_ptr<LocalBackendState> local; + std::shared_ptr<CheckpointBackend> chunk = nullptr; +}; + +// Borrowed inputs are copied into the operation before async submission. +struct CoreScanOptions +{ + Slice prefix; + Slice start; + Slice end; + Slice begin; + size_t limit = 0; + size_t byte_budget = 0; + uint64_t deadline_ms = 0; + bool reverse = false; + bool keys_only = false; + bool include_tombstones = false; + bool has_start_bound = false; + bool start_inclusive = false; + bool strict_byte_budget = false; +}; + +// Stable owner across immutable publication/source generations. The public +// facade retains it until accepted operations and all backend aliases drain. +class PublishedSnapshotContext; + +class TreeCore : public std::enable_shared_from_this<TreeCore> +{ + struct State; + + public: + static Status create(Config config, std::shared_ptr<TreeRuntimeBinding> binding, TreeCoreBootstrap bootstrap, + std::shared_ptr<TreeCore> *out); + static Status open(const Config &config, std::shared_ptr<TreeCore> *out); + static Status migrate_chunk(const Config &config, uint64_t expected_generation, std::shared_ptr<TreeCore> *out); + ~TreeCore(); + TreeCore(const TreeCore &) = delete; + TreeCore &operator=(const TreeCore &) = delete; + + // Prepare owns only input bytes and credit; arm fences its generation before + // proposal. + Status prepare_chunk_collection(std::shared_ptr<ChunkCollectionOperation> *out); + Status prepare_apply(size_t count, const detail::InlineFunction<CoreWriteInput(size_t)> &read, + detail::BudgetClass category, size_t retained_metadata_bytes, + std::shared_ptr<TreeApplyOperation> *out); + // The caller owns immutable input storage until terminal. Its owner is + // retained while the bounded replay window awaits its complete grant. + using ReplayOwnerFactory = + detail::InlineFunction<std::shared_ptr<const void>(const std::shared_ptr<detail::AllocationReservation> &)>; + Status prepare_replay_adopt(uint64_t slot, size_t count, detail::InlineFunction<CoreWriteInput(size_t)> read, + ReplayOwnerFactory retain, size_t retained_metadata_bytes, + std::shared_ptr<TreeApplyOperation> *out); + Status prepare_replay(uint64_t slot, size_t count, detail::InlineFunction<CoreWriteInput(size_t)> read, + std::shared_ptr<const void> caller_owner, size_t retained_metadata_bytes, + std::shared_ptr<TreeApplyOperation> *out); + Status apply(uint64_t slot, const Batch &batch); + Status apply_auto(const Batch &batch); + Status apply_encoded(uint64_t slot, std::vector<Crowdbtree::encoded_op> operations); + Status apply_external(uint64_t slot, std::vector<Crowdbtree::external_op> operations); + Status apply_external_inputs(uint64_t slot, size_t count, + const detail::InlineFunction<CoreWriteInput(size_t)> &read); + Status complete_noop(uint64_t slot); + void force_advance_slot(uint64_t slot) noexcept; + void request_gc_floor(uint64_t floor) noexcept; + [[nodiscard]] uint64_t requested_gc_floor() const noexcept; + Status clear(); + Status install_split_overlay(TreeCore &source, uint64_t journal_frontier); + Status clear_split_overlay(TreeCore &source); + Status begin_split_view(uint64_t *generation, uint64_t *journal_frontier); + Status release_split_view(uint64_t generation); + Status publish_split_view(uint64_t generation, uint64_t journal_frontier, TreeCore &destination, + const KeyRange &range); + Status install_portable(std::vector<leaf_entry> entries, uint64_t coverage); + Status prepare_import_credit(std::shared_ptr<detail::AllocationReservation> *out); + Status prepare_import_credit(size_t requested, std::shared_ptr<detail::AllocationReservation> *out); + Status install_readable(std::shared_ptr<const ReadablePageCatalog> catalog, + const std::shared_ptr<detail::AllocationReservation> &credit); + Status install_native(std::vector<NativeFrame> frames, NativeStreamInfo info); +#ifdef CROWDB_TREE_TEST_UTIL + void set_before_generation_install_for_tests(std::function<void()> callback); +#endif + + void get_async(Slice key, std::function<void(Status, GetView)> complete); + void scan_async(CoreScanOptions options, std::function<void(Status, ScanPackedBuf, bool)> complete); + Status get(Slice key, GetView *out); + Status scan(CoreScanOptions options, std::vector<scan_entry> *out, bool *truncated, ScanPackedBuf *packed = nullptr, + size_t *count = nullptr); + + Status prepare_flush(std::shared_ptr<FlushOperation> *out); + Status prepare_flush(std::shared_ptr<FlushOperation> *out, std::shared_ptr<TreeOperationLifetime::Ticket> accepted); + void flush_async(std::function<void(Status)> complete); + Status flush(); + Status prepare_checkpoint(const CheckpointOptions &options, std::shared_ptr<CheckpointOperation> *out); + Status prepare_checkpoint(const CheckpointOptions &options, std::shared_ptr<CheckpointOperation> *out, + std::shared_ptr<TreeOperationLifetime::Ticket> accepted); + // Runs on the shared runtime; staged completion never advances WAL coverage. + Status prepare_writeback(std::shared_ptr<CheckpointOperation> *out); + Status capture_snapshot(std::shared_ptr<PublishedSnapshotContext> *out); + Status acquire_snapshot_window(const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<TreeOperationLifetime::Ticket> *ticket, AsyncPageStore **backend); + Status reserve_snapshot_ids(const PublishedTreeState &expected, uint64_t count, + const std::shared_ptr<detail::AllocationReservation> &metadata); + Status prepare_compaction(const LocalCompactionOptions &options, std::shared_ptr<LocalCompactionOperation> *out); + Status compact_sparse_blocks(MergeGcStats *out); + size_t evict_clean_pages(PageCacheKind kind, size_t max_resident); + [[nodiscard]] uint64_t last_snapshot_pages_written() const noexcept; + [[nodiscard]] uint64_t snapshot_pages_total() const noexcept; + Status checkpoint(uint64_t *coverage = nullptr, uint64_t *sequence = nullptr); + void checkpoint_async(std::function<void(Status, uint64_t)> complete); + + // This boundary is host-only, never called from an accepted callback. + void close_and_drain() noexcept; + [[nodiscard]] bool closed() const noexcept; + [[nodiscard]] const Config &config() const noexcept; + [[nodiscard]] const std::shared_ptr<TreeRuntimeBinding> &binding() const noexcept; + [[nodiscard]] const std::shared_ptr<SharedPageCache> &cache() const noexcept; + [[nodiscard]] const std::shared_ptr<LocalBackendState> &local_backend() const noexcept; + [[nodiscard]] const std::shared_ptr<CheckpointBackend> &chunk_backend() const noexcept; + [[nodiscard]] uint64_t durable_coverage() const noexcept; + [[nodiscard]] uint64_t snapshot_sequence() const noexcept; + [[nodiscard]] AsyncPageStore *async_backend() const noexcept; + [[nodiscard]] PageLoadLimits load_limits() const noexcept; + [[nodiscard]] PublishedReadLimits read_limits() const noexcept; + [[nodiscard]] std::shared_ptr<const PublishedTreeState> capture() const; + [[nodiscard]] PublicationCoordinator &publisher() const noexcept; + [[nodiscard]] TreeOperationLifetime &operation_lifetime() const noexcept; + [[nodiscard]] TreeStatsSnapshot state_stats() const; + [[nodiscard]] BackingStatsSnapshot backing_stats() const noexcept; + [[nodiscard]] BackingStatsSampler backing_sampler() const noexcept; + [[nodiscard]] uint64_t contiguous_slot() const noexcept; + [[nodiscard]] uint64_t max_seen_slot() const noexcept; + [[nodiscard]] size_t memtable_count() const; + [[nodiscard]] SourceMemoryStats source_stats() const; + [[nodiscard]] std::array<ResourceUsage, 7> admission_resource_usage() const noexcept; + [[nodiscard]] size_t frozen_table_count() const; + Status enable_work_histogram(uint32_t stage); + + private: + void get_impl(Slice key, std::function<void(Status, GetView)> complete, bool dispatch); + void scan_impl(CoreScanOptions options, std::function<void(Status, ScanPackedBuf, bool)> complete, bool dispatch); + friend class detail::ChunkRuntimePageStore; + static Status open_chunk(const Config &config, std::shared_ptr<TreeCore> *out); + static Status open_chunk_migration(const Config &config, + const std::shared_ptr<detail::LegacyChunkReadStore> &legacy, + uint64_t snapshot_sequence, std::shared_ptr<TreeCore> *out); + friend class TreeApplyOperation; + friend class TreeGenerationLease; + friend class PreparedSplitOverlay; + // Candidate already owns a fully validated immutable graph. Null creates + // a new empty generation. All failures precede old-source closure. + Status install_generation(std::shared_ptr<const PublishedTreeState> candidate, + const SplitGeneration *split = nullptr); + Status prepare_readable(std::shared_ptr<const ReadablePageCatalog> catalog, + const std::shared_ptr<detail::AllocationReservation> &credit, + std::shared_ptr<const PublishedTreeState> *out); + Status capture_snapshot(std::shared_ptr<const PublishedTreeState> view, + std::shared_ptr<PublishedSnapshotContext> *out); + Status install_split_baseline(const SplitSourceSession &session, uint64_t identity, uint64_t end); + // Caller holds both split gates and the source ingress. + Status install_split_overlay_guarded(TreeCore &source, uint64_t journal_frontier); + explicit TreeCore(detail::ReservedUnique<State> state); + detail::ReservedUnique<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_access.h b/lib/crowdb-tree/src/btree/tree_core_access.h new file mode 100644 index 000000000..3701d157a --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_access.h @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/tree_core.h" + +namespace crowdb::tree +{ +struct TreeCoreAccess +{ + [[nodiscard]] static TreeCore *get(Crowdbtree &tree) noexcept + { + return tree.core_.get(); + } + + static void record_compaction(Crowdbtree &tree, const MergeGcStats &stats, uint64_t elapsed_ns) + { + tree.record_compaction_metrics(stats, elapsed_ns); + } + + static Status create_volatile(const Config &config, std::unique_ptr<Crowdbtree> *out) + { + std::shared_ptr<TreeRuntimeBinding> binding; + auto status = TreeRuntimeBinding::create(config.runtime, config.runtime_binding, &binding); + std::shared_ptr<TreeCore> core; + if (status.ok()) { + status = TreeCore::create(config, binding, {}, &core); + } + if (status.ok()) { + out->reset(new Crowdbtree(config, std::move(core))); + } + return status; + } +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_admitted.cpp b/lib/crowdb-tree/src/btree/tree_core_admitted.cpp new file mode 100644 index 000000000..cc2a02de9 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_admitted.cpp @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "admitted_batch.h" +#include "allocation_bound.h" +#include "tree_apply_operation.h" +#include "tree_core_internal.h" + +namespace crowdb::tree +{ +Status TreeCore::prepare_apply(size_t count, const CoreWriteReader &read, detail::BudgetClass category, + size_t retained_metadata_bytes, std::shared_ptr<TreeApplyOperation> *out) +try { + if (out == nullptr || *out || + (category != detail::BudgetClass::kAdmittedWriter && category != detail::BudgetClass::kGapRepair)) { + return Status::invalid_argument("apply admission requires an empty output and apply partition"); + } + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("apply admission is closed"); + } + const auto sources = state_->sources->catalog(); + const auto &limits = sources->active_budget->options(); + AdmittedBatch::Plan plan; + auto status = AdmittedBatch::plan(config(), limits, state_->loads.max_value_bytes, count, read, &plan); + if (!status.ok()) { + return status; + } + const auto control = TreeApplyOperation::allocation_bound(retained_metadata_bytes); + const auto complete = detail::AllocationBound(plan.staging_bytes).add(plan.source_bytes).add(control).value(); + if (!complete || plan.footprint.entries > limits.max_gap_entries || + plan.footprint.key_bytes > limits.max_gap_key_bytes || + *complete > binding()->options().limits.gap_repair_bytes) { + return Status::invalid_argument("complete proposal cannot fit the reserved gap progress partition"); + } + if (category == detail::BudgetClass::kAdmittedWriter && *complete > binding()->options().limits.writer_bytes) { + return Status::invalid_argument("complete proposal cannot fit the ordinary writer partition"); + } + auto grant = detail::ResourceBudget::try_reserve(binding()->account(), category, *complete); + if (!grant) { + return Status::unavailable("complete apply grant is currently occupied"); + } + auto staging = grant->split(plan.staging_bytes); + auto source = plan.source_bytes != 0 ? grant->split(plan.source_bytes) : std::nullopt; + if (!staging || (plan.source_bytes != 0 && !source)) { + return Status::internal_error("complete apply grant could not be partitioned"); + } + std::shared_ptr<AdmittedBatch> batch; + status = AdmittedBatch::bind_prepaid(limits, count, read, category, plan, std::move(*staging), + source ? std::move(*source) : detail::ResourceBudget::Credit{}, &batch); + if (!status.ok()) { + return status; + } + return TreeApplyOperation::bind_prepaid(shared_from_this(), std::move(batch), std::move(*grant), out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("apply admission allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_apply.cpp b/lib/crowdb-tree/src/btree/tree_core_apply.cpp new file mode 100644 index 000000000..f2cd4ae7b --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_apply.cpp @@ -0,0 +1,136 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_core_batch.h" +#include "tree_core_internal.h" + +namespace crowdb::tree +{ +Status TreeCore::apply(uint64_t slot, const Batch &batch) +try { + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return Status::unavailable("Tree ingress closed"); + } + auto status = apply_source_batch( + state_->config, *state_->sources, *state_->frontier, *state_->lifetime, state_->loads.max_value_bytes, slot, + state_->automatic_slot, batch.ops.size(), + [&](size_t index) { + const auto &operation = batch.ops[index]; + return CoreWriteInput{.key = Slice(operation.key), + .value = operation.kind == OpKind::kDelete ? Slice{} : Slice(operation.value), + .movable = nullptr, + .flags = static_cast<uint8_t>(operation.kind), + .encoded = false}; + }, + state_->apply_progress.get(), state_->source_rotation.get()); + if (status.ok()) { + state_->apply_progress->advance(); + state_->source_rotation->wait(state_->source_rotation->request()); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("Tree apply allocation failed"); +} + +Status TreeCore::apply_auto(const Batch &batch) +try { + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return Status::unavailable("Tree ingress closed"); + } + auto status = apply_source_batch( + state_->config, *state_->sources, *state_->frontier, *state_->lifetime, state_->loads.max_value_bytes, + std::nullopt, state_->automatic_slot, batch.ops.size(), + [&](size_t index) { + const auto &operation = batch.ops[index]; + return CoreWriteInput{.key = Slice(operation.key), + .value = operation.kind == OpKind::kDelete ? Slice{} : Slice(operation.value), + .movable = nullptr, + .flags = static_cast<uint8_t>(operation.kind), + .encoded = false}; + }, + state_->apply_progress.get(), state_->source_rotation.get()); + if (status.ok()) { + state_->apply_progress->advance(); + state_->source_rotation->wait(state_->source_rotation->request()); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("Tree apply allocation failed"); +} + +Status TreeCore::apply_encoded(uint64_t slot, std::vector<Crowdbtree::encoded_op> operations) +try { + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return Status::unavailable("Tree ingress closed"); + } + auto status = apply_source_batch( + state_->config, *state_->sources, *state_->frontier, *state_->lifetime, state_->loads.max_value_bytes, slot, + state_->automatic_slot, operations.size(), + [&](size_t index) { + auto &operation = operations[index]; + return CoreWriteInput{.key = Slice(operation.key), + .value = operation.cell.slice(), + .movable = &operation.cell, + .flags = 0, + .encoded = true}; + }, + state_->apply_progress.get(), state_->source_rotation.get()); + if (status.ok()) { + state_->apply_progress->advance(); + state_->source_rotation->wait(state_->source_rotation->request()); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("Tree apply allocation failed"); +} + +Status TreeCore::apply_external(uint64_t slot, std::vector<Crowdbtree::external_op> operations) +{ + return apply_external_inputs(slot, operations.size(), [&](size_t index) { + auto &operation = operations[index]; + return CoreWriteInput{.key = Slice(operation.key), + .value = operation.value.slice(), + .movable = &operation.value, + .flags = operation.flags, + .encoded = false}; + }); +} + +Status TreeCore::apply_external_inputs(uint64_t slot, size_t count, const CoreWriteReader &read) +try { + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return Status::unavailable("Tree ingress closed"); + } + auto status = apply_source_batch(state_->config, *state_->sources, *state_->frontier, *state_->lifetime, + state_->loads.max_value_bytes, slot, state_->automatic_slot, count, read, + state_->apply_progress.get(), state_->source_rotation.get()); + if (status.ok()) { + state_->apply_progress->advance(); + state_->source_rotation->wait(state_->source_rotation->request()); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("Tree apply allocation failed"); +} + +Status TreeCore::complete_noop(uint64_t slot) +{ + return apply(slot, {}); +} + +void TreeCore::force_advance_slot(uint64_t slot) noexcept +{ + State::Ingress admission(*state_); + if (admission.owner != nullptr) { + state_->frontier->force_advance(slot); + state_->apply_progress->advance(); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_batch.cpp b/lib/crowdb-tree/src/btree/tree_core_batch.cpp new file mode 100644 index 000000000..e0737b775 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_batch.cpp @@ -0,0 +1,292 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_core_batch.h" + +#include "allocation_bound.h" +#include "apply_progress.h" +#include "crowdb-tree/external_allocation.h" +#include "flush_runtime.h" +#include "source_rotation.h" + +#include <algorithm> +#include <cassert> +#include <cstring> +#include <numeric> + +namespace crowdb::tree +{ +bool primary_write_fits(const Config &config, size_t key_bytes, size_t value_bytes, bool tombstone) noexcept +{ + constexpr size_t overhead = kFrameHeaderSize + kFrameTrailerSize; + constexpr size_t inner = overhead + (2 * sizeof(uint64_t)) + kInnerSlotSize; + constexpr size_t leaf = overhead + kLeafSlotSize; + if (config.frame_bytes < inner || config.frame_bytes < leaf || key_bytes > config.frame_bytes - inner || + key_bytes > config.frame_bytes - leaf) { + return false; + } + const auto inline_bytes = config.max_inline_value == 0 ? config.frame_bytes / 4 : config.max_inline_value; + const auto available = config.frame_bytes - leaf - key_bytes; + if (tombstone) { + return kCellHeaderSize <= available; + } + if (value_bytes > inline_bytes) { + return kOverflowCellSize <= available; + } + return available >= kCellHeaderSize && value_bytes <= available - kCellHeaderSize; +} + +namespace +{ +struct StagedWrite +{ + Slice key; + AllocationLease charge; + buffer value; + uint8_t flags = 0; +}; + +size_t payload_capacity(const CoreWriteInput &input, const std::shared_ptr<AllocationBudget> &credit) +{ + if (input.encoded ? CellView(input.value).is_tombstone() : (input.flags & kFlagTombstone) != 0) { + return 0; + } + if (input.encoded && input.movable->owned()) { + return input.movable->inlined() ? 0 : input.movable->capacity(); + } + if (!input.encoded && input.movable != nullptr) { + const auto *allocation = input.movable->external_allocation(); + if (allocation != nullptr && allocation->charged_to(credit)) { + return 0; + } + } + const auto bytes = input.encoded ? input.value.size() : input.value.size() + kCellHeaderSize; + return bytes <= buffer::kInlineCap ? 0 : bytes; +} + +StagedWrite stage(CoreWriteInput input, uint64_t slot, const std::shared_ptr<detail::AllocationReservation> &credit) +{ + StagedWrite result{.key = input.key, + .charge = AllocationLease(credit, payload_capacity(input, credit), AllocationKind::kPayload), + .value = {}, + .flags = input.flags}; + if (input.encoded ? CellView(input.value).is_tombstone() : (input.flags & kFlagTombstone) != 0) { + result.value = encode_cell_buf(slot, OpKind::kDelete); + return result; + } + if (input.encoded) { + result.value = input.movable->owned() ? std::move(*input.movable) : input.movable->clone(); + return result; + } + if (input.movable != nullptr) { + const auto *allocation = input.movable->external_allocation(); + if (allocation != nullptr && allocation->charged_to(credit)) { + result.value = std::move(*input.movable); + return result; + } + } + result.value = buffer::alloc(input.value.size(), kCellHeaderSize); + for (size_t index = 0; index < sizeof(slot); ++index) { + result.value.data()[index] = static_cast<uint8_t>(slot >> (8 * index)); + } + result.value.data()[sizeof(slot)] = input.flags; + if (!input.value.empty()) { + std::memcpy(result.value.data() + kCellHeaderSize, input.value.data(), input.value.size()); + } + return result; +} + +Status validate_inputs(const Config &config, const SourceWriteOptions &options, size_t max_value_bytes, uint64_t slot, + size_t count, const CoreWriteReader &read) +{ + if (count > options.max_entries + options.max_gap_entries) { + return Status::resource_exhausted("batch exceeds bounded ingress operation count"); + } + for (size_t index = 0; index < count; ++index) { + const auto input = read(index); + if (!config.key_range.contains(input.key) || input.key.size() > options.max_key_size || + input.value.size() > SIZE_MAX - kCellHeaderSize) { + return Status::invalid_argument("batch key or value exceeds Tree limits"); + } + if (input.encoded) { + const CellView cell(input.value); + if (!cell.valid() || cell.slot() != slot) { + return Status::invalid_argument("encoded cell slot must match its batch"); + } + if ((cell.flags() & ~kFlagTombstone) != 0 || + (!cell.is_tombstone() && cell.value().size() > max_value_bytes)) { + return Status::invalid_argument("encoded value exceeds Tree value limits"); + } + } + else if ((input.flags & ~kFlagTombstone) != 0 || + ((input.flags & kFlagTombstone) == 0 && input.value.size() > max_value_bytes)) { + return Status::invalid_argument("value exceeds Tree value limits"); + } + const bool tombstone = + input.encoded ? CellView(input.value).is_tombstone() : (input.flags & kFlagTombstone) != 0; + const auto value_bytes = input.encoded ? CellView(input.value).value().size() : input.value.size(); + if (!primary_write_fits(config, input.key.size(), value_bytes, tombstone)) { + return Status::invalid_argument("batch key/cell cannot fit a primary leaf or inner separator"); + } + } + return Status::Ok(); +} + +detail::ReservedVector<size_t> select_inputs(size_t count, const CoreWriteReader &read, + const std::shared_ptr<detail::AllocationReservation> &credit) +{ + detail::ReservedVector<size_t> order(count, detail::ReservationAllocator<size_t>(credit)); + std::iota(order.begin(), order.end(), size_t{0}); + std::sort(order.begin(), order.end(), [&](size_t left, size_t right) { + const auto compare = read(left).key.compare(read(right).key); + return compare == 0 ? left < right : compare < 0; + }); + size_t selected = 0; + for (size_t index = 0; index < order.size(); ++index) { + if (index + 1 == order.size() || read(order[index]).key != read(order[index + 1]).key) { + order[selected++] = order[index]; + } + } + order.resize(selected); + return order; +} + +Status claim_auto_slot(ApplyFrontier &frontier, std::atomic<uint64_t> &automatic, uint64_t *out) noexcept +{ + auto previous = automatic.load(std::memory_order_acquire); + for (;;) { + const auto latest = std::max(previous, frontier.max_seen()); + if (latest == UINT64_MAX) { + return Status::resource_exhausted(); + } + const auto next = latest + 1; + auto status = frontier.check_slot(next); + if (!status.ok()) { + return status; + } + if (automatic.compare_exchange_weak(previous, next, std::memory_order_acq_rel)) { + *out = next; + return Status::Ok(); + } + } +} + +Status write_inputs(SourceAdmission &sources, ApplyFrontier &frontier, uint64_t completed, + std::optional<uint64_t> supplied_slot, std::atomic<uint64_t> &automatic, + const detail::ReservedVector<size_t> &order, const CoreWriteReader &read, + const std::shared_ptr<detail::AllocationReservation> &credit, detail::BudgetClass category, + ApplyProgress *progress, SourceRotation *rotation) +{ + WriteFootprint footprint{.entries = order.size()}; + detail::AllocationBound payload; + for (const auto index : order) { + const auto input = read(index); + footprint.key_bytes += input.key.size(); + payload.add(payload_capacity(input, credit)); + footprint.max_key_size = std::max(footprint.max_key_size, input.key.size()); + } + const auto bytes = payload.value(); + if (!bytes) { + return Status::resource_exhausted("batch payload allocation bound overflow"); + } + footprint.payload_bytes = *bytes; + SourceAdmission::Writer writer; + Status status; + for (;;) { + const auto observed = progress != nullptr ? progress->sequence() : 0; + bool paused = false; + status = sources.try_admit(completed, footprint, &writer, category, &paused); + if (status.ok()) { + break; + } + if (!paused && rotation != nullptr && category == detail::BudgetClass::kAdmittedWriter && + status.code() == Code::kUnavailable && !detail::FlushRuntime::on_worker_thread()) { + const auto source = sources.catalog()->active->id(); + rotation->wait(rotation->request()); + if (sources.catalog()->active->id() != source) { + continue; + } + } + if (!paused || progress == nullptr || detail::FlushRuntime::on_worker_thread()) { + return status; + } + status = progress->wait(observed); + if (!status.ok()) { + return status; + } + } + detail::ReservedVector<SourceAdmission::Writer::Prepared> prepared{ + detail::ReservationAllocator<SourceAdmission::Writer::Prepared>(credit)}; + prepared.reserve(order.size()); + for (const auto index : order) { + auto input = stage(read(index), supplied_slot.value_or(0), writer.allocation_budget()); + input.charge = {}; + prepared.emplace_back(); + status = writer.prepare_write(input.key, input.flags, std::move(input.value), &prepared.back()); + if (!status.ok()) { + return status; + } + } + uint64_t slot = supplied_slot.value_or(0); + if (!supplied_slot) { + status = claim_auto_slot(frontier, automatic, &slot); + if (!status.ok()) { + return status; + } + } + // The source permit covers every prepared write through completion. All + // payloads, node towers and maximum-size COW descriptors already exist. + for (auto &write : prepared) { + writer.commit_write(std::move(write), slot); + } + status = frontier.complete(slot); + assert(status.ok()); + return status; +} +} // namespace + +Status apply_source_batch(const Config &config, SourceAdmission &sources, ApplyFrontier &frontier, + TreeOperationLifetime &lifetime, size_t max_value_bytes, + std::optional<uint64_t> supplied_slot, std::atomic<uint64_t> &automatic, size_t count, + const CoreWriteReader &read, ApplyProgress *progress, SourceRotation *rotation) +{ + auto status = supplied_slot ? frontier.check_slot(*supplied_slot) : Status::Ok(); + if (!status.ok()) { + return status; + } + if (count == 0) { + uint64_t slot = supplied_slot.value_or(0); + if (!supplied_slot) { + status = claim_auto_slot(frontier, automatic, &slot); + } + return status.ok() ? frontier.complete(slot) : status; + } + const auto catalog = sources.catalog(); + const auto &options = catalog->active_budget->options(); + status = validate_inputs(config, options, max_value_bytes, supplied_slot.value_or(0), count, read); + if (!status.ok()) { + return status; + } + const auto bytes = detail::shared_allocation_bound<detail::AllocationReservation>() + + TreeOperationLifetime::ticket_allocation_bound() + + (count * (sizeof(size_t) + sizeof(SourceAdmission::Writer::Prepared))); + const auto completed = frontier.contiguous().load(std::memory_order_acquire); + const bool fills_gap = supplied_slot && *supplied_slot > completed && *supplied_slot <= frontier.max_seen(); + const auto category = fills_gap ? detail::BudgetClass::kGapRepair : detail::BudgetClass::kAdmittedWriter; + auto reserved = detail::ResourceBudget::try_reserve(options.account, category, bytes); + if (!reserved) { + return Status::unavailable("batch ingress credit unavailable"); + } + const auto credit = detail::AllocationReservation::create(std::move(*reserved)); + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + status = lifetime.acquire(credit, &ticket); + if (!status.ok()) { + return status; + } + auto order = select_inputs(count, read, credit); + if (supplied_slot && *supplied_slot <= catalog->floor) { + return frontier.complete(*supplied_slot); + } + return write_inputs(sources, frontier, completed, supplied_slot, automatic, order, read, credit, category, progress, + rotation); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_batch.h b/lib/crowdb-tree/src/btree/tree_core_batch.h new file mode 100644 index 000000000..6ded42db7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_batch.h @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "apply_frontier.h" +#include "crowdb-tree/btree/tree.h" +#include "inline_function.h" +#include "runtime/tree_operation_lifetime.h" +#include "source_admission.h" + +namespace crowdb::tree +{ +class ApplyProgress; +class SourceRotation; + +struct CoreWriteInput +{ + Slice key; + Slice value; + buffer *movable = nullptr; + uint8_t flags = 0; + bool encoded = false; +}; + +using CoreWriteReader = detail::InlineFunction<CoreWriteInput(size_t)>; + +// Validate both the primary leaf cell and a possible inner separator before +// admitting an input whose configured key/inline limits may exceed a frame. +[[nodiscard]] bool primary_write_fits(const Config &config, size_t key_bytes, size_t value_bytes, + bool tombstone) noexcept; + +Status apply_source_batch(const Config &config, SourceAdmission &sources, ApplyFrontier &frontier, + TreeOperationLifetime &lifetime, size_t max_value_bytes, + std::optional<uint64_t> supplied_slot, std::atomic<uint64_t> &automatic, size_t count, + const CoreWriteReader &read, ApplyProgress *progress = nullptr, + SourceRotation *rotation = nullptr); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_block_gc.cpp b/lib/crowdb-tree/src/btree/tree_core_block_gc.cpp new file mode 100644 index 000000000..a95ada71c --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_block_gc.cpp @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/block_account.h" +#include "backend/local/local_backend_state.h" +#include "backend/local/local_block_gc.h" +#include "btree/allocation_bound.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "crowdb-common/log.h" + +namespace crowdb::tree::checkpoint_detail +{ +void reclaim_free_blocks(TreeCore &core, std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<AllocationBudget> backend) noexcept +try { + auto *store = core.config().page_store; + const auto &local = core.local_backend(); + const auto unit = store != nullptr ? local_block_gc_unit(*store) : 0; + if (unit == 0 || !local || !local->ledger) { + return; + } + const LocalBlockGcLimits limits; + if (unit > limits.max_deleted_bytes) { + return; + } + using namespace detail; + const auto gc_bound = local_block_gc_metadata_bound(limits); + if (!gc_bound) { + return; + } + const auto bound = AllocationBound(gc_bound.value()).add(shared_allocation_bound<AllocationReservation>()).value(); + if (!bound) { + return; + } + const bool owned_credit = !metadata; + if (!metadata) { + auto reserved = ResourceBudget::try_reserve(core.binding()->account(), BudgetClass::kReclaim, *bound); + if (!reserved) { + return; + } + metadata = AllocationReservation::create(std::move(*reserved)); + } + + struct Seal + { + std::shared_ptr<AllocationReservation> credit; + std::shared_ptr<AllocationBudget> backend; + + ~Seal() + { + seal_backend_budget(backend); + if (credit) { + credit->seal(); + } + } + } seal{.credit = owned_credit ? metadata : nullptr, .backend = {}}; + + if (!backend) { + auto status = + reserve_backend_operation(store, core.binding()->account(), + {.workers = core.binding()->backend_cpu()->worker_count()}, &seal.backend); + if (!status.ok()) { + return; + } + backend = seal.backend; + } + + auto cursor = local->gc_cursor.load(std::memory_order_relaxed); + const auto result = collect_free_local_blocks(*store, *local->ledger, cursor, limits, metadata, backend); + (void)local->gc_cursor.compare_exchange_strong(cursor, result.complete ? 0 : result.next_block, + std::memory_order_relaxed); + if (!result.status.ok() && result.status.code() != Code::kResourceExhausted && + result.status.code() != Code::kUnavailable) { + CRB_LOG_WARN("post-checkpoint block cleanup deferred after failure: {}", result.status.to_string()); + } +} +catch (const std::bad_alloc &) { + // The durable checkpoint is already committed. Retained leases and the + // cursor make a later bounded cleanup pass safe after allocation failure. + return; +} +catch (const std::exception &error) { + CRB_LOG_WARN("post-checkpoint block cleanup deferred after exception: {}", error.what()); +} +} // namespace crowdb::tree::checkpoint_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_block_gc_async.cpp b/lib/crowdb-tree/src/btree/tree_core_block_gc_async.cpp new file mode 100644 index 000000000..b812fb7c0 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_block_gc_async.cpp @@ -0,0 +1,239 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/block_account.h" +#include "backend/local/local_sparse_gc.h" +#include "btree/allocation_bound.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "crowdb-common/log.h" + +#include <cassert> + +namespace crowdb::tree::checkpoint_detail +{ +namespace +{ +struct Cleanup : std::enable_shared_from_this<Cleanup> +{ + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + std::shared_ptr<Cleanup> owner; + + void set_value() noexcept + { + auto keep = std::move(owner); + keep->completed(Status::Ok()); + } + + void set_error(Status status) noexcept + { + auto keep = std::move(owner); + keep->completed(std::move(status)); + } + + void set_stopped() noexcept + { + auto keep = std::move(owner); + keep->completed(Status::unavailable()); + } + }; + + using Connected = stdexec::connect_result_t<detail::CallbackSender, Receiver>; + + struct Attempt + { + Attempt(detail::CallbackSender sender, std::shared_ptr<Cleanup> owner) + : connected(stdexec::connect(sender, Receiver{.owner = std::move(owner)})) + { + } + + Connected connected; + }; + + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<AllocationBudget> backend; + std::shared_ptr<LocalBackendState> local; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + TreeCore *core = nullptr; + std::shared_ptr<Cleanup> self; + bool owns_gate = false; + ProtectedExtent exclusion; + std::shared_ptr<CheckpointOperation> operation; + std::optional<Attempt> attempt; + Status outcome; + size_t remaining = 2; + + void completed(Status status) noexcept + { + outcome = std::move(status); + self = weak_from_this().lock(); + assert(self); + Status submitted; + try { + submitted = binding->backend_cpu()->try_submit(binding->backend_lane(), &Cleanup::run, this); + } + catch (...) { + submitted = Status::resource_exhausted(); + } + if (!submitted.ok()) { + self.reset(); + } + } + + void advance() + { + if (auto *memory = dynamic_cast<MemPageStore *>(core->config().page_store)) { + if (!outcome.ok()) { + return; + } + const auto result = + collect_free_local_blocks(*memory, *local->ledger, local->gc_cursor.load(), {}, metadata); + local->gc_cursor.store(result.complete ? 0 : result.next_block, std::memory_order_relaxed); + if (result.status.ok() && !result.complete) { + completed(Status::Ok()); + } + return; + } + auto *store = dynamic_cast<BlockPageStore *>(core->config().page_store); + if (store == nullptr || store->block_size() == 0 || !outcome.ok()) { + return; + } + attempt.reset(); + operation.reset(); + if (!exclusion.owner) { + reclaim_free_blocks(*core, metadata, backend); + auto candidate = select_sparse_local_block( + *store, *local->ledger, local->gc_cursor.load(std::memory_order_relaxed), metadata, 0.5, backend); + local->gc_cursor.store(candidate.next_block, std::memory_order_relaxed); + if (!candidate.status.ok()) { + return; + } + exclusion = std::move(candidate.exclusion); + } + if (remaining == 0) { + core->cache()->evict(); + // Quiescent publication roots can otherwise keep old extents + // until the epoch batch reaches its normal collection threshold. + core->publisher().reclaim(); + const LocalBlockGcLimits limits{.max_blocks = 1, .max_deleted_bytes = store->block_size()}; + (void)collect_free_local_blocks(*store, *local->ledger, exclusion.address / store->block_size(), limits, + metadata, backend); + return; + } + --remaining; + auto options = defaults(*core); + options.cause = TreeWorkCause::GarbageCollection; + options.write.frames.exclusion = {.address = exclusion.address, .length = exclusion.length}; + auto status = core->prepare_checkpoint(options, &operation); + if (!status.ok()) { + return; + } + attempt.emplace(operation->sender(), shared_from_this()); + stdexec::start(attempt->connected); + } + + ~Cleanup() + { + detail::seal_backend_budget(backend); + if (owns_gate) { + local->gc_active.store(false, std::memory_order_release); + } + if (metadata) { + metadata->seal(); + } + } + + static void run(void *context) noexcept + { + auto owner = std::move(static_cast<Cleanup *>(context)->self); + // The accepted checkpoint's Ticket keeps Core/backend lifetime valid + // while close or generation replacement drains this continuation. + try { + owner->advance(); + } + catch (const std::bad_alloc &) { + // The existing anchors and live backing still protect their bytes. + return; + } + catch (...) { + CRB_LOG_WARN("sparse block cleanup deferred after an exception"); + } + } +}; +} // namespace + +void schedule_free_blocks(TreeCore &core, std::shared_ptr<TreeOperationLifetime::Ticket> ticket) noexcept +try { + auto local = core.local_backend(); + auto *store = core.config().page_store; + const LocalBlockGcLimits limits; + const auto unit = store != nullptr ? local_block_gc_unit(*store) : 0; + if (!ticket || !local || !local->ledger || unit == 0 || unit > limits.max_deleted_bytes || + local->gc_active.exchange(true, std::memory_order_acq_rel)) { + return; + } + + struct Gate + { + LocalBackendState *local; + + ~Gate() + { + if (local != nullptr) { + local->gc_active.store(false, std::memory_order_release); + } + } + } gate{local.get()}; + + using namespace detail; + const auto bound = AllocationBound(local_block_gc_metadata_bound(limits).value_or(SIZE_MAX)) + .add(shared_allocation_bound<Cleanup>() + shared_allocation_bound<AllocationReservation>()) + .value(); + if (!bound) { + return; + } + auto reserved = ResourceBudget::try_reserve(core.binding()->account(), BudgetClass::kReclaim, *bound); + if (!reserved) { + return; + } + auto metadata = AllocationReservation::create(std::move(*reserved)); + auto cleanup = reserved_shared<Cleanup>(metadata); + cleanup->metadata = std::move(metadata); + auto backend_status = + reserve_backend_operation(store, core.binding()->account(), + {.workers = core.binding()->backend_cpu()->worker_count()}, &cleanup->backend); + if (!backend_status.ok()) { + return; + } + cleanup->local = std::move(local); + cleanup->binding = core.binding(); + cleanup->ticket = std::move(ticket); + cleanup->core = &core; + cleanup->owns_gate = true; + gate.local = nullptr; + cleanup->self = cleanup; + Status status; + try { + status = + cleanup->binding->backend_cpu()->try_submit(cleanup->binding->backend_lane(), &Cleanup::run, cleanup.get()); + } + catch (...) { + cleanup->self.reset(); + throw; + } + if (!status.ok()) { + cleanup->self.reset(); + } +} +catch (const std::bad_alloc &) { + // This checkpoint is already durable; a later pass can retry admission. + return; +} +catch (const std::exception &error) { + CRB_LOG_WARN("post-checkpoint cleanup submission deferred: {}", error.what()); +} +catch (...) { + CRB_LOG_WARN("post-checkpoint cleanup submission deferred after unknown exception"); +} +} // namespace crowdb::tree::checkpoint_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint.cpp b/lib/crowdb-tree/src/btree/tree_core_checkpoint.cpp new file mode 100644 index 000000000..f9ed1ec0b --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint.cpp @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "btree/tree_core_internal.h" +#include "crowdb-common/log.h" +#include "snapshot/operation_wait.h" + +namespace crowdb::tree +{ +Status TreeCore::prepare_checkpoint(const CheckpointOptions &options, std::shared_ptr<CheckpointOperation> *out) +{ + return prepare_checkpoint(options, out, {}); +} + +Status TreeCore::prepare_checkpoint(const CheckpointOptions &options, std::shared_ptr<CheckpointOperation> *out, + std::shared_ptr<TreeOperationLifetime::Ticket> accepted) +try { + State::Ingress admission(*state_); + if ((!accepted && admission.owner == nullptr) || (!local_backend() && !chunk_backend()) || + async_backend() == nullptr) { + return Status::unavailable("checkpoint requires an open reloadable backend"); + } + if (out == nullptr || *out) { + return Status::invalid_argument(); + } + using namespace detail; + const auto bound = AllocationBound(CheckpointOperation::control_allocation_bound()) + .add(shared_allocation_bound<checkpoint_detail::Preparation>()) + .add_product(2, shared_allocation_bound<checkpoint_detail::Credits>()) + .add(shared_allocation_bound<AllocationReservation>()) + .add(TreeOperationLifetime::ticket_allocation_bound()) + .value(); + if (!bound) { + return Status::resource_exhausted(); + } + auto control = ResourceBudget::try_reserve(binding()->account(), BudgetClass::kReclaim, *bound); + auto finish = ResourceBudget::try_reserve(binding()->account(), BudgetClass::kReclaim, + CheckpointOperation::finish_allocation_bound() + + shared_allocation_bound<AllocationReservation>()); + if (!control || !finish) { + return Status::resource_exhausted("checkpoint finish credit unavailable"); + } + auto credit = AllocationReservation::create(std::move(*control)); + auto preparation = reserved_shared<checkpoint_detail::Preparation>(credit); + preparation->control = credit; + preparation->finish = AllocationReservation::create(std::move(*finish)); + preparation->core = this; + preparation->options = options; + preparation->structure_options = state_->preparation.maintenance; + preparation->options.diagnostics = &state_->checkpoint_diagnostics; + preparation->cpu_progress = state_->apply_dispatcher; + if (accepted) { + // A finite child shares its parent's accepted lifetime; no second + // per-Tree operation slot is consumed. + preparation->ticket = std::move(accepted); + } + else { + auto status = operation_lifetime().acquire(credit, &preparation->ticket); + if (!status.ok()) { + return status; + } + } + return CheckpointOperation::create( + preparation, publisher(), local_backend(), preparation->ticket->backend(async_backend()), binding()->cpu(), + binding()->lane(), preparation->options, {.bytes = credit, .io = {}, .cpu_progress = state_->apply_dispatcher}, + preparation->finish, {}, + [preparation](std::shared_ptr<FlushOperation> *handoff) { + return preparation->core->prepare_flush(handoff, preparation->ticket); + }, + [preparation](const PublishedTreeState &view, const CapturedWritePlan *plan, CheckpointWork *work) { + return preparation->work(view, plan, work); + }, + out, [preparation] { checkpoint_detail::schedule_free_blocks(*preparation->core, preparation->ticket); }, {}, + [progress = state_->apply_progress] { progress->advance(); }, {}, chunk_backend(), + [preparation](std::shared_ptr<const PublishedTreeState> view, uint64_t pid, + std::shared_ptr<PageSplitOperation> *operation) { + return preparation->structure(std::move(view), pid, operation); + }); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status TreeCore::prepare_writeback(std::shared_ptr<CheckpointOperation> *out) +{ + auto options = checkpoint_detail::defaults(*this); + options.mode = CheckpointMode::Writeback; + options.cause = TreeWorkCause::CachePressure; + options.select_oldest_page = true; + return prepare_checkpoint(options, out); +} + +Status TreeCore::checkpoint(uint64_t *coverage, uint64_t *sequence) +{ + std::shared_ptr<CheckpointOperation> operation; + auto status = prepare_checkpoint(checkpoint_detail::defaults(*this), &operation); + if (status.ok()) { + status = detail::wait_snapshot_operation(operation); + } + if (status.ok()) { + if (coverage != nullptr) { + *coverage = operation->result().durable_coverage; + } + if (sequence != nullptr) { + *sequence = operation->result().snapshot_sequence; + } + operation.reset(); + } + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint_async.cpp b/lib/crowdb-tree/src/btree/tree_core_checkpoint_async.cpp new file mode 100644 index 000000000..0b56ff7a8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint_async.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "btree/tree_core_internal.h" +#include "crowdb-common/log.h" + +namespace crowdb::tree +{ +namespace +{ +struct CheckpointCall; + +struct CheckpointReceiver +{ + using receiver_concept = stdexec::receiver_tag; + CheckpointCall *owner; + void set_value() const noexcept; + void set_error(Status status) const noexcept; + void set_stopped() const noexcept; +}; + +struct CheckpointCall : std::enable_shared_from_this<CheckpointCall> +{ + using Connected = stdexec::connect_result_t<detail::CallbackSender, CheckpointReceiver>; + + explicit CheckpointCall(std::shared_ptr<CheckpointOperation> child) + : operation(std::move(child)), + connected(stdexec::connect(operation->sender(), CheckpointReceiver{this})) + { + } + + void start() + { + self = shared_from_this(); + stdexec::start(connected); + } + + void finish(Status status) noexcept + { + auto retained = std::move(self); + try { + callback(std::move(status), operation->result().durable_coverage); + } + catch (...) { + CRB_LOG_ERROR("Tree checkpoint receiver threw after terminal"); + } + } + + std::shared_ptr<CheckpointOperation> operation; + std::function<void(Status, uint64_t)> callback; + Connected connected; + std::shared_ptr<CheckpointCall> self; +}; + +void CheckpointReceiver::set_value() const noexcept +{ + owner->finish(Status::Ok()); +} + +void CheckpointReceiver::set_error(Status status) const noexcept +{ + owner->finish(std::move(status)); +} + +void CheckpointReceiver::set_stopped() const noexcept +{ + owner->finish(Status::unavailable()); +} +} // namespace + +void TreeCore::checkpoint_async(std::function<void(Status, uint64_t)> complete) +{ + if (!complete) { + return; + } + Status status; + uint64_t fallback_coverage = 0; + std::shared_ptr<CheckpointCall> call; + try { + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + status = Status::unavailable("Tree ingress closed"); + } + else { + fallback_coverage = durable_coverage(); + using namespace detail; + constexpr auto bytes = + shared_allocation_bound<CheckpointCall>() + shared_allocation_bound<AllocationReservation>(); + auto reserved = ResourceBudget::try_reserve(binding()->account(), BudgetClass::kReclaim, bytes); + if (!reserved) { + status = Status::resource_exhausted("checkpoint callback credit unavailable"); + } + else { + auto credit = AllocationReservation::create(std::move(*reserved)); + std::shared_ptr<CheckpointOperation> operation; + status = prepare_checkpoint(checkpoint_detail::defaults(*this), &operation); + if (status.ok()) { + call = reserved_shared<CheckpointCall>(credit, std::move(operation)); + call->callback.swap(complete); + credit->seal(); + } + } + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted(); + } + if (call) { + call->start(); + return; + } + complete(std::move(status), fallback_coverage); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint_budget.cpp b/lib/crowdb-tree/src/btree/tree_core_checkpoint_budget.cpp new file mode 100644 index 000000000..a9097d638 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint_budget.cpp @@ -0,0 +1,212 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/checkpoint_backend.h" +#include "backend/local/block_account.h" +#include "btree/allocation_bound.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "snapshot/captured_rebase.h" +#include "snapshot/native_frame_read.h" +#include "snapshot/writeback_selection.h" + +#include <algorithm> +#include <bit> + +namespace crowdb::tree::checkpoint_detail +{ +Credits::~Credits() +{ + if (snapshot) { + snapshot->seal(); + } + detail::seal_backend_budget(budget.backend); + if (budget.bytes) { + budget.bytes->seal(); + } + if (budget.io) { + budget.io->seal(); + } +} + +Preparation::~Preparation() +{ + if (control) { + control->seal(); + } + if (finish) { + finish->seal(); + } +} + +CheckpointOptions defaults(const TreeCore &core) +{ + CheckpointOptions options; + const auto &config = core.config(); + options.write.frames.encode.frame_bytes = config.frame_bytes; + options.write.frames.encode.inline_value_bytes = + config.max_inline_value != 0 ? config.max_inline_value : config.frame_bytes / 4; + options.write.write.max_frame_bytes = config.frame_bytes; + options.write.write.compression = config.compression; + if (core.local_backend()) { + options.write.write.iu_size = core.local_backend()->ledger->limits().iu_size; + options.write.write.buffer_alignment = std::bit_ceil(std::max<size_t>(4096, options.write.write.iu_size)); + } + if (core.chunk_backend()) { + options.write.write.chunk_backed = true; + options.write.write.iu_size = core.chunk_backend()->iu_size; + options.write.write.buffer_alignment = std::bit_ceil(std::max<size_t>(4096, options.write.write.iu_size)); + } + return options; +} + +Status Preparation::plan_work(const PublishedTreeState &view, const CapturedWritePlan *plan, WorkPlan *out) const +{ + using namespace detail; + if (plan != nullptr && plan->reused_primary_frames > plan->primary_frames) { + return Status::invalid_argument("checkpoint reuse count exceeds its primary count"); + } + auto frames = options.write.frames; + if (options.select_oldest_page) { + auto status = select_writeback_page(view, frames, &frames.selected_page); + if (!status.ok()) { + return status; + } + if (!frames.selected_page) { + return Status::unavailable("writeback selection has no candidate"); + } + } + const auto root = view.mapping.get(view.root_page_id); + const size_t depth = root && !frames.selected_page ? root->level + 1ULL : 1; + if (depth > frames.max_depth) { + return Status::resource_exhausted("checkpoint route depth limit"); + } + frames.max_depth = depth; + frames.page_metadata_bytes = view.mapping.copy_bounds().max_leaf_bytes; + frames.plan_only = plan == nullptr; + auto loads = core->load_limits(); + loads.max_inflight = 1; + // Captured encoders borrow inline values and retain typed raw overflow + // dependencies. They never request an assembled overflow value. + loads.max_overflow_pages = 1; + const auto workers = core->binding()->cpu()->worker_count(); + const auto retained_frames = frames.encode.max_sources; + auto bound = AllocationBound(CapturedFrameStream::allocation_bound(frames).value_or(SIZE_MAX)) + .add(AsyncStorePageLoader::read_reserve_bound(loads, depth, workers, retained_frames, 0, 0)) + .add(shared_allocation_bound<AllocationReservation>()); + if (plan == nullptr) { + bound.add(CapturedLayoutPlanner::allocation_bound(view.next_page_id, options.metadata.max_segments)); + } + else { + if (options.mode == CheckpointMode::Durable) { + if (core->chunk_backend()) { + bound.add( + core->chunk_backend()->checkpoint_allocation_bound(plan->frame_count_upper, plan->reference_heads)); + } + else { + bound.add(LocalCheckpointOperation::allocation_bound(plan->segments, options.write.write.iu_size)); + } + } + const auto reservation = core->chunk_backend() ? core->chunk_backend()->reservation_allocation_bound() + : ExtentLedger::reservation_metadata_bound(); + auto write = options.write; + write.frames = frames; + bound + .add(CapturedWriteOperation::allocation_bound(write, plan->frame_count_upper, + plan->reused_dependency_entries, plan->reference_heads)) + .add(CapturedRebaseOperation::allocation_bound( + plan->primary_frames - plan->reused_primary_frames, frames.encode.max_sources, + core->publisher().tracks_cache_residency(), plan->rebase_batches)) + .add(PageWriteOperation::allocation_bound(options.write.write)) + .add_product(plan->primary_frames, reservation) + .add_product(plan->new_overflow_frames, reservation) + .add_product(plan->resident_dependency_entries, reservation) + .add_product(plan->relocated_dependency_entries, reservation) + .add(shared_allocation_bound<ReservedOverflowIds>()); + } + if (plan != nullptr && plan->relocated_dependency_entries != 0) { + // Completed decoder callbacks may still own their controls while the + // next worker begins a read. Each worker can retain one such tail. + bound.add_product(workers + 1, NativeFrameRead::allocation_bound(loads).value_or(SIZE_MAX)); + } + const auto bytes = bound.value(); + if (!bytes) { + return Status::resource_exhausted("checkpoint reservation bound overflow"); + } + const auto write_calls = plan != nullptr ? AllocationBound(plan->frame_count_upper) + .add(plan->resident_dependency_entries) + .add(plan->relocated_dependency_entries) + .add(plan->segments) + .add(2) + .value() + : std::optional<size_t>{0}; + if (!write_calls) { + return Status::resource_exhausted("backend checkpoint write count overflow"); + } + const auto max_write = AllocationBound(options.write.write.max_frame_bytes) + .add(kDurableBlobHeader) + .add(options.write.write.iu_size) + .value(); + if (!max_write) { + return Status::resource_exhausted("backend write size overflow"); + } + const BackendOperationLimits backend{.max_io_bytes = std::max(loads.max_blob_bytes, *max_write), + .write_calls = *write_calls, + .workers = workers, + .barriers = plan != nullptr && options.mode == CheckpointMode::Durable}; + const auto backend_bytes = backend_operation_bound(core->async_backend(), backend); + if (!backend_bytes || !AllocationBound(*bytes).add(*backend_bytes).value()) { + return Status::resource_exhausted("complete checkpoint window bound overflow"); + } + *out = {.bytes = *bytes, .backend_bytes = *backend_bytes, .loads = loads, .backend = backend}; + return Status::Ok(); +} + +Status Preparation::bind_work(const WorkPlan &plan, detail::ResourceBudget::Credit bytes, + detail::ResourceBudget::Credit backend, detail::ResourceBudget::Credit io, + CheckpointWork *out) +{ + using namespace detail; + const auto account = core->binding()->account(); + if (out == nullptr || bytes.amount() != plan.bytes || bytes.allocated() || + !bytes.belongs_to(account, BudgetClass::kReclaim) || backend.amount() != plan.backend_bytes || + backend.allocated() || !backend.belongs_to(account, BudgetClass::kRouting) || io.amount() != 1 || + io.allocated() || !io.belongs_to(account, BudgetClass::kIo)) { + return Status::invalid_argument("checkpoint needs its complete prepaid partitions"); + } + auto phase = reserved_shared<Credits>(control); + phase->budget.bytes = AllocationReservation::create(std::move(bytes)); + phase->budget.io = reserved_shared<AllocationReservation>(phase->budget.bytes, std::move(io)); + phase->budget.work_metrics = core->publisher().work_metrics(); + phase->budget.cause = options.cause; + phase->budget.cpu_progress = cpu_progress; + auto status = + bind_backend_operation(core->async_backend(), plan.backend, std::move(backend), &phase->budget.backend); + if (!status.ok()) { + return status; + } + out->budget = phase->budget; + out->loader = reserved_shared<AsyncStorePageLoader>(phase->budget.bytes, ticket->backend(core->async_backend()), + core->binding()->cpu(), core->binding()->lane(), plan.loads, + OverflowBackingResolver{}, phase->budget); + out->owner = std::move(phase); + return Status::Ok(); +} + +Status Preparation::work(const PublishedTreeState &view, const CapturedWritePlan *plan, CheckpointWork *out) +{ + using namespace detail; + WorkPlan bound; + auto status = plan_work(view, plan, &bound); + if (!status.ok()) { + return status; + } + // Match handoff's routing -> reclaim -> I/O acquisition order. Backend + // allocations keep their stable Routing category through final ownership. + auto backend = ResourceBudget::try_reserve(core->binding()->account(), BudgetClass::kRouting, bound.backend_bytes); + auto bytes = ResourceBudget::try_reserve(core->binding()->account(), BudgetClass::kReclaim, bound.bytes); + auto io = ResourceBudget::try_reserve(core->binding()->account(), BudgetClass::kIo, 1); + if (!backend || !bytes || !io) { + return Status::resource_exhausted("checkpoint window credit unavailable"); + } + return bind_work(bound, std::move(*bytes), std::move(*backend), std::move(*io), out); +} +} // namespace crowdb::tree::checkpoint_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint_internal.h b/lib/crowdb-tree/src/btree/tree_core_checkpoint_internal.h new file mode 100644 index 000000000..9c1cb9e5f --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint_internal.h @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/local/block_account.h" +#include "btree/apply_dispatcher.h" +#include "btree/apply_progress.h" +#include "btree/apply_waiters.h" +#include "btree/tree_core.h" +#include "snapshot/checkpoint_operation.h" + +namespace crowdb::tree::checkpoint_detail +{ +struct Credits +{ + PageLoadBudget budget; + std::shared_ptr<detail::AllocationReservation> snapshot; + ~Credits(); +}; + +struct WorkPlan +{ + size_t bytes = 0; + size_t backend_bytes = 0; + PageLoadLimits loads; + detail::BackendOperationLimits backend; +}; + +struct Preparation +{ + TreeCore *core = nullptr; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<detail::AllocationReservation> control; + std::shared_ptr<detail::AllocationReservation> finish; + CheckpointOptions options; + PageMaintenanceOperationOptions structure_options; + Status structure(std::shared_ptr<const PublishedTreeState> view, uint64_t pid, + std::shared_ptr<PageSplitOperation> *out); + std::shared_ptr<ApplyDispatcher> cpu_progress; + std::shared_ptr<ApplyWaiters> waiters; + ~Preparation(); + Status plan_work(const PublishedTreeState &view, const CapturedWritePlan *plan, WorkPlan *out) const; + Status bind_work(const WorkPlan &plan, detail::ResourceBudget::Credit bytes, detail::ResourceBudget::Credit backend, + detail::ResourceBudget::Credit io, CheckpointWork *out); + Status work(const PublishedTreeState &view, const CapturedWritePlan *plan, CheckpointWork *out); +}; + +// One phase at a time; the checkpoint owns its captured input and retains this +// control until the grant callback and its final CPU continuation have drained. +class AsyncWork : public std::enable_shared_from_this<AsyncWork> +{ + public: + explicit AsyncWork(std::shared_ptr<Preparation> preparation); + [[nodiscard]] detail::CallbackSender prepare(const PublishedTreeState &view, const CapturedWritePlan *plan, + CheckpointWork *out); + void cancel() noexcept; + + private: + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + AsyncWork *owner; + void set_value() const noexcept; + void set_error(Status status) const noexcept; + void set_stopped() const noexcept; + }; + + struct Attempt + { + Attempt(AsyncWork *owner, detail::BudgetClass category, size_t bytes); + detail::ResourceBudget::Request request; + TreeMetricsOperation metrics; + stdexec::connect_result_t<detail::CallbackSender, Receiver> connected; + }; + + static void submit(void *source, detail::CallbackComplete complete, void *context); + static void run(void *context) noexcept; + void wake(Status status) noexcept; + void step(); + void finish(Status status) noexcept; + std::shared_ptr<Preparation> preparation_; + ApplyDispatcher::Node node_{&AsyncWork::run, this}; + WorkPlan plan_; + CheckpointWork *out_ = nullptr; + std::optional<Attempt> attempt_; + detail::ResourceBudget::Credit bytes_; + std::atomic<bool> stopped_{false}; + enum class Phase : uint8_t { kRouting, kReclaim, kIo }; + Phase phase_ = Phase::kRouting; + detail::ResourceBudget::Credit backend_; + Status status_; + detail::CallbackComplete complete_ = nullptr; + void *context_ = nullptr; +}; + +[[nodiscard]] std::optional<size_t> pressure_control_bound() noexcept; +Status prepare_pressure(TreeCore &core, std::shared_ptr<TreeOperationLifetime::Ticket> ticket, uint64_t leaf, + detail::ResourceBudget::Credit credit, std::shared_ptr<ApplyDispatcher> cpu_progress, + std::shared_ptr<PublicationLane> publication, std::shared_ptr<ApplyProgress> progress, + std::shared_ptr<ApplyWaiters> waiters, PageMaintenanceOperationOptions structure_options, + std::shared_ptr<CheckpointOperation> *out); + +CheckpointOptions defaults(const TreeCore &core); +void reclaim_free_blocks(TreeCore &core, std::shared_ptr<detail::AllocationReservation> metadata = {}, + std::shared_ptr<AllocationBudget> backend = {}) noexcept; +void schedule_free_blocks(TreeCore &core, std::shared_ptr<TreeOperationLifetime::Ticket> ticket) noexcept; +} // namespace crowdb::tree::checkpoint_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint_structure.cpp b/lib/crowdb-tree/src/btree/tree_core_checkpoint_structure.cpp new file mode 100644 index 000000000..ca302c08e --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint_structure.cpp @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/page_split_operation.h" +#include "btree/tree_core_checkpoint_internal.h" + +namespace crowdb::tree::checkpoint_detail +{ +Status Preparation::structure(std::shared_ptr<const PublishedTreeState> view, uint64_t pid, + std::shared_ptr<PageSplitOperation> *out) +try { + using namespace detail; + auto limits = structure_options; + limits.max_candidates = limits.max_visits = limits.visits_per_step = limits.max_steps = 1; + limits.publication.inner_merge_keys = 0; + limits.publication.pages.gc_floor = view->gc_floor; + const auto root = view->mapping.get(view->root_page_id); + const auto page = view->mapping.get(pid); + if (!root || !page || !page->leaf) { + return Status::unavailable("checkpoint split target changed"); + } + auto loads = core->load_limits(); + loads.max_inflight = loads.max_overflow_pages = 1; + const auto workers = core->binding()->cpu()->worker_count(); + const auto control_bytes = + PageSplitOperation::control_allocation_bound() + shared_allocation_bound<AllocationReservation>(); + const auto routing = AllocationBound() + .add(AsyncStorePageLoader::read_reserve_bound(loads, root->level + 1ULL, workers, + page->leaf->bases.size(), 0, 0)) + .add(shared_allocation_bound<AllocationReservation>()) + .value(); + const BackendOperationLimits backend_limits{ + .max_io_bytes = loads.max_blob_bytes, .write_calls = 0, .workers = workers, .barriers = false}; + const auto backend_bytes = backend_operation_bound(core->async_backend(), backend_limits); + auto account = core->binding()->account(); + if (routing && backend_bytes) { + const auto reclaim_available = ResourceBudget::available(account, BudgetClass::kReclaim); + const auto routing_available = ResourceBudget::available(account, BudgetClass::kRouting); + const auto fixed_routing = AllocationBound(*routing).add(*backend_bytes).value(); + limit_page_maintenance_window( + view->mapping, 0, reclaim_available > control_bytes ? reclaim_available - control_bytes : 0, + fixed_routing && *fixed_routing < routing_available ? routing_available - *fixed_routing : 0, + core->publisher().tracks_cache_residency(), &limits); + } + const auto maintenance = + page_maintenance_reserve_bound(view->mapping, 0, limits, core->publisher().tracks_cache_residency(), true); + const auto snapshot_bytes = page_maintenance_snapshot_reserve_bound(view->mapping, limits); + const auto reclaim = AllocationBound(maintenance.value_or(SIZE_MAX)).add(control_bytes).value(); + if (!reclaim || !routing || !backend_bytes || !snapshot_bytes) { + return Status::resource_exhausted("checkpoint split window bound overflow"); + } + auto backend = ResourceBudget::try_reserve(account, BudgetClass::kRouting, *backend_bytes); + auto read = ResourceBudget::try_reserve(account, BudgetClass::kRouting, *routing); + auto snapshot = ResourceBudget::try_reserve(account, BudgetClass::kRouting, *snapshot_bytes); + auto bytes = ResourceBudget::try_reserve(account, BudgetClass::kReclaim, *reclaim); + auto io = ResourceBudget::try_reserve(account, BudgetClass::kIo, 1); + if (!backend || !read || !snapshot || !bytes || !io) { + return Status::resource_exhausted("checkpoint split window credit unavailable"); + } + auto phase = reserved_shared<Credits>(control); + phase->snapshot = AllocationReservation::create(std::move(*snapshot)); + limits.publication.snapshot_metadata = phase->snapshot; + phase->budget.bytes = AllocationReservation::create(std::move(*read)); + phase->budget.io = reserved_shared<AllocationReservation>(phase->budget.bytes, std::move(*io)); + phase->budget.work_metrics = core->publisher().work_metrics(); + phase->budget.cause = options.cause; + phase->budget.cpu_progress = cpu_progress; + auto status = + bind_backend_operation(core->async_backend(), backend_limits, std::move(*backend), &phase->budget.backend); + if (!status.ok()) { + return status; + } + auto loader = reserved_shared<AsyncStorePageLoader>(phase->budget.bytes, ticket->backend(core->async_backend()), + core->binding()->cpu(), core->binding()->lane(), loads, + OverflowBackingResolver{}, phase->budget); + limits.prepaid = AllocationReservation::create(std::move(*bytes)); + return PageSplitOperation::create(core->publisher(), *core->binding()->cpu(), core->binding()->lane(), + std::move(view), pid, std::move(limits), std::move(loader), cpu_progress, + std::move(phase), options.cause, out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("checkpoint split allocation failed"); +} +} // namespace crowdb::tree::checkpoint_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint_work_async.cpp b/lib/crowdb-tree/src/btree/tree_core_checkpoint_work_async.cpp new file mode 100644 index 000000000..5e43d7a5c --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint_work_async.cpp @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/log.h" +#include "tree_core_checkpoint_internal.h" + +namespace crowdb::tree::checkpoint_detail +{ +AsyncWork::AsyncWork(std::shared_ptr<Preparation> preparation) : preparation_(std::move(preparation)) +{ +} + +AsyncWork::Attempt::Attempt(AsyncWork *owner, detail::BudgetClass category, size_t bytes) + : request(owner->preparation_->core->binding()->account(), category, bytes, + owner->preparation_->waiters ? owner->preparation_->waiters->get(category) : nullptr), + metrics(owner->preparation_->core->publisher().work_metrics(), TreeStage::BudgetWait, tree_metrics_now_ns()), + connected(stdexec::connect(request.sender(), Receiver{owner})) +{ +} + +void AsyncWork::Receiver::set_value() const noexcept +{ + owner->wake(Status::Ok()); +} + +void AsyncWork::Receiver::set_error(Status status) const noexcept +{ + owner->wake(std::move(status)); +} + +void AsyncWork::Receiver::set_stopped() const noexcept +{ + owner->wake(Status::unavailable("checkpoint grant cancelled")); +} + +detail::CallbackSender AsyncWork::prepare(const PublishedTreeState &view, const CapturedWritePlan *plan, + CheckpointWork *out) +{ + out_ = out; + phase_ = Phase::kRouting; + status_ = preparation_->plan_work(view, plan, &plan_); + return {this, &AsyncWork::submit}; +} + +void AsyncWork::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto &self = *static_cast<AsyncWork *>(source); + self.complete_ = complete; + self.context_ = context; + if (!self.status_.ok()) { + self.finish(std::move(self.status_)); + return; + } + self.attempt_.emplace(&self, detail::BudgetClass::kRouting, self.plan_.backend_bytes); + if (self.stopped_.load(std::memory_order_acquire)) { + self.attempt_->request.request_stop(); + } + stdexec::start(self.attempt_->connected); +} + +void AsyncWork::cancel() noexcept +{ + stopped_.store(true, std::memory_order_release); + if (attempt_) { + attempt_->request.request_stop(); + } +} + +void AsyncWork::wake(Status status) noexcept +{ + status_ = std::move(status); + preparation_->cpu_progress->request(node_, weak_from_this().lock()); +} + +void AsyncWork::run(void *context) noexcept +{ + auto &self = *static_cast<AsyncWork *>(context); + try { + self.step(); + } + catch (const std::bad_alloc &) { + self.finish(Status::resource_exhausted()); + } + catch (...) { + self.finish(Status::internal_error()); + } +} + +void AsyncWork::step() +{ + if (!attempt_) { + finish(Status::internal_error("checkpoint grant lost its request")); + return; + } + auto outcome = status_.ok() ? TreeOutcome::Success : TreeOutcome::Error; + if (stopped_.load(std::memory_order_acquire)) { + outcome = TreeOutcome::Cancelled; + } + (void)attempt_->metrics.finish(outcome, tree_metrics_now_ns()); + if (!status_.ok() || stopped_.load(std::memory_order_acquire)) { + finish(status_.ok() ? Status::unavailable("checkpoint grant cancelled") : std::move(status_)); + return; + } + auto credit = attempt_->request.take_credit(); + attempt_.reset(); + if (!credit) { + finish(Status::internal_error("checkpoint grant completed without credit")); + return; + } + if (phase_ == Phase::kRouting) { + backend_ = std::move(*credit); + phase_ = Phase::kReclaim; + attempt_.emplace(this, detail::BudgetClass::kReclaim, plan_.bytes); + stdexec::start(attempt_->connected); + return; + } + if (phase_ == Phase::kReclaim) { + bytes_ = std::move(*credit); + phase_ = Phase::kIo; + attempt_.emplace(this, detail::BudgetClass::kIo, 1); + stdexec::start(attempt_->connected); + return; + } + finish(preparation_->bind_work(plan_, std::move(bytes_), std::move(backend_), std::move(*credit), out_)); +} + +void AsyncWork::finish(Status status) noexcept +{ + attempt_.reset(); + bytes_ = {}; + backend_ = {}; + out_ = nullptr; + complete_(context_, status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError, + std::move(status)); +} +} // namespace crowdb::tree::checkpoint_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_chunk_gc.cpp b/lib/crowdb-tree/src/btree/tree_core_chunk_gc.cpp new file mode 100644 index 000000000..7a19072d4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_chunk_gc.cpp @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_backend_state.h" +#include "backend/chunk/chunk_collection_operation.h" +#include "btree/tree_core_internal.h" + +namespace crowdb::tree +{ +Status TreeCore::prepare_chunk_collection(std::shared_ptr<ChunkCollectionOperation> *out) +try { + if (out == nullptr || *out) { + return Status::invalid_argument(); + } + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable(); + } + auto backend = std::dynamic_pointer_cast<ChunkBackendState>(chunk_backend()); + if (!backend) { + return Status::not_supported(); + } + using namespace detail; + const auto bound = AllocationBound(ChunkCollectionOperation::allocation_bound()) + .add(2 * shared_allocation_bound<AllocationReservation>()) + .add(TreeOperationLifetime::ticket_allocation_bound()) + .value(); + if (!bound) { + return Status::resource_exhausted(); + } + auto bytes = ResourceBudget::try_reserve(binding()->account(), BudgetClass::kReclaim, *bound); + auto io = ResourceBudget::try_reserve(binding()->account(), BudgetClass::kIo, 1); + if (!bytes || !io) { + return Status::resource_exhausted(); + } + PageLoadBudget budget; + budget.bytes = AllocationReservation::create(std::move(*bytes)); + budget.io = reserved_shared<AllocationReservation>(budget.bytes, std::move(*io)); + + struct Credit + { + PageLoadBudget &budget; + bool transferred = false; + + ~Credit() + { + if (!transferred) { + budget.bytes->seal(); + budget.io->seal(); + } + } + } credit{.budget = budget}; + + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto status = operation_lifetime().acquire(budget.bytes, &ticket); + if (!status.ok()) { + return status; + } + status = ChunkCollectionOperation::create(backend, ticket->backend(backend->store.get()), binding()->cpu(), + binding()->lane(), budget, out); + credit.transferred = status.ok(); + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_chunk_migration.cpp b/lib/crowdb-tree/src/btree/tree_core_chunk_migration.cpp new file mode 100644 index 000000000..c90a1d70d --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_chunk_migration.cpp @@ -0,0 +1,121 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_backend_state.h" +#include "backend/chunk/legacy_chunk_recovery.h" +#include "btree/tree_core.h" +#include "snapshot/published_byte_export.h" +#include "snapshot/runtime_snapshot_import.h" + +namespace crowdb::tree +{ +namespace +{ +Status copy_snapshot(const std::shared_ptr<TreeCore> &source, const std::shared_ptr<TreeCore> &destination) +{ + std::shared_ptr<PublishedByteExport> exported; + const auto &range = source->config().key_range; + const auto format = range.is_bounded() ? snapshot_format::kPortable : snapshot_format::kNative; + auto status = PublishedByteExport::create(source, format, kSnapshotChunkBytes, &exported, range); + if (!status.ok()) { + return status; + } + std::shared_ptr<RuntimeSnapshotImport> imported; + status = RuntimeSnapshotImport::create(*destination, &imported); + bool done = false; + while (status.ok() && !done) { + SnapshotChunk chunk; + status = exported->next(&chunk, &done); + if (status.ok()) { + status = imported->feed(Slice(reinterpret_cast<const char *>(chunk.bytes.data()), chunk.bytes.size())); + } + } + uint64_t coverage = 0; + if (status.ok()) { + status = imported->finish(&coverage); + } + if (status.ok() && coverage != source->durable_coverage()) { + return Status::corruption("legacy Chunk migration changed the durable coverage"); + } + return status; +} +} // namespace + +Status TreeCore::migrate_chunk(const Config &config, uint64_t expected_generation, std::shared_ptr<TreeCore> *out) +{ + if (out == nullptr || *out || expected_generation == 0 || expected_generation == UINT64_MAX) { + return Status::invalid_argument(); + } + std::shared_ptr<detail::LegacyChunkReadStore> legacy; + std::shared_ptr<TreeCore> source; + std::shared_ptr<TreeCore> destination; + Status status; + try { + status = detail::recover_legacy_chunk_source(config, expected_generation, &legacy, &source); + if (status.ok() && !source) { + status = open_chunk(config, &destination); + if (status.ok()) { + status = legacy->release_pin(); + } + if (status.ok()) { + *out = std::move(destination); + } + return status; + } + if (status.ok()) { + status = legacy->prepare_migration(); + } + if (status.ok()) { + status = open_chunk_migration(config, legacy, source->snapshot_sequence(), &destination); + } + if (status.ok()) { + status = copy_snapshot(source, destination); + } + // The completed copy owns every imported frame; close source reads before + // taking the checkpoint's complete write window. + source.reset(); + if (status.ok()) { + status = destination->checkpoint(); + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted(); + } + catch (...) { + status = Status::internal_error(); + } + // Terminal cleanup is also required when allocation or a host callback + // throws. A confirmed unknown durable commit retains the same pin. + source.reset(); + const bool uncertain = destination && destination->chunk_backend()->needs_recovery.load(std::memory_order_acquire); + if (legacy && !uncertain) { + const auto released = legacy->release_pin(); + if (status.ok()) { + status = released; + } + } + if (status.ok()) { + *out = std::move(destination); + } + return status; +} + +Status Crowdbtree::migrate_legacy_chunk(const Config &opt, uint64_t expected_generation, + std::unique_ptr<Crowdbtree> *out) +try { + if (out == nullptr || *out) { + return Status::invalid_argument(); + } + std::shared_ptr<TreeCore> core; + auto status = TreeCore::migrate_chunk(opt, expected_generation, &core); + if (status.ok()) { + out->reset(new Crowdbtree(opt, std::move(core))); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (...) { + return Status::internal_error(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_chunk_open.cpp b/lib/crowdb-tree/src/btree/tree_core_chunk_open.cpp new file mode 100644 index 000000000..445b2c2da --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_chunk_open.cpp @@ -0,0 +1,310 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_backend_state.h" +#include "backend/chunk/chunk_protocol_operation.h" +#include "backend/chunk/legacy_chunk_read_store.h" +#include "btree/backing_identity.h" +#include "btree/tree_core.h" +#include "runtime/tree_runtime_internal.h" +#include "snapshot/chunk_root_recovery.h" +#include "snapshot/operation_wait.h" +#include "snapshot/recovered_state.h" + +namespace crowdb::tree +{ +namespace +{ +struct ChunkOpen +{ + ChunkOpen(std::shared_ptr<TreeRuntimeBinding> binding, std::shared_ptr<ChunkBackendState> backend) + : binding(std::move(binding)), + backend(std::move(backend)), + lifetime(std::in_place, 1, this->binding->lifecycle_budget(), this->backend) + { + } + + ~ChunkOpen() + { + finish(); + } + + void finish() + { + if (finished) { + return; + } + finished = true; + store.reset(); + ticket.reset(); + if (lifetime) { + lifetime->close_and_drain(); + } + lifetime.reset(); + backend.reset(); + binding->drain(); + if (budget.io) { + budget.io->seal(); + } + if (budget.bytes) { + budget.bytes->seal(); + } + budget = {}; + } + + bool finished = false; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<ChunkBackendState> backend; + std::optional<TreeOperationLifetime> lifetime; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<AsyncPageStore> store; + PageLoadBudget budget; +}; + +struct RootLoad +{ + explicit RootLoad(const std::shared_ptr<detail::AllocationReservation> &metadata) + : bytes(MetadataAllocator<uint8_t>(metadata)), + root(metadata) + { + } + + std::shared_ptr<const ChunkBackingDomain> domain; + detail::ChunkRootBytes bytes; + ChunkRootManifest root; + uint64_t generation = 0; + uint64_t latest_generation = 0; + bool loaded_current = false; + bool absent = false; + bool done = false; + + Status step(bool stopped) + { + if (stopped) { + return Status::unavailable(); + } + auto *catalog = domain->catalog->protection(); + auto status = catalog->check_read_owner(); + if (!status.ok()) { + return status; + } + const auto requested = loaded_current ? generation : 0; + status = catalog->load_root_owned(domain->tree, requested, &bytes); + if (status.code() == Code::kNotFound && generation == 0) { + absent = true; + done = true; + return Status::Ok(); + } + if (!status.ok()) { + return status; + } + root.indexes.clear(); + if (bytes.size() >= 4 && frame_u32(bytes.data(), 0) == detail::kChunkManifestFormat) { + return Status::not_supported( + "legacy Chunk root requires explicit migrate_legacy_chunk while writes are stopped"); + } + status = decode_chunk_root(bytes, domain->tree, requested, &root); + if (status.ok() && !loaded_current) { + latest_generation = root.identity.root_generation; + loaded_current = true; + if (generation != 0 && generation != latest_generation) { + return Status::Ok(); + } + } + if (status.ok()) { + status = catalog->check_read_owner(); + } + done = status.ok(); + return status; + } +}; + +Status open_credit(ChunkOpen &open) +{ + using namespace detail; + const auto &options = open.binding->options(); + if (options.limits.routing_bytes <= options.lifecycle_bytes) { + return Status::resource_exhausted(); + } + auto bytes = ResourceBudget::try_reserve(open.binding->account(), BudgetClass::kRouting, + options.limits.routing_bytes - options.lifecycle_bytes); + auto io = ResourceBudget::try_reserve(open.binding->account(), BudgetClass::kIo, 1); + if (!bytes || !io) { + return Status::resource_exhausted(); + } + open.budget.bytes = AllocationReservation::create(std::move(*bytes)); + open.budget.io = reserved_shared<AllocationReservation>(open.budget.bytes, std::move(*io)); + if (!open.lifetime) { + return Status::internal_error(); + } + auto status = open.lifetime->acquire(open.budget.bytes, &open.ticket); + if (status.ok()) { + open.store = open.ticket->backend(open.backend->store.get()); + } + return status; +} + +template <class Protocol> Status run_protocol(ChunkOpen &open, const std::shared_ptr<Protocol> &protocol) +{ + std::shared_ptr<ChunkProtocolOperation> operation; + auto status = ChunkProtocolOperation::create( + open.store, open.binding->cpu(), open.binding->lane(), open.budget, protocol, + [protocol](bool stopped) { return protocol->step(stopped); }, [protocol] { return protocol->done(); }, + &operation); + if (status.ok()) { + status = detail::wait_snapshot_operation(operation); + } + return status; +} + +Status load_root(ChunkOpen &open, std::shared_ptr<RootLoad> *out) +{ + auto root = detail::reserved_shared<RootLoad>(open.budget.bytes, open.budget.bytes); + root->domain = open.backend->domain; + root->generation = open.backend->facade->options().open_generation; + std::shared_ptr<ChunkProtocolOperation> operation; + auto status = ChunkProtocolOperation::create( + open.store, open.binding->cpu(), open.binding->lane(), open.budget, root, + [root](bool stopped) { return root->step(stopped); }, [root] { return root->done; }, &operation); + if (status.ok()) { + status = detail::wait_snapshot_operation(operation); + } + if (status.ok()) { + *out = std::move(root); + } + return status; +} + +Status recover_chunk(ChunkOpen &open, const Config &config, std::shared_ptr<const PublishedTreeState> *initial) +{ + auto status = open_credit(open); + if (!status.ok()) { + return status; + } + std::shared_ptr<RootLoad> loaded; + status = load_root(open, &loaded); + if (!status.ok() || loaded->absent) { + return status; + } + const auto identity = loaded->root.identity; + ChunkRecoveryLimits limits; + limits.max_frame_bytes = std::max(limits.max_frame_bytes, size_t{config.frame_bytes}); + limits.max_blob_bytes = round_up_to_iu(limits.max_frame_bytes + kDurableBlobHeader, open.backend->iu_size); + const auto metadata = ChunkRootRecovery::allocation_bound(loaded->root); + const auto window = ChunkRootRecovery::window_allocation_bound(limits); + if (!metadata || !window || *metadata > open.budget.bytes->available() || + *window > open.budget.bytes->available() - *metadata) { + return Status::resource_exhausted(); + } + std::shared_ptr<ChunkRootRecovery> recovery; + status = ChunkRootRecovery::create(std::move(loaded->root), open.backend->domain, open.backend->ids, limits, + open.budget.bytes, open.budget.bytes, &recovery); + loaded.reset(); + if (!status.ok()) { + return status; + } + status = run_protocol(open, recovery); + if (!status.ok()) { + return status; + } + PageLoadLimits loads; + loads.max_frame_bytes = limits.max_frame_bytes; + loads.max_blob_bytes = limits.max_blob_bytes; + loads.max_chunk_blob_bytes = limits.max_blob_bytes; + auto loader = detail::reserved_shared<AsyncStorePageLoader>(open.budget.bytes, open.store, open.binding->cpu(), + open.binding->lane(), loads, OverflowBackingResolver{}, + open.budget); + std::shared_ptr<RecoveredStateOperation> recovered; + status = RecoveredStateOperation::create(recovery->pages(), loader, open.binding->cpu(), open.binding->lane(), {}, + open.budget.bytes, &recovered); + if (!status.ok()) { + return status; + } + status = detail::wait_snapshot_operation(recovered); + if (status.ok()) { + *initial = recovered->result().state; + open.backend->max_frame_bytes = recovery->max_frame_bytes(); + open.backend->max_blob_bytes = recovery->max_blob_bytes(); + open.backend->recovered(identity); + } + return status; +} +} // namespace + +Status TreeCore::open_chunk(const Config &config, std::shared_ptr<TreeCore> *out) +{ + return open_chunk_migration(config, {}, 0, out); +} + +Status TreeCore::open_chunk_migration(const Config &config, const std::shared_ptr<detail::LegacyChunkReadStore> &legacy, + uint64_t snapshot_sequence, std::shared_ptr<TreeCore> *out) +{ + auto *facade = dynamic_cast<detail::ChunkRuntimePageStore *>(config.page_store); + if (facade == nullptr || !facade->supported() || !config.backend_owner || config.async_page_store != nullptr) { + return Status::invalid_argument(); + } + if (!config.runtime || !facade->matches_host(TreeRuntimeAccess::lifecycle(*config.runtime).get())) { + return Status::invalid_argument(); + } + if (legacy && (legacy->catalog() != facade->catalog() || legacy->transport() != facade->transport() || + facade->options().open_generation != 0)) { + return Status::invalid_argument("migration must replace the same current Chunk root"); + } + auto status = config.key_range.validate(); + if (!status.ok()) { + return status; + } + std::shared_ptr<TreeRuntimeBinding> binding; + status = TreeRuntimeBinding::create(config.runtime, config.runtime_binding, &binding); + if (!status.ok()) { + return status; + } + auto chunk = detail::reserved_shared<ChunkBackendState>(binding->lifecycle_budget()); + chunk->facade = facade; + chunk->backend_owner = config.backend_owner; + chunk->iu_size = facade->iu_size(); + if (!facade->try_claim_runtime_domain()) { + return Status::unavailable(); + } + chunk->owns_writer_domain = true; + status = facade->make_domain(detail::allocate_backend_identity(), binding->lifecycle_budget(), &chunk->domain); + if (!status.ok()) { + return status; + } + status = chunk->domain->catalog->pin_release_queue()->prepare_writer(binding->lifecycle_budget(), + chunk->domain->catalog, chunk->domain->tree, + chunk->domain->epoch, &chunk->writer_session); + if (!status.ok()) { + return status; + } + if (!chunk->domain->catalog->claim_runtime_writer()) { + return Status::unavailable(); + } + chunk->writer_session->arm(); + status = detail::ChunkRuntimeIds::create(chunk->domain->epoch, binding->lifecycle_budget(), &chunk->ids); + if (!status.ok()) { + return status; + } + status = ScheduledPageStore::create(facade, nullptr, config.backend_owner, binding, binding->lifecycle_budget(), + binding->backend_lane(), &chunk->store); + if (!status.ok()) { + return status; + } + ChunkOpen open(binding, chunk); + std::shared_ptr<const PublishedTreeState> initial; + if (legacy) { + chunk->root_generation.store(legacy->generation(), std::memory_order_relaxed); + chunk->snapshot_sequence.store(snapshot_sequence, std::memory_order_relaxed); + facade->request_wal_offset(legacy->wal_offset()); + } + else { + status = recover_chunk(open, config, &initial); + } + if (!status.ok()) { + return status; + } + // Recovery owns its completed mapping allocations, but its unused startup + // reserve must be returned before the live replay/control window is prepaid. + open.finish(); + return create(config, binding, {.initial = std::move(initial), .local = {}, .chunk = std::move(chunk)}, out); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_compact.cpp b/lib/crowdb-tree/src/btree/tree_core_compact.cpp new file mode 100644 index 000000000..b393e37b3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_compact.cpp @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/local_compaction_operation.h" +#include "btree/tree_core_internal.h" +#include "snapshot/operation_wait.h" + +namespace crowdb::tree +{ +Status TreeCore::prepare_compaction(const LocalCompactionOptions &options, + std::shared_ptr<LocalCompactionOperation> *out) +try { + if (out == nullptr || *out) { + return Status::invalid_argument(); + } + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable(); + } + const auto bound = LocalCompactionOperation::allocation_bound(); + if (!bound) { + return Status::resource_exhausted(); + } + auto reserved = detail::ResourceBudget::try_reserve(binding()->account(), detail::BudgetClass::kReclaim, *bound); + if (!reserved) { + return Status::resource_exhausted(); + } + auto credit = detail::AllocationReservation::create(std::move(*reserved)); + + struct Seal + { + std::shared_ptr<detail::AllocationReservation> credit; + bool transferred = false; + + ~Seal() + { + if (!transferred) { + credit->seal(); + } + } + } seal{.credit = credit}; + + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto status = operation_lifetime().acquire(credit, &ticket); + if (!status.ok()) { + return status; + } + status = LocalCompactionOperation::create(*this, options, state_->apply_dispatcher, credit, std::move(ticket), out); + seal.transferred = status.ok(); + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status TreeCore::compact_sparse_blocks(MergeGcStats *out) +{ + if (out == nullptr) { + return Status::invalid_argument(); + } + *out = {}; + std::shared_ptr<LocalCompactionOperation> operation; + auto status = prepare_compaction({.free_threshold = config().merge_gc_block_free_threshold, + .max_relocation_bytes = config().merge_gc_max_relocation_bytes}, + &operation); + if (!status.ok()) { + return status; + } + status = detail::wait_snapshot_operation(operation); + *out = operation->result(); + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_evict.cpp b/lib/crowdb-tree/src/btree/tree_core_evict.cpp new file mode 100644 index 000000000..81fb0bdfd --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_evict.cpp @@ -0,0 +1,252 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_internal.h" + +namespace crowdb::tree +{ +namespace +{ +bool evictable(const PublishedPage &page, PageCacheKind kind) +{ + if (page.metrics.dirty) { + return false; + } + if (kind == PageCacheKind::Inner) { + return page.level != 0 && page.inner && page.inner_backing && page.inner_backing->reloadable(); + } + if (page.level != 0 || !page.leaf || !page.leaf->overlays.empty()) { + return false; + } + bool resident = false; + for (const auto &base : page.leaf->bases) { + if (base.resident) { + if (!base.backing || !base.backing->reloadable()) { + return false; + } + resident = true; + } + } + return resident; +} + +struct CleanEviction : std::enable_shared_from_this<CleanEviction> +{ + TreeCore *core = nullptr; + PageCacheKind kind = PageCacheKind::Leaf; + size_t limit = 0; + size_t resident = 0; + size_t evicted = 0; + uint64_t cursor = 0; + uint64_t candidate = MappingView::kPageLimit; + bool counting = true; + bool waiting = false; + bool installed = false; + std::atomic<bool> ready{false}; + std::atomic<bool> done{false}; + std::shared_ptr<const PublishedTreeState> view; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<detail::AllocationReservation> controls; + std::shared_ptr<detail::AllocationReservation> publication_credit; + std::shared_ptr<detail::FlushRuntime::Progress> progress; + std::shared_ptr<PublicationRequest> request; + std::shared_ptr<CleanEviction> trim_owner; + PageCacheTrim trim; + + ~CleanEviction() + { + if (publication_credit) { + publication_credit->seal(); + } + if (controls) { + controls->seal(); + } + } + + void wake() noexcept + { + progress->request(weak_from_this().lock()); + } + + static void run(void *context) noexcept + { + auto &self = *static_cast<CleanEviction *>(context); + if (self.done.load(std::memory_order_acquire)) { + return; + } + try { + self.step(); + } + catch (...) { + self.trim_cache(); + } + } + + void trim_cache() noexcept + { + request.reset(); + if (publication_credit) { + publication_credit->seal(); + publication_credit.reset(); + } + const auto remaining = resident - evicted; + trim.kind = kind; + trim.max_pages = limit > remaining ? limit - remaining : 0; + trim.context = this; + trim.complete = [](void *context, size_t count) noexcept { + auto self = std::move(static_cast<CleanEviction *>(context)->trim_owner); + self->evicted += count; + self->done.store(true, std::memory_order_release); + self->done.notify_one(); + }; + trim_owner = weak_from_this().lock(); + core->cache()->trim(trim); + } + + void step() + { + if (waiting) { + if (!ready.load(std::memory_order_acquire)) { + return; + } + waiting = false; + evicted += static_cast<size_t>(installed); + request.reset(); + publication_credit->seal(); + publication_credit.reset(); + } + if (counting) { + const auto end = view->mapping.visit_from(cursor, 64, this, [](void *context, uint64_t, const auto &page) { + auto &self = *static_cast<CleanEviction *>(context); + self.resident += static_cast<size_t>(evictable(*page, self.kind)); + }); + if (end) { + counting = false; + cursor = 0; + } + wake(); + return; + } + if (resident - evicted <= limit) { + trim_cache(); + return; + } + candidate = MappingView::kPageLimit; + const auto end = view->mapping.visit_from(cursor, 1, this, [](void *context, uint64_t pid, const auto &page) { + auto &self = *static_cast<CleanEviction *>(context); + if (evictable(*page, self.kind)) { + self.candidate = pid; + } + }); + if (candidate != MappingView::kPageLimit) { + publish(); + } + else if (end) { + trim_cache(); + } + else { + wake(); + } + } + + void publish() + { + const auto current = core->capture(); + const auto page = current->mapping.get(candidate); + if (current->generation != view->generation || !page || !evictable(*page, kind) || + page->revision == UINT64_MAX || (page->leaf && page->leaf->representation_id == UINT64_MAX)) { + wake(); + return; + } + using namespace detail; + const auto bound = + AllocationBound(PublicationCoordinator::empty_commit_reserve_bytes()) + .add(MappingView::path_copy_bound()) + .add(current->mapping.copy_bounds().max_leaf_bytes) + .add(shared_allocation_bound<PublishedPage>() + shared_allocation_bound<PublicationRequest>() + + shared_allocation_bound<InstallRecord>() + shared_allocation_bound<AllocationReservation>() + + sizeof(PageReplacement)) + .add(PageCacheResidency::page_allocation_bound(page->leaf ? page->leaf->bases.size() : 1)) + .value(); + auto credit = bound ? ResourceBudget::try_reserve(core->binding()->account(), BudgetClass::kReclaim, *bound) + : std::nullopt; + if (!credit) { + trim_cache(); + return; + } + publication_credit = AllocationReservation::create(std::move(*credit)); + auto replacement = reserved_shared<PublishedPage>(publication_credit, *page); + ++replacement->revision; + replacement->metrics.evict_candidate = false; + if (page->leaf) { + auto leaf = reserved_shared<PageState>(publication_credit, *page->leaf, publication_credit); + ++leaf->representation_id; + for (auto &base : leaf->bases) { + base.resident.reset(); + } + replacement->leaf = std::move(leaf); + } + else { + replacement->inner.reset(); + } + request = reserved_shared<PublicationRequest>(publication_credit, publication_credit); + request->transaction.record = + reserved_shared<InstallRecord>(publication_credit, InstallToken{.generation = current->generation}); + request->transaction.pages.push_back({.page_id = candidate, .expected = page, .replacement = replacement}); + request->on_done = [self = shared_from_this()](PublicationResult result) { + self->installed = result.status.ok(); + self->ready.store(true, std::memory_order_release); + self->wake(); + }; + waiting = true; + ready.store(false, std::memory_order_release); + if (!core->publisher().try_submit(request)) { + installed = false; + ready.store(true, std::memory_order_release); + wake(); + } + } +}; +} // namespace + +size_t TreeCore::evict_clean_pages(PageCacheKind kind, size_t max_resident) +try { + State::Ingress ingress(*state_); + if (ingress.owner == nullptr || max_resident == SIZE_MAX) { + return 0; + } + using namespace detail; + const auto bound = AllocationBound(shared_allocation_bound<CleanEviction>()) + .add(shared_allocation_bound<AllocationReservation>()) + .add(TreeOperationLifetime::ticket_allocation_bound()) + .add(FlushRuntime::progress_allocation_bound()) + .value(); + auto credit = + bound ? ResourceBudget::try_reserve(binding()->account(), BudgetClass::kReclaim, *bound) : std::nullopt; + if (!credit) { + return 0; + } + auto metadata = AllocationReservation::create(std::move(*credit)); + auto operation = reserved_shared<CleanEviction>(metadata); + operation->controls = metadata; + operation->core = this; + operation->kind = kind; + operation->limit = max_resident; + operation->view = capture(); + if (!operation_lifetime().acquire(metadata, &operation->ticket).ok() || + !binding() + ->cpu() + ->reserve_progress(binding()->lane(), &CleanEviction::run, operation.get(), metadata, &operation->progress) + .ok()) { + return 0; + } + operation->wake(); + while (!operation->done.load(std::memory_order_acquire)) { + operation->done.wait(false, std::memory_order_acquire); + } + return operation->evicted; +} +catch (const std::bad_alloc &) { + return 0; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_facade.cpp b/lib/crowdb-tree/src/btree/tree_core_facade.cpp new file mode 100644 index 000000000..fefcb2005 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_facade.cpp @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/local_backend_state.h" +#include "btree/tree_core.h" + +#include <stdexcept> + +namespace crowdb::tree +{ +uint64_t Crowdbtree::last_applied_slot() const +{ + return core_->capture()->coverage; +} + +uint64_t Crowdbtree::contiguous_slot() const +{ + return core_->contiguous_slot(); +} + +uint64_t Crowdbtree::version() const +{ + return core_->capture()->commit_seq; +} + +uint64_t Crowdbtree::root_page_id() const +{ + return core_->capture()->root_page_id; +} + +uint64_t Crowdbtree::leaf_count_atomic() const +{ + return core_->capture()->stats.leaf_pages; +} + +uint64_t Crowdbtree::inner_count_atomic() const +{ + return core_->capture()->stats.inner_pages; +} + +uint64_t Crowdbtree::durable_snapshot_seq() const +{ + return core_->snapshot_sequence(); +} + +uint64_t Crowdbtree::durable_snapshot_last_applied_slot() const +{ + return core_->durable_coverage(); +} + +BackingStatsSnapshot Crowdbtree::backing_stats() const noexcept +{ + return core_->backing_stats(); +} + +TreeStatsSnapshot Crowdbtree::tree_state_stats() const +{ + return core_->state_stats(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_flush.cpp b/lib/crowdb-tree/src/btree/tree_core_flush.cpp new file mode 100644 index 000000000..265095ca7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_flush.cpp @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "crowdb-common/log.h" +#include "tree_core_internal.h" + +namespace crowdb::tree +{ +namespace +{ +struct FlushCall; + +struct FlushReceiver +{ + using receiver_concept = stdexec::receiver_tag; + FlushCall *owner; + void set_value(FlushResult result) const noexcept; + void set_error(Status status) const noexcept; + void set_stopped() const noexcept; +}; + +struct FlushCall : std::enable_shared_from_this<FlushCall> +{ + using Connected = stdexec::connect_result_t<FlushReportSender, FlushReceiver>; + + FlushCall(std::shared_ptr<TreeOperationLifetime::Ticket> lifetime, std::shared_ptr<FlushOperation> operation) + : ticket(std::move(lifetime)), + operation(std::move(operation)), + connected(stdexec::connect(this->operation->report_sender(), FlushReceiver{this})) + { + } + + void finish(Status status) noexcept + { + auto retained = std::move(self); + try { + callback(std::move(status)); + } + catch (...) { + CRB_LOG_ERROR("Tree flush receiver threw after terminal"); + } + } + + void start() + { + self = shared_from_this(); + stdexec::start(connected); + } + + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<FlushOperation> operation; + std::function<void(Status)> callback; + Connected connected; + std::shared_ptr<FlushCall> self; +}; + +void FlushReceiver::set_value(FlushResult result) const noexcept +{ + owner->finish(result.handoff_status.ok() ? result.maintenance_status : result.handoff_status); +} + +void FlushReceiver::set_error(Status status) const noexcept +{ + owner->finish(std::move(status)); +} + +void FlushReceiver::set_stopped() const noexcept +{ + owner->finish(Status::unavailable("Tree flush stopped")); +} +} // namespace + +Status TreeCore::prepare_flush(std::shared_ptr<FlushOperation> *out) +{ + return prepare_flush(out, {}); +} + +Status TreeCore::prepare_flush(std::shared_ptr<FlushOperation> *out, + std::shared_ptr<TreeOperationLifetime::Ticket> accepted) +try { + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return Status::unavailable("Tree ingress closed"); + } + auto ticket = std::move(accepted); + if (!ticket) { + auto status = state_->lifetime->acquire(binding()->lifecycle_budget(), &ticket); + if (!status.ok()) { + return status; + } + } + auto preparation = state_->preparation; + preparation.store = ticket->backend(async_backend()); + return state_->flush->prepare(preparation, out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("Tree flush preparation allocation failed"); +} + +void TreeCore::flush_async(std::function<void(Status)> complete) +{ + if (!complete) { + return; + } + std::shared_ptr<FlushCall> call; + Status status; + try { + State::Ingress admission(*state_); + constexpr auto bytes = detail::shared_allocation_bound<FlushCall>() + + detail::shared_allocation_bound<detail::AllocationReservation>(); + auto reserved = detail::ResourceBudget::try_reserve(binding()->account(), detail::BudgetClass::kHandoff, + bytes + TreeOperationLifetime::ticket_allocation_bound()); + if (admission.owner == nullptr) { + status = Status::unavailable("Tree ingress closed"); + } + else if (!reserved) { + status = Status::unavailable("Tree flush callback credit unavailable"); + } + else { + auto credit = detail::AllocationReservation::create(std::move(*reserved)); + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + status = state_->lifetime->acquire(credit, &ticket); + std::shared_ptr<FlushOperation> operation; + if (status.ok()) { + status = prepare_flush(&operation); + } + if (status.ok()) { + call = detail::reserved_shared<FlushCall>(credit, std::move(ticket), std::move(operation)); + call->callback.swap(complete); + credit->seal(); + } + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted("Tree flush callback allocation failed"); + } + if (call) { + call->start(); + return; + } + complete(std::move(status)); +} + +Status TreeCore::flush() +try { + if (detail::FlushRuntime::on_worker_thread()) { + return Status::unavailable("a runtime worker must use asynchronous flush"); + } + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return Status::unavailable("Tree ingress closed"); + } + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto status = state_->lifetime->acquire(binding()->lifecycle_budget(), &ticket); + if (!status.ok()) { + return status; + } + auto preparation = state_->preparation; + preparation.store = ticket->backend(async_backend()); + for (;;) { + const auto observed = state_->apply_progress->sequence(); + bool blocked = false; + std::shared_ptr<FlushOperation> operation; + status = state_->flush->prepare(preparation, &operation, &blocked); + if (status.ok()) { + return operation->wait(); + } + if (!blocked) { + return status; + } + status = state_->apply_progress->wait(observed); + if (!status.ok()) { + return status; + } + } +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("Tree flush preparation allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_generation.cpp b/lib/crowdb-tree/src/btree/tree_core_generation.cpp new file mode 100644 index 000000000..5ca7ace03 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_generation.cpp @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_core_internal.h" +#include "tree_generation_lease.h" + +namespace crowdb::tree +{ +Status TreeCore::install_generation(std::shared_ptr<const PublishedTreeState> candidate, const SplitGeneration *split) +{ + std::shared_ptr<TreeGenerationLease> prepared; + auto status = TreeGenerationLease::prepare(*this, std::move(candidate), split, &prepared); + if (!status.ok()) { + return status; + } + status = prepared->acquire(); + if (!status.ok()) { + return status; + } + return prepared->commit(); +} + +#ifdef CROWDB_TREE_TEST_UTIL +void TreeCore::set_before_generation_install_for_tests(std::function<void()> callback) +{ + state_->before_generation_install_for_tests = std::move(callback); +} +#endif + +Status TreeCore::clear() +{ + return install_generation({}); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_geometry.cpp b/lib/crowdb-tree/src/btree/tree_core_geometry.cpp new file mode 100644 index 000000000..1d25f7c07 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_geometry.cpp @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_geometry.h" + +#include "btree/page_maintenance.h" +#include "crowdb-protocol/frame/base.h" +#include "crowdb-tree/maptable/frame_page.h" + +#include <algorithm> +#include <limits> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kOverhead = kFrameHeaderSize + kFrameTrailerSize; +} + +size_t leaf_record_capacity(uint32_t frame_bytes) noexcept +{ + const size_t bytes = std::min<size_t>(frame_bytes, crowdb::protocol::kMaxFrameV2PayloadBytes); + return bytes > kOverhead ? bytes - kOverhead : 0; +} + +Status configure_leaf_geometry(const Config &config, PageMaintenanceLimits *pages) +{ + const size_t capacity = leaf_record_capacity(config.frame_bytes); + if (pages == nullptr || capacity == 0 || config.leaf_split_bytes == 0 || + config.leaf_split_bytes > std::numeric_limits<size_t>::max() - kOverhead || + config.leaf_merge_bytes > std::numeric_limits<size_t>::max() - kOverhead) { + return Status::invalid_argument("leaf record thresholds exceed bounded frame geometry"); + } + const size_t split = std::min(config.leaf_split_bytes, capacity); + const size_t merge = split - std::max<size_t>(1, split / 4); + pages->primary_page_bytes = capacity + kOverhead; + pages->target_page_bytes = split + kOverhead; + pages->merge_target_bytes = merge + kOverhead; + pages->merge_trigger_bytes = std::min(config.leaf_merge_bytes, merge) + kOverhead; + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_geometry.h b/lib/crowdb-tree/src/btree/tree_core_geometry.h new file mode 100644 index 000000000..727cfb050 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_geometry.h @@ -0,0 +1,11 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/config.h" + +namespace crowdb::tree +{ +struct PageMaintenanceLimits; +Status configure_leaf_geometry(const Config &config, PageMaintenanceLimits *pages); +size_t leaf_record_capacity(uint32_t frame_bytes) noexcept; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_get.cpp b/lib/crowdb-tree/src/btree/tree_core_get.cpp new file mode 100644 index 000000000..a039f8200 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_get.cpp @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_internal.h" +#include "btree/tree_core_read_internal.h" +#include "host_completion.h" + +namespace crowdb::tree +{ +namespace +{ +struct CoreGet : std::enable_shared_from_this<CoreGet> +{ + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + CoreGet *owner; + + void set_value() const noexcept + { + owner->finish(owner->read->result().status); + } + + void set_error(Status status) const noexcept + { + owner->finish(std::move(status)); + } + + void set_stopped() const noexcept + { + owner->finish(Status::unavailable("read stopped")); + } + }; + + using Operation = detail::CallbackSender::Operation<Receiver>; + core_read_detail::ReadPreparation prepared; + std::shared_ptr<PublishedGet> read; + std::function<void(Status, GetView)> complete; + std::shared_ptr<Operation> operation; + std::shared_ptr<CoreGet> owner; + + void finish(Status status) noexcept + { + auto self = std::move(owner); + GetView result; + if (status.ok() && read->result().found) { + const auto &cell = read->result().cell; + result = GetView::from_owner(cell.slot, cell.value, cell.owner); + } + detail::seal_backend_budget(prepared.backend); + prepared.io->seal(); + prepared.credit->seal(); + try { + complete(std::move(status), std::move(result)); + } + catch (...) { + // Completion is terminal; a throwing consumer cannot be retried. + return; + } + } + + static void start(void *context) noexcept + { + auto *self = static_cast<CoreGet *>(context); + auto keep = self->owner; + stdexec::start(*self->operation); + } +}; +} // namespace + +void TreeCore::get_async(Slice key, std::function<void(Status, GetView)> complete) +{ + get_impl(key, std::move(complete), true); +} + +void TreeCore::get_impl(Slice key, std::function<void(Status, GetView)> complete, bool dispatch) +{ + if (!complete) { + return; + } + std::shared_ptr<CoreGet> operation; + Status status; + bool outside_range = false; + try { + State::Ingress ingress(*state_); + core_read_detail::ReadPreparation prepared; + outside_range = ingress.owner != nullptr && !state_->config.key_range.contains(key); + if (ingress.owner == nullptr) { + status = Status::unavailable("Tree closed"); + } + else if (!outside_range) { + status = core_read_detail::prepare(*this, false, + detail::shared_allocation_bound<CoreGet>() + + detail::shared_allocation_bound<CoreGet::Operation>(), + 0, &prepared); + } + if (status.ok() && !outside_range) { + operation = detail::reserved_shared<CoreGet>(prepared.credit); + operation->prepared = std::move(prepared); + auto &ready = operation->prepared; + status = PublishedGet::create(ready.view, ready.loader, key, ready.limits, &operation->read); + } + if (status.ok() && !outside_range) { + operation->operation = + detail::reserved_construct<CoreGet::Operation>(operation->prepared.credit, [&](auto *storage) { + ::new (static_cast<void *>(storage)) CoreGet::Operation( + stdexec::connect(operation->read->sender(), CoreGet::Receiver{operation.get()})); + }); + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted("read allocation failed"); + } + if (!status.ok() || outside_range) { + complete(std::move(status), {}); + return; + } + operation->complete = std::move(complete); + operation->owner = operation; + // The synchronous host boundary already has a caller thread. Starting its + // bounded read inline avoids a worker round trip for resident data; cold + // loads retain the same asynchronous loader and operation tickets. + if (!dispatch) { + CoreGet::start(operation.get()); + return; + } + status = operation->prepared.binding->cpu()->try_submit(operation->prepared.binding->lane(), &CoreGet::start, + operation.get()); + if (!status.ok()) { + operation->finish(std::move(status)); + } +} + +Status TreeCore::get(Slice key, GetView *out) +{ + if (out == nullptr) { + return Status::invalid_argument("point read needs output"); + } + detail::HostCompletion done; + Status status; + get_impl( + key, + [&](Status outcome, GetView result) { + status = std::move(outcome); + *out = std::move(result); + done.signal(); + }, + false); + done.wait(); + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_histograms.cpp b/lib/crowdb-tree/src/btree/tree_core_histograms.cpp new file mode 100644 index 000000000..3c51e5077 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_histograms.cpp @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_internal.h" + +namespace crowdb::tree +{ +Status TreeCore::enable_work_histogram(uint32_t stage) +try { + if (stage >= kTreeStageCount) { + return Status::invalid_argument(); + } + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable(); + } + const auto selected = static_cast<TreeStage>(stage); + const auto &metrics = state_->work; + if (metrics->histogram_enabled(selected)) { + return Status::Ok(); + } + const auto bound = detail::AllocationBound() + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add(TreeWorkMetrics::histogram_allocation_bytes()) + .value(); + if (!bound) { + return Status::resource_exhausted(); + } + auto reserved = detail::ResourceBudget::try_reserve(binding()->account(), detail::BudgetClass::kRouting, *bound); + if (!reserved) { + return metrics->histogram_enabled(selected) ? Status::Ok() : Status::resource_exhausted(); + } + auto credit = detail::AllocationReservation::create(std::move(*reserved)); + const auto status = metrics->enable_histogram(selected, credit); + credit->seal(); + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_import.cpp b/lib/crowdb-tree/src/btree/tree_core_import.cpp new file mode 100644 index 000000000..f5609c18c --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_import.cpp @@ -0,0 +1,184 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "native_import.h" +#include "portable_import.h" +#include "portable_import_plan.h" +#include "runtime_range_rebuild.h" +#include "snapshot/operation_wait.h" +#include "snapshot/recovered_state.h" +#include "tree_core_internal.h" + +namespace crowdb::tree +{ +namespace +{ +struct SealImport +{ + std::shared_ptr<detail::AllocationReservation> credit; + + ~SealImport() + { + if (credit) { + credit->seal(); + } + } +}; +} // namespace + +Status TreeCore::prepare_import_credit(std::shared_ptr<detail::AllocationReservation> *out) +{ + return prepare_import_credit(SIZE_MAX, out); +} + +Status TreeCore::prepare_import_credit(size_t requested, std::shared_ptr<detail::AllocationReservation> *out) +try { + if (out == nullptr || *out) { + return Status::invalid_argument(); + } + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("Tree import ingress closed or replacing"); + } + // Diagnostic reserved/allocated samples are not coherent during credit + // conversion. Plan from admission usage and include shared host pressure. + const auto available = detail::ResourceBudget::available(binding()->account(), detail::BudgetClass::kReclaim); + if (requested == SIZE_MAX) { + requested = available; + } + if (requested > available) { + return Status::resource_exhausted("import staging exceeds available reclaim capacity"); + } + auto bytes = detail::ResourceBudget::try_reserve(binding()->account(), detail::BudgetClass::kReclaim, requested); + if (!bytes) { + return Status::resource_exhausted("import staging admission unavailable"); + } + *out = detail::AllocationReservation::create(std::move(*bytes)); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("import admission allocation failed"); +} + +Status TreeCore::install_readable(std::shared_ptr<const ReadablePageCatalog> catalog, + const std::shared_ptr<detail::AllocationReservation> &credit) +{ + std::shared_ptr<const PublishedTreeState> candidate; + auto status = prepare_readable(std::move(catalog), credit, &candidate); + return status.ok() ? install_generation(std::move(candidate)) : status; +} + +Status TreeCore::prepare_readable(std::shared_ptr<const ReadablePageCatalog> catalog, + const std::shared_ptr<detail::AllocationReservation> &credit, + std::shared_ptr<const PublishedTreeState> *out) +try { + SealImport seal{credit}; + if (!catalog || !credit || credit->account() != binding()->account() || out == nullptr || *out) { + return Status::invalid_argument(); + } + std::shared_ptr<const PublishedTreeState> candidate; + { + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("Tree import ingress closed or replacing"); + } + auto io = detail::ResourceBudget::try_reserve(binding()->account(), detail::BudgetClass::kIo, 1); + if (!io) { + return Status::resource_exhausted("import validation IO admission unavailable"); + } + auto io_credit = detail::reserved_shared<detail::AllocationReservation>(credit, std::move(*io)); + SealImport seal_io{io_credit}; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto status = state_->lifetime->acquire(credit, &ticket); + if (!status.ok()) { + return status; + } + size_t max_frame_bytes = 0; + bool all_resident = true; + for (const auto &[pid, frame] : catalog->frames) { + (void)pid; + max_frame_bytes = std::max<size_t>(max_frame_bytes, frame.frame_bytes); + all_resident = all_resident && frame.backing && frame.backing->storage == BackingStorage::kResidentFrame; + } + auto loads = all_resident ? resident_import_load_limits(state_->loads, + std::max<size_t>(max_frame_bytes, config().frame_bytes)) + : state_->loads; + auto loader = detail::reserved_shared<AsyncStorePageLoader>( + credit, ticket->backend(async_backend()), binding()->cpu(), binding()->lane(), loads, + OverflowBackingResolver{}, PageLoadBudget{.bytes = credit, .io = io_credit}); + std::shared_ptr<RecoveredStateOperation> operation; + status = RecoveredStateOperation::create_readable(catalog, std::move(loader), binding()->cpu(), + binding()->lane(), {}, credit, &operation); + if (!status.ok()) { + return status; + } + status = detail::wait_snapshot_operation(operation); + if (!status.ok()) { + return status; + } + candidate = operation->result().state; + } + // The candidate retains exact page dependencies; discard the staging catalog + // before returning unused admission and reserving replacement controls. + catalog.reset(); + credit->seal(); + *out = std::move(candidate); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("import validation allocation failed"); +} + +Status TreeCore::install_native(std::vector<NativeFrame> frames, NativeStreamInfo info) +try { + SealImport budget; + auto status = prepare_import_credit(&budget.credit); + if (!status.ok()) { + return status; + } + std::shared_ptr<ReadablePageCatalog> catalog; + status = build_native_catalog(frames, info, config(), state_->loads, {}, budget.credit, &catalog); + return status.ok() ? install_readable(std::move(catalog), budget.credit) : status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("native import staging allocation failed"); +} + +Status TreeCore::install_portable(std::vector<leaf_entry> entries, uint64_t coverage) +try { + PortableImportPlan plan(config(), state_->loads); + for (const auto &entry : entries) { + const CellView cell{Slice(entry.cell)}; + if (!cell.valid()) { + return Status::invalid_argument("portable import cell has no complete header"); + } + const auto status = plan.append(entry.key.size(), cell.value().size(), cell.is_tombstone()); + if (!status.ok()) { + return status; + } + } + const auto shape = plan.finish(); + if (!shape) { + return Status::resource_exhausted("portable import shape exceeds its bounded frame namespace"); + } + RuntimeRangeImport budget; + auto status = reserve_range_shape(*this, *shape, true, budget); + if (!status.ok()) { + return status; + } + std::shared_ptr<ReadablePageCatalog> catalog; + { + PortableImportBuilder builder(config(), state_->loads, coverage, budget.credit); + for (const auto &entry : entries) { + status = builder.append(Slice(entry.key), Slice(entry.cell)); + if (!status.ok()) { + return status; + } + } + status = builder.finish(&catalog); + } + return status.ok() ? install_readable(std::move(catalog), budget.credit) : status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("portable import staging allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_internal.h b/lib/crowdb-tree/src/btree/tree_core_internal.h new file mode 100644 index 000000000..383822c4d --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_internal.h @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/apply_dispatcher.h" +#include "btree/apply_frontier.h" +#include "btree/apply_progress.h" +#include "btree/apply_waiters.h" +#include "btree/replay_controls.h" +#include "btree/source_rotation.h" +#include "btree/tree_core.h" +#include "btree/tree_core_split_internal.h" +#include "snapshot/checkpoint_operation.h" + +namespace crowdb::tree +{ +struct TreeCore::State +{ + static constexpr uint64_t kClosed = uint64_t{1} << 63; + static constexpr uint64_t kReplacing = uint64_t{1} << 62; + static constexpr uint64_t kPrepared = uint64_t{1} << 61; + static constexpr uint64_t kParticipants = ~(kClosed | kReplacing | kPrepared); + + State(Config options, std::shared_ptr<TreeRuntimeBinding> host); + ~State(); + Status initialize(TreeCoreBootstrap bootstrap); + Status capture_split_sources(TreeCore &core, uint64_t target); + Status install_split_sources(const SplitSourceSession &session, uint64_t identity, uint64_t end); + Status prepare_split_handoff(const SplitSourceSession &session, uint64_t identity, uint64_t end); + bool enter() noexcept; + void leave() noexcept; + void join_ingress() noexcept; + bool begin_replacement() noexcept; + void finish_replacement() noexcept; + void drain_ingress() noexcept; + static std::shared_ptr<PublishedTreeState> + empty_state(const std::shared_ptr<detail::AllocationReservation> &credit); + + struct Ingress + { + explicit Ingress(State &state) : owner(state.enter() ? &state : nullptr) + { + } + + ~Ingress() + { + if (owner != nullptr) { + owner->leave(); + } + } + + State *owner; + }; + + CheckpointDiagnostics checkpoint_diagnostics; + Config config; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<detail::AllocationReservation> source_lifecycle; + std::shared_ptr<TreeResourceMetrics> resources; + std::shared_ptr<TreeWorkMetrics> work; + std::shared_ptr<SharedPageCache> cache; + std::shared_ptr<LocalBackendState> local; + std::shared_ptr<CheckpointBackend> chunk; + std::shared_ptr<AsyncPageStore> backend; + std::shared_ptr<ApplyFrontier> frontier; + detail::ReservedUnique<TreeOperationLifetime> lifetime; + detail::ReservedUnique<TreeOperationLifetime> apply_lifetime; + detail::ReservedUnique<TreeOperationLifetime> replay_lifetime; + std::shared_ptr<ReplayControls> replay_controls; + std::shared_ptr<ApplyWaiters> replay_waiters; + std::shared_ptr<ApplyProgress> apply_progress; + std::shared_ptr<ApplyDispatcher> apply_dispatcher; + std::shared_ptr<SourceRotation> source_rotation; + detail::ReservedUnique<SourceAdmission> sources; + detail::ReservedUnique<PublicationCoordinator> publisher; + detail::ReservedUnique<FlushController> flush; + PageLoadLimits loads; + PublishedReadLimits reads; + FlushPreparationOptions preparation; + std::atomic<uint64_t> ingress{0}; + std::atomic<uint64_t> automatic_slot{0}; + std::atomic<uint8_t> close_state{0}; + std::atomic_flag split_busy = ATOMIC_FLAG_INIT; + std::shared_ptr<SplitSourceSession> split_sources; + std::shared_ptr<SplitHandoff> split_handoff; + uint64_t split_sequence = 0; +#ifdef CROWDB_TREE_TEST_UTIL + std::function<void()> before_generation_install_for_tests; +#endif +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_lifecycle.cpp b/lib/crowdb-tree/src/btree/tree_core_lifecycle.cpp new file mode 100644 index 000000000..009946473 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_lifecycle.cpp @@ -0,0 +1,238 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/checkpoint_backend.h" +#include "backend/local/local_backend_state.h" +#include "tree_core_internal.h" + +namespace crowdb::tree +{ +bool TreeCore::State::enter() noexcept +{ + auto value = ingress.load(std::memory_order_acquire); + do { + if ((value & (kClosed | kReplacing | kPrepared)) != 0) { + return false; + } + } while (!ingress.compare_exchange_weak(value, value + 1, std::memory_order_acq_rel)); + return true; +} + +void TreeCore::State::leave() noexcept +{ + ingress.fetch_sub(1, std::memory_order_acq_rel); + ingress.notify_all(); +} + +void TreeCore::State::join_ingress() noexcept +{ + ingress.fetch_or(kClosed, std::memory_order_acq_rel); + // A prepared capability has no running publisher. Closing revokes it + // without waiting for an external owner to destroy its handle. + ingress.fetch_and(~kPrepared, std::memory_order_acq_rel); + auto value = ingress.load(std::memory_order_acquire); + while (value != kClosed) { + ingress.wait(value, std::memory_order_acquire); + value = ingress.load(std::memory_order_acquire); + } +} + +bool TreeCore::State::begin_replacement() noexcept +{ + auto value = ingress.load(std::memory_order_acquire); + do { + if ((value & (kClosed | kReplacing | kPrepared)) != 0) { + return false; + } + } while (!ingress.compare_exchange_weak(value, value | kReplacing, std::memory_order_acq_rel)); + source_rotation->stop(); + drain_ingress(); + source_rotation->drain(); + return true; +} + +void TreeCore::State::drain_ingress() noexcept +{ + auto value = ingress.load(std::memory_order_acquire); + while ((value & kParticipants) != 0) { + ingress.wait(value, std::memory_order_acquire); + value = ingress.load(std::memory_order_acquire); + } +} + +void TreeCore::State::finish_replacement() noexcept +{ + if ((ingress.load(std::memory_order_acquire) & kClosed) == 0) { + source_rotation->restart(); + } + ingress.fetch_and(~kReplacing, std::memory_order_acq_rel); + ingress.notify_all(); +} + +void TreeCore::close_and_drain() noexcept +{ + uint8_t expected = 0; + if (!state_->close_state.compare_exchange_strong(expected, 1, std::memory_order_acq_rel)) { + while (expected != 2) { + state_->close_state.wait(expected, std::memory_order_acquire); + expected = state_->close_state.load(std::memory_order_acquire); + } + return; + } + // Closing is visible before waiting for an in-progress replacement. Its + // release clears only kReplacing, so no install can reopen this boundary. + state_->ingress.fetch_or(State::kClosed, std::memory_order_acq_rel); + // A host ingress participant may await this unaccepted soft pause. Release + // it before joining ingress; accepted apply owns its separate flush path. + state_->source_rotation->stop(); + state_->lifetime->close(); + state_->apply_lifetime->close(); + state_->replay_lifetime->close(); + state_->join_ingress(); + // Already armed proposals can still become chosen. Their prepaid apply + // must finish while source rotation and handoff remain available. + state_->apply_lifetime->close_and_drain(); + state_->replay_lifetime->close_and_drain(); + state_->source_rotation->drain(); + state_->apply_progress->close(); + state_->split_handoff.reset(); + state_->split_sources.reset(); + state_->sources->close(); + state_->flush->close(); + state_->flush->release_pending(); + state_->lifetime->close_and_drain(); + state_->apply_dispatcher->close_and_drain(); + state_->publisher->close(); + state_->cache->close_and_drain(); + state_->binding->drain(); + if (state_->chunk) { + state_->chunk->close_writer(); + } + state_->publisher->reclaim(); + state_->sources->reclaim(); + state_->binding->account()->drain_retirements(); + state_->close_state.store(2, std::memory_order_release); + state_->close_state.notify_all(); +} + +bool TreeCore::closed() const noexcept +{ + return (state_->ingress.load(std::memory_order_acquire) & State::kClosed) != 0; +} + +const Config &TreeCore::config() const noexcept +{ + return state_->config; +} + +const std::shared_ptr<TreeRuntimeBinding> &TreeCore::binding() const noexcept +{ + return state_->binding; +} + +const std::shared_ptr<SharedPageCache> &TreeCore::cache() const noexcept +{ + return state_->cache; +} + +const std::shared_ptr<LocalBackendState> &TreeCore::local_backend() const noexcept +{ + return state_->local; +} + +const std::shared_ptr<CheckpointBackend> &TreeCore::chunk_backend() const noexcept +{ + return state_->chunk; +} + +uint64_t TreeCore::durable_coverage() const noexcept +{ + if (state_->chunk) { + return state_->chunk->durable_coverage.load(std::memory_order_acquire); + } + return state_->local ? state_->local->checkpoints->durable_coverage() : 0; +} + +uint64_t TreeCore::snapshot_sequence() const noexcept +{ + if (state_->chunk) { + return state_->chunk->snapshot_sequence.load(std::memory_order_acquire); + } + return state_->local ? state_->local->checkpoints->snapshot_sequence() : 0; +} + +AsyncPageStore *TreeCore::async_backend() const noexcept +{ + return state_->backend.get(); +} + +PageLoadLimits TreeCore::load_limits() const noexcept +{ + return state_->loads; +} + +PublishedReadLimits TreeCore::read_limits() const noexcept +{ + return state_->reads; +} + +std::shared_ptr<const PublishedTreeState> TreeCore::capture() const +{ + return state_->publisher->capture(); +} + +PublicationCoordinator &TreeCore::publisher() const noexcept +{ + return *state_->publisher; +} + +TreeOperationLifetime &TreeCore::operation_lifetime() const noexcept +{ + return *state_->lifetime; +} + +void TreeCore::request_gc_floor(uint64_t floor) noexcept +{ + State::Ingress admission(*state_); + if (admission.owner != nullptr) { + state_->publisher->request_gc_floor(floor); + } +} + +uint64_t TreeCore::requested_gc_floor() const noexcept +{ + return state_->publisher->requested_gc_floor(); +} + +uint64_t TreeCore::contiguous_slot() const noexcept +{ + return state_->frontier->contiguous().load(std::memory_order_acquire); +} + +uint64_t TreeCore::max_seen_slot() const noexcept +{ + return state_->frontier->max_seen(); +} + +size_t TreeCore::memtable_count() const +{ + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return 0; + } + size_t result = 0; + for (const auto &table : state_->sources->catalog()->tables) { + result += table->count(); + } + return result; +} + +size_t TreeCore::frozen_table_count() const +{ + State::Ingress admission(*state_); + if (admission.owner == nullptr) { + return 0; + } + const auto catalog = state_->sources->catalog(); + return catalog->tables.size() - 1; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_metrics.cpp b/lib/crowdb-tree/src/btree/tree_core_metrics.cpp new file mode 100644 index 000000000..14327a125 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_metrics.cpp @@ -0,0 +1,163 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_account.h" +#include "btree/tree_core.h" +#include "btree/tree_work_registry.h" +#include "runtime/allocation_memory_resource.h" + +namespace crowdb::tree +{ +namespace +{ +void register_source_metrics(MetricsRegistry ®istry, TreeCore &core, const std::string &prefix) +{ + auto source = [&](const char *name, uint64_t SourceMemoryStats::*field) { + registry.register_callback_gauge_inline(prefix, name, + [target = &core, field] { return target->source_stats().*field; }); + }; + source(".mt.tables.g", &SourceMemoryStats::tables); + source(".mt.frozen.g", &SourceMemoryStats::frozen_tables); + source(".mt.writers.g", &SourceMemoryStats::writers); + source(".mt.freezing.writers.g", &SourceMemoryStats::freezing_writers); + source(".mt.frozen.retained.bytes.g", &SourceMemoryStats::frozen_retained_bytes); + source(".mt.records.g", &SourceMemoryStats::records); + source(".mt.logical.bytes.g", &SourceMemoryStats::logical_bytes); + auto memory = [&](const char *name, uint64_t VersionMemory::*field) { + registry.register_callback_gauge_inline( + prefix, name, [target = &core, field] { return target->source_stats().memory.*field; }); + }; + // Linked-source summaries preserve the existing schema. Unique allocation + // capacities and old-reader retention belong to the physical owner gauges. + memory(".mt.history.live.g", &VersionMemory::history_count); + memory(".mt.history.bytes.g", &VersionMemory::history_bytes); + memory(".mt.descriptor.bytes.g", &VersionMemory::descriptor_bytes); + memory(".mt.node.bytes.g", &VersionMemory::node_bytes); + memory(".mt.payload.bytes.g", &VersionMemory::payload_bytes); + registry.register_callback_gauge_inline(prefix, ".mt.summary.known.g", [target = &core] { + return static_cast<uint64_t>(target->source_stats().known); + }); +} +} // namespace + +void Crowdbtree::init_core_metrics(const std::string &prefix) +{ + // Registry storage grows with the schema and prefix, independently of the + // fixed bootstrap reserve. Every real allocation still obeys Tree/host quota. + std::shared_ptr<detail::AllocationAccount> budget; + if (!detail::AllocationAccount::create(core_->binding()->account(), detail::BudgetClass::kRouting, &budget).ok()) { + throw std::bad_alloc(); + } + auto resource_owner = std::allocate_shared<detail::AllocationMemoryResource>( + MetadataAllocator<detail::AllocationMemoryResource>(budget), budget); + auto registry_owner = create_tree_work_registry(resource_owner, prefix, core_->binding()->work_metrics(), budget); + auto *registry = registry_owner.get(); + register_source_metrics(*registry, *core_, prefix); + auto logical = [this, registry, &prefix](const char *name, uint64_t TreeLogicalStats::*field) { + registry->register_callback_gauge_inline(prefix, name, + [this, field] { return core_->capture()->stats.*field; }); + }; + logical(".tree.live.leaf_pages", &TreeLogicalStats::leaf_pages); + logical(".tree.live.inner_pages", &TreeLogicalStats::inner_pages); + logical(".tree.live.height", &TreeLogicalStats::height); + logical(".tree.live.dirty_pages", &TreeLogicalStats::dirty_pages); + logical(".tree.live.overlay_pages", &TreeLogicalStats::overlay_pages); + logical(".tree.live.overlay_runs", &TreeLogicalStats::overlay_runs); + logical(".tree.live.overlay_entries", &TreeLogicalStats::overlay_entries); + logical(".tree.live.kv_known_count", &TreeLogicalStats::kv_known_count); + logical(".tree.live.kv_unknown_pages", &TreeLogicalStats::kv_unknown_pages); + logical(".tree.live.checkpoint_pending_pages", &TreeLogicalStats::checkpoint_pending_pages); + registry->register_callback_gauge_inline(prefix, ".tree.resident.frames", + [this] { return core_->state_stats().resources.resident_frames; }); + registry->register_callback_gauge_inline(prefix, ".tree.resident.cache_bytes", + [this] { return core_->state_stats().resources.cache_bytes; }); + registry->register_callback_gauge_inline(prefix, ".tree.resident.pinned_bytes", + [this] { return core_->state_stats().resources.cache_pinned_bytes; }); + registry->register_callback_gauge_inline(prefix, ".tree.live.proxy_pages", + [this] { return core_->state_stats().resources.proxy_pages; }); + logical(".tree.leaf.count.g", &TreeLogicalStats::leaf_pages); + logical(".tree.inner.count.g", &TreeLogicalStats::inner_pages); + logical(".tree.height.g", &TreeLogicalStats::height); + logical(".page.dirty.count.g", &TreeLogicalStats::dirty_pages); + logical(".page.overlay.count.g", &TreeLogicalStats::overlay_pages); + logical(".overlay.runs.g", &TreeLogicalStats::overlay_runs); + logical(".overlay.entries.g", &TreeLogicalStats::overlay_entries); + logical(".page.size.known.bytes.g", &TreeLogicalStats::page_size_known_bytes); + logical(".page.size.unknown.count.g", &TreeLogicalStats::page_size_unknown_pages); + logical(".page.size.estimate.bytes.g", &TreeLogicalStats::page_size_estimate_bytes); + logical(".page.size.max.observed.g", &TreeLogicalStats::page_size_max_observed_bytes); + logical(".tree.kv.known.count.g", &TreeLogicalStats::kv_known_count); + logical(".tree.kv.known.bytes.g", &TreeLogicalStats::kv_known_bytes); + logical(".tree.kv.unknown.page.count.g", &TreeLogicalStats::kv_unknown_pages); + logical(".candidate.consolidate.pages.g", &TreeLogicalStats::consolidate_candidate_pages); + logical(".candidate.split.pages.g", &TreeLogicalStats::split_candidate_pages); + logical(".candidate.split.ready.pages.g", &TreeLogicalStats::split_ready_pages); + logical(".candidate.merge.pages.g", &TreeLogicalStats::merge_candidate_pages); + logical(".candidate.merge.pairs.g", &TreeLogicalStats::merge_candidate_pairs); + logical(".candidate.merge.ready.pairs.g", &TreeLogicalStats::merge_ready_pairs); + logical(".candidate.writeback.pages.g", &TreeLogicalStats::writeback_candidate_pages); + logical(".candidate.evict.pages.g", &TreeLogicalStats::evict_candidate_pages); + logical(".page.checkpoint.pending.count.g", &TreeLogicalStats::checkpoint_pending_pages); + auto resource = [this, registry, &prefix](const char *name, uint64_t TreeResourceStats::*field) { + registry->register_callback_gauge_inline(prefix, name, + [this, field] { return core_->state_stats().resources.*field; }); + }; + resource(".memory.allocated.bytes.g", &TreeResourceStats::allocated_bytes); + resource(".memory.reserved.bytes.g", &TreeResourceStats::reserved_bytes); + resource(".memory.payload.bytes.g", &TreeResourceStats::payload_bytes); + resource(".memory.frame.bytes.g", &TreeResourceStats::frame_bytes); + resource(".memory.metadata.bytes.g", &TreeResourceStats::metadata_bytes); + resource(".memory.io.bytes.g", &TreeResourceStats::io_bytes); + resource(".memory.scratch.bytes.g", &TreeResourceStats::scratch_bytes); + resource(".memory.retired.bytes.g", &TreeResourceStats::retired_bytes); + resource(".cache.frames.g", &TreeResourceStats::resident_frames); + resource(".io.read.inflight.bytes.g", &TreeResourceStats::read_inflight_bytes); + resource(".io.write.inflight.bytes.g", &TreeResourceStats::write_inflight_bytes); + resource(".page.resident.frames.g", &TreeResourceStats::resident_frames); + resource(".cache.bytes.g", &TreeResourceStats::cache_bytes); + resource(".cache.pinned.bytes.g", &TreeResourceStats::cache_pinned_bytes); + resource(".page.proxy.count.g", &TreeResourceStats::proxy_pages); + resource(".read.inflight.bytes.g", &TreeResourceStats::read_inflight_bytes); + resource(".write.inflight.bytes.g", &TreeResourceStats::write_inflight_bytes); + resource(".materialize.inflight.bytes.g", &TreeResourceStats::materialize_inflight_bytes); + resource(".backing.allocated.bytes.g", &TreeResourceStats::allocated_backing_bytes); + resource(".maintenance.queued.g", &TreeResourceStats::maintenance_queued); + resource(".maintenance.inflight.g", &TreeResourceStats::maintenance_inflight); + registry->register_callback_gauge_inline(prefix, ".tree.page.count.g", + [this] { return core_->capture()->stats.page_count(); }); + registry->register_callback_gauge_inline(prefix, ".checkpoint.mapping.pending.g", [this] { + return static_cast<uint64_t>(core_->capture()->stats.checkpoint_mapping_pending); + }); + registry->register_callback_gauge_inline(prefix, ".candidate.oldest.age.ns.g", + [this] { return core_->state_stats().progress.candidate_oldest_age_ns; }); + registry->register_callback_gauge_inline(prefix, ".checkpoint.oldest.pending.age.ns.g", [this] { + return core_->state_stats().progress.checkpoint_oldest_pending_age_ns; + }); + registry->register_callback_gauge_inline(prefix, ".frontier.published.g", + [this] { return core_->capture()->coverage; }); + registry->register_callback_gauge_inline(prefix, ".frontier.durable.g", + [this] { return durable_snapshot_last_applied_slot(); }); + registry->register_callback_gauge_inline(prefix, ".flush.repair.pending.g", [this] { + return static_cast<uint64_t>(core_->capture()->pending_handoff.has_value()); + }); + registry->register_callback_gauge_inline(prefix, ".checkpoint.inflight.g", [this] { + return static_cast<uint64_t>(core_->capture()->checkpoint_inflight); + }); + registry->register_callback_gauge_inline(prefix, ".maintenance.oldest.age.ns.g", [this] { + return core_->state_stats().progress.maintenance_oldest_age_ns; + }); + auto dispatch = [this, registry, &prefix](const char *name, uint64_t FlushDispatchSnapshot::*field) { + registry->register_callback_gauge_inline( + prefix, name, [this, field] { return core_->binding()->work_metrics()->dispatch.snapshot().*field; }); + }; + dispatch(".flush.dispatch.inline.probes.total", &FlushDispatchSnapshot::inline_probes); + dispatch(".flush.dispatch.inline.ranges.total", &FlushDispatchSnapshot::inline_ranges); + dispatch(".flush.dispatch.queued.steps.total", &FlushDispatchSnapshot::queued_steps); + dispatch(".flush.dispatch.runs.total", &FlushDispatchSnapshot::prepared_runs); + dispatch(".flush.dispatch.entries.total", &FlushDispatchSnapshot::prepared_entries); + dispatch(".flush.dispatch.bytes.total", &FlushDispatchSnapshot::prepared_bytes); + dispatch(".flush.dispatch.cow.bytes.total", &FlushDispatchSnapshot::cow_bytes); + dispatch(".flush.dispatch.active.steps.g", &FlushDispatchSnapshot::active_steps); + dispatch(".flush.dispatch.peak.steps.g", &FlushDispatchSnapshot::peak_steps); + metrics_registry_ = std::move(registry_owner); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_open.cpp b/lib/crowdb-tree/src/btree/tree_core_open.cpp new file mode 100644 index 000000000..9ae10ce96 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_open.cpp @@ -0,0 +1,238 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/block_account.h" +#include "backend/local/local_backend_state.h" +#include "backend/local/scheduled_page_store.h" +#include "btree/allocation_bound.h" +#include "btree/backing_identity.h" +#include "btree/tree_core.h" +#include "crowdb-tree/backend/page_store.h" +#include "snapshot/local_recovery.h" +#include "snapshot/operation_wait.h" +#include "snapshot/recovered_state.h" + +#include <algorithm> +#include <bit> + +namespace crowdb::tree +{ +namespace +{ +struct OpenCredits +{ + PageLoadBudget budget; + + ~OpenCredits() + { + detail::seal_backend_budget(budget.backend); + if (budget.bytes) { + budget.bytes->seal(); + } + if (budget.io) { + budget.io->seal(); + } + } +}; + +struct OpenLifetime +{ + OpenLifetime(const std::shared_ptr<TreeRuntimeBinding> &binding, std::shared_ptr<const void> owner) + : lifetime(1, binding->lifecycle_budget(), std::move(owner)), + binding(binding) + { + } + + ~OpenLifetime() + { + store.reset(); + ticket.reset(); + lifetime.close_and_drain(); + binding->drain(); + } + + TreeOperationLifetime lifetime; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<AsyncPageStore> store; +}; + +Status reserve_open(const std::shared_ptr<TreeRuntimeBinding> &binding, const detail::BackendOpenBudget &backend, + OpenCredits *credits, std::optional<size_t> recovery_bound) +{ + const auto &options = binding->options(); + if (options.limits.routing_bytes <= options.lifecycle_bytes || options.limits.io_operations == 0) { + return Status::resource_exhausted(); + } + auto available = options.limits.routing_bytes - options.lifecycle_bytes; + for (const auto retained : {backend.owned_bytes, backend.growth_bytes}) { + if (retained >= available) { + return Status::resource_exhausted("backend metadata leaves no recovery workspace"); + } + available -= retained; + } + if (recovery_bound) { + if (*recovery_bound > available) { + return Status::resource_exhausted("empty recovery exceeds Tree routing limit"); + } + available = *recovery_bound; + } + auto bytes = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kRouting, available); + auto io = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kIo, 1); + if (!bytes || !io) { + return Status::resource_exhausted("recovery routing or IO credit unavailable"); + } + credits->budget.bytes = detail::AllocationReservation::create(std::move(*bytes)); + credits->budget.io = detail::reserved_shared<detail::AllocationReservation>(credits->budget.bytes, std::move(*io)); + return Status::Ok(); +} + +Status recover_open(const Config &config, const std::shared_ptr<TreeRuntimeBinding> &binding, + const std::shared_ptr<LocalBackendState> &local, PageLoadBudget budget, + std::shared_ptr<const PublishedTreeState> *initial) +{ + OpenLifetime lifetime(binding, local); + auto status = lifetime.lifetime.acquire(binding->lifecycle_budget(), &lifetime.ticket); + if (!status.ok()) { + return status; + } + lifetime.store = lifetime.ticket->backend(local->store.get()); + LocalRecoveryLimits limits; + limits.max_frame_bytes = std::max<size_t>(limits.max_frame_bytes, config.frame_bytes); + limits.max_blob_bytes = round_up_to_iu(limits.max_frame_bytes + kDurableBlobHeader, config.page_store->iu_size()); + ExtentLedgerLimits domain{.backend_id = detail::allocate_backend_identity(), + .region_begin = snapshot::region_base_for(config.page_store->iu_size()), + .file_size = detail::backend_size(*config.page_store, budget.backend), + .iu_size = config.page_store->iu_size()}; + if (domain.backend_id == 0) { + return Status::resource_exhausted(); + } + std::shared_ptr<LocalRecoveryOperation> recovery; + status = LocalRecoveryOperation::create(lifetime.store, binding->cpu(), binding->lane(), domain, limits, budget, + &recovery); + if (!status.ok()) { + return status; + } + status = detail::wait_snapshot_operation(recovery); + if (!status.ok()) { + return status; + } + local->ledger = recovery->result().ledger; + local->checkpoints = recovery->result().registry; + local->inherited = recovery->result().latest; + if (!local->inherited) { + initial->reset(); + return Status::Ok(); + } + PageLoadLimits loads; + loads.max_blob_bytes = limits.max_blob_bytes; + loads.max_frame_bytes = limits.max_frame_bytes; + auto loader = detail::reserved_shared<AsyncStorePageLoader>( + budget.bytes, lifetime.store, binding->cpu(), binding->lane(), loads, OverflowBackingResolver{}, budget); + std::shared_ptr<RecoveredStateOperation> recovered; + status = RecoveredStateOperation::create(local->inherited, loader, binding->cpu(), binding->lane(), {}, + budget.bytes, &recovered); + if (!status.ok()) { + return status; + } + status = detail::wait_snapshot_operation(recovered); + if (status.ok()) { + *initial = recovered->result().state; + } + return status; +} +} // namespace + +Status TreeCore::open(const Config &config, std::shared_ptr<TreeCore> *out) +try { + if (out == nullptr || !config.runtime || config.page_store == nullptr || config.page_store->iu_size() == 0 || + config.frame_bytes == 0 || config.frame_bytes % config.page_store->iu_size() != 0) { + return Status::invalid_argument(); + } + if (const auto factory = config.page_store->runtime_factory()) { + return factory(config, out); + } + if ((config.async_page_store != nullptr && !config.async_page_store->write_completion_is_reloadable()) || + config.page_store->has_shared_ownership()) { + return Status::not_supported("runtime local recovery requires an " + "independently reloadable local backend"); + } + auto status = config.key_range.validate(); + if (!status.ok()) { + return status; + } + std::shared_ptr<TreeRuntimeBinding> binding; + status = TreeRuntimeBinding::create(config.runtime, config.runtime_binding, &binding); + if (!status.ok()) { + return status; + } + auto local = detail::reserved_shared<LocalBackendState>(binding->lifecycle_budget()); + status = ScheduledPageStore::create(config.page_store, config.async_page_store, config.backend_owner, binding, + binding->lifecycle_budget(), binding->backend_lane(), &local->store); + if (!status.ok()) { + return status; + } + status = + LocalBackendDomain::claim(config.page_store, config.backend_owner, binding->lifecycle_budget(), &local->domain); + if (!status.ok()) { + return status; + } + detail::BackendOpenBudget backend_metadata; + status = detail::bind_block_account(config.page_store, binding->account(), + config.runtime->options().backend_workers, &backend_metadata); + if (!status.ok()) { + return status; + } + OpenCredits credits; + const detail::BackendOperationLimits backend_limits{ + .max_io_bytes = round_up_to_iu(std::max<size_t>(1024 * 1024, config.frame_bytes) + kDurableBlobHeader, + config.page_store->iu_size()), + .write_calls = 0, + .workers = config.runtime->options().workers + config.runtime->options().backend_workers, + .barriers = false}; + const auto backend_bound = detail::backend_operation_bound(local->store.get(), backend_limits); + if (!backend_bound) { + return Status::resource_exhausted("backend recovery bound overflow"); + } + status = detail::reserve_backend_operation(local->store.get(), binding->account(), backend_limits, + &credits.budget.backend); + if (!status.ok()) { + return status; + } + backend_metadata.growth_bytes = *backend_bound; + std::optional<size_t> recovery_bound; + // Domain ownership excludes a concurrent writer throughout this preflight + // and normal recovery. A shorter file cannot contain either commit anchor. + if (detail::backend_size(*config.page_store, credits.budget.backend) < + snapshot::superblock_slot_bytes(config.page_store->iu_size())) { + LocalRecoveryLimits limits; + limits.max_frame_bytes = std::max<size_t>(limits.max_frame_bytes, config.frame_bytes); + limits.max_blob_bytes = + round_up_to_iu(limits.max_frame_bytes + kDurableBlobHeader, config.page_store->iu_size()); + recovery_bound = detail::AllocationBound() + .add(LocalRecoveryOperation::empty_allocation_bound(limits, config.page_store->iu_size())) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .value(); + if (!recovery_bound) { + return Status::resource_exhausted("empty recovery bound overflow"); + } + } + status = reserve_open(binding, backend_metadata, &credits, recovery_bound); + if (!status.ok()) { + return status; + } + std::shared_ptr<const PublishedTreeState> initial; + status = recover_open(config, binding, local, credits.budget, &initial); + if (!status.ok()) { + return status; + } + credits.budget.bytes->seal(); + credits.budget.io->seal(); + return create(config, binding, {.initial = std::move(initial), .local = std::move(local)}, out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (const std::exception &) { + return Status::internal_error(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_pressure.cpp b/lib/crowdb-tree/src/btree/tree_core_pressure.cpp new file mode 100644 index 000000000..cde13b34b --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_pressure.cpp @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "tree_core_checkpoint_internal.h" + +namespace crowdb::tree::checkpoint_detail +{ +namespace +{ +std::optional<size_t> control_bound() noexcept +{ + using namespace detail; + return AllocationBound(CheckpointOperation::control_allocation_bound()) + .add(shared_allocation_bound<Preparation>()) + .add(shared_allocation_bound<AsyncWork>()) + .add_product(2, shared_allocation_bound<Credits>()) + .add(shared_allocation_bound<AllocationReservation>()) + .value(); +} + +size_t finish_bound() noexcept +{ + return CheckpointOperation::finish_allocation_bound() + + detail::shared_allocation_bound<detail::AllocationReservation>(); +} +} // namespace + +std::optional<size_t> pressure_control_bound() noexcept +{ + return detail::AllocationBound().add(control_bound()).add(finish_bound()).value(); +} + +Status prepare_pressure(TreeCore &core, std::shared_ptr<TreeOperationLifetime::Ticket> ticket, uint64_t leaf, + detail::ResourceBudget::Credit credit, std::shared_ptr<ApplyDispatcher> cpu_progress, + std::shared_ptr<PublicationLane> publication, std::shared_ptr<ApplyProgress> progress, + std::shared_ptr<ApplyWaiters> waiters, PageMaintenanceOperationOptions structure_options, + std::shared_ptr<CheckpointOperation> *out) +{ + using namespace detail; + const auto required = pressure_control_bound(); + const auto controls = control_bound(); + if (!required || !controls || !ticket || !cpu_progress || !publication || !progress || out == nullptr || *out || + (!core.local_backend() && !core.chunk_backend()) || credit.amount() != *required || credit.allocated() || + !credit.belongs_to(core.binding()->account(), BudgetClass::kReclaim)) { + return Status::invalid_argument("pressure writeback requires its prepaid control and owned local backend"); + } + auto control = credit.split(*controls); + if (!control) { + return Status::internal_error("pressure control split failed"); + } + auto metadata = AllocationReservation::create(std::move(*control)); + auto preparation = reserved_shared<Preparation>(metadata); + preparation->control = metadata; + preparation->finish = AllocationReservation::create(std::move(credit)); + preparation->ticket = std::move(ticket); + preparation->core = &core; + preparation->options = defaults(core); + preparation->options.mode = CheckpointMode::Writeback; + preparation->options.publication = std::move(publication); + preparation->options.write.frames.selected_page = leaf; + preparation->structure_options = std::move(structure_options); + preparation->cpu_progress = std::move(cpu_progress); + preparation->waiters = std::move(waiters); + auto work = reserved_shared<AsyncWork>(metadata, preparation); + PageLoadBudget budget{.bytes = metadata, .io = {}}; + budget.cpu_progress = preparation->cpu_progress; + return CheckpointOperation::create( + preparation, core.publisher(), core.local_backend(), preparation->ticket->backend(core.async_backend()), + core.binding()->cpu(), core.binding()->lane(), preparation->options, std::move(budget), preparation->finish, {}, + {}, {}, out, {}, + [work](const PublishedTreeState &view, const CapturedWritePlan *plan, CheckpointWork *output) { + return work->prepare(view, plan, output); + }, + {}, [work] { work->cancel(); }, core.chunk_backend(), + [preparation](std::shared_ptr<const PublishedTreeState> view, uint64_t pid, + std::shared_ptr<PageSplitOperation> *operation) { + return preparation->structure(std::move(view), pid, operation); + }); +} +} // namespace crowdb::tree::checkpoint_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_read_budget.cpp b/lib/crowdb-tree/src/btree/tree_core_read_budget.cpp new file mode 100644 index 000000000..ad2a82226 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_read_budget.cpp @@ -0,0 +1,114 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/read_value_bound.h" +#include "btree/tree_core_read_internal.h" + +#include <algorithm> + +namespace crowdb::tree::core_read_detail +{ +Status prepare(TreeCore &core, bool scan, size_t control_bytes, size_t result_bytes, ReadPreparation *out) +{ + return prepare(core, core.capture(), core.read_limits(), scan, control_bytes, result_bytes, out); +} + +Status prepare(TreeCore &core, std::shared_ptr<const PublishedTreeState> view, PublishedReadLimits limits, bool scan, + size_t control_bytes, size_t result_bytes, ReadPreparation *out) +{ + using namespace detail; + ReadPreparation ready; + ready.binding = core.binding(); + ready.view = std::move(view); + ready.limits = std::move(limits); + const auto root = ready.view->mapping.get(ready.view->root_page_id); + if (core.closed() || !root || root->level >= ready.limits.max_route_depth) { + return Status::unavailable("read requires an open bounded published root"); + } + if (ready.view->sources->tables.size() > ready.limits.max_memtables) { + return Status::resource_exhausted("captured read exceeds the source count limit"); + } + ready.limits.max_route_depth = root->level + 1ULL; + ready.limits.max_memtables = std::max<size_t>(1, ready.view->sources->tables.size()); + auto loads = core.load_limits(); + loads.max_inflight = 1; + const size_t workers = ready.binding->cpu()->worker_count(); + const auto shape = ready.view->mapping.copy_bounds(); + ready.limits.page_metadata_bytes = shape.max_leaf_bytes; + const auto captured_bases = + AllocationBound().add_product(shape.leaf_pages, shape.max_leaf_bases).value().value_or(SIZE_MAX); + const auto leaf_window = AllocationBound(shape.max_leaf_bases).add(scan ? ready.limits.max_entries : 0).value(); + if (!leaf_window) { + return Status::resource_exhausted("captured read frame window overflow"); + } + const size_t leaves = std::min(*leaf_window, scan ? captured_bases : shape.max_leaf_bases); + const size_t values = scan ? ready.limits.max_entries : 1; + const size_t cached_pages = core.cache()->retained_page_limit(); + // One point lookup visits one routing path and one logical leaf. A scan + // can retain more decoded pages, but cannot visit absent index PIDs. + const size_t cached_inner = std::min<uint64_t>(cached_pages, scan ? ready.view->stats.inner_pages : root->level); + const size_t cached_leaf = std::min(cached_pages, scan ? captured_bases : shape.max_leaf_bases); + // Only fixed L1 cells can require an overflow allocation. L0 cells lend + // their independent payload owners, including writes after this capture. + const auto value_limit = overflow_value_bytes_bound(*ready.view, loads.max_value_bytes); + const auto value_bytes = + AllocationBound(value_limit).add(scan && value_limit != 0 ? ready.limits.max_result_bytes : 0).value(); + auto bounded_loads = loads; + // Every accepted overflow frame contributes at least one value byte. This + // tightens reservation only; the loader's public validation caps stay fixed. + bounded_loads.max_overflow_pages = std::min<uint64_t>(loads.max_overflow_pages, value_limit); + auto loader_bound = + AsyncStorePageLoader::primary_reserve_bound(loads, root->level, workers, leaves, cached_leaf, cached_inner); + if (value_limit != 0) { + loader_bound = value_bytes + ? AsyncStorePageLoader::read_reserve_bound(bounded_loads, root->level, workers, leaves, values, + *value_bytes, cached_leaf, cached_inner) + : std::nullopt; + } + const auto bound = AllocationBound(control_bytes) + .add_product(2, shared_allocation_bound<AllocationReservation>()) + .add(TreeOperationLifetime::ticket_allocation_bound()) + .add(scan ? PublishedScan::allocation_bound(ready.limits, workers) + : PublishedGet::allocation_bound(ready.limits, workers)) + .add(loader_bound) + .add(core.cache()->load_reserve_bound(root->level + leaves, workers)) + .add(SharedPageCache::binding_allocation_bound()) + .add(result_bytes) + .value(); + if (!bound) { + return Status::resource_exhausted("read admission bound overflow"); + } + auto bytes = ResourceBudget::try_reserve(ready.binding->account(), BudgetClass::kRouting, *bound); + auto io = ResourceBudget::try_reserve(ready.binding->account(), BudgetClass::kIo, 1); + if (!bytes || !io) { + return Status::resource_exhausted("read progress reserve unavailable"); + } + ready.credit = AllocationReservation::create(std::move(*bytes)); + ready.io = reserved_shared<AllocationReservation>(ready.credit, std::move(*io)); + ready.limits.metadata = ready.credit; + auto backend_status = reserve_backend_operation( + core.async_backend(), ready.binding->account(), + {.max_io_bytes = loads.max_blob_bytes, .write_calls = 0, .workers = workers, .barriers = false}, + &ready.backend); + if (!backend_status.ok()) { + return backend_status; + } + auto status = core.operation_lifetime().acquire(ready.credit, &ready.ticket); + if (!status.ok()) { + return status; + } + auto loader = reserved_shared<AsyncStorePageLoader>(ready.credit, ready.ticket->backend(core.async_backend()), + ready.binding->cpu(), ready.binding->lane(), loads, + OverflowBackingResolver{}, + PageLoadBudget{.bytes = ready.credit, + .io = ready.io, + .work_metrics = core.publisher().work_metrics(), + .backend = ready.backend, + .cause = TreeWorkCause::Read}); + status = core.cache()->bind_reserved(std::move(loader), ready.credit, &ready.loader); + if (status.ok()) { + *out = std::move(ready); + } + return status; +} +} // namespace crowdb::tree::core_read_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_read_internal.h b/lib/crowdb-tree/src/btree/tree_core_read_internal.h new file mode 100644 index 000000000..0c412184b --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_read_internal.h @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/local/block_account.h" +#include "btree/tree_core.h" + +namespace crowdb::tree::core_read_detail +{ +struct ReadPreparation +{ + ReadPreparation() = default; + ReadPreparation(ReadPreparation &&) noexcept = default; + ReadPreparation &operator=(ReadPreparation &&) noexcept = default; + + ~ReadPreparation() + { + detail::seal_backend_budget(backend); + } + + std::shared_ptr<AllocationBudget> backend; + std::shared_ptr<detail::AllocationReservation> credit; + std::shared_ptr<detail::AllocationReservation> io; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<const PublishedTreeState> view; + std::shared_ptr<PageLoader> loader; + PublishedReadLimits limits; +}; + +// All admission and source capture happens before the first CPU submission. +Status prepare(TreeCore &core, bool scan, size_t control_bytes, size_t result_bytes, ReadPreparation *out); +Status prepare(TreeCore &core, std::shared_ptr<const PublishedTreeState> view, PublishedReadLimits limits, bool scan, + size_t control_bytes, size_t result_bytes, ReadPreparation *out); +} // namespace crowdb::tree::core_read_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_scan.cpp b/lib/crowdb-tree/src/btree/tree_core_scan.cpp new file mode 100644 index 000000000..20e6bae0c --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_scan.cpp @@ -0,0 +1,150 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/scan_output.h" +#include "btree/tree_core_internal.h" +#include "btree/tree_core_scan_internal.h" + +#include <algorithm> +#include <chrono> +#include <utility> + +namespace crowdb::tree::core_read_detail +{ +bool CoreScan::expired() const noexcept +{ + if (deadline_ms == 0) { + return false; + } + const auto now = + std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()) + .count(); + return std::cmp_greater_equal(now, deadline_ms); +} + +void CoreScan::finish(Status status, bool truncated) noexcept +{ + auto self = std::move(owner); + detail::seal_backend_budget(prepared.backend); + prepared.io->seal(); + prepared.credit->seal(); + try { + complete(std::move(status), std::move(packed), truncated); + } + catch (...) { + // Completion is terminal; a throwing consumer cannot be retried. + return; + } +} + +void CoreScan::step(void *context) noexcept +{ + auto *self = static_cast<CoreScan *>(context); + try { + if (self->expired()) { + self->finish(Status::Ok(), true); + return; + } + auto operation = detail::reserved_construct<Operation>(self->prepared.credit, [&](auto *storage) { + ::new (static_cast<void *>(storage)) Operation(stdexec::connect(self->read->next_sender(), Receiver{self})); + }); + self->operation = operation; + stdexec::start(*operation); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted("scan continuation allocation failed"), false); + } + catch (...) { + self->finish(Status::internal_error("scan continuation failed"), false); + } +} + +void CoreScan::window() noexcept +try { + const auto &result = read->result(); + if (!result.status.ok()) { + finish(result.status, false); + return; + } + for (const auto &entry : result.entries) { + const Slice key(entry.key); + const Slice value = entry.cell.value; + if (key.size() > UINT32_MAX || value.size() > UINT32_MAX || value.size() > value_limit) { + finish(Status::resource_exhausted("scan entry exceeds the configured value limit"), false); + return; + } + const auto size = key.size() + value.size(); + const bool exceeds = size > byte_limit - std::min(bytes, byte_limit); + if (count >= limit || (exceeds && count != 0) || expired()) { + finish(Status::Ok(), true); + return; + } + if (exceeds && strict) { + finish(Status::Ok(), true); + return; + } + detail::reserve_scan_output(packed, size, normal_capacity); + packed.pack_u32(static_cast<uint32_t>(key.size())); + packed.append(key); + packed.pack_u64(entry.cell.slot); + packed.push_back(entry.cell.tombstone() ? 1 : 0); + packed.pack_u32(static_cast<uint32_t>(value.size())); + packed.append(value); + record_work(prepared.credit, TreeWorkCounter::MaterializeCopyBytes, size + 17, TreeWorkCause::Read); + ++count; + bytes += size; + } + if (result.done) { + finish(Status::Ok(), false); + } + else if (result.byte_limit_reached || count >= limit || bytes >= byte_limit) { + finish(Status::Ok(), true); + } + else { + progress->request(owner); + } +} +catch (const std::bad_alloc &) { + finish(Status::resource_exhausted("scan output allocation failed"), false); +} +catch (...) { + finish(Status::internal_error("scan result packing failed"), false); +} +} // namespace crowdb::tree::core_read_detail + +namespace crowdb::tree +{ +void TreeCore::scan_async(CoreScanOptions options, std::function<void(Status, ScanPackedBuf, bool)> complete) +{ + scan_impl(options, std::move(complete), true); +} + +void TreeCore::scan_impl(CoreScanOptions options, std::function<void(Status, ScanPackedBuf, bool)> complete, + bool dispatch) +{ + if (!complete) { + return; + } + std::shared_ptr<core_read_detail::CoreScan> operation; + Status status; + try { + State::Ingress ingress(*state_); + status = ingress.owner == nullptr ? Status::unavailable("Tree closed") + : core_read_detail::prepare_scan(*this, options, &operation); + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted("scan admission allocation failed"); + } + if (!status.ok()) { + complete(std::move(status), {}, false); + return; + } + operation->complete = std::move(complete); + operation->owner = operation; + if (dispatch) { + operation->progress->request(operation); + } + else { + core_read_detail::CoreScan::step(operation.get()); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_scan_budget.cpp b/lib/crowdb-tree/src/btree/tree_core_scan_budget.cpp new file mode 100644 index 000000000..9f2dfd164 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_scan_budget.cpp @@ -0,0 +1,159 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_account.h" +#include "btree/allocation_bound.h" +#include "btree/tree_core_scan_internal.h" + +#include <algorithm> + +namespace crowdb::tree::core_read_detail +{ +namespace +{ +std::optional<size_t> result_count_bound(const PublishedTreeState &view, const PageLoadLimits &loads, + const TreeExecutionOptions &options, bool include_tombstones) +{ + using detail::AllocationBound; + const auto shape = view.mapping.copy_bounds(); + std::optional<size_t> count; + if (!include_tombstones && view.stats.kv_unknown_pages == 0) { + count = view.stats.kv_known_count; + } + else { + const auto bases = AllocationBound().add_product(shape.leaf_pages, shape.max_leaf_bases).value(); + if (!bases) { + return {}; + } + count = AllocationBound(view.stats.overlay_entries).add_product(*bases, loads.max_leaf_entries).value(); + } + if (!count) { + return {}; + } + AllocationBound total(*count); + // Captured sources may still accept later writes. Their hard version quota, + // not the current count, bounds those future visible keys. Core admission + // keeps these hard quotas independent of the configurable soft thresholds; + // inherited split tables are closed and drained, so use their exact count. + const auto future = AllocationBound(options.source_max_entries).add(options.source_max_gap_entries).value(); + if (!future) { + return {}; + } + for (const auto &table : view.sources->tables) { + total.add(table->closed() && table->writers() == 0 ? table->count() : *future); + } + return total.value(); +} + +PageKeyRange selection(CoreScanOptions options, const KeyRange &domain, + const std::shared_ptr<detail::AllocationReservation> &credit) +{ + PageKeyRange range; + if (!options.prefix.empty()) { + range.lower = OwnedKey(options.prefix, credit); + KeyArenaStorage upper(options.prefix.data(), options.prefix.size(), detail::ReservationAllocator<char>(credit)); + while (!upper.empty() && static_cast<uint8_t>(upper.back()) == 255) { + upper.pop_back(); + } + if (!upper.empty()) { + upper.back() = static_cast<char>(static_cast<uint8_t>(upper.back()) + 1); + range.upper = OwnedKey(Slice(upper.data(), upper.size()), credit); + } + } + if (!options.begin.empty() && (!range.lower || options.begin > Slice(*range.lower))) { + range.lower = OwnedKey(options.begin, credit); + } + if (!options.end.empty() && (!range.upper || options.end < Slice(*range.upper))) { + range.upper = OwnedKey(options.end, credit); + } + if (domain.start() && (!range.lower || Slice(*domain.start()) > Slice(*range.lower))) { + range.lower = OwnedKey(Slice(*domain.start()), credit); + } + if (domain.end() && (!range.upper || Slice(*domain.end()) < Slice(*range.upper))) { + range.upper = OwnedKey(Slice(*domain.end()), credit); + } + return range; +} +} // namespace + +Status prepare_scan(TreeCore &core, CoreScanOptions options, std::shared_ptr<CoreScan> *out) +{ + using namespace detail; + auto limits = core.read_limits(); + const auto view = core.capture(); + for (const auto key : {options.prefix, options.start, options.end, options.begin}) { + if (key.size() > limits.max_key_bytes) { + return Status::invalid_argument("scan boundary exceeds the key limit"); + } + } + const auto maximum_count = + result_count_bound(*view, core.load_limits(), core.config().execution, options.include_tombstones); + if (!maximum_count) { + return Status::resource_exhausted("scan result count bound overflow"); + } + const size_t count = options.limit == 0 ? *maximum_count : std::min(options.limit, *maximum_count); + const size_t bytes = options.byte_budget == 0 ? SIZE_MAX : options.byte_budget; + const auto window_bytes = std::min(bytes, limits.max_result_bytes); + const auto normal = AllocationBound(window_bytes).add_product(count, 17).value(); + if (!normal) { + return Status::resource_exhausted("scan result bound overflow"); + } + const auto control = + AllocationBound(shared_allocation_bound<CoreScan>()) + .add_product(core.binding()->cpu()->worker_count() + 1, shared_allocation_bound<CoreScan::Operation>()) + .add(FlushRuntime::progress_allocation_bound()) + .add_product(6, OwnedKey::allocation_overhead() + limits.max_key_bytes + 1) + .add_product(2, OwnedKey::allocation_overhead() + 1) + .add(core.config().key_range.start() ? core.config().key_range.start()->size() : 0) + .add(core.config().key_range.end() ? core.config().key_range.end()->size() : 0) + .value(); + if (!control) { + return Status::resource_exhausted("scan control bound overflow"); + } + // Keys are unique: all except the one possible empty key cost at least one + // byte. A tiny byte window cannot contain more than bytes + 1 results; + // the oversized-first exception contributes only one result. + const auto byte_count = AllocationBound(window_bytes).add(1).value().value_or(SIZE_MAX); + limits.max_entries = std::max<size_t>(1, std::min({count, limits.max_entries, byte_count})); + limits.max_result_bytes = window_bytes; + ReadPreparation ready; + auto status = prepare(core, view, limits, true, *control, 0, &ready); + if (!status.ok()) { + return status; + } + auto scan = reserved_shared<CoreScan>(ready.credit); + scan->prepared = std::move(ready); + auto &prepared = scan->prepared; + PublishedScanOptions scan_options{.start_inclusive = options.start_inclusive, + .keys_only = options.keys_only, + .include_tombstones = options.include_tombstones, + .allow_oversized_first = !options.strict_byte_budget, + .byte_budget = bytes}; + if (options.has_start_bound || !options.start.empty()) { + scan_options.start = OwnedKey(options.start, prepared.credit); + } + status = PublishedScan::create(prepared.view, prepared.loader, + selection(options, core.config().key_range, prepared.credit), options.reverse, + prepared.limits, &scan->read, std::move(scan_options)); + if (!status.ok()) { + return status; + } + scan->limit = count; + scan->byte_limit = bytes; + scan->normal_capacity = *normal; + scan->value_limit = core.load_limits().max_value_bytes; + scan->deadline_ms = options.deadline_ms; + scan->strict = options.strict_byte_budget; + std::shared_ptr<AllocationAccount> output; + status = AllocationAccount::create(prepared.binding->account(), BudgetClass::kRouting, &output); + if (!status.ok()) { + return status; + } + scan->packed = ScanPackedBuf(std::move(output)); + status = prepared.binding->cpu()->reserve_progress(prepared.binding->lane(), &CoreScan::step, scan.get(), + prepared.credit, &scan->progress); + if (status.ok()) { + *out = std::move(scan); + } + return status; +} +} // namespace crowdb::tree::core_read_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_scan_internal.h b/lib/crowdb-tree/src/btree/tree_core_scan_internal.h new file mode 100644 index 000000000..2f0983db4 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_scan_internal.h @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/tree_core_read_internal.h" + +namespace crowdb::tree::core_read_detail +{ +struct CoreScan : std::enable_shared_from_this<CoreScan> +{ + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + CoreScan *owner; + + void set_value() const noexcept + { + owner->window(); + } + + void set_error(Status status) const noexcept + { + owner->finish(std::move(status), false); + } + + void set_stopped() const noexcept + { + owner->finish(Status::unavailable("scan stopped"), false); + } + }; + + using Operation = detail::CallbackSender::Operation<Receiver>; + ReadPreparation prepared; + std::shared_ptr<PublishedScan> read; + std::shared_ptr<Operation> operation; + std::shared_ptr<detail::FlushRuntime::Progress> progress; + std::shared_ptr<CoreScan> owner; + std::function<void(Status, ScanPackedBuf, bool)> complete; + ScanPackedBuf packed; + size_t count = 0; + size_t bytes = 0; + size_t limit = 0; + size_t byte_limit = 0; + size_t normal_capacity = 0; + size_t value_limit = 0; + uint64_t deadline_ms = 0; + bool strict = false; + + void finish(Status status, bool truncated) noexcept; + void window() noexcept; + static void step(void *context) noexcept; + [[nodiscard]] bool expired() const noexcept; +}; + +Status prepare_scan(TreeCore &core, CoreScanOptions options, std::shared_ptr<CoreScan> *out); +} // namespace crowdb::tree::core_read_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_scan_sync.cpp b/lib/crowdb-tree/src/btree/tree_core_scan_sync.cpp new file mode 100644 index 000000000..bebac5870 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_scan_sync.cpp @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "host_completion.h" + +namespace crowdb::tree +{ +namespace +{ +struct PackedReader +{ + const uint8_t *position; + size_t left; + + bool integer(size_t bytes, uint64_t *value) + { + if (left < bytes) { + return false; + } + *value = 0; + for (size_t i = 0; i < bytes; ++i) { + *value |= uint64_t{position[i]} << (8 * i); + } + position += bytes; + left -= bytes; + return true; + } + + bool string(Slice *value) + { + uint64_t bytes = 0; + if (!integer(4, &bytes) || left < bytes) { + return false; + } + *value = Slice(position, bytes); + position += bytes; + left -= bytes; + return true; + } +}; + +Status unpack(const ScanPackedBuf &packed, std::vector<scan_entry> *out, size_t *count) +{ + PackedReader reader{.position = packed.data(), .left = packed.size()}; + size_t entries = 0; + while (reader.left != 0) { + Slice key; + Slice value; + uint64_t slot = 0; + uint64_t tombstone = 0; + if (!reader.string(&key) || !reader.integer(8, &slot) || !reader.integer(1, &tombstone) || + !reader.string(&value)) { + return Status::corruption("invalid internal scan record"); + } + if (out != nullptr) { + // The copying convenience API returns application-owned strings. + out->push_back( + {.key = key.to_string(), .slot = slot, .value = value.to_string(), .tombstone = tombstone != 0}); + } + ++entries; + } + if (count != nullptr) { + *count = entries; + } + return Status::Ok(); +} +} // namespace + +Status TreeCore::scan(CoreScanOptions options, std::vector<scan_entry> *out, bool *truncated, ScanPackedBuf *packed, + size_t *count) +{ + if (truncated == nullptr || (out == nullptr && packed == nullptr)) { + return Status::invalid_argument("scan needs output and truncation flag"); + } + if (out != nullptr) { + out->clear(); + } + if (count != nullptr) { + *count = 0; + } + *truncated = false; + detail::HostCompletion done; + Status status; + ScanPackedBuf result; + scan_impl( + options, + [&](Status outcome, ScanPackedBuf value, bool more) { + status = std::move(outcome); + result = std::move(value); + *truncated = more; + done.signal(); + }, + false); + done.wait(); + if (!status.ok()) { + return status; + } + try { + status = unpack(result, packed == nullptr ? out : nullptr, count); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted("scan copy allocation failed"); + } + if (status.ok() && packed != nullptr) { + *packed = std::move(result); + } + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_snapshot.cpp b/lib/crowdb-tree/src/btree/tree_core_snapshot.cpp new file mode 100644 index 000000000..9f3da6105 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_snapshot.cpp @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_internal.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +Status TreeCore::capture_snapshot(std::shared_ptr<PublishedSnapshotContext> *out) +{ + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("Tree snapshot admission closed"); + } + return PublishedSnapshotContext::capture_admitted(shared_from_this(), out); +} + +Status TreeCore::capture_snapshot(std::shared_ptr<const PublishedTreeState> view, + std::shared_ptr<PublishedSnapshotContext> *out) +{ + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("Tree snapshot admission closed"); + } + if (!view || view->generation != capture()->generation) { + return Status::unavailable("fixed snapshot belongs to an old Tree generation"); + } + return PublishedSnapshotContext::capture_admitted(shared_from_this(), out, std::move(view)); +} + +Status TreeCore::acquire_snapshot_window(const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<TreeOperationLifetime::Ticket> *ticket, + AsyncPageStore **backend) +{ + if (backend == nullptr) { + return Status::invalid_argument("snapshot read window requires backend output"); + } + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("Tree snapshot admission closed"); + } + auto status = state_->lifetime->acquire(metadata, ticket); + if (status.ok()) { + *backend = state_->backend.get(); + } + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_snapshot_ids.cpp b/lib/crowdb-tree/src/btree/tree_core_snapshot_ids.cpp new file mode 100644 index 000000000..334a0d6a5 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_snapshot_ids.cpp @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_internal.h" +#include "snapshot/operation_wait.h" + +namespace crowdb::tree +{ +Status TreeCore::reserve_snapshot_ids(const PublishedTreeState &expected, uint64_t count, + const std::shared_ptr<detail::AllocationReservation> &metadata) +try { + if (!metadata || metadata->account() != binding()->account() || metadata->category() == detail::BudgetClass::kIo || + expected.next_page_id > MappingView::kPageLimit || count > MappingView::kPageLimit - expected.next_page_id) { + return Status::invalid_argument("snapshot PID lease requires bounded same-domain metadata"); + } + detail::SnapshotWaitCompletion completion; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto request = detail::reserved_shared<PublicationRequest>(metadata, metadata); + auto &transaction = request->transaction; + transaction.record = + detail::reserved_shared<InstallRecord>(metadata, InstallToken{.generation = expected.generation}); + transaction.expected_commit_seq = expected.commit_seq; + transaction.expected_coverage = expected.coverage; + transaction.expected_next_page_id = expected.next_page_id; + transaction.next_page_id = expected.next_page_id + count; + request->on_done = [&completion](PublicationResult result) { + completion.status = std::move(result.status); + completion.done.signal(); + }; + { + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("Tree snapshot PID admission closed"); + } + auto status = state_->lifetime->acquire(metadata, &ticket); + if (!status.ok()) { + return status; + } + if (!state_->publisher->try_submit(request)) { + return Status::unavailable("snapshot PID publication queue is full"); + } + } + completion.done.wait(); + return completion.status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot PID lease allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_source_stats.cpp b/lib/crowdb-tree/src/btree/tree_core_source_stats.cpp new file mode 100644 index 000000000..40a0a152e --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_source_stats.cpp @@ -0,0 +1,28 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +SourceMemoryStats TreeCore::source_stats() const +{ + const auto view = capture(); + return view && view->sources ? view->sources->source_stats() : SourceMemoryStats{}; +} + +std::array<ResourceUsage, 7> TreeCore::admission_resource_usage() const noexcept +{ + std::array<ResourceUsage, 7> result{}; + result[0] = state_->lifetime->resource_usage(); + result[1] = state_->apply_lifetime->resource_usage(); + result[2] = state_->replay_lifetime->resource_usage(); + const auto sources = state_->sources->catalog(); + if (sources && sources->active_budget) { + const auto usage = sources->active_budget->resource_usage(); + std::copy(usage.begin(), usage.end(), result.begin() + 3); + } + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_split_baseline.cpp b/lib/crowdb-tree/src/btree/tree_core_split_baseline.cpp new file mode 100644 index 000000000..5f8fd15e8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_split_baseline.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/runtime_range_rebuild.h" +#include "btree/tree_core_internal.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +namespace +{ +Status inherit_sources(const SplitSourceSession &session, uint64_t identity, uint64_t end, + const std::shared_ptr<TreeRuntimeBinding> &binding, std::shared_ptr<const PublishedSources> *out) +{ + const auto floor = session.baseline->view()->coverage; + if (floor == end) { + return Status::Ok(); + } + const auto bytes = detail::AllocationBound(PublishedSources::allocation_bound()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add_product(session.table_count, sizeof(std::shared_ptr<MemTable>)) + .value(); + if (!bytes) { + return Status::resource_exhausted("split baseline source bound overflow"); + } + auto paid = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kHandoff, *bytes); + if (!paid) { + return Status::resource_exhausted("split baseline source admission unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*paid)); + auto sources = detail::reserved_shared<PublishedSources>(metadata, metadata); + sources->floor = floor; + sources->tables.reserve(session.table_count); + sources->inheritance = PublishedSources::Inheritance{ + .owner_identity = identity, .session = session.session, .frontier = end, .local_count = 0}; + sources->inheritance->memory.known = true; + for (size_t i = 0; i < session.table_count; ++i) { + sources->tables.push_back(session.tables[i]); + sources->inheritance->memory += session.tables[i]->source_stats(); + } + metadata->seal(); + *out = std::move(sources); + return Status::Ok(); +} +} // namespace + +Status TreeCore::install_split_baseline(const SplitSourceSession &session, uint64_t identity, uint64_t end) +{ + SplitGeneration split; + { + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("split destination ingress closed"); + } + split.expected = capture(); + const auto &view = *split.expected; + const auto root = view.mapping.get(view.root_page_id); + if (view.coverage != 0 || max_seen_slot() != 0 || view.stats.page_count() != 1 || !root || !root->leaf || + !root->leaf->bases.empty() || !root->leaf->overlays.empty() || view.sources->inheritance || + view.pending_handoff || view.flush_admission_owner != 0 || view.checkpoint_inflight) { + return Status::invalid_argument("split L1 baseline requires an empty unwritten destination"); + } + const auto floor = session.baseline->view()->coverage; + if (end < floor || !session.covers(floor, end)) { + return Status::invalid_argument("split sources do not cover the fixed L1 baseline"); + } + if (end != floor && session.table_count >= read_limits().max_memtables) { + return Status::resource_exhausted("split source catalog exceeds the destination read window"); + } + } + RuntimeRangeImport staging; + auto status = copy_snapshot_range(session.baseline, *this, config().key_range, staging); + if (!status.ok()) { + return status; + } + std::shared_ptr<const PublishedTreeState> candidate; + status = prepare_readable(std::move(staging.catalog), staging.credit, &candidate); + if (!status.ok()) { + return status; + } + status = inherit_sources(session, identity, end, binding(), &split.inherited); + if (!status.ok()) { + return status; + } + // No destination mutation has occurred. Generation replacement validates + // the original root again under its ingress fence before one atomic install + // of the independent L1 and the complete inherited source prefix. + return install_generation(std::move(candidate), &split); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_split_capture.cpp b/lib/crowdb-tree/src/btree/tree_core_split_capture.cpp new file mode 100644 index 000000000..bdafa4974 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_split_capture.cpp @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_internal.h" +#include "btree/tree_core_split_wait.h" +#include "snapshot/published_snapshot.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +#ifdef CROWDB_TREE_TEST_UTIL +thread_local std::function<void()> between_captures_for_tests; +#endif + +// The caller retains source ingress throughout this host-only wait. A busy +// prepare gate or an in-flight writer pause is waitable; quota failure is not. +Status prepare_split_capture(TreeCore &core, FlushController &controller, FlushPreparationOptions preparation, + ApplyProgress &progress, std::shared_ptr<FlushOperation> *out) +{ + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto status = core.operation_lifetime().acquire(core.binding()->lifecycle_budget(), &ticket); + if (!status.ok()) { + return status; + } + preparation.store = ticket->backend(core.async_backend()); + for (;;) { + const auto observed = progress.sequence(); + bool blocked = false; + status = controller.prepare(preparation, out, &blocked); + if (status.ok() || !blocked) { + return status; + } + status = progress.wait(observed); + if (!status.ok()) { + return status; + } + } +} + +Status advance_split_capture(const std::shared_ptr<FlushOperation> &operation) +{ + auto status = operation->wait(); +#ifdef CROWDB_TREE_TEST_UTIL + if (status.ok() && between_captures_for_tests) { + auto callback = std::move(between_captures_for_tests); + between_captures_for_tests = {}; + callback(); + } +#endif + return status; +} +} // namespace + +#ifdef CROWDB_TREE_TEST_UTIL +void set_between_split_captures_for_tests(std::function<void()> callback) +{ + between_captures_for_tests = std::move(callback); +} +#endif + +Status SplitSourceSession::retain(const FrozenSourceView &frozen) +{ + if (!frozen.catalog) { + return Status::Ok(); + } + if (frozen.catalog->generation != generation || frozen.frontier < frozen.catalog->floor) { + return Status::unavailable("split source generation changed"); + } + size_t added = 0; + for (const auto &table : frozen.catalog->tables) { + if (std::find(tables.begin(), tables.begin() + static_cast<std::ptrdiff_t>(table_count), table) == + tables.begin() + static_cast<std::ptrdiff_t>(table_count)) { + ++added; + } + } + const bool same_interval = interval_count != 0 && intervals[interval_count - 1].floor == frozen.catalog->floor && + intervals[interval_count - 1].frontier == frozen.frontier; + if (added > kMaxSources - table_count || (!same_interval && interval_count == intervals.size())) { + return Status::resource_exhausted("split source retention window is full"); + } + for (const auto &table : frozen.catalog->tables) { + if (std::find(tables.begin(), tables.begin() + static_cast<std::ptrdiff_t>(table_count), table) == + tables.begin() + static_cast<std::ptrdiff_t>(table_count)) { + tables[table_count++] = table; + } + } + if (!same_interval) { + intervals[interval_count++] = {.floor = frozen.catalog->floor, .frontier = frozen.frontier}; + } + if (frozen.catalog->active_budget) { + const auto &source_limits = frozen.catalog->active_budget->options(); + limits.max_entries = std::max(limits.max_entries, source_limits.max_entries); + limits.max_key_bytes = std::max(limits.max_key_bytes, source_limits.max_key_bytes); + limits.max_gap_entries = std::max(limits.max_gap_entries, source_limits.max_gap_entries); + limits.max_gap_key_bytes = std::max(limits.max_gap_key_bytes, source_limits.max_gap_key_bytes); + limits.max_key_size = std::max(limits.max_key_size, source_limits.max_key_size); + } + return Status::Ok(); +} + +bool SplitSourceSession::covers(uint64_t floor, uint64_t end) const noexcept +{ + auto covered = floor; + for (size_t i = 0; i < interval_count && covered < end; ++i) { + if (intervals[i].floor > covered) { + return false; + } + covered = std::max(covered, intervals[i].frontier); + } + return covered >= end; +} + +Status TreeCore::State::capture_split_sources(TreeCore &core, uint64_t target) +{ + if (detail::FlushRuntime::on_worker_thread()) { + return Status::unavailable("a runtime worker cannot wait for split capture"); + } + if (!split_sources) { + const auto bound = detail::shared_allocation_bound<SplitSourceSession>() + + detail::shared_allocation_bound<detail::AllocationReservation>(); + auto reserved = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kRouting, bound); + if (!reserved || split_sequence == UINT64_MAX) { + return Status::resource_exhausted("split retention admission unavailable"); + } + auto credit = detail::AllocationReservation::create(std::move(*reserved)); + split_sources = detail::reserved_shared<SplitSourceSession>(credit); + split_sources->generation = publisher->capture()->generation; + split_sources->session = ++split_sequence; + credit->seal(); + } + if (split_sources->generation != publisher->capture()->generation) { + return Status::unavailable("split source belongs to a replaced generation"); + } + // An existing paused capture can precede target. Finish it once, then + // capture the current successor. No source admission or epoch is held + // across the host wait; both senders run on the shared CPU runtime. + for (size_t pass = 0; pass < 2; ++pass) { + std::shared_ptr<FlushOperation> operation; + auto status = prepare_split_capture(core, *flush, preparation, *apply_progress, &operation); + if (!status.ok()) { + return status; + } + FlushResult report; + status = operation->wait_capture(&report); + if (!status.ok()) { + return status; + } + if (!report.handoff_status.ok()) { + return report.handoff_status; + } + // An unpublished first capture below the fixed target need not become + // the session baseline. Other normal handoffs may consume intervening + // sources before the successor is captured; choose its paired L1 once. + if (!split_sources->baseline && report.frontier < target) { + status = advance_split_capture(operation); + if (!status.ok()) { + return status; + } + continue; + } + const auto frozen = operation->frozen_sources(); + if (!frozen.base || (frozen.catalog && frozen.base->coverage != frozen.catalog->floor)) { + return Status::unavailable("split capture has no complete paired L1 base"); + } + if (split_sources->baseline && frozen.base->coverage > split_sources->frontier) { + // An installed overlay already fixes the older baseline. Normal + // handoffs can consume newer intervals that this session never + // retained. Return its real contiguous prefix; the caller replays + // beyond that boundary instead of inheriting unavailable history. + return Status::Ok(); + } + auto baseline = split_sources->baseline; + if (!baseline) { + status = core.capture_snapshot(frozen.base, &baseline); + if (!status.ok()) { + return status; + } + } + status = split_sources->retain(frozen); + if (!status.ok()) { + return status; + } + split_sources->baseline = std::move(baseline); + split_sources->frontier = std::max(split_sources->frontier, report.frontier); + if (report.frontier >= target) { + return Status::Ok(); + } + status = advance_split_capture(operation); + if (!status.ok()) { + return status; + } + } + return Status::unavailable("split source did not reach its fixed capture target"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_split_handoff.cpp b/lib/crowdb-tree/src/btree/tree_core_split_handoff.cpp new file mode 100644 index 000000000..c5ec17d64 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_split_handoff.cpp @@ -0,0 +1,191 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_internal.h" +#include "btree/tree_core_split_wait.h" +#include "snapshot/operation_wait.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +SourceWriteOptions combined_limits(SourceWriteOptions local, const SourceWriteOptions &source) +{ + local.max_entries = std::max(local.max_entries, source.max_entries); + local.max_key_bytes = std::max(local.max_key_bytes, source.max_key_bytes); + local.max_gap_entries = std::max(local.max_gap_entries, source.max_gap_entries); + local.max_gap_key_bytes = std::max(local.max_gap_key_bytes, source.max_gap_key_bytes); + local.max_key_size = std::max(local.max_key_size, source.max_key_size); + return local; +} +} // namespace + +Status TreeCore::State::prepare_split_handoff(const SplitSourceSession &session, uint64_t identity, uint64_t end) +{ + if (split_handoff) { + return split_handoff->source_identity == identity && split_handoff->session == session.session && + split_handoff->frontier == end + ? Status::Ok() + : Status::invalid_argument("another split handoff requires completion"); + } + const auto view = publisher->capture(); + if (!view->sources->inheritance || !session.covers(view->coverage, end)) { + return Status::unavailable("split handoff requires its retained source prefix"); + } + const auto locals = sources->catalog(); + const auto &inherited = *view->sources->inheritance; + if (inherited.owner_identity != identity || inherited.session != session.session || inherited.frontier != end) { + return Status::invalid_argument("split handoff does not match installed inheritance"); + } + const auto scratch = detail::AllocationBound(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add(detail::shared_allocation_bound<SourceWriteBudget>()) + .add_product(view->sources->tables.size(), sizeof(std::shared_ptr<MemTable>)) + .value(); + if (!scratch) { + return Status::resource_exhausted("split reserve catalog bound overflow"); + } + auto reserved = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kRouting, *scratch); + if (!reserved) { + return Status::unavailable("split reserve catalog admission unavailable"); + } + auto control = detail::AllocationReservation::create(std::move(*reserved)); + SourceCatalog catalog(control); + catalog.tables.assign(view->sources->tables.begin(), view->sources->tables.end()); + catalog.floor = view->coverage; + const auto limits = combined_limits(locals->active_budget->options(), session.limits); + catalog.active_budget = detail::reserved_shared<SourceWriteBudget>(control, limits); + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto status = lifetime->acquire(binding->lifecycle_budget(), &ticket); + if (!status.ok()) { + return status; + } + auto prepare = preparation; + prepare.store = ticket->backend(backend.get()); + const auto controls = detail::AllocationBound(detail::shared_allocation_bound<SplitHandoff>()) + .add(SourceAdmission::coverage_allocation_bound(locals->tables.size())) + .add_product(2, OwnedKey::allocation_overhead()) + .add_product(2, limits.max_key_size + 1) + .value(); + if (!controls) { + return Status::resource_exhausted("split handoff control bound overflow"); + } + FlushBatchOptions options; + options.parallelism = + std::max<size_t>(1, std::min(binding->cpu()->worker_count(), binding->options().limits.io_operations)); + options.run_limits.max_entries = 128; + options.run_limits.max_key_bytes = std::max<size_t>(256 * 1024, limits.max_key_size); + options.run_limits.key_block_bytes = 1024; + FlushBatch::Maintenance maintenance; + detail::FlushReservations reservations; + status = detail::prepare_flush_reservations(*publisher, *view, catalog, prepare, *controls, binding->lane(), + &options, &maintenance, &reservations); + if (!status.ok()) { + return status; + } + auto handoff = detail::reserved_shared<SplitHandoff>(reservations.handoff); + handoff->reservations = std::move(reservations); + handoff->source_identity = identity; + handoff->session = session.session; + handoff->frontier = end; + const auto &metadata = handoff->reservations.handoff; + status = sources->prepare_coverage(end, metadata, &handoff->coverage); + if (!status.ok()) { + return status; + } + if (config.key_range.start()) { + options.selection.lower.emplace(Slice(*config.key_range.start()), metadata); + } + if (config.key_range.end()) { + options.selection.upper.emplace(Slice(*config.key_range.end()), metadata); + } + auto inputs = detail::reserved_shared<PublishedSources>(metadata, metadata); + inputs->floor = view->coverage; + inputs->tables.assign(view->sources->tables.begin() + static_cast<std::ptrdiff_t>(inherited.local_count), + view->sources->tables.end()); + handoff->maintenance = std::move(maintenance); + auto acknowledge = [source = sources.get(), weak = std::weak_ptr<SplitHandoff>(handoff)](auto published, + auto done) { + auto current = weak.lock(); + if (!current) { + done(Status::unavailable("split handoff owner released"), 0); + return; + } + auto result = source->install_coverage(current->coverage, *published); + if (!result.ok()) { + done(std::move(result), published->stats.dirty_pages); + return; + } + current->maintenance(std::move(published), std::move(done)); + }; + status = FlushBatch::prepare(*publisher, *binding->cpu(), binding->lane(), session.session, end, std::move(options), + std::move(acknowledge), &handoff->operation, std::move(inputs), metadata); + if (!status.ok()) { + return status; + } + split_handoff = std::move(handoff); + return Status::Ok(); +} + +Status TreeCore::publish_split_view(uint64_t generation, uint64_t journal_frontier, TreeCore &destination, + const KeyRange &range) +try { + if (&destination == this || config().runtime != destination.config().runtime || !range.validate().ok() || + range.start() != destination.config().key_range.start() || + range.end() != destination.config().key_range.end()) { + return Status::invalid_argument("split publication must cover the destination's complete key range"); + } + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("split source ingress closed"); + } + detail::SplitGate source_gate(state_->split_busy); + detail::SplitGate destination_gate(destination.state_->split_busy); + if (!source_gate.held() || !destination_gate.held()) { + return Status::unavailable("split administration is busy"); + } + const auto session = state_->split_sources; + if (!session || !session->active || generation == 0 || generation != session->session || + journal_frontier > session->frontier) { + return Status::invalid_argument("split source generation or frontier is not active"); + } + // Keep the source session fixed across the destination's short ingress fence. + auto status = destination.install_split_overlay_guarded(*this, journal_frontier); + if (!status.ok()) { + return status; + } + State::Ingress destination_ingress(*destination.state_); + if (destination_ingress.owner == nullptr) { + return Status::unavailable("split destination ingress closed"); + } + if (!destination.capture()->sources->inheritance && destination.capture()->coverage == journal_frontier && + !destination.state_->split_handoff) { + return Status::Ok(); + } + status = destination.state_->prepare_split_handoff(*session, state_->sources->identity(), journal_frontier); + if (!status.ok()) { + return status; + } + const auto handoff = destination.state_->split_handoff; + if (!handoff->operation) { + return Status::Ok(); + } + status = detail::wait_snapshot_operation(handoff->operation); + if (!status.ok()) { + return status; + } + const auto &report = handoff->operation->result(); + status = report.handoff_status.ok() ? report.maintenance_status : report.handoff_status; + if (status.ok()) { + handoff->operation.reset(); + handoff->coverage.reset(); + handoff->maintenance = {}; + handoff->reservations.seal(); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("split handoff allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_split_internal.h b/lib/crowdb-tree/src/btree/tree_core_split_internal.h new file mode 100644 index 000000000..a7f1b452a --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_split_internal.h @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/flush_operation.h" + +#include <array> +#include <functional> + +namespace crowdb::tree +{ +class PublishedSnapshotContext; + +#ifdef CROWDB_TREE_TEST_UTIL +void set_between_split_captures_for_tests(std::function<void()> callback); +#endif + +struct SplitGeneration +{ + std::shared_ptr<const PublishedTreeState> expected; + std::shared_ptr<const PublishedSources> inherited; +}; + +struct SplitSourceSession +{ + static constexpr size_t kMaxSources = 64; + + struct Interval + { + uint64_t floor = 0; + uint64_t frontier = 0; + }; + + std::array<std::shared_ptr<MemTable>, kMaxSources> tables; + std::array<Interval, kMaxSources> intervals; + size_t table_count = 0; + size_t interval_count = 0; + SourceWriteOptions limits; + std::shared_ptr<PublishedSnapshotContext> baseline; + uint64_t generation = 0; + uint64_t session = 0; + uint64_t frontier = 0; + bool active = false; + + Status retain(const FrozenSourceView &frozen); + [[nodiscard]] bool covers(uint64_t floor, uint64_t end) const noexcept; +}; + +struct SplitHandoff +{ + detail::FlushReservations reservations; + std::shared_ptr<FlushBatch> operation; + std::shared_ptr<SourceAdmission::Coverage> coverage; + FlushBatch::Maintenance maintenance; + uint64_t source_identity = 0; + uint64_t session = 0; + uint64_t frontier = 0; + + ~SplitHandoff() + { + reservations.seal(); + } +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_split_overlay.cpp b/lib/crowdb-tree/src/btree/tree_core_split_overlay.cpp new file mode 100644 index 000000000..597cfa45e --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_split_overlay.cpp @@ -0,0 +1,188 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_internal.h" +#include "btree/tree_core_split_wait.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +Status TreeCore::State::install_split_sources(const SplitSourceSession &session, uint64_t identity, uint64_t end) +{ + const auto view = publisher->capture(); + if (view->sources->inheritance) { + const auto &inherited = *view->sources->inheritance; + return inherited.owner_identity == identity && inherited.session == session.session && inherited.frontier == end + ? Status::Ok() + : Status::invalid_argument("another split prefix is already installed"); + } + if (end < view->coverage || !session.covers(view->coverage, end)) { + return Status::invalid_argument("retained split sources do not cover the destination prefix"); + } + if (end == view->coverage) { + return Status::Ok(); + } + if (view->pending_handoff || view->flush_admission_owner != 0 || view->checkpoint_inflight) { + return Status::unavailable("split installation waits for destination maintenance"); + } + // The destination's sequence namespace starts at the inherited prefix. + // Its own writes may begin once this installation returns. + if (frontier->max_seen() > view->coverage) { + return Status::invalid_argument("split overlay requires an unwritten destination prefix"); + } + const auto count = view->sources->tables.size() + session.table_count; + if (count > reads.max_memtables) { + return Status::resource_exhausted("split source catalog exceeds the destination read window"); + } + const auto bytes = detail::AllocationBound(PublicationCoordinator::empty_commit_reserve_bytes()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add(PublishedSources::allocation_bound()) + .add_product(count, sizeof(std::shared_ptr<MemTable>)) + .value(); + if (!bytes) { + return Status::resource_exhausted("split catalog bound overflow"); + } + auto reserved = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kHandoff, *bytes); + if (!reserved) { + return Status::unavailable("split catalog admission unavailable"); + } + auto credit = detail::AllocationReservation::create(std::move(*reserved)); + auto next = detail::reserved_shared<PublishedSources>(credit, credit); + next->floor = view->coverage; + next->memory = view->sources->memory; + next->tables.reserve(count); + next->tables.assign(view->sources->tables.begin(), view->sources->tables.end()); + for (size_t i = 0; i < session.table_count; ++i) { + next->tables.push_back(session.tables[i]); + } + next->inheritance = PublishedSources::Inheritance{.owner_identity = identity, + .session = session.session, + .frontier = end, + .local_count = view->sources->tables.size()}; + next->inheritance->memory.known = true; + for (size_t i = 0; i < session.table_count; ++i) { + next->inheritance->memory += session.tables[i]->source_stats(); + } + auto request = detail::reserved_shared<PublicationRequest>(credit, credit); + request->transaction.record = detail::reserved_shared<InstallRecord>( + credit, + InstallToken{.generation = view->generation, .batch = session.session, .range = UINT64_MAX - 4, .sequence = 0}); + request->transaction.expected_commit_seq = view->commit_seq; + request->transaction.expected_sources = view->sources; + request->transaction.sources = std::move(next); + detail::SplitWait completion; + request->on_done = [&completion](PublicationResult result) { completion.finish(std::move(result.status)); }; + if (!publisher->try_submit(request)) { + return Status::unavailable("split publication queue unavailable"); + } + completion.wait(); + credit->seal(); + if (completion.status.ok()) { + frontier->force_advance(end); + } + return completion.status; +} + +Status TreeCore::install_split_overlay(TreeCore &source, uint64_t journal_frontier) +try { + if (&source == this || config().runtime != source.config().runtime) { + return Status::invalid_argument("split trees must be distinct and share their host runtime"); + } + State::Ingress source_ingress(*source.state_); + if (source_ingress.owner == nullptr) { + return Status::unavailable("split source ingress closed"); + } + detail::SplitGate source_gate(source.state_->split_busy); + detail::SplitGate child_gate(state_->split_busy); + if (!source_gate.held() || !child_gate.held()) { + return Status::unavailable("split administration is busy"); + } + return install_split_overlay_guarded(source, journal_frontier); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("split overlay allocation failed"); +} + +Status TreeCore::install_split_overlay_guarded(TreeCore &source, uint64_t journal_frontier) +{ + if (journal_frontier > source.contiguous_slot()) { + return Status::invalid_argument("split prefix exceeds source applied frontier"); + } + if (!source.state_->split_sources || source.state_->split_sources->frontier < journal_frontier) { + if (source.state_->split_sources && source.state_->split_sources->active) { + return Status::invalid_argument("split prefix exceeds active captured view"); + } + auto status = source.state_->capture_split_sources(source, journal_frontier); + if (!status.ok()) { + return status; + } + } + const auto &session = *source.state_->split_sources; + if (session.frontier < journal_frontier) { + return Status::unavailable("retained split sources do not reach the requested prefix"); + } + const auto current = capture(); + if (!current->sources->inheritance && session.baseline && current->coverage < session.baseline->view()->coverage) { + return install_split_baseline(session, source.state_->sources->identity(), journal_frontier); + } + if (!state_->begin_replacement()) { + return Status::unavailable("split destination ingress closed or replacing"); + } + + struct Release + { + State &state; + + ~Release() + { + state.finish_replacement(); + } + } release{*state_}; + + state_->apply_lifetime->drain_accepted(); + state_->replay_lifetime->drain_accepted(); + const auto drained = capture(); + if (drained->sources->inheritance) { + return state_->install_split_sources(*source.state_->split_sources, source.state_->sources->identity(), + journal_frontier); + } + if (drained->flush_admission_owner != 0 || drained->pending_handoff) { + return Status::unavailable("split destination has an unfinished handoff"); + } + state_->flush->release_pending(); + state_->lifetime->drain_accepted(); + state_->binding->drain(); + if (closed()) { + return Status::unavailable("split destination closed during drain"); + } + return state_->install_split_sources(*source.state_->split_sources, source.state_->sources->identity(), + journal_frontier); +} + +Status TreeCore::clear_split_overlay(TreeCore &source) +{ + if (&source == this) { + return Status::invalid_argument("split source must differ from destination"); + } + State::Ingress ingress(*state_); + State::Ingress source_ingress(*source.state_); + if (ingress.owner == nullptr || source_ingress.owner == nullptr) { + return Status::unavailable("split ingress closed"); + } + detail::SplitGate gate(state_->split_busy); + if (!gate.held()) { + return Status::unavailable("split administration is busy"); + } + const auto view = capture(); + if (view->sources->inheritance) { + return view->sources->inheritance->owner_identity != source.state_->sources->identity() + ? Status::invalid_argument("split overlay source does not match") + : Status::unavailable("split prefix publication is incomplete"); + } + if (state_->split_handoff && state_->split_handoff->source_identity != source.state_->sources->identity()) { + return Status::invalid_argument("split overlay source does not match"); + } + state_->split_handoff.reset(); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_split_session.cpp b/lib/crowdb-tree/src/btree/tree_core_split_session.cpp new file mode 100644 index 000000000..ffb5a8fc8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_split_session.cpp @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_internal.h" +#include "btree/tree_core_split_wait.h" + +namespace crowdb::tree +{ +Status TreeCore::begin_split_view(uint64_t *generation, uint64_t *journal_frontier) +try { + if (generation == nullptr || journal_frontier == nullptr) { + return Status::invalid_argument("split view requires output parameters"); + } + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("split ingress closed"); + } + detail::SplitGate gate(state_->split_busy); + if (!gate.held()) { + return Status::unavailable("split administration is busy"); + } + if (state_->split_sources && state_->split_sources->active) { + return Status::invalid_argument("a split source view is already active"); + } + auto status = state_->capture_split_sources(*this, contiguous_slot()); + if (!status.ok()) { + return status; + } + state_->split_sources->active = true; + *generation = state_->split_sources->session; + *journal_frontier = state_->split_sources->frontier; + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("split source capture allocation failed"); +} + +Status TreeCore::release_split_view(uint64_t generation) +{ + State::Ingress ingress(*state_); + if (ingress.owner == nullptr) { + return Status::unavailable("split ingress closed"); + } + detail::SplitGate gate(state_->split_busy); + if (!gate.held()) { + return Status::unavailable("split administration is busy"); + } + if (!state_->split_sources || !state_->split_sources->active || generation == 0 || + generation != state_->split_sources->session) { + return Status::invalid_argument("split source generation is not active"); + } + state_->split_sources.reset(); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_core_split_wait.h b/lib/crowdb-tree/src/btree/tree_core_split_wait.h new file mode 100644 index 000000000..83bd6d9c8 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_split_wait.h @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/flush_operation.h" +#include "host_completion.h" + +namespace crowdb::tree::detail +{ +class SplitGate +{ + public: + explicit SplitGate(std::atomic_flag &flag) : flag_(flag), held_(!flag.test_and_set(std::memory_order_acquire)) + { + } + + ~SplitGate() + { + if (held_) { + flag_.clear(std::memory_order_release); + } + } + + [[nodiscard]] bool held() const noexcept + { + return held_; + } + + private: + std::atomic_flag &flag_; + bool held_; +}; + +struct SplitWait +{ + HostCompletion done; + Status status; + FlushResult result; + + void finish(Status value) noexcept + { + status = std::move(value); + done.signal(); + } + + void wait() const noexcept + { + done.wait(); + } +}; + +struct SplitReportReceiver +{ + using receiver_concept = stdexec::receiver_tag; + SplitWait *completion; + + void set_value(FlushResult result) const noexcept + { + completion->result = std::move(result); + completion->finish(Status::Ok()); + } + + void set_error(Status status) const noexcept + { + completion->finish(std::move(status)); + } + + void set_stopped() const noexcept + { + completion->finish(Status::unavailable("split capture stopped")); + } +}; + +inline Status wait_split_report(FlushReportSender sender, FlushResult *out) +{ + SplitWait completion; + auto connected = stdexec::connect(std::move(sender), SplitReportReceiver{&completion}); + stdexec::start(connected); + completion.wait(); + *out = std::move(completion.result); + return completion.status; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/tree_core_state_stats.cpp b/lib/crowdb-tree/src/btree/tree_core_state_stats.cpp new file mode 100644 index 000000000..396b8d878 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_core_state_stats.cpp @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/checkpoint_backend.h" +#include "backend/local/local_backend_state.h" +#include "tree_core_internal.h" + +#include <algorithm> +#include <limits> + +namespace crowdb::tree +{ +BackingStatsSampler TreeCore::backing_sampler() const noexcept +{ + if (state_->chunk) { + return state_->chunk->backing_sampler(); + } + if (state_->local) { + return {.scope_id = state_->local->ledger->limits().backend_id, .metrics = state_->resources}; + } + return {}; +} + +BackingStatsSnapshot TreeCore::backing_stats() const noexcept +{ + return backing_sampler().snapshot(); +} + +TreeStatsSnapshot TreeCore::state_stats() const +{ + auto result = state_->publisher->stats(*state_->resources); + const auto budget = state_->binding->account()->snapshot(); + // I/O credit counts operations, not bytes. + result.resources.allocated_bytes = 0; + result.resources.reserved_bytes = 0; + for (size_t index = 0; index < static_cast<size_t>(detail::BudgetClass::kIo); ++index) { + result.resources.allocated_bytes += std::min<uint64_t>( + budget.allocated[index], std::numeric_limits<uint64_t>::max() - result.resources.allocated_bytes); + result.resources.reserved_bytes += std::min<uint64_t>( + budget.reserved[index], std::numeric_limits<uint64_t>::max() - result.resources.reserved_bytes); + } + result.resources.payload_bytes = budget.physical[static_cast<size_t>(AllocationKind::kPayload)]; + result.resources.frame_bytes = budget.physical[static_cast<size_t>(AllocationKind::kFrame)]; + result.resources.metadata_bytes = budget.physical[static_cast<size_t>(AllocationKind::kMetadata)]; + result.resources.io_bytes = budget.physical[static_cast<size_t>(AllocationKind::kIo)]; + result.resources.scratch_bytes = budget.physical[static_cast<size_t>(AllocationKind::kScratch)]; + const auto backing = backing_stats(); + if (backing.scope_id != 0) { + result.resources.allocated_backing_bytes = backing.allocated_bytes; + } + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_diagnostics.cpp b/lib/crowdb-tree/src/btree/tree_diagnostics.cpp new file mode 100644 index 000000000..e1aff078e --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_diagnostics.cpp @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "crowdb-common/log.h" + +namespace crowdb::tree +{ +int Crowdbtree::height() const +{ + return static_cast<int>(core_->capture()->stats.height); +} + +size_t Crowdbtree::leaf_count() const +{ + return core_->capture()->stats.leaf_pages; +} + +EngineStats Crowdbtree::stats() const +{ + const auto state = core_->state_stats(); + const auto cached = core_->cache()->snapshot(); + EngineStats result; + result.snapshot_pages_written = core_->last_snapshot_pages_written(); + result.snapshot_pages_total = core_->snapshot_pages_total(); + result.last_applied_slot = last_applied_slot(); + result.contiguous_slot = contiguous_slot(); + result.gc_watermark = gc_watermark(); + result.io_failed = io_failed(); + result.buffer_pool_hits = cached.hits; + result.buffer_pool_misses = cached.misses; + result.buffer_pool_evictions = cached.evictions; + result.buffer_pool_resident = static_cast<uint32_t>(cached.resident_pages); + result.buffer_pool_used = result.buffer_pool_resident; + result.buffer_pool_dirty = static_cast<uint32_t>(state.logical.dirty_pages); + result.leaf_count = state.logical.leaf_pages; + result.inner_count = state.logical.inner_pages; + return result; +} + +TreeSummary Crowdbtree::tree_summary() const +{ + const auto state = core_->capture(); + TreeSummary result; + if (state->summary) { + result = *state->summary; + result.root_version = state->commit_seq; + const auto sources = state->sources ? state->sources->source_stats() : SourceMemoryStats{}; + result.exact = sources.known && sources.records == 0 && !state->pending_handoff; + return result; + } + result.root_version = state->commit_seq; + result.covered_slot = state->coverage; + result.live_kv = state->stats.kv_known_count; + result.live_logical_bytes = state->stats.kv_known_bytes; + result.reachable_leaf_pages = state->stats.leaf_pages; + result.reachable_inner_pages = state->stats.inner_pages; + result.reachable_page_capacity_bytes = state->stats.page_size_known_bytes; + result.exact = state->stats.leaf_pages == 0 && state->stats.inner_pages == 0 && state->sources && + state->sources->source_stats().known && state->sources->source_stats().records == 0 && + !state->pending_handoff; + return result; +} + +ScanProfile Crowdbtree::scan_profile() const +{ + return {}; +} + +void Crowdbtree::init_metrics(const std::string &prefix, const std::string &backend_label) +{ + (void)backend_label; + init_core_metrics(prefix); +} + +std::string Crowdbtree::flush_metrics_str(double window_secs, const char *timestamp, size_t width, size_t count_w, + size_t tps_w) +{ + if (metrics_registry_ == nullptr) { + return {}; + } + char *buf = nullptr; + size_t len = 0; + FILE *fp = open_memstream(&buf, &len); + if (fp == nullptr) { + return {}; + } + try { + metrics_registry_->flush_to(fp, window_secs, timestamp, "cpp-tree", width, count_w, tps_w); + } + catch (...) { + std::fclose(fp); + std::free(buf); + throw; + } + std::fclose(fp); + std::unique_ptr<char, decltype(&std::free)> output(buf, &std::free); + return {buf, len}; +} + +size_t Crowdbtree::max_name_len() const +{ + if (metrics_registry_ == nullptr) { + return 0; + } + return metrics_registry_->max_name_len(); +} + +void Crowdbtree::record_compaction_metrics(const MergeGcStats &stats, uint64_t elapsed_ns) +{ + CRB_LOG_INFO("[{}] compact_sparse_blocks: selected={} relocated={} bytes={} deleted={} elapsed_ns={}", name_, + stats.blocks_selected, stats.pages_relocated, stats.bytes_relocated, stats.blocks_deleted, elapsed_ns); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_generation_lease.cpp b/lib/crowdb-tree/src/btree/tree_generation_lease.cpp new file mode 100644 index 000000000..167c78e04 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_generation_lease.cpp @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_generation_lease_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +TreeGenerationLease::TreeGenerationLease(detail::ReservedUnique<TreeGenerationPreparation> state) + : state_(std::move(state)) +{ +} + +TreeGenerationLease::~TreeGenerationLease() +{ + cancel(); +} + +Status TreeGenerationLease::acquire() +try { + auto &prepared = *state_; + auto &core = *prepared.owner; + auto &state = *core.state_; + if (prepared.held || prepared.committed || !state.begin_replacement()) { + return Status::unavailable("generation lease is already held or Tree is closed"); + } + prepared.held = true; + if (core.closed() || state.publisher->capture()->generation != prepared.previous_generation) { + cancel(); + return Status::unavailable("Tree generation changed during preparation"); + } + // An armed proposal can still choose a distant slot, and accepted replay + // can already be waiting for a missing slot. With ingress fenced neither + // can obtain new gap repair. Leave the old generation usable instead of + // waiting for work whose prerequisite this lease would prevent. + if (state.apply_lifetime->outstanding() != 0 || state.replay_lifetime->outstanding() != 0) { + cancel(); + return Status::unavailable("generation replacement requires completed accepted apply and replay"); + } + state.source_rotation->drain(); + if (prepared.expected && + (state.publisher->capture() != prepared.expected || state.frontier->max_seen() > prepared.expected->coverage)) { + cancel(); + return Status::unavailable("split destination changed before prepared installation"); + } + // A completed controller can retain its backend ticket. Releasing it is + // reversible: no source or controller has been permanently closed yet. + state.flush->release_pending(); + state.lifetime->drain_accepted(); + state.binding->drain(); + state.cache->evict(); +#ifdef CROWDB_TREE_TEST_UTIL + if (state.before_generation_install_for_tests) { + state.before_generation_install_for_tests(); + } +#endif + if (core.closed()) { + cancel(); + return Status::unavailable("Tree closed while preparing generation installation"); + } + assert(state.publisher->idle()); + uint64_t expected = TreeCore::State::kReplacing; + if (!state.ingress.compare_exchange_strong(expected, TreeCore::State::kPrepared, std::memory_order_acq_rel)) { + cancel(); + return Status::unavailable("Tree closed before its prepared installation became ready"); + } + return Status::Ok(); +} + +catch (const std::bad_alloc &) { + cancel(); + return Status::resource_exhausted("generation acquisition exhausted its control credit"); +} + +catch (...) { + cancel(); + return Status::internal_error("generation acquisition failed before installation"); +} + +Status TreeGenerationLease::commit() noexcept +{ + auto &prepared = *state_; + auto &core = *prepared.owner; + auto &state = *core.state_; + if (prepared.committed) { + return Status::Ok(); + } + if (!prepared.held) { + return Status::unavailable(); + } + uint64_t expected = TreeCore::State::kPrepared; + if (!state.ingress.compare_exchange_strong(expected, TreeCore::State::kReplacing, std::memory_order_acq_rel)) { + return Status::unavailable(); + } + assert(state.publisher->idle()); + state.sources->close(); + state.flush->close(); + state.split_handoff.swap(prepared.previous_split_handoff); + state.split_sources.swap(prepared.previous_split_sources); + state.sources.swap(prepared.sources); + state.flush.swap(prepared.flush); + state.source_rotation->rebind(*state.sources, *state.flush); + state.source_lifecycle.swap(prepared.source_lifecycle); + state.frontier->reset_quiescent(prepared.frontier); + state.automatic_slot.store(prepared.frontier, std::memory_order_release); + state.publisher->install_replacement(prepared.publication); + prepared.committed = true; + // Old controls stay owned by this lease until its host-side release. Do + // not reclaim epochs or destroy old controllers on the commit boundary. + state.finish_replacement(); + prepared.held = false; + return Status::Ok(); +} + +Status TreeGenerationLease::set_split_frontier(uint64_t frontier, SourceMemoryStats memory) noexcept +{ + auto &prepared = *state_; + if (!prepared.held || prepared.committed || !prepared.inherited || frontier < prepared.coverage) { + return Status::invalid_argument(); + } + auto &range = prepared.inherited->inheritance; + if (!range.has_value() || prepared.inherited->tables.empty()) { + return Status::invalid_argument(); + } + if (frontier == prepared.coverage) { + prepared.inherited->inheritance.reset(); + prepared.inherited->tables.erase(prepared.inherited->tables.begin() + 1, prepared.inherited->tables.end()); + } + else { + range->frontier = frontier; + range->memory = memory; + } + prepared.frontier = frontier; + return Status::Ok(); +} + +void TreeGenerationLease::cancel() noexcept +{ + if (!state_->held) { + return; + } + auto &state = *state_->owner->state_; + uint64_t expected = TreeCore::State::kPrepared; + // The second case is acquire failing before its host-side guard handoff. + if (state.ingress.compare_exchange_strong(expected, TreeCore::State::kReplacing, std::memory_order_acq_rel) || + (expected & TreeCore::State::kReplacing) != 0) { + state.finish_replacement(); + } + state_->held = false; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_generation_lease.h b/lib/crowdb-tree/src/btree/tree_generation_lease.h new file mode 100644 index 000000000..9375a7c08 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_generation_lease.h @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "crowdb-tree/status.h" + +namespace crowdb::tree +{ +class TreeCore; +struct PublishedTreeState; +struct SplitGeneration; +struct TreeGenerationPreparation; +struct SourceMemoryStats; + +// The caller retains TreeCore throughout this host operation. Acquiring the +// lease drains accepted work in the background; commit only installs storage +// which preparation already owns. Cancellation reopens the unchanged child. +class TreeGenerationLease +{ + public: + static Status prepare(TreeCore &owner, std::shared_ptr<const PublishedTreeState> candidate, + const SplitGeneration *split, std::shared_ptr<TreeGenerationLease> *out); + ~TreeGenerationLease(); + + Status acquire(); + Status commit() noexcept; + // Before publication only: the independent tail has stopped growing. + Status set_split_frontier(uint64_t frontier, SourceMemoryStats memory) noexcept; + void cancel() noexcept; + + private: + explicit TreeGenerationLease(detail::ReservedUnique<TreeGenerationPreparation> state); + detail::ReservedUnique<TreeGenerationPreparation> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_generation_lease_internal.h b/lib/crowdb-tree/src/btree/tree_generation_lease_internal.h new file mode 100644 index 000000000..e67eded82 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_generation_lease_internal.h @@ -0,0 +1,29 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "tree_core_internal.h" +#include "tree_generation_lease.h" + +namespace crowdb::tree +{ +struct TreeGenerationPreparation +{ + TreeCore *owner = nullptr; + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<detail::AllocationReservation> source_lifecycle; + std::shared_ptr<detail::AllocationReservation> flush_lifecycle; + detail::ReservedUnique<SourceAdmission> sources; + detail::ReservedUnique<FlushController> flush; + std::shared_ptr<PublicationCoordinator::Replacement> publication; + std::shared_ptr<const PublishedTreeState> expected; + std::shared_ptr<PublishedSources> inherited; + uint64_t previous_generation = 0; + uint64_t coverage = 0; + uint64_t frontier = 0; + std::shared_ptr<SplitSourceSession> previous_split_sources; + std::shared_ptr<SplitHandoff> previous_split_handoff; + bool held = false; + bool committed = false; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_generation_prepare.cpp b/lib/crowdb-tree/src/btree/tree_generation_prepare.cpp new file mode 100644 index 000000000..eba58ed68 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_generation_prepare.cpp @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_bound.h" +#include "tree_generation_lease_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +Status reserve_generation(const std::shared_ptr<TreeRuntimeBinding> &binding, const MappingView &mapping, bool empty, + TreeGenerationPreparation &out, size_t source_count) +{ + const auto publication = PublicationCoordinator::replacement_allocation_bound(mapping, source_count); + auto bytes = detail::AllocationBound(publication.value_or(SIZE_MAX)) + .add(sizeof(TreeGenerationPreparation)) + .add(detail::shared_allocation_bound<TreeGenerationLease>()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .add(detail::shared_allocation_bound<PublishedTreeState>()); + if (empty) { + bytes.add(MappingView::path_copy_bound()) + .add(PageState::allocation_bound()) + .add(detail::shared_allocation_bound<PublishedPage>()) + .add(2 * detail::shared_allocation_bound<PublishedTreeState>()) + .add(PublishedSources::allocation_bound()) + .add(PageCacheResidency::page_allocation_bound(0) + 256); + } + const auto bound = bytes.value(); + const auto source_bound = SourceAdmission::allocation_bound(); + const auto flush_bound = detail::AllocationBound(FlushController::allocation_bound()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .value(); + if (!bound || !source_bound || !flush_bound) { + return Status::resource_exhausted("generation preparation bound overflow"); + } + auto metadata = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kReclaim, *bound); + auto source = + detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kAdmittedWriter, *source_bound); + auto flush = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kReclaim, *flush_bound); + if (!metadata || !source || !flush) { + return Status::resource_exhausted("generation preparation credit unavailable"); + } + out.metadata = detail::AllocationReservation::create(std::move(*metadata)); + out.source_lifecycle = detail::AllocationReservation::create(std::move(*source)); + // Publication preparation is sealed after installation. The controller + // instead reuses its bounded pause control until this generation closes. + out.flush_lifecycle = detail::AllocationReservation::create(std::move(*flush)); + return Status::Ok(); +} + +FlushBatchOptions flush_options(const Config &config, const TreeRuntimeBinding &binding) +{ + FlushBatchOptions batch; + batch.parallelism = + std::max<size_t>(1, std::min(binding.cpu()->worker_count(), binding.options().limits.io_operations)); + const auto max_key = config.max_key_size != 0 ? config.max_key_size : config.frame_bytes / 2; + batch.run_limits = { + .max_entries = 128, .max_key_bytes = std::max<size_t>(256 * 1024, max_key), .key_block_bytes = 1024}; + return batch; +} +} // namespace + +Status TreeGenerationLease::prepare(TreeCore &owner, std::shared_ptr<const PublishedTreeState> candidate, + const SplitGeneration *split, std::shared_ptr<TreeGenerationLease> *out) +try { + if (out == nullptr || *out) { + return Status::invalid_argument("generation preparation requires an empty output"); + } + const bool empty_candidate = candidate && candidate->mapping.empty() && candidate->root_page_id == kInvalidPageId; + TreeGenerationPreparation prepared; + prepared.owner = &owner; + { + TreeCore::State::Ingress admission(*owner.state_); + if (admission.owner == nullptr) { + return Status::unavailable("Tree ingress closed or replacing"); + } + PublishedSources::Inheritance inherited_range; + if (split != nullptr && split->inherited) { + const auto &range = split->inherited->inheritance; + if (!range.has_value()) { + return Status::invalid_argument("split generation requires its inheritance frontier"); + } + inherited_range = *range; + inherited_range.local_count = 1; + } + const auto current = owner.state_->publisher->capture(); + if (split != nullptr && (current != split->expected || owner.state_->frontier->max_seen() > current->coverage || + current->sources->inheritance || current->pending_handoff || + current->flush_admission_owner != 0 || current->checkpoint_inflight)) { + return Status::unavailable("split destination changed before baseline preparation"); + } + if (current->generation == UINT64_MAX) { + return Status::resource_exhausted("Tree generation exhausted"); + } + prepared.previous_generation = current->generation; + prepared.expected = split != nullptr ? split->expected : nullptr; + auto status = reserve_generation( + owner.binding(), candidate ? candidate->mapping : MappingView{}, !candidate || empty_candidate, prepared, + 1 + (split != nullptr && split->inherited ? split->inherited->tables.size() : 0)); + if (!status.ok()) { + return status; + } + auto next = candidate && !empty_candidate + ? detail::reserved_shared<PublishedTreeState>(prepared.metadata, *candidate) + : TreeCore::State::empty_state(prepared.metadata); + if (empty_candidate) { + next->coverage = candidate->coverage; + } + next->generation = current->generation + 1; + prepared.coverage = next->coverage; + prepared.frontier = split != nullptr && split->inherited ? inherited_range.frontier : next->coverage; + SourceWriteOptions writers{.account = owner.binding()->account()}; + writers.max_key_size = + owner.config().max_key_size != 0 ? owner.config().max_key_size : owner.config().frame_bytes / 2; + prepared.sources = detail::reserved_unique<SourceAdmission>( + prepared.source_lifecycle, next->coverage, next->generation, writers, prepared.source_lifecycle); + prepared.flush = detail::reserved_unique<FlushController>( + prepared.flush_lifecycle, *owner.state_->publisher, *owner.binding()->cpu(), *prepared.sources, + owner.state_->frontier->contiguous(), owner.binding()->lane(), + flush_options(owner.config(), *owner.binding()), FlushBatch::Maintenance{}, prepared.flush_lifecycle, + owner.state_->apply_progress); + std::shared_ptr<PublishedSources> inherited; + if (split != nullptr && split->inherited) { + inherited = detail::reserved_shared<PublishedSources>(prepared.metadata, prepared.metadata); + inherited->floor = next->coverage; + inherited->memory = prepared.sources->catalog()->memory; + inherited->tables.reserve(1 + split->inherited->tables.size()); + inherited->tables.push_back(prepared.sources->catalog()->active); + inherited->tables.insert(inherited->tables.end(), split->inherited->tables.begin(), + split->inherited->tables.end()); + inherited->inheritance = inherited_range; + } + prepared.inherited = inherited; + status = owner.state_->publisher->prepare_replacement( + {.metadata = prepared.metadata, .initial = std::move(next), .source_catalog = prepared.sources->catalog()}, + &prepared.publication, std::move(inherited)); + if (!status.ok()) { + return status; + } + } + auto metadata = prepared.metadata; + auto state = detail::reserved_unique<TreeGenerationPreparation>(metadata, std::move(prepared)); + *out = detail::reserved_construct<TreeGenerationLease>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) TreeGenerationLease(std::move(state)); }); + metadata->seal(); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("generation preparation allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_metrics.cpp b/lib/crowdb-tree/src/btree/tree_metrics.cpp new file mode 100644 index 000000000..28eefe7a6 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_metrics.cpp @@ -0,0 +1,331 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "btree/tree_metrics.h" + +#include "crowdb-tree/memory_budget.h" + +#include <algorithm> +#include <cassert> +#include <chrono> +#include <limits> +#include <utility> + +namespace crowdb::tree +{ +namespace +{ +bool checked_delta(uint64_t &value, uint64_t removed, uint64_t added) noexcept +{ + if (value < removed || std::numeric_limits<uint64_t>::max() - (value - removed) < added) { + return false; + } + value = value - removed + added; + return true; +} + +bool valid(const PageMetricsContribution *page) noexcept +{ + return page == nullptr || + ((!page->split_ready || page->split_candidate) && (page->overlay_entries == 0 || page->overlay_runs != 0)); +} + +TreeLogicalStats contribution(const PageMetricsContribution *page) noexcept +{ + if (page == nullptr) { + return {}; + } + TreeLogicalStats out; + out.leaf_pages = static_cast<uint64_t>(page->leaf); + out.inner_pages = static_cast<uint64_t>(!page->leaf); + out.dirty_pages = static_cast<uint64_t>(page->dirty); + out.overlay_pages = static_cast<uint64_t>(page->overlay_runs != 0); + out.overlay_runs = page->overlay_runs; + out.overlay_entries = page->overlay_entries; + out.page_size_known_bytes = page->page_size_known ? page->page_size_bytes : 0; + out.page_size_unknown_pages = static_cast<uint64_t>(!page->page_size_known); + out.page_size_estimate_bytes = page->page_size_estimate_bytes; + out.kv_known_count = page->leaf && page->kv_known ? page->kv_count : 0; + out.kv_known_bytes = page->leaf && page->kv_known ? page->kv_bytes : 0; + out.kv_unknown_pages = static_cast<uint64_t>(page->leaf && !page->kv_known); + out.consolidate_candidate_pages = static_cast<uint64_t>(page->consolidate_candidate); + out.split_candidate_pages = static_cast<uint64_t>(page->split_candidate); + out.split_ready_pages = static_cast<uint64_t>(page->split_ready); + out.merge_candidate_pages = static_cast<uint64_t>(page->merge_candidate); + out.writeback_candidate_pages = static_cast<uint64_t>(page->writeback_candidate); + out.evict_candidate_pages = static_cast<uint64_t>(page->evict_candidate); + return out; +} + +constexpr std::array kPageFields{&TreeLogicalStats::leaf_pages, + &TreeLogicalStats::inner_pages, + &TreeLogicalStats::dirty_pages, + &TreeLogicalStats::overlay_pages, + &TreeLogicalStats::overlay_runs, + &TreeLogicalStats::overlay_entries, + &TreeLogicalStats::page_size_known_bytes, + &TreeLogicalStats::page_size_unknown_pages, + &TreeLogicalStats::page_size_estimate_bytes, + &TreeLogicalStats::kv_known_count, + &TreeLogicalStats::kv_known_bytes, + &TreeLogicalStats::kv_unknown_pages, + &TreeLogicalStats::consolidate_candidate_pages, + &TreeLogicalStats::split_candidate_pages, + &TreeLogicalStats::split_ready_pages, + &TreeLogicalStats::merge_candidate_pages, + &TreeLogicalStats::writeback_candidate_pages, + &TreeLogicalStats::evict_candidate_pages}; + +constexpr std::array kResourceFields{&TreeResourceStats::allocated_bytes, + &TreeResourceStats::reserved_bytes, + &TreeResourceStats::payload_bytes, + &TreeResourceStats::frame_bytes, + &TreeResourceStats::metadata_bytes, + &TreeResourceStats::io_bytes, + &TreeResourceStats::scratch_bytes, + &TreeResourceStats::retired_bytes, + &TreeResourceStats::resident_frames, + &TreeResourceStats::cache_bytes, + &TreeResourceStats::cache_pinned_bytes, + &TreeResourceStats::proxy_pages, + &TreeResourceStats::read_inflight_bytes, + &TreeResourceStats::write_inflight_bytes, + &TreeResourceStats::materialize_inflight_bytes, + &TreeResourceStats::allocated_backing_bytes, + &TreeResourceStats::maintenance_queued, + &TreeResourceStats::maintenance_inflight}; +static_assert(kResourceFields.size() == static_cast<size_t>(TreeResourceMetric::Count)); + +void observe_max(std::atomic<uint64_t> &target, uint64_t value) noexcept +{ + auto prior = target.load(std::memory_order_relaxed); + while (prior < value && !target.compare_exchange_weak(prior, value, std::memory_order_relaxed)) { + } +} +} // namespace + +bool apply_page_metrics(TreeLogicalStats &stats, const PageMetricsContribution *old_page, + const PageMetricsContribution *new_page) noexcept +{ + if (!valid(old_page) || !valid(new_page)) { + return false; + } + auto next = stats; + const auto removed = contribution(old_page); + const auto added = contribution(new_page); + for (auto field : kPageFields) { + if (!checked_delta(next.*field, removed.*field, added.*field)) { + return false; + } + } + if (std::numeric_limits<uint64_t>::max() - next.leaf_pages < next.inner_pages) { + return false; + } + if (new_page != nullptr) { + next.page_size_max_observed_bytes = + std::max({next.page_size_max_observed_bytes, new_page->page_size_estimate_bytes, + new_page->page_size_known ? new_page->page_size_bytes : 0}); + } + stats = next; + return true; +} + +bool equal_page_metrics(const TreeLogicalStats &left, const TreeLogicalStats &right) noexcept +{ + return std::all_of(kPageFields.begin(), kPageFields.end(), [&](auto field) { return left.*field == right.*field; }); +} + +bool apply_merge_pair_metrics(TreeLogicalStats &stats, uint64_t removed_pairs, uint64_t removed_ready, + uint64_t added_pairs, uint64_t added_ready) noexcept +{ + auto next = stats; + if (removed_ready > removed_pairs || added_ready > added_pairs || + !checked_delta(next.merge_candidate_pairs, removed_pairs, added_pairs) || + !checked_delta(next.merge_ready_pairs, removed_ready, added_ready) || + next.merge_ready_pairs > next.merge_candidate_pairs) { + return false; + } + stats = next; + return true; +} + +bool TreeResourceMetrics::add(TreeResourceMetric metric, uint64_t amount) noexcept +{ + const auto index = static_cast<size_t>(metric); + if (index >= values_.size()) { + return false; + } + auto &value = values_[index]; + auto prior = value.load(std::memory_order_relaxed); + do { + if (std::numeric_limits<uint64_t>::max() - prior < amount) { + return false; + } + } while (!value.compare_exchange_weak(prior, prior + amount, std::memory_order_relaxed)); + return true; +} + +bool TreeResourceMetrics::remove(TreeResourceMetric metric, uint64_t amount) noexcept +{ + const auto index = static_cast<size_t>(metric); + if (index >= values_.size()) { + return false; + } + auto &value = values_[index]; + auto prior = value.load(std::memory_order_relaxed); + do { + if (prior < amount) { + return false; + } + } while (!value.compare_exchange_weak(prior, prior - amount, std::memory_order_relaxed)); + return true; +} + +TreeResourceStats TreeResourceMetrics::snapshot() const noexcept +{ + TreeResourceStats result; + for (size_t i = 0; i < values_.size(); ++i) { + result.*kResourceFields[i] = values_[i].load(std::memory_order_relaxed); + } + return result; +} + +size_t TreeWorkMetrics::begin(TreeStage stage, uint64_t started_at_ns) noexcept +{ + auto &entry = stages_[static_cast<size_t>(stage)]; + entry.attempts.fetch_add(1, std::memory_order_relaxed); + entry.inflight.fetch_add(1, std::memory_order_relaxed); + return track_age(stage, started_at_ns); +} + +void TreeWorkMetrics::complete(TreeStage stage, TreeOutcome outcome, uint64_t ns, size_t age_slot) noexcept +{ + observe_histogram(stage, ns); + auto &entry = stages_[static_cast<size_t>(stage)]; + entry.count.fetch_add(1, std::memory_order_relaxed); + entry.sum_ns.fetch_add(ns, std::memory_order_relaxed); + entry.outcomes[static_cast<size_t>(outcome)].fetch_add(1, std::memory_order_relaxed); + observe_max(entry.max_ns, ns); + entry.window_count.fetch_add(1, std::memory_order_relaxed); + entry.window_sum_ns.fetch_add(ns, std::memory_order_relaxed); + observe_max(entry.window_max_ns, ns); + finish_age(stage, age_slot); + entry.inflight.fetch_sub(1, std::memory_order_relaxed); +} + +TreeWorkTotals TreeWorkMetrics::totals() const noexcept +{ + TreeWorkTotals result{}; + for (size_t i = 0; i < stages_.size(); ++i) { + const auto &entry = stages_[i]; + auto &out = result[i]; + out.attempts = entry.attempts.load(std::memory_order_relaxed); + out.inflight = entry.inflight.load(std::memory_order_relaxed); + out.count = entry.count.load(std::memory_order_relaxed); + out.sum_ns = entry.sum_ns.load(std::memory_order_relaxed); + out.max_ns = entry.max_ns.load(std::memory_order_relaxed); + for (size_t j = 0; j < out.outcomes.size(); ++j) { + out.outcomes[j] = entry.outcomes[j].load(std::memory_order_relaxed); + } + } + return result; +} + +TreeWorkWindow TreeWorkMetrics::drain_window() noexcept +{ + TreeWorkWindow result{}; + for (size_t i = 0; i < stages_.size(); ++i) { + auto &entry = stages_[i]; + result[i] = {.count = entry.window_count.exchange(0, std::memory_order_relaxed), + .sum_ns = entry.window_sum_ns.exchange(0, std::memory_order_relaxed), + .max_ns = entry.window_max_ns.exchange(0, std::memory_order_relaxed), + .total_count = entry.count.load(std::memory_order_relaxed)}; + } + return result; +} + +std::shared_ptr<TreeMetricsCollector> TreeWorkMetrics::claim_collector(const std::shared_ptr<AllocationBudget> &budget) +{ + auto owner = shared_from_this(); + bool expected = false; + if (!collector_claimed_.compare_exchange_strong(expected, true, std::memory_order_relaxed)) { + return nullptr; + } + MetadataAllocator<TreeMetricsCollector> allocator(budget); + TreeMetricsCollector *value = nullptr; + try { + value = allocator.allocate(1); + ::new (static_cast<void *>(value)) TreeMetricsCollector(std::move(owner)); + } + catch (...) { + if (value != nullptr) { + allocator.deallocate(value, 1); + } + collector_claimed_.store(false, std::memory_order_relaxed); + throw; + } + // shared_ptr construction calls this deleter itself if its control-block + // allocation fails. Do not reset the claim twice across that boundary. + return {value, + [allocator](TreeMetricsCollector *entry) mutable { + std::destroy_at(entry); + allocator.deallocate(entry, 1); + }, + allocator}; +} + +TreeMetricsOperation::TreeMetricsOperation(std::shared_ptr<TreeWorkMetrics> metrics, TreeStage stage, + uint64_t started_at_ns) noexcept + : metrics_(std::move(metrics)), + stage_(stage), + started_at_ns_(started_at_ns) +{ + assert(static_cast<size_t>(stage) < kTreeStageCount); + if (metrics_) { + age_slot_ = metrics_->begin(stage_, started_at_ns_); + } +} + +TreeMetricsOperation::~TreeMetricsOperation() +{ + if (!finished_.load(std::memory_order_relaxed)) { + (void)finish(TreeOutcome::Cancelled, tree_metrics_now_ns()); + } +} + +bool TreeMetricsOperation::finish(TreeOutcome outcome, uint64_t ended_at_ns) noexcept +{ + bool expected = false; + if (static_cast<size_t>(outcome) >= kTreeOutcomeCount || + !finished_.compare_exchange_strong(expected, true, std::memory_order_relaxed)) { + return false; + } + if (metrics_) { + metrics_->complete(stage_, outcome, ended_at_ns >= started_at_ns_ ? ended_at_ns - started_at_ns_ : 0, + age_slot_); + } + return true; +} + +TreeMetricsCollector::TreeMetricsCollector(std::shared_ptr<TreeWorkMetrics> metrics) : metrics_(std::move(metrics)) +{ +} + +TreeMetricsCollector::~TreeMetricsCollector() +{ + metrics_->collector_claimed_.store(false, std::memory_order_relaxed); +} + +TreeWorkWindow TreeMetricsCollector::capture_window() noexcept +{ + return metrics_->drain_window(); +} + +uint64_t tree_metrics_now_ns() noexcept +{ + return static_cast<uint64_t>( + std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_metrics.h b/lib/crowdb-tree/src/btree/tree_metrics.h new file mode 100644 index 000000000..cca7f4204 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_metrics.h @@ -0,0 +1,358 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/btree/diagnostics.h" +#include "crowdb-tree/status.h" +#include "flush_dispatch_metrics.h" +#include "pending_page_times.h" +#include "tree_wait_metrics.h" +#include "tree_work_counters.h" + +#include <array> +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <memory_resource> +#include <set> +#include <vector> + +namespace crowdb::tree +{ +namespace detail +{ +class RetirementQueue; +struct RetirementObservation; +class AllocationReservation; +} // namespace detail +class AllocationBudget; + +// Each live PID contributes once, regardless of how many old readers retain +// earlier page representations. Candidate membership is distinct from jobs. +struct PageMetricsContribution +{ + bool leaf = true; + bool dirty = false; + uint64_t overlay_runs = 0; + uint64_t overlay_entries = 0; + bool page_size_known = false; + uint64_t page_size_bytes = 0; + uint64_t page_size_estimate_bytes = 0; + bool kv_known = false; + uint64_t kv_count = 0; + uint64_t kv_bytes = 0; + bool consolidate_candidate = false; + bool split_candidate = false; + bool split_ready = false; + bool merge_candidate = false; + bool writeback_candidate = false; + bool evict_candidate = false; + + bool operator==(const PageMetricsContribution &) const noexcept = default; +}; + +// Coordinator-only: apply to an unpublished copy, then publish with mapping. +// A null contribution denotes an absent page. Invalid deltas leave stats +// intact. +[[nodiscard]] bool apply_page_metrics(TreeLogicalStats &stats, const PageMetricsContribution *old_page, + const PageMetricsContribution *new_page) noexcept; +[[nodiscard]] bool apply_merge_pair_metrics(TreeLogicalStats &stats, uint64_t removed_pairs, uint64_t removed_ready, + uint64_t added_pairs, uint64_t added_ready) noexcept; + +// Compare only additive per-page fields; frontier, pending membership and +// historical maxima require their own ledgers. +[[nodiscard]] bool equal_page_metrics(const TreeLogicalStats &left, const TreeLogicalStats &right) noexcept; + +// Coordinator-owned membership, not a collector cache. One checkpoint may be +// in flight. Changes after capture survive its commit; representation-only +// changes must not call change_page(). Allocation belongs to admitted metadata. +class CheckpointPendingMetrics +{ + struct CaptureRetirement; + + public: + // Prepare an empty set before publication. commit() swaps the old set + // into the returned retirement owner; the publisher reclaims it off-path. + class PreparedCapture + { + public: + explicit PreparedCapture(CheckpointPendingMetrics &owner, + std::shared_ptr<detail::AllocationReservation> metadata = {}, + std::shared_ptr<const void> resource_owner = {}); + [[nodiscard]] std::shared_ptr<const void> commit(uint64_t token) noexcept; + + private: + CheckpointPendingMetrics &owner_; + std::shared_ptr<CaptureRetirement> retired_; + }; + + // Only one prepared delta exists at a time under the publication gate. + // Each PID occurs at most once. Insertions are rolled back on destruction; + // removals and mapping flags change only at the no-throw commit boundary. + class PreparedDelta + { + public: + PreparedDelta(CheckpointPendingMetrics &owner, size_t max_pages); + ~PreparedDelta(); + PreparedDelta(const PreparedDelta &) = delete; + PreparedDelta &operator=(const PreparedDelta &) = delete; + void change_page(uint64_t pid); + void remove_page(uint64_t pid); + void change_mapping() noexcept; + void copy_to(TreeLogicalStats &stats) const noexcept; + void commit() noexcept; + + private: + struct Edit + { + uint64_t pid; + bool inserted_pending = false; + bool inserted_since = false; + bool removed = false; + }; + + CheckpointPendingMetrics &owner_; + std::pmr::vector<Edit> edits_; + uint64_t changed_at_ns_ = 0; + bool mapping_changed_ = false; + bool committed_ = false; + }; + + // A caller-provided resource must outlive this ledger and can be backed by + // an admitted metadata arena. The ledger is never copied on publication. + explicit CheckpointPendingMetrics(std::pmr::memory_resource *resource = std::pmr::get_default_resource()); + CheckpointPendingMetrics(const CheckpointPendingMetrics &) = delete; + CheckpointPendingMetrics &operator=(const CheckpointPendingMetrics &) = delete; + void change_page(uint64_t pid); + void remove_page(uint64_t pid); + void change_mapping() noexcept; + // Standalone convenience; publication uses PreparedCapture to defer frees. + [[nodiscard]] bool capture(uint64_t token); + [[nodiscard]] bool finish(uint64_t token, bool committed) noexcept; + [[nodiscard]] bool can_capture(uint64_t token) const noexcept; + [[nodiscard]] bool can_finish(uint64_t token) const noexcept; + void copy_to(TreeLogicalStats &stats) const noexcept; + [[nodiscard]] bool contains(uint64_t pid) const; + bool retire_step() noexcept; + [[nodiscard]] uint64_t oldest_since_ns() const noexcept; + + [[nodiscard]] bool inflight() const noexcept + { + return inflight_; + } + + private: + // Node allocation is bounded per changed PID. A small flush must never + // reserve a tree-sized hash reallocation merely to mark one more page. + PendingPageTimes pending_; + PendingPageTimes since_capture_; + uint64_t mapping_since_ns_ = 0; + uint64_t mapping_capture_since_ns_ = 0; + uint64_t token_ = 0; + bool inflight_ = false; + bool mapping_pending_ = false; + bool mapping_since_capture_ = false; +}; + +enum class TreeResourceMetric : uint8_t { + AllocatedBytes, + ReservedBytes, + PayloadBytes, + FrameBytes, + MetadataBytes, + IoBytes, + ScratchBytes, + RetiredBytes, + ResidentFrames, + CacheBytes, + CachePinnedBytes, + ProxyPages, + ReadInflightBytes, + WriteInflightBytes, + MaterializeInflightBytes, + AllocatedBackingBytes, + MaintenanceQueued, + MaintenanceInflight, + Count +}; + +class BackingMetricExtent; + +class TreeResourceMetrics +{ + public: + // The allocation/extent owner reports actual acquisition and final release. + // Reference transfers only change the overlapping referenced gauges. + [[nodiscard]] bool add(TreeResourceMetric metric, uint64_t amount) noexcept; + [[nodiscard]] bool remove(TreeResourceMetric metric, uint64_t amount) noexcept; + [[nodiscard]] TreeResourceStats snapshot() const noexcept; + + private: + friend class BackingMetricExtent; + std::array<std::atomic<uint64_t>, static_cast<size_t>(TreeResourceMetric::Count)> values_{}; +}; + +enum class TreeStage : uint8_t { + FlushBatch, + FlushAdmissionWait, + FlushCapture, + FlushWriterDrain, + FlushHandoff, + FlushRange, + PublishQueueWait, + PublishCommit, + MaintenancePlan, + OverlayCoalesce, + PageSplit, + PageMerge, + PageLayout, + PageEncode, + BudgetWait, + PageLoadPipeline, + PageWritebackPipeline, + SnapshotCapture, + SnapshotCommit, + MemoryRetireWait, + FlushRepair, + Count +}; + +enum class TreeOutcome : uint8_t { Success, Error, Cancelled, Retry, Stale, Count }; + +inline constexpr size_t kTreeStageCount = static_cast<size_t>(TreeStage::Count); +inline constexpr size_t kTreeOutcomeCount = static_cast<size_t>(TreeOutcome::Count); + +struct TreeStageTotals +{ + uint64_t attempts = 0; + uint64_t inflight = 0; + uint64_t count = 0; + uint64_t sum_ns = 0; + uint64_t max_ns = 0; + std::array<uint64_t, kTreeOutcomeCount> outcomes{}; +}; + +struct TreeStageWindow +{ + uint64_t count = 0; + uint64_t sum_ns = 0; + uint64_t max_ns = 0; + uint64_t total_count = 0; +}; + +using TreeWorkTotals = std::array<TreeStageTotals, kTreeStageCount>; +using TreeWorkWindow = std::array<TreeStageWindow, kTreeStageCount>; + +// Fixed diagnostic storage never participates in operation admission. When +// the slots are full, the sample explicitly reports incomplete age coverage. +inline constexpr size_t kTreeWorkAgeCapacity = 256; + +struct TreeStageAge +{ + uint64_t inflight = 0; + uint64_t tracked = 0; + uint64_t untracked = 0; + uint64_t oldest_age_ns = 0; + bool complete = false; +}; + +using TreeWorkAges = std::array<TreeStageAge, kTreeStageCount>; + +class TreeMetricsCollector; +class TreeMetricsOperation; +struct TreeStageHistogram; +struct TreeWorkHistograms; + +class TreeWorkMetrics : public std::enable_shared_from_this<TreeWorkMetrics> +{ + public: + ~TreeWorkMetrics(); + // Enable is additive and samples terminal events from publication onward. + // The final metrics owner frees each paid instance, including after Tree + // close. + Status enable_histogram(TreeStage stage, const std::shared_ptr<AllocationBudget> &budget); + [[nodiscard]] static size_t histogram_allocation_bytes() noexcept; + [[nodiscard]] bool histogram_enabled(TreeStage stage) const noexcept; + [[nodiscard]] TreeWorkHistograms histogram_totals() const noexcept; + FlushDispatchMetrics dispatch; + [[nodiscard]] TreeWorkTotals totals() const noexcept; + [[nodiscard]] TreeWorkAges ages(uint64_t now_ns) const noexcept; + // Exactly one draining owner. It distributes copies of the returned window + // to consumers; ordinary totals() readers never reset any observation. + [[nodiscard]] std::shared_ptr<TreeMetricsCollector> + claim_collector(const std::shared_ptr<AllocationBudget> &budget = {}); + + private: + friend class TreeMetricsCollector; + friend class TreeMetricsOperation; + friend class detail::RetirementQueue; + friend struct detail::RetirementObservation; + + struct Stage + { + std::atomic<uint64_t> attempts{0}, inflight{0}, count{0}, sum_ns{0}, max_ns{0}; + std::atomic<uint64_t> window_count{0}, window_sum_ns{0}, window_max_ns{0}; + std::array<std::atomic<uint64_t>, kTreeOutcomeCount> outcomes{}; + std::atomic<uint64_t> untracked_ages{0}; + }; + + size_t begin(TreeStage stage, uint64_t started_at_ns) noexcept; + size_t track_age(TreeStage stage, uint64_t started_at_ns) noexcept; + void finish_age(TreeStage stage, size_t slot) noexcept; + void complete(TreeStage stage, TreeOutcome outcome, uint64_t ns, size_t age_slot) noexcept; + [[nodiscard]] TreeWorkWindow drain_window() noexcept; + std::array<Stage, kTreeStageCount> stages_{}; + std::atomic<bool> collector_claimed_{false}; + std::array<std::atomic<uint64_t>, kTreeWorkAgeCapacity> active_ages_{}; + std::atomic<size_t> next_age_slot_{0}; + + public: + TreeWorkCounters counters; + TreeWaitMetrics waits; + + private: + void observe_histogram(TreeStage stage, uint64_t ns) noexcept; + [[nodiscard]] TreeWorkHistograms drain_histograms() noexcept; + // Published once, never replaced. Every observing operation retains this + // metrics owner, so no hazard allocation or shared_ptr atomic is required. + std::array<std::atomic<TreeStageHistogram *>, kTreeStageCount> histograms_{}; +}; + +// Holds the metrics owner across asynchronous completion and records only the +// first terminal notification. Destruction of unfinished work is cancellation. +class TreeMetricsOperation +{ + public: + TreeMetricsOperation(std::shared_ptr<TreeWorkMetrics> metrics, TreeStage stage, uint64_t started_at_ns) noexcept; + ~TreeMetricsOperation(); + TreeMetricsOperation(const TreeMetricsOperation &) = delete; + TreeMetricsOperation &operator=(const TreeMetricsOperation &) = delete; + [[nodiscard]] bool finish(TreeOutcome outcome, uint64_t ended_at_ns) noexcept; + + private: + std::shared_ptr<TreeWorkMetrics> metrics_; + TreeStage stage_; + uint64_t started_at_ns_; + size_t age_slot_ = kTreeWorkAgeCapacity; + std::atomic<bool> finished_{false}; +}; + +class TreeMetricsCollector +{ + public: + ~TreeMetricsCollector(); + TreeMetricsCollector(const TreeMetricsCollector &) = delete; + TreeMetricsCollector &operator=(const TreeMetricsCollector &) = delete; + [[nodiscard]] TreeWorkWindow capture_window() noexcept; + [[nodiscard]] TreeWorkHistograms capture_histograms() noexcept; + + private: + friend class TreeWorkMetrics; + explicit TreeMetricsCollector(std::shared_ptr<TreeWorkMetrics> metrics); + std::shared_ptr<TreeWorkMetrics> metrics_; +}; + +[[nodiscard]] uint64_t tree_metrics_now_ns() noexcept; + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_options.cpp b/lib/crowdb-tree/src/btree/tree_options.cpp new file mode 100644 index 000000000..6f9ed5a19 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_options.cpp @@ -0,0 +1,18 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_options.h" + +#include "btree/tree_core_geometry.h" +#include "runtime/default_runtime.h" + +namespace crowdb::tree +{ +Status configure_tree_options(Config *options) +{ + const Config defaults; + if (options->leaf_split_bytes == defaults.leaf_split_bytes) { + options->leaf_split_bytes = leaf_record_capacity(options->frame_bytes); + } + return configure_default_runtime(options); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_options.h b/lib/crowdb-tree/src/btree/tree_options.h new file mode 100644 index 000000000..01009609f --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_options.h @@ -0,0 +1,9 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/config.h" + +namespace crowdb::tree +{ +Status configure_tree_options(Config *options); +} diff --git a/lib/crowdb-tree/src/btree/tree_reads.cpp b/lib/crowdb-tree/src/btree/tree_reads.cpp new file mode 100644 index 000000000..b0074a711 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_reads.cpp @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" + +namespace crowdb::tree +{ +GetView Crowdbtree::get_view(Slice key) const +{ + GetView result; + (void)get_view(key, &result); + return result; +} + +Status Crowdbtree::get_view(Slice key, GetView *out) const +{ + const auto status = core_->get(key, out); + if (status.code() == Code::kIoError || status.code() == Code::kCorruption) { + io_failed_.store(true, std::memory_order_release); + } + return status; +} + +void Crowdbtree::get_async(Slice key, std::function<void(Status, GetView)> on_done) const +{ + core_->get_async(key, std::move(on_done)); +} + +bool Crowdbtree::get(Slice key, uint64_t *out_slot, std::string *out_value) const +{ + GetView v = get_view(key); + if (!v.found()) { + return false; + } + if (out_slot != nullptr) { + *out_slot = v.slot(); + } + if (out_value != nullptr) { + *out_value = v.value().to_string(); // Clone into caller-owned storage. + } + return true; +} + +std::vector<get_result> Crowdbtree::multi_get(const std::vector<Slice> &keys) const +{ + std::vector<get_result> results; + results.reserve(keys.size()); + for (const Slice &k : keys) { + get_result g; + g.found = get(k, &g.slot, &g.value); + results.push_back(std::move(g)); + } + return results; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_scans.cpp b/lib/crowdb-tree/src/btree/tree_scans.cpp new file mode 100644 index 000000000..8d7ab5882 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_scans.cpp @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" + +namespace crowdb::tree +{ +Status Crowdbtree::scan(Slice prefix, Slice start_after, Slice end_key, size_t limit, size_t byte_budget, + bool keys_only, uint64_t deadline_ms, std::vector<scan_entry> *out, bool *truncated, + bool include_tombstones, ScanPackedBuf *out_packed, size_t *out_count, bool has_start_bound, + bool start_inclusive, bool strict_byte_budget) const +{ + CoreScanOptions options; + options.prefix = prefix; + options.start = start_after; + options.end = end_key; + options.limit = limit; + options.byte_budget = byte_budget; + options.deadline_ms = deadline_ms; + options.keys_only = keys_only; + options.include_tombstones = include_tombstones; + options.has_start_bound = has_start_bound || !start_after.empty(); + options.start_inclusive = start_inclusive; + options.strict_byte_budget = strict_byte_budget; + return core_->scan(options, out, truncated, out_packed, out_count); +} + +Status Crowdbtree::seek_reverse(Slice start_key, bool inclusive, Slice begin_key, scan_entry *out, bool *found) const +{ + if (out == nullptr || found == nullptr) { + return Status::invalid_argument("seek_reverse output is null"); + } + CoreScanOptions options; + options.start = start_key; + options.begin = begin_key; + options.reverse = true; + options.has_start_bound = true; + options.start_inclusive = inclusive; + options.limit = 1; + std::vector<scan_entry> entries; + bool truncated = false; + const auto status = core_->scan(options, &entries, &truncated); + *found = status.ok() && !entries.empty(); + if (*found) { + *out = std::move(entries.front()); + } + return status; +} + +Status Crowdbtree::scan_reverse(Slice start_key, bool has_start_bound, bool start_inclusive, Slice begin_key, + size_t limit, size_t byte_budget, std::vector<scan_entry> *out, bool *truncated) const +{ + CoreScanOptions options; + options.start = start_key; + options.begin = begin_key; + options.reverse = true; + options.has_start_bound = has_start_bound; + options.start_inclusive = start_inclusive; + options.limit = limit; + options.byte_budget = byte_budget; + return core_->scan(options, out, truncated); +} + +void Crowdbtree::scan_async(Slice prefix, Slice start_after, Slice end_key, size_t limit, size_t byte_budget, + bool keys_only, uint64_t deadline_ms, + std::function<void(Status, ScanPackedBuf, bool)> on_done) const +{ + CoreScanOptions options; + options.prefix = prefix; + options.start = start_after; + options.end = end_key; + options.limit = limit; + options.byte_budget = byte_budget; + options.deadline_ms = deadline_ms; + options.keys_only = keys_only; + options.has_start_bound = !start_after.empty(); + core_->scan_async(options, std::move(on_done)); +} + +void Crowdbtree::scan_directional_async(Slice prefix, Slice start_after, Slice end_key, size_t limit, + size_t byte_budget, bool keys_only, uint64_t deadline_ms, bool reverse, + std::function<void(Status, ScanPackedBuf, bool)> on_done) const +{ + CoreScanOptions options; + options.prefix = prefix; + options.start = start_after; + options.end = end_key; + options.limit = limit; + options.byte_budget = byte_budget; + options.deadline_ms = deadline_ms; + options.keys_only = keys_only; + options.reverse = reverse; + options.has_start_bound = !start_after.empty(); + core_->scan_async(options, std::move(on_done)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_snapshots.cpp b/lib/crowdb-tree/src/btree/tree_snapshots.cpp new file mode 100644 index 000000000..9ea590d9e --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_snapshots.cpp @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "snapshot/published_native_collect.h" +#include "snapshot/published_snapshot_view.h" + +namespace crowdb::tree +{ +Status Crowdbtree::install_snapshot(std::vector<leaf_entry> sorted_entries, uint64_t at_slot) +{ + return core_->install_portable(std::move(sorted_entries), at_slot); +} + +Status Crowdbtree::collect_native_frames(std::vector<NativeFrame> *out, uint64_t *out_root_page_id, + uint64_t *out_at_slot, uint64_t *out_next_page_id, const KeyRange *filter, + uint64_t *out_subtrees_skipped) +{ + if (filter != nullptr) { + auto status = filter->validate(); + if (!status.ok()) { + return status; + } + } + NativeStreamInfo info; + auto status = collect_published_frames(core_, core_->binding()->options().limits.routing_bytes, out, &info); + if (!status.ok()) { + return status; + } + if (out_root_page_id != nullptr) { + *out_root_page_id = info.root_page_id; + } + if (out_at_slot != nullptr) { + *out_at_slot = info.at_slot; + } + if (out_next_page_id != nullptr) { + *out_next_page_id = info.next_page_id; + } + if (out_subtrees_skipped != nullptr) { + *out_subtrees_skipped = 0; + } + return Status::Ok(); +} + +std::shared_ptr<Snapshot> Crowdbtree::snapshot_view() +{ + std::shared_ptr<Snapshot> result; + auto status = create_published_snapshot(core_, &result); + if (!status.ok()) { + throw SnapshotReadError(status); + } + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_state_reconcile.cpp b/lib/crowdb-tree/src/btree/tree_state_reconcile.cpp new file mode 100644 index 000000000..6cd86a8fc --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_state_reconcile.cpp @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_state_reconcile.h" + +#include "logical_page.h" + +namespace crowdb::tree +{ +TreeStateReconciliation::TreeStateReconciliation(std::shared_ptr<const PublishedTreeState> view) + : view_(std::move(view)) +{ +} + +void TreeStateReconciliation::visit(void *context, uint64_t pid, const std::shared_ptr<const PublishedPage> &page) +{ + auto &self = *static_cast<TreeStateReconciliation *>(context); + ++self.visited_; + if (!self.status_.ok()) { + return; + } + if (page->metrics.leaf != (page->level == 0) || (page->level == 0 && (!page->leaf || page->inner)) || + (page->level != 0 && (page->leaf || (!page->inner && !page->inner_backing))) || + (page->metrics.split_ready && !page->metrics.split_candidate) || + (page->metrics.writeback_candidate && !page->metrics.dirty)) { + self.status_ = Status::corruption("page shape or candidate metrics disagree"); + return; + } + if (self.pending_ != nullptr) { + self.inspect_membership(pid, *page); + } + self.page_ = page; + self.next_run_ = 0; + self.overlay_entries_ = 0; + if (pid == self.view_->root_page_id) { + self.recomputed_.height = static_cast<uint64_t>(page->level) + 1; + } +} + +void TreeStateReconciliation::inspect_page() +{ + const auto &page = *page_; + if (page.leaf) { + const auto &leaf = *page.leaf; + if (page.metrics.overlay_runs != leaf.overlays.size()) { + status_ = Status::corruption("overlay run count disagrees with page metrics"); + return; + } + if (next_run_ < leaf.overlays.size()) { + const auto &run = leaf.overlays[next_run_++]; + if (!run) { + status_ = Status::corruption("overlay owner missing during reconciliation"); + return; + } + const size_t first = leaf.range.lower ? run->lower_bound(*leaf.range.lower) : 0; + const size_t last = leaf.range.upper ? run->lower_bound(*leaf.range.upper) : run->entries().size(); + if (first > last || last - first > UINT64_MAX - overlay_entries_) { + status_ = Status::corruption("overlay contribution overflow"); + return; + } + overlay_entries_ += last - first; + return; + } + if (overlay_entries_ != page.metrics.overlay_entries) { + status_ = Status::corruption("overlay entry count disagrees with page metrics"); + return; + } + } + if (!apply_page_metrics(recomputed_, nullptr, &page.metrics)) { + status_ = Status::corruption("page metrics overflow during reconciliation"); + } + page_.reset(); +} + +Status TreeStateReconciliation::step(size_t max_slots) +{ + if (!view_ || max_slots == 0) { + return Status::invalid_argument("reconciliation needs a fixed view and a positive work bound"); + } + if (ledger_mode_ && pending_ == nullptr) { + return Status::invalid_argument("ledger reconciliation must use the publication owner"); + } + if (complete_) { + return status_; + } + started_ = true; + for (size_t unit = 0; unit < max_slots && status_.ok() && !complete_; ++unit) { + if (page_) { + inspect_page(); + } + else { + complete_ = view_->mapping.visit_from(next_pid_, 1, this, &TreeStateReconciliation::visit) && !page_; + } + } + if (!status_.ok()) { + complete_ = true; + } + if (complete_ && status_.ok() && + (!equal_page_metrics(recomputed_, view_->stats) || recomputed_.height != view_->stats.height || + recomputed_.page_size_max_observed_bytes > view_->stats.page_size_max_observed_bytes || + view_->stats.checkpoint_pending_pages > recomputed_.page_count())) { + status_ = Status::corruption("published aggregate differs from its fixed mapping"); + } + if (complete_ && status_.ok() && pending_ != nullptr) { + finish_ledger(); + } + return status_; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_state_reconcile.h b/lib/crowdb-tree/src/btree/tree_state_reconcile.h new file mode 100644 index 000000000..edcf4f928 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_state_reconcile.h @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "published_state.h" + +namespace crowdb::tree +{ +// Explicit, resumable diagnostics over one fixed publication. Each call is +// bounded by radix slots; collecting ordinary metrics never invokes this. +class TreeStateReconciliation +{ + public: + explicit TreeStateReconciliation(std::shared_ptr<const PublishedTreeState> view); + Status step(size_t max_slots); + + [[nodiscard]] bool complete() const noexcept + { + return complete_; + } + + [[nodiscard]] uint64_t visited_pages() const noexcept + { + return visited_; + } + + [[nodiscard]] const TreeLogicalStats &recomputed() const noexcept + { + return recomputed_; + } + + private: + friend class PublicationCoordinator; + Status step_ledger(size_t max_slots, const PublishedTreeState ¤t, const CheckpointPendingMetrics &pending, + const MergePairMetrics &merge); + void inspect_membership(uint64_t pid, const PublishedPage &page); + void finish_ledger(); + const CheckpointPendingMetrics *pending_ = nullptr; + const MergePairMetrics *merge_ = nullptr; + bool ledger_mode_ = false; + bool started_ = false; + uint64_t pending_pages_ = 0; + uint64_t merge_pairs_ = 0; + uint64_t merge_ready_ = 0; + static void visit(void *context, uint64_t pid, const std::shared_ptr<const PublishedPage> &page); + void inspect_page(); + std::shared_ptr<const PublishedTreeState> view_; + std::shared_ptr<const PublishedPage> page_; + size_t next_run_ = 0; + uint64_t overlay_entries_ = 0; + uint64_t next_pid_ = 0; + uint64_t visited_ = 0; + TreeLogicalStats recomputed_; + Status status_; + bool complete_ = false; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_state_reconcile_ledger.cpp b/lib/crowdb-tree/src/btree/tree_state_reconcile_ledger.cpp new file mode 100644 index 000000000..e1e2aa2c1 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_state_reconcile_ledger.cpp @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "logical_page.h" +#include "tree_state_reconcile.h" + +namespace crowdb::tree +{ +Status TreeStateReconciliation::step_ledger(size_t max_slots, const PublishedTreeState ¤t, + const CheckpointPendingMetrics &pending, const MergePairMetrics &merge) +{ + if (view_.get() != ¤t) { + return Status::unavailable("reconciliation publication changed"); + } + if (started_ && !ledger_mode_) { + return Status::invalid_argument("ledger reconciliation requires a fresh cursor"); + } + ledger_mode_ = true; + pending_ = &pending; + merge_ = &merge; + Status result; + try { + result = step(max_slots); + } + catch (...) { + pending_ = nullptr; + merge_ = nullptr; + throw; + } + pending_ = nullptr; + merge_ = nullptr; + return result; +} + +void TreeStateReconciliation::inspect_membership(uint64_t pid, const PublishedPage &page) +{ + pending_pages_ += static_cast<uint64_t>(pending_->contains(pid)); + if (page.metrics.merge_candidate != merge_->contains_page(pid) || page.merge_outgoing != merge_->pair(pid)) { + status_ = Status::corruption("merge membership disagrees with the publication ledger"); + return; + } + if (!page.merge_outgoing) { + return; + } + const auto &pair = *page.merge_outgoing; + const auto right = view_->mapping.get(pair.right); + const auto parent = view_->mapping.get(pair.parent); + if (pair.left != pid || !page.leaf || !right || !right->leaf || !parent || parent->level == 0 || + page.leaf->right_sibling != pair.right || page.leaf->content_id != pair.left_content || + right->leaf->content_id != pair.right_content || page.leaf->route_generation != pair.left_route || + right->leaf->route_generation != pair.right_route || page.metrics.split_candidate || + right->metrics.split_candidate) { + status_ = Status::corruption("canonical merge pair refers to a different logical version"); + return; + } + ++merge_pairs_; + merge_ready_ += static_cast<uint64_t>(pair.ready); +} + +void TreeStateReconciliation::finish_ledger() +{ + TreeLogicalStats pending_stats; + pending_->copy_to(pending_stats); + const auto pairs = merge_->snapshot(); + if (pending_pages_ != view_->stats.checkpoint_pending_pages || + pending_pages_ != pending_stats.checkpoint_pending_pages || + pending_stats.checkpoint_mapping_pending != view_->stats.checkpoint_mapping_pending || + pending_->inflight() != view_->checkpoint_inflight || + pending_->oldest_since_ns() != view_->checkpoint_pending_since_ns || + merge_pairs_ != view_->stats.merge_candidate_pairs || merge_pairs_ != pairs.pairs || + merge_ready_ != view_->stats.merge_ready_pairs || merge_ready_ != pairs.ready || + recomputed_.merge_candidate_pages != pairs.pages || view_->merge_candidate_since_ns != pairs.oldest_since_ns) { + status_ = Status::corruption("published pending or canonical pair totals disagree with their membership"); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_wait_metrics.cpp b/lib/crowdb-tree/src/btree/tree_wait_metrics.cpp new file mode 100644 index 000000000..ba4cac5ea --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_wait_metrics.cpp @@ -0,0 +1,121 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_wait_metrics.h" + +#include "tree_metrics.h" + +#include <algorithm> +#include <cassert> +#include <limits> +#include <utility> + +namespace crowdb::tree +{ +namespace +{ +constexpr uint64_t kBits = 3; +constexpr uint64_t kMask = (1ULL << kBits) - 1; +static_assert(kTreeWaitClasses <= kMask); +} // namespace + +size_t TreeWaitMetrics::begin(TreeWaitClass kind, uint64_t started) noexcept +{ + const auto index = static_cast<size_t>(kind); + auto &summary = summaries_[index]; + summary.attempts.fetch_add(1, std::memory_order_relaxed); + summary.inflight.fetch_add(1, std::memory_order_relaxed); + if (started <= (std::numeric_limits<uint64_t>::max() >> kBits)) { + const auto value = (started << kBits) | (index + 1); + const auto first = next_age_.fetch_add(1, std::memory_order_relaxed); + for (size_t i = 0; i < ages_.size(); ++i) { + const auto slot = (first + i) % ages_.size(); + uint64_t empty = 0; + if (ages_[slot].compare_exchange_strong(empty, value, std::memory_order_release, + std::memory_order_relaxed)) { + return slot; + } + } + } + summary.untracked.fetch_add(1, std::memory_order_relaxed); + return kTreeWaitAgeCapacity; +} + +void TreeWaitMetrics::complete(TreeWaitClass kind, TreeWaitOutcome outcome, uint64_t ns, size_t slot) noexcept +{ + auto &summary = summaries_[static_cast<size_t>(kind)]; + if (slot < ages_.size()) { + ages_[slot].store(0, std::memory_order_release); + } + else { + summary.untracked.fetch_sub(1, std::memory_order_relaxed); + } + summary.inflight.fetch_sub(1, std::memory_order_relaxed); + summary.count.fetch_add(1, std::memory_order_relaxed); + summary.sum_ns.fetch_add(ns, std::memory_order_relaxed); + auto old = summary.max_ns.load(std::memory_order_relaxed); + while (old < ns && !summary.max_ns.compare_exchange_weak(old, ns, std::memory_order_relaxed)) { + } + summary.outcomes[static_cast<size_t>(outcome)].fetch_add(1, std::memory_order_relaxed); +} + +std::array<TreeWaitTotals, kTreeWaitClasses> TreeWaitMetrics::totals(uint64_t now_ns) const noexcept +{ + std::array<TreeWaitTotals, kTreeWaitClasses> result{}; + std::array<uint64_t, kTreeWaitClasses> earliest; + earliest.fill(std::numeric_limits<uint64_t>::max()); + for (size_t i = 0; i < result.size(); ++i) { + const auto &in = summaries_[i]; + auto &out = result[i]; + out.attempts = in.attempts.load(std::memory_order_relaxed); + out.inflight = in.inflight.load(std::memory_order_relaxed); + out.count = in.count.load(std::memory_order_relaxed); + out.sum_ns = in.sum_ns.load(std::memory_order_relaxed); + out.max_ns = in.max_ns.load(std::memory_order_relaxed); + out.untracked = in.untracked.load(std::memory_order_relaxed); + for (size_t o = 0; o < out.outcomes.size(); ++o) { + out.outcomes[o] = in.outcomes[o].load(std::memory_order_relaxed); + } + } + for (const auto &slot : ages_) { + const auto value = slot.load(std::memory_order_acquire); + if (value != 0) { + const auto index = static_cast<size_t>((value & kMask) - 1); + ++result[index].tracked; + earliest[index] = std::min(earliest[index], value >> kBits); + } + } + for (size_t i = 0; i < result.size(); ++i) { + auto &out = result[i]; + out.complete = out.untracked == 0 && out.tracked == out.inflight; + if (out.tracked != 0 && now_ns >= earliest[i]) { + out.oldest_age_ns = now_ns - earliest[i]; + } + } + return result; +} + +TreeWaitSpan::~TreeWaitSpan() +{ + if (metrics_) { + finish(TreeWaitOutcome::Cancelled, tree_metrics_now_ns()); + } +} + +void TreeWaitSpan::begin(std::shared_ptr<TreeWorkMetrics> metrics, TreeWaitClass kind, uint64_t started) noexcept +{ + assert(!metrics_); + metrics_ = std::move(metrics); + kind_ = kind; + started_ = started; + if (metrics_) { + slot_ = metrics_->waits.begin(kind_, started_); + } +} + +void TreeWaitSpan::finish(TreeWaitOutcome outcome, uint64_t ended) noexcept +{ + if (auto metrics = std::exchange(metrics_, {})) { + metrics->waits.complete(kind_, outcome, ended >= started_ ? ended - started_ : 0, slot_); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_wait_metrics.h b/lib/crowdb-tree/src/btree/tree_wait_metrics.h new file mode 100644 index 000000000..92f167a19 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_wait_metrics.h @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <array> +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <memory> + +namespace crowdb::tree +{ +class TreeWorkMetrics; + +// Budget classes retain their ResourceBudget order; cache joins are separate +// from both the first loader's pipeline and ordinary ready-cache hits. +enum class TreeWaitClass : uint8_t { Handoff, Routing, Reclaim, AdmittedWriter, GapRepair, Io, SharedCache, Count }; +enum class TreeWaitOutcome : uint8_t { Granted, Error, Cancelled, Count }; +inline constexpr size_t kTreeWaitClasses = static_cast<size_t>(TreeWaitClass::Count); +inline constexpr size_t kTreeWaitOutcomes = static_cast<size_t>(TreeWaitOutcome::Count); +inline constexpr size_t kTreeWaitAgeCapacity = 256; + +struct TreeWaitTotals +{ + uint64_t attempts = 0; + uint64_t inflight = 0; + uint64_t count = 0; + uint64_t sum_ns = 0; + uint64_t max_ns = 0; + std::array<uint64_t, kTreeWaitOutcomes> outcomes{}; + uint64_t tracked = 0; + uint64_t untracked = 0; + uint64_t oldest_age_ns = 0; + bool complete = false; +}; + +class TreeWaitMetrics +{ + public: + [[nodiscard]] std::array<TreeWaitTotals, kTreeWaitClasses> totals(uint64_t now_ns) const noexcept; + + private: + friend class TreeWaitSpan; + size_t begin(TreeWaitClass kind, uint64_t started) noexcept; + void complete(TreeWaitClass kind, TreeWaitOutcome outcome, uint64_t ns, size_t slot) noexcept; + + struct Summary + { + std::atomic<uint64_t> attempts{0}, inflight{0}, count{0}, sum_ns{0}, max_ns{0}, untracked{0}; + std::array<std::atomic<uint64_t>, kTreeWaitOutcomes> outcomes{}; + }; + + std::array<Summary, kTreeWaitClasses> summaries_{}; + std::array<std::atomic<uint64_t>, kTreeWaitAgeCapacity> ages_{}; + std::atomic<size_t> next_age_{0}; +}; + +// One request owner serializes begin/terminal. Repeated terminal calls do +// nothing; an unfinished owner records cancellation before releasing metrics. +class TreeWaitSpan +{ + public: + TreeWaitSpan() = default; + ~TreeWaitSpan(); + TreeWaitSpan(const TreeWaitSpan &) = delete; + TreeWaitSpan &operator=(const TreeWaitSpan &) = delete; + void begin(std::shared_ptr<TreeWorkMetrics> metrics, TreeWaitClass kind, uint64_t started) noexcept; + void finish(TreeWaitOutcome outcome, uint64_t ended) noexcept; + + private: + std::shared_ptr<TreeWorkMetrics> metrics_; + TreeWaitClass kind_ = TreeWaitClass::Handoff; + uint64_t started_ = 0; + size_t slot_ = kTreeWaitAgeCapacity; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_work_age.cpp b/lib/crowdb-tree/src/btree/tree_work_age.cpp new file mode 100644 index 000000000..aaedcccca --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_work_age.cpp @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_metrics.h" + +#include <algorithm> +#include <cassert> +#include <limits> + +namespace crowdb::tree +{ +namespace +{ +constexpr uint64_t kStageBits = 5; +constexpr uint64_t kStageMask = (1ULL << kStageBits) - 1; +static_assert(kTreeStageCount <= kStageMask); +} // namespace + +size_t TreeWorkMetrics::track_age(TreeStage stage, uint64_t started_at_ns) noexcept +{ + const auto index = static_cast<size_t>(stage); + if (started_at_ns <= (std::numeric_limits<uint64_t>::max() >> kStageBits)) { + const auto value = (started_at_ns << kStageBits) | (index + 1); + const auto first = next_age_slot_.fetch_add(1, std::memory_order_relaxed); + for (size_t i = 0; i < active_ages_.size(); ++i) { + const auto slot = (first + i) % active_ages_.size(); + uint64_t empty = 0; + if (active_ages_[slot].compare_exchange_strong(empty, value, std::memory_order_release, + std::memory_order_relaxed)) { + return slot; + } + } + } + stages_[index].untracked_ages.fetch_add(1, std::memory_order_relaxed); + return kTreeWorkAgeCapacity; +} + +void TreeWorkMetrics::finish_age(TreeStage stage, size_t slot) noexcept +{ + if (slot < active_ages_.size()) { + active_ages_[slot].store(0, std::memory_order_release); + } + else { + const auto before = stages_[static_cast<size_t>(stage)].untracked_ages.fetch_sub(1, std::memory_order_relaxed); + assert(before != 0); + (void)before; + } +} + +TreeWorkAges TreeWorkMetrics::ages(uint64_t now_ns) const noexcept +{ + TreeWorkAges result{}; + std::array<uint64_t, kTreeStageCount> earliest; + earliest.fill(std::numeric_limits<uint64_t>::max()); + for (size_t i = 0; i < stages_.size(); ++i) { + result[i].inflight = stages_[i].inflight.load(std::memory_order_relaxed); + result[i].untracked = stages_[i].untracked_ages.load(std::memory_order_relaxed); + } + for (const auto &slot : active_ages_) { + const auto value = slot.load(std::memory_order_acquire); + if (value == 0) { + continue; + } + const auto index = static_cast<size_t>((value & kStageMask) - 1); + ++result[index].tracked; + earliest[index] = std::min(earliest[index], value >> kStageBits); + } + for (size_t i = 0; i < result.size(); ++i) { + auto &out = result[i]; + out.complete = out.untracked == 0 && out.tracked == out.inflight; + if (out.tracked != 0 && now_ns >= earliest[i]) { + out.oldest_age_ns = now_ns - earliest[i]; + } + } + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_work_counters.cpp b/lib/crowdb-tree/src/btree/tree_work_counters.cpp new file mode 100644 index 000000000..36f07dfca --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_work_counters.cpp @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_work_counters.h" + +#include "allocation_reservation.h" +#include "tree_metrics.h" + +namespace crowdb::tree +{ +void TreeWorkCounters::add(TreeWorkCounter counter, uint64_t amount, TreeWorkCause cause) noexcept +{ + const auto c = static_cast<size_t>(counter); + const auto t = static_cast<size_t>(cause); + if (c < kTreeWorkCounterCount && t < kTreeWorkCauseCount) { + values_[t][c].fetch_add(amount, std::memory_order_relaxed); + } +} + +TreeCounterTotals TreeWorkCounters::snapshot() const noexcept +{ + TreeCounterTotals result{}; + for (size_t t = 0; t < kTreeWorkCauseCount; ++t) { + for (size_t c = 0; c < kTreeWorkCounterCount; ++c) { + result[t][c] = values_[t][c].load(std::memory_order_relaxed); + } + } + return result; +} + +void record_work(const std::shared_ptr<TreeWorkMetrics> &metrics, TreeWorkCounter counter, uint64_t amount, + TreeWorkCause cause) noexcept +{ + if (metrics && amount != 0) { + metrics->counters.add(counter, amount, cause); + } +} + +void record_work(const std::shared_ptr<detail::AllocationReservation> &metadata, TreeWorkCounter counter, + uint64_t amount, TreeWorkCause cause) noexcept +{ + if (metadata && amount != 0) { + record_work(metadata->account()->work_metrics(), counter, amount, cause); + } +} + +void record_work(const std::shared_ptr<AllocationBudget> &budget, TreeWorkCounter counter, uint64_t amount, + TreeWorkCause cause) noexcept +{ + if (budget && amount != 0) { + record_work(budget->retirement_metrics(), counter, amount, cause); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_work_counters.h b/lib/crowdb-tree/src/btree/tree_work_counters.h new file mode 100644 index 000000000..ce01ca9c3 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_work_counters.h @@ -0,0 +1,163 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/btree/work_cause.h" + +#include <array> +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <string_view> + +namespace crowdb::tree +{ +// Closed dimensions. Quantities count completed physical work or committed +// logical results, never planned capacity. Operation outcomes remain in stages. +enum class TreeWorkCounter : uint8_t { + FlushInputTables, + FlushRangeJobs, + FlushCandidateVersions, + FlushWinningVersions, + FlushWinningKeyBytes, + FlushWinningValueBytes, + FlushHandoffBytes, + SourceDetached, + CandidateChecks, + CandidateSkips, + PublicationMismatches, + SplitOperations, + SplitOutputPages, + MergeOperations, + MergeInputPages, + MergeOutputPages, + CoalesceOperations, + CoalesceInputRuns, + CoalesceOutputRuns, + CoalesceInputEntries, + CoalesceOutputEntries, + PayloadReferencesDropped, + MemoryReclaimedBytes, + KeyCopyBytes, + HandoffValueCopyBytes, + CowMetadataCopyBytes, + MaterializeCopyBytes, + BackendBounceCopyBytes, + EncodedFrameBytes, + EncodedPrimaryFrames, + EncodedOverflowFrames, + EncodingKeyCopyBytes, + EncodingValueCopyBytes, + ReadOperations, + ReadBytes, + ReadErrors, + WriteOperations, + WriteBytes, + WriteErrors, + ReusedFrames, + ReusedBackingBytes, + ProxyAvoidedLoads, + CacheHits, + CacheMisses, + CacheJoinedLoads, + CacheEvictions, + StaleCallbacks, + RebaseMismatches, + AdmissionRejected, + AdmissionWaits, + SnapshotRetirementDelays, + BackingDeletedBytes, + CheckpointFailures, + Retries, + LogicalRetirementBytes, + Count +}; +inline constexpr size_t kTreeWorkCounterCount = static_cast<size_t>(TreeWorkCounter::Count); +inline constexpr size_t kTreeWorkCauseCount = static_cast<size_t>(TreeWorkCause::Count); +using TreeCounterValues = std::array<uint64_t, kTreeWorkCounterCount>; +using TreeCounterTotals = std::array<TreeCounterValues, kTreeWorkCauseCount>; +inline constexpr std::array<std::string_view, kTreeWorkCounterCount> kTreeWorkCounterNames{ + "flush.input.tables", + "flush.range.jobs", + "flush.candidate.versions", + "flush.winning.versions", + "flush.winning.key.bytes", + "flush.winning.value.bytes", + "flush.handoff.bytes", + "source.detached", + "candidate.checks", + "candidate.skips", + "publication.mismatches", + "split.operations", + "split.output.pages", + "merge.operations", + "merge.input.pages", + "merge.output.pages", + "coalesce.operations", + "coalesce.input.runs", + "coalesce.output.runs", + "coalesce.input.entries", + "coalesce.output.entries", + "payload.references.dropped", + "memory.reclaimed.bytes", + "key.copy.bytes", + "handoff.value.copy.bytes", + "cow.metadata.copy.bytes", + "materialize.copy.bytes", + "backend.bounce.copy.bytes", + "encode.frame.bytes", + "encode.primary.frames", + "encode.overflow.frames", + "encode.key.copy.bytes", + "encode.value.copy.bytes", + "read.operations", + "read.bytes", + "read.errors", + "write.operations", + "write.bytes", + "write.errors", + "reused.frames", + "reused.backing.bytes", + "proxy.avoided.loads", + "cache.hits", + "cache.misses", + "cache.joined.loads", + "cache.evictions", + "callbacks.stale", + "rebase.mismatches", + "admission.rejected", + "admission.waits", + "snapshot.retirement.delays", + "backing.deleted.bytes", + "checkpoint.failures", + "retries", + "memory.logical.retirement.bytes", +}; +inline constexpr std::array<std::string_view, kTreeWorkCauseCount> kTreeWorkCauseNames{ + "other", "flush", "cache_pressure", "checkpoint", "gc", "read", "snapshot", "allocation"}; + +class TreeWorkCounters +{ + public: + void add(TreeWorkCounter counter, uint64_t amount = 1, TreeWorkCause cause = TreeWorkCause::Other) noexcept; + [[nodiscard]] TreeCounterTotals snapshot() const noexcept; + + private: + std::array<std::array<std::atomic<uint64_t>, kTreeWorkCounterCount>, kTreeWorkCauseCount> values_{}; +}; +class TreeWorkMetrics; +class AllocationBudget; + +namespace detail +{ +class AllocationReservation; +} + +void record_work(const std::shared_ptr<TreeWorkMetrics> &metrics, TreeWorkCounter counter, uint64_t amount = 1, + TreeWorkCause cause = TreeWorkCause::Other) noexcept; +void record_work(const std::shared_ptr<detail::AllocationReservation> &metadata, TreeWorkCounter counter, + uint64_t amount = 1, TreeWorkCause cause = TreeWorkCause::Other) noexcept; +void record_work(const std::shared_ptr<AllocationBudget> &budget, TreeWorkCounter counter, uint64_t amount = 1, + TreeWorkCause cause = TreeWorkCause::Other) noexcept; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_work_histograms.cpp b/lib/crowdb-tree/src/btree/tree_work_histograms.cpp new file mode 100644 index 000000000..b4939b240 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_work_histograms.cpp @@ -0,0 +1,131 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_work_histograms.h" + +#include "crowdb-common/metrics/latency_histogram.h" +#include "crowdb-tree/memory_budget.h" + +namespace crowdb::tree +{ +struct TreeStageHistogram +{ + explicit TreeStageHistogram(std::shared_ptr<AllocationBudget> budget) : budget(std::move(budget)), histogram("") + { + } + + std::shared_ptr<AllocationBudget> budget; + common::metrics::LatencyHistogram histogram; +}; + +namespace +{ +TreeHistogramSnapshot convert(const common::metrics::LatencyHistogram::Snapshot &value) noexcept +{ + return {.count = value.count, + .sum_low = value.sum_low, + .sum_high = value.sum_high, + .min_ns = value.min, + .max_ns = value.max, + .p50_ns = value.p50, + .p90_ns = value.p90, + .p95_ns = value.p95, + .p99_ns = value.p99, + .p999_ns = value.p999, + .zero_count = value.zero_count, + .overflow_count = value.overflow_count, + .total_count = value.total_count}; +} + +void dispose(TreeStageHistogram *histogram) noexcept +{ + if (histogram != nullptr) { + MetadataAllocator<TreeStageHistogram> allocator(histogram->budget); + std::destroy_at(histogram); + allocator.deallocate(histogram, 1); + } +} +} // namespace + +TreeWorkMetrics::~TreeWorkMetrics() +{ + for (auto &slot : histograms_) { + dispose(slot.load(std::memory_order_relaxed)); + } +} + +size_t TreeWorkMetrics::histogram_allocation_bytes() noexcept +{ + return sizeof(TreeStageHistogram); +} + +bool TreeWorkMetrics::histogram_enabled(TreeStage stage) const noexcept +{ + const auto index = static_cast<size_t>(stage); + return index < histograms_.size() && histograms_[index].load(std::memory_order_acquire) != nullptr; +} + +Status TreeWorkMetrics::enable_histogram(TreeStage stage, const std::shared_ptr<AllocationBudget> &budget) +{ + const auto index = static_cast<size_t>(stage); + if (index >= histograms_.size() || !budget) { + return Status::invalid_argument(); + } + auto &slot = histograms_[index]; + if (slot.load(std::memory_order_acquire) != nullptr) { + return Status::Ok(); + } + MetadataAllocator<TreeStageHistogram> allocator(budget); + TreeStageHistogram *candidate = nullptr; + try { + candidate = allocator.allocate(1); + std::construct_at(candidate, budget); + } + catch (const std::bad_alloc &) { + if (candidate != nullptr) { + allocator.deallocate(candidate, 1); + } + return slot.load(std::memory_order_acquire) != nullptr ? Status::Ok() : Status::resource_exhausted(); + } + TreeStageHistogram *expected = nullptr; + if (!slot.compare_exchange_strong(expected, candidate, std::memory_order_release, std::memory_order_relaxed)) { + dispose(candidate); + } + return Status::Ok(); +} + +void TreeWorkMetrics::observe_histogram(TreeStage stage, uint64_t ns) noexcept +{ + if (auto *histogram = histograms_[static_cast<size_t>(stage)].load(std::memory_order_acquire)) { + histogram->histogram.observe(ns); + } +} + +TreeWorkHistograms TreeWorkMetrics::histogram_totals() const noexcept +{ + TreeWorkHistograms result; + for (size_t i = 0; i < histograms_.size(); ++i) { + if (const auto *histogram = histograms_[i].load(std::memory_order_acquire)) { + result.enabled_stages |= uint64_t{1} << i; + result.stages[i] = convert(histogram->histogram.snapshot_total()); + } + } + return result; +} + +TreeWorkHistograms TreeWorkMetrics::drain_histograms() noexcept +{ + TreeWorkHistograms result; + for (size_t i = 0; i < histograms_.size(); ++i) { + if (auto *histogram = histograms_[i].load(std::memory_order_acquire)) { + result.enabled_stages |= uint64_t{1} << i; + result.stages[i] = convert(histogram->histogram.flush()); + } + } + return result; +} + +TreeWorkHistograms TreeMetricsCollector::capture_histograms() noexcept +{ + return metrics_->drain_histograms(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_work_histograms.h b/lib/crowdb-tree/src/btree/tree_work_histograms.h new file mode 100644 index 000000000..3ea329af7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_work_histograms.h @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/tree_metrics.h" + +namespace crowdb::tree +{ +// Fixed result storage only. Bucket arrays exist solely for explicitly enabled +// stages. Quantiles are common LatencyHistogram bucket upper bounds in ns. +struct TreeHistogramSnapshot +{ + uint64_t count = 0; + uint64_t sum_low = 0; + uint64_t sum_high = 0; + uint64_t min_ns = 0; + uint64_t max_ns = 0; + uint64_t p50_ns = 0; + uint64_t p90_ns = 0; + uint64_t p95_ns = 0; + uint64_t p99_ns = 0; + uint64_t p999_ns = 0; + uint64_t zero_count = 0; + uint64_t overflow_count = 0; + uint64_t total_count = 0; +}; + +struct TreeWorkHistograms +{ + uint64_t enabled_stages = 0; + std::array<TreeHistogramSnapshot, kTreeStageCount> stages{}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_work_registry.cpp b/lib/crowdb-tree/src/btree/tree_work_registry.cpp new file mode 100644 index 000000000..93c962b33 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_work_registry.cpp @@ -0,0 +1,174 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_work_registry.h" + +#include "allocation_reservation.h" + +#include <array> +#include <cstdio> + +namespace crowdb::tree +{ +namespace +{ +// Registry serializes flushes and invokes its inline callbacks in registration +// order before sorting output names. Only the first callback samples the age +// slots. These observations never claim or drain the host collector window. +struct WorkSample +{ + explicit WorkSample(std::shared_ptr<TreeWorkMetrics> value) : metrics(std::move(value)) + { + } + + std::shared_ptr<TreeWorkMetrics> metrics; + TreeWorkTotals totals{}; + TreeWorkAges ages{}; + TreeCounterTotals counters{}; + std::array<TreeWaitTotals, kTreeWaitClasses> waits{}; + + uint64_t refresh() noexcept + { + const auto now = tree_metrics_now_ns(); + totals = metrics->totals(); + ages = metrics->ages(now); + counters = metrics->counters.snapshot(); + waits = metrics->waits.totals(now); + return now; + } +}; + +constexpr std::array<std::string_view, kTreeStageCount> stages{"flush.batch", + "flush.admission_wait", + "flush.capture", + "flush.writer_drain", + "flush.handoff", + "flush.range", + "publish.queue_wait", + "publish.commit", + "maintenance.plan", + "overlay.coalesce", + "page.split", + "page.merge", + "page.layout", + "page.encode", + "budget.wait", + "page.load_pipeline", + "page.writeback_pipeline", + "snapshot.capture", + "snapshot.commit", + "memory.retire_wait", + "flush.repair"}; +constexpr std::array<std::string_view, kTreeOutcomeCount> outcomes{"success", "error", "cancelled", "retry", "stale"}; + +template <class F> +void add(crowdb::common::metrics::MetricsRegistry ®istry, std::string_view prefix, std::string_view first, + std::string_view second, F callback) +{ + std::array<char, 192> suffix{}; + const int size = std::snprintf(suffix.data(), suffix.size(), ".work.%.*s.%.*s", static_cast<int>(first.size()), + first.data(), static_cast<int>(second.size()), second.data()); + registry.register_callback_gauge_inline(prefix, std::string_view(suffix.data(), static_cast<size_t>(size)), + callback); +} +} // namespace + +std::shared_ptr<crowdb::common::metrics::MetricsRegistry> +create_tree_work_registry(std::shared_ptr<std::pmr::memory_resource> resource, std::string_view prefix, + const std::shared_ptr<TreeWorkMetrics> &metrics, + const std::shared_ptr<AllocationBudget> &metadata) +{ + struct Owner + { + Owner(std::shared_ptr<std::pmr::memory_resource> resource, std::shared_ptr<TreeWorkMetrics> metrics) + : sample(std::move(metrics)), + registry(std::move(resource)) + { + } + + WorkSample sample; + crowdb::common::metrics::MetricsRegistry registry; + }; + + auto owner = std::allocate_shared<Owner>(MetadataAllocator<Owner>(metadata), std::move(resource), metrics); + auto *sample = &owner->sample; + auto ®istry = owner->registry; + registry.register_callback_gauge_inline(prefix, ".work.sampled_at.ns", [sample] { return sample->refresh(); }); + for (size_t i = 0; i < stages.size(); ++i) { + auto total = [&](std::string_view name, uint64_t TreeStageTotals::*field) { + add(registry, prefix, stages[i], name, [sample, i, field] { return sample->totals[i].*field; }); + }; + total("attempts.c", &TreeStageTotals::attempts); + total("inflight.g", &TreeStageTotals::inflight); + total("latency.count.c", &TreeStageTotals::count); + total("latency.sum.ns.c", &TreeStageTotals::sum_ns); + total("latency.max.ns.g", &TreeStageTotals::max_ns); + for (size_t j = 0; j < outcomes.size(); ++j) { + add(registry, prefix, stages[i], outcomes[j], [sample, i, j] { return sample->totals[i].outcomes[j]; }); + } + auto age = [&](std::string_view name, uint64_t TreeStageAge::*field) { + add(registry, prefix, stages[i], name, [sample, i, field] { return sample->ages[i].*field; }); + }; + age("age.tracked.g", &TreeStageAge::tracked); + age("age.untracked.g", &TreeStageAge::untracked); + age("age.oldest.ns.g", &TreeStageAge::oldest_age_ns); + add(registry, prefix, stages[i], "age.complete.g", + [sample, i] { return static_cast<uint64_t>(sample->ages[i].complete); }); + } + constexpr std::array<std::string_view, kTreeWaitClasses> wait_names{ + "wait.handoff", "wait.routing", "wait.reclaim", "wait.admitted_writer", + "wait.gap_repair", "wait.io", "wait.shared_cache"}; + constexpr std::array<std::string_view, kTreeWaitOutcomes> wait_outcomes{"granted", "error", "cancelled"}; + for (size_t i = 0; i < wait_names.size(); ++i) { + auto total = [&](std::string_view name, uint64_t TreeWaitTotals::*field) { + add(registry, prefix, wait_names[i], name, [sample, i, field] { return sample->waits[i].*field; }); + }; + total("attempts.c", &TreeWaitTotals::attempts); + total("inflight.g", &TreeWaitTotals::inflight); + total("latency.count.c", &TreeWaitTotals::count); + total("latency.sum.ns.c", &TreeWaitTotals::sum_ns); + total("latency.max.ns.g", &TreeWaitTotals::max_ns); + total("age.tracked.g", &TreeWaitTotals::tracked); + total("age.untracked.g", &TreeWaitTotals::untracked); + total("age.oldest.ns.g", &TreeWaitTotals::oldest_age_ns); + add(registry, prefix, wait_names[i], "age.complete.g", + [sample, i] { return static_cast<uint64_t>(sample->waits[i].complete); }); + for (size_t j = 0; j < wait_outcomes.size(); ++j) { + add(registry, prefix, wait_names[i], wait_outcomes[j], + [sample, i, j] { return sample->waits[i].outcomes[j]; }); + } + } + for (size_t counter = 0; counter < kTreeWorkCounterCount; ++counter) { + std::array<char, 128> suffix{}; + const auto name = kTreeWorkCounterNames[counter]; + const int size = + std::snprintf(suffix.data(), suffix.size(), ".%.*s.c", static_cast<int>(name.size()), name.data()); + registry.register_callback_gauge_inline(prefix, std::string_view(suffix.data(), static_cast<size_t>(size)), + [sample, counter] { + uint64_t total = 0; + for (const auto &cause : sample->counters) { + total += cause[counter]; + } + return total; + }); + } + for (const auto [name, counter] : std::array{ + std::pair{std::string_view(".page.split.c"), TreeWorkCounter::SplitOperations}, + std::pair{std::string_view(".page.merge.c"), TreeWorkCounter::MergeOperations} + }) { + registry.register_callback_gauge_inline(prefix, name, [sample, counter] { + uint64_t total = 0; + for (const auto &cause : sample->counters) { + total += cause[static_cast<size_t>(counter)]; + } + return total; + }); + } + for (size_t cause = 0; cause < kTreeWorkCauseCount; ++cause) { + for (size_t counter = 0; counter < kTreeWorkCounterCount; ++counter) { + add(registry, prefix, kTreeWorkCauseNames[cause], kTreeWorkCounterNames[counter], + [sample, cause, counter] { return sample->counters[cause][counter]; }); + } + } + return {owner, &owner->registry}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/tree_work_registry.h b/lib/crowdb-tree/src/btree/tree_work_registry.h new file mode 100644 index 000000000..029dd3993 --- /dev/null +++ b/lib/crowdb-tree/src/btree/tree_work_registry.h @@ -0,0 +1,15 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-common/metrics/metrics.h" +#include "tree_metrics.h" + +#include <string_view> + +namespace crowdb::tree +{ +[[nodiscard]] std::shared_ptr<crowdb::common::metrics::MetricsRegistry> +create_tree_work_registry(std::shared_ptr<std::pmr::memory_resource> resource, std::string_view prefix, + const std::shared_ptr<TreeWorkMetrics> &metrics, + const std::shared_ptr<AllocationBudget> &metadata); +} diff --git a/lib/crowdb-tree/src/btree/writeback_metrics.cpp b/lib/crowdb-tree/src/btree/writeback_metrics.cpp new file mode 100644 index 000000000..ec2aeb80b --- /dev/null +++ b/lib/crowdb-tree/src/btree/writeback_metrics.cpp @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "writeback_metrics.h" + +#include "allocation_bound.h" + +namespace crowdb::tree +{ +WritebackMetrics::WritebackMetrics(std::shared_ptr<TreeWorkMetrics> metrics, + const std::shared_ptr<detail::AllocationReservation> &metadata, size_t max_pages) + : metrics_(std::move(metrics)), + pending_(std::less<>{}, detail::ReservationAllocator<Entry>(metadata)), + max_pages_(max_pages) +{ +} + +std::optional<size_t> WritebackMetrics::allocation_bound(size_t pages) noexcept +{ + return detail::AllocationBound(detail::shared_allocation_bound<WritebackMetrics>()) + .add_product(pages, sizeof(Entry) + (8 * sizeof(void *))) + .value(); +} + +Status WritebackMetrics::begin(uint64_t pid) +{ + if (pending_.contains(pid)) { + return Status::Ok(); + } + if (pending_.size() >= max_pages_) { + return Status::resource_exhausted("writeback metrics exceed the captured page bound"); + } + pending_.try_emplace(pid, metrics_, TreeStage::PageWritebackPipeline, tree_metrics_now_ns()); + return Status::Ok(); +} + +void WritebackMetrics::finish(uint64_t pid, TreeOutcome outcome) noexcept +{ + const auto found = pending_.find(pid); + if (found != pending_.end()) { + (void)found->second.finish(outcome, tree_metrics_now_ns()); + pending_.erase(found); + } +} + +bool WritebackMetrics::finish_some(TreeOutcome outcome) noexcept +{ + for (size_t count = 0; count < 64 && !pending_.empty(); ++count) { + const auto entry = pending_.begin(); + (void)entry->second.finish(outcome, tree_metrics_now_ns()); + pending_.erase(entry); + } + return pending_.empty(); +} + +bool WritebackMetrics::retire_step() noexcept +{ + return finish_some(TreeOutcome::Cancelled); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/writeback_metrics.h b/lib/crowdb-tree/src/btree/writeback_metrics.h new file mode 100644 index 000000000..9880695e9 --- /dev/null +++ b/lib/crowdb-tree/src/btree/writeback_metrics.h @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "allocation_reservation.h" +#include "tree_metrics.h" + +#include <map> + +namespace crowdb::tree +{ +// Transient operations owned by one captured write/rebase pipeline. Durable +// page descriptors never retain these timers or their reserved metadata. +class WritebackMetrics +{ + public: + WritebackMetrics(std::shared_ptr<TreeWorkMetrics> metrics, + const std::shared_ptr<detail::AllocationReservation> &metadata, size_t max_pages); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t pages) noexcept; + [[nodiscard]] Status begin(uint64_t pid); + void finish(uint64_t pid, TreeOutcome outcome) noexcept; + [[nodiscard]] bool finish_some(TreeOutcome outcome) noexcept; + [[nodiscard]] bool retire_step() noexcept; + + private: + using Entry = std::pair<const uint64_t, TreeMetricsOperation>; + std::shared_ptr<TreeWorkMetrics> metrics_; + std::map<uint64_t, TreeMetricsOperation, std::less<>, detail::ReservationAllocator<Entry>> pending_; + size_t max_pages_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/writeback_rebase.cpp b/lib/crowdb-tree/src/btree/writeback_rebase.cpp new file mode 100644 index 000000000..c32400b1f --- /dev/null +++ b/lib/crowdb-tree/src/btree/writeback_rebase.cpp @@ -0,0 +1,141 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "writeback_rebase.h" + +#include "snapshot/primary_backing.h" + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +bool same_base(const BaseRef &captured, const BaseRef ¤t) +{ + return captured.identity == current.identity && captured.generation == current.generation && + captured.range.lower == current.range.lower && captured.range.upper == current.range.upper && + captured.max_slot == current.max_slot && captured.backing == current.backing && + (captured.backing || captured.resident == current.resident); +} + +Status leaf_rebase(const PageState &captured, const PageState ¤t, BaseRef base, const FinalPageSummary *summary, + uint64_t coverage, const std::shared_ptr<detail::AllocationReservation> &metadata, + PublishedPage &replacement, size_t max_sources) +{ + if (captured.route_generation != current.route_generation || captured.range.lower != current.range.lower || + captured.range.upper != current.range.upper || captured.right_sibling != current.right_sibling || + current.overlays.size() < captured.overlays.size() || current.bases.size() != captured.bases.size()) { + return Status::unavailable("writeback route or captured prefix changed"); + } + const auto retained = current.overlays.size() - captured.overlays.size(); + if (retained > max_sources) { + return Status::resource_exhausted("writeback descriptor work bound exceeded"); + } + if (!std::equal(captured.bases.begin(), captured.bases.end(), current.bases.begin(), same_base) || + !std::equal(captured.overlays.begin(), captured.overlays.end(), + current.overlays.begin() + static_cast<std::ptrdiff_t>(retained))) { + return Status::unavailable("writeback base or captured run identity changed"); + } + for (const auto &run : captured.overlays) { + if (!run || run->max_slot() > coverage) { + return Status::invalid_argument("writeback capture does not cover its runs"); + } + } + if (current.representation_id == UINT64_MAX) { + return Status::resource_exhausted("writeback representation identity exhausted"); + } + auto leaf = detail::reserved_shared<PageState>(metadata, metadata); + leaf->range = current.range; + leaf->route_generation = current.route_generation; + leaf->content_id = current.content_id; + leaf->representation_id = current.representation_id + 1; + leaf->right_sibling = current.right_sibling; + leaf->bases.push_back(std::move(base)); + leaf->overlays.assign(current.overlays.begin(), current.overlays.begin() + static_cast<std::ptrdiff_t>(retained)); + auto &metrics = replacement.metrics; + metrics.dirty = retained != 0; + metrics.writeback_candidate = metrics.dirty; + metrics.evict_candidate = false; + metrics.consolidate_candidate = retained != 0 && metrics.consolidate_candidate; + metrics.overlay_runs = retained; + metrics.overlay_entries = 0; + for (const auto &run : leaf->overlays) { + if (!run) { + return Status::invalid_argument("writeback current run is absent"); + } + const auto first = leaf->range.lower ? run->lower_bound(*leaf->range.lower) : 0; + const auto last = leaf->range.upper ? run->lower_bound(*leaf->range.upper) : run->entries().size(); + if (last < first || last - first > UINT64_MAX - metrics.overlay_entries) { + return Status::invalid_argument("writeback run range is invalid"); + } + metrics.overlay_entries += last - first; + } + if (retained == 0 && summary != nullptr) { + if (summary->live_value_bytes > UINT64_MAX - summary->live_key_bytes) { + return Status::resource_exhausted("writeback logical byte total overflow"); + } + metrics.page_size_known = true; + metrics.page_size_bytes = summary->primary_used_bytes; + metrics.page_size_estimate_bytes = summary->primary_used_bytes; + metrics.kv_known = summary->live_counts_known; + metrics.kv_count = summary->live_keys; + metrics.kv_bytes = summary->live_key_bytes + summary->live_value_bytes; + metrics.split_candidate = false; + metrics.split_ready = false; + } + replacement.leaf = std::move(leaf); + return Status::Ok(); +} +} // namespace + +Status build_writeback_rebase(const std::shared_ptr<const PublishedPage> &captured, + const std::shared_ptr<const PublishedPage> ¤t, const DurableCapturedFrame &written, + uint64_t coverage, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const PublishedPage> *out, size_t max_sources, uint64_t gc_floor) +try { + if (!captured || !current || out == nullptr || *out || !written.backing || !written.backing->reloadable() || + written.page_id >= MappingView::kPageLimit || max_sources == 0 || captured->level != current->level || + (current->level == 0 ? written.type != page_type::kLeafBase : written.type != page_type::kInnerBase)) { + return Status::invalid_argument("invalid writeback receipt or page kind"); + } + if (current->revision == UINT64_MAX) { + return Status::resource_exhausted("writeback page revision exhausted"); + } + auto replacement = detail::reserved_shared<PublishedPage>(metadata, *current); + replacement->revision++; + replacement->allocation_owner.reset(); + if (current->level == 0) { + if (!captured->leaf || !current->leaf) { + return Status::invalid_argument("writeback leaf descriptor is absent"); + } + BaseRef base{.identity = written.backing->content_id, + .generation = written.backing->generation, + .range = current->leaf->range, + .max_slot = coverage, + .backing = written.backing}; + auto status = + leaf_rebase(*captured->leaf, *current->leaf, std::move(base), written.summary ? &*written.summary : nullptr, + coverage, metadata, *replacement, max_sources); + if (!status.ok()) { + return status; + } + replacement->primary = make_primary_backing(written, *captured->leaf, coverage, gc_floor, metadata); + } + else { + if (captured->inner != current->inner || captured->inner_backing != current->inner_backing) { + return Status::unavailable("writeback inner routing changed"); + } + replacement->primary = make_inner_primary_backing(written, *current, coverage, gc_floor, metadata); + replacement->inner_backing = written.backing; + replacement->metrics.dirty = false; + replacement->metrics.writeback_candidate = false; + replacement->metrics.evict_candidate = replacement->inner != nullptr; + } + *out = std::move(replacement); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("writeback rebase metadata exhausted"); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/writeback_rebase.h b/lib/crowdb-tree/src/btree/writeback_rebase.h new file mode 100644 index 000000000..a9c0d5c0b --- /dev/null +++ b/lib/crowdb-tree/src/btree/writeback_rebase.h @@ -0,0 +1,21 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "published_state.h" +#include "snapshot/captured_write.h" + +namespace crowdb::tree +{ +// The receipt must come from writing this captured page. Preparation compares +// identities against the latest page; the caller publishes with that latest +// page as its expected value. A conflict requires replan, never a blind swap. +Status build_writeback_rebase(const std::shared_ptr<const PublishedPage> &captured, + const std::shared_ptr<const PublishedPage> ¤t, const DurableCapturedFrame &written, + uint64_t coverage, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const PublishedPage> *out, size_t max_sources = 128, + uint64_t gc_floor = 0); +// A private layout can write several durable leaves for one captured source. +// Its binding proves the source identity and coverage independently of output +// PIDs. Newer overlays survive; the new base owns only the written range. +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/buffer_external.cpp b/lib/crowdb-tree/src/buffer_external.cpp new file mode 100644 index 000000000..66a4f1f15 --- /dev/null +++ b/lib/crowdb-tree/src/buffer_external.cpp @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/buffer.h" +#include "crowdb-tree/external_allocation.h" + +namespace crowdb::tree +{ +namespace +{ +struct ExternalSliceHolder +{ + + std::shared_ptr<const ExternalAllocation> allocation; +}; +} // namespace + +buffer buffer::share_external(std::shared_ptr<const ExternalAllocation> allocation, size_t offset, size_t len) +{ + if (!allocation || offset > allocation->capacity() || len > allocation->capacity() - offset) { + throw std::out_of_range("external slice exceeds allocation capacity"); + } + using Holder = ExternalSliceHolder; + MetadataAllocator<Holder> allocator(allocation->budget()); + auto *holder = allocator.allocate(1); + try { + std::construct_at(holder, std::move(allocation)); + } + catch (...) { + allocator.deallocate(holder, 1); + throw; + } + const auto *identity = holder->allocation.get(); + const auto *data = offset == 0 ? identity->data() : identity->data() + offset; + auto result = wrap_external(data, len, holder, [](void *context) { + auto *owner = static_cast<Holder *>(context); + MetadataAllocator<Holder> allocator(owner->allocation->budget()); + std::destroy_at(owner); + allocator.deallocate(owner, 1); + }); + result.ext_.allocation_ = identity; + return result; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/c_api.cpp b/lib/crowdb-tree/src/c_api.cpp index fc298ed52..5ea871c33 100644 --- a/lib/crowdb-tree/src/c_api.cpp +++ b/lib/crowdb-tree/src/c_api.cpp @@ -10,7 +10,10 @@ // hand them back to ct_free_buf regardless of allocator details. #include "crowdb-tree/c_api.h" +#include "btree/tree_core_access.h" +#include "c_api_future_internal.h" #include "c_api_internal.h" +#include "c_api_owned_budget.h" #include "crowdb-common/log.h" #include "crowdb-tree/backend/async_page_store.h" #include "crowdb-tree/backend/block_page_store.h" @@ -20,6 +23,9 @@ #include "crowdb-tree/btree/range_rebuild.h" #include "crowdb-tree/crowdb-tree.h" #include "crowdb-tree/snapshot/snapshot_io.h" +#include "runtime/tree_runtime_c_api.h" +#include "runtime/tree_tuning_c_api.h" +#include "snapshot/published_snapshot_view.h" #ifdef CROWDB_HAVE_LIBURING # include "crowdb-common/diskio_uring.h" #endif @@ -67,7 +73,7 @@ ct_buf make_buf(const void *data, size_t len) // §5's zero-copy fast path). Only for a ct_buf whose // backing memory the caller separately keeps alive for at least as long as the // buffer is in use (ct_future_poll's kGet case: the ct_future_impl's own -// EpochManager::Guard). The caller must NOT pass this to ct_free_buf. +// GetView allocation owner). The caller must NOT pass this to ct_free_buf. ct_buf make_borrowed_buf(const void *data, size_t len) { ct_buf b; @@ -156,22 +162,19 @@ struct CompletionSignal struct ct_tree { - std::unique_ptr<PageStore> store; // null for pure in-memory engine - std::shared_ptr<PageStoreBundle> injected_store; - std::unique_ptr<Crowdbtree> tree; + // The bundle owns async_store, then uring, then store through reverse + // member destruction, and captured snapshot windows share its lifetime. + std::shared_ptr<PageStoreBundle> backend; std::shared_ptr<CompletionSignal> completion = std::make_shared<CompletionSignal>(); -#ifdef CROWDB_HAVE_LIBURING - // Both null for an in-memory tree, or if opening the async twin failed - // (see ct_open) -- get_async/flush_async/snapshot_async then fall back - // to completing synchronously. Declared so `uring` - // outlives `async_store` (async_store is non-owning re: uring, - // mirroring Config' own comment) and both outlive `tree`, which is - // what actually calls into them. - std::unique_ptr<crowdb::common::DiskIOUring> uring; - std::unique_ptr<AsyncPageStore> async_store; -#endif + std::unique_ptr<Crowdbtree> tree; }; +static std::shared_ptr<ct_future_impl> make_tree_future(ct_tree *tree) +{ + auto *core = TreeCoreAccess::get(*tree->tree); + return make_ct_future(core != nullptr ? core->binding()->lifecycle_budget() : nullptr); +} + struct ct_uring { #ifdef CROWDB_HAVE_LIBURING @@ -192,33 +195,6 @@ struct ct_uring // cancel-and-free: it never has to actually reach into // the reactor to stop anything, and never risks a dangling ct_future_impl* // if the I/O completes after the caller has already abandoned the future. -struct ct_future_impl -{ - enum class Kind : std::uint8_t { kGet, kFlush, kSnapshot, kScan }; - Kind kind = Kind::kGet; - // Written once by the completion callback (release), read once by the - // first ct_future_poll that observes it (acquire) -- that acquire/ - // release pair is the only synchronization the fields below need, - // since there is exactly one writer-then-reader handoff per future. - std::atomic<bool> done{false}; - ct_status status = static_cast<ct_status>(Code::kOk); - // kGet only. May hold a live EpochManager::Guard borrowing a resident - // frame's bytes (zero-copy fast path, - // ct_future_poll deliberately does *not* delete the handle for a kGet - // future (see its updated doc comment in c_api.h); the caller must - // always follow up with ct_future_free once done reading out_value. - GetView get_result; - uint64_t slot = 0; // kSnapshot only (last_applied_slot) - // kScan only: packed record buffer (same format ct_scan produces), - // received directly from the engine via ScanPackedBuf (R57: zero-copy - // staging — no re-pack loop, no make_buf). Ownership is transferred - // to the caller via release() in ct_future_poll. - ScanPackedBuf scan_packed; - uint64_t scan_count = 0; - bool scan_truncated = false; -}; - -using ct_future_handle = std::shared_ptr<ct_future_impl>; struct ct_view { @@ -231,11 +207,6 @@ struct ct_iter size_t pos = 0; }; -struct ct_export -{ - std::unique_ptr<SnapshotExport> exp; -}; - struct ct_import { ct_tree *owner = nullptr; @@ -283,14 +254,25 @@ void ct_page_store_free(ct_page_store *store) delete store; } -ct_status ct_open(const ct_options *opt, ct_tree **out) -{ +static ct_status open_configured(const ct_options *opt, ct_runtime *runtime, const ct_tree_runtime_options *binding, + ct_tree **out, uint64_t migration_generation, CApiMigration migrate) +try { if (opt == nullptr || out == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } auto h = std::make_unique<ct_tree>(); Config o; + if (runtime != nullptr || binding != nullptr) { + auto status = configure_tree_runtime(runtime, binding, &o); + if (!status.ok()) { + return to_status(status); + } + } + auto tuning_status = configure_tree_tuning(opt->tuning, &o); + if (!tuning_status.ok()) { + return to_status(tuning_status); + } if (opt->frame_bytes != 0) { o.frame_bytes = opt->frame_bytes; } @@ -345,25 +327,39 @@ ct_status ct_open(const ct_options *opt, ct_tree **out) } const bool durable = opt->path != nullptr && opt->path[0] != '\0'; + if (opt->page_store == nullptr && (durable || runtime == nullptr)) { + h->backend = std::make_shared<PageStoreBundle>(); + h->backend->backend_label = o.backend_label; + o.backend_owner = h->backend; + } if (opt->page_store != nullptr) { if (opt->page_store->bundle == nullptr || opt->page_store->bundle->store == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } - h->injected_store = opt->page_store->bundle; - o.page_store = h->injected_store->store.get(); - o.async_page_store = h->injected_store->async_store_view != nullptr ? h->injected_store->async_store_view - : h->injected_store->async_store.get(); - o.backend_label = h->injected_store->backend_label; + h->backend = opt->page_store->bundle; + o.backend_owner = h->backend; + o.page_store = h->backend->store.get(); + o.async_page_store = + h->backend->async_store_view != nullptr ? h->backend->async_store_view : h->backend->async_store.get(); + o.backend_label = h->backend->backend_label; #ifdef CROWDB_HAVE_LIBURING - o.async_uring = h->injected_store->uring.get(); + o.async_uring = h->backend->uring.get(); #endif std::unique_ptr<Crowdbtree> t; - Status os = Crowdbtree::open(o, &t); + Status os = migrate != nullptr ? migrate(o, migration_generation, &t) : Crowdbtree::open(o, &t); if (!os.ok()) { return to_status(os); } h->tree = std::move(t); } + else if (!durable && runtime != nullptr) { + // A volatile runtime Tree has no reloadable backend or checkpoint. + // Keep it on the same immutable core as direct C++ construction. + auto status = TreeCoreAccess::create_volatile(o, &h->tree); + if (!status.ok()) { + return to_status(status); + } + } else if (!durable) { // In-memory: BlockPageStore::open_mem with IU=1 std::unique_ptr<BlockPageStore> bs; @@ -372,8 +368,8 @@ ct_status ct_open(const ct_options *opt, ct_tree **out) return to_status(s); } bs->set_sync_mode(sm); - h->store = std::move(bs); - o.page_store = h->store.get(); + h->backend->store = std::move(bs); + o.page_store = h->backend->store.get(); std::unique_ptr<Crowdbtree> t; Status os = Crowdbtree::open(o, &t); if (!os.ok()) { @@ -391,8 +387,8 @@ ct_status ct_open(const ct_options *opt, ct_tree **out) return to_status(s); } ts->set_sync_mode(sm); - h->store = std::move(ts); - o.page_store = h->store.get(); + h->backend->store = std::move(ts); + o.page_store = h->backend->store.get(); std::unique_ptr<Crowdbtree> t; Status os = Crowdbtree::open(o, &t); if (!os.ok()) { @@ -405,8 +401,8 @@ ct_status ct_open(const ct_options *opt, ct_tree **out) uint32_t iu = opt->iu_size == 0 ? 1 : opt->iu_size; auto ms = std::make_unique<MemPageStore>(iu); ms->set_sync_mode(sm); - h->store = std::move(ms); - o.page_store = h->store.get(); + h->backend->store = std::move(ms); + o.page_store = h->backend->store.get(); std::unique_ptr<Crowdbtree> t; Status os = Crowdbtree::open(o, &t); if (!os.ok()) { @@ -424,13 +420,13 @@ ct_status ct_open(const ct_options *opt, ct_tree **out) return to_status(s); } bs->set_sync_mode(sm); - h->store = std::move(bs); - o.page_store = h->store.get(); + h->backend->store = std::move(bs); + o.page_store = h->backend->store.get(); #ifdef CROWDB_HAVE_LIBURING // Wire a DiskIOUring + BlockAsyncPageStore so get_async's demand-load // miss path completes off the poll thread instead of blocking - // the caller. The async_store borrows both the store (h->store) - // and the uring (h->uring), both owned by h and outliving tree. + // the caller. The async_store borrows both the store (h->backend->store) + // and the uring (h->backend->uring), both owned by h and outliving tree. crowdb::common::Topology uring_topo; crowdb::common::PipelineConfig uring_cfg; uring_cfg.entries = 256; @@ -438,15 +434,15 @@ ct_status ct_open(const ct_options *opt, ct_tree **out) uring_topo.pipelines.push_back(uring_cfg); auto candidate = std::make_unique<crowdb::common::DiskIOUring>(std::move(uring_topo)); if (candidate->valid()) { - h->uring = std::move(candidate); + h->backend->uring = std::move(candidate); // Register all extent fds with the uring so submit_* can route. - for (int fd : static_cast<BlockPageStore *>(h->store.get())->all_extent_fds()) { - h->uring->register_fd(fd); + for (int fd : static_cast<BlockPageStore *>(h->backend->store.get())->all_extent_fds()) { + h->backend->uring->register_fd(fd); } - h->async_store = - std::make_unique<BlockAsyncPageStore>(static_cast<BlockPageStore *>(h->store.get()), h->uring.get()); - o.async_uring = h->uring.get(); - o.async_page_store = h->async_store.get(); + h->backend->async_store = std::make_unique<BlockAsyncPageStore>( + static_cast<BlockPageStore *>(h->backend->store.get()), h->backend->uring.get()); + o.async_uring = h->backend->uring.get(); + o.async_page_store = h->backend->async_store.get(); } else { // Containers may run without CAP_SYS_ADMIN, which makes the @@ -467,6 +463,57 @@ ct_status ct_open(const ct_options *opt, ct_tree **out) return static_cast<ct_status>(Code::kOk); } +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +ct_status ct_open_configured(const ct_options *opt, ct_runtime *runtime, const ct_tree_runtime_options *binding, + ct_tree **out) +{ + return open_configured(opt, runtime, binding, out, 0, nullptr); +} + +ct_status ct_migrate_configured(const ct_options *opt, ct_runtime *runtime, const ct_tree_runtime_options *binding, + uint64_t generation, CApiMigration migrate, ct_tree **out) +{ + if (out != nullptr) { + *out = nullptr; + } + if (opt == nullptr || out == nullptr || runtime == nullptr || opt->page_store == nullptr || generation == 0 || + generation == UINT64_MAX || migrate == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + return open_configured(opt, runtime, binding, out, generation, migrate); +} + +ct_status ct_open(const ct_options *opt, ct_tree **out) +{ + return ct_open_configured(opt, nullptr, nullptr, out); +} + +crowdb::tree::Crowdbtree *c_api_tree(ct_tree *tree) noexcept +{ + return tree == nullptr ? nullptr : tree->tree.get(); +} + +crowdb::tree::PageStore *c_api_page_store(ct_tree *tree) noexcept +{ + return tree != nullptr && tree->backend ? tree->backend->store.get() : nullptr; +} + +std::shared_ptr<CompletionSignal> c_api_completion_signal(ct_tree *tree) noexcept +{ + return tree->completion; +} + +void c_api_notify_completion(const std::shared_ptr<CompletionSignal> &signal) noexcept +{ + signal->notify(); +} + void ct_close(ct_tree *t) { delete t; @@ -475,14 +522,23 @@ void ct_close(ct_tree *t) ct_status ct_rebuild_range(ct_tree *source, const ct_options *destination_options, ct_tree **out, ct_range_rebuild_stats *stats) { + return ct_rebuild_configured(source, destination_options, Config{}, out, stats); +} + +ct_status ct_rebuild_configured(ct_tree *source, const ct_options *destination_options, Config options, ct_tree **out, + ct_range_rebuild_stats *stats) +try { if (source == nullptr || destination_options == nullptr || out == nullptr || destination_options->page_store == nullptr || destination_options->page_store->bundle == nullptr || destination_options->page_store->bundle->store == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } - auto handle = std::make_unique<ct_tree>(); - Config options; + auto handle = std::make_unique<ct_tree>(); + auto tuning_status = configure_tree_tuning(destination_options->tuning, &options); + if (!tuning_status.ok()) { + return to_status(tuning_status); + } if (destination_options->frame_bytes != 0) { options.frame_bytes = destination_options->frame_bytes; } @@ -512,12 +568,12 @@ ct_status ct_rebuild_range(ct_tree *source, const ct_options *destination_option options.key_range = KeyRange::bounded(std::move(start), std::move(end)); } - handle->injected_store = destination_options->page_store->bundle; - options.page_store = handle->injected_store->store.get(); - options.async_page_store = handle->injected_store->async_store_view != nullptr - ? handle->injected_store->async_store_view - : handle->injected_store->async_store.get(); - options.backend_label = handle->injected_store->backend_label; + handle->backend = destination_options->page_store->bundle; + options.backend_owner = handle->backend; + options.page_store = handle->backend->store.get(); + options.async_page_store = handle->backend->async_store_view != nullptr ? handle->backend->async_store_view + : handle->backend->async_store.get(); + options.backend_label = handle->backend->backend_label; RangeRebuildStats rebuilt; Status status = rebuild_range(*source->tree, options.key_range, options, &handle->tree, &rebuilt); if (!status.ok()) { @@ -538,6 +594,13 @@ ct_status ct_rebuild_range(ct_tree *source, const ct_options *destination_option return static_cast<ct_status>(Code::kOk); } +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + void ct_init_logging(const char *log_dir, const char *level, size_t max_file_mb, size_t max_files, const char *file_prefix) { @@ -728,7 +791,7 @@ void ct_get_tree_summary(const ct_tree *t, ct_tree_summary *out) } char *ct_flush_metrics_str(ct_tree *t, double window_secs, const char *timestamp, size_t width) -{ +try { if (t == nullptr || timestamp == nullptr) { return nullptr; } @@ -744,10 +807,13 @@ char *ct_flush_metrics_str(ct_tree *t, double window_secs, const char *timestamp out[str.size()] = '\0'; return out; } +catch (...) { + return nullptr; +} char *ct_flush_metrics_str_ext(ct_tree *t, double window_secs, const char *timestamp, size_t width, size_t count_w, size_t tps_w) -{ +try { if (t == nullptr || timestamp == nullptr) { return nullptr; } @@ -763,6 +829,9 @@ char *ct_flush_metrics_str_ext(ct_tree *t, double window_secs, const char *times out[str.size()] = '\0'; return out; } +catch (...) { + return nullptr; +} void ct_negotiate_widths(const ct_tree *t, ct_column_widths input, ct_column_widths *out) { @@ -949,8 +1018,8 @@ try { if (t == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } - std::vector<Crowdbtree::external_op> external; - external.reserve(count); + std::vector<Crowdbtree::encoded_op> encoded; + encoded.reserve(count); for (uint64_t i = 0; i < count; ++i) { const ct_ext_op &op = ops[i]; if (op.key == nullptr && op.key_len != 0) { @@ -959,19 +1028,23 @@ try { if (op.kind != 0 && op.kind != 1) { return static_cast<ct_status>(Code::kInvalidArgument); } + if (op.kind == 0 && op.value == nullptr && op.value_len != 0) { + return static_cast<ct_status>(Code::kInvalidArgument); + } std::string key(reinterpret_cast<const char *>(op.key), op.key_len); - uint8_t flags = (op.kind == 1) ? kFlagTombstone : 0; - buffer value; - if (op.kind == 0) { - // Borrow the value bytes from Rust-owned memory; crowdb-tree calls - // drop_fn(bytes_ref) when this buffer is freed (drain/overwrite). - value = buffer::wrap_external(op.value, op.value_len, op.bytes_ref, op.drop_fn); + const auto kind = op.kind == 0 ? OpKind::kPut : OpKind::kDelete; + // This legacy ABI has no complete allocation identity or capacity. + // Copy only the requested value into an owned cell, then release the + // unknown backing allocation instead of retaining it at slice cost. + const Slice value(reinterpret_cast<const char *>(op.value), op.kind == 0 ? op.value_len : 0); + auto cell = encode_cell_buf(slot, kind, value); + unclaimed.next = i + 1; + if (op.kind == 0 && op.drop_fn != nullptr) { + op.drop_fn(op.bytes_ref); } - unclaimed.next = i + 1; - // Delete: value stays default (empty); flags = kFlagTombstone. - external.push_back({.key = std::move(key), .flags = flags, .value = std::move(value)}); + encoded.push_back({.key = std::move(key), .cell = std::move(cell)}); } - return to_status(t->tree->apply_external(slot, std::move(external))); + return to_status(t->tree->apply_encoded(slot, std::move(encoded))); } catch (const std::bad_alloc &) { return static_cast<ct_status>(Code::kResourceExhausted); @@ -1017,11 +1090,11 @@ catch (...) { ct_status ct_apply_put_owned(ct_tree *t, uint64_t slot, ct_write_handle *handle) try { + std::unique_ptr<ct_write_handle> owner(handle); if (t == nullptr || handle == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } - std::unique_ptr<ct_write_handle> owner(handle); - uint8_t *p = handle->cell.data(); + uint8_t *p = handle->cell.data(); for (int i = 0; i < 8; ++i) { p[i] = static_cast<uint8_t>((slot >> (8 * i)) & 0xff); } @@ -1121,49 +1194,30 @@ ct_status ct_release_split_memtable_view(ct_tree *t, uint64_t generation) return to_status(t->tree->release_split_memtable_view(generation)); } -ct_status ct_get(ct_tree *t, const uint8_t *key, size_t klen, int32_t *found, uint64_t *slot, ct_buf *value) -{ - if (t == nullptr || found == nullptr) { - return static_cast<ct_status>(Code::kInvalidArgument); - } - Status range_status = t->tree->validate_key(Slice(reinterpret_cast<const char *>(key), klen)); - if (!range_status.ok()) { - return to_status(range_status); - } - std::string v; - uint64_t s = 0; - bool ok = t->tree->get(Slice(reinterpret_cast<const char *>(key), klen), &s, &v); - *found = ok ? 1 : 0; - if (ok) { - if (slot != nullptr) { - *slot = s; - } - if (value != nullptr) { - *value = make_buf(v.data(), v.size()); - } - } - else if (value != nullptr) { - *value = make_buf(nullptr, 0); - } - return static_cast<ct_status>(Code::kOk); -} - // ── Async data path ─────────────────────────────────────────────── ct_future *ct_get_async(ct_tree *t, const uint8_t *key, size_t klen) -{ +try { if (t == nullptr) { return nullptr; } - auto impl = std::make_shared<ct_future_impl>(); + auto impl = make_tree_future(t); impl->kind = ct_future_impl::Kind::kGet; auto signal = t->completion; Status range_status = t->tree->validate_key(Slice(reinterpret_cast<const char *>(key), klen)); + if (auto *core = TreeCoreAccess::get(*t->tree); core != nullptr && range_status.ok()) { + try { + impl->result_controls = c_api_result_controls(core->binding()->account()); + } + catch (const std::bad_alloc &) { + range_status = Status::resource_exhausted("returned owner control reserve unavailable"); + } + } if (!range_status.ok()) { impl->status = to_status(range_status); impl->done.store(true, std::memory_order_release); signal->notify(); - return reinterpret_cast<ct_future *>(new ct_future_handle(std::move(impl))); + return reinterpret_cast<ct_future *>(make_ct_future_handle(std::move(impl))); } t->tree->get_async(Slice(reinterpret_cast<const char *>(key), klen), [impl, signal](Status status, GetView view) { impl->status = to_status(status); @@ -1171,15 +1225,18 @@ ct_future *ct_get_async(ct_tree *t, const uint8_t *key, size_t klen) impl->done.store(true, std::memory_order_release); signal->notify(); }); - return reinterpret_cast<ct_future *>(new ct_future_handle(std::move(impl))); + return reinterpret_cast<ct_future *>(make_ct_future_handle(std::move(impl))); +} +catch (...) { + return nullptr; } ct_future *ct_flush_async(ct_tree *t) -{ +try { if (t == nullptr) { return nullptr; } - auto impl = std::make_shared<ct_future_impl>(); + auto impl = make_tree_future(t); impl->kind = ct_future_impl::Kind::kFlush; auto signal = t->completion; t->tree->flush_async([impl, signal](const Status &st) { @@ -1187,15 +1244,18 @@ ct_future *ct_flush_async(ct_tree *t) impl->done.store(true, std::memory_order_release); signal->notify(); }); - return reinterpret_cast<ct_future *>(new ct_future_handle(std::move(impl))); + return reinterpret_cast<ct_future *>(make_ct_future_handle(std::move(impl))); +} +catch (...) { + return nullptr; } ct_future *ct_snapshot_async(ct_tree *t) -{ +try { if (t == nullptr) { return nullptr; } - auto impl = std::make_shared<ct_future_impl>(); + auto impl = make_tree_future(t); impl->kind = ct_future_impl::Kind::kSnapshot; auto signal = t->completion; t->tree->snapshot_async([impl, signal](const Status &st, uint64_t last_applied) { @@ -1204,7 +1264,10 @@ ct_future *ct_snapshot_async(ct_tree *t) impl->done.store(true, std::memory_order_release); signal->notify(); }); - return reinterpret_cast<ct_future *>(new ct_future_handle(std::move(impl))); + return reinterpret_cast<ct_future *>(make_ct_future_handle(std::move(impl))); +} +catch (...) { + return nullptr; } ct_future *ct_scan_async(ct_tree *t, const uint8_t *prefix, size_t plen, const uint8_t *start_after, size_t salen, @@ -1218,11 +1281,11 @@ ct_future *ct_scan_async(ct_tree *t, const uint8_t *prefix, size_t plen, const u ct_future *ct_scan_directional_async(ct_tree *t, const uint8_t *prefix, size_t plen, const uint8_t *start_after, size_t salen, const uint8_t *end_key, size_t elen, size_t limit, size_t byte_budget, int keys_only, uint64_t deadline_ms, int direction) -{ +try { if (t == nullptr) { return nullptr; } - auto impl = std::make_shared<ct_future_impl>(); + auto impl = make_tree_future(t); impl->kind = ct_future_impl::Kind::kScan; t->tree->scan_directional_async( Slice(reinterpret_cast<const char *>(prefix), plen), Slice(reinterpret_cast<const char *>(start_after), salen), @@ -1237,7 +1300,10 @@ ct_future *ct_scan_directional_async(ct_tree *t, const uint8_t *prefix, size_t p impl->done.store(true, std::memory_order_release); signal->notify(); }); - return reinterpret_cast<ct_future *>(new ct_future_handle(std::move(impl))); + return reinterpret_cast<ct_future *>(make_ct_future_handle(std::move(impl))); +} +catch (...) { + return nullptr; } ct_status ct_future_poll(ct_future *f, int32_t *done, int32_t *out_found, uint64_t *out_slot, ct_buf *out_value) @@ -1247,6 +1313,10 @@ ct_status ct_future_poll(ct_future *f, int32_t *done, int32_t *out_found, uint64 } auto *handle = reinterpret_cast<ct_future_handle *>(f); auto *impl = handle->get(); + if (impl->kind == ct_future_impl::Kind::kFlushReport || impl->kind == ct_future_impl::Kind::kCheckpointReport || + impl->kind == ct_future_impl::Kind::kChunkCollection || impl->kind == ct_future_impl::Kind::kCompaction) { + return static_cast<ct_status>(Code::kInvalidArgument); + } if (!impl->done.load(std::memory_order_acquire)) { *done = 0; return static_cast<ct_status>(Code::kOk); @@ -1297,8 +1367,14 @@ ct_status ct_future_poll(ct_future *f, int32_t *done, int32_t *out_found, uint64 // ct_buf; ct_free_buf's std::free correctly frees it. size_t sz = impl->scan_packed.size(); if (sz > 0) { - out_value->data = impl->scan_packed.release(); - out_value->len = sz; + try { + out_value->data = impl->scan_packed.release(); + out_value->len = sz; + } + catch (const std::bad_alloc &) { + *out_value = {}; + status = static_cast<ct_status>(Code::kResourceExhausted); + } } else { out_value->data = nullptr; @@ -1306,7 +1382,7 @@ ct_status ct_future_poll(ct_future *f, int32_t *done, int32_t *out_found, uint64 } } } - delete handle; // Flush/Snapshot/Scan: no borrowed state; free immediately. + free_ct_future_handle(handle); // Flush/Snapshot/Scan: no borrowed state; free immediately. return status; } @@ -1315,7 +1391,7 @@ void ct_future_free(ct_future *f) if (f == nullptr) { return; } - delete reinterpret_cast<ct_future_handle *>(f); + free_ct_future_handle(reinterpret_cast<ct_future_handle *>(f)); } size_t ct_uring_eventfds(const ct_tree *t, int32_t *out_fds, size_t max_fds) @@ -1328,10 +1404,7 @@ size_t ct_uring_eventfds(const ct_tree *t, int32_t *out_fds, size_t max_fds) out_fds[0] = t->completion->fd; } #ifdef CROWDB_HAVE_LIBURING - auto *uring = t->uring.get(); - if (uring == nullptr && t->injected_store != nullptr) { - uring = t->injected_store->uring.get(); - } + auto *uring = t->backend == nullptr ? nullptr : t->backend->uring.get(); if (uring != nullptr) { const size_t room = max_fds > total ? max_fds - total : 0; const size_t count = uring->eventfds(out_fds == nullptr ? nullptr : out_fds + total, room); @@ -1454,7 +1527,7 @@ static ct_status scan_with_budget(ct_tree *t, const uint8_t *prefix, size_t plen size_t elen, size_t limit, size_t byte_budget, int keys_only, uint64_t deadline_ms, int include_tombstones, ct_buf *out_entries, uint64_t *out_count, int32_t *truncated, bool strict_byte_budget) -{ +try { if (t == nullptr || out_entries == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } @@ -1488,6 +1561,12 @@ static ct_status scan_with_budget(ct_tree *t, const uint8_t *prefix, size_t plen } return static_cast<ct_status>(Code::kOk); } +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} ct_status ct_scan(ct_tree *t, const uint8_t *prefix, size_t plen, const uint8_t *start_key, size_t sklen, int has_start_bound, int start_inclusive, const uint8_t *end_key, size_t elen, size_t limit, @@ -1577,11 +1656,15 @@ try { if (t == nullptr || out == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } + *out = nullptr; auto v = std::make_unique<ct_view>(); v->snap = t->tree->snapshot_view(); *out = v.release(); return static_cast<ct_status>(Code::kOk); } +catch (const SnapshotReadError &error) { + return static_cast<ct_status>(error.code()); +} catch (const std::bad_alloc &) { return static_cast<ct_status>(Code::kResourceExhausted); } @@ -1595,32 +1678,48 @@ uint64_t ct_view_at_slot(const ct_view *v) } ct_status ct_view_iter(ct_view *v, ct_iter **out) -{ +try { if (v == nullptr || out == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } + *out = nullptr; auto it = std::make_unique<ct_iter>(); it->snap = v->snap; it->pos = 0; *out = it.release(); return static_cast<ct_status>(Code::kOk); } +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} ct_status ct_iter_next(ct_iter *it, ct_buf *key, uint64_t *slot, uint8_t *kind, ct_buf *value, int32_t *valid) -{ +try { if (it == nullptr || valid == nullptr) { return static_cast<ct_status>(Code::kInvalidArgument); } + *valid = 0; const auto &entries = it->snap->entries(); if (it->pos >= entries.size()) { *valid = 0; return static_cast<ct_status>(Code::kOk); } - const leaf_entry &e = entries[it->pos++]; + const leaf_entry &e = entries[it->pos]; CellView cv{Slice(e.cell)}; - *valid = 1; + auto returned_key = key != nullptr ? make_buf(e.key.data(), e.key.size()) : ct_buf{}; + const auto bytes = cv.value(); + auto returned_value = value != nullptr ? make_buf(bytes.data(), bytes.size()) : ct_buf{}; + if ((returned_key.len != 0 && returned_key.data == nullptr) || + (returned_value.len != 0 && returned_value.data == nullptr)) { + ct_free_buf(&returned_key); + ct_free_buf(&returned_value); + return static_cast<ct_status>(Code::kResourceExhausted); + } if (key != nullptr) { - *key = make_buf(e.key.data(), e.key.size()); + *key = returned_key; } if (slot != nullptr) { *slot = cv.slot(); @@ -1629,92 +1728,33 @@ ct_status ct_iter_next(ct_iter *it, ct_buf *key, uint64_t *slot, uint8_t *kind, *kind = cv.is_tombstone() ? 1 : 0; } if (value != nullptr) { - Slice val = cv.value(); - *value = make_buf(val.data(), val.size()); + *value = returned_value; } + ++it->pos; + *valid = 1; return static_cast<ct_status>(Code::kOk); } - -void ct_iter_release(ct_iter *it) -{ - delete it; -} - -void ct_view_release(ct_view *v) -{ - delete v; -} - -// ── Snapshot export / import ────────────────────────────────────── - -ct_status ct_snapshot_export_begin(ct_tree *t, size_t chunk_bytes, ct_export **out) -try { - if (t == nullptr || out == nullptr || chunk_bytes == 0) { - return static_cast<ct_status>(Code::kInvalidArgument); - } - auto e = std::make_unique<ct_export>(); - Status s = snapshot_export_begin(*t->tree, snapshot_format::kPortable, chunk_bytes, &e->exp); - if (!s.ok()) { - return to_status(s); - } - *out = e.release(); - return static_cast<ct_status>(Code::kOk); +catch (const SnapshotReadError &error) { + return static_cast<ct_status>(error.code()); } catch (const std::bad_alloc &) { return static_cast<ct_status>(Code::kResourceExhausted); } -catch (const std::exception &) { +catch (...) { return static_cast<ct_status>(Code::kInternal); } -uint64_t ct_snapshot_export_at_slot(const ct_export *e) -{ - return e == nullptr ? 0 : e->exp->at_slot(); -} - -uint64_t ct_snapshot_export_total_bytes(const ct_export *e) -{ - return e == nullptr ? 0 : static_cast<uint64_t>(e->exp->total_bytes()); -} - -uint32_t ct_snapshot_export_final_crc32c(const ct_export *e) -{ - return e == nullptr ? 0 : e->exp->final_crc32c(); -} - -size_t ct_snapshot_export_chunk_bytes(const ct_export *e) -{ - return e == nullptr ? 0 : e->exp->chunk_bytes(); -} - -uint64_t ct_snapshot_export_offset(const ct_export *e) +void ct_iter_release(ct_iter *it) { - return e == nullptr ? 0 : static_cast<uint64_t>(e->exp->offset()); + delete it; } -ct_status ct_snapshot_export_next(ct_export *e, uint64_t offset, ct_buf *chunk, int32_t *done) +void ct_view_release(ct_view *v) { - if (e == nullptr || chunk == nullptr || done == nullptr) { - return static_cast<ct_status>(Code::kInvalidArgument); - } - if (offset != static_cast<uint64_t>(e->exp->offset())) { - return static_cast<ct_status>(Code::kInvalidArgument); - } - std::string out; - bool d = false; - Status s = e->exp->next_chunk(&out, &d); - if (!s.ok()) { - return to_status(s); - } - *chunk = make_buf(out.data(), out.size()); - *done = d ? 1 : 0; - return static_cast<ct_status>(Code::kOk); + delete v; } -void ct_snapshot_export_end(ct_export *e) -{ - delete e; -} +// ── Snapshot export / import ────────────────────────────────────── ct_status ct_snapshot_import_begin(ct_tree *t, ct_import **out) { diff --git a/lib/crowdb-tree/src/c_api_apply.cpp b/lib/crowdb-tree/src/c_api_apply.cpp new file mode 100644 index 000000000..f6940823d --- /dev/null +++ b/lib/crowdb-tree/src/c_api_apply.cpp @@ -0,0 +1,224 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_apply_operation.h" +#include "btree/tree_core_access.h" +#include "btree/tree_core_batch.h" +#include "c_api_future_internal.h" +#include "c_api_internal.h" +#include "crowdb-tree/apply_c_api.h" + +using namespace crowdb::tree; + +namespace +{ +struct ApplyCall : std::enable_shared_from_this<ApplyCall> +{ + using Future = std::unique_ptr<ct_future_handle, decltype(&free_ct_future_handle)>; + + ApplyCall(std::shared_ptr<TreeApplyOperation> operation, std::shared_ptr<CompletionSignal> signal) + : metadata(operation->allocation_budget()), + operation(std::move(operation)), + result(make_ct_future(metadata)), + future(make_ct_future_handle(result), free_ct_future_handle), + signal(std::move(signal)) + { + result->kind = ct_future_impl::Kind::kApply; + } + + static void completed(void *context, detail::CallbackSignal signal, Status status) noexcept + { + auto &call = *static_cast<ApplyCall *>(context); + auto retained = std::move(call.self); + call.result->status = + static_cast<ct_status>(signal == detail::CallbackSignal::kStopped ? Code::kUnavailable : status.code()); + call.operation.reset(); + call.result->done.store(true, std::memory_order_release); + c_api_notify_completion(call.signal); + } + + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<TreeApplyOperation> operation; + ct_future_handle result; + Future future; + std::shared_ptr<CompletionSignal> signal; + std::shared_ptr<ApplyCall> self; + std::atomic_flag submitted = ATOMIC_FLAG_INIT; +}; + +struct ReplayInputOwner +{ + ReplayInputOwner(void (*release)(void *), void *argument) : drop(release), context(argument) + { + } + + ReplayInputOwner(const ReplayInputOwner &) = delete; + ReplayInputOwner &operator=(const ReplayInputOwner &) = delete; + + ~ReplayInputOwner() + { + if (drop != nullptr) { + drop(context); + } + } + + AllocationLease control; + void (*drop)(void *); + void *context; +}; + +using PermitHandle = std::shared_ptr<ApplyCall>; + +PermitHandle *handle(ct_apply_permit *permit) noexcept +{ + return reinterpret_cast<PermitHandle *>(permit); +} +} // namespace + +int32_t ct_apply_prepare(ct_tree *tree, const ct_kv_ref *operations, size_t count, int32_t gap_repair, + ct_apply_permit **out) +try { + if (tree == nullptr || out == nullptr || (count != 0 && operations == nullptr) || + (gap_repair != 0 && gap_repair != 1)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = nullptr; + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + for (size_t index = 0; index < count; ++index) { + const auto &input = operations[index]; + if ((input.key == nullptr && input.key_len != 0) || (input.value == nullptr && input.value_len != 0) || + input.kind > 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + } + constexpr size_t retained = detail::shared_allocation_bound<ApplyCall>() + + detail::shared_allocation_bound<ct_future_impl>() + sizeof(ct_future_handle) + + sizeof(PermitHandle); + std::shared_ptr<TreeApplyOperation> operation; + auto status = core->prepare_apply( + count, + [operations](size_t index) { + const auto &input = operations[index]; + return CoreWriteInput{.key = Slice(input.key, input.key_len), + .value = Slice(input.value, input.value_len), + .flags = input.kind}; + }, + gap_repair != 0 ? detail::BudgetClass::kGapRepair : detail::BudgetClass::kAdmittedWriter, retained, &operation); + if (!status.ok()) { + return static_cast<int32_t>(status.code()); + } + auto metadata = operation->allocation_budget(); + auto call = detail::reserved_shared<ApplyCall>(metadata, std::move(operation), c_api_completion_signal(tree)); + detail::ReservationAllocator<PermitHandle> allocator(metadata); + auto *result = allocator.allocate(1); + std::construct_at(result, std::move(call)); + *out = reinterpret_cast<ct_apply_permit *>(result); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} + +int32_t ct_apply_arm(ct_apply_permit *permit) +{ + if (permit == nullptr || !(*handle(permit))->operation) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + return static_cast<int32_t>((*handle(permit))->operation->arm().code()); +} + +int32_t ct_apply_submit(ct_apply_permit *permit, uint64_t slot, ct_future **out) +{ + if (permit == nullptr || out == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = nullptr; + auto call = *handle(permit); + if (call->submitted.test_and_set(std::memory_order_acq_rel)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + call->self = call; + *out = reinterpret_cast<ct_future *>(call->future.release()); + call->operation->start(slot, &ApplyCall::completed, call.get()); + return 0; +} + +void ct_apply_permit_free(ct_apply_permit *permit) +{ + if (permit != nullptr) { + auto *owned = handle(permit); + detail::ReservationAllocator<PermitHandle> allocator((*owned)->metadata); + std::destroy_at(owned); + allocator.deallocate(owned, 1); + } +} + +int32_t ct_apply_replay(ct_tree *tree, uint64_t slot, size_t count, ct_replay_read read, void (*drop_context)(void *), + void *context, size_t owner_control_bytes, ct_future **out) +try { + ReplayInputOwner incoming{drop_context, context}; + if (tree == nullptr || out == nullptr || drop_context == nullptr || read == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = nullptr; + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + for (size_t index = 0; index < count; ++index) { + ct_kv_ref input{}; + read(context, index, &input); + if ((input.key == nullptr && input.key_len != 0) || (input.value == nullptr && input.value_len != 0) || + input.kind > 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + } + const auto retained = detail::AllocationBound(detail::shared_allocation_bound<ApplyCall>()) + .add(detail::shared_allocation_bound<ReplayInputOwner>()) + .add(detail::shared_allocation_bound<ct_future_impl>()) + .add(sizeof(ct_future_handle)) + .add(owner_control_bytes) + .value(); + if (!retained) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + std::shared_ptr<TreeApplyOperation> operation; + auto status = core->prepare_replay_adopt( + slot, count, + [read, context](size_t index) { + ct_kv_ref input{}; + read(context, index, &input); + return CoreWriteInput{.key = Slice(input.key, input.key_len), + .value = Slice(input.value, input.value_len), + .flags = input.kind}; + }, + [&incoming, owner_control_bytes]( + const std::shared_ptr<detail::AllocationReservation> &budget) -> std::shared_ptr<const void> { + auto owner = detail::reserved_shared<ReplayInputOwner>(budget, incoming.drop, incoming.context); + incoming.drop = nullptr; + owner->control = AllocationLease(budget, owner_control_bytes, AllocationKind::kMetadata); + return owner; + }, + *retained, &operation); + if (!status.ok()) { + return static_cast<int32_t>(status.code()); + } + auto metadata = operation->allocation_budget(); + auto call = detail::reserved_shared<ApplyCall>(metadata, std::move(operation), c_api_completion_signal(tree)); + call->self = call; + *out = reinterpret_cast<ct_future *>(call->future.release()); + call->operation->start(slot, &ApplyCall::completed, call.get()); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} diff --git a/lib/crowdb-tree/src/c_api_chunk_collection.cpp b/lib/crowdb-tree/src/c_api_chunk_collection.cpp new file mode 100644 index 000000000..af566652c --- /dev/null +++ b/lib/crowdb-tree/src/c_api_chunk_collection.cpp @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_collector.h" +#include "c_api_maintenance_internal.h" + +namespace crowdb::tree::c_api_detail +{ +void store_report(ct_future_impl &result, const detail::ChunkCollectionProgress &report, + const Status &terminal) noexcept +{ + result.maintenance = ct_chunk_collection_result_v1{.struct_size = sizeof(ct_chunk_collection_result_v1), + .version = 1, + .terminal_status = static_cast<int32_t>(terminal.code()), + .more = report.more ? 1U : 0U, + .candidates_examined = report.candidates_examined, + .reference_windows = report.reference_windows, + .stages_examined = report.stages_examined, + .pins_examined = report.pins_examined, + .pins_released = report.pins_released, + .stages_removed = report.stages_removed, + .logical_capacity_bytes = report.logical_capacity_bytes}; +} +} // namespace crowdb::tree::c_api_detail + +ct_status ct_future_poll_chunk_collection(ct_future *future, int32_t *done, ct_chunk_collection_result_v1 *out) +{ + using namespace crowdb::tree; + if (future == nullptr || done == nullptr || out == nullptr || out->version != 1 || + out->struct_size != sizeof(*out)) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + auto *handle = reinterpret_cast<ct_future_handle *>(future); + const auto &result = **handle; + if (result.kind != ct_future_impl::Kind::kChunkCollection) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + if (!result.done.load(std::memory_order_acquire)) { + *done = 0; + return 0; + } + const auto *report = std::get_if<ct_chunk_collection_result_v1>(&result.maintenance); + if (report == nullptr) { + return static_cast<ct_status>(Code::kInternal); + } + *out = *report; + *done = 1; + free_ct_future_handle(handle); + return 0; +} diff --git a/lib/crowdb-tree/src/c_api_chunk_collection_start.cpp b/lib/crowdb-tree/src/c_api_chunk_collection_start.cpp new file mode 100644 index 000000000..cb76a8faf --- /dev/null +++ b/lib/crowdb-tree/src/c_api_chunk_collection_start.cpp @@ -0,0 +1,15 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_collection_operation.h" +#include "c_api_maintenance_start.h" +using namespace crowdb::tree; +using crowdb::tree::c_api_detail::begin_report; + +int32_t ct_chunk_collect_async(ct_tree *tree, ct_future **out) +{ + return begin_report<ChunkCollectionOperation>( + tree, out, ct_future_impl::Kind::kChunkCollection, + [](TreeCore &core, std::shared_ptr<ChunkCollectionOperation> *operation) { + return core.prepare_chunk_collection(operation); + }); +} diff --git a/lib/crowdb-tree/src/c_api_compaction.cpp b/lib/crowdb-tree/src/c_api_compaction.cpp new file mode 100644 index 000000000..cb9ef74a0 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_compaction.cpp @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/local_compaction_operation.h" +#include "c_api_maintenance_start.h" + +#include <chrono> + +using namespace crowdb::tree; + +int32_t ct_compact_sparse_blocks_async(ct_tree *tree, const ct_compaction_options_v1 *options, ct_future **out) +{ + const auto started = std::chrono::steady_clock::now(); + LocalCompactionOptions policy; + if (options != nullptr) { + if (options->version != 1 || options->struct_size != sizeof(*options)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + policy = {.free_threshold = options->free_threshold, .max_relocation_bytes = options->max_relocation_bytes}; + } + return c_api_detail::begin_report<LocalCompactionOperation>( + tree, out, ct_future_impl::Kind::kCompaction, + [policy, tree, started](TreeCore &core, std::shared_ptr<LocalCompactionOperation> *operation) { + auto status = core.prepare_compaction(policy, operation); + if (status.ok()) { + auto *facade = c_api_tree(tree); + (*operation)->observe_completion([facade, started](const MergeGcStats &report) { + const auto elapsed = std::chrono::steady_clock::now() - started; + TreeCoreAccess::record_compaction( + *facade, report, + static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(elapsed).count())); + }); + } + return status; + }, + false); +} + +namespace crowdb::tree::c_api_detail +{ +void store_report(ct_future_impl &result, const MergeGcStats &report, const Status &terminal) noexcept +{ + result.maintenance = ct_compaction_result_v1{.struct_size = sizeof(ct_compaction_result_v1), + .version = 1, + .terminal_status = static_cast<int32_t>(terminal.code()), + .blocks_selected = report.blocks_selected, + .pages_relocated = report.pages_relocated, + .bytes_relocated = report.bytes_relocated, + .blocks_deleted = report.blocks_deleted}; +} +} // namespace crowdb::tree::c_api_detail + +int32_t ct_future_poll_compaction(ct_future *future, int32_t *done, ct_compaction_result_v1 *out) +{ + if (future == nullptr || done == nullptr || out == nullptr || out->version != 1 || + out->struct_size != sizeof(*out)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + auto *handle = reinterpret_cast<ct_future_handle *>(future); + const auto &result = **handle; + if (result.kind != ct_future_impl::Kind::kCompaction) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + if (!result.done.load(std::memory_order_acquire)) { + *done = 0; + return 0; + } + const auto *report = std::get_if<ct_compaction_result_v1>(&result.maintenance); + if (report == nullptr) { + return static_cast<int32_t>(Code::kInternal); + } + *out = *report; + *done = 1; + free_ct_future_handle(handle); + return 0; +} diff --git a/lib/crowdb-tree/src/c_api_future_internal.h b/lib/crowdb-tree/src/c_api_future_internal.h new file mode 100644 index 000000000..507ab6627 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_future_internal.h @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/btree/read_result.h" +#include "crowdb-tree/btree/scan_packed.h" +#include "crowdb-tree/c_api.h" +#include "crowdb-tree/chunk_runtime_c_api.h" +#include "crowdb-tree/maintenance_c_api.h" +#include "crowdb-tree/status.h" + +#include <atomic> +#include <memory> +#include <variant> + +struct ct_future_impl +{ + enum class Kind : std::uint8_t { + kGet, + kFlush, + kSnapshot, + kScan, + kApply, + kFlushReport, + kCheckpointReport, + kCompaction, + kChunkCollection + }; + Kind kind = Kind::kGet; + std::shared_ptr<crowdb::tree::AllocationBudget> metadata; + std::shared_ptr<crowdb::tree::AllocationBudget> result_controls; + // Written once by the completion callback (release), read once by the + // first ct_future_poll that observes it (acquire) -- that acquire/ + // release pair is the only synchronization the fields below need, + // since there is exactly one writer-then-reader handoff per future. + std::atomic<bool> done{false}; + ct_status status = static_cast<ct_status>(crowdb::tree::Code::kOk); + // kGet only. Retains the immutable allocation backing the borrowed bytes. + // ct_future_poll deliberately does *not* delete the handle for a kGet + // future (see its updated doc comment in c_api.h); the caller must + // always follow up with ct_future_free once done reading out_value. + crowdb::tree::GetView get_result; + uint64_t slot = 0; // kSnapshot only (last_applied_slot) + // kScan only: packed record buffer (same format ct_scan produces), + // received directly from the engine via ScanPackedBuf (R57: zero-copy + // staging — no re-pack loop, no make_buf). Ownership is transferred + // to the caller via release() in ct_future_poll. + crowdb::tree::ScanPackedBuf scan_packed; + uint64_t scan_count = 0; + bool scan_truncated = false; + std::variant<std::monostate, ct_flush_result_v1, ct_checkpoint_result_v1, ct_chunk_collection_result_v1, + ct_compaction_result_v1> + maintenance; +}; + +using ct_future_handle = std::shared_ptr<ct_future_impl>; + +inline std::shared_ptr<ct_future_impl> make_ct_future(std::shared_ptr<crowdb::tree::AllocationBudget> metadata) +{ + auto value = std::allocate_shared<ct_future_impl>(crowdb::tree::MetadataAllocator<ct_future_impl>(metadata)); + value->metadata = std::move(metadata); + return value; +} + +inline ct_future_handle *make_ct_future_handle(std::shared_ptr<ct_future_impl> value) +{ + crowdb::tree::MetadataAllocator<ct_future_handle> allocator(value->metadata); + auto *handle = allocator.allocate(1); + std::construct_at(handle, std::move(value)); + return handle; +} + +inline void free_ct_future_handle(ct_future_handle *handle) noexcept +{ + crowdb::tree::MetadataAllocator<ct_future_handle> allocator((*handle)->metadata); + std::destroy_at(handle); + allocator.deallocate(handle, 1); +} diff --git a/lib/crowdb-tree/src/c_api_get.cpp b/lib/crowdb-tree/src/c_api_get.cpp new file mode 100644 index 000000000..3a36c65f2 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_get.cpp @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "c_api_internal.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <cstdlib> +#include <cstring> + +using namespace crowdb::tree; + +ct_status ct_get(ct_tree *t, const uint8_t *key, size_t klen, int32_t *found, uint64_t *slot, ct_buf *value) +try { + if (t == nullptr || found == nullptr || (key == nullptr && klen != 0)) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *found = 0; + if (slot != nullptr) { + *slot = 0; + } + if (value != nullptr) { + *value = {}; + } + auto *tree = c_api_tree(t); + Status range_status = tree->validate_key(Slice(reinterpret_cast<const char *>(key), klen)); + if (!range_status.ok()) { + return static_cast<ct_status>(range_status.code()); + } + GetView result; + const auto status = tree->get_view(Slice(reinterpret_cast<const char *>(key), klen), &result); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + if (!result.found()) { + return static_cast<ct_status>(Code::kOk); + } + if (value != nullptr) { + const auto bytes = result.value(); + if (!bytes.empty()) { + auto *copied = static_cast<uint8_t *>(std::malloc(bytes.size())); + if (copied == nullptr) { + return static_cast<ct_status>(Code::kResourceExhausted); + } + std::memcpy(copied, bytes.data(), bytes.size()); + *value = {.data = copied, .len = bytes.size()}; + } + } + if (slot != nullptr) { + *slot = result.slot(); + } + *found = 1; + return static_cast<ct_status>(Code::kOk); +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} diff --git a/lib/crowdb-tree/src/c_api_immutable_snapshot.cpp b/lib/crowdb-tree/src/c_api_immutable_snapshot.cpp new file mode 100644 index 000000000..cdd9ec7a9 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_immutable_snapshot.cpp @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/runtime_range_rebuild.h" +#include "btree/tree_core_access.h" +#include "c_api_future_internal.h" +#include "c_api_immutable_snapshot_internal.h" +#include "c_api_internal.h" +#include "c_api_owned_budget.h" + +using namespace crowdb::tree; + +ct_status ct_immutable_snapshot_capture(ct_tree *tree, uint64_t expected_sequence, uint64_t expected_coverage, + ct_immutable_snapshot **out) +try { + if (tree == nullptr || out == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *out = nullptr; + auto *facade = c_api_tree(tree); + auto *core = TreeCoreAccess::get(*facade); + if (core == nullptr) { + return static_cast<ct_status>(Code::kNotSupported); + } + const auto matches = [&] { + return expected_sequence == 0 || (facade->durable_snapshot_seq() == expected_sequence && + facade->durable_snapshot_last_applied_slot() == expected_coverage); + }; + if (!matches()) { + return static_cast<ct_status>(Code::kUnavailable); + } + std::shared_ptr<PublishedSnapshotContext> context; + auto status = core->capture_snapshot(&context); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + if (!matches() || context->view()->coverage != expected_coverage || !context->detached()) { + return static_cast<ct_status>(Code::kUnavailable); + } + *out = new ct_immutable_snapshot(std::move(context)); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +void ct_immutable_snapshot_free(ct_immutable_snapshot *snapshot) +{ + delete snapshot; +} + +ct_status ct_immutable_snapshot_get(ct_immutable_snapshot *snapshot, const uint8_t *key, size_t size, ct_future **out) +try { + if (snapshot == nullptr || out == nullptr || (key == nullptr && size != 0)) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *out = nullptr; + const auto &context = snapshot->context; + auto value = make_ct_future(context->binding()->lifecycle_budget()); + value->result_controls = c_api_result_controls(context->binding()->account()); + auto status = context->get(Slice(key, size), &value->get_result); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + value->done.store(true, std::memory_order_release); + *out = reinterpret_cast<ct_future *>(make_ct_future_handle(std::move(value))); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +ct_status ct_immutable_snapshot_rebuild(ct_immutable_snapshot *snapshot, ct_tree *destination, + ct_range_rebuild_stats *stats) +try { + if (snapshot == nullptr || destination == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + auto *tree = c_api_tree(destination); + auto *core = TreeCoreAccess::get(*tree); + if (core == nullptr) { + return static_cast<ct_status>(Code::kNotSupported); + } + RuntimeRangeImport staging; + auto status = copy_snapshot_range(snapshot->context, *core, core->config().key_range, staging); + if (status.ok()) { + status = core->install_readable(std::move(staging.catalog), staging.credit); + } + if (status.ok()) { + status = tree->snapshot(); + } + if (status.ok() && stats != nullptr) { + const auto &value = staging.stats; + *stats = {.entries_examined = value.entries_examined, + .entries_emitted = value.entries_emitted, + .entries_filtered = value.entries_filtered, + .pages_reused = value.pages_reused, + .pages_rebuilt = value.pages_rebuilt, + .subtrees_skipped = value.subtrees_skipped}; + } + return static_cast<ct_status>(status.code()); +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} diff --git a/lib/crowdb-tree/src/c_api_immutable_snapshot_internal.h b/lib/crowdb-tree/src/c_api_immutable_snapshot_internal.h new file mode 100644 index 000000000..a79516e4f --- /dev/null +++ b/lib/crowdb-tree/src/c_api_immutable_snapshot_internal.h @@ -0,0 +1,18 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/immutable_snapshot_c_api.h" +#include "snapshot/published_snapshot.h" + +struct ct_immutable_snapshot +{ + explicit ct_immutable_snapshot(std::shared_ptr<crowdb::tree::PublishedSnapshotContext> value) + : allocation(value->binding()->lifecycle_budget(), sizeof(ct_immutable_snapshot), + crowdb::tree::AllocationKind::kMetadata), + context(std::move(value)) + { + } + + crowdb::tree::AllocationLease allocation; + std::shared_ptr<crowdb::tree::PublishedSnapshotContext> context; +}; diff --git a/lib/crowdb-tree/src/c_api_immutable_snapshot_scan.cpp b/lib/crowdb-tree/src/c_api_immutable_snapshot_scan.cpp new file mode 100644 index 000000000..b2a6bed02 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_immutable_snapshot_scan.cpp @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "c_api_future_internal.h" +#include "c_api_immutable_snapshot_internal.h" +#include "c_api_owned_budget.h" + +using namespace crowdb::tree; + +ct_status ct_immutable_snapshot_scan(ct_immutable_snapshot *snapshot, const ct_scan_options_v2 *options, + ct_scan_result **out) +try { + if (snapshot == nullptr || options == nullptr || options->struct_size != sizeof(*options) || out == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *out = nullptr; + const auto &o = *options; + if ((o.prefix == nullptr && o.prefix_len != 0) || (o.start == nullptr && o.start_len != 0) || + (o.end == nullptr && o.end_len != 0) || (o.begin == nullptr && o.begin_len != 0)) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + CoreScanOptions scan{.prefix = Slice(o.prefix, o.prefix_len), + .start = Slice(o.start, o.start_len), + .end = Slice(o.end, o.end_len), + .begin = Slice(o.begin, o.begin_len), + .limit = o.limit, + .byte_budget = o.byte_budget, + .deadline_ms = o.deadline_ms, + .reverse = o.reverse != 0, + .keys_only = o.keys_only != 0, + .include_tombstones = o.include_tombstones != 0, + .has_start_bound = o.has_start_bound != 0, + .start_inclusive = o.start_inclusive != 0, + .strict_byte_budget = o.strict_byte_budget != 0}; + const auto &context = snapshot->context; + auto value = make_ct_future(context->binding()->lifecycle_budget()); + value->kind = ct_future_impl::Kind::kScan; + value->result_controls = c_api_result_controls(context->binding()->account()); + size_t count = 0; + auto status = context->scan(scan, &value->scan_packed, &count, &value->scan_truncated); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + value->scan_count = count; + value->done.store(true, std::memory_order_release); + *out = reinterpret_cast<ct_scan_result *>(make_ct_future_handle(std::move(value))); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} diff --git a/lib/crowdb-tree/src/c_api_internal.h b/lib/crowdb-tree/src/c_api_internal.h index c306c4c0a..9c514deb5 100644 --- a/lib/crowdb-tree/src/c_api_internal.h +++ b/lib/crowdb-tree/src/c_api_internal.h @@ -6,6 +6,7 @@ #include "crowdb-tree/backend/async_page_store.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/c_api.h" +#include "crowdb-tree/memory_budget.h" #ifdef CROWDB_HAVE_LIBURING # include "crowdb-common/diskio_uring.h" @@ -16,16 +17,43 @@ struct PageStoreBundle { - std::unique_ptr<crowdb::tree::PageStore> store; - std::unique_ptr<crowdb::tree::AsyncPageStore> async_store; - crowdb::tree::AsyncPageStore *async_store_view = nullptr; + std::unique_ptr<crowdb::tree::PageStore> store; #ifdef CROWDB_HAVE_LIBURING + // Declare uring before async_store so reverse destruction order releases + // the non-owning async wrapper before its reactor. std::unique_ptr<crowdb::common::DiskIOUring> uring; #endif - std::string backend_label; + std::unique_ptr<crowdb::tree::AsyncPageStore> async_store; + crowdb::tree::AsyncPageStore *async_store_view = nullptr; + std::string backend_label; }; struct ct_page_store { + crowdb::tree::AllocationLease allocation; std::shared_ptr<PageStoreBundle> bundle; }; + +namespace crowdb::tree +{ +class Crowdbtree; +struct Config; +} // namespace crowdb::tree +struct ct_runtime; +struct ct_tree_runtime_options; +crowdb::tree::Crowdbtree *c_api_tree(ct_tree *tree) noexcept; +crowdb::tree::PageStore *c_api_page_store(ct_tree *tree) noexcept; +struct CompletionSignal; +std::shared_ptr<CompletionSignal> c_api_completion_signal(ct_tree *tree) noexcept; +void c_api_notify_completion(const std::shared_ptr<CompletionSignal> &signal) noexcept; +// Backend-specific entry points supply their implementation so ordinary open +// keeps no link-time dependency on an optional migration backend. +using CApiMigration = crowdb::tree::Status (*)(const crowdb::tree::Config &, uint64_t, + std::unique_ptr<crowdb::tree::Crowdbtree> *); +ct_status ct_migrate_configured(const ct_options *opt, ct_runtime *runtime, const ct_tree_runtime_options *binding, + uint64_t generation, CApiMigration migrate, ct_tree **out); +ct_status ct_open_configured(const ct_options *options, ct_runtime *runtime, const ct_tree_runtime_options *binding, + ct_tree **out); + +ct_status ct_rebuild_configured(ct_tree *source, const ct_options *destination_options, crowdb::tree::Config options, + ct_tree **out, ct_range_rebuild_stats *stats); diff --git a/lib/crowdb-tree/src/c_api_maintenance_internal.h b/lib/crowdb-tree/src/c_api_maintenance_internal.h new file mode 100644 index 000000000..1f25e6448 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_maintenance_internal.h @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "c_api_future_internal.h" +#include "c_api_internal.h" +#include "runtime/tree_operation_lifetime.h" +#include "snapshot/checkpoint_operation.h" +#include "stdexec_adapter.h" + +#include <type_traits> + +namespace crowdb::tree::detail +{ +struct ChunkCollectionProgress; +} + +namespace crowdb::tree::c_api_detail +{ +void store_report(ct_future_impl &result, const detail::ChunkCollectionProgress &report, + const Status &terminal) noexcept; +void store_report(ct_future_impl &result, const MergeGcStats &report, const Status &terminal) noexcept; +void store_report(ct_future_impl &result, const FlushResult &report, const Status &terminal) noexcept; +void store_report(ct_future_impl &result, const CheckpointResult &report, const Status &terminal) noexcept; + +template <class Operation> struct MaintenanceCall : std::enable_shared_from_this<MaintenanceCall<Operation>> +{ + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + MaintenanceCall *owner; + + void set_value() const noexcept + { + owner->finish(Status::Ok()); + } + + void set_value(FlushResult report) const noexcept + { + owner->finish_report(Status::Ok(), std::move(report)); + } + + void set_error(Status status) const noexcept + { + owner->finish(std::move(status)); + } + + void set_stopped() const noexcept + { + owner->finish(Status::unavailable("maintenance stopped")); + } + }; + + using Sender = + std::conditional_t<std::is_same_v<Operation, FlushOperation>, FlushReportSender, detail::CallbackSender>; + using Connected = stdexec::connect_result_t<Sender, Receiver>; + + static Sender sender(Operation &operation) noexcept + { + if constexpr (std::is_same_v<Operation, FlushOperation>) { + return operation.report_sender(); + } + else { + return operation.sender(); + } + } + + MaintenanceCall(std::shared_ptr<TreeOperationLifetime::Ticket> lifetime, std::shared_ptr<Operation> operation, + ct_future_handle result, std::shared_ptr<CompletionSignal> completion) + : ticket(std::move(lifetime)), + operation(std::move(operation)), + result(std::move(result)), + signal(std::move(completion)), + connected(stdexec::connect(sender(*this->operation), Receiver{this})) + { + } + + void start() noexcept + { + self = this->shared_from_this(); + stdexec::start(connected); + } + + template <class Report> void finish_report(Status status, const Report &report) noexcept + { + auto retained = std::move(self); + store_report(*result, report, status); + result->done.store(true, std::memory_order_release); + c_api_notify_completion(signal); + } + + void finish(Status status) noexcept + { + if constexpr (std::is_same_v<Operation, FlushOperation>) { + // A rejected concurrent connection owns no progress and must not + // observe the report being mutated by the accepted attempt. + FlushResult rejected; + rejected.handoff_status = status.without_message(); + finish_report(std::move(status), std::move(rejected)); + } + else { + finish_report(std::move(status), operation->result()); + } + } + + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<Operation> operation; + ct_future_handle result; + std::shared_ptr<CompletionSignal> signal; + Connected connected; + std::shared_ptr<MaintenanceCall> self; +}; +} // namespace crowdb::tree::c_api_detail diff --git a/lib/crowdb-tree/src/c_api_maintenance_result.cpp b/lib/crowdb-tree/src/c_api_maintenance_result.cpp new file mode 100644 index 000000000..badb6d31e --- /dev/null +++ b/lib/crowdb-tree/src/c_api_maintenance_result.cpp @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "c_api_maintenance_internal.h" +#include "crowdb-tree/btree/tree.h" + +using namespace crowdb::tree; + +uint64_t ct_durable_coverage(ct_tree *tree) +{ + return tree == nullptr ? 0 : c_api_tree(tree)->durable_snapshot_last_applied_slot(); +} + +namespace crowdb::tree::c_api_detail +{ +void store_report(ct_future_impl &result, const FlushResult &report, const Status &terminal) noexcept +{ + auto status = terminal.code(); + if (terminal.ok()) { + status = report.handoff_status.ok() ? report.maintenance_status.code() : report.handoff_status.code(); + } + const bool covered = report.phase == FlushPhase::kCovered || report.phase == FlushPhase::kMaintenance || + report.phase == FlushPhase::kComplete; + result.maintenance = + ct_flush_result_v1{.struct_size = sizeof(ct_flush_result_v1), + .version = 1, + .terminal_status = static_cast<int32_t>(status), + .handoff_status = static_cast<int32_t>(report.handoff_status.code()), + .maintenance_status = static_cast<int32_t>(report.maintenance_status.code()), + .phase = static_cast<uint32_t>(report.phase), + .batch_id = report.batch_id, + .floor = report.floor, + .frontier = report.frontier, + .published_coverage = report.published_coverage, + .deferred_pages = report.deferred_pages, + .handoff_completed = report.handoff_status.ok() && covered ? 1 : 0, + .handoff_advanced = report.handoff_advanced ? 1 : 0}; +} + +void store_report(ct_future_impl &result, const CheckpointResult &report, const Status &terminal) noexcept +{ + const auto &status = terminal.ok() ? report.status : terminal; + result.maintenance = ct_checkpoint_result_v1{.struct_size = sizeof(ct_checkpoint_result_v1), + .version = 1, + .terminal_status = static_cast<int32_t>(status.code()), + .durable_committed = report.durable_committed ? 1 : 0, + .writeback_completed = report.writeback_completed ? 1 : 0, + .durable_coverage = report.durable_coverage, + .snapshot_sequence = report.snapshot_sequence, + .checkpoint_token = report.checkpoint_token, + .captured_coverage = report.captured_coverage, + .primary_frames = report.primary_frames, + .written_frames = report.written_frames, + .reused_frames = report.reused_frames, + .rebased_pages = report.rebased_pages, + .rebase_deferred = report.rebase_deferred}; +} +} // namespace crowdb::tree::c_api_detail + +namespace +{ +template <class Report> int32_t poll_report(ct_future *future, int32_t *done, Report *out, ct_future_impl::Kind kind) +{ + if (future == nullptr || done == nullptr || out == nullptr || out->struct_size != sizeof(*out) || + out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + auto *handle = reinterpret_cast<ct_future_handle *>(future); + const auto &result = **handle; + if (result.kind != kind) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + if (!result.done.load(std::memory_order_acquire)) { + *done = 0; + return 0; + } + const auto *report = std::get_if<Report>(&result.maintenance); + if (report == nullptr) { + return static_cast<int32_t>(Code::kInternal); + } + *out = *report; + *done = 1; + free_ct_future_handle(handle); + return 0; +} +} // namespace + +int32_t ct_future_poll_flush_report(ct_future *future, int32_t *done, ct_flush_result_v1 *out) +{ + return poll_report(future, done, out, ct_future_impl::Kind::kFlushReport); +} + +int32_t ct_future_poll_checkpoint_report(ct_future *future, int32_t *done, ct_checkpoint_result_v1 *out) +{ + return poll_report(future, done, out, ct_future_impl::Kind::kCheckpointReport); +} diff --git a/lib/crowdb-tree/src/c_api_maintenance_start.cpp b/lib/crowdb-tree/src/c_api_maintenance_start.cpp new file mode 100644 index 000000000..62fcbad20 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_maintenance_start.cpp @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "c_api_maintenance_start.h" + +#include "btree/tree_core_checkpoint_internal.h" +using namespace crowdb::tree; +using crowdb::tree::c_api_detail::begin_report; + +int32_t ct_flush_report_async(ct_tree *tree, ct_future **out) +{ + return begin_report<FlushOperation>( + tree, out, ct_future_impl::Kind::kFlushReport, + [](TreeCore &core, std::shared_ptr<FlushOperation> *operation) { return core.prepare_flush(operation); }); +} + +int32_t ct_writeback_async(ct_tree *tree, ct_future **out) +{ + return begin_report<CheckpointOperation>(tree, out, ct_future_impl::Kind::kCheckpointReport, + [](TreeCore &core, std::shared_ptr<CheckpointOperation> *operation) { + return core.prepare_writeback(operation); + }); +} + +int32_t ct_checkpoint_report_async(ct_tree *tree, ct_future **out) +{ + return begin_report<CheckpointOperation>(tree, out, ct_future_impl::Kind::kCheckpointReport, + [](TreeCore &core, std::shared_ptr<CheckpointOperation> *operation) { + return core.prepare_checkpoint(checkpoint_detail::defaults(core), + operation); + }); +} diff --git a/lib/crowdb-tree/src/c_api_maintenance_start.h b/lib/crowdb-tree/src/c_api_maintenance_start.h new file mode 100644 index 000000000..9c01f4420 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_maintenance_start.h @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/tree_core_access.h" +#include "c_api_maintenance_internal.h" + +namespace crowdb::tree::c_api_detail +{ +template <class Operation, class Prepare> +int32_t begin_report(ct_tree *tree, ct_future **out, ct_future_impl::Kind kind, Prepare prepare, + bool retain_ticket = true) +try { + if (tree == nullptr || out == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = nullptr; + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + auto metadata = core->binding()->lifecycle_budget(); + auto result = make_ct_future(metadata); + result->kind = kind; + std::unique_ptr<ct_future_handle, decltype(&free_ct_future_handle)> handle(make_ct_future_handle(result), + free_ct_future_handle); + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + Status status; + if (retain_ticket) { + status = core->operation_lifetime().acquire(metadata, &ticket); + if (!status.ok()) { + return static_cast<int32_t>(status.code()); + } + } + std::shared_ptr<Operation> operation; + status = prepare(*core, &operation); + if (!status.ok()) { + return static_cast<int32_t>(status.code()); + } + using Call = c_api_detail::MaintenanceCall<Operation>; + auto call = std::allocate_shared<Call>(MetadataAllocator<Call>(metadata), std::move(ticket), std::move(operation), + result, c_api_completion_signal(tree)); + call->start(); + *out = reinterpret_cast<ct_future *>(handle.release()); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} +} // namespace crowdb::tree::c_api_detail diff --git a/lib/crowdb-tree/src/c_api_owned_allocation.cpp b/lib/crowdb-tree/src/c_api_owned_allocation.cpp new file mode 100644 index 000000000..d3b093332 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_owned_allocation.cpp @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "c_api_owned_allocation.h" + +#include "btree/allocation_account.h" +#include "btree/tree_core_access.h" +#include "c_api_internal.h" + +namespace +{ +using namespace crowdb::tree; + +// The foreign control charge follows the payload owner, not the independently +// releasable C handle. Its destructor frees the foreign object before +// refunding. +struct ForeignOwner +{ + ForeignOwner(std::shared_ptr<AllocationBudget> credit, size_t bytes, ExternalAllocation::Owner value) + : budget(std::move(credit)), + control(budget, bytes, AllocationKind::kMetadata), + context(std::move(value)) + { + } + + std::shared_ptr<AllocationBudget> budget; + AllocationLease control; + ExternalAllocation::Owner context; +}; + +void free_foreign(void *context) +{ + auto *owner = static_cast<ForeignOwner *>(context); + MetadataAllocator<ForeignOwner> allocator(owner->budget); + std::destroy_at(owner); + allocator.deallocate(owner, 1); +} + +ExternalAllocation::Owner retain_foreign(const std::shared_ptr<AllocationBudget> &budget, size_t bytes, + ExternalAllocation::Owner context) +{ + MetadataAllocator<ForeignOwner> allocator(budget); + auto *owner = allocator.allocate(1); + try { + std::construct_at(owner, budget, bytes, std::move(context)); + } + catch (...) { + allocator.deallocate(owner, 1); + throw; + } + ExternalAllocation::Owner result(owner, free_foreign); + return result; +} +} // namespace + +extern "C" ct_status ct_owned_allocation_create(ct_tree *tree, const uint8_t *data, size_t length, size_t capacity, + void (*drop_context)(void *), void *context, size_t control_bytes, + ct_owned_allocation **out) +try { + // A missing deleter is invalid and cannot consume an arbitrary pointer. + if (drop_context == nullptr) { + return static_cast<int>(Code::kInvalidArgument); + } + ExternalAllocation::Owner foreign(context, drop_context); + if (out == nullptr) { + return static_cast<int>(Code::kInvalidArgument); + } + *out = nullptr; + if (context == nullptr || length > capacity || (capacity != 0 && data == nullptr)) { + return static_cast<int>(Code::kInvalidArgument); + } + *out = nullptr; + auto *facade = c_api_tree(tree); + auto *core = facade == nullptr ? nullptr : TreeCoreAccess::get(*facade); + if (core == nullptr || core->closed()) { + return static_cast<int>(Code::kUnavailable); + } + std::shared_ptr<detail::AllocationAccount> budget; + auto status = + detail::AllocationAccount::create(core->binding()->account(), detail::BudgetClass::kAdmittedWriter, &budget); + if (!status.ok()) { + return static_cast<int>(status.code()); + } + auto retained = retain_foreign(budget, control_bytes, std::move(foreign)); + std::shared_ptr<const ExternalOwnerBridge> bridge; + status = ExternalOwnerBridge::create(data, length, capacity, std::move(retained), budget, &bridge); + if (!status.ok()) { + return static_cast<int>(status.code()); + } + MetadataAllocator<ct_owned_allocation> allocator(budget); + auto *result = allocator.allocate(1); + std::construct_at(result, std::move(budget), std::move(bridge)); + *out = result; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int>(crowdb::tree::Code::kResourceExhausted); +} +catch (...) { + return static_cast<int>(crowdb::tree::Code::kInternal); +} + +extern "C" void ct_owned_allocation_retain(ct_owned_allocation *owner) +{ + if (owner != nullptr && owner->references.fetch_add(1, std::memory_order_relaxed) == SIZE_MAX) { + std::terminate(); + } +} + +extern "C" void ct_owned_allocation_free(ct_owned_allocation *owner) +{ + if (owner == nullptr || owner->references.fetch_sub(1, std::memory_order_acq_rel) != 1) { + return; + } + crowdb::tree::MetadataAllocator<ct_owned_allocation> allocator(owner->budget); + std::destroy_at(owner); + allocator.deallocate(owner, 1); +} diff --git a/lib/crowdb-tree/src/c_api_owned_allocation.h b/lib/crowdb-tree/src/c_api_owned_allocation.h new file mode 100644 index 000000000..bf5dc1e09 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_owned_allocation.h @@ -0,0 +1,20 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/owned_allocation_c_api.h" +#include "external_owner_bridge.h" + +struct ct_owned_allocation +{ + ct_owned_allocation(std::shared_ptr<crowdb::tree::AllocationBudget> credit, + std::shared_ptr<const crowdb::tree::ExternalOwnerBridge> value) + : budget(std::move(credit)), + bridge(std::move(value)) + { + } + + std::atomic<size_t> references{1}; + std::shared_ptr<crowdb::tree::AllocationBudget> budget; + std::shared_ptr<const crowdb::tree::ExternalOwnerBridge> bridge; +}; diff --git a/lib/crowdb-tree/src/c_api_owned_batch.cpp b/lib/crowdb-tree/src/c_api_owned_batch.cpp new file mode 100644 index 000000000..a82647541 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_owned_batch.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_account.h" +#include "btree/tree_core_access.h" +#include "btree/tree_core_batch.h" +#include "c_api_internal.h" +#include "c_api_owned_allocation.h" + +namespace +{ +using namespace crowdb::tree; +using Values = std::vector<buffer, MetadataAllocator<buffer>>; +using OwnedReader = detail::InlineFunction<ct_owned_kv_ref(size_t)>; + +Status stage_values(const OwnedReader &read, Values &values) +{ + for (size_t index = 0; index < values.size(); ++index) { + const auto input = read(index); + if ((input.key_len != 0 && input.key == nullptr) || input.kind > 1) { + return Status::invalid_argument(); + } + if (input.kind == 1) { + if (input.owner != nullptr || input.offset != 0 || input.length != 0) { + return Status::invalid_argument(); + } + continue; + } + if (input.owner == nullptr) { + return Status::invalid_argument(); + } + auto status = input.owner->bridge->slice(input.offset, input.length, &values[index]); + if (!status.ok()) { + return status; + } + } + return {}; +} +ct_status apply_owned(ct_tree *tree, uint64_t slot, size_t count, const OwnedReader &read) +try { + auto *facade = c_api_tree(tree); + auto *core = facade == nullptr ? nullptr : TreeCoreAccess::get(*facade); + if (core == nullptr) { + return static_cast<int>(Code::kUnavailable); + } + std::shared_ptr<detail::AllocationAccount> budget; + auto status = + detail::AllocationAccount::create(core->binding()->account(), detail::BudgetClass::kAdmittedWriter, &budget); + if (!status.ok()) { + return static_cast<int>(status.code()); + } + Values values(count, MetadataAllocator<buffer>(budget)); + status = stage_values(read, values); + if (!status.ok()) { + return static_cast<int>(status.code()); + } + status = core->apply_external_inputs(slot, count, [&](size_t index) { + const auto input = read(index); + return CoreWriteInput{.key = Slice(input.key, input.key_len), + .value = values[index].slice(), + .movable = &values[index], + .flags = input.kind, + .encoded = false}; + }); + return static_cast<int>(status.code()); +} +catch (const std::bad_alloc &) { + return static_cast<int>(crowdb::tree::Code::kResourceExhausted); +} +catch (...) { + return static_cast<int>(crowdb::tree::Code::kInternal); +} + +} // namespace + +extern "C" ct_status ct_apply_batch_owned(ct_tree *tree, uint64_t slot, const ct_owned_kv_ref *operations, size_t count) +{ + if (count != 0 && operations == nullptr) { + return static_cast<int>(crowdb::tree::Code::kInvalidArgument); + } + return apply_owned(tree, slot, count, [operations](size_t index) { return operations[index]; }); +} + +extern "C" ct_status ct_apply_batch_owned_reader(ct_tree *tree, uint64_t slot, size_t count, ct_owned_read_fn read, + const void *context) +{ + if (count != 0 && read == nullptr) { + return static_cast<int>(crowdb::tree::Code::kInvalidArgument); + } + return apply_owned(tree, slot, count, [read, context](size_t index) { return read(context, index); }); +} diff --git a/lib/crowdb-tree/src/c_api_owned_budget.cpp b/lib/crowdb-tree/src/c_api_owned_budget.cpp new file mode 100644 index 000000000..0601eff05 --- /dev/null +++ b/lib/crowdb-tree/src/c_api_owned_budget.cpp @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "c_api_owned_budget.h" + +#include "btree/allocation_reservation.h" +#include "c_api_future_internal.h" + +using namespace crowdb::tree; + +struct ct_owned_budget +{ + std::shared_ptr<AllocationBudget> budget; +}; + +std::shared_ptr<AllocationBudget> c_api_result_controls(const std::shared_ptr<detail::ResourceBudget::Account> &account) +{ + auto credit = detail::ResourceBudget::try_reserve( + account, detail::BudgetClass::kRouting, + kReturnedOwnerControlBytes + detail::shared_allocation_bound<detail::AllocationReservation>()); + if (!credit) { + throw std::bad_alloc(); + } + return detail::AllocationReservation::create(std::move(*credit)); +} + +int32_t c_api_owned_budget(const std::shared_ptr<AllocationBudget> &budget, ct_owned_budget **out) +try { + if (out == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = nullptr; + if (!budget) { + return 0; + } + MetadataAllocator<ct_owned_budget> allocator(budget); + auto *owner = allocator.allocate(1); + std::construct_at(owner, ct_owned_budget{budget}); + *out = owner; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} + +int32_t ct_future_owned_budget(const ct_future *future, ct_owned_budget **out) +{ + if (future == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + const auto &result = **reinterpret_cast<const ct_future_handle *>(future); + if (!result.done.load(std::memory_order_acquire) || result.status != 0 || + (result.kind != ct_future_impl::Kind::kGet && result.kind != ct_future_impl::Kind::kScan)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + return c_api_owned_budget(result.result_controls, out); +} + +int32_t ct_scan_result_owned_budget(const ct_scan_result *result, ct_owned_budget **out) +{ + return ct_future_owned_budget(reinterpret_cast<const ct_future *>(result), out); +} + +int32_t ct_owned_budget_claim(ct_owned_budget *budget, size_t bytes) +{ + return budget == nullptr || budget->budget->claim(bytes) ? 0 : static_cast<int32_t>(Code::kResourceExhausted); +} + +void ct_owned_budget_record(ct_owned_budget *budget, size_t bytes, int32_t acquired) +{ + if (budget != nullptr) { + budget->budget->record_allocation(AllocationKind::kMetadata, bytes, acquired != 0); + } +} + +void ct_owned_budget_release(ct_owned_budget *budget, size_t bytes) +{ + if (budget != nullptr) { + budget->budget->release(bytes); + } +} + +void ct_owned_budget_free(ct_owned_budget *budget) +{ + if (budget != nullptr) { + MetadataAllocator<ct_owned_budget> allocator(budget->budget); + std::destroy_at(budget); + allocator.deallocate(budget, 1); + } +} diff --git a/lib/crowdb-tree/src/c_api_owned_budget.h b/lib/crowdb-tree/src/c_api_owned_budget.h new file mode 100644 index 000000000..c3c4c039f --- /dev/null +++ b/lib/crowdb-tree/src/c_api_owned_budget.h @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/resource_budget.h" +#include "crowdb-tree/owned_budget_c_api.h" + +// Bounded headroom for downstream Bytes/RPC control storage. It is reserved +// at read admission; only actual controls are marked physically allocated. +inline constexpr size_t kReturnedOwnerControlBytes = 1024; + +[[nodiscard]] std::shared_ptr<crowdb::tree::AllocationBudget> +c_api_result_controls(const std::shared_ptr<crowdb::tree::detail::ResourceBudget::Account> &account); +int32_t c_api_owned_budget(const std::shared_ptr<crowdb::tree::AllocationBudget> &budget, ct_owned_budget **out); diff --git a/lib/crowdb-tree/src/c_api_scan_owner.cpp b/lib/crowdb-tree/src/c_api_scan_owner.cpp new file mode 100644 index 000000000..6122f994a --- /dev/null +++ b/lib/crowdb-tree/src/c_api_scan_owner.cpp @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_future_internal.h" +#include "c_api_internal.h" +#include "c_api_owned_budget.h" + +#include <optional> + +using namespace crowdb::tree; + +ct_status ct_future_poll_scan_v2(ct_future *future, int32_t *done, ct_scan_result **out) +{ + if (future == nullptr || done == nullptr || out == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *done = 0; + *out = nullptr; + auto *handle = reinterpret_cast<ct_future_handle *>(future); + const auto &impl = **handle; + if (impl.kind != ct_future_impl::Kind::kScan) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + if (!impl.done.load(std::memory_order_acquire)) { + return 0; + } + *done = 1; + const auto status = impl.status; + if (status == 0) { + *out = reinterpret_cast<ct_scan_result *>(future); + } + else { + ct_future_free(future); + } + return status; +} + +ct_status ct_scan_result_data(const ct_scan_result *result, const uint8_t **data, size_t *len, uint64_t *count, + int32_t *truncated) +{ + if (result == nullptr || data == nullptr || len == nullptr || count == nullptr || truncated == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + const auto &impl = **reinterpret_cast<const ct_future_handle *>(result); + if (impl.kind != ct_future_impl::Kind::kScan || !impl.done.load(std::memory_order_acquire) || impl.status != 0) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *data = impl.scan_packed.data(); + *len = impl.scan_packed.size(); + *count = impl.scan_count; + *truncated = impl.scan_truncated ? 1 : 0; + return 0; +} + +void ct_scan_result_free(ct_scan_result *result) +{ + ct_future_free(reinterpret_cast<ct_future *>(result)); +} + +namespace +{ +ct_status prepare_scan(ct_tree *tree, const ct_scan_options_v2 *options, CoreScanOptions *scan, TreeCore **core) +{ + if (tree == nullptr || options == nullptr || options->struct_size != sizeof(*options)) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + const auto &o = *options; + if ((o.prefix == nullptr && o.prefix_len != 0) || (o.start == nullptr && o.start_len != 0) || + (o.end == nullptr && o.end_len != 0) || (o.begin == nullptr && o.begin_len != 0) || + (o.strict_byte_budget != 0 && o.byte_budget == 0)) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (*core == nullptr) { + return static_cast<ct_status>(Code::kUnavailable); + } + *scan = {.prefix = Slice(o.prefix, o.prefix_len), + .start = Slice(o.start, o.start_len), + .end = Slice(o.end, o.end_len), + .begin = Slice(o.begin, o.begin_len), + .limit = o.limit, + .byte_budget = o.byte_budget, + .deadline_ms = o.deadline_ms, + .reverse = o.reverse != 0, + .keys_only = o.keys_only != 0, + .include_tombstones = o.include_tombstones != 0, + .has_start_bound = o.has_start_bound != 0, + .start_inclusive = o.start_inclusive != 0, + .strict_byte_budget = o.strict_byte_budget != 0}; + return 0; +} + +// Validate the internal packed record framing while counting the bounded page. +std::optional<size_t> count_entries(const ScanPackedBuf &packed) +{ + size_t offset = 0; + size_t count = 0; + const auto skip_string = [&] { + if (packed.size() - offset < 4) { + return false; + } + uint32_t length = 0; + for (size_t i = 0; i < 4; ++i) { + length |= uint32_t{packed.data()[offset + i]} << (8 * i); + } + offset += 4; + if (length > packed.size() - offset) { + return false; + } + offset += length; + return true; + }; + while (offset != packed.size()) { + if (!skip_string() || packed.size() - offset < 9) { + return {}; + } + offset += 9; + if (!skip_string()) { + return {}; + } + ++count; + } + return count; +} +} // namespace + +ct_status ct_scan_v2(ct_tree *tree, const ct_scan_options_v2 *options, ct_scan_result **out) +try { + if (out == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *out = nullptr; + CoreScanOptions scan{}; + TreeCore *core = nullptr; + if (auto status = prepare_scan(tree, options, &scan, &core); status != 0) { + return status; + } + auto impl = make_ct_future(core->binding()->lifecycle_budget()); + impl->kind = ct_future_impl::Kind::kScan; + impl->result_controls = c_api_result_controls(core->binding()->account()); + size_t count = 0; + auto status = core->scan(scan, nullptr, &impl->scan_truncated, &impl->scan_packed, &count); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + impl->scan_count = count; + impl->done.store(true, std::memory_order_release); + *out = reinterpret_cast<ct_scan_result *>(make_ct_future_handle(std::move(impl))); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +ct_status ct_scan_async_v2(ct_tree *tree, const ct_scan_options_v2 *options, ct_future **out) +try { + if (out == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *out = nullptr; + CoreScanOptions scan{}; + TreeCore *core = nullptr; + if (auto status = prepare_scan(tree, options, &scan, &core); status != 0) { + return status; + } + auto impl = make_ct_future(core->binding()->lifecycle_budget()); + impl->kind = ct_future_impl::Kind::kScan; + impl->result_controls = c_api_result_controls(core->binding()->account()); + std::unique_ptr<ct_future_handle, decltype(&free_ct_future_handle)> handle(make_ct_future_handle(impl), + free_ct_future_handle); + core->scan_async( + scan, [impl, signal = c_api_completion_signal(tree)](Status status, ScanPackedBuf packed, bool truncated) { + impl->status = static_cast<ct_status>(status.code()); + if (status.ok()) { + const auto count = count_entries(packed); + if (count) { + impl->scan_count = *count; + impl->scan_packed = std::move(packed); + impl->scan_truncated = truncated; + } + else { + impl->status = static_cast<ct_status>(Code::kCorruption); + } + } + impl->done.store(true, std::memory_order_release); + c_api_notify_completion(signal); + }); + *out = reinterpret_cast<ct_future *>(handle.release()); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} diff --git a/lib/crowdb-tree/src/c_api_snapshot_export.cpp b/lib/crowdb-tree/src/c_api_snapshot_export.cpp new file mode 100644 index 000000000..8e921cc6a --- /dev/null +++ b/lib/crowdb-tree/src/c_api_snapshot_export.cpp @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "c_api_owned_budget.h" +#include "crowdb-tree/snapshot/snapshot_io.h" + +#include <cstdlib> +#include <cstring> + +using namespace crowdb::tree; + +struct ct_export +{ + std::shared_ptr<AllocationBudget> metadata; + std::unique_ptr<SnapshotExport> exp; + SnapshotChunk pending{}; + uint64_t pending_offset = 0; + bool pending_valid = false; + bool pending_done = false; + + Status prepare(uint64_t offset) + { + if (offset != (pending_valid ? pending_offset : exp->offset())) { + return Status::invalid_argument("snapshot offset is not sequential"); + } + if (pending_valid) { + return Status::Ok(); + } + auto status = exp->next_owned(&pending, &pending_done); + if (status.ok()) { + pending_offset = offset; + pending_valid = true; + } + return status; + } + + void consumed() + { + pending = {}; + pending_valid = false; + } +}; + +struct ct_snapshot_chunk +{ + SnapshotChunk value{}; +}; + +static_assert(sizeof(ct_snapshot_chunk) <= kSnapshotChunkHandleBytes); + +ct_status ct_snapshot_export_begin(ct_tree *tree, size_t chunk_bytes, ct_export **out) +try { + if (tree == nullptr || out == nullptr || chunk_bytes == 0) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *out = nullptr; + auto *native = c_api_tree(tree); + auto *core = TreeCoreAccess::get(*native); + std::shared_ptr<AllocationBudget> budget = core != nullptr ? core->binding()->lifecycle_budget() : nullptr; + std::unique_ptr<SnapshotExport> stream; + auto status = snapshot_export_begin(*native, snapshot_format::kPortable, chunk_bytes, &stream); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + MetadataAllocator<ct_export> allocator(budget); + auto *result = allocator.allocate(1); + std::construct_at(result); + result->metadata = std::move(budget); + result->exp = std::move(stream); + *out = result; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +uint64_t ct_snapshot_export_at_slot(const ct_export *e) +{ + return e == nullptr ? 0 : e->exp->at_slot(); +} + +uint64_t ct_snapshot_export_total_bytes(const ct_export *e) +{ + return e == nullptr ? 0 : static_cast<uint64_t>(e->exp->total_bytes()); +} + +uint32_t ct_snapshot_export_final_crc32c(const ct_export *e) +{ + return e == nullptr ? 0 : e->exp->final_crc32c(); +} + +size_t ct_snapshot_export_chunk_bytes(const ct_export *e) +{ + return e == nullptr ? 0 : e->exp->chunk_bytes(); +} + +uint64_t ct_snapshot_export_offset(const ct_export *e) +{ + if (e == nullptr) { + return 0; + } + return e->pending_valid ? e->pending_offset : e->exp->offset(); +} + +ct_status ct_snapshot_export_next(ct_export *e, uint64_t offset, ct_buf *chunk, int32_t *done) +try { + if (e == nullptr || chunk == nullptr || done == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + const auto status = e->prepare(offset); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + const auto bytes = e->pending.bytes; + uint8_t *data = nullptr; + if (!bytes.empty()) { + data = static_cast<uint8_t *>(std::malloc(bytes.size())); + if (data == nullptr) { + throw std::bad_alloc(); + } + std::memcpy(data, bytes.data(), bytes.size()); + } + *chunk = {.data = data, .len = bytes.size()}; + *done = e->pending_done ? 1 : 0; + e->consumed(); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +ct_status ct_snapshot_export_next_v2(ct_export *e, uint64_t offset, ct_snapshot_chunk **out, int32_t *done) +try { + if (e == nullptr || out == nullptr || done == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *out = nullptr; + const auto status = e->prepare(offset); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + MetadataAllocator<ct_snapshot_chunk> allocator(e->pending.allocation_budget); + auto *result = allocator.allocate(1); + std::construct_at(result, ct_snapshot_chunk{std::move(e->pending)}); + *out = result; + *done = e->pending_done ? 1 : 0; + e->consumed(); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +ct_status ct_snapshot_chunk_data(const ct_snapshot_chunk *chunk, const uint8_t **data, size_t *len) +{ + if (chunk == nullptr || data == nullptr || len == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + *data = chunk->value.bytes.data(); + *len = chunk->value.bytes.size(); + return 0; +} + +int32_t ct_snapshot_chunk_owned_budget(const ct_snapshot_chunk *chunk, ct_owned_budget **out) +{ + if (chunk == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + return c_api_owned_budget(chunk->value.allocation_budget, out); +} + +void ct_snapshot_chunk_free(ct_snapshot_chunk *chunk) +{ + if (chunk != nullptr) { + MetadataAllocator<ct_snapshot_chunk> allocator(chunk->value.allocation_budget); + std::destroy_at(chunk); + allocator.deallocate(chunk, 1); + } +} + +void ct_snapshot_export_end(ct_export *e) +{ + if (e != nullptr) { + MetadataAllocator<ct_export> allocator(e->metadata); + std::destroy_at(e); + allocator.deallocate(e, 1); + } +} diff --git a/lib/crowdb-tree/src/c_api_split_overlay.cpp b/lib/crowdb-tree/src/c_api_split_overlay.cpp new file mode 100644 index 000000000..5ed14864d --- /dev/null +++ b/lib/crowdb-tree/src/c_api_split_overlay.cpp @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/prepared_split_overlay.h" +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/split_overlay_c_api.h" + +struct ct_prepared_split_overlay +{ + crowdb::tree::AllocationLease allocation; + std::shared_ptr<crowdb::tree::PreparedSplitOverlay> owner; +}; + +namespace +{ +ct_status code(const crowdb::tree::Status &status) noexcept +{ + return static_cast<ct_status>(status.code()); +} +} // namespace + +extern "C" ct_status ct_arm_split_overlay(ct_tree *source, ct_tree *destination, + const ct_split_overlay_limits_v1 *limits, ct_prepared_split_overlay **out, + uint64_t *start_frontier) +try { + using namespace crowdb::tree; + auto *parent_tree = c_api_tree(source); + auto *child_tree = c_api_tree(destination); + if (parent_tree == nullptr || child_tree == nullptr || limits == nullptr || limits->version != 1 || + limits->struct_size != sizeof(*limits) || limits->max_records > SIZE_MAX || limits->max_bytes > SIZE_MAX || + out == nullptr || *out != nullptr || start_frontier == nullptr) { + return code(Status::invalid_argument()); + } + auto *parent = TreeCoreAccess::get(*parent_tree); + auto *child = TreeCoreAccess::get(*child_tree); + if (parent == nullptr || child == nullptr) { + return code(Status::invalid_argument()); + } + auto reserved = detail::ResourceBudget::try_reserve( + child->binding()->account(), detail::BudgetClass::kRouting, + sizeof(ct_prepared_split_overlay) + detail::shared_allocation_bound<detail::AllocationReservation>()); + if (!reserved) { + return code(Status::resource_exhausted()); + } + auto metadata = detail::AllocationReservation::create(std::move(*reserved)); + AllocationLease charge(metadata, sizeof(ct_prepared_split_overlay), AllocationKind::kMetadata); + auto result = std::make_unique<ct_prepared_split_overlay>(); + result->allocation = std::move(charge); + auto status = PreparedSplitOverlay::arm( + *parent, *child, + {.max_records = static_cast<size_t>(limits->max_records), .max_bytes = static_cast<size_t>(limits->max_bytes)}, + &result->owner, start_frontier); + if (!status.ok()) { + return code(status); + } + metadata->seal(); + *out = result.release(); + return code(Status::Ok()); +} +catch (const std::bad_alloc &) { + return code(crowdb::tree::Status::resource_exhausted()); +} +catch (...) { + return code(crowdb::tree::Status::internal_error()); +} + +extern "C" ct_status ct_prepared_split_overlay_prepare(ct_prepared_split_overlay *prepared, uint64_t replayed_prefix) +try { + return code(prepared != nullptr && prepared->owner ? prepared->owner->prepare(replayed_prefix) + : crowdb::tree::Status::invalid_argument()); +} +catch (const std::bad_alloc &) { + return code(crowdb::tree::Status::resource_exhausted()); +} +catch (...) { + return code(crowdb::tree::Status::internal_error()); +} + +extern "C" ct_status ct_prepared_split_overlay_try_install(ct_prepared_split_overlay *prepared, uint64_t frontier, + ct_installed_split_overlay_v1 *out) +{ + using namespace crowdb::tree; + if (prepared == nullptr || !prepared->owner || out == nullptr || out->version != 1 || + out->struct_size != sizeof(*out)) { + return code(Status::invalid_argument()); + } + InstalledSplitOverlay installed; + auto status = prepared->owner->try_install(frontier, &installed); + if (status.ok()) { + out->generation = installed.generation; + out->frontier = installed.frontier; + } + return code(status); +} + +extern "C" int ct_prepared_split_overlay_committed_frontier(const ct_prepared_split_overlay *prepared, uint64_t *out) +{ + if (prepared != nullptr && prepared->owner && out != nullptr) { + if (auto frontier = prepared->owner->committed_frontier()) { + *out = *frontier; + return 1; + } + } + return 0; +} + +extern "C" void ct_prepared_split_overlay_destroy(ct_prepared_split_overlay *prepared) +{ + delete prepared; +} diff --git a/lib/crowdb-tree/src/epoch.cpp b/lib/crowdb-tree/src/epoch.cpp index d08cc6873..6980dcd54 100644 --- a/lib/crowdb-tree/src/epoch.cpp +++ b/lib/crowdb-tree/src/epoch.cpp @@ -1,8 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -#include "crowdb-tree/epoch.h" +#include "btree/retired_shared.h" +#include "epoch_internal.h" +#include <array> #include <utility> #include <vector> @@ -11,49 +13,84 @@ namespace crowdb::tree namespace { -// Monotonic id source so a thread's per-manager cache keys never collide, even -// if a manager's heap address is reused after destruction. -std::atomic<uint64_t> g_epoch_mgr_id{1}; +// Stable thread identities survive eviction from the fixed participant cache. +std::atomic<uint64_t> g_epoch_thread_id{1}; } // namespace // ── Reader hot path (lock-free) ─────────────────────────────────── -EpochManager::Participant *EpochManager::participant_for_this_thread() +EpochManager::Participant *EpochManager::participant_for_this_thread(const std::shared_ptr<AllocationBudget> &budget) { - // Process-global per-thread cache of (manager id -> this thread's slot). A - // thread that touches several managers keeps one entry each; entries for a - // destroyed manager (unique id) are simply never matched again. - static thread_local std::vector<std::pair<uint64_t, Participant *>> cache; + // New reads of an immutable source need only its retained table owner. + // Guards accepted before the seal still exit their original participant. + if (source_sealed()) { + return nullptr; + } + // Fixed TLS storage cannot grow with the number of retired MemTables. A + // cache miss searches the manager's immutable list before allocating, so + // evicting an active nested guard's cache entry cannot create a second slot. + static thread_local std::array<std::pair<uint64_t, Participant *>, 32> cache{}; + static thread_local size_t replacement = 0; + static thread_local const uint64_t thread_id = g_epoch_thread_id.fetch_add(1, std::memory_order_relaxed); for (auto &e : cache) { if (e.first == id_) { return e.second; } } - // First use on this thread: allocate our slot and push it (lock-free) onto the - // manager's participant list. The node lives until the manager is destroyed. - auto *p = new Participant(); - Participant *head = participants_.load(std::memory_order_relaxed); - do { - p->next = head; - } while (!participants_.compare_exchange_weak(head, p, std::memory_order_release, std::memory_order_relaxed)); - cache.emplace_back(id_, p); + auto &participants = lifetime_->queue ? lifetime_->live_participants : participants_; + for (auto *p = participants.load(std::memory_order_acquire); p != nullptr; p = p->next) { + if (p->thread_id == thread_id) { + cache[replacement++ % cache.size()] = {id_, p}; + return p; + } + } + if (participants.sealed()) { + return nullptr; + } + MetadataAllocator<Participant> allocator(budget ? budget : budget_); + auto *p = allocator.allocate(1); + std::construct_at(p); + p->thread_id = thread_id; + p->budget = allocator.owner(); + const auto head = participants.registration_head(); +#ifdef CROWDB_TREE_TEST_UTIL + if (append_hook_ != nullptr) { + append_hook_(append_context_); + } +#endif + if (!participants.push(p, head)) { + destroy_participant(p); + return nullptr; + } + cache[replacement++ % cache.size()] = {id_, p}; return p; } -EpochManager::Guard EpochManager::enter() +EpochManager::Guard EpochManager::enter(std::shared_ptr<AllocationBudget> budget) { - Participant *p = participant_for_this_thread(); + Participant *p = participant_for_this_thread(budget); + if (p == nullptr) { + return {}; + } // Reentrant: only the outermost enter publishes the entry epoch. if (p->nest++ == 0) { // seq_cst pair: publish our entry epoch before any subsequent shared load, // so a concurrent reclaimer either sees this epoch (keeps the object) or we // see its swapped-in mapping (never touch the retired object). - uint64_t e = global_epoch_.load(std::memory_order_seq_cst); + auto &epoch = lifetime_->queue ? lifetime_->live_epoch : global_epoch_; + uint64_t e = epoch.load(std::memory_order_seq_cst); p->local_epoch.store(e, std::memory_order_seq_cst); } return Guard(p, lifetime_); } +void EpochManager::destroy_participant(Participant *participant) noexcept +{ + MetadataAllocator<Participant> allocator(participant->budget); + std::destroy_at(participant); + allocator.deallocate(participant, 1); +} + void EpochManager::Guard::release() { if (p_ != nullptr) { @@ -61,6 +98,9 @@ void EpochManager::Guard::release() // a plain field. No reclamation here — that is writer-driven. if (--p_->nest == 0) { p_->local_epoch.store(0, std::memory_order_release); + if (lifetime_) { + lifetime_->request_collect(); + } } p_ = nullptr; } @@ -68,43 +108,14 @@ void EpochManager::Guard::release() // ── Writer side (reclaim_mu_) ───────────────────────────────────── -// Detached borrows may outlive their manager. Once detached, no producer can -// enqueue further work; the last guard owns the remaining reclamation batch. -struct EpochManager::Lifetime -{ - Participant *participants = nullptr; - Deferred *pending = nullptr; - Deferred *collected = nullptr; - std::vector<Retired> retired; - - ~Lifetime() - { - for (auto *head : {pending, collected}) { - while (head != nullptr) { - auto *next = head->next; - head->destroy(head); - head = next; - } - } - for (auto &entry : retired) { - entry.deleter(entry.ptr); - } - while (participants != nullptr) { - auto *next = participants->next; - delete participants; - participants = next; - } - } -}; - -EpochManager::EpochManager() - : lifetime_(std::make_shared<Lifetime>()), - id_(g_epoch_mgr_id.fetch_add(1, std::memory_order_relaxed)) -{ -} - EpochManager::~EpochManager() { + if (!lifetime_) { + return; + } + if (lifetime_->queue) { + return; + } if (active_guards() != 0) { lifetime_->participants = participants_.exchange(nullptr); lifetime_->pending = deferred_.exchange(nullptr); @@ -113,8 +124,8 @@ EpochManager::~EpochManager() return; } - // By destruction time no guards must remain. Free anything still pending, then - // free the participant nodes. Drain in a loop (not once): a deleter can + // By destruction time no guards must remain. Free anything still pending, + // then free the participant nodes. Drain in a loop (not once): a deleter can // retire something else on this same manager (see reclaim_locked()'s doc // comment), which re-populates retired_ -- keep detaching+running until // nothing's left, so a nested retirement's deleter still runs instead of @@ -130,7 +141,7 @@ EpochManager::~EpochManager() r.deleter(r.ptr); } catch (...) { // NOLINT(bugprone-empty-catch) - // Destructors must not throw; swallow deleter exceptions. + // Destructors must not throw; swallow deleter exceptions. } } } @@ -138,15 +149,15 @@ EpochManager::~EpochManager() Participant *p = participants_.load(std::memory_order_acquire); while (p != nullptr) { Participant *next = p->next; - delete p; + destroy_participant(p); p = next; } } uint64_t EpochManager::min_active_epoch() { - // Oldest epoch any active reader might still be inside. seq_cst loads pair with - // enter()'s publish for the total-order argument above. + // Oldest epoch any active reader might still be inside. seq_cst loads pair + // with enter()'s publish for the total-order argument above. uint64_t min_active = global_epoch_.load(std::memory_order_seq_cst); for (Participant *p = participants_.load(std::memory_order_acquire); p != nullptr; p = p->next) { uint64_t e = p->local_epoch.load(std::memory_order_seq_cst); @@ -159,6 +170,30 @@ uint64_t EpochManager::min_active_epoch() void EpochManager::retire(void *ptr, Deleter deleter) { + if (lifetime_ && lifetime_->queue) { + struct Callback : Deferred + { + void *value; + Deleter deleter; + std::shared_ptr<AllocationBudget> budget; + }; + + MetadataAllocator<Callback> allocator(budget_); + auto *callback = allocator.allocate(1); + std::construct_at(callback); + callback->value = ptr; + callback->deleter = std::move(deleter); + callback->budget = budget_; + callback->destroy = [](Deferred *entry) noexcept { + auto *callback = static_cast<Callback *>(entry); + MetadataAllocator<Callback> allocator(callback->budget); + callback->deleter(callback->value); + std::destroy_at(callback); + allocator.deallocate(callback, 1); + }; + lifetime_->defer(callback); + return; + } std::scoped_lock lk(reclaim_mu_); // New retirements belong to the current epoch; bump so a fresh guard entering // after this cannot claim to predate the retirement. @@ -173,9 +208,7 @@ size_t EpochManager::reclaim_locked() uint64_t min_active = min_active_epoch(); // Detach the current retired list *before* running any deleter: a // deleter can legitimately call retire() again on this same manager - // (e.g. retire_orphaned_page's deferred mapping_.clear() recycling its - // now-empty MappingSegment via epoch.retire_object(seg)) -- reclaim_mu_ - // being recursive lets that nested call proceed on the same thread, but + // -- reclaim_mu_ being recursive permits same-thread re-entry, but // it would still corrupt an in-progress `for (auto &r : retired_)` (the // nested call's own push_back/swap reallocates or replaces the very // vector this loop is iterating). Iterating a local, already-swapped-out @@ -199,12 +232,20 @@ size_t EpochManager::reclaim_locked() size_t EpochManager::try_reclaim() { + if (lifetime_ && lifetime_->queue) { + const auto before = lifetime_->freed_count.load(std::memory_order_relaxed); + lifetime_->request_collect(); + return lifetime_->freed_count.load(std::memory_order_relaxed) - before; + } std::scoped_lock lk(reclaim_mu_); return reclaim_deferred_locked() + reclaim_locked(); } size_t EpochManager::pending_retired() { + if (lifetime_ && lifetime_->queue) { + return lifetime_->pending_count.load(std::memory_order_relaxed); + } std::scoped_lock lk(reclaim_mu_); size_t count = retired_.size(); // Diagnostic snapshot; tickets can only be reclaimed while this lock is held. @@ -219,6 +260,10 @@ size_t EpochManager::pending_retired() void EpochManager::defer(Deferred *entry) noexcept { + if (lifetime_ && lifetime_->queue) { + lifetime_->defer(entry); + return; + } auto *head = deferred_.load(std::memory_order_relaxed); do { entry->next = head; @@ -259,8 +304,9 @@ size_t EpochManager::reclaim_deferred_locked() size_t EpochManager::active_guards() { // Threads currently inside a guard (per-thread, not per-nested-guard). - size_t n = 0; - for (Participant *p = participants_.load(std::memory_order_acquire); p != nullptr; p = p->next) { + size_t n = 0; + auto &participants = lifetime_ && lifetime_->queue ? lifetime_->live_participants : participants_; + for (Participant *p = participants.load(std::memory_order_acquire); p != nullptr; p = p->next) { if (p->local_epoch.load(std::memory_order_seq_cst) != 0) { ++n; } diff --git a/lib/crowdb-tree/src/epoch_backend_close.cpp b/lib/crowdb-tree/src/epoch_backend_close.cpp new file mode 100644 index 000000000..296779be0 --- /dev/null +++ b/lib/crowdb-tree/src/epoch_backend_close.cpp @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "epoch_internal.h" + +#include <cassert> +#include <utility> + +namespace crowdb::tree +{ +bool EpochManager::retire_backend_step() noexcept +{ + if (!lifetime_) { + return true; + } + // Block directories are created before runtime admission. Their short + // guards cannot survive the directory's final backend-owner handoff. + assert(!lifetime_->queue && lifetime_.use_count() == 1); + if (lifetime_->participants == nullptr) { + lifetime_->participants = participants_.exchange(nullptr, std::memory_order_relaxed); + } + if (lifetime_->pending == nullptr) { + lifetime_->pending = deferred_.exchange(nullptr, std::memory_order_relaxed); + } + if (lifetime_->collected == nullptr) { + lifetime_->collected = std::exchange(collected_, nullptr); + } + if (!retired_.empty()) { + lifetime_->retired = std::move(retired_); + } + return lifetime_->retire_step(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/epoch_init.cpp b/lib/crowdb-tree/src/epoch_init.cpp new file mode 100644 index 000000000..266acc7e9 --- /dev/null +++ b/lib/crowdb-tree/src/epoch_init.cpp @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/retired_shared.h" +#include "epoch_internal.h" + +namespace crowdb::tree +{ +namespace +{ +std::atomic<uint64_t> next_epoch_identity{1}; +} + +EpochManager::EpochManager(std::shared_ptr<AllocationBudget> budget, bool enabled) + : budget_(std::move(budget)), + id_(next_epoch_identity.fetch_add(1, std::memory_order_relaxed)) +{ + if (!enabled) { + return; + } + memtable_allocation_ = std::allocate_shared<AllocationCounter>(MetadataAllocator<AllocationCounter>(budget_), 0); + if (auto queue = budget_ ? budget_->retirement_queue() : nullptr) { + lifetime_ = detail::retired_shared<Lifetime>(budget_, queue, budget_, queue); + } + else { + lifetime_ = std::allocate_shared<Lifetime>(MetadataAllocator<Lifetime>(budget_)); + } +} + +size_t EpochManager::infrastructure_allocation_bound() noexcept +{ + return sizeof(EpochManager) + detail::shared_allocation_bound<Lifetime>() + + detail::shared_allocation_bound<AllocationCounter>(); +} + +bool EpochManager::ParticipantList::push(Participant *participant, uintptr_t head) noexcept +{ + while ((head & 1) == 0) { + participant->next = pointer(head); + if (head_.compare_exchange_weak(head, reinterpret_cast<uintptr_t>(participant), std::memory_order_release, + std::memory_order_acquire)) { + return true; + } + } + return false; +} + +bool EpochManager::source_sealed() const noexcept +{ + return lifetime_ && lifetime_->queue && lifetime_->live_participants.sealed(); +} + +void EpochManager::seal_source() noexcept +{ + if (lifetime_ && lifetime_->queue) { + (void)lifetime_->live_participants.seal(); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/epoch_internal.h b/lib/crowdb-tree/src/epoch_internal.h new file mode 100644 index 000000000..dc6ddfc5a --- /dev/null +++ b/lib/crowdb-tree/src/epoch_internal.h @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/retirement_queue.h" +#include "crowdb-tree/epoch.h" + +namespace crowdb::tree +{ +struct EpochManager::Lifetime : std::enable_shared_from_this<Lifetime> +{ + explicit Lifetime(std::shared_ptr<AllocationBudget> budget = {}, + std::shared_ptr<detail::RetirementQueue> retirement = {}); + ~Lifetime(); + void request_collect() noexcept; + void defer(Deferred *entry) noexcept; + bool retire_step() noexcept; + + std::shared_ptr<detail::RetirementQueue> queue; + ParticipantList live_participants; + std::atomic<Deferred *> live_pending{nullptr}; + std::atomic<uint64_t> live_epoch{1}; + std::atomic<size_t> pending_count{0}; + std::atomic<size_t> freed_count{0}; + + // Legacy detached guards use the same bounded final destruction path. + Participant *participants = nullptr; + Deferred *pending = nullptr; + Deferred *collected = nullptr; + std::vector<Retired> retired; + + private: + struct Collector : detail::RetirementNode + { + Lifetime *owner = nullptr; + } collector; + + static bool collect(detail::RetirementNode *node) noexcept; + bool collect_step() noexcept; + void return_blocked() noexcept; + void finish_cycle() noexcept; + std::shared_ptr<Lifetime> collector_owner; + std::atomic<bool> collecting{false}; + std::atomic<bool> requested{false}; + enum class Phase : uint8_t { kStart, kParticipants, kDeferred }; + Phase phase = Phase::kStart; + Participant *scan = nullptr; + Deferred *batch = nullptr; + Deferred *blocked = nullptr; + Deferred *blocked_tail = nullptr; + uint64_t oldest = 0; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/epoch_retirement.cpp b/lib/crowdb-tree/src/epoch_retirement.cpp new file mode 100644 index 000000000..59f746d93 --- /dev/null +++ b/lib/crowdb-tree/src/epoch_retirement.cpp @@ -0,0 +1,168 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "epoch_internal.h" + +#include <cassert> +#include <utility> + +namespace crowdb::tree +{ +EpochManager::Lifetime::Lifetime(std::shared_ptr<AllocationBudget> budget, + std::shared_ptr<detail::RetirementQueue> retirement) + : queue(std::move(retirement)) +{ + collector.owner = this; + collector.retirement_budget = std::move(budget); + collector.dispose = &collect; +} + +void EpochManager::Lifetime::defer(Deferred *entry) noexcept +{ + entry->epoch = live_epoch.fetch_add(1, std::memory_order_seq_cst); + pending_count.fetch_add(1, std::memory_order_relaxed); + auto *head = live_pending.load(std::memory_order_relaxed); + do { + entry->next = head; + } while (!live_pending.compare_exchange_weak(head, entry, std::memory_order_release, std::memory_order_relaxed)); + request_collect(); +} + +void EpochManager::Lifetime::request_collect() noexcept +{ + if (!queue || pending_count.load(std::memory_order_acquire) == 0) { + return; + } + requested.store(true, std::memory_order_release); + if (!collecting.exchange(true, std::memory_order_acq_rel)) { + collector_owner = weak_from_this().lock(); + assert(collector_owner); + queue->retire(&collector); + } +} + +bool EpochManager::Lifetime::collect(detail::RetirementNode *node) noexcept +{ + auto &self = *static_cast<Collector *>(node)->owner; + auto keep = std::move(self.collector_owner); + if (!self.collect_step()) { + self.collector_owner = std::move(keep); + return false; + } + self.finish_cycle(); + return true; +} + +bool EpochManager::Lifetime::collect_step() noexcept +{ + if (phase == Phase::kStart) { + requested.exchange(false, std::memory_order_acq_rel); + batch = live_pending.exchange(nullptr, std::memory_order_acquire); + if (batch == nullptr) { + return true; + } + oldest = live_epoch.load(std::memory_order_seq_cst); + scan = live_participants.load(std::memory_order_acquire); + phase = Phase::kParticipants; + } + size_t work = 0; + while (scan != nullptr && work < 64) { + const auto epoch = scan->local_epoch.load(std::memory_order_seq_cst); + if (epoch != 0 && epoch < oldest) { + oldest = epoch; + } + scan = scan->next; + ++work; + } + if (scan != nullptr) { + return false; + } + phase = Phase::kDeferred; + while (batch != nullptr && work < 64) { + auto *entry = batch; + batch = entry->next; + if (entry->epoch < oldest) { + entry->destroy(entry); + pending_count.fetch_sub(1, std::memory_order_release); + freed_count.fetch_add(1, std::memory_order_relaxed); + } + else { + entry->next = blocked; + blocked = entry; + if (blocked_tail == nullptr) { + blocked_tail = entry; + } + } + ++work; + } + if (batch != nullptr) { + return false; + } + return_blocked(); + phase = Phase::kStart; + return true; +} + +void EpochManager::Lifetime::return_blocked() noexcept +{ + if (blocked == nullptr) { + return; + } + auto *head = live_pending.load(std::memory_order_relaxed); + do { + blocked_tail->next = head; + } while (!live_pending.compare_exchange_weak(head, blocked, std::memory_order_release, std::memory_order_relaxed)); + blocked = nullptr; + blocked_tail = nullptr; +} + +void EpochManager::Lifetime::finish_cycle() noexcept +{ + // Acquire the producer's request when its exchange observed us running. + // Otherwise the producer observes idle and schedules the next sweep. + collecting.exchange(false, std::memory_order_acq_rel); + // A live reader ends the cycle instead of keeping a worker busy. Its final + // guard exit, or a newly retired object, starts the next bounded sweep. + if (requested.load(std::memory_order_acquire)) { + request_collect(); + } +} + +bool EpochManager::Lifetime::retire_step() noexcept +{ + assert(!collecting.load(std::memory_order_relaxed)); + if (participants == nullptr) { + participants = live_participants.exchange(nullptr); + } + if (pending == nullptr) { + pending = live_pending.exchange(nullptr); + } + size_t steps = 0; + for (auto **head : {&pending, &collected}) { + while (*head != nullptr && steps < 64) { + auto *node = *head; + *head = node->next; + node->destroy(node); + ++steps; + } + } + while (!retired.empty() && steps < 64) { + auto entry = std::move(retired.back()); + retired.pop_back(); + entry.deleter(entry.ptr); + ++steps; + } + while (participants != nullptr && steps < 64) { + auto *next = participants->next; + destroy_participant(participants); + participants = next; + ++steps; + } + return pending == nullptr && collected == nullptr && retired.empty() && participants == nullptr; +} + +EpochManager::Lifetime::~Lifetime() +{ + while (!retire_step()) { + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/external_owner_bridge.cpp b/lib/crowdb-tree/src/external_owner_bridge.cpp new file mode 100644 index 000000000..841d6070f --- /dev/null +++ b/lib/crowdb-tree/src/external_owner_bridge.cpp @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "external_owner_bridge.h" + +namespace crowdb::tree +{ +Status ExternalOwnerBridge::create(const uint8_t *data, size_t initialized_bytes, size_t capacity, + ExternalAllocation::Owner owner, const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<const ExternalOwnerBridge> *out) noexcept +try { + if (out == nullptr || !budget || initialized_bytes > capacity || (capacity != 0 && data == nullptr) || !owner || + owner.get_deleter() == nullptr) { + return Status::invalid_argument(); + } + auto allocation = ExternalAllocation::create(data, capacity, std::move(owner), budget); + *out = std::allocate_shared<ExternalOwnerBridge>(MetadataAllocator<ExternalOwnerBridge>(budget), + std::move(allocation), initialized_bytes); + return {}; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (...) { + return Status::internal_error(); +} + +Status ExternalOwnerBridge::slice(size_t offset, size_t length, buffer *out) const noexcept +try { + if (out == nullptr || offset > initialized_bytes_ || length > initialized_bytes_ - offset) { + return Status::invalid_argument(); + } + *out = buffer::share_external(allocation_, offset, length); + return {}; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (...) { + return Status::internal_error(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/external_owner_bridge.h b/lib/crowdb-tree/src/external_owner_bridge.h new file mode 100644 index 000000000..8bafb0817 --- /dev/null +++ b/lib/crowdb-tree/src/external_owner_bridge.h @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/buffer.h" +#include "crowdb-tree/external_allocation.h" +#include "crowdb-tree/status.h" + +namespace crowdb::tree +{ +// A bridge created at a complete allocation boundary. Capacity is charged +// once, while slice bounds use initialized bytes and never expose spare +// capacity. A host reservation is mandatory before accepting the owner. +class ExternalOwnerBridge +{ + public: + ExternalOwnerBridge(std::shared_ptr<const ExternalAllocation> allocation, size_t initialized_bytes) + : allocation_(std::move(allocation)), + initialized_bytes_(initialized_bytes) + { + if (!allocation_ || !allocation_->budget() || initialized_bytes_ > allocation_->capacity()) { + throw std::invalid_argument("external owner requires a charged complete allocation"); + } + } + + static Status create(const uint8_t *data, size_t initialized_bytes, size_t capacity, + ExternalAllocation::Owner owner, const std::shared_ptr<AllocationBudget> &budget, + std::shared_ptr<const ExternalOwnerBridge> *out) noexcept; + + [[nodiscard]] Status slice(size_t offset, size_t length, buffer *out) const noexcept; + + [[nodiscard]] const ExternalAllocation *identity() const noexcept + { + return allocation_.get(); + } + + [[nodiscard]] size_t capacity() const noexcept + { + return allocation_->capacity(); + } + + private: + std::shared_ptr<const ExternalAllocation> allocation_; + size_t initialized_bytes_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/host_completion.h b/lib/crowdb-tree/src/host_completion.h new file mode 100644 index 000000000..520a24b1f --- /dev/null +++ b/lib/crowdb-tree/src/host_completion.h @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <atomic> +#include <thread> + +namespace crowdb::tree::detail +{ +// A one-shot bridge for a host stack waiting on an asynchronous callback. +// Publishing readiness can wake the host before notify_one returns. Keep the +// stack storage alive until the notifier's final access has completed too. +class HostCompletion +{ + public: + void signal() noexcept + { + signal_impl([] {}); + } + + void wait() const noexcept + { + ready_.wait(false, std::memory_order_acquire); + // Only the notification tail remains; it must not access this object + // after publishing released_. No operation work runs in this interval. + while (!released_.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + +#ifdef CROWDB_TREE_TEST_UTIL + [[nodiscard]] bool completed_for_tests() const noexcept + { + return released_.load(std::memory_order_acquire); + } + + void signal_for_tests(void (*before_notify)(void *), void *context) noexcept + { + signal_impl([&] { before_notify(context); }); + } +#endif + + private: + template <class BeforeNotify> void signal_impl(BeforeNotify before_notify) noexcept + { + ready_.store(true, std::memory_order_release); + before_notify(); + ready_.notify_one(); + released_.store(true, std::memory_order_release); + } + + std::atomic<bool> ready_{false}; + std::atomic<bool> released_{false}; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/maptable/compressor_bounded.cpp b/lib/crowdb-tree/src/maptable/compressor_bounded.cpp new file mode 100644 index 000000000..82584203a --- /dev/null +++ b/lib/crowdb-tree/src/maptable/compressor_bounded.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/maptable/compressor.h" + +#include <cstring> +#include <memory> + +#if CROWDB_TREE_HAVE_LZ4 +# include <lz4.h> +#endif + +namespace crowdb::tree +{ +namespace +{ +void put_u32(uint8_t *out, uint32_t value) +{ + for (unsigned i = 0; i < 4; ++i) { + out[i] = static_cast<uint8_t>(value >> (i * 8)); + } +} +} // namespace + +size_t durable_encode_scratch_bytes(compress_algo prefer) noexcept +{ +#if CROWDB_TREE_HAVE_LZ4 + return prefer == compress_algo::kLz4 ? static_cast<size_t>(LZ4_sizeofState()) + 7 : 0; +#else + (void)prefer; + return 0; +#endif +} + +Status encode_durable_page_into(std::span<const uint8_t> frame, compress_algo prefer, std::span<uint8_t> output, + std::span<uint8_t> scratch, uint32_t *logical_bytes) +{ + if (logical_bytes == nullptr || frame.empty() || frame.size() > UINT32_MAX - kDurableBlobHeader || + (prefer != compress_algo::kNone && prefer != compress_algo::kLz4)) { + return Status::invalid_argument(); + } + if (output.size() < kDurableBlobHeader + frame.size() || scratch.size() < durable_encode_scratch_bytes(prefer)) { + return Status::resource_exhausted(); + } + auto algo = compress_algo::kNone; + auto length = static_cast<uint32_t>(frame.size()); + auto *stored = output.data() + kDurableBlobHeader; +#if CROWDB_TREE_HAVE_LZ4 + if (prefer == compress_algo::kLz4) { + if (frame.size() > LZ4_MAX_INPUT_SIZE) { + return Status::invalid_argument(); + } + void *state = scratch.data(); + auto available = scratch.size(); + if (std::align(8, static_cast<size_t>(LZ4_sizeofState()), state, available) == nullptr) { + return Status::resource_exhausted(); + } + // Any output that does not strictly shrink the payload uses the raw + // representation. No second compressed-frame allocation is necessary. + const int compressed = LZ4_compress_fast_extState( + state, reinterpret_cast<const char *>(frame.data()), reinterpret_cast<char *>(stored), + static_cast<int>(frame.size()), static_cast<int>(frame.size() - 1), 1); + if (compressed > 0) { + length = static_cast<uint32_t>(compressed); + algo = compress_algo::kLz4; + } + } +#endif + if (algo == compress_algo::kNone) { + std::memcpy(stored, frame.data(), frame.size()); + } + output[0] = static_cast<uint8_t>(algo); + put_u32(output.data() + 1, static_cast<uint32_t>(frame.size())); + put_u32(output.data() + 5, length); + put_u32(output.data() + 9, crowdb::common::crc32c(stored, length)); + *logical_bytes = static_cast<uint32_t>(kDurableBlobHeader) + length; + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/maptable/frame_page.cpp b/lib/crowdb-tree/src/maptable/frame_page.cpp index 2e2f515c8..2284b72d1 100644 --- a/lib/crowdb-tree/src/maptable/frame_page.cpp +++ b/lib/crowdb-tree/src/maptable/frame_page.cpp @@ -86,18 +86,20 @@ void frame_set_inner_fence_pages(uint8_t *f, uint32_t page_bytes, uint64_t lower PageSummary frame_summary(const uint8_t *f) { PageSummary out; - if (f == nullptr || f[fh::kFormatVersion] != kFrameVersion || f[fh::kSummaryFlags] == 0) { + if (f == nullptr || f[fh::kFormatVersion] != kFrameVersion || f[fh::kSummaryFlags] != 1) { return out; } - out.live_kv = frame_u64(f, fh::kSummaryLiveKv); - out.live_key_bytes = frame_u64(f, fh::kSummaryLiveKeyBytes); - out.live_value_bytes = frame_u64(f, fh::kSummaryLiveValueBytes); - out.live_logical_bytes = frame_u64(f, fh::kSummaryLiveLogicalBytes); - out.reachable_leaf_pages = frame_u64(f, fh::kSummaryLeafPages); - out.reachable_inner_pages = frame_u64(f, fh::kSummaryInnerPages); + out.live_kv = frame_u64(f, fh::kSummaryLiveKv); + out.live_key_bytes = frame_u64(f, fh::kSummaryLiveKeyBytes); + out.live_value_bytes = frame_u64(f, fh::kSummaryLiveValueBytes); + out.live_logical_bytes = frame_u64(f, fh::kSummaryLiveLogicalBytes); + out.reachable_leaf_pages = frame_u64(f, fh::kSummaryLeafPages); + out.reachable_inner_pages = frame_u64(f, fh::kSummaryInnerPages); out.reachable_overflow_pages = frame_u64(f, fh::kSummaryOverflowPages); - out.reachable_page_capacity_bytes = frame_u64(f, fh::kSummaryPageCapacityBytes); - out.exact = true; + // Version one stores seven little-endian capacity bytes followed by the + // exact flag. Masking preserves existing frames without changing offsets. + out.reachable_page_capacity_bytes = frame_u64(f, fh::kSummaryPageCapacityBytes) & 0x00ffffffffffffffULL; + out.exact = true; return out; } @@ -111,7 +113,7 @@ void frame_set_summary(uint8_t *f, uint32_t page_bytes, const PageSummary &summa frame_put_u64(f, fh::kSummaryInnerPages, summary.reachable_inner_pages); frame_put_u64(f, fh::kSummaryOverflowPages, summary.reachable_overflow_pages); frame_put_u64(f, fh::kSummaryPageCapacityBytes, summary.reachable_page_capacity_bytes); - f[fh::kSummaryFlags] = summary.exact ? 1 : 0; + f[fh::kSummaryFlags] = summary.exact && summary.reachable_page_capacity_bytes <= 0x00ffffffffffffffULL ? 1 : 0; frame_restamp_crc(f, page_bytes); } @@ -384,8 +386,12 @@ bool LeafFrameBuilder::try_append_sorted(Slice key, Slice cell) return false; } uint32_t rec_off = free_hi_ - static_cast<uint32_t>(reclen); - std::memcpy(f_ + rec_off, key.data(), key.size()); - std::memcpy(f_ + rec_off + key.size(), cell.data(), cell.size()); + if (!key.empty()) { + std::memcpy(f_ + rec_off, key.data(), key.size()); + } + if (!cell.empty()) { + std::memcpy(f_ + rec_off + key.size(), cell.data(), cell.size()); + } uint8_t *slot = f_ + free_lo_; frame_put_u32(slot, 0, rec_off); frame_put_u32(slot, 4, static_cast<uint32_t>(key.size())); @@ -418,6 +424,12 @@ void LeafFrameBuilder::finish(uint64_t self_page_id, uint64_t right_sibling) bool inner_frame_build(uint8_t *f, uint32_t page_bytes, uint64_t self_page_id, const std::vector<uint64_t> &children, const std::vector<Slice> &separators) +{ + return inner_frame_build_span(f, page_bytes, self_page_id, children, separators); +} + +bool inner_frame_build_span(uint8_t *f, uint32_t page_bytes, uint64_t self_page_id, std::span<const uint64_t> children, + std::span<const Slice> separators) { if (children.size() != separators.size() + 1) { return false; @@ -450,7 +462,9 @@ bool inner_frame_build(uint8_t *f, uint32_t page_bytes, uint64_t self_page_id, c for (uint32_t i = 0; i < nsep; ++i) { Slice s = separators[i]; uint32_t rec_off = free_hi - static_cast<uint32_t>(s.size()); - std::memcpy(f + rec_off, s.data(), s.size()); + if (!s.empty()) { + std::memcpy(f + rec_off, s.data(), s.size()); + } uint8_t *slot = f + (free_lo + (i * kInnerSlotSize)); frame_put_u32(slot, 0, rec_off); frame_put_u32(slot, 4, static_cast<uint32_t>(s.size())); @@ -493,9 +507,13 @@ bool leaf_frame_append_deltas(const uint8_t *src, uint32_t page_bytes, const std const leaf_entry &e = entries[j]; auto reclen = static_cast<uint32_t>(e.key.size() + e.cell.size()); uint32_t rec_off = cur_hi - reclen; - std::memcpy(out + rec_off, e.key.data(), e.key.size()); // NOLINT(bugprone-not-null-terminated-result) - std::memcpy(out + (rec_off + e.key.size()), e.cell.data(), - e.cell.size()); // NOLINT(bugprone-not-null-terminated-result) + const Slice key(e.key); + if (!key.empty()) { + std::memcpy(out + rec_off, key.data(), key.size()); + } + if (!e.cell.empty()) { + std::memcpy(out + (rec_off + key.size()), e.cell.data(), e.cell.size()); + } uint8_t *slot = out + (free_lo + ((delta_count + static_cast<uint32_t>(j)) * kLeafSlotSize)); frame_put_u32(slot, 0, rec_off); frame_put_u32(slot, 4, static_cast<uint32_t>(e.key.size())); diff --git a/lib/crowdb-tree/src/memtable/admission.cpp b/lib/crowdb-tree/src/memtable/admission.cpp index 61c8d07cb..049a9bd39 100644 --- a/lib/crowdb-tree/src/memtable/admission.cpp +++ b/lib/crowdb-tree/src/memtable/admission.cpp @@ -7,11 +7,16 @@ namespace crowdb::tree { +MemTable::FrozenWaiter MemTable::frozen_sentinel_; + bool MemTable::try_enter() noexcept { auto state = writers_.load(std::memory_order_acquire); while ((state & kClosed) == 0 && state != kClosed - 1) { if (writers_.compare_exchange_weak(state, state + 1, std::memory_order_acq_rel)) { + if (list_.source_ledger_) { + list_.source_ledger_->writer_enter(); + } return true; } } @@ -32,16 +37,77 @@ bool MemTable::validate_open() noexcept void MemTable::leave() noexcept { const auto previous = writers_.fetch_sub(1, std::memory_order_acq_rel); +#ifdef CROWDB_TREE_TEST_UTIL + if (list_.test_hook_ != nullptr) { + list_.test_hook_(list_.test_context_, ConcurrentSkipList::PausePoint::kBeforeLeaveAccounting, {}); + } +#endif + if (list_.source_ledger_) { + list_.source_ledger_->writer_leave((previous & kClosed) != 0); + } assert((previous & ~kClosed) != 0); if (previous == kClosed + 1) { writers_.notify_all(); + notify_frozen(); } } void MemTable::close() noexcept { - writers_.fetch_or(kClosed, std::memory_order_acq_rel); + const auto previous = writers_.fetch_or(kClosed, std::memory_order_acq_rel); +#ifdef CROWDB_TREE_TEST_UTIL + if (list_.test_hook_ != nullptr) { + list_.test_hook_(list_.test_context_, ConcurrentSkipList::PausePoint::kBeforeCloseAccounting, {}); + } +#endif + if ((previous & kClosed) == 0 && list_.source_ledger_) { + list_.source_ledger_->table_closed(previous & ~kClosed); + } writers_.notify_all(); + notify_frozen(); +} + +bool MemTable::notify_when_frozen(FrozenWaiter &waiter) noexcept +{ + assert(waiter.complete_ != nullptr); + if (waiter.registered_.test_and_set(std::memory_order_acq_rel)) { + return false; + } + auto *head = frozen_waiters_.load(std::memory_order_acquire); + do { + if (head == &frozen_sentinel_) { + waiter.complete_(waiter.context_); + return true; + } + waiter.next_ = head; + } while ( + !frozen_waiters_.compare_exchange_weak(head, &waiter, std::memory_order_release, std::memory_order_acquire)); + notify_frozen(); + return true; +} + +void MemTable::notify_frozen() noexcept +{ + if (writers_.load(std::memory_order_acquire) != kClosed) { + return; + } + if (list_.owned_epoch_) { + list_.owned_epoch_->seal_source(); + } + if (frozen_notified_.test_and_set(std::memory_order_acq_rel)) { + return; + } + if (list_.source_ledger_) { + list_.source_ledger_->table_frozen(approx_bytes()); + } + auto *waiter = frozen_waiters_.exchange(&frozen_sentinel_, std::memory_order_acq_rel); + while (waiter != nullptr && waiter != &frozen_sentinel_) { + auto *next = waiter->next_; + auto complete = waiter->complete_; + auto *context = waiter->context_; + complete(context); + waiter = next; + } } void MemTable::wait_frozen() const noexcept diff --git a/lib/crowdb-tree/src/memtable/cell_version.cpp b/lib/crowdb-tree/src/memtable/cell_version.cpp new file mode 100644 index 000000000..1beb3059c --- /dev/null +++ b/lib/crowdb-tree/src/memtable/cell_version.cpp @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/memtable/skip_list.h" + +namespace crowdb::tree +{ +std::shared_ptr<CellVersion> CellVersion::create(buffer bytes, uint64_t version, uint8_t kind, + std::shared_ptr<AllocationCounter> counter, AllocationLease credit, + const std::shared_ptr<AllocationBudget> &budget) +{ + return std::allocate_shared<CellVersion>(MetadataAllocator<CellVersion>(budget), std::move(bytes), version, kind, + std::move(counter), std::move(credit)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/memtable/memtable.cpp b/lib/crowdb-tree/src/memtable/memtable.cpp index 055029e7b..11dbc47ca 100644 --- a/lib/crowdb-tree/src/memtable/memtable.cpp +++ b/lib/crowdb-tree/src/memtable/memtable.cpp @@ -5,6 +5,8 @@ #include "crowdb-tree/memtable/memtable.h" +#include "crowdb-tree/external_allocation.h" + #include <algorithm> #include <cstring> #include <utility> @@ -23,6 +25,25 @@ buffer buf_of(Slice s) return b; } +AllocationLease retain_payload(buffer &payload, const std::shared_ptr<AllocationBudget> &budget) +{ + if (!budget) { + return {}; + } + size_t capacity = payload.owned() ? payload.capacity() : payload.size(); + if (payload.inlined() || (!payload.owned() && capacity <= buffer::kInlineCap)) { + capacity = 0; + } + AllocationLease credit(budget, capacity, AllocationKind::kPayload); + // Legacy borrowed/external buffers expose no allocation identity/capacity. + // A budgeted source makes a controlled copy instead of treating slice + // length as the physical size of an unknown retained allocation. + if (!payload.owned()) { + payload = payload.clone(); + } + return credit; +} + // Materialize a contiguous [header][value] cell from a CellVersion (R30 split // cell support). Contiguous: clone. Split (kExternal): build header from // slot/flags + copy value. @@ -50,8 +71,12 @@ bool MemTable::upsert(Slice key, uint64_t slot, Slice cell_payload) return upsert(key, slot, buf_of(cell_payload)); } -bool MemTable::upsert(Slice key, uint64_t slot, buffer &&cell_payload, uint64_t bound, MutationStats *stats) +bool MemTable::upsert(Slice key, uint64_t slot, buffer &&cell_payload, uint64_t bound, MutationStats *stats, + std::shared_ptr<AllocationBudget> budget) { + if (!budget) { + budget = list_.budget(); + } if (stats != nullptr) { stats->admission = &writers_; } @@ -61,8 +86,10 @@ bool MemTable::upsert(Slice key, uint64_t slot, buffer &&cell_payload, uint64_t CellView cv{cell_payload.slice()}; uint64_t entry_slot = cv.valid() ? cv.slot() : slot; uint8_t flags = cv.valid() ? cv.flags() : 0; - auto ver = std::make_shared<CellVersion>(std::move(cell_payload), entry_slot, flags, epoch_->memtable_allocation()); - if (!list_.upsert(key, std::move(ver), bound, stats)) { + auto credit = retain_payload(cell_payload, budget); + auto ver = CellVersion::create(std::move(cell_payload), entry_slot, flags, epoch_->memtable_allocation(), + std::move(credit), budget); + if (!list_.upsert(key, std::move(ver), bound, stats, std::move(budget))) { return false; } update_slot_range(entry_slot); @@ -70,14 +97,41 @@ bool MemTable::upsert(Slice key, uint64_t slot, buffer &&cell_payload, uint64_t } bool MemTable::upsert_external(Slice key, uint64_t slot, uint8_t flags, buffer &&value, uint64_t bound, - MutationStats *stats) + MutationStats *stats, std::shared_ptr<AllocationBudget> budget) { + if (!budget) { + budget = list_.budget(); + } if (stats != nullptr) { stats->admission = &writers_; } if (slot <= durable_floor_.load(std::memory_order_relaxed) && !allow_old_slots_.load(std::memory_order_relaxed)) { return false; } + const auto *allocation = value.external_allocation(); + if (budget && (allocation == nullptr || !allocation->charged_to(budget))) { + if (value.size() > std::numeric_limits<size_t>::max() - kCellHeaderSize) { + throw std::bad_alloc(); + } + const auto capacity = value.size() + kCellHeaderSize; + AllocationLease credit(budget, capacity <= buffer::kInlineCap ? 0 : capacity, AllocationKind::kPayload); + auto encoded = buffer::alloc(value.size(), kCellHeaderSize); + for (size_t i = 0; i < sizeof(slot); ++i) { + encoded.data()[i] = static_cast<uint8_t>(slot >> (8 * i)); + } + encoded.data()[sizeof(slot)] = flags; + if (!value.empty()) { + std::memcpy(encoded.data() + kCellHeaderSize, value.data(), value.size()); + } + value = {}; + auto ver = CellVersion::create(std::move(encoded), slot, flags, epoch_->memtable_allocation(), + std::move(credit), budget); + if (!list_.upsert(key, std::move(ver), bound, stats, std::move(budget))) { + return false; + } + update_slot_range(slot); + return true; + } // upsert_external stores the raw value (no 9-byte cell header); the // header is reconstructed from slot/flags at read time. Tag non-kExternal // buffers as kExternal so get_view/scan treat them as split cells. @@ -91,8 +145,8 @@ bool MemTable::upsert_external(Slice key, uint64_t slot, uint8_t flags, buffer & value = buffer::wrap_external(nullptr, 0, nullptr, nullptr); } } - auto ver = std::make_shared<CellVersion>(std::move(value), slot, flags, epoch_->memtable_allocation()); - if (!list_.upsert(key, std::move(ver), bound, stats)) { + auto ver = CellVersion::create(std::move(value), slot, flags, epoch_->memtable_allocation(), {}, budget); + if (!list_.upsert(key, std::move(ver), bound, stats, std::move(budget))) { return false; } update_slot_range(slot); @@ -109,7 +163,7 @@ void MemTable::set_durable_floor(uint64_t slot) std::vector<mem_entry> MemTable::snapshot(uint64_t frontier) const { - auto guard = epoch_->enter(); + auto guard = read_guard(); std::vector<mem_entry> out; auto cur = list_.prefix_cursor(frontier); while (cur.valid()) { diff --git a/lib/crowdb-tree/src/memtable/prepared_cell.cpp b/lib/crowdb-tree/src/memtable/prepared_cell.cpp new file mode 100644 index 000000000..709d465fa --- /dev/null +++ b/lib/crowdb-tree/src/memtable/prepared_cell.cpp @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/memtable/memtable.h" + +namespace crowdb::tree +{ +MemTable::PreparedWrite MemTable::prepare_write(Slice key, buffer cell, uint8_t flags, AllocationLease payload, + uint64_t bound, size_t max_versions, + const std::shared_ptr<AllocationBudget> &budget) +{ + auto version = + CellVersion::create(std::move(cell), 0, flags, epoch_->memtable_allocation(), std::move(payload), budget); + return list_.prepare_write(key, std::move(version), bound, max_versions, budget); +} + +MemTable::PreparedWrite MemTable::prepare_reference(Slice key, std::shared_ptr<const CellVersion> cell, uint64_t bound, + size_t max_versions, + const std::shared_ptr<AllocationBudget> &budget) +{ + return list_.prepare_reference(key, std::move(cell), bound, max_versions, budget); +} + +bool MemTable::commit_write(PreparedWrite write, uint64_t slot) noexcept +{ + if (slot <= durable_floor_.load(std::memory_order_relaxed) && !allow_old_slots_.load(std::memory_order_relaxed)) { + return false; + } + const bool changed = list_.commit_write(std::move(write), slot); + if (changed) { + update_slot_range(slot); + } + return changed; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/memtable/prepared_write.cpp b/lib/crowdb-tree/src/memtable/prepared_write.cpp new file mode 100644 index 000000000..a9046c33b --- /dev/null +++ b/lib/crowdb-tree/src/memtable/prepared_write.cpp @@ -0,0 +1,208 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/memtable/skip_list.h" + +#include <algorithm> +#include <array> +#include <cassert> +#include <exception> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +void append_prepared(VersionSet &versions, const std::shared_ptr<const CellVersion> &version) noexcept +{ + // Never let an invalid admission bound turn commit into vector growth. + // shared_ptr copy itself cannot throw once capacity is already present. + if (versions.history.size() == versions.history.capacity()) { + std::terminate(); + } + try { + versions.history.push_back(version); + } + catch (...) { + std::terminate(); + } +} + +void merge_prepared(VersionSet &fresh, const VersionSet &old, const std::shared_ptr<const CellVersion> &incoming, + uint64_t bound) noexcept +{ + assert(fresh.history.capacity() > old.history.size()); + fresh.history.clear(); + fresh.bound = std::max(bound, old.bound); + fresh.current = incoming->slot > old.current->slot ? incoming : old.current; + bool anchor = fresh.current->slot <= fresh.bound; + const auto retain = [&](const auto &version) { + if (version == fresh.current) { + return; + } + if (version->slot <= fresh.bound) { + if (anchor) { + return; + } + anchor = true; + } + append_prepared(fresh, version); + }; + bool pending = incoming != fresh.current; + const auto visit = [&](const auto &version) { + if (pending && incoming->slot > version->slot) { + retain(incoming); + pending = false; + } + retain(version); + }; + visit(old.current); + for (const auto &version : old.history) { + visit(version); + } + if (pending) { + retain(incoming); + } + fresh.bytes = fresh.current->cell.size(); + for (const auto &version : fresh.history) { + fresh.bytes += version->cell.size(); + } +} +} // namespace + +void ConcurrentSkipList::PreparedWrite::NodeDeleter::operator()(Node *node) const noexcept +{ + if (node != nullptr) { + ConcurrentSkipList::free_node(node); + } +} + +ConcurrentSkipList::PreparedWrite ConcurrentSkipList::prepare_write(Slice key, std::shared_ptr<CellVersion> cell, + uint64_t bound, size_t max_versions, + const std::shared_ptr<AllocationBudget> &budget) +{ + auto result = prepare_reference(key, std::move(cell), bound, max_versions, budget); + result.initialize_slot_ = true; + return result; +} + +ConcurrentSkipList::PreparedWrite ConcurrentSkipList::prepare_reference(Slice key, + std::shared_ptr<const CellVersion> cell, + uint64_t bound, size_t max_versions, + const std::shared_ptr<AllocationBudget> &budget) +{ + if (max_versions == 0 || !cell || epoch_->source_sealed()) { + throw std::invalid_argument("prepared write needs an admitted version bound"); + } +#ifdef CROWDB_TREE_TEST_UTIL + if (test_hook_ != nullptr) { + test_hook_(test_context_, PausePoint::kBeforePrepare, key); + } +#endif + PreparedWrite result; + result.initialize_slot_ = false; + result.guard_ = epoch_->enter(budget); + result.cell_ = std::move(cell); + result.bound_ = bound; + result.initial_ = VersionSet::create(budget); + result.initial_->current = result.cell_; + result.initial_->bound = bound; + result.initial_->bytes = result.cell_->cell.size(); + result.initial_->allocation.set(epoch_->memtable_allocation(), sizeof(VersionSet)); + result.replacement_ = VersionSet::create(budget); + result.replacement_->history.reserve(max_versions); + result.replacement_->allocation.set( + epoch_->memtable_allocation(), + sizeof(VersionSet) + (result.replacement_->history.capacity() * sizeof(result.replacement_->current))); + result.candidate_.reset(alloc_node(random_height(), key, budget)); + result.candidate_->allocation.set(epoch_->memtable_allocation(), + Node::alloc_size(result.candidate_->height_, key.size())); + result.candidate_->versions_.store(result.initial_.get(), std::memory_order_relaxed); + return result; +} + +bool ConcurrentSkipList::commit_write(PreparedWrite write, uint64_t slot) noexcept +{ + assert(write.cell_ && write.candidate_); + if (epoch_->source_sealed()) { + return false; + } + if (write.initialize_slot_) { + // This is the parent's private, not-yet-published cell. Referenced + // delta writes observe the completed parent CAS and never mutate it. + auto &cell = *std::const_pointer_cast<CellVersion>(write.cell_); + cell.slot = slot; + if (cell.cell.ownership() != buffer::mode::kExternal) { + for (size_t index = 0; index < sizeof(slot); ++index) { + cell.cell.data()[index] = static_cast<uint8_t>(slot >> (8 * index)); + } + } + } + else { + assert(write.cell_->slot == slot); + } + const auto key = write.candidate_->key_slice(); + std::array<Node *, kMaxHeight> previous{}; + std::array<Node *, kMaxHeight> next{}; + Node *existing = nullptr; +#ifdef CROWDB_TREE_TEST_UTIL + if (test_hook_ != nullptr) { + test_hook_(test_context_, PausePoint::kBeforeSearch, key); + } +#endif + for (;;) { + find_ge(key, previous.data(), next.data()); + if (next[0] != nullptr && next[0]->key_slice() == key) { + existing = next[0]; + break; + } + write.candidate_->set_next(0, next[0]); + if (previous[0]->next_ptr(0)->compare_exchange_strong(next[0], write.candidate_.get(), + std::memory_order_acq_rel)) { + auto *published = write.candidate_.release(); + auto *initial = write.initial_.release(); +#ifdef CROWDB_TREE_TEST_UTIL + if (test_hook_ != nullptr) { + test_hook_(test_context_, PausePoint::kBeforeInsertAccounting, key); + } +#endif + count_.fetch_add(1, std::memory_order_relaxed); + bytes_.fetch_add(key.size() + initial->bytes, std::memory_order_relaxed); + account_node(*published, *initial); +#ifdef CROWDB_TREE_TEST_UTIL + if (test_hook_ != nullptr) { + test_hook_(test_context_, PausePoint::kAfterLevelZero, key); + } +#endif + link_upper(published); + return true; + } + } + auto *old = existing->versions_.load(std::memory_order_acquire); + const std::shared_ptr<const CellVersion> incoming = write.cell_; + for (;;) { + const auto *same = old->at(slot); + if (same != nullptr && same->slot == slot) { + return false; + } + merge_prepared(*write.replacement_, *old, incoming, write.bound_); + const bool accepted = write.replacement_->at(slot) == incoming.get(); + if (!accepted && write.replacement_->history.size() == old->history.size() && + write.replacement_->bound == old->bound) { + return false; + } + auto *replacement = write.replacement_.get(); +#ifdef CROWDB_TREE_TEST_UTIL + if (test_hook_ != nullptr) { + test_hook_(test_context_, PausePoint::kBeforeVersionCas, key); + } +#endif + if (existing->versions_.compare_exchange_weak(old, replacement, std::memory_order_acq_rel)) { + [[maybe_unused]] auto *published = write.replacement_.release(); + bytes_.fetch_add(replacement->bytes - old->bytes, std::memory_order_relaxed); + account_versions(old, *replacement); + epoch_->defer(old); + return accepted; + } + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/memtable/skip_list.cpp b/lib/crowdb-tree/src/memtable/skip_list.cpp index a187b2222..7b4bd862f 100644 --- a/lib/crowdb-tree/src/memtable/skip_list.cpp +++ b/lib/crowdb-tree/src/memtable/skip_list.cpp @@ -22,17 +22,19 @@ constexpr uint32_t kBranching = 4; // 1/4 probability of height increase // --- Node allocation --- -Node *ConcurrentSkipList::alloc_node(uint32_t height, Slice key) +Node *ConcurrentSkipList::alloc_node(uint32_t height, Slice key, const std::shared_ptr<AllocationBudget> &budget) { - size_t sz = Node::alloc_size(height, key.size()); - void *p = ::operator new(sz); + size_t sz = Node::alloc_size(height, key.size()); + AllocationLease credit(budget, sz, AllocationKind::kMetadata); + void *p = ::operator new(sz); // Construct the node metadata with // placement new — the atomics need proper initialization, not just raw // memory. The tower (next_ptr) is constructed separately below since it // lives beyond sizeof(Node). - Node *n = new (p) Node{}; - n->height_ = height; - n->key_len_ = static_cast<uint32_t>(key.size()); + Node *n = new (p) Node{}; + n->allocation_credit = std::move(credit); + n->height_ = height; + n->key_len_ = static_cast<uint32_t>(key.size()); // Construct the tower (next_[0..height-1]) with placement new. for (uint32_t i = 0; i < height; ++i) { new (n->next_ptr(i)) std::atomic<Node *>(nullptr); @@ -46,7 +48,8 @@ Node *ConcurrentSkipList::alloc_node(uint32_t height, Slice key) void ConcurrentSkipList::free_node(void *p) { - Node *n = static_cast<Node *>(p); + Node *n = static_cast<Node *>(p); + auto credit = std::move(n->allocation_credit); // Destroy the atomic objects in the tower, then the Node base, then free. for (uint32_t i = 0; i < n->height_; ++i) { n->next_ptr(i)->~atomic(); @@ -57,26 +60,36 @@ void ConcurrentSkipList::free_node(void *p) // --- ConcurrentSkipList --- -ConcurrentSkipList::ConcurrentSkipList(EpochManager *epoch) - : owned_epoch_(epoch == nullptr ? std::make_unique<EpochManager>() : nullptr), - epoch_(epoch == nullptr ? owned_epoch_.get() : epoch) +ConcurrentSkipList::ConcurrentSkipList(EpochManager *epoch, std::shared_ptr<AllocationBudget> budget) + : epoch_credit_(budget, epoch == nullptr ? sizeof(EpochManager) : 0, AllocationKind::kMetadata), + owned_epoch_(epoch == nullptr ? std::make_unique<EpochManager>(budget, true) : nullptr), + epoch_(epoch == nullptr ? owned_epoch_.get() : epoch), + budget_(std::move(budget)) { - head_ = alloc_node(kMaxHeight, Slice()); + head_ = alloc_node(kMaxHeight, Slice(), budget_); } -ConcurrentSkipList::~ConcurrentSkipList() +bool ConcurrentSkipList::retire_step() noexcept { - Node *node = head_->next(0); - while (node != nullptr) { - Node *next = node->next(0); + auto *node = head_->next(0); + for (size_t step = 0; node != nullptr && step < 64; ++step) { + auto *next = node->next(0); node->destroy = [](EpochManager::Deferred *entry) noexcept { auto *n = static_cast<Node *>(entry); - delete n->versions_.load(std::memory_order_relaxed); + VersionSet::Deleter{}(n->versions_.load(std::memory_order_relaxed)); free_node(n); }; epoch_->defer(node); node = next; } + head_->set_next(0, node); + return node == nullptr; +} + +ConcurrentSkipList::~ConcurrentSkipList() +{ + while (!retire_step()) { + } free_node(head_); } @@ -131,7 +144,7 @@ Node *ConcurrentSkipList::find_or_insert(Slice key, VersionSet *versions, bool * return next[0]; } if (candidate == nullptr) { - candidate = alloc_node(random_height(), key); + candidate = alloc_node(random_height(), key, versions->history.get_allocator().owner()); candidate->allocation.set(epoch_->memtable_allocation(), Node::alloc_size(candidate->height_, key.size())); candidate->versions_.store(versions, std::memory_order_relaxed); } @@ -139,6 +152,7 @@ Node *ConcurrentSkipList::find_or_insert(Slice key, VersionSet *versions, bool * if (prev[0]->next_ptr(0)->compare_exchange_strong(next[0], candidate, std::memory_order_acq_rel)) { count_.fetch_add(1, std::memory_order_relaxed); bytes_.fetch_add(key.size() + versions->bytes, std::memory_order_relaxed); + account_node(*candidate, *versions); #ifdef CROWDB_TREE_TEST_UTIL if (test_hook_ != nullptr) { test_hook_(test_context_, PausePoint::kAfterLevelZero, key); @@ -243,15 +257,23 @@ ConcurrentSkipList::Cursor ConcurrentSkipList::cursor_reverse(Slice start_key, b void ConcurrentSkipList::Cursor::select() { - candidate_ = nullptr; + candidate_ = nullptr; + candidate_owner_ = nullptr; while (cur_ != nullptr) { - auto *versions = cur_->versions_.load(std::memory_order_acquire); - candidate_ = versions->at(frontier_); + if (upper_.has_value() && cur_->key_slice() >= *upper_) { + cur_ = nullptr; + return; + } + auto *versions = cur_->versions_.load(std::memory_order_acquire); + candidate_owner_ = versions->owner_at(frontier_); + candidate_ = candidate_owner_ != nullptr ? candidate_owner_->get() : nullptr; if (candidate_ != nullptr && (floor_ == 0 || candidate_->slot > floor_)) { return; } cur_ = cur_->next(0); } + candidate_ = nullptr; + candidate_owner_ = nullptr; } void ConcurrentSkipList::Cursor::advance() @@ -267,4 +289,13 @@ ConcurrentSkipList::Cursor ConcurrentSkipList::prefix_cursor(uint64_t frontier, return Cursor(head_->next(0), frontier, floor); } +ConcurrentSkipList::Cursor ConcurrentSkipList::range_cursor(std::optional<Slice> lower, std::optional<Slice> upper, + uint64_t frontier, uint64_t floor) const +{ + if (floor > frontier || (lower.has_value() && upper.has_value() && *lower >= *upper)) { + return {}; + } + return Cursor(lower.has_value() ? find_ge(*lower, nullptr) : head_->next(0), frontier, floor, upper); +} + } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/memtable/source_membership.cpp b/lib/crowdb-tree/src/memtable/source_membership.cpp new file mode 100644 index 000000000..f95ece7da --- /dev/null +++ b/lib/crowdb-tree/src/memtable/source_membership.cpp @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/memtable/memtable.h" + +#include <cassert> + +namespace crowdb::tree +{ +MemTable::~MemTable() +{ + if (source_attached_) { + list_.source_ledger_->replace(source_stats(), {}); + } +} + +SourceMemoryStats MemTable::source_stats() const noexcept +{ + const auto state = writers_.load(std::memory_order_acquire); + const auto writer_count = state & ~kClosed; + const bool frozen = (state & kClosed) != 0; + return {.tables = 1, + .frozen_tables = frozen ? 1U : 0U, + .writers = writer_count, + .freezing_writers = frozen ? writer_count : 0, + .frozen_retained_bytes = frozen && writer_count == 0 ? approx_bytes() : 0, + .records = count(), + .logical_bytes = approx_bytes(), + .memory = memory(), + .known = true}; +} + +void MemTable::attach_source_stats(std::shared_ptr<SourceMemoryLedger> ledger) noexcept +{ + assert(!list_.source_ledger_ && ledger && !closed() && writers() == 0); + ledger->replace({}, source_stats()); + list_.source_ledger_ = std::move(ledger); + source_attached_ = true; +} + +void MemTable::detach_source_stats() noexcept +{ + assert(closed() && writers() == 0); + if (source_attached_) { + list_.source_ledger_->replace(source_stats(), {}); + // Keep the ledger pointer immutable: close/last-writer bookkeeping + // may finish after another thread observed the drained writer state. + // Their additive deltas still settle into this same generation. + source_attached_ = false; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/memtable/source_memory.cpp b/lib/crowdb-tree/src/memtable/source_memory.cpp new file mode 100644 index 000000000..85f402abf --- /dev/null +++ b/lib/crowdb-tree/src/memtable/source_memory.cpp @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/memtable/skip_list.h" + +#include <limits> + +namespace crowdb::tree +{ +namespace +{ +VersionMemory version_memory(const VersionSet *versions) noexcept +{ + if (versions == nullptr) { + return {}; + } + return {.history_count = versions->history.size(), + .history_bytes = versions->bytes - versions->current->cell.size(), + .descriptor_bytes = sizeof(VersionSet) + (versions->history.capacity() * sizeof(versions->current)), + .node_bytes = 0, + .payload_bytes = versions->bytes + ((versions->history.size() + 1) * sizeof(CellVersion))}; +} + +uint64_t sample(const std::atomic<uint64_t> &counter) noexcept +{ + const auto value = counter.load(std::memory_order_relaxed); + // A later CAS may subtract its predecessor before the earlier CAS adds + // it. Such an independent sample is zero, never a wrapped huge capacity. + return value <= static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) ? value : 0; +} +} // namespace + +void ConcurrentSkipList::account_versions(const VersionSet *previous, const VersionSet &next) noexcept +{ + const auto before = version_memory(previous); + const auto after = version_memory(&next); + // Deltas commute even when a later successful CAS finishes accounting + // first. The independent samples converge once accepted writers drain. + history_count_.fetch_add(after.history_count - before.history_count, std::memory_order_relaxed); + history_bytes_.fetch_add(after.history_bytes - before.history_bytes, std::memory_order_relaxed); + descriptor_bytes_.fetch_add(after.descriptor_bytes - before.descriptor_bytes, std::memory_order_relaxed); + payload_bytes_.fetch_add(after.payload_bytes - before.payload_bytes, std::memory_order_relaxed); + if (source_ledger_) { + source_ledger_->replace({.logical_bytes = previous != nullptr ? previous->bytes : 0, .memory = before}, + {.logical_bytes = next.bytes, .memory = after}); + } +} + +void ConcurrentSkipList::account_node(const Node &node, const VersionSet &versions) noexcept +{ + node_bytes_.fetch_add(Node::alloc_size(node.height_, node.key_len_), std::memory_order_relaxed); + account_versions(nullptr, versions); + if (source_ledger_) { + source_ledger_->replace({}, {.records = 1, + .logical_bytes = node.key_len_, + .memory = {.node_bytes = Node::alloc_size(node.height_, node.key_len_)}}); + } +} + +VersionMemory ConcurrentSkipList::memory() const +{ + return {.history_count = sample(history_count_), + .history_bytes = sample(history_bytes_), + .descriptor_bytes = sample(descriptor_bytes_), + .node_bytes = sample(node_bytes_), + .payload_bytes = sample(payload_bytes_)}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/memtable/source_stats.cpp b/lib/crowdb-tree/src/memtable/source_stats.cpp new file mode 100644 index 000000000..8caf2e621 --- /dev/null +++ b/lib/crowdb-tree/src/memtable/source_stats.cpp @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/memtable/source_stats.h" + +#include <limits> + +namespace crowdb::tree +{ +namespace +{ +uint64_t sample(const std::atomic<uint64_t> &value) noexcept +{ + const auto count = value.load(std::memory_order_relaxed); + return count <= static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) ? count : 0; +} +} // namespace + +SourceMemoryStats &SourceMemoryStats::operator+=(const SourceMemoryStats &other) noexcept +{ + tables += other.tables; + frozen_tables += other.frozen_tables; + writers += other.writers; + freezing_writers += other.freezing_writers; + frozen_retained_bytes += other.frozen_retained_bytes; + records += other.records; + logical_bytes += other.logical_bytes; + memory.history_count += other.memory.history_count; + memory.history_bytes += other.memory.history_bytes; + memory.descriptor_bytes += other.memory.descriptor_bytes; + memory.node_bytes += other.memory.node_bytes; + memory.payload_bytes += other.memory.payload_bytes; + known = known && other.known; + return *this; +} + +SourceMemoryStats SourceMemoryLedger::snapshot() const noexcept +{ + return { + .tables = sample(tables_), + .frozen_tables = sample(frozen_tables_), + .writers = sample(writers_), + .freezing_writers = sample(freezing_writers_), + .frozen_retained_bytes = sample(frozen_retained_bytes_), + .records = sample(records_), + .logical_bytes = sample(logical_bytes_), + .memory = + { + .history_count = sample(history_count_), + .history_bytes = sample(history_bytes_), + .descriptor_bytes = sample(descriptor_bytes_), + .node_bytes = sample(node_bytes_), + .payload_bytes = sample(payload_bytes_), + }, + .known = true, + }; +} + +void SourceMemoryLedger::replace(const SourceMemoryStats &before, const SourceMemoryStats &after) noexcept +{ + tables_.fetch_add(after.tables - before.tables, std::memory_order_relaxed); + frozen_tables_.fetch_add(after.frozen_tables - before.frozen_tables, std::memory_order_relaxed); + writers_.fetch_add(after.writers - before.writers, std::memory_order_relaxed); + freezing_writers_.fetch_add(after.freezing_writers - before.freezing_writers, std::memory_order_relaxed); + frozen_retained_bytes_.fetch_add(after.frozen_retained_bytes - before.frozen_retained_bytes, + std::memory_order_relaxed); + records_.fetch_add(after.records - before.records, std::memory_order_relaxed); + logical_bytes_.fetch_add(after.logical_bytes - before.logical_bytes, std::memory_order_relaxed); + history_count_.fetch_add(after.memory.history_count - before.memory.history_count, std::memory_order_relaxed); + history_bytes_.fetch_add(after.memory.history_bytes - before.memory.history_bytes, std::memory_order_relaxed); + descriptor_bytes_.fetch_add(after.memory.descriptor_bytes - before.memory.descriptor_bytes, + std::memory_order_relaxed); + node_bytes_.fetch_add(after.memory.node_bytes - before.memory.node_bytes, std::memory_order_relaxed); + payload_bytes_.fetch_add(after.memory.payload_bytes - before.memory.payload_bytes, std::memory_order_relaxed); +} + +void SourceMemoryLedger::writer_enter() noexcept +{ + writers_.fetch_add(1, std::memory_order_relaxed); +} + +void SourceMemoryLedger::writer_leave(bool closed) noexcept +{ + writers_.fetch_sub(1, std::memory_order_relaxed); + if (closed) { + freezing_writers_.fetch_sub(1, std::memory_order_relaxed); + } +} + +void SourceMemoryLedger::table_closed(uint64_t writers) noexcept +{ + frozen_tables_.fetch_add(1, std::memory_order_relaxed); + freezing_writers_.fetch_add(writers, std::memory_order_relaxed); +} + +void SourceMemoryLedger::table_frozen(uint64_t bytes) noexcept +{ + frozen_retained_bytes_.fetch_add(bytes, std::memory_order_relaxed); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/memtable/versions.cpp b/lib/crowdb-tree/src/memtable/versions.cpp index 24172d823..25768f516 100644 --- a/lib/crowdb-tree/src/memtable/versions.cpp +++ b/lib/crowdb-tree/src/memtable/versions.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. +#include "btree/retirement_queue.h" #include "crowdb-tree/memtable/skip_list.h" #include <algorithm> @@ -9,19 +10,59 @@ namespace crowdb::tree { -VersionSet::VersionSet() +VersionSet::VersionSet(AllocationLease credit, std::shared_ptr<AllocationBudget> budget) + : descriptor_credit(std::move(credit)), + retirement(budget ? budget->retirement_queue() : nullptr), + history(MetadataAllocator<std::shared_ptr<const CellVersion>>(budget)) { - destroy = [](EpochManager::Deferred *entry) noexcept { delete static_cast<VersionSet *>(entry); }; + destroy = [](EpochManager::Deferred *entry) noexcept { Deleter{}(static_cast<VersionSet *>(entry)); }; + retirement_budget = std::move(budget); + dispose = [](detail::RetirementNode *entry) noexcept { + auto *versions = static_cast<VersionSet *>(entry); + for (size_t i = 0; i < 64 && !versions->history.empty(); ++i) { + versions->history.pop_back(); + } + if (!versions->history.empty()) { + return false; + } + auto credit = std::move(versions->descriptor_credit); + delete versions; + return true; + }; +} + +void VersionSet::Deleter::operator()(VersionSet *versions) const noexcept +{ + if (versions != nullptr && versions->retirement) { + versions->retirement->retire(versions); + } + else if (versions != nullptr) { + auto credit = std::move(versions->descriptor_credit); + delete versions; + } +} + +VersionSet::Owner VersionSet::create(const std::shared_ptr<AllocationBudget> &budget) +{ + AllocationLease credit(budget, sizeof(VersionSet), AllocationKind::kMetadata); + auto result = Owner(new VersionSet(std::move(credit), budget)); + return result; } const CellVersion *VersionSet::at(uint64_t frontier) const +{ + const auto *owner = owner_at(frontier); + return owner != nullptr ? owner->get() : nullptr; +} + +const std::shared_ptr<const CellVersion> *VersionSet::owner_at(uint64_t frontier) const { if (current->slot <= frontier) { - return current.get(); + return ¤t; } for (const auto &version : history) { if (version->slot <= frontier) { - return version.get(); + return &version; } } return nullptr; @@ -53,11 +94,11 @@ struct CopyTimer } }; -std::unique_ptr<VersionSet> merged_versions(const VersionSet &old, const std::shared_ptr<const CellVersion> &incoming, - uint64_t bound, MutationStats *stats) +VersionSet::Owner merged_versions(const VersionSet &old, const std::shared_ptr<const CellVersion> &incoming, + uint64_t bound, MutationStats *stats, const std::shared_ptr<AllocationBudget> &budget) { CopyTimer timer(stats); - auto fresh = std::make_unique<VersionSet>(); + auto fresh = VersionSet::create(budget); fresh->bound = std::max(bound, old.bound); fresh->current = incoming != nullptr && incoming->slot > old.current->slot ? incoming : old.current; bool anchor = fresh->current->slot <= fresh->bound; @@ -71,6 +112,12 @@ std::unique_ptr<VersionSet> merged_versions(const VersionSet &old, const std::sh } anchor = true; } + if (fresh->history.empty()) { + // The immutable input bounds this candidate. Keep common one-cell + // descriptors allocation-free, and allocate one history capacity + // once a second retained version is actually needed. + fresh->history.reserve(old.history.size() + 1); + } fresh->history.push_back(version); }; bool pending = incoming != nullptr && incoming != fresh->current; @@ -97,7 +144,7 @@ std::unique_ptr<VersionSet> merged_versions(const VersionSet &old, const std::sh } // namespace bool ConcurrentSkipList::replace(Node *node, std::shared_ptr<const CellVersion> cv, uint64_t bound, - MutationStats *stats) + MutationStats *stats, const std::shared_ptr<AllocationBudget> &budget) { auto *old = node->versions_.load(std::memory_order_acquire); for (;;) { @@ -107,7 +154,7 @@ bool ConcurrentSkipList::replace(Node *node, std::shared_ptr<const CellVersion> return false; } } - auto fresh = merged_versions(*old, cv, bound, stats); + auto fresh = merged_versions(*old, cv, bound, stats, budget); const bool accepted = cv != nullptr && fresh->at(cv->slot) == cv.get(); if (!accepted && fresh->history.size() == old->history.size() && fresh->bound == old->bound) { return false; @@ -129,6 +176,7 @@ bool ConcurrentSkipList::replace(Node *node, std::shared_ptr<const CellVersion> published = fresh.release(); // Unsigned add/sub deltas commute even when a later CAS accounts first. bytes_.fetch_add(published->bytes - old->bytes, std::memory_order_relaxed); + account_versions(old, *published); if (stats != nullptr) { stats->overwrite += static_cast<uint64_t>(accepted && cv->slot > old->current->slot); const auto retained_incoming = static_cast<size_t>(accepted && cv->slot < published->current->slot); @@ -161,14 +209,21 @@ bool ConcurrentSkipList::replace(Node *node, std::shared_ptr<const CellVersion> } } -bool ConcurrentSkipList::upsert(Slice key, std::shared_ptr<const CellVersion> cv, uint64_t bound, MutationStats *stats) +bool ConcurrentSkipList::upsert(Slice key, std::shared_ptr<const CellVersion> cv, uint64_t bound, MutationStats *stats, + std::shared_ptr<AllocationBudget> budget) { - auto guard = epoch_->enter(); + if (epoch_->source_sealed()) { + return false; + } + if (!budget) { + budget = budget_; + } + auto guard = epoch_->enter(budget); auto *existing = find_ge(key, nullptr); if (existing != nullptr && existing->key_slice().compare(key) == 0) { - return replace(existing, std::move(cv), bound, stats); + return replace(existing, std::move(cv), bound, stats, budget); } - auto initial = std::make_unique<VersionSet>(); + auto initial = VersionSet::create(budget); initial->current = cv; initial->bound = bound; initial->bytes = cv->cell.size(); @@ -179,19 +234,23 @@ bool ConcurrentSkipList::upsert(Slice key, std::shared_ptr<const CellVersion> cv [[maybe_unused]] auto *published = initial.release(); return true; } - return replace(node, std::move(cv), bound, stats); + return replace(node, std::move(cv), bound, stats, budget); } void ConcurrentSkipList::prune(Slice key, uint64_t bound, MutationStats *stats) { + if (epoch_->source_sealed()) { + return; + } auto guard = epoch_->enter(); auto *node = find_ge(key, nullptr); if (node != nullptr && node->key_slice().compare(key) == 0) { - replace(node, nullptr, bound, stats); + replace(node, nullptr, bound, stats, budget_); } } -VersionMemory ConcurrentSkipList::memory() const +#ifdef CROWDB_TREE_TEST_UTIL +VersionMemory ConcurrentSkipList::scan_memory_for_tests() const { auto guard = epoch_->enter(); VersionMemory result; @@ -207,4 +266,6 @@ VersionMemory ConcurrentSkipList::memory() const } return result; } +#endif + } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/allocation_memory_resource.cpp b/lib/crowdb-tree/src/runtime/allocation_memory_resource.cpp new file mode 100644 index 000000000..de7fd62f5 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/allocation_memory_resource.cpp @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_memory_resource.h" + +#include <new> + +namespace crowdb::tree::detail +{ +void *AllocationMemoryResource::do_allocate(size_t bytes, size_t alignment) +{ + if (!budget_ || !budget_->claim(bytes)) { + throw std::bad_alloc(); + } + try { + auto *pointer = ::operator new(bytes, std::align_val_t{alignment}); + budget_->record_allocation(kind_, bytes, true); + return pointer; + } + catch (...) { + budget_->release(bytes); + throw; + } +} + +void AllocationMemoryResource::do_deallocate(void *pointer, size_t bytes, size_t alignment) +{ + ::operator delete(pointer, std::align_val_t{alignment}); + budget_->record_allocation(kind_, bytes, false); + budget_->release(bytes); +} + +bool AllocationMemoryResource::do_is_equal(const std::pmr::memory_resource &other) const noexcept +{ + return this == &other; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/runtime/allocation_memory_resource.h b/lib/crowdb-tree/src/runtime/allocation_memory_resource.h new file mode 100644 index 000000000..744b99ac7 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/allocation_memory_resource.h @@ -0,0 +1,30 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" + +#include <memory_resource> + +namespace crowdb::tree::detail +{ +// PMR ownership follows the caller's paid object. Both real allocation and +// actual free are reported; admission remains separate from observation. +class AllocationMemoryResource final : public std::pmr::memory_resource +{ + public: + explicit AllocationMemoryResource(std::shared_ptr<AllocationBudget> budget, + AllocationKind kind = AllocationKind::kMetadata) + : budget_(std::move(budget)), + kind_(kind) + { + } + + private: + void *do_allocate(size_t bytes, size_t alignment) override; + void do_deallocate(void *pointer, size_t bytes, size_t alignment) override; + [[nodiscard]] bool do_is_equal(const std::pmr::memory_resource &other) const noexcept override; + std::shared_ptr<AllocationBudget> budget_; + AllocationKind kind_; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/runtime/binding_metrics.h b/lib/crowdb-tree/src/runtime/binding_metrics.h new file mode 100644 index 000000000..b8aad75b0 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/binding_metrics.h @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" + +namespace crowdb::tree::detail +{ +// Account holds a weak metrics association. Keep its shared control on the +// host ledger, otherwise that weak control's allocator retains the same +// account and its lifecycle reservation forever. The object itself remains +// charged to the Tree and is freed before returning that charge. +template <class T> +std::shared_ptr<T> binding_metrics(const std::shared_ptr<AllocationBudget> &tree, + const std::shared_ptr<AllocationBudget> &host) +{ + struct Storage + { + explicit Storage(std::shared_ptr<AllocationBudget> budget) : budget(std::move(budget)) + { + } + + std::shared_ptr<AllocationBudget> budget; + T value; + }; + + MetadataAllocator<Storage> allocator(tree); + auto *storage = allocator.allocate(1); + try { + std::construct_at(storage, tree); + } + catch (...) { + allocator.deallocate(storage, 1); + throw; + } + return {&storage->value, + [storage](T *) noexcept { + MetadataAllocator<Storage> allocator(storage->budget); + std::destroy_at(storage); + allocator.deallocate(storage, 1); + }, + MetadataAllocator<T>(host)}; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/runtime/bootstrap_budget.cpp b/lib/crowdb-tree/src/runtime/bootstrap_budget.cpp new file mode 100644 index 000000000..61309291c --- /dev/null +++ b/lib/crowdb-tree/src/runtime/bootstrap_budget.cpp @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "bootstrap_budget.h" + +#include <cassert> +#include <stdexcept> + +namespace crowdb::tree::detail +{ +// This allocator is used for the bootstrap budget's own control block only. +// The object lives until the last weak control release, so the control can +// return its actual bytes before deleting the root accounting object. +template <class T> class BootstrapControlAllocator +{ + public: + using value_type = T; + + explicit BootstrapControlAllocator(BootstrapBudget *owner) noexcept : owner_(owner) + { + } + + template <class U> + BootstrapControlAllocator(const BootstrapControlAllocator<U> &other) noexcept : owner_(other.owner_) + { + } + + T *allocate(size_t count) + { + if (count > SIZE_MAX / sizeof(T)) { + throw std::bad_array_new_length(); + } + const auto bytes = count * sizeof(T); + if (!owner_->claim(bytes)) { + throw std::bad_alloc(); + } + try { + auto *pointer = static_cast<T *>(::operator new(bytes, std::align_val_t{alignof(T)})); + owner_->record_allocation(AllocationKind::kMetadata, bytes, true); + return pointer; + } + catch (...) { + owner_->release(bytes); + throw; + } + } + + void deallocate(T *pointer, size_t count) noexcept + { + owner_->release_control(pointer, count * sizeof(T), alignof(T)); + } + + template <class U> bool operator==(const BootstrapControlAllocator<U> &other) const noexcept + { + return owner_ == other.owner_; + } + + private: + template <class U> friend class BootstrapControlAllocator; + BootstrapBudget *owner_; +}; + +BootstrapBudget::BootstrapBudget(size_t capacity) : capacity_(capacity) +{ + physical_[static_cast<size_t>(AllocationKind::kMetadata)].store(sizeof(BootstrapBudget), std::memory_order_relaxed); +} + +BootstrapBudget::~BootstrapBudget() +{ + assert((used_.load(std::memory_order_relaxed) & ~kSealed) == sizeof(BootstrapBudget)); +} + +std::shared_ptr<BootstrapBudget> BootstrapBudget::create(size_t capacity) +{ + if (capacity >= kSealed) { + throw std::invalid_argument("host lifecycle capacity overflows accounting"); + } + if (capacity < sizeof(BootstrapBudget)) { + throw std::bad_alloc(); + } + auto *budget = new BootstrapBudget(capacity); + try { + // The control allocator performs final destruction after weak owners. + return {budget, [](BootstrapBudget *) noexcept {}, BootstrapControlAllocator<BootstrapBudget>(budget)}; + } + catch (...) { + delete budget; + throw; + } +} + +void BootstrapBudget::release_control(void *pointer, size_t bytes, size_t alignment) noexcept +{ + ::operator delete(pointer, std::align_val_t{alignment}); + record_allocation(AllocationKind::kMetadata, bytes, false); + release(bytes); + delete this; +} + +bool BootstrapBudget::claim(size_t bytes) noexcept +{ + auto current = used_.load(std::memory_order_relaxed); + while ((current & kSealed) == 0 && bytes <= capacity_ - current) { + if (used_.compare_exchange_weak(current, current + bytes, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + return true; + } + } + return false; +} + +void BootstrapBudget::release(size_t bytes) noexcept +{ + [[maybe_unused]] const auto before = used_.fetch_sub(bytes, std::memory_order_acq_rel); + assert((before & ~kSealed) >= bytes + sizeof(BootstrapBudget)); +} + +void BootstrapBudget::record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept +{ + const auto index = static_cast<size_t>(kind); + if (index >= physical_.size() || kind == AllocationKind::kUnknown) { + return; + } + if (acquired) { + physical_[index].fetch_add(bytes, std::memory_order_relaxed); + } + else { + [[maybe_unused]] const auto before = physical_[index].fetch_sub(bytes, std::memory_order_relaxed); + assert(before >= bytes); + } +} + +BootstrapBudget::Snapshot BootstrapBudget::snapshot() const noexcept +{ + const auto used = used_.load(std::memory_order_relaxed); + Snapshot result{.allocated = used & ~kSealed, .reserved = (used & kSealed) == 0 ? capacity_ - used : 0}; + for (size_t index = 0; index < physical_.size(); ++index) { + result.physical[index] = physical_[index].load(std::memory_order_relaxed); + } + return result; +} + +size_t BootstrapBudget::available() const noexcept +{ + const auto used = used_.load(std::memory_order_relaxed); + return (used & kSealed) == 0 ? capacity_ - used : 0; +} + +void BootstrapBudget::seal() noexcept +{ + used_.fetch_or(kSealed, std::memory_order_acq_rel); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/runtime/bootstrap_budget.h b/lib/crowdb-tree/src/runtime/bootstrap_budget.h new file mode 100644 index 000000000..3bdbf540f --- /dev/null +++ b/lib/crowdb-tree/src/runtime/bootstrap_budget.h @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" + +#include <array> + +namespace crowdb::tree::detail +{ +// A host lifecycle partition has no reference back to the admission ledger. +// Ledger storage, waiter slots and executor controls can therefore use this +// budget without making their allocator own the ledger it pays for. +class BootstrapBudget final : public AllocationBudget +{ + public: + struct Snapshot + { + size_t allocated = 0; + size_t reserved = 0; + std::array<size_t, static_cast<size_t>(AllocationKind::kCount)> physical{}; + }; + + [[nodiscard]] static std::shared_ptr<BootstrapBudget> create(size_t capacity); + [[nodiscard]] bool claim(size_t bytes) noexcept override; + void release(size_t bytes) noexcept override; + void record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept override; + [[nodiscard]] size_t available() const noexcept; + [[nodiscard]] Snapshot snapshot() const noexcept; + void seal() noexcept; + + private: + template <class T> friend class BootstrapControlAllocator; + explicit BootstrapBudget(size_t capacity); + ~BootstrapBudget() override; + void release_control(void *pointer, size_t bytes, size_t alignment) noexcept; + static constexpr size_t kSealed = size_t{1} << (std::numeric_limits<size_t>::digits - 1); + const size_t capacity_; + std::atomic<size_t> used_{sizeof(BootstrapBudget)}; + std::array<std::atomic<size_t>, static_cast<size_t>(AllocationKind::kCount)> physical_{}; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/runtime/default_runtime.cpp b/lib/crowdb-tree/src/runtime/default_runtime.cpp new file mode 100644 index 000000000..b119e00bb --- /dev/null +++ b/lib/crowdb-tree/src/runtime/default_runtime.cpp @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "default_runtime.h" + +#include "crowdb-tree/config.h" +#include "tree_runtime_internal.h" + +namespace crowdb::tree +{ +namespace +{ +struct DefaultRuntime +{ + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<AllocationBudget> metadata; + + static void destroy(DefaultRuntime *entry) noexcept + { + if (entry != nullptr) { + MetadataAllocator<DefaultRuntime> allocator(entry->metadata); + std::destroy_at(entry); + allocator.deallocate(entry, 1); + } + } +}; + +struct ProcessRuntime +{ + ~ProcessRuntime() + { + DefaultRuntime::destroy(value.load(std::memory_order_relaxed)); + } + + std::atomic<DefaultRuntime *> value{nullptr}; +}; + +ProcessRuntime process_runtime; + +Status default_runtime(std::shared_ptr<TreeRuntime> *out) +{ + auto *entry = process_runtime.value.load(std::memory_order_acquire); + if (entry == nullptr) { + std::shared_ptr<TreeRuntime> runtime; + auto status = TreeRuntime::create({}, &runtime); + if (!status.ok()) { + return status; + } + auto metadata = TreeRuntimeAccess::lifecycle(*runtime); + MetadataAllocator<DefaultRuntime> allocator(metadata); + auto *prepared = allocator.allocate(1); + std::construct_at(prepared, DefaultRuntime{.runtime = std::move(runtime), .metadata = std::move(metadata)}); + if (process_runtime.value.compare_exchange_strong(entry, prepared, std::memory_order_release, + std::memory_order_acquire)) { + entry = prepared; + } + else { + DefaultRuntime::destroy(prepared); + } + } + *out = entry->runtime; + return Status::Ok(); +} +} // namespace + +Status configure_default_runtime(Config *config) +try { + if (config == nullptr) { + return Status::invalid_argument(); + } + return config->runtime ? Status::Ok() : default_runtime(&config->runtime); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (...) { + return Status::internal_error(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/default_runtime.h b/lib/crowdb-tree/src/runtime/default_runtime.h new file mode 100644 index 000000000..28a650880 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/default_runtime.h @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/status.h" + +namespace crowdb::tree +{ +struct Config; + +// Standalone entry points share one process-owned host. An explicitly injected +// host retains its own limits and shutdown authority. +Status configure_default_runtime(Config *config); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/pool_storage.cpp b/lib/crowdb-tree/src/runtime/pool_storage.cpp new file mode 100644 index 000000000..f7fdd4bbe --- /dev/null +++ b/lib/crowdb-tree/src/runtime/pool_storage.cpp @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "pool_storage.h" + +#include <pthread.h> + +#include <system_error> + +namespace crowdb::tree::detail +{ +struct PoolStorage::Worker +{ + PoolStorage *storage; + void (*execute)(void *, uint32_t) noexcept; + void *context; + uint32_t index; + pthread_t thread{}; + std::thread::id id; + std::atomic<bool> ready{false}; +}; + +void *PoolStorage::do_allocate(size_t bytes, size_t alignment) +{ + if (budget_ && !budget_->claim(bytes)) { + throw std::bad_alloc(); + } + void *pointer = nullptr; + try { + pointer = ::operator new(bytes, std::align_val_t(alignment)); + } + catch (...) { + if (budget_) { + budget_->release(bytes); + } + throw; + } + if (budget_) { + budget_->record_allocation(AllocationKind::kMetadata, bytes, true); + } + return pointer; +} + +void PoolStorage::do_deallocate(void *pointer, size_t bytes, size_t alignment) +{ + ::operator delete(pointer, std::align_val_t(alignment)); + if (budget_) { + budget_->record_allocation(AllocationKind::kMetadata, bytes, false); + budget_->release(bytes); + } +} + +bool PoolStorage::do_is_equal(const std::pmr::memory_resource &other) const noexcept +{ + return this == &other; +} + +exec::pool_thread_factory PoolStorage::thread_factory() noexcept +{ + return {.context = this, .start = &start, .join = &join, .id = &identity}; +} + +size_t PoolStorage::worker_bytes() noexcept +{ + return sizeof(Worker); +} + +void *PoolStorage::start(void *resource, void (*execute)(void *, uint32_t) noexcept, void *context, uint32_t index) +{ + auto *storage = static_cast<PoolStorage *>(resource); + auto *worker = static_cast<Worker *>(storage->allocate(sizeof(Worker), alignof(Worker))); + std::construct_at(worker, storage, execute, context, index); + const auto error = pthread_create(&worker->thread, nullptr, &run, worker); + if (error != 0) { + std::destroy_at(worker); + storage->deallocate(worker, sizeof(Worker), alignof(Worker)); + throw std::system_error(error, std::generic_category(), "tree worker start"); + } + while (!worker->ready.load(std::memory_order_acquire)) { + worker->ready.wait(false, std::memory_order_acquire); + } + return worker; +} + +void *PoolStorage::run(void *pointer) noexcept +{ + auto *worker = static_cast<Worker *>(pointer); + worker->id = std::this_thread::get_id(); + worker->ready.store(true, std::memory_order_release); + worker->ready.notify_one(); + worker->execute(worker->context, worker->index); + return nullptr; +} + +void PoolStorage::join(void *pointer) noexcept +{ + auto *worker = static_cast<Worker *>(pointer); + if (pthread_join(worker->thread, nullptr) != 0) { + std::terminate(); + } + auto *storage = worker->storage; + std::destroy_at(worker); + storage->deallocate(worker, sizeof(Worker), alignof(Worker)); +} + +std::thread::id PoolStorage::identity(void *pointer) noexcept +{ + return static_cast<Worker *>(pointer)->id; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/runtime/pool_storage.h b/lib/crowdb-tree/src/runtime/pool_storage.h new file mode 100644 index 000000000..a341c7761 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/pool_storage.h @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/memory_budget.h" + +#include <exec/detail/pool_resource.hpp> +#include <memory_resource> + +namespace crowdb::tree::detail +{ +// The host owns this resource until all worker joins and pool container frees. +// Thread-system stacks/TLS are managed by pthread; engine worker controls and +// every pool container allocation use this resource. +class PoolStorage final : public std::pmr::memory_resource +{ + public: + explicit PoolStorage(std::shared_ptr<AllocationBudget> budget) : budget_(std::move(budget)) + { + } + + [[nodiscard]] exec::pool_thread_factory thread_factory() noexcept; + [[nodiscard]] static size_t worker_bytes() noexcept; + + private: + struct Worker; + static void *start(void *resource, void (*execute)(void *, uint32_t) noexcept, void *context, uint32_t index); + static void join(void *pointer) noexcept; + static std::thread::id identity(void *pointer) noexcept; + static void *run(void *pointer) noexcept; + void *do_allocate(size_t bytes, size_t alignment) override; + void do_deallocate(void *pointer, size_t bytes, size_t alignment) override; + [[nodiscard]] bool do_is_equal(const std::pmr::memory_resource &other) const noexcept override; + std::shared_ptr<AllocationBudget> budget_; +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/runtime/tree_operation_lifetime.cpp b/lib/crowdb-tree/src/runtime/tree_operation_lifetime.cpp new file mode 100644 index 000000000..0d0e6e733 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_operation_lifetime.cpp @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_operation_lifetime.h" + +#include "btree/allocation_bound.h" + +#include <stdexcept> + +namespace crowdb::tree +{ +struct TreeOperationLifetime::State +{ + static constexpr uint64_t kClosed = uint64_t{1} << 63; + + State(size_t limit, std::shared_ptr<const void> owner) : maximum(limit), host_owner(std::move(owner)) + { + } + + void release() noexcept + { + active.fetch_sub(1, std::memory_order_acq_rel); + active.notify_all(); + } + + const size_t maximum; + std::shared_ptr<const void> host_owner; + std::atomic<uint64_t> active{0}; + std::atomic<uint64_t> denied{0}; +}; + +TreeOperationLifetime::TreeOperationLifetime(size_t maximum, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const void> host_owner) +{ + if (maximum == 0 || maximum >= State::kClosed) { + throw std::invalid_argument("invalid Tree operation capacity"); + } + state_ = detail::reserved_shared<State>(metadata, maximum, std::move(host_owner)); +} + +TreeOperationLifetime::~TreeOperationLifetime() +{ + close(); +} + +Status TreeOperationLifetime::acquire(const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Ticket> *out) +{ + return acquire_impl(metadata, out, false); +} + +Status TreeOperationLifetime::try_acquire(const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Ticket> *out) +{ + return acquire_impl(metadata, out, true); +} + +Status TreeOperationLifetime::acquire_impl(const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Ticket> *out, bool temporary_capacity) +{ + if (out == nullptr || *out) { + return Status::invalid_argument("operation ticket output must be empty"); + } + auto active = state_->active.load(std::memory_order_acquire); + for (;;) { + if ((active & State::kClosed) != 0) { + return Status::unavailable("Tree operation admission closed"); + } + if (active >= state_->maximum) { + state_->denied.fetch_add(1, std::memory_order_relaxed); + return temporary_capacity ? Status::unavailable("Tree operation capacity is occupied") + : Status::resource_exhausted("Tree operation capacity exhausted"); + } + if (state_->active.compare_exchange_weak(active, active + 1, std::memory_order_acq_rel)) { + break; + } + } + bool constructed = false; + try { + *out = detail::reserved_construct<Ticket>(metadata, [&](Ticket *storage) { + ::new (static_cast<void *>(storage)) Ticket(state_); + constructed = true; + }); + } + catch (const std::bad_alloc &) { + // shared_ptr destroys an already constructed Ticket if allocating its + // control block fails; only a failed object allocation needs rollback. + if (!constructed) { + state_->release(); + } + return Status::resource_exhausted("Tree operation ticket allocation failed"); + } + return Status::Ok(); +} + +TreeOperationLifetime::Ticket::~Ticket() +{ + state_->release(); +} + +std::shared_ptr<AsyncPageStore> TreeOperationLifetime::Ticket::backend(AsyncPageStore *store) +{ + return store == nullptr ? nullptr : std::shared_ptr<AsyncPageStore>(shared_from_this(), store); +} + +size_t TreeOperationLifetime::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<State>(); +} + +size_t TreeOperationLifetime::ticket_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<Ticket>(); +} + +size_t TreeOperationLifetime::outstanding() const noexcept +{ + return state_->active.load(std::memory_order_acquire) & ~State::kClosed; +} + +ResourceUsage TreeOperationLifetime::resource_usage() const noexcept +{ + return { + .limit = state_->maximum, .used = outstanding(), .denied_tree = state_->denied.load(std::memory_order_relaxed)}; +} + +bool TreeOperationLifetime::closed() const noexcept +{ + return (state_->active.load(std::memory_order_acquire) & State::kClosed) != 0; +} + +void TreeOperationLifetime::close() noexcept +{ + state_->active.fetch_or(State::kClosed, std::memory_order_acq_rel); +} + +void TreeOperationLifetime::close_and_drain() noexcept +{ + close(); + drain_accepted(); +} + +void TreeOperationLifetime::drain_accepted() noexcept +{ + auto active = state_->active.load(std::memory_order_acquire); + while ((active & ~State::kClosed) != 0) { + state_->active.wait(active, std::memory_order_acquire); + active = state_->active.load(std::memory_order_acquire); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/tree_operation_lifetime.h b/lib/crowdb-tree/src/runtime/tree_operation_lifetime.h new file mode 100644 index 000000000..2ab863430 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_operation_lifetime.h @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_reservation.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/resource_usage.h" + +namespace crowdb::tree +{ +// Tracks accepted Tree operations through the last transport owner, including +// submissions whose acceptance is unknown. CPU drain alone cannot establish +// that a host-owned backend is safe to destroy. +class TreeOperationLifetime +{ + struct State; + + public: + class Ticket : public std::enable_shared_from_this<Ticket> + { + public: + ~Ticket(); + Ticket(const Ticket &) = delete; + Ticket &operator=(const Ticket &) = delete; + + // The backend remains host-owned. A loader/writeback holds this alias + // through every callback ticket, so close_and_drain cannot finish while + // the backend could still access its buffer or callback context. + [[nodiscard]] std::shared_ptr<AsyncPageStore> backend(AsyncPageStore *store); + + private: + friend class TreeOperationLifetime; + + explicit Ticket(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; + }; + + explicit TreeOperationLifetime(size_t maximum, const std::shared_ptr<detail::AllocationReservation> &metadata = {}, + std::shared_ptr<const void> host_owner = {}); + ~TreeOperationLifetime(); + TreeOperationLifetime(const TreeOperationLifetime &) = delete; + TreeOperationLifetime &operator=(const TreeOperationLifetime &) = delete; + + [[nodiscard]] Status acquire(const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Ticket> *out); + // Before consensus, a full operation window is temporary admission + // pressure. Allocation failure remains a resource error. + [[nodiscard]] Status try_acquire(const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<Ticket> *out); + [[nodiscard]] ResourceUsage resource_usage() const noexcept; + [[nodiscard]] static size_t allocation_bound() noexcept; + [[nodiscard]] static size_t ticket_allocation_bound() noexcept; + [[nodiscard]] size_t outstanding() const noexcept; + [[nodiscard]] bool closed() const noexcept; + void close() noexcept; + // Host thread only. Accepted operations may continue submitting child I/O + // through their existing Ticket after ingress closes. Do not call from a + // callback holding a Ticket, or before releasing controller repair owners. + void close_and_drain() noexcept; + // Caller fences new admission separately. Keeps this lifetime reusable. + void drain_accepted() noexcept; + + private: + Status acquire_impl(const std::shared_ptr<detail::AllocationReservation> &metadata, std::shared_ptr<Ticket> *out, + bool temporary_capacity); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/tree_runtime.cpp b/lib/crowdb-tree/src/runtime/tree_runtime.cpp new file mode 100644 index 000000000..85ac9a5a4 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_runtime.cpp @@ -0,0 +1,215 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "tree_runtime_internal.h" + +namespace crowdb::tree +{ +detail::ResourceBudget::Limits resource_limits(TreeResourceLimits limits) noexcept +{ + return {limits.handoff_bytes, limits.routing_bytes, limits.reclaim_bytes, + limits.writer_bytes, limits.gap_repair_bytes, limits.io_operations}; +} + +namespace +{ +detail::ResourceBudget::Limits operation_limits(const TreeRuntimeOptions &options) noexcept +{ + auto limits = resource_limits(options.limits); + limits[static_cast<size_t>(detail::BudgetClass::kRouting)] -= options.lifecycle_bytes; + return limits; +} + +TreeResourceLimits public_limits(detail::ResourceBudget::Limits limits) noexcept +{ + return {.handoff_bytes = limits[0], + .routing_bytes = limits[1], + .reclaim_bytes = limits[2], + .writer_bytes = limits[3], + .gap_repair_bytes = limits[4], + .io_operations = limits[5]}; +} +} // namespace + +TreeRuntime::State::State(TreeRuntimeOptions options, std::shared_ptr<detail::BootstrapBudget> credit) + : options(options), + lifecycle(std::move(credit)), + budget(operation_limits(options), options.max_waiters_per_class, lifecycle) +{ + const auto storage = + detail::AllocationBound() + .add(detail::FlushRuntime::allocation_bound(options.workers, options.cpu_queue_limit + 1)) + .add(detail::FlushRuntime::allocation_bound(options.backend_workers, options.backend_queue_limit)) + .add(2 * detail::shared_allocation_bound<detail::FlushRuntime>()) + .add(detail::shared_allocation_bound<TreeRuntime>()) + .add(detail::RetirementQueue::allocation_bound()) + .add(detail::shared_allocation_bound<detail::FlushRuntime::Lane>()) + .value(); + if (!storage) { + throw std::invalid_argument("runtime storage size overflow"); + } + if (*storage > lifecycle->available()) { + throw std::bad_alloc(); + } + cpu = std::allocate_shared<detail::FlushRuntime>(MetadataAllocator<detail::FlushRuntime>(lifecycle), + options.workers, options.cpu_queue_limit + 1, lifecycle); + backend = + std::allocate_shared<detail::FlushRuntime>(MetadataAllocator<detail::FlushRuntime>(lifecycle), + options.backend_workers, options.backend_queue_limit, lifecycle); + auto retirement_lane = + std::allocate_shared<detail::FlushRuntime::Lane>(MetadataAllocator<detail::FlushRuntime::Lane>(lifecycle), 1); + auto status = detail::RetirementQueue::create(*cpu, std::move(retirement_lane), lifecycle, &retirement); + if (!status.ok()) { + throw std::bad_alloc(); + } + budget.set_retirement_queue(retirement); +} + +TreeRuntime::State::~State() +{ + stopped.store(true, std::memory_order_release); + budget.close(); + retirement->close_and_drain(); + cpu->close(); + backend->close(); + cpu->drain(); + backend->drain(); + lifecycle->seal(); +} + +Status TreeRuntime::create(TreeRuntimeOptions options, std::shared_ptr<TreeRuntime> *out) +try { + if (out == nullptr || options.workers == 0 || options.cpu_queue_limit == 0 || options.cpu_queue_limit == SIZE_MAX || + options.max_waiters_per_class == 0 || options.backend_workers == 0 || options.backend_queue_limit == 0 || + options.lifecycle_bytes == 0 || options.lifecycle_bytes > options.limits.routing_bytes) { + return Status::invalid_argument(); + } + auto lifecycle = detail::BootstrapBudget::create(options.lifecycle_bytes); + auto state = std::allocate_shared<State>(MetadataAllocator<State>(lifecycle), options, lifecycle); + MetadataAllocator<TreeRuntime> allocator(lifecycle); + auto *value = allocator.allocate(1); + ::new (static_cast<void *>(value)) TreeRuntime(std::move(state)); + *out = std::shared_ptr<TreeRuntime>( + value, + [allocator](TreeRuntime *entry) mutable { + std::destroy_at(entry); + allocator.deallocate(entry, 1); + }, + allocator); + return {}; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (const std::invalid_argument &) { + return Status::invalid_argument(); +} +catch (...) { + return Status::internal_error(); +} + +TreeRuntime::~TreeRuntime() = default; + +void TreeRuntime::stop_admission() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->budget.close(); +} + +void TreeRuntime::drain_cpu() noexcept +{ + stop_admission(); + state_->cpu->close(); + state_->backend->close(); + state_->cpu->drain(); + state_->backend->drain(); +} + +bool TreeRuntime::stopped() const noexcept +{ + return state_->stopped.load(std::memory_order_acquire); +} + +TreeRuntimeSnapshot TreeRuntime::snapshot() const noexcept +{ + auto budget = state_->budget.snapshot(); + const auto bootstrap = state_->lifecycle->snapshot(); + const auto routing = static_cast<size_t>(detail::BudgetClass::kRouting); + budget.allocated[routing] += bootstrap.allocated; + budget.reserved[routing] += bootstrap.reserved; + const auto cpu = state_->cpu->snapshot(); + const auto backend = state_->backend->snapshot(); + const auto executor = [](const detail::FlushRuntime::Snapshot &value) { + return TreeExecutorSnapshot{.queued = value.queued, + .inflight = value.inflight, + .submitted = value.submitted, + .completed = value.completed, + .rejected = value.rejected, + .queue_ns = value.queue_ns, + .execute_ns = value.execute_ns}; + }; + return {.reserved = public_limits(budget.reserved), + .allocated = public_limits(budget.allocated), + .bindings = state_->bindings.load(std::memory_order_relaxed), + .queued = cpu.queued + backend.queued, + .inflight = cpu.inflight + backend.inflight, + .stopped = stopped(), + .cpu = executor(cpu), + .backend = executor(backend), + .budget_waiters = budget.waiters, + .budget_grants = budget.grants, + .budget_cancelled = budget.cancelled, + .budget_wait_ns = budget.wait_ns}; +} + +const TreeRuntimeOptions &TreeRuntime::options() const noexcept +{ + return state_->options; +} + +ResourceUsageSnapshot TreeRuntime::resource_usage() const noexcept +{ + auto result = state_->budget.resource_usage(); + const auto bootstrap = state_->lifecycle->snapshot(); + result[1].limit = state_->options.limits.routing_bytes; + result[1].reserved += bootstrap.reserved; + result[1].allocated += bootstrap.allocated; + result[1].used += bootstrap.reserved + bootstrap.allocated; + const auto cpu = state_->cpu->snapshot(); + const auto backend = state_->backend->snapshot(); + result[6] = {.limit = state_->options.cpu_queue_limit + 1, + .used = state_->cpu->queue_permits(), + .denied_host = cpu.rejected, + .waits = cpu.submitted, + .wait_ns = cpu.queue_ns, + .waiters = cpu.queued}; + result[7] = {.limit = state_->options.backend_queue_limit, + .used = state_->backend->queue_permits(), + .denied_host = backend.rejected, + .waits = backend.submitted, + .wait_ns = backend.queue_ns, + .waiters = backend.queued}; + result[8] = {.limit = state_->options.workers, .used = cpu.inflight}; + result[9] = {.limit = state_->options.backend_workers, .used = backend.inflight}; + return result; +} + +Status TreeRuntime::validate_binding(const TreeBindingOptions &options) const noexcept +{ + if (stopped()) { + return Status::unavailable(); + } + if (options.cpu_queue_limit == 0 || options.backend_queue_limit == 0 || options.lifecycle_bytes == 0 || + options.lifecycle_bytes > options.limits.routing_bytes) { + return Status::invalid_argument(); + } + const auto tree = resource_limits(options.limits); + const auto host = resource_limits(state_->options.limits); + for (size_t index = 0; index < tree.size(); ++index) { + if (tree[index] > host[index]) { + return Status::invalid_argument(); + } + } + return {}; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/tree_runtime_binding.cpp b/lib/crowdb-tree/src/runtime/tree_runtime_binding.cpp new file mode 100644 index 000000000..74daeffbd --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_runtime_binding.cpp @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_runtime_binding.h" + +#include "binding_metrics.h" +#include "btree/tree_metrics.h" +#include "tree_runtime_internal.h" + +namespace crowdb::tree +{ +TreeRuntimeBinding::TreeRuntimeBinding(std::shared_ptr<TreeRuntime> runtime, TreeBindingOptions options, + std::shared_ptr<detail::ResourceBudget::Account> account, + std::shared_ptr<detail::AllocationReservation> lifecycle, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + std::shared_ptr<detail::FlushRuntime::Lane> backend_lane) + : owner_(std::move(runtime)), + options_(options), + account_(std::move(account)), + lifecycle_(std::move(lifecycle)), + work_metrics_(detail::binding_metrics<TreeWorkMetrics>(lifecycle_, owner_->state_->lifecycle)), + resource_metrics_(detail::binding_metrics<TreeResourceMetrics>(lifecycle_, owner_->state_->lifecycle)), + cpu_(owner_->state_, owner_->state_->cpu.get()), + lane_(std::move(lane)), + backend_cpu_(owner_->state_, owner_->state_->backend.get()), + backend_lane_(std::move(backend_lane)) +{ + account_->set_work_metrics(work_metrics_); + account_->set_resource_metrics(resource_metrics_); + owner_->state_->bindings.fetch_add(1, std::memory_order_relaxed); +} + +void TreeRuntimeBinding::drain() const noexcept +{ + lane_->drain(); + backend_lane_->drain(); + account_->drain_retirements(); +} + +ResourceUsageSnapshot TreeRuntimeBinding::resource_usage() const noexcept +{ + auto result = account_->resource_usage(); + result[6] = {.limit = options_.cpu_queue_limit, .used = lane_->queue_permits()}; + result[7] = {.limit = options_.backend_queue_limit, .used = backend_lane_->queue_permits()}; + return result; +} + +TreeRuntimeBinding::~TreeRuntimeBinding() +{ + owner_->state_->bindings.fetch_sub(1, std::memory_order_relaxed); + lifecycle_->seal(); +} + +Status TreeRuntimeBinding::create(std::shared_ptr<TreeRuntime> runtime, TreeBindingOptions options, + std::shared_ptr<TreeRuntimeBinding> *out) +try { + if (!runtime || out == nullptr) { + return Status::invalid_argument(); + } + auto status = runtime->validate_binding(options); + if (!status.ok()) { + return status; + } + auto &state = *runtime->state_; + auto account = state.budget.make_account(resource_limits(options.limits), state.lifecycle); + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, options.lifecycle_bytes); + if (!credit) { + return runtime->stopped() ? Status::unavailable() : Status::resource_exhausted(); + } + auto lifecycle = detail::AllocationReservation::create(std::move(*credit)); + auto lane = detail::reserved_shared<detail::FlushRuntime::Lane>(lifecycle, options.cpu_queue_limit); + auto backend_lane = detail::reserved_shared<detail::FlushRuntime::Lane>(lifecycle, options.backend_queue_limit); + auto result = detail::reserved_construct<TreeRuntimeBinding>(lifecycle, [&](TreeRuntimeBinding *value) { + ::new (static_cast<void *>(value)) + TreeRuntimeBinding(runtime, options, account, lifecycle, std::move(lane), std::move(backend_lane)); + }); + status = + ApplyDispatcher::create(result->backend_cpu_, result->backend_lane_, 1, lifecycle, &result->backend_progress_); + if (!status.ok()) { + return status; + } + if (runtime->stopped()) { + return Status::unavailable(); + } + *out = std::move(result); + return {}; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (const std::invalid_argument &) { + return Status::invalid_argument(); +} +catch (...) { + return Status::internal_error(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/tree_runtime_binding.h b/lib/crowdb-tree/src/runtime/tree_runtime_binding.h new file mode 100644 index 000000000..1ae44c0c0 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_runtime_binding.h @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_reservation.h" +#include "btree/apply_dispatcher.h" +#include "btree/flush_runtime.h" +#include "crowdb-tree/tree_runtime.h" + +namespace crowdb::tree +{ +// Keep this owner in every Tree operation through its final backend ticket. +// It provides separate Tree quotas while preserving the host ledger domain. +class TreeRuntimeBinding +{ + public: + static Status create(std::shared_ptr<TreeRuntime> runtime, TreeBindingOptions options, + std::shared_ptr<TreeRuntimeBinding> *out); + ~TreeRuntimeBinding(); + TreeRuntimeBinding(const TreeRuntimeBinding &) = delete; + TreeRuntimeBinding &operator=(const TreeRuntimeBinding &) = delete; + + [[nodiscard]] const std::shared_ptr<detail::FlushRuntime> &cpu() const noexcept + { + return cpu_; + } + + [[nodiscard]] const std::shared_ptr<detail::FlushRuntime::Lane> &lane() const noexcept + { + return lane_; + } + + [[nodiscard]] const std::shared_ptr<detail::FlushRuntime> &backend_cpu() const noexcept + { + return backend_cpu_; + } + + [[nodiscard]] const std::shared_ptr<detail::FlushRuntime::Lane> &backend_lane() const noexcept + { + return backend_lane_; + } + + [[nodiscard]] const std::shared_ptr<ApplyDispatcher> &backend_progress() const noexcept + { + return backend_progress_; + } + + [[nodiscard]] const std::shared_ptr<TreeWorkMetrics> &work_metrics() const noexcept + { + return work_metrics_; + } + + [[nodiscard]] const std::shared_ptr<TreeResourceMetrics> &resource_metrics() const noexcept + { + return resource_metrics_; + } + + // Host-only: all operation tickets must drain before joining scheduler tails. + void drain() const noexcept; + [[nodiscard]] ResourceUsageSnapshot resource_usage() const noexcept; + + [[nodiscard]] const std::shared_ptr<detail::ResourceBudget::Account> &account() const noexcept + { + return account_; + } + + [[nodiscard]] const std::shared_ptr<detail::AllocationReservation> &lifecycle_budget() const noexcept + { + return lifecycle_; + } + + [[nodiscard]] const TreeBindingOptions &options() const noexcept + { + return options_; + } + + private: + TreeRuntimeBinding(std::shared_ptr<TreeRuntime> runtime, TreeBindingOptions options, + std::shared_ptr<detail::ResourceBudget::Account> account, + std::shared_ptr<detail::AllocationReservation> lifecycle, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + std::shared_ptr<detail::FlushRuntime::Lane> backend_lane); + + std::shared_ptr<TreeRuntime> owner_; + TreeBindingOptions options_; + std::shared_ptr<detail::ResourceBudget::Account> account_; + std::shared_ptr<detail::AllocationReservation> lifecycle_; + std::shared_ptr<TreeWorkMetrics> work_metrics_; + std::shared_ptr<TreeResourceMetrics> resource_metrics_; + std::shared_ptr<detail::FlushRuntime> cpu_; + std::shared_ptr<detail::FlushRuntime::Lane> lane_; + std::shared_ptr<detail::FlushRuntime> backend_cpu_; + std::shared_ptr<detail::FlushRuntime::Lane> backend_lane_; + std::shared_ptr<ApplyDispatcher> backend_progress_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/tree_runtime_c_api.cpp b/lib/crowdb-tree/src/runtime/tree_runtime_c_api.cpp new file mode 100644 index 000000000..022b5cd99 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_runtime_c_api.cpp @@ -0,0 +1,325 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_runtime_c_api.h" + +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/c_api.h" +#include "default_runtime.h" +#include "tree_runtime_internal.h" + +#include <algorithm> + +struct ct_runtime +{ + std::shared_ptr<crowdb::tree::TreeRuntime> owner; +}; + +namespace +{ +using namespace crowdb::tree; + +TreeResourceLimits from_c(ct_resource_limits limits) noexcept +{ + return {.handoff_bytes = limits.handoff_bytes, + .routing_bytes = limits.routing_bytes, + .reclaim_bytes = limits.reclaim_bytes, + .writer_bytes = limits.writer_bytes, + .gap_repair_bytes = limits.gap_repair_bytes, + .io_operations = limits.io_operations}; +} + +ct_resource_limits to_c(TreeResourceLimits limits) noexcept +{ + return {.handoff_bytes = limits.handoff_bytes, + .routing_bytes = limits.routing_bytes, + .reclaim_bytes = limits.reclaim_bytes, + .writer_bytes = limits.writer_bytes, + .gap_repair_bytes = limits.gap_repair_bytes, + .io_operations = limits.io_operations}; +} + +ct_executor_snapshot_v1 to_c(const TreeExecutorSnapshot &value) noexcept +{ + return {.queued = value.queued, + .inflight = value.inflight, + .submitted = value.submitted, + .completed = value.completed, + .rejected = value.rejected, + .queue_ns = value.queue_ns, + .execute_ns = value.execute_ns}; +} + +int32_t copy_usage(const ResourceUsageSnapshot &value, uint32_t count, uint64_t scope_id, + ct_resource_usage_snapshot_v1 *out) +{ + ct_resource_usage_snapshot_v1 result{.struct_size = sizeof(*out), + .version = 1, + .resource_mask = (uint32_t{1} << count) - 1, + .scope_id = scope_id, + .resources = {}}; + for (size_t i = 0; i < count; ++i) { + const auto &item = value[i]; + result.resources[i] = {.limit = item.limit, + .used = item.used, + .reserved = item.reserved, + .allocated = item.allocated, + .denied_tree = item.denied_tree, + .denied_host = item.denied_host, + .waits = item.waits, + .wait_ns = item.wait_ns, + .waiters = item.waiters}; + } + *out = result; + return 0; +} +} // namespace + +int32_t ct_runtime_resource_usage(const ct_runtime *runtime, ct_resource_usage_snapshot_v1 *out) +{ + if (runtime == nullptr || out == nullptr || out->struct_size != sizeof(*out) || out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + return copy_usage(runtime->owner->resource_usage(), 10, 0, out); +} + +int32_t ct_tree_resource_usage(ct_tree *tree, ct_resource_usage_snapshot_v1 *out) +{ + if (tree == nullptr || out == nullptr || out->struct_size != sizeof(*out) || out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + const auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + ct_resource_usage_snapshot_v1 result{}; + copy_usage(core->binding()->resource_usage(), 8, core->binding()->account()->scope_id(), &result); + const auto cache = core->cache()->snapshot(); + result.resource_mask |= (uint32_t{7} << 10); + result.resources[10].limit = core->config().buffer_pool_bytes; + result.resources[10].used = cache.resident_bytes; + result.resources[11].limit = core->cache()->retained_page_limit(); + result.resources[11].used = std::min(result.resources[11].limit, cache.resident_pages + cache.inflight); + result.resources[11].allocated = std::min(result.resources[11].used, cache.resident_pages); + result.resources[11].reserved = result.resources[11].used - result.resources[11].allocated; + result.resources[12].limit = core->binding()->options().cpu_queue_limit; + result.resources[12].used = std::min(result.resources[12].limit, cache.request_slots); + result.resources[12].reserved = std::min(result.resources[12].used, cache.reserved_waiters); + result.resources[12].allocated = result.resources[12].used - result.resources[12].reserved; + result.resources[12].waiters = cache.waiters; + const auto admission = core->admission_resource_usage(); + result.resource_mask |= uint32_t{127} << 13; + for (size_t index = 0; index < admission.size(); ++index) { + const auto &value = admission[index]; + auto &row = result.resources[13 + index]; + row.limit = value.limit; + row.used = value.used; + row.denied_tree = value.denied_tree; + } + *out = result; + return 0; +} + +int32_t ct_runtime_snapshot(const ct_runtime *runtime, ct_runtime_snapshot_v1 *out) +{ + if (runtime == nullptr || out == nullptr || out->struct_size != sizeof(*out) || out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + const auto value = runtime->owner->snapshot(); + *out = {.struct_size = sizeof(*out), + .version = 1, + .stopped = static_cast<uint32_t>(value.stopped), + .reserved = to_c(value.reserved), + .allocated = to_c(value.allocated), + .bindings = value.bindings, + .cpu = to_c(value.cpu), + .backend = to_c(value.backend), + .budget_waiters = value.budget_waiters, + .budget_grants = value.budget_grants, + .budget_cancelled = value.budget_cancelled, + .budget_wait_ns = value.budget_wait_ns}; + return 0; +} + +void ct_runtime_options_default(ct_runtime_options *out) +{ + if (out != nullptr) { + const TreeRuntimeOptions options; + *out = {.struct_size = sizeof(*out), + .workers = options.workers, + .cpu_queue_limit = options.cpu_queue_limit, + .max_waiters_per_class = options.max_waiters_per_class, + .lifecycle_bytes = options.lifecycle_bytes, + .limits = to_c(options.limits), + .backend_workers = options.backend_workers, + .backend_queue_limit = options.backend_queue_limit}; + } +} + +void ct_tree_runtime_options_default(ct_tree_runtime_options *out) +{ + if (out != nullptr) { + const TreeBindingOptions options; + *out = {.struct_size = sizeof(*out), + .cpu_queue_limit = options.cpu_queue_limit, + .lifecycle_bytes = options.lifecycle_bytes, + .limits = to_c(options.limits), + .backend_queue_limit = options.backend_queue_limit}; + } +} + +int32_t ct_runtime_create(const ct_runtime_options *options, ct_runtime **out) +try { + if (out == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = nullptr; + TreeRuntimeOptions config; + if (options != nullptr) { + if (options->struct_size != sizeof(*options)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + config = {.workers = options->workers, + .cpu_queue_limit = options->cpu_queue_limit, + .max_waiters_per_class = options->max_waiters_per_class, + .backend_workers = options->backend_workers, + .backend_queue_limit = options->backend_queue_limit, + .lifecycle_bytes = options->lifecycle_bytes, + .limits = from_c(options->limits)}; + } + std::shared_ptr<TreeRuntime> owner; + auto status = TreeRuntime::create(config, &owner); + if (!status.ok()) { + return static_cast<int32_t>(status.code()); + } + MetadataAllocator<ct_runtime> allocator(TreeRuntimeAccess::lifecycle(*owner)); + auto *handle = allocator.allocate(1); + ::new (static_cast<void *>(handle)) ct_runtime{std::move(owner)}; + *out = handle; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} + +void ct_runtime_stop_admission(ct_runtime *runtime) +{ + if (runtime != nullptr) { + runtime->owner->stop_admission(); + } +} + +void ct_runtime_drain_cpu(ct_runtime *runtime) +{ + if (runtime != nullptr) { + runtime->owner->drain_cpu(); + } +} + +void ct_runtime_free(ct_runtime *runtime) +{ + if (runtime != nullptr) { + MetadataAllocator<ct_runtime> allocator(TreeRuntimeAccess::lifecycle(*runtime->owner)); + std::destroy_at(runtime); + allocator.deallocate(runtime, 1); + } +} + +namespace crowdb::tree +{ +Status runtime_host_allocation_budget(ct_runtime *runtime, std::shared_ptr<AllocationBudget> *out) noexcept +{ + if (runtime == nullptr || out == nullptr || *out) { + return Status::invalid_argument(); + } + if (runtime->owner->stopped()) { + return Status::unavailable(); + } + *out = TreeRuntimeAccess::lifecycle(*runtime->owner); + return Status::Ok(); +} + +Status configure_tree_runtime(ct_runtime *runtime, const ct_tree_runtime_options *options, Config *out) noexcept +try { + if (out == nullptr) { + return Status::invalid_argument(); + } + TreeBindingOptions binding; + if (options != nullptr) { + if (options->struct_size != sizeof(*options)) { + return Status::invalid_argument(); + } + binding = {.cpu_queue_limit = options->cpu_queue_limit, + .backend_queue_limit = options->backend_queue_limit, + .lifecycle_bytes = options->lifecycle_bytes, + .limits = from_c(options->limits)}; + } + Config configured; + if (runtime != nullptr) { + configured.runtime = runtime->owner; + } + auto status = configure_default_runtime(&configured); + if (!status.ok()) { + return status; + } + status = configured.runtime->validate_binding(binding); + if (!status.ok()) { + return status; + } + out->runtime = std::move(configured.runtime); + out->runtime_binding = binding; + return {}; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +catch (...) { + return Status::internal_error(); +} +} // namespace crowdb::tree + +ct_status ct_open_with_runtime(const ct_options *opt, ct_runtime *runtime, const ct_tree_runtime_options *binding, + ct_tree **out) +try { + if (out == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = nullptr; + if (opt == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + return ct_open_configured(opt, runtime, binding, out); +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} + +ct_status ct_rebuild_range_with_runtime(ct_tree *source, const ct_options *opt, ct_runtime *runtime, + const ct_tree_runtime_options *binding, ct_tree **out, + ct_range_rebuild_stats *stats) +{ + if (out != nullptr) { + *out = nullptr; + } + if (stats != nullptr) { + *stats = {}; + } + if (source == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + if (opt == nullptr || out == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + Config config; + auto status = configure_tree_runtime(runtime, binding, &config); + if (!status.ok()) { + return static_cast<int32_t>(status.code()); + } + return ct_rebuild_configured(source, opt, std::move(config), out, stats); +} diff --git a/lib/crowdb-tree/src/runtime/tree_runtime_c_api.h b/lib/crowdb-tree/src/runtime/tree_runtime_c_api.h new file mode 100644 index 000000000..416c368db --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_runtime_c_api.h @@ -0,0 +1,18 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/config.h" +#include "crowdb-tree/runtime_c_api.h" + +namespace crowdb::tree +{ +class AllocationBudget; +// Borrow fixed host lifecycle credit without retaining its CPU executors. +[[nodiscard]] Status runtime_host_allocation_budget(ct_runtime *, std::shared_ptr<AllocationBudget> *) noexcept; + +// Copies host ownership (the process host when null) and validates options; +// the core creates one binding. +[[nodiscard]] Status configure_tree_runtime(ct_runtime *runtime, const ct_tree_runtime_options *options, + Config *out) noexcept; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/tree_runtime_internal.h b/lib/crowdb-tree/src/runtime/tree_runtime_internal.h new file mode 100644 index 000000000..98054e205 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_runtime_internal.h @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "bootstrap_budget.h" +#include "btree/allocation_reservation.h" +#include "btree/flush_runtime.h" +#include "btree/retirement_queue.h" +#include "crowdb-tree/tree_runtime.h" + +namespace crowdb::tree +{ +[[nodiscard]] detail::ResourceBudget::Limits resource_limits(TreeResourceLimits limits) noexcept; + +struct TreeRuntime::State +{ + State(TreeRuntimeOptions options, std::shared_ptr<detail::BootstrapBudget> credit); + ~State(); + + TreeRuntimeOptions options; + std::shared_ptr<detail::BootstrapBudget> lifecycle; + detail::ResourceBudget budget; + std::shared_ptr<detail::FlushRuntime> cpu; + std::shared_ptr<detail::FlushRuntime> backend; + std::shared_ptr<detail::RetirementQueue> retirement; + std::atomic<uint64_t> bindings{0}; + std::atomic<bool> stopped{false}; +}; + +struct TreeRuntimeAccess +{ + static std::shared_ptr<AllocationBudget> lifecycle(const TreeRuntime &runtime) noexcept + { + return runtime.state_->lifecycle; + } +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/runtime/tree_tuning_c_api.cpp b/lib/crowdb-tree/src/runtime/tree_tuning_c_api.cpp new file mode 100644 index 000000000..49f160ec5 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_tuning_c_api.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "tree_tuning_c_api.h" + +using namespace crowdb::tree; + +void ct_tree_tuning_default(ct_tree_tuning *out) +{ + if (out == nullptr) { + return; + } + const Config defaults; + *out = {.struct_size = sizeof(*out), + .memtable_flush_bytes = defaults.memtable_flush_bytes, + .memtable_flush_entries = defaults.memtable_flush_entries, + .max_memtable_count = defaults.max_memtable_count, + .leaf_split_bytes = defaults.leaf_split_bytes, + .leaf_merge_bytes = defaults.leaf_merge_bytes, + .inner_max_keys = defaults.inner_max_keys, + .inner_merge_keys = defaults.inner_merge_keys, + .max_key_size = defaults.max_key_size, + .source_max_entries = defaults.execution.source_max_entries, + .source_max_key_bytes = defaults.execution.source_max_key_bytes, + .source_max_gap_entries = defaults.execution.source_max_gap_entries, + .source_max_gap_key_bytes = defaults.execution.source_max_gap_key_bytes, + .flush_parallelism = defaults.execution.flush_parallelism, + .run_max_entries = defaults.execution.run_max_entries, + .run_max_key_bytes = defaults.execution.run_max_key_bytes, + .run_key_block_bytes = defaults.execution.run_key_block_bytes, + .run_max_bytes = defaults.execution.run_max_bytes, + .install_run_window = defaults.execution.install_run_window, + .maintenance_visits_per_step = defaults.execution.maintenance_visits_per_step, + .maintenance_max_visits = defaults.execution.maintenance_max_visits, + .maintenance_max_candidates = defaults.execution.maintenance_max_candidates, + .maintenance_max_steps = defaults.execution.maintenance_max_steps, + .maintenance_metadata_bytes = defaults.execution.maintenance_metadata_bytes, + .cache_max_pages = defaults.execution.cache_max_pages}; +} + +Status crowdb::tree::validate_tree_execution(const TreeExecutionOptions &options) noexcept +{ + if (options.source_max_entries == 0 || options.source_max_key_bytes == 0 || options.source_max_gap_entries == 0 || + options.source_max_gap_key_bytes == 0 || options.run_max_entries == 0 || options.run_max_key_bytes == 0 || + options.run_key_block_bytes == 0 || options.run_max_bytes == 0 || options.install_run_window == 0 || + options.maintenance_visits_per_step == 0 || options.maintenance_max_visits == 0 || + options.maintenance_max_candidates == 0 || options.maintenance_max_steps == 0 || + options.maintenance_metadata_bytes == 0 || options.cache_max_pages == 0) { + return Status::invalid_argument("Tree execution windows must be positive"); + } + if (options.run_key_block_bytes > options.run_max_key_bytes || + options.maintenance_visits_per_step > options.maintenance_max_visits) { + return Status::invalid_argument("Tree execution step exceeds its enclosing window"); + } + return {}; +} + +Status crowdb::tree::configure_tree_tuning(const ct_tree_tuning *options, Config *out) noexcept +{ + if (out == nullptr || (options != nullptr && options->struct_size != sizeof(*options))) { + return Status::invalid_argument("Tree tuning struct size mismatch"); + } + if (options == nullptr) { + return {}; + } + out->memtable_flush_bytes = options->memtable_flush_bytes; + out->memtable_flush_entries = options->memtable_flush_entries; + out->max_memtable_count = options->max_memtable_count; + out->leaf_split_bytes = options->leaf_split_bytes; + out->leaf_merge_bytes = options->leaf_merge_bytes; + out->inner_max_keys = options->inner_max_keys; + out->inner_merge_keys = options->inner_merge_keys; + out->max_key_size = options->max_key_size; + out->execution.source_max_entries = options->source_max_entries; + out->execution.source_max_key_bytes = options->source_max_key_bytes; + out->execution.source_max_gap_entries = options->source_max_gap_entries; + out->execution.source_max_gap_key_bytes = options->source_max_gap_key_bytes; + out->execution.flush_parallelism = options->flush_parallelism; + out->execution.run_max_entries = options->run_max_entries; + out->execution.run_max_key_bytes = options->run_max_key_bytes; + out->execution.run_key_block_bytes = options->run_key_block_bytes; + out->execution.run_max_bytes = options->run_max_bytes; + out->execution.install_run_window = options->install_run_window; + out->execution.maintenance_visits_per_step = options->maintenance_visits_per_step; + out->execution.maintenance_max_visits = options->maintenance_max_visits; + out->execution.maintenance_max_candidates = options->maintenance_max_candidates; + out->execution.maintenance_max_steps = options->maintenance_max_steps; + out->execution.maintenance_metadata_bytes = options->maintenance_metadata_bytes; + out->execution.cache_max_pages = options->cache_max_pages; + return validate_tree_execution(out->execution); +} diff --git a/lib/crowdb-tree/src/runtime/tree_tuning_c_api.h b/lib/crowdb-tree/src/runtime/tree_tuning_c_api.h new file mode 100644 index 000000000..5d41be413 --- /dev/null +++ b/lib/crowdb-tree/src/runtime/tree_tuning_c_api.h @@ -0,0 +1,11 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/config.h" +#include "crowdb-tree/tuning_c_api.h" + +namespace crowdb::tree +{ +Status configure_tree_tuning(const ct_tree_tuning *options, Config *out) noexcept; +Status validate_tree_execution(const TreeExecutionOptions &options) noexcept; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_frames.cpp b/lib/crowdb-tree/src/snapshot/captured_frames.cpp new file mode 100644 index 000000000..7118398fc --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_frames.cpp @@ -0,0 +1,244 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/page_encode_internal.h" +#include "captured_frames_internal.h" +#include "primary_backing.h" + +#include <algorithm> +#include <cassert> + +namespace crowdb::tree +{ +CapturedFrameStream::State::State(std::shared_ptr<const PublishedTreeState> captured, + std::shared_ptr<PageLoader> transport, std::shared_ptr<detail::FlushRuntime> cpu, + std::shared_ptr<detail::FlushRuntime::Lane> queue, CapturedFrameLimits bounds, + ReserveOverflow reserve) + : view(std::move(captured)), + loader(std::move(transport)), + runtime(std::move(cpu)), + lane(std::move(queue)), + limits(std::move(bounds)), + reserve_overflow(std::move(reserve)), + stack(detail::ReservationAllocator<Path>(limits.encode.metadata)), + buffer(limits.plan_only ? nullptr + : detail::reserved_shared<CapturedFrameBuffer>(limits.encode.metadata, limits.encode)) +{ + stack.reserve(limits.max_depth); + overflow_end = view->next_page_id; +} + +Status CapturedFrameStream::create(std::shared_ptr<const PublishedTreeState> captured, + std::shared_ptr<PageLoader> loader, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedFrameLimits limits, + ReserveOverflow reserve_overflow, std::shared_ptr<CapturedFrameStream> *out) +{ + if (!captured || !runtime || !lane || out == nullptr || limits.max_depth == 0 || limits.max_depth > 64 || + limits.max_pages == 0 || limits.max_pages > MappingView::kPageLimit || + limits.encode.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || limits.encode.max_sources == 0 || + limits.encode.max_entries == 0 || limits.encode.max_work_entries == 0 || + captured->gc_floor > captured->coverage || captured->next_page_id > MappingView::kPageLimit) { + return Status::invalid_argument("captured frame stream requires a fixed view and bounded work"); + } + if (captured->pending_handoff || captured->flush_admission_owner != 0 || + (captured->sources && !captured->sources->tables.empty())) { + return Status::unavailable("snapshot must capture a complete handoff " + "without retaining L0 sources"); + } + try { + const auto metadata = limits.encode.metadata; + auto state = + detail::reserved_shared<State>(metadata, std::move(captured), std::move(loader), std::move(runtime), + std::move(lane), std::move(limits), std::move(reserve_overflow)); + *out = detail::reserved_construct<CapturedFrameStream>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) CapturedFrameStream(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +std::optional<size_t> CapturedFrameStream::allocation_bound(const CapturedFrameLimits &limits) noexcept +{ + using namespace detail; + const auto page = + AllocationBound(PageState::allocation_bound()) + .add_product(limits.encode.max_sources, sizeof(BaseRef) + sizeof(std::shared_ptr<const OverlayBatch>)) + .value(); + if (!page) { + return {}; + } + return AllocationBound(shared_allocation_bound<State>() + shared_allocation_bound<CapturedFrameStream>() + + shared_allocation_bound<State::Load>() + shared_allocation_bound<FinalPageEncodeState>() + + shared_allocation_bound<FinalPageEncoder>() + shared_allocation_bound<PublishedTreeState>() + + MappingView::path_copy_bound()) + .add_product(limits.max_depth, sizeof(State::Path) + shared_allocation_bound<PublishedPage>()) + .add_product(2, std::max(*page, limits.page_metadata_bytes)) + .add(limits.encode.max_metadata_bytes) + .add(shared_allocation_bound<PrimaryReuseOutput>()) + .add(limits.encode.max_metadata_bytes) + .add(LogicalPageCursor::allocation_bound(limits.encode.max_sources)) + .add(limits.plan_only ? 0 : shared_allocation_bound<CapturedFrameBuffer>() + limits.encode.frame_bytes) + .value(); +} + +detail::CallbackSender CapturedFrameStream::sender() noexcept +{ + return {this, &CapturedFrameStream::submit}; +} + +void CapturedFrameStream::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<CapturedFrameStream *>(source)->state_->start(complete, context); +} + +const CapturedFrameResult &CapturedFrameStream::result() const noexcept +{ + return state_->report; +} + +const CapturedFrameStats &CapturedFrameStream::stats() const noexcept +{ + return state_->counters; +} + +void CapturedFrameStream::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void CapturedFrameStream::State::start(detail::CallbackComplete callback, void *receiver) noexcept +{ + uint8_t idle = 0; + if (!lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + callback(receiver, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + complete = callback; + context = receiver; + lifetime = weak_from_this().lock(); + begin_pending = true; + lifecycle.store(2, std::memory_order_release); + wake(); +} + +void CapturedFrameStream::State::wake() noexcept +{ + wake_requested.store(true, std::memory_order_release); + if (lifecycle.load(std::memory_order_acquire) != 2 || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + auto self = weak_from_this().lock(); + assert(self); + for (;;) { + queued_owner = self; + if (limits.cpu_progress) { + limits.cpu_progress->request(cpu_node, self); + return; + } + Status status; + try { + status = runtime->try_submit(lane, &State::run, this); + } + catch (...) { + status = Status::resource_exhausted(); + } + if (status.ok()) { + return; + } + queued_owner.reset(); + // No worker owns this state after rejected submission. Keep scheduled + // set through the callback so a reentrant next cannot race cleanup. + if (lifecycle.load(std::memory_order_acquire) == 2) { + finish(std::move(status), true); + } + scheduled.exchange(false, std::memory_order_acq_rel); + if (lifecycle.load(std::memory_order_acquire) != 2 || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + } +} + +void CapturedFrameStream::State::run(void *context) noexcept +{ + auto self = std::move(static_cast<State *>(context)->queued_owner); + assert(self); + self->wake_requested.exchange(false, std::memory_order_acq_rel); + try { + self->pump(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted(), true); + } + catch (...) { + self->finish(Status::internal_error(), true); + } + self->scheduled.exchange(false, std::memory_order_acq_rel); + if (self->wake_requested.load(std::memory_order_acquire) && self->lifecycle.load(std::memory_order_acquire) == 2) { + self->wake(); + } +} + +void CapturedFrameStream::State::pump() +{ + if (lifecycle.load(std::memory_order_acquire) != 2) { + return; + } + ++counters.cpu_steps; + if (begin_pending) { + begin_pending = false; + report = {}; + if (buffer && buffer.use_count() != 1) { + finish(Status::unavailable("previous snapshot frame is still pinned")); + return; + } + } + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable(), true); + return; + } + if (!failure.ok()) { + finish(failure); + return; + } + if (load) { + if (!load->ready.load(std::memory_order_acquire)) { + return; + } + loaded(); + if (lifecycle.load(std::memory_order_acquire) != 2 || begin_pending) { + return; + } + } + step(); +} + +void CapturedFrameStream::State::finish(Status status, bool fatal) noexcept +{ + if (fatal) { + // Only the status code is retained, avoiding a fallible string copy + // on allocation-failure and cancellation paths. + failure = status.without_message(); + encoder.reset(); + stack.clear(); + load.reset(); + report.frame.reset(); + report.plan.reset(); + report.complete = false; + } + report.status = std::move(status); + auto self = std::move(lifetime); + auto callback = complete; + auto *receiver = context; + auto signal = report.status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + if (stopped.load(std::memory_order_acquire)) { + signal = detail::CallbackSignal::kStopped; + } + auto outcome = report.status.without_message(); + lifecycle.store(0, std::memory_order_release); + callback(receiver, signal, std::move(outcome)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_frames.h b/lib/crowdb-tree/src/snapshot/captured_frames.h new file mode 100644 index 000000000..a00f6607a --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_frames.h @@ -0,0 +1,120 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_encode.h" +#include "btree/page_load.h" +#include "btree/resource_observation.h" + +namespace crowdb::tree +{ +struct PrimaryReuseOutput; +class WritebackMetrics; + +struct CapturedFrameLimits +{ + PageEncodeLimits encode{}; + size_t max_depth = 64; + // Derived from captured MappingView copy bounds before admission. A work + // window limits cursor state, not the source arrays retained by a leaf. + size_t page_metadata_bytes = 0; + // Maximum typed range bases in a captured leaf, derived before admission. + uint64_t max_pages = MappingView::kPageLimit; + // Traverse and prepare final layouts without copying payload or reserving + // overflow PIDs. Each result owns the returned dependency descriptors. + bool plan_only = false; + // Staged writeback can materialize one primary while retaining the complete + // captured mapping for routing and immutable overflow ownership proofs. + std::optional<uint64_t> selected_page = std::nullopt; + // Checkpoint-only: reuse verified clean primaries in this exact backend. + uint64_t reuse_backend_id = 0; + BackingExclusion exclusion{}; + std::shared_ptr<WritebackMetrics> writeback_metrics = nullptr; + std::shared_ptr<ApplyDispatcher> cpu_progress = nullptr; +}; + +struct CapturedFrameBuffer +{ + explicit CapturedFrameBuffer(const PageEncodeLimits &limits) + : resources(limits.metadata, TreeResourceMetric::MaterializeInflightBytes), + bytes(limits.frame_bytes, detail::ReservationAllocator<uint8_t, AllocationKind::kFrame>(limits.metadata)) + { + resources.begin(bytes.capacity()); + } + + ResourceObservation resources; + detail::ReservedVector<uint8_t, AllocationKind::kFrame> bytes; +}; + +struct CapturedFrame +{ + EncodedFrameInfo info; + std::shared_ptr<const CapturedFrameBuffer> buffer; + // Primary-frame dependencies keep this span and referenced old overflow + // chains alive. The sink must carry them into its durable mapping/owner. + std::span<const ReusedOverflowDependency> reused_overflow; + std::optional<FinalPageSummary> summary = std::nullopt; + std::shared_ptr<const PrimaryReuseOutput> reuse = nullptr; +}; + +struct CapturedFrameResult +{ + Status status; + std::optional<CapturedFrame> frame; + bool complete = false; + + struct Plan + { + uint64_t page_id = kInvalidPageId; + FinalPageSummary summary; + std::span<const ReusedOverflowDependency> reused_overflow; + std::shared_ptr<const void> owner; + bool reused_primary = false; + }; + + std::optional<Plan> plan = std::nullopt; + // Fatal immutable-layout result. The caller may split a still-current L1 + // representation, then build a new layout proof before capture. + std::optional<uint64_t> needs_split = std::nullopt; +}; + +struct CapturedFrameStats +{ + uint64_t pages = 0; + uint64_t frames = 0; + uint64_t loads = 0; + uint64_t cpu_steps = 0; + PageEncodeCounters encoding; +}; + +// Postorder materialization of one immutable captured L1 mapping. Each sender +// invocation yields at most one raw frame; callers write/release that window +// before requesting another. Existing overflow references are reported to the +// sink, not silently omitted from its persistence dependencies. This is not a +// self-contained native export until that sink also emits the reused chains. +class CapturedFrameStream +{ + struct State; + + public: + using ReserveOverflow = detail::InlineFunction<Status(uint64_t, ReservedOverflowIds *)>; + + static Status create(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<PageLoader> loader, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedFrameLimits limits, + ReserveOverflow reserve_overflow, std::shared_ptr<CapturedFrameStream> *out); + [[nodiscard]] static std::optional<size_t> allocation_bound(const CapturedFrameLimits &limits) noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const CapturedFrameResult &result() const noexcept; + [[nodiscard]] const CapturedFrameStats &stats() const noexcept; + void request_stop() noexcept; + + private: + explicit CapturedFrameStream(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_frames_internal.h b/lib/crowdb-tree/src/snapshot/captured_frames_internal.h new file mode 100644 index 000000000..f217a8122 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_frames_internal.h @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/apply_dispatcher.h" +#include "captured_frames.h" + +namespace crowdb::tree +{ +struct CapturedFrameStream::State : std::enable_shared_from_this<State> +{ + struct Path + { + uint64_t pid; + std::shared_ptr<const PublishedPage> page; + PageKeyRange range; + size_t child = 0; + InnerEncodeFences fences; + bool validated = false; + }; + + struct Load + { + std::atomic<bool> claimed{false}; + std::atomic<bool> ready{false}; + Status status; + size_t base = 0; + bool is_inner = false; + std::shared_ptr<const LeafBase> leaf; + std::shared_ptr<const InnerPageState> inner; + }; + + State(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<PageLoader> transport, + std::shared_ptr<detail::FlushRuntime> cpu, std::shared_ptr<detail::FlushRuntime::Lane> queue, + CapturedFrameLimits bounds, ReserveOverflow reserve); + void start(detail::CallbackComplete callback, void *receiver) noexcept; + void wake() noexcept; + static void run(void *context) noexcept; + void pump(); + void step(); + bool initialize(); + void push(uint64_t pid, PageKeyRange range, std::optional<uint32_t> expected_level); + void validate(Path &path); + void prepare(); + void prepare_encoder(); + void emit(); + void plan(); + void completed_page(bool nonempty_leaf); + bool reuse_primary(); + void load_base(const BaseLoadRequest &request); + void load_inner(); + void loaded(); + void finish(Status status, bool fatal = false) noexcept; + + std::shared_ptr<const PublishedTreeState> view; + std::shared_ptr<PageLoader> loader; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + CapturedFrameLimits limits; + ReserveOverflow reserve_overflow; + detail::ReservedVector<Path> stack; + std::shared_ptr<CapturedFrameBuffer> buffer; + detail::ReservedUnique<FinalPageEncoder> encoder; + std::shared_ptr<Load> load; + std::shared_ptr<State> lifetime; + std::shared_ptr<State> queued_owner; + ApplyDispatcher::Node cpu_node{&State::run, this}; + CapturedFrameResult report; + CapturedFrameStats counters; + uint64_t overflow_end = 0; + Status failure; + bool initialized = false; + bool encoder_ready = false; + bool begin_pending = false; + std::atomic<bool> stopped{false}; + std::atomic<bool> scheduled{false}; + std::atomic<bool> wake_requested{false}; + std::atomic<uint8_t> lifecycle{0}; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_frames_load.cpp b/lib/crowdb-tree/src/snapshot/captured_frames_load.cpp new file mode 100644 index 000000000..9fc5525a3 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_frames_load.cpp @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "captured_frames_internal.h" + +namespace crowdb::tree +{ +void CapturedFrameStream::State::load_base(const BaseLoadRequest &request) +{ + if (!loader) { + finish(Status::unavailable("snapshot needs a captured leaf loader"), true); + return; + } + auto event = detail::reserved_shared<Load>(limits.encode.metadata); + event->base = request.base_index; + load = event; + ++counters.loads; + auto self = shared_from_this(); + const auto physical = stack.back().page->leaf->bases.at(request.base_index); + loader->load_leaf(physical, [self, event](Status status, auto leaf) { + if (event->claimed.exchange(true, std::memory_order_acq_rel)) { + return; + } + event->status = std::move(status); + event->leaf = std::move(leaf); + event->ready.store(true, std::memory_order_release); + self->wake(); + }); +} + +void CapturedFrameStream::State::load_inner() +{ + const auto &backing = stack.back().page->inner_backing; + if (!backing || !backing->readable()) { + finish(Status::corruption("captured inner has no reloadable backing"), true); + return; + } + if (!loader) { + finish(Status::unavailable("snapshot needs a captured index loader"), true); + return; + } + auto event = detail::reserved_shared<Load>(limits.encode.metadata); + event->is_inner = true; + load = event; + ++counters.loads; + auto self = shared_from_this(); + loader->load_inner(backing, [self, event](Status status, auto inner) { + if (event->claimed.exchange(true, std::memory_order_acq_rel)) { + return; + } + event->status = std::move(status); + event->inner = std::move(inner); + event->ready.store(true, std::memory_order_release); + self->wake(); + }); +} + +void CapturedFrameStream::State::loaded() +{ + auto event = std::move(load); + if (!event->status.ok()) { + finish(std::move(event->status), true); + return; + } + if ((event->is_inner && !event->inner) || (!event->is_inner && !event->leaf)) { + finish(Status::corruption("snapshot loader returned no page"), true); + return; + } + auto &path = stack.back(); + auto page = detail::reserved_shared<PublishedPage>(limits.encode.metadata, *path.page); + if (event->is_inner) { + page->inner = std::move(event->inner); + } + else { + auto leaf = detail::reserved_shared<PageState>(limits.encode.metadata, *page->leaf, limits.encode.metadata); + const auto maximum = leaf->bases.at(event->base).max_slot; + if (maximum) { + for (size_t i = 0; i < event->leaf->count(); ++i) { + if (CellView(event->leaf->cell(i)).slot() > *maximum) { + finish(Status::corruption("snapshot base exceeds its captured slot bound"), true); + return; + } + } + } + leaf->bases[event->base].resident = std::move(event->leaf); + page->leaf = std::move(leaf); + } + path.page = std::move(page); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_frames_reuse.cpp b/lib/crowdb-tree/src/snapshot/captured_frames_reuse.cpp new file mode 100644 index 000000000..cdf48fa73 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_frames_reuse.cpp @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "captured_frames_internal.h" +#include "primary_backing.h" + +namespace crowdb::tree +{ +bool CapturedFrameStream::State::reuse_primary() +{ + const auto &path = stack.back(); + const bool leaf = path.page->level == 0; + if (leaf ? !can_reuse_primary(*path.page, path.pid, path.range, view->gc_floor, limits.reuse_backend_id, + limits.encode.frame_bytes) + : !can_reuse_inner_primary(*path.page, path.pid, path.fences, limits.reuse_backend_id, + limits.encode.frame_bytes)) { + return false; + } + const auto &proof = path.page->primary; + if (limits.exclusion.overlaps(*proof->frame.backing)) { + return false; + } + const auto &summary = proof->frame.summary; + if (!summary) { + return false; + } + const auto references = proof->frame.stored_overflow + ? std::span<const StoredOverflowReference>(*proof->frame.stored_overflow) + : std::span<const StoredOverflowReference>{}; + if (references.size() > limits.encode.max_entries || + references.size() > limits.encode.max_metadata_bytes / sizeof(ReusedOverflowDependency)) { + finish(Status::resource_exhausted("clean primary dependency window exceeds its bound"), true); + return true; + } + auto output = detail::reserved_shared<PrimaryReuseOutput>(limits.encode.metadata, limits.encode.metadata); + output->primary = proof; + output->dependencies.reserve(references.size()); + for (const auto &reference : references) { + output->dependencies.push_back({.head = reference.head, + .bytes = reference.bytes, + .owner = proof->frame.backing, + .backing = proof->frame.backing->overflow}); + } + if (limits.plan_only) { + report.plan = CapturedFrameResult::Plan{.page_id = path.pid, + .summary = *summary, + .reused_overflow = output->dependencies, + .owner = output, + .reused_primary = true}; + } + else { + report.frame = CapturedFrame{ + .info = {.page_id = path.pid, .type = proof->frame.type, .bytes = proof->frame.frame_bytes}, + .buffer = {}, + .reused_overflow = output->dependencies, + .summary = proof->frame.summary, + .reuse = output + }; + ++counters.frames; + } + completed_page(summary->encoded_entries != 0); + report.complete = stack.empty(); + finish(Status::Ok()); + return true; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_frames_step.cpp b/lib/crowdb-tree/src/snapshot/captured_frames_step.cpp new file mode 100644 index 000000000..78fe66393 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_frames_step.cpp @@ -0,0 +1,270 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/writeback_metrics.h" +#include "captured_frames_internal.h" +#include "writeback_selection.h" + +namespace crowdb::tree +{ +void CapturedFrameStream::State::push(uint64_t pid, PageKeyRange range, std::optional<uint32_t> expected_level) +{ + if (stack.size() == limits.max_depth || counters.pages == limits.max_pages) { + finish(Status::resource_exhausted("captured traversal exceeds its depth/page bound"), true); + return; + } + auto page = view->mapping.get(pid); + if (!page || pid >= view->next_page_id || (expected_level && page->level != *expected_level)) { + finish(Status::corruption("captured child is absent or has an invalid level"), true); + return; + } + ++counters.pages; + stack.push_back({.pid = pid, .page = std::move(page), .range = std::move(range), .fences = {}}); +} + +void CapturedFrameStream::State::validate(Path &path) +{ + if (path.page->level == 0) { + if (!path.page->leaf || path.page->inner || path.page->leaf->range.lower != path.range.lower || + path.page->leaf->range.upper != path.range.upper) { + finish(Status::corruption("captured leaf does not match its routing range"), true); + return; + } + } + else { + const auto &inner = path.page->inner; + if (!inner || path.page->leaf || inner->children.empty() || + inner->children.size() != inner->separators.size() + 1) { + finish(Status::corruption("captured index shape is invalid"), true); + return; + } + if (inner->children.size() > limits.encode.max_entries) { + finish(Status::resource_exhausted("captured index exceeds the child bound"), true); + return; + } + for (size_t i = 0; i < inner->separators.size(); ++i) { + const auto &key = inner->separators[i]; + if ((path.range.lower && key < *path.range.lower) || (path.range.upper && key > *path.range.upper) || + (i != 0 && inner->separators[i - 1] >= key)) { + finish(Status::corruption("captured index separators violate routing bounds"), true); + return; + } + } + } + path.validated = true; +} + +bool CapturedFrameStream::State::initialize() +{ + initialized = true; + if (limits.selected_page) { + const auto selected = view->mapping.get(*limits.selected_page); + if (!selected || (selected->level == 0 && !selected->leaf)) { + finish(Status::unavailable("selected writeback page was replaced"), true); + return false; + } + push(*limits.selected_page, selected->leaf ? selected->leaf->range : PageKeyRange{}, selected->level); + if (lifecycle.load(std::memory_order_acquire) != 2 || begin_pending) { + return false; + } + } + else if (view->root_page_id != kInvalidPageId) { + push(view->root_page_id, {}, {}); + if (lifecycle.load(std::memory_order_acquire) != 2 || begin_pending) { + return false; + } + } + else if (!view->mapping.empty()) { + finish(Status::corruption("captured nonempty mapping has no root"), true); + return false; + } + return true; +} + +void CapturedFrameStream::State::step() +{ + if (!initialized && !initialize()) { + return; + } + if (stack.empty()) { + report.complete = true; + finish(Status::Ok()); + return; + } + if (encoder) { + if (!encoder_ready) { + prepare_encoder(); + } + else if (limits.plan_only) { + plan(); + } + else { + emit(); + } + return; + } + auto &path = stack.back(); + if (path.page->level != 0 && !path.page->inner) { + load_inner(); + return; + } + if (!path.validated) { + validate(path); + if (lifecycle.load(std::memory_order_acquire) != 2 || begin_pending) { + return; + } + } + if (limits.selected_page && path.page->level != 0) { + auto status = writeback_inner_fences(*view, *path.page, limits.encode.max_entries, &path.fences); + if (!status.ok()) { + finish(std::move(status), true); + return; + } + } + if (!limits.selected_page && path.page->level != 0 && path.child < path.page->inner->children.size()) { + const auto &inner = *path.page->inner; + const auto index = path.child++; + auto range = path.range; + if (index != 0) { + range.lower = inner.separators[index - 1]; + } + if (index < inner.separators.size()) { + range.upper = inner.separators[index]; + } + push(inner.children[index], std::move(range), path.page->level - 1); + if (lifecycle.load(std::memory_order_acquire) == 2) { + wake(); + } + return; + } + if (!reuse_primary()) { + prepare(); + } +} + +void CapturedFrameStream::State::prepare() +{ + const auto &path = stack.back(); + if (!limits.plan_only && limits.writeback_metrics) { + auto status = limits.writeback_metrics->begin(path.pid); + if (!status.ok()) { + finish(std::move(status), true); + return; + } + } + auto input = view; + if (path.page != view->mapping.get(path.pid)) { + // Only this page's private loaded descriptor is substituted. Unvisited + // PIDs and inner child validation still use the original captured root. + auto resolved = detail::reserved_shared<PublishedTreeState>(limits.encode.metadata, *view); + MappingView::CopyStats copied; + const PageReplacement replacement{.page_id = path.pid, .expected = {}, .replacement = path.page}; + resolved->mapping = view->mapping.replace({&replacement, 1}, copied, limits.encode.metadata); + input = std::move(resolved); + } + CapturedEncodeContext context{.range = path.range, .overflow_dependencies = view}; + if (path.page->level != 0) { + context.inner_fences = path.fences; + } + auto prepared = prepare_final_page(input, path.pid, context, limits.encode); + if (!prepared.status.ok()) { + if (prepared.needs_split) { + report.needs_split = path.pid; + } + finish(std::move(prepared.status), true); + return; + } + if (prepared.needs_base && !prepared.encoder) { + load_base(*prepared.needs_base); + return; + } + if (prepared.needs_inner || !prepared.encoder) { + finish(Status::corruption("captured encoding did not resolve its page"), true); + return; + } + encoder = std::move(prepared.encoder); + prepare_encoder(); +} + +void CapturedFrameStream::State::prepare_encoder() +{ + const auto count = encoder->summary().new_overflow_frames; + if (!limits.plan_only && count != 0) { + ReservedOverflowIds ids; + auto status = reserve_overflow ? reserve_overflow(count, &ids) + : Status::unavailable("snapshot needs an owned overflow identity reservation"); + if (status.ok() && ids.first < overflow_end) { + status = Status::invalid_argument("snapshot overflow intervals must be disjoint and increasing"); + } + const auto end = ids.first <= MappingView::kPageLimit && ids.count <= MappingView::kPageLimit - ids.first + ? ids.first + ids.count + : MappingView::kPageLimit; + if (status.ok()) { + status = encoder->bind_overflow(std::move(ids)); + } + if (!status.ok()) { + finish(std::move(status), true); + return; + } + overflow_end = end; + } + encoder_ready = true; + wake(); +} + +void CapturedFrameStream::State::emit() +{ + auto output = encoder->next(buffer->bytes); + if (!output.status.ok()) { + finish(std::move(output.status), true); + return; + } + if (!output.frame) { + finish(Status::corruption("snapshot encoder produced no frame"), true); + return; + } + report.frame = CapturedFrame{.info = std::move(*output.frame), .buffer = buffer, .reused_overflow = {}}; + ++counters.frames; + if (output.complete) { + report.frame->reused_overflow = encoder->reused_overflow(); + report.frame->summary = encoder->summary(); + const auto &encoded = encoder->counters(); + counters.encoding.primary_frames += encoded.primary_frames; + counters.encoding.overflow_frames += encoded.overflow_frames; + counters.encoding.key_copy_bytes += encoded.key_copy_bytes; + counters.encoding.inline_value_copy_bytes += encoded.inline_value_copy_bytes; + counters.encoding.overflow_value_copy_bytes += encoded.overflow_value_copy_bytes; + completed_page(encoder->summary().encoded_entries != 0); + } + report.complete = stack.empty(); + finish(Status::Ok()); +} + +void CapturedFrameStream::State::plan() +{ + report.plan = CapturedFrameResult::Plan{.page_id = stack.back().pid, + .summary = encoder->summary(), + .reused_overflow = encoder->reused_overflow(), + .owner = encoder->dependency_owner()}; + completed_page(encoder->summary().encoded_entries != 0); + report.complete = stack.empty(); + finish(Status::Ok()); +} + +void CapturedFrameStream::State::completed_page(bool nonempty_leaf) +{ + auto fences = stack.back().fences; + if (stack.back().page->level == 0 && nonempty_leaf) { + fences = {.lower_leaf = stack.back().pid, .upper_leaf = stack.back().pid}; + } + encoder.reset(); + encoder_ready = false; + stack.pop_back(); + if (!stack.empty() && fences.lower_leaf != kInvalidPageId) { + auto &parent = stack.back().fences; + if (parent.lower_leaf == kInvalidPageId) { + parent.lower_leaf = fences.lower_leaf; + } + parent.upper_leaf = fences.upper_leaf; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_plan.cpp b/lib/crowdb-tree/src/snapshot/captured_plan.cpp new file mode 100644 index 000000000..7b77dc610 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_plan.cpp @@ -0,0 +1,175 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/captured_plan.h" + +#include "btree/allocation_bound.h" +#include "btree/overflow_backing.h" +#include "crowdb-tree/maptable/mapping_geometry.h" + +#include <algorithm> + +namespace crowdb::tree +{ +CapturedLayoutPlanner::CapturedLayoutPlanner(uint64_t next_page_id, uint64_t backend_id, CapturedWriteLimits limits, + size_t max_segments, std::shared_ptr<const CapturedDurablePages> inherited, + std::shared_ptr<detail::AllocationReservation> metadata) + : next_(next_page_id), + backend_(backend_id), + limits_(std::move(limits)), + max_segments_(max_segments), + inherited_(std::move(inherited)), + segments_(std::less<>{}, detail::ReservationAllocator<uint64_t>(metadata)), + rebase_segments_(std::less<>{}, detail::ReservationAllocator<uint64_t>(metadata)) +{ +} + +std::optional<size_t> CapturedLayoutPlanner::allocation_bound(uint64_t next_page_id, size_t max_segments) noexcept +{ + const auto segments = (next_page_id + kMappingSegmentSize - 1) / kMappingSegmentSize; + const auto rebase_segments = + (next_page_id / MappingView::kFanout) + static_cast<uint64_t>(next_page_id % MappingView::kFanout != 0); + const auto max_rebase_segments = + detail::AllocationBound().add_product(max_segments, kMappingSegmentSize / MappingView::kFanout).value(); + return detail::AllocationBound(detail::shared_allocation_bound<CapturedLayoutPlanner>()) + .add_product(std::min<uint64_t>(segments, max_segments), sizeof(uint64_t) + (4 * sizeof(void *))) + .add(max_rebase_segments ? detail::AllocationBound() + .add_product(std::min<uint64_t>(rebase_segments, *max_rebase_segments), + sizeof(uint64_t) + (4 * sizeof(void *))) + .value() + : std::nullopt) + .value(); +} + +Status CapturedLayoutPlanner::segment(uint64_t pid) +{ + if (pid >= next_) { + return Status::corruption("captured PID exceeds the planned namespace"); + } + const auto number = pid / kMappingSegmentSize; + if (!segments_.contains(number)) { + if (segments_.size() == max_segments_) { + return Status::resource_exhausted("checkpoint segment limit"); + } + segments_.insert(number); + } + return Status::Ok(); +} + +Status CapturedLayoutPlanner::accept(CapturedFrameResult::Plan page) +{ + if (page_ || plan_.primary_frames == limits_.max_frames || + page.summary.new_overflow_frames > limits_.max_frames - plan_.primary_frames - 1 || + plan_.new_overflow_frames > limits_.max_frames - plan_.primary_frames - 1 - page.summary.new_overflow_frames || + page.reused_overflow.size() > limits_.max_dependency_steps - plan_.reference_heads) { + return Status::resource_exhausted("checkpoint layout exceeds its descriptor bound"); + } + auto status = segment(page.page_id); + if (!status.ok()) { + return status; + } + ++plan_.primary_frames; + // Relocation can replace overflow dependencies while keeping the primary + // bytes. Such receipts still require a rebase and keep the full allowance. + if (page.reused_primary && limits_.frames.exclusion.length == 0) { + ++plan_.reused_primary_frames; + } + else { + rebase_segments_.insert(page.page_id / MappingView::kFanout); + } + plan_.new_overflow_frames += page.summary.new_overflow_frames; + plan_.reference_heads += page.reused_overflow.size(); + reference_ = 0; + pid_ = kInvalidPageId; + page_ = std::move(page); + return Status::Ok(); +} + +Status CapturedLayoutPlanner::step() +{ + if (!page_) { + return Status::invalid_argument(); + } + if (reference_ == page_->reused_overflow.size()) { + page_.reset(); + return Status::Ok(); + } + if (plan_.reused_dependency_entries == limits_.max_dependency_steps) { + return Status::resource_exhausted("checkpoint overflow dependency step limit"); + } + const auto &reference = page_->reused_overflow[reference_]; + if (pid_ == kInvalidPageId) { + pid_ = reference.head; + remaining_ = reference.bytes; + } + OverflowBackingPage old; + if (reference.backing) { + const auto found = reference.backing->pages.find(pid_); + if (found == reference.backing->pages.end()) { + return Status::corruption("missing captured overflow"); + } + old = found->second; + } + else { + if (!inherited_) { + return Status::unavailable("captured overflow has no protected descriptor"); + } + const auto found = inherited_->frames.find(pid_); + if (found == inherited_->frames.end() || found->second.type != page_type::kOverflowFrame) { + return Status::corruption("missing captured overflow"); + } + const auto &frame = found->second; + old = {.backing = frame.backing, + .next = frame.next_overflow, + .bytes = frame.overflow_bytes, + .frame_bytes = frame.frame_bytes, + .logical_bytes = frame.logical_bytes}; + } + const bool resident = old.backing && old.backing->storage == BackingStorage::kResidentFrame; + if (!old.backing || !old.backing->readable() || (!resident && old.backing->backend_id != backend_) || + (resident && + (old.backing->resident_frame.size() != old.frame_bytes || old.frame_bytes > limits_.write.max_frame_bytes)) || + old.bytes == 0 || old.bytes > remaining_ || (old.next == kInvalidPageId) != (old.bytes == remaining_)) { + return Status::corruption("captured overflow topology or owner differs from the cell"); + } + auto status = segment(pid_); + if (!status.ok()) { + return status; + } + ++plan_.reused_dependency_entries; + plan_.resident_dependency_entries += static_cast<size_t>(resident); + plan_.relocated_dependency_entries += static_cast<size_t>(limits_.frames.exclusion.overlaps(*old.backing)); + remaining_ -= old.bytes; + pid_ = old.next; + if (remaining_ == 0) { + ++reference_; + } + return Status::Ok(); +} + +Status CapturedLayoutPlanner::finish() +{ + if (pending()) { + return Status::invalid_argument(); + } + if (plan_.new_overflow_frames > MappingView::kPageLimit - next_) { + return Status::resource_exhausted("checkpoint overflow namespace exhausted"); + } + plan_.segments = segments_.size(); + plan_.rebase_batches = rebase_segments_.size(); + if (plan_.new_overflow_frames != 0) { + const auto first = next_ / kMappingSegmentSize; + const auto last = (next_ + plan_.new_overflow_frames - 1) / kMappingSegmentSize; + plan_.segments += last - first + 1 - static_cast<size_t>(segments_.contains(first)); + } + if (plan_.segments > max_segments_) { + return Status::resource_exhausted("checkpoint segment limit"); + } + plan_.frame_count_upper = plan_.primary_frames + plan_.new_overflow_frames; + plan_.frame_count_upper += std::min(limits_.max_frames - plan_.frame_count_upper, plan_.reused_dependency_entries); + // Each newly spilled value needs at most one stored reference per emitted + // overflow frame. Reused chains may share PIDs, so descriptor counts above + // are conservative; physical dependency ownership still counts each use. + plan_.reference_heads += plan_.new_overflow_frames; + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_plan.h b/lib/crowdb-tree/src/snapshot/captured_plan.h new file mode 100644 index 000000000..fd5114cc8 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_plan.h @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/captured_write.h" + +#include <set> + +namespace crowdb::tree +{ +struct CapturedWritePlan +{ + size_t primary_frames = 0; + // Exact primary receipts that survive unchanged into the new catalog. + size_t reused_primary_frames = 0; + size_t new_overflow_frames = 0; + size_t reused_dependency_entries = 0; + // Raw dependencies require extent leases but retain their existing PIDs. + // Shared chains are counted per use to keep admission conservative. + size_t resident_dependency_entries = 0; + size_t relocated_dependency_entries = 0; + size_t reference_heads = 0; + size_t frame_count_upper = 0; + size_t segments = 0; + // Publication batches use MappingView radix leaves, not durable segments. + size_t rebase_batches = 0; +}; + +// Consumes layout-only stream results. One step visits one old dependency; +// only distinct durable/radix segment numbers accumulate, never frame payloads. +class CapturedLayoutPlanner +{ + public: + CapturedLayoutPlanner(uint64_t next_page_id, uint64_t backend_id, CapturedWriteLimits limits, size_t max_segments, + std::shared_ptr<const CapturedDurablePages> inherited, + std::shared_ptr<detail::AllocationReservation> metadata); + Status accept(CapturedFrameResult::Plan page); + Status step(); + Status finish(); + + [[nodiscard]] bool pending() const noexcept + { + return page_.has_value(); + } + + [[nodiscard]] const CapturedWritePlan &result() const noexcept + { + return plan_; + } + + [[nodiscard]] static std::optional<size_t> allocation_bound(uint64_t next_page_id, size_t max_segments) noexcept; + + private: + Status segment(uint64_t pid); + uint64_t next_; + uint64_t backend_; + CapturedWriteLimits limits_; + size_t max_segments_; + std::shared_ptr<const CapturedDurablePages> inherited_; + std::set<uint64_t, std::less<>, detail::ReservationAllocator<uint64_t>> segments_; + std::set<uint64_t, std::less<>, detail::ReservationAllocator<uint64_t>> rebase_segments_; + std::optional<CapturedFrameResult::Plan> page_; + size_t reference_ = 0; + uint64_t pid_ = kInvalidPageId; + uint64_t remaining_ = 0; + CapturedWritePlan plan_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_rebase.cpp b/lib/crowdb-tree/src/snapshot/captured_rebase.cpp new file mode 100644 index 000000000..0607ed9f4 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_rebase.cpp @@ -0,0 +1,165 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/page_cache_residency.h" +#include "btree/writeback_rebase.h" +#include "snapshot/captured_rebase_internal.h" +#include "snapshot/primary_backing.h" + +#include <algorithm> + +namespace crowdb::tree +{ +Status CapturedRebaseOperation::create( + PublicationCoordinator &publisher, std::shared_ptr<const PublishedTreeState> captured, + std::shared_ptr<const CapturedDurablePages> pages, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<CapturedRebaseOperation> *out, size_t max_sources, + std::shared_ptr<WritebackMetrics> writeback_metrics, std::shared_ptr<ApplyDispatcher> cpu_progress, + std::shared_ptr<PublicationLane> publication, TreeWorkCause cause) +try { + if (!captured || !pages || !runtime || !lane || out == nullptr || *out || max_sources == 0 || + captured->generation != pages->generation || captured->coverage != pages->coverage) { + return Status::invalid_argument(); + } + auto state = detail::reserved_shared<State>(metadata, publisher); + state->captured = std::move(captured); + state->pages = std::move(pages); + state->next = state->pages->frames.begin(); + return initialize(std::move(state), std::move(runtime), std::move(lane), std::move(metadata), out, max_sources, + std::move(writeback_metrics), std::move(cpu_progress), std::move(publication), cause); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status CapturedRebaseOperation::initialize(std::shared_ptr<State> state, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<CapturedRebaseOperation> *out, size_t max_sources, + std::shared_ptr<WritebackMetrics> writeback_metrics, + std::shared_ptr<ApplyDispatcher> cpu_progress, + std::shared_ptr<PublicationLane> publication, TreeWorkCause cause) +{ + state->runtime = std::move(runtime); + state->metadata = std::move(metadata); + state->max_sources = max_sources; + state->writeback_metrics = std::move(writeback_metrics); + state->cpu_progress = std::move(cpu_progress); + state->publication = std::move(publication); + state->cause = cause; + if (!state->cpu_progress) { + auto status = state->runtime->reserve_progress(std::move(lane), &State::run, state.get(), state->metadata, + &state->progress); + if (!status.ok()) { + return status; + } + } + *out = detail::reserved_construct<CapturedRebaseOperation>(state->metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) CapturedRebaseOperation(std::move(state)); + }); + return Status::Ok(); +} + +std::optional<size_t> CapturedRebaseOperation::allocation_bound(size_t primary_frames, size_t max_sources, + bool track_residency, size_t rebase_batches) noexcept +{ + using detail::shared_allocation_bound; + if (max_sources == 0) { + return {}; + } + const auto per_page = detail::AllocationBound(detail::shared_allocation_bound<PublishedPage>()) + .add(shared_allocation_bound<PublishedPage>() + PageState::allocation_bound() + + shared_allocation_bound<PrimaryBackingMetadata>() + sizeof(BaseRef)) + .add_product(max_sources, sizeof(std::shared_ptr<const OverlayBatch>)) + .add(track_residency ? PageCacheResidency::page_allocation_bound(1) : 0) + .value(); + if (!per_page) { + return {}; + } + return detail::AllocationBound( + PublicationCoordinator::empty_commit_reserve_bytes() + shared_allocation_bound<PublicationRequest>() + + shared_allocation_bound<InstallRecord>() + shared_allocation_bound<State>() + + shared_allocation_bound<CapturedRebaseOperation>() + detail::FlushRuntime::progress_allocation_bound()) + .add_product(primary_frames, *per_page) + .add_product(std::min(primary_frames, rebase_batches), + MappingView::path_copy_bound() + PublicationCoordinator::empty_commit_reserve_bytes() + + shared_allocation_bound<PublicationRequest>() + shared_allocation_bound<InstallRecord>() + + (MappingView::kFanout * sizeof(PageReplacement))) + .value(); +} + +detail::CallbackSender CapturedRebaseOperation::sender() noexcept +{ + return {this, &CapturedRebaseOperation::submit}; +} + +const CapturedRebaseResult &CapturedRebaseOperation::result() const noexcept +{ + return state_->report; +} + +void CapturedRebaseOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void CapturedRebaseOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto &state = *static_cast<CapturedRebaseOperation *>(source)->state_; + uint8_t idle = 0; + if (!state.lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + complete(context, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + state.complete = complete; + state.context = context; + state.lifecycle.store(2, std::memory_order_release); + state.wake(); +} + +void CapturedRebaseOperation::State::wake() noexcept +{ + if (lifecycle.load(std::memory_order_acquire) == 2) { + auto self = weak_from_this().lock(); + if (cpu_progress) { + cpu_progress->request(cpu_node, std::move(self)); + } + else { + progress->request(std::move(self)); + } + } +} + +void CapturedRebaseOperation::State::run(void *context) noexcept +{ + auto &state = *static_cast<State *>(context); + if (state.lifecycle.load(std::memory_order_acquire) != 2) { + return; + } + try { + state.step(); + } + catch (const std::bad_alloc &) { + state.finish(Status::resource_exhausted()); + } + catch (...) { + state.finish(Status::internal_error()); + } +} + +void CapturedRebaseOperation::State::finish(Status status) noexcept +{ + if (lifecycle.exchange(3, std::memory_order_acq_rel) == 3) { + return; + } + report.status = std::move(status); + request.reset(); + summary_view.reset(); + captured.reset(); + pages.reset(); + complete(context, report.status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError, + report.status.without_message()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_rebase.h b/lib/crowdb-tree/src/snapshot/captured_rebase.h new file mode 100644 index 000000000..33b079286 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_rebase.h @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/captured_write.h" + +namespace crowdb::tree +{ +struct CapturedRebaseResult +{ + Status status; + uint64_t installed = 0; + uint64_t deferred = 0; + // Exact captured primary receipts require no representation replacement. + uint64_t reused = 0; +}; + +// Installs protected reloadable receipts after staged writeback or durable +// Commit. A conflict leaves the live page untouched; failure only defers +// reclamation and never advances or rolls back durable coverage. +class CapturedRebaseOperation +{ + struct State; + + public: + static Status + create(PublicationCoordinator &publisher, std::shared_ptr<const PublishedTreeState> captured, + std::shared_ptr<const CapturedDurablePages> pages, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<CapturedRebaseOperation> *out, size_t max_sources = 128, + std::shared_ptr<WritebackMetrics> writeback_metrics = {}, std::shared_ptr<ApplyDispatcher> cpu_progress = {}, + std::shared_ptr<PublicationLane> publication = {}, TreeWorkCause cause = TreeWorkCause::Checkpoint); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t primary_frames, size_t max_sources = 128, + bool track_residency = true, + size_t rebase_batches = SIZE_MAX) noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const CapturedRebaseResult &result() const noexcept; + void request_stop() noexcept; + + private: + explicit CapturedRebaseOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + static Status initialize(std::shared_ptr<State> state, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<CapturedRebaseOperation> *out, size_t max_sources, + std::shared_ptr<WritebackMetrics> writeback_metrics, + std::shared_ptr<ApplyDispatcher> cpu_progress, + std::shared_ptr<PublicationLane> publication, TreeWorkCause cause); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_rebase_batch.cpp b/lib/crowdb-tree/src/snapshot/captured_rebase_batch.cpp new file mode 100644 index 000000000..c82e4caf6 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_rebase_batch.cpp @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/writeback_rebase.h" +#include "snapshot/captured_rebase_internal.h" +#include "snapshot/primary_backing.h" + +namespace crowdb::tree +{ +namespace +{ +bool captured_receipt_is_unchanged(const PublishedPage &page, const DurableCapturedFrame &receipt) +{ + if (!page.primary || page.primary->frame.backing != receipt.backing) { + return false; + } + if (page.level != 0) { + return page.inner_backing == receipt.backing; + } + return page.leaf && page.leaf->overlays.empty() && page.leaf->bases.size() == 1 && + page.leaf->bases.front().backing == receipt.backing; +} +} // namespace + +void CapturedRebaseOperation::State::finish_batch(TreeOutcome outcome) +{ + for (const auto &page : request->transaction.pages) { + if (outcome == TreeOutcome::Success) { + ++report.installed; + } + else { + ++report.deferred; + if (outcome == TreeOutcome::Stale) { + record_work(publisher.work_metrics(), TreeWorkCounter::RebaseMismatches, 1, cause); + } + } + if (writeback_metrics) { + writeback_metrics->finish(page.page_id, outcome); + } + } +} + +void CapturedRebaseOperation::State::prepare_batch() +{ + const auto segment = next->first / MappingView::kFanout; + const auto live = publisher.capture(); + // One fixed mapping segment bounds both CPU work and radix copies. Each + // segment is visited once, even when only a subset of its receipts change. + for (; next != pages->frames.end() && next->first / MappingView::kFanout == segment; ++next) { + const auto &receipt = next->second; + if (receipt.type == page_type::kOverflowFrame) { + continue; + } + const auto original = captured->mapping.get(receipt.page_id); + if (original && captured_receipt_is_unchanged(*original, receipt)) { + ++report.reused; + if (writeback_metrics) { + writeback_metrics->finish(receipt.page_id, TreeOutcome::Success); + } + continue; + } + const auto current = live->mapping.get(receipt.page_id); + std::shared_ptr<const PublishedPage> replacement; + const auto status = live->generation != captured->generation + ? Status::unavailable() + : build_writeback_rebase(original, current, receipt, captured->coverage, metadata, + &replacement, max_sources, captured->gc_floor); + if (!status.ok()) { + defer(receipt.page_id, status); + continue; + } + add_replacement(receipt.page_id, current, std::move(replacement)); + } +} + +void CapturedRebaseOperation::State::defer(uint64_t pid, Status status) +{ + ++report.deferred; + const bool mismatch = status.code() == Code::kUnavailable; + if (mismatch) { + record_work(publisher.work_metrics(), TreeWorkCounter::RebaseMismatches, 1, cause); + } + if (writeback_metrics) { + writeback_metrics->finish(pid, mismatch ? TreeOutcome::Stale : TreeOutcome::Error); + } +} + +void CapturedRebaseOperation::State::add_replacement(uint64_t pid, std::shared_ptr<const PublishedPage> current, + std::shared_ptr<const PublishedPage> replacement) +{ + if (!request) { + request = detail::reserved_shared<PublicationRequest>(metadata, metadata); + request->transaction.record = detail::reserved_shared<InstallRecord>( + metadata, InstallToken{.generation = captured->generation, .sequence = pid}); + request->transaction.pages.reserve(MappingView::kFanout); + } + request->transaction.pages.push_back( + {.page_id = pid, .expected = std::move(current), .replacement = std::move(replacement)}); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_rebase_internal.h b/lib/crowdb-tree/src/snapshot/captured_rebase_internal.h new file mode 100644 index 000000000..da14824d0 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_rebase_internal.h @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/apply_dispatcher.h" +#include "btree/writeback_metrics.h" +#include "snapshot/captured_rebase.h" + +namespace crowdb::tree +{ +struct CapturedRebaseOperation::State : std::enable_shared_from_this<State> +{ + explicit State(PublicationCoordinator &source) : publisher(source) + { + } + + static void run(void *context) noexcept; + void wake() noexcept; + void step(); + void prepare_batch(); + void add_replacement(uint64_t pid, std::shared_ptr<const PublishedPage> current, + std::shared_ptr<const PublishedPage> replacement); + void defer(uint64_t pid, Status status); + void finish_batch(TreeOutcome outcome); + void summary_step(); + void finish(Status status) noexcept; + ApplyDispatcher::Node cpu_node{&State::run, this}; + std::shared_ptr<ApplyDispatcher> cpu_progress; + std::shared_ptr<PublicationLane> publication; + PublicationCoordinator &publisher; + std::shared_ptr<const PublishedTreeState> captured; + std::shared_ptr<const CapturedDurablePages> pages; + CapturedDurablePages::Frames::const_iterator next; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Progress> progress; + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<PublicationRequest> request; + std::shared_ptr<WritebackMetrics> writeback_metrics; + size_t max_sources = 0; + bool waiting = false; + bool summarizing = false; + bool summary_pending = false; + std::shared_ptr<const PublishedTreeState> summary_view; + TreeSummary summary; + Status publication_status; + std::atomic<bool> ready{false}; + std::atomic<bool> stopped{false}; + std::atomic<uint8_t> lifecycle{0}; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + CapturedRebaseResult report; + TreeWorkCause cause = TreeWorkCause::Checkpoint; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_rebase_step.cpp b/lib/crowdb-tree/src/snapshot/captured_rebase_step.cpp new file mode 100644 index 000000000..6240e0578 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_rebase_step.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/writeback_rebase.h" +#include "snapshot/captured_rebase_internal.h" +#include "snapshot/primary_backing.h" + +namespace crowdb::tree +{ +void CapturedRebaseOperation::State::step() +{ + if (waiting) { + if (!ready.load(std::memory_order_acquire)) { + return; + } + waiting = false; + if (summary_pending) { + finish(Status::Ok()); + return; + } + auto outcome = TreeOutcome::Success; + if (!publication_status.ok()) { + outcome = publication_status.code() == Code::kUnavailable ? TreeOutcome::Stale : TreeOutcome::Error; + } + finish_batch(outcome); + request.reset(); + } + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + if (summarizing) { + summary_step(); + return; + } + if (next == pages->frames.end()) { + summary_view = publisher.capture(); + if (report.deferred != 0 || summary_view->generation != captured->generation || + summary_view->coverage != pages->coverage || summary_view->root_page_id != pages->root_page_id || + summary_view->pending_handoff) { + finish(Status::Ok()); + return; + } + summarizing = true; + next = pages->frames.begin(); + summary.covered_slot = pages->coverage; + wake(); + return; + } + prepare_batch(); + if (!request) { + wake(); + return; + } + waiting = true; + ready.store(false, std::memory_order_release); + request->on_done = [self = shared_from_this()](PublicationResult result) { + self->publication_status = result.status.without_message(); + self->ready.store(true, std::memory_order_release); + self->wake(); + }; + const auto submitted = publication ? publisher.try_submit(request, publication) : publisher.try_submit(request); + if (!submitted) { + waiting = false; + finish_batch(TreeOutcome::Error); + record_work(publisher.work_metrics(), TreeWorkCounter::AdmissionRejected, 1, cause); + request.reset(); + wake(); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_rebase_summary.cpp b/lib/crowdb-tree/src/snapshot/captured_rebase_summary.cpp new file mode 100644 index 000000000..483f9e5b8 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_rebase_summary.cpp @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/captured_rebase_internal.h" +#include "snapshot/graph_summary.h" +#include "snapshot/primary_backing.h" + +namespace crowdb::tree +{ +namespace +{ +bool matches_frame(const PublishedTreeState &view, const ReadablePageFrame &frame) +{ + if (frame.type == page_type::kOverflowFrame) { + return true; + } + const auto page = view.mapping.get(frame.page_id); + if (!page || page->metrics.dirty || !page->primary || page->primary->frame.backing != frame.backing) { + return false; + } + if (frame.type == page_type::kLeafBase) { + return page->leaf && page->leaf->overlays.empty() && page->leaf->bases.size() == 1 && + page->leaf->bases.front().backing == frame.backing; + } + return frame.type == page_type::kInnerBase && page->level != 0 && page->inner_backing == frame.backing; +} +} // namespace + +void CapturedRebaseOperation::State::summary_step() +{ + for (size_t work = 0; next != pages->frames.end() && work < 16; ++work, ++next) { + const auto &frame = next->second; + if (!matches_frame(*summary_view, frame) || !add_graph_summary(summary, frame)) { + finish(Status::Ok()); + return; + } + } + if (next != pages->frames.end()) { + wake(); + return; + } + if (summary.reachable_leaf_pages != summary_view->stats.leaf_pages || + summary.reachable_inner_pages != summary_view->stats.inner_pages) { + finish(Status::Ok()); + return; + } + request = detail::reserved_shared<PublicationRequest>(metadata, metadata); + request->transaction.record = + detail::reserved_shared<InstallRecord>(metadata, InstallToken{.generation = summary_view->generation}); + request->transaction.summary = summary; + request->transaction.expected_commit_seq = summary_view->commit_seq; + request->on_done = [self = shared_from_this()](PublicationResult result) { + self->publication_status = result.status.without_message(); + self->ready.store(true, std::memory_order_release); + self->wake(); + }; + waiting = true; + summary_pending = true; + ready.store(false, std::memory_order_release); + const auto submitted = publication ? publisher.try_submit(request, publication) : publisher.try_submit(request); + if (!submitted) { + waiting = false; + finish(Status::Ok()); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_write.cpp b/lib/crowdb-tree/src/snapshot/captured_write.cpp new file mode 100644 index 000000000..59905ac7a --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_write.cpp @@ -0,0 +1,209 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/scheduled_page_store.h" +#include "btree/allocation_bound.h" +#include "btree/writeback_metrics.h" +#include "snapshot/captured_write_internal.h" + +namespace crowdb::tree +{ +std::optional<size_t> CapturedWriteOperation::allocation_bound(CapturedWriteLimits limits, size_t frame_count_upper, + size_t dependency_entries, + size_t reference_heads) noexcept +{ + using detail::shared_allocation_bound; + if (frame_count_upper > limits.max_frames || dependency_entries > limits.max_dependency_steps) { + return {}; + } + // Ordered-map node linkage/alignment is covered independently of payload. + // Owner vectors count dependency uses, including shared raw chains. The + // three-pointer allowance covers old and new capacity during growth. + constexpr size_t node_overhead = 8 * sizeof(void *); + return detail::AllocationBound( + shared_allocation_bound<State>() + shared_allocation_bound<CapturedWriteOperation>() + + shared_allocation_bound<CapturedDurablePages>() + shared_allocation_bound<State::Reservation>()) + .add_product(frame_count_upper, + sizeof(CapturedDurablePages::Entry) + sizeof(State::ResidentEntry) + (2 * node_overhead) + + shared_allocation_bound<State::Dependencies>() + + shared_allocation_bound<OverflowBackingSet>() + shared_allocation_bound<BackingRef>() + + shared_allocation_bound<StoredOverflowReferences>() + + PageWriteOperation::retained_allocation_bound() + sizeof(OverflowBackingSet::Entry) + + node_overhead + (3 * sizeof(std::shared_ptr<const void>))) + .add_product(dependency_entries, sizeof(OverflowBackingSet::Entry) + node_overhead + + (3 * sizeof(std::shared_ptr<const void>)) + + shared_allocation_bound<BackingRef>()) + .add(WritebackMetrics::allocation_bound(frame_count_upper)) + .add_product(reference_heads, sizeof(StoredOverflowReference)) + .value(); +} + +Status CapturedWriteOperation::create(std::shared_ptr<const PublishedTreeState> captured, + std::shared_ptr<PageLoader> loader, std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedWriteLimits limits, + PageLoadBudget budget, CapturedFrameStream::ReserveOverflow reserve_overflow, + PageWriteOperation::ReserveExtent reserve_extent, + std::shared_ptr<const CapturedDurablePages> inherited, + std::shared_ptr<CapturedWriteOperation> *out) +{ + if (!captured || !store || !runtime || !lane || !reserve_extent || out == nullptr || limits.max_frames == 0 || + limits.frames.plan_only || limits.max_dependency_steps == 0 || + limits.frames.encode.frame_bytes > limits.write.max_frame_bytes || + !PageWriteOperation::allocation_bound(limits.write) || + static_cast<bool>(budget.bytes) != static_cast<bool>(budget.io) || + (limits.frames.encode.metadata && limits.frames.encode.metadata != budget.bytes) || + (budget.bytes && + ((budget.bytes->category() != detail::BudgetClass::kRouting && + budget.bytes->category() != detail::BudgetClass::kReclaim) || + budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + return Status::invalid_argument(); + } + if (limits.write.chunk_backed ? dynamic_cast<ScheduledPageStore *>(store.get()) == nullptr + : !store->write_completion_is_reloadable()) { + return Status::not_supported(); + } + try { + limits.frames.encode.metadata = budget.bytes; + limits.frames.cpu_progress = budget.cpu_progress; + if (budget.work_metrics) { + limits.frames.encode.work_metrics = budget.work_metrics; + limits.frames.encode.cause = budget.cause; + } + auto state = detail::reserved_shared<State>(budget.bytes, budget.bytes); + state->store = std::move(store); + state->runtime = std::move(runtime); + state->lane = std::move(lane); + state->limits = std::move(limits); + state->budget = std::move(budget); + state->reserve = std::move(reserve_extent); + state->inherited = std::move(inherited); + state->pages = detail::reserved_shared<CapturedDurablePages>(state->budget.bytes, state->budget.bytes); + state->pages->root_page_id = captured->root_page_id; + state->pages->next_page_id = captured->next_page_id; + state->pages->coverage = captured->coverage; + state->pages->generation = captured->generation; + auto status = CapturedFrameStream::create(std::move(captured), std::move(loader), state->runtime, state->lane, + state->limits.frames, std::move(reserve_overflow), &state->stream); + if (!status.ok()) { + return status; + } + *out = detail::reserved_construct<CapturedWriteOperation>(state->budget.bytes, [&](auto *storage) { + ::new (static_cast<void *>(storage)) CapturedWriteOperation(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +detail::CallbackSender CapturedWriteOperation::sender() noexcept +{ + return {this, &CapturedWriteOperation::submit}; +} + +const CapturedWriteResult &CapturedWriteOperation::result() const noexcept +{ + return state_->report; +} + +void CapturedWriteOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void CapturedWriteOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<CapturedWriteOperation *>(source)->state_->start(complete, context); +} + +void CapturedWriteOperation::State::start(detail::CallbackComplete callback, void *argument) noexcept +{ + uint8_t idle = 0; + if (!lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + callback(argument, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + complete = callback; + context = argument; + lifecycle.store(2, std::memory_order_release); + wake(); +} + +void CapturedWriteOperation::State::Receiver::set_value() noexcept +{ + auto self = std::move(owner); + self->child_done(Status::Ok()); +} + +void CapturedWriteOperation::State::Receiver::set_error(Status status) noexcept +{ + auto self = std::move(owner); + self->child_done(std::move(status)); +} + +void CapturedWriteOperation::State::Receiver::set_stopped() noexcept +{ + auto self = std::move(owner); + self->child_done(Status::unavailable()); +} + +void CapturedWriteOperation::State::child_done(Status status) noexcept +{ + child_status = std::move(status); + child_ready.store(true, std::memory_order_release); + wake(); +} + +void CapturedWriteOperation::State::begin_child(detail::CallbackSender sender, Stage next) +{ + stage = next; + waiting = true; + child_ready.store(false, std::memory_order_release); + attempt.emplace(sender, shared_from_this()); + stdexec::start(attempt->connected); +} + +void CapturedWriteOperation::State::cancel_child() noexcept +{ + if (stage == Stage::Frame && stream) { + stream->request_stop(); + } + else if (stage == Stage::RelocateRead && relocation_read) { + relocation_read->request_stop(); + } + else if ((stage == Stage::Write || stage == Stage::ResidentWrite) && write) { + write->request_stop(); + } +} + +void CapturedWriteOperation::State::finish(Status status) noexcept +{ + if (lifecycle.exchange(3, std::memory_order_acq_rel) == 3) { + return; + } + const bool cancelled = stopped.load(std::memory_order_acquire); + report.status = cancelled ? Status::unavailable() : std::move(status); + if (report.status.ok()) { + report.pages = std::move(pages); + } + pages.reset(); + descriptor.backing.reset(); + resident_dependency.backing.reset(); + resident_sources.clear(); + descriptor.stored_overflow.reset(); + frame.reset(); + dependencies.reset(); + overflow_backing.reset(); + stream.reset(); + write.reset(); + relocation_read.reset(); + inherited.reset(); + auto signal = report.status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + if (cancelled) { + signal = detail::CallbackSignal::kStopped; + } + complete(context, signal, report.status.without_message()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_write.h b/lib/crowdb-tree/src/snapshot/captured_write.h new file mode 100644 index 000000000..6b6a1d411 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_write.h @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_writeback.h" +#include "snapshot/captured_frames.h" +#include "snapshot/readable_page_catalog.h" + +#include <map> + +namespace crowdb::tree +{ +// Only independently reloadable durable descriptors enter this type. The +// recovery adapter verifies that promise while validating the graph. +struct CapturedDurablePages : ReadablePageCatalog +{ + using ReadablePageCatalog::ReadablePageCatalog; +}; + +struct CapturedWriteLimits +{ + CapturedFrameLimits frames{}; + PageWriteLimits write{}; + size_t max_frames = 1024 * 1024; + size_t max_dependency_steps = 1024 * 1024; +}; + +struct CapturedWriteResult +{ + Status status; + std::shared_ptr<const CapturedDurablePages> pages; + uint64_t written_frames = 0; + uint64_t written_bytes = 0; + uint64_t reused_frames = 0; +}; + +// Materializes a captured view into protected, independently readable page +// references. It does not write a directory/anchor or advance durable coverage. +// Existing overflow cells retain verified dependencies through their primary +// backing. The inherited mapping supports older callers without typed backing +// descriptors; missing or mismatched dependencies fail explicitly. +class CapturedWriteOperation +{ + struct State; + + public: + // Sink controls and retained descriptors only. Add the stream, loader and + // one PageWriteOperation window separately. Counts come from layout preflight. + [[nodiscard]] static std::optional<size_t> allocation_bound(CapturedWriteLimits limits, size_t frame_count_upper, + size_t dependency_entries, + size_t reference_heads) noexcept; + static Status create(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<PageLoader> loader, + std::shared_ptr<AsyncPageStore> store, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedWriteLimits limits, + PageLoadBudget budget, CapturedFrameStream::ReserveOverflow reserve_overflow, + PageWriteOperation::ReserveExtent reserve_extent, + std::shared_ptr<const CapturedDurablePages> inherited, + std::shared_ptr<CapturedWriteOperation> *out); + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const CapturedWriteResult &result() const noexcept; + void request_stop() noexcept; + + private: + explicit CapturedWriteOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_write_internal.h b/lib/crowdb-tree/src/snapshot/captured_write_internal.h new file mode 100644 index 000000000..4596aaf3a --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_write_internal.h @@ -0,0 +1,146 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/apply_dispatcher.h" +#include "btree/overflow_backing.h" +#include "snapshot/captured_write.h" +#include "snapshot/native_frame_read.h" + +namespace crowdb::tree +{ +namespace captured_write_detail +{ +[[nodiscard]] bool same_durable_frame(const DurableCapturedFrame &left, const DurableCapturedFrame &right) noexcept; +} + +struct CapturedWriteOperation::State : std::enable_shared_from_this<State> +{ + enum class Stage : uint8_t { Next, Frame, Dependencies, Write, ResidentWrite, RelocateRead }; + + struct Dependencies + { + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !owners.empty(); ++i) { + owners.pop_back(); + } + return owners.empty(); + } + + explicit Dependencies(const std::shared_ptr<detail::AllocationReservation> &metadata) + : owners(detail::ReservationAllocator<std::shared_ptr<const void>>(metadata)) + { + } + + std::shared_ptr<const void> ids; + detail::ReservedVector<std::shared_ptr<const void>> owners; + }; + + struct ResidentSource + { + std::shared_ptr<const BackingRef> backing; + std::shared_ptr<Dependencies> dependencies; + }; + + using ResidentEntry = std::pair<const uint64_t, ResidentSource>; + using ResidentSources = + std::map<uint64_t, ResidentSource, std::less<>, detail::ReservationAllocator<ResidentEntry>>; + + explicit State(const std::shared_ptr<detail::AllocationReservation> &metadata) + : resident_sources(std::less<>{}, detail::ReservationAllocator<ResidentEntry>(metadata)) + { + } + + struct Reservation + { + PageWriteOperation::ExtentComplete complete; + std::shared_ptr<Dependencies> dependencies; + std::atomic<bool> claimed{false}; + }; + + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + std::shared_ptr<State> owner; + void set_value() noexcept; + void set_error(Status status) noexcept; + void set_stopped() noexcept; + }; + + using Connected = stdexec::connect_result_t<detail::CallbackSender, Receiver>; + + struct Attempt + { + Attempt(detail::CallbackSender sender, std::shared_ptr<State> owner) + : connected(stdexec::connect(sender, Receiver{.owner = std::move(owner)})) + { + } + + Connected connected; + }; + + void start(detail::CallbackComplete callback, void *argument) noexcept; + void wake() noexcept; + static void run(void *context) noexcept; + void pump(); + void begin_child(detail::CallbackSender sender, Stage next); + void child_done(Status status) noexcept; + void cancel_child() noexcept; + void received_frame(); + void reuse_step(); + void begin_write(); + PageWriteOperation::ReserveExtent reserve_page(bool protect_dependency); + void use_resident_dependency(DurableCapturedFrame value); + void begin_resident_write(DurableCapturedFrame value); + void received_resident_write(); + void begin_relocated_read(DurableCapturedFrame value); + void received_relocated_read(); + void use_relocated_dependency(DurableCapturedFrame value); + void reuse_durable_dependency(DurableCapturedFrame value); + Status rebind_primary(); + void attach_dependency(const DurableCapturedFrame &value, bool retain_owner = true); + void received_write(); + void finish_frame(); + Status insert(DurableCapturedFrame value, bool reused); + void finish(Status status) noexcept; + + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + CapturedWriteLimits limits; + PageLoadBudget budget; + PageWriteOperation::ReserveExtent reserve; + std::shared_ptr<const CapturedDurablePages> inherited; + std::shared_ptr<CapturedDurablePages> pages; + std::shared_ptr<CapturedFrameStream> stream; + std::shared_ptr<PageWriteOperation> write; + std::shared_ptr<NativeFrameRead> relocation_read; + std::shared_ptr<Dependencies> dependencies; + std::shared_ptr<OverflowBackingSet> overflow_backing; + std::optional<CapturedFrame> frame; + DurableCapturedFrame descriptor; + DurableCapturedFrame resident_dependency; + ResidentSources resident_sources; + size_t dependency_index = 0; + uint64_t dependency_pid = kInvalidPageId; + uint64_t dependency_bytes = 0; + size_t dependency_steps = 0; + bool last_frame = false; + Stage stage = Stage::Next; + bool waiting = false; + Status dispatch_failure; + Status child_status; + std::atomic<bool> child_ready{false}; + std::optional<Attempt> attempt; + std::shared_ptr<State> queued_owner; + std::atomic<uint8_t> lifecycle{0}; + std::atomic<bool> stopped{false}; + std::atomic<bool> scheduled{false}; + std::atomic<bool> wake_requested{false}; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + ApplyDispatcher::Node cpu_node{&State::run, this}; + CapturedWriteResult report; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_write_page.cpp b/lib/crowdb-tree/src/snapshot/captured_write_page.cpp new file mode 100644 index 000000000..bf5053bcb --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_write_page.cpp @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/captured_write_internal.h" +#include "snapshot/primary_backing.h" + +namespace crowdb::tree +{ +void CapturedWriteOperation::State::begin_write() +{ + if (!frame.has_value()) { + finish(Status::internal_error()); + return; + } + if (frame->reuse) { + if (limits.frames.exclusion.length != 0) { + auto rebound = rebind_primary(); + if (!rebound.ok()) { + finish(std::move(rebound)); + return; + } + } + auto status = insert(descriptor, true); + if (!status.ok()) { + finish(std::move(status)); + return; + } + finish_frame(); + return; + } + PageWriteInput input{.info = frame->info, + .frame = frame->buffer->bytes, + .owner = frame->buffer, + .reload_dependencies = dependencies}; + auto status = PageWriteOperation::create(store, runtime, lane, std::move(input), limits.write, budget, + reserve_page(descriptor.type == page_type::kOverflowFrame), &write); + if (!status.ok()) { + finish(std::move(status)); + return; + } + begin_child(write->sender(), Stage::Write); +} + +PageWriteOperation::ReserveExtent CapturedWriteOperation::State::reserve_page(bool protect_dependency) +{ + const auto weak = weak_from_this(); + return [weak, protect_dependency](uint64_t length, PageWriteOperation::ExtentComplete done) { + const auto self = weak.lock(); + if (!self) { + done(Status::unavailable(), {}); + return; + } + auto event = detail::reserved_shared<Reservation>(self->budget.bytes); + event->complete = std::move(done); + if (protect_dependency) { + event->dependencies = self->dependencies; + } + self->reserve(length, [event](Status status, ReservedWriteExtent extent) { + if (event->claimed.exchange(true, std::memory_order_acq_rel)) { + return; + } + try { + if (status.ok() && event->dependencies) { + event->dependencies->owners.push_back(extent.owner); + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted(); + } + event->complete(std::move(status), std::move(extent)); + }); + }; +} + +void CapturedWriteOperation::State::received_write() +{ + const auto &written = write->result(); + descriptor.backing = written.backing; + descriptor.logical_bytes = written.logical_bytes; + if (descriptor.type == page_type::kOverflowFrame) { + overflow_backing->pages.emplace(descriptor.page_id, + OverflowBackingPage{.backing = descriptor.backing, + .next = descriptor.next_overflow, + .bytes = descriptor.overflow_bytes, + .frame_bytes = descriptor.frame_bytes, + .logical_bytes = descriptor.logical_bytes}); + } + else if (descriptor.type == page_type::kLeafBase) { + auto primary = detail::reserved_shared<BackingRef>(budget.bytes, *descriptor.backing); + primary->overflow = overflow_backing; + descriptor.backing = std::move(primary); + } + auto status = insert(descriptor, false); + if (!status.ok()) { + finish(std::move(status)); + return; + } + ++report.written_frames; + report.written_bytes += written.physical_bytes; + finish_frame(); +} + +void CapturedWriteOperation::State::finish_frame() +{ + if (descriptor.type != page_type::kOverflowFrame) { + dependencies.reset(); + overflow_backing.reset(); + } + write.reset(); + frame.reset(); + if (last_frame) { + finish(Status::Ok()); + return; + } + stage = Stage::Next; + wake(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_write_pump.cpp b/lib/crowdb-tree/src/snapshot/captured_write_pump.cpp new file mode 100644 index 000000000..c578bbecc --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_write_pump.cpp @@ -0,0 +1,259 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/captured_write_internal.h" +#include "snapshot/primary_backing.h" +#include "snapshot/stored_overflow.h" + +#include <cassert> + +namespace crowdb::tree +{ +void CapturedWriteOperation::State::wake() noexcept +{ + wake_requested.store(true, std::memory_order_release); + if (lifecycle.load(std::memory_order_acquire) != 2 || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + auto self = weak_from_this().lock(); + assert(self); + for (;;) { + queued_owner = self; + if (budget.cpu_progress) { + budget.cpu_progress->request(cpu_node, self); + return; + } + Status status; + try { + status = runtime->try_submit(lane, &State::run, this); + } + catch (...) { + status = Status::resource_exhausted(); + } + if (status.ok()) { + return; + } + queued_owner.reset(); + dispatch_failure = std::move(status); + if (!waiting || child_ready.load(std::memory_order_acquire)) { + finish(dispatch_failure.without_message()); + scheduled.exchange(false, std::memory_order_acq_rel); + return; + } + // Keep the receiver/operation until the accepted child drains. A + // closed runtime cannot justify freeing a still-active write window. + cancel_child(); + scheduled.exchange(false, std::memory_order_acq_rel); + if (!child_ready.load(std::memory_order_acquire) || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + } +} + +void CapturedWriteOperation::State::run(void *context) noexcept +{ + auto self = std::move(static_cast<State *>(context)->queued_owner); + self->wake_requested.exchange(false, std::memory_order_acq_rel); + try { + self->pump(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted()); + } + catch (...) { + self->finish(Status::internal_error()); + } + self->scheduled.exchange(false, std::memory_order_acq_rel); + if (self->wake_requested.load(std::memory_order_acquire) && self->lifecycle.load(std::memory_order_acquire) == 2) { + self->wake(); + } +} + +void CapturedWriteOperation::State::pump() +{ + if (waiting) { + if (stopped.load(std::memory_order_acquire)) { + cancel_child(); + } + if (!child_ready.load(std::memory_order_acquire)) { + return; + } + waiting = false; + attempt.reset(); + if (!dispatch_failure.ok()) { + finish(std::move(dispatch_failure)); + return; + } + if (!child_status.ok()) { + finish(std::move(child_status)); + return; + } + } + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + switch (stage) { + case Stage::Next: + begin_child(stream->sender(), Stage::Frame); + break; + case Stage::Frame: + received_frame(); + break; + case Stage::Dependencies: + reuse_step(); + break; + case Stage::Write: + received_write(); + break; + case Stage::ResidentWrite: + received_resident_write(); + break; + case Stage::RelocateRead: + received_relocated_read(); + break; + } +} + +void CapturedWriteOperation::State::received_frame() +{ + const auto &result = stream->result(); + if (!result.frame) { + finish(result.complete ? Status::Ok() : Status::corruption()); + return; + } + frame = result.frame; + last_frame = result.complete; + descriptor = { + .page_id = frame->info.page_id, .type = frame->info.type, .frame_bytes = frame->info.bytes, .backing = {}}; + descriptor.summary = frame->summary; + if (!dependencies) { + dependencies = detail::reserved_shared<Dependencies>(budget.bytes, budget.bytes); + overflow_backing = detail::reserved_shared<OverflowBackingSet>(budget.bytes, budget.bytes); + } + if (frame->reuse) { + descriptor = frame->reuse->primary->frame; + } + else if (descriptor.type == page_type::kOverflowFrame) { + OverflowFrameView overflow(frame->buffer->bytes.data(), descriptor.frame_bytes); + descriptor.next_overflow = overflow.next_page_id(); + descriptor.overflow_bytes = overflow.chunk_len(); + dependencies->ids = frame->info.dependencies; + } + else if (descriptor.type == page_type::kLeafBase) { + const auto status = + capture_stored_overflow(LeafFrameView(frame->buffer->bytes.data(), descriptor.frame_bytes), + limits.frames.encode.max_entries, budget.bytes, &descriptor.stored_overflow); + if (!status.ok()) { + finish(status); + return; + } + } + dependency_index = 0; + dependency_pid = kInvalidPageId; + stage = Stage::Dependencies; + wake(); +} + +Status CapturedWriteOperation::State::insert(DurableCapturedFrame value, bool reused) +{ + if (!value.backing || !value.backing->reloadable() || value.page_id >= MappingView::kPageLimit || + value.backing->backend_id == 0 || (pages->backend_id != 0 && pages->backend_id != value.backing->backend_id)) { + return Status::corruption("captured write references a different or unprotected backend"); + } + auto old = pages->frames.find(value.page_id); + if (old != pages->frames.end()) { + return reused && captured_write_detail::same_durable_frame(old->second, value) + ? Status::Ok() + : Status::corruption("duplicate captured PID"); + } + if (pages->frames.size() == limits.max_frames) { + return Status::resource_exhausted(); + } + pages->backend_id = value.backing->backend_id; + pages->next_page_id = std::max(pages->next_page_id, value.page_id + 1); + const auto reused_bytes = reused ? value.backing->length : 0; + pages->frames.emplace(value.page_id, std::move(value)); + record_work(budget.work_metrics, TreeWorkCounter::ReusedFrames, static_cast<uint64_t>(reused), budget.cause); + record_work(budget.work_metrics, TreeWorkCounter::ReusedBackingBytes, reused_bytes, budget.cause); + report.reused_frames += static_cast<uint64_t>(reused); + return Status::Ok(); +} + +void CapturedWriteOperation::State::reuse_step() +{ + if (!frame.has_value()) { + finish(Status::internal_error()); + return; + } + if (dependency_index == frame->reused_overflow.size()) { + begin_write(); + return; + } + if (dependency_steps++ == limits.max_dependency_steps) { + finish(Status::resource_exhausted()); + return; + } + if (dependency_pid == kInvalidPageId) { + const auto &ref = frame->reused_overflow[dependency_index]; + dependency_pid = ref.head; + dependency_bytes = ref.bytes; + } + DurableCapturedFrame old; + const auto &captured = frame->reused_overflow[dependency_index].backing; + if (captured) { + const auto found = captured->pages.find(dependency_pid); + if (found == captured->pages.end()) { + finish(Status::corruption("captured primary has no referenced overflow PID")); + return; + } + const auto &page = found->second; + old = {.page_id = dependency_pid, + .type = page_type::kOverflowFrame, + .frame_bytes = page.frame_bytes, + .logical_bytes = page.logical_bytes, + .next_overflow = page.next, + .overflow_bytes = page.bytes, + .backing = page.backing}; + } + else { + if (!inherited) { + finish(Status::unavailable("captured overflow requires its protected durable mapping")); + return; + } + const auto found = inherited->frames.find(dependency_pid); + if (found == inherited->frames.end()) { + finish(Status::corruption("captured overflow PID is absent")); + return; + } + old = found->second; + } + if (old.page_id != dependency_pid || old.type != page_type::kOverflowFrame || old.overflow_bytes == 0 || + old.overflow_bytes > dependency_bytes || + (old.next_overflow == kInvalidPageId) != (old.overflow_bytes == dependency_bytes)) { + finish(Status::corruption("captured overflow topology does not match the cell length")); + return; + } + if (old.backing && old.backing->storage == BackingStorage::kResidentFrame) { + use_resident_dependency(std::move(old)); + return; + } + if (limits.frames.exclusion.overlaps(*old.backing)) { + use_relocated_dependency(std::move(old)); + return; + } + // A dirty primary adopts exact physical extents into its own dependency + // group. Retaining an old overflow owner would also keep deleted values + // from the old leaf alive. Clean primaries keep their existing exact graph. + if (limits.frames.exclusion.length != 0 || (!frame->reuse && old.backing->physical_owner)) { + reuse_durable_dependency(std::move(old)); + return; + } + auto status = insert(old, true); + if (!status.ok()) { + finish(std::move(status)); + return; + } + attach_dependency(old); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_write_relocate.cpp b/lib/crowdb-tree/src/snapshot/captured_write_relocate.cpp new file mode 100644 index 000000000..c7ee9785d --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_write_relocate.cpp @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/captured_write_internal.h" + +namespace crowdb::tree +{ +bool captured_write_detail::same_durable_frame(const DurableCapturedFrame &left, + const DurableCapturedFrame &right) noexcept +{ + if (!left.backing || !right.backing || left.page_id != right.page_id || left.type != right.type || + left.frame_bytes != right.frame_bytes || left.logical_bytes != right.logical_bytes || + left.next_overflow != right.next_overflow || left.overflow_bytes != right.overflow_bytes) { + return false; + } + if (left.backing == right.backing) { + return true; + } + const auto &a = *left.backing; + const auto &b = *right.backing; + return a.physical_owner && a.physical_owner == b.physical_owner && a.backend_id == b.backend_id && + a.storage == b.storage && a.address == b.address && a.length == b.length && a.content_id == b.content_id && + a.generation == b.generation && a.checksum == b.checksum && a.encoding == b.encoding && + same_chunk_backing(a.chunk, b.chunk); +} + +void CapturedWriteOperation::State::use_relocated_dependency(DurableCapturedFrame value) +{ + const auto found = pages->frames.find(value.page_id); + if (found == pages->frames.end()) { + begin_relocated_read(std::move(value)); + return; + } + const auto original = resident_sources.find(value.page_id); + if (original == resident_sources.end() || original->second.backing != value.backing || + found->second.next_overflow != value.next_overflow || found->second.overflow_bytes != value.overflow_bytes || + found->second.frame_bytes != value.frame_bytes) { + finish(Status::corruption("relocation encountered conflicting overflow identities")); + return; + } + attach_dependency(found->second); +} + +void CapturedWriteOperation::State::begin_relocated_read(DurableCapturedFrame value) +{ + PageLoadLimits loads; + loads.max_frame_bytes = limits.write.max_frame_bytes; + loads.max_blob_bytes = + ((limits.write.max_frame_bytes + kDurableBlobHeader + limits.write.iu_size - 1) / limits.write.iu_size) * + limits.write.iu_size; + loads.max_inflight = 1; + OverflowBackingPage source{.backing = value.backing, + .next = value.next_overflow, + .bytes = value.overflow_bytes, + .frame_bytes = value.frame_bytes, + .logical_bytes = value.logical_bytes}; + resident_sources.emplace(value.page_id, ResidentSource{.backing = value.backing, .dependencies = dependencies}); + resident_dependency = std::move(value); + auto status = NativeFrameRead::create(resident_dependency.page_id, std::move(source), store, runtime, lane, loads, + budget, &relocation_read); + if (!status.ok()) { + finish(std::move(status)); + return; + } + begin_child(relocation_read->sender(), Stage::RelocateRead); +} + +void CapturedWriteOperation::State::received_relocated_read() +{ + const auto &raw = relocation_read->result().frame; + if (!raw) { + finish(Status::corruption("relocation read returned no frame")); + return; + } + PageWriteInput input{ + .info = {.page_id = resident_dependency.page_id, + .type = resident_dependency.type, + .bytes = resident_dependency.frame_bytes}, + .frame = raw->bytes, + .owner = raw->owner, + .reload_dependencies = dependencies + }; + auto status = PageWriteOperation::create(store, runtime, lane, std::move(input), limits.write, budget, + reserve_page(true), &write); + if (!status.ok()) { + finish(std::move(status)); + return; + } + begin_child(write->sender(), Stage::ResidentWrite); +} + +void CapturedWriteOperation::State::reuse_durable_dependency(DurableCapturedFrame value) +{ + const auto found = pages->frames.find(value.page_id); + if (found != pages->frames.end()) { + if (!captured_write_detail::same_durable_frame(found->second, value)) { + finish(Status::corruption("captured write encountered conflicting reused overflow identities")); + return; + } + if (found->second.backing->owner == dependencies) { + attach_dependency(found->second, false); + return; + } + } + if (!value.backing->physical_owner) { + finish(Status::not_supported("dependency rebind requires independent physical extent ownership")); + return; + } + auto backing = detail::reserved_shared<BackingRef>(budget.bytes, *value.backing); + backing->owner = dependencies; + value.backing = std::move(backing); + if (found == pages->frames.end()) { + auto status = insert(value, true); + if (!status.ok()) { + finish(std::move(status)); + return; + } + } + attach_dependency(value); +} + +Status CapturedWriteOperation::State::rebind_primary() +{ + if (!descriptor.backing->physical_owner) { + return Status::not_supported("relocation requires independent primary extent ownership"); + } + auto backing = detail::reserved_shared<BackingRef>(budget.bytes, *descriptor.backing); + dependencies->owners.push_back(backing->physical_owner); + backing->owner = dependencies; + backing->overflow = overflow_backing; + descriptor.backing = std::move(backing); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_write_resident.cpp b/lib/crowdb-tree/src/snapshot/captured_write_resident.cpp new file mode 100644 index 000000000..e092dc8cb --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/captured_write_resident.cpp @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/captured_write_internal.h" + +namespace crowdb::tree +{ +void CapturedWriteOperation::State::attach_dependency(const DurableCapturedFrame &value, bool retain_owner) +{ + if (retain_owner) { + dependencies->owners.push_back(value.backing->physical_owner ? value.backing->physical_owner + : value.backing->owner); + } + overflow_backing->pages.emplace(dependency_pid, OverflowBackingPage{.backing = value.backing, + .next = value.next_overflow, + .bytes = value.overflow_bytes, + .frame_bytes = value.frame_bytes, + .logical_bytes = value.logical_bytes}); + dependency_bytes -= value.overflow_bytes; + dependency_pid = value.next_overflow; + if (dependency_bytes == 0) { + ++dependency_index; + } + stage = Stage::Dependencies; + wake(); +} + +void CapturedWriteOperation::State::use_resident_dependency(DurableCapturedFrame value) +{ + const auto written = pages->frames.find(value.page_id); + if (written == pages->frames.end()) { + begin_resident_write(std::move(value)); + return; + } + const auto original = resident_sources.find(value.page_id); + const auto &backing = value.backing; + if (!backing || original == resident_sources.end() || !backing->readable() || + original->second.backing->owner.owner_before(backing->owner) || + backing->owner.owner_before(original->second.backing->owner) || + original->second.backing->resident_frame.data() != backing->resident_frame.data() || + original->second.backing->resident_frame.size() != backing->resident_frame.size() || + written->second.type != value.type || written->second.next_overflow != value.next_overflow || + written->second.overflow_bytes != value.overflow_bytes || written->second.frame_bytes != value.frame_bytes) { + finish(Status::corruption("resident overflow PID conflicts with an already written frame")); + return; + } + // A frame written in this group already contributed its extent lease. + // Retaining its BackingRef owner would create a dependency cycle. + attach_dependency(written->second, original->second.dependencies != dependencies); +} + +void CapturedWriteOperation::State::begin_resident_write(DurableCapturedFrame value) +{ + const auto &backing = value.backing; + if (!backing || !backing->readable() || backing->storage != BackingStorage::kResidentFrame || + backing->resident_frame.size() != value.frame_bytes || value.frame_bytes > limits.write.max_frame_bytes || + !frame_validate(backing->resident_frame.data(), value.frame_bytes)) { + finish(Status::corruption("resident overflow frame has no valid independent raw owner")); + return; + } + const OverflowFrameView raw(backing->resident_frame.data(), value.frame_bytes); + if (frame_page_type(backing->resident_frame.data()) != page_type::kOverflowFrame || + raw.self_page_id() != value.page_id || raw.next_page_id() != value.next_overflow || + raw.chunk_len() != value.overflow_bytes) { + finish(Status::corruption("resident overflow descriptor differs from its frame")); + return; + } + PageWriteInput input{ + .info = {.page_id = value.page_id, .type = value.type, .bytes = value.frame_bytes}, + .frame = backing->resident_frame, + .owner = backing->owner, + .reload_dependencies = dependencies + }; + resident_sources.emplace(value.page_id, ResidentSource{.backing = backing, .dependencies = dependencies}); + resident_dependency = std::move(value); + auto status = PageWriteOperation::create(store, runtime, lane, std::move(input), limits.write, budget, + reserve_page(true), &write); + if (!status.ok()) { + finish(std::move(status)); + return; + } + begin_child(write->sender(), Stage::ResidentWrite); +} + +void CapturedWriteOperation::State::received_resident_write() +{ + const auto &result = write->result(); + resident_dependency.backing = result.backing; + resident_dependency.logical_bytes = result.logical_bytes; + auto status = insert(resident_dependency, false); + if (!status.ok()) { + finish(std::move(status)); + return; + } + ++report.written_frames; + report.written_bytes += result.physical_bytes; + attach_dependency(resident_dependency, false); + write.reset(); + relocation_read.reset(); + resident_dependency.backing.reset(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_allocation_bound.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_allocation_bound.cpp new file mode 100644 index 000000000..d58236f3d --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/checkpoint_allocation_bound.cpp @@ -0,0 +1,12 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/local_checkpoint.h" + +namespace crowdb::tree +{ +size_t ProtectedCheckpoint::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<ProtectedCheckpoint>() + + detail::shared_allocation_bound<CheckpointMetadata>(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation.cpp new file mode 100644 index 000000000..b637fd79c --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation.cpp @@ -0,0 +1,262 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/writeback_metrics.h" +#include "crowdb-common/log.h" +#include "snapshot/checkpoint_operation_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +Status CheckpointOperation::create( + std::shared_ptr<const void> owner, PublicationCoordinator &publisher, std::shared_ptr<LocalBackendState> backend, + std::shared_ptr<AsyncPageStore> operation_store, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CheckpointOptions options, PageLoadBudget budget, + std::shared_ptr<detail::AllocationReservation> finish_metadata, std::shared_ptr<PageLoader> loader, + PrepareHandoff prepare_handoff, PrepareWork prepare_work, std::shared_ptr<CheckpointOperation> *out, + PrepareCleanup prepare_cleanup, PrepareWorkAsync prepare_work_async, PrepareCleanup on_terminal, + PrepareCleanup cancel_work, std::shared_ptr<CheckpointBackend> chunk, PrepareStructure prepare_structure) +{ + const bool local_valid = backend && backend->ledger && backend->checkpoints; + const bool chunk_valid = chunk && chunk->supported(); + if ((options.mode != CheckpointMode::Durable && options.mode != CheckpointMode::Writeback) || + local_valid == chunk_valid || !operation_store || !runtime || !lane || + (!loader && !prepare_work && !prepare_work_async) || out == nullptr || + (!chunk_valid && !operation_store->write_completion_is_reloadable()) || + static_cast<bool>(budget.bytes) != static_cast<bool>(finish_metadata) || + (finish_metadata && (finish_metadata == budget.bytes || finish_metadata->account() != budget.bytes->account() || + finish_metadata->category() == detail::BudgetClass::kIo))) { + return Status::invalid_argument(); + } + if ((options.mode == CheckpointMode::Durable && + (options.write.frames.selected_page || options.select_oldest_page)) || + (options.select_oldest_page && options.write.frames.selected_page) || + options.write.write.iu_size != (chunk ? chunk->iu_size : backend->ledger->limits().iu_size) || + options.write.write.chunk_backed != chunk_valid || !PageWriteOperation::allocation_bound(options.write.write) || + options.write.frames.plan_only || + options.write.frames.encode.frame_bytes > options.write.write.max_frame_bytes || + options.write.frames.encode.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || + options.write.max_frames == 0 || options.write.max_dependency_steps == 0 || + options.metadata.max_segments == 0) { + return Status::invalid_argument("checkpoint requires matching bounded page geometry"); + } + if (chunk ? chunk->needs_recovery.load(std::memory_order_acquire) : backend->checkpoints->needs_recovery()) { + return Status::unavailable(); + } + try { + if (finish_metadata && finish_metadata->available() < finish_allocation_bound()) { + return Status::resource_exhausted(); + } + auto state = detail::reserved_shared<State>(budget.bytes, publisher); + state->owner = std::move(owner); + state->backend = std::move(backend); + state->chunk = std::move(chunk); + state->chunk_wal_offset = state->chunk ? state->chunk->requested_wal_offset() : 0; + state->store = std::move(operation_store); + state->runtime = std::move(runtime); + state->lane = std::move(lane); + state->options = std::move(options); + state->max_layout_depth = state->options.write.frames.max_depth; + state->budget = std::move(budget); + state->finish_metadata = std::move(finish_metadata); + state->loader = std::move(loader); + state->prepare_handoff = std::move(prepare_handoff); + state->prepare_structure = std::move(prepare_structure); + state->prepare_work = std::move(prepare_work); + state->prepare_cleanup = std::move(prepare_cleanup); + state->prepare_work_async = std::move(prepare_work_async); + state->on_terminal = std::move(on_terminal); + state->cancel_work = std::move(cancel_work); + state->cpu_progress = state->budget.cpu_progress; + state->control_metadata = state->budget.bytes; + state->capture_request = detail::reserved_shared<PublicationRequest>(state->budget.bytes, state->budget.bytes); + state->finish_request = + detail::reserved_shared<PublicationRequest>(state->finish_metadata, state->finish_metadata); + if (state->options.mode == CheckpointMode::Durable) { + state->baseline = detail::reserved_shared<SnapshotBaseline>(state->budget.bytes); + } + if (!state->cpu_progress) { + auto status = state->runtime->reserve_progress(state->lane, &State::run, state.get(), + state->finish_metadata, &state->progress); + if (!status.ok()) { + return status; + } + } + *out = detail::reserved_construct<CheckpointOperation>(state->budget.bytes, [&](auto *storage) { + ::new (static_cast<void *>(storage)) CheckpointOperation(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +size_t CheckpointOperation::control_allocation_bound() noexcept +{ + using detail::shared_allocation_bound; + return shared_allocation_bound<State>() + shared_allocation_bound<CheckpointOperation>() + + shared_allocation_bound<PublicationRequest>() + shared_allocation_bound<InstallRecord>() + + (4 * shared_allocation_bound<PublishedTreeState>()) + (3 * PublishedSources::allocation_bound()) + + PublicationCoordinator::empty_commit_reserve_bytes() + shared_allocation_bound<SnapshotBaseline>(); +} + +size_t CheckpointOperation::finish_allocation_bound() noexcept +{ + return detail::shared_allocation_bound<PublicationRequest>() + detail::shared_allocation_bound<InstallRecord>() + + detail::FlushRuntime::progress_allocation_bound() + PublicationCoordinator::empty_commit_reserve_bytes(); +} + +detail::CallbackSender CheckpointOperation::sender() noexcept +{ + return {this, &CheckpointOperation::submit}; +} + +const CheckpointResult &CheckpointOperation::result() const noexcept +{ + return state_->report; +} + +void CheckpointOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void CheckpointOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<CheckpointOperation *>(source)->state_->start(complete, context); +} + +void CheckpointOperation::State::start(detail::CallbackComplete callback, void *argument) noexcept +{ + uint8_t idle = 0; + if (!lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + callback(argument, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + complete = callback; + context = argument; + lifecycle.store(2, std::memory_order_release); + wake(); +} + +void CheckpointOperation::State::wake() noexcept +{ + if (lifecycle.load(std::memory_order_acquire) == 2) { + auto self = weak_from_this().lock(); + assert(self); + if (cpu_progress) { + cpu_progress->request(cpu_node, std::move(self)); + } + else { + progress->request(std::move(self)); + } + } +} + +void CheckpointOperation::State::run(void *context) noexcept +{ + auto *self = static_cast<State *>(context); + if (self->lifecycle.load(std::memory_order_acquire) != 2) { + return; + } + try { + self->pump(); + } + catch (const std::bad_alloc &) { + self->fail(Status::resource_exhausted()); + } + catch (...) { + self->fail(Status::internal_error()); + } +} + +void CheckpointOperation::State::finish(Status status) noexcept +{ + if (stage != Stage::FinishMetrics) { + final_status = std::move(status); + final_outcome = final_status.ok() ? TreeOutcome::Stale : TreeOutcome::Error; + if (!report.durable_committed && stopped.load(std::memory_order_acquire)) { + final_outcome = TreeOutcome::Cancelled; + } + stage = Stage::FinishMetrics; + const auto terminal = final_status.ok() ? TreeOutcome::Success : final_outcome; + const auto now = tree_metrics_now_ns(); + if (capture_metrics) { + (void)capture_metrics->finish(terminal, now); + } + if (commit_metrics) { + (void)commit_metrics->finish(terminal, now); + } + } + if (writeback_metrics && !writeback_metrics->finish_some(final_outcome)) { + wake(); + return; + } + if (lifecycle.exchange(3, std::memory_order_acq_rel) == 3) { + return; + } + status = std::move(final_status); + options.write.frames.writeback_metrics.reset(); + writeback_metrics.reset(); + report.status = std::move(status); + if (!report.status.ok()) { + record_work(publisher.work_metrics(), TreeWorkCounter::CheckpointFailures, 1, options.cause); + } + if (!report.durable_committed) { + report.durable_coverage = + chunk ? chunk->durable_coverage.load(std::memory_order_acquire) : backend->checkpoints->durable_coverage(); + report.snapshot_sequence = chunk ? chunk->snapshot_sequence.load(std::memory_order_acquire) + : backend->checkpoints->snapshot_sequence(); + } + view.reset(); + capture_request.reset(); + finish_request.reset(); + handoff.reset(); + structure.reset(); + write.reset(); + checkpoint.reset(); + chunk_checkpoint.reset(); + rebase.reset(); + written_pages.reset(); + baseline.reset(); + planning_stream.reset(); + planner.reset(); + loader.reset(); + budget = {}; + work_owner.reset(); + store.reset(); + prepare_handoff = {}; + prepare_structure = {}; + prepare_work = {}; + prepare_work_async = {}; + prepared_work = {}; + if (report.durable_committed && prepare_cleanup) { + try { + prepare_cleanup(); + } + catch (...) { + CRB_LOG_WARN("post-checkpoint cleanup submission failed; retaining extents for a later pass"); + } + } + prepare_cleanup = {}; + if (on_terminal) { + try { + on_terminal(); + } + catch (...) { + CRB_LOG_ERROR("checkpoint terminal observer threw"); + } + } + on_terminal = {}; + cancel_work = {}; + owner.reset(); + auto signal = report.status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + if (!report.durable_committed && stopped.load(std::memory_order_acquire)) { + signal = detail::CallbackSignal::kStopped; + } + complete(context, signal, report.status.without_message()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation.h b/lib/crowdb-tree/src/snapshot/checkpoint_operation.h new file mode 100644 index 000000000..c66d1950f --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation.h @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/local/local_backend_state.h" +#include "btree/flush_operation.h" +#include "btree/page_split_operation.h" +#include "snapshot/captured_plan.h" + +namespace crowdb::tree +{ +class CheckpointBackend; + +enum class CheckpointMode : uint8_t { Durable, Writeback }; + +struct CheckpointDiagnostics +{ + std::atomic<uint64_t> written_frames{0}; + std::atomic<uint64_t> total_written_frames{0}; +}; + +struct CheckpointOptions +{ + CheckpointMode mode = CheckpointMode::Durable; + // Core's accepted ticket retains this state through terminal. Only durable + // commits update it, while their publication capture token is still held. + CheckpointDiagnostics *diagnostics = nullptr; + // Select a bounded host window from the immutable view after handoff. + bool select_oldest_page = false; + CapturedWriteLimits write; + LocalCheckpointLimits metadata; + std::shared_ptr<PublicationLane> publication = nullptr; + TreeWorkCause cause = TreeWorkCause::Checkpoint; +}; + +struct CheckpointWork +{ + PageLoadBudget budget; + std::shared_ptr<PageLoader> loader; + std::shared_ptr<const void> owner; +}; + +struct CheckpointResult +{ + Status status; + uint64_t durable_coverage = 0; + uint64_t snapshot_sequence = 0; + uint64_t checkpoint_token = 0; + uint64_t captured_coverage = 0; + uint64_t primary_frames = 0; + uint64_t structural_windows = 0; + uint64_t written_frames = 0; + uint64_t written_bytes = 0; + uint64_t reused_frames = 0; + bool durable_committed = false; + bool writeback_completed = false; + uint64_t rebased_pages = 0; + uint64_t rebase_deferred = 0; +}; + +// Serializes a finite handoff, immutable publication capture, protected page +// materialization, local anchor commit and publication reconciliation. The +// finish partition and CPU continuation are reserved before closing sources. +// A successful durable commit must be reconciled even after cancellation. +// Writeback mode stops before anchor commit, installs reloadable +// representations and aborts the capture token without advancing the durable +// frontier. +class CheckpointOperation +{ + struct State; + + public: + using PrepareHandoff = detail::InlineFunction<Status(std::shared_ptr<FlushOperation> *)>; + // Called after handoff for a read/layout window, then after that window + // drains for the observed materialization bound. A null plan means preflight. + using PrepareWork = + detail::InlineFunction<Status(const PublishedTreeState &, const CapturedWritePlan *, CheckpointWork *)>; + + // The sender owns one complete phase grant and writes the output before terminal. + // The captured view, plan and output stay stable through that terminal. + using PrepareWorkAsync = detail::InlineFunction<detail::CallbackSender( + const PublishedTreeState &, const CapturedWritePlan *, CheckpointWork *)>; + using PrepareCleanup = detail::InlineFunction<void()>; + using PrepareStructure = detail::InlineFunction<Status(std::shared_ptr<const PublishedTreeState>, uint64_t, + std::shared_ptr<PageSplitOperation> *)>; + + static Status create(std::shared_ptr<const void> owner, PublicationCoordinator &publisher, + std::shared_ptr<LocalBackendState> backend, std::shared_ptr<AsyncPageStore> operation_store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CheckpointOptions options, + PageLoadBudget budget, std::shared_ptr<detail::AllocationReservation> finish_metadata, + std::shared_ptr<PageLoader> loader, PrepareHandoff prepare_handoff, PrepareWork prepare_work, + std::shared_ptr<CheckpointOperation> *out, PrepareCleanup prepare_cleanup = {}, + PrepareWorkAsync prepare_work_async = {}, PrepareCleanup on_terminal = {}, + PrepareCleanup cancel_work = {}, std::shared_ptr<CheckpointBackend> chunk = {}, + PrepareStructure prepare_structure = {}); + [[nodiscard]] static size_t control_allocation_bound() noexcept; + [[nodiscard]] static size_t finish_allocation_bound() noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const CheckpointResult &result() const noexcept; + void request_stop() noexcept; + + private: + explicit CheckpointOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation_backend.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation_backend.cpp new file mode 100644 index 000000000..763c43e12 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation_backend.cpp @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/checkpoint_operation_internal.h" + +namespace crowdb::tree +{ +bool CheckpointOperation::State::backend_needs_recovery() const noexcept +{ + return chunk ? chunk->needs_recovery.load(std::memory_order_acquire) : backend->checkpoints->needs_recovery(); +} + +uint64_t CheckpointOperation::State::backend_identity() const noexcept +{ + return chunk ? chunk->backend_identity() : backend->ledger->limits().backend_id; +} + +std::shared_ptr<const CapturedDurablePages> CheckpointOperation::State::inherited_pages() const +{ + return backend ? backend->inherited : nullptr; +} + +void CheckpointOperation::State::begin_chunk_checkpoint() +{ + auto status = chunk->prepare_checkpoint({.pages = written_pages, + .view = *view, + .wal_offset = chunk_wal_offset, + .max_dependency_steps = options.write.max_dependency_steps, + .store = store, + .runtime = runtime, + .lane = lane, + .budget = budget, + .owner = owner}, + &chunk_checkpoint); + if (!status.ok()) { + reconcile(std::move(status)); + return; + } + write.reset(); + begin_child(chunk_checkpoint->sender(), Stage::Checkpoint); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation_internal.h b/lib/crowdb-tree/src/snapshot/checkpoint_operation_internal.h new file mode 100644 index 000000000..eeee343c2 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation_internal.h @@ -0,0 +1,165 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/checkpoint_backend.h" +#include "btree/apply_dispatcher.h" +#include "snapshot/captured_rebase.h" +#include "snapshot/checkpoint_operation.h" +#include "snapshot/snapshot_baseline.h" + +namespace crowdb::tree +{ +struct CheckpointOperation::State : std::enable_shared_from_this<State> +{ + enum class Stage : uint8_t { + Begin, + Handoff, + PrepareLayout, + PrepareWrite, + PlanPage, + PlanDependencies, + Structure, + Capture, + Write, + Checkpoint, + Reconcile, + Rebase, + FinishMetrics + }; + + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + std::shared_ptr<State> owner; + void set_value() noexcept; + void set_value(FlushResult report) noexcept; + void set_error(Status status) noexcept; + void set_stopped() noexcept; + }; + + using Connected = stdexec::connect_result_t<detail::CallbackSender, Receiver>; + + struct Attempt + { + Attempt(detail::CallbackSender sender, std::shared_ptr<State> owner) + : connected(stdexec::connect(sender, Receiver{.owner = std::move(owner)})) + { + } + + Connected connected; + }; + + struct HandoffAttempt + { + HandoffAttempt(FlushReportSender sender, std::shared_ptr<State> owner) + : connected(stdexec::connect(sender, Receiver{.owner = std::move(owner)})) + { + } + + stdexec::connect_result_t<FlushReportSender, Receiver> connected; + }; + + explicit State(PublicationCoordinator &coordinator) : publisher(coordinator) + { + } + + void start(detail::CallbackComplete callback, void *argument) noexcept; + void wake() noexcept; + static void run(void *context) noexcept; + void pump(); + void begin_child(detail::CallbackSender sender, Stage next); + void begin_handoff(); + void child_done(Status status) noexcept; + void cancel_child(); + void preflight(); + void begin_structure(uint64_t pid); + void structured(); + void prepare_phase(bool writing); + void phase_ready(bool writing); + void begin_layout(); + void begin_materialization(); + void planned_page(); + void plan_dependencies(); + void planned(); + void capture(); + void captured(); + void written(); + void checkpointed(); + void begin_chunk_checkpoint(); + [[nodiscard]] bool backend_needs_recovery() const noexcept; + [[nodiscard]] uint64_t backend_identity() const noexcept; + [[nodiscard]] std::shared_ptr<const CapturedDurablePages> inherited_pages() const; + void reconcile(Status status); + void begin_rebase(); + void record_rebase_result() noexcept; + void finish_rebase(Status status); + void fail(Status status) noexcept; + void finish(Status status) noexcept; + + ApplyDispatcher::Node cpu_node{&State::run, this}; + std::shared_ptr<ApplyDispatcher> cpu_progress; + PrepareWorkAsync prepare_work_async; + PrepareCleanup on_terminal; + PrepareCleanup cancel_work; + CheckpointWork prepared_work; + PublicationCoordinator &publisher; + std::shared_ptr<const void> owner; + std::shared_ptr<LocalBackendState> backend; + std::shared_ptr<CheckpointBackend> chunk; + uint64_t chunk_wal_offset = 0; + std::shared_ptr<BackendCheckpoint> chunk_checkpoint; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<detail::FlushRuntime::Progress> progress; + CheckpointOptions options; + size_t max_layout_depth = 64; + PageLoadBudget budget; + std::shared_ptr<detail::AllocationReservation> finish_metadata; + std::shared_ptr<PageLoader> loader; + PrepareHandoff prepare_handoff; + PrepareStructure prepare_structure; + std::shared_ptr<PageSplitOperation> structure; + std::optional<uint64_t> fixed_coverage; + std::optional<uint64_t> fixed_generation; + std::optional<uint64_t> structure_l1_seq; + PrepareWork prepare_work; + PrepareCleanup prepare_cleanup; + std::shared_ptr<const void> work_owner; + std::shared_ptr<detail::AllocationReservation> control_metadata; + std::shared_ptr<CapturedFrameStream> planning_stream; + detail::ReservedUnique<CapturedLayoutPlanner> planner; + CapturedWritePlan plan; + bool last_plan = false; + uint64_t overflow_begin = 0; + std::shared_ptr<PublicationRequest> capture_request; + std::shared_ptr<PublicationRequest> finish_request; + PublicationResult publication; + std::shared_ptr<const PublishedTreeState> view; + std::shared_ptr<FlushOperation> handoff; + std::shared_ptr<CapturedWriteOperation> write; + std::shared_ptr<LocalCheckpointOperation> checkpoint; + std::shared_ptr<const CapturedDurablePages> written_pages; + std::shared_ptr<SnapshotBaseline> baseline; + std::shared_ptr<CapturedRebaseOperation> rebase; + std::optional<Attempt> attempt; + std::optional<HandoffAttempt> handoff_attempt; + Stage stage = Stage::Begin; + bool waiting = false; + bool capture_accepted = false; + Status child_status; + Status outcome; + std::atomic<bool> child_ready{false}; + std::atomic<bool> stopped{false}; + std::atomic<uint8_t> lifecycle{0}; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + CheckpointResult report; + std::shared_ptr<WritebackMetrics> writeback_metrics; + Status final_status; + TreeOutcome final_outcome = TreeOutcome::Cancelled; + std::optional<TreeMetricsOperation> capture_metrics; + std::optional<TreeMetricsOperation> commit_metrics; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation_plan.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation_plan.cpp new file mode 100644 index 000000000..12d1e14c2 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation_plan.cpp @@ -0,0 +1,194 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/writeback_metrics.h" +#include "snapshot/checkpoint_operation_internal.h" +#include "snapshot/writeback_selection.h" + +namespace crowdb::tree +{ +void CheckpointOperation::State::preflight() +{ + auto current = publisher.capture(); + if ((fixed_coverage && current->coverage != *fixed_coverage) || + (fixed_generation && current->generation != *fixed_generation) || + (structure_l1_seq && current->l1_seq != *structure_l1_seq)) { + finish(Status::unavailable("checkpoint fixed frontier or structural view changed")); + return; + } + fixed_coverage = current->coverage; + fixed_generation = current->generation; + structure_l1_seq.reset(); + if (current->pending_handoff || current->flush_admission_owner != 0 || current->checkpoint_inflight || + backend_needs_recovery()) { + finish(Status::unavailable()); + return; + } + if (options.select_oldest_page) { + auto status = select_writeback_page(*current, options.write.frames, &options.write.frames.selected_page); + if (!status.ok()) { + finish(std::move(status)); + return; + } + if (!options.write.frames.selected_page) { + report.captured_coverage = current->coverage; + report.writeback_completed = true; + finish(Status::Ok()); + return; + } + } + if (current->checkpoint_token == UINT64_MAX) { + finish(Status::resource_exhausted()); + return; + } + report.checkpoint_token = current->checkpoint_token + 1; + const auto record = InstallToken{.generation = current->generation, .sequence = report.checkpoint_token}; + capture_request->transaction.record = detail::reserved_shared<InstallRecord>(control_metadata, record); + finish_request->transaction.record = detail::reserved_shared<InstallRecord>(finish_metadata, record); + capture_request->transaction.checkpoint = {.action = CheckpointAction::Capture, .token = report.checkpoint_token}; + auto fixed = detail::reserved_shared<PublishedTreeState>(control_metadata, *current); + // Source writers may continue above this frontier. The immutable L1 map + // and Capture's expected L1 sequence keep this layout proof exact. + auto sources = detail::reserved_shared<PublishedSources>(control_metadata, control_metadata); + sources->floor = fixed->coverage; + fixed->sources = std::move(sources); + view = std::move(fixed); + prepare_phase(false); +} + +void CheckpointOperation::State::prepare_phase(bool writing) +{ + if (prepare_work_async) { + prepared_work = {}; + begin_child(prepare_work_async(*view, writing ? &plan : nullptr, &prepared_work), + writing ? Stage::PrepareWrite : Stage::PrepareLayout); + return; + } + if (prepare_work) { + auto status = prepare_work(*view, writing ? &plan : nullptr, &prepared_work); + if (!status.ok()) { + finish(std::move(status)); + return; + } + phase_ready(writing); + return; + } + if (writing) { + begin_materialization(); + } + else { + begin_layout(); + } +} + +void CheckpointOperation::State::phase_ready(bool writing) +{ + budget = std::move(prepared_work.budget); + loader = std::move(prepared_work.loader); + work_owner = std::move(prepared_work.owner); + if (writing) { + begin_materialization(); + } + else { + begin_layout(); + } +} + +void CheckpointOperation::State::begin_layout() +{ + const auto root = view->mapping.get(view->root_page_id); + const size_t depth = root && !options.write.frames.selected_page ? root->level + 1ULL : 1; + if (depth > max_layout_depth) { + finish(Status::resource_exhausted()); + return; + } + options.write.frames.max_depth = depth; + options.write.frames.reuse_backend_id = backend_identity(); + auto limits = options.write.frames; + limits.max_depth = depth; + limits.plan_only = true; + limits.encode.metadata = budget.bytes; + limits.cpu_progress = budget.cpu_progress; + planner = detail::reserved_unique<CapturedLayoutPlanner>(budget.bytes, view->next_page_id, backend_identity(), + options.write, options.metadata.max_segments, + inherited_pages(), budget.bytes); + auto status = CapturedFrameStream::create(view, loader, runtime, lane, std::move(limits), {}, &planning_stream); + if (!status.ok()) { + finish(std::move(status)); + return; + } + begin_child(planning_stream->sender(), Stage::PlanPage); +} + +void CheckpointOperation::State::planned_page() +{ + const auto &result = planning_stream->result(); + last_plan = result.complete; + if (!result.plan) { + if (last_plan) { + planned(); + } + else { + finish(Status::corruption("layout stream produced no plan")); + } + return; + } + // A split window leaves its unvisited suffix explicitly marked. Complete + // that layout proof even when the suffix already fits a physical frame. + const auto page = view->mapping.get(result.plan->page_id); + if (prepare_structure && page && page->leaf && page->metrics.split_candidate) { + begin_structure(result.plan->page_id); + return; + } + auto status = planner->accept(*result.plan); + if (!status.ok()) { + finish(std::move(status)); + return; + } + stage = Stage::PlanDependencies; + wake(); +} + +void CheckpointOperation::State::plan_dependencies() +{ + auto status = planner->step(); + if (!status.ok()) { + finish(std::move(status)); + return; + } + if (planner->pending()) { + wake(); + return; + } + if (last_plan) { + planned(); + return; + } + begin_child(planning_stream->sender(), Stage::PlanPage); +} + +void CheckpointOperation::State::planned() +{ + auto status = planner->finish(); + if (!status.ok()) { + finish(std::move(status)); + return; + } + plan = planner->result(); + planning_stream.reset(); + planner.reset(); + if (prepare_work || prepare_work_async) { + loader.reset(); + budget = {}; + work_owner.reset(); + } + prepare_phase(true); +} + +void CheckpointOperation::State::begin_materialization() +{ + writeback_metrics = detail::reserved_shared<WritebackMetrics>(budget.bytes, publisher.work_metrics(), budget.bytes, + plan.primary_frames); + options.write.frames.writeback_metrics = writeback_metrics; + capture(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp new file mode 100644 index 000000000..4938a2292 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp @@ -0,0 +1,405 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/extent_writeback.h" +#include "snapshot/checkpoint_operation_internal.h" + +namespace crowdb::tree +{ +void CheckpointOperation::State::Receiver::set_value() noexcept +{ + auto self = std::move(owner); + self->child_done(Status::Ok()); +} + +void CheckpointOperation::State::Receiver::set_value(FlushResult report) noexcept +{ + auto self = std::move(owner); + self->fixed_coverage = report.frontier; + self->child_done(report.handoff_status.ok() ? std::move(report.maintenance_status) + : std::move(report.handoff_status)); +} + +void CheckpointOperation::State::Receiver::set_error(Status status) noexcept +{ + auto self = std::move(owner); + self->child_done(std::move(status)); +} + +void CheckpointOperation::State::Receiver::set_stopped() noexcept +{ + auto self = std::move(owner); + self->child_done(Status::unavailable()); +} + +void CheckpointOperation::State::child_done(Status status) noexcept +{ + child_status = std::move(status); + child_ready.store(true, std::memory_order_release); + wake(); +} + +void CheckpointOperation::State::begin_child(detail::CallbackSender sender, Stage next) +{ + stage = next; + waiting = true; + child_ready.store(false, std::memory_order_release); + attempt.emplace(sender, shared_from_this()); + stdexec::start(attempt->connected); +} + +void CheckpointOperation::State::begin_handoff() +{ + stage = Stage::Handoff; + waiting = true; + child_ready.store(false, std::memory_order_release); + handoff_attempt.emplace(handoff->report_sender(), shared_from_this()); + stdexec::start(handoff_attempt->connected); +} + +void CheckpointOperation::State::cancel_child() +{ + if ((stage == Stage::PrepareLayout || stage == Stage::PrepareWrite) && cancel_work) { + cancel_work(); + } + else if (stage == Stage::Structure && structure) { + structure->request_stop(); + } + else if (stage == Stage::Handoff && handoff) { + handoff->request_stop(); + } + else if (stage == Stage::PlanPage && planning_stream) { + planning_stream->request_stop(); + } + else if (stage == Stage::Rebase && rebase) { + rebase->request_stop(); + } + else if (stage == Stage::Write && write) { + write->request_stop(); + } + else if (stage == Stage::Checkpoint && chunk_checkpoint) { + chunk_checkpoint->request_stop(); + } + else if (stage == Stage::Checkpoint && checkpoint) { + checkpoint->request_stop(); + } +} + +void CheckpointOperation::State::pump() +{ + if (stage == Stage::FinishMetrics) { + finish(Status::Ok()); + return; + } + if (waiting) { + if (stopped.load(std::memory_order_acquire)) { + cancel_child(); + } + if (!child_ready.load(std::memory_order_acquire)) { + return; + } + waiting = false; + attempt.reset(); + handoff_attempt.reset(); + if (!child_status.ok() && stage == Stage::Rebase) { + finish_rebase(child_status.without_message()); + return; + } + if (!child_status.ok() && stage == Stage::PlanPage && planning_stream && prepare_structure && + planning_stream->result().needs_split && !stopped.load(std::memory_order_acquire)) { + begin_structure(*planning_stream->result().needs_split); + return; + } + if (!child_status.ok()) { + fail(child_status.without_message()); + return; + } + } + if (stopped.load(std::memory_order_acquire) && stage != Stage::Capture && stage != Stage::Checkpoint && + stage != Stage::Reconcile && stage != Stage::Rebase) { + fail(Status::unavailable()); + return; + } + switch (stage) { + case Stage::Begin: + if (prepare_handoff) { + auto status = prepare_handoff(&handoff); + if (!status.ok()) { + finish(std::move(status)); + return; + } + if (!handoff) { + finish(Status::internal_error()); + return; + } + begin_handoff(); + } + else { + preflight(); + } + break; + case Stage::Handoff: + handoff.reset(); + preflight(); + break; + case Stage::PrepareLayout: + phase_ready(false); + break; + case Stage::PrepareWrite: + phase_ready(true); + break; + case Stage::PlanPage: + planned_page(); + break; + case Stage::PlanDependencies: + plan_dependencies(); + break; + case Stage::Structure: + structured(); + break; + case Stage::Capture: + captured(); + break; + case Stage::Write: + written(); + break; + case Stage::Checkpoint: + checkpointed(); + break; + case Stage::Reconcile: + if (options.mode == CheckpointMode::Durable && report.durable_committed) { + begin_rebase(); + } + else { + finish(outcome); + } + break; + case Stage::Rebase: + finish_rebase(Status::Ok()); + break; + case Stage::FinishMetrics: + finish(Status::Ok()); + break; + } +} + +void CheckpointOperation::State::capture() +{ + capture_metrics.emplace(publisher.work_metrics(), TreeStage::SnapshotCapture, tree_metrics_now_ns()); + if (view->pending_handoff || view->flush_admission_owner != 0 || backend_needs_recovery()) { + finish(Status::unavailable()); + return; + } + auto &transaction = capture_request->transaction; + transaction.expected_coverage = view->coverage; + transaction.expected_l1_seq = view->l1_seq; + transaction.expected_next_page_id = view->next_page_id; + transaction.next_page_id = view->next_page_id + plan.new_overflow_frames; + overflow_begin = view->next_page_id; + stage = Stage::Capture; + waiting = true; + child_ready.store(false, std::memory_order_release); + capture_request->on_done = [self = shared_from_this()](PublicationResult result) { + const auto status = result.status.without_message(); + self->publication = std::move(result); + self->child_done(status); + }; + const auto submitted = options.publication ? publisher.try_submit(capture_request, options.publication) + : publisher.try_submit(capture_request); + if (!submitted) { + waiting = false; + finish(Status::unavailable()); + } +} + +void CheckpointOperation::State::captured() +{ + capture_accepted = true; + view = std::move(publication.checkpoint_view); + capture_request.reset(); + if (!view) { + reconcile(Status::internal_error()); + return; + } + if (capture_metrics) { + (void)capture_metrics->finish(TreeOutcome::Success, tree_metrics_now_ns()); + } + if (options.mode == CheckpointMode::Durable) { + commit_metrics.emplace(publisher.work_metrics(), TreeStage::SnapshotCommit, tree_metrics_now_ns()); + } + report.captured_coverage = view->coverage; + report.primary_frames = plan.primary_frames; + if (stopped.load(std::memory_order_acquire)) { + reconcile(Status::unavailable()); + return; + } + auto artifact = detail::reserved_shared<PublishedTreeState>(control_metadata, *view); + artifact->next_page_id = overflow_begin; + view = std::move(artifact); + const auto end = overflow_begin + plan.new_overflow_frames; + // The publication owns the advanced high-water mark. A tiny independent + // owner, rather than this operation, protects the emitted interval. + auto lease = detail::reserved_shared<ReservedOverflowIds>(budget.bytes); + lease->first = overflow_begin; + lease->count = plan.new_overflow_frames; + auto reserve = [lease, next = overflow_begin, end](uint64_t count, ReservedOverflowIds *out) mutable { + if (count > end - next) { + return Status::corruption("encoded overflow exceeds its planned lease"); + } + *out = {.first = next, .count = count, .owner = lease}; + next += count; + return Status::Ok(); + }; + auto status = CapturedWriteOperation::create( + view, loader, store, runtime, lane, options.write, budget, std::move(reserve), + chunk ? chunk->reserve(budget.bytes) : extent_write_reservation(backend->ledger, budget.bytes), + inherited_pages(), &write); + if (!status.ok()) { + reconcile(std::move(status)); + return; + } + begin_child(write->sender(), Stage::Write); +} + +void CheckpointOperation::State::written() +{ + written_pages = write->result().pages; + report.written_frames = write->result().written_frames; + report.reused_frames = write->result().reused_frames; + report.written_bytes = write->result().written_bytes; + report.writeback_completed = true; + if (options.mode == CheckpointMode::Writeback) { + write.reset(); + begin_rebase(); + return; + } + if (chunk) { + begin_chunk_checkpoint(); + return; + } + auto status = LocalCheckpointOperation::create(written_pages, backend->checkpoints, store, runtime, lane, + options.metadata, budget, &checkpoint); + if (!status.ok()) { + reconcile(std::move(status)); + return; + } + write.reset(); + begin_child(checkpoint->sender(), Stage::Checkpoint); +} + +void CheckpointOperation::State::checkpointed() +{ + if (chunk) { + chunk_checkpoint->completed(); + if (!chunk_checkpoint->committed()) { + reconcile(chunk_checkpoint->status()); + return; + } + } + report.durable_committed = true; + if (commit_metrics) { + (void)commit_metrics->finish(TreeOutcome::Success, tree_metrics_now_ns()); + } + reconcile(Status::Ok()); +} + +void CheckpointOperation::State::reconcile(Status status) +{ + if (report.durable_committed) { + // The capture token excludes a later checkpoint until reconciliation. + // Record this checkpoint's pair before releasing that serialization. + report.durable_coverage = + chunk ? chunk->durable_coverage.load(std::memory_order_acquire) : backend->checkpoints->durable_coverage(); + report.snapshot_sequence = chunk ? chunk->snapshot_sequence.load(std::memory_order_acquire) + : backend->checkpoints->snapshot_sequence(); + baseline->generation = view->generation; + baseline->capture_token = report.checkpoint_token; + baseline->coverage = report.durable_coverage; + baseline->snapshot_sequence = report.snapshot_sequence; + baseline->backend_id = backend_identity(); + baseline->gc_floor = view->gc_floor; + baseline->wal_offset = chunk_wal_offset; + baseline->pages = written_pages; + if (options.diagnostics != nullptr) { + options.diagnostics->written_frames.store(report.written_frames, std::memory_order_release); + options.diagnostics->total_written_frames.fetch_add(report.written_frames, std::memory_order_relaxed); + } + } + outcome = std::move(status); + stage = Stage::Reconcile; + waiting = true; + child_ready.store(false, std::memory_order_release); + finish_request->transaction.checkpoint = {.action = report.durable_committed ? CheckpointAction::Commit + : CheckpointAction::Abort, + .token = report.checkpoint_token, + .baseline = report.durable_committed ? baseline : nullptr}; + finish_request->on_done = [self = shared_from_this()](PublicationResult result) { + self->child_done(result.status.without_message()); + }; + if (!publisher.try_submit_checkpoint_completion(finish_request)) { + waiting = false; + finish(Status::unavailable()); + } +} + +void CheckpointOperation::State::begin_rebase() +{ + if (stopped.load(std::memory_order_acquire)) { + finish_rebase(Status::unavailable()); + return; + } + auto status = CapturedRebaseOperation::create(publisher, view, written_pages, runtime, lane, budget.bytes, &rebase, + options.write.frames.encode.max_sources, writeback_metrics, + cpu_progress, options.publication, options.cause); + if (!status.ok()) { + finish_rebase(std::move(status)); + return; + } + begin_child(rebase->sender(), Stage::Rebase); +} + +void CheckpointOperation::State::record_rebase_result() noexcept +{ + report.rebased_pages = rebase ? rebase->result().installed : 0; + const auto reused = rebase ? rebase->result().reused : 0; + report.rebase_deferred = plan.primary_frames - report.rebased_pages - reused; +} + +void CheckpointOperation::State::finish_rebase(Status status) +{ + record_rebase_result(); + if (options.mode == CheckpointMode::Writeback) { + // The protected receipts stay alive through the last publication. Abort + // retains checkpoint-pending changes; staged data is not a WAL cutoff. + reconcile(std::move(status)); + } + else { + finish(Status::Ok()); + } +} + +void CheckpointOperation::State::fail(Status status) noexcept +{ + if (chunk && stage == Stage::Checkpoint && chunk_checkpoint) { + chunk_checkpoint->completed(); + if (chunk_checkpoint->committed()) { + report.durable_committed = true; + status = Status::Ok(); + } + } + if (report.durable_committed && stage == Stage::Rebase) { + record_rebase_result(); + finish(Status::Ok()); + } + else if (capture_accepted && stage != Stage::Reconcile) { + try { + reconcile(std::move(status)); + } + catch (...) { + finish(Status::internal_error()); + } + } + else { + finish(std::move(status)); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation_structure.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation_structure.cpp new file mode 100644 index 000000000..873265579 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation_structure.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/checkpoint_operation_internal.h" + +namespace crowdb::tree +{ +void CheckpointOperation::State::begin_structure(uint64_t pid) +{ + // Drop the rejected proof and all its phase grants before admitting the + // next bounded split/load window. The fixed L1 view is control-owned. + planning_stream.reset(); + planner.reset(); + loader.reset(); + budget = {}; + work_owner.reset(); + const auto current = publisher.capture(); + if (current->l1_seq != view->l1_seq || current->generation != view->generation || + current->coverage != view->coverage || current->pending_handoff || current->flush_admission_owner != 0 || + current->checkpoint_inflight || backend_needs_recovery()) { + finish(Status::unavailable("checkpoint split input changed")); + return; + } + auto status = prepare_structure(view, pid, &structure); + if (!status.ok() || !structure) { + finish(status.ok() ? Status::internal_error("checkpoint split operation absent") : std::move(status)); + return; + } + // The split window owns any source-free descriptor it needed to create. + // Reusing the checkpoint's fixed control credit would accumulate deferred + // descriptor retirement over a long sequence of suffix windows. + view = structure->captured_view(); + begin_child(structure->sender(), Stage::Structure); +} + +void CheckpointOperation::State::structured() +{ + structure_l1_seq = structure->l1_sequence(); + if (*structure_l1_seq <= view->l1_seq) { + finish(Status::internal_error("checkpoint split did not publish progress")); + return; + } + ++report.structural_windows; + std::optional<uint64_t> next; + auto current = publisher.capture(); + // A full checkpoint must finish the split suffix before its layout proof. + // Single-page writeback keeps its existing bounded page selection. + if (!options.write.frames.selected_page) { + if (current->l1_seq == *structure_l1_seq && current->generation == view->generation && + current->coverage == view->coverage) { + for (const auto pid : structure->output_leaf_pids()) { + const auto page = current->mapping.get(pid); + if (page && page->level == 0 && page->leaf && page->metrics.split_candidate) { + next = pid; + break; + } + } + } + } + structure.reset(); + if (next) { + view = std::move(current); + begin_structure(*next); + return; + } + view.reset(); + // No second handoff: future L0 writers stay above the initial frontier. + // A concurrent L1 writer invalidates this attempt instead of extending it. + preflight(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_index_codec.cpp b/lib/crowdb-tree/src/snapshot/chunk_index_codec.cpp new file mode 100644 index 000000000..704766138 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_index_codec.cpp @@ -0,0 +1,174 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "snapshot/chunk_root_codec_internal.h" + +namespace crowdb::tree +{ +namespace +{ +bool valid_frame(const ChunkIndexFrame &frame) +{ + if (frame.page_id == kInvalidPageId || frame.frame_bytes < kFrameHeaderSize + kFrameTrailerSize || + frame.logical_bytes == 0 || frame.logical_bytes > frame.pack.length || + !chunk_codec::valid_pack(frame.pack, PagePurpose::kBtreePage)) { + return false; + } + if (frame.type == page_type::kOverflowFrame) { + return frame.stored_reference_words.empty() && frame.overflow_bytes != 0 && + frame.overflow_bytes <= frame.frame_bytes - kFrameHeaderSize - kFrameTrailerSize && + frame.next_overflow != frame.page_id; + } + return (frame.type == page_type::kLeafBase || frame.type == page_type::kInnerBase) && + frame.next_overflow == kInvalidPageId && frame.overflow_bytes == 0 && + (frame.type != page_type::kInnerBase || frame.stored_reference_words.empty()); +} + +Status frame_size(const DurableCapturedFrame &frame, const ChunkRootIdentity &identity, size_t *bytes) +{ + if (!frame.backing || frame.backing->storage != BackingStorage::kChunkPack || !frame.backing->reloadable() || + frame.backing->chunk->domain->tree != identity.tree || + (frame.type == page_type::kLeafBase && !frame.stored_overflow)) { + return Status::invalid_argument("Chunk index needs owned same-tree pack references"); + } + const auto refs = frame.stored_overflow ? frame.stored_overflow->size() : 0; + if (refs > kChunkIndexReferencesPerFrame || (frame.type != page_type::kLeafBase && refs != 0)) { + return Status::resource_exhausted("Chunk index stored-reference bound exceeded"); + } + ChunkIndexFrame geometry{.page_id = frame.page_id, + .type = frame.type, + .frame_bytes = frame.frame_bytes, + .logical_bytes = frame.logical_bytes, + .next_overflow = frame.next_overflow, + .overflow_bytes = frame.overflow_bytes, + .pack = frame.backing->chunk->pack, + .stored_reference_words = {}}; + if (!valid_frame(geometry)) { + return Status::invalid_argument("Chunk index frame geometry is invalid"); + } + if (frame.stored_overflow) { + for (const auto &ref : *frame.stored_overflow) { + if (ref.head == kInvalidPageId || ref.head >= identity.next_page_id || ref.bytes == 0) { + return Status::invalid_argument("Chunk index overflow reference is invalid"); + } + } + } + *bytes = kChunkIndexRecordBytes + (refs * 16); + return Status::Ok(); +} +} // namespace + +Status encode_chunk_index(const ChunkRootIdentity &identity, uint64_t ordinal, + std::span<const DurableCapturedFrame *const> frames, ChunkMetadataBytes *out) +try { + using namespace chunk_codec; + if (out == nullptr || !out->empty() || identity.tree == 0 || identity.root_generation == 0 || frames.empty() || + frames.size() > kChunkIndexFrameLimit) { + return Status::invalid_argument(); + } + size_t size = kChunkIndexHeaderBytes + 4; + uint64_t previous = 0; + for (size_t i = 0; i < frames.size(); ++i) { + if (frames[i] == nullptr || frames[i]->page_id >= identity.next_page_id || + (i != 0 && frames[i]->page_id <= previous)) { + return Status::invalid_argument("Chunk index page identities must be ordered"); + } + size_t bytes = 0; + auto status = frame_size(*frames[i], identity, &bytes); + if (!status.ok()) { + return status; + } + size += bytes; + previous = frames[i]->page_id; + } + out->resize(size); + auto bytes = std::span(*out); + const uint64_t header[]{kIndexMagic, 1, identity.tree, identity.root_generation, ordinal, frames.size()}; + for (size_t i = 0; i < std::size(header); ++i) { + put(bytes, i, header[i]); + } + size_t offset = kChunkIndexHeaderBytes; + for (const auto *frame : frames) { + auto record = bytes.subspan(offset, kChunkIndexRecordBytes); + const uint64_t words[]{frame->page_id, static_cast<uint32_t>(frame->type), + frame->frame_bytes, frame->logical_bytes, + frame->next_overflow, frame->overflow_bytes}; + for (size_t i = 0; i < std::size(words); ++i) { + put(record, i, words[i]); + } + pack(record.subspan(48), frame->backing->chunk->pack); + const auto refs = frame->stored_overflow ? frame->stored_overflow->size() : 0; + put(record, 15, refs); + offset += kChunkIndexRecordBytes; + if (frame->stored_overflow) { + for (const auto &ref : *frame->stored_overflow) { + put(bytes.subspan(offset, 16), 0, ref.head); + put(bytes.subspan(offset, 16), 1, ref.bytes); + offset += 16; + } + } + } + seal(bytes); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status decode_chunk_index_header(std::span<const uint8_t> bytes, const ChunkRootIdentity &identity, uint64_t ordinal, + uint64_t *count, size_t *offset) +{ + using namespace chunk_codec; + if (count == nullptr || offset == nullptr) { + return Status::invalid_argument(); + } + if (bytes.size() > kChunkIndexWindowBytes || !verified(bytes, kChunkIndexHeaderBytes) || + word(bytes, 0) != kIndexMagic || word(bytes, 1) != 1 || word(bytes, 2) != identity.tree || + word(bytes, 3) != identity.root_generation || word(bytes, 4) != ordinal || word(bytes, 5) == 0 || + word(bytes, 5) > kChunkIndexFrameLimit) { + return Status::corruption("Chunk index checksum, format or identity mismatch"); + } + *count = word(bytes, 5); + *offset = kChunkIndexHeaderBytes; + return Status::Ok(); +} + +Status decode_chunk_index_frame(std::span<const uint8_t> bytes, size_t *offset, ChunkIndexFrame *out) +{ + using namespace chunk_codec; + if (offset == nullptr || out == nullptr || *offset < kChunkIndexHeaderBytes || *offset > bytes.size()) { + return Status::invalid_argument(); + } + if (bytes.size() - *offset < kChunkIndexRecordBytes + 4) { + return Status::corruption("Chunk index record is truncated"); + } + const auto record = bytes.subspan(*offset, kChunkIndexRecordBytes); + if (word(record, 1) > UINT8_MAX || word(record, 2) > UINT32_MAX || word(record, 3) > UINT32_MAX || + word(record, 5) > UINT32_MAX || word(record, 15) > kChunkIndexReferencesPerFrame || + !unpack(record.subspan(48), PagePurpose::kBtreePage, &out->pack)) { + return Status::corruption("Chunk index frame fields exceed their encoded widths"); + } + const auto reference_bytes = word(record, 15) * 16; + if (reference_bytes > bytes.size() - *offset - kChunkIndexRecordBytes - 4) { + return Status::corruption("Chunk index stored references are truncated"); + } + out->page_id = word(record, 0); + out->type = static_cast<page_type>(word(record, 1)); + out->frame_bytes = static_cast<uint32_t>(word(record, 2)); + out->logical_bytes = static_cast<uint32_t>(word(record, 3)); + out->next_overflow = word(record, 4); + out->overflow_bytes = static_cast<uint32_t>(word(record, 5)); + out->stored_reference_words = bytes.subspan(*offset + kChunkIndexRecordBytes, reference_bytes); + if (!valid_frame(*out)) { + return Status::corruption("Chunk index frame geometry is invalid"); + } + for (size_t i = 0; i < reference_bytes; i += 16) { + const auto ref = out->stored_reference_words.subspan(i, 16); + if (word(ref, 0) == kInvalidPageId || word(ref, 1) == 0) { + return Status::corruption("Chunk index stored reference is invalid"); + } + } + *offset += kChunkIndexRecordBytes + reference_bytes; + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_codec.cpp b/lib/crowdb-tree/src/snapshot/chunk_root_codec.cpp new file mode 100644 index 000000000..a4de545e8 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_codec.cpp @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "snapshot/chunk_root_codec_internal.h" + +namespace crowdb::tree +{ +namespace +{ +bool valid_identity(const ChunkRootIdentity &id, uint64_t frames) +{ + return id.tree != 0 && id.root_generation != 0 && id.owner_epoch != 0 && id.snapshot_sequence != 0 && + id.page_generation != 0 && id.gc_floor <= id.coverage && id.next_page_id <= MappingView::kPageLimit && + ((frames == 0 && id.root_page_id == kInvalidPageId) || (frames != 0 && id.root_page_id < id.next_page_id)); +} + +bool valid_indexes(const ChunkRootManifest &root) +{ + uint64_t count = 0; + uint64_t previous = 0; + for (const auto &entry : root.indexes) { + if (entry.frame_count == 0 || entry.frame_count > kChunkIndexFrameLimit || entry.first_page > entry.last_page || + entry.last_page >= root.identity.next_page_id || (count != 0 && entry.first_page <= previous) || + !chunk_codec::valid_pack(entry.pack, PagePurpose::kPageIndex) || + entry.pack.length > kChunkIndexWindowBytes || count > UINT64_MAX - entry.frame_count) { + return false; + } + count += entry.frame_count; + previous = entry.last_page; + } + return count == root.frame_count && root.stored_references <= detail::kMaxCandidateReferences && + root.dependency_entries <= detail::kMaxCandidateReferences && + root.dependency_entries >= root.stored_references && + (root.stored_references != 0 || root.dependency_entries == 0); +} +} // namespace + +Status encode_chunk_root(const ChunkRootManifest &root, ChunkMetadataBytes *out) +try { + using namespace chunk_codec; + if (out == nullptr || !out->empty() || !valid_identity(root.identity, root.frame_count) || !valid_indexes(root)) { + return Status::invalid_argument("invalid Chunk root identity or directory"); + } + const auto size = detail::AllocationBound(kChunkRootHeaderBytes + 4) + .add_product(root.indexes.size(), kChunkRootEntryBytes) + .value(); + if (!size || *size > detail::kMaxCandidateManifestBytes) { + return Status::resource_exhausted(); + } + out->resize(*size); + auto bytes = std::span(*out); + const auto &id = root.identity; + const uint64_t words[]{kRootMagic, + 1, + id.tree, + id.root_generation, + id.owner_epoch, + id.snapshot_sequence, + id.page_generation, + id.coverage, + id.gc_floor, + id.root_page_id, + id.next_page_id, + id.wal_replay_offset, + root.frame_count, + root.indexes.size(), + root.stored_references, + root.dependency_entries}; + for (size_t i = 0; i < std::size(words); ++i) { + put(bytes, i, words[i]); + } + size_t offset = kChunkRootHeaderBytes; + for (const auto &entry : root.indexes) { + auto encoded = bytes.subspan(offset, kChunkRootEntryBytes); + put(encoded, 0, entry.first_page); + put(encoded, 1, entry.last_page); + put(encoded, 2, entry.frame_count); + pack(encoded.subspan(24), entry.pack); + offset += kChunkRootEntryBytes; + } + seal(bytes); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status decode_chunk_root(std::span<const uint8_t> bytes, uint64_t tree, uint64_t exact_generation, + ChunkRootManifest *out) +try { + using namespace chunk_codec; + if (out == nullptr || !out->indexes.empty() || tree == 0) { + return Status::invalid_argument(); + } + if (bytes.size() > detail::kMaxCandidateManifestBytes || !verified(bytes, kChunkRootHeaderBytes) || + word(bytes, 0) != kRootMagic || word(bytes, 1) != 1 || word(bytes, 2) != tree || + (exact_generation != 0 && word(bytes, 3) != exact_generation)) { + return Status::corruption("Chunk root checksum, format or identity mismatch"); + } + const auto count = word(bytes, 13); + if (count > (bytes.size() - kChunkRootHeaderBytes - 4) / kChunkRootEntryBytes || + bytes.size() != kChunkRootHeaderBytes + 4 + (count * kChunkRootEntryBytes)) { + return Status::corruption("Chunk root directory length mismatch"); + } + out->identity = {.tree = tree, + .root_generation = word(bytes, 3), + .owner_epoch = word(bytes, 4), + .snapshot_sequence = word(bytes, 5), + .page_generation = word(bytes, 6), + .coverage = word(bytes, 7), + .gc_floor = word(bytes, 8), + .root_page_id = word(bytes, 9), + .next_page_id = word(bytes, 10), + .wal_replay_offset = word(bytes, 11)}; + out->frame_count = word(bytes, 12); + out->stored_references = word(bytes, 14); + out->dependency_entries = word(bytes, 15); + out->indexes.reserve(count); + for (uint64_t i = 0; i < count; ++i) { + const auto encoded = bytes.subspan(kChunkRootHeaderBytes + (i * kChunkRootEntryBytes), kChunkRootEntryBytes); + ChunkIndexReference entry{ + .first_page = word(encoded, 0), .last_page = word(encoded, 1), .frame_count = word(encoded, 2), .pack = {}}; + if (!unpack(encoded.subspan(24), PagePurpose::kPageIndex, &entry.pack)) { + return Status::corruption("Chunk root index pack identity mismatch"); + } + out->indexes.push_back(entry); + } + return valid_identity(out->identity, out->frame_count) && valid_indexes(*out) + ? Status::Ok() + : Status::corruption("Chunk root directory is unordered or incomplete"); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_codec.h b/lib/crowdb-tree/src/snapshot/chunk_root_codec.h new file mode 100644 index 000000000..8ebbf2158 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_codec.h @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_backing.h" +#include "snapshot/readable_page_catalog.h" + +namespace crowdb::tree +{ +inline constexpr size_t kChunkIndexFrameLimit = 64; +inline constexpr size_t kChunkIndexReferencesPerFrame = 4096; +inline constexpr size_t kChunkIndexHeaderBytes = 48; +inline constexpr size_t kChunkIndexRecordBytes = 128; +inline constexpr size_t kChunkRootHeaderBytes = 128; +inline constexpr size_t kChunkRootEntryBytes = 96; +inline constexpr size_t kChunkIndexWindowBytes = + kChunkIndexHeaderBytes + 4 + + (kChunkIndexFrameLimit * (kChunkIndexRecordBytes + (16 * kChunkIndexReferencesPerFrame))); + +struct ChunkRootIdentity +{ + uint64_t tree = 0; + uint64_t root_generation = 0; + uint64_t owner_epoch = 0; + uint64_t snapshot_sequence = 0; + uint64_t page_generation = 0; + uint64_t coverage = 0; + uint64_t gc_floor = 0; + uint64_t root_page_id = kInvalidPageId; + uint64_t next_page_id = 0; + uint64_t wal_replay_offset = 0; +}; + +struct ChunkIndexReference +{ + uint64_t first_page = 0; + uint64_t last_page = 0; + uint64_t frame_count = 0; + detail::ChunkFixedPackRef pack; +}; + +struct ChunkRootManifest +{ + explicit ChunkRootManifest(const std::shared_ptr<detail::AllocationReservation> &budget) + : indexes(detail::ReservationAllocator<ChunkIndexReference>(budget)) + { + } + + ChunkRootIdentity identity; + uint64_t frame_count = 0; + uint64_t stored_references = 0; + uint64_t dependency_entries = 0; + detail::ReservedVector<ChunkIndexReference> indexes; +}; + +using ChunkMetadataBytes = detail::ReservedVector<uint8_t>; + +Status encode_chunk_root(const ChunkRootManifest &root, ChunkMetadataBytes *out); +Status decode_chunk_root(std::span<const uint8_t> bytes, uint64_t tree, uint64_t exact_generation, + ChunkRootManifest *out); +Status encode_chunk_index(const ChunkRootIdentity &identity, uint64_t ordinal, + std::span<const DurableCapturedFrame *const> frames, ChunkMetadataBytes *out); + +// Parsing does no I/O or pin registration. Callers acquire exact pack pins on +// the backend lane before turning these records into readable BackingRefs. +struct ChunkIndexFrame +{ + uint64_t page_id = 0; + page_type type = page_type::kLeafBase; + uint32_t frame_bytes = 0; + uint32_t logical_bytes = 0; + uint64_t next_overflow = kInvalidPageId; + uint32_t overflow_bytes = 0; + detail::ChunkFixedPackRef pack; + std::span<const uint8_t> stored_reference_words; +}; + +Status decode_chunk_index_header(std::span<const uint8_t> bytes, const ChunkRootIdentity &identity, uint64_t ordinal, + uint64_t *count, size_t *offset); +Status decode_chunk_index_frame(std::span<const uint8_t> bytes, size_t *offset, ChunkIndexFrame *out); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_codec_internal.h b/lib/crowdb-tree/src/snapshot/chunk_root_codec_internal.h new file mode 100644 index 000000000..689ca9c19 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_codec_internal.h @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_frame.h" +#include "crowdb-common/crc32c.h" +#include "snapshot/chunk_root_codec.h" + +namespace crowdb::tree::chunk_codec +{ +inline constexpr uint64_t kRootMagic = 0x33544F4F524B4843ULL; +inline constexpr uint64_t kIndexMagic = 0x335845444E494843ULL; + +inline uint64_t word(std::span<const uint8_t> input, size_t index) +{ + uint64_t value = 0; + for (size_t byte = 0; byte < 8; ++byte) { + value |= uint64_t{input[(index * 8) + byte]} << (byte * 8); + } + return value; +} + +inline void put(std::span<uint8_t> out, size_t index, uint64_t value) +{ + for (size_t byte = 0; byte < 8; ++byte) { + out[(index * 8) + byte] = static_cast<uint8_t>(value >> (byte * 8)); + } +} + +inline void seal(std::span<uint8_t> out) +{ + const auto crc = crowdb::common::crc32c(out.data(), out.size() - 4); + for (size_t i = 0; i < 4; ++i) { + out[out.size() - 4 + i] = static_cast<uint8_t>(crc >> (8 * i)); + } +} + +inline bool verified(std::span<const uint8_t> input, size_t header) +{ + if (input.size() < header + 4) { + return false; + } + uint32_t saved = 0; + for (size_t i = 0; i < 4; ++i) { + saved |= uint32_t{input[input.size() - 4 + i]} << (i * 8); + } + return saved == crowdb::common::crc32c(input.data(), input.size() - 4); +} + +inline bool valid_pack(const detail::ChunkFixedPackRef &pack, PagePurpose purpose) +{ + return pack.stage.owner_epoch != 0 && (pack.stage.operation_high != 0 || pack.stage.operation_low != 0) && + pack.purpose == purpose && detail::chunk_purpose(pack.chunk) == purpose && pack.length != 0 && + pack.length <= detail::kMaxChunkPackBytes && + pack.offset <= UINT64_MAX - detail::chunk_framed_length(pack.length); +} + +inline void pack(std::span<uint8_t> out, const detail::ChunkFixedPackRef &ref) +{ + const uint64_t words[]{ref.stage.owner_epoch, + ref.stage.operation_high, + ref.stage.operation_low, + ref.chunk.high, + ref.chunk.low, + ref.offset, + ref.length, + ref.checksum, + static_cast<uint32_t>(ref.purpose)}; + for (size_t i = 0; i < std::size(words); ++i) { + put(out, i, words[i]); + } +} + +inline bool unpack(std::span<const uint8_t> input, PagePurpose purpose, detail::ChunkFixedPackRef *out) +{ + if (word(input, 6) > UINT32_MAX || word(input, 7) > UINT32_MAX || + word(input, 8) != static_cast<uint32_t>(purpose)) { + return false; + } + *out = { + .stage = {.owner_epoch = word(input, 0), .operation_high = word(input, 1), .operation_low = word(input, 2)}, + .chunk = {word(input, 3), word(input, 4)}, + .offset = word(input, 5), + .length = static_cast<uint32_t>(word(input, 6)), + .checksum = static_cast<uint32_t>(word(input, 7)), + .purpose = purpose + }; + return valid_pack(*out, purpose); +} +} // namespace crowdb::tree::chunk_codec diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_commit.cpp b/lib/crowdb-tree/src/snapshot/chunk_root_commit.cpp new file mode 100644 index 000000000..c59c62a16 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_commit.cpp @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/chunk_root_commit.h" + +#include "backend/chunk/chunk_physical_registry.h" +#include "btree/allocation_bound.h" + +#include <algorithm> + +namespace crowdb::tree +{ +ChunkRootCommit::ChunkRootCommit(ConstructionKey /*unused*/, std::shared_ptr<detail::AllocationReservation> metadata) + : metadata_(std::move(metadata)), + root_(metadata_), + manifest_(detail::ReservationAllocator<uint8_t>(metadata_)), + index_pins_(detail::ReservationAllocator<std::shared_ptr<const ChunkPackBacking>>(metadata_)), + references_(std::less<>{}, detail::ReservationAllocator<detail::ChunkStageId>(metadata_)) +{ +} + +Status ChunkRootCommit::create(std::shared_ptr<const CapturedDurablePages> pages, + std::shared_ptr<const ChunkBackingDomain> domain, + std::shared_ptr<detail::ChunkRuntimeIds> ids, ChunkRootIdentity identity, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<detail::AllocationReservation> window, + std::shared_ptr<ChunkRootCommit> *out, size_t max_dependency_steps) +try { + if (max_dependency_steps == 0 || max_dependency_steps > detail::kMaxCandidateReferences || !pages || !domain || + !domain->supported() || !ids || !metadata || !window || out == nullptr || *out || + identity.tree != domain->tree || identity.owner_epoch != domain->epoch || identity.root_generation == 0 || + identity.page_generation != pages->generation || identity.coverage != pages->coverage || + identity.root_page_id != pages->root_page_id || identity.next_page_id != pages->next_page_id || + (!pages->frames.empty() && pages->backend_id != domain->backend_id) || + !allocation_bound(pages->frames.size())) { + return Status::invalid_argument(); + } + auto result = detail::reserved_shared<ChunkRootCommit>(metadata, ConstructionKey{}, metadata); + result->max_dependency_steps_ = max_dependency_steps; + result->window_ = std::move(window); + result->pages_ = std::move(pages); + result->domain_ = std::move(domain); + result->ids_ = std::move(ids); + result->root_.identity = identity; + result->root_.frame_count = result->pages_->frames.size(); + const auto indexes = (result->root_.frame_count + kChunkIndexFrameLimit - 1) / kChunkIndexFrameLimit; + result->root_.indexes.reserve(indexes); + result->index_pins_.reserve(indexes); + detail::ChunkId unused; + auto status = result->ids_->next(PagePurpose::kPageIndex, &result->candidate_, &unused); + if (!status.ok()) { + return status; + } + result->page_ = result->pages_->frames.begin(); + *out = std::move(result); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +std::optional<size_t> ChunkRootCommit::allocation_bound(size_t frames) noexcept +{ + using namespace detail; + if (frames > kMaxCandidateReferences) { + return {}; + } + const auto indexes = (frames + kChunkIndexFrameLimit - 1) / kChunkIndexFrameLimit; + const auto root_bytes = + AllocationBound(kChunkRootHeaderBytes + 4).add_product(indexes, kChunkRootEntryBytes).value(); + if (!root_bytes || *root_bytes > kMaxCandidateManifestBytes || indexes > kMaxCandidateReferences - frames) { + return {}; + } + return AllocationBound(shared_allocation_bound<ChunkRootCommit>()) + .add(*root_bytes) + + .add_product(frames + indexes, sizeof(ChunkStageId) + (8 * sizeof(void *))) + .add_product(indexes, sizeof(ChunkIndexReference) + sizeof(std::shared_ptr<const ChunkPackBacking>) + + chunk_backing_allocation_bound()) + .value(); +} + +std::optional<size_t> ChunkRootCommit::window_allocation_bound(size_t stored_references) noexcept +{ + const auto refs = std::min(stored_references, kChunkIndexFrameLimit * kChunkIndexReferencesPerFrame); + const auto bytes = kChunkIndexHeaderBytes + 4 + (kChunkIndexFrameLimit * kChunkIndexRecordBytes) + (16 * refs); + return detail::AllocationBound(bytes).add(detail::ChunkFixedPackStore::write_allocation_bound(bytes)).value(); +} + +bool ChunkRootCommit::retire_step() noexcept +{ + for (size_t i = 0; i < 64 && !references_.empty(); ++i) { + references_.erase(references_.begin()); + } + for (size_t i = 0; i < 64 && !index_pins_.empty(); ++i) { + index_pins_.pop_back(); + } + return references_.empty() && index_pins_.empty(); +} + +Status ChunkRootCommit::step(bool stop_requested) noexcept +try { + if (done()) { + return status_.without_message(); + } + if (stop_requested && stage_ != Stage::QueryCommit && stage_ != Stage::QueryBegin && + stage_ != Stage::QueryPrepare && stage_ != Stage::Abort && stage_ != Stage::QueryAbort && + stage_ != Stage::Release) { + return fail(Status::unavailable("Chunk checkpoint stopped before root commit")); + } + return advance(); +} +catch (const std::bad_alloc &) { + return fail(Status::resource_exhausted()); +} +catch (...) { + return fail(Status::internal_error()); +} + +Status ChunkRootCommit::advance() +{ + switch (stage_) { + case Stage::Index: + return index(); + case Stage::Dependencies: + return dependencies(); + case Stage::Checksum: + return checksum(); + case Stage::Begin: + return begin(); + case Stage::Prepare: + return prepare(); + case Stage::Commit: + return finish_candidate(true); + case Stage::Abort: + return finish_candidate(false); + case Stage::QueryBegin: + case Stage::QueryPrepare: + case Stage::QueryCommit: + case Stage::QueryAbort: + return query(); + case Stage::Release: + return release(); + case Stage::Done: + return status_.without_message(); + } + return finish(Status::internal_error()); +} + +Status ChunkRootCommit::fail(Status status) +{ + if (status_.ok()) { + status_ = std::move(status); + } + if (stage_ == Stage::QueryCommit || stage_ == Stage::QueryBegin || stage_ == Stage::QueryPrepare || + stage_ == Stage::QueryAbort || stage_ == Stage::Abort || stage_ == Stage::Release) { + return finish(status_.without_message(), true); + } + if (candidate_started_) { + stage_ = Stage::Abort; + return Status::Ok(); + } + return finish(status_.without_message()); +} + +Status ChunkRootCommit::finish(Status status, bool uncertain) +{ + status_ = std::move(status); + uncertain_ = uncertain; + stage_ = Stage::Done; + return status_.without_message(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_commit.h b/lib/crowdb-tree/src/snapshot/chunk_root_commit.h new file mode 100644 index 000000000..7ef738ceb --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_commit.h @@ -0,0 +1,123 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_runtime_ids.h" +#include "snapshot/captured_write.h" +#include "snapshot/chunk_root_codec.h" + +#include <set> + +namespace crowdb::tree +{ +// A backend-lane protocol. Each step handles at most one index pack or one +// bounded reference authority window. The caller yields between steps; no CPU task +// performs catalog waits, and an unknown mutation is queried by its exact ID. +class ChunkRootCommit +{ + struct ConstructionKey + { + }; + + public: + ChunkRootCommit(ConstructionKey, std::shared_ptr<detail::AllocationReservation> metadata); + static Status create(std::shared_ptr<const CapturedDurablePages> pages, + std::shared_ptr<const ChunkBackingDomain> domain, std::shared_ptr<detail::ChunkRuntimeIds> ids, + ChunkRootIdentity identity, std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<detail::AllocationReservation> window, std::shared_ptr<ChunkRootCommit> *out, + size_t max_dependency_steps = 1024 * 1024); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t frames) noexcept; + [[nodiscard]] static std::optional<size_t> window_allocation_bound(size_t stored_references) noexcept; + Status step(bool stop_requested = false) noexcept; + + [[nodiscard]] bool done() const noexcept + { + return stage_ == Stage::Done; + } + + [[nodiscard]] bool committed() const noexcept + { + return committed_; + } + + [[nodiscard]] bool needs_recovery() const noexcept + { + return uncertain_; + } + + [[nodiscard]] const Status &status() const noexcept + { + return status_; + } + + [[nodiscard]] const ChunkRootIdentity &identity() const noexcept + { + return root_.identity; + } + + [[nodiscard]] detail::ChunkStageId candidate() const noexcept + { + return candidate_; + } + + bool retire_step() noexcept; + + private: + enum class Stage : uint8_t { + Index, + Dependencies, + Checksum, + Begin, + Prepare, + Commit, + QueryBegin, + QueryPrepare, + QueryCommit, + Abort, + QueryAbort, + Release, + Done + }; + using References = std::set<detail::ChunkStageId, std::less<>, detail::ReservationAllocator<detail::ChunkStageId>>; + Status advance(); + Status index(); + Status checksum(); + Status dependencies(); + Status begin(); + Status prepare(); + Status finish_candidate(bool commit); + Status query(); + Status release(); + Status fail(Status status); + Status finish(Status status, bool uncertain = false); + Status query_result(const detail::ChunkCandidateStatus ¤t); + void accepted_window(); + + std::shared_ptr<detail::AllocationReservation> metadata_; + std::shared_ptr<detail::AllocationReservation> window_; + std::shared_ptr<const CapturedDurablePages> pages_; + std::shared_ptr<const ChunkBackingDomain> domain_; + std::shared_ptr<detail::ChunkRuntimeIds> ids_; + + ChunkRootManifest root_; + ChunkMetadataBytes manifest_; + detail::ReservedVector<std::shared_ptr<const ChunkPackBacking>> index_pins_; + References references_; + CapturedDurablePages::Frames::const_iterator page_; + References::const_iterator reference_; + detail::ChunkStageId candidate_{}; + std::array<detail::ChunkStageId, detail::kMaxReferenceWindow> pending_{}; + size_t pending_count_ = 0; + uint64_t prepared_ = 0; + uint64_t released_ = 0; + uint32_t checksum_ = 0; + size_t dependency_ = 0; + size_t max_dependency_steps_ = 0; + uint64_t chain_pid_ = kInvalidPageId; + uint64_t chain_bytes_ = 0; + Stage stage_ = Stage::Index; + bool candidate_started_ = false; + bool committed_ = false; + bool uncertain_ = false; + Status status_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_commit_dependencies.cpp b/lib/crowdb-tree/src/snapshot/chunk_root_commit_dependencies.cpp new file mode 100644 index 000000000..2fe9005c3 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_commit_dependencies.cpp @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/chunk_root_commit.h" + +namespace crowdb::tree +{ +Status ChunkRootCommit::dependencies() +{ + for (size_t work = 0; work < detail::kMaxStageScan && page_ != pages_->frames.end(); ++work) { + const auto &frame = page_->second; + if (frame.type != page_type::kLeafBase || dependency_ == frame.stored_overflow->size()) { + ++page_; + dependency_ = 0; + continue; + } + if (root_.dependency_entries == max_dependency_steps_) { + return fail(Status::resource_exhausted()); + } + if (chain_pid_ == kInvalidPageId) { + const auto &reference = (*frame.stored_overflow)[dependency_]; + chain_pid_ = reference.head; + chain_bytes_ = reference.bytes; + } + const auto found = pages_->frames.find(chain_pid_); + if (found == pages_->frames.end() || found->second.type != page_type::kOverflowFrame || + found->second.overflow_bytes == 0 || found->second.overflow_bytes > chain_bytes_ || + (found->second.next_overflow == kInvalidPageId) != (found->second.overflow_bytes == chain_bytes_)) { + return fail(Status::corruption("Chunk checkpoint overflow dependency is incomplete")); + } + ++root_.dependency_entries; + chain_bytes_ -= found->second.overflow_bytes; + chain_pid_ = found->second.next_overflow; + if (chain_bytes_ == 0) { + ++dependency_; + } + } + if (page_ == pages_->frames.end()) { + reference_ = references_.begin(); + stage_ = Stage::Checksum; + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_commit_finish.cpp b/lib/crowdb-tree/src/snapshot/chunk_root_commit_finish.cpp new file mode 100644 index 000000000..febf7bc99 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_commit_finish.cpp @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/chunk_root_commit.h" + +#include <algorithm> + +namespace crowdb::tree +{ +Status ChunkRootCommit::finish_candidate(bool commit) +{ + stage_ = commit ? Stage::QueryCommit : Stage::QueryAbort; + auto status = domain_->catalog->protection()->finish_candidate( + domain_->tree, candidate_, commit ? CT_CHUNK_CANDIDATE_COMMIT : CT_CHUNK_CANDIDATE_ABORT); + if (!status.ok()) { + if (status_.ok()) { + status_ = std::move(status); + } + return Status::Ok(); + } + if (commit) { + committed_ = true; + return finish(Status::Ok()); + } + stage_ = Stage::Release; + return Status::Ok(); +} + +Status ChunkRootCommit::query() +{ + std::optional<detail::ChunkCandidateStatus> current; + auto status = domain_->catalog->protection()->load_candidate(domain_->tree, candidate_, ¤t); + if (!status.ok()) { + return finish(std::move(status), true); + } + if (!current) { + return stage_ == Stage::QueryBegin ? finish(status_.without_message()) + : finish(Status::corruption("Chunk candidate disappeared"), true); + } + if (current->id != candidate_ || current->generation != root_.identity.root_generation || + current->reference_count != references_.size() || current->prepared_references > references_.size() || + current->prepared_windows > + (references_.size() + detail::kMaxReferenceWindow - 1) / detail::kMaxReferenceWindow || + current->released_windows > current->prepared_windows) { + return finish(Status::corruption("Chunk candidate identity changed"), true); + } + const auto prepared = + std::min<uint64_t>(references_.size(), current->prepared_windows * detail::kMaxReferenceWindow); + const bool all_prepared = current->prepared_references == references_.size(); + using Candidate = detail::ChunkCandidateState; + if (current->prepared_references != prepared || + ((current->state == Candidate::Ready || current->state == Candidate::Committed) && !all_prepared) || + (current->state == Candidate::Preparing && all_prepared) || + (current->state <= Candidate::Committed && current->released_windows != 0)) { + return finish(Status::corruption("Chunk candidate reference proof is inconsistent"), true); + } + return query_result(*current); +} + +Status ChunkRootCommit::query_result(const detail::ChunkCandidateStatus ¤t) +{ + using Candidate = detail::ChunkCandidateState; + if (current.state == Candidate::Committed) { + committed_ = true; + return finish(Status::Ok()); + } + if (current.state == Candidate::Aborted || current.state == Candidate::Reclaimed) { + prepared_ = current.prepared_windows; + released_ = current.released_windows; + if (status_.ok()) { + status_ = Status::unavailable("Chunk candidate was aborted"); + } + stage_ = Stage::Release; + return Status::Ok(); + } + if (stage_ == Stage::QueryBegin && current.prepared_windows == 0 && current.prepared_references == 0) { + status_ = Status::Ok(); + stage_ = references_.empty() ? Stage::Commit : Stage::Prepare; + return Status::Ok(); + } + const auto accepted = std::min<uint64_t>(references_.size(), (prepared_ + 1) * detail::kMaxReferenceWindow); + if (stage_ == Stage::QueryPrepare && current.prepared_windows == prepared_ + 1 && + current.prepared_references == accepted) { + accepted_window(); + return Status::Ok(); + } + // A successful linearizable query proves the failed mutation did not + // commit. Abort the same candidate; never generate a replacement ID here. + if (stage_ == Stage::QueryAbort) { + return finish(status_.without_message(), true); + } + prepared_ = current.prepared_windows; + released_ = current.released_windows; + stage_ = Stage::Abort; + return Status::Ok(); +} + +Status ChunkRootCommit::release() +{ + if (released_ == prepared_) { + return finish(status_.without_message()); + } + auto status = domain_->catalog->protection()->release_references(domain_->tree, candidate_, released_); + if (!status.ok()) { + return finish(std::move(status), true); + } + ++released_; + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_commit_index.cpp b/lib/crowdb-tree/src/snapshot/chunk_root_commit_index.cpp new file mode 100644 index 000000000..c48432454 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_commit_index.cpp @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/chunk_root_commit.h" + +namespace crowdb::tree +{ +Status ChunkRootCommit::index() +{ + if (page_ == pages_->frames.end()) { + page_ = pages_->frames.begin(); + stage_ = Stage::Dependencies; + return Status::Ok(); + } + std::array<const DurableCapturedFrame *, kChunkIndexFrameLimit> frames{}; + size_t count = 0; + auto next = page_; + while (next != pages_->frames.end() && count < frames.size()) { + if (next->first != next->second.page_id) { + return fail(Status::corruption("Chunk checkpoint page identity mismatch")); + } + frames[count++] = &next->second; + ++next; + } + ChunkMetadataBytes bytes{detail::ReservationAllocator<uint8_t>(window_)}; + auto status = encode_chunk_index(root_.identity, root_.indexes.size(), std::span(frames).first(count), &bytes); + if (!status.ok()) { + return fail(std::move(status)); + } + detail::ChunkStageId id; + detail::ChunkId chunk; + status = ids_->next(PagePurpose::kPageIndex, &id, &chunk); + if (!status.ok()) { + return fail(std::move(status)); + } + std::shared_ptr<ChunkPackBacking> backing; + status = prepare_chunk_backing(domain_, id, chunk, static_cast<uint32_t>(bytes.size()), PagePurpose::kPageIndex, + metadata_, &backing); + if (!status.ok()) { + return fail(std::move(status)); + } + status = write_chunk_backing(*backing, bytes, window_); + if (!status.ok()) { + return fail(std::move(status)); + } + for (size_t i = 0; i < count; ++i) { + references_.insert(frames[i]->backing->chunk->pack.stage); + root_.stored_references += frames[i]->stored_overflow ? frames[i]->stored_overflow->size() : 0; + } + if (root_.stored_references > detail::kMaxCandidateReferences) { + return fail(Status::resource_exhausted()); + } + references_.insert(id); + root_.indexes.push_back({.first_page = frames[0]->page_id, + .last_page = frames[count - 1]->page_id, + .frame_count = count, + .pack = backing->pack}); + index_pins_.push_back(std::move(backing)); + page_ = next; + return Status::Ok(); +} + +Status ChunkRootCommit::checksum() +{ + std::array<detail::ChunkStageId, detail::kMaxStageScan> window{}; + size_t count = 0; + while (reference_ != references_.end() && count < window.size()) { + window[count++] = *reference_++; + } + checksum_ = detail::chunk_reference_checksum(checksum_, std::span(window).first(count)); + if (reference_ != references_.end()) { + return Status::Ok(); + } + auto status = encode_chunk_root(root_, &manifest_); + if (!status.ok()) { + return fail(std::move(status)); + } + reference_ = references_.begin(); + stage_ = Stage::Begin; + return Status::Ok(); +} + +Status ChunkRootCommit::begin() +{ + const detail::ChunkManifestCandidate candidate{.id = candidate_, + .expected_generation = root_.identity.root_generation - 1, + .reference_count = references_.size(), + .reference_checksum = checksum_, + .manifest = manifest_}; + auto *catalog = domain_->catalog->protection(); + auto status = catalog->check_read_owner(); + if (!status.ok()) { + return fail(std::move(status)); + } + candidate_started_ = true; + stage_ = Stage::QueryBegin; + status = catalog->begin_candidate(domain_->tree, candidate); + if (!status.ok()) { + status_ = std::move(status); + return Status::Ok(); + } + stage_ = references_.empty() ? Stage::Commit : Stage::Prepare; + return Status::Ok(); +} + +Status ChunkRootCommit::prepare() +{ + pending_count_ = 0; + auto next = reference_; + while (next != references_.end() && pending_count_ < pending_.size()) { + pending_[pending_count_++] = *next++; + } + stage_ = Stage::QueryPrepare; + auto status = domain_->catalog->protection()->prepare_references(domain_->tree, candidate_, prepared_, + std::span(pending_).first(pending_count_)); + if (!status.ok()) { + status_ = std::move(status); + return Status::Ok(); + } + accepted_window(); + return Status::Ok(); +} + +void ChunkRootCommit::accepted_window() +{ + for (size_t i = 0; i < pending_count_; ++i) { + ++reference_; + } + ++prepared_; + pending_count_ = 0; + status_ = Status::Ok(); + stage_ = reference_ == references_.end() ? Stage::Commit : Stage::Prepare; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_recovery.cpp b/lib/crowdb-tree/src/snapshot/chunk_root_recovery.cpp new file mode 100644 index 000000000..3aabdc95e --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_recovery.cpp @@ -0,0 +1,144 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/chunk_root_recovery.h" + +#include "backend/chunk/chunk_physical_registry.h" +#include "btree/allocation_bound.h" + +namespace crowdb::tree +{ +ChunkRootRecovery::ChunkRootRecovery(ConstructionKey /*unused*/, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<detail::AllocationReservation> window) + : metadata_(std::move(metadata)), + window_(std::move(window)), + root_(metadata_), + index_bytes_(detail::ReservationAllocator<uint8_t>(window_)), + blob_(detail::ReservationAllocator<uint8_t, AllocationKind::kIo>(window_)), + raw_(detail::ReservationAllocator<uint8_t, AllocationKind::kScratch>(window_)) +{ +} + +std::optional<size_t> ChunkRootRecovery::allocation_bound(const ChunkRootManifest &root) noexcept +{ + using namespace detail; + constexpr size_t node_links = 4 * sizeof(void *); + return AllocationBound(shared_allocation_bound<ChunkRootRecovery>() + + shared_allocation_bound<CapturedDurablePages>()) + .add(chunk_pin_validation_bound()) + .add_product(root.indexes.size(), sizeof(ChunkIndexReference) + chunk_backing_allocation_bound()) + .add_product(root.frame_count, + sizeof(CapturedDurablePages::Entry) + node_links + (2 * shared_allocation_bound<BackingRef>()) + + chunk_backing_allocation_bound() + shared_allocation_bound<StoredOverflowReferences>() + + shared_allocation_bound<OverflowBackingSet>() + shared_allocation_bound<LeafProtection>()) + .add_product(root.stored_references, sizeof(StoredOverflowReference)) + .add_product(root.dependency_entries, sizeof(OverflowBackingSet::Entry) + node_links) + .value(); +} + +std::optional<size_t> ChunkRootRecovery::window_allocation_bound(ChunkRecoveryLimits limits) noexcept +{ + using namespace detail; + return AllocationBound(kChunkIndexWindowBytes) + .add(limits.max_blob_bytes) + .add(limits.max_frame_bytes) + .add(chunk_backing_allocation_bound()) + .add(chunk_pin_validation_bound()) + .add(ChunkFixedPackStore::read_into_allocation_bound(std::max(kChunkIndexWindowBytes, limits.max_blob_bytes))) + .value(); +} + +Status ChunkRootRecovery::create(ChunkRootManifest root, std::shared_ptr<const ChunkBackingDomain> domain, + std::shared_ptr<detail::ChunkRuntimeIds> ids, ChunkRecoveryLimits limits, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<detail::AllocationReservation> window, + std::shared_ptr<ChunkRootRecovery> *out) +try { + if (!domain || !domain->supported() || !ids || !metadata || !window || out == nullptr || *out || + root.identity.tree != domain->tree || root.identity.root_generation == 0 || + limits.max_frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || limits.max_blob_bytes < kDurableBlobHeader) { + return Status::invalid_argument(); + } + if (root.frame_count > limits.max_pages || root.stored_references > limits.max_references || + root.dependency_entries > limits.max_dependency_steps) { + return Status::resource_exhausted(); + } + auto result = detail::reserved_shared<ChunkRootRecovery>(metadata, ConstructionKey{}, metadata, window); + result->domain_ = std::move(domain); + result->ids_ = std::move(ids); + result->limits_ = limits; + result->root_ = std::move(root); + result->pages_ = detail::reserved_shared<CapturedDurablePages>(metadata, metadata); + const auto &identity = result->root_.identity; + auto &pages = *result->pages_; + pages.root_page_id = identity.root_page_id; + pages.next_page_id = identity.next_page_id; + pages.coverage = identity.coverage; + pages.generation = identity.page_generation; + pages.backend_id = result->domain_->backend_id; + // Fixed capacities eliminate vector growth overlap across later windows. + result->index_bytes_.reserve(kChunkIndexWindowBytes); + result->blob_.reserve(limits.max_blob_bytes); + result->raw_.reserve(limits.max_frame_bytes); + *out = std::move(result); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted(); +} + +Status ChunkRootRecovery::next_pin(detail::ChunkStageId *out) +{ + detail::ChunkId unused; + return ids_->next(PagePurpose::kBtreePage, out, &unused); +} + +Status ChunkRootRecovery::step(bool stopped) noexcept +try { + if (done_) { + return status_; + } + if (stopped) { + return finish(Status::unavailable()); + } + auto status = domain_->catalog->protection()->check_read_owner(); + if (status.ok()) { + if (dependency_phase_) { + status = dependencies(); + } + else if (left_ != 0) { + status = frame(); + } + else { + status = index(); + } + } + if (!status.ok()) { + return finish(std::move(status)); + } + return status; +} +catch (const std::bad_alloc &) { + return finish(Status::resource_exhausted()); +} +catch (...) { + return finish(Status::internal_error()); +} + +Status ChunkRootRecovery::finish(Status status) +{ + status_ = std::move(status); + done_ = true; + if (!status_.ok()) { + pages_.reset(); + } + return status_; +} + +bool ChunkRootRecovery::retire_step() noexcept +{ + // The catalog owns its own bounded retirement; no synchronous map walk. + pages_.reset(); + return true; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_recovery.h b/lib/crowdb-tree/src/snapshot/chunk_root_recovery.h new file mode 100644 index 000000000..603715d0c --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_recovery.h @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_runtime_ids.h" +#include "btree/overflow_backing.h" +#include "snapshot/captured_write.h" +#include "snapshot/chunk_root_codec.h" + +namespace crowdb::tree +{ +struct ChunkRecoveryLimits +{ + size_t max_pages = 1024 * 1024; + size_t max_frame_bytes = 1024 * 1024; + size_t max_blob_bytes = 2 * 1024 * 1024; + size_t max_references = 1024 * 1024; + size_t max_dependency_steps = 1024 * 1024; +}; + +// One exact index, frame, or bounded dependency slice per backend step. The +// root is loaded first; its validated counts determine the admission bound. +// A successful result still requires RecoveredStateOperation graph validation. +class ChunkRootRecovery +{ + struct ConstructionKey + { + }; + + public: + ChunkRootRecovery(ConstructionKey, std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<detail::AllocationReservation> window); + static Status create(ChunkRootManifest root, std::shared_ptr<const ChunkBackingDomain> domain, + std::shared_ptr<detail::ChunkRuntimeIds> ids, ChunkRecoveryLimits limits, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<detail::AllocationReservation> window, + std::shared_ptr<ChunkRootRecovery> *out); + [[nodiscard]] static std::optional<size_t> allocation_bound(const ChunkRootManifest &root) noexcept; + [[nodiscard]] static std::optional<size_t> window_allocation_bound(ChunkRecoveryLimits limits) noexcept; + Status step(bool stopped = false) noexcept; + + [[nodiscard]] bool done() const noexcept + { + return done_; + } + + [[nodiscard]] const Status &status() const noexcept + { + return status_; + } + + [[nodiscard]] const std::shared_ptr<CapturedDurablePages> &pages() const noexcept + { + return pages_; + } + + [[nodiscard]] size_t max_frame_bytes() const noexcept + { + return max_frame_bytes_; + } + + [[nodiscard]] size_t max_blob_bytes() const noexcept + { + return max_blob_bytes_; + } + + bool retire_step() noexcept; + + private: + struct LeafProtection + { + std::shared_ptr<const void> primary; + std::shared_ptr<const OverflowBackingSet> overflow; + }; + + Status index(); + Status frame(); + Status validate_frame(DurableCapturedFrame &descriptor, std::span<const uint8_t> words); + Status dependencies(); + Status next_pin(detail::ChunkStageId *out); + Status finish(Status status); + std::shared_ptr<detail::AllocationReservation> metadata_; + std::shared_ptr<detail::AllocationReservation> window_; + std::shared_ptr<const ChunkBackingDomain> domain_; + std::shared_ptr<detail::ChunkRuntimeIds> ids_; + ChunkRecoveryLimits limits_; + + ChunkRootManifest root_; + std::shared_ptr<CapturedDurablePages> pages_; + ChunkMetadataBytes index_bytes_; + detail::ReservedVector<uint8_t, AllocationKind::kIo> blob_; + detail::ReservedVector<uint8_t, AllocationKind::kScratch> raw_; + size_t max_frame_bytes_ = 0; + size_t max_blob_bytes_ = 0; + size_t index_ = 0; + size_t offset_ = 0; + uint64_t left_ = 0; + uint64_t last_pid_ = 0; + uint64_t references_ = 0; + uint64_t steps_ = 0; + bool have_pid_ = false; + bool dependency_phase_ = false; + bool done_ = false; + Status status_; + CapturedDurablePages::Frames::iterator leaf_; + size_t reference_ = 0; + uint64_t pid_ = kInvalidPageId; + uint64_t remaining_ = 0; + std::shared_ptr<OverflowBackingSet> overflow_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_recovery_dependencies.cpp b/lib/crowdb-tree/src/snapshot/chunk_root_recovery_dependencies.cpp new file mode 100644 index 000000000..1ac0138d5 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_recovery_dependencies.cpp @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/chunk_root_recovery.h" + +namespace crowdb::tree +{ +Status ChunkRootRecovery::dependencies() +{ + for (size_t work = 0; work < 64; ++work) { + if (leaf_ == pages_->frames.end()) { + return finish(steps_ == root_.dependency_entries ? Status::Ok() : Status::corruption()); + } + auto &leaf = leaf_->second; + if (leaf.type != page_type::kLeafBase || reference_ == leaf.stored_overflow->size()) { + ++leaf_; + reference_ = 0; + overflow_.reset(); + continue; + } + if (steps_ == root_.dependency_entries || steps_ == limits_.max_dependency_steps) { + return Status::corruption(); + } + if (!overflow_) { + overflow_ = detail::reserved_shared<OverflowBackingSet>(metadata_, metadata_); + auto owner = detail::reserved_shared<LeafProtection>(metadata_); + owner->primary = leaf.backing->physical_owner; + owner->overflow = overflow_; + auto backing = detail::reserved_shared<BackingRef>(metadata_, *leaf.backing); + backing->owner = std::move(owner); + backing->overflow = overflow_; + leaf.backing = std::move(backing); + } + if (pid_ == kInvalidPageId) { + const auto &ref = (*leaf.stored_overflow)[reference_]; + pid_ = ref.head; + remaining_ = ref.bytes; + } + const auto found = pages_->frames.find(pid_); + if (found == pages_->frames.end()) { + return Status::corruption(); + } + const auto &frame = found->second; + if (frame.type != page_type::kOverflowFrame || frame.overflow_bytes == 0 || frame.overflow_bytes > remaining_ || + (frame.next_overflow == kInvalidPageId) != (frame.overflow_bytes == remaining_)) { + return Status::corruption(); + } + overflow_->pages.emplace(pid_, OverflowBackingPage{.backing = frame.backing, + .next = frame.next_overflow, + .bytes = frame.overflow_bytes, + .frame_bytes = frame.frame_bytes, + .logical_bytes = frame.logical_bytes}); + ++steps_; + remaining_ -= frame.overflow_bytes; + pid_ = frame.next_overflow; + if (remaining_ == 0) { + ++reference_; + } + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/chunk_root_recovery_frames.cpp b/lib/crowdb-tree/src/snapshot/chunk_root_recovery_frames.cpp new file mode 100644 index 000000000..3aec2f6b3 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/chunk_root_recovery_frames.cpp @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/chunk_root_codec_internal.h" +#include "snapshot/chunk_root_recovery.h" +#include "snapshot/stored_overflow.h" + +namespace crowdb::tree +{ +Status ChunkRootRecovery::index() +{ + if (index_ == root_.indexes.size()) { + if (pages_->frames.size() != root_.frame_count || references_ != root_.stored_references) { + return Status::corruption(); + } + leaf_ = pages_->frames.begin(); + dependency_phase_ = true; + return Status::Ok(); + } + const auto &expected = root_.indexes[index_]; + if (expected.pack.length > kChunkIndexWindowBytes) { + return Status::corruption(); + } + detail::ChunkStageId pin; + auto status = next_pin(&pin); + if (!status.ok()) { + return status; + } + std::shared_ptr<const ChunkPackBacking> backing; + status = pin_chunk_backing(domain_, expected.pack, pin, metadata_, &backing); + if (!status.ok()) { + return status; + } + index_bytes_.resize(expected.pack.length); + status = read_chunk_backing(*backing, index_bytes_, window_); + if (!status.ok()) { + return status; + } + status = decode_chunk_index_header(index_bytes_, root_.identity, index_, &left_, &offset_); + if (!status.ok()) { + return status; + } + if (left_ != expected.frame_count) { + return Status::corruption(); + } + return Status::Ok(); +} + +Status ChunkRootRecovery::frame() +{ + ChunkIndexFrame record; + auto status = decode_chunk_index_frame(index_bytes_, &offset_, &record); + if (!status.ok()) { + return status; + } + const auto &expected = root_.indexes[index_]; + if ((have_pid_ && record.page_id <= last_pid_) || record.page_id >= root_.identity.next_page_id || + (left_ == expected.frame_count && record.page_id != expected.first_page) || + (left_ == 1 && (record.page_id != expected.last_page || offset_ != index_bytes_.size() - 4))) { + return Status::corruption(); + } + if (record.pack.length > limits_.max_blob_bytes || record.frame_bytes > limits_.max_frame_bytes) { + return Status::resource_exhausted(); + } + detail::ChunkStageId pin; + status = next_pin(&pin); + if (!status.ok()) { + return status; + } + std::shared_ptr<const ChunkPackBacking> pack; + status = pin_chunk_backing(domain_, record.pack, pin, metadata_, &pack); + if (!status.ok()) { + return status; + } + auto backing = detail::reserved_shared<BackingRef>(metadata_); + backing->backend_id = domain_->backend_id; + backing->length = record.pack.length; + backing->generation = domain_->epoch; + backing->checksum = record.pack.checksum; + backing->durability = BackingDurability::kCheckpointCommitted; + backing->storage = BackingStorage::kChunkPack; + backing->owner = pack; + backing->physical_owner = pack; + backing->physical_metrics = pack->physical_metrics; + backing->chunk = std::move(pack); + DurableCapturedFrame descriptor{.page_id = record.page_id, + .type = record.type, + .frame_bytes = record.frame_bytes, + .logical_bytes = record.logical_bytes, + .next_overflow = record.next_overflow, + .overflow_bytes = record.overflow_bytes, + .backing = std::move(backing), + .stored_overflow = {}, + .summary = {}}; + blob_.resize(record.pack.length); + status = read_chunk_backing(*descriptor.backing->chunk, blob_, window_); + if (!status.ok()) { + return status; + } + status = validate_frame(descriptor, record.stored_reference_words); + if (!status.ok()) { + return status; + } + pages_->frames.emplace(record.page_id, std::move(descriptor)); + max_frame_bytes_ = std::max<size_t>(max_frame_bytes_, record.frame_bytes); + max_blob_bytes_ = std::max<size_t>(max_blob_bytes_, record.pack.length); + last_pid_ = record.page_id; + have_pid_ = true; + if (--left_ == 0) { + ++index_; + } + return Status::Ok(); +} + +Status ChunkRootRecovery::validate_frame(DurableCapturedFrame &descriptor, std::span<const uint8_t> words) +{ + if (durable_blob_raw_len(blob_.data(), blob_.size()) != descriptor.frame_bytes || + durable_blob_logical_len(blob_.data(), blob_.size()) != descriptor.logical_bytes) { + return Status::corruption(); + } + raw_.resize(descriptor.frame_bytes); + auto status = decode_durable_page(blob_.data(), blob_.size(), raw_.data(), raw_.size()); + if (!status.ok()) { + return status; + } + if (!frame_validate(raw_.data(), raw_.size()) || frame_u64(raw_.data(), fh::kSelfpage_id) != descriptor.page_id || + frame_page_type(raw_.data()) != descriptor.type) { + return Status::corruption(); + } + if (descriptor.type == page_type::kOverflowFrame) { + const OverflowFrameView overflow(raw_.data(), raw_.size()); + return overflow.chunk_len() == descriptor.overflow_bytes && overflow.next_page_id() == descriptor.next_overflow + ? Status::Ok() + : Status::corruption(); + } + if (descriptor.type != page_type::kLeafBase) { + return Status::Ok(); + } + const LeafFrameView leaf(raw_.data(), raw_.size()); + const uint64_t cells = uint64_t{leaf.count()} + leaf.delta_count(); + size_t stored = 0; + for (uint64_t i = 0; i < cells; ++i) { + const auto raw = i < leaf.count() ? leaf.cell(static_cast<uint32_t>(i)) + : leaf.delta_cell(static_cast<uint32_t>(i - leaf.count())); + const CellView cell(raw); + if (!cell.valid() || cell.slot() > pages_->coverage || + (cell.flags() != 0 && cell.flags() != kFlagTombstone && cell.flags() != kFlagOverflow) || + (cell.is_tombstone() && raw.size() != kCellHeaderSize)) { + return Status::corruption(); + } + stored += cell.is_overflow() ? 1 : 0; + } + if (stored != words.size() / 16 || stored > root_.stored_references - references_) { + return Status::corruption(); + } + status = capture_stored_overflow(leaf, kChunkIndexReferencesPerFrame, metadata_, &descriptor.stored_overflow); + if (!status.ok()) { + return status; + } + const auto &refs = *descriptor.stored_overflow; + if (refs.size() != words.size() / 16 || refs.size() > root_.stored_references - references_) { + return Status::corruption(); + } + for (size_t i = 0; i < refs.size(); ++i) { + const auto encoded = words.subspan(i * 16, 16); + if (refs[i].head != chunk_codec::word(encoded, 0) || refs[i].bytes != chunk_codec::word(encoded, 1)) { + return Status::corruption(); + } + } + references_ += refs.size(); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/commit_anchor.cpp b/lib/crowdb-tree/src/snapshot/commit_anchor.cpp index 02b1babb8..c69eaea3c 100644 --- a/lib/crowdb-tree/src/snapshot/commit_anchor.cpp +++ b/lib/crowdb-tree/src/snapshot/commit_anchor.cpp @@ -11,7 +11,7 @@ namespace crowdb::tree::snapshot { namespace { -constexpr size_t kAnchorFixedFields = 4 + 4 + (8 * 4) + 4 + 8 + 4 + 4 + 4 + 8 + 8 + 4; +constexpr size_t kAnchorFixedFields = 4 + 4 + (8 * 4) + 4 + 8 + 4 + 4 + 8 + 8 + 4; void put_u32(std::vector<uint8_t> *out, uint32_t v) { @@ -66,23 +66,22 @@ void encode_anchor(const CommitAnchor &anchor, uint64_t slot_bytes, std::vector< out->resize(slot_bytes, 0); // zero pad to the IU-aligned slot size } -namespace -{ -bool decode_anchor(const uint8_t *buf, CommitAnchor *a) +bool decode_anchor(std::span<const uint8_t> input, CommitAnchor *out) { - if (get_u32(buf) != kAnchorMagic) { + if (input.size() < kAnchorFixedFields || out == nullptr) { return false; } - uint32_t stored_crc = get_u32(buf + (kAnchorFixedFields - 4)); - if (crowdb::common::crc32c(buf, kAnchorFixedFields - 4) != stored_crc) { + const auto *buf = input.data(); + if (get_u32(buf) != kAnchorMagic || get_u32(buf + 4) != kFormatVersion) { return false; } - a->magic = get_u32(buf); - a->format_version = get_u32(buf + 4); - if (a->format_version != kFormatVersion) { - // Clean-break format: no older format to accept. + uint32_t stored_crc = get_u32(buf + (kAnchorFixedFields - 4)); + if (crowdb::common::crc32c(buf, kAnchorFixedFields - 4) != stored_crc) { return false; } + auto *a = out; + a->magic = get_u32(buf); + a->format_version = get_u32(buf + 4); a->snapshot_seq = get_u64(buf + 8); a->root_page_id = get_u64(buf + 16); a->last_applied_slot = get_u64(buf + 24); @@ -95,7 +94,6 @@ bool decode_anchor(const uint8_t *buf, CommitAnchor *a) a->inner_count = get_u64(buf + 68); return true; } -} // namespace Status read_valid_anchors(const PageStore &store, uint32_t iu, std::vector<CommitAnchor> *out) { @@ -115,7 +113,7 @@ Status read_valid_anchors(const PageStore &store, uint32_t iu, std::vector<Commi return status; } CommitAnchor a; - if (!decode_anchor(buf.data(), &a)) { + if (!decode_anchor(buf, &a)) { continue; } out->push_back(a); diff --git a/lib/crowdb-tree/src/snapshot/commit_anchor.h b/lib/crowdb-tree/src/snapshot/commit_anchor.h index d169602ea..95198d0bb 100644 --- a/lib/crowdb-tree/src/snapshot/commit_anchor.h +++ b/lib/crowdb-tree/src/snapshot/commit_anchor.h @@ -5,6 +5,7 @@ #include "crowdb-tree/backend/block_page_store.h" #include "crowdb-tree/backend/page_store.h" +#include <span> #include <vector> namespace crowdb::tree::snapshot @@ -34,7 +35,7 @@ struct CommitAnchor uint64_t root_page_id = 0; uint64_t last_applied_slot = 0; uint64_t next_page_id = 0; - uint32_t segment_slots = 0; // MappingTable::kSegmentSize at persist time (format guard) + uint32_t segment_slots = 0; // kMappingSegmentSize at persist time (format guard) uint64_t segdir_addr = 0; uint32_t segdir_len = 0; uint32_t segdir_crc = 0; @@ -42,6 +43,9 @@ struct CommitAnchor uint64_t inner_count = 0; // live inner pages (O(1) gauge, restored on open) }; -void encode_anchor(const CommitAnchor &anchor, uint64_t slot_bytes, std::vector<uint8_t> *out); +void encode_anchor(const CommitAnchor &anchor, uint64_t slot_bytes, std::vector<uint8_t> *out); +// Allocation-free validation of the fixed fields; trailing IU padding is +// ignored. Returns false for short input/null output and leaves output intact. +bool decode_anchor(std::span<const uint8_t> input, CommitAnchor *out); Status read_valid_anchors(const PageStore &store, uint32_t iu, std::vector<CommitAnchor> *out); } // namespace crowdb::tree::snapshot diff --git a/lib/crowdb-tree/src/snapshot/graph_summary.cpp b/lib/crowdb-tree/src/snapshot/graph_summary.cpp new file mode 100644 index 000000000..7e635ba39 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/graph_summary.cpp @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/graph_summary.h" + +namespace crowdb::tree +{ +namespace +{ +bool add(uint64_t &value, uint64_t amount) noexcept +{ + if (amount > UINT64_MAX - value) { + return false; + } + value += amount; + return true; +} +} // namespace + +bool add_graph_summary(TreeSummary &summary, const ReadablePageFrame &frame, + const FinalPageSummary *verified_leaf) noexcept +{ + auto next = summary; + if (frame.frame_bytes == 0 || !add(next.reachable_page_capacity_bytes, frame.frame_bytes)) { + return false; + } + switch (frame.type) { + case page_type::kLeafBase: { + const auto *leaf = verified_leaf; + if (leaf == nullptr && frame.summary) { + leaf = &*frame.summary; + } + if (leaf == nullptr || !leaf->live_counts_known || !add(next.reachable_leaf_pages, 1) || + !add(next.live_kv, leaf->live_keys) || !add(next.live_key_bytes, leaf->live_key_bytes) || + !add(next.live_value_bytes, leaf->live_value_bytes) || + !add(next.live_logical_bytes, leaf->live_key_bytes) || + !add(next.live_logical_bytes, leaf->live_value_bytes)) { + return false; + } + break; + } + case page_type::kInnerBase: + if (!add(next.reachable_inner_pages, 1)) { + return false; + } + break; + case page_type::kOverflowFrame: + if (!add(next.reachable_overflow_pages, 1)) { + return false; + } + break; + default: + return false; + } + summary = next; + return true; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/graph_summary.h b/lib/crowdb-tree/src/snapshot/graph_summary.h new file mode 100644 index 000000000..f7e42811f --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/graph_summary.h @@ -0,0 +1,13 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/readable_page_catalog.h" + +namespace crowdb::tree +{ +// The caller visits each reachable PID exactly once. Recovery supplies its +// validated leaf counts; writeback uses the encoder's receipt. No I/O occurs. +[[nodiscard]] bool add_graph_summary(TreeSummary &summary, const ReadablePageFrame &frame, + const FinalPageSummary *verified_leaf = nullptr) noexcept; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp new file mode 100644 index 000000000..4fa870572 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp @@ -0,0 +1,173 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "snapshot/local_checkpoint_internal.h" + +#include <algorithm> +#include <bit> +#include <cassert> + +namespace crowdb::tree +{ +LocalCheckpointOperation::State::Window::Window(size_t size, size_t alignment, + const std::shared_ptr<detail::AllocationReservation> &budget) + : storage(size + alignment - 1, detail::ReservationAllocator<uint8_t, AllocationKind::kIo>(budget)) +{ + void *data = storage.data(); + auto capacity = storage.size(); + auto *aligned = std::align(alignment, size, data, capacity); + assert(aligned != nullptr); + bytes = {static_cast<uint8_t *>(aligned), size}; +} + +LocalCheckpointOperation::State::State(PageLoadBudget credits) + : budget(std::move(credits)), + directory(detail::ReservationAllocator<DirEntry>(budget.bytes)) +{ +} + +Status LocalCheckpointOperation::create(std::shared_ptr<const CapturedDurablePages> pages, + std::shared_ptr<LocalCheckpointRegistry> registry, + std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, LocalCheckpointLimits limits, + PageLoadBudget budget, std::shared_ptr<LocalCheckpointOperation> *out) +{ + if (!pages || !registry || !store || !runtime || !lane || out == nullptr || limits.max_segments == 0 || + limits.max_segments > kMaxMappingSegments || static_cast<bool>(budget.bytes) != static_cast<bool>(budget.io) || + (budget.bytes && + ((budget.bytes->category() != detail::BudgetClass::kRouting && + budget.bytes->category() != detail::BudgetClass::kReclaim) || + budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + return Status::invalid_argument(); + } + if (!store->write_completion_is_reloadable()) { + return Status::not_supported(); + } + try { + auto state = detail::reserved_shared<State>(budget.bytes, budget); + state->limits = limits; + state->registry = std::move(registry); + state->store = std::move(store); + state->runtime = std::move(runtime); + state->lane = std::move(lane); + state->image = detail::reserved_shared<ProtectedCheckpoint>(budget.bytes, budget.bytes); + state->image->pages = std::move(pages); + *out = detail::reserved_construct<LocalCheckpointOperation>(budget.bytes, [&](auto *storage) { + ::new (static_cast<void *>(storage)) LocalCheckpointOperation(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +std::optional<size_t> LocalCheckpointOperation::allocation_bound(size_t segments, uint32_t iu) noexcept +{ + using namespace detail; + if (iu == 0 || segments > kMaxMappingSegments) { + return {}; + } + const auto bytes = round_up_to_iu(segment_image_encoded_size(kMappingSegmentSize), iu); + return AllocationBound(shared_allocation_bound<State>() + shared_allocation_bound<LocalCheckpointOperation>() + + ProtectedCheckpoint::allocation_bound() + shared_allocation_bound<State::Window>() + + shared_allocation_bound<State::IoStep>()) + .add(std::max<uint64_t>(bytes, snapshot::superblock_slot_bytes(iu))) + .add(std::bit_ceil(std::max<size_t>(4096, iu)) - 1) + .add_product(3 * (segments + 1), sizeof(DirEntry) + sizeof(ProtectedExtent)) + .add_product(segments + 1, ExtentLedger::reservation_metadata_bound()) + .value(); +} + +detail::CallbackSender LocalCheckpointOperation::sender() noexcept +{ + return {this, &LocalCheckpointOperation::submit}; +} + +const LocalCheckpointResult &LocalCheckpointOperation::result() const noexcept +{ + return state_->report; +} + +void LocalCheckpointOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void LocalCheckpointOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<LocalCheckpointOperation *>(source)->state_->start(complete, context); +} + +void LocalCheckpointOperation::State::start(detail::CallbackComplete callback, void *argument) noexcept +{ + uint8_t idle = 0; + if (!lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + callback(argument, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + complete = callback; + context = argument; + lifecycle.store(2, std::memory_order_release); + wake(); +} + +void LocalCheckpointOperation::State::begin() +{ + const auto &pages = *image->pages; + constexpr auto page_limit = kMappingSegmentSize * kMaxMappingSegments; + if (pages.next_page_id > page_limit || + (pages.backend_id != registry->ledger_->limits().backend_id && !pages.frames.empty()) || + (pages.frames.empty() ? pages.root_page_id != kInvalidPageId : !pages.frames.contains(pages.root_page_id))) { + finish(Status::invalid_argument()); + return; + } + auto status = registry->begin(pages.coverage, pages.generation, &image->anchor.snapshot_seq); + if (!status.ok()) { + finish(std::move(status)); + return; + } + admitted = true; + iu = registry->ledger_->limits().iu_size; + segment_bytes = round_up_to_iu(segment_image_encoded_size(kMappingSegmentSize), iu); + window = detail::reserved_shared<Window>(budget.bytes, + std::max<uint64_t>(segment_bytes, snapshot::superblock_slot_bytes(iu)), + std::bit_ceil(std::max<size_t>(4096, iu)), budget.bytes); + auto &anchor = image->anchor; + anchor.magic = snapshot::kAnchorMagic; + anchor.format_version = snapshot::kFormatVersion; + anchor.root_page_id = pages.root_page_id; + anchor.last_applied_slot = pages.coverage; + anchor.next_page_id = pages.next_page_id; + anchor.segment_slots = kMappingSegmentSize; + position = pages.frames.begin(); + stage = Stage::Segment; + wake(); +} + +void LocalCheckpointOperation::State::finish(Status status) noexcept +{ + if (lifecycle.exchange(3, std::memory_order_acq_rel) == 3) { + return; + } + const bool cancelled = !anchor_touched && stopped.load(std::memory_order_acquire); + report.status = cancelled ? Status::unavailable() : std::move(status); + if (admitted) { + registry->finish(image, report.status.ok()); + } + if (report.status.ok()) { + report.checkpoint = std::move(image); + } + image.reset(); + window.reset(); + reserved.owner.reset(); + store.reset(); + auto signal = report.status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + if (cancelled) { + signal = detail::CallbackSignal::kStopped; + } + complete(context, signal, report.status.without_message()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint.h b/lib/crowdb-tree/src/snapshot/local_checkpoint.h new file mode 100644 index 000000000..07c85913e --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint.h @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/local/extent_ledger.h" +#include "snapshot/captured_write.h" +#include "snapshot/commit_anchor.h" + +#include <array> + +namespace crowdb::tree +{ +struct CheckpointMetadata +{ + explicit CheckpointMetadata(const std::shared_ptr<detail::AllocationReservation> &budget) + : extents(detail::ReservationAllocator<ProtectedExtent>(budget)) + { + } + + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !extents.empty(); ++i) { + extents.pop_back(); + } + return extents.empty(); + } + + detail::ReservedVector<ProtectedExtent> extents; +}; + +struct ProtectedCheckpoint +{ + explicit ProtectedCheckpoint(const std::shared_ptr<detail::AllocationReservation> &budget) + : metadata(detail::reserved_shared<CheckpointMetadata>(budget, budget)) + { + } + + bool retire_step() noexcept + { + return true; + } + + [[nodiscard]] static size_t allocation_bound() noexcept; + + snapshot::CommitAnchor anchor; + std::shared_ptr<const CapturedDurablePages> pages; + std::shared_ptr<CheckpointMetadata> metadata; +}; + +class LocalCheckpointOperation; + +// One registry per local allocation domain, retained for the domain lifetime. +// Both valid anchors pin their complete extent sets. An uncertain anchor write +// additionally pins its candidate and prevents another commit until recovery +// reconstructs a new registry from durable storage. +class LocalCheckpointRegistry +{ + public: + static Status create(std::shared_ptr<ExtentLedger> ledger, std::shared_ptr<detail::AllocationReservation> metadata, + std::array<std::shared_ptr<const ProtectedCheckpoint>, 2> recovered, + std::shared_ptr<LocalCheckpointRegistry> *out); + + [[nodiscard]] uint64_t durable_coverage() const noexcept + { + return coverage_.load(std::memory_order_acquire); + } + + [[nodiscard]] uint64_t snapshot_sequence() const noexcept + { + return sequence_.load(std::memory_order_acquire); + } + + [[nodiscard]] bool needs_recovery() const noexcept + { + return uncertain_.load(std::memory_order_acquire); + } + + private: + friend class LocalCheckpointOperation; + LocalCheckpointRegistry() = default; + Status begin(uint64_t coverage, uint64_t generation, uint64_t *sequence) noexcept; + void touched(std::shared_ptr<const ProtectedCheckpoint> candidate) noexcept; + void finish(std::shared_ptr<const ProtectedCheckpoint> candidate, bool committed) noexcept; + + std::shared_ptr<ExtentLedger> ledger_; + std::array<std::shared_ptr<const ProtectedCheckpoint>, 2> anchors_; + std::shared_ptr<const ProtectedCheckpoint> candidate_; + std::atomic<bool> busy_{false}; + std::atomic<bool> uncertain_{false}; + std::atomic<uint64_t> coverage_{0}; + std::atomic<uint64_t> sequence_{0}; + uint64_t generation_ = 0; +}; + +struct LocalCheckpointLimits +{ + size_t max_segments = 1U << 16; +}; + +struct LocalCheckpointResult +{ + Status status; + std::shared_ptr<const ProtectedCheckpoint> checkpoint; +}; + +// Writes the existing segment/directory/anchor format through one aligned +// metadata window. The page barrier precedes the anchor, and durable coverage +// advances only after the anchor barrier. Cancellation after touching the +// anchor completes that barrier and reports the actual commit outcome. +class LocalCheckpointOperation +{ + struct State; + + public: + static Status create(std::shared_ptr<const CapturedDurablePages> pages, + std::shared_ptr<LocalCheckpointRegistry> registry, std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, LocalCheckpointLimits limits, + PageLoadBudget budget, std::shared_ptr<LocalCheckpointOperation> *out); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t segments, uint32_t iu) noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const LocalCheckpointResult &result() const noexcept; + void request_stop() noexcept; + + private: + explicit LocalCheckpointOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_internal.h b/lib/crowdb-tree/src/snapshot/local_checkpoint_internal.h new file mode 100644 index 000000000..9318a3fd6 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_internal.h @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/maptable/mapping_geometry.h" +#include "snapshot/local_checkpoint.h" +#include "snapshot/metadata_codec.h" + +namespace crowdb::tree +{ +struct LocalCheckpointOperation::State : std::enable_shared_from_this<State> +{ + enum class Stage : uint8_t { + Begin, + Segment, + SegmentReserved, + SegmentWritten, + DirectoryReserved, + DirectoryWrite, + DirectoryWritten, + BeforeAnchor, + AnchorWritten, + Commit + }; + + struct Window + { + Window(size_t size, size_t alignment, const std::shared_ptr<detail::AllocationReservation> &budget); + detail::ReservedVector<uint8_t, AllocationKind::kIo> storage; + std::span<uint8_t> bytes; + }; + + // Each submission has its own identity and retains its buffer/extent even + // when submission throws after transport acceptance. An inline callback + // cannot advance the state machine until submit has returned successfully. + struct IoStep + { + explicit IoStep(const PageLoadBudget &budget) : credit(budget.io, 1) + { + } + + static void completed(void *context, Status status) noexcept; + void returned(Status status) noexcept; + void publish() noexcept; + std::shared_ptr<State> owner; + std::shared_ptr<Window> window; + std::shared_ptr<const ProtectedCheckpoint> checkpoint; + AllocationLease credit; + Status completion_status; + Status submission_status; + std::atomic<unsigned> events{0}; + std::atomic<bool> claimed{false}; + std::atomic<bool> reported{false}; + }; + + explicit State(PageLoadBudget credits); + void start(detail::CallbackComplete callback, void *argument) noexcept; + void wake() noexcept; + static void run(void *context) noexcept; + void pump(); + void begin(); + void segment(); + Status pack_frame(const DurableCapturedFrame &frame, uint64_t *word); + void directory_window(); + void anchor(); + void reserve(uint64_t length, Stage next); + void io(bool barrier, uint64_t address, size_t bytes, Stage next, bool anchor = false); + void event(Status status) noexcept; + void finish(Status status) noexcept; + + PageLoadBudget budget; + LocalCheckpointLimits limits; + std::shared_ptr<LocalCheckpointRegistry> registry; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<ProtectedCheckpoint> image; + std::shared_ptr<Window> window; + detail::ReservedVector<DirEntry> directory; + std::optional<snapshot::DirectoryEncoder> encoder; + std::array<uint64_t, kMappingSegmentSize> words{}; + CapturedDurablePages::Frames::const_iterator position; + DirEntry segment_entry; + ProtectedExtent reserved; + uint64_t directory_address = 0; + uint64_t directory_offset = 0; + size_t segment_bytes = 0; + uint32_t iu = 0; + Stage stage = Stage::Begin; + bool admitted = false; + bool anchor_touched = false; + bool waiting = false; + Status event_status; + Status dispatch_failure; + std::atomic<bool> event_ready{false}; + std::atomic<bool> stopped{false}; + std::atomic<bool> scheduled{false}; + std::atomic<bool> wake_requested{false}; + std::atomic<uint8_t> lifecycle{0}; + std::shared_ptr<State> queued_owner; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + LocalCheckpointResult report; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_io.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_io.cpp new file mode 100644 index 000000000..870111d63 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_io.cpp @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/local_checkpoint_internal.h" + +namespace crowdb::tree +{ +void LocalCheckpointOperation::State::IoStep::completed(void *context, Status status) noexcept +{ + auto &step = *static_cast<IoStep *>(context); + if (step.claimed.exchange(true, std::memory_order_acq_rel)) { + return; + } + step.completion_status = std::move(status); + step.credit.reset(); + step.events.fetch_or(1, std::memory_order_acq_rel); + step.publish(); +} + +void LocalCheckpointOperation::State::IoStep::returned(Status status) noexcept +{ + submission_status = std::move(status); + events.fetch_or(2, std::memory_order_acq_rel); + publish(); +} + +void LocalCheckpointOperation::State::IoStep::publish() noexcept +{ + const auto observed = events.load(std::memory_order_acquire); + if ((observed & 2U) == 0 || (submission_status.ok() && (observed & 1U) == 0) || + reported.exchange(true, std::memory_order_acq_rel)) { + return; + } + owner->event(submission_status.ok() ? completion_status.without_message() : submission_status.without_message()); +} + +void LocalCheckpointOperation::State::io(bool barrier, uint64_t address, size_t bytes, Stage next, bool anchor) +{ + auto step = detail::reserved_shared<IoStep>(budget.bytes, budget); + step->owner = shared_from_this(); + step->window = window; + step->checkpoint = image; + stage = next; + waiting = true; + event_ready.store(false, std::memory_order_release); + if (anchor) { + registry->touched(image); + anchor_touched = true; + } + AsyncCompletion callback{.context = step.get(), + .complete_fn = &IoStep::completed, + .owner = step, + .allocations = budget.backend, + .cause = budget.cause}; + Status status; + // A callback may finish the operation on another worker after submit + // returns; keep the backend itself alive across the entire call. + auto backend = store; + try { + if (barrier) { + status = backend->submit_fsync(std::move(callback)); + } + else { + backend->submit_typed_write(PagePurpose::kPageIndex, address, window->bytes.data(), bytes, + std::move(callback)); + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted(); + } + catch (...) { + status = Status::io_error(); + } + step->returned(std::move(status)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp new file mode 100644 index 000000000..4f52a862c --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/maptable/mapping_slot.h" +#include "snapshot/local_checkpoint_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +Status LocalCheckpointOperation::State::pack_frame(const DurableCapturedFrame &frame, uint64_t *word) +{ + const auto &domain = registry->ledger_->limits(); + const auto &backing = frame.backing; + if (!backing || backing->storage != BackingStorage::kDurableBlob || !backing->reloadable() || + backing->backend_id != domain.backend_id || backing->generation != domain.generation || + backing->address < domain.region_begin || backing->address % iu != 0 || backing->length % iu != 0 || + backing->address > UINT64_MAX - backing->length || backing->address / iu > slot_word::kMaxIuIndex || + backing->length / iu > slot_word::kMaxIuCount || frame.logical_bytes == 0 || + frame.logical_bytes > backing->length || frame.page_id >= image->pages->next_page_id) { + return Status::corruption(); + } + if (frame.type == page_type::kLeafBase) { + ++image->anchor.leaf_count; + } + else if (frame.type == page_type::kInnerBase) { + ++image->anchor.inner_count; + } + else if (frame.type != page_type::kOverflowFrame || frame.page_id == image->pages->root_page_id) { + return Status::corruption(); + } + *word = slot_word::pack_unloaded(backing->address / iu, static_cast<uint32_t>(backing->length / iu)); + return Status::Ok(); +} + +void LocalCheckpointOperation::State::segment() +{ + if (position == image->pages->frames.end()) { + encoder.emplace(directory); + image->anchor.segdir_len = encoder->logical_bytes(); + reserve(round_up_to_iu(encoder->logical_bytes(), iu), Stage::DirectoryReserved); + return; + } + if (directory.size() == limits.max_segments) { + finish(Status::resource_exhausted()); + return; + } + const auto index = position->first / kMappingSegmentSize; + if (index >= kMaxMappingSegments) { + finish(Status::invalid_argument()); + return; + } + words.fill(0); + uint32_t live = 0; + while (position != image->pages->frames.end() && position->first / kMappingSegmentSize == index) { + const auto &[pid, frame] = *position; + auto status = + pid == frame.page_id ? pack_frame(frame, &words[pid % kMappingSegmentSize]) : Status::corruption(); + if (!status.ok()) { + finish(std::move(status)); + return; + } + ++live; + ++position; + } + const SegmentImageHeader header{.seg_idx = static_cast<uint32_t>(index), + .generation = image->anchor.snapshot_seq, + .slot_count = kMappingSegmentSize, + .live_count = live}; + std::ranges::fill(window->bytes, uint8_t{0}); + segment_entry = {.seg_idx = header.seg_idx, + .generation = header.generation, + .image_len = static_cast<uint32_t>(segment_image_encoded_size(header.slot_count))}; + auto status = snapshot::encode_segment_into(header, words, window->bytes, &segment_entry.image_crc); + if (!status.ok()) { + finish(std::move(status)); + return; + } + reserve(segment_bytes, Stage::SegmentReserved); +} + +void LocalCheckpointOperation::State::directory_window() +{ + if (!encoder) { + finish(Status::internal_error()); + return; + } + std::ranges::fill(window->bytes, uint8_t{0}); + size_t written = 0; + auto status = encoder->next(window->bytes, &written); + if (!status.ok() || written == 0) { + finish(status.ok() ? Status::corruption() : std::move(status)); + return; + } + const auto address = directory_address + directory_offset; + const auto bytes = round_up_to_iu(written, iu); + directory_offset += bytes; + io(false, address, bytes, Stage::DirectoryWritten); +} + +void LocalCheckpointOperation::State::anchor() +{ + auto status = snapshot::encode_anchor_into(image->anchor, window->bytes); + if (!status.ok()) { + finish(std::move(status)); + return; + } + const auto slot_bytes = snapshot::superblock_slot_bytes(iu); + const auto address = image->anchor.snapshot_seq % 2 == 0 ? slot_bytes : 0; + io(false, address, slot_bytes, Stage::AnchorWritten, true); +} + +void LocalCheckpointOperation::State::reserve(uint64_t length, Stage next) +{ + stage = next; + waiting = true; + event_ready.store(false, std::memory_order_release); + auto status = + registry->ledger_->reserve(length, budget.bytes, [self = shared_from_this()](ExtentLedgerResult result) { + self->reserved = std::move(result.extent); + self->event(std::move(result.status)); + }); + if (!status.ok()) { + event(std::move(status)); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_pump.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_pump.cpp new file mode 100644 index 000000000..544e184d6 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_pump.cpp @@ -0,0 +1,136 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/local_checkpoint_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +void LocalCheckpointOperation::State::wake() noexcept +{ + wake_requested.store(true, std::memory_order_release); + if (lifecycle.load(std::memory_order_acquire) != 2 || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + auto self = weak_from_this().lock(); + assert(self); + for (;;) { + queued_owner = self; + Status status; + try { + status = runtime->try_submit(lane, &State::run, this); + } + catch (...) { + status = Status::resource_exhausted(); + } + if (status.ok()) { + return; + } + queued_owner.reset(); + dispatch_failure = std::move(status); + if (!waiting || event_ready.load(std::memory_order_acquire)) { + finish(dispatch_failure.without_message()); + scheduled.exchange(false, std::memory_order_acq_rel); + return; + } + scheduled.exchange(false, std::memory_order_acq_rel); + if (!event_ready.load(std::memory_order_acquire) || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + } +} + +void LocalCheckpointOperation::State::run(void *context) noexcept +{ + auto self = std::move(static_cast<State *>(context)->queued_owner); + self->wake_requested.exchange(false, std::memory_order_acq_rel); + try { + self->pump(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted()); + } + catch (...) { + self->finish(Status::internal_error()); + } + self->scheduled.exchange(false, std::memory_order_acq_rel); + if (self->wake_requested.load(std::memory_order_acquire) && self->lifecycle.load(std::memory_order_acquire) == 2) { + self->wake(); + } +} + +void LocalCheckpointOperation::State::event(Status status) noexcept +{ + event_status = std::move(status); + event_ready.store(true, std::memory_order_release); + wake(); +} + +void LocalCheckpointOperation::State::pump() +{ + if (waiting) { + if (!event_ready.load(std::memory_order_acquire)) { + return; + } + waiting = false; + if (!dispatch_failure.ok() || !event_status.ok()) { + finish(dispatch_failure.ok() ? event_status.without_message() : dispatch_failure.without_message()); + return; + } + } + if (!anchor_touched && stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + switch (stage) { + case Stage::Begin: + begin(); + break; + case Stage::Segment: + segment(); + break; + case Stage::SegmentReserved: + image->metadata->extents.push_back(reserved); + segment_entry.image_addr = reserved.address; + directory.push_back(segment_entry); + io(false, reserved.address, segment_bytes, Stage::SegmentWritten); + break; + case Stage::SegmentWritten: + stage = Stage::Segment; + wake(); + break; + case Stage::DirectoryReserved: + image->metadata->extents.push_back(reserved); + directory_address = reserved.address; + image->anchor.segdir_addr = reserved.address; + directory_window(); + break; + case Stage::DirectoryWrite: + directory_window(); + break; + case Stage::DirectoryWritten: + if (!encoder) { + finish(Status::internal_error()); + break; + } + if (encoder->complete()) { + image->anchor.segdir_crc = encoder->crc(); + io(true, 0, 0, Stage::BeforeAnchor); + } + else { + stage = Stage::DirectoryWrite; + wake(); + } + break; + case Stage::BeforeAnchor: + anchor(); + break; + case Stage::AnchorWritten: + io(true, 0, 0, Stage::Commit); + break; + case Stage::Commit: + finish(Status::Ok()); + break; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_registry.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_registry.cpp new file mode 100644 index 000000000..fbf962619 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_registry.cpp @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/local_checkpoint.h" + +namespace crowdb::tree +{ +Status LocalCheckpointRegistry::create(std::shared_ptr<ExtentLedger> ledger, + std::shared_ptr<detail::AllocationReservation> metadata, + std::array<std::shared_ptr<const ProtectedCheckpoint>, 2> recovered, + std::shared_ptr<LocalCheckpointRegistry> *out) +{ + if (!ledger || out == nullptr || ledger->limits().iu_size == 0 || + ledger->limits().region_begin < snapshot::region_base_for(ledger->limits().iu_size) || + (metadata && metadata->category() == detail::BudgetClass::kIo)) { + return Status::invalid_argument(); + } + try { + auto registry = detail::reserved_construct<LocalCheckpointRegistry>( + metadata, [](auto *storage) { ::new (static_cast<void *>(storage)) LocalCheckpointRegistry; }); + registry->ledger_ = std::move(ledger); + for (size_t slot = 0; slot < recovered.size(); ++slot) { + const auto &image = recovered[slot]; + if (!image) { + continue; + } + const auto &anchor = image->anchor; + if (!image->pages || anchor.snapshot_seq == 0 || (anchor.snapshot_seq % 2 == 0 ? 1U : 0U) != slot || + anchor.magic != snapshot::kAnchorMagic || anchor.format_version != snapshot::kFormatVersion || + anchor.last_applied_slot != image->pages->coverage || + (!image->pages->frames.empty() && image->pages->backend_id != registry->ledger_->limits().backend_id)) { + return Status::invalid_argument(); + } + if (anchor.snapshot_seq > registry->sequence_.load()) { + registry->sequence_.store(anchor.snapshot_seq); + registry->coverage_.store(anchor.last_applied_slot); + registry->generation_ = image->pages->generation; + } + } + registry->anchors_ = std::move(recovered); + *out = std::move(registry); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +Status LocalCheckpointRegistry::begin(uint64_t coverage, uint64_t generation, uint64_t *sequence) noexcept +{ + if (busy_.exchange(true, std::memory_order_acq_rel)) { + return Status::unavailable(); + } + if (uncertain_.load(std::memory_order_acquire) || generation < generation_ || + (generation == generation_ && coverage < coverage_.load()) || sequence_.load() == UINT64_MAX) { + busy_.store(false, std::memory_order_release); + return Status::unavailable(); + } + *sequence = sequence_.load() + 1; + return Status::Ok(); +} + +void LocalCheckpointRegistry::touched(std::shared_ptr<const ProtectedCheckpoint> candidate) noexcept +{ + candidate_ = std::move(candidate); + uncertain_.store(true, std::memory_order_release); +} + +void LocalCheckpointRegistry::finish(std::shared_ptr<const ProtectedCheckpoint> candidate, bool committed) noexcept +{ + if (committed) { + const auto &anchor = candidate->anchor; + generation_ = candidate->pages->generation; + anchors_[anchor.snapshot_seq % 2 == 0 ? 1 : 0] = std::move(candidate); + sequence_.store(anchor.snapshot_seq, std::memory_order_release); + coverage_.store(anchor.last_applied_slot, std::memory_order_release); + candidate_.reset(); + uncertain_.store(false, std::memory_order_release); + } + busy_.store(false, std::memory_order_release); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_recovery.cpp b/lib/crowdb-tree/src/snapshot/local_recovery.cpp new file mode 100644 index 000000000..c36bb126f --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_recovery.cpp @@ -0,0 +1,178 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "snapshot/local_recovery_internal.h" + +#include <algorithm> +#include <bit> +#include <cassert> + +namespace crowdb::tree +{ +LocalRecoveryOperation::State::Window::Window(size_t blob_bytes, size_t frame_bytes, size_t alignment, + const std::shared_ptr<detail::AllocationReservation> &metadata) + : storage(blob_bytes + alignment - 1, detail::ReservationAllocator<uint8_t, AllocationKind::kIo>(metadata)), + frame(frame_bytes, detail::ReservationAllocator<uint8_t, AllocationKind::kScratch>(metadata)) +{ + void *data = storage.data(); + auto available = storage.size(); + auto *aligned = std::align(alignment, blob_bytes, data, available); + assert(aligned != nullptr); + bytes = {static_cast<uint8_t *>(aligned), blob_bytes}; +} + +LocalRecoveryOperation::State::State(PageLoadBudget credits) + : budget(std::move(credits)), + directory(detail::ReservationAllocator<DirEntry>(budget.bytes)), + references(detail::ReservationAllocator<LeafReference>(budget.bytes)), + report(detail::reserved_shared<LocalRecoveryResult>(budget.bytes)) +{ +} + +std::optional<size_t> LocalRecoveryOperation::empty_allocation_bound(LocalRecoveryLimits limits, uint32_t iu) noexcept +{ + if (iu == 0) { + return {}; + } + const auto metadata_window = round_up_to_iu(segment_image_encoded_size(kMappingSegmentSize), iu); + using detail::shared_allocation_bound; + return detail::AllocationBound( + shared_allocation_bound<State>() + shared_allocation_bound<LocalRecoveryOperation>() + + shared_allocation_bound<State::Window>() + shared_allocation_bound<State::ReadStep>() + + shared_allocation_bound<LocalCheckpointRegistry>() + shared_allocation_bound<LocalRecoveryResult>()) + .add(ExtentLedger::creation_metadata_bound()) + .add(ExtentLedger::reservation_metadata_bound()) + .add(std::max(limits.max_blob_bytes, metadata_window)) + .add(std::bit_ceil(std::max<size_t>(4096, iu)) - 1) + .add(limits.max_frame_bytes) + .value(); +} + +Status LocalRecoveryOperation::create(std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, ExtentLedgerLimits domain, + LocalRecoveryLimits limits, PageLoadBudget budget, + std::shared_ptr<LocalRecoveryOperation> *out) +{ + if (!store || !runtime || !lane || out == nullptr || domain.iu_size == 0 || + domain.region_begin < snapshot::region_base_for(domain.iu_size) || limits.max_segments == 0 || + limits.max_segments > kMaxMappingSegments || limits.max_frames == 0 || + limits.max_blob_bytes < kDurableBlobHeader || limits.max_blob_bytes > UINT32_MAX || + limits.max_frame_bytes < kFrameHeaderSize + kFrameTrailerSize || limits.max_frame_bytes > UINT32_MAX || + limits.max_overflow_references == 0 || limits.max_dependency_steps == 0 || + static_cast<bool>(budget.bytes) != static_cast<bool>(budget.io) || + (budget.bytes && + ((budget.bytes->category() != detail::BudgetClass::kRouting && + budget.bytes->category() != detail::BudgetClass::kReclaim) || + budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + return Status::invalid_argument(); + } + try { + auto state = detail::reserved_shared<State>(budget.bytes, budget); + state->readable_bytes = domain.file_size; + domain.file_size = std::max(domain.file_size, domain.region_begin); + state->domain = domain; + state->limits = limits; + state->store = std::move(store); + state->runtime = std::move(runtime); + state->lane = std::move(lane); + auto status = ExtentLedger::create(domain, budget.bytes, &state->ledger); + if (!status.ok()) { + return status; + } + state->metadata_window = round_up_to_iu(segment_image_encoded_size(kMappingSegmentSize), domain.iu_size); + state->window = detail::reserved_shared<State::Window>( + budget.bytes, std::max(limits.max_blob_bytes, state->metadata_window), limits.max_frame_bytes, + std::bit_ceil(std::max<size_t>(4096, domain.iu_size)), budget.bytes); + *out = detail::reserved_construct<LocalRecoveryOperation>(budget.bytes, [&](auto *storage) { + ::new (static_cast<void *>(storage)) LocalRecoveryOperation(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +LocalRecoveryOperation::LocalRecoveryOperation(std::shared_ptr<State> state) + : state_(std::move(state)), + result_(state_->report) +{ +} + +detail::CallbackSender LocalRecoveryOperation::sender() noexcept +{ + return {this, &LocalRecoveryOperation::submit}; +} + +const LocalRecoveryResult &LocalRecoveryOperation::result() const noexcept +{ + return *result_; +} + +void LocalRecoveryOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void LocalRecoveryOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<LocalRecoveryOperation *>(source)->state_->start(complete, context); +} + +void LocalRecoveryOperation::State::start(detail::CallbackComplete callback, void *argument) noexcept +{ + uint8_t idle = 0; + if (!lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + callback(argument, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + complete = callback; + context = argument; + lifecycle.store(2, std::memory_order_release); + wake(); +} + +void LocalRecoveryOperation::State::finish(Status status) noexcept +{ + if (lifecycle.exchange(3, std::memory_order_acq_rel) == 3) { + return; + } + const bool cancelled = stopped.load(std::memory_order_acquire); + report->status = cancelled ? Status::unavailable() : std::move(status); + if (report->status.ok()) { + report->ledger = ledger; + report->anchors = std::move(anchors); + uint64_t latest = 0; + for (const auto &anchor : report->anchors) { + if (anchor && anchor->anchor.snapshot_seq > latest) { + latest = anchor->anchor.snapshot_seq; + report->latest = anchor->pages; + } + } + } + else { + report->registry.reset(); + } + anchors = {}; + image.reset(); + pages.reset(); + window.reset(); + recovered.owner.reset(); + protection.reset(); + overflow_backing.reset(); + ledger.reset(); + store.reset(); + auto signal = report->status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + if (cancelled) { + signal = detail::CallbackSignal::kStopped; + } + const auto result_status = report->status.without_message(); + // Completed backend callbacks may still retain State. Transfer the result + // to the public operation before notifying its consumer so those callback + // tails cannot retain the recovered checkpoint and its protected extents. + report.reset(); + complete(context, signal, result_status); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_recovery.h b/lib/crowdb-tree/src/snapshot/local_recovery.h new file mode 100644 index 000000000..27cf3c82b --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_recovery.h @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/local_checkpoint.h" + +namespace crowdb::tree +{ +struct LocalRecoveryLimits +{ + bool accept_legacy_chunk_refs = false; + size_t max_segments = 1U << 16; + size_t max_frames = 1024 * 1024; + size_t max_blob_bytes = 1024 * 1024; + size_t max_frame_bytes = 1024 * 1024; + size_t max_overflow_references = 1024 * 1024; + size_t max_dependency_steps = 1024 * 1024; +}; + +struct LocalRecoveryResult +{ + Status status; + std::shared_ptr<ExtentLedger> ledger; + std::shared_ptr<LocalCheckpointRegistry> registry; + std::array<std::shared_ptr<const ProtectedCheckpoint>, 2> anchors; + std::shared_ptr<const CapturedDurablePages> latest; +}; + +// Startup-only bootstrap of an exclusive local allocation domain. Every +// retained anchor's metadata and page/overflow extents are protected before +// enabling allocation. Frames are checked through bounded windows to rebuild +// precise overflow dependencies; raw payloads are not retained. Routing/live +// publication construction is a separate consumer of the recovered mapping. +class LocalRecoveryOperation +{ + struct State; + + public: + static Status create(std::shared_ptr<AsyncPageStore> store, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, ExtentLedgerLimits domain, + LocalRecoveryLimits limits, PageLoadBudget budget, + std::shared_ptr<LocalRecoveryOperation> *out); + // Valid only when the exclusive domain is shorter than the first anchor + // slot. The normal recovery still validates and initializes that domain. + [[nodiscard]] static std::optional<size_t> empty_allocation_bound(LocalRecoveryLimits limits, uint32_t iu) noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const LocalRecoveryResult &result() const noexcept; + void request_stop() noexcept; + + private: + explicit LocalRecoveryOperation(std::shared_ptr<State> state); + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; + std::shared_ptr<LocalRecoveryResult> result_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_frames.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_frames.cpp new file mode 100644 index 000000000..3c657e77c --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_recovery_frames.cpp @@ -0,0 +1,216 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/recovered_frame_identity.h" +#include "crowdb-tree/btree/cell.h" +#include "crowdb-tree/maptable/compressor.h" +#include "snapshot/local_recovery_internal.h" +#include "snapshot/stored_overflow.h" + +namespace crowdb::tree +{ +void LocalRecoveryOperation::State::next_frame() +{ + if (frame_position == pages->frames.end()) { + const auto root = pages->frames.find(pages->root_page_id); + const bool valid_root = pages->frames.empty() + ? pages->root_page_id == kInvalidPageId + : root != pages->frames.end() && root->second.type != page_type::kOverflowFrame; + if (!valid_root || leaf_count != image->anchor.leaf_count || inner_count != image->anchor.inner_count) { + finish(Status::corruption()); + return; + } + stage = Stage::Dependencies; + wake(); + return; + } + const auto &backing = frame_position->second.backing; + if (!backing || !backing->reloadable() || backing->encoding != 0 || backing->backend_id != domain.backend_id || + backing->generation != domain.generation) { + finish(Status::corruption()); + return; + } + if (backing->length > limits.max_blob_bytes) { + finish(Status::resource_exhausted()); + return; + } + read(backing->address, static_cast<size_t>(backing->length), Stage::FrameRead); +} + +Status LocalRecoveryOperation::State::collect_cell(uint64_t page_id, Slice raw) +{ + const CellView cell(raw); + if (!cell.valid() || cell.slot() > pages->coverage || + (cell.flags() != 0 && cell.flags() != kFlagTombstone && cell.flags() != kFlagOverflow) || + (cell.is_tombstone() && raw.size() != kCellHeaderSize)) { + return Status::corruption(); + } + if (!cell.is_overflow()) { + return Status::Ok(); + } + if (raw.size() != kOverflowCellSize || cell.overflow_head() == kInvalidPageId || cell.overflow_len() == 0) { + return Status::corruption(); + } + if (references.size() == limits.max_overflow_references) { + return Status::resource_exhausted(); + } + references.push_back({.page_id = page_id, .head = cell.overflow_head(), .bytes = cell.overflow_len()}); + return Status::Ok(); +} + +void LocalRecoveryOperation::State::frame_read() +{ + if (read_bytes < kDurableBlobHeader) { + finish(Status::corruption()); + return; + } + const uint64_t logical = kDurableBlobHeader + uint64_t{frame_u32(window->bytes.data(), 5)}; + const uint32_t bytes = durable_blob_raw_len(window->bytes.data(), read_bytes); + if (logical > read_bytes || logical > UINT32_MAX || bytes <= kFrameHeaderSize + kFrameTrailerSize) { + finish(Status::corruption()); + return; + } + if (bytes > limits.max_frame_bytes) { + finish(Status::resource_exhausted()); + return; + } + window->frame.resize(bytes); + auto status = decode_durable_page(window->bytes.data(), read_bytes, window->frame.data(), bytes); + if (!status.ok()) { + finish(status.without_message()); + return; + } + auto &descriptor = frame_position->second; + const auto *frame = window->frame.data(); + if (!frame_validate(frame, bytes) || + !recovered_frame_identity_matches(*descriptor.backing, frame_page_type(frame), + frame_u64(frame, fh::kSelfpage_id), descriptor.page_id)) { + finish(Status::corruption()); + return; + } + descriptor.frame_bytes = bytes; + descriptor.logical_bytes = static_cast<uint32_t>(logical); + descriptor.type = frame_page_type(frame); + if (descriptor.type == page_type::kLeafBase) { + const LeafFrameView leaf(frame, bytes); + for (uint32_t i = 0; i < leaf.count(); ++i) { + status = collect_cell(descriptor.page_id, leaf.cell(i)); + if (!status.ok()) { + finish(std::move(status)); + return; + } + } + for (uint32_t i = 0; i < leaf.delta_count(); ++i) { + status = collect_cell(descriptor.page_id, leaf.delta_cell(i)); + if (!status.ok()) { + finish(std::move(status)); + return; + } + } + status = + capture_stored_overflow(leaf, limits.max_overflow_references, budget.bytes, &descriptor.stored_overflow); + if (!status.ok()) { + finish(std::move(status)); + return; + } + ++leaf_count; + } + else if (descriptor.type == page_type::kInnerBase) { + ++inner_count; + } + else { + const OverflowFrameView overflow(frame, bytes); + if (overflow.chunk_len() == 0 || overflow.next_page_id() == descriptor.page_id) { + finish(Status::corruption()); + return; + } + descriptor.overflow_bytes = overflow.chunk_len(); + descriptor.next_overflow = overflow.next_page_id(); + } + ++frame_position; + stage = Stage::NextFrame; + wake(); +} + +void LocalRecoveryOperation::State::dependencies() +{ + if (reference_index == references.size()) { + image_complete(); + return; + } + if (dependency_steps == limits.max_dependency_steps) { + finish(Status::resource_exhausted()); + return; + } + ++dependency_steps; + const auto &reference = references[reference_index]; + if (chain_pid == kInvalidPageId) { + chain_pid = reference.head; + chain_bytes = reference.bytes; + } + if (protected_leaf != reference.page_id) { + auto leaf = pages->frames.find(reference.page_id); + if (leaf == pages->frames.end() || leaf->second.type != page_type::kLeafBase || !leaf->second.backing) { + finish(Status::corruption()); + return; + } + protection = detail::reserved_shared<LeafProtection>(budget.bytes, budget.bytes); + overflow_backing = detail::reserved_shared<OverflowBackingSet>(budget.bytes, budget.bytes); + protection->primary = leaf->second.backing->owner; + auto backing = detail::reserved_shared<BackingRef>(budget.bytes, *leaf->second.backing); + backing->owner = protection; + backing->overflow = overflow_backing; + leaf->second.backing = std::move(backing); + protected_leaf = reference.page_id; + } + const auto found = pages->frames.find(chain_pid); + if (found == pages->frames.end()) { + finish(Status::corruption()); + return; + } + const auto &overflow = found->second; + if (overflow.type != page_type::kOverflowFrame || !overflow.backing || !overflow.backing->reloadable() || + overflow.overflow_bytes == 0 || overflow.overflow_bytes > chain_bytes || + (overflow.next_overflow == kInvalidPageId) != (overflow.overflow_bytes == chain_bytes)) { + finish(Status::corruption()); + return; + } + protection->overflow.push_back(overflow.backing->owner); + overflow_backing->pages.emplace(chain_pid, OverflowBackingPage{.backing = overflow.backing, + .next = overflow.next_overflow, + .bytes = overflow.overflow_bytes, + .frame_bytes = overflow.frame_bytes, + .logical_bytes = overflow.logical_bytes}); + chain_bytes -= overflow.overflow_bytes; + chain_pid = overflow.next_overflow; + if (chain_bytes == 0) { + ++reference_index; + } + wake(); +} + +void LocalRecoveryOperation::State::image_complete() +{ + image->pages = std::move(pages); + + anchors[slot] = std::move(image); + references.clear(); + directory.clear(); + decoder.reset(); + recovered = {}; + protection.reset(); + overflow_backing.reset(); + reference_index = 0; + chain_pid = kInvalidPageId; + chain_bytes = 0; + dependency_steps = 0; + protected_leaf = kInvalidPageId; + leaf_count = 0; + inner_count = 0; + directory_offset = 0; + segment_index = 0; + word_index = 0; + ++slot; + stage = Stage::NextAnchor; + wake(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_internal.h b/lib/crowdb-tree/src/snapshot/local_recovery_internal.h new file mode 100644 index 000000000..9c98681df --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_recovery_internal.h @@ -0,0 +1,157 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/overflow_backing.h" +#include "crowdb-tree/maptable/mapping_geometry.h" +#include "snapshot/local_recovery.h" +#include "snapshot/metadata_decode.h" + +namespace crowdb::tree +{ +struct LocalRecoveryOperation::State : std::enable_shared_from_this<State> +{ + enum class Stage : uint8_t { + NextAnchor, + AnchorRead, + DirectoryProtected, + DirectoryRead, + Segment, + SegmentProtected, + SegmentRead, + PageExtent, + PageProtected, + NextFrame, + FrameRead, + Dependencies, + Activated + }; + + struct Window + { + Window(size_t blob_bytes, size_t frame_bytes, size_t alignment, + const std::shared_ptr<detail::AllocationReservation> &metadata); + detail::ReservedVector<uint8_t, AllocationKind::kIo> storage; + detail::ReservedVector<uint8_t, AllocationKind::kScratch> frame; + std::span<uint8_t> bytes; + }; + + struct ReadStep + { + explicit ReadStep(const PageLoadBudget &budget) : credit(budget.io, 1) + { + } + + static void completed(void *context, Status status) noexcept; + void returned(Status status) noexcept; + void publish() noexcept; + std::shared_ptr<State> owner; + std::shared_ptr<Window> window; + AllocationLease credit; + Status completion_status; + Status submission_status; + std::atomic<unsigned> events{0}; + std::atomic<bool> claimed{false}; + std::atomic<bool> reported{false}; + }; + + struct LeafReference + { + uint64_t page_id; + uint64_t head; + uint64_t bytes; + }; + + struct LeafProtection + { + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !overflow.empty(); ++i) { + overflow.pop_back(); + } + return overflow.empty(); + } + + explicit LeafProtection(const std::shared_ptr<detail::AllocationReservation> &budget) + : overflow(detail::ReservationAllocator<std::shared_ptr<const void>>(budget)) + { + } + + std::shared_ptr<const void> primary; + detail::ReservedVector<std::shared_ptr<const void>> overflow; + }; + + explicit State(PageLoadBudget credits); + void start(detail::CallbackComplete callback, void *argument) noexcept; + void wake() noexcept; + static void run(void *context) noexcept; + void pump(); + void next_anchor(); + void anchor_read(); + void read_directory(); + void directory_read(); + void segment(); + void segment_read(); + void page_extent(); + void page_protected(); + bool valid_extent(uint64_t address, uint64_t bytes) const noexcept; + void protect(uint64_t address, uint64_t bytes, Stage next); + void read(uint64_t address, size_t bytes, Stage next); + void event(Status status) noexcept; + void finish(Status status) noexcept; + + // Frame/bootstrap implementation lives in local_recovery_frames.cpp. + void next_frame(); + void frame_read(); + Status collect_cell(uint64_t page_id, Slice raw); + void dependencies(); + void image_complete(); + + PageLoadBudget budget; + LocalRecoveryLimits limits; + ExtentLedgerLimits domain; + uint64_t readable_bytes = 0; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<ExtentLedger> ledger; + std::shared_ptr<Window> window; + std::array<std::shared_ptr<const ProtectedCheckpoint>, 2> anchors; + std::shared_ptr<ProtectedCheckpoint> image; + std::shared_ptr<CapturedDurablePages> pages; + detail::ReservedVector<DirEntry> directory; + std::optional<snapshot::DirectoryDecoder> decoder; + std::array<uint64_t, kMappingSegmentSize> words{}; + ProtectedExtent recovered; + size_t slot = 0; + size_t directory_offset = 0; + size_t segment_index = 0; + size_t word_index = 0; + size_t read_bytes = 0; + size_t metadata_window = 0; + CapturedDurablePages::Frames::iterator frame_position; + detail::ReservedVector<LeafReference> references; + size_t reference_index = 0; + uint64_t chain_pid = kInvalidPageId; + uint64_t chain_bytes = 0; + size_t dependency_steps = 0; + uint64_t protected_leaf = kInvalidPageId; + std::shared_ptr<LeafProtection> protection; + std::shared_ptr<OverflowBackingSet> overflow_backing; + uint64_t leaf_count = 0; + uint64_t inner_count = 0; + Stage stage = Stage::NextAnchor; + bool waiting = false; + Status event_status; + Status dispatch_failure; + std::atomic<bool> event_ready{false}; + std::atomic<bool> stopped{false}; + std::atomic<bool> scheduled{false}; + std::atomic<bool> wake_requested{false}; + std::atomic<uint8_t> lifecycle{0}; + std::shared_ptr<State> queued_owner; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + std::shared_ptr<LocalRecoveryResult> report; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_io.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_io.cpp new file mode 100644 index 000000000..9a7c49eab --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_recovery_io.cpp @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/local_recovery_internal.h" + +namespace crowdb::tree +{ +void LocalRecoveryOperation::State::ReadStep::completed(void *context, Status status) noexcept +{ + auto &step = *static_cast<ReadStep *>(context); + if (step.claimed.exchange(true, std::memory_order_acq_rel)) { + return; + } + step.completion_status = std::move(status); + step.credit.reset(); + step.events.fetch_or(1, std::memory_order_acq_rel); + step.publish(); +} + +void LocalRecoveryOperation::State::ReadStep::returned(Status status) noexcept +{ + submission_status = std::move(status); + events.fetch_or(2, std::memory_order_acq_rel); + publish(); +} + +void LocalRecoveryOperation::State::ReadStep::publish() noexcept +{ + const auto observed = events.load(std::memory_order_acquire); + if ((observed & 2U) == 0 || (submission_status.ok() && (observed & 1U) == 0) || + reported.exchange(true, std::memory_order_acq_rel)) { + return; + } + owner->event(submission_status.ok() ? completion_status.without_message() : submission_status.without_message()); +} + +void LocalRecoveryOperation::State::read(uint64_t address, size_t bytes, Stage next) +{ + if (bytes > window->bytes.size()) { + finish(Status::resource_exhausted()); + return; + } + auto step = detail::reserved_shared<ReadStep>(budget.bytes, budget); + step->owner = shared_from_this(); + step->window = window; + read_bytes = bytes; + stage = next; + waiting = true; + event_ready.store(false, std::memory_order_release); + Status status; + auto backend = store; + try { + backend->submit_read( + address, window->bytes.data(), bytes, + {.context = step.get(), .complete_fn = &ReadStep::completed, .owner = step, .allocations = budget.backend}); + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted(); + } + catch (...) { + status = Status::io_error(); + } + step->returned(std::move(status)); +} + +void LocalRecoveryOperation::State::protect(uint64_t address, uint64_t bytes, Stage next) +{ + if (!valid_extent(address, bytes)) { + finish(Status::corruption()); + return; + } + stage = next; + waiting = true; + event_ready.store(false, std::memory_order_release); + auto status = ledger->recover(address, bytes, budget.bytes, [self = shared_from_this()](ExtentLedgerResult result) { + self->recovered = std::move(result.extent); + self->event(std::move(result.status)); + }); + if (!status.ok()) { + event(std::move(status)); + } +} + +bool LocalRecoveryOperation::State::valid_extent(uint64_t address, uint64_t bytes) const noexcept +{ + return address >= domain.region_begin && bytes != 0 && address % domain.iu_size == 0 && + bytes % domain.iu_size == 0 && address <= readable_bytes && bytes <= readable_bytes - address; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp new file mode 100644 index 000000000..676c96abc --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp @@ -0,0 +1,194 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/maptable/mapping_slot.h" +#include "snapshot/local_recovery_internal.h" + +#include <algorithm> + +namespace crowdb::tree +{ +void LocalRecoveryOperation::State::next_anchor() +{ + if (slot == anchors.size()) { + stage = Stage::Activated; + waiting = true; + event_ready.store(false, std::memory_order_release); + auto status = ledger->activate(budget.bytes, [self = shared_from_this()](ExtentLedgerResult result) { + self->event(std::move(result.status)); + }); + if (!status.ok()) { + event(std::move(status)); + } + return; + } + const auto bytes = snapshot::superblock_slot_bytes(domain.iu_size); + if ((slot + 1) * bytes > readable_bytes) { + ++slot; + wake(); + return; + } + read(slot * bytes, bytes, Stage::AnchorRead); +} + +void LocalRecoveryOperation::State::anchor_read() +{ + snapshot::CommitAnchor anchor; + if (event_status.code() == Code::kNotFound || !snapshot::decode_anchor(window->bytes.first(read_bytes), &anchor)) { + ++slot; + stage = Stage::NextAnchor; + wake(); + return; + } + if (anchor.snapshot_seq == 0 || (anchor.snapshot_seq % 2 == 0 ? 1U : 0U) != slot || + anchor.segment_slots != kMappingSegmentSize || + anchor.next_page_id > kMappingSegmentSize * kMaxMappingSegments || anchor.segdir_len < 20 || + (anchor.segdir_len - 20) % 32 != 0 || (anchor.segdir_len - 20) / 32 > limits.max_segments) { + finish(Status::corruption()); + return; + } + image = detail::reserved_shared<ProtectedCheckpoint>(budget.bytes, budget.bytes); + image->anchor = anchor; + pages = detail::reserved_shared<CapturedDurablePages>(budget.bytes, budget.bytes); + pages->root_page_id = anchor.root_page_id; + pages->next_page_id = anchor.next_page_id; + pages->coverage = anchor.last_applied_slot; + pages->generation = anchor.snapshot_seq; + pages->backend_id = domain.backend_id; + directory_offset = 0; + segment_index = 0; + protect(anchor.segdir_addr, round_up_to_iu(anchor.segdir_len, domain.iu_size), Stage::DirectoryProtected); +} + +void LocalRecoveryOperation::State::read_directory() +{ + const auto &anchor = image->anchor; + const auto bytes = + std::min<uint64_t>(metadata_window, round_up_to_iu(anchor.segdir_len, domain.iu_size) - directory_offset); + read(anchor.segdir_addr + directory_offset, bytes, Stage::DirectoryRead); +} + +void LocalRecoveryOperation::State::directory_read() +{ + const auto &anchor = image->anchor; + if (directory_offset == 0) { + uint32_t count = 0; + auto status = snapshot::directory_entry_count(window->bytes.first(read_bytes), anchor.segdir_len, &count); + if (!status.ok()) { + finish(std::move(status)); + return; + } + directory.resize(count); + decoder.emplace(directory, anchor.segdir_len); + } + if (!decoder) { + finish(Status::internal_error()); + return; + } + const auto logical = std::min<size_t>(read_bytes, anchor.segdir_len - directory_offset); + auto status = decoder->next(window->bytes.first(logical)); + if (!status.ok()) { + finish(std::move(status)); + return; + } + directory_offset += read_bytes; + if (!decoder->complete()) { + read_directory(); + return; + } + if (!decoder->finish().ok() || decoder->crc() != anchor.segdir_crc) { + finish(Status::corruption()); + return; + } + stage = Stage::Segment; + wake(); +} + +void LocalRecoveryOperation::State::segment() +{ + if (segment_index == directory.size()) { + frame_position = pages->frames.begin(); + stage = Stage::NextFrame; + wake(); + return; + } + const auto &entry = directory[segment_index]; + if (entry.seg_idx >= kMaxMappingSegments || + (segment_index > 0 && directory[segment_index - 1].seg_idx >= entry.seg_idx) || + entry.image_len != segment_image_encoded_size(kMappingSegmentSize)) { + finish(Status::corruption()); + return; + } + protect(entry.image_addr, round_up_to_iu(entry.image_len, domain.iu_size), Stage::SegmentProtected); +} + +void LocalRecoveryOperation::State::segment_read() +{ + SegmentImageHeader header; + auto status = snapshot::decode_segment_into(window->bytes.first(read_bytes), &header, words); + const auto &entry = directory[segment_index]; + if (!status.ok() || header.seg_idx != entry.seg_idx || header.generation != entry.generation || + header.slot_count != kMappingSegmentSize || header.body_crc != entry.image_crc) { + finish(Status::corruption()); + return; + } + word_index = 0; + stage = Stage::PageExtent; + wake(); +} + +void LocalRecoveryOperation::State::page_extent() +{ + while (word_index < words.size() && slot_word::is_empty(words[word_index])) { + ++word_index; + } + if (word_index == words.size()) { + ++segment_index; + stage = Stage::Segment; + wake(); + return; + } + const auto word = words[word_index]; + const bool legacy_ref = limits.accept_legacy_chunk_refs && slot_word::is_page_ref(word); + if (!slot_word::is_byte_location(word) && !legacy_ref) { + finish(Status::not_supported()); + return; + } + const auto pid = (uint64_t{directory[segment_index].seg_idx} * kMappingSegmentSize) + word_index; + if (pid >= pages->next_page_id || pages->frames.size() == limits.max_frames) { + finish(pid >= pages->next_page_id ? Status::corruption() : Status::resource_exhausted()); + return; + } + const auto iu_index = legacy_ref ? slot_word::page_ref_ordinal(word) : slot_word::unloaded_iu_index(word); + const auto iu_count = legacy_ref ? slot_word::page_ref_iu_count(word) : slot_word::unloaded_iu_count(word); + if (iu_index > UINT64_MAX / domain.iu_size) { + finish(Status::corruption()); + return; + } + const auto address = iu_index * domain.iu_size; + const auto bytes = uint64_t{iu_count} * domain.iu_size; + if (bytes > limits.max_blob_bytes) { + finish(Status::resource_exhausted()); + return; + } + protect(address, bytes, Stage::PageProtected); +} + +void LocalRecoveryOperation::State::page_protected() +{ + auto backing = detail::reserved_shared<BackingRef>(budget.bytes); + backing->backend_id = recovered.backend_id; + backing->generation = recovered.generation; + backing->content_id = recovered.identity; + backing->address = recovered.address; + backing->length = recovered.length; + backing->durability = BackingDurability::kCheckpointCommitted; + backing->owner = recovered.owner; + backing->physical_owner = recovered.owner; + backing->physical_metrics = recovered.physical_metrics.lock(); + const auto pid = (uint64_t{directory[segment_index].seg_idx} * kMappingSegmentSize) + word_index; + pages->frames.emplace(pid, DurableCapturedFrame{.page_id = pid, .backing = std::move(backing)}); + ++word_index; + stage = Stage::PageExtent; + wake(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp new file mode 100644 index 000000000..8cfe8f685 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/local_recovery_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +void LocalRecoveryOperation::State::wake() noexcept +{ + wake_requested.store(true, std::memory_order_release); + if (lifecycle.load(std::memory_order_acquire) != 2 || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + auto self = weak_from_this().lock(); + assert(self); + for (;;) { + queued_owner = self; + Status status; + try { + status = runtime->try_submit(lane, &State::run, this); + } + catch (...) { + status = Status::resource_exhausted(); + } + if (status.ok()) { + return; + } + queued_owner.reset(); + dispatch_failure = std::move(status); + if (!waiting || event_ready.load(std::memory_order_acquire)) { + finish(dispatch_failure.without_message()); + scheduled.exchange(false, std::memory_order_acq_rel); + return; + } + scheduled.exchange(false, std::memory_order_acq_rel); + if (!event_ready.load(std::memory_order_acquire) || scheduled.exchange(true, std::memory_order_acq_rel)) { + return; + } + } +} + +void LocalRecoveryOperation::State::run(void *context) noexcept +{ + auto self = std::move(static_cast<State *>(context)->queued_owner); + self->wake_requested.exchange(false, std::memory_order_acq_rel); + try { + self->pump(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted()); + } + catch (...) { + self->finish(Status::internal_error()); + } + self->scheduled.exchange(false, std::memory_order_acq_rel); + if (self->wake_requested.load(std::memory_order_acquire) && self->lifecycle.load(std::memory_order_acquire) == 2) { + self->wake(); + } +} + +void LocalRecoveryOperation::State::event(Status status) noexcept +{ + event_status = std::move(status); + event_ready.store(true, std::memory_order_release); + wake(); +} + +void LocalRecoveryOperation::State::pump() +{ + if (waiting) { + if (!event_ready.load(std::memory_order_acquire)) { + return; + } + waiting = false; + if (!dispatch_failure.ok() || + (!event_status.ok() && (stage != Stage::AnchorRead || event_status.code() != Code::kNotFound))) { + finish(dispatch_failure.ok() ? event_status.without_message() : dispatch_failure.without_message()); + return; + } + } + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + switch (stage) { + case Stage::NextAnchor: + next_anchor(); + break; + case Stage::AnchorRead: + anchor_read(); + break; + case Stage::DirectoryProtected: + image->metadata->extents.push_back(recovered); + read_directory(); + break; + case Stage::DirectoryRead: + directory_read(); + break; + case Stage::Segment: + segment(); + break; + case Stage::SegmentProtected: + image->metadata->extents.push_back(recovered); + read(recovered.address, recovered.length, Stage::SegmentRead); + break; + case Stage::SegmentRead: + segment_read(); + break; + case Stage::PageExtent: + page_extent(); + break; + case Stage::PageProtected: + page_protected(); + break; + case Stage::NextFrame: + next_frame(); + break; + case Stage::FrameRead: + frame_read(); + break; + case Stage::Dependencies: + dependencies(); + break; + case Stage::Activated: + finish(LocalCheckpointRegistry::create(ledger, budget.bytes, anchors, &report->registry)); + break; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/metadata_codec.cpp b/lib/crowdb-tree/src/snapshot/metadata_codec.cpp new file mode 100644 index 000000000..b0605dab3 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/metadata_codec.cpp @@ -0,0 +1,127 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/metadata_codec.h" + +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/maptable/mapping_slot.h" + +#include <algorithm> +#include <array> + +namespace crowdb::tree::snapshot +{ +namespace +{ +template <class T> void put(std::span<uint8_t> output, size_t offset, T value) +{ + for (size_t i = 0; i < sizeof(T); ++i) { + output[offset + i] = static_cast<uint8_t>(value >> (8 * i)); + } +} +} // namespace + +Status encode_segment_into(const SegmentImageHeader &header, std::span<const uint64_t> words, std::span<uint8_t> output, + uint32_t *body_crc) +{ + if (words.size() != header.slot_count || header.live_count > header.slot_count || body_crc == nullptr || + output.size() < segment_image_encoded_size(header.slot_count)) { + return Status::invalid_argument(); + } + bool refs = false; + for (const auto word : words) { + if (!slot_word::is_empty(word) && !slot_word::is_unloaded(word)) { + return Status::invalid_argument(); + } + refs = refs || slot_word::is_page_ref(word); + } + put(output, 0, uint32_t{0x534D5443}); + put(output, 4, refs ? uint16_t{2} : uint16_t{1}); + put(output, 6, refs ? uint16_t{1} : uint16_t{0}); + put(output, 8, header.seg_idx); + put(output, 12, header.generation); + put(output, 20, header.slot_count); + put(output, 24, header.live_count); + put(output, 28, common::crc32c(output.data(), 28)); + for (size_t i = 0; i < words.size(); ++i) { + put(output, 32 + (i * sizeof(uint64_t)), words[i]); + } + *body_crc = common::crc32c(output.data() + 32, words.size_bytes()); + put(output, 32 + words.size_bytes(), *body_crc); + return Status::Ok(); +} + +Status encode_anchor_into(const CommitAnchor &anchor, std::span<uint8_t> output) +{ + if (output.size() < kAnchorBytes) { + return Status::invalid_argument(); + } + std::ranges::fill(output, uint8_t{0}); + put(output, 0, anchor.magic); + put(output, 4, anchor.format_version); + put(output, 8, anchor.snapshot_seq); + put(output, 16, anchor.root_page_id); + put(output, 24, anchor.last_applied_slot); + put(output, 32, anchor.next_page_id); + put(output, 40, anchor.segment_slots); + put(output, 44, anchor.segdir_addr); + put(output, 52, anchor.segdir_len); + put(output, 56, anchor.segdir_crc); + put(output, 60, anchor.leaf_count); + put(output, 68, anchor.inner_count); + put(output, 76, common::crc32c(output.data(), 76)); + return Status::Ok(); +} + +uint32_t DirectoryEncoder::logical_bytes() const noexcept +{ + return entries_.size() > (UINT32_MAX - 20) / 32 ? 0 : static_cast<uint32_t>(20 + (entries_.size() * 32)); +} + +Status DirectoryEncoder::next(std::span<uint8_t> output, size_t *written) +{ + if (written == nullptr || output.empty() || logical_bytes() == 0) { + return Status::invalid_argument(); + } + *written = 0; + const size_t body_end = 16 + (entries_.size() * 32); + while (*written < output.size() && position_ < logical_bytes()) { + std::array<uint8_t, 32> piece{}; + size_t length = 0; + size_t offset = 0; + bool body = false; + if (position_ < 16) { + put(std::span(piece), 0, uint32_t{0x44535443}); + put(std::span(piece), 4, uint16_t{1}); + put(std::span(piece), 8, static_cast<uint32_t>(entries_.size())); + put(std::span(piece), 12, common::crc32c(piece.data(), 12)); + length = 16; + offset = position_; + } + else if (position_ < body_end) { + const auto &entry = entries_[(position_ - 16) / 32]; + put(std::span(piece), 0, entry.seg_idx); + put(std::span(piece), 8, entry.generation); + put(std::span(piece), 16, entry.image_addr); + put(std::span(piece), 24, entry.image_len); + put(std::span(piece), 28, entry.image_crc); + length = 32; + offset = (position_ - 16) % 32; + body = true; + } + else { + put(std::span(piece), 0, body_crc_); + length = 4; + offset = position_ - body_end; + } + const auto bytes = std::min(length - offset, output.size() - *written); + std::copy_n(piece.data() + offset, bytes, output.data() + *written); + if (body) { + body_crc_ = common::crc32c_update(body_crc_, piece.data() + offset, bytes); + } + full_crc_ = common::crc32c_update(full_crc_, piece.data() + offset, bytes); + position_ += bytes; + *written += bytes; + } + return Status::Ok(); +} +} // namespace crowdb::tree::snapshot diff --git a/lib/crowdb-tree/src/snapshot/metadata_codec.h b/lib/crowdb-tree/src/snapshot/metadata_codec.h new file mode 100644 index 000000000..8358c2afa --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/metadata_codec.h @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/maptable/mapping_persist.h" +#include "snapshot/commit_anchor.h" + +#include <span> + +namespace crowdb::tree::snapshot +{ +// Bounded encoders for the existing segment v1/v2, directory v1 and anchor v3 +// formats. Output ownership/alignment and IU padding belong to the caller. +Status encode_segment_into(const SegmentImageHeader &header, std::span<const uint64_t> words, std::span<uint8_t> output, + uint32_t *body_crc); +Status encode_anchor_into(const CommitAnchor &anchor, std::span<uint8_t> output); + +class DirectoryEncoder +{ + public: + explicit DirectoryEncoder(std::span<const DirEntry> entries) : entries_(entries) + { + } + + [[nodiscard]] uint32_t logical_bytes() const noexcept; + + [[nodiscard]] uint32_t crc() const noexcept + { + return full_crc_; + } + + [[nodiscard]] bool complete() const noexcept + { + return logical_bytes() != 0 && position_ == logical_bytes(); + } + + // Fills only logical bytes. The last window's unused tail is untouched. + Status next(std::span<uint8_t> output, size_t *written); + + private: + std::span<const DirEntry> entries_; + size_t position_ = 0; + uint32_t body_crc_ = 0; + uint32_t full_crc_ = 0; +}; +} // namespace crowdb::tree::snapshot diff --git a/lib/crowdb-tree/src/snapshot/metadata_decode.cpp b/lib/crowdb-tree/src/snapshot/metadata_decode.cpp new file mode 100644 index 000000000..c4ffe6bcd --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/metadata_decode.cpp @@ -0,0 +1,184 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/metadata_decode.h" + +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/maptable/mapping_slot.h" + +#include <algorithm> + +namespace crowdb::tree::snapshot +{ +namespace +{ +template <class T> T get(std::span<const uint8_t> input, size_t offset) +{ + T value = 0; + for (size_t i = 0; i < sizeof(T); ++i) { + value |= static_cast<T>(input[offset + i]) << (8 * i); + } + return value; +} + +bool valid_word(uint64_t word, uint16_t version) +{ + if (slot_word::is_empty(word)) { + return true; + } + if (slot_word::is_page_ref(word)) { + return version == 2 && slot_word::page_ref_iu_count(word) != 0; + } + return slot_word::is_byte_location(word) && slot_word::unloaded_iu_count(word) != 0; +} +} // namespace + +Status decode_segment_into(std::span<const uint8_t> input, SegmentImageHeader *header, std::span<uint64_t> words) +{ + if (header == nullptr) { + return Status::invalid_argument(); + } + if (input.size() < 32 || get<uint32_t>(input, 0) != 0x534D5443 || + get<uint32_t>(input, 28) != common::crc32c(input.data(), 28)) { + return Status::corruption(); + } + const SegmentImageHeader decoded{.format_version = get<uint16_t>(input, 4), + .flags = get<uint16_t>(input, 6), + .seg_idx = get<uint32_t>(input, 8), + .generation = get<uint64_t>(input, 12), + .slot_count = get<uint32_t>(input, 20), + .live_count = get<uint32_t>(input, 24)}; + if (((decoded.format_version != 1 || decoded.flags != 0) && (decoded.format_version != 2 || decoded.flags != 1)) || + decoded.live_count > decoded.slot_count || input.size() < segment_image_encoded_size(decoded.slot_count)) { + return Status::corruption(); + } + if (words.size() < decoded.slot_count) { + return Status::invalid_argument(); + } + const size_t body_bytes = static_cast<size_t>(decoded.slot_count) * sizeof(uint64_t); + const auto body_crc = get<uint32_t>(input, 32 + body_bytes); + if (body_crc != common::crc32c(input.data() + 32, body_bytes)) { + return Status::corruption(); + } + uint32_t live = 0; + for (uint32_t i = 0; i < decoded.slot_count; ++i) { + const auto word = get<uint64_t>(input, 32 + (static_cast<size_t>(i) * sizeof(uint64_t))); + if (!valid_word(word, decoded.format_version)) { + return Status::corruption(); + } + live += slot_word::is_empty(word) ? 0 : 1; + } + if (live != decoded.live_count) { + return Status::corruption(); + } + for (uint32_t i = 0; i < decoded.slot_count; ++i) { + words[i] = get<uint64_t>(input, 32 + (static_cast<size_t>(i) * sizeof(uint64_t))); + } + *header = decoded; + header->body_crc = body_crc; + return Status::Ok(); +} + +Status directory_entry_count(std::span<const uint8_t> header, uint32_t logical_bytes, uint32_t *count) +{ + if (count == nullptr) { + return Status::invalid_argument(); + } + if (header.size() < 16 || get<uint32_t>(header, 0) != 0x44535443 || get<uint16_t>(header, 4) != 1 || + get<uint16_t>(header, 6) != 0 || get<uint32_t>(header, 12) != common::crc32c(header.data(), 12)) { + return Status::corruption(); + } + const auto entries = get<uint32_t>(header, 8); + if (20 + (uint64_t{entries} * 32) != logical_bytes) { + return Status::corruption(); + } + *count = entries; + return Status::Ok(); +} + +DirectoryDecoder::DirectoryDecoder(std::span<DirEntry> entries, uint32_t logical_bytes) + : entries_(entries), + logical_bytes_(logical_bytes) +{ + if (logical_bytes < 20 || (logical_bytes - 20) % 32 != 0) { + error_ = Code::kCorruption; + } + else if (entries.size() < (logical_bytes - 20) / 32) { + error_ = Code::kInvalidArgument; + } +} + +Status DirectoryDecoder::finish() const +{ + if (error_ == Code::kInvalidArgument) { + return Status::invalid_argument(); + } + return complete() ? Status::Ok() : Status::corruption(); +} + +Status DirectoryDecoder::finish_piece(size_t length) +{ + if (length == 16) { + uint32_t count = 0; + return directory_entry_count(piece_, logical_bytes_, &count); + } + if (length == 4) { + return get<uint32_t>(piece_, 0) == body_crc_ ? Status::Ok() : Status::corruption(); + } + if (get<uint32_t>(piece_, 4) != 0) { + return Status::corruption(); + } + const size_t index = ((position_ - 16) / 32) - 1; + entries_[index] = {.seg_idx = get<uint32_t>(piece_, 0), + .generation = get<uint64_t>(piece_, 8), + .image_addr = get<uint64_t>(piece_, 16), + .image_len = get<uint32_t>(piece_, 24), + .image_crc = get<uint32_t>(piece_, 28)}; + return Status::Ok(); +} + +Status DirectoryDecoder::next(std::span<const uint8_t> input) +{ + if (error_ != Code::kOk) { + return finish(); + } + if (input.size() > logical_bytes_ - position_) { + error_ = Code::kCorruption; + return finish(); + } + const uint32_t body_end = logical_bytes_ - 4; + while (!input.empty()) { + const bool header = position_ < 16; + const bool body = !header && position_ < body_end; + size_t length = 0; + size_t offset = 0; + if (header) { + length = 16; + offset = position_; + } + else if (body) { + length = 32; + offset = (position_ - 16) % 32; + } + else { + length = 4; + offset = position_ - body_end; + } + const size_t bytes = std::min(length - offset, input.size()); + std::copy_n(input.data(), bytes, piece_.data() + offset); + full_crc_ = common::crc32c_update(full_crc_, input.data(), bytes); + if (body) { + body_crc_ = common::crc32c_update(body_crc_, input.data(), bytes); + } + position_ += static_cast<uint32_t>(bytes); + input = input.subspan(bytes); + if (offset + bytes == length) { + const auto status = finish_piece(length); + if (!status.ok()) { + error_ = status.code(); + return status; + } + } + } + return Status::Ok(); +} +} // namespace crowdb::tree::snapshot diff --git a/lib/crowdb-tree/src/snapshot/metadata_decode.h b/lib/crowdb-tree/src/snapshot/metadata_decode.h new file mode 100644 index 000000000..ffb497a46 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/metadata_decode.h @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/maptable/mapping_persist.h" + +#include <array> +#include <span> + +namespace crowdb::tree::snapshot +{ +// Decodes into caller-owned storage without allocating. IU padding is ignored; +// only slot_count words are written. Both outputs remain unchanged on failure. +Status decode_segment_into(std::span<const uint8_t> input, SegmentImageHeader *header, std::span<uint64_t> words); + +// Reads the first 16 bytes and checks that the count describes exactly +// logical_bytes. This allows charging output storage before streaming the body. +Status directory_entry_count(std::span<const uint8_t> header, uint32_t logical_bytes, uint32_t *count); + +class DirectoryDecoder +{ + public: + DirectoryDecoder(std::span<DirEntry> entries, uint32_t logical_bytes); + + // Consumes the entire window, including header/trailer fragments. The caller + // must exclude IU padding. An error is sticky; output is valid only after + // complete() and finish().ok(), and may be partially written on failure. + Status next(std::span<const uint8_t> input); + [[nodiscard]] Status finish() const; + + [[nodiscard]] bool complete() const noexcept + { + return error_ == Code::kOk && position_ == logical_bytes_; + } + + [[nodiscard]] uint32_t crc() const noexcept + { + return full_crc_; + } + + private: + Status finish_piece(size_t length); + + std::span<DirEntry> entries_; + std::array<uint8_t, 32> piece_{}; + uint32_t logical_bytes_ = 0; + uint32_t position_ = 0; + uint32_t body_crc_ = 0; + uint32_t full_crc_ = 0; + Code error_ = Code::kOk; +}; +} // namespace crowdb::tree::snapshot diff --git a/lib/crowdb-tree/src/snapshot/native_frame_read.cpp b/lib/crowdb-tree/src/snapshot/native_frame_read.cpp new file mode 100644 index 000000000..1b9b43635 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/native_frame_read.cpp @@ -0,0 +1,270 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/native_frame_read.h" + +#include "btree/allocation_bound.h" +#include "btree/page_load_internal.h" +#include "btree/page_load_lifetime.h" +#include "btree/recovered_frame_identity.h" +#include "crowdb-tree/maptable/compressor.h" + +namespace crowdb::tree +{ +namespace +{ +struct NativeReadBuffer +{ + NativeReadBuffer(size_t bytes, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const BackingRef> extent) + : frame(bytes, detail::ReservationAllocator<uint8_t, AllocationKind::kFrame>(metadata)), + backing(std::move(extent)) + { + } + + detail::ReservedVector<uint8_t, AllocationKind::kFrame> frame; + std::shared_ptr<const BackingRef> backing; +}; +} // namespace + +struct NativeFrameRead::State : PageLoadProgress, std::enable_shared_from_this<State> +{ + State(const PageLoadBudget &budget, uint64_t started_at_ns) + : PageLoadProgress(budget, started_at_ns), + metadata(budget.bytes), + blob(detail::ReservationAllocator<uint8_t>(metadata)) + { + } + + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<AsyncPageStore> store; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<State> queued_owner; + std::shared_ptr<State> lifetime; + std::shared_ptr<NativeReadBuffer> buffer; + AlignedLoadBytes blob; + OverflowBackingPage descriptor; + uint64_t pid = kInvalidPageId; + std::atomic<bool> started{false}; + std::atomic<bool> terminal{false}; + std::atomic<bool> stopped{false}; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + NativeFrameReadResult report; + + void finish(Status status) noexcept + { + if (terminal.exchange(true, std::memory_order_acq_rel)) { + return; + } + release_if_idle(); + if (stopped.load(std::memory_order_acquire)) { + status = Status::unavailable(); + } + auto outcome = status.ok() ? TreeOutcome::Success : TreeOutcome::Error; + if (stopped.load(std::memory_order_acquire)) { + outcome = TreeOutcome::Cancelled; + } + (void)metrics.finish(outcome, tree_metrics_now_ns()); + report.status = std::move(status); + if (!report.status.ok()) { + report.frame.reset(); + } + auto keep = std::move(lifetime); + complete(context, report.status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError, + report.status.without_message()); + } + + static void start(void *context) noexcept + { + auto self = std::move(static_cast<State *>(context)->queued_owner); + try { + if (self->stopped.load(std::memory_order_acquire)) { + self->finish(Status::unavailable("native frame read stopped")); + return; + } + if (self->descriptor.backing->storage == BackingStorage::kResidentFrame) { + self->decode_frame(); + return; + } + self->blob.resize(self->descriptor.backing->length); + self->buffer = detail::reserved_shared<NativeReadBuffer>(self->metadata, self->descriptor.frame_bytes, + self->metadata, self->descriptor.backing); + submit_page_read(self, self->descriptor.backing, self->blob.data(), self->blob.size()); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted("native frame read allocation failed")); + } + catch (...) { + self->finish(Status::io_error("native frame submission threw")); + } + } + + static void decode(void *context) noexcept + { + auto self = std::move(static_cast<State *>(context)->queued_owner); + try { + self->decode_frame(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted("native frame decode allocation failed")); + } + catch (...) { + self->finish(Status::internal_error("native frame decode threw")); + } + } + + void decode_frame() + { + const auto &expected = descriptor; + std::span<const uint8_t> bytes; + std::shared_ptr<const void> owner; + if (expected.backing->storage == BackingStorage::kResidentFrame) { + bytes = expected.backing->resident_frame; + owner = expected.backing; + } + else { + const auto raw = durable_blob_raw_len(blob.data(), blob.size()); + const auto logical = durable_blob_logical_len(blob.data(), blob.size()); + if (raw != expected.frame_bytes || logical != expected.logical_bytes) { + finish(Status::corruption("native blob length changed from its captured descriptor")); + return; + } + auto status = decode_durable_page(blob.data(), blob.size(), buffer->frame.data(), expected.frame_bytes); + if (!status.ok()) { + finish(std::move(status)); + return; + } + bytes = buffer->frame; + owner = buffer; + } + const auto *frame = bytes.data(); + if (!frame_validate(frame, expected.frame_bytes) || frame_page_type(frame) != page_type::kOverflowFrame || + !recovered_frame_identity_matches(*expected.backing, page_type::kOverflowFrame, + frame_u64(frame, fh::kSelfpage_id), pid)) { + finish(Status::corruption("native captured overflow frame has invalid CRC, type or PID")); + return; + } + const OverflowFrameView overflow(frame, expected.frame_bytes); + if (overflow.next_page_id() != expected.next || overflow.chunk_len() != expected.bytes) { + finish(Status::corruption("native overflow topology changed from its captured descriptor")); + return; + } + // Native transfer emits canonical identities. The local decoded window + // is already owned and charged; the legacy durable bytes stay immutable. + if (frame_u64(frame, fh::kSelfpage_id) == kInvalidPageId) { + frame_put_u64(buffer->frame.data(), fh::kSelfpage_id, pid); + frame_restamp_crc(buffer->frame.data(), expected.frame_bytes); + } + report.frame = NativeFrameView{.page_id = pid, + .bytes = bytes, + .owner = std::move(owner), + .durable_addr = kNoAddr, + .durable_plen = 0, + .inherited = false}; + finish(Status::Ok()); + } +}; + +std::optional<size_t> NativeFrameRead::allocation_bound(PageLoadLimits limits) noexcept +{ + return detail::AllocationBound( + detail::shared_allocation_bound<State>() + detail::shared_allocation_bound<NativeFrameRead>() + + detail::shared_allocation_bound<NativeReadBuffer>() + detail::shared_allocation_bound<PageReadTicket>()) + .add(limits.max_blob_bytes) + .add(chunk_load_window_bound(limits)) + .add(AlignedLoadBytes::kAlignment - 1) + .add(limits.max_frame_bytes) + .value(); +} + +Status NativeFrameRead::create(uint64_t pid, OverflowBackingPage page, std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageLoadLimits limits, + PageLoadBudget budget, std::shared_ptr<NativeFrameRead> *out) +{ + const auto started_at_ns = tree_metrics_now_ns(); + std::shared_ptr<State> state; + auto status = [&]() -> Status { + try { + if (out == nullptr || *out || pid == kInvalidPageId || !runtime || !lane || !budget.bytes || + budget.bytes->category() == detail::BudgetClass::kIo || !page.backing || !page.backing->readable() || + page.backing->encoding != 0 || page.bytes == 0 || + page.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || + page.bytes > overflow_chunk_cap(page.frame_bytes) || page.next == pid) { + return Status::invalid_argument("native frame read requires a protected bounded descriptor"); + } + const bool resident = page.backing->storage == BackingStorage::kResidentFrame; + if (resident) { + if (page.backing->durability != BackingDurability::kResident || + page.backing->resident_frame.size() != page.frame_bytes || page.logical_bytes != page.frame_bytes) { + return Status::invalid_argument("native resident descriptor has inconsistent physical geometry"); + } + } + else if (!store || !budget.io || budget.io->category() != detail::BudgetClass::kIo || + budget.bytes->account() != budget.io->account() || page.logical_bytes < kDurableBlobHeader || + page.logical_bytes > page.backing->length) { + return Status::invalid_argument( + "native durable frame read needs matching I/O credit and blob geometry"); + } + if (page.backing->length > limits.max_blob_bytes || page.frame_bytes > limits.max_frame_bytes || + !chunk_load_allowed(*page.backing, limits)) { + return Status::resource_exhausted("native captured frame exceeds read bounds"); + } + auto progress = budget; + if (resident) { + progress.io.reset(); + } + state = detail::reserved_shared<State>(budget.bytes, progress, started_at_ns); + state->pid = pid; + state->descriptor = std::move(page); + state->store = std::move(store); + state->runtime = std::move(runtime); + state->lane = std::move(lane); + *out = detail::reserved_construct<NativeFrameRead>( + budget.bytes, [&](auto *storage) { ::new (static_cast<void *>(storage)) NativeFrameRead(state); }); + return Status::Ok(); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted("native frame read admission failed"); + } + }(); + if (!status.ok()) { + if (state) { + (void)state->metrics.finish(TreeOutcome::Error, tree_metrics_now_ns()); + } + else { + record_page_load_terminal(budget, started_at_ns, status); + } + } + return status; +} + +detail::CallbackSender NativeFrameRead::sender() noexcept +{ + return {this, &NativeFrameRead::submit}; +} + +const NativeFrameReadResult &NativeFrameRead::result() const noexcept +{ + return state_->report; +} + +void NativeFrameRead::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); +} + +void NativeFrameRead::submit(void *source, detail::CallbackComplete callback, void *context) +{ + auto state = static_cast<NativeFrameRead *>(source)->state_; + if (state->started.exchange(true, std::memory_order_acq_rel)) { + callback(context, detail::CallbackSignal::kError, Status::unavailable("native frame read is single-use")); + return; + } + state->complete = callback; + state->context = context; + state->lifetime = state; + queue_page_load(state, &State::start); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/native_frame_read.h b/lib/crowdb-tree/src/snapshot/native_frame_read.h new file mode 100644 index 000000000..4f334843c --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/native_frame_read.h @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/overflow_backing.h" +#include "btree/page_load.h" +#include "crowdb-tree/snapshot/native_frame.h" +#include "stdexec_adapter.h" + +namespace crowdb::tree +{ +struct NativeFrameReadResult +{ + Status status; + std::optional<NativeFrameView> frame; +}; + +// Reads one captured physical overflow frame without assembling its value. +// The result owns both charged frame bytes and that exact extent protection. +class NativeFrameRead +{ + struct State; + + public: + static Status create(uint64_t pid, OverflowBackingPage page, std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, PageLoadLimits limits, PageLoadBudget budget, + std::shared_ptr<NativeFrameRead> *out); + static std::optional<size_t> allocation_bound(PageLoadLimits limits) noexcept; + detail::CallbackSender sender() noexcept; + [[nodiscard]] const NativeFrameReadResult &result() const noexcept; + void request_stop() noexcept; + + private: + explicit NativeFrameRead(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete callback, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/native_layout.cpp b/lib/crowdb-tree/src/snapshot/native_layout.cpp new file mode 100644 index 000000000..f42ed8eb9 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/native_layout.cpp @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/native_layout.h" + +#include "btree/allocation_bound.h" +#include "crowdb-tree/maptable/compressor.h" + +#include <algorithm> + +namespace crowdb::tree +{ +NativeLayoutPlanner::NativeLayoutPlanner(std::shared_ptr<const PublishedTreeState> view, NativeLayoutLimits limits, + std::shared_ptr<detail::AllocationReservation> metadata, uint64_t backend_id, + uint64_t incarnation) + : view_(std::move(view)), + limits_(limits), + backend_id_(backend_id), + incarnation_(incarnation), + old_(metadata), + primary_(std::less<>{}, detail::ReservationAllocator<uint64_t>(std::move(metadata))) +{ +} + +std::optional<size_t> NativeLayoutPlanner::allocation_bound(uint64_t next_page_id, NativeLayoutLimits limits) noexcept +{ + if (next_page_id > MappingView::kPageLimit || limits.max_frames == 0 || + limits.max_frames > MappingView::kPageLimit || limits.max_dependency_steps == 0) { + return {}; + } + // Primary and old-overflow PIDs are disjoint. The larger map-node bound + // therefore also covers every primary set node without counting twice. + return detail::AllocationBound(detail::shared_allocation_bound<NativeLayoutPlanner>()) + .add_product(std::min(next_page_id, limits.max_frames), + detail::shared_allocation_bound<OverflowBackingSet::Entry>()) + .value(); +} + +Status NativeLayoutPlanner::accept(CapturedFrameResult::Plan page) +{ + if (!view_ || page_ || limits_.max_frames == 0 || limits_.max_frames > MappingView::kPageLimit || + limits_.max_dependency_steps == 0) { + return Status::invalid_argument("native layout has no bounded idle captured view"); + } + if (page.page_id >= view_->next_page_id || primary_.contains(page.page_id) || old_.pages.contains(page.page_id)) { + return Status::corruption("native primary PID is duplicate or outside the captured namespace"); + } + const auto current = frame_count(); + if (current >= limits_.max_frames || page.summary.new_overflow_frames > limits_.max_frames - current - 1) { + return Status::resource_exhausted("native frame count exceeds its bound"); + } + primary_.insert(page.page_id); + new_overflow_ += page.summary.new_overflow_frames; + reference_ = 0; + pid_ = kInvalidPageId; + remaining_ = 0; + page_ = std::move(page); + return Status::Ok(); +} + +Status NativeLayoutPlanner::include(uint64_t pid, const OverflowBackingPage &page) +{ + if (pid >= view_->next_page_id || primary_.contains(pid) || view_->mapping.get(pid)) { + return Status::corruption("native overflow PID aliases a primary or exceeds its namespace"); + } + const auto found = old_.pages.find(pid); + if (found != old_.pages.end()) { + const auto &old = found->second; + if (old.next != page.next || old.bytes != page.bytes || old.frame_bytes != page.frame_bytes || + old.logical_bytes != page.logical_bytes || old.backing->backend_id != page.backing->backend_id || + old.backing->generation != page.backing->generation || old.backing->address != page.backing->address || + old.backing->length != page.backing->length || old.backing->encoding != page.backing->encoding || + old.backing->storage != page.backing->storage || + !same_chunk_backing(old.backing->chunk, page.backing->chunk) || + old.backing->resident_frame.data() != page.backing->resident_frame.data() || + old.backing->resident_frame.size() != page.backing->resident_frame.size()) { + return Status::corruption("one native PID names conflicting captured extents"); + } + } + else { + if (frame_count() == limits_.max_frames) { + return Status::resource_exhausted("native old overflow count exceeds its bound"); + } + old_.pages.emplace(pid, page); + } + return Status::Ok(); +} + +Status NativeLayoutPlanner::step() +{ + if (!page_) { + return Status::invalid_argument("native dependency step has no current page"); + } + if (reference_ == page_->reused_overflow.size()) { + page_.reset(); + return Status::Ok(); + } + if (steps_ == limits_.max_dependency_steps) { + return Status::resource_exhausted("native overflow dependency work exceeds its bound"); + } + const auto &reference = page_->reused_overflow[reference_]; + if (!reference.backing) { + return Status::unavailable("native overflow has no typed physical descriptors"); + } + if (remaining_ == 0) { + if (reference.head == kInvalidPageId || reference.bytes == 0) { + return Status::corruption("native overflow reference has no head or declared length"); + } + pid_ = reference.head; + remaining_ = reference.bytes; + } + const auto found = reference.backing->pages.find(pid_); + if (found == reference.backing->pages.end()) { + return Status::corruption("native overflow is missing from its primary dependency catalog"); + } + const auto &old = found->second; + if (!old.backing || !old.backing->readable() || old.backing->encoding != 0 || old.bytes == 0 || + old.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || old.bytes > overflow_chunk_cap(old.frame_bytes) || + old.bytes > remaining_ || old.next == pid_ || ((old.next == kInvalidPageId) != (old.bytes == remaining_))) { + return Status::corruption("native overflow descriptor disagrees with its logical chain"); + } + if (old.backing->storage == BackingStorage::kResidentFrame) { + if (old.backing->durability != BackingDurability::kResident || + old.backing->resident_frame.size() != old.frame_bytes || old.logical_bytes != old.frame_bytes) { + return Status::corruption("native resident overflow has inconsistent physical geometry"); + } + } + else if (old.backing->backend_id != backend_id_ || old.backing->generation != incarnation_ || + old.logical_bytes < kDurableBlobHeader || old.logical_bytes > old.backing->length) { + return Status::corruption("native durable overflow has inconsistent domain or blob geometry"); + } + auto status = include(pid_, old); + if (!status.ok()) { + return status; + } + ++steps_; + remaining_ -= old.bytes; + pid_ = old.next; + if (remaining_ == 0) { + ++reference_; + } + return Status::Ok(); +} + +Status NativeLayoutPlanner::finish() +{ + if (!view_ || pending()) { + return Status::invalid_argument("native layout is not complete"); + } + if (view_->next_page_id > MappingView::kPageLimit || + new_overflow_ > MappingView::kPageLimit - view_->next_page_id) { + return Status::resource_exhausted("native overflow namespace exhausted"); + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/native_layout.h b/lib/crowdb-tree/src/snapshot/native_layout.h new file mode 100644 index 000000000..f32819f36 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/native_layout.h @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/overflow_backing.h" +#include "snapshot/captured_frames.h" + +#include <set> + +namespace crowdb::tree +{ +struct NativeLayoutLimits +{ + uint64_t max_frames = 1024 * 1024; + uint64_t max_dependency_steps = 1024 * 1024; +}; + +// Counts and owns physical old overflow descriptors, never their payloads. +// One step checks one dependency; the caller controls CPU continuation work. +class NativeLayoutPlanner +{ + public: + NativeLayoutPlanner(std::shared_ptr<const PublishedTreeState> view, NativeLayoutLimits limits, + std::shared_ptr<detail::AllocationReservation> metadata, uint64_t backend_id = 0, + uint64_t incarnation = 0); + static std::optional<size_t> allocation_bound(uint64_t next_page_id, NativeLayoutLimits limits) noexcept; + Status accept(CapturedFrameResult::Plan page); + Status step(); + Status finish(); + + [[nodiscard]] bool pending() const noexcept + { + return page_.has_value(); + } + + [[nodiscard]] uint64_t new_overflow_frames() const noexcept + { + return new_overflow_; + } + + [[nodiscard]] uint64_t frame_count() const noexcept + { + return primary_.size() + new_overflow_ + old_.pages.size(); + } + + [[nodiscard]] const OverflowBackingSet::Pages &old_frames() const noexcept + { + return old_.pages; + } + + private: + Status include(uint64_t pid, const OverflowBackingPage &page); + std::shared_ptr<const PublishedTreeState> view_; + NativeLayoutLimits limits_; + uint64_t backend_id_; + uint64_t incarnation_; + OverflowBackingSet old_; + std::set<uint64_t, std::less<>, detail::ReservationAllocator<uint64_t>> primary_; + std::optional<CapturedFrameResult::Plan> page_; + size_t reference_ = 0; + uint64_t pid_ = kInvalidPageId; + uint64_t remaining_ = 0; + uint64_t steps_ = 0; + uint64_t new_overflow_ = 0; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/native_plan.cpp b/lib/crowdb-tree/src/snapshot/native_plan.cpp new file mode 100644 index 000000000..a0ecbe3e9 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/native_plan.cpp @@ -0,0 +1,213 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/native_plan.h" + +#include "btree/allocation_bound.h" + +namespace crowdb::tree +{ +struct NativePlanOperation::State : std::enable_shared_from_this<State> +{ + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + std::shared_ptr<State> owner; + + void set_error(Status status) noexcept + { + auto self = std::move(owner); + self->child_status = std::move(status); + self->ready.store(true, std::memory_order_release); + self->wake(); + } + + void set_value() noexcept + { + set_error(Status::Ok()); + } + + void set_stopped() noexcept + { + set_error(Status::unavailable()); + } + }; + + using Connected = stdexec::connect_result_t<detail::CallbackSender, Receiver>; + + struct Attempt + { + Attempt(detail::CallbackSender sender, std::shared_ptr<State> owner) + : connected(stdexec::connect(sender, Receiver{.owner = std::move(owner)})) + { + } + + Connected connected; + }; + + std::shared_ptr<CapturedFrameStream> stream; + std::shared_ptr<NativeLayoutPlanner> planner; + std::shared_ptr<detail::FlushRuntime::Progress> progress; + std::optional<Attempt> attempt; + std::atomic<uint8_t> lifecycle{0}; + std::atomic<bool> stopped{false}; + std::atomic<bool> ready{false}; + bool waiting = false; + bool last_page = false; + Status child_status; + Status outcome; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + + void wake() noexcept + { + if (lifecycle.load(std::memory_order_acquire) == 2) { + progress->request(weak_from_this().lock()); + } + } + + void finish(Status status) noexcept + { + lifecycle.store(3, std::memory_order_release); + outcome = std::move(status); + stream.reset(); + planner.reset(); + complete(context, outcome.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError, + outcome.without_message()); + } + + void begin() + { + waiting = true; + ready.store(false, std::memory_order_release); + attempt.emplace(stream->sender(), shared_from_this()); + stdexec::start(attempt->connected); + } + + void pump() + { + if (waiting) { + if (stopped.load(std::memory_order_acquire)) { + stream->request_stop(); + } + if (!ready.load(std::memory_order_acquire)) { + return; + } + waiting = false; + attempt.reset(); + if (!child_status.ok()) { + finish(std::move(child_status)); + return; + } + const auto &result = stream->result(); + last_page = result.complete; + if (result.plan) { + auto status = planner->accept(*result.plan); + if (!status.ok()) { + finish(std::move(status)); + return; + } + } + else if (!last_page) { + finish(Status::corruption("native planning stream returned no page")); + return; + } + } + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + if (planner->pending()) { + auto status = planner->step(); + if (!status.ok()) { + finish(std::move(status)); + return; + } + wake(); + return; + } + if (last_page) { + finish(planner->finish()); + return; + } + begin(); + } + + static void run(void *context) noexcept + { + auto *self = static_cast<State *>(context); + if (self->lifecycle.load(std::memory_order_acquire) != 2) { + return; + } + try { + self->pump(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted()); + } + catch (...) { + self->finish(Status::internal_error()); + } + } +}; + +Status NativePlanOperation::create(std::shared_ptr<CapturedFrameStream> stream, + std::shared_ptr<NativeLayoutPlanner> planner, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<NativePlanOperation> *out) +try { + if (!stream || !planner || !runtime || !lane || !metadata || out == nullptr || *out) { + return Status::invalid_argument("native planning requires admitted bounded inputs"); + } + auto state = detail::reserved_shared<State>(metadata); + state->stream = std::move(stream); + state->planner = std::move(planner); + auto status = runtime->reserve_progress(lane, &State::run, state.get(), metadata, &state->progress); + if (!status.ok()) { + return status; + } + *out = detail::reserved_construct<NativePlanOperation>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) NativePlanOperation(std::move(state)); }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("native planning allocation failed"); +} + +size_t NativePlanOperation::allocation_bound() noexcept +{ + return detail::shared_allocation_bound<State>() + detail::shared_allocation_bound<NativePlanOperation>() + + detail::FlushRuntime::progress_allocation_bound(); +} + +detail::CallbackSender NativePlanOperation::sender() noexcept +{ + return {this, &NativePlanOperation::submit}; +} + +const Status &NativePlanOperation::result() const noexcept +{ + return state_->outcome; +} + +void NativePlanOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void NativePlanOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + auto self = static_cast<NativePlanOperation *>(source)->state_; + uint8_t idle = 0; + if (!self->lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + complete(context, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + self->complete = complete; + self->context = context; + self->lifecycle.store(2, std::memory_order_release); + self->wake(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/native_plan.h b/lib/crowdb-tree/src/snapshot/native_plan.h new file mode 100644 index 000000000..22161be47 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/native_plan.h @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/native_layout.h" + +namespace crowdb::tree +{ +// Runs the layout pass with one bounded dependency step per CPU continuation. +// The completed planner retains only charged physical descriptors. +class NativePlanOperation +{ + struct State; + + public: + static Status create(std::shared_ptr<CapturedFrameStream> stream, std::shared_ptr<NativeLayoutPlanner> planner, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<NativePlanOperation> *out); + static size_t allocation_bound() noexcept; + detail::CallbackSender sender() noexcept; + [[nodiscard]] const Status &result() const noexcept; + void request_stop() noexcept; + + private: + explicit NativePlanOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/operation_wait.h b/lib/crowdb-tree/src/snapshot/operation_wait.h new file mode 100644 index 000000000..7f518e621 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/operation_wait.h @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "host_completion.h" +#include "stdexec_adapter.h" + +namespace crowdb::tree::detail +{ +struct SnapshotWaitCompletion +{ + HostCompletion done; + Status status; +}; + +struct SnapshotWaitReceiver +{ + using receiver_concept = stdexec::receiver_tag; + SnapshotWaitCompletion *completion; + + void set_error(Status status) const noexcept + { + completion->status = std::move(status); + completion->done.signal(); + } + + void set_value() noexcept + { + set_error(Status::Ok()); + } + + void set_stopped() noexcept + { + set_error(Status::unavailable()); + } +}; + +// Synchronous public facades call this only on their host thread. All CPU and +// backend work still runs through the operation's admitted asynchronous path. +template <class Operation> Status wait_snapshot_operation(const std::shared_ptr<Operation> &operation) +{ + SnapshotWaitCompletion completion; + auto connected = stdexec::connect(operation->sender(), SnapshotWaitReceiver{.completion = &completion}); + stdexec::start(connected); + completion.done.wait(); + return completion.status; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/snapshot/persist.cpp b/lib/crowdb-tree/src/snapshot/persist.cpp index 9fd503382..8d6daf751 100644 --- a/lib/crowdb-tree/src/snapshot/persist.cpp +++ b/lib/crowdb-tree/src/snapshot/persist.cpp @@ -1,888 +1,59 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. - -// B+tree snapshot and recovery implementation. - -// snapshot and recovery (plan-tree #14c/#14d: mapping-table on-disk format). -// -// On-device layout owned here: -// [anchor slot A][anchor slot B][page/segment-image/directory region] -// Each anchor slot is at least 4 KiB and rounded up to the store IU; the -// region begins after the two A/B slots. -// Each snapshot writes only *dirty* base pages (clean pages keep their prior -// addr) plus a fresh image for each *dirty* mapping-table segment (empty/ -// unloaded/resident slots packed verbatim; a resident slot converts to an -// unloaded descriptor at its durable addr -- resident pointers are never -// persisted) plus a fresh segment directory (every present segment's latest -// generation + image location), then commits by writing the inactive anchor -// slot (chosen by seq parity) and syncing. New writes land in space that is -// **dead w.r.t. the committed snapshot** (reused gaps) or appended past EOF — -// never over the committed image, so a crash mid-snapshot falls back intact -// to the last committed anchor. Space freed by the committed snapshot -// becomes reusable only after the next snapshot commits (two-generation -// safety). -// -// Unlike the pre-#14c manifest scheme, there is no reachable-page tree walk -// here: prepare_snapshot_locked() discovers dirty pages/segments by -// enumerating MappingTable's present segments directly (bounded by -// MappingTable::kMaxSegments, not resident-tree size) and inspecting each -// dirty segment's own slot words -- a page's *own* segment is dirty exactly -// when the page was created/mutated/retired (every mapping_.store*/clear -// call bumps its segment's write_seq), so this is complete without walking -// PID structural references (root -> children -> overflow chains) at all. -// -// Key work: segment-scan-driven dirty-page/segment discovery, crash-safe -// free-space reuse, page/segment-image/directory framing, anchor A/B -// commit, best-anchor recovery, lazy mapping-table rebuild. -#include "async_completion_adapter.h" -#include "crowdb-common/crc32c.h" +#include "btree/tree_core.h" +#include "btree/tree_options.h" #include "crowdb-common/log.h" -#include "crowdb-tree/backend/async_page_store.h" -#include "crowdb-tree/backend/block_page_store.h" #include "crowdb-tree/backend/page_store.h" -#include "crowdb-tree/crowdb-tree.h" -#include "crowdb-tree/maptable/compressor.h" -#include "crowdb-tree/maptable/mapping_persist.h" -#include "snapshot/commit_anchor.h" - -#include <algorithm> -#include <chrono> -#include <cstring> -#include <functional> -#include <limits> -#include <map> -#include <memory> -#include <mutex> -#include <thread> -#include <unordered_map> -#include <utility> -#include <vector> namespace crowdb::tree { - -namespace -{ - -std::string make_metrics_prefix(const Config &opt) -{ - return "s." + std::to_string(opt.store_id) + ".g." + std::to_string(opt.group_id) + ".tree"; -} - -using snapshot::CommitAnchor; -using snapshot::encode_anchor; -using snapshot::kAnchorMagic; -using snapshot::kFormatVersion; -using snapshot::read_valid_anchors; -using snapshot::region_base_for; -using snapshot::superblock_slot_bytes; - -// Crash-safe append/reuse allocator. `gaps` are byte ranges that are dead w.r.t. -// the committed snapshot (safe to overwrite); `append` is the grow cursor at -// (or past) EOF. First-fit reuse, else append. Pages are uniform frame_bytes in -// the common case, so freed gaps fit later rewrites exactly. -struct SpaceAllocator -{ - std::vector<std::pair<uint64_t, uint64_t>> gaps; // (addr, len), sorted by addr - uint64_t append = 0; // set by build_allocator (region base or EOF) - uint32_t iu = 1; // every extent is IU-aligned + IU-sized (PT9) - std::set<uint32_t> empty_blocks; // block indices with zero live bytes (block compaction) - - uint64_t alloc(uint64_t len) - { - len = round_up_to_iu(len, iu); // reserve an IU-multiple so the next addr stays aligned - for (auto &g : gaps) { - if (g.second >= len) { - uint64_t a = g.first; - g.first += len; - g.second -= len; - return a; - } - } - uint64_t a = append; - append += len; - return a; - } -}; - -// Live byte ranges of the committed snapshot `anchor`: the directory image, -// every live segment's image, and every *unloaded* (durable, on-disk) slot's -// page frame described by those images. These must never be overwritten -// (they are the crash fallback). Returns false if the directory or any -// segment image can't be read/validated. -bool collect_live_extents_from_directory(const PageStore &store, const CommitAnchor &anchor, uint32_t iu, - std::vector<std::pair<uint64_t, uint64_t>> *out) -{ - std::vector<uint8_t> dbuf(round_up_to_iu(anchor.segdir_len, iu)); - if (!store.read_at(anchor.segdir_addr, dbuf.data(), dbuf.size()).ok()) { - return false; - } - std::vector<DirEntry> entries; - if (!decode_segment_directory(dbuf.data(), dbuf.size(), &entries).ok()) { - return false; - } - out->emplace_back(anchor.segdir_addr, round_up_to_iu(anchor.segdir_len, iu)); - - for (const DirEntry &e : entries) { - out->emplace_back(e.image_addr, round_up_to_iu(e.image_len, iu)); - - std::vector<uint8_t> ibuf(round_up_to_iu(e.image_len, iu)); - if (!store.read_at(e.image_addr, ibuf.data(), ibuf.size()).ok()) { - return false; - } - SegmentImageHeader hdr; - std::vector<uint64_t> words; - if (!decode_segment_image(ibuf.data(), ibuf.size(), &hdr, &words).ok()) { - return false; - } - if (hdr.body_crc != e.image_crc) { - return false; // directory entry points at the wrong/stale image - } - for (uint64_t w : words) { - if (!slot_word::is_unloaded(w)) { - continue; // empty, or (impossible on disk) resident - } - uint64_t addr = 0; - uint32_t plen = 0; - if (!store.decode_mapping_location(w, &addr, &plen).ok()) { - return false; - } - out->emplace_back(addr, plen); - } - } - return true; -} - -// Default sparse-block threshold used when the configured value is zero. -constexpr double kSparseBlockThreshold = 0.70; - -// build the allocator: free = the complement of `live` within -// [kRegionBase, file_size); append grows past EOF. When block_size > 0 -// (array-of-blocks mode), gaps above the configured sparse-block threshold -// are excluded so new writes don't reuse space in nearly-empty blocks. -// Also populates `empty_blocks` with block indices that have zero live bytes. -SpaceAllocator build_allocator(std::vector<std::pair<uint64_t, uint64_t>> live, uint64_t file_size, uint32_t iu, - uint64_t region_base, uint64_t block_size, double sparse_block_threshold, - const std::string &name) -{ - SpaceAllocator a; - a.iu = iu; - std::ranges::sort(live); - uint64_t prev_end = region_base; - for (const auto &e : live) { - if (e.first > prev_end) { - a.gaps.emplace_back(prev_end, e.first - prev_end); - } - prev_end = std::max(prev_end, e.first + e.second); - } - uint64_t eof = file_size < region_base ? region_base : file_size; - eof = round_up_to_iu(eof, iu); // keep the append cursor IU-aligned - if (eof > prev_end) { - a.gaps.emplace_back(prev_end, eof - prev_end); // dead tail - } - a.append = eof; - - if (block_size > 0) { - // Compute per-block live bytes to identify empty blocks. - // The anchor region [0, region_base) is always live (block 0). - std::map<uint32_t, uint64_t> live_per_block; - live_per_block[0] += region_base; // anchor + superblock slots - for (const auto &e : live) { - uint64_t addr = e.first; - uint64_t remaining = e.second; - auto blk = static_cast<uint32_t>(addr / block_size); - // Split cross-block extents across all covered blocks. - while (remaining > 0) { - uint64_t blk_end = (static_cast<uint64_t>(blk) + 1) * block_size; - uint64_t bytes_in_blk = std::min(remaining, blk_end - addr); - live_per_block[blk] += bytes_in_blk; - remaining -= bytes_in_blk; - addr += bytes_in_blk; - ++blk; - } - } - auto max_blk = eof == 0 ? 0U : static_cast<uint32_t>((eof - 1) / block_size); - for (uint32_t i = 0; i <= max_blk; ++i) { - if (!live_per_block.contains(i)) { - a.empty_blocks.insert(i); - } - } - CRB_LOG_INFO("[{}] build_allocator: live_extents={} empty_blocks={} max_blk={} block_size={}", name, - live.size(), a.empty_blocks.size(), max_blk, block_size); - - // Exclude gaps in sparse blocks from the gap list. - if (!a.gaps.empty()) { - std::vector<std::pair<uint64_t, uint64_t>> filtered; - filtered.reserve(a.gaps.size()); - for (const auto &[gap_addr, gap_len] : a.gaps) { - uint64_t addr = gap_addr; - uint64_t remaining = gap_len; - while (remaining > 0) { - uint64_t blk_end = ((addr / block_size) + 1) * block_size; - uint64_t len = std::min(remaining, blk_end - addr); - auto blk = static_cast<uint32_t>(addr / block_size); - uint64_t live_bytes = live_per_block.contains(blk) ? live_per_block.at(blk) : 0; - double free_ratio = 1.0 - (static_cast<double>(live_bytes) / static_cast<double>(block_size)); - if (free_ratio <= sparse_block_threshold) { - filtered.emplace_back(addr, len); - } - addr += len; - remaining -= len; - } - } - size_t gaps_before = a.gaps.size(); - a.gaps = std::move(filtered); - CRB_LOG_INFO("[{}] build_allocator: gap filtering {} -> {} (sparse-block threshold {})", name, gaps_before, - a.gaps.size(), sparse_block_threshold); - } - } - - return a; -} - -std::set<uint32_t> select_sparse_blocks(const std::vector<std::pair<uint64_t, uint64_t>> &live, uint64_t block_size, - double sparse_threshold, uint64_t byte_budget) -{ - std::unordered_map<uint32_t, uint64_t> live_bytes; - for (const auto &[extent_addr, extent_len] : live) { - uint64_t addr = extent_addr; - uint64_t left = extent_len; - while (left > 0) { - auto block = static_cast<uint32_t>(addr / block_size); - uint64_t end = (static_cast<uint64_t>(block) + 1) * block_size; - uint64_t len = std::min(left, end - addr); - live_bytes[block] += len; - addr += len; - left -= len; - } - } - - std::vector<std::pair<uint32_t, uint64_t>> candidates; - for (const auto &[block, bytes] : live_bytes) { - double free_ratio = 1.0 - (static_cast<double>(bytes) / static_cast<double>(block_size)); - if (block != 0 && free_ratio > sparse_threshold) { - candidates.emplace_back(block, bytes); - } - } - std::ranges::sort(candidates, [block_size](const auto &a, const auto &b) { - double free_a = 1.0 - (static_cast<double>(a.second) / static_cast<double>(block_size)); - double free_b = 1.0 - (static_cast<double>(b.second) / static_cast<double>(block_size)); - return free_a == free_b ? a.first < b.first : free_a > free_b; - }); - - std::set<uint32_t> selected; - uint64_t accumulated = 0; - for (const auto &[block, bytes] : candidates) { - if (!selected.empty() && byte_budget > 0 && accumulated + bytes > byte_budget) { - break; - } - selected.insert(block); - accumulated += bytes; - } - return selected; -} - -Status snapshot_frame(PageBase *page, const uint8_t **frame, uint32_t *frame_len) -{ - switch (page->type) { - case page_type::kLeafBase: - *frame = static_cast<LeafBase *>(page)->frame(); - *frame_len = static_cast<LeafBase *>(page)->page_bytes(); - return Status::Ok(); - case page_type::kInnerBase: - *frame = static_cast<InnerBase *>(page)->frame(); - *frame_len = static_cast<InnerBase *>(page)->page_bytes(); - return Status::Ok(); - case page_type::kOverflowFrame: - *frame = static_cast<OverflowBase *>(page)->frame(); - *frame_len = static_cast<OverflowBase *>(page)->page_bytes(); - return Status::Ok(); - default: - return Status::internal_error("snapshot: unexpected resident page type"); - } -} - -} // namespace - -struct Crowdbtree::SnapshotPrepareContext -{ - PreparedSnapshot *out; - PageStore *store; - uint32_t iu; - uint64_t gc; - uint64_t block_size; - bool have_prev; - CommitAnchor prev; - SpaceAllocator alloc; - std::set<uint32_t> relocation_blocks; - std::vector<PrefetchedPage> prefetched; - std::unordered_map<uint64_t, PrefetchedPage *> prefetch_by_page_id; - std::set<uint64_t> forced_segment_images; - std::unordered_map<uint64_t, std::pair<uint64_t, uint32_t>> pending_addr; - std::vector<DirEntry> directory_entries; - uint64_t pages_written = 0; - uint64_t pages_relocated = 0; - uint64_t bytes_relocated = 0; - uint64_t segments_written = 0; - uint64_t live_page_count = 0; -}; - -Status Crowdbtree::prepare_snapshot_locked(PreparedSnapshot *out, std::vector<PrefetchedPage> prefetched, - std::set<uint32_t> relocation_blocks) -{ - if (publication_incomplete_) { - return Status::unavailable("flush publication requires repair"); - } - - PageStore *store = opt_.page_store; - if (store == nullptr) { - return Status::invalid_argument("snapshot: no page_store"); - } - - const uint32_t iu = store->iu_size(); - const uint64_t gc = gc_floor_.load(); - // Geometry (PT9 §9.2): the pool frame must be IU-aligned. The anchor slot - // is rounded up to the IU (superblock_slot_bytes), so larger-IU stores are - // now supported (16/64 KiB etc.) — no fixed 4096 cap. - if (iu > 1 && (opt_.frame_bytes % iu != 0)) { - return Status::invalid_argument("snapshot: frame_bytes must be IU-aligned"); - } - const uint64_t region_base = region_base_for(iu); - - // build the crash-safe allocator from the committed snapshot: its page - // frames, segment images, and directory are off-limits (the crash - // fallback); every other byte in the file is dead and reusable. The - // first snapshot (no committed anchor) just appends. Reusing only - // committed-dead space gives two-generation safety. - std::vector<CommitAnchor> anchors; - Status anchor_status = read_valid_anchors(*store, iu, &anchors); - if (!anchor_status.ok()) { - return anchor_status; - } - bool have_prev = !anchors.empty(); - CommitAnchor prev; - std::vector<std::pair<uint64_t, uint64_t>> live; - if (have_prev) { - prev = anchors.back(); - for (const CommitAnchor &anchor : anchors) { - if (!collect_live_extents_from_directory(*store, anchor, iu, &live)) { - return Status::corruption("snapshot: committed segment directory unreadable"); - } - } - } - double sparse_threshold = - opt_.merge_gc_block_free_threshold > 0.0 ? opt_.merge_gc_block_free_threshold : kSparseBlockThreshold; - SpaceAllocator alloc = - build_allocator(std::move(live), store->size(), iu, region_base, store->block_size(), sparse_threshold, name_); - out->empty_blocks = alloc.empty_blocks; - - const uint64_t block_size = store->block_size(); - out->blocks_selected = relocation_blocks.size(); - - SnapshotPrepareContext ctx{ - .out = out, - .store = store, - .iu = iu, - .gc = gc, - .block_size = block_size, - .have_prev = have_prev, - .prev = prev, - .alloc = std::move(alloc), - .relocation_blocks = std::move(relocation_blocks), - .prefetched = std::move(prefetched), - .prefetch_by_page_id = {}, - .forced_segment_images = {}, - .pending_addr = {}, - .directory_entries = {}, - }; - for (auto &pf : ctx.prefetched) { - ctx.prefetch_by_page_id[pf.page_id] = &pf; - ctx.forced_segment_images.insert(pf.page_id / MappingTable::kSegmentSize); - } - - // Fold and refresh all summaries before any page blob is encoded. - for (uint64_t seg_idx = 0; seg_idx < MappingTable::kMaxSegments; ++seg_idx) { - MappingSegment *segment = mapping_.segment_at(seg_idx); - if (segment == nullptr) { - continue; - } - for (uint32_t slot = 0; slot < segment->slot_count; ++slot) { - const uint64_t page_id = (seg_idx * MappingTable::kSegmentSize) + slot; - uint64_t word = segment->slots[slot].load(std::memory_order_relaxed); - if (!slot_word::is_resident(word)) { - continue; - } - PageBase *page = slot_word::resident_ptr(word); - Status fold = fold_snapshot_page_locked(page_id, gc, &page); - if (!fold.ok()) { - return fold; - } - } - } - bool mapping_dirty = false; - for (uint64_t seg_idx = 0; seg_idx < MappingTable::kMaxSegments && !mapping_dirty; ++seg_idx) { - auto *segment = mapping_.segment_at(seg_idx); - mapping_dirty = segment != nullptr && segment->is_dirty(); - } - if (mapping_dirty) { - refresh_subtree_summaries_locked(root_page_id_.load(std::memory_order_acquire)); - } - Status pages = prepare_snapshot_pages_locked(ctx); - if (!pages.ok()) { - return pages; - } - Status segments = prepare_snapshot_segments_locked(ctx); - if (!segments.ok()) { - return segments; - } - prepare_snapshot_metadata_locked(ctx); - return Status::Ok(); -} - -Status Crowdbtree::fold_snapshot_page_locked(uint64_t page_id, uint64_t gc, PageBase **page) -{ - if ((*page)->type == page_type::kBatchDelta) { - PageBase *base = *page; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - if (base == nullptr || base->type != page_type::kLeafBase) { - return Status::internal_error("snapshot: delta chain without leaf base"); - } - uint64_t right = static_cast<LeafBase *>(base)->right_sibling(); - std::vector<uint64_t> dead_overflow; - LeafBase *fresh = build_leaf_spilling_locked(resolve_leaf_chain_for_rebuild(*page, gc, &dead_overflow), right); - store_preserving_parent_locked(page_id, fresh); - for (PageBase *node = *page; node != nullptr;) { - PageBase *next = node->next; - retire_page(node); - node = next; - } - for (uint64_t head : dead_overflow) { - retire_overflow_chain_locked(head); - } - *page = fresh; - return Status::Ok(); - } - if ((*page)->type != page_type::kLeafBase) { - return Status::Ok(); - } - - auto *leaf = static_cast<LeafBase *>(*page); - LeafFrameView view = leaf->view(); - bool eligible_tombstone = false; - for (uint32_t entry = 0; entry < view.count() && !eligible_tombstone; ++entry) { - CellView cell{view.cell(entry)}; - eligible_tombstone = cell.is_tombstone() && cell.slot() <= gc; - } - for (uint32_t entry = 0; entry < view.delta_count() && !eligible_tombstone; ++entry) { - CellView cell{view.delta_cell(entry)}; - eligible_tombstone = cell.is_tombstone() && cell.slot() <= gc; - } - if (!eligible_tombstone) { - return Status::Ok(); - } - - std::vector<uint64_t> dead_overflow; - size_t dropped = 0; - LeafBase *fresh = build_leaf_spilling_locked(resolve_leaf_chain_for_rebuild(*page, gc, &dead_overflow, &dropped), - leaf->right_sibling()); - store_preserving_parent_locked(page_id, fresh); - retire_page(*page); - for (uint64_t head : dead_overflow) { - retire_overflow_chain_locked(head); - } - if (metrics_.gc_tombstones_c != nullptr && dropped > 0) { - metrics_.gc_tombstones_c->inc_by(dropped); - } - *page = fresh; - return Status::Ok(); -} - -Status Crowdbtree::queue_snapshot_page_locked(SnapshotPrepareContext &ctx, uint64_t page_id, PageBase *page, - const uint8_t *frame, uint32_t frame_len, bool relocate, uint64_t *addr, - uint32_t *logical_len) const -{ - if (page->durable_addr != kNoAddr && !relocate) { - *addr = page->durable_addr; - *logical_len = page->durable_plen; - if (metrics_.snapshot_page_write_cache_c != nullptr) { - metrics_.snapshot_page_write_cache_c->inc(); - } - return Status::Ok(); - } - - std::vector<uint8_t> blob; - encode_durable_page(frame, frame_len, opt_.compression, &blob); - auto logical = static_cast<uint32_t>(blob.size()); - *addr = ctx.alloc.alloc(logical); - *logical_len = logical; - ctx.out->page_writes.push_back(PreparedPageWrite{ - .page_id = page_id, - .page = page, - .prior_addr = page->durable_addr, - .addr = *addr, - .logical_len = logical, - .blob = std::move(blob), - }); - ++ctx.pages_written; - if (relocate) { - ++ctx.pages_relocated; - ctx.bytes_relocated += logical; - } - return Status::Ok(); -} - -Status Crowdbtree::prepare_snapshot_resident_page_locked(SnapshotPrepareContext &ctx, uint64_t page_id, - uint64_t seg_idx) -{ - uint64_t word = mapping_.get_word(page_id); - if (!slot_word::is_resident(word)) { - return Status::Ok(); - } - PageBase *page = slot_word::resident_ptr(word); - Status fold = fold_snapshot_page_locked(page_id, ctx.gc, &page); - if (!fold.ok()) { - return fold; - } - const uint8_t *frame = nullptr; - uint32_t frame_len = 0; - Status resolved = snapshot_frame(page, &frame, &frame_len); - if (!resolved.ok()) { - return resolved; - } - bool relocate = page->durable_addr != kNoAddr && ctx.block_size > 0 && - ctx.relocation_blocks.contains(static_cast<uint32_t>(page->durable_addr / ctx.block_size)); - uint64_t addr = 0; - uint32_t logical_len = 0; - Status queued = queue_snapshot_page_locked(ctx, page_id, page, frame, frame_len, relocate, &addr, &logical_len); - if (!queued.ok()) { - return queued; - } - if (page->durable_addr == kNoAddr || relocate) { - ctx.pending_addr[page_id] = {addr, logical_len}; - if (relocate) { - ctx.forced_segment_images.insert(seg_idx); - } - } - return Status::Ok(); -} - -Status Crowdbtree::prepare_snapshot_pages_locked(SnapshotPrepareContext &ctx) -{ - // Ascending order discovers pages created by folds later in the same pass. - for (uint64_t seg_idx = 0; seg_idx < MappingTable::kMaxSegments; ++seg_idx) { - MappingSegment *segment = mapping_.segment_at(seg_idx); - if (segment == nullptr) { - continue; - } - for (uint32_t slot = 0; slot < segment->slot_count; ++slot) { - uint64_t page_id = (seg_idx * MappingTable::kSegmentSize) + slot; - Status status = prepare_snapshot_resident_page_locked(ctx, page_id, seg_idx); - if (!status.ok()) { - return status; - } - } - } - snapshot_pages_written_.store(ctx.pages_written); - snapshot_pages_total_.fetch_add(ctx.pages_written, std::memory_order_relaxed); - if (metrics_.snapshot_pages_c != nullptr && ctx.pages_written > 0) { - metrics_.snapshot_pages_c->inc_by(ctx.pages_written); - } - return Status::Ok(); -} - -Status Crowdbtree::prepare_snapshot_slot_locked(SnapshotPrepareContext &ctx, uint64_t page_id, uint64_t word, - uint64_t *durable_word, uint32_t *live_count) -{ - if (slot_word::is_empty(word)) { - *durable_word = slot_word::kEmpty; - return Status::Ok(); - } - if (slot_word::is_resident(word)) { - PageBase *page = slot_word::resident_ptr(word); - auto pending = ctx.pending_addr.find(page_id); - uint64_t addr = pending == ctx.pending_addr.end() ? page->durable_addr : pending->second.first; - uint32_t len = pending == ctx.pending_addr.end() ? page->durable_plen : pending->second.second; - if (addr == kNoAddr) { - return Status::internal_error("snapshot: dirty resident page missing pending write"); - } - Status encode_status = ctx.store->encode_mapping_location(addr, len, durable_word); - if (!encode_status.ok()) { - return encode_status; - } - ++*live_count; - return Status::Ok(); - } - - uint64_t old_addr = 0; - uint32_t padded_len = 0; - Status decode_status = ctx.store->decode_mapping_location(word, &old_addr, &padded_len); - if (!decode_status.ok()) { - return decode_status; - } - auto block = ctx.block_size == 0 ? 0U : static_cast<uint32_t>(old_addr / ctx.block_size); - if (ctx.block_size == 0 || !ctx.relocation_blocks.contains(block)) { - *durable_word = word; - ++*live_count; - return Status::Ok(); - } - std::vector<uint8_t> blob; - auto prefetched = ctx.prefetch_by_page_id.find(page_id); - if (prefetched != ctx.prefetch_by_page_id.end() && prefetched->second->old_word == word) { - blob = prefetched->second->blob; - } - else if (prefetched != ctx.prefetch_by_page_id.end()) { - *durable_word = word; - ++*live_count; - return Status::Ok(); - } - else { - blob.resize(padded_len); - Status read = ctx.store->read_at(old_addr, blob.data(), blob.size()); - if (!read.ok()) { - return read; - } - } - - uint64_t new_addr = ctx.alloc.alloc(padded_len); - uint64_t new_word = slot_word::kEmpty; - Status encode_status = ctx.store->encode_mapping_location(new_addr, padded_len, &new_word); - if (!encode_status.ok()) { - return encode_status; - } - ctx.out->page_writes.push_back(PreparedPageWrite{ - .page_id = page_id, - .page = nullptr, - .prior_addr = old_addr, - .addr = new_addr, - .logical_len = padded_len, - .blob = std::move(blob), - }); - ctx.out->unloaded_relocations.push_back( - PreparedUnloadedRelocation{.page_id = page_id, .old_word = word, .new_word = new_word}); - *durable_word = new_word; - ++*live_count; - ++ctx.pages_relocated; - ctx.bytes_relocated += padded_len; - return Status::Ok(); -} - -Status Crowdbtree::prepare_snapshot_segment_locked(SnapshotPrepareContext &ctx, uint64_t seg_idx, - MappingSegment *segment) -{ - if (!segment->is_dirty() && !ctx.forced_segment_images.contains(seg_idx)) { - ctx.directory_entries.push_back(DirEntry{ - .seg_idx = static_cast<uint32_t>(seg_idx), - .generation = segment->generation.load(std::memory_order_relaxed), - .image_addr = segment->image_addr, - .image_len = segment->image_len, - .image_crc = segment->image_crc, - }); - return Status::Ok(); - } - - std::vector<uint64_t> words(segment->slot_count); - uint32_t live_count = 0; - for (uint32_t slot = 0; slot < segment->slot_count; ++slot) { - uint64_t page_id = (seg_idx * MappingTable::kSegmentSize) + slot; - uint64_t word = segment->slots[slot].load(std::memory_order_relaxed); - Status status = prepare_snapshot_slot_locked(ctx, page_id, word, &words[slot], &live_count); - if (!status.ok()) { - return status; - } - } - - uint64_t seen_write_seq = segment->write_seq.load(std::memory_order_relaxed); - uint64_t generation = segment->generation.load(std::memory_order_relaxed) + 1; - SegmentImageHeader header{ - .seg_idx = static_cast<uint32_t>(seg_idx), - .generation = generation, - .slot_count = segment->slot_count, - .live_count = live_count, - }; - std::vector<uint8_t> image; - uint32_t image_crc = 0; - encode_segment_image(header, words, &image, &image_crc); - auto image_len = static_cast<uint32_t>(image.size()); - uint64_t image_addr = ctx.alloc.alloc(image_len); - image.resize(round_up_to_iu(image_len, ctx.iu), 0); - ctx.out->segment_writes.push_back(PreparedSegmentWrite{ - .seg_idx = seg_idx, - .seg = segment, - .seen_write_seq = seen_write_seq, - .new_generation = generation, - .addr = image_addr, - .logical_len = image_len, - .image_crc = image_crc, - .blob = std::move(image), - }); - ctx.directory_entries.push_back(DirEntry{ - .seg_idx = static_cast<uint32_t>(seg_idx), - .generation = generation, - .image_addr = image_addr, - .image_len = image_len, - .image_crc = image_crc, - }); - ++ctx.segments_written; - ctx.live_page_count += live_count; - return Status::Ok(); -} - -Status Crowdbtree::prepare_snapshot_segments_locked(SnapshotPrepareContext &ctx) -{ - for (uint64_t seg_idx = 0; seg_idx < MappingTable::kMaxSegments; ++seg_idx) { - MappingSegment *segment = mapping_.segment_at(seg_idx); - if (segment == nullptr) { - continue; - } - Status status = prepare_snapshot_segment_locked(ctx, seg_idx, segment); - if (!status.ok()) { - return status; - } +Status Crowdbtree::open(const Config &opt, std::unique_ptr<Crowdbtree> *out) +try { + if (out == nullptr) { + return Status::invalid_argument("open requires output"); + } + Config options = opt; + auto status = configure_tree_options(&options); + std::shared_ptr<TreeCore> core; + if (status.ok()) { + status = TreeCore::open(options, &core); + } + if (status.ok()) { + auto tree = std::unique_ptr<Crowdbtree>(new Crowdbtree(options, std::move(core))); + tree->init_metrics("s." + std::to_string(options.store_id) + ".g." + std::to_string(options.group_id) + ".tree", + options.backend_label); + CRB_LOG_INFO("[{}] open: frame_bytes={}", tree->name_, options.frame_bytes); + *out = std::move(tree); } - return Status::Ok(); + return status; } - -void Crowdbtree::prepare_snapshot_metadata_locked(SnapshotPrepareContext &ctx) -{ - PreparedSnapshot *out = ctx.out; - const uint32_t iu = ctx.iu; - const bool have_prev = ctx.have_prev; - const CommitAnchor &prev = ctx.prev; - auto &alloc = ctx.alloc; - auto &directory_entries = ctx.directory_entries; - const uint64_t pages_written = ctx.pages_written; - const uint64_t pages_relocated = ctx.pages_relocated; - const uint64_t bytes_relocated = ctx.bytes_relocated; - const uint64_t segments_written = ctx.segments_written; - const uint64_t live_page_count = ctx.live_page_count; - - std::vector<uint8_t> directory; - encode_segment_directory(directory_entries, &directory); - uint64_t segdir_len = directory.size(); // logical (recorded in the anchor) - uint64_t segdir_addr = alloc.alloc(segdir_len); - directory.resize(round_up_to_iu(segdir_len, iu), 0); // pad to the IU extent (PT9) - out->directory_write = PreparedSnapshotWrite{.addr = segdir_addr, .blob = std::move(directory)}; - - uint64_t seq = have_prev ? prev.snapshot_seq + 1 : 1; - - CommitAnchor anchor; - anchor.magic = kAnchorMagic; - anchor.format_version = kFormatVersion; - anchor.snapshot_seq = seq; - anchor.root_page_id = root_page_id_.load(); - anchor.last_applied_slot = last_applied_slot_.load(); - anchor.next_page_id = mapping_.next_page_id(); - anchor.segment_slots = static_cast<uint32_t>(MappingTable::kSegmentSize); - anchor.segdir_addr = segdir_addr; - anchor.segdir_len = static_cast<uint32_t>(segdir_len); - anchor.segdir_crc = crowdb::common::crc32c(out->directory_write.blob.data(), segdir_len); - anchor.leaf_count = leaf_count_.load(std::memory_order_relaxed); - anchor.inner_count = inner_count_.load(std::memory_order_relaxed); - - std::vector<uint8_t> abuf; - encode_anchor(anchor, superblock_slot_bytes(iu), &abuf); - uint64_t anchor_slot = (seq & 1) != 0 ? 0 : superblock_slot_bytes(iu); // alternate A/B by parity - out->anchor_write = PreparedSnapshotWrite{.addr = anchor_slot, .blob = std::move(abuf)}; - out->last_applied_slot = anchor.last_applied_slot; - out->seq = seq; - out->live_page_count = live_page_count; - out->pages_written = pages_written; - out->segdir_len = segdir_len; - out->pages_relocated = pages_relocated; - out->bytes_relocated = bytes_relocated; - snapshot_segments_written_.store(segments_written); +catch (const std::bad_alloc &) { + return Status::resource_exhausted("Tree open allocation failed"); } - -void Crowdbtree::commit_prepared_snapshot(const PreparedSnapshot &prepared) -{ - { - std::scoped_lock lk(write_mutex_); - for (const auto &pw : prepared.page_writes) { - PageBase *v = mapping_.get_resident(pw.page_id); - // Identity check (not just durable_addr == kNoAddr): `v` may be a - // *different*, independently-dirty page if a concurrent - // consolidate/flush/split replaced this page_id's mapping entry - // since prepare_snapshot_locked() ran -- see PreparedPageWrite's - // doc comment. A mismatch just skips this entry (harmless). - if (pw.page != nullptr && v == pw.page && v->durable_addr == pw.prior_addr) { - v->durable_addr = pw.addr; - v->durable_plen = pw.logical_len; - } - } - for (const auto &sw : prepared.segment_writes) { - // commit_segment_persist() re-checks identity *and* write_seq -- - // see PreparedSegmentWrite's doc comment for why a segment needs - // both, not just identity like a page write. A refusal just - // leaves the segment dirty for the next snapshot (harmless). - mapping_.commit_segment_persist(sw.seg_idx, sw.seg, sw.seen_write_seq, sw.new_generation, sw.addr, - sw.logical_len, sw.image_crc); - } - for (const auto &relocation : prepared.unloaded_relocations) { - if (mapping_.get_word(relocation.page_id) == relocation.old_word) { - mapping_.store_word(relocation.page_id, relocation.new_word); - } - } - } - version_.fetch_add(1); - last_applied_slot_.store(prepared.last_applied_slot, std::memory_order_release); - snapshot_total_.fetch_add(1, std::memory_order_relaxed); - CRB_LOG_INFO("[{}] snapshot committed: seq={} last_applied={} live_pages={} written={} segdir_len={}", name_, - prepared.seq, prepared.last_applied_slot, prepared.live_page_count, prepared.pages_written, - prepared.segdir_len); +catch (const std::exception &error) { + return Status::internal_error(error.what()); } -void Crowdbtree::finalize_prepared_snapshot(PreparedSnapshot &prepared) +Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_seq) { - commit_prepared_snapshot(prepared); - durable_snapshot_last_applied_slot_.store(prepared.last_applied_slot, std::memory_order_release); - durable_snapshot_seq_.store(prepared.seq, std::memory_order_release); - if (opt_.page_store->block_size() == 0) { - return; - } - std::vector<std::pair<uint64_t, uint64_t>> live; - std::vector<CommitAnchor> anchors; - Status anchor_status = read_valid_anchors(*opt_.page_store, opt_.page_store->iu_size(), &anchors); - if (!anchor_status.ok()) { - CRB_LOG_WARN("[{}] block compaction: anchor read failed; skipping deletion: {}", name_, - anchor_status.to_string()); - return; - } - for (const CommitAnchor &anchor : anchors) { - if (!collect_live_extents_from_directory(*opt_.page_store, anchor, opt_.page_store->iu_size(), &live)) { - CRB_LOG_WARN("[{}] block compaction: retained anchor unreadable; skipping deletion", name_); - return; + uint64_t applied = 0; + uint64_t sequence = 0; + auto status = core_->checkpoint(&applied, &sequence); + if (status.ok()) { + if (out_last_applied != nullptr) { + *out_last_applied = applied; } - } - double sparse_threshold = - opt_.merge_gc_block_free_threshold > 0.0 ? opt_.merge_gc_block_free_threshold : kSparseBlockThreshold; - SpaceAllocator state = build_allocator(std::move(live), opt_.page_store->size(), opt_.page_store->iu_size(), - region_base_for(opt_.page_store->iu_size()), opt_.page_store->block_size(), - sparse_threshold, name_); - uint64_t deleted = 0; - for (uint32_t block : state.empty_blocks) { - Status ds = opt_.page_store->delete_block(block); - if (ds.ok()) { - ++deleted; - } - else { - CRB_LOG_WARN("[{}] block compaction: delete_block({}) failed: {}", name_, block, ds.to_string()); + if (out_snapshot_seq != nullptr) { + *out_snapshot_seq = sequence; } + CRB_LOG_INFO("[{}] snapshot committed: slot={} sequence={}", name_, applied, sequence); } - prepared.blocks_deleted = deleted; -} - -void Crowdbtree::acquire_snapshot_slot() -{ - while (snapshot_inflight_.exchange(true, std::memory_order_acquire)) { - std::this_thread::sleep_for(std::chrono::microseconds(50)); - } + return status; } -void Crowdbtree::release_snapshot_slot() +void Crowdbtree::snapshot_async(std::function<void(Status, uint64_t)> on_done) { - snapshot_inflight_.store(false, std::memory_order_release); + core_->checkpoint_async(std::move(on_done)); } Status Crowdbtree::materialize_ownership(uint64_t *bytes_written, bool *complete) @@ -890,421 +61,20 @@ Status Crowdbtree::materialize_ownership(uint64_t *bytes_written, bool *complete if (opt_.page_store == nullptr || bytes_written == nullptr || complete == nullptr) { return Status::invalid_argument("materialize_ownership: invalid argument"); } - if (opt_.page_store->has_shared_ownership()) { - constexpr size_t kPagesPerPass = 1024; - constexpr size_t kSegmentsPerPass = 1; - bool mapping_ready = false; - bool needs_snapshot = false; - uint64_t pruned_version = std::numeric_limits<uint64_t>::max(); - { - std::scoped_lock lock(write_mutex_); - const uint64_t current_version = version_.load(); - if (mapping_pruned_version_ == current_version) { - mapping_ready = true; - } - else { - if (mapping_materialization_ == nullptr || mapping_materialization_->version != current_version) { - mapping_materialization_ = std::make_unique<MappingMaterializationState>(); - mapping_materialization_->version = current_version; - mapping_materialization_->stack.push_back(root_page_id_.load()); - mapping_materialization_->reachable.resize((mapping_.next_page_id() + 63) / 64); - } - MappingMaterializationState &state = *mapping_materialization_; - size_t work = 0; - while (!state.stack.empty() && work < kPagesPerPass) { - const uint64_t page_id = state.stack.back(); - state.stack.pop_back(); - if (page_id >= mapping_.next_page_id()) { - continue; - } - uint64_t &word = state.reachable[page_id / 64]; - const uint64_t bit = uint64_t{1} << (page_id % 64); - if ((word & bit) != 0) { - continue; - } - word |= bit; - ++work; - PageBase *page = resident(page_id); - if (page == nullptr) { - mapping_materialization_.reset(); - return Status::corruption("mapping materialization: reachable page is missing"); - } - while (page->type == page_type::kBatchDelta) { - page = page->next; - if (page == nullptr) { - mapping_materialization_.reset(); - return Status::corruption("mapping materialization: delta chain has no base"); - } - } - if (page->type == page_type::kInnerBase) { - for (uint64_t child : static_cast<InnerBase *>(page)->children()) { - state.stack.push_back(child); - } - } - else if (page->type == page_type::kLeafBase) { - LeafFrameView leaf = static_cast<LeafBase *>(page)->view(); - for (uint32_t index = 0; index < leaf.count(); ++index) { - CellView cell{leaf.cell(index)}; - if (cell.is_overflow()) { - state.stack.push_back(cell.overflow_head()); - } - } - } - else if (page->type == page_type::kOverflowFrame) { - const uint64_t next = static_cast<OverflowBase *>(page)->next_page_id(); - if (next != kInvalidPageId) { - state.stack.push_back(next); - } - } - else { - mapping_materialization_.reset(); - return Status::corruption("mapping materialization: unexpected page type"); - } - } - if (!state.stack.empty()) { - *bytes_written = 0; - *complete = false; - return Status::Ok(); - } - const uint64_t segment_limit = - (mapping_.next_page_id() + MappingTable::kSegmentSize - 1) / MappingTable::kSegmentSize; - size_t segments = 0; - while (state.next_segment < segment_limit && segments < kSegmentsPerPass) { - const uint64_t segment_index = state.next_segment++; - MappingSegment *segment = mapping_.segment_at(segment_index); - ++segments; - if (segment == nullptr) { - continue; - } - std::vector<uint64_t> unreachable; - for (uint32_t slot = 0; slot < segment->slot_count; ++slot) { - const uint64_t page_id = (segment_index * MappingTable::kSegmentSize) + slot; - const bool reachable = page_id < mapping_.next_page_id() && - (state.reachable[page_id / 64] & (uint64_t{1} << (page_id % 64))) != 0; - if (!reachable && !slot_word::is_empty(segment->slots[slot].load(std::memory_order_relaxed))) { - unreachable.push_back(page_id); - } - } - for (uint64_t page_id : unreachable) { - const uint64_t word = mapping_.get_word(page_id); - mapping_.clear(page_id); - if (slot_word::is_resident(word)) { - for (PageBase *page = slot_word::resident_ptr(word); page != nullptr;) { - PageBase *next = page->next; - retire_page(page); - page = next; - } - } - } - } - if (state.next_segment == segment_limit) { - mapping_materialization_.reset(); - mapping_ready = true; - needs_snapshot = true; - pruned_version = current_version; - } - } - } - if (!mapping_ready) { - *bytes_written = 0; - *complete = false; - return Status::Ok(); - } - if (needs_snapshot) { - Status snapshot_status = snapshot(); - if (!snapshot_status.ok()) { - return snapshot_status; - } - std::scoped_lock lock(write_mutex_); - if (version_.load() != pruned_version + 1) { - *bytes_written = 0; - *complete = false; - return Status::Ok(); - } - mapping_pruned_version_ = version_.load(); - } - } - - acquire_snapshot_slot(); - if (opt_.page_store->has_shared_ownership()) { - std::vector<std::pair<uint64_t, uint64_t>> live{ - {0, region_base_for(opt_.page_store->iu_size())}, - }; - std::vector<CommitAnchor> anchors; - Status anchor_status = read_valid_anchors(*opt_.page_store, opt_.page_store->iu_size(), &anchors); - if (!anchor_status.ok()) { - release_snapshot_slot(); - return anchor_status; - } - if (anchors.empty()) { - release_snapshot_slot(); - return Status::corruption("materialize_ownership: current snapshot is unreadable"); - } - for (const CommitAnchor &anchor : anchors) { - if (!collect_live_extents_from_directory(*opt_.page_store, anchor, opt_.page_store->iu_size(), &live)) { - release_snapshot_slot(); - return Status::corruption("materialize_ownership: retained snapshot is unreadable"); - } - } - opt_.page_store->set_materialization_live_extents(std::move(live)); - } - Status status = opt_.page_store->materialize_ownership(bytes_written, complete); - release_snapshot_slot(); - return status; -} - -Status Crowdbtree::prefetch_sparse_pages(std::vector<PrefetchedPage> *out, std::set<uint32_t> *selected_blocks) -{ - selected_blocks->clear(); - const uint32_t iu = opt_.page_store->iu_size(); - const uint64_t block_size = opt_.page_store->block_size(); - std::vector<CommitAnchor> anchors; - Status anchor_status = read_valid_anchors(*opt_.page_store, iu, &anchors); - if (!anchor_status.ok()) { - return anchor_status; - } - if (anchors.empty()) { - return Status::Ok(); - } - - double sparse_threshold = - opt_.merge_gc_block_free_threshold > 0.0 ? opt_.merge_gc_block_free_threshold : kSparseBlockThreshold; - std::vector<std::pair<uint64_t, uint64_t>> current_live; - if (!collect_live_extents_from_directory(*opt_.page_store, anchors.back(), iu, ¤t_live)) { - CRB_LOG_WARN("[{}] compact_sparse_blocks: current anchor unreadable", name_); - return Status::corruption("compact_sparse_blocks: current anchor unreadable"); - } - *selected_blocks = - select_sparse_blocks(current_live, block_size, sparse_threshold, opt_.merge_gc_max_relocation_bytes); - if (selected_blocks->empty()) { - return Status::Ok(); - } - - // Reads stay outside write_mutex_; prepare revalidates each mapping word. - for (uint64_t seg_idx = 0; seg_idx < MappingTable::kMaxSegments; ++seg_idx) { - MappingSegment *seg = mapping_.segment_at(seg_idx); - if (seg == nullptr) { - continue; - } - for (uint32_t i = 0; i < seg->slot_count; ++i) { - uint64_t page_id = (seg_idx * MappingTable::kSegmentSize) + i; - uint64_t word = seg->slots[i].load(std::memory_order_relaxed); - if (!slot_word::is_unloaded(word)) { - continue; - } - uint64_t old_addr = 0; - uint32_t padded_len = 0; - Status decode = opt_.page_store->decode_mapping_location(word, &old_addr, &padded_len); - if (!decode.ok()) { - continue; - } - if (!selected_blocks->contains(static_cast<uint32_t>(old_addr / block_size))) { - continue; - } - std::vector<uint8_t> blob(padded_len); - Status read = opt_.page_store->read_at(old_addr, blob.data(), blob.size()); - if (!read.ok()) { - CRB_LOG_WARN("[{}] compact_sparse_blocks: prefetch read failed for page {}: {}", name_, page_id, - read.to_string()); - continue; - } - out->push_back(PrefetchedPage{.page_id = page_id, .old_word = word, .blob = std::move(blob)}); - } - } + *bytes_written = 0; + *complete = true; return Status::Ok(); } -void Crowdbtree::record_compaction_metrics(const MergeGcStats &stats, uint64_t elapsed_ns) -{ - if (metrics_.merge_gc_blocks_c != nullptr && stats.blocks_selected > 0) { - metrics_.merge_gc_blocks_c->inc_by(stats.blocks_selected); - } - if (metrics_.merge_gc_relocated_c != nullptr && stats.pages_relocated > 0) { - metrics_.merge_gc_relocated_c->inc_by(stats.pages_relocated); - } - if (metrics_.merge_gc_deleted_c != nullptr && stats.blocks_deleted > 0) { - metrics_.merge_gc_deleted_c->inc_by(stats.blocks_deleted); - } - if (metrics_.merge_gc_l != nullptr) { - metrics_.merge_gc_l->observe(elapsed_ns); - } - CRB_LOG_INFO("[{}] compact_sparse_blocks: selected={} relocated={} bytes={} deleted={}", name_, - stats.blocks_selected, stats.pages_relocated, stats.bytes_relocated, stats.blocks_deleted); -} - -Status Crowdbtree::compact_sparse_blocks(MergeGcStats *out_stats) +Status Crowdbtree::compact_sparse_blocks(MergeGcStats *out) { - auto started_at = std::chrono::steady_clock::now(); - if (out_stats == nullptr) { - return Status::invalid_argument("compact_sparse_blocks: null output"); - } - *out_stats = {}; - if (opt_.page_store == nullptr) { - return Status::invalid_argument("compact_sparse_blocks: no page_store"); + if (out == nullptr || opt_.page_store == nullptr) { + return Status::invalid_argument("compaction requires backend and output"); } + *out = {}; if (opt_.page_store->block_size() == 0 || opt_.merge_gc_max_relocation_bytes == 0) { return Status::Ok(); } - - std::vector<PrefetchedPage> prefetched; - std::set<uint32_t> selected_blocks; - Status prefetch = prefetch_sparse_pages(&prefetched, &selected_blocks); - if (!prefetch.ok() || selected_blocks.empty()) { - return prefetch; - } - - PreparedSnapshot prepared; - Status persist = persist_compaction_snapshot(std::move(prefetched), std::move(selected_blocks), &prepared); - if (!persist.ok()) { - return persist; - } - *out_stats = MergeGcStats{ - .blocks_selected = prepared.blocks_selected, - .pages_relocated = prepared.pages_relocated, - .bytes_relocated = prepared.bytes_relocated, - .blocks_deleted = prepared.blocks_deleted, - }; - auto elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - started_at); - record_compaction_metrics(*out_stats, static_cast<uint64_t>(elapsed.count())); - return Status::Ok(); -} - -Status Crowdbtree::open(const Config &opt, std::unique_ptr<Crowdbtree> *out) -{ - Status range_status = opt.key_range.validate(); - if (!range_status.ok()) { - return range_status; - } - if (opt.page_store == nullptr) { - return Status::invalid_argument("open: no page_store"); - } - PageStore *store = opt.page_store; - const uint32_t iu = store->iu_size(); - // Logging is now process-global: the application calls init_logging() - // (via ct_init_logging) at startup before any Crowdbtree::open(). This - // ensures all engine instances share one logger without resetting - // each other's. - CRB_LOG_INFO("[{}] open: iu={} frame_bytes={} store_size={}", opt.name, iu, opt.frame_bytes, store->size()); - // Geometry validation (PT9 §9.2): the pool frame must be IU-aligned. The - // superblock slot is IU-rounded (superblock_slot_bytes), so any IU is supported. - if (iu > 1 && (opt.frame_bytes % iu != 0)) { - return Status::invalid_argument("open: frame_bytes must be IU-aligned"); - } - - auto tree = std::make_unique<Crowdbtree>(opt); - - std::vector<CommitAnchor> anchors; - Status anchor_status = read_valid_anchors(*store, iu, &anchors); - if (!anchor_status.ok()) { - return anchor_status; - } - if (anchors.empty()) { - // No valid snapshot: fresh empty tree (already constructed). - CRB_LOG_INFO("[{}] open: no committed anchor; starting empty", opt.name); - tree->init_metrics(make_metrics_prefix(opt), opt.backend_label); - *out = std::move(tree); - return Status::Ok(); - } - const CommitAnchor &anchor = anchors.back(); - if (anchor.segment_slots != MappingTable::kSegmentSize) { - return Status::corruption("open: anchor segment_slots does not match this build's MappingTable::kSegmentSize"); - } - - // Read + verify the segment directory. The physical extent is IU-padded - // (PT9); read the rounded span but parse over the logical length. - std::vector<uint8_t> dbuf(round_up_to_iu(anchor.segdir_len, iu)); - Status dr = store->read_at(anchor.segdir_addr, dbuf.data(), dbuf.size()); - if (!dr.ok()) { - return dr; - } - if (crowdb::common::crc32c(dbuf.data(), anchor.segdir_len) != anchor.segdir_crc) { - return Status::corruption("open: segment directory CRC mismatch"); - } - std::vector<DirEntry> entries; - Status dds = decode_segment_directory(dbuf.data(), dbuf.size(), &entries); - if (!dds.ok()) { - return dds; - } - - // Drop the freshly-built empty root before installing recovered - // segments. open() is single-threaded (the tree is not yet published), - // so free immediately. - tree->free_subtree(tree->root_page_id_.load(), /*retire=*/false); - - // Lazy recovery: install each segment's packed words verbatim (zero - // decode -- design's point: the mapping table IS the persistent - // structure). Base pages are demand-loaded (and CRC-checked) on first - // access via resident(). - for (const DirEntry &e : entries) { - std::vector<uint8_t> ibuf(round_up_to_iu(e.image_len, iu)); - Status ir = store->read_at(e.image_addr, ibuf.data(), ibuf.size()); - if (!ir.ok()) { - return ir; - } - SegmentImageHeader hdr; - std::vector<uint64_t> words; - Status ids = decode_segment_image(ibuf.data(), ibuf.size(), &hdr, &words); - if (!ids.ok()) { - return ids; - } - // Cross-check the directory's copy of this image's body CRC against - // what was actually read -- catches a directory entry pointing at - // the wrong/stale address (a self-consistent-but-different image - // would otherwise decode cleanly). - if (hdr.body_crc != e.image_crc) { - return Status::corruption("open: segment image CRC does not match its directory entry"); - } - if (hdr.slot_count != MappingTable::kSegmentSize) { - return Status::corruption("open: segment image slot_count does not match kSegmentSize"); - } - tree->mapping_.install_recovered_segment(e.seg_idx, hdr.generation, hdr.live_count, words, e.image_addr, - e.image_len, e.image_crc); - } - - tree->mapping_.set_next_page_id(anchor.next_page_id); - tree->root_page_id_.store(anchor.root_page_id); - tree->routing_fences_trusted_.store(false, std::memory_order_release); - tree->last_applied_slot_.store(anchor.last_applied_slot); - tree->contiguous_slot_.store(anchor.last_applied_slot); - tree->version_.store(anchor.snapshot_seq); - tree->durable_snapshot_last_applied_slot_.store(anchor.last_applied_slot, std::memory_order_release); - tree->durable_snapshot_seq_.store(anchor.snapshot_seq, std::memory_order_release); - tree->leaf_count_.store(anchor.leaf_count, std::memory_order_relaxed); - tree->inner_count_.store(anchor.inner_count, std::memory_order_relaxed); - - // Reconstruct the cached summary from the selected durable root. This is - // startup work only; ordinary reads never scan the tree for statistics. - if (PageBase *root = tree->resident(anchor.root_page_id); root != nullptr) { - PageSummary summary; - if (root->type == page_type::kLeafBase) { - summary = static_cast<LeafBase *>(root)->summary(); - } - else if (root->type == page_type::kInnerBase) { - summary = static_cast<InnerBase *>(root)->summary(); - } - if (summary.exact) { - tree->summary_live_kv_.store(summary.live_kv, std::memory_order_release); - tree->summary_live_key_bytes_.store(summary.live_key_bytes, std::memory_order_release); - tree->summary_live_value_bytes_.store(summary.live_value_bytes, std::memory_order_release); - tree->summary_live_logical_bytes_.store(summary.live_logical_bytes, std::memory_order_release); - tree->summary_reachable_overflow_pages_.store(summary.reachable_overflow_pages, std::memory_order_release); - tree->summary_reachable_page_capacity_bytes_.store(summary.reachable_page_capacity_bytes, - std::memory_order_release); - tree->summary_covered_slot_.store(anchor.last_applied_slot, std::memory_order_release); - tree->summary_root_version_.store(anchor.snapshot_seq, std::memory_order_release); - tree->summary_available_.store(true, std::memory_order_release); - } - // Reopen must keep lazy recovery lazy: the root was loaded only to - // read its persisted summary, so release it before returning. - (void)tree->evict_clean_inner(0); - (void)tree->evict_clean_leaves(0); - } - - CRB_LOG_INFO("[{}] open: recovered seq={} last_applied={} root_pid={} segments={}", opt.name, anchor.snapshot_seq, - anchor.last_applied_slot, anchor.root_page_id, entries.size()); - tree->init_metrics(make_metrics_prefix(opt), opt.backend_label); - *out = std::move(tree); - return Status::Ok(); + return core_->compact_sparse_blocks(out); } - } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/primary_backing.cpp b/lib/crowdb-tree/src/snapshot/primary_backing.cpp new file mode 100644 index 000000000..eacb18733 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/primary_backing.cpp @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "primary_backing.h" + +namespace crowdb::tree +{ +std::shared_ptr<const PrimaryBackingMetadata> +make_primary_backing(const DurableCapturedFrame &frame, const PageState &page, uint64_t coverage, uint64_t gc_floor, + const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + if (frame.type != page_type::kLeafBase || !frame.backing || !frame.backing->reloadable() || !frame.summary || + !frame.stored_overflow || frame.frame_bytes == 0 || frame.logical_bytes == 0) { + return {}; + } + auto proof = detail::reserved_shared<PrimaryBackingMetadata>(metadata); + proof->frame = frame; + auto summary = *frame.summary; + summary.new_overflow_frames = 0; + summary.dropped_tombstones = 0; + proof->frame.summary = summary; + proof->range = page.range; + proof->right_sibling = page.right_sibling; + proof->coverage = coverage; + proof->gc_floor = gc_floor; + return proof; +} + +std::shared_ptr<const PrimaryBackingMetadata> +make_inner_primary_backing(const DurableCapturedFrame &frame, const PublishedPage &page, uint64_t coverage, + uint64_t gc_floor, const std::shared_ptr<detail::AllocationReservation> &metadata) +{ + if (frame.type != page_type::kInnerBase || page.level == 0 || !frame.backing || !frame.backing->reloadable() || + !frame.summary || !frame.summary->inner_fences || frame.frame_bytes == 0 || frame.logical_bytes == 0) { + return {}; + } + auto proof = detail::reserved_shared<PrimaryBackingMetadata>(metadata); + proof->frame = frame; + proof->coverage = coverage; + proof->gc_floor = gc_floor; + proof->inner_route = page.inner; + return proof; +} + +bool can_reuse_inner_primary(const PublishedPage &page, uint64_t pid, const InnerEncodeFences &fences, uint64_t backend, + uint32_t frame_bytes) noexcept +{ + if (backend == 0 || page.level == 0 || page.leaf || !page.inner || !page.primary) { + return false; + } + const auto &proof = *page.primary; + const auto &frame = proof.frame; + if (frame.page_id != pid || frame.type != page_type::kInnerBase || frame.frame_bytes != frame_bytes || + !frame.backing || !frame.backing->reloadable() || frame.backing->backend_id != backend || + frame.backing != page.inner_backing || !frame.summary || !frame.summary->inner_fences) { + return false; + } + const auto &stored = *frame.summary->inner_fences; + if (stored.lower_leaf != fences.lower_leaf || stored.upper_leaf != fences.upper_leaf) { + return false; + } + if (proof.inner_route.lock() == page.inner) { + return true; + } + const auto &decoded = page.inner->frame_proof; + return decoded && decoded->self_page_id == pid && decoded->frame_bytes == frame_bytes && decoded->page_id_fences && + decoded->has_fences == (fences.lower_leaf != kInvalidPageId) && + (!decoded->has_fences || + (decoded->lower_leaf == fences.lower_leaf && decoded->upper_leaf == fences.upper_leaf)); +} + +bool can_reuse_primary(const PublishedPage &page, uint64_t pid, const PageKeyRange &range, uint64_t gc_floor, + uint64_t backend, uint32_t frame_bytes) noexcept +{ + if (backend == 0 || page.level != 0 || !page.leaf || !page.primary || !page.leaf->overlays.empty() || + page.leaf->bases.size() != 1) { + return false; + } + const auto &proof = *page.primary; + const auto &frame = proof.frame; + const auto &base = page.leaf->bases.front(); + // Encoding keeps one winner per key. An exact live-only summary proves + // that advancing retention cannot remove a cell from this stored frame. + const bool live_only = + frame.summary && frame.summary->live_counts_known && frame.summary->encoded_entries == frame.summary->live_keys; + const bool compatible_floor = proof.gc_floor == gc_floor || (proof.gc_floor < gc_floor && live_only); + return frame.page_id == pid && frame.type == page_type::kLeafBase && frame.frame_bytes == frame_bytes && + frame.backing && frame.backing->reloadable() && frame.backing->backend_id == backend && + frame.backing == base.backing && frame.summary && frame.stored_overflow && compatible_floor && + proof.right_sibling == page.leaf->right_sibling && proof.range.lower == range.lower && + proof.range.upper == range.upper && base.range.lower == range.lower && base.range.upper == range.upper && + (!base.max_slot || *base.max_slot >= proof.coverage); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/primary_backing.h b/lib/crowdb-tree/src/snapshot/primary_backing.h new file mode 100644 index 000000000..10af25018 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/primary_backing.h @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/captured_write.h" + +namespace crowdb::tree +{ +// This receipt owns one primary and its exact overflow dependencies, never an +// entire checkpoint. Route and base must match. A higher GC floor remains +// compatible when the exact frame summary proves that no tombstones remain. +struct PrimaryBackingMetadata +{ + DurableCapturedFrame frame; + PageKeyRange range; + uint64_t coverage = 0; + uint64_t gc_floor = 0; + uint64_t right_sibling = kInvalidPageId; + // Allows a resident route to be identified without retaining its cache + // payload. + std::weak_ptr<const InnerPageState> inner_route; +}; + +struct PrimaryReuseOutput +{ + explicit PrimaryReuseOutput(const std::shared_ptr<detail::AllocationReservation> &metadata) + : dependencies(detail::ReservationAllocator<ReusedOverflowDependency>(metadata)) + { + } + + std::shared_ptr<const PrimaryBackingMetadata> primary; + detail::ReservedVector<ReusedOverflowDependency> dependencies; +}; + +std::shared_ptr<const PrimaryBackingMetadata> +make_primary_backing(const DurableCapturedFrame &frame, const PageState &page, uint64_t coverage, uint64_t gc_floor, + const std::shared_ptr<detail::AllocationReservation> &metadata); + +std::shared_ptr<const PrimaryBackingMetadata> +make_inner_primary_backing(const DurableCapturedFrame &frame, const PublishedPage &page, uint64_t coverage, + uint64_t gc_floor, const std::shared_ptr<detail::AllocationReservation> &metadata); + +bool can_reuse_inner_primary(const PublishedPage &page, uint64_t pid, const InnerEncodeFences &fences, uint64_t backend, + uint32_t frame_bytes) noexcept; + +bool can_reuse_primary(const PublishedPage &page, uint64_t pid, const PageKeyRange &range, uint64_t gc_floor, + uint64_t backend, uint32_t frame_bytes) noexcept; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_byte_export.cpp b/lib/crowdb-tree/src/snapshot/published_byte_export.cpp new file mode 100644 index 000000000..4e37a86d4 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_byte_export.cpp @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core.h" +#include "snapshot/published_byte_export_internal.h" +#include "snapshot/snapshot_wire.h" + +namespace crowdb::tree +{ +void PublishedByteExport::State::encode_header() +{ + snapshot_detail::put_u32(header.data(), 0x4e535443U); + snapshot_detail::put_u32(header.data() + 4, 1); + header[8] = static_cast<uint8_t>(format); + snapshot_detail::put_u64(header.data() + 9, info.slot); + if (format == snapshot_format::kPortable) { + snapshot_detail::put_u64(header.data() + 17, count); + header_size = 25; + } + else { + snapshot_detail::put_u64(header.data() + 17, root); + snapshot_detail::put_u64(header.data() + 25, count); + header_size = 33; + } +} + +Status PublishedByteExport::create(const std::shared_ptr<TreeCore> &core, snapshot_format format, size_t chunk_bytes, + std::shared_ptr<PublishedByteExport> *out) +{ + return create(core, format, chunk_bytes, out, KeyRange::unbounded()); +} + +Status PublishedByteExport::create(const std::shared_ptr<TreeCore> &core, snapshot_format format, size_t chunk_bytes, + std::shared_ptr<PublishedByteExport> *out, const KeyRange &range) +try { + if (!core || out == nullptr || *out || chunk_bytes > kSnapshotChunkBytes) { + return Status::invalid_argument("snapshot export needs an empty output and a bounded chunk size"); + } + if (!range.validate().ok() || (format == snapshot_format::kNative && range.is_bounded())) { + return Status::invalid_argument("a selected key range requires portable export"); + } + if (format != snapshot_format::kPortable && format != snapshot_format::kNative) { + return Status::not_supported("unknown snapshot wire format"); + } + using detail::shared_allocation_bound; + const auto bound = shared_allocation_bound<State>() + shared_allocation_bound<PublishedByteExport>() + + shared_allocation_bound<detail::AllocationReservation>() + sizeof(SnapshotExport); + auto credit = detail::ResourceBudget::try_reserve(core->binding()->account(), detail::BudgetClass::kRouting, bound); + if (!credit) { + return Status::resource_exhausted("snapshot export control reserve unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + auto state = detail::reserved_shared<State>(metadata); + state->metadata = metadata; + state->binding = core->binding(); + state->format = format; + state->info.chunk_bytes = chunk_bytes == 0 ? kSnapshotChunkBytes : chunk_bytes; + Status status; + if (format == snapshot_format::kNative) { + status = PublishedNativeCursor::create(core, {}, &state->native); + if (status.ok()) { + const auto &native = state->native->info(); + state->info.slot = native.at_slot; + state->root = native.root_page_id; + state->count = native.frame_count; + } + } + else { + status = core->flush(); + if (status.ok()) { + status = core->capture_snapshot(&state->snapshot); + } + if (status.ok()) { + state->info.slot = state->snapshot->view()->coverage; + } + } + if (status.ok()) { + status = state->preflight(range); + } + if (!status.ok()) { + return status; + } + *out = detail::reserved_construct<PublishedByteExport>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) PublishedByteExport(std::move(state)); }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot export allocation failed"); +} + +const SnapshotByteInfo &PublishedByteExport::info() const noexcept +{ + return state_->info; +} + +const std::shared_ptr<detail::AllocationReservation> &PublishedByteExport::metadata() const noexcept +{ + return state_->metadata; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_byte_export.h b/lib/crowdb-tree/src/snapshot/published_byte_export.h new file mode 100644 index 000000000..534ea42a8 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_byte_export.h @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/snapshot/snapshot_io.h" +#include "snapshot/published_native.h" +#include "snapshot/published_snapshot_entries.h" + +namespace crowdb::tree +{ +struct SnapshotByteInfo +{ + uint64_t slot = 0; + size_t total_bytes = 0; + size_t chunk_bytes = 0; + size_t offset = 0; + uint32_t final_crc = 0; +}; + +class PublishedByteExport +{ + struct State; + + public: + static Status create(const std::shared_ptr<TreeCore> &core, snapshot_format format, size_t chunk_bytes, + std::shared_ptr<PublishedByteExport> *out); + static Status create(const std::shared_ptr<TreeCore> &core, snapshot_format format, size_t chunk_bytes, + std::shared_ptr<PublishedByteExport> *out, const KeyRange &range); + Status next(SnapshotChunk *out, bool *done); + [[nodiscard]] const SnapshotByteInfo &info() const noexcept; + [[nodiscard]] const std::shared_ptr<detail::AllocationReservation> &metadata() const noexcept; + + private: + explicit PublishedByteExport(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_byte_export_internal.h b/lib/crowdb-tree/src/snapshot/published_byte_export_internal.h new file mode 100644 index 000000000..a65ba1109 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_byte_export_internal.h @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/published_byte_export.h" + +#include <array> + +namespace crowdb::tree +{ +struct PublishedByteExport::State +{ + ~State() + { + if (metadata) { + metadata->seal(); + } + } + enum class Phase : uint8_t { Header, Records, Trailer, Done }; + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<PublishedSnapshotContext> snapshot; + std::shared_ptr<PublishedSnapshotEntries> entries; + std::shared_ptr<PublishedNativeCursor> native; + snapshot_format format = snapshot_format::kPortable; + SnapshotByteInfo info; + uint64_t count = 0; + uint64_t root = kInvalidPageId; + std::array<uint8_t, 33> header{}; + size_t header_size = 0; + std::array<uint8_t, 17> fields{}; + std::array<uint8_t, 4> trailer{}; + std::optional<PublishedScanEntry> entry; + std::optional<NativeFrameView> frame; + bool source_done = false; + Phase phase = Phase::Header; + unsigned part = 0; + size_t piece_offset = 0; + uint32_t emitted_crc = 0; + std::atomic<bool> busy{false}; + Status failure; + + void encode_header(); + Status preflight(const KeyRange &range); + Status portable_preflight(const KeyRange &range); + Status native_preflight(); + Status next_piece(std::span<const uint8_t> *out); + Status record_piece(std::span<const uint8_t> *out); + void consumed_piece(); + Status next(SnapshotChunk *out, bool *done); +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_byte_export_next.cpp b/lib/crowdb-tree/src/snapshot/published_byte_export_next.cpp new file mode 100644 index 000000000..b4dff80f8 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_byte_export_next.cpp @@ -0,0 +1,228 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "snapshot/published_byte_export_internal.h" +#include "snapshot/snapshot_wire.h" + +#include <cstring> + +namespace crowdb::tree +{ +namespace +{ +struct ChunkOutput +{ + ChunkOutput(size_t capacity, const std::shared_ptr<detail::AllocationReservation> &metadata) + : bytes(capacity, detail::ReservationAllocator<uint8_t, AllocationKind::kPayload>(metadata)) + { + } + + detail::ReservedVector<uint8_t, AllocationKind::kPayload> bytes; +}; +} // namespace + +Status PublishedByteExport::State::record_piece(std::span<const uint8_t> *out) +{ + if (format == snapshot_format::kNative) { + while (!frame && !source_done) { + auto status = native->next(&frame, &source_done); + if (!status.ok()) { + return status; + } + } + if (frame) { + snapshot_detail::put_u64(fields.data(), frame->page_id); + snapshot_detail::put_u32(fields.data() + 8, static_cast<uint32_t>(frame->bytes.size())); + *out = part == 0 ? std::span<const uint8_t>(fields).first(12) : frame->bytes; + return Status::Ok(); + } + } + else { + while (!entry && !source_done) { + auto status = entries->next(&entry, &source_done); + if (!status.ok()) { + return status; + } + } + if (entry) { + auto status = snapshot_detail::entry_fields(*entry, &fields); + if (!status.ok()) { + return status; + } + switch (part) { + case 0: + *out = std::span<const uint8_t>(fields).first(4); + break; + case 1: + *out = snapshot_detail::bytes(Slice(entry->key)); + break; + case 2: + *out = std::span<const uint8_t>(fields).subspan(4); + break; + default: + *out = snapshot_detail::bytes(entry->cell.value); + break; + } + return Status::Ok(); + } + } + phase = Phase::Trailer; + return next_piece(out); +} + +Status PublishedByteExport::State::next_piece(std::span<const uint8_t> *out) +{ + switch (phase) { + case Phase::Header: + *out = std::span(header).first(header_size); + return Status::Ok(); + case Phase::Records: + return record_piece(out); + case Phase::Trailer: + if (emitted_crc != info.final_crc || info.offset != info.total_bytes - 4 + piece_offset) { + return Status::corruption("snapshot emission differs from its fixed preflight stream"); + } + *out = trailer; + return Status::Ok(); + case Phase::Done: + *out = {}; + return Status::Ok(); + } + return Status::internal_error("invalid snapshot wire phase"); +} + +void PublishedByteExport::State::consumed_piece() +{ + piece_offset = 0; + switch (phase) { + case Phase::Header: + phase = Phase::Records; + break; + case Phase::Records: + ++part; + if (part == (format == snapshot_format::kNative ? 2U : 4U)) { + part = 0; + entry.reset(); + frame.reset(); + if (source_done) { + phase = Phase::Trailer; + } + } + break; + case Phase::Trailer: + phase = Phase::Done; + break; + case Phase::Done: + break; + } +} + +Status PublishedByteExport::State::next(SnapshotChunk *out, bool *done) +{ + if (phase == Phase::Done) { + *done = true; + return Status::Ok(); + } + // Cursor advancement may change stream fields, but never the capacity + // allocated for this output window. Keep its subtraction bound local. + const size_t capacity = info.chunk_bytes; + if (capacity == 0 || capacity > kSnapshotChunkBytes || info.total_bytes < 4 || info.offset > info.total_bytes) { + return Status::corruption("snapshot output has invalid fixed window geometry"); + } + using detail::shared_allocation_bound; + const auto bound = + detail::AllocationBound(shared_allocation_bound<ChunkOutput>() + + shared_allocation_bound<detail::AllocationReservation>() + kSnapshotChunkHandleBytes) + .add(capacity) + .value(); + if (!bound) { + return Status::resource_exhausted("snapshot chunk allocation bound overflow"); + } + auto credit = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kRouting, *bound); + if (!credit) { + return Status::resource_exhausted("snapshot chunk reserve unavailable"); + } + auto budget = detail::AllocationReservation::create(std::move(*credit)); + auto buffer = detail::reserved_shared<ChunkOutput>(budget, capacity, budget); + size_t used = 0; + while (used < capacity && phase != Phase::Done) { + std::span<const uint8_t> piece; + auto status = next_piece(&piece); + if (!status.ok()) { + return status; + } + if (piece_offset > piece.size()) { + return Status::corruption("snapshot piece cursor exceeds its owner"); + } + const auto take = std::min(capacity - used, piece.size() - piece_offset); + if (info.offset > info.total_bytes || take > info.total_bytes - info.offset) { + return Status::corruption("snapshot emission exceeds its preflight byte count"); + } + if (used > buffer->bytes.size() || take > buffer->bytes.size() - used || take > kSnapshotChunkBytes) { + return Status::corruption("snapshot piece exceeds its allocated output window"); + } + const auto bytes = piece.subspan(piece_offset, take); + if (take != 0) { + const auto copy_bytes = static_cast<uint32_t>(take); + std::memcpy(buffer->bytes.data() + used, bytes.data(), copy_bytes); + record_work(budget, TreeWorkCounter::MaterializeCopyBytes, copy_bytes, TreeWorkCause::Snapshot); + } + if (phase != Phase::Trailer) { + emitted_crc = snapshot_detail::crc_bytes(emitted_crc, bytes); + } + used += take; + piece_offset += take; + info.offset += take; + if (piece_offset == piece.size()) { + consumed_piece(); + } + } + *done = phase == Phase::Done; + *out = SnapshotChunk{.bytes = std::span<const uint8_t>(buffer->bytes).first(used), + .owner = std::move(buffer), + .allocation_budget = std::move(budget)}; + if (*done) { + entries.reset(); + native.reset(); + snapshot.reset(); + } + return Status::Ok(); +} + +Status PublishedByteExport::next(SnapshotChunk *out, bool *done) +{ + if (out == nullptr || done == nullptr) { + return Status::invalid_argument("snapshot chunk requires result outputs"); + } + *out = {}; + *done = false; + if (state_->busy.exchange(true, std::memory_order_acq_rel)) { + return Status::unavailable("snapshot exporter already has an active window"); + } + + struct Leave + { + std::atomic<bool> &busy; + + ~Leave() + { + busy.store(false, std::memory_order_release); + } + } leave{state_->busy}; + + if (!state_->failure.ok()) { + return state_->failure; + } + try { + auto status = state_->next(out, done); + if (!status.ok()) { + state_->failure = status.without_message(); + } + return status; + } + catch (const std::bad_alloc &) { + state_->failure = Status::resource_exhausted(); + return state_->failure; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_byte_export_plan.cpp b/lib/crowdb-tree/src/snapshot/published_byte_export_plan.cpp new file mode 100644 index 000000000..421222c29 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_byte_export_plan.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/published_byte_export_internal.h" +#include "snapshot/snapshot_crc.h" +#include "snapshot/snapshot_wire.h" + +namespace crowdb::tree +{ +Status PublishedByteExport::State::portable_preflight(const KeyRange &range) +{ + auto status = PublishedSnapshotEntries::create(snapshot, &entries, range); + if (!status.ok()) { + return status; + } + uint32_t body_crc = 0; + size_t body_bytes = 0; + bool complete = false; + while (!complete) { + std::optional<PublishedScanEntry> item; + status = entries->next(&item, &complete); + if (!status.ok()) { + return status; + } + if (!item) { + continue; + } + status = snapshot_detail::entry_fields(*item, &fields); + if (!status.ok()) { + return status; + } + const auto key = snapshot_detail::bytes(Slice(item->key)); + const auto value = snapshot_detail::bytes(item->cell.value); + const auto bytes = uint64_t{17} + key.size() + value.size(); + if (count == UINT64_MAX || bytes > SIZE_MAX - body_bytes - 29) { + return Status::resource_exhausted("portable stream length overflow"); + } + ++count; + body_bytes += bytes; + body_crc = snapshot_detail::crc_bytes(body_crc, std::span(fields).first(4)); + body_crc = snapshot_detail::crc_bytes(body_crc, key); + body_crc = snapshot_detail::crc_bytes(body_crc, std::span(fields).subspan(4)); + body_crc = snapshot_detail::crc_bytes(body_crc, value); + } + encode_header(); + const auto header_crc = snapshot_detail::crc_bytes(0, std::span(header).first(header_size)); + info.final_crc = snapshot_detail::crc_concatenate(header_crc, body_crc, body_bytes); + info.total_bytes = header_size + body_bytes + 4; + entries.reset(); + snapshot->lane()->drain(); + return PublishedSnapshotEntries::create(snapshot, &entries, range); +} + +Status PublishedByteExport::State::native_preflight() +{ + encode_header(); + uint32_t crc = snapshot_detail::crc_bytes(0, std::span(header).first(header_size)); + size_t length = header_size; + bool complete = false; + while (!complete) { + std::optional<NativeFrameView> item; + auto status = native->next(&item, &complete); + if (!status.ok()) { + return status; + } + if (!item) { + continue; + } + if (item->bytes.size() > UINT32_MAX || length > SIZE_MAX - 16 || item->bytes.size() > SIZE_MAX - length - 16) { + return Status::resource_exhausted("native stream length overflow"); + } + snapshot_detail::put_u64(fields.data(), item->page_id); + snapshot_detail::put_u32(fields.data() + 8, static_cast<uint32_t>(item->bytes.size())); + crc = snapshot_detail::crc_bytes(crc, std::span(fields).first(12)); + crc = snapshot_detail::crc_bytes(crc, item->bytes); + length += 12 + item->bytes.size(); + } + info.total_bytes = length + 4; + info.final_crc = crc; + return native->rewind(); +} + +Status PublishedByteExport::State::preflight(const KeyRange &range) +{ + auto status = format == snapshot_format::kPortable ? portable_preflight(range) : native_preflight(); + if (status.ok()) { + snapshot_detail::put_u32(trailer.data(), info.final_crc); + } + return status; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_native.cpp b/lib/crowdb-tree/src/snapshot/published_native.cpp new file mode 100644 index 000000000..8280371d9 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_native.cpp @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/checkpoint_backend.h" +#include "backend/local/local_backend_state.h" +#include "btree/allocation_bound.h" +#include "btree/tree_core.h" +#include "snapshot/native_frame_read.h" +#include "snapshot/native_plan.h" +#include "snapshot/operation_wait.h" +#include "snapshot/published_native_internal.h" +#include "snapshot/published_snapshot_budget.h" + +namespace crowdb::tree +{ +namespace +{ +Status plan_native(const std::shared_ptr<PublishedSnapshotContext> &snapshot, CapturedFrameLimits frames, + NativeLayoutLimits limits, uint64_t backend_id, uint64_t incarnation, size_t load_bound, + std::shared_ptr<NativeLayoutPlanner> *out) +{ + frames.plan_only = true; + const auto bound = detail::AllocationBound(load_bound) + .add(CapturedFrameStream::allocation_bound(frames)) + .add(NativeLayoutPlanner::allocation_bound(snapshot->view()->next_page_id, limits)) + .add(NativePlanOperation::allocation_bound()) + .value(); + if (!bound) { + return Status::resource_exhausted("native layout admission bound overflow"); + } + SnapshotReadWindow window; + auto status = snapshot->prepare_window(*bound, &window); + if (!status.ok()) { + return status; + } + frames.encode.metadata = window.budget.bytes; + auto planner = detail::reserved_shared<NativeLayoutPlanner>(window.budget.bytes, snapshot->view(), limits, + window.budget.bytes, backend_id, incarnation); + std::shared_ptr<CapturedFrameStream> stream; + status = CapturedFrameStream::create(snapshot->view(), window.loader, snapshot->binding()->cpu(), snapshot->lane(), + frames, {}, &stream); + if (!status.ok()) { + return status; + } + std::shared_ptr<NativePlanOperation> plan; + status = NativePlanOperation::create(std::move(stream), planner, snapshot->binding()->cpu(), snapshot->lane(), + window.budget.bytes, &plan); + if (status.ok()) { + status = detail::wait_snapshot_operation(plan); + if (!status.ok()) { + status = plan->result(); + } + } + if (status.ok()) { + *out = std::move(planner); + } + window.seal(); + return status; +} + +std::optional<size_t> loader_bound(const PublishedSnapshotContext &snapshot, size_t depth, size_t sources) +{ + // This traversal loads primary frames only; old overflow frames have + // their own one-frame reader and are never materialized as full values. + auto loads = snapshot.load_limits(); + loads.max_overflow_pages = 1; + return AsyncStorePageLoader::read_reserve_bound(loads, depth, snapshot.binding()->cpu()->worker_count(), sources, 0, + 0); +} +} // namespace + +std::optional<size_t> PublishedNativeCursor::State::control_bound(const CapturedFrameLimits &limits) noexcept +{ + using detail::shared_allocation_bound; + return detail::AllocationBound( + shared_allocation_bound<State>() + shared_allocation_bound<PublishedNativeCursor>() + + sizeof(NativeFrameIterator) + shared_allocation_bound<detail::AllocationReservation>() + + shared_allocation_bound<SnapshotPageLoader>() + shared_allocation_bound<ReservedOverflowIds>() + + shared_allocation_bound<PublicationRequest>() + shared_allocation_bound<InstallRecord>() + + TreeOperationLifetime::ticket_allocation_bound() + PublicationCoordinator::empty_commit_reserve_bytes()) + .add(CapturedFrameStream::allocation_bound(limits)) + .value(); +} + +Status PublishedNativeCursor::create(const std::shared_ptr<TreeCore> &core, NativeLayoutLimits limits, + std::shared_ptr<PublishedNativeCursor> *out) +try { + if (!core || out == nullptr || *out || !NativeLayoutPlanner::allocation_bound(0, limits)) { + return Status::invalid_argument("native cursor requires bounded admission and an empty output"); + } + auto status = core->flush(); + if (!status.ok()) { + return status; + } + std::shared_ptr<PublishedSnapshotContext> snapshot; + status = core->capture_snapshot(&snapshot); + if (!status.ok()) { + return status; + } + CapturedFrameLimits frames; + frames.encode.frame_bytes = core->config().frame_bytes; + frames.encode.work_metrics = core->publisher().work_metrics(); + frames.encode.cause = TreeWorkCause::Snapshot; + frames.encode.inline_value_bytes = + core->config().max_inline_value != 0 ? core->config().max_inline_value : frames.encode.frame_bytes / 4; + const auto reads = snapshot->read_limits(); + frames.encode.max_sources = reads.max_page_sources; + frames.page_metadata_bytes = reads.page_metadata_bytes; + const auto root = snapshot->view()->mapping.get(snapshot->view()->root_page_id); + frames.max_depth = root ? root->level + 1ULL : 1; + frames.max_pages = limits.max_frames; + if (frames.max_depth > 64) { + return Status::resource_exhausted("native root exceeds traversal depth bound"); + } + const auto leaves = snapshot_leaf_window(*snapshot->view(), frames.encode.max_entries); + const auto loads = + leaves ? loader_bound(*snapshot, frames.max_depth, std::max(frames.encode.max_sources, *leaves)) : std::nullopt; + const auto controls = State::control_bound(frames); + if (!loads || !controls) { + return Status::resource_exhausted("native cursor admission bound overflow"); + } + uint64_t backend_id = 0; + uint64_t incarnation = 0; + if (const auto &local = core->local_backend()) { + backend_id = local->ledger->limits().backend_id; + incarnation = local->ledger->limits().generation; + } + else if (const auto &chunk = core->chunk_backend()) { + backend_id = chunk->backend_identity(); + incarnation = chunk->incarnation(); + } + std::shared_ptr<NativeLayoutPlanner> layout; + status = plan_native(snapshot, frames, limits, backend_id, incarnation, *loads, &layout); + if (!status.ok()) { + return status; + } + auto credit = + detail::ResourceBudget::try_reserve(snapshot->binding()->account(), detail::BudgetClass::kRouting, *controls); + if (!credit) { + return Status::resource_exhausted("native cursor control reserve unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + auto state = detail::reserved_shared<State>(metadata); + state->metadata = metadata; + state->snapshot = std::move(snapshot); + state->layout = std::move(layout); + state->old = state->layout->old_frames().begin(); + state->limits = std::move(frames); + state->limits.encode.metadata = metadata; + state->read_bound = *loads; + state->loader = detail::reserved_shared<SnapshotPageLoader>(metadata); + const auto &view = state->snapshot->view(); + const auto count = state->layout->new_overflow_frames(); + const auto end = view->next_page_id + count; + state->info = {.root_page_id = view->root_page_id, + .at_slot = view->coverage, + .next_page_id = end, + .frame_count = state->layout->frame_count(), + .frame_count_known = true}; + status = state->restart(); + if (!status.ok()) { + return status; + } + auto result = detail::reserved_construct<PublishedNativeCursor>( + metadata, [&](auto *storage) { ::new (static_cast<void *>(storage)) PublishedNativeCursor(state); }); + // All persistent controls are prepared before the shared PID authority + // advances. A changed publication rejects this exact layout proof. + status = core->reserve_snapshot_ids(*view, count, metadata); + if (status.ok()) { + *out = std::move(result); + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("native cursor allocation failed"); +} + +const NativeStreamInfo &PublishedNativeCursor::info() const noexcept +{ + return state_->info; +} + +const std::shared_ptr<detail::AllocationReservation> &PublishedNativeCursor::metadata() const noexcept +{ + return state_->metadata; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_native.h b/lib/crowdb-tree/src/snapshot/published_native.h new file mode 100644 index 000000000..0627325e5 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_native.h @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/snapshot/native_frame.h" +#include "snapshot/native_layout.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +// Host-thread cursor over a fixed immutable generation. Each next() owns a +// separate charged output window; idle cursors retain no I/O reservation. +class PublishedNativeCursor +{ + struct State; + + public: + static Status create(const std::shared_ptr<TreeCore> &core, NativeLayoutLimits limits, + std::shared_ptr<PublishedNativeCursor> *out); + Status next(std::optional<NativeFrameView> *out, bool *complete); + Status rewind(); + [[nodiscard]] const NativeStreamInfo &info() const noexcept; + [[nodiscard]] const std::shared_ptr<detail::AllocationReservation> &metadata() const noexcept; + + private: + explicit PublishedNativeCursor(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_native_collect.cpp b/lib/crowdb-tree/src/snapshot/published_native_collect.cpp new file mode 100644 index 000000000..9e45d9464 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_native_collect.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/published_native_collect.h" + +#include "btree/allocation_bound.h" +#include "btree/tree_core.h" + +namespace crowdb::tree +{ +namespace +{ +struct CopiedFrameOwner +{ + CopiedFrameOwner(std::shared_ptr<detail::AllocationReservation> metadata, size_t capacity) + : bytes(std::move(metadata), capacity, AllocationKind::kFrame) + { + } + + AllocationLease bytes; +}; + +Status measure_frames(PublishedNativeCursor &cursor, size_t max_bytes, size_t *bytes) +{ + detail::AllocationBound bound(detail::shared_allocation_bound<detail::AllocationReservation>()); + bound.add_product(cursor.info().frame_count, + (sizeof(NativeFrame) * 2) + detail::shared_allocation_bound<CopiedFrameOwner>()); + if (!bound.value() || *bound.value() > max_bytes) { + return Status::resource_exhausted("native compatibility result metadata exceeds its capacity limit"); + } + bool done = false; + while (!done) { + std::optional<NativeFrameView> frame; + auto status = cursor.next(&frame, &done); + if (!status.ok()) { + return status; + } + if (frame) { + bound.add(frame->bytes.size()); + if (!bound.value() || *bound.value() > max_bytes) { + return Status::resource_exhausted("native compatibility result exceeds its capacity limit"); + } + } + } + *bytes = *bound.value(); + return cursor.rewind(); +} + +Status copy_frames(PublishedNativeCursor &cursor, const std::shared_ptr<detail::AllocationReservation> &metadata, + std::vector<NativeFrame> *result) +{ + bool done = false; + while (!done) { + std::optional<NativeFrameView> frame; + auto status = cursor.next(&frame, &done); + if (!status.ok()) { + return status; + } + if (!frame) { + continue; + } + if (result->size() == cursor.info().frame_count) { + return Status::corruption("native compatibility result exceeds its planned count"); + } + NativeFrame value{.page_id = frame->page_id, + .frame = std::vector<uint8_t>(frame->bytes.begin(), frame->bytes.end()), + .durable_addr = kNoAddr, + .durable_plen = 0, + .inherited = false}; + value.allocation_owner = detail::reserved_shared<CopiedFrameOwner>(metadata, metadata, value.frame.capacity()); + result->push_back(std::move(value)); + } + return result->size() == cursor.info().frame_count + ? Status::Ok() + : Status::corruption("native compatibility result omits a planned frame"); +} +} // namespace + +Status collect_published_frames(const std::shared_ptr<TreeCore> &core, size_t max_bytes, std::vector<NativeFrame> *out, + NativeStreamInfo *info) +try { + if (!core || max_bytes == 0 || out == nullptr || info == nullptr) { + return Status::invalid_argument("native compatibility collection requires bounded inputs and outputs"); + } + std::shared_ptr<PublishedNativeCursor> cursor; + auto status = PublishedNativeCursor::create(core, {}, &cursor); + if (!status.ok()) { + return status; + } + size_t bytes = 0; + status = measure_frames(*cursor, max_bytes, &bytes); + if (!status.ok()) { + return status; + } + auto credit = detail::ResourceBudget::try_reserve(core->binding()->account(), detail::BudgetClass::kRouting, bytes); + if (!credit) { + return Status::resource_exhausted("native compatibility result reserve unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + AllocationLease container; + std::vector<NativeFrame> result; + result.reserve(cursor->info().frame_count); + container = AllocationLease(metadata, result.capacity() * sizeof(NativeFrame), AllocationKind::kMetadata); + status = copy_frames(*cursor, metadata, &result); + if (!status.ok()) { + return status; + } + metadata->seal(); + *info = cursor->info(); + out->swap(result); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("native compatibility collection allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_native_collect.h b/lib/crowdb-tree/src/snapshot/published_native_collect.h new file mode 100644 index 000000000..7494a0d02 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_native_collect.h @@ -0,0 +1,14 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/published_native.h" + +namespace crowdb::tree +{ +// Compatibility copy boundary. During construction all capacities are charged. +// On successful publication the outer vector belongs to the caller; each raw +// frame keeps its own engine payload charge through allocation_owner. +Status collect_published_frames(const std::shared_ptr<TreeCore> &core, size_t max_bytes, std::vector<NativeFrame> *out, + NativeStreamInfo *info); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_native_internal.h b/lib/crowdb-tree/src/snapshot/published_native_internal.h new file mode 100644 index 000000000..037a9388f --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_native_internal.h @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/published_native.h" + +namespace crowdb::tree +{ +struct PublishedNativeCursor::State +{ + ~State() + { + if (metadata) { + metadata->seal(); + } + } + + std::shared_ptr<PublishedSnapshotContext> snapshot; + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<SnapshotPageLoader> loader; + std::shared_ptr<CapturedFrameStream> stream; + std::shared_ptr<const NativeLayoutPlanner> layout; + OverflowBackingSet::Pages::const_iterator old; + CapturedFrameLimits limits; + NativeStreamInfo info; + size_t read_bound = 0; + uint64_t emitted = 0; + bool generated = false; + std::atomic<bool> busy{false}; + Status failure; + + Status restart(); + static std::optional<size_t> control_bound(const CapturedFrameLimits &limits) noexcept; + Status generated_frame(const SnapshotReadWindow &window, std::optional<NativeFrameView> *out); + Status old_frame(const SnapshotReadWindow &window, std::optional<NativeFrameView> *out); + Status next(std::optional<NativeFrameView> *out, bool *complete); +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_native_next.cpp b/lib/crowdb-tree/src/snapshot/published_native_next.cpp new file mode 100644 index 000000000..4bac0b653 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_native_next.cpp @@ -0,0 +1,177 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "snapshot/native_frame_read.h" +#include "snapshot/operation_wait.h" +#include "snapshot/published_native_internal.h" + +namespace crowdb::tree +{ +namespace +{ +struct NativeOutput +{ + NativeOutput(std::span<const uint8_t> frame, const std::shared_ptr<detail::AllocationReservation> &metadata) + : bytes(frame.begin(), frame.end(), detail::ReservationAllocator<uint8_t, AllocationKind::kFrame>(metadata)) + { + } + + detail::ReservedVector<uint8_t, AllocationKind::kFrame> bytes; +}; + +struct ActiveLoader +{ + std::shared_ptr<SnapshotPageLoader> loader; + + ~ActiveLoader() + { + loader->end(); + } +}; +} // namespace + +Status PublishedNativeCursor::State::generated_frame(const SnapshotReadWindow &window, + std::optional<NativeFrameView> *out) +{ + auto status = loader->begin(window.loader); + if (!status.ok()) { + return status; + } + ActiveLoader active{loader}; + status = detail::wait_snapshot_operation(stream); + if (!status.ok()) { + return stream->result().status; + } + const auto &result = stream->result(); + generated = result.complete; + if (result.frame) { + const auto &frame = *result.frame; + if (!frame.buffer || frame.info.bytes == 0 || frame.info.bytes > frame.buffer->bytes.size()) { + return Status::corruption("native encoder returned an invalid byte window"); + } + // The encoder deliberately reuses one buffer. Copy only this frame + // into an independently charged owner before advancing that buffer. + auto bytes = detail::reserved_shared<NativeOutput>( + window.budget.bytes, std::span<const uint8_t>(frame.buffer->bytes.data(), frame.info.bytes), + window.budget.bytes); + *out = NativeFrameView{.page_id = frame.info.page_id, + .bytes = bytes->bytes, + .owner = std::move(bytes), + .durable_addr = kNoAddr, + .durable_plen = 0, + .inherited = false}; + } + else if (!generated) { + return Status::corruption("native encoder returned neither a frame nor completion"); + } + if (generated) { + stream.reset(); + } + return Status::Ok(); +} + +Status PublishedNativeCursor::State::old_frame(const SnapshotReadWindow &window, std::optional<NativeFrameView> *out) +{ + std::shared_ptr<NativeFrameRead> read; + auto status = NativeFrameRead::create(old->first, old->second, window.backend, snapshot->binding()->cpu(), + snapshot->lane(), snapshot->load_limits(), window.budget, &read); + if (status.ok()) { + status = detail::wait_snapshot_operation(read); + if (!status.ok()) { + return read->result().status; + } + } + if (status.ok()) { + if (!read->result().frame) { + return Status::corruption("native old-frame reader produced no frame"); + } + *out = read->result().frame; + ++old; + } + return status; +} + +Status PublishedNativeCursor::State::next(std::optional<NativeFrameView> *out, bool *complete) +{ + if (!failure.ok()) { + return failure; + } + if (generated && old == layout->old_frames().end()) { + if (emitted != info.frame_count) { + return Status::corruption("native output count differs from its layout proof"); + } + *complete = true; + return Status::Ok(); + } + auto bound = detail::AllocationBound(read_bound) + .add(NativeFrameRead::allocation_bound(snapshot->load_limits())) + .add(detail::shared_allocation_bound<NativeOutput>()) + .add(limits.encode.frame_bytes) + .value(); + if (!bound) { + return Status::resource_exhausted("native frame window bound overflow"); + } + SnapshotReadWindow window; + auto status = snapshot->prepare_window(*bound, &window); + if (!status.ok()) { + return status; + } + if (!generated) { + status = generated_frame(window, out); + } + else { + status = old_frame(window, out); + } + if (!status.ok()) { + return status; + } + if (*out) { + if (emitted == info.frame_count) { + out->reset(); + return Status::corruption("native output exceeds its planned frame count"); + } + ++emitted; + } + *complete = generated && old == layout->old_frames().end(); + if (*complete && emitted != info.frame_count) { + out->reset(); + return Status::corruption("native output omits a planned frame"); + } + window.seal(); + return Status::Ok(); +} + +Status PublishedNativeCursor::next(std::optional<NativeFrameView> *out, bool *complete) +{ + if (out == nullptr || complete == nullptr) { + return Status::invalid_argument("native cursor requires frame and completion outputs"); + } + out->reset(); + *complete = false; + if (state_->busy.exchange(true, std::memory_order_acq_rel)) { + return Status::unavailable("native cursor already has an active window"); + } + + struct Leave + { + std::atomic<bool> &busy; + + ~Leave() + { + busy.store(false, std::memory_order_release); + } + } leave{state_->busy}; + + try { + auto status = state_->next(out, complete); + if (!status.ok()) { + state_->failure = status.without_message(); + } + return status; + } + catch (const std::bad_alloc &) { + state_->failure = Status::resource_exhausted(); + return state_->failure; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_native_rewind.cpp b/lib/crowdb-tree/src/snapshot/published_native_rewind.cpp new file mode 100644 index 000000000..84bc09878 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_native_rewind.cpp @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/published_native_internal.h" + +namespace crowdb::tree +{ +Status PublishedNativeCursor::State::restart() +{ + const auto &view = snapshot->view(); + auto lease = detail::reserved_shared<ReservedOverflowIds>(metadata); + lease->first = view->next_page_id; + lease->count = layout->new_overflow_frames(); + auto reserve = [lease, next = view->next_page_id, end = info.next_page_id](uint64_t count, + ReservedOverflowIds *out) mutable { + if (count > end - next) { + return Status::corruption("native encoder exceeds its planned PID interval"); + } + *out = {.first = next, .count = count, .owner = lease}; + next += count; + return Status::Ok(); + }; + auto status = CapturedFrameStream::create(view, loader, snapshot->binding()->cpu(), snapshot->lane(), limits, + std::move(reserve), &stream); + if (status.ok()) { + emitted = 0; + generated = false; + old = layout->old_frames().begin(); + } + return status; +} + +Status PublishedNativeCursor::rewind() +{ + if (state_->busy.exchange(true, std::memory_order_acq_rel)) { + return Status::unavailable("native cursor already has an active window"); + } + + struct Leave + { + std::atomic<bool> &busy; + + ~Leave() + { + busy.store(false, std::memory_order_release); + } + } leave{state_->busy}; + + if (!state_->failure.ok()) { + return state_->failure; + } + if (!state_->generated || state_->old != state_->layout->old_frames().end()) { + return Status::unavailable("native rewind requires a completed pass"); + } + // Release CPU-tail metadata from the first pass before spending the same + // fixed stream reserve again. No additional PID publication is performed. + state_->snapshot->lane()->drain(); + state_->stream.reset(); + try { + return state_->restart(); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted("native rewind allocation failed"); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot.cpp new file mode 100644 index 000000000..a73635aef --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot.cpp @@ -0,0 +1,297 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/published_snapshot.h" + +#include "backend/local/block_account.h" +#include "backend/local/local_backend_state.h" +#include "backend/local/scheduled_page_store.h" +#include "btree/allocation_bound.h" +#include "btree/tree_core.h" + +namespace crowdb::tree +{ +namespace +{ +struct SnapshotBackendOwner +{ + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<LocalBackendDomain> domain; + std::shared_ptr<const void> backend; +}; +} // namespace + +struct PublishedSnapshotContext::State +{ + ~State() + { + if (lifetime) { + lifetime->close_and_drain(); + lifetime.reset(); + } + // A terminal callback may precede its CPU job's final owner release. + // Snapshot contexts are host-owned, so the binding outlives that tail. + if (lane) { + lane->drain(); + } + if (backend_lane) { + backend_lane->drain(); + } + } + + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<const PublishedTreeState> view; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<SnapshotBackendOwner> owner; + std::shared_ptr<detail::FlushRuntime::Lane> backend_lane; + std::shared_ptr<AsyncPageStore> scheduled; + detail::ReservedUnique<TreeOperationLifetime> lifetime; + std::weak_ptr<TreeCore> core; + std::shared_ptr<TreeWorkMetrics> work_metrics; + AsyncPageStore *backend = nullptr; + PageLoadLimits loads; + PublishedReadLimits reads; + PageKeyRange range; + bool detached = false; +}; + +SnapshotReadWindow::~SnapshotReadWindow() +{ + seal(); +} + +SnapshotReadWindow &SnapshotReadWindow::operator=(SnapshotReadWindow &&other) noexcept +{ + if (this != &other) { + seal(); + loader.reset(); + backend.reset(); + ticket.reset(); + budget = {}; + host_core.reset(); + host_core = std::move(other.host_core); + budget = std::move(other.budget); + ticket = std::move(other.ticket); + backend = std::move(other.backend); + loader = std::move(other.loader); + } + return *this; +} + +void SnapshotReadWindow::seal() noexcept +{ + detail::seal_backend_budget(budget.backend); + if (budget.io) { + budget.io->seal(); + } + if (budget.bytes) { + budget.bytes->seal(); + } +} + +size_t PublishedSnapshotContext::allocation_bound() noexcept +{ + using namespace detail; + return shared_allocation_bound<State>() + shared_allocation_bound<PublishedSnapshotContext>() + + shared_allocation_bound<AllocationReservation>() + shared_allocation_bound<PublishedTreeState>() + + PublishedSources::allocation_bound() + shared_allocation_bound<FlushRuntime::Lane>() + + shared_allocation_bound<SnapshotBackendOwner>() + sizeof(TreeOperationLifetime) + + TreeOperationLifetime::allocation_bound() + TreeOperationLifetime::ticket_allocation_bound(); +} + +size_t PublishedSnapshotContext::window_control_bound() noexcept +{ + return (2 * detail::shared_allocation_bound<detail::AllocationReservation>()) + + TreeOperationLifetime::ticket_allocation_bound(); +} + +Status PublishedSnapshotContext::capture(const std::shared_ptr<TreeCore> &core, + std::shared_ptr<PublishedSnapshotContext> *out) +{ + return core ? core->capture_snapshot(out) : Status::invalid_argument("snapshot capture requires an open Core"); +} + +Status PublishedSnapshotContext::capture_admitted(const std::shared_ptr<TreeCore> &core, + std::shared_ptr<PublishedSnapshotContext> *out, + std::shared_ptr<const PublishedTreeState> captured) +try { + using namespace detail; + if (!core || out == nullptr || *out) { + return Status::invalid_argument("snapshot capture requires an open Core and empty output"); + } + auto binding = core->binding(); + auto bound = AllocationBound(allocation_bound()) + .add(ScheduledPageStore::allocation_bound(binding->options().limits.io_operations)) + .add(shared_allocation_bound<FlushRuntime::Lane>()) + .add_product(2, OwnedKey::allocation_overhead() + 1) + .add(core->config().key_range.start() ? core->config().key_range.start()->size() : 0) + .add(core->config().key_range.end() ? core->config().key_range.end()->size() : 0) + .value(); + if (!bound) { + return Status::resource_exhausted("snapshot backend control bound overflow"); + } + auto credit = ResourceBudget::try_reserve(binding->account(), BudgetClass::kRouting, *bound); + if (!credit) { + return Status::resource_exhausted("snapshot context reserve unavailable"); + } + auto metadata = AllocationReservation::create(std::move(*credit)); + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + auto status = core->operation_lifetime().acquire(metadata, &ticket); + if (!status.ok()) { + return status; + } + auto state = reserved_shared<State>(metadata); + state->metadata = metadata; + if (core->config().key_range.start()) { + state->range.lower = OwnedKey(Slice(*core->config().key_range.start()), metadata); + } + if (core->config().key_range.end()) { + state->range.upper = OwnedKey(Slice(*core->config().key_range.end()), metadata); + } + state->work_metrics = core->publisher().work_metrics(); + state->binding = std::move(binding); + state->lane = reserved_shared<FlushRuntime::Lane>(metadata, state->binding->options().cpu_queue_limit); + if (!captured) { + captured = core->publisher().capture_complete(); + } + auto view = reserved_shared<PublishedTreeState>(metadata, *captured); + auto sources = reserved_shared<PublishedSources>(metadata, metadata); + sources->floor = view->coverage; + view->sources = std::move(sources); + state->view = std::move(view); + state->loads = core->load_limits(); + state->loads.max_inflight = 1; + state->reads = core->read_limits(); + state->reads.max_memtables = 1; + state->reads.metadata.reset(); + state->backend = core->config().async_page_store; + state->detached = (state->backend == nullptr && core->config().page_store == nullptr) || + static_cast<bool>(core->config().backend_owner); + if (state->detached) { + state->owner = reserved_shared<SnapshotBackendOwner>(metadata); + state->owner->binding = state->binding; + state->owner->backend = core->config().backend_owner; + if (const auto &local = core->local_backend()) { + state->owner->domain = local->domain; + } + if (core->config().page_store != nullptr || state->backend != nullptr) { + state->backend_lane = + reserved_shared<FlushRuntime::Lane>(metadata, state->binding->options().backend_queue_limit); + status = ScheduledPageStore::create(core->config().page_store, state->backend, core->config().backend_owner, + state->binding, metadata, state->backend_lane, &state->scheduled); + if (!status.ok()) { + return status; + } + state->backend = state->scheduled.get(); + } + state->lifetime = reserved_unique<TreeOperationLifetime>(metadata, state->binding->options().cpu_queue_limit, + metadata, state->owner); + } + else { + state->core = core; + } + *out = reserved_construct<PublishedSnapshotContext>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) PublishedSnapshotContext(std::move(state)); + }); + metadata->seal(); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot context allocation failed"); +} + +Status PublishedSnapshotContext::prepare_window(size_t bytes, SnapshotReadWindow *out) const +try { + using namespace detail; + if (out == nullptr || out->host_core || out->budget.bytes || out->loader || out->ticket) { + return Status::invalid_argument("snapshot read window output must be empty"); + } + auto bound = AllocationBound(bytes).add(window_control_bound()).value(); + if (!bound) { + return Status::resource_exhausted("snapshot window bound overflow"); + } + auto credit = ResourceBudget::try_reserve(state_->binding->account(), BudgetClass::kRouting, *bound); + auto io = ResourceBudget::try_reserve(state_->binding->account(), BudgetClass::kIo, 1); + if (!credit || !io) { + return Status::resource_exhausted("snapshot read window reserve unavailable"); + } + SnapshotReadWindow window; + window.budget.bytes = AllocationReservation::create(std::move(*credit)); + window.budget.io = reserved_shared<AllocationReservation>(window.budget.bytes, std::move(*io)); + window.budget.work_metrics = state_->work_metrics; + window.budget.cause = TreeWorkCause::Snapshot; + Status status; + AsyncPageStore *backend = state_->backend; + if (state_->detached) { + status = state_->lifetime->acquire(window.budget.bytes, &window.ticket); + } + else { + window.host_core = state_->core.lock(); + if (window.host_core) { + status = window.host_core->acquire_snapshot_window(window.budget.bytes, &window.ticket, &backend); + } + else { + status = Status::unavailable("snapshot source Tree closed"); + } + } + if (!status.ok()) { + return status; + } + status = reserve_backend_operation(backend, state_->binding->account(), + {.max_io_bytes = state_->loads.max_blob_bytes, + .write_calls = 0, + .workers = state_->binding->cpu()->worker_count(), + .barriers = false}, + &window.budget.backend); + if (!status.ok()) { + return status; + } + window.backend = window.ticket->backend(backend); + window.loader = + reserved_shared<AsyncStorePageLoader>(window.budget.bytes, window.backend, state_->binding->cpu(), state_->lane, + state_->loads, OverflowBackingResolver{}, window.budget); + *out = std::move(window); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot read window allocation failed"); +} + +const std::shared_ptr<const PublishedTreeState> &PublishedSnapshotContext::view() const noexcept +{ + return state_->view; +} + +const std::shared_ptr<TreeRuntimeBinding> &PublishedSnapshotContext::binding() const noexcept +{ + return state_->binding; +} + +const std::shared_ptr<detail::FlushRuntime::Lane> &PublishedSnapshotContext::lane() const noexcept +{ + return state_->lane; +} + +PageLoadLimits PublishedSnapshotContext::load_limits() const noexcept +{ + return state_->loads; +} + +PublishedReadLimits PublishedSnapshotContext::read_limits() const noexcept +{ + auto limits = state_->reads; + const auto shape = state_->view->mapping.copy_bounds(); + limits.page_metadata_bytes = shape.max_leaf_bytes; + return limits; +} + +const PageKeyRange &PublishedSnapshotContext::range() const noexcept +{ + return state_->range; +} + +bool PublishedSnapshotContext::detached() const noexcept +{ + return state_->detached; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot.h b/lib/crowdb-tree/src/snapshot/published_snapshot.h new file mode 100644 index 000000000..c36de1b64 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot.h @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/published_read.h" +#include "crowdb-tree/btree/read_result.h" +#include "runtime/tree_operation_lifetime.h" +#include "runtime/tree_runtime_binding.h" + +namespace crowdb::tree +{ +class TreeCore; +struct CoreScanOptions; +class ScanPackedBuf; + +// One accepted window. Idle cursor handles keep no I/O reserve or Tree +// ticket. Lower-level callback owners independently retain accepted I/O. +// Construct and destroy this host scope off runtime workers. +struct SnapshotReadWindow +{ + // Host-only guard; released after the loader and ticket, so a last Core + // release cannot synchronously close while waiting on this same ticket. + std::shared_ptr<TreeCore> host_core; + PageLoadBudget budget; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + std::shared_ptr<AsyncPageStore> backend; + std::shared_ptr<PageLoader> loader; + + SnapshotReadWindow() = default; + ~SnapshotReadWindow(); + SnapshotReadWindow(const SnapshotReadWindow &) = delete; + SnapshotReadWindow &operator=(const SnapshotReadWindow &) = delete; + SnapshotReadWindow(SnapshotReadWindow &&) noexcept = default; + SnapshotReadWindow &operator=(SnapshotReadWindow &&other) noexcept; + void seal() noexcept; +}; + +// A captured L1 header/mapping with no live L0 catalog. Shared backends can +// detach from the Core; raw backends admit each window through a weak Core. +class PublishedSnapshotContext +{ + struct State; + + public: + static Status capture(const std::shared_ptr<TreeCore> &core, std::shared_ptr<PublishedSnapshotContext> *out); + [[nodiscard]] static size_t allocation_bound() noexcept; + [[nodiscard]] static size_t window_control_bound() noexcept; + // bytes covers the caller's operation, loader and output capacities. + // This adds reservation-control/ticket overhead and reserves one I/O. + Status prepare_window(size_t bytes, SnapshotReadWindow *out) const; + Status get(Slice key, GetView *out) const; + Status scan(CoreScanOptions options, ScanPackedBuf *out, size_t *count, bool *truncated) const; + + [[nodiscard]] const std::shared_ptr<const PublishedTreeState> &view() const noexcept; + [[nodiscard]] const std::shared_ptr<TreeRuntimeBinding> &binding() const noexcept; + [[nodiscard]] const std::shared_ptr<detail::FlushRuntime::Lane> &lane() const noexcept; + [[nodiscard]] PageLoadLimits load_limits() const noexcept; + [[nodiscard]] PublishedReadLimits read_limits() const noexcept; + [[nodiscard]] bool detached() const noexcept; + [[nodiscard]] const PageKeyRange &range() const noexcept; + + private: + friend class TreeCore; + static Status capture_admitted(const std::shared_ptr<TreeCore> &core, + std::shared_ptr<PublishedSnapshotContext> *out, + std::shared_ptr<const PublishedTreeState> captured = {}); + + explicit PublishedSnapshotContext(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; + +// A long-lived traversal can retain this delegate across idle windows. Its +// active loader is installed only while the serialized next operation owns +// a SnapshotReadWindow. Completed leaf/value owners never retain it. +class SnapshotPageLoader final : public PageLoader +{ + public: + Status begin(std::shared_ptr<PageLoader> loader); + void end() noexcept; + void load_leaf(BaseRef base, LeafComplete complete) override; + void load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete) override; + void load_overflow(std::shared_ptr<const BackingRef> primary, uint64_t head, uint64_t length, + ValueComplete complete) override; + + private: + std::shared_ptr<PageLoader> active_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_budget.h b/lib/crowdb-tree/src/snapshot/published_snapshot_budget.h new file mode 100644 index 000000000..726d2f04f --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_budget.h @@ -0,0 +1,25 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_bound.h" +#include "btree/published_state.h" + +#include <algorithm> + +namespace crowdb::tree +{ +// A range source retains its current frame while result cells can keep earlier +// frames alive. A point lookup needs only one physical frame per logical base. +inline std::optional<size_t> snapshot_leaf_window(const PublishedTreeState &view, size_t entries) noexcept +{ + const auto shape = view.mapping.copy_bounds(); + if (entries == 0) { + return shape.max_leaf_bases; + } + const auto available = + detail::AllocationBound().add_product(shape.leaf_pages, shape.max_leaf_bases).value().value_or(SIZE_MAX); + const auto window = detail::AllocationBound(shape.max_leaf_bases).add(entries).value(); + return window ? std::optional<size_t>(std::min(*window, available)) : std::nullopt; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_entries.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot_entries.cpp new file mode 100644 index 000000000..91dcbb67f --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_entries.cpp @@ -0,0 +1,201 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/published_snapshot_entries.h" + +#include "btree/allocation_bound.h" +#include "btree/read_value_bound.h" +#include "snapshot/operation_wait.h" +#include "snapshot/published_snapshot_budget.h" + +namespace crowdb::tree +{ +struct PublishedSnapshotEntries::State +{ + ~State() + { + if (metadata) { + metadata->seal(); + } + } + + std::shared_ptr<PublishedSnapshotContext> snapshot; + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<SnapshotPageLoader> loader; + std::shared_ptr<PublishedScan> scan; + size_t read_bound = 0; + std::atomic<bool> busy{false}; + Status failure; + bool complete = false; +}; + +Status PublishedSnapshotEntries::create(std::shared_ptr<PublishedSnapshotContext> snapshot, + std::shared_ptr<PublishedSnapshotEntries> *out, const KeyRange &range) +try { + if (!snapshot || out == nullptr || *out) { + return Status::invalid_argument("snapshot entry cursor requires fixed inputs and empty output"); + } + auto status = range.validate(); + if (!status.ok()) { + return status; + } + auto limits = snapshot->read_limits(); + if ((range.start() && range.start()->size() > limits.max_key_bytes) || + (range.end() && range.end()->size() > limits.max_key_bytes)) { + return Status::invalid_argument("snapshot range exceeds key budget"); + } + limits.max_entries = 1; + // Snapshot capture removes L0. Its fixed mapping cannot observe future + // source writes, so neither source cursors nor their key copies are needed. + limits.max_memtables = 1; + const auto root = snapshot->view()->mapping.get(snapshot->view()->root_page_id); + limits.max_route_depth = root ? root->level + 1ULL : 1; + if (limits.max_route_depth > 64) { + return Status::resource_exhausted("snapshot entry cursor exceeds route depth bound"); + } + const auto workers = snapshot->binding()->cpu()->worker_count(); + const auto loads = snapshot->load_limits(); + const auto &view = *snapshot->view(); + const auto leaves = snapshot_leaf_window(view, limits.max_entries); + if (!leaves) { + return Status::resource_exhausted("snapshot entry frame window overflow"); + } + const auto value_bytes = overflow_value_bytes_bound(view, loads.max_value_bytes); + auto bounded_loads = loads; + // Each accepted overflow frame supplies at least one value byte. Keep the + // loader's validation limits unchanged; only this fixed window is smaller. + bounded_loads.max_overflow_pages = std::min<uint64_t>(loads.max_overflow_pages, value_bytes); + const auto read_bound = + value_bytes == 0 ? AsyncStorePageLoader::primary_reserve_bound(loads, limits.max_route_depth, workers, *leaves) + : AsyncStorePageLoader::read_reserve_bound(bounded_loads, limits.max_route_depth, workers, + *leaves, 1, value_bytes); + using detail::shared_allocation_bound; + const auto bound = + detail::AllocationBound(shared_allocation_bound<State>() + shared_allocation_bound<PublishedSnapshotEntries>() + + shared_allocation_bound<SnapshotPageLoader>() + + shared_allocation_bound<detail::AllocationReservation>()) + .add(PublishedScan::allocation_bound(limits, workers)) + .add_product(2, OwnedKey::allocation_overhead() + limits.max_key_bytes + 1) + .value(); + if (!bound || !read_bound) { + return Status::resource_exhausted("snapshot entry cursor bound overflow"); + } + auto credit = + detail::ResourceBudget::try_reserve(snapshot->binding()->account(), detail::BudgetClass::kRouting, *bound); + if (!credit) { + return Status::resource_exhausted("snapshot entry control reserve unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + auto state = detail::reserved_shared<State>(metadata); + state->snapshot = std::move(snapshot); + state->metadata = metadata; + state->loader = detail::reserved_shared<SnapshotPageLoader>(metadata); + state->read_bound = *read_bound; + limits.metadata = metadata; + PageKeyRange selection; + if (range.start()) { + selection.lower = OwnedKey(Slice(*range.start()), metadata); + } + if (range.end()) { + selection.upper = OwnedKey(Slice(*range.end()), metadata); + } + const auto &domain = state->snapshot->range(); + if (domain.lower && (!selection.lower || *domain.lower > *selection.lower)) { + selection.lower = domain.lower; + } + if (domain.upper && (!selection.upper || *domain.upper < *selection.upper)) { + selection.upper = domain.upper; + } + status = + PublishedScan::create(state->snapshot->view(), state->loader, std::move(selection), false, limits, &state->scan, + {.start = {}, .include_tombstones = true, .allow_oversized_first = true}); + if (!status.ok()) { + return status; + } + *out = detail::reserved_construct<PublishedSnapshotEntries>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) PublishedSnapshotEntries(std::move(state)); + }); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot entry cursor allocation failed"); +} + +Status PublishedSnapshotEntries::next(std::optional<PublishedScanEntry> *out, bool *complete) +{ + if (out == nullptr || complete == nullptr) { + return Status::invalid_argument("snapshot entry cursor needs result outputs"); + } + out->reset(); + *complete = false; + if (state_->busy.exchange(true, std::memory_order_acq_rel)) { + return Status::unavailable("snapshot entry cursor already has an active window"); + } + + struct Leave + { + State &state; + + ~Leave() + { + state.busy.store(false, std::memory_order_release); + } + } leave{*state_}; + + if (!state_->failure.ok()) { + return state_->failure; + } + if (state_->complete) { + *complete = true; + return Status::Ok(); + } + try { + SnapshotReadWindow window; + + struct ActiveWindow + { + SnapshotPageLoader &loader; + + ~ActiveWindow() + { + loader.end(); + } + } active{*state_->loader}; + + auto status = state_->snapshot->prepare_window(state_->read_bound, &window); + if (status.ok()) { + status = state_->loader->begin(window.loader); + } + if (!status.ok()) { + state_->failure = status.without_message(); + return status; + } + detail::SnapshotWaitCompletion completion; + auto connected = + stdexec::connect(state_->scan->next_sender(), detail::SnapshotWaitReceiver{.completion = &completion}); + stdexec::start(connected); + completion.done.wait(); + const auto &result = state_->scan->result(); + if (!result.status.ok()) { + state_->failure = result.status.without_message(); + return result.status; + } + if (!result.entries.empty()) { + *out = result.entries.front(); + } + state_->complete = result.done; + *complete = result.done; + state_->loader->end(); + window.seal(); + return Status::Ok(); + } + catch (const std::bad_alloc &) { + state_->failure = Status::resource_exhausted(); + return state_->failure; + } +} + +uint64_t PublishedSnapshotEntries::skipped_subtrees() const noexcept +{ + return state_->scan->skipped_subtrees(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_entries.h b/lib/crowdb-tree/src/snapshot/published_snapshot_entries.h new file mode 100644 index 000000000..3a59bd4a8 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_entries.h @@ -0,0 +1,30 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/btree/key_range.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +// One logical entry per host window, including tombstones. The scan owns only +// a bounded path/current leaf; returned keys and values have independent +// owners. +class PublishedSnapshotEntries +{ + struct State; + + public: + static Status create(std::shared_ptr<PublishedSnapshotContext> snapshot, + std::shared_ptr<PublishedSnapshotEntries> *out, const KeyRange &range = KeyRange::unbounded()); + Status next(std::optional<PublishedScanEntry> *out, bool *complete); + [[nodiscard]] uint64_t skipped_subtrees() const noexcept; + + private: + explicit PublishedSnapshotEntries(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_materialize.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot_materialize.cpp new file mode 100644 index 000000000..742557be9 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_materialize.cpp @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/published_snapshot_materialize.h" + +#include "btree/allocation_bound.h" + +#include <climits> + +namespace crowdb::tree +{ +namespace +{ +struct MaterializePlan +{ + size_t count = 0; + size_t bytes = sizeof(SnapshotMaterialized) + detail::shared_allocation_bound<detail::AllocationReservation>(); + + Status accept(const PublishedScanEntry &entry, size_t limit) + { + // Direct string construction has no old buffer. Reserve conservative + // growth room, then transfer only actual heap capacity to allocation. + const auto bound = detail::AllocationBound(bytes) + .add((sizeof(leaf_entry) * 2) + sizeof(std::string) + 1 + 9) + .add_product(entry.key.size(), 2) + .add(entry.cell.value.size()) + .value(); + if (!bound || *bound > limit || count == INT_MAX) { + return Status::resource_exhausted("snapshot compatibility result exceeds its capacity limit"); + } + ++count; + bytes = *bound; + return Status::Ok(); + } +}; + +size_t key_heap_bytes(const std::string &key) +{ + const auto object = reinterpret_cast<uintptr_t>(&key); + const auto data = reinterpret_cast<uintptr_t>(key.data()); + const auto object_end = reinterpret_cast<uintptr_t>(std::addressof(key) + 1); + return data >= object && data < object_end ? 0 : key.capacity() + 1; +} + +Status plan_materialization(const std::shared_ptr<PublishedSnapshotContext> &snapshot, size_t max_bytes, + MaterializePlan *plan) +{ + if (plan->bytes > max_bytes) { + return Status::resource_exhausted("snapshot compatibility result cannot fit its control state"); + } + std::shared_ptr<PublishedSnapshotEntries> cursor; + auto status = PublishedSnapshotEntries::create(snapshot, &cursor); + if (!status.ok()) { + return status; + } + bool done = false; + while (!done) { + std::optional<PublishedScanEntry> entry; + status = cursor->next(&entry, &done); + if (!status.ok()) { + return status; + } + if (entry) { + status = plan->accept(*entry, max_bytes); + if (!status.ok()) { + return status; + } + } + } + return Status::Ok(); +} + +Status copy_materialization(const std::shared_ptr<PublishedSnapshotContext> &snapshot, size_t count, + SnapshotMaterialized *result) +{ + result->entries.reserve(count); + result->charge(result->entries.capacity() * sizeof(leaf_entry)); + std::shared_ptr<PublishedSnapshotEntries> cursor; + auto status = PublishedSnapshotEntries::create(snapshot, &cursor); + if (!status.ok()) { + return status; + } + bool done = false; + while (!done) { + std::optional<PublishedScanEntry> entry; + status = cursor->next(&entry, &done); + if (!status.ok()) { + return status; + } + if (!entry) { + continue; + } + if (result->entries.size() == count) { + return Status::corruption("snapshot compatibility result grew beyond its fixed preflight"); + } + leaf_entry copied{.key = std::string(entry->key.data(), entry->key.size()), + .cell = encode_cell_buf(entry->cell.slot, + entry->cell.tombstone() ? OpKind::kDelete : OpKind::kPut, + entry->cell.value)}; + result->charge(key_heap_bytes(copied.key) + (copied.cell.inlined() ? 0 : copied.cell.capacity()), + AllocationKind::kPayload); + result->entries.push_back(std::move(copied)); + } + return result->entries.size() == count + ? Status::Ok() + : Status::corruption("snapshot compatibility result shortened after fixed preflight"); +} +} // namespace + +SnapshotMaterialized::~SnapshotMaterialized() +{ + // Release actual allocations before returning their budget to the host. + std::vector<leaf_entry>().swap(entries); + metadata->record_allocation(AllocationKind::kMetadata, metadata_bytes, false); + metadata->record_allocation(AllocationKind::kPayload, payload_bytes, false); + metadata->release(metadata_bytes + payload_bytes); +} + +void SnapshotMaterialized::charge(size_t bytes, AllocationKind kind) +{ + if (!metadata->claim(bytes)) { + throw std::bad_alloc(); + } + metadata->record_allocation(kind, bytes, true); + if (kind == AllocationKind::kPayload) { + payload_bytes += bytes; + } + else { + metadata_bytes += bytes; + } +} + +Status materialize_snapshot(const std::shared_ptr<PublishedSnapshotContext> &snapshot, size_t max_bytes, + detail::ReservedUnique<SnapshotMaterialized> *out) +try { + if (!snapshot || out == nullptr || *out) { + return Status::invalid_argument("snapshot materialization requires a fixed view and empty output"); + } + MaterializePlan plan; + auto status = plan_materialization(snapshot, max_bytes, &plan); + if (!status.ok()) { + return status; + } + snapshot->lane()->drain(); + auto credit = + detail::ResourceBudget::try_reserve(snapshot->binding()->account(), detail::BudgetClass::kRouting, plan.bytes); + if (!credit) { + return Status::resource_exhausted("snapshot compatibility result reserve unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + auto result = detail::reserved_unique<SnapshotMaterialized>(metadata, metadata); + status = copy_materialization(snapshot, plan.count, result.get()); + if (!status.ok()) { + return status; + } + metadata->seal(); + *out = std::move(result); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot compatibility result allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_materialize.h b/lib/crowdb-tree/src/snapshot/published_snapshot_materialize.h new file mode 100644 index 000000000..c93558bb7 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_materialize.h @@ -0,0 +1,29 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/published_snapshot_entries.h" + +namespace crowdb::tree +{ +// The legacy Snapshot API returns std::vector<leaf_entry>. Its allocator type +// cannot change, so these leases account actual retained vector/key/cell +// capacities while the fixed reservation bounds construction. +struct SnapshotMaterialized +{ + explicit SnapshotMaterialized(std::shared_ptr<detail::AllocationReservation> budget) : metadata(std::move(budget)) + { + } + + ~SnapshotMaterialized(); + void charge(size_t bytes, AllocationKind kind = AllocationKind::kMetadata); + + std::shared_ptr<detail::AllocationReservation> metadata; + size_t metadata_bytes = 0; + size_t payload_bytes = 0; + std::vector<leaf_entry> entries; +}; + +Status materialize_snapshot(const std::shared_ptr<PublishedSnapshotContext> &snapshot, size_t max_bytes, + detail::ReservedUnique<SnapshotMaterialized> *out); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_read.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot_read.cpp new file mode 100644 index 000000000..202ae0457 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_read.cpp @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/read_value_bound.h" +#include "snapshot/operation_wait.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +Status PublishedSnapshotContext::get(Slice key, GetView *out) const +try { + if (out == nullptr) { + return Status::invalid_argument("snapshot point read needs output"); + } + *out = {}; + if (!range().contains(key)) { + return Status::Ok(); + } + auto limits = read_limits(); + const auto root = view()->mapping.get(view()->root_page_id); + if (!root || root->level >= limits.max_route_depth) { + return Status::corruption("snapshot has no bounded published root"); + } + limits.max_route_depth = root->level + 1ULL; + const auto loads = load_limits(); + const auto workers = binding()->cpu()->worker_count(); + const auto leaves = view()->mapping.copy_bounds().max_leaf_bases; + const auto values = overflow_value_bytes_bound(*view(), loads.max_value_bytes); + auto bounded_loads = loads; + bounded_loads.max_overflow_pages = std::min<uint64_t>(loads.max_overflow_pages, values); + const auto loader = + values == 0 ? AsyncStorePageLoader::primary_reserve_bound(loads, root->level, workers, leaves) + : AsyncStorePageLoader::read_reserve_bound(bounded_loads, root->level, workers, leaves, 1, values); + const auto bound = + detail::AllocationBound().add(PublishedGet::allocation_bound(limits, workers)).add(loader).value(); + if (!bound) { + return Status::resource_exhausted("snapshot point read bound overflow"); + } + SnapshotReadWindow window; + auto status = prepare_window(*bound, &window); + if (!status.ok()) { + return status; + } + limits.metadata = window.budget.bytes; + std::shared_ptr<PublishedGet> read; + status = PublishedGet::create(view(), window.loader, key, limits, &read); + if (status.ok()) { + status = detail::wait_snapshot_operation(read); + } + if (status.ok() && read->result().found) { + const auto &cell = read->result().cell; + *out = GetView::from_owner(cell.slot, cell.value, cell.owner); + } + window.seal(); + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot point read allocation failed"); +} + +Status SnapshotPageLoader::begin(std::shared_ptr<PageLoader> loader) +{ + if (active_ || !loader) { + return Status::unavailable("snapshot loader already has an active window"); + } + active_ = std::move(loader); + return Status::Ok(); +} + +void SnapshotPageLoader::end() noexcept +{ + active_.reset(); +} + +void SnapshotPageLoader::load_leaf(BaseRef base, LeafComplete complete) +{ + if (auto loader = active_) { + loader->load_leaf(std::move(base), std::move(complete)); + } + else { + complete(Status::unavailable("snapshot loader has no active window"), {}); + } +} + +void SnapshotPageLoader::load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete) +{ + if (auto loader = active_) { + loader->load_inner(std::move(backing), std::move(complete)); + } + else { + complete(Status::unavailable("snapshot loader has no active window"), {}); + } +} + +void SnapshotPageLoader::load_overflow(std::shared_ptr<const BackingRef> primary, uint64_t head, uint64_t length, + ValueComplete complete) +{ + if (auto loader = active_) { + loader->load_overflow(std::move(primary), head, length, std::move(complete)); + } + else { + complete(Status::unavailable("snapshot loader has no active window"), {}); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_scan.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot_scan.cpp new file mode 100644 index 000000000..bffc826fb --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_scan.cpp @@ -0,0 +1,224 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_account.h" +#include "btree/allocation_bound.h" +#include "btree/read_value_bound.h" +#include "btree/scan_output.h" +#include "btree/tree_core.h" +#include "snapshot/operation_wait.h" +#include "snapshot/published_snapshot.h" +#include "snapshot/published_snapshot_budget.h" + +#include <algorithm> +#include <chrono> +#include <utility> + +namespace crowdb::tree +{ +namespace +{ +std::optional<size_t> result_count_bound(const PublishedTreeState &view, const PageLoadLimits &loads, + bool include_tombstones) +{ + // Snapshot capture removes the live source catalog. Exact live-key counts + // exclude tombstones; otherwise each captured base contributes at most its + // configured leaf capacity in addition to the immutable overlay entries. + if (!include_tombstones && view.stats.kv_unknown_pages == 0) { + return view.stats.kv_known_count; + } + const auto shape = view.mapping.copy_bounds(); + const auto bases = detail::AllocationBound().add_product(shape.leaf_pages, shape.max_leaf_bases).value(); + if (!bases) { + return {}; + } + return detail::AllocationBound(view.stats.overlay_entries).add_product(*bases, loads.max_leaf_entries).value(); +} + +PageKeyRange scan_range(CoreScanOptions options, const PageKeyRange &domain, + const std::shared_ptr<detail::AllocationReservation> &credit) +{ + PageKeyRange range; + if (!options.prefix.empty()) { + range.lower = OwnedKey(options.prefix, credit); + KeyArenaStorage upper(options.prefix.data(), options.prefix.size(), detail::ReservationAllocator<char>(credit)); + while (!upper.empty() && static_cast<uint8_t>(upper.back()) == 255) { + upper.pop_back(); + } + if (!upper.empty()) { + upper.back() = static_cast<char>(static_cast<uint8_t>(upper.back()) + 1); + range.upper = OwnedKey(Slice(upper.data(), upper.size()), credit); + } + } + if (!options.begin.empty() && (!range.lower || options.begin > Slice(*range.lower))) { + range.lower = OwnedKey(options.begin, credit); + } + if (!options.end.empty() && (!range.upper || options.end < Slice(*range.upper))) { + range.upper = OwnedKey(options.end, credit); + } + if (domain.lower && (!range.lower || *domain.lower > *range.lower)) { + range.lower = domain.lower; + } + if (domain.upper && (!range.upper || *domain.upper < *range.upper)) { + range.upper = domain.upper; + } + return range; +} + +bool expired(uint64_t deadline) +{ + return deadline != 0 && std::cmp_greater_equal(std::chrono::duration_cast<std::chrono::milliseconds>( + std::chrono::system_clock::now().time_since_epoch()) + .count(), + deadline); +} + +Status next_scan(const std::shared_ptr<PublishedScan> &scan) +{ + detail::SnapshotWaitCompletion completion; + auto operation = stdexec::connect(scan->next_sender(), detail::SnapshotWaitReceiver{.completion = &completion}); + stdexec::start(operation); + completion.done.wait(); + return completion.status.ok() ? scan->result().status : completion.status; +} + +Status pack_window(const PublishedScanResult &result, CoreScanOptions options, size_t bytes_limit, + size_t normal_capacity, size_t value_limit, ScanPackedBuf *packed, size_t *count, size_t *bytes, + bool *truncated) +{ + for (const auto &entry : result.entries) { + const Slice key(entry.key); + const Slice value = entry.cell.value; + if (key.size() > UINT32_MAX || value.size() > UINT32_MAX || value.size() > value_limit) { + return Status::resource_exhausted("snapshot scan entry exceeds configured limits"); + } + const auto size = key.size() + value.size(); + const bool exceeds = size > bytes_limit - std::min(*bytes, bytes_limit); + if (*count >= options.limit || (exceeds && *count != 0) || expired(options.deadline_ms)) { + *truncated = true; + return Status::Ok(); + } + if (exceeds && options.strict_byte_budget) { + *truncated = true; + return Status::Ok(); + } + detail::reserve_scan_output(*packed, size, normal_capacity); + packed->pack_u32(static_cast<uint32_t>(key.size())); + packed->append(key); + packed->pack_u64(entry.cell.slot); + packed->push_back(entry.cell.tombstone() ? 1 : 0); + packed->pack_u32(static_cast<uint32_t>(value.size())); + packed->append(value); + ++*count; + *bytes += size; + } + *truncated = !result.done && (result.byte_limit_reached || *count >= options.limit || *bytes >= bytes_limit); + return Status::Ok(); +} +} // namespace + +Status PublishedSnapshotContext::scan(CoreScanOptions options, ScanPackedBuf *out, size_t *count, bool *truncated) const +try { + if (out == nullptr || count == nullptr || truncated == nullptr || + (options.strict_byte_budget && options.byte_budget == 0)) { + return Status::invalid_argument("snapshot scan requires bounded output"); + } + *out = {}; + *count = 0; + *truncated = false; + auto limits = read_limits(); + for (const auto key : {options.prefix, options.start, options.end, options.begin}) { + if (key.size() > limits.max_key_bytes) { + return Status::invalid_argument("snapshot scan boundary exceeds key limit"); + } + } + const auto root = view()->mapping.get(view()->root_page_id); + if (!root || root->level >= limits.max_route_depth) { + return Status::corruption("snapshot scan has no bounded root"); + } + using detail::AllocationBound; + const auto loads = load_limits(); + const auto maximum_count = result_count_bound(*view(), loads, options.include_tombstones); + if (!maximum_count) { + return Status::resource_exhausted("snapshot scan result count bound overflow"); + } + options.limit = options.limit == 0 ? *maximum_count : std::min(options.limit, *maximum_count); + limits.max_route_depth = root->level + 1ULL; + const auto byte_limit = options.byte_budget == 0 ? SIZE_MAX : options.byte_budget; + limits.max_result_bytes = std::min(byte_limit, limits.max_result_bytes); + // The public limit bounds the whole packed result. Each read still uses a + // fixed work window, with at most one zero-byte empty-key result. + const auto byte_count = AllocationBound(limits.max_result_bytes).add(1).value().value_or(SIZE_MAX); + limits.max_entries = std::max<size_t>(1, std::min({options.limit, limits.max_entries, byte_count})); + const auto workers = binding()->cpu()->worker_count(); + const auto normal = AllocationBound(limits.max_result_bytes).add_product(options.limit, 17).value(); + const auto value_limit = overflow_value_bytes_bound(*view(), loads.max_value_bytes); + const auto values = AllocationBound(value_limit).add(value_limit == 0 ? 0 : limits.max_result_bytes).value(); + if (!normal || !values) { + return Status::resource_exhausted("snapshot scan result bound overflow"); + } + const auto leaves = snapshot_leaf_window(*view(), limits.max_entries); + if (!leaves) { + return Status::resource_exhausted("snapshot scan frame window overflow"); + } + auto bounded_loads = loads; + bounded_loads.max_overflow_pages = std::min<uint64_t>(loads.max_overflow_pages, value_limit); + const auto loader = value_limit == 0 + ? AsyncStorePageLoader::primary_reserve_bound(loads, root->level, workers, *leaves) + : AsyncStorePageLoader::read_reserve_bound(bounded_loads, root->level, workers, *leaves, + limits.max_entries, *values); + const auto bound = AllocationBound() + .add(PublishedScan::allocation_bound(limits, workers)) + .add(loader) + .add_product(6, OwnedKey::allocation_overhead() + limits.max_key_bytes + 1) + .value(); + if (!bound) { + return Status::resource_exhausted("snapshot scan read bound overflow"); + } + SnapshotReadWindow window; + auto status = prepare_window(*bound, &window); + if (!status.ok()) { + return status; + } + limits.metadata = window.budget.bytes; + PublishedScanOptions selection{.start_inclusive = options.start_inclusive, + .keys_only = options.keys_only, + .include_tombstones = options.include_tombstones, + .allow_oversized_first = !options.strict_byte_budget, + .byte_budget = byte_limit}; + if (options.has_start_bound || !options.start.empty()) { + selection.start = OwnedKey(options.start, limits.metadata); + } + std::shared_ptr<PublishedScan> scan; + status = PublishedScan::create(view(), window.loader, scan_range(options, range(), limits.metadata), + options.reverse, limits, &scan, std::move(selection)); + std::shared_ptr<detail::AllocationAccount> output; + if (status.ok()) { + status = detail::AllocationAccount::create(binding()->account(), detail::BudgetClass::kRouting, &output); + } + ScanPackedBuf packed(std::move(output)); + size_t bytes = 0; + while (status.ok()) { + if (expired(options.deadline_ms)) { + *truncated = true; + break; + } + status = next_scan(scan); + if (!status.ok()) { + break; + } + status = pack_window(scan->result(), options, byte_limit, *normal, loads.max_value_bytes, &packed, count, + &bytes, truncated); + if (scan->result().done || *truncated) { + break; + } + } + if (status.ok()) { + *out = std::move(packed); + } + window.seal(); + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot scan allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_view.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot_view.cpp new file mode 100644 index 000000000..639077d5d --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_view.cpp @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/published_snapshot_view.h" + +#include "btree/allocation_bound.h" +#include "btree/tree_core.h" + +#include <stdexcept> + +namespace crowdb::tree +{ +PublishedSnapshotView::PublishedSnapshotView(std::shared_ptr<PublishedSnapshotContext> context, size_t max_bytes) + : Snapshot(context->view()->coverage, {}), + context_(std::move(context)), + max_bytes_(max_bytes) +{ +} + +PublishedSnapshotView::~PublishedSnapshotView() +{ + if (auto *stored = materialized_.load(std::memory_order_relaxed)) { + detail::ReservationDeleter<SnapshotMaterialized>{stored->metadata}(stored); + } +} + +Status PublishedSnapshotView::create(std::shared_ptr<PublishedSnapshotContext> context, size_t max_bytes, + std::shared_ptr<Snapshot> *out) +try { + if (!context || max_bytes == 0 || out == nullptr || *out) { + return Status::invalid_argument("snapshot compatibility view requires bounded fixed inputs and empty output"); + } + const auto bound = detail::shared_allocation_bound<PublishedSnapshotView>() + + detail::shared_allocation_bound<detail::AllocationReservation>(); + auto credit = + detail::ResourceBudget::try_reserve(context->binding()->account(), detail::BudgetClass::kRouting, bound); + if (!credit) { + return Status::resource_exhausted("snapshot compatibility control reserve unavailable"); + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + auto result = detail::reserved_construct<PublishedSnapshotView>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) PublishedSnapshotView(std::move(context), max_bytes); + }); + if (!result->context_->detached()) { + detail::ReservedUnique<SnapshotMaterialized> values; + auto status = materialize_snapshot(result->context_, max_bytes, &values); + if (!status.ok()) { + return status; + } + result->materialized_.store(values.release(), std::memory_order_release); + result->context_.reset(); + } + metadata->seal(); + *out = std::move(result); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot compatibility allocation failed"); +} + +const std::vector<leaf_entry> &PublishedSnapshotView::entries() const +{ + if (const auto *stored = materialized_.load(std::memory_order_acquire)) { + return stored->entries; + } + detail::ReservedUnique<SnapshotMaterialized> values; + auto status = materialize_snapshot(context_, max_bytes_, &values); + if (!status.ok()) { + throw SnapshotReadError(status); + } + auto *expected = static_cast<SnapshotMaterialized *>(nullptr); + if (materialized_.compare_exchange_strong(expected, values.get(), std::memory_order_acq_rel)) { + return values.release()->entries; + } + return expected->entries; +} + +bool PublishedSnapshotView::get(Slice key, uint64_t *out_slot, std::string *out_value) const +{ + if (materialized_.load(std::memory_order_acquire) != nullptr) { + return Snapshot::get(key, out_slot, out_value); + } + GetView value; + auto status = context_->get(key, &value); + if (!status.ok()) { + throw SnapshotReadError(status); + } + if (!value.found()) { + return false; + } + if (out_slot != nullptr) { + *out_slot = value.slot(); + } + if (out_value != nullptr) { + *out_value = value.value().to_string(); + } + return true; +} + +Status create_published_snapshot(const std::shared_ptr<TreeCore> &core, std::shared_ptr<Snapshot> *out) +{ + if (!core || out == nullptr || *out) { + return Status::invalid_argument("snapshot compatibility view requires a Core and empty output"); + } + std::shared_ptr<PublishedSnapshotContext> context; + auto status = core->capture_snapshot(&context); + if (!status.ok()) { + return status; + } + return PublishedSnapshotView::create(std::move(context), core->binding()->options().limits.routing_bytes, out); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_view.h b/lib/crowdb-tree/src/snapshot/published_snapshot_view.h new file mode 100644 index 000000000..1738b37e4 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_view.h @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/snapshot/snapshot.h" +#include "snapshot/published_snapshot_materialize.h" + +#include <stdexcept> + +namespace crowdb::tree +{ +// Compatibility methods return references/bools and therefore report failure +// by exception. ABI adapters preserve this status without unwinding into FFI. +class SnapshotReadError final : public std::runtime_error +{ + public: + explicit SnapshotReadError(const Status &status) : std::runtime_error(status.to_string()), code_(status.code()) + { + } + + [[nodiscard]] Code code() const noexcept + { + return code_; + } + + private: + Code code_; +}; + +// Compatibility facade: point reads use the fixed immutable publication; +// whole-result methods explicitly materialize within a caller-supplied cap. +class PublishedSnapshotView final : public Snapshot +{ + public: + static Status create(std::shared_ptr<PublishedSnapshotContext> context, size_t max_bytes, + std::shared_ptr<Snapshot> *out); + ~PublishedSnapshotView() override; + [[nodiscard]] const std::vector<leaf_entry> &entries() const override; + [[nodiscard]] bool get(Slice key, uint64_t *out_slot, std::string *out_value) const override; + + private: + PublishedSnapshotView(std::shared_ptr<PublishedSnapshotContext> context, size_t max_bytes); + std::shared_ptr<PublishedSnapshotContext> context_; + size_t max_bytes_; + mutable std::atomic<SnapshotMaterialized *> materialized_{nullptr}; +}; + +// Captures published L1 exactly as snapshot_view historically did: it never +// hands off the live source table or advances coverage. +Status create_published_snapshot(const std::shared_ptr<TreeCore> &core, std::shared_ptr<Snapshot> *out); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/readable_page_catalog.h b/lib/crowdb-tree/src/snapshot/readable_page_catalog.h new file mode 100644 index 000000000..c5841ab9f --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/readable_page_catalog.h @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_writeback.h" + +#include <map> + +namespace crowdb::tree +{ +struct StoredOverflowReference +{ + uint64_t head = kInvalidPageId; + uint64_t bytes = 0; +}; + +using StoredOverflowReferences = detail::ReservedVector<StoredOverflowReference>; + +struct ReadablePageFrame +{ + uint64_t page_id = kInvalidPageId; + page_type type = page_type::kLeafBase; + uint32_t frame_bytes = 0; + uint32_t logical_bytes = 0; + uint64_t next_overflow = kInvalidPageId; + uint32_t overflow_bytes = 0; + std::shared_ptr<const BackingRef> backing; + // Every encoded leaf cell, including overwritten in-frame delta history. + // An empty owned list proves no stored overflow references; null is unknown. + std::shared_ptr<const StoredOverflowReferences> stored_overflow = nullptr; + std::optional<FinalPageSummary> summary = std::nullopt; +}; + +using DurableCapturedFrame = ReadablePageFrame; + +// A fixed readable graph used for validation. Resident frames and protected +// durable blobs both retain independent owners; this catalog promises no +// checkpoint durability. +struct ReadablePageCatalog +{ + static constexpr bool observe_retirement = true; + + bool retire_step() noexcept + { + for (size_t i = 0; i < 64 && !frames.empty(); ++i) { + frames.erase(frames.begin()); + } + return frames.empty(); + } + + using Entry = std::pair<const uint64_t, DurableCapturedFrame>; + using Frames = std::map<uint64_t, DurableCapturedFrame, std::less<>, detail::ReservationAllocator<Entry>>; + + explicit ReadablePageCatalog(const std::shared_ptr<detail::AllocationReservation> &metadata) + : frames(std::less<>{}, detail::ReservationAllocator<Entry>(metadata)) + { + } + + Frames frames; + uint64_t root_page_id = kInvalidPageId; + uint64_t next_page_id = 0; + uint64_t coverage = 0; + uint64_t generation = 0; + uint64_t backend_id = 0; +}; + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/recovered_state.cpp b/lib/crowdb-tree/src/snapshot/recovered_state.cpp new file mode 100644 index 000000000..d13c19f21 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/recovered_state.cpp @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/recovered_state_internal.h" + +#include <cassert> + +namespace crowdb::tree +{ +RecoveredStateOperation::State::State(std::shared_ptr<detail::AllocationReservation> credit) + : metadata(std::move(credit)), + path(detail::ReservationAllocator<Path>(metadata)), + reached(std::less<>{}, detail::ReservationAllocator<uint64_t>(metadata)) +{ +} + +std::optional<size_t> RecoveredStateOperation::allocation_bound(size_t frames, size_t primary_pages, + size_t max_key_bytes) noexcept +{ + using detail::shared_allocation_bound; + const auto page = detail::AllocationBound(shared_allocation_bound<PublishedPage>()) + .add(PageState::allocation_bound()) + .add(sizeof(BaseRef)) + .add(MappingView::path_copy_bound()) + .add_product(2, OwnedKey::allocation_overhead()) + .add_product(2, max_key_bytes) + .value(); + if (!page || primary_pages > frames || frames > RecoveredStateLimits{}.max_pages) { + return {}; + } + return detail::AllocationBound(shared_allocation_bound<State>() + + shared_allocation_bound<RecoveredStateOperation>() + + shared_allocation_bound<PublishedTreeState>() + PublishedSources::allocation_bound()) + .add(detail::FlushRuntime::progress_allocation_bound()) + .add_product(frames, shared_allocation_bound<State::Load>()) + .add_product(RecoveredStateLimits{}.max_depth, sizeof(State::Path)) + .add_product(frames, shared_allocation_bound<uint64_t>()) + .add_product(primary_pages, *page) + .value(); +} + +Status RecoveredStateOperation::create(std::shared_ptr<const CapturedDurablePages> captured, + std::shared_ptr<PageLoader> loader, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, RecoveredStateLimits limits, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<RecoveredStateOperation> *out) +{ + auto status = create_readable(std::move(captured), std::move(loader), std::move(runtime), std::move(lane), limits, + std::move(metadata), out); + if (status.ok()) { + (*out)->state_->require_durable = true; + } + return status; +} + +Status RecoveredStateOperation::create_readable(std::shared_ptr<const ReadablePageCatalog> captured, + std::shared_ptr<PageLoader> loader, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, + RecoveredStateLimits limits, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<RecoveredStateOperation> *out) +{ + if (!captured || !loader || !runtime || !lane || out == nullptr || limits.max_pages == 0 || limits.max_depth == 0 || + limits.max_depth > 64 || limits.max_children == 0 || limits.max_children > 4096 || + limits.max_leaf_entries == 0 || limits.max_step_entries == 0 || limits.max_dependency_steps == 0 || + captured->next_page_id > MappingView::kPageLimit) { + return Status::invalid_argument(); + } + if (captured->frames.size() > limits.max_pages) { + return Status::resource_exhausted(); + } + try { + auto state = detail::reserved_shared<State>(metadata, metadata); + state->captured = std::move(captured); + state->loader = std::move(loader); + state->runtime = std::move(runtime); + state->lane = std::move(lane); + state->limits = limits; + state->path.reserve(limits.max_depth); + state->building = detail::reserved_shared<PublishedTreeState>(metadata, metadata, state->captured->coverage); + state->building->root_page_id = state->captured->root_page_id; + state->building->next_page_id = state->captured->next_page_id; + state->building->generation = state->captured->generation == 0 ? 1 : state->captured->generation; + state->building->coverage = state->captured->coverage; + state->building->summary.emplace(); + state->building->summary->covered_slot = state->captured->coverage; + state->position = state->captured->frames.begin(); + const auto status = + state->runtime->reserve_progress(state->lane, &State::run, state.get(), metadata, &state->progress); + if (!status.ok()) { + return status; + } + *out = detail::reserved_construct<RecoveredStateOperation>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) RecoveredStateOperation(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + +detail::CallbackSender RecoveredStateOperation::sender() noexcept +{ + return {this, &RecoveredStateOperation::submit}; +} + +const RecoveredStateResult &RecoveredStateOperation::result() const noexcept +{ + return state_->report; +} + +void RecoveredStateOperation::request_stop() noexcept +{ + state_->stopped.store(true, std::memory_order_release); + state_->wake(); +} + +void RecoveredStateOperation::submit(void *source, detail::CallbackComplete complete, void *context) +{ + static_cast<RecoveredStateOperation *>(source)->state_->start(complete, context); +} + +void RecoveredStateOperation::State::start(detail::CallbackComplete callback, void *argument) noexcept +{ + uint8_t idle = 0; + if (!lifecycle.compare_exchange_strong(idle, 1, std::memory_order_acq_rel)) { + callback(argument, detail::CallbackSignal::kError, Status::unavailable()); + return; + } + complete = callback; + context = argument; + lifecycle.store(2, std::memory_order_release); + wake(); +} + +void RecoveredStateOperation::State::wake() noexcept +{ + if (lifecycle.load(std::memory_order_acquire) == 2) { + auto self = weak_from_this().lock(); + assert(self); + progress->request(std::move(self)); + } +} + +void RecoveredStateOperation::State::run(void *context) noexcept +{ + auto *self = static_cast<State *>(context); + if (self->lifecycle.load(std::memory_order_acquire) != 2) { + return; + } + try { + self->pump(); + } + catch (const std::bad_alloc &) { + self->finish(Status::resource_exhausted()); + } + catch (...) { + self->finish(Status::internal_error()); + } +} + +void RecoveredStateOperation::State::finish(Status status) noexcept +{ + if (lifecycle.exchange(3, std::memory_order_acq_rel) == 3) { + return; + } + const bool cancelled = stopped.load(std::memory_order_acquire); + report.status = cancelled ? Status::unavailable() : std::move(status); + if (report.status.ok()) { + report.state = std::move(building); + } + building.reset(); + path.clear(); + reached.clear(); + load.reset(); + captured.reset(); + loader.reset(); + auto signal = report.status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + if (cancelled) { + signal = detail::CallbackSignal::kStopped; + } + complete(context, signal, report.status.without_message()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/recovered_state.h b/lib/crowdb-tree/src/snapshot/recovered_state.h new file mode 100644 index 000000000..92e313c4d --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/recovered_state.h @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/captured_write.h" + +namespace crowdb::tree +{ +struct RecoveredStateLimits +{ + size_t max_pages = 1024 * 1024; + size_t max_depth = 64; + size_t max_children = 4096; + size_t max_leaf_entries = 4096; + size_t max_key_bytes = 256 * 1024; + size_t max_step_entries = 128; + size_t max_dependency_steps = 1024 * 1024; +}; + +struct RecoveredStateResult +{ + Status status; + std::shared_ptr<const PublishedTreeState> state; + uint64_t pages = 0; + uint64_t loads = 0; + uint64_t cpu_steps = 0; +}; + +// Validates a recovered immutable namespace before its one-time installation. +// Only a bounded decoded path and one leaf are retained. The resulting mapping +// owns cold proxies; overflow extents remain protected by each leaf's backing. +class RecoveredStateOperation +{ + struct State; + + public: + static Status create(std::shared_ptr<const CapturedDurablePages> captured, std::shared_ptr<PageLoader> loader, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, RecoveredStateLimits limits, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<RecoveredStateOperation> *out); + static Status create_readable(std::shared_ptr<const ReadablePageCatalog> captured, + std::shared_ptr<PageLoader> loader, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, RecoveredStateLimits limits, + std::shared_ptr<detail::AllocationReservation> metadata, + std::shared_ptr<RecoveredStateOperation> *out); + [[nodiscard]] static std::optional<size_t> allocation_bound(size_t frames, size_t primary_pages, + size_t max_key_bytes) noexcept; + [[nodiscard]] detail::CallbackSender sender() noexcept; + [[nodiscard]] const RecoveredStateResult &result() const noexcept; + void request_stop() noexcept; + + private: + explicit RecoveredStateOperation(std::shared_ptr<State> state) : state_(std::move(state)) + { + } + + static void submit(void *source, detail::CallbackComplete complete, void *context); + std::shared_ptr<State> state_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/recovered_state_graph.cpp b/lib/crowdb-tree/src/snapshot/recovered_state_graph.cpp new file mode 100644 index 000000000..e462979c7 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/recovered_state_graph.cpp @@ -0,0 +1,218 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/graph_summary.h" +#include "snapshot/primary_backing.h" +#include "snapshot/recovered_state_internal.h" + +namespace crowdb::tree +{ +void RecoveredStateOperation::State::push(uint64_t pid, PageKeyRange range) +{ + if (path.size() == limits.max_depth || report.pages == limits.max_pages) { + finish(Status::resource_exhausted()); + return; + } + const auto found = captured->frames.find(pid); + if (pid >= captured->next_page_id || found == captured->frames.end() || found->second.page_id != pid || + (found->second.type != page_type::kInnerBase && found->second.type != page_type::kLeafBase) || + !found->second.backing || + (!found->second.backing->readable() || (require_durable && !found->second.backing->reloadable())) || + found->second.backing->backend_id != captured->backend_id || !reached.insert(pid).second) { + finish(Status::corruption()); + return; + } + Path next; + next.pid = pid; + next.descriptor = &found->second; + next.range = std::move(range); + path.push_back(std::move(next)); + ++report.pages; + wake(); +} + +void RecoveredStateOperation::State::inner_step() +{ + auto &top = path.back(); + const auto &keys = top.inner->separators; + size_t work = 0; + while (top.index < keys.size() && work++ < limits.max_step_entries) { + const auto &key = keys[top.index]; + if (key.size() > limits.max_key_bytes - top.key_bytes) { + finish(Status::resource_exhausted()); + return; + } + top.key_bytes += key.size(); + if ((top.index != 0 && keys[top.index - 1] >= key) || (top.range.lower && key < *top.range.lower) || + (top.range.upper && key > *top.range.upper)) { + finish(Status::corruption()); + return; + } + ++top.index; + } + if (top.index == keys.size()) { + const auto &stored = top.inner->frame_proof; + if (!stored.has_value()) { + finish(Status::corruption()); + return; + } + const auto &proof = *stored; + if (proof.has_fences && !proof.page_id_fences && + (proof.lower.size() > limits.max_key_bytes - top.key_bytes || + proof.upper.size() > limits.max_key_bytes - top.key_bytes - proof.lower.size())) { + finish(Status::resource_exhausted()); + return; + } + top.index = 0; + top.phase = Phase::Children; + } + wake(); +} + +void RecoveredStateOperation::State::child_step() +{ + auto &top = path.back(); + if (top.index == top.inner->children.size()) { + top.phase = Phase::Finish; + wake(); + return; + } + const auto index = top.index++; + auto range = top.range; + if (index != 0) { + range.lower = top.inner->separators[index - 1]; + } + if (index < top.inner->separators.size()) { + range.upper = top.inner->separators[index]; + } + push(top.inner->children[index], std::move(range)); +} + +void RecoveredStateOperation::State::complete_page() +{ + auto &top = path.back(); + uint32_t level = 0; + if (top.inner) { + if (!top.child_level || *top.child_level >= limits.max_depth) { + finish(Status::corruption()); + return; + } + level = *top.child_level + 1; + const auto &stored = top.inner->frame_proof; + if (!stored.has_value()) { + finish(Status::corruption()); + return; + } + const auto &proof = *stored; + // Old native frames may omit fences; the traversal reconstructs them. + if (proof.has_fences && + (!top.bounds.lower || !top.bounds.upper || + (proof.page_id_fences && + (proof.lower_leaf != top.bounds.lower_leaf || proof.upper_leaf != top.bounds.upper_leaf)) || + (!proof.page_id_fences && (proof.lower != *top.bounds.lower || proof.upper != *top.bounds.upper)))) { + finish(Status::corruption()); + return; + } + } + if (path.size() > 1) { + auto &parent = path[path.size() - 2]; + if (parent.child_level && *parent.child_level != level) { + finish(Status::corruption()); + return; + } + parent.child_level = level; + } + if (!install_page(level)) { + return; + } + if (path.size() > 1) { + auto &parent = path[path.size() - 2].bounds; + if (!parent.lower && top.bounds.lower) { + parent.lower = std::move(top.bounds.lower); + parent.lower_leaf = top.bounds.lower_leaf; + } + if (top.bounds.upper) { + parent.upper = std::move(top.bounds.upper); + parent.upper_leaf = top.bounds.upper_leaf; + } + } + path.pop_back(); + wake(); +} + +bool RecoveredStateOperation::State::install_page(uint32_t level) +{ + const auto &top = path.back(); + auto page = detail::reserved_shared<PublishedPage>(metadata); + page->revision = 1; + page->level = level; + page->metrics = top.metrics; + if (top.inner) { + page->inner_backing = top.descriptor->backing; + auto descriptor = *top.descriptor; + descriptor.summary.emplace(); + descriptor.summary->encoded_entries = top.inner->separators.size(); + descriptor.summary->primary_used_bytes = top.metrics.page_size_bytes; + descriptor.summary->inner_fences = {.lower_leaf = top.bounds.lower_leaf, .upper_leaf = top.bounds.upper_leaf}; + page->primary = make_inner_primary_backing(descriptor, *page, captured->coverage, building->gc_floor, metadata); + } + else { + auto leaf = detail::reserved_shared<PageState>(metadata, metadata); + leaf->range = top.range; + leaf->route_generation = 1; + leaf->content_id = 1; + leaf->representation_id = 1; + leaf->right_sibling = top.leaf->right_sibling(); + leaf->bases.push_back({.identity = top.pid + 1, + .generation = captured->generation, + .range = top.range, + .max_slot = captured->coverage, + .backing = top.descriptor->backing}); + auto descriptor = *top.descriptor; + descriptor.summary = top.summary; + page->primary = make_primary_backing(descriptor, *leaf, captured->coverage, building->gc_floor, metadata); + page->leaf = std::move(leaf); + } + if (!building->summary || + !add_graph_summary(*building->summary, *top.descriptor, top.inner ? nullptr : &top.summary)) { + finish(Status::corruption("recovered summary overflow or invalid frame")); + return false; + } + MappingView::CopyStats copied; + const PageReplacement replacement{.page_id = top.pid, .expected = {}, .replacement = page}; + building->mapping = building->mapping.replace({&replacement, 1}, copied, metadata); + if (top.pid == captured->root_page_id) { + building->stats.height = static_cast<uint64_t>(level) + 1; + } + if (!apply_page_metrics(building->stats, nullptr, &page->metrics)) { + finish(Status::corruption()); + return false; + } + return true; +} + +void RecoveredStateOperation::State::verify_reachable() +{ + size_t work = 0; + while (position != captured->frames.end() && work++ < limits.max_step_entries) { + if (reached.erase(position->first) == 0) { + finish(Status::corruption()); + return; + } + if (!building->summary || (position->second.type == page_type::kOverflowFrame && + !add_graph_summary(*building->summary, position->second))) { + finish(Status::corruption("recovered overflow summary is invalid")); + return; + } + ++position; + } + if (position != captured->frames.end()) { + wake(); + return; + } + if (have_leaf && previous_sibling != kInvalidPageId) { + finish(Status::corruption()); + return; + } + finish(Status::Ok()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/recovered_state_internal.h b/lib/crowdb-tree/src/snapshot/recovered_state_internal.h new file mode 100644 index 000000000..7a5b97b07 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/recovered_state_internal.h @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/recovered_state.h" + +#include <set> + +namespace crowdb::tree +{ +struct RecoveredStateOperation::State : std::enable_shared_from_this<State> +{ + enum class Phase : uint8_t { Load, Inner, Children, Leaf, Dependencies, Finish }; + + struct Bounds + { + std::optional<OwnedKey> lower; + std::optional<OwnedKey> upper; + uint64_t lower_leaf = kInvalidPageId; + uint64_t upper_leaf = kInvalidPageId; + }; + + struct Path + { + uint64_t pid = kInvalidPageId; + const DurableCapturedFrame *descriptor = nullptr; + PageKeyRange range; + Bounds bounds; + std::shared_ptr<const InnerPageState> inner; + std::shared_ptr<const LeafBase> leaf; + PageMetricsContribution metrics; + FinalPageSummary summary; + std::optional<uint32_t> child_level; + size_t index = 0; + size_t key_bytes = 0; + size_t dependency = 0; + uint64_t chain_pid = kInvalidPageId; + uint64_t chain_bytes = 0; + Phase phase = Phase::Load; + }; + + struct Load + { + static constexpr unsigned kReturned = 1; + static constexpr unsigned kCallback = 2; + static constexpr unsigned kFailed = 4; + Status status; + Status submission; + std::shared_ptr<const LeafBase> leaf; + std::shared_ptr<const InnerPageState> inner; + std::atomic<bool> claimed{false}; + std::atomic<unsigned> events{0}; + }; + + using Pids = std::set<uint64_t, std::less<>, detail::ReservationAllocator<uint64_t>>; + explicit State(std::shared_ptr<detail::AllocationReservation> credit); + void start(detail::CallbackComplete callback, void *argument) noexcept; + void wake() noexcept; + static void run(void *context) noexcept; + void pump(); + void push(uint64_t pid, PageKeyRange range); + void begin_load(); + void loaded(); + void inner_step(); + void child_step(); + void leaf_step(); + void dependency_step(); + void complete_page(); + bool install_page(uint32_t level); + void verify_reachable(); + void finish(Status status) noexcept; + + std::shared_ptr<detail::AllocationReservation> metadata; + std::shared_ptr<const ReadablePageCatalog> captured; + std::shared_ptr<PageLoader> loader; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane; + std::shared_ptr<detail::FlushRuntime::Progress> progress; + RecoveredStateLimits limits; + std::shared_ptr<PublishedTreeState> building; + detail::ReservedVector<Path> path; + Pids reached; + std::shared_ptr<Load> load; + ReadablePageCatalog::Frames::const_iterator position; + uint64_t previous_sibling = kInvalidPageId; + bool have_leaf = false; + bool initialized = false; + bool require_durable = false; + size_t dependency_steps = 0; + std::atomic<uint8_t> lifecycle{0}; + std::atomic<bool> stopped{false}; + detail::CallbackComplete complete = nullptr; + void *context = nullptr; + RecoveredStateResult report; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/recovered_state_leaf.cpp b/lib/crowdb-tree/src/snapshot/recovered_state_leaf.cpp new file mode 100644 index 000000000..cfface443 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/recovered_state_leaf.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/recovered_state_internal.h" + +namespace crowdb::tree +{ +namespace +{ +bool valid_recovered_cell(Slice raw, uint64_t coverage) +{ + const CellView cell(raw); + return cell.valid() && cell.slot() <= coverage && + (cell.flags() == 0 || cell.flags() == kFlagTombstone || cell.flags() == kFlagOverflow) && + (!cell.is_tombstone() || raw.size() == kCellHeaderSize) && + (!cell.is_overflow() || + (raw.size() == kOverflowCellSize && cell.overflow_head() != kInvalidPageId && cell.overflow_len() != 0)); +} +} // namespace + +void RecoveredStateOperation::State::leaf_step() +{ + auto &top = path.back(); + size_t work = 0; + while (top.index < top.leaf->count() && work++ < limits.max_step_entries) { + const auto key = top.leaf->key(top.index); + const auto raw = top.leaf->cell(top.index); + if (key.size() > limits.max_key_bytes - top.key_bytes) { + finish(Status::resource_exhausted()); + return; + } + top.key_bytes += key.size(); + if (!top.range.contains(key) || !valid_recovered_cell(raw, captured->coverage) || + (top.index != 0 && top.leaf->key(top.index - 1) >= key)) { + finish(Status::corruption()); + return; + } + const CellView cell(raw); + const uint64_t primary_bytes = key.size() + raw.size() + kLeafSlotSize; + if (primary_bytes > UINT64_MAX - top.metrics.page_size_bytes) { + finish(Status::resource_exhausted()); + return; + } + top.metrics.page_size_bytes += primary_bytes; + if (!cell.is_tombstone()) { + const uint64_t value_bytes = cell.is_overflow() ? cell.overflow_len() : cell.value().size(); + if (key.size() > UINT64_MAX - value_bytes || key.size() + value_bytes > UINT64_MAX - top.metrics.kv_bytes) { + finish(Status::resource_exhausted()); + return; + } + ++top.metrics.kv_count; + top.summary.live_key_bytes += key.size(); + top.summary.live_value_bytes += value_bytes; + top.metrics.kv_bytes += key.size() + value_bytes; + } + ++top.index; + } + if (top.index == top.leaf->count()) { + if (top.index != 0) { + top.bounds.lower = OwnedKey(top.leaf->key(0), metadata); + top.bounds.upper = OwnedKey(top.leaf->key(top.index - 1), metadata); + top.bounds.lower_leaf = top.pid; + top.bounds.upper_leaf = top.pid; + } + top.metrics.page_size_estimate_bytes = top.metrics.page_size_bytes; + top.summary.live_counts_known = true; + top.summary.encoded_entries = top.index; + top.summary.live_keys = top.metrics.kv_count; + top.summary.primary_used_bytes = top.metrics.page_size_bytes; + top.phase = Phase::Dependencies; + } + wake(); +} + +void RecoveredStateOperation::State::dependency_step() +{ + auto &top = path.back(); + const auto &references = *top.descriptor->stored_overflow; + size_t work = 0; + while (top.dependency < references.size() && work++ < limits.max_step_entries) { + if (dependency_steps++ == limits.max_dependency_steps) { + finish(Status::resource_exhausted()); + return; + } + if (top.chain_pid == kInvalidPageId) { + const auto &reference = references[top.dependency]; + top.chain_pid = reference.head; + top.chain_bytes = reference.bytes; + } + const auto found = captured->frames.find(top.chain_pid); + if (top.chain_pid >= captured->next_page_id || found == captured->frames.end()) { + finish(Status::corruption()); + return; + } + const auto &frame = found->second; + if (frame.page_id != top.chain_pid || frame.type != page_type::kOverflowFrame || !frame.backing || + (!frame.backing->readable() || (require_durable && !frame.backing->reloadable())) || + frame.backing->backend_id != captured->backend_id || frame.overflow_bytes == 0 || + frame.overflow_bytes > top.chain_bytes || + (frame.next_overflow == kInvalidPageId) != (frame.overflow_bytes == top.chain_bytes)) { + finish(Status::corruption()); + return; + } + reached.insert(top.chain_pid); + top.chain_bytes -= frame.overflow_bytes; + top.chain_pid = frame.next_overflow; + if (top.chain_bytes == 0) { + ++top.dependency; + } + } + if (top.dependency == references.size()) { + top.phase = Phase::Finish; + } + wake(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/recovered_state_load.cpp b/lib/crowdb-tree/src/snapshot/recovered_state_load.cpp new file mode 100644 index 000000000..818133c63 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/recovered_state_load.cpp @@ -0,0 +1,159 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/recovered_frame_identity.h" +#include "snapshot/recovered_state_internal.h" + +namespace crowdb::tree +{ +void RecoveredStateOperation::State::begin_load() +{ + auto event = detail::reserved_shared<Load>(metadata); + load = event; + auto self = shared_from_this(); + const auto &top = path.back(); + ++report.loads; + try { + if (top.descriptor->type == page_type::kInnerBase) { + loader->load_inner(top.descriptor->backing, [self, event](Status status, auto inner) { + if (!event->claimed.exchange(true, std::memory_order_acq_rel)) { + event->status = std::move(status); + event->inner = std::move(inner); + event->events.fetch_or(Load::kCallback, std::memory_order_release); + self->wake(); + } + }); + } + else { + BaseRef base{.identity = top.pid + 1, + .generation = captured->generation, + .range = top.range, + .max_slot = captured->coverage, + .backing = top.descriptor->backing}; + loader->load_leaf(std::move(base), [self, event](Status status, auto leaf) { + if (!event->claimed.exchange(true, std::memory_order_acq_rel)) { + event->status = std::move(status); + event->leaf = std::move(leaf); + event->events.fetch_or(Load::kCallback, std::memory_order_release); + self->wake(); + } + }); + } + } + catch (...) { + event->submission = Status::io_error(); + event->events.fetch_or(Load::kFailed, std::memory_order_release); + } + event->events.fetch_or(Load::kReturned, std::memory_order_release); + wake(); +} + +void RecoveredStateOperation::State::loaded() +{ + auto event = std::move(load); + const auto flags = event->events.load(std::memory_order_acquire); + if ((flags & Load::kFailed) != 0 || !event->status.ok()) { + finish((flags & Load::kFailed) != 0 ? event->submission : event->status); + return; + } + auto &top = path.back(); + if (top.descriptor->type == page_type::kInnerBase) { + if (!event->inner || !event->inner->frame_proof || event->inner->children.empty() || + event->inner->children.size() != event->inner->separators.size() + 1 || + !recovered_frame_identity_matches(*top.descriptor->backing, page_type::kInnerBase, + event->inner->frame_proof->self_page_id, top.pid) || + event->inner->frame_proof->frame_bytes != top.descriptor->frame_bytes) { + finish(Status::corruption()); + return; + } + if (event->inner->children.size() > limits.max_children) { + finish(Status::resource_exhausted()); + return; + } + top.inner = std::move(event->inner); + top.phase = Phase::Inner; + top.metrics.leaf = false; + top.metrics.page_size_estimate_bytes = top.descriptor->frame_bytes; + } + else { + if (!event->leaf || + !recovered_frame_identity_matches(*top.descriptor->backing, page_type::kLeafBase, + event->leaf->view().self_page_id(), top.pid) || + event->leaf->page_bytes() != top.descriptor->frame_bytes || event->leaf->view().delta_count() != 0 || + !top.descriptor->stored_overflow) { + finish(Status::corruption()); + return; + } + if (event->leaf->count() > limits.max_leaf_entries || + top.descriptor->stored_overflow->size() > limits.max_leaf_entries) { + finish(Status::resource_exhausted()); + return; + } + if (have_leaf && previous_sibling != top.pid) { + finish(Status::corruption()); + return; + } + have_leaf = true; + previous_sibling = event->leaf->right_sibling(); + top.leaf = std::move(event->leaf); + top.phase = Phase::Leaf; + top.metrics.page_size_known = true; + top.metrics.page_size_bytes = kFrameHeaderSize + kFrameTrailerSize; + top.metrics.kv_known = true; + } + wake(); +} + +void RecoveredStateOperation::State::pump() +{ + ++report.cpu_steps; + if (load) { + const auto flags = load->events.load(std::memory_order_acquire); + if ((flags & Load::kReturned) == 0 || (flags & (Load::kCallback | Load::kFailed)) == 0) { + return; + } + } + if (stopped.load(std::memory_order_acquire)) { + finish(Status::unavailable()); + return; + } + if (load) { + loaded(); + return; + } + if (!initialized) { + initialized = true; + if (captured->root_page_id != kInvalidPageId) { + push(captured->root_page_id, {}); + return; + } + if (!captured->frames.empty()) { + finish(Status::corruption()); + return; + } + } + if (path.empty()) { + verify_reachable(); + return; + } + switch (path.back().phase) { + case Phase::Load: + begin_load(); + break; + case Phase::Inner: + inner_step(); + break; + case Phase::Children: + child_step(); + break; + case Phase::Leaf: + leaf_step(); + break; + case Phase::Dependencies: + dependency_step(); + break; + case Phase::Finish: + complete_page(); + break; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/retry.cpp b/lib/crowdb-tree/src/snapshot/retry.cpp deleted file mode 100644 index e83b9da85..000000000 --- a/lib/crowdb-tree/src/snapshot/retry.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. - -#include "async_completion_adapter.h" -#include "crowdb-common/log.h" -#include "crowdb-tree/backend/async_page_store.h" -#include "crowdb-tree/crowdb-tree.h" - -#include <memory> -#include <mutex> -#include <utility> - -namespace crowdb::tree -{ - -Status Crowdbtree::retry_snapshot_anchor() -{ - if (!snapshot_anchor_pending_) { - return Status::Ok(); - } - Status status = opt_.page_store->sync(); - if (status.ok()) { - snapshot_anchor_pending_ = false; - } - else { - CRB_LOG_ERROR("[{}] snapshot pending anchor sync failed: {}", name_, status.to_string()); - } - return status; -} - -void Crowdbtree::snapshot_prepare_async(std::function<void(Status, uint64_t)> on_done) -{ - if (snapshot_anchor_pending_) { - Status submitted = opt_.async_page_store->submit_fsync( - detail::own_async_completion([this, on_done](const Status &status) mutable { - if (!status.ok()) { - CRB_LOG_ERROR("[{}] snapshot pending anchor sync failed: {}", name_, status.to_string()); - release_snapshot_slot(); - on_done(status, 0); - return; - } - snapshot_anchor_pending_ = false; - snapshot_prepare_async(std::move(on_done)); - })); - if (!submitted.ok()) { - release_snapshot_slot(); - on_done(std::move(submitted), 0); - } - return; - } - auto prepared = std::make_shared<PreparedSnapshot>(); - Status status; - { - std::scoped_lock lock(write_mutex_); - status = prepare_snapshot_locked(prepared.get()); - } - if (!status.ok()) { - CRB_LOG_ERROR("[{}] snapshot prepare failed: {}", name_, status.to_string()); - release_snapshot_slot(); - on_done(std::move(status), 0); - return; - } - snapshot_write_next_async(std::move(prepared), 0, std::move(on_done)); -} - -} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/runtime_snapshot_import.cpp b/lib/crowdb-tree/src/snapshot/runtime_snapshot_import.cpp new file mode 100644 index 000000000..43d31f7dd --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/runtime_snapshot_import.cpp @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "runtime_snapshot_import.h" + +namespace crowdb::tree +{ +Status RuntimeSnapshotImport::create(TreeCore &core, std::shared_ptr<RuntimeSnapshotImport> *out) +try { + if (out == nullptr || *out) { + return Status::invalid_argument(); + } + std::shared_ptr<detail::AllocationReservation> metadata; + auto status = core.prepare_import_credit(&metadata); + if (!status.ok()) { + return status; + } + *out = detail::reserved_shared<RuntimeSnapshotImport>(metadata, core, metadata); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot import control allocation failed"); +} + +RuntimeSnapshotImport::RuntimeSnapshotImport(TreeCore &core, std::shared_ptr<detail::AllocationReservation> metadata) + : core_(core.weak_from_this()), + metadata_(std::move(metadata)), + loads_(core.load_limits()), + max_key_(core.config().max_key_size != 0 ? core.config().max_key_size : core.config().frame_bytes / 2), + key_(detail::ReservationAllocator<uint8_t, AllocationKind::kScratch>(metadata_)), + window_(detail::ReservationAllocator<uint8_t, AllocationKind::kScratch>(metadata_)) +{ +} + +RuntimeSnapshotImport::~RuntimeSnapshotImport() +{ + metadata_->seal(); +} + +Status RuntimeSnapshotImport::fail(Status status) +{ + if (status_.ok()) { + status_ = std::move(status); + } + portable_.reset(); + catalog_.reset(); + key_ = detail::ReservedVector<uint8_t, AllocationKind::kScratch>(key_.get_allocator()); + window_ = detail::ReservedVector<uint8_t, AllocationKind::kScratch>(window_.get_allocator()); + metadata_->seal(); + return status_; +} + +Status RuntimeSnapshotImport::feed(Slice bytes) +try { + if (!status_.ok()) { + return status_; + } + if (finishing_) { + return Status::unavailable("snapshot import is already finished"); + } + auto core = core_.lock(); + if (!core || core->closed()) { + return fail(Status::unavailable("snapshot import Tree is closed")); + } + auto status = parse({reinterpret_cast<const uint8_t *>(bytes.data()), bytes.size()}, *core); + return status.ok() ? status : fail(std::move(status)); +} +catch (const std::bad_alloc &) { + return fail(Status::resource_exhausted("snapshot import staging allocation failed")); +} + +Status RuntimeSnapshotImport::finish(uint64_t *coverage) +try { + if (!status_.ok()) { + return status_; + } + if (finishing_) { + return Status::unavailable("snapshot import is already finished"); + } + finishing_ = true; + if (part_ != Part::Complete || parsed_ != expected_) { + return fail(Status::invalid_argument("snapshot import stream is truncated")); + } + auto core = core_.lock(); + if (!core || core->closed()) { + return fail(Status::unavailable("snapshot import Tree is closed")); + } + if (native_) { + status_ = attach_native_dependencies(*catalog_, {}, metadata_); + } + else { + status_ = portable_->finish(&catalog_); + } + portable_.reset(); + key_ = detail::ReservedVector<uint8_t, AllocationKind::kScratch>(key_.get_allocator()); + window_ = detail::ReservedVector<uint8_t, AllocationKind::kScratch>(window_.get_allocator()); + if (status_.ok()) { + status_ = core->install_readable(std::move(catalog_), metadata_); + } + if (!status_.ok()) { + return fail(status_); + } + if (coverage != nullptr) { + *coverage = coverage_; + } + metadata_->seal(); + return Status::Ok(); +} +catch (const std::bad_alloc &) { + return fail(Status::resource_exhausted("snapshot import validation allocation failed")); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/runtime_snapshot_import.h b/lib/crowdb-tree/src/snapshot/runtime_snapshot_import.h new file mode 100644 index 000000000..7ee7d08a9 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/runtime_snapshot_import.h @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/portable_import.h" +#include "btree/tree_core.h" + +namespace crowdb::tree +{ +// A private staging stream holds byte admission but no accepted Tree ticket, +// backend alias, or IO permit while it waits for the caller's next chunk. +class RuntimeSnapshotImport +{ + public: + static Status create(TreeCore &core, std::shared_ptr<RuntimeSnapshotImport> *out); + RuntimeSnapshotImport(TreeCore &core, std::shared_ptr<detail::AllocationReservation> metadata); + ~RuntimeSnapshotImport(); + Status feed(Slice bytes); + Status finish(uint64_t *coverage); + + private: + enum class Part : uint8_t { + Header, + KeyLength, + Key, + Slot, + Kind, + ValueLength, + Value, + PageId, + FrameLength, + Frame, + Trailer, + Complete + }; + Status parse(std::span<const uint8_t> bytes, TreeCore &core); + Status advance(TreeCore &core); + Status advance_portable(); + Status advance_native(TreeCore &core); + Status begin(TreeCore &core); + void consume_fixed(std::span<const uint8_t> &bytes, bool include_crc); + void consume_window(std::span<const uint8_t> &bytes, + detail::ReservedVector<uint8_t, AllocationKind::kScratch> &window, size_t offset); + void expect(Part part, size_t bytes); + void completed_entry(); + void expect_entry(); + Status fail(Status status); + + std::weak_ptr<TreeCore> core_; + std::shared_ptr<detail::AllocationReservation> metadata_; + PageLoadLimits loads_; + size_t max_key_; + detail::ReservedUnique<PortableImportBuilder> portable_; + std::shared_ptr<ReadablePageCatalog> catalog_; + detail::ReservedVector<uint8_t, AllocationKind::kScratch> key_; + detail::ReservedVector<uint8_t, AllocationKind::kScratch> window_; + std::array<uint8_t, 33> fixed_{}; + size_t fixed_used_ = 0; + size_t required_ = 9; + size_t copied_ = 0; + uint64_t coverage_ = 0; + uint64_t expected_ = 0; + uint64_t parsed_ = 0; + uint64_t slot_ = 0; + uint64_t pid_ = kInvalidPageId; + uint32_t crc_ = 0; + uint8_t kind_ = 0; + bool native_ = false; + bool finishing_ = false; + Part part_ = Part::Header; + Status status_; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/runtime_snapshot_import_fields.cpp b/lib/crowdb-tree/src/snapshot/runtime_snapshot_import_fields.cpp new file mode 100644 index 000000000..3c0bf5a30 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/runtime_snapshot_import_fields.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "runtime_snapshot_import.h" + +namespace crowdb::tree +{ +Status RuntimeSnapshotImport::advance_portable() +{ + switch (part_) { + case Part::KeyLength: + required_ = frame_u32(fixed_.data(), 0); + if (required_ > max_key_) { + return Status::resource_exhausted("snapshot key exceeds Tree limit"); + } + key_.resize(required_); + expect(Part::Key, required_); + break; + case Part::Key: + expect(Part::Slot, 8); + break; + case Part::Slot: + slot_ = frame_u64(fixed_.data(), 0); + if (slot_ > coverage_) { + return Status::corruption("snapshot cell exceeds its coverage"); + } + expect(Part::Kind, 1); + break; + case Part::Kind: + kind_ = fixed_[0]; + if (kind_ > 1) { + return Status::corruption("snapshot cell kind is invalid"); + } + expect(Part::ValueLength, 4); + break; + case Part::ValueLength: + required_ = frame_u32(fixed_.data(), 0); + if (required_ > loads_.max_value_bytes) { + return Status::resource_exhausted("snapshot value exceeds Tree limit"); + } + if (kind_ != 0 && required_ != 0) { + return Status::corruption("snapshot tombstone contains a value"); + } + window_.resize(kCellHeaderSize + required_); + frame_put_u64(window_.data(), 0, slot_); + window_[sizeof(uint64_t)] = kind_ != 0 ? kFlagTombstone : 0; + expect(Part::Value, required_); + break; + case Part::Value: { + auto status = portable_->append(Slice(key_.data(), key_.size()), Slice(window_.data(), window_.size())); + if (!status.ok()) { + return status; + } + completed_entry(); + break; + } + default: + return Status::internal_error("invalid portable parser state"); + } + return Status::Ok(); +} + +Status RuntimeSnapshotImport::advance_native(TreeCore &core) +{ + switch (part_) { + case Part::PageId: + pid_ = frame_u64(fixed_.data(), 0); + expect(Part::FrameLength, 4); + break; + case Part::FrameLength: + required_ = frame_u32(fixed_.data(), 0); + if (required_ > loads_.max_frame_bytes || required_ < kFrameHeaderSize + kFrameTrailerSize) { + return Status::resource_exhausted("snapshot frame size exceeds Tree limit"); + } + window_.resize(required_); + expect(Part::Frame, required_); + break; + case Part::Frame: { + auto status = append_native_frame(*catalog_, pid_, window_, core.config(), loads_, {}, metadata_); + if (!status.ok()) { + return status; + } + completed_entry(); + break; + } + default: + return Status::internal_error("invalid native parser state"); + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/runtime_snapshot_import_parse.cpp b/lib/crowdb-tree/src/snapshot/runtime_snapshot_import_parse.cpp new file mode 100644 index 000000000..6e7f9febc --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/runtime_snapshot_import_parse.cpp @@ -0,0 +1,146 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "runtime_snapshot_import.h" +#include "snapshot_wire.h" + +namespace crowdb::tree +{ +void RuntimeSnapshotImport::expect(Part part, size_t bytes) +{ + part_ = part; + required_ = bytes; + fixed_used_ = 0; + copied_ = 0; +} + +void RuntimeSnapshotImport::consume_fixed(std::span<const uint8_t> &bytes, bool include_crc) +{ + const auto take = std::min(bytes.size(), required_ - fixed_used_); + std::copy_n(bytes.data(), take, fixed_.data() + fixed_used_); + if (include_crc) { + crc_ = snapshot_detail::crc_bytes(crc_, bytes.first(take)); + } + fixed_used_ += take; + bytes = bytes.subspan(take); +} + +void RuntimeSnapshotImport::consume_window(std::span<const uint8_t> &bytes, + detail::ReservedVector<uint8_t, AllocationKind::kScratch> &window, + size_t offset) +{ + const auto take = std::min(bytes.size(), required_ - copied_); + if (take == 0) { + return; + } + std::copy_n(bytes.data(), take, window.data() + offset + copied_); + crc_ = snapshot_detail::crc_bytes(crc_, bytes.first(take)); + copied_ += take; + bytes = bytes.subspan(take); +} + +void RuntimeSnapshotImport::expect_entry() +{ + if (parsed_ == expected_) { + expect(Part::Trailer, 4); + } + else { + expect(native_ ? Part::PageId : Part::KeyLength, native_ ? 8 : 4); + } +} + +void RuntimeSnapshotImport::completed_entry() +{ + ++parsed_; + expect_entry(); +} + +Status RuntimeSnapshotImport::begin(TreeCore &core) +{ + coverage_ = frame_u64(fixed_.data(), 9); + expected_ = frame_u64(fixed_.data(), native_ ? 25 : 17); + const auto max_entries = uint64_t{NativeImportLimits{}.max_frames} * loads_.max_leaf_entries; + if (expected_ > (native_ ? NativeImportLimits{}.max_frames : max_entries)) { + return Status::resource_exhausted("snapshot import count exceeds its frame limits"); + } + if (native_) { + catalog_ = detail::reserved_shared<ReadablePageCatalog>(metadata_, metadata_); + catalog_->coverage = coverage_; + catalog_->generation = 1; + catalog_->root_page_id = frame_u64(fixed_.data(), 17); + } + else { + portable_ = + detail::reserved_unique<PortableImportBuilder>(metadata_, core.config(), loads_, coverage_, metadata_); + } + expect_entry(); + return Status::Ok(); +} + +Status RuntimeSnapshotImport::advance(TreeCore &core) +{ + switch (part_) { + case Part::Header: + if (required_ == 9) { + if (frame_u32(fixed_.data(), 0) != 0x4E535443) { + return Status::corruption("snapshot: bad magic"); + } + if (frame_u32(fixed_.data(), 4) != 1) { + return Status::not_supported("snapshot: version"); + } + if (fixed_[8] > 1) { + return Status::not_supported("snapshot: unknown format"); + } + native_ = fixed_[8] == 1; + required_ = native_ ? 33 : 25; + return Status::Ok(); + } + return begin(core); + case Part::KeyLength: + case Part::Key: + case Part::Slot: + case Part::Kind: + case Part::ValueLength: + case Part::Value: + return advance_portable(); + case Part::PageId: + case Part::FrameLength: + case Part::Frame: + return advance_native(core); + case Part::Trailer: + if (frame_u32(fixed_.data(), 0) != crc_) { + return Status::corruption("snapshot: CRC mismatch"); + } + expect(Part::Complete, 0); + break; + case Part::Complete: + return Status::corruption("snapshot: trailing bytes"); + } + return Status::Ok(); +} + +Status RuntimeSnapshotImport::parse(std::span<const uint8_t> bytes, TreeCore &core) +{ + for (;;) { + const bool window = part_ == Part::Key || part_ == Part::Value || part_ == Part::Frame; + if (part_ == Part::Complete) { + return bytes.empty() ? Status::Ok() : Status::corruption("snapshot: trailing bytes"); + } + if (bytes.empty() && (window ? copied_ : fixed_used_) != required_) { + return Status::Ok(); + } + if (window) { + consume_window(bytes, part_ == Part::Key ? key_ : window_, part_ == Part::Value ? kCellHeaderSize : 0); + } + else { + consume_fixed(bytes, part_ != Part::Trailer); + } + if ((window ? copied_ : fixed_used_) != required_) { + return Status::Ok(); + } + auto status = advance(core); + if (!status.ok()) { + return status; + } + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/snapshot_baseline.h b/lib/crowdb-tree/src/snapshot/snapshot_baseline.h new file mode 100644 index 000000000..4db28c4ec --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/snapshot_baseline.h @@ -0,0 +1,25 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <cstdint> +#include <memory> + +namespace crowdb::tree +{ +struct CapturedDurablePages; + +// Durable receipts have an owner independent of live representation rebase. +// This object deliberately retains neither source mappings nor overlay runs. +struct SnapshotBaseline +{ + uint64_t generation = 0; + uint64_t capture_token = 0; + uint64_t coverage = 0; + uint64_t snapshot_sequence = 0; + uint64_t backend_id = 0; + uint64_t gc_floor = 0; + uint64_t wal_offset = 0; + std::shared_ptr<const CapturedDurablePages> pages; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/snapshot_crc.h b/lib/crowdb-tree/src/snapshot/snapshot_crc.h new file mode 100644 index 000000000..ddd0d8011 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/snapshot_crc.h @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include <array> +#include <cstddef> +#include <cstdint> + +namespace crowdb::tree::snapshot_detail +{ +// Concatenate two independently accumulated CRC32C values without retaining +// their bytes. Powers of the reflected byte transition bound work to 64 steps. +inline uint32_t crc_concatenate(uint32_t prefix, uint32_t suffix, uint64_t suffix_bytes) noexcept +{ + using Matrix = std::array<uint32_t, 32>; + auto apply = [](const Matrix &matrix, uint32_t bits) { + uint32_t result = 0; + for (size_t i = 0; i < matrix.size(); ++i) { + if ((bits & (uint32_t{1} << i)) != 0) { + result ^= matrix[i]; + } + } + return result; + }; + Matrix byte{}; + for (size_t i = 0; i < byte.size(); ++i) { + uint32_t state = uint32_t{1} << i; + for (unsigned bit = 0; bit != 8; ++bit) { + state = (state >> 1) ^ ((state & 1U) != 0 ? 0x82f63b78U : 0U); + } + byte[i] = state; + } + while (suffix_bytes != 0) { + if ((suffix_bytes & 1U) != 0) { + prefix = apply(byte, prefix); + } + suffix_bytes >>= 1; + if (suffix_bytes != 0) { + Matrix squared{}; + for (size_t i = 0; i < byte.size(); ++i) { + squared[i] = apply(byte, byte[i]); + } + byte = squared; + } + } + return prefix ^ suffix; +} +} // namespace crowdb::tree::snapshot_detail diff --git a/lib/crowdb-tree/src/snapshot/snapshot_export_materialized.cpp b/lib/crowdb-tree/src/snapshot/snapshot_export_materialized.cpp new file mode 100644 index 000000000..eba34b896 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/snapshot_export_materialized.cpp @@ -0,0 +1,191 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/btree/cell.h" +#include "crowdb-tree/snapshot/snapshot.h" +#include "crowdb-tree/snapshot/snapshot_io.h" + +#include <algorithm> +#include <array> +#include <limits> + +namespace crowdb::tree +{ +namespace +{ +constexpr uint32_t kSnapMagic = 0x4E535443; +constexpr uint32_t kSnapVersion = 1; +constexpr size_t kSnapHeader = 4 + 4 + 1 + 8 + 8; +constexpr size_t kSnapTrailer = 4; + +void encode_u32(uint32_t v, std::array<uint8_t, 4> *out) +{ + for (int i = 0; i < 4; ++i) { + (*out)[static_cast<size_t>(i)] = static_cast<uint8_t>((v >> (8 * i)) & 0xff); + } +} + +void encode_u64(uint64_t v, std::array<uint8_t, 8> *out) +{ + for (int i = 0; i < 8; ++i) { + (*out)[static_cast<size_t>(i)] = static_cast<uint8_t>((v >> (8 * i)) & 0xff); + } +} + +template <typename Fn> void for_each_portable_piece(const Snapshot &snapshot, Fn &&fn) +{ + std::array<uint8_t, kSnapHeader> header{}; + std::array<uint8_t, 4> u32{}; + std::array<uint8_t, 8> u64{}; + encode_u32(kSnapMagic, &u32); + std::copy(u32.begin(), u32.end(), header.begin()); + encode_u32(kSnapVersion, &u32); + std::copy(u32.begin(), u32.end(), header.begin() + 4); + header[8] = static_cast<uint8_t>(snapshot_format::kPortable); + encode_u64(snapshot.at_slot(), &u64); + std::copy(u64.begin(), u64.end(), header.begin() + 9); + encode_u64(static_cast<uint64_t>(snapshot.entries().size()), &u64); + std::copy(u64.begin(), u64.end(), header.begin() + 17); + fn(header.data(), header.size()); + + for (const leaf_entry &entry : snapshot.entries()) { + CellView value{Slice(entry.cell)}; + encode_u32(static_cast<uint32_t>(entry.key.size()), &u32); + fn(u32.data(), u32.size()); + fn(reinterpret_cast<const uint8_t *>(entry.key.data()), entry.key.size()); + encode_u64(value.slot(), &u64); + fn(u64.data(), u64.size()); + const uint8_t kind = value.is_tombstone() ? 1 : 0; + fn(&kind, 1); + Slice bytes = value.value(); + encode_u32(static_cast<uint32_t>(bytes.size()), &u32); + fn(u32.data(), u32.size()); + fn(reinterpret_cast<const uint8_t *>(bytes.data()), bytes.size()); + } +} + +} // namespace + +SnapshotExport::SnapshotExport(std::shared_ptr<Snapshot> snapshot, size_t chunk_bytes) + : snapshot_(std::move(snapshot)), + chunk_bytes_(chunk_bytes == 0 ? kSnapshotChunkBytes : chunk_bytes), + at_slot_(snapshot_->at_slot()) +{ + uint32_t crc = 0; + size_t len = 0; + for_each_portable_piece(*snapshot_, [&](const uint8_t *data, size_t size) { + if (size > std::numeric_limits<size_t>::max() - len) { + len = std::numeric_limits<size_t>::max(); + return; + } + len += size; + crc = crowdb::common::crc32c_update(crc, data, size); + }); + final_crc32c_ = crc; + total_bytes_ = len + kSnapTrailer; +} + +Status SnapshotExport::next_portable_chunk(std::string *out, bool *done) +{ + out->clear(); + out->reserve(chunk_bytes_); + const size_t body_piece_count = 1 + (snapshot_->entries().size() * 6); + const size_t trailer_piece = body_piece_count; + + auto append_piece = [&](const uint8_t *data, size_t size) { + const size_t available = chunk_bytes_ - out->size(); + const size_t remaining = size - piece_offset_; + const size_t take = std::min(available, remaining); + out->append(reinterpret_cast<const char *>(data + piece_offset_), take); + piece_offset_ += take; + pos_ += take; + if (piece_offset_ == size) { + piece_offset_ = 0; + ++piece_index_; + } + }; + + while (out->size() < chunk_bytes_ && piece_index_ <= trailer_piece) { + std::array<uint8_t, kSnapHeader> header{}; + std::array<uint8_t, 4> u32{}; + std::array<uint8_t, 8> u64{}; + const uint8_t *data = nullptr; + size_t size = 0; + + if (piece_index_ == 0) { + encode_u32(kSnapMagic, &u32); + std::copy(u32.begin(), u32.end(), header.begin()); + encode_u32(kSnapVersion, &u32); + std::copy(u32.begin(), u32.end(), header.begin() + 4); + header[8] = static_cast<uint8_t>(snapshot_format::kPortable); + encode_u64(at_slot_, &u64); + std::copy(u64.begin(), u64.end(), header.begin() + 9); + encode_u64(static_cast<uint64_t>(snapshot_->entries().size()), &u64); + std::copy(u64.begin(), u64.end(), header.begin() + 17); + data = header.data(); + size = header.size(); + } + else if (piece_index_ == trailer_piece) { + encode_u32(final_crc32c_, &u32); + data = u32.data(); + size = u32.size(); + } + else { + const size_t encoded = piece_index_ - 1; + const leaf_entry &entry = snapshot_->entries()[encoded / 6]; + const size_t part = encoded % 6; + CellView value{Slice(entry.cell)}; + switch (part) { + case 0: + encode_u32(static_cast<uint32_t>(entry.key.size()), &u32); + data = u32.data(); + size = u32.size(); + break; + case 1: + data = reinterpret_cast<const uint8_t *>(entry.key.data()); + size = entry.key.size(); + break; + case 2: + encode_u64(value.slot(), &u64); + data = u64.data(); + size = u64.size(); + break; + case 3: { + static constexpr uint8_t kPut = 0; + static constexpr uint8_t kDelete = 1; + data = value.is_tombstone() ? &kDelete : &kPut; + size = 1; + break; + } + case 4: { + Slice bytes = value.value(); + encode_u32(static_cast<uint32_t>(bytes.size()), &u32); + data = u32.data(); + size = u32.size(); + break; + } + case 5: { + Slice bytes = value.value(); + data = reinterpret_cast<const uint8_t *>(bytes.data()); + size = bytes.size(); + break; + } + default: + return Status::corruption("snapshot export: invalid encoder state"); + } + } + if (size == 0) { + ++piece_index_; + piece_offset_ = 0; + continue; + } + append_piece(data, size); + } + if (done != nullptr) { + *done = piece_index_ > trailer_piece; + } + return Status::Ok(); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/snapshot_export_owned.cpp b/lib/crowdb-tree/src/snapshot/snapshot_export_owned.cpp new file mode 100644 index 000000000..bb5ca1280 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/snapshot_export_owned.cpp @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/snapshot/snapshot_io.h" +#include "snapshot/published_byte_export.h" + +namespace crowdb::tree +{ +SnapshotExport::SnapshotExport(std::shared_ptr<PublishedByteExport> runtime) + : chunk_bytes_(runtime->info().chunk_bytes), + at_slot_(runtime->info().slot), + total_bytes_(runtime->info().total_bytes), + final_crc32c_(runtime->info().final_crc), + runtime_(std::move(runtime)), + facade_credit_(runtime_->metadata(), sizeof(SnapshotExport), AllocationKind::kMetadata) +{ + runtime_->metadata()->seal(); +} + +Status SnapshotExport::next_owned(SnapshotChunk *out, bool *done) +try { + if (out == nullptr || done == nullptr) { + return Status::invalid_argument("snapshot chunk requires output and completion"); + } + *out = {}; + if (runtime_) { + auto status = runtime_->next(out, done); + pos_ = runtime_->info().offset; + return status; + } + auto bytes = std::make_shared<std::string>(); + auto status = next_chunk(bytes.get(), done); + if (status.ok()) { + *out = { + .bytes = {reinterpret_cast<const uint8_t *>(bytes->data()), bytes->size()}, + .owner = std::move(bytes), + .allocation_budget = {} + }; + } + return status; +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot chunk allocation failed"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/snapshot_io.cpp b/lib/crowdb-tree/src/snapshot/snapshot_io.cpp index 2c80beaa9..1a2ec8a9e 100644 --- a/lib/crowdb-tree/src/snapshot/snapshot_io.cpp +++ b/lib/crowdb-tree/src/snapshot/snapshot_io.cpp @@ -1,171 +1,30 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// B+tree snapshot stream implementation. - #include "crowdb-tree/snapshot/snapshot_io.h" -#include "crowdb-common/crc32c.h" -#include "crowdb-tree/btree/cell.h" -#include "crowdb-tree/crowdb-tree.h" -#include "crowdb-tree/maptable/page_types.h" +#include "btree/tree_core_access.h" +#include "snapshot/published_byte_export.h" +#include "snapshot/runtime_snapshot_import.h" #include <array> -#include <cstring> #include <fstream> -#include <limits> -#include <vector> namespace crowdb::tree { - -namespace -{ - -constexpr uint32_t kSnapMagic = 0x4E535443; // 'CTSN' little-endian -constexpr uint32_t kSnapVersion = 1; -// Portable header: magic + version + format + at_slot + entry_count. -constexpr size_t kSnapHeader = 4 + 4 + 1 + 8 + 8; -constexpr size_t kSnapTrailer = 4; // whole-stream CRC32C - -// Native header: magic + version + format + at_slot + root_page_id + frame_count. -constexpr size_t kNativeHeader = 4 + 4 + 1 + 8 + 8 + 8; - -void put_u32(std::string *o, uint32_t v) -{ - for (int i = 0; i < 4; ++i) { - o->push_back(static_cast<char>((v >> (8 * i)) & 0xff)); - } -} - -void put_u64(std::string *o, uint64_t v) -{ - for (int i = 0; i < 8; ++i) { - o->push_back(static_cast<char>((v >> (8 * i)) & 0xff)); - } -} - -void encode_u32(uint32_t v, std::array<uint8_t, 4> *out) -{ - for (int i = 0; i < 4; ++i) { - (*out)[static_cast<size_t>(i)] = static_cast<uint8_t>((v >> (8 * i)) & 0xff); - } -} - -void encode_u64(uint64_t v, std::array<uint8_t, 8> *out) -{ - for (int i = 0; i < 8; ++i) { - (*out)[static_cast<size_t>(i)] = static_cast<uint8_t>((v >> (8 * i)) & 0xff); - } -} - -template <typename Fn> void for_each_portable_piece(const Snapshot &snapshot, Fn &&fn) -{ - std::array<uint8_t, kSnapHeader> header{}; - std::array<uint8_t, 4> u32{}; - std::array<uint8_t, 8> u64{}; - encode_u32(kSnapMagic, &u32); - std::copy(u32.begin(), u32.end(), header.begin()); - encode_u32(kSnapVersion, &u32); - std::copy(u32.begin(), u32.end(), header.begin() + 4); - header[8] = static_cast<uint8_t>(snapshot_format::kPortable); - encode_u64(snapshot.at_slot(), &u64); - std::copy(u64.begin(), u64.end(), header.begin() + 9); - encode_u64(static_cast<uint64_t>(snapshot.entries().size()), &u64); - std::copy(u64.begin(), u64.end(), header.begin() + 17); - fn(header.data(), header.size()); - - for (const leaf_entry &entry : snapshot.entries()) { - CellView value{Slice(entry.cell)}; - encode_u32(static_cast<uint32_t>(entry.key.size()), &u32); - fn(u32.data(), u32.size()); - fn(reinterpret_cast<const uint8_t *>(entry.key.data()), entry.key.size()); - encode_u64(value.slot(), &u64); - fn(u64.data(), u64.size()); - const uint8_t kind = value.is_tombstone() ? 1 : 0; - fn(&kind, 1); - Slice bytes = value.value(); - encode_u32(static_cast<uint32_t>(bytes.size()), &u32); - fn(u32.data(), u32.size()); - fn(reinterpret_cast<const uint8_t *>(bytes.data()), bytes.size()); - } -} - -uint32_t get_u32(const uint8_t *p) -{ - uint32_t v = 0; - for (int i = 0; i < 4; ++i) { - v |= static_cast<uint32_t>(p[i]) << (8 * i); - } - return v; -} - -uint64_t get_u64(const uint8_t *p) -{ - uint64_t v = 0; - for (int i = 0; i < 8; ++i) { - v |= static_cast<uint64_t>(p[i]) << (8 * i); - } - return v; -} - -Status snapshot_export_begin_portable(Crowdbtree &tree, size_t chunk_bytes, std::unique_ptr<SnapshotExport> *out) -{ - // v1 always exports the current durable view (its last_applied_slot is recorded - // in the stream header). An arbitrary historical pin is deferred until - // path-copy COW RootVersions exist. - std::shared_ptr<Snapshot> snap = tree.snapshot_view(); - auto exp = std::make_unique<SnapshotExport>(std::move(snap), chunk_bytes); - *out = std::move(exp); - return Status::Ok(); -} - -// plan-tree #16: native format -- raw leaf/inner/overflow frame bytes -// tagged with their PID, no cell decode/tuple encode. Body: -// `[u64 page_id][u32 frame_len][frame_len bytes] * frame_count`. -Status snapshot_export_begin_native(Crowdbtree &tree, size_t chunk_bytes, std::unique_ptr<SnapshotExport> *out) -{ - std::vector<NativeFrame> frames; - uint64_t root_page_id = 0; - uint64_t slot = 0; - Status cs = tree.collect_native_frames(&frames, &root_page_id, &slot); - if (!cs.ok()) { - return cs; - } - - std::string s; - put_u32(&s, kSnapMagic); - put_u32(&s, kSnapVersion); - s.push_back(static_cast<char>(snapshot_format::kNative)); - put_u64(&s, slot); - put_u64(&s, root_page_id); - put_u64(&s, static_cast<uint64_t>(frames.size())); - for (const NativeFrame &f : frames) { - put_u64(&s, f.page_id); - put_u32(&s, static_cast<uint32_t>(f.frame.size())); - s.append(reinterpret_cast<const char *>(f.frame.data()), f.frame.size()); - } - uint32_t crc = crowdb::common::crc32c(reinterpret_cast<const uint8_t *>(s.data()), s.size()); - put_u32(&s, crc); - - auto exp = std::make_unique<SnapshotExport>(std::move(s), chunk_bytes, slot); - *out = std::move(exp); - return Status::Ok(); -} - -} // namespace - Status snapshot_export_begin(Crowdbtree &tree, snapshot_format fmt, size_t chunk_bytes, std::unique_ptr<SnapshotExport> *out) try { - switch (fmt) { - case snapshot_format::kPortable: - return snapshot_export_begin_portable(tree, chunk_bytes, out); - case snapshot_format::kNative: - return snapshot_export_begin_native(tree, chunk_bytes, out); - default: - return Status::not_supported("snapshot export: unknown format"); + if (out == nullptr) { + return Status::invalid_argument("snapshot export needs output"); } + auto *core = TreeCoreAccess::get(tree); + std::shared_ptr<PublishedByteExport> runtime; + auto status = PublishedByteExport::create(core->shared_from_this(), fmt, chunk_bytes, &runtime); + if (status.ok()) { + *out = std::make_unique<SnapshotExport>(std::move(runtime)); + } + return status; } catch (const std::bad_alloc &) { return Status::resource_exhausted("snapshot export allocation failed"); @@ -176,6 +35,24 @@ catch (const std::exception &error) { Status SnapshotExport::next_chunk(std::string *out, bool *done) { + if (out == nullptr) { + return Status::invalid_argument("snapshot chunk needs output"); + } + if (runtime_) { + SnapshotChunk chunk; + bool complete = false; + auto status = next_owned(&chunk, &complete); + out->clear(); + if (status.ok()) { + if (!chunk.bytes.empty()) { + out->assign(reinterpret_cast<const char *>(chunk.bytes.data()), chunk.bytes.size()); + } + if (done != nullptr) { + *done = complete; + } + } + return status; + } if (snapshot_ != nullptr) { return next_portable_chunk(out, done); } @@ -190,127 +67,6 @@ Status SnapshotExport::next_chunk(std::string *out, bool *done) return Status::Ok(); } -SnapshotExport::SnapshotExport(std::shared_ptr<Snapshot> snapshot, size_t chunk_bytes) - : snapshot_(std::move(snapshot)), - chunk_bytes_(chunk_bytes == 0 ? kSnapshotChunkBytes : chunk_bytes), - at_slot_(snapshot_->at_slot()) -{ - uint32_t crc = 0; - size_t len = 0; - for_each_portable_piece(*snapshot_, [&](const uint8_t *data, size_t size) { - if (size > std::numeric_limits<size_t>::max() - len) { - len = std::numeric_limits<size_t>::max(); - return; - } - len += size; - crc = crowdb::common::crc32c_update(crc, data, size); - }); - final_crc32c_ = crc; - total_bytes_ = len + kSnapTrailer; -} - -Status SnapshotExport::next_portable_chunk(std::string *out, bool *done) -{ - out->clear(); - out->reserve(chunk_bytes_); - const size_t body_piece_count = 1 + (snapshot_->entries().size() * 6); - const size_t trailer_piece = body_piece_count; - - auto append_piece = [&](const uint8_t *data, size_t size) { - const size_t available = chunk_bytes_ - out->size(); - const size_t remaining = size - piece_offset_; - const size_t take = std::min(available, remaining); - out->append(reinterpret_cast<const char *>(data + piece_offset_), take); - piece_offset_ += take; - pos_ += take; - if (piece_offset_ == size) { - piece_offset_ = 0; - ++piece_index_; - } - }; - - while (out->size() < chunk_bytes_ && piece_index_ <= trailer_piece) { - std::array<uint8_t, kSnapHeader> header{}; - std::array<uint8_t, 4> u32{}; - std::array<uint8_t, 8> u64{}; - const uint8_t *data = nullptr; - size_t size = 0; - - if (piece_index_ == 0) { - encode_u32(kSnapMagic, &u32); - std::copy(u32.begin(), u32.end(), header.begin()); - encode_u32(kSnapVersion, &u32); - std::copy(u32.begin(), u32.end(), header.begin() + 4); - header[8] = static_cast<uint8_t>(snapshot_format::kPortable); - encode_u64(at_slot_, &u64); - std::copy(u64.begin(), u64.end(), header.begin() + 9); - encode_u64(static_cast<uint64_t>(snapshot_->entries().size()), &u64); - std::copy(u64.begin(), u64.end(), header.begin() + 17); - data = header.data(); - size = header.size(); - } - else if (piece_index_ == trailer_piece) { - encode_u32(final_crc32c_, &u32); - data = u32.data(); - size = u32.size(); - } - else { - const size_t encoded = piece_index_ - 1; - const leaf_entry &entry = snapshot_->entries()[encoded / 6]; - const size_t part = encoded % 6; - CellView value{Slice(entry.cell)}; - switch (part) { - case 0: - encode_u32(static_cast<uint32_t>(entry.key.size()), &u32); - data = u32.data(); - size = u32.size(); - break; - case 1: - data = reinterpret_cast<const uint8_t *>(entry.key.data()); - size = entry.key.size(); - break; - case 2: - encode_u64(value.slot(), &u64); - data = u64.data(); - size = u64.size(); - break; - case 3: { - static constexpr uint8_t kPut = 0; - static constexpr uint8_t kDelete = 1; - data = value.is_tombstone() ? &kDelete : &kPut; - size = 1; - break; - } - case 4: { - Slice bytes = value.value(); - encode_u32(static_cast<uint32_t>(bytes.size()), &u32); - data = u32.data(); - size = u32.size(); - break; - } - case 5: { - Slice bytes = value.value(); - data = reinterpret_cast<const uint8_t *>(bytes.data()); - size = bytes.size(); - break; - } - default: - return Status::corruption("snapshot export: invalid encoder state"); - } - } - if (size == 0) { - ++piece_index_; - piece_offset_ = 0; - continue; - } - append_piece(data, size); - } - if (done != nullptr) { - *done = piece_index_ > trailer_piece; - } - return Status::Ok(); -} - Status snapshot_dump_to_file(Crowdbtree &tree, snapshot_format fmt, const std::string &path) { std::unique_ptr<SnapshotExport> exp; @@ -343,162 +99,6 @@ Status snapshot_dump_to_file(Crowdbtree &tree, snapshot_format fmt, const std::s return Status::Ok(); } -struct SnapshotImport::PortableState -{ - enum class Part : uint8_t { kKeyLen, kKey, kSlot, kKind, kValueLen, kValue, kTrailer, kComplete }; - - uint64_t at_slot = 0; - uint64_t expected = 0; - uint64_t parsed = 0; - uint32_t crc = 0; - Part part = Part::kKeyLen; - std::array<uint8_t, 8> fixed{}; - size_t fixed_used = 0; - uint32_t key_len = 0; - uint64_t slot = 0; - uint8_t kind = 0; - uint32_t value_len = 0; - std::string key; - std::string value; - std::vector<leaf_entry> entries; - - Status begin(const uint8_t *header) - { - if (get_u32(header) != kSnapMagic) { - return Status::corruption("snapshot: bad magic"); - } - if (get_u32(header + 4) != kSnapVersion) { - return Status::not_supported("snapshot: version"); - } - if (static_cast<snapshot_format>(header[8]) != snapshot_format::kPortable) { - return Status::not_supported("snapshot: unknown format"); - } - at_slot = get_u64(header + 9); - expected = get_u64(header + 17); - crc = crowdb::common::crc32c_update(0, header, kSnapHeader); - part = expected == 0 ? Part::kTrailer : Part::kKeyLen; - return Status::Ok(); - } - - void reset_fixed() - { - fixed_used = 0; - } - - void consume_fixed(const uint8_t **data, size_t *len, size_t need, bool include_crc) - { - size_t take = std::min(*len, need - fixed_used); - std::copy_n(*data, take, fixed.data() + fixed_used); - if (include_crc) { - crc = crowdb::common::crc32c_update(crc, *data, take); - } - fixed_used += take; - *data += take; - *len -= take; - } - - void consume_string(const uint8_t **data, size_t *len, std::string *out, size_t need) - { - size_t take = std::min(*len, need - out->size()); - out->append(reinterpret_cast<const char *>(*data), take); - crc = crowdb::common::crc32c_update(crc, *data, take); - *data += take; - *len -= take; - } - - Status feed(const uint8_t *data, size_t len) - { - while (len > 0) { - switch (part) { - case Part::kKeyLen: - consume_fixed(&data, &len, 4, true); - if (fixed_used == 4) { - key_len = get_u32(fixed.data()); - key.clear(); - reset_fixed(); - part = key_len == 0 ? Part::kSlot : Part::kKey; - } - break; - case Part::kKey: - consume_string(&data, &len, &key, key_len); - if (key.size() == key_len) { - part = Part::kSlot; - } - break; - case Part::kSlot: - consume_fixed(&data, &len, 8, true); - if (fixed_used == 8) { - slot = get_u64(fixed.data()); - reset_fixed(); - part = Part::kKind; - } - break; - case Part::kKind: - consume_fixed(&data, &len, 1, true); - if (fixed_used == 1) { - kind = fixed[0]; - reset_fixed(); - if (kind > 1) { - return Status::corruption("snapshot: invalid cell kind"); - } - part = Part::kValueLen; - } - break; - case Part::kValueLen: - consume_fixed(&data, &len, 4, true); - if (fixed_used == 4) { - value_len = get_u32(fixed.data()); - value.clear(); - reset_fixed(); - part = Part::kValue; - } - break; - case Part::kValue: - consume_string(&data, &len, &value, value_len); - if (value.size() == value_len) { - buffer cell = encode_cell_buf(slot, kind != 0 ? OpKind::kDelete : OpKind::kPut, Slice(value)); - entries.push_back({.key = std::move(key), .cell = std::move(cell)}); - value.clear(); - ++parsed; - if (parsed > expected) { - return Status::corruption("snapshot: entry count overflow"); - } - part = parsed == expected ? Part::kTrailer : Part::kKeyLen; - } - break; - case Part::kTrailer: - consume_fixed(&data, &len, kSnapTrailer, false); - if (fixed_used == kSnapTrailer) { - if (get_u32(fixed.data()) != crc) { - return Status::corruption("snapshot: CRC mismatch"); - } - reset_fixed(); - part = Part::kComplete; - } - break; - case Part::kComplete: - return Status::corruption("snapshot: trailing bytes"); - } - } - return Status::Ok(); - } - - Status finish(Crowdbtree &tree, uint64_t *out_at_slot) - { - if (part != Part::kComplete || parsed != expected) { - return Status::invalid_argument("snapshot: stream truncated"); - } - Status installed = tree.install_snapshot(std::move(entries), at_slot); - if (!installed.ok()) { - return installed; - } - if (out_at_slot != nullptr) { - *out_at_slot = at_slot; - } - return Status::Ok(); - } -}; - SnapshotImport::SnapshotImport(Crowdbtree &tree) : tree_(tree) { } @@ -507,116 +107,21 @@ SnapshotImport::~SnapshotImport() = default; Status SnapshotImport::feed(Slice chunk) { - const auto *data = reinterpret_cast<const uint8_t *>(chunk.data()); - size_t len = chunk.size(); - if (!format_selected_) { - size_t take = std::min(len, kSnapHeader - buf_.size()); - buf_.append(reinterpret_cast<const char *>(data), take); - data += take; - len -= take; - if (buf_.size() < kSnapHeader) { - return Status::Ok(); - } - const auto *header = reinterpret_cast<const uint8_t *>(buf_.data()); - if (get_u32(header) != kSnapMagic) { - return Status::corruption("snapshot: bad magic"); - } - if (get_u32(header + 4) != kSnapVersion) { - return Status::not_supported("snapshot: version"); - } - auto fmt = static_cast<snapshot_format>(header[8]); - format_selected_ = true; - if (fmt == snapshot_format::kNative) { - native_ = true; - } - else if (fmt == snapshot_format::kPortable) { - portable_ = std::make_unique<PortableState>(); - Status status = portable_->begin(header); - if (!status.ok()) { - return status; - } - buf_.clear(); - } - else { - return Status::not_supported("snapshot: unknown format"); - } - } - if (native_) { - buf_.append(reinterpret_cast<const char *>(data), len); - return Status::Ok(); - } - return portable_->feed(data, len); -} - -Status SnapshotImport::finish_native(const uint8_t *p, size_t len, uint64_t *out_at_slot) -{ - if (len < kNativeHeader + kSnapTrailer) { - return Status::invalid_argument("snapshot: native stream too short"); - } - // Verify the whole-stream CRC over everything but the trailing 4 bytes. - uint32_t want_crc = get_u32(p + (len - kSnapTrailer)); - if (crowdb::common::crc32c(p, len - kSnapTrailer) != want_crc) { - return Status::corruption("snapshot: CRC mismatch"); - } - - uint64_t at_slot = get_u64(p + 9); - uint64_t root_page_id = get_u64(p + 17); - uint64_t count = get_u64(p + 25); - - std::vector<NativeFrame> frames; - frames.reserve(count); - size_t pos = kNativeHeader; - const size_t body_end = len - kSnapTrailer; - for (uint64_t i = 0; i < count; ++i) { - if (pos + 8 + 4 > body_end) { - return Status::corruption("snapshot: truncated native frame header"); - } - uint64_t page_id = get_u64(p + pos); - pos += 8; - uint32_t flen = get_u32(p + pos); - pos += 4; - if (pos + flen > body_end) { - return Status::corruption("snapshot: truncated native frame body"); + if (!runtime_) { + auto status = RuntimeSnapshotImport::create(*TreeCoreAccess::get(tree_), &runtime_); + if (!status.ok()) { + return status; } - frames.push_back(NativeFrame{.page_id = page_id, .frame = std::vector<uint8_t>(p + pos, p + pos + flen)}); - pos += flen; - } - if (pos != body_end) { - return Status::corruption("snapshot: trailing bytes"); - } - - Status is = tree_.install_snapshot_native(std::move(frames), root_page_id, at_slot); - if (!is.ok()) { - return is; - } - if (out_at_slot != nullptr) { - *out_at_slot = at_slot; } - return Status::Ok(); + return runtime_->feed(chunk); } Status SnapshotImport::finish(uint64_t *out_at_slot) { - if (portable_ != nullptr) { - return portable_->finish(tree_, out_at_slot); - } - const size_t len = buf_.size(); - if (len < kSnapHeader + kSnapTrailer) { - return Status::invalid_argument("snapshot: stream too short"); - } - const auto *p = reinterpret_cast<const uint8_t *>(buf_.data()); - - if (get_u32(p) != kSnapMagic) { - return Status::corruption("snapshot: bad magic"); - } - if (get_u32(p + 4) != kSnapVersion) { - return Status::not_supported("snapshot: version"); - } - auto fmt = static_cast<snapshot_format>(p[8]); - if (fmt == snapshot_format::kNative) { - return finish_native(p, len, out_at_slot); + if (!runtime_) { + return Status::invalid_argument("snapshot: stream truncated"); } - return Status::not_supported("snapshot: unknown format"); + return runtime_->finish(out_at_slot); } Status snapshot_load_from_file(Crowdbtree &tree, const std::string &path) diff --git a/lib/crowdb-tree/src/snapshot/snapshot_wire.h b/lib/crowdb-tree/src/snapshot/snapshot_wire.h new file mode 100644 index 000000000..bf1132527 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/snapshot_wire.h @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-common/crc32c.h" +#include "snapshot/published_snapshot_entries.h" + +#include <algorithm> +#include <array> +#include <span> + +namespace crowdb::tree::snapshot_detail +{ +inline void put_u32(uint8_t *out, uint32_t value) +{ + for (unsigned i = 0; i != 4; ++i) { + out[i] = static_cast<uint8_t>(value >> (8 * i)); + } +} + +inline void put_u64(uint8_t *out, uint64_t value) +{ + for (unsigned i = 0; i != 8; ++i) { + out[i] = static_cast<uint8_t>(value >> (8 * i)); + } +} + +inline std::span<const uint8_t> bytes(Slice value) +{ + return {reinterpret_cast<const uint8_t *>(value.data()), value.size()}; +} + +inline uint32_t crc_bytes(uint32_t crc, std::span<const uint8_t> bytes) +{ + // The shared SIMD CRC API takes an int length. Keep every call bounded + // even when a configured logical-value limit exceeds that parameter. + while (!bytes.empty()) { + const size_t take = std::min(bytes.size(), size_t{65536}); + crc = crowdb::common::crc32c_update(crc, bytes.data(), take); + bytes = bytes.subspan(take); + } + return crc; +} + +inline Status entry_fields(const PublishedScanEntry &entry, std::array<uint8_t, 17> *fields) +{ + if (entry.key.size() > UINT32_MAX || entry.cell.value.size() > UINT32_MAX) { + return Status::resource_exhausted("portable tuple exceeds wire length fields"); + } + put_u32(fields->data(), static_cast<uint32_t>(entry.key.size())); + put_u64(fields->data() + 4, entry.cell.slot); + (*fields)[12] = entry.cell.tombstone() ? 1 : 0; + put_u32(fields->data() + 13, static_cast<uint32_t>(entry.cell.value.size())); + return Status::Ok(); +} +} // namespace crowdb::tree::snapshot_detail diff --git a/lib/crowdb-tree/src/snapshot/stored_overflow.cpp b/lib/crowdb-tree/src/snapshot/stored_overflow.cpp new file mode 100644 index 000000000..d3d8b6fa1 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/stored_overflow.cpp @@ -0,0 +1,45 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/stored_overflow.h" + +namespace crowdb::tree +{ +Status capture_stored_overflow(const LeafFrameView &leaf, size_t max_references, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const StoredOverflowReferences> *out) +{ + const size_t count = static_cast<size_t>(leaf.count()) + leaf.delta_count(); + auto cell_at = [&leaf](size_t index) { + return index < leaf.count() ? leaf.cell(static_cast<uint32_t>(index)) + : leaf.delta_cell(static_cast<uint32_t>(index - leaf.count())); + }; + size_t references = 0; + for (size_t i = 0; i < count; ++i) { + const Slice raw = cell_at(i); + const CellView cell(raw); + if (!cell.valid()) { + return Status::corruption(); + } + if (cell.is_overflow()) { + if (raw.size() != kOverflowCellSize || cell.overflow_head() == kInvalidPageId || cell.overflow_len() == 0) { + return Status::corruption(); + } + if (references == max_references) { + return Status::resource_exhausted(); + } + ++references; + } + } + auto result = detail::reserved_shared<StoredOverflowReferences>( + metadata, detail::ReservationAllocator<StoredOverflowReference>(metadata)); + result->reserve(references); + for (size_t i = 0; i < count; ++i) { + const CellView cell(cell_at(i)); + if (cell.is_overflow()) { + result->push_back({.head = cell.overflow_head(), .bytes = cell.overflow_len()}); + } + } + *out = std::move(result); + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/stored_overflow.h b/lib/crowdb-tree/src/snapshot/stored_overflow.h new file mode 100644 index 000000000..5d0209094 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/stored_overflow.h @@ -0,0 +1,12 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/captured_write.h" + +namespace crowdb::tree +{ +Status capture_stored_overflow(const LeafFrameView &leaf, size_t max_references, + const std::shared_ptr<detail::AllocationReservation> &metadata, + std::shared_ptr<const StoredOverflowReferences> *out); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/write.cpp b/lib/crowdb-tree/src/snapshot/write.cpp deleted file mode 100644 index f0502ab6a..000000000 --- a/lib/crowdb-tree/src/snapshot/write.cpp +++ /dev/null @@ -1,365 +0,0 @@ -// Copyright 2026-present Gian <crow.db@outlook.com> -// Licensed under the Apache License, Version 2.0. - -#include "async_completion_adapter.h" -#include "crowdb-common/log.h" -#include "crowdb-tree/backend/async_page_store.h" -#include "crowdb-tree/crowdb-tree.h" - -#include <chrono> -#include <functional> -#include <memory> -#include <mutex> -#include <set> -#include <utility> -#include <vector> - -namespace crowdb::tree -{ -Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_seq) -{ - if (opt_.page_store == nullptr) { - return Status::invalid_argument("snapshot: no page_store"); - } - auto snap_t0 = std::chrono::steady_clock::now(); - acquire_snapshot_slot(); - Status pending = retry_snapshot_anchor(); - if (!pending.ok()) { - release_snapshot_slot(); - return pending; - } - PreparedSnapshot prepared; - Status ps; - { - auto apply_t0 = std::chrono::steady_clock::now(); - std::scoped_lock lk(write_mutex_); - ps = prepare_snapshot_locked(&prepared); - if (metrics_.snapshot_apply_l != nullptr) { - auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - apply_t0) - .count(); - metrics_.snapshot_apply_l->observe(static_cast<uint64_t>(ns)); - } - } - if (!ps.ok()) { - CRB_LOG_ERROR("[{}] snapshot: prepare_snapshot_locked failed: {}", name_, ps.to_string()); - release_snapshot_slot(); - return ps; - } - for (auto &w : prepared.page_writes) { - w.blob.resize(round_up_to_iu(w.logical_len, opt_.page_store->iu_size()), 0); - auto write_t0 = std::chrono::steady_clock::now(); - Status s = opt_.page_store->write_at(w.addr, w.blob.data(), w.blob.size()); - if (metrics_.snapshot_page_write_l != nullptr) { - auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - write_t0) - .count(); - metrics_.snapshot_page_write_l->observe(static_cast<uint64_t>(ns)); - } - if (metrics_.snapshot_page_write_bw != nullptr) { - metrics_.snapshot_page_write_bw->observe(w.blob.size()); - } - if (!s.ok()) { - CRB_LOG_ERROR("[{}] snapshot page write failed: addr={} status={}", name_, w.addr, s.to_string()); - release_snapshot_slot(); - return s; - } - std::vector<uint8_t>().swap(w.blob); - } - for (auto &sw : prepared.segment_writes) { - Status s = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size()); - if (!s.ok()) { - CRB_LOG_ERROR("[{}] snapshot segment write failed: addr={} status={}", name_, sw.addr, s.to_string()); - release_snapshot_slot(); - return s; - } - } - Status dw = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.directory_write.addr, - prepared.directory_write.blob.data(), prepared.directory_write.blob.size()); - if (!dw.ok()) { - CRB_LOG_ERROR("[{}] snapshot directory write failed: addr={} status={}", name_, prepared.directory_write.addr, - dw.to_string()); - release_snapshot_slot(); - return dw; - } - // Barrier: pages + segment images + directory durable before the anchor - // that references them. - auto fsync_t0 = std::chrono::steady_clock::now(); - Status sync1 = opt_.page_store->sync(); - if (metrics_.fsync_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - fsync_t0).count(); - metrics_.fsync_l->observe(static_cast<uint64_t>(ns)); - } - if (!sync1.ok()) { - CRB_LOG_ERROR("[{}] snapshot data sync failed: {}", name_, sync1.to_string()); - release_snapshot_slot(); - return sync1; - } - Status aw = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.anchor_write.addr, - prepared.anchor_write.blob.data(), prepared.anchor_write.blob.size()); - if (!aw.ok()) { - CRB_LOG_ERROR("[{}] snapshot anchor write failed: addr={} status={}", name_, prepared.anchor_write.addr, - aw.to_string()); - release_snapshot_slot(); - return aw; - } - auto fsync2_t0 = std::chrono::steady_clock::now(); - snapshot_anchor_pending_ = true; - Status sync2 = opt_.page_store->sync(); - if (metrics_.fsync_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - fsync2_t0).count(); - metrics_.fsync_l->observe(static_cast<uint64_t>(ns)); - } - if (!sync2.ok()) { - CRB_LOG_ERROR("[{}] snapshot anchor sync failed: {}", name_, sync2.to_string()); - commit_prepared_snapshot(prepared); - release_snapshot_slot(); - return sync2; - } - snapshot_anchor_pending_ = false; - - // Observe metadata write bytes: segments + directory + anchor. - if (metrics_.snapshot_meta_write_bw != nullptr) { - uint64_t meta_bytes = 0; - for (const auto &sw : prepared.segment_writes) { - meta_bytes += sw.blob.size(); - } - meta_bytes += prepared.directory_write.blob.size(); - meta_bytes += prepared.anchor_write.blob.size(); - metrics_.snapshot_meta_write_bw->observe(meta_bytes); - } - - finalize_prepared_snapshot(prepared); - - // Publish the exact logical summary only after the root anchor is durable. - if (PageBase *root = resident(root_page_id_.load(std::memory_order_acquire)); root != nullptr) { - PageSummary summary; - if (root->type == page_type::kLeafBase) { - summary = static_cast<LeafBase *>(root)->summary(); - } - else if (root->type == page_type::kInnerBase) { - summary = static_cast<InnerBase *>(root)->summary(); - } - if (summary.exact) { - summary_live_kv_.store(summary.live_kv, std::memory_order_release); - summary_live_key_bytes_.store(summary.live_key_bytes, std::memory_order_release); - summary_live_value_bytes_.store(summary.live_value_bytes, std::memory_order_release); - summary_live_logical_bytes_.store(summary.live_logical_bytes, std::memory_order_release); - summary_reachable_overflow_pages_.store(summary.reachable_overflow_pages, std::memory_order_release); - summary_reachable_page_capacity_bytes_.store(summary.reachable_page_capacity_bytes, - std::memory_order_release); - summary_covered_slot_.store(prepared.last_applied_slot, std::memory_order_release); - summary_root_version_.store(prepared.seq, std::memory_order_release); - summary_available_.store(true, std::memory_order_release); - } - } - - release_snapshot_slot(); - if (out_last_applied != nullptr) { - *out_last_applied = prepared.last_applied_slot; - } - if (out_snapshot_seq != nullptr) { - *out_snapshot_seq = prepared.seq; - } - if (metrics_.snapshot_l != nullptr) { - auto ns = - std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - snap_t0).count(); - metrics_.snapshot_l->observe(static_cast<uint64_t>(ns)); - } - return Status::Ok(); -} - -void Crowdbtree::snapshot_async( - std::function<void(Status, uint64_t)> on_done) // NOLINT(performance-unnecessary-value-param) -{ - if (opt_.page_store == nullptr) { - on_done(Status::invalid_argument("snapshot: no page_store"), 0); - return; - } - if (opt_.async_page_store != nullptr) { - acquire_snapshot_slot(); - snapshot_prepare_async(std::move(on_done)); - return; - } - // No async backend wired -- run the synchronous path in - // this stack frame; still correct, just not genuinely async. - uint64_t last_applied = 0; - Status st = snapshot(&last_applied); - on_done(st, last_applied); -} - -void Crowdbtree::snapshot_write_next_async( // NOLINT(readability-convert-member-functions-to-static) - std::shared_ptr<PreparedSnapshot> prepared, // NOLINT(performance-unnecessary-value-param) - size_t idx, std::function<void(Status, uint64_t)> on_done) // NOLINT(performance-unnecessary-value-param) -{ - // snapshot_inflight_ (acquired by snapshot_async() before - // prepare_snapshot_locked()) stays held across this entire async chain - // -- see snapshot_async's doc comment on crowdb-tree.h for why an atomic - // spin-gate is used here instead of write_mutex_ (which cannot be - // unlocked from a different thread than the one that locked it, and - // this chain's completions run on the Reactor thread). `prepared` is a - // shared_ptr so each hop's lambda can carry it to the next hop after - // this call's own stack frame returns. - if (idx < prepared->page_writes.size()) { - PreparedPageWrite &w = prepared->page_writes[idx]; - w.blob.resize(round_up_to_iu(w.logical_len, opt_.page_store->iu_size()), 0); - opt_.async_page_store->submit_write( - w.addr, w.blob.data(), w.blob.size(), - detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - std::vector<uint8_t>().swap(prepared->page_writes[idx].blob); - snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); - })); - return; - } - size_t seg_idx_in_list = idx - prepared->page_writes.size(); - if (seg_idx_in_list < prepared->segment_writes.size()) { - const PreparedSegmentWrite &sw = prepared->segment_writes[seg_idx_in_list]; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size(), - detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); - })); - return; - } - - const PreparedSnapshotWrite &dw = prepared->directory_write; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, dw.addr, dw.blob.data(), dw.blob.size(), - detail::own_async_completion([this, prepared, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - // Barrier: pages + segment images + directory durable before the anchor - // that references them. - Status fs1 = opt_.async_page_store->submit_fsync( - detail::own_async_completion([this, prepared, on_done](const Status &st2) mutable { - if (!st2.ok()) { - release_snapshot_slot(); - on_done(st2, 0); - return; - } - const PreparedSnapshotWrite &aw = prepared->anchor_write; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, aw.addr, aw.blob.data(), aw.blob.size(), - detail::own_async_completion([this, prepared, on_done](const Status &st3) mutable { - if (!st3.ok()) { - release_snapshot_slot(); - on_done(st3, 0); - return; - } - snapshot_anchor_pending_ = true; - Status fs2 = opt_.async_page_store->submit_fsync( - detail::own_async_completion([this, prepared, on_done](const Status &st4) mutable { - if (!st4.ok()) { - CRB_LOG_ERROR("[{}] snapshot anchor sync failed: {}", name_, st4.to_string()); - commit_prepared_snapshot(*prepared); - release_snapshot_slot(); - on_done(st4, 0); - return; - } - snapshot_anchor_pending_ = false; - finalize_prepared_snapshot(*prepared); - uint64_t last_applied = prepared->last_applied_slot; - release_snapshot_slot(); - on_done(Status::Ok(), last_applied); - })); - if (!fs2.ok()) { - commit_prepared_snapshot(*prepared); - release_snapshot_slot(); - on_done(fs2, 0); - } - })); - })); - if (!fs1.ok()) { - release_snapshot_slot(); - on_done(fs1, 0); - } - })); -} - -Status Crowdbtree::persist_compaction_snapshot(std::vector<PrefetchedPage> prefetched, - std::set<uint32_t> selected_blocks, PreparedSnapshot *prepared) -{ - acquire_snapshot_slot(); - Status pending = retry_snapshot_anchor(); - if (!pending.ok()) { - release_snapshot_slot(); - return pending; - } - Status prepare; - { - std::scoped_lock lk(write_mutex_); - prepare = prepare_snapshot_locked(prepared, std::move(prefetched), std::move(selected_blocks)); - } - if (!prepare.ok()) { - CRB_LOG_ERROR("[{}] compact_sparse_blocks: prepare failed: {}", name_, prepare.to_string()); - release_snapshot_slot(); - return prepare; - } - - for (auto &write : prepared->page_writes) { - write.blob.resize(round_up_to_iu(write.logical_len, opt_.page_store->iu_size()), 0); - Status status = opt_.page_store->write_at(write.addr, write.blob.data(), write.blob.size()); - if (!status.ok()) { - release_snapshot_slot(); - return status; - } - std::vector<uint8_t>().swap(write.blob); - } - for (auto &write : prepared->segment_writes) { - Status status = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, write.addr, write.blob.data(), write.blob.size()); - if (!status.ok()) { - release_snapshot_slot(); - return status; - } - } - Status directory = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->directory_write.addr, - prepared->directory_write.blob.data(), prepared->directory_write.blob.size()); - if (!directory.ok()) { - release_snapshot_slot(); - return directory; - } - Status durable_contents = opt_.page_store->sync(); - if (!durable_contents.ok()) { - release_snapshot_slot(); - return durable_contents; - } - Status anchor = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, - prepared->anchor_write.blob.data(), prepared->anchor_write.blob.size()); - if (!anchor.ok()) { - release_snapshot_slot(); - return anchor; - } - Status committed = opt_.page_store->sync(); - if (!committed.ok()) { - std::vector<uint8_t> invalid_anchor(prepared->anchor_write.blob.size(), 0); - Status rollback = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, - invalid_anchor.data(), invalid_anchor.size()); - if (rollback.ok()) { - (void)opt_.page_store->sync(); - } - release_snapshot_slot(); - return committed; - } - finalize_prepared_snapshot(*prepared); - release_snapshot_slot(); - return Status::Ok(); -} - -} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/writeback_selection.cpp b/lib/crowdb-tree/src/snapshot/writeback_selection.cpp new file mode 100644 index 000000000..7206418e3 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/writeback_selection.cpp @@ -0,0 +1,165 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "writeback_selection.h" + +#include "snapshot/primary_backing.h" + +namespace crowdb::tree +{ +namespace +{ +bool resident_child_fences(const PublishedPage &page, uint64_t pid, InnerEncodeFences *out) +{ + auto backing = page.inner_backing; + if (page.level == 0) { + if (!page.leaf || !page.leaf->overlays.empty() || page.leaf->bases.size() != 1) { + return false; + } + const auto &base = page.leaf->bases.front(); + if (base.range.lower != page.leaf->range.lower || base.range.upper != page.leaf->range.upper) { + return false; + } + backing = base.backing; + } + if (!backing || backing->storage != BackingStorage::kResidentFrame || !backing->readable() || + backing->resident_frame.size() < kFrameHeaderSize + kFrameTrailerSize) { + return false; + } + // Import verified the entire graph before publishing this immutable raw + // frame. Header access borrows its existing owner and performs no I/O. + const auto *frame = backing->resident_frame.data(); + if (frame_u64(frame, fh::kSelfpage_id) != pid) { + return false; + } + if (page.level == 0 && frame_page_type(frame) == page_type::kLeafBase) { + const bool nonempty = frame_u32(frame, fh::kSlotCount) != 0 || frame_u32(frame, fh::kDeltaCount) != 0; + *out = nonempty ? InnerEncodeFences{.lower_leaf = pid, .upper_leaf = pid} : InnerEncodeFences{}; + return true; + } + if (page.level == 0 || frame_page_type(frame) != page_type::kInnerBase || !frame_fences_are_page_ids(frame)) { + return false; + } + *out = {.lower_leaf = frame_lower_fence_page_id(frame), .upper_leaf = frame_upper_fence_page_id(frame)}; + return true; +} + +bool child_fences(const PublishedPage &page, uint64_t pid, InnerEncodeFences *out) +{ + if (page.metrics.dirty) { + return false; + } + if (!page.primary || !page.primary->frame.backing) { + return resident_child_fences(page, pid, out); + } + const auto &proof = *page.primary; + const auto &frame = proof.frame; + if (!frame.summary) { + return resident_child_fences(page, pid, out); + } + const auto &summary = *frame.summary; + if (page.level == 0) { + if (!page.leaf || !can_reuse_primary(page, pid, page.leaf->range, proof.gc_floor, frame.backing->backend_id, + frame.frame_bytes)) { + return false; + } + *out = summary.encoded_entries != 0 ? InnerEncodeFences{.lower_leaf = pid, .upper_leaf = pid} + : InnerEncodeFences{}; + return true; + } + if (!summary.inner_fences || frame.page_id != pid || frame.backing != page.inner_backing || + !frame.backing->reloadable()) { + return false; + } + const auto fences = *summary.inner_fences; + if (page.inner && !can_reuse_inner_primary(page, pid, fences, frame.backing->backend_id, frame.frame_bytes)) { + return false; + } + *out = fences; + return true; +} + +Status check_inner(const PublishedPage &page, size_t max_children) +{ + if (page.level == 0 || page.leaf || !page.inner || page.inner->children.empty() || + page.inner->children.size() != page.inner->separators.size() + 1) { + return Status::corruption("writeback candidate has no resident routing descriptor"); + } + if (page.inner->children.size() > max_children) { + return Status::resource_exhausted("writeback child window exceeds its page bound"); + } + return Status::Ok(); +} + +Status missing_child(const PublishedTreeState &view, const PublishedPage &page, size_t max_children, + std::optional<uint64_t> *out) +{ + auto status = check_inner(page, max_children); + if (!status.ok()) { + return status; + } + for (const auto pid : page.inner->children) { + const auto child = view.mapping.get(pid); + if (!child || child->level + uint64_t{1} != page.level) { + return Status::corruption("writeback child is absent or has the wrong captured level"); + } + InnerEncodeFences fences; + if (!child_fences(*child, pid, &fences)) { + *out = pid; + return Status::Ok(); + } + } + out->reset(); + return Status::Ok(); +} +} // namespace + +Status select_writeback_page(const PublishedTreeState &view, const CapturedFrameLimits &limits, + std::optional<uint64_t> *out) +{ + *out = view.mapping.oldest_writeback_page(); + for (size_t depth = 0; out->has_value() && depth < limits.max_depth; ++depth) { + const auto page = view.mapping.get(out->value()); + if (!page) { + return Status::corruption("writeback candidate is absent from captured mapping"); + } + if (page->level == 0) { + return Status::Ok(); + } + std::optional<uint64_t> child; + auto status = missing_child(view, *page, limits.encode.max_entries, &child); + if (!status.ok() || !child) { + return status; + } + *out = child; + } + return out->has_value() ? Status::resource_exhausted("writeback dependency route exceeds its depth bound") + : Status::Ok(); +} + +Status writeback_inner_fences(const PublishedTreeState &view, const PublishedPage &page, size_t max_children, + InnerEncodeFences *out) +{ + auto status = check_inner(page, max_children); + if (!status.ok()) { + return status; + } + *out = {}; + for (const auto pid : page.inner->children) { + const auto child = view.mapping.get(pid); + if (!child || child->level + uint64_t{1} != page.level) { + return Status::corruption("writeback child is absent or has the wrong captured level"); + } + InnerEncodeFences fences; + if (!child_fences(*child, pid, &fences)) { + return Status::unavailable("writeback inner needs its captured child receipts"); + } + if (fences.lower_leaf != kInvalidPageId) { + if (out->lower_leaf == kInvalidPageId) { + out->lower_leaf = fences.lower_leaf; + } + out->upper_leaf = fences.upper_leaf; + } + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/writeback_selection.h b/lib/crowdb-tree/src/snapshot/writeback_selection.h new file mode 100644 index 000000000..32710d702 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/writeback_selection.h @@ -0,0 +1,15 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "snapshot/captured_frames.h" + +namespace crowdb::tree +{ +// Select from this exact immutable view. Dirty leaves are materialized first; +// an inner's missing child receipt is completed before that inner is selected. +Status select_writeback_page(const PublishedTreeState &view, const CapturedFrameLimits &limits, + std::optional<uint64_t> *out); +Status writeback_inner_fences(const PublishedTreeState &view, const PublishedPage &page, size_t max_children, + InnerEncodeFences *out); +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/tree_backing_stats_c_api.cpp b/lib/crowdb-tree/src/tree_backing_stats_c_api.cpp new file mode 100644 index 000000000..ce26b541b --- /dev/null +++ b/lib/crowdb-tree/src/tree_backing_stats_c_api.cpp @@ -0,0 +1,28 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "c_api_internal.h" +#include "crowdb-tree/backing_stats_c_api.h" +#include "crowdb-tree/btree/tree.h" +#include "tree_backing_stats_c_api_internal.h" + +int32_t crowdb::tree::copy_backing_stats(BackingStatsSnapshot value, ct_backing_stats_v1 *out) noexcept +{ + if (out == nullptr || out->version != 1 || out->struct_size != sizeof(*out)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + if (value.scope_id == 0) { + return static_cast<int32_t>(Code::kUnavailable); + } + *out = {.struct_size = sizeof(*out), + .version = 1, + .scope_id = value.scope_id, + .allocated_bytes = value.allocated_bytes}; + return 0; +} + +int32_t ct_get_backing_stats(ct_tree *tree, ct_backing_stats_v1 *out) +{ + using namespace crowdb::tree; + return tree == nullptr ? static_cast<int32_t>(Code::kInvalidArgument) + : copy_backing_stats(c_api_tree(tree)->backing_stats(), out); +} diff --git a/lib/crowdb-tree/src/tree_backing_stats_c_api_internal.h b/lib/crowdb-tree/src/tree_backing_stats_c_api_internal.h new file mode 100644 index 000000000..b17259035 --- /dev/null +++ b/lib/crowdb-tree/src/tree_backing_stats_c_api_internal.h @@ -0,0 +1,10 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/backing_stats_c_api.h" +#include "crowdb-tree/btree/diagnostics.h" + +namespace crowdb::tree +{ +int32_t copy_backing_stats(BackingStatsSnapshot value, ct_backing_stats_v1 *out) noexcept; +} diff --git a/lib/crowdb-tree/src/tree_progress_c_api.cpp b/lib/crowdb-tree/src/tree_progress_c_api.cpp new file mode 100644 index 000000000..b50f0d4b3 --- /dev/null +++ b/lib/crowdb-tree/src/tree_progress_c_api.cpp @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "c_api_internal.h" +#include "crowdb-tree/btree/tree.h" +#include "crowdb-tree/tree_stats_c_api.h" + +using namespace crowdb::tree; + +int32_t ct_get_tree_progress_stats(ct_tree *tree, ct_tree_progress_snapshot_v1 *out) +try { + if (tree == nullptr || out == nullptr || out->struct_size != sizeof(*out) || out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + const auto snapshot = c_api_tree(tree)->tree_state_stats(); + if (snapshot.instance_generation == 0) { + return static_cast<int32_t>(Code::kUnavailable); + } + *out = {.struct_size = sizeof(*out), + .version = 1, + .instance_generation = snapshot.instance_generation, + .commit_seq = snapshot.commit_seq, + .sampled_at_ns = snapshot.sampled_at_ns, + .candidate_oldest_age_ns = snapshot.progress.candidate_oldest_age_ns, + .checkpoint_oldest_pending_age_ns = snapshot.progress.checkpoint_oldest_pending_age_ns, + .maintenance_oldest_age_ns = snapshot.progress.maintenance_oldest_age_ns}; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} diff --git a/lib/crowdb-tree/src/tree_stats_c_api.cpp b/lib/crowdb-tree/src/tree_stats_c_api.cpp new file mode 100644 index 000000000..c4e3573f2 --- /dev/null +++ b/lib/crowdb-tree/src/tree_stats_c_api.cpp @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/tree_stats_c_api.h" + +#include "c_api_internal.h" +#include "crowdb-tree/btree/tree.h" + +using namespace crowdb::tree; + +int32_t ct_get_tree_state_stats(ct_tree *tree, ct_tree_stats_snapshot_v1 *out) +try { + if (tree == nullptr || out == nullptr || out->struct_size != sizeof(*out) || out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + const auto snapshot = c_api_tree(tree)->tree_state_stats(); + if (snapshot.instance_generation == 0) { + return static_cast<int32_t>(Code::kUnavailable); + } + ct_tree_stats_snapshot_v1 result{}; + result.struct_size = sizeof(result); + result.version = 1; + result.instance_generation = snapshot.instance_generation; + result.commit_seq = snapshot.commit_seq; + result.sampled_at_ns = snapshot.sampled_at_ns; + result.logical.leaf_pages = snapshot.logical.leaf_pages; + result.logical.inner_pages = snapshot.logical.inner_pages; + result.logical.height = snapshot.logical.height; + result.logical.dirty_pages = snapshot.logical.dirty_pages; + result.logical.overlay_pages = snapshot.logical.overlay_pages; + result.logical.overlay_runs = snapshot.logical.overlay_runs; + result.logical.overlay_entries = snapshot.logical.overlay_entries; + result.logical.page_size_known_bytes = snapshot.logical.page_size_known_bytes; + result.logical.page_size_unknown_pages = snapshot.logical.page_size_unknown_pages; + result.logical.page_size_estimate_bytes = snapshot.logical.page_size_estimate_bytes; + result.logical.page_size_max_observed_bytes = snapshot.logical.page_size_max_observed_bytes; + result.logical.kv_known_count = snapshot.logical.kv_known_count; + result.logical.kv_known_bytes = snapshot.logical.kv_known_bytes; + result.logical.kv_unknown_pages = snapshot.logical.kv_unknown_pages; + result.logical.consolidate_candidate_pages = snapshot.logical.consolidate_candidate_pages; + result.logical.split_candidate_pages = snapshot.logical.split_candidate_pages; + result.logical.split_ready_pages = snapshot.logical.split_ready_pages; + result.logical.merge_candidate_pages = snapshot.logical.merge_candidate_pages; + result.logical.merge_candidate_pairs = snapshot.logical.merge_candidate_pairs; + result.logical.merge_ready_pairs = snapshot.logical.merge_ready_pairs; + result.logical.writeback_candidate_pages = snapshot.logical.writeback_candidate_pages; + result.logical.evict_candidate_pages = snapshot.logical.evict_candidate_pages; + result.logical.checkpoint_pending_pages = snapshot.logical.checkpoint_pending_pages; + result.logical.checkpoint_mapping_pending = snapshot.logical.checkpoint_mapping_pending ? 1 : 0; + result.resources.allocated_bytes = snapshot.resources.allocated_bytes; + result.resources.reserved_bytes = snapshot.resources.reserved_bytes; + result.resources.payload_bytes = snapshot.resources.payload_bytes; + result.resources.frame_bytes = snapshot.resources.frame_bytes; + result.resources.metadata_bytes = snapshot.resources.metadata_bytes; + result.resources.io_bytes = snapshot.resources.io_bytes; + result.resources.scratch_bytes = snapshot.resources.scratch_bytes; + result.resources.retired_bytes = snapshot.resources.retired_bytes; + result.resources.resident_frames = snapshot.resources.resident_frames; + result.resources.cache_bytes = snapshot.resources.cache_bytes; + result.resources.cache_pinned_bytes = snapshot.resources.cache_pinned_bytes; + result.resources.proxy_pages = snapshot.resources.proxy_pages; + result.resources.read_inflight_bytes = snapshot.resources.read_inflight_bytes; + result.resources.write_inflight_bytes = snapshot.resources.write_inflight_bytes; + result.resources.materialize_inflight_bytes = snapshot.resources.materialize_inflight_bytes; + result.resources.allocated_backing_bytes = snapshot.resources.allocated_backing_bytes; + result.resources.maintenance_queued = snapshot.resources.maintenance_queued; + result.resources.maintenance_inflight = snapshot.resources.maintenance_inflight; + *out = result; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} diff --git a/lib/crowdb-tree/src/tree_wait_metrics_c_api.cpp b/lib/crowdb-tree/src/tree_wait_metrics_c_api.cpp new file mode 100644 index 000000000..682095665 --- /dev/null +++ b/lib/crowdb-tree/src/tree_wait_metrics_c_api.cpp @@ -0,0 +1,50 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "tree_wait_metrics_c_api_internal.h" + +namespace crowdb::tree +{ +static_assert(CT_TREE_WAIT_CLASSES_V1 == kTreeWaitClasses); +static_assert(CT_TREE_WAIT_OUTCOMES_V1 == kTreeWaitOutcomes); + +int32_t copy_tree_wait_totals(const TreeWorkMetrics *metrics, ct_tree_wait_snapshot_v1 *out) noexcept +{ + if (metrics == nullptr || out == nullptr || out->struct_size != sizeof(*out) || out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + ct_tree_wait_snapshot_v1 result{}; + result.struct_size = sizeof(result); + result.version = 1; + result.sampled_at_ns = tree_metrics_now_ns(); + const auto values = metrics->waits.totals(result.sampled_at_ns); + for (size_t i = 0; i < values.size(); ++i) { + const auto &in = values[i]; + auto &value = result.classes[i]; + value.attempts = in.attempts; + value.inflight = in.inflight; + value.total_count = in.count; + value.total_sum_ns = in.sum_ns; + value.total_max_ns = in.max_ns; + for (size_t outcome = 0; outcome < in.outcomes.size(); ++outcome) { + value.outcomes[outcome] = in.outcomes[outcome]; + } + value.tracked = in.tracked; + value.untracked = in.untracked; + value.observed_oldest_age_ns = in.oldest_age_ns; + value.complete = static_cast<uint64_t>(in.complete); + } + *out = result; + return 0; +} +} // namespace crowdb::tree + +int32_t ct_tree_wait_totals(ct_tree *tree, ct_tree_wait_snapshot_v1 *out) +{ + if (tree == nullptr) { + return static_cast<int32_t>(crowdb::tree::Code::kInvalidArgument); + } + auto *core = crowdb::tree::TreeCoreAccess::get(*c_api_tree(tree)); + return crowdb::tree::copy_tree_wait_totals(core->publisher().work_metrics().get(), out); +} diff --git a/lib/crowdb-tree/src/tree_wait_metrics_c_api_internal.h b/lib/crowdb-tree/src/tree_wait_metrics_c_api_internal.h new file mode 100644 index 000000000..00a317642 --- /dev/null +++ b/lib/crowdb-tree/src/tree_wait_metrics_c_api_internal.h @@ -0,0 +1,10 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/tree_wait_metrics_c_api.h" + +namespace crowdb::tree +{ +class TreeWorkMetrics; +int32_t copy_tree_wait_totals(const TreeWorkMetrics *metrics, ct_tree_wait_snapshot_v1 *out) noexcept; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/tree_work_histograms_c_api.cpp b/lib/crowdb-tree/src/tree_work_histograms_c_api.cpp new file mode 100644 index 000000000..46d0612e6 --- /dev/null +++ b/lib/crowdb-tree/src/tree_work_histograms_c_api.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/tree_work_histograms_c_api.h" + +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "tree_work_histograms_c_api_internal.h" + +namespace crowdb::tree +{ +ct_tree_work_histograms_v1 copy_work_histograms(const TreeWorkHistograms &value, uint64_t sequence) noexcept +{ + ct_tree_work_histograms_v1 result{}; + result.struct_size = sizeof(result); + result.version = 1; + result.window_sequence = sequence; + result.sampled_at_ns = tree_metrics_now_ns(); + result.enabled_stages = value.enabled_stages; + for (size_t i = 0; i < value.stages.size(); ++i) { + const auto &stage = value.stages[i]; + result.stages[i] = {.count = stage.count, + .sum_low = stage.sum_low, + .sum_high = stage.sum_high, + .min_ns = stage.min_ns, + .max_ns = stage.max_ns, + .p50_ns = stage.p50_ns, + .p90_ns = stage.p90_ns, + .p95_ns = stage.p95_ns, + .p99_ns = stage.p99_ns, + .p999_ns = stage.p999_ns, + .zero_count = stage.zero_count, + .overflow_count = stage.overflow_count, + .total_count = stage.total_count}; + } + return result; +} +} // namespace crowdb::tree + +using namespace crowdb::tree; + +int32_t ct_work_histogram_enable_v1(ct_tree *tree, uint32_t stage) +{ + if (tree == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + return core == nullptr ? static_cast<int32_t>(Code::kUnavailable) + : static_cast<int32_t>(core->enable_work_histogram(stage).code()); +} + +int32_t ct_work_histograms_totals_v1(ct_tree *tree, ct_tree_work_histograms_v1 *out) +{ + if (tree == nullptr || out == nullptr || out->struct_size != sizeof(*out) || out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + *out = copy_work_histograms(core->publisher().work_metrics()->histogram_totals(), 0); + return 0; +} diff --git a/lib/crowdb-tree/src/tree_work_histograms_c_api_internal.h b/lib/crowdb-tree/src/tree_work_histograms_c_api_internal.h new file mode 100644 index 000000000..1c546ad0d --- /dev/null +++ b/lib/crowdb-tree/src/tree_work_histograms_c_api_internal.h @@ -0,0 +1,10 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/tree_work_histograms.h" +#include "crowdb-tree/tree_work_histograms_c_api.h" + +namespace crowdb::tree +{ +ct_tree_work_histograms_v1 copy_work_histograms(const TreeWorkHistograms &value, uint64_t sequence) noexcept; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/tree_work_metrics_c_api.cpp b/lib/crowdb-tree/src/tree_work_metrics_c_api.cpp new file mode 100644 index 000000000..6a3e899a3 --- /dev/null +++ b/lib/crowdb-tree/src/tree_work_metrics_c_api.cpp @@ -0,0 +1,316 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/tree_work_metrics_c_api.h" + +#include "btree/backing_stats_sampler.h" +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "tree_backing_stats_c_api_internal.h" +#include "tree_wait_metrics_c_api_internal.h" +#include "tree_work_histograms_c_api_internal.h" + +using namespace crowdb::tree; + +static_assert(CT_TREE_WORK_STAGES_V1 == kTreeStageCount); +static_assert(CT_TREE_WORK_OUTCOMES_V1 == kTreeOutcomeCount); +static_assert(CT_TREE_WORK_COUNTERS_V1 == kTreeWorkCounterCount); +static_assert(CT_TREE_WORK_CAUSES_V1 == kTreeWorkCauseCount); + +struct ct_work_metrics_collector +{ + std::shared_ptr<AllocationBudget> metadata; + std::shared_ptr<TreeWorkMetrics> metrics; + std::shared_ptr<TreeMetricsCollector> owner; + BackingStatsSampler backing; + std::shared_ptr<TreeResourceMetrics> resources; + std::atomic_flag capturing = ATOMIC_FLAG_INIT; + uint64_t sequence = 0; +}; + +namespace +{ +bool valid(const ct_tree_work_snapshot_v1 *out) +{ + return out != nullptr && out->struct_size == sizeof(*out) && out->version == 1; +} + +bool valid(const ct_tree_work_ages_v1 *out) +{ + return out != nullptr && out->struct_size == sizeof(*out) && out->version == 1; +} + +bool valid(const ct_tree_work_counters_v1 *out) +{ + return out != nullptr && out->struct_size == sizeof(*out) && out->version == 1; +} + +ct_tree_work_counters_v1 sample_counters(const TreeWorkMetrics &metrics) +{ + ct_tree_work_counters_v1 result{}; + result.struct_size = sizeof(result); + result.version = 1; + result.sampled_at_ns = tree_metrics_now_ns(); + const auto values = metrics.counters.snapshot(); + for (size_t t = 0; t < kTreeWorkCauseCount; ++t) { + for (size_t c = 0; c < kTreeWorkCounterCount; ++c) { + result.values[t][c] = values[t][c]; + } + } + return result; +} + +ct_tree_work_ages_v1 sample_ages(const TreeWorkMetrics &metrics) +{ + ct_tree_work_ages_v1 result{}; + result.struct_size = sizeof(result); + result.version = 1; + result.sampled_at_ns = tree_metrics_now_ns(); + const auto ages = metrics.ages(result.sampled_at_ns); + for (size_t i = 0; i < ages.size(); ++i) { + result.stages[i] = {.inflight = ages[i].inflight, + .tracked = ages[i].tracked, + .untracked = ages[i].untracked, + .observed_oldest_age_ns = ages[i].oldest_age_ns, + .complete = static_cast<uint64_t>(ages[i].complete)}; + } + return result; +} + +ct_tree_work_snapshot_v1 sample(const TreeWorkMetrics &metrics) +{ + ct_tree_work_snapshot_v1 result{}; + result.struct_size = sizeof(result); + result.version = 1; + result.sampled_at_ns = tree_metrics_now_ns(); + const auto totals = metrics.totals(); + for (size_t i = 0; i < totals.size(); ++i) { + auto &stage = result.stages[i]; + stage.attempts = totals[i].attempts; + stage.inflight = totals[i].inflight; + stage.total_count = totals[i].count; + stage.total_sum_ns = totals[i].sum_ns; + stage.total_max_ns = totals[i].max_ns; + for (size_t j = 0; j < totals[i].outcomes.size(); ++j) { + stage.outcomes[j] = totals[i].outcomes[j]; + } + } + return result; +} +} // namespace + +int32_t ct_work_metrics_collector_create(ct_tree *tree, ct_work_metrics_collector **out) +try { + if (tree == nullptr || out == nullptr) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = nullptr; + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + const auto &budget = core->binding()->lifecycle_budget(); + const auto &metrics = core->publisher().work_metrics(); + auto owner = metrics->claim_collector(budget); + if (!owner) { + return static_cast<int32_t>(Code::kUnavailable); + } + MetadataAllocator<ct_work_metrics_collector> allocator(budget); + auto *collector = allocator.allocate(1); + std::construct_at(collector); + collector->backing = core->backing_sampler(); + collector->resources = core->binding()->resource_metrics(); + collector->metadata = budget; + collector->metrics = metrics; + collector->owner = std::move(owner); + *out = collector; + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<int32_t>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<int32_t>(Code::kInternal); +} + +void ct_work_metrics_collector_free(ct_work_metrics_collector *collector) +{ + if (collector != nullptr) { + MetadataAllocator<ct_work_metrics_collector> allocator(collector->metadata); + std::destroy_at(collector); + allocator.deallocate(collector, 1); + } +} + +int32_t ct_work_metrics_capture(ct_work_metrics_collector *collector, ct_tree_work_snapshot_v1 *out) +{ + if (collector == nullptr || !valid(out)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + if (collector->capturing.test_and_set(std::memory_order_acquire)) { + return static_cast<int32_t>(Code::kUnavailable); + } + auto result = sample(*collector->metrics); + const auto window = collector->owner->capture_window(); + // Every ABI advances the same collector window, including enabled + // distributions that this older output layout cannot represent. + (void)collector->owner->capture_histograms(); + result.window_sequence = ++collector->sequence; + for (size_t i = 0; i < window.size(); ++i) { + result.stages[i].window_count = window[i].count; + result.stages[i].window_sum_ns = window[i].sum_ns; + result.stages[i].window_max_ns = window[i].max_ns; + } + collector->capturing.clear(std::memory_order_release); + *out = result; + return 0; +} + +int32_t ct_work_metrics_totals(ct_tree *tree, ct_tree_work_snapshot_v1 *out) +{ + if (tree == nullptr || !valid(out)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + *out = sample(*core->publisher().work_metrics()); + return 0; +} + +int32_t ct_flush_dispatch_totals(ct_tree *tree, ct_tree_flush_dispatch_v1 *out) +{ + if (tree == nullptr || out == nullptr || out->struct_size != sizeof(*out) || out->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + const auto value = core->publisher().work_metrics()->dispatch.snapshot(); + *out = {.struct_size = sizeof(*out), + .version = 1, + .sampled_at_ns = tree_metrics_now_ns(), + .inline_probes = value.inline_probes, + .inline_ranges = value.inline_ranges, + .queued_steps = value.queued_steps, + .prepared_runs = value.prepared_runs, + .prepared_entries = value.prepared_entries, + .prepared_bytes = value.prepared_bytes, + .cow_bytes = value.cow_bytes, + .active_steps = value.active_steps, + .peak_steps = value.peak_steps}; + return 0; +} + +int32_t ct_work_metrics_ages(ct_tree *tree, ct_tree_work_ages_v1 *out) +{ + if (tree == nullptr || !valid(out)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + *out = sample_ages(*core->publisher().work_metrics()); + return 0; +} + +int32_t ct_work_metrics_collector_ages(ct_work_metrics_collector *collector, ct_tree_work_ages_v1 *out) +{ + if (collector == nullptr || !valid(out)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = sample_ages(*collector->metrics); + return 0; +} + +int32_t ct_work_metrics_collector_backing_stats(ct_work_metrics_collector *collector, ct_backing_stats_v1 *out) +{ + return collector == nullptr ? static_cast<int32_t>(Code::kInvalidArgument) + : copy_backing_stats(collector->backing.snapshot(), out); +} + +int32_t ct_work_counters_totals(ct_tree *tree, ct_tree_work_counters_v1 *out) +{ + if (tree == nullptr || !valid(out)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + if (core == nullptr) { + return static_cast<int32_t>(Code::kUnavailable); + } + *out = sample_counters(*core->publisher().work_metrics()); + return 0; +} + +int32_t ct_work_metrics_collector_counters(ct_work_metrics_collector *collector, ct_tree_work_counters_v1 *out) +{ + if (collector == nullptr || !valid(out)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + *out = sample_counters(*collector->metrics); + return 0; +} + +static int32_t capture_all(ct_work_metrics_collector *collector, ct_tree_work_snapshot_v1 *out, + ct_tree_work_ages_v1 *ages, ct_tree_work_counters_v1 *counters, + ct_tree_work_histograms_v1 *histograms) +{ + if (collector == nullptr || !valid(out) || !valid(ages) || !valid(counters)) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + if (collector->capturing.test_and_set(std::memory_order_acquire)) { + return static_cast<int32_t>(Code::kUnavailable); + } + auto result = sample(*collector->metrics); + const auto age_result = sample_ages(*collector->metrics); + const auto counter_result = sample_counters(*collector->metrics); + const auto window = collector->owner->capture_window(); + const auto histogram_window = collector->owner->capture_histograms(); + result.window_sequence = ++collector->sequence; + for (size_t i = 0; i < window.size(); ++i) { + result.stages[i].window_count = window[i].count; + result.stages[i].window_sum_ns = window[i].sum_ns; + result.stages[i].window_max_ns = window[i].max_ns; + } + if (histograms != nullptr) { + *histograms = copy_work_histograms(histogram_window, result.window_sequence); + } + *out = result; + *ages = age_result; + *counters = counter_result; + collector->capturing.clear(std::memory_order_release); + return 0; +} + +int32_t ct_work_metrics_capture_v2(ct_work_metrics_collector *collector, ct_tree_work_snapshot_v1 *out, + ct_tree_work_ages_v1 *ages, ct_tree_work_counters_v1 *counters) +{ + return capture_all(collector, out, ages, counters, nullptr); +} + +int32_t ct_work_metrics_capture_v3(ct_work_metrics_collector *collector, ct_tree_work_snapshot_v1 *out, + ct_tree_work_ages_v1 *ages, ct_tree_work_counters_v1 *counters, + ct_tree_work_histograms_v1 *histograms) +{ + if (histograms == nullptr || histograms->struct_size != sizeof(*histograms) || histograms->version != 1) { + return static_cast<int32_t>(Code::kInvalidArgument); + } + return capture_all(collector, out, ages, counters, histograms); +} + +const char *ct_work_counter_name_v1(uint32_t index) +{ + return index < kTreeWorkCounterCount ? kTreeWorkCounterNames[index].data() : nullptr; +} + +const char *ct_work_cause_name_v1(uint32_t index) +{ + return index < kTreeWorkCauseCount ? kTreeWorkCauseNames[index].data() : nullptr; +} + +int32_t ct_work_metrics_collector_wait_totals(ct_work_metrics_collector *collector, ct_tree_wait_snapshot_v1 *out) +{ + return copy_tree_wait_totals(collector != nullptr ? collector->metrics.get() : nullptr, out); +} diff --git a/lib/crowdb-tree/tests/.clang-tidy b/lib/crowdb-tree/tests/.clang-tidy index ad1090c72..eefe00286 100644 --- a/lib/crowdb-tree/tests/.clang-tidy +++ b/lib/crowdb-tree/tests/.clang-tidy @@ -1,7 +1,13 @@ -# Inherit every repository check; only drop diagnostics whose cause lives in -# a third-party test-framework macro body instead of CROWDB code. +# Inherit repository checks, with exceptions for test-framework macros and +# deliberate regression-test behavior. Production checks remain enabled. InheritParentConfig: true # gtest's TEST()/TEST_F() expand to a static `test_info_` initialized from a # throwing std::string constructor. clang-tidy reports the warning at the # TEST() line in our file, but there is no fix available in CROWDB code. -Checks: '-bugprone-throwing-static-initialization' +# Fixed seeds keep randomized cases reproducible. Credit ownership tests also +# inspect the documented empty state of moved-from objects. +Checks: >- + -bugprone-throwing-static-initialization, + -bugprone-random-generator-seed, + -bugprone-use-after-move, + -clang-analyzer-cplusplus.Move diff --git a/lib/crowdb-tree/tests/allocation_account_test_support.h b/lib/crowdb-tree/tests/allocation_account_test_support.h new file mode 100644 index 000000000..a4c895c02 --- /dev/null +++ b/lib/crowdb-tree/tests/allocation_account_test_support.h @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/allocation_account.h" + +#include <gtest/gtest.h> + +#include <type_traits> + +namespace crowdb::tree::detail +{ +struct AllocationAccountTestAccess +{ + using Step = AllocationAccount::Step; + + template <class F> static void release_for_tests(AllocationAccount &budget, size_t bytes, F &&callback) + { + budget.release( + bytes, + [](void *context, Step step, size_t kind) noexcept { + (*static_cast<std::remove_reference_t<F> *>(context))(step, kind); + }, + &callback); + } + + template <class F> + static Status rebind_for_tests(AllocationAccount &budget, std::shared_ptr<ResourceBudget::Account> account, + F &&callback) + { + return budget.rebind( + std::move(account), + [](void *context, Step step, size_t kind) noexcept { + (*static_cast<std::remove_reference_t<F> *>(context))(step, kind); + }, + &callback); + } +}; + +namespace test_allocation_account +{ +constexpr size_t kRouting = static_cast<size_t>(BudgetClass::kRouting); +constexpr size_t kMetadata = static_cast<size_t>(AllocationKind::kMetadata); +constexpr size_t kFrame = static_cast<size_t>(AllocationKind::kFrame); +using Step = AllocationAccountTestAccess::Step; + +inline ResourceBudget::Limits limits(size_t amount) +{ + ResourceBudget::Limits result{}; + result.fill(amount); + return result; +} + +struct TestAccounts +{ + ResourceBudget host{limits(16384), 1}; + std::shared_ptr<ResourceBudget::Account> old = host.make_account(limits(8192)); + std::shared_ptr<ResourceBudget::Account> next = host.make_account(limits(8192)); + std::shared_ptr<AllocationAccount> budget; + size_t base = 0; + + TestAccounts() + { + const auto status = AllocationAccount::create(old, BudgetClass::kRouting, &budget); + EXPECT_TRUE(status.ok()) << status.to_string(); + base = old->snapshot().allocated[kRouting]; + EXPECT_GE(base, sizeof(AllocationAccount)); + EXPECT_EQ(old->snapshot().physical[kMetadata], base); + } + + void expect_only_control(const std::shared_ptr<ResourceBudget::Account> ¤t) const + { + EXPECT_EQ(host.snapshot().allocated[kRouting], base); + EXPECT_EQ(host.snapshot().reserved[kRouting], 0U); + EXPECT_EQ(current->snapshot().allocated[kRouting], base); + EXPECT_EQ(current->snapshot().physical[kMetadata], base); + EXPECT_EQ(current->snapshot().physical[kFrame], 0U); + } + + void finish() + { + budget.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(host.snapshot().reserved[kRouting], 0U); + for (const auto &account : {old, next}) { + EXPECT_EQ(account->snapshot().allocated[kRouting], 0U); + for (const auto bytes : account->snapshot().physical) { + EXPECT_EQ(bytes, 0U); + } + } + } +}; +} // namespace test_allocation_account +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/maptable/buffer_pool.cpp b/lib/crowdb-tree/tests/buffer_pool_test_support.cpp similarity index 86% rename from lib/crowdb-tree/src/maptable/buffer_pool.cpp rename to lib/crowdb-tree/tests/buffer_pool_test_support.cpp index 3fa13339d..a2fcd2866 100644 --- a/lib/crowdb-tree/src/maptable/buffer_pool.cpp +++ b/lib/crowdb-tree/tests/buffer_pool_test_support.cpp @@ -3,7 +3,7 @@ // Mapping-table page-buffer service. -#include "crowdb-tree/maptable/buffer_pool.h" +#include "buffer_pool_test_support.h" #include "crowdb-common/metrics/metrics.h" #include "crowdb-tree/maptable/frame_page.h" @@ -26,14 +26,14 @@ size_t next_pow2(size_t v) } } // namespace -// ── FrameRef ────────────────────────────────────────────────────── +// ── TestPoolFrameRef ────────────────────────────────────────────────────── -FrameRef::~FrameRef() +TestPoolFrameRef::~TestPoolFrameRef() { release(); } -FrameRef &FrameRef::operator=(FrameRef &&o) noexcept +TestPoolFrameRef &TestPoolFrameRef::operator=(TestPoolFrameRef &&o) noexcept { if (this != &o) { release(); @@ -48,7 +48,7 @@ FrameRef &FrameRef::operator=(FrameRef &&o) noexcept return *this; } -void FrameRef::release() +void TestPoolFrameRef::release() { if (pool_ != nullptr) { pool_->unpin(idx_); @@ -58,9 +58,9 @@ void FrameRef::release() } } -// ── BufferPool ──────────────────────────────────────────────────── +// ── TestBufferPool ──────────────────────────────────────────────────── -BufferPool::BufferPool(size_t capacity_bytes, uint32_t page_bytes, PageStore *store) +TestBufferPool::TestBufferPool(size_t capacity_bytes, uint32_t page_bytes, PageStore *store) : page_bytes_(page_bytes), store_(store) { @@ -80,7 +80,7 @@ BufferPool::BufferPool(size_t capacity_bytes, uint32_t page_bytes, PageStore *st ht_mask_ = ht_cap - 1; } -void BufferPool::ht_insert(uint64_t page_id, uint32_t idx) +void TestBufferPool::ht_insert(uint64_t page_id, uint32_t idx) { size_t i = page_id & ht_mask_; while (ht_key_[i] != kInvalidPageId) { @@ -90,7 +90,7 @@ void BufferPool::ht_insert(uint64_t page_id, uint32_t idx) ht_val_[i] = idx; } -int64_t BufferPool::ht_find(uint64_t page_id) const +int64_t TestBufferPool::ht_find(uint64_t page_id) const { size_t i = page_id & ht_mask_; while (ht_key_[i] != kInvalidPageId) { @@ -102,7 +102,7 @@ int64_t BufferPool::ht_find(uint64_t page_id) const return -1; } -void BufferPool::ht_erase(uint64_t page_id) +void TestBufferPool::ht_erase(uint64_t page_id) { size_t i = page_id & ht_mask_; while (ht_key_[i] != kInvalidPageId && ht_key_[i] != page_id) { @@ -129,7 +129,7 @@ void BufferPool::ht_erase(uint64_t page_id) } } -Status BufferPool::write_back_io(uint32_t idx, PageAddr addr) +Status TestBufferPool::write_back_io(uint32_t idx, PageAddr addr) { auto t0 = std::chrono::steady_clock::now(); Status s = store_->write_at(addr, frame_bytes(idx), page_bytes_); @@ -146,7 +146,7 @@ Status BufferPool::write_back_io(uint32_t idx, PageAddr addr) return Status::Ok(); } -int64_t BufferPool::acquire_victim() +int64_t TestBufferPool::acquire_victim() { // Two full sweeps: first honoring the ref bit, second forcing a clean victim. for (uint32_t scanned = 0; scanned < num_frames_ * 2; ++scanned) { @@ -168,7 +168,7 @@ int64_t BufferPool::acquire_victim() return -1; // everything pinned } -Status BufferPool::pin(uint64_t page_id, PageAddr addr, FrameRef *out) +Status TestBufferPool::pin(uint64_t page_id, PageAddr addr, TestPoolFrameRef *out) { std::unique_lock lk(mu_); for (;;) { @@ -183,14 +183,14 @@ Status BufferPool::pin(uint64_t page_id, PageAddr addr, FrameRef *out) ++m.pin; m.ref = 1; ++stats_.hits; - *out = FrameRef(this, idx, frame_bytes(idx), page_id); + *out = TestPoolFrameRef(this, idx, frame_bytes(idx), page_id); return Status::Ok(); } ++stats_.misses; int64_t v = acquire_victim(); if (v < 0) { - return Status::internal_error("BufferPool: no evictable frame (all pinned)"); + return Status::internal_error("TestBufferPool: no evictable frame (all pinned)"); } auto idx = static_cast<uint32_t>(v); FrameMeta &m = frames_[idx]; @@ -239,24 +239,24 @@ Status BufferPool::pin(uint64_t page_id, PageAddr addr, FrameRef *out) m.ref = 0; m.state = FrameMeta::State::kReady; state_cv_.notify_all(); - return rs.ok() ? Status::corruption("BufferPool: page CRC on load") : rs; + return rs.ok() ? Status::corruption("TestBufferPool: page CRC on load") : rs; } m.state = FrameMeta::State::kReady; state_cv_.notify_all(); - *out = FrameRef(this, idx, frame_bytes(idx), page_id); + *out = TestPoolFrameRef(this, idx, frame_bytes(idx), page_id); return Status::Ok(); } } -Status BufferPool::pin_new(uint64_t page_id, PageAddr addr, FrameRef *out) +Status TestBufferPool::pin_new(uint64_t page_id, PageAddr addr, TestPoolFrameRef *out) { std::unique_lock lk(mu_); if (ht_find(page_id) >= 0) { - return Status::invalid_argument("BufferPool: page_id already resident"); + return Status::invalid_argument("TestBufferPool: page_id already resident"); } int64_t v = acquire_victim(); if (v < 0) { - return Status::internal_error("BufferPool: no evictable frame (all pinned)"); + return Status::internal_error("TestBufferPool: no evictable frame (all pinned)"); } auto idx = static_cast<uint32_t>(v); FrameMeta &m = frames_[idx]; @@ -295,16 +295,16 @@ Status BufferPool::pin_new(uint64_t page_id, PageAddr addr, FrameRef *out) m.state = FrameMeta::State::kReady; ht_insert(page_id, idx); state_cv_.notify_all(); - *out = FrameRef(this, idx, frame_bytes(idx), page_id); + *out = TestPoolFrameRef(this, idx, frame_bytes(idx), page_id); return Status::Ok(); } -Status BufferPool::acquire_frame(uint32_t *out_idx, uint8_t **out_bytes) +Status TestBufferPool::acquire_frame(uint32_t *out_idx, uint8_t **out_bytes) { std::unique_lock lk(mu_); int64_t v = acquire_victim(); if (v < 0) { - return Status::internal_error("BufferPool: no evictable frame (all pinned)"); + return Status::internal_error("TestBufferPool: no evictable frame (all pinned)"); } auto idx = static_cast<uint32_t>(v); FrameMeta &m = frames_[idx]; @@ -347,7 +347,7 @@ Status BufferPool::acquire_frame(uint32_t *out_idx, uint8_t **out_bytes) return Status::Ok(); } -void BufferPool::release_frame(uint32_t idx) +void TestBufferPool::release_frame(uint32_t idx) { std::unique_lock lk(mu_); FrameMeta &m = frames_[idx]; @@ -362,7 +362,7 @@ void BufferPool::release_frame(uint32_t idx) m.dirty = false; // empty + evictable again } -void BufferPool::unpin(uint32_t idx) +void TestBufferPool::unpin(uint32_t idx) { std::scoped_lock lk(mu_); if (frames_[idx].pin > 0) { @@ -370,7 +370,7 @@ void BufferPool::unpin(uint32_t idx) } } -void BufferPool::mark_dirty(uint64_t page_id) +void TestBufferPool::mark_dirty(uint64_t page_id) { std::unique_lock lk(mu_); for (;;) { @@ -387,7 +387,7 @@ void BufferPool::mark_dirty(uint64_t page_id) } } -Status BufferPool::flush_dirty() +Status TestBufferPool::flush_dirty() { for (uint32_t idx = 0; idx < num_frames_; ++idx) { std::unique_lock lk(mu_); @@ -416,7 +416,7 @@ Status BufferPool::flush_dirty() return Status::Ok(); } -BufferPool::Stats BufferPool::stats() const +TestBufferPool::Stats TestBufferPool::stats() const { std::scoped_lock lk(mu_); Stats s = stats_; diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h b/lib/crowdb-tree/tests/buffer_pool_test_support.h similarity index 69% rename from lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h rename to lib/crowdb-tree/tests/buffer_pool_test_support.h index 47fbd979a..b9a125a97 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h +++ b/lib/crowdb-tree/tests/buffer_pool_test_support.h @@ -1,19 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// Buffer pool: crowdb-tree's bounded, explicitly-managed cache of page frames. -// The pool is the only holder of base-page -// memory: a single contiguous arena of equal-size frames plus an -// open-addressing page_id->frame table (no std::unordered_map on the hot path). -// Frames are pinned while in use (never evicted), evicted by a CLOCK sweep, and -// dirty frames are written back to the PageStore before reuse. -// -// v1 is mutex-guarded for pool operations (pin/evict/insert); reading a pinned -// frame's bytes needs no lock since a pinned frame cannot move or be evicted. -// Lock-free hot-path reads land with the core migration (PT6c). -// -// Key work: frame arena, open-addressing page table, pin/unpin (RAII FrameRef), -// CLOCK eviction, dirty write-back, stats. +// Legacy synchronous pool model used only by low-level regression tests. +// The production Tree uses SharedPageCache and independently owned frames. #pragma once #include "crowdb-common/metrics/metrics.h" @@ -27,13 +16,14 @@ #include <mutex> #include <vector> -// Forward-declare metric types for BufferPool handles. The metrics core -// moved to crowdb-common::metrics (R12); bridge the moved types into `crowdb-tree` -// with per-type using-declarations so existing `Counter*`/`Gauge*` -// references compile unchanged. (Not a `namespace crowdb::tree = -// crowdb::common::metrics;` alias — only the specific types are bridged.) The -// forward declarations live at the root namespace so they resolve to -// `crowdb::common::metrics::Counter` rather than nesting under `crowdb::tree::`. +// Forward-declare metric types for TestBufferPool handles. The metrics core +// moved to crowdb-common::metrics (R12); bridge the moved types into +// `crowdb-tree` with per-type using-declarations so existing +// `Counter*`/`Gauge*` references compile unchanged. (Not a `namespace +// crowdb::tree = crowdb::common::metrics;` alias — only the specific types are +// bridged.) The forward declarations live at the root namespace so they resolve +// to `crowdb::common::metrics::Counter` rather than nesting under +// `crowdb::tree::`. namespace crowdb::common::metrics { class Counter; @@ -49,16 +39,16 @@ using crowdb::common::metrics::Counter; using crowdb::common::metrics::Gauge; using crowdb::common::metrics::LatencySummary; -class BufferPool; +class TestBufferPool; // RAII pin handle. Keeps a frame resident until destroyed/released. Movable, // non-copyable. bytes() is valid for the handle's lifetime. -class FrameRef +class TestPoolFrameRef { public: - FrameRef() = default; + TestPoolFrameRef() = default; - FrameRef(BufferPool *pool, uint32_t idx, uint8_t *bytes, uint64_t page_id) + TestPoolFrameRef(TestBufferPool *pool, uint32_t idx, uint8_t *bytes, uint64_t page_id) : pool_(pool), idx_(idx), bytes_(bytes), @@ -66,17 +56,17 @@ class FrameRef { } - ~FrameRef(); + ~TestPoolFrameRef(); - FrameRef(const FrameRef &) = delete; - FrameRef &operator=(const FrameRef &) = delete; + TestPoolFrameRef(const TestPoolFrameRef &) = delete; + TestPoolFrameRef &operator=(const TestPoolFrameRef &) = delete; - FrameRef(FrameRef &&o) noexcept + TestPoolFrameRef(TestPoolFrameRef &&o) noexcept { *this = std::move(o); } - FrameRef &operator=(FrameRef &&o) noexcept; + TestPoolFrameRef &operator=(TestPoolFrameRef &&o) noexcept; [[nodiscard]] bool valid() const { @@ -101,13 +91,13 @@ class FrameRef void release(); private: - BufferPool *pool_ = nullptr; - uint32_t idx_ = 0; - uint8_t *bytes_ = nullptr; - uint64_t page_id_ = kInvalidPageId; + TestBufferPool *pool_ = nullptr; + uint32_t idx_ = 0; + uint8_t *bytes_ = nullptr; + uint64_t page_id_ = kInvalidPageId; }; -class BufferPool +class TestBufferPool { public: struct Stats @@ -123,20 +113,23 @@ class BufferPool }; // capacity_bytes / page_bytes frames (>= 1). `store` is non-owning. - BufferPool(size_t capacity_bytes, uint32_t page_bytes, PageStore *store); + TestBufferPool(size_t capacity_bytes, uint32_t page_bytes, PageStore *store); - // pin the frame for `page_id`, demand-loading from `addr` on a miss (CRC-checked). - Status pin(uint64_t page_id, PageAddr addr, FrameRef *out); + // pin the frame for `page_id`, demand-loading from `addr` on a miss + // (CRC-checked). + Status pin(uint64_t page_id, PageAddr addr, TestPoolFrameRef *out); // pin a fresh zeroed frame for a new page that will live at `addr`. No load. - Status pin_new(uint64_t page_id, PageAddr addr, FrameRef *out); + Status pin_new(uint64_t page_id, PageAddr addr, TestPoolFrameRef *out); // Acquire a fresh zeroed, anonymous frame for a freshly built base page (no // page_id mapping, no durable addr; dirty until a snapshot assigns one). The - // frame is pinned-resident until release_frame so it is never evicted. Returns - // an error (caller should fall back to a heap buffer) if no frame is free. - // The returned `out_bytes` window is valid until release_frame(*out_idx). + // frame is pinned-resident until release_frame so it is never evicted. + // Returns an error (caller should fall back to a heap buffer) if no frame is + // free. The returned `out_bytes` window is valid until + // release_frame(*out_idx). Status acquire_frame(uint32_t *out_idx, uint8_t **out_bytes); - // Return an owned frame (from acquire_frame, or a page_id-mapped base) to the pool. + // Return an owned frame (from acquire_frame, or a page_id-mapped base) to the + // pool. void release_frame(uint32_t idx); void mark_dirty(uint64_t page_id); @@ -163,7 +156,7 @@ class BufferPool } private: - friend class FrameRef; + friend class TestPoolFrameRef; struct FrameMeta { @@ -184,7 +177,8 @@ class BufferPool void unpin(uint32_t idx); - // open-addressing page_id->frame index table (linear probe, backward-shift erase). + // open-addressing page_id->frame index table (linear probe, backward-shift + // erase). void ht_insert(uint64_t page_id, uint32_t idx); [[nodiscard]] int64_t ht_find(uint64_t page_id) const; void ht_erase(uint64_t page_id); diff --git a/lib/crowdb-tree/tests/captured_write_test_support.h b/lib/crowdb-tree/tests/captured_write_test_support.h new file mode 100644 index 000000000..cb7ac9f1d --- /dev/null +++ b/lib/crowdb-tree/tests/captured_write_test_support.h @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/local/extent_writeback.h" +#include "extent_ledger_test_support.h" +#include "page_writeback_test_support.h" +#include "snapshot/captured_write.h" + +namespace crowdb::tree::test_write +{ +inline std::shared_ptr<PublishedTreeState> captured_leaf(std::shared_ptr<const PageState> leaf) +{ + auto result = std::make_shared<PublishedTreeState>(*view(std::move(leaf), {}, 10)); + result->next_page_id = 1; + return result; +} + +struct TestCapturedWrite +{ + LoadCredits credits; + std::shared_ptr<detail::FlushRuntime> runtime = std::make_shared<detail::FlushRuntime>(1, 8); + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<TestWriteStore> store = std::make_shared<TestWriteStore>(); + std::shared_ptr<ExtentLedger> ledger; + std::shared_ptr<CapturedWriteOperation> operation; + uint64_t next_overflow = 1; + + TestCapturedWrite() + { + EXPECT_TRUE(ExtentLedger::create({.backend_id = 9}, credits.credits.bytes, &ledger).ok()); + EXPECT_TRUE(await_ledger([&](auto done) { + return ledger->activate(credits.credits.bytes, std::move(done)); + }).status.ok()); + } + + Status prepare(std::shared_ptr<const PublishedTreeState> captured, + std::shared_ptr<const CapturedDurablePages> inherited = {}, + CapturedWriteLimits limits = {.frames = {.encode = {.frame_bytes = 512, .inline_value_bytes = 16}}, + .write = {.max_frame_bytes = 512}}) + { + next_overflow = captured->next_page_id; + return CapturedWriteOperation::create( + std::move(captured), {}, store, runtime, lane, std::move(limits), credits.credits, + [this](uint64_t count, ReservedOverflowIds *out) { + *out = {.first = next_overflow, .count = count, .owner = ledger}; + next_overflow += count; + return Status::Ok(); + }, + extent_write_reservation(ledger, credits.credits.bytes), std::move(inherited), &operation); + } + + std::vector<uint8_t> read(const DurableCapturedFrame &frame) + { + std::vector<uint8_t> blob(frame.backing->length); + std::vector<uint8_t> raw(frame.frame_bytes); + EXPECT_TRUE(store->device->read_at(frame.backing->address, blob.data(), blob.size()).ok()); + EXPECT_TRUE(decode_durable_page(blob.data(), blob.size(), raw.data(), raw.size()).ok()); + EXPECT_TRUE(frame_validate(raw.data(), raw.size())); + EXPECT_EQ(frame_u64(raw.data(), fh::kSelfpage_id), frame.page_id); + return raw; + } + + void run(Completion *done) + { + auto connected = stdexec::connect(operation->sender(), Receiver{.done = done}); + stdexec::start(connected); + runtime->drain(); + } +}; +} // namespace crowdb::tree::test_write diff --git a/lib/crowdb-tree/tests/checkpoint_structure_test_support.h b/lib/crowdb-tree/tests/checkpoint_structure_test_support.h new file mode 100644 index 000000000..f76aa31aa --- /dev/null +++ b/lib/crowdb-tree/tests/checkpoint_structure_test_support.h @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/allocation_bound.h" +#include "btree/apply_dispatcher.h" +#include "btree/page_split_operation.h" +#include "published_read_test_support.h" + +#include <future> + +namespace crowdb::tree::test_structure +{ +struct TestCheckpointStructure +{ + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + TestCheckpointStructure *owner; + + void finish(Status status) const noexcept + { + owner->terminal = std::move(status); + owner->done.store(true, std::memory_order_release); + } + + void set_value() const noexcept + { + finish(Status::Ok()); + } + + void set_error(Status status) const noexcept + { + finish(std::move(status)); + } + + void set_stopped() const noexcept + { + finish(Status::unavailable()); + } + }; + + struct Attempt + { + Attempt(detail::CallbackSender sender, TestCheckpointStructure *owner) + : connected(stdexec::connect(sender, Receiver{owner})) + { + } + + stdexec::connect_result_t<detail::CallbackSender, Receiver> connected; + }; + + std::optional<Attempt> attempt; + + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits value; + value.fill(64 * 1024 * 1024); + return value; + } + + detail::ResourceBudget budget{limits(), 8}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + PublicationCoordinator publisher; + std::shared_ptr<detail::FlushRuntime> runtime = std::make_shared<detail::FlushRuntime>(2, 16); + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(16); + std::shared_ptr<ApplyDispatcher> progress; + std::shared_ptr<test_read::TestLoader> loader = std::make_shared<test_read::TestLoader>(); + std::shared_ptr<PageSplitOperation> operation; + std::atomic<bool> done{false}; + Status terminal; + uint64_t sequence = 0; + + TestCheckpointStructure() + { + auto reserved = detail::ResourceBudget::try_reserve( + account, detail::BudgetClass::kRouting, + ApplyDispatcher::allocation_bound() + detail::shared_allocation_bound<detail::AllocationReservation>()); + EXPECT_TRUE(reserved.has_value()); + if (reserved) { + auto credit = detail::AllocationReservation::create(std::move(*reserved)); + EXPECT_TRUE(ApplyDispatcher::create(runtime, lane, 8, credit, &progress).ok()); + credit->seal(); + } + } + + ~TestCheckpointStructure() + { + try { + if (operation) { + operation->request_stop(); + } + if (!loader->pending.empty()) { + loader->finish(Status::unavailable()); + } + runtime->drain(); + operation.reset(); + if (progress) { + progress->close_and_drain(); + } + runtime->drain(); + } + catch (const std::exception &error) { + ADD_FAILURE() << error.what(); + } + catch (...) { + ADD_FAILURE() << "checkpoint fixture cleanup failed"; + } + } + + Status publish(std::shared_ptr<const PageState> logical, bool split_candidate = false) + { + const auto old = publisher.capture()->mapping.get(0); + auto replacement = std::make_shared<PublishedPage>(); + replacement->leaf = std::move(logical); + replacement->revision = old ? old->revision + 1 : 1; + replacement->metrics.dirty = true; + replacement->metrics.split_candidate = split_candidate; + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 10; + auto request = std::make_shared<PublicationRequest>(); + request->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{ + .generation = publisher.capture()->generation, .sequence = ++sequence}), + .pages = {{.page_id = 0, .expected = old, .replacement = replacement}}, + .root_page_id = 0, + .coverage = 10, + .expected_sources = publisher.capture()->sources, + .sources = std::move(sources), + .expected_next_page_id = publisher.capture()->next_page_id, + .next_page_id = 1}; + std::promise<Status> completed; + auto future = completed.get_future(); + request->on_done = [&](PublicationResult result) { completed.set_value(result.status); }; + auto status = publisher.try_submit(request) ? future.get() : Status::unavailable(); + if (!status.ok()) { + ADD_FAILURE() << status.to_string(); + } + return status; + } + + Status prepare(std::shared_ptr<const PublishedTreeState> view, size_t max_output_pages = 4, + size_t max_work_entries = 16) + { + PageMaintenanceOperationOptions options; + options.max_candidates = options.max_steps = options.max_visits = options.visits_per_step = 1; + options.publication.pages.primary_page_bytes = 512; + options.publication.pages.target_page_bytes = 256; + options.publication.pages.merge_trigger_bytes = 64; + options.publication.pages.merge_target_bytes = 128; + options.publication.pages.max_work_entries = max_work_entries; + options.publication.pages.max_output_pages = max_output_pages; + options.publication.pages.max_metadata_bytes = 1024 * 1024; + options.publication.max_inner_bytes = 512; + const auto bound = page_maintenance_reserve_bound(view->mapping, 0, options); + if (!bound) { + return Status::resource_exhausted(); + } + const auto complete = detail::AllocationBound(*bound) + .add(PageSplitOperation::control_allocation_bound()) + .add(detail::shared_allocation_bound<detail::AllocationReservation>()) + .value(); + if (!complete) { + return Status::resource_exhausted(); + } + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, *complete); + if (!credit) { + return Status::resource_exhausted(); + } + options.prepaid = detail::AllocationReservation::create(std::move(*credit)); + return PageSplitOperation::create(publisher, *runtime, lane, std::move(view), 0, std::move(options), loader, + progress, {}, TreeWorkCause::Checkpoint, &operation); + } + + void begin() + { + attempt.emplace(operation->sender(), this); + stdexec::start(attempt->connected); + runtime->drain(); + } +}; + +inline std::shared_ptr<const PageState> structural_input(bool cold = false) +{ + auto leaf = std::make_shared<PageState>(*test_read::page({ + {.key = "a", .value = std::string(80, 'a')}, + {.key = "c", .value = std::string(80, 'c')}, + {.key = "d", .value = std::string(80, 'd')}, + {.key = "e", .value = std::string(80, 'e')}, + {.key = "f", .value = std::string(80, 'f')}, + {.key = "g", .value = std::string(80, 'g')} + })); + if (cold) { + leaf->bases.push_back(test_read::cold(1)); + } + return leaf; +} + +} // namespace crowdb::tree::test_structure diff --git a/lib/crowdb-tree/tests/common/test_chunk_backing.h b/lib/crowdb-tree/tests/common/test_chunk_backing.h new file mode 100644 index 000000000..41d7ab52b --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_chunk_backing.h @@ -0,0 +1,144 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_backing.h" +#include "backend/local/scheduled_page_store.h" +#include "common/test_chunk_stage_backend.h" +#include "load_budget_test_support.h" + +namespace crowdb::tree::test_chunk +{ +class TestProtectedCatalog final : public detail::TestStageCatalog, public detail::ChunkProtectionCatalog +{ + public: + std::shared_ptr<detail::ChunkPinReleaseQueue> queue; + mutable size_t reader_checks = 0; + size_t fence_at = SIZE_MAX; + size_t unpins = 0; + bool fail_unpin = false; + + detail::ChunkProtectionCatalog *protection() noexcept override + { + return this; + } + + std::shared_ptr<detail::ChunkPinReleaseQueue> pin_release_queue() const noexcept override + { + return queue; + } + + Status check_read_owner() const override + { + return ++reader_checks >= fence_at ? Status::unavailable() : Status::Ok(); + } + + Status change_pin(uint64_t /*tree*/, detail::ChunkStageId /*stage*/, detail::ChunkStageId /*pin*/, + bool acquire) override + { + if (!acquire) { + ++unpins; + } + return !acquire && fail_unpin ? Status::unavailable() : Status::Ok(); + } + + Status writer_quiesced(uint64_t /*tree*/, uint64_t /*epoch*/) override + { + return Status::Ok(); + } + + Status load_root_bytes(uint64_t /*tree*/, uint64_t /*generation*/, std::vector<uint8_t> * /*out*/) const override + { + return Status::not_supported(); + } + + Status begin_candidate(uint64_t /*tree*/, const detail::ChunkManifestCandidate & /*unused*/) override + { + return Status::not_supported(); + } + + Status load_candidate(uint64_t /*tree*/, detail::ChunkStageId /*unused*/, + std::optional<detail::ChunkCandidateStatus> * /*out*/) const override + { + return Status::not_supported(); + } + + Status list_candidates(uint64_t /*tree*/, std::optional<detail::ChunkStageId> /*after*/, uint32_t /*limit*/, + std::vector<detail::ChunkCandidateStatus> * /*out*/) const override + { + return Status::not_supported(); + } + + Status prepare_references(uint64_t /*tree*/, detail::ChunkStageId /*unused*/, uint64_t /*window*/, + std::span<const detail::ChunkStageId> /*unused*/) override + { + return Status::not_supported(); + } + + Status finish_candidate(uint64_t /*tree*/, detail::ChunkStageId /*unused*/, uint32_t /*action*/) override + { + return Status::not_supported(); + } + + Status release_references(uint64_t /*tree*/, detail::ChunkStageId /*unused*/, uint64_t /*window*/) override + { + return Status::not_supported(); + } + + Status prune_pins(uint64_t /*tree*/, detail::ChunkStageId /*unused*/, std::optional<detail::ChunkStageId> /*after*/, + uint32_t /*limit*/, detail::ChunkPinProgress * /*out*/) override + { + return Status::not_supported(); + } +}; + +struct TestChunkBackend +{ + test_read::LoadCredits credits; + std::shared_ptr<TestProtectedCatalog> catalog = std::make_shared<TestProtectedCatalog>(); + std::shared_ptr<detail::TestFixedTransport> transport = std::make_shared<detail::TestFixedTransport>(catalog); + std::shared_ptr<ChunkBackingDomain> domain = std::make_shared<ChunkBackingDomain>(); + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<AsyncPageStore> scheduled; + MemPageStore empty_device{4096}; + + TestChunkBackend() + { + EXPECT_TRUE( + detail::ChunkPinReleaseQueue::create( + credits.credits.bytes, 16, std::make_shared<int>(1), [](void *) noexcept {}, nullptr, &catalog->queue) + .ok()); + domain->backend_id = 17; + domain->tree = 19; + domain->epoch = 1; + domain->mirrors = 2; + domain->catalog = catalog; + domain->transport = transport; + TreeRuntimeOptions options; + options.workers = 1; + options.backend_workers = 1; + EXPECT_TRUE(TreeRuntime::create(options, &runtime).ok()); + EXPECT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + EXPECT_TRUE(ScheduledPageStore::create(&empty_device, nullptr, {}, binding, binding->lifecycle_budget(), + binding->backend_lane(), &scheduled) + .ok()); + } + + ~TestChunkBackend() + { + binding->drain(); + catalog->fail_unpin = false; + catalog->queue->stop_admission(); + (void)catalog->queue->pump(16); + EXPECT_TRUE(catalog->queue->drained()); + catalog->record.reset(); + } + + Status prepare(uint32_t bytes, std::shared_ptr<ChunkPackBacking> *out) const + { + return prepare_chunk_backing(domain, {.owner_epoch = 1, .operation_high = 4, .operation_low = 5}, + {uint64_t{2} << 56U, 1}, bytes, PagePurpose::kBtreePage, credits.credits.bytes, + out); + } +}; +} // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_chunk_collector.h b/lib/crowdb-tree/tests/common/test_chunk_collector.h new file mode 100644 index 000000000..fd4231ba1 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_chunk_collector.h @@ -0,0 +1,165 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_collector.h" +#include "backend/chunk/chunk_pin_release.h" +#include "common/test_chunk_stage_backend.h" + +namespace crowdb::tree::detail +{ +class TestCollectorCatalog final : public TestStageCatalog, public ChunkProtectionCatalog +{ + public: + std::shared_ptr<ChunkPinReleaseQueue> queue; + + std::shared_ptr<ChunkPinReleaseQueue> pin_release_queue() const noexcept override + { + return queue; + } + + Status load_root_owned(uint64_t /*unused*/, uint64_t /*unused*/, ChunkRootBytes * /*unused*/) const override + { + return Status::not_found(); + } + + std::optional<ChunkCandidateStatus> candidate; + uint64_t references = 0; + uint64_t pins = 0; + bool recovered = true; + bool writer_live = false; + bool lose_release = false; + size_t pin_calls = 0; + size_t release_calls = 0; + uint32_t largest_pin_window = 0; + + ChunkProtectionCatalog *protection() noexcept override + { + return this; + } + + Status check_read_owner() const override + { + return Status::Ok(); + } + + Status writer_quiesced(uint64_t /*tree*/, uint64_t /*epoch*/) override + { + return Status::Ok(); + } + + Status load_root_bytes(uint64_t /*tree*/, uint64_t /*generation*/, std::vector<uint8_t> * /*out*/) const override + { + return Status::not_supported(); + } + + Status begin_candidate(uint64_t /*tree*/, const ChunkManifestCandidate & /*unused*/) override + { + return Status::not_supported(); + } + + Status prepare_references(uint64_t /*tree*/, ChunkStageId /*unused*/, uint64_t /*window*/, + std::span<const ChunkStageId> /*unused*/) override + { + return Status::not_supported(); + } + + Status list_candidates(uint64_t /*tree*/, std::optional<ChunkStageId> /*after*/, uint32_t /*limit*/, + std::vector<ChunkCandidateStatus> * /*out*/) const override + { + return Status::not_supported(); + } + + Status load_candidate(uint64_t /*tree*/, ChunkStageId id, std::optional<ChunkCandidateStatus> *out) const override + { + *out = candidate && candidate->id == id ? candidate : std::nullopt; + return Status::Ok(); + } + + Status next_candidate(uint64_t /*unused*/, std::optional<ChunkStageId> after, + std::optional<ChunkCandidateStatus> *out) const override + { + *out = candidate && (!after || candidate->id > *after) ? candidate : std::nullopt; + return Status::Ok(); + } + + Status next_stage(uint64_t /*unused*/, std::optional<ChunkStageId> after, + const std::shared_ptr<AllocationBudget> & /*unused*/, + std::optional<ChunkStageRecord> *out) const override + { + *out = record && (!after || record->id > *after) ? record : std::nullopt; + return Status::Ok(); + } + + Status finish_candidate(uint64_t /*tree*/, ChunkStageId /*unused*/, uint32_t action) override + { + EXPECT_EQ(action, CT_CHUNK_CANDIDATE_RECOVER); + if (!recovered || writer_live || !candidate) { + return Status::unavailable(); + } + candidate->state = ChunkCandidateState::Aborted; + return Status::Ok(); + } + + Status release_references(uint64_t /*tree*/, ChunkStageId id, uint64_t window) override + { + if (!candidate || candidate->id != id || window != candidate->released_windows || + window >= candidate->prepared_windows || references == 0) { + return Status::corruption(); + } + ++release_calls; + ++candidate->released_windows; + --references; + return lose_release ? Status::unavailable() : Status::Ok(); + } + + Status prune_pins(uint64_t /*tree*/, ChunkStageId /*unused*/, std::optional<ChunkStageId> after, uint32_t limit, + ChunkPinProgress *out) override + { + if (!recovered) { + return Status::unavailable(); + } + ++pin_calls; + largest_pin_window = std::max(largest_pin_window, limit); + const uint64_t first = after ? after->operation_low : 0; + const uint64_t count = std::min<uint64_t>(pins - std::min(pins, first), limit); + *out = {.after = count == 0 ? after + : std::optional<ChunkStageId>( + {.owner_epoch = 1, .operation_high = 1, .operation_low = first + count}), + .examined = static_cast<uint32_t>(count), + .released = 0, + .more = first + count < pins}; + return Status::Ok(); + } + + Status change_pin(uint64_t /*tree*/, ChunkStageId /*stage*/, ChunkStageId /*pin*/, bool /*acquire*/) override + { + return Status::not_supported(); + } + + Status compare_stage(uint64_t tree, uint64_t epoch, const std::optional<ChunkStageRecord> &expected, + const std::optional<ChunkStageRecord> &next) override + { + if (!recovered || references != 0 || pins != 0) { + return Status::unavailable(); + } + return TestStageCatalog::compare_stage(tree, epoch, expected, next); + } +}; + +inline ChunkStageRecord test_collector_stage() +{ + return { + .tree_id = 17, + .id = {.owner_epoch = 1, .operation_high = 10, .operation_low = 20}, + .revision = 2, + .chunk_high = uint64_t{2} + << 56, + .chunk_low = 1, + .capacity = 4096, + .manifest_generation = 0, + .purpose = 2, + .state = ChunkStageState::Reloadable, + .packs = {{.offset = 0, .length = 4096, .checksum = 1}} + }; +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/common/test_chunk_physical.h b/lib/crowdb-tree/tests/common/test_chunk_physical.h new file mode 100644 index 000000000..46d8b88f3 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_chunk_physical.h @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_physical_registry.h" +#include "btree/retirement_queue.h" +#include "btree/tree_metrics.h" +#include "runtime/bootstrap_budget.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::test_chunk +{ +struct TestPhysicalBudget : AllocationBudget +{ + std::atomic<int> allowances{-1}; + size_t limit = SIZE_MAX; + std::atomic<size_t> claimed{0}; + std::atomic<bool> reject{false}; + std::weak_ptr<detail::RetirementQueue> queue; + std::shared_ptr<TreeResourceMetrics> metrics = std::make_shared<TreeResourceMetrics>(); + + bool claim(size_t bytes) noexcept override + { + auto permitted = allowances.load(); + while (permitted >= 0) { + if (permitted == 0) { + return false; + } + if (allowances.compare_exchange_weak(permitted, permitted - 1)) { + break; + } + } + if (reject.load()) { + return false; + } + auto previous = claimed.load(); + do { + if (bytes > limit - previous) { + return false; + } + } while (!claimed.compare_exchange_weak(previous, previous + bytes)); + return true; + } + + void release(size_t bytes) noexcept override + { + claimed.fetch_sub(bytes); + } + + [[nodiscard]] std::shared_ptr<detail::RetirementQueue> retirement_queue() const noexcept override + { + return queue.lock(); + } + + [[nodiscard]] std::shared_ptr<TreeResourceMetrics> resource_metrics() const noexcept override + { + return metrics; + } +}; + +struct TestPhysicalHost +{ + std::shared_ptr<TestPhysicalBudget> budget = std::make_shared<TestPhysicalBudget>(); + detail::FlushRuntime cpu{1, 4, budget}; + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(4); + std::shared_ptr<detail::RetirementQueue> queue; + std::shared_ptr<ChunkPhysicalRegistry> registry; + + TestPhysicalHost() + { + // Keep the queue allocator independent of budget's weak queue reference. + auto lifecycle = detail::BootstrapBudget::create(65536); + EXPECT_TRUE(detail::RetirementQueue::create(cpu, lane, lifecycle, &queue).ok()); + budget->queue = queue; + EXPECT_TRUE(ChunkPhysicalRegistry::create(budget, ®istry).ok()); + } + + ~TestPhysicalHost() + { + registry.reset(); + lane->drain(); + queue->close_and_drain(); + } + + static constexpr detail::ChunkStageId stage{.owner_epoch = 1, .operation_high = 10, .operation_low = 20}; + static constexpr detail::ChunkId chunk{uint64_t{2} << 56, 1}; + static constexpr detail::ChunkAllocationIdentity receipt{ + .chunk = chunk, .owner_epoch = 1, .created_ms = 37, .physical_bytes = 8192}; + static constexpr std::array<uint64_t, 8> identity{1, 10, 20, chunk.high, 1, 0, 32, 2}; + + std::shared_ptr<ChunkPhysicalPack> prepare() + { + std::shared_ptr<ChunkPhysicalPack> result; + EXPECT_TRUE(registry->prepare(stage, chunk, budget, &result).ok()); + return result; + } +}; +} // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_chunk_root.h b/lib/crowdb-tree/tests/common/test_chunk_root.h new file mode 100644 index 000000000..899b3892f --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_chunk_root.h @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "common/test_root_transport.h" +#include "load_budget_test_support.h" +#include "snapshot/chunk_root_commit.h" + +#include <map> + +namespace crowdb::tree::test_chunk +{ +struct TestRootBackend +{ + test_read::LoadCredits credits; + std::shared_ptr<TestRootCatalog> catalog = std::make_shared<TestRootCatalog>(credits.credits.bytes); + std::shared_ptr<TestRootTransport> transport = std::make_shared<TestRootTransport>(catalog); + std::shared_ptr<ChunkBackingDomain> domain = std::make_shared<ChunkBackingDomain>(); + std::shared_ptr<detail::ChunkRuntimeIds> ids = + std::make_shared<detail::ChunkRuntimeIds>(1, 23, uint64_t{45} << 32U); + + TestRootBackend() + { + EXPECT_TRUE( + detail::ChunkPinReleaseQueue::create( + credits.credits.bytes, 256, std::make_shared<int>(1), [](void *) noexcept {}, nullptr, &catalog->queue) + .ok()); + domain->backend_id = 17; + domain->tree = 19; + domain->epoch = 1; + domain->mirrors = 2; + domain->catalog = catalog; + domain->transport = transport; + } + + ~TestRootBackend() + { + catalog->queue->stop_admission(); + (void)catalog->queue->pump(catalog->queue->snapshot().pending); + EXPECT_TRUE(catalog->queue->drained()); + catalog->records.clear(); + } + + std::shared_ptr<CapturedDurablePages> pages(size_t count) + { + auto pages = detail::reserved_shared<CapturedDurablePages>(credits.credits.bytes, credits.credits.bytes); + pages->generation = 3; + pages->coverage = 15; + pages->root_page_id = count != 0 ? 0 : kInvalidPageId; + pages->next_page_id = count; + pages->backend_id = 17; + for (uint64_t i = 0; i < count; ++i) { + detail::ChunkStageId stage; + detail::ChunkId chunk; + EXPECT_TRUE(ids->next(PagePurpose::kBtreePage, &stage, &chunk).ok()); + std::shared_ptr<ChunkPackBacking> pack; + EXPECT_TRUE( + prepare_chunk_backing(domain, stage, chunk, 8192, PagePurpose::kBtreePage, credits.credits.bytes, &pack) + .ok()); + std::array<uint8_t, 8192> raw{}; + EXPECT_TRUE(write_chunk_backing(*pack, raw, credits.credits.bytes).ok()); + auto backing = detail::reserved_shared<BackingRef>(credits.credits.bytes); + backing->backend_id = 17; + backing->length = 8192; + backing->owner = pack; + backing->physical_owner = pack; + backing->storage = BackingStorage::kChunkPack; + backing->chunk = std::move(pack); + auto refs = detail::reserved_shared<StoredOverflowReferences>( + credits.credits.bytes, detail::ReservationAllocator<StoredOverflowReference>(credits.credits.bytes)); + pages->frames.emplace(i, DurableCapturedFrame{.page_id = i, + .type = page_type::kLeafBase, + .frame_bytes = 4096, + .logical_bytes = 4132, + .next_overflow = kInvalidPageId, + .overflow_bytes = 0, + .backing = std::move(backing), + .stored_overflow = std::move(refs), + .summary = {}}); + } + return pages; + } + + [[nodiscard]] std::shared_ptr<ChunkRootCommit> + commit(const std::shared_ptr<const CapturedDurablePages> &pages) const + { + ChunkRootIdentity identity{.tree = 19, + .root_generation = 7, + .owner_epoch = 1, + .snapshot_sequence = 4, + .page_generation = pages->generation, + .coverage = pages->coverage, + .gc_floor = 8, + .root_page_id = pages->root_page_id, + .next_page_id = pages->next_page_id, + .wal_replay_offset = 4096}; + std::shared_ptr<ChunkRootCommit> result; + EXPECT_TRUE( + ChunkRootCommit::create(pages, domain, ids, identity, credits.credits.bytes, credits.credits.bytes, &result) + .ok()); + return result; + } +}; +} // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_chunk_root_catalog.h b/lib/crowdb-tree/tests/common/test_chunk_root_catalog.h new file mode 100644 index 000000000..440afa7c4 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_chunk_root_catalog.h @@ -0,0 +1,288 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_pin_release.h" +#include "common/test_stage_catalog.h" + +#include <map> + +namespace crowdb::tree::test_chunk +{ +class TestRootCatalog : public detail::TestStageCatalog, public detail::ChunkProtectionCatalog +{ + public: + explicit TestRootCatalog(std::shared_ptr<AllocationBudget> storage_budget) + : storage_budget_(std::move(storage_budget)) + { + } + + std::shared_ptr<detail::ChunkPinReleaseQueue> queue; + std::map<detail::ChunkStageId /*unused*/, detail::ChunkStageRecord> records; + std::optional<detail::ChunkCandidateStatus> candidate; + std::vector<uint8_t> manifest; + std::vector<uint8_t> committed_manifest; + std::map<uint64_t, std::vector<uint8_t>> retained_roots; + size_t writer_releases = 0; + bool fail_writer_release = false; + uint64_t current_generation = 6; + uint32_t checksum = 0; + uint32_t expected_checksum = 0; + size_t begin_calls = 0; + size_t prepare_calls = 0; + size_t commit_calls = 0; + size_t abort_calls = 0; + size_t acquire_calls = 0; + size_t release_calls = 0; + bool lose_begin = false; + bool lose_prepare = false; + bool lose_commit = false; + bool lose_commit_outcome = false; + bool fail_commit = false; + bool fail_query = false; + bool delete_before_acquire = false; + + detail::ChunkProtectionCatalog *protection() noexcept override + { + return this; + } + + std::shared_ptr<detail::ChunkPinReleaseQueue> pin_release_queue() const noexcept override + { + return queue; + } + + Status check_read_owner() const override + { + return Status::Ok(); + } + + Status writer_quiesced(uint64_t /*unused*/, uint64_t /*unused*/) override + { + ++writer_releases; + return fail_writer_release ? Status::unavailable() : Status::Ok(); + } + + [[nodiscard]] std::shared_ptr<TestRootCatalog> next_owner() const + { + auto next = std::make_shared<TestRootCatalog>(storage_budget_); + next->queue = queue; + next->records = records; + next->candidate = candidate; + next->manifest = manifest; + next->committed_manifest = committed_manifest; + next->retained_roots = retained_roots; + next->current_generation = current_generation; + next->checksum = checksum; + next->expected_checksum = expected_checksum; + return next; + } + + Status load_root_bytes(uint64_t /*unused*/, uint64_t /*unused*/, std::vector<uint8_t> *out) const override + { + *out = manifest; + return Status::Ok(); + } + + Status load_root_owned(uint64_t /*tree*/, uint64_t generation, detail::ChunkRootBytes *out) const override + { + if (current_generation == 0 || committed_manifest.empty()) { + return Status::not_found(); + } + if (generation != 0 && generation != current_generation) { + const auto old = retained_roots.find(generation); + if (old == retained_roots.end()) { + return Status::not_found(); + } + out->assign(old->second.begin(), old->second.end()); + return Status::Ok(); + } + out->assign(committed_manifest.begin(), committed_manifest.end()); + return Status::Ok(); + } + + Status load_stage(uint64_t /*unused*/, detail::ChunkStageId id, + std::optional<detail::ChunkStageRecord> *out) const override + { + const auto found = records.find(id); + if (found == records.end()) { + out->reset(); + } + else { + *out = found->second; + } + return Status::Ok(); + } + + Status compare_stage(uint64_t /*unused*/, uint64_t /*unused*/, + const std::optional<detail::ChunkStageRecord> &expected, + const std::optional<detail::ChunkStageRecord> &next) override + { + auto status = detail::validate_stage_transition(expected, next); + if (!status.ok()) { + return status; + } + if (!next && !expected) { + return Status::invalid_argument(); + } + const auto id = next ? next->id : expected->id; + std::optional<detail::ChunkStageRecord> current; + (void)load_stage(19, id, ¤t); + if (current == next) { + return Status::Ok(); + } + if (current != expected) { + return Status::unavailable(); + } + if (next) { + // The mock durable catalog owns its persisted bytes independently + // of the submitting checkpoint's eventually sealed reservation. + detail::ChunkStageRecord persisted{.tree_id = next->tree_id, + .id = next->id, + .revision = next->revision, + .chunk_high = next->chunk_high, + .chunk_low = next->chunk_low, + .capacity = next->capacity, + .manifest_generation = next->manifest_generation, + .purpose = next->purpose, + .state = next->state, + .packs = decltype(detail::ChunkStageRecord::packs)( + MetadataAllocator<detail::ChunkStagePack>(storage_budget_))}; + persisted.packs.assign(next->packs.begin(), next->packs.end()); + records.insert_or_assign(id, std::move(persisted)); + } + else { + records.erase(id); + } + return Status::Ok(); + } + + Status begin_candidate(uint64_t /*unused*/, const detail::ChunkManifestCandidate &request) override + { + ++begin_calls; + if ((candidate && candidate->state != detail::ChunkCandidateState::Committed && + candidate->state != detail::ChunkCandidateState::Aborted) || + request.expected_generation != current_generation) { + return Status::unavailable(); + } + checksum = 0; + candidate = {.id = request.id, + .generation = request.expected_generation + 1, + .reference_count = request.reference_count, + .prepared_references = 0, + .prepared_windows = 0, + .released_windows = 0, + .state = request.reference_count == 0 ? detail::ChunkCandidateState::Ready + : detail::ChunkCandidateState::Preparing}; + expected_checksum = request.reference_checksum; + manifest.assign(request.manifest.begin(), request.manifest.end()); + return lose_begin ? Status::unavailable() : Status::Ok(); + } + + Status load_candidate(uint64_t /*unused*/, detail::ChunkStageId id, + std::optional<detail::ChunkCandidateStatus> *out) const override + { + if (fail_query) { + return Status::unavailable(); + } + *out = candidate && candidate->id == id ? candidate : std::nullopt; + return Status::Ok(); + } + + Status prepare_references(uint64_t /*unused*/, detail::ChunkStageId id, uint64_t window, + std::span<const detail::ChunkStageId> refs) override + { + ++prepare_calls; + if (!candidate || candidate->id != id || candidate->prepared_windows != window || refs.size() > 64) { + return Status::invalid_argument(); + } + for (auto stage : refs) { + const auto found = records.find(stage); + if (found == records.end() || found->second.state != detail::ChunkStageState::Reloadable) { + return Status::corruption(); + } + } + checksum = detail::chunk_reference_checksum(checksum, refs); + ++candidate->prepared_windows; + candidate->prepared_references += refs.size(); + if (candidate->prepared_references == candidate->reference_count) { + if (checksum != expected_checksum) { + return Status::corruption(); + } + candidate->state = detail::ChunkCandidateState::Ready; + } + return lose_prepare ? Status::unavailable() : Status::Ok(); + } + + Status finish_candidate(uint64_t /*unused*/, detail::ChunkStageId id, uint32_t action) override + { + if (!candidate || candidate->id != id) { + return Status::not_found(); + } + if (action == CT_CHUNK_CANDIDATE_COMMIT) { + ++commit_calls; + if (fail_commit) { + return Status::unavailable(); + } + if (candidate->state != detail::ChunkCandidateState::Ready) { + return Status::invalid_argument(); + } + candidate->state = detail::ChunkCandidateState::Committed; + retained_roots[current_generation] = committed_manifest; + current_generation = candidate->generation; + committed_manifest = manifest; + if (lose_commit_outcome) { + fail_query = true; + return Status::unavailable("injected lost commit and query replies"); + } + return lose_commit ? Status::unavailable() : Status::Ok(); + } + ++abort_calls; + if (candidate->state == detail::ChunkCandidateState::Committed) { + return Status::unavailable(); + } + candidate->state = detail::ChunkCandidateState::Aborted; + return Status::Ok(); + } + + Status release_references(uint64_t /*unused*/, detail::ChunkStageId /*unused*/, uint64_t window) override + { + if (!candidate || candidate->state != detail::ChunkCandidateState::Aborted || + candidate->released_windows != window) { + return Status::invalid_argument(); + } + ++candidate->released_windows; + ++release_calls; + return Status::Ok(); + } + + Status change_pin(uint64_t /*unused*/, detail::ChunkStageId stage, detail::ChunkStageId /*unused*/, + bool acquire) override + { + if (!acquire) { + return Status::Ok(); + } + ++acquire_calls; + if (delete_before_acquire) { + records.at(stage).state = detail::ChunkStageState::Deleting; + } + return records.at(stage).state == detail::ChunkStageState::Reloadable ? Status::Ok() : Status::unavailable(); + } + + Status list_candidates(uint64_t /*unused*/, std::optional<detail::ChunkStageId> /*unused*/, uint32_t /*unused*/, + std::vector<detail::ChunkCandidateStatus> * /*unused*/) const override + { + return Status::not_supported(); + } + + Status prune_pins(uint64_t /*unused*/, detail::ChunkStageId /*unused*/, + std::optional<detail::ChunkStageId> /*unused*/, uint32_t /*unused*/, + detail::ChunkPinProgress * /*unused*/) override + { + return Status::not_supported(); + } + + private: + std::shared_ptr<AllocationBudget> storage_budget_; +}; + +} // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_chunk_route_service.h b/lib/crowdb-tree/tests/common/test_chunk_route_service.h index 49a11395e..4e28f6b84 100644 --- a/lib/crowdb-tree/tests/common/test_chunk_route_service.h +++ b/lib/crowdb-tree/tests/common/test_chunk_route_service.h @@ -31,6 +31,10 @@ struct TestChunkRouteService TestSealReply seal_reply = TestSealReply::Exact; std::atomic<uint64_t> acknowledged{0}; bool drop_allocation = false; + std::atomic<uint64_t> requested_low{0}; + std::atomic<uint32_t> deletes{0}; + std::atomic<bool> fence_missing{false}; + bool unfinished_delete = false; static void handle(uint64_t request_id, uint64_t /*unused*/, uint16_t message, const uint8_t *control, uint32_t length, const uint8_t * /*unused*/, uint32_t /*unused*/, void *connection, void *frame, @@ -51,6 +55,11 @@ struct TestChunkRouteService if (!verifier.VerifyBuffer<FBAllocateChunkRequest>(nullptr)) { return; } + const auto *allocation = flatbuffers::GetRoot<FBAllocateChunkRequest>(control); + if (allocation->chunk_id() != nullptr) { + low = allocation->chunk_id()->low(); + requested_low.store(low); + } allocations.fetch_add(1); if (drop_allocation) { return; @@ -71,6 +80,15 @@ struct TestChunkRouteService acknowledged.store(flatbuffers::GetRoot<FBAdvanceChunkWriteRequest>(control)->acknowledged_cursor()); advances.fetch_add(1); } + else if (message == FBMsgType_EDeleteChunkRequest) { + if (!verifier.VerifyBuffer<FBDeleteChunkRequest>(nullptr)) { + return; + } + const auto *request = flatbuffers::GetRoot<FBDeleteChunkRequest>(control); + low = request->chunk_id()->low(); + fence_missing.store(request->fence_missing()); + deletes.fetch_add(1); + } else { if (!verifier.VerifyBuffer<FBSealChunkRequest>(nullptr)) { return; @@ -100,6 +118,14 @@ struct TestChunkRouteService builder.Finish(CreateFBAdvanceChunkWriteResponse(builder, request_id, 0, code, 0, 0, 0, chunk)); response_type = FBMsgType_EAdvanceChunkWriteResponse; } + else if (message == FBMsgType_EDeleteChunkRequest) { + const FBInt128 id(0x0200'0000'0000'0042ULL, low); + const auto deleted = unfinished_delete ? chunk + : CreateFBChunk(builder, &id, 5, FBChunkState_Deleted, 1, 2, 0, 0, 0, + FBChunkType_BtreePage, 17, 0); + builder.Finish(CreateFBDeleteChunkResponse(builder, request_id, 0, code, 0, 0, 0, deleted)); + response_type = FBMsgType_EDeleteChunkResponse; + } else { builder.Finish(CreateFBSealChunkResponse(builder, request_id, 0, code, 0, 0, 0, chunk)); response_type = FBMsgType_ESealChunkResponse; @@ -150,8 +176,9 @@ struct TestChunkRouteFixture service.parity = index; service.server = crowdb_rpc_server_create(nullptr); static_cast<void>(crowdb_rpc_server_listen(service.server, "127.0.0.1", 0)); - for (auto message : {FBMsgType_EAllocateChunkRequest, FBMsgType_EQueryChunkRequest, - FBMsgType_EAdvanceChunkWriteRequest, FBMsgType_ESealChunkRequest}) { + for (auto message : + {FBMsgType_EAllocateChunkRequest, FBMsgType_EQueryChunkRequest, FBMsgType_EAdvanceChunkWriteRequest, + FBMsgType_ESealChunkRequest, FBMsgType_EDeleteChunkRequest}) { crowdb_rpc_server_register_handler(service.server, message, TestChunkRouteService::handle, &service, nullptr); } diff --git a/lib/crowdb-tree/tests/common/test_chunk_runtime_c_api.cpp b/lib/crowdb-tree/tests/common/test_chunk_runtime_c_api.cpp new file mode 100644 index 000000000..1deacc858 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_chunk_runtime_c_api.cpp @@ -0,0 +1,120 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_c_api_internal.h" +#include "common/test_root_transport.h" +#include "common/test_runtime_catalog.h" +#include "runtime/tree_runtime_c_api.h" + +using namespace crowdb::tree; +using namespace crowdb::tree::detail; +using namespace crowdb::tree::test_chunk; + +namespace +{ +constexpr size_t kTestPinCapacity = 4096; +} + +extern "C" ct_status ct_test_chunk_runtime_create(ct_runtime *runtime, ct_root_catalog **catalog, + ct_chunk_transport **transport) +try { + if (catalog == nullptr || *catalog != nullptr || transport == nullptr || *transport != nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + std::shared_ptr<AllocationBudget> budget; + auto status = runtime_host_allocation_budget(runtime, &budget); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + auto state = std::make_shared<TestRuntimeCatalog>(budget); + status = ChunkPinReleaseQueue::create( + budget, kTestPinCapacity, std::make_shared<int>(1), [](void *) noexcept {}, nullptr, &state->queue); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + auto io = std::make_shared<TestRootTransport>(state); + auto catalog_handle = std::make_unique<ct_root_catalog>(); + catalog_handle->catalog = state; + catalog_handle->transport = io; + auto transport_handle = std::make_unique<ct_chunk_transport>(); + transport_handle->transport = io; + *catalog = catalog_handle.release(); + *transport = transport_handle.release(); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +extern "C" ct_status ct_test_chunk_runtime_next(ct_root_catalog *old, ct_root_catalog **out) +try { + if (old == nullptr || out == nullptr || *out != nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + auto state = std::dynamic_pointer_cast<TestRuntimeCatalog>(old->catalog); + auto io = std::dynamic_pointer_cast<TestRootTransport>(old->transport); + if (!state || !io) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + // All prior Tree work must already have drained on the host thread. Pump + // exact queued writer/pin releases before replacing its catalog incarnation. + const auto progress = state->queue->pump(kTestPinCapacity); + if (progress.failed != 0 || progress.more || state->writer_releases == 0) { + return static_cast<ct_status>(Code::kUnavailable); + } + auto next = state->next_writer(); + io->catalog = next; + auto handle = std::make_unique<ct_root_catalog>(); + handle->catalog = std::move(next); + handle->transport = io; + *out = handle.release(); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} + +extern "C" ct_status ct_test_chunk_runtime_finish(ct_root_catalog *catalog) +{ + if (catalog == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + auto state = std::dynamic_pointer_cast<TestRuntimeCatalog>(catalog->catalog); + if (!state) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + state->queue->stop_admission(); + const auto progress = state->queue->pump(kTestPinCapacity); + return progress.failed == 0 && state->queue->drained() ? 0 : static_cast<ct_status>(Code::kUnavailable); +} + +// A valid drop callback transfers context ownership even when installation fails. +extern "C" ct_status ct_test_chunk_runtime_root_load(ct_root_catalog *catalog, bool (*callback)(void *), + void (*drop_context)(void *), void *context) +try { + if (drop_context == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + std::shared_ptr<void> owner(context, drop_context); + if (catalog == nullptr || callback == nullptr) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + auto state = std::dynamic_pointer_cast<TestRuntimeCatalog>(catalog->catalog); + if (!state) { + return static_cast<ct_status>(Code::kInvalidArgument); + } + state->before_root_load = callback; + state->root_load_owner = std::move(owner); + return 0; +} +catch (const std::bad_alloc &) { + return static_cast<ct_status>(Code::kResourceExhausted); +} +catch (...) { + return static_cast<ct_status>(Code::kInternal); +} diff --git a/lib/crowdb-tree/tests/common/test_chunk_stage_backend.h b/lib/crowdb-tree/tests/common/test_chunk_stage_backend.h new file mode 100644 index 000000000..83f1c4e34 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_chunk_stage_backend.h @@ -0,0 +1,136 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "common/test_stage_catalog.h" + +#include <gtest/gtest.h> + +#include <functional> + +namespace crowdb::tree::detail +{ +class TestFixedTransport final : public ChunkTransport +{ + public: + explicit TestFixedTransport(std::shared_ptr<TestStageCatalog> catalog) : catalog(std::move(catalog)) + { + } + + std::shared_ptr<TestStageCatalog> catalog; + MemoryChunkTransport memory; + bool unknown_allocate = false; + bool fail_fence = false; + bool fail_write = false; + size_t allocations = 0; + + uint64_t physical_epoch = 1; + std::function<void()> before_drain; + bool absent_after_delete = false; + bool fail_drain = false; + mutable size_t drains = 0; + bool unknown_delete = false; + bool physically_deleted = false; + size_t deletions = 0; + + [[nodiscard]] Status drain_fixed_chunk(ChunkId /*unused*/) const override + { + ++drains; + if (before_drain) { + before_drain(); + } + if (physically_deleted && absent_after_delete) { + return Status::not_found(); + } + return fail_drain ? Status::unavailable() : Status::Ok(); + } + + Status inspect_fixed_allocation(ChunkId id, uint64_t epoch, ChunkAllocationIdentity *out) const override + { + if (physically_deleted && absent_after_delete) { + return Status::not_found(); + } + if (epoch != physical_epoch) { + return Status::unavailable(); + } + *out = {.chunk = id, .owner_epoch = epoch, .created_ms = 1, .physical_bytes = 8192}; + return Status::Ok(); + } + + Status delete_chunk(ChunkId /*id*/) override + { + ++deletions; + physically_deleted = true; + return unknown_delete ? Status::unavailable() : Status::Ok(); + } + + bool fixed_staging_supported() const noexcept override + { + return true; + } + + Status allocate_fixed_chunk(ChunkId id, uint64_t capacity, uint64_t epoch, PagePurpose purpose) override + { + if (!catalog->record.has_value()) { + ADD_FAILURE() << "allocation submitted before its durable stage"; + return Status::corruption(); + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Reserved); + EXPECT_EQ(catalog->record->chunk_high, id.high); + EXPECT_EQ(catalog->record->chunk_low, id.low); + ++allocations; + ChunkId result; + auto status = memory.allocate_mirror_chunk(capacity, epoch, &result, purpose); + EXPECT_EQ(result, id); + return unknown_allocate ? Status::unavailable() : status; + } + + Status allocate_mirror_chunk(uint64_t /*unused*/, uint64_t /*unused*/, ChunkId * /*unused*/, + PagePurpose /*unused*/) override + { + return Status::not_supported(); + } + + [[nodiscard]] Status verify_fixed_read_fence(ChunkId /*unused*/, uint32_t /*unused*/, + ChunkIoContext /*unused*/) const override + { + return fail_fence ? Status::not_supported() : Status::Ok(); + } + + Status write_fixed_mirror(ChunkId id, uint32_t mirror, uint64_t offset, const uint8_t *data, size_t length, + ChunkIoContext /*unused*/) override + { + return write_mirror(id, mirror, offset, data, length); + } + + Status read_fixed_mirror(ChunkId id, uint32_t mirror, uint64_t offset, uint8_t *data, size_t length, + ChunkIoContext /*unused*/) const override + { + return read_mirror(id, mirror, offset, data, length); + } + + Status write_mirror(ChunkId id, uint32_t mirror, uint64_t offset, const uint8_t *data, size_t length) override + { + return fail_write ? Status::unavailable() : memory.write_mirror(id, mirror, offset, data, length); + } + + Status advance_write(ChunkId id, uint64_t expected, uint64_t bytes) override + { + return memory.advance_write(id, expected, bytes); + } + + Status read_mirror(ChunkId id, uint32_t mirror, uint64_t offset, uint8_t *data, size_t length) const override + { + return memory.read_mirror(id, mirror, offset, data, length); + } + + Status query_chunk(ChunkId id, ChunkLayout *out) const override + { + return memory.query_chunk(id, out); + } + + Status seal_chunk(ChunkId id, uint64_t epoch, uint64_t bytes) override + { + return memory.seal_chunk(id, epoch, bytes); + } +}; +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/common/test_legacy_chunk.h b/lib/crowdb-tree/tests/common/test_legacy_chunk.h new file mode 100644 index 000000000..b4b5c11c4 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_legacy_chunk.h @@ -0,0 +1,177 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/legacy_chunk_read_store.h" +#include "common/test_chunk_root.h" + +#include <filesystem> +#include <fstream> + +namespace crowdb::tree::test_chunk +{ +class TestLegacyCatalog final : public TestRootCatalog +{ + public: + explicit TestLegacyCatalog(std::shared_ptr<AllocationBudget> budget) + : TestRootCatalog(budget), + storage_(std::move(budget)) + { + } + + [[nodiscard]] const std::shared_ptr<AllocationBudget> &storage() const + { + return storage_; + } + + std::shared_ptr<AllocationBudget> storage_; + std::map<std::pair<uint64_t, uint64_t>, std::vector<uint8_t>> segments; + std::map<std::pair<uint64_t, uint64_t>, uint64_t> generation_pins; + + std::map<uint64_t, std::vector<uint8_t>> migrations; + + [[nodiscard]] std::shared_ptr<TestLegacyCatalog> next_legacy_owner() const + { + auto next = std::make_shared<TestLegacyCatalog>(storage()); + next->queue = queue; + next->records = records; + next->candidate = candidate; + next->manifest = manifest; + next->committed_manifest = committed_manifest; + next->retained_roots = retained_roots; + next->current_generation = current_generation; + next->checksum = checksum; + next->expected_checksum = expected_checksum; + next->segments = segments; + next->generation_pins = generation_pins; + next->migrations = migrations; + return next; + } + + Status prepare_legacy_migration(uint64_t tree, uint64_t generation, std::span<const uint8_t> bytes) override + { + if (tree != 42 || generation != current_generation || + !std::equal(bytes.begin(), bytes.end(), committed_manifest.begin(), committed_manifest.end())) { + return Status::unavailable(); + } + const auto [found, inserted] = migrations.emplace(generation, std::vector<uint8_t>(bytes.begin(), bytes.end())); + return inserted || found->second == std::vector<uint8_t>(bytes.begin(), bytes.end()) ? Status::Ok() + : Status::corruption(); + } + + Status load_legacy_migration_owned(uint64_t tree, uint64_t generation, detail::ChunkRootBytes *out) const override + { + const auto found = migrations.find(generation); + if (tree != 42 || found == migrations.end()) { + return Status::not_found(); + } + out->assign(found->second.begin(), found->second.end()); + return Status::Ok(); + } + + Status load_legacy_reference_owned(uint64_t tree, uint64_t object, detail::ChunkRootBytes *out) const override + { + const auto found = segments.find({tree, object}); + if (found == segments.end()) { + + return Status::not_found(); + } + out->assign(found->second.begin(), found->second.end()); + return Status::Ok(); + } + + Status pin_generation(uint64_t tree, uint64_t high, uint64_t low, uint64_t generation) override + { + if (tree != 42 || generation != current_generation) { + + return Status::unavailable(); + } + const auto [position, inserted] = generation_pins.emplace(std::pair{high, low}, generation); + return inserted || position->second == generation ? Status::Ok() : Status::unavailable(); + } + + Status unpin_generation(uint64_t tree, uint64_t high, uint64_t low) override + { + if (tree != 42) { + + return Status::invalid_argument(); + } + generation_pins.erase({high, low}); + return Status::Ok(); + } +}; + +struct TestLegacyChunk +{ + test_read::LoadCredits credits; + std::shared_ptr<TestLegacyCatalog> catalog = std::make_shared<TestLegacyCatalog>(credits.credits.bytes); + std::shared_ptr<TestRootTransport> transport = std::make_shared<TestRootTransport>(catalog); + std::vector<uint8_t> fixture; + std::map<std::array<uint64_t, 2>, std::vector<uint8_t>> physical; + detail::LegacyChunkLayout layout; + + TestLegacyChunk() + { + const auto path = std::filesystem::path(__FILE__).parent_path().parent_path() / "data/legacy-chunk-v7.bin"; + std::ifstream input(path, std::ios::binary); + fixture.assign(std::istreambuf_iterator<char>(input), {}); + if (fixture.size() < 8 || std::string_view(reinterpret_cast<const char *>(fixture.data()), 7) != "CTLEGV7") { + throw std::runtime_error("legacy Chunk fixture is missing"); + } + size_t offset = 8; + const auto number = [&] { + if (fixture.size() - offset < 8) { + throw std::runtime_error("legacy fixture number truncated"); + } + const auto value = frame_u64(fixture.data(), offset); + offset += 8; + return value; + }; + const auto bytes = [&] { + const auto size = number(); + if (size > fixture.size() - offset) { + throw std::runtime_error("legacy fixture bytes truncated"); + } + std::vector<uint8_t> value(fixture.data() + offset, fixture.data() + offset + size); + offset += size; + return value; + }; + catalog->committed_manifest = bytes(); + EXPECT_TRUE(detail::LegacyChunkLayout::parse(catalog->committed_manifest, 42, 7, &layout).ok()); + catalog->current_generation = layout.generation(); + const auto segment_count = number(); + for (uint64_t i = 0; i < segment_count; ++i) { + const auto tree = number(); + const auto object = number(); + catalog->segments.emplace(std::pair{tree, object}, bytes()); + } + const auto pack_count = number(); + for (uint64_t i = 0; i < pack_count; ++i) { + const auto high = number(); + const auto low = number(); + const auto begin = number(); + const auto value = bytes(); + auto &chunk = physical[{high, low}]; + chunk.resize(std::max(chunk.size(), begin + value.size())); + std::copy(value.begin(), value.end(), chunk.data() + begin); + } + EXPECT_EQ(offset, fixture.size()); + for (const auto &[id, chunk] : physical) { + detail::ChunkId allocated; + EXPECT_TRUE(transport->memory + .allocate_mirror_chunk(chunk.size(), 7, &allocated, detail::chunk_purpose({id[0], id[1]})) + .ok()); + transport->ids.emplace(id, allocated); + for (uint32_t mirror = 0; mirror < 2; ++mirror) { + EXPECT_TRUE(transport->memory.write_mirror(allocated, mirror, 0, chunk.data(), chunk.size()).ok()); + } + EXPECT_TRUE(transport->memory.advance_write(allocated, 0, chunk.size()).ok()); + EXPECT_TRUE(transport->memory.seal_chunk(allocated, 7, chunk.size()).ok()); + } + } + + [[nodiscard]] std::shared_ptr<detail::LegacyChunkReadStore> reader() const + { + return std::make_shared<detail::LegacyChunkReadStore>(catalog, transport, credits.credits.bytes, 42, 7, 1, 2); + } +}; +} // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_root_transport.h b/lib/crowdb-tree/tests/common/test_root_transport.h new file mode 100644 index 000000000..312ad6615 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_root_transport.h @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "common/test_chunk_root_catalog.h" + +#include <functional> + +namespace crowdb::tree::test_chunk +{ +class TestRootTransport : public detail::ChunkTransport +{ + public: + explicit TestRootTransport(std::shared_ptr<TestRootCatalog> catalog) : catalog(std::move(catalog)) + { + } + + std::shared_ptr<TestRootCatalog> catalog; + detail::MemoryChunkTransport memory; + std::map<std::array<uint64_t, 2>, detail::ChunkAllocationIdentity> receipts; + std::function<void()> before_allocate; + std::map<std::array<uint64_t /*unused*/, 2>, detail::ChunkId> ids; + + bool fixed_staging_supported() const noexcept override + { + return true; + } + + Status allocate_fixed_chunk(detail::ChunkId id, uint64_t bytes, uint64_t epoch, PagePurpose purpose) override + { + if (before_allocate) { + before_allocate(); + } + bool protected_allocation = false; + for (const auto &[stage, record] : catalog->records) { + (void)stage; + protected_allocation |= record.chunk_high == id.high && record.chunk_low == id.low && + record.state == detail::ChunkStageState::Reserved; + } + if (!protected_allocation) { + return Status::corruption(); + } + detail::ChunkId physical; + auto status = memory.allocate_mirror_chunk(bytes, epoch, &physical, purpose); + if (status.ok()) { + ids.emplace(std::array{id.high, id.low}, physical); + receipts.emplace(std::array{id.high, id.low}, detail::ChunkAllocationIdentity{.chunk = id, + .owner_epoch = epoch, + .created_ms = physical.low, + .physical_bytes = bytes * 2}); + } + return status; + } + + Status inspect_fixed_allocation(detail::ChunkId id, uint64_t epoch, + detail::ChunkAllocationIdentity *out) const override + { + auto found = receipts.find({id.high, id.low}); + if (found == receipts.end()) { + return Status::not_found(); + } + if (found->second.owner_epoch != epoch) { + return Status::unavailable(); + } + *out = found->second; + return {}; + } + + Status allocate_mirror_chunk(uint64_t /*unused*/, uint64_t /*unused*/, detail::ChunkId * /*unused*/, + PagePurpose /*unused*/) override + { + return Status::not_supported(); + } + + Status verify_fixed_read_fence(detail::ChunkId /*unused*/, uint32_t /*unused*/, + detail::ChunkIoContext /*unused*/) const override + { + return Status::Ok(); + } + + Status write_fixed_mirror(detail::ChunkId id, uint32_t mirror, uint64_t offset, const uint8_t *data, size_t size, + detail::ChunkIoContext /*unused*/) override + { + return write_mirror(id, mirror, offset, data, size); + } + + Status read_fixed_mirror(detail::ChunkId id, uint32_t mirror, uint64_t offset, uint8_t *data, size_t size, + detail::ChunkIoContext /*unused*/) const override + { + return read_mirror(id, mirror, offset, data, size); + } + + Status write_mirror(detail::ChunkId id, uint32_t mirror, uint64_t offset, const uint8_t *data, size_t size) override + { + return memory.write_mirror(ids.at({id.high, id.low}), mirror, offset, data, size); + } + + Status read_mirror(detail::ChunkId id, uint32_t mirror, uint64_t offset, uint8_t *data, size_t size) const override + { + return memory.read_mirror(ids.at({id.high, id.low}), mirror, offset, data, size); + } + + Status advance_write(detail::ChunkId id, uint64_t expected, uint64_t bytes) override + { + return memory.advance_write(ids.at({id.high, id.low}), expected, bytes); + } + + Status seal_chunk(detail::ChunkId id, uint64_t epoch, uint64_t bytes) override + { + return memory.seal_chunk(ids.at({id.high, id.low}), epoch, bytes); + } + + Status query_chunk(detail::ChunkId id, detail::ChunkLayout *out) const override + { + return memory.query_chunk(ids.at({id.high, id.low}), out); + } +}; + +} // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_runtime_catalog.h b/lib/crowdb-tree/tests/common/test_runtime_catalog.h new file mode 100644 index 000000000..7bfb45216 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_runtime_catalog.h @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "common/test_chunk_root_catalog.h" + +#include <set> + +namespace crowdb::tree::test_chunk +{ +class TestRuntimeCatalog final : public TestRootCatalog +{ + public: + explicit TestRuntimeCatalog(std::shared_ptr<AllocationBudget> budget) + : TestRootCatalog(budget), + budget(std::move(budget)) + { + current_generation = 0; + } + + std::shared_ptr<AllocationBudget> budget; + std::map<std::pair<uint64_t, uint64_t>, uint64_t> pins; + + std::map<detail::ChunkStageId, std::set<detail::ChunkStageId>> stage_pins; + bool (*before_root_load)(void *) = nullptr; + std::shared_ptr<void> root_load_owner; + + Status load_root_owned(uint64_t tree, uint64_t generation, detail::ChunkRootBytes *out) const override + { + if (before_root_load != nullptr && !before_root_load(root_load_owner.get())) { + return Status::internal_error("test root load callback failed"); + } + return TestRootCatalog::load_root_owned(tree, generation, out); + } + + [[nodiscard]] std::shared_ptr<TestRuntimeCatalog> next_writer() const + { + auto next = std::make_shared<TestRuntimeCatalog>(budget); + next->queue = queue; + next->records = records; + next->candidate = candidate; + next->manifest = manifest; + next->committed_manifest = committed_manifest; + next->retained_roots = retained_roots; + next->current_generation = current_generation; + next->checksum = checksum; + next->expected_checksum = expected_checksum; + next->pins = pins; + next->stage_pins = stage_pins; + next->before_root_load = before_root_load; + next->root_load_owner = root_load_owner; + return next; + } + + Status change_pin(uint64_t /*tree*/, detail::ChunkStageId stage, detail::ChunkStageId pin, bool acquire) override + { + if (!acquire) { + const auto found = stage_pins.find(stage); + if (found != stage_pins.end()) { + found->second.erase(pin); + if (found->second.empty()) { + stage_pins.erase(found); + } + } + return Status::Ok(); + } + const auto record = records.find(stage); + if (record == records.end() || record->second.state != detail::ChunkStageState::Reloadable) { + return Status::unavailable(); + } + stage_pins[stage].insert(pin); + return Status::Ok(); + } + + Status pin_generation(uint64_t /*tree*/, uint64_t high, uint64_t low, uint64_t generation) override + { + if (generation == 0 || (generation != current_generation && !retained_roots.contains(generation))) { + return Status::not_found(); + } + const auto [found, inserted] = pins.emplace(std::pair{high, low}, generation); + return inserted || found->second == generation ? Status::Ok() : Status::invalid_argument(); + } + + Status unpin_generation(uint64_t /*tree*/, uint64_t high, uint64_t low) override + { + pins.erase({high, low}); + return Status::Ok(); + } + + uint64_t reclaim_before(uint64_t /*tree*/, uint64_t generation) override + { + for (const auto &[id, pinned] : pins) { + (void)id; + generation = std::min(generation, pinned); + } + const auto before = retained_roots.size(); + std::erase_if(retained_roots, + [generation](const auto &entry) { return entry.first != 0 && entry.first < generation; }); + return before - retained_roots.size(); + } +}; +} // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_stage_catalog.h b/lib/crowdb-tree/tests/common/test_stage_catalog.h new file mode 100644 index 000000000..a400c3a62 --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_stage_catalog.h @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "backend/chunk/chunk_fixed_pack.h" + +namespace crowdb::tree::detail +{ +class TestStageCatalog : public RootCatalog +{ + public: + std::optional<ChunkStageRecord> record; + mutable size_t manifest_loads = 0; + + bool stage_schema_supported() const noexcept override + { + return true; + } + + Status load_stage(uint64_t /*unused*/, ChunkStageId /*unused*/, std::optional<ChunkStageRecord> *out) const override + { + *out = record; + return Status::Ok(); + } + + Status compare_stage(uint64_t /*unused*/, uint64_t /*unused*/, const std::optional<ChunkStageRecord> &expected, + const std::optional<ChunkStageRecord> &next) override + { + auto status = validate_stage_transition(expected, next); + if (!status.ok()) { + return status; + } + if (record == next) { + return Status::Ok(); + } + if (record != expected) { + return Status::unavailable(); + } + record = next; + return Status::Ok(); + } + + std::shared_ptr<const ChunkManifest> load(uint64_t /*unused*/) const override + { + ++manifest_loads; + return {}; + } + + std::shared_ptr<const ChunkManifest> load_generation(uint64_t /*unused*/, uint64_t /*unused*/) const override + { + return {}; + } + + Status persist_reference_segment(uint64_t /*unused*/, + std::shared_ptr<const ChunkReferenceSegmentImage> /*segment*/) override + { + return Status::not_supported(); + } + + std::shared_ptr<const ChunkReferenceSegmentImage> load_reference_segment(uint64_t /*unused*/, + uint64_t /*unused*/) const override + { + return {}; + } + + uint64_t allocate_reference_segment_id(uint64_t /*unused*/) override + { + return 0; + } + + uint64_t discard_reference_segments(uint64_t /*unused*/, const std::vector<uint64_t> & /*objects*/) override + { + return 0; + } + + Status publish(uint64_t /*unused*/, uint64_t /*unused*/, uint64_t /*unused*/, + std::shared_ptr<const ChunkManifest> /*manifest*/) override + { + return Status::not_supported(); + } + + Status pin_generation(uint64_t /*unused*/, uint64_t /*unused*/, uint64_t /*unused*/, uint64_t /*unused*/) override + { + return Status::not_supported(); + } + + Status unpin_generation(uint64_t /*unused*/, uint64_t /*unused*/, uint64_t /*unused*/) override + { + return Status::not_supported(); + } + + uint64_t reclaim_before(uint64_t /*unused*/, uint64_t /*unused*/) override + { + return 0; + } + + uint64_t retained_manifest_count(uint64_t /*unused*/) const override + { + return 0; + } + + uint64_t pinned_bytes(uint64_t /*unused*/) const override + { + return 0; + } + + uint64_t oldest_pin_age_ms(uint64_t /*unused*/) const override + { + return 0; + } +}; + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/common/test_tree_core_chunk.h b/lib/crowdb-tree/tests/common/test_tree_core_chunk.h new file mode 100644 index 000000000..6ff47482c --- /dev/null +++ b/lib/crowdb-tree/tests/common/test_tree_core_chunk.h @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/chunk/chunk_backend_state.h" +#include "btree/tree_core_access.h" +#include "common/test_chunk_root.h" + +namespace crowdb::tree::test_chunk +{ +struct TestCoreChunk +{ + test_chunk::TestRootBackend backend; + Config config; + std::shared_ptr<detail::ChunkRuntimePageStore> store; + std::unique_ptr<Crowdbtree> tree; + bool attempted_open = false; + + TestCoreChunk() + { + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + EXPECT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + backend.catalog->current_generation = 0; + configure_store(0); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + } + + ~TestCoreChunk() + { + tree.reset(); + config.backend_owner.reset(); + store.reset(); + config.runtime.reset(); + } + + void configure_store(uint64_t generation) + { + store = std::make_shared<detail::ChunkRuntimePageStore>( + detail::ChunkRuntimeOptions{.tree = 19, + .owner_epoch = 1, + .open_generation = generation, + .iu_size = 4096, + .mirrors = 2, + .max_chunk_bytes = 256 * 1024 * 1024}, + backend.catalog, backend.transport, backend.credits.credits.bytes); + config.page_store = store.get(); + config.backend_owner = store; + } + + Status open(uint64_t generation = 0) + { + if (attempted_open) { + backend.catalog = backend.catalog->next_owner(); + backend.transport->catalog = backend.catalog; + configure_store(generation); + } + attempted_open = true; + return Crowdbtree::open(config, &tree); + } + + [[nodiscard]] Status put(uint64_t slot, const std::string &value) const + { + return tree->apply(slot, Batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = value}}}); + } +}; +} // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/compaction_fault_store.h b/lib/crowdb-tree/tests/compaction_fault_store.h new file mode 100644 index 000000000..45213eedf --- /dev/null +++ b/lib/crowdb-tree/tests/compaction_fault_store.h @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/backend/block_page_store.h" + +#include <atomic> + +namespace crowdb::tree +{ +// The actual BlockPageStore remains the allocator/domain authority. Its async +// adapter runs on ScheduledPageStore's backend lane and preserves each grant. +class TestCompactionFaultStore final : public AsyncPageStore +{ + public: + explicit TestCompactionFaultStore(BlockPageStore &store) : store_(store) + { + } + + void arm(int index, bool sync) + { + writes_.store(0); + syncs_.store(0); + fail_write_.store(sync ? -1 : index); + fail_sync_.store(sync ? index : -1); + } + + [[nodiscard]] unsigned faults() const + { + return faults_.load(); + } + + [[nodiscard]] bool write_completion_is_reloadable() const noexcept override + { + return true; + } + + uint64_t submit_read(PageAddr addr, void *data, size_t size, AsyncCompletion complete) override + { + complete.complete(store_.read_at(addr, static_cast<uint8_t *>(data), size, complete.allocations)); + return 0; + } + + uint64_t submit_write(PageAddr addr, const void *data, size_t size, AsyncCompletion complete) override + { + if (writes_.fetch_add(1) == fail_write_.load()) { + faults_.fetch_add(1); + complete.complete(Status::io_error("injected compaction write failure")); + } + else { + complete.complete(store_.write_at(addr, static_cast<const uint8_t *>(data), size, complete.allocations)); + } + return 0; + } + + Status submit_fsync(AsyncCompletion complete) override + { + if (syncs_.fetch_add(1) == fail_sync_.load()) { + faults_.fetch_add(1); + complete.complete(Status::io_error("injected compaction sync failure")); + } + else { + complete.complete(store_.sync(complete.allocations)); + } + return Status::Ok(); + } + + void cancel(uint64_t /*op_id*/) override + { + } + + private: + BlockPageStore &store_; + std::atomic<int> writes_{0}; + std::atomic<int> syncs_{0}; + std::atomic<int> fail_write_{-1}; + std::atomic<int> fail_sync_{-1}; + std::atomic<unsigned> faults_{0}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/data/legacy-chunk-v7.bin b/lib/crowdb-tree/tests/data/legacy-chunk-v7.bin new file mode 100644 index 0000000000000000000000000000000000000000..1fab0d596a3216e5afe269181a769983f02e42e2 GIT binary patch literal 115472 zcmeI*dvF%j8OQO>CP=LYA|j-sHX<}~85;wL2&IXyMwm*-NQz9E!L)!EXf23Xs&!1n zKg0nGjx$KhD7131?bI-!EgdLEIx@CJE>n8Jbbw)?9eR-oilrCQvpMHEIcMD%AmHZm zJu~F&*|X2?zMnVmlK02HAyawh?RQKb89aJX(A2uCuY>hskY)O4UDOtC&};s6YE3u2 z#{arEM~^do?;Rku@qJ0f)oS4%FNRkH(L&T@Lp}P)*KFCLoyl{G>}0gExrJf#vR-nF z>|}wrl`XnXcIFmk%eF<b(A3+avI4W-Eix`aP+}%^XNw+?ow-GpxFWe|vFyw(GS6ZV zR3#U^Dm!zF<oONeBp2<Kow-Gyl5MMHp{ciz=>yrBTXeB(yHpnZqWl`O-YxpHY`aPp zntFSr(aX$wx9AetwpJGWqG_uWiY}FH-;jl--X5uDgIVt$DOa}LEen3p%XJAwJ!IR1 zve4AqBkkB@*1JdQDcc^G1;6Onp@gDn2fAg+Le$$M<u;hr?vXB&P4kkAibAvAE$Ssj z1<6H|dL<NHE=5JjMUV7PDC#XmCCNoAiW7>ikfMs@qAj;36!np!s^p>rcO?|{m7+Px zMd9>>qI@Z;PA(caFQMo&QnWO=Xxu`*en`~G74}L~G-R1qlx;6<vQziT?I%U6lZz5B zmRCuU`HRC0?UCeSIVGBq{mN;|_%?H1GB!K0e{sl8<MToDcfS2*m)fK`?-A$?ZJnHG zQk{44j@5bB=-8ZhDUQwUt<o!!j^|xiV|Cv3GdAa4CS!Bn)i5^aUG!pe-nA_@=Uu{L zbKaFIHs@WCVsqYgCpPC@USf0JRU|g&T`XdA-ZddM=e_h}bKWaFHs`&NV{_i?RnOTM ziyo@4&EMB{F5!FE&dK*~^dgfVw#>dh@<WyEC+w#FuF11M=VdB0rO~S}{t}bxu)W}^ zA9N;~*ZOdCjCjrRL#G}yQ*;O*fB*srAb<b@2q1s}0tg_`jsy-)yrFw69hA5AHX@VL zwGl#&7a_<rJpu?IfB*srAb<b@2q1s}0tj?ofur|4SZZ&$)DKgJHbRi-K>9EqAw+jR zs6UV2*U*ruyt6DTD?5m8L5Oa%dd1v^*BEQ>suUu+{W6-jw_&QW+`2GTEk~_-knZ0# zz2qBuZ4lTS!Svl1S+d{g4y>)}7}4FD(eg{qa*eY*-C4ffSuS*zFLsvyrtT1weeQIY zUv-wBQ_Im~ijMoRv;J;p{VmS&)y{I@EFU&D(SF`?mc=YuPeT9!1Q0*~0R#|0009JA z6_`Eq+oR-TU!lB<sy1b6@@DxUnfffCZqDRL`Dj`w@7J8ko8-f8p?rwQnH(V>p9|$< zNzUYOc_$dk2c4YBVe&8Cq1<TiOcoc(iG=12XAl$wO-((PV*8d>K1t@3-Z)B5Gn5af zxwALRd4}?NH+OcVoM<SP?(9u+rlDZEvm?~o%BB{IraL=ay%E&2!s*TqlSdhfr#o9L z4>J@{clJhkn4yTevqkbS?c0BMR^EQg=ga~%IsV1o9|~}wn+)wf<iidvMhH=z(%M^* z^Su5fg&KU}i4%31%88S6)KRPTj_uc2oX}_|+PEMO(0rOB>uV{uHa;B65kLR|1Q0*~ z0R#|0009ILXh#BhPY+ltnSnVi8QKUzexJw3MhM<xQV~Mg@~5`hJfY&~(f8BN6K1E} zJRk-LAb<b@2q1s}0tg_000IbfG=U|X$~H@8V9s5JHbPijqG!%LLhuw+p0K`Tm%2kh zEku6?WXk)cohN)f<>mn~KmY**5I_I{1Q0*~0R#|0paTdTx@5*G$qdZt%Fsp#^1p+e zV}#%-s0g7U_vCDwCrsJZZ&ccO!Z%WG9uNZr5I_I{1Q0*~0R#|0009Krr@-M~UNlxR z19OHlv=M^*C!*&gLhuw+p0I9Xz50C>=Kh5GsYmWfJ5P8p<>mn~KmY**5I_I{1Q0*~ z0R#|0pxp?R7H7X8nSnVe8QKWpgE@NUyhjM0f{GEQ%>CgCn<uRLL+#^f=Lz3Txp_bg z5I_I{1Q0*~0R#|0009ILNK@doG0V#(Gce~NLmMIF*XWrG8zFcKDo@DWwdkPD6aF&c zk3UX3PneT(^MDv2fB*srAb<b@2q1s}0tg`R2@7QP`T0u849uZrXd{Ht%k<1A7$JBH zDo>bQ-uRZy6TV-!WKr6A!rYXb2gCpY1Q0*~0R#|0009ILKmdUYFYuo&{~RWnfjOEC zZG<pwwVp{ULhuw+o-p?KzUKcg^0Qx__g>n0!b2%H4~PK*2q1s}0tg_000IagfB*s) zMBv|<nd>DpFb9yKjSy-!=$UpAA$STZPpB*Y(Qtcz!t_6VZfDwg!n~B52gCpY1Q0*~ z0R#|0009ILKmdXBCD3?s@FK|!%sypkBZQah^i2DU5IhByCp^EndV<Xp`hMrx{%Pk4 z52xHbAO;8^fB*srAb<b@2q1s}0tmE0VEgq4PfKQCb|XU@A?(<rXWDs$;3=p);hnn8 zg*HzpJ+bz1+IhnCl$!^{009ILKmY**5I_I{1Q0*~fwL3XGHCk}$qY=T3~hvP?2w-6 zND+dkpz?&v2mQ49_c0zF|Eph1p0J>&w`tI5S`UA9n@3r)2X!*4WzDA{fB*srAb<b@ z2q1s}0tg_`p#<umnm<!A1M?<VhBiXTZO}6<L<k+~@y-v_Tln8B-uGX7f5OVKZ*OWX zPl&|~2I|~9@`R4{O~ctDfB*srAb<b@2q1s}0thI9Y1gdUBAG!;5kgU@-|}{f2*G>S zqdcKz&l8(%o=~;*i@%UOA<w?i*D!K_g7oK}Ao!K`Fx;QuJvlP)wHP6Q00IagfB*sr zAb<b@2&5&@y?)oW)*^&Sy*@TVNbA(k`*GD<_^*uKvB2gDBPRE_RPu!C*wG|MkUlnj z&ck?~FeBx5&JTn_1Q0*~0R#|0009ILKmdWxCh&u;&(yaTAw1H*YeWd%bF1=%@iUhH z#O4X(_fNec?L1*-%FP2}fB*srAb<b@2q1s}0tg_`#RLu=J@9sG5yFb%qzK{EsWsiY z3YN+fuG+tKpUo3?maiF?cAju=%B`zB_dHGn5I_I{1Q0*~0R#|000ASA(|hQntwji1 zZtdz3g7@vB@`S_rZ%(s$!iG1G&q_N_xG&}A0Wm-T0R#|0009ILKmY**5a^Tw4UH%B zTZ<45+(m@Y(vz?9gxA;pwb14Xzk4YExwP|y`%`W$S+O1g1Q0*~0R#|0009ILK%hel ze188s*S8iSgwu%-≥2dBWXK);?qNg!QK<)TW&$JdkobBRDof009ILKmY**5I_I{ z1Q6%|0-NTSz0g{OFmN6b!r4ESDo+T<eD8$K6Nb*(`1`c;gjp%Kvsa`70R#|0009IL zKmY**5I~^a2)y68ZD?x|!nlPxLMYc=5a#p@hGcc6U#HmSr%UAt|2utT!MZVjFKIzL zJ3Dre>r_qD!friP3}Ll1l@lg~fqo{;^AQ9&LC{xiy1wSx*Jf1*L4jIUeQuHNuPo7h zVTJC8SLwcVj_%8=bw72f?(biv`}wuHfAS68e|NX;S02>;E5~*JdUQJKc-!-Izq>&9 ze=gGfu@c>XSfTrzuj>A?xw^lmM)foV5I_I{1Q0*~0R#|0009ILa0I^ev%#^bK!!F# zkmx}AScKs0j70<xKmY**5I_I{1Q0*~0R#}}qyih)JbO$s1M?7NXd{HO0zLDX*&vD# zrUdpb1hclq?v}znPiHD8-4^}Be)Q*CYkoUMHkN<vAF@m@4JZhLX#2iL*45X&H}G0H J7qfl#{{igL`;!0w literal 0 HcmV?d00001 diff --git a/lib/crowdb-tree/tests/data/legacy-chunk-v7.json b/lib/crowdb-tree/tests/data/legacy-chunk-v7.json new file mode 100644 index 000000000..c63304536 --- /dev/null +++ b/lib/crowdb-tree/tests/data/legacy-chunk-v7.json @@ -0,0 +1,24 @@ +{ + "format": 7, + "writer_base_commit": "57c8303bad4d372ddbf20d7259474ad3df45e591", + "fixture_sha256": "70615da50d499a0d6158e514f9ff05a222831a3cabfb5fc8a0879773a0da10bd", + "tree_id": 42, + "owner_epoch": 7, + "root_generation": 1, + "coverage": 20, + "wal_replay_offset": 1234, + "records": 20, + "primary_and_overflow_frames": 23, + "legacy_packs": 29, + "logical_bytes": 110979, + "configuration": { + "frame_bytes": 4096, + "leaf_split_bytes": 512, + "max_inline_value": 64, + "iu_size": 1, + "pack_bytes": 4096, + "mirror_copies": 2 + }, + "values": "key1..key19 have 128-byte values; key20 has 8192 bytes of u; values use slots matching key numbers", + "purpose": "Frozen output of the real historical Crowdbtree + ChunkPageStore writer; tests require only the read-only migration codecs." +} diff --git a/lib/crowdb-tree/tests/extent_ledger_test_support.h b/lib/crowdb-tree/tests/extent_ledger_test_support.h new file mode 100644 index 000000000..c4f65979e --- /dev/null +++ b/lib/crowdb-tree/tests/extent_ledger_test_support.h @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "backend/local/extent_ledger.h" +#include "load_budget_test_support.h" + +namespace crowdb::tree::test_read +{ +template <class Submit> ExtentLedgerResult await_ledger(Submit submit) +{ + ExtentLedgerResult result; + std::atomic<bool> done{false}; + auto status = submit([&](ExtentLedgerResult value) { + result = std::move(value); + done.store(true, std::memory_order_release); + done.notify_one(); + }); + if (!status.ok()) { + return {.status = std::move(status), .extent = {}}; + } + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + return result; +} + +struct LedgerFixture +{ + LoadCredits credits; + std::shared_ptr<ExtentLedger> ledger; + + explicit LedgerFixture(ExtentLedgerLimits limits = {.backend_id = 1}) + { + EXPECT_TRUE(ExtentLedger::create(limits, credits.credits.bytes, &ledger).ok()); + EXPECT_LE(credits.budget.snapshot().allocated[LoadCredits::kBytes], ExtentLedger::creation_metadata_bound()); + } + + ExtentLedgerResult activate() + { + return await_ledger([&](auto done) { return ledger->activate(credits.credits.bytes, std::move(done)); }); + } + + ExtentLedgerResult recover(uint64_t address, uint64_t length) + { + return await_ledger( + [&](auto done) { return ledger->recover(address, length, credits.credits.bytes, std::move(done)); }); + } + + ExtentLedgerResult reserve(uint64_t length) + { + return await_ledger([&](auto done) { return ledger->reserve(length, credits.credits.bytes, std::move(done)); }); + } +}; +} // namespace crowdb::tree::test_read diff --git a/lib/crowdb-tree/tests/flush_preparation_test_support.h b/lib/crowdb-tree/tests/flush_preparation_test_support.h new file mode 100644 index 000000000..0b92c3ac9 --- /dev/null +++ b/lib/crowdb-tree/tests/flush_preparation_test_support.h @@ -0,0 +1,179 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/allocation_bound.h" +#include "btree/apply_progress.h" +#include "btree/flush_operation.h" +#include "btree/page_cache.h" +#include "inner_load_test_support.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +struct TestPreparedFlushTree +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(16 * 1024 * 1024); + result[static_cast<size_t>(detail::BudgetClass::kIo)] = 1; + return result; + } + + static std::shared_ptr<detail::AllocationReservation> + reserve_lifecycle(const std::shared_ptr<detail::ResourceBudget::Account> &account) + { + const auto bytes = SourceAdmission::allocation_bound(); + if (!bytes) { + throw std::bad_alloc(); + } + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kAdmittedWriter, *bytes); + if (!credit) { + throw std::bad_alloc(); + } + return detail::AllocationReservation::create(std::move(*credit)); + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + std::shared_ptr<detail::AllocationReservation> lifecycle = reserve_lifecycle(account); + SourceAdmission sources{ + 0, + 1, + {.account = account, + .max_entries = 64, + .max_key_bytes = 4096, + .max_gap_entries = 8, + .max_gap_key_bytes = 256, + .max_key_size = 256}, + lifecycle + }; + std::shared_ptr<detail::FlushRuntime> runtime; + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(32); + std::shared_ptr<TreeResourceMetrics> resources; + std::shared_ptr<SharedPageCache> cache; + PublicationCoordinator publisher; + std::shared_ptr<test_read::TestInnerStore> store = std::make_shared<test_read::TestInnerStore>(); + std::atomic<uint64_t> frontier{0}; + FlushBatchOptions batch; + FlushPreparationOptions preparation; + std::unique_ptr<FlushController> controller; + + explicit TestPreparedFlushTree(size_t workers = 1, bool cold = false, bool metrics = false) + : runtime(std::make_shared<detail::FlushRuntime>(workers, 32)), + resources(metrics ? std::make_shared<TreeResourceMetrics>() : nullptr), + cache(metrics ? std::make_shared<SharedPageCache>(PageCacheLimits{.max_pages = 2, .max_waiters = 1}, nullptr, + resources, runtime.get(), lane) + : nullptr), + publisher(64, 1, {}, std::pmr::get_default_resource(), cache) + { + batch.run_limits = {.max_entries = 4, .max_key_bytes = 256, .key_block_bytes = 64}; + batch.max_sources = 8; + batch.parallelism = 4; + batch.routing_leaves_per_step = 1; + preparation.runtime = runtime; + preparation.store = store; + preparation.cache = cache; + preparation.loads.max_blob_bytes = 128 * 1024; + preparation.loads.max_frame_bytes = 128 * 1024; + preparation.loads.max_inner_children = 64; + preparation.loads.max_inner_key_bytes = 4096; + auto &maintenance = preparation.maintenance; + maintenance.max_candidates = 8; + maintenance.max_visits = 16; + maintenance.max_steps = 32; + maintenance.visits_per_step = 2; + maintenance.publication.max_changed_pages = 16; + maintenance.publication.max_path_pages = 8; + auto &pages = maintenance.publication.pages; + pages.max_metadata_bytes = 4096; + pages.max_work_entries = 128; + pages.max_output_pages = 8; + pages.primary_page_bytes = 512; + pages.target_page_bytes = 384; + pages.merge_trigger_bytes = 64; + pages.merge_target_bytes = 128; + + auto left = std::make_shared<PageState>(); + left->range.upper = "m"; + left->right_sibling = 1; + auto right = std::make_shared<PageState>(); + right->range.lower = "m"; + if (metrics) { + left->bases.push_back(cold_leaf(0, 10)); + right->bases.push_back(cold_leaf(1, 11)); + } + auto first = std::make_shared<PublishedPage>(); + first->leaf = left; + auto second = std::make_shared<PublishedPage>(); + second->leaf = right; + auto root = std::make_shared<PublishedPage>(); + root->level = 1; + root->metrics.leaf = false; + const auto inner = test_read::inner({"m"}, {0, 1}); + if (cold) { + root->inner_backing = test_read::inner_backing(store, 0, *inner); + } + else { + root->inner = inner; + } + auto catalog = std::make_shared<PublishedSources>(); + catalog->tables.assign(sources.catalog()->tables.begin(), sources.catalog()->tables.end()); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = { + .record = std::make_shared<InstallRecord>(InstallToken{} + ), + .pages = {{.page_id = 0, .expected = {}, .replacement = first}, + {.page_id = 1, .expected = {}, .replacement = second}, + {.page_id = 2, .expected = {}, .replacement = root}}, + .root_page_id = 2, + .expected_sources = publisher.capture()->sources, + .sources = catalog + }; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + EXPECT_TRUE(publisher.try_submit(request)); + runtime->drain(); + controller = std::make_unique<FlushController>(publisher, *runtime, sources, frontier, lane, batch); + } + + BaseRef cold_leaf(uint64_t identity, uint64_t address) + { + std::array<uint8_t, 512> frame{}; + LeafFrameBuilder builder(frame.data(), frame.size()); + builder.finish(identity, kInvalidPageId); + PageSummary summary; + summary.exact = true; + summary.reachable_leaf_pages = 1; + summary.reachable_page_capacity_bytes = frame.size(); + frame_set_summary(frame.data(), frame.size(), summary); + auto &blob = store->blobs[address]; + encode_durable_page(frame.data(), frame.size(), compress_algo::kNone, &blob); + auto backing = std::make_shared<BackingRef>(); + backing->address = address; + backing->length = blob.size(); + backing->owner = store; + return {.identity = identity, .backing = std::move(backing)}; + } + + [[nodiscard]] detail::ResourceBudget::Limits lifecycle_reservations() const noexcept + { + detail::ResourceBudget::Limits result{}; + result[static_cast<size_t>(lifecycle->category())] = lifecycle->available(); + return result; + } + + std::array<std::optional<detail::ResourceBudget::Credit>, detail::ResourceBudget::kClasses> exhaust_unused() const + { + std::array<std::optional<detail::ResourceBudget::Credit>, detail::ResourceBudget::kClasses> blockers; + const auto snapshot = budget.snapshot(); + for (size_t category = 0; category < blockers.size(); ++category) { + const auto remaining = limits()[category] - snapshot.allocated[category] - snapshot.reserved[category]; + blockers[category] = + detail::ResourceBudget::try_reserve(account, static_cast<detail::BudgetClass>(category), remaining); + EXPECT_TRUE(blockers[category]); + } + return blockers; + } +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/inner_load_test_support.h b/lib/crowdb-tree/tests/inner_load_test_support.h new file mode 100644 index 000000000..8e78287b6 --- /dev/null +++ b/lib/crowdb-tree/tests/inner_load_test_support.h @@ -0,0 +1,179 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/published_route.h" +#include "crowdb-tree/maptable/compressor.h" +#include "published_read_test_support.h" + +#include <cstring> + +namespace crowdb::tree::test_read +{ +class TestInnerStore final : public AsyncPageStore +{ + public: + std::map<uint64_t, std::vector<uint8_t>> blobs; + std::deque<AsyncCompletion> pending; + std::atomic<size_t> reads{0}; + bool deferred = false; + + uint64_t submit_read(PageAddr address, void *destination, size_t length, AsyncCompletion complete) override + { + const auto &blob = blobs.at(address); + EXPECT_EQ(length, blob.size()); + std::memcpy(destination, blob.data(), std::min(length, blob.size())); + const uint64_t id = reads.fetch_add(1) + 1; + if (deferred) { + pending.push_back(complete); + } + else { + complete.complete(Status::Ok()); + } + return id; + } + + uint64_t submit_write(PageAddr /*addr*/, const void * /*buf*/, size_t /*len*/, AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return 0; + } + + Status submit_fsync(AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return Status::not_supported(); + } + + void cancel(uint64_t /*op_id*/) override + { + } + + void finish(Status status = Status::Ok()) + { + ASSERT_FALSE(pending.empty()); + const auto callback = pending.front(); + pending.pop_front(); + callback.complete(std::move(status)); + } +}; + +inline std::shared_ptr<InnerPageState> inner(std::vector<std::string> separators, std::vector<uint64_t> children) +{ + auto result = std::make_shared<InnerPageState>(); + result->separators.assign(separators.begin(), separators.end()); + result->children.assign(children.begin(), children.end()); + return result; +} + +inline std::shared_ptr<const BackingRef> inner_backing(const std::shared_ptr<TestInnerStore> &store, uint64_t id, + const InnerPageState &state) +{ + std::vector<std::string> separators; + for (const auto &key : state.separators) { + separators.push_back(Slice(key).to_string()); + } + const std::vector<uint64_t> children(state.children.begin(), state.children.end()); + const std::unique_ptr<InnerBase> frame(InnerBase::build(separators, children)); + auto &blob = store->blobs[id]; + encode_durable_page(frame->frame(), frame->page_bytes(), compress_algo::kNone, &blob); + auto backing = std::make_shared<BackingRef>(); + backing->address = id; + backing->length = blob.size(); + backing->owner = std::make_shared<int>(1); + return backing; +} + +class TestInnerLoader final : public PageLoader +{ + public: + bool deferred = false; + size_t calls = 0; + std::map<uint64_t, std::shared_ptr<const InnerPageState>> states; + std::deque<InnerComplete> pending; + std::deque<uint64_t> ids; + + void load_leaf(BaseRef /*base*/, LeafComplete complete) override + { + complete(Status::not_supported(), {}); + } + + void load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete) override + { + ++calls; + if (deferred) { + pending.push_back(std::move(complete)); + ids.push_back(backing->address); + } + else { + complete(Status::Ok(), states.at(backing->address)); + } + } + + void finish(Status status = Status::Ok()) + { + ASSERT_FALSE(pending.empty()); + auto callback = std::move(pending.front()); + const auto id = ids.front(); + pending.pop_front(); + ids.pop_front(); + callback(std::move(status), states.at(id)); + } +}; + +struct TestColdTree +{ + std::shared_ptr<TestInnerStore> store = std::make_shared<TestInnerStore>(); + std::shared_ptr<TestInnerLoader> loader = std::make_shared<TestInnerLoader>(); + std::shared_ptr<PublishedTreeState> view = std::make_shared<PublishedTreeState>(); + + TestColdTree() + { + loader->states[0] = inner({"m"}, {1, 2}); + loader->states[1] = inner({"g"}, {3, 4}); + loader->states[2] = inner({"t"}, {5, 6}); + std::vector<PageReplacement> changes; + changes.reserve(7); + for (uint64_t id = 0; id < 3; ++id) { + auto index = std::make_shared<PublishedPage>(); + index->level = id == 0 ? 2 : 1; + index->inner_backing = inner_backing(store, id, *loader->states.at(id)); + changes.push_back({.page_id = id, .expected = {}, .replacement = std::move(index)}); + } + changes.push_back({ + .page_id = 3, + .expected = {}, + .replacement = + published(page({{.key = "", .value = "empty"}, {.key = "a", .value = "a"}}, + {.upper = "g"} + )) + }); + changes.push_back({ + .page_id = 4, + .expected = {}, + .replacement = published( + page({{.key = "g", .value = "g"}, {.key = "l", .value = "l"}}, + {.lower = "g", .upper = "m"} + )) + }); + changes.push_back({ + .page_id = 5, + .expected = {}, + .replacement = published( + page({{.key = "m", .value = "m"}, {.key = "n", .value = "n"}}, + {.lower = "m", .upper = "t"} + )) + }); + changes.push_back({ + .page_id = 6, + .expected = {}, + .replacement = published(page({{.key = "t", .value = "t"}, {.key = "z", .value = "z"}}, + {.lower = "t"} + )) + }); + MappingView::CopyStats copied; + view->mapping = view->mapping.replace(changes, copied); + view->root_page_id = 0; + } +}; +} // namespace crowdb::tree::test_read diff --git a/lib/crowdb-tree/tests/integration/alignment_test.cpp b/lib/crowdb-tree/tests/integration/alignment_test.cpp index 52196d394..b05da8c04 100644 --- a/lib/crowdb-tree/tests/integration/alignment_test.cpp +++ b/lib/crowdb-tree/tests/integration/alignment_test.cpp @@ -88,7 +88,7 @@ TEST(Alignment, Iu4096AllocatorReuseStaysAligned) // Rewrite the same keys repeatedly; aligned gaps are reused so the file stays // a 4 KiB multiple and roughly flat. - uint64_t slot = 100000; + uint64_t slot = 81; for (int round = 0; round < 30; ++round) { for (int i : {1, 2, 3, 4, 5}) { ASSERT_TRUE(t.apply(slot, put_one(make_key(i), "r" + std::to_string(round))).ok()); diff --git a/lib/crowdb-tree/tests/integration/async_get_test.cpp b/lib/crowdb-tree/tests/integration/async_get_test.cpp index e04f2de6d..796dd2632 100644 --- a/lib/crowdb-tree/tests/integration/async_get_test.cpp +++ b/lib/crowdb-tree/tests/integration/async_get_test.cpp @@ -65,9 +65,9 @@ ct_status poll_until_done(ct_future *f, int32_t *out_found, uint64_t *out_slot, } // namespace -// Case 1: a resident L1 hit's future is already done on -// the very first ct_future_poll call -- no reactor round trip needed. -TEST(AsyncGet, FastPathHitCompletesSynchronously) +// Resident values complete through the shared CPU runtime and retain their +// owner until the caller releases the future. +TEST(AsyncGet, ResidentHitCompletesWithAnOwnedValue) { ct_options opt = {}; ct_tree *t = nullptr; @@ -78,13 +78,11 @@ TEST(AsyncGet, FastPathHitCompletesSynchronously) ct_future *f = ct_get_async(t, reinterpret_cast<const uint8_t *>(k.data()), k.size()); ASSERT_NE(f, nullptr); - int32_t done = 0; int32_t found = 0; uint64_t slot = 0; ct_buf val = {}; - ct_status st = ct_future_poll(f, &done, &found, &slot, &val); + ct_status st = poll_until_done(f, &found, &slot, &val); ASSERT_EQ(st, 0); - EXPECT_EQ(done, 1) << "L1-resident hit must complete on the first poll"; EXPECT_EQ(found, 1); EXPECT_EQ(std::string(reinterpret_cast<char *>(val.data), val.len), "hello"); // zero-copy fast path: `val` may @@ -100,8 +98,7 @@ TEST(AsyncGet, FastPathHitCompletesSynchronously) // Regression for the above: ct_future_poll on a resolved kGet future must // NOT free it (unlike flush_async/snapshot_async) -- polling it again // before the caller's own ct_future_free is safe and idempotent, since the -// borrowed value's backing frame is kept alive by the future's own epoch -// guard the whole time. +// borrowed value's independent owner is kept alive by the future. TEST(AsyncGet, FastPathValueSurvivesRepeatedPollsUntilExplicitFree) { ct_options opt = {}; @@ -113,6 +110,7 @@ TEST(AsyncGet, FastPathValueSurvivesRepeatedPollsUntilExplicitFree) ct_future *f = ct_get_async(t, reinterpret_cast<const uint8_t *>(k.data()), k.size()); ASSERT_NE(f, nullptr); + ASSERT_EQ(poll_until_done(f, nullptr, nullptr, nullptr), 0); for (int i = 0; i < 3; ++i) { int32_t done = 0; int32_t found = 0; @@ -152,8 +150,16 @@ TEST(AsyncGet, MissAfterEvictionCompletesViaReactor) ASSERT_EQ(ct_snapshot(t, &durable), 0); EXPECT_EQ(durable, 20U); + const auto warm_key = make_key(5); + int32_t warm_found = 0; + ct_buf warm_value{}; + ASSERT_EQ(ct_get(t, reinterpret_cast<const uint8_t *>(warm_key.data()), warm_key.size(), &warm_found, nullptr, + &warm_value), + 0); + ASSERT_EQ(warm_found, 1); + ct_free_buf(&warm_value); uint64_t evicted = ct_evict_clean_leaves(t, 0); - EXPECT_GT(evicted, 0U) << "snapshot should have made every leaf clean and evictable"; + EXPECT_GT(evicted, 0U) << "the explicitly loaded clean leaf must be evictable"; std::string k = make_key(5); ct_future *f = ct_get_async(t, reinterpret_cast<const uint8_t *>(k.data()), k.size()); diff --git a/lib/crowdb-tree/tests/integration/async_scan_test.cpp b/lib/crowdb-tree/tests/integration/async_scan_test.cpp index 5ab1370d0..edad5ff61 100644 --- a/lib/crowdb-tree/tests/integration/async_scan_test.cpp +++ b/lib/crowdb-tree/tests/integration/async_scan_test.cpp @@ -96,10 +96,9 @@ std::map<std::string, std::string> unpack_entries(const ct_buf &buf, uint64_t co } // namespace -// Fast path: every leaf already resident -- the future is done on the very -// first poll, and the packed output matches ct_scan's own synchronous result -// byte for byte. -TEST(AsyncScan, FastPathAllResidentCompletesSynchronously) +// Resident results also use the shared runtime. Completion must return every +// packed entry without depending on which worker runs before the first poll. +TEST(AsyncScan, ResidentScanCompletesWithAllEntries) { ct_options opt = {}; ct_tree *t = nullptr; @@ -111,13 +110,11 @@ TEST(AsyncScan, FastPathAllResidentCompletesSynchronously) ct_future *f = ct_scan_async(t, nullptr, 0, nullptr, 0, nullptr, 0, 0, 0, 0, 0); ASSERT_NE(f, nullptr); - int32_t done = 0; int32_t truncated = 0; uint64_t count = 0; ct_buf entries = {}; - ct_status st = ct_future_poll(f, &done, &truncated, &count, &entries); + ct_status st = poll_scan_until_done(f, &truncated, &count, &entries); ASSERT_EQ(st, 0); - EXPECT_EQ(done, 1) << "an all-resident scan must complete on the first poll"; EXPECT_EQ(truncated, 0); ASSERT_EQ(count, 10U); @@ -192,8 +189,14 @@ TEST(AsyncScan, MissAfterEvictionCompletesViaReactor) ASSERT_EQ(ct_snapshot(t, &durable), 0); EXPECT_EQ(durable, 40U); + ct_buf warm_entries{}; + uint64_t warm_count = 0; + ASSERT_EQ(ct_scan(t, nullptr, 0, nullptr, 0, 0, 0, nullptr, 0, 0, 0, 0, 0, 0, &warm_entries, &warm_count, nullptr), + 0); + ASSERT_EQ(warm_count, 40U); + ct_free_buf(&warm_entries); uint64_t evicted = ct_evict_clean_leaves(t, 0); - EXPECT_GT(evicted, 0U) << "snapshot should have made every leaf clean and evictable"; + EXPECT_GT(evicted, 0U) << "the explicitly loaded clean leaves must be evictable"; ct_future *f = ct_scan_async(t, nullptr, 0, nullptr, 0, nullptr, 0, 0, 0, 0, 0); ASSERT_NE(f, nullptr); @@ -241,10 +244,8 @@ TEST(AsyncScan, FutureFreeBeforeCompletionDoesNotCrashOrLeak) ct_close(t); } -// Empty-tree scan_async resolves immediately with zero entries, not an -// infinite retry loop (root_page_id_ == kInvalidPageId is handled the same -// way scan() itself handles it -- no page to ever be "pending"). -TEST(AsyncScan, EmptyTreeResolvesImmediately) +// An empty scan must terminate without repeatedly scheduling an empty window. +TEST(AsyncScan, EmptyTreeResolvesWithoutRetryingForever) { ct_options opt = {}; ct_tree *t = nullptr; @@ -252,11 +253,9 @@ TEST(AsyncScan, EmptyTreeResolvesImmediately) ct_future *f = ct_scan_async(t, nullptr, 0, nullptr, 0, nullptr, 0, 0, 0, 0, 0); ASSERT_NE(f, nullptr); - int32_t done = 0; uint64_t count = 0; - ct_status st = ct_future_poll(f, &done, nullptr, &count, nullptr); + ct_status st = poll_scan_until_done(f, nullptr, &count, nullptr); ASSERT_EQ(st, 0); - EXPECT_EQ(done, 1); EXPECT_EQ(count, 0U); ct_close(t); diff --git a/lib/crowdb-tree/tests/integration/backing_stats_c_api_test.cpp b/lib/crowdb-tree/tests/integration/backing_stats_c_api_test.cpp new file mode 100644 index 000000000..4eba386d3 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/backing_stats_c_api_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/backing_stats_c_api.h" +#include "crowdb-tree/tree_work_metrics_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(BackingStatsCAbi, ClosedLocalTreeCollectorSamplesWithoutRetainingPagesOrBackend) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_page_store *store = nullptr; + ASSERT_EQ(ct_page_store_open_mem(1, &store), 0); + ct_options options{}; + options.page_store = store; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + auto *facade = c_api_tree(tree); + ASSERT_TRUE(facade->put("key", "value").ok()); + ASSERT_TRUE(facade->snapshot().ok()); + auto cpu = TreeCoreAccess::get(*facade)->binding()->cpu(); + cpu->drain(); + ct_work_metrics_collector *collector = nullptr; + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + ct_backing_stats_v1 value{}; + value.struct_size = sizeof(value); + value.version = 2; + value.allocated_bytes = 99; + EXPECT_EQ(ct_get_backing_stats(tree, &value), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(value.allocated_bytes, 99U); + value.version = 1; + --value.struct_size; + EXPECT_EQ(ct_work_metrics_collector_backing_stats(collector, &value), static_cast<int32_t>(Code::kInvalidArgument)); + ++value.struct_size; + EXPECT_EQ(ct_get_backing_stats(nullptr, &value), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_get_backing_stats(tree, nullptr), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_work_metrics_collector_backing_stats(nullptr, &value), static_cast<int32_t>(Code::kInvalidArgument)); + ASSERT_EQ(ct_get_backing_stats(tree, &value), 0); + ASSERT_NE(value.scope_id, 0U); + ASSERT_GT(value.allocated_bytes, 0U); + const auto scope = value.scope_id; + auto old = facade->snapshot_view(); + auto *device = c_api_page_store(tree); + ct_close(tree); + cpu->drain(); + ASSERT_EQ(ct_work_metrics_collector_backing_stats(collector, &value), 0); + EXPECT_EQ(value.scope_id, scope); + EXPECT_GT(value.allocated_bytes, 0U); + EXPECT_FALSE(device->try_claim_runtime_domain()); + old.reset(); + cpu->drain(); + EXPECT_TRUE(device->try_claim_runtime_domain()); + device->release_runtime_domain(); + ct_page_store_free(store); + cpu->drain(); + cpu.reset(); + ct_runtime_free(runtime); + ASSERT_EQ(ct_work_metrics_collector_backing_stats(collector, &value), 0); + EXPECT_EQ(value.scope_id, scope); + EXPECT_EQ(value.allocated_bytes, 0U); + ct_work_metrics_collector_free(collector); +} + +TEST(BackingStatsCAbi, VolatileTreeHasNoBackendScope) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + ct_backing_stats_v1 value{}; + value.struct_size = sizeof(value); + value.version = 1; + EXPECT_EQ(ct_get_backing_stats(tree, &value), static_cast<int32_t>(Code::kUnavailable)); + ct_close(tree); + ct_runtime_free(runtime); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/borrowed_read_budget_test.cpp b/lib/crowdb-tree/tests/integration/borrowed_read_budget_test.cpp new file mode 100644 index 000000000..4a2e7e757 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/borrowed_read_budget_test.cpp @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "btree/tree_core_read_internal.h" +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/crowdb-tree.h" +#include "snapshot/published_snapshot_entries.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(BorrowedReadBudget, InlineWritebackAndNewUnknownOverlaysAdmitConcurrentReads) +{ + MemPageStore store(1); + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = &store; + config.frame_bytes = 65536; + config.runtime_binding.limits.routing_bytes = 128 * 1024 * 1024; + // Each accepted read owns a loader ticket and backend admission together. + config.runtime_binding.limits.io_operations = 64; + Crowdbtree tree(config); + const std::string value(16384, 'v'); + ASSERT_TRUE(tree.apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = value}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + auto &core = *TreeCoreAccess::get(tree); + ASSERT_TRUE(core.checkpoint().ok()); + ASSERT_TRUE(tree.apply(2, Batch{.ops = {{.key = "b", .kind = OpKind::kPut, .value = value}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + core.binding()->drain(); + auto captured = std::make_shared<PublishedTreeState>(*core.capture()); + // A different, still-unmeasured leaf can coexist with the written base. + captured->stats.kv_unknown_pages = 1; + ASSERT_GT(captured->mapping.copy_bounds().max_leaf_bases, 0U); + ASSERT_GT(captured->stats.kv_unknown_pages, 0U); + std::array<core_read_detail::ReadPreparation, 16> accepted; + size_t readers = 0; + for (auto &read : accepted) { + const auto status = core_read_detail::prepare(core, captured, core.read_limits(), false, 0, 0, &read); + ASSERT_TRUE(status.ok()) << "readers=" << readers << ' ' << status.to_string(); + ++readers; + } +} + +TEST(BorrowedReadBudget, InlineOverlayValuesDoNotReserveAnOverflowAssemblyWindow) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.runtime_binding.limits.routing_bytes = 16 * 1024 * 1024; + Crowdbtree tree(config); + const std::string first(1024 * 1024, 'a'); + const std::string second(1024 * 1024, 'b'); + auto status = tree.apply(1, Batch{ + .ops = {{.key = "a", .kind = OpKind::kPut, .value = first}, + {.key = "b", .kind = OpKind::kPut, .value = second}} + }); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = tree.flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto &core = *TreeCoreAccess::get(tree); + core.binding()->drain(); + std::shared_ptr<PublishedSnapshotContext> snapshot; + status = core.capture_snapshot(&snapshot); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::shared_ptr<PublishedSnapshotEntries> cursor; + status = PublishedSnapshotEntries::create(snapshot, &cursor); + ASSERT_TRUE(status.ok()) << status.to_string(); + bool complete = false; + size_t entries = 0; + while (!complete) { + std::optional<PublishedScanEntry> entry; + status = cursor->next(&entry, &complete); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (entry) { + EXPECT_EQ(entry->cell.value.to_string(), entries == 0 ? first : second); + ++entries; + } + } + EXPECT_EQ(entries, 2U); + + GetView found; + status = snapshot->get(Slice("b"), &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), second); + ScanPackedBuf packed; + size_t count = 0; + bool truncated = false; + CoreScanOptions options; + options.limit = 2; + options.byte_budget = 3 * 1024 * 1024; + status = snapshot->scan(options, &packed, &count, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(count, 2U); + EXPECT_FALSE(truncated); + status = core.get(Slice("a"), &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), first); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/budgeted_flush_test.cpp b/lib/crowdb-tree/tests/integration/budgeted_flush_test.cpp new file mode 100644 index 000000000..8783342b9 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/budgeted_flush_test.cpp @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_operation.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(BudgetedFlush, WriterPayloadRemainsChargedAfterHandoffAndReleasesWithLastMappingOwner) +{ + detail::ResourceBudget::Limits limits; + limits.fill(4 * 1024 * 1024); + detail::ResourceBudget budget(limits, 4); + auto account = budget.make_account(limits); + constexpr auto writer_class = static_cast<size_t>(detail::BudgetClass::kAdmittedWriter); + { + const auto lifecycle_bound = SourceAdmission::allocation_bound(); + if (!lifecycle_bound) { + FAIL() << "source lifecycle bound overflow"; + } + auto lifecycle_credit = + detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kAdmittedWriter, *lifecycle_bound); + if (!lifecycle_credit) { + FAIL() << "source lifecycle credit unavailable"; + } + auto lifecycle = detail::AllocationReservation::create(std::move(*lifecycle_credit)); + SourceAdmission sources(0, 1, + {.account = account, + .max_entries = 16, + .max_key_bytes = 1024, + .max_gap_entries = 2, + .max_gap_key_bytes = 128}, + lifecycle); + PublicationCoordinator publisher; + auto published_sources = std::make_shared<PublishedSources>(); + published_sources->tables.push_back(sources.catalog()->active); + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{}); + request->transaction.pages.push_back({.page_id = 0, .expected = {}, .replacement = page}); + request->transaction.root_page_id = 0; + request->transaction.expected_sources = publisher.capture()->sources; + request->transaction.sources = std::move(published_sources); + request->on_done = [](PublicationResult result) { + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + }; + ASSERT_TRUE(publisher.try_submit(request)); + request.reset(); + SourceAdmission::Writer writer; + constexpr size_t payload_bytes = 8192; + ASSERT_TRUE(sources.try_admit(0, {8, 8, 8 * (payload_bytes + kCellHeaderSize)}, &writer).ok()); + std::weak_ptr<MemTable> old_source = writer.table(); + for (uint64_t slot = 1; slot <= 8; ++slot) { + ASSERT_TRUE(writer + .upsert(std::to_string(slot), slot, + encode_cell_buf(slot, OpKind::kPut, std::string(payload_bytes, 'v'))) + .ok()); + } + writer.release(); + EXPECT_EQ(budget.snapshot().reserved[writer_class], lifecycle->available()); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> frontier{8}; + FlushController controller(publisher, runtime, sources, frontier, lane); + auto reserved = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, 1024 * 1024); + if (!reserved) { + FAIL() << "test handoff credit unavailable"; + } + auto metadata = std::make_shared<detail::AllocationReservation>(std::move(*reserved)); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(controller.prepare(metadata, &flush).ok()); + ASSERT_TRUE(stdexec::sync_wait(flush->sender())); + runtime.drain(); + ASSERT_TRUE(flush->result().handoff_status.ok()) << flush->result().handoff_status.to_string(); + EXPECT_EQ(flush->result().published_coverage, 8U); + publisher.reclaim(); + sources.reclaim(); + EXPECT_TRUE(old_source.expired()); + EXPECT_GE(budget.snapshot().allocated[writer_class], 8 * payload_bytes); + // Retired source infrastructure returns its bytes to the still-live + // lifecycle owner. All separate writer reservations must be sealed. + EXPECT_EQ(budget.snapshot().reserved[writer_class], lifecycle->available()); + auto view = publisher.capture(); + EXPECT_EQ(view->mapping.get(0)->leaf->lookup("1").cell.value.size(), payload_bytes); + } + for (auto allocated : budget.snapshot().allocated) { + EXPECT_EQ(allocated, 0U); + } + for (auto reserved : budget.snapshot().reserved) { + EXPECT_EQ(reserved, 0U); + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/c_api_test.cpp b/lib/crowdb-tree/tests/integration/c_api_test.cpp index 52abb7e83..2aded705a 100644 --- a/lib/crowdb-tree/tests/integration/c_api_test.cpp +++ b/lib/crowdb-tree/tests/integration/c_api_test.cpp @@ -16,6 +16,7 @@ #include <vector> #ifdef __linux__ +# include <poll.h> # include <unistd.h> #endif @@ -101,6 +102,9 @@ TEST(CApi, InjectedStoreRetainedAndFutureSignalsCompletionFd) int32_t fd = -1; ASSERT_GE(ct_uring_eventfds(t, &fd, 1), 1U); ASSERT_GE(fd, 0); + pollfd ready{.fd = fd, .events = POLLIN, .revents = 0}; + ASSERT_EQ(::poll(&ready, 1, 2000), 1); + ASSERT_NE(ready.revents & POLLIN, 0); uint64_t completions = 0; ASSERT_EQ(::read(fd, &completions, sizeof(completions)), static_cast<ssize_t>(sizeof(completions))); EXPECT_EQ(completions, 1U); diff --git a/lib/crowdb-tree/tests/integration/captured_frame_window_test.cpp b/lib/crowdb-tree/tests/integration/captured_frame_window_test.cpp new file mode 100644 index 000000000..9f916593d --- /dev/null +++ b/lib/crowdb-tree/tests/integration/captured_frame_window_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "load_budget_test_support.h" +#include "published_read_test_support.h" +#include "snapshot/captured_frames.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +TEST(CapturedFrameWindow, WarmAndColdOversizedLeavesUseExactPrepaidDescriptorBound) +{ + for (bool cold_base : {false, true}) { + auto logical = std::make_shared<PageState>(); + if (cold_base) { + logical->bases.push_back(cold(1)); + } + const auto run = page({ + {.key = "z", .value = "overlay"} + }) + ->overlays.front(); + logical->overlays.assign(8192, run); + auto captured = std::make_shared<PublishedTreeState>(*view(logical, {}, 10)); + captured->next_page_id = 1; + LoadCredits credits; + credits.credits.bytes->seal(); + CapturedFrameLimits limits; + limits.max_depth = 1; + limits.page_metadata_bytes = captured->mapping.copy_bounds().max_leaf_bytes; + limits.encode.frame_bytes = 512; + limits.encode.max_sources = 1; + limits.encode.max_entries = 2; + limits.encode.max_work_entries = 8; + limits.encode.max_metadata_bytes = 4096; + const auto bound = CapturedFrameStream::allocation_bound(limits); + if (!bound) { + FAIL() << "captured descriptor allocation bound overflow"; + } + limits.encode.metadata = credits.reserve(detail::BudgetClass::kRouting, *bound); + auto loader = std::make_shared<TestLoader>(); + loader->frames[1] = base({ + {.key = "a", .value = "cold"} + }); + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<CapturedFrameStream> stream; + ASSERT_TRUE(CapturedFrameStream::create(captured, loader, runtime, lane, limits, {}, &stream).ok()); + Completion done; + auto operation = stdexec::connect(stream->sender(), Receiver{.done = &done}); + stdexec::start(operation); + runtime->drain(); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + const auto &frame = stream->result().frame; + if (!frame) { + FAIL() << "snapshot did not emit a primary frame"; + } + const auto &summary = frame->summary; + if (!summary) { + FAIL() << "primary frame lacks its final summary"; + } + EXPECT_EQ(summary->encoded_entries, cold_base ? 2U : 1U); + EXPECT_EQ(loader->calls, cold_base ? 1U : 0U); + EXPECT_TRUE(stream->result().complete); + } +} + +TEST(CapturedFrameWindow, DescriptorGeometryOverflowRejectsBeforeAllocation) +{ + CapturedFrameLimits limits; + limits.page_metadata_bytes = SIZE_MAX; + EXPECT_FALSE(CapturedFrameStream::allocation_bound(limits)); + limits.page_metadata_bytes = 0; + limits.encode.max_sources = SIZE_MAX; + EXPECT_FALSE(CapturedFrameStream::allocation_bound(limits)); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/captured_frames_test.cpp b/lib/crowdb-tree/tests/integration/captured_frames_test.cpp new file mode 100644 index 000000000..f3c5c4bdc --- /dev/null +++ b/lib/crowdb-tree/tests/integration/captured_frames_test.cpp @@ -0,0 +1,648 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "inner_load_test_support.h" +#include "load_budget_test_support.h" +#include "snapshot/captured_frames.h" + +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +struct TestStream +{ + std::shared_ptr<detail::FlushRuntime> runtime = std::make_shared<detail::FlushRuntime>(1, 8); + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<CapturedFrameStream> stream; + + [[nodiscard]] const CapturedFrame &frame() const + { + const auto &value = stream->result().frame; + if (!value) { + throw std::runtime_error("snapshot did not emit a frame"); + } + return *value; + } + + Status create(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<PageLoader> loader = {}, + CapturedFrameLimits limits = {}, CapturedFrameStream::ReserveOverflow reserve = {}) + { + return CapturedFrameStream::create(std::move(captured), std::move(loader), runtime, lane, std::move(limits), + std::move(reserve), &stream); + } + + void next(Completion *done) + { + auto operation = stdexec::connect(stream->sender(), Receiver{.done = done}); + stdexec::start(operation); + runtime->drain(); + EXPECT_EQ(done->values.load() + done->errors.load() + done->stopped.load(), 1U); + } +}; + +std::shared_ptr<PublishedTreeState> snapshot_leaf(std::shared_ptr<const PageState> page, uint64_t coverage = 10) +{ + auto result = std::make_shared<PublishedTreeState>(*view(std::move(page), {}, coverage)); + result->next_page_id = 1; + return result; +} + +std::shared_ptr<PublishedTreeState> snapshot_tree(size_t leaves, size_t value_bytes = 1) +{ + auto result = std::make_shared<PublishedTreeState>(); + std::vector<PageReplacement> changes; + auto index = std::make_shared<InnerPageState>(); + for (size_t i = 0; i < leaves; ++i) { + const auto key = std::string(4 - std::to_string(i).size(), '0') + std::to_string(i); + const auto upper = std::string(4 - std::to_string(i + 1).size(), '0') + std::to_string(i + 1); + PageKeyRange range; + if (i != 0) { + range.lower = key; + index->separators.emplace_back(key); + } + if (i + 1 != leaves) { + range.upper = upper; + } + index->children.push_back(i); + changes.push_back({ + .page_id = i, + .expected = { }, + .replacement = published(page({{.key = key, .value = std::string(value_bytes, 'v')}}, + range)) + }); + } + auto root = std::make_shared<PublishedPage>(); + root->level = 1; + root->metrics.leaf = false; + root->inner = index; + changes.push_back({.page_id = leaves, .expected = {}, .replacement = root}); + MappingView::CopyStats copied; + result->mapping = result->mapping.replace(changes, copied); + result->root_page_id = leaves; + result->next_page_id = leaves + 1; + result->coverage = 10; + return result; +} + +TEST(CapturedFrames, MaterializationBytesFollowTheLastOutputOwnerAfterStreamDestruction) +{ + LoadCredits credits; + TestStream test; + ASSERT_TRUE(test.create(snapshot_leaf(page({ + {.key = "a", .value = "captured"} + })), + {}, {.encode = {.frame_bytes = 4096, .metadata = credits.credits.bytes}}) + .ok()); + Completion done; + test.next(&done); + ASSERT_EQ(done.values.load(), 1U); + auto retained = test.frame().buffer; + ASSERT_TRUE(retained); + EXPECT_EQ(credits.resources->snapshot().materialize_inflight_bytes, retained->bytes.capacity()); + EXPECT_EQ(retained->bytes.capacity(), 4096U); + test.stream.reset(); + test.runtime->drain(); + credits.seal(); + EXPECT_EQ(credits.resources->snapshot().materialize_inflight_bytes, 4096U); + retained.reset(); + EXPECT_EQ(credits.resources->snapshot().materialize_inflight_bytes, 0U); +} + +TEST(CapturedFrames, CapturedMappingSurvivesUnvisitedReplacementAndFiltersFutureVersions) +{ + auto captured = snapshot_tree(2); + PublicationCoordinator publisher; + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{}); + for (uint64_t pid = 0; pid < 3; ++pid) { + request->transaction.pages.push_back( + {.page_id = pid, .expected = {}, .replacement = captured->mapping.get(pid)}); + } + request->transaction.root_page_id = 2; + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 10; + sources->tables.push_back(std::make_shared<MemTable>()); + request->transaction.expected_sources = publisher.capture()->sources; + request->transaction.sources = sources; + request->transaction.coverage = 10; + request->on_done = [](auto result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(publisher.try_submit(request)); + auto capture = std::make_shared<PublicationRequest>(); + capture->transaction.record = std::make_shared<InstallRecord>(InstallToken{}); + capture->transaction.expected_coverage = 10; + capture->transaction.checkpoint = {.action = CheckpointAction::Capture, .token = 1}; + std::shared_ptr<const PublishedTreeState> fixed; + capture->on_done = [&](auto result) { + EXPECT_TRUE(result.status.ok()); + fixed = std::move(result.checkpoint_view); + }; + ASSERT_TRUE(publisher.try_submit(capture)); + ASSERT_TRUE(fixed); + EXPECT_TRUE(fixed->sources->tables.empty()); + TestStream test; + ASSERT_TRUE(test.create(fixed, {}, {.encode = {.frame_bytes = 512}}).ok()); + Completion first; + test.next(&first); + ASSERT_EQ(first.values.load(), 1U); + ASSERT_TRUE(test.stream->result().frame); + EXPECT_EQ(test.frame().info.page_id, 0U); + auto replace = std::make_shared<PublicationRequest>(); + replace->transaction.record = std::make_shared<InstallRecord>(InstallToken{}); + replace->transaction.pages.push_back({ + .page_id = 1, + .expected = publisher.capture()->mapping.get(1), + .replacement = published(page({{.key = "0001", .slot = 11, .value = "new"}}, + {.lower = "0001" } + )) + }); + replace->on_done = [](auto result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(publisher.try_submit(replace)); + Completion second; + test.next(&second); + ASSERT_EQ(second.values.load(), 1U); + ASSERT_TRUE(test.stream->result().frame); + const auto &bytes = test.frame().buffer->bytes; + LeafFrameView leaf(bytes.data(), bytes.size()); + ASSERT_EQ(leaf.count(), 1U); + EXPECT_EQ(CellView(leaf.cell(0)).value().to_string(), "v"); + EXPECT_EQ(CellView(leaf.cell(0)).slot(), 1U); + Completion root; + test.next(&root); + ASSERT_EQ(root.values.load(), 1U); + EXPECT_EQ(test.frame().info.page_id, 2U); + EXPECT_TRUE(test.stream->result().complete); + EXPECT_EQ(publisher.capture()->mapping.get(1)->leaf->lookup("0001").cell.slot, 11U); +} + +TEST(CapturedFrames, ColdIndexTraversalLoadsOnDemandWithOneCpuWorker) +{ + TestColdTree cold_tree; + cold_tree.view->coverage = 10; + cold_tree.view->next_page_id = 7; + cold_tree.store->deferred = true; + TestStream test; + auto loader = std::make_shared<AsyncStorePageLoader>(cold_tree.store, test.runtime, test.lane); + ASSERT_TRUE(test.create(cold_tree.view, loader, {.encode = {.frame_bytes = 512}}).ok()); + std::vector<uint64_t> ids; + for (size_t i = 0; i < 7; ++i) { + Completion done; + auto operation = stdexec::connect(test.stream->sender(), Receiver{.done = &done}); + stdexec::start(operation); + test.runtime->drain(); + for (size_t load = 0; load < 3 && done.values.load() + done.errors.load() == 0; ++load) { + ASSERT_FALSE(cold_tree.store->pending.empty()); + cold_tree.store->finish(); + test.runtime->drain(); + } + ASSERT_EQ(done.values.load(), 1U) << test.stream->result().status.to_string(); + ASSERT_TRUE(test.stream->result().frame); + const auto &frame = test.frame(); + ids.push_back(frame.info.page_id); + EXPECT_TRUE(frame_validate(frame.buffer->bytes.data(), frame.info.bytes)); + } + EXPECT_EQ(ids, (std::vector<uint64_t>{3, 4, 1, 5, 6, 2, 0})); + EXPECT_EQ(cold_tree.store->reads.load(), 3U); + EXPECT_EQ(test.stream->stats().loads, 3U); + EXPECT_TRUE(test.stream->result().complete); + EXPECT_FALSE(cold_tree.view->mapping.get(0)->inner); +} + +TEST(CapturedFrames, PinnedOutputAppliesBackpressureWithoutOverwritingOrAdvancing) +{ + TestStream test; + ASSERT_TRUE(test.create(snapshot_tree(2), {}, {.encode = {.frame_bytes = 512}}).ok()); + Completion first; + test.next(&first); + ASSERT_EQ(first.values.load(), 1U); + auto pinned = std::make_unique<CapturedFrame>(test.frame()); + ASSERT_TRUE(pinned); + const auto crc = frame_u32(pinned->buffer->bytes.data(), pinned->info.bytes - kFrameTrailerSize); + Completion rejected; + test.next(&rejected); + EXPECT_EQ(rejected.errors.load(), 1U); + EXPECT_EQ(test.stream->result().status.code(), Code::kUnavailable); + EXPECT_EQ(frame_u32(pinned->buffer->bytes.data(), pinned->info.bytes - kFrameTrailerSize), crc); + EXPECT_EQ(test.stream->stats().frames, 1U); + pinned.reset(); + Completion second; + test.next(&second); + ASSERT_EQ(second.values.load(), 1U); + EXPECT_EQ(test.frame().info.page_id, 1U); +} + +TEST(CapturedFrames, OverflowStreamsThroughOneWindowWithOwnedDisjointIdentities) +{ + TestStream test; + const std::string value(4096, 'x'); + auto captured = snapshot_leaf(page({ + {.key = "a", .value = value}, + {.key = "b", .slot = 11, .value = "future"} + })); + auto owner = std::make_shared<int>(0); + uint64_t next = captured->next_page_id; + ASSERT_TRUE(test.create(captured, + { + }, + {.encode = {.frame_bytes = 256, .inline_value_bytes = 8}}, + [owner, &next](uint64_t count, ReservedOverflowIds *out) { + *out = {.first = next, .count = count, .owner = owner}; + next += count; + return Status::Ok(); + }) + .ok()); + const uint8_t *address = nullptr; + std::string restored; + for (size_t i = 0; i < 100; ++i) { + Completion done; + test.next(&done); + ASSERT_EQ(done.values.load(), 1U) << test.stream->result().status.to_string(); + ASSERT_TRUE(test.stream->result().frame); + const auto &frame = test.frame(); + if (address == nullptr) { + address = frame.buffer->bytes.data(); + } + EXPECT_EQ(frame.buffer->bytes.data(), address); + EXPECT_TRUE(frame_validate(address, frame.info.bytes)); + if (frame.info.type == page_type::kOverflowFrame) { + OverflowFrameView overflow(address, frame.info.bytes); + restored += overflow.payload().to_string(); + } + else { + LeafFrameView leaf(address, frame.info.bytes); + ASSERT_EQ(leaf.count(), 1U); + EXPECT_EQ(CellView(leaf.cell(0)).overflow_head(), 1U); + EXPECT_EQ(CellView(leaf.cell(0)).overflow_len(), value.size()); + } + if (test.stream->result().complete) { + break; + } + } + EXPECT_TRUE(test.stream->result().complete); + EXPECT_EQ(restored, value); + EXPECT_EQ(test.stream->stats().encoding.overflow_value_copy_bytes, value.size()); + EXPECT_EQ(test.stream->stats().encoding.primary_frames, 1U); +} + +TEST(CapturedFrames, CancelledColdLeafRetainsCallbackOwnershipAndDoesNotProduceLateFrames) +{ + LoadCredits credits; + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(cold(1)); + auto loader = std::make_shared<TestLoader>(); + loader->deferred = true; + loader->frames[1] = base({ + {.key = "a", .value = "value"} + }); + TestStream test; + ASSERT_TRUE(test.create(snapshot_leaf(leaf), loader, + { + .encode = {.frame_bytes = 512, .metadata = credits.credits.bytes} + }) + .ok()); + Completion done; + auto operation = stdexec::connect(test.stream->sender(), Receiver{.done = &done}); + stdexec::start(operation); + test.runtime->drain(); + EXPECT_EQ(loader->calls, 1U); + EXPECT_EQ(done.values.load(), 0U); + test.stream->request_stop(); + test.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + test.stream.reset(); + credits.seal(); + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + loader->finish(); + test.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_EQ(done.values.load(), 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(CapturedFrames, OutputMemoryIsBoundedAcrossManyPagesUnderExhaustedHostBudget) +{ + LoadCredits credits; + TestStream test; + ASSERT_TRUE(test.create(snapshot_tree(80), + { + }, + {.encode = {.frame_bytes = 4096, .max_entries = 128, .metadata = credits.credits.bytes}}) + .ok()); + size_t peak = 0; + size_t outputs = 0; + for (size_t i = 0; i < 81; ++i) { + Completion done; + test.next(&done); + ASSERT_EQ(done.values.load(), 1U) << test.stream->result().status.to_string(); + ASSERT_TRUE(test.stream->result().frame); + ++outputs; + peak = std::max(peak, credits.budget.snapshot().allocated[LoadCredits::kBytes]); + } + EXPECT_EQ(outputs, 81U); + EXPECT_TRUE(test.stream->result().complete); + EXPECT_LT(peak, outputs * 4096 / 2); + RecordProperty("peak_stream_allocated_bytes", static_cast<int>(peak)); + RecordProperty("encoded_output_bytes", static_cast<int>(outputs * 4096)); + test.stream.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(CapturedFrames, RejectsPartialCaptureAndOversizedLayoutBeforeEmittingPage) +{ + TestStream test; + auto captured = snapshot_leaf(page({ + {.key = "a", .value = std::string(500, 'v')} + })); + captured->pending_handoff = {.batch = 1, .frontier = 11}; + EXPECT_EQ(test.create(captured).code(), Code::kUnavailable); + captured->pending_handoff.reset(); + ASSERT_TRUE(test.create(captured, + { + }, + {.encode = {.frame_bytes = 256, .inline_value_bytes = 1000}}) + .ok()); + Completion done; + test.next(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.stream->result().status.code(), Code::kResourceExhausted); + EXPECT_FALSE(test.stream->result().frame); + EXPECT_EQ(test.stream->stats().frames, 0U); +} + +TEST(CapturedFrames, RuntimeShutdownWhileLoadingReturnsErrorWithoutLateOutput) +{ + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(cold(1)); + auto loader = std::make_shared<TestLoader>(); + loader->deferred = true; + loader->frames[1] = base({ + {.key = "a", .value = "value"} + }); + TestStream test; + ASSERT_TRUE(test.create(snapshot_leaf(leaf), loader).ok()); + Completion done; + auto operation = stdexec::connect(test.stream->sender(), Receiver{.done = &done}); + stdexec::start(operation); + test.runtime->drain(); + ASSERT_EQ(loader->calls, 1U); + test.runtime->close(); + loader->finish(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_FALSE(test.stream->result().frame); + EXPECT_EQ(test.stream->stats().frames, 0U); +} + +TEST(CapturedFrames, GcEmptyLeavesDoNotBecomeInnerFenceEndpoints) +{ + auto captured = snapshot_tree(3); + captured->gc_floor = 10; + MappingView::CopyStats copied; + const std::array<PageReplacement, 2> changes{ + {{.page_id = 0, + .expected = {}, + .replacement = + published(page({{.key = "0000", .slot = 2, .value = "", .tombstone = true}}, {.upper = "0001"}))}, + {.page_id = 2, + .expected = {}, + .replacement = + published(page({{.key = "0002", .slot = 2, .value = "", .tombstone = true}}, {.lower = "0002"}))}} + }; + captured->mapping = captured->mapping.replace(changes, copied); + TestStream test; + ASSERT_TRUE(test.create(captured, {}, {.encode = {.frame_bytes = 512}}).ok()); + for (size_t i = 0; i < 4; ++i) { + Completion done; + test.next(&done); + ASSERT_EQ(done.values.load(), 1U) << test.stream->result().status.to_string(); + } + const auto &frame = test.frame(); + EXPECT_EQ(frame.info.page_id, 3U); + EXPECT_EQ(frame_lower_fence_page_id(frame.buffer->bytes.data()), 1U); + EXPECT_EQ(frame_upper_fence_page_id(frame.buffer->bytes.data()), 1U); +} + +TEST(CapturedFrames, ReusedOverflowDependencyOutlivesStreamAndCapturedInput) +{ + auto logical = std::make_shared<PageState>(); + auto backing = std::make_shared<BackingRef>(); + auto protection = std::make_shared<int>(1); + std::weak_ptr<int> weak = protection; + backing->owner = protection; + backing->length = 4096; + std::vector<leaf_entry> entries; + entries.push_back({.key = "a", .cell = encode_overflow_cell_buf(2, 40, 1000)}); + auto base = std::shared_ptr<const LeafBase>(LeafBase::build(entries)); + logical->bases.push_back({.identity = 1, .resident = base, .backing = backing}); + auto captured = snapshot_leaf(logical); + captured->next_page_id = 100; + TestStream test; + ASSERT_TRUE(test.create(captured, {}, {.encode = {.frame_bytes = 512}}).ok()); + Completion done; + test.next(&done); + ASSERT_EQ(done.values.load(), 1U); + auto frame = std::make_unique<CapturedFrame>(test.frame()); + ASSERT_EQ(frame->reused_overflow.size(), 1U); + EXPECT_EQ(frame->reused_overflow[0].head, 40U); + EXPECT_EQ(frame->reused_overflow[0].bytes, 1000U); + EXPECT_EQ(test.stream->stats().encoding.overflow_frames, 0U); + test.stream.reset(); + captured.reset(); + logical.reset(); + backing.reset(); + protection.reset(); + EXPECT_FALSE(weak.expired()); + EXPECT_EQ(frame->reused_overflow[0].head, 40U); + frame.reset(); + EXPECT_TRUE(weak.expired()); +} + +TEST(CapturedFrames, ColdLeafFailureAndInvalidSlotProofNeverBecomeEmptySuccess) +{ + for (const bool io_failure : {false, true}) { + auto logical = std::make_shared<PageState>(); + logical->bases.push_back(cold(1, 2)); + auto loader = std::make_shared<TestLoader>(); + loader->deferred = true; + loader->frames[1] = base({ + {.key = "a", .slot = 3, .value = "value"} + }); + TestStream test; + ASSERT_TRUE(test.create(snapshot_leaf(logical), loader).ok()); + Completion done; + auto operation = stdexec::connect(test.stream->sender(), Receiver{.done = &done}); + stdexec::start(operation); + test.runtime->drain(); + loader->finish(io_failure ? Status::io_error() : Status::Ok()); + test.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.stream->result().status.code(), io_failure ? Code::kIoError : Code::kCorruption); + EXPECT_FALSE(test.stream->result().frame); + EXPECT_FALSE(test.stream->result().complete); + EXPECT_EQ(test.stream->stats().frames, 0U); + } +} + +TEST(CapturedFrames, ReusedNewOverflowIntervalFailsBeforeItCanOverwriteEarlierFrames) +{ + auto captured = snapshot_tree(2, 512); + auto owner = std::make_shared<int>(0); + TestStream test; + ASSERT_TRUE(test.create(captured, + { + }, + {.encode = {.frame_bytes = 256, .inline_value_bytes = 8}}, + [owner](uint64_t count, ReservedOverflowIds *out) { + *out = {.first = 10, .count = count, .owner = owner}; + return Status::Ok(); + }) + .ok()); + size_t errors = 0; + for (size_t i = 0; i < 16; ++i) { + Completion done; + test.next(&done); + if (done.errors.load() != 0) { + ++errors; + break; + } + ASSERT_EQ(done.values.load(), 1U); + } + EXPECT_EQ(errors, 1U); + EXPECT_EQ(test.stream->result().status.code(), Code::kInvalidArgument); + EXPECT_FALSE(test.stream->result().frame); + EXPECT_EQ(test.stream->stats().encoding.primary_frames, 1U); +} + +struct TestNextReceiver +{ + using receiver_concept = stdexec::receiver_tag; + detail::InlineFunction<void()> next; + detail::InlineFunction<void(Status)> error; + + void set_value() const noexcept + { + try { + next(); + } + catch (const std::exception &error) { + ADD_FAILURE() << error.what(); + } + catch (...) { + ADD_FAILURE() << "stream receiver threw"; + } + } + + void set_error(Status status) const noexcept + { + try { + if (error) { + error(std::move(status)); + } + else { + ADD_FAILURE() << status.to_string(); + } + } + catch (...) { + ADD_FAILURE() << "stream error receiver threw"; + } + } + + void set_stopped() const noexcept + { + ADD_FAILURE() << "unexpected stream cancellation"; + } +}; + +struct TestNextOperation +{ + TestNextOperation(detail::CallbackSender sender, TestNextReceiver receiver) + : operation(stdexec::connect(sender, std::move(receiver))) + { + } + + stdexec::connect_result_t<detail::CallbackSender, TestNextReceiver> operation; +}; + +TEST(CapturedFrames, ReceiverCanRequestNextAndDropLastHandleInsideCompletion) +{ + TestStream test; + ASSERT_TRUE(test.create(snapshot_tree(64), {}, {.encode = {.frame_bytes = 4096}}).ok()); + std::vector<std::unique_ptr<TestNextOperation>> operations; + operations.reserve(65); + size_t completed = 0; + for (size_t i = 0; i < 65; ++i) { + operations.push_back(std::make_unique<TestNextOperation>( + test.stream->sender(), TestNextReceiver{.next = + [&] { + EXPECT_EQ(test.frame().info.page_id, completed); + ++completed; + if (completed == 65) { + EXPECT_TRUE(test.stream->result().complete); + test.stream.reset(); + } + else { + stdexec::start(operations[completed]->operation); + } + }, + .error = {}})); + } + stdexec::start(operations[0]->operation); + test.runtime->drain(); + EXPECT_EQ(completed, 65U); + EXPECT_FALSE(test.stream); +} + +TEST(CapturedFrames, ReentrantErrorReceiverCannotTurnFailedColdLoadIntoEmptyCompletion) +{ + auto logical = std::make_shared<PageState>(); + logical->bases.push_back(cold(1)); + auto loader = std::make_shared<TestLoader>(); + loader->deferred = true; + loader->frames[1] = base({ + {.key = "a", .value = "value"} + }); + TestStream test; + ASSERT_TRUE(test.create(snapshot_leaf(logical), loader).ok()); + std::vector<std::unique_ptr<TestNextOperation>> operations; + operations.reserve(2); + size_t errors = 0; + for (size_t i = 0; i < 2; ++i) { + operations.push_back(std::make_unique<TestNextOperation>( + test.stream->sender(), TestNextReceiver{.next = [] { ADD_FAILURE() << "failed stream returned success"; }, + .error = + [&](Status status) { + EXPECT_EQ(status.code(), Code::kIoError); + ++errors; + if (errors == 1) { + stdexec::start(operations[1]->operation); + } + }})); + } + stdexec::start(operations[0]->operation); + test.runtime->drain(); + loader->finish(Status::io_error()); + test.runtime->drain(); + EXPECT_EQ(errors, 2U); + EXPECT_FALSE(test.stream->result().frame); + EXPECT_FALSE(test.stream->result().complete); +} + +TEST(CapturedFrames, RejectedNextAfterSuccessfulFrameClearsPreviousOutput) +{ + TestStream test; + ASSERT_TRUE(test.create(snapshot_tree(2)).ok()); + Completion first; + test.next(&first); + ASSERT_EQ(first.values.load(), 1U); + ASSERT_TRUE(test.stream->result().frame); + test.runtime->close(); + Completion failed; + test.next(&failed); + EXPECT_EQ(failed.errors.load(), 1U); + EXPECT_FALSE(test.stream->result().frame); + EXPECT_FALSE(test.stream->result().complete); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/captured_write_backend_test.cpp b/lib/crowdb-tree/tests/integration/captured_write_backend_test.cpp new file mode 100644 index 000000000..a6f940a98 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/captured_write_backend_test.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "captured_write_test_support.h" +#include "test_tmp.h" + +#include <future> + +namespace crowdb::tree +{ +using namespace test_write; + +#ifdef CROWDB_HAVE_LIBURING +TEST(CapturedWrite, RealBlockReopenReloadsPrimaryAndOverflowThroughItsCapturedMapping) +{ + crowdb::tree_test::TempDir dir("captured_write_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 3, 4096, 4096, &file).ok()); + crowdb::common::Topology topology; + topology.pipelines.emplace_back().entries = 64; + crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + auto store = std::make_shared<BlockAsyncPageStore>(file.get(), &uring); + TestCapturedWrite test; + const std::string value(3000, 'b'); + uint64_t next = 1; + ASSERT_TRUE( + CapturedWriteOperation::create( + captured_leaf(page({ + {.key = "a", .value = value} + })), + {}, store, test.runtime, test.lane, + {.frames = {.encode = {.frame_bytes = 512, .inline_value_bytes = 16}}, .write = {.max_frame_bytes = 512}}, + test.credits.credits, + [owner = test.ledger, &next](uint64_t count, ReservedOverflowIds *out) { + *out = {.first = next, .count = count, .owner = owner}; + next += count; + return Status::Ok(); + }, + extent_write_reservation(test.ledger, test.credits.credits.bytes), {}, &test.operation) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); + test.runtime->drain(); + auto pages = test.operation->result().pages; + ASSERT_TRUE(pages); + test.operation.reset(); + ASSERT_TRUE(file->sync().ok()); + store.reset(); + file.reset(); + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 3, 4096, 4096, &file).ok()); + store = std::make_shared<BlockAsyncPageStore>(file.get(), &uring); + auto loader = std::make_shared<AsyncStorePageLoader>( + store, test.runtime, test.lane, + PageLoadLimits{.max_inflight = 1, .max_blob_bytes = 4096, .max_frame_bytes = 512}, + [pages](uint64_t pid, std::shared_ptr<const BackingRef> *out) { + auto frame = pages->frames.find(pid); + if (frame == pages->frames.end()) { + return Status::not_found(); + } + *out = frame->second.backing; + return Status::Ok(); + }, + test.credits.credits); + std::promise<std::shared_ptr<const LeafBase>> leaf_result; + loader->load_leaf({.identity = 1, .backing = pages->frames.at(0).backing}, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()) << status.to_string(); + leaf_result.set_value(std::move(leaf)); + }); + auto leaf = leaf_result.get_future().get(); + ASSERT_TRUE(leaf); + const CellView cell(leaf->cell(0)); + std::promise<std::shared_ptr<const buffer>> value_result; + loader->load_overflow(cell.overflow_head(), cell.overflow_len(), [&](Status status, auto restored) { + EXPECT_TRUE(status.ok()) << status.to_string(); + value_result.set_value(std::move(restored)); + }); + auto restored = value_result.get_future().get(); + ASSERT_TRUE(restored); + EXPECT_EQ(Slice(restored->data(), restored->size()).to_string(), value); + test.runtime->drain(); + leaf.reset(); + restored.reset(); + pages.reset(); + loader.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +#endif +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/captured_write_dependency_lifetime_test.cpp b/lib/crowdb-tree/tests/integration/captured_write_dependency_lifetime_test.cpp new file mode 100644 index 000000000..b5beb3870 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/captured_write_dependency_lifetime_test.cpp @@ -0,0 +1,184 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/overflow_backing.h" +#include "captured_write_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_write; + +TEST(CapturedWriteDependencies, ReusingOneValueDoesNotKeepTheDeletedValueFromTheOldLeafAlive) +{ + TestCapturedWrite test; + const std::string kept_value(3000, 'k'); + const std::string deleted_value(4000, 'd'); + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = kept_value }, + {.key = "b", .value = deleted_value} + }))) + .ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U) << first.status.to_string(); + auto old = test.operation->result().pages; + ASSERT_NE(old, nullptr); + const auto raw = test.read(old->frames.at(0)); + const LeafFrameView original(raw.data(), raw.size()); + ASSERT_EQ(original.count(), 2U); + const CellView kept(original.cell(0)); + const CellView deleted(original.cell(1)); + std::vector<std::weak_ptr<const void>> deleted_owners; + for (auto pid = deleted.overflow_head(); pid != kInvalidPageId;) { + const auto &frame = old->frames.at(pid); + deleted_owners.emplace_back(frame.backing->physical_owner); + pid = frame.next_overflow; + } + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back( + {.identity = 1, + .max_slot = 10, + .resident = std::shared_ptr<const LeafBase>(LeafBase::from_frame_copy(raw.data(), raw.size())), + .backing = old->frames.at(0).backing}); + const auto delta = page({ + {.key = "b", .slot = 10, .value = "", .tombstone = true} + }); + leaf->overlays.push_back(delta->overlays.front()); + auto next = captured_leaf(leaf); + next->next_page_id = old->next_page_id; + test.operation.reset(); + ASSERT_TRUE(test.prepare(next, old).ok()); + Completion second; + test.run(&second); + ASSERT_EQ(second.values.load(), 1U) << second.status.to_string(); + auto result = test.operation->result(); + ASSERT_NE(result.pages, nullptr); + EXPECT_EQ(result.written_frames, 1U); + const auto updated = test.read(result.pages->frames.at(0)); + const LeafFrameView current(updated.data(), updated.size()); + ASSERT_EQ(current.count(), 2U); + EXPECT_TRUE(CellView(current.cell(1)).is_tombstone()); + EXPECT_LT(result.pages->frames.size(), old->frames.size()); + test.operation.reset(); + next.reset(); + leaf.reset(); + old.reset(); + for (const auto &owner : deleted_owners) { + EXPECT_TRUE(owner.expired()) << "the retained chain still owns the deleted value's extent"; + } + deleted_owners.clear(); + EXPECT_EQ(test.ledger->stats().protected_ranges, result.pages->frames.size()); + std::string restored; + for (auto pid = kept.overflow_head(); pid != kInvalidPageId;) { + const auto bytes = test.read(result.pages->frames.at(pid)); + const OverflowFrameView chunk(bytes.data(), bytes.size()); + restored += chunk.payload().to_string(); + pid = chunk.next_page_id(); + } + EXPECT_EQ(restored, kept_value); + result = {}; + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +std::shared_ptr<const PublishedTreeState> shared_chain_input(const CapturedDurablePages &old, Slice encoded) +{ + std::vector<leaf_entry> entries; + entries.push_back({.key = "a", .cell = buffer::copy_of(encoded)}); + entries.push_back({.key = "c", .cell = buffer::copy_of(encoded)}); + auto shared = std::shared_ptr<const LeafBase>(LeafBase::build(entries)); + const auto right_pid = old.next_page_id; + const auto root_pid = right_pid + 1; + auto left = std::make_shared<PageState>(*page({ + {.key = "b", .slot = 10, .value = std::string(4000, 'b')} + })); + left->range.upper = "c"; + left->right_sibling = right_pid; + left->bases.push_back( + {.identity = 1, .range = left->range, .max_slot = 10, .resident = shared, .backing = old.frames.at(0).backing}); + auto right = std::make_shared<PageState>(); + right->range.lower = "c"; + right->bases.push_back({.identity = 1, + .range = right->range, + .max_slot = 10, + .resident = shared, + .backing = old.frames.at(0).backing}); + auto inner = std::make_shared<InnerPageState>(); + inner->children = {0, right_pid}; + inner->separators.emplace_back("c"); + auto root = std::make_shared<PublishedPage>(); + root->level = 1; + root->inner = inner; + const std::array<PageReplacement, 3> changes{ + {{.page_id = 0, .replacement = published(left)}, + {.page_id = right_pid, .replacement = published(right)}, + {.page_id = root_pid, .replacement = root}} + }; + auto captured = std::make_shared<PublishedTreeState>(); + MappingView::CopyStats copied; + captured->mapping = captured->mapping.replace(changes, copied); + captured->root_page_id = root_pid; + captured->next_page_id = root_pid + 1; + captured->coverage = 10; + return captured; +} + +TEST(CapturedWriteDependencies, SharedChainsDoNotJoinTheDependencyOwnersOfDifferentOutputLeaves) +{ + TestCapturedWrite test; + const std::string value(3000, 's'); + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = value} + }))) + .ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U) << first.status.to_string(); + auto old = test.operation->result().pages; + ASSERT_NE(old, nullptr); + const auto raw = test.read(old->frames.at(0)); + const auto encoded = LeafFrameView(raw.data(), raw.size()).cell(0); + const auto right_pid = old->next_page_id; + auto captured = shared_chain_input(*old, encoded); + test.operation.reset(); + ASSERT_TRUE(test.prepare(captured, old).ok()); + Completion second; + test.run(&second); + ASSERT_EQ(second.values.load(), 1U) << second.status.to_string(); + auto result = test.operation->result(); + ASSERT_NE(result.pages, nullptr); + auto retained = result.pages->frames.at(right_pid).backing; + ASSERT_NE(retained->overflow, nullptr); + const auto protected_count = retained->overflow->pages.size() + 1; + test.operation.reset(); + captured.reset(); + old.reset(); + result = {}; + EXPECT_EQ(test.ledger->stats().protected_ranges, protected_count); + const CellView cell{Slice(encoded)}; + std::string restored; + for (auto pid = cell.overflow_head(); pid != kInvalidPageId;) { + const auto &frame = retained->overflow->pages.at(pid); + const auto bytes = test.read({.page_id = pid, + .type = page_type::kOverflowFrame, + .frame_bytes = frame.frame_bytes, + .logical_bytes = frame.logical_bytes, + .next_overflow = frame.next, + .overflow_bytes = frame.bytes, + .backing = frame.backing}); + const OverflowFrameView chunk(bytes.data(), bytes.size()); + restored += chunk.payload().to_string(); + pid = chunk.next_page_id(); + } + EXPECT_EQ(restored, value); + retained.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/captured_write_resident_test.cpp b/lib/crowdb-tree/tests/integration/captured_write_resident_test.cpp new file mode 100644 index 000000000..2c60fd3ee --- /dev/null +++ b/lib/crowdb-tree/tests/integration/captured_write_resident_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/overflow_backing.h" +#include "captured_write_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<PublishedTreeState> shared_resident_chain() +{ + auto overflow = std::make_shared<OverflowBackingSet>(nullptr); + for (uint64_t pid = 1; pid <= 2; ++pid) { + auto bytes = std::make_shared<std::vector<uint8_t>>(512); + const std::string value(300, 'v'); + const auto next = pid == 1 ? 2 : kInvalidPageId; + overflow_frame_build(bytes->data(), bytes->size(), pid, next, reinterpret_cast<const uint8_t *>(value.data()), + value.size()); + auto backing = std::make_shared<BackingRef>(); + backing->owner = bytes; + backing->storage = BackingStorage::kResidentFrame; + backing->resident_frame = *bytes; + overflow->pages.emplace( + pid, + OverflowBackingPage{ + .backing = std::move(backing), .next = next, .bytes = 300, .frame_bytes = 512, .logical_bytes = 0}); + } + auto raw = std::make_shared<std::vector<uint8_t>>(512); + const auto cell = encode_overflow_cell_buf(10, 1, 600); + LeafFrameBuilder builder(raw->data(), raw->size()); + EXPECT_TRUE(builder.try_append_sorted("a", Slice(cell))); + EXPECT_TRUE(builder.try_append_sorted("b", Slice(cell))); + builder.finish(0, kInvalidPageId); + auto backing = std::make_shared<BackingRef>(); + backing->owner = raw; + backing->storage = BackingStorage::kResidentFrame; + backing->resident_frame = *raw; + backing->overflow = std::move(overflow); + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back( + {.identity = 1, + .generation = 1, + .max_slot = 10, + .resident = std::shared_ptr<const LeafBase>(LeafBase::from_frame_copy(raw->data(), raw->size())), + .backing = std::move(backing)}); + auto captured = test_write::captured_leaf(std::move(leaf)); + captured->next_page_id = 3; + return captured; +} + +TEST(CapturedWriteResident, DuplicateReferencesShareWritesAndReleaseEveryExtent) +{ + test_write::TestCapturedWrite test; + auto captured = shared_resident_chain(); + std::weak_ptr<const void> raw_owner = + captured->mapping.get(0)->leaf->bases.front().backing->overflow->pages.at(1).backing->owner; + ASSERT_TRUE(test.prepare(captured).ok()); + test_write::Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + auto pages = test.operation->result().pages; + ASSERT_NE(pages, nullptr); + EXPECT_EQ(test.operation->result().written_frames, 3U); + EXPECT_EQ(pages->frames.size(), 3U); + EXPECT_EQ(pages->frames.at(0).backing->overflow->pages.size(), 2U); + auto retained = pages->frames.at(0).backing; + test.operation.reset(); + captured.reset(); + pages.reset(); + EXPECT_TRUE(raw_owner.expired()); + EXPECT_EQ(test.ledger->stats().protected_ranges, 3U); + retained.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[test_write::LoadCredits::kBytes], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/captured_write_test.cpp b/lib/crowdb-tree/tests/integration/captured_write_test.cpp new file mode 100644 index 000000000..514bfbab2 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/captured_write_test.cpp @@ -0,0 +1,312 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/overflow_backing.h" +#include "captured_write_test_support.h" + +namespace crowdb::tree +{ +using namespace test_write; + +TEST(CapturedWrite, OverflowAndPrimaryUseOneWindowAndKeepProtectedReferencesAfterInputsEnd) +{ + TestCapturedWrite test; + const std::string value(16000, 'v'); + auto captured = captured_leaf(page({ + {.key = "a", .value = value} + })); + ASSERT_TRUE(test.prepare(captured).ok()); + test.store->deferred = true; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + size_t completed = 0; + size_t peak = 0; + while (done.values.load() + done.errors.load() == 0 && completed < 100) { + ASSERT_TRUE(test.store->pending); + EXPECT_EQ(test.store->writes, completed + 1); + peak = std::max(peak, test.credits.budget.snapshot().allocated[LoadCredits::kBytes]); + test.store->finish(); + ++completed; + test.runtime->drain(); + } + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + auto result = test.operation->result(); + ASSERT_TRUE(result.pages); + EXPECT_EQ(result.pages->coverage, 10U); + EXPECT_EQ(result.written_frames, result.pages->frames.size()); + EXPECT_GT(result.written_frames, 40U); + EXPECT_EQ(result.written_bytes, result.written_frames * 4096); + EXPECT_LT(peak, result.written_bytes); + RecordProperty("allocated_peak", peak); + RecordProperty("written_bytes", result.written_bytes); + test.operation.reset(); + captured.reset(); + const auto raw = test.read(result.pages->frames.at(0)); + LeafFrameView leaf(raw.data(), raw.size()); + const CellView cell(leaf.cell(0)); + std::string restored; + uint64_t pid = cell.overflow_head(); + while (pid != kInvalidPageId) { + const auto bytes = test.read(result.pages->frames.at(pid)); + OverflowFrameView chunk(bytes.data(), bytes.size()); + restored += chunk.payload().to_string(); + pid = chunk.next_page_id(); + } + EXPECT_EQ(restored, value); + EXPECT_EQ(test.ledger->stats().protected_ranges, result.written_frames); + result = {}; + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(CapturedWrite, CancellationWaitsForPendingWriteAndDoesNotPublishPartialMapping) +{ + TestCapturedWrite test; + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = std::string(2000, 'v')} + }))) + .ok()); + test.store->deferred = true; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + ASSERT_TRUE(test.store->pending); + test.operation->request_stop(); + test.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 0U); + EXPECT_EQ(test.ledger->stats().protected_ranges, 1U); + test.store->finish(); + test.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_FALSE(test.operation->result().pages); + EXPECT_EQ(test.store->writes, 1U); + test.operation.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); +} + +TEST(CapturedWrite, ClosedRuntimeDrainsPendingWriteBeforeReportingDispatchFailure) +{ + TestCapturedWrite test; + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = "v"} + }))) + .ok()); + test.store->deferred = true; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + ASSERT_TRUE(test.store->pending); + test.runtime->close(); + test.store->finish(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_FALSE(test.operation->result().pages); + test.operation.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); +} + +TEST(CapturedWrite, UnknownWriteSubmissionKeepsPhysicalProtectionUntilLateCompletion) +{ + TestCapturedWrite test; + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = "v"} + }))) + .ok()); + test.store->throw_after_capture = true; + Completion done; + test.run(&done); + ASSERT_EQ(done.errors.load(), 1U); + EXPECT_FALSE(test.operation->result().pages); + test.operation.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 1U); + test.store->finish(); + test.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); +} + +TEST(CapturedWrite, FrameLimitFailsAndReleasesAllCompletedOrphanWrites) +{ + TestCapturedWrite test; + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = std::string(2000, 'v')} + })), + {}, + {.frames = {.encode = {.frame_bytes = 512, .inline_value_bytes = 16}}, + .write = {.max_frame_bytes = 512}, + .max_frames = 2}) + .ok()); + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(test.operation->result().pages); + test.operation.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); +} + +TEST(CapturedWrite, ReusesOnlyTheCapturedOverflowChainAndReleasesTheOldPrimary) +{ + TestCapturedWrite test; + const std::string value(3000, 'r'); + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = value} + }))) + .ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U) << first.status.to_string(); + auto old = test.operation->result().pages; + ASSERT_TRUE(old); + const auto raw = test.read(old->frames.at(0)); + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back( + {.identity = 1, + .generation = 1, + .max_slot = 10, + .resident = std::shared_ptr<const LeafBase>(LeafBase::from_frame_copy(raw.data(), raw.size())), + .backing = old->frames.at(0).backing}); + auto next = captured_leaf(leaf); + next->next_page_id = old->next_page_id; + test.operation.reset(); + ASSERT_TRUE(test.prepare(next, old).ok()); + Completion second; + test.run(&second); + ASSERT_EQ(second.values.load(), 1U) << second.status.to_string(); + auto result = test.operation->result(); + ASSERT_TRUE(result.pages); + EXPECT_EQ(result.written_frames, 1U); + EXPECT_EQ(result.reused_frames, old->frames.size() - 1); + for (const auto &[pid, frame] : old->frames) { + if (pid != 0) { + const auto &adopted = result.pages->frames.at(pid).backing; + EXPECT_NE(frame.backing, adopted); + EXPECT_EQ(frame.backing->physical_owner, adopted->physical_owner); + EXPECT_EQ(frame.backing->address, adopted->address); + EXPECT_EQ(frame.backing->length, adopted->length); + } + } + const auto expected = result.pages->frames.size(); + test.operation.reset(); + old.reset(); + next.reset(); + leaf.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, expected); + const CellView cell(LeafFrameView(raw.data(), raw.size()).cell(0)); + std::string restored; + for (auto pid = cell.overflow_head(); pid != kInvalidPageId;) { + const auto bytes = test.read(result.pages->frames.at(pid)); + OverflowFrameView chunk(bytes.data(), bytes.size()); + restored += chunk.payload().to_string(); + pid = chunk.next_page_id(); + } + EXPECT_EQ(restored, value); + result = {}; + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(CapturedWrite, MissingOrCyclicInheritedOverflowCannotBecomeSuccessfulOutput) +{ + TestCapturedWrite test; + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = std::string(1000, 'r')} + }))) + .ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U); + auto old = test.operation->result().pages; + const auto raw = test.read(old->frames.at(0)); + const CellView cell(LeafFrameView(raw.data(), raw.size()).cell(0)); + for (bool typed : {false, true}) { + for (bool cycle : {false, true}) { + SCOPED_TRACE(::testing::Message() << "typed=" << typed << " cycle=" << cycle); + auto bad = std::make_shared<CapturedDurablePages>(nullptr); + bad->frames.insert(old->frames.begin(), old->frames.end()); + auto backing = std::make_shared<BackingRef>(*old->frames.at(0).backing); + backing->overflow.reset(); + if (typed) { + auto dependencies = std::make_shared<OverflowBackingSet>(nullptr); + dependencies->pages = old->frames.at(0).backing->overflow->pages; + if (cycle) { + dependencies->pages.at(cell.overflow_head()).next = cell.overflow_head(); + } + else { + dependencies->pages.erase(cell.overflow_head()); + } + backing->overflow = std::move(dependencies); + } + else if (cycle) { + bad->frames.at(cell.overflow_head()).next_overflow = cell.overflow_head(); + } + else { + bad->frames.erase(cell.overflow_head()); + } + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back( + {.identity = 1, + .max_slot = 10, + .resident = std::shared_ptr<const LeafBase>(LeafBase::from_frame_copy(raw.data(), raw.size())), + .backing = std::move(backing)}); + auto next = captured_leaf(leaf); + next->next_page_id = old->next_page_id; + test.operation.reset(); + const auto writes = test.store->writes; + ASSERT_TRUE(test.prepare(next, bad).ok()); + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kCorruption); + EXPECT_FALSE(test.operation->result().pages); + EXPECT_EQ(test.store->writes, writes); + } + } +} + +TEST(CapturedWrite, EmptyCaptureCompletesWithoutReservingAnExtent) +{ + TestCapturedWrite test; + ASSERT_TRUE(test.prepare(std::make_shared<PublishedTreeState>()).ok()); + Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U); + ASSERT_TRUE(test.operation->result().pages); + EXPECT_TRUE(test.operation->result().pages->frames.empty()); + EXPECT_EQ(test.store->writes, 0U); +} + +TEST(CapturedWrite, MetadataExhaustionAfterAcceptedIoReleasesThePartialImage) +{ + TestCapturedWrite test; + ASSERT_TRUE(test.prepare(captured_leaf(page({ + {.key = "a", .value = std::string(2000, 'v')} + }))) + .ok()); + test.store->deferred = true; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + ASSERT_TRUE(test.store->pending); + test.credits.credits.bytes->seal(); + test.store->finish(); + test.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(test.operation->result().pages); + EXPECT_EQ(test.store->writes, 1U); + test.operation.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/checkpoint_handoff_report_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_handoff_report_test.cpp new file mode 100644 index 000000000..8c708b82a --- /dev/null +++ b/lib/crowdb-tree/tests/integration/checkpoint_handoff_report_test.cpp @@ -0,0 +1,206 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_checkpoint_test_support.h" +#include "snapshot/checkpoint_operation.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_write; + +struct TestHandoffCheckpoint +{ + TestCheckpoint local; + SourceAdmission sources{10}; + std::shared_ptr<LocalBackendState> backend = std::make_shared<LocalBackendState>(); + std::shared_ptr<const PublishedTreeState> initial = [this] { + auto state = captured_leaf(page({ + {.key = "a", .value = "old"} + })); + EXPECT_TRUE(apply_page_metrics(state->stats, nullptr, &state->mapping.get(0)->metrics)); + auto catalog = sources.catalog(); + auto inputs = std::make_shared<PublishedSources>(); + inputs->floor = catalog->floor; + inputs->tables.assign(catalog->tables.begin(), catalog->tables.end()); + state->sources = std::move(inputs); + return state; + }(); + PublicationCoordinator publisher{ + 4, 1, + {}, + std::pmr::get_default_resource(), + {}, + {.metadata = {}, .initial = initial, .source_catalog = sources.catalog()} + }; + std::atomic<uint64_t> contiguous{11}; + std::shared_ptr<detail::AllocationReservation> handoff_credit; + std::shared_ptr<detail::AllocationReservation> finish_credit; + std::shared_ptr<FlushOperation> handoff; + std::shared_ptr<CheckpointOperation> operation; + + TestHandoffCheckpoint() + { + backend->store = local.store; + backend->ledger = local.source.ledger; + backend->checkpoints = local.registry; + handoff_credit = partition(1024 * 1024); + finish_credit = partition(CheckpointOperation::finish_allocation_bound() + 512); + } + + std::shared_ptr<detail::AllocationReservation> partition(size_t bytes) + { + auto credit = local.source.credits.credits.bytes->take_reserved(bytes); + if (!credit) { + ADD_FAILURE() << "test credit unavailable"; + return {}; + } + return detail::AllocationReservation::create(std::move(credit.value())); + } + + Status prepare() + { + CheckpointOptions options; + options.write.frames.encode.frame_bytes = 4096; + options.write.frames.encode.inline_value_bytes = 16; + options.write.write.max_frame_bytes = 4096; + return CheckpointOperation::create( + {}, publisher, backend, local.store, local.source.runtime, local.source.lane, options, + local.source.credits.credits, finish_credit, std::make_shared<test_read::TestLoader>(), + [this](std::shared_ptr<FlushOperation> *out) { + *out = handoff; + return Status::Ok(); + }, + {}, &operation); + } + + void expect_no_durable_work(Code expected) + { + Completion done; + auto connected = stdexec::connect(operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + local.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.values.load(), 0U); + EXPECT_EQ(done.status.code(), expected); + EXPECT_FALSE(operation->result().durable_committed); + EXPECT_EQ(local.registry->durable_coverage(), 0U); + EXPECT_EQ(local.registry->snapshot_sequence(), 0U); + EXPECT_FALSE(publisher.capture()->checkpoint_inflight); + EXPECT_TRUE(local.store->events.empty()); + EXPECT_EQ(local.source.store->writes, 0U); + } +}; + +TEST(CheckpointHandoffReport, FailedAdmissionNeverCommitsTheOlderCoverage) +{ + TestHandoffCheckpoint test; + FlushController controller(test.publisher, *test.local.source.runtime, test.sources, test.contiguous, + test.local.source.lane); + const auto before = test.publisher.capture(); + ASSERT_TRUE(controller.prepare(test.handoff_credit, &test.handoff, before->commit_seq + 1).ok()); + ASSERT_TRUE(test.prepare().ok()); + test.expect_no_durable_work(Code::kUnavailable); + EXPECT_EQ(test.publisher.capture(), before); + EXPECT_EQ(test.publisher.capture()->coverage, 10U); + EXPECT_FALSE(test.sources.catalog()->active->closed()); +} + +TEST(CheckpointHandoffReport, MaintenanceFailureKeepsHandoffProgressButDoesNotWriteAnAnchor) +{ + TestHandoffCheckpoint test; + SourceAdmission::Writer writer; + ASSERT_TRUE(test.sources.try_admit(10, &writer).ok()); + ASSERT_TRUE(writer.table()->upsert("b", 11, encode_cell_buf(11, OpKind::kPut, "new"))); + writer.release(); + FlushController controller(test.publisher, *test.local.source.runtime, test.sources, test.contiguous, + test.local.source.lane, {}, + [](auto, auto complete) { complete(Status::io_error(), 7); }); + ASSERT_TRUE(controller.prepare(test.handoff_credit, &test.handoff).ok()); + ASSERT_TRUE(test.prepare().ok()); + test.expect_no_durable_work(Code::kIoError); + EXPECT_EQ(test.publisher.capture()->coverage, 11U); + EXPECT_EQ(test.sources.catalog()->floor, 11U); +} + +struct TestWorkerGate +{ + std::atomic<bool> entered{false}, released{false}; + + ~TestWorkerGate() + { + release(); + } + + void release() + { + released.store(true); + released.notify_all(); + } + + void wait() const + { + while (!entered.load()) { + entered.wait(false); + } + } + + static void run(void *context) noexcept + { + auto &self = *static_cast<TestWorkerGate *>(context); + self.entered.store(true); + self.entered.notify_all(); + while (!self.released.load()) { + self.released.wait(false); + } + } +}; + +TEST(CheckpointHandoffReport, ReconnectCannotChangeTheDeliveredFrontier) +{ + TestHandoffCheckpoint test; + auto handoff_runtime = std::make_shared<detail::FlushRuntime>(1, 8); + TestWorkerGate handoff_gate; + TestWorkerGate checkpoint_gate; + FlushController controller(test.publisher, *handoff_runtime, test.sources, test.contiguous, test.local.source.lane); + SourceAdmission::Writer writer; + ASSERT_TRUE(test.sources.try_admit(10, &writer).ok()); + ASSERT_TRUE(writer.upsert("b", 11, encode_cell_buf(11, OpKind::kPut, "eleven")).ok()); + writer.release(); + ASSERT_TRUE(controller.prepare(test.handoff_credit, &test.handoff).ok()); + ASSERT_TRUE(test.prepare().ok()); + ASSERT_TRUE(handoff_runtime->try_submit(test.local.source.lane, &TestWorkerGate::run, &handoff_gate).ok()); + handoff_gate.wait(); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + ASSERT_TRUE( + test.local.source.runtime->try_submit(test.local.source.lane, &TestWorkerGate::run, &checkpoint_gate).ok()); + checkpoint_gate.wait(); + handoff_gate.release(); + handoff_runtime->drain(); + EXPECT_EQ(test.handoff->result().frontier, 11U); + + // Keep the checkpoint continuation queued while a later flush and a split + // capture reconnect change the shared handoff object's mutable report. + ASSERT_TRUE(test.sources.try_admit(11, &writer).ok()); + ASSERT_TRUE(writer.upsert("c", 12, encode_cell_buf(12, OpKind::kPut, "twelve")).ok()); + writer.release(); + test.contiguous.store(12); + std::shared_ptr<FlushOperation> next; + ASSERT_TRUE(controller.prepare(test.partition(1024 * 1024), &next).ok()); + ASSERT_TRUE(next->wait().ok()); + FlushResult recaptured; + ASSERT_TRUE(test.handoff->wait_capture(&recaptured).ok()); + EXPECT_EQ(recaptured.frontier, 12U); + checkpoint_gate.release(); + test.local.source.runtime->drain(); + EXPECT_EQ(done.status.code(), Code::kUnavailable); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_FALSE(test.operation->result().durable_committed); + EXPECT_EQ(test.local.registry->durable_coverage(), 0U); + EXPECT_TRUE(test.local.store->events.empty()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/checkpoint_operation_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_operation_test.cpp new file mode 100644 index 000000000..f934c6ca0 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/checkpoint_operation_test.cpp @@ -0,0 +1,342 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_checkpoint_test_support.h" +#include "snapshot/checkpoint_operation.h" +#include "snapshot/snapshot_baseline.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_write; + +struct TestCheckpointPipeline +{ + TestCheckpoint local; + std::shared_ptr<LocalBackendState> backend = std::make_shared<LocalBackendState>(); + std::shared_ptr<const PublishedTreeState> initial = [] { + auto result = captured_leaf(page({ + {.key = "a", .value = "v"} + })); + EXPECT_TRUE(apply_page_metrics(result->stats, nullptr, &result->mapping.get(0)->metrics)); + return result; + }(); + PublicationCoordinator publisher{ + 4, 1, {}, + std::pmr::get_default_resource(), {}, + {.metadata = {}, .initial = initial, .source_catalog = {}} + }; + std::shared_ptr<detail::AllocationReservation> finish; + std::shared_ptr<CheckpointOperation> operation; + + TestCheckpointPipeline() + { + backend->store = local.store; + backend->ledger = local.source.ledger; + backend->checkpoints = local.registry; + auto credit = + local.source.credits.credits.bytes->take_reserved(CheckpointOperation::finish_allocation_bound() + 512); + EXPECT_TRUE(credit.has_value()); + if (credit) { + finish = detail::AllocationReservation::create(std::move(*credit)); + } + } + + Status prepare(CheckpointOperation::PrepareWork work = {}, std::optional<CheckpointOptions> supplied = {}) + { + CheckpointOptions options; + options.write.frames.encode.frame_bytes = 4096; + options.write.frames.encode.inline_value_bytes = 16; + options.write.write.max_frame_bytes = 4096; + if (supplied) { + options = *supplied; + } + auto loader = std::make_shared<test_read::TestLoader>(); + return CheckpointOperation::create({}, publisher, backend, local.store, local.source.runtime, local.source.lane, + options, local.source.credits.credits, finish, loader, {}, std::move(work), + &operation); + } + + void replace_live() + { + auto before = publisher.capture(); + auto leaf = std::make_shared<PageState>(*page({ + {.key = "a", .value = "new"} + })); + leaf->content_id = before->mapping.get(0)->leaf->content_id + 1; + auto replacement = std::make_shared<PublishedPage>(*before->mapping.get(0)); + replacement->leaf = std::move(leaf); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{.sequence = 99}); + request->transaction.pages.push_back( + {.page_id = 0, .expected = before->mapping.get(0), .replacement = replacement}); + request->on_done = [](PublicationResult result) { + EXPECT_TRUE(result.status.ok()) << result.status.to_string(); + }; + ASSERT_TRUE(publisher.try_submit(request)); + } +}; + +TEST(CheckpointOperation, DurableCommitReconcilesOnlyItsCapturedView) +{ + TestCheckpointPipeline test; + test.local.store->defer_at = 1; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + ASSERT_TRUE(test.publisher.capture()->checkpoint_inflight); + const auto waiting_metrics = test.publisher.work_metrics()->totals(); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::SnapshotCapture)].count, 1U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::SnapshotCommit)].count, 0U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::SnapshotCommit)].inflight, 1U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)].inflight, 1U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)].count, 0U); + test.replace_live(); + const auto live = test.publisher.capture()->mapping.get(0); + test.local.store->finish(); + test.local.source.runtime->drain(); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_TRUE(test.operation->result().durable_committed); + EXPECT_EQ(test.operation->result().durable_coverage, 10U); + EXPECT_EQ(test.operation->result().snapshot_sequence, 1U); + EXPECT_EQ(test.publisher.capture()->mapping.get(0), live); + EXPECT_EQ(test.publisher.capture()->stats.checkpoint_pending_pages, 1U); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + const auto baseline = test.publisher.capture()->snapshot_baseline; + ASSERT_TRUE(baseline); + ASSERT_TRUE(baseline->pages); + EXPECT_EQ(baseline->coverage, 10U); + EXPECT_EQ(baseline->snapshot_sequence, 1U); + EXPECT_EQ(baseline->pages->root_page_id, 0U); + EXPECT_EQ(baseline->pages->frames.size(), 1U); + ASSERT_TRUE(test.publisher.capture()->snapshot_changes.get(0)); + EXPECT_EQ(test.publisher.capture()->snapshot_changes.get(0)->after.content, live->leaf->content_id); + const auto committed_metrics = test.publisher.work_metrics()->totals(); + EXPECT_EQ(committed_metrics[static_cast<size_t>(TreeStage::SnapshotCommit)].count, 1U); + EXPECT_EQ(committed_metrics[static_cast<size_t>(TreeStage::SnapshotCommit)].inflight, 0U); + EXPECT_EQ(committed_metrics[static_cast<size_t>(TreeStage::SnapshotCommit)] + .outcomes[static_cast<size_t>(TreeOutcome::Success)], + 1U); + EXPECT_EQ(committed_metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)].count, 1U); + EXPECT_EQ(committed_metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)].inflight, 0U); + EXPECT_EQ(committed_metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)] + .outcomes[static_cast<size_t>(TreeOutcome::Stale)], + 1U); + EXPECT_EQ(test.publisher.work_metrics()->counters.snapshot()[static_cast<size_t>( + TreeWorkCause::Checkpoint)][static_cast<size_t>(TreeWorkCounter::RebaseMismatches)], + 1U); + test.local.expect_read("v"); +} + +TEST(CheckpointOperation, FailedMetadataAbortsPublicationWithoutAdvancingDurability) +{ + TestCheckpointPipeline test; + test.local.store->fail_at = 2; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.publisher.work_metrics()->counters.snapshot()[static_cast<size_t>( + TreeWorkCause::Checkpoint)][static_cast<size_t>(TreeWorkCounter::CheckpointFailures)], + 1U); + EXPECT_EQ(test.operation->result().status.code(), Code::kIoError); + EXPECT_FALSE(test.operation->result().durable_committed); + EXPECT_EQ(test.local.registry->durable_coverage(), 0U); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_FALSE(test.local.registry->needs_recovery()); + EXPECT_FALSE(test.publisher.capture()->snapshot_baseline); + EXPECT_FALSE(test.publisher.capture()->snapshot_changes.empty()); + const auto failed_metrics = test.publisher.work_metrics()->totals(); + EXPECT_EQ(failed_metrics[static_cast<size_t>(TreeStage::SnapshotCommit)].count, 1U); + EXPECT_EQ(failed_metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)].inflight, 0U); + EXPECT_EQ(failed_metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)] + .outcomes[static_cast<size_t>(TreeOutcome::Error)], + 1U); + EXPECT_EQ(failed_metrics[static_cast<size_t>(TreeStage::SnapshotCommit)] + .outcomes[static_cast<size_t>(TreeOutcome::Error)], + 1U); +} + +TEST(CheckpointOperation, UnknownAnchorKeepsRegistryFencedAfterPublicationAbort) +{ + TestCheckpointPipeline test; + test.local.store->throw_at = 5; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_TRUE(test.local.registry->needs_recovery()); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_EQ(test.local.registry->durable_coverage(), 0U); + ASSERT_TRUE(test.local.store->pending); + test.local.store->finish(); + test.local.source.runtime->drain(); + EXPECT_EQ(test.prepare().code(), Code::kUnavailable); +} + +TEST(CheckpointOperation, CancellationBeforeAnchorWaitsForWriteAndAbortsCapture) +{ + TestCheckpointPipeline test; + test.local.store->defer_at = 1; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + test.operation->request_stop(); + test.local.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 0U); + test.local.store->finish(); + test.local.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_EQ(test.local.registry->durable_coverage(), 0U); +} + +TEST(CheckpointOperation, CancellationAfterAnchorCompletesCommitEvenAfterPublisherClose) +{ + TestCheckpointPipeline test; + test.local.store->defer_at = 6; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + ASSERT_TRUE(test.local.store->pending); + ASSERT_EQ(test.local.store->events.back(), "barrier"); + test.publisher.close(); + test.operation->request_stop(); + test.local.source.runtime->drain(); + test.local.store->finish(); + test.local.source.runtime->drain(); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_TRUE(test.operation->result().durable_committed); + EXPECT_EQ(test.local.registry->durable_coverage(), 10U); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); +} + +TEST(CheckpointOperation, ExhaustedMaterializationStillHasIndependentAbortCredit) +{ + TestCheckpointPipeline test; + test.local.store->defer_at = 1; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + auto credit = test.local.source.credits.credits.bytes; + const auto remaining = credit->available(); + ASSERT_TRUE(credit->claim(remaining)); + test.local.store->finish(); + test.local.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.operation->result().status.code(), Code::kResourceExhausted); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + credit->release(remaining); +} + +TEST(CheckpointOperation, ChangedPublicationAfterLayoutIsRejectedBeforeCapture) +{ + TestCheckpointPipeline test; + ASSERT_TRUE(test.prepare([&test](const PublishedTreeState &, const CapturedWritePlan *plan, CheckpointWork *work) { + work->budget = test.local.source.credits.credits; + work->loader = std::make_shared<test_read::TestLoader>(); + if (plan != nullptr) { + test.replace_live(); + } + return Status::Ok(); + }) + .ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kUnavailable); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_TRUE(test.local.store->events.empty()); + EXPECT_EQ(test.backend->checkpoints->durable_coverage(), 0U); +} + +TEST(CheckpointOperation, NewL0CatalogBetweenLayoutAndCapturePreservesTheFixedPrefix) +{ + TestCheckpointPipeline test; + bool published = false; + ASSERT_TRUE(test.prepare([&](const PublishedTreeState &, const CapturedWritePlan *plan, CheckpointWork *work) { + work->budget = test.local.source.credits.credits; + work->loader = std::make_shared<test_read::TestLoader>(); + if (plan != nullptr) { + const auto before = test.publisher.capture(); + auto sources = std::make_shared<PublishedSources>(*before->sources); + sources->tables.push_back(std::make_shared<MemTable>()); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{.sequence = 99}); + request->transaction.expected_sources = before->sources; + request->transaction.sources = std::move(sources); + request->on_done = [&](PublicationResult result) { + EXPECT_TRUE(result.status.ok()) << result.status.to_string(); + published = result.status.ok(); + }; + EXPECT_TRUE(test.publisher.try_submit(request)); + } + return Status::Ok(); + }) + .ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + EXPECT_TRUE(published); + ASSERT_EQ(done.values.load(), 1U) << test.operation->result().status.to_string(); + EXPECT_TRUE(test.operation->result().durable_committed); + EXPECT_EQ(test.operation->result().durable_coverage, 10U); + EXPECT_EQ(test.publisher.capture()->sources->tables.size(), 1U); +} + +TEST(CheckpointOperation, PlannedWorkAdmissionFailureDoesNotCaptureOrWrite) +{ + TestCheckpointPipeline test; + ASSERT_TRUE(test.prepare([&test](const PublishedTreeState &, const CapturedWritePlan *plan, CheckpointWork *work) { + if (plan != nullptr) { + return Status::resource_exhausted(); + } + work->budget = test.local.source.credits.credits; + work->loader = std::make_shared<test_read::TestLoader>(); + return Status::Ok(); + }) + .ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_TRUE(test.local.store->events.empty()); +} + +TEST(CheckpointOperation, InvalidGeometryIsRejectedBeforeAnyCaptureOrIo) +{ + TestCheckpointPipeline test; + CheckpointOptions options; + options.write.write.iu_size = 8192; + options.write.write.buffer_alignment = 8192; + EXPECT_EQ(test.prepare({}, options).code(), Code::kInvalidArgument); + EXPECT_FALSE(test.operation); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_TRUE(test.local.store->events.empty()); + options.write.write.iu_size = 4096; + options.write.frames.encode.frame_bytes = options.write.write.max_frame_bytes + 1; + EXPECT_EQ(test.prepare({}, options).code(), Code::kInvalidArgument); + EXPECT_FALSE(test.operation); + EXPECT_TRUE(test.local.store->events.empty()); +} + +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/checkpoint_retention_reuse_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_retention_reuse_test.cpp new file mode 100644 index 000000000..123f77bf4 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/checkpoint_retention_reuse_test.cpp @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +void check_live_only_reuse(size_t value_bytes) +{ + MemPageStore store; + Config config; + config.page_store = &store; + config.frame_bytes = 65536; + const std::string value(value_bytes, 'v'); + { + Crowdbtree tree(config); + for (uint64_t slot = 1; slot <= 3; ++slot) { + ASSERT_TRUE( + tree.apply(slot, + Batch{.ops = {{.key = "key" + std::to_string(slot), .kind = OpKind::kPut, .value = value}}}) + .ok()); + } + uint64_t durable = 0; + ASSERT_TRUE(tree.snapshot(&durable).ok()); + ASSERT_EQ(durable, 3U); + tree.set_gc_watermark(3, 3); + ASSERT_TRUE(tree.snapshot(&durable).ok()); + EXPECT_EQ(durable, 3U); + EXPECT_EQ(tree.last_snapshot_pages_written(), 0U); + } + Crowdbtree recovered(config); + for (uint64_t slot = 1; slot <= 3; ++slot) { + auto result = recovered.get_view("key" + std::to_string(slot)); + ASSERT_TRUE(result.found()); + EXPECT_EQ(result.slot(), slot); + EXPECT_EQ(result.value().to_string(), value); + } +} + +} // namespace + +TEST(CheckpointRetentionReuse, AdvancingFloorWithoutDeletesDoesNotRewriteStoredValues) +{ + check_live_only_reuse(16384); + check_live_only_reuse(98304); +} + +TEST(CheckpointRetentionReuse, AdvancingFloorStillRewritesAStoredTombstoneBeforeRecovery) +{ + MemPageStore store; + Config config; + config.page_store = &store; + config.frame_bytes = 65536; + { + Crowdbtree tree(config); + ASSERT_TRUE(tree.apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "old"}}}).ok()); + ASSERT_TRUE(tree.apply(2, Batch{.ops = {{.key = "b", .kind = OpKind::kPut, .value = "live"}}}).ok()); + ASSERT_TRUE(tree.apply(3, Batch{.ops = {{.key = "a", .kind = OpKind::kDelete, .value = ""}}}).ok()); + uint64_t durable = 0; + ASSERT_TRUE(tree.snapshot(&durable).ok()); + ASSERT_EQ(durable, 3U); + tree.set_gc_watermark(3, 3); + ASSERT_TRUE(tree.snapshot(&durable).ok()); + EXPECT_GT(tree.last_snapshot_pages_written(), 0U); + EXPECT_EQ(durable, 3U); + } + Crowdbtree recovered(config); + EXPECT_FALSE(recovered.get_view("a").found()); + auto result = recovered.get_view("b"); + ASSERT_TRUE(result.found()); + EXPECT_EQ(result.slot(), 2U); + EXPECT_EQ(result.value().to_string(), "live"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/checkpoint_split_progress_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_split_progress_test.cpp new file mode 100644 index 000000000..e37ee3d4a --- /dev/null +++ b/lib/crowdb-tree/tests/integration/checkpoint_split_progress_test.cpp @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "snapshot/operation_wait.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +uint64_t checkpoint_jobs(size_t entries) +{ + MemPageStore store(1); + Config config; + config.page_store = &store; + config.frame_bytes = 65536; + config.memtable_flush_bytes = 64 * 1024 * 1024; + config.execution.maintenance_max_visits = 1; + config.execution.maintenance_visits_per_step = 1; + config.execution.maintenance_max_candidates = 1; + config.execution.maintenance_max_steps = 1; + config.runtime_binding.limits.handoff_bytes = 128 * 1024 * 1024; + config.runtime_binding.limits.reclaim_bytes = 128 * 1024 * 1024; + TreeRuntimeOptions host; + host.workers = 1; + auto status = TreeRuntime::create(host, &config.runtime); + EXPECT_TRUE(status.ok()) << status.to_string(); + Crowdbtree tree(config); + const std::string value(16384, 'v'); + for (uint64_t i = 1; i <= entries; ++i) { + status = tree.apply( + i, Batch{.ops = {{.key = "key" + std::to_string(10000 + i), .kind = OpKind::kPut, .value = value}}}); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + status = tree.flush(); + EXPECT_TRUE(status.ok()) << status.to_string(); + auto *core = TreeCoreAccess::get(tree); + core->binding()->account()->drain_retirements(); + core->binding()->lane()->drain(); + const auto before = config.runtime->snapshot().cpu.completed; + std::shared_ptr<CheckpointOperation> operation; + status = core->prepare_checkpoint(checkpoint_detail::defaults(*core), &operation); + EXPECT_TRUE(status.ok()) << status.to_string(); + if (!status.ok()) { + return 0; + } + status = detail::wait_snapshot_operation(operation); + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(operation->result().durable_coverage, entries); + EXPECT_GT(operation->result().structural_windows, 1U); + operation.reset(); + core->binding()->account()->drain_retirements(); + core->binding()->lane()->drain(); + const auto jobs = config.runtime->snapshot().cpu.completed - before; + for (uint64_t i = 1; i <= entries; ++i) { + const auto result = tree.get_view("key" + std::to_string(10000 + i)); + EXPECT_TRUE(result.found()); + EXPECT_EQ(result.value().to_string(), value); + } + return jobs; +} +} // namespace + +TEST(CheckpointSplitProgress, DoublingOneOversizedLeafKeepsCheckpointWorkBelowQuadraticGrowth) +{ + const auto small = checkpoint_jobs(512); + const auto large = checkpoint_jobs(1024); + ASSERT_GT(small, 0U); + EXPECT_LT(large, 3 * small) << "checkpoint jobs: 512 keys=" << small << ", 1024 keys=" << large; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/checkpoint_structure_public_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_structure_public_test.cpp new file mode 100644 index 000000000..723e34146 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/checkpoint_structure_public_test.cpp @@ -0,0 +1,195 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "published_read_test_support.h" +#include "runtime_tree_access.h" +#include "snapshot/operation_wait.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestCheckpointContinuation +{ + MemPageStore store{1}; + + static Config config(PageStore *store) + { + Config value; + value.page_store = store; + value.frame_bytes = 4096; + value.leaf_split_bytes = 160; + return value; + } + + Crowdbtree tree{config(&store)}; + TreeCore &core = RuntimeTreeAccess_for_tests::core(tree); + std::shared_ptr<CheckpointOperation> operation; + std::shared_ptr<ApplyDispatcher> progress; + + ~TestCheckpointContinuation() + { + operation.reset(); + if (progress) { + progress->close_and_drain(); + } + } + + Status large_leaf() const + { + auto before = core.capture(); + auto replacement = std::make_shared<PublishedPage>(); + replacement->revision = before->mapping.get(0)->revision + 1; + replacement->leaf = test_read::page({ + {.key = "a", .value = std::string(80, 'a')}, + {.key = "b", .value = std::string(80, 'b')}, + {.key = "c", .value = std::string(80, 'c')}, + {.key = "d", .value = std::string(80, 'd')}, + {.key = "e", .value = std::string(80, 'e')}, + {.key = "f", .value = std::string(80, 'f')} + }); + replacement->metrics.dirty = replacement->metrics.split_candidate = true; + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = + std::make_shared<InstallRecord>(InstallToken{.generation = before->generation, .sequence = 99}); + request->transaction.pages.push_back( + {.page_id = 0, .expected = before->mapping.get(0), .replacement = std::move(replacement)}); + std::atomic<bool> done{false}; + Status result; + request->on_done = [&](PublicationResult completed) { + result = std::move(completed.status); + done.store(true, std::memory_order_release); + done.notify_all(); + }; + if (!core.publisher().try_submit(request)) { + return Status::unavailable(); + } + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + return result; + } + + Status prepare(std::function<Status()> before_split, size_t *handoffs) + { + using namespace detail; + auto reserved = ResourceBudget::try_reserve(core.binding()->account(), BudgetClass::kReclaim, 1024 * 1024); + if (!reserved) { + return Status::resource_exhausted(); + } + auto credit = AllocationReservation::create(std::move(*reserved)); + auto preparation = reserved_shared<checkpoint_detail::Preparation>(credit); + preparation->core = &core; + preparation->control = credit; + preparation->options = checkpoint_detail::defaults(core); + auto finish = credit->take_reserved(CheckpointOperation::finish_allocation_bound() + 512); + if (!finish) { + return Status::resource_exhausted(); + } + preparation->finish = reserved_shared<AllocationReservation>(credit, std::move(*finish)); + auto status = core.operation_lifetime().acquire(credit, &preparation->ticket); + if (!status.ok()) { + return status; + } + status = ApplyDispatcher::create(core.binding()->cpu(), core.binding()->lane(), 8, credit, &progress); + if (!status.ok()) { + return status; + } + preparation->cpu_progress = progress; + auto &limits = preparation->structure_options; + limits.publication.pages.primary_page_bytes = 4096; + limits.publication.pages.target_page_bytes = 256; + limits.publication.pages.merge_trigger_bytes = 64; + limits.publication.pages.merge_target_bytes = 128; + limits.publication.pages.max_work_entries = 16; + limits.publication.pages.max_metadata_bytes = 1024 * 1024; + limits.publication.max_inner_bytes = 4096; + return CheckpointOperation::create( + preparation, core.publisher(), core.local_backend(), preparation->ticket->backend(core.async_backend()), + core.binding()->cpu(), core.binding()->lane(), preparation->options, + {.bytes = credit, .io = {}, .cpu_progress = progress}, preparation->finish, {}, + [preparation, handoffs](std::shared_ptr<FlushOperation> *out) { + ++*handoffs; + return preparation->core->prepare_flush(out, preparation->ticket); + }, + [preparation](const PublishedTreeState &view, const CapturedWritePlan *plan, CheckpointWork *out) { + return preparation->work(view, plan, out); + }, + &operation, {}, {}, {}, {}, {}, + [preparation, before_split = std::move(before_split)](std::shared_ptr<const PublishedTreeState> view, + uint64_t pid, + std::shared_ptr<PageSplitOperation> *out) { + auto status = before_split(); + return status.ok() ? preparation->structure(std::move(view), pid, out) : status; + }); + } +}; + +TEST(CheckpointContinuation, ConcurrentNewL0IsOutsideTheSingleCapturedFrontier) +{ + TestCheckpointContinuation test; + ASSERT_TRUE(test.tree.apply(1, Batch{{{.key = "seed", .kind = OpKind::kPut, .value = "v"}}}).ok()); + ASSERT_TRUE(test.tree.flush().ok()); + ASSERT_TRUE(test.large_leaf().ok()); + size_t handoffs = 0; + size_t windows = 0; + ASSERT_TRUE(test.prepare( + [&] { + ++windows; + if (windows == 1) { + Batch batch{{{.key = "new", .kind = OpKind::kPut, .value = "later"}}}; + return test.tree.apply(2, batch); + } + return Status::Ok(); + }, + &handoffs) + .ok()); + const auto status = detail::wait_snapshot_operation(test.operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(handoffs, 1U); + EXPECT_GT(windows, 0U); + EXPECT_EQ(test.operation->result().structural_windows, windows); + EXPECT_EQ(test.operation->result().durable_coverage, 1U); + EXPECT_EQ(test.core.capture()->coverage, 1U); + EXPECT_EQ(test.tree.contiguous_slot(), 2U); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(test.tree.get("new", &slot, &value)); + EXPECT_EQ(value, "later"); + EXPECT_EQ(slot, 2U); + test.operation.reset(); + ASSERT_TRUE(test.tree.flush().ok()); + EXPECT_EQ(test.core.capture()->coverage, 2U); + ASSERT_TRUE(test.tree.snapshot().ok()); + EXPECT_EQ(test.core.durable_coverage(), 2U); +} + +TEST(CheckpointContinuation, FailedStructureAdmissionPreservesThePreviousDurableCheckpoint) +{ + TestCheckpointContinuation test; + ASSERT_TRUE(test.tree.apply(1, Batch{{{.key = "seed", .kind = OpKind::kPut, .value = "v"}}}).ok()); + ASSERT_TRUE(test.tree.snapshot().ok()); + const auto sequence = test.core.snapshot_sequence(); + ASSERT_TRUE(test.large_leaf().ok()); + size_t handoffs = 0; + size_t windows = 0; + ASSERT_TRUE(test.prepare( + [&] { + ++windows; + return Status::resource_exhausted(); + }, + &handoffs) + .ok()); + const auto status = detail::wait_snapshot_operation(test.operation); + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_EQ(handoffs, 1U); + EXPECT_EQ(windows, 1U); + EXPECT_FALSE(test.operation->result().durable_committed); + EXPECT_EQ(test.core.durable_coverage(), 1U); + EXPECT_EQ(test.core.snapshot_sequence(), sequence); + EXPECT_FALSE(test.core.capture()->checkpoint_inflight); + EXPECT_FALSE(test.core.local_backend()->checkpoints->needs_recovery()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/checkpoint_structure_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_structure_test.cpp new file mode 100644 index 000000000..1653264d1 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/checkpoint_structure_test.cpp @@ -0,0 +1,173 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "checkpoint_structure_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_structure; + +TEST(CheckpointStructure, OneWindowHasExactCreditAndPublishesOnlyItsFixedTarget) +{ + TestCheckpointStructure test; + ASSERT_TRUE(test.publish(structural_input()).ok()); + const auto before = test.publisher.capture(); + ASSERT_TRUE(test.prepare(before).ok()); + test.begin(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + ASSERT_TRUE(test.terminal.ok()) << test.terminal.to_string(); + const auto after = test.publisher.capture(); + EXPECT_EQ(after->commit_seq, before->commit_seq + 1); + EXPECT_EQ(after->coverage, before->coverage); + EXPECT_GT(after->stats.leaf_pages, 1U); + EXPECT_LE(after->stats.leaf_pages, 4U); + EXPECT_EQ(test.operation->commit_sequence(), after->commit_seq); + EXPECT_EQ(before->mapping.get(0)->leaf->lookup("a").cell.value.to_string(), std::string(80, 'a')); + EXPECT_EQ(test.publisher.work_metrics()->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Checkpoint)] + [static_cast<size_t>(TreeWorkCounter::SplitOperations)], + 1U); +} + +TEST(CheckpointStructure, UnchangedTargetIsExplicitFailureAndNeverLoops) +{ + TestCheckpointStructure test; + ASSERT_TRUE(test.publish(test_read::page({ + {.key = "a", .value = "v"} + })) + .ok()); + const auto before = test.publisher.capture(); + ASSERT_TRUE(test.prepare(before).ok()); + test.begin(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + EXPECT_EQ(test.terminal.code(), Code::kResourceExhausted); + EXPECT_EQ(test.publisher.capture(), before); +} + +TEST(CheckpointStructure, UnvisitedSuffixMayClassifyExactlyOnceWithoutSplitting) +{ + TestCheckpointStructure test; + ASSERT_TRUE(test.publish(test_read::page({ + {.key = "a", .value = "v"} + }), + true) + .ok()); + const auto before = test.publisher.capture(); + ASSERT_TRUE(test.prepare(before).ok()); + test.begin(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + ASSERT_TRUE(test.terminal.ok()) << test.terminal.to_string(); + const auto after = test.publisher.capture(); + EXPECT_EQ(after->commit_seq, before->commit_seq + 1); + EXPECT_EQ(after->stats.leaf_pages, before->stats.leaf_pages); + EXPECT_FALSE(after->mapping.get(0)->metrics.split_candidate); + EXPECT_TRUE(after->mapping.get(0)->metrics.page_size_known); + EXPECT_TRUE(after->mapping.get(0)->metrics.kv_known); + EXPECT_EQ(test.publisher.work_metrics()->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Checkpoint)] + [static_cast<size_t>(TreeWorkCounter::SplitOperations)], + 0U); + test.attempt.reset(); + test.operation.reset(); + test.done.store(false, std::memory_order_release); + ASSERT_TRUE(test.prepare(after).ok()); + test.begin(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + EXPECT_EQ(test.terminal.code(), Code::kResourceExhausted); + EXPECT_EQ(test.publisher.capture(), after); +} + +TEST(CheckpointStructure, ChangedViewRejectsBeforeReadingOrPublishing) +{ + TestCheckpointStructure test; + ASSERT_TRUE(test.publish(structural_input()).ok()); + ASSERT_TRUE(test.prepare(test.publisher.capture()).ok()); + ASSERT_TRUE(test.publish(test_read::page({ + {.key = "new", .value = "v"} + })) + .ok()); + const auto current = test.publisher.capture(); + test.begin(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + EXPECT_EQ(test.terminal.code(), Code::kUnavailable); + EXPECT_EQ(test.publisher.capture(), current); + EXPECT_EQ(test.loader->calls, 0U); +} + +TEST(CheckpointStructure, ColdBaseParticipatesInTheSuccessfulSplit) +{ + TestCheckpointStructure test; + ASSERT_TRUE(test.publish(structural_input(true)).ok()); + test.loader->frames.emplace(1, test_read::base({ + {.key = "b", .value = std::string(80, 'b')} + })); + ASSERT_TRUE(test.prepare(test.publisher.capture()).ok()); + test.begin(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + ASSERT_TRUE(test.terminal.ok()) << test.terminal.to_string(); + EXPECT_EQ(test.loader->calls, 1U); + const auto view = test.publisher.capture(); + uint64_t count = 0; + for (uint64_t pid = 0; pid < view->next_page_id; ++pid) { + const auto page = view->mapping.get(pid); + if (page && page->leaf) { + auto found = page->leaf->lookup("b"); + if (found.state == LogicalLookupState::kValue) { + EXPECT_EQ(found.cell.value.to_string(), std::string(80, 'b')); + ++count; + } + } + } + EXPECT_EQ(count, 1U); +} + +TEST(CheckpointStructure, ConcurrentPublicationDuringColdReadRejectsTheWholeWindow) +{ + TestCheckpointStructure test; + ASSERT_TRUE(test.publish(structural_input(true)).ok()); + test.loader->deferred = true; + test.loader->frames.emplace(1, test_read::base({ + {.key = "b", .value = std::string(80, 'b')} + })); + ASSERT_TRUE(test.prepare(test.publisher.capture()).ok()); + test.begin(); + ASSERT_FALSE(test.done.load(std::memory_order_acquire)); + ASSERT_EQ(test.loader->pending.size(), 1U); + ASSERT_TRUE(test.publish(test_read::page({ + {.key = "new", .value = "v"} + })) + .ok()); + const auto current = test.publisher.capture(); + test.loader->finish(); + test.runtime->drain(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + EXPECT_EQ(test.terminal.code(), Code::kUnavailable); + EXPECT_EQ(test.publisher.capture(), current); +} + +TEST(CheckpointStructure, CancellationWaitsForAcceptedColdReadAndPreservesTheOldView) +{ + TestCheckpointStructure test; + ASSERT_TRUE(test.publish(structural_input(true)).ok()); + const auto before = test.publisher.capture(); + test.loader->deferred = true; + test.loader->frames.emplace(1, test_read::base({ + {.key = "b", .value = std::string(80, 'b')} + })); + ASSERT_TRUE(test.prepare(before).ok()); + test.begin(); + ASSERT_FALSE(test.done.load(std::memory_order_acquire)); + ASSERT_EQ(test.loader->pending.size(), 1U); + test.operation->request_stop(); + test.runtime->drain(); + EXPECT_FALSE(test.done.load(std::memory_order_acquire)); + test.loader->finish(); + test.runtime->drain(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + EXPECT_EQ(test.terminal.code(), Code::kUnavailable); + EXPECT_EQ(test.publisher.capture(), before); + test.operation.reset(); + test.runtime->drain(); + EXPECT_EQ(test.account->snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kReclaim)], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_backing_pin_test.cpp b/lib/crowdb-tree/tests/integration/chunk_backing_pin_test.cpp new file mode 100644 index 000000000..3a9b751d5 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_backing_pin_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_chunk_root.h" + +namespace crowdb::tree +{ +TEST(ChunkBackingPin, RejectsStageAndPackIdentityMismatchBeforeAcquiringProtection) +{ + test_chunk::TestRootBackend test; + auto pages = test.pages(1); + const auto original = pages->frames.begin()->second.backing->chunk->pack; + for (size_t fault = 0; fault < 5; ++fault) { + auto invalid = original; + switch (fault) { + case 0: + ++invalid.chunk.low; + break; + case 1: + ++invalid.offset; + break; + case 2: + --invalid.length; + break; + case 3: + ++invalid.checksum; + break; + case 4: + invalid.purpose = PagePurpose::kPageIndex; + invalid.chunk.high = uint64_t{3} << 56U; + break; + default: + FAIL() << "unexpected corruption case"; + } + detail::ChunkStageId pin; + detail::ChunkId unused; + ASSERT_TRUE(test.ids->next(PagePurpose::kBtreePage, &pin, &unused).ok()); + std::shared_ptr<const ChunkPackBacking> owner; + EXPECT_EQ(pin_chunk_backing(test.domain, invalid, pin, test.credits.credits.bytes, &owner).code(), + Code::kCorruption); + EXPECT_FALSE(owner); + } + EXPECT_EQ(test.catalog->acquire_calls, 0U); +} + +TEST(ChunkBackingPin, DeletingTransitionBetweenExactValidationAndAcquireRejectsTheReader) +{ + test_chunk::TestRootBackend test; + auto pages = test.pages(1); + test.catalog->delete_before_acquire = true; + detail::ChunkStageId pin; + detail::ChunkId unused; + ASSERT_TRUE(test.ids->next(PagePurpose::kBtreePage, &pin, &unused).ok()); + std::shared_ptr<const ChunkPackBacking> owner; + EXPECT_EQ(pin_chunk_backing(test.domain, pages->frames.begin()->second.backing->chunk->pack, pin, + test.credits.credits.bytes, &owner) + .code(), + Code::kUnavailable); + EXPECT_FALSE(owner); + EXPECT_EQ(test.catalog->acquire_calls, 1U); + EXPECT_EQ(test.catalog->queue->pump(1).completed, 1U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_backing_test.cpp b/lib/crowdb-tree/tests/integration/chunk_backing_test.cpp new file mode 100644 index 000000000..8e273be63 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_backing_test.cpp @@ -0,0 +1,142 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/backing_io.h" +#include "btree/page_cache_identity.h" +#include "common/test_chunk_backing.h" +#include "page_writeback_test_support.h" + +#include <future> + +namespace crowdb::tree +{ +using test_chunk::TestChunkBackend; + +TEST(ChunkBacking, SealedWriteCreditProducesDirectLeafReloadWithoutActiveManifest) +{ + TestChunkBackend test; + std::shared_ptr<ChunkPackBacking> pack; + ASSERT_TRUE(test.prepare(8192, &pack).ok()); + PageWriteLimits limits; + limits.max_frame_bytes = 4096; + limits.compression = compress_algo::kNone; + limits.chunk_backed = true; + std::shared_ptr<PageWriteOperation> write; + ASSERT_TRUE(PageWriteOperation::create( + test.scheduled, test.binding->cpu(), test.binding->lane(), test_write::write_input(), limits, + test.credits.credits, + [pack](uint64_t length, PageWriteOperation::ExtentComplete complete) { + complete(Status::Ok(), {.backend_id = 17, + .address = 0, + .length = length, + .content_id = 0, + .generation = 1, + .owner = pack, + .physical_metrics = {}, + .chunk = pack}); + }, + &write) + .ok()); + test.credits.seal(); + ASSERT_TRUE(stdexec::sync_wait(write->sender())); + test.binding->drain(); + const auto backing = write->result().backing; + ASSERT_TRUE(backing); + EXPECT_EQ(backing->storage, BackingStorage::kChunkPack); + EXPECT_TRUE(backing->reloadable()); + EXPECT_EQ(test.empty_device.size(), 0U); + EXPECT_EQ(test.catalog->manifest_loads, 0U); + test_read::LoadCredits reads; + PageLoadLimits read_limits; + read_limits.max_blob_bytes = 8192; + read_limits.max_frame_bytes = 4096; + read_limits.max_chunk_blob_bytes = 8192; + AsyncStorePageLoader loader(test.scheduled, test.binding->cpu(), test.binding->lane(), read_limits, {}, + reads.credits); + std::promise<Status> completed; + std::shared_ptr<const LeafBase> loaded; + loader.load_leaf({.identity = 7, .max_slot = 3, .backing = backing}, [&](Status status, auto leaf) { + loaded = std::move(leaf); + completed.set_value(std::move(status)); + }); + ASSERT_TRUE(completed.get_future().get().ok()); + test.binding->drain(); + ASSERT_TRUE(loaded); + EXPECT_EQ(CellView(loaded->cell(0)).value(), Slice("value")); + EXPECT_EQ(test.catalog->manifest_loads, 0U); + EXPECT_EQ(test.catalog->reader_checks, 4U); +} + +TEST(ChunkBacking, ProcessFenceAfterIoRejectsTheReadBeforeExposingBytes) +{ + TestChunkBackend test; + std::shared_ptr<ChunkPackBacking> pack; + ASSERT_TRUE(test.prepare(32, &pack).ok()); + std::array<uint8_t, 32> input{}; + ASSERT_TRUE(write_chunk_backing(*pack, input, test.credits.credits.bytes).ok()); + test.catalog->fence_at = 4; + auto backing = std::make_shared<BackingRef>(); + backing->backend_id = 17; + backing->length = input.size(); + backing->storage = BackingStorage::kChunkPack; + backing->chunk = pack; + backing->owner = pack; + std::array<uint8_t, 32> output{}; + std::promise<Status> completed; + submit_backing_read( + *test.scheduled, backing, output.data(), output.size(), test.credits.credits.bytes, + {.context = &completed, + .complete_fn = [](void *raw, + Status status) { static_cast<std::promise<Status> *>(raw)->set_value(std::move(status)); }, + .owner = backing}); + EXPECT_EQ(completed.get_future().get().code(), Code::kUnavailable); + test.binding->drain(); + EXPECT_EQ(test.catalog->reader_checks, 4U); +} + +TEST(ChunkBacking, UnknownAllocationRetainsItsExactPaidPinUntilHostReleaseSucceeds) +{ + TestChunkBackend test; + std::shared_ptr<ChunkPackBacking> pack; + ASSERT_TRUE(test.prepare(32, &pack).ok()); + test.transport->unknown_allocate = true; + std::array<uint8_t, 32> input{}; + EXPECT_EQ(write_chunk_backing(*pack, input, test.credits.credits.bytes).code(), Code::kUnavailable); + if (!test.catalog->record) { + FAIL() << "unknown Allocate lost its StageReserved record"; + } + EXPECT_EQ(test.catalog->record->state, detail::ChunkStageState::Reserved); + EXPECT_EQ(test.catalog->queue->snapshot().registered, 1U); + test.catalog->fail_unpin = true; + pack.reset(); + EXPECT_EQ(test.catalog->queue->pump(1).failed, 1U); + EXPECT_EQ(test.catalog->queue->snapshot().registered, 1U); + EXPECT_EQ(test.transport->allocations, 1U); + test.catalog->fail_unpin = false; + EXPECT_EQ(test.catalog->queue->pump(1).completed, 1U); + EXPECT_EQ(test.catalog->queue->snapshot().registered, 0U); +} + +TEST(ChunkBacking, CacheIdentityIncludesExactChunkAndStageInsteadOfSyntheticAddress) +{ + TestChunkBackend test; + std::shared_ptr<ChunkPackBacking> pack; + ASSERT_TRUE(test.prepare(32, &pack).ok()); + auto a = std::make_shared<BackingRef>(); + a->storage = BackingStorage::kChunkPack; + a->backend_id = 17; + a->length = 32; + a->owner = pack; + a->chunk = pack; + auto other = std::make_shared<ChunkPackBacking>(*pack); + auto b = std::make_shared<BackingRef>(*a); + b->chunk = other; + const auto key = page_cache_key(PageCacheKind::Leaf, {.identity = 7, .backing = a}); + EXPECT_EQ(key, page_cache_key(PageCacheKind::Leaf, {.identity = 7, .backing = b})); + ++other->pack.chunk.low; + EXPECT_NE(key, page_cache_key(PageCacheKind::Leaf, {.identity = 7, .backing = b})); + other->pack = pack->pack; + ++other->pack.stage.operation_high; + EXPECT_NE(key, page_cache_key(PageCacheKind::Leaf, {.identity = 7, .backing = b})); + EXPECT_FALSE((BackingExclusion{.address = 0, .length = 4096}).overlaps(*a)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_backing_visibility_test.cpp b/lib/crowdb-tree/tests/integration/chunk_backing_visibility_test.cpp new file mode 100644 index 000000000..29a4549f7 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_backing_visibility_test.cpp @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_physical_registry.h" +#include "btree/backing_stats_sampler.h" +#include "common/test_tree_core_chunk.h" + +namespace crowdb::tree +{ +TEST(ChunkBackingVisibility, PublicNextOwnerKeepsTheLedgerAfterThePreviousTreeCollectorIsDestroyed) +{ + test_chunk::TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, std::string(9000, 'a')).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + auto *core = TreeCoreAccess::get(*test.tree); + core->binding()->cpu()->drain(); + const auto sampler = core->backing_sampler(); + const auto before = sampler.snapshot(); + ASSERT_NE(before.scope_id, 0U); + ASSERT_GT(before.allocated_bytes, 0U); + auto old = test.tree->snapshot_view(); + ASSERT_NE(old, nullptr); + test.tree.reset(); + EXPECT_EQ(sampler.snapshot().scope_id, before.scope_id); + EXPECT_EQ(sampler.snapshot().allocated_bytes, before.allocated_bytes); + ASSERT_TRUE(test.open().ok()); + core = TreeCoreAccess::get(*test.tree); + core->binding()->cpu()->drain(); + const auto reopened = test.tree->backing_stats(); + EXPECT_EQ(reopened.scope_id, before.scope_id); + EXPECT_EQ(reopened.allocated_bytes, before.allocated_bytes); + EXPECT_EQ(test.tree->tree_state_stats().resources.allocated_backing_bytes, before.allocated_bytes); + EXPECT_GT(test.tree->tree_state_stats().resources.allocated_bytes, 0U); + ASSERT_TRUE(test.put(2, std::string(12000, 'b')).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + core->binding()->cpu()->drain(); + const auto newer = test.tree->backing_stats(); + EXPECT_EQ(newer.scope_id, before.scope_id); + EXPECT_GT(newer.allocated_bytes, before.allocated_bytes); + uint64_t actual = 0; + for (const auto &[identity, receipt] : test.backend.transport->receipts) { + (void)identity; + actual += receipt.physical_bytes; + } + EXPECT_EQ(newer.allocated_bytes, actual); + EXPECT_EQ(sampler.snapshot().allocated_bytes, actual); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(old->get("key", &slot, &value)); + EXPECT_EQ(value, std::string(9000, 'a')); + EXPECT_EQ(slot, 1U); + old.reset(); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + TreeCoreAccess::get(*test.tree)->binding()->cpu()->drain(); + EXPECT_EQ(test.tree->backing_stats().scope_id, before.scope_id); + EXPECT_EQ(test.tree->backing_stats().allocated_bytes, actual); + EXPECT_EQ(test.tree->get_view("key").value().to_string(), std::string(12000, 'b')); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_collection_c_api_test.cpp b/lib/crowdb-tree/tests/integration/chunk_collection_c_api_test.cpp new file mode 100644 index 000000000..6b062955a --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_collection_c_api_test.cpp @@ -0,0 +1,123 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_c_api_internal.h" +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "common/test_chunk_collector.h" +#include "crowdb-tree/btree/tree.h" +#include "crowdb-tree/chunk_runtime_c_api.h" +#include "load_budget_test_support.h" + +#include <latch> + +namespace crowdb::tree +{ +namespace +{ +struct TestCollectionCAbi +{ + test_read::LoadCredits credits; + std::shared_ptr<detail::TestCollectorCatalog> catalog = std::make_shared<detail::TestCollectorCatalog>(); + std::shared_ptr<detail::TestFixedTransport> transport = std::make_shared<detail::TestFixedTransport>(catalog); + ct_runtime *runtime = nullptr; + ct_page_store *store = nullptr; + ct_tree *tree = nullptr; + + ~TestCollectionCAbi() + { + if (tree != nullptr) { + ct_close(tree); + } + if (store != nullptr) { + ct_page_store_free(store); + } + if (runtime != nullptr) { + ct_runtime_free(runtime); + } + if (catalog->queue) { + catalog->queue->stop_admission(); + (void)catalog->queue->pump(32); + EXPECT_TRUE(catalog->queue->drained()); + } + } + + ct_status open() + { + auto status = detail::ChunkPinReleaseQueue::create( + credits.credits.bytes, 32, std::make_shared<int>(1), [](void * /*unused*/) noexcept {}, nullptr, + &catalog->queue); + if (!status.ok()) { + return static_cast<ct_status>(status.code()); + } + auto result = ct_runtime_create(nullptr, &runtime); + if (result != 0) { + return result; + } + ct_root_catalog roots{.catalog = catalog, .transport = transport}; + ct_chunk_transport chunks{.transport = transport}; + const ct_chunk_runtime_options options{.version = 1, + .struct_bytes = sizeof(ct_chunk_runtime_options), + .tree_id = 17, + .owner_epoch = 1, + .open_generation = 0, + .iu_size = 4096, + .mirror_copies = 2, + .max_chunk_bytes = 256 * 1024 * 1024}; + result = ct_chunk_runtime_page_store_open(runtime, &options, &roots, &chunks, &store); + if (result != 0) { + return result; + } + ct_options config{}; + config.page_store = store; + config.frame_bytes = 4096; + return ct_open_with_runtime(&config, runtime, nullptr, &tree); + } +}; +} // namespace + +TEST(ChunkCollectionCAbi, TypedReportSurvivesTreeDrainAndRejectsWrongVersion) +{ + TestCollectionCAbi test; + ASSERT_EQ(test.open(), 0); + test.catalog->record = detail::test_collector_stage(); + ct_future *future = nullptr; + ASSERT_EQ(ct_chunk_collect_async(test.tree, &future), 0); + std::unique_ptr<ct_future, decltype(&ct_future_free)> handle(future, &ct_future_free); + ct_chunk_collection_result_v1 result{}; + int32_t done = 0; + EXPECT_EQ(ct_future_poll_chunk_collection(handle.get(), &done, &result), + static_cast<ct_status>(Code::kInvalidArgument)); + TreeCoreAccess::get(*c_api_tree(test.tree))->close_and_drain(); + result.struct_size = sizeof(result); + result.version = 1; + ASSERT_EQ(ct_future_poll_chunk_collection(handle.get(), &done, &result), 0); + ASSERT_EQ(done, 1); + EXPECT_EQ(handle.release(), future); + EXPECT_EQ(result.terminal_status, 0); + EXPECT_EQ(result.stages_removed, 1U); + EXPECT_EQ(result.logical_capacity_bytes, 4096U); + EXPECT_EQ(test.transport->deletions, 1U); +} + +TEST(ChunkCollectionCAbi, DroppedFutureRetainsAcceptedDeleteThroughClose) +{ + TestCollectionCAbi test; + ASSERT_EQ(test.open(), 0); + test.catalog->record = detail::test_collector_stage(); + std::latch entered(1); + std::latch release(1); + test.transport->before_drain = [&] { + entered.count_down(); + release.wait(); + }; + ct_future *future = nullptr; + ASSERT_EQ(ct_chunk_collect_async(test.tree, &future), 0); + entered.wait(); + ct_future_free(future); + release.count_down(); + ct_close(test.tree); + test.tree = nullptr; + EXPECT_EQ(test.transport->deletions, 1U); + EXPECT_FALSE(test.catalog->record); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_collection_operation_test.cpp b/lib/crowdb-tree/tests/integration/chunk_collection_operation_test.cpp new file mode 100644 index 000000000..cbd1f7eb8 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_collection_operation_test.cpp @@ -0,0 +1,168 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_collection_operation.h" +#include "backend/chunk/chunk_runtime_page_store.h" +#include "btree/tree_core_access.h" +#include "common/test_chunk_collector.h" +#include "load_budget_test_support.h" +#include "snapshot/operation_wait.h" + +#include <future> +#include <latch> + +namespace crowdb::tree +{ +namespace +{ +struct TestCollectionCore +{ + test_read::LoadCredits credits; + std::shared_ptr<detail::TestCollectorCatalog> catalog = std::make_shared<detail::TestCollectorCatalog>(); + std::shared_ptr<detail::TestFixedTransport> transport = std::make_shared<detail::TestFixedTransport>(catalog); + Config config; + std::shared_ptr<detail::ChunkRuntimePageStore> store; + std::unique_ptr<Crowdbtree> tree; + + TestCollectionCore() + { + EXPECT_TRUE( + detail::ChunkPinReleaseQueue::create( + credits.credits.bytes, 32, std::make_shared<int>(1), [](void *) noexcept {}, nullptr, &catalog->queue) + .ok()); + TreeRuntimeOptions options; + options.workers = 1; + options.backend_workers = 1; + EXPECT_TRUE(TreeRuntime::create(options, &config.runtime).ok()); + store = std::make_shared<detail::ChunkRuntimePageStore>( + detail::ChunkRuntimeOptions{.tree = 17, + .owner_epoch = 1, + .open_generation = 0, + .iu_size = 4096, + .mirrors = 2, + .max_chunk_bytes = 256 * 1024 * 1024}, + catalog, transport, credits.credits.bytes); + config.page_store = store.get(); + config.backend_owner = store; + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.runtime_binding.limits.io_operations = 1; + } + + ~TestCollectionCore() + { + tree.reset(); + config.backend_owner.reset(); + store.reset(); + config.runtime.reset(); + catalog->queue->stop_admission(); + (void)catalog->queue->pump(32); + EXPECT_TRUE(catalog->queue->drained()); + } + + Status open() + { + return Crowdbtree::open(config, &tree); + } + + Status prepare(std::shared_ptr<ChunkCollectionOperation> *out) const + { + return TreeCoreAccess::get(*tree)->prepare_chunk_collection(out); + } +}; +} // namespace + +TEST(ChunkCollectionOperation, SingleIoCoreCollectionDeletesOnlyAfterPinsLeave) +{ + TestCollectionCore test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.catalog->record = detail::test_collector_stage(); + test.catalog->pins = 65; + std::shared_ptr<ChunkCollectionOperation> operation; + ASSERT_TRUE(test.prepare(&operation).ok()); + status = detail::wait_snapshot_operation(operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(test.transport->deletions, 0U); + EXPECT_EQ(operation->result().pins_examined, 65U); + EXPECT_LE(test.catalog->largest_pin_window, 64U); + operation.reset(); + test.catalog->pins = 0; + ASSERT_TRUE(test.prepare(&operation).ok()); + status = detail::wait_snapshot_operation(operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(operation->result().stages_removed, 1U); + EXPECT_EQ(operation->result().logical_capacity_bytes, 4096U); + EXPECT_EQ(test.transport->drains, 1U); + EXPECT_EQ(test.transport->deletions, 1U); + EXPECT_FALSE(test.catalog->record); +} + +TEST(ChunkCollectionOperation, CancelBeforeStartKeepsStageAndReturnsItsAdmission) +{ + TestCollectionCore test; + ASSERT_TRUE(test.open().ok()); + test.catalog->record = detail::test_collector_stage(); + std::shared_ptr<ChunkCollectionOperation> operation; + ASSERT_TRUE(test.prepare(&operation).ok()); + operation->request_stop(); + EXPECT_EQ(detail::wait_snapshot_operation(operation).code(), Code::kUnavailable); + EXPECT_EQ(test.transport->deletions, 0U); + EXPECT_TRUE(test.catalog->record); + operation.reset(); + ASSERT_TRUE(test.prepare(&operation).ok()); + EXPECT_TRUE(detail::wait_snapshot_operation(operation).ok()); + EXPECT_EQ(test.transport->deletions, 1U); +} + +TEST(ChunkCollectionOperation, CloseDrainsAcceptedDeleteWithoutWaitingForCallerHandle) +{ + TestCollectionCore test; + ASSERT_TRUE(test.open().ok()); + test.catalog->record = detail::test_collector_stage(); + std::latch entered(1); + std::latch release(1); + std::latch close_started(1); + test.transport->before_drain = [&] { + entered.count_down(); + release.wait(); + }; + std::shared_ptr<ChunkCollectionOperation> operation; + ASSERT_TRUE(test.prepare(&operation).ok()); + auto run = std::async(std::launch::async, [&] { return detail::wait_snapshot_operation(operation); }); + entered.wait(); + auto close = std::async(std::launch::async, [&] { + close_started.count_down(); + TreeCoreAccess::get(*test.tree)->close_and_drain(); + }); + close_started.wait(); + EXPECT_EQ(close.wait_for(std::chrono::milliseconds(1)), std::future_status::timeout); + release.count_down(); + EXPECT_TRUE(run.get().ok()); + close.get(); + EXPECT_EQ(operation->result().stages_removed, 1U); + EXPECT_EQ(test.transport->deletions, 1U); + EXPECT_TRUE(operation); +} + +TEST(ChunkCollectionOperation, LostDeleteAckKeepsDeletingAndAbsentRetryConverges) +{ + TestCollectionCore test; + ASSERT_TRUE(test.open().ok()); + test.catalog->record = detail::test_collector_stage(); + test.transport->unknown_delete = true; + test.transport->absent_after_delete = true; + std::shared_ptr<ChunkCollectionOperation> operation; + ASSERT_TRUE(test.prepare(&operation).ok()); + EXPECT_EQ(detail::wait_snapshot_operation(operation).code(), Code::kUnavailable); + ASSERT_TRUE(test.catalog->record); + EXPECT_EQ(test.catalog->record->state, detail::ChunkStageState::Deleting); + EXPECT_EQ(operation->result().stages_removed, 0U); + operation.reset(); + test.transport->unknown_delete = false; + ASSERT_TRUE(test.prepare(&operation).ok()); + EXPECT_TRUE(detail::wait_snapshot_operation(operation).ok()); + EXPECT_EQ(operation->result().stages_removed, 1U); + EXPECT_EQ(test.transport->deletions, 2U); + EXPECT_FALSE(test.catalog->record); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_legacy_runtime_open_test.cpp b/lib/crowdb-tree/tests/integration/chunk_legacy_runtime_open_test.cpp new file mode 100644 index 000000000..7101c0549 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_legacy_runtime_open_test.cpp @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +TEST(ChunkLegacyRuntimeOpen, RejectsTheMigrationOnlyStoreWithAndWithoutAsyncInjection) +{ + for (const bool asynchronous : {false, true}) { + SCOPED_TRACE(asynchronous); + auto catalog = std::make_shared<MemoryRootCatalog>(1); + auto transport = std::make_shared<MemoryChunkTransport>(); + ChunkPageStore store({.tree_id = 29, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, transport); + ASSERT_TRUE(store.open_status().ok()); + crowdb::tree::Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = &store; + config.async_page_store = asynchronous ? &store : nullptr; + config.frame_bytes = 4096; + std::unique_ptr<Crowdbtree> tree; + const auto status = Crowdbtree::open(config, &tree); + EXPECT_EQ(status.code(), Code::kNotSupported) << status.to_string(); + EXPECT_NE(status.to_string().find("migrate_legacy_chunk"), std::string::npos); + EXPECT_FALSE(tree); + EXPECT_EQ(store.stats().generations_published, 0U); + } +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp b/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp index 5d7d84dcf..41ea69614 100644 --- a/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp @@ -22,13 +22,6 @@ namespace crowdb::tree::detail namespace { -Batch put(uint64_t /*unused*/, std::string key, std::string value) -{ - Batch batch; - batch.ops.push_back({.key = std::move(key), .kind = OpKind::kPut, .value = std::move(value)}); - return batch; -} - struct CompletionState { std::atomic<bool> done{false}; @@ -556,24 +549,22 @@ TEST(ChunkPageStore, ShutdownCancelsAndDrainsBlockedPackWrites) EXPECT_EQ(catalog->load(34), nullptr); } -TEST(ChunkPageStore, SnapshotPublishesBoundedChecksummedPacksAndReopens) +TEST(ChunkPageStore, LegacyTypedPacksRemainBoundedChecksummedAndReopen) { auto catalog = std::make_shared<MemoryRootCatalog>(7); auto transport = std::make_shared<MemoryChunkTransport>(); ChunkPageStore::Config config{ .tree_id = 42, .owner_epoch = 7, .pack_bytes = 4096, .mirror_copies = 3, .iu_size = 1}; { - ChunkPageStore store(config, catalog, transport); - Config options; - options.page_store = &store; - options.frame_bytes = 4096; - options.leaf_split_bytes = 512; - Crowdbtree tree(options); - for (uint64_t i = 1; i <= 80; ++i) { - ASSERT_TRUE(tree.apply(i, put(i, "key" + std::to_string(i), std::string(256, 'x'))).ok()); - } - ASSERT_TRUE(tree.flush().ok()); - ASSERT_TRUE(tree.snapshot().ok()); + ChunkPageStore store(config, catalog, transport); + const std::vector<uint8_t> page_bytes(12000, 'x'); + const std::vector<uint8_t> index_bytes(6000, 'i'); + ASSERT_TRUE(store.write_typed_at(PagePurpose::kBtreePage, 8192, page_bytes.data(), page_bytes.size()).ok()); + ASSERT_TRUE(store.write_typed_at(PagePurpose::kPageIndex, 24576, index_bytes.data(), index_bytes.size()).ok()); + ASSERT_TRUE(store.sync().ok()); + const uint8_t anchor = 1; + ASSERT_TRUE(store.write_at(0, &anchor, 1).ok()); + ASSERT_TRUE(store.sync().ok()); auto manifest = catalog->load(42); ASSERT_NE(manifest, nullptr); ASSERT_GT(manifest->packs.size(), 1U); @@ -605,19 +596,13 @@ TEST(ChunkPageStore, SnapshotPublishesBoundedChecksummedPacksAndReopens) EXPECT_EQ(store.stats().mirror_write_attempts, manifest->packs.size() * 3U); } - ChunkPageStore reopened_store(config, catalog, transport); - Config reopened_options; - reopened_options.page_store = &reopened_store; - reopened_options.frame_bytes = 4096; - reopened_options.leaf_split_bytes = 512; - std::unique_ptr<Crowdbtree> reopened; - ASSERT_TRUE(Crowdbtree::open(reopened_options, &reopened).ok()); - EXPECT_TRUE(slot_word::is_page_ref(reopened->mapping().get_word(reopened->root_page_id()))); - std::string value; - uint64_t slot = 0; - ASSERT_TRUE(reopened->get(Slice("key40"), &slot, &value)); - EXPECT_EQ(slot, 40U); - EXPECT_EQ(value, std::string(256, 'x')); + ChunkPageStore reopened_store(config, catalog, transport); + std::vector<uint8_t> page_bytes(12000); + std::vector<uint8_t> index_bytes(6000); + ASSERT_TRUE(reopened_store.read_at(8192, page_bytes.data(), page_bytes.size()).ok()); + ASSERT_TRUE(reopened_store.read_at(24576, index_bytes.data(), index_bytes.size()).ok()); + EXPECT_EQ(page_bytes, std::vector<uint8_t>(12000, 'x')); + EXPECT_EQ(index_bytes, std::vector<uint8_t>(6000, 'i')); } TEST(ChunkPageStore, CheckpointCowsOnlyChangedLogicalPacks) @@ -1188,29 +1173,25 @@ TEST(ChunkPageStore, EpochFailureLeavesPriorRootAndAccountsOrphans) auto catalog = std::make_shared<MemoryRootCatalog>(3); auto transport = std::make_shared<MemoryChunkTransport>(); ChunkPageStore store({.tree_id = 9, .owner_epoch = 3, .pack_bytes = 4096, .iu_size = 1}, catalog, transport); - Config options; - options.page_store = &store; - Crowdbtree tree(options); - ASSERT_TRUE(tree.apply(1, put(1, "a", "old")).ok()); - ASSERT_TRUE(tree.flush().ok()); - ASSERT_TRUE(tree.snapshot().ok()); + publish_raw_generation(&store, 1); auto prior = catalog->load(9); ASSERT_NE(prior, nullptr); - ASSERT_TRUE(tree.apply(2, put(2, "a", "new")).ok()); - ASSERT_TRUE(tree.flush().ok()); + const uint8_t changed = 2; + ASSERT_TRUE(store.write_at(8192, &changed, 1).ok()); + ASSERT_TRUE(store.sync().ok()); + ASSERT_TRUE(store.write_at(0, &changed, 1).ok()); catalog->set_owner_epoch(4); - Status failed = tree.snapshot(); + const Status failed = store.sync(); EXPECT_EQ(failed.code(), Code::kUnavailable); EXPECT_EQ(catalog->load(9)->generation, prior->generation); EXPECT_GT(store.stats().orphan_bytes, 0U); EXPECT_GT(catalog->reference_segment_count(9), prior->reference_segments.size()); EXPECT_GT(store.reclaim_orphans(), 0U); EXPECT_EQ(catalog->reference_segment_count(9), prior->reference_segments.size()); - std::string value; - uint64_t slot = 0; - ASSERT_TRUE(tree.get(Slice("a"), &slot, &value)); - EXPECT_EQ(value, "new"); + uint8_t value = 0; + ASSERT_TRUE(store.read_at(8192, &value, 1).ok()); + EXPECT_EQ(value, 1U); } TEST(ChunkPageStore, FailedAllReusedPublicationDoesNotAccountSourceBytesAsOrphans) @@ -1663,44 +1644,21 @@ TEST(ChunkPageStore, AsyncUnavailableRemainsTypedAndDoesNotLatchCorruption) auto catalog = std::make_shared<MemoryRootCatalog>(1); auto transport = std::make_shared<MemoryChunkTransport>(); ChunkPageStore store({.tree_id = 15, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, transport); - Config options; - options.page_store = &store; - options.async_page_store = &store; - options.frame_bytes = 4096; - Crowdbtree tree(options); - ASSERT_TRUE(tree.apply(1, put(1, "key", "value")).ok()); - ASSERT_TRUE(tree.flush().ok()); - ASSERT_TRUE(tree.snapshot().ok()); - ASSERT_GT(tree.evict_clean_leaves(0), 0U); - + publish_raw_generation(&store, 9); + uint8_t value = 0; store.inject_unavailable(true); - Status unavailable; - GetView missing; - std::atomic<bool> unavailable_done{false}; - tree.get_async(Slice("key"), [&](Status status, GetView result) { - unavailable = std::move(status); - missing = std::move(result); - unavailable_done.store(true, std::memory_order_release); - unavailable_done.notify_one(); - }); - unavailable_done.wait(false, std::memory_order_acquire); - EXPECT_EQ(unavailable.code(), Code::kUnavailable); - EXPECT_FALSE(missing.found()); - EXPECT_FALSE(tree.io_failed()); + CompletionState unavailable; + ASSERT_NE(store.submit_read(8192, &value, 1, {.context = &unavailable, .complete_fn = &record_completion}), 0U); + unavailable.done.wait(false, std::memory_order_acquire); + EXPECT_EQ(unavailable.code.load(std::memory_order_relaxed), static_cast<int>(Code::kUnavailable)); + EXPECT_EQ(value, 0U); store.inject_unavailable(false); - Status recovered; - GetView found; - std::atomic<bool> recovered_done{false}; - tree.get_async(Slice("key"), [&](Status status, GetView result) { - recovered = std::move(status); - found = std::move(result); - recovered_done.store(true, std::memory_order_release); - recovered_done.notify_one(); - }); - recovered_done.wait(false, std::memory_order_acquire); - EXPECT_TRUE(recovered.ok()) << recovered.to_string(); - EXPECT_TRUE(found.found()); + CompletionState recovered; + ASSERT_NE(store.submit_read(8192, &value, 1, {.context = &recovered, .complete_fn = &record_completion}), 0U); + recovered.done.wait(false, std::memory_order_acquire); + EXPECT_EQ(recovered.code.load(std::memory_order_relaxed), static_cast<int>(Code::kOk)); + EXPECT_EQ(value, 9U); } TEST(ChunkPageStore, AsyncQueueIsBoundedAndRunsOffSubmitter) @@ -1773,7 +1731,7 @@ TEST(ChunkPageStore, CompletionMayReleaseFinalStoreOwner) completion.done.wait(false, std::memory_order_acquire); } -TEST(ChunkPageStore, CApiFactoryInjectsBackendWithoutChangingOpen) +TEST(ChunkPageStore, CApiLegacyBackendRequiresExplicitMigration) { ct_chunk_transport *invalid_transport = nullptr; ct_chunk_rpc_transport_options invalid_options = {}; @@ -1799,45 +1757,13 @@ TEST(ChunkPageStore, CApiFactoryInjectsBackendWithoutChangingOpen) options.page_store = store; options.frame_bytes = 4096; ct_tree *tree = nullptr; - ASSERT_EQ(ct_open(&options, &tree), 0); - ASSERT_EQ(ct_apply_put(tree, 1, reinterpret_cast<const uint8_t *>("key"), 3, - reinterpret_cast<const uint8_t *>("value"), 5), - 0); - ASSERT_EQ(ct_flush(tree), 0); - uint64_t slot = 0; - ASSERT_EQ(ct_snapshot(tree, &slot), 0); - EXPECT_EQ(slot, 1U); - uint64_t materialized_bytes = 1; - int32_t materialized = 0; - ASSERT_EQ(ct_materialize_ownership(tree, &materialized_bytes, &materialized), 0); - EXPECT_EQ(materialized_bytes, 0U); - EXPECT_EQ(materialized, 1); - ASSERT_GT(ct_evict_clean_leaves(tree, 0), 0U); - ct_future *future = ct_get_async(tree, reinterpret_cast<const uint8_t *>("key"), 3); - ASSERT_NE(future, nullptr); - int32_t done = 0; - int32_t found = 0; - ct_buf value = {}; - ct_status future_status = 0; - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do { - future_status = ct_future_poll(future, &done, &found, &slot, &value); - if (done == 0) { - std::this_thread::yield(); - } - } while (done == 0 && std::chrono::steady_clock::now() < deadline); - ASSERT_EQ(future_status, 0); - EXPECT_EQ(done, 1); - EXPECT_EQ(found, 1); - EXPECT_EQ(std::string(reinterpret_cast<char *>(value.data), value.len), "value"); - ct_future_free(future); + EXPECT_EQ(ct_open(&options, &tree), static_cast<ct_status>(Code::kNotSupported)); + EXPECT_EQ(tree, nullptr); ct_chunk_page_store_stats stats = {}; ASSERT_EQ(ct_chunk_page_store_get_stats(store, &stats), 0); - EXPECT_EQ(stats.generations_published, 1U); - EXPECT_GT(stats.packs_written, 0U); - EXPECT_EQ(stats.mirror_write_attempts, stats.packs_written * 5U); + EXPECT_EQ(stats.generations_published, 0U); + EXPECT_EQ(stats.packs_written, 0U); ct_page_store_free(store); - ct_close(tree); ct_root_catalog_free(catalog); } diff --git a/lib/crowdb-tree/tests/integration/chunk_physical_core_test.cpp b/lib/crowdb-tree/tests/integration/chunk_physical_core_test.cpp new file mode 100644 index 000000000..75d7956d4 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_physical_core_test.cpp @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_tree_core_chunk.h" + +namespace crowdb::tree +{ +TEST(ChunkPhysicalCore, SingleIoCheckpointReportsAllPhysicalMirrorAllocationsIncludingIndexPacks) +{ + test_chunk::TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, std::string(20000, 'x')).ok()); + auto status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + uint64_t expected = 0; + bool index = false; + for (const auto &[identity, receipt] : test.backend.transport->receipts) { + (void)identity; + expected += receipt.physical_bytes; + index |= detail::chunk_purpose(receipt.chunk) == PagePurpose::kPageIndex; + } + ASSERT_TRUE(index); + ASSERT_GT(expected, 0U); + EXPECT_EQ(test.tree->tree_state_stats().resources.allocated_backing_bytes, expected); + const auto first = expected; + ASSERT_TRUE(test.put(2, "small").ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + expected = 0; + for (const auto &[identity, receipt] : test.backend.transport->receipts) { + (void)identity; + expected += receipt.physical_bytes; + } + EXPECT_GT(expected, first); + EXPECT_EQ(test.tree->tree_state_stats().resources.allocated_backing_bytes, expected); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_root_commit_test.cpp b/lib/crowdb-tree/tests/integration/chunk_root_commit_test.cpp new file mode 100644 index 000000000..ee314e6f2 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_root_commit_test.cpp @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "common/test_chunk_root.h" +#include "snapshot/chunk_root_codec_internal.h" + +namespace crowdb::tree +{ +namespace +{ +void complete(const std::shared_ptr<ChunkRootCommit> &operation) +{ + for (size_t i = 0; i < 100 && !operation->done(); ++i) { + (void)operation->step(); + } + ASSERT_TRUE(operation->done()); +} +} // namespace + +TEST(ChunkRootCommit, LostMutationRepliesResolveTheSameCandidateWithoutRepeatingReferenceWindows) +{ + test_chunk::TestRootBackend test; + auto pages = test.pages(65); + auto operation = test.commit(pages); + test.catalog->lose_begin = true; + test.catalog->lose_prepare = true; + test.catalog->lose_commit = true; + complete(operation); + ASSERT_TRUE(operation->status().ok()) << operation->status().to_string(); + EXPECT_TRUE(operation->committed()); + EXPECT_FALSE(operation->needs_recovery()); + EXPECT_EQ(test.catalog->begin_calls, 1U); + EXPECT_EQ(test.catalog->prepare_calls, 2U); + EXPECT_EQ(test.catalog->commit_calls, 1U); + EXPECT_EQ(test.catalog->abort_calls, 0U); + ChunkRootManifest root(test.credits.credits.bytes); + ASSERT_TRUE(decode_chunk_root(test.catalog->manifest, 19, 7, &root).ok()); + EXPECT_EQ(root.frame_count, 65U); + EXPECT_EQ(root.indexes.size(), 2U); + EXPECT_EQ(root.identity.coverage, 15U); + EXPECT_EQ(root.identity.wal_replay_offset, 4096U); + EXPECT_EQ(root.stored_references, 0U); + EXPECT_EQ(root.dependency_entries, 0U); + detail::ChunkStageId pin; + detail::ChunkId unused; + ASSERT_TRUE(test.ids->next(PagePurpose::kPageIndex, &pin, &unused).ok()); + std::shared_ptr<const ChunkPackBacking> index; + ASSERT_TRUE(pin_chunk_backing(test.domain, root.indexes[0].pack, pin, test.credits.credits.bytes, &index).ok()); + std::vector<uint8_t> bytes(index->pack.length); + ASSERT_TRUE(read_chunk_backing(*index, bytes, test.credits.credits.bytes).ok()); + uint64_t count = 0; + size_t offset = 0; + ASSERT_TRUE(decode_chunk_index_header(bytes, root.identity, 0, &count, &offset).ok()); + EXPECT_EQ(count, 64U); +} + +TEST(ChunkRootCommit, KnownFailedCommitAbortsAndReleasesEveryAcceptedWindow) +{ + test_chunk::TestRootBackend test; + auto operation = test.commit(test.pages(65)); + test.catalog->fail_commit = true; + complete(operation); + EXPECT_EQ(operation->status().code(), Code::kUnavailable); + EXPECT_FALSE(operation->committed()); + EXPECT_FALSE(operation->needs_recovery()); + EXPECT_EQ(test.catalog->abort_calls, 1U); + EXPECT_EQ(test.catalog->release_calls, 2U); +} + +TEST(ChunkRootCommit, UnqueryableCommitKeepsCandidateProtectionAndDoesNotReportDurability) +{ + test_chunk::TestRootBackend test; + auto operation = test.commit(test.pages(1)); + test.catalog->lose_commit = true; + test.catalog->fail_query = true; + complete(operation); + EXPECT_EQ(operation->status().code(), Code::kUnavailable); + EXPECT_FALSE(operation->committed()); + EXPECT_TRUE(operation->needs_recovery()); + if (!test.catalog->candidate) { + FAIL() << "candidate protection disappeared"; + } + EXPECT_EQ(test.catalog->candidate->state, detail::ChunkCandidateState::Committed); + EXPECT_EQ(test.catalog->abort_calls, 0U); + EXPECT_EQ(test.catalog->release_calls, 0U); +} + +TEST(ChunkRootCommit, CancellationReleasesOnlyTheWindowsThatWereActuallyPrepared) +{ + test_chunk::TestRootBackend test; + auto operation = test.commit(test.pages(65)); + for (size_t i = 0; i < 30 && test.catalog->prepare_calls == 0; ++i) { + ASSERT_TRUE(operation->step().ok()); + } + ASSERT_EQ(test.catalog->prepare_calls, 1U); + ASSERT_TRUE(operation->step(true).ok()); + complete(operation); + EXPECT_EQ(operation->status().code(), Code::kUnavailable); + EXPECT_EQ(test.catalog->prepare_calls, 1U); + EXPECT_EQ(test.catalog->commit_calls, 0U); + EXPECT_EQ(test.catalog->release_calls, 1U); +} + +TEST(ChunkRootCommit, AdmittedMetadataAndOneWindowFinishWithHostCreditExhausted) +{ + test_chunk::TestRootBackend test; + auto pages = test.pages(65); + const auto metadata = ChunkRootCommit::allocation_bound(65); + const auto window = ChunkRootCommit::window_allocation_bound(0); + if (!metadata || !window) { + FAIL() << "checkpoint bound overflow"; + } + detail::ResourceBudget::Limits limits{}; + const auto category = static_cast<size_t>(detail::BudgetClass::kRouting); + limits[category] = *metadata + *window + detail::shared_allocation_bound<detail::AllocationReservation>(); + detail::ResourceBudget budget(limits, 4); + auto account = budget.make_account(limits); + auto grant = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, limits[category]); + if (!grant) { + FAIL() << "checkpoint reservation failed"; + } + auto credit = detail::AllocationReservation::create(std::move(*grant)); + EXPECT_FALSE(detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, 1)); + ChunkRootIdentity identity{.tree = 19, + .root_generation = 7, + .owner_epoch = 1, + .snapshot_sequence = 4, + .page_generation = 3, + .coverage = 15, + .gc_floor = 8, + .root_page_id = 0, + .next_page_id = 65, + .wal_replay_offset = 4096}; + std::shared_ptr<ChunkRootCommit> operation; + ASSERT_TRUE(ChunkRootCommit::create(pages, test.domain, test.ids, identity, credit, credit, &operation).ok()); + complete(operation); + ASSERT_TRUE(operation->status().ok()) << operation->status().to_string(); + EXPECT_TRUE(operation->committed()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_runtime_c_api_test.cpp b/lib/crowdb-tree/tests/integration/chunk_runtime_c_api_test.cpp new file mode 100644 index 000000000..47d38412d --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_runtime_c_api_test.cpp @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_c_api_internal.h" +#include "c_api_internal.h" +#include "common/test_chunk_root.h" +#include "crowdb-tree/chunk_runtime_c_api.h" +#include "runtime/tree_runtime_c_api.h" + +namespace crowdb::tree +{ +namespace +{ +ct_chunk_runtime_options runtime_options() +{ + return {.version = 1, + .struct_bytes = sizeof(ct_chunk_runtime_options), + .tree_id = 19, + .owner_epoch = 1, + .open_generation = 0, + .iu_size = 4096, + .mirror_copies = 2, + .max_chunk_bytes = 256 * 1024 * 1024}; +} +} // namespace + +TEST(ChunkRuntimeCAbi, VersionedFactoryChargesAndReleasesItsActualOwners) +{ + test_chunk::TestRootBackend backend; + ct_runtime *raw_runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &raw_runtime), 0); + std::unique_ptr<ct_runtime, decltype(&ct_runtime_free)> runtime(raw_runtime, &ct_runtime_free); + Config config; + ASSERT_TRUE(configure_tree_runtime(runtime.get(), nullptr, &config).ok()); + const auto baseline = config.runtime->snapshot().allocated.routing_bytes; + ct_root_catalog catalog{.catalog = backend.catalog, .transport = backend.transport}; + ct_chunk_transport transport{.transport = backend.transport}; + auto options = runtime_options(); + ct_page_store *raw_store = nullptr; + --options.struct_bytes; + EXPECT_EQ(ct_chunk_runtime_page_store_open(runtime.get(), &options, &catalog, &transport, &raw_store), + static_cast<ct_status>(Code::kInvalidArgument)); + EXPECT_EQ(raw_store, nullptr); + EXPECT_EQ(config.runtime->snapshot().allocated.routing_bytes, baseline); + ++options.struct_bytes; + ASSERT_EQ(ct_chunk_runtime_page_store_open(runtime.get(), &options, &catalog, &transport, &raw_store), 0); + EXPECT_GT(config.runtime->snapshot().allocated.routing_bytes, baseline); + ct_page_store_free(raw_store); + EXPECT_EQ(config.runtime->snapshot().allocated.routing_bytes, baseline); +} + +TEST(ChunkRuntimeCAbi, PublicTreeUsesSameHostAndCommitsWalWithTheRoot) +{ + test_chunk::TestRootBackend backend; + backend.catalog->current_generation = 0; + ct_runtime *raw_runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &raw_runtime), 0); + std::unique_ptr<ct_runtime, decltype(&ct_runtime_free)> runtime(raw_runtime, &ct_runtime_free); + ct_root_catalog catalog{.catalog = backend.catalog, .transport = backend.transport}; + ct_chunk_transport transport{.transport = backend.transport}; + const auto chunk_options = runtime_options(); + ct_page_store *raw_store = nullptr; + ASSERT_EQ(ct_chunk_runtime_page_store_open(runtime.get(), &chunk_options, &catalog, &transport, &raw_store), 0); + std::unique_ptr<ct_page_store, decltype(&ct_page_store_free)> store(raw_store, &ct_page_store_free); + ct_options options{}; + options.page_store = store.get(); + options.frame_bytes = 4096; + options.max_inline_value = 64; + ct_runtime *raw_other = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &raw_other), 0); + std::unique_ptr<ct_runtime, decltype(&ct_runtime_free)> other(raw_other, &ct_runtime_free); + ct_tree *raw_tree = nullptr; + EXPECT_EQ(ct_open_with_runtime(&options, other.get(), nullptr, &raw_tree), + static_cast<ct_status>(Code::kInvalidArgument)); + ASSERT_EQ(ct_open_with_runtime(&options, runtime.get(), nullptr, &raw_tree), 0); + std::unique_ptr<ct_tree, decltype(&ct_close)> tree(raw_tree, &ct_close); + const uint8_t key = 'k'; + const std::string value(20000, 'v'); + ASSERT_EQ(ct_apply_put(tree.get(), 1, &key, 1, reinterpret_cast<const uint8_t *>(value.data()), value.size()), 0); + ASSERT_EQ(ct_chunk_page_store_set_wal_replay_offset(store.get(), 4096), 0); + uint64_t coverage = 0; + ASSERT_EQ(ct_snapshot(tree.get(), &coverage), 0); + EXPECT_EQ(coverage, 1U); + uint64_t wal = 0; + uint64_t root = 0; + ASSERT_EQ(ct_chunk_page_store_get_wal_replay_offset(store.get(), &wal), 0); + ASSERT_EQ(ct_chunk_page_store_get_manifest_generation(store.get(), &root), 0); + EXPECT_EQ(wal, 4096U); + EXPECT_EQ(root, 1U); + tree.reset(); + raw_tree = nullptr; + EXPECT_EQ(ct_open_with_runtime(&options, runtime.get(), nullptr, &raw_tree), + static_cast<ct_status>(Code::kUnavailable)); + EXPECT_EQ(backend.catalog->writer_releases, 0U); + EXPECT_GT(backend.catalog->queue->pump(64).completed, 0U); + EXPECT_EQ(backend.catalog->writer_releases, 1U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_snapshot_export_test.cpp b/lib/crowdb-tree/tests/integration/chunk_snapshot_export_test.cpp new file mode 100644 index 000000000..111c0a286 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_snapshot_export_test.cpp @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_tree_core_chunk.h" +#include "crowdb-tree/snapshot/snapshot_io.h" + +namespace crowdb::tree +{ +namespace +{ +class ChunkSnapshotExport : public testing::TestWithParam<snapshot_format> +{ +}; + +TEST_P(ChunkSnapshotExport, ColdCapturedStreamImportsAfterSourceWriterReplacementAndClose) +{ + test_chunk::TestCoreChunk source; + ASSERT_TRUE(source.open().ok()); + const std::string overflow(20'000, 'v'); + Batch batch; + for (size_t i = 0; i < 32; ++i) { + batch.ops.push_back({.key = "key" + std::to_string(1000 + i), .kind = OpKind::kPut, .value = "old"}); + } + batch.ops.push_back({.key = "overflow", .kind = OpKind::kPut, .value = overflow}); + batch.ops.push_back({.key = "gone", .kind = OpKind::kPut, .value = "deleted"}); + ASSERT_TRUE(source.tree->apply(1, batch).ok()); + ASSERT_TRUE(source.tree->apply(2, Batch{.ops = {{.key = "gone", .kind = OpKind::kDelete, .value = ""}}}).ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + source.tree.reset(); + ASSERT_TRUE(source.open().ok()); + std::unique_ptr<SnapshotExport> exporter; + auto status = snapshot_export_begin(*source.tree, GetParam(), 4093, &exporter); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(exporter->at_slot(), 2U); + ASSERT_TRUE(source.tree + ->apply(3, + Batch{ + .ops = {{.key = "overflow", .kind = OpKind::kPut, .value = "new"}, + {.key = "later", .kind = OpKind::kPut, .value = "new"}} + }) + .ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + source.tree.reset(); + + test_chunk::TestCoreChunk destination; + ASSERT_TRUE(destination.open().ok()); + SnapshotImport importer(*destination.tree); + uint64_t bytes = 0; + bool done = false; + while (!done) { + SnapshotChunk chunk; + status = exporter->next_owned(&chunk, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_LE(chunk.bytes.size(), 4093U); + if (!chunk.bytes.empty()) { + ASSERT_TRUE(chunk.owner); + ASSERT_TRUE(chunk.allocation_budget); + status = + importer.feed(std::string_view(reinterpret_cast<const char *>(chunk.bytes.data()), chunk.bytes.size())); + ASSERT_TRUE(status.ok()) << status.to_string(); + bytes += chunk.bytes.size(); + } + EXPECT_EQ(source.config.runtime->snapshot().reserved.io_operations, 0U); + } + EXPECT_EQ(bytes, exporter->total_bytes()); + uint64_t slot = 0; + status = importer.finish(&slot); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(slot, 2U); + exporter.reset(); + ASSERT_TRUE(destination.tree->snapshot().ok()); + destination.tree.reset(); + ASSERT_TRUE(destination.open().ok()); + EXPECT_EQ(destination.tree->durable_snapshot_last_applied_slot(), 2U); + EXPECT_EQ(destination.tree->get_view("overflow").value().to_string(), overflow); + EXPECT_FALSE(destination.tree->get_view("gone").found()); + EXPECT_FALSE(destination.tree->get_view("later").found()); + for (size_t i = 0; i < 32; ++i) { + const auto value = destination.tree->get_view("key" + std::to_string(1000 + i)); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.value().to_string(), "old"); + } +} + +INSTANTIATE_TEST_SUITE_P(PublicFormats, ChunkSnapshotExport, + testing::Values(snapshot_format::kPortable, snapshot_format::kNative)); +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp b/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp index e9fb5ea6e..dc5410223 100644 --- a/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -#include "backend/chunk/chunk_page_store.h" +#include "common/test_tree_core_chunk.h" #include "crowdb-tree/crowdb-tree.h" #include <gtest/gtest.h> @@ -32,65 +32,106 @@ Status snapshot(Crowdbtree &tree, bool asynchronous) return result; } -void check_snapshot_retry(bool asynchronous, bool write_while_unavailable = false) +class TestFailingMirrorTransport final : public test_chunk::TestRootTransport { - auto catalog = std::make_shared<MemoryRootCatalog>(1); - auto transport = std::make_shared<MemoryChunkTransport>(); - ChunkPageStore store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, transport); - Config options; - options.page_store = &store; - options.async_page_store = asynchronous ? &store : nullptr; - Crowdbtree tree(options); - ASSERT_TRUE(tree.apply(1, Batch{{{"key", OpKind::kPut, "old"}}}).ok()); - ASSERT_TRUE(tree.flush().ok()); - ASSERT_TRUE(snapshot(tree, asynchronous).ok()); - auto prior = catalog->load(201); - ASSERT_NE(prior, nullptr); - - ASSERT_TRUE(tree.apply(2, Batch{{{"key", OpKind::kPut, "new"}}}).ok()); - ASSERT_TRUE(tree.flush().ok()); - store.inject_mirror_write_failures(1U << 1U); - const Status failed = snapshot(tree, asynchronous); - ASSERT_EQ(failed.code(), Code::kUnavailable) << failed.to_string(); - ASSERT_EQ(catalog->load(201)->generation, prior->generation); + public: + using TestRootTransport::TestRootTransport; + bool fail_mirror = false; + + Status write_fixed_mirror(ChunkId id, uint32_t mirror, uint64_t offset, const uint8_t *data, size_t length, + ChunkIoContext context) override { - ChunkPageStore durable_store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, - transport); - Config durable_options; - durable_options.page_store = &durable_store; - std::unique_ptr<Crowdbtree> durable; - ASSERT_TRUE(Crowdbtree::open(durable_options, &durable).ok()); - std::string value; - uint64_t slot = 0; - ASSERT_TRUE(durable->get(Slice("key"), &slot, &value)); - EXPECT_EQ(slot, 1U); - EXPECT_EQ(value, "old"); + return fail_mirror && mirror == 1 + ? Status::unavailable() + : TestRootTransport::write_fixed_mirror(id, mirror, offset, data, length, context); } +}; + +void check_snapshot_retry(bool asynchronous, bool write_while_unavailable = false) +{ + test_chunk::TestCoreChunk test; + auto transport = std::make_shared<TestFailingMirrorTransport>(test.backend.catalog); + test.backend.transport = transport; + test.configure_store(0); + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "old").ok()); + ASSERT_TRUE(snapshot(*test.tree, asynchronous).ok()); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + const auto prior = test.backend.catalog->current_generation; + auto old_snapshot = test.tree->snapshot_view(); + + ASSERT_TRUE(test.put(2, "new").ok()); + ASSERT_TRUE(test.tree->flush().ok()); + transport->fail_mirror = true; + const auto failed = snapshot(*test.tree, asynchronous); + ASSERT_EQ(failed.code(), Code::kUnavailable) << failed.to_string(); + ASSERT_EQ(test.backend.catalog->current_generation, prior); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(old_snapshot->get(Slice("key"), &slot, &value)); + EXPECT_EQ(slot, 1U); + EXPECT_EQ(value, "old"); if (write_while_unavailable) { - ASSERT_TRUE(tree.apply(3, Batch{{{"key", OpKind::kPut, "newest"}}}).ok()); - ASSERT_TRUE(tree.flush().ok()); + ASSERT_TRUE(test.put(3, "newest").ok()); + ASSERT_TRUE(test.tree->flush().ok()); } - const Status still_unavailable = snapshot(tree, asynchronous); + const auto still_unavailable = snapshot(*test.tree, asynchronous); ASSERT_EQ(still_unavailable.code(), Code::kUnavailable) << still_unavailable.to_string(); - ASSERT_EQ(catalog->load(201)->generation, prior->generation); - store.inject_mirror_write_failures(0); + ASSERT_EQ(test.backend.catalog->current_generation, prior); + transport->fail_mirror = false; - const Status retried = snapshot(tree, asynchronous); + const auto retried = snapshot(*test.tree, asynchronous); ASSERT_TRUE(retried.ok()) << retried.to_string(); - ASSERT_GT(catalog->load(201)->generation, prior->generation); - ChunkPageStore reopened_store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, - transport); - options.page_store = &reopened_store; - options.async_page_store = asynchronous ? &reopened_store : nullptr; - std::unique_ptr<Crowdbtree> reopened; - ASSERT_TRUE(Crowdbtree::open(options, &reopened).ok()); - std::string value; - uint64_t slot = 0; - ASSERT_TRUE(reopened->get(Slice("key"), &slot, &value)); + ASSERT_GT(test.backend.catalog->current_generation, prior); + ASSERT_TRUE(old_snapshot->get(Slice("key"), &slot, &value)); + EXPECT_EQ(slot, 1U); + EXPECT_EQ(value, "old"); + old_snapshot.reset(); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.tree->get(Slice("key"), &slot, &value)); EXPECT_EQ(slot, write_while_unavailable ? 3U : 2U); EXPECT_EQ(value, write_while_unavailable ? "newest" : "new"); } +TEST(ChunkSnapshotRetry, SmallFramesCheckpointLargeValues) +{ + test_chunk::TestCoreChunk test; + std::shared_ptr<ChunkPinReleaseQueue> releases; + ASSERT_TRUE(ChunkPinReleaseQueue::create( + test.backend.credits.credits.bytes, 1024, std::make_shared<int>(1), [](void *) noexcept {}, nullptr, + &releases) + .ok()); + test.backend.catalog->queue = std::move(releases); + ASSERT_TRUE(test.open().ok()); + for (uint64_t slot = 1; slot <= 32; ++slot) { + ASSERT_TRUE(test.tree + ->apply(slot, Batch{.ops = {{.key = std::to_string(slot), + .kind = OpKind::kPut, + .value = std::string(64 * 1024, 'v')}}}) + .ok()); + } + ASSERT_TRUE(test.tree->flush().ok()); + const auto status = test.tree->snapshot(); + const auto pins = test.backend.catalog->queue->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string() << " registered=" << pins.registered << " pending=" << pins.pending; + EXPECT_GE(test.tree->last_snapshot_pages_written(), 512U); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 32U); + test.tree.reset(); + test.config.backend_owner.reset(); + test.store.reset(); + const auto released = test.backend.catalog->queue->pump(test.backend.catalog->queue->snapshot().pending); + ASSERT_EQ(released.failed, 0U); + ASSERT_TRUE(test.open().ok()); + for (uint64_t slot = 1; slot <= 32; ++slot) { + const auto value = test.tree->get_view(std::to_string(slot)); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.slot(), slot); + EXPECT_EQ(value.value().to_string(), std::string(64 * 1024, 'v')); + } +} + TEST(ChunkSnapshotRetry, PublicationFailureDoesNotPoisonSynchronousSnapshotRetry) { check_snapshot_retry(false); diff --git a/lib/crowdb-tree/tests/integration/compression_test.cpp b/lib/crowdb-tree/tests/integration/compression_test.cpp index b4b703407..c6bcd13d9 100644 --- a/lib/crowdb-tree/tests/integration/compression_test.cpp +++ b/lib/crowdb-tree/tests/integration/compression_test.cpp @@ -4,6 +4,7 @@ // PT10.3: end-to-end page compression. snapshot writes compressed durable // blobs; reopen/demand-load/eviction decode them transparently; a tampered // stored byte fails CRC on reload. +#include "btree/tree_core_access.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" #include "crowdb-tree/maptable/compressor.h" @@ -41,9 +42,14 @@ std::string make_val(int i) void fill_buf(Crowdbtree *t, int K, std::map<std::string, std::string> *oracle) { for (int i = 0; i < K; ++i) { - std::string v = make_val(i); - ASSERT_TRUE(t->apply(i + 1, put_one(make_key(i), v)).ok()); - ASSERT_TRUE(t->flush().ok()); + std::string v = make_val(i); + const auto applied = t->apply(i + 1, put_one(make_key(i), v)); + ASSERT_TRUE(applied.ok()) << "key=" << i << " " << applied.to_string(); + const auto flushed = t->flush(); + ASSERT_TRUE(flushed.ok()) << "key=" << i << " " << flushed.to_string(); + // Flush completion acknowledges publication. Bound this codec test's + // accepted retirement windows before producing another complete flush. + TreeCoreAccess::get(*t)->binding()->account()->drain_retirements(); (*oracle)[make_key(i)] = v; } } @@ -70,7 +76,8 @@ TEST(Compression, CheckpointReopenWithCompressedPages) } std::unique_ptr<Crowdbtree> t2; - ASSERT_TRUE(Crowdbtree::open(opt, &t2).ok()); + const auto reopened = Crowdbtree::open(opt, &t2); + ASSERT_TRUE(reopened.ok()) << reopened.to_string(); EXPECT_GT(t2->height(), 1); for (const auto &kv : oracle) { std::string v; @@ -135,7 +142,7 @@ TEST(Compression, CrcTamperRejectedOnReopen) } // Corrupt a page's stored bytes deep in the page region (past both 4 KiB - // superblock slots). The blob CRC must reject it on demand-load. + // superblock slots). Recovery verifies the blob CRC before publishing it. uint64_t off = (2 * 4096) + 64; // inside the first page's stored area uint8_t b = 0; ASSERT_TRUE(store.read_at(off, &b, 1).ok()); @@ -143,11 +150,7 @@ TEST(Compression, CrcTamperRejectedOnReopen) ASSERT_TRUE(store.write_at(off, &b, 1).ok()); std::unique_ptr<Crowdbtree> t2; - ASSERT_TRUE(Crowdbtree::open(opt, &t2).ok()); // lazy: tags only - // Demand-load fails CRC -> the page reads as a miss, but the media fault is - // latched so a caller can detect the corruption. - std::string v; - uint64_t s; - EXPECT_FALSE(t2->get(Slice("a"), &s, &v)); - EXPECT_TRUE(t2->io_failed()); + const auto reopened = Crowdbtree::open(opt, &t2); + EXPECT_EQ(reopened.code(), Code::kCorruption) << reopened.to_string(); + EXPECT_FALSE(t2); } diff --git a/lib/crowdb-tree/tests/integration/crash_recovery_test.cpp b/lib/crowdb-tree/tests/integration/crash_recovery_test.cpp index 0149c7408..b0f9f5ff7 100644 --- a/lib/crowdb-tree/tests/integration/crash_recovery_test.cpp +++ b/lib/crowdb-tree/tests/integration/crash_recovery_test.cpp @@ -8,6 +8,9 @@ #include "crowdb-tree/backend/block_page_store.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" +#include "crowdb-tree/maptable/mapping_persist.h" +#include "runtime_tree_access.h" +#include "snapshot/commit_anchor.h" #include "test_tmp.h" #include <gtest/gtest.h> @@ -79,6 +82,7 @@ TEST(CrashRecovery, FaultInjectedWriteFailureLeavesPreviousGenerationIntact) // gen2's anchor was never written -- gen1 is still the last committed // generation. Reopen (fault already disarmed -- one-shot) and confirm. + t.reset(); std::unique_ptr<Crowdbtree> t2; ASSERT_TRUE(Crowdbtree::open(opt, &t2).ok()); for (int i = 0; i < 20; ++i) { @@ -89,47 +93,38 @@ TEST(CrashRecovery, FaultInjectedWriteFailureLeavesPreviousGenerationIntact) } } -// A dirty mapping-table segment's image is silently lost (as if the write -// never landed pre-crash) but every *later* write in this generation -- -// including the anchor itself -- still succeeds and commits normally. This -// is the #14c/#14d-specific corruption class the recovery spec -// calls out: "a segment or directory image failing CRC while its anchor was -// committed indicates media corruption -> fail the node out" (§9), as -// opposed to the anchor itself being torn (already covered by the -// byte-corruption tests above, and by construction unaffected by #14c/#14d -// since the anchor keeps the same A/B slot geometry). -TEST(CrashRecovery, DroppedSegmentImageFailsReopenEvenThoughAnchorCommitted) +// A committed anchor whose segment image fails CRC is media corruption; +// recovery must not silently fall back to an older logical image. +TEST(CrashRecovery, CorruptSegmentImageFailsReopenEvenThoughAnchorCommitted) { - MemPageStore mem(1); - FaultyPageStore store(&mem); - Config opt; - opt.page_store = &store; - - std::unique_ptr<Crowdbtree> t; - ASSERT_TRUE(Crowdbtree::open(opt, &t).ok()); - for (int i = 0; i < 5; ++i) { - ASSERT_TRUE(t->apply(i + 1, put_one(make_key(i), "gen1-" + std::to_string(i))).ok()); + MemPageStore store(1); + Config options; + options.page_store = &store; + std::unique_ptr<Crowdbtree> tree; + ASSERT_TRUE(Crowdbtree::open(options, &tree).ok()); + for (int index = 0; index < 5; ++index) { + ASSERT_TRUE(tree->apply(index + 1, put_one(make_key(index), "first")).ok()); } - ASSERT_TRUE(t->flush().ok()); - ASSERT_TRUE(t->snapshot(nullptr).ok()); // gen1 commits; small tree -> exactly 1 segment - - // A single-key edit (no split) dirties exactly one page and one segment; - // this generation's write sequence is [page write, segment write, - // directory write]. Drop write #1 (the segment image) specifically. - int gen1_writes = store.write_count(); - uint64_t slot = 100; - ASSERT_TRUE(t->apply(slot, put_one(make_key(0), "gen2-updated")).ok()); - t->force_advance_slot(slot); // slot 100 is non-contiguous with gen1's 1..5 - ASSERT_TRUE(t->flush().ok()); - store.arm_write_fault(gen1_writes + 1, FaultyPageStore::Fault::kDrop); - ASSERT_TRUE(t->snapshot(nullptr).ok()) << "the drop is silent -- snapshot() itself must still report success"; - ASSERT_EQ(t->last_snapshot_segments_written(), 1U) << "expected exactly the touched segment to be re-imaged"; - - // The anchor now points at a segment image address whose bytes were - // never actually written -- reopen must detect this as corruption, not - // silently fall back or (worse) serve wrong data. - std::unique_ptr<Crowdbtree> t2; - EXPECT_FALSE(Crowdbtree::open(opt, &t2).ok()); + ASSERT_TRUE(tree->snapshot().ok()); + ASSERT_TRUE(tree->put(make_key(0), "second").ok()); + ASSERT_TRUE(tree->snapshot().ok()); + std::vector<snapshot::CommitAnchor> anchors; + ASSERT_TRUE(snapshot::read_valid_anchors(store, store.iu_size(), &anchors).ok()); + ASSERT_EQ(anchors.size(), 2U); + const auto anchor = *std::max_element(anchors.begin(), anchors.end(), [](const auto &left, const auto &right) { + return left.snapshot_seq < right.snapshot_seq; + }); + std::vector<uint8_t> directory(anchor.segdir_len); + ASSERT_TRUE(store.read_at(anchor.segdir_addr, directory.data(), directory.size()).ok()); + std::vector<DirEntry> entries; + ASSERT_TRUE(decode_segment_directory(directory.data(), directory.size(), &entries).ok()); + ASSERT_FALSE(entries.empty()); + const auto damaged = entries.front(); + tree.reset(); + std::vector<uint8_t> zeros(damaged.image_len, 0); + ASSERT_TRUE(store.write_at(damaged.image_addr, zeros.data(), zeros.size()).ok()); + EXPECT_FALSE(Crowdbtree::open(options, &tree).ok()); + EXPECT_FALSE(tree); } // ckpt1 (large overflow values + compression) commits; a second snapshot then @@ -232,6 +227,7 @@ TEST(CrashRecovery, DemandLoadCorruptionLatched) { MemPageStore store(1); Config opt; + uint64_t off = 0; opt.page_store = &store; opt.frame_bytes = 4096; opt.max_delta_len = 1; @@ -243,17 +239,31 @@ TEST(CrashRecovery, DemandLoadCorruptionLatched) ASSERT_TRUE(t.flush().ok()); } ASSERT_TRUE(t.snapshot(nullptr).ok()); + RuntimeTreeAccess_for_tests::capture(t)->mapping.visit( + &off, [](void *raw, uint64_t, const std::shared_ptr<const PublishedPage> &page) { + auto &offset = *static_cast<uint64_t *>(raw); + if (offset == 0 && page->leaf && !page->leaf->bases.empty()) { + const auto &backing = page->leaf->bases.front().backing; + if (backing && backing->length > 1) { + offset = backing->address + (backing->length / 2); + } + } + }); } - // Corrupt deep in the page region (well past both superblock slots). - uint64_t off = (2 * kSbBytes) + 200; - uint8_t b = 0; + ASSERT_GT(off, 2 * kSbBytes); + uint8_t b = 0; ASSERT_TRUE(store.read_at(off, &b, 1).ok()); - b ^= 0xff; - ASSERT_TRUE(store.write_at(off, &b, 1).ok()); + const uint8_t corrupt = b ^ 0xff; + ASSERT_TRUE(store.write_at(off, &corrupt, 1).ok()); std::unique_ptr<Crowdbtree> t2; + // Recovery verifies the persisted graph before exposing its cold mapping. + EXPECT_EQ(Crowdbtree::open(opt, &t2).code(), Code::kCorruption); + ASSERT_FALSE(t2); + ASSERT_TRUE(store.write_at(off, &b, 1).ok()); ASSERT_TRUE(Crowdbtree::open(opt, &t2).ok()); EXPECT_FALSE(t2->io_failed()); // nothing loaded yet (lazy recovery) + ASSERT_TRUE(store.write_at(off, &corrupt, 1).ok()); // Read every key: the corrupted page demand-load fails CRC and latches. for (int i = 0; i < 60; ++i) { diff --git a/lib/crowdb-tree/tests/integration/double_buffer_test.cpp b/lib/crowdb-tree/tests/integration/double_buffer_test.cpp index 61f174f68..eb4911142 100644 --- a/lib/crowdb-tree/tests/integration/double_buffer_test.cpp +++ b/lib/crowdb-tree/tests/integration/double_buffer_test.cpp @@ -3,6 +3,7 @@ // plan-tree #3: MemTable double buffering (active_ + frozen_). See the // active_/frozen_ member comment in crowdb-tree.h for the full design. +#include "btree/tree_core_access.h" #include "crowdb-tree/crowdb-tree.h" #include <gtest/gtest.h> @@ -51,7 +52,8 @@ std::string make_key(int i) TEST(DoubleBuffer, ThresholdSwapKeepsAllEntriesReadable) { Config opt; - opt.memtable_flush_entries = 2; // force a freeze every couple of applies + opt.memtable_flush_entries = 2; // force a freeze every couple of applies + opt.max_memtable_count = 32; // isolate rotation from count-triggered handoff Crowdbtree t(opt); const int K = 20; @@ -139,7 +141,7 @@ TEST(DoubleBuffer, SupportsMoreThanTwoBuffersAndFlushesAllOfThemCorrectly) // these applies stays non-contiguous and none of the freezes it triggers // get drained until the very last flush() below. for (int i = 0; i < K; ++i) { - auto s = static_cast<uint64_t>((i + 1) * 10); + auto s = static_cast<uint64_t>(i + 1) * 10; ASSERT_TRUE(t.apply(s, put_one(make_key(i), "v" + std::to_string(i))).ok()); } EXPECT_EQ(t.last_applied_slot(), 0U); @@ -148,7 +150,7 @@ TEST(DoubleBuffer, SupportsMoreThanTwoBuffersAndFlushesAllOfThemCorrectly) EXPECT_EQ(get_or(t, make_key(i), "?"), "v" + std::to_string(i)); } - t.force_advance_slot(static_cast<uint64_t>(K * 10)); + t.force_advance_slot(static_cast<uint64_t>(K) * 10); ASSERT_TRUE(t.flush().ok()); EXPECT_EQ(t.memtable_count(), 0U); for (int i = 0; i < K; ++i) { @@ -198,9 +200,16 @@ TEST(DoubleBuffer, GetAndScanResolveHighestSlotAcrossOutOfOrderFreezeBoundary) // TSan/ASan to catch races/UAF in the active_/frozen_ swap + drain path. TEST(DoubleBuffer, ConcurrentReadersDuringFrequentFreezeAndDrainNoCorruption) { - Config opt; - opt.memtable_flush_entries = 4; // freeze/drain every few applies - opt.max_memtable_count = 3; + Config opt; + TreeRuntimeOptions host; + host.limits.routing_bytes = 1536ULL * 1024 * 1024; + ASSERT_TRUE(TreeRuntime::create(host, &opt.runtime).ok()); + // Four concurrent scans each reserve oversized-first output and bounded + // decoder/overflow state. Give this four-reader stress fixture explicit + // aggregate capacity without changing the production defaults. + opt.runtime_binding.limits.routing_bytes = 1024ULL * 1024 * 1024; + opt.memtable_flush_entries = 4; // freeze/drain every few applies + opt.max_memtable_count = 3; Crowdbtree t(opt); const int K = 200; @@ -215,10 +224,11 @@ TEST(DoubleBuffer, ConcurrentReadersDuringFrequentFreezeAndDrainNoCorruption) for (int r = 0; r < 4; ++r) { readers.emplace_back([&, r] { std::mt19937 rng(9000 + r); - std::string v; - uint64_t s; while (!stop.load(std::memory_order_relaxed) && !bad.load(std::memory_order_relaxed)) { - if (!t.get(Slice("zz-sentinel"), &s, &v) || v != "always-visible") { + GetView sentinel; + const auto status = TreeCoreAccess::get(t)->get(Slice("zz-sentinel"), &sentinel); + EXPECT_TRUE(status.ok()) << status.to_string(); + if (!status.ok() || !sentinel.found() || sentinel.value().to_string() != "always-visible") { bad.store(true); return; } @@ -226,11 +236,19 @@ TEST(DoubleBuffer, ConcurrentReadersDuringFrequentFreezeAndDrainNoCorruption) // Random keys have no stable value oracle because the writer // mutates them concurrently. The sentinel above separately // proves an immutable key never disappears during a drain. - (void)t.get(Slice(make_key(static_cast<int>(rng() % K))), &s, &v); + GetView value; + const auto read = TreeCoreAccess::get(t)->get(Slice(make_key(static_cast<int>(rng() % K))), &value); + EXPECT_TRUE(read.ok()) << read.to_string(); + if (!read.ok()) { + bad.store(true); + return; + } } std::vector<scan_entry> out; bool trunc = false; - if (!t.scan(Slice("key"), Slice(), Slice(), 16, 0, false, 0, &out, &trunc).ok()) { + const auto scan = t.scan(Slice("key"), Slice(), Slice(), 16, 0, false, 0, &out, &trunc); + EXPECT_TRUE(scan.ok()) << scan.to_string(); + if (!scan.ok()) { bad.store(true); return; } @@ -354,7 +372,7 @@ TEST(DoubleBuffer, ScanMergeLoserTreeDistinctKeysAcrossFrozenMemtables) for (int gen = 0; gen < 3; ++gen) { for (int i = 0; i < 3; ++i) { int key_idx = gen + (i * 3); - auto s = static_cast<uint64_t>((key_idx + 1) * 10); + auto s = static_cast<uint64_t>(key_idx + 1) * 10; ASSERT_TRUE(t.apply(s, put_one(make_key(key_idx), "v" + std::to_string(s))).ok()); } } @@ -492,11 +510,8 @@ TEST(DoubleBuffer, SortAwareDescentAcrossLeafBoundariesWithSplits) } } -// Frozen queue full: when max_memtable_count is small and we write fast -// without flushing, the frozen queue fills up and maybe_freeze_active -// returns false — active_ keeps growing past the threshold. All entries -// must stay readable (they're in active_). This verifies the backpressure -// behavior documented in the maybe_freeze_active error log path. +// A full soft source queue requests finite handoff. Every committed entry +// remains readable while source ownership moves into the published mapping. TEST(DoubleBuffer, FrozenQueueFullActiveKeepsGrowingEntriesReadable) { Config opt; @@ -506,16 +521,14 @@ TEST(DoubleBuffer, FrozenQueueFullActiveKeepsGrowingEntriesReadable) // Write enough to fill the frozen queue (1 slot) and overflow active_. // After 3 entries: active_ freezes → frozen_ has 1, active_ is fresh. - // After 6 entries: active_ tries to freeze but frozen_ is full → - // maybe_freeze_active returns false, active_ keeps growing. + // After 6 entries the full queue requests a finite handoff. const int N = 20; for (int i = 0; i < N; ++i) { ASSERT_TRUE(t.apply(static_cast<uint64_t>(i + 1), put_one(make_key(i), "v" + std::to_string(i))).ok()); } - // Without flush(), all entries must still be readable from L0 - // (active_ + frozen_). The frozen queue full condition does NOT - // lose data — it just delays the freeze. + // Automatic handoff may have moved entries into L1; every key remains + // readable across the source and mapping boundary. for (int i = 0; i < N; ++i) { EXPECT_EQ(get_or(t, make_key(i), "?"), "v" + std::to_string(i)); } @@ -641,9 +654,8 @@ TEST(DoubleBuffer, FlushDrainsAllFrozenMemtablesInOneCall) } } -// Iteration cap: when more memtables freeze than max_memtable_count, flush() -// exits at the cap and remaining tables stay in frozen_ for the next flush(). -// No data loss — the next flush() drains the rest. +// Count-triggered handoffs remain finite. Explicit maintenance after the +// writer stops drains the final source set, and the next write can proceed. TEST(DoubleBuffer, FlushIterationCapExitsCleanly) { Config opt; @@ -651,12 +663,7 @@ TEST(DoubleBuffer, FlushIterationCapExitsCleanly) opt.max_memtable_count = 2; // 1 active + 1 frozen slot; cap = 2 Crowdbtree t(opt); - // Write a lot without flushing. With max_memtable_count=2, the frozen - // queue fills at 1 slot; further freezes are skipped (active_ grows). - // So frozen_ has at most 1 entry — the cap is not exercised by writes - // alone. Instead, verify the cap path is correct: flush() drains what's - // frozen and leaves frozen_ empty (the cap is a safety net, not the - // common path here). + // Repeated batches fill and drain the bounded source queue. const int N = 30; for (int i = 0; i < N; ++i) { ASSERT_TRUE(t.apply(static_cast<uint64_t>(i + 1), put_one(make_key(i), "v" + std::to_string(i))).ok()); diff --git a/lib/crowdb-tree/tests/integration/eviction_test.cpp b/lib/crowdb-tree/tests/integration/eviction_test.cpp index 15e8e9e52..da931bf12 100644 --- a/lib/crowdb-tree/tests/integration/eviction_test.cpp +++ b/lib/crowdb-tree/tests/integration/eviction_test.cpp @@ -1,12 +1,11 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// PT6c-5.4: writer-driven eviction of clean resident bases. An -// evicted leaf re-tags its mapping slot `unloaded` and epoch-retires the page; -// the next access demand-loads it. Run under TSan for the eviction-vs-reader -// race (epoch-deferred frame reuse). +// Clean cache eviction preserves the immutable publication and independent +// read owners; the next uncached access reloads the same backing. #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" +#include "runtime_tree_access.h" #include <gtest/gtest.h> @@ -44,6 +43,17 @@ void fill_buf(Crowdbtree *t, int K, std::map<std::string, std::string> *oracle) } } +void warm_cache(Crowdbtree &tree, const std::map<std::string, std::string> &oracle) +{ + for (const auto &[key, value] : oracle) { + GetView result; + const auto status = RuntimeTreeAccess_for_tests::core(tree).get(Slice(key), &result); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(result.found()) << key; + ASSERT_EQ(result.value().to_string(), value); + } +} + // Counts read_at calls so a test can observe whether a specific page was // demand-loaded (evicted, then reloaded) vs. still resident (plan-tree #17 // recency-ranked eviction). @@ -77,14 +87,15 @@ TEST(Eviction, EvictedLeavesFreeMemoryAndReloadCorrectly) std::map<std::string, std::string> oracle; fill_buf(&t, 200, &oracle); ASSERT_TRUE(t.snapshot(nullptr).ok()); // all reachable pages now clean + ASSERT_NO_FATAL_FAILURE(warm_cache(t, oracle)); - uint32_t before = t.buffer_pool()->stats().used; + uint32_t before = t.stats().buffer_pool_used; size_t evicted = t.evict_clean_leaves(2); // keep at most 2 resident leaves EXPECT_GT(evicted, 0U); // No reader guards are open, so the epoch manager reclaims the retired pages // synchronously and their frames return to the pool: residency drops. - uint32_t after = t.buffer_pool()->stats().used; + uint32_t after = t.stats().buffer_pool_used; EXPECT_LT(after, before); // Every value is still readable — evicted leaves demand-load on access. @@ -113,6 +124,7 @@ TEST(Eviction, EvictIsIdempotentAndSkipsDirty) EXPECT_EQ(t.evict_clean_leaves(0), 0U); ASSERT_TRUE(t.snapshot(nullptr).ok()); // pages become clean + ASSERT_NO_FATAL_FAILURE(warm_cache(t, oracle)); size_t first = t.evict_clean_leaves(1); EXPECT_GT(first, 0U); // A second pass with everything already unloaded evicts nothing more. @@ -134,6 +146,13 @@ TEST(Eviction, ConcurrentReadersWhileEvicting) opt.max_delta_len = 1; opt.leaf_split_bytes = 160; opt.frame_bytes = 4096; + // Six simultaneous point windows each prepay the supported 64 MiB + // overflow value plus decoder/control storage. Quota exhaustion is + // covered separately; this case exercises readers racing cache eviction. + TreeRuntimeOptions runtime; + runtime.limits.routing_bytes = 1024ULL * 1024 * 1024; + opt.runtime_binding.limits.routing_bytes = 768ULL * 1024 * 1024; + ASSERT_TRUE(TreeRuntime::create(runtime, &opt.runtime).ok()); Crowdbtree t(opt); const int K = 250; @@ -148,11 +167,12 @@ TEST(Eviction, ConcurrentReadersWhileEvicting) for (int r = 0; r < 6; ++r) { readers.emplace_back([&, r] { std::mt19937 rng(9000 + r); - std::string v; - uint64_t s; while (!stop.load(std::memory_order_relaxed)) { - int i = static_cast<int>(rng() % K); - if (!t.get(Slice(make_key(i)), &s, &v) || v != "val" + std::to_string(i)) { + int i = static_cast<int>(rng() % K); + GetView result; + const auto status = RuntimeTreeAccess_for_tests::core(t).get(Slice(make_key(i)), &result); + if (!status.ok() || !result.found() || result.value().to_string() != "val" + std::to_string(i)) { + ADD_FAILURE() << "reader " << r << ", key " << i << ": " << status.to_string(); fail.store(true); return; } @@ -187,7 +207,8 @@ TEST(Eviction, RecentlyTouchedLeafSurvivesEvictionOverColderOnes) std::map<std::string, std::string> oracle; fill_buf(&t, 200, &oracle); - ASSERT_TRUE(t.snapshot(nullptr).ok()); // clean + resident, touch order == snapshot's DFS walk + ASSERT_TRUE(t.snapshot(nullptr).ok()); + ASSERT_NO_FATAL_FAILURE(warm_cache(t, oracle)); // Re-touch the very first key's leaf so it becomes the *most* recently // touched -- recency ranking should keep it resident longer than leaves @@ -229,6 +250,7 @@ TEST(Eviction, EvictCleanInnerNeverTouchesLeaves) std::map<std::string, std::string> oracle; fill_buf(&t, 300, &oracle); // enough keys for several inner levels ASSERT_TRUE(t.snapshot(nullptr).ok()); + ASSERT_NO_FATAL_FAILURE(warm_cache(t, oracle)); int reads_before = store.reads.load(); // Aggressive inner budget: keep at most 1 inner base resident. Leaves @@ -356,13 +378,14 @@ TEST(Eviction, InstallSnapshotReclaimsResidentLeavesEvenWhenInnerAncestorWasEvic std::map<std::string, std::string> oracle; fill_buf(&t, 300, &oracle); ASSERT_TRUE(t.snapshot(nullptr).ok()); + ASSERT_NO_FATAL_FAILURE(warm_cache(t, oracle)); // Evict every inner base -- every leaf remains fully resident underneath // a now-entirely-unloaded inner chain, including the root itself. size_t inner_evicted = t.evict_clean_inner(0); EXPECT_GT(inner_evicted, 0U); - uint32_t used_before = t.buffer_pool()->stats().used; + uint32_t used_before = t.stats().buffer_pool_used; EXPECT_GT(used_before, 0U) << "every leaf should still be resident (never evicted)"; // Replaces the whole live tree -- must reclaim every still-resident @@ -372,7 +395,7 @@ TEST(Eviction, InstallSnapshotReclaimsResidentLeavesEvenWhenInnerAncestorWasEvic // No reader guards are open, so the epoch manager reclaims synchronously // (same assumption the leaf-eviction tests above rely on): only the // freshly-built empty root should remain resident. - EXPECT_EQ(t.buffer_pool()->stats().used, 1U) - << "install_snapshot must reclaim the old tree's still-resident leaves " - "even though their ancestor chain was unloaded"; + RuntimeTreeAccess_for_tests::reclaim(t); + EXPECT_EQ(t.stats().buffer_pool_used, 0U) << "install_snapshot must reclaim the old tree's still-resident leaves " + "even though their ancestor chain was unloaded"; } diff --git a/lib/crowdb-tree/tests/integration/extent_writeback_test.cpp b/lib/crowdb-tree/tests/integration/extent_writeback_test.cpp new file mode 100644 index 000000000..37f6e7fc6 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/extent_writeback_test.cpp @@ -0,0 +1,195 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/extent_writeback.h" +#include "extent_ledger_test_support.h" +#include "page_writeback_test_support.h" +#include "test_tmp.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_write; + +struct LedgerWrites : TestWriter +{ + std::shared_ptr<ExtentLedger> ledger; + + LedgerWrites() + { + EXPECT_TRUE( + ExtentLedger::create({.backend_id = 3, .address_limit = 65536}, credits.credits.bytes, &ledger).ok()); + EXPECT_TRUE(await_ledger([&](auto done) { + return ledger->activate(credits.credits.bytes, std::move(done)); + }).status.ok()); + } + + ExtentLedgerResult allocate(uint64_t bytes) + { + return await_ledger([&](auto done) { return ledger->reserve(bytes, credits.credits.bytes, std::move(done)); }); + } +}; + +ReservedWriteExtent for_write(ProtectedExtent extent) +{ + return {.backend_id = extent.backend_id, + .address = extent.address, + .length = extent.length, + .content_id = extent.identity, + .generation = extent.generation, + .owner = std::move(extent.owner), + .physical_metrics = extent.physical_metrics, + .chunk = {}}; +} + +#ifdef CROWDB_HAVE_LIBURING +TEST(LedgerWriteback, RealBlockPageStaysReadableAndCannotBeReusedUntilLastBackingOwner) +{ + crowdb::tree_test::TempDir dir("ledger_write_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 4096, &file).ok()); + crowdb::common::Topology topology; + topology.pipelines.emplace_back().entries = 64; + crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + auto store = std::make_shared<BlockAsyncPageStore>(file.get(), &uring); + LedgerWrites test; + ASSERT_TRUE( + PageWriteOperation::create(store, test.runtime, test.lane, write_input(), + {.max_frame_bytes = 4096, .compression = compress_algo::kNone}, test.credits.credits, + extent_write_reservation(test.ledger, test.credits.credits.bytes), &test.operation) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); + test.runtime->drain(); + auto reader = test.operation->result().backing; + ASSERT_TRUE(reader); + EXPECT_EQ(reader->address, 8192U); + EXPECT_EQ(reader->length, 8192U); + const auto identity = reader->content_id; + test.operation.reset(); + auto later = test.allocate(8192); + ASSERT_TRUE(later.status.ok()); + EXPECT_EQ(later.extent.address, 16384U); + std::vector<uint8_t> replacement(8192, 0xCC); + ASSERT_TRUE(file->write_at(later.extent.address, replacement.data(), replacement.size()).ok()); + std::vector<uint8_t> blob(reader->length); + std::vector<uint8_t> frame(4096); + ASSERT_TRUE(file->read_at(reader->address, blob.data(), blob.size()).ok()); + ASSERT_TRUE(decode_durable_page(blob.data(), blob.size(), frame.data(), frame.size()).ok()); + LeafFrameView leaf(frame.data(), frame.size()); + EXPECT_EQ(CellView(leaf.cell(0)).value().to_string(), "value"); + reader.reset(); + auto reused = test.allocate(8192); + ASSERT_TRUE(reused.status.ok()); + EXPECT_EQ(reused.extent.address, 8192U); + EXPECT_NE(reused.extent.identity, identity); + later = {}; + reused = {}; + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +#endif + +TEST(LedgerWriteback, PendingAllocationLeavesCpuWorkerFreeAndCancellationDrainsItsOwner) +{ + LedgerWrites test; + PageWriteOperation::ExtentComplete pending; + uint64_t bytes = 0; + ASSERT_TRUE(PageWriteOperation::create( + test.store, test.runtime, test.lane, write_input(), {}, test.credits.credits, + [&](uint64_t length, PageWriteOperation::ExtentComplete done) { + bytes = length; + pending = std::move(done); + }, + &test.operation) + .ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + ASSERT_TRUE(pending); + std::atomic<bool> progressed{false}; + ASSERT_TRUE(test.runtime + ->try_submit( + test.lane, + [](void *context) noexcept { static_cast<std::atomic<bool> *>(context)->store(true); }, + &progressed) + .ok()); + test.runtime->drain(); + EXPECT_TRUE(progressed.load()); + test.operation->request_stop(); + auto allocated = test.allocate(bytes); + ASSERT_TRUE(allocated.status.ok()); + pending(Status::Ok(), for_write(std::move(allocated.extent))); + pending = {}; + test.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_EQ(test.store->writes, 0U); + test.operation.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(LedgerWriteback, UnknownAllocationSubmissionDropsLateExtentWithoutIssuingWrite) +{ + LedgerWrites test; + PageWriteOperation::ExtentComplete pending; + uint64_t bytes = 0; + ASSERT_TRUE(PageWriteOperation::create( + test.store, test.runtime, test.lane, write_input(), {}, test.credits.credits, + [&](uint64_t length, PageWriteOperation::ExtentComplete complete) { + bytes = length; + pending = std::move(complete); + throw std::runtime_error("unknown allocation submission"); + }, + &test.operation) + .ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + ASSERT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.store->writes, 0U); + test.operation.reset(); + auto allocated = test.allocate(bytes); + ASSERT_TRUE(allocated.status.ok()); + pending(Status::Ok(), for_write(std::move(allocated.extent))); + pending = {}; + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.store->writes, 0U); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(LedgerWriteback, InlineAllocationSuccessThenThrowDoesNotStartIo) +{ + LedgerWrites test; + auto allocated = test.allocate(8192); + ASSERT_TRUE(allocated.status.ok()); + ASSERT_TRUE(PageWriteOperation::create( + test.store, test.runtime, test.lane, write_input(), + {.max_frame_bytes = 4096, .compression = compress_algo::kNone}, test.credits.credits, + [extent = std::make_shared<ProtectedExtent>(allocated.extent)]( + uint64_t, PageWriteOperation::ExtentComplete complete) { + complete(Status::Ok(), for_write(*extent)); + throw std::runtime_error("after inline allocation"); + }, + &test.operation) + .ok()); + allocated = {}; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.store->writes, 0U); + test.operation.reset(); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/external_ownership_test.cpp b/lib/crowdb-tree/tests/integration/external_ownership_test.cpp index 1e31517cf..1c1d9013a 100644 --- a/lib/crowdb-tree/tests/integration/external_ownership_test.cpp +++ b/lib/crowdb-tree/tests/integration/external_ownership_test.cpp @@ -34,7 +34,7 @@ TEST(ExternalOwnership, EmptyValueReleasesItsTransferredOwner) int drops = 0; auto op = external(&drops, nullptr, 0); ASSERT_EQ(ct_apply_batch_external(tree, 1, &op, 1), 0); - EXPECT_EQ(drops, 0); + EXPECT_EQ(drops, 1); ct_close(tree); EXPECT_EQ(drops, 1); } @@ -65,3 +65,65 @@ TEST(ExternalOwnership, InvalidLaterOperationReleasesEveryTransferredOwner) EXPECT_EQ(count, 1); } } + +TEST(ExternalOwnership, NullTreeAndInvalidValueReleaseTransferredOwners) +{ + int drops = 0; + auto op = external(&drops, nullptr, 12); + EXPECT_NE(ct_apply_batch_external(nullptr, 1, &op, 1), 0); + EXPECT_EQ(drops, 1); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + EXPECT_NE(ct_apply_batch_external(tree, 1, &op, 1), 0); + EXPECT_EQ(drops, 2); + ct_close(tree); + EXPECT_EQ(drops, 2); +} + +TEST(ExternalOwnership, ControlledCopyDoesNotRetainOrAliasUnknownPayload) +{ + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + std::array<uint8_t, 128> payload{}; + payload.fill('v'); + int drops = 0; + auto op = external(&drops, payload.data() + 32, 64); + ASSERT_EQ(ct_apply_batch_external(tree, 1, &op, 1), 0); + EXPECT_EQ(drops, 1); + payload.fill('x'); + int32_t found = 0; + uint64_t slot = 0; + ct_buf value{}; + ASSERT_EQ(ct_get(tree, op.key, op.key_len, &found, &slot, &value), 0); + EXPECT_EQ(found, 1); + EXPECT_EQ(slot, 1U); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(value.data), value.len), std::string(64, 'v')); + ct_free_buf(&value); + ct_close(tree); + EXPECT_EQ(drops, 1); +} + +TEST(ExternalOwnership, ControlledCopyAllocationFailureReleasesInput) +{ + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + int drops = 0; + const uint8_t value = 'v'; + auto op = external(&drops, &value, std::numeric_limits<size_t>::max()); + EXPECT_NE(ct_apply_batch_external(tree, 1, &op, 1), 0); + EXPECT_EQ(drops, 1); + ct_close(tree); + EXPECT_EQ(drops, 1); +} + +TEST(ExternalOwnership, OwnedHandleIsConsumedWhenTreeIsNull) +{ + ct_write_handle *handle = nullptr; + ct_write_ptrs pointers{}; + ASSERT_EQ(ct_alloc(nullptr, 3, 1024, &handle, &pointers), 0); + ASSERT_NE(handle, nullptr); + EXPECT_NE(ct_apply_put_owned(nullptr, 1, handle), 0); +} diff --git a/lib/crowdb-tree/tests/integration/facade_checkpoint_diagnostics_test.cpp b/lib/crowdb-tree/tests/integration/facade_checkpoint_diagnostics_test.cpp new file mode 100644 index 000000000..3b44b7bb8 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/facade_checkpoint_diagnostics_test.cpp @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "local_checkpoint_test_support.h" +#include "snapshot/operation_wait.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestFacadeCheckpoint +{ + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + Config config; + std::unique_ptr<Crowdbtree> tree; + + TestFacadeCheckpoint() + { + store.device = device; + config.page_store = device.get(); + config.async_page_store = &store; + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.max_inline_value = 64; + } + + void open() + { + const auto status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + } +}; + +TEST(FacadeCheckpointDiagnostics, PhysicalWritesExcludeCleanReuseAndIncludeOverflow) +{ + TestFacadeCheckpoint test; + test.open(); + ASSERT_TRUE(test.tree); + EXPECT_EQ(test.tree->last_snapshot_pages_written(), 0U); + ASSERT_TRUE(test.tree->put("large", std::string(12000, 'v')).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + EXPECT_GT(test.tree->last_snapshot_pages_written(), 1U); + EXPECT_EQ(test.tree->stats().snapshot_pages_written, test.tree->last_snapshot_pages_written()); + const auto written = test.tree->last_snapshot_pages_written(); + EXPECT_EQ(test.tree->stats().snapshot_pages_total, written); + ASSERT_TRUE(test.tree->snapshot().ok()); + EXPECT_EQ(test.tree->last_snapshot_pages_written(), 0U); + EXPECT_EQ(test.tree->stats().snapshot_pages_total, written); + EXPECT_EQ(test.tree->last_snapshot_segments_written(), 0U); + EXPECT_EQ(test.tree->scan_profile().count, 0U); + test.tree.reset(); + test.open(); + ASSERT_TRUE(test.tree); + EXPECT_EQ(test.tree->get_view("large").value().to_string(), std::string(12000, 'v')); +} + +TEST(FacadeCheckpointDiagnostics, FailedAndStagedWritesPreserveLastDurableObservation) +{ + TestFacadeCheckpoint test; + test.open(); + ASSERT_TRUE(test.tree); + ASSERT_TRUE(test.tree->put("a", "one").ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + const auto first = test.tree->last_snapshot_pages_written(); + ASSERT_GT(first, 0U); + ASSERT_TRUE(test.tree->put("a", "two").ok()); + test.store.fail_at = test.store.events.size() + 1; + EXPECT_FALSE(test.tree->snapshot().ok()); + EXPECT_EQ(test.tree->last_snapshot_pages_written(), first); + test.store.fail_at = 0; + auto *core = TreeCoreAccess::get(*test.tree); + std::shared_ptr<CheckpointOperation> writeback; + ASSERT_TRUE(core->prepare_writeback(&writeback).ok()); + const auto status = detail::wait_snapshot_operation(writeback); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(writeback->result().writeback_completed); + EXPECT_EQ(test.tree->last_snapshot_pages_written(), first); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 1U); + writeback.reset(); + ASSERT_TRUE(test.tree->snapshot().ok()); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 2U); + EXPECT_EQ(test.tree->last_snapshot_pages_written(), 0U); +} + +TEST(FacadeCheckpointDiagnostics, SmallerDefaultFrameDerivesSplitButRejectsExplicitInvalidGeometry) +{ + TestFacadeCheckpoint test; + test.config.leaf_split_bytes = Config{}.leaf_split_bytes; + test.open(); + ASSERT_TRUE(test.tree); + EXPECT_LT(TreeCoreAccess::get(*test.tree)->config().leaf_split_bytes, test.config.frame_bytes); + ASSERT_TRUE(test.tree->put("key", "value").ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + test.tree.reset(); + test.config.frame_bytes = 256; + auto status = Crowdbtree::open(test.config, &test.tree); + EXPECT_FALSE(status.ok()); + EXPECT_FALSE(test.tree); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/flush_capture_report_test.cpp b/lib/crowdb-tree/tests/integration/flush_capture_report_test.cpp new file mode 100644 index 000000000..5a3a1a009 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/flush_capture_report_test.cpp @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +class FlushCaptureReport : public testing::Test +{ + protected: + void SetUp() override + { + Config options; + ASSERT_TRUE(TreeRuntime::create({}, &options.runtime).ok()); + options.memtable_flush_bytes = SIZE_MAX; + options.memtable_flush_entries = UINT32_MAX; + tree = std::make_unique<Crowdbtree>(options); + core = TreeCoreAccess::get(*tree); + ASSERT_TRUE(tree->put("key", "value").ok()); + } + + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; +}; + +TEST_F(FlushCaptureReport, CoveredCaptureKeepsItsFrontierAndNormalCompletion) +{ + ASSERT_TRUE(tree->flush().ok()); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(core->prepare_flush(&operation).ok()); + ASSERT_TRUE(tree->put("later", "successor").ok()); + FlushResult capture; + ASSERT_TRUE(operation->wait_capture(&capture).ok()); + ASSERT_TRUE(capture.handoff_status.ok()); + EXPECT_EQ(capture.frontier, 1U); + EXPECT_EQ(operation->frozen_sources().frontier, capture.frontier); + EXPECT_EQ(operation->frozen_sources().base->coverage, capture.frontier); + ASSERT_TRUE(operation->wait().ok()); + EXPECT_EQ(operation->result().phase, FlushPhase::kComplete); + EXPECT_FALSE(operation->result().handoff_advanced); + EXPECT_EQ(core->capture()->coverage, 1U); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(core->capture()->coverage, 2U); +} + +TEST_F(FlushCaptureReport, CapturingACompletedHandoffDoesNotCreditItAgain) +{ + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(core->prepare_flush(&operation).ok()); + ASSERT_TRUE(operation->wait().ok()); + EXPECT_TRUE(operation->result().handoff_advanced); + FlushResult capture; + ASSERT_TRUE(operation->wait_capture(&capture).ok()); + EXPECT_EQ(capture.frontier, 1U); + ASSERT_TRUE(operation->wait().ok()); + EXPECT_EQ(operation->result().phase, FlushPhase::kComplete); + EXPECT_FALSE(operation->result().handoff_advanced); +} + +TEST_F(FlushCaptureReport, RepeatedCapturePreservesTheOriginalPairAfterHandoff) +{ + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(core->prepare_flush(&operation).ok()); + FlushResult capture; + ASSERT_TRUE(operation->wait_capture(&capture).ok()); + const auto frozen = operation->frozen_sources(); + ASSERT_TRUE(frozen.catalog); + ASSERT_TRUE(operation->wait().ok()); + EXPECT_TRUE(operation->result().handoff_advanced); + ASSERT_TRUE(operation->wait_capture(&capture).ok()); + ASSERT_TRUE(operation->wait().ok()); + EXPECT_EQ(operation->result().phase, FlushPhase::kComplete); + EXPECT_FALSE(operation->result().handoff_advanced); + EXPECT_EQ(operation->frozen_sources().catalog, frozen.catalog); + EXPECT_EQ(operation->frozen_sources().base, frozen.base); + EXPECT_EQ(operation->frozen_sources().frontier, frozen.frontier); +} + +TEST_F(FlushCaptureReport, CoveredFlushReportsRemainingDirtyPages) +{ + ASSERT_TRUE(tree->flush().ok()); + const auto remaining = core->capture()->stats.dirty_pages; + ASSERT_GT(remaining, 0U); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(core->prepare_flush(&operation).ok()); + ASSERT_TRUE(operation->wait().ok()); + EXPECT_EQ(operation->result().deferred_pages, remaining); + EXPECT_FALSE(operation->result().handoff_advanced); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/flush_continuation_test.cpp b/lib/crowdb-tree/tests/integration/flush_continuation_test.cpp new file mode 100644 index 000000000..19268c9fe --- /dev/null +++ b/lib/crowdb-tree/tests/integration/flush_continuation_test.cpp @@ -0,0 +1,188 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_batch.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +void initialize(PublicationCoordinator &publisher, const std::vector<std::shared_ptr<MemTable>> &tables) +{ + auto leaf = std::make_shared<PublishedPage>(); + leaf->leaf = std::make_shared<PageState>(); + auto sources = std::make_shared<PublishedSources>(); + sources->tables.assign(tables.begin(), tables.end()); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{}), + .pages = {{.page_id = 0, .expected = {}, .replacement = leaf}}, + .root_page_id = 0, + .expected_sources = publisher.capture()->sources, + .sources = sources}; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(request)); +} + +std::shared_ptr<MemTable> make_source(const std::vector<std::string> &keys, uint64_t slot, Slice value) +{ + auto table = std::make_shared<MemTable>(); + for (const auto &key : keys) { + EXPECT_TRUE(table->upsert(key, slot, encode_cell_buf(slot, OpKind::kPut, value))); + } + table->close(); + return table; +} + +TEST(FlushContinuation, EntryAndArenaWindowsPreserveBinaryKeysAndAllSourceWinners) +{ + for (const size_t workers : {1U, 4U}) { + for (const RunLimits limits : { + RunLimits{.max_entries = 1}, + RunLimits{.max_entries = 100, .max_key_bytes = 7, .key_block_bytes = 4} + }) { + const std::vector<std::string> keys{"", std::string("\0", 1), std::string("a\0b", 3), "aaaa", "zz"}; + auto first = make_source(keys, 1, "older"); + auto second = make_source(keys, 2, "newer"); + auto future = make_source(keys, 3, "after-frontier"); + detail::FlushRuntime runtime(workers, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize(publisher, {first, second, future}); + const auto before = publisher.capture(); + FlushBatchOptions options; + options.run_limits = limits; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 2, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + const auto after = publisher.capture(); + EXPECT_EQ(after->coverage, 2U); + EXPECT_EQ(after->stats.overlay_entries, keys.size()); + const auto &runs = after->mapping.get(0)->leaf->overlays; + ASSERT_GT(runs.size(), 1U); + std::vector<std::string> actual; + for (const auto &run : runs) { + EXPECT_LE(run->entries().size(), limits.max_entries); + EXPECT_LE(run->key_bytes(), limits.max_key_bytes); + for (const auto &entry : run->entries()) { + actual.push_back(entry.key().to_string()); + EXPECT_EQ(entry.slot(), 2U); + EXPECT_EQ(entry.value().to_string(), "newer"); + auto guard = second->read_guard(); + EXPECT_EQ(entry.owner.get(), second->find(entry.key())); + } + } + EXPECT_EQ(actual, keys); + EXPECT_TRUE(before->mapping.get(0)->leaf->overlays.empty()); + EXPECT_EQ(before->sources->tables.size(), 3U); + ASSERT_EQ(after->sources->tables.size(), 1U); + EXPECT_EQ(after->sources->tables.front(), future); + } + } +} + +TEST(FlushContinuation, OversizedKeyDoesNotPublishOrLoopAfterPreparedPrefix) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize(publisher, {make_source({"a", "b", "cccc"}, 1, "v")}); + FlushBatchOptions options; + options.run_limits.max_entries = 1; + options.run_limits.max_key_bytes = 3; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + for (size_t attempt = 0; attempt < 2; ++attempt) { + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_EQ(batch->result().handoff_status.code(), Code::kResourceExhausted); + EXPECT_EQ(batch->result().phase, FlushPhase::kRepairRequired); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 0U); + EXPECT_EQ(publisher.capture()->sources->tables.size(), 1U); + } +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(FlushContinuation, RejectedContinuationRetainsPrefixAndYieldsWorkerBetweenRuns) +{ + detail::FlushRuntime runtime(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + PublicationCoordinator publisher; + initialize(publisher, {make_source({"a", "b", "c"}, 1, "v")}); + FlushBatchOptions options; + options.run_limits.max_entries = 1; + std::vector<size_t> prepared; + std::vector<uint64_t> completed; + bool other_work_ran = false; + options.after_run_for_tests = [&](size_t, size_t runs) { + prepared.push_back(runs); + completed.push_back(runtime.snapshot().completed); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 0U); + if (runs == 1) { + EXPECT_TRUE( + runtime + .try_submit( + lane, [](void *context) noexcept { *static_cast<bool *>(context) = true; }, &other_work_ran) + .ok()); + } + }; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(other_work_ran); + EXPECT_EQ(batch->result().handoff_status.code(), Code::kResourceExhausted); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_EQ(prepared, (std::vector<size_t>{1})); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(batch->result().handoff_status.ok()); + EXPECT_EQ(prepared, (std::vector<size_t>{1, 2, 3})); + ASSERT_EQ(completed.size(), 3U); + EXPECT_GT(completed[1], completed[0]); + EXPECT_GT(completed[2], completed[1]); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 3U); + EXPECT_EQ(publisher.capture()->coverage, 1U); +} + +TEST(FlushContinuation, CancellationBetweenRunsOrBeforePublicationResumesWithoutRebuilding) +{ + for (const size_t cancel_after : {1U, 3U}) { + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize(publisher, {make_source({"a", "b", "c"}, 1, "v")}); + FlushBatchOptions options; + options.run_limits.max_entries = 1; + std::shared_ptr<FlushBatch> batch; + std::vector<size_t> prepared; + options.after_run_for_tests = [&](size_t, size_t runs) { + prepared.push_back(runs); + if (runs == cancel_after) { + batch->request_stop(); + } + }; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_EQ(batch->result().phase, FlushPhase::kRepairRequired); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_EQ(publisher.capture()->sources->tables.size(), 1U); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 0U); + EXPECT_EQ(prepared.size(), cancel_after); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(batch->result().handoff_status.ok()); + EXPECT_EQ(prepared, (std::vector<size_t>{1, 2, 3})); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 3U); + EXPECT_EQ(publisher.capture()->coverage, 1U); + EXPECT_TRUE(publisher.capture()->sources->tables.empty()); + } +} +#endif +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/flush_dispatch_test.cpp b/lib/crowdb-tree/tests/integration/flush_dispatch_test.cpp new file mode 100644 index 000000000..ab1b922be --- /dev/null +++ b/lib/crowdb-tree/tests/integration/flush_dispatch_test.cpp @@ -0,0 +1,157 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_batch.h" + +#include <gtest/gtest.h> + +#include <future> +#include <latch> + +namespace crowdb::tree +{ +namespace +{ +void initialize_dispatch(PublicationCoordinator &publisher, size_t ranges, size_t entries, size_t value_bytes = 1) +{ + auto table = std::make_shared<MemTable>(); + auto inner = std::make_shared<InnerPageState>(); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{}); + for (size_t index = 0; index < ranges; ++index) { + auto logical = std::make_shared<PageState>(); + if (index != 0) { + logical->range.lower = OwnedKey(std::to_string(1000 + (100 * index))); + inner->separators.push_back(*logical->range.lower); + } + if (index + 1 < ranges) { + logical->range.upper = OwnedKey(std::to_string(1100 + (100 * index))); + logical->right_sibling = index + 1; + } + inner->children.push_back(index); + auto page = std::make_shared<PublishedPage>(); + page->leaf = logical; + request->transaction.pages.push_back({.page_id = index, .expected = {}, .replacement = page}); + for (size_t entry = 0; entry < entries; ++entry) { + const auto key = std::to_string(1000 + (100 * index) + entry); + ASSERT_TRUE(table->upsert(key, 1, encode_cell_buf(1, OpKind::kPut, std::string(value_bytes, 'v')))); + } + } + table->close(); + auto sources = std::make_shared<PublishedSources>(); + sources->tables.push_back(table); + request->transaction.expected_sources = publisher.capture()->sources; + request->transaction.sources = sources; + auto root = std::make_shared<PublishedPage>(); + root->level = 1; + root->metrics.leaf = false; + root->inner = inner; + request->transaction.pages.push_back({.page_id = ranges, .expected = {}, .replacement = root}); + request->transaction.root_page_id = ranges; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(request)); +} + +TEST(FlushDispatch, SmallRangesFinishWithinTheRoutingTask) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize_dispatch(publisher, 2, 2); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, {}, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()); + const auto metrics = publisher.work_metrics()->dispatch.snapshot(); + EXPECT_EQ(metrics.inline_probes, 2U); + EXPECT_EQ(metrics.inline_ranges, 2U); + EXPECT_EQ(metrics.queued_steps, 0U); + EXPECT_EQ(metrics.prepared_entries, 4U); + EXPECT_EQ(metrics.active_steps, 0U); + EXPECT_EQ(metrics.peak_steps, 1U); + EXPECT_GT(metrics.cow_bytes, 0U); +} + +TEST(FlushDispatch, LargeValuesYieldAfterEachByteBoundedRun) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize_dispatch(publisher, 1, 5, 128); + FlushBatchOptions options; + options.inline_bytes = 64; + options.max_run_bytes = 64; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()); + const auto metrics = publisher.work_metrics()->dispatch.snapshot(); + EXPECT_EQ(metrics.inline_probes, 1U); + EXPECT_EQ(metrics.inline_ranges, 0U); + EXPECT_EQ(metrics.queued_steps, 4U); + EXPECT_EQ(metrics.prepared_runs, 5U); + EXPECT_EQ(metrics.prepared_entries, 5U); + EXPECT_EQ(metrics.prepared_bytes, 5U * (4 + 128 + kCellHeaderSize)); + EXPECT_EQ(publisher.capture()->coverage, 1U); + for (const auto &run : publisher.capture()->mapping.get(0)->leaf->overlays) { + EXPECT_EQ(run->entries().size(), 1U); + } +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(FlushDispatch, InlineQuotaBoundsEachCpuTurn) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize_dispatch(publisher, 4, 1); + FlushBatchOptions options; + options.inline_ranges_per_step = 1; + + std::vector<uint64_t> completed; + options.after_run_for_tests = [&](size_t, size_t) { completed.push_back(runtime.snapshot().completed); }; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()); + ASSERT_EQ(completed.size(), 4U); + for (size_t index = 1; index < completed.size(); ++index) { + EXPECT_GT(completed[index], completed[index - 1]); + } + EXPECT_GT(publisher.work_metrics()->dispatch.snapshot().queued_steps, 0U); +} + +TEST(FlushDispatch, ActualParallelStepsRemainSeparateFromCoverage) +{ + detail::FlushRuntime runtime(2, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize_dispatch(publisher, 2, 2); + FlushBatchOptions options; + options.inline_ranges_per_step = 0; + std::latch entered(2); + std::latch release(1); + options.after_run_for_tests = [&](size_t, size_t) { + entered.count_down(); + release.wait(); + }; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + auto done = std::async(std::launch::async, [&] { return stdexec::sync_wait(batch->sender()).has_value(); }); + entered.wait(); + const auto metrics = publisher.work_metrics()->dispatch.snapshot(); + EXPECT_EQ(metrics.active_steps, 2U); + EXPECT_GE(metrics.peak_steps, 2U); + EXPECT_EQ(publisher.capture()->coverage, 0U); + release.count_down(); + EXPECT_TRUE(done.get()); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()); + EXPECT_EQ(publisher.work_metrics()->dispatch.snapshot().active_steps, 0U); + EXPECT_EQ(publisher.capture()->coverage, 1U); +} +#endif +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/flush_operation_test.cpp b/lib/crowdb-tree/tests/integration/flush_operation_test.cpp new file mode 100644 index 000000000..d6cf31cb6 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/flush_operation_test.cpp @@ -0,0 +1,394 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_operation.h" + +#include <gtest/gtest.h> + +#include <future> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct FlushCredits +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(4 * 1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> reserve() const + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, 1024 * 1024); + if (!credit.has_value()) { + ADD_FAILURE() << "test reservation failed"; + return {}; + } + return std::make_shared<detail::AllocationReservation>(std::move(credit.value())); + } +}; + +void initialize_sources(PublicationCoordinator &publisher, SourceAdmission &admission) +{ + auto sources = std::make_shared<PublishedSources>(); + const auto catalog = admission.catalog(); + sources->floor = catalog->floor; + sources->tables.assign(catalog->tables.begin(), catalog->tables.end()); + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = { + .record = std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 0, .range = 0, .sequence = 0}), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0, + .expected_sources = publisher.capture()->sources, + .sources = std::move(sources)}; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(request)); +} + +struct OperationReceiver +{ + using receiver_concept = stdexec::receiver_tag; + std::promise<void> *done; + + void set_value() const noexcept + { + done->set_value(); + } + + void set_error(Status status) const noexcept + { + ADD_FAILURE() << status.to_string(); + done->set_value(); + } + + void set_stopped() const noexcept + { + ADD_FAILURE() << "expected progress-bearing value"; + done->set_value(); + } +}; + +TEST(FlushOperation, CapturedWriterDrainAndSuccessorWritesCompleteAsOneAsyncFlow) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + FlushCredits credits; + FlushController controller(publisher, runtime, sources, contiguous, lane); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + EXPECT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "first"))); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(controller.prepare(credits.reserve(), &flush).ok()); + std::promise<void> done; + auto terminal = done.get_future(); + auto operation = stdexec::connect(flush->sender(), OperationReceiver{&done}); + stdexec::start(operation); + runtime.drain(); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_EQ(publisher.capture()->sources->tables.size(), 2U); + EXPECT_NE(publisher.capture()->flush_admission_owner, 0U); + SourceAdmission::Writer successor; + ASSERT_TRUE(sources.try_admit(1, &successor).ok()); + EXPECT_NE(successor.table(), writer.table()); + EXPECT_TRUE(successor.table()->upsert("b", 2, encode_cell_buf(2, OpKind::kPut, "next"))); + successor.release(); + contiguous.store(2); + writer.release(); + terminal.get(); + runtime.drain(); + EXPECT_EQ(flush->result().frontier, 1U); + EXPECT_EQ(flush->result().published_coverage, 1U); + EXPECT_EQ(flush->result().phase, FlushPhase::kComplete); + EXPECT_EQ(sources.catalog()->floor, 1U); + EXPECT_EQ(publisher.capture()->sources->floor, 1U); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 0U); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("a").cell.value.to_string(), "first"); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("b").state, LogicalLookupState::kMissing); + std::shared_ptr<FlushOperation> next; + ASSERT_TRUE(controller.prepare(credits.reserve(), &next).ok()); + EXPECT_NE(next, flush); + ASSERT_TRUE(stdexec::sync_wait(next->sender()).has_value()); + runtime.drain(); + EXPECT_EQ(next->result().published_coverage, 2U); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("b").cell.value.to_string(), "next"); +} + +TEST(FlushOperation, IncompatibleSourceKeyLimitRejectsBeforePublicationAdmissionOrSourceClosure) +{ + FlushCredits credits; + SourceAdmission sources(0, 1, {.account = credits.account, .max_key_size = 8}); + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + FlushBatchOptions options; + options.run_limits.max_key_bytes = 4; + FlushController controller(publisher, runtime, sources, contiguous, lane, options); + const auto before = publisher.capture(); + const auto active = sources.catalog()->active; + auto metadata = credits.reserve(); + const auto available = metadata->available(); + std::shared_ptr<FlushOperation> operation; + EXPECT_EQ(controller.prepare(metadata, &operation).code(), Code::kInvalidArgument); + EXPECT_FALSE(operation); + EXPECT_FALSE(active->closed()); + EXPECT_EQ(publisher.capture(), before); + EXPECT_EQ(metadata->available(), available); +} + +TEST(FlushOperation, NoProgressReturnsCreditWithoutRotatingSourcesOrPublishing) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{0}; + FlushCredits credits; + FlushController controller(publisher, runtime, sources, contiguous, lane); + const auto before = sources.catalog(); + const auto tree_before = publisher.capture(); + auto metadata = credits.reserve(); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(metadata, &operation).ok()); + ASSERT_TRUE(stdexec::sync_wait(operation->sender()).has_value()); + EXPECT_EQ(operation->result().phase, FlushPhase::kComplete); + EXPECT_EQ(operation->result().published_coverage, 0U); + EXPECT_EQ(metadata->available(), 0U); + EXPECT_EQ(sources.catalog(), before); + EXPECT_FALSE(before->active->closed()); + EXPECT_EQ(publisher.capture(), tree_before); +} + +TEST(FlushOperation, MaintenanceRetryReusesBatchAfterCoverageAckAndMetadataSeal) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + FlushCredits credits; + auto metadata = credits.reserve(); + size_t attempts = 0; + FlushController controller(publisher, runtime, sources, contiguous, lane, {}, [&](auto view, auto done) { + EXPECT_EQ(sources.catalog()->floor, view->coverage); + EXPECT_EQ(metadata->available(), 0U); + ++attempts; + done(attempts == 1 ? Status::io_error("controlled maintenance error") : Status::Ok(), 1); + }); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + EXPECT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "value"))); + std::weak_ptr<MemTable> old_source = writer.table(); + writer.release(); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(metadata, &operation).ok()); + ASSERT_TRUE(stdexec::sync_wait(operation->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()); + EXPECT_EQ(operation->result().maintenance_status.code(), Code::kIoError); + EXPECT_EQ(operation->result().published_coverage, 1U); + auto *original = operation.get(); + const auto batch = operation->result().batch_id; + auto covered = publisher.capture(); + operation.reset(); + ASSERT_TRUE(controller.prepare(credits.reserve(), &operation).ok()); + EXPECT_EQ(operation.get(), original); + ASSERT_TRUE(stdexec::sync_wait(operation->sender()).has_value()); + runtime.drain(); + EXPECT_EQ(operation->result().batch_id, batch); + EXPECT_TRUE(operation->result().maintenance_status.ok()); + EXPECT_EQ(attempts, 2U); + EXPECT_EQ(publisher.capture(), covered); + publisher.reclaim(); + sources.reclaim(); + EXPECT_TRUE(old_source.expired()); +} + +TEST(FlushOperation, CancellationDuringDrainKeepsRepairOwnerAndFixedFrontier) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + FlushCredits credits; + FlushController controller(publisher, runtime, sources, contiguous, lane); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + EXPECT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "value"))); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(credits.reserve(), &operation).ok()); + std::promise<void> done; + auto terminal = done.get_future(); + auto connected = stdexec::connect(operation->sender(), OperationReceiver{&done}); + stdexec::start(connected); + runtime.drain(); + operation->request_stop(); + writer.release(); + terminal.get(); + EXPECT_EQ(operation->result().phase, FlushPhase::kRepairRequired); + EXPECT_EQ(operation->result().published_coverage, 0U); + EXPECT_NE(publisher.capture()->flush_admission_owner, 0U); + const auto batch = operation->result().batch_id; + contiguous.store(10); + std::shared_ptr<FlushOperation> retry; + ASSERT_TRUE(controller.prepare(credits.reserve(), &retry).ok()); + EXPECT_EQ(retry, operation); + ASSERT_TRUE(stdexec::sync_wait(retry->sender()).has_value()); + runtime.drain(); + EXPECT_EQ(retry->result().batch_id, batch); + EXPECT_EQ(retry->result().frontier, 1U); + EXPECT_EQ(retry->result().published_coverage, 1U); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 0U); +} + +TEST(FlushOperation, StaleReservationFailsBeforeSourceClosureAndCanBeReprepared) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + FlushCredits credits; + FlushController controller(publisher, runtime, sources, contiguous, lane); + const auto before = sources.catalog(); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(credits.reserve(), &operation, publisher.capture()->commit_seq - 1).ok()); + ASSERT_TRUE(stdexec::sync_wait(operation->sender()).has_value()); + runtime.drain(); + EXPECT_EQ(operation->result().handoff_status.code(), Code::kUnavailable); + EXPECT_EQ(sources.catalog(), before); + EXPECT_FALSE(before->active->closed()); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 0U); + std::shared_ptr<FlushOperation> next; + ASSERT_TRUE(controller.prepare(credits.reserve(), &next).ok()); + EXPECT_NE(next, operation); + ASSERT_TRUE(stdexec::sync_wait(next->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(next->result().handoff_status.ok()) << next->result().handoff_status.to_string(); + EXPECT_EQ(next->result().published_coverage, 1U); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 0U); +} + +TEST(FlushOperation, CaptureAllocationFailureReleasesGateUsingIndependentCredit) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + FlushCredits credits; + auto metadata = credits.reserve(); + const auto before = sources.catalog(); + std::optional<AllocationLease> exhausted; + FlushBatchOptions options; + options.before_capture_for_tests = [&] { + EXPECT_NE(publisher.capture()->flush_admission_owner, 0U); + EXPECT_FALSE(before->active->closed()); + exhausted.emplace(metadata, metadata->available()); + }; + FlushController controller(publisher, runtime, sources, contiguous, lane, options); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(metadata, &operation).ok()); + ASSERT_TRUE(stdexec::sync_wait(operation->sender()).has_value()); + runtime.drain(); + EXPECT_EQ(operation->result().handoff_status.code(), Code::kResourceExhausted); + EXPECT_EQ(sources.catalog(), before); + EXPECT_FALSE(before->active->closed()); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 0U); + EXPECT_EQ(publisher.capture()->coverage, 0U); + exhausted.reset(); +} + +TEST(FlushOperation, MissingPublishedSourceCatalogRejectsBeforeClosingActive) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + FlushCredits credits; + FlushController controller(publisher, runtime, sources, contiguous, lane); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(credits.reserve(), &operation).ok()); + ASSERT_TRUE(stdexec::sync_wait(operation->sender()).has_value()); + EXPECT_EQ(operation->result().handoff_status.code(), Code::kUnavailable); + EXPECT_FALSE(sources.catalog()->active->closed()); + EXPECT_EQ(sources.catalog()->tables.size(), 1U); + EXPECT_EQ(publisher.capture()->coverage, 0U); +} + +TEST(FlushOperation, NoProgressCompletionRunsOnTheSharedCpuRuntime) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{0}; + FlushCredits credits; + FlushController controller(publisher, runtime, sources, contiguous, lane); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(credits.reserve(), &operation).ok()); + std::thread::id completion_thread; + ASSERT_TRUE(stdexec::sync_wait(stdexec::then(operation->sender(), [&] { + completion_thread = std::this_thread::get_id(); + })).has_value()); + runtime.drain(); + EXPECT_NE(completion_thread, std::this_thread::get_id()); + EXPECT_GT(runtime.snapshot().submitted, 0U); +} + +TEST(FlushOperation, ClosingIngressPreservesAlreadyCapturedWriterDrainAndCoverage) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + FlushCredits credits; + FlushController controller(publisher, runtime, sources, contiguous, lane); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + ASSERT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "value"))); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(credits.reserve(), &operation).ok()); + std::promise<void> done; + auto terminal = done.get_future(); + auto connected = stdexec::connect(operation->sender(), OperationReceiver{&done}); + stdexec::start(connected); + runtime.drain(); + ASSERT_EQ(sources.catalog()->tables.size(), 2U); + controller.close(); + sources.close(); + SourceAdmission::Writer rejected; + EXPECT_EQ(sources.try_admit(1, &rejected).code(), Code::kUnavailable); + writer.release(); + terminal.get(); + runtime.drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()); + EXPECT_EQ(operation->result().published_coverage, 1U); + EXPECT_EQ(sources.catalog()->floor, 1U); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("a").cell.value.to_string(), "value"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/flush_range_scaling_test.cpp b/lib/crowdb-tree/tests/integration/flush_range_scaling_test.cpp new file mode 100644 index 000000000..c4d1a957d --- /dev/null +++ b/lib/crowdb-tree/tests/integration/flush_range_scaling_test.cpp @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_batch.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +void seed_wide_tree(PublicationCoordinator &publisher, const std::shared_ptr<MemTable> &source, size_t leaves) +{ + auto request = std::make_shared<PublicationRequest>(); + std::vector<std::pair<uint64_t, std::optional<OwnedKey>>> level; + for (size_t index = 0; index < leaves; ++index) { + const auto key = "k" + std::to_string(10000 + index); + ASSERT_TRUE(source->upsert(key, 1, encode_cell_buf(1, OpKind::kPut, "value"))); + auto leaf = std::make_shared<PageState>(); + if (index != 0) { + leaf->range.lower = key; + } + if (index + 1 != leaves) { + leaf->range.upper = "k" + std::to_string(10001 + index); + leaf->right_sibling = index + 1; + } + auto page = std::make_shared<PublishedPage>(); + page->leaf = leaf; + level.emplace_back(index, leaf->range.lower); + request->transaction.pages.push_back({.page_id = index, .replacement = page}); + } + uint64_t next_pid = leaves; + uint32_t depth = 0; + while (level.size() > 1) { + ++depth; + std::vector<std::pair<uint64_t, std::optional<OwnedKey>>> parents; + for (size_t first = 0; first < level.size(); first += 64) { + auto inner = std::make_shared<InnerPageState>(); + for (size_t child = first; child < std::min(first + 64, level.size()); ++child) { + inner->children.push_back(level[child].first); + if (child != first) { + const auto &lower = level[child].second; + if (!lower) { + FAIL() << "a nonfirst child needs a lower separator"; + } + inner->separators.push_back(*lower); + } + } + auto page = std::make_shared<PublishedPage>(); + page->level = depth; + page->metrics.leaf = false; + page->inner = std::move(inner); + parents.emplace_back(next_pid, level[first].second); + request->transaction.pages.push_back({.page_id = next_pid++, .replacement = page}); + } + level = std::move(parents); + } + source->close(); + auto inputs = std::make_shared<PublishedSources>(); + inputs->tables.push_back(source); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{}); + request->transaction.root_page_id = level.front().first; + request->transaction.expected_next_page_id = publisher.capture()->next_page_id; + request->transaction.next_page_id = next_pid; + request->transaction.expected_sources = publisher.capture()->sources; + request->transaction.sources = inputs; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(request)); +} + +TEST(FlushRangeScaling, PrepaidHandoffCrosses4096LeavesWithoutATreeSizeLimit) +{ + constexpr size_t leaves = 4097; + detail::FlushRuntime runtime(4, 32); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(32); + PublicationCoordinator publisher; + auto source = std::make_shared<MemTable>(); + seed_wide_tree(publisher, source, leaves); + const auto view = publisher.capture(); + ASSERT_EQ(view->mapping.copy_bounds().leaf_pages, leaves); + FlushBatchOptions options; + // Admission includes future writers, so its range bound may exceed the + // currently linked entries as well as the old 4096-range ceiling. + const auto bound = FlushBatch::metadata_reserve_bound(view->mapping, 1, 8192, 8192 * 6, 4, options); + if (!bound) { + FAIL() << "the finite range metadata bound must fit size_t"; + } + detail::ResourceBudget::Limits limits{}; + limits[static_cast<size_t>(detail::BudgetClass::kHandoff)] = *bound; + detail::ResourceBudget budget(limits, 4); + auto account = budget.make_account(limits); + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, *bound); + if (!credit) { + FAIL() << "the exact prepaid handoff grant must fit: " << *bound; + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch, {}, metadata).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + EXPECT_EQ(publisher.capture()->coverage, 1U); + EXPECT_TRUE(publisher.capture()->sources->tables.empty()); + for (size_t index = 0; index < leaves; ++index) { + const auto key = "k" + std::to_string(10000 + index); + EXPECT_EQ(publisher.capture()->mapping.get(index)->leaf->lookup(key).cell.value.to_string(), "value"); + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp b/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp new file mode 100644 index 000000000..3695f93ef --- /dev/null +++ b/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp @@ -0,0 +1,638 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_preparation_test_support.h" + +#include <future> + +namespace crowdb::tree +{ +namespace +{ + +struct Receiver +{ + using receiver_concept = stdexec::receiver_tag; + std::promise<void> *terminal; + + void set_value() const noexcept + { + terminal->set_value(); + } + + void set_error(Status status) const noexcept + { + ADD_FAILURE() << status.to_string(); + terminal->set_value(); + } + + void set_stopped() const noexcept + { + ADD_FAILURE() << "flush must return its progress result"; + terminal->set_value(); + } +}; + +TEST(FlushReservation, AdmittedUnappliedWriterAndMaintenanceFinishWithAllUnusedPartitionsExhausted) +{ + for (const size_t workers : {1U, 4U}) { + TestPreparedFlushTree tree(workers); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {16, 32, 0, 2}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a0", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + tree.frontier.store(1); + std::shared_ptr<FlushOperation> operation; + const auto admitted = tree.controller->prepare(tree.preparation, &operation); + ASSERT_TRUE(admitted.ok()) << admitted.to_string(); + auto blockers = tree.exhaust_unused(); + // The reservation predates these writes. Only after they have applied + // may the host advance the completed frontier captured by flush. + for (uint64_t i = 2; i <= 16; ++i) { + const auto key = std::string(i % 2 == 0 ? "a" : "z") + static_cast<char>('A' + i); + ASSERT_TRUE(writer.upsert(key, i, encode_cell_buf(i, OpKind::kPut, "v")).ok()); + } + tree.frontier.store(16); + std::promise<void> terminal; + auto done = terminal.get_future(); + auto connected = stdexec::connect(operation->sender(), Receiver{&terminal}); + stdexec::start(connected); + tree.runtime->drain(); + EXPECT_TRUE(writer.table()->closed()); + EXPECT_EQ(tree.publisher.capture()->coverage, 0U); + writer.release(); + done.get(); + tree.runtime->drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()) << operation->result().handoff_status.to_string(); + EXPECT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.to_string(); + EXPECT_EQ(operation->result().published_coverage, 16U); + EXPECT_EQ(tree.publisher.capture()->flush_admission_owner, 0U); + const auto snapshot = tree.budget.snapshot(); + const auto lifecycle_reserved = tree.lifecycle_reservations(); + for (size_t category = 0; category < blockers.size(); ++category) { + const auto &blocker = blockers[category]; + if (!blocker.has_value()) { + ADD_FAILURE() << "test unused credit blocker missing"; + return; + } + EXPECT_EQ(snapshot.reserved[category], blocker->amount() + lifecycle_reserved[category]); + EXPECT_LE(snapshot.reserved[category] + snapshot.allocated[category], + TestPreparedFlushTree::limits()[category]); + } + } +} + +TEST(FlushReservation, ColdRoutingReserveFollowsCapturedBaseCount) +{ + TestPreparedFlushTree tree; + auto view = *tree.publisher.capture(); + const auto sources = tree.sources.catalog(); + const auto bound = [&](size_t source_limit) { + auto preparation = tree.preparation; + preparation.maintenance.publication.pages.max_sources = source_limit; + detail::FlushReserveBounds result; + const auto status = + detail::flush_reserve_bounds(tree.publisher, view, *sources, preparation, 256 * 1024, tree.batch, &result); + EXPECT_TRUE(status.ok()) << status.to_string(); + return result.amounts[1]; + }; + const auto empty = bound(1); + EXPECT_EQ(bound(64), empty); + EXPECT_LT(empty, TestPreparedFlushTree::limits()[static_cast<size_t>(detail::BudgetClass::kRouting)]); + + auto page = std::make_shared<PublishedPage>(*view.mapping.get(0)); + auto leaf = std::make_shared<PageState>(*page->leaf); + for (uint64_t pid = 0; pid < 4; ++pid) { + leaf->bases.push_back(test_read::cold(pid)); + } + page->leaf = std::move(leaf); + const std::array replacements{ + PageReplacement{.page_id = 0, .expected = view.mapping.get(0), .replacement = std::move(page)} + }; + MappingView::CopyStats copied; + view.mapping = view.mapping.replace(replacements, copied); + EXPECT_GT(bound(4), empty); + EXPECT_EQ(bound(64), bound(4)); + // Primary maintenance retains overflow references without reading chains. + const auto primary_only = bound(64); + tree.preparation.loads.max_overflow_pages = SIZE_MAX; + EXPECT_EQ(bound(64), primary_only); + EXPECT_FALSE(AsyncStorePageLoader::read_reserve_bound(tree.preparation.loads, 1, 1, 4, 0, 0)); +} + +TEST(FlushReservation, GrantedCompletePartitionsBindWithoutReacquiringUnderHostPressure) +{ + TestPreparedFlushTree tree(1, true); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {1, 1, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + writer.release(); + tree.frontier.store(1); + const auto view = tree.publisher.capture(); + const auto sources = tree.sources.catalog(); + detail::FlushReserveBounds bounds; + // This caller supplies a finite control arena for the operation factory. + constexpr size_t controls = 256 * 1024; + const auto before = tree.budget.snapshot(); + ASSERT_TRUE( + detail::flush_reserve_bounds(tree.publisher, *view, *sources, tree.preparation, controls, tree.batch, &bounds) + .ok()); + EXPECT_EQ(tree.budget.snapshot().reserved, before.reserved); + EXPECT_EQ(tree.budget.snapshot().allocated, before.allocated); + detail::FlushCredits credits; + for (size_t index = 0; index < credits.size(); ++index) { + detail::ResourceBudget::Request request(tree.account, detail::FlushReserveBounds::categories[index], + bounds.amounts[index]); + ASSERT_TRUE(stdexec::sync_wait(request.sender())); + auto paid = request.take_credit(); + if (!paid) { + FAIL() << "missing paid test prerequisite"; + } + credits[index] = std::move(*paid); + } + auto blockers = tree.exhaust_unused(); + auto batch = tree.batch; + FlushBatch::Maintenance maintenance; + detail::FlushReservations reserved; + ASSERT_TRUE(detail::bind_flush_reservations(tree.publisher, tree.preparation, tree.account, tree.lane, bounds, + std::move(credits), &batch, &maintenance, &reserved) + .ok()); + FlushController controller(tree.publisher, *tree.runtime, tree.sources, tree.frontier, tree.lane, std::move(batch), + std::move(maintenance)); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(reserved.handoff, &operation, view->commit_seq).ok()); + ASSERT_TRUE(stdexec::sync_wait(operation->report_sender())); + tree.runtime->drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()) << operation->result().handoff_status.to_string(); + EXPECT_EQ(tree.publisher.capture()->coverage, 1U); + EXPECT_GT(tree.store->reads.load(), 0U); + reserved.seal(); +} + +TEST(FlushReservation, BindingRejectsWrongPartitionBeforeConstructingConsumers) +{ + TestPreparedFlushTree tree; + const auto view = tree.publisher.capture(); + const auto sources = tree.sources.catalog(); + detail::FlushReserveBounds bounds; + ASSERT_TRUE( + detail::flush_reserve_bounds(tree.publisher, *view, *sources, tree.preparation, 256 * 1024, tree.batch, &bounds) + .ok()); + detail::FlushCredits credits; + for (size_t index = 0; index < credits.size(); ++index) { + auto paid = detail::ResourceBudget::try_reserve( + tree.account, index == 3 ? detail::BudgetClass::kRouting : detail::FlushReserveBounds::categories[index], + bounds.amounts[index]); + if (!paid) { + FAIL() << "missing paid test prerequisite"; + } + credits[index] = std::move(*paid); + } + const auto before = tree.budget.snapshot(); + auto batch = tree.batch; + FlushBatch::Maintenance maintenance; + detail::FlushReservations reserved; + EXPECT_EQ(detail::bind_flush_reservations(tree.publisher, tree.preparation, tree.account, tree.lane, bounds, + std::move(credits), &batch, &maintenance, &reserved) + .code(), + Code::kInvalidArgument); + EXPECT_FALSE(batch.loader); + EXPECT_FALSE(maintenance); + EXPECT_FALSE(sources->active->closed()); + EXPECT_EQ(tree.budget.snapshot().allocated, before.allocated); + EXPECT_EQ(tree.budget.snapshot().reserved, tree.lifecycle_reservations()); +} + +detail::FlushCredits grant_plan(TestPreparedFlushTree &tree, const FlushReservePlan &plan) +{ + detail::FlushCredits credits; + for (size_t index = 0; index < credits.size(); ++index) { + detail::ResourceBudget::Request request(tree.account, detail::FlushReserveBounds::categories[index], + plan.bounds.amounts[index]); + const auto completed = stdexec::sync_wait(request.sender()); + if (!completed) { + throw std::runtime_error("test flush grant stopped"); + } + auto paid = request.take_credit(); + if (!paid) { + throw std::runtime_error("test flush grant is missing"); + } + credits[index] = std::move(*paid); + } + return credits; +} + +TEST(FlushReservation, ControllerConsumesPlannedGrantsAndReusesTheExactPendingOperation) +{ + TestPreparedFlushTree tree(1, true); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {1, 1, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + writer.release(); + tree.frontier.store(1); + const auto before = tree.budget.snapshot(); + FlushReservePlan plan; + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(tree.controller->prepare_or_plan(tree.preparation, &operation, &plan).ok()); + EXPECT_FALSE(operation); + ASSERT_TRUE(plan.view); + EXPECT_EQ(tree.budget.snapshot().reserved, before.reserved); + auto credits = grant_plan(tree, plan); + auto blockers = tree.exhaust_unused(); + ASSERT_TRUE(tree.controller->prepare_prepaid(tree.preparation, plan, std::move(credits), &operation).ok()); + FlushReservePlan unused; + std::shared_ptr<FlushOperation> same; + ASSERT_TRUE(tree.controller->prepare_or_plan(tree.preparation, &same, &unused).ok()); + EXPECT_EQ(same, operation); + EXPECT_FALSE(unused.view); + auto completed = stdexec::sync_wait(operation->report_sender()); + if (!completed) { + FAIL() << "missing completed test prerequisite"; + } + const auto &report = std::get<0>(*completed); + EXPECT_TRUE(report.handoff_status.ok()) << report.handoff_status.to_string(); + EXPECT_EQ(report.published_coverage, 1U); + tree.runtime->drain(); +} + +TEST(FlushReservation, StalePlanReturnsAllGrantsBeforeSourceClosure) +{ + TestPreparedFlushTree tree; + tree.frontier.store(1); + FlushReservePlan plan; + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(tree.controller->prepare_or_plan(tree.preparation, &operation, &plan).ok()); + auto credits = grant_plan(tree, plan); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{}); + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(tree.publisher.try_submit(request)); + tree.runtime->drain(); + EXPECT_EQ(tree.controller->prepare_prepaid(tree.preparation, plan, std::move(credits), &operation).code(), + Code::kUnavailable); + EXPECT_FALSE(operation); + EXPECT_FALSE(plan.sources->active->closed()); + EXPECT_EQ(tree.budget.snapshot().reserved, tree.lifecycle_reservations()); +} + +TEST(FlushReservation, UncontendedPlanningIsNotAnApplyProgressTransition) +{ + TestPreparedFlushTree tree; + tree.frontier.store(1); + auto paid = detail::ResourceBudget::try_reserve(tree.account, detail::BudgetClass::kHandoff, 16384); + if (!paid) { + FAIL() << "missing paid test prerequisite"; + } + auto metadata = detail::AllocationReservation::create(std::move(*paid)); + std::shared_ptr<ApplyProgress> progress; + ASSERT_TRUE(ApplyProgress::create(2, metadata, &progress).ok()); + FlushController controller(tree.publisher, *tree.runtime, tree.sources, tree.frontier, tree.lane, tree.batch, {}, + {}, progress); + const auto sequence = progress->sequence(); + for (size_t attempt = 0; attempt < 4; ++attempt) { + FlushReservePlan plan; + std::shared_ptr<FlushOperation> existing; + ASSERT_TRUE(controller.prepare_or_plan(tree.preparation, &existing, &plan).ok()); + EXPECT_FALSE(existing); + EXPECT_EQ(progress->sequence(), sequence); + } +} + +TEST(FlushReservation, EachMissingProgressPartitionRollsBackBeforeSourceClosure) +{ + for (const auto category : {detail::BudgetClass::kHandoff, detail::BudgetClass::kRouting, + detail::BudgetClass::kReclaim, detail::BudgetClass::kIo}) { + TestPreparedFlushTree tree; + tree.frontier.store(1); + const auto before = tree.publisher.capture(); + const auto active = tree.sources.catalog()->active; + auto blocker = detail::ResourceBudget::try_reserve( + tree.account, category, TestPreparedFlushTree::limits()[static_cast<size_t>(category)]); + ASSERT_TRUE(blocker); + const auto budget_before = tree.budget.snapshot(); + std::shared_ptr<FlushOperation> operation; + EXPECT_EQ(tree.controller->prepare(tree.preparation, &operation).code(), Code::kUnavailable); + EXPECT_FALSE(operation); + EXPECT_FALSE(active->closed()); + EXPECT_EQ(tree.publisher.capture(), before); + EXPECT_EQ(tree.budget.snapshot().reserved, budget_before.reserved); + EXPECT_EQ(tree.budget.snapshot().allocated, budget_before.allocated); + } +} + +TEST(FlushReservation, ColdRoutingUsesHeldBytesAndIoUnderCompleteHostPressure) +{ + TestPreparedFlushTree tree(1, true); + tree.preparation.cache = std::make_shared<SharedPageCache>(PageCacheLimits{.max_pages = 2, .max_waiters = 1}); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {2, 2, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + ASSERT_TRUE(writer.upsert("z", 2, encode_cell_buf(2, OpKind::kPut, "v")).ok()); + writer.release(); + tree.frontier.store(2); + std::shared_ptr<FlushOperation> operation; + auto status = tree.controller->prepare(tree.preparation, &operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 1U); + auto blockers = tree.exhaust_unused(); + ASSERT_TRUE(stdexec::sync_wait(operation->sender())); + tree.runtime->drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()) << operation->result().handoff_status.to_string(); + EXPECT_EQ(operation->result().published_coverage, 2U); + EXPECT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.to_string(); + EXPECT_GT(operation->result().deferred_pages, 0U); + // Handoff caches the root once; cold maintenance has its own raw loader. + EXPECT_EQ(tree.store->reads.load(), 2U); + EXPECT_EQ(tree.budget.snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kIo)], 0U); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); + EXPECT_EQ(tree.preparation.cache->snapshot().misses, 1U); +} + +TEST(FlushReservation, DescendingColdRoutePidsUseHeldCreditForEveryLoadedLevel) +{ + TestPreparedFlushTree tree(1, true); + tree.preparation.cache = std::make_shared<SharedPageCache>(PageCacheLimits{.max_pages = 2, .max_waiters = 1}); + const auto before = tree.publisher.capture(); + auto root = std::make_shared<PublishedPage>(); + root->level = 2; + root->metrics.leaf = false; + root->inner_backing = test_read::inner_backing(tree.store, 1, *test_read::inner({}, {2})); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 2}), + .pages = {{.page_id = 3, .expected = {}, .replacement = std::move(root)}}, + .root_page_id = 3, + .expected_root_page_id = before->root_page_id}; + std::promise<Status> published; + auto installed = published.get_future(); + request->on_done = [&](PublicationResult result) { published.set_value(std::move(result.status)); }; + ASSERT_TRUE(tree.publisher.try_submit(request)); + ASSERT_TRUE(installed.get().ok()); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {2, 2, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + ASSERT_TRUE(writer.upsert("z", 2, encode_cell_buf(2, OpKind::kPut, "v")).ok()); + writer.release(); + tree.frontier.store(2); + std::shared_ptr<FlushOperation> operation; + const auto status = tree.controller->prepare(tree.preparation, &operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto blockers = tree.exhaust_unused(); + ASSERT_TRUE(stdexec::sync_wait(operation->sender())); + tree.runtime->drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()) << operation->result().handoff_status.to_string(); + EXPECT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.to_string(); + EXPECT_EQ(operation->result().published_coverage, 2U); + // Handoff and the separate inner-repair/layout windows each read the two + // descending route PIDs. Both maintenance windows use their held credit. + EXPECT_EQ(tree.store->reads.load(), 6U); + EXPECT_EQ(tree.preparation.cache->snapshot().misses, 2U); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); + EXPECT_EQ(tree.budget.snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kIo)], 0U); +} + +TEST(FlushReservation, LiveResidencyAndColdProxyPublicationUseHeldCreditUnderCompleteHostPressure) +{ + for (const size_t workers : {1U, 4U}) { + TestPreparedFlushTree tree(workers, true, true); + EXPECT_EQ(tree.resources->snapshot().proxy_pages, 3U); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {2, 2, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + ASSERT_TRUE(writer.upsert("z", 2, encode_cell_buf(2, OpKind::kPut, "v")).ok()); + writer.release(); + tree.frontier.store(2); + std::shared_ptr<FlushOperation> operation; + const auto status = tree.controller->prepare(tree.preparation, &operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto blockers = tree.exhaust_unused(); + ASSERT_TRUE(stdexec::sync_wait(operation->sender())); + tree.runtime->drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()) << operation->result().handoff_status.to_string(); + EXPECT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.to_string(); + EXPECT_EQ(operation->result().published_coverage, 2U); + // Handoff caches the inner. Maintenance reads the inner and first leaf + // directly, then publishes that decoded leaf with its exact summary. + EXPECT_EQ(tree.store->reads.load(), 3U); + EXPECT_EQ(tree.cache->snapshot().misses, 1U); + const auto view = tree.publisher.capture(); + ASSERT_TRUE(view->mapping.get(0)->leaf->bases.front().resident); + EXPECT_FALSE(view->mapping.get(1)->leaf->bases.front().resident); + EXPECT_FALSE(view->mapping.get(2)->inner); + EXPECT_EQ(tree.resources->snapshot().proxy_pages, 1U); + EXPECT_EQ(tree.resources->snapshot().resident_frames, 1U); + EXPECT_EQ(tree.resources->snapshot().cache_pinned_bytes, 0U); + EXPECT_EQ(tree.cache->snapshot().reserved_waiters, 0U); + EXPECT_EQ(tree.budget.snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kIo)], 0U); + tree.cache->evict(); + tree.runtime->drain(); + EXPECT_EQ(tree.resources->snapshot().proxy_pages, 2U); + EXPECT_EQ(tree.resources->snapshot().resident_frames, 0U); + EXPECT_EQ(tree.resources->snapshot().cache_bytes, 0U); + } +} + +TEST(FlushReservation, StaleMappingRejectsAndReleasesAllUnspentReservations) +{ + TestPreparedFlushTree tree; + tree.frontier.store(1); + const auto active = tree.sources.catalog()->active; + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &operation).ok()); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{}); + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(tree.publisher.try_submit(request)); + ASSERT_TRUE(stdexec::sync_wait(operation->sender())); + tree.runtime->drain(); + EXPECT_EQ(operation->result().handoff_status.code(), Code::kUnavailable); + EXPECT_FALSE(active->closed()); + EXPECT_EQ(tree.budget.snapshot().reserved, tree.lifecycle_reservations()); +} + +TEST(FlushReservation, BoundsRejectOverflowAndNeverWrapIntoSmallAdmission) +{ + EXPECT_FALSE(detail::AllocationBound(SIZE_MAX).add(1).value()); + EXPECT_FALSE(detail::AllocationBound().add_product(SIZE_MAX, 2).add_product(0, 1).value()); + TestPreparedFlushTree tree; + const auto view = tree.publisher.capture(); + EXPECT_FALSE(FlushBatch::metadata_reserve_bound(view->mapping, 1, SIZE_MAX, SIZE_MAX, 1, tree.batch)); + EXPECT_FALSE(SourceAdmission::capture_allocation_bound(SIZE_MAX)); + auto loads = tree.preparation.loads; + loads.max_blob_bytes = SIZE_MAX; + EXPECT_FALSE(AsyncStorePageLoader::routing_reserve_bound(loads, 1, 1)); + auto maintenance = tree.preparation.maintenance; + maintenance.max_candidates = SIZE_MAX; + EXPECT_FALSE(page_maintenance_reserve_bound(view->mapping, 0, maintenance)); +} + +TEST(FlushReservation, ClosedSourcesUseActualKeyBoundsOnlyAfterAcceptedWritersDrain) +{ + TestPreparedFlushTree tree; + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {2, 2, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "first")).ok()); + const auto sources = tree.sources.catalog(); + const auto view = tree.publisher.capture(); + detail::FlushReserveBounds open; + ASSERT_TRUE( + detail::flush_reserve_bounds(tree.publisher, *view, *sources, tree.preparation, 0, tree.batch, &open).ok()); + writer.table()->close(); + detail::FlushReserveBounds draining; + ASSERT_TRUE( + detail::flush_reserve_bounds(tree.publisher, *view, *sources, tree.preparation, 0, tree.batch, &draining).ok()); + EXPECT_EQ(draining.amounts, open.amounts); + ASSERT_TRUE(writer.upsert("z", 2, encode_cell_buf(2, OpKind::kPut, "last")).ok()); + writer.release(); + detail::FlushReserveBounds frozen; + ASSERT_TRUE( + detail::flush_reserve_bounds(tree.publisher, *view, *sources, tree.preparation, 0, tree.batch, &frozen).ok()); + EXPECT_LT(frozen.amounts[0], open.amounts[0]); + EXPECT_EQ(sources->tables.front()->source_stats().records, 2U); +} + +TEST(FlushReservation, CoveredPreparationFixesItsPrefixWithoutBlockingLaterWrites) +{ + for (const bool planned : {false, true}) { + TestPreparedFlushTree tree; + std::array<std::optional<detail::ResourceBudget::Credit>, 3> blockers; + constexpr std::array categories{detail::BudgetClass::kRouting, detail::BudgetClass::kReclaim, + detail::BudgetClass::kIo}; + for (size_t i = 0; i < categories.size(); ++i) { + blockers[i] = detail::ResourceBudget::try_reserve( + tree.account, categories[i], TestPreparedFlushTree::limits()[static_cast<size_t>(categories[i])]); + ASSERT_TRUE(blockers[i]); + } + const auto catalog = tree.sources.catalog(); + const auto view = tree.publisher.capture(); + std::shared_ptr<FlushOperation> operation; + FlushReservePlan plan; + const auto prepared = planned ? tree.controller->prepare_or_plan(tree.preparation, &operation, &plan) + : tree.controller->prepare(tree.preparation, &operation); + ASSERT_TRUE(prepared.ok()) << prepared.to_string(); + ASSERT_TRUE(operation); + EXPECT_FALSE(plan.view); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {1, 1, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "new")).ok()); + writer.release(); + tree.frontier.store(1, std::memory_order_release); + auto completed = stdexec::sync_wait(operation->report_sender()); + if (!completed) { + FAIL() << "covered operation did not complete"; + } + const auto &report = std::get<0>(*completed); + EXPECT_TRUE(report.handoff_status.ok()); + EXPECT_TRUE(report.maintenance_status.ok()); + EXPECT_EQ(report.floor, 0U); + EXPECT_EQ(report.frontier, 0U); + EXPECT_EQ(report.published_coverage, 0U); + EXPECT_FALSE(report.handoff_advanced); + EXPECT_EQ(tree.sources.catalog(), catalog); + EXPECT_FALSE(catalog->active->closed()); + EXPECT_EQ(tree.publisher.capture(), view); + EXPECT_EQ(tree.store->reads.load(), 0U); + blockers = {}; + std::shared_ptr<FlushOperation> following; + const auto next = tree.controller->prepare(tree.preparation, &following); + ASSERT_TRUE(next.ok()) << next.to_string(); + ASSERT_NE(following, operation); + ASSERT_TRUE(stdexec::sync_wait(following->sender())); + tree.runtime->drain(); + EXPECT_TRUE(following->result().handoff_status.ok()) << following->result().handoff_status.to_string(); + EXPECT_EQ(following->result().frontier, 1U); + EXPECT_EQ(tree.publisher.capture()->coverage, 1U); + const auto value = tree.publisher.capture()->mapping.get(0)->leaf->lookup("a"); + ASSERT_EQ(value.state, LogicalLookupState::kValue); + EXPECT_EQ(value.cell.value.to_string(), "new"); + } +} + +TEST(FlushReservation, NoProgressReleasesEveryPartitionWithoutRotatingSources) +{ + TestPreparedFlushTree tree; + tree.preparation.cache = std::make_shared<SharedPageCache>(PageCacheLimits{.max_pages = 1, .max_waiters = 1}); + const auto catalog = tree.sources.catalog(); + const auto view = tree.publisher.capture(); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &operation).ok()); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); + ASSERT_TRUE(stdexec::sync_wait(operation->sender())); + tree.runtime->drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()); + EXPECT_EQ(operation->result().published_coverage, 0U); + EXPECT_EQ(tree.sources.catalog(), catalog); + EXPECT_EQ(tree.publisher.capture(), view); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); + EXPECT_EQ(tree.budget.snapshot().reserved, tree.lifecycle_reservations()); +} + +TEST(FlushReservation, CancelledClosedBatchRetriesWithOriginalCreditsWhileHostIsFull) +{ + TestPreparedFlushTree tree; + tree.preparation.cache = std::make_shared<SharedPageCache>(PageCacheLimits{.max_pages = 1, .max_waiters = 1}); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {1, 1, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + tree.frontier.store(1); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &operation).ok()); + auto blockers = tree.exhaust_unused(); + std::promise<void> terminal; + auto done = terminal.get_future(); + auto connected = stdexec::connect(operation->sender(), Receiver{&terminal}); + stdexec::start(connected); + tree.runtime->drain(); + ASSERT_TRUE(writer.table()->closed()); + operation->request_stop(); + writer.release(); + done.get(); + tree.runtime->drain(); + EXPECT_EQ(operation->result().phase, FlushPhase::kRepairRequired); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 1U); + const auto identity = operation->result().batch_id; + const auto before = tree.budget.snapshot(); + const auto owner = tree.publisher.capture()->flush_admission_owner; + ASSERT_NE(owner, 0U); + std::shared_ptr<FlushOperation> retry; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &retry).ok()); + EXPECT_EQ(retry, operation); + EXPECT_EQ(tree.budget.snapshot().reserved, before.reserved); + EXPECT_EQ(tree.publisher.capture()->flush_admission_owner, owner); + tree.frontier.store(10); + ASSERT_TRUE(stdexec::sync_wait(retry->sender())); + tree.runtime->drain(); + EXPECT_TRUE(retry->result().handoff_status.ok()) << retry->result().handoff_status.to_string(); + EXPECT_TRUE(retry->result().maintenance_status.ok()) << retry->result().maintenance_status.to_string(); + EXPECT_EQ(retry->result().batch_id, identity); + EXPECT_EQ(retry->result().frontier, 1U); + EXPECT_EQ(retry->result().published_coverage, 1U); + EXPECT_EQ(tree.publisher.capture()->flush_admission_owner, 0U); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); +} + +TEST(FlushReservation, CacheWaiterPressureRollsBackBeforeSourceClosure) +{ + TestPreparedFlushTree tree; + tree.frontier.store(1); + tree.preparation.cache = std::make_shared<SharedPageCache>(PageCacheLimits{.max_pages = 1, .max_waiters = 1}); + auto upstream = std::make_shared<test_read::TestLoader>(); + std::shared_ptr<PageLoader> occupied; + ASSERT_TRUE(tree.preparation.cache->bind_reserved(upstream, {}, &occupied).ok()); + const auto catalog = tree.sources.catalog(); + const auto view = tree.publisher.capture(); + const auto before = tree.budget.snapshot(); + std::shared_ptr<FlushOperation> operation; + EXPECT_EQ(tree.controller->prepare(tree.preparation, &operation).code(), Code::kUnavailable); + EXPECT_FALSE(operation); + EXPECT_FALSE(catalog->active->closed()); + EXPECT_EQ(tree.sources.catalog(), catalog); + EXPECT_EQ(tree.publisher.capture(), view); + EXPECT_EQ(tree.budget.snapshot().reserved, before.reserved); + EXPECT_EQ(tree.budget.snapshot().allocated, before.allocated); + occupied.reset(); + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &operation).ok()); + ASSERT_TRUE(stdexec::sync_wait(operation->sender())); + tree.runtime->drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/flush_routing_test.cpp b/lib/crowdb-tree/tests/integration/flush_routing_test.cpp new file mode 100644 index 000000000..009f1bf18 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/flush_routing_test.cpp @@ -0,0 +1,190 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_batch.h" +#include "btree/page_load.h" + +#include <gtest/gtest.h> + +#include <future> + +namespace crowdb::tree +{ +namespace +{ +class RoutingLoader final : public PageLoader +{ + public: + std::promise<void> entered; + InnerComplete pending; + size_t calls = 0; + + void load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete) override + { + EXPECT_TRUE(backing->reloadable()); + ++calls; + pending = std::move(complete); + entered.set_value(); + } + + void load_leaf(BaseRef base, LeafComplete complete) override + { + (void)base; + ADD_FAILURE() << "handoff must not load cold leaves"; + complete(Status::internal_error(), {}); + } + + void finish(Status status = Status::Ok()) + { + auto inner = std::make_shared<InnerPageState>(); + inner->children = {0, 1}; + inner->separators = {"m"}; + auto callback = std::move(pending); + entered = std::promise<void>(); + callback(std::move(status), std::move(inner)); + } +}; + +void seed(PublicationCoordinator &publisher) +{ + auto left = std::make_shared<PageState>(); + left->range.upper = "m"; + auto right = std::make_shared<PageState>(); + right->range.lower = "m"; + auto backing = std::make_shared<BackingRef>(); + backing->length = 4096; + backing->owner = std::make_shared<int>(0); + right->bases.push_back({.range = right->range, .max_slot = 0, .backing = backing}); + auto first = std::make_shared<PublishedPage>(); + first->leaf = left; + auto second = std::make_shared<PublishedPage>(); + second->leaf = right; + auto root = std::make_shared<PublishedPage>(); + root->level = 1; + root->metrics.leaf = false; + root->inner_backing = backing; + auto table = std::make_shared<MemTable>(); + ASSERT_TRUE(table->upsert("z", 1, encode_cell_buf(1, OpKind::kPut, "payload"))); + table->close(); + auto sources = std::make_shared<PublishedSources>(); + sources->tables.push_back(std::move(table)); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = { + .record = std::make_shared<InstallRecord>(InstallToken{} + ), + .pages = {{.page_id = 0, .expected = {}, .replacement = first}, + {.page_id = 1, .expected = {}, .replacement = second}, + {.page_id = 2, .expected = {}, .replacement = root}}, + .root_page_id = 2, + .expected_sources = publisher.capture()->sources, + .sources = sources + }; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(request)); +} + +struct Receiver +{ + using receiver_concept = stdexec::receiver_tag; + std::promise<void> *done; + + void set_value() const noexcept + { + done->set_value(); + } + + void set_error(Status status) const noexcept + { + ADD_FAILURE() << status.to_string(); + done->set_value(); + } + + void set_stopped() const noexcept + { + ADD_FAILURE() << "flush must return its progress result"; + done->set_value(); + } +}; + +TEST(FlushRouting, ColdIndexWaitReleasesSingleWorkerAndNeverLoadsLeaf) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + seed(publisher); + auto loader = std::make_shared<RoutingLoader>(); + FlushBatchOptions options; + options.loader = loader; + options.routing_leaves_per_step = 1; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + std::promise<void> done; + auto terminal = done.get_future(); + auto entered = loader->entered.get_future(); + auto operation = stdexec::connect(batch->sender(), Receiver{&done}); + stdexec::start(operation); + entered.get(); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_TRUE(publisher.capture()->pending_handoff); + std::promise<void> spare; + auto spare_task = spare.get_future(); + ASSERT_TRUE( + runtime + .try_submit( + lane, [](void *context) noexcept { static_cast<std::promise<void> *>(context)->set_value(); }, &spare) + .ok()); + spare_task.get(); + loader->finish(); + terminal.get(); + runtime.drain(); + EXPECT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + EXPECT_EQ(batch->result().published_coverage, 1U); + EXPECT_EQ(loader->calls, 1U); + EXPECT_TRUE(publisher.capture()->mapping.get(0)->leaf->overlays.empty()); + EXPECT_EQ(publisher.capture()->mapping.get(1)->leaf->lookup("z").cell.value.to_string(), "payload"); + EXPECT_FALSE(publisher.capture()->mapping.get(2)->inner); + EXPECT_FALSE(publisher.capture()->pending_handoff); +} + +TEST(FlushRouting, LoadFailureAndCancellationRetainRepairIdentity) +{ + for (bool cancel : {false, true}) { + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + seed(publisher); + auto loader = std::make_shared<RoutingLoader>(); + FlushBatchOptions options; + options.loader = loader; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 7, 1, options, {}, &batch).ok()); + for (size_t attempt = 0; attempt < 2; ++attempt) { + auto entered = loader->entered.get_future(); + std::promise<void> done; + auto terminal = done.get_future(); + auto operation = stdexec::connect(batch->sender(), Receiver{&done}); + stdexec::start(operation); + entered.get(); + if (attempt == 0 && cancel) { + batch->request_stop(); + } + loader->finish(attempt == 0 && !cancel ? Status::io_error("controlled cold index failure") : Status::Ok()); + terminal.get(); + runtime.drain(); + if (attempt == 0) { + EXPECT_FALSE(batch->result().handoff_status.ok()); + EXPECT_EQ(batch->result().published_coverage, 0U); + const auto pending = publisher.capture()->pending_handoff; + ASSERT_TRUE(pending.has_value()); + if (pending) { + EXPECT_EQ(pending->batch, 7U); + } + EXPECT_EQ(publisher.capture()->sources->tables.size(), 1U); + } + } + EXPECT_TRUE(batch->result().handoff_status.ok()); + EXPECT_EQ(batch->result().published_coverage, 1U); + EXPECT_FALSE(publisher.capture()->pending_handoff); + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/flush_window_test.cpp b/lib/crowdb-tree/tests/integration/flush_window_test.cpp new file mode 100644 index 000000000..08566c7b6 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/flush_window_test.cpp @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_batch.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +void initialize_windows(PublicationCoordinator &publisher, size_t entries) +{ + auto table = std::make_shared<MemTable>(); + for (size_t i = 0; i < entries; ++i) { + const auto key = std::to_string(1000 + i); + ASSERT_TRUE(table->upsert(key, 1, encode_cell_buf(1, OpKind::kPut, "old"))); + ASSERT_TRUE(table->upsert(key, 2, encode_cell_buf(2, OpKind::kPut, "latest"))); + } + table->close(); + auto sources = std::make_shared<PublishedSources>(); + sources->tables.push_back(std::move(table)); + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{}), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0, + .expected_sources = publisher.capture()->sources, + .sources = sources}; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(publisher.try_submit(request)); +} + +void verify_window_values(const std::shared_ptr<const PublishedTreeState> &view, size_t count) +{ + EXPECT_EQ(view->stats.overlay_entries, count); + const auto page = view->mapping.get(0); + for (size_t i = 0; i < count; ++i) { + const auto value = page->leaf->lookup(std::to_string(1000 + i)); + ASSERT_EQ(value.state, LogicalLookupState::kValue); + EXPECT_EQ(value.cell.value.to_string(), "latest"); + } + ASSERT_EQ(page->leaf->overlays.size(), count); + for (size_t i = 0; i < count; ++i) { + EXPECT_EQ(page->leaf->overlays[i]->entries().front().key().to_string(), std::to_string(1000 + i)); + } +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(FlushWindow, GeometricInstallationsPublishPrefixesBeforeGlobalCoverage) +{ + for (size_t workers : {1U, 4U}) { + detail::FlushRuntime runtime(workers, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize_windows(publisher, 6); + const auto old = publisher.capture(); + FlushBatchOptions options; + options.install_run_window = 1; + options.run_limits.max_entries = 1; + std::vector<size_t> prefixes; + options.after_run_for_tests = [&](size_t, size_t) { + const auto view = publisher.capture(); + prefixes.push_back(view->stats.overlay_entries); + EXPECT_EQ(view->coverage, 0U); + ASSERT_TRUE(view->pending_handoff); + EXPECT_FALSE(view->sources->tables.empty()); + }; + bool maintained = false; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare( + publisher, runtime, lane, 1, 2, options, + [&](auto view, auto done) { + maintained = true; + EXPECT_EQ(view->coverage, 2U); + verify_window_values(view, 6); + done(Status::Ok(), 0); + }, + &batch) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + EXPECT_EQ(prefixes, (std::vector<size_t>{0, 1, 2, 2, 4, 4})); + EXPECT_TRUE(maintained); + EXPECT_EQ(publisher.capture()->coverage, 2U); + EXPECT_TRUE(publisher.capture()->sources->tables.empty()); + EXPECT_EQ(old->stats.overlay_entries, 0U); + verify_window_values(publisher.capture(), 6); + } +} + +TEST(FlushWindow, CancelAfterCommittedPrefixResumesPreparedSuffixExactlyOnce) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize_windows(publisher, 6); + FlushBatchOptions options; + options.install_run_window = 1; + options.run_limits.max_entries = 1; + std::shared_ptr<FlushBatch> batch; + std::vector<size_t> prepared; + options.after_run_for_tests = [&](size_t, size_t count) { + prepared.push_back(count); + if (count == 3) { + batch->request_stop(); + } + }; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 2, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + EXPECT_EQ(batch->result().phase, FlushPhase::kRepairRequired); + EXPECT_EQ(publisher.capture()->coverage, 0U); + verify_window_values(publisher.capture(), 2); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()); + EXPECT_EQ(prepared, (std::vector<size_t>{1, 2, 3, 4, 5, 6})); + verify_window_values(publisher.capture(), 6); + EXPECT_EQ(publisher.capture()->coverage, 2U); +} + +TEST(FlushWindow, RejectedContinuationKeepsCommittedWindowAndStableNextRecord) +{ + detail::FlushRuntime runtime(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + PublicationCoordinator publisher; + initialize_windows(publisher, 4); + FlushBatchOptions options; + options.install_run_window = 1; + options.run_limits.max_entries = 1; + std::vector<size_t> prepared; + bool other_ran = false; + options.after_run_for_tests = [&](size_t, size_t count) { + prepared.push_back(count); + if (count == 1) { + EXPECT_TRUE( + runtime.try_submit( + lane, [](void *raw) noexcept { *static_cast<bool *>(raw) = true; }, &other_ran) + .ok()); + } + }; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 2, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + EXPECT_TRUE(other_ran); + EXPECT_EQ(batch->result().phase, FlushPhase::kRepairRequired); + verify_window_values(publisher.capture(), 1); + EXPECT_EQ(publisher.capture()->coverage, 0U); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()); + EXPECT_EQ(prepared, (std::vector<size_t>{1, 2, 3, 4})); + verify_window_values(publisher.capture(), 4); +} +#endif + +TEST(FlushWindow, AllInstallRootsAndArraysFitThePrepaidBoundAtTheHardLimit) +{ + detail::ResourceBudget::Limits limits; + limits.fill(64 * 1024 * 1024); + detail::ResourceBudget budget(limits, 4); + auto account = budget.make_account(limits); + constexpr auto category = detail::BudgetClass::kHandoff; + { + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize_windows(publisher, 64); + FlushBatchOptions options; + options.install_run_window = 2; + options.run_limits.max_entries = 1; + const auto bound = FlushBatch::metadata_reserve_bound(publisher.capture()->mapping, 1, 64, 256, 1, options); + if (!bound) { + FAIL() << "window footprint exceeds representable allocation bound"; + } + auto credit = detail::ResourceBudget::try_reserve(account, category, *bound); + if (!credit) { + FAIL() << "window footprint exceeds configured handoff quota"; + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + auto remaining = + detail::ResourceBudget::try_reserve(account, category, limits[static_cast<size_t>(category)] - *bound); + ASSERT_TRUE(remaining); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 2, options, {}, &batch, {}, metadata).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + verify_window_values(publisher.capture(), 64); + metadata->seal(); + } + EXPECT_EQ(budget.snapshot().allocated[static_cast<size_t>(category)], 0U); + EXPECT_EQ(budget.snapshot().reserved[static_cast<size_t>(category)], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/host_writeback_window_test.cpp b/lib/crowdb-tree/tests/integration/host_writeback_window_test.cpp new file mode 100644 index 000000000..71a03569f --- /dev/null +++ b/lib/crowdb-tree/tests/integration/host_writeback_window_test.cpp @@ -0,0 +1,209 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "local_checkpoint_test_support.h" +#include "native_import_test_support.h" +#include "snapshot/checkpoint_operation.h" +#include "snapshot/operation_wait.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestHostWriteback +{ + Config config; + std::shared_ptr<MemPageStore> store = std::make_shared<MemPageStore>(4096); + std::shared_ptr<TreeCore> core; + + Status open() + { + if (!config.runtime) { + auto status = TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime); + if (!status.ok()) { + return status; + } + } + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.page_store = store.get(); + config.backend_owner = store; + config.runtime_binding.limits.io_operations = 1; + return TreeCore::open(config, &core); + } + + void populate() + { + const std::string value(512, 'v'); + for (uint64_t slot = 1; slot <= 256; ++slot) { + const auto status = core->apply( + slot, + Batch{.ops = {{.key = "key" + std::to_string(1000 + slot), .kind = OpKind::kPut, .value = value}}}); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + const auto status = core->flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + } +}; + +TEST(HostWritebackWindow, OnePrimaryPerPassConvergesAfterTheLastLeafAndRemainsCheckpointable) +{ + TestHostWriteback test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.populate(); + auto initial = test.core->capture(); + ASSERT_GT(initial->stats.writeback_candidate_pages, 1U); + ASSERT_GT(initial->stats.inner_pages, 0U); + const auto pending = initial->stats.checkpoint_pending_pages; + size_t passes = 0; + bool saw_inner_only = false; + while (test.core->capture()->stats.writeback_candidate_pages != 0 && passes < 64) { + const auto before = test.core->capture(); + if (!before->mapping.oldest_writeback_leaf()) { + saw_inner_only = true; + } + std::shared_ptr<CheckpointOperation> operation; + status = test.core->prepare_writeback(&operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = detail::wait_snapshot_operation(operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(operation->result().writeback_completed); + EXPECT_FALSE(operation->result().durable_committed); + EXPECT_EQ(operation->result().primary_frames, 1U); + EXPECT_EQ(operation->result().rebased_pages, 1U); + EXPECT_EQ(operation->result().durable_coverage, 0U); + EXPECT_EQ(operation->result().captured_coverage, 256U); + const auto after = test.core->capture(); + EXPECT_LT(after->stats.writeback_candidate_pages, before->stats.writeback_candidate_pages); + EXPECT_EQ(after->stats.checkpoint_pending_pages, pending); + ++passes; + } + EXPECT_GT(passes, 1U); + EXPECT_TRUE(saw_inner_only); + EXPECT_EQ(test.core->capture()->stats.writeback_candidate_pages, 0U); + EXPECT_EQ(test.core->durable_coverage(), 0U); + ASSERT_TRUE(test.core->checkpoint().ok()); + initial.reset(); + test.core.reset(); + status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + GetView found; + ASSERT_TRUE(test.core->get("key1256", &found).ok()); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), std::string(512, 'v')); + EXPECT_EQ(test.core->durable_coverage(), 256U); +} + +TEST(HostWritebackWindow, NoCandidateCompletesWithoutCapturingOrAdvancingDurableState) +{ + TestHostWriteback test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto before = test.core->capture(); + for (size_t pass = 0; pass < 3; ++pass) { + std::shared_ptr<CheckpointOperation> operation; + ASSERT_TRUE(test.core->prepare_writeback(&operation).ok()); + status = detail::wait_snapshot_operation(operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto &result = operation->result(); + EXPECT_TRUE(result.writeback_completed); + EXPECT_FALSE(result.durable_committed); + EXPECT_EQ(result.primary_frames, 0U); + EXPECT_EQ(result.written_frames, 0U); + EXPECT_EQ(result.durable_coverage, 0U); + EXPECT_EQ(result.snapshot_sequence, 0U); + EXPECT_EQ(test.core->capture()->checkpoint_token, before->checkpoint_token); + EXPECT_FALSE(test.core->capture()->checkpoint_inflight); + } +} + +TEST(HostWritebackWindow, ImportedRawChildrenKeepTheirCapturedFenceProofWithoutBackendReads) +{ + TestHostWriteback test; + ASSERT_TRUE(test.open().ok()); + std::vector<NativeFrame> frames; + frames.push_back({.page_id = 0, .frame = std::vector<uint8_t>(4096)}); + ASSERT_TRUE(inner_frame_build(frames[0].frame.data(), 4096, 0, {1, 2}, {"m"})); + frame_set_inner_fence_pages(frames[0].frame.data(), 4096, 1, 2); + const auto left = encode_cell_buf(7, OpKind::kPut, "old-left"); + frames.push_back(test_import::leaf(1, "a", Slice(left), 2)); + frames.push_back(test_import::inline_leaf(2, "z", "old-right")); + auto info = test_import::info(frames); + auto status = test.core->install_native(std::move(frames), info); + ASSERT_TRUE(status.ok()) << status.to_string(); + for (uint64_t slot = 8; slot <= 263; ++slot) { + status = test.core->apply(slot, Batch{.ops = {{.key = "key" + std::to_string(1000 + slot), + .kind = OpKind::kPut, + .value = std::string(512, 'v')}}}); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + status = test.core->flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + for (size_t pass = 0; test.core->capture()->stats.writeback_candidate_pages != 0 && pass < 64; ++pass) { + std::shared_ptr<CheckpointOperation> operation; + ASSERT_TRUE(test.core->prepare_writeback(&operation).ok()); + status = detail::wait_snapshot_operation(operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(operation->result().primary_frames, 1U); + EXPECT_EQ(operation->result().durable_coverage, 0U); + } + EXPECT_EQ(test.core->capture()->stats.writeback_candidate_pages, 0U); + status = test.core->checkpoint(); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.core.reset(); + ASSERT_TRUE(test.open().ok()); + GetView found; + ASSERT_TRUE(test.core->get("z", &found).ok()); + EXPECT_EQ(found.value().to_string(), "old-right"); + EXPECT_EQ(test.core->durable_coverage(), 263U); +} + +TEST(HostWritebackWindow, FailedAndCancelledWindowsLeaveTheCandidateForALaterPass) +{ + test_write::TestCheckpointStore store; + TestHostWriteback test; + store.device = test.store; + test.config.async_page_store = &store; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "value"}}}).ok()); + ASSERT_TRUE(test.core->flush().ok()); + const auto before = test.core->capture(); + ASSERT_EQ(before->stats.writeback_candidate_pages, 1U); + store.fail_at = 1; + std::shared_ptr<CheckpointOperation> operation; + ASSERT_TRUE(test.core->prepare_writeback(&operation).ok()); + EXPECT_EQ(detail::wait_snapshot_operation(operation).code(), Code::kIoError); + EXPECT_FALSE(operation->result().writeback_completed); + EXPECT_EQ(test.core->capture()->mapping.get(before->root_page_id), before->mapping.get(before->root_page_id)); + operation.reset(); + store.fail_at = 0; + store.defer_at = store.events.size() + 1; + ASSERT_TRUE(test.core->prepare_writeback(&operation).ok()); + test_write::Completion done; + auto connected = stdexec::connect(operation->sender(), test_write::Receiver{.done = &done}); + stdexec::start(connected); + test.core->binding()->cpu()->drain(); + // Backend submission runs on a separate worker from the CPU continuation. + store.wait_deferred(); + ASSERT_TRUE(store.pending); + operation->request_stop(); + test.core->binding()->cpu()->drain(); + EXPECT_EQ(done.stopped.load(), 0U); + store.finish(); + test.core->binding()->cpu()->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_EQ(test.core->capture()->stats.writeback_candidate_pages, 1U); + EXPECT_EQ(test.core->durable_coverage(), 0U); + operation.reset(); + store.defer_at = 0; + ASSERT_TRUE(test.core->prepare_writeback(&operation).ok()); + ASSERT_TRUE(detail::wait_snapshot_operation(operation).ok()); + EXPECT_EQ(operation->result().primary_frames, 1U); + EXPECT_EQ(test.core->capture()->stats.writeback_candidate_pages, 0U); + EXPECT_EQ(test.core->durable_coverage(), 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/immutable_chunk_snapshot_test.cpp b/lib/crowdb-tree/tests/integration/immutable_chunk_snapshot_test.cpp new file mode 100644 index 000000000..eecca6cdc --- /dev/null +++ b/lib/crowdb-tree/tests/integration/immutable_chunk_snapshot_test.cpp @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/runtime_range_rebuild.h" +#include "common/test_tree_core_chunk.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +TEST(ImmutableChunkSnapshot, ColdPackCanRebuildFixedRangeAfterSourceWriterCloses) +{ + test_chunk::TestCoreChunk source; + ASSERT_TRUE(source.open().ok()); + Batch batch{ + .ops = {{.key = "a", .kind = OpKind::kPut, .value = "left"}, + {.key = "z", .kind = OpKind::kPut, .value = std::string(9000, 'v')}} + }; + ASSERT_TRUE(source.tree->apply(1, batch).ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + source.tree.reset(); + ASSERT_TRUE(source.open().ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(TreeCoreAccess::get(*source.tree)->capture_snapshot(&snapshot).ok()); + ASSERT_TRUE(source.tree->apply(2, Batch{.ops = {{.key = "z", .kind = OpKind::kPut, .value = "new"}}}).ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + source.tree.reset(); + + GetView value; + ASSERT_TRUE(snapshot->get("z", &value).ok()); + EXPECT_EQ(value.value().to_string(), std::string(9000, 'v')); + test_chunk::TestCoreChunk destination; + destination.config.key_range = KeyRange::bounded("m", {}); + ASSERT_TRUE(destination.open().ok()); + auto *core = TreeCoreAccess::get(*destination.tree); + RuntimeRangeImport staging; + // The fixed-range planner reserves its observed import shape. + auto status = copy_snapshot_range(snapshot, *core, destination.config.key_range, staging); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(core->install_readable(std::move(staging.catalog), staging.credit).ok()); + ASSERT_TRUE(destination.tree->snapshot().ok()); + EXPECT_EQ(staging.stats.entries_emitted, 1U); + EXPECT_EQ(staging.stats.pages_reused, 0U); + EXPECT_GT(staging.stats.pages_rebuilt, 0U); + snapshot.reset(); + value = {}; + destination.tree.reset(); + ASSERT_TRUE(destination.open().ok()); + EXPECT_EQ(destination.tree->get_view("z").value().to_string(), std::string(9000, 'v')); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/incremental_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/incremental_checkpoint_test.cpp index 5b2cbd26e..48b710c5c 100644 --- a/lib/crowdb-tree/tests/integration/incremental_checkpoint_test.cpp +++ b/lib/crowdb-tree/tests/integration/incremental_checkpoint_test.cpp @@ -5,10 +5,13 @@ // durable addrs, and reopens to identical state. #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" +#include "runtime_tree_access.h" +#include "snapshot/primary_backing.h" #include <gtest/gtest.h> #include <array> +#include <chrono> #include <map> #include <memory> #include <string> @@ -30,16 +33,56 @@ std::string key(int i) return buf.data(); } -// build a multi-level tree by inserting K keys, flushing incrementally so leaves -// stay small (one bulk flush would make a single oversized leaf). -void fill(Crowdbtree *t, int K, std::map<std::string, std::string> *oracle) +// Bounded batches exercise real byte-aware splitting without relying on one +// oversized leaf or a pointer-delta chain. +void fill(Crowdbtree *tree, int count, std::map<std::string, std::string> *oracle) { - for (int i = 0; i < K; ++i) { - std::string v = "val" + std::to_string(i); - ASSERT_TRUE(t->apply(i + 1, put_one(key(i), v)).ok()); - ASSERT_TRUE(t->flush().ok()); - (*oracle)[key(i)] = v; + for (int first = 0; first < count; first += 100) { + Batch batch; + const int last = std::min(first + 100, count); + for (int index = first; index < last; ++index) { + auto value = "val" + std::to_string(index); + batch.ops.push_back({.key = key(index), .kind = OpKind::kPut, .value = value}); + (*oracle)[key(index)] = value; + } + ASSERT_TRUE(tree->apply(static_cast<uint64_t>(last), batch).ok()); + tree->force_advance_slot(static_cast<uint64_t>(last)); + ASSERT_TRUE(tree->flush().ok()); + } +} + +using DurableLocations = std::map<uint64_t, std::pair<uint64_t, uint64_t>>; + +DurableLocations durable_locations(Crowdbtree &tree) +{ + DurableLocations result; + RuntimeTreeAccess_for_tests::capture(tree)->mapping.visit( + &result, [](void *raw, uint64_t pid, const std::shared_ptr<const PublishedPage> &page) { + if (page->primary && page->primary->frame.backing) { + auto &out = *static_cast<DurableLocations *>(raw); + const auto &backing = *page->primary->frame.backing; + out.emplace(pid, std::pair{backing.address, backing.length}); + } + }); + return result; +} + +void expect_untouched_locations(const DurableLocations &before, const DurableLocations &after) +{ + size_t reused = 0; + size_t changed = 0; + for (const auto &[pid, location] : before) { + auto current = after.find(pid); + if (current != after.end() && current->second == location) { + ++reused; + } + else { + ++changed; + } } + EXPECT_GT(reused, 0U); + EXPECT_GT(changed, 0U); + EXPECT_LT(changed, before.size()); } } // namespace @@ -83,7 +126,7 @@ TEST(IncrementalCheckpoint, SingleKeyEditRewritesOnlyItsPath) ASSERT_GT(total, 4U); // several leaves + inner level(s) // Touch exactly one key, flush, snapshot again. - uint64_t slot = 100000; + uint64_t slot = t.contiguous_slot() + 1; ASSERT_TRUE(t.apply(slot, put_one(key(7), "updated")).ok()); t.force_advance_slot(slot); ASSERT_TRUE(t.flush().ok()); @@ -95,71 +138,58 @@ TEST(IncrementalCheckpoint, SingleKeyEditRewritesOnlyItsPath) EXPECT_LT(rewritten, total); // unchanged leaves/inners were retained } -// Regression (plan-tree #14c/#14d/#18 D4): snapshot's dirty-tracking is -// segment-level, not page-level -- confirm only the *dirty* segment(s) get a -// fresh image written, not every present segment. With kSegmentSize == 1024 -// PIDs, a 200-key tree comfortably fits in one segment, so this also -// exercises the single-segment steady-state case explicitly. -TEST(IncrementalCheckpoint, OnlyDirtySegmentsAreRewritten) +TEST(IncrementalCheckpoint, UnchangedCheckpointPreservesExactPageLocations) { MemPageStore store(1); - Config opt; - opt.page_store = &store; - opt.max_delta_len = 1; - opt.leaf_split_bytes = 160; - opt.frame_bytes = 4096; - Crowdbtree t(opt); - + Config options; + options.page_store = &store; + options.leaf_split_bytes = 160; + options.frame_bytes = 4096; + Crowdbtree tree(options); std::map<std::string, std::string> oracle; - fill(&t, 200, &oracle); - - ASSERT_TRUE(t.snapshot(nullptr).ok()); - uint64_t first_segments = t.last_snapshot_segments_written(); - EXPECT_GE(first_segments, 1U); // first snapshot: at least the one segment in use - - // No mutations -> no segment should need re-imaging. - ASSERT_TRUE(t.snapshot(nullptr).ok()); - EXPECT_EQ(t.last_snapshot_segments_written(), 0U); - - // A single-key edit still only touches PIDs within one segment (this - // tree never grows past kSegmentSize PIDs) -- expect exactly that one - // segment re-imaged, not zero and not "every segment". - uint64_t slot = 100000; - ASSERT_TRUE(t.apply(slot, put_one(key(7), "updated")).ok()); - t.force_advance_slot(slot); - ASSERT_TRUE(t.flush().ok()); - ASSERT_TRUE(t.snapshot(nullptr).ok()); - EXPECT_EQ(t.last_snapshot_segments_written(), 1U); + fill(&tree, 200, &oracle); + ASSERT_TRUE(tree.snapshot().ok()); + const auto before = durable_locations(tree); + ASSERT_GT(before.size(), 4U); + ASSERT_TRUE(tree.snapshot().ok()); + EXPECT_EQ(durable_locations(tree), before); + EXPECT_EQ(tree.last_snapshot_pages_written(), 0U); + ASSERT_TRUE(tree.apply(tree.contiguous_slot() + 1, put_one(key(7), "updated")).ok()); + ASSERT_TRUE(tree.snapshot().ok()); + expect_untouched_locations(before, durable_locations(tree)); } -// Same regression, but with a tree big enough (kSegmentSize == 1024 PIDs per -// segment) to actually span multiple mapping-table segments: a single-key -// edit must re-image only the one segment holding the touched leaf's PID, -// leaving every other segment's image/generation untouched. -TEST(IncrementalCheckpoint, MultiSegmentTreeOnlyRewritesTouchedSegment) +TEST(IncrementalCheckpoint, LargeTreeEditPreservesUntouchedPhysicalPages) { MemPageStore store(1); - Config opt; - opt.page_store = &store; - opt.max_delta_len = 1; - opt.leaf_split_bytes = 160; - opt.frame_bytes = 4096; - Crowdbtree t(opt); - + Config options; + options.page_store = &store; + options.leaf_split_bytes = 160; + options.frame_bytes = 4096; + Crowdbtree tree(options); std::map<std::string, std::string> oracle; - fill(&t, 3000, &oracle); // enough splits to allocate well past 1024 PIDs - - ASSERT_TRUE(t.snapshot(nullptr).ok()); - size_t total_segments = t.mapping().segments_allocated(); - ASSERT_GT(total_segments, 1U) << "test needs a multi-segment tree to be meaningful"; - - uint64_t slot = 1000000; - ASSERT_TRUE(t.apply(slot, put_one(key(7), "updated")).ok()); - t.force_advance_slot(slot); - ASSERT_TRUE(t.flush().ok()); - ASSERT_TRUE(t.snapshot(nullptr).ok()); - EXPECT_EQ(t.last_snapshot_segments_written(), 1U); - EXPECT_LT(t.last_snapshot_segments_written(), total_segments); + fill(&tree, 4096, &oracle); + const auto work = RuntimeTreeAccess_for_tests::core(tree).publisher().work_metrics(); + const auto before_splits = work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Checkpoint)] + [static_cast<size_t>(TreeWorkCounter::SplitOperations)]; + const auto started = std::chrono::steady_clock::now(); + const auto initial = tree.snapshot(); + const auto elapsed = + std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - started); + ASSERT_TRUE(initial.ok()) << initial.to_string(); + const auto windows = work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Checkpoint)] + [static_cast<size_t>(TreeWorkCounter::SplitOperations)] - + before_splits; + EXPECT_GT(windows, 0U); + RecordProperty("checkpoint_split_transactions", std::to_string(windows)); + RecordProperty("initial_checkpoint_ms", std::to_string(elapsed.count())); + const auto before = durable_locations(tree); + ASSERT_GT(before.size(), 1024U); + ASSERT_TRUE(tree.apply(tree.contiguous_slot() + 1, put_one(key(7), "updated")).ok()); + const auto updated = tree.snapshot(); + ASSERT_TRUE(updated.ok()) << updated.to_string(); + expect_untouched_locations(before, durable_locations(tree)); + EXPECT_LT(tree.last_snapshot_pages_written(), before.size()); } TEST(IncrementalCheckpoint, SpaceIsReusedAcrossManyCheckpoints) @@ -281,8 +311,7 @@ TEST(IncrementalCheckpoint, SnapshotPagesTotalAccumulates) EXPECT_EQ(t.stats().snapshot_pages_total, first_total); // Touch one key, flush, snapshot again: cumulative increases. - ASSERT_TRUE(t.apply(100000, put_one(key(7), "updated")).ok()); - t.force_advance_slot(100000); + ASSERT_TRUE(t.apply(t.contiguous_slot() + 1, put_one(key(7), "updated")).ok()); ASSERT_TRUE(t.flush().ok()); ASSERT_TRUE(t.snapshot(nullptr).ok()); uint64_t third_written = t.last_snapshot_pages_written(); diff --git a/lib/crowdb-tree/tests/integration/large_value_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/large_value_checkpoint_test.cpp new file mode 100644 index 000000000..053c89273 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/large_value_checkpoint_test.cpp @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +void checkpoint_large_values(size_t source_entries) +{ + MemPageStore store(1); + Config config; + config.page_store = &store; + config.frame_bytes = 65536; + config.execution.source_max_entries = source_entries; + // Retained cold-page descriptors coexist with the next complete maintenance window. + config.runtime_binding.limits.reclaim_bytes = 128 * 1024 * 1024; + config.runtime_binding.limits.handoff_bytes = 128 * 1024 * 1024; + config.runtime_binding.limits.writer_bytes = 256 * 1024 * 1024; + const std::string value(16384, 'v'); + { + Crowdbtree tree(config); + for (uint64_t pass = 0; pass < 2; ++pass) { + for (uint64_t index = 1; index <= 4096; ++index) { + const uint64_t slot = (pass * 4096) + index; + auto status = tree.apply(slot, Batch{.ops = {{.key = "key" + std::to_string(10000 + index), + .kind = OpKind::kPut, + .value = value}}}); + ASSERT_TRUE(status.ok()) << slot << ": " << status.to_string(); + } + auto status = tree.flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + uint64_t durable = 0; + status = tree.snapshot(&durable); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(durable, (pass + 1) * 4096); + tree.set_gc_watermark(durable, durable); + } + } + Crowdbtree reopened(config); + for (uint64_t slot = 1; slot <= 4096; ++slot) { + auto result = reopened.get_view("key" + std::to_string(10000 + slot)); + ASSERT_TRUE(result.found()) << slot; + EXPECT_EQ(result.value().to_string(), value) << slot; + } +} + +} // namespace + +TEST(LargeValueCheckpoint, ConfiguredBudgetPersistsAndReloadsMultipleMappingSegments) +{ + checkpoint_large_values(4096); +} + +TEST(LargeValueCheckpoint, SmallSourceWindowPersistsOversizedLeafSources) +{ + checkpoint_large_values(64); +} + +TEST(SourceBatchCapacity, WholeBatchRotatesBeforePartiallyRemainingEntryOrKeyCapacity) +{ + for (bool key_limited : {false, true}) { + Config config; + config.execution.source_max_entries = key_limited ? 4096 : 5; + config.execution.source_max_key_bytes = key_limited ? 128 : 1024 * 1024; + const size_t batch_size = key_limited ? 2 : 3; + Crowdbtree tree(config); + for (uint64_t slot = 1; slot <= 8; ++slot) { + Batch batch; + for (size_t item = 0; item < batch_size; ++item) { + const auto suffix = std::to_string((slot * batch_size) + item); + batch.ops.push_back( + {.key = std::string(48 - suffix.size(), 'k') + suffix, .kind = OpKind::kPut, .value = "value"}); + } + const auto status = tree.apply(slot, batch); + ASSERT_TRUE(status.ok()) << "key_limited=" << key_limited << " slot=" << slot << ' ' << status.to_string(); + } + ASSERT_TRUE(tree.flush().ok()); + for (uint64_t slot = 1; slot <= 8; ++slot) { + for (size_t item = 0; item < batch_size; ++item) { + const auto suffix = std::to_string((slot * batch_size) + item); + const auto result = tree.get_view(std::string(48 - suffix.size(), 'k') + suffix); + ASSERT_TRUE(result.found()); + EXPECT_EQ(result.slot(), slot); + EXPECT_EQ(result.value().to_string(), "value"); + } + } + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/legacy_chunk_migration_test.cpp b/lib/crowdb-tree/tests/integration/legacy_chunk_migration_test.cpp new file mode 100644 index 000000000..0244e0cb5 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/legacy_chunk_migration_test.cpp @@ -0,0 +1,279 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_frame.h" +#include "backend/chunk/chunk_runtime_page_store.h" +#include "backend/chunk/legacy_chunk_recovery.h" +#include "c_api_internal.h" +#include "common/test_legacy_chunk.h" +#include "crowdb-tree/chunk_migration_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +Config configuration(test_chunk::TestLegacyChunk &fixture) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + EXPECT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + EXPECT_TRUE(detail::ChunkPinReleaseQueue::create( + fixture.credits.credits.bytes, 256, std::make_shared<int>(1), [](void *) noexcept {}, nullptr, + &fixture.catalog->queue) + .ok()); + auto store = std::make_shared<detail::ChunkRuntimePageStore>( + detail::ChunkRuntimeOptions{.tree = 42, .owner_epoch = 7, .iu_size = 1, .mirrors = 2}, fixture.catalog, + fixture.transport, fixture.credits.credits.bytes); + config.page_store = store.get(); + config.backend_owner = std::move(store); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + return config; +} + +void close_queue(test_chunk::TestLegacyChunk &fixture) +{ + fixture.catalog->queue->stop_admission(); + (void)fixture.catalog->queue->pump(256); + EXPECT_TRUE(fixture.catalog->queue->drained()); +} +} // namespace + +TEST(LegacyChunkMigration, ReadOnlyAdapterVerifiesActualLegacyWriterFramesAndPinsGeneration) +{ + test_chunk::TestLegacyChunk fixture; + auto reader = fixture.reader(); + const auto status = reader->open(1); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(fixture.catalog->generation_pins.size(), 1U); + EXPECT_EQ(reader->size(), 110979U); + for (size_t i = 0; i < fixture.layout.pack_count(); ++i) { + const auto pack = fixture.layout.pack(i); + std::vector<uint8_t> value(pack.ref.length); + const auto loaded = reader->read_at(pack.logical_offset, value.data(), value.size()); + ASSERT_TRUE(loaded.ok()) << loaded.to_string(); + const auto &physical = fixture.physical.at({pack.ref.chunk_id.high, pack.ref.chunk_id.low}); + EXPECT_TRUE( + std::equal(value.begin(), value.end(), physical.begin() + pack.ref.offset + protocol::kFrameV2HeaderBytes)); + } + const uint8_t byte = 0; + EXPECT_EQ(reader->write_at(0, &byte, 1).code(), Code::kNotSupported); + ASSERT_TRUE(reader->release_pin().ok()); + EXPECT_TRUE(fixture.catalog->generation_pins.empty()); +} + +TEST(LegacyChunkMigration, CorruptedReferenceSegmentAndWrongGenerationDoNotPermitReads) +{ + test_chunk::TestLegacyChunk fixture; + auto reader = fixture.reader(); + EXPECT_EQ(reader->open(2).code(), Code::kUnavailable); + EXPECT_TRUE(fixture.catalog->generation_pins.empty()); + fixture.catalog->segments.begin()->second.back() ^= 1; + EXPECT_EQ(reader->open(1).code(), Code::kCorruption); + EXPECT_EQ(fixture.catalog->generation_pins.size(), 1U); + uint8_t byte = 0; + EXPECT_EQ(reader->read_at(0, &byte, 1).code(), Code::kInvalidArgument); + ASSERT_TRUE(reader->release_pin().ok()); +} + +TEST(LegacyChunkMigration, ActualLegacyInnerAndOverflowRecoverIntoReadOnlyCapturedGraph) +{ + test_chunk::TestLegacyChunk fixture; + { + auto config = configuration(fixture); + std::shared_ptr<detail::LegacyChunkReadStore> reader; + std::shared_ptr<TreeCore> source; + const auto status = detail::recover_legacy_chunk_source(config, 1, &reader, &source); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(source->durable_coverage(), 20U); + GetView view; + ASSERT_TRUE(source->get("key20", &view).ok()); + EXPECT_EQ(view.slot(), 20U); + EXPECT_EQ(view.value().to_string(), std::string(8192, 'u')); + view = {}; + source.reset(); + ASSERT_TRUE(reader->release_pin().ok()); + reader.reset(); + } + close_queue(fixture); +} + +TEST(LegacyChunkMigration, ExplicitMigrationCommitsTheSameRootAndReopensWithNewFormat) +{ + test_chunk::TestLegacyChunk fixture; + { + auto config = configuration(fixture); + std::unique_ptr<Crowdbtree> tree; + const auto status = Crowdbtree::migrate_legacy_chunk(config, 1, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(fixture.catalog->current_generation, 2U); + EXPECT_TRUE(fixture.catalog->generation_pins.empty()); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(tree->get("key20", &slot, &value)); + EXPECT_EQ(slot, 20U); + EXPECT_EQ(value, std::string(8192, 'u')); + std::vector<scan_entry> rows; + bool truncated = false; + ASSERT_TRUE(tree->scan({}, {}, {}, 0, 1024 * 1024, false, 0, &rows, &truncated).ok()); + EXPECT_FALSE(truncated); + ASSERT_EQ(rows.size(), 20U); + tree.reset(); + (void)fixture.catalog->queue->pump(256); + auto next = fixture.catalog->next_owner(); + fixture.transport->catalog = next; + auto store = std::make_shared<detail::ChunkRuntimePageStore>( + detail::ChunkRuntimeOptions{.tree = 42, .owner_epoch = 7, .iu_size = 1, .mirrors = 2}, next, + fixture.transport, fixture.credits.credits.bytes); + config.page_store = store.get(); + config.backend_owner = store; + const auto reopened = Crowdbtree::open(config, &tree); + ASSERT_TRUE(reopened.ok()) << reopened.to_string(); + ASSERT_TRUE(tree->get("key20", &slot, &value)); + EXPECT_EQ(slot, 20U); + EXPECT_EQ(value, std::string(8192, 'u')); + EXPECT_EQ(store->durable_wal_offset(), 1234U); + } + close_queue(fixture); +} + +TEST(LegacyChunkMigration, OrdinaryOpenRejectsOldFormatWithoutChangingItsRoot) +{ + test_chunk::TestLegacyChunk fixture; + { + auto config = configuration(fixture); + const auto original = fixture.catalog->committed_manifest; + std::unique_ptr<Crowdbtree> tree; + const auto status = Crowdbtree::open(config, &tree); + EXPECT_EQ(status.code(), Code::kNotSupported) << status.to_string(); + EXPECT_FALSE(tree); + EXPECT_EQ(fixture.catalog->committed_manifest, original); + EXPECT_EQ(fixture.catalog->current_generation, 1U); + EXPECT_TRUE(fixture.catalog->generation_pins.empty()); + EXPECT_TRUE(fixture.catalog->migrations.empty()); + } + close_queue(fixture); +} + +TEST(LegacyChunkMigration, RejectedCommitKeepsOriginalRootRecoverableAndReleasesItsAttemptPin) +{ + test_chunk::TestLegacyChunk fixture; + { + auto config = configuration(fixture); + const auto original = fixture.catalog->committed_manifest; + fixture.catalog->fail_commit = true; + std::unique_ptr<Crowdbtree> tree; + const auto status = Crowdbtree::migrate_legacy_chunk(config, 1, &tree); + EXPECT_EQ(status.code(), Code::kUnavailable) << status.to_string(); + EXPECT_FALSE(tree); + EXPECT_EQ(fixture.catalog->committed_manifest, original); + EXPECT_EQ(fixture.catalog->current_generation, 1U); + EXPECT_TRUE(fixture.catalog->generation_pins.empty()); + EXPECT_EQ(fixture.catalog->migrations.size(), 1U); + auto reader = fixture.reader(); + ASSERT_TRUE(reader->open(1).ok()); + uint8_t byte = 0; + EXPECT_TRUE(reader->read_at(fixture.layout.pack(0).logical_offset, &byte, 1).ok()); + EXPECT_TRUE(reader->release_pin().ok()); + } + close_queue(fixture); +} + +TEST(LegacyChunkMigration, UnknownCommitRetainsOldPinAndRetryResolvesDurableReceipt) +{ + test_chunk::TestLegacyChunk fixture; + { + auto config = configuration(fixture); + fixture.catalog->lose_commit_outcome = true; + std::unique_ptr<Crowdbtree> tree; + const auto status = Crowdbtree::migrate_legacy_chunk(config, 1, &tree); + EXPECT_EQ(status.code(), Code::kUnavailable) << status.to_string(); + EXPECT_FALSE(tree); + EXPECT_EQ(fixture.catalog->current_generation, 2U); + EXPECT_EQ(fixture.catalog->generation_pins.size(), 1U); + EXPECT_EQ(fixture.catalog->migrations.size(), 1U); + (void)fixture.catalog->queue->pump(256); + fixture.catalog = fixture.catalog->next_legacy_owner(); + fixture.transport->catalog = fixture.catalog; + auto store = std::make_shared<detail::ChunkRuntimePageStore>( + detail::ChunkRuntimeOptions{.tree = 42, .owner_epoch = 7, .iu_size = 1, .mirrors = 2}, fixture.catalog, + fixture.transport, fixture.credits.credits.bytes); + config.page_store = store.get(); + config.backend_owner = store; + const auto retried = Crowdbtree::migrate_legacy_chunk(config, 1, &tree); + ASSERT_TRUE(retried.ok()) << retried.to_string(); + EXPECT_TRUE(fixture.catalog->generation_pins.empty()); + EXPECT_EQ(fixture.catalog->current_generation, 2U); + EXPECT_EQ(fixture.catalog->commit_calls, 0U); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(tree->get("key20", &slot, &value)); + EXPECT_EQ(value, std::string(8192, 'u')); + EXPECT_EQ(slot, 20U); + } + close_queue(fixture); +} + +TEST(LegacyChunkMigration, BoundedChildCopiesOnlyItsLogicalRangeFromSharedLegacyFrames) +{ + test_chunk::TestLegacyChunk fixture; + { + auto config = configuration(fixture); + config.key_range = KeyRange::bounded("key20", "key3"); + std::unique_ptr<Crowdbtree> tree; + const auto status = Crowdbtree::migrate_legacy_chunk(config, 1, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->tree_state_stats().logical.kv_known_count, 1U); + std::string value; + uint64_t slot = 0; + EXPECT_FALSE(tree->get("key2", &slot, &value)); + EXPECT_FALSE(tree->get("key3", &slot, &value)); + ASSERT_TRUE(tree->get("key20", &slot, &value)); + EXPECT_EQ(value, std::string(8192, 'u')); + EXPECT_EQ(slot, 20U); + } + close_queue(fixture); +} + +TEST(LegacyChunkMigration, PublicCEntryUsesInjectedRuntimeBackendAndPreservesTheRecoveredValue) +{ + test_chunk::TestLegacyChunk fixture; + ASSERT_TRUE(detail::ChunkPinReleaseQueue::create( + fixture.credits.credits.bytes, 256, std::make_shared<int>(1), [](void *) noexcept {}, nullptr, + &fixture.catalog->queue) + .ok()); + ct_runtime *raw_runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &raw_runtime), 0); + const std::unique_ptr<ct_runtime, decltype(&ct_runtime_free)> runtime(raw_runtime, &ct_runtime_free); + auto store = std::make_unique<ct_page_store>(); + store->bundle = std::make_shared<PageStoreBundle>(); + store->bundle->store = std::make_unique<detail::ChunkRuntimePageStore>( + detail::ChunkRuntimeOptions{.tree = 42, .owner_epoch = 7, .iu_size = 1, .mirrors = 2}, fixture.catalog, + fixture.transport, fixture.credits.credits.bytes); + ct_options options{}; + options.page_store = store.get(); + options.frame_bytes = 4096; + options.max_inline_value = 64; + ct_tree *raw_tree = nullptr; + const auto status = ct_migrate_legacy_chunk_with_runtime(&options, runtime.get(), nullptr, 1, &raw_tree); + ASSERT_EQ(status, 0); + { + const std::unique_ptr<ct_tree, decltype(&ct_close)> tree(raw_tree, &ct_close); + uint64_t slot = 0; + ct_buf value{}; + int32_t found = 0; + EXPECT_EQ(ct_get(tree.get(), reinterpret_cast<const uint8_t *>("key20"), 5, &found, &slot, &value), 0); + EXPECT_EQ(found, 1); + EXPECT_EQ(slot, 20U); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(value.data), value.len), std::string(8192, 'u')); + ct_free_buf(&value); + } + close_queue(fixture); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_block_gc_test.cpp b/lib/crowdb-tree/tests/integration/local_block_gc_test.cpp new file mode 100644 index 000000000..a9fa70e5b --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_block_gc_test.cpp @@ -0,0 +1,166 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/local_block_gc.h" +#include "btree/tree_core.h" +#include "btree/tree_core_access.h" +#include "extent_ledger_test_support.h" +#include "local_checkpoint_test_support.h" +#include "test_tmp.h" + +#include <future> + +namespace crowdb::tree +{ +using namespace test_read; + +namespace +{ +struct TestBlockGc +{ + tree_test::TempDir directory{"block_gc_"}; + std::unique_ptr<BlockPageStore> store; + LedgerFixture ledger; + + explicit TestBlockGc(uint64_t size = 6 * 4096, uint64_t block_size = 4096) + : ledger({.backend_id = 1, .file_size = size, .address_limit = size}) + { + EXPECT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, block_size, 1, &store).ok()); + const std::vector<uint8_t> bytes(size, 42); + EXPECT_TRUE(store->write_at(0, bytes.data(), bytes.size()).ok()); + } +}; + +TEST(LocalBlockGc, BoundedPassRetainsAnchorsAndReadersThenReusesDeletedHoles) +{ + TestBlockGc test; + auto anchor = test.ledger.recover(12288, 4096); + ASSERT_TRUE(anchor.status.ok()); + auto reader = anchor.extent.owner; + ASSERT_TRUE(test.ledger.activate().status.ok()); + const LocalBlockGcLimits limits{.max_blocks = 2, .max_deleted_bytes = 4096}; + auto first = + collect_free_local_blocks(*test.store, *test.ledger.ledger, 0, limits, test.ledger.credits.credits.bytes); + ASSERT_TRUE(first.status.ok()); + EXPECT_EQ(first.deleted_blocks, 1U); + EXPECT_EQ(first.next_block, 3U); + EXPECT_FALSE(first.complete); + uint64_t offset = 0; + EXPECT_TRUE(test.store->file_for_offset(0, &offset)); + EXPECT_TRUE(test.store->file_for_offset(4096, &offset)); + EXPECT_FALSE(test.store->file_for_offset(8192, &offset)); + anchor = {}; + auto second = collect_free_local_blocks(*test.store, *test.ledger.ledger, first.next_block, limits, + test.ledger.credits.credits.bytes); + ASSERT_TRUE(second.status.ok()); + EXPECT_EQ(second.protected_blocks, 1U); + EXPECT_EQ(second.deleted_blocks, 1U); + EXPECT_EQ(second.next_block, 5U); + EXPECT_TRUE(test.store->file_for_offset(12288, &offset)); + reader.reset(); + auto third = + collect_free_local_blocks(*test.store, *test.ledger.ledger, 3, limits, test.ledger.credits.credits.bytes); + ASSERT_TRUE(third.status.ok()); + EXPECT_EQ(third.deleted_blocks, 1U); + EXPECT_FALSE(test.store->file_for_offset(12288, &offset)); + auto reused = test.ledger.reserve(8192); + ASSERT_TRUE(reused.status.ok()); + EXPECT_EQ(reused.extent.address, 8192U); + const uint8_t value = 9; + ASSERT_TRUE(test.store->write_at(reused.extent.address, &value, 1).ok()); + uint8_t actual = 0; + ASSERT_TRUE(test.store->read_at(reused.extent.address, &actual, 1).ok()); + EXPECT_EQ(actual, value); +} + +TEST(LocalBlockGc, MetadataFailureAndPartialTailCannotDeleteUnleasedStorage) +{ + TestBlockGc test(5 * 4096, 8192); + ASSERT_TRUE(test.ledger.activate().status.ok()); + test.ledger.credits.seal(); + auto rejected = + collect_free_local_blocks(*test.store, *test.ledger.ledger, 0, {}, test.ledger.credits.credits.bytes); + EXPECT_EQ(rejected.status.code(), Code::kResourceExhausted); + EXPECT_EQ(rejected.next_block, 1U); + EXPECT_EQ(test.store->num_extents(), 3U); + const LocalBlockGcLimits too_small{.max_blocks = 8, .max_deleted_bytes = 4096}; + auto invalid = collect_free_local_blocks(*test.store, *test.ledger.ledger, 0, too_small, {}); + EXPECT_EQ(invalid.status.code(), Code::kResourceExhausted); + auto collected = collect_free_local_blocks(*test.store, *test.ledger.ledger, 0, {}, {}); + ASSERT_TRUE(collected.status.ok()); + EXPECT_TRUE(collected.complete); + EXPECT_EQ(collected.deleted_blocks, 1U); + uint64_t offset = 0; + EXPECT_TRUE(test.store->file_for_offset(16384, &offset)); +} + +TEST(LocalBlockGc, PublicCheckpointReclaimsCrashTailWithoutLosingRecovery) +{ + tree_test::TempDir dir("checkpoint_gc_"); + std::unique_ptr<BlockPageStore> opened; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 2, 3, 8192, 4096, &opened).ok()); + std::shared_ptr<BlockPageStore> device = std::move(opened); + const uint8_t orphan = 37; + ASSERT_TRUE(device->write_at((31 * 8192) - 1, &orphan, 1).ok()); + test_write::TestCheckpointStore store; + store.device = device; + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = device.get(); + config.async_page_store = &store; + config.frame_bytes = 4096; + std::unique_ptr<Crowdbtree> tree; + const auto opened_status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(opened_status.ok()) << opened_status.to_string(); + ASSERT_TRUE(tree->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "value"}}}).ok()); + uint64_t coverage = 0; + auto status = tree->snapshot(&coverage); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 1U); + TreeCoreAccess::get(*tree)->operation_lifetime().drain_accepted(); + TreeCoreAccess::get(*tree)->binding()->drain(); + uint64_t offset = 0; + EXPECT_FALSE(device->file_for_offset(30 * 8192, &offset)); + tree.reset(); + status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + GetView value; + ASSERT_TRUE(TreeCoreAccess::get(*tree)->get("a", &value).ok()); + EXPECT_EQ(value.value().to_string(), "value"); +} + +TEST(LocalBlockGc, PublicAsyncCheckpointCleanupDrainsBeforeRawBackendClose) +{ + tree_test::TempDir dir("async_checkpoint_gc_"); + std::unique_ptr<BlockPageStore> device; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 4, 5, 8192, 4096, &device).ok()); + const uint8_t orphan = 23; + ASSERT_TRUE(device->write_at((31 * 8192) - 1, &orphan, 1).ok()); + Config config; + TreeRuntimeOptions runtime; + runtime.workers = 1; + runtime.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(runtime, &config.runtime).ok()); + config.page_store = device.get(); + config.frame_bytes = 4096; + std::shared_ptr<TreeCore> core; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "value"}}}).ok()); + std::promise<Status> completion; + auto done = completion.get_future(); + core->checkpoint_async([&](Status status, uint64_t coverage) { + EXPECT_EQ(coverage, 1U); + completion.set_value(std::move(status)); + }); + ASSERT_EQ(done.wait_for(std::chrono::seconds(5)), std::future_status::ready); + const auto status = done.get(); + ASSERT_TRUE(status.ok()) << status.to_string(); + core.reset(); + uint64_t offset = 0; + EXPECT_FALSE(device->file_for_offset(30 * 8192, &offset)); + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + GetView value; + ASSERT_TRUE(core->get("a", &value).ok()); + EXPECT_EQ(value.value().to_string(), "value"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_backend_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_backend_test.cpp new file mode 100644 index 000000000..11e26a6e0 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_backend_test.cpp @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_checkpoint_test_support.h" +#include "test_tmp.h" + +namespace crowdb::tree +{ +using namespace test_write; + +#ifdef CROWDB_HAVE_LIBURING +TEST(LocalCheckpoint, RealBlockReopenUsesOnlyAnchorAndDurableMapping) +{ + crowdb::tree_test::TempDir dir("local_checkpoint_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 3, 4096, 4096, &file).ok()); + crowdb::common::Topology topology; + topology.pipelines.emplace_back().entries = 64; + crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + auto store = std::make_shared<BlockAsyncPageStore>(file.get(), &uring); + TestCheckpoint test; + uint64_t next = 1; + const std::string value(12000, 'r'); + ASSERT_TRUE( + CapturedWriteOperation::create( + captured_leaf(page({ + {.key = "a", .value = value} + })), + {}, store, test.source.runtime, test.source.lane, + {.frames = {.encode = {.frame_bytes = 4096, .inline_value_bytes = 16}}, .write = {.max_frame_bytes = 4096}}, + test.source.credits.credits, + [owner = test.source.ledger, &next](uint64_t count, ReservedOverflowIds *out) { + *out = {.first = next, .count = count, .owner = owner}; + next += count; + return Status::Ok(); + }, + extent_write_reservation(test.source.ledger, test.source.credits.credits.bytes), {}, &test.source.operation) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(test.source.operation->sender())); + test.source.runtime->drain(); + ASSERT_TRUE(LocalCheckpointOperation::create(test.source.operation->result().pages, test.registry, store, + test.source.runtime, test.source.lane, {}, test.source.credits.credits, + &test.operation) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); + test.source.runtime->drain(); + EXPECT_EQ(test.registry->durable_coverage(), 10U); + test.operation.reset(); + test.source.operation.reset(); + test.registry.reset(); + test.source.ledger.reset(); + store.reset(); + file.reset(); + test.source.credits.seal(); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 3, 4096, 4096, &file).ok()); + Config config; + config.page_store = file.get(); + config.frame_bytes = 4096; + std::unique_ptr<Crowdbtree> tree; + const auto opened = Crowdbtree::open(config, &tree); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + std::string restored; + uint64_t slot = 0; + ASSERT_TRUE(tree->get(Slice("a"), &slot, &restored)); + EXPECT_EQ(restored, value); + EXPECT_EQ(slot, 1U); +} +#endif + +TEST(LocalCheckpoint, ClosedRuntimeWaitsForAcceptedIoBeforeReleasingTheCommitGate) +{ + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + test.store->defer_at = 1; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.source.runtime->drain(); + ASSERT_TRUE(test.store->pending); + test.source.runtime->close(); + test.operation->request_stop(); + EXPECT_EQ(done.errors.load() + done.stopped.load(), 0U); + test.store->finish(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_FALSE(test.registry->needs_recovery()); + EXPECT_EQ(test.registry->snapshot_sequence(), 0U); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp new file mode 100644 index 000000000..7260c0a70 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/maptable/mapping_geometry.h" +#include "local_checkpoint_test_support.h" +#include "runtime_tree_access.h" + +namespace crowdb::tree +{ +using namespace test_write; + +namespace +{ +std::shared_ptr<PublishedTreeState> sparse_captured_tree(size_t leaves) +{ + auto captured = std::make_shared<PublishedTreeState>(); + auto index = std::make_shared<InnerPageState>(); + std::vector<PageReplacement> replacements; + for (size_t i = 0; i < leaves; ++i) { + auto key = std::to_string(1000 + i); + PageKeyRange range; + if (i != 0) { + range.lower = key; + index->separators.emplace_back(key); + } + if (i + 1 < leaves) { + range.upper = std::to_string(1001 + i); + } + const auto pid = i * kMappingSegmentSize; + index->children.push_back(pid); + auto leaf = std::make_shared<PageState>(*page( + { + {.key = key, .value = "v"} + }, + range)); + leaf->right_sibling = i + 1 == leaves ? kInvalidPageId : (i + 1) * kMappingSegmentSize; + replacements.push_back({.page_id = pid, .expected = {}, .replacement = published(std::move(leaf))}); + } + auto root = std::make_shared<PublishedPage>(); + root->level = 1; + root->inner = index; + root->metrics.leaf = false; + captured->root_page_id = leaves * kMappingSegmentSize; + captured->next_page_id = captured->root_page_id + 1; + captured->coverage = 10; + replacements.push_back({.page_id = captured->root_page_id, .expected = {}, .replacement = root}); + MappingView::CopyStats copied; + captured->mapping = captured->mapping.replace(replacements, copied); + return captured; +} +} // namespace + +TEST(LocalCheckpoint, SparseMappingStreamsDirectoryAcrossWindowsAndEnforcesSegmentCap) +{ + TestCheckpoint test; + // 385 segment entries require 12,340 logical directory bytes, crossing + // the single 12,288-byte aligned metadata window. + constexpr size_t leaves = 384; + ASSERT_TRUE(test.source + .prepare(sparse_captured_tree(leaves), {}, + {.frames = {.encode = {.frame_bytes = 16384}}, .write = {.max_frame_bytes = 16384}}) + .ok()); + Completion materialized; + test.source.run(&materialized); + ASSERT_EQ(materialized.values.load(), 1U) << materialized.status.to_string(); + auto pages = test.source.operation->result().pages; + ASSERT_TRUE(test.prepare(pages).ok()); + Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + const auto &anchor = test.operation->result().checkpoint->anchor; + EXPECT_EQ(anchor.segdir_len, 12340U); + EXPECT_EQ(anchor.leaf_count, leaves); + EXPECT_EQ(anchor.inner_count, 1U); + EXPECT_EQ(test.store->events.size(), leaves + 6); + Config config; + config.page_store = test.store->device.get(); + config.frame_bytes = 16384; + config.inner_max_keys = leaves - 1; + std::unique_ptr<Crowdbtree> tree; + const auto opened = Crowdbtree::open(config, &tree); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + for (size_t i = 0; i < leaves; ++i) { + const auto key = std::to_string(1000 + i); + GetView result; + const auto read = RuntimeTreeAccess_for_tests::core(*tree).get(Slice(key), &result); + ASSERT_TRUE(read.ok()) << key << ": " << read.to_string(); + ASSERT_TRUE(result.found()) << key; + EXPECT_EQ(result.value().to_string(), "v"); + } + const auto protected_before = test.source.ledger->stats().protected_ranges; + ASSERT_TRUE(test.prepare(pages, {.max_segments = 2}).ok()); + Completion capped; + test.run(&capped); + EXPECT_EQ(capped.errors.load(), 1U); + EXPECT_EQ(capped.status.code(), Code::kResourceExhausted); + EXPECT_EQ(test.registry->snapshot_sequence(), 1U); + EXPECT_FALSE(test.registry->needs_recovery()); + EXPECT_EQ(test.source.ledger->stats().protected_ranges, protected_before); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_failure_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_failure_test.cpp new file mode 100644 index 000000000..515ad0565 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_failure_test.cpp @@ -0,0 +1,132 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_checkpoint_test_support.h" + +namespace crowdb::tree +{ +using namespace test_write; + +TEST(LocalCheckpoint, CancellationDrainsMetadataButFinishesAnAttemptedAnchor) +{ + for (const size_t pause : {1U, 4U}) { + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + test.store->defer_at = pause; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.source.runtime->drain(); + ASSERT_TRUE(test.store->pending); + test.operation->request_stop(); + test.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 0U); + EXPECT_EQ(done.values.load(), 0U); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + test.store->finish(); + test.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), pause == 1 ? 1U : 0U); + EXPECT_EQ(done.values.load(), pause == 4 ? 1U : 0U); + EXPECT_EQ(test.registry->durable_coverage(), pause == 4 ? 10U : 0U); + EXPECT_FALSE(test.registry->needs_recovery()); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + } +} + +TEST(LocalCheckpoint, UnknownSubmissionRetainsBufferAndExtentsUntilLateCallback) +{ + for (const size_t unknown : {1U, 4U}) { + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + test.store->throw_at = unknown; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.source.runtime->drain(); + ASSERT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.registry->durable_coverage(), 0U); + EXPECT_EQ(test.registry->needs_recovery(), unknown == 4); + test.operation.reset(); + test.source.operation.reset(); + EXPECT_EQ(test.source.ledger->stats().protected_ranges, unknown == 1 ? 2U : 3U); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + auto duplicate = test.store->pending; + test.store->finish(); + duplicate.complete(Status::Ok()); + duplicate = {}; + test.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.store->events.size(), unknown); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_EQ(test.source.ledger->stats().protected_ranges, unknown == 1 ? 0U : 3U); + test.registry.reset(); + test.source.ledger.reset(); + test.source.credits.seal(); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + } +} + +TEST(LocalCheckpoint, InlineSuccessFollowedBySubmissionFailureCannotAdvance) +{ + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + test.store->inline_throw_at = 1; + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.store->events.size(), 1U); + EXPECT_FALSE(test.registry->needs_recovery()); + EXPECT_EQ(test.registry->snapshot_sequence(), 0U); +} + +TEST(LocalCheckpoint, ConcurrentCommitAndCoverageRegressionAreRejected) +{ + TestCheckpoint test; + auto pages = test.materialize(); + ASSERT_TRUE(test.prepare(pages).ok()); + test.store->defer_at = 1; + auto first = test.operation; + Completion first_done; + auto connected = stdexec::connect(first->sender(), Receiver{.done = &first_done}); + stdexec::start(connected); + test.source.runtime->drain(); + ASSERT_TRUE(test.prepare(pages).ok()); + Completion rejected; + test.run(&rejected); + EXPECT_EQ(rejected.errors.load(), 1U); + EXPECT_EQ(test.store->events.size(), 1U); + test.store->finish(); + test.source.runtime->drain(); + ASSERT_EQ(first_done.values.load(), 1U); + auto older = std::make_shared<CapturedDurablePages>(*pages); + older->coverage = 9; + ASSERT_TRUE(test.prepare(older).ok()); + Completion regressed; + test.run(®ressed); + EXPECT_EQ(regressed.errors.load(), 1U); + EXPECT_EQ(test.registry->durable_coverage(), 10U); + EXPECT_EQ(test.store->events.size(), 5U); +} + +TEST(LocalCheckpoint, AllocationFailureAfterAnchorKeepsRecoveryProtection) +{ + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + test.store->defer_at = 4; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.source.runtime->drain(); + ASSERT_TRUE(test.store->pending); + auto retained_callback = test.store->pending; + AllocationLease pressure(test.source.credits.credits.bytes, test.source.credits.credits.bytes->available()); + test.store->finish(); + test.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kResourceExhausted); + EXPECT_TRUE(test.registry->needs_recovery()); + EXPECT_EQ(test.registry->snapshot_sequence(), 0U); + EXPECT_EQ(test.source.ledger->stats().protected_ranges, 3U); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_TRUE(retained_callback); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp new file mode 100644 index 000000000..5c81871dd --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp @@ -0,0 +1,150 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_checkpoint_test_support.h" + +namespace crowdb::tree +{ +using namespace test_write; + +TEST(LocalCheckpoint, PersistsExistingFormatAndAdvancesOnlyAfterSecondBarrier) +{ + TestCheckpoint test; + auto pages = test.materialize(std::string(3000, 'v')); + ASSERT_TRUE(test.prepare(pages).ok()); + test.store->defer_at = 5; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.source.runtime->drain(); + EXPECT_EQ(test.store->events, (std::vector<std::string>{"metadata", "metadata", "barrier", "anchor", "barrier"})); + EXPECT_EQ(done.values.load(), 0U); + EXPECT_EQ(test.registry->durable_coverage(), 0U); + test.store->finish(); + test.source.runtime->drain(); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(test.registry->durable_coverage(), 10U); + EXPECT_EQ(test.registry->snapshot_sequence(), 1U); + EXPECT_FALSE(test.registry->needs_recovery()); + test.expect_read(std::string(3000, 'v')); +} + +TEST(LocalCheckpoint, RotationRetainsBothAnchorsAndReaderOwners) +{ + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U); + auto reader = test.operation->result().checkpoint; + std::weak_ptr<const void> first_extent = reader->metadata->extents[0].owner; + ASSERT_TRUE(test.prepare(test.materialize("second")).ok()); + Completion second; + test.run(&second); + ASSERT_EQ(second.values.load(), 1U); + EXPECT_FALSE(first_extent.expired()); + ASSERT_TRUE(test.prepare(test.materialize("third")).ok()); + Completion third; + test.run(&third); + ASSERT_EQ(third.values.load(), 1U); + EXPECT_FALSE(first_extent.expired()); + reader.reset(); + EXPECT_TRUE(first_extent.expired()); + EXPECT_EQ(test.registry->snapshot_sequence(), 3U); + std::vector<snapshot::CommitAnchor> anchors; + ASSERT_TRUE(snapshot::read_valid_anchors(*test.store->device, 4096, &anchors).ok()); + ASSERT_EQ(anchors.size(), 2U); + EXPECT_EQ(anchors[0].snapshot_seq, 2U); + EXPECT_EQ(anchors[1].snapshot_seq, 3U); + test.expect_read("third"); +} + +TEST(LocalCheckpoint, PreAnchorFailureKeepsPreviousSnapshotAndPermitsRetry) +{ + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U); + auto pages = test.materialize("next"); + ASSERT_TRUE(test.prepare(pages).ok()); + test.store->fail_at = test.store->events.size() + 3; + Completion failed; + test.run(&failed); + EXPECT_EQ(failed.errors.load(), 1U); + EXPECT_EQ(test.registry->snapshot_sequence(), 1U); + EXPECT_FALSE(test.registry->needs_recovery()); + test.expect_read("v"); + ASSERT_TRUE(test.prepare(pages).ok()); + Completion retry; + test.run(&retry); + EXPECT_EQ(retry.values.load(), 1U) << retry.status.to_string(); + test.expect_read("next"); +} + +TEST(LocalCheckpoint, FinalBarrierFailureProtectsCandidateAndRequiresRecovery) +{ + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + test.store->fail_at = 5; + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_TRUE(test.registry->needs_recovery()); + EXPECT_EQ(test.registry->durable_coverage(), 0U); + EXPECT_EQ(test.registry->snapshot_sequence(), 0U); + test.operation.reset(); + test.source.operation.reset(); + EXPECT_EQ(test.source.ledger->stats().protected_ranges, 3U); + ASSERT_TRUE(test.prepare().ok()); + Completion rejected; + test.run(&rejected); + EXPECT_EQ(rejected.errors.load(), 1U); + EXPECT_EQ(test.store->events.size(), 5U); +} + +TEST(LocalCheckpoint, CoverageMayResetOnlyWhenTheLogicalGenerationAdvances) +{ + TestCheckpoint test; + auto first_pages = test.materialize(); + ASSERT_TRUE(test.prepare(first_pages).ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U) << first.status.to_string(); + auto empty = detail::reserved_shared<CapturedDurablePages>(test.source.credits.credits.bytes, + test.source.credits.credits.bytes); + empty->generation = first_pages->generation; + ASSERT_TRUE(test.prepare(empty).ok()); + const auto before = test.store->events.size(); + Completion same_generation; + test.run(&same_generation); + EXPECT_EQ(same_generation.status.code(), Code::kUnavailable); + EXPECT_EQ(test.store->events.size(), before); + ++empty->generation; + ASSERT_TRUE(test.prepare(empty).ok()); + Completion next_generation; + test.run(&next_generation); + ASSERT_EQ(next_generation.values.load(), 1U) << next_generation.status.to_string(); + EXPECT_EQ(test.registry->snapshot_sequence(), 2U); + EXPECT_EQ(test.registry->durable_coverage(), 0U); + ASSERT_TRUE(test.prepare(first_pages).ok()); + Completion old_generation; + test.run(&old_generation); + EXPECT_EQ(old_generation.status.code(), Code::kUnavailable); + EXPECT_EQ(test.registry->snapshot_sequence(), 2U); + EXPECT_EQ(test.registry->durable_coverage(), 0U); +} + +TEST(LocalCheckpoint, EmptyCapturedViewCommitsAnEmptyDirectory) +{ + TestCheckpoint test; + auto pages = std::make_shared<CapturedDurablePages>(test.source.credits.credits.bytes); + pages->coverage = 12; + ASSERT_TRUE(test.prepare(pages).ok()); + Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(test.registry->durable_coverage(), 12U); + EXPECT_EQ(test.operation->result().checkpoint->anchor.segdir_len, 20U); + EXPECT_EQ(test.source.ledger->stats().protected_ranges, 1U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_compaction_async_test.cpp b/lib/crowdb-tree/tests/integration/local_compaction_async_test.cpp new file mode 100644 index 000000000..0346dca4b --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_compaction_async_test.cpp @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/local_compaction_operation.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "snapshot/operation_wait.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +#include <future> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct TestBackendGate +{ + ApplyDispatcher::Node node{&TestBackendGate::run, this}; + std::promise<void> entered; + std::atomic<bool> released{false}; + + static void run(void *context) noexcept + { + auto &gate = *static_cast<TestBackendGate *>(context); + gate.entered.set_value(); + while (!gate.released.load(std::memory_order_acquire)) { + gate.released.wait(false); + } + } + + void release() + { + released.store(true, std::memory_order_release); + released.notify_all(); + } +}; + +struct TestGateRelease +{ + std::shared_ptr<TestBackendGate> gate; + + ~TestGateRelease() + { + gate->release(); + } +}; + +struct TestCompletion +{ + using receiver_concept = stdexec::receiver_tag; + std::promise<Status> *result; + + void set_value() const noexcept + { + set_error(Status::Ok()); + } + + void set_stopped() const noexcept + { + set_error(Status::unavailable()); + } + + void set_error(Status status) const noexcept + { + result->set_value(std::move(status)); + } +}; + +TEST(LocalCompactionAsync, BlockedBackendLeavesSingleCpuFreeForResidentReads) +{ + tree_test::TempDir directory("async_compact_cpu_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.page_store = store.get(); + config.frame_bytes = 4096; + std::shared_ptr<TreeCore> core; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "resident"}}}).ok()); + auto gate = std::make_shared<TestBackendGate>(); + TestGateRelease release{gate}; + auto entered = gate->entered.get_future(); + core->binding()->backend_progress()->request(gate->node, gate); + ASSERT_EQ(entered.wait_for(std::chrono::seconds(2)), std::future_status::ready); + std::shared_ptr<LocalCompactionOperation> operation; + ASSERT_TRUE(core->prepare_compaction({}, &operation).ok()); + std::promise<Status> terminal; + auto completed = terminal.get_future(); + auto connection = stdexec::connect(operation->sender(), TestCompletion{.result = &terminal}); + stdexec::start(connection); + // Starting the operation synchronously queues its paid CPU continuation. + // Drain that lane on another host thread: it must finish while the sole + // backend worker is still stopped, before the foreground read is issued. + std::promise<void> cpu_terminal; + auto cpu_drained = cpu_terminal.get_future(); + std::jthread cpu_wait([&] { + core->binding()->lane()->drain(); + cpu_terminal.set_value(); + }); + const auto cpu_status = cpu_drained.wait_for(std::chrono::seconds(2)); + std::promise<Status> read; + auto readable = read.get_future(); + core->get_async("key", [&](Status status, GetView value) { + if (status.ok() && (!value.found() || value.value().to_string() != "resident")) { + status = Status::corruption(); + } + read.set_value(std::move(status)); + }); + const auto read_status = readable.wait_for(std::chrono::seconds(2)); + const auto pending = completed.wait_for(std::chrono::milliseconds(0)); + gate->release(); + cpu_wait.join(); + EXPECT_EQ(cpu_status, std::future_status::ready); + ASSERT_EQ(read_status, std::future_status::ready); + EXPECT_TRUE(readable.get().ok()); + EXPECT_EQ(pending, std::future_status::timeout); + ASSERT_EQ(completed.wait_for(std::chrono::seconds(2)), std::future_status::ready); + EXPECT_TRUE(completed.get().ok()); + operation.reset(); +} + +TEST(LocalCompactionAsync, StopBeforeStartCompletesOnceAndReleasesTheSelectionGate) +{ + tree_test::TempDir directory("async_compact_cancel_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.page_store = store.get(); + config.runtime_binding.cpu_queue_limit = 2; + std::shared_ptr<TreeCore> core; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + std::shared_ptr<LocalCompactionOperation> operation; + ASSERT_TRUE(core->prepare_compaction({}, &operation).ok()); + std::atomic<size_t> observed{0}; + operation->observe_completion([&observed](const MergeGcStats &) { observed.fetch_add(1); }); + operation->request_stop(); + EXPECT_EQ(detail::wait_snapshot_operation(operation).code(), Code::kUnavailable); + EXPECT_FALSE(core->local_backend()->gc_active.load()); + EXPECT_EQ(detail::wait_snapshot_operation(operation).code(), Code::kUnavailable); + EXPECT_EQ(operation->result().blocks_selected, 0U); + EXPECT_EQ(observed.load(), 1U); + operation.reset(); + core->binding()->drain(); + ASSERT_TRUE(core->prepare_compaction({}, &operation).ok()); + EXPECT_TRUE(detail::wait_snapshot_operation(operation).ok()); + operation.reset(); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_recovery_backend_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_backend_test.cpp new file mode 100644 index 000000000..8605ece9c --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_recovery_backend_test.cpp @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_recovery_test_support.h" +#include "test_tmp.h" + +#include <future> + +namespace crowdb::tree +{ +using namespace test_write; + +#ifdef CROWDB_HAVE_LIBURING +namespace +{ +Status await_status(detail::CallbackSender sender) +{ + try { + return stdexec::sync_wait(sender) ? Status::Ok() : Status::unavailable(); + } + catch (const Status &status) { + return status; + } +} +} // namespace + +TEST(LocalRecovery, ReopenedBlockFilesRebuildProtectedPrimaryAndOverflowOwners) +{ + crowdb::tree_test::TempDir dir("local_recovery_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 3, 4096, 4096, &file).ok()); + crowdb::common::Topology topology; + topology.pipelines.emplace_back().entries = 64; + crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + auto store = std::make_shared<BlockAsyncPageStore>(file.get(), &uring); + TestCheckpoint test; + uint64_t next = 1; + const std::string value(12000, 'r'); + ASSERT_TRUE( + CapturedWriteOperation::create( + captured_leaf(page({ + {.key = "a", .value = value} + })), + {}, store, test.source.runtime, test.source.lane, + {.frames = {.encode = {.frame_bytes = 4096, .inline_value_bytes = 16}}, .write = {.max_frame_bytes = 4096}}, + test.source.credits.credits, + [owner = test.source.ledger, &next](uint64_t count, ReservedOverflowIds *out) { + *out = {.first = next, .count = count, .owner = owner}; + next += count; + return Status::Ok(); + }, + extent_write_reservation(test.source.ledger, test.source.credits.credits.bytes), {}, &test.source.operation) + .ok()); + auto status = await_status(test.source.operation->sender()); + ASSERT_TRUE(status.ok()) << "captured write: " << status.to_string(); + test.source.runtime->drain(); + ASSERT_TRUE(LocalCheckpointOperation::create(test.source.operation->result().pages, test.registry, store, + test.source.runtime, test.source.lane, {}, test.source.credits.credits, + &test.operation) + .ok()); + status = await_status(test.operation->sender()); + ASSERT_TRUE(status.ok()) << "checkpoint commit: " << status.to_string(); + test.source.runtime->drain(); + test.operation.reset(); + test.source.operation.reset(); + test.registry.reset(); + test.source.ledger.reset(); + store.reset(); + file.reset(); + test.source.credits.seal(); + EXPECT_EQ(test.source.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 3, 4096, 4096, &file).ok()); + store = std::make_shared<BlockAsyncPageStore>(file.get(), &uring); + LoadCredits credits; + std::shared_ptr<LocalRecoveryOperation> recovery; + ASSERT_TRUE(LocalRecoveryOperation::create(store, test.source.runtime, test.source.lane, + {.backend_id = 9, .generation = 2, .file_size = file->size()}, {}, + credits.credits, &recovery) + .ok()); + status = await_status(recovery->sender()); + ASSERT_TRUE(status.ok()) << "local recovery: " << status.to_string(); + test.source.runtime->drain(); + auto pages = recovery->result().latest; + auto ledger = recovery->result().ledger; + ASSERT_TRUE(pages); + EXPECT_EQ(recovery->result().registry->durable_coverage(), 10U); + EXPECT_EQ(ledger->stats().protected_ranges, pages->frames.size() + 2); + recovery.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, pages->frames.size()); + auto loader = std::make_shared<AsyncStorePageLoader>( + store, test.source.runtime, test.source.lane, + PageLoadLimits{.max_inflight = 1, .max_blob_bytes = 4096, .max_frame_bytes = 4096}, + [pages](uint64_t pid, std::shared_ptr<const BackingRef> *out) { + const auto frame = pages->frames.find(pid); + if (frame == pages->frames.end()) { + return Status::not_found(); + } + *out = frame->second.backing; + return Status::Ok(); + }, + credits.credits); + std::promise<std::shared_ptr<const LeafBase>> leaf_result; + loader->load_leaf({.identity = 1, .backing = pages->frames.at(0).backing}, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()) << status.to_string(); + leaf_result.set_value(std::move(leaf)); + }); + auto leaf = leaf_result.get_future().get(); + ASSERT_TRUE(leaf); + const CellView cell(leaf->cell(0)); + std::promise<std::shared_ptr<const buffer>> value_result; + loader->load_overflow(cell.overflow_head(), cell.overflow_len(), [&](Status status, auto restored) { + EXPECT_TRUE(status.ok()) << status.to_string(); + value_result.set_value(std::move(restored)); + }); + auto restored = value_result.get_future().get(); + ASSERT_TRUE(restored); + EXPECT_EQ(Slice(restored->data(), restored->size()).to_string(), value); + test.source.runtime->drain(); + leaf.reset(); + restored.reset(); + pages.reset(); + loader.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, 0U); + ledger.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +#endif +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_recovery_failure_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_failure_test.cpp new file mode 100644 index 000000000..c65cdfa72 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_recovery_failure_test.cpp @@ -0,0 +1,149 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_recovery_test_support.h" + +namespace crowdb::tree +{ +using namespace test_write; + +TEST(LocalRecoveryFailure, CancellationDrainsAcceptedReadBeforeReleasingItsBuffer) +{ + for (const size_t pause : {1U, 2U, 3U, 4U}) { + TestRecovery test; + test.checkpoint(); + ASSERT_TRUE(test.prepare().ok()); + test.store->defer_at = pause; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.original.source.runtime->drain(); + ASSERT_TRUE(test.store->pending); + test.operation->request_stop(); + test.original.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + test.store->finish(); + test.original.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_EQ(done.errors.load(), 0U); + EXPECT_EQ(test.store->reads, pause); + EXPECT_FALSE(test.operation->result().ledger); + EXPECT_FALSE(test.operation->result().registry); + test.operation.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + } +} + +TEST(LocalRecoveryFailure, UnknownAcceptanceRetainsWindowUntilLateDuplicateCallbacksEnd) +{ + for (const size_t unknown : {1U, 4U}) { + TestRecovery test; + test.checkpoint(); + ASSERT_TRUE(test.prepare().ok()); + test.store->throw_at = unknown; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.original.source.runtime->drain(); + ASSERT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kIoError); + EXPECT_FALSE(test.operation->result().ledger); + test.operation.reset(); + test.credits.seal(); + EXPECT_GE(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 2U * 1024 * 1024); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + auto duplicate = test.store->pending; + test.store->finish(); + duplicate.complete(Status::corruption()); + duplicate = {}; + test.original.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.store->reads, unknown); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + } +} + +TEST(LocalRecoveryFailure, InlineReadFollowedBySubmissionExceptionCannotAdvance) +{ + TestRecovery test; + test.checkpoint(); + ASSERT_TRUE(test.prepare().ok()); + test.store->inline_throw_at = 1; + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kIoError); + EXPECT_EQ(test.store->reads, 1U); + EXPECT_FALSE(test.operation->result().latest); + EXPECT_FALSE(test.operation->result().registry); +} + +TEST(LocalRecoveryFailure, ReadErrorsNeverTurnIntoAnEmptyOrOlderRecovery) +{ + for (const size_t failure : {1U, 2U, 3U, 4U}) { + TestRecovery test; + test.checkpoint(); + ASSERT_TRUE(test.prepare().ok()); + test.store->fail_at = failure; + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kIoError); + EXPECT_EQ(test.store->reads, failure); + EXPECT_FALSE(test.operation->result().latest); + EXPECT_FALSE(test.operation->result().registry); + } +} + +TEST(LocalRecoveryFailure, ClosingRuntimeDrainsPendingReadAndRejectsFurtherRecovery) +{ + TestRecovery test; + test.checkpoint(); + ASSERT_TRUE(test.prepare().ok()); + test.store->defer_at = 1; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.original.source.runtime->drain(); + ASSERT_TRUE(test.store->pending); + test.original.source.runtime->close(); + test.operation->request_stop(); + EXPECT_EQ(done.stopped.load(), 0U); + test.store->finish(); + test.original.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_EQ(test.store->reads, 1U); + test.operation.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} + +TEST(LocalRecoveryFailure, BudgetExhaustionAfterReadCannotPublishPartialOwnership) +{ + TestRecovery test; + test.checkpoint(); + ASSERT_TRUE(test.prepare().ok()); + test.store->defer_at = 1; + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.original.source.runtime->drain(); + ASSERT_TRUE(test.store->pending); + AllocationLease pressure(test.credits.credits.bytes, test.credits.credits.bytes->available()); + test.store->finish(); + test.original.source.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(test.operation->result().ledger); + EXPECT_FALSE(test.operation->result().registry); + EXPECT_EQ(test.store->reads, 1U); + pressure.reset(); + test.operation.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_recovery_frames_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_frames_test.cpp new file mode 100644 index 000000000..79024b720 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_recovery_frames_test.cpp @@ -0,0 +1,271 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_recovery_test_support.h" + +namespace crowdb::tree +{ +using namespace test_write; + +namespace +{ +std::vector<uint8_t> read_frame(TestRecovery &test, uint64_t pid) +{ + const auto &[address, length] = test.locations.at(pid); + std::vector<uint8_t> blob(length); + EXPECT_TRUE(test.store->device->read_at(address, blob.data(), blob.size()).ok()); + std::vector<uint8_t> raw(durable_blob_raw_len(blob.data(), blob.size())); + EXPECT_TRUE(decode_durable_page(blob.data(), blob.size(), raw.data(), raw.size()).ok()); + return raw; +} + +void write_frame(TestRecovery &test, uint64_t pid, std::vector<uint8_t> raw, bool restamp = true) +{ + if (restamp) { + frame_restamp_crc(raw.data(), raw.size()); + } + std::vector<uint8_t> blob; + encode_durable_page(raw.data(), raw.size(), compress_algo::kLz4, &blob); + const auto &[address, length] = test.locations.at(pid); + ASSERT_LE(blob.size(), length); + blob.resize(length); + ASSERT_TRUE(test.store->device->write_at(address, blob.data(), blob.size()).ok()); +} + +size_t first_cell(std::span<const uint8_t> raw) +{ + const LeafFrameView leaf(raw.data(), raw.size()); + return static_cast<size_t>(leaf.cell(0).bytes() - raw.data()); +} + +void expect_failure(TestRecovery &test, Code code = Code::kCorruption, LocalRecoveryLimits limits = {}) +{ + ASSERT_TRUE(test.prepare(limits).ok()); + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), code); + EXPECT_FALSE(test.operation->result().latest); + EXPECT_FALSE(test.operation->result().registry); + test.operation.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace + +TEST(LocalRecoveryFrames, RejectsBlobCrcFrameCrcSelfPidAndPageTypeCorruption) +{ + for (size_t mode = 0; mode < 4; ++mode) { + TestRecovery test; + test.checkpoint(); + if (mode == 0) { + const auto &[address, length] = test.locations.at(0); + std::vector<uint8_t> blob(length); + ASSERT_TRUE(test.store->device->read_at(address, blob.data(), blob.size()).ok()); + blob[kDurableBlobHeader] ^= 1; + ASSERT_TRUE(test.store->device->write_at(address, blob.data(), blob.size()).ok()); + } + else { + auto raw = read_frame(test, 0); + if (mode == 3) { + raw[fh::kType] = 0xFF; + } + else { + frame_put_u64(raw.data(), fh::kSelfpage_id, 9); + } + write_frame(test, 0, std::move(raw), mode != 1); + } + expect_failure(test); + } +} + +TEST(LocalRecoveryFrames, RejectsInvalidLeafFlagsSlotsAndOverflowCells) +{ + for (size_t mode = 0; mode < 6; ++mode) { + TestRecovery test; + test.checkpoint(std::string(2000, 'v')); + auto raw = read_frame(test, 0); + auto *cell = raw.data() + first_cell(raw); + if (mode == 0) { + frame_put_u64(cell, 0, test.anchor.last_applied_slot + 1); + } + else if (mode == 1) { + cell[8] = 0x80; + } + else if (mode == 2) { + cell[8] = kFlagTombstone; + } + else if (mode == 3) { + cell[8] = kFlagTombstone | kFlagOverflow; + } + else if (mode == 4) { + frame_put_u64(cell, kCellHeaderSize, kInvalidPageId); + } + else { + frame_put_u64(cell, kCellHeaderSize + sizeof(uint64_t), 0); + } + write_frame(test, 0, std::move(raw)); + expect_failure(test); + } +} + +TEST(LocalRecoveryFrames, RejectsMissingShortLongCyclicAndEmptyOverflowChains) +{ + for (size_t mode = 0; mode < 5; ++mode) { + TestRecovery test; + test.checkpoint(std::string(8000, 'v')); + auto raw = read_frame(test, 0); + const CellView cell(LeafFrameView(raw.data(), raw.size()).cell(0)); + const uint64_t head = cell.overflow_head(); + if (mode == 0 || mode == 1) { + auto *encoded = raw.data() + first_cell(raw); + if (mode == 0) { + frame_put_u64(encoded, kCellHeaderSize, test.anchor.next_page_id); + } + else { + frame_put_u64(encoded, kCellHeaderSize + sizeof(uint64_t), cell.overflow_len() - 1); + } + write_frame(test, 0, std::move(raw)); + } + else { + raw = read_frame(test, head); + if (mode == 2) { + frame_put_u64(raw.data(), fh::kRightSibling, kInvalidPageId); + } + else if (mode == 3) { + frame_put_u64(raw.data(), fh::kRightSibling, head); + } + else { + frame_put_u32(raw.data(), fh::kSlotCount, 0); + } + write_frame(test, head, std::move(raw)); + } + expect_failure(test); + } +} + +TEST(LocalRecoveryFrames, DependencyStepsAndReferenceCountAreBounded) +{ + for (const bool limit_references : {false, true}) { + TestRecovery test; + test.checkpoint(std::string(8000, 'v')); + LocalRecoveryLimits limits; + if (limit_references) { + const auto raw = read_frame(test, 0); + const auto cell = LeafFrameView(raw.data(), raw.size()).cell(0); + std::vector<uint8_t> rewritten(raw.size()); + LeafFrameBuilder builder(rewritten.data(), rewritten.size()); + ASSERT_TRUE(builder.try_append_sorted("a", cell)); + ASSERT_TRUE(builder.try_append_sorted("b", cell)); + builder.finish(0, kInvalidPageId); + write_frame(test, 0, std::move(rewritten)); + limits.max_overflow_references = 1; + } + else { + limits.max_dependency_steps = 1; + } + expect_failure(test, Code::kResourceExhausted, limits); + } +} + +TEST(LocalRecoveryFrames, LeafBackingAloneKeepsItsPrimaryAndActualOverflowChainAfterRegistryEnds) +{ + TestRecovery test; + test.checkpoint(std::string(6000, 'o')); + test.checkpoint(std::string(12000, 'v')); + ASSERT_TRUE(test.prepare().ok()); + Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + auto ledger = test.operation->result().ledger; + auto pages = test.operation->result().latest; + ASSERT_TRUE(pages); + const auto count = pages->frames.size(); + auto leaf = pages->frames.at(0).backing; + std::vector<std::weak_ptr<const void>> overflow; + for (const auto &[pid, frame] : pages->frames) { + if (pid != 0) { + overflow.push_back(frame.backing->owner); + } + } + pages.reset(); + test.operation.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, count); + for (const auto &owner : overflow) { + EXPECT_FALSE(owner.expired()); + } + leaf.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, 0U); + for (const auto &owner : overflow) { + EXPECT_TRUE(owner.expired()); + } + overflow.clear(); + ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(LocalRecoveryFrames, RecoveredOverflowCanBeReusedAndDoesNotRetainItsOldPrimary) +{ + TestRecovery test; + test.checkpoint(std::string(12000, 'r')); + const auto raw = read_frame(test, 0); + ASSERT_TRUE(test.prepare().ok()); + Completion recovered; + test.run(&recovered); + ASSERT_EQ(recovered.values.load(), 1U) << recovered.status.to_string(); + auto inherited = test.operation->result().latest; + auto ledger = test.operation->result().ledger; + std::weak_ptr<const void> old_primary = inherited->frames.at(0).backing->owner; + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back( + {.identity = 1, + .max_slot = inherited->coverage, + .resident = std::shared_ptr<const LeafBase>(LeafBase::from_frame_copy(raw.data(), raw.size())), + .backing = inherited->frames.at(0).backing}); + auto captured = captured_leaf(leaf); + captured->next_page_id = inherited->next_page_id; + std::shared_ptr<CapturedWriteOperation> writer; + ASSERT_TRUE( + CapturedWriteOperation::create( + captured, {}, test.store, test.original.source.runtime, test.original.source.lane, + {.frames = {.encode = {.frame_bytes = 4096, .inline_value_bytes = 16}}, .write = {.max_frame_bytes = 4096}}, + test.credits.credits, [](uint64_t, ReservedOverflowIds *) { return Status::not_supported(); }, + extent_write_reservation(ledger, test.credits.credits.bytes), inherited, &writer) + .ok()); + Completion written; + auto connected = stdexec::connect(writer->sender(), Receiver{.done = &written}); + stdexec::start(connected); + test.original.source.runtime->drain(); + ASSERT_EQ(written.values.load(), 1U) << written.status.to_string(); + auto pages = writer->result().pages; + EXPECT_EQ(writer->result().written_frames, 1U); + EXPECT_EQ(writer->result().reused_frames, inherited->frames.size() - 1); + for (const auto &[pid, frame] : inherited->frames) { + if (pid != 0) { + const auto &adopted = pages->frames.at(pid).backing; + EXPECT_NE(adopted, frame.backing); + EXPECT_EQ(adopted->physical_owner, frame.backing->physical_owner); + EXPECT_EQ(adopted->address, frame.backing->address); + EXPECT_EQ(adopted->length, frame.backing->length); + } + } + writer.reset(); + inherited.reset(); + captured.reset(); + leaf.reset(); + test.operation.reset(); + EXPECT_TRUE(old_primary.expired()); + EXPECT_EQ(ledger->stats().protected_ranges, pages->frames.size()); + std::string restored; + const CellView cell(LeafFrameView(raw.data(), raw.size()).cell(0)); + for (auto pid = cell.overflow_head(); pid != kInvalidPageId;) { + const auto bytes = read_frame(test, pid); + const OverflowFrameView chunk(bytes.data(), bytes.size()); + restored += chunk.payload().to_string(); + pid = chunk.next_page_id(); + } + EXPECT_EQ(restored, std::string(12000, 'r')); + pages.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_recovery_lifetime_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_lifetime_test.cpp new file mode 100644 index 000000000..d15fb019b --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_recovery_lifetime_test.cpp @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "local_recovery_test_support.h" + +namespace crowdb::tree +{ +using namespace test_write; + +TEST(LocalRecovery, CompletedIoCallbackDoesNotRetainRecoveredCheckpointMetadata) +{ + TestRecovery test; + test.checkpoint(); + ASSERT_TRUE(test.prepare().ok()); + test.store->retain_completed = true; + Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + auto ledger = test.operation->result().ledger; + auto pages = test.operation->result().latest; + ASSERT_TRUE(ledger); + ASSERT_TRUE(pages); + ASSERT_TRUE(test.store->retained_completion.owner); + EXPECT_EQ(ledger->stats().protected_ranges, pages->frames.size() + 2); + test.operation.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, pages->frames.size()); + pages.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, 0U); + test.store->retained_completion = {}; + ledger.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_recovery_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_test.cpp new file mode 100644 index 000000000..dbea0d53b --- /dev/null +++ b/lib/crowdb-tree/tests/integration/local_recovery_test.cpp @@ -0,0 +1,299 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/maptable/mapping_persist.h" +#include "crowdb-tree/maptable/mapping_slot.h" +#include "local_recovery_test_support.h" + +namespace crowdb::tree +{ +using namespace test_write; + +namespace +{ +Status recover(TestRecovery &test, LocalRecoveryLimits limits = {}) +{ + auto status = test.prepare(limits); + if (!status.ok()) { + return status; + } + Completion done; + test.run(&done); + EXPECT_EQ(done.values.load() + done.errors.load(), 1U); + return test.operation->result().status; +} + +std::vector<DirEntry> read_directory(TestRecovery &test) +{ + std::vector<uint8_t> bytes(round_up_to_iu(test.anchor.segdir_len, 4096)); + EXPECT_TRUE(test.store->device->read_at(test.anchor.segdir_addr, bytes.data(), bytes.size()).ok()); + std::vector<DirEntry> entries; + EXPECT_TRUE(decode_segment_directory(bytes.data(), bytes.size(), &entries).ok()); + return entries; +} + +void write_directory(TestRecovery &test, const std::vector<DirEntry> &entries) +{ + std::vector<uint8_t> bytes; + encode_segment_directory(entries, &bytes); + test.anchor.segdir_len = static_cast<uint32_t>(bytes.size()); + test.anchor.segdir_crc = common::crc32c(bytes.data(), bytes.size()); + bytes.resize(round_up_to_iu(bytes.size(), 4096)); + ASSERT_TRUE(test.store->device->write_at(test.anchor.segdir_addr, bytes.data(), bytes.size()).ok()); + bytes.clear(); + snapshot::encode_anchor(test.anchor, 4096, &bytes); + ASSERT_TRUE( + test.store->device->write_at(test.anchor.snapshot_seq % 2 == 0 ? 4096 : 0, bytes.data(), bytes.size()).ok()); +} + +std::vector<uint64_t> read_segment(TestRecovery &test, const DirEntry &entry, SegmentImageHeader *header) +{ + std::vector<uint8_t> bytes(round_up_to_iu(entry.image_len, 4096)); + EXPECT_TRUE(test.store->device->read_at(entry.image_addr, bytes.data(), bytes.size()).ok()); + std::vector<uint64_t> words; + EXPECT_TRUE(decode_segment_image(bytes.data(), bytes.size(), header, &words).ok()); + return words; +} + +void write_segment(TestRecovery &test, DirEntry &entry, const SegmentImageHeader &header, + const std::vector<uint64_t> &words) +{ + std::vector<uint8_t> bytes; + encode_segment_image(header, words, &bytes, &entry.image_crc); + entry.image_len = static_cast<uint32_t>(bytes.size()); + bytes.resize(round_up_to_iu(bytes.size(), 4096)); + ASSERT_TRUE(test.store->device->write_at(entry.image_addr, bytes.data(), bytes.size()).ok()); +} + +void expect_corruption(TestRecovery &test) +{ + EXPECT_EQ(recover(test).code(), Code::kCorruption); + ASSERT_TRUE(test.operation); + EXPECT_FALSE(test.operation->result().latest); + EXPECT_FALSE(test.operation->result().registry); + test.operation.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace + +TEST(LocalRecovery, FreshDomainActivatesAnEmptyRegistryBeforeItsFirstAllocation) +{ + TestRecovery test; + ASSERT_TRUE(recover(test).ok()); + const auto &result = test.operation->result(); + ASSERT_TRUE(result.ledger); + ASSERT_TRUE(result.registry); + EXPECT_FALSE(result.latest); + EXPECT_FALSE(result.anchors[0]); + EXPECT_FALSE(result.anchors[1]); + EXPECT_EQ(result.registry->snapshot_sequence(), 0U); + EXPECT_EQ(result.ledger->stats().protected_ranges, 0U); + auto allocated = await_ledger( + [&](auto done) { return result.ledger->reserve(4096, test.credits.credits.bytes, std::move(done)); }); + ASSERT_TRUE(allocated.status.ok()); + EXPECT_EQ(allocated.extent.address, 8192U); + EXPECT_EQ(allocated.extent.generation, 2U); + EXPECT_EQ(result.ledger->stats().appended_allocations, 1U); +} + +TEST(LocalRecovery, ChoosesTheNewestAnchorAcrossParityRotationAndReleasesUnrelatedOwners) +{ + TestRecovery test; + test.checkpoint("first"); + test.checkpoint("second"); + test.checkpoint("third"); + ASSERT_TRUE(recover(test).ok()); + auto ledger = test.operation->result().ledger; + const auto &result = test.operation->result(); + ASSERT_TRUE(result.anchors[0]); + ASSERT_TRUE(result.anchors[1]); + EXPECT_EQ(result.anchors[0]->anchor.snapshot_seq, 3U); + EXPECT_EQ(result.anchors[1]->anchor.snapshot_seq, 2U); + EXPECT_EQ(result.latest, result.anchors[0]->pages); + EXPECT_EQ(result.registry->snapshot_sequence(), 3U); + EXPECT_EQ(ledger->stats().protected_ranges, 6U); + auto leaf = result.latest->frames.at(0).backing; + EXPECT_EQ(leaf->generation, 2U); + test.original.expect_read("third"); + test.operation.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, 1U); + leaf.reset(); + EXPECT_EQ(ledger->stats().protected_ranges, 0U); +} + +TEST(LocalRecovery, NewestSequenceWinsAfterAChangedGenerationResetsCoverage) +{ + TestRecovery test; + test.checkpoint("old generation"); + auto empty = detail::reserved_shared<CapturedDurablePages>(test.original.source.credits.credits.bytes, + test.original.source.credits.credits.bytes); + empty->generation = test.original.operation->result().checkpoint->pages->generation + 1; + ASSERT_TRUE(test.original.prepare(empty).ok()); + Completion cleared; + test.original.run(&cleared); + ASSERT_EQ(cleared.values.load(), 1U) << cleared.status.to_string(); + empty.reset(); + const auto status = recover(test); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto &result = test.operation->result(); + ASSERT_TRUE(result.anchors[0]); + ASSERT_TRUE(result.anchors[1]); + EXPECT_EQ(result.anchors[0]->anchor.last_applied_slot, 10U); + EXPECT_EQ(result.anchors[1]->anchor.last_applied_slot, 0U); + EXPECT_EQ(result.latest, result.anchors[1]->pages); + EXPECT_EQ(result.registry->snapshot_sequence(), 2U); + EXPECT_EQ(result.registry->durable_coverage(), 0U); + EXPECT_EQ(result.latest->root_page_id, kInvalidPageId); + EXPECT_TRUE(result.latest->frames.empty()); +} + +TEST(LocalRecovery, RecoveredRegistryCanCommitTheNextCheckpointAndAdvanceDurableCoverage) +{ + TestRecovery test; + test.checkpoint("reopened"); + ASSERT_TRUE(recover(test).ok()); + const auto &result = test.operation->result(); + auto pages = detail::reserved_shared<CapturedDurablePages>(test.credits.credits.bytes, *result.latest); + ++pages->coverage; + ++pages->generation; + std::shared_ptr<LocalCheckpointOperation> checkpoint; + ASSERT_TRUE(LocalCheckpointOperation::create(pages, result.registry, test.store, test.original.source.runtime, + test.original.source.lane, {}, test.credits.credits, &checkpoint) + .ok()); + Completion done; + auto connected = stdexec::connect(checkpoint->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.original.source.runtime->drain(); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(result.registry->snapshot_sequence(), 2U); + EXPECT_EQ(result.registry->durable_coverage(), 11U); + EXPECT_EQ(checkpoint->result().checkpoint->anchor.last_applied_slot, 11U); + EXPECT_EQ(checkpoint->result().checkpoint->pages->frames.at(0).backing, result.latest->frames.at(0).backing); + test.original.expect_read("reopened"); +} + +TEST(LocalRecovery, SegmentGenerationIsIndependentOfTheAnchorSequence) +{ + for (const uint64_t generation : {uint64_t{0}, uint64_t{77}}) { + TestRecovery test; + test.checkpoint(); + auto entries = read_directory(test); + ASSERT_EQ(entries.size(), 1U); + SegmentImageHeader header; + const auto words = read_segment(test, entries[0], &header); + header.generation = generation; + entries[0].generation = generation; + write_segment(test, entries[0], header, words); + write_directory(test, entries); + ASSERT_TRUE(recover(test).ok()); + EXPECT_EQ(test.operation->result().registry->snapshot_sequence(), 1U); + test.original.expect_read("v"); + } +} + +TEST(LocalRecovery, ValidAnchorWithCorruptDirectoryOrSegmentNeverFallsBackSilently) +{ + for (const bool older : {false, true}) { + for (const bool segment : {false, true}) { + TestRecovery test; + test.checkpoint("first"); + const auto first = test.anchor; + test.checkpoint("second"); + if (older) { + test.anchor = first; + } + const auto entries = read_directory(test); + ASSERT_EQ(entries.size(), 1U); + const auto address = segment ? entries[0].image_addr : test.anchor.segdir_addr; + std::array<uint8_t, 4096> bytes{}; + ASSERT_TRUE(test.store->device->read_at(address, bytes.data(), bytes.size()).ok()); + bytes[0] ^= 1; + ASSERT_TRUE(test.store->device->write_at(address, bytes.data(), bytes.size()).ok()); + expect_corruption(test); + } + } +} + +TEST(LocalRecovery, ChecksSegmentIdentityGenerationBodyCrcAndLiveCountAgainstDirectory) +{ + for (size_t mode = 0; mode < 4; ++mode) { + TestRecovery test; + test.checkpoint(); + auto entries = read_directory(test); + SegmentImageHeader header; + const auto words = read_segment(test, entries.at(0), &header); + if (mode == 0) { + ++header.seg_idx; + } + else if (mode == 1) { + ++header.generation; + } + else if (mode == 3) { + ++header.live_count; + } + write_segment(test, entries[0], header, words); + if (mode == 2) { + entries[0].image_crc ^= 1; + } + write_directory(test, entries); + expect_corruption(test); + } +} + +TEST(LocalRecovery, RejectsMappingExtentsThatOverlapProtectedMetadata) +{ + TestRecovery test; + test.checkpoint(); + auto entries = read_directory(test); + SegmentImageHeader header; + auto words = read_segment(test, entries.at(0), &header); + words[0] = slot_word::pack_unloaded((entries[0].image_addr + 4096) / 4096, 1); + write_segment(test, entries[0], header, words); + write_directory(test, entries); + expect_corruption(test); +} + +TEST(LocalRecovery, DeclaredSegmentAndFrameLimitsCannotSelectAnOlderSnapshot) +{ + for (const bool segments : {false, true}) { + TestRecovery test; + test.checkpoint(); + test.checkpoint(std::string(8000, 'v')); + LocalRecoveryLimits limits; + if (segments) { + auto entries = read_directory(test); + DirEntry extra{.seg_idx = 1, .generation = 1, .image_addr = test.store->device->size()}; + write_segment(test, extra, {.seg_idx = 1, .generation = 1, .slot_count = 1024}, + std::vector<uint64_t>(1024)); + entries.push_back(extra); + write_directory(test, entries); + limits.max_segments = 1; + } + else { + limits.max_frames = 1; + } + EXPECT_FALSE(recover(test, limits).ok()); + EXPECT_FALSE(test.operation->result().latest); + EXPECT_FALSE(test.operation->result().registry); + } +} + +TEST(LocalRecovery, UnreferencedCrashTailBecomesReusableAfterAllAnchorsAreProtected) +{ + TestRecovery test; + test.checkpoint(); + const auto orphan = test.store->device->size(); + const std::array<uint8_t, 4096> bytes{}; + ASSERT_TRUE(test.store->device->write_at(orphan, bytes.data(), bytes.size()).ok()); + ASSERT_TRUE(recover(test).ok()); + auto ledger = test.operation->result().ledger; + auto reused = + await_ledger([&](auto done) { return ledger->reserve(4096, test.credits.credits.bytes, std::move(done)); }); + ASSERT_TRUE(reused.status.ok()); + EXPECT_EQ(reused.extent.address, orphan); + EXPECT_EQ(ledger->stats().reused_allocations, 1U); + EXPECT_EQ(ledger->stats().appended_allocations, 0U); + test.original.expect_read("v"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/maintenance_cold_test.cpp b/lib/crowdb-tree/tests/integration/maintenance_cold_test.cpp new file mode 100644 index 000000000..159205f14 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/maintenance_cold_test.cpp @@ -0,0 +1,132 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_operation.h" +#include "published_read_test_support.h" + +#include <future> + +namespace crowdb::tree +{ +namespace +{ +struct TestColdMaintenance +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(64 * 1024 * 1024); + return result; + } + + Status publish(std::shared_ptr<const PublishedPage> expected, std::shared_ptr<const PublishedPage> page) + { + auto request = std::make_shared<PublicationRequest>(); + request->transaction = { + .record = std::make_shared<InstallRecord>(InstallToken{.generation = 1}), + .pages = {{.page_id = 0, .expected = std::move(expected), .replacement = std::move(page)}}, + .root_page_id = 0}; + std::promise<Status> completed; + auto terminal = completed.get_future(); + request->on_done = [&](PublicationResult result) { completed.set_value(std::move(result.status)); }; + return publisher.try_submit(request) ? terminal.get() : Status::unavailable(); + } + + void prepare() + { + auto leaf = std::make_shared<PageState>(*test_read::page({ + {.key = "a", .value = std::string(80, 'a')}, + {.key = "c", .value = std::string(80, 'c')} + })); + leaf->bases.push_back(test_read::cold(1)); + ASSERT_TRUE(publish({}, test_read::published(std::move(leaf))).ok()); + loader->deferred = true; + loader->frames.emplace(1, test_read::base({ + {.key = "b", .value = std::string(80, 'b')} + })); + PageMaintenanceOperationOptions options; + options.max_steps = 1; + options.max_stale_replans = 0; + options.publication.pages.primary_page_bytes = 512; + options.publication.pages.target_page_bytes = 384; + options.publication.pages.merge_trigger_bytes = 64; + options.publication.pages.merge_target_bytes = 128; + options.publication.pages.max_work_entries = 8; + options.acquire = [this](auto *out) { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, 16 * 1024 * 1024); + if (!credit) { + return Status::resource_exhausted(); + } + *out = detail::AllocationReservation::create(std::move(*credit)); + return Status::Ok(); + }; + maintenance = make_page_maintenance(publisher, runtime, lane, std::move(options), loader, {}); + } + + void begin() + { + done.store(false); + maintenance(publisher.capture(), [this](Status result, uint64_t) { + status = std::move(result); + done.store(true, std::memory_order_release); + }); + runtime.drain(); + } + + detail::ResourceBudget budget{limits(), 2}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + PublicationCoordinator publisher; + detail::FlushRuntime runtime{2, 8}; + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<test_read::TestLoader> loader = std::make_shared<test_read::TestLoader>(); + FlushBatch::Maintenance maintenance; + std::atomic<bool> done{false}; + Status status; +}; + +TEST(ColdPageMaintenance, StaleLoadedViewDoesNotReplaceTheConcurrentWriter) +{ + TestColdMaintenance test; + test.prepare(); + test.begin(); + ASSERT_FALSE(test.done.load(std::memory_order_acquire)); + ASSERT_EQ(test.loader->pending.size(), 1U); + const auto before = test.publisher.capture()->mapping.get(0); + auto changed = std::make_shared<PublishedPage>(*before); + changed->revision++; + changed->leaf = test_read::page({ + {.key = "new", .slot = 2, .value = "concurrent"} + }); + ASSERT_TRUE(test.publish(before, changed).ok()); + test.loader->finish(); + test.runtime.drain(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + ASSERT_TRUE(test.status.ok()) << test.status.to_string(); + EXPECT_EQ(test.publisher.capture()->mapping.get(0), changed); + EXPECT_EQ(test.loader->calls, 1U); +} + +TEST(ColdPageMaintenance, FailedReadRetainsTheOriginalCandidateAndNextPassRetries) +{ + TestColdMaintenance test; + test.prepare(); + const auto before = test.publisher.capture(); + test.begin(); + ASSERT_FALSE(test.done.load(std::memory_order_acquire)); + ASSERT_EQ(test.loader->pending.size(), 1U); + test.loader->finish(Status::io_error("injected cold read failure")); + test.runtime.drain(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + EXPECT_EQ(test.status.code(), Code::kIoError); + EXPECT_EQ(test.publisher.capture(), before); + test.begin(); + ASSERT_FALSE(test.done.load(std::memory_order_acquire)); + ASSERT_EQ(test.loader->pending.size(), 1U); + test.loader->finish(); + test.runtime.drain(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + ASSERT_TRUE(test.status.ok()) << test.status.to_string(); + EXPECT_GT(test.publisher.capture()->stats.leaf_pages, 1U); + EXPECT_EQ(test.loader->calls, 2U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/maintenance_operation_test.cpp b/lib/crowdb-tree/tests/integration/maintenance_operation_test.cpp new file mode 100644 index 000000000..ba3035de6 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/maintenance_operation_test.cpp @@ -0,0 +1,641 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_operation.h" +#include "btree/flush_reservation_internal.h" +#include "btree/maintenance_operation.h" + +#include <gtest/gtest.h> + +#include <algorithm> +#include <future> + +namespace crowdb::tree +{ +namespace +{ +struct MaintenanceCredits +{ + MaintenanceCredits() + { + account->set_resource_metrics(resources); + } + + std::shared_ptr<TreeResourceMetrics> resources = std::make_shared<TreeResourceMetrics>(); + + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(64 * 1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + size_t acquired = 0; + + Status reserve(std::shared_ptr<detail::AllocationReservation> *out, + detail::BudgetClass category = detail::BudgetClass::kReclaim) + { + auto credit = detail::ResourceBudget::try_reserve(account, category, 4 * 1024 * 1024); + if (!credit) { + return Status::resource_exhausted(); + } + *out = std::make_shared<detail::AllocationReservation>(std::move(*credit)); + ++acquired; + return Status::Ok(); + } + + PageMaintenanceOperationOptions options() + { + PageMaintenanceOperationOptions result; + result.publication.pages.primary_page_bytes = 512; + result.publication.pages.target_page_bytes = 256; + result.publication.pages.merge_trigger_bytes = 64; + result.publication.pages.merge_target_bytes = 128; + result.publication.pages.max_work_entries = 4; + result.acquire = [this](auto *out) { return reserve(out); }; + return result; + } +}; + +Status publish(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<Status> completed; + auto terminal = completed.get_future(); + request->on_done = [&](PublicationResult result) { completed.set_value(std::move(result.status)); }; + if (!publisher.try_submit(request)) { + return Status::unavailable(); + } + return terminal.get(); +} + +std::shared_ptr<InstallRecord> record() +{ + return std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 0, .range = 0, .sequence = 0}); +} + +std::shared_ptr<PublishedPage> leaf(size_t count, size_t start = 0) +{ + OverlayBuilder builder({}); + for (size_t i = start; i < start + count; ++i) { + auto cell = std::make_shared<CellVersion>(encode_cell_buf(1, OpKind::kPut, std::string(16, 'v')), 1, 0); + EXPECT_TRUE(builder.append(std::to_string(1000 + i), std::move(cell)).ok()); + } + std::shared_ptr<const OverlayBatch> run; + EXPECT_TRUE(builder.finish(1, &run).ok()); + auto logical = std::make_shared<PageState>(); + logical->overlays.push_back(std::move(run)); + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::move(logical); + page->metrics.dirty = true; + page->metrics.overlay_runs = 1; + page->metrics.overlay_entries = count; + return page; +} + +std::pair<Status, uint64_t> maintain(FlushBatch::Maintenance &callback, PublicationCoordinator &publisher) +{ + std::promise<std::pair<Status, uint64_t>> completed; + auto terminal = completed.get_future(); + callback(publisher.capture(), + [&](Status status, uint64_t deferred) { completed.set_value({std::move(status), deferred}); }); + return terminal.get(); +} + +std::shared_ptr<const PublishedPage> route(const PublishedTreeState &view, const std::string &key) +{ + auto page = view.mapping.get(view.root_page_id); + for (size_t depth = 0; depth < 64 && page && page->inner; ++depth) { + auto child = + static_cast<size_t>(std::upper_bound(page->inner->separators.begin(), page->inner->separators.end(), key) - + page->inner->separators.begin()); + page = view.mapping.get(page->inner->children[child]); + } + return page; +} + +TEST(PageMaintenanceOperation, FreshFactoriesContinuePastThePreviousBoundedWindow) +{ + PublicationCoordinator publisher; + auto left = leaf(1); + auto right = leaf(1, 1); + auto left_state = std::make_shared<PageState>(*left->leaf); + auto right_state = std::make_shared<PageState>(*right->leaf); + left_state->range.upper = OwnedKey("1001"); + left_state->right_sibling = 1; + right_state->range.lower = OwnedKey("1001"); + left->leaf = left_state; + right->leaf = right_state; + auto inner = std::make_shared<InnerPageState>(); + inner->children = {0, 1}; + inner->separators.emplace_back("1001"); + auto root = std::make_shared<PublishedPage>(); + root->level = 1; + root->inner = inner; + root->metrics.leaf = false; + ASSERT_TRUE(publish(publisher, + { + .record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = left}, + {.page_id = 1, .expected = {}, .replacement = right}, + {.page_id = 2, .expected = {}, .replacement = root}}, + .root_page_id = 2, + .expected_next_page_id = 0, + .next_page_id = 3 + }) + .ok()); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + for (uint64_t pid = 0; pid < 2; ++pid) { + auto options = credits.options(); + options.max_visits = options.max_candidates = options.max_steps = 1; + auto fresh = make_page_maintenance(publisher, runtime, lane, std::move(options)); + const auto result = maintain(fresh, publisher); + ASSERT_TRUE(result.first.ok()) << result.first.to_string(); + runtime.drain(); + EXPECT_TRUE(publisher.capture()->mapping.get(pid)->metrics.page_size_known) << pid; + } +} + +TEST(PageMaintenanceOperation, QueueObservationEndsAtDispatchAndTerminalReleasesAllWork) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = leaf(1)}}, + .root_page_id = 0, + .expected_next_page_id = 0, + .next_page_id = 1}) + .ok()); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto maintenance = make_page_maintenance(publisher, runtime, lane, credits.options()); + + struct Blocker + { + std::promise<void> entered; + std::promise<void> release; + + static void run(void *raw) noexcept + { + auto &self = *static_cast<Blocker *>(raw); + self.entered.set_value(); + self.release.get_future().wait(); + } + } blocker; + + ASSERT_TRUE(runtime.try_submit(lane, &Blocker::run, &blocker).ok()); + blocker.entered.get_future().wait(); + std::promise<Status> done; + auto finished = done.get_future(); + maintenance(publisher.capture(), [&](Status status, uint64_t) { done.set_value(std::move(status)); }); + const auto queued = credits.resources->snapshot(); + EXPECT_EQ(queued.maintenance_queued, 1U); + EXPECT_EQ(queued.maintenance_inflight, 0U); + EXPECT_GT(publisher.oldest_maintenance_since_ns(), 0U); + EXPECT_GT(publisher.stats(*credits.resources).progress.maintenance_oldest_age_ns, 0U); + blocker.release.set_value(); + EXPECT_TRUE(finished.get().ok()); + runtime.drain(); + const auto ended = credits.resources->snapshot(); + EXPECT_EQ(ended.maintenance_queued, 0U); + EXPECT_EQ(ended.maintenance_inflight, 0U); + EXPECT_EQ(publisher.oldest_maintenance_since_ns(), 0U); + EXPECT_EQ(publisher.stats(*credits.resources).progress.maintenance_oldest_age_ns, 0U); +} + +TEST(PageMaintenanceOperation, SingleStepPublishesBulkRangesWithinAdvertisedCreditWithoutEncoding) +{ + PublicationCoordinator publisher; + const auto original = leaf(12); + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = original}}, + .root_page_id = 0, + .expected_next_page_id = 0, + .next_page_id = 1}) + .ok()); + const auto before = publisher.capture(); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto options = credits.options(); + options.max_steps = 1; + options.publication.pages.max_work_entries = 16; + options.publication.pages.max_output_pages = 4; + options.publication.pages.max_metadata_bytes = 1024 * 1024; + const auto bound = page_maintenance_reserve_bound(before->mapping, 0, options); + if (!bound) { + FAIL() << "maintenance reserve bound must be representable"; + } + ASSERT_LE(*bound, MaintenanceCredits::limits()[static_cast<size_t>(detail::BudgetClass::kReclaim)]); + options.acquire = [&](auto *out) { + auto credit = detail::ResourceBudget::try_reserve(credits.account, detail::BudgetClass::kReclaim, *bound); + if (!credit) { + return Status::resource_exhausted(); + } + *out = detail::AllocationReservation::create(std::move(*credit)); + return Status::Ok(); + }; + auto maintenance = make_page_maintenance(publisher, runtime, lane, std::move(options)); + const auto result = maintain(maintenance, publisher); + ASSERT_TRUE(result.first.ok()) << result.first.to_string(); + runtime.drain(); + const auto after = publisher.capture(); + EXPECT_EQ(after->commit_seq, before->commit_seq + 1); + EXPECT_EQ(after->stats.leaf_pages, 4U); + EXPECT_EQ(after->stats.overlay_entries, 12U); + for (size_t i = 0; i < 12; ++i) { + const auto key = std::to_string(1000 + i); + const auto page = route(*after, key); + ASSERT_TRUE(page && page->leaf); + EXPECT_EQ(page->leaf->lookup(key).cell.owner, original->leaf->lookup(key).cell.owner); + } + const auto totals = publisher.work_metrics()->totals(); + EXPECT_EQ(totals[static_cast<size_t>(TreeStage::PageLayout)].count, 1U); + EXPECT_EQ(totals[static_cast<size_t>(TreeStage::PageSplit)].count, 1U); + EXPECT_EQ(totals[static_cast<size_t>(TreeStage::PageEncode)].count, 0U); + EXPECT_EQ(before->mapping.get(0), original); + EXPECT_EQ(before->stats.leaf_pages, 1U); +} + +TEST(PageMaintenanceOperation, IdleAdmissionAppliesItsNewWindowBeforeBuildingTheSplit) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = leaf(64)}}, + .root_page_id = 0, + .expected_next_page_id = 0, + .next_page_id = 1}) + .ok()); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto options = credits.options(); + options.max_steps = 1; + options.publication.pages.max_work_entries = 128; + options.publication.pages.max_output_pages = 7; + options.publication.pages.max_metadata_bytes = 1024 * 1024; + detail::ReclaimAdmission admission{.publisher = &publisher, .account = credits.account, .options = options}; + options.publication.pages.max_output_pages = 2; + options.acquire_window = [&](const auto &mapping, auto *publication, auto *credit) { + return admission.acquire_window(mapping, publication, credit); + }; + auto maintenance = make_page_maintenance(publisher, runtime, lane, std::move(options)); + const auto result = maintain(maintenance, publisher); + ASSERT_TRUE(result.first.ok()) << result.first.to_string(); + runtime.drain(); + EXPECT_EQ(publisher.capture()->stats.leaf_pages, 7U); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 64U); + EXPECT_EQ(credits.acquired, 0U); +} + +TEST(PageMaintenanceOperation, FullTreeMaintenanceWindowDefersBeforeQueueingAndCanRetry) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = leaf(1)}}, + .root_page_id = 0, + .expected_next_page_id = 0, + .next_page_id = 1}) + .ok()); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto maintenance = make_page_maintenance(publisher, runtime, lane, credits.options()); + const auto before = publisher.capture(); + std::array<size_t, PublicationCoordinator::kMaintenanceSlots> slots{}; + for (auto &slot : slots) { + slot = publisher.start_maintenance(tree_metrics_now_ns()); + ASSERT_LT(slot, slots.size()); + } + bool deferred = false; + maintenance(before, [&](Status status, uint64_t dirty) { + EXPECT_TRUE(status.ok()); + EXPECT_EQ(dirty, before->stats.dirty_pages); + deferred = true; + }); + EXPECT_TRUE(deferred); + EXPECT_EQ(publisher.capture(), before); + EXPECT_EQ(credits.resources->snapshot().maintenance_queued, 0U); + for (const auto slot : slots) { + publisher.finish_maintenance(slot); + } + std::promise<Status> done; + auto result = done.get_future(); + maintenance(publisher.capture(), [&](Status status, uint64_t) { done.set_value(std::move(status)); }); + EXPECT_TRUE(result.get().ok()); + runtime.drain(); + EXPECT_EQ(publisher.oldest_maintenance_since_ns(), 0U); +} + +TEST(PageMaintenanceOperation, SingleWorkerFlushSplitsClipsRunsAndPreservesOldView) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + auto initial = std::make_shared<PublishedPage>(); + initial->leaf = std::make_shared<PageState>(); + auto catalog = std::make_shared<PublishedSources>(); + const auto source_view = sources.catalog(); + catalog->tables.assign(source_view->tables.begin(), source_view->tables.end()); + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = initial}}, + .root_page_id = 0, + .expected_sources = publisher.capture()->sources, + .sources = catalog}) + .ok()); + const auto before = publisher.capture(); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto callback = make_page_maintenance(publisher, runtime, lane, credits.options()); + std::atomic<uint64_t> contiguous{1}; + FlushBatchOptions flush_options; + flush_options.run_limits.max_entries = 2; + FlushController controller(publisher, runtime, sources, contiguous, lane, flush_options, callback); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + for (size_t i = 0; i < 12; ++i) { + ASSERT_TRUE(writer.table()->upsert(std::to_string(1000 + i), 1, + encode_cell_buf(1, OpKind::kPut, std::string(16, 'v')))); + } + writer.release(); + std::shared_ptr<detail::AllocationReservation> metadata; + ASSERT_TRUE(credits.reserve(&metadata, detail::BudgetClass::kHandoff).ok()); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(controller.prepare(metadata, &operation).ok()); + ASSERT_TRUE(stdexec::sync_wait(operation->sender()).has_value()); + runtime.drain(); + ASSERT_TRUE(operation->result().handoff_status.ok()); + ASSERT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.message(); + const auto after = publisher.capture(); + EXPECT_GT(after->stats.leaf_pages, 1U); + EXPECT_EQ(operation->result().deferred_pages, after->stats.dirty_pages); + EXPECT_GT(operation->result().deferred_pages, 0U); + for (size_t i = 0; i < 12; ++i) { + const auto key = std::to_string(1000 + i); + auto page = route(*after, key); + ASSERT_TRUE(page && page->leaf); + EXPECT_EQ(page->leaf->lookup(key).cell.value.to_string(), std::string(16, 'v')); + for (const auto &run : page->leaf->overlays) { + for (const auto &entry : run->entries()) { + EXPECT_TRUE(page->leaf->range.contains(entry.key())); + } + } + } + EXPECT_EQ(before->mapping.get(0), initial); + EXPECT_EQ(before->mapping.get(0)->leaf->lookup("1000").state, LogicalLookupState::kMissing); +} + +TEST(PageMaintenanceOperation, IdleMaintenanceMergesExactSiblingsAndKeepsOldRoot) +{ + PublicationCoordinator publisher; + auto left = leaf(1); + auto right = leaf(1, 1); + auto left_logical = std::make_shared<PageState>(*left->leaf); + left_logical->range.upper = "1001"; + left_logical->right_sibling = 1; + left->leaf = left_logical; + auto right_logical = std::make_shared<PageState>(*right->leaf); + right_logical->range.lower = "1001"; + right->leaf = right_logical; + auto inner = std::make_shared<InnerPageState>(); + inner->children = {0, 1}; + inner->separators = {"1001"}; + auto parent = std::make_shared<PublishedPage>(); + parent->level = 1; + parent->inner = inner; + parent->metrics.leaf = false; + ASSERT_TRUE(publish(publisher, + { + .record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = left}, + {.page_id = 1, .expected = {}, .replacement = right}, + {.page_id = 2, .expected = {}, .replacement = parent}}, + .root_page_id = 2 + }) + .ok()); + const auto before = publisher.capture(); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto options = credits.options(); + options.publication.pages.target_page_bytes = 400; + options.publication.pages.merge_trigger_bytes = 256; + options.publication.pages.merge_target_bytes = 350; + auto callback = make_page_maintenance(publisher, runtime, lane, options); + EXPECT_TRUE(maintain(callback, publisher).first.ok()); + runtime.drain(); + const auto after = publisher.capture(); + EXPECT_EQ(after->stats.leaf_pages, 1U); + EXPECT_EQ(after->root_page_id, 0U); + EXPECT_EQ(after->mapping.get(0)->leaf->lookup("1001").cell.owner, right->leaf->lookup("1001").cell.owner); + EXPECT_EQ(before->mapping.get(2), parent); + EXPECT_EQ(before->mapping.get(1), right); + const auto counters = publisher.work_metrics()->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Flush)]; + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::MergeOperations)], 1U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::MergeInputPages)], 2U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::MergeOutputPages)], 1U); +} + +TEST(PageMaintenanceOperation, CommittedCoalesceCountsInputOutputWhileOldReadersKeepPayload) +{ + PublicationCoordinator publisher; + auto original = leaf(2); + auto logical = std::make_shared<PageState>(*original->leaf); + logical->overlays.push_back(logical->overlays.front()); + original->leaf = logical; + original->metrics.overlay_runs = 2; + original->metrics.overlay_entries = 4; + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = original}}, + .root_page_id = 0}) + .ok()); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto callback = make_page_maintenance(publisher, runtime, lane, credits.options()); + const auto status = maintain(callback, publisher).first; + ASSERT_TRUE(status.ok()) << status.to_string(); + runtime.drain(); + auto counters = publisher.work_metrics()->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Flush)]; + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::CoalesceOperations)], 1U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::CoalesceInputRuns)], 2U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::CoalesceOutputRuns)], 1U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::CoalesceInputEntries)], 4U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::CoalesceOutputEntries)], 2U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::PayloadReferencesDropped)], 2U); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("1000").cell.owner, + original->leaf->lookup("1000").cell.owner); + ASSERT_TRUE(maintain(callback, publisher).first.ok()); + runtime.drain(); + EXPECT_EQ(publisher.work_metrics()->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Flush)] + [static_cast<size_t>(TreeWorkCounter::CoalesceOperations)], + 1U); +} + +TEST(PageMaintenanceOperation, SmallStepBudgetMakesProgressAcrossIdleInvocations) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = leaf(16)}}, + .root_page_id = 0}) + .ok()); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto options = credits.options(); + options.max_steps = 1; + options.publication.pages.max_work_entries = 2; + auto callback = make_page_maintenance(publisher, runtime, lane, options); + EXPECT_TRUE(maintain(callback, publisher).first.ok()); + runtime.drain(); + EXPECT_GT(publisher.capture()->stats.page_size_unknown_pages, 0U); + const auto first_pages = publisher.capture()->stats.leaf_pages; + for (size_t i = 0; i < 40; ++i) { + ASSERT_TRUE(maintain(callback, publisher).first.ok()); + } + runtime.drain(); + const auto after = publisher.capture(); + EXPECT_GT(after->stats.leaf_pages, first_pages); + EXPECT_EQ(after->stats.page_size_unknown_pages, 0U); + EXPECT_EQ(after->stats.kv_known_count, 16U); + EXPECT_EQ(credits.acquired, 41U); + for (size_t i = 0; i < 16; ++i) { + const auto key = std::to_string(1000 + i); + EXPECT_EQ(route(*after, key)->leaf->lookup(key).state, LogicalLookupState::kValue); + } +} + +TEST(PageMaintenanceOperation, ColdBaseDefersWithoutChangingTheLogicalPage) +{ + PublicationCoordinator publisher; + auto page = leaf(1); + auto logical = std::make_shared<PageState>(*page->leaf); + auto backing = std::make_shared<BackingRef>(); + backing->length = 4096; + backing->owner = std::make_shared<int>(1); + logical->bases.push_back({.identity = 1, .backing = backing}); + page->leaf = logical; + ASSERT_TRUE( + publish(publisher, + {.record = record(), .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, .root_page_id = 0}) + .ok()); + const auto before = publisher.capture(); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto callback = make_page_maintenance(publisher, runtime, lane, credits.options()); + auto result = maintain(callback, publisher); + runtime.drain(); + EXPECT_TRUE(result.first.ok()); + EXPECT_EQ(result.second, 1U); + EXPECT_EQ(publisher.capture(), before); + EXPECT_FALSE(logical->bases.front().resident); +} + +TEST(PageMaintenanceOperation, EmptyTreeAndCorruptRoutingReturnErrors) +{ + PublicationCoordinator publisher; + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto callback = make_page_maintenance(publisher, runtime, lane, credits.options()); + EXPECT_EQ(maintain(callback, publisher).first.code(), Code::kInvalidArgument); + auto invalid = std::make_shared<PublishedPage>(); + invalid->level = 1; + invalid->inner = std::make_shared<InnerPageState>(); + invalid->metrics.leaf = false; + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = invalid}}, + .root_page_id = 0}) + .ok()); + EXPECT_EQ(maintain(callback, publisher).first.code(), Code::kCorruption); + runtime.drain(); +} + +TEST(PageMaintenanceOperation, PrepaidCreditRunsWithNoUnreservedCapacityAndLaterInvocationReacquires) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = leaf(4)}}, + .root_page_id = 0}) + .ok()); + const auto before = publisher.capture()->commit_seq; + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto options = credits.options(); + ASSERT_TRUE(credits.reserve(&options.prepaid).ok()); + size_t acquire_calls = 0; + options.acquire = [&](auto *) { + ++acquire_calls; + return Status::unavailable(); + }; + auto callback = make_page_maintenance(publisher, runtime, lane, std::move(options)); + constexpr auto category = static_cast<size_t>(detail::BudgetClass::kReclaim); + const auto snapshot = credits.budget.snapshot(); + const auto remaining = + MaintenanceCredits::limits()[category] - snapshot.allocated[category] - snapshot.reserved[category]; + auto blocker = detail::ResourceBudget::try_reserve(credits.account, detail::BudgetClass::kReclaim, remaining); + ASSERT_TRUE(blocker); + auto result = maintain(callback, publisher); + runtime.drain(); + EXPECT_TRUE(result.first.ok()) << result.first.to_string(); + EXPECT_GT(publisher.capture()->commit_seq, before); + EXPECT_EQ(acquire_calls, 0U); + EXPECT_EQ(credits.budget.snapshot().reserved[category], remaining); + EXPECT_TRUE(maintain(callback, publisher).first.ok()); + runtime.drain(); + EXPECT_EQ(acquire_calls, 1U); +} + +TEST(PageMaintenanceOperation, PrepaidCreditRejectsHandoffPartitionBeforeConsumption) +{ + PublicationCoordinator publisher; + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto options = credits.options(); + ASSERT_TRUE(credits.reserve(&options.prepaid, detail::BudgetClass::kHandoff).ok()); + const auto available = options.prepaid->available(); + EXPECT_THROW((void)make_page_maintenance(publisher, runtime, lane, options), std::invalid_argument); + EXPECT_EQ(options.prepaid->available(), available); +} + +TEST(PageMaintenanceOperation, FailedAcquisitionReturnsTerminalAndLeavesFactoryAvailableForRetry) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(publish(publisher, {.record = record(), + .pages = {{.page_id = 0, .expected = {}, .replacement = leaf(4)}}, + .root_page_id = 0}) + .ok()); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + MaintenanceCredits credits; + auto options = credits.options(); + size_t attempts = 0; + options.acquire = [&](auto *out) -> Status { + ++attempts; + if (attempts == 1) { + throw std::bad_alloc(); + } + if (attempts == 2) { + throw std::runtime_error("injected admission failure"); + } + return credits.reserve(out); + }; + auto callback = make_page_maintenance(publisher, runtime, lane, std::move(options)); + const auto before = publisher.capture(); + EXPECT_TRUE(maintain(callback, publisher).first.ok()); + EXPECT_EQ(maintain(callback, publisher).first.code(), Code::kInternal); + EXPECT_EQ(publisher.capture(), before); + EXPECT_TRUE(maintain(callback, publisher).first.ok()); + runtime.drain(); + EXPECT_EQ(attempts, 3U); + EXPECT_GT(publisher.capture()->commit_seq, before->commit_seq); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/memory_backing_gc_test.cpp b/lib/crowdb-tree/tests/integration/memory_backing_gc_test.cpp new file mode 100644 index 000000000..fe1413acc --- /dev/null +++ b/lib/crowdb-tree/tests/integration/memory_backing_gc_test.cpp @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/local_backend_state.h" +#include "backend/local/local_block_gc.h" +#include "btree/tree_core.h" +#include "btree/tree_core_access.h" +#include "crowdb-tree/crowdb-tree.h" +#include "extent_ledger_test_support.h" + +#include <array> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; +constexpr uint64_t kPage = MemPageStore::kBackingPageBytes; + +TEST(MemoryBackingGc, PartialPagesAndReadersRemainProtectedUntilTheirLastOwnerDrops) +{ + MemPageStore store; + const std::vector<uint8_t> input(3 * kPage, 42); + ASSERT_TRUE(store.write_at(0, input.data(), input.size()).ok()); + LedgerFixture ledger({.backend_id = 1, .file_size = input.size(), .address_limit = input.size()}); + auto anchor = ledger.recover(kPage + 4096, 4096); + ASSERT_TRUE(anchor.status.ok()); + auto reader = anchor.extent.owner; + ASSERT_TRUE(ledger.activate().status.ok()); + const LocalBlockGcLimits limits{.max_blocks = 1, .max_deleted_bytes = kPage}; + auto first = collect_free_local_blocks(store, *ledger.ledger, 0, limits, ledger.credits.credits.bytes); + ASSERT_TRUE(first.status.ok()); + EXPECT_EQ(first.protected_blocks, 1U); + EXPECT_FALSE(first.complete); + anchor = {}; + auto second = + collect_free_local_blocks(store, *ledger.ledger, first.next_block, limits, ledger.credits.credits.bytes); + ASSERT_TRUE(second.status.ok()); + EXPECT_TRUE(second.complete); + EXPECT_EQ(second.deleted_bytes, kPage); + EXPECT_EQ(store.resident_bytes(), 2 * kPage); + uint8_t value = 0; + ASSERT_TRUE(store.read_at(kPage + 4096, &value, 1).ok()); + EXPECT_EQ(value, 42); + reader.reset(); + auto third = collect_free_local_blocks(store, *ledger.ledger, 1, limits, ledger.credits.credits.bytes); + ASSERT_TRUE(third.status.ok()); + EXPECT_EQ(third.deleted_blocks, 1U); + EXPECT_EQ(store.resident_bytes(), kPage); + EXPECT_EQ(store.size(), input.size()); + ASSERT_TRUE(store.read_at(kPage + 4096, &value, 1).ok()); + EXPECT_EQ(value, 0); + ASSERT_TRUE(store.read_at(0, &value, 1).ok()); + EXPECT_EQ(value, 42); +} + +TEST(MemoryBackingGc, PendingWritesAndExclusiveDeletionLeasesPreventAddressReuse) +{ + MemPageStore store; + LedgerFixture ledger({.backend_id = 1, .region_begin = kPage, .file_size = 2 * kPage, .address_limit = 2 * kPage}); + const uint8_t value = 71; + ASSERT_TRUE(store.write_at((2 * kPage) - 1, &value, 1).ok()); + ASSERT_TRUE(ledger.activate().status.ok()); + auto pending = ledger.reserve(kPage); + ASSERT_TRUE(pending.status.ok()); + auto held = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.bytes); + ASSERT_TRUE(held.status.ok()); + EXPECT_EQ(held.protected_blocks, 1U); + EXPECT_EQ(held.deleted_bytes, 0U); + pending = {}; + auto deletion = await_ledger([&](auto complete) { + return ledger.ledger->claim_free(kPage, kPage, ledger.credits.credits.bytes, std::move(complete)); + }); + ASSERT_TRUE(deletion.status.ok()); + EXPECT_EQ(ledger.reserve(kPage).status.code(), Code::kResourceExhausted); + auto concurrent = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.bytes); + ASSERT_TRUE(concurrent.status.ok()); + EXPECT_EQ(concurrent.deleted_bytes, 0U); + deletion = {}; + auto released = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.bytes); + ASSERT_TRUE(released.status.ok()); + EXPECT_EQ(released.deleted_bytes, kPage); + EXPECT_EQ(store.resident_bytes(), 0U); + auto reuse = ledger.reserve(kPage); + ASSERT_TRUE(reuse.status.ok()); + EXPECT_EQ(reuse.extent.address, kPage); + ASSERT_TRUE(store.write_at(reuse.extent.address, &value, 1).ok()); + uint8_t actual = 0; + ASSERT_TRUE(store.read_at(reuse.extent.address, &actual, 1).ok()); + EXPECT_EQ(actual, value); +} + +TEST(MemoryBackingGc, FailedMetadataAdmissionAndPartialTailDoNotReleaseBacking) +{ + MemPageStore store; + const std::vector<uint8_t> input((2 * kPage) + 4096, 19); + ASSERT_TRUE(store.write_at(0, input.data(), input.size()).ok()); + LedgerFixture ledger({.backend_id = 1, .file_size = input.size(), .address_limit = input.size()}); + ASSERT_TRUE(ledger.activate().status.ok()); + ledger.credits.seal(); + auto failed = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.bytes); + EXPECT_EQ(failed.status.code(), Code::kResourceExhausted); + EXPECT_EQ(failed.next_block, 1U); + EXPECT_EQ(store.resident_bytes(), 3 * kPage); + auto collected = collect_free_local_blocks(store, *ledger.ledger, 0, {}, {}); + ASSERT_TRUE(collected.status.ok()); + EXPECT_TRUE(collected.complete); + EXPECT_EQ(collected.deleted_blocks, 1U); + EXPECT_EQ(store.resident_bytes(), 2 * kPage); + uint8_t tail = 0; + ASSERT_TRUE(store.read_at(2 * kPage, &tail, 1).ok()); + EXPECT_EQ(tail, 19); +} + +TEST(MemoryBackingGc, OnePublicSnapshotReclaimsMultipleWindowsWithoutRewritingTheTree) +{ + auto store = std::make_shared<MemPageStore>(4096); + const std::vector<uint8_t> orphan(160 * kPage, 37); + ASSERT_TRUE(store->write_at(0, orphan.data(), orphan.size()).ok()); + // Anchors must start absent; only the remaining written backing is orphaned. + const std::array<uint8_t, 8192> anchors{}; + ASSERT_TRUE(store->write_at(0, anchors.data(), anchors.size()).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = store.get(); + config.frame_bytes = 4096; + std::unique_ptr<Crowdbtree> tree; + ASSERT_TRUE(Crowdbtree::open(config, &tree).ok()); + ASSERT_TRUE(tree->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "value"}}}).ok()); + uint64_t coverage = 0; + const auto status = tree->snapshot(&coverage); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 1U); + auto *core = TreeCoreAccess::get(*tree); + core->operation_lifetime().drain_accepted(); + core->binding()->drain(); + const auto sequence = core->local_backend()->checkpoints->snapshot_sequence(); + EXPECT_EQ(sequence, 1U); + EXPECT_LT(store->resident_bytes(), 3 * kPage); + EXPECT_GE(store->size(), orphan.size()); + tree.reset(); + ASSERT_TRUE(Crowdbtree::open(config, &tree).ok()); + GetView actual; + ASSERT_TRUE(TreeCoreAccess::get(*tree)->get("a", &actual).ok()); + EXPECT_EQ(actual.value().to_string(), "value"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/memtable_failure_test.cpp b/lib/crowdb-tree/tests/integration/memtable_failure_test.cpp index 81ca59339..ee3443d03 100644 --- a/lib/crowdb-tree/tests/integration/memtable_failure_test.cpp +++ b/lib/crowdb-tree/tests/integration/memtable_failure_test.cpp @@ -27,7 +27,7 @@ Batch values(const std::string &value) void fail_second_key(void * /*context*/, Point point, Slice key) { - if (point == Point::kBeforeVersionCas && key == Slice("b")) { + if (point == Point::kBeforePrepare && key == Slice("b")) { throw std::bad_alloc(); } } @@ -52,7 +52,7 @@ Status apply_variant(Crowdbtree &tree, int variant, uint64_t slot) } } // namespace -TEST(MemTableFailure, PartialApplyReleasesAdmissionWithoutCreditingItsSlot) +TEST(MemTableFailure, PreparationFailurePublishesNoKeysAndDoesNotCreditItsSlot) { for (int variant = 0; variant < 3; ++variant) { Crowdbtree tree; @@ -62,9 +62,8 @@ TEST(MemTableFailure, PartialApplyReleasesAdmissionWithoutCreditingItsSlot) EXPECT_EQ(apply_variant(tree, variant, 2).code(), Code::kResourceExhausted); EXPECT_EQ(table->writers(), 0U); EXPECT_EQ(tree.contiguous_slot(), 1U); - EXPECT_EQ(tree.get_view("a").slot(), 2U); + EXPECT_EQ(tree.get_view("a").slot(), 1U); EXPECT_EQ(tree.get_view("b").slot(), 1U); - EXPECT_EQ(Access::counters(tree).failed_batches.load(), 1U); table->set_hook_for_tests(nullptr, nullptr); ASSERT_TRUE(tree.flush().ok()); uint64_t slot = 0; @@ -80,27 +79,24 @@ TEST(MemTableFailure, PartialApplyReleasesAdmissionWithoutCreditingItsSlot) } } -TEST(MemTableFailure, OptionalPruningFailureKeepsSuccessfulCompletion) +TEST(MemTableFailure, FailureBeforeSecondPreparedKeyReturnsItsPermitForRetry) { Crowdbtree tree; - ASSERT_TRUE(tree.apply(1, Batch{{{"a", OpKind::kPut, "old"}}}).ok()); - auto table = Access::active(tree); - int calls = 0; - table->set_hook_for_tests(&calls, [](void *context, Point point, Slice) { - if (point == Point::kBeforeVersionCas && ++*static_cast<int *>(context) == 2) { - throw std::bad_alloc(); - } - }); - ASSERT_TRUE(tree.apply(2, Batch{{{"a", OpKind::kPut, "new"}}}).ok()); - EXPECT_EQ(tree.contiguous_slot(), 2U); + auto table = Access::active(tree); + table->set_hook_for_tests(nullptr, fail_second_key); + EXPECT_EQ(tree.apply(1, values("unpublished")).code(), Code::kResourceExhausted); EXPECT_EQ(table->writers(), 0U); - EXPECT_EQ(table->memory().history_count, 1U); + EXPECT_EQ(table->count(), 0U); + EXPECT_EQ(tree.contiguous_slot(), 0U); table->set_hook_for_tests(nullptr, nullptr); - ASSERT_TRUE(tree.apply(3, Batch{{{"a", OpKind::kPut, "next"}}}).ok()); - EXPECT_EQ(table->memory().history_count, 0U); + ASSERT_TRUE(tree.apply(1, values("retry")).ok()); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_EQ(tree.get_view("a").value().to_string(), "retry"); + EXPECT_EQ(tree.get_view("b").value().to_string(), "retry"); + EXPECT_EQ(tree.last_applied_slot(), 1U); } -TEST(MemTableFailure, IncompleteL1PublicationBlocksExportsUntilRetry) +TEST(MemTableFailure, FailedPublicationPreservesAtomicOldSnapshotAndRepairableSources) { MemPageStore store; Config options; @@ -108,98 +104,90 @@ TEST(MemTableFailure, IncompleteL1PublicationBlocksExportsUntilRetry) Crowdbtree tree(options); ASSERT_TRUE(tree.apply(1, values("old")).ok()); ASSERT_TRUE(tree.flush().ok()); + auto snapshot = tree.snapshot_view(); ASSERT_TRUE(tree.apply(2, values("new")).ok()); - Access::fail_publication(tree, [] { throw std::bad_alloc(); }); + auto &publisher = RuntimeTreeAccess_for_tests::core(tree).publisher(); + publisher.set_before_root_for_tests([] { throw std::bad_alloc(); }); EXPECT_FALSE(tree.flush().ok()); EXPECT_EQ(tree.last_applied_slot(), 1U); EXPECT_EQ(tree.get_view("b").value().to_string(), "new"); - EXPECT_FALSE(tree.snapshot().ok()); - EXPECT_THROW((void)tree.snapshot_view(), std::runtime_error); - std::unique_ptr<SnapshotExport> portable; - EXPECT_FALSE(snapshot_export_begin(tree, snapshot_format::kPortable, 4096, &portable).ok()); - EXPECT_EQ(portable, nullptr); - std::vector<NativeFrame> frames; - EXPECT_FALSE(tree.collect_native_frames(&frames, nullptr, nullptr).ok()); - std::unique_ptr<NativeFrameIterator> iterator; - EXPECT_FALSE(tree.open_native_frame_iterator(nullptr, &iterator).ok()); - Access::fail_publication(tree, nullptr); + uint64_t slot = 0; + std::string value; + ASSERT_TRUE(snapshot->get("b", &slot, &value)); + EXPECT_EQ(slot, 1U); + EXPECT_EQ(value, "old"); + publisher.set_before_root_for_tests({}); ASSERT_TRUE(tree.flush().ok()); ASSERT_TRUE(tree.snapshot().ok()); EXPECT_EQ(tree.last_applied_slot(), 2U); + EXPECT_EQ(tree.get_view("b").value().to_string(), "new"); + ASSERT_TRUE(snapshot->get("b", &slot, &value)); + EXPECT_EQ(value, "old"); } -TEST(MemTableFailure, ReplacementDrainsOldBatchesBeforeAdmittingNewGeneration) +TEST(MemTableFailure, ReplacementDrainsOldBatchesBeforeInstallingNewGeneration) { Crowdbtree tree; + auto table = Access::active(tree); std::promise<void> entered; std::promise<void> release; auto resume = release.get_future(); - auto old = std::async(std::launch::async, [&] { - auto batch = Access::admit(tree); - entered.set_value(); - resume.wait(); - batch.table->upsert("old", 1, encode_cell_buf(1, OpKind::kPut, "old"), batch.bound); - Access::finish(tree, 1); + + struct Pause + { + std::promise<void> &entered; + std::future<void> &resume; + } pause{.entered = entered, .resume = resume}; + + table->set_hook_for_tests(&pause, [](void *argument, Point point, Slice) { + if (point == Point::kBeforePrepare) { + auto &pause = *static_cast<Pause *>(argument); + pause.entered.set_value(); + pause.resume.wait(); + } }); + auto old = std::async(std::launch::async, [&] { return tree.put("old", "old"); }); entered.get_future().wait(); - auto reset = std::async(std::launch::async, [&] { return tree.clear(); }); - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - while (!Access::replacing(tree) && std::chrono::steady_clock::now() < deadline) { - std::this_thread::yield(); - } - EXPECT_TRUE(Access::replacing(tree)); - auto next = std::async(std::launch::async, [&] { return tree.apply(1, values("new")); }); - EXPECT_EQ(next.wait_for(std::chrono::milliseconds(0)), std::future_status::timeout); + auto reset = std::async(std::launch::async, [&] { return tree.clear(); }); + EXPECT_EQ(reset.wait_for(std::chrono::milliseconds(0)), std::future_status::timeout); release.set_value(); - old.get(); + EXPECT_TRUE(old.get().ok()); EXPECT_TRUE(reset.get().ok()); - EXPECT_TRUE(next.get().ok()); + table->set_hook_for_tests(nullptr, nullptr); + ASSERT_TRUE(tree.apply(1, values("new")).ok()); EXPECT_FALSE(tree.get_view("old").found()); EXPECT_EQ(tree.get_view("a").value().to_string(), "new"); EXPECT_EQ(tree.contiguous_slot(), 1U); } -TEST(MemTableFailure, MetricsSeparateLogicalMergeFromDelayedPhysicalReclamation) +TEST(MemTableFailure, LogicalReplacementDoesNotFreeBorrowedPayload) { - Crowdbtree tree; - ASSERT_TRUE(tree.apply(1, values("one")).ok()); - auto borrowed = tree.get_view("a"); - ASSERT_TRUE(tree.apply(2, values("two")).ok()); - ASSERT_TRUE(tree.apply(2, values("replay")).ok()); - auto &metrics = Access::counters(tree); - EXPECT_EQ(metrics.overwrite.load(), 2U); - EXPECT_EQ(metrics.keep.load(), 2U); - EXPECT_EQ(metrics.merged.load(), 2U); - EXPECT_EQ(metrics.keep_pending.load(), 2U); - EXPECT_EQ(Access::active(tree)->memory().history_count, 0U); - auto &epoch = Access::epoch(tree); - const auto resident = epoch.memtable_allocation()->load(); - epoch.try_reclaim(); - EXPECT_EQ(epoch.memtable_allocation()->load(), resident); + GetView borrowed; + { + Crowdbtree tree; + ASSERT_TRUE(tree.apply(1, values("one")).ok()); + borrowed = tree.get_view("a"); + ASSERT_TRUE(tree.apply(2, values("two")).ok()); + ASSERT_TRUE(tree.apply(2, values("two")).ok()); + ASSERT_TRUE(tree.flush().ok()); + RuntimeTreeAccess_for_tests::reclaim(tree); + EXPECT_EQ(borrowed.value().to_string(), "one"); + EXPECT_EQ(tree.get_view("a").value().to_string(), "two"); + } EXPECT_EQ(borrowed.value().to_string(), "one"); - borrowed = {}; - epoch.try_reclaim(); - EXPECT_LT(epoch.memtable_allocation()->load(), resident); } -TEST(MemTableFailure, DescriptorCopyLatencyIncludesAbortedPreparation) +TEST(MemTableFailure, AbortedPreparationDoesNotConsumeAutomaticSlot) { Crowdbtree tree; - tree.init_metrics("copy-test", "memory"); - auto *latency = Access::copy_latency(tree); - ASSERT_NE(latency, nullptr); - ASSERT_TRUE(tree.apply(1, values("old")).ok()); - auto first = latency->flush(); - // Each key's completion prune reconstructs its existing descriptor. - EXPECT_EQ(first.count, 2U); + ASSERT_TRUE(tree.batch_put(values("old")).ok()); auto table = Access::active(tree); table->set_hook_for_tests(nullptr, fail_second_key); - EXPECT_EQ(tree.apply(2, values("new")).code(), Code::kResourceExhausted); + EXPECT_EQ(tree.batch_put(values("failed")).code(), Code::kResourceExhausted); table->set_hook_for_tests(nullptr, nullptr); - auto failed = latency->flush(); - // Both candidates copied current + anchor; only the first CAS published. - EXPECT_EQ(failed.count, 2U); - EXPECT_GE(failed.sum, failed.max); - EXPECT_EQ(failed.total_count, first.count + failed.count); + ASSERT_TRUE(tree.batch_put(values("next")).ok()); + EXPECT_EQ(tree.get_view("a").slot(), 2U); + EXPECT_EQ(tree.get_view("b").slot(), 2U); + EXPECT_EQ(tree.contiguous_slot(), 2U); } #endif diff --git a/lib/crowdb-tree/tests/integration/memtable_handoff_test.cpp b/lib/crowdb-tree/tests/integration/memtable_handoff_test.cpp index 77b9b7847..b252c4ef8 100644 --- a/lib/crowdb-tree/tests/integration/memtable_handoff_test.cpp +++ b/lib/crowdb-tree/tests/integration/memtable_handoff_test.cpp @@ -19,10 +19,21 @@ Batch put(std::string key, std::string value) return Batch{{{.key = std::move(key), .kind = OpKind::kPut, .value = std::move(value)}}}; } +bool wait_closed(const std::shared_ptr<MemTable> &table) +{ + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!table->closed() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + return table->closed(); +} + void expect_value(Crowdbtree &tree, uint64_t expected, const std::string &value) { - auto found = tree.get_view("x"); - ASSERT_TRUE(found.found()); + GetView found; + auto status = RuntimeTreeAccess_for_tests::core(tree).get("x", &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(found.found()) << "coverage=" << tree.last_applied_slot() << " expected=" << expected; EXPECT_EQ(found.slot(), expected); EXPECT_EQ(found.value().to_string(), value); } @@ -35,22 +46,42 @@ TEST(MemTableHandoff, AcknowledgedResidualNeverRelocatesOrLeaksIntoL1) tree.force_advance_slot(843); ASSERT_TRUE(tree.flush().ok()); tree.force_advance_slot(852); - auto boundary = [&] { - auto batch = MemTableAccess_for_tests::admit(tree); - batch.table->upsert("x", 853, encode_cell_buf(853, OpKind::kPut, "cursor24040"), batch.bound); - auto captured = MemTableAccess_for_tests::capture(tree); - MemTableAccess_for_tests::finish(tree, 853); - return captured; - }(); - auto old_sources = MemTableAccess_for_tests::sources(tree); - auto borrowed = tree.get_view("x"); + auto old_table = MemTableAccess_for_tests::active(tree); + std::promise<void> entered; + std::promise<void> release; + auto resume = release.get_future(); + old_table->set_hook_for_tests(nullptr, nullptr); + + struct Pause + { + std::promise<void> &entered; + std::future<void> &resume; + } pause{.entered = entered, .resume = resume}; + + old_table->set_hook_for_tests(&pause, [](void *argument, ConcurrentSkipList::PausePoint point, Slice) { + if (point == ConcurrentSkipList::PausePoint::kBeforePrepare) { + auto &pause = *static_cast<Pause *>(argument); + pause.entered.set_value(); + pause.resume.wait(); + } + }); + auto writer = std::async(std::launch::async, [&] { return tree.apply(853, put("x", "cursor24040")); }); + entered.get_future().wait(); + auto capturing = std::async(std::launch::async, [&] { return MemTableAccess_for_tests::capture(tree); }); + const bool closed = wait_closed(old_table); + release.set_value(); + ASSERT_TRUE(writer.get().ok()); + auto boundary = capturing.get(); + ASSERT_TRUE(closed); + old_table->set_hook_for_tests(nullptr, nullptr); + auto borrowed = tree.get_view("x"); ASSERT_EQ(boundary.frontier, 852U); expect_value(tree, 853, "cursor24040"); MemTableAccess_for_tests::publish(tree, boundary); expect_value(tree, 853, "cursor24040"); EXPECT_EQ(tree.last_applied_slot(), 852U); - ASSERT_NE(old_sources.front()->find("x"), nullptr); - EXPECT_EQ(old_sources.front()->find("x")->slot, 853U); + ASSERT_NE(old_table->find("x"), nullptr); + EXPECT_EQ(old_table->find("x")->slot, 853U); auto snapshot = tree.snapshot_view(); uint64_t published_slot = 0; std::string published_value; @@ -92,32 +123,51 @@ TEST(MemTableHandoff, CapturedPrefixSurvivesOverwriteInEitherOrder) TEST(MemTableHandoff, EmptyAdmittedWriterSurvivesCloseAndSuccessorProgresses) { - Crowdbtree tree; - std::promise<void> admitted; - std::promise<void> resume; - auto continuation = resume.get_future(); - std::thread writer([&] { - auto batch = MemTableAccess_for_tests::admit(tree); - admitted.set_value(); - continuation.wait(); - batch.table->upsert("x", 1, encode_cell_buf(1, OpKind::kPut, "old-writer"), batch.bound); - MemTableAccess_for_tests::finish(tree, 1); + Crowdbtree tree; + tree.force_advance_slot(1); + auto old = MemTableAccess_for_tests::active(tree); + std::promise<void> entered; + std::promise<void> release; + auto resume = release.get_future(); + + struct Pause + { + std::promise<void> &entered; + std::future<void> &resume; + } pause{.entered = entered, .resume = resume}; + + old->set_hook_for_tests(&pause, [](void *argument, ConcurrentSkipList::PausePoint point, Slice) { + if (point == ConcurrentSkipList::PausePoint::kBeforePrepare) { + auto &pause = *static_cast<Pause *>(argument); + pause.entered.set_value(); + pause.resume.wait(); + } }); - admitted.get_future().wait(); - auto boundary = MemTableAccess_for_tests::capture(tree); - EXPECT_EQ(boundary.tables.back()->writers(), 1U); - EXPECT_TRUE(boundary.tables.back()->closed()); - EXPECT_FALSE(boundary.tables.back()->try_enter()); - EXPECT_FALSE(boundary.tables.back()->validate_open()); - ASSERT_TRUE(tree.apply(2, put("b", "successor")).ok()); - resume.set_value(); - writer.join(); + auto writer = std::async(std::launch::async, [&] { return tree.apply(2, put("x", "old-writer")); }); + entered.get_future().wait(); + auto capture = std::async(std::launch::async, [&] { return MemTableAccess_for_tests::capture(tree); }); + if (!wait_closed(old)) { + release.set_value(); + writer.wait(); + capture.wait(); + FAIL() << "source did not close before the bounded deadline"; + } + EXPECT_EQ(old->writers(), 1U); + EXPECT_FALSE(old->try_enter()); + EXPECT_FALSE(old->validate_open()); + RuntimeTreeAccess_for_tests::core(tree).binding()->drain(); + EXPECT_TRUE(tree.apply(3, put("b", "successor")).ok()); + release.set_value(); + EXPECT_TRUE(writer.get().ok()); + auto boundary = capture.get(); + EXPECT_EQ(boundary.frontier, 1U); + old->set_hook_for_tests(nullptr, nullptr); MemTableAccess_for_tests::publish(tree, boundary); - EXPECT_EQ(tree.last_applied_slot(), 0U); - expect_value(tree, 1, "old-writer"); + EXPECT_EQ(tree.last_applied_slot(), 1U); + expect_value(tree, 2, "old-writer"); ASSERT_TRUE(tree.flush().ok()); - EXPECT_EQ(tree.last_applied_slot(), 2U); - expect_value(tree, 1, "old-writer"); + EXPECT_EQ(tree.last_applied_slot(), 3U); + expect_value(tree, 2, "old-writer"); } TEST(MemTableHandoff, SuccessorCompletionCannotWidenCapturedFrontier) @@ -150,26 +200,37 @@ TEST(MemTableHandoff, BorrowedResultOutlivesItsTreeAndGeneration) TEST(MemTableHandoff, AsyncFlushCanWaitWithoutBlockingTheCaller) { - Crowdbtree tree; + Crowdbtree tree; + tree.force_advance_slot(1); + auto old = MemTableAccess_for_tests::active(tree); std::promise<void> entered; std::promise<void> release; - auto proceed = release.get_future(); - std::thread writer([&] { - auto batch = MemTableAccess_for_tests::admit(tree); - entered.set_value(); - proceed.wait(); - batch.table->upsert("x", 1, encode_cell_buf(1, OpKind::kPut, "late"), batch.bound); - MemTableAccess_for_tests::finish(tree, 1); + auto resume = release.get_future(); + + struct Pause + { + std::promise<void> &entered; + std::future<void> &resume; + } pause{.entered = entered, .resume = resume}; + + old->set_hook_for_tests(&pause, [](void *argument, ConcurrentSkipList::PausePoint point, Slice) { + if (point == ConcurrentSkipList::PausePoint::kBeforePrepare) { + auto &pause = *static_cast<Pause *>(argument); + pause.entered.set_value(); + pause.resume.wait(); + } }); + auto writer = std::async(std::launch::async, [&] { return tree.apply(2, put("x", "late")); }); entered.get_future().wait(); std::promise<Status> completion; auto done = completion.get_future(); tree.flush_async([&](Status status) { completion.set_value(std::move(status)); }); - // Reaching this line must not depend on releasing the old writer. + EXPECT_EQ(done.wait_for(std::chrono::milliseconds(0)), std::future_status::timeout); release.set_value(); - writer.join(); + EXPECT_TRUE(writer.get().ok()); EXPECT_TRUE(done.get().ok()); - expect_value(tree, 1, "late"); + old->set_hook_for_tests(nullptr, nullptr); + expect_value(tree, 2, "late"); } TEST(MemTableHandoff, EncodedMismatchCannotCreditUnpublishedFutureSlot) diff --git a/lib/crowdb-tree/tests/integration/native_import_export_test.cpp b/lib/crowdb-tree/tests/integration/native_import_export_test.cpp new file mode 100644 index 000000000..89656ad22 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/native_import_export_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/crowdb-tree.h" +#include "crowdb-tree/maptable/frame_page.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(NativeImportExport, ImportedResidentOverflowExportsAfterTreeClosesWithoutABackend) +{ + Config source_config; + source_config.frame_bytes = 4096; + source_config.max_inline_value = 64; + Crowdbtree source(source_config); + const std::string value(16000, 'i'); + ASSERT_TRUE(source.put("a", value).ok()); + ASSERT_TRUE(source.flush().ok()); + std::vector<NativeFrame> original; + uint64_t root = 0; + uint64_t slot = 0; + uint64_t next = 0; + auto status = source.collect_native_frames(&original, &root, &slot, &next); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_GT(original.size(), 1U); + ASSERT_EQ(frame_page_type(original[1].frame.data()), page_type::kOverflowFrame); + Config target_config; + target_config.frame_bytes = 4096; + target_config.max_inline_value = 64; + ASSERT_TRUE(TreeRuntime::create({}, &target_config.runtime).ok()); + auto target = std::make_unique<Crowdbtree>(target_config); + status = target->install_snapshot_native(std::move(original), root, slot, next); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::unique_ptr<NativeFrameIterator> cursor; + status = target->open_native_frame_iterator(nullptr, &cursor); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto info = cursor->stream_info(); + target.reset(); + std::vector<NativeFrame> frames; + std::vector<NativeFrameView> retained; + bool done = false; + while (!done) { + std::optional<NativeFrameView> frame; + status = cursor->next_owned(&frame, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (!frame) { + continue; + } + EXPECT_EQ(frame->durable_addr, kNoAddr); + EXPECT_FALSE(frame->inherited); + frames.push_back({.page_id = frame->page_id, + .frame = std::vector<uint8_t>(frame->bytes.begin(), frame->bytes.end()), + .durable_addr = kNoAddr, + .durable_plen = 0, + .inherited = false}); + retained.push_back(*frame); + } + EXPECT_EQ(frames.size(), info.frame_count); + cursor.reset(); + for (const auto &frame : retained) { + EXPECT_TRUE(frame_validate(frame.bytes.data(), frame.bytes.size())); + } + Crowdbtree roundtrip(source_config); + status = roundtrip.install_snapshot_native(std::move(frames), info.root_page_id, info.at_slot, info.next_page_id); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::string actual; + ASSERT_TRUE(roundtrip.get("a", nullptr, &actual)); + EXPECT_EQ(actual, value); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/page_writeback_backend_test.cpp b/lib/crowdb-tree/tests/integration/page_writeback_backend_test.cpp new file mode 100644 index 000000000..babe4c353 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/page_writeback_backend_test.cpp @@ -0,0 +1,175 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "async_completion_adapter.h" +#include "page_writeback_test_support.h" +#include "test_tmp.h" + +#include <future> + +namespace crowdb::tree +{ +using namespace test_write; + +#ifdef CROWDB_HAVE_LIBURING +TEST(PageWriteback, RealUringWriteAndColdLoadCrossExtentBoundary) +{ + crowdb::tree_test::TempDir dir("writeback_uring_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 4096, &file).ok()); + crowdb::common::Topology topology; + topology.pipelines.emplace_back().entries = 64; + crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + auto store = std::make_shared<BlockAsyncPageStore>(file.get(), &uring); + ASSERT_TRUE(store->write_completion_is_reloadable()); + TestWriter test; + ASSERT_TRUE(PageWriteOperation::create( + store, test.runtime, test.lane, write_input(), + {.max_frame_bytes = 4096, .compression = compress_algo::kNone}, test.credits.credits, + [owner = test.extent](uint64_t length, PageWriteOperation::ExtentComplete done) { + ReservedWriteExtent extent{.backend_id = 1, + .address = 4096, + .length = length, + .content_id = 3, + .generation = 1, + .owner = owner, + .physical_metrics = {}, + .chunk = {}}; + done(Status::Ok(), std::move(extent)); + }, + &test.operation) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); + test.runtime->drain(); + auto backing = test.operation->result().backing; + ASSERT_TRUE(backing); + EXPECT_EQ(backing->length, 8192U); + AsyncStorePageLoader loader(store, test.runtime, test.lane, {}, {}, test.credits.credits); + std::atomic<bool> done{false}; + std::shared_ptr<const LeafBase> loaded; + loader.load_leaf({.identity = 7, .backing = backing}, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()) << status.to_string(); + loaded = std::move(leaf); + done.store(true, std::memory_order_release); + }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!done.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + ASSERT_TRUE(done.load(std::memory_order_acquire)); + ASSERT_TRUE(loaded); + EXPECT_EQ(CellView(loaded->cell(0)).value().to_string(), "value"); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} + +TEST(PageWriteback, CrossExtentInlineFailureCompletesOnceAfterAllSubmissions) +{ + crowdb::tree_test::TempDir dir("writeback_inline_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 4096, &file).ok()); + // An empty topology deterministically rejects every chunk inline. + crowdb::common::DiskIOUring uring({}); + BlockAsyncPageStore store(file.get(), &uring); + std::vector<uint8_t> buffer(3 * 4096, 0); + size_t terminals = 0; + store.submit_write(0, buffer.data(), buffer.size(), detail::own_async_completion([&](Status status) { + ++terminals; + EXPECT_EQ(status.code(), Code::kIoError); + })); + EXPECT_EQ(terminals, 1U); + terminals = 0; + store.submit_read(0, buffer.data(), buffer.size(), detail::own_async_completion([&](Status status) { + ++terminals; + EXPECT_EQ(status.code(), Code::kIoError); + })); + EXPECT_EQ(terminals, 1U); +} + +TEST(PageWriteback, FsyncReleasesItsOwnedContextAfterInlineFailure) +{ + crowdb::tree_test::TempDir dir("writeback_fsync_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 4096, &file).ok()); + ASSERT_TRUE(file->ensure_extents(0, 3 * 4096).ok()); + crowdb::common::DiskIOUring uring({}); + BlockAsyncPageStore store(file.get(), &uring); + auto calls = std::make_shared<size_t>(0); + std::weak_ptr<size_t> retained = calls; + ASSERT_TRUE(store + .submit_fsync({.context = calls.get(), + .complete_fn = + [](void *context, Status status) { + ++*static_cast<size_t *>(context); + EXPECT_EQ(status.code(), Code::kIoError); + }, + .owner = calls}) + .ok()); + EXPECT_EQ(*calls, 1U); + ASSERT_TRUE(store + .submit_fsync({.context = calls.get(), + .complete_fn = + [](void *context, Status status) { + ++*static_cast<size_t *>(context); + EXPECT_EQ(status.code(), Code::kIoError); + }, + .owner = calls}) + .ok()); + EXPECT_EQ(*calls, 2U); + calls.reset(); + EXPECT_TRUE(retained.expired()); +} + +TEST(PageWriteback, MissingLaterReadExtentDrainsEarlierIoBeforeReportingFailure) +{ + crowdb::tree_test::TempDir dir("writeback_partial_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 4096, &file).ok()); + std::vector<uint8_t> input(4096, 42); + ASSERT_TRUE(file->write_at(0, input.data(), input.size()).ok()); + crowdb::common::Topology topology; + topology.pipelines.emplace_back().entries = 64; + crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + BlockAsyncPageStore store(file.get(), &uring); + std::vector<uint8_t> output(8192, 0); + // The second extent was never written. Absence must agree with the + // synchronous store while accepted I/O for the first extent still drains. + ASSERT_EQ(file->read_at(0, output.data(), output.size()).code(), Code::kNotFound); + std::fill(output.begin(), output.end(), 0); + std::atomic<size_t> calls{0}; + store.submit_read(0, output.data(), output.size(), detail::own_async_completion([&](Status status) { + EXPECT_EQ(status.code(), Code::kNotFound); + EXPECT_TRUE(std::equal(input.begin(), input.end(), output.begin())); + calls.fetch_add(1, std::memory_order_release); + })); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (calls.load(std::memory_order_acquire) == 0 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + EXPECT_EQ(calls.load(std::memory_order_acquire), 1U); +} + +TEST(BlockAsyncPageStore, UnwrittenAndShortExtentsMatchSynchronousNotFound) +{ + crowdb::tree_test::TempDir dir("readback_short_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 8192, 4096, &file).ok()); + std::vector<uint8_t> buffer(8192); + ASSERT_TRUE(file->write_at(0, buffer.data(), 4096).ok()); + crowdb::common::Topology topology; + topology.pipelines.emplace_back().entries = 64; + crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + BlockAsyncPageStore store(file.get(), &uring); + for (const size_t offset : {0U, 4096U, 8192U, 16384U}) { + const auto bytes = offset == 0 ? buffer.size() : size_t{4096}; + ASSERT_EQ(file->read_at(offset, buffer.data(), bytes).code(), Code::kNotFound); + std::promise<Status> completed; + auto result = completed.get_future(); + store.submit_read(offset, buffer.data(), bytes, + detail::own_async_completion([&](Status status) { completed.set_value(std::move(status)); })); + EXPECT_EQ(result.get().code(), Code::kNotFound) << "read offset " << offset; + } +} +#endif +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/page_writeback_test.cpp b/lib/crowdb-tree/tests/integration/page_writeback_test.cpp new file mode 100644 index 000000000..54664ae28 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/page_writeback_test.cpp @@ -0,0 +1,219 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "page_writeback_test_support.h" +#include "snapshot/captured_frames.h" +#include "test_tmp.h" + +#include <algorithm> + +namespace crowdb::tree +{ +using namespace test_write; + +TEST(PageWriteback, HeldCreditPaysAlignedCodecAndIoAfterSealAndReturnsReloadableBacking) +{ + TestWriter test; + ASSERT_TRUE(test.prepare().ok()); + const auto bound = PageWriteOperation::allocation_bound({}); + if (!bound) { + ADD_FAILURE() << "write allocation bound missing"; + return; + } + EXPECT_LE(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], bound.value()); + test.credits.seal(); + ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); + test.runtime->drain(); + const auto backing = test.operation->result().backing; + ASSERT_TRUE(backing); + EXPECT_TRUE(backing->reloadable()); + EXPECT_EQ(backing->durability, BackingDurability::kReloadable); + EXPECT_EQ(backing->address, 8192U); + EXPECT_EQ(test.store->purpose, PagePurpose::kBtreePage); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + std::vector<uint8_t> blob(backing->length); + std::vector<uint8_t> frame(4096); + ASSERT_TRUE(test.store->device->read_at(backing->address, blob.data(), blob.size()).ok()); + ASSERT_TRUE(decode_durable_page(blob.data(), blob.size(), frame.data(), frame.size()).ok()); + LeafFrameView decoded(frame.data(), frame.size()); + ASSERT_EQ(decoded.count(), 1U); + EXPECT_EQ(CellView(decoded.cell(0)).value().to_string(), "value"); + EXPECT_TRUE( + std::all_of(blob.begin() + test.operation->result().logical_bytes, blob.end(), [](auto b) { return b == 0; })); +} + +TEST(PageWriteback, CancellationWaitsForWriteAndKeepsItsPermitAndExtentAlive) +{ + TestWriter test; + test.store->deferred = true; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + ASSERT_EQ(test.store->writes, 1U); + test.operation->request_stop(); + EXPECT_EQ(done.stopped.load(), 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + EXPECT_EQ(test.credits.resources->snapshot().write_inflight_bytes, + test.credits.account->snapshot().physical[static_cast<size_t>(AllocationKind::kIo)]); + EXPECT_GT(test.credits.resources->snapshot().write_inflight_bytes, test.store->bytes); + test.store->finish(); + EXPECT_EQ(test.credits.resources->snapshot().write_inflight_bytes, 0U); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_FALSE(test.operation->result().backing); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + test.operation.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageWriteback, UnknownSubmissionOutcomeKeepsOwnedCallbackBufferUntilActualCompletion) +{ + TestWriter test; + test.store->throw_after_capture = true; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + ASSERT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kIoError); + EXPECT_FALSE(test.operation->result().backing); + test.operation.reset(); + test.credits.seal(); + EXPECT_GT(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + EXPECT_EQ(test.credits.resources->snapshot().write_inflight_bytes, + test.credits.account->snapshot().physical[static_cast<size_t>(AllocationKind::kIo)]); + EXPECT_GT(test.credits.resources->snapshot().write_inflight_bytes, test.store->bytes); + test.store->finish(); + EXPECT_EQ(test.credits.resources->snapshot().write_inflight_bytes, 0U); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} + +TEST(PageWriteback, BackendCapabilityAndMissingProgressPermitRejectBeforeWrite) +{ + TestWriter test; + test.store->reloadable = false; + EXPECT_EQ(test.prepare().code(), Code::kNotSupported); + EXPECT_EQ(test.store->writes, 0U); + test.store->reloadable = true; + AllocationLease occupied(test.credits.credits.io, 1); + EXPECT_EQ(test.prepare().code(), Code::kResourceExhausted); + EXPECT_FALSE(test.operation); + EXPECT_EQ(test.store->writes, 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageWriteback, OverflowDependencyRetainsBackingWithoutRetainingRawInput) +{ + TestWriter test; + auto input = write_input(4096, true); + std::weak_ptr<const void> raw = input.owner; + auto protection = std::make_shared<int>(2); + std::weak_ptr<int> retained = protection; + input.reload_dependencies = protection; + ASSERT_TRUE(test.prepare(std::move(input)).ok()); + ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); + test.runtime->drain(); + auto backing = test.operation->result().backing; + ASSERT_TRUE(backing); + EXPECT_TRUE(raw.expired()); + test.operation.reset(); + protection.reset(); + EXPECT_FALSE(retained.expired()); + backing.reset(); + EXPECT_TRUE(retained.expired()); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageWriteback, MissingOverflowProtectionAndCorruptInputNeverAllocateOrWrite) +{ + for (const bool overflow : {false, true}) { + TestWriter test; + auto input = write_input(4096, overflow); + if (!overflow) { + ++input.info.page_id; + } + ASSERT_TRUE(test.prepare(std::move(input)).ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), overflow ? Code::kInvalidArgument : Code::kCorruption); + EXPECT_EQ(test.next_address, 8192U); + EXPECT_EQ(test.store->writes, 0U); + } +} + +TEST(PageWriteback, WriteErrorNeverProducesBackingAndDropsAllOperationCharges) +{ + TestWriter test; + test.store->deferred = true; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.runtime->drain(); + test.store->finish(Status::io_error()); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_FALSE(test.operation->result().backing); + test.operation.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} + +TEST(PageWriteback, CapturedFrameWritesToFileAndReloadsThroughExistingDurableCodec) +{ + crowdb::tree_test::TempDir dir("captured_write_"); + ASSERT_FALSE(dir.path.empty()); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 64 * 1024, 4096, &file).ok()); + TestWriter test; + test.store->device = std::shared_ptr<PageStore>(std::move(file)); + auto captured = std::make_shared<PublishedTreeState>(*view(page({ + {.key = "a", .value = "captured"} + }), + {}, 10)); + captured->next_page_id = 1; + std::shared_ptr<CapturedFrameStream> stream; + ASSERT_TRUE(CapturedFrameStream::create(captured, {}, test.runtime, test.lane, {.encode = {.frame_bytes = 4096}}, + {}, &stream) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(stream->sender())); + test.runtime->drain(); + const auto &frame = stream->result().frame; + if (!frame) { + FAIL() << "captured frame missing"; + } + ASSERT_TRUE( + test.prepare( + {.info = frame->info, .frame = frame->buffer->bytes, .owner = frame->buffer, .reload_dependencies = {}}) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); + test.runtime->drain(); + ASSERT_TRUE(test.store->device->sync().ok()); + auto backing = test.operation->result().backing; + ASSERT_TRUE(backing); + // This is page persistence/reload evidence, not an anchor checkpoint. + std::unique_ptr<BlockPageStore> reopened; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 64 * 1024, 4096, &reopened).ok()); + auto reader = std::make_shared<TestWriteStore>(); + reader->device = std::shared_ptr<PageStore>(std::move(reopened)); + AsyncStorePageLoader loader(reader, test.runtime, test.lane); + std::shared_ptr<const LeafBase> loaded; + loader.load_leaf({.identity = 0, .backing = backing}, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + loaded = std::move(leaf); + }); + test.runtime->drain(); + ASSERT_TRUE(loaded); + EXPECT_EQ(CellView(loaded->cell(0)).value().to_string(), "captured"); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/parallel_flush_test.cpp b/lib/crowdb-tree/tests/integration/parallel_flush_test.cpp new file mode 100644 index 000000000..9e09b4430 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/parallel_flush_test.cpp @@ -0,0 +1,404 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_batch.h" + +#include <gtest/gtest.h> + +#include <future> +#include <latch> + +namespace crowdb::tree +{ +namespace +{ +void initialize(PublicationCoordinator &publisher, std::vector<std::shared_ptr<MemTable>> tables) +{ + auto left = std::make_shared<PageState>(); + left->range.upper = "m"; + left->right_sibling = 1; + auto right = std::make_shared<PageState>(); + right->range.lower = "m"; + auto first = std::make_shared<PublishedPage>(); + first->leaf = left; + auto second = std::make_shared<PublishedPage>(); + second->leaf = right; + auto index = std::make_shared<InnerPageState>(); + index->separators = {"m"}; + index->children = {0, 1}; + auto root = std::make_shared<PublishedPage>(); + root->inner = index; + root->level = 1; + root->metrics.leaf = false; + auto catalog = std::make_shared<PublishedSources>(); + catalog->tables.assign(tables.begin(), tables.end()); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = { + .record = std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 0, .range = 0, .sequence = 0} + ), + .pages = {{.page_id = 0, .expected = {}, .replacement = first}, + {.page_id = 1, .expected = {}, .replacement = second}, + {.page_id = 2, .expected = {}, .replacement = root}}, + .root_page_id = 2, + .expected_sources = publisher.capture()->sources, + .sources = catalog + }; + request->on_done = [](const auto &result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(request)); +} + +std::shared_ptr<MemTable> table(std::initializer_list<std::pair<std::string, uint64_t>> entries) +{ + auto source = std::make_shared<MemTable>(); + for (const auto &[key, slot] : entries) { + EXPECT_TRUE(source->upsert(key, slot, encode_cell_buf(slot, OpKind::kPut, key + std::to_string(slot)))); + } + source->close(); + return source; +} + +struct Receiver +{ + using receiver_concept = stdexec::receiver_tag; + std::promise<void> *done; + + void set_value() const noexcept + { + done->set_value(); + } + + void set_error(Status error) const noexcept + { + ADD_FAILURE() << error.to_string(); + done->set_value(); + } + + void set_stopped() const noexcept + { + ADD_FAILURE() << "unexpected unreported stop"; + done->set_value(); + } +}; + +TEST(ParallelFlush, AllMemTablesMergeByRangeBeforeMaintenanceWithSingleOrMultipleWorkers) +{ + for (const size_t workers : {1U, 4U}) { + detail::FlushRuntime runtime(workers, 16); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(16); + PublicationCoordinator publisher; + auto older = table({ + {"a", 1}, + {"b", 1}, + {"c", 1}, + {"d", 1}, + {"e", 1}, + {"m", 2}, + {"z", 8} + }); + auto newer = table({ + {"a", 3}, + {"m", 4}, + {"n", 4} + }); + const void *payload; + { + auto guard = newer->read_guard(); + payload = newer->find("a")->cell.data() + kCellHeaderSize; + } + initialize(publisher, {older, newer}); + auto before = publisher.capture(); + bool maintenance_ran = false; + FlushBatchOptions options; + options.run_limits.max_entries = 2; + std::shared_ptr<FlushBatch> batch; + auto status = FlushBatch::prepare( + publisher, runtime, lane, 1, 4, options, + [&](auto view, auto done) { + EXPECT_EQ(view->coverage, 4U); + EXPECT_EQ(view->mapping.get(0)->leaf->lookup("a").cell.slot, 3U); + EXPECT_EQ(view->mapping.get(1)->leaf->lookup("m").cell.slot, 4U); + maintenance_ran = true; + done(Status::Ok(), 2); + }, + &batch); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(maintenance_ran); + EXPECT_EQ(batch->result().phase, FlushPhase::kComplete); + EXPECT_EQ(batch->result().published_coverage, 4U); + const auto after = publisher.capture(); + for (const auto pid : {0U, 1U}) { + ASSERT_TRUE(after->snapshot_changes.get(pid)); + EXPECT_EQ(after->snapshot_changes.get(pid)->effect, + SnapshotChangeEffect::FullContent | SnapshotChangeEffect::AcceptedPrefix); + } + EXPECT_EQ(after->stats.leaf_pages, 2U); + EXPECT_EQ(after->stats.height, 2U); + EXPECT_EQ(after->stats.dirty_pages, 2U); + EXPECT_EQ(after->stats.overlay_entries, 7U); + EXPECT_EQ(after->mapping.get(0)->leaf->lookup("a").cell.value.data(), payload); + EXPECT_EQ(after->mapping.get(1)->leaf->lookup("z").state, LogicalLookupState::kMissing); + ASSERT_EQ(after->sources->tables.size(), 1U); + EXPECT_EQ(after->sources->tables.front(), older); + EXPECT_EQ(after->sources->floor, 4U); + EXPECT_EQ(before->coverage, 0U); + EXPECT_EQ(before->sources->tables.size(), 2U); + EXPECT_EQ(before->mapping.get(0)->leaf->lookup("a").state, LogicalLookupState::kMissing); + } +} + +TEST(ParallelFlush, PartialHandoffRetainsSourcesAndRetryDoesNotDuplicateInstalledRuns) +{ + detail::FlushRuntime runtime(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + PublicationCoordinator publisher; + initialize(publisher, { + table({{"a", 1}, {"m", 1}} + ) + }); + std::latch entered(1); + std::latch release(1); + + struct Blocker + { + std::latch &entered; + std::latch &release; + } blocker{.entered = entered, .release = release}; + + ASSERT_TRUE(runtime + .try_submit( + lane, + [](void *context) noexcept { + auto *block = static_cast<Blocker *>(context); + block->entered.count_down(); + block->release.wait(); + }, + &blocker) + .ok()); + entered.wait(); + std::shared_ptr<FlushBatch> batch; + FlushBatchOptions options; + // This case exercises queued-child admission failure. + options.inline_ranges_per_step = 0; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + std::promise<void> done; + auto terminal = done.get_future(); + auto operation = stdexec::connect(batch->sender(), Receiver{&done}); + stdexec::start(operation); + release.count_down(); + terminal.get(); + runtime.drain(); + EXPECT_EQ(batch->result().phase, FlushPhase::kRepairRequired); + EXPECT_FALSE(batch->result().handoff_status.ok()); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_EQ(publisher.capture()->sources->tables.size(), 1U); + auto installed = publisher.capture()->mapping.get(0); + ASSERT_EQ(installed->leaf->overlays.size(), 1U); + ASSERT_TRUE(publisher.capture()->pending_handoff); + auto checkpoint = std::make_shared<PublicationRequest>(); + checkpoint->transaction = { + .record = std::make_shared<InstallRecord>(InstallToken{.batch = 2} + ), + .expected_coverage = 0, + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = 1} + }; + checkpoint->on_done = [](PublicationResult result) { EXPECT_EQ(result.status.code(), Code::kUnavailable); }; + ASSERT_TRUE(publisher.try_submit(checkpoint)); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(batch->result().handoff_status.ok()); + EXPECT_EQ(batch->result().published_coverage, 1U); + EXPECT_EQ(publisher.capture()->mapping.get(0), installed); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 2U); + EXPECT_TRUE(publisher.capture()->sources->tables.empty()); + EXPECT_FALSE(publisher.capture()->pending_handoff); +} + +TEST(ParallelFlush, MaintenanceFailurePreservesCoverageAndRetriesOnlyMaintenance) +{ + detail::FlushRuntime runtime(2, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize(publisher, { + table({{"a", 1}, {"m", 1}} + ) + }); + std::shared_ptr<FlushBatch> batch; + size_t attempts = 0; + std::function<void(Status, uint64_t)> late; + ASSERT_TRUE(FlushBatch::prepare( + publisher, runtime, lane, 1, 1, {}, + [&](auto, auto done) { + ++attempts; + if (attempts == 1) { + late = done; + } + else { + late(Status::corruption("stale maintenance completion"), 999); + } + done(attempts == 1 ? Status::io_error("controlled writeback failure") : Status::Ok(), 2); + }, + &batch) + .ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(batch->result().handoff_status.ok()); + EXPECT_EQ(batch->result().maintenance_status.code(), Code::kIoError); + EXPECT_EQ(batch->result().published_coverage, 1U); + auto covered = publisher.capture(); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(batch->result().maintenance_status.ok()); + EXPECT_EQ(batch->result().deferred_pages, 2U); + EXPECT_EQ(attempts, 2U); + EXPECT_EQ(publisher.capture(), covered); + const auto work = publisher.work_metrics()->totals(); + EXPECT_EQ(work[static_cast<size_t>(TreeStage::FlushBatch)].count, 2U); + EXPECT_EQ(work[static_cast<size_t>(TreeStage::FlushHandoff)].count, 1U); + EXPECT_EQ(work[static_cast<size_t>(TreeStage::MaintenancePlan)].count, 2U); +} + +TEST(ParallelFlush, PreStartCancellationRetainsCapturedBatchForRetry) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize(publisher, { + table({{"a", 1}, {"m", 1}} + ) + }); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, {}, {}, &batch).ok()); + batch->request_stop(); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + EXPECT_EQ(batch->result().phase, FlushPhase::kRepairRequired); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 0U); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_EQ(batch->result().published_coverage, 1U); + EXPECT_EQ(publisher.capture()->stats.overlay_entries, 2U); +} + +TEST(ParallelFlush, ColdLeafAcceptsOverlayWithoutMaterializingBase) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize(publisher, { + table({{"a", 1}} + ) + }); + auto original = publisher.capture()->mapping.get(0); + auto leaf = std::make_shared<PageState>(*original->leaf); + auto backing = std::make_shared<BackingRef>(); + backing->length = 4096; + backing->owner = std::make_shared<uint8_t>(0); + leaf->bases.push_back({.identity = 1, .range = leaf->range, .max_slot = 0, .backing = backing}); + auto proxy = std::make_shared<PublishedPage>(*original); + proxy->leaf = leaf; + auto request = std::make_shared<PublicationRequest>(); + request->transaction = { + .record = std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 0, .range = 1, .sequence = 0}), + .pages = {{.page_id = 0, .expected = original, .replacement = proxy}}}; + ASSERT_TRUE(publisher.try_submit(request)); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, {}, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + auto current = publisher.capture()->mapping.get(0)->leaf; + EXPECT_EQ(current->bases.front().backing, backing); + EXPECT_FALSE(current->bases.front().resident); + EXPECT_EQ(current->lookup("a").cell.value.to_string(), "a1"); + EXPECT_EQ(current->lookup("b").state, LogicalLookupState::kNeedsBase); +} + +TEST(ParallelFlush, DetachedSourceIndexCanDieWhileOverlayValueRemainsReadable) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + auto source = table({ + {"a", 1}, + {"m", 1} + }); + std::weak_ptr<MemTable> weak = source; + initialize(publisher, {source}); + source.reset(); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, {}, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + publisher.reclaim(); + EXPECT_TRUE(weak.expired()); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("a").cell.value.to_string(), "a1"); +} + +TEST(ParallelFlush, SuccessorRemainsVisibleAndDoesNotExpandCapturedFrontier) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + auto source = table({ + {"a", 1} + }); + auto successor = std::make_shared<MemTable>(); + ASSERT_TRUE(successor->upsert("n", 2, encode_cell_buf(2, OpKind::kPut, "new"))); + initialize(publisher, {source, successor}); + auto inputs = std::make_shared<PublishedSources>(); + inputs->tables = {source}; + FlushBatchOptions options; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch, inputs).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(batch->result().handoff_status.ok()); + auto current = publisher.capture(); + EXPECT_EQ(current->coverage, 1U); + ASSERT_EQ(current->sources->tables.size(), 1U); + EXPECT_EQ(current->sources->tables.front(), successor); + EXPECT_EQ(current->mapping.get(1)->leaf->lookup("n").state, LogicalLookupState::kMissing); + EXPECT_EQ(current->stats.overlay_entries, 1U); + auto guard = successor->read_guard(); + ASSERT_NE(successor->find("n"), nullptr); + EXPECT_EQ(successor->find("n")->slot, 2U); +} + +TEST(ParallelFlush, EmptyInputAdvancesCoverageWithoutInstallingEmptyRuns) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + initialize(publisher, {table({})}); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 8, {}, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + EXPECT_TRUE(batch->result().handoff_status.ok()); + auto current = publisher.capture(); + EXPECT_EQ(current->coverage, 8U); + EXPECT_EQ(current->stats.overlay_runs, 0U); + EXPECT_TRUE(current->sources->tables.empty()); +} + +TEST(ParallelFlush, CannotDropAnotherClosedTableFromBatchInputs) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher; + auto source = table({ + {"a", 1} + }); + initialize(publisher, { + source, table({{"m", 2}} + ) + }); + auto inputs = std::make_shared<PublishedSources>(); + inputs->tables = {source}; + std::shared_ptr<FlushBatch> batch; + EXPECT_EQ(FlushBatch::prepare(publisher, runtime, lane, 1, 2, {}, {}, &batch, inputs).code(), + Code::kInvalidArgument); + EXPECT_EQ(publisher.capture()->coverage, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/parity_test.cpp b/lib/crowdb-tree/tests/integration/parity_test.cpp index 89febf7f8..a879c3b7a 100644 --- a/lib/crowdb-tree/tests/integration/parity_test.cpp +++ b/lib/crowdb-tree/tests/integration/parity_test.cpp @@ -3,6 +3,7 @@ // CT14: randomized parity against a std::map oracle (in-order + out-of-order), // exercising flush / consolidate / split / merge. +#include "btree/tree_core_access.h" #include "crowdb-tree/crowdb-tree.h" #include <gtest/gtest.h> @@ -120,6 +121,7 @@ TEST(Parity, InOrderRandomOpsWithPeriodicCompare) TEST(Parity, OutOfOrderConvergesToHighestSlot) { Crowdbtree t(small_tree()); + const auto account = TreeCoreAccess::get(t)->binding()->account(); struct Op { @@ -155,15 +157,38 @@ TEST(Parity, OutOfOrderConvergesToHighestSlot) } } - // apply in shuffled order; contiguous stays 0 (nothing flushes), so all live - // in L0 with highest-slot-wins. scan must already match the oracle. - std::vector<Op> shuffled = ops; - std::shuffle(shuffled.begin(), shuffled.end(), rng); // NOLINT(cert-msc50-cpp) - for (const auto &op : shuffled) { - Batch b{{{.key = op.key, .kind = op.del ? OpKind::kDelete : OpKind::kPut, .value = op.value}}}; - ASSERT_TRUE(t.apply(op.slot, b).ok()); + // Keep every operation while bounding the outstanding out-of-order prefix. + // The admission contract reserves 64 gap entries. Each shuffled window + // resolves highest slots before and after handoff without exceeding it. + constexpr size_t window = 64; + std::map<std::string, std::string> expected; + for (size_t begin = 0; begin < ops.size(); begin += window) { + const auto end = std::min(begin + window, ops.size()); + std::vector<Op> shuffled(ops.begin() + static_cast<ptrdiff_t>(begin), + ops.begin() + static_cast<ptrdiff_t>(end)); + std::shuffle(shuffled.begin(), shuffled.end(), rng); // NOLINT(cert-msc50-cpp) + for (const auto &op : shuffled) { + Batch b{{{.key = op.key, .kind = op.del ? OpKind::kDelete : OpKind::kPut, .value = op.value}}}; + const auto status = t.apply(op.slot, b); + ASSERT_TRUE(status.ok()) << status.to_string() << " at slot " << op.slot; + } + for (size_t index = begin; index < end; ++index) { + const auto &op = ops[index]; + if (op.del) { + expected.erase(op.key); + } + else { + expected[op.key] = op.value; + } + } + expect_parity(t, expected); + ASSERT_TRUE(t.flush().ok()); + expect_parity(t, expected); + // The next shuffled window needs its independent gap reserve. Finish + // the accepted retirement work from this fully resolved window first; + // successful publication alone does not release retired allocation. + account->drain_retirements(); } - expect_parity(t, oracle); // pre-flush (L0 only) // Now make everything contiguous and flush; state must still match. t.force_advance_slot(M); diff --git a/lib/crowdb-tree/tests/integration/persist_test.cpp b/lib/crowdb-tree/tests/integration/persist_test.cpp index 01f47c3a4..4853d8ab6 100644 --- a/lib/crowdb-tree/tests/integration/persist_test.cpp +++ b/lib/crowdb-tree/tests/integration/persist_test.cpp @@ -2,9 +2,15 @@ // Licensed under the Apache License, Version 2.0. // PT3-PT5: snapshot + recovery + durable round-trip integration tests. +#include "backend/local/block_account.h" +#include "backend/local/local_backend_state.h" +#include "backend/local/local_block_gc.h" +#include "btree/allocation_bound.h" +#include "compaction_fault_store.h" #include "crowdb-tree/backend/block_page_store.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" +#include "runtime_tree_access.h" #include "test_tmp.h" #include <gtest/gtest.h> @@ -137,15 +143,15 @@ TEST(Persist, LazyRecoveryDemandLoadsOnAccess) std::unique_ptr<Crowdbtree> t2; ASSERT_TRUE(Crowdbtree::open(opt, &t2).ok()); // Lazy: recovery only recorded page_id->addr tags; nothing is resident yet. - ASSERT_NE(t2->buffer_pool(), nullptr); - EXPECT_EQ(t2->buffer_pool()->stats().used, 0U); + ASSERT_GT(t2->leaf_count_atomic(), 0U); + EXPECT_EQ(t2->stats().buffer_pool_used, 0U); // First access demand-loads the pages along the descent path into the pool. std::string v; uint64_t s; ASSERT_TRUE(t2->get(Slice(make_key(0)), &s, &v)); EXPECT_EQ(v, "val0"); - EXPECT_GT(t2->buffer_pool()->stats().used, 0U); + EXPECT_GT(t2->stats().buffer_pool_used, 0U); // Every key reads back correctly through demand load. for (const auto &kv : oracle) { @@ -182,7 +188,7 @@ TEST(Persist, CheckpointThenReopenRestoresKeys) } } -TEST(Persist, TreeSummaryTracksDurableRootAndRejectsUnknownAsZero) +TEST(Persist, TreeSummaryTracksPublishedRootAndRejectsUnknownAsZero) { MemPageStore store(1); Config opt; @@ -201,8 +207,12 @@ TEST(Persist, TreeSummaryTracksDurableRootAndRejectsUnknownAsZero) ASSERT_TRUE(t.apply(3, del_one("a")).ok()); ASSERT_TRUE(t.flush().ok()); - // The old durable root remains authoritative until the next snapshot. - EXPECT_EQ(t.tree_summary().live_kv, 2U); + // Publication updates known live counts; exact graph sizes await a checkpoint. + const auto changed = t.tree_summary(); + EXPECT_FALSE(changed.exact); + EXPECT_EQ(changed.live_kv, 1U); + EXPECT_EQ(changed.covered_slot, 3U); + EXPECT_EQ(t.durable_snapshot_last_applied_slot(), 2U); ASSERT_TRUE(t.snapshot().ok()); EXPECT_EQ(t.tree_summary().live_kv, 1U); } @@ -210,10 +220,11 @@ TEST(Persist, TreeSummaryTracksDurableRootAndRejectsUnknownAsZero) TEST(Persist, TreeSummaryAggregatesPersistedInnerPages) { MemPageStore store(1); - Config opt; - opt.page_store = &store; + Config opt; + opt.page_store = &store; opt.leaf_split_bytes = 256; - Crowdbtree t(opt); + auto owner = std::make_unique<Crowdbtree>(opt); + auto &t = *owner; for (uint64_t slot = 1; slot <= 80; ++slot) { ASSERT_TRUE(t.apply(slot, put_one(make_key(static_cast<int>(slot)), "value")).ok()); } @@ -225,11 +236,13 @@ TEST(Persist, TreeSummaryAggregatesPersistedInnerPages) EXPECT_GT(summary.reachable_leaf_pages, 1U); EXPECT_GT(summary.reachable_inner_pages, 0U); EXPECT_EQ(summary.live_logical_bytes, summary.live_key_bytes + summary.live_value_bytes); + EXPECT_EQ(summary.root_version, t.version()); + owner.reset(); std::unique_ptr<Crowdbtree> reopened; ASSERT_TRUE(Crowdbtree::open(opt, &reopened).ok()); const auto reopened_summary = reopened->tree_summary(); - EXPECT_EQ(reopened_summary.root_version, summary.root_version); + EXPECT_EQ(reopened_summary.root_version, reopened->version()); EXPECT_EQ(reopened_summary.covered_slot, summary.covered_slot); EXPECT_EQ(reopened_summary.live_kv, summary.live_kv); EXPECT_EQ(reopened_summary.live_logical_bytes, summary.live_logical_bytes); @@ -333,7 +346,8 @@ TEST(Persist, MultiLevelTreeSurvivesAndComparesEqual) } std::unique_ptr<Crowdbtree> t2; - ASSERT_TRUE(Crowdbtree::open(opt, &t2).ok()); + const auto reopened = Crowdbtree::open(opt, &t2); + ASSERT_TRUE(reopened.ok()) << reopened.to_string(); EXPECT_GT(t2->height(), 1); auto after = t2->snapshot_view(); EXPECT_TRUE(before->compare(*after).empty()); @@ -540,13 +554,8 @@ TEST(Persist, BlockDeviceBackendRoundTrip) } } -// plan-tree #8: write_mutex_ must be released before snapshot()'s I/O phase -// (prepare_snapshot_locked's CPU-only walk holds it; the actual -// write_at/sync calls must not). Park the first write_at() call mid-flight -// on a background thread, then prove apply()+flush() -- flush() needs -// write_mutex_ -- still completes on the main thread without waiting for -// the parked write to unblock. If write_mutex_ were still held across the -// I/O phase, flush() would deadlock against this test's own timeout. +// Park checkpoint I/O and verify that concurrent apply/flush can publish a +// newer prefix while the captured checkpoint retains its original coverage. TEST(Persist, WriteMutexNotHeldDuringSnapshotIo) { MemPageStore mem(1); @@ -560,15 +569,15 @@ TEST(Persist, WriteMutexNotHeldDuringSnapshotIo) std::thread snap_thread([&] { Status s = t.snapshot(nullptr); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << s.to_string(); }); store.wait_until_blocked(); // snapshot()'s I/O phase is now parked - // If write_mutex_ were still held here, this would block until - // snap_thread's I/O unblocks and commit_prepared_snapshot() releases it - // -- defeating the whole point of the async-writable persistence phase. - ASSERT_TRUE(t.apply(2, put_one("b", "2")).ok()); - ASSERT_TRUE(t.flush().ok()); + // Keep failure diagnostics nonfatal until the parked thread is released. + const auto applied = t.apply(2, put_one("b", "2")); + EXPECT_TRUE(applied.ok()) << applied.to_string(); + const auto flushed = t.flush(); + EXPECT_TRUE(flushed.ok()) << flushed.to_string(); std::string v; uint64_t s; EXPECT_TRUE(t.get(Slice("b"), &s, &v)); @@ -576,6 +585,9 @@ TEST(Persist, WriteMutexNotHeldDuringSnapshotIo) store.release(); snap_thread.join(); + EXPECT_EQ(t.last_applied_slot(), 2U) << "snapshot completion must not regress concurrent flush coverage"; + EXPECT_EQ(t.durable_snapshot_last_applied_slot(), 1U) + << "durable snapshot frontier remains the prepared snapshot frontier"; } // Task 6: Array-of-blocks BlockPageStore integration test. Snapshot with @@ -949,9 +961,13 @@ TEST(Persist, CompactSparseBlocksRestartSafety) ASSERT_FALSE(tmp.path.empty()); constexpr uint64_t blk = 8 * 1024; + // Preserve complete old leaves as well as the spaced survivors. A 4 KiB + // frame shares an 8 KiB block with pages invalidated by the deletes. std::set<std::string> kept_keys; - for (int i = 0; i < 200; i += 20) { - kept_keys.insert(make_key(i)); + for (int i = 0; i < 200; ++i) { + if (i < 20 || i % 20 == 0) { + kept_keys.insert(make_key(i)); + } } { @@ -959,10 +975,13 @@ TEST(Persist, CompactSparseBlocksRestartSafety) ASSERT_TRUE(BlockPageStore::open_blocks(tmp.path, 0, 0, blk, 1, &store).ok()); Config opt; opt.page_store = store.get(); + opt.frame_bytes = 4096; opt.leaf_split_bytes = 256; opt.merge_gc_block_free_threshold = 0.30; opt.merge_gc_max_relocation_bytes = 8 * 1024 * 1024; Crowdbtree t(opt); + auto local = RuntimeTreeAccess_for_tests::core(t).local_backend(); + local->gc_active.store(true); uint64_t slot = 0; for (int i = 0; i < 200; ++i) { @@ -974,19 +993,37 @@ TEST(Persist, CompactSparseBlocksRestartSafety) // Delete most keys to create sparse blocks. for (int i = 0; i < 200; ++i) { - if (i % 20 != 0) { + if (!kept_keys.contains(make_key(i))) { ++slot; ASSERT_TRUE(t.apply(slot, del_one(make_key(i))).ok()); } } ASSERT_TRUE(t.flush().ok()); - t.set_gc_watermark(slot, slot); + // Keep the GC floor fixed so unrelated clean leaves reuse their old + // extents; advancing it would rewrite every page into a dense suffix. ASSERT_TRUE(t.snapshot().ok()); - // Run compaction. + // Rotate both durable anchors before selecting sparse extents, and + // leave this fixture's only compaction to the explicit call below. + ASSERT_TRUE(t.snapshot().ok()); + local->gc_active.store(false); MergeGcStats stats; - ASSERT_TRUE(t.compact_sparse_blocks(&stats).ok()); + // A pass scans at most eight blocks, including fully free blocks + // removed before sparse selection. Follow its cursor for at most one + // initial-domain traversal rather than assuming the first window wins. + const auto initial_blocks = local->ledger->stats().address_high_water / blk; + for (uint64_t visited = 0; visited < initial_blocks && stats.blocks_selected == 0; visited += 8) { + const auto previous_cursor = local->gc_cursor.load(); + const auto compacted = t.compact_sparse_blocks(&stats); + ASSERT_TRUE(compacted.ok()) << compacted.to_string(); + const auto next_cursor = local->gc_cursor.load(); + if (stats.blocks_selected != 0 || next_cursor == 0) { + break; + } + ASSERT_GT(next_cursor, previous_cursor); + } EXPECT_GE(stats.blocks_selected, 1U); + EXPECT_GT(stats.pages_relocated, 0U); // Verify surviving keys. for (const auto &k : kept_keys) { @@ -1001,120 +1038,186 @@ TEST(Persist, CompactSparseBlocksRestartSafety) std::unique_ptr<BlockPageStore> store; ASSERT_TRUE(BlockPageStore::open_blocks(tmp.path, 0, 0, blk, 1, &store).ok()); Config opt; - opt.page_store = store.get(); + opt.page_store = store.get(); + opt.frame_bytes = 4096; std::unique_ptr<Crowdbtree> t; ASSERT_TRUE(Crowdbtree::open(opt, &t).ok()); - for (const auto &k : kept_keys) { - std::string v; - uint64_t s; - EXPECT_TRUE(t->get(Slice(k), &s, &v)) << "missing " << k << " after reopen"; + for (int i = 0; i < 200; ++i) { + const auto key = make_key(i); + std::string value; + uint64_t recovered_slot; + const bool found = t->get(Slice(key), &recovered_slot, &value); + EXPECT_EQ(found, kept_keys.contains(key)) << key << " after reopen"; + if (found) { + EXPECT_EQ(value, "v" + std::to_string(i)); + } + } + } +} + +namespace +{ +void prepare_sparse_compaction(const std::string &path, const std::set<std::string> &kept_keys) +{ + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(path, 0, 0, 8192, 1, &store).ok()); + Config opt; + opt.page_store = store.get(); + opt.frame_bytes = 4096; + opt.leaf_split_bytes = 256; + Crowdbtree t(opt); + auto local = RuntimeTreeAccess_for_tests::core(t).local_backend(); + local->gc_active.store(true); + uint64_t slot = 0; + for (int i = 0; i < 200; ++i) { + ASSERT_TRUE(t.apply(++slot, put_one(make_key(i), "v" + std::to_string(i))).ok()); + } + ASSERT_TRUE(t.flush().ok()); + ASSERT_TRUE(t.snapshot().ok()); + for (int i = 0; i < 200; ++i) { + if (!kept_keys.contains(make_key(i))) { + ASSERT_TRUE(t.apply(++slot, del_one(make_key(i))).ok()); } } + ASSERT_TRUE(t.flush().ok()); + // Keep clean old leaves in their original blocks and rotate both + // anchors; automatic GC must not consume the intended candidate. + ASSERT_TRUE(t.snapshot().ok()); + ASSERT_TRUE(t.snapshot().ok()); } -// R129: compact_sparse_blocks failure injection — a write or sync failure -// at any stage of the compaction snapshot must leave the previous -// generation fully recoverable. No source block is deleted before the -// finalizer, so a crash mid-compaction falls back to the prior anchor -// with all data intact. This parameterized test arms a fault at each -// write/sync call index in the compaction pass and verifies reopen. +void remove_free_compaction_blocks(TreeCore &core, BlockPageStore &store) +{ + const auto &local = core.local_backend(); + core.publisher().reclaim(); + core.binding()->account()->drain_retirements(); + const auto initial_blocks = local->ledger->stats().address_high_water / store.block_size(); + // Remove already-free blocks before arming the fault. The strict zero + // deletion assertion below then concerns the selected live source. + const LocalBlockGcLimits limits; + const auto metadata_bound = local_block_gc_metadata_bound(limits); + if (!metadata_bound) { + ADD_FAILURE() << "free-block metadata bound overflow"; + return; + } + auto reserve = detail::ResourceBudget::try_reserve( + core.binding()->account(), detail::BudgetClass::kReclaim, + *metadata_bound + detail::shared_allocation_bound<detail::AllocationReservation>()); + if (!reserve) { + ADD_FAILURE() << "free-block metadata reserve unavailable"; + return; + } + auto metadata = detail::AllocationReservation::create(std::move(*reserve)); + std::shared_ptr<AllocationBudget> backend; + ASSERT_TRUE(reserve_backend_operation(&store, core.binding()->account(), + {.workers = core.binding()->backend_cpu()->worker_count()}, &backend) + .ok()); + uint64_t cursor = 0; + bool complete = false; + for (uint64_t visited = 0; visited < initial_blocks; visited += limits.max_blocks) { + const auto result = collect_free_local_blocks(store, *local->ledger, cursor, limits, metadata, backend); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + if (result.complete) { + complete = true; + break; + } + ASSERT_GT(result.next_block, cursor); + cursor = result.next_block; + } + detail::seal_backend_budget(backend); + backend.reset(); + metadata->seal(); + metadata.reset(); + ASSERT_TRUE(complete); +} +} // namespace + +// A real sparse source block must survive a failed relocation checkpoint. class CompactSparseBlocksFailureInjectionTest : public ::testing::TestWithParam<std::tuple<int, bool>> { - // std::get<0> = write index to fail (-1 = don't fail writes) - // std::get<1> = fail sync (true) or write (false) }; TEST_P(CompactSparseBlocksFailureInjectionTest, ReopenFromPriorAnchorIsClean) { - auto [fail_write_idx, fail_sync] = GetParam(); + const auto [fail_index, fail_sync] = GetParam(); crowdb::tree_test::TempDir tmp("cmpfail_"); ASSERT_FALSE(tmp.path.empty()); - constexpr uint64_t blk = 8 * 1024; - + constexpr uint64_t blk = 8 * 1024; std::set<std::string> kept_keys; - for (int i = 0; i < 200; i += 20) { - kept_keys.insert(make_key(i)); - } - - // Phase 1: write data, snapshot, delete most keys, snapshot (sparse blocks). - { - std::unique_ptr<BlockPageStore> store; - ASSERT_TRUE(BlockPageStore::open_blocks(tmp.path, 0, 0, blk, 1, &store).ok()); - Config opt; - opt.page_store = store.get(); - opt.leaf_split_bytes = 256; - Crowdbtree t(opt); - - uint64_t slot = 0; - for (int i = 0; i < 200; ++i) { - ++slot; - ASSERT_TRUE(t.apply(slot, put_one(make_key(i), "v" + std::to_string(i))).ok()); + for (int i = 0; i < 200; ++i) { + if (i < 20 || i % 20 == 0) { + kept_keys.insert(make_key(i)); } - ASSERT_TRUE(t.flush().ok()); - ASSERT_TRUE(t.snapshot().ok()); - - for (int i = 0; i < 200; ++i) { - if (i % 20 != 0) { - ++slot; - ASSERT_TRUE(t.apply(slot, del_one(make_key(i))).ok()); - } - } - ASSERT_TRUE(t.flush().ok()); - t.set_gc_watermark(slot, slot); - ASSERT_TRUE(t.snapshot().ok()); } - // Phase 2: reopen with FaultyPageStore, arm the fault, run compaction. - // The compaction must fail (not silently succeed), and no block is - // deleted because the finalizer never runs. + ASSERT_NO_FATAL_FAILURE(prepare_sparse_compaction(tmp.path, kept_keys)); + { std::unique_ptr<BlockPageStore> store; ASSERT_TRUE(BlockPageStore::open_blocks(tmp.path, 0, 0, blk, 1, &store).ok()); - FaultyPageStore faulty(store.get()); - Config opt; - opt.page_store = &faulty; + TestCompactionFaultStore faulty(*store); + Config opt; + opt.page_store = store.get(); + opt.async_page_store = &faulty; + opt.frame_bytes = 4096; opt.leaf_split_bytes = 256; opt.merge_gc_block_free_threshold = 0.30; opt.merge_gc_max_relocation_bytes = 8 * 1024 * 1024; Crowdbtree t(opt); - - if (fail_sync) { - faulty.arm_sync_fault(fail_write_idx, FaultyPageStore::Fault::kFail); - } - else { - faulty.arm_write_fault(fail_write_idx, FaultyPageStore::Fault::kFail); - } - + auto &core = RuntimeTreeAccess_for_tests::core(t); + auto local = core.local_backend(); + local->gc_active.store(true); + ASSERT_NO_FATAL_FAILURE(remove_free_compaction_blocks(core, *store)); + const auto initial_blocks = local->ledger->stats().address_high_water / blk; + local->gc_cursor.store(0); + local->gc_active.store(false); + faulty.arm(fail_index, fail_sync); MergeGcStats stats; - Status status = t.compact_sparse_blocks(&stats); - EXPECT_FALSE(status.ok()); - // blocks_deleted must be 0 — the finalizer did not complete. + Status status; + for (uint64_t visited = 0; visited < initial_blocks; visited += 8) { + const auto previous = local->gc_cursor.load(); + status = t.compact_sparse_blocks(&stats); + EXPECT_EQ(stats.blocks_deleted, 0U); + if (!status.ok()) { + break; + } + const auto next = local->gc_cursor.load(); + if (next == 0) { + break; + } + ASSERT_GT(next, previous); + } + EXPECT_EQ(faulty.faults(), 1U); + EXPECT_EQ(status.code(), Code::kIoError) << status.to_string(); + EXPECT_EQ(stats.blocks_selected, 1U); EXPECT_EQ(stats.blocks_deleted, 0U); } - // Phase 3: reopen cleanly — prior anchor must be intact, all kept keys - // readable, no corruption from the partial compaction. { std::unique_ptr<BlockPageStore> store; ASSERT_TRUE(BlockPageStore::open_blocks(tmp.path, 0, 0, blk, 1, &store).ok()); Config opt; - opt.page_store = store.get(); - std::unique_ptr<Crowdbtree> t; - ASSERT_TRUE(Crowdbtree::open(opt, &t).ok()); - for (const auto &k : kept_keys) { - std::string v; - uint64_t s; - EXPECT_TRUE(t->get(Slice(k), &s, &v)) << "missing " << k << " after failed compaction + reopen"; + opt.page_store = store.get(); + opt.frame_bytes = 4096; + std::unique_ptr<Crowdbtree> tree; + const auto opened = Crowdbtree::open(opt, &tree); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + for (int i = 0; i < 200; ++i) { + const auto key = make_key(i); + std::string value; + uint64_t slot = 0; + const bool found = tree->get(Slice(key), &slot, &value); + EXPECT_EQ(found, kept_keys.contains(key)) << key << " after failed compaction"; + if (found) { + EXPECT_EQ(value, "v" + std::to_string(i)); + } } } } -// Fail at page/metadata writes and either durability barrier. INSTANTIATE_TEST_SUITE_P(CompactSparseBlocksFailureStages, CompactSparseBlocksFailureInjectionTest, - ::testing::Values(std::make_tuple(0, false), // first page write - std::make_tuple(2, false), // segment image - std::make_tuple(0, true), // first sync barrier - std::make_tuple(1, true))); // second sync barrier + ::testing::Values(std::make_tuple(0, false), std::make_tuple(2, false), + std::make_tuple(0, true), std::make_tuple(1, true))); // leaf/inner counts are persisted in the commit anchor and restored on open. TEST(Persist, LeafInnerCountSurvivesSnapshotOpen) diff --git a/lib/crowdb-tree/tests/integration/prepared_split_overlay_test.cpp b/lib/crowdb-tree/tests/integration/prepared_split_overlay_test.cpp new file mode 100644 index 000000000..78a4e421c --- /dev/null +++ b/lib/crowdb-tree/tests/integration/prepared_split_overlay_test.cpp @@ -0,0 +1,305 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/prepared_split_overlay.h" +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +#include <chrono> +#include <future> +#include <semaphore> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct CreditReceiver +{ + using receiver_concept = stdexec::receiver_tag; + std::binary_semaphore *entered; + std::binary_semaphore *release; + + void set_value() const noexcept + { + entered->release(); + release->acquire(); + } + + void set_error(Status /*unused*/) const noexcept + { + } + + void set_stopped() const noexcept + { + } +}; + +class PreparedSplit : public testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions runtime; + runtime.workers = 1; + ASSERT_TRUE(TreeRuntime::create(runtime, &options.runtime).ok()); + parent = std::make_unique<Crowdbtree>(options); + child = std::make_unique<Crowdbtree>(options); + } + + static Status put(Crowdbtree &tree, uint64_t slot, const std::string &key, const std::string &value) + { + return tree.apply(slot, Batch{.ops = {{.key = key, .kind = OpKind::kPut, .value = value}}}); + } + + Status arm(size_t records = 64, size_t bytes = 64 * 1024) + { + return PreparedSplitOverlay::arm(*TreeCoreAccess::get(*parent), *TreeCoreAccess::get(*child), + {.max_records = records, .max_bytes = bytes}, &token, &start); + } + + Config options; + std::unique_ptr<Crowdbtree> parent, child; + std::shared_ptr<PreparedSplitOverlay> token; + uint64_t start = UINT64_MAX; +}; + +TEST_F(PreparedSplit, ContinuousParentRotationAndFlushCannotLoseThePreparedTail) +{ + ASSERT_TRUE(put(*parent, 1, "base", "one").ok()); + ASSERT_TRUE(put(*child, 1, "base", "one").ok()); + auto status = arm(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(start, 1U); + ASSERT_TRUE(put(*parent, 2, "replayed", "two").ok()); + ASSERT_TRUE(put(*child, 2, "replayed", "two").ok()); + ASSERT_TRUE(parent->flush().ok()); + status = token->prepare(2); + ASSERT_TRUE(status.ok()) << status.to_string(); + for (uint64_t slot = 3; slot <= 8; ++slot) { + ASSERT_TRUE(put(*parent, slot, "key" + std::to_string(slot), std::to_string(slot)).ok()); + ASSERT_TRUE(parent->flush().ok()); + } + const auto active = TreeCoreAccess::get(*parent)->capture()->sources->tables.back(); + ASSERT_FALSE(active->closed()); + InstalledSplitOverlay installed; + status = token->try_install(8, &installed); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_FALSE(active->closed()); + EXPECT_EQ(installed.frontier, 8U); + EXPECT_EQ(child->get_view("base").value().to_string(), "one"); + for (uint64_t slot = 3; slot <= 8; ++slot) { + auto value = child->get_view("key" + std::to_string(slot)); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.slot(), slot); + } + ASSERT_TRUE(put(*parent, 9, "late", "parent").ok()); + ASSERT_TRUE(put(*child, 9, "late", "child").ok()); + EXPECT_EQ(child->get_view("late").value().to_string(), "child"); + status = parent->publish_split_memtable_view(installed.generation, 8, *child, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(child->clear_split_memtable_overlay(*parent).ok()); + ASSERT_TRUE(parent->release_split_memtable_view(installed.generation).ok()); + token.reset(); + parent.reset(); + EXPECT_EQ(child->get_view("late").value().to_string(), "child"); + ASSERT_TRUE(child->flush().ok()); +} + +TEST_F(PreparedSplit, InstallDoesNotWaitForCpuAndIsIdempotentOnlyForItsExactFrontier) +{ + auto status = arm(); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = token->prepare(0); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(put(*parent, 1, "tail", "one").ok()); + auto *core = TreeCoreAccess::get(*parent); + + struct Blocked + { + std::binary_semaphore started{0}, release{0}; + } blocked; + + ASSERT_TRUE(core->binding() + ->cpu() + ->try_submit( + core->binding()->lane(), + [](void *context) noexcept { + auto &wait = *static_cast<Blocked *>(context); + wait.started.release(); + wait.release.acquire(); + }, + &blocked) + .ok()); + blocked.started.acquire(); + InstalledSplitOverlay installed; + std::thread caller([&] { status = token->try_install(1, &installed); }); + caller.join(); // must not need the blocked native worker + blocked.release.release(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(token->committed_frontier(), 1U); + EXPECT_TRUE(token->try_install(1, &installed).ok()); + EXPECT_FALSE(token->try_install(2, &installed).ok()); + EXPECT_EQ(child->get_view("tail").value().to_string(), "one"); +} + +TEST_F(PreparedSplit, InstallDoesNotRunCreditWaitCallbacks) +{ + auto status = arm(); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = token->prepare(0); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto *core = TreeCoreAccess::get(*child); + core->binding()->account()->drain_retirements(); + core->binding()->drain(); + + struct Blocked + { + std::binary_semaphore entered{0}, release{0}; + } cpu, callback; + + ASSERT_TRUE(core->binding() + ->cpu() + ->try_submit( + core->binding()->lane(), + [](void *context) noexcept { + auto &wait = *static_cast<Blocked *>(context); + wait.entered.release(); + wait.release.acquire(); + }, + &cpu) + .ok()); + cpu.entered.acquire(); + constexpr auto category = detail::BudgetClass::kRouting; + constexpr auto index = static_cast<size_t>(category); + const auto account = core->binding()->account(); + const auto snapshot = account->snapshot(); + auto occupied = detail::ResourceBudget::try_reserve(account, category, + detail::ResourceBudget::capacity(account, category) - + snapshot.allocated[index] - snapshot.reserved[index]); + + detail::ResourceBudget::Request request(account, category, 1); + auto operation = + stdexec::connect(request.sender(), CreditReceiver{.entered = &callback.entered, .release = &callback.release}); + stdexec::start(operation); + std::promise<Status> result; + auto done = result.get_future(); + std::thread installer([&] { + InstalledSplitOverlay installed; + result.set_value(token->try_install(0, &installed)); + }); + const bool returned = done.wait_for(std::chrono::seconds(1)) == std::future_status::ready; + // Always unblock both participants before reporting an installation regression. + if (!returned) { + callback.release.release(); + cpu.release.release(); + } + installer.join(); + EXPECT_TRUE(returned); + EXPECT_TRUE(done.get().ok()); + if (returned) { + std::thread cleanup([&] { token.reset(); }); + callback.entered.acquire(); + callback.release.release(); + cpu.release.release(); + cleanup.join(); + } + else { + token.reset(); + } +} + +TEST_F(PreparedSplit, RecordOverflowExpiresTheTokenWithoutRejectingParentWrites) +{ + auto status = arm(1); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = token->prepare(0); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto before = TreeCoreAccess::get(*child)->capture(); + ASSERT_TRUE(put(*parent, 1, "a", "one").ok()); + ASSERT_TRUE(put(*parent, 2, "b", "two").ok()); + InstalledSplitOverlay installed; + EXPECT_FALSE(token->try_install(2, &installed).ok()); + EXPECT_EQ(TreeCoreAccess::get(*child)->capture(), before); + token.reset(); + EXPECT_TRUE(put(*child, 1, "own", "works").ok()); + EXPECT_EQ(parent->get_view("b").value().to_string(), "two"); +} + +TEST_F(PreparedSplit, ByteOverflowAndExplicitChildCloseLeaveItsRootUnchanged) +{ + auto status = arm(8, 16); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = token->prepare(0); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto before = TreeCoreAccess::get(*child)->capture(); + ASSERT_TRUE(put(*parent, 1, "a", std::string(64, 'v')).ok()); + InstalledSplitOverlay installed; + EXPECT_FALSE(token->try_install(1, &installed).ok()); + EXPECT_EQ(TreeCoreAccess::get(*child)->capture(), before); + token.reset(); + status = arm(); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(put(*child, 1, "a", std::string(64, 'v')).ok()); + status = token->prepare(1); + ASSERT_TRUE(status.ok()) << status.to_string(); + TreeCoreAccess::get(*child)->close_and_drain(); + EXPECT_FALSE(token->try_install(1, &installed).ok()); + token.reset(); + ASSERT_TRUE(put(*parent, 2, "alive", "yes").ok()); +} + +TEST_F(PreparedSplit, EmptyTailAndFacadeClosureKeepTheCapabilityMemorySafe) +{ + auto status = arm(); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = token->prepare(0); + ASSERT_TRUE(status.ok()) << status.to_string(); + InstalledSplitOverlay installed; + status = token->try_install(0, &installed); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(child->contiguous_slot(), 0U); + EXPECT_FALSE(TreeCoreAccess::get(*child)->capture()->sources->inheritance); + EXPECT_TRUE(token->try_install(0, &installed).ok()); + token.reset(); + ASSERT_TRUE(parent->release_split_memtable_view(installed.generation).ok()); + status = arm(); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = token->prepare(0); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::weak_ptr<TreeCore> parent_owner = TreeCoreAccess::get(*parent)->shared_from_this(); + std::weak_ptr<TreeCore> child_owner = TreeCoreAccess::get(*child)->shared_from_this(); + parent.reset(); + child.reset(); + EXPECT_FALSE(parent_owner.expired()); + EXPECT_FALSE(child_owner.expired()); + EXPECT_FALSE(token->try_install(0, &installed).ok()); + token.reset(); + EXPECT_TRUE(parent_owner.expired()); + EXPECT_TRUE(child_owner.expired()); +} + +TEST_F(PreparedSplit, RegistrationHighWaterCoversPreexistingOutOfOrderVersions) +{ + ASSERT_TRUE(put(*parent, 3, "future", "three").ok()); + ASSERT_TRUE(put(*parent, 1, "first", "one").ok()); + ASSERT_EQ(parent->contiguous_slot(), 1U); + auto status = arm(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(start, 3U); + ASSERT_TRUE(put(*parent, 2, "gap", "two").ok()); + ASSERT_TRUE(put(*child, 1, "first", "one").ok()); + ASSERT_TRUE(put(*child, 2, "gap", "two").ok()); + ASSERT_TRUE(put(*child, 3, "future", "three").ok()); + status = token->prepare(3); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(put(*parent, 4, "tail", "four").ok()); + InstalledSplitOverlay installed; + status = token->try_install(4, &installed); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(child->get_view("future").value().to_string(), "three"); + EXPECT_EQ(child->get_view("future").slot(), 3U); + EXPECT_EQ(child->get_view("tail").slot(), 4U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/primary_layout_admission_test.cpp b/lib/crowdb-tree/tests/integration/primary_layout_admission_test.cpp new file mode 100644 index 000000000..35d311e8a --- /dev/null +++ b/lib/crowdb-tree/tests/integration/primary_layout_admission_test.cpp @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/admitted_batch.h" +#include "btree/tree_apply_operation.h" +#include "btree/tree_core.h" +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +class PrimaryLayoutAdmission : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.frame_bytes = 4096; + config.max_key_size = 4096; + config.max_inline_value = 4096; + tree = std::make_unique<Crowdbtree>(config); + } + + Config config; + std::unique_ptr<Crowdbtree> tree; +}; + +TEST_F(PrimaryLayoutAdmission, UnrepresentableBatchLeavesEveryInputAndFrontierUnchanged) +{ + Batch invalid{ + .ops = {{.key = "first", .kind = OpKind::kPut, .value = "valid"}, + {.key = "oversized", .kind = OpKind::kPut, .value = std::string(4096, 'v')}} + }; + EXPECT_EQ(tree->apply(1, invalid).code(), Code::kInvalidArgument); + EXPECT_FALSE(tree->get_view("first").found()); + EXPECT_EQ(tree->contiguous_slot(), 0U); + EXPECT_EQ(tree->put(std::string(4096, 'k'), "value").code(), Code::kInvalidArgument); + EXPECT_EQ(tree->contiguous_slot(), 0U); + ASSERT_TRUE(tree->put("accepted", "value").ok()); + EXPECT_EQ(tree->contiguous_slot(), 1U); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(tree->get_view("accepted").value().to_string(), "value"); +} + +TEST_F(PrimaryLayoutAdmission, EncodedAndExternalInputsUseTheSameLayoutValidation) +{ + std::vector<Crowdbtree::encoded_op> encoded; + encoded.push_back({.key = "encoded", .cell = encode_cell_buf(1, OpKind::kPut, std::string(4096, 'v'))}); + EXPECT_EQ(tree->apply_encoded(1, std::move(encoded)).code(), Code::kInvalidArgument); + std::vector<Crowdbtree::external_op> external; + auto value = buffer::alloc(4096); + external.push_back({.key = "external", .flags = 0, .value = std::move(value)}); + EXPECT_EQ(tree->apply_external(1, std::move(external)).code(), Code::kInvalidArgument); + EXPECT_EQ(tree->contiguous_slot(), 0U); + EXPECT_FALSE(tree->get_view("encoded").found()); + EXPECT_FALSE(tree->get_view("external").found()); +} + +TEST_F(PrimaryLayoutAdmission, PreproposalAndReplayRejectBeforeChargingOrClaimingSource) +{ + auto *core = TreeCoreAccess::get(*tree); + const std::string value(4096, 'v'); + const auto input = [&](size_t) { return CoreWriteInput{.key = "key", .value = value}; }; + core->binding()->drain(); + const auto baseline = core->binding()->account()->snapshot(); + for (const auto category : {detail::BudgetClass::kAdmittedWriter, detail::BudgetClass::kGapRepair}) { + std::shared_ptr<TreeApplyOperation> operation; + EXPECT_EQ(core->prepare_apply(1, input, category, 0, &operation).code(), Code::kInvalidArgument); + EXPECT_FALSE(operation); + SourceWriteOptions options{.account = core->binding()->account()}; + std::shared_ptr<AdmittedBatch> batch; + EXPECT_EQ(AdmittedBatch::create(config, options, 20'000, 1, input, category, &batch).code(), + Code::kInvalidArgument); + EXPECT_FALSE(batch); + } + core->binding()->drain(); + const auto after = core->binding()->account()->snapshot(); + EXPECT_EQ(after.allocated, baseline.allocated); + EXPECT_EQ(after.reserved, baseline.reserved); + EXPECT_EQ(tree->contiguous_slot(), 0U); + EXPECT_EQ(core->capture()->sources->tables.front()->count(), 0U); +} + +TEST_F(PrimaryLayoutAdmission, OverflowAndTombstoneNeedOnlyTheirActualPrimaryCells) +{ + const std::string large(20'000, 'v'); + ASSERT_TRUE(tree->put("overflow", large).ok()); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(tree->get_view("overflow").value().to_string(), large); + Batch erase{.ops = {{.key = "overflow", .kind = OpKind::kDelete, .value = std::string(4096, 'x')}}}; + ASSERT_TRUE(tree->apply(2, erase).ok()); + EXPECT_FALSE(tree->get_view("overflow").found()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_byte_export_test.cpp b/lib/crowdb-tree/tests/integration/published_byte_export_test.cpp new file mode 100644 index 000000000..e675baf4c --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_byte_export_test.cpp @@ -0,0 +1,218 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/snapshot/snapshot_io.h" +#include "local_checkpoint_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestByteBackend +{ + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + + TestByteBackend() + { + store.device = device; + } +}; + +struct TestByteExport +{ + std::shared_ptr<TestByteBackend> backend = std::make_shared<TestByteBackend>(); + Config config; + std::unique_ptr<Crowdbtree> tree; + + TestByteExport() + { + EXPECT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = backend->device.get(); + config.async_page_store = &backend->store; + config.backend_owner = backend; + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + } + + [[nodiscard]] Status open() + { + return Crowdbtree::open(config, &tree); + } + + static Status collect(SnapshotExport &exporter, std::string *wire, std::vector<SnapshotChunk> *retained = nullptr) + { + bool done = false; + while (!done) { + SnapshotChunk chunk; + auto status = exporter.next_owned(&chunk, &done); + if (!status.ok()) { + return status; + } + if (!chunk.bytes.empty()) { + if (!chunk.owner || !chunk.allocation_budget) { + return Status::internal_error("runtime export returned unaccounted bytes"); + } + wire->append(reinterpret_cast<const char *>(chunk.bytes.data()), chunk.bytes.size()); + } + if (retained != nullptr) { + retained->push_back(std::move(chunk)); + } + } + return Status::Ok(); + } +}; + +TEST(PublishedByteExport, PortableMatchesExistingWireAndRetainsOneCapturedCoverage) +{ + TestByteExport test; + auto opened = test.open(); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + const std::string value(12000, 'v'); + ASSERT_TRUE(test.tree + ->apply(1, + Batch{ + .ops = {{.key = "a", .kind = OpKind::kPut, .value = value}, + {.key = "empty", .kind = OpKind::kPut, .value = ""}, + {.key = "gone", .kind = OpKind::kPut, .value = "old"}} + }) + .ok()); + ASSERT_TRUE(test.tree->apply(2, Batch{.ops = {{.key = "gone", .kind = OpKind::kDelete, .value = ""}}}).ok()); + std::unique_ptr<SnapshotExport> exporter; + auto status = snapshot_export_begin(*test.tree, snapshot_format::kPortable, 7, &exporter); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(exporter->at_slot(), 2U); + ASSERT_TRUE(test.tree->apply(3, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "changed"}}}).ok()); + ASSERT_TRUE(test.tree->flush().ok()); + std::vector<leaf_entry> expected; + expected.push_back({.key = "a", .cell = encode_cell_buf(1, OpKind::kPut, value)}); + expected.push_back({.key = "empty", .cell = encode_cell_buf(1, OpKind::kPut, "")}); + expected.push_back({.key = "gone", .cell = encode_cell_buf(2, OpKind::kDelete)}); + SnapshotExport reference(std::make_shared<Snapshot>(2, std::move(expected)), 4096); + std::string expected_wire; + bool done = false; + while (!done) { + std::string piece; + ASSERT_TRUE(reference.next_chunk(&piece, &done).ok()); + expected_wire += piece; + } + EXPECT_EQ(exporter->total_bytes(), expected_wire.size()); + EXPECT_EQ(exporter->final_crc32c(), reference.final_crc32c()); + std::string wire; + std::vector<SnapshotChunk> retained; + status = TestByteExport::collect(*exporter, &wire, &retained); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(wire, expected_wire); + EXPECT_EQ(exporter->offset(), exporter->total_bytes()); + EXPECT_EQ(test.config.runtime->snapshot().reserved.io_operations, 0U); + exporter.reset(); + test.tree.reset(); + std::string owned_again; + for (const auto &chunk : retained) { + if (!chunk.bytes.empty()) { + owned_again.append(reinterpret_cast<const char *>(chunk.bytes.data()), chunk.bytes.size()); + } + } + EXPECT_EQ(owned_again, wire); +} + +TEST(PublishedByteExport, NativeColdOverflowUsesTheSamePidLeaseForBothPasses) +{ + TestByteExport test; + auto opened = test.open(); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + const std::string value(14000, 'n'); + ASSERT_TRUE(test.tree->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = value}}}).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + std::unique_ptr<SnapshotExport> exporter; + auto status = snapshot_export_begin(*test.tree, snapshot_format::kNative, 4093, &exporter); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto next = TreeCoreAccess::get(*test.tree)->capture()->next_page_id; + std::string wire; + status = TestByteExport::collect(*exporter, &wire); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(TreeCoreAccess::get(*test.tree)->capture()->next_page_id, next); + EXPECT_EQ(wire.size(), exporter->total_bytes()); + const auto crc = crowdb::common::crc32c(reinterpret_cast<const uint8_t *>(wire.data()), wire.size() - 4); + EXPECT_EQ(crc, exporter->final_crc32c()); + Config target; + target.frame_bytes = 4096; + Crowdbtree copy(target); + SnapshotImport importer(copy); + ASSERT_TRUE(importer.feed(wire).ok()); + uint64_t slot = 0; + status = importer.finish(&slot); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(slot, 1U); + std::string restored; + ASSERT_TRUE(copy.get("a", &slot, &restored)); + EXPECT_EQ(restored, value); +} + +TEST(PublishedByteExport, NativeFreshOverflowRewindsWithoutReservingAnotherInterval) +{ + TestByteExport test; + auto opened = test.open(); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + const std::string value(14000, 'f'); + ASSERT_TRUE(test.tree->put("fresh", value).ok()); + ASSERT_TRUE(test.tree->flush().ok()); + auto *core = TreeCoreAccess::get(*test.tree); + const auto before = core->capture()->next_page_id; + std::unique_ptr<SnapshotExport> exporter; + auto status = snapshot_export_begin(*test.tree, snapshot_format::kNative, 4093, &exporter); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto leased = core->capture()->next_page_id; + EXPECT_GT(leased, before); + std::string wire; + status = TestByteExport::collect(*exporter, &wire); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(core->capture()->next_page_id, leased); + Config target; + target.frame_bytes = 4096; + Crowdbtree copy(target); + SnapshotImport importer(copy); + ASSERT_TRUE(importer.feed(wire).ok()); + status = importer.finish(nullptr); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::string restored; + uint64_t slot = 0; + ASSERT_TRUE(copy.get("fresh", &slot, &restored)); + EXPECT_EQ(restored, value); +} + +TEST(PublishedByteExport, EmptyPortableSupportsEveryHeaderAndTrailerByteBoundary) +{ + TestByteExport test; + auto opened = test.open(); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + std::unique_ptr<SnapshotExport> exporter; + auto status = snapshot_export_begin(*test.tree, snapshot_format::kPortable, 1, &exporter); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::string wire; + status = TestByteExport::collect(*exporter, &wire); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(wire.size(), 29U); + EXPECT_EQ(exporter->total_bytes(), 29U); + EXPECT_EQ(exporter->offset(), 29U); +} + +TEST(PublishedByteExport, OversizeChunkFailsBeforeHandoffOrCapture) +{ + TestByteExport test; + auto opened = test.open(); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + ASSERT_TRUE(test.tree->put("a", "pending").ok()); + auto *core = TreeCoreAccess::get(*test.tree); + const auto before = core->capture(); + std::unique_ptr<SnapshotExport> exporter; + auto status = snapshot_export_begin(*test.tree, snapshot_format::kPortable, kSnapshotChunkBytes + 1, &exporter); + EXPECT_EQ(status.code(), Code::kInvalidArgument); + EXPECT_FALSE(exporter); + EXPECT_EQ(core->capture(), before); + EXPECT_EQ(core->capture()->coverage, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_native_collect_test.cpp b/lib/crowdb-tree/tests/integration/published_native_collect_test.cpp new file mode 100644 index 000000000..4be78c4a8 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_native_collect_test.cpp @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/crowdb-tree.h" +#include "snapshot/published_native_collect.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestNativeCollect +{ + Config config; + std::unique_ptr<Crowdbtree> tree; + + TestNativeCollect() + { + EXPECT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.frame_bytes = 4096; + config.max_inline_value = 64; + } + + [[nodiscard]] Status open() + { + tree = std::make_unique<Crowdbtree>(config); + return Status::Ok(); + } +}; + +TEST(PublishedNativeCollect, PublicCompatibilityPayloadsRemainChargedAfterTreeClose) +{ + TestNativeCollect test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(12000, 'c'); + ASSERT_TRUE(test.tree->put("a", value).ok()); + std::vector<NativeFrame> frames; + uint64_t root = 0; + uint64_t slot = 0; + uint64_t next = 0; + status = test.tree->collect_native_frames(&frames, &root, &slot, &next); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(frames.empty()); + size_t capacity = 0; + for (const auto &frame : frames) { + EXPECT_TRUE(frame.allocation_owner); + EXPECT_EQ(frame.durable_addr, kNoAddr); + capacity += frame.frame.capacity(); + } + test.tree.reset(); + EXPECT_GE(test.config.runtime->snapshot().allocated.routing_bytes, capacity); + Config destination; + destination.frame_bytes = 4096; + Crowdbtree copy(destination); + status = copy.install_snapshot_native(std::move(frames), root, slot, next); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::string actual; + ASSERT_TRUE(copy.get("a", nullptr, &actual)); + EXPECT_EQ(actual, value); +} + +TEST(PublishedNativeCollect, CapacityFailurePreservesEveryCallerOutput) +{ + TestNativeCollect test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(test.tree->put("a", std::string(12000, 'b')).ok()); + auto core = TreeCoreAccess::get(*test.tree)->shared_from_this(); + std::vector<NativeFrame> frames; + frames.push_back({ + .page_id = 987, .frame = {1, 2, 3}, + .durable_addr = kNoAddr, .durable_plen = 0, .inherited = false + }); + NativeStreamInfo info{ + .root_page_id = 111, .at_slot = 222, .next_page_id = 333, .frame_count = 444, .frame_count_known = true}; + status = collect_published_frames(core, 1024, &frames, &info); + EXPECT_EQ(status.code(), Code::kResourceExhausted); + ASSERT_EQ(frames.size(), 1U); + EXPECT_EQ(frames[0].page_id, 987U); + EXPECT_EQ(frames[0].frame, (std::vector<uint8_t>{1, 2, 3})); + EXPECT_EQ(info.root_page_id, 111U); + EXPECT_EQ(info.at_slot, 222U); + EXPECT_EQ(info.next_page_id, 333U); + EXPECT_EQ(info.frame_count, 444U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_native_test.cpp b/lib/crowdb-tree/tests/integration/published_native_test.cpp new file mode 100644 index 000000000..ea94f486e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_native_test.cpp @@ -0,0 +1,241 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/maptable/compressor.h" +#include "local_checkpoint_test_support.h" +#include "snapshot/published_native.h" + +#include <set> + +namespace crowdb::tree +{ +namespace +{ +struct TestNativeBackend +{ + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + + TestNativeBackend() + { + store.device = device; + } +}; + +struct TestNativeExport +{ + std::shared_ptr<TestNativeBackend> backend = std::make_shared<TestNativeBackend>(); + Config config; + std::unique_ptr<Crowdbtree> tree; + + explicit TestNativeExport(bool owned = true) + { + EXPECT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = backend->device.get(); + config.async_page_store = &backend->store; + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + if (owned) { + config.backend_owner = backend; + } + } + + Status open() + { + return Crowdbtree::open(config, &tree); + } + + [[nodiscard]] Status put(uint64_t slot, std::string key, std::string value) const + { + return tree->apply(slot, + Batch{.ops = {{.key = std::move(key), .kind = OpKind::kPut, .value = std::move(value)}}}); + } + + static void append(const NativeFrameView &frame, std::vector<NativeFrame> *out) + { + ASSERT_TRUE(frame.owner); + ASSERT_FALSE(frame.bytes.empty()); + EXPECT_TRUE(frame_validate(frame.bytes.data(), frame.bytes.size())); + EXPECT_EQ(frame_u64(frame.bytes.data(), fh::kSelfpage_id), frame.page_id); + EXPECT_EQ(frame.durable_addr, kNoAddr); + EXPECT_EQ(frame.durable_plen, 0U); + EXPECT_FALSE(frame.inherited); + out->push_back({.page_id = frame.page_id, + .frame = std::vector<uint8_t>(frame.bytes.begin(), frame.bytes.end()), + .durable_addr = kNoAddr, + .durable_plen = 0, + .inherited = false}); + } + + static void import_and_read(std::vector<NativeFrame> frames, NativeStreamInfo info, const std::string &expected) + { + Config destination; + destination.frame_bytes = 4096; + Crowdbtree copy(destination); + auto status = + copy.install_snapshot_native(std::move(frames), info.root_page_id, info.at_slot, info.next_page_id); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(copy.get("a", &slot, &value)); + EXPECT_EQ(value, expected); + EXPECT_EQ(slot, 1U); + } +}; + +TEST(PublishedNative, FreshOverflowHasExactFrameCountAndStableOwnedWindows) +{ + TestNativeExport test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string old_value(14000, 'v'); + ASSERT_TRUE(test.put(1, "a", old_value).ok()); + std::unique_ptr<NativeFrameIterator> iterator; + status = test.tree->open_native_frame_iterator(nullptr, &iterator); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto info = iterator->stream_info(); + ASSERT_TRUE(info.frame_count_known); + EXPECT_EQ(info.at_slot, 1U); + EXPECT_GT(info.frame_count, 2U); + const auto leased_next = TreeCoreAccess::get(*test.tree)->capture()->next_page_id; + EXPECT_EQ(info.next_page_id, leased_next); + ASSERT_TRUE(test.put(2, "a", "newer").ok()); + ASSERT_TRUE(test.tree->flush().ok()); + + std::vector<NativeFrameView> retained; + std::vector<NativeFrame> frames; + std::set<uint64_t> pids; + bool done = false; + while (!done) { + std::optional<NativeFrameView> frame; + status = iterator->next_owned(&frame, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (frame) { + EXPECT_TRUE(pids.insert(frame->page_id).second); + TestNativeExport::append(*frame, &frames); + retained.push_back(std::move(*frame)); + } + EXPECT_EQ(test.config.runtime->snapshot().reserved.io_operations, 0U); + } + ASSERT_EQ(frames.size(), info.frame_count); + for (size_t i = 0; i < retained.size(); ++i) { + EXPECT_EQ(std::vector<uint8_t>(retained[i].bytes.begin(), retained[i].bytes.end()), frames[i].frame); + } + TestNativeExport::import_and_read(std::move(frames), info, old_value); +} + +TEST(PublishedNative, RecoveredOverflowRemainsReadableAfterOwnedBackendTreeCloses) +{ + TestNativeExport test; + ASSERT_TRUE(test.open().ok()); + const std::string value(12000, 'c'); + ASSERT_TRUE(test.put(1, "a", value).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + std::unique_ptr<NativeFrameIterator> iterator; + auto status = test.tree->open_native_frame_iterator(nullptr, &iterator); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto info = iterator->stream_info(); + test.tree.reset(); + EXPECT_EQ(test.open().code(), Code::kUnavailable); + std::vector<NativeFrame> frames; + bool done = false; + while (!done) { + std::optional<NativeFrameView> frame; + status = iterator->next_owned(&frame, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (frame) { + TestNativeExport::append(*frame, &frames); + } + } + EXPECT_EQ(frames.size(), info.frame_count); + TestNativeExport::import_and_read(std::move(frames), info, value); + iterator.reset(); + status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +TEST(PublishedNative, RawBackendIdleCursorDoesNotBlockCloseAndKeepsReturnedBytes) +{ + TestNativeExport test(false); + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "a", std::string(12000, 'r')).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + std::unique_ptr<NativeFrameIterator> iterator; + auto status = test.tree->open_native_frame_iterator(nullptr, &iterator); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::optional<NativeFrameView> first; + bool done = false; + status = iterator->next_owned(&first, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (!first.has_value()) { + FAIL() << "native cursor returned no owned frame"; + } + ASSERT_FALSE(done); + const auto &retained = first.value(); + const std::vector<uint8_t> copy(retained.bytes.begin(), retained.bytes.end()); + test.tree.reset(); + std::optional<NativeFrameView> after; + status = iterator->next_owned(&after, &done); + EXPECT_EQ(status.code(), Code::kUnavailable) << status.to_string(); + EXPECT_FALSE(after); + EXPECT_EQ(std::vector<uint8_t>(retained.bytes.begin(), retained.bytes.end()), copy); + status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +TEST(PublishedNative, FrameCountLimitRejectsBeforeReservingOverflowIds) +{ + TestNativeExport test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "a", std::string(12000, 'b')).ok()); + ASSERT_TRUE(test.tree->flush().ok()); + auto core = TreeCoreAccess::get(*test.tree)->shared_from_this(); + const auto before = core->capture()->next_page_id; + std::shared_ptr<PublishedNativeCursor> cursor; + auto status = PublishedNativeCursor::create(core, {.max_frames = 1, .max_dependency_steps = 16}, &cursor); + EXPECT_EQ(status.code(), Code::kResourceExhausted) << status.to_string(); + EXPECT_FALSE(cursor); + EXPECT_EQ(core->capture()->next_page_id, before); + EXPECT_EQ(test.config.runtime->snapshot().reserved.io_operations, 0U); +} + +TEST(PublishedNative, CorruptColdOverflowCannotBeReportedAsCompleteOutput) +{ + TestNativeExport test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "a", std::string(12000, 'x')).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + auto *core = TreeCoreAccess::get(*test.tree); + const auto view = core->capture(); + const auto page = view->mapping.get(view->root_page_id); + ASSERT_TRUE(page && page->leaf && !page->leaf->bases.empty()); + const auto primary = page->leaf->bases.front().backing; + ASSERT_TRUE(primary && primary->overflow && !primary->overflow->pages.empty()); + const auto broken = primary->overflow->pages.begin()->second.backing; + std::unique_ptr<NativeFrameIterator> iterator; + auto status = test.tree->open_native_frame_iterator(nullptr, &iterator); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::vector<uint8_t> blob(broken->length); + ASSERT_TRUE(test.backend->device->read_at(broken->address, blob.data(), blob.size()).ok()); + blob.back() ^= 0xff; + // Corrupt inside the logical blob rather than IU alignment padding. + blob[kDurableBlobHeader + 32] ^= 0xff; + ASSERT_TRUE(test.backend->device->write_at(broken->address, blob.data(), blob.size()).ok()); + bool complete = false; + uint64_t emitted = 0; + do { + std::optional<NativeFrameView> frame; + status = iterator->next_owned(&frame, &complete); + if (frame) { + ++emitted; + } + } while (status.ok() && !complete); + EXPECT_EQ(status.code(), Code::kCorruption) << status.to_string(); + EXPECT_FALSE(complete); + EXPECT_LT(emitted, iterator->stream_info().frame_count); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_read_delta_test.cpp b/lib/crowdb-tree/tests/integration/published_read_delta_test.cpp new file mode 100644 index 000000000..8f530c3a0 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_read_delta_test.cpp @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "page_load_delta_test_support.h" + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(PublishedReadDelta, GetUsesFoldedSnapshotAndCoverageOnlyFiltersMemTables) +{ + TestDeltaLeaf fixture; + auto loader = fixture.loader(); + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(fixture.reference); + auto table = std::make_shared<MemTable>(); + ASSERT_TRUE(table->upsert("a", 7, encode_cell_buf(7, OpKind::kPut, "covered"))); + ASSERT_TRUE(table->upsert("b", 8, encode_cell_buf(8, OpKind::kPut, "active"))); + const auto captured = view(leaf, {table}, 7); + for (const auto &[key, value] : std::array{ + std::pair{"a", "delta" }, + std::pair{"b", "active" }, + std::pair{"new", "first" }, + std::pair{"tie", "base-tie"} + }) { + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(captured, loader, key, {}, &request).ok()); + fixture.complete(request->sender()); + ASSERT_TRUE(request->result().found); + EXPECT_EQ(request->result().cell.value, Slice(value)); + } + for (const auto *key : {"c", "zero"}) { + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(captured, loader, key, {}, &request).ok()); + fixture.complete(request->sender()); + EXPECT_FALSE(request->result().found); + EXPECT_TRUE(request->result().cell.tombstone()); + } +} + +TEST(PublishedReadDelta, ForwardAndReverseScanMergeBaseDeltaAndDeletes) +{ + TestDeltaLeaf fixture; + auto loader = fixture.loader(); + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(fixture.reference); + for (bool reverse : {false, true}) { + std::shared_ptr<PublishedScan> request; + ASSERT_TRUE(PublishedScan::create(view(leaf, {}, 7), loader, {}, reverse, {.max_entries = 1}, &request).ok()); + std::vector<std::pair<std::string, std::string>> actual; + for (size_t i = 0; i < 20; ++i) { + fixture.complete(request->next_sender()); + for (const auto &entry : request->result().entries) { + actual.emplace_back(Slice(entry.key).to_string(), entry.cell.value.to_string()); + } + if (request->result().done) { + break; + } + } + ASSERT_TRUE(request->result().done); + std::vector<std::pair<std::string, std::string>> expected{ + {"a", "delta" }, + {"b", "base-newer"}, + {"new", "first" }, + {"tie", "base-tie" } + }; + if (reverse) { + std::reverse(expected.begin(), expected.end()); + } + EXPECT_EQ(actual, expected); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_snapshot_context_test.cpp b/lib/crowdb-tree/tests/integration/published_snapshot_context_test.cpp new file mode 100644 index 000000000..6c2f53138 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_snapshot_context_test.cpp @@ -0,0 +1,153 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "local_checkpoint_test_support.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestSnapshotBackend +{ + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + + TestSnapshotBackend() + { + store.device = device; + } +}; + +struct TestSnapshotContext +{ + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<TestSnapshotBackend> backend = std::make_shared<TestSnapshotBackend>(); + Config config; + std::shared_ptr<TreeCore> core; + + explicit TestSnapshotContext(bool owned) + { + EXPECT_TRUE(TreeRuntime::create({}, &runtime).ok()); + config.runtime = runtime; + config.page_store = backend->device.get(); + config.async_page_store = &backend->store; + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + if (owned) { + config.backend_owner = backend; + } + } + + Status open() + { + return TreeCore::open(config, &core); + } + + Status put(uint64_t slot, const std::string &value) + { + return core->apply(slot, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = value}}}); + } +}; + +TEST(PublishedSnapshotContext, ExcludesLiveSourcesAndKeepsOneImmutableL1Coverage) +{ + TestSnapshotContext test(true); + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(test.put(1, "old").ok()); + ASSERT_TRUE(test.core->flush().ok()); + ASSERT_TRUE(test.put(2, "new").ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + status = PublishedSnapshotContext::capture(test.core, &snapshot); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(snapshot->view()->coverage, 1U); + EXPECT_TRUE(snapshot->view()->sources->tables.empty()); + EXPECT_EQ(snapshot->view()->sources->floor, 1U); + ASSERT_TRUE(test.core->flush().ok()); + GetView result; + status = snapshot->get("a", &result); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(result.found()); + EXPECT_EQ(result.slot(), 1U); + EXPECT_EQ(result.value().to_string(), "old"); +} + +TEST(PublishedSnapshotContext, SharedBackendAndDomainOutliveCoreWithoutPinningRegistry) +{ + TestSnapshotContext test(true); + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(12000, 'x'); + ASSERT_TRUE(test.put(1, value).ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + status = PublishedSnapshotContext::capture(test.core, &snapshot); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(snapshot->detached()); + std::weak_ptr<LocalBackendState> old_local = test.core->local_backend(); + test.core.reset(); + EXPECT_TRUE(old_local.expired()); + EXPECT_EQ(test.open().code(), Code::kUnavailable); + GetView result; + status = snapshot->get("a", &result); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(result.found()); + EXPECT_EQ(result.value().to_string(), value); + EXPECT_EQ(result.slot(), 1U); + snapshot.reset(); + status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(result.value().to_string(), value); +} + +TEST(PublishedSnapshotContext, RawBackendIdleHandleDoesNotBlockCloseAndOwnedReadSurvives) +{ + TestSnapshotContext test(false); + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(test.put(1, "retained").ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + status = PublishedSnapshotContext::capture(test.core, &snapshot); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(snapshot->detached()); + GetView owned; + status = snapshot->get("a", &owned); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(owned.found()); + test.core.reset(); + GetView after; + status = snapshot->get("a", &after); + EXPECT_EQ(status.code(), Code::kUnavailable) << status.to_string(); + EXPECT_FALSE(after.found()); + EXPECT_EQ(owned.value().to_string(), "retained"); + status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +TEST(PublishedSnapshotContext, OversizeWindowFailsBeforeIoOrAdmissionAndSingleIoGetsCanRepeat) +{ + TestSnapshotContext test(true); + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(test.put(1, "one").ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(PublishedSnapshotContext::capture(test.core, &snapshot).ok()); + SnapshotReadWindow window; + EXPECT_EQ(snapshot->prepare_window(SIZE_MAX, &window).code(), Code::kResourceExhausted); + EXPECT_FALSE(window.loader); + EXPECT_FALSE(window.ticket); + GetView first; + status = snapshot->get("a", &first); + ASSERT_TRUE(status.ok()) << status.to_string(); + GetView second; + status = snapshot->get("a", &second); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(first.value().to_string(), "one"); + EXPECT_EQ(second.value().to_string(), "one"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_snapshot_domain_test.cpp b/lib/crowdb-tree/tests/integration/published_snapshot_domain_test.cpp new file mode 100644 index 000000000..5cbe53dae --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_snapshot_domain_test.cpp @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/btree/range_rebuild.h" +#include "crowdb-tree/btree/scan_packed.h" +#include "snapshot/published_snapshot_entries.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestSnapshotDomain +{ + std::shared_ptr<MemPageStore> store = std::make_shared<MemPageStore>(1); + Config config; + std::unique_ptr<Crowdbtree> tree; + + Status open() + { + auto status = TreeRuntime::create({}, &config.runtime); + if (!status.ok()) { + return status; + } + config.page_store = store.get(); + config.backend_owner = store; + status = Crowdbtree::open(config, &tree); + if (!status.ok()) { + return status; + } + for (const auto *key : {"", "b", "m", "z"}) { + status = tree->put(key, "value"); + if (!status.ok()) { + return status; + } + } + status = tree->snapshot(); + if (!status.ok()) { + return status; + } + tree.reset(); + config.key_range = KeyRange::bounded("a", "m"); + return Crowdbtree::open(config, &tree); + } +}; + +TEST(PublishedSnapshotDomain, DetachedGetScanAndEntriesKeepTheCapturedLogicalRange) +{ + TestSnapshotDomain test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(TreeCoreAccess::get(*test.tree)->capture_snapshot(&snapshot).ok()); + auto public_view = test.tree->snapshot_view(); + test.tree.reset(); + test.config.key_range = KeyRange::unbounded(); + GetView value; + ASSERT_TRUE(snapshot->get("b", &value).ok()); + EXPECT_EQ(value.value().to_string(), "value"); + ASSERT_TRUE(snapshot->get("z", &value).ok()); + EXPECT_FALSE(value.found()); + EXPECT_FALSE(public_view->get("z", nullptr, nullptr)); + ASSERT_EQ(public_view->entries().size(), 1U); + EXPECT_EQ(public_view->entries()[0].key, "b"); + for (const bool reverse : {false, true}) { + ScanPackedBuf packed; + size_t count = 0; + bool truncated = false; + CoreScanOptions options; + options.reverse = reverse; + const auto result = snapshot->scan(options, &packed, &count, &truncated); + ASSERT_TRUE(result.ok()) << result.to_string(); + EXPECT_EQ(count, 1U); + EXPECT_FALSE(truncated); + } + std::shared_ptr<PublishedSnapshotEntries> entries; + ASSERT_TRUE(PublishedSnapshotEntries::create(snapshot, &entries).ok()); + bool complete = false; + std::vector<std::string> keys; + while (!complete) { + std::optional<PublishedScanEntry> entry; + ASSERT_TRUE(entries->next(&entry, &complete).ok()); + if (entry) { + keys.push_back(Slice(entry->key).to_string()); + } + } + EXPECT_EQ(keys, (std::vector<std::string>{"b"})); +} + +TEST(PublishedSnapshotDomain, UnboundedRebuildCannotWidenABoundedRecoveredSource) +{ + TestSnapshotDomain test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + MemPageStore target(1); + Config config; + config.page_store = ⌖ + std::unique_ptr<Crowdbtree> destination; + status = rebuild_range(*test.tree, KeyRange::unbounded(), config, &destination); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(destination->get("b", nullptr, nullptr)); + EXPECT_FALSE(destination->get("z", nullptr, nullptr)); + EXPECT_EQ(destination->snapshot_view()->entries().size(), 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_snapshot_entries_range_test.cpp b/lib/crowdb-tree/tests/integration/published_snapshot_entries_range_test.cpp new file mode 100644 index 000000000..084d0d98b --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_snapshot_entries_range_test.cpp @@ -0,0 +1,183 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/crowdb-tree.h" +#include "snapshot/published_snapshot_entries.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestSnapshotRange +{ + Config config; + std::unique_ptr<Crowdbtree> tree; + std::shared_ptr<PublishedSnapshotContext> snapshot; + + Status prepare() + { + auto status = TreeRuntime::create({}, &config.runtime); + if (!status.ok()) { + return status; + } + tree = std::make_unique<Crowdbtree>(config); + status = tree->apply(1, Batch{ + .ops = {{.key = "", .kind = OpKind::kPut, .value = "empty"}, + {.key = "a", .kind = OpKind::kPut, .value = "first"}, + {.key = "b", .kind = OpKind::kDelete, .value = ""}, + {.key = "c", .kind = OpKind::kPut, .value = "last"}} + }); + if (status.ok()) { + status = tree->flush(); + } + return status.ok() ? TreeCoreAccess::get(*tree)->capture_snapshot(&snapshot) : status; + } + + Status keys(const KeyRange &range, std::vector<std::string> *out) const + { + std::shared_ptr<PublishedSnapshotEntries> cursor; + auto status = PublishedSnapshotEntries::create(snapshot, &cursor, range); + if (!status.ok()) { + return status; + } + bool done = false; + while (!done) { + std::optional<PublishedScanEntry> entry; + status = cursor->next(&entry, &done); + if (!status.ok()) { + return status; + } + if (entry) { + out->push_back(Slice(entry->key).to_string()); + EXPECT_EQ(entry->cell.tombstone(), Slice(entry->key) == Slice("b")); + } + } + return Status::Ok(); + } +}; + +TEST(PublishedSnapshotEntriesRange, HalfOpenAndPresentEmptyBoundsKeepTheirMeaning) +{ + TestSnapshotRange test; + auto status = test.prepare(); + ASSERT_TRUE(status.ok()) << status.to_string(); + for (const auto &bounds : { + std::pair{"a", "c"}, + std::pair{"", "a"}, + std::pair{"b", "b"} + }) { + std::vector<std::string> keys; + status = test.keys(KeyRange::bounded(bounds.first, bounds.second), &keys); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (std::string_view(bounds.first) == "a") { + EXPECT_EQ(keys, (std::vector<std::string>{"a", "b"})); + } + else if (std::string_view(bounds.first).empty()) { + EXPECT_EQ(keys, (std::vector<std::string>{""})); + } + else { + EXPECT_TRUE(keys.empty()); + } + } +} + +TEST(PublishedSnapshotEntriesRange, FixedSmallViewFitsTwoCursorsWithOnlySixteenMiBRemaining) +{ + TestSnapshotRange test; + auto status = test.prepare(); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto &core = *TreeCoreAccess::get(*test.tree); + // This newer source value is outside the fixed snapshot. It cannot enlarge + // the snapshot's L1 decode bound or add an L0 iterator to either cursor. + ASSERT_TRUE( + test.tree->apply(2, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = std::string(1024 * 1024, 'n')}}}) + .ok()); + core.binding()->drain(); + const auto account = core.binding()->account(); + const auto used = account->snapshot(); + constexpr auto category = detail::BudgetClass::kRouting; + constexpr auto index = static_cast<size_t>(category); + constexpr size_t available = 16 * 1024 * 1024; + const auto capacity = detail::ResourceBudget::capacity(account, category); + ASSERT_GT(capacity, used.allocated[index] + used.reserved[index] + available); + auto blocker = detail::ResourceBudget::try_reserve( + account, category, capacity - used.allocated[index] - used.reserved[index] - available); + ASSERT_TRUE(blocker); + std::shared_ptr<PublishedSnapshotEntries> first; + std::shared_ptr<PublishedSnapshotEntries> second; + status = PublishedSnapshotEntries::create(test.snapshot, &first); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = PublishedSnapshotEntries::create(test.snapshot, &second); + ASSERT_TRUE(status.ok()) << status.to_string(); + for (const auto &cursor : {first, second}) { + bool done = false; + std::vector<std::string> keys; + while (!done) { + std::optional<PublishedScanEntry> entry; + status = cursor->next(&entry, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (entry) { + keys.push_back(Slice(entry->key).to_string()); + if (Slice(entry->key) == Slice("a")) { + EXPECT_EQ(entry->cell.value.to_string(), "first"); + } + } + } + EXPECT_EQ(keys, (std::vector<std::string>{"", "a", "b", "c"})); + } +} + +TEST(PublishedSnapshotEntriesRange, FullValueCapRemainsReadableFromTheFixedView) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + constexpr size_t bytes = 64 * 1024 * 1024; + auto status = + tree.apply(1, Batch{.ops = {{.key = "large", .kind = OpKind::kPut, .value = std::string(bytes, 'v')}}}); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = tree.flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(TreeCoreAccess::get(tree)->capture_snapshot(&snapshot).ok()); + std::shared_ptr<PublishedSnapshotEntries> cursor; + status = PublishedSnapshotEntries::create(snapshot, &cursor); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::optional<PublishedScanEntry> entry; + bool done = false; + do { + status = cursor->next(&entry, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + } while (!entry && !done); + ASSERT_TRUE(entry); + EXPECT_EQ(Slice(entry->key).to_string(), "large"); + ASSERT_EQ(entry->cell.value.size(), bytes); + EXPECT_EQ(entry->cell.value.data()[0], 'v'); + EXPECT_EQ(entry->cell.value.data()[bytes - 1], 'v'); +} + +TEST(PublishedSnapshotEntriesRange, InvalidBoundsFailBeforeAnyControlAdmission) +{ + TestSnapshotRange test; + auto status = test.prepare(); + ASSERT_TRUE(status.ok()) << status.to_string(); + // Complete capture's deferred releases before comparing the two resource + // partitions; returning an allocation to its reserve changes both gauges. + TreeCoreAccess::get(*test.tree)->binding()->drain(); + const auto before = test.config.runtime->snapshot(); + std::shared_ptr<PublishedSnapshotEntries> cursor; + status = PublishedSnapshotEntries::create(test.snapshot, &cursor, KeyRange::bounded("c", "a")); + EXPECT_EQ(status.code(), Code::kInvalidArgument); + EXPECT_FALSE(cursor); + status = PublishedSnapshotEntries::create( + test.snapshot, &cursor, + KeyRange::bounded(std::string(test.snapshot->read_limits().max_key_bytes + 1, 'a'), {})); + EXPECT_EQ(status.code(), Code::kInvalidArgument); + EXPECT_FALSE(cursor); + EXPECT_EQ(test.config.runtime->snapshot().reserved.routing_bytes, before.reserved.routing_bytes); + EXPECT_EQ(test.config.runtime->snapshot().allocated.routing_bytes, before.allocated.routing_bytes); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_snapshot_scan_test.cpp b/lib/crowdb-tree/tests/integration/published_snapshot_scan_test.cpp new file mode 100644 index 000000000..c8d3c2c69 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_snapshot_scan_test.cpp @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/btree/scan_packed.h" +#include "snapshot/published_snapshot.h" + +#include <gtest/gtest.h> + +#include <array> +#include <cstring> + +namespace crowdb::tree +{ +namespace +{ +std::string key(size_t index) +{ + std::array<char, 32> text{}; + snprintf(text.data(), text.size(), "key%04zu", index); + return text.data(); +} + +void expect_entries(const ScanPackedBuf &packed, size_t count, bool reverse, bool tombstones) +{ + size_t offset = 0; + for (size_t index = 0; index < count; ++index) { + const auto expected_index = reverse ? 512 - index : index; + const auto expected_key = key(expected_index); + const bool tombstone = tombstones && (expected_index % 2 != 0); + const auto value_size = tombstone ? 0U : 5U; + ASSERT_GE(packed.size() - offset, expected_key.size() + value_size + 17); + EXPECT_EQ(packed.data()[offset], expected_key.size()); + EXPECT_EQ(std::memcmp(packed.data() + offset + 4, expected_key.data(), expected_key.size()), 0); + offset += 4 + expected_key.size() + 8; + EXPECT_EQ(packed.data()[offset], tombstone ? 1 : 0); + EXPECT_EQ(packed.data()[offset + 1], value_size); + offset += 5; + if (!tombstone) { + EXPECT_EQ(std::memcmp(packed.data() + offset, "value", value_size), 0); + } + offset += value_size; + } + EXPECT_EQ(offset, packed.size()); +} + +TEST(PublishedSnapshotScan, TotalCountsSpanFixedWindowsInBothDirectionsAfterTreeClose) +{ + auto tree = std::make_unique<Crowdbtree>(); + for (size_t index = 0; index < 513; ++index) { + ASSERT_TRUE(tree->put(key(index), "value").ok()); + } + ASSERT_TRUE(tree->flush().ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(TreeCoreAccess::get(*tree)->capture_snapshot(&snapshot).ok()); + ASSERT_TRUE(tree->put("later", "outside the captured L1").ok()); + tree.reset(); + for (const size_t limit : {size_t{0}, size_t{129}, size_t{257}}) { + for (const bool reverse : {false, true}) { + CoreScanOptions options; + options.limit = limit; + options.reverse = reverse; + ScanPackedBuf packed; + size_t count = 0; + bool truncated = false; + const auto status = snapshot->scan(options, &packed, &count, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(count, limit == 0 ? 513 : limit); + EXPECT_EQ(truncated, limit != 0); + expect_entries(packed, count, reverse, false); + } + } +} + +TEST(PublishedSnapshotScan, TombstonesUseThePhysicalCountBoundAcrossWindows) +{ + Crowdbtree tree; + Batch batch; + for (size_t index = 0; index < 513; ++index) { + const bool tombstone = index % 2 != 0; + batch.ops.push_back( + {.key = key(index), .kind = tombstone ? OpKind::kDelete : OpKind::kPut, .value = tombstone ? "" : "value"}); + } + ASSERT_TRUE(tree.apply(1, batch).ok()); + ASSERT_TRUE(tree.flush().ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(TreeCoreAccess::get(tree)->capture_snapshot(&snapshot).ok()); + for (const size_t limit : {size_t{0}, size_t{129}, size_t{257}}) { + for (const bool reverse : {false, true}) { + CoreScanOptions options; + options.limit = limit; + options.reverse = reverse; + options.include_tombstones = true; + ScanPackedBuf packed; + size_t count = 0; + bool truncated = false; + const auto status = snapshot->scan(options, &packed, &count, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(count, limit == 0 ? 513 : limit); + EXPECT_EQ(truncated, limit != 0); + expect_entries(packed, count, reverse, true); + } + } +} + +TEST(PublishedSnapshotScan, PublicByteLimitsSpanWindowsAndPermitOneLargerWindowEntry) +{ + Crowdbtree tree; + std::vector<size_t> sizes{400 * 1024, 400 * 1024, 1500 * 1024, 400 * 1024, 400 * 1024}; + for (size_t index = 0; index < sizes.size(); ++index) { + ASSERT_TRUE(tree.put(key(index), std::string(sizes[index], 'v')).ok()); + } + ASSERT_TRUE(tree.flush().ok()); + auto *core = TreeCoreAccess::get(tree); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(core->capture_snapshot(&snapshot).ok()); + for (const bool reverse : {false, true}) { + for (const size_t budget : {size_t{0}, size_t{3 * 1024 * 1024}}) { + CoreScanOptions options; + options.reverse = reverse; + options.byte_budget = budget; + options.strict_byte_budget = budget != 0; + size_t expected_count = 0; + size_t expected_bytes = 0; + for (size_t index = 0; index < sizes.size(); ++index) { + const auto next = sizes[reverse ? sizes.size() - 1 - index : index] + key(index).size(); + if (budget != 0 && next > budget - expected_bytes) { + break; + } + ++expected_count; + expected_bytes += next; + } + for (const bool fixed : {false, true}) { + ScanPackedBuf packed; + size_t count = 0; + bool truncated = false; + const auto status = fixed ? snapshot->scan(options, &packed, &count, &truncated) + : core->scan(options, nullptr, &truncated, &packed, &count); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(count, expected_count); + EXPECT_EQ(packed.size(), expected_bytes + (expected_count * 17)); + EXPECT_EQ(truncated, expected_count != sizes.size()); + } + } + } +} + +TEST(PublishedSnapshotScan, StrictFirstOverflowReturnsEmptyTruncatedAndResumes) +{ + Crowdbtree tree; + ASSERT_TRUE(tree.put("a", std::string(128 * 1024, 'v')).ok()); + ASSERT_TRUE(tree.put("b", "small").ok()); + ASSERT_TRUE(tree.flush().ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(TreeCoreAccess::get(tree)->capture_snapshot(&snapshot).ok()); + CoreScanOptions options; + options.byte_budget = 1024; + options.strict_byte_budget = true; + ScanPackedBuf packed; + size_t count = 0; + bool truncated = false; + auto status = snapshot->scan(options, &packed, &count, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(count, 0U); + EXPECT_EQ(packed.size(), 0U); + EXPECT_TRUE(truncated); + options.start = "a"; + options.has_start_bound = true; + options.byte_budget = 6; + status = snapshot->scan(options, &packed, &count, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(count, 1U); + EXPECT_EQ(packed.size(), 6U + 17); + EXPECT_FALSE(truncated); + packed = {}; + options.start = {}; + options.has_start_bound = false; + options.byte_budget = 1024; + options.strict_byte_budget = false; + status = snapshot->scan(options, &packed, &count, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(count, 1U); + EXPECT_EQ(packed.size(), (128U * 1024) + 1 + 17); + EXPECT_TRUE(truncated); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/published_snapshot_view_test.cpp b/lib/crowdb-tree/tests/integration/published_snapshot_view_test.cpp new file mode 100644 index 000000000..ceaea2bec --- /dev/null +++ b/lib/crowdb-tree/tests/integration/published_snapshot_view_test.cpp @@ -0,0 +1,192 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "local_checkpoint_test_support.h" +#include "snapshot/published_snapshot_view.h" + +#include <barrier> +#include <stdexcept> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct TestViewBackend +{ + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + + TestViewBackend() + { + store.device = device; + } +}; + +struct TestSnapshotView +{ + Config config; + std::shared_ptr<TestViewBackend> backend; + std::unique_ptr<Crowdbtree> tree; + + explicit TestSnapshotView(bool storage = true, bool owned = true) + { + EXPECT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.frame_bytes = 4096; + config.max_inline_value = 64; + if (storage) { + backend = std::make_shared<TestViewBackend>(); + config.page_store = backend->device.get(); + config.async_page_store = &backend->store; + if (owned) { + config.backend_owner = backend; + } + } + } + + [[nodiscard]] Status open() + { + if (config.page_store != nullptr) { + return Crowdbtree::open(config, &tree); + } + tree = std::make_unique<Crowdbtree>(config); + return Status::Ok(); + } + + [[nodiscard]] Status capture(std::shared_ptr<Snapshot> *out) const + { + try { + *out = tree->snapshot_view(); + return Status::Ok(); + } + catch (const std::exception &error) { + return Status::internal_error(error.what()); + } + } +}; + +TEST(PublishedSnapshotView, FixedL1GetAndEntriesSurviveSharedBackendTreeClose) +{ + TestSnapshotView test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(12000, 'v'); + ASSERT_TRUE(test.tree + ->apply(1, + Batch{ + .ops = {{.key = "a", .kind = OpKind::kPut, .value = value}, + {.key = "gone", .kind = OpKind::kDelete, .value = ""}} + }) + .ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + ASSERT_TRUE(test.tree->put("a", "pending").ok()); + std::shared_ptr<Snapshot> snapshot; + status = test.capture(&snapshot); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(snapshot->at_slot(), 1U); + EXPECT_EQ(TreeCoreAccess::get(*test.tree)->capture()->coverage, 1U); + test.tree.reset(); + uint64_t slot = 0; + std::string actual; + ASSERT_TRUE(snapshot->get("a", &slot, &actual)); + EXPECT_EQ(slot, 1U); + EXPECT_EQ(actual, value); + EXPECT_FALSE(snapshot->get("gone", nullptr, nullptr)); + const auto &entries = snapshot->entries(); + ASSERT_EQ(entries.size(), 2U); + EXPECT_EQ(entries[0].key, "a"); + EXPECT_EQ(CellView(Slice(entries[0].cell)).value().to_string(), value); + EXPECT_TRUE(CellView(Slice(entries[1].cell)).is_tombstone()); + EXPECT_EQ(&snapshot->entries(), &entries); +} + +TEST(PublishedSnapshotView, RawBackendMaterializesBeforeReturningAndReleasesDomain) +{ + TestSnapshotView test(true, false); + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(12000, 'r'); + ASSERT_TRUE(test.tree->put("a", value).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + std::shared_ptr<Snapshot> snapshot; + status = test.capture(&snapshot); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.tree.reset(); + status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::string actual; + ASSERT_TRUE(snapshot->get("a", nullptr, &actual)); + EXPECT_EQ(actual, value); + EXPECT_EQ(snapshot->size(), 1U); + test.tree.reset(); + test.backend.reset(); + EXPECT_EQ(CellView(Slice(snapshot->entries()[0].cell)).value().to_string(), value); +} + +TEST(PublishedSnapshotView, ConcurrentWholeResultCallsPublishOneStableVector) +{ + TestSnapshotView test(false); + test.config.runtime_binding.limits.routing_bytes = test.config.runtime->options().limits.routing_bytes; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(test.tree->put("a", std::string(16000, 'x')).ok()); + ASSERT_TRUE(test.tree->flush().ok()); + std::shared_ptr<Snapshot> snapshot; + status = test.capture(&snapshot); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::barrier start(3); + const std::vector<leaf_entry> *first = nullptr; + const std::vector<leaf_entry> *second = nullptr; + std::exception_ptr first_error; + std::exception_ptr second_error; + std::jthread left([&] { + start.arrive_and_wait(); + try { + first = &snapshot->entries(); + } + catch (...) { + first_error = std::current_exception(); + } + }); + std::jthread right([&] { + start.arrive_and_wait(); + try { + second = &snapshot->entries(); + } + catch (...) { + second_error = std::current_exception(); + } + }); + start.arrive_and_wait(); + left.join(); + right.join(); + EXPECT_FALSE(first_error); + EXPECT_FALSE(second_error); + ASSERT_NE(first, nullptr); + EXPECT_EQ(first, second); + EXPECT_EQ(first->size(), 1U); +} + +TEST(PublishedSnapshotView, WholeResultLimitFailsExplicitlyWhilePointReadsStayUsable) +{ + TestSnapshotView test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(test.tree->put("a", std::string(12000, 'l')).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + std::shared_ptr<PublishedSnapshotContext> context; + ASSERT_TRUE(TreeCoreAccess::get(*test.tree)->capture_snapshot(&context).ok()); + std::shared_ptr<Snapshot> snapshot; + status = PublishedSnapshotView::create(context, 1024, &snapshot); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_THROW(static_cast<void>(snapshot->entries()), std::runtime_error); + std::string actual; + ASSERT_TRUE(snapshot->get("a", nullptr, &actual)); + EXPECT_EQ(actual, std::string(12000, 'l')); + detail::ReservedUnique<SnapshotMaterialized> materialized; + status = materialize_snapshot(context, 1024, &materialized); + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(materialized); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/r6_test.cpp b/lib/crowdb-tree/tests/integration/r6_test.cpp index fd0fdc6d3..abc26a7c4 100644 --- a/lib/crowdb-tree/tests/integration/r6_test.cpp +++ b/lib/crowdb-tree/tests/integration/r6_test.cpp @@ -1,13 +1,13 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// R6: cross-thread page refcount tests. Verifies the three scenarios: -// (1) get_async slow path returns a borrowed Slice (no copy), -// (2) PinnedSnapshot stays consistent across install_snapshot, -// (3) stale-root pages are freed when the last pin drops (refcount GC). +// Independent immutable read owners survive asynchronous delivery and +// generation replacement; their final release retires the old publication. #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" #include "crowdb-tree/snapshot/snapshot.h" +#include "runtime_tree_access.h" +#include "snapshot/published_snapshot_view.h" #include <gtest/gtest.h> @@ -32,12 +32,8 @@ std::string make_key(int i) } } // namespace -// R6 scenario 1: get_async on an evicted (demand-loaded) page returns a -// borrowed Slice pointing into the resident frame, not an owned copy. -// frame_base() != nullptr proves the value is borrowed. On macOS (no -// liburing) the demand-load is synchronous and the borrow is via the epoch -// guard; on Linux (liburing) the Reactor thread resolves the miss and the -// borrow is via the R6 per-page refcount pin. Either way, no copy. +// A cold asynchronous read hands an independent immutable value owner to the +// callback, without carrying a caller-thread epoch guard. TEST(R6, GetAsyncSlowPathReturnsBorrowedSlice) { MemPageStore store(1); @@ -53,6 +49,11 @@ TEST(R6, GetAsyncSlowPathReturnsBorrowedSlice) ASSERT_TRUE(t.flush().ok()); ASSERT_TRUE(t.snapshot(nullptr).ok()); // clean + evictable + GetView warm; + ASSERT_TRUE(RuntimeTreeAccess_for_tests::core(t).get(Slice(make_key(0)), &warm).ok()); + ASSERT_TRUE(warm.found()); + warm = {}; + // Evict all evictable leaves so the next get_async demand-loads (slow path). size_t evicted = t.evict_clean_leaves(0); ASSERT_GT(evicted, 0U) << "test requires at least one evictable leaf"; @@ -80,14 +81,13 @@ TEST(R6, GetAsyncSlowPathReturnsBorrowedSlice) EXPECT_TRUE(borrowed.load()) << "slow path must return a borrowed Slice (frame_base != nullptr), not an owned copy"; } -// R6 scenario 2: snapshot_view returns a PinnedSnapshot that stays -// consistent across a concurrent install_snapshot. The pinned view's -// entries must not be truncated or mixed by the slot clears. -TEST(R6, PinnedSnapshotStaysConsistentAcrossInstallSnapshot) +// An immutable snapshot keeps its fixed mapping through generation install. +TEST(R6, PublishedSnapshotStaysConsistentAcrossInstallSnapshot) { - MemPageStore store(1); - Config opt; - opt.page_store = &store; + auto store = std::make_shared<MemPageStore>(1); + Config opt; + opt.page_store = store.get(); + opt.backend_owner = store; opt.leaf_split_bytes = 160; opt.frame_bytes = 4096; Crowdbtree t(opt); @@ -98,20 +98,17 @@ TEST(R6, PinnedSnapshotStaysConsistentAcrossInstallSnapshot) ASSERT_TRUE(t.flush().ok()); ASSERT_TRUE(t.snapshot(nullptr).ok()); - // Take a pinned snapshot of the 200-key tree. - auto snap = t.snapshot_view(); - // R6: snapshot_view() now returns a PinnedSnapshot (zero-copy, page - // refcount pins keep frames alive). Verify it's actually a PinnedSnapshot. - EXPECT_NE(dynamic_cast<PinnedSnapshot *>(snap.get()), nullptr); + std::weak_ptr<const PublishedPage> old_root = RuntimeTreeAccess_for_tests::root(t); + auto snap = t.snapshot_view(); + ASSERT_NE(dynamic_cast<PublishedSnapshotView *>(snap.get()), nullptr); ASSERT_EQ(snap->size(), 200U); - // Wipe the tree via install_snapshot. The old pages are retired, but - // the PinnedSnapshot's refcount pins keep them alive. + // The old mapping remains owned by the independent snapshot. ASSERT_TRUE(t.install_snapshot({}, 0).ok()); // The pinned snapshot must still see all 200 keys (leaf chain not // truncated/mixed by the slot clears in install_snapshot). - EXPECT_EQ(snap->size(), 200U) << "PinnedSnapshot must stay consistent across install_snapshot"; + EXPECT_EQ(snap->size(), 200U) << "Published snapshot must stay consistent across install_snapshot"; for (int i = 0; i < 200; ++i) { uint64_t s; std::string v; @@ -119,17 +116,11 @@ TEST(R6, PinnedSnapshotStaysConsistentAcrossInstallSnapshot) EXPECT_EQ(v, "val" + std::to_string(i)); } - // Drop the snapshot. The old pages' refcounts drop to zero and are freed - // (the retire_with_pins deleter deferred them). Verify the buffer pool - // reclaims them. - uint32_t used_before = t.buffer_pool()->stats().used; + RuntimeTreeAccess_for_tests::reclaim(t); + EXPECT_FALSE(old_root.expired()); snap.reset(); - // After the shared_ptr drops, the PinnedSnapshot dtor unpins. The pages - // were already retired by install_snapshot, so the last unpin frees them. - // Force a reclamation sweep (the retire deleter runs via epoch_.try_reclaim). - // The used count should drop (old tree's pages freed). - EXPECT_LE(t.buffer_pool()->stats().used, used_before) - << "dropping the PinnedSnapshot should free the old tree's pages"; + RuntimeTreeAccess_for_tests::reclaim(t); + EXPECT_TRUE(old_root.expired()); } // R6 scenario 3: concurrent readers + install_snapshot churn, no UAF. @@ -143,6 +134,12 @@ TEST(R6, ConcurrentReadersAndInstallSnapshotNoUAF) opt.page_store = &store; opt.leaf_split_bytes = 160; opt.frame_bytes = 4096; + // Raw-backend snapshots materialize independently. Four concurrent + // cursors must each fund a full supported value and its output owner. + TreeRuntimeOptions runtime; + runtime.limits.routing_bytes = 1024ULL * 1024 * 1024; + opt.runtime_binding.limits.routing_bytes = 768ULL * 1024 * 1024; + ASSERT_TRUE(TreeRuntime::create(runtime, &opt.runtime).ok()); Crowdbtree t(opt); for (int i = 0; i < 100; ++i) { @@ -151,8 +148,9 @@ TEST(R6, ConcurrentReadersAndInstallSnapshotNoUAF) ASSERT_TRUE(t.flush().ok()); ASSERT_TRUE(t.snapshot(nullptr).ok()); - std::atomic<bool> stop{false}; - std::atomic<bool> bad{false}; + std::atomic<bool> stop{false}; + std::atomic<bool> bad{false}; + std::atomic<size_t> ready{0}; // Reader threads: snapshot_view in a loop. Each snapshot is a // materialized copy (today) or a PinnedSnapshot (after T8-T11). The @@ -162,8 +160,24 @@ TEST(R6, ConcurrentReadersAndInstallSnapshotNoUAF) readers.reserve(4); for (int r = 0; r < 4; ++r) { readers.emplace_back([&] { + bool reported_ready = false; while (!stop.load(std::memory_order_relaxed) && !bad.load(std::memory_order_relaxed)) { - auto snap = t.snapshot_view(); + std::shared_ptr<Snapshot> snap; + const auto status = create_published_snapshot(TreeCoreAccess::get(t)->shared_from_this(), &snap); + if (status.code() == Code::kUnavailable) { + // Replacement closes new admission while draining the + // preceding generation; already accepted owners remain valid. + continue; + } + if (!status.ok()) { + ADD_FAILURE() << status.to_string(); + bad.store(true, std::memory_order_relaxed); + return; + } + if (!reported_ready) { + reported_ready = true; + ready.fetch_add(1, std::memory_order_release); + } if (snap == nullptr || snap->size() == 0) { continue; } @@ -184,6 +198,14 @@ TEST(R6, ConcurrentReadersAndInstallSnapshotNoUAF) }); } + for (size_t attempt = 0; attempt < 2000 && ready.load(std::memory_order_acquire) < 4 && !bad.load(); ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(ready.load(), 4U); + if (ready.load() != 4) { + bad.store(true); + } + // Writer thread: repeatedly install_snapshot to churn the tree. std::thread writer([&] { for (int i = 0; i < 50 && !bad.load(std::memory_order_relaxed); ++i) { @@ -193,7 +215,9 @@ TEST(R6, ConcurrentReadersAndInstallSnapshotNoUAF) std::string v = "v" + std::to_string(i) + "_" + std::to_string(j); entries.push_back({.key = k, .cell = encode_cell_buf((i * 100) + j, OpKind::kPut, Slice(v))}); } - if (!t.install_snapshot(std::move(entries), (i * 100) + 20).ok()) { + const auto installed = t.install_snapshot(std::move(entries), (i * 100) + 20); + if (!installed.ok()) { + ADD_FAILURE() << "install iteration " << i << ": " << installed.to_string(); bad.store(true, std::memory_order_relaxed); } } diff --git a/lib/crowdb-tree/tests/integration/range_rebuild_chunk_test.cpp b/lib/crowdb-tree/tests/integration/range_rebuild_chunk_test.cpp new file mode 100644 index 000000000..d225e616f --- /dev/null +++ b/lib/crowdb-tree/tests/integration/range_rebuild_chunk_test.cpp @@ -0,0 +1,202 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_tree_core_chunk.h" +#include "crowdb-tree/btree/range_rebuild.h" +#include "runtime_tree_access.h" +#include "snapshot/primary_backing.h" + +#include <gtest/gtest.h> + +#include <map> + +namespace crowdb::tree +{ +namespace +{ +using test_chunk::TestCoreChunk; + +std::map<std::string, std::string> contents(Crowdbtree &tree) +{ + std::vector<scan_entry> entries; + bool truncated = false; + const auto status = tree.scan({}, {}, {}, 0, 0, false, 0, &entries, &truncated); + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_FALSE(truncated); + std::map<std::string, std::string> result; + for (auto &entry : entries) { + result.emplace(std::move(entry.key), std::move(entry.value)); + } + return result; +} + +void fill(TestCoreChunk &test) +{ + ASSERT_TRUE(test.open().ok()); + for (size_t index = 0; index < 80; ++index) { + ASSERT_TRUE(test.tree->put("k" + std::to_string(1000 + index), std::string(128, 'v')).ok()); + } + ASSERT_TRUE(test.tree->snapshot().ok()); +} + +Status rebuild(TestCoreChunk &source, TestCoreChunk &child, const KeyRange &range, RangeRebuildStats *stats = nullptr) +{ + auto status = rebuild_range(*source.tree, range, child.config, &child.tree, stats); + child.attempted_open = true; + return status; +} + +void next_owner(TestCoreChunk &child) +{ + child.tree.reset(); + child.backend.catalog = child.backend.catalog->next_owner(); + child.backend.transport->catalog = child.backend.catalog; + child.configure_store(child.backend.catalog->current_generation); +} + +TEST(RangeRebuildChunk, ChildPublishesIndependentPaidBackingAndKeepsParentUnchanged) +{ + TestCoreChunk source; + TestCoreChunk child; + fill(source); + RangeRebuildStats stats; + ASSERT_TRUE(rebuild(source, child, KeyRange::unbounded(), &stats).ok()); + EXPECT_EQ(stats.pages_reused, 0U); + EXPECT_GT(stats.pages_rebuilt, 0U); + EXPECT_EQ(contents(*child.tree), contents(*source.tree)); + const auto old_source = source.backend.catalog->committed_manifest; + auto source_root = RuntimeTreeAccess_for_tests::root(*source.tree); + auto child_root = RuntimeTreeAccess_for_tests::root(*child.tree); + ASSERT_TRUE(source_root->primary); + ASSERT_TRUE(child_root->primary); + EXPECT_NE(source_root->primary->frame.backing, child_root->primary->frame.backing); + ASSERT_TRUE(child.tree->put("new-key", "child-only").ok()); + const auto checkpoint_status = child.tree->snapshot(); + ASSERT_TRUE(checkpoint_status.ok()) << checkpoint_status.to_string(); + EXPECT_EQ(source.backend.catalog->committed_manifest, old_source); + EXPECT_FALSE(contents(*source.tree).contains("new-key")); + EXPECT_EQ(contents(*child.tree).at("new-key"), "child-only"); +} + +TEST(RangeRebuildChunk, ImportedGenerationCanCheckpointLaterWritesAndReopenExactCoverage) +{ + TestCoreChunk source; + TestCoreChunk child; + ASSERT_TRUE(source.open().ok()); + ASSERT_TRUE(source.tree->put("b", "base").ok()); + ASSERT_TRUE(source.tree->flush().ok()); + auto status = rebuild(source, child, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(child.tree->durable_snapshot_last_applied_slot(), 1U); + EXPECT_EQ(child.tree->durable_snapshot_seq(), 1U); + for (uint64_t slot = 2; slot <= 3; ++slot) { + ASSERT_TRUE( + child.tree->apply(slot, Batch{.ops = {{.key = "c", .kind = OpKind::kPut, .value = std::to_string(slot)}}}) + .ok()); + status = child.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(child.tree->durable_snapshot_last_applied_slot(), slot); + EXPECT_EQ(child.tree->durable_snapshot_seq(), slot); + } + const auto generation = child.backend.catalog->current_generation; + child.tree.reset(); + status = child.open(generation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(child.tree->durable_snapshot_last_applied_slot(), 3U); + EXPECT_EQ(contents(*child.tree), (std::map<std::string, std::string>{ + {"b", "base"}, + {"c", "3" } + })); +} + +TEST(RangeRebuildChunk, RetryReplacesAnAlreadyCommittedChildThroughItsAuthority) +{ + TestCoreChunk source; + TestCoreChunk child; + ASSERT_TRUE(source.open().ok()); + ASSERT_TRUE(source.tree->put("a", "first").ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + ASSERT_TRUE(rebuild(source, child, KeyRange::unbounded()).ok()); + const auto generation = child.backend.catalog->current_generation; + next_owner(child); + ASSERT_TRUE(source.tree->put("b", "second").ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + ASSERT_TRUE(rebuild(source, child, KeyRange::unbounded()).ok()); + EXPECT_GT(child.backend.catalog->current_generation, generation); + EXPECT_EQ(contents(*child.tree), (std::map<std::string, std::string>{ + {"a", "first" }, + {"b", "second"} + })); +} + +TEST(RangeRebuildChunk, BoundedChildHasOnlyItsReachableGraphAndReopens) +{ + TestCoreChunk source; + TestCoreChunk child; + fill(source); + const auto range = KeyRange::bounded("k1020", "k1060"); + ASSERT_TRUE(rebuild(source, child, range).ok()); + child.config.key_range = range; + const auto view = RuntimeTreeAccess_for_tests::capture(*child.tree); + size_t count = 0; + view->mapping.visit(&count, [](void *context, uint64_t, const std::shared_ptr<const PublishedPage> &) { + ++*static_cast<size_t *>(context); + }); + EXPECT_EQ(count, view->stats.leaf_pages + view->stats.inner_pages); + EXPECT_EQ(contents(*child.tree).size(), 40U); + uint64_t bytes = UINT64_MAX; + bool complete = false; + ASSERT_TRUE(child.tree->materialize_ownership(&bytes, &complete).ok()); + EXPECT_TRUE(complete); + EXPECT_EQ(bytes, 0U); + child.tree.reset(); + ASSERT_TRUE(child.open(child.backend.catalog->current_generation).ok()); + EXPECT_EQ(contents(*child.tree).size(), 40U); + EXPECT_FALSE(contents(*child.tree).contains("k1019")); + EXPECT_FALSE(contents(*child.tree).contains("k1060")); + ASSERT_TRUE(child.tree->put("k1030x", "after-reopen").ok()); + ASSERT_TRUE(child.tree->snapshot().ok()); + child.tree.reset(); + ASSERT_TRUE(child.open(child.backend.catalog->current_generation).ok()); + EXPECT_EQ(contents(*child.tree).at("k1030x"), "after-reopen"); +} + +TEST(RangeRebuildChunk, UncheckpointedPublishedSourceIsCopiedAndDurableInChild) +{ + TestCoreChunk source; + TestCoreChunk child; + ASSERT_TRUE(source.open().ok()); + ASSERT_TRUE(source.tree->put("key", "old").ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + const auto durable = source.tree->durable_snapshot_last_applied_slot(); + ASSERT_TRUE(source.tree->put("key", "new").ok()); + ASSERT_TRUE(source.tree->flush().ok()); + ASSERT_TRUE(rebuild(source, child, KeyRange::unbounded()).ok()); + EXPECT_EQ(source.tree->durable_snapshot_last_applied_slot(), durable); + EXPECT_EQ(contents(*child.tree).at("key"), "new"); + child.tree.reset(); + ASSERT_TRUE(child.open(child.backend.catalog->current_generation).ok()); + EXPECT_EQ(contents(*child.tree).at("key"), "new"); +} + +TEST(RangeRebuildChunk, FailedReplacementKeepsOldRootRecoverable) +{ + TestCoreChunk source; + TestCoreChunk child; + ASSERT_TRUE(source.open().ok()); + ASSERT_TRUE(source.tree->put("key", "old").ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + ASSERT_TRUE(rebuild(source, child, KeyRange::unbounded()).ok()); + const auto generation = child.backend.catalog->current_generation; + next_owner(child); + ASSERT_TRUE(source.tree->put("key", "new").ok()); + ASSERT_TRUE(source.tree->snapshot().ok()); + child.backend.catalog->fail_commit = true; + EXPECT_FALSE(rebuild(source, child, KeyRange::unbounded()).ok()); + EXPECT_FALSE(child.tree); + EXPECT_EQ(child.backend.catalog->current_generation, generation); + child.backend.catalog->fail_commit = false; + ASSERT_TRUE(child.open(generation).ok()); + EXPECT_EQ(contents(*child.tree).at("key"), "old"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/range_rebuild_test.cpp b/lib/crowdb-tree/tests/integration/range_rebuild_test.cpp index 01a884a72..165dcdc49 100644 --- a/lib/crowdb-tree/tests/integration/range_rebuild_test.cpp +++ b/lib/crowdb-tree/tests/integration/range_rebuild_test.cpp @@ -1,10 +1,10 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -#include "backend/chunk/chunk_page_store.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/btree/range_rebuild.h" #include "crowdb-tree/c_api.h" +#include "runtime_tree_access.h" #include <gtest/gtest.h> @@ -62,6 +62,24 @@ NativeFrame make_inner_frame(uint64_t page_id, const std::vector<uint64_t> &chil return frame; } +TEST(RangeRebuild, UnboundedVolatileSourceBuildsIntoAnotherSharedHost) +{ + Crowdbtree source; + ASSERT_TRUE(source.put("a", "one").ok()); + ASSERT_TRUE(source.put("z", "two").ok()); + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + MemPageStore store(1); + Config config; + config.runtime = runtime; + config.page_store = &store; + std::unique_ptr<Crowdbtree> destination; + const auto status = rebuild_range(source, KeyRange::unbounded(), config, &destination); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(live_entries(*destination), live_entries(source)); + EXPECT_EQ(destination->durable_snapshot_last_applied_slot(), source.last_applied_slot()); +} + TEST(RangeRebuild, AdjacentChildrenHaveExactUnionAndEmptyIntersection) { MemPageStore source_store(1); @@ -104,10 +122,12 @@ TEST(RangeRebuild, AdjacentChildrenHaveExactUnionAndEmptyIntersection) EXPECT_EQ(left_entries, expected); EXPECT_EQ(left->get(Slice("k1050"), nullptr, nullptr), false); EXPECT_EQ(right->get(Slice("k1049"), nullptr, nullptr), false); - EXPECT_GT(left_stats.pages_reused, 0U); + EXPECT_EQ(left_stats.pages_reused, 0U); EXPECT_GT(left_stats.pages_rebuilt, 0U); - EXPECT_GT(left_stats.subtrees_skipped, 0U); - EXPECT_GT(right_stats.pages_reused, 0U); + EXPECT_GT(left_stats.subtrees_skipped, 0U) + << "source leaves=" << RuntimeTreeAccess_for_tests::capture(source)->stats.leaf_pages + << " rootlevel=" << RuntimeTreeAccess_for_tests::root(source)->level; + EXPECT_EQ(right_stats.pages_reused, 0U); EXPECT_GT(right_stats.pages_rebuilt, 0U); EXPECT_GT(right_stats.subtrees_skipped, 0U); } @@ -139,7 +159,7 @@ TEST(RangeRebuild, OversizedSourceLeafSplitsIntoFixedDestinationFrames) MemPageStore source_store(1); Config options; options.page_store = &source_store; - options.frame_bytes = 4096; + options.frame_bytes = 65536; options.leaf_split_bytes = 65536; options.max_inline_value = 2048; Crowdbtree source(options); @@ -155,7 +175,9 @@ TEST(RangeRebuild, OversizedSourceLeafSplitsIntoFixedDestinationFrames) })); MemPageStore destination_store(1); - options.page_store = &destination_store; + options.page_store = &destination_store; + options.frame_bytes = 4096; + options.leaf_split_bytes = 3000; std::unique_ptr<Crowdbtree> destination; ASSERT_TRUE( rebuild_range(source, KeyRange::bounded(std::string("k1012"), std::string("k1036")), options, &destination) @@ -333,14 +355,14 @@ TEST(RangeRebuild, ConcurrentWorkersPublishIndependentTrees) std::async(std::launch::async, worker, &high_store, KeyRange::bounded(std::string("k20"), std::nullopt)); auto low = low_future.get(); auto high = high_future.get(); - ASSERT_TRUE(low.first.ok()); - ASSERT_TRUE(high.first.ok()); + ASSERT_TRUE(low.first.ok()) << low.first.to_string(); + ASSERT_TRUE(high.first.ok()) << high.first.to_string(); EXPECT_FALSE(live_entries(*low.second).empty()); EXPECT_FALSE(live_entries(*high.second).empty()); EXPECT_EQ(live_entries(source).size(), 20U); } -TEST(RangeRebuild, WhollyContainedTreeReusesNativePageFrames) +TEST(RangeRebuild, WhollyContainedTreeCopiesIntoIndependentPaidFrames) { MemPageStore source_store(1); Config options; @@ -361,255 +383,12 @@ TEST(RangeRebuild, WhollyContainedTreeReusesNativePageFrames) ASSERT_TRUE(rebuild_range(source, KeyRange::bounded(std::string("k100"), std::string("k999")), options, &destination, &stats) .ok()); - EXPECT_GT(stats.pages_reused, 0U); - EXPECT_EQ(stats.pages_rebuilt, 0U); + EXPECT_EQ(stats.pages_reused, 0U); + EXPECT_GT(stats.pages_rebuilt, 0U); EXPECT_EQ(stats.entries_emitted, 40U); EXPECT_EQ(live_entries(*destination), live_entries(source)); } -TEST(RangeRebuild, ChunkChildPublishesIndependentManifestWithSharedPacks) -{ - auto catalog = std::make_shared<detail::MemoryRootCatalog>(1); - auto transport = std::make_shared<detail::MemoryChunkTransport>(); - detail::ChunkPageStore source_store( - {.tree_id = 60, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config source_options; - source_options.page_store = &source_store; - source_options.frame_bytes = 4096; - source_options.leaf_split_bytes = 256; - Crowdbtree source(source_options); - for (uint64_t index = 0; index < 80; ++index) { - ASSERT_TRUE(source.put(Slice("k" + std::to_string(index + 1000)), Slice(std::string(128, 'v'))).ok()); - } - ASSERT_TRUE(source.flush().ok()); - ASSERT_TRUE(source.snapshot().ok()); - auto source_manifest = catalog->load(60); - ASSERT_NE(source_manifest, nullptr); - - detail::ChunkPageStore child_store( - {.tree_id = 61, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config child_options = source_options; - child_options.page_store = &child_store; - std::unique_ptr<Crowdbtree> child; - ASSERT_TRUE(rebuild_range(source, KeyRange::unbounded(), child_options, &child).ok()); - MappingSegment *source_segment = source.mapping().segment_at(0); - MappingSegment *child_segment = child->mapping().segment_at(0); - ASSERT_NE(source_segment, nullptr); - ASSERT_NE(child_segment, nullptr); - EXPECT_EQ(child_segment->image_addr, source_segment->image_addr); - EXPECT_EQ(child_segment->image_crc, source_segment->image_crc); - - auto child_manifest = catalog->load(61); - ASSERT_NE(child_manifest, nullptr); - EXPECT_NE(child_manifest, source_manifest); - EXPECT_GT(child_manifest->packs_reused, 0U); - EXPECT_GT(child_store.stats().pack_bytes_reused, 0U); - bool shares_pack = false; - for (const detail::ChunkPagePack &child_pack : child_manifest->packs) { - shares_pack = shares_pack || std::any_of(source_manifest->packs.begin(), source_manifest->packs.end(), - [&](const detail::ChunkPagePack &source_pack) { - return child_pack.ref.chunk_id == source_pack.ref.chunk_id && - child_pack.ref.offset == source_pack.ref.offset; - }); - } - EXPECT_TRUE(shares_pack); - - ASSERT_TRUE(child->put(Slice("new-key"), Slice("child-only")).ok()); - ASSERT_TRUE(child->flush().ok()); - ASSERT_TRUE(child->snapshot().ok()); - EXPECT_NE(child->mapping().segment_at(0)->image_addr, source_segment->image_addr); - EXPECT_EQ(live_entries(source).contains("new-key"), false); - EXPECT_EQ(live_entries(*child).at("new-key"), "child-only"); - EXPECT_GT(child_store.stats().packs_reused, child_manifest->packs_reused); -} - -TEST(RangeRebuild, RetryAtomicallyReplacesAnAlreadyPublishedChunkChild) -{ - auto catalog = std::make_shared<detail::MemoryRootCatalog>(1); - auto transport = std::make_shared<detail::MemoryChunkTransport>(); - detail::ChunkPageStore source_store( - {.tree_id = 62, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config source_options; - source_options.page_store = &source_store; - source_options.frame_bytes = 4096; - Crowdbtree source(source_options); - ASSERT_TRUE(source.put(Slice("a"), Slice("first")).ok()); - ASSERT_TRUE(source.flush().ok()); - ASSERT_TRUE(source.snapshot().ok()); - - detail::ChunkPageStore first_child_store( - {.tree_id = 63, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config child_options = source_options; - child_options.page_store = &first_child_store; - std::unique_ptr<Crowdbtree> child; - ASSERT_TRUE(rebuild_range(source, KeyRange::unbounded(), child_options, &child).ok()); - const auto first_manifest = catalog->load(63); - ASSERT_NE(first_manifest, nullptr); - child.reset(); - - ASSERT_TRUE(source.put(Slice("b"), Slice("second")).ok()); - ASSERT_TRUE(source.flush().ok()); - ASSERT_TRUE(source.snapshot().ok()); - detail::ChunkPageStore retry_child_store( - {.tree_id = 63, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - child_options.page_store = &retry_child_store; - ASSERT_TRUE(rebuild_range(source, KeyRange::unbounded(), child_options, &child).ok()); - EXPECT_EQ(live_entries(*child), (std::map<std::string, std::string>{ - {"a", "first" }, - {"b", "second"} - })); - const auto retry_manifest = catalog->load(63); - ASSERT_NE(retry_manifest, nullptr); - EXPECT_GT(retry_manifest->generation, first_manifest->generation); -} - -TEST(RangeRebuild, MappingMaterializationClearsInheritedUnreachableSlots) -{ - auto catalog = std::make_shared<detail::MemoryRootCatalog>(1); - auto transport = std::make_shared<detail::MemoryChunkTransport>(); - detail::ChunkPageStore source_store( - {.tree_id = 70, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config source_options; - source_options.page_store = &source_store; - source_options.frame_bytes = 4096; - source_options.leaf_split_bytes = 256; - Crowdbtree source(source_options); - for (uint64_t index = 0; index < 80; ++index) { - ASSERT_TRUE(source.put(Slice("k" + std::to_string(index + 1000)), Slice(std::string(128, 'v'))).ok()); - } - ASSERT_TRUE(source.flush().ok()); - ASSERT_TRUE(source.snapshot().ok()); - - detail::ChunkPageStore child_store( - {.tree_id = 71, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config child_options = source_options; - child_options.page_store = &child_store; - const KeyRange range = KeyRange::bounded(std::string("k1020"), std::string("k1060")); - std::unique_ptr<Crowdbtree> child; - ASSERT_TRUE(rebuild_range(source, range, child_options, &child).ok()); - MappingSegment *segment = child->mapping().segment_at(0); - ASSERT_NE(segment, nullptr); - const uint32_t inherited_live_slots = segment->live_count.load(); - - bool complete = false; - for (uint32_t pass = 0; pass < 16 && !complete; ++pass) { - uint64_t bytes_written = 0; - ASSERT_TRUE(child->materialize_ownership(&bytes_written, &complete).ok()); - } - ASSERT_TRUE(complete); - ASSERT_NE(child->mapping().segment_at(0), nullptr); - EXPECT_LT(child->mapping().segment_at(0)->live_count.load(), inherited_live_slots); - EXPECT_EQ(live_entries(*child).size(), 40U); - EXPECT_FALSE(live_entries(*child).contains("k1019")); - EXPECT_FALSE(live_entries(*child).contains("k1060")); - - child.reset(); - ASSERT_TRUE(Crowdbtree::open(child_options, &child).ok()); - EXPECT_EQ(live_entries(*child).size(), 40U); - EXPECT_FALSE(live_entries(*child).contains("k1019")); - EXPECT_FALSE(live_entries(*child).contains("k1060")); - Batch after_repack; - after_repack.ops.push_back({.key = "k1030x", .kind = OpKind::kPut, .value = "after-repack"}); - ASSERT_TRUE(child->apply(81, after_repack).ok()); - ASSERT_TRUE(child->flush().ok()); - ASSERT_TRUE(child->snapshot().ok()); - child.reset(); - ASSERT_TRUE(Crowdbtree::open(child_options, &child).ok()); - EXPECT_EQ(live_entries(*child).at("k1030x"), "after-repack"); -} - -TEST(RangeRebuild, UnsnapshottedSourcePageIsCopiedInsteadOfInherited) -{ - auto catalog = std::make_shared<detail::MemoryRootCatalog>(1); - auto transport = std::make_shared<detail::MemoryChunkTransport>(); - detail::ChunkPageStore source_store( - {.tree_id = 72, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config source_options; - source_options.page_store = &source_store; - source_options.frame_bytes = 4096; - Crowdbtree source(source_options); - ASSERT_TRUE(source.put(Slice("key"), Slice("old")).ok()); - ASSERT_TRUE(source.flush().ok()); - ASSERT_TRUE(source.snapshot().ok()); - const uint64_t source_image = source.mapping().segment_at(0)->image_addr; - - ASSERT_TRUE(source.put(Slice("key"), Slice("new")).ok()); - ASSERT_TRUE(source.flush().ok()); - - detail::ChunkPageStore child_store( - {.tree_id = 73, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config child_options = source_options; - child_options.page_store = &child_store; - std::unique_ptr<Crowdbtree> child; - ASSERT_TRUE(rebuild_range(source, KeyRange::unbounded(), child_options, &child).ok()); - EXPECT_EQ(live_entries(*child).at("key"), "new"); - EXPECT_NE(child->mapping().segment_at(0)->image_addr, source_image); - - child.reset(); - ASSERT_TRUE(Crowdbtree::open(child_options, &child).ok()); - EXPECT_EQ(live_entries(*child).at("key"), "new"); -} - -TEST(RangeRebuild, MappingMaterializationRestartsAfterConcurrentFlush) -{ - auto catalog = std::make_shared<detail::MemoryRootCatalog>(1); - auto transport = std::make_shared<detail::MemoryChunkTransport>(); - detail::ChunkPageStore source_store( - {.tree_id = 74, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config source_options; - source_options.page_store = &source_store; - source_options.frame_bytes = 4096; - source_options.leaf_split_bytes = 256; - Crowdbtree source(source_options); - for (uint64_t index = 0; index < 80; ++index) { - ASSERT_TRUE(source.put(Slice("k" + std::to_string(index + 1000)), Slice(std::string(128, 'v'))).ok()); - } - ASSERT_TRUE(source.flush().ok()); - ASSERT_TRUE(source.snapshot().ok()); - - detail::ChunkPageStore child_store( - {.tree_id = 75, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); - Config child_options = source_options; - child_options.page_store = &child_store; - const KeyRange range = KeyRange::bounded(std::string("k1020"), std::string("k1060")); - std::unique_ptr<Crowdbtree> child; - ASSERT_TRUE(rebuild_range(source, range, child_options, &child).ok()); - - catalog->block_next_publish_for_tests(); - Status first_status; - uint64_t first_bytes = 0; - bool first_complete = true; - std::thread materializer([&] { first_status = child->materialize_ownership(&first_bytes, &first_complete); }); - catalog->wait_for_blocked_publish_for_tests(); - Status mutation_status; - for (uint64_t index = 20; index < 40; ++index) { - Batch batch; - batch.ops.push_back({.key = "k" + std::to_string(index + 1000), .kind = OpKind::kDelete, .value = {}}); - mutation_status = child->apply(81 + index - 20, batch); - if (!mutation_status.ok()) { - break; - } - } - if (mutation_status.ok()) { - mutation_status = child->flush(); - } - catalog->release_blocked_publish_for_tests(); - materializer.join(); - ASSERT_TRUE(mutation_status.ok()); - ASSERT_TRUE(first_status.ok()); - EXPECT_FALSE(first_complete); - EXPECT_EQ(first_bytes, 0U); - EXPECT_TRUE(child_store.has_shared_ownership()); - - bool complete = false; - for (uint32_t pass = 0; pass < 16 && !complete; ++pass) { - uint64_t bytes_written = 0; - ASSERT_TRUE(child->materialize_ownership(&bytes_written, &complete).ok()); - } - ASSERT_TRUE(complete); - EXPECT_EQ(live_entries(*child).size(), 20U); -} - TEST(RangeRebuild, CopiesOnlyOverflowChainsReferencedByTheChildRange) { MemPageStore source_store(1); @@ -638,13 +417,13 @@ TEST(RangeRebuild, CopiesOnlyOverflowChainsReferencedByTheChildRange) const auto rebuilt = live_entries(*destination); ASSERT_EQ(rebuilt.size(), 1U); EXPECT_EQ(rebuilt.at("m"), kept_value); - EXPECT_EQ(stats.entries_examined, 3U); + EXPECT_EQ(stats.entries_examined, 1U); EXPECT_EQ(stats.entries_emitted, 1U); - EXPECT_EQ(stats.entries_filtered, 2U); + EXPECT_EQ(stats.entries_filtered, 0U); EXPECT_EQ(live_entries(source).size(), 3U); } -TEST(RangeRebuild, RewrittenRootAllocatesAboveSourcePageIdHighWater) +TEST(RangeRebuild, RewrittenRootHasAnIndependentValidPageIdNamespace) { MemPageStore source_store(1); Config options; @@ -673,7 +452,10 @@ TEST(RangeRebuild, RewrittenRootAllocatesAboveSourcePageIdHighWater) std::vector<NativeFrame> destination_frames; uint64_t destination_root = kInvalidPageId; ASSERT_TRUE(destination->collect_native_frames(&destination_frames, &destination_root, nullptr).ok()); - EXPECT_GE(destination_root, source_highwater); + const auto destination_view = RuntimeTreeAccess_for_tests::capture(*destination); + EXPECT_LT(destination_root, destination_view->next_page_id); + EXPECT_EQ(live_entries(*destination).size(), 42U); + EXPECT_NE(RuntimeTreeAccess_for_tests::root(source), RuntimeTreeAccess_for_tests::root(*destination)); EXPECT_EQ(source_root < source_highwater, true); } @@ -934,7 +716,7 @@ TEST(RangeRebuild, NativeInstallUpgradesLegacyInnerFramesWithoutFences) EXPECT_EQ(live_entries(destination).size(), 2U); } -TEST(RangeRebuild, RecoveredTreeValidatesBeforeEnablingSubtreePruning) +TEST(RangeRebuild, RecoveredTreeValidatesBeforeFirstRangePruning) { MemPageStore store(1); Config options; @@ -960,7 +742,7 @@ TEST(RangeRebuild, RecoveredTreeValidatesBeforeEnablingSubtreePruning) std::unique_ptr<Crowdbtree> first; RangeRebuildStats first_stats; ASSERT_TRUE(rebuild_range(*recovered, range, child_options, &first, &first_stats).ok()); - EXPECT_EQ(first_stats.subtrees_skipped, 0U); + EXPECT_GT(first_stats.subtrees_skipped, 0U); MemPageStore second_store(1); child_options.page_store = &second_store; @@ -1026,9 +808,9 @@ TEST(RangeRebuild, CApiBuildsAnIndependentTreeOnAnInjectedStore) ct_range_rebuild_stats stats = {}; ASSERT_EQ(ct_rebuild_range(source, &destination_options, &destination, &stats), 0); ct_page_store_free(store); - EXPECT_EQ(stats.entries_examined, 4U); + EXPECT_EQ(stats.entries_examined, 1U); EXPECT_EQ(stats.entries_emitted, 1U); - EXPECT_EQ(stats.entries_filtered, 3U); + EXPECT_EQ(stats.entries_filtered, 0U); int32_t found = 0; uint64_t slot = 0; diff --git a/lib/crowdb-tree/tests/integration/range_runtime_binding_test.cpp b/lib/crowdb-tree/tests/integration/range_runtime_binding_test.cpp new file mode 100644 index 000000000..62a498e0a --- /dev/null +++ b/lib/crowdb-tree/tests/integration/range_runtime_binding_test.cpp @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/btree/range_rebuild.h" +#include "runtime/default_runtime.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(RangeRuntimeBinding, UnfundedDefaultDestinationKeepsCallerBindingAndOutput) +{ + Config source_options; + ASSERT_TRUE(TreeRuntime::create({}, &source_options.runtime).ok()); + Crowdbtree source(source_options); + ASSERT_TRUE(source.put("key", "value").ok()); + MemPageStore store(1); + Config destination_options; + destination_options.page_store = &store; + destination_options.runtime_binding.lifecycle_bytes = 1; + std::unique_ptr<Crowdbtree> destination; + const auto status = rebuild_range(source, KeyRange::unbounded(), destination_options, &destination); + EXPECT_EQ(status.code(), Code::kResourceExhausted) << status.to_string(); + EXPECT_EQ(destination, nullptr); + std::string value; + EXPECT_TRUE(source.get("key", nullptr, &value)); + EXPECT_EQ(value, "value"); +} + +TEST(RangeRuntimeBinding, BothCopyPathsUseTheProcessHostAndCallerLimits) +{ + Config source_options; + ASSERT_TRUE(TreeRuntime::create({}, &source_options.runtime).ok()); + source_options.frame_bytes = 4096; + Crowdbtree source(source_options); + ASSERT_TRUE(source.put("a", "one").ok()); + ASSERT_TRUE(source.put("z", "two").ok()); + Config process_options; + ASSERT_TRUE(configure_default_runtime(&process_options).ok()); + ASSERT_NE(process_options.runtime, source_options.runtime); + for (const bool bounded : {false, true}) { + SCOPED_TRACE(bounded); + MemPageStore store(1); + Config destination_options; + destination_options.frame_bytes = 4096; + destination_options.page_store = &store; + destination_options.runtime_binding.limits.writer_bytes = 1024 * 1024; + const auto range = bounded ? KeyRange::bounded(std::string("a"), std::string("m")) : KeyRange::unbounded(); + std::unique_ptr<Crowdbtree> destination; + const auto status = rebuild_range(source, range, destination_options, &destination); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto *core = TreeCoreAccess::get(*destination); + EXPECT_EQ(core->config().runtime, process_options.runtime); + EXPECT_EQ(core->binding()->options().limits.writer_bytes, 1024U * 1024U); + std::string value; + EXPECT_TRUE(destination->get("a", nullptr, &value)); + EXPECT_EQ(value, "one"); + EXPECT_EQ(destination->get("z", nullptr, &value), !bounded); + EXPECT_EQ(destination->durable_snapshot_last_applied_slot(), source.last_applied_slot()); + } +} + +TEST(RangeRuntimeBinding, DefaultDestinationStillRequiresAReloadableBackend) +{ + Crowdbtree source; + ASSERT_TRUE(source.put("key", "value").ok()); + std::unique_ptr<Crowdbtree> destination; + const auto status = rebuild_range(source, KeyRange::unbounded(), {}, &destination); + EXPECT_EQ(status.code(), Code::kInvalidArgument) << status.to_string(); + EXPECT_EQ(destination, nullptr); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/recovered_frame_identity_test.cpp b/lib/crowdb-tree/tests/integration/recovered_frame_identity_test.cpp new file mode 100644 index 000000000..489b8c600 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/recovered_frame_identity_test.cpp @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "local_recovery_test_support.h" +#include "native_import_test_support.h" +#include "recovered_state_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +TEST(RecoveredFrameIdentity, ConcreteLeafAndOverflowPidMismatchesRemainCorruption) +{ + for (const uint64_t pid : {0U, 1U}) { + test_write::TestRecovery test; + test.checkpoint(std::string(8000, 'v')); + const auto [address, length] = test.locations.at(pid); + std::vector<uint8_t> blob(length); + ASSERT_TRUE(test.store->device->read_at(address, blob.data(), blob.size()).ok()); + std::vector<uint8_t> raw(durable_blob_raw_len(blob.data(), blob.size())); + ASSERT_TRUE(decode_durable_page(blob.data(), blob.size(), raw.data(), raw.size()).ok()); + frame_put_u64(raw.data(), fh::kSelfpage_id, pid + 99); + frame_restamp_crc(raw.data(), raw.size()); + encode_durable_page(raw.data(), raw.size(), compress_algo::kLz4, &blob); + ASSERT_LE(blob.size(), length); + blob.resize(length); + ASSERT_TRUE(test.store->device->write_at(address, blob.data(), blob.size()).ok()); + ASSERT_TRUE(test.prepare().ok()); + test_write::Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kCorruption); + EXPECT_FALSE(test.operation->result().latest); + } +} + +TEST(RecoveredFrameIdentity, ConcreteInnerPidMismatchIsRejectedBeforePublication) +{ + test_read::TestRecoveredGraph test; + test.balanced(); + std::vector<uint8_t> raw(4096); + ASSERT_TRUE(inner_frame_build(raw.data(), raw.size(), 99, {1, 2}, {"m"})); + test.add(0, raw); + ASSERT_TRUE(test.prepare().ok()); + test_read::Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kCorruption); + EXPECT_FALSE(test.operation->result().state); +} + +TEST(RecoveredFrameIdentity, NativeImportStillRejectsEveryConcreteFrameIdentityMismatch) +{ + for (const auto type : {page_type::kLeafBase, page_type::kInnerBase, page_type::kOverflowFrame}) { + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.frame_bytes = 4096; + Crowdbtree tree(config); + auto frames = test_import::overflow_tree(std::string(8000, 'v')); + if (type == page_type::kInnerBase) { + frames = { + NativeFrame{.page_id = 0, .frame = std::vector<uint8_t>(4096)}, + test_import::inline_leaf(1) + }; + ASSERT_TRUE(inner_frame_build(frames[0].frame.data(), 4096, 0, {1}, {})); + } + const auto count = frames.size(); + auto status = tree.install_snapshot_native(frames, 0, 7, count); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto original = TreeCoreAccess::get(tree)->capture(); + size_t modified = 0; + for (auto &frame : frames) { + if (frame_page_type(frame.frame.data()) == type) { + frame_put_u64(frame.frame.data(), fh::kSelfpage_id, frame.page_id + 99); + frame_restamp_crc(frame.frame.data(), frame.frame.size()); + ++modified; + } + } + ASSERT_GT(modified, 0U); + status = tree.install_snapshot_native(std::move(frames), 0, 7, count); + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_EQ(TreeCoreAccess::get(tree)->capture(), original); + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/recovered_state_backend_test.cpp b/lib/crowdb-tree/tests/integration/recovered_state_backend_test.cpp new file mode 100644 index 000000000..474cb8aa3 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/recovered_state_backend_test.cpp @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/overflow_backing.h" +#include "local_recovery_test_support.h" +#include "recovered_state_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +std::shared_ptr<PublishedTreeState> boundary_tree() +{ + auto result = std::make_shared<PublishedTreeState>(); + result->coverage = 10; + result->root_page_id = 0; + result->next_page_id = 7; + std::vector<PageReplacement> replacements; + for (uint64_t pid = 0; pid < 3; ++pid) { + auto index = std::make_shared<PublishedPage>(); + index->level = pid == 0 ? 2 : 1; + index->inner = inner({"m"}, {(2 * pid) + 1, (2 * pid) + 2}); + index->metrics.leaf = false; + replacements.push_back({.page_id = pid, .expected = {}, .replacement = index}); + } + for (uint64_t pid = 3; pid < 7; ++pid) { + auto logical = std::make_shared<PageState>(); + if (pid != 3) { + logical->range.lower = "m"; + } + if (pid != 6) { + logical->range.upper = "m"; + } + logical->right_sibling = pid == 6 ? kInvalidPageId : pid + 1; + auto resident = base({}); + if (pid == 3) { + resident = base({ + {.key = "a", .value = "left"} + }); + } + else if (pid == 6) { + resident = base({ + {.key = "z", .value = "right"} + }); + } + logical->bases.push_back( + {.identity = pid + 1, .range = logical->range, .max_slot = 10, .resident = std::move(resident)}); + replacements.push_back({.page_id = pid, .expected = {}, .replacement = published(logical)}); + } + MappingView::CopyStats copied; + result->mapping = result->mapping.replace(replacements, copied); + return result; +} + +void complete_with(const std::shared_ptr<detail::FlushRuntime> &runtime, detail::CallbackSender sender) +{ + Completion done; + auto connected = stdexec::connect(sender, Receiver{&done}); + stdexec::start(connected); + runtime->drain(); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(done.errors.load(), 0U); +} +} // namespace + +TEST(RecoveredStateBackend, CheckpointRecoveryPreservesBoundaryEmptyChildrenForReadsAndRecheckpoint) +{ + test_write::TestRecovery test; + ASSERT_TRUE(test.original.source + .prepare(boundary_tree(), {}, + {.frames = {.encode = {.frame_bytes = 4096}}, .write = {.max_frame_bytes = 4096}}) + .ok()); + Completion written; + test.original.source.run(&written); + ASSERT_EQ(written.values.load(), 1U) << written.status.to_string(); + ASSERT_TRUE(test.original.prepare(test.original.source.operation->result().pages).ok()); + Completion committed; + test.original.run(&committed); + ASSERT_EQ(committed.values.load(), 1U) << committed.status.to_string(); + ASSERT_TRUE(test.prepare().ok()); + Completion recovered; + test.run(&recovered); + ASSERT_EQ(recovered.values.load(), 1U) << recovered.status.to_string(); + auto runtime = test.original.source.runtime; + auto lane = test.original.source.lane; + auto loader = std::make_shared<AsyncStorePageLoader>(test.store, runtime, lane, PageLoadLimits{}, + OverflowBackingResolver{}, test.credits.credits); + std::shared_ptr<RecoveredStateOperation> operation; + ASSERT_TRUE(RecoveredStateOperation::create(test.operation->result().latest, loader, runtime, lane, + {.max_step_entries = 1}, test.credits.credits.bytes, &operation) + .ok()); + complete_with(runtime, operation->sender()); + const auto state = operation->result().state; + ASSERT_NE(state, nullptr); + EXPECT_TRUE(state->mapping.get(4)->leaf->range.empty()); + EXPECT_TRUE(state->mapping.get(5)->leaf->range.empty()); + EXPECT_EQ(state->stats.leaf_pages, 4U); + for (const auto *key : {"a", "z", "m"}) { + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(state, loader, key, {}, &get).ok()); + complete_with(runtime, get->sender()); + EXPECT_EQ(get->result().found, std::string_view(key) != "m"); + } + for (bool reverse : {false, true}) { + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(state, loader, {}, reverse, {.max_entries = 1}, &scan).ok()); + std::vector<std::string> keys; + for (size_t i = 0; i < 16; ++i) { + complete_with(runtime, scan->next_sender()); + for (const auto &entry : scan->result().entries) { + keys.push_back(Slice(entry.key).to_string()); + } + if (scan->result().done) { + break; + } + } + EXPECT_TRUE(scan->result().done); + EXPECT_EQ(keys, reverse ? (std::vector<std::string>{"z", "a"}) : (std::vector<std::string>{"a", "z"})); + } + std::shared_ptr<CapturedFrameStream> stream; + ASSERT_TRUE( + CapturedFrameStream::create(state, loader, runtime, lane, + { + .encode = {.frame_bytes = 4096, .metadata = test.credits.credits.bytes} + }, + {}, &stream) + .ok()); + for (size_t i = 0; i < 7; ++i) { + complete_with(runtime, stream->sender()); + ASSERT_TRUE(stream->result().frame); + } + EXPECT_TRUE(stream->result().complete); +} + +TEST(RecoveredStateBackend, LegacyMissingFencesRemainCompatibleButWrongByteFencesFail) +{ + for (bool wrong : {false, true}) { + TestRecoveredGraph test; + test.balanced(); + std::vector<uint8_t> raw(4096); + ASSERT_TRUE(inner_frame_build(raw.data(), raw.size(), 0, {1, 2}, {"m"})); + if (wrong) { + const Slice lower("a"); + const Slice upper("y"); + ASSERT_TRUE(frame_set_fences(raw.data(), raw.size(), &lower, &upper)); + } + test.add(0, raw); + ASSERT_TRUE(test.prepare().ok()); + Completion done; + test.run(&done); + EXPECT_EQ(done.values.load(), wrong ? 0U : 1U); + EXPECT_EQ(done.errors.load(), wrong ? 1U : 0U); + } +} + +TEST(RecoveredStateBackend, OverflowReadAndRecaptureNeedOnlyPrimaryDependencies) +{ + test_write::TestRecovery test; + const std::string value(12000, 'v'); + test.checkpoint(value); + ASSERT_TRUE(test.prepare().ok()); + Completion recovered; + test.run(&recovered); + ASSERT_EQ(recovered.values.load(), 1U) << recovered.status.to_string(); + auto runtime = test.original.source.runtime; + auto lane = test.original.source.lane; + auto ledger = test.operation->result().ledger; + auto loader = std::make_shared<AsyncStorePageLoader>(test.store, runtime, lane, PageLoadLimits{}, + OverflowBackingResolver{}, test.credits.credits); + std::shared_ptr<RecoveredStateOperation> operation; + ASSERT_TRUE(RecoveredStateOperation::create(test.operation->result().latest, loader, runtime, lane, {}, + test.credits.credits.bytes, &operation) + .ok()); + complete_with(runtime, operation->sender()); + auto state = operation->result().state; + ASSERT_TRUE(state); + const auto &primary = state->mapping.get(state->root_page_id)->leaf->bases[0].backing; + ASSERT_TRUE(primary->overflow); + EXPECT_FALSE(primary->overflow->pages.empty()); + operation.reset(); + test.operation.reset(); + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(state, loader, "a", {}, &get).ok()); + complete_with(runtime, get->sender()); + ASSERT_TRUE(get->result().found); + EXPECT_EQ(get->result().cell.value.to_string(), value); + std::shared_ptr<CapturedWriteOperation> write; + ASSERT_TRUE(CapturedWriteOperation::create( + state, loader, test.store, runtime, lane, + {.frames = {.encode = {.frame_bytes = 4096}}, .write = {.max_frame_bytes = 4096}}, + test.credits.credits, {}, extent_write_reservation(ledger, test.credits.credits.bytes), {}, &write) + .ok()); + complete_with(runtime, write->sender()); + ASSERT_TRUE(write->result().pages); + EXPECT_EQ(write->result().written_frames, 1U); + EXPECT_EQ(write->result().reused_frames, primary->overflow->pages.size()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/recovered_state_test.cpp b/lib/crowdb-tree/tests/integration/recovered_state_test.cpp new file mode 100644 index 000000000..3c7333451 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/recovered_state_test.cpp @@ -0,0 +1,253 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "recovered_state_test_support.h" + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(RecoveredState, BuildsColdMappingAndExactLeafMetricsThroughBoundedSteps) +{ + TestRecoveredGraph test; + test.balanced(); + ASSERT_TRUE(test.prepare({.max_step_entries = 1}).ok()); + Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + const auto &result = test.operation->result(); + ASSERT_NE(result.state, nullptr); + EXPECT_EQ(result.pages, 3U); + EXPECT_EQ(result.loads, 3U); + EXPECT_GT(result.cpu_steps, 12U); + EXPECT_EQ(result.state->generation, 3U); + EXPECT_EQ(result.state->coverage, 10U); + EXPECT_EQ(result.state->sources->floor, 10U); + EXPECT_TRUE(result.state->sources->tables.empty()); + EXPECT_EQ(result.state->next_page_id, 3U); + EXPECT_EQ(result.state->stats.leaf_pages, 2U); + EXPECT_EQ(result.state->stats.inner_pages, 1U); + EXPECT_EQ(result.state->stats.kv_known_count, 2U); + EXPECT_EQ(result.state->stats.kv_known_bytes, 8U); + EXPECT_EQ(result.state->stats.kv_unknown_pages, 0U); + EXPECT_EQ(result.state->stats.page_size_unknown_pages, 1U); + EXPECT_EQ(result.state->stats.dirty_pages, 0U); + EXPECT_EQ(result.state->mapping.get(0)->level, 1U); + EXPECT_FALSE(result.state->mapping.get(0)->inner); + EXPECT_TRUE(result.state->mapping.get(0)->inner_backing); + for (uint64_t pid : {1, 2}) { + const auto &base = result.state->mapping.get(pid)->leaf->bases.front(); + EXPECT_FALSE(base.resident); + EXPECT_EQ(base.max_slot, 10U); + EXPECT_EQ(base.backing, test.pages->frames.at(pid).backing); + } + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(result.state, test.loader, "z", {}, &request).ok()); + test.complete(request->sender()); + EXPECT_TRUE(request->result().found); + EXPECT_EQ(request->result().cell.value, Slice("two")); +} + +TEST(RecoveredState, RejectsMissingRepeatedCyclicAndUnevenTreeEdges) +{ + for (size_t scenario = 0; scenario < 4; ++scenario) { + SCOPED_TRACE(scenario); + TestRecoveredGraph test; + test.balanced(); + if (scenario == 0) { + test.pages->frames.erase(2); + } + if (scenario == 1) { + test.index(0, {1, 1}, {"m"}); + } + if (scenario == 2) { + test.index(0, {0, 2}, {"m"}); + } + if (scenario == 3) { + test.index(2, {3}, {}); + test.leaf(1, + { + {.key = "a", .value = "one"} + }, + 3); + test.leaf(3, { + {.key = "z", .value = "two"} + }); + } + ASSERT_TRUE(test.prepare().ok()); + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kCorruption); + EXPECT_FALSE(test.operation->result().state); + } +} + +TEST(RecoveredState, RejectsBadSiblingFencesOutOfRangeKeysAndUnreachableFrames) +{ + for (size_t scenario = 0; scenario < 5; ++scenario) { + SCOPED_TRACE(scenario); + TestRecoveredGraph test; + test.balanced(); + if (scenario == 0) { + test.leaf(1, + { + {.key = "a", .value = "one"} + }, + 1); + } + if (scenario == 1) { + test.leaf(2, + { + {.key = "z", .value = "two"} + }, + 1); + } + if (scenario == 2) { + test.index(0, {1, 2}, {"m"}, std::pair{2, 1}); + } + if (scenario == 3) { + test.leaf(1, + { + {.key = "m", .value = "out"} + }, + 2); + } + if (scenario == 4) { + test.leaf(3, {}); + } + ASSERT_TRUE(test.prepare().ok()); + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kCorruption); + EXPECT_FALSE(test.operation->result().state); + } +} + +TEST(RecoveredState, RejectsMissingStoredDependencyProofAndEnforcesBounds) +{ + for (size_t scenario = 0; scenario < 5; ++scenario) { + SCOPED_TRACE(scenario); + TestRecoveredGraph test; + test.balanced(); + RecoveredStateLimits limits; + if (scenario == 0) { + test.pages->frames.at(1).stored_overflow.reset(); + } + if (scenario == 1) { + limits.max_depth = 1; + } + if (scenario == 2) { + limits.max_children = 1; + } + if (scenario == 3) { + limits.max_leaf_entries = 1; + } + if (scenario == 4) { + limits.max_key_bytes = 0; + } + ASSERT_TRUE(test.prepare(limits).ok()); + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), scenario == 0 ? Code::kCorruption : Code::kResourceExhausted); + } +} + +TEST(RecoveredState, PreservesStoredOverflowHistoryHiddenByAnInlineDelta) +{ + TestRecoveredGraph test; + std::vector<uint8_t> raw(4096); + LeafFrameBuilder builder(raw.data(), raw.size()); + const auto overflow = encode_overflow_cell_buf(2, 1, 3); + ASSERT_TRUE(builder.try_append_sorted("a", Slice(overflow))); + builder.finish(0, kInvalidPageId); + std::vector<leaf_entry> delta; + delta.push_back({.key = "a", .cell = encode_cell_buf(3, OpKind::kPut, "new")}); + std::vector<uint8_t> appended(raw.size()); + ASSERT_TRUE(leaf_frame_append_deltas(raw.data(), raw.size(), delta, appended.data())); + test.add(0, appended); + overflow_frame_build(raw.data(), raw.size(), 1, kInvalidPageId, Slice("old").bytes(), 3); + test.add(1, raw); + ASSERT_TRUE(test.prepare().ok()); + Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(test.operation->result().state->stats.kv_known_count, 1U); + EXPECT_EQ(test.operation->result().state->stats.kv_known_bytes, 4U); + EXPECT_FALSE(test.operation->result().state->mapping.get(1)); + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(test.operation->result().state, test.loader, "a", {}, &request).ok()); + test.complete(request->sender()); + EXPECT_EQ(request->result().cell.value, Slice("new")); +} + +TEST(RecoveredState, RejectsDuplicateAndEscapingSeparators) +{ + for (bool duplicate : {false, true}) { + TestRecoveredGraph test; + test.balanced(); + if (duplicate) { + std::vector<uint8_t> raw(4096); + ASSERT_TRUE(inner_frame_build(raw.data(), raw.size(), 0, {1, 2, 3}, {"m", "m"})); + encode_durable_page(raw.data(), raw.size(), compress_algo::kNone, &test.store->blobs[0]); + } + else { + test.index(1, {3, 4}, {"z"}); + test.leaf(3, + { + {.key = "a", .value = "one"} + }, + 4); + test.leaf(4, {}, 2); + } + ASSERT_TRUE(test.prepare().ok()); + Completion done; + test.run(&done); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(done.status.code(), Code::kCorruption); + } +} + +TEST(RecoveredState, ExhaustedAdmissionFailsWithoutStartingIo) +{ + TestRecoveredGraph test; + test.balanced(); + const auto held = test.credits.credits.bytes->available(); + ASSERT_TRUE(test.credits.credits.bytes->claim(held)); + EXPECT_EQ(test.prepare().code(), Code::kResourceExhausted); + EXPECT_EQ(test.store->reads.load(), 0U); + EXPECT_FALSE(test.operation); + test.credits.credits.bytes->release(held); + test.loader.reset(); + test.pages.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(RecoveredState, CancelWaitsForAcceptedReadAndReleasesAllChargedMetadata) +{ + TestRecoveredGraph test; + test.balanced(); + test.store->deferred = true; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{&done}); + stdexec::start(connected); + test.runtime->drain(); + ASSERT_EQ(test.store->pending.size(), 1U); + test.operation->request_stop(); + test.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 0U); + test.store->finish(); + test.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_FALSE(test.operation->result().state); + test.operation.reset(); + test.pages.reset(); + test.loader.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/rpc_chunk_collection_test.cpp b/lib/crowdb-tree/tests/integration/rpc_chunk_collection_test.cpp new file mode 100644 index 000000000..34b8978d1 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/rpc_chunk_collection_test.cpp @@ -0,0 +1,208 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_stage_gc.h" +#include "backend/chunk/rpc_chunk_transport.h" +#include "common/test_chunk_physical.h" +#include "common/test_chunk_route_service.h" +#include "common/test_chunk_stage_backend.h" +#include "diskio_generated.h" + +namespace crowdb::tree::detail +{ +namespace +{ +struct TestChunkCollectionService +{ + crowdb_rpc_server_t server = nullptr; + std::atomic<bool> deleted{false}; + std::atomic<bool> lose_delete{false}; + std::atomic<uint64_t> epoch{17}; + std::atomic<size_t> deletes{0}; + std::atomic<size_t> physical_deletes{0}; + + static void handle(uint64_t id, uint64_t /*unused*/, uint16_t message, const uint8_t *control, uint32_t length, + const uint8_t * /*unused*/, uint32_t /*unused*/, void *connection, void *frame, void *context) + { + static_cast<TestChunkCollectionService *>(context)->respond(id, message, control, length, connection); + crowdb_rpc_frame_release(frame); + } + + void respond(uint64_t id, uint16_t message, const uint8_t *control, uint32_t length, void *connection) + { + using namespace crowdb::chunkdb::proto; + using namespace crowdb::rpc::proto; + flatbuffers::Verifier verifier(control, length); + flatbuffers::FlatBufferBuilder builder; + uint16_t response = 0; + const FBInt128 chunk_id(0x0200'0000'0000'0042ULL, 144); + const FBInt128 disk(9, 10); + if (message == FBMsgType_EQueryChunkRequest) { + ASSERT_TRUE(verifier.VerifyBuffer<FBQueryChunkRequest>(nullptr)); + flatbuffers::Offset<FBChunk> chunk; + if (deleted.load()) { + chunk = CreateFBChunk(builder, &chunk_id, 2, FBChunkState_Deleted, 123, 0, 0, 0, 0, + FBChunkType_BtreePage, epoch.load(), 0); + } + else { + const std::vector<crowdb::diskdb::proto::FBSegment> segments{ + {disk, chunk_id, 0, 1, 2, 7}, + {disk, chunk_id, 9, 1, 2, 9} + }; + auto mirror = CreateFBMirrorStrip(builder, builder.CreateVectorOfStructs(segments)); + auto strip = CreateFBChunkStrip(builder, 0, 0, 4, 4, 4, 0, 0, FBStripType_Mirror, + FBStripBody_FBMirrorStrip, mirror.Union()); + auto strips = builder.CreateVector(std::vector<flatbuffers::Offset<FBChunkStrip>>{strip}); + chunk = CreateFBChunk(builder, &chunk_id, 1, FBChunkState_Active, 123, 0, 4, 0, strips, + FBChunkType_BtreePage, epoch.load(), 0); + } + builder.Finish(CreateFBQueryChunkResponse(builder, id, 0, FBChunkdbRetCode_Success, 0, 0, 0, chunk, 0)); + response = FBMsgType_EQueryChunkResponse; + } + else if (message == FBMsgType_EDeleteChunkRequest) { + ASSERT_TRUE(verifier.VerifyBuffer<FBDeleteChunkRequest>(nullptr)); + EXPECT_TRUE(flatbuffers::GetRoot<FBDeleteChunkRequest>(control)->fence_missing()); + ++deletes; + if (!deleted.exchange(true)) { + ++physical_deletes; + } + if (lose_delete.exchange(false)) { + return; + } + builder.Finish(CreateFBDeleteChunkResponse(builder, id, 0, FBChunkdbRetCode_NotFound)); + response = FBMsgType_EDeleteChunkResponse; + } + else { + namespace dio = crowdb::diskio::proto; + ASSERT_TRUE(verifier.VerifyBuffer<dio::FBDiskWriteRequest>(nullptr)); + const auto *request = flatbuffers::GetRoot<dio::FBDiskWriteRequest>(control); + EXPECT_TRUE(request->sync()); + EXPECT_EQ(request->size(), 0U); + builder.Finish(dio::CreateFBDiskWriteResponse(builder, id, 0, dio::FBDiskIoRetCode_Success, true)); + response = FBMsgType_EDiskWriteResponse; + } + static_cast<void>(crowdb_rpc_server_submit_response(server, connection, builder.GetBufferPointer(), + builder.GetSize(), nullptr, 0, response, id)); + } +}; + +class RpcChunkCollection : public ::testing::Test +{ + protected: + TestChunkRouteFixture fixture; + TestChunkCollectionService service; + std::unique_ptr<RpcChunkTransport> transport; + std::shared_ptr<TestStageCatalog> catalog = std::make_shared<TestStageCatalog>(); + ChunkStageId stage{.owner_epoch = 17, .operation_high = 10, .operation_low = 20}; + ChunkId id{0x0200'0000'0000'0042ULL, 144}; + + void SetUp() override + { + service.server = fixture.services[0].server; + for (const auto message : + {crowdb::rpc::proto::FBMsgType_EQueryChunkRequest, crowdb::rpc::proto::FBMsgType_EDeleteChunkRequest, + crowdb::rpc::proto::FBMsgType_EDiskWriteRequest}) { + crowdb_rpc_server_register_handler(service.server, message, TestChunkCollectionService::handle, &service, + nullptr); + } + const ct_chunk_rpc_disk_route disk{.disk_id_high = 9, .disk_id_low = 10, .route = fixture.routes[0]}; + ct_chunk_rpc_transport_options options{}; + options.disk_routes = &disk; + options.disk_route_count = 1; + options.rpc_timeout_ms = 50; + options.chunkdb_resolver = fixture.resolver(); + transport = std::make_unique<RpcChunkTransport>(options); + catalog->record = ChunkStageRecord{.tree_id = 19, + .id = stage, + .revision = 2, + .chunk_high = id.high, + .chunk_low = id.low, + .capacity = 4096, + .manifest_generation = 0, + .purpose = 2, + .state = ChunkStageState::Retired, + .packs = {}}; + } +}; +} // namespace + +TEST_F(RpcChunkCollection, PhysicalReceiptCountsRoundedSegmentsAcrossAllMirrors) +{ + ChunkAllocationIdentity receipt; + const auto status = transport->inspect_fixed_allocation(id, 17, &receipt); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(receipt.chunk, id); + EXPECT_EQ(receipt.owner_epoch, 17U); + EXPECT_EQ(receipt.created_ms, 123U); + EXPECT_EQ(receipt.physical_bytes, (7U + 9U) * 4U * 1024U); + if (!catalog->record) { + FAIL() << "stage unexpectedly absent"; + return; + } + EXPECT_NE(receipt.physical_bytes, catalog->record->capacity); +} + +TEST_F(RpcChunkCollection, LostDeleteReplyAndDurableDeletingNotFoundConverge) +{ + service.lose_delete.store(true); + auto result = reclaim_retired_stage(*catalog, *transport, 19, 17, stage); + EXPECT_EQ(result.status.code(), Code::kUnavailable); + EXPECT_FALSE(result.deletion_acknowledged); + EXPECT_FALSE(result.stage_removed); + if (!catalog->record) { + FAIL() << "stage unexpectedly absent"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Deleting); + EXPECT_EQ(service.physical_deletes.load(), 1U); + result = reclaim_retired_stage(*catalog, *transport, 19, 17, stage); + EXPECT_TRUE(result.status.ok()) << result.status.to_string(); + EXPECT_TRUE(result.deletion_acknowledged); + EXPECT_TRUE(result.stage_removed); + EXPECT_EQ(service.deletes.load(), 2U); + EXPECT_EQ(service.physical_deletes.load(), 1U); + EXPECT_FALSE(catalog->record); +} + +TEST_F(RpcChunkCollection, ANewPhysicalOwnerCannotBeDeletedByAnOldStage) +{ + service.epoch.store(18); + const auto result = reclaim_retired_stage(*catalog, *transport, 19, 18, stage); + EXPECT_EQ(result.status.code(), Code::kUnavailable); + EXPECT_FALSE(result.deletion_acknowledged); + EXPECT_FALSE(result.stage_removed); + EXPECT_EQ(service.deletes.load(), 0U); + EXPECT_EQ(service.physical_deletes.load(), 0U); + if (!catalog->record) { + FAIL() << "stage unexpectedly absent"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Deleting); +} + +TEST_F(RpcChunkCollection, PhysicalAllocationRemainsCountedUntilActualDeleteAcknowledgement) +{ + test_chunk::TestPhysicalHost host; + std::shared_ptr<ChunkPhysicalRegistry> registry; + ASSERT_TRUE(transport->physical_registry(19, host.budget, ®istry).ok()); + const auto initial_scope = registry->backing_stats().scope_id; + ChunkAllocationIdentity receipt; + ASSERT_TRUE(transport->inspect_fixed_allocation(id, 17, &receipt).ok()); + std::shared_ptr<ChunkPhysicalPack> physical; + ASSERT_TRUE(registry->prepare(stage, id, host.budget, &physical).ok()); + ASSERT_TRUE(registry->observe(physical, receipt, host.budget).ok()); + service.lose_delete.store(true); + auto result = reclaim_retired_stage(*catalog, *transport, 19, 17, stage, host.budget, registry); + EXPECT_EQ(result.status.code(), Code::kUnavailable); + EXPECT_EQ(registry->backing_metrics()->snapshot().allocated_backing_bytes, receipt.physical_bytes); + result = reclaim_retired_stage(*catalog, *transport, 19, 17, stage, host.budget, registry); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + EXPECT_TRUE(result.deletion_acknowledged); + EXPECT_EQ(service.physical_deletes.load(), 1U); + host.cpu.drain(); + EXPECT_EQ(registry->backing_metrics()->snapshot().allocated_backing_bytes, 0U); + registry.reset(); + ASSERT_TRUE(transport->physical_registry(19, host.budget, ®istry).ok()); + EXPECT_EQ(registry->backing_stats().scope_id, initial_scope); + EXPECT_EQ(registry->backing_stats().allocated_bytes, 0U); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/rpc_chunk_fixed_io_test.cpp b/lib/crowdb-tree/tests/integration/rpc_chunk_fixed_io_test.cpp new file mode 100644 index 000000000..77f9ad8a4 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/rpc_chunk_fixed_io_test.cpp @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/rpc_chunk_transport.h" +#include "common/test_chunk_route_service.h" +#include "diskio_generated.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +namespace +{ +struct TestFixedIoService +{ + crowdb_rpc_server_t server = nullptr; + std::atomic<bool> age_receipt{false}; + std::atomic<bool> sync_receipt{false}; + std::atomic<uint64_t> timestamp{0}; + std::atomic<bool> sync_requested{false}; + std::atomic<uint32_t> write_size{UINT32_MAX}; + std::atomic<uint64_t> ordering_offset{UINT64_MAX}; + + static void handle(uint64_t request_id, uint64_t /*unused*/, uint16_t message, const uint8_t *control, + uint32_t length, const uint8_t * /*unused*/, uint32_t /*unused*/, void *connection, void *frame, + void *context) + { + auto &self = *static_cast<TestFixedIoService *>(context); + self.respond(request_id, message, control, length, connection); + crowdb_rpc_frame_release(frame); + } + + void respond(uint64_t id, uint16_t message, const uint8_t *control, uint32_t length, void *connection) + { + namespace dio = crowdb::diskio::proto; + namespace rpc = crowdb::rpc::proto; + flatbuffers::Verifier verifier(control, length); + flatbuffers::FlatBufferBuilder builder; + uint16_t response = 0; + uint32_t bytes = 0; + const std::array<uint8_t, 4> data{1, 2, 3, 4}; + if (message == rpc::FBMsgType_EDiskReadRequest) { + ASSERT_TRUE(verifier.VerifyBuffer<dio::FBDiskReadRequest>(nullptr)); + const auto *request = flatbuffers::GetRoot<dio::FBDiskReadRequest>(control); + timestamp.store(request->read_create_time_ms()); + bytes = request->size(); + ASSERT_LE(bytes, data.size()); + builder.Finish( + dio::CreateFBDiskReadResponse(builder, id, 0, dio::FBDiskIoRetCode_Success, age_receipt.load())); + response = rpc::FBMsgType_EDiskReadResponse; + } + else { + ASSERT_TRUE(verifier.VerifyBuffer<dio::FBDiskWriteRequest>(nullptr)); + const auto *request = flatbuffers::GetRoot<dio::FBDiskWriteRequest>(control); + timestamp.store(request->write_create_time_ms()); + sync_requested.store(request->sync()); + write_size.store(request->size()); + ordering_offset.store(request->ordering_zone_offset()); + builder.Finish( + dio::CreateFBDiskWriteResponse(builder, id, 0, dio::FBDiskIoRetCode_Success, sync_receipt.load())); + response = rpc::FBMsgType_EDiskWriteResponse; + } + static_cast<void>(crowdb_rpc_server_submit_response(server, connection, builder.GetBufferPointer(), + builder.GetSize(), bytes == 0 ? nullptr : data.data(), + bytes, response, id)); + } +}; + +class RpcChunkFixedIo : public ::testing::Test +{ + protected: + TestChunkRouteFixture fixture; + TestFixedIoService io; + std::unique_ptr<RpcChunkTransport> transport; + const ChunkId id{0x0200'0000'0000'0042ULL, 144}; + + void SetUp() override + { + io.server = fixture.services[0].server; + for (auto message : + {crowdb::rpc::proto::FBMsgType_EDiskReadRequest, crowdb::rpc::proto::FBMsgType_EDiskWriteRequest}) { + crowdb_rpc_server_register_handler(io.server, message, TestFixedIoService::handle, &io, nullptr); + } + const ct_chunk_rpc_disk_route disk{.disk_id_high = 9, .disk_id_low = 10, .route = fixture.routes[0]}; + ct_chunk_rpc_transport_options options{}; + options.disk_routes = &disk; + options.disk_route_count = 1; + options.rpc_timeout_ms = 50; + options.chunkdb_resolver = fixture.resolver(); + transport = std::make_unique<RpcChunkTransport>(options); + ASSERT_TRUE(transport->allocate_fixed_chunk(id, 4096, 17, PagePurpose::kBtreePage).ok()); + } +}; +} // namespace + +TEST_F(RpcChunkFixedIo, ReadRequiresNewServerReceiptAndKeepsOriginalRequestAge) +{ + const ChunkIoContext context{.created_ms = 1234}; + EXPECT_EQ(transport->verify_fixed_read_fence(id, 1, context).code(), Code::kNotSupported); + io.age_receipt.store(true); + ASSERT_TRUE(transport->verify_fixed_read_fence(id, 1, context).ok()); + EXPECT_EQ(io.timestamp.load(), context.created_ms); + ASSERT_TRUE(transport->advance_write(id, 0, 4).ok()); + std::array<uint8_t, 4> data{}; + ASSERT_TRUE(transport->read_fixed_mirror(id, 0, 0, data.data(), data.size(), context).ok()); + EXPECT_EQ(io.timestamp.load(), context.created_ms); + io.age_receipt.store(false); + EXPECT_EQ(transport->read_fixed_mirror(id, 0, 0, data.data(), data.size(), context).code(), Code::kNotSupported); + EXPECT_TRUE(transport->read_mirror(id, 0, 0, data.data(), data.size()).ok()); +} + +TEST_F(RpcChunkFixedIo, DrainRequiresZeroByteSynchronizedBarrier) +{ + EXPECT_EQ(transport->drain_fixed_chunk(id).code(), Code::kNotSupported); + EXPECT_TRUE(io.sync_requested.load()); + EXPECT_EQ(io.write_size.load(), 0U); + EXPECT_NE(io.ordering_offset.load(), UINT64_MAX); + io.sync_receipt.store(true); + EXPECT_TRUE(transport->drain_fixed_chunk(id).ok()); + EXPECT_EQ(io.write_size.load(), 0U); +} + +TEST_F(RpcChunkFixedIo, WriteRequiresSynchronizedReceiptAndKeepsOriginalRequestAge) +{ + const ChunkIoContext context{.created_ms = 5678}; + const std::array<uint8_t, 4> data{1, 2, 3, 4}; + EXPECT_EQ(transport->write_fixed_mirror(id, 0, 0, data.data(), data.size(), context).code(), Code::kNotSupported); + EXPECT_TRUE(io.sync_requested.load()); + EXPECT_EQ(io.timestamp.load(), context.created_ms); + io.sync_receipt.store(true); + EXPECT_TRUE(transport->write_fixed_mirror(id, 0, 0, data.data(), data.size(), context).ok()); + EXPECT_EQ(io.timestamp.load(), context.created_ms); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/rpc_chunk_staging_test.cpp b/lib/crowdb-tree/tests/integration/rpc_chunk_staging_test.cpp new file mode 100644 index 000000000..3ffcb5f0d --- /dev/null +++ b/lib/crowdb-tree/tests/integration/rpc_chunk_staging_test.cpp @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/rpc_chunk_transport.h" +#include "common/test_chunk_route_service.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +TEST(RpcChunkStaging, FixedIdentityIsInTheSubmittedAllocation) +{ + TestChunkRouteFixture fixture; + const ct_chunk_rpc_disk_route disk{.disk_id_high = 9, .disk_id_low = 10, .route = fixture.routes[0]}; + ct_chunk_rpc_transport_options options{}; + options.disk_routes = &disk; + options.disk_route_count = 1; + options.rpc_timeout_ms = 50; + options.chunkdb_resolver = fixture.resolver(); + RpcChunkTransport transport(options); + const ChunkId id(0x0200'0000'0000'0042ULL, 144); + ASSERT_TRUE(transport.allocate_fixed_chunk(id, 4096, 17, PagePurpose::kBtreePage).ok()); + EXPECT_EQ(fixture.services[0].requested_low.load(), id.low); + EXPECT_EQ(fixture.services[0].allocations.load(), 1U); + EXPECT_EQ(fixture.services[1].allocations.load(), 0U); +} + +TEST(RpcChunkStaging, UnknownAllocationDoesNotRetryWithAnotherIdentity) +{ + TestChunkRouteFixture fixture; + fixture.services[1].drop_allocation = true; + const ct_chunk_rpc_disk_route disk{.disk_id_high = 9, .disk_id_low = 10, .route = fixture.routes[0]}; + ct_chunk_rpc_transport_options options{}; + options.disk_routes = &disk; + options.disk_route_count = 1; + options.rpc_timeout_ms = 50; + options.chunkdb_resolver = fixture.resolver(); + RpcChunkTransport transport(options); + const ChunkId id(0x0200'0000'0000'0042ULL, 145); + EXPECT_FALSE(transport.allocate_fixed_chunk(id, 4096, 17, PagePurpose::kBtreePage).ok()); + EXPECT_EQ(fixture.services[1].requested_low.load(), id.low); + EXPECT_EQ(fixture.services[1].allocations.load(), 1U); + EXPECT_EQ(fixture.services[0].allocations.load(), 0U); +} + +TEST(RpcChunkStaging, DeleteRequiresACompleteTombstoneAndFencesMissingAllocations) +{ + TestChunkRouteFixture fixture; + const ct_chunk_rpc_disk_route disk{.disk_id_high = 9, .disk_id_low = 10, .route = fixture.routes[0]}; + ct_chunk_rpc_transport_options options{}; + options.disk_routes = &disk; + options.disk_route_count = 1; + options.rpc_timeout_ms = 50; + options.chunkdb_resolver = fixture.resolver(); + RpcChunkTransport transport(options); + const ChunkId id(0x0200'0000'0000'0042ULL, 144); + fixture.services[0].unfinished_delete = true; + EXPECT_FALSE(transport.delete_chunk(id).ok()); + EXPECT_TRUE(fixture.services[0].fence_missing.load()); + fixture.services[0].unfinished_delete = false; + EXPECT_TRUE(transport.delete_chunk(id).ok()); + EXPECT_EQ(fixture.services[0].deletes.load(), 2U); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/runtime_block_compaction_test.cpp b/lib/crowdb-tree/tests/integration/runtime_block_compaction_test.cpp new file mode 100644 index 000000000..5b0adf8d1 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_block_compaction_test.cpp @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_checkpoint_internal.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(RuntimeBlockCompaction, DisabledPublicFacadeDoesNotWriteACheckpoint) +{ + tree_test::TempDir directory("runtime_gc_disabled_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.page_store = store.get(); + std::unique_ptr<Crowdbtree> tree; + ASSERT_TRUE(Crowdbtree::open(config, &tree).ok()); + MergeGcStats stats{.blocks_selected = 99}; + ASSERT_TRUE(tree->compact_sparse_blocks(&stats).ok()); + EXPECT_EQ(stats.blocks_selected, 0U); + EXPECT_EQ(tree->durable_snapshot_seq(), 0U); +} + +TEST(RuntimeBlockCompaction, PublicPassRelocatesSparseStorageAndReopens) +{ + tree_test::TempDir directory("runtime_gc_public_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.page_store = store.get(); + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.compression = compress_algo::kNone; + config.merge_gc_max_relocation_bytes = 1; // The first block is always allowed. + config.runtime_binding.limits.io_operations = 1; + config.runtime_binding.cpu_queue_limit = 3; + std::shared_ptr<TreeCore> core; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + core->local_backend()->gc_active.store(true); + ASSERT_TRUE( + core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = std::string(100000, 'a')}}}).ok()); + ASSERT_TRUE(core->checkpoint().ok()); + core->binding()->drain(); + ASSERT_TRUE(core->apply(2, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "kept"}}}).ok()); + ASSERT_TRUE(core->checkpoint().ok()); + core->binding()->drain(); + ASSERT_TRUE(core->checkpoint().ok()); + core->binding()->drain(); + core->local_backend()->gc_active.store(false); + MergeGcStats stats; + auto status = core->compact_sparse_blocks(&stats); + ASSERT_TRUE(status.ok()) << status.to_string(); + core->binding()->drain(); + EXPECT_GT(stats.blocks_deleted, 0U); + EXPECT_EQ(stats.blocks_selected, 1U); + EXPECT_GT(stats.pages_relocated, 0U); + EXPECT_EQ(core->local_backend()->checkpoints->durable_coverage(), 2U); + GetView value; + ASSERT_TRUE(core->get("a", &value).ok()); + EXPECT_EQ(value.value().to_string(), "kept"); + value = {}; + core.reset(); + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + ASSERT_TRUE(core->get("a", &value).ok()); + EXPECT_EQ(value.value().to_string(), "kept"); +} + +TEST(RuntimeBlockCompaction, ConcurrentCleanupAndInvalidThresholdLeaveCoverageUnchanged) +{ + tree_test::TempDir directory("runtime_gc_busy_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.page_store = store.get(); + config.merge_gc_max_relocation_bytes = 1; + std::shared_ptr<TreeCore> core; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + core->local_backend()->gc_active.store(true); + MergeGcStats stats; + EXPECT_EQ(core->compact_sparse_blocks(&stats).code(), Code::kUnavailable); + EXPECT_EQ(stats.blocks_selected, 0U); + core->local_backend()->gc_active.store(false); + EXPECT_EQ(core->local_backend()->checkpoints->durable_coverage(), 0U); + core.reset(); + config.merge_gc_block_free_threshold = 1.0; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + EXPECT_EQ(core->compact_sparse_blocks(&stats).code(), Code::kInvalidArgument); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/runtime_eviction_test.cpp b/lib/crowdb-tree/tests/integration/runtime_eviction_test.cpp new file mode 100644 index 000000000..dfe7e8c67 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_eviction_test.cpp @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_tree_core_chunk.h" + +namespace crowdb::tree +{ +TEST(RuntimeEviction, CleanRoutingAndLeafCacheEvictIndependentlyWithoutLosingOldReaders) +{ + test_chunk::TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + Batch batch; + for (size_t i = 0; i < 80; ++i) { + batch.ops.push_back({.key = std::to_string(1000 + i), .kind = OpKind::kPut, .value = std::string(60, 'x')}); + } + ASSERT_TRUE(test.tree->apply(1, batch).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + auto *const core = TreeCoreAccess::get(*test.tree); + const auto old = core->capture(); + ASSERT_GT(old->stats.inner_pages, 0U); + ASSERT_TRUE(old->mapping.get(old->root_page_id)->inner); + EXPECT_EQ(test.tree->evict_clean_inner(old->stats.inner_pages), 0U); + EXPECT_GT(test.tree->evict_clean_inner(0), 0U); + EXPECT_FALSE(core->capture()->mapping.get(old->root_page_id)->inner); + EXPECT_TRUE(old->mapping.get(old->root_page_id)->inner); + uint64_t slot = 0; + std::string value; + ASSERT_TRUE(test.tree->get("1000", &slot, &value)); + EXPECT_EQ(value, std::string(60, 'x')); + const auto metrics = core->publisher().work_metrics(); + const auto loads = metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count; + const auto encodes = metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count; + EXPECT_GT(test.tree->evict_clean_leaves(0), 0U); + ASSERT_TRUE(test.tree->get("1000", &slot, &value)); + EXPECT_EQ(value, std::string(60, 'x')); + EXPECT_GT(metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count, loads); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count, encodes); + EXPECT_EQ(test.tree->last_applied_slot(), 1U); +} + +TEST(RuntimeEviction, UnbackedDirtyDataSurvivesBothZeroBudgets) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + Crowdbtree tree(config); + ASSERT_TRUE(tree.apply(1, Batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "value"}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_EQ(tree.evict_clean_leaves(0), 0U); + EXPECT_EQ(tree.evict_clean_inner(0), 0U); + uint64_t slot = 0; + std::string value; + ASSERT_TRUE(tree.get("key", &slot, &value)); + EXPECT_EQ(slot, 1U); + EXPECT_EQ(value, "value"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/runtime_inspection_test.cpp b/lib/crowdb-tree/tests/integration/runtime_inspection_test.cpp new file mode 100644 index 000000000..2509b32d8 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_inspection_test.cpp @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/backend/page_store.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestInspection +{ + Config config; + std::shared_ptr<MemPageStore> store = std::make_shared<MemPageStore>(4096); + std::unique_ptr<Crowdbtree> tree; + uint64_t runtime_bytes = 0; + + TestInspection() + { + EXPECT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + runtime_bytes = config.runtime->snapshot().allocated.routing_bytes; + config.page_store = store.get(); + config.backend_owner = store; + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.runtime_binding.limits.io_operations = 1; + EXPECT_TRUE(Crowdbtree::open(config, &tree).ok()); + } +}; + +TEST(RuntimeInspection, LogicalOverlayIsEncodedWithoutChangingPublicationOrAllocatingPids) +{ + TestInspection test; + ASSERT_TRUE(test.tree->apply(1, Batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "value"}}}).ok()); + ASSERT_TRUE(test.tree->flush().ok()); + auto metrics = TreeCoreAccess::get(*test.tree)->publisher().work_metrics(); + const auto encoding_before = metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count; + const auto before = test.tree->version(); + NativeFrame frame; + uint64_t version = 0; + uint64_t root = 0; + uint32_t deltas = 0; + auto status = test.tree->inspect_page({}, before, &frame, &version, &root, &deltas); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count, encoding_before + 1); + metrics.reset(); + EXPECT_EQ(before, version); + EXPECT_EQ(before, test.tree->version()); + EXPECT_EQ(root, frame.page_id); + ASSERT_TRUE(frame_validate(frame.frame.data(), frame.frame.size())); + LeafFrameView leaf(frame.frame.data(), frame.frame.size()); + ASSERT_EQ(leaf.count(), 1U); + EXPECT_EQ(leaf.key(0).to_string(), "key"); + EXPECT_EQ(CellView(leaf.cell(0)).value().to_string(), "value"); + test.tree.reset(); + EXPECT_EQ(CellView(leaf.cell(0)).value().to_string(), "value"); + EXPECT_GE(test.config.runtime->snapshot().allocated.routing_bytes, test.runtime_bytes + frame.frame.capacity()); + frame = {}; + EXPECT_EQ(test.config.runtime->snapshot().allocated.routing_bytes, test.runtime_bytes); +} + +TEST(RuntimeInspection, ColdIndexAndLeafUseOneIoPermitAndRejectStaleOrInvalidPaths) +{ + TestInspection test; + Batch batch; + for (size_t i = 0; i < 100; ++i) { + batch.ops.push_back({.key = std::to_string(1000 + i), .kind = OpKind::kPut, .value = std::string(100, 'v')}); + } + ASSERT_TRUE(test.tree->apply(1, batch).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + test.tree.reset(); + ASSERT_TRUE(Crowdbtree::open(test.config, &test.tree).ok()); + NativeFrame frame; + uint64_t version = 0; + uint64_t root = 0; + uint32_t deltas = 0; + auto status = test.tree->inspect_page({}, kInvalidPageId, &frame, &version, &root, &deltas); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(frame_page_type(frame.frame.data()), page_type::kInnerBase); + const auto revision = version; + status = test.tree->inspect_page({0}, revision, &frame, &version, &root, &deltas); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(frame_page_type(frame.frame.data()), page_type::kLeafBase); + EXPECT_EQ(test.tree->inspect_page({UINT32_MAX}, revision, &frame, &version, &root, &deltas).code(), + Code::kInvalidArgument); + ASSERT_TRUE(test.tree->apply(2, {}).ok()); + ASSERT_TRUE(test.tree->flush().ok()); + EXPECT_EQ(test.tree->inspect_page({}, revision, &frame, &version, &root, &deltas).code(), Code::kUnavailable); + EXPECT_EQ( + test.tree->inspect_page(std::vector<uint32_t>(33), kInvalidPageId, &frame, &version, &root, &deltas).code(), + Code::kInvalidArgument); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/runtime_migration_test.cpp b/lib/crowdb-tree/tests/integration/runtime_migration_test.cpp new file mode 100644 index 000000000..30dec122f --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_migration_test.cpp @@ -0,0 +1,185 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +void expect_canonical_native(Crowdbtree &tree, bool expect_inner, bool expect_overflow) +{ + std::unique_ptr<NativeFrameIterator> cursor; + auto status = tree.open_native_frame_iterator(nullptr, &cursor); + ASSERT_TRUE(status.ok()) << status.to_string(); + uint64_t inner = 0; + uint64_t overflow = 0; + uint64_t frames = 0; + bool done = false; + while (!done && frames < 1024) { + std::optional<NativeFrameView> frame; + status = cursor->next_owned(&frame, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (!frame) { + continue; + } + ++frames; + EXPECT_TRUE(frame_validate(frame->bytes.data(), frame->bytes.size())); + EXPECT_EQ(frame_u64(frame->bytes.data(), fh::kSelfpage_id), frame->page_id); + inner += static_cast<uint64_t>(frame_page_type(frame->bytes.data()) == page_type::kInnerBase); + overflow += static_cast<uint64_t>(frame_page_type(frame->bytes.data()) == page_type::kOverflowFrame); + } + EXPECT_TRUE(done); + EXPECT_GT(frames, 0U); + EXPECT_EQ(inner != 0, expect_inner); + EXPECT_EQ(overflow != 0, expect_overflow); +} + +TEST(RuntimeMigration, LegacyBlockCheckpointOpensOnTheSamePathAndRemainsDurable) +{ + tree_test::TempDir directory("runtime_migrate_legacy_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + Config config; + config.page_store = store.get(); + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.compression = compress_algo::kNone; + std::unique_ptr<Crowdbtree> tree; + auto status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string large(12000, 'v'); + ASSERT_TRUE(tree->apply(1, Batch{.ops = {{.key = "alpha", .kind = OpKind::kPut, .value = large}}}).ok()); + ASSERT_TRUE(tree->apply(2, Batch{.ops = {{.key = "beta", .kind = OpKind::kPut, .value = "deleted"}}}).ok()); + ASSERT_TRUE(tree->apply(3, Batch{.ops = {{.key = "beta", .kind = OpKind::kDelete, .value = {}}}}).ok()); + ASSERT_TRUE(tree->flush().ok()); + uint64_t coverage = 0; + uint64_t sequence = 0; + status = tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(coverage, 3U); + tree.reset(); + store.reset(); + + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + config.page_store = store.get(); + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(tree->durable_snapshot_last_applied_slot(), 3U); + auto *core = TreeCoreAccess::get(*tree); + ASSERT_NE(core, nullptr); + GetView found; + status = core->get("alpha", &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), large); + EXPECT_EQ(found.slot(), 1U); + found = {}; + status = core->get("beta", &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_FALSE(found.found()); + std::vector<scan_entry> rows; + bool truncated = true; + status = tree->scan({}, {}, {}, 16, 64 * 1024, false, 0, &rows, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(rows.size(), 1U); + EXPECT_FALSE(truncated); + EXPECT_EQ(rows.front().key, "alpha"); + EXPECT_EQ(rows.front().value, large); + expect_canonical_native(*tree, false, true); + // A checkpoint with no writes reuses the legacy primary and overflow extents. + status = tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(coverage, 3U); + tree.reset(); + store.reset(); + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + config.page_store = store.get(); + status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + expect_canonical_native(*tree, false, true); + ASSERT_TRUE(tree->apply(4, Batch{.ops = {{.key = "gamma", .kind = OpKind::kPut, .value = "runtime"}}}).ok()); + status = tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 4U); + tree.reset(); + store.reset(); + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + config.page_store = store.get(); + status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->durable_snapshot_last_applied_slot(), 4U); + core = TreeCoreAccess::get(*tree); + status = core->get("gamma", &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), "runtime"); +} + +TEST(RuntimeMigration, LegacyInnerTreeRemainsColdReadableAndExportsCanonicalFrames) +{ + tree_test::TempDir directory("runtime_migrate_inner_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + Config config; + config.page_store = store.get(); + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.leaf_split_bytes = 512; + config.max_delta_len = 0; + config.leaf_merge_bytes = 128; + config.inner_max_keys = 3; + config.compression = compress_algo::kNone; + std::unique_ptr<Crowdbtree> tree; + ASSERT_TRUE(Crowdbtree::open(config, &tree).ok()); + const std::string value(32, 'v'); + for (uint64_t slot = 1; slot <= 64; ++slot) { + ASSERT_TRUE( + tree->apply( + slot, + Batch{.ops = {{.key = "key" + std::to_string(1000 + slot), .kind = OpKind::kPut, .value = value}}}) + .ok()); + } + ASSERT_TRUE(tree->flush().ok()); + uint64_t coverage = 0; + ASSERT_TRUE(tree->snapshot(&coverage).ok()); + ASSERT_EQ(coverage, 64U); + ASSERT_GT(tree->tree_summary().reachable_inner_pages, 0U); + tree.reset(); + store.reset(); + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + for (size_t reopen = 0; reopen < 2; ++reopen) { + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &store).ok()); + config.page_store = store.get(); + auto status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->durable_snapshot_last_applied_slot(), 64U); + EXPECT_GT(tree->tree_state_stats().logical.inner_pages, 0U); + GetView found; + status = TreeCoreAccess::get(*tree)->get("key1064", &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), value); + EXPECT_EQ(found.slot(), 64U); + found = {}; + std::vector<scan_entry> rows; + bool truncated = true; + status = tree->scan({}, {}, {}, 64, 64 * 1024, false, 0, &rows, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_FALSE(truncated); + EXPECT_EQ(rows.size(), 64U); + expect_canonical_native(*tree, true, false); + status = tree->snapshot(&coverage); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 64U); + tree.reset(); + store.reset(); + } +} + +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/runtime_range_rebuild_test.cpp b/lib/crowdb-tree/tests/integration/runtime_range_rebuild_test.cpp new file mode 100644 index 000000000..4d60a7ca6 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_range_rebuild_test.cpp @@ -0,0 +1,167 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/btree/range_rebuild.h" +#include "local_checkpoint_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestRangeBackend +{ + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + + TestRangeBackend() + { + store.device = device; + } + + Config config(std::shared_ptr<TreeRuntime> runtime = {}) + { + Config out; + out.runtime = std::move(runtime); + out.frame_bytes = 4096; + out.max_inline_value = 32; + out.page_store = device.get(); + out.async_page_store = &store; + out.runtime_binding.limits.io_operations = 1; + return out; + } +}; + +class RuntimeRangeRebuild : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + source_config = source_backend->config(runtime); + source_config.backend_owner = source_backend; + auto status = Crowdbtree::open(source_config, &source); + ASSERT_TRUE(status.ok()) << status.to_string(); + Batch batch; + for (size_t i = 0; i != 100; ++i) { + const auto key = std::string(3 - std::to_string(i).size(), '0') + std::to_string(i); + batch.ops.push_back({.key = key, .kind = OpKind::kPut, .value = value}); + } + ASSERT_TRUE(source->apply(1, batch).ok()); + ASSERT_TRUE(source->apply(2, Batch{.ops = {{.key = "050", .kind = OpKind::kDelete, .value = ""}}}).ok()); + ASSERT_TRUE(source->snapshot().ok()); + } + + Config target_options(bool explicit_runtime = true) + { + auto config = target_backend->config(explicit_runtime ? runtime : nullptr); + config.backend_owner = target_backend; + return config; + } + + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<TestRangeBackend> source_backend = std::make_shared<TestRangeBackend>(); + std::shared_ptr<TestRangeBackend> target_backend = std::make_shared<TestRangeBackend>(); + Config source_config; + std::unique_ptr<Crowdbtree> source; + const std::string value = std::string(9000, 'r'); +}; + +TEST_F(RuntimeRangeRebuild, BoundedRebuildPersistsOnlyItsRangeAndRemainsIndependentlyMutable) +{ + auto options = target_options(); + const auto range = KeyRange::bounded(std::string("030"), std::string("070")); + std::unique_ptr<Crowdbtree> target; + RangeRebuildStats stats; + auto status = rebuild_range(*source, range, options, &target, &stats); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(stats.entries_emitted, 40U); + EXPECT_EQ(stats.pages_reused, 0U); + EXPECT_GT(stats.pages_rebuilt, 0U); + EXPECT_NE(TreeCoreAccess::get(*source)->binding()->account(), TreeCoreAccess::get(*target)->binding()->account()); + EXPECT_EQ(target->get_view("030").value().to_string(), value); + EXPECT_EQ(target->get_view("069").value().to_string(), value); + EXPECT_FALSE(target->get_view("029").found()); + EXPECT_FALSE(target->get_view("070").found()); + EXPECT_FALSE(target->get_view("050").found()); + ASSERT_TRUE(source->put("040", "source changed").ok()); + EXPECT_EQ(target->get_view("040").value().to_string(), value); + ASSERT_TRUE(target->put("040", "target changed").ok()); + EXPECT_EQ(target->get_view("040").slot(), 3U); + EXPECT_EQ(target->put("outside", "rejected").code(), Code::kInvalidArgument); + ASSERT_TRUE(target->snapshot().ok()); + target.reset(); + options.key_range = range; + status = Crowdbtree::open(options, &target); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(target->get_view("040").value().to_string(), "target changed"); + EXPECT_FALSE(target->get_view("050").found()); + EXPECT_EQ(source->get_view("040").value().to_string(), "source changed"); +} + +TEST_F(RuntimeRangeRebuild, WholeNativeCopyUsesTheRequestedHostAndANewBackendDomain) +{ + auto options = target_options(); + std::unique_ptr<Crowdbtree> target; + RangeRebuildStats stats; + auto status = rebuild_range(*source, KeyRange::unbounded(), options, &target, &stats); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto *src = TreeCoreAccess::get(*source); + auto *dst = TreeCoreAccess::get(*target); + ASSERT_NE(dst, nullptr); + EXPECT_EQ(dst->config().runtime, runtime); + EXPECT_NE(src->binding()->account(), dst->binding()->account()); + EXPECT_EQ(stats.entries_emitted, 100U); + EXPECT_EQ(stats.pages_reused, 0U); + EXPECT_GT(stats.pages_rebuilt, 0U); + EXPECT_EQ(target->get_view("099").value().to_string(), value); + auto retained = target->get_view("099"); + source.reset(); + target.reset(); + EXPECT_EQ(retained.value().to_string(), value); + options.runtime = runtime; + status = Crowdbtree::open(options, &target); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(target->get_view("099").value().to_string(), value); +} + +TEST_F(RuntimeRangeRebuild, DifferentGeometryAndEmptyRangePreserveCapturedCoverage) +{ + auto options = target_options(); + options.frame_bytes = 8192; + const auto range = KeyRange::bounded(std::string("050"), std::string("050")); + std::unique_ptr<Crowdbtree> target; + RangeRebuildStats stats; + auto status = rebuild_range(*source, range, options, &target, &stats); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(stats.entries_emitted, 0U); + EXPECT_EQ(target->contiguous_slot(), 2U); + EXPECT_FALSE(target->get_view("030").found()); + EXPECT_EQ(target->put("050", "outside").code(), Code::kInvalidArgument); + target.reset(); + options.key_range = range; + status = Crowdbtree::open(options, &target); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(target->durable_snapshot_last_applied_slot(), 2U); +} + +TEST_F(RuntimeRangeRebuild, FailedDestinationStagingPreservesExistingOutputAndSourceValues) +{ + auto options = target_options(); + options.runtime_binding.limits.reclaim_bytes = 512; + Config sentinel_config; + sentinel_config.runtime = runtime; + auto target = std::make_unique<Crowdbtree>(sentinel_config); + auto *original = target.get(); + ASSERT_TRUE(target->put("retained", "old output").ok()); + RangeRebuildStats stats; + stats.entries_emitted = 99; + auto status = + rebuild_range(*source, KeyRange::bounded(std::string("030"), std::string("070")), options, &target, &stats); + EXPECT_EQ(status.code(), Code::kResourceExhausted) << status.to_string(); + EXPECT_EQ(target.get(), original); + EXPECT_EQ(target->get_view("retained").value().to_string(), "old output"); + EXPECT_EQ(stats.entries_emitted, 99U); + EXPECT_EQ(source->get_view("040").value().to_string(), value); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/runtime_read_window_test.cpp b/lib/crowdb-tree/tests/integration/runtime_read_window_test.cpp new file mode 100644 index 000000000..9abef5ae9 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_read_window_test.cpp @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "crowdb-tree/backend/page_store.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(RuntimeReadWindow, DefaultFrameGeometryReadsColdInnerAndLeafWindowsWithinItsOriginalQuota) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + const auto store = std::make_shared<MemPageStore>(4096); + config.page_store = store.get(); + config.backend_owner = store; + config.runtime_binding.limits.io_operations = 1; + std::shared_ptr<TreeCore> core; + auto status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(80, 'v'); + for (size_t part = 0; part < 12; ++part) { + Batch batch; + for (size_t i = part * 100; i < (part + 1) * 100; ++i) { + batch.ops.push_back({.key = "key" + std::to_string(10000 + i), .kind = OpKind::kPut, .value = value}); + } + status = core->apply(part + 1, batch); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + status = core->checkpoint(); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_GT(core->capture()->stats.inner_pages, 0U); + core.reset(); + status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + GetView found; + status = core->get("key11199", &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), value); + size_t count = 0; + std::string cursor; + bool truncated = true; + while (truncated) { + CoreScanOptions options; + options.start = cursor; + options.has_start_bound = !cursor.empty(); + options.limit = 128; + std::vector<scan_entry> entries; + status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + for (const auto &entry : entries) { + EXPECT_EQ(entry.key, "key" + std::to_string(10000 + count)); + EXPECT_EQ(entry.value, value); + ++count; + } + if (!entries.empty()) { + cursor = entries.back().key; + } + ASSERT_TRUE(!truncated || !entries.empty()); + } + EXPECT_EQ(count, 1200U); + EXPECT_EQ(core->durable_coverage(), 12U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/runtime_snapshot_import_test.cpp b/lib/crowdb-tree/tests/integration/runtime_snapshot_import_test.cpp new file mode 100644 index 000000000..68d6ca2e6 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_snapshot_import_test.cpp @@ -0,0 +1,265 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/snapshot/snapshot_io.h" +#include "native_import_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +void test_u32(std::string &wire, uint32_t value) +{ + for (unsigned i = 0; i != 4; ++i) { + wire.push_back(static_cast<char>(value >> (8 * i))); + } +} + +void test_u64(std::string &wire, uint64_t value) +{ + for (unsigned i = 0; i != 8; ++i) { + wire.push_back(static_cast<char>(value >> (8 * i))); + } +} + +void test_crc(std::string &wire) +{ + test_u32(wire, crowdb::common::crc32c(reinterpret_cast<const uint8_t *>(wire.data()), wire.size())); +} + +struct TestTuple +{ + std::string key; + uint64_t slot; + uint8_t kind; + std::string value; +}; + +std::string test_portable(const std::vector<TestTuple> &entries, uint64_t coverage = 17) +{ + std::string wire; + test_u32(wire, 0x4E535443); + test_u32(wire, 1); + wire.push_back(0); + test_u64(wire, coverage); + test_u64(wire, entries.size()); + for (const auto &e : entries) { + test_u32(wire, e.key.size()); + wire += e.key; + test_u64(wire, e.slot); + wire.push_back(static_cast<char>(e.kind)); + test_u32(wire, e.value.size()); + wire += e.value; + } + test_crc(wire); + return wire; +} + +std::string test_native(const std::vector<NativeFrame> &frames, uint64_t coverage = 17) +{ + std::string wire; + test_u32(wire, 0x4E535443); + test_u32(wire, 1); + wire.push_back(1); + test_u64(wire, coverage); + test_u64(wire, frames.empty() ? kInvalidPageId : 0); + test_u64(wire, frames.size()); + for (const auto &f : frames) { + test_u64(wire, f.page_id); + test_u32(wire, f.frame.size()); + wire.append(reinterpret_cast<const char *>(f.frame.data()), f.frame.size()); + } + test_crc(wire); + return wire; +} + +class RuntimeSnapshotImportTest : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.frame_bytes = 4096; + config.max_inline_value = 32; + config.runtime_binding.limits.io_operations = 1; + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + ASSERT_TRUE(tree->put("old", "retained").ok()); + } + + static Status feed_chunks(SnapshotImport &importer, const std::string &wire, size_t chunk) + { + for (size_t offset = 0; offset < wire.size(); offset += chunk) { + auto status = importer.feed(Slice(wire.data() + offset, std::min(chunk, wire.size() - offset))); + if (!status.ok()) { + return status; + } + } + return Status::Ok(); + } + + Config config; + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; +}; + +TEST_F(RuntimeSnapshotImportTest, PublicPortableInstallReplacesGenerationWithBalancedOverflowGraph) +{ + const auto old = tree->get_view("old"); + std::vector<leaf_entry> entries; + for (size_t i = 0; i != 200; ++i) { + const auto key = std::string(3 - std::to_string(i).size(), '0') + std::to_string(i); + entries.push_back({.key = key, .cell = encode_cell_buf(i + 1, OpKind::kPut, std::string(9000, 'v'))}); + } + const auto status = tree->install_snapshot(std::move(entries), 250); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(core->capture()->stats.kv_known_count, 200U); + EXPECT_GT(core->capture()->stats.height, 1U); + EXPECT_EQ(tree->get_view("199").value().size(), 9000U); + EXPECT_EQ(old.value().to_string(), "retained"); + ASSERT_TRUE(tree->put("next", "value").ok()); + EXPECT_EQ(tree->get_view("next").slot(), 251U); +} + +TEST_F(RuntimeSnapshotImportTest, PortableEveryByteBoundaryPreservesEmptyKeyTombstoneAndLargeValue) +{ + const std::string value(9000, 'v'); + const auto wire = test_portable({ + {.key = "", .slot = 3, .kind = 0, .value = "empty"}, + {.key = "gone", .slot = 5, .kind = 1, .value = "" }, + {.key = "key", .slot = 7, .kind = 0, .value = value } + }); + const auto old = tree->get_view("old"); + SnapshotImport importer(*tree); + auto status = feed_chunks(importer, wire, 1); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->get_view("old").value().to_string(), "retained"); + uint64_t coverage = 0; + status = importer.finish(&coverage); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 17U); + EXPECT_EQ(tree->get_view("").value().to_string(), "empty"); + EXPECT_FALSE(tree->get_view("gone").found()); + EXPECT_EQ(tree->get_view("key").value().to_string(), value); + EXPECT_EQ(old.value().to_string(), "retained"); + ASSERT_TRUE(tree->put("next", "value").ok()); + EXPECT_EQ(tree->get_view("next").slot(), 18U); +} + +TEST_F(RuntimeSnapshotImportTest, CrcTruncationOrderAndFieldFailuresNeverReplaceTheOldRoot) +{ + const auto original = core->capture(); + std::vector<std::string> invalid; + auto crc = test_portable({ + {.key = "key", .slot = 7, .kind = 0, .value = "value"} + }); + crc.back() ^= 1; + invalid.push_back(std::move(crc)); + auto truncated = test_portable({ + {.key = "key", .slot = 7, .kind = 0, .value = "value"} + }); + truncated.pop_back(); + invalid.push_back(std::move(truncated)); + auto trailing = test_portable({}); + trailing.push_back('x'); + invalid.push_back(std::move(trailing)); + invalid.push_back(test_portable({ + {.key = "z", .slot = 7, .kind = 0, .value = "value"}, + {.key = "a", .slot = 7, .kind = 0, .value = "value"} + })); + invalid.push_back(test_portable({ + {.key = "future", .slot = 18, .kind = 0, .value = "value"} + })); + invalid.push_back(test_portable({ + {.key = "bad", .slot = 7, .kind = 2, .value = "value"} + })); + invalid.push_back(test_portable({ + {.key = "bad", .slot = 7, .kind = 1, .value = "value"} + })); + auto oversized = test_portable({ + {.key = "key", .slot = 7, .kind = 0, .value = "value"} + }); + const auto length_offset = 25 + 4 + 3 + 8 + 1; + for (size_t i = 0; i != 4; ++i) { + oversized[length_offset + i] = static_cast<char>(0xff); + } + invalid.push_back(std::move(oversized)); + for (const auto &wire : invalid) { + SnapshotImport importer(*tree); + auto status = feed_chunks(importer, wire, 13); + if (status.ok()) { + status = importer.finish(nullptr); + } + EXPECT_FALSE(status.ok()); + EXPECT_EQ(core->capture(), original); + EXPECT_EQ(tree->get_view("old").value().to_string(), "retained"); + } + ASSERT_TRUE(tree->put("after", "works").ok()); + EXPECT_EQ(tree->get_view("after").slot(), 2U); +} + +TEST_F(RuntimeSnapshotImportTest, NativeStreamsRetainOnlyChargedFramesBeforeAtomicInstall) +{ + const std::string value(9000, 'n'); + const auto wire = test_native(test_import::overflow_tree(value)); + const auto account = core->binding()->account(); + const auto reclaim = static_cast<size_t>(detail::BudgetClass::kReclaim); + SnapshotImport importer(*tree); + auto status = feed_chunks(importer, wire, 17); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_GT(account->snapshot().allocated[reclaim], wire.size()); + EXPECT_EQ(config.runtime->snapshot().reserved.io_operations, 0U); + EXPECT_EQ(core->capture()->coverage, 0U); + status = importer.finish(nullptr); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->get_view("key").value().to_string(), value); + EXPECT_EQ(core->capture()->coverage, 17U); +} + +TEST_F(RuntimeSnapshotImportTest, IdlePartialImportHoldsNoIoTicketAndCloseDoesNotWaitForItsOwner) +{ + const auto account = core->binding()->account(); + const auto wire = test_portable({ + {.key = "key", .slot = 7, .kind = 0, .value = "value"} + }); + { + SnapshotImport importer(*tree); + ASSERT_TRUE(importer.feed(Slice(wire.data(), 9)).ok()); + auto io = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kIo, 1); + ASSERT_TRUE(io); + io.reset(); + core->close_and_drain(); + EXPECT_EQ(importer.feed(Slice(wire.data() + 9, wire.size() - 9)).code(), Code::kUnavailable); + } + const auto reclaim = static_cast<size_t>(detail::BudgetClass::kReclaim); + EXPECT_EQ(account->snapshot().allocated[reclaim], 0U); + EXPECT_EQ(account->snapshot().reserved[reclaim], 0U); +} + +TEST_F(RuntimeSnapshotImportTest, EmptyStreamsKeepCoverageAndAdmissionFailureLeavesSourceUsable) +{ + for (bool native : {false, true}) { + SnapshotImport importer(*tree); + const auto wire = native ? test_native({}, 23) : test_portable({}, 23); + ASSERT_TRUE(feed_chunks(importer, wire, 1).ok()); + auto status = importer.finish(nullptr); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->contiguous_slot(), 23U); + ASSERT_TRUE(tree->put("after", "value").ok()); + EXPECT_EQ(tree->get_view("after").slot(), 24U); + } + const auto original = core->capture(); + const auto account = core->binding()->account(); + const auto index = static_cast<size_t>(detail::BudgetClass::kReclaim); + const auto sample = account->snapshot(); + auto pressure = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, + config.runtime_binding.limits.reclaim_bytes - + sample.allocated[index] - sample.reserved[index]); + ASSERT_TRUE(pressure); + SnapshotImport importer(*tree); + EXPECT_EQ(importer.feed(test_portable({})).code(), Code::kResourceExhausted); + EXPECT_EQ(core->capture(), original); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/runtime_split_hint_test.cpp b/lib/crowdb-tree/tests/integration/runtime_split_hint_test.cpp new file mode 100644 index 000000000..d8c15a193 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_split_hint_test.cpp @@ -0,0 +1,51 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(RuntimeSplitHint, OverlayHintDoesNotLoadEncodeOrAdvanceCoverage) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + Crowdbtree tree(config); + EXPECT_FALSE(tree.approximate_split_key()); + ASSERT_TRUE(tree.apply(1, + Batch{ + .ops = {{.key = "a", .kind = OpKind::kPut, .value = "a"}, + {.key = "b", .kind = OpKind::kPut, .value = "b"}, + {.key = "c", .kind = OpKind::kPut, .value = "c"}} + }) + .ok()); + EXPECT_FALSE(tree.approximate_split_key()); + ASSERT_TRUE(tree.flush().ok()); + const auto before = tree.version(); + const auto metrics = TreeCoreAccess::get(tree)->publisher().work_metrics(); + const auto loads = metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count; + const auto encodes = metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count; + EXPECT_EQ(tree.approximate_split_key(), std::optional<std::string>("b")); + EXPECT_EQ(tree.version(), before); + EXPECT_EQ(tree.last_applied_slot(), 1U); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count, loads); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count, encodes); +} + +TEST(RuntimeSplitHint, EmptyAndBinaryRangeEndpointsAreExcluded) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.key_range = KeyRange::bounded(std::string{}, std::string("\0c", 2)); + Crowdbtree tree(config); + const std::string middle("\0b", 2); + ASSERT_TRUE(tree.apply(1, + Batch{ + .ops = {{.key = "", .kind = OpKind::kPut, .value = "a"}, + {.key = middle, .kind = OpKind::kPut, .value = "b"}} + }) + .ok()); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_EQ(tree.approximate_split_key(), std::optional(middle)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/runtime_summary_test.cpp b/lib/crowdb-tree/tests/integration/runtime_summary_test.cpp new file mode 100644 index 000000000..6be6f060e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/runtime_summary_test.cpp @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_tree_core_chunk.h" + +namespace crowdb::tree +{ +namespace +{ +Batch summary_data() +{ + Batch batch; + for (size_t i = 0; i < 80; ++i) { + batch.ops.push_back( + {.key = std::to_string(1000 + i), .kind = OpKind::kPut, .value = std::string(i == 0 ? 20000 : 60, 'x')}); + } + return batch; +} + +void check_summary(const Crowdbtree &tree, uint64_t keys, uint64_t key_bytes, uint64_t values) +{ + const auto summary = tree.tree_summary(); + ASSERT_TRUE(summary.exact); + EXPECT_EQ(summary.live_kv, keys); + EXPECT_EQ(summary.live_key_bytes, key_bytes); + EXPECT_EQ(summary.live_value_bytes, values); + EXPECT_EQ(summary.live_logical_bytes, key_bytes + values); + EXPECT_GT(summary.reachable_leaf_pages, 1U); + EXPECT_GT(summary.reachable_inner_pages, 0U); + EXPECT_GT(summary.reachable_overflow_pages, 0U); + EXPECT_EQ(summary.reachable_page_capacity_bytes, + 4096 * (summary.reachable_leaf_pages + summary.reachable_inner_pages + summary.reachable_overflow_pages)); +} +} // namespace + +TEST(RuntimeSummary, CheckpointAndRecoveryExposeCompleteCountsWithoutLoadingPages) +{ + test_chunk::TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.tree->apply(1, summary_data()).ok()); + EXPECT_FALSE(test.tree->tree_summary().exact); + ASSERT_TRUE(test.tree->snapshot().ok()); + check_summary(*test.tree, 80, 320, 20000 + (79 * 60)); + const auto saved = test.tree->tree_summary(); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + const auto metrics = TreeCoreAccess::get(*test.tree)->publisher().work_metrics(); + const auto loads = metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count; + const auto encodes = metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count; + check_summary(*test.tree, 80, 320, 20000 + (79 * 60)); + EXPECT_EQ(test.tree->tree_summary().reachable_page_capacity_bytes, saved.reachable_page_capacity_bytes); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count, loads); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count, encodes); +} + +TEST(RuntimeSummary, UnflushedAndPublishedWritesInvalidateOnlyTheCurrentSummary) +{ + test_chunk::TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.tree->apply(1, summary_data()).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + auto old = TreeCoreAccess::get(*test.tree)->capture(); + if (!old->summary) { + FAIL() << "fixed summary missing after checkpoint"; + } + ASSERT_TRUE(test.tree->apply(2, Batch{.ops = {{.key = "1001", .kind = OpKind::kDelete, .value = {}}}}).ok()); + EXPECT_FALSE(test.tree->tree_summary().exact); + ASSERT_TRUE(test.tree->flush().ok()); + EXPECT_FALSE(test.tree->tree_summary().exact); + ASSERT_TRUE(test.tree->snapshot().ok()); + check_summary(*test.tree, 79, 316, 20000 + (78 * 60)); + EXPECT_EQ(old->summary->live_kv, 80U); + EXPECT_EQ(old->summary->live_value_bytes, 20000 + (79 * 60)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/snapshot_change_effect_test.cpp b/lib/crowdb-tree/tests/integration/snapshot_change_effect_test.cpp new file mode 100644 index 000000000..35a267bd7 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/snapshot_change_effect_test.cpp @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "checkpoint_structure_test_support.h" +#include "snapshot_publication_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_snapshot_publication; + +TEST(SnapshotChangeEffect, RealMaintenanceSplitRecordsEquivalentTopologyWithoutFullContent) +{ + test_structure::TestCheckpointStructure test; + ASSERT_TRUE(test.publish(test_structure::structural_input()).ok()); + ASSERT_TRUE(checkpoint(test.publisher, CheckpointAction::Capture, 1).status.ok()); + ASSERT_TRUE(checkpoint(test.publisher, CheckpointAction::Commit, 1).status.ok()); + EXPECT_TRUE(test.publisher.capture()->snapshot_changes.empty()); + ASSERT_TRUE(test.prepare(test.publisher.capture()).ok()); + test.begin(); + ASSERT_TRUE(test.done.load(std::memory_order_acquire)); + ASSERT_TRUE(test.terminal.ok()) << test.terminal.to_string(); + const auto view = test.publisher.capture(); + EXPECT_GT(view->snapshot_changes.size(), 1); + uint64_t next = 0; + bool complete = false; + while (!complete) { + complete = view->snapshot_changes.visit_from(next, 4, nullptr, [](void *, const SnapshotPageChange &change) { + EXPECT_EQ(change.effect, SnapshotChangeEffect::SourceStructure); + }); + } +} + +TEST(SnapshotChangeEffect, CapturedStructureCannotEraseLaterContentWhenTheOlderSnapshotCommits) +{ + PublicationCoordinator publisher; + auto first = leaf(1); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .replacement = first}}, .root_page_id = 0}).status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + auto second = leaf(2); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .expected = first, .replacement = second}}, + .snapshot_effect = SnapshotChangeEffect::SourceStructure}) + .status.ok()); + const auto captured = checkpoint(publisher, CheckpointAction::Capture, 2); + ASSERT_TRUE(captured.status.ok()); + auto third = leaf(3); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .expected = second, .replacement = third}}}).status.ok()); + auto fourth = leaf(4); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .expected = third, .replacement = fourth}}, + .snapshot_effect = SnapshotChangeEffect::SourceStructure}) + .status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 2).status.ok()); + const auto view = publisher.capture(); + ASSERT_TRUE(view->snapshot_changes.get(0)); + EXPECT_EQ(view->snapshot_changes.get(0)->effect, + SnapshotChangeEffect::FullContent | SnapshotChangeEffect::SourceStructure); + EXPECT_EQ(captured.checkpoint_view->snapshot_changes.get(0)->effect, SnapshotChangeEffect::SourceStructure); +} + +TEST(SnapshotChangeEffect, UnfencedAcceptedDeltaCannotChangeThePublishedState) +{ + PublicationCoordinator publisher; + auto first = leaf(1); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .replacement = first}}, .root_page_id = 0}).status.ok()); + const auto before = publisher.capture(); + const auto reject = [&](SnapshotChangeEffect effect) { + return publish(publisher, {.pages = {{.page_id = 0, .expected = first, .replacement = leaf(2)}}, + .snapshot_effect = effect}) + .status.code(); + }; + EXPECT_EQ(reject(SnapshotChangeEffect::AcceptedPrefix), Code::kInvalidArgument); + EXPECT_EQ(publisher.capture(), before); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/snapshot_export_test.cpp b/lib/crowdb-tree/tests/integration/snapshot_export_test.cpp index c63461bee..bf7f7b9eb 100644 --- a/lib/crowdb-tree/tests/integration/snapshot_export_test.cpp +++ b/lib/crowdb-tree/tests/integration/snapshot_export_test.cpp @@ -4,6 +4,7 @@ // PT7: snapshot export / import (portable stream + file wrappers). #include "crowdb-common/crc32c.h" #include "crowdb-tree/crowdb-tree.h" +#include "crowdb-tree/maptable/frame_page.h" #include "crowdb-tree/snapshot/snapshot_io.h" #include "test_tmp.h" @@ -446,11 +447,18 @@ TEST(SnapshotExport, NativeCrcTamperRejected) ASSERT_TRUE(exp->next_chunk(&c, &done).ok()); stream += c; } - ASSERT_GT(stream.size(), 64U); - stream[40] = static_cast<char>(stream[40] ^ 0xff); // flip a byte in the frame body + // Native header (33), frame record header (12), then the frame body. + constexpr size_t body_offset = 33 + 12 + kFrameHeaderSize; + ASSERT_GT(stream.size(), body_offset); + stream[body_offset] = static_cast<char>(stream[body_offset] ^ 0xff); - Crowdbtree b(opt); + Crowdbtree b(opt); + ASSERT_TRUE(b.apply(1, Batch{{{.key = "untouched", .kind = OpKind::kPut, .value = "v"}}}).ok()); + ASSERT_TRUE(b.flush().ok()); SnapshotImport imp(b); - ASSERT_TRUE(imp.feed(Slice(stream)).ok()); + const auto status = imp.feed(Slice(stream)); + EXPECT_EQ(status.code(), Code::kCorruption) << status.to_string(); EXPECT_EQ(imp.finish(nullptr).code(), Code::kCorruption); + EXPECT_EQ(b.get_view("untouched").value().to_string(), "v"); + EXPECT_EQ(b.last_applied_slot(), 1U); } diff --git a/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp b/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp index 3a6ff5a41..6bd3adb27 100644 --- a/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp +++ b/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. #include "backend/chunk/chunk_page_staging.h" +#include "buffer_pool_test_support.h" #include "crowdb-tree/backend/async_page_store.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" @@ -9,6 +10,7 @@ #include <gtest/gtest.h> #include <array> +#include <atomic> #include <cstring> #include <fstream> #include <iostream> @@ -45,6 +47,11 @@ class TestSnapshotBufferStore final : public PageStore, public AsyncPageStore { } + [[nodiscard]] bool write_completion_is_reloadable() const noexcept override + { + return true; + } + Status write_at(uint64_t off, const uint8_t *buf, size_t len) override { if (baseline > 0 && !prepared_bytes.has_value()) { @@ -101,6 +108,7 @@ class TestSnapshotBufferStore final : public PageStore, public AsyncPageStore uint64_t submit_write(PageAddr addr, const void *buf, size_t len, AsyncCompletion done) override { pending_ = Pending{.addr = addr, .data = buf, .length = len, .done = done, .typed = false}; + wake(); return 0; } @@ -108,6 +116,7 @@ class TestSnapshotBufferStore final : public PageStore, public AsyncPageStore AsyncCompletion done) override { pending_ = Pending{.addr = addr, .data = buf, .length = len, .done = done, .typed = true}; + wake(); return 0; } @@ -121,13 +130,21 @@ class TestSnapshotBufferStore final : public PageStore, public AsyncPageStore { } + void wake() + { + ready_.store(true, std::memory_order_release); + ready_.notify_one(); + } + bool complete_next() { + ready_.wait(false, std::memory_order_acquire); if (!pending_.has_value()) { return false; } const auto write = pending_.value(); pending_.reset(); + ready_.store(false, std::memory_order_release); const auto *data = static_cast<const uint8_t *>(write.data); auto status = write.typed ? write_typed_at(PagePurpose::kPageIndex, write.addr, data, write.length) : write_at(write.addr, data, write.length); @@ -152,6 +169,7 @@ class TestSnapshotBufferStore final : public PageStore, public AsyncPageStore std::vector<uint8_t> bytes_; uint64_t size_ = 0; std::optional<Pending> pending_; + std::atomic<bool> ready_{false}; }; class SnapshotMemoryTest : public ::testing::TestWithParam<bool> @@ -177,14 +195,17 @@ TEST_P(SnapshotMemoryTest, ReleasesCompletedPageBuffersBeforeWritingMetadata) store.baseline = resident_bytes(); ASSERT_GT(store.baseline, 0U); if (GetParam()) { - bool complete = false; + std::atomic<bool> complete{false}; tree.snapshot_async([&](Status status, uint64_t applied) { EXPECT_TRUE(status.ok()) << status.to_string(); EXPECT_EQ(applied, 256U); - complete = true; + complete.store(true, std::memory_order_release); + store.wake(); }); - while (!complete) { - ASSERT_TRUE(store.complete_next()); + while (!complete.load(std::memory_order_acquire)) { + if (!store.complete_next()) { + ASSERT_TRUE(complete.load(std::memory_order_acquire)); + } } } else { @@ -251,11 +272,11 @@ TEST(SnapshotMemory, PaddedOverlayKeepsZeroCoverageWithoutRetainingPadding) TEST(SnapshotMemory, UnusedFrameCapacityDoesNotPrefaultTheArena) { - MemPageStore store(65536); - const auto before = resident_bytes(); - BufferPool pool(256U * 1024 * 1024, 65536, &store); - const auto current = resident_bytes(); - const auto growth = current > before ? current - before : 0; + MemPageStore store(65536); + const auto before = resident_bytes(); + TestBufferPool pool(256U * 1024 * 1024, 65536, &store); + const auto current = resident_bytes(); + const auto growth = current > before ? current - before : 0; std::cout << "Unused frame-cache RSS growth: " << growth << " bytes\n"; EXPECT_LT(growth, 8U * 1024 * 1024); EXPECT_EQ(pool.stats().num_frames, 4096U); diff --git a/lib/crowdb-tree/tests/integration/source_generation_rotation_test.cpp b/lib/crowdb-tree/tests/integration/source_generation_rotation_test.cpp new file mode 100644 index 000000000..5556b9e41 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/source_generation_rotation_test.cpp @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "btree/tree_generation_lease.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeSourceControls, SoftHandoffKeepsProgressAfterClearAndPreparedReplacement) +{ + Config config; + TreeRuntimeOptions runtime; + runtime.workers = 1; + ASSERT_TRUE(TreeRuntime::create(runtime, &config.runtime).ok()); + config.memtable_flush_entries = 1; + config.memtable_flush_bytes = SIZE_MAX; + config.max_memtable_count = 2; + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + ASSERT_TRUE(tree.put("old", "value").ok()); + for (size_t round = 0; round < 2; ++round) { + if (round == 0) { + ASSERT_TRUE(tree.clear().ok()); + } + else { + std::shared_ptr<TreeGenerationLease> lease; + ASSERT_TRUE(TreeGenerationLease::prepare(*core, {}, nullptr, &lease).ok()); + ASSERT_TRUE(lease->acquire().ok()); + ASSERT_TRUE(lease->commit().ok()); + // Drop the old controller before any automatic maintenance starts. + lease.reset(); + } + EXPECT_FALSE(tree.get_view("old").found()); + EXPECT_EQ(core->capture()->coverage, 0U); + for (size_t i = 0; i < 24; ++i) { + const auto status = tree.put("key" + std::to_string(i), "value"); + ASSERT_TRUE(status.ok()) << "round=" << round << " key=" << i << " " << status.to_string(); + EXPECT_LT(tree.frozen_table_count(), config.max_memtable_count); + } + // No explicit flush drives the assertion: the replacement's soft + // controller must reach its finite handoff by itself. + EXPECT_GT(core->capture()->coverage, 0U); + for (size_t i = 0; i < 24; ++i) { + auto value = tree.get_view("key" + std::to_string(i)); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.value().to_string(), "value"); + } + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/source_host_wait_test.cpp b/lib/crowdb-tree/tests/integration/source_host_wait_test.cpp new file mode 100644 index 000000000..18bfbc893 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/source_host_wait_test.cpp @@ -0,0 +1,420 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_controller_test.h" +#include "btree/tree_core_access.h" +#include "btree/tree_core_split_internal.h" +#include "btree/tree_core_split_wait.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +#include <barrier> +#include <chrono> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ + +TEST(SourceHostWait, SplitCaptureWaitsForThePreparingController) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.memtable_flush_bytes = SIZE_MAX; + config.memtable_flush_entries = UINT32_MAX; + Crowdbtree tree(config); + ASSERT_TRUE(tree.put("key", "value").ok()); + auto *core = TreeCoreAccess::get(tree); + core->binding()->drain(); + + struct Gate + { + std::atomic<bool> held{false}; + std::atomic<bool> waiting{false}; + std::atomic<bool> release{false}; + } gate; + + auto hold = [](void *context, bool acquired) noexcept { + auto &state = *static_cast<Gate *>(context); + if (!acquired) { + return; + } + state.held.store(true, std::memory_order_release); + state.held.notify_one(); + while (!state.release.load(std::memory_order_acquire)) { + state.release.wait(false); + } + }; + auto observe = [](void *context, bool acquired) noexcept { + if (!acquired) { + auto &state = *static_cast<Gate *>(context); + state.waiting.store(true, std::memory_order_release); + state.waiting.notify_one(); + } + }; + Status prepared; + std::shared_ptr<FlushOperation> operation; + std::thread holder([&] { + set_flush_prepare_hook_for_tests(hold, &gate); + prepared = core->prepare_flush(&operation); + set_flush_prepare_hook_for_tests(nullptr, nullptr); + }); + while (!gate.held.load(std::memory_order_acquire)) { + gate.held.wait(false); + } + Status captured; + uint64_t session = 0; + uint64_t frontier = 0; + std::thread capture([&] { + set_flush_prepare_hook_for_tests(observe, &gate); + captured = tree.begin_split_memtable_view(&session, &frontier); + set_flush_prepare_hook_for_tests(nullptr, nullptr); + }); + while (!gate.waiting.load(std::memory_order_acquire)) { + gate.waiting.wait(false); + } + gate.release.store(true, std::memory_order_release); + gate.release.notify_one(); + holder.join(); + capture.join(); + ASSERT_TRUE(prepared.ok()) << prepared.to_string(); + ASSERT_TRUE(captured.ok()) << captured.to_string(); + EXPECT_EQ(frontier, 1U); + ASSERT_TRUE(tree.release_split_memtable_view(session).ok()); + ASSERT_TRUE(operation->wait().ok()); + EXPECT_EQ(tree.get_view("key").value().to_string(), "value"); +} + +TEST(SourceHostWait, SplitCaptureReturnsQuotaFailureWithoutWaiting) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.runtime_binding.limits.reclaim_bytes = 4096; + config.memtable_flush_bytes = SIZE_MAX; + config.memtable_flush_entries = UINT32_MAX; + Crowdbtree tree(config); + ASSERT_TRUE(tree.put("key", "value").ok()); + uint64_t session = 71; + uint64_t frontier = 72; + const auto status = tree.begin_split_memtable_view(&session, &frontier); + EXPECT_EQ(status.code(), Code::kResourceExhausted) << status.to_string(); + EXPECT_EQ(session, 71U); + EXPECT_EQ(frontier, 72U); + EXPECT_EQ(tree.get_view("key").value().to_string(), "value"); +} + +TEST(SourceHostWait, SplitCaptureRejectsWorkerWaitBeforeCreatingASession) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + Crowdbtree tree(config); + ASSERT_TRUE(tree.put("key", "value").ok()); + auto *core = TreeCoreAccess::get(tree); + + struct Request + { + Crowdbtree *tree; + uint64_t session = 71; + uint64_t frontier = 72; + Status status; + std::atomic<bool> done{false}; + } request{.tree = &tree, .status = Status::Ok()}; + + auto capture = [](void *context) noexcept { + auto &state = *static_cast<Request *>(context); + state.status = state.tree->begin_split_memtable_view(&state.session, &state.frontier); + state.done.store(true, std::memory_order_release); + state.done.notify_one(); + }; + ASSERT_TRUE(core->binding()->cpu()->try_submit(core->binding()->lane(), capture, &request).ok()); + while (!request.done.load(std::memory_order_acquire)) { + request.done.wait(false); + } + EXPECT_EQ(request.status.code(), Code::kUnavailable); + EXPECT_EQ(request.session, 71U); + EXPECT_EQ(request.frontier, 72U); + ASSERT_TRUE(tree.begin_split_memtable_view(&request.session, &request.frontier).ok()); + EXPECT_EQ(request.frontier, 1U); + EXPECT_TRUE(tree.release_split_memtable_view(request.session).ok()); +} + +TEST(SourceHostWait, PreparedFlushPreservesInputsAndResumesAnOrdinaryWriter) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.memtable_flush_bytes = SIZE_MAX; + config.memtable_flush_entries = UINT32_MAX; + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + ASSERT_TRUE(tree.put("first", "before").ok()); + core->binding()->drain(); + std::shared_ptr<FlushOperation> prepared; + ASSERT_TRUE(core->prepare_flush(&prepared).ok()); + const auto baseline = core->operation_lifetime().outstanding(); + std::atomic<bool> done{false}; + Status written; + std::thread writer([&] { + written = tree.put("second", "unmoved"); + done.store(true, std::memory_order_release); + }); + const auto until = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (core->operation_lifetime().outstanding() <= baseline && !done.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < until) { + std::this_thread::yield(); + } + EXPECT_GT(core->operation_lifetime().outstanding(), baseline); + EXPECT_FALSE(done.load(std::memory_order_acquire)); + const auto flushed = prepared->wait(); + writer.join(); + ASSERT_TRUE(flushed.ok()) << flushed.to_string(); + ASSERT_TRUE(written.ok()) << written.to_string(); + EXPECT_EQ(tree.contiguous_slot(), 2U); + EXPECT_EQ(tree.get_view("second").value().to_string(), "unmoved"); +} + +TEST(SourceHostWait, SplitCaptureJoinsAnAlreadyRunningFlush) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + Crowdbtree tree(config); + ASSERT_TRUE(tree.put("key", "value").ok()); + auto *core = TreeCoreAccess::get(tree); + std::shared_ptr<FlushOperation> operation; + ASSERT_TRUE(core->prepare_flush(&operation).ok()); + + struct Gate + { + std::atomic<bool> entered{false}; + std::atomic<bool> release{false}; + FlushOperation *operation = nullptr; + Status worker_wait; + } gate; + + gate.operation = operation.get(); + + auto block = [](void *raw) noexcept { + auto &state = *static_cast<Gate *>(raw); + FlushResult ignored; + state.worker_wait = state.operation->wait_capture(&ignored); + state.entered.store(true, std::memory_order_release); + state.entered.notify_one(); + while (!state.release.load(std::memory_order_acquire)) { + state.release.wait(false); + } + }; + ASSERT_TRUE(core->binding()->cpu()->try_submit(core->binding()->lane(), block, &gate).ok()); + while (!gate.entered.load(std::memory_order_acquire)) { + gate.entered.wait(false); + } + std::atomic<bool> flushed{false}; + Status flush_status; + tree.flush_async([&](Status status) { + flush_status = std::move(status); + flushed.store(true, std::memory_order_release); + flushed.notify_one(); + }); + FlushResult duplicate_report; + const auto duplicate = detail::wait_split_report(operation->capture_sender(), &duplicate_report); + EXPECT_EQ(duplicate.code(), Code::kUnavailable); + EXPECT_EQ(gate.worker_wait.code(), Code::kUnavailable); + std::atomic<bool> capture_entered{false}; + Status captured; + uint64_t generation = 0; + uint64_t frontier = 0; + std::thread capture([&] { + capture_entered.store(true, std::memory_order_release); + capture_entered.notify_one(); + captured = tree.begin_split_memtable_view(&generation, &frontier); + }); + while (!capture_entered.load(std::memory_order_acquire)) { + capture_entered.wait(false); + } + gate.release.store(true, std::memory_order_release); + gate.release.notify_one(); + capture.join(); + while (!flushed.load(std::memory_order_acquire)) { + flushed.wait(false); + } + ASSERT_TRUE(flush_status.ok()) << flush_status.to_string(); + ASSERT_TRUE(captured.ok()) << captured.to_string(); + EXPECT_EQ(frontier, 1U); + ASSERT_TRUE(tree.release_split_memtable_view(generation).ok()); + EXPECT_EQ(tree.get_view("key").value().to_string(), "value"); + FlushResult initial; + ASSERT_TRUE(operation->wait_capture(&initial).ok()); + const auto fixed = operation->frozen_sources(); + ASSERT_TRUE(fixed.base); + EXPECT_FALSE(fixed.catalog); + EXPECT_EQ(fixed.base->coverage, 1U); + ASSERT_TRUE(tree.put("later", "second").ok()); + ASSERT_TRUE(tree.flush().ok()); + FlushResult again; + ASSERT_TRUE(operation->wait_capture(&again).ok()); + EXPECT_EQ(again.frontier, 1U); + EXPECT_EQ(operation->frozen_sources().base, fixed.base); + EXPECT_EQ(core->capture()->coverage, 2U); +} + +TEST(SourceHostWait, NewSplitChoosesItsBaselineAfterInterveningNormalHandoffs) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.memtable_flush_bytes = SIZE_MAX; + config.memtable_flush_entries = UINT32_MAX; + Crowdbtree source(config); + auto *core = TreeCoreAccess::get(source); + ASSERT_TRUE(source.put("first", "one").ok()); + std::shared_ptr<FlushOperation> old; + ASSERT_TRUE(core->prepare_flush(&old).ok()); + FlushResult first; + ASSERT_TRUE(old->wait_capture(&first).ok()); + EXPECT_EQ(first.frontier, 1U); + ASSERT_TRUE(source.put("second", "two").ok()); + Status interleaved; + size_t handoffs = 0; + set_between_split_captures_for_tests([&] { + for (const auto &key : {"third", "fourth"}) { + interleaved = source.put(key, key); + if (!interleaved.ok()) { + return; + } + interleaved = source.flush(); + if (!interleaved.ok()) { + return; + } + ++handoffs; + } + }); + uint64_t session = 0; + uint64_t frontier = 0; + const auto captured = source.begin_split_memtable_view(&session, &frontier); + set_between_split_captures_for_tests({}); + ASSERT_TRUE(interleaved.ok()) << interleaved.to_string(); + ASSERT_TRUE(captured.ok()) << captured.to_string(); + EXPECT_EQ(handoffs, 2U); + EXPECT_EQ(frontier, 4U); + Crowdbtree child(config); + const auto published = source.publish_split_memtable_view(session, frontier, child, config.key_range); + ASSERT_TRUE(published.ok()) << published.to_string(); + uint64_t slot = 1; + for (const auto &[key, expected] : { + std::pair{"first", "one" }, + std::pair{"second", "two" }, + std::pair{"third", "third" }, + std::pair{"fourth", "fourth"} + }) { + auto value = child.get_view(key); + EXPECT_EQ(value.slot(), slot++); + EXPECT_EQ(value.value().to_string(), expected); + } + EXPECT_EQ(child.contiguous_slot(), 4U); + ASSERT_TRUE(source.release_split_memtable_view(session).ok()); +} + +TEST(SourceHostWait, ExistingSplitPrefixSurvivesLaterNormalHandoffs) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.memtable_flush_bytes = SIZE_MAX; + config.memtable_flush_entries = UINT32_MAX; + Crowdbtree source(config); + Crowdbtree child(config); + ASSERT_TRUE(source.put("key", "inherited").ok()); + ASSERT_TRUE(child.install_split_memtable_overlay(source, 1).ok()); + ASSERT_TRUE(child.put("key", "child").ok()); + ASSERT_TRUE(source.put("key", "parent-newer").ok()); + ASSERT_TRUE(source.flush().ok()); + ASSERT_TRUE(source.put("later", "outside-prefix").ok()); + ASSERT_TRUE(source.flush().ok()); + Crowdbtree explicit_target(config); + auto status = explicit_target.install_split_memtable_overlay(source, 3); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(explicit_target.contiguous_slot(), 0U); + EXPECT_FALSE(explicit_target.get_view("key").found()); + uint64_t session = 0; + uint64_t frontier = 0; + status = source.begin_split_memtable_view(&session, &frontier); + ASSERT_TRUE(status.ok()) << status.to_string(); + Crowdbtree fresh(config); + status = source.publish_split_memtable_view(session, frontier, fresh, config.key_range); + ASSERT_TRUE(status.ok()) << "frontier=" << frontier << " " << status.to_string(); + EXPECT_EQ(frontier, 1U); + EXPECT_EQ(fresh.get_view("key").slot(), 1U); + EXPECT_EQ(fresh.get_view("key").value().to_string(), "inherited"); + EXPECT_FALSE(fresh.get_view("later").found()); + ASSERT_TRUE(fresh.put("key", "parent-newer").ok()); + ASSERT_TRUE(fresh.put("later", "outside-prefix").ok()); + EXPECT_EQ(fresh.get_view("key").slot(), 2U); + EXPECT_EQ(fresh.get_view("key").value().to_string(), "parent-newer"); + EXPECT_EQ(fresh.get_view("later").slot(), 3U); + EXPECT_EQ(fresh.get_view("later").value().to_string(), "outside-prefix"); + // The originally installed child still publishes its fixed cutover while + // retaining its independently accepted later write. + status = source.publish_split_memtable_view(session, 1, child, config.key_range); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(child.clear_split_memtable_overlay(source).ok()); + EXPECT_EQ(child.get_view("key").slot(), 2U); + EXPECT_EQ(child.get_view("key").value().to_string(), "child"); + EXPECT_FALSE(child.get_view("later").found()); + ASSERT_TRUE(source.release_split_memtable_view(session).ok()); +} + +TEST(SourceHostWait, ConcurrentFlushCallersJoinFiniteWorkWithoutFailingAdmission) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.memtable_flush_bytes = SIZE_MAX; + config.memtable_flush_entries = UINT32_MAX; + Crowdbtree tree(config); + for (size_t i = 0; i < 100; ++i) { + ASSERT_TRUE(tree.put(std::to_string(i), "value").ok()); + } + std::barrier start(4); + std::array<Status, 4> results; + std::array<std::thread, 4> workers; + for (size_t i = 0; i < workers.size(); ++i) { + workers[i] = std::thread([&, i] { + start.arrive_and_wait(); + for (size_t step = 0; step < 16; ++step) { + results[i] = tree.flush(); + if (!results[i].ok()) { + return; + } + } + }); + } + for (auto &worker : workers) { + worker.join(); + } + for (const auto &result : results) { + EXPECT_TRUE(result.ok()) << result.to_string(); + } + EXPECT_EQ(TreeCoreAccess::get(tree)->capture()->coverage, 100U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/source_rotation_report_test.cpp b/lib/crowdb-tree/tests/integration/source_rotation_report_test.cpp new file mode 100644 index 000000000..3493b8ca8 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/source_rotation_report_test.cpp @@ -0,0 +1,261 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_rotation.h" +#include "flush_preparation_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestSoftFlush : TestPreparedFlushTree +{ + std::shared_ptr<detail::AllocationReservation> bootstrap; + std::shared_ptr<ApplyProgress> progress; + std::shared_ptr<ApplyDispatcher> dispatcher; + std::shared_ptr<SourceRotation> rotation; + + explicit TestSoftFlush(bool cold = false) : TestPreparedFlushTree(1, cold) + { + auto paid = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, 1024 * 1024); + if (!paid) { + throw std::bad_alloc(); + } + bootstrap = detail::AllocationReservation::create(std::move(*paid)); + EXPECT_TRUE(ApplyProgress::create(8, bootstrap, &progress).ok()); + EXPECT_TRUE(ApplyDispatcher::create(runtime, lane, 4, bootstrap, &dispatcher).ok()); + preparation.cpu_progress = dispatcher; + batch.cpu_progress = dispatcher; + } + + ~TestSoftFlush() + { + if (rotation) { + rotation->stop(); + runtime->drain(); + while (!store->pending.empty()) { + store->finish(Status::io_error()); + runtime->drain(); + } + rotation->drain(); + runtime->drain(); + rotation.reset(); + } + controller->release_pending(); + dispatcher->close_and_drain(); + } + + void configure(FlushBatch::Maintenance maintenance = {}) + { + controller = std::make_unique<FlushController>(publisher, *runtime, sources, frontier, lane, batch, + std::move(maintenance), bootstrap, progress); + rotation = std::make_shared<SourceRotation>( + sources, publisher, dispatcher, progress, account, + SourceRotationOptions{.bytes = 1, .entries = 1, .tables = 2, .source_window = 2}); + rotation->set_async_flush(*controller, [this](FlushPreparationOptions *out) { + *out = preparation; + return Status::Ok(); + }); + } + + void write(const char *key, uint64_t slot) + { + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(slot - 1, {1, 1, 1, 1}, &writer).ok()); + ASSERT_TRUE(writer.upsert(key, slot, encode_cell_buf(slot, OpKind::kPut, "v")).ok()); + writer.release(); + frontier.store(slot); + } +}; + +TEST(SourceRotationReport, StaleAdmissionReplansWithoutAnotherWrite) +{ + TestSoftFlush tree; + tree.configure(); + tree.write("a", 1); + std::shared_ptr<FlushOperation> stale; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &stale).ok()); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>( + InstallToken{.generation = tree.publisher.capture()->generation, .batch = 998, .range = 0, .sequence = 0}); + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(tree.publisher.try_submit(request)); + const auto sequence = tree.rotation->request(); + ASSERT_NE(sequence, 0U); + tree.runtime->drain(); + tree.rotation->wait(sequence); + EXPECT_EQ(stale->result().handoff_status.code(), Code::kUnavailable); + EXPECT_TRUE(stale->result().admission_stale); + EXPECT_EQ(tree.publisher.capture()->coverage, 1U); + EXPECT_EQ(tree.publisher.capture()->mapping.get(0)->leaf->lookup("a").cell.value.to_string(), "v"); +} + +TEST(SourceRotationReport, CancelledAdmissionDoesNotRestartItself) +{ + TestSoftFlush tree; + tree.configure(); + tree.write("a", 1); + std::shared_ptr<FlushOperation> cancelled; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &cancelled).ok()); + cancelled->request_stop(); + const auto sequence = tree.rotation->request(); + ASSERT_NE(sequence, 0U); + tree.runtime->drain(); + tree.rotation->wait(sequence); + EXPECT_EQ(cancelled->result().handoff_status.code(), Code::kUnavailable); + EXPECT_FALSE(cancelled->result().admission_stale); + EXPECT_EQ(tree.publisher.capture()->coverage, 0U); + EXPECT_FALSE(tree.sources.catalog()->active->closed()); +} + +TEST(SourceRotationReport, ReadFailureRetainsRepairWithoutRetryingItsOwnProgress) +{ + for (const auto code : {Code::kUnavailable, Code::kIoError}) { + TestSoftFlush tree(true); + tree.store->deferred = true; + tree.configure(); + tree.write("a", 1); + std::shared_ptr<FlushOperation> failed; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &failed).ok()); + const auto sequence = tree.rotation->request(); + ASSERT_NE(sequence, 0U); + tree.runtime->drain(); + ASSERT_EQ(tree.store->pending.size(), 1U); + tree.store->finish(code == Code::kUnavailable ? Status::unavailable() : Status::io_error()); + tree.runtime->drain(); + tree.rotation->wait(sequence); + EXPECT_EQ(failed->result().handoff_status.code(), code); + EXPECT_EQ(failed->result().phase, FlushPhase::kRepairRequired); + EXPECT_FALSE(failed->result().admission_stale); + EXPECT_EQ(tree.store->reads.load(), 1U); + EXPECT_TRUE(tree.store->pending.empty()); + EXPECT_EQ(tree.publisher.capture()->coverage, 0U); + const auto batch = failed->result().batch_id; + std::shared_ptr<FlushOperation> retry; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &retry).ok()); + ASSERT_EQ(retry, failed); + tree.store->deferred = false; + ASSERT_TRUE(retry->wait().ok()); + EXPECT_EQ(retry->result().batch_id, batch); + EXPECT_EQ(tree.publisher.capture()->coverage, 1U); + } +} + +TEST(SourceRotationReport, MaintenanceFailureKeepsCoverageWithoutRepeatingHandoff) +{ + TestSoftFlush tree; + size_t calls = 0; + tree.configure([&](auto, auto done) { + ++calls; + done(Status::unavailable(), 1); + }); + tree.write("a", 1); + std::shared_ptr<FlushOperation> failed; + ASSERT_TRUE(tree.controller->prepare(tree.bootstrap, &failed).ok()); + const auto sequence = tree.rotation->request(); + ASSERT_NE(sequence, 0U); + tree.runtime->drain(); + tree.rotation->wait(sequence); + EXPECT_EQ(calls, 1U); + EXPECT_TRUE(failed->result().handoff_status.ok()); + EXPECT_EQ(failed->result().maintenance_status.code(), Code::kUnavailable); + EXPECT_FALSE(failed->result().admission_stale); + EXPECT_EQ(tree.publisher.capture()->coverage, 1U); + EXPECT_EQ(tree.publisher.capture()->stats.overlay_entries, 1U); +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(SourceRotationReport, PartialHandoffRetryKeepsInstalledRunsAndBatchIdentity) +{ + TestSoftFlush tree; + std::shared_ptr<FlushOperation> failed; + bool stop = true; + tree.batch.parallelism = 1; + tree.batch.after_run_for_tests = [&](size_t range, size_t) { + if (stop && range == 1) { + failed->request_stop(); + } + }; + tree.configure(); + tree.write("a", 1); + tree.write("z", 2); + ASSERT_TRUE(tree.controller->prepare(tree.bootstrap, &failed).ok()); + const auto sequence = tree.rotation->request(); + ASSERT_NE(sequence, 0U); + tree.runtime->drain(); + tree.rotation->wait(sequence); + ASSERT_EQ(failed->result().phase, FlushPhase::kRepairRequired); + EXPECT_FALSE(failed->result().admission_stale); + EXPECT_EQ(tree.publisher.capture()->coverage, 0U); + ASSERT_EQ(tree.publisher.capture()->stats.overlay_entries, 1U); + const auto installed = tree.publisher.capture()->mapping.get(0); + const auto batch = failed->result().batch_id; + std::shared_ptr<FlushOperation> retry; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &retry).ok()); + ASSERT_EQ(retry, failed); + stop = false; + ASSERT_TRUE(retry->wait().ok()); + EXPECT_EQ(retry->result().batch_id, batch); + EXPECT_EQ(tree.publisher.capture()->mapping.get(0), installed); + EXPECT_EQ(tree.publisher.capture()->stats.overlay_entries, 2U); + EXPECT_EQ(tree.publisher.capture()->coverage, 2U); +} +#endif + +struct TestReceiver +{ + using receiver_concept = stdexec::receiver_tag; + std::atomic<bool> *done; + + void set_value(FlushResult report) const noexcept + { + EXPECT_TRUE(report.handoff_status.ok()); + done->store(true); + } + + void set_error(Status status) const noexcept + { + ADD_FAILURE() << status.to_string(); + done->store(true); + } + + void set_stopped() const noexcept + { + ADD_FAILURE() << "accepted handoff was cancelled"; + done->store(true); + } +}; + +TEST(SourceRotationReport, CloseCancelsBusyConnectionWaitAndDrainsAcceptedHandoff) +{ + TestSoftFlush tree; + tree.configure(); + SourceAdmission::Writer writer; + ASSERT_TRUE(tree.sources.try_admit(0, {1, 1, 1, 1}, &writer).ok()); + ASSERT_TRUE(writer.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + tree.frontier.store(1); + std::shared_ptr<FlushOperation> accepted; + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &accepted).ok()); + + std::atomic<bool> done{false}; + auto connected = stdexec::connect(accepted->report_sender(), TestReceiver{&done}); + tree.rotation->set_async_flush(*tree.controller, [&](FlushPreparationOptions *out) { + *out = tree.preparation; + stdexec::start(connected); + return Status::Ok(); + }); + const auto sequence = tree.rotation->request(); + EXPECT_NE(sequence, 0U); + tree.runtime->drain(); + EXPECT_FALSE(done.load()); + EXPECT_TRUE(writer.table()->closed()); + tree.rotation->stop(); + tree.runtime->drain(); + tree.rotation->wait(sequence); + EXPECT_FALSE(done.load()); + writer.release(); + tree.runtime->drain(); + EXPECT_TRUE(done.load()); + EXPECT_EQ(tree.publisher.capture()->coverage, 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/source_rotation_wait_test.cpp b/lib/crowdb-tree/tests/integration/source_rotation_wait_test.cpp new file mode 100644 index 000000000..a429a15a3 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/source_rotation_wait_test.cpp @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +#include <chrono> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +class SourceRotationWait : public ::testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.memtable_flush_entries = 1; + config.memtable_flush_bytes = SIZE_MAX; + config.max_memtable_count = 2; + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + ASSERT_TRUE(tree->put("first", "value").ok()); + const auto account = core->binding()->account(); + const auto snapshot = account->snapshot(); + constexpr auto index = static_cast<size_t>(detail::BudgetClass::kHandoff); + const auto available = detail::ResourceBudget::capacity(account, detail::BudgetClass::kHandoff) - + snapshot.allocated[index] - snapshot.reserved[index]; + blocker = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, available); + ASSERT_TRUE(blocker); + const auto initial_waits = wait_count(); + tree->get_async("first", [&](Status read, GetView value) { + EXPECT_TRUE(read.ok()) << read.to_string(); + EXPECT_TRUE(value.found()); + written = tree->put("second", "next"); + done.store(true, std::memory_order_release); + done.notify_all(); + }); + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + ASSERT_TRUE(written.ok()) << written.to_string(); + const auto until = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (wait_count() == initial_waits && std::chrono::steady_clock::now() < until) { + std::this_thread::yield(); + } + ASSERT_GT(wait_count(), initial_waits); + } + + [[nodiscard]] uint64_t wait_count() const + { + return core->publisher().work_metrics()->counters.snapshot()[static_cast<size_t>( + TreeWorkCause::Allocation)][static_cast<size_t>(TreeWorkCounter::AdmissionWaits)]; + } + + Config config; + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; + std::optional<detail::ResourceBudget::Credit> blocker; + std::atomic<bool> done{false}; + Status written; +}; + +TEST_F(SourceRotationWait, BudgetReleaseResumesFiniteHandoffWithoutAnotherWrite) +{ + blocker.reset(); + // Scheduler-tail drain requires operation tickets to have finished first. + // Observe automatic publication without submitting another write or flush. + const auto until = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (core->capture()->coverage != 2 && std::chrono::steady_clock::now() < until) { + std::this_thread::yield(); + } + ASSERT_EQ(core->capture()->coverage, 2U); + core->binding()->drain(); + EXPECT_EQ(tree->get_view("second").value().to_string(), "next"); + ASSERT_TRUE(tree->put("third", "continued").ok()); +} + +TEST_F(SourceRotationWait, CloseCancelsOnlyTheUnacceptedSoftBudgetWait) +{ + core->close_and_drain(); + EXPECT_TRUE(core->closed()); + EXPECT_EQ(tree->contiguous_slot(), 2U); + blocker.reset(); +} + +TEST_F(SourceRotationWait, CloseReleasesAHostWriterWaitingForTheSoftPause) +{ + core->binding()->drain(); + const auto baseline = core->operation_lifetime().outstanding(); + std::atomic<bool> written_done{false}; + Status status; + std::thread writer([&] { + status = tree->put("third", "accepted ingress"); + written_done.store(true, std::memory_order_release); + }); + const auto until = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (core->operation_lifetime().outstanding() <= baseline && !written_done.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < until) { + std::this_thread::yield(); + } + EXPECT_GT(core->operation_lifetime().outstanding(), baseline); + EXPECT_FALSE(written_done.load(std::memory_order_acquire)); + core->close_and_drain(); + writer.join(); + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->contiguous_slot(), 3U); + blocker.reset(); +} + +TEST(SourceRotationProgress, RetainedReclaimCannotStrandAnAlreadyAppliedHostWrite) +{ + Config config; + config.memtable_flush_entries = 1; + config.memtable_flush_bytes = SIZE_MAX; + config.max_memtable_count = 2; + Crowdbtree tree(config); + auto *const core = TreeCoreAccess::get(tree); + ASSERT_TRUE(tree.put("first", "value").ok()); + core->binding()->drain(); + const auto account = core->binding()->account(); + const auto available = detail::ResourceBudget::available(account, detail::BudgetClass::kReclaim); + auto reserved = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, available); + if (!reserved) { + FAIL() << "Reclaim credit unavailable"; + } + auto retained = reserved->allocate(available); + ASSERT_TRUE(retained); + std::atomic<bool> done{false}; + Status status; + std::thread writer([&] { + status = tree.put("second", "accepted"); + done.store(true, std::memory_order_release); + }); + const auto until = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!done.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < until) { + std::this_thread::yield(); + } + const bool returned_with_retained_owner = done.load(std::memory_order_acquire); + retained.reset(); + reserved.reset(); + writer.join(); + EXPECT_TRUE(returned_with_retained_owner); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_EQ(tree.get_view("second").value().to_string(), "accepted"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/sparse_block_relocation_test.cpp b/lib/crowdb-tree/tests/integration/sparse_block_relocation_test.cpp new file mode 100644 index 000000000..b7528ee34 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/sparse_block_relocation_test.cpp @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/local_sparse_gc.h" +#include "btree/overflow_backing.h" +#include "btree/tree_core_checkpoint_internal.h" +#include "extent_ledger_test_support.h" +#include "snapshot/operation_wait.h" +#include "snapshot/primary_backing.h" +#include "test_tmp.h" + +namespace crowdb::tree +{ +namespace +{ +TEST(SparseBlockRelocation, SelectionSkipsDenseEmptyAndMissingBlocksAndKeepsItsFence) +{ + tree_test::TempDir directory("sparse_candidate_"); + std::unique_ptr<BlockPageStore> device; + constexpr uint64_t block = 16384; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, block, 4096, &device).ok()); + ASSERT_TRUE(device->ensure_extents(0, 6 * block).ok()); + ASSERT_TRUE(device->delete_block(3).ok()); + test_read::LedgerFixture test({.backend_id = 1, .file_size = 6 * block, .address_limit = 6 * block}); + auto dense = test.recover(block, block); + auto sparse = test.recover(4 * block, 4096); + ASSERT_TRUE(dense.status.ok() && sparse.status.ok()); + ASSERT_TRUE(test.activate().status.ok()); + auto selected = select_sparse_local_block(*device, *test.ledger, 0, test.credits.credits.bytes); + ASSERT_TRUE(selected.status.ok()) << selected.status.to_string(); + EXPECT_EQ(selected.exclusion.address, 4 * block); + EXPECT_EQ(selected.exclusion.length, block); + EXPECT_EQ(selected.next_block, 5U); + auto first = test.reserve(3 * block); + EXPECT_EQ(first.status.code(), Code::kResourceExhausted); + selected = {}; + auto retry = test.reserve(3 * block); + EXPECT_EQ(retry.status.code(), Code::kResourceExhausted); + sparse = {}; + auto available = test.reserve(3 * block); + ASSERT_TRUE(available.status.ok()); + EXPECT_EQ(available.extent.address, 2 * block); +} + +TEST(SparseBlockRelocation, CopiesOnlyTheSelectedOverflowFramesAndRetainsOldReadersThroughTwoAnchors) +{ + tree_test::TempDir directory("sparse_relocation_"); + std::unique_ptr<BlockPageStore> device; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &device).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.page_store = device.get(); + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.compression = compress_algo::kNone; + config.runtime_binding.limits.io_operations = 1; + std::shared_ptr<TreeCore> core; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + auto local = core->local_backend(); + local->gc_active.store(true); // This test drives the two rotations explicitly. + const std::string payload(100000, 'v'); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = payload}}}).ok()); + ASSERT_TRUE(core->checkpoint().ok()); + auto old = core->capture(); + auto leaf = old->mapping.get(old->root_page_id); + ASSERT_TRUE(leaf->primary && leaf->primary->frame.backing->overflow); + const BackingExclusion interval{.address = 64 * 1024, .length = 64 * 1024}; + size_t old_inside = 0; + size_t old_outside = 0; + for (const auto &[pid, page] : leaf->primary->frame.backing->overflow->pages) { + (void)pid; + interval.overlaps(*page.backing) ? ++old_inside : ++old_outside; + } + ASSERT_GT(old_inside, 0U); + ASSERT_GT(old_outside, 0U); + auto fence = test_read::await_ledger([&](auto done) { + return local->ledger->exclude_allocations(interval.address, interval.length, {}, std::move(done)); + }); + ASSERT_TRUE(fence.status.ok()) << fence.status.to_string(); + auto options = checkpoint_detail::defaults(*core); + options.write.frames.exclusion = interval; + std::shared_ptr<CheckpointOperation> moved; + ASSERT_TRUE(core->prepare_checkpoint(options, &moved).ok()); + auto status = detail::wait_snapshot_operation(moved); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_GE(moved->result().written_frames, old_inside); + EXPECT_GT(moved->result().reused_frames, 0U); + auto current = core->capture()->mapping.get(old->root_page_id); + ASSERT_TRUE(current->primary && current->primary->frame.backing->overflow); + EXPECT_FALSE(interval.overlaps(*current->primary->frame.backing)); + for (const auto &[pid, page] : current->primary->frame.backing->overflow->pages) { + (void)pid; + EXPECT_FALSE(interval.overlaps(*page.backing)); + } + moved.reset(); + ASSERT_TRUE(core->prepare_checkpoint(options, &moved).ok()); + ASSERT_TRUE(detail::wait_snapshot_operation(moved).ok()); + EXPECT_EQ(moved->result().written_frames, 0U); + moved.reset(); + core->cache()->evict(); + core->binding()->cpu()->drain(); + auto claim = [&] { + return test_read::await_ledger([&](auto done) { + return local->ledger->claim_free(interval.address, interval.length, {}, std::move(done)); + }); + }; + EXPECT_EQ(claim().status.code(), Code::kNotFound); + old.reset(); + leaf.reset(); + core->publisher().reclaim(); + core->binding()->account()->drain_retirements(); + auto deletion = claim(); + ASSERT_TRUE(deletion.status.ok()) << deletion.status.to_string(); + ASSERT_TRUE(device->delete_block(1).ok()); + deletion = {}; + fence = {}; + current.reset(); + GetView actual; + ASSERT_TRUE(core->get("a", &actual).ok()); + EXPECT_EQ(actual.value().to_string(), payload); + actual = {}; + local.reset(); + core.reset(); + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + ASSERT_TRUE(core->get("a", &actual).ok()); + EXPECT_EQ(actual.value().to_string(), payload); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/split_merge_test.cpp b/lib/crowdb-tree/tests/integration/split_merge_test.cpp index 57fab2539..a1f34255f 100644 --- a/lib/crowdb-tree/tests/integration/split_merge_test.cpp +++ b/lib/crowdb-tree/tests/integration/split_merge_test.cpp @@ -4,6 +4,7 @@ // CT12: page split & merge integration tests. #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" +#include "runtime_tree_access.h" #include <gtest/gtest.h> @@ -79,15 +80,16 @@ TEST(SplitMerge, SimilarKeysKeepSplittingOneHotRange) opt.leaf_merge_bytes = 64; Crowdbtree t(opt); - // Keep every new key in one narrow lexical interval. Each flush has to - // route through the repeatedly splitting hot range instead of distributing - // the writes across the tree. + // Append every round to the same narrow hot tail. Four records exceed one + // target leaf, so every flush must split that range even if the prior leaf + // has spare capacity. Interleaving four separated key bands would not + // guarantee growth on every round. std::map<std::string, std::string> oracle; uint64_t slot = 0; size_t leaves_before = t.leaf_count(); for (uint64_t round = 0; round < 40; ++round) { for (uint64_t offset = 0; offset < 4; ++offset) { - const uint64_t sequence = (offset * 40) + round; + const uint64_t sequence = (round * 4) + offset; std::array<char, 32> key{}; snprintf(key.data(), key.size(), "hot/500/%020llu", static_cast<unsigned long long>(sequence)); const std::string value = @@ -169,7 +171,8 @@ TEST(SplitMerge, SnapshotSucceedsAfterHeavyMergeAndRootCollapse) opt.leaf_split_bytes = 200; opt.leaf_merge_bytes = 60; opt.inner_max_keys = 4; // force inner splits/merges too, not just leaves - Crowdbtree t(opt); + auto tree = std::make_unique<Crowdbtree>(opt); + auto &t = *tree; const int N = 200; uint64_t slot = 0; @@ -190,8 +193,10 @@ TEST(SplitMerge, SnapshotSucceedsAfterHeavyMergeAndRootCollapse) ASSERT_TRUE(t.snapshot().ok()); + tree.reset(); // The writer domain must close before reopening the same store. std::unique_ptr<Crowdbtree> t2; - ASSERT_TRUE(Crowdbtree::open(opt, &t2).ok()); + const auto status = Crowdbtree::open(opt, &t2); + ASSERT_TRUE(status.ok()) << status.to_string(); std::string v; uint64_t s; EXPECT_TRUE(t2->get(Slice(make_key(0)), &s, &v)); @@ -368,27 +373,21 @@ TEST(SplitMerge, ConsolidationSplitsIterativelyToThreshold) } // Count the inner pages reachable from the root (test helper). -static size_t inner_count_walk(Crowdbtree &t) +static size_t inner_count_walk(Crowdbtree &tree) { - std::function<size_t(uint64_t)> rec = [&](uint64_t page_id) -> size_t { - PageBase *head = t.mapping().get_resident(page_id); - if (head == nullptr) { - return 0; - } - PageBase *base = head; - while (base != nullptr && base->type == page_type::kBatchDelta) { - base = base->next; - } - if (base == nullptr || base->type == page_type::kLeafBase) { + auto view = RuntimeTreeAccess_for_tests::capture(tree); + std::function<size_t(uint64_t)> walk = [&](uint64_t id) -> size_t { + auto page = view->mapping.get(id); + if (!page || !page->inner) { return 0; } - size_t n = 1; - for (uint64_t c : static_cast<InnerBase *>(base)->children()) { - n += rec(c); + size_t result = 1; + for (auto child : page->inner->children) { + result += walk(child); } - return n; + return result; }; - return rec(t.root_page_id()); + return walk(view->root_page_id); } // O(1) atomic leaf/inner counters must match the tree walk after splits. @@ -563,6 +562,28 @@ TEST(SplitMerge, SplitOverlayServesBeforeSourceMemtableSeal) EXPECT_EQ(value, "right-after"); } +namespace +{ +struct SplitWorkers +{ + std::atomic<bool> &stop; + std::thread &writer; + std::thread &reader; + std::thread &flusher; + + ~SplitWorkers() + { + stop.store(true, std::memory_order_release); + for (auto *thread : {&writer, &reader, &flusher}) { + if (thread->joinable()) { + thread->join(); + } + } + } +}; + +} // namespace + TEST(SplitMerge, RepeatedSharedViewsKeepBothReplayFrontiersContinuous) { constexpr uint64_t kRounds = 32; @@ -573,12 +594,21 @@ TEST(SplitMerge, RepeatedSharedViewsKeepBothReplayFrontiersContinuous) source_opt.memtable_flush_bytes = 1ULL << 10; source_opt.memtable_flush_entries = 32; source_opt.max_memtable_count = 1'024; + // This fixture admits a complete maintenance window while prior versions + // and writer histories retire while two split children retain mappings. Keep that peak + // finite and explicit; quota-pressure behavior has separate regressions. + TreeRuntimeOptions runtime_options; + runtime_options.limits.reclaim_bytes = 384ULL * 1024 * 1024; + ASSERT_TRUE(TreeRuntime::create(runtime_options, &source_opt.runtime).ok()); + source_opt.runtime_binding.limits.reclaim_bytes = 256ULL * 1024 * 1024; + source_opt.runtime_binding.limits.writer_bytes = 256ULL * 1024 * 1024; Crowdbtree source(source_opt); source.init_metrics("split-mixed", ""); for (uint64_t slot = 1; slot <= kSeedKeys; ++slot) { ASSERT_TRUE(source.apply(slot, put_one("seed-" + std::to_string(slot), "tree-resident")).ok()); } ASSERT_TRUE(source.flush().ok()); + std::atomic<bool> stop{false}; std::atomic<bool> writer_failed{false}; std::atomic<bool> reader_failed{false}; std::atomic<bool> writer_done{false}; @@ -590,12 +620,14 @@ TEST(SplitMerge, RepeatedSharedViewsKeepBothReplayFrontiersContinuous) write_latencies_us.reserve(kWrites); read_latencies_us.reserve(kReads); std::thread writer([&] { - for (uint64_t write = 1; write <= kWrites; ++write) { + for (uint64_t write = 1; write <= kWrites && !stop.load(std::memory_order_acquire); ++write) { const uint64_t slot = kSeedKeys + write; const std::string prefix = write % 2 == 0 ? "left-" : "right-"; written.store(slot, std::memory_order_release); const auto started = std::chrono::steady_clock::now(); - if (!source.apply(slot, put_one(prefix + std::to_string(slot), "injected")).ok()) { + const auto status = source.apply(slot, put_one(prefix + std::to_string(slot), "injected")); + if (!status.ok()) { + ADD_FAILURE() << "writer slot=" << slot << " " << status.to_string(); writer_failed.store(true); writer_done.store(true, std::memory_order_release); return; @@ -606,12 +638,15 @@ TEST(SplitMerge, RepeatedSharedViewsKeepBothReplayFrontiersContinuous) writer_done.store(true, std::memory_order_release); }); std::thread reader([&] { - for (uint64_t read = 0; read < kReads || completed_splits.load(std::memory_order_acquire) < kRounds; ++read) { + for (uint64_t read = 0; !stop.load(std::memory_order_acquire) && + (read < kReads || completed_splits.load(std::memory_order_acquire) < kRounds); + ++read) { uint64_t revision = 0; std::string value; - const std::string key = "seed-" + std::to_string(read % kSeedKeys + 1); + const std::string key = "seed-" + std::to_string((read % kSeedKeys) + 1); const auto started = std::chrono::steady_clock::now(); if (!source.get(Slice(key), &revision, &value) || value != "tree-resident") { + ADD_FAILURE() << "reader key=" << key << " revision=" << revision << " value=" << value; reader_failed.store(true); return; } @@ -621,8 +656,10 @@ TEST(SplitMerge, RepeatedSharedViewsKeepBothReplayFrontiersContinuous) } }); std::thread flusher([&] { - while (!writer_done.load(std::memory_order_acquire)) { - if (!source.flush().ok()) { + while (!stop.load(std::memory_order_acquire) && !writer_done.load(std::memory_order_acquire)) { + const auto status = source.flush(); + if (!status.ok()) { + ADD_FAILURE() << "flusher written=" << written.load() << " " << status.to_string(); flush_failed.store(true); return; } @@ -630,6 +667,8 @@ TEST(SplitMerge, RepeatedSharedViewsKeepBothReplayFrontiersContinuous) } }); + SplitWorkers join_workers{.stop = stop, .writer = writer, .reader = reader, .flusher = flusher}; + while (written.load(std::memory_order_acquire) == 0) { std::this_thread::yield(); } @@ -637,25 +676,33 @@ TEST(SplitMerge, RepeatedSharedViewsKeepBothReplayFrontiersContinuous) for (uint64_t round = 0; round < kRounds; ++round) { uint64_t generation = 0; uint64_t journal_frontier = 0; - ASSERT_TRUE(source.begin_split_memtable_view(&generation, &journal_frontier).ok()); + auto status = source.begin_split_memtable_view(&generation, &journal_frontier); + ASSERT_TRUE(status.ok()) << "begin round=" << round << " " << status.to_string(); EXPECT_GE(journal_frontier, previous_frontier); EXPECT_LE(journal_frontier, written.load(std::memory_order_acquire)); Config left_opt; + left_opt.runtime = source_opt.runtime; left_opt.key_range = KeyRange::bounded(std::nullopt, std::string("m")); Config right_opt; + right_opt.runtime = source_opt.runtime; right_opt.key_range = KeyRange::bounded(std::string("m"), std::nullopt); Crowdbtree left(left_opt); Crowdbtree right(right_opt); - ASSERT_TRUE(source.publish_split_memtable_view(generation, journal_frontier, left, left_opt.key_range).ok()); - ASSERT_TRUE(source.publish_split_memtable_view(generation, journal_frontier, right, right_opt.key_range).ok()); + status = source.publish_split_memtable_view(generation, journal_frontier, left, left_opt.key_range); + ASSERT_TRUE(status.ok()) << "left publish round=" << round << " " << status.to_string(); + status = source.publish_split_memtable_view(generation, journal_frontier, right, right_opt.key_range); + ASSERT_TRUE(status.ok()) << "right publish round=" << round << " " << status.to_string(); const uint64_t replay_slot = journal_frontier + 1; - ASSERT_TRUE(left.apply(replay_slot, put_one("left-replay-" + std::to_string(round), "replayed")).ok()); - ASSERT_TRUE(right.apply(replay_slot, put_one("right-replay-" + std::to_string(round), "replayed")).ok()); + status = left.apply(replay_slot, put_one("left-replay-" + std::to_string(round), "replayed")); + ASSERT_TRUE(status.ok()) << "left replay round=" << round << " " << status.to_string(); + status = right.apply(replay_slot, put_one("right-replay-" + std::to_string(round), "replayed")); + ASSERT_TRUE(status.ok()) << "right replay round=" << round << " " << status.to_string(); EXPECT_EQ(left.contiguous_slot(), replay_slot); EXPECT_EQ(right.contiguous_slot(), replay_slot); - ASSERT_TRUE(source.release_split_memtable_view(generation).ok()); + status = source.release_split_memtable_view(generation); + ASSERT_TRUE(status.ok()) << "release round=" << round << " " << status.to_string(); previous_frontier = journal_frontier; completed_splits.store(round + 1, std::memory_order_release); } @@ -677,5 +724,9 @@ TEST(SplitMerge, RepeatedSharedViewsKeepBothReplayFrontiersContinuous) EXPECT_LT(read_latencies_us[read_p99_index], 10'000U); EXPECT_LT(read_latencies_us.back(), 100'000U); const std::string metrics = source.flush_metrics_str(1.0, "test"); - EXPECT_NE(metrics.find("split-mixed.l1.get.c"), std::string::npos); + EXPECT_FALSE(metrics.empty()); + EXPECT_NE(metrics.find("split-mixed.read.inflight.bytes.g"), std::string::npos); + EXPECT_EQ(source.tree_state_stats().resources.read_inflight_bytes, 0U); + const auto work = RuntimeTreeAccess_for_tests::core(source).publisher().work_metrics()->totals(); + EXPECT_GT(work[static_cast<size_t>(TreeStage::FlushBatch)].count, 0U); } diff --git a/lib/crowdb-tree/tests/integration/staged_writeback_test.cpp b/lib/crowdb-tree/tests/integration/staged_writeback_test.cpp new file mode 100644 index 000000000..fa40b26d5 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/staged_writeback_test.cpp @@ -0,0 +1,208 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "local_checkpoint_test_support.h" +#include "snapshot/checkpoint_operation.h" +#include "snapshot/operation_wait.h" +#include "test_tmp.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_write; + +struct TestStagedWriteback +{ + TestCheckpoint local; + std::shared_ptr<LocalBackendState> backend = std::make_shared<LocalBackendState>(); + std::shared_ptr<const PublishedTreeState> initial = [] { + auto result = captured_leaf(page({ + {.key = "a", .value = "old"} + })); + EXPECT_TRUE(apply_page_metrics(result->stats, nullptr, &result->mapping.get(0)->metrics)); + return result; + }(); + PublicationCoordinator publisher{ + 4, 1, {}, + std::pmr::get_default_resource(), {}, + {.metadata = {}, .initial = initial, .source_catalog = {}} + }; + std::shared_ptr<CheckpointOperation> operation; + + TestStagedWriteback() + { + backend->store = local.store; + backend->ledger = local.source.ledger; + backend->checkpoints = local.registry; + } + + Status prepare() + { + auto reserved = + local.source.credits.credits.bytes->take_reserved(CheckpointOperation::finish_allocation_bound() + 512); + if (!reserved) { + return Status::resource_exhausted(); + } + auto finish = detail::AllocationReservation::create(std::move(*reserved)); + CheckpointOptions options; + options.mode = CheckpointMode::Writeback; + options.write.frames.encode.frame_bytes = 4096; + options.write.write.max_frame_bytes = 4096; + auto loader = std::make_shared<test_read::TestLoader>(); + return CheckpointOperation::create({}, publisher, backend, local.store, local.source.runtime, local.source.lane, + options, local.source.credits.credits, finish, loader, {}, {}, &operation); + } +}; + +TEST(StagedWriteback, ReloadablePagesDoNotAdvanceTheDurableCheckpoint) +{ + TestStagedWriteback test; + ASSERT_TRUE(test.prepare().ok()); + const auto status = detail::wait_snapshot_operation(test.operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto &result = test.operation->result(); + EXPECT_TRUE(result.writeback_completed); + EXPECT_FALSE(result.durable_committed); + EXPECT_EQ(result.captured_coverage, 10U); + EXPECT_EQ(result.rebased_pages, 1U); + EXPECT_EQ(result.durable_coverage, 0U); + EXPECT_EQ(result.snapshot_sequence, 0U); + EXPECT_EQ(test.local.store->events, std::vector<std::string>{"metadata"}); + auto view = test.publisher.capture(); + EXPECT_FALSE(view->checkpoint_inflight); + ASSERT_TRUE(view->mapping.get(0)->leaf->bases.front().backing->reloadable()); + EXPECT_FALSE(view->mapping.get(0)->metrics.dirty); + EXPECT_EQ(view->stats.checkpoint_pending_pages, test.initial->stats.checkpoint_pending_pages); + const auto metrics = test.publisher.work_metrics()->totals(); + EXPECT_EQ(metrics[static_cast<size_t>(TreeStage::SnapshotCommit)].attempts, 0U); + EXPECT_EQ(metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)].inflight, 0U); + EXPECT_EQ(metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)] + .outcomes[static_cast<size_t>(TreeOutcome::Success)], + 1U); + test.operation.reset(); + ASSERT_TRUE(test.prepare().ok()); + ASSERT_TRUE(detail::wait_snapshot_operation(test.operation).ok()); + EXPECT_EQ(test.publisher.work_metrics()->totals()[static_cast<size_t>(TreeStage::PageWritebackPipeline)].attempts, + metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)].attempts); +} + +TEST(StagedWriteback, ANewerPublicationSurvivesDelayedWriteback) +{ + TestStagedWriteback test; + test.local.store->defer_at = 1; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + ASSERT_TRUE(test.local.store->pending); + auto before = test.publisher.capture()->mapping.get(0); + auto replacement = std::make_shared<PublishedPage>(*before); + auto leaf = std::make_shared<PageState>(*page({ + {.key = "a", .value = "new"} + })); + leaf->content_id = before->leaf->content_id + 1; + replacement->leaf = leaf; + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{.sequence = 123}); + request->transaction.pages.push_back({.page_id = 0, .expected = before, .replacement = replacement}); + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(test.publisher.try_submit(request)); + test.local.store->finish(); + test.local.source.runtime->drain(); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(test.operation->result().rebased_pages, 0U); + EXPECT_EQ(test.operation->result().rebase_deferred, 1U); + EXPECT_EQ(test.publisher.capture()->mapping.get(0), replacement); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_EQ(test.local.registry->durable_coverage(), 0U); +} + +TEST(StagedWriteback, CancelRetainsTheAcceptedWriteUntilItsCallback) +{ + TestStagedWriteback test; + test.local.store->defer_at = 1; + ASSERT_TRUE(test.prepare().ok()); + Completion done; + auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + test.local.source.runtime->drain(); + ASSERT_TRUE(test.local.store->pending); + test.operation->request_stop(); + test.local.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 0U); + EXPECT_TRUE(test.publisher.capture()->checkpoint_inflight); + test.local.store->finish(); + test.local.source.runtime->drain(); + EXPECT_EQ(done.stopped.load(), 1U); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_EQ(test.local.registry->durable_coverage(), 0U); + EXPECT_EQ(test.publisher.capture()->mapping.get(0), test.initial->mapping.get(0)); + const auto metrics = test.publisher.work_metrics()->totals(); + EXPECT_EQ(metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)].inflight, 0U); + EXPECT_EQ(metrics[static_cast<size_t>(TreeStage::PageWritebackPipeline)] + .outcomes[static_cast<size_t>(TreeOutcome::Cancelled)], + 1U); +} + +TEST(StagedWriteback, FailureAbortsTheCaptureAndKeepsItsInput) +{ + TestStagedWriteback test; + test.local.store->fail_at = 1; + ASSERT_TRUE(test.prepare().ok()); + const auto status = detail::wait_snapshot_operation(test.operation); + EXPECT_EQ(status.code(), Code::kIoError); + EXPECT_FALSE(test.operation->result().writeback_completed); + EXPECT_FALSE(test.publisher.capture()->checkpoint_inflight); + EXPECT_EQ(test.publisher.capture()->mapping.get(0), test.initial->mapping.get(0)); + EXPECT_EQ(test.local.registry->durable_coverage(), 0U); +} + +TEST(StagedWriteback, BlockReopenUsesTheAnchorUntilStagedPagesAreCommitted) +{ + tree_test::TempDir directory("staged_writeback_"); + std::unique_ptr<BlockPageStore> device; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &device).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({.workers = 1, .backend_workers = 1}, &config.runtime).ok()); + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.page_store = device.get(); + config.runtime_binding.limits.io_operations = 1; + std::shared_ptr<TreeCore> core; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "old"}}}).ok()); + const auto initial_checkpoint = core->checkpoint(); + ASSERT_TRUE(initial_checkpoint.ok()) << initial_checkpoint.to_string(); + const std::string value(16000, 'v'); + ASSERT_TRUE(core->apply(2, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = value}}}).ok()); + std::shared_ptr<CheckpointOperation> writeback; + ASSERT_TRUE(core->prepare_writeback(&writeback).ok()); + const auto staged = detail::wait_snapshot_operation(writeback); + ASSERT_TRUE(staged.ok()) << staged.to_string(); + EXPECT_EQ(writeback->result().captured_coverage, 2U); + EXPECT_EQ(writeback->result().durable_coverage, 1U); + EXPECT_GT(core->capture()->stats.checkpoint_pending_pages, 0U); + GetView actual; + ASSERT_TRUE(core->get("a", &actual).ok()); + EXPECT_EQ(actual.value().to_string(), value); + actual = {}; + writeback.reset(); + core.reset(); + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + ASSERT_TRUE(core->get("a", &actual).ok()); + EXPECT_EQ(actual.value().to_string(), "old"); + ASSERT_TRUE(core->apply(2, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = value}}}).ok()); + ASSERT_TRUE(core->prepare_writeback(&writeback).ok()); + ASSERT_TRUE(detail::wait_snapshot_operation(writeback).ok()); + ASSERT_TRUE(core->checkpoint().ok()); + actual = {}; + writeback.reset(); + core.reset(); + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + ASSERT_TRUE(core->get("a", &actual).ok()); + EXPECT_EQ(actual.value().to_string(), value); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/stress_test.cpp b/lib/crowdb-tree/tests/integration/stress_test.cpp index 635a23c03..f9c2e08a8 100644 --- a/lib/crowdb-tree/tests/integration/stress_test.cpp +++ b/lib/crowdb-tree/tests/integration/stress_test.cpp @@ -3,6 +3,7 @@ // CT14: concurrent readers while a single writer applies/flushes/splits/merges. // Run under TSan/ASan to catch races and use-after-free in epoch reclamation. +#include "btree/tree_core_access.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" @@ -11,6 +12,7 @@ #include <array> #include <atomic> #include <cstdio> +#include <exception> #include <map> #include <memory> #include <random> @@ -58,7 +60,7 @@ TEST(Stress, ConcurrentDemandLoadAfterRecovery) std::unique_ptr<Crowdbtree> t2; ASSERT_TRUE(Crowdbtree::open(opt, &t2).ok()); - EXPECT_EQ(t2->buffer_pool()->stats().used, 0U); // nothing loaded yet + EXPECT_EQ(t2->stats().buffer_pool_used, 0U); // nothing loaded yet std::atomic<bool> fail{false}; std::vector<std::thread> readers; @@ -81,7 +83,7 @@ TEST(Stress, ConcurrentDemandLoadAfterRecovery) th.join(); } EXPECT_FALSE(fail.load()); - EXPECT_GT(t2->buffer_pool()->stats().used, 0U); // demand-loaded into the pool + EXPECT_GT(t2->stats().buffer_pool_used, 0U); // demand-loaded into the pool } TEST(Stress, ConcurrentReadersSingleWriter) @@ -184,13 +186,21 @@ TEST(Stress, ConcurrentReadersSingleWriter) // (run under TSan/ASan). TEST(Stress, ConcurrentScanDuringChurnNoCorruption) { - Config opt; - opt.max_delta_len = 2; - opt.leaf_split_bytes = 160; - opt.leaf_merge_bytes = 50; - opt.inner_max_keys = 8; - opt.inner_merge_keys = 3; + Config opt; + TreeRuntimeOptions host; + // Four concurrent scans each retain a complete bounded read window. + host.limits.routing_bytes = 1536ULL * 1024 * 1024; + ASSERT_TRUE(TreeRuntime::create(host, &opt.runtime).ok()); + opt.runtime_binding.limits.routing_bytes = 1024ULL * 1024 * 1024; + // A complete flush window and roots awaiting concurrent readers coexist. + opt.runtime_binding.limits.reclaim_bytes = 128ULL * 1024 * 1024; + opt.max_delta_len = 2; + opt.leaf_split_bytes = 160; + opt.leaf_merge_bytes = 50; + opt.inner_max_keys = 8; + opt.inner_merge_keys = 3; Crowdbtree t(opt); + const auto account = TreeCoreAccess::get(t)->binding()->account(); const int K = 300; std::atomic<bool> stop{false}; @@ -233,55 +243,73 @@ TEST(Stress, ConcurrentScanDuringChurnNoCorruption) } } scans.fetch_add(1, std::memory_order_relaxed); + // Release the completed scan before admitting another read + // window; publication does not drain asynchronous retirement. + out.clear(); + account->drain_retirements(); } }); } - std::mt19937 rng(43); - uint64_t slot = 0; - for (int step = 0; step < 6000; ++step) { - ++slot; - std::string key = make_key(static_cast<int>(rng() % K)); - if ((rng() % 4) == 0) { - ASSERT_TRUE(t.apply(slot, Batch{{{.key = key, .kind = OpKind::kDelete, .value = ""}}}).ok()); - } - else { - std::string val = "v" + std::to_string(slot); - ASSERT_TRUE(t.apply(slot, Batch{{{.key = key, .kind = OpKind::kPut, .value = val}}}).ok()); - } - if ((rng() % 6) == 0) { - ASSERT_TRUE(t.flush().ok()); + int failed_step = -1; + const auto status = [&]() -> Status { + std::mt19937 rng(43); + uint64_t slot = 0; + for (int step = 0; step < 6000; ++step) { + failed_step = step; + ++slot; + std::string key = make_key(static_cast<int>(rng() % K)); + Status applied; + if ((rng() % 4) == 0) { + applied = t.apply(slot, Batch{{{.key = key, .kind = OpKind::kDelete, .value = ""}}}); + } + else { + std::string val = "v" + std::to_string(slot); + applied = t.apply(slot, Batch{{{.key = key, .kind = OpKind::kPut, .value = val}}}); + } + if (!applied.ok()) { + return applied; + } + if ((rng() % 6) == 0) { + auto flushed = t.flush(); + if (!flushed.ok()) { + return flushed; + } + } } - } - ASSERT_TRUE(t.flush().ok()); + return t.flush(); + }(); stop.store(true); for (auto &th : scanners) { th.join(); } + EXPECT_TRUE(status.ok()) << "step=" << failed_step << " " << status.to_string(); EXPECT_FALSE(bad.load()); EXPECT_GT(scans.load(), 0); } -// plan-tree #8: snapshot_view() no longer holds write_mutex_ (epoch guard -// only), walking L1 via the same right_sibling technique as scan(). This -// hammers snapshot_view() concurrently with heavy split/merge/flush churn and -// checks the invariants that must hold for any torn/mid-mutation snapshot: -// sorted order, no duplicate keys, well-formed values, and that at_slot is a -// valid *lower bound* (every key/value the snapshot's own get() resolves to -// is consistent with some slot <= the highest slot applied so far -- entries -// may run ahead of at_slot, per the at_slot-captured-before-the-walk -// argument in snapshot_view()'s header comment, but must never show a value -// that couldn't have existed at any real point in time). +// Four viewers retain complete L1 snapshots while one writer applies, flushes, +// splits and merges. Every snapshot must stay sorted and contain only values +// from successful writes, including when a handoff publishes several ranges. TEST(Stress, ConcurrentSnapshotViewDuringChurnNoCorruption) { - Config opt; - opt.max_delta_len = 2; - opt.leaf_split_bytes = 160; - opt.leaf_merge_bytes = 50; - opt.inner_max_keys = 8; - opt.inner_merge_keys = 3; + Config opt; + TreeRuntimeOptions host; + // Four simultaneous snapshot materializers each own a complete bounded + // read window. Keep the original concurrency under an explicit host quota. + host.limits.routing_bytes = 1536ULL * 1024 * 1024; + ASSERT_TRUE(TreeRuntime::create(host, &opt.runtime).ok()); + opt.runtime_binding.limits.routing_bytes = 1024ULL * 1024 * 1024; + // Keep the complete flush window funded while old views retire. + opt.runtime_binding.limits.reclaim_bytes = 128ULL * 1024 * 1024; + opt.max_delta_len = 2; + opt.leaf_split_bytes = 160; + opt.leaf_merge_bytes = 50; + opt.inner_max_keys = 8; + opt.inner_merge_keys = 3; Crowdbtree t(opt); + const auto account = TreeCoreAccess::get(t)->binding()->account(); const int K = 300; std::atomic<bool> stop{false}; @@ -289,64 +317,93 @@ TEST(Stress, ConcurrentSnapshotViewDuringChurnNoCorruption) std::atomic<bool> bad{false}; std::atomic<uint64_t> max_slot_seen{0}; - std::vector<std::thread> viewers; + std::array<std::exception_ptr, 4> failures; + std::vector<std::thread> viewers; viewers.reserve(4); - for (int r = 0; r < 4; ++r) { + for (auto &failure : failures) { viewers.emplace_back([&] { - while (!stop.load(std::memory_order_relaxed) && !bad.load(std::memory_order_relaxed)) { - auto snap = t.snapshot_view(); - if (snap == nullptr) { - bad.store(true); - return; - } - uint64_t ceiling = max_slot_seen.load(std::memory_order_acquire); - const auto &entries = snap->entries(); - for (size_t i = 0; i < entries.size(); ++i) { - if (i > 0 && !(Slice(entries[i - 1].key).compare(Slice(entries[i].key)) < 0)) { - bad.store(true); // not strictly increasing -> duplicate or out of order - return; - } - CellView v{Slice(entries[i].cell)}; - // Every applied value/tombstone carries a slot that was - // real at some point; it can never exceed the highest - // slot any apply() call had returned from so far. - if (v.slot() > ceiling + 1) { // +1: benign race with the counter's own update below + try { + while (!stop.load(std::memory_order_relaxed) && !bad.load(std::memory_order_relaxed)) { + auto snap = t.snapshot_view(); + if (snap == nullptr) { bad.store(true); return; } - if (!v.is_tombstone() && (v.value().empty() || v.value().data()[0] != 'v')) { - bad.store(true); // corrupted value - return; + uint64_t ceiling = max_slot_seen.load(std::memory_order_acquire); + const auto &entries = snap->entries(); + for (size_t i = 0; i < entries.size(); ++i) { + if (i > 0 && !(Slice(entries[i - 1].key).compare(Slice(entries[i].key)) < 0)) { + bad.store(true); // not strictly increasing -> duplicate or out of order + return; + } + CellView v{Slice(entries[i].cell)}; + // Every applied value/tombstone carries a slot that was + // real at some point; it can never exceed the highest + // slot any apply() call had returned from so far. + if (v.slot() > ceiling + 1) { // +1: benign race with the counter's own update below + bad.store(true); + return; + } + if (!v.is_tombstone() && (v.value().empty() || v.value().data()[0] != 'v')) { + bad.store(true); // corrupted value + return; + } } + views.fetch_add(1, std::memory_order_relaxed); + // Complete this accepted read window before admitting the + // next. All four host viewers release their snapshots and + // wait here, so retired windows cannot grow without bound. + snap.reset(); + account->drain_retirements(); } - views.fetch_add(1, std::memory_order_relaxed); + } + catch (...) { + failure = std::current_exception(); + bad.store(true); } }); } - std::mt19937 rng(44); - uint64_t slot = 0; - for (int step = 0; step < 6000; ++step) { - ++slot; - std::string key = make_key(static_cast<int>(rng() % K)); - if ((rng() % 4) == 0) { - ASSERT_TRUE(t.apply(slot, Batch{{{.key = key, .kind = OpKind::kDelete, .value = ""}}}).ok()); - } - else { - std::string val = "v" + std::to_string(slot); - ASSERT_TRUE(t.apply(slot, Batch{{{.key = key, .kind = OpKind::kPut, .value = val}}}).ok()); - } - max_slot_seen.store(slot, std::memory_order_release); - if ((rng() % 6) == 0) { - ASSERT_TRUE(t.flush().ok()); + int failed_step = -1; + const auto status = [&]() -> Status { + std::mt19937 rng(44); + uint64_t slot = 0; + for (int step = 0; step < 6000; ++step) { + failed_step = step; + ++slot; + std::string key = make_key(static_cast<int>(rng() % K)); + Status applied; + if ((rng() % 4) == 0) { + applied = t.apply(slot, Batch{{{.key = key, .kind = OpKind::kDelete, .value = ""}}}); + } + else { + std::string val = "v" + std::to_string(slot); + applied = t.apply(slot, Batch{{{.key = key, .kind = OpKind::kPut, .value = val}}}); + } + if (!applied.ok()) { + return applied; + } + max_slot_seen.store(slot, std::memory_order_release); + if ((rng() % 6) == 0) { + auto flushed = t.flush(); + if (!flushed.ok()) { + return flushed; + } + } } - } - ASSERT_TRUE(t.flush().ok()); + return t.flush(); + }(); stop.store(true); for (auto &th : viewers) { th.join(); } + for (const auto &failure : failures) { + if (failure) { + EXPECT_NO_THROW(std::rethrow_exception(failure)); + } + } + EXPECT_TRUE(status.ok()) << status.to_string() << " step=" << failed_step; EXPECT_FALSE(bad.load()); EXPECT_GT(views.load(), 0); } diff --git a/lib/crowdb-tree/tests/integration/tree_apply_grant_test.cpp b/lib/crowdb-tree/tests/integration/tree_apply_grant_test.cpp new file mode 100644 index 000000000..73abaedbd --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_apply_grant_test.cpp @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_apply_operation.h" +#include "btree/tree_core_access.h" +#include "btree/tree_core_batch.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +class TreeApplyGrant : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + } + + Config config; + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; +}; + +TEST_F(TreeApplyGrant, CompleteGapBoundIncludesBothCopiedInputAndFutureCell) +{ + // A single 8 MiB value fits the read limit and one source's payload, + // but two owned copies plus descriptors cannot fit the 16 MiB gap budget. + const std::string value(8 * 1024 * 1024, 'v'); + const auto before = core->binding()->account()->snapshot(); + std::shared_ptr<TreeApplyOperation> operation; + const auto status = core->prepare_apply( + 1, [&](size_t) { return CoreWriteInput{.key = Slice("key"), .value = Slice(value)}; }, + detail::BudgetClass::kAdmittedWriter, 0, &operation); + EXPECT_EQ(status.code(), Code::kInvalidArgument); + EXPECT_FALSE(operation); + const auto after = core->binding()->account()->snapshot(); + EXPECT_EQ(after.reserved, before.reserved); + EXPECT_EQ(after.allocated, before.allocated); + EXPECT_EQ(core->contiguous_slot(), 0U); +} + +TEST_F(TreeApplyGrant, RetainedCallerControlsArePartOfTheSameCompleteBound) +{ + std::shared_ptr<TreeApplyOperation> operation; + EXPECT_EQ(core->prepare_apply(0, {}, detail::BudgetClass::kAdmittedWriter, + config.runtime_binding.limits.gap_repair_bytes, &operation) + .code(), + Code::kInvalidArgument); + EXPECT_FALSE(operation); + EXPECT_TRUE(core->prepare_apply(0, {}, detail::BudgetClass::kAdmittedWriter, 256, &operation).ok()); + EXPECT_TRUE(operation); +} + +TEST_F(TreeApplyGrant, PermanentOrdinaryLimitDoesNotBecomeAnUnfinishableWait) +{ + Config smaller = config; + smaller.runtime_binding.limits.writer_bytes = 1024 * 1024; + Crowdbtree local(smaller); + const std::string value(600 * 1024, 'v'); + std::shared_ptr<TreeApplyOperation> operation; + EXPECT_EQ(TreeCoreAccess::get(local) + ->prepare_apply( + 1, [&](size_t) { return CoreWriteInput{.key = Slice("key"), .value = Slice(value)}; }, + detail::BudgetClass::kAdmittedWriter, 0, &operation) + .code(), + Code::kInvalidArgument); + EXPECT_FALSE(operation); +} + +TEST_F(TreeApplyGrant, DuplicateKeysKeepLastCellButChargeAllRetainedInputBytes) +{ + const std::array<std::string, 3> values{"first", "second", "last"}; + std::shared_ptr<TreeApplyOperation> operation; + ASSERT_TRUE( + core->prepare_apply( + values.size(), + [&](size_t index) { return CoreWriteInput{.key = Slice("same"), .value = Slice(values[index])}; }, + detail::BudgetClass::kAdmittedWriter, 0, &operation) + .ok()); + ASSERT_TRUE(operation->arm().ok()); + + struct Completion + { + std::atomic<bool> done{false}; + Status status; + } completion; + + operation->start( + 1, + [](void *raw, detail::CallbackSignal, Status status) noexcept { + auto &value = *static_cast<Completion *>(raw); + value.status = std::move(status); + value.done.store(true, std::memory_order_release); + value.done.notify_all(); + }, + &completion); + while (!completion.done.load(std::memory_order_acquire)) { + completion.done.wait(false, std::memory_order_acquire); + } + EXPECT_TRUE(completion.status.ok()); + EXPECT_EQ(tree->get_view("same").value().to_string(), "last"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_admitted_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_admitted_test.cpp new file mode 100644 index 000000000..71d723773 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_admitted_test.cpp @@ -0,0 +1,297 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_apply_operation.h" +#include "btree/tree_core_access.h" +#include "btree/tree_core_batch.h" + +#include <gtest/gtest.h> + +#include <array> +#include <thread> +#include <vector> + +namespace crowdb::tree +{ +namespace +{ +struct TestApplyCompletion +{ + std::atomic<bool> done{false}; + Status status; + + static void complete(void *context, detail::CallbackSignal signal, Status result) noexcept + { + auto &self = *static_cast<TestApplyCompletion *>(context); + self.status = signal == detail::CallbackSignal::kStopped ? Status::unavailable() : std::move(result); + self.done.store(true, std::memory_order_release); + self.done.notify_all(); + } + + void wait() const + { + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + } +}; + +class TreeCoreAdmitted : public testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions host; + host.workers = 2; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.buffer_pool_bytes = 4 * 1024 * 1024; + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + } + + Status prepare(Slice key, Slice value, std::shared_ptr<TreeApplyOperation> *out) + { + return core->prepare_apply( + 1, [=](size_t) { return CoreWriteInput{.key = key, .value = value}; }, detail::BudgetClass::kAdmittedWriter, + 0, out); + } + + Status fill_full_source() + { + const auto batch_for = [](uint64_t slot) { + Batch batch; + for (size_t index = 0; index < 64; ++index) { + batch.ops.push_back( + {.key = "k" + std::to_string((slot * 64) + index), .kind = OpKind::kPut, .value = "v"}); + } + return batch; + }; + for (uint64_t slot = 1; slot < 64; ++slot) { + const auto status = tree->apply(slot, batch_for(slot)); + if (!status.ok()) { + return Status::internal_error("fill slot " + std::to_string(slot) + ": " + status.to_string()); + } + // Preparation targets a full source and retained Handoff credit, + // not temporary history candidates awaiting asynchronous disposal. + // The final CPU callback below must not join its own retirement work. + core->binding()->account()->drain_retirements(); + } + // The final batch commits from a CPU callback. Reaching the hard source + // cap starts asynchronous soft maintenance, while this callback cannot + // wait for the Handoff credit intentionally retained by the test. + TestApplyCompletion completion; + tree->get_async("k64", [&](Status read, GetView) { + auto status = read.ok() ? tree->apply(64, batch_for(64)) : read; + TestApplyCompletion::complete(&completion, detail::CallbackSignal::kValue, std::move(status)); + }); + completion.wait(); + core->binding()->lane()->drain(); + return completion.status; + } + + Config config; + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; +}; + +TEST_F(TreeCoreAdmitted, PreparationOwnsCallerBytesAndDoesNotBlockReplacementOrClose) +{ + std::shared_ptr<TreeApplyOperation> stale; + ASSERT_TRUE(prepare("before", "value", &stale).ok()); + ASSERT_TRUE(core->clear().ok()); + EXPECT_EQ(stale->arm().code(), Code::kUnavailable); + stale.reset(); + std::shared_ptr<TreeApplyOperation> unsubmitted; + ASSERT_TRUE(prepare("after", "value", &unsubmitted).ok()); + tree.reset(); + EXPECT_EQ(unsubmitted->arm().code(), Code::kUnavailable); +} + +TEST_F(TreeCoreAdmitted, FullBackendWindowReturnsTemporaryPressureAndRollsBackTheApplyTicket) +{ + auto &lifetime = core->operation_lifetime(); + const auto maximum = lifetime.resource_usage().limit; + std::vector<std::shared_ptr<TreeOperationLifetime::Ticket>> occupied; + occupied.reserve(maximum); + for (uint64_t index = 0; index < maximum; ++index) { + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + ASSERT_TRUE(lifetime.acquire(core->binding()->lifecycle_budget(), &ticket).ok()); + occupied.push_back(std::move(ticket)); + } + std::shared_ptr<TreeApplyOperation> operation; + ASSERT_TRUE(prepare("admitted", "value", &operation).ok()); + EXPECT_EQ(operation->arm().code(), Code::kUnavailable); + auto usage = core->admission_resource_usage(); + EXPECT_EQ(usage[0].used, maximum); + EXPECT_EQ(usage[0].denied_tree, 1U); + EXPECT_EQ(usage[1].used, 0U); + occupied.pop_back(); + ASSERT_TRUE(operation->arm().ok()); + usage = core->admission_resource_usage(); + EXPECT_EQ(usage[0].used, maximum); + EXPECT_EQ(usage[1].used, 1U); + TestApplyCompletion completed; + operation->start(1, &TestApplyCompletion::complete, &completed); + completed.wait(); + core->binding()->lane()->drain(); + ASSERT_TRUE(completed.status.ok()) << completed.status.to_string(); + usage = core->admission_resource_usage(); + EXPECT_EQ(usage[0].used, maximum - 1); + EXPECT_EQ(usage[1].used, 0U); + EXPECT_EQ(usage[3].used, 1U); + EXPECT_EQ(usage[5].used, 0U); +} + +TEST_F(TreeCoreAdmitted, ArmedApplyFinishesAfterCloseFencesIngressAndCallerDropsTheHandle) +{ + std::shared_ptr<TreeApplyOperation> operation; + std::string key = "retained"; + std::string value = "owned before consensus"; + ASSERT_TRUE(prepare(key, value, &operation).ok()); + key.assign("changed"); + value.clear(); + ASSERT_TRUE(operation->arm().ok()); + std::atomic<bool> closed{false}; + std::jthread closer([&] { + core->close_and_drain(); + closed.store(true, std::memory_order_release); + }); + while (!core->closed()) { + std::this_thread::yield(); + } + EXPECT_FALSE(closed.load(std::memory_order_acquire)); + TestApplyCompletion completed; + operation->start(1, &TestApplyCompletion::complete, &completed); + operation.reset(); + completed.wait(); + closer.join(); + EXPECT_TRUE(completed.status.ok()) << completed.status.to_string(); + EXPECT_TRUE(closed.load(std::memory_order_acquire)); + EXPECT_EQ(core->contiguous_slot(), 1U); +} + +TEST_F(TreeCoreAdmitted, DistantNoopAwaitsAnActualFrontierTransitionWithoutSkippingTheGap) +{ + std::shared_ptr<TreeApplyOperation> operation; + ASSERT_TRUE(core->prepare_apply(0, {}, detail::BudgetClass::kAdmittedWriter, 0, &operation).ok()); + ASSERT_TRUE(operation->arm().ok()); + TestApplyCompletion completed; + operation->start(ApplyFrontier::kMaxWindow + 1, &TestApplyCompletion::complete, &completed); + core->binding()->lane()->drain(); + EXPECT_FALSE(completed.done.load(std::memory_order_acquire)); + EXPECT_EQ(core->contiguous_slot(), 0U); + core->force_advance_slot(1); + completed.wait(); + EXPECT_TRUE(completed.status.ok()); + EXPECT_EQ(core->contiguous_slot(), 1U); + EXPECT_EQ(core->max_seen_slot(), ApplyFrontier::kMaxWindow + 1); +} + +TEST_F(TreeCoreAdmitted, FullSourceAwaitsCompleteFlushCreditThenConsumesThePrepaidWrite) +{ + tree.reset(); + config.max_memtable_count = 2; + config.memtable_flush_entries = UINT32_MAX; + config.memtable_flush_bytes = SIZE_MAX; + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + // Fill one complete hard source window with Handoff credit held. The + // final callback write starts soft backpressure without blocking itself. + auto pressure = detail::ResourceBudget::try_reserve(core->binding()->account(), detail::BudgetClass::kHandoff, + config.runtime_binding.limits.handoff_bytes); + ASSERT_TRUE(pressure); + std::shared_ptr<TreeApplyOperation> operation; + std::string value = "owned"; + ASSERT_TRUE(prepare("last", value, &operation).ok()); + value.clear(); + ASSERT_TRUE(operation->arm().ok()); + const auto filled = fill_full_source(); + ASSERT_TRUE(filled.ok()) << filled.to_string(); + ASSERT_EQ(core->capture()->sources->tables.back()->count(), 4096U); + TestApplyCompletion completed; + operation->start(65, &TestApplyCompletion::complete, &completed); + operation.reset(); + core->binding()->lane()->drain(); + EXPECT_FALSE(completed.done.load(std::memory_order_acquire)) + << completed.status.to_string() << " tables=" << core->capture()->sources->tables.size() + << " active=" << core->capture()->sources->tables.back()->count(); + const auto waiting = core->publisher().work_metrics()->totals(); + EXPECT_EQ(waiting[static_cast<size_t>(TreeStage::FlushAdmissionWait)].inflight, 1U); + EXPECT_EQ(waiting[static_cast<size_t>(TreeStage::BudgetWait)].inflight, 1U); + pressure.reset(); + completed.wait(); + ASSERT_TRUE(completed.status.ok()) << completed.status.to_string(); + EXPECT_EQ(tree->get_view("last").value().to_string(), "owned"); + EXPECT_EQ(core->contiguous_slot(), 65U); + const auto terminal = core->publisher().work_metrics()->totals(); + EXPECT_EQ(terminal[static_cast<size_t>(TreeStage::FlushAdmissionWait)].count, 1U); + EXPECT_EQ(terminal[static_cast<size_t>(TreeStage::BudgetWait)].count, 4U); + EXPECT_EQ(terminal[static_cast<size_t>(TreeStage::BudgetWait)].inflight, 0U); +} + +TEST_F(TreeCoreAdmitted, CompleteBatchMustFitTheGapSourceBeforeItCanBeProposed) +{ + std::shared_ptr<TreeApplyOperation> operation; + auto status = core->prepare_apply( + 65, + [](size_t index) { + static const std::array<std::string, 65> keys = [] { + std::array<std::string, 65> result; + for (size_t item = 0; item < result.size(); ++item) { + result[item] = std::to_string(item); + } + return result; + }(); + return CoreWriteInput{.key = Slice(keys[index]), .value = Slice("value")}; + }, + detail::BudgetClass::kAdmittedWriter, 0, &operation); + EXPECT_EQ(status.code(), Code::kInvalidArgument); + EXPECT_FALSE(operation); + EXPECT_EQ(core->contiguous_slot(), 0U); +} + +TEST_F(TreeCoreAdmitted, FullOrdinaryCpuQueueCannotRejectAcceptedHandoff) +{ + tree.reset(); + config.max_memtable_count = 2; + config.memtable_flush_entries = UINT32_MAX; + config.memtable_flush_bytes = SIZE_MAX; + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + auto pressure = detail::ResourceBudget::try_reserve(core->binding()->account(), detail::BudgetClass::kHandoff, + config.runtime_binding.limits.handoff_bytes); + ASSERT_TRUE(pressure); + std::shared_ptr<TreeApplyOperation> operation; + ASSERT_TRUE(prepare("accepted", "paid", &operation).ok()); + ASSERT_TRUE(operation->arm().ok()); + const auto filled = fill_full_source(); + ASSERT_TRUE(filled.ok()) << filled.to_string(); + ASSERT_EQ(core->capture()->sources->tables.back()->count(), 4096U); + auto runtime = core->binding()->cpu(); + auto other = std::make_shared<detail::FlushRuntime::Lane>(1024); + std::vector<std::shared_ptr<detail::FlushRuntime::Progress>> occupied; + for (;;) { + std::shared_ptr<detail::FlushRuntime::Progress> slot; + auto status = runtime->reserve_progress(other, [](void *) noexcept {}, nullptr, {}, &slot); + if (!status.ok()) { + ASSERT_EQ(status.code(), Code::kResourceExhausted); + break; + } + occupied.push_back(std::move(slot)); + } + ASSERT_FALSE(occupied.empty()); + EXPECT_EQ( + runtime->try_submit(core->binding()->lane(), [](void *) noexcept {}, nullptr).code(), Code::kResourceExhausted); + pressure.reset(); + TestApplyCompletion completed; + operation->start(65, &TestApplyCompletion::complete, &completed); + completed.wait(); + ASSERT_TRUE(completed.status.ok()) << completed.status.to_string(); + EXPECT_EQ(core->contiguous_slot(), 65U); + occupied.clear(); + EXPECT_EQ(tree->get_view("accepted").value().to_string(), "paid"); +} + +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_auto_write_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_auto_write_test.cpp new file mode 100644 index 000000000..6b7903a78 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_auto_write_test.cpp @@ -0,0 +1,182 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/apply_frontier.h" +#include "btree/tree_core_access.h" +#include "crowdb-tree/btree/tree.h" + +#include <gtest/gtest.h> + +#include <array> +#include <barrier> +#include <set> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +class TreeCoreAutoWrite : public testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions host; + host.workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.buffer_pool_bytes = 4 * 1024 * 1024; + tree = std::make_unique<Crowdbtree>(config); + } + + static Batch values(const std::string &value) + { + return { + .ops = {{.key = "a", .kind = OpKind::kPut, .value = value}, + {.key = "b", .kind = OpKind::kPut, .value = value}} + }; + } + + Config config; + std::unique_ptr<Crowdbtree> tree; +}; + +TEST_F(TreeCoreAutoWrite, ExplicitWritesNoopsAndForceAdvancePrecedeAutomaticSlots) +{ + ASSERT_TRUE(tree->apply(1, values("explicit")).ok()); + ASSERT_TRUE(tree->put("a", "auto").ok()); + EXPECT_EQ(tree->get_view("a").slot(), 2U); + ASSERT_TRUE(tree->apply(4, {}).ok()); + ASSERT_TRUE(tree->batch_put(values("batch")).ok()); + EXPECT_EQ(tree->get_view("a").slot(), 5U); + EXPECT_EQ(tree->get_view("b").slot(), 5U); + EXPECT_EQ(tree->contiguous_slot(), 2U); + ASSERT_TRUE(tree->apply(3, {}).ok()); + EXPECT_EQ(tree->contiguous_slot(), 5U); + ASSERT_TRUE(tree->del("a").ok()); + EXPECT_EQ(tree->contiguous_slot(), 6U); + EXPECT_FALSE(tree->get_view("a").found()); + tree->force_advance_slot(100); + ASSERT_TRUE(tree->put("a", "after-force").ok()); + EXPECT_EQ(tree->get_view("a").slot(), 101U); + EXPECT_EQ(tree->contiguous_slot(), 101U); +} + +TEST_F(TreeCoreAutoWrite, RejectedBatchDoesNotConsumeAnAutomaticSlot) +{ + tree.reset(); + config.runtime_binding.limits.writer_bytes = 2 * 1024 * 1024; + tree = std::make_unique<Crowdbtree>(config); + EXPECT_EQ(tree->put(std::string(config.frame_bytes, 'k'), "invalid").code(), Code::kInvalidArgument); + auto oversized = values("unused"); + oversized.ops[1].value.assign(config.runtime_binding.limits.writer_bytes, 'x'); + EXPECT_FALSE(tree->batch_put(oversized).ok()); + EXPECT_EQ(tree->contiguous_slot(), 0U); + EXPECT_FALSE(tree->get_view("a").found()); + ASSERT_TRUE(tree->put("a", "first").ok()); + EXPECT_EQ(tree->get_view("a").slot(), 1U); + ASSERT_TRUE(tree->batch_put({}).ok()); + EXPECT_EQ(tree->contiguous_slot(), 2U); +} + +TEST_F(TreeCoreAutoWrite, SlotWindowAndIntegerExhaustionCannotPublishAPartialBatch) +{ + ASSERT_TRUE(tree->apply(ApplyFrontier::kMaxWindow, {}).ok()); + EXPECT_EQ(tree->batch_put(values("outside-window")).code(), Code::kResourceExhausted); + EXPECT_FALSE(tree->get_view("a").found()); + tree->force_advance_slot(ApplyFrontier::kMaxWindow); + ASSERT_TRUE(tree->batch_put(values("next")).ok()); + EXPECT_EQ(tree->get_view("a").slot(), ApplyFrontier::kMaxWindow + 1); + tree->force_advance_slot(UINT64_MAX); + EXPECT_EQ(tree->batch_put(values("wrapped")).code(), Code::kResourceExhausted); + EXPECT_EQ(tree->get_view("a").value().to_string(), "next"); +} + +TEST_F(TreeCoreAutoWrite, ConcurrentBatchesUseDistinctSlotsAndLeaveNoGap) +{ + constexpr size_t threads = 4; + constexpr size_t per_thread = 16; + std::barrier start(threads); + std::array<std::thread, threads> writers; + for (size_t thread = 0; thread < threads; ++thread) { + writers[thread] = std::thread([&, thread] { + start.arrive_and_wait(); + for (size_t index = 0; index < per_thread; ++index) { + const auto key = std::to_string(thread) + "/" + std::to_string(index); + const auto status = tree->put(Slice(key), "v"); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + }); + } + for (auto &writer : writers) { + writer.join(); + } + EXPECT_EQ(tree->contiguous_slot(), threads * per_thread); + std::set<uint64_t> slots; + for (size_t thread = 0; thread < threads; ++thread) { + for (size_t index = 0; index < per_thread; ++index) { + const auto key = std::to_string(thread) + "/" + std::to_string(index); + auto value = tree->get_view(Slice(key)); + ASSERT_TRUE(value.found()); + slots.insert(value.slot()); + } + } + EXPECT_EQ(slots.size(), threads * per_thread); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(tree->last_applied_slot(), threads * per_thread); +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST_F(TreeCoreAutoWrite, SecondPreparationFailureKeepsAllKeysAndAutomaticSlotUnchanged) +{ + ASSERT_TRUE(tree->apply(1, values("old")).ok()); + auto table = TreeCoreAccess::get(*tree)->capture()->sources->tables.back(); + table->set_hook_for_tests(nullptr, [](void *, ConcurrentSkipList::PausePoint point, Slice key) { + if (point == ConcurrentSkipList::PausePoint::kBeforePrepare && key == Slice("b")) { + throw std::bad_alloc(); + } + }); + EXPECT_EQ(tree->batch_put(values("new")).code(), Code::kResourceExhausted); + EXPECT_EQ(tree->get_view("a").value().to_string(), "old"); + EXPECT_EQ(tree->get_view("b").value().to_string(), "old"); + EXPECT_EQ(tree->contiguous_slot(), 1U); + EXPECT_EQ(table->writers(), 0U); + table->set_hook_for_tests(nullptr, nullptr); + ASSERT_TRUE(tree->batch_put(values("new")).ok()); + EXPECT_EQ(tree->get_view("a").slot(), 2U); + EXPECT_EQ(tree->get_view("b").slot(), 2U); + EXPECT_EQ(tree->contiguous_slot(), 2U); +} + +TEST_F(TreeCoreAutoWrite, ExplicitEncodedAndExternalPreparationFailuresRemainFullyRetryable) +{ + ASSERT_TRUE(tree->apply(1, values("old")).ok()); + auto table = TreeCoreAccess::get(*tree)->capture()->sources->tables.back(); + table->set_hook_for_tests(nullptr, [](void *, ConcurrentSkipList::PausePoint point, Slice key) { + if (point == ConcurrentSkipList::PausePoint::kBeforePrepare && key == Slice("b")) { + throw std::bad_alloc(); + } + }); + EXPECT_EQ(tree->apply(2, values("new")).code(), Code::kResourceExhausted); + std::vector<Crowdbtree::encoded_op> encoded; + std::vector<Crowdbtree::external_op> external; + for (const auto *key : {"a", "b"}) { + encoded.push_back({.key = key, .cell = encode_cell_buf(2, OpKind::kPut, "encoded")}); + external.push_back({.key = key, .flags = 0, .value = buffer::copy_of("external")}); + } + EXPECT_EQ(tree->apply_encoded(2, std::move(encoded)).code(), Code::kResourceExhausted); + EXPECT_EQ(tree->apply_external(2, std::move(external)).code(), Code::kResourceExhausted); + EXPECT_EQ(tree->get_view("a").slot(), 1U); + EXPECT_EQ(tree->get_view("b").slot(), 1U); + EXPECT_EQ(tree->contiguous_slot(), 1U); + EXPECT_EQ(table->writers(), 0U); + table->set_hook_for_tests(nullptr, nullptr); + ASSERT_TRUE(tree->apply(2, values("retry")).ok()); + ASSERT_TRUE(tree->apply(2, values("duplicate")).ok()); + EXPECT_EQ(tree->get_view("a").value().to_string(), "retry"); + EXPECT_EQ(tree->get_view("b").value().to_string(), "retry"); + EXPECT_EQ(tree->contiguous_slot(), 2U); +} +#endif +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_block_account_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_block_account_test.cpp new file mode 100644 index 000000000..f12c02658 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_block_account_test.cpp @@ -0,0 +1,145 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "snapshot/commit_anchor.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kRouting = static_cast<size_t>(detail::BudgetClass::kRouting); +constexpr size_t kMetadata = static_cast<size_t>(AllocationKind::kMetadata); + +TEST(TreeCoreBlockAccount, ConstructorRecoversAndTransfersExistingBackendWithMonotonicAutoSlots) +{ + tree_test::TempDir directory("core_block_budget_"); + std::unique_ptr<BlockPageStore> device; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 4, 5, 65536, 1, &device).ok()); + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 2; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.page_store = device.get(); + config.frame_bytes = 4096; + std::shared_ptr<detail::ResourceBudget::Account> first; + { + Crowdbtree tree(config); + first = TreeCoreAccess::get(tree)->binding()->account(); + ASSERT_TRUE(tree.apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "first"}}}).ok()); + ASSERT_TRUE(tree.snapshot().ok()); + } + EXPECT_GT(first->snapshot().allocated[kRouting], 0U); + { + Crowdbtree reopened(config); + EXPECT_EQ(first->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(first->snapshot().physical[kMetadata], 0U); + EXPECT_EQ(reopened.contiguous_slot(), 1U); + GetView old; + ASSERT_TRUE(TreeCoreAccess::get(reopened)->get("a", &old).ok()); + ASSERT_TRUE(old.found()); + EXPECT_EQ(old.value().to_string(), "first"); + EXPECT_EQ(old.slot(), 1U); + ASSERT_TRUE(reopened.put("a", "second").ok()); + EXPECT_EQ(reopened.contiguous_slot(), 2U); + ASSERT_TRUE(reopened.snapshot().ok()); + } + { + Crowdbtree again(config); + EXPECT_EQ(again.contiguous_slot(), 2U); + ASSERT_TRUE(again.put("a", "third").ok()); + GetView value; + ASSERT_TRUE(TreeCoreAccess::get(again)->get("a", &value).ok()); + EXPECT_EQ(value.slot(), 3U); + EXPECT_EQ(value.value().to_string(), "third"); + } +} + +TEST(TreeCoreBlockAccount, FailedAdoptionReleasesDomainAndPrivateCreateCannotBypassIt) +{ + tree_test::TempDir directory("core_block_fail_"); + std::unique_ptr<BlockPageStore> device; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 4, 5, 65536, 1, &device).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = device.get(); + config.frame_bytes = 4096; + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(config.runtime, config.runtime_binding, &binding).ok()); + std::shared_ptr<TreeCore> bypass; + EXPECT_EQ(TreeCore::create(config, binding, {}, &bypass).code(), Code::kInvalidArgument); + EXPECT_FALSE(bypass); + binding.reset(); + const auto baseline = config.runtime->snapshot(); + const auto original_limit = config.runtime_binding.limits.routing_bytes; + config.runtime_binding.limits.routing_bytes = config.runtime_binding.lifecycle_bytes + 1; + bool rejected = false; + try { + Crowdbtree candidate(config); + } + catch (const std::runtime_error &) { + rejected = true; + } + EXPECT_TRUE(rejected); + EXPECT_EQ(config.runtime->snapshot().allocated.routing_bytes, baseline.allocated.routing_bytes); + EXPECT_EQ(config.runtime->snapshot().reserved.routing_bytes, baseline.reserved.routing_bytes); + ASSERT_TRUE(device->try_claim_runtime_domain()); + device->release_runtime_domain(); + config.runtime_binding.limits.routing_bytes = original_limit; + { + Crowdbtree accepted(config); + ASSERT_TRUE(accepted.put("a", "value").ok()); + ASSERT_TRUE(accepted.snapshot().ok()); + } + ASSERT_TRUE(device->try_claim_runtime_domain()); + device->release_runtime_domain(); +} + +TEST(TreeCoreBlockAccount, FailedRecoveryRetainsRealBackendChargeAndReleasesDomainForRetry) +{ + tree_test::TempDir directory("core_block_recovery_fail_"); + std::unique_ptr<BlockPageStore> device; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 4, 5, 65536, 1, &device).ok()); + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = device.get(); + config.frame_bytes = 4096; + const auto before_backend = config.runtime->snapshot().allocated.routing_bytes; + { + Crowdbtree original(config); + ASSERT_TRUE(original.put("a", "retained").ok()); + ASSERT_TRUE(original.snapshot().ok()); + } + std::vector<snapshot::CommitAnchor> anchors; + ASSERT_TRUE(snapshot::read_valid_anchors(*device, device->iu_size(), &anchors).ok()); + ASSERT_EQ(anchors.size(), 1U); + uint8_t original = 0; + ASSERT_TRUE(device->read_at(anchors[0].segdir_addr, &original, 1).ok()); + const uint8_t corrupted = original ^ 1; + ASSERT_TRUE(device->write_at(anchors[0].segdir_addr, &corrupted, 1).ok()); + std::shared_ptr<TreeCore> rejected; + const auto status = TreeCore::open(config, &rejected); + EXPECT_FALSE(status.ok()) << status.to_string(); + EXPECT_FALSE(rejected); + EXPECT_GT(config.runtime->snapshot().allocated.routing_bytes, before_backend); + ASSERT_TRUE(device->try_claim_runtime_domain()); + device->release_runtime_domain(); + ASSERT_TRUE(device->write_at(anchors[0].segdir_addr, &original, 1).ok()); + { + Crowdbtree retry(config); + GetView found; + ASSERT_TRUE(TreeCoreAccess::get(retry)->get("a", &found).ok()); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), "retained"); + } + device.reset(); + EXPECT_EQ(config.runtime->snapshot().allocated.routing_bytes, before_backend); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_checkpoint_test.cpp new file mode 100644 index 000000000..3745aed97 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_checkpoint_test.cpp @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "btree/tree_core_access.h" +#include "local_checkpoint_test_support.h" +#include "snapshot/checkpoint_operation.h" +#include "snapshot/operation_wait.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestPublicCheckpoint +{ + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + Config config; + std::unique_ptr<Crowdbtree> tree; + Status last_open; + + TestPublicCheckpoint() + { + EXPECT_TRUE(TreeRuntime::create({}, &runtime).ok()); + store.device = device; + config.runtime = runtime; + config.page_store = device.get(); + config.async_page_store = &store; + config.frame_bytes = 4096; + config.max_inline_value = 64; + } + + Status open() + { + last_open = Crowdbtree::open(config, &tree); + return last_open; + } + + Status get(Slice key, GetView *out) const + { + std::atomic<bool> done{false}; + Status status; + tree->get_async(key, [&](Status result, GetView view) { + status = std::move(result); + *out = std::move(view); + done.store(true, std::memory_order_release); + done.notify_one(); + }); + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + return status; + } + + void read(const std::string &expected, uint64_t expected_slot) const + { + GetView view; + const auto status = get("a", &view); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(view.found()); + EXPECT_EQ(view.value().to_string(), expected); + EXPECT_EQ(view.slot(), expected_slot); + } +}; + +TEST(TreeCoreCheckpoint, PublicFreshCheckpointColdReadAndReopen) +{ + TestPublicCheckpoint test; + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + const std::string large(20000, 'v'); + ASSERT_TRUE(test.tree->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = large}}}).ok()); + uint64_t coverage = 0; + uint64_t sequence = 0; + auto status = test.tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 1U); + EXPECT_EQ(sequence, 1U); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 1U); + test.read(large, 1); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + test.read(large, 1); + ASSERT_TRUE(test.tree->apply(2, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "new"}}}).ok()); + status = test.tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 2U); + EXPECT_EQ(sequence, 2U); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + test.read("new", 2); +} + +TEST(TreeCoreCheckpoint, ActualWritebackCheckpointReuseAndColdReadKeepSeparateCauses) +{ + TestPublicCheckpoint test; + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + const std::string value(20000, 'v'); + ASSERT_TRUE(test.tree->put("a", value).ok()); + auto *core = TreeCoreAccess::get(*test.tree); + auto work = core->publisher().work_metrics(); + std::shared_ptr<CheckpointOperation> writeback; + ASSERT_TRUE(core->prepare_writeback(&writeback).ok()); + auto status = detail::wait_snapshot_operation(writeback); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_GT(writeback->result().written_frames, 0U); + const auto written = writeback->result().written_frames; + auto pressure = work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::CachePressure)]; + EXPECT_EQ(pressure[static_cast<size_t>(TreeWorkCounter::WriteOperations)], written); + EXPECT_EQ(pressure[static_cast<size_t>(TreeWorkCounter::EncodedFrameBytes)], written * test.config.frame_bytes); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 0U); + writeback.reset(); + test.read(value, 1); + EXPECT_GT(work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Read)] + [static_cast<size_t>(TreeWorkCounter::ReadBytes)], + 0U); + status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto checkpoint = work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Checkpoint)]; + EXPECT_EQ(checkpoint[static_cast<size_t>(TreeWorkCounter::ReusedFrames)], written); + EXPECT_GT(checkpoint[static_cast<size_t>(TreeWorkCounter::ReusedBackingBytes)], 0U); + EXPECT_GT(checkpoint[static_cast<size_t>(TreeWorkCounter::WriteBytes)], 0U); + EXPECT_EQ(checkpoint[static_cast<size_t>(TreeWorkCounter::EncodedFrameBytes)], 0U); + EXPECT_EQ(work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::CachePressure)] + [static_cast<size_t>(TreeWorkCounter::WriteOperations)], + written); +} + +TEST(TreeCoreCheckpoint, BackendDomainRejectsConcurrentOpenAndReleasesAtLastOwner) +{ + TestPublicCheckpoint test; + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + std::unique_ptr<Crowdbtree> second; + auto status = Crowdbtree::open(test.config, &second); + EXPECT_EQ(status.code(), Code::kUnavailable); + EXPECT_FALSE(second); + test.tree.reset(); + status = Crowdbtree::open(test.config, &second); + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +TEST(TreeCoreCheckpoint, FailedOpenReturnsBackendDomain) +{ + TestPublicCheckpoint test; + test.config.runtime_binding.limits.routing_bytes = test.config.runtime_binding.lifecycle_bytes; + EXPECT_FALSE(test.open().ok()); + EXPECT_FALSE(test.tree); + test.config.runtime_binding = {}; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); +} + +TEST(TreeCoreCheckpoint, SingleIoCreditDoesNotOverlapHandoffAndCheckpoint) +{ + TestPublicCheckpoint test; + test.config.runtime_binding.limits.io_operations = 1; + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + ASSERT_TRUE(test.tree->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "one"}}}).ok()); + auto status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + // The second handoff retains the first checkpoint's cold base. Its + // preflight must release the read window before reserving write I/O. + ASSERT_TRUE(test.tree->apply(2, Batch{.ops = {{.key = "b", .kind = OpKind::kPut, .value = "two"}}}).ok()); + status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.read("one", 1); + GetView view; + status = test.get("b", &view); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(view.found()); + EXPECT_EQ(view.slot(), 2U); + EXPECT_EQ(view.value().to_string(), "two"); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 2U); +} + +TEST(TreeCoreCheckpoint, RecoveredOverflowCanBeCheckpointedWithoutGlobalResolver) +{ + TestPublicCheckpoint test; + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + const std::string large(12000, 'r'); + ASSERT_TRUE(test.tree->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = large}}}).ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + ASSERT_TRUE(test.tree->apply(2, Batch{.ops = {{.key = "b", .kind = OpKind::kPut, .value = "other"}}}).ok()); + auto status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()) << test.last_open.to_string(); + test.read(large, 1); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 2U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_chunk_read_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_chunk_read_test.cpp new file mode 100644 index 000000000..683cb38c7 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_chunk_read_test.cpp @@ -0,0 +1,148 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_tree_core_chunk.h" + +#include <map> + +namespace crowdb::tree +{ +namespace +{ +using test_chunk::TestCoreChunk; +using TestContents = std::map<std::string, std::string>; + +TestContents seed(TestCoreChunk &test) +{ + TestContents expected; + Batch batch; + for (size_t i = 0; i < 96; ++i) { + const auto key = std::to_string(1000 + i); + const auto value = std::string(i % 31 == 0 ? 12000 : 48, static_cast<char>('a' + (i % 26))); + expected.emplace(key, value); + batch.ops.push_back({.key = key, .kind = OpKind::kPut, .value = value}); + } + EXPECT_TRUE(test.tree->apply(1, batch).ok()); + const auto status = test.tree->snapshot(); + EXPECT_TRUE(status.ok()) << status.to_string(); + return expected; +} + +void check_scan(const Crowdbtree &tree, const TestContents &expected, bool reverse, bool keys_only = false) +{ + TestContents actual; + std::string cursor; + bool truncated = true; + size_t windows = 0; + while (truncated && windows++ <= expected.size()) { + std::vector<scan_entry> entries; + const auto status = reverse ? tree.scan_reverse(cursor, !cursor.empty(), false, {}, 7, 0, &entries, &truncated) + : tree.scan({}, cursor, {}, 7, 0, keys_only, 0, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (truncated) { + ASSERT_FALSE(entries.empty()); + } + for (const auto &entry : entries) { + if (!cursor.empty()) { + EXPECT_TRUE(reverse ? entry.key < cursor : entry.key > cursor); + } + cursor = entry.key; + EXPECT_TRUE(actual.emplace(entry.key, entry.value).second); + } + } + ASSERT_FALSE(truncated); + ASSERT_EQ(actual.size(), expected.size()); + for (const auto &[key, value] : expected) { + const auto found = actual.find(key); + ASSERT_NE(found, actual.end()) << key; + EXPECT_EQ(found->second, keys_only ? std::string{} : value) << key; + } +} + +TEST(TreeCoreChunkRead, ColdForwardReverseAndKeyOnlyWindowsMatchTheCommittedKeyspace) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + const auto expected = seed(test); + ASSERT_GT(test.tree->tree_state_stats().logical.leaf_pages, 1U); + test.tree.reset(); + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto metrics = TreeCoreAccess::get(*test.tree)->publisher().work_metrics(); + const auto encoded = metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count; + check_scan(*test.tree, expected, false, true); + check_scan(*test.tree, expected, false); + check_scan(*test.tree, expected, true); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].count, encoded); + const auto summary = test.tree->tree_summary(); + EXPECT_TRUE(summary.exact); + EXPECT_EQ(summary.live_kv, expected.size()); + EXPECT_EQ(test.tree->tree_state_stats().logical.kv_unknown_pages, 0U); +} + +TEST(TreeCoreChunkRead, OverlayTombstonesAndOldSnapshotKeepDifferentFixedViewsAfterClose) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + const auto previous = seed(test); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + auto snapshot = test.tree->snapshot_view(); + ASSERT_NE(snapshot, nullptr); + EXPECT_EQ(snapshot->at_slot(), 1U); + auto expected = previous; + Batch batch; + for (size_t i = 0; i < 96; i += 7) { + const auto key = std::to_string(1000 + i); + batch.ops.push_back({.key = key, .kind = OpKind::kDelete, .value = {}}); + expected.erase(key); + } + batch.ops.push_back({.key = "1031", .kind = OpKind::kPut, .value = "new"}); + expected["1031"] = "new"; + ASSERT_TRUE(test.tree->apply(2, batch).ok()); + ASSERT_TRUE(test.tree->flush().ok()); + EXPECT_FALSE(test.tree->tree_summary().exact); + check_scan(*test.tree, expected, false); + check_scan(*test.tree, expected, true); + ASSERT_TRUE(test.tree->snapshot().ok()); + test.tree.reset(); + for (const auto &[key, value] : previous) { + std::string actual; + uint64_t slot = 0; + ASSERT_TRUE(snapshot->get(key, &slot, &actual)) << key; + EXPECT_EQ(slot, 1U); + EXPECT_EQ(actual, value) << key; + } + snapshot.reset(); + ASSERT_TRUE(test.open().ok()); + check_scan(*test.tree, expected, false); + check_scan(*test.tree, expected, true); +} + +TEST(TreeCoreChunkRead, ColdInspectionUsesOneIoPermitAndPreservesTheCapturedRevision) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + const auto expected = seed(test); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + NativeFrame frame; + uint64_t version = 0; + uint64_t root = 0; + uint32_t deltas = 0; + auto status = test.tree->inspect_page({}, kInvalidPageId, &frame, &version, &root, &deltas); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(frame_page_type(frame.frame.data()), page_type::kInnerBase); + const auto captured = version; + status = test.tree->inspect_page({0}, captured, &frame, &version, &root, &deltas); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(frame_page_type(frame.frame.data()), page_type::kLeafBase); + ASSERT_TRUE(frame_validate(frame.frame.data(), frame.frame.size())); + LeafFrameView leaf(frame.frame.data(), frame.frame.size()); + ASSERT_GT(leaf.count(), 0U); + EXPECT_TRUE(expected.contains(leaf.key(0).to_string())); + ASSERT_TRUE(test.tree->apply(2, {}).ok()); + ASSERT_TRUE(test.tree->flush().ok()); + EXPECT_EQ(test.tree->inspect_page({}, captured, &frame, &version, &root, &deltas).code(), Code::kUnavailable); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_chunk_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_chunk_test.cpp new file mode 100644 index 000000000..c9723e0b6 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_chunk_test.cpp @@ -0,0 +1,215 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_backend_state.h" +#include "btree/tree_core_access.h" +#include "common/test_tree_core_chunk.h" +#include "snapshot/chunk_root_recovery.h" +#include "snapshot/operation_wait.h" + +namespace crowdb::tree +{ +using test_chunk::TestCoreChunk; + +TEST(TreeCoreChunk, SingleIoCheckpointColdReadAndReopenKeepWalCoveragePair) +{ + TestCoreChunk test; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(20000, 'v'); + ASSERT_TRUE(test.put(1, value).ok()); + test.store->request_wal_offset(8192); + uint64_t coverage = 0; + uint64_t sequence = 0; + status = test.tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 1U); + EXPECT_EQ(sequence, 1U); + EXPECT_EQ(test.store->durable_wal_offset(), 8192U); + EXPECT_EQ(test.store->manifest_generation(), 1U); + auto held = test.tree->get_view("key"); + ASSERT_TRUE(held.found()); + EXPECT_EQ(held.value().to_string(), value); + test.tree.reset(); + EXPECT_EQ(held.value().to_string(), value); + status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto recovered_limits = TreeCoreAccess::get(*test.tree)->load_limits(); + EXPECT_EQ(recovered_limits.max_frame_bytes, test.config.frame_bytes); + EXPECT_EQ(recovered_limits.max_blob_bytes, 8192U); + GetView reopened; + status = TreeCoreAccess::get(*test.tree)->get("key", &reopened); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(reopened.found()); + EXPECT_EQ(reopened.value().to_string(), value); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 1U); + ASSERT_TRUE(test.put(2, "next").ok()); + test.store->request_wal_offset(12288); + status = test.tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 2U); + EXPECT_EQ(sequence, 2U); + EXPECT_EQ(test.store->durable_wal_offset(), 12288U); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + EXPECT_EQ(test.tree->get_view("key").value().to_string(), "next"); +} + +TEST(TreeCoreChunk, LostCommitReplyResolvesOneRootWithoutDuplicateWindows) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "value").ok()); + test.backend.catalog->lose_commit = true; + auto status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(test.backend.catalog->commit_calls, 1U); + EXPECT_EQ(test.backend.catalog->prepare_calls, 1U); + EXPECT_EQ(test.backend.catalog->abort_calls, 0U); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 1U); +} + +TEST(TreeCoreChunk, ANewWalRequestCannotChangeThePairOfAnAcceptedCheckpoint) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "value").ok()); + test.store->request_wal_offset(4096); + test.backend.transport->before_allocate = [&] { test.store->request_wal_offset(8192); }; + auto status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(test.store->requested_wal_offset(), 8192U); + EXPECT_EQ(test.store->durable_wal_offset(), 4096U); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 1U); +} + +TEST(TreeCoreChunk, UnqueryableCommitBlocksAnotherCheckpointUntilValidatedReopen) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "value").ok()); + test.store->request_wal_offset(4096); + test.backend.catalog->lose_commit = true; + test.backend.catalog->fail_query = true; + EXPECT_EQ(test.tree->snapshot().code(), Code::kUnavailable); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 0U); + EXPECT_EQ(test.store->durable_wal_offset(), 0U); + EXPECT_EQ(test.tree->snapshot().code(), Code::kUnavailable); + EXPECT_EQ(test.backend.catalog->begin_calls, 1U); + test.tree.reset(); + test.backend.catalog->fail_query = false; + auto status = test.open(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 1U); + EXPECT_EQ(test.store->durable_wal_offset(), 4096U); + EXPECT_EQ(test.tree->get_view("key").value().to_string(), "value"); +} + +TEST(TreeCoreChunk, ASecondWriterCannotClaimTheSameOpenFacade) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + std::unique_ptr<Crowdbtree> second; + EXPECT_EQ(Crowdbtree::open(test.config, &second).code(), Code::kUnavailable); + EXPECT_FALSE(second); + test.tree.reset(); + EXPECT_EQ(Crowdbtree::open(test.config, &second).code(), Code::kUnavailable); + ASSERT_TRUE(test.open().ok()); +} + +TEST(TreeCoreChunk, DetachedNativeCursorReadsExactPacksAfterTreeClose) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, std::string(20000, 's')).ok()); + auto status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::unique_ptr<NativeFrameIterator> cursor; + status = test.tree->open_native_frame_iterator(nullptr, &cursor); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto info = cursor->stream_info(); + test.tree.reset(); + size_t count = 0; + bool done = false; + while (!done) { + std::optional<NativeFrameView> frame; + status = cursor->next_owned(&frame, &done); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (frame) { + ++count; + EXPECT_TRUE(frame_validate(frame->bytes.data(), frame->bytes.size())); + } + } + EXPECT_GT(count, 1U); + EXPECT_EQ(count, info.frame_count); +} + +TEST(TreeCoreChunk, ReopenRejectsCorruptPhysicalPackEvenWhenCatalogMetadataIsValid) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, std::string(12000, 'x')).ok()); + auto status = test.tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.tree.reset(); + const auto found = + std::find_if(test.backend.catalog->records.begin(), test.backend.catalog->records.end(), [](const auto &item) { + return item.second.purpose == static_cast<uint32_t>(PagePurpose::kBtreePage); + }); + ASSERT_NE(found, test.backend.catalog->records.end()); + const auto physical = test.backend.transport->ids.at({found->second.chunk_high, found->second.chunk_low}); + test.backend.transport->memory.corrupt_mirror(physical, 0, 64); + test.backend.transport->memory.corrupt_mirror(physical, 1, 64); + status = test.open(); + EXPECT_EQ(status.code(), Code::kCorruption) << status.to_string(); + EXPECT_FALSE(test.tree); +} + +TEST(TreeCoreChunk, ExactBootstrapReadsOnlyNamedRootAndCannotPublishADivergentSuccessor) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "first").ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + ASSERT_TRUE(test.put(2, "second").ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + test.tree.reset(); + auto status = test.open(1); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(test.store->manifest_generation(), 1U); + EXPECT_EQ(test.tree->get_view("key").value().to_string(), "first"); + ASSERT_TRUE(test.put(2, "branch").ok()); + EXPECT_EQ(test.tree->snapshot().code(), Code::kUnavailable); + EXPECT_EQ(test.backend.catalog->current_generation, 2U); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + EXPECT_EQ(test.tree->get_view("key").value().to_string(), "second"); +} + +TEST(TreeCoreChunk, ClosedWriterQueuesQuiescenceWhileIndependentReaderRemainsUsable) +{ + TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "value").ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + std::unique_ptr<NativeFrameIterator> cursor; + ASSERT_TRUE(test.tree->open_native_frame_iterator(nullptr, &cursor).ok()); + auto owner = test.backend.catalog; + TreeCoreAccess::get(*test.tree)->close_and_drain(); + EXPECT_EQ(owner->writer_releases, 0U); + EXPECT_GT(owner->queue->snapshot().pending, 0U); + owner->fail_writer_release = true; + const auto failed = owner->queue->pump(64); + EXPECT_EQ(failed.failed, 1U); + owner->fail_writer_release = false; + EXPECT_GT(owner->queue->pump(64).completed, 0U); + EXPECT_EQ(owner->writer_releases, 2U); + std::optional<NativeFrameView> frame; + bool done = false; + ASSERT_TRUE(cursor->next_owned(&frame, &done).ok()); + ASSERT_TRUE(frame); + if (frame) { + EXPECT_TRUE(frame_validate(frame->bytes.data(), frame->bytes.size())); + } +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_chunk_writeback_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_chunk_writeback_test.cpp new file mode 100644 index 000000000..ea8e1bec4 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_chunk_writeback_test.cpp @@ -0,0 +1,144 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_tree_core_chunk.h" +#include "snapshot/checkpoint_operation.h" +#include "snapshot/operation_wait.h" + +#include <future> + +namespace crowdb::tree +{ +namespace +{ +struct TestPendingAllocation +{ + std::promise<void> entered; + std::atomic<bool> release{false}; + + ~TestPendingAllocation() + { + resume(); + } + + void resume() noexcept + { + release.store(true, std::memory_order_release); + release.notify_all(); + } + + void wait() + { + if (release.load(std::memory_order_acquire)) { + return; + } + entered.set_value(); + while (!release.load(std::memory_order_acquire)) { + release.wait(false, std::memory_order_acquire); + } + } +}; + +struct TestResumeAllocation +{ + std::shared_ptr<TestPendingAllocation> gate; + + ~TestResumeAllocation() + { + gate->resume(); + } +}; +} // namespace + +TEST(TreeCoreChunkWriteback, ColdUncommittedPacksStayReadableButReopenUsesTheDurableWalPair) +{ + test_chunk::TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "committed").ok()); + test.store->request_wal_offset(4096); + ASSERT_TRUE(test.tree->snapshot().ok()); + auto *core = TreeCoreAccess::get(*test.tree); + ASSERT_TRUE(test.put(2, std::string(20000, 's')).ok()); + test.store->request_wal_offset(8192); + std::shared_ptr<CheckpointOperation> writeback; + ASSERT_TRUE(core->prepare_writeback(&writeback).ok()); + const auto staged = detail::wait_snapshot_operation(writeback); + ASSERT_TRUE(staged.ok()) << staged.to_string(); + EXPECT_TRUE(writeback->result().writeback_completed); + EXPECT_FALSE(writeback->result().durable_committed); + EXPECT_EQ(writeback->result().captured_coverage, 2U); + EXPECT_EQ(writeback->result().durable_coverage, 1U); + EXPECT_EQ(test.store->durable_wal_offset(), 4096U); + EXPECT_EQ(test.store->manifest_generation(), 1U); + EXPECT_EQ(test.backend.catalog->commit_calls, 1U); + EXPECT_GT(core->capture()->stats.checkpoint_pending_pages, 0U); + writeback.reset(); + (void)test.tree->evict_clean_leaves(0); + const auto metrics = core->publisher().work_metrics(); + const auto loads = metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count; + GetView actual; + const auto cold = core->get("key", &actual); + ASSERT_TRUE(cold.ok()) << cold.to_string(); + EXPECT_EQ(actual.value().to_string(), std::string(20000, 's')); + EXPECT_GT(metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count, loads); + actual = {}; + test.tree.reset(); + const auto reopened = test.open(); + ASSERT_TRUE(reopened.ok()) << reopened.to_string(); + EXPECT_EQ(test.tree->get_view("key").value().to_string(), "committed"); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 1U); + EXPECT_EQ(test.store->durable_wal_offset(), 4096U); +} + +TEST(TreeCoreChunkWriteback, PendingAllocationRejectsAnotherCheckpointAndCancellationKeepsNewerApply) +{ + test_chunk::TestCoreChunk test; + ASSERT_TRUE(test.open().ok()); + ASSERT_TRUE(test.put(1, "committed").ok()); + ASSERT_TRUE(test.tree->snapshot().ok()); + ASSERT_TRUE(test.put(2, std::string(12000, 's')).ok()); + auto gate = std::make_shared<TestPendingAllocation>(); + auto entered = gate->entered.get_future(); + test.backend.transport->before_allocate = [gate] { gate->wait(); }; + auto *core = TreeCoreAccess::get(*test.tree); + std::shared_ptr<CheckpointOperation> writeback; + ASSERT_TRUE(core->prepare_writeback(&writeback).ok()); + detail::SnapshotWaitCompletion completion; + auto connected = stdexec::connect(writeback->sender(), detail::SnapshotWaitReceiver{.completion = &completion}); + TestResumeAllocation resume{gate}; + stdexec::start(connected); + const auto pending = entered.wait_for(std::chrono::seconds(2)); + if (pending != std::future_status::ready) { + gate->resume(); + } + ASSERT_EQ(pending, std::future_status::ready); + EXPECT_TRUE(core->capture()->checkpoint_inflight); + EXPECT_FALSE(completion.done.completed_for_tests()); + auto second = std::make_shared<std::promise<Status>>(); + auto second_result = second->get_future(); + test.tree->snapshot_async([second](Status status, uint64_t) { second->set_value(std::move(status)); }); + const auto rejected = second_result.wait_for(std::chrono::seconds(2)); + if (rejected != std::future_status::ready) { + gate->resume(); + } + ASSERT_EQ(rejected, std::future_status::ready); + EXPECT_EQ(second_result.get().code(), Code::kUnavailable); + ASSERT_TRUE(test.put(3, "newer").ok()); + writeback->request_stop(); + EXPECT_FALSE(completion.done.completed_for_tests()); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 1U); + gate->resume(); + completion.done.wait(); + EXPECT_EQ(completion.status.code(), Code::kUnavailable); + EXPECT_FALSE(core->capture()->checkpoint_inflight); + EXPECT_EQ(test.tree->get_view("key").value().to_string(), "newer"); + EXPECT_EQ(test.backend.catalog->commit_calls, 1U); + writeback.reset(); + test.backend.transport->before_allocate = {}; + const auto committed = test.tree->snapshot(); + ASSERT_TRUE(committed.ok()) << committed.to_string(); + EXPECT_EQ(test.tree->durable_snapshot_last_applied_slot(), 3U); + test.tree.reset(); + ASSERT_TRUE(test.open().ok()); + EXPECT_EQ(test.tree->get_view("key").value().to_string(), "newer"); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_empty_open_budget_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_empty_open_budget_test.cpp new file mode 100644 index 000000000..c29cc1c3e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_empty_open_budget_test.cpp @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/resource_budget.h" +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/crowdb-tree.h" +#include "runtime/tree_runtime_binding.h" + +#include <gtest/gtest.h> + +#include <future> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeCoreEmptyOpenBudget, ConcurrentEmptyDomainsPayOnlyTheirRecoveryWindows) +{ + TreeRuntimeOptions options; + options.limits.routing_bytes = 16 * 1024 * 1024; + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create(options, &runtime).ok()); + TreeBindingOptions binding_options; + binding_options.limits.routing_bytes = 8 * 1024 * 1024; + std::shared_ptr<TreeRuntimeBinding> other_tree; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, binding_options, &other_tree).ok()); + auto retained = + detail::ResourceBudget::try_reserve(other_tree->account(), detail::BudgetClass::kRouting, 4 * 1024 * 1024); + ASSERT_TRUE(retained); + MemPageStore first_store(1); + MemPageStore second_store(1); + auto open = [&](MemPageStore *store) { + Config config; + config.runtime = runtime; + config.runtime_binding = binding_options; + config.page_store = store; + std::unique_ptr<Crowdbtree> tree; + auto status = Crowdbtree::open(config, &tree); + return std::pair{status, std::move(tree)}; + }; + auto first = std::async(std::launch::async, open, &first_store); + auto second = std::async(std::launch::async, open, &second_store); + auto left = first.get(); + auto right = second.get(); + ASSERT_TRUE(left.first.ok()) << left.first.to_string(); + ASSERT_TRUE(right.first.ok()) << right.first.to_string(); + EXPECT_TRUE(left.second->put("left", "value").ok()); + EXPECT_TRUE(right.second->put("right", "value").ok()); + EXPECT_EQ(left.second->memtable_count(), 1U); + EXPECT_EQ(right.second->memtable_count(), 1U); + EXPECT_EQ(runtime->snapshot().bindings, 3U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_flush_read_budget_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_flush_read_budget_test.cpp new file mode 100644 index 000000000..0a8bcd726 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_flush_read_budget_test.cpp @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "snapshot/operation_wait.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(TreeCoreFlushReadBudget, CapturedLeafOnlyTreeLeavesDefaultRoutingCreditForReads) +{ + Config options; + ASSERT_TRUE(TreeRuntime::create({}, &options.runtime).ok()); + Crowdbtree tree(options); + ASSERT_TRUE(tree.put("key", "value").ok()); + auto *core = TreeCoreAccess::get(tree); + ASSERT_EQ(core->capture()->stats.inner_pages, 0U); + ASSERT_EQ(core->cache()->retained_page_limit(), 256U); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(core->prepare_flush(&flush).ok()); + auto capture = stdexec::sync_wait(flush->capture_sender()); + if (!capture) { + FAIL() << "flush capture did not complete"; + } + ASSERT_TRUE(std::get<0>(*capture).handoff_status.ok()); + GetView value; + const auto read = core->get("key", &value); + EXPECT_TRUE(read.ok()) << read.to_string(); + EXPECT_EQ(value.value().to_string(), "value"); + // Always finish accepted capture before the host's blocking close. + EXPECT_TRUE(detail::wait_snapshot_operation(flush).ok()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_generation_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_generation_checkpoint_test.cpp new file mode 100644 index 000000000..55e325dff --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_generation_checkpoint_test.cpp @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "local_checkpoint_test_support.h" +#include "snapshot/published_snapshot.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestGenerationBackend +{ + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + + TestGenerationBackend() + { + store.device = device; + } +}; + +TEST(TreeCoreGenerationCheckpoint, ClearPersistsZeroCoverageWithoutInvalidatingOldOwnedViews) +{ + auto backend = std::make_shared<TestGenerationBackend>(); + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = backend->device.get(); + config.async_page_store = &backend->store; + config.backend_owner = backend; + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + std::unique_ptr<Crowdbtree> tree; + auto status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(12000, 'p'); + ASSERT_TRUE(tree->apply(1, Batch{.ops = {{.key = "old", .kind = OpKind::kPut, .value = value}}}).ok()); + uint64_t coverage = 0; + uint64_t sequence = 0; + status = tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(coverage, 1U); + ASSERT_EQ(sequence, 1U); + + auto *core = TreeCoreAccess::get(*tree); + const auto generation = core->capture()->generation; + std::shared_ptr<PublishedSnapshotContext> captured; + status = core->capture_snapshot(&captured); + ASSERT_TRUE(status.ok()) << status.to_string(); + GetView retained; + status = captured->get("old", &retained); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(retained.found()); + + status = tree->clear(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(core->capture()->generation, generation + 1); + EXPECT_EQ(core->capture()->coverage, 0U); + status = tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 0U); + EXPECT_EQ(sequence, 2U); + EXPECT_EQ(tree->durable_snapshot_last_applied_slot(), 0U); + GetView absent; + status = core->get("old", &absent); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_FALSE(absent.found()); + GetView old_again; + status = captured->get("old", &old_again); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(old_again.found()); + EXPECT_EQ(old_again.value().to_string(), value); + EXPECT_EQ(retained.value().to_string(), value); + + tree.reset(); + captured.reset(); + status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->durable_snapshot_last_applied_slot(), 0U); + EXPECT_EQ(tree->contiguous_slot(), 0U); + core = TreeCoreAccess::get(*tree); + status = core->get("old", &absent); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_FALSE(absent.found()); + EXPECT_EQ(retained.value().to_string(), value); + ASSERT_TRUE(tree->put("new", "after-clear").ok()); + status = tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 1U); + EXPECT_EQ(sequence, 3U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_generation_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_generation_test.cpp new file mode 100644 index 000000000..0746eef00 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_generation_test.cpp @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/btree/tree.h" + +#include <gtest/gtest.h> + +#include <semaphore> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +class TreeCoreGeneration : public testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions host; + host.workers = 2; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.buffer_pool_bytes = 4 * 1024 * 1024; + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + } + + Config config; + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; +}; + +TEST_F(TreeCoreGeneration, ClearResetsSourcesSlotsAndGcWithoutInvalidatingBorrowedValues) +{ + ASSERT_TRUE(tree->put("old", "retained").ok()); + ASSERT_TRUE(tree->flush().ok()); + auto borrowed = tree->get_view("old"); + auto original = core->capture(); + tree->set_gc_watermark(100, 100); + ASSERT_EQ(tree->gc_watermark(), 100U); + ASSERT_TRUE(tree->apply(3, {}).ok()); + ASSERT_TRUE(tree->clear().ok()); + const auto empty = core->capture(); + EXPECT_EQ(empty->generation, original->generation + 1); + EXPECT_GT(empty->commit_seq, original->commit_seq); + EXPECT_EQ(empty->coverage, 0U); + EXPECT_EQ(empty->gc_floor, 0U); + EXPECT_EQ(tree->gc_watermark(), 0U); + EXPECT_EQ(tree->contiguous_slot(), 0U); + EXPECT_EQ(core->max_seen_slot(), 0U); + EXPECT_EQ(core->memtable_count(), 0U); + EXPECT_EQ(empty->sources->floor, 0U); + EXPECT_EQ(empty->sources->tables.size(), 1U); + EXPECT_FALSE(tree->get_view("old").found()); + EXPECT_EQ(borrowed.value().to_string(), "retained"); + EXPECT_EQ(original->coverage, 1U); + ASSERT_TRUE(tree->put("new", "value").ok()); + EXPECT_EQ(tree->get_view("new").slot(), 1U); + ASSERT_TRUE(tree->apply(2, {}).ok()); + EXPECT_EQ(tree->contiguous_slot(), 2U); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(core->capture()->coverage, 2U); +} + +TEST_F(TreeCoreGeneration, ReplacedControllerReusesPauseCreditAndReleasesItAtClose) +{ + const auto account = core->binding()->account(); + ASSERT_TRUE(tree->put("old", "old-generation").ok()); + ASSERT_TRUE(tree->clear().ok()); + for (size_t round = 0; round < 3; ++round) { + ASSERT_TRUE(tree->put("key", std::to_string(round)).ok()); + const auto status = tree->flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(core->capture()->coverage, round + 1); + EXPECT_EQ(tree->get_view("key").value().to_string(), std::to_string(round)); + } + tree.reset(); + account->drain_retirements(); + const auto closed = account->snapshot(); + for (size_t category = 0; category < detail::ResourceBudget::kClasses; ++category) { + EXPECT_EQ(closed.allocated[category], 0U) << category; + EXPECT_EQ(closed.reserved[category], 0U) << category; + } +} + +TEST_F(TreeCoreGeneration, RejectedPreparationPreservesOriginalGenerationAndFutureWrites) +{ + ASSERT_TRUE(tree->put("old", "value").ok()); + const auto original = core->capture(); + auto pressure = detail::ResourceBudget::try_reserve(core->binding()->account(), detail::BudgetClass::kReclaim, + config.runtime_binding.limits.reclaim_bytes); + ASSERT_TRUE(pressure); + EXPECT_EQ(tree->clear().code(), Code::kResourceExhausted); + EXPECT_EQ(core->capture(), original); + EXPECT_EQ(tree->get_view("old").value().to_string(), "value"); + pressure.reset(); + ASSERT_TRUE(tree->put("next", "value").ok()); + EXPECT_EQ(tree->get_view("next").slot(), 2U); + ASSERT_TRUE(tree->clear().ok()); + EXPECT_EQ(core->capture()->generation, original->generation + 1); +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST_F(TreeCoreGeneration, ReplacementWaitsForAnAlreadyAdmittedWriterBeforeResettingItsFrontier) +{ + ASSERT_TRUE(tree->put("old", "value").ok()); + auto table = core->capture()->sources->tables.back(); + std::binary_semaphore entered(0); + std::binary_semaphore resume(0); + + struct Pause + { + std::binary_semaphore &entered; + std::binary_semaphore &resume; + } pause{.entered = entered, .resume = resume}; + + table->set_hook_for_tests(&pause, [](void *argument, ConcurrentSkipList::PausePoint point, Slice) { + if (point == ConcurrentSkipList::PausePoint::kBeforePrepare) { + auto &pause = *static_cast<Pause *>(argument); + pause.entered.release(); + pause.resume.acquire(); + } + }); + Status write; + Status cleared; + std::atomic<bool> installed{false}; + core->set_before_generation_install_for_tests([&] { installed.store(true, std::memory_order_release); }); + std::jthread writer([&] { write = tree->put("accepted", "old-generation"); }); + entered.acquire(); + std::jthread replacement([&] { cleared = tree->clear(); }); + // The writer remains an ingress participant, so publication cannot yet + // reach the generation install hook. + EXPECT_FALSE(installed.load(std::memory_order_acquire)); + resume.release(); + writer.join(); + replacement.join(); + ASSERT_TRUE(write.ok()) << write.to_string(); + ASSERT_TRUE(cleared.ok()) << cleared.to_string(); + EXPECT_TRUE(installed.load(std::memory_order_acquire)); + EXPECT_FALSE(tree->get_view("accepted").found()); + ASSERT_TRUE(tree->put("new", "value").ok()); + EXPECT_EQ(tree->get_view("new").slot(), 1U); +} + +TEST_F(TreeCoreGeneration, CloseWinsWhileReplacementIsDrainingAndAdmissionNeverReopens) +{ + ASSERT_TRUE(tree->put("old", "value").ok()); + const auto original = core->capture(); + std::binary_semaphore before_install(0); + std::binary_semaphore resume(0); + std::atomic<bool> reached_install{false}; + core->set_before_generation_install_for_tests([&] { + reached_install.store(true, std::memory_order_release); + before_install.release(); + resume.acquire(); + }); + Status cleared; + std::jthread replacement([&] { + cleared = tree->clear(); + if (!reached_install.load(std::memory_order_acquire)) { + before_install.release(); + } + }); + before_install.acquire(); + if (!reached_install.load(std::memory_order_acquire)) { + replacement.join(); + ASSERT_TRUE(cleared.ok()) << cleared.to_string(); + FAIL() << "replacement unexpectedly skipped its install boundary"; + } + std::atomic<bool> closed{false}; + std::jthread closer([&] { + core->close_and_drain(); + closed.store(true, std::memory_order_release); + }); + while (!core->closed()) { + std::this_thread::yield(); + } + EXPECT_FALSE(closed.load(std::memory_order_acquire)); + EXPECT_EQ(tree->put("rejected", "value").code(), Code::kUnavailable); + EXPECT_EQ(tree->clear().code(), Code::kUnavailable); + resume.release(); + replacement.join(); + closer.join(); + EXPECT_EQ(cleared.code(), Code::kUnavailable); + EXPECT_TRUE(closed.load(std::memory_order_acquire)); + EXPECT_EQ(core->capture(), original); + EXPECT_EQ(tree->put("still-closed", "value").code(), Code::kUnavailable); +} +#endif +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_native_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_native_checkpoint_test.cpp new file mode 100644 index 000000000..5f33e5133 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_native_checkpoint_test.cpp @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/overflow_backing.h" +#include "btree/tree_core_access.h" +#include "local_checkpoint_test_support.h" +#include "native_import_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestNativeBackend +{ + std::shared_ptr<MemPageStore> device = std::make_shared<MemPageStore>(4096); + test_write::TestCheckpointStore store; + + TestNativeBackend() + { + store.device = device; + } +}; + +TEST(TreeCoreNativeCheckpoint, ImportedResidentOverflowBecomesDurableAndReopens) +{ + auto backend = std::make_shared<TestNativeBackend>(); + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = backend->device.get(); + config.async_page_store = &backend->store; + config.backend_owner = backend; + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + std::unique_ptr<Crowdbtree> tree; + auto status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(12000, 'p'); + auto frames = test_import::overflow_tree(value); + const auto count = frames.size(); + status = tree->install_snapshot_native(std::move(frames), 0, 7, count); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto retained = tree->get_view("key"); + ASSERT_TRUE(retained.found()); + uint64_t coverage = 0; + uint64_t sequence = 0; + status = tree->snapshot(&coverage, &sequence); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(coverage, 7U); + EXPECT_EQ(sequence, 1U); + const auto page = TreeCoreAccess::get(*tree)->capture()->mapping.get(0); + ASSERT_NE(page, nullptr); + const auto primary = page->leaf->bases.front().backing; + ASSERT_NE(primary, nullptr); + EXPECT_TRUE(primary->reloadable()); + ASSERT_NE(primary->overflow, nullptr); + EXPECT_EQ(primary->overflow->pages.size(), count - 1); + for (const auto &[pid, dependency] : primary->overflow->pages) { + EXPECT_NE(pid, 0U); + EXPECT_TRUE(dependency.backing->reloadable()); + EXPECT_EQ(dependency.backing->backend_id, primary->backend_id); + } + tree.reset(); + status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->durable_snapshot_last_applied_slot(), 7U); + EXPECT_EQ(tree->get_view("key").value().to_string(), value); + EXPECT_EQ(retained.value().to_string(), value); + ASSERT_TRUE(tree->put("after", "value").ok()); + EXPECT_EQ(tree->get_view("after").slot(), 8U); +} + +TEST(TreeCoreNativeCheckpoint, SharedResidentChainIsWrittenOnceWithoutPinningTheRawImport) +{ + auto backend = std::make_shared<TestNativeBackend>(); + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = backend->device.get(); + config.async_page_store = &backend->store; + config.backend_owner = backend; + config.frame_bytes = 4096; + config.max_inline_value = 64; + std::unique_ptr<Crowdbtree> tree; + auto status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + NativeFrame index{.page_id = 0, .frame = std::vector<uint8_t>(4096)}; + ASSERT_TRUE(inner_frame_build(index.frame.data(), 4096, 0, {1, 2}, {Slice("m")})); + const std::string value(1000, 's'); + const auto cell = encode_overflow_cell_buf(7, 3, value.size()); + NativeFrame overflow{.page_id = 3, .frame = std::vector<uint8_t>(4096)}; + overflow_frame_build(overflow.frame.data(), 4096, 3, kInvalidPageId, + reinterpret_cast<const uint8_t *>(value.data()), value.size()); + std::vector<NativeFrame> frames; + frames.push_back(std::move(index)); + frames.push_back(test_import::leaf(1, "a", Slice(cell), 2)); + frames.push_back(test_import::leaf(2, "z", Slice(cell))); + frames.push_back(std::move(overflow)); + status = tree->install_snapshot_native(std::move(frames), 0, 7, 4); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto *core = TreeCoreAccess::get(*tree); + std::weak_ptr<const void> raw_owner = + core->capture()->mapping.get(1)->leaf->bases.front().backing->overflow->pages.at(3).backing->owner; + status = tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto first = core->capture()->mapping.get(1)->leaf->bases.front().backing; + const auto second = core->capture()->mapping.get(2)->leaf->bases.front().backing; + ASSERT_NE(first->overflow, nullptr); + ASSERT_NE(second->overflow, nullptr); + EXPECT_EQ(first->overflow->pages.at(3).backing, second->overflow->pages.at(3).backing); + EXPECT_TRUE(first->overflow->pages.at(3).backing->reloadable()); + tree.reset(); + EXPECT_TRUE(raw_owner.expired()); + status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->get_view("a").value().to_string(), value); + EXPECT_EQ(tree->get_view("z").value().to_string(), value); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_native_import_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_native_import_test.cpp new file mode 100644 index 000000000..8e3958d7c --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_native_import_test.cpp @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/snapshot/snapshot_io.h" +#include "native_import_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +class TreeCoreNativeImport : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.frame_bytes = 4096; + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + } + + Status install(std::vector<NativeFrame> frames, uint64_t coverage = 7) + { + const auto count = frames.size(); + return tree->install_snapshot_native(std::move(frames), count == 0 ? kInvalidPageId : 0, coverage, count); + } + + Config config; + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; +}; + +TEST_F(TreeCoreNativeImport, OwnedNativeFramesReplaceGenerationAndResumeAutomaticSlots) +{ + ASSERT_TRUE(tree->put("old", "retained").ok()); + const auto old = tree->get_view("old"); + const auto generation = core->capture()->generation; + const std::string value(9000, 'n'); + auto status = install(test_import::overflow_tree(value)); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(core->capture()->generation, generation + 1); + EXPECT_EQ(tree->contiguous_slot(), 7U); + EXPECT_EQ(core->capture()->coverage, 7U); + EXPECT_EQ(old.value().to_string(), "retained"); + EXPECT_FALSE(tree->get_view("old").found()); + EXPECT_EQ(tree->get_view("key").value().to_string(), value); + const auto primary = core->capture()->mapping.get(0)->leaf->bases.front().backing; + ASSERT_NE(primary, nullptr); + EXPECT_TRUE(primary->readable()); + EXPECT_FALSE(primary->reloadable()); + ASSERT_TRUE(tree->put("next", "value").ok()); + EXPECT_EQ(tree->get_view("next").slot(), 8U); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(tree->get_view("key").value().to_string(), value); +} + +TEST_F(TreeCoreNativeImport, EmptyImportKeepsCoverageAndBuildsAUsableEmptyRoot) +{ + ASSERT_TRUE(tree->put("old", "value").ok()); + const auto status = install({}, 17); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->contiguous_slot(), 17U); + EXPECT_FALSE(tree->get_view("old").found()); + ASSERT_TRUE(tree->put("new", "value").ok()); + EXPECT_EQ(tree->get_view("new").slot(), 18U); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(core->capture()->coverage, 18U); +} + +TEST_F(TreeCoreNativeImport, InvalidFrameAndGraphLeaveTheOldRootAndWriterUsable) +{ + ASSERT_TRUE(tree->put("old", "value").ok()); + const auto original = core->capture(); + for (size_t variant = 0; variant != 5; ++variant) { + std::vector<NativeFrame> frames{test_import::inline_leaf()}; + if (variant == 0) { + frames[0].frame.back() ^= 1; + } + else if (variant == 1) { + frames.push_back(frames[0]); + } + else if (variant == 2) { + frames.push_back(test_import::inline_leaf(1, "unreachable")); + } + else if (variant == 3) { + frames[0] = test_import::inline_leaf(0, "future", "value", 8); + } + else { + ASSERT_TRUE(inner_frame_build(frames[0].frame.data(), 4096, 0, {0}, {})); + } + const auto status = install(std::move(frames)); + EXPECT_EQ(status.code(), Code::kCorruption) << variant << ": " << status.to_string(); + EXPECT_EQ(core->capture(), original); + EXPECT_EQ(tree->get_view("old").value().to_string(), "value"); + } + ASSERT_TRUE(tree->put("new", "value").ok()); + EXPECT_EQ(tree->get_view("new").slot(), 2U); +} + +TEST_F(TreeCoreNativeImport, StreamingImportUsesRemainingHostCapacity) +{ + auto peer_config = config; + const auto host_limit = TreeRuntimeOptions{}.limits.reclaim_bytes; + peer_config.runtime_binding.limits.reclaim_bytes = host_limit; + Crowdbtree peer(peer_config); + auto pressure = detail::ResourceBudget::try_reserve(TreeCoreAccess::get(peer)->binding()->account(), + detail::BudgetClass::kReclaim, + host_limit - (config.runtime_binding.limits.reclaim_bytes / 2)); + ASSERT_TRUE(pressure); + std::unique_ptr<SnapshotExport> exported; + ASSERT_TRUE(snapshot_export_begin(peer, snapshot_format::kNative, 4096, &exported).ok()); + SnapshotImport imported(*tree); + bool done = false; + while (!done) { + std::string bytes; + ASSERT_TRUE(exported->next_chunk(&bytes, &done).ok()); + const auto status = imported.feed(Slice(bytes)); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + ASSERT_TRUE(imported.finish(nullptr).ok()); + EXPECT_EQ(core->contiguous_slot(), 0U); +} + +TEST_F(TreeCoreNativeImport, FailedStagingAdmissionPreservesTheOriginalGeneration) +{ + ASSERT_TRUE(tree->put("old", "value").ok()); + const auto original = core->capture(); + auto pressure = detail::ResourceBudget::try_reserve(core->binding()->account(), detail::BudgetClass::kReclaim, + config.runtime_binding.limits.reclaim_bytes); + ASSERT_TRUE(pressure); + EXPECT_EQ(install({test_import::inline_leaf()}).code(), Code::kResourceExhausted); + EXPECT_EQ(core->capture(), original); + pressure.reset(); + const auto status = install({test_import::inline_leaf()}); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree->get_view("key").value().to_string(), "imported"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_primary_reuse_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_primary_reuse_test.cpp new file mode 100644 index 000000000..54764b6f8 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_primary_reuse_test.cpp @@ -0,0 +1,216 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "crowdb-tree/backend/page_store.h" +#include "snapshot/checkpoint_operation.h" +#include "snapshot/operation_wait.h" +#include "snapshot/primary_backing.h" +#include "snapshot/published_native.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestCountedStore : MemPageStore +{ + TestCountedStore() : MemPageStore(4096) + { + } + + mutable std::atomic<size_t> reads{0}; + std::atomic<size_t> primary_writes{0}; + + Status read_at(uint64_t address, uint8_t *data, size_t bytes) const override + { + reads.fetch_add(1); + return MemPageStore::read_at(address, data, bytes); + } + + Status write_typed_at(PagePurpose purpose, uint64_t address, const uint8_t *data, size_t bytes) override + { + if (purpose == PagePurpose::kBtreePage) { + primary_writes.fetch_add(1); + } + return MemPageStore::write_at(address, data, bytes); + } +}; + +struct TestPrimaryReuse +{ + Config config; + std::shared_ptr<TestCountedStore> store = std::make_shared<TestCountedStore>(); + std::shared_ptr<TreeCore> core; + + TestPrimaryReuse() + { + EXPECT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.page_store = store.get(); + config.backend_owner = store; + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + const auto status = TreeCore::open(config, &core); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + [[nodiscard]] std::shared_ptr<const PublishedPage> page() const + { + auto view = core->capture(); + return view->mapping.get(view->root_page_id); + } +}; + +TEST(TreeCorePrimaryReuse, ColdInlineAndOverflowCheckpointReusesWithoutPrimaryIo) +{ + TestPrimaryReuse test; + const std::string large(12000, 'v'); + ASSERT_TRUE(test.core + ->apply(1, + Batch{ + .ops = {{.key = "a", .kind = OpKind::kPut, .value = large}, + {.key = "b", .kind = OpKind::kPut, .value = "small"}} + }) + .ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + const auto before = test.page(); + ASSERT_TRUE(before->primary); + const auto reads = test.store->reads.load(); + const auto writes = test.store->primary_writes.load(); + ASSERT_TRUE(test.core->checkpoint().ok()); + EXPECT_EQ(test.store->reads.load(), reads); + EXPECT_EQ(test.store->primary_writes.load(), writes); + EXPECT_EQ(test.page()->leaf->bases.front().backing, before->leaf->bases.front().backing); + EXPECT_EQ(test.page(), before); + test.core.reset(); + auto status = TreeCore::open(test.config, &test.core); + ASSERT_TRUE(status.ok()) << status.to_string(); + GetView value; + ASSERT_TRUE(test.core->get("a", &value).ok()); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.value().to_string(), large); + const auto reopened_reads = test.store->reads.load(); + ASSERT_TRUE(test.core->checkpoint().ok()); + EXPECT_EQ(test.store->reads.load(), reopened_reads); + EXPECT_EQ(test.store->primary_writes.load(), writes); +} + +TEST(TreeCorePrimaryReuse, ExactReceiptReuseIsNotReportedAsDeferredRepresentationWork) +{ + TestPrimaryReuse test; + ASSERT_TRUE(test.core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "value"}}}).ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + CheckpointOptions options; + options.write.frames.encode.frame_bytes = 4096; + options.write.frames.encode.inline_value_bytes = 64; + options.write.write.max_frame_bytes = 4096; + std::shared_ptr<CheckpointOperation> operation; + ASSERT_TRUE(test.core->prepare_checkpoint(options, &operation).ok()); + ASSERT_TRUE(detail::wait_snapshot_operation(operation).ok()); + EXPECT_TRUE(operation->result().durable_committed); + EXPECT_EQ(operation->result().primary_frames, 1U); + EXPECT_EQ(operation->result().reused_frames, 1U); + EXPECT_EQ(operation->result().written_frames, 0U); + EXPECT_EQ(operation->result().rebased_pages, 0U); + EXPECT_EQ(operation->result().rebase_deferred, 0U); +} + +TEST(TreeCorePrimaryReuse, LaterOverlayAndGcFloorInvalidateOldReceipts) +{ + TestPrimaryReuse test; + ASSERT_TRUE(test.core + ->apply(1, + Batch{ + .ops = {{.key = "a", .kind = OpKind::kPut, .value = "old"}, + {.key = "b", .kind = OpKind::kDelete, .value = ""}} + }) + .ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + auto old = test.page(); + ASSERT_TRUE(old->primary); + const auto &old_summary = old->primary->frame.summary; + if (!old_summary) { + FAIL() << "checkpoint omitted its primary summary"; + } + EXPECT_EQ(old_summary->encoded_entries, 2U); + auto writes = test.store->primary_writes.load(); + ASSERT_TRUE(test.core->apply(2, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "new"}}}).ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + EXPECT_GT(test.store->primary_writes.load(), writes); + EXPECT_NE(test.page()->leaf->bases.front().backing, old->leaf->bases.front().backing); + writes = test.store->primary_writes.load(); + test.core->request_gc_floor(2); + ASSERT_TRUE(test.core->complete_noop(3).ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + EXPECT_GT(test.store->primary_writes.load(), writes); + EXPECT_EQ(test.core->capture()->gc_floor, 2U); + const auto latest = test.page(); + ASSERT_TRUE(latest->primary); + const auto &summary = latest->primary->frame.summary; + if (!summary) { + FAIL() << "checkpoint omitted its primary summary"; + } + EXPECT_EQ(summary->encoded_entries, 1U); + GetView value; + ASSERT_TRUE(test.core->get("a", &value).ok()); + EXPECT_EQ(value.value().to_string(), "new"); + ASSERT_TRUE(test.core->get("b", &value).ok()); + EXPECT_FALSE(value.found()); +} + +TEST(TreeCorePrimaryReuse, NativeExportStillReturnsActualFramesAfterCleanCheckpoint) +{ + TestPrimaryReuse test; + ASSERT_TRUE(test.core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "value"}}}).ok()); + ASSERT_TRUE(test.core->checkpoint().ok()); + std::shared_ptr<PublishedNativeCursor> cursor; + const auto status = PublishedNativeCursor::create(test.core, {}, &cursor); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::optional<NativeFrameView> frame; + bool done = false; + auto next = cursor->next(&frame, &done); + ASSERT_TRUE(next.ok()) << next.to_string(); + if (!frame) { + FAIL() << "native cursor omitted its frame"; + } + EXPECT_EQ(frame->bytes.size(), test.config.frame_bytes); + EXPECT_TRUE(frame_validate(frame->bytes.data(), frame->bytes.size())); + EXPECT_TRUE(frame->owner); +} + +TEST(TreeCorePrimaryReuse, InnerAndLeavesKeepTheirDurableFramesAcrossCheckpointAndRecovery) +{ + TestPrimaryReuse test; + Batch batch; + for (size_t i = 0; i < 160; ++i) { + batch.ops.push_back({.key = std::string(4 - std::to_string(i).size(), '0') + std::to_string(i), + .kind = OpKind::kPut, + .value = std::string(32, 'v')}); + } + ASSERT_TRUE(test.core->apply(1, batch).ok()); + const auto first = test.core->checkpoint(); + ASSERT_TRUE(first.ok()) << first.to_string(); + ASSERT_GT(test.page()->level, 0U); + ASSERT_TRUE(test.page()->primary); + const auto writes = test.store->primary_writes.load(); + const auto backing = test.page()->inner_backing; + ASSERT_TRUE(test.core->checkpoint().ok()); + EXPECT_EQ(test.store->primary_writes.load(), writes); + EXPECT_EQ(test.page()->inner_backing, backing); + test.core.reset(); + const auto open = TreeCore::open(test.config, &test.core); + ASSERT_TRUE(open.ok()) << open.to_string(); + ASSERT_TRUE(test.page()->primary); + EXPECT_FALSE(test.page()->inner); + const auto next = test.core->checkpoint(); + ASSERT_TRUE(next.ok()) << next.to_string(); + EXPECT_EQ(test.store->primary_writes.load(), writes); + GetView value; + ASSERT_TRUE(test.core->get("0159", &value).ok()); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.value().to_string(), std::string(32, 'v')); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_public_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_public_test.cpp new file mode 100644 index 000000000..18b0a0bde --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_public_test.cpp @@ -0,0 +1,201 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "btree/tree_core_access.h" +#include "crowdb-tree/btree/tree.h" +#include "crowdb-tree/external_allocation.h" + +#include <gtest/gtest.h> + +#include <array> +#include <future> + +namespace crowdb::tree +{ +namespace +{ +class TreeCorePublic : public testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions host; + host.workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.buffer_pool_bytes = 4 * 1024 * 1024; + tree = std::make_unique<Crowdbtree>(config); + } + + void put(uint64_t slot, std::string key, std::string value) + { + const auto status = + tree->apply(slot, Batch{.ops = {{.key = std::move(key), .kind = OpKind::kPut, .value = std::move(value)}}}); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + Config config; + std::unique_ptr<Crowdbtree> tree; +}; + +TEST_F(TreeCorePublic, OutOfOrderSlotsAndNoopKeepFutureValuesAcrossFiniteFlush) +{ + put(2, "future", "second"); + EXPECT_EQ(tree->last_applied_slot(), 0U); + EXPECT_EQ(tree->contiguous_slot(), 0U); + ASSERT_TRUE(tree->apply(4, {}).ok()); + EXPECT_EQ(tree->contiguous_slot(), 0U); + put(1, "first", "one"); + EXPECT_EQ(tree->contiguous_slot(), 2U); + const auto flushed = tree->flush(); + ASSERT_TRUE(flushed.ok()) << flushed.to_string(); + EXPECT_EQ(tree->get_view("future").value().to_string(), "second"); + EXPECT_EQ(tree->get_view("first").value().to_string(), "one"); + ASSERT_TRUE(tree->apply(3, {}).ok()); + EXPECT_EQ(tree->contiguous_slot(), 4U); + EXPECT_EQ(tree->last_applied_slot(), 2U); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(tree->get_view("future").slot(), 2U); +} + +TEST_F(TreeCorePublic, BatchLastKeyWinsAndInvalidEncodedBatchPublishesNothing) +{ + Batch batch{ + .ops = {{.key = "same", .kind = OpKind::kPut, .value = "old"}, + {.key = "same", .kind = OpKind::kPut, .value = "latest"}, + {.key = "gone", .kind = OpKind::kDelete, .value = ""}} + }; + ASSERT_TRUE(tree->apply(1, batch).ok()); + EXPECT_EQ(tree->get_view("same").value().to_string(), "latest"); + EXPECT_FALSE(tree->get_view("gone").found()); + std::vector<Crowdbtree::encoded_op> invalid; + invalid.push_back({.key = "new", .cell = encode_cell_buf(2, OpKind::kPut, Slice("valid"))}); + invalid.push_back({.key = "same", .cell = encode_cell_buf(3, OpKind::kPut, Slice("invalid-slot"))}); + EXPECT_EQ(tree->apply_encoded(2, std::move(invalid)).code(), Code::kInvalidArgument); + EXPECT_FALSE(tree->get_view("new").found()); + EXPECT_EQ(tree->get_view("same").value().to_string(), "latest"); + EXPECT_EQ(tree->contiguous_slot(), 1U); +} + +TEST_F(TreeCorePublic, SingleWorkerAsyncFlushAndRetainedReadOutliveFacade) +{ + put(1, "key", std::string(1024, 'v')); + auto held = tree->get_view("key"); + ASSERT_TRUE(held.found()); + std::promise<Status> complete; + auto done = complete.get_future(); + tree->flush_async([&](Status status) { complete.set_value(std::move(status)); }); + const auto flushed = done.get(); + ASSERT_TRUE(flushed.ok()) << flushed.to_string(); + tree.reset(); + EXPECT_EQ(held.value().to_string(), std::string(1024, 'v')); + EXPECT_GT(config.runtime->snapshot().allocated.writer_bytes, 0U); + held = {}; + config.runtime->drain_cpu(); + EXPECT_EQ(config.runtime->snapshot().allocated.writer_bytes, 0U); +} + +TEST_F(TreeCorePublic, UnfundedBatchDoesNotChangeAnyKeyOrFrontier) +{ + tree.reset(); + config.runtime_binding.limits.writer_bytes = 2 * 1024 * 1024; + tree = std::make_unique<Crowdbtree>(config); + put(1, "existing", "before"); + const auto large = config.runtime_binding.limits.writer_bytes; + Batch batch{ + .ops = {{.key = "existing", .kind = OpKind::kPut, .value = "after"}, + {.key = "oversized", .kind = OpKind::kPut, .value = std::string(large, 'x')}} + }; + EXPECT_FALSE(tree->apply(2, batch).ok()); + EXPECT_EQ(tree->get_view("existing").value().to_string(), "before"); + EXPECT_FALSE(tree->get_view("oversized").found()); + EXPECT_EQ(tree->last_applied_slot(), 0U); + EXPECT_EQ(tree->contiguous_slot(), 1U); +} + +TEST_F(TreeCorePublic, DefaultMaximumValueBatchUsesOnePayloadReservation) +{ + const auto maximum = TreeCoreAccess::get(*tree)->load_limits().max_value_bytes; + Batch batch{ + .ops = {{.key = "large", .kind = OpKind::kPut, .value = std::string(maximum, 'v')}, + {.key = "small", .kind = OpKind::kPut, .value = "same-batch"}} + }; + const auto status = tree->apply(1, batch); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto value = tree->get_view("large"); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.value().size(), maximum); + EXPECT_EQ(value.value().data()[0], 'v'); + EXPECT_EQ(value.value().data()[maximum - 1], 'v'); + EXPECT_EQ(tree->get_view("small").value().to_string(), "same-batch"); + EXPECT_EQ(tree->contiguous_slot(), 1U); + batch.ops[0].value.push_back('x'); + batch.ops[1].value = "must-not-apply"; + EXPECT_EQ(tree->apply(2, batch).code(), Code::kInvalidArgument); + EXPECT_EQ(tree->get_view("small").value().to_string(), "same-batch"); + EXPECT_EQ(tree->contiguous_slot(), 1U); + std::vector<Crowdbtree::encoded_op> encoded; + encoded.push_back({.key = "oversized", .cell = encode_cell_buf(2, OpKind::kPut, Slice(batch.ops[0].value))}); + EXPECT_EQ(tree->apply_encoded(2, std::move(encoded)).code(), Code::kInvalidArgument); + std::vector<Crowdbtree::external_op> external; + const auto borrowed = Slice(batch.ops[0].value); + external.push_back({.key = "oversized", .flags = 0, .value = buffer::wrap(borrowed.bytes(), borrowed.size())}); + EXPECT_EQ(tree->apply_external(2, std::move(external)).code(), Code::kInvalidArgument); + batch.ops[0].kind = OpKind::kDelete; + batch.ops[1].value = "deleted"; + ASSERT_TRUE(tree->apply(2, batch).ok()); + EXPECT_FALSE(tree->get_view("large").found()); + EXPECT_EQ(tree->get_view("small").value().to_string(), "deleted"); +} + +TEST_F(TreeCorePublic, KnownExternalCapacityIsNotReservedAgainDuringBatchStaging) +{ + tree.reset(); + config.runtime_binding.limits.writer_bytes = 2 * 1024 * 1024; + tree = std::make_unique<Crowdbtree>(config); + + struct Payload + { + std::array<uint8_t, 1536 * 1024> bytes; + size_t *drops; + }; + + size_t drops = 0; + auto *payload = new Payload{.bytes = {}, .drops = &drops}; + payload->bytes.fill('v'); + ExternalAllocation::Owner owner(payload, [](void *context) { + auto *owned = static_cast<Payload *>(context); + ++*owned->drops; + delete owned; + }); + auto credit = detail::ResourceBudget::try_reserve(TreeCoreAccess::get(*tree)->binding()->account(), + detail::BudgetClass::kAdmittedWriter, sizeof(Payload) + 4096); + if (!credit) { + FAIL() << "external allocation credit rejected"; + } + auto reservation = detail::AllocationReservation::create(std::move(credit.value())); + auto allocation = ExternalAllocation::create(payload->bytes.data(), sizeof(Payload), std::move(owner), reservation); + std::vector<Crowdbtree::external_op> operations; + operations.push_back( + {.key = "external", .flags = 0, .value = buffer::share_external(allocation, 0, payload->bytes.size())}); + reservation->seal(); + reservation.reset(); + allocation.reset(); + const auto status = tree->apply_external(1, std::move(operations)); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto held = tree->get_view("external"); + ASSERT_TRUE(held.found()); + EXPECT_EQ(held.value().data(), reinterpret_cast<const char *>(payload->bytes.data())); + EXPECT_EQ(drops, 0U); + tree.reset(); + EXPECT_EQ(drops, 0U); + EXPECT_EQ(held.value().size(), 1536U * 1024); + held = {}; + EXPECT_EQ(drops, 1U); + // Reference metadata is retired after the final external reader exits. + config.runtime->drain_cpu(); + EXPECT_EQ(config.runtime->snapshot().allocated.writer_bytes, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_replay_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_replay_test.cpp new file mode 100644 index 000000000..1304cd96d --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_replay_test.cpp @@ -0,0 +1,321 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/local_backend_state.h" +#include "btree/tree_apply_operation.h" +#include "btree/tree_core.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +#include <array> + +namespace crowdb::tree +{ +namespace +{ +struct TestReplayCompletion +{ + std::atomic<bool> done{false}; + Status status; + + static void complete(void *context, detail::CallbackSignal signal, Status status) noexcept + { + auto &result = *static_cast<TestReplayCompletion *>(context); + result.status = signal == detail::CallbackSignal::kStopped ? Status::unavailable() : std::move(status); + result.done.store(true, std::memory_order_release); + result.done.notify_all(); + } + + void wait() const + { + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + } +}; + +struct TestReplayInput +{ + std::string key; + std::string value; +}; + +class TreeCoreReplay : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 64 * 1024, 4096, &device).ok()); + TreeRuntimeOptions host; + host.workers = 2; + // Replay and source rotation retain fixed paths; one proposal must also fit. + host.max_waiters_per_class = 3; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.leaf_split_bytes = 3000; + config.page_store = device.get(); + config.runtime_binding.limits.io_operations = 1; + config.runtime_binding.limits.gap_repair_bytes = 1024 * 1024; + ASSERT_TRUE(TreeCore::open(config, &core).ok()); + } + + Status prepare(uint64_t slot, std::string key, std::string value, std::shared_ptr<TreeApplyOperation> *out) + { + auto input = std::make_shared<TestReplayInput>(std::move(key), std::move(value)); + return core->prepare_replay( + slot, 1, + [raw = input.get()](size_t) { return CoreWriteInput{.key = Slice(raw->key), .value = Slice(raw->value)}; }, + input, 0, out); + } + + Status apply(uint64_t slot, std::string key, std::string value) + { + std::shared_ptr<TreeApplyOperation> operation; + auto status = prepare(slot, std::move(key), std::move(value), &operation); + if (!status.ok()) { + return status; + } + TestReplayCompletion completed; + operation->start(slot, &TestReplayCompletion::complete, &completed); + operation.reset(); + completed.wait(); + return completed.status; + } + + tree_test::TempDir directory{"tree_replay_"}; + std::unique_ptr<BlockPageStore> device; + Config config; + std::shared_ptr<TreeCore> core; +}; + +TEST_F(TreeCoreReplay, DefaultInlineGeometryReleasesChosenPayloadWithoutSnapshot) +{ + core.reset(); + auto store = std::make_shared<MemPageStore>(1); + Config large; + large.runtime = config.runtime; + large.page_store = store.get(); + large.backend_owner = store; + large.frame_bytes = 65536; + auto status = TreeCore::open(large, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(16384, 'v'); + for (uint64_t slot = 1; slot <= 4096; ++slot) { + status = apply(slot, "key" + std::to_string(10000 + slot), value); + ASSERT_TRUE(status.ok()) << slot << ": " << status.to_string(); + } + EXPECT_EQ(core->contiguous_slot(), 4096U); + EXPECT_EQ(core->durable_coverage(), 0U); + for (const auto slot : {1, 1764, 4096}) { + GetView found; + status = core->get("key" + std::to_string(10000 + slot), &found); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), value); + } +} + +TEST_F(TreeCoreReplay, CoalescedInlineReplaySplitsLogicalLeavesUnderGapPressure) +{ + core.reset(); + auto store = std::make_shared<MemPageStore>(1); + Config large; + large.runtime = config.runtime; + large.page_store = store.get(); + large.backend_owner = store; + large.frame_bytes = 65536; + large.runtime_binding.limits.gap_repair_bytes = 1024 * 1024; + auto status = TreeCore::open(large, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string value(16384, 'v'); + for (uint64_t slot = 1; slot <= 1024; ++slot) { + auto input = std::make_shared<std::array<TestReplayInput, 2>>(); + for (size_t index = 0; index < input->size(); ++index) { + (*input)[index] = {.key = "key" + std::to_string(10000 + (2 * slot) + index), .value = value}; + } + std::shared_ptr<TreeApplyOperation> operation; + status = core->prepare_replay( + slot, input->size(), + [raw = input.get()](size_t index) { + return CoreWriteInput{.key = Slice((*raw)[index].key), .value = Slice((*raw)[index].value)}; + }, + input, 0, &operation); + ASSERT_TRUE(status.ok()) << slot << ": " << status.to_string(); + TestReplayCompletion completed; + operation->start(slot, &TestReplayCompletion::complete, &completed); + operation.reset(); + completed.wait(); + ASSERT_TRUE(completed.status.ok()) << slot << ": " << completed.status.to_string(); + } + EXPECT_EQ(core->contiguous_slot(), 1024U); + EXPECT_EQ(core->durable_coverage(), 0U); + EXPECT_GT(core->publisher().work_metrics()->totals()[static_cast<size_t>(TreeStage::PageWritebackPipeline)].count, + 0U); + for (const auto index : {10002, 11025, 12049}) { + GetView found; + ASSERT_TRUE(core->get("key" + std::to_string(index), &found).ok()); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.value().to_string(), value); + } +} + +TEST_F(TreeCoreReplay, DistantChosenInputsDoNotConsumeTheGapPayloadReserve) +{ + std::array<std::shared_ptr<TreeApplyOperation>, 3> operations; + std::array<TestReplayCompletion, 3> completed; + const auto before = core->binding()->account()->snapshot(); + for (size_t index = 0; index < operations.size(); ++index) { + ASSERT_TRUE(prepare(index + 2, "key" + std::to_string(index), std::string(4096, 'v'), &operations[index]).ok()); + operations[index]->start(index + 2, &TestReplayCompletion::complete, &completed[index]); + } + core->binding()->lane()->drain(); + const auto waiting = core->binding()->account()->snapshot(); + const auto gap = static_cast<size_t>(detail::BudgetClass::kGapRepair); + EXPECT_EQ(waiting.allocated[gap] + waiting.reserved[gap], before.allocated[gap] + before.reserved[gap]); + for (const auto &result : completed) { + EXPECT_FALSE(result.done.load(std::memory_order_acquire)); + } + ASSERT_TRUE(apply(1, "first", "value").ok()); + for (const auto &result : completed) { + result.wait(); + EXPECT_TRUE(result.status.ok()) << result.status.to_string(); + } + EXPECT_EQ(core->contiguous_slot(), 4U); +} + +TEST_F(TreeCoreReplay, WholeGapGrantWaitsForActualQuotaRelease) +{ + auto pressure = detail::ResourceBudget::try_reserve(core->binding()->account(), detail::BudgetClass::kGapRepair, + config.runtime_binding.limits.gap_repair_bytes); + ASSERT_TRUE(pressure); + std::shared_ptr<TreeApplyOperation> operation; + ASSERT_TRUE(prepare(1, "key", "value", &operation).ok()); + TestReplayCompletion completed; + operation->start(1, &TestReplayCompletion::complete, &completed); + operation.reset(); + core->binding()->lane()->drain(); + EXPECT_FALSE(completed.done.load(std::memory_order_acquire)); + EXPECT_EQ(core->contiguous_slot(), 0U); + pressure.reset(); + completed.wait(); + EXPECT_TRUE(completed.status.ok()) << completed.status.to_string(); + EXPECT_EQ(core->contiguous_slot(), 1U); +} + +TEST_F(TreeCoreReplay, BoundedWritebackReleasesGapPayloadUnderFullOrdinaryCpuAndSingleIo) +{ + // Fill every spare registration after the replay and source rotation paths. + // Chosen pressure must use its startup-reserved path even when all classes are full. + std::vector<detail::ResourceBudget::Registration> slots; + for (auto category : {detail::BudgetClass::kHandoff, detail::BudgetClass::kRouting, detail::BudgetClass::kReclaim, + detail::BudgetClass::kGapRepair, detail::BudgetClass::kIo}) { + while (auto slot = detail::ResourceBudget::try_register(core->binding()->account(), category)) { + slots.push_back(std::move(*slot)); + } + EXPECT_FALSE(detail::ResourceBudget::try_register(core->binding()->account(), category)); + } + ASSERT_FALSE(slots.empty()); + const std::string value(256 * 1024, 'v'); + ASSERT_TRUE(apply(1, "a", value).ok()); + ASSERT_TRUE(apply(2, "b", value).ok()); + auto runtime = core->binding()->cpu(); + auto other = std::make_shared<detail::FlushRuntime::Lane>(1024); + std::vector<std::shared_ptr<detail::FlushRuntime::Progress>> occupied; + for (;;) { + std::shared_ptr<detail::FlushRuntime::Progress> slot; + auto status = runtime->reserve_progress(other, [](void *) noexcept {}, nullptr, {}, &slot); + if (!status.ok()) { + ASSERT_EQ(status.code(), Code::kResourceExhausted); + break; + } + occupied.push_back(std::move(slot)); + } + ASSERT_FALSE(occupied.empty()); + ASSERT_TRUE(apply(3, "c", value).ok()); + EXPECT_EQ(core->contiguous_slot(), 3U); + EXPECT_EQ(core->local_backend()->checkpoints->durable_coverage(), 0U); + EXPECT_GT(core->publisher().work_metrics()->totals()[static_cast<size_t>(TreeStage::PageWritebackPipeline)].count, + 0U); + occupied.clear(); + for (const auto *key : {"a", "b", "c"}) { + GetView result; + ASSERT_TRUE(core->get(key, &result).ok()); + EXPECT_EQ(result.value().to_string(), value); + } +} + +TEST_F(TreeCoreReplay, DuplicateCurrentGapSerializesItsPrepaidWaitPath) +{ + auto pressure = detail::ResourceBudget::try_reserve(core->binding()->account(), detail::BudgetClass::kGapRepair, + config.runtime_binding.limits.gap_repair_bytes); + ASSERT_TRUE(pressure); + std::array<std::shared_ptr<TreeApplyOperation>, 2> operations; + std::array<TestReplayCompletion, 2> completed; + for (size_t index = 0; index < operations.size(); ++index) { + ASSERT_TRUE(prepare(1, "key", "same-chosen-value", &operations[index]).ok()); + operations[index]->start(1, &TestReplayCompletion::complete, &completed[index]); + } + core->binding()->lane()->drain(); + for (const auto &result : completed) { + EXPECT_FALSE(result.done.load(std::memory_order_acquire)); + } + pressure.reset(); + for (const auto &result : completed) { + result.wait(); + EXPECT_TRUE(result.status.ok()) << result.status.to_string(); + } + EXPECT_EQ(core->contiguous_slot(), 1U); +} + +TEST_F(TreeCoreReplay, ProposalRejectsBeforeArmingWhenWaiterRegistrationIsOccupied) +{ + std::shared_ptr<TreeApplyOperation> first; + std::shared_ptr<TreeApplyOperation> second; + const CoreWriteReader empty = [](size_t) { return CoreWriteInput{}; }; + const auto first_status = core->prepare_apply(0, empty, detail::BudgetClass::kAdmittedWriter, 0, &first); + ASSERT_TRUE(first_status.ok()) << first_status.to_string(); + EXPECT_EQ(core->prepare_apply(0, empty, detail::BudgetClass::kAdmittedWriter, 0, &second).code(), + Code::kUnavailable); + EXPECT_EQ(core->contiguous_slot(), 0U); + first.reset(); + ASSERT_TRUE(core->prepare_apply(0, empty, detail::BudgetClass::kAdmittedWriter, 0, &second).ok()); +} + +TEST_F(TreeCoreReplay, ReplacementRejectsWithoutClosingTheMissingSlotAdmission) +{ + std::shared_ptr<TreeApplyOperation> operation; + ASSERT_TRUE(prepare(2, "second", "value", &operation).ok()); + TestReplayCompletion completed; + operation->start(2, &TestReplayCompletion::complete, &completed); + core->binding()->lane()->drain(); + const auto generation = core->capture()->generation; + EXPECT_EQ(core->clear().code(), Code::kUnavailable); + EXPECT_EQ(core->capture()->generation, generation); + EXPECT_FALSE(completed.done.load(std::memory_order_acquire)); + ASSERT_TRUE(apply(1, "first", "value").ok()); + completed.wait(); + ASSERT_TRUE(completed.status.ok()) << completed.status.to_string(); + core->binding()->lane()->drain(); + EXPECT_EQ(core->contiguous_slot(), 2U); + ASSERT_TRUE(core->clear().ok()); + EXPECT_EQ(core->capture()->generation, generation + 1); +} + +TEST_F(TreeCoreReplay, ReplacementRejectsAnArmedProposalUntilItsOwnerCompletes) +{ + std::shared_ptr<TreeApplyOperation> operation; + const CoreWriteReader empty = [](size_t) { return CoreWriteInput{}; }; + ASSERT_TRUE(core->prepare_apply(0, empty, detail::BudgetClass::kAdmittedWriter, 0, &operation).ok()); + ASSERT_TRUE(operation->arm().ok()); + EXPECT_EQ(core->clear().code(), Code::kUnavailable); + TestReplayCompletion completed; + operation->start(1, &TestReplayCompletion::complete, &completed); + completed.wait(); + ASSERT_TRUE(completed.status.ok()); + core->binding()->lane()->drain(); + ASSERT_TRUE(core->clear().ok()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_retirement_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_retirement_test.cpp new file mode 100644 index 000000000..9ecf2616e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_retirement_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/retirement_queue.h" +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeCoreRetirement, PublishedOwnersRemainValidAfterTreeAndExecutorDestruction) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + std::weak_ptr<TreeRuntime> runtime = config.runtime; + auto tree = std::make_unique<Crowdbtree>(config); + auto *core = TreeCoreAccess::get(*tree); + auto account = core->binding()->account(); + for (size_t index = 0; index < 1024; ++index) { + ASSERT_TRUE(tree->put(std::to_string(index), "retained value").ok()); + } + auto view = core->capture(); + GetView value; + ASSERT_TRUE(core->get("512", &value).ok()); + ASSERT_TRUE(value.found()); + tree.reset(); + EXPECT_EQ(value.value().to_string(), "retained value"); + EXPECT_NE(view->sources->tables.back()->find("512"), nullptr); + config.runtime.reset(); + EXPECT_TRUE(runtime.expired()); + view.reset(); + value = {}; + account->drain_retirements(); + const auto after = account->snapshot(); + for (size_t i = 0; i < detail::ResourceBudget::kClasses; ++i) { + EXPECT_EQ(after.allocated[i], 0U) << i; + EXPECT_EQ(after.reserved[i], 0U) << i; + } + for (const auto physical : after.physical) { + EXPECT_EQ(physical, 0U); + } +} + +TEST(TreeCoreRetirement, OldPublicationRootsAreCollectedWhileTreeRemainsOpen) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + auto old = core->capture(); + std::weak_ptr<const PublishedTreeState> retired = old; + old.reset(); + ASSERT_TRUE(tree.put("key", "value").ok()); + const auto flushed = tree.flush(); + ASSERT_TRUE(flushed.ok()) << flushed.to_string(); + core->binding()->drain(); + EXPECT_TRUE(retired.expired()); + const auto work = core->binding()->lifecycle_budget()->retirement_queue()->snapshot(); + EXPECT_GT(work.enqueued, 0U); + EXPECT_EQ(work.enqueued, work.freed); + EXPECT_LE(work.max_step_nodes, detail::RetirementQueue::kNodesPerStep); + const auto totals = core->publisher().work_metrics()->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)]; + EXPECT_GT(totals.attempts, 0U); + EXPECT_EQ(totals.inflight, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_source_stats_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_source_stats_test.cpp new file mode 100644 index 000000000..637c4e87b --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_source_stats_test.cpp @@ -0,0 +1,134 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +#include <sstream> + +namespace crowdb::tree +{ +namespace +{ +class TreeCoreSourceStats : public testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions host; + host.workers = 2; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.buffer_pool_bytes = 4 * 1024 * 1024; + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + } + + static void expect_current(TreeCore &core) + { + SourceMemoryStats expected{.known = true}; + for (const auto &table : core.capture()->sources->tables) { + expected += table->source_stats(); + } + EXPECT_EQ(core.source_stats(), expected); + } + + Config config; + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; +}; + +TEST_F(TreeCoreSourceStats, FlushAndGenerationReplacementKeepOldOwnersOutOfCurrentSummary) +{ + EXPECT_TRUE(core->source_stats().known); + EXPECT_EQ(core->source_stats().tables, 1U); + EXPECT_EQ(core->source_stats().records, 0U); + ASSERT_TRUE(tree->put("a", "one").ok()); + ASSERT_TRUE(tree->put("b", "two").ok()); + ASSERT_TRUE(tree->put("a", "new").ok()); + expect_current(*core); + const auto previous = core->capture(); + ASSERT_TRUE(tree->flush().ok()); + expect_current(*core); + EXPECT_EQ(core->source_stats().records, 0U); + EXPECT_EQ(previous->sources->tables.front()->count(), 2U); + ASSERT_TRUE(tree->put("c", "three").ok()); + const auto before_clear = core->capture(); + ASSERT_TRUE(tree->clear().ok()); + EXPECT_NE(core->capture()->sources->memory, before_clear->sources->memory); + expect_current(*core); + EXPECT_EQ(core->source_stats().records, 0U); + EXPECT_EQ(before_clear->sources->memory->snapshot().records, 1U); + ASSERT_TRUE(tree->put("d", "new-generation").ok()); + EXPECT_EQ(core->source_stats().records, 1U); + EXPECT_EQ(before_clear->sources->memory->snapshot().records, 1U); +} + +TEST_F(TreeCoreSourceStats, InheritedFrozenSummaryTracksOnlyItsFixedParentPrefix) +{ + Crowdbtree child(config); + auto *child_core = TreeCoreAccess::get(child); + ASSERT_TRUE(tree->put("a", "parent-a").ok()); + ASSERT_TRUE(tree->put("b", "parent-b").ok()); + auto status = child.install_split_memtable_overlay(*tree, 2); + ASSERT_TRUE(status.ok()) << status.to_string(); + expect_current(*child_core); + EXPECT_EQ(child_core->source_stats().records, 2U); + ASSERT_TRUE(child.put("a", "child-a").ok()); + ASSERT_TRUE(tree->put("c", "parent-c").ok()); + EXPECT_EQ(child_core->source_stats().records, 3U); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(tree->begin_split_memtable_view(&generation, &frontier).ok()); + status = tree->publish_split_memtable_view(generation, 2, child, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + expect_current(*child_core); + EXPECT_EQ(child_core->source_stats().records, 1U); + ASSERT_TRUE(child.clear_split_memtable_overlay(*tree).ok()); + ASSERT_TRUE(tree->release_split_memtable_view(generation).ok()); + ASSERT_TRUE(child.flush().ok()); + expect_current(*child_core); + EXPECT_EQ(child_core->source_stats().records, 0U); +} + +TEST_F(TreeCoreSourceStats, RegisteredGaugesFollowCurrentSourcesWhileOldReadersRemainAlive) +{ + tree->init_metrics("source", "mem"); + ASSERT_TRUE(tree->put("a", "first").ok()); + ASSERT_TRUE(tree->put("a", "second").ok()); + const auto held = core->capture(); + auto expect_gauges = [&] { + const auto stats = core->source_stats(); + const auto output = tree->flush_metrics_str(1.0, "now", 0); + const std::pair<const char *, uint64_t> expected[] = { + {"source.mt.summary.known.g", 1 }, + {"source.mt.tables.g", stats.tables }, + {"source.mt.records.g", stats.records }, + {"source.mt.frozen.g", stats.frozen_tables }, + {"source.mt.writers.g", stats.writers }, + {"source.mt.freezing.writers.g", stats.freezing_writers }, + {"source.mt.frozen.retained.bytes.g", stats.frozen_retained_bytes }, + {"source.mt.history.live.g", stats.memory.history_count }, + {"source.mt.history.bytes.g", stats.memory.history_bytes }, + {"source.mt.descriptor.bytes.g", stats.memory.descriptor_bytes}, + {"source.mt.node.bytes.g", stats.memory.node_bytes }, + {"source.mt.payload.bytes.g", stats.memory.payload_bytes } + }; + for (const auto &[name, value] : expected) { + const auto at = output.find(name); + ASSERT_NE(at, std::string::npos) << name; + std::istringstream input(output.substr(at + std::char_traits<char>::length(name))); + uint64_t actual = UINT64_MAX; + ASSERT_TRUE(static_cast<bool>(input >> actual)) << name; + EXPECT_EQ(actual, value) << name; + } + }; + expect_gauges(); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(held->sources->tables.front()->count(), 1U); + EXPECT_EQ(core->source_stats().records, 0U); + expect_gauges(); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_split_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_split_test.cpp new file mode 100644 index 000000000..8486e4e27 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_split_test.cpp @@ -0,0 +1,465 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "btree/tree_core_access.h" +#include "local_checkpoint_test_support.h" + +#include <gtest/gtest.h> + +#include <future> + +namespace crowdb::tree +{ +namespace +{ +class TreeCoreSplit : public testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions options; + options.workers = 1; + ASSERT_TRUE(TreeRuntime::create(options, &config.runtime).ok()); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + config.buffer_pool_bytes = 4 * 1024 * 1024; + parent = std::make_unique<Crowdbtree>(config); + } + + std::unique_ptr<Crowdbtree> child(KeyRange range = KeyRange::unbounded()) + { + auto options = config; + options.key_range = std::move(range); + return std::make_unique<Crowdbtree>(options); + } + + static Status put(Crowdbtree &tree, uint64_t slot, std::string key, std::string value) + { + return tree.apply(slot, + Batch{.ops = {{.key = std::move(key), .kind = OpKind::kPut, .value = std::move(value)}}}); + } + + Config config; + std::unique_ptr<Crowdbtree> parent; +}; + +TEST_F(TreeCoreSplit, ContinuingParentKeepsChildPrefixAndIndependentFutureWrites) +{ + auto destination = child(); + ASSERT_TRUE(put(*parent, 1, "x", "inherited").ok()); + ASSERT_TRUE(put(*parent, 2, "a", "apple").ok()); + auto status = destination->install_split_memtable_overlay(*parent, 2); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(destination->last_applied_slot(), 0U); + EXPECT_EQ(destination->contiguous_slot(), 2U); + EXPECT_EQ(destination->get_view("x").value().to_string(), "inherited"); + EXPECT_FALSE(destination->flush().ok()); + EXPECT_FALSE(destination->clear_split_memtable_overlay(*parent).ok()); + ASSERT_TRUE(put(*destination, 3, "x", "child").ok()); + ASSERT_TRUE(put(*parent, 3, "a", "banana").ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + status = parent->begin_split_memtable_view(&generation, &frontier); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(frontier, 3U); + status = parent->publish_split_memtable_view(generation, 2, *destination, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(destination->last_applied_slot(), 2U); + EXPECT_EQ(destination->contiguous_slot(), 3U); + EXPECT_EQ(destination->get_view("x").value().to_string(), "child"); + EXPECT_EQ(destination->get_view("a").value().to_string(), "apple"); + ASSERT_TRUE(destination->clear_split_memtable_overlay(*parent).ok()); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + ASSERT_TRUE(destination->flush().ok()); + EXPECT_EQ(destination->last_applied_slot(), 3U); + ASSERT_TRUE(parent->flush().ok()); + EXPECT_EQ(parent->get_view("a").value().to_string(), "banana"); +} + +TEST_F(TreeCoreSplit, TwoChildrenPublishOnlyTheirHalfOpenRanges) +{ + const auto left_range = KeyRange::bounded({}, "m"); + const auto right_range = KeyRange::bounded("m", {}); + auto left = child(left_range); + auto right = child(right_range); + ASSERT_TRUE(put(*parent, 1, "a", "left").ok()); + ASSERT_TRUE(put(*parent, 2, "m", "right").ok()); + ASSERT_TRUE(left->install_split_memtable_overlay(*parent, 2).ok()); + ASSERT_TRUE(right->install_split_memtable_overlay(*parent, 2).ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + EXPECT_EQ(frontier, 2U); + auto status = parent->publish_split_memtable_view(generation, 2, *left, left_range); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = parent->publish_split_memtable_view(generation, 2, *right, right_range); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(left->get_view("a").value().to_string(), "left"); + EXPECT_FALSE(left->get_view("m").found()); + EXPECT_EQ(right->get_view("m").value().to_string(), "right"); + EXPECT_FALSE(right->get_view("a").found()); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + ASSERT_TRUE(parent->flush().ok()); +} + +TEST_F(TreeCoreSplit, InheritedVersionsAndReverseScanSurviveSourceDestruction) +{ + auto destination = child(); + ASSERT_TRUE(put(*parent, 4, "x", "future-parent").ok()); + ASSERT_TRUE(put(*parent, 1, "x", "inherited").ok()); + ASSERT_TRUE(put(*parent, 2, "a", "first").ok()); + ASSERT_TRUE(destination->install_split_memtable_overlay(*parent, 2).ok()); + parent.reset(); + EXPECT_EQ(destination->get_view("x").value().to_string(), "inherited"); + for (bool reverse : {false, true}) { + std::vector<scan_entry> entries; + bool truncated = false; + const auto status = + TreeCoreAccess::get(*destination) + ->scan({.prefix = {}, .start = {}, .end = {}, .begin = {}, .limit = 8, .reverse = reverse}, &entries, + &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), 2U); + EXPECT_FALSE(truncated); + EXPECT_EQ(entries[reverse ? 0 : 1].key, "x"); + EXPECT_EQ(entries[reverse ? 0 : 1].slot, 1U); + } +} + +TEST_F(TreeCoreSplit, InvalidSessionAndRangeLeaveDestinationUntouched) +{ + auto destination = child(KeyRange::bounded("a", "z")); + ASSERT_TRUE(put(*parent, 1, "x", "value").ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + EXPECT_FALSE( + parent->publish_split_memtable_view(generation + 1, 1, *destination, KeyRange::bounded("a", "z")).ok()); + EXPECT_FALSE(parent->publish_split_memtable_view(generation, 1, *destination, KeyRange::bounded("a", "m")).ok()); + EXPECT_EQ(destination->contiguous_slot(), 0U); + EXPECT_FALSE(TreeCoreAccess::get(*destination)->capture()->sources->inheritance); + EXPECT_FALSE(parent->release_split_memtable_view(generation + 1).ok()); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); +} + +TEST_F(TreeCoreSplit, ExistingDestinationCannotCreditAMissingParentPrefix) +{ + auto destination = child(); + ASSERT_TRUE(put(*destination, 1, "own", "destination").ok()); + ASSERT_TRUE(destination->flush().ok()); + const auto before = TreeCoreAccess::get(*destination)->capture(); + ASSERT_TRUE(put(*parent, 1, "old", "committed-parent").ok()); + ASSERT_TRUE(put(*parent, 2, "other", "second-parent").ok()); + ASSERT_TRUE(parent->flush().ok()); + ASSERT_TRUE(put(*parent, 3, "new", "source-only").ok()); + const auto status = destination->install_split_memtable_overlay(*parent, 3); + EXPECT_EQ(status.code(), Code::kInvalidArgument) << status.to_string(); + EXPECT_EQ(TreeCoreAccess::get(*destination)->capture(), before); + EXPECT_EQ(destination->contiguous_slot(), 1U); + EXPECT_EQ(destination->get_view("own").value().to_string(), "destination"); + EXPECT_FALSE(destination->get_view("old").found()); +} + +TEST_F(TreeCoreSplit, EmptyChildrenInheritTheFixedL1AndCapturedL0Prefix) +{ + ASSERT_TRUE(put(*parent, 1, "a", "base-left").ok()); + ASSERT_TRUE(put(*parent, 2, "z", "base-right").ok()); + ASSERT_TRUE(parent->flush().ok()); + ASSERT_TRUE(put(*parent, 3, "a", "captured-left").ok()); + ASSERT_TRUE(put(*parent, 4, "n", "captured-right").ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + ASSERT_EQ(frontier, 4U); + ASSERT_TRUE(put(*parent, 5, "z", "future-parent").ok()); + ASSERT_TRUE(parent->flush().ok()); + ASSERT_TRUE(parent->flush().ok()); + EXPECT_EQ(parent->get_view("z").value().to_string(), "future-parent"); + const auto left_range = KeyRange::bounded({}, "m"); + const auto right_range = KeyRange::bounded("m", {}); + auto left = child(left_range); + auto right = child(right_range); + auto status = parent->publish_split_memtable_view(generation, frontier, *left, left_range); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = parent->publish_split_memtable_view(generation, frontier, *right, right_range); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(left->get_view("a").value().to_string(), "captured-left"); + EXPECT_FALSE(left->get_view("z").found()); + EXPECT_EQ(right->get_view("z").value().to_string(), "base-right"); + EXPECT_EQ(right->get_view("n").value().to_string(), "captured-right"); + EXPECT_FALSE(right->get_view("a").found()); + EXPECT_EQ(left->last_applied_slot(), frontier); + EXPECT_EQ(right->contiguous_slot(), frontier); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + parent.reset(); + ASSERT_TRUE(put(*right, 5, "z", "child").ok()); + status = right->flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(right->get_view("z").value().to_string(), "child"); + EXPECT_EQ(left->get_view("a").value().to_string(), "captured-left"); +} + +TEST_F(TreeCoreSplit, FullyFlushedPrefixInstallsIntoAnEmptyChildWithoutAnL0Interval) +{ + ASSERT_TRUE(put(*parent, 1, "x", "base").ok()); + ASSERT_TRUE(parent->flush().ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + auto destination = child(); + auto status = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(destination->get_view("x").value().to_string(), "base"); + EXPECT_EQ(destination->last_applied_slot(), frontier); + EXPECT_FALSE(TreeCoreAccess::get(*destination)->capture()->sources->inheritance); + const auto installed = TreeCoreAccess::get(*destination)->capture(); + ASSERT_TRUE(parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()).ok()); + EXPECT_EQ(TreeCoreAccess::get(*destination)->capture(), installed); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); +} + +TEST_F(TreeCoreSplit, MultiWindowBaselineCopiesEveryRecordAtTheFixedCoverage) +{ + constexpr uint64_t count = 513; + for (uint64_t slot = 1; slot <= count; ++slot) { + ASSERT_TRUE(put(*parent, slot, "key-" + std::to_string(slot), std::string(80, 'v')).ok()); + } + ASSERT_TRUE(parent->flush().ok()); + EXPECT_GT(TreeCoreAccess::get(*parent)->capture()->stats.page_count(), 1U); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + auto destination = child(); + const auto status = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + parent.reset(); + std::vector<scan_entry> entries; + bool truncated = false; + ASSERT_TRUE(destination->scan({}, {}, {}, 0, 0, false, 0, &entries, &truncated).ok()); + ASSERT_EQ(entries.size(), count); + EXPECT_FALSE(truncated); + for (const auto &entry : entries) { + EXPECT_EQ(entry.key, "key-" + std::to_string(entry.slot)); + EXPECT_EQ(entry.value, std::string(80, 'v')); + } + EXPECT_EQ(destination->last_applied_slot(), count); +} + +TEST_F(TreeCoreSplit, BaselineQuotaFailureLeavesTheDestinationUnchangedAndCanRetry) +{ + ASSERT_TRUE(put(*parent, 1, "x", "base").ok()); + ASSERT_TRUE(parent->flush().ok()); + ASSERT_TRUE(put(*parent, 2, "y", "captured").ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + for (const auto category : {detail::BudgetClass::kReclaim, detail::BudgetClass::kHandoff}) { + auto destination = child(); + auto *core = TreeCoreAccess::get(*destination); + const auto before = core->capture(); + const auto account = core->binding()->account(); + const auto sample = account->snapshot(); + const auto index = static_cast<size_t>(category); + const auto used = sample.allocated[index] + sample.reserved[index]; + const auto limit = category == detail::BudgetClass::kReclaim ? core->binding()->options().limits.reclaim_bytes + : core->binding()->options().limits.handoff_bytes; + auto held = detail::ResourceBudget::try_reserve(account, category, limit - used); + ASSERT_TRUE(held.has_value()); + auto status = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + EXPECT_EQ(status.code(), Code::kResourceExhausted) << status.to_string(); + EXPECT_EQ(core->capture(), before); + EXPECT_EQ(destination->contiguous_slot(), 0U); + EXPECT_FALSE(destination->get_view("x").found()); + EXPECT_FALSE(destination->get_view("y").found()); + held.reset(); + status = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(destination->get_view("x").value().to_string(), "base"); + EXPECT_EQ(destination->get_view("y").value().to_string(), "captured"); + } + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); +} + +TEST_F(TreeCoreSplit, ColdOverflowBaselineBecomesIndependentOfTheParentBackend) +{ + auto device = std::make_shared<MemPageStore>(4096); + auto backend = std::make_shared<test_write::TestCheckpointStore>(); + backend->device = device; + auto options = config; + options.page_store = device.get(); + options.async_page_store = backend.get(); + options.backend_owner = backend; + options.max_inline_value = 64; + parent.reset(); + ASSERT_TRUE(Crowdbtree::open(options, &parent).ok()); + const std::string value(12000, 'v'); + ASSERT_TRUE(put(*parent, 1, "x", value).ok()); + ASSERT_TRUE(parent->snapshot().ok()); + // Checkpoint rebase already publishes a reloadable cold base. + { + const auto view = TreeCoreAccess::get(*parent)->capture(); + const auto leaf = view->mapping.get(view->root_page_id)->leaf; + ASSERT_EQ(leaf->bases.size(), 1U); + EXPECT_FALSE(leaf->bases.front().resident); + ASSERT_TRUE(leaf->bases.front().backing); + EXPECT_TRUE(leaf->bases.front().backing->reloadable()); + } + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + auto destination = child(); + const auto status = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + parent.reset(); + std::weak_ptr<test_write::TestCheckpointStore> old_backend = backend; + options.backend_owner.reset(); + backend.reset(); + device.reset(); + EXPECT_TRUE(old_backend.expired()); + EXPECT_EQ(destination->get_view("x").value().to_string(), value); + EXPECT_EQ(destination->last_applied_slot(), 1U); +} + +TEST_F(TreeCoreSplit, CompletedPrefixCanBeRetriedWithoutDuplicateHandoff) +{ + auto destination = child(); + ASSERT_TRUE(put(*parent, 1, "x", "value").ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + auto status = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto sequence = TreeCoreAccess::get(*destination)->capture()->commit_seq; + ASSERT_TRUE(parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()).ok()); + EXPECT_EQ(TreeCoreAccess::get(*destination)->capture()->commit_seq, sequence); + ASSERT_TRUE(destination->clear_split_memtable_overlay(*parent).ok()); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); +} + +TEST_F(TreeCoreSplit, PartialCoverageFailureRetriesTheSameBatchAndKeepsChildWrites) +{ + auto destination = child(); + ASSERT_TRUE(put(*parent, 1, "x", "inherited").ok()); + ASSERT_TRUE(destination->install_split_memtable_overlay(*parent, 1).ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + auto &publisher = TreeCoreAccess::get(*destination)->publisher(); + size_t publications = 0; + publisher.set_before_root_for_tests([&] { + if (++publications == 3) { + throw std::bad_alloc(); + } + }); + const auto failed = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + publisher.set_before_root_for_tests({}); + EXPECT_FALSE(failed.ok()); + EXPECT_EQ(publications, 3U); + EXPECT_EQ(destination->last_applied_slot(), 0U); + EXPECT_TRUE(TreeCoreAccess::get(*destination)->capture()->pending_handoff); + EXPECT_EQ(destination->get_view("x").value().to_string(), "inherited"); + ASSERT_TRUE(put(*destination, 2, "x", "child").ok()); + const auto status = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(destination->last_applied_slot(), 1U); + EXPECT_EQ(destination->get_view("x").value().to_string(), "child"); + ASSERT_TRUE(destination->clear_split_memtable_overlay(*parent).ok()); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + ASSERT_TRUE(destination->flush().ok()); +} + +TEST_F(TreeCoreSplit, UnwrittenEmptyPrefixNeedsNoInheritedSourceOrCoverageMutation) +{ + auto destination = child(); + uint64_t generation = 0; + uint64_t frontier = 123; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + EXPECT_EQ(frontier, 0U); + const auto sequence = TreeCoreAccess::get(*destination)->capture()->commit_seq; + const auto status = parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(TreeCoreAccess::get(*destination)->capture()->commit_seq, sequence); + EXPECT_FALSE(TreeCoreAccess::get(*destination)->capture()->sources->inheritance); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + ASSERT_TRUE(put(*destination, 1, "x", "first").ok()); +} + +TEST_F(TreeCoreSplit, ChildWriterCanCompleteWhileInheritedCoverageIsPublishing) +{ + auto destination = child(); + ASSERT_TRUE(put(*parent, 1, "x", "inherited").ok()); + ASSERT_TRUE(destination->install_split_memtable_overlay(*parent, 1).ok()); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + std::promise<void> coverage; + std::promise<void> resume; + auto boundary = coverage.get_future(); + auto resumed = resume.get_future(); + auto &publisher = TreeCoreAccess::get(*destination)->publisher(); + size_t publications = 0; + publisher.set_before_root_for_tests([&] { + if (++publications == 3) { + coverage.set_value(); + resumed.wait(); + } + }); + auto published = std::async(std::launch::async, [&] { + return parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + }); + boundary.wait(); + const auto written = put(*destination, 2, "x", "concurrent-child"); + resume.set_value(); + const auto status = published.get(); + publisher.set_before_root_for_tests({}); + ASSERT_TRUE(written.ok()) << written.to_string(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(destination->last_applied_slot(), 1U); + EXPECT_EQ(destination->contiguous_slot(), 2U); + EXPECT_EQ(destination->get_view("x").value().to_string(), "concurrent-child"); + ASSERT_TRUE(destination->clear_split_memtable_overlay(*parent).ok()); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + ASSERT_TRUE(destination->flush().ok()); +} + +TEST_F(TreeCoreSplit, ParentPartialHandoffRemainsCapturedUntilItsNormalRetry) +{ + auto destination = child(); + ASSERT_TRUE(put(*parent, 1, "x", "inherited").ok()); + auto &publisher = TreeCoreAccess::get(*parent)->publisher(); + size_t publications = 0; + publisher.set_before_root_for_tests([&] { + if (++publications == 5) { + throw std::bad_alloc(); + } + }); + const auto failed = parent->flush(); + publisher.set_before_root_for_tests({}); + ASSERT_FALSE(failed.ok()); + const auto repairing_metrics = publisher.work_metrics()->totals(); + EXPECT_EQ(repairing_metrics[static_cast<size_t>(TreeStage::FlushRepair)].inflight, 1U); + EXPECT_EQ(repairing_metrics[static_cast<size_t>(TreeStage::FlushRepair)].count, 0U); + ASSERT_TRUE(TreeCoreAccess::get(*parent)->capture()->pending_handoff); + const auto sequence = TreeCoreAccess::get(*parent)->capture()->commit_seq; + const auto installed = destination->install_split_memtable_overlay(*parent, 1); + ASSERT_TRUE(installed.ok()) << installed.to_string(); + EXPECT_EQ(TreeCoreAccess::get(*parent)->capture()->commit_seq, sequence); + EXPECT_EQ(parent->last_applied_slot(), 0U); + uint64_t generation = 0; + uint64_t frontier = 0; + ASSERT_TRUE(parent->begin_split_memtable_view(&generation, &frontier).ok()); + const auto published = + parent->publish_split_memtable_view(generation, frontier, *destination, KeyRange::unbounded()); + ASSERT_TRUE(published.ok()) << published.to_string(); + ASSERT_TRUE(parent->release_split_memtable_view(generation).ok()); + ASSERT_TRUE(parent->flush().ok()); + EXPECT_EQ(parent->last_applied_slot(), 1U); + EXPECT_EQ(destination->get_view("x").value().to_string(), "inherited"); + const auto repaired_metrics = publisher.work_metrics()->totals(); + EXPECT_EQ(repaired_metrics[static_cast<size_t>(TreeStage::FlushRepair)].inflight, 0U); + EXPECT_EQ(repaired_metrics[static_cast<size_t>(TreeStage::FlushRepair)].count, 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_core_sync_backend_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_sync_backend_test.cpp new file mode 100644 index 000000000..94d98e71e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_core_sync_backend_test.cpp @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "crowdb-tree/backend/page_store.h" +#include "snapshot/published_snapshot.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeCoreSyncBackend, OwnedSynchronousBackendCheckpointsReopensAndOutlivesCore) +{ + Config config; + TreeRuntimeOptions runtime_options; + runtime_options.workers = 1; + runtime_options.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(runtime_options, &config.runtime).ok()); + auto store = std::make_shared<MemPageStore>(4096); + config.page_store = store.get(); + config.backend_owner = store; + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.runtime_binding.limits.io_operations = 1; + std::shared_ptr<TreeCore> core; + auto status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string large(12000, 'v'); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = large}}}).ok()); + status = core->checkpoint(); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(core->capture_snapshot(&snapshot).ok()); + EXPECT_TRUE(snapshot->detached()); + core.reset(); + GetView old; + status = snapshot->get("a", &old); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(old.found()); + EXPECT_EQ(old.value().to_string(), large); + snapshot.reset(); + status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + GetView recovered; + status = core->get("a", &recovered); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(recovered.found()); + EXPECT_EQ(recovered.value().to_string(), large); + EXPECT_EQ(core->capture()->coverage, 1U); +} + +TEST(TreeCoreSyncBackend, RawSynchronousSnapshotCannotOutliveItsBackendDomain) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + MemPageStore store(4096); + config.page_store = &store; + config.frame_bytes = 4096; + std::shared_ptr<TreeCore> core; + auto status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = "value"}}}).ok()); + ASSERT_TRUE(core->checkpoint().ok()); + std::shared_ptr<PublishedSnapshotContext> snapshot; + ASSERT_TRUE(core->capture_snapshot(&snapshot).ok()); + EXPECT_FALSE(snapshot->detached()); + GetView value; + ASSERT_TRUE(snapshot->get("a", &value).ok()); + core.reset(); + EXPECT_EQ(snapshot->get("a", &value).code(), Code::kUnavailable); + status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_generation_lease_test.cpp b/lib/crowdb-tree/tests/integration/tree_generation_lease_test.cpp new file mode 100644 index 000000000..f189d746e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_generation_lease_test.cpp @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "btree/tree_generation_lease.h" + +#include <gtest/gtest.h> + +#include <semaphore> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +class PreparedGenerationLease : public testing::Test +{ + protected: + void SetUp() override + { + TreeRuntimeOptions options; + options.workers = 1; + ASSERT_TRUE(TreeRuntime::create(options, &config.runtime).ok()); + tree = std::make_unique<Crowdbtree>(config); + core = TreeCoreAccess::get(*tree); + } + + Config config; + std::unique_ptr<Crowdbtree> tree; + TreeCore *core = nullptr; +}; + +TEST_F(PreparedGenerationLease, CancelReopensTheUnchangedSourcesAndRoot) +{ + ASSERT_TRUE(tree->put("old", "retained").ok()); + ASSERT_TRUE(tree->flush().ok()); + const auto original = core->capture(); + std::shared_ptr<TreeGenerationLease> lease; + ASSERT_TRUE(TreeGenerationLease::prepare(*core, {}, nullptr, &lease).ok()); + ASSERT_TRUE(lease->acquire().ok()); + EXPECT_EQ(core->capture(), original); + EXPECT_EQ(tree->put("blocked", "value").code(), Code::kUnavailable); + lease->cancel(); + lease.reset(); + EXPECT_EQ(core->capture(), original); + EXPECT_EQ(tree->get_view("old").value().to_string(), "retained"); + ASSERT_TRUE(tree->put("next", "accepted").ok()); + ASSERT_TRUE(tree->flush().ok()); + EXPECT_EQ(tree->get_view("next").value().to_string(), "accepted"); +} + +TEST_F(PreparedGenerationLease, CommitDoesNotWaitForCpuOrReadmitStorage) +{ + ASSERT_TRUE(tree->put("old", "retained").ok()); + auto borrowed = tree->get_view("old"); + const auto generation = core->capture()->generation; + std::shared_ptr<TreeGenerationLease> lease; + ASSERT_TRUE(TreeGenerationLease::prepare(*core, {}, nullptr, &lease).ok()); + ASSERT_TRUE(lease->acquire().ok()); + + struct BlockedWorker + { + std::binary_semaphore entered{0}; + std::binary_semaphore release{0}; + } blocked; + + ASSERT_TRUE(core->binding() + ->cpu() + ->try_submit( + core->binding()->lane(), + [](void *context) noexcept { + auto &worker = *static_cast<BlockedWorker *>(context); + worker.entered.release(); + worker.release.acquire(); + }, + &blocked) + .ok()); + blocked.entered.acquire(); + config.runtime->stop_admission(); + // A new caller thread has no existing publication Epoch participant. The + // prepared commit must not attempt a capture or register one here. + Status installed; + std::jthread caller([&] { installed = lease->commit(); }); + caller.join(); + blocked.release.release(); + ASSERT_TRUE(installed.ok()) << installed.to_string(); + EXPECT_EQ(core->capture()->generation, generation + 1); + EXPECT_EQ(core->capture()->coverage, 0U); + EXPECT_EQ(core->contiguous_slot(), 0U); + EXPECT_EQ(borrowed.value().to_string(), "retained"); +} + +TEST_F(PreparedGenerationLease, CancelBeforeAcquireDoesNotFenceTheTree) +{ + const auto original = core->capture(); + std::shared_ptr<TreeGenerationLease> lease; + ASSERT_TRUE(TreeGenerationLease::prepare(*core, {}, nullptr, &lease).ok()); + lease.reset(); + EXPECT_EQ(core->capture(), original); + ASSERT_TRUE(tree->put("key", "value").ok()); + EXPECT_EQ(tree->get_view("key").slot(), 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_host_progress_test.cpp b/lib/crowdb-tree/tests/integration/tree_host_progress_test.cpp new file mode 100644 index 000000000..d0bea71a9 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_host_progress_test.cpp @@ -0,0 +1,184 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/local_backend_state.h" +#include "btree/tree_apply_operation.h" +#include "btree/tree_core.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +#include <array> +#include <future> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct TestHostInput +{ + std::string key; + std::string value; +}; + +struct TestHostCompletion +{ + std::promise<Status> result; + std::future<Status> future = result.get_future(); + + static void complete(void *context, detail::CallbackSignal signal, Status status) noexcept + { + auto &self = *static_cast<TestHostCompletion *>(context); + self.result.set_value(signal == detail::CallbackSignal::kStopped ? Status::unavailable() : std::move(status)); + } +}; + +struct TestHostBackendGate +{ + ApplyDispatcher::Node node{&TestHostBackendGate::run, this}; + std::promise<void> entered; + std::atomic<bool> released{false}; + + static void run(void *context) noexcept + { + auto &gate = *static_cast<TestHostBackendGate *>(context); + gate.entered.set_value(); + while (!gate.released.load(std::memory_order_acquire)) { + gate.released.wait(false); + } + } + + void release() + { + released.store(true, std::memory_order_release); + released.notify_all(); + } +}; + +struct TestHostRelease +{ + std::shared_ptr<TestHostBackendGate> gate; + + ~TestHostRelease() + { + gate->release(); + } +}; + +Status prepare(TreeCore &core, uint64_t slot, std::string key, std::string value, + std::shared_ptr<TreeApplyOperation> *out) +{ + auto input = std::make_shared<TestHostInput>(std::move(key), std::move(value)); + return core.prepare_replay( + slot, 1, + [raw = input.get()](size_t) { return CoreWriteInput{.key = Slice(raw->key), .value = Slice(raw->value)}; }, + input, 0, out); +} + +Status apply(TreeCore &core, uint64_t slot, std::string key, std::string value) +{ + std::shared_ptr<TreeApplyOperation> operation; + auto status = prepare(core, slot, std::move(key), std::move(value), &operation); + if (!status.ok()) { + return status; + } + TestHostCompletion completed; + operation->start(slot, &TestHostCompletion::complete, &completed); + operation.reset(); + return completed.future.get(); +} + +TEST(TreeHostProgress, OneCpuPreservesOtherTreeProgressDuringSlowChosenWritebackAndGapRepair) +{ + tree_test::TempDir directory("shared_host_pressure_"); + TreeRuntimeOptions options; + options.workers = 1; + options.backend_workers = 1; + options.max_waiters_per_class = 4; + options.limits.gap_repair_bytes = 2 * 1024 * 1024; + options.limits.io_operations = 2; + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create(options, &runtime).ok()); + std::array<std::unique_ptr<BlockPageStore>, 2> stores; + std::array<std::shared_ptr<TreeCore>, 2> cores; + for (size_t index = 0; index < cores.size(); ++index) { + std::filesystem::create_directories(directory.path + "/" + std::to_string(index)); + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path + "/" + std::to_string(index), 1, index + 1, 64 * 1024, + 4096, &stores[index]) + .ok()); + Config config; + config.runtime = runtime; + config.page_store = stores[index].get(); + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.leaf_split_bytes = 3000; + config.runtime_binding.cpu_queue_limit = 4; + config.runtime_binding.limits.io_operations = 1; + config.runtime_binding.limits.gap_repair_bytes = 1024 * 1024; + auto status = TreeCore::open(config, &cores[index]); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + auto &slow = *cores[0]; + auto &fast = *cores[1]; + const std::string value(256 * 1024, 'v'); + ASSERT_TRUE(apply(slow, 1, "a", value).ok()); + ASSERT_TRUE(apply(slow, 2, "b", value).ok()); + + auto gate = std::make_shared<TestHostBackendGate>(); + TestHostRelease release{gate}; + auto entered = gate->entered.get_future(); + slow.binding()->backend_progress()->request(gate->node, gate); + ASSERT_EQ(entered.wait_for(std::chrono::seconds(2)), std::future_status::ready); + std::array<std::shared_ptr<TreeApplyOperation>, 4> operations; + std::array<TestHostCompletion, 4> completed; + // A durable later chosen value waits for its preceding gap without taking + // the gap payload reserve. The missing predecessor drives bounded writeback. + ASSERT_TRUE(prepare(slow, 4, "d", value, operations.data()).ok()); + operations[0]->start(4, &TestHostCompletion::complete, completed.data()); + ASSERT_TRUE(prepare(slow, 3, "c", value, &operations[1]).ok()); + operations[1]->start(3, &TestHostCompletion::complete, &completed[1]); + + std::promise<void> cpu_terminal; + auto cpu_drained = cpu_terminal.get_future(); + std::jthread cpu_wait([&] { + slow.binding()->lane()->drain(); + cpu_terminal.set_value(); + }); + const auto cpu_status = cpu_drained.wait_for(std::chrono::seconds(2)); + // The other Tree shares the one CPU and host budget, but its ready prefix + // and independent gap must still complete while slow storage is stopped. + ASSERT_TRUE(prepare(fast, 2, "second", "two", &operations[2]).ok()); + operations[2]->start(2, &TestHostCompletion::complete, &completed[2]); + ASSERT_TRUE(prepare(fast, 1, "first", "one", &operations[3]).ok()); + operations[3]->start(1, &TestHostCompletion::complete, &completed[3]); + const auto fast_status = completed[2].future.wait_for(std::chrono::seconds(2)); + const auto pending = completed[1].future.wait_for(std::chrono::milliseconds(0)); + const auto local = slow.binding()->account()->snapshot(); + gate->release(); + cpu_wait.join(); + EXPECT_EQ(cpu_status, std::future_status::ready); + EXPECT_EQ(fast_status, std::future_status::ready); + EXPECT_EQ(pending, std::future_status::timeout); + const auto gap = static_cast<size_t>(detail::BudgetClass::kGapRepair); + const auto io = static_cast<size_t>(detail::BudgetClass::kIo); + EXPECT_LE(local.allocated[gap] + local.reserved[gap], 1024U * 1024U); + EXPECT_LE(local.allocated[io] + local.reserved[io], 1U); + for (auto &completion : completed) { + ASSERT_EQ(completion.future.wait_for(std::chrono::seconds(2)), std::future_status::ready); + const auto status = completion.future.get(); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + operations = {}; + EXPECT_EQ(slow.contiguous_slot(), 4U); + EXPECT_EQ(fast.contiguous_slot(), 2U); + EXPECT_EQ(slow.durable_coverage(), 0U); + EXPECT_EQ(fast.durable_coverage(), 0U); + GetView result; + ASSERT_TRUE(fast.get("second", &result).ok()); + EXPECT_EQ(result.value().to_string(), "two"); + EXPECT_GT(slow.publisher().work_metrics()->totals()[static_cast<size_t>(TreeStage::PageWritebackPipeline)].count, + 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_point_read_bound_test.cpp b/lib/crowdb-tree/tests/integration/tree_point_read_bound_test.cpp new file mode 100644 index 000000000..6f3bcc207 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_point_read_bound_test.cpp @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "btree/tree_core_read_internal.h" +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +#include <array> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreePointReadBound, SixteenTreesRetainPointWindowsOnOneDefaultHost) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.runtime_binding.limits.routing_bytes = 16ULL * 1024 * 1024; + std::array<std::unique_ptr<Crowdbtree>, 16> trees; + std::array<core_read_detail::ReadPreparation, 16> reads; + for (size_t index = 0; index < trees.size(); ++index) { + trees[index] = std::make_unique<Crowdbtree>(config); + ASSERT_TRUE(trees[index]->put("key", "value").ok()); + auto *core = TreeCoreAccess::get(*trees[index]); + auto status = core_read_detail::prepare(*core, false, 256, 0, &reads[index]); + ASSERT_TRUE(status.ok()) << index << ": " << status.to_string(); + } + for (const auto &tree : trees) { + GetView value; + const auto status = TreeCoreAccess::get(*tree)->get("key", &value); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(value.value().to_string(), "value"); + } + reads = {}; +} + +TEST(TreePointReadBound, MaximumL0ValueBorrowsPayloadWithSmallRoutingAllowance) +{ + Config config; + config.runtime_binding.limits.routing_bytes = 16ULL * 1024 * 1024; + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + ASSERT_EQ(core->load_limits().max_value_bytes, 64ULL * 1024 * 1024); + ASSERT_TRUE(tree.put("key", std::string(core->load_limits().max_value_bytes, 'v')).ok()); + GetView value; + const auto status = core->get("key", &value); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(value.value().size(), core->load_limits().max_value_bytes); + EXPECT_EQ(value.value().data()[value.value().size() - 1], 'v'); + core->close_and_drain(); + EXPECT_EQ(value.value().data()[0], 'v'); +} + +TEST(TreePointReadBound, ColdKnownValueBoundsReadWhileUnknownSummaryRemainsConservative) +{ + auto store = std::make_shared<MemPageStore>(4096); + Config config; + config.page_store = store.get(); + config.backend_owner = store; + config.frame_bytes = 4096; + config.max_inline_value = 64; + config.runtime_binding.limits.routing_bytes = 32ULL * 1024 * 1024; + std::unique_ptr<Crowdbtree> tree; + auto status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + const std::string large(20000, 'v'); + ASSERT_TRUE(tree->put("key", large).ok()); + status = tree->snapshot(); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto *core = TreeCoreAccess::get(*tree); + auto view = core->capture(); + ASSERT_EQ(view->stats.kv_unknown_pages, 0U); + ASSERT_GT(view->mapping.copy_bounds().max_leaf_bases, 0U); + (void)tree->evict_clean_leaves(0); + GetView value; + status = core->get("key", &value); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(value.value().to_string(), large); + auto unknown = std::make_shared<PublishedTreeState>(*view); + unknown->stats.kv_unknown_pages = 1; + unknown->stats.kv_known_bytes = 0; + core_read_detail::ReadPreparation prepared; + status = core_read_detail::prepare(*core, std::move(unknown), core->read_limits(), false, 256, 0, &prepared); + EXPECT_EQ(status.code(), Code::kResourceExhausted); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_retirement_observation_test.cpp b/lib/crowdb-tree/tests/integration/tree_retirement_observation_test.cpp new file mode 100644 index 000000000..6ce46140e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_retirement_observation_test.cpp @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeRetirementObservation, LogicalReplacementStartsBeforeOldViewIsReleased) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + auto old = core->capture(); + ASSERT_TRUE(tree.put("key", "value").ok()); + const auto status = tree.flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + core->binding()->drain(); + const auto work = core->binding()->work_metrics(); + const auto resources = core->binding()->resource_metrics(); + const auto held = work->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)]; + EXPECT_GT(held.inflight, 0U); + const auto held_age = work->ages(tree_metrics_now_ns())[static_cast<size_t>(TreeStage::MemoryRetireWait)]; + EXPECT_EQ(held_age.inflight, held.inflight); + EXPECT_EQ(held_age.tracked, held.inflight); + EXPECT_TRUE(held_age.complete); + EXPECT_GT(held_age.oldest_age_ns, 0U); + EXPECT_GT(resources->snapshot().retired_bytes, 0U); + old.reset(); + core->binding()->drain(); + const auto released = work->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)]; + EXPECT_EQ(released.inflight, 0U); + const auto released_age = work->ages(tree_metrics_now_ns())[static_cast<size_t>(TreeStage::MemoryRetireWait)]; + EXPECT_EQ(released_age.tracked, 0U); + EXPECT_EQ(released_age.untracked, 0U); + EXPECT_EQ(released_age.oldest_age_ns, 0U); + EXPECT_TRUE(released_age.complete); + EXPECT_GT(released.count, held.count); + EXPECT_EQ(resources->snapshot().retired_bytes, 0U); +} + +TEST(TreeRetirementObservation, ResourceMetricsOwnerHasNoWeakControlCycle) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + const auto baseline = runtime->snapshot(); + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + auto account = binding->account(); + auto resources = binding->resource_metrics(); + EXPECT_EQ(account->resource_metrics(), resources); + binding.reset(); + EXPECT_GT(account->snapshot().allocated[1], 0U); + resources.reset(); + EXPECT_EQ(account->snapshot().allocated[1], 0U); + account.reset(); + EXPECT_EQ(runtime->snapshot().allocated.routing_bytes, baseline.allocated.routing_bytes); + EXPECT_EQ(runtime->snapshot().reserved.routing_bytes, baseline.reserved.routing_bytes); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_scan_total_window_test.cpp b/lib/crowdb-tree/tests/integration/tree_scan_total_window_test.cpp new file mode 100644 index 000000000..3fb68f7de --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_scan_total_window_test.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +#include <array> + +namespace crowdb::tree +{ +namespace +{ +std::string key(size_t index) +{ + std::array<char, 32> text{}; + snprintf(text.data(), text.size(), "key%04zu", index); + return text.data(); +} + +TEST(TreeScanTotalWindow, UnlimitedAndExplicitCountsSpanWindowsInBothDirections) +{ + Crowdbtree tree; + for (size_t index = 0; index < 513; ++index) { + ASSERT_TRUE(tree.put(key(index), "value").ok()); + } + auto *core = TreeCoreAccess::get(tree); + for (const bool materialized : {false, true}) { + if (materialized) { + ASSERT_TRUE(tree.flush().ok()); + } + for (const size_t limit : {size_t{0}, size_t{129}, size_t{257}}) { + for (const bool reverse : {false, true}) { + CoreScanOptions options; + options.limit = limit; + options.reverse = reverse; + std::vector<scan_entry> entries; + bool truncated = false; + const auto status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), limit == 0 ? 513 : limit); + EXPECT_EQ(truncated, limit != 0); + for (size_t index = 0; index < entries.size(); ++index) { + EXPECT_EQ(entries[index].key, key(reverse ? 512 - index : index)); + EXPECT_EQ(entries[index].value, "value"); + } + } + } + } +} + +TEST(TreeScanTotalWindow, SmallWorkWindowsRespectThePublicByteBudget) +{ + Config config; + TreeRuntimeOptions host; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + // Read admission pays for reachable decoded references. Packed output + // independently claims each actual capacity before allocation. + config.runtime_binding.limits.routing_bytes = 192ULL * 1024 * 1024; + Crowdbtree tree(config); + for (size_t index = 0; index < 300; ++index) { + ASSERT_TRUE(tree.put(key(index), "value").ok()); + } + CoreScanOptions options; + options.byte_budget = 25; + std::vector<scan_entry> entries; + bool truncated = false; + auto *core = TreeCoreAccess::get(tree); + auto status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), 2U); + EXPECT_TRUE(truncated); + options.byte_budget = 1; + status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), 1U); + EXPECT_TRUE(truncated); + options.strict_byte_budget = true; + status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(entries.empty()); + EXPECT_TRUE(truncated); +} + +TEST(TreeScanTotalWindow, MaximumValueRemainsValidAsAnOversizedFirstResult) +{ + Config config; + config.runtime_binding.limits.routing_bytes = 192ULL * 1024 * 1024; + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + const auto maximum = core->load_limits().max_value_bytes; + ASSERT_EQ(maximum, 64ULL * 1024 * 1024); + ASSERT_TRUE(tree.put("large", std::string(maximum, 'v')).ok()); + CoreScanOptions options; + options.byte_budget = 1; + bool truncated = false; + ScanPackedBuf packed; + size_t count = 0; + const auto status = core->scan(options, nullptr, &truncated, &packed, &count); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(count, 1U); + EXPECT_EQ(packed.size(), maximum + 5 + 17); + EXPECT_EQ(packed.data()[packed.size() - 1], 'v'); + EXPECT_FALSE(truncated); + packed = {}; + options.strict_byte_budget = true; + const auto strict_status = core->scan(options, nullptr, &truncated, &packed, &count); + ASSERT_TRUE(strict_status.ok()) << strict_status.to_string(); + EXPECT_EQ(count, 0U); + EXPECT_EQ(packed.size(), 0U); + EXPECT_TRUE(truncated); +} + +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_source_controls_test.cpp b/lib/crowdb-tree/tests/integration/tree_source_controls_test.cpp new file mode 100644 index 000000000..602f21180 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_source_controls_test.cpp @@ -0,0 +1,190 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +Batch put_batch(std::string key) +{ + Batch batch; + batch.ops.push_back({.key = std::move(key), .kind = OpKind::kPut, .value = "value"}); + return batch; +} + +TEST(TreeSourceControls, EntryThresholdAndFiniteCountHandoffAllowRepeatedBatches) +{ + Config config; + config.memtable_flush_entries = 2; + config.memtable_flush_bytes = SIZE_MAX; + config.max_memtable_count = 3; + Crowdbtree tree(config); + ASSERT_TRUE(tree.put("a", "value").ok()); + EXPECT_EQ(tree.frozen_table_count(), 0U); + ASSERT_TRUE(tree.put("b", "value").ok()); + EXPECT_EQ(tree.frozen_table_count(), 1U); + for (size_t i = 0; i < 30; ++i) { + const auto status = tree.put("key" + std::to_string(i), "value"); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_LT(tree.frozen_table_count(), config.max_memtable_count); + } + EXPECT_GT(TreeCoreAccess::get(tree)->capture()->coverage, 0U); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_EQ(tree.frozen_table_count(), 0U); + for (size_t i = 0; i < 30; ++i) { + EXPECT_TRUE(tree.get("key" + std::to_string(i), nullptr, nullptr)); + } + ASSERT_TRUE(tree.put("after-maintenance", "value").ok()); + EXPECT_TRUE(tree.get("after-maintenance", nullptr, nullptr)); +} + +TEST(TreeSourceControls, ByteThresholdAndGenerationReplacementKeepIndependentControls) +{ + Config config; + config.memtable_flush_bytes = 1; + config.memtable_flush_entries = UINT32_MAX; + config.max_memtable_count = 4; + Crowdbtree tree(config); + ASSERT_TRUE(tree.put("before", "value").ok()); + EXPECT_EQ(tree.frozen_table_count(), 1U); + ASSERT_TRUE(tree.clear().ok()); + EXPECT_EQ(tree.frozen_table_count(), 0U); + EXPECT_FALSE(tree.get("before", nullptr, nullptr)); + ASSERT_TRUE(tree.put("after", "value").ok()); + EXPECT_EQ(tree.frozen_table_count(), 1U); + EXPECT_TRUE(tree.get("after", nullptr, nullptr)); +} + +TEST(TreeSourceControls, MissingPrefixKeepsGapAdmissionAndResumesFiniteDrain) +{ + Config config; + config.memtable_flush_entries = 1; + config.memtable_flush_bytes = SIZE_MAX; + config.max_memtable_count = 2; + Crowdbtree tree(config); + ASSERT_TRUE(tree.apply(2, put_batch("two")).ok()); + ASSERT_TRUE(tree.apply(3, put_batch("three")).ok()); + EXPECT_EQ(TreeCoreAccess::get(tree)->capture()->coverage, 0U); + EXPECT_TRUE(tree.get("two", nullptr, nullptr)); + EXPECT_TRUE(tree.get("three", nullptr, nullptr)); + const auto status = tree.apply(1, put_batch("one")); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_EQ(TreeCoreAccess::get(tree)->capture()->coverage, 3U); + EXPECT_TRUE(tree.get("one", nullptr, nullptr)); + ASSERT_TRUE(tree.apply(4, put_batch("four")).ok()); +} + +TEST(TreeSourceControls, SingleWorkerReadCallbackCanWriteAcrossTheSoftThreshold) +{ + Config config; + TreeRuntimeOptions runtime; + runtime.workers = 1; + runtime.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(runtime, &config.runtime).ok()); + config.memtable_flush_entries = 2; + config.memtable_flush_bytes = SIZE_MAX; + config.max_memtable_count = 4; + Crowdbtree tree(config); + ASSERT_TRUE(tree.put("seed", "value").ok()); + std::atomic<bool> done{false}; + Status written; + tree.get_async("seed", [&](Status status, GetView value) { + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(value.found()); + EXPECT_TRUE(detail::FlushRuntime::on_worker_thread()); + written = tree.put("callback", "value"); + done.store(true, std::memory_order_release); + done.notify_all(); + }); + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + ASSERT_TRUE(written.ok()) << written.to_string(); + TreeCoreAccess::get(tree)->binding()->drain(); + EXPECT_EQ(tree.frozen_table_count(), 1U); + EXPECT_TRUE(tree.get("callback", nullptr, nullptr)); + EXPECT_FALSE(detail::FlushRuntime::on_worker_thread()); + + struct BackendWrite + { + Crowdbtree &tree; + std::atomic<bool> done{false}; + Status status; + + static void run(void *context) noexcept + { + auto &call = *static_cast<BackendWrite *>(context); + EXPECT_TRUE(detail::FlushRuntime::on_worker_thread()); + call.status = call.tree.put("backend-one", "value"); + if (call.status.ok()) { + call.status = call.tree.put("backend-two", "value"); + } + call.done.store(true, std::memory_order_release); + call.done.notify_all(); + } + } call{.tree = tree, .done = false, .status = {}}; + + const auto &binding = TreeCoreAccess::get(tree)->binding(); + ASSERT_TRUE(binding->backend_cpu()->try_submit(binding->backend_lane(), &BackendWrite::run, &call).ok()); + while (!call.done.load(std::memory_order_acquire)) { + call.done.wait(false, std::memory_order_acquire); + } + ASSERT_TRUE(call.status.ok()) << call.status.to_string(); + binding->drain(); + EXPECT_TRUE(tree.get("backend-two", nullptr, nullptr)); +} + +TEST(TreeSourceControls, RejectedPublicationLeavesTheCommittedWriteInItsOriginalSource) +{ + Config config; + config.memtable_flush_entries = 1; + Crowdbtree tree(config); + TreeCoreAccess::get(tree)->publisher().close(); + ASSERT_TRUE(tree.put("first", "value").ok()); + EXPECT_EQ(tree.frozen_table_count(), 0U); + ASSERT_TRUE(tree.put("second", "value").ok()); + EXPECT_TRUE(tree.get("first", nullptr, nullptr)); + EXPECT_TRUE(tree.get("second", nullptr, nullptr)); +} + +TEST(TreeSourceControls, CloseDrainsRotationQueuedByTheLastAcceptedCallback) +{ + Config config; + TreeRuntimeOptions runtime; + runtime.workers = 1; + ASSERT_TRUE(TreeRuntime::create(runtime, &config.runtime).ok()); + config.memtable_flush_entries = 2; + auto tree = std::make_unique<Crowdbtree>(config); + ASSERT_TRUE(tree->put("seed", "value").ok()); + std::atomic<bool> ready{false}; + std::atomic<bool> release{false}; + Status written; + tree->get_async("seed", [&](Status status, GetView value) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(value.found()); + written = tree->put("last", "value"); + ready.store(true, std::memory_order_release); + ready.notify_all(); + while (!release.load(std::memory_order_acquire)) { + release.wait(false, std::memory_order_acquire); + } + }); + while (!ready.load(std::memory_order_acquire)) { + ready.wait(false, std::memory_order_acquire); + } + ASSERT_TRUE(written.ok()) << written.to_string(); + std::thread close([&] { tree.reset(); }); + release.store(true, std::memory_order_release); + release.notify_all(); + close.join(); + EXPECT_FALSE(tree); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_source_window_test.cpp b/lib/crowdb-tree/tests/integration/tree_source_window_test.cpp new file mode 100644 index 000000000..9f52cc38f --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_source_window_test.cpp @@ -0,0 +1,177 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/crowdb-tree.h" + +#include <gtest/gtest.h> + +#include <latch> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeSourceWindow, LargeConfiguredCountMakesContinuousProgressWithinTheReadWindow) +{ + Config config; + config.memtable_flush_entries = 32; + config.memtable_flush_bytes = SIZE_MAX; + config.max_memtable_count = 1024; + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + for (size_t index = 0; index < 4096; ++index) { + const auto status = tree.put("key-" + std::to_string(index), "value"); + ASSERT_TRUE(status.ok()) << index << ": " << status.to_string(); + ASSERT_LE(core->capture()->sources->tables.size(), core->read_limits().max_memtables - 1); + } + EXPECT_EQ(core->config().max_memtable_count, 1024U); + EXPECT_GT(core->capture()->coverage, 0U); + auto status = tree.flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + for (size_t index = 0; index < 4096; ++index) { + GetView value; + status = core->get("key-" + std::to_string(index), &value); + ASSERT_TRUE(status.ok()) << index << ": " << status.to_string(); + ASSERT_EQ(value.value().to_string(), "value"); + } + ASSERT_TRUE(tree.put("after-drain", "next").ok()); +} + +TEST(TreeSourceWindow, FiveThousandSequentialVersionsFlushWithDefaultQuotas) +{ + Crowdbtree tree(Config{}); + auto *core = TreeCoreAccess::get(tree); + for (uint64_t slot = 1; slot <= 5000; ++slot) { + Batch batch; + batch.ops.push_back({.key = "key", .kind = OpKind::kPut, .value = std::to_string(slot)}); + const auto status = tree.apply(slot, batch); + ASSERT_TRUE(status.ok()) << "slot " << slot << ": " << status.to_string(); + // Source-window coverage requires each previous write's physical + // retirement to finish before admitting the next bounded allocation. + core->binding()->account()->drain_retirements(); + } + const auto status = tree.flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree.contiguous_slot(), 5000U); + EXPECT_EQ(tree.get_view("key").value().to_string(), "5000"); +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(TreeSourceWindow, WriterCreditReturnsOnlyAfterPhysicalRetirement) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.runtime_binding.limits.writer_bytes = 2 * 1024 * 1024; + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + const auto binding = core->binding(); + binding->drain(); + + struct Gate + { + std::latch entered{1}, release{1}; + } gate; + + auto unblock = [binding](Gate *gate) { + gate->release.count_down(); + binding->drain(); + }; + std::unique_ptr<Gate, decltype(unblock)> cleanup(&gate, unblock); + ASSERT_TRUE(binding->cpu() + ->try_submit( + binding->lane(), + [](void *context) noexcept { + auto &gate = *static_cast<Gate *>(context); + gate.entered.count_down(); + gate.release.wait(); + }, + &gate) + .ok()); + gate.entered.wait(); + uint64_t rejected = 0; + for (uint64_t slot = 1; slot <= 128; ++slot) { + Batch batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = std::to_string(slot)}}}; + const auto status = tree.apply(slot, batch); + if (!status.ok()) { + EXPECT_EQ(status.code(), Code::kUnavailable) << status.to_string(); + rejected = slot; + break; + } + } + ASSERT_GT(rejected, 1U); + EXPECT_EQ(tree.contiguous_slot(), rejected - 1); + EXPECT_GT(binding->account()->retirements_pending_for_tests(), 0U); + constexpr auto category = static_cast<size_t>(detail::BudgetClass::kAdmittedWriter); + const auto held = binding->account()->snapshot().allocated[category]; + EXPECT_GT(held, config.runtime_binding.limits.writer_bytes / 2); + cleanup.reset(); + binding->account()->drain_retirements(); + EXPECT_LT(binding->account()->snapshot().allocated[category], held / 2); + EXPECT_EQ(tree.get_view("key").value().to_string(), std::to_string(rejected - 1)); + const Batch next{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "continued"}}}; + ASSERT_TRUE(tree.apply(rejected, next).ok()); + EXPECT_EQ(tree.contiguous_slot(), rejected); + EXPECT_EQ(tree.get_view("key").value().to_string(), "continued"); +} +#endif + +TEST(TreeSourceWindow, CapturedPrefixLeavesRoomForChildReplayAndSuccessor) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + config.memtable_flush_entries = 1; + config.memtable_flush_bytes = SIZE_MAX; + config.max_memtable_count = 1024; + Crowdbtree source(config); + for (size_t index = 0; index < 61; ++index) { + ASSERT_TRUE(source.put("key-" + std::to_string(index), "value").ok()); + } + ASSERT_EQ(TreeCoreAccess::get(source)->capture()->sources->tables.size(), 62U); + uint64_t generation = 0; + uint64_t frontier = 0; + auto status = source.begin_split_memtable_view(&generation, &frontier); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(frontier, 61U); + EXPECT_EQ(TreeCoreAccess::get(source)->capture()->sources->tables.size(), 63U); + Crowdbtree child(config); + status = child.install_split_memtable_overlay(source, frontier); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto *core = TreeCoreAccess::get(child); + ASSERT_EQ(core->capture()->sources->tables.size(), 63U); + std::atomic<bool> done{false}; + Status written; + child.get_async("key-0", [&](Status read, GetView value) { + EXPECT_TRUE(read.ok()) << read.to_string(); + EXPECT_TRUE(value.found()); + written = child.put("replayed", "child"); + done.store(true, std::memory_order_release); + done.notify_one(); + }); + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + ASSERT_TRUE(written.ok()) << written.to_string(); + core->binding()->drain(); + EXPECT_LE(core->capture()->sources->tables.size(), core->read_limits().max_memtables); + status = source.publish_split_memtable_view(generation, frontier, child, KeyRange::unbounded()); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(child.clear_split_memtable_overlay(source).ok()); + ASSERT_TRUE(source.release_split_memtable_view(generation).ok()); + status = child.flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(child.contiguous_slot(), 62U); + for (size_t index = 0; index < 61; ++index) { + GetView value; + status = core->get("key-" + std::to_string(index), &value); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(value.value().to_string(), "value"); + } + EXPECT_EQ(child.get_view("replayed").value().to_string(), "child"); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_state_reconcile_test.cpp b/lib/crowdb-tree/tests/integration/tree_state_reconcile_test.cpp new file mode 100644 index 000000000..1332273fa --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_state_reconcile_test.cpp @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "btree/tree_state_reconcile.h" +#include "published_read_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +std::shared_ptr<PublishedTreeState> sparse_state() +{ + auto state = std::make_shared<PublishedTreeState>(); + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(cold(7)); + auto entry = published(leaf); + entry->metrics.dirty = false; + const std::array<uint64_t, 3> ids{0, uint64_t{1} << 29, MappingView::kPageLimit - 1}; + std::vector<PageReplacement> changes; + for (auto pid : ids) { + changes.push_back({.page_id = pid, .expected = {}, .replacement = entry}); + EXPECT_TRUE(apply_page_metrics(state->stats, nullptr, &entry->metrics)); + } + MappingView::CopyStats copies; + state->mapping = state->mapping.replace(changes, copies); + state->root_page_id = 0; + state->stats.height = 1; + return state; +} + +TEST(TreeStateReconciliation, SparseColdMappingYieldsAndSkipsUnallocatedSubtrees) +{ + auto state = sparse_state(); + TreeStateReconciliation audit(state); + for (size_t steps = 0; steps < 1024 && !audit.complete(); ++steps) { + const auto before = audit.visited_pages(); + const auto status = audit.step(1); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_LE(audit.visited_pages() - before, 1U); + } + ASSERT_TRUE(audit.complete()); + EXPECT_EQ(audit.visited_pages(), 3U); + EXPECT_EQ(audit.recomputed().kv_unknown_pages, 3U); + EXPECT_EQ(audit.recomputed().dirty_pages, 0U); +} + +TEST(TreeStateReconciliation, AggregateCorruptionAndInvalidWorkBoundAreReported) +{ + auto state = sparse_state(); + ++state->stats.dirty_pages; + TreeStateReconciliation audit(state); + EXPECT_EQ(audit.step(0).code(), Code::kInvalidArgument); + EXPECT_FALSE(audit.complete()); + Status status; + for (size_t i = 0; i < 1024 && !audit.complete(); ++i) { + status = audit.step(1); + } + ASSERT_TRUE(audit.complete()); + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_EQ(audit.step(1).code(), Code::kCorruption); +} + +TEST(TreeStateReconciliation, OverlayDescriptorsAreCheckedAgainstTheActualRange) +{ + auto logical = page( + { + {.key = "a", .value = "one"}, + {.key = "z", .value = "two"} + }, + {.lower = "m"}); + auto entry = published(logical); + entry->metrics.overlay_runs = 1; + entry->metrics.overlay_entries = 2; // Only z belongs to this logical range. + auto state = std::make_shared<PublishedTreeState>(); + MappingView::CopyStats copies; + const std::array<PageReplacement, 1> changes{{{.page_id = 0, .expected = {}, .replacement = entry}}}; + state->mapping = state->mapping.replace(changes, copies); + ASSERT_TRUE(apply_page_metrics(state->stats, nullptr, &entry->metrics)); + state->root_page_id = 0; + state->stats.height = 1; + TreeStateReconciliation audit(state); + ASSERT_TRUE(audit.step(1).ok()); + ASSERT_TRUE(audit.step(1).ok()); // Bounded inspection of one run. + EXPECT_FALSE(audit.complete()); + EXPECT_EQ(audit.step(1).code(), Code::kCorruption); +} + +TEST(TreeStateReconciliation, CapturedRuntimeGenerationsStayConsistentAcrossLaterFlushes) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + ASSERT_TRUE(tree.put("a", "first").ok()); + ASSERT_TRUE(tree.flush().ok()); + TreeStateReconciliation old(core->capture()); + ASSERT_TRUE(tree.put("b", "second").ok()); + ASSERT_TRUE(tree.flush().ok()); + TreeStateReconciliation current(core->capture()); + for (auto *audit : {&old, ¤t}) { + for (size_t i = 0; i < 1024 && !audit->complete(); ++i) { + const auto status = audit->step(4); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + EXPECT_TRUE(audit->complete()); + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_work_counters_test.cpp b/lib/crowdb-tree/tests/integration/tree_work_counters_test.cpp new file mode 100644 index 000000000..893ca0b9b --- /dev/null +++ b/lib/crowdb-tree/tests/integration/tree_work_counters_test.cpp @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/tree_work_metrics_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeWorkCountersIntegration, RegistryUsesPaidStorageBeyondTheBootstrapReserve) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + core->binding()->drain(); + const auto &bootstrap = core->binding()->lifecycle_budget(); + const auto bytes = bootstrap->available(); + ASSERT_TRUE(bootstrap->claim(bytes)); + auto release = [bytes](detail::AllocationReservation *owner) { owner->release(bytes); }; + std::unique_ptr<detail::AllocationReservation, decltype(release)> remaining(bootstrap.get(), release); + const auto available = bootstrap->available(); + const auto &account = core->binding()->account(); + const auto kind = static_cast<size_t>(AllocationKind::kMetadata); + const auto before = account->snapshot().physical[kind]; + ASSERT_NO_THROW(tree.init_metrics("paid-registry", "")); + EXPECT_GT(account->snapshot().physical[kind], before); + EXPECT_GE(bootstrap->available(), available); + remaining.reset(); + const auto output = tree.flush_metrics_str(1, "test"); + EXPECT_NE(output.find("paid-registry.work.flush.batch.attempts.c"), std::string::npos); + EXPECT_NE(output.find("paid-registry.memory.metadata.bytes.g"), std::string::npos); +} + +TEST(TreeWorkCountersIntegration, ActualFlushAndSplitExposeCommittedOutputsWithoutPayloadCopies) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + config.frame_bytes = 4096; + config.leaf_split_bytes = 3000; + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + const auto work = core->publisher().work_metrics(); + ASSERT_TRUE(tree.put("a", "old").ok()); + ASSERT_TRUE(tree.put("a", "new").ok()); + ASSERT_TRUE(tree.put("b", "two").ok()); + ASSERT_TRUE(tree.flush().ok()); + auto counters = work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Flush)]; + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::FlushWinningVersions)], 2U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::FlushCandidateVersions)], 2U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::FlushWinningKeyBytes)], 2U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::FlushWinningValueBytes)], 6U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::HandoffValueCopyBytes)], 0U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::SourceDetached)], 1U); + for (size_t i = 0; i < 100; ++i) { + ASSERT_TRUE(tree.put("key-" + std::to_string(i), std::string(96, 'x')).ok()); + } + const auto status = tree.flush(); + ASSERT_TRUE(status.ok()) << status.to_string(); + counters = work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Flush)]; + const auto splits = counters[static_cast<size_t>(TreeWorkCounter::SplitOperations)]; + EXPECT_GT(splits, 0U); + EXPECT_GE(counters[static_cast<size_t>(TreeWorkCounter::SplitOutputPages)], splits * 2); + EXPECT_EQ( + work->totals()[static_cast<size_t>(TreeStage::PageSplit)].outcomes[static_cast<size_t>(TreeOutcome::Success)], + splits); +} + +TEST(TreeWorkCountersIntegration, AdditiveCaptureRejectsInvalidHeaderWithoutDrainAndSurvivesClose) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_tree *tree = nullptr; + ct_options options{}; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + auto work = core->publisher().work_metrics(); + ct_work_metrics_collector *collector = nullptr; + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + ct_tree_work_snapshot_v1 window{}; + window.struct_size = sizeof(window); + window.version = 1; + ct_tree_work_ages_v1 ages{}; + ages.struct_size = sizeof(ages); + ages.version = 1; + ct_tree_work_counters_v1 counters{}; + counters.struct_size = sizeof(counters); + counters.version = 1; + ASSERT_EQ(ct_work_metrics_capture_v2(collector, &window, &ages, &counters), 0); + ASSERT_TRUE(c_api_tree(tree)->put("a", "value").ok()); + ASSERT_TRUE(c_api_tree(tree)->flush().ok()); + counters.version = 2; + const auto seq = window.window_sequence; + EXPECT_EQ(ct_work_metrics_capture_v2(collector, &window, &ages, &counters), + static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(window.window_sequence, seq); + counters.version = 1; + ASSERT_EQ(ct_work_counters_totals(tree, &counters), 0); + const auto index = static_cast<size_t>(TreeWorkCounter::FlushWinningValueBytes); + const auto cause = static_cast<size_t>(TreeWorkCause::Flush); + EXPECT_EQ(counters.values[cause][index], 5U); + ct_close(tree); + ct_runtime_free(runtime); + ASSERT_EQ(ct_work_metrics_capture_v2(collector, &window, &ages, &counters), 0); + EXPECT_EQ(window.window_sequence, seq + 1); + EXPECT_GT(window.stages[static_cast<size_t>(TreeStage::FlushBatch)].window_count, 0U); + EXPECT_EQ(counters.values[cause][index], 5U); + EXPECT_STREQ(ct_work_counter_name_v1(static_cast<uint32_t>(index)), "flush.winning.value.bytes"); + EXPECT_EQ(ct_work_counter_name_v1(CT_TREE_WORK_COUNTERS_V1), nullptr); + EXPECT_EQ(ct_work_cause_name_v1(CT_TREE_WORK_CAUSES_V1), nullptr); + ct_work_metrics_collector_free(collector); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/write_path_test.cpp b/lib/crowdb-tree/tests/integration/write_path_test.cpp index cf86865e1..916d5bec3 100644 --- a/lib/crowdb-tree/tests/integration/write_path_test.cpp +++ b/lib/crowdb-tree/tests/integration/write_path_test.cpp @@ -34,7 +34,7 @@ std::string get_or(Crowdbtree &t, const std::string &k, const std::string &dflt) } // namespace -TEST(WritePath, BasePagesLiveInBufferPool) +TEST(WritePath, LogicalPagesRetainPaidOwnersWithoutEncodingFrames) { Config opt; opt.max_delta_len = 1; // consolidate into base frames quickly @@ -47,13 +47,15 @@ TEST(WritePath, BasePagesLiveInBufferPool) ASSERT_TRUE(t.apply(s, put_one(key(i), "payload-" + std::to_string(i))).ok()); ASSERT_TRUE(t.flush().ok()); } - // The tree split into multiple leaves under one or more inner pages; every - // such base page is built into a pool frame (held resident by its page). + // Flush splits logical pages without encoding frames. Independent paid + // overlay owners supply values until writeback creates reloadable bases. ASSERT_GT(t.leaf_count(), 1U); - ASSERT_NE(t.buffer_pool(), nullptr); - auto s = t.buffer_pool()->stats(); - EXPECT_GE(s.used, t.leaf_count()); // at least one frame per live leaf - // Values remain correct when read straight out of the frames. + ASSERT_GT(t.leaf_count_atomic(), 0U); + const auto stats = t.tree_state_stats(); + // These short values are inline in the charged CellVersion metadata. + EXPECT_GT(stats.resources.metadata_bytes, 0U); + EXPECT_GT(stats.resources.allocated_bytes, 0U); + EXPECT_EQ(t.stats().buffer_pool_used, 0U); for (int i = 0; i < 60; ++i) { EXPECT_EQ(get_or(t, key(i), "?"), "payload-" + std::to_string(i)); } @@ -92,7 +94,7 @@ TEST(WritePath, WriteStormToBoundedKeySetKeepsDirtyMemoryBounded) // threshold crossing, not the unbounded-growth-with-write-count this // test is checking for). auto fixed_value = [](int round) { - std::array<char, 8> b{}; + std::array<char, 16> b{}; snprintf(b.data(), b.size(), "v%06d", round); return std::string(b.data()); }; @@ -102,7 +104,7 @@ TEST(WritePath, WriteStormToBoundedKeySetKeepsDirtyMemoryBounded) ASSERT_TRUE(t.apply(i + 1, put_one(key(i), fixed_value(0))).ok()); } ASSERT_TRUE(t.flush().ok()); - uint32_t dirty_after_initial_fill = t.buffer_pool()->stats().dirty; + uint32_t dirty_after_initial_fill = t.stats().buffer_pool_dirty; ASSERT_GT(dirty_after_initial_fill, 0U); // never snapshotted -- still all dirty ASSERT_EQ(t.leaf_count(), 1U); // sanity: this test wants exactly one leaf throughout @@ -120,7 +122,7 @@ TEST(WritePath, WriteStormToBoundedKeySetKeepsDirtyMemoryBounded) // Dirty frame count must not have grown with the number of writes -- // only with the number of *distinct* keys, which didn't change. - EXPECT_EQ(t.buffer_pool()->stats().dirty, dirty_after_initial_fill); + EXPECT_EQ(t.stats().buffer_pool_dirty, dirty_after_initial_fill); for (int i = 0; i < kKeys; ++i) { EXPECT_EQ(get_or(t, key(i), "?"), fixed_value(500)); @@ -179,7 +181,7 @@ TEST(WritePath, FlushOnlyContiguousPrefix) t.force_advance_slot(2); ASSERT_TRUE(t.flush().ok()); EXPECT_EQ(t.last_applied_slot(), 2U); - EXPECT_EQ(t.memtable_count(), 1U); // b@5 retained in L0 + EXPECT_EQ(t.memtable_count(), 2U); // retained gap source and fresh writable successor // Both still readable (a from L1, b from L0). EXPECT_EQ(get_or(t, "a", "?"), "A2"); EXPECT_EQ(get_or(t, "b", "?"), "B5"); diff --git a/lib/crowdb-tree/tests/load_budget_test_support.h b/lib/crowdb-tree/tests/load_budget_test_support.h new file mode 100644 index 000000000..56d13e02a --- /dev/null +++ b/lib/crowdb-tree/tests/load_budget_test_support.h @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_load.h" +#include "btree/tree_metrics.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::test_read +{ +struct LoadCredits +{ + LoadCredits() + { + account->set_resource_metrics(resources); + } + + std::shared_ptr<TreeResourceMetrics> resources = std::make_shared<TreeResourceMetrics>(); + + static constexpr size_t kBytes = static_cast<size_t>(detail::BudgetClass::kRouting); + static constexpr size_t kIo = static_cast<size_t>(detail::BudgetClass::kIo); + + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result{}; + result[kBytes] = 4 * 1024 * 1024; + result[kIo] = 1; + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + PageLoadBudget credits{.bytes = reserve(detail::BudgetClass::kRouting, limits()[kBytes]), + .io = reserve(detail::BudgetClass::kIo, limits()[kIo])}; + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> reserve(detail::BudgetClass category, + size_t amount) const + { + auto credit = detail::ResourceBudget::try_reserve(account, category, amount); + if (!credit) { + ADD_FAILURE() << "test load reservation failed"; + return {}; + } + return std::make_shared<detail::AllocationReservation>(std::move(credit.value())); + } + + void seal() const + { + credits.bytes->seal(); + credits.io->seal(); + } +}; +} // namespace crowdb::tree::test_read diff --git a/lib/crowdb-tree/tests/local_checkpoint_test_support.h b/lib/crowdb-tree/tests/local_checkpoint_test_support.h new file mode 100644 index 000000000..cbd003337 --- /dev/null +++ b/lib/crowdb-tree/tests/local_checkpoint_test_support.h @@ -0,0 +1,175 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "captured_write_test_support.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "crowdb-tree/crowdb-tree.h" +#include "snapshot/local_checkpoint.h" + +#include <atomic> + +namespace crowdb::tree::test_write +{ +class TestCheckpointStore final : public AsyncPageStore +{ + public: + std::shared_ptr<PageStore> device; + std::vector<std::string> events; + size_t defer_at = 0; + size_t fail_at = 0; + size_t throw_at = 0; + size_t inline_throw_at = 0; + AsyncCompletion pending; + std::atomic<bool> deferred{false}; + uint64_t address = 0; + const uint8_t *input = nullptr; + size_t bytes = 0; + bool barrier = false; + + [[nodiscard]] bool write_completion_is_reloadable() const noexcept override + { + return true; + } + + uint64_t submit_read(PageAddr at, void *out, size_t size, AsyncCompletion callback) override + { + auto *block = dynamic_cast<BlockPageStore *>(device.get()); + callback.complete(block != nullptr ? block->read_at(at, static_cast<uint8_t *>(out), size, callback.allocations) + : device->read_at(at, static_cast<uint8_t *>(out), size)); + return 0; + } + + uint64_t submit_write(PageAddr at, const void *data, size_t size, AsyncCompletion callback) override + { + EXPECT_EQ(reinterpret_cast<uintptr_t>(data) % 4096, 0U); + EXPECT_EQ(at % 4096, 0U); + EXPECT_EQ(size % 4096, 0U); + events.emplace_back(at < 8192 ? "anchor" : "metadata"); + address = at; + input = static_cast<const uint8_t *>(data); + bytes = size; + barrier = false; + accept(std::move(callback)); + return events.size(); + } + + Status submit_fsync(AsyncCompletion callback) override + { + events.emplace_back("barrier"); + barrier = true; + accept(std::move(callback)); + return Status::Ok(); + } + + void cancel(uint64_t /*id*/) override + { + } + + void accept(AsyncCompletion callback) + { + ASSERT_FALSE(pending); + pending = std::move(callback); + if (events.size() == throw_at) { + throw std::runtime_error("unknown acceptance"); + } + if (events.size() != defer_at) { + finish(); + } + else { + deferred.store(true, std::memory_order_release); + deferred.notify_all(); + } + if (events.size() == inline_throw_at) { + throw std::runtime_error("inline completion then failed submission"); + } + } + + void finish() + { + deferred.store(false, std::memory_order_release); + auto callback = std::exchange(pending, {}); + ASSERT_TRUE(callback); + auto status = Status::io_error(); + if (events.size() != fail_at) { + auto *block = dynamic_cast<BlockPageStore *>(device.get()); + if (block != nullptr) { + status = barrier ? block->sync(callback.allocations) + : block->write_at(address, input, bytes, callback.allocations); + } + else { + status = barrier ? device->sync() : device->write_at(address, input, bytes); + } + } + callback.complete(status); + } + + void wait_deferred() const + { + while (!deferred.load(std::memory_order_acquire)) { + deferred.wait(false, std::memory_order_acquire); + } + } +}; + +struct TestCheckpoint +{ + TestCapturedWrite source; + std::shared_ptr<TestCheckpointStore> store = std::make_shared<TestCheckpointStore>(); + std::shared_ptr<LocalCheckpointRegistry> registry; + std::shared_ptr<LocalCheckpointOperation> operation; + + TestCheckpoint() + { + store->device = source.store->device; + EXPECT_TRUE(LocalCheckpointRegistry::create(source.ledger, source.credits.credits.bytes, {}, ®istry).ok()); + } + + std::shared_ptr<const CapturedDurablePages> materialize(std::string value = "v", uint64_t coverage = 10) + { + auto captured = captured_leaf(page({ + {.key = "a", .value = std::move(value)} + })); + captured->coverage = coverage; + EXPECT_TRUE(source + .prepare(std::move(captured), {}, + {.frames = {.encode = {.frame_bytes = 4096, .inline_value_bytes = 16}}, + .write = {.max_frame_bytes = 4096}}) + .ok()); + Completion done; + source.run(&done); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + return source.operation->result().pages; + } + + Status prepare(std::shared_ptr<const CapturedDurablePages> pages = {}, LocalCheckpointLimits limits = {}) + { + if (!pages) { + pages = materialize(); + } + return LocalCheckpointOperation::create(std::move(pages), registry, store, source.runtime, source.lane, limits, + source.credits.credits, &operation); + } + + void run(Completion *done) + { + auto connected = stdexec::connect(operation->sender(), Receiver{.done = done}); + stdexec::start(connected); + source.runtime->drain(); + } + + void expect_read(const std::string &expected) + { + Config config; + config.page_store = store->device.get(); + config.frame_bytes = 4096; + std::unique_ptr<Crowdbtree> tree; + const auto status = Crowdbtree::open(config, &tree); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(tree->get(Slice("a"), &slot, &value)); + EXPECT_EQ(value, expected); + } +}; +} // namespace crowdb::tree::test_write diff --git a/lib/crowdb-tree/tests/local_recovery_test_support.h b/lib/crowdb-tree/tests/local_recovery_test_support.h new file mode 100644 index 000000000..5658fbd10 --- /dev/null +++ b/lib/crowdb-tree/tests/local_recovery_test_support.h @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "local_checkpoint_test_support.h" +#include "snapshot/local_recovery.h" + +namespace crowdb::tree::test_write +{ +class TestRecoveryStore final : public AsyncPageStore +{ + public: + std::shared_ptr<PageStore> device; + size_t reads = 0; + size_t defer_at = 0; + size_t fail_at = 0; + size_t throw_at = 0; + size_t inline_throw_at = 0; + AsyncCompletion pending; + AsyncCompletion retained_completion; + bool retain_completed = false; + PageAddr address = 0; + uint8_t *output = nullptr; + size_t bytes = 0; + + [[nodiscard]] bool write_completion_is_reloadable() const noexcept override + { + return true; + } + + uint64_t submit_read(PageAddr at, void *out, size_t size, AsyncCompletion callback) override + { + ++reads; + EXPECT_FALSE(pending); + EXPECT_EQ(reinterpret_cast<uintptr_t>(out) % 4096, 0U); + address = at; + output = static_cast<uint8_t *>(out); + bytes = size; + pending = std::move(callback); + if (reads == throw_at) { + throw std::runtime_error("unknown read acceptance"); + } + if (reads != defer_at) { + finish(); + } + if (reads == inline_throw_at) { + throw std::runtime_error("read completed before submission failure"); + } + return reads; + } + + uint64_t submit_write(PageAddr at, const void *data, size_t size, AsyncCompletion callback) override + { + callback.complete(device->write_at(at, static_cast<const uint8_t *>(data), size)); + return 0; + } + + Status submit_fsync(AsyncCompletion callback) override + { + auto status = device->sync(); + callback.complete(status); + return status; + } + + void cancel(uint64_t /*id*/) override + { + } + + void finish() + { + auto callback = std::exchange(pending, {}); + ASSERT_TRUE(callback); + if (retain_completed) { + retained_completion = callback; + } + callback.complete(reads == fail_at ? Status::io_error() : device->read_at(address, output, bytes)); + } +}; + +struct TestRecovery +{ + TestCheckpoint original; + LoadCredits credits; + std::shared_ptr<TestRecoveryStore> store = std::make_shared<TestRecoveryStore>(); + std::shared_ptr<LocalRecoveryOperation> operation; + snapshot::CommitAnchor anchor; + std::map<uint64_t, std::pair<uint64_t, uint64_t>> locations; + + TestRecovery() + { + store->device = original.store->device; + } + + void checkpoint(std::string value = "v") + { + ASSERT_TRUE(original.prepare(original.materialize(std::move(value))).ok()); + Completion done; + original.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + const auto image = original.operation->result().checkpoint; + anchor = image->anchor; + locations.clear(); + for (const auto &[pid, frame] : image->pages->frames) { + locations.emplace(pid, std::pair(frame.backing->address, frame.backing->length)); + } + } + + Status prepare(LocalRecoveryLimits limits = {}) + { + // Simulate process ownership ending before opening a new allocation + // incarnation. Only durable bytes and plain locations survive. + original.operation.reset(); + original.source.operation.reset(); + original.registry.reset(); + original.source.ledger.reset(); + return LocalRecoveryOperation::create(store, original.source.runtime, original.source.lane, + {.backend_id = 9, .generation = 2, .file_size = store->device->size()}, + limits, credits.credits, &operation); + } + + void run(Completion *done) + { + auto connected = stdexec::connect(operation->sender(), Receiver{.done = done}); + stdexec::start(connected); + original.source.runtime->drain(); + } +}; +} // namespace crowdb::tree::test_write diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_segment.h b/lib/crowdb-tree/tests/mapping_segment_test_support.h similarity index 54% rename from lib/crowdb-tree/include/crowdb-tree/maptable/mapping_segment.h rename to lib/crowdb-tree/tests/mapping_segment_test_support.h index 5f0d783e6..f4c164a5b 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_segment.h +++ b/lib/crowdb-tree/tests/mapping_segment_test_support.h @@ -1,25 +1,8 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// Standalone mapping-table segment struct (plan-tree #14a). -// -// Holds `slot_count` packed -// slot words (crowdb::tree::slot_word encoding, see mapping_slot.h) plus the -// per-segment bookkeeping segment recycling (#14b, done) and incremental -// persistence (#14c/#14d) need: a live-slot count (0 => recyclable), a -// generation counter (bumped when persisted, for image versioning), the -// image's last-persisted durable location, and a write-vs-persisted -// sequence pair that tells the snapshot path whether this segment has any -// change not yet captured in `image_addr` (see is_dirty()). -// -// `write_seq`/`persisted_seq` replace a plain dirty bool because -// prepare-then-commit snapshotting (persist.cpp) has a gap between -// capturing a segment's image and durably committing it, during which a -// concurrent write can dirty the segment again; a single bool can't tell -// "still exactly the state we imaged" apart from "changed again since". See -// mapping_table.h's commit_segment_persist(). -// -// Live in `MappingTable` (mapping_table.h). +// Mutable segment model retained only for low-level legacy mapping tests. +// Sequence checks model a write that races a prepared image commit. #pragma once #include <atomic> @@ -33,10 +16,9 @@ namespace crowdb::tree // Sentinel image_addr meaning "this segment has never been persisted". inline constexpr uint64_t kNoSegmentImageAddr = std::numeric_limits<uint64_t>::max(); -struct MappingSegment +struct TestMappingSegment { - explicit MappingSegment(uint32_t slot_count) - // NOLINTNEXTLINE(modernize-avoid-c-arrays) + explicit TestMappingSegment(uint32_t slot_count) : slots(std::make_unique<std::atomic<uint64_t>[]>(slot_count)), slot_count(slot_count) { @@ -45,8 +27,8 @@ struct MappingSegment } } - MappingSegment(const MappingSegment &) = delete; - MappingSegment &operator=(const MappingSegment &) = delete; + TestMappingSegment(const TestMappingSegment &) = delete; + TestMappingSegment &operator=(const TestMappingSegment &) = delete; // True if any slot has changed since the last successful persist. [[nodiscard]] bool is_dirty() const @@ -54,7 +36,6 @@ struct MappingSegment return write_seq.load(std::memory_order_relaxed) != persisted_seq.load(std::memory_order_relaxed); } - // NOLINTNEXTLINE(modernize-avoid-c-arrays) std::unique_ptr<std::atomic<uint64_t>[]> slots; // packed slot words (mapping_slot.h) uint32_t slot_count; std::atomic<uint32_t> live_count{0}; // non-empty slots; 0 => recyclable @@ -67,7 +48,7 @@ struct MappingSegment std::atomic<uint64_t> persisted_seq{0}; // Durable location of the image matching `persisted_seq`/`generation`. - // Writer-only bookkeeping (persist.cpp); no reader ever touches these. + // Writer-only bookkeeping in the test model. uint64_t image_addr = kNoSegmentImageAddr; uint32_t image_len = 0; uint32_t image_crc = 0; diff --git a/lib/crowdb-tree/src/maptable/mapping_table.cpp b/lib/crowdb-tree/tests/mapping_table_test_support.cpp similarity index 70% rename from lib/crowdb-tree/src/maptable/mapping_table.cpp rename to lib/crowdb-tree/tests/mapping_table_test_support.cpp index d689fc191..82ebbd8bd 100644 --- a/lib/crowdb-tree/src/maptable/mapping_table.cpp +++ b/lib/crowdb-tree/tests/mapping_table_test_support.cpp @@ -3,7 +3,7 @@ // Mapping-table implementation. -#include "crowdb-tree/maptable/mapping_table.h" +#include "mapping_table_test_support.h" #include "crowdb-tree/backend/page_store.h" // round_up_to_iu @@ -12,38 +12,38 @@ namespace crowdb::tree { -MappingTable::MappingTable() : segments_(kMaxSegments) +TestMappingTable::TestMappingTable(bool enabled) : segments_(enabled ? kMaxSegments : 0) { for (auto &s : segments_) { s.store(nullptr, std::memory_order_relaxed); } } -void MappingTable::swap_quiescent(MappingTable &other) noexcept +void TestMappingTable::swap_quiescent(TestMappingTable &other) noexcept { segments_.swap(other.segments_); std::swap(next_page_id_, other.next_page_id_); } -MappingTable::~MappingTable() +TestMappingTable::~TestMappingTable() { // The mapping table does not own resident pages (the epoch manager frees // them). With packed slot words there are no heap-allocated unloaded // descriptors to free — the descriptor is inline in the 64-bit word. for (auto &s : segments_) { - MappingSegment *seg = s.load(std::memory_order_relaxed); + TestMappingSegment *seg = s.load(std::memory_order_relaxed); delete seg; } } -MappingSegment *MappingTable::ensure_segment(uint64_t seg_idx) +TestMappingSegment *TestMappingTable::ensure_segment(uint64_t seg_idx) { - MappingSegment *seg = segments_[seg_idx].load(std::memory_order_acquire); + TestMappingSegment *seg = segments_[seg_idx].load(std::memory_order_acquire); if (seg != nullptr) { return seg; } - auto *fresh = new MappingSegment(static_cast<uint32_t>(kSegmentSize)); - MappingSegment *expected = nullptr; + auto *fresh = new TestMappingSegment(static_cast<uint32_t>(kSegmentSize)); + TestMappingSegment *expected = nullptr; if (segments_[seg_idx].compare_exchange_strong(expected, fresh, std::memory_order_acq_rel)) { return fresh; } @@ -51,34 +51,34 @@ MappingSegment *MappingTable::ensure_segment(uint64_t seg_idx) return expected; } -uint64_t MappingTable::get_word(uint64_t page_id) const +uint64_t TestMappingTable::get_word(uint64_t page_id) const { if (page_id == kInvalidPageId) { return slot_word::kEmpty; } uint64_t seg_idx = page_id / kSegmentSize; - if (seg_idx >= kMaxSegments) { + if (seg_idx >= segments_.size()) { return slot_word::kEmpty; } - MappingSegment *seg = segments_[seg_idx].load(std::memory_order_acquire); + TestMappingSegment *seg = segments_[seg_idx].load(std::memory_order_acquire); if (seg == nullptr) { return slot_word::kEmpty; } return seg->slots[page_id % kSegmentSize].load(std::memory_order_acquire); } -PageBase *MappingTable::get_resident(uint64_t page_id) const +PageBase *TestMappingTable::get_resident(uint64_t page_id) const { uint64_t w = get_word(page_id); return slot_word::is_resident(w) ? slot_word::resident_ptr(w) : nullptr; } -void MappingTable::store(uint64_t page_id, PageBase *page) +void TestMappingTable::store(uint64_t page_id, PageBase *page) { assert(page_id != kInvalidPageId); uint64_t seg_idx = page_id / kSegmentSize; assert(seg_idx < kMaxSegments); - MappingSegment *seg = segments_[seg_idx].load(std::memory_order_acquire); + TestMappingSegment *seg = segments_[seg_idx].load(std::memory_order_acquire); if (seg == nullptr) { std::scoped_lock lk(alloc_mu_); seg = ensure_segment(seg_idx); @@ -100,19 +100,19 @@ void MappingTable::store(uint64_t page_id, PageBase *page) return; } // store(page_id, nullptr) clears a slot, same as store_word/clear -- keep - // segment recycling (#14b) consistent across every path that empties a slot. + // segment recycling consistent across every path that empties a slot. uint32_t prev_live = seg->live_count.fetch_sub(1, std::memory_order_acq_rel); if (prev_live == 1) { recycle_segment_if_empty(seg_idx, seg); } } -void MappingTable::store_word(uint64_t page_id, uint64_t word) +void TestMappingTable::store_word(uint64_t page_id, uint64_t word) { assert(page_id != kInvalidPageId); uint64_t seg_idx = page_id / kSegmentSize; assert(seg_idx < kMaxSegments); - MappingSegment *seg = segments_[seg_idx].load(std::memory_order_acquire); + TestMappingSegment *seg = segments_[seg_idx].load(std::memory_order_acquire); if (seg == nullptr) { std::scoped_lock lk(alloc_mu_); seg = ensure_segment(seg_idx); @@ -136,21 +136,23 @@ void MappingTable::store_word(uint64_t page_id, uint64_t word) } } -void MappingTable::recycle_segment_if_empty(uint64_t seg_idx, MappingSegment *seg) +void TestMappingTable::recycle_segment_if_empty(uint64_t seg_idx, TestMappingSegment *seg) { - MappingSegment *expected = seg; + TestMappingSegment *expected = seg; if (!segments_[seg_idx].compare_exchange_strong(expected, nullptr, std::memory_order_acq_rel)) { - return; // shouldn't happen under the single-writer invariant; stay defensive + return; // shouldn't happen under the single-writer invariant; stay + // defensive } if (epoch_ != nullptr) { epoch_->retire_object(seg); // freed once no in-flight reader guard could see it } else { - delete seg; // no epoch wired -- no concurrent readers to protect against (e.g. unit tests) + delete seg; // no epoch wired -- no concurrent readers to protect against + // (e.g. unit tests) } } -void MappingTable::store_unloaded(uint64_t page_id, uint64_t addr, uint32_t plen, uint32_t iu) +void TestMappingTable::store_unloaded(uint64_t page_id, uint64_t addr, uint32_t plen, uint32_t iu) { assert(iu >= 1); uint64_t iu_index = addr / iu; @@ -159,12 +161,12 @@ void MappingTable::store_unloaded(uint64_t page_id, uint64_t addr, uint32_t plen store_word(page_id, slot_word::pack_unloaded(iu_index, iu_count)); } -void MappingTable::clear(uint64_t page_id) +void TestMappingTable::clear(uint64_t page_id) { store_word(page_id, slot_word::kEmpty); } -uint64_t MappingTable::allocate_page_id() +uint64_t TestMappingTable::allocate_page_id() { std::scoped_lock lk(alloc_mu_); uint64_t page_id = next_page_id_++; @@ -176,19 +178,19 @@ uint64_t MappingTable::allocate_page_id() return page_id; } -void MappingTable::set_next_page_id(uint64_t next) +void TestMappingTable::set_next_page_id(uint64_t next) { std::scoped_lock lk(alloc_mu_); next_page_id_ = next; } -uint64_t MappingTable::next_page_id() const +uint64_t TestMappingTable::next_page_id() const { std::scoped_lock lk(alloc_mu_); return next_page_id_; } -size_t MappingTable::segments_allocated() const +size_t TestMappingTable::segments_allocated() const { size_t n = 0; for (const auto &s : segments_) { @@ -199,7 +201,7 @@ size_t MappingTable::segments_allocated() const return n; } -MappingSegment *MappingTable::segment_at(uint64_t seg_idx) const +TestMappingSegment *TestMappingTable::segment_at(uint64_t seg_idx) const { if (seg_idx >= kMaxSegments) { return nullptr; @@ -207,9 +209,9 @@ MappingSegment *MappingTable::segment_at(uint64_t seg_idx) const return segments_[seg_idx].load(std::memory_order_acquire); } -bool MappingTable::commit_segment_persist(uint64_t seg_idx, MappingSegment *expected, uint64_t seen_write_seq, - uint64_t new_generation, uint64_t new_image_addr, uint32_t new_image_len, - uint32_t new_image_crc) +bool TestMappingTable::commit_segment_persist(uint64_t seg_idx, TestMappingSegment *expected, uint64_t seen_write_seq, + uint64_t new_generation, uint64_t new_image_addr, uint32_t new_image_len, + uint32_t new_image_crc) { if (seg_idx >= kMaxSegments) { return false; @@ -218,7 +220,8 @@ bool MappingTable::commit_segment_persist(uint64_t seg_idx, MappingSegment *expe return false; // recycled (or replaced) since prepare captured it } if (expected->write_seq.load(std::memory_order_relaxed) != seen_write_seq) { - return false; // written again since prepare -- still dirty, leave for next snapshot + return false; // written again since prepare -- still dirty, leave for next + // snapshot } expected->generation.store(new_generation, std::memory_order_relaxed); expected->image_addr = new_image_addr; @@ -228,13 +231,13 @@ bool MappingTable::commit_segment_persist(uint64_t seg_idx, MappingSegment *expe return true; } -void MappingTable::install_recovered_segment(uint64_t seg_idx, uint64_t generation, uint32_t live_count, - const std::vector<uint64_t> &words, uint64_t image_addr, - uint32_t image_len, uint32_t image_crc) +void TestMappingTable::install_recovered_segment(uint64_t seg_idx, uint64_t generation, uint32_t live_count, + const std::vector<uint64_t> &words, uint64_t image_addr, + uint32_t image_len, uint32_t image_crc) { assert(seg_idx < kMaxSegments); assert(words.size() == kSegmentSize); - MappingSegment *seg = ensure_segment(seg_idx); + TestMappingSegment *seg = ensure_segment(seg_idx); for (uint64_t i = 0; i < words.size(); ++i) { seg->slots[i].store(words[i], std::memory_order_relaxed); } diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_table.h b/lib/crowdb-tree/tests/mapping_table_test_support.h similarity index 53% rename from lib/crowdb-tree/include/crowdb-tree/maptable/mapping_table.h rename to lib/crowdb-tree/tests/mapping_table_test_support.h index 4066c7b86..484c7ab2f 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/mapping_table.h +++ b/lib/crowdb-tree/tests/mapping_table_test_support.h @@ -1,29 +1,16 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. -// Mapping table. -// -// PID -> atomic<uint64_t> packed slot word (plan-tree #14b). All structural -// references (root, sibling links, inner children) are PIDs, so a page can be -// replaced by swapping one slot. Readers do a lock-free atomic load; the single -// writer (flusher) does a plain atomic store (no CAS, per D2). PID allocation -// is mutex-guarded. -// -// Slot word encoding: see mapping_slot.h (crowdb::tree::slot_word). -// word == 0 -> empty (dead or never-allocated PID) -// (word & 7) == 0 (and !=0) -> resident: PageBase* (8-byte aligned) -// (word & 1) == 1 -> unloaded local byte descriptor -// (word & 7) == 2 -> unloaded immutable-store page reference -// -// The unloaded descriptor is inline in the word (no heap allocation), unlike the -// old tagged-pointer scheme that allocated an unloaded_page struct. +// Legacy mutable mapping model for allocation, packed slots, segment recycling +// and prepare/commit identity regressions. Production uses immutable MappingView. #pragma once #include "crowdb-common/metrics/metrics.h" #include "crowdb-tree/epoch.h" -#include "crowdb-tree/maptable/mapping_segment.h" +#include "crowdb-tree/maptable/mapping_geometry.h" #include "crowdb-tree/maptable/mapping_slot.h" #include "crowdb-tree/maptable/page.h" +#include "mapping_segment_test_support.h" #include <atomic> #include <cstdint> @@ -34,20 +21,23 @@ namespace crowdb::tree { -class MappingTable +using crowdb::common::metrics::Counter; + +// Legacy mutable mapping model retained only for low-level regression tests. +class TestMappingTable { public: - static constexpr uint64_t kSegmentSize = 1024; // PIDs per segment - static constexpr uint64_t kMaxSegments = 1U << 16; // -> 64M PIDs + static constexpr uint64_t kSegmentSize = kMappingSegmentSize; + static constexpr uint64_t kMaxSegments = kMaxMappingSegments; - MappingTable(); - ~MappingTable(); + explicit TestMappingTable(bool enabled = true); + ~TestMappingTable(); - MappingTable(const MappingTable &) = delete; - MappingTable &operator=(const MappingTable &) = delete; + TestMappingTable(const TestMappingTable &) = delete; + TestMappingTable &operator=(const TestMappingTable &) = delete; // Both owners must exclude operations before exchanging generations. - void swap_quiescent(MappingTable &other) noexcept; + void swap_quiescent(TestMappingTable &other) noexcept; // Reader: lock-free atomic load of the raw packed slot word. // Use slot_word::is_empty / is_resident / is_unloaded to classify. @@ -89,9 +79,7 @@ class MappingTable alloc_counter_ = c; } - // Allocate a fresh PID. Monotonic -- PIDs are never recycled (plan-tree - // #14 D1: a reused PID could be seen by a stale reader as the new page, - // i.e. silent wrong data). + // Allocate a fresh monotonic PID; stale readers must never see a reused identity. uint64_t allocate_page_id(); // Number of segments currently allocated (diagnostics). @@ -101,51 +89,36 @@ class MappingTable void set_next_page_id(uint64_t next); [[nodiscard]] uint64_t next_page_id() const; - // -- #14c/#14d: segment-image persistence (persist.cpp) -------------- - // - // Raw atomic load of the top-level segment pointer; nullptr if `seg_idx` - // was never allocated or has been recycled (#14b). The snapshot path - // (persist.cpp) uses this to enumerate every present segment (bounded by - // kMaxSegments, cheap) and inspect is_dirty()/slots[] directly -- no - // separate iteration API needed since MappingSegment's fields are - // public. - [[nodiscard]] MappingSegment *segment_at(uint64_t seg_idx) const; - - // Commit phase for a snapshot's segment-image write (mirrors the page - // write identity check in commit_prepared_snapshot). Call once the - // segment's image bytes are durable. `expected` and `seen_write_seq` are - // whatever prepare captured *before* releasing write_mutex_ for the I/O - // phase (see MappingSegment's doc comment). Returns false (no-op) if - // either changed in the meantime -- the segment either recycled away or - // was written to again, so the image we just persisted is stale; the - // segment stays dirty and the next snapshot re-images it. Never loses - // data: at worst it wastes one redundant re-image later. - bool commit_segment_persist(uint64_t seg_idx, MappingSegment *expected, uint64_t seen_write_seq, + // Inspect the test model's current segment, or nullptr after recycling. + [[nodiscard]] TestMappingSegment *segment_at(uint64_t seg_idx) const; + + // Model a prepared segment image commit. Changed identity or write sequence + // rejects the stale image without clearing newer dirty state. + bool commit_segment_persist(uint64_t seg_idx, TestMappingSegment *expected, uint64_t seen_write_seq, uint64_t new_generation, uint64_t new_image_addr, uint32_t new_image_len, uint32_t new_image_crc); // Recovery: install a segment's full slot-word array plus its // bookkeeping directly (no live_count/write_seq transition logic -- // recovery has no concurrent readers yet). `words.size()` must equal - // kSegmentSize (the only slot count this table currently supports; see - // Config::mapping_segment_slots's TODO). + // kSegmentSize, matching the durable segment geometry. void install_recovered_segment(uint64_t seg_idx, uint64_t generation, uint32_t live_count, const std::vector<uint64_t> &words, uint64_t image_addr, uint32_t image_len, uint32_t image_crc); private: - MappingSegment *ensure_segment(uint64_t seg_idx); + TestMappingSegment *ensure_segment(uint64_t seg_idx); // Called by the writer when a slot's last live entry in `seg` goes empty - // (plan-tree #14b). CAS's the top-level slot to nullptr -- final and safe - // because PIDs are monotonic (D1), so a freed segment's PID range is dead + // CAS the top-level slot to nullptr -- final and safe + // because PIDs are monotonic, so a freed segment's PID range is dead // forever and no allocation will ever revisit `seg_idx` -- then hands the // segment to the epoch manager (if wired) so any in-flight lock-free // reader that already loaded `seg` keeps a valid pointer until its guard // drains. - void recycle_segment_if_empty(uint64_t seg_idx, MappingSegment *seg); + void recycle_segment_if_empty(uint64_t seg_idx, TestMappingSegment *seg); - std::vector<std::atomic<MappingSegment *>> segments_; // fixed-size top-level array + std::vector<std::atomic<TestMappingSegment *>> segments_; // fixed-size top-level array mutable std::mutex alloc_mu_; uint64_t next_page_id_ = 0; diff --git a/lib/crowdb-tree/tests/memtable_access.h b/lib/crowdb-tree/tests/memtable_access.h index 9ad6177a6..a821b7d71 100644 --- a/lib/crowdb-tree/tests/memtable_access.h +++ b/lib/crowdb-tree/tests/memtable_access.h @@ -1,68 +1,81 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. #pragma once - -#include "crowdb-tree/crowdb-tree.h" - +#include "runtime_tree_access.h" +#include "snapshot/operation_wait.h" #ifdef CROWDB_TREE_TEST_UTIL namespace crowdb::tree { -// Deterministic access to the production capture/publication boundaries. struct MemTableAccess_for_tests { - static auto admit(Crowdbtree &tree) - { - return tree.admit_batch(); - } - - static auto capture(Crowdbtree &tree) + struct Capture { - return tree.capture_flush_locked(); - } - - static auto sources(Crowdbtree &tree) - { - return tree.all_memtables(); - } + std::shared_ptr<FlushOperation> operation; + uint64_t frontier = 0; + SourceCatalog::Tables tables; + }; - static auto active(Crowdbtree &tree) + struct Receiver { - return tree.current_active(); - } + using receiver_concept = stdexec::receiver_tag; + detail::SnapshotWaitCompletion *done; - static auto &counters(Crowdbtree &tree) - { - return tree.memtable_counters_; - } + void set_value(FlushResult report) const noexcept + { + set_error(report.handoff_status); + } - static auto copy_latency(Crowdbtree &tree) - { - return tree.metrics_.mt_version_copy_l; - } + void set_error(Status status) const noexcept + { + done->status = std::move(status); + done->done.signal(); + } - static auto &epoch(Crowdbtree &tree) - { - return tree.epoch_; - } + void set_stopped() const noexcept + { + set_error(Status::unavailable()); + } + }; - static bool replacing(Crowdbtree &tree) + static Capture capture(Crowdbtree &tree) { - return tree.generation_.closed_for_tests(); + Capture result; + auto status = RuntimeTreeAccess_for_tests::core(tree).prepare_flush(&result.operation); + if (!status.ok()) { + throw std::runtime_error(status.to_string()); + } + detail::SnapshotWaitCompletion done; + auto operation = stdexec::connect(result.operation->capture_sender(), Receiver{&done}); + stdexec::start(operation); + done.done.wait(); + if (!done.status.ok()) { + throw std::runtime_error(done.status.to_string()); + } + auto frozen = result.operation->frozen_sources(); + if (!frozen.catalog) { + throw std::runtime_error("capture requested without a materializable contiguous frontier"); + } + result.frontier = frozen.frontier; + result.tables.assign(frozen.catalog->tables.begin(), frozen.catalog->tables.end()); + return result; } - static void fail_publication(Crowdbtree &tree, void (*hook)()) + static auto active(Crowdbtree &tree) { - tree.after_flush_group_for_tests_ = hook; + return RuntimeTreeAccess_for_tests::capture(tree)->sources->tables.back(); } - static void finish(Crowdbtree &tree, uint64_t slot) + static auto sources(Crowdbtree &tree) { - tree.note_applied_slot(slot); + return RuntimeTreeAccess_for_tests::capture(tree)->sources->tables; } - static void publish(Crowdbtree &tree, Crowdbtree::FlushBoundary &boundary) + static void publish([[maybe_unused]] Crowdbtree &tree, Capture &capture) { - tree.publish_flush_locked(boundary); + auto status = detail::wait_snapshot_operation(capture.operation); + if (!status.ok()) { + throw std::runtime_error(status.to_string()); + } } }; } // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/native_import_test_support.h b/lib/crowdb-tree/tests/native_import_test_support.h new file mode 100644 index 000000000..7c498f9ed --- /dev/null +++ b/lib/crowdb-tree/tests/native_import_test_support.h @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/native_import.h" +#include "crowdb-tree/maptable/frame_page.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::test_import +{ +inline NativeFrame leaf(uint64_t pid, Slice key, Slice cell, uint64_t sibling = kInvalidPageId) +{ + NativeFrame result{.page_id = pid, .frame = std::vector<uint8_t>(4096)}; + LeafFrameBuilder builder(result.frame.data(), result.frame.size()); + EXPECT_TRUE(builder.try_append_sorted(key, cell)); + builder.finish(pid, sibling); + return result; +} + +inline NativeFrame inline_leaf(uint64_t pid = 0, Slice key = "key", Slice value = "imported", uint64_t slot = 7) +{ + const auto cell = encode_cell_buf(slot, OpKind::kPut, value); + return leaf(pid, key, Slice(cell)); +} + +inline std::vector<NativeFrame> overflow_tree(Slice value, uint64_t slot = 7) +{ + const auto cell = encode_overflow_cell_buf(slot, 1, value.size()); + std::vector<NativeFrame> result; + result.push_back(leaf(0, "key", Slice(cell))); + size_t written = 0; + while (written != value.size()) { + const auto count = std::min<size_t>(value.size() - written, overflow_chunk_cap(4096)); + const auto pid = result.size(); + const auto next = written + count == value.size() ? kInvalidPageId : pid + 1; + NativeFrame frame{.page_id = pid, .frame = std::vector<uint8_t>(4096)}; + overflow_frame_build(frame.frame.data(), frame.frame.size(), pid, next, + reinterpret_cast<const uint8_t *>(value.data() + written), count); + result.push_back(std::move(frame)); + written += count; + } + return result; +} + +inline NativeStreamInfo info(const std::vector<NativeFrame> &frames, uint64_t coverage = 7) +{ + return {.root_page_id = frames.empty() ? kInvalidPageId : frames.front().page_id, + .at_slot = coverage, + .next_page_id = frames.size(), + .frame_count = frames.size(), + .frame_count_known = true}; +} +} // namespace crowdb::tree::test_import diff --git a/lib/crowdb-tree/tests/overflow_load_test_support.h b/lib/crowdb-tree/tests/overflow_load_test_support.h new file mode 100644 index 000000000..9891a4b88 --- /dev/null +++ b/lib/crowdb-tree/tests/overflow_load_test_support.h @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/maptable/compressor.h" +#include "published_read_test_support.h" + +#include <cstring> + +namespace crowdb::tree::test_read +{ +class TestOverflowStore final : public AsyncPageStore +{ + public: + std::map<uint64_t, std::vector<uint8_t>> blobs; + std::deque<AsyncCompletion> pending; + std::atomic<size_t> reads{0}; + bool deferred = false; + + uint64_t submit_read(PageAddr address, void *destination, size_t length, AsyncCompletion complete) override + { + const auto &blob = blobs.at(address); + EXPECT_EQ(length, blob.size()); + std::memcpy(destination, blob.data(), std::min(length, blob.size())); + const uint64_t id = reads.fetch_add(1) + 1; + if (deferred) { + pending.push_back(complete); + } + else { + complete.complete(Status::Ok()); + } + return id; + } + + uint64_t submit_write(PageAddr /*addr*/, const void * /*buf*/, size_t /*len*/, AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return 0; + } + + Status submit_fsync(AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return Status::not_supported(); + } + + void cancel(uint64_t /*op_id*/) override + { + } + + void finish(Status status = Status::Ok()) + { + ASSERT_FALSE(pending.empty()); + const auto callback = pending.front(); + pending.pop_front(); + callback.complete(std::move(status)); + } +}; + +using TestBackingMap = std::map<uint64_t, std::shared_ptr<const BackingRef>>; + +inline void add_overflow(const std::shared_ptr<TestOverflowStore> &store, TestBackingMap *captured, uint64_t id, + uint64_t next, Slice value) +{ + const std::unique_ptr<OverflowBase> frame(OverflowBase::build(next, value.bytes(), value.size())); + std::vector<uint8_t> raw(frame->frame(), frame->frame() + frame->page_bytes()); + frame_put_u64(raw.data(), fh::kSelfpage_id, id); + frame_restamp_crc(raw.data(), raw.size()); + auto &blob = store->blobs[id]; + encode_durable_page(raw.data(), raw.size(), compress_algo::kNone, &blob); + auto backing = std::make_shared<BackingRef>(); + backing->address = id; + backing->length = blob.size(); + backing->owner = std::make_shared<int>(1); + captured->emplace(id, std::move(backing)); +} + +inline OverflowBackingResolver resolver(std::shared_ptr<const TestBackingMap> captured) +{ + return [captured = std::move(captured)](uint64_t id, std::shared_ptr<const BackingRef> *out) { + const auto at = captured->find(id); + if (at == captured->end()) { + return Status::not_found("captured overflow PID absent"); + } + *out = at->second; + return Status::Ok(); + }; +} + +inline std::shared_ptr<const PageState> overflow_leaf() +{ + auto result = std::make_shared<PageState>(); + OverlayBuilder builder({}); + auto owner = std::make_shared<CellVersion>(encode_overflow_cell_buf(5, 1, 10), 5, kFlagOverflow); + EXPECT_TRUE(builder.append("key", std::move(owner)).ok()); + std::shared_ptr<const OverlayBatch> overlay; + EXPECT_TRUE(builder.finish(1, &overlay).ok()); + result->overlays.push_back(std::move(overlay)); + return result; +} +} // namespace crowdb::tree::test_read diff --git a/lib/crowdb-tree/tests/page_cache_residency_test_support.h b/lib/crowdb-tree/tests/page_cache_residency_test_support.h new file mode 100644 index 000000000..1689dbaa5 --- /dev/null +++ b/lib/crowdb-tree/tests/page_cache_residency_test_support.h @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_cache.h" +#include "published_read_test_support.h" + +namespace crowdb::tree::test_cache_residency +{ +using namespace test_read; + +inline std::shared_ptr<PublishedPage> published(std::initializer_list<BaseRef> refs) +{ + auto leaf = std::make_shared<PageState>(); + leaf->bases.assign(refs.begin(), refs.end()); + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::move(leaf); + return page; +} + +struct TestCache +{ + detail::FlushRuntime runtime{1, 8}; + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + bool synchronize = true; + std::shared_ptr<TreeResourceMetrics> resources = std::make_shared<TreeResourceMetrics>(); + std::shared_ptr<SharedPageCache> cache = + std::make_shared<SharedPageCache>(PageCacheLimits{}, nullptr, resources, &runtime, lane); + PublicationCoordinator publisher{64, 1, {}, std::pmr::get_default_resource(), cache}; + std::shared_ptr<TestLoader> upstream = std::make_shared<TestLoader>(); + std::shared_ptr<PageLoader> loader = cache->bind(upstream); + uint64_t sequence = 0; + + PublicationResult publish(PublicationTransaction::Pages pages, std::optional<uint64_t> root = {}, + std::shared_ptr<detail::AllocationReservation> metadata = {}) + { + auto request = std::make_shared<PublicationRequest>(); + request->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{.sequence = ++sequence}), + .pages = std::move(pages), + .root_page_id = root, + .metadata = std::move(metadata)}; + PublicationResult result; + size_t calls = 0; + request->on_done = [&](PublicationResult value) { + result = std::move(value); + ++calls; + }; + EXPECT_TRUE(publisher.try_submit(request)); + EXPECT_EQ(calls, 1U); + if (synchronize) { + runtime.drain(); + } + return result; + } + + std::shared_ptr<const LeafBase> load(BaseRef input) + { + std::shared_ptr<const LeafBase> result; + loader->load_leaf(std::move(input), [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()) << status.to_string(); + result = std::move(leaf); + }); + runtime.drain(); + return result; + } + + void frame(uint64_t identity) + { + upstream->frames[identity] = base({ + {.key = "a", .value = "value"} + }); + } +}; +} // namespace crowdb::tree::test_cache_residency diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/descent.h b/lib/crowdb-tree/tests/page_descent_test_support.h similarity index 73% rename from lib/crowdb-tree/include/crowdb-tree/btree/descent.h rename to lib/crowdb-tree/tests/page_descent_test_support.h index 5f2ddbb36..ab9f24ca7 100644 --- a/lib/crowdb-tree/include/crowdb-tree/btree/descent.h +++ b/lib/crowdb-tree/tests/page_descent_test_support.h @@ -6,19 +6,19 @@ // (deltas) are resolved by the caller; descent only follows base inner pages. #pragma once -#include "crowdb-tree/maptable/mapping_table.h" #include "crowdb-tree/maptable/page.h" #include "crowdb-tree/slice.h" namespace crowdb::tree { -// Returns the leaf PID that should contain `key`, or kInvalidPageId if the tree is -// empty / malformed. `resolve(page_id)` maps a PID to its resident chain head, -// demand-loading an unloaded slot; it returns a real PageBase* or +// Returns the leaf PID that should contain `key`, or kInvalidPageId if the tree +// is empty / malformed. `resolve(page_id)` maps a PID to its resident chain +// head, demand-loading an unloaded slot; it returns a real PageBase* or // nullptr. `max_depth` guards against accidental cycles. template <class Resolve> -[[nodiscard]] inline uint64_t find_leaf_page_id(Resolve &&resolve, uint64_t root_page_id, Slice key, int max_depth = 64) +[[nodiscard]] inline uint64_t find_leaf_page_id_for_tests(Resolve &&resolve, uint64_t root_page_id, Slice key, + int max_depth = 64) { uint64_t page_id = root_page_id; for (int d = 0; d < max_depth; ++d) { @@ -26,8 +26,9 @@ template <class Resolve> if (page == nullptr) { return kInvalidPageId; } - // A leaf's mapping slot may point at a delta chain head; the chain shares the - // leaf's PID, so the PID is already the answer once we reach a leaf level. + // A leaf's mapping slot may point at a delta chain head; the chain shares + // the leaf's PID, so the PID is already the answer once we reach a leaf + // level. PageBase *node = page; while (node != nullptr && node->type == page_type::kBatchDelta) { node = node->next; diff --git a/lib/crowdb-tree/tests/page_load_delta_test_support.h b/lib/crowdb-tree/tests/page_load_delta_test_support.h new file mode 100644 index 000000000..02beb98c0 --- /dev/null +++ b/lib/crowdb-tree/tests/page_load_delta_test_support.h @@ -0,0 +1,123 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-tree/maptable/compressor.h" +#include "published_read_test_support.h" + +#include <cstring> + +namespace crowdb::tree::test_read +{ +class TestDeltaStore final : public AsyncPageStore +{ + public: + std::vector<uint8_t> blob; + bool deferred = false; + AsyncCompletion pending; + + uint64_t submit_read(PageAddr /*address*/, void *destination, size_t length, AsyncCompletion complete) override + { + EXPECT_EQ(length, blob.size()); + std::memcpy(destination, blob.data(), std::min(length, blob.size())); + if (deferred) { + pending = std::move(complete); + } + else { + complete.complete(Status::Ok()); + } + return 1; + } + + uint64_t submit_write(PageAddr /*address*/, const void * /*source*/, size_t /*length*/, + AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return 0; + } + + Status submit_fsync(AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return Status::not_supported(); + } + + void cancel(uint64_t /*id*/) override + { + } +}; + +struct TestDeltaLeaf +{ + std::shared_ptr<detail::FlushRuntime> runtime = std::make_shared<detail::FlushRuntime>(1, 8); + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<TestDeltaStore> store = std::make_shared<TestDeltaStore>(); + std::vector<uint8_t> frame = std::vector<uint8_t>(4096); + BaseRef reference; + + explicit TestDeltaLeaf(bool overflow = false) + { + LeafFrameBuilder builder(frame.data(), static_cast<uint32_t>(frame.size())); + for (const auto &record : std::initializer_list<Record>{ + {.key = "a", .slot = 1, .value = "old"}, + {.key = "b", .slot = 4, .value = "base-newer"}, + {.key = "c", .slot = 2, .value = "deleted"}, + {.key = "tie", .slot = 7, .value = "base-tie"}, + {.key = "zero", .slot = 0, .value = {}, .tombstone = true} + }) { + const auto cell = + encode_cell_buf(record.slot, record.tombstone ? OpKind::kDelete : OpKind::kPut, Slice(record.value)); + EXPECT_TRUE(builder.try_append_sorted(record.key, Slice(cell))); + } + builder.finish(17, 23); + std::vector<leaf_entry> deltas; + for (const auto &record : std::initializer_list<Record>{ + {.key = "c", .slot = 3, .value = {}, .tombstone = true}, + {.key = "a", .slot = 6, .value = "delta"}, + {.key = "new", .slot = 5, .value = "first"}, + {.key = "new", .slot = 5, .value = "second"}, + {.key = "b", .slot = 3, .value = "stale"}, + {.key = "tie", .slot = 7, .value = "delta-tie"}, + {.key = "new", .slot = 4, .value = "older"} + }) { + deltas.push_back({.key = record.key, + .cell = encode_cell_buf(record.slot, record.tombstone ? OpKind::kDelete : OpKind::kPut, + Slice(record.value))}); + } + if (overflow) { + deltas.push_back({.key = "overflow", .cell = encode_overflow_cell_buf(7, 41, 16384)}); + } + std::vector<uint8_t> output(frame.size()); + EXPECT_TRUE(leaf_frame_append_deltas(frame.data(), static_cast<uint32_t>(frame.size()), deltas, output.data())); + frame.swap(output); + auto backing = std::make_shared<BackingRef>(); + backing->owner = std::make_shared<int>(1); + reference = {.identity = 1, .max_slot = 7, .backing = backing}; + encode(); + } + + void encode() + { + EXPECT_TRUE(frame_validate(frame.data(), static_cast<uint32_t>(frame.size()))); + encode_durable_page(frame.data(), static_cast<uint32_t>(frame.size()), compress_algo::kNone, &store->blob); + auto backing = std::make_shared<BackingRef>(*reference.backing); + backing->length = store->blob.size(); + reference.backing = std::move(backing); + } + + std::shared_ptr<AsyncStorePageLoader> loader(PageLoadLimits limits = {}, PageLoadBudget budget = {}) + { + return std::make_shared<AsyncStorePageLoader>(store, runtime, lane, limits, OverflowBackingResolver{}, budget); + } + + void complete(detail::CallbackSender sender) const + { + Completion done; + auto operation = stdexec::connect(sender, Receiver{&done}); + stdexec::start(operation); + runtime->drain(); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(done.errors.load(), 0U); + } +}; +} // namespace crowdb::tree::test_read diff --git a/lib/crowdb-tree/tests/page_writeback_test_support.h b/lib/crowdb-tree/tests/page_writeback_test_support.h new file mode 100644 index 000000000..b2f29e301 --- /dev/null +++ b/lib/crowdb-tree/tests/page_writeback_test_support.h @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/page_writeback.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "load_budget_test_support.h" +#include "published_read_test_support.h" + +#include <stdexcept> + +namespace crowdb::tree::test_write +{ +using namespace test_read; + +class TestWriteStore final : public AsyncPageStore +{ + public: + std::shared_ptr<PageStore> device = std::make_shared<MemPageStore>(4096); + bool reloadable = true; + bool deferred = false; + bool throw_after_capture = false; + size_t writes = 0; + PagePurpose purpose = PagePurpose::kInvalid; + uint64_t address = 0; + const uint8_t *data = nullptr; + size_t bytes = 0; + AsyncCompletion pending; + + [[nodiscard]] bool write_completion_is_reloadable() const noexcept override + { + return reloadable; + } + + uint64_t submit_read(PageAddr addr, void *out, size_t size, AsyncCompletion complete) override + { + complete.complete(device->read_at(addr, static_cast<uint8_t *>(out), size)); + return 0; + } + + uint64_t submit_typed_write(PagePurpose kind, PageAddr addr, const void *input, size_t size, + AsyncCompletion complete) override + { + purpose = kind; + return submit_write(addr, input, size, std::move(complete)); + } + + uint64_t submit_write(PageAddr addr, const void *input, size_t size, AsyncCompletion complete) override + { + ++writes; + address = addr; + data = static_cast<const uint8_t *>(input); + bytes = size; + EXPECT_EQ(reinterpret_cast<uintptr_t>(input) % 4096, 0U); + EXPECT_EQ(size % 4096, 0U); + pending = std::move(complete); + if (throw_after_capture) { + throw std::runtime_error("unknown transport acceptance"); + } + if (!deferred) { + finish(); + } + return writes; + } + + Status submit_fsync(AsyncCompletion complete) override + { + auto status = device->sync(); + complete.complete(status); + return status; + } + + void cancel(uint64_t /*id*/) override + { + } + + void finish(Status status = Status::Ok()) + { + auto complete = std::exchange(pending, {}); + ASSERT_TRUE(complete); + if (status.ok()) { + status = device->write_at(address, data, bytes); + } + complete.complete(std::move(status)); + } +}; + +inline PageWriteInput write_input(size_t bytes = 4096, bool overflow = false) +{ + auto owner = std::make_shared<std::vector<uint8_t>>(bytes); + LeafFrameBuilder builder(owner->data(), bytes); + const auto cell = overflow ? encode_overflow_cell(3, 11, 1000) : encode_cell(3, OpKind::kPut, "value"); + EXPECT_TRUE(builder.try_append_sorted("a", Slice(cell))); + builder.finish(7, kInvalidPageId); + return { + .info = {.page_id = 7, .type = page_type::kLeafBase, .bytes = static_cast<uint32_t>(bytes)}, + .frame = *owner, + .owner = owner, + .reload_dependencies = {} + }; +} + +struct TestWriter +{ + LoadCredits credits; + std::shared_ptr<detail::FlushRuntime> runtime = std::make_shared<detail::FlushRuntime>(1, 8); + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<TestWriteStore> store = std::make_shared<TestWriteStore>(); + std::shared_ptr<PageWriteOperation> operation; + std::shared_ptr<int> extent = std::make_shared<int>(1); + uint64_t next_address = 8192; + + Status prepare(PageWriteInput input = write_input(), PageWriteLimits limits = {}) + { + return PageWriteOperation::create( + store, runtime, lane, std::move(input), limits, credits.credits, + [this](uint64_t length, PageWriteOperation::ExtentComplete done) { + ReservedWriteExtent reserved{.backend_id = 1, + .address = next_address, + .length = length, + .content_id = 10, + .generation = 2, + .owner = extent, + .physical_metrics = {}, + .chunk = {}}; + next_address += length; + done(Status::Ok(), std::move(reserved)); + }, + &operation); + } +}; + +} // namespace crowdb::tree::test_write diff --git a/lib/crowdb-tree/tests/published_read_test_support.h b/lib/crowdb-tree/tests/published_read_test_support.h new file mode 100644 index 000000000..bbec9b47a --- /dev/null +++ b/lib/crowdb-tree/tests/published_read_test_support.h @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/published_read.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <deque> +#include <map> + +namespace crowdb::tree::test_read +{ +struct Record +{ + std::string key; + uint64_t slot = 1; + std::string value; + bool tombstone = false; +}; + +inline std::shared_ptr<const PageState> page(std::initializer_list<Record> records, PageKeyRange range = {}) +{ + auto result = std::make_shared<PageState>(); + result->range = std::move(range); + OverlayBuilder builder({}); + for (const auto &record : records) { + const auto kind = record.tombstone ? OpKind::kDelete : OpKind::kPut; + auto owner = std::make_shared<CellVersion>(encode_cell_buf(record.slot, kind, Slice(record.value)), record.slot, + record.tombstone ? kFlagTombstone : 0); + EXPECT_TRUE(builder.append(Slice(record.key), std::move(owner)).ok()); + } + std::shared_ptr<const OverlayBatch> overlay; + EXPECT_TRUE(builder.finish(1, &overlay).ok()); + result->overlays.push_back(std::move(overlay)); + return result; +} + +inline std::shared_ptr<const LeafBase> base(std::initializer_list<Record> records) +{ + std::vector<leaf_entry> entries; + entries.reserve(records.size()); + for (const auto &record : records) { + entries.push_back({.key = record.key, + .cell = encode_cell_buf(record.slot, record.tombstone ? OpKind::kDelete : OpKind::kPut, + Slice(record.value))}); + } + return std::shared_ptr<const LeafBase>(LeafBase::build(entries)); +} + +inline std::shared_ptr<PublishedPage> published(std::shared_ptr<const PageState> leaf) +{ + auto result = std::make_shared<PublishedPage>(); + result->leaf = std::move(leaf); + result->metrics.dirty = true; + return result; +} + +inline std::shared_ptr<const PublishedTreeState> +view(std::shared_ptr<const PageState> leaf, std::vector<std::shared_ptr<MemTable>> tables = {}, uint64_t floor = 0) +{ + auto result = std::make_shared<PublishedTreeState>(); + auto sources = std::make_shared<PublishedSources>(); + sources->tables.assign(tables.begin(), tables.end()); + sources->floor = floor; + result->sources = std::move(sources); + result->coverage = floor; + result->root_page_id = 0; + MappingView::CopyStats copied; + const std::array<PageReplacement, 1> changes{ + {{.page_id = 0, .expected = {}, .replacement = published(std::move(leaf))}}}; + result->mapping = result->mapping.replace(changes, copied); + return result; +} + +inline BaseRef cold(uint64_t identity, std::optional<uint64_t> max_slot = std::nullopt) +{ + auto backing = std::make_shared<BackingRef>(); + backing->length = 4096; + backing->owner = std::make_shared<int>(1); + return {.identity = identity, .max_slot = max_slot, .backing = std::move(backing)}; +} + +class TestLoader final : public PageLoader +{ + public: + bool deferred = false; + size_t calls = 0; + std::map<uint64_t, std::shared_ptr<const LeafBase>> frames; + std::deque<LeafComplete> pending; + std::deque<uint64_t> identities; + + void load_leaf(BaseRef requested, LeafComplete complete) override + { + ++calls; + if (deferred) { + pending.push_back(std::move(complete)); + identities.push_back(requested.identity); + } + else { + complete(Status::Ok(), frames.at(requested.identity)); + } + } + + void finish(Status status = Status::Ok()) + { + ASSERT_FALSE(pending.empty()); + auto complete = std::move(pending.front()); + const auto id = identities.front(); + pending.pop_front(); + identities.pop_front(); + complete(std::move(status), frames.at(id)); + } +}; + +struct Completion +{ + std::atomic<size_t> values{0}; + std::atomic<size_t> errors{0}; + std::atomic<size_t> stopped{0}; + Status status; +}; + +struct Receiver +{ + using receiver_concept = stdexec::receiver_tag; + Completion *done; + + void set_value() const noexcept + { + done->values.fetch_add(1); + } + + void set_error(Status status) const noexcept + { + done->status = std::move(status); + done->errors.fetch_add(1); + } + + void set_stopped() const noexcept + { + done->stopped.fetch_add(1); + } +}; + +inline void complete_inline(detail::CallbackSender sender) +{ + Completion done; + auto operation = stdexec::connect(sender, Receiver{.done = &done}); + stdexec::start(operation); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(done.errors.load(), 0U); +} + +inline std::vector<PublishedScanEntry> collect(const std::shared_ptr<PublishedScan> &scan) +{ + std::vector<PublishedScanEntry> result; + for (size_t window = 0; window < 1000; ++window) { + complete_inline(scan->next_sender()); + if (!scan->result().status.ok()) { + return result; + } + result.insert(result.end(), scan->result().entries.begin(), scan->result().entries.end()); + if (scan->result().done) { + return result; + } + } + ADD_FAILURE() << "bounded scan did not terminate"; + return result; +} +} // namespace crowdb::tree::test_read diff --git a/lib/crowdb-tree/tests/recovered_state_test_support.h b/lib/crowdb-tree/tests/recovered_state_test_support.h new file mode 100644 index 000000000..e76d4ed7d --- /dev/null +++ b/lib/crowdb-tree/tests/recovered_state_test_support.h @@ -0,0 +1,120 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "inner_load_test_support.h" +#include "load_budget_test_support.h" +#include "snapshot/recovered_state.h" +#include "snapshot/stored_overflow.h" + +namespace crowdb::tree::test_read +{ +struct TestRecoveredGraph +{ + LoadCredits credits; + std::shared_ptr<detail::FlushRuntime> runtime = std::make_shared<detail::FlushRuntime>(1, 8); + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<TestInnerStore> store = std::make_shared<TestInnerStore>(); + std::shared_ptr<CapturedDurablePages> pages = + detail::reserved_shared<CapturedDurablePages>(credits.credits.bytes, credits.credits.bytes); + std::shared_ptr<AsyncStorePageLoader> loader = std::make_shared<AsyncStorePageLoader>( + store, runtime, lane, PageLoadLimits{}, OverflowBackingResolver{}, credits.credits); + std::shared_ptr<RecoveredStateOperation> operation; + + TestRecoveredGraph() + { + pages->root_page_id = 0; + pages->coverage = 10; + pages->generation = 3; + pages->backend_id = 9; + } + + void add(uint64_t pid, const std::vector<uint8_t> &raw) + { + ASSERT_TRUE(frame_validate(raw.data(), raw.size())); + auto &blob = store->blobs[pid]; + encode_durable_page(raw.data(), raw.size(), compress_algo::kNone, &blob); + auto backing = std::make_shared<BackingRef>(); + backing->backend_id = 9; + backing->address = pid; + backing->length = blob.size(); + backing->owner = std::make_shared<int>(1); + DurableCapturedFrame frame{.page_id = pid, + .type = frame_page_type(raw.data()), + .frame_bytes = static_cast<uint32_t>(raw.size()), + .backing = backing}; + if (frame.type == page_type::kLeafBase) { + ASSERT_TRUE(capture_stored_overflow(LeafFrameView(raw.data(), raw.size()), 4096, credits.credits.bytes, + &frame.stored_overflow) + .ok()); + } + else if (frame.type == page_type::kOverflowFrame) { + const OverflowFrameView overflow(raw.data(), raw.size()); + frame.next_overflow = overflow.next_page_id(); + frame.overflow_bytes = overflow.chunk_len(); + } + pages->frames.insert_or_assign(pid, std::move(frame)); + pages->next_page_id = std::max(pages->next_page_id, pid + 1); + } + + void leaf(uint64_t pid, std::initializer_list<Record> records, uint64_t sibling = kInvalidPageId) + { + std::vector<uint8_t> raw(4096); + LeafFrameBuilder builder(raw.data(), raw.size()); + for (const auto &record : records) { + const auto cell = + encode_cell_buf(record.slot, record.tombstone ? OpKind::kDelete : OpKind::kPut, Slice(record.value)); + ASSERT_TRUE(builder.try_append_sorted(record.key, Slice(cell))); + } + builder.finish(pid, sibling); + add(pid, raw); + } + + void index(uint64_t pid, std::vector<uint64_t> children, std::initializer_list<Slice> keys, + std::optional<std::pair<uint64_t, uint64_t>> fences = {}) + { + std::vector<uint8_t> raw(4096); + ASSERT_TRUE(inner_frame_build(raw.data(), raw.size(), pid, children, std::vector<Slice>(keys))); + if (fences) { + frame_set_inner_fence_pages(raw.data(), raw.size(), fences->first, fences->second); + } + add(pid, raw); + } + + void balanced() + { + index(0, {1, 2}, {"m"}, std::pair{1, 2}); + leaf(1, + { + {.key = "a", .slot = 2, .value = "one"}, + {.key = "d", .slot = 3, .value = {}, .tombstone = true} + }, + 2); + leaf(2, { + {.key = "z", .slot = 4, .value = "two"} + }); + } + + Status prepare(RecoveredStateLimits limits = {}) + { + return RecoveredStateOperation::create(pages, loader, runtime, lane, limits, credits.credits.bytes, &operation); + } + + void run(Completion *done) const + { + auto connected = stdexec::connect(operation->sender(), Receiver{done}); + stdexec::start(connected); + runtime->drain(); + } + + void complete(detail::CallbackSender sender) const + { + Completion done; + auto connected = stdexec::connect(sender, Receiver{&done}); + stdexec::start(connected); + runtime->drain(); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(done.errors.load(), 0U); + } +}; +} // namespace crowdb::tree::test_read diff --git a/lib/crowdb-tree/tests/runtime_tree_access.h b/lib/crowdb-tree/tests/runtime_tree_access.h new file mode 100644 index 000000000..6e9e416bf --- /dev/null +++ b/lib/crowdb-tree/tests/runtime_tree_access.h @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once +#include "btree/tree_core_access.h" +#ifdef CROWDB_TREE_TEST_UTIL +namespace crowdb::tree +{ +struct RuntimeTreeAccess_for_tests +{ + static TreeCore &core(Crowdbtree &tree) + { + return *TreeCoreAccess::get(tree); + } + + static auto capture(Crowdbtree &tree) + { + return core(tree).capture(); + } + + static auto root(Crowdbtree &tree) + { + auto view = capture(tree); + return view->mapping.get(view->root_page_id); + } + + static void reclaim(Crowdbtree &tree) + { + core(tree).publisher().reclaim(); + core(tree).binding()->drain(); + } +}; +} // namespace crowdb::tree +#endif diff --git a/lib/crowdb-tree/tests/snapshot_publication_test_support.h b/lib/crowdb-tree/tests/snapshot_publication_test_support.h new file mode 100644 index 000000000..7f5edc65e --- /dev/null +++ b/lib/crowdb-tree/tests/snapshot_publication_test_support.h @@ -0,0 +1,50 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <future> + +namespace crowdb::tree::test_snapshot_publication +{ +inline std::shared_ptr<PublishedPage> leaf(uint64_t content) +{ + auto page = std::make_shared<PublishedPage>(); + auto state = std::make_shared<PageState>(); + state->content_id = content; + state->route_generation = 1; + state->representation_id = 1; + page->leaf = std::move(state); + page->metrics.dirty = true; + return page; +} + +inline PublicationResult publish(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + transaction.record = std::make_shared<InstallRecord>(InstallToken{.generation = publisher.capture()->generation}); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> completed; + auto result = completed.get_future(); + request->on_done = [&](PublicationResult value) { completed.set_value(std::move(value)); }; + if (!publisher.try_submit(request)) { + ADD_FAILURE() << "snapshot publication rejected"; + return {.status = Status::unavailable()}; + } + return result.get(); +} + +inline PublicationResult checkpoint(PublicationCoordinator &publisher, CheckpointAction action, uint64_t token) +{ + PublicationTransaction transaction; + transaction.checkpoint = {.action = action, .token = token}; + if (action == CheckpointAction::Capture) { + transaction.expected_coverage = publisher.capture()->coverage; + } + return publish(publisher, std::move(transaction)); +} +} // namespace crowdb::tree::test_snapshot_publication diff --git a/lib/crowdb-tree/tests/stdexec_overlay_test.py b/lib/crowdb-tree/tests/stdexec_overlay_test.py new file mode 100644 index 000000000..11c18df73 --- /dev/null +++ b/lib/crowdb-tree/tests/stdexec_overlay_test.py @@ -0,0 +1,48 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +import importlib.util +from pathlib import Path +import shutil +import sys +import tempfile +import unittest + +sys.dont_write_bytecode = True +ENGINE = Path(sys.argv.pop(1)).resolve() +VENDOR = Path(sys.argv.pop(1)).resolve() +SPEC = importlib.util.spec_from_file_location("prepare_stdexec", ENGINE / "cmake/prepare_stdexec.py") +PREPARE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PREPARE) + + +class StdexecOverlay(unittest.TestCase): + def test_generation_is_idempotent_and_does_not_mutate_vendor(self): + original = (VENDOR / "exec/static_thread_pool.hpp").read_bytes() + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "overlay" + PREPARE.prepare(VENDOR, output) + patched = output / "include/exec/static_thread_pool.hpp" + self.assertIn(b"pool_thread_factory", patched.read_bytes()) + self.assertNotEqual(original, patched.read_bytes()) + first = patched.stat().st_mtime_ns + PREPARE.prepare(VENDOR, output) + self.assertEqual(first, patched.stat().st_mtime_ns) + self.assertEqual(original, (VENDOR / "exec/static_thread_pool.hpp").read_bytes()) + + def test_changed_baseline_preserves_previous_valid_output(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + vendor = root / "vendor" + output = root / "overlay" + shutil.copytree(VENDOR, vendor) + PREPARE.prepare(vendor, output) + stamp = (output / "stamp").read_bytes() + target = vendor / "exec/detail/bwos_lifo_queue.hpp" + target.write_bytes(target.read_bytes() + b"\n// changed baseline\n") + with self.assertRaisesRegex(RuntimeError, "baseline changed"): + PREPARE.prepare(vendor, output) + self.assertEqual(stamp, (output / "stamp").read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/lib/crowdb-tree/tests/unit/admitted_batch_test.cpp b/lib/crowdb-tree/tests/unit/admitted_batch_test.cpp new file mode 100644 index 000000000..509b992dc --- /dev/null +++ b/lib/crowdb-tree/tests/unit/admitted_batch_test.cpp @@ -0,0 +1,227 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/admitted_batch.h" + +#include <gtest/gtest.h> + +#include <array> + +namespace crowdb::tree +{ +namespace +{ +constexpr auto kWriter = detail::BudgetClass::kAdmittedWriter; +constexpr auto kWriterIndex = static_cast<size_t>(kWriter); + +struct TestAdmittedInputs +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(4 * 1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + SourceWriteOptions options{ + .account = account, .max_entries = 4, .max_key_bytes = 1024, .max_gap_entries = 2, .max_gap_key_bytes = 512}; + Config config; + + Status admit(const Batch &batch, std::shared_ptr<AdmittedBatch> *out) const + { + return AdmittedBatch::create( + config, options, 1024 * 1024, batch.ops.size(), + [&](size_t index) { + const auto &op = batch.ops[index]; + return CoreWriteInput{ + .key = Slice(op.key), .value = Slice(op.value), .flags = static_cast<uint8_t>(op.kind)}; + }, + kWriter, out); + } + + [[nodiscard]] size_t usage() const + { + const auto snapshot = budget.snapshot(); + return snapshot.allocated[kWriterIndex] + snapshot.reserved[kWriterIndex]; + } +}; + +TEST(AdmittedBatch, OwnsInputsAndLastDuplicateBeforeConsensusWithoutSourceEntry) +{ + TestAdmittedInputs fixture; + SourceAdmission sources(0, 1, fixture.options); + std::shared_ptr<ApplyFrontier> frontier; + ASSERT_TRUE(ApplyFrontier::create(0, 8, {}, &frontier).ok()); + std::shared_ptr<AdmittedBatch> admitted; + { + Batch input; + input.ops.push_back({.key = "k", .kind = OpKind::kPut, .value = "old"}); + input.ops.push_back({.key = "k", .kind = OpKind::kPut, .value = "chosen"}); + input.ops.push_back({.key = "z", .kind = OpKind::kPut, .value = "tail"}); + ASSERT_TRUE(fixture.admit(input, &admitted).ok()); + } + EXPECT_EQ(sources.catalog()->active->count(), 0U); + EXPECT_EQ(admitted->size(), 2U); + auto blocker = detail::ResourceBudget::try_reserve(fixture.account, kWriter, + TestAdmittedInputs::limits()[kWriterIndex] - fixture.usage()); + ASSERT_TRUE(blocker); + ASSERT_TRUE(admitted->try_commit(sources, *frontier, 1).ok()); + EXPECT_TRUE(admitted->consumed()); + EXPECT_EQ(frontier->contiguous().load(), 1U); + const auto table = sources.catalog()->active; + auto guard = table->read_guard(); + const auto *cell = table->find("k"); + ASSERT_NE(cell, nullptr); + EXPECT_EQ(CellView(cell->cell.slice()).value(), Slice("chosen")); + EXPECT_EQ(admitted->try_commit(sources, *frontier, 2).code(), Code::kInvalidArgument); +} + +TEST(AdmittedBatch, BusySourcePreservesInputsAndReservationUntilSuccessorIsPublished) +{ + TestAdmittedInputs fixture; + fixture.options.max_entries = 1; + SourceAdmission sources(0, 1, fixture.options); + std::shared_ptr<ApplyFrontier> frontier; + ASSERT_TRUE(ApplyFrontier::create(0, 8, {}, &frontier).ok()); + Batch input; + input.ops.push_back({.key = "a", .kind = OpKind::kPut, .value = "one"}); + std::shared_ptr<AdmittedBatch> first; + ASSERT_TRUE(fixture.admit(input, &first).ok()); + ASSERT_TRUE(first->try_commit(sources, *frontier, 1).ok()); + std::shared_ptr<AdmittedBatch> second; + ASSERT_TRUE(fixture.admit(input, &second).ok()); + EXPECT_EQ(second->try_commit(sources, *frontier, 2).code(), Code::kUnavailable); + EXPECT_FALSE(second->consumed()); + auto reserved = detail::ResourceBudget::try_reserve(fixture.account, detail::BudgetClass::kHandoff, 128 * 1024); + if (!reserved) { + FAIL() << "handoff credit unavailable"; + } + auto credit = detail::AllocationReservation::create(std::move(reserved.value())); + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(sources.try_capture(frontier->contiguous(), credit, &capture).ok()); + ASSERT_TRUE(stdexec::sync_wait(capture->drain_sender()).has_value()); + ASSERT_TRUE(sources.activate_successor(capture, capture->batch_id()).ok()); + ASSERT_TRUE(second->try_commit(sources, *frontier, 2).ok()); + EXPECT_EQ(frontier->contiguous().load(), 2U); +} + +TEST(AdmittedBatch, ChosenGapKeepsItsOrdinaryMemoryReservation) +{ + TestAdmittedInputs fixture; + fixture.options.max_entries = 1; + SourceAdmission sources(0, 1, fixture.options); + std::shared_ptr<ApplyFrontier> frontier; + ASSERT_TRUE(ApplyFrontier::create(0, 8, {}, &frontier).ok()); + Batch input; + input.ops.push_back({.key = "k", .kind = OpKind::kPut, .value = "value"}); + std::shared_ptr<AdmittedBatch> gap; + std::shared_ptr<AdmittedBatch> ahead; + ASSERT_TRUE(fixture.admit(input, &gap).ok()); + ASSERT_TRUE(fixture.admit(input, &ahead).ok()); + ASSERT_TRUE(ahead->try_commit(sources, *frontier, 2).ok()); + EXPECT_EQ(frontier->contiguous().load(), 0U); + ASSERT_TRUE(gap->try_commit(sources, *frontier, 1).ok()); + EXPECT_EQ(frontier->contiguous().load(), 2U); + const auto category = static_cast<size_t>(detail::BudgetClass::kGapRepair); + EXPECT_EQ(fixture.budget.snapshot().reserved[category] + fixture.budget.snapshot().allocated[category], 0U); +} + +TEST(AdmittedBatch, HigherGapCannotConsumeTheLastContiguousSourceAllowance) +{ + TestAdmittedInputs fixture; + fixture.options.max_entries = fixture.options.max_gap_entries = 1; + SourceAdmission sources(0, 1, fixture.options); + std::shared_ptr<ApplyFrontier> frontier; + ASSERT_TRUE(ApplyFrontier::create(0, 8, {}, &frontier).ok()); + Batch input; + input.ops.push_back({.key = "k", .kind = OpKind::kPut, .value = "value"}); + std::array<std::shared_ptr<AdmittedBatch>, 3> admitted; + for (auto &batch : admitted) { + ASSERT_TRUE(fixture.admit(input, &batch).ok()); + } + // The newest slot uses the ordinary source allowance. A later execution + // of slot 2 must leave the independent gap allowance for missing slot 1. + ASSERT_TRUE(admitted[2]->try_commit(sources, *frontier, 3).ok()); + EXPECT_EQ(admitted[1]->try_commit(sources, *frontier, 2).code(), Code::kUnavailable); + EXPECT_FALSE(admitted[1]->consumed()); + EXPECT_EQ(sources.catalog()->active_budget->resource_usage()[2].used, 0U); + ASSERT_TRUE(admitted[0]->try_commit(sources, *frontier, 1).ok()); + EXPECT_EQ(frontier->contiguous().load(), 1U); + EXPECT_EQ(sources.catalog()->active_budget->resource_usage()[2].used, 1U); + + auto reserved = detail::ResourceBudget::try_reserve(fixture.account, detail::BudgetClass::kHandoff, 128 * 1024); + if (!reserved) { + FAIL() << "handoff credit unavailable"; + } + auto credit = detail::AllocationReservation::create(std::move(reserved.value())); + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(sources.try_capture(frontier->contiguous(), credit, &capture).ok()); + ASSERT_TRUE(stdexec::sync_wait(capture->drain_sender())); + ASSERT_TRUE(sources.activate_successor(capture, capture->batch_id()).ok()); + ASSERT_TRUE(admitted[1]->try_commit(sources, *frontier, 2).ok()); + EXPECT_EQ(frontier->contiguous().load(), 3U); +} + +TEST(AdmittedBatch, HigherGapsUseFreeOrdinaryCapacityWithoutTakingTheProgressAllowance) +{ + TestAdmittedInputs fixture; + fixture.options.max_gap_entries = 1; + SourceAdmission sources(0, 1, fixture.options); + std::shared_ptr<ApplyFrontier> frontier; + ASSERT_TRUE(ApplyFrontier::create(0, 8, {}, &frontier).ok()); + Batch input; + input.ops.push_back({.key = "k", .kind = OpKind::kPut, .value = "value"}); + std::array<std::shared_ptr<AdmittedBatch>, 3> admitted; + for (auto &batch : admitted) { + ASSERT_TRUE(fixture.admit(input, &batch).ok()); + } + ASSERT_TRUE(admitted[2]->try_commit(sources, *frontier, 3).ok()); + ASSERT_TRUE(admitted[1]->try_commit(sources, *frontier, 2).ok()); + auto usage = sources.catalog()->active_budget->resource_usage(); + EXPECT_EQ(usage[0].used, 2U); + EXPECT_EQ(usage[2].used, 0U); + ASSERT_TRUE(admitted[0]->try_commit(sources, *frontier, 1).ok()); + EXPECT_EQ(frontier->contiguous().load(), 3U); + usage = sources.catalog()->active_budget->resource_usage(); + EXPECT_EQ(usage[0].used, 2U); + EXPECT_EQ(usage[2].used, 1U); +} + +TEST(AdmittedBatch, CancellationAndInvalidInputsReturnAllPreproposalCredit) +{ + TestAdmittedInputs fixture; + const auto baseline = fixture.usage(); + { + Batch input; + input.ops.push_back({.key = "k", .kind = OpKind::kPut, .value = std::string(4096, 'v')}); + std::shared_ptr<AdmittedBatch> admitted; + ASSERT_TRUE(fixture.admit(input, &admitted).ok()); + EXPECT_GT(fixture.usage(), baseline); + } + EXPECT_EQ(fixture.usage(), baseline); + Batch invalid; + invalid.ops.push_back( + {.key = std::string(fixture.options.max_key_size + 1, 'k'), .kind = OpKind::kPut, .value = "v"}); + std::shared_ptr<AdmittedBatch> rejected; + EXPECT_EQ(fixture.admit(invalid, &rejected).code(), Code::kInvalidArgument); + EXPECT_FALSE(rejected); + EXPECT_EQ(fixture.usage(), baseline); +} + +TEST(AdmittedBatch, NoOpRespectsTheBoundedFrontierAndDoesNotSkipGaps) +{ + TestAdmittedInputs fixture; + SourceAdmission sources(0, 1, fixture.options); + std::shared_ptr<ApplyFrontier> frontier; + ASSERT_TRUE(ApplyFrontier::create(0, 2, {}, &frontier).ok()); + std::shared_ptr<AdmittedBatch> noop; + ASSERT_TRUE(fixture.admit({}, &noop).ok()); + EXPECT_EQ(noop->try_commit(sources, *frontier, 3).code(), Code::kResourceExhausted); + EXPECT_FALSE(noop->consumed()); + ASSERT_TRUE(noop->try_commit(sources, *frontier, 2).ok()); + EXPECT_EQ(frontier->contiguous().load(), 0U); + EXPECT_EQ(sources.catalog()->active->count(), 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/allocation_account_adoption_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_adoption_test.cpp new file mode 100644 index 000000000..1dcda9561 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/allocation_account_adoption_test.cpp @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" + +#include <limits> + +namespace crowdb::tree::detail +{ +namespace +{ +using namespace test_allocation_account; + +TEST(AllocationAccountAdoption, FirstBindChargesEveryExistingOwnerBeforeFurtherGrowth) +{ + ResourceBudget host(limits(16384), 1); + auto account = host.make_account(limits(8192)); + std::shared_ptr<AllocationAccount> budget; + ASSERT_TRUE(AllocationAccount::create_unbound(&budget).ok()); + EXPECT_EQ(budget->domain(), nullptr); + { + MetadataAllocator<uint8_t, AllocationAccount> allocator(budget); + auto *data = allocator.allocate(103); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + ASSERT_TRUE(budget->rebind(account).ok()); + EXPECT_NE(budget->domain(), nullptr); + const auto bound = host.snapshot().allocated[kRouting]; + EXPECT_GT(bound, 103U + sizeof(AllocationAccount)); + EXPECT_EQ(account->snapshot().physical[kMetadata], bound); + EXPECT_FALSE(budget->claim(8192 - bound + 1)); + allocator.deallocate(data, 103); + EXPECT_EQ(account->snapshot().allocated[kRouting], bound - 103); + } + budget.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(account->snapshot().physical[kMetadata], 0U); +} + +TEST(AllocationAccountAdoption, InsufficientFirstQuotaLeavesUnboundStorageAndHostUntouched) +{ + ResourceBudget host(limits(16384), 1); + auto small = host.make_account(limits(1)); + auto adequate = host.make_account(limits(8192)); + std::shared_ptr<AllocationAccount> budget; + ASSERT_TRUE(AllocationAccount::create_unbound(&budget).ok()); + ASSERT_TRUE(budget->claim(113)); + budget->record_allocation(AllocationKind::kFrame, 113, true); + EXPECT_FALSE(budget->rebind(small).ok()); + EXPECT_EQ(budget->domain(), nullptr); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + ASSERT_TRUE(budget->rebind(adequate).ok()); + EXPECT_EQ(adequate->snapshot().physical[kFrame], 113U); + budget->record_allocation(AllocationKind::kFrame, 113, false); + budget->release(113); + budget.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(adequate->snapshot().physical[kFrame], 0U); + EXPECT_EQ(adequate->snapshot().physical[kMetadata], 0U); +} + +TEST(AllocationAccountAdoption, UnboundCapacityCannotWrapAndMayBeFreedWithoutBinding) +{ + std::shared_ptr<AllocationAccount> budget; + ASSERT_TRUE(AllocationAccount::create_unbound(&budget).ok()); + EXPECT_FALSE(budget->claim(std::numeric_limits<size_t>::max())); + ASSERT_TRUE(budget->claim(127)); + budget->record_allocation(AllocationKind::kFrame, 127, true); + budget->record_allocation(AllocationKind::kFrame, 127, false); + budget->release(127); + budget.reset(); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/allocation_account_grant_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_grant_test.cpp new file mode 100644 index 000000000..66bfb78c4 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/allocation_account_grant_test.cpp @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" +#include "btree/allocation_account_grant.h" + +namespace crowdb::tree::detail +{ +namespace +{ +using namespace test_allocation_account; + +TEST(BackendAllocationGrant, UsesExistingAdmissionAndRecyclesTemporaryWindows) +{ + TestAccounts test; + auto credit = ResourceBudget::try_reserve(test.old, BudgetClass::kRouting, 8192 - test.base); + ASSERT_TRUE(credit); + if (!credit) { + return; + } + std::shared_ptr<BackendAllocationGrant> grant; + ASSERT_TRUE(BackendAllocationGrant::create(test.budget, std::move(*credit), &grant).ok()); + const auto control = test.old->snapshot().allocated[kRouting] - test.base; + ASSERT_LE(control, BackendAllocationGrant::allocation_bound()); + const auto window = 8192 - test.base - control; + EXPECT_FALSE(test.budget->claim(1)); + { + MetadataAllocator<uint8_t, BackendAllocationGrant> allocator(grant); + for (size_t i = 0; i != 3; ++i) { + auto *bytes = allocator.allocate(window); + EXPECT_EQ(test.host.snapshot().allocated[kRouting], 8192U); + EXPECT_EQ(test.old->snapshot().physical[kMetadata], 8192U); + allocator.deallocate(bytes, window); + EXPECT_EQ(test.host.snapshot().reserved[kRouting], window); + } + } + grant->seal(); + EXPECT_EQ(test.host.snapshot().reserved[kRouting], 0U); + grant.reset(); + test.expect_only_control(test.old); + test.finish(); +} + +TEST(BackendAllocationGrant, RetainedAllocationMovesWithBackendAfterOperationSeals) +{ + TestAccounts test; + auto credit = ResourceBudget::try_reserve(test.old, BudgetClass::kRouting, 2048); + ASSERT_TRUE(credit); + if (!credit) { + return; + } + std::shared_ptr<BackendAllocationGrant> grant; + ASSERT_TRUE(BackendAllocationGrant::create(test.budget, std::move(*credit), &grant).ok()); + { + MetadataAllocator<uint8_t, BackendAllocationGrant> allocator(grant); + auto *bytes = allocator.allocate(133); + grant->seal(); + const auto retained = test.host.snapshot().allocated[kRouting]; + ASSERT_TRUE(test.budget->rebind(test.next).ok()); + EXPECT_EQ(test.old->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(test.next->snapshot().physical[kMetadata], retained); + grant.reset(); + allocator.deallocate(bytes, 133); + } + test.expect_only_control(test.next); + test.finish(); +} + +TEST(BackendAllocationGrant, TargetQuotaRefusalReturnsSourceCreditWithoutDoubleCharge) +{ + TestAccounts test; + auto blocker = ResourceBudget::try_reserve(test.old, BudgetClass::kRouting, 8192 - test.base); + ASSERT_TRUE(blocker); + auto source = ResourceBudget::try_reserve(test.next, BudgetClass::kRouting, 2048); + ASSERT_TRUE(source); + if (!source) { + return; + } + const auto host_before = test.host.snapshot(); + std::shared_ptr<BackendAllocationGrant> grant; + EXPECT_FALSE(BackendAllocationGrant::create(test.budget, std::move(*source), &grant).ok()); + EXPECT_FALSE(grant); + EXPECT_EQ(test.next->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(test.next->snapshot().reserved[kRouting], 0U); + EXPECT_EQ(test.host.snapshot().allocated[kRouting], host_before.allocated[kRouting]); + EXPECT_EQ(test.host.snapshot().reserved[kRouting], host_before.reserved[kRouting] - 2048); + blocker.reset(); + test.finish(); +} + +TEST(BackendAllocationGrant, FinalFreeDuringTransferUsesPendingDeltaAndNeverRevivesReserve) +{ + TestAccounts test; + auto credit = ResourceBudget::try_reserve(test.old, BudgetClass::kRouting, 2048); + ASSERT_TRUE(credit); + if (!credit) { + return; + } + std::shared_ptr<BackendAllocationGrant> grant; + ASSERT_TRUE(BackendAllocationGrant::create(test.budget, std::move(*credit), &grant).ok()); + { + MetadataAllocator<uint8_t, BackendAllocationGrant> allocator(grant); + auto *bytes = allocator.allocate(149); + grant->seal(); + bool freed = false; + ASSERT_TRUE(AllocationAccountTestAccess::rebind_for_tests(*test.budget, test.next, [&](Step step, size_t) { + if (step == Step::kClosed) { + allocator.deallocate(bytes, 149); + freed = true; + } + }).ok()); + EXPECT_TRUE(freed); + EXPECT_EQ(test.host.snapshot().reserved[kRouting], 0U); + } + grant.reset(); + test.expect_only_control(test.next); + test.finish(); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/allocation_account_interleaving_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_interleaving_test.cpp new file mode 100644 index 000000000..786b077fe --- /dev/null +++ b/lib/crowdb-tree/tests/unit/allocation_account_interleaving_test.cpp @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" + +#include <atomic> +#include <thread> + +namespace crowdb::tree::detail +{ +namespace +{ +using namespace test_allocation_account; + +void publish(std::atomic<bool> &flag) noexcept +{ + flag.store(true, std::memory_order_release); + flag.notify_one(); +} + +void await(std::atomic<bool> &flag) noexcept +{ + flag.wait(false, std::memory_order_acquire); +} + +TEST(AllocationAccountInterleaving, ActiveAccountAccessRejectsTransferWithoutSealing) +{ + TestAccounts test; + ASSERT_TRUE(test.budget->claim(59)); + std::atomic<bool> entered{false}; + std::atomic<bool> resume{false}; + std::thread writer([&] { + AllocationAccountTestAccess::release_for_tests(*test.budget, 59, [&](Step, size_t) noexcept { + publish(entered); + await(resume); + }); + }); + await(entered); + const auto status = test.budget->rebind(test.next); + EXPECT_EQ(status.code(), Code::kUnavailable); + EXPECT_EQ(test.next->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(test.old->snapshot().allocated[kRouting], test.base + 59); + const bool accepted = test.budget->claim(1); + EXPECT_TRUE(accepted); + if (accepted) { + test.budget->release(1); + } + publish(resume); + writer.join(); + ASSERT_TRUE(test.budget->rebind(test.next).ok()); + test.expect_only_control(test.next); + test.finish(); +} + +TEST(AllocationAccountInterleaving, ReopenWaitsForEnteredWriterBeforeAcceptingNewClaims) +{ + TestAccounts test; + ASSERT_TRUE(test.budget->claim(61)); + std::atomic<bool> release_started{false}; + std::atomic<bool> writer_entered{false}; + std::atomic<bool> resume_writer{false}; + std::atomic<bool> reopen_waiting{false}; + std::atomic<bool> completed{false}; + Status result; + std::thread writer([&] { + await(release_started); + AllocationAccountTestAccess::release_for_tests(*test.budget, 61, [&](Step, size_t) noexcept { + publish(writer_entered); + await(resume_writer); + }); + }); + std::thread transfer([&] { + bool injected = false; + result = + AllocationAccountTestAccess::rebind_for_tests(*test.budget, test.next, [&](Step step, size_t) noexcept { + if (!injected && step == Step::kBeforeReopen) { + injected = true; + publish(release_started); + await(writer_entered); + } + if (step == Step::kWaitForWriter) { + publish(reopen_waiting); + } + }); + publish(completed); + }); + await(reopen_waiting); + EXPECT_FALSE(completed.load(std::memory_order_acquire)); + EXPECT_FALSE(test.budget->claim(1)); + publish(resume_writer); + writer.join(); + transfer.join(); + ASSERT_TRUE(result.ok()) << result.to_string(); + EXPECT_TRUE(test.budget->claim(1)); + test.budget->release(1); + test.expect_only_control(test.next); + test.finish(); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/allocation_account_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_test.cpp new file mode 100644 index 000000000..39564898d --- /dev/null +++ b/lib/crowdb-tree/tests/unit/allocation_account_test.cpp @@ -0,0 +1,104 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" + +namespace crowdb::tree::detail +{ +namespace +{ +using namespace test_allocation_account; + +TEST(AllocationAccount, TransferAtFullHostCapacityMovesActualTreeOwnership) +{ + TestAccounts test; + { + BudgetAllocator<uint8_t, AllocationAccount, AllocationKind::kFrame> allocator(test.budget); + auto *bytes = allocator.allocate(128); + auto filler = test.host.make_account(limits(16384)); + auto capacity = ResourceBudget::try_reserve(filler, BudgetClass::kRouting, 16384 - test.base - 128); + ASSERT_TRUE(capacity); + EXPECT_FALSE(test.budget->claim(1)); + const auto before = test.host.snapshot(); + ASSERT_TRUE(test.budget->rebind(test.next).ok()); + EXPECT_EQ(test.host.snapshot().allocated[kRouting], before.allocated[kRouting]); + EXPECT_EQ(test.host.snapshot().reserved[kRouting], before.reserved[kRouting]); + EXPECT_EQ(test.old->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(test.old->snapshot().physical[kMetadata], 0U); + EXPECT_EQ(test.old->snapshot().physical[kFrame], 0U); + EXPECT_EQ(test.next->snapshot().allocated[kRouting], test.base + 128); + EXPECT_EQ(test.next->snapshot().physical[kFrame], 128U); + allocator.deallocate(bytes, 128); + capacity.reset(); + } + test.expect_only_control(test.next); + test.finish(); +} + +TEST(AllocationAccount, TargetQuotaFailureLeavesOriginalAccountUsable) +{ + TestAccounts test; + auto small = test.host.make_account(limits(test.base - 1)); + EXPECT_FALSE(test.budget->rebind(small).ok()); + test.expect_only_control(test.old); + EXPECT_EQ(small->snapshot().allocated[kRouting], 0U); + ASSERT_TRUE(test.budget->claim(17)); + test.budget->release(17); + ASSERT_TRUE(test.budget->rebind(test.next).ok()); + EXPECT_FALSE(test.budget->claim(8192 - test.base + 1)); + ASSERT_TRUE(test.budget->claim(8192 - test.base)); + test.budget->release(8192 - test.base); + test.finish(); +} + +TEST(AllocationAccount, ClaimRecordAndFreeReleaseMayStraddleBothTransfers) +{ + TestAccounts test; + ASSERT_TRUE(test.budget->claim(37)); + ASSERT_TRUE(test.budget->rebind(test.next).ok()); + EXPECT_EQ(test.next->snapshot().allocated[kRouting], test.base + 37); + EXPECT_EQ(test.next->snapshot().physical[kFrame], 0U); + test.budget->record_allocation(AllocationKind::kFrame, 37, true); + EXPECT_EQ(test.next->snapshot().physical[kFrame], 37U); + test.budget->record_allocation(AllocationKind::kFrame, 37, false); + ASSERT_TRUE(test.budget->rebind(test.old).ok()); + EXPECT_EQ(test.old->snapshot().allocated[kRouting], test.base + 37); + EXPECT_EQ(test.old->snapshot().physical[kFrame], 0U); + test.budget->release(37); + test.expect_only_control(test.old); + test.finish(); +} + +TEST(AllocationAccount, WeakControlRetainsItsActualChargeUntilFinalFree) +{ + TestAccounts test; + std::weak_ptr<AllocationAccount> weak = test.budget; + ASSERT_TRUE(test.budget->rebind(test.next).ok()); + test.budget.reset(); + EXPECT_TRUE(weak.expired()); + test.expect_only_control(test.next); + weak.reset(); + test.finish(); +} + +TEST(AllocationAccount, ForeignHostAndFailedBootstrapDoNotLeakAdmission) +{ + TestAccounts test; + ResourceBudget foreign(limits(16384), 1); + auto destination = foreign.make_account(limits(16384)); + EXPECT_FALSE(test.budget->rebind(destination).ok()); + test.expect_only_control(test.old); + EXPECT_EQ(foreign.snapshot().allocated[kRouting], 0U); + auto small = test.host.make_account(limits(sizeof(AllocationAccount))); + std::shared_ptr<AllocationAccount> result; + EXPECT_FALSE(AllocationAccount::create(small, BudgetClass::kRouting, &result).ok()); + EXPECT_FALSE(result); + EXPECT_EQ(small->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(small->snapshot().reserved[kRouting], 0U); + EXPECT_EQ(small->snapshot().physical[kMetadata], 0U); + EXPECT_FALSE(AllocationAccount::create(test.old, BudgetClass::kIo, &result).ok()); + EXPECT_FALSE(AllocationAccount::create(test.old, BudgetClass::kRouting, nullptr).ok()); + test.expect_only_control(test.old); + test.finish(); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/allocation_account_transfer_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_transfer_test.cpp new file mode 100644 index 000000000..def124180 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/allocation_account_transfer_test.cpp @@ -0,0 +1,127 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" + +#include <array> +#include <thread> + +namespace crowdb::tree::detail +{ +namespace +{ +using namespace test_allocation_account; + +TEST(AllocationAccountTransfer, ClosedDeltasFollowTheSuccessfulDestination) +{ + TestAccounts test; + ASSERT_TRUE(test.budget->claim(72)); + test.budget->record_allocation(AllocationKind::kFrame, 40, true); + const auto status = + AllocationAccountTestAccess::rebind_for_tests(*test.budget, test.next, [&](Step step, size_t) noexcept { + if (step == Step::kClosed) { + EXPECT_FALSE(test.budget->claim(1)); + test.budget->record_allocation(AllocationKind::kFrame, 32, true); + test.budget->record_allocation(AllocationKind::kFrame, 40, false); + test.budget->release(40); + } + if (step == Step::kTransferred) { + test.budget->record_allocation(AllocationKind::kFrame, 32, false); + test.budget->release(32); + } + }); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.expect_only_control(test.next); + test.finish(); +} + +TEST(AllocationAccountTransfer, AcquireAfterDrainCannotUnderflowOrLoseMatchingFree) +{ + TestAccounts test; + ASSERT_TRUE(test.budget->claim(29)); + bool injected = false; + const auto status = + AllocationAccountTestAccess::rebind_for_tests(*test.budget, test.next, [&](Step step, size_t kind) noexcept { + if (!injected && step == Step::kAcquiredDrained && kind == kFrame) { + injected = true; + test.budget->record_allocation(AllocationKind::kFrame, 29, true); + test.budget->record_allocation(AllocationKind::kFrame, 29, false); + test.budget->release(29); + } + }); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(injected); + test.expect_only_control(test.next); + test.finish(); +} + +TEST(AllocationAccountTransfer, FreeAfterLastDrainForcesAnotherPassBeforeReopen) +{ + TestAccounts test; + ASSERT_TRUE(test.budget->claim(31)); + test.budget->record_allocation(AllocationKind::kFrame, 31, true); + bool injected = false; + const auto status = + AllocationAccountTestAccess::rebind_for_tests(*test.budget, test.next, [&](Step step, size_t) noexcept { + if (!injected && step == Step::kBeforeReopen) { + injected = true; + test.budget->record_allocation(AllocationKind::kFrame, 31, false); + test.budget->release(31); + } + }); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(injected); + test.expect_only_control(test.next); + test.finish(); +} + +TEST(AllocationAccountTransfer, FailedTransferAppliesConcurrentFreesToOriginalAccount) +{ + TestAccounts test; + ASSERT_TRUE(test.budget->claim(43)); + test.budget->record_allocation(AllocationKind::kFrame, 43, true); + auto small = test.host.make_account(limits(test.base - 1)); + const auto status = + AllocationAccountTestAccess::rebind_for_tests(*test.budget, small, [&](Step step, size_t) noexcept { + if (step == Step::kClosed) { + test.budget->record_allocation(AllocationKind::kFrame, 43, false); + test.budget->release(43); + } + }); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(small->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(small->snapshot().physical[kMetadata], 0U); + test.expect_only_control(test.old); + ASSERT_TRUE(test.budget->claim(43)); + test.budget->release(43); + test.finish(); +} + +TEST(AllocationAccountTransfer, ConcurrentClosedWritersLeaveNoPendingCharges) +{ + TestAccounts test; + constexpr size_t count = 8; + constexpr size_t bytes = 47; + ASSERT_TRUE(test.budget->claim(count * bytes)); + const auto status = + AllocationAccountTestAccess::rebind_for_tests(*test.budget, test.next, [&](Step step, size_t) noexcept { + if (step != Step::kClosed) { + return; + } + std::array<std::thread, count> writers; + for (auto &writer : writers) { + writer = std::thread([&] { + test.budget->record_allocation(AllocationKind::kFrame, bytes, true); + test.budget->record_allocation(AllocationKind::kFrame, bytes, false); + test.budget->release(bytes); + }); + } + for (auto &writer : writers) { + writer.join(); + } + }); + ASSERT_TRUE(status.ok()) << status.to_string(); + test.expect_only_control(test.next); + test.finish(); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/allocation_observation_test.cpp b/lib/crowdb-tree/tests/unit/allocation_observation_test.cpp new file mode 100644 index 000000000..a7dbf5303 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/allocation_observation_test.cpp @@ -0,0 +1,120 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "crowdb-tree/external_allocation.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +namespace +{ +size_t observed(const std::shared_ptr<ResourceBudget::Account> &account, AllocationKind kind) +{ + return account->snapshot().physical[static_cast<size_t>(kind)]; +} + +class AllocationObservation : public testing::Test +{ + protected: + ResourceBudget budget{ + {4096, 4096, 4096, 4096, 4096, 2}, + 2 + }; + std::shared_ptr<ResourceBudget::Account> account = budget.make_account({4096, 4096, 4096, 4096, 4096, 2}); + + std::shared_ptr<AllocationReservation> reserve(BudgetClass category, size_t bytes) + { + auto credit = ResourceBudget::try_reserve(account, category, bytes); + if (!credit) { + ADD_FAILURE() << "test reservation must fit"; + return nullptr; + } + return AllocationReservation::create(std::move(*credit)); + } +}; + +TEST_F(AllocationObservation, SharedControlRemainsMetadataUntilTheFinalWeakOwnerReleases) +{ + auto owner = reserve(BudgetClass::kRouting, 1024); + ASSERT_TRUE(owner); + const auto bootstrap = observed(account, AllocationKind::kMetadata); + EXPECT_GT(bootstrap, 0U); + auto value = reserved_shared<uint64_t>(owner, 42); + const auto allocated = observed(account, AllocationKind::kMetadata); + EXPECT_GT(allocated, bootstrap + sizeof(uint64_t)); + std::weak_ptr<uint64_t> weak = value; + owner->seal(); + value.reset(); + EXPECT_EQ(observed(account, AllocationKind::kMetadata), allocated); + owner.reset(); + EXPECT_EQ(observed(account, AllocationKind::kMetadata), allocated); + weak.reset(); + EXPECT_EQ(observed(account, AllocationKind::kMetadata), 0U); + EXPECT_EQ(account->snapshot().allocated[static_cast<size_t>(BudgetClass::kRouting)], 0U); +} + +TEST_F(AllocationObservation, VectorCapacityAndFailedGrowthUseTheActualAllocationKind) +{ + auto owner = reserve(BudgetClass::kHandoff, 1024); + ASSERT_TRUE(owner); + using Allocator = ReservationAllocator<uint8_t, AllocationKind::kFrame>; + { + std::vector<uint8_t, Allocator> frame(96, Allocator(owner)); + EXPECT_EQ(observed(account, AllocationKind::kFrame), frame.capacity()); + frame.reserve(192); + EXPECT_EQ(observed(account, AllocationKind::kFrame), frame.capacity()); + const auto held = frame.capacity(); + EXPECT_THROW(frame.reserve(4096), std::bad_alloc); + EXPECT_EQ(frame.capacity(), held); + EXPECT_EQ(observed(account, AllocationKind::kFrame), held); + owner->seal(); + } + EXPECT_EQ(observed(account, AllocationKind::kFrame), 0U); + owner.reset(); + EXPECT_EQ(observed(account, AllocationKind::kMetadata), 0U); +} + +TEST_F(AllocationObservation, SharedExternalPayloadIsChargedOnceAndSurvivesReservationSeal) +{ + auto owner = reserve(BudgetClass::kRouting, 2048); + ASSERT_TRUE(owner); + auto *bytes = new uint8_t[512]; + auto allocation = ExternalAllocation::create( + bytes, 512, ExternalAllocation::Owner(bytes, [](void *value) { delete[] static_cast<uint8_t *>(value); }), + owner); + auto clone = allocation; + EXPECT_EQ(observed(account, AllocationKind::kPayload), 512U); + owner->seal(); + allocation.reset(); + owner.reset(); + EXPECT_EQ(observed(account, AllocationKind::kPayload), 512U); + clone.reset(); + EXPECT_EQ(observed(account, AllocationKind::kPayload), 0U); + EXPECT_EQ(observed(account, AllocationKind::kMetadata), 0U); +} + +TEST_F(AllocationObservation, ClassifiedCreditTransfersWithoutDoubleCounting) +{ + auto reserved = ResourceBudget::try_reserve(account, BudgetClass::kRouting, 512); + if (!reserved) { + ADD_FAILURE() << "reservation must fit"; + return; + } + auto allocation = reserved->allocate(128); + if (!allocation) { + ADD_FAILURE() << "allocation must fit reservation"; + return; + } + ASSERT_TRUE(allocation->mark_allocation(AllocationKind::kScratch)); + EXPECT_FALSE(allocation->mark_allocation(AllocationKind::kPayload)); + auto part = allocation->split(32); + ASSERT_TRUE(part); + EXPECT_EQ(observed(account, AllocationKind::kScratch), 128U); + allocation.reset(); + EXPECT_EQ(observed(account, AllocationKind::kScratch), 32U); + part.reset(); + EXPECT_EQ(observed(account, AllocationKind::kScratch), 0U); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/allocation_reservation_test.cpp b/lib/crowdb-tree/tests/unit/allocation_reservation_test.cpp new file mode 100644 index 000000000..46dfdd263 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/allocation_reservation_test.cpp @@ -0,0 +1,474 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <barrier> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +using detail::AllocationReservation; +using detail::BudgetClass; +using detail::ResourceBudget; +constexpr size_t kMetadata = static_cast<size_t>(BudgetClass::kHandoff); + +ResourceBudget::Limits limits(size_t bytes) +{ + ResourceBudget::Limits result{}; + result[kMetadata] = bytes; + return result; +} + +std::shared_ptr<AllocationReservation> reserve(ResourceBudget &budget, size_t bytes) +{ + auto credit = ResourceBudget::try_reserve(budget.make_account(limits(bytes)), BudgetClass::kHandoff, bytes); + if (!credit.has_value()) { + ADD_FAILURE() << "test metadata reservation was rejected"; + return {}; + } + return std::make_shared<AllocationReservation>(std::move(credit.value())); +} + +TEST(AllocationReservation, ActualAllocationTransfersCreditAndOutlivesSeal) +{ + ResourceBudget budget(limits(4096), 4); + auto reservation = reserve(budget, 4096); + auto value = detail::reserved_shared<std::array<uint64_t, 32>>(reservation); + const auto allocated = budget.snapshot().allocated[kMetadata]; + EXPECT_GE(allocated, sizeof(*value)); + EXPECT_EQ(budget.snapshot().reserved[kMetadata] + allocated, 4096U); + reservation->seal(); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 0U); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], allocated); + EXPECT_THROW(detail::reserved_shared<uint64_t>(reservation), std::bad_alloc); + reservation.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], allocated); + value.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, RepeatedChildWindowsReturnUnusedAndReleasedBytesToTheirParent) +{ + ResourceBudget budget(limits(4096), 4); + auto parent = reserve(budget, 4096); + for (size_t step = 0; step < 1000; ++step) { + auto child = AllocationReservation::create(parent, 1024); + auto value = detail::reserved_shared<std::array<uint64_t, 16>>(child); + child->seal(); + value.reset(); + child.reset(); + EXPECT_EQ(parent->available(), 4096U) << step; + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U) << step; + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 4096U) << step; + } +} + +TEST(AllocationReservation, ChildRetainsAcceptedCreditAcrossParentSealAndReturnsEveryLateFree) +{ + ResourceBudget budget(limits(4096), 4); + auto parent = reserve(budget, 4096); + auto child = AllocationReservation::create(parent, 1024); + parent->seal(); + auto value = detail::reserved_shared<std::array<uint64_t, 16>>(child); + child->seal(); + value.reset(); + child.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 0U); +} + +TEST(AllocationReservation, FailedChildControlAllocationRestoresTheOriginalWindow) +{ + ResourceBudget budget(limits(4096), 4); + auto parent = reserve(budget, 4096); + EXPECT_THROW((void)AllocationReservation::create(parent, 4096), std::bad_alloc); + EXPECT_EQ(parent->available(), 4096U); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 4096U); +} + +TEST(AllocationReservation, ParentSealRacesChildRefundWithoutLosingOrDuplicatingCredit) +{ + ResourceBudget budget(limits(4096), 4); + for (size_t pass = 0; pass < 128; ++pass) { + auto parent = reserve(budget, 4096); + auto child = AllocationReservation::create(parent, 1024); + auto value = detail::reserved_shared<std::array<uint64_t, 16>>(child); + std::barrier ready(2); + std::thread sealing([parent, &ready] { + ready.arrive_and_wait(); + parent->seal(); + }); + ready.arrive_and_wait(); + child->seal(); + value.reset(); + child.reset(); + sealing.join(); + parent.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U) << pass; + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 0U) << pass; + } +} + +TEST(AllocationReservation, BootstrapChargesItsControlBlockUntilLastWeakOwnerReleases) +{ + ResourceBudget budget(limits(4096), 4); + auto credit = ResourceBudget::try_reserve(budget.make_account(limits(4096)), BudgetClass::kHandoff, 4096); + if (!credit.has_value()) { + ADD_FAILURE() << "test bootstrap credit unavailable"; + return; + } + auto reservation = AllocationReservation::create(std::move(credit.value())); + const auto bootstrap = budget.snapshot().allocated[kMetadata]; + EXPECT_GE(bootstrap, sizeof(AllocationReservation)); + EXPECT_EQ(reservation->available() + bootstrap, 4096U); + std::weak_ptr<AllocationReservation> weak = reservation; + auto value = detail::reserved_shared<uint64_t>(reservation, 17); + reservation->seal(); + reservation.reset(); + EXPECT_FALSE(weak.expired()); + EXPECT_EQ(*value, 17U); + value.reset(); + EXPECT_TRUE(weak.expired()); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], bootstrap); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 0U); + weak.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, BootstrapFailureReturnsTheEntireOriginalCredit) +{ + ResourceBudget budget(limits(1), 4); + auto credit = ResourceBudget::try_reserve(budget.make_account(limits(1)), BudgetClass::kHandoff, 1); + if (!credit.has_value()) { + ADD_FAILURE() << "test bootstrap credit unavailable"; + return; + } + EXPECT_THROW((void)AllocationReservation::create(std::move(credit.value())), std::bad_alloc); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 0U); +} + +TEST(AllocationReservation, FailedAllocationReturnsReusableCreditBeforeSeal) +{ + ResourceBudget budget(limits(64), 4); + auto reservation = reserve(budget, 64); + detail::ReservationAllocator<uint64_t> allocator(reservation); + EXPECT_THROW((void)allocator.allocate(9), std::bad_alloc); + auto *memory = allocator.allocate(8); + EXPECT_EQ(reservation->available(), 0U); + EXPECT_THROW((void)allocator.allocate(1), std::bad_alloc); + allocator.deallocate(memory, 8); + EXPECT_EQ(reservation->available(), 64U); + memory = allocator.allocate(8); + allocator.deallocate(memory, 8); + reservation->seal(); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 0U); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, ConcurrentAllocationsAndSealReturnEveryByteOnce) +{ + ResourceBudget budget(limits(8192), 4); + auto reservation = reserve(budget, 8192); + std::barrier ready(5); + std::vector<std::jthread> workers; + workers.reserve(4); + for (size_t i = 0; i < 4; ++i) { + workers.emplace_back([&] { + detail::ReservationAllocator<uint64_t> allocator(reservation); + auto *memory = allocator.allocate(64); + ready.arrive_and_wait(); + allocator.deallocate(memory, 64); + }); + } + ready.arrive_and_wait(); + reservation->seal(); + workers.clear(); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 0U); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, DetachedProgressCreditRemainsReservedAcrossParentSeal) +{ + ResourceBudget budget(limits(4096), 4); + auto reservation = reserve(budget, 4096); + auto progress = reservation->take_reserved(1024); + if (!progress) { + FAIL() << "progress credit was not split"; + } + EXPECT_EQ(reservation->available(), 3072U); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 4096U); + EXPECT_FALSE(reservation->take_reserved(3073).has_value()); + reservation->seal(); + EXPECT_FALSE(reservation->take_reserved(1).has_value()); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 1024U); + auto child = std::make_shared<AllocationReservation>(std::move(progress.value())); + auto value = detail::reserved_shared<std::array<char, 512>>(child); + child->seal(); + EXPECT_EQ(budget.snapshot().reserved[kMetadata], 0U); + EXPECT_GE(budget.snapshot().allocated[kMetadata], 512U); + value.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, ImmutableRangeCopiesRetainOneChargedBinaryKeyAllocation) +{ + ResourceBudget budget(limits(16384), 4); + auto reservation = reserve(budget, 16384); + std::string bytes(8192, 'k'); + bytes[31] = '\0'; + PageKeyRange range{.lower = OwnedKey(Slice(bytes), reservation)}; + const auto allocated = budget.snapshot().allocated[kMetadata]; + EXPECT_GT(allocated, bytes.size()); + reservation->seal(); + PageKeyRange copy(range); + if (!copy.lower || !range.lower) { + FAIL() << "range copy lost its lower bound"; + } + EXPECT_EQ(copy.lower->data(), range.lower->data()); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], allocated); + range = {}; + EXPECT_EQ(Slice(*copy.lower).to_string(), bytes); + EXPECT_TRUE(copy.contains(Slice(bytes))); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], allocated); + copy = {}; + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, NewOwnedKeyFailsBeforeUnreservedCapacityAndPreservesOldKey) +{ + ResourceBudget budget(limits(1024), 4); + auto reservation = reserve(budget, 1024); + OwnedKey key(Slice(std::string(256, 'a')), reservation); + const auto allocated = budget.snapshot().allocated[kMetadata]; + EXPECT_THROW(key = OwnedKey(Slice(std::string(1024, 'b')), reservation), std::bad_alloc); + EXPECT_EQ(Slice(key).to_string(), std::string(256, 'a')); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], allocated); + OwnedKey moved(std::move(key)); + EXPECT_EQ(moved.size(), 256U); +} + +TEST(AllocationReservation, SharedRoutingSeparatorSurvivesIndexOwnerRelease) +{ + auto inner = std::make_shared<InnerPageState>(); + inner->separators.emplace_back(std::string(8192, 'm')); + const auto *address = inner->separators.front().data(); + std::weak_ptr<const InnerPageState> old = inner; + std::optional<OwnedKey> boundary(inner->separators.front()); + inner.reset(); + EXPECT_TRUE(old.expired()); + EXPECT_EQ(boundary->data(), address); + EXPECT_EQ(Slice(*boundary).to_string(), std::string(8192, 'm')); + boundary.reset(); + EXPECT_TRUE(old.expired()); + PageKeyRange empty_lower{.lower = OwnedKey("")}; + EXPECT_TRUE(empty_lower.lower.has_value()); + EXPECT_TRUE(empty_lower.contains("")); + EXPECT_FALSE(PageKeyRange{}.lower.has_value()); +} + +TEST(AllocationReservation, PrivateHandleControlBlockFailureDestroysTheConstructedObject) +{ + struct Handle + { + size_t *destroyed; + + ~Handle() + { + ++*destroyed; + } + }; + + ResourceBudget budget(limits(sizeof(Handle)), 4); + auto reservation = reserve(budget, sizeof(Handle)); + size_t destroyed = 0; + EXPECT_THROW((void)detail::reserved_construct<Handle>( + reservation, + [&](Handle *storage) { ::new (static_cast<void *>(storage)) Handle{.destroyed = &destroyed}; }), + std::bad_alloc); + EXPECT_EQ(destroyed, 1U); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); + EXPECT_EQ(reservation->available(), sizeof(Handle)); +} + +TEST(AllocationReservation, PublicationArraysKeepTheirBudgetAfterTransactionAssignment) +{ + ResourceBudget budget(limits(8192), 4); + auto reservation = reserve(budget, 8192); + auto request = detail::reserved_shared<PublicationRequest>(reservation, reservation); + request->transaction = {.expected_coverage = 0, .metadata = reservation}; + const auto before = budget.snapshot().allocated[kMetadata]; + request->transaction.pages.resize(16); + request->transaction.required_installs.resize(32); + EXPECT_GE(budget.snapshot().allocated[kMetadata] - before, + (16 * sizeof(PageReplacement)) + (32 * sizeof(std::shared_ptr<InstallRecord>))); + const auto *original = request->transaction.pages.data(); + EXPECT_THROW(request->transaction.pages.reserve(8192), std::bad_alloc); + EXPECT_EQ(request->transaction.pages.data(), original); + reservation->seal(); + EXPECT_GT(budget.snapshot().allocated[kMetadata], before); + request.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, InlineCallbacksRetainCapturedOwnersAfterSealAndReleaseTheLastCopy) +{ + ResourceBudget budget(limits(4096), 4); + auto reservation = reserve(budget, 4096); + auto value = detail::reserved_shared<uint64_t>(reservation, 10); + const auto allocated = budget.snapshot().allocated[kMetadata]; + reservation->seal(); + detail::InlineFunction<uint64_t(uint64_t)> callback([value](uint64_t delta) { return *value + delta; }); + auto copy = callback; + auto moved = std::move(callback); + value.reset(); + EXPECT_EQ(copy(3), 13U); + EXPECT_EQ(moved(4), 14U); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], allocated); + copy.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], allocated); + moved.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); + EXPECT_THROW(moved(1), std::bad_function_call); +} + +TEST(AllocationReservation, InlineCallbackCopyFailurePreservesThePreviousTarget) +{ + struct ThrowingCopy + { + bool *fail; + + explicit ThrowingCopy(bool *flag) : fail(flag) + { + } + + ThrowingCopy(ThrowingCopy &&) noexcept = default; + + ThrowingCopy(const ThrowingCopy &other) : fail(other.fail) + { + if (*fail) { + throw std::bad_alloc(); + } + } + + int operator()(int value) const + { + return value + 10; + } + }; + + bool fail = false; + detail::InlineFunction<int(int)> source{ThrowingCopy(&fail)}; + detail::InlineFunction<int(int)> target([](int value) { return value + 1; }); + fail = true; + EXPECT_THROW(target = source, std::bad_alloc); + EXPECT_EQ(target(5), 6); + EXPECT_EQ(source(5), 15); + target = std::move(source); + EXPECT_EQ(target(5), 15); + + struct Oversized + { + std::array<char, 256> data; + + void operator()() const + { + } + }; + + static_assert(!std::is_constructible_v<detail::InlineFunction<void()>, Oversized>); +} + +TEST(AllocationReservation, MappingOldViewRetainsItsChargeAfterReplacement) +{ + ResourceBudget budget(limits(65536), 4); + auto reservation = reserve(budget, 65536); + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + std::array changes{ + PageReplacement{.page_id = 0, .expected = {}, .replacement = page} + }; + MappingView::CopyStats copied; + auto before = MappingView{}.replace(changes, copied, reservation); + const auto first_bytes = budget.snapshot().allocated[kMetadata]; + EXPECT_LE(first_bytes, MappingView::path_copy_bound()); + changes[0].expected = page; + changes[0].replacement = std::make_shared<PublishedPage>(*page); + auto after = before.replace(changes, copied, reservation); + reservation->seal(); + EXPECT_GT(budget.snapshot().allocated[kMetadata], first_bytes); + after = {}; + EXPECT_EQ(budget.snapshot().allocated[kMetadata], first_bytes); + before = {}; + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, PublicationBudgetFailureDoesNotCommitOrDetach) +{ + ResourceBudget budget(limits(64), 4); + auto reservation = reserve(budget, 64); + PublicationCoordinator publisher; + auto before = publisher.capture(); + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + auto record = std::make_shared<InstallRecord>(InstallToken{}); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = record; + request->transaction.pages.push_back({.page_id = 0, .expected = {}, .replacement = page}); + request->transaction.root_page_id = 0; + request->transaction.metadata = reservation; + bool completed = false; + request->on_done = [&](PublicationResult result) { + EXPECT_EQ(result.status.code(), Code::kResourceExhausted); + completed = true; + }; + ASSERT_TRUE(publisher.try_submit(request)); + EXPECT_TRUE(completed); + EXPECT_EQ(publisher.capture(), before); + EXPECT_FALSE(record->committed.load()); + EXPECT_EQ(reservation->available(), 64U); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} + +TEST(AllocationReservation, OverlayKeepsMetadataChargedAndSharesLargePayload) +{ + ResourceBudget budget(limits(4096), 4); + auto reservation = reserve(budget, 4096); + auto table = std::make_shared<MemTable>(); + ASSERT_TRUE(table->upsert("key", 1, encode_cell_buf(1, OpKind::kPut, std::string(16 * 1024, 'x')))); + std::shared_ptr<const OverlayBatch> overlay; + const void *payload; + { + auto guard = table->read_guard(); + auto cursor = table->prefix_cursor(1); + auto owner = cursor.cell_owner(); + payload = owner->cell.data() + kCellHeaderSize; + OverlayBuilder builder({.max_entries = 16, .max_key_bytes = 1024, .key_block_bytes = 128}, reservation); + ASSERT_TRUE(builder.append(cursor.key(), owner).ok()); + ASSERT_TRUE(builder.finish(1, &overlay).ok()); + } + table.reset(); + reservation->seal(); + EXPECT_GT(budget.snapshot().allocated[kMetadata], 128U); + EXPECT_LT(budget.snapshot().allocated[kMetadata], 4096U); + EXPECT_EQ(overlay->entries().front().value().data(), payload); + EXPECT_EQ(overlay->entries().front().value().size(), 16 * 1024U); + auto page = std::make_shared<PageState>(); + page->overlays.push_back(overlay); + auto copy = std::make_shared<PageState>(*page); + EXPECT_EQ(copy->overlays.front(), overlay); + overlay.reset(); + page.reset(); + copy.reset(); + EXPECT_EQ(budget.snapshot().allocated[kMetadata], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/apply_dispatcher_test.cpp b/lib/crowdb-tree/tests/unit/apply_dispatcher_test.cpp new file mode 100644 index 000000000..f958e6063 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/apply_dispatcher_test.cpp @@ -0,0 +1,157 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/apply_dispatcher.h" +#include "load_budget_test_support.h" + +#include <array> +#include <latch> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct TestDispatcherTask +{ + TestDispatcherTask(std::atomic<size_t> &completed, std::atomic<size_t> &destroyed) + : completed(completed), + destroyed(destroyed), + node(&TestDispatcherTask::run, this) + { + } + + ~TestDispatcherTask() + { + destroyed.fetch_add(1); + } + + static void run(void *context) noexcept + { + static_cast<TestDispatcherTask *>(context)->completed.fetch_add(1); + } + + std::atomic<size_t> &completed; + std::atomic<size_t> &destroyed; + ApplyDispatcher::Node node; +}; + +TEST(ApplyDispatcher, ConcurrentProducersRetainAndExecuteEachQueuedOwnerOnce) +{ + test_read::LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(2, 4); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + std::shared_ptr<ApplyDispatcher> dispatcher; + ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 3, credits.credits.bytes, &dispatcher).ok()); + credits.seal(); + std::atomic<size_t> completed{0}; + std::atomic<size_t> destroyed{0}; + std::array<std::jthread, 8> producers; + for (auto &producer : producers) { + producer = std::jthread([&] { + for (size_t index = 0; index < 128; ++index) { + auto owner = std::make_shared<TestDispatcherTask>(completed, destroyed); + dispatcher->request(owner->node, owner); + } + }); + } + for (auto &producer : producers) { + producer.join(); + } + dispatcher->close_and_drain(); + lane->drain(); + EXPECT_EQ(completed.load(), 1024U); + EXPECT_EQ(destroyed.load(), 1024U); + dispatcher.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], 0U); +} + +TEST(ApplyDispatcher, ReentrantContinuationKeepsFifoOrderAcrossBoundedTurns) +{ + test_read::LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 2); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + std::shared_ptr<ApplyDispatcher> dispatcher; + ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 1, credits.credits.bytes, &dispatcher).ok()); + + struct Context : std::enable_shared_from_this<Context> + { + explicit Context(std::shared_ptr<ApplyDispatcher> target) + : dispatcher(std::move(target)), + first(&Context::run_first, this), + second(&Context::run_second, this) + { + } + + static void run_first(void *argument) noexcept + { + auto &self = *static_cast<Context *>(argument); + self.order[self.count++] = 1; + if (self.count == 1) { + self.dispatcher->request(self.second, self.weak_from_this().lock()); + self.dispatcher->request(self.first, self.weak_from_this().lock()); + } + } + + static void run_second(void *argument) noexcept + { + auto &self = *static_cast<Context *>(argument); + self.order[self.count++] = 2; + } + + std::shared_ptr<ApplyDispatcher> dispatcher; + ApplyDispatcher::Node first; + ApplyDispatcher::Node second; + std::array<size_t, 3> order{}; + size_t count = 0; + }; + + auto owner = std::make_shared<Context>(dispatcher); + dispatcher->request(owner->first, owner); + dispatcher->close_and_drain(); + EXPECT_EQ(owner->count, 3U); + EXPECT_EQ(owner->order, (std::array<size_t, 3>{1, 2, 1})); + EXPECT_GE(runtime->snapshot().completed, 3U); +} + +TEST(ApplyDispatcher, DuplicateWakeupsCoalesceAndPrepaidProgressSurvivesRuntimeClose) +{ + test_read::LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 2); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + std::shared_ptr<ApplyDispatcher> dispatcher; + ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 1, credits.credits.bytes, &dispatcher).ok()); + + struct Gate + { + std::latch entered{1}; + std::latch release{1}; + } gate; + + ASSERT_TRUE(runtime + ->try_submit( + std::make_shared<detail::FlushRuntime::Lane>(1), + [](void *context) noexcept { + auto &state = *static_cast<Gate *>(context); + state.entered.count_down(); + state.release.wait(); + }, + &gate) + .ok()); + gate.entered.wait(); + runtime->close(); + std::atomic<size_t> completed{0}; + std::atomic<size_t> destroyed{0}; + auto owner = std::make_shared<TestDispatcherTask>(completed, destroyed); + for (size_t wakeup = 0; wakeup < 32; ++wakeup) { + dispatcher->request(owner->node, owner); + } + owner.reset(); + EXPECT_EQ(destroyed.load(), 0U); + gate.release.count_down(); + dispatcher->close_and_drain(); + runtime->drain(); + EXPECT_EQ(completed.load(), 1U); + EXPECT_EQ(destroyed.load(), 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/apply_frontier_test.cpp b/lib/crowdb-tree/tests/unit/apply_frontier_test.cpp new file mode 100644 index 000000000..533deb16a --- /dev/null +++ b/lib/crowdb-tree/tests/unit/apply_frontier_test.cpp @@ -0,0 +1,134 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/apply_frontier.h" +#include "load_budget_test_support.h" + +#include <thread> + +namespace crowdb::tree +{ +using test_read::LoadCredits; + +TEST(ApplyFrontier, NoOpCannotSkipEarlierIncompleteSlots) +{ + std::shared_ptr<ApplyFrontier> tracker; + ASSERT_TRUE(ApplyFrontier::create(0, 4, {}, &tracker).ok()); + ASSERT_TRUE(tracker->complete(3).ok()); + EXPECT_EQ(tracker->contiguous().load(), 0U); + EXPECT_EQ(tracker->max_seen(), 3U); + ASSERT_TRUE(tracker->complete(1).ok()); + EXPECT_EQ(tracker->contiguous().load(), 1U); + ASSERT_TRUE(tracker->complete(2).ok()); + EXPECT_EQ(tracker->contiguous().load(), 3U); +} + +TEST(ApplyFrontier, FullGapWindowRejectsBeforeMutationButStillAdmitsGapRepair) +{ + std::shared_ptr<ApplyFrontier> tracker; + ASSERT_TRUE(ApplyFrontier::create(10, 4, {}, &tracker).ok()); + for (uint64_t slot = 12; slot <= 14; ++slot) { + ASSERT_TRUE(tracker->complete(slot).ok()); + } + EXPECT_EQ(tracker->check_slot(15).code(), Code::kResourceExhausted); + EXPECT_EQ(tracker->complete(15).code(), Code::kResourceExhausted); + EXPECT_EQ(tracker->max_seen(), 14U); + ASSERT_TRUE(tracker->complete(11).ok()); + EXPECT_EQ(tracker->contiguous().load(), 14U); + EXPECT_TRUE(tracker->check_slot(15).ok()); +} + +TEST(ApplyFrontier, LateReplaysCannotEraseNewCompletionsAfterRingReuse) +{ + std::shared_ptr<ApplyFrontier> tracker; + ASSERT_TRUE(ApplyFrontier::create(0, 4, {}, &tracker).ok()); + for (uint64_t first = 1; first <= 1000; first += 4) { + for (uint64_t offset : {3U, 2U, 1U}) { + ASSERT_TRUE(tracker->complete(first + offset).ok()); + } + if (first > 4) { + for (uint64_t old = first - 4; old < first; ++old) { + ASSERT_TRUE(tracker->complete(old).ok()); + } + } + EXPECT_EQ(tracker->contiguous().load(), first - 1); + ASSERT_TRUE(tracker->complete(first).ok()); + EXPECT_EQ(tracker->contiguous().load(), first + 3); + } +} + +TEST(ApplyFrontier, ConcurrentCompletionsPublishTheirWritesBeforeTheFrontier) +{ + std::shared_ptr<ApplyFrontier> tracker; + ASSERT_TRUE(ApplyFrontier::create(0, 256, {}, &tracker).ok()); + std::array<uint64_t, 256> payload{}; + std::vector<std::jthread> workers; + workers.reserve(4); + for (size_t worker = 0; worker < 4; ++worker) { + workers.emplace_back([&, worker] { + for (size_t i = worker; i < payload.size(); i += 4) { + payload[i] = i + 1; + EXPECT_TRUE(tracker->complete(i + 1).ok()); + } + }); + } + uint64_t checked = 0; + while (checked != payload.size()) { + const auto frontier = tracker->contiguous().load(std::memory_order_acquire); + while (checked < frontier) { + EXPECT_EQ(payload[checked], checked + 1); + ++checked; + } + std::this_thread::yield(); + } + workers.clear(); + EXPECT_EQ(tracker->max_seen(), payload.size()); +} + +TEST(ApplyFrontier, ExplicitRecoveryAdvanceFoldsCompletedSuffixWithoutOverflow) +{ + std::shared_ptr<ApplyFrontier> tracker; + ASSERT_TRUE(ApplyFrontier::create(UINT64_MAX - 5, 4, {}, &tracker).ok()); + EXPECT_EQ(tracker->check_slot(UINT64_MAX).code(), Code::kResourceExhausted); + ASSERT_TRUE(tracker->complete(UINT64_MAX - 2).ok()); + tracker->force_advance(UINT64_MAX - 3); + EXPECT_EQ(tracker->contiguous().load(), UINT64_MAX - 2); + ASSERT_TRUE(tracker->complete(UINT64_MAX).ok()); + ASSERT_TRUE(tracker->complete(UINT64_MAX - 1).ok()); + EXPECT_EQ(tracker->contiguous().load(), UINT64_MAX); + tracker->force_advance(0); + EXPECT_TRUE(tracker->complete(1).ok()); + EXPECT_EQ(tracker->contiguous().load(), UINT64_MAX); +} + +TEST(ApplyFrontier, FixedStorageIsChargedOnceAndCompletionsWorkAfterSealing) +{ + LoadCredits credits; + std::shared_ptr<ApplyFrontier> tracker; + ASSERT_TRUE(ApplyFrontier::create(0, 8, credits.credits.bytes, &tracker).ok()); + const auto allocated = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + EXPECT_GE(allocated, sizeof(ApplyFrontier) + (8 * sizeof(std::atomic<uint64_t>))); + const auto bound = ApplyFrontier::allocation_bound(8); + ASSERT_TRUE(bound.has_value()); + EXPECT_LE(allocated, bound.value_or(0)); + credits.seal(); + for (uint64_t slot = 1; slot <= 100; ++slot) { + ASSERT_TRUE(tracker->complete(slot).ok()); + } + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], allocated); + tracker.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(ApplyFrontier, InvalidWindowAndAllocationFailureLeaveOutputUnchanged) +{ + LoadCredits credits; + std::shared_ptr<ApplyFrontier> tracker; + ASSERT_TRUE(ApplyFrontier::create(0, 4, {}, &tracker).ok()); + const auto original = tracker; + EXPECT_EQ(ApplyFrontier::create(0, 0, {}, &tracker).code(), Code::kInvalidArgument); + EXPECT_EQ(ApplyFrontier::create(0, SIZE_MAX, {}, &tracker).code(), Code::kInvalidArgument); + AllocationLease pressure(credits.credits.bytes, credits.credits.bytes->available()); + EXPECT_EQ(ApplyFrontier::create(0, 4, credits.credits.bytes, &tracker).code(), Code::kResourceExhausted); + EXPECT_EQ(tracker, original); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/apply_progress_test.cpp b/lib/crowdb-tree/tests/unit/apply_progress_test.cpp new file mode 100644 index 000000000..74d78ad38 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/apply_progress_test.cpp @@ -0,0 +1,168 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/apply_progress.h" +#include "load_budget_test_support.h" + +#include <barrier> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct TestProgressResult +{ + std::atomic<size_t> values{0}; + std::atomic<size_t> errors{0}; + std::atomic<size_t> stopped{0}; +}; + +struct TestProgressReceiver +{ + using receiver_concept = stdexec::receiver_tag; + TestProgressResult *result; + + void set_value() const noexcept + { + result->values.fetch_add(1); + } + + void set_error(Status /*unused*/) const noexcept + { + result->errors.fetch_add(1); + } + + void set_stopped() const noexcept + { + result->stopped.fetch_add(1); + } +}; + +TEST(ApplyProgress, WakeupCoversTransitionsBeforeAndAfterRegistration) +{ + test_read::LoadCredits credits; + std::shared_ptr<ApplyProgress> progress; + ASSERT_TRUE(ApplyProgress::create(1, credits.credits.bytes, &progress).ok()); + const auto initial = progress->sequence(); + ApplyProgress::Request before(*progress, initial); + TestProgressResult first; + auto op_first = stdexec::connect(before.sender(), TestProgressReceiver{&first}); + progress->advance(); + stdexec::start(op_first); + EXPECT_EQ(first.values.load(), 1U); + ApplyProgress::Request after(*progress, progress->sequence()); + TestProgressResult second; + auto op_second = stdexec::connect(after.sender(), TestProgressReceiver{&second}); + stdexec::start(op_second); + EXPECT_EQ(second.values.load(), 0U); + progress->advance(); + progress->advance(); + EXPECT_EQ(second.values.load(), 1U); + EXPECT_EQ(first.errors.load() + second.errors.load(), 0U); +} + +TEST(ApplyProgress, BoundedWaitersReturnTheirSlotsOnCancellation) +{ + test_read::LoadCredits credits; + std::shared_ptr<ApplyProgress> progress; + ASSERT_TRUE(ApplyProgress::create(1, credits.credits.bytes, &progress).ok()); + ApplyProgress::Request first(*progress, progress->sequence()); + ApplyProgress::Request overflow(*progress, progress->sequence()); + TestProgressResult a; + TestProgressResult b; + auto op_a = stdexec::connect(first.sender(), TestProgressReceiver{&a}); + auto op_b = stdexec::connect(overflow.sender(), TestProgressReceiver{&b}); + stdexec::start(op_a); + stdexec::start(op_b); + EXPECT_EQ(b.errors.load(), 1U); + first.request_stop(); + first.request_stop(); + EXPECT_EQ(a.stopped.load(), 1U); + ApplyProgress::Request replacement(*progress, progress->sequence()); + TestProgressResult c; + auto op_c = stdexec::connect(replacement.sender(), TestProgressReceiver{&c}); + stdexec::start(op_c); + progress->close(); + progress->advance(); + EXPECT_EQ(c.stopped.load(), 1U); + EXPECT_EQ(c.errors.load() + c.values.load(), 0U); +} + +TEST(ApplyProgress, LastIngressOwnerStopsPendingRequestsAndPreservesTheirChargedState) +{ + test_read::LoadCredits credits; + std::shared_ptr<ApplyProgress> progress; + ASSERT_TRUE(ApplyProgress::create(2, credits.credits.bytes, &progress).ok()); + auto request = std::make_unique<ApplyProgress::Request>(*progress, progress->sequence()); + TestProgressResult result; + auto operation = stdexec::connect(request->sender(), TestProgressReceiver{&result}); + stdexec::start(operation); + credits.seal(); + progress.reset(); + EXPECT_EQ(result.stopped.load(), 1U); + EXPECT_GT(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], 0U); + request.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], 0U); +} + +struct TestDestroyingReceiver +{ + using receiver_concept = stdexec::receiver_tag; + std::unique_ptr<ApplyProgress::Request> *request; + TestProgressResult *result; + + void set_value() const noexcept + { + result->values.fetch_add(1); + request->reset(); + } + + void set_error(Status /*unused*/) const noexcept + { + result->errors.fetch_add(1); + } + + void set_stopped() const noexcept + { + result->stopped.fetch_add(1); + } +}; + +TEST(ApplyProgress, TerminalMayDestroyTheRequestInline) +{ + test_read::LoadCredits credits; + std::shared_ptr<ApplyProgress> progress; + ASSERT_TRUE(ApplyProgress::create(1, credits.credits.bytes, &progress).ok()); + auto request = std::make_unique<ApplyProgress::Request>(*progress, progress->sequence()); + TestProgressResult result; + auto operation = + stdexec::connect(request->sender(), TestDestroyingReceiver{.request = &request, .result = &result}); + stdexec::start(operation); + progress->advance(); + EXPECT_FALSE(request); + EXPECT_EQ(result.values.load(), 1U); +} + +TEST(ApplyProgress, ConcurrentRegistrationAndTransitionNeverLoseTheWakeup) +{ + test_read::LoadCredits credits; + std::shared_ptr<ApplyProgress> progress; + ASSERT_TRUE(ApplyProgress::create(1, credits.credits.bytes, &progress).ok()); + for (size_t iteration = 0; iteration < 64; ++iteration) { + ApplyProgress::Request request(*progress, progress->sequence()); + TestProgressResult result; + auto operation = stdexec::connect(request.sender(), TestProgressReceiver{&result}); + std::barrier start(2); + std::jthread transition([&] { + start.arrive_and_wait(); + progress->advance(); + }); + start.arrive_and_wait(); + stdexec::start(operation); + transition.join(); + EXPECT_EQ(result.values.load(), 1U); + EXPECT_EQ(result.errors.load() + result.stopped.load(), 0U); + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/backend_factory_test.cpp b/lib/crowdb-tree/tests/unit/backend_factory_test.cpp new file mode 100644 index 000000000..62968cb31 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/backend_factory_test.cpp @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "crowdb-tree/backend/page_store.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +class FactoryPageStore final : public PageStore +{ + public: + static Status open(const Config &config, std::shared_ptr<TreeCore> *out) + { + if (config.page_store == nullptr || out == nullptr || *out) { + return Status::invalid_argument(); + } + return Status::unavailable("injected backend factory"); + } + + [[nodiscard]] RuntimeFactory runtime_factory() const noexcept override + { + return &open; + } + + Status write_at(uint64_t /*off*/, const uint8_t * /*buf*/, size_t /*len*/) override + { + return Status::internal_error(); + } + + Status read_at(uint64_t /*off*/, uint8_t * /*buf*/, size_t /*len*/) const override + { + return Status::internal_error(); + } + + Status sync() override + { + return Status::internal_error(); + } + + [[nodiscard]] uint64_t size() const override + { + return 0; + } + + [[nodiscard]] uint32_t iu_size() const override + { + return 1; + } +}; +} // namespace + +TEST(BackendFactory, InjectedFactoryReceivesRuntimeOpenWithoutByteRecovery) +{ + FactoryPageStore store; + Config config; + config.page_store = &store; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + std::shared_ptr<TreeCore> core; + const auto status = TreeCore::open(config, &core); + EXPECT_EQ(status.code(), Status::unavailable().code()); + EXPECT_EQ(status.message(), "injected backend factory"); + EXPECT_FALSE(core); +} + +TEST(BackendFactory, LocalStoreDoesNotAdvertiseAnImmutableAuthority) +{ + MemPageStore store(1); + EXPECT_EQ(store.runtime_factory(), nullptr); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/backend_release_c_api_test.cpp b/lib/crowdb-tree/tests/unit/backend_release_c_api_test.cpp new file mode 100644 index 000000000..e1dd2d015 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/backend_release_c_api_test.cpp @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "c_api_internal.h" +#include "crowdb-tree/backend/backend_release_service.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "crowdb-tree/backend_release_c_api.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct TestWake +{ + std::atomic<size_t> wakes{0}; + std::atomic<size_t> drops{0}; + + static void wake(void *p) + { + static_cast<TestWake *>(p)->wakes.fetch_add(1); + } + + static void drop(void *p) + { + static_cast<TestWake *>(p)->drops.fetch_add(1); + } +}; + +TEST(BackendReleaseCAbi, HostHandleOutlivesTreeAndDropsContextOnlyAfterDisposal) +{ + tree_test::TempDir path("block_release_abi_"); + ct_options options{}; + options.path = path.path.c_str(); + options.backend = CT_BACKEND_BLOCK; + options.iu_size = 4096; + options.frame_bytes = 4096; + options.block_size = 1 << 20; + options.buffer_pool_bytes = 1 << 20; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + TestWake wake; + ct_backend_release_service *service = nullptr; + ASSERT_EQ(ct_tree_backend_release_service(tree, TestWake::wake, TestWake::drop, &wake, &service), 0); + TestWake refused; + ct_backend_release_service *duplicate = nullptr; + EXPECT_NE(ct_tree_backend_release_service(tree, TestWake::wake, TestWake::drop, &refused, &duplicate), 0); + EXPECT_EQ(refused.drops.load(), 0U); + EXPECT_EQ(duplicate, nullptr); + std::thread callback([tree] { ct_close(tree); }); + callback.join(); + EXPECT_EQ(wake.drops.load(), 0U); + size_t work = 0; + bool drained = false; + while (work++ < 1024) { + const auto step = ct_backend_release_service_pump(service, 1); + EXPECT_LE(step.visited, 1U); + if (step.drained != 0) { + drained = true; + break; + } + } + EXPECT_TRUE(drained); + ct_backend_release_service_free(service); + EXPECT_EQ(wake.drops.load(), 1U); + EXPECT_GT(wake.wakes.load(), 0U); +} + +TEST(BackendReleaseCAbi, UnsupportedBackendDoesNotConsumeWakeContext) +{ + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + TestWake wake; + ct_backend_release_service *service = nullptr; + EXPECT_EQ(ct_tree_backend_release_service(tree, TestWake::wake, TestWake::drop, &wake, &service), + static_cast<ct_status>(Code::kNotFound)); + EXPECT_EQ(service, nullptr); + EXPECT_EQ(wake.drops.load(), 0U); + ct_close(tree); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/backend_release_service_test.cpp b/lib/crowdb-tree/tests/unit/backend_release_service_test.cpp new file mode 100644 index 000000000..0c3baf313 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/backend_release_service_test.cpp @@ -0,0 +1,187 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/backend/backend_release_service.h" +#include "runtime/bootstrap_budget.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <latch> +#include <thread> +#include <vector> + +namespace crowdb::tree +{ +namespace +{ +struct TestRelease : BackendReleaseService::Node +{ + std::shared_ptr<AllocationBudget> budget; + std::atomic<size_t> *completed; + size_t remaining = 0; + std::latch *entered = nullptr; + std::latch *proceed = nullptr; + + TestRelease(std::shared_ptr<AllocationBudget> credit, std::atomic<size_t> *done, size_t steps) + : budget(std::move(credit)), + completed(done), + remaining(steps) + { + dispose = [](BackendReleaseService::Node *base) noexcept { + auto *node = static_cast<TestRelease *>(base); + if (node->remaining != 0) { + --node->remaining; + return false; + } + if (node->entered != nullptr) { + node->entered->count_down(); + node->proceed->wait(); + } + auto allocator = MetadataAllocator<TestRelease>(node->budget); + node->completed->fetch_add(1); + std::destroy_at(node); + allocator.deallocate(node, 1); + return true; + }; + } +}; + +struct TestReleaseHost +{ + std::shared_ptr<detail::BootstrapBudget> budget = detail::BootstrapBudget::create(1024 * 1024); + std::shared_ptr<BackendReleaseService> service; + std::atomic<size_t> completed{0}; + + explicit TestReleaseHost(size_t capacity = 64) + { + EXPECT_TRUE(BackendReleaseService::create(budget, capacity, {}, nullptr, nullptr, &service).ok()); + } + + TestRelease *prepare(size_t steps = 0) + { + MetadataAllocator<TestRelease> allocator(budget); + auto *node = allocator.allocate(1); + std::construct_at(node, budget, &completed, steps); + auto status = service->register_node(*node); + if (!status.ok()) { + std::destroy_at(node); + allocator.deallocate(node, 1); + return nullptr; + } + return node; + } +}; + +TEST(BackendReleaseService, FinalOwnerOnlyQueuesAndHostPumpOwnsActualDisposal) +{ + TestReleaseHost host; + const auto baseline = host.budget->snapshot().allocated; + auto *node = host.prepare(); + ASSERT_NE(node, nullptr); + const auto held = host.budget->snapshot().allocated; + EXPECT_GT(held, baseline); + std::thread reader([node] { BackendReleaseService::enqueue(*node); }); + reader.join(); + EXPECT_EQ(host.completed.load(), 0U); + EXPECT_EQ(host.budget->snapshot().allocated, held); + EXPECT_EQ(host.service->snapshot().registered, 1U); + EXPECT_EQ(host.service->pump(1).completed, 1U); + EXPECT_EQ(host.completed.load(), 1U); + EXPECT_EQ(host.budget->snapshot().allocated, baseline); +} + +TEST(BackendReleaseService, StopRejectsNewNodesButRetainedReaderCanReleaseLater) +{ + TestReleaseHost host(1); + auto *node = host.prepare(); + ASSERT_NE(node, nullptr); + EXPECT_EQ(host.prepare(), nullptr); + host.service->stop_admission(); + EXPECT_FALSE(host.service->drained()); + EXPECT_EQ(host.prepare(), nullptr); + std::weak_ptr<BackendReleaseService> weak = host.service; + host.service.reset(); + ASSERT_FALSE(weak.expired()); + BackendReleaseService::enqueue(*node); + auto pump = weak.lock(); + ASSERT_NE(pump, nullptr); + EXPECT_EQ(pump->pump(1).completed, 1U); + EXPECT_TRUE(pump->drained()); + pump.reset(); + EXPECT_TRUE(weak.expired()); +} + +TEST(BackendReleaseService, FiniteBatchesPreventDeferredAndIncomingStarvation) +{ + TestReleaseHost host; + auto *slow = host.prepare(2); + auto *fast = host.prepare(); + ASSERT_NE(slow, nullptr); + ASSERT_NE(fast, nullptr); + BackendReleaseService::enqueue(*fast); + BackendReleaseService::enqueue(*slow); + EXPECT_EQ(host.service->pump(1).completed, 0U); + auto *newer = host.prepare(); + ASSERT_NE(newer, nullptr); + BackendReleaseService::enqueue(*newer); + EXPECT_EQ(host.service->pump(1).completed, 1U); + EXPECT_EQ(host.service->pump(1).completed, 0U); + EXPECT_EQ(host.service->pump(1).completed, 1U); + EXPECT_EQ(host.service->pump(1).completed, 1U); + EXPECT_EQ(host.completed.load(), 3U); +} + +TEST(BackendReleaseService, DrainedIsPublishedOnlyAfterActualNodeFree) +{ + TestReleaseHost host; + const auto baseline = host.budget->snapshot().allocated; + auto *node = host.prepare(); + ASSERT_NE(node, nullptr); + std::latch entered(1); + std::latch proceed(1); + node->entered = &entered; + node->proceed = &proceed; + BackendReleaseService::enqueue(*node); + host.service->stop_admission(); + std::thread pump([&] { (void)host.service->pump(1); }); + entered.wait(); + EXPECT_FALSE(host.service->drained()); + EXPECT_GT(host.budget->snapshot().allocated, baseline); + EXPECT_TRUE(host.service->pump(1).more); + proceed.count_down(); + pump.join(); + EXPECT_TRUE(host.service->drained()); + EXPECT_EQ(host.budget->snapshot().allocated, baseline); +} + +TEST(BackendReleaseService, ConcurrentFinalOwnersUseTheirPreallocatedNodes) +{ + TestReleaseHost host; + std::vector<TestRelease *> nodes; + nodes.reserve(64); + for (size_t index = 0; index < 64; ++index) { + nodes.push_back(host.prepare()); + } + const auto held = host.budget->snapshot().allocated; + std::vector<std::thread> readers; + readers.reserve(4); + for (size_t thread = 0; thread < 4; ++thread) { + readers.emplace_back([&, thread] { + for (size_t index = thread; index < nodes.size(); index += 4) { + BackendReleaseService::enqueue(*nodes[index]); + } + }); + } + for (auto &reader : readers) { + reader.join(); + } + EXPECT_EQ(host.budget->snapshot().allocated, held); + host.service->stop_admission(); + while (!host.service->drained()) { + const auto step = host.service->pump(3); + EXPECT_LE(step.visited, 3U); + } + EXPECT_EQ(host.completed.load(), 64U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/block_account_grant_test.cpp b/lib/crowdb-tree/tests/unit/block_account_grant_test.cpp new file mode 100644 index 000000000..0c18318ec --- /dev/null +++ b/lib/crowdb-tree/tests/unit/block_account_grant_test.cpp @@ -0,0 +1,214 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" +#include "backend/local/block_account.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +using namespace detail; +using namespace detail::test_allocation_account; +constexpr size_t kCapacity = 1U << 20; +constexpr size_t kIoBytes = static_cast<size_t>(AllocationKind::kIo); + +TEST(BlockAccountGrant, SparseGrowthAndBounceUsePrepaidCapacityAtTheHardLimit) +{ + tree_test::TempDir directory("block_grant_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 65536, 4096, &store).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto account = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + ASSERT_TRUE(bind_block_account(store.get(), account, 1, &adopted).ok()); + std::shared_ptr<AllocationBudget> grant; + const BackendOperationLimits work{.max_io_bytes = 512, .write_calls = 1, .workers = 1, .barriers = false}; + ASSERT_TRUE(reserve_backend_operation(store.get(), account, work, &grant).ok()); + const auto admitted = account->snapshot(); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kRouting, + kCapacity - admitted.allocated[kRouting] - admitted.reserved[kRouting]); + ASSERT_TRUE(occupied); + const uint8_t value = 73; + const uint64_t offset = (65536ULL * 1000) + 17; + EXPECT_EQ(store->write_at(offset, &value, 1).code(), Code::kResourceExhausted); + ASSERT_TRUE(store->write_at(offset, &value, 1, grant).ok()); + EXPECT_EQ(account->snapshot().physical[kIoBytes], 0U); + for (size_t i = 0; i < 3; ++i) { + uint8_t actual = 0; + ASSERT_TRUE(store->read_at(offset, &actual, 1, grant).ok()); + EXPECT_EQ(actual, value); + EXPECT_EQ(account->snapshot().physical[kIoBytes], 0U); + } + EXPECT_GT(account->snapshot().allocated[kRouting], adopted.owned_bytes); + seal_backend_budget(grant); + occupied.reset(); + EXPECT_EQ(account->snapshot().reserved[kRouting], 0U); + store.reset(); + EXPECT_GT(account->snapshot().allocated[kRouting], 0U); + grant.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(account->snapshot().physical[kMetadata], 0U); + EXPECT_EQ(account->snapshot().physical[kIoBytes], 0U); +} + +TEST(BlockAccountGrant, CompletePrepaidBoundBindsAfterAllUnreservedCapacityIsOccupied) +{ + tree_test::TempDir directory("block_prepaid_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 65536, 4096, &store).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto account = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + ASSERT_TRUE(bind_block_account(store.get(), account, 1, &adopted).ok()); + const BackendOperationLimits work{.max_io_bytes = 512, .write_calls = 1, .workers = 1, .barriers = false}; + const auto bound = backend_operation_bound(store.get(), work); + if (!bound) { + FAIL() << "missing bound test prerequisite"; + } + auto paid = ResourceBudget::try_reserve(account, BudgetClass::kRouting, *bound); + if (!paid) { + FAIL() << "missing paid test prerequisite"; + } + const auto before = account->snapshot(); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kRouting, + kCapacity - before.allocated[kRouting] - before.reserved[kRouting]); + ASSERT_TRUE(occupied); + std::shared_ptr<AllocationBudget> grant; + ASSERT_TRUE(bind_backend_operation(store.get(), work, std::move(*paid), &grant).ok()); + const uint8_t value = 11; + ASSERT_TRUE(store->write_at((65536ULL * 900) + 1, &value, 1, grant).ok()); + uint8_t actual = 0; + ASSERT_TRUE(store->read_at((65536ULL * 900) + 1, &actual, 1, grant).ok()); + EXPECT_EQ(actual, value); + seal_backend_budget(grant); + grant.reset(); + occupied.reset(); + store.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(host.snapshot().reserved[kRouting], 0U); +} + +TEST(BlockAccountGrant, IncompletePrepaidBoundFailsWithoutAllocatingOrGrowing) +{ + tree_test::TempDir directory("block_prepaid_small_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 65536, 4096, &store).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto account = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + ASSERT_TRUE(bind_block_account(store.get(), account, 1, &adopted).ok()); + const BackendOperationLimits work{.max_io_bytes = 512, .write_calls = 1, .workers = 1, .barriers = false}; + const auto bound = backend_operation_bound(store.get(), work); + if (!bound) { + FAIL() << "missing bound test prerequisite"; + } + auto paid = ResourceBudget::try_reserve(account, BudgetClass::kRouting, *bound - 1); + if (!paid) { + FAIL() << "missing paid test prerequisite"; + } + const auto before = account->snapshot(); + std::shared_ptr<AllocationBudget> grant; + EXPECT_EQ(bind_backend_operation(store.get(), work, std::move(*paid), &grant).code(), Code::kResourceExhausted); + EXPECT_FALSE(grant); + EXPECT_EQ(account->snapshot().allocated, before.allocated); + EXPECT_EQ(account->snapshot().reserved[kRouting], 0U); + EXPECT_EQ(store->num_extents(), 1U); +} + +TEST(BlockAccountGrant, WrongBackendGrantAndOverflowBoundAreRejectedBeforeGrowth) +{ + tree_test::TempDir directory("block_grant_wrong_"); + std::unique_ptr<BlockPageStore> left; + std::unique_ptr<BlockPageStore> right; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 1, 4096, 1, &left).ok()); + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 1, 2, 4096, 1, &right).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto account = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + ASSERT_TRUE(bind_block_account(left.get(), account, 1, &adopted).ok()); + ASSERT_TRUE(bind_block_account(right.get(), account, 1, &adopted).ok()); + std::shared_ptr<AllocationBudget> grant; + ASSERT_TRUE(reserve_backend_operation( + left.get(), account, {.max_io_bytes = 1, .write_calls = 1, .workers = 1, .barriers = false}, &grant) + .ok()); + const uint8_t value = 3; + EXPECT_EQ(right->write_at(4096, &value, 1, grant).code(), Code::kInvalidArgument); + EXPECT_EQ(right->num_extents(), 1U); + EXPECT_FALSE(backend_operation_bound( + left.get(), {.max_io_bytes = SIZE_MAX, .write_calls = SIZE_MAX, .workers = SIZE_MAX, .barriers = true})); + seal_backend_budget(grant); + grant.reset(); + left.reset(); + right.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); +} + +TEST(BlockAccountGrant, DeletionOnNewBackendThreadUsesPrepaidMetadata) +{ + tree_test::TempDir directory("block_grant_delete_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 4096, 1, &store).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto account = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + ASSERT_TRUE(bind_block_account(store.get(), account, 1, &adopted).ok()); + std::shared_ptr<AllocationBudget> grant; + ASSERT_TRUE(reserve_backend_operation(store.get(), account, {}, &grant).ok()); + const auto admitted = account->snapshot(); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kRouting, + kCapacity - admitted.allocated[kRouting] - admitted.reserved[kRouting]); + ASSERT_TRUE(occupied); + Status deleted; + std::thread backend([&] { deleted = store->delete_block(0, grant); }); + backend.join(); + ASSERT_TRUE(deleted.ok()) << deleted.to_string(); + EXPECT_EQ(store->num_extents(), 0U); + seal_backend_budget(grant); + occupied.reset(); + store.reset(); + grant.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(account->snapshot().physical[kMetadata], 0U); +} + +TEST(BlockAccountGrant, OpeningThreadMetadataUsesItsPrepaidParticipant) +{ + tree_test::TempDir directory("block_grant_thread_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 4096, 1, &store).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto account = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + ASSERT_TRUE(bind_block_account(store.get(), account, 1, &adopted).ok()); + std::shared_ptr<AllocationBudget> grant; + ASSERT_TRUE(reserve_backend_operation(store.get(), account, + {.max_io_bytes = 1, .write_calls = 0, .workers = 1, .barriers = false}, + &grant) + .ok()); + const auto admitted = account->snapshot(); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kRouting, + kCapacity - admitted.allocated[kRouting] - admitted.reserved[kRouting]); + ASSERT_TRUE(occupied); + bool succeeded = false; + std::thread opening([&] { + try { + succeeded = backend_size(*store, grant) == 0; + } + catch (const std::bad_alloc &) { + succeeded = false; + } + }); + opening.join(); + EXPECT_TRUE(succeeded); + seal_backend_budget(grant); + occupied.reset(); + store.reset(); + grant.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/block_account_test.cpp b/lib/crowdb-tree/tests/unit/block_account_test.cpp new file mode 100644 index 000000000..3a18813d0 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/block_account_test.cpp @@ -0,0 +1,129 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" +#include "backend/local/block_account.h" +#include "crowdb-tree/backend/backend_release_service.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "crowdb-tree/epoch.h" +#include "test_tmp.h" + +#include <fcntl.h> + +#include <cerrno> +#include <filesystem> + +namespace crowdb::tree +{ +namespace +{ +using namespace detail; +using namespace detail::test_allocation_account; +constexpr size_t kCapacity = 1U << 20; + +TEST(BlockAccount, StartupTracksSparseNodesControlAndEpochBeforeFirstBinding) +{ + tree_test::TempDir directory("block_account_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 4096, 1, &store).ok()); + const uint8_t value = 19; + ASSERT_TRUE(store->write_at(uint64_t{UINT32_MAX} * 4096, &value, 1).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto account = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + ASSERT_TRUE(bind_block_account(store.get(), account, 2, &adopted).ok()); + EXPECT_GT(adopted.owned_bytes, sizeof(BlockPageStore)); + EXPECT_EQ(adopted.growth_bytes, 3 * EpochManager::participant_allocation_bytes()); + EXPECT_EQ(host.snapshot().allocated[kRouting], adopted.owned_bytes); + EXPECT_EQ(account->snapshot().physical[kMetadata], adopted.owned_bytes); + EXPECT_EQ(store->num_extents(), 2U); + store.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(account->snapshot().physical[kMetadata], 0U); +} + +TEST(BlockAccount, SameHostReopenTransfersFdOwnersAndRejectsSmallerOrForeignAccount) +{ + tree_test::TempDir directory("block_transfer_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 4096, 1, &store).ok()); + const uint8_t value = 23; + ASSERT_TRUE(store->write_at(4096, &value, 1).ok()); + uint64_t local = 0; + auto held = store->file_for_offset(4096, &local); + auto release = store->release_service(); + ASSERT_TRUE(held); + const auto fd = held->fd(); + ResourceBudget host(limits(kCapacity), 1); + auto old = host.make_account(limits(kCapacity)); + auto next = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + ASSERT_TRUE(bind_block_account(store.get(), old, 1, &adopted).ok()); + ASSERT_TRUE(store->delete_block(1).ok()); + const auto retained = old->snapshot().allocated[kRouting]; + auto small = host.make_account(limits(retained - 1)); + EXPECT_EQ(bind_block_account(store.get(), small, 1, &adopted).code(), Code::kResourceExhausted); + EXPECT_EQ(old->snapshot().allocated[kRouting], retained); + EXPECT_EQ(small->snapshot().allocated[kRouting], 0U); + ResourceBudget foreign(limits(kCapacity), 1); + auto elsewhere = foreign.make_account(limits(kCapacity)); + EXPECT_EQ(bind_block_account(store.get(), elsewhere, 1, &adopted).code(), Code::kInvalidArgument); + ASSERT_TRUE(bind_block_account(store.get(), next, 1, &adopted).ok()); + EXPECT_EQ(old->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(old->snapshot().physical[kMetadata], 0U); + EXPECT_EQ(next->snapshot().allocated[kRouting], retained); + EXPECT_EQ(host.snapshot().allocated[kRouting], retained); + store.reset(); + EXPECT_GT(next->snapshot().allocated[kRouting], 0U); + EXPECT_EQ(release->snapshot().registered, 1U); + EXPECT_EQ(release->snapshot().pending, 0U); + EXPECT_FALSE(release->drained()); + uint8_t actual = 0; + ASSERT_TRUE(held->pread_at(0, &actual, 1).ok()); + EXPECT_EQ(actual, value); + held.reset(); + // This standalone host owns the explicit release boundary after its late + // file reader. Final owner destruction itself must not close the fd. + EXPECT_GE(::fcntl(fd, F_GETFD), 0); + EXPECT_EQ(release->snapshot().pending, 1U); + EXPECT_EQ(release->pump(1).completed, 1U); + EXPECT_TRUE(release->drained()); + errno = 0; + EXPECT_EQ(::fcntl(fd, F_GETFD), -1); + EXPECT_EQ(errno, EBADF); + release.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(next->snapshot().physical[kMetadata], 0U); +} + +TEST(BlockAccount, FailedFirstAdoptionAndRejectedGrowthLeaveFilesAndExistingReadsIntact) +{ + tree_test::TempDir directory("block_quota_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 4096, 1, &store).ok()); + const uint8_t value = 29; + ASSERT_TRUE(store->write_at(0, &value, 1).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto small = host.make_account(limits(1)); + BackendOpenBudget adopted; + EXPECT_EQ(bind_block_account(store.get(), small, 1, &adopted).code(), Code::kResourceExhausted); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + auto account = host.make_account(limits(kCapacity)); + ASSERT_TRUE(bind_block_account(store.get(), account, 1, &adopted).ok()); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kRouting, kCapacity - adopted.owned_bytes); + ASSERT_TRUE(occupied); + EXPECT_EQ(store->write_at(4096000, &value, 1).code(), Code::kResourceExhausted); + EXPECT_FALSE(std::filesystem::exists(directory.path + "/3-4.blk-1000")); + EXPECT_EQ(store->num_extents(), 1U); + uint8_t actual = 0; + ASSERT_TRUE(store->read_at(0, &actual, 1).ok()); + EXPECT_EQ(actual, value); + occupied.reset(); + ASSERT_TRUE(store->write_at(4096000, &value, 1).ok()); + EXPECT_GT(account->snapshot().allocated[kRouting], adopted.owned_bytes); + store.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(account->snapshot().physical[kMetadata], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/block_async_account_test.cpp b/lib/crowdb-tree/tests/unit/block_async_account_test.cpp new file mode 100644 index 000000000..cbb0d3705 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/block_async_account_test.cpp @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#ifdef CROWDB_HAVE_LIBURING +# include "allocation_account_test_support.h" +# include "backend/local/block_account.h" +# include "crowdb-tree/backend/async_page_store.h" +# include "crowdb-tree/backend/block_page_store.h" +# include "test_tmp.h" + +# include <chrono> +# include <thread> + +namespace crowdb::tree +{ +namespace +{ +using namespace detail; +using namespace detail::test_allocation_account; +constexpr size_t kCapacity = 1U << 20; + +struct TestCompletion +{ + std::atomic<bool> done{false}; + Status status; + + static void completed(void *context, Status status) + { + auto &self = *static_cast<TestCompletion *>(context); + self.status = std::move(status); + self.done.store(true, std::memory_order_release); + } + + AsyncCompletion callback(const std::shared_ptr<AllocationBudget> &budget) + { + done.store(false); + return {.context = this, .complete_fn = &completed, .owner = {}, .allocations = budget}; + } + + [[nodiscard]] bool wait() const + { + for (size_t i = 0; i < 200; ++i) { + if (done.load(std::memory_order_acquire)) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return done.load(std::memory_order_acquire); + } +}; + +TEST(BlockAsyncAccount, SplitControlsBounceAndBarrierSpendPrepaidCapacityAtTheHardLimit) +{ + tree_test::TempDir directory("block_async_account_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 3, 4, 4096, 4096, &store).ok()); + ResourceBudget host(limits(kCapacity), 1); + auto account = host.make_account(limits(kCapacity)); + BackendOpenBudget adopted; + ASSERT_TRUE(bind_block_account(store.get(), account, 1, &adopted).ok()); + std::shared_ptr<AllocationBudget> grant; + ASSERT_TRUE(reserve_backend_operation( + store.get(), account, + {.max_io_bytes = 8192, .write_calls = 1, .workers = 1, .barriers = true, .asynchronous = true}, + &grant) + .ok()); + const auto totals = account->snapshot(); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kRouting, + kCapacity - totals.allocated[kRouting] - totals.reserved[kRouting]); + ASSERT_TRUE(occupied); + { + ::crowdb::common::Topology topology; + topology.pipelines.push_back({.entries = 16, .mode = ::crowdb::common::PollingMode::Classic}); + ::crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + BlockAsyncPageStore backend(store.get(), &uring); + std::vector<uint8_t> input(8193, 37); + std::vector<uint8_t> output(8193, 0); + TestCompletion completion; + backend.submit_write(0, input.data() + 1, 8192, completion.callback(grant)); + ASSERT_TRUE(completion.wait()); + ASSERT_TRUE(completion.status.ok()) << completion.status.to_string(); + for (size_t i = 0; i < 3; ++i) { + backend.submit_read(0, output.data() + 1, 8192, completion.callback(grant)); + ASSERT_TRUE(completion.wait()); + ASSERT_TRUE(completion.status.ok()) << completion.status.to_string(); + EXPECT_TRUE(std::equal(input.begin() + 1, input.end(), output.begin() + 1)); + } + ASSERT_TRUE(backend.submit_fsync(completion.callback(grant)).ok()); + ASSERT_TRUE(completion.wait()); + EXPECT_TRUE(completion.status.ok()) << completion.status.to_string(); + } + seal_backend_budget(grant); + occupied.reset(); + EXPECT_EQ(account->snapshot().reserved[kRouting], 0U); + EXPECT_EQ(account->snapshot().physical[static_cast<size_t>(AllocationKind::kIo)], 0U); + store.reset(); + grant.reset(); + EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); + EXPECT_EQ(account->snapshot().physical[kMetadata], 0U); +} +} // namespace +} // namespace crowdb::tree +#endif diff --git a/lib/crowdb-tree/tests/unit/block_directory_test.cpp b/lib/crowdb-tree/tests/unit/block_directory_test.cpp new file mode 100644 index 000000000..e0e3fffd8 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/block_directory_test.cpp @@ -0,0 +1,114 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +#include <array> +#include <barrier> +#include <thread> + +namespace crowdb::tree +{ +TEST(BlockDirectory, SparseMaximumIndexDoesNotCreateDensePlaceholders) +{ + tree_test::TempDir dir("block_sparse_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 3, 4, 4096, 1, &store).ok()); + const uint8_t value = 7; + const uint64_t offset = uint64_t{UINT32_MAX} * 4096; + ASSERT_TRUE(store->write_at(offset, &value, 1).ok()); + EXPECT_EQ(store->num_extents(), 2U); + uint8_t actual = 0; + ASSERT_TRUE(store->read_at(offset, &actual, 1).ok()); + EXPECT_EQ(actual, value); + EXPECT_EQ(store->size(), offset + 1); + ASSERT_TRUE(store->sync().ok()); + store.reset(); + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 3, 4, 4096, 1, &store).ok()); + EXPECT_EQ(store->num_extents(), 2U); + ASSERT_TRUE(store->read_at(offset, &actual, 1).ok()); + EXPECT_EQ(actual, value); + EXPECT_EQ(store->write_at(offset + 4096, &value, 1).code(), Code::kInvalidArgument); + EXPECT_EQ(store->ensure_extents(UINT64_MAX, 2).code(), Code::kInvalidArgument); +} + +TEST(BlockDirectory, DeletedInteriorAndLastBlockCanBeRecreatedWithoutReusingOldFd) +{ + tree_test::TempDir dir("block_recreate_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 3, 4, 4096, 1, &store).ok()); + uint8_t value = 7; + ASSERT_TRUE(store->write_at(8192, &value, 1).ok()); + ASSERT_TRUE(store->write_at(4096, &value, 1).ok()); + uint64_t local = 0; + auto held = store->file_for_offset(4096, &local); + ASSERT_TRUE(held); + ASSERT_TRUE(store->delete_block(1).ok()); + value = 9; + ASSERT_TRUE(store->write_at(4096, &value, 1).ok()); + auto replacement = store->file_for_offset(4096, &local); + EXPECT_NE(held, replacement); + uint8_t old = 0; + ASSERT_TRUE(held->pread_at(0, &old, 1).ok()); + EXPECT_EQ(old, 7); + uint8_t current = 0; + ASSERT_TRUE(store->read_at(4096, ¤t, 1).ok()); + EXPECT_EQ(current, 9); + ASSERT_TRUE(store->delete_block(2).ok()); + ASSERT_TRUE(store->delete_block(1).ok()); + ASSERT_TRUE(store->delete_block(0).ok()); + EXPECT_EQ(store->num_extents(), 0U); + EXPECT_EQ(store->size(), 0U); + EXPECT_EQ(store->read_at(0, ¤t, 1).code(), Code::kNotFound); + ASSERT_TRUE(store->write_at(0, &value, 1).ok()); + EXPECT_EQ(store->num_extents(), 1U); +} + +TEST(BlockDirectory, ConcurrentGrowthDeletionAndEnumerationKeepReadersStable) +{ + tree_test::TempDir dir("block_concurrent_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 3, 4, 4096, 1, &store).ok()); + const uint8_t value = 37; + ASSERT_TRUE(store->write_at(0, &value, 1).ok()); + std::barrier start(4); + std::array<std::thread, 3> workers; + workers[0] = std::thread([&] { + start.arrive_and_wait(); + for (uint64_t i = 1; i <= 64; ++i) { + EXPECT_TRUE(store->write_at(i * 4096, &value, 1).ok()); + EXPECT_TRUE(store->delete_block(static_cast<uint32_t>(i)).ok()); + } + }); + workers[1] = std::thread([&] { + start.arrive_and_wait(); + for (uint64_t i = 65; i <= 128; ++i) { + EXPECT_TRUE(store->write_at(i * 4096, &value, 1).ok()); + } + }); + workers[2] = std::thread([&] { + start.arrive_and_wait(); + for (size_t i = 0; i < 128; ++i) { + const auto targets = store->sync_targets(); + for (const auto &target : targets) { + EXPECT_GE(target.medium->fd(), 0); + } + (void)store->size(); + (void)store->all_extent_fds(); + } + }); + start.arrive_and_wait(); + for (size_t i = 0; i < 256; ++i) { + uint8_t actual = 0; + EXPECT_TRUE(store->read_at(0, &actual, 1).ok()); + EXPECT_EQ(actual, value); + } + for (auto &worker : workers) { + worker.join(); + } + EXPECT_EQ(store->num_extents(), 65U); + EXPECT_TRUE(store->sync().ok()); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/block_release_service_test.cpp b/lib/crowdb-tree/tests/unit/block_release_service_test.cpp new file mode 100644 index 000000000..80bfe4d92 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/block_release_service_test.cpp @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" +#include "crowdb-tree/backend/backend_release_service.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include <fcntl.h> +#include <gtest/gtest.h> + +#include <atomic> +#include <cerrno> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +TEST(BlockReleaseService, BackendFinalOwnerQueuesDirectoryAndLateFileWithoutClosingOnCallback) +{ + tree_test::TempDir path("block_release_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(path.path, 3, 4, 4096, 1, &store).ok()); + const uint8_t value = 37; + ASSERT_TRUE(store->write_at(uint64_t{UINT32_MAX} * 4096, &value, 1).ok()); + auto service = store->release_service(); + auto notifications = std::make_shared<std::atomic<size_t>>(0); + ASSERT_TRUE(service + ->bind_wake( + notifications, + [](void *context) noexcept { static_cast<std::atomic<size_t> *>(context)->fetch_add(1); }, + notifications.get()) + .ok()); + uint64_t local = 0; + auto reader = store->file_for_offset(uint64_t{UINT32_MAX} * 4096, &local); + ASSERT_NE(reader, nullptr); + const auto fd = reader->fd(); + std::thread callback([owned = std::move(store)]() mutable { owned.reset(); }); + callback.join(); + EXPECT_GE(::fcntl(fd, F_GETFD), 0); + EXPECT_FALSE(service->drained()); + EXPECT_EQ(service->snapshot().pending, 1U); + size_t steps = 0; + while (service->snapshot().pending != 0 && steps < 256) { + const auto step = service->pump(1); + EXPECT_LE(step.visited, 1U); + ++steps; + } + EXPECT_LT(steps, 256U); + EXPECT_GT(steps, 1U); + EXPECT_EQ(service->snapshot().registered, 1U); + uint8_t actual = 0; + EXPECT_TRUE(reader->pread_at(0, &actual, 1).ok()); + EXPECT_EQ(actual, value); + std::thread late([owned = std::move(reader)]() mutable { owned.reset(); }); + late.join(); + EXPECT_GE(::fcntl(fd, F_GETFD), 0); + EXPECT_EQ(service->snapshot().pending, 1U); + EXPECT_EQ(service->pump(1).completed, 1U); + EXPECT_TRUE(service->drained()); + errno = 0; + EXPECT_EQ(::fcntl(fd, F_GETFD), -1); + EXPECT_EQ(errno, EBADF); + EXPECT_GT(notifications->load(), 0U); +} + +TEST(BlockReleaseService, ManualHostDestructionDrainsCurrentWorkAndPreservesLateReaders) +{ + tree_test::TempDir path("block_manual_release_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(path.path, 3, 4, 4096, 1, &store).ok()); + auto service = store->release_service(); + auto account = std::dynamic_pointer_cast<detail::AllocationAccount>(service->allocation_budget()); + ASSERT_NE(account, nullptr); + detail::ResourceBudget host(detail::test_allocation_account::limits(1 << 20), 1); + auto ledger = host.make_account(detail::test_allocation_account::limits(1 << 20)); + ASSERT_TRUE(account->rebind(ledger).ok()); + uint64_t local = 0; + auto reader = store->file_for_offset(0, &local); + ASSERT_NE(reader, nullptr); + const auto fd = reader->fd(); + store.reset(); + EXPECT_EQ(service->snapshot().pending, 0U); + EXPECT_EQ(service->snapshot().registered, 1U); + EXPECT_FALSE(service->drained()); + reader.reset(); + EXPECT_GE(::fcntl(fd, F_GETFD), 0); + EXPECT_EQ(service->pump(1).completed, 1U); + EXPECT_TRUE(service->drained()); + EXPECT_GT(account->owned_bytes(), 0U); + service.reset(); + // The stable account pays for its own object/control until its last owner. + std::shared_ptr<detail::AllocationAccount> empty_account; + ASSERT_TRUE(detail::AllocationAccount::create_unbound(&empty_account).ok()); + EXPECT_EQ(account->owned_bytes(), empty_account->owned_bytes()); + account.reset(); + EXPECT_EQ(ledger->snapshot().allocated[detail::test_allocation_account::kRouting], 0U); + EXPECT_EQ(ledger->snapshot().physical[detail::test_allocation_account::kMetadata], 0U); +} + +TEST(BlockReleaseService, FailedSingleFileOpenReturnsAllQueuedCloseControls) +{ + tree_test::TempDir path("block_open_release_"); + std::unique_ptr<BlockPageStore> store; + const auto status = BlockPageStore::open(path.path + "/missing/file", 1, &store); + EXPECT_FALSE(status.ok()); + EXPECT_FALSE(store); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/block_sync_generation_test.cpp b/lib/crowdb-tree/tests/unit/block_sync_generation_test.cpp new file mode 100644 index 000000000..5c2cd3c11 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/block_sync_generation_test.cpp @@ -0,0 +1,134 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "async_completion_adapter.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +#include <array> +#include <future> + +namespace crowdb::tree +{ +TEST(BlockSyncGeneration, OlderBarrierCannotAcknowledgeLaterWrite) +{ + crowdb::tree_test::TempDir dir("sync_generation_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 1, &store).ok()); + const std::array<uint8_t, 1> bytes{42}; + ASSERT_TRUE(store->write_at(0, bytes.data(), bytes.size()).ok()); + auto first = store->sync_targets(); + ASSERT_EQ(first.size(), 1U); + ASSERT_TRUE(store->write_at(1, bytes.data(), bytes.size()).ok()); + auto second = store->sync_targets(); + ASSERT_EQ(second.size(), 1U); + first[0].medium->acknowledge_sync(first[0].generation); + EXPECT_EQ(store->dirty_fds().size(), 1U); + second[0].medium->acknowledge_sync(second[0].generation); + EXPECT_TRUE(store->dirty_fds().empty()); + // Late completion of the older barrier cannot undo a newer success. + first[0].medium->acknowledge_sync(first[0].generation); + EXPECT_TRUE(store->dirty_fds().empty()); +} + +TEST(BlockSyncGeneration, WriteCompletingAfterBarrierRemainsDirty) +{ + crowdb::tree_test::TempDir dir("sync_inflight_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 1, &store).ok()); + ASSERT_TRUE(store->ensure_extents(0, 1).ok()); + auto barrier = store->sync_targets(); + ASSERT_EQ(barrier.size(), 1U); + barrier[0].medium->acknowledge_sync(barrier[0].generation); + EXPECT_TRUE(store->dirty_fds().empty()); + // A submitted write may still be in flight when that barrier completes. + barrier[0].medium->mark_dirty(); + EXPECT_EQ(store->dirty_fds().size(), 1U); + ASSERT_TRUE(store->sync().ok()); + EXPECT_TRUE(store->dirty_fds().empty()); +} + +TEST(BlockSyncGeneration, CapturedFdSurvivesDeleteAndDoesNotAcknowledgeReplacement) +{ + crowdb::tree_test::TempDir dir("sync_incarnation_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 1, &store).ok()); + const std::array<uint8_t, 1> bytes{42}; + ASSERT_TRUE(store->write_at(4096, bytes.data(), bytes.size()).ok()); + auto old = store->sync_targets(); + ASSERT_EQ(old.size(), 1U); + std::weak_ptr<FileMedium> retained = old[0].medium; + ASSERT_TRUE(store->delete_block(1).ok()); + EXPECT_FALSE(retained.expired()); + ASSERT_TRUE(store->write_at(4096, bytes.data(), bytes.size()).ok()); + auto current = store->sync_targets(); + ASSERT_EQ(current.size(), 1U); + EXPECT_NE(current[0].medium, old[0].medium); + ASSERT_TRUE(old[0].medium->fsync().ok()); + old[0].medium->acknowledge_sync(old[0].generation); + EXPECT_EQ(store->dirty_fds().size(), 1U); + old.clear(); + EXPECT_TRUE(retained.expired()); + ASSERT_TRUE(store->sync().ok()); + EXPECT_TRUE(store->dirty_fds().empty()); +} + +TEST(BlockSyncGeneration, SingleFileSyncAcknowledgesOnlySuccessfulWrites) +{ + crowdb::tree_test::TempDir dir("sync_single_"); + std::unique_ptr<BlockPageStore> store; + ASSERT_TRUE(BlockPageStore::open(dir.path + "/store", 4096, &store).ok()); + const std::array<uint8_t, 1> bytes{42}; + EXPECT_TRUE(store->dirty_fds().empty()); + ASSERT_TRUE(store->write_at(0, bytes.data(), bytes.size()).ok()); + EXPECT_EQ(store->dirty_fds().size(), 1U); + ASSERT_TRUE(store->sync().ok()); + EXPECT_TRUE(store->dirty_fds().empty()); +} + +#ifdef CROWDB_HAVE_LIBURING +TEST(BlockSyncGeneration, AsyncBarrierAcknowledgesSuccessAndRetainsFailure) +{ + crowdb::tree_test::TempDir dir("sync_async_"); + std::unique_ptr<BlockPageStore> file; + ASSERT_TRUE(BlockPageStore::open_blocks(dir.path, 1, 2, 4096, 4096, &file).ok()); + crowdb::common::Topology topology; + topology.pipelines.emplace_back().entries = 64; + crowdb::common::DiskIOUring uring(std::move(topology)); + ASSERT_TRUE(uring.valid()); + BlockAsyncPageStore store(file.get(), &uring); + std::array<uint8_t, 8192> bytes{}; + std::promise<Status> written; + auto write = written.get_future(); + store.submit_write(0, bytes.data(), bytes.size(), detail::own_async_completion([&written](Status status) { + written.set_value(std::move(status)); + })); + ASSERT_EQ(write.wait_for(std::chrono::seconds(5)), std::future_status::ready); + ASSERT_TRUE(write.get().ok()); + ASSERT_EQ(file->dirty_fds().size(), 2U); + std::promise<Status> synced; + auto sync = synced.get_future(); + ASSERT_TRUE(store + .submit_fsync( + detail::own_async_completion([&synced](Status status) { synced.set_value(std::move(status)); })) + .ok()); + ASSERT_EQ(sync.wait_for(std::chrono::seconds(5)), std::future_status::ready); + ASSERT_TRUE(sync.get().ok()); + EXPECT_TRUE(file->dirty_fds().empty()); + ASSERT_TRUE(file->ensure_extents(4096, 4096).ok()); + crowdb::common::DiskIOUring unavailable({}); + BlockAsyncPageStore rejected(file.get(), &unavailable); + size_t calls = 0; + ASSERT_TRUE(rejected + .submit_fsync(detail::own_async_completion([&calls](Status status) { + EXPECT_EQ(status.code(), Code::kIoError); + ++calls; + })) + .ok()); + EXPECT_EQ(calls, 1U); + EXPECT_EQ(file->dirty_fds().size(), 1U); +} +#endif +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/budget_memory_resource_test.cpp b/lib/crowdb-tree/tests/unit/budget_memory_resource_test.cpp new file mode 100644 index 000000000..70c7b2d04 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/budget_memory_resource_test.cpp @@ -0,0 +1,149 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "btree/budget_memory_resource.h" + +#include <gtest/gtest.h> + +#include <functional> +#include <vector> + +namespace crowdb::tree::detail +{ +namespace +{ +constexpr size_t kMetadata = static_cast<size_t>(BudgetClass::kHandoff); + +struct TestBudget +{ + static ResourceBudget::Limits limits() + { + ResourceBudget::Limits result{}; + result[kMetadata] = 65536; + return result; + } + + ResourceBudget budget{limits(), 2}; + std::shared_ptr<ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] std::shared_ptr<AllocationReservation> reserve(size_t bytes = 16384) const + { + auto credit = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, bytes); + if (!credit) { + throw std::bad_alloc(); + } + return std::make_shared<AllocationReservation>(std::move(*credit)); + } + + [[nodiscard]] size_t allocated() const + { + return budget.snapshot().allocated[kMetadata]; + } +}; + +class TestUpstream final : public std::pmr::memory_resource +{ + public: + bool fail = false; + std::function<void()> before_free; + + private: + void *do_allocate(size_t bytes, size_t alignment) override + { + if (fail) { + throw std::bad_alloc(); + } + return std::pmr::new_delete_resource()->allocate(bytes, alignment); + } + + void do_deallocate(void *pointer, size_t bytes, size_t alignment) override + { + if (before_free) { + before_free(); + } + std::pmr::new_delete_resource()->deallocate(pointer, bytes, alignment); + } + + [[nodiscard]] bool do_is_equal(const std::pmr::memory_resource &other) const noexcept override + { + return this == &other; + } +}; + +TEST(BudgetMemoryResource, AlignedBlockKeepsItsOriginalReservationUntilPhysicalFree) +{ + TestBudget budget; + TestUpstream upstream; + BudgetMemoryResource resource(&upstream); + auto first = budget.reserve(); + auto second = budget.reserve(); + void *pointer; + { + BudgetMemoryResource::Scope scope(resource, first); + pointer = resource.allocate(3, 128); + } + EXPECT_EQ(reinterpret_cast<uintptr_t>(pointer) % 128, 0U); + const auto charged = budget.allocated(); + EXPECT_GT(charged, 3U); + EXPECT_EQ(charged, resource.retained_bytes()); + first->seal(); + upstream.before_free = [&] { EXPECT_EQ(budget.allocated(), charged); }; + { + BudgetMemoryResource::Scope scope(resource, second); + resource.deallocate(pointer, 3, 128); + } + EXPECT_EQ(second->available(), 16384U); + EXPECT_EQ(budget.allocated(), 0U); + EXPECT_EQ(resource.retained_bytes(), 0U); +} + +TEST(BudgetMemoryResource, UpstreamAndAdmissionFailuresReturnAllCredit) +{ + TestBudget budget; + TestUpstream upstream; + BudgetMemoryResource resource(&upstream); + auto credit = budget.reserve(); + BudgetMemoryResource::Scope scope(resource, credit); + upstream.fail = true; + EXPECT_THROW((void)resource.allocate(128, 64), std::bad_alloc); + EXPECT_EQ(credit->available(), 16384U); + EXPECT_EQ(budget.allocated(), 0U); + upstream.fail = false; + // Exercise runtime overflow rejection without a constant oversized allocation diagnostic. + volatile size_t overflowing_bytes = SIZE_MAX; + EXPECT_THROW((void)resource.allocate(overflowing_bytes, 64), std::bad_alloc); + EXPECT_THROW((void)resource.allocate(65536, 64), std::bad_alloc); + EXPECT_EQ(budget.allocated(), 0U); + EXPECT_EQ(resource.retained_bytes(), 0U); +} + +TEST(BudgetMemoryResource, ContainerGrowthUsesNewCreditAndFreesSealedOldCapacity) +{ + TestBudget budget; + TestUpstream upstream; + BudgetMemoryResource resource(&upstream); + auto first = budget.reserve(); + auto second = budget.reserve(); + { + std::pmr::vector<uint64_t> values(&resource); + { + BudgetMemoryResource::Scope scope(resource, first); + values.resize(64, 7); + } + const auto old_bytes = budget.allocated(); + first->seal(); + upstream.before_free = [&] { EXPECT_GT(budget.allocated(), old_bytes + (128 * sizeof(uint64_t))); }; + { + BudgetMemoryResource::Scope scope(resource, second); + values.reserve(128); + } + upstream.before_free = {}; + EXPECT_EQ(values.front(), 7U); + EXPECT_GT(resource.retained_bytes(), 128 * sizeof(uint64_t)); + second->seal(); + } + EXPECT_EQ(budget.allocated(), 0U); + EXPECT_EQ(resource.retained_bytes(), 0U); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/buffer_pool_test.cpp b/lib/crowdb-tree/tests/unit/buffer_pool_test.cpp index 50952f320..28ff92488 100644 --- a/lib/crowdb-tree/tests/unit/buffer_pool_test.cpp +++ b/lib/crowdb-tree/tests/unit/buffer_pool_test.cpp @@ -2,9 +2,9 @@ // Licensed under the Apache License, Version 2.0. // PT6b: buffer pool manager tests. +#include "buffer_pool_test_support.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/btree/cell.h" -#include "crowdb-tree/maptable/buffer_pool.h" #include "crowdb-tree/maptable/frame_page.h" #include <gtest/gtest.h> @@ -32,7 +32,7 @@ std::string key(int i) } // build a one-entry leaf into a frame and mark it dirty. -void build_leaf(BufferPool *pool, FrameRef *ref, uint64_t page_id, int k) +void build_leaf(TestBufferPool *pool, TestPoolFrameRef *ref, uint64_t page_id, int k) { LeafFrameBuilder b(ref->bytes(), kPb); std::string cell = encode_cell(static_cast<uint64_t>(k), OpKind::kPut, Slice("v")); @@ -110,21 +110,22 @@ class BlockingReadStore final : public PageStore TEST(BufferPool, MissLoadsFromStoreAfterEviction) { - MemPageStore store(1); - BufferPool pool(static_cast<size_t>(2) * kPb, kPb, &store); // 2 frames + MemPageStore store(1); + TestBufferPool pool(static_cast<size_t>(2) * kPb, kPb, &store); // 2 frames // Write 4 pages through the pool; with 2 frames, evictions write dirty pages // back to the store. for (int i = 0; i < 4; ++i) { - FrameRef r; + TestPoolFrameRef r; ASSERT_TRUE(pool.pin_new(i, addr(i), &r).ok()); build_leaf(&pool, &r, i, i); r.release(); } ASSERT_TRUE(pool.flush_dirty().ok()); - // pin page_id 0 again: it was evicted, so this is a miss that reloads from store. - FrameRef r; + // pin page_id 0 again: it was evicted, so this is a miss that reloads from + // store. + TestPoolFrameRef r; ASSERT_TRUE(pool.pin(0, addr(0), &r).ok()); LeafFrameView v(r.bytes(), kPb); ASSERT_EQ(v.count(), 1U); @@ -139,15 +140,15 @@ TEST(BufferPool, MissLoadsFromStoreAfterEviction) TEST(BufferPool, HitDoesNotMiss) { - MemPageStore store(1); - BufferPool pool(static_cast<size_t>(4) * kPb, kPb, &store); + MemPageStore store(1); + TestBufferPool pool(static_cast<size_t>(4) * kPb, kPb, &store); { - FrameRef r; + TestPoolFrameRef r; ASSERT_TRUE(pool.pin_new(5, addr(5), &r).ok()); build_leaf(&pool, &r, 5, 5); } // released - auto before = pool.stats(); - FrameRef r; + auto before = pool.stats(); + TestPoolFrameRef r; ASSERT_TRUE(pool.pin(5, addr(5), &r).ok()); // still resident -> hit auto after = pool.stats(); EXPECT_EQ(after.misses, before.misses); @@ -157,9 +158,9 @@ TEST(BufferPool, HitDoesNotMiss) TEST(BufferPool, UnrelatedHitContinuesWhileMissReadIsBlocked) { BlockingReadStore store; - BufferPool pool(static_cast<size_t>(2) * kPb, kPb, &store); + TestBufferPool pool(static_cast<size_t>(2) * kPb, kPb, &store); { - FrameRef resident; + TestPoolFrameRef resident; ASSERT_TRUE(pool.pin_new(1, addr(1), &resident).ok()); build_leaf(&pool, &resident, 1, 1); } @@ -173,13 +174,13 @@ TEST(BufferPool, UnrelatedHitContinuesWhileMissReadIsBlocked) store.block(addr(2)); auto miss = std::async(std::launch::async, [&] { - FrameRef loaded; + TestPoolFrameRef loaded; return pool.pin(2, addr(2), &loaded); }); store.wait_until_blocked(); auto hit = std::async(std::launch::async, [&] { - FrameRef resident; + TestPoolFrameRef resident; return pool.pin(1, addr(1), &resident); }); EXPECT_EQ(hit.wait_for(std::chrono::milliseconds(100)), std::future_status::ready); @@ -191,14 +192,14 @@ TEST(BufferPool, UnrelatedHitContinuesWhileMissReadIsBlocked) TEST(BufferPool, PinnedFramesAreNotEvicted) { - MemPageStore store(1); - BufferPool pool(static_cast<size_t>(2) * kPb, kPb, &store); // 2 frames - FrameRef a; + MemPageStore store(1); + TestBufferPool pool(static_cast<size_t>(2) * kPb, kPb, &store); // 2 frames + TestPoolFrameRef a; ASSERT_TRUE(pool.pin_new(0, addr(0), &a).ok()); - FrameRef b; + TestPoolFrameRef b; ASSERT_TRUE(pool.pin_new(1, addr(1), &b).ok()); - FrameRef c; - Status s = pool.pin_new(2, addr(2), &c); + TestPoolFrameRef c; + Status s = pool.pin_new(2, addr(2), &c); EXPECT_FALSE(s.ok()); EXPECT_FALSE(c.valid()); // release one; now it succeeds. @@ -208,10 +209,10 @@ TEST(BufferPool, PinnedFramesAreNotEvicted) TEST(BufferPool, DirtyFlushWritesBack) { - MemPageStore store(1); - BufferPool pool(static_cast<size_t>(4) * kPb, kPb, &store); + MemPageStore store(1); + TestBufferPool pool(static_cast<size_t>(4) * kPb, kPb, &store); { - FrameRef r; + TestPoolFrameRef r; ASSERT_TRUE(pool.pin_new(9, addr(9), &r).ok()); build_leaf(&pool, &r, 9, 9); } @@ -224,10 +225,10 @@ TEST(BufferPool, DirtyFlushWritesBack) TEST(BufferPool, CorruptPageRejectedOnLoad) { - MemPageStore store(1); - BufferPool pool(static_cast<size_t>(2) * kPb, kPb, &store); + MemPageStore store(1); + TestBufferPool pool(static_cast<size_t>(2) * kPb, kPb, &store); for (int i = 0; i < 4; ++i) { - FrameRef r; + TestPoolFrameRef r; ASSERT_TRUE(pool.pin_new(i, addr(i), &r).ok()); build_leaf(&pool, &r, i, i); r.release(); @@ -236,15 +237,16 @@ TEST(BufferPool, CorruptPageRejectedOnLoad) // Corrupt page_id 0's durable image, then force a reload. std::vector<uint8_t> garbage(kPb, 0x5a); ASSERT_TRUE(store.write_at(addr(0), garbage.data(), garbage.size()).ok()); - FrameRef r; + TestPoolFrameRef r; EXPECT_EQ(pool.pin(0, addr(0), &r).code(), Code::kCorruption); } TEST(BufferPool, AcquireFrameIsAnonymousDirtyAndResident) { - BufferPool pool(static_cast<size_t>(4) * kPb, kPb, nullptr); // no store: anon frames only - uint32_t idx = 0; - uint8_t *bytes = nullptr; + TestBufferPool pool(static_cast<size_t>(4) * kPb, kPb, + nullptr); // no store: anon frames only + uint32_t idx = 0; + uint8_t *bytes = nullptr; ASSERT_TRUE(pool.acquire_frame(&idx, &bytes).ok()); ASSERT_NE(bytes, nullptr); // build a leaf directly into the acquired frame and read it back zero-copy. @@ -264,11 +266,11 @@ TEST(BufferPool, AcquireFrameIsAnonymousDirtyAndResident) TEST(BufferPool, AcquireFrameFailsWhenFull) { - BufferPool pool(static_cast<size_t>(2) * kPb, kPb, nullptr); // 2 frames - uint32_t i1 = 0; - uint32_t i2 = 0; - uint32_t i3 = 0; - uint8_t *b = nullptr; + TestBufferPool pool(static_cast<size_t>(2) * kPb, kPb, nullptr); // 2 frames + uint32_t i1 = 0; + uint32_t i2 = 0; + uint32_t i3 = 0; + uint8_t *b = nullptr; ASSERT_TRUE(pool.acquire_frame(&i1, &b).ok()); ASSERT_TRUE(pool.acquire_frame(&i2, &b).ok()); // Both frames pinned-resident: a third acquire must fail (caller falls back @@ -280,11 +282,12 @@ TEST(BufferPool, AcquireFrameFailsWhenFull) TEST(BufferPool, ManyPagesThroughSmallPool) { - MemPageStore store(1); - BufferPool pool(static_cast<size_t>(3) * kPb, kPb, &store); // 3 frames, many pages - const int N = 200; + MemPageStore store(1); + TestBufferPool pool(static_cast<size_t>(3) * kPb, kPb, + &store); // 3 frames, many pages + const int N = 200; for (int i = 0; i < N; ++i) { - FrameRef r; + TestPoolFrameRef r; ASSERT_TRUE(pool.pin_new(i, addr(i), &r).ok()); build_leaf(&pool, &r, i, i); r.release(); @@ -292,7 +295,7 @@ TEST(BufferPool, ManyPagesThroughSmallPool) ASSERT_TRUE(pool.flush_dirty().ok()); // Every page reloads correctly through the small pool (page table churn). for (int i = 0; i < N; ++i) { - FrameRef r; + TestPoolFrameRef r; ASSERT_TRUE(pool.pin(i, addr(i), &r).ok()); EXPECT_EQ(LeafFrameView(r.bytes(), kPb).key(0).to_string(), key(i)); } diff --git a/lib/crowdb-tree/tests/unit/c_api_admission_test.cpp b/lib/crowdb-tree/tests/unit/c_api_admission_test.cpp new file mode 100644 index 000000000..1607836d8 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/c_api_admission_test.cpp @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/apply_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +class AdmissionCAbi : public testing::Test +{ + protected: + ct_runtime *runtime = nullptr; + ct_tree *tree = nullptr; + + void SetUp() override + { + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options config{}; + ASSERT_EQ(ct_open_with_runtime(&config, runtime, nullptr, &tree), 0); + } + + void TearDown() override + { + ct_close(tree); + ct_runtime_free(runtime); + } + + TreeCore &core() + { + return *TreeCoreAccess::get(*c_api_tree(tree)); + } +}; + +TEST_F(AdmissionCAbi, UnarmedHandleOutlivesTreeWithoutBlockingClose) +{ + ct_apply_permit *permit = nullptr; + ASSERT_EQ(ct_apply_prepare(tree, nullptr, 0, 0, &permit), 0); + ct_close(tree); + tree = nullptr; + ct_runtime_free(runtime); + runtime = nullptr; + EXPECT_EQ(ct_apply_arm(permit), static_cast<int32_t>(Code::kUnavailable)); + ct_apply_permit_free(permit); +} + +TEST_F(AdmissionCAbi, AcceptedCopyOutlivesBothCallerHandles) +{ + uint8_t key = 'k'; + uint8_t value = 'v'; + ct_kv_ref input{.key = &key, .key_len = 1, .value = &value, .value_len = 1, .kind = 0}; + ct_apply_permit *permit = nullptr; + ASSERT_EQ(ct_apply_prepare(tree, &input, 1, 0, &permit), 0); + key = 'x'; + value = 'y'; + ASSERT_EQ(ct_apply_arm(permit), 0); + ct_future *future = nullptr; + ASSERT_EQ(ct_apply_submit(permit, 1, &future), 0); + ASSERT_NE(future, nullptr); + ct_apply_permit_free(permit); + ct_future_free(future); + core().binding()->lane()->drain(); + GetView result; + ASSERT_TRUE(core().get(Slice("k", 1), &result).ok()); + ASSERT_TRUE(result.found()); + EXPECT_EQ(result.value().to_view(), "v"); + EXPECT_EQ(core().contiguous_slot(), 1U); +} + +TEST_F(AdmissionCAbi, PollConsumesOneTerminalAndCancellationReleasesArm) +{ + ct_apply_permit *cancelled = nullptr; + ASSERT_EQ(ct_apply_prepare(tree, nullptr, 0, 0, &cancelled), 0); + ASSERT_EQ(ct_apply_arm(cancelled), 0); + ct_apply_permit_free(cancelled); + ASSERT_TRUE(core().clear().ok()); + ct_apply_permit *permit = nullptr; + ASSERT_EQ(ct_apply_prepare(tree, nullptr, 0, 1, &permit), 0); + ASSERT_EQ(ct_apply_arm(permit), 0); + ct_future *future = nullptr; + ASSERT_EQ(ct_apply_submit(permit, 1, &future), 0); + ct_future *duplicate = nullptr; + EXPECT_EQ(ct_apply_submit(permit, 1, &duplicate), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(duplicate, nullptr); + core().binding()->lane()->drain(); + int32_t done = 0; + EXPECT_EQ(ct_future_poll(future, &done, nullptr, nullptr, nullptr), 0); + EXPECT_EQ(done, 1); + ct_apply_permit_free(permit); + EXPECT_EQ(core().contiguous_slot(), 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/c_api_maintenance_report_test.cpp b/lib/crowdb-tree/tests/unit/c_api_maintenance_report_test.cpp new file mode 100644 index 000000000..adb682775 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/c_api_maintenance_report_test.cpp @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_maintenance_internal.h" + +#include <gtest/gtest.h> + +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +class MaintenanceReportCAbi : public testing::Test +{ + protected: + ct_runtime *runtime = nullptr; + ct_tree *tree = nullptr; + + void SetUp() override + { + ct_runtime_options options{}; + ct_runtime_options_default(&options); + options.workers = 1; + ASSERT_EQ(ct_runtime_create(&options, &runtime), 0); + ct_options config{}; + config.backend = CT_BACKEND_MEM_BLOCK; + ct_page_store *store = nullptr; + ASSERT_EQ(ct_page_store_open_mem(4096, &store), 0); + std::unique_ptr<ct_page_store, decltype(&ct_page_store_free)> owned_store(store, &ct_page_store_free); + config.page_store = store; + config.block_size = 65536; + config.frame_bytes = 4096; + config.iu_size = 4096; + config.max_inline_value = 64; + config.sync_mode = CT_SYNC_SKIP; + ASSERT_EQ(ct_open_with_runtime(&config, runtime, nullptr, &tree), 0); + } + + void TearDown() override + { + ct_close(tree); + ct_runtime_free(runtime); + } + + TreeCore &core() + { + return *TreeCoreAccess::get(*c_api_tree(tree)); + } + + void put(uint64_t slot, uint8_t key) + { + const std::string value(8192, static_cast<char>(key)); + ASSERT_EQ(ct_apply_put(tree, slot, &key, 1, reinterpret_cast<const uint8_t *>(value.data()), value.size()), 0); + } +}; + +TEST_F(MaintenanceReportCAbi, FlushReportRetainsFrontiersAndRejectsWrongPollWithoutConsuming) +{ + put(1, 'a'); + put(2, 'b'); + ct_future *future = nullptr; + ASSERT_EQ(ct_flush_report_async(tree, &future), 0); + ASSERT_NE(future, nullptr); + int32_t done = 9; + EXPECT_EQ(ct_future_poll(future, &done, nullptr, nullptr, nullptr), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(done, 9); + ct_flush_result_v1 report{}; + report.struct_size = sizeof(report); + report.version = 2; + EXPECT_EQ(ct_future_poll_flush_report(future, &done, &report), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(report.version, 2U); + report.version = 1; + core().binding()->lane()->drain(); + ASSERT_EQ(ct_future_poll_flush_report(future, &done, &report), 0); + ASSERT_EQ(done, 1); + EXPECT_EQ(report.terminal_status, 0); + EXPECT_EQ(report.handoff_status, 0); + EXPECT_EQ(report.maintenance_status, 0); + EXPECT_EQ(report.handoff_completed, 1); + EXPECT_EQ(report.handoff_advanced, 1); + EXPECT_EQ(report.floor, 0U); + EXPECT_EQ(report.frontier, 2U); + EXPECT_EQ(report.published_coverage, 2U); +} + +TEST_F(MaintenanceReportCAbi, WritebackAndDurableCheckpointKeepDistinctReports) +{ + put(1, 'a'); + ct_future *future = nullptr; + ASSERT_EQ(ct_writeback_async(tree, &future), 0); + ct_checkpoint_result_v1 report{}; + report.struct_size = sizeof(report); + report.version = 1; + int32_t done = 0; + while (done == 0) { + ASSERT_EQ(ct_future_poll_checkpoint_report(future, &done, &report), 0); + if (done == 0) { + std::this_thread::yield(); + } + } + ASSERT_EQ(report.terminal_status, 0); + EXPECT_EQ(report.captured_coverage, 1U); + EXPECT_EQ(report.writeback_completed, 1); + EXPECT_EQ(report.durable_committed, 0); + EXPECT_EQ(report.durable_coverage, 0U); + EXPECT_EQ(ct_last_applied_slot(tree), 1U); + EXPECT_EQ(ct_durable_coverage(tree), 0U); + EXPECT_GT(report.written_frames, report.primary_frames); + EXPECT_GT(report.rebased_pages, 0U); + ASSERT_EQ(ct_checkpoint_report_async(tree, &future), 0); + done = 0; + while (done == 0) { + ASSERT_EQ(ct_future_poll_checkpoint_report(future, &done, &report), 0); + if (done == 0) { + std::this_thread::yield(); + } + } + ASSERT_EQ(report.terminal_status, 0); + EXPECT_EQ(report.durable_committed, 1); + EXPECT_EQ(ct_durable_coverage(tree), 1U); + EXPECT_EQ(report.durable_coverage, 1U); + EXPECT_GT(report.snapshot_sequence, 0U); +} + +TEST_F(MaintenanceReportCAbi, AbandonedPendingFutureKeepsItsConnectedOperationUntilTreeClose) +{ + struct Gate + { + std::atomic<bool> entered{false}; + std::atomic<bool> release{false}; + } gate; + + put(1, 'a'); + ASSERT_TRUE(core() + .binding() + ->cpu() + ->try_submit( + core().binding()->lane(), + [](void *context) noexcept { + auto &wait = *static_cast<Gate *>(context); + wait.entered.store(true, std::memory_order_release); + wait.entered.notify_one(); + while (!wait.release.load(std::memory_order_acquire)) { + wait.release.wait(false); + } + }, + &gate) + .ok()); + while (!gate.entered.load(std::memory_order_acquire)) { + gate.entered.wait(false); + } + ct_future *future = nullptr; + const auto started = ct_flush_report_async(tree, &future); + if (started != 0) { + gate.release.store(true, std::memory_order_release); + gate.release.notify_one(); + FAIL() << "flush start rejected: " << started; + } + ct_future_free(future); + std::jthread close([this] { ct_close(tree); }); + gate.release.store(true, std::memory_order_release); + gate.release.notify_one(); + close.join(); + tree = nullptr; +} + +TEST(MaintenanceReportMapping, MaintenanceFailurePreservesCompletedHandoff) +{ + auto future = make_ct_future({}); + future->kind = ct_future_impl::Kind::kFlushReport; + FlushResult source; + source.batch_id = 5; + source.floor = 10; + source.frontier = 20; + source.published_coverage = 20; + source.maintenance_status = Status::io_error(); + source.deferred_pages = 7; + source.phase = FlushPhase::kComplete; + source.handoff_advanced = true; + c_api_detail::store_report(*future, source, Status::Ok()); + future->done.store(true, std::memory_order_release); + auto *handle = reinterpret_cast<ct_future *>(make_ct_future_handle(std::move(future))); + ct_flush_result_v1 report{}; + report.struct_size = sizeof(report); + report.version = 1; + int32_t done = 0; + ASSERT_EQ(ct_future_poll_flush_report(handle, &done, &report), 0); + EXPECT_EQ(done, 1); + EXPECT_EQ(report.terminal_status, static_cast<int32_t>(Code::kIoError)); + EXPECT_EQ(report.handoff_status, 0); + EXPECT_EQ(report.handoff_completed, 1); + EXPECT_EQ(report.handoff_advanced, 1); + EXPECT_EQ(report.frontier, 20U); + EXPECT_EQ(report.published_coverage, 20U); + EXPECT_EQ(report.deferred_pages, 7U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/c_api_owned_budget_test.cpp b/lib/crowdb-tree/tests/unit/c_api_owned_budget_test.cpp new file mode 100644 index 000000000..6260ed438 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/c_api_owned_budget_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "c_api_owned_budget.h" +#include "crowdb-tree/runtime_c_api.h" + +#include <gtest/gtest.h> + +#include <limits> + +namespace crowdb::tree +{ +TEST(OwnedBudgetCAbi, ReturnedScanControlWindowOutlivesEveryOriginalHandle) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + auto account = TreeCoreAccess::get(*c_api_tree(tree))->binding()->account(); + const uint8_t key = 'k'; + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(tree, 1, &key, 1, &value, 1), 0); + ct_scan_options_v2 scan{}; + scan.struct_size = sizeof(scan); + ct_scan_result *result = nullptr; + ASSERT_EQ(ct_scan_v2(tree, &scan, &result), 0); + ct_owned_budget *budget = nullptr; + ASSERT_EQ(ct_scan_result_owned_budget(result, &budget), 0); + ASSERT_NE(budget, nullptr); + ct_scan_result_free(result); + ct_close(tree); + ct_runtime_free(runtime); + const auto index = static_cast<size_t>(AllocationKind::kMetadata); + const auto before = account->snapshot().physical[index]; + ASSERT_EQ(ct_owned_budget_claim(budget, 64), 0); + auto *control = new uint8_t[64]; + ct_owned_budget_record(budget, 64, 1); + EXPECT_EQ(account->snapshot().physical[index], before + 64); + delete[] control; + ct_owned_budget_record(budget, 64, 0); + ct_owned_budget_release(budget, 64); + EXPECT_EQ(account->snapshot().physical[index], before); + EXPECT_NE(ct_owned_budget_claim(budget, std::numeric_limits<size_t>::max()), 0); + EXPECT_EQ(account->snapshot().physical[index], before); + ct_owned_budget_free(budget); + EXPECT_EQ(account->snapshot().physical[index], 0U); +} + +TEST(OwnedBudgetCAbi, LegacyCallerOwnedBudgetDoesNotClaimNativeCapacity) +{ + ct_owned_budget *budget = nullptr; + ASSERT_EQ(c_api_owned_budget({}, &budget), 0); + EXPECT_EQ(budget, nullptr); + EXPECT_EQ(ct_owned_budget_claim(budget, 64), 0); + ct_owned_budget_record(budget, 64, 1); + ct_owned_budget_record(budget, 64, 0); + ct_owned_budget_release(budget, 64); + ct_owned_budget_free(budget); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/c_api_read_budget_test.cpp b/lib/crowdb-tree/tests/unit/c_api_read_budget_test.cpp new file mode 100644 index 000000000..6bfb7c6ac --- /dev/null +++ b/lib/crowdb-tree/tests/unit/c_api_read_budget_test.cpp @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(CAbiReadBudget, LazySnapshotIterationReturnsPressureInsteadOfThrowingAcrossTheAbi) +{ + ct_page_store *raw_store = nullptr; + ASSERT_EQ(ct_page_store_open_mem(1, &raw_store), 0); + std::unique_ptr<ct_page_store, decltype(&ct_page_store_free)> store(raw_store, &ct_page_store_free); + ct_options options{}; + options.page_store = store.get(); + ct_tree *raw = nullptr; + ASSERT_EQ(ct_open(&options, &raw), 0); + std::unique_ptr<ct_tree, decltype(&ct_close)> tree(raw, &ct_close); + const uint8_t key = 'k'; + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(tree.get(), 1, &key, 1, &value, 1), 0); + ASSERT_EQ(ct_flush(tree.get()), 0); + ct_view *raw_view = nullptr; + ASSERT_EQ(ct_snapshot_view(tree.get(), &raw_view), 0); + std::unique_ptr<ct_view, decltype(&ct_view_release)> view(raw_view, &ct_view_release); + ct_iter *raw_iterator = nullptr; + ASSERT_EQ(ct_view_iter(view.get(), &raw_iterator), 0); + std::unique_ptr<ct_iter, decltype(&ct_iter_release)> iterator(raw_iterator, &ct_iter_release); + auto *core = TreeCoreAccess::get(*c_api_tree(tree.get())); + core->binding()->drain(); + const auto account = core->binding()->account(); + auto blocker = + detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, + detail::ResourceBudget::available(account, detail::BudgetClass::kRouting)); + ASSERT_TRUE(blocker); + int32_t valid = 7; + ct_status status = 0; + EXPECT_NO_THROW(status = ct_iter_next(iterator.get(), nullptr, nullptr, nullptr, nullptr, &valid)); + EXPECT_EQ(status, static_cast<ct_status>(Code::kResourceExhausted)); + EXPECT_EQ(valid, 0); + blocker.reset(); + ct_buf returned_key{}; + ct_buf returned_value{}; + ASSERT_EQ(ct_iter_next(iterator.get(), &returned_key, nullptr, nullptr, &returned_value, &valid), 0); + ASSERT_EQ(valid, 1); + EXPECT_EQ(returned_key.len, 1U); + EXPECT_EQ(returned_value.len, 1U); + EXPECT_EQ(returned_key.data[0], key); + EXPECT_EQ(returned_value.data[0], value); + ct_free_buf(&returned_key); + ct_free_buf(&returned_value); +} + +TEST(CAbiReadBudget, PressureReturnsResourceExhaustedAndDoesNotLatchIoFailure) +{ + ct_options options{}; + ct_tree *raw = nullptr; + ASSERT_EQ(ct_open(&options, &raw), 0); + std::unique_ptr<ct_tree, decltype(&ct_close)> tree(raw, &ct_close); + const uint8_t key = 'k'; + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(tree.get(), 1, &key, 1, &value, 1), 0); + const auto account = TreeCoreAccess::get(*c_api_tree(tree.get()))->binding()->account(); + auto blocker = + detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, + detail::ResourceBudget::available(account, detail::BudgetClass::kRouting)); + ASSERT_TRUE(blocker); + int32_t found = 7; + uint64_t slot = 7; + ct_buf result{}; + EXPECT_EQ(ct_get(tree.get(), &key, 1, &found, &slot, &result), static_cast<int32_t>(Code::kResourceExhausted)); + EXPECT_EQ(found, 0); + EXPECT_EQ(slot, 0U); + EXPECT_EQ(result.data, nullptr); + EXPECT_EQ(result.len, 0U); + EXPECT_FALSE(c_api_tree(tree.get())->get_view("k").found()); + ct_stats stats{}; + ct_get_stats(tree.get(), &stats); + EXPECT_EQ(stats.io_failed, 0); + blocker.reset(); + ASSERT_EQ(ct_get(tree.get(), &key, 1, &found, &slot, &result), 0); + EXPECT_EQ(found, 1); + EXPECT_EQ(slot, 1U); + ASSERT_EQ(result.len, 1U); + EXPECT_EQ(result.data[0], value); + ct_free_buf(&result); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/c_api_scan_async_v2_test.cpp b/lib/crowdb-tree/tests/unit/c_api_scan_async_v2_test.cpp new file mode 100644 index 000000000..c9c41ff94 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/c_api_scan_async_v2_test.cpp @@ -0,0 +1,104 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/c_api.h" +#include "crowdb-tree/status.h" + +#include <gtest/gtest.h> + +#include <memory> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +class ScanAsyncV2CAbi : public ::testing::Test +{ + protected: + void SetUp() override + { + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + scan.struct_size = sizeof(scan); + scan.limit = 1; + } + + void TearDown() override + { + ct_close(tree); + ct_runtime_free(runtime); + } + + void expect_key(const std::string &key, bool more) + { + ct_future *future = nullptr; + ASSERT_EQ(ct_scan_async_v2(tree, &scan, &future), 0); + ASSERT_NE(future, nullptr); + int32_t done = 0; + ct_scan_result *result = nullptr; + while (done == 0) { + ASSERT_EQ(ct_future_poll_scan_v2(future, &done, &result), 0); + std::this_thread::yield(); + } + std::unique_ptr<ct_scan_result, decltype(&ct_scan_result_free)> owner(result, ct_scan_result_free); + const uint8_t *data = nullptr; + size_t len = 0; + uint64_t count = 0; + int32_t truncated = 0; + ASSERT_EQ(ct_scan_result_data(result, &data, &len, &count, &truncated), 0); + ASSERT_EQ(count, 1U); + ASSERT_GE(len, key.size() + 17); + EXPECT_EQ(data[0], key.size()); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(data + 4), key.size()), key); + EXPECT_EQ(truncated != 0, more); + } + + ct_runtime *runtime = nullptr; + ct_tree *tree = nullptr; + ct_scan_options_v2 scan{}; +}; +} // namespace + +TEST_F(ScanAsyncV2CAbi, EmptyCursorResumesWithoutRepeatingTheFirstWindow) +{ + const uint8_t key = 'a'; + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(tree, 1, nullptr, 0, &value, 1), 0); + ASSERT_EQ(ct_apply_put(tree, 2, &key, 1, &value, 1), 0); + expect_key("", true); + scan.has_start_bound = 1; + expect_key("a", false); + scan.reverse = 1; + scan.start_inclusive = 1; + expect_key("", false); +} + +TEST_F(ScanAsyncV2CAbi, InvalidOptionsFailBeforeReturningAnOwnedFuture) +{ + ct_future *future = nullptr; + --scan.struct_size; + EXPECT_EQ(ct_scan_async_v2(tree, &scan, &future), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(future, nullptr); + ++scan.struct_size; + scan.strict_byte_budget = 1; + EXPECT_EQ(ct_scan_async_v2(tree, &scan, &future), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(future, nullptr); + scan.strict_byte_budget = 0; + scan.start_len = 1; + EXPECT_EQ(ct_scan_async_v2(tree, &scan, &future), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(future, nullptr); +} + +TEST_F(ScanAsyncV2CAbi, AbandonedFutureRetainsItsAcceptedOperationUntilClose) +{ + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(tree, 1, nullptr, 0, &value, 1), 0); + ct_future *future = nullptr; + ASSERT_EQ(ct_scan_async_v2(tree, &scan, &future), 0); + ASSERT_NE(future, nullptr); + ct_future_free(future); + ct_close(tree); + tree = nullptr; +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/c_api_scan_owner_test.cpp b/lib/crowdb-tree/tests/unit/c_api_scan_owner_test.cpp new file mode 100644 index 000000000..f6684e036 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/c_api_scan_owner_test.cpp @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/c_api.h" +#include "runtime/tree_runtime_c_api.h" + +#include <gtest/gtest.h> + +#include <thread> + +namespace crowdb::tree +{ +TEST(ScanOwnerCAbi, PackedBytesAndChargeOutliveTreeAndRuntimeHandles) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + Config config; + ASSERT_TRUE(configure_tree_runtime(runtime, nullptr, &config).ok()); + const auto baseline = config.runtime->snapshot().allocated.routing_bytes; + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + const std::string key = "key"; + const std::string value = "value"; + ASSERT_EQ(ct_apply_put(tree, 1, reinterpret_cast<const uint8_t *>(key.data()), key.size(), + reinterpret_cast<const uint8_t *>(value.data()), value.size()), + 0); + ct_scan_options_v2 scan{}; + scan.struct_size = sizeof(scan); + ct_scan_result *result = nullptr; + ASSERT_EQ(ct_scan_v2(tree, &scan, &result), 0); + ASSERT_NE(result, nullptr); + ct_close(tree); + ct_runtime_free(runtime); + const uint8_t *data = nullptr; + size_t len = 0; + uint64_t count = 0; + int32_t truncated = 0; + ASSERT_EQ(ct_scan_result_data(result, &data, &len, &count, &truncated), 0); + ASSERT_EQ(count, 1U); + ASSERT_EQ(len, key.size() + value.size() + 17); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(data + len - value.size()), value.size()), value); + EXPECT_GT(config.runtime->snapshot().allocated.routing_bytes, baseline); + ct_scan_result_free(result); + EXPECT_LT(config.runtime->snapshot().allocated.routing_bytes, baseline); +} + +TEST(ScanOwnerCAbi, AsyncPollTransfersOwnerAndAbandonedFutureDrainsSafely) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + const uint8_t key = 'k'; + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(tree, 1, &key, 1, &value, 1), 0); + auto *abandoned = ct_scan_async(tree, nullptr, 0, nullptr, 0, nullptr, 0, 1, 128, 0, 0); + ASSERT_NE(abandoned, nullptr); + ct_future_free(abandoned); + ct_close(tree); + tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + ASSERT_EQ(ct_apply_put(tree, 1, &key, 1, &value, 1), 0); + auto *future = ct_scan_async(tree, nullptr, 0, nullptr, 0, nullptr, 0, 1, 128, 0, 0); + ASSERT_NE(future, nullptr); + int32_t done = 0; + ct_scan_result *result = nullptr; + while (done == 0) { + ASSERT_EQ(ct_future_poll_scan_v2(future, &done, &result), 0); + std::this_thread::yield(); + } + ASSERT_NE(result, nullptr); + ct_close(tree); + ct_runtime_free(runtime); + const uint8_t *data = nullptr; + size_t len = 0; + uint64_t count = 0; + int32_t truncated = 0; + ASSERT_EQ(ct_scan_result_data(result, &data, &len, &count, &truncated), 0); + EXPECT_EQ(count, 1U); + EXPECT_EQ(data[len - 1], value); + ct_scan_result_free(result); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/c_api_snapshot_backend_owner_test.cpp b/lib/crowdb-tree/tests/unit/c_api_snapshot_backend_owner_test.cpp new file mode 100644 index 000000000..2d4d892b4 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/c_api_snapshot_backend_owner_test.cpp @@ -0,0 +1,106 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/local_backend_state.h" +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/runtime_c_api.h" +#include "test_tmp.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +void continue_after_close(bool injected) +{ + tree_test::TempDir directory("snapshot_backend_owner_"); + ct_runtime *raw_runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &raw_runtime), 0); + std::unique_ptr<ct_runtime, decltype(&ct_runtime_free)> runtime(raw_runtime, &ct_runtime_free); + ct_page_store *raw_store = nullptr; + if (injected) { + ASSERT_EQ(ct_page_store_open_mem(4096, &raw_store), 0); + } + std::unique_ptr<ct_page_store, decltype(&ct_page_store_free)> store(raw_store, &ct_page_store_free); + ct_options options{}; + options.page_store = store.get(); + options.path = injected ? nullptr : directory.path.c_str(); + options.backend = CT_BACKEND_BLOCK; + options.block_size = 65536; + options.iu_size = 4096; + options.frame_bytes = 4096; + options.max_inline_value = 64; + options.sync_mode = CT_SYNC_SKIP; + ct_tree *raw_tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime.get(), nullptr, &raw_tree), 0); + std::unique_ptr<ct_tree, decltype(&ct_close)> tree(raw_tree, &ct_close); + const std::string value(32768, 'v'); + for (uint8_t key : {uint8_t{'a'}, uint8_t{'b'}}) { + ASSERT_EQ(ct_apply_put(tree.get(), static_cast<uint64_t>(key - 'a') + 1, &key, 1, + reinterpret_cast<const uint8_t *>(value.data()), value.size()), + 0); + } + uint64_t coverage = 0; + ASSERT_EQ(ct_snapshot(tree.get(), &coverage), 0); + ASSERT_EQ(coverage, 2U); + tree.reset(); + raw_tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime.get(), nullptr, &raw_tree), 0); + tree.reset(raw_tree); + store.reset(); + auto *core = TreeCoreAccess::get(*c_api_tree(tree.get())); + ASSERT_NE(core, nullptr); + std::weak_ptr<const void> backend = core->config().backend_owner; + std::weak_ptr<const void> domain = core->local_backend()->domain; + ASSERT_FALSE(backend.expired()); + ASSERT_FALSE(domain.expired()); + ct_export *raw_export = nullptr; + ASSERT_EQ(ct_snapshot_export_begin(tree.get(), 64, &raw_export), 0); + std::unique_ptr<ct_export, decltype(&ct_snapshot_export_end)> stream(raw_export, &ct_snapshot_export_end); + const auto total = ct_snapshot_export_total_bytes(stream.get()); + ct_snapshot_chunk *first = nullptr; + int32_t done = 0; + ASSERT_EQ(ct_snapshot_export_next_v2(stream.get(), 0, &first, &done), 0); + ASSERT_EQ(done, 0); + const uint8_t *first_data = nullptr; + size_t returned = 0; + ASSERT_EQ(ct_snapshot_chunk_data(first, &first_data, &returned), 0); + ASSERT_GT(returned, 0U); + ASSERT_LE(returned, 64U); + ASSERT_NE(first_data, nullptr); + ct_snapshot_chunk_free(first); + tree.reset(); + runtime.reset(); + ASSERT_FALSE(backend.expired()); + ASSERT_FALSE(domain.expired()); + while (done == 0) { + ct_snapshot_chunk *chunk = nullptr; + ASSERT_EQ(ct_snapshot_export_next_v2(stream.get(), returned, &chunk, &done), 0); + std::unique_ptr<ct_snapshot_chunk, decltype(&ct_snapshot_chunk_free)> held(chunk, &ct_snapshot_chunk_free); + const uint8_t *data = nullptr; + size_t length = 0; + ASSERT_EQ(ct_snapshot_chunk_data(held.get(), &data, &length), 0); + ASSERT_GT(length, 0U); + ASSERT_LE(length, 64U); + ASSERT_NE(data, nullptr); + returned += length; + ASSERT_LE(returned, total); + } + EXPECT_EQ(returned, total); + stream.reset(); + EXPECT_TRUE(backend.expired()); + EXPECT_TRUE(domain.expired()); +} + +TEST(SnapshotBackendOwnerCAbi, OwnedBlockStreamContinuesColdReadsAfterTreeAndRuntimeClose) +{ + continue_after_close(false); +} + +TEST(SnapshotBackendOwnerCAbi, InjectedStoreStreamRetainsItsIndependentBackendAndDomain) +{ + continue_after_close(true); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/c_api_snapshot_owner_test.cpp b/lib/crowdb-tree/tests/unit/c_api_snapshot_owner_test.cpp new file mode 100644 index 000000000..097d132e5 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/c_api_snapshot_owner_test.cpp @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/c_api.h" +#include "runtime/tree_runtime_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(SnapshotOwnerCAbi, ChunkAndChargeOutliveTheStreamTreeAndRuntimeHandles) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + Config config; + ASSERT_TRUE(configure_tree_runtime(runtime, nullptr, &config).ok()); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + const uint8_t key = 'k'; + const std::string value(8192, 'v'); + ASSERT_EQ(ct_apply_put(tree, 1, &key, 1, reinterpret_cast<const uint8_t *>(value.data()), value.size()), 0); + ASSERT_EQ(ct_flush(tree), 0); + ct_export *stream = nullptr; + ASSERT_EQ(ct_snapshot_export_begin(tree, 64, &stream), 0); + ct_snapshot_chunk *chunk = nullptr; + int32_t done = 0; + ASSERT_EQ(ct_snapshot_export_next_v2(stream, 0, &chunk, &done), 0); + EXPECT_EQ(done, 0); + const uint8_t *data = nullptr; + size_t length = 0; + ASSERT_EQ(ct_snapshot_chunk_data(chunk, &data, &length), 0); + ASSERT_EQ(length, 64U); + const std::string retained(reinterpret_cast<const char *>(data), length); + ct_buf copied{}; + ASSERT_EQ(ct_snapshot_export_next(stream, length, &copied, &done), 0); + EXPECT_EQ(copied.len, 64U); + ct_free_buf(&copied); + ct_snapshot_export_end(stream); + ct_close(tree); + ct_runtime_free(runtime); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(data), length), retained); + const auto held = config.runtime->snapshot().allocated.routing_bytes; + ct_snapshot_chunk_free(chunk); + EXPECT_LT(config.runtime->snapshot().allocated.routing_bytes, held); +} + +TEST(SnapshotOwnerCAbi, InvalidOffsetDoesNotAdvanceAndLegacyCopiedBuffersRemainOwned) +{ + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + const uint8_t key = 'k'; + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(tree, 1, &key, 1, &value, 1), 0); + ASSERT_EQ(ct_flush(tree), 0); + ct_export *stream = nullptr; + ASSERT_EQ(ct_snapshot_export_begin(tree, 16, &stream), 0); + ct_snapshot_chunk *chunk = nullptr; + int32_t done = 0; + EXPECT_EQ(ct_snapshot_export_next_v2(stream, 1, &chunk, &done), static_cast<ct_status>(Code::kInvalidArgument)); + EXPECT_EQ(chunk, nullptr); + EXPECT_EQ(ct_snapshot_export_offset(stream), 0U); + ct_buf copied{}; + ASSERT_EQ(ct_snapshot_export_next(stream, 0, &copied, &done), 0); + ASSERT_GT(copied.len, 0U); + const std::string retained(reinterpret_cast<const char *>(copied.data), copied.len); + ct_snapshot_export_end(stream); + ct_close(tree); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(copied.data), copied.len), retained); + ct_free_buf(&copied); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/captured_rebase_reuse_test.cpp b/lib/crowdb-tree/tests/unit/captured_rebase_reuse_test.cpp new file mode 100644 index 000000000..90c53f29c --- /dev/null +++ b/lib/crowdb-tree/tests/unit/captured_rebase_reuse_test.cpp @@ -0,0 +1,159 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "load_budget_test_support.h" +#include "published_read_test_support.h" +#include "snapshot/captured_plan.h" +#include "snapshot/captured_rebase.h" +#include "snapshot/primary_backing.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +TEST(CapturedRebaseReuse, LayoutCountsOnlyExactPrimariesWithoutRelocation) +{ + for (bool relocation : {false, true}) { + CapturedWriteLimits limits; + limits.frames.exclusion.length = relocation ? 4096 : 0; + CapturedLayoutPlanner planner(2, 1, limits, 1, {}, {}); + for (uint64_t pid = 0; pid < 2; ++pid) { + ASSERT_TRUE( + planner + .accept( + {.page_id = pid, .summary = {}, .reused_overflow = {}, .owner = {}, .reused_primary = pid == 0}) + .ok()); + ASSERT_TRUE(planner.step().ok()); + } + ASSERT_TRUE(planner.finish().ok()); + EXPECT_EQ(planner.result().primary_frames, 2U); + EXPECT_EQ(planner.result().reused_primary_frames, relocation ? 0U : 1U); + EXPECT_EQ(planner.result().frame_count_upper, 2U); + } +} + +TEST(CapturedRebaseReuse, LayoutSeparatesRadixBatchesFromDurableSegments) +{ + CapturedLayoutPlanner planner(1025, 1, {}, 2, {}, {}); + for (uint64_t pid : {0U, 63U, 64U, 128U, 1023U, 1024U}) { + ASSERT_TRUE( + planner + .accept( + {.page_id = pid, .summary = {}, .reused_overflow = {}, .owner = {}, .reused_primary = pid == 128}) + .ok()); + ASSERT_TRUE(planner.step().ok()); + } + ASSERT_TRUE(planner.finish().ok()); + EXPECT_EQ(planner.result().primary_frames, 6U); + EXPECT_EQ(planner.result().reused_primary_frames, 1U); + EXPECT_EQ(planner.result().segments, 2U); + EXPECT_EQ(planner.result().rebase_batches, 4U); +} + +struct TestReuse +{ + explicit TestReuse(bool later_overlay) + { + auto base_ref = cold(33, 10); + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(base_ref); + frame = {.page_id = 0, + .type = page_type::kLeafBase, + .frame_bytes = 4096, + .logical_bytes = 4096, + .backing = base_ref.backing}; + frame.summary.emplace(); + frame.summary->live_counts_known = true; + frame.summary->live_keys = 1; + frame.summary->live_key_bytes = 1; + frame.summary->live_value_bytes = 3; + frame.stored_overflow = std::make_shared<StoredOverflowReferences>(); + auto page = published(leaf); + page->metrics.dirty = false; + page->primary = make_primary_backing(frame, *leaf, 10, 0, {}); + EXPECT_TRUE(page->primary); + initial = std::make_shared<PublishedTreeState>(nullptr, 10); + initial->coverage = 10; + initial->root_page_id = 0; + initial->next_page_id = 1; + MappingView::CopyStats copied; + initial->mapping = initial->mapping.replace( + std::array<PageReplacement, 1>{{{.page_id = 0, .expected = {}, .replacement = page}}}, copied); + EXPECT_TRUE(apply_page_metrics(initial->stats, nullptr, &page->metrics)); + current = std::make_shared<PublishedTreeState>(*initial); + if (later_overlay) { + auto changed_leaf = std::make_shared<PageState>(*leaf); + auto run = test_read::page({ + {.key = "a", .slot = 11, .value = "new"} + }); + changed_leaf->overlays.push_back(run->overlays.front()); + auto changed = std::make_shared<PublishedPage>(*page); + changed->leaf = changed_leaf; + changed->metrics.dirty = true; + current->mapping = current->mapping.replace( + std::array<PageReplacement, 1>{{{.page_id = 0, .expected = page, .replacement = changed}}}, copied); + } + } + + DurableCapturedFrame frame; + std::shared_ptr<PublishedTreeState> initial; + std::shared_ptr<PublishedTreeState> current; +}; + +TEST(CapturedRebaseReuse, ZeroReplacementCreditPreservesExactPageAndLaterOverlay) +{ + for (bool later_overlay : {false, true}) { + TestReuse fixture(later_overlay); + LoadCredits credits; + const auto bound = CapturedRebaseOperation::allocation_bound(0); + if (!bound) { + FAIL() << "zero-replacement rebase bound overflow"; + } + auto paid = credits.credits.bytes->take_reserved( + *bound + detail::shared_allocation_bound<detail::AllocationReservation>()); + if (!paid) { + FAIL() << "zero-replacement rebase credit unavailable"; + } + auto metadata = detail::AllocationReservation::create(std::move(*paid)); + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher(4, 1, {}, std::pmr::get_default_resource(), {}, + {.metadata = {}, .initial = fixture.current, .source_catalog = {}}); + auto pages = std::make_shared<CapturedDurablePages>(nullptr); + pages->coverage = 10; + pages->generation = fixture.initial->generation; + pages->root_page_id = 0; + pages->frames.emplace(0, fixture.frame); + std::shared_ptr<CapturedRebaseOperation> operation; + ASSERT_TRUE( + CapturedRebaseOperation::create(publisher, fixture.initial, pages, runtime, lane, metadata, &operation) + .ok()); + Completion done; + auto connected = stdexec::connect(operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + runtime->drain(); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(operation->result().installed, 0U); + EXPECT_EQ(operation->result().deferred, 0U); + EXPECT_EQ(operation->result().reused, 1U); + const auto actual = publisher.capture()->mapping.get(0); + EXPECT_EQ(actual, fixture.current->mapping.get(0)); + EXPECT_EQ(actual->leaf->overlays.size(), later_overlay ? 1U : 0U); + if (later_overlay) { + auto loader = std::make_shared<TestLoader>(); + loader->frames[33] = base({ + {.key = "a", .value = "old"} + }); + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(publisher.capture(), loader, Slice("a"), {}, &get).ok()); + complete_inline(get->sender()); + ASSERT_TRUE(get->result().found); + EXPECT_EQ(get->result().cell.slot, 11U); + EXPECT_EQ(get->result().cell.value.to_string(), "new"); + } + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/chunk_collector_test.cpp b/lib/crowdb-tree/tests/unit/chunk_collector_test.cpp new file mode 100644 index 000000000..b41c8805c --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_collector_test.cpp @@ -0,0 +1,114 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_chunk_collector.h" +#include "runtime/bootstrap_budget.h" + +namespace crowdb::tree::detail +{ +TEST(ChunkCollector, ReferencedOrPinnedStageRemainsUntilActualOwnersRelease) +{ + auto catalog = std::make_shared<TestCollectorCatalog>(); + catalog->record = test_collector_stage(); + catalog->references = 1; + catalog->pins = 65; + TestFixedTransport transport(catalog); + ChunkCollector collector; + auto budget = BootstrapBudget::create(1024 * 1024); + ChunkCollectionProgress progress; + for (size_t step = 0; step < 12; ++step) { + ASSERT_TRUE(collector.step(*catalog, transport, 17, 1, budget, &progress).ok()); + } + EXPECT_EQ(transport.deletions, 0U); + EXPECT_EQ(catalog->largest_pin_window, 64U); + EXPECT_GT(catalog->pin_calls, 1U); + catalog->references = 0; + for (size_t step = 0; step < 12; ++step) { + ASSERT_TRUE(collector.step(*catalog, transport, 17, 1, budget, &progress).ok()); + } + EXPECT_EQ(transport.deletions, 0U); + catalog->pins = 0; + for (size_t step = 0; step < 12 && catalog->record; ++step) { + ASSERT_TRUE(collector.step(*catalog, transport, 17, 1, budget, &progress).ok()); + } + EXPECT_FALSE(catalog->record); + EXPECT_EQ(transport.drains, 1U); + EXPECT_EQ(transport.deletions, 1U); + EXPECT_EQ(progress.logical_capacity_bytes, 4096U); + EXPECT_EQ(progress.stages_removed, 1U); +} + +TEST(ChunkCollector, UnknownReferenceReleaseReloadsSameCandidateBeforeAdvancing) +{ + auto catalog = std::make_shared<TestCollectorCatalog>(); + catalog->record = test_collector_stage(); + catalog->references = 2; + catalog->candidate = { + .id = {.owner_epoch = 1, .operation_high = 3, .operation_low = 4}, + .generation = 1, + .reference_count = 65, + .prepared_references = 65, + .prepared_windows = 2, + .released_windows = 0, + .state = ChunkCandidateState::Aborted + }; + TestFixedTransport transport(catalog); + ChunkCollector collector; + auto budget = BootstrapBudget::create(1024 * 1024); + ChunkCollectionProgress progress; + catalog->lose_release = true; + EXPECT_EQ(collector.step(*catalog, transport, 17, 1, budget, &progress).code(), Code::kUnavailable); + EXPECT_EQ(catalog->references, 1U); + catalog->lose_release = false; + for (size_t step = 0; step < 20 && catalog->record; ++step) { + ASSERT_TRUE(collector.step(*catalog, transport, 17, 1, budget, &progress).ok()); + } + EXPECT_EQ(catalog->references, 0U); + EXPECT_EQ(catalog->release_calls, 2U); + EXPECT_FALSE(catalog->record); + EXPECT_EQ(transport.deletions, 1U); +} + +TEST(ChunkCollector, UnknownPhysicalDeleteKeepsIdentityForTheNextAcceptedPass) +{ + auto catalog = std::make_shared<TestCollectorCatalog>(); + catalog->record = test_collector_stage(); + catalog->record->state = ChunkStageState::Retired; + TestFixedTransport transport(catalog); + transport.unknown_delete = true; + ChunkCollector collector; + auto budget = BootstrapBudget::create(1024 * 1024); + ChunkCollectionProgress progress; + ASSERT_TRUE(collector.step(*catalog, transport, 17, 2, budget, &progress).ok()); + EXPECT_EQ(collector.step(*catalog, transport, 17, 2, budget, &progress).code(), Code::kUnavailable); + ASSERT_TRUE(catalog->record); + if (catalog->record) { + EXPECT_EQ(catalog->record->state, ChunkStageState::Deleting); + } + EXPECT_EQ(progress.logical_capacity_bytes, 0U); + transport.unknown_delete = false; + ASSERT_TRUE(collector.step(*catalog, transport, 17, 2, budget, &progress).ok()); + EXPECT_FALSE(catalog->record); + EXPECT_EQ(progress.logical_capacity_bytes, 4096U); + EXPECT_EQ(transport.deletions, 2U); +} + +TEST(ChunkCollector, MissingRecoveryProofCannotRetireOrDeleteOlderStages) +{ + auto catalog = std::make_shared<TestCollectorCatalog>(); + catalog->record = test_collector_stage(); + catalog->recovered = false; + TestFixedTransport transport(catalog); + ChunkCollector collector; + auto budget = BootstrapBudget::create(1024 * 1024); + ChunkCollectionProgress progress; + ASSERT_TRUE(collector.step(*catalog, transport, 17, 2, budget, &progress).ok()); + EXPECT_EQ(collector.step(*catalog, transport, 17, 2, budget, &progress).code(), Code::kUnavailable); + EXPECT_EQ(transport.deletions, 0U); + catalog->recovered = true; + for (size_t step = 0; step < 8 && catalog->record; ++step) { + ASSERT_TRUE(collector.step(*catalog, transport, 17, 2, budget, &progress).ok()); + } + EXPECT_FALSE(catalog->record); + EXPECT_EQ(transport.deletions, 1U); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/chunk_fixed_pack_test.cpp b/lib/crowdb-tree/tests/unit/chunk_fixed_pack_test.cpp new file mode 100644 index 000000000..7f710450e --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_fixed_pack_test.cpp @@ -0,0 +1,140 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_chunk_stage_backend.h" + +#include <array> + +namespace crowdb::tree::detail +{ +namespace +{ +struct TestPackBudget +{ + static constexpr ResourceBudget::Limits limits{1U << 20, 1U << 20, 1U << 20, 1U << 20, 1U << 20, 16}; + ResourceBudget host{limits, 4}; + std::shared_ptr<ResourceBudget::Account> account = host.make_account(limits); + + [[nodiscard]] std::shared_ptr<AllocationReservation> bytes(size_t count = 1U << 20) const + { + auto value = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, count); + if (!value) { + throw std::bad_alloc(); + } + return AllocationReservation::create(std::move(*value)); + } +}; + +constexpr ChunkId fixed{uint64_t{2} << 56, 1}; +constexpr ChunkStageId operation{.owner_epoch = 1, .operation_high = 10, .operation_low = 20}; +} // namespace + +TEST(ChunkFixedPack, UnknownAllocateRetainsPreexistingStageIdentity) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + transport->unknown_allocate = true; + ChunkFixedPackStore store(17, 1, 2, catalog, transport); + TestPackBudget budget; + auto credit = budget.bytes(); + std::array<uint8_t, 32> input{}; + ChunkFixedPackRef output; + EXPECT_FALSE(store.write(operation, fixed, PagePurpose::kBtreePage, input, credit, &output).ok()); + if (!catalog->record.has_value()) { + FAIL() << "durable stage missing"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Reserved); + EXPECT_EQ(catalog->record->chunk_low, fixed.low); + EXPECT_EQ(transport->allocations, 1U); + EXPECT_EQ(output.length, 0U); + ChunkLayout physical; + EXPECT_TRUE(transport->query_chunk(fixed, &physical).ok()); +} + +TEST(ChunkFixedPack, UnsupportedReadFenceRetainsAllocationWithoutPublishingBacking) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + transport->fail_fence = true; + ChunkFixedPackStore store(17, 1, 2, catalog, transport); + TestPackBudget budget; + auto credit = budget.bytes(); + std::array<uint8_t, 32> input{}; + ChunkFixedPackRef output; + EXPECT_EQ(store.write(operation, fixed, PagePurpose::kBtreePage, input, credit, &output).code(), + Code::kNotSupported); + if (!catalog->record.has_value()) { + FAIL() << "durable stage missing"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Reserved); + EXPECT_EQ(transport->allocations, 1U); + EXPECT_EQ(output.length, 0U); + ChunkLayout physical; + EXPECT_TRUE(transport->query_chunk(fixed, &physical).ok()); + EXPECT_EQ(physical.acknowledged_bytes, 0U); +} + +TEST(ChunkFixedPack, AdmissionFailurePrecedesStageAndAllocation) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + ChunkFixedPackStore store(17, 1, 2, catalog, transport); + TestPackBudget budget; + auto credit = budget.bytes(1024); + std::array<uint8_t, 4096> input{}; + ChunkFixedPackRef output; + EXPECT_FALSE(store.write(operation, fixed, PagePurpose::kBtreePage, input, credit, &output).ok()); + EXPECT_FALSE(catalog->record); + EXPECT_EQ(transport->allocations, 0U); +} + +TEST(ChunkFixedPack, ReloadsFixedPackWithoutAManifestAndFallsBackFromCorruptMirror) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + ChunkFixedPackStore store(17, 1, 2, catalog, transport); + TestPackBudget budget; + auto credit = budget.bytes(); + std::vector<uint8_t> input(70'000, 87); + ChunkFixedPackRef output; + ASSERT_TRUE(store.write(operation, fixed, PagePurpose::kBtreePage, input, credit, &output).ok()); + if (!catalog->record.has_value()) { + FAIL() << "durable stage missing"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Reloadable); + transport->memory.corrupt_mirror(fixed, 0, 20); + std::shared_ptr<const ChunkPackWindow> loaded; + ASSERT_TRUE(store.read(output, credit, &loaded).ok()); + EXPECT_EQ(std::vector<uint8_t>(loaded->bytes.begin(), loaded->bytes.end()), input); + EXPECT_EQ(catalog->manifest_loads, 0U); + loaded.reset(); + ++output.checksum; + EXPECT_FALSE(store.read(output, credit, &loaded).ok()); + catalog->record.reset(); + credit->seal(); + credit.reset(); + EXPECT_EQ(budget.host.snapshot().allocated[0], 0U); +} + +TEST(ChunkFixedPack, WriteFailureKeepsProvisionalReferencesProtected) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + transport->fail_write = true; + ChunkFixedPackStore store(17, 1, 2, catalog, transport); + TestPackBudget budget; + auto credit = budget.bytes(); + std::array<uint8_t, 32> input{}; + ChunkFixedPackRef output; + EXPECT_FALSE(store.write(operation, fixed, PagePurpose::kBtreePage, input, credit, &output).ok()); + if (!catalog->record.has_value()) { + FAIL() << "durable stage missing"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Writing); + EXPECT_EQ(catalog->record->packs.size(), 1U); + EXPECT_EQ(output.length, 0U); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/chunk_physical_gc_test.cpp b/lib/crowdb-tree/tests/unit/chunk_physical_gc_test.cpp new file mode 100644 index 000000000..04afb3095 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_physical_gc_test.cpp @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_stage_gc.h" +#include "common/test_chunk_physical.h" +#include "common/test_chunk_stage_backend.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_chunk; +using namespace detail; + +ChunkStageRecord retired() +{ + return {.tree_id = 17, + .id = TestPhysicalHost::stage, + .revision = 2, + .chunk_high = TestPhysicalHost::chunk.high, + .chunk_low = TestPhysicalHost::chunk.low, + .capacity = 4096, + .manifest_generation = 0, + .purpose = 2, + .state = ChunkStageState::Retired, + .packs = {}}; +} + +TEST(ChunkPhysicalGc, LostDeleteReplyKeepsPhysicalChargeUntilDeletingNotFoundRetryAcknowledges) +{ + TestPhysicalHost host; + auto catalog = std::make_shared<TestStageCatalog>(); + catalog->record = retired(); + TestFixedTransport transport(catalog); + transport.unknown_delete = true; + transport.absent_after_delete = true; + auto result = + reclaim_retired_stage(*catalog, transport, 17, 1, TestPhysicalHost::stage, host.budget, host.registry); + EXPECT_FALSE(result.status.ok()); + EXPECT_FALSE(result.deletion_acknowledged); + EXPECT_TRUE(transport.physically_deleted); + if (!catalog->record) { + FAIL() << "expected protected stage"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Deleting); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 8192U); + transport.unknown_delete = false; + result = reclaim_retired_stage(*catalog, transport, 17, 1, TestPhysicalHost::stage, host.budget, host.registry); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + EXPECT_TRUE(result.stage_removed); + EXPECT_TRUE(result.deletion_acknowledged); + EXPECT_EQ(transport.deletions, 2U); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 0U); +} + +TEST(ChunkPhysicalGc, ReusedChunkEpochCannotAuthorizeDeleteOrRemoveItsPhysicalCharge) +{ + TestPhysicalHost host; + auto catalog = std::make_shared<TestStageCatalog>(); + catalog->record = retired(); + TestFixedTransport transport(catalog); + transport.physical_epoch = 2; + auto stage = TestPhysicalHost::stage; + ++stage.owner_epoch; + auto receipt = TestPhysicalHost::receipt; + ++receipt.owner_epoch; + std::shared_ptr<ChunkPhysicalPack> pack; + ASSERT_TRUE(host.registry->prepare(stage, TestPhysicalHost::chunk, host.budget, &pack).ok()); + ASSERT_TRUE(host.registry->observe(pack, receipt, host.budget).ok()); + const auto result = + reclaim_retired_stage(*catalog, transport, 17, 2, TestPhysicalHost::stage, host.budget, host.registry); + EXPECT_EQ(result.status.code(), Code::kUnavailable); + EXPECT_EQ(transport.deletions, 0U); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 8192U); +} + +TEST(ChunkPhysicalGc, UnknownAllocateReplyAlreadyAccountsTheExactMirrorsWhileStageRemainsReserved) +{ + TestPhysicalHost host; + auto catalog = std::make_shared<TestStageCatalog>(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + transport->unknown_allocate = true; + ChunkFixedPackStore store(17, 1, 2, catalog, transport, host.registry); + std::array<uint8_t, 32> input{}; + ChunkFixedPackRef output; + std::shared_ptr<ChunkPhysicalPack> physical; + const auto status = store.write(TestPhysicalHost::stage, TestPhysicalHost::chunk, PagePurpose::kBtreePage, input, + host.budget, &output, &physical); + EXPECT_EQ(status.code(), Code::kUnavailable); + if (!catalog->record) { + FAIL() << "expected protected stage"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Reserved); + EXPECT_EQ(transport->allocations, 1U); + EXPECT_NE(catalog->record->capacity, 8192U); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 8192U); + EXPECT_EQ(output.length, 0U); +} + +TEST(ChunkPhysicalGc, MissingCommandCreditRejectsBeforeExternalAllocation) +{ + TestPhysicalHost host; + auto catalog = std::make_shared<TestStageCatalog>(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + ChunkFixedPackStore store(17, 1, 2, catalog, transport, host.registry); + host.budget->reject = true; + std::array<uint8_t, 32> input{}; + ChunkFixedPackRef output; + std::shared_ptr<ChunkPhysicalPack> physical; + EXPECT_EQ(store + .write(TestPhysicalHost::stage, TestPhysicalHost::chunk, PagePurpose::kBtreePage, input, host.budget, + &output, &physical) + .code(), + Code::kResourceExhausted); + EXPECT_EQ(transport->allocations, 0U); + EXPECT_FALSE(catalog->record); + host.budget->reject = false; +} +} // namespace + +TEST(ChunkPhysicalGc, AdvertisedWriteBoundPaysCanonicalOwnersAtTheHardLimit) +{ + TestPhysicalHost host; + auto catalog = std::make_shared<TestStageCatalog>(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + ChunkFixedPackStore store(17, 1, 2, catalog, transport, host.registry); + const std::array<uint8_t, 4096> input{}; + const auto bound = ChunkFixedPackStore::write_allocation_bound(input.size()); + if (!bound) { + FAIL() << "expected bounded admission"; + return; + } + auto credit = std::make_shared<TestPhysicalBudget>(); + credit->queue = host.queue; + credit->limit = *bound; + ChunkFixedPackRef output; + std::shared_ptr<ChunkPhysicalPack> physical; + const auto status = store.write(TestPhysicalHost::stage, TestPhysicalHost::chunk, PagePurpose::kBtreePage, input, + credit, &output, &physical); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_LE(credit->claimed.load(), *bound); + EXPECT_EQ(host.registry->backing_stats().allocated_bytes, 8192U); + ASSERT_TRUE(host.registry->deleted(TestPhysicalHost::stage, TestPhysicalHost::chunk, {}, credit).ok()); + physical.reset(); + catalog->record.reset(); + host.lane->drain(); + EXPECT_EQ(credit->claimed.load(), 0U); +} + +TEST(ChunkPhysicalGc, RejectedStageReturnsAllPreallocatedRegistryMetadataWithoutAllocatingRollback) +{ + TestPhysicalHost host; + auto catalog = std::make_shared<TestStageCatalog>(); + catalog->record = retired(); + auto transport = std::make_shared<TestFixedTransport>(catalog); + ChunkFixedPackStore store(17, 1, 2, catalog, transport, host.registry); + auto credit = std::make_shared<TestPhysicalBudget>(); + credit->queue = host.queue; + std::array<uint8_t, 32> input{}; + ChunkFixedPackRef output; + std::shared_ptr<ChunkPhysicalPack> physical; + const auto status = store.write(TestPhysicalHost::stage, TestPhysicalHost::chunk, PagePurpose::kBtreePage, input, + credit, &output, &physical); + EXPECT_EQ(status.code(), Code::kUnavailable); + EXPECT_EQ(transport->allocations, 0U); + credit->reject = true; + physical.reset(); + host.lane->drain(); + EXPECT_EQ(credit->claimed.load(), 0U); + credit->reject = false; +} + +TEST(ChunkPhysicalGc, DurableReclaimedReceiptEndsObservationBeforeInterruptedMetadataEraseRetries) +{ + TestPhysicalHost host; + auto pack = host.prepare(); + ASSERT_TRUE(host.registry->observe(pack, TestPhysicalHost::receipt, host.budget).ok()); + auto catalog = std::make_shared<TestStageCatalog>(); + catalog->record = retired(); + catalog->record->state = ChunkStageState::Reclaimed; + catalog->record->revision = 4; + TestFixedTransport transport(catalog); + transport.physically_deleted = true; + transport.absent_after_delete = true; + const auto result = + reclaim_retired_stage(*catalog, transport, 17, 1, TestPhysicalHost::stage, host.budget, host.registry); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + EXPECT_TRUE(result.stage_removed); + EXPECT_EQ(transport.deletions, 0U); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/chunk_physical_registry_test.cpp b/lib/crowdb-tree/tests/unit/chunk_physical_registry_test.cpp new file mode 100644 index 000000000..c523fd8cc --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_physical_registry_test.cpp @@ -0,0 +1,195 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_runtime_page_store.h" +#include "common/test_chunk_physical.h" +#include "common/test_chunk_root.h" + +#include <latch> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_chunk; + +TEST(ChunkPhysicalRegistry, RepeatedObserversCountAllMirrorCapacityOnceUntilActualDelete) +{ + TestPhysicalHost host; + auto pack = host.prepare(); + auto repeated = host.prepare(); + ASSERT_EQ(pack, repeated); + ASSERT_TRUE(host.registry->observe(pack, TestPhysicalHost::receipt, host.budget).ok()); + ASSERT_TRUE(host.registry->observe(pack, TestPhysicalHost::receipt, host.budget).ok()); + auto tail = ChunkPhysicalRegistry::extent(pack); + pack.reset(); + repeated.reset(); + host.lane->drain(); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 8192U); + ASSERT_TRUE( + host.registry->deleted(TestPhysicalHost::stage, TestPhysicalHost::chunk, TestPhysicalHost::receipt, host.budget) + .ok()); + // Metadata observers are still alive, but the physical allocation is gone. + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 0U); + ASSERT_TRUE( + host.registry->deleted(TestPhysicalHost::stage, TestPhysicalHost::chunk, TestPhysicalHost::receipt, host.budget) + .ok()); + tail.reset(); + host.lane->drain(); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 0U); +} + +TEST(ChunkPhysicalRegistry, OnePageStoreReopensIntoTheOriginalCanonicalPhysicalLedger) +{ + TestPhysicalHost host; + TestRootBackend backend; + detail::ChunkRuntimePageStore store({.tree = 19, .owner_epoch = 1}, backend.catalog, backend.transport, + host.budget); + auto other = std::make_shared<TestPhysicalBudget>(); + other->queue = host.queue; + std::shared_ptr<const ChunkBackingDomain> first; + std::shared_ptr<const ChunkBackingDomain> second; + ASSERT_TRUE(store.make_domain(51, host.budget, &first).ok()); + ASSERT_TRUE(store.make_domain(52, other, &second).ok()); + ASSERT_EQ(first->physical_registry, second->physical_registry); + std::shared_ptr<ChunkPhysicalPack> one; + std::shared_ptr<ChunkPhysicalPack> two; + ASSERT_TRUE( + first->physical_registry->prepare(TestPhysicalHost::stage, TestPhysicalHost::chunk, host.budget, &one).ok()); + ASSERT_TRUE(first->physical_registry->observe(one, TestPhysicalHost::receipt, host.budget).ok()); + one.reset(); + host.lane->drain(); + ASSERT_TRUE(second->physical_registry->prepare(TestPhysicalHost::stage, TestPhysicalHost::chunk, other, &two).ok()); + ASSERT_TRUE(second->physical_registry->observe(two, TestPhysicalHost::receipt, other).ok()); + EXPECT_EQ(second->physical_registry->backing_metrics()->snapshot().allocated_backing_bytes, 8192U); + EXPECT_EQ(other->metrics->snapshot().allocated_backing_bytes, 0U); + ASSERT_TRUE(second->physical_registry + ->deleted(TestPhysicalHost::stage, TestPhysicalHost::chunk, TestPhysicalHost::receipt, other) + .ok()); + EXPECT_EQ(second->physical_registry->backing_metrics()->snapshot().allocated_backing_bytes, 0U); +} + +TEST(ChunkPhysicalRegistry, ExistingPageStoreCannotMixCanonicalRolesAcrossHostQueues) +{ + TestPhysicalHost first_host; + TestPhysicalHost other_host; + TestRootBackend backend; + detail::ChunkRuntimePageStore store({.tree = 19, .owner_epoch = 1}, backend.catalog, backend.transport, + first_host.budget); + std::shared_ptr<const ChunkBackingDomain> first; + std::shared_ptr<const ChunkBackingDomain> rejected; + ASSERT_TRUE(store.make_domain(51, first_host.budget, &first).ok()); + EXPECT_EQ(store.make_domain(52, other_host.budget, &rejected).code(), Code::kInvalidArgument); + EXPECT_FALSE(rejected); + EXPECT_TRUE(first->physical_registry->uses_queue(first_host.queue.get())); +} + +TEST(ChunkPhysicalRegistry, ChangedIncarnationCannotRemoveAnotherPhysicalAllocation) +{ + TestPhysicalHost host; + auto old = host.prepare(); + ASSERT_TRUE(host.registry->observe(old, TestPhysicalHost::receipt, host.budget).ok()); + auto wrong = TestPhysicalHost::receipt; + ++wrong.created_ms; + EXPECT_EQ(host.registry->deleted(TestPhysicalHost::stage, TestPhysicalHost::chunk, wrong, host.budget).code(), + Code::kCorruption); + auto stage = TestPhysicalHost::stage; + ++stage.owner_epoch; + auto receipt = TestPhysicalHost::receipt; + ++receipt.owner_epoch; + ++receipt.created_ms; + std::shared_ptr<ChunkPhysicalPack> fresh; + ASSERT_TRUE(host.registry->prepare(stage, TestPhysicalHost::chunk, host.budget, &fresh).ok()); + ASSERT_TRUE(host.registry->observe(fresh, receipt, host.budget).ok()); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 16384U); + ASSERT_TRUE(host.registry->deleted(TestPhysicalHost::stage, TestPhysicalHost::chunk, {}, host.budget).ok()); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 8192U); + EXPECT_EQ(host.registry->deleted(stage, TestPhysicalHost::chunk, TestPhysicalHost::receipt, host.budget).code(), + Code::kCorruption); + ASSERT_TRUE(host.registry->deleted(stage, TestPhysicalHost::chunk, receipt, host.budget).ok()); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 0U); +} + +TEST(ChunkPhysicalRegistry, SingleBackendWorkerWaitsOnlyOnTheIndependentHostRetirementLane) +{ + TestPhysicalHost host; + detail::FlushRuntime backend(1, 1, host.budget); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + + struct Call + { + TestPhysicalHost *host; + Status status; + std::latch done{1}; + } call{.host = &host, .status = {}}; + + ASSERT_TRUE(backend + .try_submit( + lane, + [](void *ptr) noexcept { + auto &call = *static_cast<Call *>(ptr); + std::shared_ptr<ChunkPhysicalPack> pack; + call.status = call.host->registry->prepare(TestPhysicalHost::stage, TestPhysicalHost::chunk, + call.host->budget, &pack); + if (call.status.ok()) + call.status = + call.host->registry->observe(pack, TestPhysicalHost::receipt, call.host->budget); + call.done.count_down(); + }, + &call) + .ok()); + call.done.wait(); + lane->drain(); + ASSERT_TRUE(call.status.ok()) << call.status.to_string(); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 8192U); + EXPECT_GT(host.queue->snapshot().cpu_steps, 0U); +} + +TEST(ChunkPhysicalRegistry, CommandAdmissionRejectsBeforeEnqueueAndClosedHostAcceptsOnlyMemoryRetirement) +{ + TestPhysicalHost host; + const auto before = host.budget->claimed.load(); + host.budget->reject = true; + std::shared_ptr<ChunkPhysicalPack> pack; + EXPECT_EQ(host.registry->prepare(TestPhysicalHost::stage, TestPhysicalHost::chunk, host.budget, &pack).code(), + Code::kResourceExhausted); + EXPECT_FALSE(pack); + EXPECT_EQ(host.budget->claimed.load(), before); + host.budget->reject = false; + pack = host.prepare(); + ASSERT_TRUE(host.registry->observe(pack, TestPhysicalHost::receipt, host.budget).ok()); + host.queue->close_and_drain(); + ASSERT_TRUE( + host.registry->deleted(TestPhysicalHost::stage, TestPhysicalHost::chunk, TestPhysicalHost::receipt, host.budget) + .ok()); + EXPECT_EQ(host.registry->backing_metrics()->snapshot().allocated_backing_bytes, 0U); +} +} // namespace + +TEST(ChunkPhysicalRegistry, EveryPreparationAllocationFailureReleasesUnallocatedCanonicalMetadata) +{ + TestPhysicalHost host; + bool reached_success = false; + for (int allowance = 0; allowance < 8; ++allowance) { + auto credit = std::make_shared<TestPhysicalBudget>(); + credit->queue = host.queue; + credit->allowances = allowance; + std::shared_ptr<ChunkPhysicalPack> pack; + { + ChunkPhysicalAdmission admission; + const auto status = + host.registry->prepare(TestPhysicalHost::stage, TestPhysicalHost::chunk, credit, &pack, &admission); + if (status.ok()) { + reached_success = true; + } + else { + EXPECT_EQ(status.code(), Code::kResourceExhausted) << status.to_string(); + } + credit->reject = true; + } + pack.reset(); + host.lane->drain(); + EXPECT_EQ(credit->claimed.load(), 0U) << allowance; + } + EXPECT_TRUE(reached_success); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/chunk_protection_callback_test.cpp b/lib/crowdb-tree/tests/unit/chunk_protection_callback_test.cpp new file mode 100644 index 000000000..c1a6b403b --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_protection_callback_test.cpp @@ -0,0 +1,352 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_page_store.h" +#include "backend/chunk/chunk_pin_release.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +namespace +{ +struct TestProtection +{ + int drops = 0; + int free_blobs = 0; + int pins = 0; + ct_status pin_error = 0; + uint32_t action = 0; + bool corrupt = false; + ct_status reader = -8; + std::array<uint8_t, 3> root{1, 2, 3}; + + static ct_status load_root(void *p, int32_t /*kind*/, uint64_t /*tree*/, uint64_t /*object*/, const uint8_t **out, + size_t *length) + { + auto *self = static_cast<TestProtection *>(p); + *out = self->root.data(); + *length = self->root.size(); + return 0; + } + + static ct_status load(void *p, uint64_t /*tree*/, ct_chunk_stage_id id, ct_chunk_candidate_status *out) + { + auto *self = static_cast<TestProtection *>(p); + *out = {.id = id, + .generation = 1, + .reference_count = 1, + .prepared_references = 1, + .prepared_windows = 1, + .released_windows = 0, + .state = self->corrupt ? 99U : 2U}; + return 0; + } + + static ct_status list(void *p, uint64_t tree, const ct_chunk_stage_id * /*after*/, uint32_t /*limit*/, + ct_chunk_candidate_status *out, size_t *count) + { + *count = 1; + return load(p, tree, {.owner_epoch = 1, .operation_high = 2, .operation_low = 3}, out); + } + + ct_root_catalog_callbacks base() + { + return {.load = &load_root, + .free_blob = +[](void *p, const uint8_t *, size_t) { ++static_cast<TestProtection *>(p)->free_blobs; }, + .store = +[](void *, int32_t, uint64_t, uint64_t, const uint8_t *, size_t) -> ct_status { return 0; }, + .publish = +[](void *, uint64_t, uint64_t, uint64_t, uint64_t, const uint8_t *, size_t) -> ct_status { + return 0; + }, + .allocate_reference_segment_id = +[](void *, uint64_t, uint64_t *out) -> ct_status { + *out = 1; + return 0; + }, + .discard_reference_segments = nullptr, + .reclaim_before = nullptr, + .pin_generation = nullptr, + .unpin_generation = nullptr, + .drop_context = +[](void *p) { ++static_cast<TestProtection *>(p)->drops; }}; + } + + static ct_root_catalog_staging_callbacks stages() + { + return {.version = 1, + .struct_bytes = sizeof(ct_root_catalog_staging_callbacks), + .capabilities = CT_CHUNK_STAGE_SCHEMA_V1, + .load = + +[](void *, uint64_t, ct_chunk_stage_id, const uint8_t **, size_t *) -> ct_status { return -1; }, + .compare = +[](void *, uint64_t, uint64_t, const uint8_t *, size_t, const uint8_t *, + size_t) -> ct_status { return 0; }, + .list = +[](void *, uint64_t, const ct_chunk_stage_id *, uint32_t, const uint8_t **, + size_t *) -> ct_status { return -5; }}; + } + + static ct_root_catalog_protection_callbacks protection() + { + return {.version = 1, + .struct_bytes = sizeof(ct_root_catalog_protection_callbacks), + .capabilities = CT_CHUNK_PROTECTION_SCHEMA_V1, + .check_reader = +[](void *p) -> ct_status { return static_cast<TestProtection *>(p)->reader; }, + .writer_quiesced = +[](void *, uint64_t, uint64_t) -> ct_status { return 0; }, + .begin = +[](void *, uint64_t, ct_chunk_stage_id, uint64_t, uint64_t, uint32_t, const uint8_t *, + size_t) -> ct_status { return 0; }, + .load = &load, + .list = &list, + .prepare = +[](void *, uint64_t, ct_chunk_stage_id, uint64_t, const ct_chunk_stage_id *, + size_t) -> ct_status { return 0; }, + .finish = +[](void *p, uint64_t, ct_chunk_stage_id, uint32_t action) -> ct_status { + static_cast<TestProtection *>(p)->action = action; + return 0; + }, + .release = +[](void *, uint64_t, ct_chunk_stage_id, uint64_t) -> ct_status { return 0; }, + .pin = +[](void *p, uint64_t, ct_chunk_stage_id, ct_chunk_stage_id, uint8_t acquire) -> ct_status { + if (static_cast<TestProtection *>(p)->pin_error != 0) { + return static_cast<TestProtection *>(p)->pin_error; + } + static_cast<TestProtection *>(p)->pins += acquire ? 1 : -1; + return 0; + }, + .prune = +[](void *, uint64_t, ct_chunk_stage_id, const ct_chunk_stage_id *, uint32_t, + ct_chunk_pin_progress *out) -> ct_status { + *out = { + .after = {.owner_epoch = 1, .operation_high = 2, .operation_low = 3}, + .examined = 1, + .released = 1, + .has_after = 1, + .more = 0 + }; + return 0; + }}; + } +}; + +constexpr ChunkStageId stage{.owner_epoch = 1, .operation_high = 2, .operation_low = 3}; +} // namespace + +TEST(ChunkProtectionCallback, OpaqueRootAndFencedReaderKeepOneContextOwner) +{ + TestProtection context; + { + CallbackRootCatalog catalog(context.base(), TestProtection::stages(), TestProtection::protection(), &context); + ASSERT_NE(catalog.protection(), nullptr); + EXPECT_EQ(catalog.check_read_owner().code(), Code::kUnavailable); + context.reader = 0; + EXPECT_TRUE(catalog.check_read_owner().ok()); + std::vector<uint8_t> bytes; + ASSERT_TRUE(catalog.load_root_bytes(42, 0, &bytes).ok()); + EXPECT_EQ(bytes, (std::vector<uint8_t>{1, 2, 3})); + EXPECT_EQ(context.free_blobs, 1); + EXPECT_TRUE(catalog.change_pin(42, stage, stage, true).ok()); + EXPECT_EQ(context.pins, 1); + EXPECT_TRUE(catalog.writer_quiesced(42, 1).ok()); + EXPECT_EQ(context.pins, 1); + EXPECT_TRUE(catalog.change_pin(42, stage, stage, false).ok()); + EXPECT_EQ(context.pins, 0); + } + EXPECT_EQ(context.drops, 1); +} + +TEST(ChunkProtectionCallback, CandidateValidationRejectsCorruptOrUnboundedReplies) +{ + TestProtection context; + CallbackRootCatalog catalog(context.base(), TestProtection::stages(), TestProtection::protection(), &context); + std::optional<ChunkCandidateStatus> candidate; + ASSERT_TRUE(catalog.load_candidate(42, stage, &candidate).ok()); + ASSERT_TRUE(candidate); + if (candidate.has_value()) { + EXPECT_EQ(candidate->state, ChunkCandidateState::Ready); + } + context.corrupt = true; + EXPECT_EQ(catalog.load_candidate(42, stage, &candidate).code(), Code::kCorruption); + context.corrupt = false; + std::vector<ChunkCandidateStatus> list; + EXPECT_EQ(catalog.list_candidates(42, {}, 65, &list).code(), Code::kInvalidArgument); + EXPECT_EQ(catalog.list_candidates(42, stage, 64, &list).code(), Code::kCorruption); + EXPECT_TRUE(catalog.finish_candidate(42, stage, CT_CHUNK_CANDIDATE_RECOVER).ok()); + EXPECT_EQ(context.action, CT_CHUNK_CANDIDATE_RECOVER); + EXPECT_EQ(catalog.finish_candidate(42, stage, 99).code(), Code::kInvalidArgument); +} + +TEST(ChunkProtectionCallback, ReferenceWindowsAndPinCursorRemainFiniteAndOrdered) +{ + TestProtection context; + CallbackRootCatalog catalog(context.base(), TestProtection::stages(), TestProtection::protection(), &context); + std::array<ChunkStageId, 2> duplicate{stage, stage}; + EXPECT_EQ(catalog.prepare_references(42, stage, 0, duplicate).code(), Code::kInvalidArgument); + EXPECT_TRUE(catalog.prepare_references(42, stage, 0, {duplicate.data(), 1}).ok()); + ChunkPinProgress progress; + EXPECT_TRUE(catalog.prune_pins(42, stage, {}, 64, &progress).ok()); + EXPECT_EQ(progress.released, 1); + EXPECT_EQ(progress.after, stage); + EXPECT_EQ(catalog.prune_pins(42, stage, stage, 64, &progress).code(), Code::kCorruption); + const std::array<ChunkStageId, 2> ids{ + stage, ChunkStageId{.owner_epoch = 1, .operation_high = 2, .operation_low = 4} + }; + EXPECT_EQ(chunk_reference_checksum(0, ids), + chunk_reference_checksum(chunk_reference_checksum(0, {ids.data(), 1}), {ids.data() + 1, 1})); +} + +namespace +{ +class TestPinBudget final : public AllocationBudget +{ + public: + bool claim(size_t bytes) noexcept override + { + if (deny) { + return false; + } + used += bytes; + return true; + } + + void release(size_t bytes) noexcept override + { + used -= bytes; + } + + size_t used = 0; + bool deny = false; +}; + +struct TestPinQueue +{ + std::shared_ptr<TestPinBudget> budget = std::make_shared<TestPinBudget>(); + std::shared_ptr<size_t> wakes = std::make_shared<size_t>(0); + std::shared_ptr<ChunkPinReleaseQueue> queue; + + TestPinQueue() + { + EXPECT_TRUE(ChunkPinReleaseQueue::create( + budget, 1, wakes, +[](void *context) noexcept { ++*static_cast<size_t *>(context); }, + wakes.get(), &queue) + .ok()); + } +}; +} // namespace + +TEST(ChunkPinRelease, LastOwnerOnlyEnqueuesAndUnknownUnpinKeepsPaidIdentity) +{ + TestPinQueue service; + TestProtection context; + auto catalog = std::make_shared<CallbackRootCatalog>(context.base(), TestProtection::stages(), + TestProtection::protection(), &context); + const auto baseline = service.budget->used; + std::shared_ptr<ChunkPinOwner> owner; + ASSERT_TRUE(service.queue->prepare(service.budget, catalog, 42, stage, stage, &owner).ok()); + owner->arm(); + context.pins = 1; + service.budget->deny = true; + owner.reset(); + EXPECT_EQ(context.pins, 1); + EXPECT_GT(service.budget->used, baseline); + EXPECT_EQ(service.queue->snapshot().pending, 1); + context.pin_error = -8; + auto first = service.queue->pump(8); + EXPECT_EQ(first.failed, 1); + EXPECT_EQ(first.completed, 0); + EXPECT_EQ(service.queue->snapshot().registered, 1); + context.pin_error = 0; + auto second = service.queue->pump(8); + EXPECT_EQ(second.completed, 1); + EXPECT_EQ(context.pins, 0); + EXPECT_EQ(service.budget->used, baseline); + service.queue->stop_admission(); + EXPECT_TRUE(service.queue->drained()); +} + +TEST(ChunkPinRelease, HostStopPreservesLateReaderAndCatalogUntilRealRelease) +{ + TestPinQueue service; + TestProtection context; + auto catalog = std::make_shared<CallbackRootCatalog>(context.base(), TestProtection::stages(), + TestProtection::protection(), &context); + std::shared_ptr<ChunkPinOwner> retained; + ASSERT_TRUE(service.queue->prepare(service.budget, catalog, 42, stage, stage, &retained).ok()); + retained->arm(); + context.pins = 1; + ASSERT_TRUE(catalog->writer_quiesced(42, 1).ok()); + catalog.reset(); + service.queue->stop_admission(); + EXPECT_FALSE(service.queue->drained()); + EXPECT_EQ(context.drops, 0); + EXPECT_EQ(service.queue->pump(8).completed, 0); + retained.reset(); + EXPECT_EQ(context.drops, 0); + EXPECT_FALSE(service.queue->drained()); + EXPECT_EQ(service.queue->pump(8).completed, 1); + EXPECT_TRUE(service.queue->drained()); + EXPECT_EQ(context.drops, 1); +} + +TEST(ChunkPinRelease, FixedCapacityAndUnarmedFailureReleaseWithoutMetadata) +{ + TestPinQueue service; + TestProtection context; + auto catalog = std::make_shared<CallbackRootCatalog>(context.base(), TestProtection::stages(), + TestProtection::protection(), &context); + const auto baseline = service.budget->used; + const auto empty = service.queue->snapshot(); + EXPECT_EQ(empty.capacity, 1U); + EXPECT_EQ(empty.registered, 0U); + EXPECT_EQ(empty.pending, 0U); + EXPECT_EQ(empty.denied, 0U); + std::shared_ptr<ChunkPinOwner> first; + std::shared_ptr<ChunkPinOwner> second; + ASSERT_TRUE(service.queue->prepare(service.budget, catalog, 42, stage, stage, &first).ok()); + const auto paid = service.budget->used; + for (size_t sample = 0; sample < 10; ++sample) { + const auto held = service.queue->snapshot(); + EXPECT_EQ(held.registered, 1U); + EXPECT_EQ(held.pending, 0U); + EXPECT_EQ(held.denied, 0U); + } + EXPECT_EQ(service.budget->used, paid); + EXPECT_EQ(service.queue->prepare(service.budget, catalog, 42, stage, stage, &second).code(), + Code::kResourceExhausted); + EXPECT_EQ(service.queue->snapshot().denied, 1U); + service.queue->stop_admission(); + EXPECT_EQ(service.queue->prepare(service.budget, catalog, 42, stage, stage, &second).code(), Code::kUnavailable); + first.reset(); + EXPECT_EQ(service.budget->used, baseline); + EXPECT_EQ(context.pins, 0); + EXPECT_TRUE(service.queue->drained()); + EXPECT_EQ(service.queue->snapshot().denied, 1U); + EXPECT_EQ(service.queue->pump(8).completed, 0); +} + +TEST(ChunkPinRelease, FailedIdentityCannotStarveAnUntouchedOlderRelease) +{ + auto budget = std::make_shared<TestPinBudget>(); + auto wakes = std::make_shared<size_t>(0); + std::shared_ptr<ChunkPinReleaseQueue> queue; + ASSERT_TRUE( + ChunkPinReleaseQueue::create( + budget, 2, wakes, +[](void *context) noexcept { ++*static_cast<size_t *>(context); }, wakes.get(), &queue) + .ok()); + TestProtection healthy; + TestProtection unavailable; + healthy.pins = unavailable.pins = 1; + unavailable.pin_error = -8; + auto good_catalog = std::make_shared<CallbackRootCatalog>(healthy.base(), TestProtection::stages(), + TestProtection::protection(), &healthy); + auto bad_catalog = std::make_shared<CallbackRootCatalog>(unavailable.base(), TestProtection::stages(), + TestProtection::protection(), &unavailable); + std::shared_ptr<ChunkPinOwner> good; + std::shared_ptr<ChunkPinOwner> bad; + ASSERT_TRUE(queue->prepare(budget, good_catalog, 42, stage, stage, &good).ok()); + ASSERT_TRUE(queue->prepare(budget, bad_catalog, 42, stage, stage, &bad).ok()); + good->arm(); + bad->arm(); + good.reset(); + bad.reset(); + EXPECT_EQ(queue->pump(1).failed, 1); + EXPECT_EQ(queue->pump(1).completed, 1); + EXPECT_EQ(healthy.pins, 0); + EXPECT_EQ(unavailable.pins, 1); + unavailable.pin_error = 0; + EXPECT_EQ(queue->pump(1).completed, 1); + queue->stop_admission(); + EXPECT_TRUE(queue->drained()); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/chunk_registry_scope_test.cpp b/lib/crowdb-tree/tests/unit/chunk_registry_scope_test.cpp new file mode 100644 index 000000000..6e9c1736a --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_registry_scope_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "common/test_chunk_physical.h" +#include "common/test_chunk_root.h" + +#include <thread> + +namespace crowdb::tree +{ +TEST(ChunkRegistryScope, HostAndLogicalTreeScopeRemainDistinctAcrossClosedHostQueues) +{ + test_chunk::TestRootBackend backend; + uint64_t first_scope = 0; + { + test_chunk::TestPhysicalHost first; + std::shared_ptr<ChunkPhysicalRegistry> one; + std::shared_ptr<ChunkPhysicalRegistry> same; + std::shared_ptr<ChunkPhysicalRegistry> other_tree; + ASSERT_TRUE(backend.transport->physical_registry(19, first.budget, &one).ok()); + ASSERT_TRUE(backend.transport->physical_registry(19, first.budget, &same).ok()); + ASSERT_TRUE(backend.transport->physical_registry(20, first.budget, &other_tree).ok()); + EXPECT_EQ(one, same); + EXPECT_NE(one->backing_stats().scope_id, other_tree->backing_stats().scope_id); + first_scope = one->backing_stats().scope_id; + } + test_chunk::TestPhysicalHost second; + std::shared_ptr<ChunkPhysicalRegistry> next_host; + ASSERT_TRUE(backend.transport->physical_registry(19, second.budget, &next_host).ok()); + EXPECT_NE(next_host->backing_stats().scope_id, first_scope); + EXPECT_EQ(next_host->backing_stats().allocated_bytes, 0U); +} + +TEST(ChunkRegistryScope, ConcurrentFacadesShareOneCanonicalLedger) +{ + test_chunk::TestPhysicalHost host; + test_chunk::TestRootBackend backend; + std::shared_ptr<ChunkPhysicalRegistry> first; + std::shared_ptr<ChunkPhysicalRegistry> second; + Status one; + Status two; + std::jthread a([&] { one = backend.transport->physical_registry(19, host.budget, &first); }); + std::jthread b([&] { two = backend.transport->physical_registry(19, host.budget, &second); }); + a.join(); + b.join(); + ASSERT_TRUE(one.ok()); + ASSERT_TRUE(two.ok()); + EXPECT_EQ(first, second); +} + +TEST(ChunkRegistryScope, EveryStartupAllocationFailureReturnsItsPaidMetadataWithoutExternalAllocation) +{ + test_chunk::TestPhysicalHost host; + bool accepted = false; + for (int allowance = 0; allowance < 16; ++allowance) { + test_chunk::TestRootBackend backend; + auto credit = std::make_shared<test_chunk::TestPhysicalBudget>(); + credit->queue = host.queue; + credit->allowances = allowance; + std::shared_ptr<ChunkPhysicalRegistry> registry; + const auto status = backend.transport->physical_registry(19, credit, ®istry); + if (status.ok()) { + accepted = true; + ASSERT_TRUE(registry); + } + else { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(registry); + } + EXPECT_TRUE(backend.transport->receipts.empty()); + credit->reject = true; + registry.reset(); + backend.domain->transport.reset(); + backend.transport.reset(); + host.cpu.drain(); + EXPECT_EQ(credit->claimed.load(), 0U) << allowance; + } + EXPECT_TRUE(accepted); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/chunk_root_codec_test.cpp b/lib/crowdb-tree/tests/unit/chunk_root_codec_test.cpp new file mode 100644 index 000000000..0c1434d9e --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_root_codec_test.cpp @@ -0,0 +1,171 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_runtime_ids.h" +#include "backend/chunk/chunk_runtime_page_store.h" +#include "common/test_chunk_backing.h" +#include "snapshot/chunk_root_codec_internal.h" + +namespace crowdb::tree +{ +namespace +{ +ChunkRootIdentity test_identity() +{ + return {.tree = 19, + .root_generation = 7, + .owner_epoch = 1, + .snapshot_sequence = 4, + .page_generation = 3, + .coverage = 15, + .gc_floor = 8, + .root_page_id = 0, + .next_page_id = 10, + .wal_replay_offset = 4096}; +} + +detail::ChunkFixedPackRef test_index_pack(uint32_t bytes) +{ + return { + .stage = {.owner_epoch = 1, .operation_high = 2, .operation_low = 3}, + .chunk = {uint64_t{3} << 56U, 9}, + .offset = 0, + .length = bytes, + .checksum = 17, + .purpose = PagePurpose::kPageIndex + }; +} +} // namespace + +TEST(ChunkRootCodec, RoundTripKeepsRootSequenceAndWalDistinctFromPageGeneration) +{ + test_read::LoadCredits credits; + ChunkRootManifest root(credits.credits.bytes); + root.identity = test_identity(); + root.frame_count = 2; + root.indexes.push_back({.first_page = 0, .last_page = 9, .frame_count = 2, .pack = test_index_pack(512)}); + ChunkMetadataBytes bytes{detail::ReservationAllocator<uint8_t>(credits.credits.bytes)}; + ASSERT_TRUE(encode_chunk_root(root, &bytes).ok()); + ChunkRootManifest decoded(credits.credits.bytes); + ASSERT_TRUE(decode_chunk_root(bytes, 19, 7, &decoded).ok()); + EXPECT_EQ(decoded.identity.root_generation, 7U); + EXPECT_EQ(decoded.identity.snapshot_sequence, 4U); + EXPECT_EQ(decoded.identity.page_generation, 3U); + EXPECT_EQ(decoded.identity.coverage, 15U); + EXPECT_EQ(decoded.identity.wal_replay_offset, 4096U); + EXPECT_EQ(decoded.indexes.size(), 1U); + ChunkRootManifest wrong_root(credits.credits.bytes); + EXPECT_EQ(decode_chunk_root(bytes, 19, 8, &wrong_root).code(), Code::kCorruption); + chunk_codec::put(bytes, 13, UINT64_MAX); + chunk_codec::seal(bytes); + EXPECT_EQ(decode_chunk_root(bytes, 19, 7, &wrong_root).code(), Code::kCorruption); +} + +TEST(ChunkRootCodec, IndexValidatesExactNamespaceAndRejectsOversizedPackFields) +{ + test_chunk::TestChunkBackend test; + std::shared_ptr<ChunkPackBacking> pack; + ASSERT_TRUE(test.prepare(8192, &pack).ok()); + auto backing = detail::reserved_shared<BackingRef>(test.credits.credits.bytes); + backing->backend_id = 17; + backing->length = 8192; + backing->owner = pack; + backing->chunk = pack; + backing->storage = BackingStorage::kChunkPack; + auto refs = detail::reserved_shared<StoredOverflowReferences>( + test.credits.credits.bytes, detail::ReservationAllocator<StoredOverflowReference>(test.credits.credits.bytes)); + refs->push_back({.head = 4, .bytes = 123}); + DurableCapturedFrame frame{.page_id = 0, + .type = page_type::kLeafBase, + .frame_bytes = 4096, + .logical_bytes = 4132, + .next_overflow = kInvalidPageId, + .overflow_bytes = 0, + .backing = backing, + .stored_overflow = refs, + .summary = {}}; + const DurableCapturedFrame *frames[]{&frame}; + ChunkMetadataBytes bytes{detail::ReservationAllocator<uint8_t>(test.credits.credits.bytes)}; + ASSERT_TRUE(encode_chunk_index(test_identity(), 2, frames, &bytes).ok()); + uint64_t count = 0; + size_t offset = 0; + EXPECT_EQ(decode_chunk_index_header(bytes, test_identity(), 3, &count, &offset).code(), Code::kCorruption); + ASSERT_TRUE(decode_chunk_index_header(bytes, test_identity(), 2, &count, &offset).ok()); + ASSERT_EQ(count, 1U); + ChunkIndexFrame parsed; + ASSERT_TRUE(decode_chunk_index_frame(bytes, &offset, &parsed).ok()); + EXPECT_EQ(parsed.page_id, 0U); + EXPECT_EQ(parsed.stored_reference_words.size(), 16U); + EXPECT_EQ(offset + 4, bytes.size()); + chunk_codec::put(std::span(bytes).subspan(kChunkIndexHeaderBytes), 12, uint64_t{1} << 32U); + chunk_codec::seal(bytes); + ASSERT_TRUE(decode_chunk_index_header(bytes, test_identity(), 2, &count, &offset).ok()); + EXPECT_EQ(decode_chunk_index_frame(bytes, &offset, &parsed).code(), Code::kCorruption); +} + +TEST(ChunkRuntimeIds, PurposeAndCounterShareOneNamespaceWithoutReusingOperations) +{ + detail::ChunkRuntimeIds ids(7, 23, uint64_t{45} << 32U); + detail::ChunkStageId first; + detail::ChunkStageId second; + detail::ChunkId a; + detail::ChunkId b; + ASSERT_TRUE(ids.next(PagePurpose::kBtreePage, &first, &a).ok()); + ASSERT_TRUE(ids.next(PagePurpose::kPageIndex, &second, &b).ok()); + EXPECT_NE(first, second); + EXPECT_EQ(first.owner_epoch, 7U); + EXPECT_EQ(detail::chunk_purpose(a), PagePurpose::kBtreePage); + EXPECT_EQ(detail::chunk_purpose(b), PagePurpose::kPageIndex); + EXPECT_NE(a.low, b.low); +} + +TEST(ChunkRuntimePageStore, UnsupportedCatalogDoesNotExposeAnIndependentAddressSpace) +{ + test_read::LoadCredits credits; + auto catalog = std::make_shared<detail::MemoryRootCatalog>(1); + auto transport = std::make_shared<detail::MemoryChunkTransport>(); + detail::ChunkRuntimePageStore store({.tree = 19, .owner_epoch = 1}, catalog, transport, credits.credits.bytes); + EXPECT_FALSE(store.supported()); + std::shared_ptr<const ChunkBackingDomain> domain; + EXPECT_EQ(store.make_domain(17, credits.credits.bytes, &domain).code(), Code::kNotSupported); + EXPECT_FALSE(domain); + EXPECT_EQ(store.sync().code(), Code::kNotSupported); + EXPECT_EQ(store.write_at(0, nullptr, 0).code(), Code::kNotSupported); + EXPECT_EQ(store.read_at(0, nullptr, 0).code(), Code::kNotSupported); +} + +TEST(ChunkStageCodec, OwnedEncodingAndDecodingRetainTheCallerAllocator) +{ + test_read::LoadCredits credits; + using Record = detail::ChunkStageRecord; + Record record{ + .tree_id = 19, + .id = {.owner_epoch = 1, .operation_high = 2, .operation_low = 3}, + .revision = 3, + .chunk_high = uint64_t{2} + << 56U, + .chunk_low = 1, + .capacity = 128, + .manifest_generation = 0, + .purpose = 2, + .state = detail::ChunkStageState::Reloadable, + .packs = decltype(Record::packs)(MetadataAllocator<detail::ChunkStagePack>(credits.credits.bytes)) + }; + record.packs.push_back({.offset = 0, .length = 32, .checksum = 9}); + auto decoded = record; + decoded.packs.clear(); + decoded.packs.shrink_to_fit(); + const auto before = credits.credits.bytes->available(); + { + detail::ChunkStageBytes bytes{MetadataAllocator<uint8_t>(credits.credits.bytes)}; + ASSERT_TRUE(detail::encode_stage_record_owned(record, &bytes).ok()); + ASSERT_TRUE(detail::decode_stage_record(bytes, &decoded).ok()); + EXPECT_EQ(decoded.packs.get_allocator().owner(), credits.credits.bytes); + EXPECT_EQ(decoded, record); + EXPECT_EQ(before - credits.credits.bytes->available(), + bytes.capacity() + (decoded.packs.capacity() * sizeof(detail::ChunkStagePack))); + } + decoded.packs.clear(); + decoded.packs.shrink_to_fit(); + EXPECT_EQ(credits.credits.bytes->available(), before); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/chunk_stage_gc_test.cpp b/lib/crowdb-tree/tests/unit/chunk_stage_gc_test.cpp new file mode 100644 index 000000000..284be6619 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_stage_gc_test.cpp @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_stage_gc.h" +#include "common/test_chunk_stage_backend.h" + +namespace crowdb::tree::detail +{ +namespace +{ +ChunkStageRecord retired_stage() +{ + return { + .tree_id = 17, + .id = {.owner_epoch = 1, .operation_high = 10, .operation_low = 20}, + .revision = 2, + .chunk_high = uint64_t{2} + << 56, + .chunk_low = 1, + .capacity = 4096, + .manifest_generation = 0, + .purpose = 2, + .state = ChunkStageState::Retired, + .packs = {} + }; +} +} // namespace + +TEST(ChunkStageGc, UnknownDeleteKeepsDurableProtectionUntilRetryConfirmsPhysicalDeletion) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + catalog->record = retired_stage(); + TestFixedTransport transport(catalog); + transport.unknown_delete = true; + const auto id = retired_stage().id; + auto result = reclaim_retired_stage(*catalog, transport, 17, 1, id); + EXPECT_FALSE(result.status.ok()); + EXPECT_FALSE(result.stage_removed); + EXPECT_FALSE(result.deletion_acknowledged); + EXPECT_TRUE(transport.physically_deleted); + if (!catalog->record.has_value()) { + FAIL() << "unknown delete dropped stage"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Deleting); + transport.unknown_delete = false; + result = reclaim_retired_stage(*catalog, transport, 17, 1, id); + EXPECT_TRUE(result.status.ok()); + EXPECT_TRUE(result.deletion_acknowledged); + EXPECT_TRUE(result.stage_removed); + EXPECT_FALSE(catalog->record.has_value()); + EXPECT_EQ(transport.deletions, 2U); +} + +TEST(ChunkStageGc, UnconfirmedDiskDrainRetainsDeletingStageWithoutPhysicalDelete) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + catalog->record = retired_stage(); + TestFixedTransport transport(catalog); + transport.fail_drain = true; + const auto id = retired_stage().id; + auto result = reclaim_retired_stage(*catalog, transport, 17, 1, id); + EXPECT_FALSE(result.status.ok()); + EXPECT_FALSE(result.stage_removed); + EXPECT_FALSE(result.deletion_acknowledged); + EXPECT_EQ(transport.deletions, 0U); + if (!catalog->record.has_value()) { + FAIL() << "unconfirmed disk drain dropped stage"; + return; + } + EXPECT_EQ(catalog->record->state, ChunkStageState::Deleting); + transport.fail_drain = false; + result = reclaim_retired_stage(*catalog, transport, 17, 1, id); + EXPECT_TRUE(result.status.ok()); + EXPECT_TRUE(result.stage_removed); + EXPECT_EQ(transport.drains, 2U); + EXPECT_EQ(transport.deletions, 1U); +} + +TEST(ChunkStageGc, LiveStageAndFutureEpochCannotAuthorizeDelete) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + auto record = retired_stage(); + record.state = ChunkStageState::Reserved; + record.revision = 1; + catalog->record = record; + TestFixedTransport transport(catalog); + EXPECT_FALSE(reclaim_retired_stage(*catalog, transport, 17, 1, record.id).status.ok()); + catalog->record = retired_stage(); + EXPECT_FALSE(reclaim_retired_stage(*catalog, transport, 17, 0, record.id).status.ok()); + EXPECT_EQ(transport.deletions, 0U); + EXPECT_TRUE(catalog->record.has_value()); +} + +TEST(ChunkStageGc, ChangedPhysicalEpochCannotDeleteUsingTheOldStage) +{ + auto catalog = std::make_shared<TestStageCatalog>(); + catalog->record = retired_stage(); + TestFixedTransport transport(catalog); + transport.physical_epoch = 2; + const auto result = reclaim_retired_stage(*catalog, transport, 17, 2, retired_stage().id); + EXPECT_EQ(result.status.code(), Code::kUnavailable); + EXPECT_FALSE(result.deletion_acknowledged); + EXPECT_FALSE(result.stage_removed); + EXPECT_EQ(transport.drains, 0U); + EXPECT_EQ(transport.deletions, 0U); + ASSERT_TRUE(catalog->record); + EXPECT_EQ(catalog->record->state, ChunkStageState::Deleting); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/chunk_stage_test.cpp b/lib/crowdb-tree/tests/unit/chunk_stage_test.cpp new file mode 100644 index 000000000..889ddcdfa --- /dev/null +++ b/lib/crowdb-tree/tests/unit/chunk_stage_test.cpp @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/chunk/chunk_page_store.h" +#include "backend/chunk/chunk_stage.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +namespace +{ +ChunkStageRecord reserved() +{ + return { + .tree_id = 42, + .id = {.owner_epoch = 7, .operation_high = 8, .operation_low = 9}, + .revision = 1, + .chunk_high = uint64_t{2} + << 56, + .chunk_low = 123, + .capacity = 4096, + .manifest_generation = 0, + .purpose = 2, + .state = ChunkStageState::Reserved, + .packs = {} + }; +} + +TEST(ChunkStage, CodecChecksEveryBoundBeforeAllocatingPackRecords) +{ + auto record = reserved(); + record.revision = 2; + record.state = ChunkStageState::Writing; + record.packs.push_back({.offset = 0, .length = 4096, .checksum = 0x12345678}); + std::vector<uint8_t> encoded; + ASSERT_TRUE(encode_stage_record(record, &encoded).ok()); + ASSERT_EQ(encoded.size(), 116U); + EXPECT_EQ(std::string(encoded.begin(), encoded.begin() + 4), "CTSG"); + ChunkStageRecord decoded; + ASSERT_TRUE(decode_stage_record(encoded, &decoded).ok()); + EXPECT_EQ(decoded, record); + for (size_t offset : {8, 40, 64, 84, 88, 96, 112}) { + auto corrupt = encoded; + corrupt[offset] ^= 1; + EXPECT_EQ(decode_stage_record(corrupt, &decoded).code(), Code::kCorruption); + } + encoded[4] = 2; + EXPECT_EQ(decode_stage_record(encoded, &decoded).code(), Code::kNotSupported); +} + +TEST(ChunkStage, ProvisionalIdentityAndReferencesNeverChangeDuringTransition) +{ + const auto first = reserved(); + auto writing = first; + writing.revision = 2; + writing.state = ChunkStageState::Writing; + EXPECT_TRUE(validate_stage_transition({}, first).ok()); + EXPECT_TRUE(validate_stage_transition(first, writing).ok()); + EXPECT_FALSE(validate_stage_transition(writing, {}).ok()); + auto changed = writing; + ++changed.chunk_low; + EXPECT_FALSE(validate_stage_transition(first, changed).ok()); + auto ready = writing; + ready.revision = 3; + ready.state = ChunkStageState::Reloadable; + EXPECT_FALSE(validate_stage_transition(writing, ready).ok()); + ready.packs.push_back({.offset = 0, .length = 4096, .checksum = 0}); + EXPECT_TRUE(validate_stage_transition(writing, ready).ok()); + ready.packs.push_back({.offset = 1, .length = 1, .checksum = 0}); + EXPECT_FALSE(validate_stage_record(ready).ok()); +} + +TEST(ChunkStage, OldCatalogKeepsIndependentStagingUnavailable) +{ + MemoryRootCatalog catalog(7); + EXPECT_FALSE(catalog.stage_schema_supported()); + const auto first = reserved(); + std::optional<ChunkStageRecord> loaded; + EXPECT_EQ(catalog.load_stage(42, first.id, &loaded).code(), Code::kNotSupported); + EXPECT_EQ(catalog.compare_stage(42, 7, {}, first).code(), Code::kNotSupported); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/consolidation_test.cpp b/lib/crowdb-tree/tests/unit/consolidation_test.cpp index fdc001260..5219a5cf1 100644 --- a/lib/crowdb-tree/tests/unit/consolidation_test.cpp +++ b/lib/crowdb-tree/tests/unit/consolidation_test.cpp @@ -4,7 +4,7 @@ // CT10: consolidation tests (fold by highest slot, triggers, tombstone keep, // old-chain retirement via the epoch manager). #include "crowdb-tree/crowdb-tree.h" -#include "crowdb-tree/maptable/page.h" +#include "runtime_tree_access.h" #include <gtest/gtest.h> @@ -19,33 +19,25 @@ Batch put_one(const std::string &k, const std::string &v) return Batch{{{.key = k, .kind = OpKind::kPut, .value = v}}}; } -page_type head_type(Crowdbtree &t) +uint64_t overlay_runs(Crowdbtree &tree) { - return t.mapping().get_resident(t.root_page_id())->type; + return RuntimeTreeAccess_for_tests::root(tree)->metrics.overlay_runs; } } // namespace -TEST(Consolidation, FoldsChainAtDeltaLenThreshold) +TEST(Consolidation, FiniteFlushCoalescesOverlayRunsWithoutLosingEarlierKeys) { - Config opt; - opt.max_delta_len = 4; - Crowdbtree t(opt); - // Each flush adds one delta to the single root leaf. - for (uint64_t s = 1; s <= 4; ++s) { - ASSERT_TRUE(t.apply(s, put_one("k" + std::to_string(s), "v")).ok()); - ASSERT_TRUE(t.flush().ok()); - } - // 4 deltas: not yet over threshold (> 4). - EXPECT_EQ(head_type(t), page_type::kBatchDelta); - // 5th delta trips consolidation -> head becomes a fresh LeafBase. - ASSERT_TRUE(t.apply(5, put_one("k5", "v")).ok()); - ASSERT_TRUE(t.flush().ok()); - EXPECT_EQ(head_type(t), page_type::kLeafBase); - // All keys survive the fold. - for (int i = 1; i <= 5; ++i) { - std::string v; - uint64_t slot; - EXPECT_TRUE(t.get(Slice("k" + std::to_string(i)), &slot, &v)); + Crowdbtree tree; + for (uint64_t slot = 1; slot <= 5; ++slot) { + ASSERT_TRUE(tree.apply(slot, put_one("k" + std::to_string(slot), "v")).ok()); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_LE(overlay_runs(tree), 1U); + EXPECT_EQ(tree.last_applied_slot(), slot); + for (uint64_t prior = 1; prior <= slot; ++prior) { + auto found = tree.get_view("k" + std::to_string(prior)); + ASSERT_TRUE(found.found()); + EXPECT_EQ(found.slot(), prior); + } } } @@ -86,48 +78,32 @@ TEST(Consolidation, TombstonePreservedThroughFold) EXPECT_FALSE(t.get(Slice("a"), &slot, &v)); } -TEST(Consolidation, OldChainRetiredViaEpoch) +TEST(Consolidation, OldPublishedPageRemainsAliveUntilCapturedViewIsReleased) { - Config opt; - opt.max_delta_len = 3; - Crowdbtree t(opt); - for (uint64_t s = 1; s <= 3; ++s) { - ASSERT_TRUE(t.apply(s, put_one("k", "v" + std::to_string(s))).ok()); - ASSERT_TRUE(t.flush().ok()); - } - // Hold a read guard so retired pages cannot be freed during consolidation. - { - EpochManager::Guard g = t.epoch().enter(); - ASSERT_TRUE(t.apply(4, put_one("k", "v4")).ok()); - ASSERT_TRUE(t.flush().ok()); // consolidation retires the old chain - EXPECT_GT(t.epoch().pending_retired(), 0U); - } - // Guard dropped -> retired pages become reclaimable. - t.epoch().try_reclaim(); - EXPECT_EQ(t.epoch().pending_retired(), 0U); + Crowdbtree tree; + ASSERT_TRUE(tree.put("k", "old").ok()); + ASSERT_TRUE(tree.flush().ok()); + auto old = RuntimeTreeAccess_for_tests::capture(tree); + std::weak_ptr<const PublishedPage> retained = old->mapping.get(old->root_page_id); + ASSERT_TRUE(tree.put("k", "new").ok()); + ASSERT_TRUE(tree.flush().ok()); + RuntimeTreeAccess_for_tests::reclaim(tree); + EXPECT_FALSE(retained.expired()); + old.reset(); + RuntimeTreeAccess_for_tests::reclaim(tree); + EXPECT_TRUE(retained.expired()); + EXPECT_EQ(tree.get_view("k").value().to_string(), "new"); } -// Retired count drops after reclaim: consolidation retires pages, the -// pending count is > 0 while a guard holds them, then drops to 0 after the -// guard is released and try_reclaim() runs. -TEST(Consolidation, RetiredCountAfterFold) +TEST(Consolidation, BorrowedValueSurvivesRetiredPageReclamation) { - Config opt; - opt.max_delta_len = 2; - Crowdbtree t(opt); - // Build a small tree with a few flushes so the delta chain grows. - for (uint64_t s = 1; s <= 3; ++s) { - ASSERT_TRUE(t.apply(s, put_one("k" + std::to_string(s), "v")).ok()); - ASSERT_TRUE(t.flush().ok()); - } - // Hold a guard so retired pages cannot be freed during consolidation. - { - EpochManager::Guard g = t.epoch().enter(); - ASSERT_TRUE(t.apply(4, put_one("k4", "v")).ok()); - ASSERT_TRUE(t.flush().ok()); // consolidation retires the old chain - EXPECT_GT(t.epoch().pending_retired(), 0U); - } - // Guard dropped -> retired pages become reclaimable. - t.epoch().try_reclaim(); - EXPECT_EQ(t.epoch().pending_retired(), 0U); + Crowdbtree tree; + ASSERT_TRUE(tree.put("k", "old").ok()); + ASSERT_TRUE(tree.flush().ok()); + auto borrowed = tree.get_view("k"); + ASSERT_TRUE(tree.put("k", "new").ok()); + ASSERT_TRUE(tree.flush().ok()); + RuntimeTreeAccess_for_tests::reclaim(tree); + EXPECT_EQ(borrowed.value().to_string(), "old"); + EXPECT_EQ(tree.get_view("k").value().to_string(), "new"); } diff --git a/lib/crowdb-tree/tests/unit/default_runtime_test.cpp b/lib/crowdb-tree/tests/unit/default_runtime_test.cpp new file mode 100644 index 000000000..8cfb4cb12 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/default_runtime_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/btree/tree.h" +#include "crowdb-tree/config.h" +#include "runtime/default_runtime.h" + +#include <gtest/gtest.h> + +#include <array> +#include <barrier> +#include <thread> +#include <vector> + +namespace crowdb::tree +{ +TEST(DefaultTreeRuntime, DefaultGeometryFlushesWithinTheBindingReclaimLimit) +{ + Crowdbtree tree; + ASSERT_TRUE(tree.put("key", "value").ok()); + const auto flushed = tree.flush(); + ASSERT_TRUE(flushed.ok()) << flushed.to_string(); + EXPECT_EQ(tree.get_view("key").value().to_string(), "value"); +} + +TEST(DefaultTreeRuntime, ConcurrentStandaloneConfigurationsShareOneHost) +{ + std::array<Config, 8> configs; + std::array<Status, 8> statuses; + std::barrier ready(8); + std::vector<std::thread> workers; + workers.reserve(configs.size()); + for (size_t index = 0; index < configs.size(); ++index) { + workers.emplace_back([&, index] { + ready.arrive_and_wait(); + statuses[index] = configure_default_runtime(&configs[index]); + }); + } + for (auto &worker : workers) { + worker.join(); + } + ASSERT_NE(configs.front().runtime, nullptr); + for (size_t index = 0; index < configs.size(); ++index) { + ASSERT_TRUE(statuses[index].ok()) << statuses[index].to_string(); + EXPECT_EQ(configs[index].runtime, configs.front().runtime); + } + const auto shared = configs.front().runtime; + for (auto &config : configs) { + config.runtime.reset(); + } + Config reopened; + ASSERT_TRUE(configure_default_runtime(&reopened).ok()); + EXPECT_EQ(reopened.runtime, shared); + EXPECT_FALSE(reopened.runtime->stopped()); +} + +TEST(DefaultTreeRuntime, ExplicitHostAndBindingLimitsArePreserved) +{ + Config config; + TreeRuntimeOptions options; + options.workers = 1; + ASSERT_TRUE(TreeRuntime::create(options, &config.runtime).ok()); + const auto explicit_host = config.runtime; + config.runtime_binding.cpu_queue_limit = 3; + ASSERT_TRUE(configure_default_runtime(&config).ok()); + EXPECT_EQ(config.runtime, explicit_host); + EXPECT_EQ(config.runtime->options().workers, 1U); + EXPECT_EQ(config.runtime_binding.cpu_queue_limit, 3U); + EXPECT_EQ(configure_default_runtime(nullptr).code(), Code::kInvalidArgument); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/epoch_budget_test.cpp b/lib/crowdb-tree/tests/unit/epoch_budget_test.cpp new file mode 100644 index 000000000..af38715d9 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/epoch_budget_test.cpp @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "crowdb-tree/epoch.h" + +#include <gtest/gtest.h> + +#include <array> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kBudget = static_cast<size_t>(detail::BudgetClass::kHandoff); + +struct TestCredit +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result{}; + result[kBudget] = 65536; + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> reserve(size_t bytes = 16384) const + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, bytes); + if (!credit) { + throw std::bad_alloc(); + } + return std::make_shared<detail::AllocationReservation>(std::move(*credit)); + } + + [[nodiscard]] size_t allocated() const + { + return budget.snapshot().allocated[kBudget]; + } +}; + +TEST(EpochBudget, DetachedGuardRetainsChargedLifetimeAndParticipantAfterManagerDestruction) +{ + TestCredit credit; + auto reservation = credit.reserve(); + auto manager = std::make_unique<EpochManager>(reservation); + const auto infrastructure = credit.allocated(); + EXPECT_GT(infrastructure, 0U); + auto guard = manager->enter(); + EXPECT_GT(credit.allocated(), infrastructure); + reservation->seal(); + manager.reset(); + EXPECT_GT(credit.allocated(), 0U); + guard = {}; + EXPECT_EQ(credit.allocated(), 0U); +} + +TEST(EpochBudget, CacheEvictionPreservesNestedParticipantWithoutAllocationAfterSeal) +{ + TestCredit credit; + auto reservation = credit.reserve(); + { + EpochManager manager(reservation); + auto outer = manager.enter(); + const auto allocated = credit.allocated(); + reservation->seal(); + std::array<std::unique_ptr<EpochManager>, 64> others; + for (auto &other : others) { + other = std::make_unique<EpochManager>(); + auto guard = other->enter(); + } + auto nested = manager.enter(); + EXPECT_EQ(manager.active_guards(), 1U); + EXPECT_EQ(credit.allocated(), allocated); + outer = {}; + EXPECT_EQ(manager.active_guards(), 1U); + nested = {}; + EXPECT_EQ(manager.active_guards(), 0U); + } + EXPECT_EQ(credit.allocated(), 0U); +} + +TEST(EpochBudget, NewWorkerUsesItsCurrentCreditAndFailedRegistrationIsRetryable) +{ + TestCredit credit; + auto initial = credit.reserve(); + auto work = credit.reserve(); + { + EpochManager manager(initial); + initial->seal(); + const auto before = credit.allocated(); + std::jthread worker([&] { + EXPECT_THROW(manager.enter(), std::bad_alloc); + EXPECT_EQ(credit.allocated(), before); + auto guard = manager.enter(work); + EXPECT_GT(credit.allocated(), before); + work->seal(); + auto nested = manager.enter(); + EXPECT_EQ(manager.active_guards(), 1U); + }); + worker.join(); + EXPECT_EQ(manager.active_guards(), 0U); + EXPECT_GT(credit.allocated(), before); + } + EXPECT_EQ(credit.allocated(), 0U); +} + +TEST(EpochBudget, ConstructionFailureReturnsAllPreviouslyClaimedInfrastructure) +{ + TestCredit credit; + auto tiny = credit.reserve(64); + EXPECT_THROW(EpochManager manager(tiny), std::bad_alloc); + EXPECT_EQ(credit.allocated(), 0U); + EXPECT_EQ(tiny->available(), 64U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/extent_exclusion_test.cpp b/lib/crowdb-tree/tests/unit/extent_exclusion_test.cpp new file mode 100644 index 000000000..545c4ad47 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/extent_exclusion_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "extent_ledger_test_support.h" + +namespace crowdb::tree +{ +using namespace test_read; + +namespace +{ +ExtentLedgerResult exclude(LedgerFixture &test, uint64_t address, uint64_t length) +{ + return await_ledger([&](auto done) { + return test.ledger->exclude_allocations(address, length, test.credits.credits.bytes, std::move(done)); + }); +} + +TEST(ExtentExclusion, ExistingReadersRemainProtectedAndNewWritesAvoidTheWholeInterval) +{ + LedgerFixture test({.backend_id = 1, .file_size = 65536, .address_limit = 65536}); + auto data = test.recover(16384, 4096); + ASSERT_TRUE(data.status.ok()); + ASSERT_TRUE(test.activate().status.ok()); + auto frozen = exclude(test, 8192, 24576); + ASSERT_TRUE(frozen.status.ok()); + EXPECT_EQ(frozen.occupied_bytes, 4096U); + EXPECT_EQ(test.ledger->stats().protected_bytes, 4096U); + auto retained = await_ledger( + [&](auto done) { return test.ledger->retain(16384, 4096, test.credits.credits.bytes, std::move(done)); }); + ASSERT_TRUE(retained.status.ok()); + EXPECT_EQ(retained.extent.owner, data.extent.owner); + EXPECT_EQ(exclude(test, 32768, 8192).status.code(), Code::kUnavailable); + auto outside = test.reserve(4096); + ASSERT_TRUE(outside.status.ok()); + EXPECT_GE(outside.extent.address, 32768U); + data = {}; + retained = {}; + auto after_release = test.reserve(4096); + ASSERT_TRUE(after_release.status.ok()); + EXPECT_GE(after_release.extent.address, 32768U); + frozen = {}; + auto reused = test.reserve(4096); + ASSERT_TRUE(reused.status.ok()); + EXPECT_EQ(reused.extent.address, 8192U); +} + +TEST(ExtentExclusion, FreeDeletionLeaseIsAllowedWhileFutureAllocationStaysFenced) +{ + LedgerFixture test({.backend_id = 1, .file_size = 32768, .address_limit = 32768}); + ASSERT_TRUE(test.activate().status.ok()); + auto frozen = exclude(test, 8192, 24576); + ASSERT_TRUE(frozen.status.ok()); + EXPECT_EQ(frozen.occupied_bytes, 0U); + EXPECT_EQ(test.reserve(4096).status.code(), Code::kResourceExhausted); + auto deletion = await_ledger( + [&](auto done) { return test.ledger->claim_free(8192, 24576, test.credits.credits.bytes, std::move(done)); }); + ASSERT_TRUE(deletion.status.ok()); + deletion = {}; + EXPECT_EQ(test.reserve(4096).status.code(), Code::kResourceExhausted); + test.credits.seal(); + frozen = {}; + test.ledger.reset(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(ExtentExclusion, GeometryAndCreditFailureLeaveTheAllocationIndexesUnchanged) +{ + LedgerFixture test({.backend_id = 1, .file_size = 32768, .address_limit = 32768}); + EXPECT_EQ(exclude(test, 8192, 4096).status.code(), Code::kUnavailable); + ASSERT_TRUE(test.activate().status.ok()); + EXPECT_EQ(exclude(test, 0, 4096).status.code(), Code::kInvalidArgument); + EXPECT_EQ(exclude(test, 32768, 4096).status.code(), Code::kInvalidArgument); + EXPECT_EQ(exclude(test, 8192, 1).status.code(), Code::kInvalidArgument); + test.credits.seal(); + EXPECT_EQ(exclude(test, 8192, 4096).status.code(), Code::kResourceExhausted); + auto allocated = await_ledger([&](auto done) { return test.ledger->reserve(4096, {}, std::move(done)); }); + ASSERT_TRUE(allocated.status.ok()); + EXPECT_EQ(allocated.extent.address, 8192U); + EXPECT_EQ(test.ledger->stats().address_high_water, 32768U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/extent_gc_lease_test.cpp b/lib/crowdb-tree/tests/unit/extent_gc_lease_test.cpp new file mode 100644 index 000000000..b0597f5b3 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/extent_gc_lease_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "extent_ledger_test_support.h" + +namespace crowdb::tree +{ +using namespace test_read; + +namespace +{ +ExtentLedgerResult claim(LedgerFixture &test, uint64_t address, uint64_t length) +{ + return await_ledger([&](auto done) { + return test.ledger->claim_free(address, length, test.credits.credits.bytes, std::move(done)); + }); +} + +TEST(ExtentGcLease, RequiresActivatedFreeRangeAndNeverBorrowsDataOwners) +{ + LedgerFixture test({.backend_id = 1, .file_size = 32768, .address_limit = 32768}); + auto live = test.recover(16384, 4096); + ASSERT_TRUE(live.status.ok()); + EXPECT_EQ(claim(test, 8192, 4096).status.code(), Code::kUnavailable); + ASSERT_TRUE(test.activate().status.ok()); + EXPECT_EQ(claim(test, 16384, 4096).status.code(), Code::kNotFound); + EXPECT_EQ(claim(test, 12288, 8192).status.code(), Code::kNotFound); + EXPECT_EQ(claim(test, 8192, 24576).status.code(), Code::kNotFound); + EXPECT_EQ(claim(test, 32768, 4096).status.code(), Code::kNotFound); + EXPECT_EQ(test.ledger->stats().address_high_water, 32768U); + auto leased = claim(test, 8192, 8192); + ASSERT_TRUE(leased.status.ok()); + EXPECT_EQ(leased.extent.address, 8192U); + EXPECT_NE(leased.extent.owner, live.extent.owner); + EXPECT_EQ(claim(test, 8192, 8192).status.code(), Code::kNotFound); + auto retained = await_ledger( + [&](auto done) { return test.ledger->retain(8192, 4096, test.credits.credits.bytes, std::move(done)); }); + EXPECT_EQ(retained.status.code(), Code::kNotFound); + auto allocated = test.reserve(8192); + ASSERT_TRUE(allocated.status.ok()); + EXPECT_GE(allocated.extent.address, 20480U); + leased = {}; + auto reused = test.reserve(8192); + ASSERT_TRUE(reused.status.ok()); + EXPECT_EQ(reused.extent.address, 8192U); +} + +TEST(ExtentGcLease, OldReaderPreventsDeletionUntilItsFinalOwnerReleases) +{ + LedgerFixture test({.backend_id = 1, .file_size = 16384, .address_limit = 16384}); + auto anchor = test.recover(8192, 8192); + ASSERT_TRUE(anchor.status.ok()); + auto reader = anchor.extent.owner; + ASSERT_TRUE(test.activate().status.ok()); + anchor = {}; + EXPECT_EQ(claim(test, 8192, 8192).status.code(), Code::kNotFound); + reader.reset(); + auto leased = claim(test, 8192, 8192); + ASSERT_TRUE(leased.status.ok()); + EXPECT_EQ(test.reserve(4096).status.code(), Code::kResourceExhausted); + test.credits.seal(); + leased = {}; + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + test.ledger.reset(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(ExtentGcLease, ExhaustedMetadataLeavesFreeIndexAndFrontierUnchanged) +{ + LedgerFixture test({.backend_id = 1, .file_size = 32768, .address_limit = 32768}); + ASSERT_TRUE(test.activate().status.ok()); + test.credits.seal(); + auto denied = claim(test, 12288, 8192); + EXPECT_EQ(denied.status.code(), Code::kResourceExhausted); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + EXPECT_EQ(test.ledger->stats().address_high_water, 32768U); + auto leased = await_ledger([&](auto done) { return test.ledger->claim_free(8192, 24576, {}, std::move(done)); }); + ASSERT_TRUE(leased.status.ok()); + EXPECT_EQ(leased.extent.address, 8192U); + EXPECT_EQ(leased.extent.length, 24576U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/extent_ledger_test.cpp b/lib/crowdb-tree/tests/unit/extent_ledger_test.cpp new file mode 100644 index 000000000..84a366ec1 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/extent_ledger_test.cpp @@ -0,0 +1,269 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "extent_ledger_test_support.h" + +#include <set> +#include <thread> + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(ExtentLedger, RecoveryProtectsBothAnchorOwnersBeforeAllocationAndReaderPinsSurviveRotation) +{ + LedgerFixture test({.backend_id = 7, .file_size = 65536, .address_limit = 65536}); + auto first = test.recover(8192, 8192); + auto second = test.recover(8192, 8192); + auto other = test.recover(24576, 4096); + ASSERT_TRUE(first.status.ok()); + ASSERT_TRUE(second.status.ok()); + ASSERT_TRUE(other.status.ok()); + EXPECT_EQ(first.extent.owner, second.extent.owner); + EXPECT_EQ(test.ledger->stats().protected_bytes, 12288U); + EXPECT_EQ(test.reserve(4096).status.code(), Code::kUnavailable); + EXPECT_EQ(test.recover(12288, 8192).status.code(), Code::kCorruption); + ASSERT_TRUE(test.activate().status.ok()); + auto gap = test.reserve(4096); + ASSERT_TRUE(gap.status.ok()); + EXPECT_EQ(gap.extent.address, 16384U); + auto reader = first.extent; + first = {}; + second = {}; + auto later = test.reserve(8192); + ASSERT_TRUE(later.status.ok()); + EXPECT_GE(later.extent.address, 28672U); + const auto old_identity = reader.identity; + reader = {}; + auto reused = test.reserve(8192); + ASSERT_TRUE(reused.status.ok()); + EXPECT_EQ(reused.extent.address, 8192U); + EXPECT_NE(reused.extent.identity, old_identity); + EXPECT_EQ(reused.extent.backend_id, 7U); +} + +TEST(ExtentLedger, LastOwnerReleaseCoalescesBothNeighborsWithoutAllocatingAfterSeal) +{ + LedgerFixture test({.backend_id = 1, .file_size = 24576, .address_limit = 24576}); + ASSERT_TRUE(test.activate().status.ok()); + auto a = test.reserve(4096); + auto b = test.reserve(4096); + auto c = test.reserve(4096); + ASSERT_TRUE(a.status.ok()); + ASSERT_TRUE(b.status.ok()); + ASSERT_TRUE(c.status.ok()); + EXPECT_EQ(test.ledger->stats().protected_ranges, 3U); + test.credits.seal(); + a = {}; + c = {}; + b = {}; + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + EXPECT_EQ(test.ledger->stats().protected_bytes, 0U); + // The free-space index stays charged until the allocation domain ends. + EXPECT_GT(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + test.ledger.reset(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(ExtentLedger, RecoveryRejectsPartialOverlapAndCapacityNeverWrapsOrInventsSpace) +{ + LedgerFixture test({.backend_id = 1, .file_size = 16384, .address_limit = 20480}); + auto recovered = test.recover(8192, 4096); + ASSERT_TRUE(recovered.status.ok()); + EXPECT_EQ(test.recover(8192, 8192).status.code(), Code::kCorruption); + EXPECT_EQ(test.recover(1, 4096).status.code(), Code::kInvalidArgument); + ASSERT_TRUE(test.activate().status.ok()); + auto tail = test.reserve(8192); + EXPECT_EQ(tail.status.code(), Code::kResourceExhausted); + auto gap = test.reserve(4096); + auto append = test.reserve(4096); + ASSERT_TRUE(gap.status.ok()); + ASSERT_TRUE(append.status.ok()); + EXPECT_EQ(gap.extent.address, 12288U); + EXPECT_EQ(append.extent.address, 16384U); + EXPECT_EQ(test.reserve(4096).status.code(), Code::kResourceExhausted); + EXPECT_EQ(test.reserve(UINT64_MAX).status.code(), Code::kInvalidArgument); + EXPECT_EQ(test.recover(12288, 4096).status.code(), Code::kUnavailable); + auto retained = await_ledger( + [&](auto done) { return test.ledger->retain(8192, 4096, test.credits.credits.bytes, std::move(done)); }); + EXPECT_TRUE(retained.status.ok()); + EXPECT_EQ(retained.extent.owner, recovered.extent.owner); +} + +TEST(ExtentLedger, MetadataBoundPaysForReserveAndAllocationFailuresLeaveIndexesUnchanged) +{ + bool rejected = false; + bool failed_after_submit = false; + bool succeeded = false; + for (size_t amount = 64; amount <= ExtentLedger::reservation_metadata_bound(); amount += 64) { + LoadCredits credits; + auto carved = credits.credits.bytes->take_reserved(amount); + if (!carved.has_value()) { + FAIL() << "test credit cannot fund reservation"; + } + auto limited = std::make_shared<detail::AllocationReservation>(std::move(carved.value())); + std::shared_ptr<ExtentLedger> ledger; + ASSERT_TRUE(ExtentLedger::create({.backend_id = 1, .file_size = 65536}, {}, &ledger).ok()); + ASSERT_TRUE(await_ledger([&](auto done) { return ledger->activate({}, std::move(done)); }).status.ok()); + bool called = false; + ExtentLedgerResult outcome; + auto submitted = ledger->reserve(4096, limited, [&](ExtentLedgerResult result) { + called = true; + outcome = std::move(result); + }); + if (!submitted.ok()) { + rejected = true; + EXPECT_FALSE(called); + } + else if (!outcome.status.ok()) { + failed_after_submit = true; + EXPECT_EQ(ledger->stats().protected_ranges, 0U); + auto retry = await_ledger([&](auto done) { return ledger->reserve(4096, {}, std::move(done)); }); + ASSERT_TRUE(retry.status.ok()); + EXPECT_EQ(retry.extent.address, 8192U); + } + else { + succeeded = true; + EXPECT_EQ(outcome.extent.address, 8192U); + } + outcome = {}; + ledger.reset(); + limited->seal(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + } + EXPECT_TRUE(rejected); + EXPECT_TRUE(failed_after_submit); + EXPECT_TRUE(succeeded); +} + +TEST(ExtentLedger, ConcurrentReservationsAndCrossThreadReleasesShareOneReuseDomain) +{ + LedgerFixture test({.backend_id = 1, .address_limit = 2 * 1024 * 1024}); + ASSERT_TRUE(test.activate().status.ok()); + std::vector<ProtectedExtent> extents(200); + std::vector<std::thread> workers; + workers.reserve(4); + for (size_t worker = 0; worker < 4; ++worker) { + workers.emplace_back([&, worker] { + for (size_t i = worker; i < extents.size(); i += 4) { + auto result = test.reserve(4096); + EXPECT_TRUE(result.status.ok()); + extents[i] = std::move(result.extent); + } + }); + } + for (auto &worker : workers) { + worker.join(); + } + std::set<uint64_t> addresses; + std::set<uint64_t> identities; + for (const auto &extent : extents) { + addresses.insert(extent.address); + identities.insert(extent.identity); + } + EXPECT_EQ(addresses.size(), extents.size()); + EXPECT_EQ(identities.size(), extents.size()); + EXPECT_EQ(test.ledger->stats().protected_ranges, extents.size()); + workers.clear(); + for (size_t worker = 0; worker < 4; ++worker) { + workers.emplace_back([&, worker] { + for (size_t i = worker; i < extents.size(); i += 4) { + extents[i] = {}; + } + }); + } + for (auto &worker : workers) { + worker.join(); + } + auto reused = test.reserve(4096); + ASSERT_TRUE(reused.status.ok()); + EXPECT_EQ(reused.extent.address, 8192U); + EXPECT_EQ(test.ledger->stats().protected_ranges, 1U); +} + +TEST(ExtentLedger, RangeLimitRejectsSplitsWithoutLosingTheOriginalGap) +{ + LedgerFixture test({.backend_id = 1, .file_size = 24576, .address_limit = 24576, .max_ranges = 2}); + EXPECT_EQ(test.recover(12288, 4096).status.code(), Code::kResourceExhausted); + EXPECT_EQ(test.ledger->stats().protected_ranges, 0U); + auto first = test.recover(8192, 4096); + ASSERT_TRUE(first.status.ok()); + ASSERT_TRUE(test.activate().status.ok()); + EXPECT_EQ(test.reserve(4096).status.code(), Code::kResourceExhausted); + auto rest = test.reserve(12288); + ASSERT_TRUE(rest.status.ok()); + EXPECT_EQ(rest.extent.address, 12288U); + first = {}; + rest = {}; + auto whole = test.reserve(16384); + ASSERT_TRUE(whole.status.ok()); + EXPECT_EQ(whole.extent.address, 8192U); +} + +TEST(ExtentLedger, MiddleRecoveryFitsItsBoundAndEveryFailedAllocationRollsBack) +{ + const auto bound = ExtentLedger::reservation_metadata_bound(); + for (size_t amount = 64; amount <= bound; amount = std::min(amount + 64, bound)) { + LoadCredits credits; + auto carved = credits.credits.bytes->take_reserved(amount); + if (!carved.has_value()) { + FAIL() << "test credit cannot fund recovery"; + } + auto limited = std::make_shared<detail::AllocationReservation>(std::move(carved.value())); + std::shared_ptr<ExtentLedger> ledger; + ASSERT_TRUE(ExtentLedger::create({.backend_id = 1, .file_size = 32768}, {}, &ledger).ok()); + auto result = await_ledger([&](auto done) { return ledger->recover(16384, 4096, limited, std::move(done)); }); + if (amount == bound) { + ASSERT_TRUE(result.status.ok()); + } + if (!result.status.ok()) { + EXPECT_EQ(result.status.code(), Code::kResourceExhausted); + EXPECT_EQ(ledger->stats().protected_ranges, 0U); + result = await_ledger([&](auto done) { return ledger->recover(8192, 24576, {}, std::move(done)); }); + ASSERT_TRUE(result.status.ok()); + } + result = {}; + ledger.reset(); + limited->seal(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + if (amount == bound) { + break; + } + } +} + +TEST(ExtentLedger, QueueReentryAndThrowingReceiverDoNotStrandAllocationOrRelease) +{ + LedgerFixture test({.backend_id = 1, .queue_capacity = 1}); + ASSERT_TRUE(test.activate().status.ok()); + size_t delivered = 0; + ProtectedExtent retained; + ASSERT_TRUE( + test.ledger + ->reserve(4096, test.credits.credits.bytes, + [&](ExtentLedgerResult first) { + EXPECT_TRUE(first.status.ok()); + EXPECT_TRUE(test.ledger + ->reserve(4096, test.credits.credits.bytes, + [&](ExtentLedgerResult next) { + EXPECT_TRUE(next.status.ok()); + retained = std::move(next.extent); + ++delivered; + }) + .ok()); + EXPECT_EQ( + test.ledger->reserve(4096, test.credits.credits.bytes, [](ExtentLedgerResult) {}).code(), + Code::kResourceExhausted); + throw std::runtime_error("controlled receiver failure"); + }) + .ok()); + EXPECT_EQ(delivered, 1U); + EXPECT_EQ(test.ledger->stats().protected_ranges, 1U); + test.ledger->close(); + test.ledger.reset(); + test.credits.seal(); + retained = {}; + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/external_allocation_test.cpp b/lib/crowdb-tree/tests/unit/external_allocation_test.cpp new file mode 100644 index 000000000..a0f05544b --- /dev/null +++ b/lib/crowdb-tree/tests/unit/external_allocation_test.cpp @@ -0,0 +1,176 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "crowdb-tree/external_allocation.h" +#include "crowdb-tree/memtable/memtable.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kWriter = static_cast<size_t>(detail::BudgetClass::kAdmittedWriter); + +struct TestBudget +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result{}; + result[kWriter] = 1024 * 1024; + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> reserve(size_t bytes = 256 * 1024) const + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kAdmittedWriter, bytes); + if (!credit) { + throw std::bad_alloc(); + } + return std::make_shared<detail::AllocationReservation>(std::move(*credit)); + } + + [[nodiscard]] size_t allocated() const + { + return budget.snapshot().allocated[kWriter]; + } +}; + +struct TestPayload +{ + std::array<uint8_t, 8192> bytes{}; + size_t *drops; +}; + +std::shared_ptr<const ExternalAllocation> payload(const std::shared_ptr<AllocationBudget> &budget, size_t &drops) +{ + auto *owner = new TestPayload{.drops = &drops}; + owner->bytes.fill('v'); + ExternalAllocation::Owner handle(owner, [](void *context) { + auto *owner = static_cast<TestPayload *>(context); + ++*owner->drops; + delete owner; + }); + // The owner allocation includes the drop-counter pointer too. + return ExternalAllocation::create(owner->bytes.data(), sizeof(TestPayload), std::move(handle), budget); +} + +TEST(ExternalAllocation, SharedSlicesKeepOneCapacityChargeAndSurviveSourceDetachment) +{ + TestBudget budget; + auto payload_credit = budget.reserve(); + auto source_credit = budget.reserve(); + size_t drops = 0; + auto allocation = payload(payload_credit, drops); + auto table = std::make_shared<MemTable>(1, nullptr, source_credit); + const auto before = budget.allocated(); + const auto *base = allocation->data(); + for (size_t index = 0; index < 4; ++index) { + auto value = buffer::share_external(allocation, index * 1024, 1024); + ASSERT_TRUE(table->upsert_external(std::to_string(index), 1, 0, std::move(value), 0)); + } + // Four 1 KiB views retain one 8 KiB allocation, plus bounded descriptors. + EXPECT_LT(budget.allocated() - before, 4096U); + std::shared_ptr<const CellVersion> retained; + { + auto guard = table->read_guard(); + retained = table->prefix_cursor(1).cell_owner(); + } + ASSERT_NE(retained, nullptr); + EXPECT_EQ(retained->cell.data(), base); + EXPECT_EQ(retained->cell.external_allocation(), allocation.get()); + allocation.reset(); + payload_credit->seal(); + source_credit->seal(); + table.reset(); + EXPECT_EQ(drops, 0U); + EXPECT_GE(budget.allocated(), sizeof(TestPayload)); + EXPECT_EQ(retained->cell.slice().to_string(), std::string(1024, 'v')); + retained.reset(); + EXPECT_EQ(drops, 1U); + EXPECT_EQ(budget.allocated(), 0U); +} + +TEST(ExternalAllocation, SeparateTreeAccountsShareOneHostPayloadCharge) +{ + TestBudget budget; + auto payload_credit = budget.reserve(); + auto second_account = budget.budget.make_account(TestBudget::limits()); + auto credit = detail::ResourceBudget::try_reserve(second_account, detail::BudgetClass::kAdmittedWriter, 65536); + if (!credit) { + FAIL() << "test source credit unavailable"; + } + auto source_credit = std::make_shared<detail::AllocationReservation>(std::move(*credit)); + size_t drops = 0; + auto allocation = payload(payload_credit, drops); + const auto *data = allocation->data(); + auto first = buffer::share_external(allocation, 0, 1024); + auto second = buffer::share_external(allocation, 1024, 1024); + MemTable table(1, nullptr, source_credit); + const auto before = budget.allocated(); + ASSERT_TRUE(table.upsert_external("key", 1, 0, std::move(second), 0)); + EXPECT_LT(budget.allocated() - before, 1024U); + allocation.reset(); + first = {}; + auto guard = table.read_guard(); + ASSERT_NE(table.find("key"), nullptr); + EXPECT_EQ(table.find("key")->cell.data(), data + 1024); + EXPECT_EQ(drops, 0U); +} + +TEST(ExternalAllocation, ForeignHostCopiesAndReleasesOriginalOwner) +{ + TestBudget first; + TestBudget second; + auto payload_credit = first.reserve(); + auto source_credit = second.reserve(); + size_t drops = 0; + auto allocation = payload(payload_credit, drops); + auto value = buffer::share_external(allocation, 0, 1024); + allocation.reset(); + MemTable table(1, nullptr, source_credit); + ASSERT_TRUE(table.upsert_external("key", 1, 0, std::move(value), 0)); + EXPECT_EQ(drops, 1U); + EXPECT_EQ(first.allocated(), 0U); + auto guard = table.read_guard(); + ASSERT_NE(table.find("key"), nullptr); + EXPECT_EQ(table.find("key")->cell.ownership(), buffer::mode::kOwned); + EXPECT_EQ(CellView(table.find("key")->cell.slice()).value().to_string(), std::string(1024, 'v')); +} + +TEST(ExternalAllocation, SliceBoundsAndMovesDoNotLeakOrDuplicateOwner) +{ + TestBudget budget; + auto credit = budget.reserve(); + size_t drops = 0; + auto allocation = payload(credit, drops); + const auto before = budget.allocated(); + EXPECT_THROW(buffer::share_external(allocation, SIZE_MAX, 1), std::out_of_range); + EXPECT_THROW(buffer::share_external(allocation, 1, SIZE_MAX), std::out_of_range); + EXPECT_EQ(budget.allocated(), before); + auto value = buffer::share_external(allocation, 0, 1024); + auto moved = std::move(value); + EXPECT_EQ(moved.external_allocation(), allocation.get()); + allocation.reset(); + EXPECT_EQ(drops, 0U); + moved = {}; + EXPECT_EQ(drops, 1U); + EXPECT_EQ(budget.allocated(), 0U); +} + +TEST(ExternalAllocation, FailedDescriptorOrPayloadReservationDropsInputExactlyOnce) +{ + for (size_t bytes : {size_t{0}, size_t{1024}}) { + TestBudget budget; + auto credit = budget.reserve(bytes); + size_t drops = 0; + EXPECT_THROW(payload(credit, drops), std::bad_alloc); + EXPECT_EQ(drops, 1U); + EXPECT_EQ(budget.allocated(), 0U); + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/external_owner_bridge_test.cpp b/lib/crowdb-tree/tests/unit/external_owner_bridge_test.cpp new file mode 100644 index 000000000..eaf2b3217 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/external_owner_bridge_test.cpp @@ -0,0 +1,207 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "crowdb-tree/memtable/memtable.h" +#include "external_owner_bridge.h" + +#include <gtest/gtest.h> + +#include <array> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kWriter = static_cast<size_t>(detail::BudgetClass::kAdmittedWriter); + +struct TestHost +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result{}; + result[kWriter] = 1024 * 1024; + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + + std::shared_ptr<detail::AllocationReservation> reserve(size_t bytes = 65536) + { + auto account = budget.make_account(limits()); + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kAdmittedWriter, bytes); + if (!credit) { + throw std::bad_alloc(); + } + return std::make_shared<detail::AllocationReservation>(std::move(*credit)); + } + + [[nodiscard]] size_t allocated() const + { + return budget.snapshot().allocated[kWriter]; + } +}; + +struct TestPayload +{ + std::array<uint8_t, 8192> bytes{}; + size_t *drops; +}; + +Status create_owner(const std::shared_ptr<AllocationBudget> &budget, size_t &drops, + std::shared_ptr<const ExternalOwnerBridge> *out) +{ + auto payload = std::make_unique<TestPayload>(); + payload->bytes.fill('v'); + payload->drops = &drops; + const auto *data = payload->bytes.data(); + ExternalAllocation::Owner owner(payload.release(), [](void *context) { + auto *payload = static_cast<TestPayload *>(context); + ++*payload->drops; + delete payload; + }); + return ExternalOwnerBridge::create(data, 4096, sizeof(TestPayload), std::move(owner), budget, out); +} + +TEST(ExternalOwnerBridge, SharedSlicesChargeCompleteCapacityAndFinalDropOnce) +{ + TestHost host; + auto credit = host.reserve(); + size_t drops = 0; + std::shared_ptr<const ExternalOwnerBridge> owner; + ASSERT_TRUE(create_owner(credit, drops, &owner).ok()); + EXPECT_EQ(owner->capacity(), sizeof(TestPayload)); + EXPECT_GE(host.allocated(), sizeof(TestPayload)); + const auto *identity = owner->identity(); + buffer first; + buffer second; + const auto before = host.allocated(); + ASSERT_TRUE(owner->slice(0, 1024, &first).ok()); + ASSERT_TRUE(owner->slice(1024, 1024, &second).ok()); + EXPECT_LT(host.allocated() - before, 256U); + EXPECT_EQ(first.external_allocation(), identity); + EXPECT_EQ(second.external_allocation(), identity); + owner.reset(); + credit->seal(); + first = {}; + EXPECT_EQ(drops, 0U); + EXPECT_EQ(second.slice().to_string(), std::string(1024, 'v')); + second = {}; + EXPECT_EQ(drops, 1U); + EXPECT_EQ(host.allocated(), 0U); +} + +TEST(ExternalOwnerBridge, SliceRejectsSpareCapacityAndOverflowWithoutConsumingCredit) +{ + TestHost host; + auto credit = host.reserve(); + size_t drops = 0; + std::shared_ptr<const ExternalOwnerBridge> owner; + ASSERT_TRUE(create_owner(credit, drops, &owner).ok()); + buffer slice; + const auto before = host.allocated(); + EXPECT_EQ(owner->slice(4096, 1, &slice).code(), Code::kInvalidArgument); + EXPECT_EQ(owner->slice(SIZE_MAX, 1, &slice).code(), Code::kInvalidArgument); + EXPECT_EQ(owner->slice(1, SIZE_MAX, &slice).code(), Code::kInvalidArgument); + EXPECT_EQ(host.allocated(), before); + ASSERT_TRUE(owner->slice(4096, 0, &slice).ok()); +} + +TEST(ExternalOwnerBridge, MissingHostAndExhaustedReservationsReleaseOwner) +{ + size_t drops = 0; + std::shared_ptr<const ExternalOwnerBridge> owner; + EXPECT_EQ(create_owner({}, drops, &owner).code(), Code::kInvalidArgument); + EXPECT_EQ(drops, 1U); + TestHost host; + for (size_t capacity : {size_t{0}, size_t{1024}}) { + auto credit = host.reserve(capacity); + EXPECT_EQ(create_owner(credit, drops, &owner).code(), Code::kResourceExhausted); + EXPECT_EQ(owner, nullptr); + EXPECT_EQ(host.allocated(), 0U); + } + EXPECT_EQ(drops, 3U); +} + +TEST(ExternalOwnerBridge, InvalidCompleteAllocationDescriptorDropsWithoutCharging) +{ + TestHost host; + auto credit = host.reserve(); + size_t drops = 0; + std::array<uint8_t, 4> bytes{}; + std::shared_ptr<const ExternalOwnerBridge> owner; + const auto release = [](void *context) { ++*static_cast<size_t *>(context); }; + EXPECT_EQ(ExternalOwnerBridge::create(bytes.data(), 5, 4, {&drops, release}, credit, &owner).code(), + Code::kInvalidArgument); + EXPECT_EQ(ExternalOwnerBridge::create(nullptr, 1, 4, {&drops, release}, credit, &owner).code(), + Code::kInvalidArgument); + EXPECT_EQ(ExternalOwnerBridge::create(bytes.data(), 1, 4, {&drops, release}, credit, nullptr).code(), + Code::kInvalidArgument); + EXPECT_EQ(drops, 3U); + EXPECT_EQ(host.allocated(), 0U); +} + +TEST(ExternalOwnerBridge, TwoTreeAccountsRetainIdentityWithOneHostCapacityCharge) +{ + TestHost host; + auto payload_credit = host.reserve(); + auto first_credit = host.reserve(); + auto second_credit = host.reserve(); + size_t drops = 0; + std::shared_ptr<const ExternalOwnerBridge> owner; + ASSERT_TRUE(create_owner(payload_credit, drops, &owner).ok()); + const auto *data = owner->identity()->data(); + { + MemTable first(1, nullptr, first_credit); + MemTable second(1, nullptr, second_credit); + buffer left; + buffer right; + ASSERT_TRUE(owner->slice(0, 1024, &left).ok()); + ASSERT_TRUE(owner->slice(1024, 1024, &right).ok()); + const auto before = host.allocated(); + ASSERT_TRUE(first.upsert_external("left", 1, 0, std::move(left), 0)); + ASSERT_TRUE(second.upsert_external("right", 1, 0, std::move(right), 0)); + EXPECT_LT(host.allocated() - before, 4096U); + owner.reset(); + auto left_guard = first.read_guard(); + auto right_guard = second.read_guard(); + EXPECT_EQ(first.find("left")->cell.data(), data); + EXPECT_EQ(second.find("right")->cell.data(), data + 1024); + EXPECT_EQ(drops, 0U); + } + EXPECT_EQ(drops, 1U); + EXPECT_EQ(host.allocated(), 0U); +} + +TEST(ExternalOwnerBridge, ForeignHostCopiesAndCancellationCanReleaseOnAnotherThread) +{ + TestHost first; + TestHost second; + auto payload_credit = first.reserve(); + auto source_credit = second.reserve(); + size_t drops = 0; + std::shared_ptr<const ExternalOwnerBridge> owner; + ASSERT_TRUE(create_owner(payload_credit, drops, &owner).ok()); + buffer value; + ASSERT_TRUE(owner->slice(0, 1024, &value).ok()); + owner.reset(); + { + MemTable table(1, nullptr, source_credit); + ASSERT_TRUE(table.upsert_external("key", 1, 0, std::move(value), 0)); + EXPECT_EQ(drops, 1U); + EXPECT_EQ(first.allocated(), 0U); + auto guard = table.read_guard(); + EXPECT_TRUE(table.find("key")->cell.owned()); + } + EXPECT_EQ(second.allocated(), 0U); + ASSERT_TRUE(create_owner(payload_credit, drops, &owner).ok()); + ASSERT_TRUE(owner->slice(0, 1, &value).ok()); + owner.reset(); + payload_credit->seal(); + std::thread release([value = std::move(value)]() mutable { value = {}; }); + release.join(); + EXPECT_EQ(drops, 2U); + EXPECT_EQ(first.allocated(), 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/flush_report_sender_test.cpp b/lib/crowdb-tree/tests/unit/flush_report_sender_test.cpp new file mode 100644 index 000000000..d8b28629f --- /dev/null +++ b/lib/crowdb-tree/tests/unit/flush_report_sender_test.cpp @@ -0,0 +1,382 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/apply_progress.h" +#include "btree/flush_operation.h" + +#include <gtest/gtest.h> + +#include <future> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +class FlushReportTest : public testing::Test +{ + protected: + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(4 * 1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + SourceAdmission sources; + PublicationCoordinator publisher; + detail::FlushRuntime runtime{2, 8}; + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::atomic<uint64_t> contiguous{1}; + + void SetUp() override + { + auto inputs = std::make_shared<PublishedSources>(); + const auto catalog = sources.catalog(); + inputs->floor = catalog->floor; + inputs->tables.assign(catalog->tables.begin(), catalog->tables.end()); + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{.generation = 1}), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0, + .expected_sources = publisher.capture()->sources, + .sources = std::move(inputs)}; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(publisher.try_submit(request)); + } + + void TearDown() override + { + runtime.drain(); + } + + std::shared_ptr<detail::AllocationReservation> reserve() + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, 1024 * 1024); + if (!credit) { + ADD_FAILURE() << "test reservation failed"; + return {}; + } + return std::make_shared<detail::AllocationReservation>(std::move(credit.value())); + } +}; + +struct ReportReceiver +{ + using receiver_concept = stdexec::receiver_tag; + std::promise<FlushResult> *done; + + void set_value(FlushResult report) const noexcept + { + done->set_value(std::move(report)); + } + + void set_error(Status status) const noexcept + { + FlushResult report; + report.handoff_status = std::move(status); + done->set_value(std::move(report)); + } + + void set_stopped() const noexcept + { + set_error(Status::unavailable()); + } +}; + +TEST_F(FlushReportTest, ConcurrentConnectionRejectsWithoutObservingActiveProgress) +{ + FlushController controller(publisher, runtime, sources, contiguous, lane); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + ASSERT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "value"))); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(controller.prepare(reserve(), &flush).ok()); + std::promise<FlushResult> accepted; + auto finished = accepted.get_future(); + auto first = stdexec::connect(flush->report_sender(), ReportReceiver{&accepted}); + stdexec::start(first); + runtime.drain(); + const auto waiting_metrics = publisher.work_metrics()->totals(); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::FlushCapture)].count, 1U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::FlushWriterDrain)].inflight, 1U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::FlushRange)].attempts, 0U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::FlushBatch)].attempts, 1U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::FlushBatch)].inflight, 1U); + EXPECT_EQ(waiting_metrics[static_cast<size_t>(TreeStage::FlushBatch)].count, 0U); + std::promise<FlushResult> rejected; + auto rejected_result = rejected.get_future(); + auto second = stdexec::connect(flush->report_sender(), ReportReceiver{&rejected}); + stdexec::start(second); + const auto report = rejected_result.get(); + EXPECT_EQ(report.handoff_status.code(), Code::kUnavailable); + EXPECT_EQ(report.frontier, 0U); + EXPECT_FALSE(report.handoff_advanced); + writer.release(); + const auto completed = finished.get(); + EXPECT_EQ(completed.frontier, 1U); + EXPECT_TRUE(completed.handoff_status.ok()); + EXPECT_TRUE(completed.handoff_advanced); + const auto completed_metrics = publisher.work_metrics()->totals(); + EXPECT_EQ(completed_metrics[static_cast<size_t>(TreeStage::FlushBatch)].attempts, 1U); + EXPECT_EQ(completed_metrics[static_cast<size_t>(TreeStage::FlushBatch)].count, 1U); + EXPECT_EQ(completed_metrics[static_cast<size_t>(TreeStage::FlushBatch)].inflight, 0U); + EXPECT_EQ(completed_metrics[static_cast<size_t>(TreeStage::FlushWriterDrain)].count, 1U); + EXPECT_EQ(completed_metrics[static_cast<size_t>(TreeStage::FlushWriterDrain)].inflight, 0U); + EXPECT_EQ(completed_metrics[static_cast<size_t>(TreeStage::FlushRange)].count, 1U); +} + +struct PausedReportReceiver +{ + using receiver_concept = stdexec::receiver_tag; + std::atomic<bool> *entered; + std::atomic<bool> *release; + std::promise<FlushResult> *done; + + void set_value(FlushResult report) const noexcept + { + entered->store(true, std::memory_order_release); + entered->notify_one(); + while (!release->load(std::memory_order_acquire)) { + release->wait(false); + } + done->set_value(std::move(report)); + } + + void set_error(Status status) const noexcept + { + ADD_FAILURE() << status.to_string(); + FlushResult rejected; + rejected.handoff_status = std::move(status); + set_value(std::move(rejected)); + } + + void set_stopped() const noexcept + { + set_error(Status::unavailable()); + } +}; + +TEST_F(FlushReportTest, ReconnectCannotOverwriteAnUndeliveredTerminalValue) +{ + size_t attempts = 0; + FlushController controller(publisher, runtime, sources, contiguous, lane, {}, [&](auto, auto complete) { + ++attempts; + complete(attempts == 1 ? Status::io_error() : Status::Ok(), attempts == 1 ? 7 : 0); + }); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + ASSERT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "value"))); + writer.release(); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(controller.prepare(reserve(), &flush).ok()); + std::promise<FlushResult> accepted; + auto finished = accepted.get_future(); + std::atomic<bool> entered{false}; + std::atomic<bool> release{false}; + auto first = stdexec::connect(flush->report_sender(), + PausedReportReceiver{.entered = &entered, .release = &release, .done = &accepted}); + stdexec::start(first); + while (!entered.load(std::memory_order_acquire)) { + entered.wait(false); + } + + struct ReleaseOnExit + { + std::atomic<bool> &gate; + + ~ReleaseOnExit() + { + gate.store(true, std::memory_order_release); + gate.notify_one(); + } + } release_on_exit{release}; + + auto retry = stdexec::sync_wait(flush->report_sender()); + release.store(true, std::memory_order_release); + release.notify_one(); + const auto original = finished.get(); + runtime.drain(); + if (!retry.has_value()) { + FAIL() << "retry stopped without a progress report"; + } + const auto &retried = std::get<0>(retry.value()); + EXPECT_TRUE(retried.maintenance_status.ok()); + EXPECT_EQ(retried.deferred_pages, 0U); + EXPECT_FALSE(retried.handoff_advanced); + EXPECT_EQ(original.maintenance_status.code(), Code::kIoError); + EXPECT_EQ(original.deferred_pages, 7U); + EXPECT_TRUE(original.handoff_advanced); + EXPECT_EQ(original.batch_id, retried.batch_id); + EXPECT_EQ(original.frontier, retried.frontier); +} + +TEST_F(FlushReportTest, NoProgressDoesNotCountAHandoff) +{ + contiguous.store(0); + FlushController controller(publisher, runtime, sources, contiguous, lane); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(controller.prepare(reserve(), &flush).ok()); + auto completed = stdexec::sync_wait(flush->report_sender()); + runtime.drain(); + if (!completed.has_value()) { + FAIL() << "flush stopped without a progress report"; + } + const auto &report = std::get<0>(completed.value()); + EXPECT_EQ(report.phase, FlushPhase::kComplete); + EXPECT_FALSE(report.handoff_advanced); + EXPECT_EQ(report.frontier, report.floor); +} + +TEST_F(FlushReportTest, CaptureDrainsWritersWithoutCoverageAndNormalSenderResumesTheSamePrefix) +{ + FlushController controller(publisher, runtime, sources, contiguous, lane); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + ASSERT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "inherited"))); + const auto original = writer.table(); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(controller.prepare(reserve(), &flush).ok()); + std::promise<FlushResult> terminal; + auto future = terminal.get_future(); + auto capture = stdexec::connect(flush->capture_sender(), ReportReceiver{&terminal}); + stdexec::start(capture); + runtime.drain(); + EXPECT_EQ(future.wait_for(std::chrono::seconds(0)), std::future_status::timeout); + EXPECT_EQ(publisher.capture()->coverage, 0U); + writer.release(); + const auto report = future.get(); + ASSERT_TRUE(report.handoff_status.ok()); + EXPECT_EQ(report.phase, FlushPhase::kCaptured); + EXPECT_EQ(report.frontier, 1U); + EXPECT_EQ(report.published_coverage, 0U); + EXPECT_FALSE(report.handoff_advanced); + const auto frozen = flush->frozen_sources(); + ASSERT_TRUE(frozen.catalog); + ASSERT_TRUE(frozen.base); + EXPECT_EQ(frozen.base->coverage, frozen.catalog->floor); + ASSERT_EQ(frozen.catalog->tables.size(), 1U); + EXPECT_EQ(frozen.catalog->tables.front(), original); + EXPECT_TRUE(original->closed()); + EXPECT_EQ(original->writers(), 0U); + + SourceAdmission::Writer successor; + ASSERT_TRUE(sources.try_admit(1, &successor).ok()); + EXPECT_NE(successor.table(), original); + ASSERT_TRUE(successor.table()->upsert("a", 2, encode_cell_buf(2, OpKind::kPut, "parent-later"))); + successor.release(); + contiguous.store(2); + const auto resumed = stdexec::sync_wait(flush->report_sender()); + if (!resumed) { + FAIL() << "flush sender stopped"; + } + EXPECT_TRUE(std::get<0>(*resumed).handoff_status.ok()); + EXPECT_TRUE(std::get<0>(*resumed).handoff_advanced); + EXPECT_EQ(std::get<0>(*resumed).published_coverage, 1U); + EXPECT_EQ(publisher.capture()->coverage, 1U); + EXPECT_EQ(flush->frozen_sources().catalog, frozen.catalog); + EXPECT_EQ(flush->frozen_sources().base, frozen.base); + auto guard = original->read_guard(); + auto cursor = original->prefix_cursor(1); + ASSERT_TRUE(cursor.valid()); + EXPECT_EQ(CellView(cursor.cell_version()->cell.slice()).value().to_string(), "inherited"); +} + +TEST_F(FlushReportTest, RepeatedCaptureHasTheSameFrozenIdentityAndDoesNotPublish) +{ + FlushController controller(publisher, runtime, sources, contiguous, lane); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + ASSERT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "value"))); + writer.release(); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(controller.prepare(reserve(), &flush).ok()); + auto first = stdexec::sync_wait(flush->capture_sender()); + if (!first) { + FAIL() << "flush sender stopped"; + } + ASSERT_TRUE(std::get<0>(*first).handoff_status.ok()); + const auto frozen = flush->frozen_sources(); + const auto version = publisher.capture()->commit_seq; + auto second = stdexec::sync_wait(flush->capture_sender()); + if (!second) { + FAIL() << "flush sender stopped"; + } + EXPECT_TRUE(std::get<0>(*second).handoff_status.ok()); + EXPECT_EQ(publisher.capture()->commit_seq, version); + EXPECT_EQ(publisher.capture()->coverage, 0U); + EXPECT_EQ(flush->frozen_sources().catalog, frozen.catalog); + EXPECT_EQ(flush->frozen_sources().base, frozen.base); + EXPECT_EQ(flush->frozen_sources().frontier, 1U); + EXPECT_EQ(flush->frozen_sources().batch_id, frozen.batch_id); +} + +struct ProgressReceiver +{ + using receiver_concept = stdexec::receiver_tag; + std::atomic<size_t> *count; + + void set_value() const noexcept + { + count->fetch_add(1, std::memory_order_release); + } + + void set_error(Status status) const noexcept + { + ADD_FAILURE() << status.to_string(); + } + + void set_stopped() const noexcept + { + ADD_FAILURE() << "unexpected stopped progress waiter"; + } +}; + +TEST_F(FlushReportTest, ProgressWakeupsDistinguishSuccessorCaptureAndHandoff) +{ + std::shared_ptr<ApplyProgress> progress; + ASSERT_TRUE(ApplyProgress::create(1, reserve(), &progress).ok()); + FlushController controller(publisher, runtime, sources, contiguous, lane, {}, {}, {}, progress); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + ASSERT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v"))); + std::shared_ptr<FlushOperation> flush; + ASSERT_TRUE(controller.prepare(reserve(), &flush).ok()); + std::atomic<size_t> successor_ready{0}; + ApplyProgress::Request successor(*progress, progress->sequence()); + auto successor_wait = stdexec::connect(successor.sender(), ProgressReceiver{&successor_ready}); + stdexec::start(successor_wait); + std::promise<FlushResult> terminal; + auto future = terminal.get_future(); + auto capture = stdexec::connect(flush->capture_sender(), ReportReceiver{&terminal}); + stdexec::start(capture); + runtime.drain(); + EXPECT_EQ(successor_ready.load(std::memory_order_acquire), 1U); + EXPECT_EQ(future.wait_for(std::chrono::seconds(0)), std::future_status::timeout); + EXPECT_EQ(publisher.capture()->coverage, 0U); + std::atomic<size_t> capture_ready{0}; + ApplyProgress::Request capture_done(*progress, progress->sequence()); + auto capture_wait = stdexec::connect(capture_done.sender(), ProgressReceiver{&capture_ready}); + stdexec::start(capture_wait); + writer.release(); + const auto captured = future.get(); + EXPECT_EQ(captured.phase, FlushPhase::kCaptured); + EXPECT_FALSE(captured.handoff_advanced); + EXPECT_EQ(capture_ready.load(std::memory_order_acquire), 1U); + std::atomic<size_t> handoff_ready{0}; + ApplyProgress::Request handoff_done(*progress, progress->sequence()); + auto handoff_wait = stdexec::connect(handoff_done.sender(), ProgressReceiver{&handoff_ready}); + stdexec::start(handoff_wait); + const auto completed = stdexec::sync_wait(flush->report_sender()); + if (!completed) { + FAIL() << "flush sender stopped"; + } + EXPECT_TRUE(std::get<0>(completed.value()).handoff_advanced); + EXPECT_EQ(handoff_ready.load(std::memory_order_acquire), 1U); + EXPECT_EQ(publisher.capture()->coverage, 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/flush_runtime_progress_test.cpp b/lib/crowdb-tree/tests/unit/flush_runtime_progress_test.cpp new file mode 100644 index 000000000..c255c3612 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/flush_runtime_progress_test.cpp @@ -0,0 +1,273 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_runtime.h" +#include "load_budget_test_support.h" + +#include <chrono> +#include <future> +#include <latch> +#include <thread> + +namespace crowdb::tree::detail +{ +namespace +{ +struct TestProgressGate +{ + std::latch entered{1}; + std::latch release{1}; + + static void wait(void *context) noexcept + { + auto &self = *static_cast<TestProgressGate *>(context); + self.entered.count_down(); + self.release.wait(); + } +}; + +struct TestProgressOwner : std::enable_shared_from_this<TestProgressOwner> +{ + explicit TestProgressOwner(std::atomic<size_t> &calls) : calls(calls) + { + } + + std::atomic<size_t> &calls; + std::shared_ptr<FlushRuntime::Progress> progress; + size_t remaining = 1; + + static void run(void *context) noexcept + { + auto &self = *static_cast<TestProgressOwner *>(context); + self.calls.fetch_add(1); + if (--self.remaining != 0) { + for (size_t i = 0; i < 100; ++i) { + self.progress->request(self.weak_from_this().lock()); + } + } + } +}; + +TEST(FlushRuntime, ReservedProgressConsumesLaneAndGlobalCapacityAndSurvivesFullQueueClose) +{ + FlushRuntime runtime(1, 2); + auto progress_lane = std::make_shared<FlushRuntime::Lane>(1); + auto ordinary_lane = std::make_shared<FlushRuntime::Lane>(2); + std::atomic<size_t> calls{0}; + std::atomic<size_t> ordinary{0}; + auto owner = std::make_shared<TestProgressOwner>(calls); + ASSERT_TRUE( + runtime.reserve_progress(progress_lane, &TestProgressOwner::run, owner.get(), {}, &owner->progress).ok()); + auto increment = [](void *context) noexcept { static_cast<std::atomic<size_t> *>(context)->fetch_add(1); }; + EXPECT_EQ(runtime.try_submit(progress_lane, increment, &ordinary).code(), Code::kResourceExhausted); + TestProgressGate gate; + ASSERT_TRUE(runtime.try_submit(ordinary_lane, &TestProgressGate::wait, &gate).ok()); + gate.entered.wait(); + const auto queued = runtime.try_submit(ordinary_lane, increment, &ordinary); + EXPECT_TRUE(queued.ok()); + EXPECT_EQ(runtime.try_submit(ordinary_lane, increment, &ordinary).code(), Code::kResourceExhausted); + runtime.close(); + for (size_t i = 0; i < 100; ++i) { + owner->progress->request(owner); + } + std::weak_ptr<TestProgressOwner> retained = owner; + owner.reset(); + EXPECT_FALSE(retained.expired()); + EXPECT_EQ(runtime.snapshot().queued, 2U); + gate.release.count_down(); + runtime.drain(); + EXPECT_TRUE(retained.expired()); + EXPECT_EQ(calls.load(), 1U); + EXPECT_EQ(ordinary.load(), 1U); + EXPECT_EQ(runtime.snapshot().completed, 3U); +} + +TEST(FlushRuntime, ReservedProgressRerunsUseSealedStorageAndDrainReleasesLastContextOwner) +{ + test_read::LoadCredits credits; + FlushRuntime runtime(1, 2, credits.credits.bytes); + auto lane = std::make_shared<FlushRuntime::Lane>(2); + const auto baseline = credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes]; + std::atomic<size_t> calls{0}; + auto owner = reserved_shared<TestProgressOwner>(credits.credits.bytes, calls); + owner->remaining = 1000; + ASSERT_TRUE( + runtime.reserve_progress(lane, &TestProgressOwner::run, owner.get(), credits.credits.bytes, &owner->progress) + .ok()); + std::weak_ptr<TestProgressOwner> retained = owner; + credits.seal(); + owner->progress->request(owner); + owner.reset(); + runtime.drain(); + EXPECT_EQ(calls.load(), 1000U); + EXPECT_TRUE(retained.expired()); + retained.reset(); + EXPECT_EQ(runtime.snapshot().submitted, 1000U); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], baseline); +} + +TEST(FlushRuntime, FailedProgressRegistrationReturnsSlotAndQueueCredits) +{ + test_read::LoadCredits credits; + FlushRuntime runtime(1, 1); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + std::shared_ptr<FlushRuntime::Progress> progress; + std::atomic<size_t> calls{0}; + auto increment = [](void *context) noexcept { static_cast<std::atomic<size_t> *>(context)->fetch_add(1); }; + credits.seal(); + EXPECT_EQ(runtime.reserve_progress(lane, increment, &calls, credits.credits.bytes, &progress).code(), + Code::kResourceExhausted); + EXPECT_FALSE(progress); + ASSERT_TRUE(runtime.try_submit(lane, increment, &calls).ok()); + runtime.drain(); + ASSERT_TRUE(runtime.reserve_progress(lane, increment, &calls, {}, &progress).ok()); + progress.reset(); + ASSERT_TRUE(runtime.try_submit(lane, increment, &calls).ok()); + runtime.drain(); + EXPECT_EQ(calls.load(), 2U); + runtime.close(); + EXPECT_EQ(runtime.reserve_progress(lane, increment, &calls, {}, &progress).code(), Code::kUnavailable); +} + +TEST(FlushRuntime, ClosedProgressOutlivesRuntimeWithoutWaitingForAnotherTree) +{ + auto runtime = std::make_unique<FlushRuntime>(2, 4); + auto lane = std::make_shared<FlushRuntime::Lane>(2); + auto other_lane = std::make_shared<FlushRuntime::Lane>(2); + std::atomic<size_t> calls{0}; + auto owner = std::make_shared<TestProgressOwner>(calls); + ASSERT_TRUE(runtime->reserve_progress(lane, &TestProgressOwner::run, owner.get(), {}, &owner->progress).ok()); + TestProgressGate other; + ASSERT_TRUE(runtime->try_submit(other_lane, &TestProgressGate::wait, &other).ok()); + other.entered.wait(); + owner->progress->request(owner); + auto closed = std::async(std::launch::async, [&] { owner->progress->close_and_drain(); }); + const auto result = closed.wait_for(std::chrono::seconds(5)); + other.release.count_down(); + EXPECT_EQ(result, std::future_status::ready); + closed.get(); + EXPECT_EQ(calls.load(), 1U); + runtime.reset(); + owner->progress->request(owner); + owner->progress->close_and_drain(); + EXPECT_EQ(calls.load(), 1U); +} + +struct TestCompletionBoundary +{ + std::latch entered{1}; + std::latch release{1}; + std::atomic<size_t> calls{0}; + + static void run(void *context) noexcept + { + auto &self = *static_cast<TestCompletionBoundary *>(context); + if (self.calls.fetch_add(1) == 0) { + self.entered.count_down(); + self.release.wait(); + } + } +}; + +TEST(FlushRuntime, RequestsBeforeAndAfterCompletionKeepTheirWake) +{ + FlushRuntime runtime(1, 1); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + auto owner = std::make_shared<TestCompletionBoundary>(); + std::shared_ptr<FlushRuntime::Progress> progress; + ASSERT_TRUE(runtime.reserve_progress(lane, &TestCompletionBoundary::run, owner.get(), {}, &progress).ok()); + progress->request(owner); + owner->entered.wait(); + for (size_t i = 0; i < 100; ++i) { + progress->request(owner); + } + owner->release.count_down(); + progress->drain(); + EXPECT_EQ(owner->calls.load(), 2U); + progress->request(owner); + progress->drain(); + EXPECT_EQ(owner->calls.load(), 3U); + progress->close_and_drain(); + progress->request(owner); + lane->drain(); + EXPECT_EQ(owner->calls.load(), 3U); +} + +TEST(FlushRuntime, CloseDrainsTheCoalescedRequestAcceptedBeforeCompletion) +{ + FlushRuntime runtime(1, 1); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + auto owner = std::make_shared<TestCompletionBoundary>(); + std::shared_ptr<FlushRuntime::Progress> progress; + ASSERT_TRUE(runtime.reserve_progress(lane, &TestCompletionBoundary::run, owner.get(), {}, &progress).ok()); + progress->request(owner); + owner->entered.wait(); + for (size_t i = 0; i < 100; ++i) { + progress->request(owner); + } + auto closed = std::async(std::launch::async, [&] { progress->close_and_drain(); }); + EXPECT_EQ(closed.wait_for(std::chrono::milliseconds(10)), std::future_status::timeout); + owner->release.count_down(); + closed.get(); + progress->request(owner); + lane->drain(); + EXPECT_EQ(owner->calls.load(), 2U); + EXPECT_EQ(runtime.snapshot().completed, 2U); +} + +TEST(FlushRuntime, ConcurrentHostRequestsDuringCompletionNeverBecomeIdleAndPending) +{ + struct Observed + { + std::atomic<size_t> requested{0}; + std::atomic<size_t> completed{0}; + + static void run(void *context) noexcept + { + auto &self = *static_cast<Observed *>(context); + self.completed.store(self.requested.load(std::memory_order_acquire), std::memory_order_release); + } + }; + + FlushRuntime runtime(1, 1); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + auto owner = std::make_shared<Observed>(); + std::shared_ptr<FlushRuntime::Progress> progress; + ASSERT_TRUE(runtime.reserve_progress(lane, &Observed::run, owner.get(), {}, &progress).ok()); + constexpr size_t requests = 10000; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + size_t acknowledged = 0; + for (size_t i = 1; i <= requests; ++i) { + owner->requested.store(i, std::memory_order_release); + progress->request(owner); + while (owner->completed.load(std::memory_order_acquire) < i && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + if (owner->completed.load(std::memory_order_acquire) != i) { + break; + } + acknowledged = i; + } + progress->close_and_drain(); + lane->drain(); + EXPECT_EQ(acknowledged, requests); + EXPECT_EQ(owner->completed.load(), requests); +} + +TEST(FlushRuntime, ClosingProgressWaitsForItsAcceptedTask) +{ + FlushRuntime runtime(1, 2); + auto lane = std::make_shared<FlushRuntime::Lane>(2); + auto gate = std::make_shared<TestProgressGate>(); + std::shared_ptr<FlushRuntime::Progress> progress; + ASSERT_TRUE(runtime.reserve_progress(lane, &TestProgressGate::wait, gate.get(), {}, &progress).ok()); + progress->request(gate); + gate->entered.wait(); + auto closed = std::async(std::launch::async, [&] { progress->close_and_drain(); }); + EXPECT_EQ(closed.wait_for(std::chrono::milliseconds(10)), std::future_status::timeout); + gate->release.count_down(); + closed.get(); + runtime.drain(); + EXPECT_EQ(runtime.snapshot().completed, 1U); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/flush_runtime_scheduler_test.cpp b/lib/crowdb-tree/tests/unit/flush_runtime_scheduler_test.cpp new file mode 100644 index 000000000..333922f48 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/flush_runtime_scheduler_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_runtime.h" +#include "load_budget_test_support.h" + +#include <array> +#include <thread> + +namespace crowdb::tree::detail +{ +namespace +{ +void count_task(void *context) noexcept +{ + static_cast<std::atomic<size_t> *>(context)->fetch_add(1, std::memory_order_relaxed); +} + +TEST(FlushRuntimeScheduler, ShortLivedSubmittersReuseAdmittedControlStorage) +{ + test_read::LoadCredits credits; + auto runtime = std::make_unique<FlushRuntime>(4, 16, credits.credits.bytes); + auto lane = std::make_shared<FlushRuntime::Lane>(16); + const auto allocated = credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes]; + std::atomic<size_t> completed{0}; + for (size_t batch = 0; batch < 8; ++batch) { + std::array<Status, 8> outcomes; + std::array<std::jthread, 8> submitters; + for (size_t index = 0; index < submitters.size(); ++index) { + submitters[index] = + std::jthread([&, index] { outcomes[index] = runtime->try_submit(lane, &count_task, &completed); }); + } + for (auto &thread : submitters) { + thread.join(); + } + lane->drain(); + for (const auto &status : outcomes) { + EXPECT_TRUE(status.ok()) << status.to_string(); + } + EXPECT_EQ(completed.load(), (batch + 1) * submitters.size()); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], allocated); + } + runtime.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], 0U); +} + +TEST(FlushRuntimeScheduler, ReservedProgressCanMoveAcrossFreshHostThreads) +{ + test_read::LoadCredits credits; + auto runtime = std::make_unique<FlushRuntime>(2, 1, credits.credits.bytes); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + auto completed = std::make_shared<std::atomic<size_t>>(0); + std::shared_ptr<FlushRuntime::Progress> progress; + ASSERT_TRUE(runtime->reserve_progress(lane, &count_task, completed.get(), credits.credits.bytes, &progress).ok()); + const auto allocated = credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes]; + for (size_t iteration = 0; iteration < 32; ++iteration) { + std::jthread submitter([&] { progress->request(completed); }); + submitter.join(); + lane->drain(); + EXPECT_EQ(completed->load(), iteration + 1); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], allocated); + } + progress->close_and_drain(); + progress.reset(); + runtime.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], 0U); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/flush_runtime_test.cpp b/lib/crowdb-tree/tests/unit/flush_runtime_test.cpp new file mode 100644 index 000000000..c302927b0 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/flush_runtime_test.cpp @@ -0,0 +1,348 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "btree/allocation_reservation.h" +#include "btree/flush_runtime.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <latch> +#include <optional> +#include <stdexcept> +#include <thread> +#include <vector> + +namespace crowdb::tree::detail +{ +namespace +{ +struct JoinResult +{ + std::atomic<size_t> values{0}; + std::atomic<size_t> errors{0}; + std::atomic<size_t> stopped{0}; +}; + +struct JoinReceiver +{ + using receiver_concept = stdexec::receiver_tag; + JoinResult *result; + + void set_value() const noexcept + { + result->values.fetch_add(1); + } + + void set_error(Status /*unused*/) const noexcept + { + result->errors.fetch_add(1); + } + + void set_stopped() const noexcept + { + result->stopped.fetch_add(1); + } +}; + +TEST(FlushRuntime, SingleWorkerCanSubmitNestedWorkWithOneQueueCredit) +{ + FlushRuntime runtime(1, 1); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + + struct Context + { + FlushRuntime *runtime; + std::shared_ptr<FlushRuntime::Lane> lane; + std::atomic<size_t> completed{0}; + std::atomic<bool> child_accepted{false}; + } context{.runtime = &runtime, .lane = lane}; + + auto parent = [](void *argument) noexcept { + auto &state = *static_cast<Context *>(argument); + auto child = [](void *inner) noexcept { static_cast<Context *>(inner)->completed.fetch_add(1); }; + state.child_accepted.store(state.runtime->try_submit(state.lane, child, &state).ok()); + }; + ASSERT_TRUE(runtime.try_submit(lane, parent, &context).ok()); + runtime.drain(); + EXPECT_TRUE(context.child_accepted.load()); + EXPECT_EQ(context.completed.load(), 1U); + auto stats = runtime.snapshot(); + EXPECT_EQ(stats.submitted, 2U); + EXPECT_EQ(stats.completed, 2U); + EXPECT_EQ(stats.queued, 0U); + EXPECT_EQ(stats.inflight, 0U); +} + +TEST(FlushRuntime, SealedJobStorageSupportsRepeatedNestedDispatchWithoutNewCredit) +{ + ResourceBudget::Limits limits{}; + const auto category = static_cast<size_t>(BudgetClass::kHandoff); + limits[category] = 65536; + ResourceBudget budget(limits, 4); + auto credit = ResourceBudget::try_reserve(budget.make_account(limits), BudgetClass::kHandoff, limits[category]); + if (!credit) { + FAIL() << "runtime storage credit rejected"; + } + auto reservation = std::make_shared<AllocationReservation>(std::move(credit.value())); + { + FlushRuntime runtime(1, 1, reservation); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + const auto allocated = budget.snapshot().allocated[category]; + EXPECT_GT(allocated, 0U); + reservation->seal(); + + struct Context + { + FlushRuntime &runtime; + std::shared_ptr<FlushRuntime::Lane> lane; + std::atomic<size_t> completed{0}; + std::atomic<size_t> failures{0}; + + static void step(void *argument) noexcept + { + auto &self = *static_cast<Context *>(argument); + if (self.completed.fetch_add(1) + 1 < 1000 && + !self.runtime.try_submit(self.lane, &Context::step, argument).ok()) { + self.failures.fetch_add(1); + } + } + } context{.runtime = runtime, .lane = lane}; + + ASSERT_TRUE(runtime.try_submit(lane, &Context::step, &context).ok()); + runtime.drain(); + EXPECT_EQ(context.completed.load(), 1000U); + EXPECT_EQ(context.failures.load(), 0U); + EXPECT_EQ(budget.snapshot().allocated[category], allocated); + EXPECT_EQ(budget.snapshot().reserved[category], 0U); + } + EXPECT_EQ(budget.snapshot().allocated[category], 0U); +} + +TEST(FlushRuntime, FailedStorageAdmissionReturnsAllClaimedCapacity) +{ + ResourceBudget::Limits limits{}; + const auto category = static_cast<size_t>(BudgetClass::kHandoff); + limits[category] = 4096; + ResourceBudget budget(limits, 4); + auto credit = ResourceBudget::try_reserve(budget.make_account(limits), BudgetClass::kHandoff, limits[category]); + if (!credit) { + FAIL() << "runtime storage credit rejected"; + } + auto reservation = std::make_shared<AllocationReservation>(std::move(credit.value())); + EXPECT_THROW((void)FlushRuntime(1, 1024, reservation), std::bad_alloc); + EXPECT_EQ(reservation->available(), limits[category]); + EXPECT_EQ(budget.snapshot().allocated[category], 0U); +} + +TEST(FlushRuntime, QueueAndLaneAreBoundedAndCloseDrainsAcceptedTasks) +{ + FlushRuntime runtime(1, 2); + auto lane_a = std::make_shared<FlushRuntime::Lane>(1); + auto lane_b = std::make_shared<FlushRuntime::Lane>(1); + + struct Gate + { + std::latch entered{1}; + std::latch release{1}; + } gate; + + auto blocker = [](void *context) noexcept { + auto &g = *static_cast<Gate *>(context); + g.entered.count_down(); + g.release.wait(); + }; + auto noop = [](void *) noexcept {}; + ASSERT_TRUE(runtime.try_submit(lane_a, blocker, &gate).ok()); + gate.entered.wait(); + EXPECT_TRUE(runtime.try_submit(lane_a, noop, nullptr).ok()); + EXPECT_EQ(runtime.try_submit(lane_a, noop, nullptr).code(), Code::kResourceExhausted); + EXPECT_TRUE(runtime.try_submit(lane_b, noop, nullptr).ok()); + EXPECT_EQ(runtime.try_submit(std::make_shared<FlushRuntime::Lane>(1), noop, nullptr).code(), + Code::kResourceExhausted); + runtime.close(); + EXPECT_EQ(runtime.try_submit(lane_b, noop, nullptr).code(), Code::kUnavailable); + gate.release.count_down(); + runtime.drain(); + EXPECT_EQ(runtime.snapshot().completed, 3U); + EXPECT_EQ(runtime.snapshot().rejected, 3U); +} + +TEST(FlushRuntime, LaneDrainIncludesProgressOwnerDestructionWithoutWaitingForAnotherTree) +{ + FlushRuntime runtime(2, 4); + auto lane = std::make_shared<FlushRuntime::Lane>(2); + auto other = std::make_shared<FlushRuntime::Lane>(2); + + struct Gate + { + std::latch entered{1}; + std::latch release{1}; + } unrelated, destruction; + + struct Owner + { + Gate *gate; + std::atomic<bool> *destroyed; + + ~Owner() + { + gate->entered.count_down(); + gate->release.wait(); + destroyed->store(true, std::memory_order_release); + } + }; + + ASSERT_TRUE(runtime + .try_submit( + other, + [](void *context) noexcept { + auto &gate = *static_cast<Gate *>(context); + gate.entered.count_down(); + gate.release.wait(); + }, + &unrelated) + .ok()); + unrelated.entered.wait(); + std::atomic<bool> destroyed{false}; + auto owner = std::make_shared<Owner>(&destruction, &destroyed); + std::shared_ptr<FlushRuntime::Progress> progress; + ASSERT_TRUE(runtime.reserve_progress(lane, [](void *) noexcept {}, nullptr, {}, &progress).ok()); + progress->request(std::move(owner)); + destruction.entered.wait(); + std::thread release([&] { destruction.release.count_down(); }); + lane->drain(); + EXPECT_TRUE(destroyed.load(std::memory_order_acquire)); + EXPECT_EQ(runtime.snapshot().inflight, 1U); + release.join(); + progress->close_and_drain(); + progress.reset(); + unrelated.release.count_down(); + other->drain(); +} + +TEST(BoundedRangeJoin, InlineChildrenReuseSlotsWithoutRecursiveStackGrowth) +{ + size_t count = 0; + BoundedRangeJoin join(10000, 1, [&](size_t index, BoundedRangeJoin::Completion complete) { + EXPECT_EQ(index, count++); + complete(); + complete(Status::io_error("duplicate")); + }); + JoinResult result; + auto operation = stdexec::connect(join.sender(), JoinReceiver{&result}); + stdexec::start(operation); + EXPECT_EQ(count, 10000U); + EXPECT_EQ(result.values.load(), 1U); + EXPECT_EQ(result.errors.load(), 0U); +} + +TEST(BoundedRangeJoin, FirstErrorStopsDispatchAndWaitsForEveryStartedChild) +{ + std::vector<BoundedRangeJoin::Completion> children; + BoundedRangeJoin join(8, 2, [&](size_t, BoundedRangeJoin::Completion complete) { children.push_back(complete); }); + JoinResult result; + auto operation = stdexec::connect(join.sender(), JoinReceiver{&result}); + stdexec::start(operation); + ASSERT_EQ(children.size(), 2U); + children[1](Status::io_error("injected")); + EXPECT_EQ(result.errors.load(), 0U); + children[0](); + EXPECT_EQ(children.size(), 2U); + EXPECT_EQ(result.errors.load(), 1U); + children[1](); + EXPECT_EQ(result.errors.load(), 1U); +} + +TEST(BoundedRangeJoin, StaleSlotCompletionCannotCompleteItsReplacement) +{ + std::vector<BoundedRangeJoin::Completion> children; + BoundedRangeJoin join(2, 1, [&](size_t, BoundedRangeJoin::Completion complete) { children.push_back(complete); }); + JoinResult result; + auto operation = stdexec::connect(join.sender(), JoinReceiver{&result}); + stdexec::start(operation); + // Copy before completing: the starter appends and may relocate the vector. + auto first = children[0]; + first(); + ASSERT_EQ(children.size(), 2U); + first(Status::io_error("late duplicate")); + EXPECT_EQ(result.values.load(), 0U); + children[1](); + EXPECT_EQ(result.values.load(), 1U); + EXPECT_EQ(result.errors.load(), 0U); +} + +TEST(BoundedRangeJoin, StopDrainsLateCallbacksAndPreStartStopStartsNoChildren) +{ + std::optional<BoundedRangeJoin::Completion> child; + BoundedRangeJoin join(10, 1, [&](size_t, BoundedRangeJoin::Completion complete) { child = complete; }); + JoinResult result; + auto operation = stdexec::connect(join.sender(), JoinReceiver{&result}); + stdexec::start(operation); + join.request_stop(); + EXPECT_EQ(result.stopped.load(), 0U); + if (!child.has_value()) { + FAIL() << "range was not dispatched"; + } + child.value()(); + EXPECT_EQ(result.stopped.load(), 1U); + + BoundedRangeJoin stopped(10, 1, [](size_t, BoundedRangeJoin::Completion) { ADD_FAILURE(); }); + stopped.request_stop(); + JoinResult stopped_result; + auto stopped_operation = stdexec::connect(stopped.sender(), JoinReceiver{&stopped_result}); + stdexec::start(stopped_operation); + EXPECT_EQ(stopped_result.stopped.load(), 1U); +} + +TEST(BoundedRangeJoin, EmptyAndThrowingRangesHaveOneTerminalSignal) +{ + BoundedRangeJoin empty(0, 1, [](size_t, BoundedRangeJoin::Completion) { ADD_FAILURE(); }); + JoinResult result; + auto operation = stdexec::connect(empty.sender(), JoinReceiver{&result}); + stdexec::start(operation); + EXPECT_EQ(result.values.load(), 1U); + + BoundedRangeJoin throwing(5, 1, [](size_t, BoundedRangeJoin::Completion) { throw std::runtime_error("injected"); }); + JoinResult failure; + auto failed = stdexec::connect(throwing.sender(), JoinReceiver{&failure}); + stdexec::start(failed); + EXPECT_EQ(failure.errors.load(), 1U); +} + +TEST(BoundedRangeJoin, ReusableSlotsRunOnSharedSingleWorkerWithoutWaiting) +{ + FlushRuntime runtime(1, 4); + auto lane = std::make_shared<FlushRuntime::Lane>(4); + std::atomic<size_t> processed{0}; + + struct Child + { + BoundedRangeJoin::Completion complete; + std::atomic<size_t> *processed; + }; + + BoundedRangeJoin join(100, 2, [&](size_t, BoundedRangeJoin::Completion complete) { + auto *child = new Child{.complete = complete, .processed = &processed}; + auto run = [](void *context) noexcept { + std::unique_ptr<Child> current(static_cast<Child *>(context)); + current->processed->fetch_add(1); + current->complete(); + }; + auto status = runtime.try_submit(lane, run, child); + if (!status.ok()) { + delete child; + complete(std::move(status)); + } + }); + JoinResult result; + auto operation = stdexec::connect(join.sender(), JoinReceiver{&result}); + stdexec::start(operation); + runtime.drain(); + EXPECT_EQ(processed.load(), 100U); + EXPECT_EQ(result.values.load(), 1U); + EXPECT_EQ(result.errors.load(), 0U); +} + +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/flush_runtime_usage_test.cpp b/lib/crowdb-tree/tests/unit/flush_runtime_usage_test.cpp new file mode 100644 index 000000000..354c36b56 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/flush_runtime_usage_test.cpp @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_runtime.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +namespace +{ +struct TestCleanupOwner +{ + TestCleanupOwner(std::atomic<bool> &entered, std::atomic<bool> &released) : entered(entered), released(released) + { + } + + ~TestCleanupOwner() + { + entered.store(true, std::memory_order_release); + entered.notify_one(); + released.wait(false, std::memory_order_acquire); + } + + std::atomic<bool> &entered; + std::atomic<bool> &released; +}; + +TEST(FlushRuntimeUsage, WorkerRemainsOccupiedUntilItsContextOwnerCleanupCompletes) +{ + FlushRuntime runtime(1, 8); + auto lane = std::make_shared<FlushRuntime::Lane>(8); + std::shared_ptr<FlushRuntime::Progress> progress; + std::atomic<bool> published{false}; + ASSERT_TRUE(runtime + .reserve_progress( + lane, + [](void *context) noexcept { + static_cast<std::atomic<bool> *>(context)->wait(false, std::memory_order_acquire); + }, + &published, {}, &progress) + .ok()); + std::atomic<bool> entered{false}; + std::atomic<bool> released{false}; + auto owner = std::make_shared<TestCleanupOwner>(entered, released); + progress->request(owner); + owner.reset(); + published.store(true, std::memory_order_release); + published.notify_one(); + entered.wait(false, std::memory_order_acquire); + const auto during = runtime.snapshot(); + EXPECT_EQ(during.inflight, 1U); + EXPECT_EQ(during.completed, 0U); + released.store(true, std::memory_order_release); + released.notify_one(); + progress->close_and_drain(); + runtime.drain(); + EXPECT_EQ(runtime.snapshot().inflight, 0U); + EXPECT_EQ(runtime.snapshot().completed, 1U); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/frame_page_test.cpp b/lib/crowdb-tree/tests/unit/frame_page_test.cpp index a63e54bf5..59ea6cdbc 100644 --- a/lib/crowdb-tree/tests/unit/frame_page_test.cpp +++ b/lib/crowdb-tree/tests/unit/frame_page_test.cpp @@ -65,6 +65,25 @@ TEST(FramePage, LeafBuildViewRoundTrip) EXPECT_EQ(frame_upper_fence(frame.data()).to_string(), "c"); } +TEST(FramePage, NullBackedEmptyKeysAndSeparatorsRemainValid) +{ + constexpr uint32_t bytes = 4096; + std::vector<uint8_t> frame(bytes); + const auto cell = make_cell(1, ""); + LeafFrameBuilder builder(frame.data(), bytes); + ASSERT_TRUE(builder.try_append_sorted(Slice{}, Slice(cell))); + builder.finish(1, kInvalidPageId); + ASSERT_TRUE(frame_validate(frame.data(), bytes)); + const LeafFrameView leaf(frame.data(), bytes); + ASSERT_EQ(leaf.count(), 1U); + EXPECT_TRUE(leaf.key(0).empty()); + EXPECT_TRUE(CellView{leaf.cell(0)}.value().empty()); + + ASSERT_TRUE(inner_frame_build(frame.data(), bytes, 3, {1, 2}, {Slice{}})); + ASSERT_TRUE(frame_validate(frame.data(), bytes)); + EXPECT_TRUE(InnerFrameView(frame.data(), bytes).separator_at(0).empty()); +} + TEST(FramePage, LeafFindAndLowerBound) { const uint32_t pb = 4096; diff --git a/lib/crowdb-tree/tests/unit/frame_summary_test.cpp b/lib/crowdb-tree/tests/unit/frame_summary_test.cpp new file mode 100644 index 000000000..9e1e1b5cd --- /dev/null +++ b/lib/crowdb-tree/tests/unit/frame_summary_test.cpp @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/maptable/frame_page.h" + +#include <gtest/gtest.h> + +#include <array> + +using namespace crowdb::tree; + +TEST(FrameSummary, ReadsExistingVersionOneCapacityWithoutExactFlagByte) +{ + std::array<uint8_t, 4096> frame{}; + LeafFrameBuilder builder(frame.data(), frame.size()); + builder.finish(7, kInvalidPageId); + // These are the bytes produced by the original version-one writer: + // the last byte of the capacity word is overwritten with the exact flag. + frame_put_u64(frame.data(), 64, 3); + frame_put_u64(frame.data(), 120, 4096); + frame[127] = 1; + frame_restamp_crc(frame.data(), frame.size()); + ASSERT_TRUE(frame_validate(frame.data(), frame.size())); + const auto summary = frame_summary(frame.data()); + EXPECT_TRUE(summary.exact); + EXPECT_EQ(summary.live_kv, 3U); + EXPECT_EQ(summary.reachable_page_capacity_bytes, 4096U); +} + +TEST(FrameSummary, CapacityRoundTripsThroughEveryRepresentableByte) +{ + std::array<uint8_t, 4096> frame{}; + LeafFrameBuilder builder(frame.data(), frame.size()); + builder.finish(7, kInvalidPageId); + for (const uint64_t capacity : + {uint64_t{0}, uint64_t{4096}, uint64_t{0x00f1e2d3c4b5a697}, uint64_t{0x00ffffffffffffff}}) { + const PageSummary input{.live_kv = 2, + .live_key_bytes = 5, + .live_value_bytes = 8, + .live_logical_bytes = 13, + .reachable_leaf_pages = 1, + .reachable_inner_pages = 2, + .reachable_overflow_pages = 3, + .reachable_page_capacity_bytes = capacity, + .exact = true}; + frame_set_summary(frame.data(), frame.size(), input); + ASSERT_TRUE(frame_validate(frame.data(), frame.size())); + const auto output = frame_summary(frame.data()); + EXPECT_TRUE(output.exact); + EXPECT_EQ(output.live_kv, input.live_kv); + EXPECT_EQ(output.live_key_bytes, input.live_key_bytes); + EXPECT_EQ(output.live_value_bytes, input.live_value_bytes); + EXPECT_EQ(output.live_logical_bytes, input.live_logical_bytes); + EXPECT_EQ(output.reachable_leaf_pages, input.reachable_leaf_pages); + EXPECT_EQ(output.reachable_inner_pages, input.reachable_inner_pages); + EXPECT_EQ(output.reachable_overflow_pages, input.reachable_overflow_pages); + EXPECT_EQ(output.reachable_page_capacity_bytes, capacity); + } +} + +TEST(FrameSummary, UnrepresentableCapacityAndUnknownFlagsCannotClaimExactCounts) +{ + std::array<uint8_t, 4096> frame{}; + LeafFrameBuilder builder(frame.data(), frame.size()); + builder.finish(7, kInvalidPageId); + for (const uint64_t capacity : {uint64_t{1} << 56, UINT64_MAX}) { + frame_set_summary(frame.data(), frame.size(), + {.live_kv = 5, .reachable_page_capacity_bytes = capacity, .exact = true}); + EXPECT_TRUE(frame_validate(frame.data(), frame.size())); + EXPECT_FALSE(frame_summary(frame.data()).exact); + } + frame_set_summary(frame.data(), frame.size(), + {.live_kv = 5, .reachable_page_capacity_bytes = 4096, .exact = false}); + EXPECT_FALSE(frame_summary(frame.data()).exact); + for (const uint8_t flag : {2, 3, 255}) { + frame[127] = flag; + frame_restamp_crc(frame.data(), frame.size()); + EXPECT_TRUE(frame_validate(frame.data(), frame.size())); + EXPECT_FALSE(frame_summary(frame.data()).exact); + } +} diff --git a/lib/crowdb-tree/tests/unit/gc_test.cpp b/lib/crowdb-tree/tests/unit/gc_test.cpp index 370171558..25f972acc 100644 --- a/lib/crowdb-tree/tests/unit/gc_test.cpp +++ b/lib/crowdb-tree/tests/unit/gc_test.cpp @@ -4,9 +4,11 @@ // R129: tombstone folding is part of snapshot preparation; the standalone // resident-tree GC sweep is removed. These tests verify the watermark // contract and that snapshot folding drops eligible tombstones. +#include "backend/local/local_backend_state.h" #include "crowdb-tree/backend/block_page_store.h" #include "crowdb-tree/backend/page_store.h" #include "crowdb-tree/crowdb-tree.h" +#include "runtime_tree_access.h" #include "test_tmp.h" #include <gtest/gtest.h> @@ -28,9 +30,9 @@ Batch del_one(const std::string &k) return Batch{{{.key = k, .kind = OpKind::kDelete, .value = ""}}}; } -page_type head_type(Crowdbtree &t) +bool has_overlay(Crowdbtree &tree) { - return t.mapping().get_resident(t.root_page_id())->type; + return RuntimeTreeAccess_for_tests::root(tree)->metrics.overlay_runs != 0; } } // namespace @@ -72,10 +74,10 @@ TEST(Gc, SnapshotFoldingBelowWatermarkPreservesTombstone) // Now raise the watermark past the tombstone's slot and snapshot again. // This second snapshot must fold the tombstone (it's now eligible). t.set_gc_watermark(2, 2); - PageBase *before = t.mapping().get_resident(t.root_page_id()); + auto before = RuntimeTreeAccess_for_tests::root(t); ASSERT_TRUE(t.snapshot().ok()); // The leaf was rebuilt (eligible tombstone folded). - EXPECT_NE(t.mapping().get_resident(t.root_page_id()), before); + EXPECT_NE(RuntimeTreeAccess_for_tests::root(t), before); EXPECT_FALSE(t.get(Slice("a"), &slot, &v)); } @@ -99,15 +101,16 @@ TEST(Gc, SnapshotFoldsEligibleTombstoneInCleanLeaf) // clean LeafBase with no BatchDelta chain on top. ASSERT_TRUE(t.apply(5, del_one("k1")).ok()); ASSERT_TRUE(t.flush().ok()); - ASSERT_EQ(head_type(t), page_type::kLeafBase); + ASSERT_TRUE(t.snapshot().ok()); + ASSERT_FALSE(has_overlay(t)); // Advance the watermark past the delete's slot, then snapshot to fold. t.set_gc_watermark(5, 5); - PageBase *before = t.mapping().get_resident(t.root_page_id()); + auto before = RuntimeTreeAccess_for_tests::root(t); ASSERT_TRUE(t.snapshot().ok()); // The leaf was rebuilt (eligible tombstone folded). - EXPECT_NE(t.mapping().get_resident(t.root_page_id()), before); - ASSERT_EQ(head_type(t), page_type::kLeafBase); + EXPECT_NE(RuntimeTreeAccess_for_tests::root(t), before); + ASSERT_FALSE(has_overlay(t)); // Delete still honored (not resurrected); other keys unaffected. std::string v; @@ -135,14 +138,14 @@ TEST(Gc, SnapshotFoldingReclaimsTombstoneAfterOpen) } ASSERT_TRUE(t->apply(5, del_one("k1")).ok()); ASSERT_TRUE(t->flush().ok()); - ASSERT_EQ(head_type(*t), page_type::kLeafBase); - PageBase *before = t->mapping().get_resident(t->root_page_id()); + ASSERT_TRUE(t->snapshot().ok()); + ASSERT_FALSE(has_overlay(*t)); + auto before = RuntimeTreeAccess_for_tests::root(*t); t->set_gc_watermark(5, 5); ASSERT_TRUE(t->snapshot().ok()); - EXPECT_NE(t->mapping().get_resident(t->root_page_id()), before) - << "snapshot folding never swept the stale tombstone"; + EXPECT_NE(RuntimeTreeAccess_for_tests::root(*t), before) << "snapshot folding never swept the stale tombstone"; std::string v; uint64_t slot; @@ -162,7 +165,8 @@ TEST(Gc, CompactSparseBlocksNoOpOnMemStore) ASSERT_TRUE(t.snapshot().ok()); MergeGcStats stats; - ASSERT_TRUE(t.compact_sparse_blocks(&stats).ok()); + const auto status = t.compact_sparse_blocks(&stats); + ASSERT_TRUE(status.ok()) << status.to_string(); EXPECT_EQ(stats.blocks_selected, 0U); EXPECT_EQ(stats.pages_relocated, 0U); EXPECT_EQ(stats.bytes_relocated, 0U); @@ -207,6 +211,9 @@ TEST(Gc, CompactSparseBlocksProtectsAnchorBlock) opt.merge_gc_block_free_threshold = 0.01; { Crowdbtree t(opt); + // This fixture drives relocation explicitly instead of racing checkpoint cleanup. + auto local = RuntimeTreeAccess_for_tests::core(t).local_backend(); + local->gc_active.store(true); for (int i = 0; i < 50; ++i) { ASSERT_TRUE(t.apply(i + 1, put_one("k" + std::to_string(i), "v")).ok()); } @@ -222,7 +229,9 @@ TEST(Gc, CompactSparseBlocksProtectsAnchorBlock) ASSERT_TRUE(t.snapshot().ok()); // Run compaction — must not touch block 0 (anchor). MergeGcStats stats; - ASSERT_TRUE(t.compact_sparse_blocks(&stats).ok()); + local->gc_active.store(false); + const auto status = t.compact_sparse_blocks(&stats); + ASSERT_TRUE(status.ok()) << status.to_string(); EXPECT_GE(stats.blocks_selected, 0U); // Surviving keys readable. for (int i = 0; i < 50; i += 10) { @@ -266,6 +275,9 @@ TEST(Gc, CompactSparseBlocksRespectsByteBudget) // Very small budget: only the first block should be selected. opt.merge_gc_max_relocation_bytes = blk / 4; Crowdbtree t(opt); + // This fixture drives relocation explicitly instead of racing checkpoint cleanup. + auto local = RuntimeTreeAccess_for_tests::core(t).local_backend(); + local->gc_active.store(true); // Write enough data to span multiple blocks. uint64_t slot = 0; @@ -273,8 +285,10 @@ TEST(Gc, CompactSparseBlocksRespectsByteBudget) ++slot; ASSERT_TRUE(t.apply(slot, put_one("k" + std::to_string(i), std::string(128, 'x'))).ok()); } - ASSERT_TRUE(t.flush().ok()); - ASSERT_TRUE(t.snapshot().ok()); + const auto flush_1 = t.flush(); + ASSERT_TRUE(flush_1.ok()) << flush_1.to_string(); + const auto snapshot_1 = t.snapshot(); + ASSERT_TRUE(snapshot_1.ok()) << snapshot_1.to_string(); // Delete most keys to make blocks sparse. for (int i = 0; i < 200; ++i) { @@ -283,11 +297,19 @@ TEST(Gc, CompactSparseBlocksRespectsByteBudget) ASSERT_TRUE(t.apply(slot, del_one("k" + std::to_string(i))).ok()); } } - ASSERT_TRUE(t.flush().ok()); - ASSERT_TRUE(t.snapshot().ok()); + const auto flush_2 = t.flush(); + ASSERT_TRUE(flush_2.ok()) << flush_2.to_string(); + const auto snapshot_2 = t.snapshot(); + ASSERT_TRUE(snapshot_2.ok()) << snapshot_2.to_string(); + // Both recovery anchors must describe the deleted-key view before the + // superseded checkpoint's extents can become sparse GC candidates. + const auto retired_anchor = t.snapshot(); + ASSERT_TRUE(retired_anchor.ok()) << retired_anchor.to_string(); MergeGcStats stats; - ASSERT_TRUE(t.compact_sparse_blocks(&stats).ok()); + local->gc_active.store(false); + const auto status = t.compact_sparse_blocks(&stats); + ASSERT_TRUE(status.ok()) << status.to_string(); // At least one block selected (the first eligible is always allowed). EXPECT_GE(stats.blocks_selected, 1U); // Surviving keys are still readable. @@ -319,14 +341,19 @@ TEST(Gc, CompactSparseBlocksMaintainsDataIntegrity) opt.merge_gc_block_free_threshold = 0.30; opt.merge_gc_max_relocation_bytes = 8 * 1024 * 1024; Crowdbtree t(opt); + // This fixture drives relocation explicitly instead of racing checkpoint cleanup. + auto local = RuntimeTreeAccess_for_tests::core(t).local_backend(); + local->gc_active.store(true); uint64_t slot = 0; for (int i = 0; i < 200; ++i) { ++slot; ASSERT_TRUE(t.apply(slot, put_one("k" + std::to_string(i), std::string(128, 'x'))).ok()); } - ASSERT_TRUE(t.flush().ok()); - ASSERT_TRUE(t.snapshot().ok()); + const auto flush_1 = t.flush(); + ASSERT_TRUE(flush_1.ok()) << flush_1.to_string(); + const auto snapshot_1 = t.snapshot(); + ASSERT_TRUE(snapshot_1.ok()) << snapshot_1.to_string(); // Delete most keys to create sparse blocks. for (int i = 0; i < 200; ++i) { @@ -335,19 +362,30 @@ TEST(Gc, CompactSparseBlocksMaintainsDataIntegrity) ASSERT_TRUE(t.apply(slot, del_one("k" + std::to_string(i))).ok()); } } - ASSERT_TRUE(t.flush().ok()); - ASSERT_TRUE(t.snapshot().ok()); + const auto flush_2 = t.flush(); + ASSERT_TRUE(flush_2.ok()) << flush_2.to_string(); + const auto snapshot_2 = t.snapshot(); + ASSERT_TRUE(snapshot_2.ok()) << snapshot_2.to_string(); + // Both recovery anchors must describe the deleted-key view before the + // superseded checkpoint's extents can become sparse GC candidates. + const auto retired_anchor = t.snapshot(); + ASSERT_TRUE(retired_anchor.ok()) << retired_anchor.to_string(); // Run compaction. MergeGcStats stats; - ASSERT_TRUE(t.compact_sparse_blocks(&stats).ok()); + local->gc_active.store(false); + const auto status = t.compact_sparse_blocks(&stats); + ASSERT_TRUE(status.ok()) << status.to_string(); // Some blocks should be selected and relocated. EXPECT_GE(stats.blocks_selected, 1U); + local->gc_active.store(true); ++slot; ASSERT_TRUE(t.apply(slot, put_one("after-compaction", "still-writable")).ok()); - ASSERT_TRUE(t.flush().ok()); - ASSERT_TRUE(t.snapshot().ok()); + const auto flush_3 = t.flush(); + ASSERT_TRUE(flush_3.ok()) << flush_3.to_string(); + const auto snapshot_3 = t.snapshot(); + ASSERT_TRUE(snapshot_3.ok()) << snapshot_3.to_string(); // All surviving keys must be readable after compaction. for (int i = 0; i < 200; i += 20) { @@ -364,7 +402,9 @@ TEST(Gc, CompactSparseBlocksMaintainsDataIntegrity) // A second compaction pass should be idempotent (no more sparse blocks // or at least no data loss). MergeGcStats stats2; - ASSERT_TRUE(t.compact_sparse_blocks(&stats2).ok()); + local->gc_active.store(false); + const auto repeated_compaction = t.compact_sparse_blocks(&stats2); + ASSERT_TRUE(repeated_compaction.ok()) << repeated_compaction.to_string(); (void)stats2; for (int i = 0; i < 200; i += 20) { std::string v; diff --git a/lib/crowdb-tree/tests/unit/graph_summary_test.cpp b/lib/crowdb-tree/tests/unit/graph_summary_test.cpp new file mode 100644 index 000000000..096cb85c5 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/graph_summary_test.cpp @@ -0,0 +1,50 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/graph_summary.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(GraphSummary, CountsEachValidatedGraphFrameAndItsFullCapacity) +{ + TreeSummary summary; + ReadablePageFrame leaf{.type = page_type::kLeafBase, .frame_bytes = 4096, .backing = {}}; + leaf.summary.emplace(); + leaf.summary->live_counts_known = true; + leaf.summary->live_keys = 3; + leaf.summary->live_key_bytes = 9; + leaf.summary->live_value_bytes = 10000; + ASSERT_TRUE(add_graph_summary(summary, leaf)); + ASSERT_TRUE(add_graph_summary(summary, {.type = page_type::kInnerBase, .frame_bytes = 2048, .backing = {}})); + ASSERT_TRUE(add_graph_summary(summary, {.type = page_type::kOverflowFrame, .frame_bytes = 4096, .backing = {}})); + ASSERT_TRUE(add_graph_summary(summary, {.type = page_type::kOverflowFrame, .frame_bytes = 4096, .backing = {}})); + EXPECT_EQ(summary.live_kv, 3U); + EXPECT_EQ(summary.live_key_bytes, 9U); + EXPECT_EQ(summary.live_value_bytes, 10000U); + EXPECT_EQ(summary.live_logical_bytes, 10009U); + EXPECT_EQ(summary.reachable_leaf_pages, 1U); + EXPECT_EQ(summary.reachable_inner_pages, 1U); + EXPECT_EQ(summary.reachable_overflow_pages, 2U); + EXPECT_EQ(summary.reachable_page_capacity_bytes, 14336U); +} + +TEST(GraphSummary, UnknownOrOverflowingReceiptLeavesEveryCounterUnchanged) +{ + TreeSummary summary; + summary.live_value_bytes = UINT64_MAX; + ReadablePageFrame leaf{.type = page_type::kLeafBase, .frame_bytes = 4096, .backing = {}}; + EXPECT_FALSE(add_graph_summary(summary, leaf)); + leaf.summary.emplace(); + leaf.summary->live_counts_known = true; + leaf.summary->live_keys = 1; + leaf.summary->live_key_bytes = 4; + leaf.summary->live_value_bytes = 1; + EXPECT_FALSE(add_graph_summary(summary, leaf)); + EXPECT_EQ(summary.live_value_bytes, UINT64_MAX); + EXPECT_EQ(summary.live_kv, 0U); + EXPECT_EQ(summary.live_key_bytes, 0U); + EXPECT_EQ(summary.reachable_page_capacity_bytes, 0U); + EXPECT_EQ(summary.reachable_leaf_pages, 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/host_completion_test.cpp b/lib/crowdb-tree/tests/unit/host_completion_test.cpp new file mode 100644 index 000000000..66b301789 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/host_completion_test.cpp @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "host_completion.h" + +#include <gtest/gtest.h> + +#include <memory> +#include <thread> + +using crowdb::tree::detail::HostCompletion; + +TEST(HostCompletion, SignalBeforeWaitPublishesTheResult) +{ + HostCompletion completion; + completion.signal(); + completion.wait(); +} + +TEST(HostCompletion, WaiterCanReleaseStorageBeforeJoiningTheNotifier) +{ + for (size_t attempt = 0; attempt < 256; ++attempt) { + auto completion = std::make_unique<HostCompletion>(); + int result = 0; + std::thread notifier([signal = completion.get(), &result] { + result = 42; + signal->signal(); + }); + completion->wait(); + EXPECT_EQ(result, 42); + completion.reset(); + notifier.join(); + } +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(HostCompletion, ReadyWaiterStillRetainsStorageDuringNotification) +{ + struct TestNotification + { + std::atomic<bool> entered{false}; + std::atomic<bool> release{false}; + } notification; + + HostCompletion completion; + std::atomic<bool> waiting{false}; + std::atomic<bool> returned{false}; + std::thread notifier([&] { + completion.signal_for_tests( + [](void *context) { + auto &state = *static_cast<TestNotification *>(context); + state.entered.store(true, std::memory_order_release); + state.entered.notify_one(); + state.release.wait(false, std::memory_order_acquire); + }, + ¬ification); + }); + notification.entered.wait(false, std::memory_order_acquire); + std::thread waiter([&] { + waiting.store(true, std::memory_order_release); + waiting.notify_one(); + completion.wait(); + returned.store(true, std::memory_order_release); + }); + waiting.wait(false, std::memory_order_acquire); + for (size_t attempt = 0; attempt < 1'000; ++attempt) { + std::this_thread::yield(); + } + EXPECT_FALSE(returned.load(std::memory_order_acquire)); + notification.release.store(true, std::memory_order_release); + notification.release.notify_one(); + waiter.join(); + notifier.join(); + EXPECT_TRUE(returned.load(std::memory_order_acquire)); +} +#endif diff --git a/lib/crowdb-tree/tests/unit/immutable_snapshot_c_api_test.cpp b/lib/crowdb-tree/tests/unit/immutable_snapshot_c_api_test.cpp new file mode 100644 index 000000000..706170566 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/immutable_snapshot_c_api_test.cpp @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/immutable_snapshot_c_api.h" +#include "runtime/tree_runtime_c_api.h" + +#include <gtest/gtest.h> + +#include <utility> + +namespace crowdb::tree +{ +namespace +{ +struct TestSnapshotTree +{ + ct_runtime *runtime = nullptr; + ct_tree *tree = nullptr; + ct_immutable_snapshot *snapshot = nullptr; + + TestSnapshotTree() + { + EXPECT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + EXPECT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + } + + ~TestSnapshotTree() + { + ct_immutable_snapshot_free(snapshot); + ct_close(tree); + ct_runtime_free(runtime); + } + + void put(uint64_t slot, const std::string &key, const std::string &value) const + { + ASSERT_EQ(ct_apply_put(tree, slot, reinterpret_cast<const uint8_t *>(key.data()), key.size(), + reinterpret_cast<const uint8_t *>(value.data()), value.size()), + 0); + } +}; +} // namespace + +TEST(ImmutableSnapshotCAbi, FixedViewAndResultOwnersSurviveTreeAndRuntimeClose) +{ + TestSnapshotTree test; + test.put(1, "", "empty"); + test.put(2, "key", "old"); + ASSERT_EQ(ct_flush(test.tree), 0); + ASSERT_EQ(ct_immutable_snapshot_capture(test.tree, 0, 2, &test.snapshot), 0); + test.put(3, "key", "new"); + ASSERT_EQ(ct_flush(test.tree), 0); + ct_close(std::exchange(test.tree, nullptr)); + ct_runtime_free(std::exchange(test.runtime, nullptr)); + + ct_future *get = nullptr; + const uint8_t key[] = {'k', 'e', 'y'}; + ASSERT_EQ(ct_immutable_snapshot_get(test.snapshot, key, sizeof(key), &get), 0); + int32_t done = 0; + int32_t found = 0; + uint64_t slot = 0; + ct_buf bytes{}; + ASSERT_EQ(ct_future_poll(get, &done, &found, &slot, &bytes), 0); + ASSERT_EQ(done, 1); + ASSERT_EQ(found, 1); + EXPECT_EQ(slot, 2U); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(bytes.data), bytes.len), "old"); + ct_scan_options_v2 scan{}; + scan.struct_size = sizeof(scan); + scan.has_start_bound = 1; + scan.start_inclusive = 1; + scan.limit = 1; + ct_scan_result *result = nullptr; + ASSERT_EQ(ct_immutable_snapshot_scan(test.snapshot, &scan, &result), 0); + ct_immutable_snapshot_free(std::exchange(test.snapshot, nullptr)); + const uint8_t *packed = nullptr; + size_t len = 0; + uint64_t count = 0; + int32_t truncated = 0; + ASSERT_EQ(ct_scan_result_data(result, &packed, &len, &count, &truncated), 0); + EXPECT_EQ(count, 1U); + EXPECT_EQ(truncated, 1); + ASSERT_GE(len, 5U); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(packed + len - 5), 5), "empty"); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(bytes.data), bytes.len), "old"); + ct_scan_result_free(result); + ct_future_free(get); +} + +TEST(ImmutableSnapshotCAbi, CaptureRejectsWrongCoverageAndStrictScanTruncatesOversizedFirstEntry) +{ + TestSnapshotTree test; + test.put(1, "key", "value"); + ASSERT_EQ(ct_flush(test.tree), 0); + EXPECT_EQ(ct_immutable_snapshot_capture(test.tree, 0, 2, &test.snapshot), + static_cast<ct_status>(Code::kUnavailable)); + EXPECT_EQ(test.snapshot, nullptr); + ASSERT_EQ(ct_immutable_snapshot_capture(test.tree, 0, 1, &test.snapshot), 0); + ct_scan_options_v2 scan{}; + scan.struct_size = sizeof(scan); + scan.strict_byte_budget = 1; + scan.byte_budget = 1; + ct_scan_result *result = nullptr; + ASSERT_EQ(ct_immutable_snapshot_scan(test.snapshot, &scan, &result), 0); + const uint8_t *packed = nullptr; + size_t len = 0; + uint64_t count = 0; + int32_t truncated = 0; + ASSERT_EQ(ct_scan_result_data(result, &packed, &len, &count, &truncated), 0); + EXPECT_EQ(count, 0U); + EXPECT_EQ(truncated, 1); + ct_scan_result_free(result); + result = nullptr; + scan.strict_byte_budget = 0; + scan.reverse = 1; + ASSERT_EQ(ct_immutable_snapshot_scan(test.snapshot, &scan, &result), 0); + ct_scan_result_free(result); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/inner_page_test.cpp b/lib/crowdb-tree/tests/unit/inner_page_test.cpp index ab1be83e9..a49ea829e 100644 --- a/lib/crowdb-tree/tests/unit/inner_page_test.cpp +++ b/lib/crowdb-tree/tests/unit/inner_page_test.cpp @@ -2,9 +2,9 @@ // Licensed under the Apache License, Version 2.0. // CT4: inner base page + tree descent tests. -#include "crowdb-tree/btree/descent.h" -#include "crowdb-tree/maptable/mapping_table.h" #include "crowdb-tree/maptable/page.h" +#include "mapping_table_test_support.h" +#include "page_descent_test_support.h" #include <gtest/gtest.h> @@ -49,17 +49,19 @@ TEST(InnerPage, child_index_for) TEST(Descent, SingleLeafRoot) { - MappingTable mt; - uint64_t leaf_page_id = mt.allocate_page_id(); + TestMappingTable mt; + uint64_t leaf_page_id = mt.allocate_page_id(); mt.store(leaf_page_id, LeafBase::build(make_entries(make_entry("a")))); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, leaf_page_id, "a"), leaf_page_id); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, leaf_page_id, "zzz"), leaf_page_id); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, leaf_page_id, "a"), + leaf_page_id); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, leaf_page_id, "zzz"), + leaf_page_id); delete mt.get_resident(leaf_page_id); } TEST(Descent, TwoLevelTree) { - MappingTable mt; + TestMappingTable mt; // Three leaves. uint64_t l0 = mt.allocate_page_id(); uint64_t l1 = mt.allocate_page_id(); @@ -71,12 +73,12 @@ TEST(Descent, TwoLevelTree) uint64_t root = mt.allocate_page_id(); mt.store(root, InnerBase::build({"k", "q"}, {l0, l1, l2})); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "a"), l0); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "j"), l0); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "k"), l1); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "p"), l1); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "q"), l2); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "zz"), l2); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "a"), l0); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "j"), l0); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "k"), l1); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "p"), l1); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "q"), l2); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "zz"), l2); for (uint64_t page_id : {l0, l1, l2, root}) { delete mt.get_resident(page_id); @@ -85,7 +87,7 @@ TEST(Descent, TwoLevelTree) TEST(Descent, ThreeLevelTree) { - MappingTable mt; + TestMappingTable mt; // Leaves under two inner nodes, joined by a root. uint64_t la = mt.allocate_page_id(); uint64_t lb = mt.allocate_page_id(); @@ -102,12 +104,12 @@ TEST(Descent, ThreeLevelTree) uint64_t root = mt.allocate_page_id(); // sep [m] -> [left, right] mt.store(root, InnerBase::build({"m"}, {left, right})); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "a"), la); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "e"), lb); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "f"), lb); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "m"), lc); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "t"), ld); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, root, "zz"), ld); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "a"), la); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "e"), lb); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "f"), lb); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "m"), lc); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "t"), ld); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, root, "zz"), ld); for (uint64_t page_id : {la, lb, lc, ld, left, right, root}) { delete mt.get_resident(page_id); @@ -116,8 +118,9 @@ TEST(Descent, ThreeLevelTree) TEST(Descent, EmptyRoot) { - MappingTable mt; - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, kInvalidPageId, "a"), kInvalidPageId); - EXPECT_EQ(find_leaf_page_id([&](uint64_t p) { return mt.get_resident(p); }, 999, "a"), + TestMappingTable mt; + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, kInvalidPageId, "a"), + kInvalidPageId); + EXPECT_EQ(find_leaf_page_id_for_tests([&](uint64_t p) { return mt.get_resident(p); }, 999, "a"), kInvalidPageId); // unset page_id } diff --git a/lib/crowdb-tree/tests/unit/logical_page_cursor_window_test.cpp b/lib/crowdb-tree/tests/unit/logical_page_cursor_window_test.cpp new file mode 100644 index 000000000..9fdc2ec8e --- /dev/null +++ b/lib/crowdb-tree/tests/unit/logical_page_cursor_window_test.cpp @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_publication.h" +#include "btree/page_maintenance.h" +#include "published_read_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +TEST(LogicalPageCursorWindow, ManyRunsKeepConstantCursorStorageAndMatchForwardReverseFrontiers) +{ + auto page = std::make_shared<PageState>(); + for (uint64_t slot = 1; slot <= 129; ++slot) { + const auto run = test_read::page({ + {.key = "b", .slot = slot, .value = std::to_string(slot)}, + {.key = "c", .slot = slot, .value = "later" } + }); + page->overlays.push_back(run->overlays.front()); + } + page->bases.push_back({ + .identity = 1, + .resident = + test_read::base({{.key = "a", .slot = 1, .value = "base"}, {.key = "d", .slot = 1, .value = "base"}} + ) + }); + detail::ResourceBudget::Limits limits; + const auto bytes = LogicalPageCursor::allocation_bound(1).value_or(0) + + detail::shared_allocation_bound<detail::AllocationReservation>(); + limits.fill(bytes); + detail::ResourceBudget budget(limits, 1); + auto account = budget.make_account(limits); + for (bool reverse : {false, true}) { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, bytes); + if (!credit) { + FAIL() << "cursor credit unavailable"; + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + LogicalPageCursor cursor(page, {}, 64, 1, reverse, metadata); + ASSERT_TRUE(cursor.status().ok()) << cursor.status().to_string(); + std::vector<std::string> keys; + while (cursor.valid()) { + keys.push_back(cursor.key().to_string()); + if (cursor.key() == Slice("b")) { + EXPECT_EQ(cursor.cell().slot, 64U); + EXPECT_EQ(cursor.cell().value.to_string(), "64"); + } + cursor.next(); + } + EXPECT_EQ(keys, (reverse ? std::vector<std::string>{"d", "c", "b", "a"} + : std::vector<std::string>{"a", "b", "c", "d"})); + } +} + +TEST(LogicalPageCursorWindow, PartialSplitRetainsAnOversizedQueryableRemainder) +{ + auto page = std::make_shared<PageState>(); + for (int index = 0; index < 40; ++index) { + const auto run = test_read::page({ + {.key = "key" + std::to_string(100 + index), .slot = 1, .value = std::string(40, 'v')} + }); + page->overlays.push_back(run->overlays.front()); + } + auto published = std::make_shared<PublishedTreeState>(*test_read::view(page)); + published->next_page_id = 1; + MaintenancePublicationOptions options; + options.pages.max_sources = 2; + options.pages.max_work_entries = 8; + options.pages.primary_page_bytes = 256; + options.pages.target_page_bytes = 220; + options.pages.inline_value_bytes = 64; + auto record = std::make_shared<InstallRecord>(InstallToken{.generation = published->generation}); + auto result = build_split_step_publication(published, 0, record, options); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_GT(result.output_leaf_pids.size(), 1U); + size_t keys = 0; + for (const auto &change : result.transaction.pages) { + if (!change.replacement || !change.replacement->leaf) { + continue; + } + LogicalPageCursor cursor(change.replacement->leaf, {}, UINT64_MAX, 2); + ASSERT_TRUE(cursor.status().ok()); + while (cursor.valid()) { + ++keys; + EXPECT_EQ(cursor.cell().value.to_string(), std::string(40, 'v')); + cursor.next(); + } + } + EXPECT_EQ(keys, 40U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/logical_page_test.cpp b/lib/crowdb-tree/tests/unit/logical_page_test.cpp new file mode 100644 index 000000000..6d054004f --- /dev/null +++ b/lib/crowdb-tree/tests/unit/logical_page_test.cpp @@ -0,0 +1,401 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <thread> + +using namespace crowdb::tree; + +namespace +{ +std::shared_ptr<const CellVersion> make_cell(uint64_t slot, Slice value, OpKind kind = OpKind::kPut) +{ + return std::make_shared<CellVersion>(encode_cell_buf(slot, kind, value), slot, + kind == OpKind::kDelete ? kFlagTombstone : 0); +} + +void put(MemTable &table, Slice key, uint64_t slot, Slice value, OpKind kind = OpKind::kPut) +{ + ASSERT_TRUE(table.upsert(key, slot, encode_cell_buf(slot, kind, value), 0)); +} + +std::shared_ptr<const OverlayBatch> +make_run(uint64_t id, const std::vector<std::pair<std::string, std::shared_ptr<const CellVersion>>> &entries) +{ + OverlayBuilder builder({}); + for (const auto &[key, cell] : entries) { + EXPECT_TRUE(builder.append(Slice(key), cell).ok()); + } + std::shared_ptr<const OverlayBatch> run; + EXPECT_TRUE(builder.finish(id, &run).ok()); + return run; +} + +std::shared_ptr<const LeafBase> make_base() +{ + std::vector<leaf_entry> entries; + for (const auto *key : {"a", "b", "c", "d"}) { + entries.push_back({.key = key, .cell = encode_cell_buf(3, OpKind::kPut, Slice("base"))}); + } + return std::shared_ptr<const LeafBase>(LeafBase::build(entries)); +} + +std::shared_ptr<const BackingRef> make_backing() +{ + auto backing = std::make_shared<BackingRef>(); + backing->length = 4096; + backing->owner = std::make_shared<int>(1); + return backing; +} +} // namespace + +TEST(LogicalPage, CellOwnerOutlivesSourceAndRetainsOnlyOneVersion) +{ + EpochManager epoch; + auto source = std::make_shared<MemTable>(1, &epoch); + put(*source, "key", 5, "prefix"); + put(*source, "key", 9, "successor"); + EntryRef entry; + { + MemTableRangeMerge merge({source}, {}, 5, 0, {}, 1); + ASSERT_TRUE(merge.status().ok()); + ASSERT_TRUE(merge.next(&entry).ok()); + EXPECT_EQ(entry.slot(), 5U); + auto guard = source->read_guard(); + auto cursor = source->prefix_cursor(5); + EXPECT_EQ(entry.owner.get(), cursor.cell_version()); + } + std::weak_ptr<MemTable> table = source; + source.reset(); + epoch.try_reclaim(); + EXPECT_TRUE(table.expired()); + EXPECT_EQ(epoch.memtable_allocation()->load(), sizeof(CellVersion) + entry.owner->cell.size()); + EXPECT_EQ(entry.key().to_string(), "key"); + EXPECT_EQ(entry.value().to_string(), "prefix"); + entry = {}; + EXPECT_EQ(epoch.memtable_allocation()->load(), 0U); +} + +TEST(LogicalPage, SelectedOwnerIsStableAcrossPruneAndOverwrite) +{ + MemTable source; + put(source, "key", 5, "prefix"); + put(source, "key", 9, "newer"); + std::shared_ptr<const CellVersion> owner; + { + auto guard = source.read_guard(); + auto cursor = source.prefix_cursor(5); + const auto *selected = cursor.cell_version(); + source.prune("key", 9); + put(source, "key", 12, "latest"); + owner = cursor.cell_owner(); + ASSERT_EQ(owner.get(), selected); + } + EXPECT_EQ(owner->slot, 5U); + EXPECT_EQ(CellView(owner->cell.slice()).value().to_string(), "prefix"); +} + +TEST(LogicalPage, ExternalValueHandoffDoesNotCopyAndDropsOnceAcrossThreads) +{ + struct Payload + { + std::string value; + std::atomic<int> *drops; + }; + + std::atomic<int> drops{0}; + auto *payload = new Payload{.value = std::string(64 * 1024, 'v'), .drops = &drops}; + const char *data = payload->value.data(); + EntryRef entry; + { + auto source = std::make_shared<MemTable>(); + auto value = buffer::wrap_external(reinterpret_cast<const uint8_t *>(data), payload->value.size(), payload, + [](void *context) { + auto *owned = static_cast<Payload *>(context); + owned->drops->fetch_add(1); + delete owned; + }); + ASSERT_TRUE(source->upsert_external("key", 7, 0, std::move(value), 0)); + MemTableRangeMerge merge({source}, {}, 7, 0, {}, 1); + ASSERT_TRUE(merge.next(&entry).ok()); + EXPECT_EQ(entry.value().data(), data); + } + EXPECT_EQ(drops.load(), 0); + std::thread consumer([owned = std::move(entry), data] { + EXPECT_EQ(owned.value().data(), data); + EXPECT_EQ(owned.value().size(), 64U * 1024); + }); + consumer.join(); + EXPECT_EQ(drops.load(), 1); +} + +TEST(LogicalPage, RangeMergeCombinesAllTablesBySlotAndStableSourceOrder) +{ + auto first = std::make_shared<MemTable>(); + auto second = std::make_shared<MemTable>(); + put(*first, "a", 2, "covered"); + put(*first, "b", 5, "prefix"); + put(*first, "b", 11, "after-frontier"); + put(*first, "c", 7, "stable-first"); + put(*first, "e", 8, "outside"); + put(*second, "b", 6, "newer-prefix"); + put(*second, "c", 7, "stable-second"); + put(*second, "d", 8, {}, OpKind::kDelete); + MemTableRangeMerge merge({first, second}, {.lower = "b", .upper = "e"}, 8, 2, {}, 2); + std::vector<EntryRef> entries; + while (merge.valid()) { + EntryRef entry; + ASSERT_TRUE(merge.next(&entry).ok()); + entries.push_back(std::move(entry)); + } + ASSERT_EQ(entries.size(), 3U); + EXPECT_EQ(entries[0].key().to_string(), "b"); + EXPECT_EQ(entries[0].value().to_string(), "newer-prefix"); + EXPECT_EQ(entries[1].key().to_string(), "c"); + EXPECT_EQ(entries[1].value().to_string(), "stable-first"); + EXPECT_EQ(entries[2].key().to_string(), "d"); + EXPECT_EQ(entries[2].flags(), kFlagTombstone); +} + +TEST(LogicalPage, IndependentRangeCursorsDoNotLoseSeparatorOrEmptyKey) +{ + auto source = std::make_shared<MemTable>(); + for (const auto *key : {"", "a", "b", "c"}) { + put(*source, key, 1, key); + } + MemTableRangeMerge left({source}, {.upper = "b"}, 1, 0, {}, 1); + MemTableRangeMerge right({source}, {.lower = "b"}, 1, 0, {}, 1); + std::vector<std::string> keys; + for (auto *merge : {&left, &right}) { + while (merge->valid()) { + EntryRef entry; + ASSERT_TRUE(merge->next(&entry).ok()); + keys.push_back(entry.key().to_string()); + } + } + EXPECT_EQ(keys, (std::vector<std::string>{"", "a", "b", "c"})); + MemTableRangeMerge empty({source}, {.lower = "b", .upper = "b"}, 1, 0, {}, 1); + EXPECT_FALSE(empty.valid()); + MemTableRangeMerge no_sources({}, {}, 1, 0, {}, 0); + EXPECT_TRUE(no_sources.status().ok()); + EXPECT_FALSE(no_sources.valid()); +} + +TEST(LogicalPage, MergeAndArenaBudgetsFailWithoutConsumingCandidate) +{ + auto source = std::make_shared<MemTable>(); + put(*source, "large-key", 1, "value"); + MemTableRangeMerge rejected({source}, {}, 1, 0, {}, 0); + EXPECT_EQ(rejected.status().code(), Code::kResourceExhausted); + MemTableRangeMerge merge({source}, {}, 1, 0, {.max_key_bytes = 3}, 1); + EntryRef entry; + EXPECT_EQ(merge.next(&entry).code(), Code::kResourceExhausted); + EXPECT_TRUE(merge.valid()); + EXPECT_EQ(entry.owner, nullptr); +} + +TEST(LogicalPage, BoundedArenaSupportsLongKeysAndNeverRelocatesViews) +{ + RunLimits limits{.max_entries = 4, .max_key_bytes = 1024, .key_block_bytes = 16}; + KeyArenaBuilder arena(limits); + EntryRef first; + EntryRef second; + ASSERT_TRUE(arena.append("a", make_cell(1, "one"), &first).ok()); + const char *key_data = first.key().data(); + const std::string long_key(600, 'z'); + ASSERT_TRUE(arena.append(Slice(long_key), make_cell(2, "two"), &second).ok()); + EXPECT_EQ(first.key().data(), key_data); + EXPECT_EQ(first.key().to_string(), "a"); + EXPECT_EQ(second.key().to_string(), long_key); + EXPECT_NE(first.key_arena, second.key_arena); +} + +TEST(LogicalPage, MultipleArenaWindowsContinueWithoutRewindingSources) +{ + auto source = std::make_shared<MemTable>(); + for (const auto *key : {"a", "b", "c", "d", "e"}) { + put(*source, key, 1, key); + } + MemTableRangeMerge merge({source}, {}, 1, 0, {.max_entries = 2}, 1); + std::vector<EntryRef> entries; + size_t windows = 1; + while (merge.valid()) { + EntryRef entry; + auto status = merge.next(&entry); + if (status.code() == Code::kResourceExhausted) { + merge.rotate_arena(); + ++windows; + ASSERT_TRUE(merge.next(&entry).ok()); + } + else { + ASSERT_TRUE(status.ok()); + } + entries.push_back(std::move(entry)); + } + EXPECT_EQ(windows, 3U); + ASSERT_EQ(entries.size(), 5U); + for (size_t i = 0; i < entries.size(); ++i) { + EXPECT_EQ(entries[i].key().to_string(), std::string(1, static_cast<char>('a' + i))); + } + EXPECT_EQ(entries[0].key_arena, entries[1].key_arena); + EXPECT_NE(entries[1].key_arena, entries[2].key_arena); +} + +TEST(LogicalPage, OverlayValidatesOrderAndSharesValues) +{ + auto value = make_cell(7, std::string(1024, 'v')); + OverlayBuilder builder({}); + ASSERT_TRUE(builder.append("b", value).ok()); + EXPECT_EQ(builder.append("a", value).code(), Code::kInvalidArgument); + EXPECT_EQ(builder.append("b", value).code(), Code::kInvalidArgument); + std::shared_ptr<const OverlayBatch> run; + ASSERT_TRUE(builder.finish(42, &run).ok()); + ASSERT_NE(run->find("b"), nullptr); + EXPECT_EQ(run->find("b")->owner, value); + EXPECT_EQ(run->find("b")->value().data(), CellView(value->cell.slice()).value().data()); + EXPECT_EQ(run->find("c"), nullptr); + EXPECT_EQ(run->id(), 42U); + EXPECT_EQ(run->min_slot(), 7U); + EXPECT_EQ(run->max_slot(), 7U); +} + +TEST(LogicalPage, LookupPreservesTombstoneOverColdBaseAndRequestsUnknownData) +{ + PageState page; + page.bases.push_back({.identity = 1, .max_slot = 3, .backing = make_backing()}); + page.overlays.push_back(make_run(1, { + {"b", make_cell(7, {}, OpKind::kDelete)} + })); + auto result = page.lookup("b"); + EXPECT_EQ(result.state, LogicalLookupState::kTombstone); + EXPECT_EQ(result.cell.slot, 7U); + EXPECT_EQ(page.lookup("a").state, LogicalLookupState::kNeedsBase); + EXPECT_EQ(page.lookup("b", 5).state, LogicalLookupState::kNeedsBase); + page.bases[0].max_slot.reset(); + EXPECT_EQ(page.lookup("b").state, LogicalLookupState::kNeedsBase); + page.bases[0].max_slot = 9; + EXPECT_EQ(page.lookup("b").state, LogicalLookupState::kNeedsBase); +} + +TEST(LogicalPage, SharedBaseRangeViewsFilterLookupWithoutCopy) +{ + auto base = make_base(); + PageState left; + left.range = {.upper = "c"}; + left.bases.push_back({.identity = 1, .range = {.upper = "c"}, .resident = base}); + PageState right; + right.range = {.lower = "c"}; + right.bases.push_back({.identity = 1, .range = {.lower = "c"}, .resident = base}); + EXPECT_EQ(left.lookup("c").state, LogicalLookupState::kMissing); + EXPECT_EQ(right.lookup("b").state, LogicalLookupState::kMissing); + auto value = right.lookup("c"); + EXPECT_EQ(value.state, LogicalLookupState::kValue); + EXPECT_EQ(value.cell.value.data(), CellView(base->cell(2)).value().data()); + EXPECT_EQ(value.cell.owner.get(), base.get()); +} + +TEST(LogicalPage, CursorMergesBasesAndRunsForwardReverseAndAtFrontier) +{ + auto page = std::make_shared<PageState>(); + page->bases.push_back({.identity = 1, .resident = make_base()}); + page->overlays.push_back(make_run(2, { + {"b", make_cell(7, {}, OpKind::kDelete)}, + {"c", make_cell(9, "latest")} + })); + page->overlays.push_back(make_run(1, { + {"c", make_cell(6, "prefix")} + })); + for (bool reverse : {false, true}) { + LogicalPageCursor cursor(page, {.lower = "b", .upper = "d"}, 7, 3, reverse); + ASSERT_TRUE(cursor.status().ok()); + std::vector<std::string> keys; + while (cursor.valid()) { + const auto key = cursor.key().to_string(); + const auto cell = cursor.cell(); + keys.push_back(key); + if (key == "b") { + EXPECT_TRUE(cell.tombstone()); + EXPECT_EQ(cell.slot, 7U); + } + else { + EXPECT_EQ(cell.value.to_string(), "prefix"); + EXPECT_EQ(cell.slot, 6U); + } + cursor.next(); + } + EXPECT_EQ(keys, (reverse ? std::vector<std::string>{"c", "b"} : std::vector<std::string>{"b", "c"})); + } +} + +TEST(LogicalPage, CursorReportsColdBaseAndBudgetsWithoutStartingPartialScan) +{ + auto page = std::make_shared<PageState>(); + page->bases.push_back({.identity = 1, .range = {.upper = "c"}, .backing = make_backing()}); + page->overlays.push_back(make_run(1, { + {"z", make_cell(7, "overlay")} + })); + LogicalPageCursor cold(page, {}, UINT64_MAX, 2); + EXPECT_TRUE(cold.status().ok()); + EXPECT_EQ(cold.needs_base(), 0U); + EXPECT_FALSE(cold.valid()); + LogicalPageCursor outside(page, {.lower = "c"}, UINT64_MAX, 2); + EXPECT_FALSE(outside.needs_base().has_value()); + ASSERT_TRUE(outside.valid()); + EXPECT_EQ(outside.key().to_string(), "z"); + LogicalPageCursor windowed(page, {}, UINT64_MAX, 1); + EXPECT_TRUE(windowed.status().ok()); + EXPECT_EQ(windowed.needs_base(), 0U); + EXPECT_FALSE(windowed.valid()); +} + +TEST(LogicalPage, BackingNeedsIndependentOwnerAndCapabilities) +{ + BackingRef backing; + backing.length = 4096; + EXPECT_FALSE(backing.reloadable()); + backing.owner = std::make_shared<int>(1); + EXPECT_TRUE(backing.reloadable()); + EXPECT_FALSE(BackingCapabilities{}.can_stage()); + EXPECT_FALSE((BackingCapabilities{true, false}).can_stage()); + EXPECT_TRUE((BackingCapabilities{true, true}).can_stage()); +} + +TEST(LogicalPage, ColdBaseSlotTieRespectsSourceOrderBeforeSkippingLoad) +{ + PageState page; + page.bases.push_back({.identity = 1, .max_slot = 3, .backing = make_backing()}); + page.bases.push_back({.identity = 2, .resident = make_base()}); + auto result = page.lookup("a"); + EXPECT_EQ(result.state, LogicalLookupState::kNeedsBase); + EXPECT_EQ(result.base_index, 0U); + // An overlay precedes every base, so an equal-slot overlay proves the + // outcome even when the first base is cold. + page.overlays.push_back(make_run(1, { + {"a", make_cell(3, "overlay")} + })); + result = page.lookup("a"); + EXPECT_EQ(result.state, LogicalLookupState::kValue); + EXPECT_EQ(result.cell.value.to_string(), "overlay"); + page.overlays.clear(); + std::swap(page.bases[0], page.bases[1]); + result = page.lookup("a"); + EXPECT_EQ(result.state, LogicalLookupState::kValue); + EXPECT_EQ(result.cell.value.to_string(), "base"); +} + +TEST(LogicalPage, InvalidColdBaseCannotMasqueradeAsLoadableOrMissing) +{ + auto page = std::make_shared<PageState>(); + page->bases.push_back({.identity = 1}); + auto result = page->lookup("a"); + EXPECT_EQ(result.state, LogicalLookupState::kInvalid); + EXPECT_EQ(result.status.code(), Code::kCorruption); + EXPECT_EQ(result.base_index, 0U); + LogicalPageCursor cursor(page, {}, UINT64_MAX, 1); + EXPECT_EQ(cursor.status().code(), Code::kCorruption); + EXPECT_FALSE(cursor.needs_base().has_value()); + EXPECT_FALSE(cursor.valid()); +} diff --git a/lib/crowdb-tree/tests/unit/maintenance_budget_test.cpp b/lib/crowdb-tree/tests/unit/maintenance_budget_test.cpp new file mode 100644 index 000000000..3dabe0e53 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/maintenance_budget_test.cpp @@ -0,0 +1,223 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_publication.h" +#include "btree/page_encode.h" + +#include <gtest/gtest.h> + +#include <array> + +namespace crowdb::tree +{ +namespace +{ +struct ReclaimBudget +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result{}; + result[static_cast<size_t>(detail::BudgetClass::kReclaim)] = 1024 * 1024; + return result; + } + + detail::ResourceBudget budget{limits(), 1}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + std::shared_ptr<detail::AllocationReservation> reservation; + + ReclaimBudget() + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, 1024 * 1024); + if (credit) { + reservation = detail::AllocationReservation::create(std::move(*credit)); + } + EXPECT_TRUE(reservation); + } + + [[nodiscard]] size_t allocated() const + { + return budget.snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kReclaim)]; + } +}; + +std::string key(size_t index) +{ + std::string result(64, static_cast<char>('a' + index)); + return result; +} + +std::shared_ptr<PageState> input_page() +{ + auto page = std::make_shared<PageState>(); + OverlayBuilder builder({}); + for (size_t i = 0; i < 4; ++i) { + auto cell = std::make_shared<CellVersion>(encode_cell_buf(i + 1, OpKind::kPut, "value"), i + 1, 0); + EXPECT_TRUE(builder.append(key(i), cell).ok()); + } + std::shared_ptr<const OverlayBatch> run; + EXPECT_TRUE(builder.finish(1, &run).ok()); + page->overlays.push_back(std::move(run)); + return page; +} + +TEST(MaintenanceBudget, EveryCoalescedRunKeepsItsCreditAndSharedPayloadAfterWindowMoves) +{ + ReclaimBudget budget; + auto input = input_page(); + const auto value = input->lookup(key(3)).cell; + PageMaintenanceLimits limits; + limits.metadata = budget.reservation; + limits.run.max_entries = 1; + const auto baseline = budget.allocated(); + auto result = coalesce_overlays(input, limits, 2, 10); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_EQ(result.page->overlays.size(), 4U); + EXPECT_EQ(result.page->overlays.get_allocator().owner(), budget.reservation); + EXPECT_GE(budget.allocated() - baseline, 4 * (sizeof(OverlayBatch) + sizeof(EntryRef))); + input.reset(); + budget.reservation->seal(); + EXPECT_EQ(result.page->lookup(key(3)).cell.owner, value.owner); + EXPECT_EQ(result.page->lookup(key(3)).cell.value.to_string(), "value"); + const auto charged = budget.allocated(); + result.page.reset(); + EXPECT_LT(budget.allocated(), charged); + EXPECT_EQ(budget.allocated(), baseline); +} + +TEST(MaintenanceBudget, LayoutSeparatorsAndSplitDescriptorsRetainActualCapacity) +{ + ReclaimBudget budget; + PageMaintenanceLimits limits; + limits.metadata = budget.reservation; + limits.primary_page_bytes = 320; + limits.target_page_bytes = 260; + const auto baseline = budget.allocated(); + auto input = input_page(); + auto layout = plan_leaf_layout(input, limits); + ASSERT_TRUE(layout.status.ok()) << layout.status.to_string(); + ASSERT_EQ(layout.primary_bytes.size(), 4U); + EXPECT_EQ(layout.separators.get_allocator().owner(), budget.reservation); + EXPECT_EQ(layout.primary_bytes.get_allocator().owner(), budget.reservation); + std::array<SplitTarget, 4> targets{}; + for (size_t i = 0; i < targets.size(); ++i) { + targets[i] = { + .page_id = i, .version = {.route_generation = 1, .content_id = 1, .representation_id = 1} + }; + } + auto split = split_leaf_ranges(layout, targets, limits); + ASSERT_TRUE(split.status.ok()) << split.status.to_string(); + ASSERT_EQ(split.pages.size(), 4U); + for (const auto &page : split.pages) { + EXPECT_EQ(page->overlays.get_allocator().owner(), budget.reservation); + EXPECT_EQ(page->bases.get_allocator().owner(), budget.reservation); + } + { + auto dropped = std::move(layout); + } + input.reset(); + budget.reservation->seal(); + EXPECT_GT(budget.allocated(), baseline); + EXPECT_EQ(split.pages[2]->range.lower, std::optional<OwnedKey>(key(2))); + EXPECT_EQ(split.pages[2]->lookup(key(2)).cell.value.to_string(), "value"); + // Destroy the result containers too, so their retained capacity is freed. + { + auto dropped = std::move(split); + } + EXPECT_EQ(budget.allocated(), baseline); +} + +TEST(MaintenanceBudget, CursorScratchUsesCurrentCreditAndSealedCreditCannotAllocate) +{ + ReclaimBudget budget; + auto page = input_page(); + const auto baseline = budget.allocated(); + { + LogicalPageCursor cursor(page, {}, UINT64_MAX, 8, false, budget.reservation); + ASSERT_TRUE(cursor.valid()); + EXPECT_GT(budget.allocated(), baseline); + } + EXPECT_EQ(budget.allocated(), baseline); + budget.reservation->seal(); + EXPECT_THROW((LogicalPageCursor(page, {}, UINT64_MAX, 8, false, budget.reservation)), std::bad_alloc); +} + +TEST(MaintenanceBudget, StructureTransactionAndPublishedPagesRetainCreditUntilLastView) +{ + ReclaimBudget budget; + const auto baseline = budget.allocated(); + std::shared_ptr<const PublishedTreeState> retained; + { + PublicationCoordinator publisher; + auto page = std::make_shared<PublishedPage>(); + page->leaf = input_page(); + auto seed = std::make_shared<PublicationRequest>(); + seed->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{}), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0}; + seed->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(publisher.try_submit(seed)); + MaintenancePublicationOptions options; + options.metadata = budget.reservation; + options.pages.primary_page_bytes = 320; + options.pages.target_page_bytes = 260; + auto record = std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 2}); + auto result = build_split_step_publication(publisher.capture(), 0, record, options); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + ASSERT_TRUE(result.changed()); + EXPECT_EQ(result.transaction.pages.get_allocator().owner(), budget.reservation); + EXPECT_EQ(result.output_leaf_pids.get_allocator().owner(), budget.reservation); + for (const auto &replacement : result.transaction.pages) { + if (replacement.replacement->inner) { + EXPECT_EQ(replacement.replacement->inner->children.get_allocator().owner(), budget.reservation); + EXPECT_EQ(replacement.replacement->inner->separators.get_allocator().owner(), budget.reservation); + } + } + auto request = detail::reserved_shared<PublicationRequest>(budget.reservation, budget.reservation); + request->transaction = std::move(result.transaction); + request->on_done = [](PublicationResult published) { EXPECT_TRUE(published.status.ok()); }; + ASSERT_TRUE(publisher.try_submit(request)); + retained = publisher.capture(); + budget.reservation->seal(); + auto denied = build_split_step_publication(retained, 1, record, options); + EXPECT_EQ(denied.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(denied.changed()); + EXPECT_EQ(publisher.capture(), retained); + } + EXPECT_GT(budget.allocated(), baseline); + EXPECT_EQ(retained->mapping.get(0)->leaf->lookup(key(0)).cell.value.to_string(), "value"); + retained.reset(); + EXPECT_EQ(budget.allocated(), baseline); +} + +TEST(MaintenanceBudget, EncoderRunsAfterSealAndFrameDependencyKeepsItsCharge) +{ + ReclaimBudget budget; + const auto baseline = budget.allocated(); + auto view = std::make_shared<PublishedTreeState>(); + auto page = std::make_shared<PublishedPage>(); + page->leaf = input_page(); + const std::array<PageReplacement, 1> changes{{{.page_id = 0, .expected = {}, .replacement = page}}}; + MappingView::CopyStats copied; + view->mapping = view->mapping.replace(changes, copied); + view->root_page_id = 0; + view->next_page_id = 1; + view->coverage = 4; + auto prepared = prepare_final_page(view, 0, {}, {.frame_bytes = 1024, .metadata = budget.reservation}); + ASSERT_TRUE(prepared.status.ok()) << prepared.status.to_string(); + ASSERT_TRUE(prepared.encoder); + EXPECT_EQ(prepared.encoder.get_deleter().owner, budget.reservation); + budget.reservation->seal(); + std::array<uint8_t, 1024> frame{}; + auto step = prepared.encoder->next(frame); + ASSERT_TRUE(step.status.ok()) << step.status.to_string(); + ASSERT_TRUE(step.frame); + EXPECT_TRUE(step.complete); + EXPECT_TRUE(frame_validate(frame.data(), frame.size())); + const auto before = budget.allocated(); + prepared.encoder.reset(); + EXPECT_EQ(before - budget.allocated(), sizeof(FinalPageEncoder)); + EXPECT_GT(budget.allocated(), baseline); + step.frame.reset(); + EXPECT_EQ(budget.allocated(), baseline); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/maintenance_inner_merge_test.cpp b/lib/crowdb-tree/tests/unit/maintenance_inner_merge_test.cpp new file mode 100644 index 000000000..1fc4d61d4 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/maintenance_inner_merge_test.cpp @@ -0,0 +1,262 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_operation.h" +#include "btree/maintenance_publication.h" +#include "btree/page_load.h" + +#include <gtest/gtest.h> + +#include <future> +#include <map> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<InstallRecord> token(uint64_t sequence) +{ + return std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 10, .sequence = sequence}); +} + +PublicationResult publish(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> result; + auto future = result.get_future(); + request->on_done = [&](auto value) { result.set_value(std::move(value)); }; + EXPECT_TRUE(publisher.try_submit(request)); + return future.get(); +} + +std::shared_ptr<PublishedPage> inner(uint32_t level, std::vector<uint64_t> children, + std::vector<std::string> separators) +{ + auto route = std::make_shared<InnerPageState>(); + route->children.assign(children.begin(), children.end()); + route->separators.assign(separators.begin(), separators.end()); + auto page = std::make_shared<PublishedPage>(); + page->revision = 1; + page->level = level; + page->inner = std::move(route); + page->metrics.leaf = false; + return page; +} + +struct MergeTree +{ + PublicationCoordinator publisher; + MaintenancePublicationOptions options; + + MergeTree() + { + options.max_inner_children = 4; + options.inner_merge_keys = 2; + options.pages.primary_page_bytes = 256; + options.pages.target_page_bytes = 220; + options.pages.merge_trigger_bytes = 190; + options.pages.merge_target_bytes = 210; + PublicationTransaction seed{.record = token(0)}; + const std::array<std::string, 4> keys{"a", "b", "m", "n"}; + for (size_t i = 0; i < keys.size(); ++i) { + auto leaf = std::make_shared<PageState>(); + leaf->route_generation = leaf->content_id = leaf->representation_id = 1; + if (i != 0) { + leaf->range.lower = OwnedKey(keys[i]); + } + if (i + 1 < keys.size()) { + leaf->range.upper = OwnedKey(keys[i + 1]); + } + leaf->right_sibling = i + 1 < keys.size() ? i + 1 : kInvalidPageId; + OverlayBuilder builder({}); + auto cell = std::make_shared<CellVersion>(encode_cell_buf(1, OpKind::kPut, "v"), 1, 0); + EXPECT_TRUE(builder.append(keys[i], cell).ok()); + std::shared_ptr<const OverlayBatch> run; + EXPECT_TRUE(builder.finish(i + 1, &run).ok()); + leaf->overlays.push_back(std::move(run)); + auto page = std::make_shared<PublishedPage>(); + page->revision = 1; + page->leaf = std::move(leaf); + seed.pages.push_back({.page_id = i, .expected = {}, .replacement = page}); + } + seed.pages.push_back({.page_id = 10, .expected = {}, .replacement = inner(1, {0, 1}, {"b"})}); + seed.pages.push_back({.page_id = 11, .expected = {}, .replacement = inner(1, {2, 3}, {"n"})}); + seed.pages.push_back({.page_id = 20, .expected = {}, .replacement = inner(2, {10, 11}, {"m"})}); + seed.root_page_id = 20; + EXPECT_TRUE(publish(publisher, std::move(seed)).status.ok()); + } +}; + +TEST(MaintenanceInnerMerge, RepairsUntouchedUnderfullRouteAndKeepsOldView) +{ + MergeTree tree; + const auto before = tree.publisher.capture(); + auto plan = build_inner_merge_publication(before, 2, token(1), tree.options); + ASSERT_TRUE(plan.status.ok()) << plan.status.to_string(); + ASSERT_TRUE(plan.changed()); + EXPECT_EQ(plan.merged_inner_pairs, 1U); + ASSERT_TRUE(publish(tree.publisher, std::move(plan.transaction)).status.ok()); + const auto after = tree.publisher.capture(); + EXPECT_EQ(before->mapping.copy_bounds().max_level, 2U); + EXPECT_EQ(after->mapping.copy_bounds().max_level, 1U); + EXPECT_EQ(after->root_page_id, 10U); + EXPECT_FALSE(after->mapping.get(11)); + EXPECT_FALSE(after->mapping.get(20)); + const auto route = after->mapping.get(10)->inner; + EXPECT_EQ(route->children.size(), 4U); + EXPECT_EQ(route->separators[1], OwnedKey("m")); + EXPECT_EQ(before->mapping.get(20)->inner->children.size(), 2U); + EXPECT_EQ(before->mapping.get(11)->inner->children.size(), 2U); + EXPECT_EQ(before->mapping.get(2)->leaf->lookup("m").cell.value.to_string(), "v"); +} + +TEST(MaintenanceInnerMerge, LeafMergeRepairsItsParentInTheSameTransaction) +{ + MergeTree tree; + auto plan = build_merge_publication(tree.publisher.capture(), 2, 3, token(1), 10, tree.options); + ASSERT_TRUE(plan.status.ok()) << plan.status.to_string(); + ASSERT_TRUE(plan.changed()); + EXPECT_EQ(plan.merged_inner_pairs, 1U); + ASSERT_TRUE(publish(tree.publisher, std::move(plan.transaction)).status.ok()); + const auto after = tree.publisher.capture(); + EXPECT_EQ(after->root_page_id, 10U); + EXPECT_EQ(after->mapping.get(10)->inner->children.size(), 3U); + EXPECT_FALSE(after->mapping.get(3)); + EXPECT_EQ(after->mapping.get(2)->leaf->lookup("n").cell.value.to_string(), "v"); +} + +TEST(MaintenanceInnerMerge, FanoutAndEncodedBytesPreventOversizedMerge) +{ + MergeTree tree; + tree.options.max_inner_children = 3; + auto plan = build_inner_merge_publication(tree.publisher.capture(), 2, token(1), tree.options); + ASSERT_TRUE(plan.status.ok()); + EXPECT_FALSE(plan.changed()); + tree.options.max_inner_children = 4; + tree.options.max_inner_bytes = kFrameHeaderSize + kFrameTrailerSize + (2 * sizeof(uint64_t)) + kInnerSlotSize + 1; + plan = build_inner_merge_publication(tree.publisher.capture(), 2, token(2), tree.options); + ASSERT_TRUE(plan.status.ok()); + EXPECT_FALSE(plan.changed()); +} + +TEST(MaintenanceInnerMerge, ChangedPageBoundRejectsWithoutPublication) +{ + MergeTree tree; + const auto before = tree.publisher.capture(); + tree.options.max_changed_pages = 1; + const auto plan = build_inner_merge_publication(before, 2, token(1), tree.options); + EXPECT_EQ(plan.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(plan.changed()); + EXPECT_EQ(tree.publisher.capture(), before); +} + +TEST(MaintenanceInnerMerge, ChangedSiblingInvalidatesWholePreparedTransaction) +{ + MergeTree tree; + const auto before = tree.publisher.capture(); + auto plan = build_inner_merge_publication(before, 2, token(1), tree.options); + ASSERT_TRUE(plan.status.ok()); + auto changed = std::make_shared<PublishedPage>(*before->mapping.get(10)); + ++changed->revision; + ASSERT_TRUE(publish(tree.publisher, + {.record = token(2), + .pages = {{.page_id = 10, .expected = before->mapping.get(10), .replacement = changed}}}) + .status.ok()); + const auto intervening = tree.publisher.capture(); + EXPECT_EQ(publish(tree.publisher, std::move(plan.transaction)).status.code(), Code::kUnavailable); + EXPECT_EQ(tree.publisher.capture(), intervening); +} + +class TestInnerLoader final : public PageLoader +{ + public: + std::map<uint64_t, std::shared_ptr<const InnerPageState>> frames; + size_t calls = 0; + bool fail_sibling = false; + + void load_leaf(BaseRef /*base*/, LeafComplete complete) override + { + ADD_FAILURE() << "inner repair must not load leaf payload"; + complete(Status::internal_error(), {}); + } + + void load_inner(std::shared_ptr<const BackingRef> backing, InnerComplete complete) override + { + ++calls; + if (fail_sibling && backing->address == 11) { + complete(Status::io_error("injected sibling failure"), {}); + return; + } + complete(Status::Ok(), frames.at(backing->address)); + } +}; + +void cold_repair(bool fail) +{ + MergeTree tree; + auto loader = std::make_shared<TestInnerLoader>(); + loader->fail_sibling = fail; + PublicationTransaction evict{.record = token(3)}; + const auto before = tree.publisher.capture(); + for (const uint64_t pid : {10, 11, 20}) { + const auto old = before->mapping.get(pid); + loader->frames.emplace(pid, old->inner); + auto page = std::make_shared<PublishedPage>(*old); + page->inner.reset(); + auto backing = std::make_shared<BackingRef>(); + backing->address = pid; + backing->length = 512; + backing->owner = std::make_shared<int>(1); + page->inner_backing = std::move(backing); + evict.pages.push_back({.page_id = pid, .expected = old, .replacement = std::move(page)}); + } + ASSERT_TRUE(publish(tree.publisher, std::move(evict)).status.ok()); + const auto captured = tree.publisher.capture(); + EXPECT_EQ(captured->mapping.copy_bounds().max_level, 2U); + detail::ResourceBudget::Limits limits{}; + limits.fill(16 * 1024 * 1024); + detail::ResourceBudget budget(limits, 1); + auto account = budget.make_account(limits); + detail::FlushRuntime runtime(1, 4); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(4); + PageMaintenanceOperationOptions options; + options.publication = tree.options; + options.max_steps = options.max_candidates = 1; + options.acquire = [&](auto *out) { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, 16 * 1024 * 1024); + if (!credit) { + return Status::resource_exhausted(); + } + *out = detail::AllocationReservation::create(std::move(*credit)); + return Status::Ok(); + }; + auto maintenance = make_page_maintenance(tree.publisher, runtime, lane, std::move(options), loader, {}); + std::promise<Status> completion; + auto result = completion.get_future(); + maintenance(captured, [&](Status status, uint64_t) { completion.set_value(std::move(status)); }); + const auto status = result.get(); + runtime.drain(); + EXPECT_EQ(loader->calls, 3U); + if (fail) { + EXPECT_EQ(status.code(), Code::kIoError); + EXPECT_EQ(tree.publisher.capture(), captured); + } + else { + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(tree.publisher.capture()->root_page_id, 10U); + EXPECT_FALSE(captured->mapping.get(20)->inner); + EXPECT_EQ(tree.publisher.capture()->mapping.get(10)->inner->children.size(), 4U); + } +} + +TEST(MaintenanceInnerMerge, ColdSiblingAndDescendingRoutePublishOnOneWorker) +{ + cold_repair(false); +} + +TEST(MaintenanceInnerMerge, ColdSiblingFailureKeepsCapturedTree) +{ + cold_repair(true); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/maintenance_publication_test.cpp b/lib/crowdb-tree/tests/unit/maintenance_publication_test.cpp new file mode 100644 index 000000000..718946577 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/maintenance_publication_test.cpp @@ -0,0 +1,383 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_publication.h" + +#include <gtest/gtest.h> + +#include <algorithm> +#include <future> +#include <string> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<InstallRecord> maintenance_record(uint64_t sequence) +{ + return std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 9, .range = 0, .sequence = sequence}); +} + +PublicationResult publish_maintenance(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> completed; + auto result = completed.get_future(); + request->on_done = [&](PublicationResult value) { completed.set_value(std::move(value)); }; + if (!publisher.try_submit(request)) { + ADD_FAILURE() << "publication rejected by ingress"; + return {.status = Status::unavailable()}; + } + return result.get(); +} + +std::shared_ptr<PublishedPage> maintenance_leaf(std::vector<std::string> keys, size_t value_bytes, + PageKeyRange range = {}, uint64_t sibling = kInvalidPageId) +{ + auto logical = std::make_shared<PageState>(); + logical->range = std::move(range); + logical->right_sibling = sibling; + logical->route_generation = 1; + logical->content_id = 7; + logical->representation_id = 4; + OverlayBuilder builder({}); + for (const auto &key : keys) { + auto cell = + std::make_shared<CellVersion>(encode_cell_buf(1, OpKind::kPut, std::string(value_bytes, 'v')), 1, 0); + EXPECT_TRUE(builder.append(Slice(key), std::move(cell)).ok()); + } + std::shared_ptr<const OverlayBatch> run; + EXPECT_TRUE(builder.finish(1, &run).ok()); + logical->overlays.push_back(std::move(run)); + auto page = std::make_shared<PublishedPage>(); + page->revision = 1; + page->leaf = std::move(logical); + page->metrics.dirty = true; + page->metrics.overlay_runs = 1; + page->metrics.overlay_entries = keys.size(); + return page; +} + +std::shared_ptr<PublishedPage> maintenance_inner(std::vector<uint64_t> children, std::vector<std::string> separators, + uint32_t level = 1) +{ + auto inner = std::make_shared<InnerPageState>(); + inner->children.assign(children.begin(), children.end()); + inner->separators.assign(separators.begin(), separators.end()); + auto page = std::make_shared<PublishedPage>(); + page->revision = 1; + page->level = level; + page->inner = std::move(inner); + page->metrics.leaf = false; + return page; +} + +MaintenancePublicationOptions maintenance_limits() +{ + MaintenancePublicationOptions options; + options.pages.primary_page_bytes = 256; + options.pages.target_page_bytes = 220; + options.pages.merge_trigger_bytes = 190; + options.pages.merge_target_bytes = 210; + options.pages.inline_value_bytes = 64; + options.max_inner_children = 2; + return options; +} + +std::shared_ptr<const PublishedPage> route(const PublishedTreeState &view, const std::string &key) +{ + auto page = view.mapping.get(view.root_page_id); + for (size_t depth = 0; depth < 64 && page && page->inner; ++depth) { + EXPECT_EQ(page->inner->children.size(), page->inner->separators.size() + 1); + const auto index = + static_cast<size_t>(std::upper_bound(page->inner->separators.begin(), page->inner->separators.end(), key) - + page->inner->separators.begin()); + const auto next = view.mapping.get(page->inner->children[index]); + EXPECT_NE(next, nullptr); + if (!next) { + return {}; + } + EXPECT_EQ(next->level + 1, page->level); + page = next; + } + return page; +} + +void seed_leaf(PublicationCoordinator &publisher, const std::shared_ptr<PublishedPage> &page) +{ + EXPECT_TRUE(publish_maintenance(publisher, {.record = maintenance_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0}) + .status.ok()); +} + +TEST(MaintenancePublication, MultiwayRootSplitPublishesAllLevelsAndKeepsOldView) +{ + PublicationCoordinator publisher; + auto original = maintenance_leaf({"a", "b", "c", "d"}, 30); + seed_leaf(publisher, original); + const auto before = publisher.capture(); + auto plan = build_split_publication(before, 0, maintenance_record(1), maintenance_limits()); + ASSERT_TRUE(plan.status.ok()) << plan.status.message(); + ASSERT_TRUE(plan.changed()); + EXPECT_TRUE(std::ranges::equal(plan.output_leaf_pids, std::array<uint64_t, 4>{0, 1, 2, 3})); + ASSERT_TRUE(publish_maintenance(publisher, std::move(plan.transaction)).status.ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->stats.leaf_pages, 4U); + EXPECT_EQ(after->stats.inner_pages, 3U); + EXPECT_EQ(after->stats.height, 3U); + EXPECT_EQ(after->stats.dirty_pages, 7U); + EXPECT_EQ(after->stats.writeback_candidate_pages, 7U); + EXPECT_EQ(after->stats.checkpoint_pending_pages, 7U); + EXPECT_TRUE(after->stats.checkpoint_mapping_pending); + EXPECT_EQ(after->stats.page_size_unknown_pages, 0U); + EXPECT_EQ(after->stats.kv_known_count, 4U); + EXPECT_EQ(after->stats.kv_known_bytes, 124U); + for (size_t i = 0; i < 4; ++i) { + const std::string key(1, static_cast<char>('a' + i)); + const auto leaf = route(*after, key); + ASSERT_TRUE(leaf && leaf->leaf); + EXPECT_EQ(leaf->leaf->lookup(Slice(key)).cell.owner, original->leaf->lookup(Slice(key)).cell.owner); + EXPECT_EQ(after->mapping.get(i)->leaf->right_sibling, i == 3 ? kInvalidPageId : i + 1); + } + publisher.reclaim(); + EXPECT_EQ(before->mapping.get(0), original); + EXPECT_EQ(before->stats.height, 1U); + EXPECT_EQ(before->mapping.get(0)->leaf->lookup("d").state, LogicalLookupState::kValue); +} + +TEST(MaintenancePublication, SplittingExistingChildRecursivelySplitsParentAndPreservesRightNeighbor) +{ + PublicationCoordinator publisher; + auto left = maintenance_leaf({"a", "b", "c", "d"}, 30, {.upper = "m"}, 1); + auto right = maintenance_leaf({"m", "n"}, 30, {.lower = "m"}); + auto parent = maintenance_inner({0, 1}, {"m"}); + ASSERT_TRUE(publish_maintenance(publisher, + { + .record = maintenance_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = left}, + {.page_id = 1, .expected = {}, .replacement = right}, + {.page_id = 2, .expected = {}, .replacement = parent}}, + .root_page_id = 2 + }) + .status.ok()); + const auto before = publisher.capture(); + auto plan = build_split_publication(before, 0, maintenance_record(1), maintenance_limits()); + ASSERT_TRUE(plan.status.ok()) << plan.status.message(); + const auto last = plan.output_leaf_pids.back(); + ASSERT_TRUE(publish_maintenance(publisher, std::move(plan.transaction)).status.ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->stats.leaf_pages, 5U); + EXPECT_EQ(after->stats.height, 4U); + EXPECT_EQ(after->mapping.get(last)->leaf->right_sibling, 1U); + EXPECT_EQ(after->mapping.get(1), right); + for (const auto *key : {"a", "b", "c", "d", "m", "n"}) { + auto page = route(*after, key); + ASSERT_TRUE(page && page->leaf); + EXPECT_EQ(page->leaf->lookup(key).state, LogicalLookupState::kValue); + } + EXPECT_EQ(before->mapping.get(2)->inner->children, (InnerPageState::Children{0, 1})); +} + +TEST(MaintenancePublication, SameParentMergeRemovesRightAndContractsRootAtomically) +{ + PublicationCoordinator publisher; + auto left = maintenance_leaf({"a"}, 1, {.upper = "b"}, 1); + auto right = maintenance_leaf({"b"}, 1, {.lower = "b"}); + auto parent = maintenance_inner({0, 1}, {"b"}); + ASSERT_TRUE(publish_maintenance(publisher, + { + .record = maintenance_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = left}, + {.page_id = 1, .expected = {}, .replacement = right}, + {.page_id = 2, .expected = {}, .replacement = parent}}, + .root_page_id = 2 + }) + .status.ok()); + const auto before = publisher.capture(); + auto plan = build_merge_publication(before, 0, 1, maintenance_record(1), 100, maintenance_limits()); + ASSERT_TRUE(plan.status.ok()) << plan.status.message(); + ASSERT_TRUE(plan.changed()); + ASSERT_TRUE(publish_maintenance(publisher, std::move(plan.transaction)).status.ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->root_page_id, 0U); + EXPECT_EQ(after->next_page_id, 3U); + EXPECT_EQ(after->stats.leaf_pages, 1U); + EXPECT_EQ(after->stats.inner_pages, 0U); + EXPECT_EQ(after->stats.height, 1U); + EXPECT_EQ(after->stats.dirty_pages, 1U); + EXPECT_EQ(after->stats.checkpoint_pending_pages, 1U); + EXPECT_TRUE(after->stats.checkpoint_mapping_pending); + EXPECT_EQ(after->stats.kv_known_count, 2U); + EXPECT_EQ(after->stats.kv_known_bytes, 4U); + EXPECT_EQ(after->mapping.get(1), nullptr); + EXPECT_EQ(after->mapping.get(2), nullptr); + EXPECT_EQ(after->mapping.get(0)->leaf->right_sibling, kInvalidPageId); + EXPECT_EQ(route(*after, "b")->leaf->lookup("b").cell.owner, right->leaf->lookup("b").cell.owner); + EXPECT_EQ(before->mapping.get(2), parent); + EXPECT_EQ(before->mapping.get(1), right); +} + +TEST(MaintenancePublication, CoalesceUpdatesExactPrimarySizeWhenItRemovesAnEligibleTombstone) +{ + PublicationCoordinator publisher; + auto leaf = maintenance_leaf({"a"}, 1); + auto logical = std::make_shared<PageState>(*leaf->leaf); + OverlayBuilder builder({}); + auto tombstone = std::make_shared<CellVersion>(encode_cell_buf(2, OpKind::kDelete, {}), 2, kFlagTombstone); + ASSERT_TRUE(builder.append("b", std::move(tombstone)).ok()); + std::shared_ptr<const OverlayBatch> run; + ASSERT_TRUE(builder.finish(2, &run).ok()); + logical->overlays.push_back(std::move(run)); + leaf->leaf = logical; + const auto layout = plan_leaf_layout(logical, maintenance_limits().pages); + ASSERT_TRUE(layout.status.ok()); + leaf->metrics.page_size_known = true; + leaf->metrics.page_size_bytes = layout.total_primary_bytes; + leaf->metrics.page_size_estimate_bytes = layout.total_primary_bytes; + leaf->metrics.kv_known = true; + leaf->metrics.kv_count = 1; + leaf->metrics.kv_bytes = 2; + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 2; + publisher.request_gc_floor(2); + ASSERT_TRUE(publish_maintenance(publisher, {.record = maintenance_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = leaf}}, + .root_page_id = 0, + .coverage = 2, + .expected_sources = publisher.capture()->sources, + .sources = std::move(sources)}) + .status.ok()); + const auto before = publisher.capture(); + auto plan = build_coalesce_publication(before, 0, maintenance_record(1), 100, maintenance_limits()); + ASSERT_TRUE(plan.status.ok()) << plan.status.to_string(); + ASSERT_TRUE(publish_maintenance(publisher, std::move(plan.transaction)).status.ok()); + const auto after = publisher.capture()->mapping.get(0); + EXPECT_TRUE(after->metrics.page_size_known); + EXPECT_EQ(after->metrics.page_size_bytes, + kFrameHeaderSize + kFrameTrailerSize + kLeafSlotSize + kCellHeaderSize + 2); + EXPECT_EQ(after->metrics.kv_count, 1U); + EXPECT_EQ(after->metrics.kv_bytes, 2U); + EXPECT_EQ(after->leaf->lookup("b").state, LogicalLookupState::kMissing); + EXPECT_EQ(before->mapping.get(0)->leaf->lookup("b").state, LogicalLookupState::kTombstone); +} + +TEST(MaintenancePublication, CrossParentMergeAndMetadataExhaustionLeaveTreeUnchanged) +{ + PublicationCoordinator publisher; + seed_leaf(publisher, maintenance_leaf({"a", "b", "c", "d"}, 30)); + auto options = maintenance_limits(); + const auto original = publisher.capture(); + options.max_changed_pages = 1; + auto denied = build_split_publication(original, 0, maintenance_record(1), options); + EXPECT_EQ(denied.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(denied.changed()); + EXPECT_EQ(publisher.capture(), original); + options.max_changed_pages = 256; + auto split = build_split_publication(original, 0, maintenance_record(2), options); + ASSERT_TRUE(split.status.ok()); + ASSERT_TRUE(publish_maintenance(publisher, std::move(split.transaction)).status.ok()); + const auto before = publisher.capture(); + auto merged = build_merge_publication(before, 1, 2, maintenance_record(3), 100, options); + EXPECT_EQ(merged.status.code(), Code::kInvalidArgument); + EXPECT_FALSE(merged.changed()); + EXPECT_EQ(publisher.capture(), before); +} + +TEST(MaintenancePublication, StaleLeafPlanCannotPublishAnyOfItsStructuralChanges) +{ + PublicationCoordinator publisher; + auto original = maintenance_leaf({"a", "b", "c", "d"}, 30); + seed_leaf(publisher, original); + auto plan = build_split_publication(publisher.capture(), 0, maintenance_record(1), maintenance_limits()); + ASSERT_TRUE(plan.status.ok()); + auto newer = std::make_shared<PublishedPage>(*original); + ++newer->revision; + ASSERT_TRUE(publish_maintenance(publisher, {.record = maintenance_record(2), + .pages = {{.page_id = 0, .expected = original, .replacement = newer}}}) + .status.ok()); + const auto before = publisher.capture(); + EXPECT_EQ(publish_maintenance(publisher, std::move(plan.transaction)).status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture(), before); + EXPECT_EQ(before->stats.leaf_pages, 1U); + EXPECT_EQ(before->next_page_id, 1U); +} + +TEST(MaintenancePublication, RootPreconditionRejectsReparentingEvenWhenTargetLeafIsUnchanged) +{ + PublicationCoordinator publisher; + auto original = maintenance_leaf({"a"}, 1); + seed_leaf(publisher, original); + auto plan = build_coalesce_publication(publisher.capture(), 0, maintenance_record(1), 100); + ASSERT_TRUE(plan.status.ok()); + ASSERT_TRUE(plan.changed()); + auto parent = maintenance_inner({0}, {}); + ASSERT_TRUE(publish_maintenance(publisher, {.record = maintenance_record(2), + .pages = {{.page_id = 1, .expected = {}, .replacement = parent}}, + .root_page_id = 1}) + .status.ok()); + const auto before = publisher.capture(); + EXPECT_EQ(publish_maintenance(publisher, std::move(plan.transaction)).status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture(), before); + EXPECT_EQ(before->root_page_id, 1U); +} + +TEST(MaintenancePublication, CoalescePreservesUnknownSummaryAndLogicalIdentity) +{ + PublicationCoordinator publisher; + auto original = maintenance_leaf({"a", "b"}, 1); + auto logical = std::make_shared<PageState>(*original->leaf); + logical->overlays.push_back(logical->overlays.front()); + original->leaf = logical; + original->metrics.overlay_runs = 2; + original->metrics.overlay_entries = 4; + original->metrics.consolidate_candidate = true; + seed_leaf(publisher, original); + ASSERT_TRUE( + publish_maintenance(publisher, + { + .record = maintenance_record(1000), + .expected_coverage = 0, + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = 1} + }) + .status.ok()); + ASSERT_TRUE( + publish_maintenance(publisher, + { + .record = maintenance_record(1001), + .checkpoint = CheckpointPublication{.action = CheckpointAction::Commit, .token = 1} + }) + .status.ok()); + auto plan = build_coalesce_publication(publisher.capture(), 0, maintenance_record(1), 100); + ASSERT_TRUE(plan.status.ok()); + ASSERT_TRUE(publish_maintenance(publisher, std::move(plan.transaction)).status.ok()); + const auto after = publisher.capture(); + const auto page = after->mapping.get(0); + EXPECT_EQ(page->leaf->content_id, logical->content_id); + EXPECT_EQ(page->leaf->route_generation, logical->route_generation); + EXPECT_EQ(page->leaf->representation_id, logical->representation_id + 1); + EXPECT_EQ(after->stats.overlay_runs, 1U); + EXPECT_EQ(after->stats.overlay_entries, 2U); + EXPECT_EQ(after->stats.consolidate_candidate_pages, 0U); + EXPECT_EQ(after->stats.page_size_unknown_pages, 1U); + EXPECT_EQ(after->stats.kv_unknown_pages, 1U); + EXPECT_EQ(after->stats.checkpoint_pending_pages, 0U); + EXPECT_FALSE(after->stats.checkpoint_mapping_pending); +} + +TEST(MaintenancePublication, InnerSeparatorByteLimitRejectsBeforePublication) +{ + PublicationCoordinator publisher; + seed_leaf( + publisher, + maintenance_leaf({std::string(20, 'a'), std::string(20, 'b'), std::string(20, 'c'), std::string(20, 'd')}, 30)); + const auto before = publisher.capture(); + auto options = maintenance_limits(); + options.max_inner_bytes = kFrameHeaderSize + kFrameTrailerSize + (2 * sizeof(uint64_t)) + kInnerSlotSize + 1; + auto plan = build_split_publication(before, 0, maintenance_record(1), options); + EXPECT_EQ(plan.status.code(), Code::kInvalidArgument); + EXPECT_FALSE(plan.changed()); + EXPECT_EQ(publisher.capture(), before); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/maintenance_split_step_test.cpp b/lib/crowdb-tree/tests/unit/maintenance_split_step_test.cpp new file mode 100644 index 000000000..dcfc857a3 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/maintenance_split_step_test.cpp @@ -0,0 +1,216 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/maintenance_publication.h" + +#include <gtest/gtest.h> + +#include <algorithm> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<InstallRecord> record(uint64_t sequence) +{ + return std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 1, .range = 0, .sequence = sequence}); +} + +Status publish(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + Status status = Status::unavailable(); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + request->on_done = [&](PublicationResult result) { status = std::move(result.status); }; + EXPECT_TRUE(publisher.try_submit(request)); + return status; +} + +std::shared_ptr<PublishedPage> seed(PublicationCoordinator &publisher, size_t count) +{ + OverlayBuilder builder({}); + for (size_t i = 0; i < count; ++i) { + std::string key = std::to_string(1000 + i); + auto cell = std::make_shared<CellVersion>(encode_cell_buf(1, OpKind::kPut, std::string(16, 'v')), 1, 0); + EXPECT_TRUE(builder.append(key, cell).ok()); + } + std::shared_ptr<const OverlayBatch> run; + EXPECT_TRUE(builder.finish(1, &run).ok()); + auto logical = std::make_shared<PageState>(); + logical->overlays.push_back(run); + auto page = std::make_shared<PublishedPage>(); + page->leaf = logical; + page->metrics.dirty = true; + page->metrics.overlay_entries = count; + page->metrics.overlay_runs = 1; + EXPECT_TRUE(publish(publisher, {.record = record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0}) + .ok()); + return page; +} + +TEST(MaintenanceSplitStep, OversizedLeafMakesBoundedProgressWithSharedPayloadsAndOldView) +{ + PublicationCoordinator publisher; + auto original = seed(publisher, 80); + auto before = publisher.capture(); + MaintenancePublicationOptions options; + options.pages.max_work_entries = 3; + options.pages.max_output_pages = 2; + options.pages.primary_page_bytes = 512; + options.pages.target_page_bytes = 256; + options.max_inner_children = 3; + auto full = build_split_publication(before, 0, record(1), options); + EXPECT_EQ(full.status.code(), Code::kResourceExhausted); + EXPECT_EQ(publisher.capture(), before); + uint64_t remaining = 0; + size_t steps = 0; + for (; steps < 100; ++steps) { + auto plan = build_split_step_publication(publisher.capture(), remaining, record(steps + 2), options); + ASSERT_TRUE(plan.status.ok()) << plan.status.message(); + if (!plan.changed()) { + break; + } + const auto next = plan.output_leaf_pids.back(); + ASSERT_TRUE(publish(publisher, std::move(plan.transaction)).ok()); + auto current = publisher.capture(); + EXPECT_EQ(current->stats.overlay_entries, 80U); + if (plan.output_leaf_pids.size() == 2) { + EXPECT_FALSE(current->mapping.get(next)->metrics.page_size_known); + EXPECT_TRUE(current->mapping.get(next)->metrics.split_candidate); + } + remaining = next; + } + ASSERT_LT(steps, 100U); + const auto after = publisher.capture(); + EXPECT_GT(after->stats.leaf_pages, 2U); + EXPECT_EQ(after->stats.page_size_unknown_pages, 0U); + EXPECT_EQ(after->stats.kv_unknown_pages, 0U); + EXPECT_EQ(after->stats.kv_known_count, 80U); + size_t seen = 0; + uint64_t pid = 0; + while (pid != kInvalidPageId) { + const auto page = after->mapping.get(pid); + ASSERT_TRUE(page && page->leaf); + EXPECT_LE(page->metrics.page_size_bytes, options.pages.primary_page_bytes); + LogicalPageCursor cursor(page->leaf, page->leaf->range, UINT64_MAX, 64); + while (cursor.valid()) { + const auto key = std::to_string(1000 + seen++); + EXPECT_EQ(cursor.key().to_string(), key); + EXPECT_EQ(cursor.cell().owner, original->leaf->lookup(key).cell.owner); + cursor.next(); + } + pid = page->leaf->right_sibling; + } + EXPECT_EQ(seen, 80U); + EXPECT_EQ(before->mapping.get(0), original); + EXPECT_EQ(before->stats.leaf_pages, 1U); +} + +TEST(MaintenanceSplitStep, ColdInputRequestsLoadAndTinyTransactionBudgetLeavesRootUntouched) +{ + PublicationCoordinator publisher; + seed(publisher, 16); + auto before = publisher.capture(); + MaintenancePublicationOptions options; + options.pages.max_work_entries = 1; + options.max_changed_pages = 1; + auto denied = build_split_step_publication(before, 0, record(1), options); + EXPECT_EQ(denied.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(denied.changed()); + EXPECT_EQ(publisher.capture(), before); + + auto old = before->mapping.get(0); + auto logical = std::make_shared<PageState>(*old->leaf); + auto backing = std::make_shared<BackingRef>(); + backing->length = 4096; + backing->owner = std::make_shared<int>(1); + logical->bases.push_back({.identity = 1, .backing = backing}); + auto proxy = std::make_shared<PublishedPage>(*old); + proxy->leaf = logical; + ASSERT_TRUE( + publish(publisher, {.record = record(2), .pages = {{.page_id = 0, .expected = old, .replacement = proxy}}}) + .ok()); + auto current = publisher.capture(); + auto needs = build_split_step_publication(current, 0, record(3)); + ASSERT_TRUE(needs.status.ok()); + EXPECT_EQ(needs.readiness, MaintenanceReadiness::kNeedsBase); + EXPECT_FALSE(needs.changed()); + ASSERT_EQ(needs.loads.size(), 1U); + EXPECT_EQ(needs.loads.front().page, logical); + EXPECT_EQ(publisher.capture(), current); +} + +TEST(MaintenanceSplitStep, OneWindowPublishesSeveralExactRangesAndAnUnvisitedSuffix) +{ + PublicationCoordinator publisher; + const auto original = seed(publisher, 80); + const auto before = publisher.capture(); + MaintenancePublicationOptions options; + options.pages.primary_page_bytes = 512; + options.pages.target_page_bytes = 256; + options.pages.max_work_entries = 80; + options.pages.max_output_pages = 8; + auto plan = build_split_step_publication(before, 0, record(1), options); + ASSERT_TRUE(plan.status.ok()) << plan.status.to_string(); + ASSERT_EQ(plan.output_leaf_pids.size(), options.pages.max_output_pages); + ASSERT_TRUE(publish(publisher, std::move(plan.transaction)).ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->commit_seq, before->commit_seq + 1); + EXPECT_EQ(after->stats.leaf_pages, options.pages.max_output_pages); + EXPECT_EQ(after->stats.page_size_unknown_pages, 1U); + EXPECT_EQ(after->stats.overlay_entries, 80U); + size_t seen = 0; + for (size_t i = 0; i < plan.output_leaf_pids.size(); ++i) { + const auto page = after->mapping.get(plan.output_leaf_pids[i]); + ASSERT_TRUE(page && page->leaf); + if (i + 1 < plan.output_leaf_pids.size()) { + EXPECT_TRUE(page->metrics.page_size_known); + EXPECT_LE(page->metrics.page_size_bytes, options.pages.target_page_bytes); + EXPECT_FALSE(page->metrics.split_candidate); + } + else { + EXPECT_FALSE(page->metrics.page_size_known); + EXPECT_TRUE(page->metrics.split_candidate); + } + LogicalPageCursor cursor(page->leaf, page->leaf->range, UINT64_MAX, 64); + while (cursor.valid()) { + const auto key = std::to_string(1000 + seen++); + EXPECT_EQ(cursor.key().to_string(), key); + EXPECT_EQ(cursor.cell().owner, original->leaf->lookup(key).cell.owner); + cursor.next(); + } + } + EXPECT_EQ(seen, 80U); + EXPECT_EQ(before->mapping.get(0), original); + EXPECT_EQ(before->stats.leaf_pages, 1U); +} + +TEST(MaintenanceSplitStep, WorkLimitCountsAllOutputRangesAndTransactionFailurePublishesNothing) +{ + PublicationCoordinator publisher; + seed(publisher, 80); + const auto before = publisher.capture(); + MaintenancePublicationOptions options; + options.pages.primary_page_bytes = 512; + options.pages.target_page_bytes = 256; + options.pages.max_work_entries = 5; + auto plan = build_split_step_publication(before, 0, record(1), options); + ASSERT_TRUE(plan.status.ok()) << plan.status.to_string(); + const auto suffix_pid = plan.output_leaf_pids.back(); + const auto suffix = std::find_if(plan.transaction.pages.begin(), plan.transaction.pages.end(), + [&](const auto &page) { return page.page_id == suffix_pid; }); + ASSERT_NE(suffix, plan.transaction.pages.end()); + if (!suffix->replacement->leaf->range.lower) { + FAIL() << "split suffix must retain its lower boundary"; + } + EXPECT_EQ(Slice(*suffix->replacement->leaf->range.lower).to_string(), "1005"); + options.max_changed_pages = 2; + const auto rejected = build_split_step_publication(before, 0, record(2), options); + EXPECT_EQ(rejected.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(rejected.changed()); + EXPECT_TRUE(rejected.output_leaf_pids.empty()); + EXPECT_EQ(publisher.capture(), before); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/maintenance_window_test.cpp b/lib/crowdb-tree/tests/unit/maintenance_window_test.cpp new file mode 100644 index 000000000..5e9b5a4ce --- /dev/null +++ b/lib/crowdb-tree/tests/unit/maintenance_window_test.cpp @@ -0,0 +1,149 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_reservation_internal.h" +#include "flush_preparation_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +TEST(MaintenanceWindow, UsesTheLargestCompletelyFundedOutputWindow) +{ + TestPreparedFlushTree tree; + const auto view = tree.publisher.capture(); + auto limits = tree.preparation.maintenance; + limits.publication.pages.max_output_pages = 7; + const auto bytes = page_maintenance_reserve_bound(view->mapping, 0, limits, false, true); + if (!bytes) { + FAIL() << "maintenance bound missing"; + } + limits.publication.pages.max_output_pages = 64; + limit_page_maintenance_window(view->mapping, 0, *bytes, SIZE_MAX, false, &limits); + EXPECT_EQ(limits.publication.pages.max_output_pages, 7U); + EXPECT_EQ(page_maintenance_reserve_bound(view->mapping, 0, limits, false, true), bytes); + limits.publication.pages.max_output_pages = 8; + EXPECT_GT(page_maintenance_reserve_bound(view->mapping, 0, limits, false, true), bytes); +} + +TEST(MaintenanceWindow, PreservesMinimumProgressAndNeverExceedsTheWorkLimit) +{ + TestPreparedFlushTree tree; + const auto view = tree.publisher.capture(); + auto limits = tree.preparation.maintenance; + limits.publication.pages.max_output_pages = 64; + limit_page_maintenance_window(view->mapping, 0, 0, 0, false, &limits); + EXPECT_EQ(limits.publication.pages.max_output_pages, 2U); + limits.publication.pages.max_output_pages = 64; + limits.publication.pages.max_work_entries = 5; + limit_page_maintenance_window(view->mapping, 0, SIZE_MAX, SIZE_MAX, false, &limits); + EXPECT_EQ(limits.publication.pages.max_output_pages, 6U); +} + +TEST(MaintenanceWindow, FlushBindsTheChosenWindowAndFinishesUsingItsHeldCredit) +{ + TestPreparedFlushTree tree; + tree.frontier.store(1); + tree.preparation.maintenance.publication.pages.max_output_pages = 64; + const auto view = tree.publisher.capture(); + auto limits = tree.preparation.maintenance; + limits.publication.pages.max_output_pages = 7; + // Ordinary admission is paused, but the missing-slot allowance stays open. + const auto gap_entries = tree.sources.catalog()->active_budget->options().max_gap_entries; + const auto bytes = page_maintenance_reserve_bound(view->mapping, gap_entries, limits, false, true); + if (!bytes) { + FAIL() << "maintenance bound missing"; + } + const auto available = detail::ResourceBudget::available(tree.account, detail::BudgetClass::kReclaim); + ASSERT_GT(available, *bytes); + auto blocker = detail::ResourceBudget::try_reserve(tree.account, detail::BudgetClass::kReclaim, available - *bytes); + ASSERT_TRUE(blocker); + FlushReservePlan plan; + std::shared_ptr<FlushOperation> operation; + auto status = tree.controller->prepare_or_plan(tree.preparation, &operation, &plan); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(operation); + ASSERT_EQ(plan.bounds.maintenance_output_pages, 7U); + detail::FlushCredits credits; + for (size_t index = 0; index < credits.size(); ++index) { + auto credit = detail::ResourceBudget::try_reserve(tree.account, detail::FlushReserveBounds::categories[index], + plan.bounds.amounts[index]); + if (!credit) { + FAIL() << "flush grant missing: " << index; + } + credits[index] = std::move(*credit); + } + status = tree.controller->prepare_prepaid(tree.preparation, plan, std::move(credits), &operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(stdexec::sync_wait(operation->sender())); + tree.runtime->drain(); + EXPECT_TRUE(operation->result().handoff_status.ok()) << operation->result().handoff_status.to_string(); + EXPECT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.to_string(); + EXPECT_EQ(operation->result().published_coverage, 1U); +} + +TEST(MaintenanceWindow, IdleReacquisitionExpandsAndShrinksWithCurrentPressure) +{ + TestPreparedFlushTree tree; + const auto view = tree.publisher.capture(); + auto options = tree.preparation.maintenance; + options.publication.pages.max_output_pages = 64; + detail::ReclaimAdmission admission{.publisher = &tree.publisher, .account = tree.account, .options = options}; + auto seven = options; + seven.publication.pages.max_output_pages = 7; + const auto bound = page_maintenance_reserve_bound(view->mapping, 0, seven, false, true); + if (!bound) { + FAIL() << "missing pressure fixture bound"; + } + const auto available = detail::ResourceBudget::available(tree.account, detail::BudgetClass::kReclaim); + ASSERT_GT(available, *bound); + auto blocker = detail::ResourceBudget::try_reserve(tree.account, detail::BudgetClass::kReclaim, available - *bound); + ASSERT_TRUE(blocker); + MaintenancePublicationOptions selected; + std::shared_ptr<detail::AllocationReservation> credit; + auto status = admission.acquire_window(view->mapping, &selected, &credit); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(selected.pages.max_output_pages, 7U); + ASSERT_TRUE(credit); + EXPECT_GT(credit->available(), 0U); + EXPECT_EQ(detail::ResourceBudget::available(tree.account, detail::BudgetClass::kReclaim), 0U); + credit.reset(); + selected.snapshot_metadata.reset(); + blocker.reset(); + auto expected = options; + limit_page_maintenance_window( + view->mapping, 0, detail::ResourceBudget::available(tree.account, detail::BudgetClass::kReclaim), + detail::ResourceBudget::available(tree.account, detail::BudgetClass::kRouting), false, &expected); + ASSERT_GT(expected.publication.pages.max_output_pages, 7U); + status = admission.acquire_window(view->mapping, &selected, &credit); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(selected.pages.max_output_pages, expected.publication.pages.max_output_pages); + credit.reset(); + selected.snapshot_metadata.reset(); + blocker = detail::ResourceBudget::try_reserve(tree.account, detail::BudgetClass::kReclaim, available - *bound); + ASSERT_TRUE(blocker); + status = admission.acquire_window(view->mapping, &selected, &credit); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(selected.pages.max_output_pages, 7U); +} + +TEST(MaintenanceWindow, FailedIdleAdmissionDoesNotChangeTheWindowOrHoldPartialCredit) +{ + TestPreparedFlushTree tree; + const auto view = tree.publisher.capture(); + detail::ReclaimAdmission admission{ + .publisher = &tree.publisher, .account = tree.account, .options = tree.preparation.maintenance}; + const auto available = detail::ResourceBudget::available(tree.account, detail::BudgetClass::kReclaim); + auto blocker = detail::ResourceBudget::try_reserve(tree.account, detail::BudgetClass::kReclaim, available); + ASSERT_TRUE(blocker); + const auto routing = detail::ResourceBudget::available(tree.account, detail::BudgetClass::kRouting); + MaintenancePublicationOptions selected; + selected.pages.max_output_pages = 7; + std::shared_ptr<detail::AllocationReservation> credit; + EXPECT_EQ(admission.acquire_window(view->mapping, &selected, &credit).code(), Code::kUnavailable); + EXPECT_FALSE(credit); + EXPECT_FALSE(selected.snapshot_metadata); + EXPECT_EQ(selected.pages.max_output_pages, 7U); + EXPECT_EQ(detail::ResourceBudget::available(tree.account, detail::BudgetClass::kRouting), routing); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/mapping_segment_test.cpp b/lib/crowdb-tree/tests/unit/mapping_segment_test.cpp index 72d6d6431..102999125 100644 --- a/lib/crowdb-tree/tests/unit/mapping_segment_test.cpp +++ b/lib/crowdb-tree/tests/unit/mapping_segment_test.cpp @@ -2,8 +2,8 @@ // Licensed under the Apache License, Version 2.0. // #14a: standalone mapping-table segment struct. -#include "crowdb-tree/maptable/mapping_segment.h" #include "crowdb-tree/maptable/mapping_slot.h" +#include "mapping_segment_test_support.h" #include <gtest/gtest.h> @@ -11,7 +11,7 @@ using namespace crowdb::tree; TEST(MappingSegment, ConstructsAllSlotsEmpty) { - MappingSegment seg(128); + TestMappingSegment seg(128); EXPECT_EQ(seg.slot_count, 128U); for (uint32_t i = 0; i < seg.slot_count; ++i) { EXPECT_TRUE(slot_word::is_empty(seg.slots[i].load())); @@ -23,8 +23,8 @@ TEST(MappingSegment, ConstructsAllSlotsEmpty) TEST(MappingSegment, SlotsAreIndependentlyMutable) { - MappingSegment seg(4); - uint64_t w = slot_word::pack_unloaded(42, 7); + TestMappingSegment seg(4); + uint64_t w = slot_word::pack_unloaded(42, 7); seg.slots[2].store(w); seg.live_count.fetch_add(1); seg.write_seq.fetch_add(1); @@ -41,7 +41,7 @@ TEST(MappingSegment, SlotsAreIndependentlyMutable) TEST(MappingSegment, GenerationBumpsIndependentlyOfSlots) { - MappingSegment seg(8); + TestMappingSegment seg(8); seg.generation.fetch_add(1); seg.generation.fetch_add(1); EXPECT_EQ(seg.generation.load(), 2U); @@ -55,6 +55,6 @@ TEST(MappingSegment, GenerationBumpsIndependentlyOfSlots) TEST(MappingSegment, DefaultSizeMatchesOptionsDefault) { // Config.mapping_segment_slots default is 1024. - MappingSegment seg(1024); + TestMappingSegment seg(1024); EXPECT_EQ(seg.slot_count, 1024U); } diff --git a/lib/crowdb-tree/tests/unit/mapping_table_test.cpp b/lib/crowdb-tree/tests/unit/mapping_table_test.cpp index 62e399f5c..4b819cf5a 100644 --- a/lib/crowdb-tree/tests/unit/mapping_table_test.cpp +++ b/lib/crowdb-tree/tests/unit/mapping_table_test.cpp @@ -4,8 +4,8 @@ // CT5: mapping table tests (alloc/free/recycle, growth, atomic store/load). #include "crowdb-tree/epoch.h" #include "crowdb-tree/maptable/mapping_slot.h" -#include "crowdb-tree/maptable/mapping_table.h" #include "crowdb-tree/maptable/page.h" +#include "mapping_table_test_support.h" #include <gtest/gtest.h> @@ -27,8 +27,8 @@ PageBase *make_leaf() TEST(MappingTable, AllocStoreGet) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); EXPECT_NE(page_id, kInvalidPageId); EXPECT_EQ(mt.get_resident(page_id), nullptr); @@ -41,7 +41,7 @@ TEST(MappingTable, AllocStoreGet) TEST(MappingTable, Invalidpage_id) { - MappingTable mt; + TestMappingTable mt; EXPECT_EQ(mt.get_resident(kInvalidPageId), nullptr); EXPECT_EQ(mt.get_resident(123456), nullptr); // never allocated } @@ -49,8 +49,8 @@ TEST(MappingTable, Invalidpage_id) TEST(MappingTable, AllocatePageIdIsMonotonic) { // Plan-tree #14 D1: PIDs are never recycled, even across many allocations. - MappingTable mt; - uint64_t prev = mt.allocate_page_id(); + TestMappingTable mt; + uint64_t prev = mt.allocate_page_id(); for (int i = 0; i < 100; ++i) { uint64_t next = mt.allocate_page_id(); EXPECT_GT(next, prev); @@ -60,17 +60,17 @@ TEST(MappingTable, AllocatePageIdIsMonotonic) TEST(MappingTable, SegmentGrowth) { - MappingTable mt; + TestMappingTable mt; // Allocate enough PIDs to span multiple segments. std::vector<uint64_t> pids; - pids.reserve((MappingTable::kSegmentSize * 3) + 5); - for (uint64_t i = 0; i < (MappingTable::kSegmentSize * 3) + 5; ++i) { + pids.reserve((TestMappingTable::kSegmentSize * 3) + 5); + for (uint64_t i = 0; i < (TestMappingTable::kSegmentSize * 3) + 5; ++i) { pids.push_back(mt.allocate_page_id()); } EXPECT_GE(mt.segments_allocated(), 4U); // store + read back across segment boundaries. PageBase *page = make_leaf(); - uint64_t cross = pids[MappingTable::kSegmentSize + 1]; + uint64_t cross = pids[TestMappingTable::kSegmentSize + 1]; mt.store(cross, page); EXPECT_EQ(mt.get_resident(cross), page); delete page; @@ -78,9 +78,9 @@ TEST(MappingTable, SegmentGrowth) TEST(MappingTable, ConcurrentReadersSingleWriter) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); - PageBase *p1 = make_leaf(); + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); + PageBase *p1 = make_leaf(); mt.store(page_id, p1); std::atomic<bool> stop{false}; @@ -118,8 +118,8 @@ TEST(MappingTable, ConcurrentReadersSingleWriter) TEST(MappingTable, StoreUnloadedGetWord) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); // Store an unloaded descriptor (addr=4096, plen=512, iu=512). uint32_t iu = 512; uint64_t addr = 4096; @@ -139,11 +139,11 @@ TEST(MappingTable, StoreUnloadedGetWord) TEST(MappingTable, StoreUnloadedRoundUpIu) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); - uint32_t iu = 4096; - uint64_t addr = 8192; - uint32_t plen = 100; // smaller than one IU -> rounds up to 1 IU + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); + uint32_t iu = 4096; + uint64_t addr = 8192; + uint32_t plen = 100; // smaller than one IU -> rounds up to 1 IU mt.store_unloaded(page_id, addr, plen, iu); uint64_t w = mt.get_word(page_id); @@ -154,9 +154,9 @@ TEST(MappingTable, StoreUnloadedRoundUpIu) TEST(MappingTable, ClearSlot) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); - PageBase *page = make_leaf(); + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); + PageBase *page = make_leaf(); mt.store(page_id, page); EXPECT_TRUE(is_resident(mt.get_word(page_id))); @@ -168,9 +168,9 @@ TEST(MappingTable, ClearSlot) TEST(MappingTable, StoreWordPackedUnloaded) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); - uint64_t word = pack_unloaded(42, 7); + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); + uint64_t word = pack_unloaded(42, 7); mt.store_word(page_id, word); uint64_t got = mt.get_word(page_id); @@ -186,12 +186,12 @@ TEST(MappingTable, SegmentRecycledWhenLastSlotCleared) { // No epoch manager wired: recycled segments are deleted immediately on // the writer thread (fine -- no concurrent readers in this test). - MappingTable mt; + TestMappingTable mt; std::vector<uint64_t> pids; std::vector<PageBase *> pages; - pids.reserve(MappingTable::kSegmentSize); - pages.reserve(MappingTable::kSegmentSize); - for (uint64_t i = 0; i < MappingTable::kSegmentSize; ++i) { + pids.reserve(TestMappingTable::kSegmentSize); + pages.reserve(TestMappingTable::kSegmentSize); + for (uint64_t i = 0; i < TestMappingTable::kSegmentSize; ++i) { uint64_t pid = mt.allocate_page_id(); PageBase *p = make_leaf(); mt.store(pid, p); @@ -203,7 +203,8 @@ TEST(MappingTable, SegmentRecycledWhenLastSlotCleared) for (uint64_t i = 0; i + 1 < pids.size(); ++i) { mt.clear(pids[i]); } - EXPECT_EQ(mt.segments_allocated(), 1U); // one live slot left -> not recyclable + EXPECT_EQ(mt.segments_allocated(), + 1U); // one live slot left -> not recyclable mt.clear(pids.back()); // last live slot -> segment recycles EXPECT_EQ(mt.segments_allocated(), 0U); @@ -219,11 +220,11 @@ TEST(MappingTable, SegmentRecycledWhenLastSlotCleared) TEST(MappingTable, SegmentNotRecycledWhilePartiallyLive) { - MappingTable mt; - uint64_t a = mt.allocate_page_id(); - uint64_t b = mt.allocate_page_id(); - PageBase *pa = make_leaf(); - PageBase *pb = make_leaf(); + TestMappingTable mt; + uint64_t a = mt.allocate_page_id(); + uint64_t b = mt.allocate_page_id(); + PageBase *pa = make_leaf(); + PageBase *pb = make_leaf(); mt.store(a, pa); mt.store(b, pb); ASSERT_EQ(mt.segments_allocated(), 1U); @@ -243,13 +244,13 @@ TEST(MappingTable, SegmentNotRecycledWhilePartiallyLive) TEST(MappingTable, SegmentAtReflectsAllocationAndRecycling) { - MappingTable mt; + TestMappingTable mt; EXPECT_EQ(mt.segment_at(0), nullptr); uint64_t page_id = mt.allocate_page_id(); PageBase *page = make_leaf(); mt.store(page_id, page); - MappingSegment *seg = mt.segment_at(0); + TestMappingSegment *seg = mt.segment_at(0); ASSERT_NE(seg, nullptr); EXPECT_TRUE(seg->is_dirty()); @@ -260,12 +261,12 @@ TEST(MappingTable, SegmentAtReflectsAllocationAndRecycling) TEST(MappingTable, CommitSegmentPersistMarksClean) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); - PageBase *page = make_leaf(); + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); + PageBase *page = make_leaf(); mt.store(page_id, page); - MappingSegment *seg = mt.segment_at(0); + TestMappingSegment *seg = mt.segment_at(0); ASSERT_NE(seg, nullptr); uint64_t seen_seq = seg->write_seq.load(); ASSERT_TRUE(seg->is_dirty()); @@ -283,16 +284,17 @@ TEST(MappingTable, CommitSegmentPersistMarksClean) TEST(MappingTable, CommitSegmentPersistSkippedIfWrittenAgainDuringGap) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); - PageBase *page = make_leaf(); + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); + PageBase *page = make_leaf(); mt.store(page_id, page); - MappingSegment *seg = mt.segment_at(0); - uint64_t seen_seq = seg->write_seq.load(); - uint64_t page_id_2 = mt.allocate_page_id(); - PageBase *page2 = make_leaf(); - mt.store(page_id_2, page2); // "concurrent" write during the prepare->commit gap + TestMappingSegment *seg = mt.segment_at(0); + uint64_t seen_seq = seg->write_seq.load(); + uint64_t page_id_2 = mt.allocate_page_id(); + PageBase *page2 = make_leaf(); + mt.store(page_id_2, + page2); // "concurrent" write during the prepare->commit gap // The image we "prepared" (at seen_seq) is now stale -- commit must refuse. EXPECT_FALSE(mt.commit_segment_persist(0, seg, seen_seq, 1, 4096, 8224, 0xABCD)); @@ -304,13 +306,13 @@ TEST(MappingTable, CommitSegmentPersistSkippedIfWrittenAgainDuringGap) TEST(MappingTable, CommitSegmentPersistSkippedIfRecycledDuringGap) { - MappingTable mt; - uint64_t page_id = mt.allocate_page_id(); - PageBase *page = make_leaf(); + TestMappingTable mt; + uint64_t page_id = mt.allocate_page_id(); + PageBase *page = make_leaf(); mt.store(page_id, page); - MappingSegment *seg = mt.segment_at(0); - uint64_t seen_seq = seg->write_seq.load(); + TestMappingSegment *seg = mt.segment_at(0); + uint64_t seen_seq = seg->write_seq.load(); mt.clear(page_id); // recycles segment 0 (no epoch wired -> deleted immediately) EXPECT_EQ(mt.segment_at(0), nullptr); @@ -325,14 +327,15 @@ TEST(MappingTable, CommitSegmentPersistSkippedIfRecycledDuringGap) TEST(MappingTable, InstallRecoveredSegmentIsNotDirty) { - MappingTable mt; - std::vector<uint64_t> words(MappingTable::kSegmentSize, 0); + TestMappingTable mt; + std::vector<uint64_t> words(TestMappingTable::kSegmentSize, 0); words[5] = pack_unloaded(42, 3); - mt.install_recovered_segment(2, /*generation=*/7, /*live_count=*/1, words, /*image_addr=*/8192, + mt.install_recovered_segment(2, /*generation=*/7, /*live_count=*/1, words, + /*image_addr=*/8192, /*image_len=*/8224, /*image_crc=*/0x99); - MappingSegment *seg = mt.segment_at(2); + TestMappingSegment *seg = mt.segment_at(2); ASSERT_NE(seg, nullptr); EXPECT_FALSE(seg->is_dirty()); EXPECT_EQ(seg->generation.load(), 7U); @@ -342,7 +345,7 @@ TEST(MappingTable, InstallRecoveredSegmentIsNotDirty) EXPECT_EQ(seg->image_crc, 0x99U); // Recovered PID 2*kSegmentSize + 5 reads back as the installed descriptor. - uint64_t recovered_pid = (2 * MappingTable::kSegmentSize) + 5; + uint64_t recovered_pid = (2 * TestMappingTable::kSegmentSize) + 5; uint64_t w = mt.get_word(recovered_pid); EXPECT_TRUE(is_unloaded(w)); EXPECT_EQ(unloaded_iu_index(w), 42U); @@ -351,8 +354,8 @@ TEST(MappingTable, InstallRecoveredSegmentIsNotDirty) TEST(MappingTable, RecycledSegmentFreedOnlyAfterEpochGuardDrains) { - MappingTable mt; - EpochManager epoch; + TestMappingTable mt; + EpochManager epoch; mt.set_epoch_manager(&epoch); uint64_t page_id = mt.allocate_page_id(); @@ -368,7 +371,8 @@ TEST(MappingTable, RecycledSegmentFreedOnlyAfterEpochGuardDrains) mt.clear(page_id); // last live slot -> segment CAS'd out and retired EXPECT_EQ(mt.segments_allocated(), 0U); EXPECT_EQ(epoch.pending_retired(), 1U); - EXPECT_EQ(epoch.try_reclaim(), 0U); // the open guard could still see it -> not freed + EXPECT_EQ(epoch.try_reclaim(), + 0U); // the open guard could still see it -> not freed } EXPECT_EQ(epoch.try_reclaim(), 1U); // now safe to free EXPECT_EQ(epoch.pending_retired(), 0U); diff --git a/lib/crowdb-tree/tests/unit/memory_backing_failure_test.cpp b/lib/crowdb-tree/tests/unit/memory_backing_failure_test.cpp new file mode 100644 index 000000000..26e1bd728 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/memory_backing_failure_test.cpp @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "backend/local/memory_backing_test.h" +#include "crowdb-tree/backend/page_store.h" + +#include <gtest/gtest.h> + +#include <array> +#include <vector> + +using namespace crowdb::tree; + +TEST(MemoryBacking, FailedGrowthReleasesEveryPreparedPageAndPreservesStoredBytes) +{ + MemPageStore store; + const std::array<uint8_t, 3> prefix{1, 2, 3}; + ASSERT_TRUE(store.write_at(0, prefix.data(), prefix.size()).ok()); + const auto before = store.resident_bytes(); + std::vector<uint8_t> payload(3 * 64 * 1024, 0x55); + set_memory_backing_failure_after_for_tests(1); + const auto failed = store.write_at(64 * 1024, payload.data(), payload.size()); + set_memory_backing_failure_after_for_tests(std::nullopt); + ASSERT_EQ(failed.code(), Code::kResourceExhausted); + EXPECT_EQ(store.resident_bytes(), before); + EXPECT_EQ(store.size(), prefix.size()); + std::array<uint8_t, 3> first{}; + ASSERT_TRUE(store.read_at(0, first.data(), first.size()).ok()); + EXPECT_EQ(first, prefix); + ASSERT_TRUE(store.write_at(64 * 1024, payload.data(), payload.size()).ok()); + std::vector<uint8_t> actual(payload.size()); + ASSERT_TRUE(store.read_at(64 * 1024, actual.data(), actual.size()).ok()); + EXPECT_EQ(actual, payload); + ASSERT_TRUE(store.read_at(0, first.data(), first.size()).ok()); + EXPECT_EQ(first, prefix); +} diff --git a/lib/crowdb-tree/tests/unit/memory_backing_test.cpp b/lib/crowdb-tree/tests/unit/memory_backing_test.cpp new file mode 100644 index 000000000..748553931 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/memory_backing_test.cpp @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-tree/backend/page_store.h" + +#include <gtest/gtest.h> + +#include <algorithm> +#include <array> +#include <limits> +#include <vector> + +using namespace crowdb::tree; + +TEST(MemoryBacking, SparseGrowthAllocatesWrittenPagesWithoutMaterializingTheAddressSpace) +{ + MemPageStore store; + const std::array<uint8_t, 8> prefix{1, 2, 3, 4, 5, 6, 7, 8}; + const uint8_t suffix = 0xab; + constexpr uint64_t offset = 200ULL * 1024 * 1024; + ASSERT_TRUE(store.write_at(0, prefix.data(), prefix.size()).ok()); + ASSERT_TRUE(store.write_at(offset, &suffix, 1).ok()); + EXPECT_EQ(store.size(), offset + 1); + EXPECT_LT(store.resident_bytes(), 1024 * 1024); + std::array<uint8_t, 8> read{}; + ASSERT_TRUE(store.read_at(0, read.data(), read.size()).ok()); + EXPECT_EQ(read, prefix); + read.fill(0xff); + ASSERT_TRUE(store.read_at(offset / 2, read.data(), read.size()).ok()); + EXPECT_EQ(read, (std::array<uint8_t, 8>{})); + uint8_t last = 0; + ASSERT_TRUE(store.read_at(offset, &last, 1).ok()); + EXPECT_EQ(last, suffix); +} + +TEST(MemoryBacking, CrossPageWritesAndOverwritesPreserveBothEnds) +{ + MemPageStore store; + constexpr uint64_t offset = (64 * 1024) - 3; + std::vector<uint8_t> input((3 * 64 * 1024) + 7); + for (size_t index = 0; index < input.size(); ++index) { + input[index] = static_cast<uint8_t>(index % 251); + } + ASSERT_TRUE(store.write_at(offset, input.data(), input.size()).ok()); + const std::array<uint8_t, 9> patch{91, 92, 93, 94, 95, 96, 97, 98, 99}; + ASSERT_TRUE(store.write_at((128 * 1024) - 4, patch.data(), patch.size()).ok()); + std::copy(patch.begin(), patch.end(), input.begin() + ((64 * 1024) - 1)); + std::vector<uint8_t> output(input.size()); + ASSERT_TRUE(store.read_at(offset, output.data(), output.size()).ok()); + EXPECT_EQ(output, input); + uint8_t hole = 0xff; + ASSERT_TRUE(store.read_at(offset - 1, &hole, 1).ok()); + EXPECT_EQ(hole, 0); +} + +TEST(MemoryBacking, LargeSparseAddressesAndRangeOverflowRemainBounded) +{ + MemPageStore store; + constexpr auto end = std::numeric_limits<uint64_t>::max(); + const uint8_t value = 0xcd; + ASSERT_TRUE(store.write_at(end - 1, &value, 1).ok()); + EXPECT_EQ(store.size(), end); + EXPECT_LT(store.resident_bytes(), 1024 * 1024); + uint8_t read = 0; + ASSERT_TRUE(store.read_at(end - 1, &read, 1).ok()); + EXPECT_EQ(read, value); + EXPECT_EQ(store.write_at(end, &value, 1).code(), Code::kInvalidArgument); + EXPECT_EQ(store.write_at(end - 1, &value, 2).code(), Code::kInvalidArgument); + EXPECT_FALSE(store.read_at(end - 1, &read, 2).ok()); + EXPECT_EQ(store.size(), end); +} + +TEST(MemoryBacking, EmptyAndInvalidTransfersDoNotGrowTheBacking) +{ + MemPageStore store; + EXPECT_TRUE(store.write_at(100, nullptr, 0).ok()); + EXPECT_TRUE(store.read_at(0, nullptr, 0).ok()); + EXPECT_EQ(store.write_at(0, nullptr, 1).code(), Code::kInvalidArgument); + EXPECT_EQ(store.size(), 0); + EXPECT_EQ(store.resident_bytes(), 0); +} diff --git a/lib/crowdb-tree/tests/unit/memtable_budget_test.cpp b/lib/crowdb-tree/tests/unit/memtable_budget_test.cpp new file mode 100644 index 000000000..00e1dfb5f --- /dev/null +++ b/lib/crowdb-tree/tests/unit/memtable_budget_test.cpp @@ -0,0 +1,272 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "crowdb-tree/memtable/memtable.h" +#include "runtime/tree_runtime_binding.h" + +#include <gtest/gtest.h> + +#include <semaphore> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kWriter = static_cast<size_t>(detail::BudgetClass::kAdmittedWriter); + +detail::ResourceBudget::Limits limits() +{ + detail::ResourceBudget::Limits result{}; + result[kWriter] = 1024 * 1024; + return result; +} + +struct TestCredit +{ + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::AllocationReservation> reservation; + + explicit TestCredit(std::shared_ptr<detail::RetirementQueue> queue = {}) + { + if (queue) { + budget.set_retirement_queue(std::move(queue)); + } + auto credit = detail::ResourceBudget::try_reserve(budget.make_account(limits()), + detail::BudgetClass::kAdmittedWriter, limits()[kWriter]); + if (!credit) { + throw std::bad_alloc(); + } + reservation = std::make_shared<detail::AllocationReservation>(std::move(*credit)); + } + + [[nodiscard]] size_t allocated() const + { + return budget.snapshot().allocated[kWriter]; + } +}; + +class TestFailBudget final : public AllocationBudget +{ + public: + explicit TestFailBudget(std::shared_ptr<AllocationBudget> owner) : owner_(std::move(owner)) + { + } + + size_t fail_after = SIZE_MAX; + + bool claim(size_t bytes) noexcept override + { + if (fail_after == 0) { + return false; + } + --fail_after; + return owner_->claim(bytes); + } + + void release(size_t bytes) noexcept override + { + owner_->release(bytes); + } + + private: + std::shared_ptr<AllocationBudget> owner_; +}; + +TEST(MemTableBudget, PayloadCapacityStaysChargedThroughIndependentCellOwner) +{ + TestCredit credit; + auto table = std::make_shared<MemTable>(1, nullptr, credit.reservation); + auto payload = buffer::alloc(4096); + const auto encoded = encode_cell_buf(1, OpKind::kPut, "value"); + std::memcpy(payload.data(), encoded.data(), encoded.size()); + payload.set_size(encoded.size()); + ASSERT_TRUE(table->upsert("key", 1, std::move(payload))); + EXPECT_GE(credit.allocated(), 4096U + sizeof(CellVersion)); + std::shared_ptr<const CellVersion> cell; + { + auto guard = table->read_guard(); + cell = table->prefix_cursor(1).cell_owner(); + } + credit.reservation->seal(); + table.reset(); + EXPECT_GE(credit.allocated(), 4096U + sizeof(CellVersion)); + EXPECT_EQ(CellView(cell->cell.slice()).value().to_string(), "value"); + EXPECT_EQ(credit.budget.snapshot().reserved[kWriter], 0U); + cell.reset(); + EXPECT_EQ(credit.allocated(), 0U); +} + +TEST(MemTableBudget, FrozenReaderNeedsNoNewAllocationAndRetainsItsChargedPayload) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + TestCredit credit(binding->lifecycle_budget()->retirement_queue()); + auto table = std::make_shared<MemTable>(1, nullptr, credit.reservation); + ASSERT_TRUE(table->upsert("key", 1, encode_cell_buf(1, OpKind::kPut, "value"), 0)); + auto guard = table->read_guard(); + table->close(); + credit.reservation->seal(); + const auto allocated = credit.allocated(); + std::jthread reader([owner = table] { + auto frozen_guard = owner->read_guard(); + ASSERT_NE(owner->find("key"), nullptr); + EXPECT_EQ(CellView(owner->find("key")->cell.slice()).value(), Slice("value")); + }); + reader.join(); + EXPECT_EQ(credit.allocated(), allocated); + table.reset(); + credit.reservation->account()->drain_retirements(); + EXPECT_GT(credit.allocated(), 0U); + guard = {}; + credit.reservation->account()->drain_retirements(); + EXPECT_EQ(credit.allocated(), 0U); +} + +TEST(MemTableBudget, FreezeRejectsAnEpochRegistrationThatAlreadyReadTheOldHead) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + TestCredit credit(binding->lifecycle_budget()->retirement_queue()); + auto table = std::make_shared<MemTable>(1, nullptr, credit.reservation); + ASSERT_TRUE(table->upsert("key", 1, encode_cell_buf(1, OpKind::kPut, "value"), 0)); + auto old_guard = table->read_guard(); + auto &epoch = table->epoch_for_tests(); + std::binary_semaphore reached{0}; + std::binary_semaphore proceed{0}; + + struct Pause + { + std::binary_semaphore &reached; + std::binary_semaphore &proceed; + } pause{.reached = reached, .proceed = proceed}; + + epoch.set_before_participant_append_for_tests( + [](void *context) { + auto &pause = *static_cast<Pause *>(context); + pause.reached.release(); + pause.proceed.acquire(); + }, + &pause); + std::jthread reader([owner = table] { + auto guard = owner->read_guard(); + ASSERT_NE(owner->find("key"), nullptr); + EXPECT_EQ(owner->find("key")->slot, 1U); + }); + reached.acquire(); + table->close(); + proceed.release(); + reader.join(); + epoch.set_before_participant_append_for_tests(nullptr, nullptr); + EXPECT_TRUE(epoch.source_sealed()); + EXPECT_EQ(epoch.active_guards(), 1U); + old_guard = {}; + EXPECT_EQ(epoch.active_guards(), 0U); + credit.reservation->seal(); + table.reset(); + credit.reservation->account()->drain_retirements(); + EXPECT_EQ(credit.allocated(), 0U); +} + +TEST(MemTableBudget, EveryOverwriteAllocationFailureKeepsPublishedVersionAndReturnsCredit) +{ + size_t failures = 0; + for (size_t fail_after = 0; fail_after < 8; ++fail_after) { + TestCredit credit; + auto failing = std::make_shared<TestFailBudget>(credit.reservation); + EpochManager epoch; + // The externally owned epoch is outside this table's data account. + { + auto registration = epoch.enter(); + } + auto table = std::make_shared<MemTable>(1, &epoch, failing); + ASSERT_TRUE(table->upsert("key", 1, encode_cell_buf(1, OpKind::kPut, "previous"), 0)); + const auto allocated = credit.allocated(); + failing->fail_after = fail_after; + bool failed = false; + try { + ASSERT_TRUE(table->upsert("key", 2, encode_cell_buf(2, OpKind::kPut, std::string(4096, 'v')), 0)); + } + catch (const std::bad_alloc &) { + failed = true; + ++failures; + } + { + auto guard = table->read_guard(); + ASSERT_NE(table->find("key"), nullptr); + EXPECT_EQ(table->find("key")->slot, failed ? 1U : 2U); + } + if (failed) { + EXPECT_EQ(credit.allocated(), allocated); + } + table.reset(); + epoch.try_reclaim(); + credit.reservation->seal(); + EXPECT_EQ(credit.allocated(), 0U); + } + EXPECT_GE(failures, 4U); +} + +TEST(MemTableBudget, RetiredHistoryRemainsChargedUntilReadGuardLeaves) +{ + TestCredit credit; + EpochManager epoch; + { + auto registration = epoch.enter(); + } + auto table = std::make_shared<MemTable>(1, &epoch, credit.reservation); + ASSERT_TRUE(table->upsert("key", 1, encode_cell_buf(1, OpKind::kPut, std::string(1024, 'a')), 0)); + size_t retained; + { + auto guard = table->read_guard(); + for (uint64_t slot = 2; slot <= 8; ++slot) { + ASSERT_TRUE(table->upsert("key", slot, encode_cell_buf(slot, OpKind::kPut, std::string(1024, 'b')), 0)); + } + retained = credit.allocated(); + table->prune("key", 8); + epoch.try_reclaim(); + EXPECT_GE(credit.allocated(), retained); + } + epoch.try_reclaim(); + EXPECT_LT(credit.allocated(), retained); + table.reset(); + epoch.try_reclaim(); + credit.reservation->seal(); + EXPECT_EQ(credit.allocated(), 0U); +} + +TEST(MemTableBudget, UnknownExternalCapacityIsCopiedBeforeRetainingInBudgetedSource) +{ + TestCredit credit; + auto table = std::make_shared<MemTable>(1, nullptr, credit.reservation); + + struct TestExternal + { + std::string value; + size_t *drops; + }; + + size_t drops = 0; + auto *owner = new TestExternal{.value = std::string(8192, 'v'), .drops = &drops}; + auto value = + buffer::wrap_external(reinterpret_cast<const uint8_t *>(owner->value.data()), 1024, owner, [](void *context) { + auto *data = static_cast<TestExternal *>(context); + ++*data->drops; + delete data; + }); + ASSERT_TRUE(table->upsert_external("key", 1, 0, std::move(value), 0)); + EXPECT_EQ(drops, 1U); + auto guard = table->read_guard(); + const auto *cell = table->find("key"); + ASSERT_NE(cell, nullptr); + EXPECT_EQ(cell->cell.ownership(), buffer::mode::kOwned); + EXPECT_EQ(CellView(cell->cell.slice()).value().to_string(), std::string(1024, 'v')); + EXPECT_GE(credit.allocated(), 1024U + kCellHeaderSize); + EXPECT_LT(credit.allocated(), 8192U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/merge_pair_metrics_test.cpp b/lib/crowdb-tree/tests/unit/merge_pair_metrics_test.cpp new file mode 100644 index 000000000..08a5fe69d --- /dev/null +++ b/lib/crowdb-tree/tests/unit/merge_pair_metrics_test.cpp @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/merge_pair_metrics.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +void insert(MergePairMetrics &ledger, uint64_t left, uint64_t right, uint64_t parent, uint64_t when, bool ready = true) +{ + MergePairMetrics::Prepared update(ledger, {}, {}); + update.set_pair({.left = left, .right = right, .parent = parent, .ready = ready}, when); + update.finish(); + (void)update.commit(); +} + +TEST(MergePairMetrics, OverlappingAdjacentPairsCountEachPidOnce) +{ + MergePairMetrics ledger; + insert(ledger, 1, 2, 10, 5); + insert(ledger, 2, 3, 10, 8, false); + EXPECT_EQ(ledger.snapshot(), (MergePairMetrics::Snapshot{3, 2, 1, 5})); + EXPECT_TRUE(ledger.consistent()); + MergePairMetrics::Prepared update(ledger, {}, {}); + update.invalidate_page(1); + update.finish(); + EXPECT_EQ(update.snapshot(), (MergePairMetrics::Snapshot{2, 1, 0, 8})); + EXPECT_EQ(ledger.snapshot().pages, 3U); + (void)update.commit(); + EXPECT_FALSE(ledger.contains_page(1)); + EXPECT_TRUE(ledger.contains_page(2)); + EXPECT_TRUE(ledger.consistent()); +} + +TEST(MergePairMetrics, ReadinessRetryPreservesTheFirstCandidateAge) +{ + MergePairMetrics ledger; + insert(ledger, 1, 2, 10, 5, false); + insert(ledger, 1, 2, 10, 20, true); + EXPECT_EQ(ledger.snapshot(), (MergePairMetrics::Snapshot{2, 1, 1, 5})); + EXPECT_TRUE(ledger.consistent()); +} + +TEST(MergePairMetrics, ChangedEndpointAndParentInvalidateOnlyTheirOwnPairs) +{ + MergePairMetrics ledger; + insert(ledger, 1, 2, 10, 5); + insert(ledger, 2, 3, 10, 6); + insert(ledger, 4, 5, 11, 7); + MergePairMetrics::Prepared update(ledger, {}, {}); + update.invalidate_page(2); + update.invalidate_parent(10); + update.invalidate_page(3); + update.finish(); + (void)update.commit(); + EXPECT_EQ(ledger.snapshot(), (MergePairMetrics::Snapshot{2, 1, 1, 7})); + EXPECT_TRUE(ledger.consistent()); +} + +TEST(MergePairMetrics, AbandonedPreparedChangeLeavesTheLiveIndexesIntact) +{ + MergePairMetrics ledger; + insert(ledger, 1, 2, 10, 5); + { + MergePairMetrics::Prepared update(ledger, {}, {}); + update.invalidate_parent(10); + update.set_pair({.left = 3, .right = 4, .parent = 11}, 8); + update.finish(); + } + EXPECT_EQ(ledger.snapshot(), (MergePairMetrics::Snapshot{2, 1, 1, 5})); + EXPECT_TRUE(ledger.consistent()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/metrics_registry_allocation_test.cpp b/lib/crowdb-tree/tests/unit/metrics_registry_allocation_test.cpp new file mode 100644 index 000000000..c52964d81 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/metrics_registry_allocation_test.cpp @@ -0,0 +1,164 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/metrics/metrics.h" +#include "runtime/allocation_memory_resource.h" + +#include <gtest/gtest.h> + +#include <array> +#include <cstdio> +#include <memory> +#include <string> + +namespace crowdb::tree::detail +{ +namespace +{ +using common::metrics::MetricsRegistry; + +class TestRegistryBudget final : public AllocationBudget +{ + public: + bool claim(size_t bytes) noexcept override + { + if (calls++ == fail_at) { + return false; + } + used += bytes; + return true; + } + + void release(size_t bytes) noexcept override + { + used -= bytes; + } + + void record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept override + { + EXPECT_EQ(kind, AllocationKind::kMetadata); + if (acquired) { + physical += bytes; + } + else { + physical -= bytes; + } + } + + size_t used = 0; + size_t physical = 0; + size_t calls = 0; + size_t fail_at = SIZE_MAX; +}; + +template <class F> bool throws_bad_alloc(F function) +{ + try { + function(); + } + catch (const std::bad_alloc &) { + return true; + } + return false; +} + +std::string output(MetricsRegistry ®istry) +{ + std::unique_ptr<FILE, decltype(&std::fclose)> file(std::tmpfile(), &std::fclose); + if (!file) { + return {}; + } + registry.flush_to(file.get(), 1, "test"); + EXPECT_EQ(std::fseek(file.get(), 0, SEEK_SET), 0); + std::string result; + std::array<char, 512> bytes{}; + while (const auto count = std::fread(bytes.data(), 1, bytes.size(), file.get())) { + result.append(bytes.data(), count); + } + return result; +} + +TEST(MetricsRegistryAllocation, NamesNodesRegistryAndSortWindowUseActualBudget) +{ + auto budget = std::make_shared<TestRegistryBudget>(); + { + auto resource = + std::allocate_shared<AllocationMemoryResource>(MetadataAllocator<AllocationMemoryResource>(budget), budget); + auto registry = std::allocate_shared<MetricsRegistry>(MetadataAllocator<MetricsRegistry>(budget), resource); + const auto before = budget->used; + uint64_t value = 41; + auto *first = + registry->register_callback_gauge_inline("a.long.tree.instance.prefix", ".z", [&value] { return value; }); + const auto with_first = budget->used; + EXPECT_GT(with_first, before); + registry->register_callback_gauge_inline("a.long.tree.instance.prefix", ".a", [] { return 7; }); + EXPECT_EQ(first->get(), 41U); + const auto retained = budget->used; + const auto calls = budget->calls; + const auto rendered = output(*registry); + EXPECT_LT(rendered.find("prefix.a"), rendered.find("prefix.z")); + EXPECT_NE(rendered.find("41"), std::string::npos); + EXPECT_GT(budget->calls, calls); + EXPECT_EQ(budget->used, retained); + EXPECT_EQ(budget->physical, retained); + const auto *same = registry->register_callback_gauge_inline("a.long.tree.instance.prefix.z", [] { return 99; }); + EXPECT_EQ(same, first); + EXPECT_EQ(first->get(), 99U); + EXPECT_EQ(budget->used, retained); + resource.reset(); + EXPECT_EQ(first->get(), 99U); + } + EXPECT_EQ(budget->used, 0U); + EXPECT_EQ(budget->physical, 0U); +} + +TEST(MetricsRegistryAllocation, RegistrationFailureAtEveryAllocationLeavesExistingGaugeUsable) +{ + for (size_t fail = 0; fail < 2; ++fail) { + auto budget = std::make_shared<TestRegistryBudget>(); + { + auto resource = std::make_shared<AllocationMemoryResource>(budget); + MetricsRegistry registry(resource); + auto *existing = registry.register_callback_gauge_inline("live", [] { return 5; }); + const auto retained = budget->used; + budget->fail_at = budget->calls + fail; + EXPECT_TRUE(throws_bad_alloc( + [&] { registry.register_callback_gauge_inline("another.long.paid.tree.name", [] { return 17; }); })); + EXPECT_EQ(existing->get(), 5U); + EXPECT_EQ(registry.max_name_len(), 4U); + EXPECT_EQ(budget->used, retained); + EXPECT_EQ(budget->physical, retained); + } + EXPECT_EQ(budget->used, 0U); + } +} + +TEST(MetricsRegistryAllocation, FlushScratchRefusalRetainsCallbacksWithoutUnchargedFallback) +{ + auto budget = std::make_shared<TestRegistryBudget>(); + { + auto resource = std::make_shared<AllocationMemoryResource>(budget); + MetricsRegistry registry(resource); + registry.register_callback_gauge_inline("retained", [] { return 73; }); + const auto retained = budget->used; + budget->fail_at = budget->calls; + EXPECT_TRUE(throws_bad_alloc([&] { (void)output(registry); })); + EXPECT_EQ(budget->used, retained); + EXPECT_NE(output(registry).find("73"), std::string::npos); + } + EXPECT_EQ(budget->used, 0U); + EXPECT_EQ(budget->physical, 0U); +} + +TEST(MetricsRegistryAllocation, InlineGaugeCoexistsWithLegacyGaugeAndCallback) +{ + MetricsRegistry registry; + registry.register_gauge("b.regular")->set(8); + registry.register_callback_gauge("c.callback", [] { return 9; }); + registry.register_callback_gauge_inline("a.inline", [] { return 7; }); + const auto rendered = output(registry); + EXPECT_LT(rendered.find("a.inline"), rendered.find("b.regular")); + EXPECT_LT(rendered.find("b.regular"), rendered.find("c.callback")); + EXPECT_EQ(registry.max_name_len(), 10U); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/metrics_test.cpp b/lib/crowdb-tree/tests/unit/metrics_test.cpp index 2aad878e5..2dd310547 100644 --- a/lib/crowdb-tree/tests/unit/metrics_test.cpp +++ b/lib/crowdb-tree/tests/unit/metrics_test.cpp @@ -258,12 +258,11 @@ TEST(MetricsRegistry, FlushMetricsStrFormat) std::string out = t.flush_metrics_str(5.0, "2026-07-15T16:30:05.123Z", 0); ASSERT_FALSE(out.empty()); EXPECT_NE(out.find("cpp-tree\n"), std::string::npos); - // Latency section should use us units. - EXPECT_NE(out.find("us"), std::string::npos); - // Bandwidth section should use MB. - EXPECT_NE(out.find("MB"), std::string::npos); - // tps column should be present. - EXPECT_NE(out.find("tps"), std::string::npos); + // Native registry exports cumulative fixed-schema gauges; the host + // collector computes rates from its independently owned window. + EXPECT_NE(out.find("s.0.g.0.memory.allocated.bytes.g"), std::string::npos); + EXPECT_NE(out.find("s.0.g.0.work.flush.batch.latency.sum.ns.c"), std::string::npos); + EXPECT_NE(out.find("s.0.g.0.frontier.published.g"), std::string::npos); // max_name_len should be non-zero after init_metrics. EXPECT_GT(t.max_name_len(), 0U); } diff --git a/lib/crowdb-tree/tests/unit/native_import_test.cpp b/lib/crowdb-tree/tests/unit/native_import_test.cpp new file mode 100644 index 000000000..263999cb6 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/native_import_test.cpp @@ -0,0 +1,173 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/overflow_backing.h" +#include "inner_load_test_support.h" +#include "load_budget_test_support.h" +#include "native_import_test_support.h" +#include "snapshot/operation_wait.h" +#include "snapshot/recovered_state.h" + +namespace crowdb::tree +{ +namespace +{ +class NativeImport : public testing::Test +{ + protected: + Status catalog(const std::vector<NativeFrame> &frames, std::shared_ptr<ReadablePageCatalog> *out) const + { + return build_native_catalog(frames, test_import::info(frames), {}, {}, {}, credits.credits.bytes, out); + } + + test_read::LoadCredits credits; + std::shared_ptr<detail::FlushRuntime> runtime = std::make_shared<detail::FlushRuntime>(1, 8); + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<test_read::TestInnerStore> store = std::make_shared<test_read::TestInnerStore>(); + std::shared_ptr<AsyncStorePageLoader> loader = std::make_shared<AsyncStorePageLoader>( + store, runtime, lane, PageLoadLimits{}, OverflowBackingResolver{}, credits.credits); +}; + +TEST_F(NativeImport, ResidentLeafAndOverflowUseOwnedFramesWithoutBackendIo) +{ + loader = std::make_shared<AsyncStorePageLoader>(nullptr, runtime, lane, PageLoadLimits{}, OverflowBackingResolver{}, + credits.credits); + const std::string value(9000, 'v'); + auto frames = test_import::overflow_tree(value); + std::shared_ptr<ReadablePageCatalog> pages; + auto status = catalog(frames, &pages); + ASSERT_TRUE(status.ok()) << status.to_string(); + auto primary = pages->frames.at(0).backing; + EXPECT_TRUE(primary->readable()); + EXPECT_FALSE(primary->reloadable()); + EXPECT_EQ(primary->address, 0U); + EXPECT_EQ(primary->length, 0U); + frames.clear(); + pages.reset(); + std::shared_ptr<const LeafBase> leaf; + loader->load_leaf({.identity = 1, .max_slot = 7, .backing = primary}, [&](Status result, auto loaded) { + EXPECT_TRUE(result.ok()) << result.to_string(); + leaf = std::move(loaded); + }); + runtime->drain(); + ASSERT_NE(leaf, nullptr); + EXPECT_EQ(leaf->view().self_page_id(), 0U); + std::shared_ptr<const buffer> loaded_value; + loader->load_overflow(primary, 1, value.size(), [&](Status result, auto loaded) { + EXPECT_TRUE(result.ok()) << result.to_string(); + loaded_value = std::move(loaded); + }); + runtime->drain(); + ASSERT_NE(loaded_value, nullptr); + EXPECT_EQ(Slice(*loaded_value).to_string(), value); + EXPECT_EQ(store->reads.load(), 0U); + primary.reset(); + leaf.reset(); + loaded_value.reset(); + loader.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], 0U); +} + +TEST_F(NativeImport, ReadableGraphLoadsResidentIndexButDurableGraphRejectsIt) +{ + loader = std::make_shared<AsyncStorePageLoader>(nullptr, runtime, lane, PageLoadLimits{}, OverflowBackingResolver{}, + credits.credits); + NativeFrame index{.page_id = 0, .frame = std::vector<uint8_t>(4096)}; + ASSERT_TRUE(inner_frame_build(index.frame.data(), index.frame.size(), 0, {1, 2}, {Slice("m")})); + const auto cell = encode_cell_buf(7, OpKind::kPut, "left"); + std::vector<NativeFrame> frames; + frames.push_back(std::move(index)); + frames.push_back(test_import::leaf(1, "a", Slice(cell), 2)); + frames.push_back(test_import::inline_leaf(2, "z", "right")); + std::shared_ptr<ReadablePageCatalog> pages; + auto status = catalog(frames, &pages); + ASSERT_TRUE(status.ok()) << status.to_string(); + std::shared_ptr<RecoveredStateOperation> operation; + ASSERT_TRUE( + RecoveredStateOperation::create_readable(pages, loader, runtime, lane, {}, credits.credits.bytes, &operation) + .ok()); + status = detail::wait_snapshot_operation(operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(operation->result().state->stats.height, 2U); + EXPECT_EQ(operation->result().state->stats.kv_known_count, 2U); + EXPECT_EQ(store->reads.load(), 0U); + auto durable = detail::reserved_shared<CapturedDurablePages>(credits.credits.bytes, credits.credits.bytes); + static_cast<ReadablePageCatalog &>(*durable) = *pages; + operation.reset(); + ASSERT_TRUE( + RecoveredStateOperation::create(durable, loader, runtime, lane, {}, credits.credits.bytes, &operation).ok()); + EXPECT_EQ(detail::wait_snapshot_operation(operation).code(), Code::kCorruption); +} + +TEST_F(NativeImport, DurableReadsStillRequireABackend) +{ + auto backing = std::make_shared<BackingRef>(); + backing->length = 4096; + backing->owner = std::make_shared<int>(0); + OverflowBackingResolver resolve = [backing](uint64_t, std::shared_ptr<const BackingRef> *out) { + *out = backing; + return Status::Ok(); + }; + AsyncStorePageLoader missing(nullptr, runtime, lane, {}, std::move(resolve), credits.credits); + size_t completed = 0; + auto failed = [&](Status status, const auto &result) { + EXPECT_EQ(status.code(), Code::kInvalidArgument) << status.to_string(); + EXPECT_FALSE(result); + ++completed; + }; + missing.load_leaf({.identity = 1, .max_slot = 7, .backing = backing}, failed); + missing.load_inner(backing, failed); + missing.load_overflow(1, 10, failed); + runtime->drain(); + EXPECT_EQ(completed, 3U); + EXPECT_EQ(store->reads.load(), 0U); +} + +TEST_F(NativeImport, OverwrittenOverflowCellStillRetainsItsPhysicalDependency) +{ + auto frames = test_import::overflow_tree("old-overflow"); + std::vector<uint8_t> updated(4096); + std::vector<leaf_entry> delta; + delta.push_back({.key = "key", .cell = encode_cell_buf(7, OpKind::kPut, "latest")}); + ASSERT_TRUE(leaf_frame_append_deltas(frames[0].frame.data(), 4096, delta, updated.data())); + frames[0].frame = std::move(updated); + std::shared_ptr<ReadablePageCatalog> pages; + auto status = catalog(frames, &pages); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(pages->frames.at(0).stored_overflow->size(), 1U); + std::shared_ptr<RecoveredStateOperation> operation; + ASSERT_TRUE( + RecoveredStateOperation::create_readable(pages, loader, runtime, lane, {}, credits.credits.bytes, &operation) + .ok()); + status = detail::wait_snapshot_operation(operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto leaf = operation->result().state->mapping.get(0)->leaf; + EXPECT_EQ(leaf->bases.front().backing->overflow->pages.size(), 1U); + EXPECT_EQ(store->reads.load(), 0U); +} + +TEST_F(NativeImport, ResidentLoaderValidatesCrcAndRetainsNoChargeAfterFailure) +{ + auto raw = std::make_shared<std::vector<uint8_t>>(test_import::inline_leaf().frame); + raw->back() ^= 1; + auto backing = std::make_shared<BackingRef>(); + backing->storage = BackingStorage::kResidentFrame; + backing->durability = BackingDurability::kResident; + backing->resident_frame = *raw; + backing->owner = raw; + size_t completed = 0; + loader->load_leaf({.identity = 1, .max_slot = 7, .backing = backing}, [&](Status status, auto result) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_FALSE(result); + ++completed; + }); + runtime->drain(); + EXPECT_EQ(completed, 1U); + EXPECT_EQ(store->reads.load(), 0U); + loader.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes], 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[test_read::LoadCredits::kIo], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/native_layout_test.cpp b/lib/crowdb-tree/tests/unit/native_layout_test.cpp new file mode 100644 index 000000000..48561c580 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/native_layout_test.cpp @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/maptable/compressor.h" +#include "snapshot/native_layout.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestNativeLayout +{ + std::shared_ptr<PublishedTreeState> view = std::make_shared<PublishedTreeState>(); + std::shared_ptr<OverflowBackingSet> dependencies = std::make_shared<OverflowBackingSet>(nullptr); + std::vector<ReusedOverflowDependency> references; + + TestNativeLayout() + { + view->next_page_id = 8; + add(1, kInvalidPageId, 100); + references.push_back({.head = 1, .bytes = 100, .owner = dependencies, .backing = dependencies}); + } + + void add(uint64_t pid, uint64_t next, uint32_t bytes) + { + auto backing = std::make_shared<BackingRef>(); + backing->backend_id = 7; + backing->generation = 9; + backing->address = 8192 * pid; + backing->length = 8192; + backing->owner = std::make_shared<int>(0); + dependencies->pages[pid] = {.backing = std::move(backing), + .next = next, + .bytes = bytes, + .frame_bytes = 4096, + .logical_bytes = 4096 + kDurableBlobHeader}; + } + + CapturedFrameResult::Plan page(uint64_t pid = 0, uint64_t fresh = 0) + { + return {.page_id = pid, + .summary = {.new_overflow_frames = fresh}, + .reused_overflow = references, + .owner = dependencies}; + } + + static Status drain(NativeLayoutPlanner &planner) + { + while (planner.pending()) { + auto status = planner.step(); + if (!status.ok()) { + return status; + } + } + return planner.finish(); + } +}; + +TEST(NativeLayout, DeduplicatesPhysicalChainsAndCountsNewFramesWithoutPayloads) +{ + TestNativeLayout test; + test.references.push_back(test.references.front()); + NativeLayoutPlanner planner(test.view, {}, {}, 7, 9); + ASSERT_TRUE(planner.accept(test.page(0, 3)).ok()); + auto status = TestNativeLayout::drain(planner); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(planner.frame_count(), 5U); + EXPECT_EQ(planner.new_overflow_frames(), 3U); + EXPECT_EQ(planner.old_frames().size(), 1U); + EXPECT_EQ(planner.old_frames().at(1).backing, test.dependencies->pages.at(1).backing); +} + +TEST(NativeLayout, RejectsMissingCycleLengthAndForeignDomainDescriptors) +{ + for (unsigned fault = 0; fault != 4; ++fault) { + TestNativeLayout test; + if (fault == 0) { + test.dependencies->pages.clear(); + } + if (fault == 1) { + test.dependencies->pages.at(1).next = 1; + } + if (fault == 2) { + test.references.front().bytes = 101; + } + if (fault == 3) { + auto foreign = std::make_shared<BackingRef>(*test.dependencies->pages.at(1).backing); + ++foreign->generation; + test.dependencies->pages.at(1).backing = std::move(foreign); + } + NativeLayoutPlanner planner(test.view, {}, {}, 7, 9); + ASSERT_TRUE(planner.accept(test.page()).ok()); + EXPECT_EQ(planner.step().code(), Code::kCorruption) << "fault=" << fault; + } +} + +TEST(NativeLayout, RejectsConflictingAliasesAndStopsAtTheDependencyStepLimit) +{ + TestNativeLayout test; + test.add(2, kInvalidPageId, 50); + test.dependencies->pages.at(1).next = 2; + test.dependencies->pages.at(1).bytes = 50; + NativeLayoutPlanner bounded(test.view, {.max_frames = 8, .max_dependency_steps = 1}, {}, 7, 9); + ASSERT_TRUE(bounded.accept(test.page()).ok()); + ASSERT_TRUE(bounded.step().ok()); + EXPECT_EQ(bounded.step().code(), Code::kResourceExhausted); + + NativeLayoutPlanner conflicting(test.view, {}, {}, 7, 9); + ASSERT_TRUE(conflicting.accept(test.page()).ok()); + ASSERT_TRUE(TestNativeLayout::drain(conflicting).ok()); + auto changed = std::make_shared<BackingRef>(*test.dependencies->pages.at(1).backing); + changed->address += 4096; + test.dependencies->pages.at(1).backing = std::move(changed); + ASSERT_TRUE(conflicting.accept(test.page(3)).ok()); + EXPECT_EQ(conflicting.step().code(), Code::kCorruption); +} + +TEST(NativeLayout, BoundRejectsCountAndNamespaceOverflow) +{ + TestNativeLayout test; + NativeLayoutPlanner planner(test.view, {.max_frames = 1, .max_dependency_steps = 1}, {}, 7, 9); + EXPECT_EQ(planner.accept(test.page(0, 1)).code(), Code::kResourceExhausted); + EXPECT_EQ(planner.frame_count(), 0U); + EXPECT_FALSE(NativeLayoutPlanner::allocation_bound(UINT64_MAX, {})); + EXPECT_FALSE(NativeLayoutPlanner::allocation_bound(1, {.max_frames = UINT64_MAX})); + EXPECT_FALSE(NativeLayoutPlanner::allocation_bound(1, {.max_frames = 0})); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/native_readable_test.cpp b/lib/crowdb-tree/tests/unit/native_readable_test.cpp new file mode 100644 index 000000000..48d54cdf9 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/native_readable_test.cpp @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_bound.h" +#include "runtime/tree_runtime_binding.h" +#include "snapshot/native_frame_read.h" +#include "snapshot/native_layout.h" +#include "snapshot/operation_wait.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestNativeReadable +{ + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<std::vector<uint8_t>> raw = std::make_shared<std::vector<uint8_t>>(4096); + std::shared_ptr<BackingRef> backing = std::make_shared<BackingRef>(); + OverflowBackingPage descriptor; + PageLoadLimits limits; + std::shared_ptr<TreeWorkMetrics> metrics = std::make_shared<TreeWorkMetrics>(); + + TestNativeReadable() + { + EXPECT_TRUE(TreeRuntime::create({}, &runtime).ok()); + EXPECT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + const std::string value(100, 'r'); + overflow_frame_build(raw->data(), raw->size(), 1, kInvalidPageId, + reinterpret_cast<const uint8_t *>(value.data()), value.size()); + backing->storage = BackingStorage::kResidentFrame; + backing->durability = BackingDurability::kResident; + backing->resident_frame = *raw; + backing->owner = raw; + descriptor = { + .backing = backing, .next = kInvalidPageId, .bytes = 100, .frame_bytes = 4096, .logical_bytes = 4096}; + limits.max_blob_bytes = 8192; + limits.max_frame_bytes = 4096; + } + + ~TestNativeReadable() + { + binding->drain(); + } + + [[nodiscard]] Status create(std::shared_ptr<NativeFrameRead> *out, bool seal_credit = false) const + { + const auto bound = NativeFrameRead::allocation_bound(limits); + if (!bound) { + return Status::resource_exhausted("native read test bound overflow"); + } + const auto bytes = *bound + detail::shared_allocation_bound<detail::AllocationReservation>(); + auto credit = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kRouting, bytes); + if (!credit) { + return Status::resource_exhausted(); + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + if (seal_credit) { + metadata->seal(); + } + return NativeFrameRead::create(1, descriptor, {}, binding->cpu(), binding->lane(), limits, + {.bytes = std::move(metadata), .io = {}, .work_metrics = metrics}, out); + } +}; + +TEST(NativeReadable, RawOverflowNeedsNoBackendOrIoCreditAndRetainsExactOwner) +{ + TestNativeReadable test; + std::shared_ptr<NativeFrameRead> read; + auto status = test.create(&read); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = detail::wait_snapshot_operation(read); + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto &frame = read->result().frame; + if (!frame.has_value()) { + FAIL() << "native reader omitted its successful frame"; + } + auto retained = frame.value(); + EXPECT_EQ(retained.bytes.data(), test.raw->data()); + EXPECT_EQ(retained.durable_addr, kNoAddr); + EXPECT_FALSE(retained.inherited); + std::weak_ptr<std::vector<uint8_t>> weak = test.raw; + test.descriptor = {}; + test.backing.reset(); + test.raw.reset(); + read.reset(); + test.binding->drain(); + EXPECT_FALSE(weak.expired()); + EXPECT_TRUE(frame_validate(retained.bytes.data(), retained.bytes.size())); + EXPECT_EQ(test.runtime->snapshot().reserved.io_operations, 0U); + EXPECT_EQ(test.runtime->snapshot().allocated.io_operations, 0U); + retained = {}; + EXPECT_TRUE(weak.expired()); +} + +TEST(NativeReadable, RawOverflowChecksCrcSelfPidAndPhysicalTopology) +{ + for (unsigned fault = 0; fault != 3; ++fault) { + TestNativeReadable test; + if (fault == 0) { + test.raw->at(100) ^= 0xff; + } + if (fault == 1) { + const std::string value(100, 'r'); + overflow_frame_build(test.raw->data(), test.raw->size(), 2, kInvalidPageId, + reinterpret_cast<const uint8_t *>(value.data()), value.size()); + } + if (fault == 2) { + test.descriptor.next = 2; + } + std::shared_ptr<NativeFrameRead> read; + auto status = test.create(&read); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(detail::wait_snapshot_operation(read).code(), Code::kCorruption) << fault; + EXPECT_FALSE(read->result().frame); + } +} + +TEST(NativeReadable, LayoutAcceptsResidentCatalogWithoutDurableDomainIdentity) +{ + TestNativeReadable test; + auto view = std::make_shared<PublishedTreeState>(); + view->next_page_id = 2; + auto catalog = std::make_shared<OverflowBackingSet>(nullptr); + catalog->pages.emplace(1, test.descriptor); + const std::vector<ReusedOverflowDependency> references{ + {.head = 1, .bytes = 100, .owner = catalog, .backing = catalog} + }; + NativeLayoutPlanner planner(view, {}, {}, 7, 9); + ASSERT_TRUE(planner.accept({.page_id = 0, .summary = {}, .reused_overflow = references, .owner = catalog}).ok()); + while (planner.pending()) { + auto status = planner.step(); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + ASSERT_TRUE(planner.finish().ok()); + EXPECT_EQ(planner.frame_count(), 2U); + EXPECT_EQ(planner.new_overflow_frames(), 0U); +} + +TEST(NativeReadable, MetricsIncludeValidationAdmissionAndAbandonedRead) +{ + TestNativeReadable test; + std::shared_ptr<NativeFrameRead> read; + test.descriptor.next = 1; + EXPECT_EQ(test.create(&read).code(), Code::kInvalidArgument); + test.descriptor.next = kInvalidPageId; + EXPECT_EQ(test.create(&read, true).code(), Code::kResourceExhausted); + auto totals = test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)]; + EXPECT_EQ(totals.attempts, 2U); + EXPECT_EQ(totals.inflight, 0U); + EXPECT_EQ(totals.outcomes[static_cast<size_t>(TreeOutcome::Error)], 2U); + ASSERT_TRUE(test.create(&read).ok()); + EXPECT_EQ(test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].inflight, 1U); + read.reset(); + totals = test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)]; + EXPECT_EQ(totals.count, 3U); + EXPECT_EQ(totals.inflight, 0U); + EXPECT_EQ(totals.outcomes[static_cast<size_t>(TreeOutcome::Cancelled)], 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/owned_allocation_c_api_test.cpp b/lib/crowdb-tree/tests/unit/owned_allocation_c_api_test.cpp new file mode 100644 index 000000000..422e227c2 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/owned_allocation_c_api_test.cpp @@ -0,0 +1,245 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_write_budget.h" +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "c_api_owned_allocation.h" +#include "crowdb-tree/owned_allocation_c_api.h" +#include "crowdb-tree/runtime_c_api.h" + +#include <gtest/gtest.h> + +#include <array> +#include <set> + +namespace crowdb::tree +{ +namespace +{ +struct TestHost +{ + TestHost() + { + ct_options options{}; + if (ct_runtime_create(nullptr, &runtime) != 0 || ct_open_with_runtime(&options, runtime, nullptr, &tree) != 0) { + throw std::runtime_error("runtime Tree initialization failed"); + } + } + + ~TestHost() + { + ct_close(tree); + ct_runtime_free(runtime); + } + + [[nodiscard]] TreeCore *core() const + { + return TreeCoreAccess::get(*c_api_tree(tree)); + } + + ct_runtime *runtime = nullptr; + ct_tree *tree = nullptr; +}; + +struct TestPayload +{ + std::array<uint8_t, 8192> bytes{}; + size_t *drops; +}; + +void drop_payload(void *context) +{ + auto *payload = static_cast<TestPayload *>(context); + ++*payload->drops; + delete payload; +} + +ct_status create_owner(ct_tree *tree, size_t *drops, ct_owned_allocation **out, const uint8_t **data = nullptr) +{ + auto *payload = new TestPayload{.bytes = {}, .drops = drops}; + payload->bytes.fill('v'); + if (data != nullptr) { + *data = payload->bytes.data(); + } + return ct_owned_allocation_create(tree, payload->bytes.data(), 256, payload->bytes.size(), drop_payload, payload, + sizeof(payload->drops), out); +} + +ct_owned_kv_ref input(ct_owned_allocation *owner, size_t offset = 0, size_t length = 128) +{ + static constexpr uint8_t key[] = {'k'}; + return {.key = key, .key_len = 1, .owner = owner, .offset = offset, .length = length, .kind = 0}; +} + +size_t payload_bytes(const std::shared_ptr<detail::ResourceBudget::Account> &account) +{ + return account->snapshot().physical[static_cast<size_t>(AllocationKind::kPayload)]; +} + +TEST(OwnedAllocationCAbi, SlicesRetainOneCapacityAndFinalForeignDrop) +{ + TestHost host; + const auto account = host.core()->binding()->account(); + const auto before = payload_bytes(account); + size_t drops = 0; + ct_owned_allocation *owner = nullptr; + const uint8_t *data = nullptr; + ASSERT_EQ(create_owner(host.tree, &drops, &owner, &data), 0); + EXPECT_EQ(payload_bytes(account) - before, 8192U); + const auto operation = input(owner, 16, 128); + ASSERT_EQ(ct_apply_batch_owned(host.tree, 1, &operation, 1), 0); + host.core()->binding()->drain(); + ct_owned_allocation_retain(owner); + ct_owned_allocation_free(owner); + GetView view; + ASSERT_TRUE(host.core()->get(Slice("k"), &view).ok()); + EXPECT_EQ(view.value().bytes(), data + 16); + view = {}; + ct_owned_allocation_free(owner); + EXPECT_EQ(drops, 0U); + ct_close(std::exchange(host.tree, nullptr)); + EXPECT_EQ(drops, 1U); + EXPECT_EQ(payload_bytes(account), before); +} + +TEST(OwnedAllocationCAbi, CrossHostCopiesAndSourceOwnerCanBeDropped) +{ + TestHost source; + TestHost target; + size_t drops = 0; + ct_owned_allocation *owner = nullptr; + ASSERT_EQ(create_owner(source.tree, &drops, &owner), 0); + const auto operation = input(owner); + const auto before = target.core()->state_stats().resources.payload_bytes; + ASSERT_EQ(ct_apply_batch_owned(target.tree, 1, &operation, 1), 0); + EXPECT_EQ(target.core()->state_stats().resources.payload_bytes - before, 128U + kCellHeaderSize); + ct_owned_allocation_free(owner); + EXPECT_EQ(drops, 1U); + GetView view; + ASSERT_TRUE(target.core()->get(Slice("k"), &view).ok()); + EXPECT_EQ(view.value().to_string(), std::string(128, 'v')); +} + +TEST(OwnedAllocationCAbi, InvalidSliceAndAdmissionFailureReleaseExactlyOnce) +{ + TestHost host; + size_t drops = 0; + ct_owned_allocation *owner = nullptr; + ASSERT_EQ(create_owner(host.tree, &drops, &owner), 0); + const auto operation = input(owner, 256, 1); + EXPECT_EQ(ct_apply_batch_owned(host.tree, 1, &operation, 1), static_cast<int>(Code::kInvalidArgument)); + EXPECT_EQ(drops, 0U); + ct_owned_allocation_free(owner); + EXPECT_EQ(drops, 1U); + ct_runtime_stop_admission(host.runtime); + EXPECT_NE(create_owner(host.tree, &drops, &owner), 0); + EXPECT_EQ(owner, nullptr); + EXPECT_EQ(drops, 2U); +} + +TEST(OwnedAllocationCAbi, UnsubmittedControlAndPayloadOutliveTreeAndRuntime) +{ + TestHost host; + const auto account = host.core()->binding()->account(); + size_t drops = 0; + ct_owned_allocation *owner = nullptr; + ASSERT_EQ(create_owner(host.tree, &drops, &owner), 0); + ct_close(std::exchange(host.tree, nullptr)); + ct_runtime_free(std::exchange(host.runtime, nullptr)); + EXPECT_EQ(drops, 0U); + EXPECT_EQ(payload_bytes(account), 8192U); + ct_owned_allocation_free(owner); + EXPECT_EQ(drops, 1U); + EXPECT_EQ(payload_bytes(account), 0U); + // The retained observation account has no remaining foreign allocations. + EXPECT_EQ(account->snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kAdmittedWriter)], 0U); +} + +TEST(OwnedAllocationCAbi, FrozenSourceKeepsExternalSlicesAfterTreeClose) +{ + TestHost host; + size_t drops = 0; + ct_owned_allocation *owner = nullptr; + const uint8_t *data = nullptr; + ASSERT_EQ(create_owner(host.tree, &drops, &owner, &data), 0); + auto account = host.core()->binding()->account(); + std::array<ct_owned_kv_ref, 2> operations{input(owner, 0, 128), input(owner, 128, 128)}; + static constexpr uint8_t second_key[] = {'z'}; + operations[1].key = second_key; + ASSERT_EQ(ct_apply_batch_owned(host.tree, 1, operations.data(), operations.size()), 0); + auto catalog = host.core()->capture()->sources; + auto runtime = host.core()->binding(); + ct_owned_allocation_free(owner); + ct_close(std::exchange(host.tree, nullptr)); + runtime->drain(); + EXPECT_EQ(drops, 0U); + EXPECT_EQ(payload_bytes(account), 8192U); + size_t cells = 0; + for (const auto &table : catalog->tables) { + auto guard = table->read_guard(); + for (auto cursor = table->prefix_cursor(1); cursor.valid(); cursor.advance()) { + const auto cell = cursor.cell_owner(); + ASSERT_TRUE(cell); + EXPECT_EQ(cell->cell.data(), data + (cells * 128)); + EXPECT_EQ(cell->cell.size(), 128U); + ++cells; + } + } + EXPECT_EQ(cells, 2U); + catalog.reset(); + runtime->drain(); + EXPECT_EQ(drops, 1U); + EXPECT_EQ(payload_bytes(account), 0U); +} + +TEST(OwnedAllocationCAbi, ExactWriterCreditPreparesOwnersBeforeCommit) +{ + TestHost host; + const auto initial = host.core()->binding()->account()->snapshot().allocated[static_cast<size_t>( + detail::BudgetClass::kAdmittedWriter)]; + size_t drops = 0; + ct_owned_allocation *owner = nullptr; + ASSERT_EQ(create_owner(host.tree, &drops, &owner), 0); + auto account = host.core()->binding()->account(); + const auto category = detail::BudgetClass::kAdmittedWriter; + const auto baseline = account->snapshot().allocated[static_cast<size_t>(category)]; + buffer value; + ASSERT_TRUE(owner->bridge->slice(0, 128, &value).ok()); + SourceWriteOptions options; + options.account = account; + options.max_entries = 4; + options.max_gap_entries = 1; + const WriteFootprint footprint{.entries = 1, .key_bytes = 3, .payload_bytes = 0}; + const auto bound = SourceWriteBudget::allocation_bound(options, footprint); + if (!bound) { + FAIL() << "writer allocation bound overflow"; + } + auto reserved = detail::ResourceBudget::try_reserve(account, category, *bound); + if (!reserved) { + FAIL() << "writer allocation reserve unavailable"; + } + auto metadata = detail::AllocationReservation::create(std::move(*reserved)); + { + MemTable table(1, nullptr, host.core()->binding()->lifecycle_budget()); + auto prepared = table.prepare_write("key", std::move(value), 0, {}, 0, 5, metadata); + metadata->seal(); + ASSERT_TRUE(table.commit_write(std::move(prepared), 1)); + auto guard = table.read_guard(); + auto cell = table.prefix_cursor(1).cell_owner(); + ASSERT_TRUE(cell); + EXPECT_EQ(cell->cell.size(), 128U); + } + metadata.reset(); + host.core()->binding()->drain(); + // The independent slice holder has also left; all writer preparation + // allocations have returned after their last owner was released. + EXPECT_EQ(account->snapshot().allocated[static_cast<size_t>(category)], baseline); + ct_owned_allocation_free(owner); + host.core()->binding()->drain(); + EXPECT_EQ(drops, 1U); + EXPECT_EQ(account->snapshot().allocated[static_cast<size_t>(category)], initial); + ct_close(std::exchange(host.tree, nullptr)); + EXPECT_EQ(account->snapshot().allocated[static_cast<size_t>(category)], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/owned_key_test.cpp b/lib/crowdb-tree/tests/unit/owned_key_test.cpp new file mode 100644 index 000000000..12afb611f --- /dev/null +++ b/lib/crowdb-tree/tests/unit/owned_key_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/owned_key.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(OwnedKey, ShortKeysNeedNoHeapCreditBeyondTheirContainingObject) +{ + detail::ResourceBudget::Limits limits{}; + limits.fill(1024 * 1024); + detail::ResourceBudget budget(limits, 1); + auto account = budget.make_account(limits); + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, 4096); + if (!credit) { + FAIL() << "short-key test control admission failed"; + } + auto metadata = detail::AllocationReservation::create(std::move(*credit)); + metadata->seal(); + const auto before = account->snapshot().physical; + const std::string value(24, 'k'); + ASSERT_NO_THROW({ + OwnedKey key(Slice(value), metadata); + EXPECT_EQ(Slice(key).to_string(), value); + }); + EXPECT_EQ(account->snapshot().physical, before); +} + +TEST(OwnedKey, CopiesAndMovesOwnBinaryBytesAcrossStorageTransitions) +{ + for (const size_t size : {0U, 1U, 23U, 24U, 25U, 64U, 4096U}) { + std::string value(size, '\0'); + for (size_t i = 0; i < size; ++i) { + value[i] = static_cast<char>(i % 256); + } + OwnedKey original{Slice(value)}; + auto copy = original; + auto moved = std::exchange(original, OwnedKey{}); + EXPECT_EQ(original.size(), 0U); + original = "replacement"; + EXPECT_EQ(Slice(copy).to_string(), value); + EXPECT_EQ(Slice(moved).to_string(), value); + OwnedKey assigned(std::string(4096, 'a')); + assigned = copy; + EXPECT_EQ(Slice(assigned).to_string(), value); + assigned = std::exchange(moved, OwnedKey{}); + EXPECT_EQ(moved.size(), 0U); + EXPECT_EQ(Slice(assigned).to_string(), value); + assigned = "short"; + EXPECT_EQ(assigned, "short"); + assigned = copy; + EXPECT_EQ(Slice(assigned).to_string(), value); + EXPECT_EQ(assigned <=> copy, std::strong_ordering::equal); + } +} + +TEST(OwnedKey, ContainerRelocationKeepsEachOwnedKeyValid) +{ + std::vector<OwnedKey> keys; + // Keep growth deliberate: this regression must relocate existing keys. + keys.reserve(1); + for (size_t i = 0; i < 256; ++i) { + keys.emplace_back(std::string(i, static_cast<char>(i))); + } + for (size_t i = 0; i < keys.size(); ++i) { + EXPECT_EQ(Slice(keys[i]).to_string(), std::string(i, static_cast<char>(i))); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_cache_residency_budget_test.cpp b/lib/crowdb-tree/tests/unit/page_cache_residency_budget_test.cpp new file mode 100644 index 000000000..9baf2ce4b --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_cache_residency_budget_test.cpp @@ -0,0 +1,225 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "inner_load_test_support.h" +#include "load_budget_test_support.h" +#include "page_cache_residency_test_support.h" + +#include <latch> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_cache_residency; + +TEST(PageCacheResidency, ActualDecodedAllocationSurvivesEvictionUntilLastReaderPin) +{ + LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestInnerStore>(); + auto source = base({ + {.key = "a", .value = "value"} + }); + encode_durable_page(source->frame(), source->page_bytes(), compress_algo::kNone, &store->blobs[0]); + auto backing = std::make_shared<BackingRef>(); + backing->length = store->blobs[0].size(); + backing->owner = store; + auto upstream = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, + OverflowBackingResolver{}, credits.credits); + auto metrics = std::make_shared<TreeResourceMetrics>(); + { + SharedPageCache cache({.max_pages = 1, .max_waiters = 2}, credits.credits.bytes, metrics, runtime.get(), lane); + auto loader = cache.bind(upstream, credits.credits.bytes); + const auto infrastructure = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + std::shared_ptr<const LeafBase> pin; + loader->load_leaf({.identity = 1, .backing = backing}, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + pin = std::move(leaf); + }); + runtime->drain(); + ASSERT_TRUE(pin); + const auto bytes = pin->page_bytes(); + EXPECT_EQ(metrics->snapshot().resident_frames, 1U); + EXPECT_EQ(metrics->snapshot().cache_bytes, bytes); + credits.seal(); + cache.evict(); + runtime->drain(); + EXPECT_EQ(metrics->snapshot().cache_pinned_bytes, bytes); + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], infrastructure); + pin.reset(); + EXPECT_EQ(metrics->snapshot().resident_frames, 0U); + EXPECT_EQ(metrics->snapshot().cache_bytes, 0U); + EXPECT_EQ(metrics->snapshot().cache_pinned_bytes, 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], infrastructure); + } + runtime->drain(); + upstream.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageCacheResidency, PinAdmissionFailureReturnsErrorWithoutDiscardingReadyFrame) +{ + TestCache fixture; + fixture.frame(1); + const auto input = cold(1); + fixture.load(input).reset(); + LoadCredits credits; + auto loader = fixture.cache->bind(fixture.upstream, credits.credits.bytes); + credits.seal(); + size_t done = 0; + loader->load_leaf(input, [&](Status status, auto frame) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(frame); + ++done; + }); + EXPECT_EQ(done, 1U); + EXPECT_EQ(fixture.upstream->calls, 1U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 1U); + EXPECT_EQ(fixture.resources->snapshot().cache_pinned_bytes, 0U); +} + +TEST(PageCacheResidency, MissAdmissionFailureOccursBeforeIoAndCanRetry) +{ + TestCache fixture; + fixture.frame(1); + LoadCredits credits; + auto loader = fixture.cache->bind(fixture.upstream, credits.credits.bytes); + credits.seal(); + size_t failed = 0; + const auto input = cold(1); + loader->load_leaf(input, [&](Status status, auto frame) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(frame); + ++failed; + }); + EXPECT_EQ(failed, 1U); + EXPECT_EQ(fixture.upstream->calls, 0U); + EXPECT_EQ(fixture.cache->snapshot().waiters, 0U); + ASSERT_TRUE(fixture.load(input)); + EXPECT_EQ(fixture.upstream->calls, 1U); +} + +TEST(PageCacheResidency, PublisherShutdownRemovesOnlyLiveMembershipAndKeepsCachedFrames) +{ + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto resources = std::make_shared<TreeResourceMetrics>(); + auto cache = std::make_shared<SharedPageCache>(PageCacheLimits{}, nullptr, resources, &runtime, lane); + std::shared_ptr<const PublishedTreeState> retained; + { + PublicationCoordinator publisher(4, 1, {}, std::pmr::get_default_resource(), cache); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{}), + .pages = {{.page_id = 0, .expected = {}, .replacement = published({cold(1)})}}, + .root_page_id = 0}; + ASSERT_TRUE(publisher.try_submit(request)); + retained = publisher.capture(); + runtime.drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, 1U); + EXPECT_THROW((PublicationCoordinator(4, 1, {}, std::pmr::get_default_resource(), cache)), + std::invalid_argument); + } + runtime.drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, 0U); + EXPECT_TRUE(retained->mapping.get(0)); +} + +TEST(PageCacheResidency, QueuedCommandsRetainTheirOwnersAfterCacheAndPublisherDestruction) +{ + LoadCredits credits; + detail::FlushRuntime runtime(1, 4); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(4); + auto resources = std::make_shared<TreeResourceMetrics>(); + + struct Gate + { + std::latch entered{1}, release{1}; + } gate; + + auto blocker = [](void *context) noexcept { + auto &self = *static_cast<Gate *>(context); + self.entered.count_down(); + self.release.wait(); + }; + ASSERT_TRUE(runtime.try_submit(lane, blocker, &gate).ok()); + gate.entered.wait(); + { + auto cache = std::make_shared<SharedPageCache>(PageCacheLimits{.max_pages = 1, .max_waiters = 1}, + credits.credits.bytes, resources, &runtime, lane); + PublicationCoordinator publisher(4, 1, {}, std::pmr::get_default_resource(), cache); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = {.record = std::make_shared<InstallRecord>(InstallToken{}), + .pages = {{.page_id = 0, .expected = {}, .replacement = published({cold(1)})}}, + .root_page_id = 0, + .metadata = credits.credits.bytes}; + EXPECT_TRUE(publisher.try_submit(request)); + auto upstream = std::make_shared<TestLoader>(); + upstream->frames[1] = base({ + {.key = "a", .value = "value"} + }); + auto loader = cache->bind(upstream, credits.credits.bytes); + loader->load_leaf(cold(1), [](Status status, auto frame) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(frame); + }); + credits.seal(); + runtime.close(); + } + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + gate.release.count_down(); + runtime.drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, 0U); + EXPECT_EQ(resources->snapshot().resident_frames, 0U); + EXPECT_EQ(resources->snapshot().cache_pinned_bytes, 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageCacheResidency, ConcurrentCacheEventsAndMappingReplacementsLeaveExactLiveMembership) +{ + TestCache fixture; + fixture.frame(1); + auto input = cold(1); + auto current = published({input}); + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = current} + }, + 0) + .status.ok()); + fixture.synchronize = false; + std::atomic<bool> finished{false}; + std::jthread reader([&] { + while (!finished.load(std::memory_order_acquire)) { + fixture.loader->load_leaf(input, [](Status status, auto frame) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(frame); + }); + fixture.cache->evict(); + } + }); + for (size_t i = 0; i < 100; ++i) { + auto next = published({input, input}); + const auto status = fixture + .publish({ + {.page_id = 0, .expected = current, .replacement = next} + }) + .status; + EXPECT_TRUE(status.ok()); + if (!status.ok()) { + break; + } + current = std::move(next); + } + finished.store(true, std::memory_order_release); + reader.join(); + fixture.cache->evict(); + fixture.runtime.drain(); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 1U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 0U); + EXPECT_EQ(fixture.resources->snapshot().cache_bytes, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_cache_residency_test.cpp b/lib/crowdb-tree/tests/unit/page_cache_residency_test.cpp new file mode 100644 index 000000000..9e06af34e --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_cache_residency_test.cpp @@ -0,0 +1,288 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "load_budget_test_support.h" +#include "page_cache_residency_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_cache_residency; + +TEST(PageCacheResidency, CloseDetachesRuntimeWhileOldPinsAndPreparedUpdatesRemainAlive) +{ + auto fixture = std::make_unique<TestCache>(); + fixture->frame(1); + auto input = cold(1); + auto page = published({input}); + ASSERT_TRUE(fixture + ->publish( + { + {.page_id = 0, .expected = {}, .replacement = page} + }, + 0) + .status.ok()); + auto pin = fixture->load(input); + ASSERT_TRUE(pin); + auto resources = fixture->resources; + auto residency = fixture->cache->residency(); + const std::array<PageReplacement, 1> edits{{{.page_id = 0, .expected = page, .replacement = {}}}}; + auto late_update = residency->prepare(edits, {}); + auto late_ready = residency->prepare_ready(page_cache_key(PageCacheKind::Leaf, input), {}); + fixture->cache->close_and_drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, 0U); + fixture.reset(); + PageCacheResidency::publish(std::move(late_update)); + PageCacheResidency::ready(late_ready); + PageCacheResidency::evict(late_ready); + residency->clear(); + late_ready.reset(); + residency.reset(); + EXPECT_EQ(resources->snapshot().resident_frames, 1U); + pin.reset(); + EXPECT_EQ(resources->snapshot().resident_frames, 0U); + EXPECT_EQ(resources->snapshot().cache_pinned_bytes, 0U); +} + +TEST(PageCacheResidency, SplitViewsShareOneFrameAndOnlyLivePagesContributeToProxyCount) +{ + TestCache fixture; + fixture.frame(1); + auto input = cold(1); + auto left_ref = input; + left_ref.range.upper = "m"; + auto right_ref = input; + right_ref.range.lower = "m"; + auto left = published({left_ref}); + auto right = published({right_ref}); + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = left }, + {.page_id = 1, .expected = {}, .replacement = right} + }, + 0) + .status.ok()); + const auto old_view = fixture.publisher.capture(); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 2U); + auto pin = fixture.load(input); + ASSERT_TRUE(pin); + const auto bytes = pin->page_bytes(); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 0U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 1U); + EXPECT_EQ(fixture.resources->snapshot().cache_bytes, bytes); + EXPECT_EQ(fixture.resources->snapshot().cache_pinned_bytes, bytes); + auto second_pin = fixture.load(right_ref); + EXPECT_EQ(fixture.resources->snapshot().cache_pinned_bytes, bytes); + fixture.cache->evict(); + fixture.runtime.drain(); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 2U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 1U); + EXPECT_EQ(fixture.resources->snapshot().cache_bytes, bytes); + auto overlay = published({}); + ASSERT_TRUE(fixture + .publish({ + {.page_id = 0, .expected = left, .replacement = overlay}, + {.page_id = 1, .expected = right, .replacement = {} } + }) + .status.ok()); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 0U); + EXPECT_EQ(old_view->mapping.get(1), right); + pin.reset(); + EXPECT_EQ(fixture.resources->snapshot().cache_pinned_bytes, bytes); + second_pin.reset(); + EXPECT_EQ(fixture.resources->snapshot().cache_pinned_bytes, 0U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 0U); + EXPECT_EQ(fixture.resources->snapshot().cache_bytes, 0U); +} + +TEST(PageCacheResidency, MultiBaseAndRepeatedBaseReferencesCountEachLogicalPageOnce) +{ + TestCache fixture; + fixture.frame(1); + fixture.frame(2); + auto first = cold(1); + auto second = cold(2); + auto page = published({first, second, first}); + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = page} + }, + 0) + .status.ok()); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 1U); + fixture.load(first).reset(); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 1U); + fixture.load(second).reset(); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 0U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 2U); + EXPECT_EQ(fixture.resources->snapshot().cache_pinned_bytes, 0U); + fixture.cache->evict(); + fixture.runtime.drain(); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 1U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 0U); +} + +TEST(PageCacheResidency, ReadyBaseInstalledIntoNewLiveViewsDoesNotBecomeAProxy) +{ + TestCache fixture; + fixture.frame(1); + const auto input = cold(1); + fixture.load(input).reset(); + auto page = published({input}); + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = page} + }, + 0) + .status.ok()); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 0U); + const auto loaded = fixture.resources->snapshot(); + const auto before = fixture.publisher.capture(); + auto duplicate = published({input}); + ASSERT_TRUE(fixture + .publish({ + {.page_id = 0, .expected = page, .replacement = duplicate} + }) + .status.ok()); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, loaded.resident_frames); + EXPECT_EQ(fixture.resources->snapshot().cache_bytes, loaded.cache_bytes); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 0U); + EXPECT_EQ(before->mapping.get(0), page); +} + +TEST(PageCacheResidency, LateLoadOfOldGenerationDoesNotMarkReplacementResident) +{ + TestCache fixture; + fixture.frame(1); + fixture.upstream->deferred = true; + const auto input = cold(1); + auto old_page = published({input}); + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = old_page} + }, + 0) + .status.ok()); + size_t done = 0; + fixture.loader->load_leaf(input, [&](Status status, auto frame) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(frame); + ++done; + }); + auto newer = input; + ++newer.generation; + auto replacement = published({newer}); + ASSERT_TRUE(fixture + .publish({ + {.page_id = 0, .expected = old_page, .replacement = replacement} + }) + .status.ok()); + fixture.upstream->finish(); + fixture.runtime.drain(); + EXPECT_EQ(done, 1U); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 1U); + fixture.loader->load_leaf(newer, [&](Status status, auto frame) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(frame); + ++done; + }); + fixture.upstream->finish(); + fixture.runtime.drain(); + EXPECT_EQ(done, 2U); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 0U); +} + +TEST(PageCacheResidency, FailedAndStalePublicationsLeaveLiveMembershipUnchanged) +{ + TestCache fixture; + auto page = published({cold(1)}); + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = page} + }, + 0) + .status.ok()); + auto replacement = published({}); + EXPECT_EQ(fixture + .publish({ + {.page_id = 0, .expected = {}, .replacement = replacement} + }) + .status.code(), + Code::kUnavailable); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 1U); + fixture.publisher.set_before_root_for_tests([] { throw std::bad_alloc(); }); + EXPECT_EQ(fixture + .publish({ + {.page_id = 0, .expected = page, .replacement = replacement} + }) + .status.code(), + Code::kResourceExhausted); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 1U); + EXPECT_EQ(fixture.publisher.capture()->mapping.get(0), page); + fixture.publisher.set_before_root_for_tests({}); + ASSERT_TRUE(fixture + .publish({ + {.page_id = 0, .expected = page, .replacement = replacement} + }) + .status.ok()); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 0U); +} + +TEST(PageCacheResidency, ExistingResidentAndOverlayOnlyPagesNeverContributeColdRefs) +{ + TestCache fixture; + fixture.frame(1); + auto resident = cold(1); + resident.resident = fixture.upstream->frames[1]; + auto page = published({resident}); + auto overlay = published({}); + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = page }, + {.page_id = 1, .expected = {}, .replacement = overlay} + }, + 0) + .status.ok()); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 0U); + fixture.cache->evict(); + fixture.runtime.drain(); + EXPECT_EQ(fixture.cache->snapshot().proxy_pages, 0U); + EXPECT_EQ(fixture.upstream->calls, 0U); + const auto sample = fixture.publisher.stats(*fixture.resources); + EXPECT_EQ(sample.logical.page_count(), 2U); + EXPECT_EQ(sample.resources.proxy_pages, 0U); +} + +TEST(PageCacheResidency, PreparedNodesRollbackAndReleaseAfterBudgetSealing) +{ + LoadCredits credits; + auto resources = std::make_shared<TreeResourceMetrics>(); + detail::FlushRuntime runtime(1, 4); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(4); + PageCacheResidency residency(resources, {}, &runtime, lane); + auto page = published({cold(1), cold(2)}); + const std::array<PageReplacement, 1> edits{{{.page_id = 0, .expected = {}, .replacement = page}}}; + auto update = residency.prepare(edits, credits.credits.bytes); + const auto allocated = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + EXPECT_GT(allocated, 0U); + EXPECT_EQ(resources->snapshot().proxy_pages, 0U); + update.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + update = residency.prepare(edits, credits.credits.bytes); + credits.seal(); + PageCacheResidency::publish(std::move(update)); + runtime.drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, 1U); + residency.clear(); + runtime.drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_cache_test.cpp b/lib/crowdb-tree/tests/unit/page_cache_test.cpp new file mode 100644 index 000000000..7381905e5 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_cache_test.cpp @@ -0,0 +1,572 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_cache.h" +#include "inner_load_test_support.h" +#include "load_budget_test_support.h" + +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +TEST(SharedPageCache, ConcurrentRangeViewsAndBindingsJoinOneLoadAndValidateEachSlotProof) +{ + SharedPageCache cache({.max_pages = 2, .max_waiters = 32}); + auto first = std::make_shared<TestLoader>(); + auto second = std::make_shared<TestLoader>(); + first->deferred = second->deferred = true; + first->frames[1] = second->frames[1] = base({ + {.key = "a", .slot = 7, .value = "value"} + }); + auto left = cache.bind(first); + auto right = cache.bind(second); + auto input = cold(1, 2); + input.generation = 3; + input.range.upper = "m"; + size_t invalid = 0; + left->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_FALSE(leaf); + ++invalid; + }); + input.max_slot = 10; + input.range = {.lower = "m"}; + std::atomic<size_t> done{0}; + std::vector<std::jthread> readers; + readers.reserve(16); + for (size_t i = 0; i < 16; ++i) { + readers.emplace_back([&, input] { + right->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_EQ(leaf, first->frames[1]); + done.fetch_add(1); + }); + }); + } + readers.clear(); + EXPECT_EQ(first->calls, 1U); + EXPECT_EQ(second->calls, 0U); + EXPECT_EQ(cache.snapshot().joined, 16U); + first->finish(); + EXPECT_EQ(invalid, 1U); + EXPECT_EQ(done.load(), 16U); + EXPECT_EQ(cache.snapshot().waiters, 0U); + EXPECT_EQ(cache.snapshot().resident_pages, 1U); + right->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_EQ(leaf, first->frames[1]); + done.fetch_add(1); + }); + EXPECT_EQ(done.load(), 17U); + EXPECT_EQ(cache.snapshot().hits, 1U); +} + +TEST(SharedPageCache, CancelledReaderDoesNotCancelOtherWaiterOrOverwriteNewOverlay) +{ + SharedPageCache cache; + auto upstream = std::make_shared<TestLoader>(); + upstream->deferred = true; + upstream->frames[1] = base({ + {.key = "a", .value = "base"} + }); + auto loader = cache.bind(upstream); + auto logical = std::make_shared<PageState>(); + logical->bases.push_back(cold(1)); + auto old_view = view(logical); + auto changed = std::make_shared<PageState>(*page({ + {.key = "b", .slot = 2, .value = "overlay"} + })); + changed->bases = logical->bases; + auto new_view = view(changed); + std::shared_ptr<PublishedGet> first; + std::shared_ptr<PublishedGet> second; + ASSERT_TRUE(PublishedGet::create(old_view, loader, "a", {}, &first).ok()); + ASSERT_TRUE(PublishedGet::create(new_view, loader, "a", {}, &second).ok()); + Completion a; + Completion b; + auto op_a = stdexec::connect(first->sender(), Receiver{.done = &a}); + auto op_b = stdexec::connect(second->sender(), Receiver{.done = &b}); + stdexec::start(op_a); + stdexec::start(op_b); + EXPECT_EQ(upstream->calls, 1U); + first->request_stop(); + EXPECT_EQ(a.stopped.load(), 1U); + upstream->finish(); + EXPECT_EQ(a.stopped.load(), 1U); + EXPECT_EQ(a.values.load(), 0U); + EXPECT_EQ(b.values.load(), 1U); + EXPECT_EQ(second->result().cell.value.to_string(), "base"); + EXPECT_EQ(new_view->mapping.get(0)->leaf, changed); + EXPECT_EQ(changed->lookup("b").cell.value.to_string(), "overlay"); + EXPECT_FALSE(changed->bases[0].resident); +} + +class TestReplayLoader final : public PageLoader +{ + public: + std::vector<LeafComplete> callbacks; + + void load_leaf(BaseRef /*base*/, LeafComplete done) override + { + callbacks.push_back(std::move(done)); + } +}; + +TEST(SharedPageCache, FailureRetriesAndLateDuplicateCannotCompleteRecycledEntry) +{ + SharedPageCache cache({.max_pages = 1, .max_waiters = 4}); + auto upstream = std::make_shared<TestReplayLoader>(); + auto loader = cache.bind(upstream); + auto input = cold(1); + size_t failed = 0; + size_t completed = 0; + loader->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kIoError); + EXPECT_FALSE(leaf); + ++failed; + }); + ASSERT_EQ(upstream->callbacks.size(), 1U); + auto obsolete = upstream->callbacks[0]; + obsolete(Status::io_error(), {}); + EXPECT_EQ(failed, 1U); + EXPECT_EQ(cache.snapshot().resident_pages, 0U); + loader->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + ASSERT_TRUE(leaf); + EXPECT_EQ(CellView(leaf->cell(0)).value().to_string(), "new"); + ++completed; + }); + ASSERT_EQ(upstream->callbacks.size(), 2U); + obsolete(Status::Ok(), base({ + {.key = "a", .value = "obsolete"} + })); + EXPECT_EQ(completed, 0U); + upstream->callbacks[1](Status::Ok(), base({ + {.key = "a", .value = "new"} + })); + EXPECT_EQ(completed, 1U); + EXPECT_EQ(cache.snapshot().inflight, 0U); + EXPECT_EQ(cache.snapshot().waiters, 0U); +} + +TEST(SharedPageCache, IdentityIncludesGenerationAndBackendProtection) +{ + SharedPageCache cache({.max_pages = 4, .max_waiters = 8}); + auto upstream = std::make_shared<TestLoader>(); + upstream->frames[1] = base({ + {.key = "a", .value = "one"} + }); + auto loader = cache.bind(upstream); + auto input = cold(1); + auto read = [&](const BaseRef &base) { + loader->load_leaf(base, [](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + }); + }; + read(input); + read(input); + EXPECT_EQ(upstream->calls, 1U); + ++input.generation; + read(input); + EXPECT_EQ(upstream->calls, 2U); + auto backing = std::make_shared<BackingRef>(*input.backing); + backing->owner = std::make_shared<int>(2); + input.backing = backing; + read(input); + EXPECT_EQ(upstream->calls, 3U); + auto newer = std::make_shared<BackingRef>(*backing); + ++newer->generation; + input.backing = newer; + read(input); + EXPECT_EQ(upstream->calls, 4U); +} + +TEST(SharedPageCache, FixedCapacityRejectsBeforeIoAndCallbacksCanReenter) +{ + SharedPageCache cache({.max_pages = 1, .max_waiters = 2}); + auto upstream = std::make_shared<TestLoader>(); + upstream->deferred = true; + upstream->frames[1] = upstream->frames[2] = base({ + {.key = "a", .value = "value"} + }); + auto loader = cache.bind(upstream); + auto input = cold(1); + size_t done = 0; + loader->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++done; + loader->load_leaf(input, [&](Status nested, auto frame) { + EXPECT_TRUE(nested.ok()); + EXPECT_TRUE(frame); + ++done; + }); + }); + EXPECT_EQ(upstream->calls, 1U); + loader->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++done; + }); + loader->load_leaf(input, [](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kUnavailable); + EXPECT_FALSE(leaf); + }); + upstream->finish(); + EXPECT_EQ(done, 3U); + EXPECT_EQ(cache.snapshot().waiters, 0U); + EXPECT_EQ(cache.snapshot().rejected, 1U); +} + +TEST(SharedPageCache, ReservedWaiterSurvivesOrdinaryPressureAndWaitsForInFlightPageCapacity) +{ + LoadCredits credits; + SharedPageCache cache({.max_pages = 1, .max_waiters = 2}, credits.credits.bytes); + auto upstream = std::make_shared<TestLoader>(); + upstream->deferred = true; + upstream->frames[1] = upstream->frames[2] = base({ + {.key = "a", .value = "value"} + }); + auto ordinary = cache.bind(upstream, credits.credits.bytes); + std::shared_ptr<PageLoader> reserved; + ASSERT_TRUE(cache.bind_reserved(upstream, credits.credits.bytes, &reserved).ok()); + EXPECT_EQ(cache.snapshot().reserved_waiters, 1U); + credits.seal(); + size_t ordinary_done = 0; + size_t reserved_done = 0; + ordinary->load_leaf(cold(1), [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++ordinary_done; + }); + ordinary->load_leaf(cold(1), [](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kUnavailable); + EXPECT_FALSE(leaf); + }); + reserved->load_leaf(cold(2), [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++reserved_done; + }); + EXPECT_EQ(cache.snapshot().waiters, 2U); + EXPECT_EQ(upstream->calls, 1U); + upstream->finish(); + EXPECT_EQ(ordinary_done, 1U); + EXPECT_EQ(reserved_done, 0U); + EXPECT_EQ(upstream->calls, 2U); + upstream->finish(); + EXPECT_EQ(reserved_done, 1U); + EXPECT_EQ(cache.snapshot().waiters, 0U); + reserved.reset(); + EXPECT_EQ(cache.snapshot().reserved_waiters, 0U); +} + +TEST(SharedPageCache, EvictionReturnsCacheReferenceButPinnedFrameRemainsCharged) +{ + LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestInnerStore>(); + auto frame = base({ + {.key = "a", .value = "value"} + }); + auto &blob = store->blobs[0]; + encode_durable_page(frame->frame(), frame->page_bytes(), compress_algo::kNone, &blob); + auto backing = std::make_shared<BackingRef>(); + backing->length = blob.size(); + backing->owner = store; + auto upstream = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, + OverflowBackingResolver{}, credits.credits); + { + SharedPageCache cache({.max_pages = 2, .max_waiters = 8}, credits.credits.bytes); + auto loader = cache.bind(upstream, credits.credits.bytes); + const auto infrastructure = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + std::shared_ptr<const LeafBase> pinned; + loader->load_leaf({.identity = 1, .backing = backing}, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + pinned = std::move(leaf); + }); + runtime->drain(); + ASSERT_TRUE(pinned); + credits.seal(); + const auto allocated = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + EXPECT_GT(allocated, infrastructure); + EXPECT_EQ(cache.snapshot().resident_pages, 1U); + loader->load_leaf({.identity = 1, .backing = backing}, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_EQ(leaf, pinned); + }); + EXPECT_EQ(store->reads.load(), 1U); + cache.evict(); + EXPECT_EQ(cache.snapshot().resident_pages, 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], allocated); + EXPECT_EQ(CellView(pinned->cell(0)).value().to_string(), "value"); + pinned.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], infrastructure); + } + upstream.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(SharedPageCache, FailedBindingAllocationReturnsReservedSlot) +{ + LoadCredits credits; + SharedPageCache cache({.max_pages = 1, .max_waiters = 1}); + auto upstream = std::make_shared<TestLoader>(); + upstream->frames[1] = base({ + {.key = "a", .value = "value"} + }); + credits.seal(); + std::shared_ptr<PageLoader> binding; + EXPECT_EQ(cache.bind_reserved(upstream, credits.credits.bytes, &binding).code(), Code::kResourceExhausted); + EXPECT_FALSE(binding); + EXPECT_EQ(cache.snapshot().reserved_waiters, 0U); + ASSERT_TRUE(cache.bind_reserved(upstream, {}, &binding).ok()); + size_t done = 0; + binding->load_leaf(cold(1), [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++done; + }); + EXPECT_EQ(done, 1U); + binding.reset(); + EXPECT_EQ(cache.snapshot().reserved_waiters, 0U); +} + +TEST(SharedPageCache, ResidentByteLimitEvictsWithoutInvalidatingPinnedFrame) +{ + auto upstream = std::make_shared<TestLoader>(); + auto frame = base({ + {.key = "a", .value = "value"} + }); + upstream->frames[1] = upstream->frames[2] = frame; + SharedPageCache cache({.max_pages = 4, .max_waiters = 2, .max_resident_bytes = frame->page_bytes()}); + auto loader = cache.bind(upstream); + const auto first = cold(1); + const auto second = cold(2); + std::shared_ptr<const LeafBase> pinned; + loader->load_leaf(first, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + pinned = std::move(leaf); + }); + loader->load_leaf(second, [](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + }); + EXPECT_EQ(cache.snapshot().resident_pages, 1U); + EXPECT_EQ(cache.snapshot().resident_bytes, frame->page_bytes()); + EXPECT_EQ(cache.snapshot().evictions, 1U); + ASSERT_TRUE(pinned); + EXPECT_EQ(CellView(pinned->cell(0)).value().to_string(), "value"); + loader->load_leaf(second, [](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + }); + EXPECT_EQ(upstream->calls, 2U); + loader->load_leaf(first, [](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + }); + EXPECT_EQ(upstream->calls, 3U); +} + +TEST(SharedPageCache, MissingSuccessfulPayloadFailsWaitersAndAllowsRetry) +{ + SharedPageCache cache({.max_pages = 1, .max_waiters = 2}); + auto upstream = std::make_shared<TestReplayLoader>(); + auto loader = cache.bind(upstream); + const auto input = cold(1); + size_t failed = 0; + for (size_t i = 0; i < 2; ++i) { + loader->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_FALSE(leaf); + ++failed; + }); + } + ASSERT_EQ(upstream->callbacks.size(), 1U); + upstream->callbacks[0](Status::Ok(), {}); + EXPECT_EQ(failed, 2U); + EXPECT_EQ(cache.snapshot().resident_pages, 0U); + EXPECT_EQ(cache.snapshot().waiters, 0U); + size_t done = 0; + loader->load_leaf(input, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++done; + }); + ASSERT_EQ(upstream->callbacks.size(), 2U); + upstream->callbacks[1](Status::Ok(), base({ + {.key = "a", .value = "value"} + })); + EXPECT_EQ(done, 1U); +} + +TEST(SharedPageCache, CloseAndEvictionBeforeCompletionDoNotRetainLateResult) +{ + for (const bool close : {false, true}) { + SharedPageCache cache; + auto upstream = std::make_shared<TestLoader>(); + upstream->deferred = true; + upstream->frames[1] = base({ + {.key = "a", .value = "value"} + }); + auto loader = cache.bind(upstream); + size_t done = 0; + loader->load_leaf(cold(1), [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++done; + }); + if (close) { + cache.close(); + } + else { + cache.evict(); + } + upstream->finish(); + EXPECT_EQ(done, 1U); + EXPECT_EQ(cache.snapshot().resident_pages, 0U); + EXPECT_EQ(cache.snapshot().inflight, 0U); + if (close) { + loader->load_leaf(cold(1), [](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kUnavailable); + EXPECT_FALSE(leaf); + }); + } + } +} + +TEST(SharedPageCache, TwoColdIndexReadersShareBothLoadsWithOneWorker) +{ + TestColdTree tree; + tree.store->deferred = true; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto upstream = std::make_shared<AsyncStorePageLoader>(tree.store, runtime, lane); + SharedPageCache cache; + auto loader = cache.bind(upstream); + std::shared_ptr<PublishedGet> a; + std::shared_ptr<PublishedGet> b; + ASSERT_TRUE(PublishedGet::create(tree.view, loader, "a", {}, &a).ok()); + ASSERT_TRUE(PublishedGet::create(tree.view, loader, "g", {}, &b).ok()); + Completion first; + Completion second; + auto op_a = stdexec::connect(a->sender(), Receiver{.done = &first}); + auto op_b = stdexec::connect(b->sender(), Receiver{.done = &second}); + stdexec::start(op_a); + stdexec::start(op_b); + EXPECT_EQ(tree.store->reads.load(), 1U); + tree.store->finish(); + runtime->drain(); + EXPECT_EQ(tree.store->reads.load(), 2U); + tree.store->finish(); + runtime->drain(); + EXPECT_EQ(first.values.load(), 1U); + EXPECT_EQ(second.values.load(), 1U); + EXPECT_EQ(a->result().cell.value.to_string(), "a"); + EXPECT_EQ(b->result().cell.value.to_string(), "g"); + EXPECT_EQ(cache.snapshot().misses, 2U); + EXPECT_EQ(cache.snapshot().joined, 2U); +} + +TEST(SharedPageCache, HighWaterPressureReclaimsToLowWaterAndKeepsReadPinsValid) +{ + auto upstream = std::make_shared<TestLoader>(); + auto frame = base({ + {.key = "a", .value = "value"} + }); + const auto bytes = frame->page_bytes(); + SharedPageCache cache( + {.max_pages = 8, .max_waiters = 2, .max_resident_bytes = 4 * bytes, .low_resident_bytes = 2 * bytes}); + auto loader = cache.bind(upstream); + std::shared_ptr<const LeafBase> pinned; + for (uint64_t pid = 1; pid <= 5; ++pid) { + upstream->frames[pid] = frame; + loader->load_leaf(cold(pid), [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + if (!pinned) { + pinned = std::move(leaf); + } + }); + } + EXPECT_EQ(cache.snapshot().resident_bytes, 2 * bytes); + EXPECT_EQ(cache.snapshot().resident_pages, 2U); + EXPECT_EQ(cache.snapshot().evictions, 3U); + ASSERT_TRUE(pinned); + EXPECT_EQ(CellView(pinned->cell(0)).value().to_string(), "value"); +} + +TEST(SharedPageCache, ForegroundReadsHaveBoundedPriorityOverQueuedMaintenance) +{ + SharedPageCache cache({.max_pages = 1, .max_waiters = 8, .max_read_burst = 2}); + auto upstream = std::make_shared<TestReplayLoader>(); + auto read = cache.bind(upstream); + std::shared_ptr<PageLoader> maintenance; + ASSERT_TRUE(cache.bind_reserved(upstream, {}, &maintenance, PageLoadPriority::Maintenance).ok()); + std::vector<uint64_t> completed; + auto load = [&](const std::shared_ptr<PageLoader> &loader, uint64_t pid) { + loader->load_leaf(cold(pid), [&, pid](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + completed.push_back(pid); + }); + }; + load(read, 1); + load(maintenance, 2); + load(read, 3); + load(read, 4); + load(read, 5); + auto frame = base({ + {.key = "a", .value = "value"} + }); + for (size_t i = 0; i < 5; ++i) { + ASSERT_EQ(upstream->callbacks.size(), i + 1); + auto done = upstream->callbacks[i]; + done(Status::Ok(), frame); + } + EXPECT_EQ(completed, (std::vector<uint64_t>{1, 3, 4, 2, 5})); + EXPECT_EQ(cache.snapshot().waiters, 0U); +} + +TEST(SharedPageCache, MaintenanceFramesAreEvictedBeforeRecentlyReadFrames) +{ + SharedPageCache cache({.max_pages = 2, .max_waiters = 4}); + auto upstream = std::make_shared<TestLoader>(); + auto frame = base({ + {.key = "a", .value = "value"} + }); + for (uint64_t pid = 1; pid <= 3; ++pid) { + upstream->frames[pid] = frame; + } + auto read = cache.bind(upstream); + auto maintenance = cache.bind(upstream, {}, PageLoadPriority::Maintenance); + auto load = [](const std::shared_ptr<PageLoader> &loader, const BaseRef &input) { + loader->load_leaf(input, [](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + }); + }; + const auto hot = cold(1); + const auto background = cold(2); + load(read, hot); + load(maintenance, background); + load(maintenance, cold(3)); + EXPECT_EQ(upstream->calls, 3U); + load(read, hot); + EXPECT_EQ(upstream->calls, 3U); + load(read, background); + EXPECT_EQ(upstream->calls, 4U); +} + +TEST(SharedPageCache, InvalidWatermarksAndUnboundedReadPriorityRejectAtConstruction) +{ + EXPECT_THROW(SharedPageCache({.max_resident_bytes = 10, .low_resident_bytes = 11}), std::invalid_argument); + EXPECT_THROW(SharedPageCache({.max_read_burst = 0}), std::invalid_argument); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_cache_trim_test.cpp b/lib/crowdb-tree/tests/unit/page_cache_trim_test.cpp new file mode 100644 index 000000000..f6be4cc79 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_cache_trim_test.cpp @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_cache.h" +#include "inner_load_test_support.h" + +#include <array> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +struct TestTrim +{ + size_t count = SIZE_MAX; + PageCacheTrim request; + + TestTrim(PageCacheKind kind, size_t retained) + : request{.kind = kind, + .max_pages = retained, + .complete = [](void *context, + size_t evicted) noexcept { static_cast<TestTrim *>(context)->count = evicted; }, + .context = this} + { + } +}; + +class TestTrimLoader final : public PageLoader +{ + public: + std::shared_ptr<const LeafBase> leaf = base({ + {.key = "key", .value = "value"} + }); + std::shared_ptr<InnerPageState> inner = std::make_shared<InnerPageState>(); + size_t leaves = 0; + size_t indexes = 0; + + TestTrimLoader() + { + inner->children.push_back(10); + } + + void load_leaf(BaseRef /*base*/, LeafComplete complete) override + { + ++leaves; + complete(Status::Ok(), leaf); + } + + void load_inner(std::shared_ptr<const BackingRef> /*backing*/, InnerComplete complete) override + { + ++indexes; + complete(Status::Ok(), inner); + } +}; + +TEST(PageCacheTrim, LeafAndInnerBudgetsAreIndependentAndKeepRecentPinsValid) +{ + SharedPageCache cache({.max_pages = 8, .max_waiters = 4}); + auto upstream = std::make_shared<TestTrimLoader>(); + auto loader = cache.bind(upstream); + const std::array<BaseRef, 4> frames{cold(1), cold(2), cold(3), cold(4)}; + std::shared_ptr<const LeafBase> pinned; + auto leaf = [&](uint64_t pid) { + loader->load_leaf(frames.at(pid - 1), [&](Status status, auto value) { + ASSERT_TRUE(status.ok()); + pinned = std::move(value); + }); + }; + auto inner = [&](uint64_t pid) { + loader->load_inner(frames.at(pid - 1).backing, [](Status status, auto value) { + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(value); + }); + }; + leaf(1); + leaf(2); + inner(3); + inner(4); + leaf(1); + ASSERT_EQ(cache.snapshot().resident_pages, 4U); + TestTrim leaves(PageCacheKind::Leaf, 1); + cache.trim(leaves.request); + EXPECT_EQ(leaves.count, 1U); + EXPECT_EQ(cache.snapshot().resident_pages, 3U); + leaf(1); + inner(3); + inner(4); + EXPECT_EQ(upstream->leaves, 2U); + EXPECT_EQ(upstream->indexes, 2U); + TestTrim indexes(PageCacheKind::Inner, 0); + cache.trim(indexes.request); + EXPECT_EQ(indexes.count, 2U); + EXPECT_EQ(cache.snapshot().resident_pages, 1U); + EXPECT_EQ(CellView(pinned->cell(0)).value().to_string(), "value"); + leaf(2); + EXPECT_EQ(upstream->leaves, 3U); +} + +TEST(PageCacheTrim, ReentrantCompletionQueuesTrimWithoutBlockingTheLoadOwner) +{ + SharedPageCache cache; + auto upstream = std::make_shared<TestTrimLoader>(); + auto loader = cache.bind(upstream); + TestTrim trim(PageCacheKind::Leaf, 0); + loader->load_leaf(cold(1), [&](Status status, auto leaf) { + ASSERT_TRUE(status.ok()); + cache.trim(trim.request); + EXPECT_EQ(trim.count, SIZE_MAX); + EXPECT_EQ(CellView(leaf->cell(0)).value().to_string(), "value"); + }); + EXPECT_EQ(trim.count, 1U); + EXPECT_EQ(cache.snapshot().resident_pages, 0U); + EXPECT_EQ(cache.snapshot().waiters, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_cache_wake_test.cpp b/lib/crowdb-tree/tests/unit/page_cache_wake_test.cpp new file mode 100644 index 000000000..21153a0cb --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_cache_wake_test.cpp @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_runtime.h" +#include "btree/page_cache.h" +#include "btree/page_load_internal.h" +#include "host_completion.h" +#include "published_read_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +class TestAsyncLoader final : public PageLoader +{ + public: + void load_leaf(BaseRef /*base*/, LeafComplete complete) override + { + auto pending = + std::make_shared<Pending>(Pending{.complete = std::move(complete), .leaf = leaf_, .owner = nullptr}); + pending->owner = pending; + auto status = runtime_.try_submit(lane_, &Pending::run, pending.get()); + if (!status.ok()) { + pending->owner.reset(); + complete_page_load(pending->complete, std::move(status), std::shared_ptr<const LeafBase>{}); + } + } + + private: + struct Pending + { + LeafComplete complete; + std::shared_ptr<const LeafBase> leaf; + std::shared_ptr<Pending> owner; + + static void run(void *context) noexcept + { + auto self = std::move(static_cast<Pending *>(context)->owner); + complete_page_load(self->complete, Status::Ok(), std::move(self->leaf)); + } + }; + + detail::FlushRuntime runtime_{1, 8}; + std::shared_ptr<detail::FlushRuntime::Lane> lane_ = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<const LeafBase> leaf_ = test_read::base({ + {.key = "a", .value = "value"} + }); +}; + +TEST(SharedPageCacheWake, CrossThreadCompletionAndNextColdRequestCannotLeaveIdleIncomingWork) +{ + SharedPageCache cache({.max_pages = 1, .max_waiters = 4}); + auto upstream = std::make_shared<TestAsyncLoader>(); + auto binding = cache.bind(upstream); + const std::array<BaseRef, 2> inputs{test_read::cold(1), test_read::cold(2)}; + constexpr size_t iterations = 50000; + for (size_t index = 0; index < iterations; ++index) { + detail::HostCompletion done; + Status status; + std::shared_ptr<const LeafBase> leaf; + binding->load_leaf(inputs[index % inputs.size()], [&](Status outcome, auto result) { + status = std::move(outcome); + leaf = std::move(result); + done.signal(); + }); + done.wait(); + ASSERT_TRUE(status.ok()) << index << ' ' << status.to_string(); + ASSERT_TRUE(leaf) << index; + ASSERT_EQ(leaf->count(), 1U); + } + EXPECT_EQ(cache.snapshot().misses, iterations); + EXPECT_EQ(cache.snapshot().waiters, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_compression_test.cpp b/lib/crowdb-tree/tests/unit/page_compression_test.cpp index edae22326..9931f27ab 100644 --- a/lib/crowdb-tree/tests/unit/page_compression_test.cpp +++ b/lib/crowdb-tree/tests/unit/page_compression_test.cpp @@ -8,6 +8,7 @@ #include <gtest/gtest.h> +#include <algorithm> #include <array> #include <cstdio> #include <random> @@ -69,7 +70,7 @@ TEST(PageCompression, CompressibleShrinksWhenLz4Available) TEST(PageCompression, IncompressibleStoredRaw) { std::vector<uint8_t> frame(kPb); - std::mt19937 rng(1234); + std::mt19937 rng(1234); // NOLINT(bugprone-random-generator-seed) repeatable incompressible input for (auto &b : frame) { b = static_cast<uint8_t>(rng()); } @@ -109,3 +110,66 @@ TEST(PageCompression, ShortBlobRejected) std::vector<uint8_t> back(kPb); EXPECT_EQ(decode_durable_page(b.data(), b.size(), back.data(), kPb).code(), Code::kInvalidArgument); } + +TEST(PageCompression, BoundedEncoderUsesUnalignedCallerScratchAndExistingDecoder) +{ + const auto frame = compressible_frame(); + for (const auto algorithm : {compress_algo::kNone, compress_algo::kLz4}) { + std::vector<uint8_t> output(kDurableBlobHeader + frame.size() + 2, 0xA5); + std::vector<uint8_t> scratch(durable_encode_scratch_bytes(algorithm) + 1); + uint32_t length = 0; + ASSERT_TRUE(encode_durable_page_into(frame, algorithm, std::span<uint8_t>(output).subspan(1, output.size() - 2), + std::span<uint8_t>(scratch).subspan(1), &length) + .ok()); + EXPECT_EQ(output.front(), 0xA5); + EXPECT_EQ(output.back(), 0xA5); + std::vector<uint8_t> restored(frame.size()); + ASSERT_TRUE(decode_durable_page(output.data() + 1, length, restored.data(), restored.size()).ok()); + EXPECT_EQ(restored, frame); + if (algorithm == compress_algo::kLz4 && lz4_available()) { + EXPECT_LT(length, frame.size()); + } + else { + EXPECT_EQ(length, kDurableBlobHeader + frame.size()); + } + } +} + +TEST(PageCompression, BoundedEncoderRawFallbackOverwritesPartialCompressedOutput) +{ + std::vector<uint8_t> frame(kPb); + std::mt19937 random(918); // NOLINT(bugprone-random-generator-seed) repeatable incompressible input + for (auto &byte : frame) { + byte = static_cast<uint8_t>(random()); + } + std::vector<uint8_t> output(kDurableBlobHeader + frame.size()); + std::vector<uint8_t> scratch(durable_encode_scratch_bytes(compress_algo::kLz4)); + uint32_t length = 0; + ASSERT_TRUE(encode_durable_page_into(frame, compress_algo::kLz4, output, scratch, &length).ok()); + EXPECT_EQ(output[0], static_cast<uint8_t>(compress_algo::kNone)); + std::vector<uint8_t> legacy; + encode_durable_page(frame.data(), frame.size(), compress_algo::kNone, &legacy); + EXPECT_EQ(output, legacy); + EXPECT_EQ(length, output.size()); +} + +TEST(PageCompression, BoundedEncoderRejectsInsufficientWindowsBeforeMutation) +{ + const auto frame = compressible_frame(); + std::vector<uint8_t> output(kDurableBlobHeader + frame.size(), 0xA5); + std::vector<uint8_t> scratch(durable_encode_scratch_bytes(compress_algo::kLz4)); + uint32_t length = 123; + EXPECT_EQ(encode_durable_page_into(frame, compress_algo::kLz4, std::span<uint8_t>(output).first(output.size() - 1), + scratch, &length) + .code(), + Code::kResourceExhausted); + EXPECT_EQ(length, 123U); + EXPECT_TRUE(std::all_of(output.begin(), output.end(), [](auto byte) { return byte == 0xA5; })); + if (!scratch.empty()) { + EXPECT_EQ(encode_durable_page_into(frame, compress_algo::kLz4, output, + std::span<uint8_t>(scratch).first(scratch.size() - 1), &length) + .code(), + Code::kResourceExhausted); + EXPECT_EQ(length, 123U); + } +} diff --git a/lib/crowdb-tree/tests/unit/page_encode_test.cpp b/lib/crowdb-tree/tests/unit/page_encode_test.cpp new file mode 100644 index 000000000..953ecf2b6 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_encode_test.cpp @@ -0,0 +1,352 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_encode.h" +#include "btree/tree_metrics.h" + +#include <gtest/gtest.h> + +#include <array> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<const CellVersion> encode_cell_owner(uint64_t slot, std::string value, bool deleted = false) +{ + return std::make_shared<CellVersion>(encode_cell_buf(slot, deleted ? OpKind::kDelete : OpKind::kPut, value), slot, + deleted ? kFlagTombstone : 0); +} + +std::shared_ptr<PublishedPage> +encoding_leaf(const std::vector<std::pair<std::string, std::shared_ptr<const CellVersion>>> &entries, + PageKeyRange range = {}) +{ + OverlayBuilder builder({}); + for (const auto &[key, cell] : entries) { + EXPECT_TRUE(builder.append(Slice(key), cell).ok()); + } + std::shared_ptr<const OverlayBatch> run; + EXPECT_TRUE(builder.finish(1, &run).ok()); + auto logical = std::make_shared<PageState>(); + logical->range = std::move(range); + if (run) { + logical->overlays.push_back(std::move(run)); + } + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::move(logical); + return page; +} + +std::shared_ptr<PublishedTreeState> encoding_capture(std::vector<std::shared_ptr<PublishedPage>> pages, + uint64_t root = 0, uint64_t coverage = 10, uint64_t gc_floor = 0) +{ + auto view = std::make_shared<PublishedTreeState>(); + std::vector<PageReplacement> replacements; + replacements.reserve(pages.size()); + for (size_t index = 0; index < pages.size(); ++index) { + replacements.push_back({.page_id = index, .expected = {}, .replacement = std::move(pages[index])}); + } + MappingView::CopyStats copied; + view->mapping = view->mapping.replace(replacements, copied); + view->root_page_id = root; + view->next_page_id = pages.size(); + view->coverage = coverage; + view->gc_floor = gc_floor; + return view; +} + +TEST(FinalPageEncoder, LeafResolvesCapturedWinnersAndGcWithoutIntermediateValueCopies) +{ + auto page = encoding_leaf({ + {"a", encode_cell_owner(3, "new")}, + {"b", encode_cell_owner(2, "", true)}, + {"c", encode_cell_owner(4, "", true)}, + {"d", encode_cell_owner(11, "future")} + }); + auto old = encoding_leaf({ + {"a", encode_cell_owner(1, "old") }, + {"d", encode_cell_owner(5, "captured")} + }); + auto merged = std::make_shared<PageState>(*page->leaf); + merged->overlays.push_back(old->leaf->overlays.front()); + page->leaf = merged; + auto captured = encoding_capture({page}, 0, 10, 2); + auto prepared = prepare_final_page(captured, 0, {}, {.frame_bytes = 512}); + ASSERT_TRUE(prepared.status.ok()); + ASSERT_NE(prepared.encoder, nullptr); + EXPECT_EQ(prepared.encoder->summary().encoded_entries, 3U); + EXPECT_TRUE(prepared.encoder->summary().live_counts_known); + EXPECT_EQ(prepared.encoder->summary().dropped_tombstones, 1U); + EXPECT_EQ(prepared.encoder->counters().inline_value_copy_bytes, 0U); + std::array<uint8_t, 512> frame{}; + auto step = prepared.encoder->next(frame); + ASSERT_TRUE(step.status.ok()); + ASSERT_TRUE(step.complete); + EXPECT_TRUE(frame_validate(frame.data(), frame.size())); + LeafFrameView leaf(frame.data(), frame.size()); + ASSERT_EQ(leaf.count(), 3U); + EXPECT_EQ(CellView(leaf.cell(0)).value().to_string(), "new"); + EXPECT_TRUE(CellView(leaf.cell(1)).is_tombstone()); + EXPECT_EQ(CellView(leaf.cell(2)).value().to_string(), "captured"); + EXPECT_EQ(frame_lower_fence(frame.data()).to_string(), "a"); + EXPECT_EQ(frame_upper_fence(frame.data()).to_string(), "d"); + EXPECT_EQ(prepared.encoder->counters().inline_value_copy_bytes, 11U); + EXPECT_EQ(prepared.encoder->counters().key_copy_bytes, 3U); + EXPECT_FALSE(prepared.encoder->next(frame).frame.has_value()); +} + +TEST(FinalPageEncoder, LargeExternalValueStreamsThroughOneCallerFrameAndRetainsOwner) +{ + auto payload = std::make_shared<std::string>(16 * 1024, 'v'); + auto cell = std::make_shared<CellVersion>( + buffer::wrap_external(reinterpret_cast<const uint8_t *>(payload->data()), payload->size(), + new std::shared_ptr<std::string>(payload), + [](void *owner) { delete static_cast<std::shared_ptr<std::string> *>(owner); }), + 7, 0); + std::weak_ptr<const CellVersion> lifetime = cell; + auto captured = encoding_capture({ + encoding_leaf({{"k", cell}} + ) + }); + auto prepared = prepare_final_page(captured, 0, {}, {.frame_bytes = 256, .inline_value_bytes = 16}); + ASSERT_TRUE(prepared.status.ok()); + ASSERT_NE(prepared.encoder, nullptr); + captured.reset(); + cell.reset(); + EXPECT_FALSE(lifetime.expired()); + const auto count = prepared.encoder->summary().new_overflow_frames; + EXPECT_EQ(count, (payload->size() + overflow_chunk_cap(256) - 1) / overflow_chunk_cap(256)); + std::array<uint8_t, 257> frame{}; + frame.back() = 0xAA; + EXPECT_EQ(prepared.encoder->next(frame).status.code(), Code::kUnavailable); + EXPECT_EQ(prepared.encoder->bind_overflow({.first = 0, .count = count, .owner = payload}).code(), + Code::kInvalidArgument); + ASSERT_TRUE(prepared.encoder->bind_overflow({.first = 100, .count = count, .owner = payload}).ok()); + EXPECT_EQ(prepared.encoder->next(std::span<uint8_t>(frame.data(), 255)).status.code(), Code::kInvalidArgument); + size_t offset = 0; + for (uint64_t index = 0; index < count; ++index) { + auto step = prepared.encoder->next(frame); + ASSERT_TRUE(step.status.ok()); + if (!step.frame.has_value()) { + ADD_FAILURE() << "overflow stream did not emit a frame"; + return; + } + EXPECT_FALSE(step.complete); + EXPECT_EQ(step.frame->page_id, 100 + index); + EXPECT_TRUE(frame_validate(frame.data(), 256)); + OverflowFrameView overflow(frame.data(), 256); + EXPECT_EQ(overflow.payload(), Slice(payload->data() + offset, overflow.chunk_len())); + EXPECT_EQ(overflow.next_page_id(), index + 1 == count ? kInvalidPageId : 101 + index); + offset += overflow.chunk_len(); + EXPECT_EQ(frame.back(), 0xAA); + } + EXPECT_EQ(offset, payload->size()); + auto primary = prepared.encoder->next(frame); + ASSERT_TRUE(primary.status.ok()); + EXPECT_TRUE(primary.complete); + EXPECT_TRUE(frame_validate(frame.data(), 256)); + LeafFrameView leaf(frame.data(), 256); + EXPECT_EQ(CellView(leaf.cell(0)).overflow_head(), 100U); + EXPECT_EQ(CellView(leaf.cell(0)).overflow_len(), payload->size()); + EXPECT_EQ(prepared.encoder->counters().overflow_value_copy_bytes, payload->size()); + EXPECT_EQ(prepared.encoder->counters().inline_value_copy_bytes, 0U); + EXPECT_EQ(prepared.encoder->counters().primary_frames, 1U); + prepared.encoder.reset(); + EXPECT_FALSE(lifetime.expired()); + primary.frame.reset(); + EXPECT_TRUE(lifetime.expired()); +} + +TEST(FinalPageEncoder, ExistingOverflowRequiresCapturedProtectionAndCopiesNoPayload) +{ + auto cell = std::make_shared<CellVersion>(encode_overflow_cell_buf(2, 40, 99999), 2, kFlagOverflow); + auto captured = encoding_capture({ + encoding_leaf({{"k", cell}} + ) + }); + captured->next_page_id = 100; + EXPECT_EQ(prepare_final_page(captured, 0, {}).status.code(), Code::kInvalidArgument); + auto protection = std::make_shared<int>(1); + auto prepared = prepare_final_page(captured, 0, {.overflow_dependencies = protection}, {.frame_bytes = 256}); + ASSERT_TRUE(prepared.status.ok()); + ASSERT_NE(prepared.encoder, nullptr); + ASSERT_EQ(prepared.encoder->reused_overflow().size(), 1U); + EXPECT_EQ(prepared.encoder->reused_overflow()[0].owner, protection); + EXPECT_EQ(prepared.encoder->summary().new_overflow_frames, 0U); + EXPECT_EQ(prepared.encoder->summary().live_value_bytes, 99999U); + std::array<uint8_t, 256> frame{}; + EXPECT_TRUE(prepared.encoder->next(frame).complete); + LeafFrameView leaf(frame.data(), frame.size()); + EXPECT_EQ(CellView(leaf.cell(0)).overflow_head(), 40U); + EXPECT_EQ(prepared.encoder->counters().overflow_value_copy_bytes, 0U); + EXPECT_EQ(prepared.encoder->counters().inline_value_copy_bytes, 0U); +} + +TEST(FinalPageEncoder, OversizedPrimaryAndBoundedWorkFailBeforeOutput) +{ + auto captured = encoding_capture({ + encoding_leaf({{"a", encode_cell_owner(1, std::string(30, 'x'))}, + {"b", encode_cell_owner(1, std::string(30, 'x'))}, + {"c", encode_cell_owner(1, std::string(30, 'x'))}} + ) + }); + auto oversized = prepare_final_page(captured, 0, {}, {.frame_bytes = 256}); + EXPECT_EQ(oversized.status.code(), Code::kResourceExhausted); + EXPECT_EQ(oversized.encoder, nullptr); + EXPECT_TRUE(oversized.needs_split); + const auto work = prepare_final_page(captured, 0, {}, {.max_work_entries = 1}); + EXPECT_EQ(work.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(work.needs_split); + const auto metadata = prepare_final_page(captured, 0, {}, {.max_metadata_bytes = 1}); + EXPECT_EQ(metadata.status.code(), Code::kResourceExhausted); + EXPECT_FALSE(metadata.needs_split); + EXPECT_EQ(prepare_final_page(captured, 0, {.range = {.lower = "a"}}).status.code(), Code::kInvalidArgument); +} + +TEST(FinalPageEncoder, ColdLeafAndInnerReturnOwnedLoadRequestsWithoutEncoding) +{ + auto backing = std::make_shared<BackingRef>(); + backing->length = 512; + backing->owner = std::make_shared<int>(1); + auto logical = std::make_shared<PageState>(); + logical->bases.push_back({.identity = 1, .generation = 1, .backing = backing}); + auto leaf = std::make_shared<PublishedPage>(); + leaf->leaf = logical; + auto captured = encoding_capture({leaf}); + auto leaf_plan = prepare_final_page(captured, 0, {}); + ASSERT_TRUE(leaf_plan.status.ok()); + if (!leaf_plan.needs_base.has_value()) { + ADD_FAILURE() << "cold leaf did not return its load request"; + return; + } + EXPECT_EQ(leaf_plan.needs_base->page, logical); + EXPECT_EQ(leaf_plan.encoder, nullptr); + auto inner = std::make_shared<PublishedPage>(); + inner->level = 1; + inner->inner_backing = backing; + captured = encoding_capture({inner}); + auto inner_plan = prepare_final_page(captured, 0, {}); + ASSERT_TRUE(inner_plan.status.ok()); + if (!inner_plan.needs_inner.has_value()) { + ADD_FAILURE() << "cold inner did not return its load request"; + return; + } + EXPECT_EQ(inner_plan.needs_inner->backing, backing); + EXPECT_EQ(inner_plan.encoder, nullptr); +} + +TEST(FinalPageEncoder, InnerUsesOnlyCapturedRoutingAndExplicitDescendantFences) +{ + auto inner = std::make_shared<InnerPageState>(); + inner->children = {0, 1}; + inner->separators = {"m"}; + auto root = std::make_shared<PublishedPage>(); + root->level = 1; + root->inner = inner; + auto captured = encoding_capture( + { + encoding_leaf({{"a", encode_cell_owner(1, "v")}}, + {.upper = "m" } + ), + encoding_leaf({{"m", encode_cell_owner(1, "v")}}, + {.lower = "m" } + ), root + }, + 2); + EXPECT_EQ(prepare_final_page(captured, 2, {}).status.code(), Code::kInvalidArgument); + auto prepared = prepare_final_page(captured, 2, + { + .inner_fences = InnerEncodeFences{.lower_leaf = 0, .upper_leaf = 1} + }, + {.frame_bytes = 256}); + ASSERT_TRUE(prepared.status.ok()); + ASSERT_NE(prepared.encoder, nullptr); + captured.reset(); + std::array<uint8_t, 256> frame{}; + EXPECT_TRUE(prepared.encoder->next(frame).complete); + EXPECT_TRUE(frame_validate(frame.data(), frame.size())); + InnerFrameView encoded(frame.data(), frame.size()); + EXPECT_EQ(encoded.child_at(0), 0U); + EXPECT_EQ(encoded.child_at(1), 1U); + EXPECT_EQ(encoded.separator_at(0).to_string(), "m"); + EXPECT_FALSE(prepared.encoder->summary().live_counts_known); + EXPECT_EQ(frame_lower_fence_page_id(frame.data()), 0U); + EXPECT_EQ(frame_upper_fence_page_id(frame.data()), 1U); +} + +TEST(FinalPageEncoder, GcCanProduceAnEmptyValidLeafFrame) +{ + auto captured = encoding_capture( + { + encoding_leaf({{"a", encode_cell_owner(1, "", true)}} + ) + }, + 0, 10, 1); + auto prepared = prepare_final_page(captured, 0, {}, {.frame_bytes = 256}); + ASSERT_TRUE(prepared.status.ok()); + ASSERT_NE(prepared.encoder, nullptr); + std::array<uint8_t, 256> frame{}; + EXPECT_TRUE(prepared.encoder->next(frame).complete); + EXPECT_TRUE(frame_validate(frame.data(), frame.size())); + EXPECT_EQ(LeafFrameView(frame.data(), frame.size()).count(), 0U); + EXPECT_FALSE(frame_has_lower_fence(frame.data())); +} + +TEST(PageEncodeMetrics, CountsActualFrameWorkAndErrorsButNotCompletedPolls) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto captured = encoding_capture({ + encoding_leaf({{"k", encode_cell_owner(1, "value")}} + ) + }); + PageEncodeLimits limits; + limits.frame_bytes = 512; + limits.work_metrics = metrics; + auto prepared = prepare_final_page(captured, 0, {}, limits); + ASSERT_TRUE(prepared.status.ok()); + ASSERT_NE(prepared.encoder, nullptr); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)].attempts, 0U); + std::array<uint8_t, 512> frame{}; + EXPECT_EQ(prepared.encoder->next(std::span<uint8_t>(frame.data(), 511)).status.code(), Code::kInvalidArgument); + ASSERT_TRUE(prepared.encoder->next(frame).status.ok()); + EXPECT_FALSE(prepared.encoder->next(frame).frame.has_value()); + const auto result = metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)]; + EXPECT_EQ(result.attempts, 2U); + EXPECT_EQ(result.count, 2U); + EXPECT_EQ(result.inflight, 0U); + EXPECT_EQ(result.outcomes[static_cast<size_t>(TreeOutcome::Error)], 1U); + EXPECT_EQ(result.outcomes[static_cast<size_t>(TreeOutcome::Success)], 1U); +} + +TEST(PageEncodeMetrics, OverflowAndPrimaryEachRecordTheirOwnEncodingWork) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto captured = encoding_capture({ + encoding_leaf({{"k", encode_cell_owner(1, std::string(1024, 'v'))}} + ) + }); + PageEncodeLimits limits; + limits.frame_bytes = 256; + limits.inline_value_bytes = 16; + limits.work_metrics = metrics; + auto prepared = prepare_final_page(captured, 0, {}, limits); + ASSERT_TRUE(prepared.status.ok()); + ASSERT_NE(prepared.encoder, nullptr); + const auto frames = prepared.encoder->summary().new_overflow_frames; + ASSERT_GT(frames, 1U); + ASSERT_TRUE(prepared.encoder->bind_overflow({.first = 100, .count = frames, .owner = captured}).ok()); + std::array<uint8_t, 256> output{}; + for (uint64_t i = 0; i <= frames; ++i) { + auto step = prepared.encoder->next(output); + ASSERT_TRUE(step.status.ok()); + ASSERT_TRUE(step.frame.has_value()); + EXPECT_EQ(step.complete, i == frames); + } + const auto result = metrics->totals()[static_cast<size_t>(TreeStage::PageEncode)]; + EXPECT_EQ(result.attempts, frames + 1); + EXPECT_EQ(result.count, frames + 1); + EXPECT_EQ(result.outcomes[static_cast<size_t>(TreeOutcome::Success)], frames + 1); + EXPECT_EQ(result.inflight, 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_load_delta_test.cpp b/lib/crowdb-tree/tests/unit/page_load_delta_test.cpp new file mode 100644 index 000000000..6f5d96e6a --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_load_delta_test.cpp @@ -0,0 +1,174 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/btree/leaf_cursor.h" +#include "load_budget_test_support.h" +#include "page_load_delta_test_support.h" + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(PageLoadDelta, NormalizesUsingFoldWinnersAndKeepsAllTombstones) +{ + TestDeltaLeaf fixture(true); + auto loader = fixture.loader(); + std::shared_ptr<const LeafBase> result; + loader->load_leaf(fixture.reference, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()) << status.to_string(); + result = std::move(leaf); + }); + fixture.runtime->drain(); + ASSERT_NE(result, nullptr); + EXPECT_EQ(result->view().delta_count(), 0U); + EXPECT_EQ(result->view().self_page_id(), 17U); + EXPECT_EQ(result->right_sibling(), 23U); + EXPECT_LE(result->page_bytes(), fixture.frame.size()); + EXPECT_TRUE(frame_validate(result->frame(), result->page_bytes())); + const auto raw = std::unique_ptr<LeafBase>(LeafBase::from_frame_copy(fixture.frame.data(), fixture.frame.size())); + LeafChainCursor expected(raw.get(), 0); + size_t index = 0; + while (expected.valid()) { + ASSERT_LT(index, result->count()); + EXPECT_EQ(result->key(index), expected.key()); + EXPECT_EQ(result->cell(index), expected.cell()); + ++index; + expected.next(); + } + ASSERT_EQ(result->count(), index + 1); + EXPECT_EQ(result->key(index), Slice("zero")); + EXPECT_TRUE(CellView(result->cell(index)).is_tombstone()); + EXPECT_EQ(CellView(result->cell(index)).slot(), 0U); + EXPECT_EQ(CellView(result->cell(result->find("a"))).value(), Slice("delta")); + EXPECT_EQ(CellView(result->cell(result->find("b"))).value(), Slice("base-newer")); + EXPECT_EQ(CellView(result->cell(result->find("new"))).value(), Slice("first")); + EXPECT_EQ(CellView(result->cell(result->find("tie"))).value(), Slice("base-tie")); +} + +TEST(PageLoadDelta, SlotBoundValidatesMainAndDeltaInsteadOfFilteringHistory) +{ + for (const auto bound : {0U, 5U, 6U}) { + TestDeltaLeaf fixture; + fixture.reference.max_slot = bound; + auto loader = fixture.loader(); + size_t calls = 0; + loader->load_leaf(fixture.reference, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_FALSE(leaf); + ++calls; + }); + fixture.runtime->drain(); + EXPECT_EQ(calls, 1U); + } + TestDeltaLeaf fixture; + auto *raw = const_cast<uint8_t *>(LeafFrameView(fixture.frame.data(), fixture.frame.size()).delta_cell(1).bytes()); + frame_put_u64(raw, 0, 8); + frame_restamp_crc(fixture.frame.data(), fixture.frame.size()); + fixture.encode(); + auto loader = fixture.loader(); + loader->load_leaf(fixture.reference, [](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_FALSE(leaf); + }); + fixture.runtime->drain(); +} + +TEST(PageLoadDelta, RejectsMalformedCellsEvenWhenTheyLoseTheFold) +{ + for (bool delta : {false, true}) { + TestDeltaLeaf fixture; + const LeafFrameView view(fixture.frame.data(), fixture.frame.size()); + const auto cell = delta ? view.delta_cell(4) : view.cell(0); + fixture.frame[static_cast<size_t>(cell.bytes() - fixture.frame.data()) + 8] = 0x80; + frame_restamp_crc(fixture.frame.data(), fixture.frame.size()); + fixture.encode(); + auto loader = fixture.loader(); + size_t calls = 0; + loader->load_leaf(fixture.reference, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_FALSE(leaf); + ++calls; + }); + fixture.runtime->drain(); + EXPECT_EQ(calls, 1U); + } +} + +TEST(PageLoadDelta, EntryAndKeyWindowsCountTheEntireEncodedFrame) +{ + for (bool entries : {false, true}) { + TestDeltaLeaf fixture; + PageLoadLimits limits; + if (entries) { + limits.max_leaf_entries = 11; + } + else { + limits.max_leaf_key_bytes = 16; + } + auto loader = fixture.loader(limits); + size_t calls = 0; + loader->load_leaf(fixture.reference, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(leaf); + ++calls; + }); + fixture.runtime->drain(); + EXPECT_EQ(calls, 1U); + } +} + +TEST(PageLoadDelta, LeafRetainsOnlyItsBackingProtectionAndActualFrameCharge) +{ + TestDeltaLeaf fixture(true); + LoadCredits credits; + auto loader = fixture.loader({}, credits.credits); + std::weak_ptr<const void> protection = fixture.reference.backing->owner; + std::weak_ptr<TestDeltaStore> store = fixture.store; + std::shared_ptr<const LeafBase> result; + loader->load_leaf(fixture.reference, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()) << status.to_string(); + result = std::move(leaf); + }); + fixture.runtime->drain(); + ASSERT_NE(result, nullptr); + loader.reset(); + fixture.reference = {}; + fixture.store.reset(); + credits.seal(); + EXPECT_TRUE(store.expired()); + EXPECT_FALSE(protection.expired()); + EXPECT_EQ(credits.budget.snapshot().reserved[LoadCredits::kBytes], 0U); + EXPECT_GE(credits.budget.snapshot().allocated[LoadCredits::kBytes], result->page_bytes() + sizeof(LeafBase)); + const CellView overflow(result->cell(result->find("overflow"))); + EXPECT_EQ(overflow.overflow_head(), 41U); + EXPECT_EQ(overflow.overflow_len(), 16384U); + result.reset(); + EXPECT_TRUE(protection.expired()); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageLoadDelta, IndexAllocationExhaustionReleasesDecodeAndIoCredit) +{ + TestDeltaLeaf fixture; + fixture.store->deferred = true; + LoadCredits credits; + auto loader = fixture.loader({}, credits.credits); + size_t calls = 0; + loader->load_leaf(fixture.reference, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(leaf); + ++calls; + }); + const auto delta_bytes = LeafFrameView(fixture.frame.data(), fixture.frame.size()).delta_count() * sizeof(uint32_t); + const auto held = credits.credits.bytes->available() - fixture.frame.size() - delta_bytes + 1; + ASSERT_TRUE(credits.credits.bytes->claim(held)); + fixture.store->pending.complete(Status::Ok()); + fixture.store->pending = {}; + fixture.runtime->drain(); + EXPECT_EQ(calls, 1U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + loader.reset(); + credits.credits.bytes->release(held); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_load_inner_proof_test.cpp b/lib/crowdb-tree/tests/unit/page_load_inner_proof_test.cpp new file mode 100644 index 000000000..b886bbc96 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_load_inner_proof_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "recovered_state_test_support.h" + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(InnerLoadProof, OwnsByteFencesAndDropsProofWhenRoutingIsCopied) +{ + TestRecoveredGraph test; + std::vector<uint8_t> raw(4096); + ASSERT_TRUE(inner_frame_build(raw.data(), raw.size(), 0, {1, 2}, {"m"})); + const Slice lower("a"); + const Slice upper("z"); + ASSERT_TRUE(frame_set_fences(raw.data(), raw.size(), &lower, &upper)); + test.add(0, raw); + std::shared_ptr<const InnerPageState> result; + test.loader->load_inner(test.pages->frames.at(0).backing, [&](Status status, auto loaded) { + EXPECT_TRUE(status.ok()) << status.to_string(); + result = std::move(loaded); + }); + test.runtime->drain(); + ASSERT_NE(result, nullptr); + const auto &stored = result->frame_proof; + if (!stored.has_value()) { + FAIL() << "decoded index has no frame proof"; + return; + } + const auto &proof = *stored; + EXPECT_TRUE(proof.has_fences); + EXPECT_FALSE(proof.page_id_fences); + EXPECT_EQ(proof.self_page_id, 0U); + EXPECT_EQ(proof.frame_bytes, 4096U); + EXPECT_EQ(proof.lower, "a"); + EXPECT_EQ(proof.upper, "z"); + auto copied = + detail::reserved_shared<InnerPageState>(test.credits.credits.bytes, *result, test.credits.credits.bytes); + EXPECT_FALSE(copied->frame_proof); + copied.reset(); + result.reset(); + test.loader.reset(); + test.pages.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(InnerLoadProof, FenceKeysConsumeTheSameBoundAsSeparators) +{ + TestRecoveredGraph test; + std::vector<uint8_t> raw(4096); + ASSERT_TRUE(inner_frame_build(raw.data(), raw.size(), 0, {1, 2}, {"m"})); + const Slice lower("a"); + const Slice upper("z"); + ASSERT_TRUE(frame_set_fences(raw.data(), raw.size(), &lower, &upper)); + test.add(0, raw); + PageLoadLimits limits; + limits.max_inner_key_bytes = 2; + auto loader = std::make_shared<AsyncStorePageLoader>(test.store, test.runtime, test.lane, limits, + OverflowBackingResolver{}, test.credits.credits); + size_t calls = 0; + loader->load_inner(test.pages->frames.at(0).backing, [&](Status status, auto loaded) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(loaded); + ++calls; + }); + test.runtime->drain(); + EXPECT_EQ(calls, 1U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_load_inner_test.cpp b/lib/crowdb-tree/tests/unit/page_load_inner_test.cpp new file mode 100644 index 000000000..ee5c7ed3a --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_load_inner_test.cpp @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "inner_load_test_support.h" +#include "load_budget_test_support.h" + +#include <thread> + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(InnerLoad, SharedSeparatorKeepsOnlyItsChargedCapacityAfterIndexRelease) +{ + LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestInnerStore>(); + const std::string key(4096, 'm'); + auto backing = inner_backing(store, 7, *inner({key, std::string(4096, 't')}, {1, 2, 3})); + auto loader = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, + OverflowBackingResolver{}, credits.credits); + std::shared_ptr<const InnerPageState> result; + loader->load_inner(backing, [&](Status status, auto index) { + EXPECT_TRUE(status.ok()) << status.to_string(); + result = std::move(index); + }); + runtime->drain(); + ASSERT_NE(result, nullptr); + ASSERT_EQ(result->separators.size(), 2U); + OwnedKey retained = result->separators.front(); + loader.reset(); + credits.seal(); + const auto all = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + EXPECT_GT(all, 8192U); + result.reset(); + EXPECT_LT(credits.budget.snapshot().allocated[LoadCredits::kBytes], all); + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], key.size()); + EXPECT_EQ(Slice(retained).to_string(), key); + retained = {}; + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} + +TEST(InnerLoad, InlineIoDecodesOnCpuAndReturnsIndependentImmutableRouting) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + auto store = std::make_shared<TestInnerStore>(); + auto backing = inner_backing(store, 0, *inner({"m", "t"}, {1, 2, 3})); + AsyncStorePageLoader loader(store, runtime, lane); + std::shared_ptr<const InnerPageState> result; + std::atomic<size_t> calls{0}; + const auto caller = std::this_thread::get_id(); + loader.load_inner(backing, [&](Status status, std::shared_ptr<const InnerPageState> loaded) { + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_NE(std::this_thread::get_id(), caller); + result = std::move(loaded); + ++calls; + }); + runtime->drain(); + store->blobs.clear(); + backing.reset(); + ASSERT_NE(result, nullptr); + EXPECT_EQ(result->separators, (InnerPageState::Keys{"m", "t"})); + EXPECT_EQ(result->children, (InnerPageState::Children{1, 2, 3})); + EXPECT_EQ(calls.load(), 1U); +} + +TEST(InnerLoad, ValidatesChecksumChildIdsAndMetadataBounds) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + for (size_t scenario = 0; scenario < 4; ++scenario) { + auto store = std::make_shared<TestInnerStore>(); + auto state = inner({"m"}, {1, scenario == 1 ? kInvalidPageId : 2}); + auto backing = inner_backing(store, 0, *state); + if (scenario == 0) { + store->blobs[0].back() ^= 1; + } + PageLoadLimits limits; + limits.max_inner_children = scenario == 2 ? 1 : 2; + limits.max_inner_key_bytes = scenario == 3 ? 0 : 1; + AsyncStorePageLoader loader(store, runtime, lane, limits); + std::atomic<size_t> calls{0}; + loader.load_inner(backing, [&](Status status, std::shared_ptr<const InnerPageState> loaded) { + EXPECT_EQ(status.code(), scenario < 2 ? Code::kCorruption : Code::kResourceExhausted); + EXPECT_EQ(loaded, nullptr); + ++calls; + }); + runtime->drain(); + EXPECT_EQ(calls.load(), 1U); + } +} + +TEST(InnerLoad, SlowIoDoesNotHoldCpuWorkerAndLateErrorReturnsPermit) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + auto store = std::make_shared<TestInnerStore>(); + store->deferred = true; + auto backing = inner_backing(store, 0, *inner({"m"}, {1, 2})); + AsyncStorePageLoader loader(store, runtime, lane, {.max_inflight = 1}); + std::vector<Code> outcomes; + auto complete = [&](Status status, std::shared_ptr<const InnerPageState>) { outcomes.push_back(status.code()); }; + loader.load_inner(backing, complete); + std::atomic<bool> progressed{false}; + ASSERT_TRUE(runtime + ->try_submit( + lane, [](void *context) noexcept { static_cast<std::atomic<bool> *>(context)->store(true); }, + &progressed) + .ok()); + runtime->drain(); + EXPECT_TRUE(progressed.load()); + EXPECT_TRUE(outcomes.empty()); + loader.load_inner(backing, complete); + store->finish(Status::io_error("first read failed")); + loader.load_inner(backing, complete); + store->finish(Status::io_error("second read failed")); + EXPECT_EQ(outcomes, (std::vector<Code>{Code::kResourceExhausted, Code::kIoError, Code::kIoError})); + EXPECT_EQ(store->reads.load(), 2U); +} + +TEST(InnerLoad, RejectsWrongFrameTypeAndCpuQueueShutdown) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + auto store = std::make_shared<TestInnerStore>(); + auto backing = std::make_shared<BackingRef>(); + const auto leaf = base({ + {.key = "a", .value = "leaf"} + }); + encode_durable_page(leaf->frame(), leaf->page_bytes(), compress_algo::kNone, &store->blobs[0]); + backing->length = store->blobs[0].size(); + backing->owner = std::make_shared<int>(1); + AsyncStorePageLoader loader(store, runtime, lane); + std::atomic<size_t> calls{0}; + loader.load_inner(backing, [&](Status status, std::shared_ptr<const InnerPageState> loaded) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_EQ(loaded, nullptr); + ++calls; + }); + runtime->drain(); + runtime->close(); + loader.load_inner(backing, [&](Status status, std::shared_ptr<const InnerPageState> loaded) { + EXPECT_EQ(status.code(), Code::kUnavailable); + EXPECT_EQ(loaded, nullptr); + ++calls; + }); + EXPECT_EQ(calls.load(), 2U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_load_lifetime_test.cpp b/lib/crowdb-tree/tests/unit/page_load_lifetime_test.cpp new file mode 100644 index 000000000..47aca267f --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_load_lifetime_test.cpp @@ -0,0 +1,389 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/apply_dispatcher.h" +#include "btree/tree_metrics.h" +#include "crowdb-tree/maptable/compressor.h" +#include "load_budget_test_support.h" +#include "published_read_test_support.h" +#include "runtime/tree_operation_lifetime.h" + +#include <cstring> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; +enum class Kind : uint8_t { kLeaf, kInner, kOverflow }; + +class UncertainReadStore final : public AsyncPageStore +{ + public: + struct Pending + { + uint64_t address; + void *buffer; + size_t bytes; + AsyncCompletion complete; + }; + + bool throw_after_accept = true; + bool inline_completion = false; + bool allocation_failure = false; + std::map<uint64_t, std::vector<uint8_t>> blobs; + std::deque<Pending> pending; + size_t reads = 0; + + uint64_t submit_read(PageAddr addr, void *buffer, size_t bytes, AsyncCompletion complete) override + { + ++reads; + EXPECT_TRUE(complete.owner); + pending.push_back({.address = addr, .buffer = buffer, .bytes = bytes, .complete = std::move(complete)}); + if (inline_completion) { + finish(); + } + if (throw_after_accept) { + if (allocation_failure) { + throw std::bad_alloc(); + } + throw std::runtime_error("read accepted before failure"); + } + return reads; + } + + uint64_t submit_write(PageAddr /*addr*/, const void * /*buffer*/, size_t /*bytes*/, + AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return 0; + } + + Status submit_fsync(AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return Status::not_supported(); + } + + void cancel(uint64_t /*id*/) override + { + } + + void finish() + { + ASSERT_FALSE(pending.empty()); + auto read = std::move(pending.front()); + pending.pop_front(); + const auto &blob = blobs.at(read.address); + ASSERT_EQ(read.bytes, blob.size()); + std::memcpy(read.buffer, blob.data(), blob.size()); + read.complete.complete(Status::Ok()); + read.complete.complete(Status::io_error()); // a duplicate must not report another terminal + } +}; + +struct Reads +{ + LoadCredits credits; + std::shared_ptr<TreeWorkMetrics> metrics = std::make_shared<TreeWorkMetrics>(); + std::shared_ptr<detail::FlushRuntime> runtime = std::make_shared<detail::FlushRuntime>(1, 8); + std::shared_ptr<detail::FlushRuntime::Lane> lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<UncertainReadStore> store = std::make_shared<UncertainReadStore>(); + std::map<uint64_t, std::shared_ptr<const BackingRef>> backings; + std::shared_ptr<AsyncStorePageLoader> loader; + size_t calls = 0; + Code last = Code::kOk; + + explicit Reads(Kind kind, bool chain = false, bool prepaid = false) + { + credits.credits.work_metrics = metrics; + if (prepaid) { + EXPECT_TRUE( + ApplyDispatcher::create(runtime, lane, 1, credits.credits.bytes, &credits.credits.cpu_progress).ok()); + } + if (kind == Kind::kLeaf) { + auto leaf = base({ + {.key = "key", .value = "data"} + }); + add(11, leaf->frame(), leaf->page_bytes()); + } + else if (kind == Kind::kInner) { + const std::unique_ptr<InnerBase> inner(InnerBase::build({"m"}, {1, 2})); + add(11, inner->frame(), inner->page_bytes()); + } + else { + const std::unique_ptr<OverflowBase> first( + OverflowBase::build(chain ? 12 : kInvalidPageId, Slice("a").bytes(), 1)); + add(11, first->frame(), first->page_bytes()); + if (chain) { + const std::unique_ptr<OverflowBase> second(OverflowBase::build(kInvalidPageId, Slice("b").bytes(), 1)); + add(12, second->frame(), second->page_bytes()); + } + } + loader = std::make_shared<AsyncStorePageLoader>( + store, runtime, lane, PageLoadLimits{.max_inflight = 1}, + [captured = backings](uint64_t id, std::shared_ptr<const BackingRef> *out) { + *out = captured.at(id); + return Status::Ok(); + }, + credits.credits); + } + + void add(uint64_t id, const uint8_t *frame, uint32_t bytes) + { + std::vector<uint8_t> raw(frame, frame + bytes); + frame_put_u64(raw.data(), fh::kSelfpage_id, id); + frame_restamp_crc(raw.data(), raw.size()); + auto &blob = store->blobs[id]; + encode_durable_page(raw.data(), raw.size(), compress_algo::kNone, &blob); + auto backing = std::make_shared<BackingRef>(); + backing->address = id; + backing->length = blob.size(); + backing->owner = std::make_shared<int>(1); + backings[id] = std::move(backing); + } + + void start(Kind kind, uint64_t length = 1) + { + auto complete = [this](Status status, auto result) { + ++calls; + last = status.code(); + EXPECT_EQ(status.ok(), static_cast<bool>(result)); + }; + if (kind == Kind::kLeaf) { + loader->load_leaf({.identity = 11, .backing = backings.at(11)}, complete); + } + else if (kind == Kind::kInner) { + loader->load_inner(backings.at(11), complete); + } + else { + loader->load_overflow(11, length, complete); + } + runtime->drain(); + } +}; + +TEST(PageLoadLifetime, PrepaidDecodeContinuesAcrossColdReadsWithAllOrdinaryCpuSlotsOccupied) +{ + for (auto kind : {Kind::kLeaf, Kind::kInner, Kind::kOverflow}) { + Reads test(kind, true, true); + test.store->throw_after_accept = false; + test.store->inline_completion = true; + auto lane = std::make_shared<detail::FlushRuntime::Lane>(16); + std::vector<std::shared_ptr<detail::FlushRuntime::Progress>> occupied; + for (;;) { + std::shared_ptr<detail::FlushRuntime::Progress> slot; + const auto status = test.runtime->reserve_progress(lane, [](void *) noexcept {}, nullptr, {}, &slot); + if (!status.ok()) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + break; + } + occupied.push_back(std::move(slot)); + } + EXPECT_FALSE(occupied.empty()); + test.start(kind, kind == Kind::kOverflow ? 2 : 1); + EXPECT_EQ(test.calls, 1U); + EXPECT_EQ(test.last, Code::kOk); + EXPECT_EQ(test.store->reads, kind == Kind::kOverflow ? 2U : 1U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + test.credits.credits.cpu_progress->close_and_drain(); + } +} + +TEST(PageLoadLifetime, UnknownAcceptancePinsWindowAndProgressUntilLateCallback) +{ + for (auto kind : {Kind::kLeaf, Kind::kInner, Kind::kOverflow}) { + Reads test(kind); + test.start(kind); + ASSERT_EQ(test.calls, 1U); + EXPECT_EQ(test.last, Code::kIoError); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + test.start(kind); + EXPECT_EQ(test.calls, 2U); + EXPECT_EQ(test.last, Code::kResourceExhausted); + EXPECT_EQ(test.store->reads, 1U); + test.loader.reset(); + test.credits.seal(); + EXPECT_GT(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(test.credits.resources->snapshot().read_inflight_bytes, + test.credits.account->snapshot().physical[static_cast<size_t>(AllocationKind::kIo)]); + EXPECT_GT(test.credits.resources->snapshot().read_inflight_bytes, 0U); + test.store->finish(); + EXPECT_EQ(test.credits.resources->snapshot().read_inflight_bytes, 0U); + test.runtime->drain(); + EXPECT_EQ(test.calls, 2U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + } +} + +TEST(PageLoadLifetime, TreeShutdownWaitsForUnknownTransportOwnershipAfterLogicalTerminalAndCpuDrain) +{ + for (const bool callback : {true, false}) { + Reads test(Kind::kLeaf); + TreeOperationLifetime lifetime(1, test.credits.credits.bytes); + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + ASSERT_TRUE(lifetime.acquire(test.credits.credits.bytes, &ticket).ok()); + test.loader = std::make_shared<AsyncStorePageLoader>(ticket->backend(test.store.get()), test.runtime, test.lane, + PageLoadLimits{.max_inflight = 1}, + OverflowBackingResolver{}, test.credits.credits); + test.start(Kind::kLeaf); + ASSERT_EQ(test.calls, 1U); + EXPECT_EQ(test.last, Code::kIoError); + ticket.reset(); + test.loader.reset(); + lifetime.close(); + test.runtime->drain(); + EXPECT_EQ(lifetime.outstanding(), 1U); + if (callback) { + test.store->finish(); + } + else { + test.store->pending.clear(); + } + EXPECT_EQ(lifetime.outstanding(), 0U); + lifetime.close_and_drain(); + EXPECT_EQ(test.calls, 1U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + } +} + +TEST(PageLoadLifetime, InlineSuccessThenThrowDoesNotDecodeOrPublishSuccess) +{ + for (auto kind : {Kind::kLeaf, Kind::kInner, Kind::kOverflow}) { + Reads test(kind); + test.store->inline_completion = true; + test.start(kind); + EXPECT_EQ(test.calls, 1U); + EXPECT_EQ(test.last, Code::kIoError); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + test.loader.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + } +} + +TEST(PageLoadLifetime, AcceptedReadAllocationFailureAlsoRetainsItsProgressCredit) +{ + for (auto kind : {Kind::kLeaf, Kind::kInner, Kind::kOverflow}) { + Reads test(kind); + test.store->allocation_failure = true; + test.start(kind); + EXPECT_EQ(test.calls, 1U); + EXPECT_EQ(test.last, Code::kResourceExhausted); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + test.loader.reset(); + test.credits.seal(); + test.store->finish(); + test.runtime->drain(); + EXPECT_EQ(test.calls, 1U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + } +} + +TEST(PageLoadLifetime, TransportDiscardOfUnknownCallbackReleasesAdmissionForNextRead) +{ + for (auto kind : {Kind::kLeaf, Kind::kInner, Kind::kOverflow}) { + Reads test(kind); + test.start(kind); + test.store->pending.clear(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + test.store->throw_after_accept = false; + test.store->inline_completion = true; + test.start(kind); + EXPECT_EQ(test.calls, 2U); + EXPECT_EQ(test.last, Code::kOk); + EXPECT_EQ(test.store->reads, 2U); + test.loader.reset(); + test.credits.seal(); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + } +} + +TEST(PageLoadLifetime, PriorOverflowCallbackCannotAffectNextInflightFrame) +{ + Reads test(Kind::kOverflow, true); + test.store->throw_after_accept = false; + test.start(Kind::kOverflow, 2); + ASSERT_EQ(test.store->pending.size(), 1U); + auto duplicate = test.store->pending.front().complete; + test.store->finish(); + test.runtime->drain(); + ASSERT_EQ(test.store->pending.size(), 1U); + EXPECT_EQ(test.calls, 0U); + duplicate.complete(Status::io_error()); + test.runtime->drain(); + EXPECT_EQ(test.calls, 0U); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + test.store->finish(); + test.runtime->drain(); + EXPECT_EQ(test.calls, 1U); + EXPECT_EQ(test.last, Code::kOk); + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + test.loader.reset(); + test.credits.seal(); + duplicate = {}; + EXPECT_EQ(test.credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageLoadMetrics, ErrorAndBudgetRejectionHaveOneTerminalBeforeLateDuplicateCallback) +{ + for (auto kind : {Kind::kLeaf, Kind::kInner, Kind::kOverflow}) { + Reads test(kind); + test.start(kind); + test.start(kind); + const auto terminal = test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)]; + EXPECT_EQ(terminal.attempts, 2U); + EXPECT_EQ(terminal.count, 2U); + EXPECT_EQ(terminal.inflight, 0U); + EXPECT_EQ(terminal.outcomes[static_cast<size_t>(TreeOutcome::Error)], 2U); + test.loader.reset(); + test.store->finish(); + test.runtime->drain(); + const auto late = test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)]; + EXPECT_EQ(late.count, terminal.count); + EXPECT_EQ(late.sum_ns, terminal.sum_ns); + EXPECT_EQ(test.calls, 2U); + } +} + +TEST(PageLoadMetrics, DemandRemainsInflightUntilDecodedValueIsReady) +{ + for (auto kind : {Kind::kLeaf, Kind::kInner, Kind::kOverflow}) { + Reads test(kind); + test.store->throw_after_accept = false; + test.start(kind); + const auto accepted = test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)]; + EXPECT_EQ(accepted.attempts, 1U); + EXPECT_EQ(accepted.inflight, 1U); + EXPECT_EQ(accepted.count, 0U); + test.store->finish(); + test.runtime->drain(); + const auto ready = test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)]; + EXPECT_EQ(ready.attempts, 1U); + EXPECT_EQ(ready.inflight, 0U); + EXPECT_EQ(ready.count, 1U); + EXPECT_EQ(ready.outcomes[static_cast<size_t>(TreeOutcome::Success)], 1U); + EXPECT_EQ(test.calls, 1U); + } +} + +TEST(PageLoadMetrics, DiscardingAcceptedReadOwnerCountsCancellationOnce) +{ + Reads test(Kind::kLeaf); + test.store->throw_after_accept = false; + test.start(Kind::kLeaf); + test.loader.reset(); + test.store->pending.clear(); + const auto cancelled = test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)]; + EXPECT_EQ(cancelled.attempts, 1U); + EXPECT_EQ(cancelled.inflight, 0U); + EXPECT_EQ(cancelled.count, 1U); + EXPECT_EQ(cancelled.outcomes[static_cast<size_t>(TreeOutcome::Cancelled)], 1U); + EXPECT_EQ(test.calls, 0U); + test.store.reset(); + EXPECT_EQ(test.metrics->totals()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count, 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_load_overflow_test.cpp b/lib/crowdb-tree/tests/unit/page_load_overflow_test.cpp new file mode 100644 index 000000000..926a3dea2 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_load_overflow_test.cpp @@ -0,0 +1,255 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/overflow_backing.h" +#include "load_budget_test_support.h" +#include "overflow_load_test_support.h" + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(OverflowLoad, PrimaryDependenciesRetainOnlyItsChainWithoutGlobalResolver) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestOverflowStore>(); + auto captured = std::make_shared<TestBackingMap>(); + add_overflow(store, captured.get(), 1, 2, "hello"); + add_overflow(store, captured.get(), 2, kInvalidPageId, "world"); + add_overflow(store, captured.get(), 3, kInvalidPageId, "unrelated"); + std::weak_ptr<const BackingRef> unrelated = captured->at(3); + auto dependencies = std::make_shared<OverflowBackingSet>(nullptr); + dependencies->pages.emplace(1, OverflowBackingPage{.backing = captured->at(1)}); + dependencies->pages.emplace(2, OverflowBackingPage{.backing = captured->at(2)}); + auto primary = std::make_shared<BackingRef>(); + primary->overflow = dependencies; + captured.reset(); + EXPECT_TRUE(unrelated.expired()); + AsyncStorePageLoader loader(store, runtime, lane); + size_t calls = 0; + loader.load_overflow(primary, 1, 10, [&](Status status, std::shared_ptr<const buffer> value) { + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(value); + EXPECT_EQ(value->slice().to_string(), "helloworld"); + ++calls; + }); + runtime->drain(); + EXPECT_EQ(calls, 1U); +} + +TEST(OverflowLoad, PrimaryCannotResolveForeignPidThroughFallback) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestOverflowStore>(); + auto captured = std::make_shared<TestBackingMap>(); + add_overflow(store, captured.get(), 1, kInvalidPageId, "hello"); + auto primary = std::make_shared<BackingRef>(); + primary->overflow = std::make_shared<OverflowBackingSet>(nullptr); + AsyncStorePageLoader loader(store, runtime, lane, {}, resolver(captured)); + size_t calls = 0; + loader.load_overflow(primary, 1, 5, [&](Status status, std::shared_ptr<const buffer> value) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_FALSE(value); + ++calls; + }); + runtime->drain(); + EXPECT_EQ(calls, 1U); + EXPECT_EQ(store->reads.load(), 0U); +} + +TEST(OverflowLoad, ValueCapacityRemainsChargedAfterDecodeScratchAndLoaderAreGone) +{ + LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestOverflowStore>(); + auto captured = std::make_shared<TestBackingMap>(); + const std::string first(4096, 'a'); + const std::string second(4096, 'b'); + add_overflow(store, captured.get(), 1, 2, first); + add_overflow(store, captured.get(), 2, kInvalidPageId, second); + auto loader = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, resolver(captured), + credits.credits); + std::shared_ptr<const buffer> result; + loader->load_overflow(1, 8192, [&](Status status, auto value) { + EXPECT_TRUE(status.ok()) << status.to_string(); + result = std::move(value); + }); + runtime->drain(); + ASSERT_NE(result, nullptr); + loader.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], result->capacity()); + EXPECT_LT(credits.budget.snapshot().allocated[LoadCredits::kBytes], result->capacity() + 512); + EXPECT_EQ(result->slice().to_string(), first + second); + result.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(OverflowLoad, CapturedResolverAndBackingSurviveDeferredTransport) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestOverflowStore>(); + store->deferred = true; + auto captured = std::make_shared<TestBackingMap>(); + add_overflow(store, captured.get(), 1, 2, "hello"); + add_overflow(store, captured.get(), 2, kInvalidPageId, "world"); + std::weak_ptr<const TestBackingMap> retained = captured; + auto loader = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, resolver(captured)); + std::shared_ptr<const buffer> result; + std::atomic<size_t> calls{0}; + loader->load_overflow(1, 10, [&](Status status, std::shared_ptr<const buffer> value) { + EXPECT_TRUE(status.ok()) << status.to_string(); + result = std::move(value); + ++calls; + }); + runtime->drain(); + captured.reset(); + loader.reset(); + EXPECT_FALSE(retained.expired()); + store->finish(); + runtime->drain(); + EXPECT_EQ(calls.load(), 0U); + store->finish(); + runtime->drain(); + EXPECT_EQ(calls.load(), 1U); + ASSERT_NE(result, nullptr); + EXPECT_EQ(result->slice().to_string(), "helloworld"); + EXPECT_TRUE(retained.expired()); +} + +TEST(OverflowLoad, RejectsShortLongAndCyclicChains) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + for (uint64_t length : {9, 11, 12}) { + auto store = std::make_shared<TestOverflowStore>(); + auto captured = std::make_shared<TestBackingMap>(); + add_overflow(store, captured.get(), 1, 2, "hello"); + add_overflow(store, captured.get(), 2, length == 12 ? 1 : kInvalidPageId, "world"); + AsyncStorePageLoader loader(store, runtime, lane, {}, resolver(captured)); + std::atomic<size_t> errors{0}; + loader.load_overflow(1, length, [&](Status status, std::shared_ptr<const buffer> value) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_EQ(value, nullptr); + ++errors; + }); + runtime->drain(); + EXPECT_EQ(errors.load(), 1U); + EXPECT_LE(store->reads.load(), 2U); + } +} + +TEST(OverflowLoad, BoundsValueBytesAndPagesAndValidatesChecksum) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + for (size_t scenario = 0; scenario < 3; ++scenario) { + auto store = std::make_shared<TestOverflowStore>(); + auto captured = std::make_shared<TestBackingMap>(); + add_overflow(store, captured.get(), 1, 2, "hello"); + add_overflow(store, captured.get(), 2, kInvalidPageId, "world"); + PageLoadLimits limits; + limits.max_value_bytes = scenario == 0 ? 9 : 10; + limits.max_overflow_pages = scenario == 1 ? 1 : 2; + if (scenario == 2) { + store->blobs[2].back() ^= 1; + } + AsyncStorePageLoader loader(store, runtime, lane, limits, resolver(captured)); + std::atomic<size_t> errors{0}; + loader.load_overflow(1, 10, [&](Status status, std::shared_ptr<const buffer> value) { + EXPECT_EQ(status.code(), scenario == 2 ? Code::kCorruption : Code::kResourceExhausted); + EXPECT_EQ(value, nullptr); + ++errors; + }); + runtime->drain(); + EXPECT_EQ(errors.load(), 1U); + EXPECT_EQ(store->reads.load(), scenario); + } +} + +TEST(OverflowLoad, GetAndScanMaterializeTheSameCapturedChain) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestOverflowStore>(); + auto captured = std::make_shared<TestBackingMap>(); + add_overflow(store, captured.get(), 1, 2, "hello"); + add_overflow(store, captured.get(), 2, kInvalidPageId, "world"); + auto loader = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, resolver(captured)); + auto snapshot = view(overflow_leaf()); + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(snapshot, loader, "key", {}, &get).ok()); + Completion got; + auto point = stdexec::connect(get->sender(), Receiver{.done = &got}); + stdexec::start(point); + runtime->drain(); + EXPECT_EQ(got.values.load(), 1U); + EXPECT_EQ(got.errors.load(), 0U); + EXPECT_TRUE(get->result().found); + EXPECT_FALSE(get->result().cell.overflow()); + EXPECT_EQ(get->result().cell.value.to_string(), "helloworld"); + for (bool reverse : {false, true}) { + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(snapshot, loader, {}, reverse, {.max_entries = 1}, &scan).ok()); + Completion scanned; + auto range = stdexec::connect(scan->next_sender(), Receiver{.done = &scanned}); + stdexec::start(range); + runtime->drain(); + EXPECT_EQ(scanned.values.load(), 1U); + ASSERT_EQ(scan->result().entries.size(), 1U); + EXPECT_EQ(scan->result().entries[0].cell.value.to_string(), "helloworld"); + complete_inline(scan->next_sender()); + EXPECT_TRUE(scan->result().done); + } +} + +TEST(OverflowLoad, InlineChainUsesBoundedCpuContinuations) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + auto store = std::make_shared<TestOverflowStore>(); + auto captured = std::make_shared<TestBackingMap>(); + constexpr size_t pages = 128; + for (size_t id = 1; id <= pages; ++id) { + add_overflow(store, captured.get(), id, id == pages ? kInvalidPageId : id + 1, "x"); + } + AsyncStorePageLoader loader(store, runtime, lane, {}, resolver(captured)); + std::atomic<size_t> calls{0}; + loader.load_overflow(1, pages, [&](Status status, std::shared_ptr<const buffer> value) { + EXPECT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(value, nullptr); + EXPECT_EQ(value->slice().to_string(), std::string(pages, 'x')); + ++calls; + }); + runtime->drain(); + EXPECT_EQ(calls.load(), 1U); + EXPECT_EQ(store->reads.load(), pages); + EXPECT_EQ(runtime->snapshot().completed, pages + 1); +} + +TEST(OverflowLoad, LateReadErrorReleasesSharedInflightPermit) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestOverflowStore>(); + store->deferred = true; + auto captured = std::make_shared<TestBackingMap>(); + add_overflow(store, captured.get(), 1, kInvalidPageId, "hello"); + AsyncStorePageLoader loader(store, runtime, lane, {.max_inflight = 1}, resolver(captured)); + std::vector<Code> outcomes; + auto complete = [&](Status status, std::shared_ptr<const buffer>) { outcomes.push_back(status.code()); }; + loader.load_overflow(1, 5, complete); + runtime->drain(); + loader.load_overflow(1, 5, complete); + store->finish(Status::io_error("deferred failure")); + loader.load_overflow(1, 5, complete); + runtime->drain(); + store->finish(Status::io_error("second failure")); + EXPECT_EQ(outcomes, (std::vector<Code>{Code::kResourceExhausted, Code::kIoError, Code::kIoError})); + EXPECT_EQ(store->reads.load(), 2U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_load_test.cpp b/lib/crowdb-tree/tests/unit/page_load_test.cpp new file mode 100644 index 000000000..382372835 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_load_test.cpp @@ -0,0 +1,325 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/maptable/compressor.h" +#include "load_budget_test_support.h" +#include "published_read_test_support.h" + +#include <cstring> +#include <thread> + +namespace crowdb::tree +{ +using namespace test_read; + +namespace +{ +class TestAsyncStore final : public AsyncPageStore +{ + public: + bool deferred = false; + size_t reads = 0; + std::vector<uint8_t> blob; + std::deque<AsyncCompletion> pending; + + uint64_t submit_read(PageAddr /*addr*/, void *destination, size_t size, AsyncCompletion complete) override + { + ++reads; + EXPECT_EQ(size, blob.size()); + EXPECT_EQ(reinterpret_cast<uintptr_t>(destination) % 4096, 0U); + std::memcpy(destination, blob.data(), std::min(size, blob.size())); + if (deferred) { + pending.push_back(complete); + } + else { + complete.complete(Status::Ok()); + } + return reads; + } + + uint64_t submit_write(PageAddr /*addr*/, const void * /*buf*/, size_t /*len*/, AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return 0; + } + + Status submit_fsync(AsyncCompletion complete) override + { + complete.complete(Status::not_supported()); + return Status::not_supported(); + } + + void cancel(uint64_t /*op_id*/) override + { + } + + void finish(Status status = Status::Ok()) + { + ASSERT_FALSE(pending.empty()); + auto callback = pending.front(); + pending.pop_front(); + callback.complete(std::move(status)); + } +}; + +BaseRef test_backing(const std::shared_ptr<TestAsyncStore> &store) +{ + const auto frame = base({ + {.key = "key", .slot = 5, .value = "disk"} + }); + encode_durable_page(frame->frame(), frame->page_bytes(), compress_algo::kNone, &store->blob); + auto backing = std::make_shared<BackingRef>(); + backing->length = store->blob.size(); + backing->owner = store; + return {.identity = 1, .max_slot = 5, .backing = std::move(backing)}; +} +} // namespace + +TEST(PageLoad, InlineTransportDecodesAndCompletesOnCpuPool) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + auto backing = test_backing(store); + AsyncStorePageLoader loader(store, runtime, lane); + std::atomic<size_t> calls{0}; + const auto caller = std::this_thread::get_id(); + loader.load_leaf(backing, [&](Status status, std::shared_ptr<const LeafBase> leaf) { + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_NE(std::this_thread::get_id(), caller); + ASSERT_NE(leaf, nullptr); + EXPECT_EQ(CellView(leaf->cell(0)).value().to_string(), "disk"); + ++calls; + }); + runtime->drain(); + EXPECT_EQ(calls.load(), 1U); + EXPECT_EQ(store->reads, 1U); +} + +TEST(PageLoad, DecodedLeafRetainsItsActualFrameChargeAfterLoaderAndReserveAreReleased) +{ + LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + auto backing = test_backing(store); + auto loader = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, + OverflowBackingResolver{}, credits.credits); + std::shared_ptr<const LeafBase> result; + loader->load_leaf(backing, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()) << status.to_string(); + result = std::move(leaf); + }); + runtime->drain(); + ASSERT_NE(result, nullptr); + loader.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().reserved[LoadCredits::kBytes], 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_GE(credits.budget.snapshot().allocated[LoadCredits::kBytes], result->page_bytes() + sizeof(LeafBase)); + EXPECT_EQ(CellView(result->cell(0)).value().to_string(), "disk"); + result.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageLoad, IndependentIoCreditSurvivesLateFailureAndIsReusable) +{ + LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + store->deferred = true; + auto backing = test_backing(store); + auto loader = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, + OverflowBackingResolver{}, credits.credits); + size_t failures = 0; + loader->load_leaf(backing, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kIoError); + EXPECT_FALSE(leaf); + ++failures; + }); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + auto second = backing; + auto second_backing = std::make_shared<BackingRef>(*backing.backing); + ++second_backing->address; + second.backing = std::move(second_backing); + loader->load_leaf(second, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(leaf); + ++failures; + }); + EXPECT_EQ(store->reads, 1U); + store->finish(Status::io_error("controlled transport failure")); + EXPECT_EQ(failures, 2U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + size_t successes = 0; + loader->load_leaf(backing, [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_NE(leaf, nullptr); + ++successes; + }); + store->finish(); + runtime->drain(); + EXPECT_EQ(successes, 1U); + EXPECT_EQ(store->reads, 2U); + loader.reset(); + credits.seal(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageLoad, SealedDecodeCreditRetainsInflightBlobUntilTransportTerminal) +{ + LoadCredits credits; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + store->deferred = true; + auto backing = test_backing(store); + auto loader = std::make_shared<AsyncStorePageLoader>(store, runtime, lane, PageLoadLimits{}, + OverflowBackingResolver{}, credits.credits); + size_t calls = 0; + loader->load_leaf(backing, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(leaf); + ++calls; + }); + loader.reset(); + credits.seal(); + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], backing.backing->length + 4095); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 1U); + store->finish(); + runtime->drain(); + EXPECT_EQ(calls, 1U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PageLoad, RejectsMixedAccountsAndByteIoUnitsBeforeAllocatingLoaderState) +{ + LoadCredits first; + LoadCredits second; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + for (const auto &invalid : { + PageLoadBudget{.bytes = first.credits.bytes, .io = second.credits.io}, + PageLoadBudget{.bytes = first.credits.io, .io = first.credits.io }, + PageLoadBudget{.bytes = first.credits.bytes, .io = {} } + }) { + EXPECT_THROW((void)AsyncStorePageLoader(store, runtime, lane, {}, {}, invalid), std::invalid_argument); + } + EXPECT_EQ(first.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + EXPECT_EQ(first.budget.snapshot().allocated[LoadCredits::kIo], 0U); + EXPECT_EQ(second.budget.snapshot().allocated[LoadCredits::kIo], 0U); +} + +TEST(PageLoad, SlowReadLeavesSingleCpuWorkerAvailable) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + store->deferred = true; + auto backing = test_backing(store); + auto loader = std::make_shared<AsyncStorePageLoader>(store, runtime, lane); + std::atomic<size_t> completed{0}; + loader->load_leaf(backing, [&](Status status, std::shared_ptr<const LeafBase> leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_NE(leaf, nullptr); + ++completed; + }); + std::atomic<bool> unrelated{false}; + ASSERT_TRUE(runtime + ->try_submit( + lane, [](void *context) noexcept { static_cast<std::atomic<bool> *>(context)->store(true); }, + &unrelated) + .ok()); + runtime->drain(); + EXPECT_TRUE(unrelated.load()); + EXPECT_EQ(completed.load(), 0U); + loader.reset(); + store->finish(); + runtime->drain(); + EXPECT_EQ(completed.load(), 1U); +} + +TEST(PageLoad, InflightBudgetIsReleasedOnLateIoFailure) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + store->deferred = true; + auto backing = test_backing(store); + AsyncStorePageLoader loader(store, runtime, lane, {.max_inflight = 1}); + std::vector<Code> codes; + auto complete = [&](Status status, std::shared_ptr<const LeafBase>) { codes.push_back(status.code()); }; + loader.load_leaf(backing, complete); + loader.load_leaf(backing, complete); + EXPECT_EQ(codes, (std::vector<Code>{Code::kResourceExhausted})); + store->finish(Status::io_error("late failure")); + loader.load_leaf(backing, complete); + store->finish(Status::io_error("second failure")); + EXPECT_EQ(codes, (std::vector<Code>{Code::kResourceExhausted, Code::kIoError, Code::kIoError})); + EXPECT_EQ(store->reads, 2U); +} + +TEST(PageLoad, CorruptionAndProvenSlotBoundsRejectLoadedFrames) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + for (bool corrupt_bytes : {false, true}) { + auto store = std::make_shared<TestAsyncStore>(); + auto backing = test_backing(store); + if (corrupt_bytes) { + store->blob.back() ^= 1; + } + else { + backing.max_slot = 4; + } + AsyncStorePageLoader loader(store, runtime, lane); + std::atomic<size_t> errors{0}; + loader.load_leaf(backing, [&](Status status, std::shared_ptr<const LeafBase> leaf) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_EQ(leaf, nullptr); + ++errors; + }); + runtime->drain(); + EXPECT_EQ(errors.load(), 1U); + } +} + +TEST(PageLoad, RejectedCpuQueueFailsWithoutDecodingOnIoThread) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + auto backing = test_backing(store); + runtime->close(); + AsyncStorePageLoader loader(store, runtime, lane); + size_t calls = 0; + loader.load_leaf(backing, [&](Status status, std::shared_ptr<const LeafBase> leaf) { + EXPECT_EQ(status.code(), Code::kUnavailable); + EXPECT_EQ(leaf, nullptr); + ++calls; + }); + EXPECT_EQ(calls, 1U); +} + +TEST(PageLoad, BoundsAndMissingRuntimeRejectBeforeIo) +{ + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto store = std::make_shared<TestAsyncStore>(); + auto backing = test_backing(store); + AsyncStorePageLoader bounded(store, runtime, lane, {.max_blob_bytes = 1}); + bounded.load_leaf(backing, [](Status status, std::shared_ptr<const LeafBase>) { + EXPECT_EQ(status.code(), Code::kResourceExhausted); + }); + AsyncStorePageLoader missing(store, {}, lane); + missing.load_leaf(backing, [](Status status, std::shared_ptr<const LeafBase>) { + EXPECT_EQ(status.code(), Code::kInvalidArgument); + }); + missing.load_overflow( + 1, 4, [](Status status, std::shared_ptr<const buffer>) { EXPECT_EQ(status.code(), Code::kNotSupported); }); + EXPECT_EQ(store->reads, 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/page_maintenance_test.cpp b/lib/crowdb-tree/tests/unit/page_maintenance_test.cpp new file mode 100644 index 000000000..34bcb17f0 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/page_maintenance_test.cpp @@ -0,0 +1,356 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_maintenance.h" + +#include <gtest/gtest.h> + +using namespace crowdb::tree; + +namespace +{ +std::shared_ptr<const CellVersion> cell(uint64_t slot, Slice value, OpKind kind = OpKind::kPut) +{ + return std::make_shared<CellVersion>(encode_cell_buf(slot, kind, value), slot, + kind == OpKind::kDelete ? kFlagTombstone : 0); +} + +std::shared_ptr<const OverlayBatch> +run(uint64_t id, std::initializer_list<std::pair<std::string, std::shared_ptr<const CellVersion>>> entries) +{ + OverlayBuilder builder({}); + for (const auto &[key, owner] : entries) { + EXPECT_TRUE(builder.append(Slice(key), owner).ok()); + } + std::shared_ptr<const OverlayBatch> output; + EXPECT_TRUE(builder.finish(id, &output).ok()); + return output; +} + +std::shared_ptr<const BackingRef> cold_backing() +{ + auto backing = std::make_shared<BackingRef>(); + backing->length = 4096; + backing->owner = std::make_shared<int>(1); + return backing; +} + +PageMaintenanceLimits small_layout() +{ + PageMaintenanceLimits limits; + limits.primary_page_bytes = 256; + limits.target_page_bytes = 220; + limits.merge_trigger_bytes = 190; + limits.merge_target_bytes = 210; + limits.inline_value_bytes = 64; + return limits; +} + +std::shared_ptr<PageState> dense_page() +{ + auto page = std::make_shared<PageState>(); + page->route_generation = 1; + page->content_id = 1; + page->right_sibling = 99; + page->overlays.push_back(run(1, { + {"a", cell(1, std::string(30, 'a'))}, + {"b", cell(2, std::string(30, 'b'))}, + {"c", cell(3, std::string(30, 'c'))}, + {"d", cell(4, std::string(30, 'd'))} + })); + return page; +} +} // namespace + +TEST(PageMaintenance, ColdOverlayCoalescePreservesTombstoneAndDoesNotLoadBase) +{ + auto page = std::make_shared<PageState>(); + page->content_id = 9; + page->bases.push_back({.identity = 1, .max_slot = 2, .backing = cold_backing()}); + page->overlays.push_back(run(1, { + {"key", cell(2, "old")} + })); + const auto tombstone = cell(3, {}, OpKind::kDelete); + page->overlays.push_back(run(2, { + {"key", tombstone} + })); + PageMaintenanceLimits limits; + limits.gc_floor = 3; + auto result = coalesce_overlays(page, limits, 7, 100); + ASSERT_TRUE(result.status.ok()); + ASSERT_NE(result.page, nullptr); + EXPECT_EQ(result.readiness, MaintenanceReadiness::kReady); + EXPECT_TRUE(result.loads.empty()); + ASSERT_EQ(result.page->overlays.size(), 1U); + EXPECT_EQ(result.page->overlays[0]->find("key")->owner, tombstone); + EXPECT_EQ(result.page->lookup("key").state, LogicalLookupState::kTombstone); + EXPECT_EQ(result.page->bases[0].backing, page->bases[0].backing); + EXPECT_EQ(result.page->content_id, 9U); + EXPECT_EQ(result.page->representation_id, 7U); + EXPECT_EQ(page->overlays.size(), 2U); + EXPECT_EQ(result.removed_primary_bytes, 0U); +} + +TEST(PageMaintenance, OverlayOnlyGcKeepsNewerDeletesAndPreservesCapturedPages) +{ + auto page = std::make_shared<PageState>(); + page->overlays.push_back(run(1, { + {"a", cell(1, "old")}, + {"b", cell(2, "old")} + })); + page->overlays.push_back(run(2, { + {"a", cell(3, {}, OpKind::kDelete)}, + {"b", cell(4, {}, OpKind::kDelete)} + })); + PageMaintenanceLimits limits; + limits.gc_floor = 3; + auto result = coalesce_overlays(page, limits, 7, 100); + ASSERT_TRUE(result.status.ok()); + ASSERT_NE(result.page, nullptr); + EXPECT_EQ(result.page->lookup("a").state, LogicalLookupState::kMissing); + EXPECT_EQ(result.page->lookup("b").state, LogicalLookupState::kTombstone); + EXPECT_EQ(page->lookup("a").state, LogicalLookupState::kTombstone); + EXPECT_EQ(page->overlays.front()->find("a")->value().to_string(), "old"); + EXPECT_EQ(result.removed_primary_bytes, kCellHeaderSize + kLeafSlotSize + 1); +} + +TEST(PageMaintenance, CoalesceKeepsStableTiesAndProducesBoundedRuns) +{ + auto page = std::make_shared<PageState>(); + const auto first = cell(7, "first"); + page->overlays.push_back(run(1, { + {"a", first}, + {"c", cell(3, "c")} + })); + page->overlays.push_back(run(2, { + {"a", cell(7, "second")}, + {"b", cell(2, "b") } + })); + PageMaintenanceLimits limits; + limits.run.max_entries = 1; + auto result = coalesce_overlays(page, limits, 10, 100); + ASSERT_TRUE(result.status.ok()); + ASSERT_EQ(result.page->overlays.size(), 3U); + EXPECT_EQ(result.page->overlays[0]->entries()[0].owner, first); + EXPECT_EQ(result.page->overlays[0]->id(), 100U); + EXPECT_EQ(result.page->overlays[2]->id(), 102U); + EXPECT_EQ(result.page->lookup("b").cell.value.to_string(), "b"); +} + +TEST(PageMaintenance, CoalesceBudgetFailureDoesNotPublishPartialOutput) +{ + auto page = dense_page(); + PageMaintenanceLimits limits; + limits.max_work_entries = 2; + auto result = coalesce_overlays(page, limits, 2, 10); + EXPECT_EQ(result.status.code(), Code::kResourceExhausted); + EXPECT_EQ(result.readiness, MaintenanceReadiness::kDeferred); + EXPECT_EQ(result.page, nullptr); + EXPECT_EQ(page->overlays[0]->entries().size(), 4U); +} + +TEST(PageMaintenance, ColdLayoutRequestsOneBaseWithoutInventingPageSize) +{ + auto page = dense_page(); + page->bases.push_back({.identity = 1, .backing = cold_backing()}); + page->bases.push_back({.identity = 2, .backing = cold_backing()}); + auto layout = plan_leaf_layout(page, small_layout()); + EXPECT_TRUE(layout.status.ok()); + EXPECT_EQ(layout.readiness, MaintenanceReadiness::kNeedsBase); + ASSERT_EQ(layout.loads.size(), 1U); + EXPECT_EQ(layout.loads[0].base_index, 0U); + EXPECT_TRUE(layout.primary_bytes.empty()); + EXPECT_TRUE(layout.separators.empty()); +} + +TEST(PageMaintenance, ByteLayoutSplitsManyWaysWithoutCopyingValues) +{ + auto page = dense_page(); + const auto limits = small_layout(); + auto layout = plan_leaf_layout(page, limits); + ASSERT_TRUE(layout.status.ok()); + ASSERT_EQ(layout.readiness, MaintenanceReadiness::kReady); + EXPECT_TRUE(std::ranges::equal(layout.separators, std::array{"b", "c", "d"}, + [](const auto &actual, const char *expected) { return actual == expected; })); + ASSERT_EQ(layout.primary_bytes.size(), 4U); + std::vector<SplitTarget> targets; + targets.reserve(layout.primary_bytes.size()); + for (size_t i = 0; i < layout.primary_bytes.size(); ++i) { + EXPECT_LE(layout.primary_bytes[i], limits.primary_page_bytes); + targets.push_back({ + .page_id = 10 + i, .version = {.route_generation = 2, .content_id = 10 + i, .representation_id = 10 + i} + }); + } + auto split = split_leaf_ranges(layout, targets, limits); + ASSERT_TRUE(split.status.ok()); + ASSERT_EQ(split.pages.size(), 4U); + for (size_t i = 0; i < split.pages.size(); ++i) { + const std::string key(1, static_cast<char>('a' + i)); + auto before = page->lookup(Slice(key)); + auto after = split.pages[i]->lookup(Slice(key)); + EXPECT_EQ(before.cell.owner, after.cell.owner); + EXPECT_EQ(before.cell.value.data(), after.cell.value.data()); + EXPECT_EQ(split.pages[i]->right_sibling, i == 3 ? 99U : 11 + i); + for (size_t j = 0; j < split.pages.size(); ++j) { + if (j != i) { + EXPECT_EQ(split.pages[j]->lookup(Slice(key)).state, LogicalLookupState::kMissing); + } + } + } + EXPECT_EQ(page->right_sibling, 99U); + EXPECT_EQ(page->overlays[0]->entries().size(), 4U); +} + +TEST(PageMaintenance, SplitSharesResidentBaseAndClipsEveryView) +{ + auto page = std::make_shared<PageState>(); + page->route_generation = 1; + std::vector<leaf_entry> entries; + for (const auto *key : {"a", "b", "c", "d"}) { + entries.push_back({.key = key, .cell = encode_cell_buf(1, OpKind::kPut, std::string(30, 'v'))}); + } + auto base = std::shared_ptr<const LeafBase>(LeafBase::build(entries)); + page->bases.push_back({.identity = 42, .resident = base}); + auto layout = plan_leaf_layout(page, small_layout()); + ASSERT_TRUE(layout.status.ok()); + std::vector<SplitTarget> targets; + targets.reserve(layout.primary_bytes.size()); + for (size_t i = 0; i < layout.primary_bytes.size(); ++i) { + targets.push_back({ + .page_id = i + 1, .version = {.route_generation = 2, .content_id = i + 1, .representation_id = i + 1} + }); + } + auto split = split_leaf_ranges(layout, targets, small_layout()); + ASSERT_TRUE(split.status.ok()); + ASSERT_EQ(split.pages.size(), 4U); + for (size_t i = 0; i < split.pages.size(); ++i) { + ASSERT_EQ(split.pages[i]->bases.size(), 1U); + EXPECT_EQ(split.pages[i]->bases[0].resident, base); + EXPECT_EQ(split.pages[i]->bases[0].identity, 42U); + LogicalPageCursor cursor(split.pages[i], {}, UINT64_MAX, 1); + ASSERT_TRUE(cursor.valid()); + EXPECT_EQ(cursor.key().to_string(), std::string(1, static_cast<char>('a' + i))); + cursor.next(); + EXPECT_FALSE(cursor.valid()); + } +} + +TEST(PageMaintenance, LargeValuePlansOverflowAndOversizeKeyIsExplicitlyRejected) +{ + auto page = std::make_shared<PageState>(); + page->overlays.push_back(run(1, { + {"a", cell(1, std::string(100000, 'v'))} + })); + auto layout = plan_leaf_layout(page, small_layout()); + ASSERT_TRUE(layout.status.ok()); + EXPECT_EQ(layout.overflow_values, 1U); + EXPECT_EQ(layout.primary_bytes.size(), 1U); + EXPECT_EQ(layout.total_primary_bytes, kFrameHeaderSize + kFrameTrailerSize + kLeafSlotSize + 1 + kOverflowCellSize); + page->overlays = { + run(2, {{std::string(200, 'z'), cell(2, "x")}} + ) + }; + layout = plan_leaf_layout(page, small_layout()); + EXPECT_EQ(layout.status.code(), Code::kInvalidArgument); + EXPECT_TRUE(layout.primary_bytes.empty()); +} + +TEST(PageMaintenance, SplitTargetBudgetDoesNotRejectOnePageThatAlreadyFits) +{ + auto page = std::make_shared<PageState>(); + page->overlays.push_back(run(1, { + {"a", cell(1, std::string(30, 'a'))}, + {"b", cell(2, std::string(30, 'b'))} + })); + auto limits = small_layout(); + limits.max_output_pages = 1; + auto layout = plan_leaf_layout(page, limits); + ASSERT_TRUE(layout.status.ok()); + EXPECT_TRUE(std::ranges::equal(layout.primary_bytes, std::array<size_t, 1>{240})); + EXPECT_TRUE(layout.separators.empty()); + layout = plan_leaf_layout(dense_page(), limits); + EXPECT_EQ(layout.status.code(), Code::kResourceExhausted); + EXPECT_EQ(layout.readiness, MaintenanceReadiness::kDeferred); + EXPECT_TRUE(layout.primary_bytes.empty()); +} + +TEST(PageMaintenance, MergeRequiresSameParentAdjacentRangesAndHysteresis) +{ + auto left = std::make_shared<PageState>(); + auto right = std::make_shared<PageState>(); + left->range = {.upper = "b"}; + right->range = {.lower = "b"}; + left->right_sibling = 2; + right->right_sibling = 8; + left->overlays = { + run(1, {{"a", cell(1, "x")}} + ) + }; + right->overlays = { + run(2, {{"b", cell(2, "y")}} + ) + }; + LeafMergeInput input{ + .left = left, .right = right, .left_page_id = 1, .right_page_id = 2, .left_parent = 5, .right_parent = 5}; + auto merged = + plan_leaf_merge(input, small_layout(), {.route_generation = 1, .content_id = 3, .representation_id = 3}, 100); + ASSERT_TRUE(merged.status.ok()); + ASSERT_NE(merged.page, nullptr); + EXPECT_EQ(merged.page->lookup("a").cell.value.to_string(), "x"); + EXPECT_EQ(merged.page->lookup("b").cell.value.to_string(), "y"); + EXPECT_EQ(merged.page->right_sibling, 8U); + input.right_parent = 6; + EXPECT_EQ(plan_leaf_merge(input, small_layout(), {1, 3, 3}, 100).status.code(), Code::kInvalidArgument); + input.right_parent = 5; + left->right_sibling = 3; + EXPECT_EQ(plan_leaf_merge(input, small_layout(), {1, 3, 3}, 100).status.code(), Code::kInvalidArgument); +} + +TEST(PageMaintenance, MergeClipsHiddenBaseKeysBeforeCombiningRanges) +{ + std::vector<leaf_entry> entries; + entries.push_back({.key = "a", .cell = encode_cell_buf(1, OpKind::kPut, "left")}); + entries.push_back({.key = "b", .cell = encode_cell_buf(99, OpKind::kPut, "hidden")}); + auto base = std::shared_ptr<const LeafBase>(LeafBase::build(entries)); + auto left = std::make_shared<PageState>(); + left->range = {.upper = "b"}; + left->right_sibling = 2; + left->bases.push_back({.identity = 1, .resident = base}); + auto right = std::make_shared<PageState>(); + right->range = {.lower = "b"}; + right->overlays.push_back(run(2, { + {"b", cell(2, "visible")} + })); + auto merged = plan_leaf_merge( + {.left = left, .right = right, .left_page_id = 1, .right_page_id = 2, .left_parent = 5, .right_parent = 5}, + small_layout(), {.route_generation = 1, .content_id = 3, .representation_id = 3}, 100); + ASSERT_TRUE(merged.status.ok()); + ASSERT_NE(merged.page, nullptr); + EXPECT_EQ(merged.page->lookup("b").cell.value.to_string(), "visible"); +} + +TEST(PageMaintenance, PartialSourceWindowPreservesTombstonesMaskingUntouchedRuns) +{ + auto page = std::make_shared<PageState>(); + page->overlays.push_back(run(1, { + {"key", cell(1, "older")} + })); + page->overlays.push_back(run(2, { + {"key", cell(3, {}, OpKind::kDelete)} + })); + page->overlays.push_back(run(3, { + {"other", cell(2, "value")} + })); + PageMaintenanceLimits limits; + limits.max_sources = 2; + limits.gc_floor = 3; + auto partial = coalesce_overlays(page, limits, 2, 4); + ASSERT_TRUE(partial.status.ok()) << partial.status.to_string(); + ASSERT_TRUE(partial.page); + EXPECT_EQ(partial.page->overlays.size(), 2U); + EXPECT_EQ(partial.page->lookup("key").state, LogicalLookupState::kTombstone); + EXPECT_EQ(partial.page->lookup("other").cell.value.to_string(), "value"); + auto complete = coalesce_overlays(partial.page, limits, 3, 5); + ASSERT_TRUE(complete.status.ok()) << complete.status.to_string(); + EXPECT_EQ(complete.page->lookup("key").state, LogicalLookupState::kMissing); + EXPECT_EQ(complete.page->lookup("other").cell.value.to_string(), "value"); +} diff --git a/lib/crowdb-tree/tests/unit/portable_import_bound_test.cpp b/lib/crowdb-tree/tests/unit/portable_import_bound_test.cpp new file mode 100644 index 000000000..2790e6f7e --- /dev/null +++ b/lib/crowdb-tree/tests/unit/portable_import_bound_test.cpp @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/portable_import.h" +#include "btree/portable_import_plan.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(PortableImportBound, PlannedFrameAndKeyWindowsCompleteWithExactlyTheirPrepaidCredit) +{ + Config config; + config.frame_bytes = 512; + config.max_inline_value = 16; + config.inner_max_keys = 2; + PageLoadLimits loads; + loads.max_leaf_entries = 3; + loads.max_leaf_key_bytes = 12; + PortableImportPlan plan(config, loads); + for (size_t i = 0; i < 37; ++i) { + ASSERT_TRUE(plan.append(3, i % 2 == 0 ? 1000 : 0, i % 2 != 0).ok()); + } + const auto shape = plan.finish(); + if (!shape) { + ADD_FAILURE() << "shape is missing"; + return; + } + const auto bound = detail::AllocationBound() + .add(native_import_allocation_bound(shape.value())) + .add(PortableImportBuilder::allocation_bound(shape.value())) + .value(); + if (!bound) { + ADD_FAILURE() << "bound is missing"; + return; + } + detail::ResourceBudget::Limits limits{}; + limits[static_cast<size_t>(detail::BudgetClass::kReclaim)] = bound.value(); + detail::ResourceBudget budget(limits, 1); + auto account = budget.make_account(limits); + auto reserved = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, bound.value()); + if (!reserved) { + ADD_FAILURE() << "reserved is missing"; + return; + } + auto credit = detail::AllocationReservation::create(std::move(reserved.value())); + std::shared_ptr<ReadablePageCatalog> catalog; + { + PortableImportBuilder builder(config, loads, 40, credit); + for (size_t i = 0; i < 37; ++i) { + const auto key = std::string(3 - std::to_string(i).size(), '0') + std::to_string(i); + const auto cell = encode_cell_buf(i + 1, i % 2 == 0 ? OpKind::kPut : OpKind::kDelete, + i % 2 == 0 ? std::string(1000, 'v') : std::string{}); + const auto status = builder.append(key, Slice(cell)); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + const auto status = builder.finish(&catalog); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + EXPECT_LE(catalog->frames.size(), shape.value().frames); + EXPECT_EQ(shape.value().leaf_pages, 13U); + catalog.reset(); + credit->seal(); + credit.reset(); + EXPECT_EQ(budget.snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kReclaim)], 0U); +} + +TEST(PortableImportBound, OversizedInputsAndArithmeticOverflowCannotBecomeSmallReservations) +{ + Config config; + config.frame_bytes = 4096; + PortableImportPlan plan(config, {}); + EXPECT_EQ(plan.append(SIZE_MAX, 1, false).code(), Code::kResourceExhausted); + EXPECT_EQ(plan.append(1, SIZE_MAX, false).code(), Code::kResourceExhausted); + NativeImportShape shape; + shape.frames = 1; + shape.primary_pages = 1; + shape.max_frame_bytes = SIZE_MAX; + EXPECT_FALSE(native_import_allocation_bound(shape)); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/portable_import_test.cpp b/lib/crowdb-tree/tests/unit/portable_import_test.cpp new file mode 100644 index 000000000..81d8a0035 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/portable_import_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/portable_import.h" +#include "recovered_state_test_support.h" +#include "snapshot/operation_wait.h" + +namespace crowdb::tree +{ +namespace +{ +TEST(PortableImport, SortedCellsBuildABoundedBalancedReadableGraphWithOverflow) +{ + test_read::TestRecoveredGraph test; + Config config; + config.frame_bytes = 512; + config.max_inline_value = 16; + config.inner_max_keys = 2; + PortableImportBuilder builder(config, {}, 300, test.credits.credits.bytes); + for (size_t i = 0; i != 200; ++i) { + const auto key = std::string(3 - std::to_string(i).size(), '0') + std::to_string(i); + const auto cell = encode_cell_buf(i + 1, OpKind::kPut, std::string(1000, static_cast<char>('a' + (i % 26)))); + const auto status = builder.append(key, Slice(cell)); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + std::shared_ptr<ReadablePageCatalog> catalog; + auto status = builder.finish(&catalog); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_NE(catalog, nullptr); + EXPECT_GT(catalog->frames.size(), 200U); + ASSERT_TRUE(RecoveredStateOperation::create_readable(catalog, test.loader, test.runtime, test.lane, {}, + test.credits.credits.bytes, &test.operation) + .ok()); + status = detail::wait_snapshot_operation(test.operation); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(test.operation->result().state->stats.kv_known_count, 200U); + EXPECT_GT(test.operation->result().state->stats.height, 2U); + EXPECT_EQ(test.operation->result().state->coverage, 300U); + EXPECT_EQ(test.store->reads.load(), 0U); +} + +TEST(PortableImport, FailurePoisonsStagingAndDoesNotReturnAPartialCatalog) +{ + test_read::LoadCredits credits; + Config config; + PortableImportBuilder builder(config, {}, 7, credits.credits.bytes); + auto cell = encode_cell_buf(7, OpKind::kPut, "value"); + ASSERT_TRUE(builder.append("z", Slice(cell)).ok()); + EXPECT_EQ(builder.append("a", Slice(cell)).code(), Code::kInvalidArgument); + EXPECT_EQ(builder.append("zz", Slice(cell)).code(), Code::kInvalidArgument); + std::shared_ptr<ReadablePageCatalog> catalog; + EXPECT_EQ(builder.finish(&catalog).code(), Code::kInvalidArgument); + EXPECT_FALSE(catalog); +} + +TEST(PortableImport, EmptyKeyAndTombstonePreserveSlotsAndEmptyImportPreservesCoverage) +{ + test_read::LoadCredits credits; + PortableImportBuilder builder({}, {}, 17, credits.credits.bytes); + const auto empty = encode_cell_buf(5, OpKind::kPut, "empty"); + const auto deleted = encode_cell_buf(9, OpKind::kDelete); + ASSERT_TRUE(builder.append("", Slice(empty)).ok()); + ASSERT_TRUE(builder.append("removed", Slice(deleted)).ok()); + std::shared_ptr<ReadablePageCatalog> catalog; + ASSERT_TRUE(builder.finish(&catalog).ok()); + const auto &raw = catalog->frames.at(catalog->root_page_id).backing->resident_frame; + const LeafFrameView leaf(raw.data(), raw.size()); + ASSERT_EQ(leaf.count(), 2U); + EXPECT_TRUE(leaf.key(0).empty()); + EXPECT_EQ(CellView(leaf.cell(0)).slot(), 5U); + EXPECT_TRUE(CellView(leaf.cell(1)).is_tombstone()); + EXPECT_EQ(CellView(leaf.cell(1)).slot(), 9U); + PortableImportBuilder empty_builder({}, {}, 23, credits.credits.bytes); + catalog.reset(); + ASSERT_TRUE(empty_builder.finish(&catalog).ok()); + EXPECT_TRUE(catalog->frames.empty()); + EXPECT_EQ(catalog->coverage, 23U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/prepared_split_c_api_test.cpp b/lib/crowdb-tree/tests/unit/prepared_split_c_api_test.cpp new file mode 100644 index 000000000..d0d52bfc0 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/prepared_split_c_api_test.cpp @@ -0,0 +1,153 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/split_overlay_c_api.h" +#include "crowdb-tree/status.h" + +#include <gtest/gtest.h> + +#include <array> +#include <utility> + +namespace crowdb::tree +{ +namespace +{ +class PreparedSplitCAbi : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &source), 0); + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &destination), 0); + } + + void TearDown() override + { + ct_prepared_split_overlay_destroy(token); + ct_close(source); + ct_close(destination); + ct_runtime_free(runtime); + } + + static ct_installed_split_overlay_v1 output() + { + return {.version = 1, .struct_size = sizeof(ct_installed_split_overlay_v1), .generation = 71, .frontier = 79}; + } + + static constexpr ct_status invalid = static_cast<ct_status>(Code::kInvalidArgument); + ct_runtime *runtime = nullptr; + ct_tree *source = nullptr; + ct_tree *destination = nullptr; + ct_prepared_split_overlay *token = nullptr; + ct_split_overlay_limits_v1 limits{ + .version = 1, .struct_size = sizeof(ct_split_overlay_limits_v1), .max_records = 8, .max_bytes = 4096}; +}; + +TEST_F(PreparedSplitCAbi, ExactLimitsAndNullArgumentsLeaveOutputsUnchanged) +{ + uint64_t start = 97; + EXPECT_EQ(ct_arm_split_overlay(nullptr, destination, &limits, &token, &start), invalid); + EXPECT_EQ(ct_arm_split_overlay(source, nullptr, &limits, &token, &start), invalid); + EXPECT_EQ(ct_arm_split_overlay(source, destination, nullptr, &token, &start), invalid); + EXPECT_EQ(ct_arm_split_overlay(source, destination, &limits, nullptr, &start), invalid); + EXPECT_EQ(ct_arm_split_overlay(source, destination, &limits, &token, nullptr), invalid); + for (const auto version : std::array<uint32_t, 2>{0, 2}) { + auto bad = limits; + bad.version = version; + EXPECT_EQ(ct_arm_split_overlay(source, destination, &bad, &token, &start), invalid); + EXPECT_EQ(token, nullptr); + EXPECT_EQ(start, 97U); + } + for (const auto size : std::array<uint32_t, 2>{sizeof(limits) - 1, sizeof(limits) + 1}) { + auto bad = limits; + bad.struct_size = size; + EXPECT_EQ(ct_arm_split_overlay(source, destination, &bad, &token, &start), invalid); + EXPECT_EQ(token, nullptr); + EXPECT_EQ(start, 97U); + } + EXPECT_EQ(ct_prepared_split_overlay_prepare(nullptr, 0), invalid); + EXPECT_EQ(ct_prepared_split_overlay_committed_frontier(nullptr, &start), 0); + EXPECT_EQ(start, 97U); + ASSERT_EQ(ct_arm_split_overlay(source, destination, &limits, &token, &start), 0); + ASSERT_NE(token, nullptr); + auto *same = token; + start = 97; + EXPECT_EQ(ct_arm_split_overlay(source, destination, &limits, &token, &start), invalid); + EXPECT_EQ(token, same); + EXPECT_EQ(start, 97U); +} + +TEST_F(PreparedSplitCAbi, ExactInstallOutputAndFixedFrontierArePreserved) +{ + auto out = output(); + EXPECT_EQ(ct_prepared_split_overlay_try_install(nullptr, 0, &out), invalid); + EXPECT_EQ(out.generation, 71U); + EXPECT_EQ(out.frontier, 79U); + uint64_t start = 97; + ASSERT_EQ(ct_arm_split_overlay(source, destination, &limits, &token, &start), 0); + EXPECT_EQ(start, 0U); + ASSERT_EQ(ct_prepared_split_overlay_prepare(token, 0), 0); + const uint8_t key = 'k'; + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(source, 1, &key, 1, &value, 1), 0); + EXPECT_EQ(ct_prepared_split_overlay_try_install(token, 1, nullptr), invalid); + for (const auto version : std::array<uint32_t, 2>{0, 2}) { + out = output(); + out.version = version; + EXPECT_EQ(ct_prepared_split_overlay_try_install(token, 1, &out), invalid); + EXPECT_EQ(out.version, version); + EXPECT_EQ(out.generation, 71U); + EXPECT_EQ(out.frontier, 79U); + } + for (const auto size : std::array<uint32_t, 2>{sizeof(out) - 1, sizeof(out) + 1}) { + out = output(); + out.struct_size = size; + EXPECT_EQ(ct_prepared_split_overlay_try_install(token, 1, &out), invalid); + EXPECT_EQ(out.struct_size, size); + EXPECT_EQ(out.generation, 71U); + EXPECT_EQ(out.frontier, 79U); + } + EXPECT_EQ(ct_prepared_split_overlay_committed_frontier(token, &start), 0); + out = output(); + ASSERT_EQ(ct_prepared_split_overlay_try_install(token, 1, &out), 0); + EXPECT_EQ(out.frontier, 1U); + EXPECT_GT(out.generation, 0U); + const auto generation = out.generation; + ASSERT_EQ(ct_prepared_split_overlay_try_install(token, 1, &out), 0); + EXPECT_EQ(out.generation, generation); + EXPECT_EQ(ct_prepared_split_overlay_committed_frontier(token, &start), 1); + EXPECT_EQ(start, 1U); + EXPECT_EQ(ct_prepared_split_overlay_committed_frontier(token, nullptr), 0); + out = output(); + EXPECT_EQ(ct_prepared_split_overlay_try_install(token, 2, &out), static_cast<ct_status>(Code::kUnavailable)); + EXPECT_EQ(out.generation, 71U); + EXPECT_EQ(out.frontier, 79U); + int32_t found = 0; + uint64_t slot = 0; + ct_buf bytes{}; + ASSERT_EQ(ct_get(destination, &key, 1, &found, &slot, &bytes), 0); + EXPECT_EQ(found, 1); + EXPECT_EQ(slot, 1U); + ASSERT_EQ(bytes.len, 1U); + EXPECT_EQ(bytes.data[0], value); + ct_free_buf(&bytes); +} + +TEST_F(PreparedSplitCAbi, PreparedHandleSafelyOutlivesBothTreeFacades) +{ + uint64_t start = 97; + ASSERT_EQ(ct_arm_split_overlay(source, destination, &limits, &token, &start), 0); + ASSERT_EQ(ct_prepared_split_overlay_prepare(token, 0), 0); + ct_close(std::exchange(source, nullptr)); + ct_close(std::exchange(destination, nullptr)); + auto out = output(); + EXPECT_EQ(ct_prepared_split_overlay_try_install(token, 0, &out), static_cast<ct_status>(Code::kUnavailable)); + EXPECT_EQ(out.generation, 71U); + EXPECT_EQ(out.frontier, 79U); + EXPECT_EQ(ct_prepared_split_overlay_committed_frontier(token, &start), 0); + ct_prepared_split_overlay_destroy(std::exchange(token, nullptr)); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/primary_backing_test.cpp b/lib/crowdb-tree/tests/unit/primary_backing_test.cpp new file mode 100644 index 000000000..db45b2fe4 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/primary_backing_test.cpp @@ -0,0 +1,173 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "published_read_test_support.h" +#include "snapshot/primary_backing.h" + +namespace crowdb::tree +{ +namespace +{ +struct TestPrimary +{ + std::shared_ptr<PageState> leaf = std::make_shared<PageState>(); + PublishedPage page; + DurableCapturedFrame frame; + + TestPrimary() + { + auto backing = std::make_shared<BackingRef>(); + backing->backend_id = 7; + backing->content_id = 5; + backing->generation = 1; + backing->length = 4096; + backing->owner = std::make_shared<int>(1); + leaf->bases.push_back({.identity = 5, .generation = 1, .max_slot = 10, .backing = backing}); + frame.page_id = 3; + frame.frame_bytes = 4096; + frame.logical_bytes = 4096; + frame.backing = std::move(backing); + frame.summary.emplace(); + frame.summary->encoded_entries = 1; + frame.summary->new_overflow_frames = 2; + frame.summary->dropped_tombstones = 4; + frame.stored_overflow = std::make_shared<StoredOverflowReferences>(); + page.leaf = leaf; + page.primary = make_primary_backing(frame, *leaf, 10, 2, {}); + } + + [[nodiscard]] bool eligible() const + { + return can_reuse_primary(page, 3, {}, 2, 7, 4096); + } +}; + +TEST(PrimaryBacking, ReceiptForgetsOldWorkCountersAndRetainsOnlyItsPage) +{ + TestPrimary test; + ASSERT_TRUE(test.page.primary); + EXPECT_TRUE(test.eligible()); + const auto &reused = test.page.primary->frame.summary; + const auto &original = test.frame.summary; + if (!reused || !original) { + FAIL() << "primary summary is absent"; + } + EXPECT_EQ(reused->new_overflow_frames, 0U); + EXPECT_EQ(reused->dropped_tombstones, 0U); + EXPECT_EQ(original->new_overflow_frames, 2U); + EXPECT_EQ(test.page.primary->frame.backing, test.frame.backing); +} + +TEST(PrimaryBacking, IdentityGeometryFloorAndSlotFilterMustMatch) +{ + TestPrimary test; + EXPECT_FALSE(can_reuse_primary(test.page, 4, {}, 2, 7, 4096)); + EXPECT_FALSE(can_reuse_primary(test.page, 3, {}, 2, 8, 4096)); + EXPECT_FALSE(can_reuse_primary(test.page, 3, {}, 2, 0, 4096)); + EXPECT_FALSE(can_reuse_primary(test.page, 3, {}, 2, 7, 8192)); + EXPECT_FALSE(can_reuse_primary(test.page, 3, {}, 3, 7, 4096)); + test.leaf->bases.front().max_slot = 9; + EXPECT_FALSE(test.eligible()); + test.leaf->bases.front().max_slot = 11; + EXPECT_TRUE(test.eligible()); + test.leaf->bases.front().max_slot.reset(); + EXPECT_TRUE(test.eligible()); + test.leaf->bases.front().backing = std::make_shared<BackingRef>(*test.frame.backing); + EXPECT_FALSE(test.eligible()); +} + +TEST(PrimaryBacking, LaterOverlaySiblingAndRoutingSliceInvalidateReuse) +{ + TestPrimary test; + test.leaf->overlays = test_read::page({ + {.key = "a", .value = "later"} + }) + ->overlays; + EXPECT_FALSE(test.eligible()); + test.leaf->overlays.clear(); + test.leaf->right_sibling = 8; + EXPECT_FALSE(test.eligible()); + test.leaf->right_sibling = kInvalidPageId; + test.leaf->bases.front().range.lower = OwnedKey("a"); + EXPECT_FALSE(test.eligible()); + test.leaf->bases.front().range.lower.reset(); + PageKeyRange narrowed; + narrowed.upper = OwnedKey("z"); + EXPECT_FALSE(can_reuse_primary(test.page, 3, narrowed, 2, 7, 4096)); + EXPECT_TRUE(test.eligible()); +} + +TEST(PrimaryBacking, AdvancedFloorReusesOnlyACompleteLiveOnlyReceipt) +{ + TestPrimary test; + if (!test.frame.summary) { + FAIL() << "missing frame summary"; + } + test.frame.summary->live_counts_known = true; + test.frame.summary->live_keys = test.frame.summary->encoded_entries; + test.page.primary = make_primary_backing(test.frame, *test.leaf, 10, 2, {}); + EXPECT_TRUE(can_reuse_primary(test.page, 3, {}, 10, 7, 4096)); + EXPECT_FALSE(can_reuse_primary(test.page, 3, {}, 1, 7, 4096)); + + test.frame.summary->encoded_entries++; + test.page.primary = make_primary_backing(test.frame, *test.leaf, 10, 2, {}); + EXPECT_FALSE(can_reuse_primary(test.page, 3, {}, 10, 7, 4096)); + EXPECT_TRUE(test.eligible()); + + test.frame.summary->encoded_entries--; + test.frame.summary->live_counts_known = false; + test.page.primary = make_primary_backing(test.frame, *test.leaf, 10, 2, {}); + EXPECT_FALSE(can_reuse_primary(test.page, 3, {}, 10, 7, 4096)); +} + +TEST(PrimaryBacking, MissingStoredReferenceProofCannotBeInvented) +{ + TestPrimary test; + test.frame.stored_overflow.reset(); + EXPECT_FALSE(make_primary_backing(test.frame, *test.leaf, 10, 2, {})); + test.frame.stored_overflow = std::make_shared<StoredOverflowReferences>(); + test.frame.summary.reset(); + EXPECT_FALSE(make_primary_backing(test.frame, *test.leaf, 10, 2, {})); +} + +TEST(PrimaryBacking, InnerRequiresUnchangedRouteAndCapturedDescendantFences) +{ + TestPrimary test; + auto inner = std::make_shared<InnerPageState>(); + inner->children = {1, 2}; + inner->separators.emplace_back("middle"); + test.page.leaf.reset(); + test.page.level = 1; + test.page.inner = inner; + test.page.inner_backing = test.frame.backing; + test.frame.type = page_type::kInnerBase; + test.frame.stored_overflow.reset(); + auto &summary = test.frame.summary; + if (!summary) { + FAIL() << "primary summary is absent"; + } + summary->inner_fences = {.lower_leaf = 1, .upper_leaf = 2}; + test.page.primary = make_inner_primary_backing(test.frame, test.page, 10, 2, {}); + ASSERT_TRUE(test.page.primary); + EXPECT_TRUE(can_reuse_inner_primary(test.page, 3, {1, 2}, 7, 4096)); + EXPECT_FALSE(can_reuse_inner_primary(test.page, 3, {2, 2}, 7, 4096)); + EXPECT_FALSE(can_reuse_inner_primary(test.page, 3, {}, 7, 4096)); + test.page.inner = std::make_shared<InnerPageState>(*inner); + EXPECT_FALSE(can_reuse_inner_primary(test.page, 3, {1, 2}, 7, 4096)); + auto decoded = std::make_shared<InnerPageState>(*inner); + decoded->frame_proof = {.self_page_id = 3, + .frame_bytes = 4096, + .has_fences = true, + .page_id_fences = true, + .lower = {}, + .upper = {}, + .lower_leaf = 1, + .upper_leaf = 2}; + test.page.inner = decoded; + inner.reset(); + EXPECT_TRUE(test.page.primary->inner_route.expired()); + EXPECT_TRUE(can_reuse_inner_primary(test.page, 3, {1, 2}, 7, 4096)); + decoded->frame_proof->self_page_id = 4; + EXPECT_FALSE(can_reuse_inner_primary(test.page, 3, {1, 2}, 7, 4096)); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_admission_test.cpp b/lib/crowdb-tree/tests/unit/publication_admission_test.cpp new file mode 100644 index 000000000..a1960f0af --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_admission_test.cpp @@ -0,0 +1,243 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <future> + +namespace crowdb::tree +{ +namespace +{ +PublicationResult publish(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + static std::atomic<uint64_t> next_record{1}; + if (!transaction.record) { + transaction.record = std::make_shared<InstallRecord>( + InstallToken{.generation = 1, .batch = 1, .range = 0, .sequence = next_record.fetch_add(1)}); + } + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> done; + auto result = done.get_future(); + request->on_done = [&](PublicationResult value) { done.set_value(std::move(value)); }; + if (!publisher.try_submit(request)) { + ADD_FAILURE() << "admission test publication rejected"; + return {.status = Status::unavailable()}; + } + return result.get(); +} + +PublicationResult acquire(PublicationCoordinator &publisher, uint64_t sequence, uint64_t owner = 42) +{ + return publish(publisher, { + .expected_commit_seq = sequence, + .flush_admission = FlushAdmissionPublication{.action = FlushAdmissionAction::Acquire, + .owner = owner}, + .flush_admission_owner = owner + }); +} + +PublicationResult release(PublicationCoordinator &publisher, uint64_t owner = 42) +{ + return publish(publisher, { + .flush_admission = FlushAdmissionPublication{.action = FlushAdmissionAction::Release, + .owner = owner}, + .flush_admission_owner = owner + }); +} + +std::shared_ptr<PublishedPage> leaf() +{ + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + return page; +} + +TEST(PublicationAdmission, StaleReserveCannotAcquirePublicationOwnership) +{ + PublicationCoordinator publisher; + const auto before = publisher.capture(); + ASSERT_TRUE( + publish(publisher, {.pages = {{.page_id = 0, .expected = {}, .replacement = leaf()}}, .root_page_id = 0}) + .status.ok()); + const auto changed = publisher.capture(); + EXPECT_EQ(acquire(publisher, before->commit_seq).status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture(), changed); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 0U); + EXPECT_EQ(publish(publisher, + { + .flush_admission = FlushAdmissionPublication{FlushAdmissionAction::Acquire, 42}, + .flush_admission_owner = 42 + }) + .status.code(), + Code::kInvalidArgument); +} + +TEST(PublicationAdmission, OnlyAdmittedWriterCanChangeMappingWhileOldReadersRemainValid) +{ + PublicationCoordinator publisher; + auto original = leaf(); + ASSERT_TRUE( + publish(publisher, {.pages = {{.page_id = 0, .expected = {}, .replacement = original}}, .root_page_id = 0}) + .status.ok()); + const auto old = publisher.capture(); + ASSERT_TRUE(acquire(publisher, old->commit_seq).status.ok()); + auto replacement = leaf(); + auto blocked = publish(publisher, {.pages = {{.page_id = 0, .expected = original, .replacement = replacement}}}); + EXPECT_EQ(blocked.status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture()->mapping.get(0), original); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .expected = original, .replacement = replacement}}, + .flush_admission_owner = 42}) + .status.ok()); + EXPECT_EQ(old->mapping.get(0), original); + EXPECT_EQ(publisher.capture()->mapping.get(0), replacement); + EXPECT_EQ(release(publisher, 43).status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 42U); + ASSERT_TRUE(release(publisher).status.ok()); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 0U); + EXPECT_TRUE(publish(publisher, {}).status.ok()); +} + +TEST(PublicationAdmission, PartialHandoffKeepsGateUntilAtomicCoverageCommit) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(acquire(publisher, 0).status.ok()); + ASSERT_TRUE(publish(publisher, + { + .handoff = PendingHandoff{.batch = 1, .frontier = 1}, + .flush_admission_owner = 42 + }) + .status.ok()); + EXPECT_EQ(release(publisher).status.code(), Code::kUnavailable); + auto before = publisher.capture(); + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 1; + PublicationTransaction coverage{ + .expected_coverage = 0, + .coverage = 1, + .expected_sources = before->sources, + .sources = sources, + .handoff = PendingHandoff{.batch = 1, .frontier = 1}, + .flush_admission_owner = 43 + }; + EXPECT_EQ(publish(publisher, coverage).status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture(), before); + coverage.flush_admission_owner = 42; + ASSERT_TRUE(publish(publisher, std::move(coverage)).status.ok()); + EXPECT_EQ(publisher.capture()->coverage, 1U); + EXPECT_EQ(publisher.capture()->flush_admission_owner, 0U); + EXPECT_FALSE(publisher.capture()->pending_handoff); +} + +TEST(PublicationAdmission, SnapshotCaptureKeepsTheSourceFreeCompleteViewDuringPartialHandoff) +{ + PublicationCoordinator publisher; + auto original = leaf(); + auto initial_sources = std::make_shared<PublishedSources>(); + initial_sources->tables.push_back(std::make_shared<MemTable>()); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .expected = {}, .replacement = original}}, + .root_page_id = 0, + .expected_sources = publisher.capture()->sources, + .sources = initial_sources}) + .status.ok()); + const auto before = publisher.capture_complete(); + ASSERT_TRUE(acquire(publisher, before->commit_seq).status.ok()); + const auto complete = publisher.capture_complete(); + EXPECT_EQ(complete->coverage, before->coverage); + EXPECT_EQ(complete->mapping.get(0), original); + EXPECT_TRUE(complete->sources->tables.empty()); + EXPECT_EQ(complete->sources->floor, complete->coverage); + auto replacement = leaf(); + ASSERT_TRUE(publish(publisher, + { + .pages = {{.page_id = 0, .expected = original, .replacement = replacement}}, + .handoff = PendingHandoff{.batch = 1, .frontier = 1}, + .flush_admission_owner = 42 + }) + .status.ok()); + EXPECT_EQ(publisher.capture()->mapping.get(0), replacement); + EXPECT_EQ(publisher.capture_complete(), complete); + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 1; + ASSERT_TRUE(publish(publisher, + { + .expected_coverage = 0, + .coverage = 1, + .expected_sources = publisher.capture()->sources, + .sources = sources, + .handoff = PendingHandoff{.batch = 1, .frontier = 1}, + .flush_admission_owner = 42 + }) + .status.ok()); + EXPECT_EQ(publisher.capture_complete(), publisher.capture()); + EXPECT_EQ(publisher.capture_complete()->coverage, 1U); + EXPECT_EQ(publisher.capture_complete()->mapping.get(0), replacement); + EXPECT_EQ(complete->mapping.get(0), original); +} + +TEST(PublicationAdmission, StandalonePartialHandoffAlsoRetainsTheLastCompleteView) +{ + PublicationCoordinator publisher; + auto original = leaf(); + ASSERT_TRUE( + publish(publisher, {.pages = {{.page_id = 0, .expected = {}, .replacement = original}}, .root_page_id = 0}) + .status.ok()); + auto replacement = leaf(); + ASSERT_TRUE(publish(publisher, + { + .pages = {{.page_id = 0, .expected = original, .replacement = replacement}}, + .handoff = PendingHandoff{.batch = 1, .frontier = 1} + }) + .status.ok()); + EXPECT_EQ(publisher.capture_complete()->mapping.get(0), original); + EXPECT_EQ(publisher.capture_complete()->coverage, 0U); + EXPECT_TRUE(publisher.capture_complete()->sources->tables.empty()); + EXPECT_EQ(publisher.capture()->mapping.get(0), replacement); +} + +TEST(PublicationAdmission, AdmissionReleaseDropsTheUnusedCompleteView) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(acquire(publisher, publisher.capture()->commit_seq).status.ok()); + std::weak_ptr<const PublishedTreeState> complete = publisher.capture_complete(); + EXPECT_FALSE(complete.expired()); + ASSERT_TRUE(release(publisher).status.ok()); + EXPECT_EQ(publisher.capture_complete(), publisher.capture()); + (void)publisher.reclaim(); + EXPECT_TRUE(complete.expired()); +} + +TEST(PublicationAdmission, OldCheckpointMayFinishButNewCaptureWaits) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(publish(publisher, + { + .expected_coverage = 0, .checkpoint = CheckpointPublication{CheckpointAction::Capture, 1} + }) + .status.ok()); + ASSERT_TRUE(acquire(publisher, publisher.capture()->commit_seq).status.ok()); + ASSERT_TRUE(publish(publisher, + { + .checkpoint = CheckpointPublication{CheckpointAction::Commit, 1} + }) + .status.ok()); + EXPECT_EQ(publish(publisher, + { + .expected_coverage = 0, + .checkpoint = CheckpointPublication{CheckpointAction::Capture, 2}, + .flush_admission_owner = 42 + }) + .status.code(), + Code::kUnavailable); + ASSERT_TRUE(release(publisher).status.ok()); + EXPECT_TRUE(publish(publisher, + { + .expected_coverage = 0, .checkpoint = CheckpointPublication{CheckpointAction::Capture, 2} + }) + .status.ok()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_bootstrap_test.cpp b/lib/crowdb-tree/tests/unit/publication_bootstrap_test.cpp new file mode 100644 index 000000000..ded1f475f --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_bootstrap_test.cpp @@ -0,0 +1,181 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_operation.h" +#include "load_budget_test_support.h" +#include "page_cache_residency_test_support.h" + +#include <algorithm> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_cache_residency; + +TEST(PublicationBootstrap, LifecycleChargesSourceControllerEpochAndInitialPublicationUntilLastViewRelease) +{ + LoadCredits credits; + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + std::shared_ptr<const PublishedTreeState> retained; + { + auto sources = detail::reserved_unique<SourceAdmission>( + credits.credits.bytes, 0, 1, SourceWriteOptions{.account = credits.account}, credits.credits.bytes); + const auto source_bytes = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + EXPECT_GT(source_bytes, sizeof(SourceAdmission)); + EXPECT_LE(source_bytes, SourceAdmission::allocation_bound().value_or(0)); + auto publisher = detail::reserved_unique<PublicationCoordinator>( + credits.credits.bytes, 8, 1, std::shared_ptr<TreeWorkMetrics>{}, std::pmr::get_default_resource(), + std::shared_ptr<SharedPageCache>{}, + PublicationBootstrap{ + .metadata = credits.credits.bytes, .initial = {}, .source_catalog = sources->catalog()}); + retained = publisher->capture(); + EXPECT_TRUE(std::ranges::equal(retained->sources->tables, sources->catalog()->tables)); + const auto publication_bytes = credits.budget.snapshot().allocated[LoadCredits::kBytes] - source_bytes; + EXPECT_GT(publication_bytes, sizeof(PublicationCoordinator)); + EXPECT_LE(publication_bytes, PublicationCoordinator::allocation_bound(1).value_or(0)); + std::atomic<uint64_t> frontier{0}; + { + auto controller = detail::reserved_unique<FlushController>( + credits.credits.bytes, *publisher, runtime, *sources, frontier, lane, FlushBatchOptions{}, + FlushBatch::Maintenance{}, credits.credits.bytes); + const auto controller_bytes = + credits.budget.snapshot().allocated[LoadCredits::kBytes] - source_bytes - publication_bytes; + EXPECT_GT(controller_bytes, sizeof(FlushController)); + EXPECT_LE(controller_bytes, FlushController::allocation_bound()); + } + credits.seal(); + } + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + retained.reset(); + runtime.drain(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PublicationBootstrap, SparseRecoveredMappingInstallsExactSourcesAndAllColdResidencyBatches) +{ + LoadCredits credits; + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto resources = std::make_shared<TreeResourceMetrics>(); + auto cache = std::make_shared<SharedPageCache>(PageCacheLimits{}, credits.credits.bytes, resources, &runtime, lane); + SourceAdmission sources(42, 9, {}, credits.credits.bytes); + auto initial = std::make_shared<PublishedTreeState>(nullptr, 42); + initial->generation = 9; + initial->coverage = 42; + initial->root_page_id = 0; + const auto input = cold(1); + std::vector<PageReplacement> edits; + edits.reserve(71); + for (uint64_t i = 0; i < 70; ++i) { + edits.push_back({.page_id = i * 64, .expected = {}, .replacement = published({input})}); + } + const uint64_t last = (uint64_t{1} << 29) + 3; + edits.push_back({.page_id = last, .expected = {}, .replacement = published({input, input})}); + MappingView::CopyStats copied; + initial->mapping = initial->mapping.replace(edits, copied); + initial->next_page_id = last + 1; + for (const auto &edit : edits) { + ASSERT_TRUE(apply_page_metrics(initial->stats, nullptr, &edit.replacement->metrics)); + } + const auto baseline = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + { + PublicationCoordinator publisher( + 8, 9, {}, std::pmr::get_default_resource(), cache, + {.metadata = credits.credits.bytes, .initial = initial, .source_catalog = sources.catalog()}); + runtime.drain(); + auto view = publisher.capture(); + EXPECT_EQ(view->coverage, 42U); + EXPECT_EQ(view->generation, 9U); + EXPECT_EQ(view->commit_seq, 0U); + EXPECT_EQ(view->mapping.get(last), edits.back().replacement); + EXPECT_TRUE(std::ranges::equal(view->sources->tables, sources.catalog()->tables)); + EXPECT_EQ(view->stats.page_count(), edits.size()); + EXPECT_EQ(resources->snapshot().proxy_pages, edits.size()); + EXPECT_LE(credits.budget.snapshot().allocated[LoadCredits::kBytes] - baseline, + PublicationCoordinator::allocation_bound(1, 64, &initial->mapping, true).value_or(0)); + auto upstream = std::make_shared<TestLoader>(); + upstream->frames[1] = base({ + {.key = "a", .value = "value"} + }); + auto loader = cache->bind(upstream, credits.credits.bytes); + loader->load_leaf(input, [](Status status, auto frame) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(frame); + }); + runtime.drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, 0U); + cache->evict(); + runtime.drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, edits.size()); + } + runtime.drain(); + EXPECT_EQ(resources->snapshot().proxy_pages, 0U); + EXPECT_EQ(resources->snapshot().resident_frames, 0U); +} + +TEST(PublicationBootstrap, FailedInitialPublicationDoesNotClaimCacheOrLeaveAllocations) +{ + LoadCredits credits; + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + auto resources = std::make_shared<TreeResourceMetrics>(); + auto cache = std::make_shared<SharedPageCache>(PageCacheLimits{}, nullptr, resources, &runtime, lane); + SourceAdmission wrong(7, 1); + const auto initial = std::make_shared<PublishedTreeState>(); + EXPECT_THROW((PublicationCoordinator( + 8, 1, {}, std::pmr::get_default_resource(), cache, + {.metadata = credits.credits.bytes, .initial = initial, .source_catalog = wrong.catalog()})), + std::invalid_argument); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + SourceAdmission matching; + { + PublicationCoordinator publisher( + 8, 1, {}, std::pmr::get_default_resource(), cache, + {.metadata = credits.credits.bytes, .initial = initial, .source_catalog = matching.catalog()}); + EXPECT_TRUE(std::ranges::equal(publisher.capture()->sources->tables, matching.catalog()->tables)); + } + runtime.drain(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PublicationBootstrap, ExhaustedLifecycleRejectsNewThreadBeforeWriterAdmission) +{ + LoadCredits credits; + SourceAdmission sources(0, 1, {}, credits.credits.bytes); + const auto catalog = sources.catalog(); + credits.seal(); + Status status; + std::jthread caller([&] { + SourceAdmission::Writer writer; + status = sources.try_admit(1, &writer); + EXPECT_FALSE(writer.table()); + }); + caller.join(); + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_EQ(sources.catalog(), catalog); +} + +TEST(PublicationBootstrap, RejectedControlAllocationsReturnAllPartialCharges) +{ + LoadCredits credits; + auto token = credits.credits.bytes->take_reserved(16); + if (!token) { + FAIL() << "tiny control reservation unavailable"; + } + auto tiny = std::make_shared<detail::AllocationReservation>(std::move(*token)); + EXPECT_THROW((SourceAdmission(0, 1, {}, tiny)), std::bad_alloc); + EXPECT_THROW((PublicationCoordinator(8, 1, {}, std::pmr::get_default_resource(), {}, + {.metadata = tiny, .initial = {}, .source_catalog = {}})), + std::bad_alloc); + SourceAdmission sources; + PublicationCoordinator publisher; + detail::FlushRuntime runtime(1, 4); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(4); + std::atomic<uint64_t> frontier{0}; + EXPECT_THROW((FlushController(publisher, runtime, sources, frontier, lane, {}, {}, tiny)), std::bad_alloc); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_checkpoint_capacity_test.cpp b/lib/crowdb-tree/tests/unit/publication_checkpoint_capacity_test.cpp new file mode 100644 index 000000000..ee523ba7e --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_checkpoint_capacity_test.cpp @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <future> +#include <latch> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<PublicationRequest> request(uint64_t sequence) +{ + auto value = std::make_shared<PublicationRequest>(); + value->transaction.record = + std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 1, .range = 0, .sequence = sequence}); + return value; +} + +PublicationResult submit(PublicationCoordinator &publisher, const std::shared_ptr<PublicationRequest> &value) +{ + std::promise<PublicationResult> completed; + auto future = completed.get_future(); + value->on_done = [&](PublicationResult result) { completed.set_value(std::move(result)); }; + if (!publisher.try_submit(value)) { + ADD_FAILURE() << "publication rejected"; + return {.status = Status::unavailable()}; + } + return future.get(); +} + +TEST(PublicationCheckpointCapacity, CompletionHasIndependentCapacityAndSurvivesClose) +{ + PublicationCoordinator publisher(1); + auto capture = request(0); + capture->transaction.expected_coverage = 0; + capture->transaction.checkpoint = {.action = CheckpointAction::Capture, .token = 7}; + ASSERT_TRUE(submit(publisher, capture).status.ok()); + + std::latch entered{1}; + std::latch release{1}; + auto blocked = request(1); + blocked->on_done = [&](PublicationResult result) { + EXPECT_TRUE(result.status.ok()); + entered.count_down(); + release.wait(); + }; + std::thread first([&] { EXPECT_TRUE(publisher.try_submit(blocked)); }); + entered.wait(); + auto queued = request(2); + EXPECT_TRUE(publisher.try_submit(queued)); + EXPECT_FALSE(publisher.try_submit(request(3))); + publisher.close(); + + auto finish = request(4); + finish->transaction.checkpoint = {.action = CheckpointAction::Commit, .token = 7}; + std::promise<PublicationResult> completed; + auto outcome = completed.get_future(); + finish->on_done = [&](PublicationResult result) { completed.set_value(std::move(result)); }; + EXPECT_TRUE(publisher.try_submit_checkpoint_completion(finish)); + auto duplicate = request(5); + duplicate->transaction.checkpoint = {.action = CheckpointAction::Abort, .token = 7}; + EXPECT_FALSE(publisher.try_submit_checkpoint_completion(duplicate)); + release.count_down(); + first.join(); + EXPECT_TRUE(outcome.get().status.ok()); + EXPECT_FALSE(publisher.capture()->checkpoint_inflight); + EXPECT_FALSE(publisher.try_submit_checkpoint_completion(duplicate)); +} + +TEST(PublicationCheckpointCapacity, CaptureReservesSharedPidAuthorityWithoutLogicalMutation) +{ + PublicationCoordinator publisher; + const auto before = publisher.capture(); + auto capture = request(0); + capture->transaction.expected_coverage = 0; + capture->transaction.expected_commit_seq = before->commit_seq; + capture->transaction.checkpoint = {.action = CheckpointAction::Capture, .token = 1}; + capture->transaction.expected_next_page_id = before->next_page_id; + capture->transaction.next_page_id = before->next_page_id + 10; + const auto outcome = submit(publisher, capture); + ASSERT_TRUE(outcome.status.ok()) << outcome.status.to_string(); + ASSERT_TRUE(outcome.checkpoint_view); + EXPECT_EQ(publisher.capture()->next_page_id, before->next_page_id + 10); + EXPECT_EQ(outcome.checkpoint_view->next_page_id, before->next_page_id + 10); + EXPECT_EQ(publisher.capture()->stats.leaf_pages, before->stats.leaf_pages); + EXPECT_EQ(publisher.capture()->stats.checkpoint_mapping_pending, before->stats.checkpoint_mapping_pending); + auto stale = request(1); + stale->transaction.expected_next_page_id = before->next_page_id; + EXPECT_EQ(submit(publisher, stale).status.code(), Code::kUnavailable); + auto backwards = request(2); + backwards->transaction.expected_next_page_id = before->next_page_id + 10; + backwards->transaction.next_page_id = before->next_page_id; + EXPECT_EQ(submit(publisher, backwards).status.code(), Code::kInvalidArgument); + auto finish = request(3); + finish->transaction.checkpoint = {.action = CheckpointAction::Commit, .token = 1}; + ASSERT_TRUE(submit(publisher, finish).status.ok()); + EXPECT_FALSE(publisher.capture()->stats.checkpoint_mapping_pending); + EXPECT_EQ(publisher.capture()->next_page_id, before->next_page_id + 10); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_checkpoint_failure_test.cpp b/lib/crowdb-tree/tests/unit/publication_checkpoint_failure_test.cpp new file mode 100644 index 000000000..15ac567d0 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_checkpoint_failure_test.cpp @@ -0,0 +1,235 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <algorithm> +#include <functional> +#include <future> +#include <memory_resource> + +namespace crowdb::tree +{ +namespace +{ +class CheckpointResource : public std::pmr::memory_resource +{ + public: + size_t allocations = 0; + size_t deallocations = 0; + size_t fail_at = 0; + size_t largest_allocation = 0; + std::function<void()> before_allocate; + + private: + void *do_allocate(size_t bytes, size_t alignment) override + { + largest_allocation = std::max(largest_allocation, bytes); + if (before_allocate) { + before_allocate(); + } + if (++allocations == fail_at) { + throw std::bad_alloc(); + } + return std::pmr::new_delete_resource()->allocate(bytes, alignment); + } + + void do_deallocate(void *pointer, size_t bytes, size_t alignment) override + { + ++deallocations; + std::pmr::new_delete_resource()->deallocate(pointer, bytes, alignment); + } + + [[nodiscard]] bool do_is_equal(const std::pmr::memory_resource &other) const noexcept override + { + return this == &other; + } +}; + +std::shared_ptr<InstallRecord> pending_record(uint64_t sequence) +{ + return std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 1, .range = 0, .sequence = sequence}); +} + +std::shared_ptr<PublishedPage> pending_leaf() +{ + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + return page; +} + +PublicationResult pending_publish(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> completed; + auto result = completed.get_future(); + request->on_done = [&](PublicationResult value) { completed.set_value(std::move(value)); }; + if (!publisher.try_submit(request)) { + ADD_FAILURE() << "checkpoint test publication rejected"; + return {.status = Status::unavailable()}; + } + return result.get(); +} + +PublicationResult pending_checkpoint(PublicationCoordinator &publisher, CheckpointAction action, uint64_t token) +{ + PublicationTransaction transaction; + transaction.record = pending_record(token); + transaction.checkpoint = {.action = action, .token = token}; + if (action == CheckpointAction::Capture) { + transaction.expected_coverage = publisher.capture()->coverage; + } + return pending_publish(publisher, std::move(transaction)); +} + +TEST(PublicationCheckpointFailure, EveryMultiPageAllocationFailureRollsBackMembershipsAndDeletion) +{ + size_t failures = 0; + for (size_t offset = 1; offset <= 8; ++offset) { + CheckpointResource resource; + PublicationCoordinator publisher(8, 1, {}, &resource); + auto original = pending_leaf(); + ASSERT_TRUE(pending_publish(publisher, {.record = pending_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = original}}, + .root_page_id = 0}) + .status.ok()); + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + const auto before = publisher.capture(); + auto record = pending_record(2); + resource.fail_at = resource.allocations + offset; + auto result = + pending_publish(publisher, { + .record = record, + .pages = {{.page_id = 0, .expected = original, .replacement = {}}, + {.page_id = 1, .expected = {}, .replacement = pending_leaf()}, + {.page_id = 2, .expected = {}, .replacement = pending_leaf()}}, + .root_page_id = 1 + }); + resource.fail_at = 0; + if (!result.status.ok()) { + ++failures; + EXPECT_EQ(result.status.code(), Code::kResourceExhausted); + EXPECT_EQ(publisher.capture(), before); + EXPECT_FALSE(record->committed.load()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 1U); + } + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, result.status.ok() ? 2U : 0U); + EXPECT_EQ(publisher.capture()->stats.checkpoint_mapping_pending, result.status.ok()); + } + EXPECT_GE(failures, 4U); +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(PublicationCheckpointFailure, FinalRootAllocationFailureRollsBackAlreadyPreparedLedger) +{ + detail::ResourceBudget::Limits limits{}; + const auto category = static_cast<size_t>(detail::BudgetClass::kHandoff); + limits[category] = 1024 * 1024; + detail::ResourceBudget budget(limits, 4); + { + CheckpointResource resource; + PublicationCoordinator publisher(8, 1, {}, &resource); + auto original = pending_leaf(); + ASSERT_TRUE(pending_publish(publisher, {.record = pending_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = original}}, + .root_page_id = 0}) + .status.ok()); + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + auto credit = detail::ResourceBudget::try_reserve(budget.make_account(limits), detail::BudgetClass::kHandoff, + limits[category]); + if (!credit) { + FAIL() << "checkpoint metadata reservation was rejected"; + } + auto reservation = std::make_shared<detail::AllocationReservation>(std::move(*credit)); + bool reached_pending = false; + bool reached_root = false; + resource.before_allocate = [&] { reached_pending = true; }; + publisher.set_before_root_for_tests([&] { + reached_root = true; + reservation->seal(); + }); + const auto before = publisher.capture(); + auto record = pending_record(2); + const auto result = + pending_publish(publisher, {.record = record, + .pages = {{.page_id = 1, .expected = {}, .replacement = pending_leaf()}}, + .metadata = reservation}); + publisher.set_before_root_for_tests({}); + resource.before_allocate = {}; + EXPECT_TRUE(reached_pending); + EXPECT_TRUE(reached_root); + EXPECT_EQ(result.status.code(), Code::kResourceExhausted); + EXPECT_EQ(publisher.capture(), before); + EXPECT_FALSE(record->committed.load()); + // Node-based membership has no retained hash capacity. Rollback frees + // every allocation made by the failed delta, including both sets. + EXPECT_EQ(budget.snapshot().allocated[category], 0U); + EXPECT_EQ(budget.snapshot().reserved[category], 0U); + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); + EXPECT_FALSE(publisher.capture()->stats.checkpoint_mapping_pending); + } + EXPECT_EQ(budget.snapshot().allocated[category], 0U); +} +#endif + +TEST(PublicationCheckpointFailure, CapturedSetNodesAreReleasedOnlyByExplicitReclaim) +{ + CheckpointResource resource; + PublicationCoordinator publisher(8, 1, {}, &resource); + ASSERT_TRUE(pending_publish(publisher, {.record = pending_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = pending_leaf()}}, + .root_page_id = 0}) + .status.ok()); + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + const auto frees = resource.deallocations; + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Capture, 2).status.ok()); + EXPECT_EQ(resource.deallocations, frees); + publisher.reclaim(); + EXPECT_GT(resource.deallocations, frees); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); +} + +TEST(PublicationCheckpointFailure, CapturePreparationFailureDoesNotConsumeItsToken) +{ + CheckpointResource resource; + PublicationCoordinator publisher(8, 1, {}, &resource); + const auto before = publisher.capture(); + resource.fail_at = resource.allocations + 1; + EXPECT_EQ(pending_checkpoint(publisher, CheckpointAction::Capture, 1).status.code(), Code::kResourceExhausted); + EXPECT_EQ(publisher.capture(), before); + resource.fail_at = 0; + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + EXPECT_TRUE(publisher.capture()->checkpoint_inflight); + ASSERT_TRUE(pending_checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); +} + +TEST(PublicationCheckpointFailure, OneChangedPageNeverAllocatesTreeSizedMembershipCapacity) +{ + CheckpointResource resource; + CheckpointPendingMetrics pending(&resource); + for (uint64_t pid = 0; pid < 4096; ++pid) { + const auto before = resource.allocations; + pending.change_page(pid); + EXPECT_EQ(resource.allocations - before, 2U); + } + EXPECT_LE(resource.largest_allocation, 256U); + ASSERT_TRUE(pending.capture(1)); + for (uint64_t pid = 4096; pid < 8192; ++pid) { + const auto before = resource.allocations; + pending.change_page(pid); + EXPECT_EQ(resource.allocations - before, 3U); + } + EXPECT_LE(resource.largest_allocation, 256U); + ASSERT_TRUE(pending.finish(1, true)); + TreeLogicalStats stats; + pending.copy_to(stats); + EXPECT_EQ(stats.checkpoint_pending_pages, 4096U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_checkpoint_test.cpp b/lib/crowdb-tree/tests/unit/publication_checkpoint_test.cpp new file mode 100644 index 000000000..fb404a603 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_checkpoint_test.cpp @@ -0,0 +1,294 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <future> +#include <memory_resource> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<InstallRecord> checkpoint_record(uint64_t sequence) +{ + return std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 1, .range = 0, .sequence = sequence}); +} + +std::shared_ptr<PublishedPage> checkpoint_leaf(uint64_t content = 1, uint64_t route = 1, uint64_t representation = 1) +{ + auto state = std::make_shared<PageState>(); + state->content_id = content; + state->route_generation = route; + state->representation_id = representation; + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::move(state); + page->metrics.dirty = true; + return page; +} + +PublicationResult publish_checkpoint(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> completed; + auto result = completed.get_future(); + request->on_done = [&](PublicationResult value) { completed.set_value(std::move(value)); }; + if (!publisher.try_submit(request)) { + ADD_FAILURE() << "checkpoint publication was rejected"; + return {.status = Status::unavailable()}; + } + return result.get(); +} + +PublicationResult checkpoint(PublicationCoordinator &publisher, CheckpointAction action, uint64_t token) +{ + PublicationTransaction transaction; + transaction.record = checkpoint_record(token); + transaction.checkpoint = {.action = action, .token = token}; + if (action == CheckpointAction::Capture) { + transaction.expected_coverage = publisher.capture()->coverage; + } + return publish_checkpoint(publisher, std::move(transaction)); +} + +void seed_checkpoint(PublicationCoordinator &publisher, const std::shared_ptr<PublishedPage> &page) +{ + ASSERT_TRUE(publish_checkpoint(publisher, {.record = checkpoint_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0}) + .status.ok()); +} + +TEST(PublicationCheckpoint, CaptureArtifactReleasesL0AndRetryReturnsTheSamePinnedView) +{ + PublicationCoordinator publisher; + auto table = std::make_shared<MemTable>(); + std::weak_ptr<MemTable> retained = table; + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 3; + sources->tables.push_back(table); + auto page = checkpoint_leaf(); + ASSERT_TRUE(publish_checkpoint(publisher, {.record = checkpoint_record(0), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0, + .coverage = 3, + .expected_sources = publisher.capture()->sources, + .sources = sources}) + .status.ok()); + PublicationTransaction capture{ + .record = checkpoint_record(1), + .expected_coverage = 3, + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = 10} + }; + auto first = publish_checkpoint(publisher, capture); + ASSERT_TRUE(first.status.ok()); + ASSERT_TRUE(first.checkpoint_view); + EXPECT_EQ(first.checkpoint_view->mapping.get(0), page); + EXPECT_TRUE(first.checkpoint_view->sources->tables.empty()); + EXPECT_EQ(first.checkpoint_view->sources->floor, 3U); + EXPECT_EQ(first.checkpoint_view->coverage, 3U); + auto detached = std::make_shared<PublishedSources>(); + detached->floor = 3; + ASSERT_TRUE(publish_checkpoint(publisher, + {.record = checkpoint_record(2), .expected_sources = sources, .sources = detached}) + .status.ok()); + sources.reset(); + table.reset(); + publisher.reclaim(); + EXPECT_TRUE(retained.expired()); + auto retry = publish_checkpoint(publisher, capture); + EXPECT_TRUE(retry.already_committed); + EXPECT_EQ(retry.checkpoint_view, first.checkpoint_view); + EXPECT_EQ(retry.commit_seq, first.commit_seq); +} + +TEST(PublicationCheckpoint, L1FenceAllowsNewSourcesAndRejectsChangedPages) +{ + PublicationCoordinator publisher; + auto first = checkpoint_leaf(); + seed_checkpoint(publisher, first); + const auto before = publisher.capture(); + auto sources = std::make_shared<PublishedSources>(); + sources->tables.push_back(std::make_shared<MemTable>()); + const auto catalog = publish_checkpoint( + publisher, {.record = checkpoint_record(1), .expected_sources = before->sources, .sources = sources}); + ASSERT_TRUE(catalog.status.ok()); + EXPECT_GT(catalog.commit_seq, before->commit_seq); + EXPECT_EQ(catalog.l1_seq, before->l1_seq); + PublicationTransaction change{.record = checkpoint_record(2), + .pages = {{.page_id = 0, .expected = first, .replacement = checkpoint_leaf(2)}}, + .expected_l1_seq = before->l1_seq}; + const auto committed = publish_checkpoint(publisher, change); + ASSERT_TRUE(committed.status.ok()); + EXPECT_EQ(committed.l1_seq, before->l1_seq + 1); + const auto retry = publish_checkpoint(publisher, change); + EXPECT_TRUE(retry.already_committed); + EXPECT_EQ(retry.l1_seq, committed.l1_seq); + EXPECT_EQ(publish_checkpoint(publisher, {.record = checkpoint_record(3), .expected_l1_seq = before->l1_seq}) + .status.code(), + Code::kUnavailable); +} + +TEST(PublicationCheckpoint, ExhaustedL1SequenceStillAllowsSourcesButRejectsGraphMutation) +{ + auto initial = std::make_shared<PublishedTreeState>(); + initial->l1_seq = UINT64_MAX; + PublicationCoordinator publisher{ + 4, 1, {}, + std::pmr::get_default_resource(), {}, + {.metadata = {}, .initial = initial, .source_catalog = {}} + }; + auto sources = std::make_shared<PublishedSources>(); + sources->tables.push_back(std::make_shared<MemTable>()); + ASSERT_TRUE( + publish_checkpoint(publisher, + {.record = checkpoint_record(1), .expected_sources = initial->sources, .sources = sources}) + .status.ok()); + auto first = checkpoint_leaf(); + const auto rejected = + publish_checkpoint(publisher, {.record = checkpoint_record(2), + .pages = {{.page_id = 0, .expected = {}, .replacement = first}}, + .root_page_id = 0}); + EXPECT_EQ(rejected.status.code(), Code::kResourceExhausted); + EXPECT_EQ(publisher.capture()->l1_seq, UINT64_MAX); + EXPECT_EQ(publisher.capture()->sources, sources); + EXPECT_TRUE(publisher.capture()->mapping.empty()); +} + +TEST(PublicationCheckpoint, CommitRetainsLaterChangesAndRejectsWrongTokens) +{ + PublicationCoordinator publisher; + auto first = checkpoint_leaf(); + seed_checkpoint(publisher, first); + auto captured = checkpoint(publisher, CheckpointAction::Capture, 10); + ASSERT_TRUE(captured.status.ok()); + auto next = checkpoint_leaf(2); + ASSERT_TRUE(publish_checkpoint(publisher, {.record = checkpoint_record(1), + .pages = {{.page_id = 0, .expected = first, .replacement = next}}}) + .status.ok()); + const auto before_stale = publisher.capture(); + EXPECT_EQ(checkpoint(publisher, CheckpointAction::Commit, 9).status.code(), Code::kUnavailable); + EXPECT_EQ(checkpoint(publisher, CheckpointAction::Capture, 11).status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture(), before_stale); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 10).status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 1U); + EXPECT_TRUE(publisher.capture()->stats.checkpoint_mapping_pending); + EXPECT_EQ(captured.checkpoint_view->mapping.get(0), first); + EXPECT_EQ(checkpoint(publisher, CheckpointAction::Commit, 10).status.code(), Code::kUnavailable); + EXPECT_EQ(checkpoint(publisher, CheckpointAction::Capture, 10).status.code(), Code::kUnavailable); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 11).status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 11).status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); + EXPECT_FALSE(publisher.capture()->stats.checkpoint_mapping_pending); + EXPECT_EQ(publisher.capture()->stats.dirty_pages, 1U); + EXPECT_FALSE(publisher.capture()->checkpoint_inflight); +} + +TEST(PublicationCheckpoint, RepresentationAndCleanWritebackDoNotCreateLogicalChanges) +{ + PublicationCoordinator publisher; + auto first = checkpoint_leaf(); + seed_checkpoint(publisher, first); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + auto clean = checkpoint_leaf(1, 1, 2); + clean->metrics.dirty = false; + ASSERT_TRUE(publish_checkpoint(publisher, {.record = checkpoint_record(2), + .pages = {{.page_id = 0, .expected = first, .replacement = clean}}}) + .status.ok()); + EXPECT_EQ(publisher.capture()->stats.dirty_pages, 0U); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 1U); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); + auto loaded = checkpoint_leaf(1, 1, 3); + loaded->metrics.dirty = false; + ASSERT_TRUE(publish_checkpoint(publisher, {.record = checkpoint_record(3), + .pages = {{.page_id = 0, .expected = clean, .replacement = loaded}}}) + .status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); + EXPECT_FALSE(publisher.capture()->stats.checkpoint_mapping_pending); + auto routed = checkpoint_leaf(1, 2, 4); + ASSERT_TRUE(publish_checkpoint(publisher, {.record = checkpoint_record(4), + .pages = {{.page_id = 0, .expected = loaded, .replacement = routed}}}) + .status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 1U); +} + +TEST(PublicationCheckpoint, DeleteAndCoverageOnlyAdvanceKeepAnchorPending) +{ + PublicationCoordinator publisher; + auto first = checkpoint_leaf(); + seed_checkpoint(publisher, first); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + ASSERT_TRUE(publish_checkpoint(publisher, {.record = checkpoint_record(2), + .pages = {{.page_id = 0, .expected = first, .replacement = {}}}, + .root_page_id = UINT64_MAX}) + .status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); + EXPECT_TRUE(publisher.capture()->stats.checkpoint_mapping_pending); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 2).status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 2).status.ok()); + EXPECT_FALSE(publisher.capture()->stats.checkpoint_mapping_pending); + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 9; + ASSERT_TRUE(publish_checkpoint(publisher, {.record = checkpoint_record(3), + .coverage = 9, + .expected_sources = publisher.capture()->sources, + .sources = sources}) + .status.ok()); + EXPECT_TRUE(publisher.capture()->stats.checkpoint_mapping_pending); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); + const auto before = publisher.capture(); + EXPECT_EQ( + publish_checkpoint(publisher, + { + .record = checkpoint_record(4), + .expected_coverage = 8, + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = 3} + }) + .status.code(), + Code::kUnavailable); + EXPECT_EQ(publisher.capture(), before); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 3).status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Abort, 3).status.ok()); + EXPECT_TRUE(publisher.capture()->stats.checkpoint_mapping_pending); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 4).status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 4).status.ok()); + EXPECT_FALSE(publisher.capture()->stats.checkpoint_mapping_pending); + const auto totals = publisher.work_metrics()->totals(); + // Publication-only bookkeeping has no durable I/O pipeline to time. + EXPECT_EQ(totals[static_cast<size_t>(TreeStage::SnapshotCommit)].attempts, 0U); +} + +TEST(PublicationCheckpoint, CaptureRejectsMixedMutationAndMissingHandoffFrontier) +{ + PublicationCoordinator publisher; + auto page = checkpoint_leaf(); + const auto before = publisher.capture(); + EXPECT_EQ( + publish_checkpoint(publisher, + { + .record = checkpoint_record(1), + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = 1} + }) + .status.code(), + Code::kInvalidArgument); + EXPECT_EQ( + publish_checkpoint(publisher, + { + .record = checkpoint_record(2), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0, + .expected_coverage = 0, + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = 1} + }) + .status.code(), + Code::kInvalidArgument); + EXPECT_EQ(publisher.capture(), before); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_gc_floor_test.cpp b/lib/crowdb-tree/tests/unit/publication_gc_floor_test.cpp new file mode 100644 index 000000000..0987a68d5 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_gc_floor_test.cpp @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_state.h" +#include "btree/tree_core.h" +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +PublicationResult publish_floor(PublicationCoordinator &publisher, uint64_t sequence, + std::optional<uint64_t> coverage = {}, + std::optional<CheckpointPublication> checkpoint = {}) +{ + const auto current = publisher.capture(); + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>( + InstallToken{.generation = current->generation, .batch = 1, .sequence = sequence}); + request->transaction.expected_coverage = current->coverage; + if (coverage) { + auto sources = std::make_shared<PublishedSources>(); + sources->floor = *coverage; + request->transaction.coverage = coverage; + request->transaction.expected_sources = current->sources; + request->transaction.sources = std::move(sources); + } + request->transaction.checkpoint = checkpoint; + PublicationResult result; + request->on_done = [&](PublicationResult value) { result = std::move(value); }; + EXPECT_TRUE(publisher.try_submit(request)); + return result; +} + +TEST(PublicationGcFloor, ClampsToCoverageAndLeavesCapturedViewsUnchanged) +{ + PublicationCoordinator publisher; + const auto initial = publisher.capture(); + publisher.request_gc_floor(4); + ASSERT_TRUE(publish_floor(publisher, 1, 2).status.ok()); + const auto captured = publisher.capture(); + EXPECT_EQ(captured->gc_floor, 2U); + publisher.request_gc_floor(1); + ASSERT_TRUE(publish_floor(publisher, 2, 6).status.ok()); + EXPECT_EQ(publisher.capture()->gc_floor, 4U); + publisher.request_gc_floor(10); + ASSERT_TRUE(publish_floor(publisher, 3).status.ok()); + EXPECT_EQ(publisher.capture()->gc_floor, 6U); + EXPECT_EQ(captured->gc_floor, 2U); + EXPECT_EQ(initial->gc_floor, 0U); +} + +TEST(PublicationGcFloor, FloorOnlyCommitDoesNotCreatePendingLogicalChanges) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(publish_floor(publisher, 1, 10).status.ok()); + ASSERT_TRUE(publish_floor(publisher, 2, {}, CheckpointPublication{.action = CheckpointAction::Capture, .token = 1}) + .status.ok()); + ASSERT_TRUE(publish_floor(publisher, 3, {}, CheckpointPublication{.action = CheckpointAction::Commit, .token = 1}) + .status.ok()); + const auto before = publisher.capture(); + publisher.request_gc_floor(8); + ASSERT_TRUE(publish_floor(publisher, 4).status.ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->gc_floor, 8U); + EXPECT_EQ(after->stats.checkpoint_pending_pages, before->stats.checkpoint_pending_pages); + EXPECT_EQ(after->stats.checkpoint_mapping_pending, before->stats.checkpoint_mapping_pending); + publisher.request_gc_floor(9); + const auto captured = + publish_floor(publisher, 5, {}, CheckpointPublication{.action = CheckpointAction::Capture, .token = 2}); + ASSERT_TRUE(captured.status.ok()); + ASSERT_TRUE(captured.checkpoint_view); + EXPECT_EQ(captured.checkpoint_view->gc_floor, 9U); + EXPECT_EQ(after->gc_floor, 8U); +} + +TEST(PublicationGcFloor, PublicWatermarkUsesBothLimitsAtFiniteHandoff) +{ + Config config; + TreeRuntimeOptions runtime; + runtime.workers = 1; + ASSERT_TRUE(TreeRuntime::create(runtime, &config.runtime).ok()); + Crowdbtree tree(config); + ASSERT_TRUE(tree.apply(1, {}).ok()); + ASSERT_TRUE(tree.apply(2, {}).ok()); + auto *const core = TreeCoreAccess::get(tree); + const auto old = core->capture(); + tree.set_gc_watermark(10, 1); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_EQ(core->capture()->coverage, 2U); + EXPECT_EQ(core->capture()->gc_floor, 1U); + EXPECT_EQ(old->gc_floor, 0U); + tree.set_gc_watermark(1, 10); + EXPECT_EQ(tree.gc_watermark(), 1U); + tree.set_gc_watermark(10, 10); + ASSERT_TRUE(tree.apply(3, {}).ok()); + ASSERT_TRUE(tree.flush().ok()); + EXPECT_EQ(core->capture()->gc_floor, 3U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_generation_test.cpp b/lib/crowdb-tree/tests/unit/publication_generation_test.cpp new file mode 100644 index 000000000..d44901fd2 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_generation_test.cpp @@ -0,0 +1,97 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_admission.h" +#include "page_cache_residency_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_cache_residency; + +std::shared_ptr<PublicationCoordinator::Replacement> replacement(TestCache &fixture, size_t pages, const BaseRef &base) +{ + auto candidate = std::make_shared<PublishedTreeState>(); + candidate->generation = fixture.publisher.capture()->generation + 1; + candidate->root_page_id = 0; + candidate->next_page_id = pages; + PublicationTransaction::Pages edits; + for (size_t i = 0; i < pages; ++i) { + edits.push_back({.page_id = i, .expected = {}, .replacement = published({base})}); + } + MappingView::CopyStats copied; + candidate->mapping = candidate->mapping.replace(edits, copied); + SourceAdmission sources(0, candidate->generation); + std::shared_ptr<PublicationCoordinator::Replacement> result; + EXPECT_TRUE( + fixture.publisher + .prepare_replacement({.metadata = {}, .initial = candidate, .source_catalog = sources.catalog()}, &result) + .ok()); + return result; +} + +TEST(PublicationGeneration, NewNamespaceClearsOldMembershipWithoutDroppingIndependentReadyFrames) +{ + TestCache fixture; + fixture.frame(1); + const auto base = cold(1); + auto old_page = published({base}); + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = old_page} + }, + 0) + .status.ok()); + auto old_view = fixture.publisher.capture(); + auto pin = fixture.load(base); + ASSERT_TRUE(pin); + auto prepared = replacement(fixture, 97, cold(2)); + ASSERT_TRUE(prepared); + fixture.publisher.request_gc_floor(100); + fixture.publisher.install_replacement(std::move(prepared)); + fixture.runtime.drain(); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 97U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 1U); + EXPECT_EQ(old_view->mapping.get(0), old_page); + EXPECT_EQ(fixture.publisher.requested_gc_floor(), 0U); + auto current = fixture.publisher.capture(); + EXPECT_EQ(current->generation, 2U); + EXPECT_GT(current->commit_seq, old_view->commit_seq); + EXPECT_EQ(current->stats.checkpoint_pending_pages, 97U); + EXPECT_TRUE(current->stats.checkpoint_mapping_pending); + fixture.publisher.install_replacement(replacement(fixture, 2, base)); + fixture.runtime.drain(); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 0U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 1U); + EXPECT_EQ(fixture.publisher.capture()->stats.checkpoint_pending_pages, 2U); + fixture.cache->evict(); + fixture.runtime.drain(); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 2U); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 1U); + pin.reset(); + EXPECT_EQ(fixture.resources->snapshot().resident_frames, 0U); +} + +TEST(PublicationGeneration, DiscardedPreparationLeavesTheOldCheckpointLedgerAndRootUnchanged) +{ + TestCache fixture; + ASSERT_TRUE(fixture + .publish( + { + {.page_id = 0, .expected = {}, .replacement = published({cold(1)})} + }, + 0) + .status.ok()); + auto original = fixture.publisher.capture(); + auto prepared = replacement(fixture, 64, cold(2)); + ASSERT_TRUE(prepared); + prepared.reset(); + fixture.runtime.drain(); + EXPECT_EQ(fixture.publisher.capture(), original); + EXPECT_EQ(fixture.resources->snapshot().proxy_pages, 1U); + EXPECT_EQ(fixture.publisher.capture()->stats.checkpoint_pending_pages, 1U); + fixture.publisher.reclaim(); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_handoff_test.cpp b/lib/crowdb-tree/tests/unit/publication_handoff_test.cpp new file mode 100644 index 000000000..660e8801c --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_handoff_test.cpp @@ -0,0 +1,169 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<InstallRecord> record(uint64_t batch, uint64_t sequence) +{ + return std::make_shared<InstallRecord>(InstallToken{.batch = batch, .sequence = sequence}); +} + +PublicationResult publish(PublicationCoordinator &publisher, PublicationTransaction transaction) +{ + PublicationResult result; + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + request->on_done = [&](PublicationResult value) { result = std::move(value); }; + EXPECT_TRUE(publisher.try_submit(request)); + return result; +} + +std::shared_ptr<PublishedPage> seed(PublicationCoordinator &publisher) +{ + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + EXPECT_TRUE(publish(publisher, {.record = record(0, 0), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0}) + .status.ok()); + return page; +} + +PublicationTransaction capture(uint64_t token) +{ + return { + .record = record(9, token), + .expected_coverage = 0, + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = token} + }; +} + +TEST(PublicationHandoff, PartialInstallBlocksCheckpointAndForeignMutationUntilCoverage) +{ + PublicationCoordinator publisher; + auto before = seed(publisher); + auto after = std::make_shared<PublishedPage>(*before); + auto state = std::make_shared<PageState>(*before->leaf); + state->content_id = 2; + after->leaf = state; + auto installed = record(1, 0); + PublicationTransaction install{ + .record = installed, + .pages = {{.page_id = 0, .expected = before, .replacement = after}}, + .expected_coverage = 0, + .handoff = PendingHandoff{.batch = 1, .frontier = 3} + }; + ASSERT_TRUE(publish(publisher, install).status.ok()); + auto partial = publisher.capture(); + ASSERT_TRUE(partial->pending_handoff); + EXPECT_EQ(partial->coverage, 0U); + EXPECT_EQ(publish(publisher, capture(1)).status.code(), Code::kUnavailable); + EXPECT_EQ(publish(publisher, + {.record = record(2, 0), .pages = {{.page_id = 0, .expected = after, .replacement = before}}}) + .status.code(), + Code::kUnavailable); + install.record = record(2, 1); + install.handoff->batch = 2; + EXPECT_EQ(publish(publisher, install).status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture(), partial); + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 3; + auto unfinished = record(1, 1); + PublicationTransaction coverage{ + .record = record(1, 2), + .expected_coverage = 0, + .coverage = 3, + .expected_sources = partial->sources, + .sources = sources, + .required_installs = {installed, unfinished }, + .handoff = PendingHandoff{.batch = 1, .frontier = 3} + }; + EXPECT_EQ(publish(publisher, coverage).status.code(), Code::kUnavailable); + EXPECT_EQ(publisher.capture(), partial); + ASSERT_TRUE( + publish(publisher, + { + .record = unfinished, .expected_coverage = 0, .handoff = PendingHandoff{.batch = 1, .frontier = 3} + }) + .status.ok()); + ASSERT_TRUE(publish(publisher, coverage).status.ok()); + EXPECT_FALSE(publisher.capture()->pending_handoff); + EXPECT_TRUE(partial->pending_handoff); + EXPECT_EQ(partial->mapping.get(0), after); + auto checkpoint = capture(2); + checkpoint.expected_coverage = 3; + auto result = publish(publisher, checkpoint); + ASSERT_TRUE(result.status.ok()); + ASSERT_TRUE(result.checkpoint_view); + EXPECT_EQ(result.checkpoint_view->coverage, 3U); + EXPECT_EQ(result.checkpoint_view->mapping.get(0), after); +} + +TEST(PublicationHandoff, PriorCheckpointCanCommitDuringNewHandoff) +{ + PublicationCoordinator publisher; + auto page = seed(publisher); + auto captured = publish(publisher, capture(1)); + ASSERT_TRUE(captured.status.ok()); + auto changed = std::make_shared<PublishedPage>(*page); + auto state = std::make_shared<PageState>(); + state->content_id = 9; + changed->leaf = state; + ASSERT_TRUE(publish(publisher, + { + .record = record(1, 0), + .pages = {{.page_id = 0, .expected = page, .replacement = changed}}, + .handoff = PendingHandoff{.batch = 1, .frontier = 2} + }) + .status.ok()); + ASSERT_TRUE(publish(publisher, + { + .record = record(9, 2), + .checkpoint = CheckpointPublication{.action = CheckpointAction::Commit, .token = 1} + }) + .status.ok()); + EXPECT_TRUE(publisher.capture()->pending_handoff); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 1U); + EXPECT_EQ(captured.checkpoint_view->mapping.get(0), page); + EXPECT_EQ(publish(publisher, capture(2)).status.code(), Code::kUnavailable); +} + +TEST(PublicationHandoff, RejectsMismatchedBatchAndCoverageWithoutPublishing) +{ + PublicationCoordinator publisher; + seed(publisher); + const auto before = publisher.capture(); + EXPECT_EQ(publish(publisher, + { + .record = record(1, 0), .handoff = PendingHandoff{.batch = 2, .frontier = 4} + }) + .status.code(), + Code::kInvalidArgument); + EXPECT_EQ(publish(publisher, + { + .record = record(1, 0), .handoff = PendingHandoff{.batch = 1, .frontier = 0} + }) + .status.code(), + Code::kInvalidArgument); + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 3; + EXPECT_EQ(publish(publisher, + { + .record = record(1, 0), + .coverage = 3, + .expected_sources = before->sources, + .sources = sources, + .handoff = PendingHandoff{.batch = 1, .frontier = 4} + }) + .status.code(), + Code::kInvalidArgument); + EXPECT_EQ(publisher.capture(), before); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_lane_test.cpp b/lib/crowdb-tree/tests/unit/publication_lane_test.cpp new file mode 100644 index 000000000..9c7fa184b --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_lane_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/publication_lane.h" +#include "btree/published_state.h" +#include "load_budget_test_support.h" + +#include <latch> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<PublicationRequest> request(uint64_t sequence) +{ + auto value = std::make_shared<PublicationRequest>(); + value->transaction.record = + std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 1, .range = 0, .sequence = sequence}); + return value; +} + +TEST(PublicationLane, PrepaidWindowProgressesWhenOrdinaryQueueIsFullAndReturnsBeforeCallback) +{ + test_read::LoadCredits credits; + PublicationCoordinator publisher(2); + std::shared_ptr<PublicationLane> lane; + ASSERT_TRUE(publisher.prepare_lane(2, credits.credits.bytes, &lane).ok()); + std::latch entered(1); + std::latch release(1); + auto blocker = request(1); + blocker->on_done = [&](auto result) { + EXPECT_TRUE(result.status.ok()); + entered.count_down(); + release.wait(); + }; + std::jthread worker([&] { EXPECT_TRUE(publisher.try_submit(blocker)); }); + entered.wait(); + EXPECT_TRUE(publisher.try_submit(request(2))); + EXPECT_TRUE(publisher.try_submit(request(3))); + EXPECT_FALSE(publisher.try_submit(request(4))); + auto first = request(5); + auto second = request(6); + auto reentrant = request(7); + std::atomic<size_t> completed{0}; + reentrant->on_done = [&](auto result) { + EXPECT_TRUE(result.status.ok()); + completed.fetch_add(1); + }; + first->on_done = [&](auto result) { + EXPECT_TRUE(result.status.ok()); + EXPECT_TRUE(publisher.try_submit(reentrant, lane)); + completed.fetch_add(1); + }; + second->on_done = [&](auto result) { + EXPECT_TRUE(result.status.ok()); + completed.fetch_add(1); + }; + EXPECT_TRUE(publisher.try_submit(first, lane)); + EXPECT_TRUE(publisher.try_submit(second, lane)); + EXPECT_FALSE(publisher.try_submit(request(8), lane)); + release.count_down(); + worker.join(); + EXPECT_EQ(completed.load(), 3U); + EXPECT_TRUE(publisher.idle()); +} + +TEST(PublicationLane, WindowIsFixedUniqueAndBoundToItsPublisher) +{ + test_read::LoadCredits credits; + PublicationCoordinator publisher(2); + PublicationCoordinator other(2); + std::shared_ptr<PublicationLane> lane; + std::shared_ptr<PublicationLane> duplicate; + EXPECT_EQ(publisher.prepare_lane(3, credits.credits.bytes, &lane).code(), Code::kInvalidArgument); + ASSERT_TRUE(publisher.prepare_lane(2, credits.credits.bytes, &lane).ok()); + EXPECT_EQ(publisher.prepare_lane(1, credits.credits.bytes, &duplicate).code(), Code::kUnavailable); + EXPECT_FALSE(other.try_submit(request(1), lane)); + publisher.close(); + EXPECT_FALSE(publisher.try_submit(request(2), lane)); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_merge_pair_test.cpp b/lib/crowdb-tree/tests/unit/publication_merge_pair_test.cpp new file mode 100644 index 000000000..8494e4e97 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_merge_pair_test.cpp @@ -0,0 +1,398 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_batch.h" +#include "btree/logical_page.h" +#include "btree/maintenance_operation.h" +#include "btree/published_state.h" +#include "btree/tree_state_reconcile.h" + +#include <gtest/gtest.h> + +#include <future> + +namespace crowdb::tree +{ +namespace +{ +class PublicationMergePair : public ::testing::Test +{ + protected: + PublicationCoordinator publisher{8, 1}; + uint64_t sequence = 0; + + PublicationResult publish(PublicationTransaction transaction) + { + transaction.record = std::make_shared<InstallRecord>( + InstallToken{.generation = 1, .batch = 1, .range = 0, .sequence = ++sequence}); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> done; + auto result = done.get_future(); + request->on_done = [&](PublicationResult value) { done.set_value(std::move(value)); }; + if (!publisher.try_submit(request)) { + ADD_FAILURE() << "merge test publication rejected"; + return {.status = Status::unavailable()}; + } + return result.get(); + } + + void SetUp() override + { + PublicationTransaction transaction; + for (uint64_t pid = 1; pid <= 3; ++pid) { + auto logical = std::make_shared<PageState>(); + logical->content_id = pid; + logical->route_generation = 1; + logical->right_sibling = pid == 3 ? UINT64_MAX : pid + 1; + if (pid > 1) { + logical->range.lower = OwnedKey(pid == 2 ? "m" : "t"); + } + if (pid < 3) { + logical->range.upper = OwnedKey(pid == 1 ? "m" : "t"); + } + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::move(logical); + page->metrics.page_size_known = true; + page->metrics.page_size_bytes = kFrameHeaderSize + kFrameTrailerSize; + page->metrics.kv_known = true; + transaction.pages.push_back({.page_id = pid, .expected = {}, .replacement = page}); + } + auto inner = std::make_shared<InnerPageState>(); + inner->children = {1, 2, 3}; + inner->separators = {OwnedKey("m"), OwnedKey("t")}; + auto page = std::make_shared<PublishedPage>(); + page->inner = std::move(inner); + page->level = 1; + page->metrics.leaf = false; + transaction.pages.push_back({.page_id = 10, .expected = {}, .replacement = page}); + transaction.root_page_id = 10; + ASSERT_TRUE(publish(std::move(transaction)).status.ok()); + } + + PublicationTransaction classification(uint64_t left, bool ready = true) + { + const auto view = publisher.capture(); + const auto a = view->mapping.get(left); + const auto b = view->mapping.get(left + 1); + PublicationTransaction transaction; + transaction.merge_pair = MergePairPublication{ + .left = left, + .expected_left = a, + .expected_right = b, + .expected_parent = view->mapping.get(10), + .candidate = MergePairProof{.left = left, + .right = left + 1, + .parent = 10, + .left_content = a->leaf->content_id, + .right_content = b->leaf->content_id, + .left_route = a->leaf->route_generation, + .right_route = b->leaf->route_generation, + .ready = ready} + }; + return transaction; + } + + void reconcile(const std::shared_ptr<const PublishedTreeState> &view) + { + TreeStateReconciliation audit(view); + for (size_t steps = 0; steps < 1024 && !audit.complete(); ++steps) { + ASSERT_TRUE(audit.step(1).ok()); + } + EXPECT_TRUE(audit.complete()); + } +}; + +TEST_F(PublicationMergePair, AdjacentPairsPublishUniqueMembershipAndPreserveOldReaders) +{ + const auto old = publisher.capture(); + const auto pending = old->stats.checkpoint_pending_pages; + ASSERT_TRUE(publish(classification(1)).status.ok()); + ASSERT_TRUE(publish(classification(2, false)).status.ok()); + const auto view = publisher.capture(); + EXPECT_EQ(view->stats.merge_candidate_pages, 3U); + EXPECT_EQ(view->stats.merge_candidate_pairs, 2U); + EXPECT_EQ(view->stats.merge_ready_pairs, 1U); + EXPECT_EQ(view->stats.checkpoint_pending_pages, pending); + EXPECT_EQ(view->mapping.copy_bounds().merge_candidate_pages, 3U); + EXPECT_EQ(old->stats.merge_candidate_pages, 0U); + for (uint64_t pid = 1; pid <= 3; ++pid) { + EXPECT_TRUE(view->mapping.get(pid)->metrics.merge_candidate); + EXPECT_FALSE(old->mapping.get(pid)->metrics.merge_candidate); + EXPECT_EQ(view->mapping.get(pid)->merge_outgoing.has_value(), pid < 3); + EXPECT_FALSE(old->mapping.get(pid)->merge_outgoing); + } + reconcile(old); + reconcile(view); +} + +TEST_F(PublicationMergePair, ReadinessAndRepresentationChangesPreserveCandidateAge) +{ + ASSERT_TRUE(publish(classification(1, false)).status.ok()); + const auto before = publisher.capture(); + ASSERT_GT(before->merge_candidate_since_ns, 0U); + auto page = std::make_shared<PublishedPage>(*before->mapping.get(1)); + ++page->revision; + page->metrics.merge_candidate = false; // Recomputed logical summary omits pair membership. + ASSERT_TRUE(publish({.pages = {{1, before->mapping.get(1), page}}}).status.ok()); + EXPECT_TRUE(publisher.capture()->mapping.get(1)->metrics.merge_candidate); + ASSERT_TRUE(publish(classification(1, true)).status.ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->merge_candidate_since_ns, before->merge_candidate_since_ns); + EXPECT_EQ(after->stats.merge_ready_pairs, 1U); + const auto proof = after->mapping.get(1)->merge_outgoing; + if (!proof) { + FAIL() << "published outgoing proof missing"; + } + EXPECT_TRUE(proof->ready); + reconcile(after); +} + +TEST_F(PublicationMergePair, ChangedMiddlePageInvalidatesBothPairsAtomically) +{ + ASSERT_TRUE(publish(classification(1)).status.ok()); + ASSERT_TRUE(publish(classification(2)).status.ok()); + const auto before = publisher.capture(); + auto page = std::make_shared<PublishedPage>(*before->mapping.get(2)); + auto logical = std::make_shared<PageState>(*page->leaf); + ++logical->content_id; + page->leaf = std::move(logical); + ASSERT_TRUE(publish({.pages = {{2, before->mapping.get(2), page}}}).status.ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->stats.merge_candidate_pairs, 0U); + EXPECT_EQ(after->stats.merge_candidate_pages, 0U); + EXPECT_EQ(after->merge_candidate_since_ns, 0U); + EXPECT_EQ(before->stats.merge_candidate_pages, 3U); + EXPECT_FALSE(after->mapping.get(1)->merge_outgoing); + EXPECT_FALSE(after->mapping.get(2)->merge_outgoing); + reconcile(after); +} + +TEST_F(PublicationMergePair, SplitPriorityInvalidatesPairWithoutChangingLogicalVersion) +{ + ASSERT_TRUE(publish(classification(1)).status.ok()); + const auto before = publisher.capture(); + auto page = std::make_shared<PublishedPage>(*before->mapping.get(2)); + page->metrics.split_candidate = true; + ASSERT_TRUE(publish({.pages = {{2, before->mapping.get(2), page}}}).status.ok()); + EXPECT_EQ(publisher.capture()->stats.merge_candidate_pairs, 0U); + EXPECT_EQ(publish(classification(1)).status.code(), Code::kUnavailable); + reconcile(publisher.capture()); +} + +TEST_F(PublicationMergePair, SiblingInvalidationDoesNotRejectAlreadyPreparedPage) +{ + ASSERT_TRUE(publish(classification(1)).status.ok()); + const auto before = publisher.capture(); + auto left = std::make_shared<PublishedPage>(*before->mapping.get(1)); + auto right = std::make_shared<PublishedPage>(*before->mapping.get(2)); + auto left_logical = std::make_shared<PageState>(*left->leaf); + auto right_logical = std::make_shared<PageState>(*right->leaf); + ++left_logical->content_id; + ++right_logical->content_id; + left->leaf = std::move(left_logical); + right->leaf = std::move(right_logical); + ASSERT_TRUE(publish({.pages = {{1, before->mapping.get(1), left}}}).status.ok()); + ASSERT_NE(publisher.capture()->mapping.get(2), before->mapping.get(2)); + const auto result = publish({.pages = {{.page_id = 2, .expected = before->mapping.get(2), .replacement = right}}}); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + const auto after = publisher.capture(); + EXPECT_EQ(after->mapping.get(2)->leaf, right->leaf); + EXPECT_EQ(after->stats.merge_candidate_pages, 0U); + EXPECT_EQ(after->stats.merge_candidate_pairs, 0U); + reconcile(before); + reconcile(after); +} + +TEST_F(PublicationMergePair, ClassificationDoesNotRejectPreparedPageButRepresentationChangeDoes) +{ + const auto before = publisher.capture(); + auto page = std::make_shared<PublishedPage>(*before->mapping.get(2)); + ++page->revision; + ASSERT_TRUE(publish(classification(1)).status.ok()); + const auto result = publish({.pages = {{.page_id = 2, .expected = before->mapping.get(2), .replacement = page}}}); + ASSERT_TRUE(result.status.ok()) << result.status.to_string(); + const auto after = publisher.capture(); + EXPECT_EQ(after->stats.merge_candidate_pages, 2U); + EXPECT_EQ(after->stats.merge_candidate_pairs, 1U); + EXPECT_TRUE(after->mapping.get(2)->metrics.merge_candidate); + EXPECT_EQ(publish({.pages = {{2, before->mapping.get(2), page}}}).status.code(), Code::kUnavailable); + reconcile(after); +} + +TEST_F(PublicationMergePair, WindowedSiblingFlushKeepsCoverageAndStatisticsConsistent) +{ + auto table = std::make_shared<MemTable>(); + for (const auto *key : {"a", "b", "n", "o"}) { + ASSERT_TRUE(table->upsert(key, 2, encode_cell_buf(2, OpKind::kPut, key))); + } + table->close(); + auto sources = std::make_shared<PublishedSources>(); + sources->tables.push_back(std::move(table)); + ASSERT_TRUE(publish({.expected_sources = publisher.capture()->sources, .sources = sources}).status.ok()); + ASSERT_TRUE(publish(classification(1)).status.ok()); + detail::FlushRuntime runtime(2, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + FlushBatchOptions options; + options.install_run_window = 1; + options.run_limits.max_entries = 1; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 20, 2, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + const auto after = publisher.capture(); + EXPECT_EQ(after->coverage, 2U); + EXPECT_EQ(after->stats.overlay_entries, 4U); + EXPECT_EQ(after->stats.merge_candidate_pairs, 0U); + EXPECT_EQ(after->stats.merge_candidate_pages, 0U); + EXPECT_TRUE(after->sources->tables.empty()); + for (const auto *key : {"a", "b", "n", "o"}) { + const auto pid = key[0] < 'm' ? 1 : 2; + const auto value = after->mapping.get(pid)->leaf->lookup(key); + ASSERT_EQ(value.state, LogicalLookupState::kValue); + EXPECT_EQ(value.cell.value.to_string(), key); + } + reconcile(after); +} + +TEST_F(PublicationMergePair, ParentRouteChangeInvalidatesOnlyItsDirectPairProofs) +{ + ASSERT_TRUE(publish(classification(1)).status.ok()); + ASSERT_TRUE(publish(classification(2)).status.ok()); + const auto before = publisher.capture(); + auto page = std::make_shared<PublishedPage>(*before->mapping.get(10)); + auto inner = std::make_shared<InnerPageState>(*page->inner); + inner->separators[0] = OwnedKey("n"); + page->inner = std::move(inner); + ASSERT_TRUE(publish({.pages = {{10, before->mapping.get(10), page}}}).status.ok()); + EXPECT_EQ(publisher.capture()->stats.merge_candidate_pages, 0U); + reconcile(publisher.capture()); +} + +TEST_F(PublicationMergePair, BoundedMaintenanceClassifiesAdjacentPairsBeforeStructuralWork) +{ + detail::ResourceBudget::Limits limits{}; + limits.fill(16 * 1024 * 1024); + detail::ResourceBudget budget(limits, 4); + auto account = budget.make_account(limits); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PageMaintenanceOperationOptions options; + options.max_steps = 2; + options.publication.pages.merge_trigger_bytes = 256; + options.publication.pages.merge_target_bytes = 512; + options.acquire = [&](auto *out) { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, 4 * 1024 * 1024); + if (!credit) { + return Status::resource_exhausted(); + } + *out = std::make_shared<detail::AllocationReservation>(std::move(*credit)); + return Status::Ok(); + }; + auto maintenance = make_page_maintenance(publisher, runtime, lane, options); + std::promise<Status> done; + auto terminal = done.get_future(); + maintenance(publisher.capture(), [&](Status status, uint64_t) { done.set_value(std::move(status)); }); + const auto status = terminal.get(); + ASSERT_TRUE(status.ok()) << status.to_string(); + runtime.drain(); + const auto after = publisher.capture(); + EXPECT_EQ(after->stats.leaf_pages, 3U); + EXPECT_EQ(after->stats.merge_candidate_pages, 3U); + EXPECT_EQ(after->stats.merge_candidate_pairs, 2U); + EXPECT_EQ(after->stats.merge_ready_pairs, 2U); + reconcile(after); +} + +TEST_F(PublicationMergePair, RepeatedSingleStepMaintenanceAdvancesPastUnchangedClassifications) +{ + detail::ResourceBudget::Limits limits{}; + limits.fill(16 * 1024 * 1024); + detail::ResourceBudget budget(limits, 4); + auto account = budget.make_account(limits); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PageMaintenanceOperationOptions options; + options.max_steps = 1; + options.publication.pages.merge_trigger_bytes = 256; + options.publication.pages.merge_target_bytes = 512; + options.acquire = [&](auto *out) { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kReclaim, 4 * 1024 * 1024); + if (!credit) { + return Status::resource_exhausted(); + } + *out = std::make_shared<detail::AllocationReservation>(std::move(*credit)); + return Status::Ok(); + }; + auto maintenance = make_page_maintenance(publisher, runtime, lane, options); + for (size_t attempt = 0; attempt < 8 && publisher.capture()->stats.leaf_pages == 3; ++attempt) { + std::promise<Status> done; + auto terminal = done.get_future(); + maintenance(publisher.capture(), [&](Status status, uint64_t) { done.set_value(std::move(status)); }); + const auto status = terminal.get(); + ASSERT_TRUE(status.ok()) << status.to_string(); + runtime.drain(); + } + const auto after = publisher.capture(); + EXPECT_LT(after->stats.leaf_pages, 3U); + reconcile(after); +} + +TEST_F(PublicationMergePair, IndependentBootstrapClearsOnlyItsNewGenerationMembership) +{ + ASSERT_TRUE(publish(classification(1)).status.ok()); + const auto before = publisher.capture(); + PublicationBootstrap bootstrap; + bootstrap.initial = before; + PublicationCoordinator other(8, 1, {}, std::pmr::get_default_resource(), {}, std::move(bootstrap)); + const auto after = other.capture(); + EXPECT_EQ(after->stats.merge_candidate_pairs, 0U); + EXPECT_EQ(after->stats.merge_candidate_pages, 0U); + EXPECT_EQ(after->mapping.copy_bounds().merge_candidate_pages, 0U); + EXPECT_EQ(before->stats.merge_candidate_pages, 2U); + EXPECT_TRUE(before->mapping.get(1)->metrics.merge_candidate); + EXPECT_TRUE(before->mapping.get(1)->merge_outgoing); + EXPECT_FALSE(after->mapping.get(1)->merge_outgoing); + reconcile(before); + reconcile(after); +} + +TEST_F(PublicationMergePair, ClearingOutgoingPairKeepsTheSharedIncomingEndpoint) +{ + ASSERT_TRUE(publish(classification(1)).status.ok()); + ASSERT_TRUE(publish(classification(2)).status.ok()); + auto clear = classification(2); + if (!clear.merge_pair) { + FAIL() << "classification missing"; + } + clear.merge_pair->candidate.reset(); + ASSERT_TRUE(publish(std::move(clear)).status.ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->stats.merge_candidate_pairs, 1U); + EXPECT_EQ(after->stats.merge_candidate_pages, 2U); + EXPECT_TRUE(after->mapping.get(2)->metrics.merge_candidate); + EXPECT_FALSE(after->mapping.get(3)->metrics.merge_candidate); + EXPECT_FALSE(after->mapping.get(2)->merge_outgoing); + reconcile(after); +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST_F(PublicationMergePair, FailureAfterMembershipPreparationLeavesBothLedgersAndRootUnchanged) +{ + ASSERT_TRUE(publish(classification(1, false)).status.ok()); + const auto before = publisher.capture(); + publisher.set_before_root_for_tests([] { throw std::bad_alloc(); }); + EXPECT_EQ(publish(classification(2)).status.code(), Code::kResourceExhausted); + publisher.set_before_root_for_tests({}); + EXPECT_EQ(publisher.capture(), before); + ASSERT_TRUE(publish(classification(1, true)).status.ok()); + const auto after = publisher.capture(); + EXPECT_EQ(after->stats.merge_candidate_pages, 2U); + EXPECT_EQ(after->stats.merge_candidate_pairs, 1U); + EXPECT_EQ(after->stats.merge_ready_pairs, 1U); + EXPECT_EQ(after->merge_candidate_since_ns, before->merge_candidate_since_ns); + reconcile(after); +} +#endif +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_metrics_test.cpp b/lib/crowdb-tree/tests/unit/publication_metrics_test.cpp new file mode 100644 index 000000000..f0653f8b9 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_metrics_test.cpp @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <future> +#include <latch> +#include <memory> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<InstallRecord> metric_record(uint64_t sequence, uint64_t generation = 1) +{ + return std::make_shared<InstallRecord>( + InstallToken{.generation = generation, .batch = 1, .range = sequence, .sequence = 0}); +} + +std::shared_ptr<PublishedPage> metric_page(uint64_t revision) +{ + auto page = std::make_shared<PublishedPage>(); + page->revision = revision; + page->leaf = std::make_shared<PageState>(); + page->metrics.dirty = revision % 2 != 0; + page->metrics.kv_known = true; + page->metrics.kv_count = revision; + return page; +} + +PublicationResult metric_publish(PublicationCoordinator &coordinator, PublicationTransaction transaction) +{ + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> done; + auto result = done.get_future(); + request->on_done = [&done](const auto &value) { done.set_value(value); }; + if (!coordinator.try_submit(request)) { + ADD_FAILURE() << "publication was not accepted"; + return {.status = Status::unavailable()}; + } + return result.get(); +} + +TEST(PublicationMetrics, RetryObservesAnAttemptWithoutRepeatingLogicalDeltas) +{ + auto work = std::make_shared<TreeWorkMetrics>(); + PublicationCoordinator coordinator(8, 1, work); + TreeResourceMetrics resources; + auto page = metric_page(1); + PublicationTransaction transaction{ + .record = metric_record(1), .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, .root_page_id = 0}; + ASSERT_TRUE(metric_publish(coordinator, transaction).status.ok()); + const auto first = coordinator.stats(resources); + ASSERT_TRUE(metric_publish(coordinator, transaction).already_committed); + auto stale = + metric_publish(coordinator, {.record = metric_record(2), + .pages = {{.page_id = 0, .expected = {}, .replacement = metric_page(2)}}}); + EXPECT_EQ(stale.status.code(), Code::kUnavailable); + const auto after = coordinator.stats(resources); + EXPECT_EQ(after.commit_seq, first.commit_seq); + EXPECT_EQ(after.logical.page_count(), 1U); + EXPECT_EQ(after.logical.kv_known_count, 1U); + EXPECT_EQ(after.logical.dirty_pages, 1U); + + const auto totals = work->totals(); + const auto commit = totals[static_cast<size_t>(TreeStage::PublishCommit)]; + EXPECT_EQ(commit.attempts, 3U); + EXPECT_EQ(commit.count, 3U); + EXPECT_EQ(commit.inflight, 0U); + EXPECT_EQ(commit.outcomes[static_cast<size_t>(TreeOutcome::Success)], 1U); + EXPECT_EQ(commit.outcomes[static_cast<size_t>(TreeOutcome::Retry)], 1U); + EXPECT_EQ(commit.outcomes[static_cast<size_t>(TreeOutcome::Stale)], 1U); + EXPECT_EQ(totals[static_cast<size_t>(TreeStage::PublishQueueWait)].count, 3U); +} + +TEST(PublicationMetrics, SnapshotCapturesOneLogicalCommitAndIndependentResourceGauges) +{ + PublicationCoordinator coordinator(8, 17); + TreeResourceMetrics resources; + ASSERT_TRUE(resources.add(TreeResourceMetric::AllocatedBytes, 8192)); + ASSERT_TRUE(resources.add(TreeResourceMetric::ProxyPages, 1)); + auto page = metric_page(1); + ASSERT_TRUE(metric_publish(coordinator, {.record = metric_record(1, 17), + .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, + .root_page_id = 0}) + .status.ok()); + std::atomic<bool> finished{false}; + std::thread reader([&] { + do { + const auto sampled = coordinator.stats(resources); + EXPECT_EQ(sampled.instance_generation, 17U); + EXPECT_EQ(sampled.logical.kv_known_count, sampled.commit_seq); + EXPECT_EQ(sampled.logical.dirty_pages, sampled.commit_seq % 2); + EXPECT_EQ(sampled.logical.height, 1U); + EXPECT_EQ(sampled.logical.page_size_unknown_pages, 1U); + EXPECT_EQ(sampled.resources.allocated_bytes, 8192U); + EXPECT_EQ(sampled.resources.proxy_pages, 1U); + } while (!finished.load(std::memory_order_acquire)); + }); + for (uint64_t revision = 2; revision <= 100; ++revision) { + auto next = metric_page(revision); + EXPECT_TRUE(metric_publish(coordinator, {.record = metric_record(revision, 17), + .pages = {{.page_id = 0, .expected = page, .replacement = next}}}) + .status.ok()); + page = std::move(next); + } + finished.store(true, std::memory_order_release); + reader.join(); + EXPECT_EQ(coordinator.stats(resources).commit_seq, 100U); +} + +TEST(PublicationMetrics, QueuedWorkIsVisibleBeforeItsCommitStarts) +{ + auto work = std::make_shared<TreeWorkMetrics>(); + PublicationCoordinator coordinator(2, 1, work); + std::latch first_completed(1); + std::latch release_callback(1); + auto first = std::make_shared<PublicationRequest>(); + first->transaction.record = metric_record(1); + first->on_done = [&](const auto &result) { + EXPECT_TRUE(result.status.ok()); + first_completed.count_down(); + release_callback.wait(); + }; + std::thread publisher([&] { EXPECT_TRUE(coordinator.try_submit(first)); }); + first_completed.wait(); + auto second = std::make_shared<PublicationRequest>(); + second->transaction.record = metric_record(2); + EXPECT_TRUE(coordinator.try_submit(second)); + const auto waiting = work->totals(); + const auto queue = waiting[static_cast<size_t>(TreeStage::PublishQueueWait)]; + EXPECT_EQ(queue.attempts, 2U); + EXPECT_EQ(queue.inflight, 1U); + EXPECT_EQ(queue.count, 1U); + EXPECT_EQ(waiting[static_cast<size_t>(TreeStage::PublishCommit)].inflight, 0U); + release_callback.count_down(); + publisher.join(); + const auto complete = work->totals(); + EXPECT_EQ(complete[static_cast<size_t>(TreeStage::PublishQueueWait)].count, 2U); + EXPECT_EQ(complete[static_cast<size_t>(TreeStage::PublishQueueWait)].inflight, 0U); + EXPECT_EQ(complete[static_cast<size_t>(TreeStage::PublishCommit)].count, 2U); +} + +TEST(PublicationMetrics, InvalidDeltaRecordsErrorAndKeepsPreviousLogicalSnapshot) +{ + auto work = std::make_shared<TreeWorkMetrics>(); + PublicationCoordinator coordinator(2, 1, work); + TreeResourceMetrics resources; + auto invalid = metric_page(1); + invalid->metrics.split_ready = true; + invalid->metrics.split_candidate = false; + const auto result = metric_publish(coordinator, {.record = metric_record(1), + .pages = {{.page_id = 0, .expected = {}, .replacement = invalid}}, + .root_page_id = 0}); + EXPECT_EQ(result.status.code(), Code::kInvalidArgument); + const auto stats = coordinator.stats(resources); + EXPECT_EQ(stats.commit_seq, 0U); + EXPECT_EQ(stats.logical.page_count(), 0U); + EXPECT_EQ(stats.logical.split_ready_pages, 0U); + const auto totals = work->totals()[static_cast<size_t>(TreeStage::PublishCommit)]; + EXPECT_EQ(totals.outcomes[static_cast<size_t>(TreeOutcome::Error)], 1U); + EXPECT_EQ(totals.inflight, 0U); +} + +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_reconcile_test.cpp b/lib/crowdb-tree/tests/unit/publication_reconcile_test.cpp new file mode 100644 index 000000000..129d49d90 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_reconcile_test.cpp @@ -0,0 +1,178 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/tree_state_reconcile.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +class PublicationReconcile : public ::testing::Test +{ + protected: + PublicationCoordinator publisher{8, 1}; + + PublicationResult publish(PublicationTransaction transaction) + { + transaction.record = std::make_shared<InstallRecord>(InstallToken{.generation = 1}); + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + PublicationResult result; + bool completed = false; + request->on_done = [&](PublicationResult value) { + result = std::move(value); + completed = true; + }; + EXPECT_TRUE(publisher.try_submit(request)); + EXPECT_TRUE(completed); + return result; + } + + Status advance(const std::shared_ptr<TreeStateReconciliation> &audit, size_t slots = 1) + { + auto request = std::make_shared<PublicationRequest>(); + request->reconciliation = audit; + request->reconciliation_slots = slots; + Status status; + bool completed = false; + request->on_done = [&](PublicationResult result) { + status = result.status; + completed = true; + }; + EXPECT_TRUE(publisher.try_submit(request)); + EXPECT_TRUE(completed); + return status; + } + + void reconcile() + { + const auto view = publisher.capture(); + auto audit = std::make_shared<TreeStateReconciliation>(view); + for (size_t i = 0; i < 1024 && !audit->complete(); ++i) { + const auto before = audit->visited_pages(); + const auto status = advance(audit); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_LE(audit->visited_pages() - before, 1U); + } + EXPECT_TRUE(audit->complete()); + EXPECT_EQ(publisher.capture(), view); + } + + void SetUp() override + { + PublicationTransaction transaction; + for (uint64_t pid = 1; pid <= 2; ++pid) { + auto leaf = std::make_shared<PageState>(); + leaf->content_id = pid; + leaf->route_generation = 1; + leaf->right_sibling = pid == 1 ? 2 : UINT64_MAX; + auto page = std::make_shared<PublishedPage>(); + page->leaf = leaf; + transaction.pages.push_back({.page_id = pid, .expected = {}, .replacement = page}); + } + auto inner = std::make_shared<InnerPageState>(); + inner->children = {1, 2}; + inner->separators = {OwnedKey("m")}; + auto parent = std::make_shared<PublishedPage>(); + parent->level = 1; + parent->inner = inner; + parent->metrics.leaf = false; + transaction.pages.push_back({.page_id = 10, .expected = {}, .replacement = parent}); + transaction.root_page_id = 10; + ASSERT_TRUE(publish(std::move(transaction)).status.ok()); + } +}; + +TEST_F(PublicationReconcile, PendingMembershipTracksCaptureLaterChangesCommitAndAbort) +{ + reconcile(); + ASSERT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 3U); + ASSERT_TRUE(publish({ + .expected_coverage = 0, + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = 1} + }) + .status.ok()); + const auto before = publisher.capture()->mapping.get(1); + auto leaf = std::make_shared<PageState>(*before->leaf); + ++leaf->content_id; + auto after = std::make_shared<PublishedPage>(*before); + after->leaf = leaf; + ASSERT_TRUE(publish({.pages = {{.page_id = 1, .expected = before, .replacement = after}}}).status.ok()); + reconcile(); + ASSERT_TRUE(publish({ + .checkpoint = CheckpointPublication{.action = CheckpointAction::Commit, .token = 1} + }) + .status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 1U); + reconcile(); + ASSERT_TRUE(publish({ + .expected_coverage = 0, + .checkpoint = CheckpointPublication{.action = CheckpointAction::Capture, .token = 2} + }) + .status.ok()); + ASSERT_TRUE(publish({ + .checkpoint = CheckpointPublication{.action = CheckpointAction::Abort, .token = 2} + }) + .status.ok()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 1U); + reconcile(); +} + +TEST_F(PublicationReconcile, CanonicalPairsAndUniqueEndpointsMatchTheirPublication) +{ + const auto view = publisher.capture(); + ASSERT_TRUE(publish({ + .merge_pair = MergePairPublication{.left = 1, + .expected_left = view->mapping.get(1), + .expected_right = view->mapping.get(2), + .expected_parent = view->mapping.get(10), + .candidate = MergePairProof{.left = 1, + .right = 2, + .parent = 10, + .left_content = 1, + .right_content = 2, + .left_route = 1, + .right_route = 1}} + }) + .status.ok()); + EXPECT_EQ(publisher.capture()->stats.merge_candidate_pages, 2U); + EXPECT_EQ(publisher.capture()->stats.merge_candidate_pairs, 1U); + reconcile(); + const auto classified = publisher.capture(); + ASSERT_TRUE(publish({ + .merge_pair = MergePairPublication{.left = 1, .expected_left = classified->mapping.get(1)} + }) + .status.ok()); + reconcile(); +} + +TEST_F(PublicationReconcile, ConcurrentPublicationRejectsTheOldAuditWithoutMutatingItsView) +{ + const auto old = publisher.capture(); + auto audit = std::make_shared<TreeStateReconciliation>(old); + ASSERT_TRUE(advance(audit).ok()); + auto page = std::make_shared<PublishedPage>(*old->mapping.get(1)); + auto leaf = std::make_shared<PageState>(*page->leaf); + ++leaf->content_id; + page->leaf = leaf; + ASSERT_TRUE(publish({.pages = {{.page_id = 1, .expected = old->mapping.get(1), .replacement = page}}}).status.ok()); + EXPECT_EQ(advance(audit).code(), Code::kUnavailable); + EXPECT_EQ(old->mapping.get(1)->leaf->content_id, 1U); + EXPECT_EQ(publisher.capture()->mapping.get(1)->leaf->content_id, 2U); + reconcile(); +} + +TEST_F(PublicationReconcile, InvalidBoundDoesNotConsumeCursorOrPublishAnEmptyTransaction) +{ + const auto view = publisher.capture(); + auto audit = std::make_shared<TreeStateReconciliation>(view); + EXPECT_EQ(advance(audit, 0).code(), Code::kInvalidArgument); + EXPECT_EQ(advance(audit, 65).code(), Code::kInvalidArgument); + EXPECT_EQ(audit->visited_pages(), 0U); + EXPECT_EQ(publisher.capture(), view); + reconcile(); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_safety_test.cpp b/lib/crowdb-tree/tests/unit/publication_safety_test.cpp new file mode 100644 index 000000000..4200c7d39 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_safety_test.cpp @@ -0,0 +1,232 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <array> +#include <atomic> +#include <barrier> +#include <future> +#include <memory> +#include <thread> +#include <utility> +#include <vector> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<InstallRecord> safety_record(uint64_t sequence) +{ + return std::make_shared<InstallRecord>(InstallToken{.generation = 1, .batch = 1, .range = sequence, .sequence = 0}); +} + +std::shared_ptr<PublishedPage> safety_page(uint64_t revision) +{ + auto page = std::make_shared<PublishedPage>(); + page->revision = revision; + page->leaf = std::make_shared<PageState>(); + return page; +} + +PublicationResult safety_publish(PublicationCoordinator &coordinator, PublicationTransaction transaction) +{ + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> done; + auto result = done.get_future(); + request->on_done = [&done](PublicationResult value) { done.set_value(std::move(value)); }; + if (!coordinator.try_submit(request)) { + ADD_FAILURE() << "publication was not accepted"; + return {.status = Status::unavailable()}; + } + return result.get(); +} + +TEST(PublicationSafety, SharedInstallRecordBindsToExactlyOneConcurrentPublisher) +{ + PublicationCoordinator first; + PublicationCoordinator second; + auto record = safety_record(1); + std::barrier start(2); + PublicationResult a; + PublicationResult b; + std::thread one([&] { + start.arrive_and_wait(); + a = safety_publish(first, {.record = record}); + }); + std::thread two([&] { + start.arrive_and_wait(); + b = safety_publish(second, {.record = record}); + }); + one.join(); + two.join(); + EXPECT_NE(a.status.ok(), b.status.ok()); + EXPECT_EQ(a.status.ok() ? b.status.code() : a.status.code(), Code::kUnavailable); + EXPECT_EQ(first.capture()->commit_seq + second.capture()->commit_seq, 1U); + EXPECT_FALSE(a.already_committed); + EXPECT_FALSE(b.already_committed); +} + +TEST(PublicationSafety, ForeignInstallCannotProveSourceCoverage) +{ + PublicationCoordinator first; + PublicationCoordinator second; + auto installed = safety_record(1); + ASSERT_TRUE(safety_publish(first, {.record = installed}).status.ok()); + const auto before = second.capture(); + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 42; + const auto result = safety_publish(second, {.record = safety_record(2), + .coverage = 42, + .expected_sources = before->sources, + .sources = sources, + .required_installs = {installed}}); + EXPECT_EQ(result.status.code(), Code::kUnavailable); + EXPECT_EQ(second.capture(), before); + EXPECT_EQ(second.capture()->sources->floor, 0U); +} + +TEST(PublicationSafety, FailedMultiPageChangeLeavesMappingStatsAndRecordUncommitted) +{ + PublicationCoordinator coordinator; + auto a = safety_page(1); + auto b = safety_page(1); + ASSERT_TRUE(safety_publish(coordinator, + { + .record = safety_record(1), + .pages = {{.page_id = 0, .expected = {}, .replacement = a}, + {.page_id = 1, .expected = {}, .replacement = b}}, + .root_page_id = 0 + }) + .status.ok()); + const auto before = coordinator.capture(); + auto good = safety_page(2); + auto invalid = safety_page(2); + invalid->metrics.split_ready = true; + auto record = safety_record(2); + auto result = safety_publish(coordinator, { + .record = record, + .pages = {{.page_id = 0, .expected = a, .replacement = good}, + {.page_id = 1, .expected = b, .replacement = invalid}} + }); + EXPECT_EQ(result.status.code(), Code::kInvalidArgument); + EXPECT_EQ(coordinator.capture(), before); + EXPECT_EQ(coordinator.capture()->mapping.get(0), a); + EXPECT_EQ(coordinator.capture()->stats.leaf_pages, 2U); + EXPECT_FALSE(record->committed.load()); + + auto corrected = safety_page(2); + result = safety_publish(coordinator, { + .record = record, + .pages = {{.page_id = 0, .expected = a, .replacement = good}, + {.page_id = 1, .expected = b, .replacement = corrected}} + }); + EXPECT_TRUE(result.status.ok()); + EXPECT_EQ(coordinator.capture()->mapping.get(0), good); + EXPECT_TRUE(record->committed.load()); +} + +TEST(PublicationSafety, RemovingAnAbsentPageCannotAdvancePidAllocation) +{ + PublicationCoordinator coordinator; + const auto before = coordinator.capture(); + const auto result = safety_publish( + coordinator, {.record = safety_record(1), .pages = {{.page_id = 999, .expected = {}, .replacement = {}}}}); + EXPECT_EQ(result.status.code(), Code::kInvalidArgument); + EXPECT_EQ(coordinator.capture(), before); + EXPECT_EQ(coordinator.capture()->next_page_id, 0U); +} + +TEST(PublicationSafety, RemovingLastEntriesPrunesRadixPathsAndRetainsOldViews) +{ + MappingView initial; + auto a = safety_page(1); + auto b = safety_page(2); + auto c = safety_page(3); + const uint64_t distant = MappingView::kPageLimit - 1; + MappingView::CopyStats copied; + std::vector<PageReplacement> add{ + {.page_id = 1, .expected = {}, .replacement = a}, + {.page_id = 65, .expected = {}, .replacement = b}, + {.page_id = distant, .expected = {}, .replacement = c} + }; + auto live = initial.replace(add, copied); + ASSERT_FALSE(live.empty()); + std::vector<PageReplacement> remove{ + {.page_id = 1, .expected = a, .replacement = {}}, + {.page_id = distant, .expected = c, .replacement = {}} + }; + auto partial = live.replace(remove, copied); + EXPECT_FALSE(partial.empty()); + EXPECT_EQ(partial.get(65), b); + EXPECT_EQ(partial.get(1), nullptr); + EXPECT_EQ(partial.get(distant), nullptr); + EXPECT_EQ(live.get(distant), c); + remove = { + {.page_id = 65, .expected = b, .replacement = {}} + }; + auto empty = partial.replace(remove, copied); + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(partial.get(65), b); + EXPECT_EQ(live.get(1), a); +} + +TEST(PublicationSafety, CapturedRootOutlivesPublisherAndRetiredRootReclamation) +{ + std::shared_ptr<const PublishedTreeState> captured; + auto old = safety_page(1); + { + PublicationCoordinator coordinator; + ASSERT_TRUE(safety_publish(coordinator, {.record = safety_record(1), + .pages = {{.page_id = 0, .expected = {}, .replacement = old}}, + .root_page_id = 0}) + .status.ok()); + captured = coordinator.capture(); + auto next = safety_page(2); + ASSERT_TRUE(safety_publish(coordinator, {.record = safety_record(2), + .pages = {{.page_id = 0, .expected = old, .replacement = next}}}) + .status.ok()); + coordinator.reclaim(); + EXPECT_EQ(captured->mapping.get(0), old); + } + EXPECT_EQ(captured->commit_seq, 1U); + EXPECT_EQ(captured->mapping.get(0), old); + EXPECT_EQ(captured->stats.leaf_pages, 1U); +} + +TEST(PublicationSafety, ConcurrentIngressCannotStrandAcceptedRequests) +{ + constexpr size_t workers = 4; + constexpr size_t requests_per_worker = 128; + PublicationCoordinator coordinator(workers * requests_per_worker); + std::atomic<size_t> completed{0}; + std::barrier start(static_cast<ptrdiff_t>(workers)); + std::array<std::thread, workers> producers; + for (size_t worker = 0; worker < workers; ++worker) { + producers[worker] = std::thread([&, worker] { + start.arrive_and_wait(); + for (size_t i = 0; i < requests_per_worker; ++i) { + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = safety_record((worker * requests_per_worker) + i); + request->on_done = [&](PublicationResult result) { + EXPECT_TRUE(result.status.ok()); + completed.fetch_add(1, std::memory_order_relaxed); + }; + EXPECT_TRUE(coordinator.try_submit(request)); + } + }); + } + for (auto &producer : producers) { + producer.join(); + } + EXPECT_TRUE(coordinator.idle()); + EXPECT_EQ(completed.load(), workers * requests_per_worker); + EXPECT_EQ(coordinator.capture()->commit_seq, workers * requests_per_worker); +} + +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/publication_test.cpp b/lib/crowdb-tree/tests/unit/publication_test.cpp new file mode 100644 index 000000000..a1f3d9937 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/publication_test.cpp @@ -0,0 +1,262 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/published_state.h" + +#include <gtest/gtest.h> + +#include <barrier> +#include <future> +#include <latch> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<const PublishedPage> leaf(uint64_t revision) +{ + auto result = std::make_shared<PublishedPage>(); + result->revision = revision; + result->leaf = std::make_shared<PageState>(); + result->metrics.dirty = true; + return result; +} + +std::shared_ptr<InstallRecord> record(uint64_t sequence, uint64_t generation = 1) +{ + return std::make_shared<InstallRecord>( + InstallToken{.generation = generation, .batch = 7, .range = sequence, .sequence = 0}); +} + +PublicationResult publish(PublicationCoordinator &coordinator, PublicationTransaction transaction) +{ + auto request = std::make_shared<PublicationRequest>(); + request->transaction = std::move(transaction); + std::promise<PublicationResult> done; + auto result = done.get_future(); + request->on_done = [&done](const auto &value) { done.set_value(value); }; + if (!coordinator.try_submit(request)) { + ADD_FAILURE() << "publication was not accepted"; + return {.status = Status::unavailable()}; + } + return result.get(); +} + +TEST(Publication, PersistentMappingCopiesEachTouchedSegmentOnce) +{ + MappingView initial; + auto a = leaf(1); + auto b = leaf(2); + auto c = leaf(3); + std::vector<PageReplacement> changes{ + {.page_id = 1, .expected = {}, .replacement = a}, + {.page_id = 2, .expected = {}, .replacement = b}, + {.page_id = 65, .expected = {}, .replacement = c} + }; + MappingView::CopyStats copied; + auto next = initial.replace(changes, copied); + EXPECT_FALSE(initial.get(1)); + EXPECT_EQ(next.get(1), a); + EXPECT_EQ(next.get(65), c); + EXPECT_EQ(copied.directories, 4U); + EXPECT_EQ(copied.segments, 2U); + copied = {}; + changes = { + {.page_id = 2, .expected = b, .replacement = c} + }; + auto later = next.replace(changes, copied); + EXPECT_EQ(next.get(2), b); + EXPECT_EQ(later.get(1), a); + EXPECT_EQ(later.get(2), c); + EXPECT_EQ(copied.segments, 1U); + EXPECT_FALSE(later.get(MappingView::kPageLimit)); +} + +TEST(Publication, DescriptorBoundsFollowOnlyChangedRadixPathsAndShrinkAfterCoalescing) +{ + auto small = leaf(1); + auto large = std::make_shared<PublishedPage>(); + auto large_state = std::make_shared<PageState>(); + large_state->bases.resize(16); + large_state->overlays.resize(128); + large->leaf = large_state; + std::vector<PageReplacement> changes{ + {.page_id = 1, .expected = {}, .replacement = small}, + {.page_id = 65, .expected = {}, .replacement = large}, + {.page_id = uint64_t{1} << 24, .expected = {}, .replacement = small} + }; + MappingView::CopyStats copied; + auto before = MappingView{}.replace(changes, copied); + const auto bounds = before.copy_bounds(); + EXPECT_EQ(bounds.leaf_pages, 3U); + EXPECT_EQ(bounds.max_leaf_bases, 16U); + EXPECT_EQ(bounds.max_leaf_append_bytes, bounds.max_leaf_bytes); + EXPECT_GT(bounds.max_leaf_bytes, sizeof(PageState) + (16 * sizeof(BaseRef))); + changes = { + {.page_id = 65, .expected = large, .replacement = small} + }; + auto after = before.replace(changes, copied); + EXPECT_EQ(after.copy_bounds().leaf_pages, 3U); + EXPECT_LT(after.copy_bounds().max_leaf_bytes, bounds.max_leaf_bytes); + EXPECT_EQ(after.copy_bounds().max_leaf_bases, 0U); + EXPECT_EQ(before.copy_bounds().max_leaf_bases, 16U); + EXPECT_EQ(after.copy_bounds().leaf_bytes, 3 * after.copy_bounds().max_leaf_bytes); + EXPECT_EQ(before.copy_bounds().max_leaf_bytes, bounds.max_leaf_bytes); + changes = { + {.page_id = 1, .expected = small, .replacement = {}}, + {.page_id = 65, .expected = small, .replacement = {}}, + {.page_id = uint64_t{1} << 24, .expected = small, .replacement = {}} + }; + auto empty = after.replace(changes, copied); + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(empty.copy_bounds().leaf_pages, 0U); + EXPECT_EQ(empty.copy_bounds().leaf_bytes, 0U); + EXPECT_EQ(empty.copy_bounds().max_leaf_bytes, 0U); + EXPECT_EQ(empty.copy_bounds().max_leaf_append_bytes, 0U); + EXPECT_EQ(empty.copy_bounds().max_leaf_bases, 0U); +} + +TEST(Publication, AtomicCoverageAndCatalogKeepOldViewAlive) +{ + PublicationCoordinator coordinator; + auto before = coordinator.capture(); + auto installed = record(1); + auto page = leaf(1); + auto first = publish( + coordinator, + {.record = installed, .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, .root_page_id = 0}); + ASSERT_TRUE(first.status.ok()); + auto handoff = coordinator.capture(); + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 42; + auto result = publish(coordinator, {.record = record(2), + .expected_coverage = 0, + .coverage = 42, + .expected_sources = handoff->sources, + .sources = sources, + .required_installs = {installed}}); + ASSERT_TRUE(result.status.ok()); + const auto after = coordinator.capture(); + EXPECT_EQ(before->root_page_id, UINT64_MAX); + EXPECT_EQ(handoff->coverage, 0U); + EXPECT_EQ(handoff->sources->floor, 0U); + EXPECT_EQ(after->coverage, 42U); + EXPECT_EQ(after->sources->floor, 42U); + EXPECT_EQ(after->mapping.get(0), page); + EXPECT_EQ(after->stats.leaf_pages, 1U); + EXPECT_EQ(after->stats.height, 1U); + coordinator.reclaim(); + EXPECT_EQ(handoff->mapping.get(0), page); +} + +TEST(Publication, RetryIsResourceIdempotentAndConflictsDoNotPublish) +{ + PublicationCoordinator coordinator; + auto token = record(1); + auto page = leaf(1); + PublicationTransaction tx{ + .record = token, .pages = {{.page_id = 0, .expected = {}, .replacement = page}}, .root_page_id = 0}; + auto first = publish(coordinator, tx); + ASSERT_TRUE(first.status.ok()); + auto repeated = publish(coordinator, tx); + EXPECT_TRUE(repeated.status.ok()); + EXPECT_TRUE(repeated.already_committed); + EXPECT_EQ(first.commit_seq, repeated.commit_seq); + auto rejected = + publish(coordinator, {.record = record(2), .pages = {{.page_id = 0, .expected = {}, .replacement = leaf(2)}}}); + EXPECT_EQ(rejected.status.code(), Code::kUnavailable); + auto state = coordinator.capture(); + EXPECT_EQ(state->commit_seq, 1U); + EXPECT_EQ(state->stats.leaf_pages, 1U); + EXPECT_EQ(state->stats.dirty_pages, 1U); +} + +TEST(Publication, ConcurrentPagesInSameSegmentMergeIntoLatestRoot) +{ + constexpr size_t count = 12; + PublicationCoordinator coordinator(32); + std::vector<PageReplacement> initial; + initial.reserve(count); + for (size_t i = 0; i < count; ++i) { + initial.push_back({.page_id = i, .expected = {}, .replacement = leaf(1)}); + } + ASSERT_TRUE(publish(coordinator, + { + .record = record(0), .pages = {initial.begin(), initial.end()}, + .root_page_id = 0 + }) + .status.ok()); + auto old = coordinator.capture(); + std::barrier start(static_cast<ptrdiff_t>(count)); + std::vector<std::thread> writers; + writers.reserve(count); + for (size_t i = 0; i < count; ++i) { + writers.emplace_back([&, i] { + auto changed = leaf(2); + start.arrive_and_wait(); + EXPECT_TRUE(publish(coordinator, + {.record = record(i + 1), + .pages = {{.page_id = i, .expected = old->mapping.get(i), .replacement = changed}}}) + .status.ok()); + }); + } + for (auto &writer : writers) { + writer.join(); + } + const auto after = coordinator.capture(); + for (size_t i = 0; i < count; ++i) { + EXPECT_EQ(after->mapping.get(i)->revision, 2U); + EXPECT_EQ(old->mapping.get(i)->revision, 1U); + } + EXPECT_EQ(after->stats.leaf_pages, count); + EXPECT_EQ(after->commit_seq, count + 1); +} + +TEST(Publication, IncompleteCoverageAndWrongGenerationAreRejected) +{ + PublicationCoordinator coordinator; + const auto old = coordinator.capture(); + auto sources = std::make_shared<PublishedSources>(); + sources->floor = 10; + auto result = publish(coordinator, {.record = record(1), + .coverage = 10, + .expected_sources = old->sources, + .sources = sources, + .required_installs = {record(2)}}); + EXPECT_EQ(result.status.code(), Code::kUnavailable); + EXPECT_EQ(coordinator.capture(), old); + result = publish(coordinator, {.record = record(3, 2)}); + EXPECT_EQ(result.status.code(), Code::kUnavailable); + EXPECT_EQ(coordinator.capture(), old); +} + +TEST(Publication, BoundedQueueDrainsReentrantSubmissionsAndCloseRejects) +{ + PublicationCoordinator coordinator(1); + std::latch callback_entered(1); + std::latch release_callback(1); + auto first = std::make_shared<PublicationRequest>(); + first->transaction.record = record(1); + first->on_done = [&](const auto &result) { + EXPECT_TRUE(result.status.ok()); + callback_entered.count_down(); + release_callback.wait(); + }; + std::thread publisher([&] { EXPECT_TRUE(coordinator.try_submit(first)); }); + callback_entered.wait(); + auto second = std::make_shared<PublicationRequest>(); + second->transaction.record = record(2); + EXPECT_TRUE(coordinator.try_submit(second)); + auto third = std::make_shared<PublicationRequest>(); + third->transaction.record = record(3); + EXPECT_FALSE(coordinator.try_submit(third)); + coordinator.close(); + release_callback.count_down(); + publisher.join(); + EXPECT_TRUE(coordinator.idle()); + EXPECT_EQ(coordinator.capture()->commit_seq, 2U); + EXPECT_FALSE(coordinator.try_submit(third)); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/published_inner_read_test.cpp b/lib/crowdb-tree/tests/unit/published_inner_read_test.cpp new file mode 100644 index 000000000..0ce7dedf7 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/published_inner_read_test.cpp @@ -0,0 +1,194 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "inner_load_test_support.h" + +#include <algorithm> + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(PublishedInnerRead, StandaloneRouteOwnsMappingAndResumesExactPath) +{ + TestColdTree tree; + PublishedRouteCursor route(tree.view->root_page_id, tree.view->mapping, 8, false); + tree.view.reset(); + ASSERT_TRUE(route.seek("m", false).ok()); + ASSERT_NE(route.needs_inner(), nullptr); + EXPECT_EQ(route.needs_inner()->address, 0U); + ASSERT_TRUE(route.resume_inner(tree.loader->states[0]).ok()); + ASSERT_NE(route.needs_inner(), nullptr); + EXPECT_EQ(route.needs_inner()->address, 2U); + ASSERT_TRUE(route.resume_inner(tree.loader->states[2]).ok()); + EXPECT_EQ(route.needs_inner(), nullptr); + ASSERT_NE(route.leaf(), nullptr); + EXPECT_EQ(route.leaf()->lookup("m").cell.value.to_string(), "m"); + ASSERT_TRUE(route.next().ok()); + EXPECT_EQ(route.leaf()->lookup("t").cell.value.to_string(), "t"); + EXPECT_EQ(route.resume_inner(tree.loader->states[0]).code(), Code::kInvalidArgument); +} + +TEST(PublishedInnerRead, PointInlineLoadsBothLevelsWithoutMutatingCapturedMapping) +{ + TestColdTree tree; + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(tree.view, tree.loader, "m", {}, &request).ok()); + complete_inline(request->sender()); + EXPECT_TRUE(request->result().found); + EXPECT_EQ(request->result().cell.value.to_string(), "m"); + EXPECT_EQ(tree.loader->calls, 2U); + EXPECT_EQ(tree.view->mapping.get(0)->inner, nullptr); + EXPECT_EQ(tree.view->mapping.get(2)->inner, nullptr); +} + +TEST(PublishedInnerRead, LoadedRootIsConsumedBeforeTraversingAResidentChild) +{ + TestColdTree tree; + auto child = std::make_shared<PublishedPage>(*tree.view->mapping.get(2)); + child->inner = tree.loader->states[2]; + const PageReplacement change{.page_id = 2, .expected = tree.view->mapping.get(2), .replacement = child}; + MappingView::CopyStats copied; + tree.view->mapping = tree.view->mapping.replace({&change, 1}, copied); + for (bool reverse : {false, true}) { + PublishedRouteCursor route(tree.view->root_page_id, tree.view->mapping, 3, reverse); + ASSERT_TRUE(route.seek("m", false).ok()); + ASSERT_NE(route.needs_inner(), nullptr); + ASSERT_TRUE(route.resume_inner(tree.loader->states[0]).ok()); + EXPECT_EQ(route.needs_inner(), nullptr); + EXPECT_EQ(route.leaf_page_id(), 5U); + ASSERT_NE(route.leaf(), nullptr); + EXPECT_EQ(route.leaf()->lookup("m").cell.value.to_string(), "m"); + } +} + +TEST(PublishedInnerRead, ScanCrossesColdIndexesInBothDirectionsWithSingleWorker) +{ + TestColdTree tree; + auto runtime = std::make_shared<detail::FlushRuntime>(1, 1); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(1); + auto loader = std::make_shared<AsyncStorePageLoader>(tree.store, runtime, lane, PageLoadLimits{.max_inflight = 1}); + for (bool reverse : {false, true}) { + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(tree.view, loader, {.lower = "g", .upper = "t"}, reverse, + {.max_work = 2, .max_entries = 1}, &scan) + .ok()); + std::vector<std::string> keys; + for (size_t window = 0; window < 40; ++window) { + Completion done; + auto operation = stdexec::connect(scan->next_sender(), Receiver{.done = &done}); + stdexec::start(operation); + runtime->drain(); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + ASSERT_EQ(done.errors.load(), 0U); + EXPECT_LE(scan->result().work, 2U); + for (const auto &entry : scan->result().entries) { + keys.push_back(Slice(entry.key).to_string()); + } + if (scan->result().done) { + break; + } + } + EXPECT_TRUE(scan->result().done); + std::vector<std::string> expected{"g", "l", "m", "n"}; + if (reverse) { + std::reverse(expected.begin(), expected.end()); + } + EXPECT_EQ(keys, expected); + } + EXPECT_EQ(tree.store->reads.load(), 6U); +} + +TEST(PublishedInnerRead, LateLoadUsesOldMappingAfterReplacement) +{ + TestColdTree tree; + tree.loader->deferred = true; + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(tree.view, tree.loader, "m", {}, &request).ok()); + Completion done; + auto operation = stdexec::connect(request->sender(), Receiver{.done = &done}); + stdexec::start(operation); + auto changed = std::make_shared<PublishedTreeState>(*tree.view); + const std::array<PageReplacement, 1> changes{ + {{.page_id = 5, + .expected = tree.view->mapping.get(5), + .replacement = published(page({{.key = "m", .slot = 2, .value = "new"}}, {.lower = "m", .upper = "t"}))}} + }; + MappingView::CopyStats copied; + changed->mapping = changed->mapping.replace(changes, copied); + tree.view = std::move(changed); + tree.loader->finish(); + EXPECT_EQ(done.values.load(), 0U); + tree.loader->finish(); + EXPECT_EQ(done.values.load(), 1U); + EXPECT_EQ(request->result().cell.value.to_string(), "m"); +} + +TEST(PublishedInnerRead, LateFailureAndCancellationCompleteExactlyOnce) +{ + for (bool stopped : {false, true}) { + TestColdTree tree; + tree.loader->deferred = true; + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(tree.view, tree.loader, "m", {}, &request).ok()); + Completion done; + auto operation = stdexec::connect(request->sender(), Receiver{.done = &done}); + stdexec::start(operation); + if (stopped) { + request->request_stop(); + request.reset(); + } + tree.loader->finish(Status::io_error("index read failed")); + EXPECT_EQ(done.values.load(), 0U); + EXPECT_EQ(done.errors.load(), stopped ? 0U : 1U); + EXPECT_EQ(done.stopped.load(), stopped ? 1U : 0U); + EXPECT_EQ(tree.loader->calls, 1U); + } +} + +TEST(PublishedInnerRead, RejectsMissingChildInvalidSeparatorsAndNullLoadedState) +{ + for (size_t scenario = 0; scenario < 3; ++scenario) { + TestColdTree tree; + if (scenario == 0) { + tree.loader->states[0] = inner({"m"}, {1, 99}); + } + else if (scenario == 1) { + tree.loader->states[2] = inner({"a"}, {5, 6}); + } + else { + tree.loader->states[0].reset(); + } + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(tree.view, tree.loader, "m", {}, &request).ok()); + Completion done; + auto operation = stdexec::connect(request->sender(), Receiver{.done = &done}); + stdexec::start(operation); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(request->result().status.code(), Code::kCorruption); + } +} + +TEST(PublishedInnerRead, MissingTransportReturnsUnavailableAndScanCancelsPendingIndex) +{ + TestColdTree tree; + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(tree.view, {}, "m", {}, &request).ok()); + Completion done; + auto operation = stdexec::connect(request->sender(), Receiver{.done = &done}); + stdexec::start(operation); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_EQ(request->result().status.code(), Code::kUnavailable); + tree.loader->deferred = true; + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(tree.view, tree.loader, {}, true, {}, &scan).ok()); + Completion scanned; + auto range = stdexec::connect(scan->next_sender(), Receiver{.done = &scanned}); + stdexec::start(range); + scan->request_stop(); + tree.loader->finish(); + EXPECT_EQ(scanned.stopped.load(), 1U); + EXPECT_EQ(scanned.values.load(), 0U); + EXPECT_EQ(scanned.errors.load(), 0U); + EXPECT_EQ(tree.loader->calls, 1U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/published_read_budget_test.cpp b/lib/crowdb-tree/tests/unit/published_read_budget_test.cpp new file mode 100644 index 000000000..b58280894 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/published_read_budget_test.cpp @@ -0,0 +1,167 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "load_budget_test_support.h" +#include "published_read_test_support.h" + +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +TEST(PublishedReadBudget, FreshWorkerReadsSealedSourceWithItsOwnCreditAndRetainsResultKey) +{ + LoadCredits credits; + auto raw = credits.credits.bytes->take_reserved(16384); + if (!raw) { + FAIL() << "initial source credit unavailable"; + } + auto initial = detail::AllocationReservation::create(std::move(*raw)); + auto table = std::make_shared<MemTable>(1, nullptr, initial); + const std::string key(80, 'k'); + ASSERT_TRUE(table->upsert(key, 1, encode_cell_buf(1, OpKind::kPut, "value"))); + table->close(); + initial->seal(); + initial.reset(); + auto captured = view(page({}), {table}); + std::optional<PublishedScanEntry> retained; + std::jthread worker([&] { + PublishedReadLimits limits; + limits.metadata = credits.credits.bytes; + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(captured, {}, key, limits, &get).ok()); + complete_inline(get->sender()); + ASSERT_TRUE(get->result().found); + EXPECT_EQ(get->result().cell.value.to_string(), "value"); + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(captured, {}, {}, false, limits, &scan).ok()); + complete_inline(scan->next_sender()); + ASSERT_EQ(scan->result().entries.size(), 1U); + EXPECT_EQ(scan->result().entries.get_allocator().owner(), credits.credits.bytes); + retained = scan->result().entries.front(); + }); + worker.join(); + captured.reset(); + table.reset(); + credits.seal(); + if (!retained) { + FAIL() << "scan did not return an owned result"; + } + EXPECT_EQ(retained->key, key); + EXPECT_EQ(retained->cell.value.to_string(), "value"); + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + retained.reset(); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PublishedReadBudget, ColdReadCancellationKeepsChargedCallbackUntilLateCompletion) +{ + LoadCredits credits; + auto logical = std::make_shared<PageState>(); + logical->bases.push_back(cold(1)); + auto captured = view(logical); + auto loader = std::make_shared<TestLoader>(); + loader->deferred = true; + loader->frames[1] = base({ + {.key = "a", .value = "value"} + }); + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(captured, loader, "a", {.metadata = credits.credits.bytes}, &get).ok()); + Completion terminal; + { + auto connected = stdexec::connect(get->sender(), Receiver{.done = &terminal}); + stdexec::start(connected); + ASSERT_EQ(loader->pending.size(), 1U); + get->request_stop(); + EXPECT_EQ(terminal.stopped.load(), 1U); + } + get.reset(); + credits.seal(); + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + loader->finish(); + EXPECT_EQ(terminal.stopped.load(), 1U); + EXPECT_EQ(terminal.values.load(), 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PublishedReadBudget, ExhaustedAdmissionReturnsStatusWithoutUnbudgetedHandleFallback) +{ + LoadCredits credits; + auto captured = view(page({ + {.key = "a", .value = "value"} + })); + credits.seal(); + PublishedReadLimits limits; + limits.metadata = credits.credits.bytes; + std::shared_ptr<PublishedGet> get; + EXPECT_EQ(PublishedGet::create(captured, {}, "a", limits, &get).code(), Code::kResourceExhausted); + EXPECT_FALSE(get); + std::shared_ptr<PublishedScan> scan; + EXPECT_EQ(PublishedScan::create(captured, {}, {}, false, limits, &scan).code(), Code::kResourceExhausted); + EXPECT_FALSE(scan); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} + +TEST(PublishedReadBudget, CapturedDescriptorOverflowRejectsAdmission) +{ + PublishedReadLimits limits; + limits.page_metadata_bytes = SIZE_MAX; + EXPECT_FALSE(PublishedGet::allocation_bound(limits, 1)); + EXPECT_FALSE(PublishedScan::allocation_bound(limits, 1)); +} + +TEST(PublishedReadBudget, ColdLeafBeyondSourceWindowUsesCapturedDescriptorBound) +{ + auto logical = std::make_shared<PageState>(); + logical->bases.push_back(cold(1)); + auto run = page({ + {.key = "z", .value = "overlay"} + }) + ->overlays.front(); + logical->overlays.assign(8192, run); + auto captured = view(logical); + for (bool scan_read : {false, true}) { + LoadCredits credits; + credits.credits.bytes->seal(); + PublishedReadLimits limits; + limits.max_key_bytes = 1; + limits.max_memtables = 1; + limits.max_route_depth = 1; + limits.max_entries = 2; + limits.max_result_bytes = 32; + limits.page_metadata_bytes = captured->mapping.copy_bounds().max_leaf_bytes; + const auto bound = + scan_read ? PublishedScan::allocation_bound(limits, 1) : PublishedGet::allocation_bound(limits, 1); + if (!bound) { + FAIL() << "read bound overflow"; + } + limits.metadata = credits.reserve(detail::BudgetClass::kRouting, *bound); + auto loader = std::make_shared<TestLoader>(); + loader->frames[1] = base({ + {.key = "a", .value = "cold"} + }); + if (scan_read) { + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(captured, loader, {}, false, limits, &scan).ok()); + const auto entries = collect(scan); + ASSERT_TRUE(scan->result().status.ok()) << scan->result().status.to_string(); + ASSERT_EQ(entries.size(), 2U); + EXPECT_EQ(entries.front().key, "a"); + EXPECT_EQ(entries.front().cell.value.to_string(), "cold"); + EXPECT_EQ(entries.back().key, "z"); + } + else { + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(captured, loader, "a", limits, &get).ok()); + complete_inline(get->sender()); + ASSERT_TRUE(get->result().status.ok()) << get->result().status.to_string(); + ASSERT_TRUE(get->result().found); + EXPECT_EQ(get->result().cell.value.to_string(), "cold"); + } + EXPECT_EQ(loader->calls, 1U); + } +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/published_read_test.cpp b/lib/crowdb-tree/tests/unit/published_read_test.cpp new file mode 100644 index 000000000..2ecc22bce --- /dev/null +++ b/lib/crowdb-tree/tests/unit/published_read_test.cpp @@ -0,0 +1,171 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "published_read_test_support.h" + +#include <thread> + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(PublishedRead, PointMergesTwoMemTablesOverlayAndBaseWithCapturedFloor) +{ + auto first = std::make_shared<MemTable>(); + auto second = std::make_shared<MemTable>(); + ASSERT_TRUE(first->upsert("key", 5, encode_cell_buf(5, OpKind::kPut, "covered"))); + ASSERT_TRUE(first->upsert("key", 11, encode_cell_buf(11, OpKind::kPut, "first"), 0)); + ASSERT_TRUE(second->upsert("key", 12, encode_cell_buf(12, OpKind::kPut, "second"))); + auto leaf = std::make_shared<PageState>(*page({ + {.key = "key", .slot = 10, .value = "overlay"} + })); + leaf->bases.push_back({ + .identity = 1, .resident = base({{.key = "key", .slot = 3, .value = "base"}} + ) + }); + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(view(leaf, {first, second}, 8), {}, "key", {}, &request).ok()); + complete_inline(request->sender()); + EXPECT_TRUE(request->result().found); + EXPECT_EQ(request->result().cell.slot, 12U); + EXPECT_EQ(request->result().cell.value.to_string(), "second"); +} + +TEST(PublishedRead, PointDistinguishesMissingAndTombstoneAndBorrowsValue) +{ + const auto leaf = page({ + {.key = "a", .slot = 4, .value = "value"}, + {.key = "b", .slot = 5, .value = {}, .tombstone = true} + }); + for (const auto *key : {"a", "b", "c"}) { + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(view(leaf), {}, key, {}, &request).ok()); + complete_inline(request->sender()); + EXPECT_EQ(request->result().found, std::string_view(key) == "a"); + if (std::string_view(key) == "a") { + EXPECT_EQ(request->result().cell.value.data(), leaf->lookup("a").cell.value.data()); + } + if (std::string_view(key) == "b") { + EXPECT_TRUE(request->result().cell.tombstone()); + EXPECT_EQ(request->result().cell.slot, 5U); + } + } +} + +TEST(PublishedRead, ColdGetCompletesInlineWithoutRecursiveRestart) +{ + auto leaf = std::make_shared<PageState>(); + auto loader = std::make_shared<TestLoader>(); + for (uint64_t id = 1; id <= 20; ++id) { + leaf->bases.push_back(cold(id)); + loader->frames[id] = base({ + {.key = "key", .slot = id, .value = std::to_string(id)} + }); + } + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(view(leaf), loader, "key", {}, &request).ok()); + complete_inline(request->sender()); + EXPECT_EQ(loader->calls, 20U); + EXPECT_EQ(request->result().cell.value.to_string(), "20"); + EXPECT_EQ(leaf->bases[0].resident, nullptr); +} + +TEST(PublishedRead, ColdLateCompletionHoldsNoThreadBoundGuard) +{ + EpochManager epoch; + auto table = std::make_shared<MemTable>(1, &epoch); + ASSERT_TRUE(table->upsert("key", 5, encode_cell_buf(5, OpKind::kPut, "l0"))); + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(cold(1)); + auto loader = std::make_shared<TestLoader>(); + loader->deferred = true; + loader->frames[1] = base({ + {.key = "key", .slot = 7, .value = "disk"} + }); + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(view(leaf, {table}), loader, "key", {}, &request).ok()); + Completion done; + auto operation = stdexec::connect(request->sender(), Receiver{&done}); + stdexec::start(operation); + EXPECT_EQ(done.values.load(), 0U); + EXPECT_EQ(epoch.active_guards(), 0U); + std::thread worker([loader] { loader->finish(); }); + worker.join(); + EXPECT_EQ(done.values.load(), 1U); + EXPECT_EQ(request->result().cell.value.to_string(), "disk"); +} + +TEST(PublishedRead, LateIoErrorAndStopAreTerminalExactlyOnce) +{ + for (bool stopped : {false, true}) { + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(cold(1)); + auto loader = std::make_shared<TestLoader>(); + loader->deferred = true; + loader->frames[1] = base({ + {.key = "key", .slot = 1, .value = "disk"} + }); + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(view(leaf), loader, "key", {}, &request).ok()); + Completion done; + auto operation = stdexec::connect(request->sender(), Receiver{&done}); + stdexec::start(operation); + if (stopped) { + request->request_stop(); + EXPECT_EQ(done.stopped.load(), 1U); + } + loader->finish(Status::io_error("read failed")); + EXPECT_EQ(done.values.load(), 0U); + EXPECT_EQ(done.errors.load(), stopped ? 0U : 1U); + EXPECT_EQ(done.stopped.load(), stopped ? 1U : 0U); + } +} + +TEST(PublishedRead, ActiveMemTableAboveCoverageIsVisibleAndEqualSlotUsesSourceOrder) +{ + auto table = std::make_shared<MemTable>(); + ASSERT_TRUE(table->upsert("key", 20, encode_cell_buf(20, OpKind::kPut, "active"))); + auto leaf = page({ + {.key = "key", .slot = 20, .value = "overlay"} + }); + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(view(leaf, {table}, 10), {}, "key", {}, &request).ok()); + complete_inline(request->sender()); + EXPECT_EQ(request->result().cell.value.to_string(), "active"); +} + +TEST(PublishedRead, OldCapturedMappingSurvivesAConcurrentPublication) +{ + PublicationCoordinator coordinator; + auto original = published(page({ + {.key = "key", .slot = 1, .value = "old"} + })); + auto install = std::make_shared<PublicationRequest>(); + install->transaction.record = std::make_shared<InstallRecord>(InstallToken{.batch = 1}); + install->transaction.pages = { + {.page_id = 0, .expected = {}, .replacement = original} + }; + install->transaction.root_page_id = 0; + install->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(coordinator.try_submit(install)); + const auto captured = coordinator.capture(); + auto replace = std::make_shared<PublicationRequest>(); + replace->transaction.record = std::make_shared<InstallRecord>(InstallToken{.batch = 2}); + replace->transaction.pages = { + {.page_id = 0, + .expected = original, + .replacement = published(page({{.key = "key", .slot = 2, .value = "new"}}))} + }; + replace->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()); }; + ASSERT_TRUE(coordinator.try_submit(replace)); + coordinator.reclaim(); + for (const auto &[snapshot, expected] : std::array{ + std::pair{captured, std::string("old")}, + std::pair{coordinator.capture(), std::string("new")} + }) { + std::shared_ptr<PublishedGet> request; + ASSERT_TRUE(PublishedGet::create(snapshot, {}, "key", {}, &request).ok()); + complete_inline(request->sender()); + EXPECT_EQ(request->result().cell.value.to_string(), expected); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/published_route_pruning_test.cpp b/lib/crowdb-tree/tests/unit/published_route_pruning_test.cpp new file mode 100644 index 000000000..c2ab2d825 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/published_route_pruning_test.cpp @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_route.h" +#include "published_read_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<PublishedTreeState> routing_tree() +{ + auto tree = std::make_shared<PublishedTreeState>(); + auto root = std::make_shared<PublishedPage>(); + auto inner = std::make_shared<InnerPageState>(); + inner->children = {1, 2, 3}; + inner->separators = {"m", "t"}; + root->level = 1; + root->inner = inner; + auto middle = test_read::published(test_read::page( + { + {.key = "n", .value = "kept"} + }, + {.lower = "m", .upper = "t"})); + // Omitted children deliberately have no mapping descriptor: a bounded + // route must skip their roots without loading or inspecting descendants. + const std::array<PageReplacement, 2> replacements{ + {{.page_id = 0, .expected = {}, .replacement = root}, {.page_id = 2, .expected = {}, .replacement = middle}} + }; + MappingView::CopyStats copied; + tree->mapping = tree->mapping.replace(replacements, copied); + tree->root_page_id = 0; + return tree; +} + +TEST(PublishedRoutePruning, CountsExcludedRootsWithoutTouchingTheirDescendants) +{ + const auto tree = routing_tree(); + for (bool reverse : {false, true}) { + PublishedRouteCursor cursor(0, tree->mapping, 2, reverse, {}, {.lower = "m", .upper = "t"}); + const auto status = cursor.seek(reverse ? Slice("t") : Slice("m"), reverse); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(cursor.leaf_page_id(), 2U); + EXPECT_EQ(cursor.skipped_subtrees(), 1U); + ASSERT_TRUE(cursor.next().ok()); + EXPECT_FALSE(cursor.leaf()); + EXPECT_EQ(cursor.skipped_subtrees(), 2U); + } +} + +TEST(PublishedRoutePruning, EmptyRangeSkipsTheRootWithoutReadingItsIndex) +{ + const auto tree = routing_tree(); + PublishedRouteCursor cursor(0, tree->mapping, 2, false, {}, {.lower = "m", .upper = "m"}); + ASSERT_TRUE(cursor.seek(Slice("m"), false).ok()); + EXPECT_FALSE(cursor.leaf()); + EXPECT_FALSE(cursor.needs_inner()); + EXPECT_EQ(cursor.skipped_subtrees(), 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/published_scan_options_test.cpp b/lib/crowdb-tree/tests/unit/published_scan_options_test.cpp new file mode 100644 index 000000000..eeb5730ae --- /dev/null +++ b/lib/crowdb-tree/tests/unit/published_scan_options_test.cpp @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "published_read_test_support.h" + +namespace crowdb::tree +{ +using namespace test_read; + +TEST(PublishedScanOptions, InclusiveReverseEmptyKeyIsARealBoundary) +{ + auto captured = view(page({ + {.key = "", .value = "empty"}, + {.key = "a", .value = "a" } + })); + for (bool inclusive : {false, true}) { + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(captured, {}, {}, true, {}, &scan, + {.start = OwnedKey(""), .start_inclusive = inclusive}) + .ok()); + const auto entries = collect(scan); + EXPECT_EQ(entries.size(), inclusive ? 1U : 0U); + } +} + +TEST(PublishedScanOptions, KeysOnlyIncludesRequestedTombstoneWithinStrictBytes) +{ + auto captured = view(page({ + {.key = "a", .value = std::string(100, 'x') }, + {.key = "b", .value = "", .tombstone = true} + })); + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(captured, {}, {}, false, {.max_result_bytes = 2}, &scan, + {.keys_only = true, .include_tombstones = true}) + .ok()); + const auto entries = collect(scan); + ASSERT_EQ(entries.size(), 2U); + EXPECT_TRUE(entries[0].cell.value.empty()); + EXPECT_TRUE(entries[1].cell.tombstone()); +} + +TEST(PublishedScanOptions, ExactEntryLimitReportsEndWithoutAnExtraWindow) +{ + auto captured = view(page({ + {.key = "a", .value = "value"} + })); + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(captured, {}, {}, false, {.max_entries = 1}, &scan).ok()); + complete_inline(scan->next_sender()); + ASSERT_EQ(scan->result().entries.size(), 1U); + EXPECT_TRUE(scan->result().done); +} + +TEST(PublishedScanOptions, FullWindowProbeNeverLoadsTheNextOverflowValue) +{ + auto table = std::make_shared<MemTable>(); + ASSERT_TRUE(table->upsert("z", 2, encode_overflow_cell_buf(2, 10, 1024))); + auto captured = view(page({ + {.key = "a", .value = "value"} + }), + {table}); + std::shared_ptr<PublishedScan> scan; + // No loader is installed: materializing the lookahead value would fail. + ASSERT_TRUE(PublishedScan::create(captured, {}, {}, false, {.max_entries = 1}, &scan).ok()); + complete_inline(scan->next_sender()); + ASSERT_EQ(scan->result().entries.size(), 1U); + EXPECT_EQ(Slice(scan->result().entries[0].key).to_string(), "a"); + EXPECT_FALSE(scan->result().done); +} + +TEST(PublishedScanOptions, StrictFirstOverflowIsTruncatedWithoutLoadingOrConsumingIt) +{ + auto table = std::make_shared<MemTable>(); + ASSERT_TRUE(table->upsert("a", 2, encode_overflow_cell_buf(2, 10, 128 * 1024))); + auto captured = view(page({}), {table}); + std::shared_ptr<PublishedScan> scan; + // A missing loader makes any attempted overflow materialization fail. + ASSERT_TRUE(PublishedScan::create(captured, {}, {}, false, {.max_result_bytes = 1024}, &scan).ok()); + for (size_t attempt = 0; attempt < 2; ++attempt) { + complete_inline(scan->next_sender()); + EXPECT_TRUE(scan->result().status.ok()); + EXPECT_TRUE(scan->result().entries.empty()); + EXPECT_TRUE(scan->result().byte_limit_reached); + EXPECT_FALSE(scan->result().done); + } +} + +TEST(PublishedScanOptions, PublicBudgetSpansBoundedWindowsAndAllowsLargerSingleEntry) +{ + auto captured = view(page({ + {.key = "a", .value = "1234"}, + {.key = "b", .value = "1234"}, + {.key = "c", .value = std::string(20, 'v')}, + {.key = "d", .value = "1234"} + })); + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(captured, {}, {}, false, {.max_entries = 2, .max_result_bytes = 10}, &scan, + {.byte_budget = 36}) + .ok()); + for (const size_t count : {size_t{2}, size_t{1}, size_t{1}}) { + complete_inline(scan->next_sender()); + ASSERT_TRUE(scan->result().status.ok()); + EXPECT_EQ(scan->result().entries.size(), count); + EXPECT_FALSE(scan->result().byte_limit_reached); + } + EXPECT_TRUE(scan->result().done); +} + +TEST(PublishedReadBounds, RejectsOverflowedLimitsAndFundsDefaultOperations) +{ + PublishedReadLimits limits; + ASSERT_TRUE(PublishedGet::allocation_bound(limits, 4)); + ASSERT_TRUE(PublishedScan::allocation_bound(limits, 4)); + limits.max_entries = SIZE_MAX; + EXPECT_FALSE(PublishedScan::allocation_bound(limits, 4)); + PageLoadLimits loads; + EXPECT_TRUE(AsyncStorePageLoader::read_reserve_bound(loads, 1, 4, 4, 1, loads.max_value_bytes)); + EXPECT_FALSE(AsyncStorePageLoader::read_reserve_bound(loads, 1, SIZE_MAX, 4, 1, 0)); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/published_scan_test.cpp b/lib/crowdb-tree/tests/unit/published_scan_test.cpp new file mode 100644 index 000000000..4fb8aee85 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/published_scan_test.cpp @@ -0,0 +1,217 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "published_read_test_support.h" + +#include <algorithm> + +namespace crowdb::tree +{ +using namespace test_read; + +namespace +{ +std::shared_ptr<const PublishedTreeState> test_branch() +{ + auto result = std::make_shared<PublishedTreeState>(); + auto root = std::make_shared<PublishedPage>(); + auto index = std::make_shared<InnerPageState>(); + index->separators = {"m", "t"}; + index->children = {1, 2, 3}; + root->level = 1; + root->inner = index; + auto left = published(page( + { + {.key = "", .value = "empty"}, + {.key = "a", .value = "a" }, + {.key = "l", .value = "l" } + }, + {.upper = "m"})); + auto middle = published(page( + { + {.key = "m", .value = "m"}, + {.key = "n", .value = "n"} + }, + {.lower = "m", .upper = "t"})); + auto right = published(page( + { + {.key = "t", .value = "t"}, + {.key = "z", .value = "z"} + }, + {.lower = "t"})); + const std::array<PageReplacement, 4> changes{ + { + {.page_id = 0, .expected = {}, .replacement = root}, + {.page_id = 1, .expected = {}, .replacement = left}, + {.page_id = 2, .expected = {}, .replacement = middle}, + {.page_id = 3, .expected = {}, .replacement = right}, + } + }; + MappingView::CopyStats copied; + result->mapping = result->mapping.replace(changes, copied); + result->root_page_id = 0; + result->sources = std::make_shared<PublishedSources>(); + return result; +} + +std::vector<std::string> keys(const std::vector<PublishedScanEntry> &entries) +{ + std::vector<std::string> result; + result.reserve(entries.size()); + for (const auto &entry : entries) { + result.push_back(Slice(entry.key).to_string()); + } + return result; +} +} // namespace + +TEST(PublishedScan, BothDirectionsMergeTwoMemTablesOverlayAndBase) +{ + auto first = std::make_shared<MemTable>(); + auto second = std::make_shared<MemTable>(); + ASSERT_TRUE(first->upsert("b", 12, encode_cell_buf(12, OpKind::kPut, "first"))); + ASSERT_TRUE(first->upsert("c", 2, encode_cell_buf(2, OpKind::kPut, "covered"))); + ASSERT_TRUE(second->upsert("b", 13, encode_cell_buf(13, OpKind::kPut, "second"))); + ASSERT_TRUE(second->upsert("d", 14, encode_cell_buf(14, OpKind::kDelete, {}), kFlagTombstone)); + auto leaf = std::make_shared<PageState>(*page({ + {.key = "b", .slot = 10, .value = "overlay"}, + {.key = "c", .slot = 11, .value = "live" } + })); + leaf->bases.push_back({ + .identity = 1, + .resident = + base({{.key = "a", .value = "base"}, {.key = "b", .value = "old"}, {.key = "d", .value = "removed"}} + ) + }); + for (bool reverse : {false, true}) { + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(view(leaf, {first, second}, 8), {}, {}, reverse, + {.max_work = 3, .max_entries = 1}, &scan) + .ok()); + auto entries = collect(scan); + if (reverse) { + std::reverse(entries.begin(), entries.end()); + } + EXPECT_EQ(keys(entries), (std::vector<std::string>{"a", "b", "c"})); + ASSERT_EQ(entries.size(), 3U); + EXPECT_EQ(entries[1].cell.slot, 13U); + EXPECT_EQ(entries[1].cell.value.to_string(), "second"); + EXPECT_EQ(entries[2].cell.value.to_string(), "live"); + } +} + +TEST(PublishedScan, SeparatorBoundsAreHalfOpenInBothDirections) +{ + const auto snapshot = test_branch(); + for (bool reverse : {false, true}) { + for (const auto &range : std::array<PageKeyRange, 4>{ + {{}, {.upper = "m"}, {.lower = "m", .upper = "t"}, {.lower = "t"}} + }) { + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE( + PublishedScan::create(snapshot, {}, range, reverse, {.max_work = 1, .max_entries = 1}, &scan).ok()); + auto actual = keys(collect(scan)); + std::vector<std::string> expected; + for (const auto *key : {"", "a", "l", "m", "n", "t", "z"}) { + if (range.contains(key)) { + expected.emplace_back(key); + } + } + if (reverse) { + std::reverse(expected.begin(), expected.end()); + } + EXPECT_EQ(actual, expected); + } + } + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(snapshot, {}, "m", {}, &get).ok()); + complete_inline(get->sender()); + EXPECT_EQ(get->result().cell.value.to_string(), "m"); +} + +TEST(PublishedScan, CoveredEntriesAndTombstonesYieldBoundedEmptyWindows) +{ + auto table = std::make_shared<MemTable>(); + auto leaf = std::make_shared<PageState>(); + OverlayBuilder builder({}); + for (size_t i = 0; i < 80; ++i) { + const auto key = "k" + std::to_string(100 + i); + ASSERT_TRUE(table->upsert(Slice(key), 2, encode_cell_buf(2, OpKind::kPut, "covered"))); + auto cell = std::make_shared<CellVersion>(encode_cell_buf(3, OpKind::kDelete, {}), 3, kFlagTombstone); + ASSERT_TRUE(builder.append(Slice(key), std::move(cell)).ok()); + } + std::shared_ptr<const OverlayBatch> overlay; + ASSERT_TRUE(builder.finish(1, &overlay).ok()); + leaf->overlays.push_back(std::move(overlay)); + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE( + PublishedScan::create(view(leaf, {table}, 2), {}, {}, false, {.max_work = 4, .max_entries = 1}, &scan).ok()); + size_t windows = 0; + do { + complete_inline(scan->next_sender()); + EXPECT_TRUE(scan->result().entries.empty()); + EXPECT_LE(scan->result().work, 4U); + ASSERT_LT(++windows, 100U); + } while (!scan->result().done); + EXPECT_GT(windows, 20U); +} + +TEST(PublishedScan, ContinuationKeepsCapturedMappingAndOwners) +{ + auto original = test_branch(); + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(original, {}, {}, false, {.max_entries = 1}, &scan).ok()); + complete_inline(scan->next_sender()); + ASSERT_EQ(scan->result().entries.size(), 1U); + auto kept = scan->result().entries[0]; + original.reset(); + EXPECT_EQ(keys(collect(scan)), (std::vector<std::string>{"a", "l", "m", "n", "t", "z"})); + scan.reset(); + EXPECT_EQ(kept.cell.value.to_string(), "empty"); +} + +TEST(PublishedScan, ColdLateErrorTerminatesAndRemainsSticky) +{ + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(cold(1)); + auto loader = std::make_shared<TestLoader>(); + loader->deferred = true; + loader->frames[1] = base({ + {.key = "a", .value = "disk"} + }); + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(view(leaf), loader, {}, false, {}, &scan).ok()); + Completion done; + auto operation = stdexec::connect(scan->next_sender(), Receiver{.done = &done}); + stdexec::start(operation); + EXPECT_EQ(done.values.load(), 0U); + loader->finish(Status::io_error("cold read failed")); + EXPECT_EQ(done.errors.load(), 1U); + EXPECT_FALSE(scan->result().status.ok()); + Completion again; + auto next = stdexec::connect(scan->next_sender(), Receiver{.done = &again}); + stdexec::start(next); + EXPECT_EQ(again.errors.load(), 1U); + EXPECT_EQ(loader->calls, 1U); +} + +TEST(PublishedScan, ByteWindowRetainsPendingCandidateAndReportsOversizedRecord) +{ + const auto snapshot = view(page({ + {.key = "a", .value = "123"}, + {.key = "b", .value = "456"} + })); + std::shared_ptr<PublishedScan> scan; + ASSERT_TRUE(PublishedScan::create(snapshot, {}, {}, false, {.max_result_bytes = 5}, &scan).ok()); + EXPECT_EQ(keys(collect(scan)), (std::vector<std::string>{"a", "b"})); + ASSERT_TRUE(PublishedScan::create(snapshot, {}, {}, false, {.max_result_bytes = 3}, &scan).ok()); + Completion done; + auto operation = stdexec::connect(scan->next_sender(), Receiver{.done = &done}); + stdexec::start(operation); + EXPECT_EQ(done.values.load(), 1U); + EXPECT_EQ(done.errors.load(), 0U); + EXPECT_TRUE(scan->result().status.ok()); + EXPECT_TRUE(scan->result().byte_limit_reached); + EXPECT_TRUE(scan->result().entries.empty()); + EXPECT_FALSE(scan->result().done); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/read_value_bound_test.cpp b/lib/crowdb-tree/tests/unit/read_value_bound_test.cpp new file mode 100644 index 000000000..47c4b68a5 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/read_value_bound_test.cpp @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/overflow_backing.h" +#include "btree/read_value_bound.h" +#include "published_read_test_support.h" +#include "snapshot/primary_backing.h" + +namespace crowdb::tree +{ +TEST(ReadValueBound, OnlyExactMatchingDependencyProofsRemoveColdBaseAllowance) +{ + auto base = test_read::cold(7); + auto backing = std::make_shared<BackingRef>(*base.backing); + base.backing = backing; + auto leaf = std::make_shared<PageState>(); + leaf->bases.push_back(base); + auto page = test_read::published(leaf); + auto primary = std::make_shared<PrimaryBackingMetadata>(); + page->primary = primary; + auto refs = std::make_shared<StoredOverflowReferences>(); + primary->frame.stored_overflow = refs; + auto bound = [&] { + auto captured = std::make_shared<PublishedTreeState>(*test_read::view(leaf)); + MappingView::CopyStats copied; + const std::array<PageReplacement, 1> changes{ + {{.page_id = 0, .expected = captured->mapping.get(0), .replacement = page}}}; + captured->mapping = captured->mapping.replace(changes, copied); + captured->stats.kv_unknown_pages = 1; + return overflow_value_bytes_bound(*captured, 65536); + }; + EXPECT_EQ(bound(), 65536U); + primary->frame.backing = backing; + EXPECT_EQ(bound(), 0U); + refs->push_back({.head = 17, .bytes = 65536}); + EXPECT_EQ(bound(), 65536U); + primary->frame.stored_overflow.reset(); + EXPECT_EQ(bound(), 65536U); + auto dependencies = std::make_shared<OverflowBackingSet>(nullptr); + backing->overflow = dependencies; + EXPECT_EQ(bound(), 0U); + dependencies->pages.emplace(17, OverflowBackingPage{.backing = backing}); + EXPECT_EQ(bound(), 65536U); + // Avoid an artificial self-owned dependency in this metadata-only fixture. + dependencies->pages.clear(); +} + +TEST(ReadValueBound, UnknownLiveCountsDoNotRequireOverflowStorageForBorrowedCells) +{ + auto captured = std::make_shared<PublishedTreeState>(*test_read::view(test_read::page({ + {.key = "a", .value = "borrowed"} + }))); + captured->stats.kv_unknown_pages = 1; + EXPECT_FALSE(captured->mapping.copy_bounds().may_have_overflow_values); + EXPECT_EQ(overflow_value_bytes_bound(*captured, 65536), 0U); +} + +TEST(ReadValueBound, EncodedOverlayAndBaseInputsRetainTheirOverflowAllowance) +{ + auto logical = std::make_shared<PageState>(); + OverlayBuilder builder({}); + auto owner = std::make_shared<CellVersion>(encode_overflow_cell_buf(2, 41, 16384), 2, kFlagOverflow); + ASSERT_TRUE(builder.append(Slice("a"), std::move(owner)).ok()); + std::shared_ptr<const OverlayBatch> run; + ASSERT_TRUE(builder.finish(1, &run).ok()); + ASSERT_TRUE(run->has_overflow_values()); + logical->overlays.push_back(std::move(run)); + auto captured = std::make_shared<PublishedTreeState>(*test_read::view(logical)); + captured->stats.kv_unknown_pages = 1; + EXPECT_TRUE(captured->mapping.copy_bounds().may_have_overflow_values); + EXPECT_EQ(overflow_value_bytes_bound(*captured, 65536), 65536U); + captured->stats.kv_unknown_pages = 0; + captured->stats.kv_known_bytes = 16385; + EXPECT_EQ(overflow_value_bytes_bound(*captured, 65536), 16385U); + auto cold = std::make_shared<PageState>(); + cold->bases.push_back(test_read::cold(7)); + auto replacement = test_read::published(cold); + const std::array<PageReplacement, 1> changes{ + {{.page_id = 0, .expected = captured->mapping.get(0), .replacement = replacement}}}; + MappingView::CopyStats copied; + captured->mapping = captured->mapping.replace(changes, copied); + EXPECT_TRUE(captured->mapping.copy_bounds().may_have_overflow_values); + const std::array<PageReplacement, 1> clear{ + {{.page_id = 0, .expected = replacement, .replacement = test_read::published(test_read::page({}))}}}; + captured->mapping = captured->mapping.replace(clear, copied); + EXPECT_FALSE(captured->mapping.copy_bounds().may_have_overflow_values); + EXPECT_EQ(overflow_value_bytes_bound(*captured, 65536), 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/resource_account_stats_test.cpp b/lib/crowdb-tree/tests/unit/resource_account_stats_test.cpp new file mode 100644 index 000000000..8fbf05547 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/resource_account_stats_test.cpp @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +TEST(ResourceAccountStats, ReservationTransfersAndSealsRemainAccountLocal) +{ + ResourceBudget budget({1024, 1024, 1024, 1024, 1024, 16}, 4); + auto first = budget.make_account({1024, 1024, 1024, 1024, 1024, 16}); + auto second = budget.make_account({1024, 1024, 1024, 1024, 1024, 16}); + constexpr auto category = static_cast<size_t>(BudgetClass::kRouting); + auto a = ResourceBudget::try_reserve(first, BudgetClass::kRouting, 300); + auto b = ResourceBudget::try_reserve(second, BudgetClass::kRouting, 500); + if (!a || !b) { + ADD_FAILURE() << "both account reservations must succeed"; + return; + } + auto allocated = a.value().allocate(100); + ASSERT_TRUE(allocated); + auto split = a.value().split(50); + ASSERT_TRUE(split); + EXPECT_EQ(first->snapshot().reserved[category], 200U); + EXPECT_EQ(first->snapshot().allocated[category], 100U); + split.reset(); + AllocationReservation remaining(std::move(a.value())); + ASSERT_TRUE(remaining.claim(64)); + EXPECT_EQ(first->snapshot().reserved[category], 86U); + EXPECT_EQ(first->snapshot().allocated[category], 164U); + auto transferred = remaining.take_reserved(16); + ASSERT_TRUE(transferred); + EXPECT_EQ(first->snapshot().reserved[category], 86U); + transferred.reset(); + remaining.release(64); + remaining.seal(); + EXPECT_EQ(first->snapshot().reserved[category], 0U); + EXPECT_EQ(first->snapshot().allocated[category], 100U); + EXPECT_EQ(second->snapshot().reserved[category], 500U); + EXPECT_EQ(second->snapshot().allocated[category], 0U); + allocated.reset(); + EXPECT_EQ(first->snapshot().allocated[category], 0U); + b.reset(); + EXPECT_EQ(second->snapshot().reserved[category], 0U); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/resource_budget_progress_test.cpp b/lib/crowdb-tree/tests/unit/resource_budget_progress_test.cpp new file mode 100644 index 000000000..b986f72d8 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/resource_budget_progress_test.cpp @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/resource_budget.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +namespace +{ +ResourceBudget::Limits limits(size_t bytes) +{ + ResourceBudget::Limits result; + result.fill(bytes); + return result; +} + +struct TestCompletion +{ + using receiver_concept = stdexec::receiver_tag; + Status *status; + bool *done; + + void set_value() const noexcept + { + *done = true; + } + + void set_error(Status error) const noexcept + { + *status = std::move(error); + *done = true; + } + + void set_stopped() const noexcept + { + *status = Status::unavailable(); + *done = true; + } +}; + +TEST(ResourceBudgetProgress, ReservedCreditCanReleaseButConversionToLiveAllocationRejectsSoftWork) +{ + ResourceBudget budget(limits(100), 2); + auto account = budget.make_account(limits(100)); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kReclaim, 100); + if (!occupied) { + FAIL() << "initial credit unavailable"; + } + ResourceBudget::Request request(account, BudgetClass::kReclaim, 80, nullptr, + ResourceBudget::Request::WaitPolicy::ReservationsOnly); + Status status; + bool done = false; + auto operation = stdexec::connect(request.sender(), TestCompletion{.status = &status, .done = &done}); + stdexec::start(operation); + EXPECT_FALSE(done); + const auto before = account->resource_usage()[static_cast<size_t>(BudgetClass::kReclaim)].denied_tree; + auto live = occupied->allocate(25); + ASSERT_TRUE(live); + EXPECT_TRUE(done); + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(request.take_credit()); + EXPECT_EQ(budget.snapshot().waiters, 0U); + EXPECT_EQ(account->resource_usage()[static_cast<size_t>(BudgetClass::kReclaim)].denied_tree, before + 1); +} + +TEST(ResourceBudgetProgress, OtherTreeLiveAllocationsAlsoLimitSoftWorkHeadroom) +{ + ResourceBudget budget(limits(100), 2); + auto first = budget.make_account(limits(100)); + auto second = budget.make_account(limits(100)); + auto occupied = ResourceBudget::try_reserve(first, BudgetClass::kReclaim, 100); + if (!occupied) { + FAIL() << "initial credit unavailable"; + } + auto live = occupied->allocate(60); + ASSERT_TRUE(live); + occupied.reset(); + ResourceBudget::Request request(second, BudgetClass::kReclaim, 50, nullptr, + ResourceBudget::Request::WaitPolicy::ReservationsOnly); + Status status; + bool done = false; + auto operation = stdexec::connect(request.sender(), TestCompletion{.status = &status, .done = &done}); + stdexec::start(operation); + EXPECT_TRUE(done); + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_EQ(budget.snapshot().waiters, 0U); + EXPECT_EQ(second->resource_usage()[static_cast<size_t>(BudgetClass::kReclaim)].denied_host, 1U); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/resource_budget_test.cpp b/lib/crowdb-tree/tests/unit/resource_budget_test.cpp new file mode 100644 index 000000000..2575d8e0a --- /dev/null +++ b/lib/crowdb-tree/tests/unit/resource_budget_test.cpp @@ -0,0 +1,303 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "btree/resource_budget.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <limits> +#include <thread> + +namespace crowdb::tree::detail +{ +namespace +{ +constexpr size_t kHandoff = static_cast<size_t>(BudgetClass::kHandoff); + +ResourceBudget::Limits limits(size_t amount) +{ + ResourceBudget::Limits result; + result.fill(amount); + return result; +} + +struct BudgetResult +{ + std::atomic<size_t> values{0}; + std::atomic<size_t> errors{0}; + std::atomic<size_t> stopped{0}; +}; + +struct BudgetReceiver +{ + using receiver_concept = stdexec::receiver_tag; + BudgetResult *result; + + void set_value() const noexcept + { + result->values.fetch_add(1); + } + + void set_error(Status /*unused*/) const noexcept + { + result->errors.fetch_add(1); + } + + void set_stopped() const noexcept + { + result->stopped.fetch_add(1); + } +}; + +struct ReentrantBudgetReceiver +{ + using receiver_concept = stdexec::receiver_tag; + BudgetReceiver receiver; + ResourceBudget::Request *first; + CallbackSender::Operation<BudgetReceiver> *next; + + void set_value() const noexcept + { + receiver.set_value(); + first->take_credit().reset(); + stdexec::start(*next); + } + + void set_error(Status status) const noexcept + { + receiver.set_error(std::move(status)); + } + + void set_stopped() const noexcept + { + receiver.set_stopped(); + } +}; + +TEST(ResourceBudget, ReservationConvertsToAllocationWithoutDoubleCharging) +{ + ResourceBudget budget(limits(100), 4); + auto account = budget.make_account(limits(100)); + auto reservation = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 100); + if (!reservation.has_value()) { + FAIL() << "reservation rejected"; + } + auto allocation = reservation.value().allocate(60); + if (!allocation.has_value()) { + FAIL() << "allocation transfer rejected"; + } + EXPECT_EQ(budget.snapshot().reserved[kHandoff], 40U); + EXPECT_EQ(budget.snapshot().allocated[kHandoff], 60U); + EXPECT_FALSE(ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 1)); + reservation.reset(); + auto moved = std::move(allocation.value()); + allocation.reset(); + EXPECT_EQ(budget.snapshot().allocated[kHandoff], 60U); + auto split = moved.split(20); + ASSERT_TRUE(split); + moved.release(); + EXPECT_EQ(budget.snapshot().allocated[kHandoff], 20U); + split.reset(); + EXPECT_EQ(budget.snapshot().allocated[kHandoff], 0U); +} + +TEST(ResourceBudget, HostTreeAndProgressPartitionsAreIndependentlyBounded) +{ + ResourceBudget budget(limits(100), 4); + auto a = budget.make_account(limits(60)); + auto b = budget.make_account(limits(60)); + auto a_credit = ResourceBudget::try_reserve(a, BudgetClass::kHandoff, 60); + ASSERT_TRUE(a_credit); + EXPECT_FALSE(ResourceBudget::try_reserve(a, BudgetClass::kHandoff, 1)); + EXPECT_FALSE(ResourceBudget::try_reserve(b, BudgetClass::kHandoff, 41)); + auto b_credit = ResourceBudget::try_reserve(b, BudgetClass::kHandoff, 40); + ASSERT_TRUE(b_credit); + EXPECT_TRUE(ResourceBudget::try_reserve(a, BudgetClass::kReclaim, 60)); + EXPECT_TRUE(ResourceBudget::try_reserve(a, BudgetClass::kGapRepair, 60)); + EXPECT_EQ(budget.snapshot().reserved[kHandoff], 100U); +} + +TEST(ResourceBudget, AvailableCapacityIncludesHostUsageAndCreditConversion) +{ + ResourceBudget budget(limits(100), 2); + auto a = budget.make_account(limits(80)); + auto b = budget.make_account(limits(80)); + auto first = ResourceBudget::try_reserve(a, BudgetClass::kHandoff, 60); + auto second = ResourceBudget::try_reserve(b, BudgetClass::kHandoff, 30); + if (!first.has_value()) { + FAIL() << "initial reservation rejected"; + } + ASSERT_TRUE(second); + EXPECT_EQ(ResourceBudget::available(a, BudgetClass::kHandoff), 10U); + auto allocated = first.value().allocate(60); + ASSERT_TRUE(allocated); + EXPECT_EQ(ResourceBudget::available(a, BudgetClass::kHandoff), 10U); + second.reset(); + EXPECT_EQ(ResourceBudget::available(a, BudgetClass::kHandoff), 20U); + EXPECT_EQ(ResourceBudget::available(b, BudgetClass::kHandoff), 40U); + budget.close(); + EXPECT_EQ(ResourceBudget::available(a, BudgetClass::kHandoff), 0U); +} + +TEST(ResourceBudget, ReleasingAnotherTreeWakesRegisteredWaiter) +{ + ResourceBudget budget(limits(100), 2); + auto a = budget.make_account(limits(100)); + auto b = budget.make_account(limits(100)); + auto occupied = ResourceBudget::try_reserve(a, BudgetClass::kHandoff, 100); + ResourceBudget::Request request(b, BudgetClass::kHandoff, 100); + BudgetResult result; + auto operation = stdexec::connect(request.sender(), BudgetReceiver{&result}); + stdexec::start(operation); + EXPECT_EQ(result.values.load(), 0U); + EXPECT_EQ(budget.snapshot().waiters, 1U); + occupied.reset(); + EXPECT_EQ(result.values.load(), 1U); + EXPECT_EQ(budget.snapshot().waiters, 0U); + auto credit = request.take_credit(); + if (!credit.has_value()) { + FAIL() << "completed waiter did not receive credit"; + } + EXPECT_FALSE(request.take_credit()); + EXPECT_EQ(credit.value().amount(), 100U); + credit.reset(); + EXPECT_EQ(budget.snapshot().reserved[kHandoff], 0U); +} + +TEST(ResourceBudget, CancellationAndCloseDrainWaitersButKeepOwnedAllocation) +{ + ResourceBudget budget(limits(10), 2); + auto account = budget.make_account(limits(10)); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 10); + if (!occupied.has_value()) { + FAIL() << "initial reservation rejected"; + } + auto allocation = occupied.value().allocate(10); + ResourceBudget::Request first(account, BudgetClass::kHandoff, 1); + ResourceBudget::Request second(account, BudgetClass::kHandoff, 1); + BudgetResult a; + BudgetResult b; + auto op_a = stdexec::connect(first.sender(), BudgetReceiver{&a}); + auto op_b = stdexec::connect(second.sender(), BudgetReceiver{&b}); + stdexec::start(op_a); + stdexec::start(op_b); + first.request_stop(); + first.request_stop(); + EXPECT_EQ(a.stopped.load(), 1U); + EXPECT_EQ(b.stopped.load(), 0U); + budget.close(); + EXPECT_EQ(b.stopped.load(), 1U); + EXPECT_EQ(budget.snapshot().allocated[kHandoff], 10U); + EXPECT_FALSE(ResourceBudget::try_reserve(account, BudgetClass::kReclaim, 1)); + allocation.reset(); + EXPECT_EQ(budget.snapshot().allocated[kHandoff], 0U); +} + +TEST(ResourceBudget, ImpossibleAndExcessWaitersFailInsteadOfWaitingForever) +{ + ResourceBudget budget(limits(10), 1); + auto account = budget.make_account(limits(10)); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 10); + ResourceBudget::Request waiting(account, BudgetClass::kHandoff, 1); + ResourceBudget::Request overflow(account, BudgetClass::kHandoff, 1); + ResourceBudget::Request impossible(account, BudgetClass::kHandoff, std::numeric_limits<size_t>::max()); + BudgetResult a; + BudgetResult b; + BudgetResult c; + auto op_a = stdexec::connect(waiting.sender(), BudgetReceiver{&a}); + auto op_b = stdexec::connect(overflow.sender(), BudgetReceiver{&b}); + auto op_c = stdexec::connect(impossible.sender(), BudgetReceiver{&c}); + stdexec::start(op_a); + stdexec::start(op_b); + stdexec::start(op_c); + EXPECT_EQ(b.errors.load(), 1U); + EXPECT_EQ(c.errors.load(), 1U); + waiting.request_stop(); + EXPECT_EQ(a.stopped.load(), 1U); +} + +TEST(ResourceBudget, ConcurrentRegistrationAndReleaseCannotLoseWakeup) +{ + ResourceBudget budget(limits(1), 1); + auto account = budget.make_account(limits(1)); + for (size_t attempt = 0; attempt < 100; ++attempt) { + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 1); + ASSERT_TRUE(occupied); + ResourceBudget::Request request(account, BudgetClass::kHandoff, 1); + BudgetResult result; + auto operation = stdexec::connect(request.sender(), BudgetReceiver{&result}); + std::thread register_waiter([&] { stdexec::start(operation); }); + occupied.reset(); + register_waiter.join(); + EXPECT_EQ(result.values.load(), 1U); + request.request_stop(); + } + EXPECT_EQ(budget.snapshot().reserved[kHandoff], 0U); + EXPECT_EQ(budget.snapshot().waiters, 0U); +} + +TEST(ResourceBudget, CompletionCanRegisterAnotherWaiterAfterTheQueueBecomesEmpty) +{ + ResourceBudget budget(limits(1), 1); + auto account = budget.make_account(limits(1)); + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 1); + ResourceBudget::Request first(account, BudgetClass::kHandoff, 1); + ResourceBudget::Request second(account, BudgetClass::kHandoff, 1); + BudgetResult result; + auto next = stdexec::connect(second.sender(), BudgetReceiver{&result}); + + auto operation = stdexec::connect(first.sender(), + ReentrantBudgetReceiver{.receiver = {&result}, .first = &first, .next = &next}); + stdexec::start(operation); + EXPECT_EQ(result.values.load(), 0U); + occupied.reset(); + EXPECT_EQ(result.values.load(), 2U); + EXPECT_EQ(result.errors.load(), 0U); + EXPECT_EQ(result.stopped.load(), 0U); + EXPECT_EQ(budget.snapshot().waiters, 0U); + EXPECT_TRUE(second.take_credit()); +} + +TEST(ResourceBudget, HandoffWaitersCannotConsumeReclaimRegistrationSlots) +{ + ResourceBudget budget(limits(1), 1); + auto account = budget.make_account(limits(1)); + auto handoff = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 1); + auto reclaim = ResourceBudget::try_reserve(account, BudgetClass::kReclaim, 1); + ResourceBudget::Request handoff_request(account, BudgetClass::kHandoff, 1); + ResourceBudget::Request reclaim_request(account, BudgetClass::kReclaim, 1); + BudgetResult a; + BudgetResult b; + auto op_a = stdexec::connect(handoff_request.sender(), BudgetReceiver{&a}); + auto op_b = stdexec::connect(reclaim_request.sender(), BudgetReceiver{&b}); + stdexec::start(op_a); + stdexec::start(op_b); + EXPECT_EQ(b.errors.load(), 0U); + reclaim.reset(); + EXPECT_EQ(b.values.load(), 1U); + EXPECT_EQ(a.values.load(), 0U); + handoff_request.request_stop(); +} + +TEST(ResourceBudget, AllocationOwnerCanOutliveClosedBudget) +{ + std::optional<ResourceBudget::Credit> allocation; + { + ResourceBudget budget(limits(100), 1); + auto account = budget.make_account(limits(100)); + auto reservation = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 100); + if (!reservation.has_value()) { + FAIL() << "initial reservation rejected"; + } + allocation = reservation.value().allocate(100); + } + if (!allocation.has_value()) { + FAIL() << "allocation transfer rejected"; + } + EXPECT_EQ(allocation.value().amount(), 100U); + allocation.reset(); +} + +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/resource_registration_test.cpp b/lib/crowdb-tree/tests/unit/resource_registration_test.cpp new file mode 100644 index 000000000..2f5c919b3 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/resource_registration_test.cpp @@ -0,0 +1,137 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/resource_budget.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <thread> + +namespace crowdb::tree::detail +{ +namespace +{ +ResourceBudget::Limits limits(size_t value) +{ + ResourceBudget::Limits result; + result.fill(value); + return result; +} + +struct Result +{ + std::atomic<size_t> value{0}; + std::atomic<size_t> error{0}; + std::atomic<size_t> stopped{0}; +}; + +struct Receiver +{ + using receiver_concept = stdexec::receiver_tag; + Result *result; + + void set_value() const noexcept + { + result->value.fetch_add(1); + } + + void set_error(Status /*unused*/) const noexcept + { + result->error.fetch_add(1); + } + + void set_stopped() const noexcept + { + result->stopped.fetch_add(1); + } +}; + +TEST(ResourceRegistration, PrepaidSlotSurvivesFullOrdinaryRegistrationAndCanBeReused) +{ + ResourceBudget budget(limits(1), 2); + auto account = budget.make_account(limits(1)); + auto registration = ResourceBudget::try_register(account, BudgetClass::kReclaim); + if (!registration) { + FAIL() << "budget registration unavailable"; + } + auto &slot = *registration; + for (size_t round = 0; round < 2; ++round) { + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kReclaim, 1); + ASSERT_TRUE(occupied); + ResourceBudget::Request ordinary(account, BudgetClass::kReclaim, 1); + ResourceBudget::Request excess(account, BudgetClass::kReclaim, 1); + ResourceBudget::Request chosen(account, BudgetClass::kReclaim, 1, &slot); + Result a; + Result b; + Result c; + auto op_a = stdexec::connect(ordinary.sender(), Receiver{&a}); + auto op_b = stdexec::connect(excess.sender(), Receiver{&b}); + auto op_c = stdexec::connect(chosen.sender(), Receiver{&c}); + stdexec::start(op_a); + stdexec::start(op_b); + stdexec::start(op_c); + EXPECT_EQ(b.error.load(), 1U); + EXPECT_EQ(c.error.load(), 0U); + EXPECT_EQ(budget.snapshot().waiters, 2U); + ordinary.request_stop(); + occupied.reset(); + EXPECT_EQ(a.stopped.load(), 1U); + EXPECT_EQ(c.value.load(), 1U); + auto granted = chosen.take_credit(); + ASSERT_TRUE(granted); + granted.reset(); + } + registration.reset(); + EXPECT_TRUE(ResourceBudget::try_register(account, BudgetClass::kReclaim)); +} + +TEST(ResourceRegistration, CancelReturnsTheSameSlotWithoutAHostRegistrationRace) +{ + ResourceBudget budget(limits(1), 1); + auto account = budget.make_account(limits(1)); + auto registration = ResourceBudget::try_register(account, BudgetClass::kGapRepair); + if (!registration) { + FAIL() << "budget registration unavailable"; + } + auto &slot = *registration; + auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kGapRepair, 1); + ASSERT_TRUE(occupied); + for (size_t round = 0; round < 100; ++round) { + ResourceBudget::Request request(account, BudgetClass::kGapRepair, 1, &slot); + Result result; + auto operation = stdexec::connect(request.sender(), Receiver{&result}); + stdexec::start(operation); + std::thread cancel([&request] { request.request_stop(); }); + cancel.join(); + EXPECT_EQ(result.stopped.load(), 1U); + EXPECT_EQ(result.error.load(), 0U); + EXPECT_EQ(budget.snapshot().waiters, 0U); + } + budget.close(); + EXPECT_FALSE(ResourceBudget::try_register(account, BudgetClass::kGapRepair)); + ResourceBudget::Request closed(account, BudgetClass::kGapRepair, 1, &slot); + Result result; + auto operation = stdexec::connect(closed.sender(), Receiver{&result}); + stdexec::start(operation); + EXPECT_EQ(result.stopped.load(), 1U); +} + +TEST(ResourceRegistration, StartupReservationFailureDoesNotConsumeAnotherCategory) +{ + ResourceBudget budget(limits(1), 1); + auto account = budget.make_account(limits(1)); + auto registration = ResourceBudget::try_register(account, BudgetClass::kReclaim); + if (!registration) { + FAIL() << "budget registration unavailable"; + } + auto &slot = *registration; + EXPECT_FALSE(ResourceBudget::try_register(account, BudgetClass::kReclaim)); + EXPECT_TRUE(ResourceBudget::try_register(account, BudgetClass::kIo)); + auto moved = std::move(slot); + registration.reset(); + EXPECT_FALSE(ResourceBudget::try_register(account, BudgetClass::kReclaim)); + moved = {}; + EXPECT_TRUE(ResourceBudget::try_register(account, BudgetClass::kReclaim)); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/resource_usage_test.cpp b/lib/crowdb-tree/tests/unit/resource_usage_test.cpp new file mode 100644 index 000000000..eb91bcf67 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/resource_usage_test.cpp @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/resource_budget.h" +#include "crowdb-tree/c_api.h" +#include "crowdb-tree/runtime_c_api.h" +#include "published_read_test_support.h" + +namespace crowdb::tree +{ +TEST(ResourceUsage, ClassifiesTreeAndHostPressureAndRetainsQueuedWaits) +{ + detail::ResourceBudget::Limits host{}; + host.fill(100); + auto limits = host; + limits[0] = 80; + detail::ResourceBudget budget(host, 4); + auto first = budget.make_account(limits); + auto second = budget.make_account(limits); + auto occupied = detail::ResourceBudget::try_reserve(first, detail::BudgetClass::kHandoff, 70); + if (!occupied) { + FAIL() << "initial credit unavailable"; + } + auto allocation = occupied->allocate(20); + ASSERT_TRUE(allocation); + EXPECT_FALSE(detail::ResourceBudget::try_reserve(first, detail::BudgetClass::kHandoff, 11)); + EXPECT_FALSE(detail::ResourceBudget::try_reserve(second, detail::BudgetClass::kHandoff, 40)); + auto observed = first->resource_usage()[0]; + EXPECT_EQ(observed.limit, 80U); + EXPECT_EQ(observed.used, 70U); + EXPECT_EQ(observed.reserved, 50U); + EXPECT_EQ(observed.allocated, 20U); + EXPECT_EQ(observed.denied_tree, 1U); + EXPECT_EQ(second->resource_usage()[0].denied_host, 1U); + EXPECT_EQ(budget.resource_usage()[0].used, 70U); + detail::ResourceBudget::Request request(second, detail::BudgetClass::kHandoff, 40); + test_read::Completion done; + auto operation = stdexec::connect(request.sender(), test_read::Receiver{.done = &done}); + stdexec::start(operation); + EXPECT_EQ(second->resource_usage()[0].waiters, 1U); + EXPECT_EQ(budget.resource_usage()[0].waits, 1U); + occupied.reset(); + EXPECT_EQ(done.values.load(), 1U); + EXPECT_EQ(second->resource_usage()[0].waiters, 0U); + EXPECT_GT(second->resource_usage()[0].wait_ns, 0U); + EXPECT_EQ(second->resource_usage()[1].waits, 0U); +} + +TEST(ResourceUsage, VersionedHostAndTreeSnapshotsAreNonDraining) +{ + ct_runtime *raw_runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &raw_runtime), 0); + std::unique_ptr<ct_runtime, decltype(&ct_runtime_free)> runtime(raw_runtime, &ct_runtime_free); + ct_options options{}; + ct_tree *raw_tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime.get(), nullptr, &raw_tree), 0); + std::unique_ptr<ct_tree, decltype(&ct_close)> tree(raw_tree, &ct_close); + ct_resource_usage_snapshot_v1 host{ + .struct_size = sizeof(host), .version = 1, .resource_mask = 0, .scope_id = 0, .resources = {}}; + auto local = host; + ASSERT_EQ(ct_runtime_resource_usage(runtime.get(), &host), 0); + ASSERT_EQ(ct_tree_resource_usage(tree.get(), &local), 0); + EXPECT_EQ(host.resource_mask, 0x3ffU); + EXPECT_EQ(local.resource_mask, 0xffcffU); + EXPECT_EQ(local.resources[12].reserved, local.resources[12].used); + EXPECT_NE(local.scope_id, 0U); + EXPECT_GT(local.resources[1].used, 0U); + EXPECT_GE(host.resources[1].used, local.resources[1].used); + EXPECT_GT(local.resources[6].used, 0U); + EXPECT_GT(host.resources[8].limit, 0U); + EXPECT_EQ(local.resources[13].limit, local.resources[6].limit); + EXPECT_EQ(local.resources[14].limit, local.resources[6].limit); + EXPECT_EQ(local.resources[15].limit, local.resources[6].limit + 2); + EXPECT_EQ(local.resources[16].limit, 4096U); + EXPECT_EQ(local.resources[17].limit, 1024U * 1024U); + EXPECT_EQ(local.resources[18].limit, 64U); + EXPECT_EQ(local.resources[19].limit, 128U * 1024U); + const auto saved = local; + ASSERT_EQ(ct_tree_resource_usage(tree.get(), &local), 0); + EXPECT_EQ(local.resources[0].denied_host, saved.resources[0].denied_host); + local.version = 2; + EXPECT_NE(ct_tree_resource_usage(tree.get(), &local), 0); + EXPECT_EQ(local.version, 2U); + EXPECT_EQ(local.scope_id, saved.scope_id); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/retirement_queue_test.cpp b/lib/crowdb-tree/tests/unit/retirement_queue_test.cpp new file mode 100644 index 000000000..61e341eac --- /dev/null +++ b/lib/crowdb-tree/tests/unit/retirement_queue_test.cpp @@ -0,0 +1,449 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "btree/retired_shared.h" +#include "btree/tree_metrics.h" +#include "crowdb-tree/memtable/memtable.h" + +#include <gtest/gtest.h> + +#include <chrono> +#include <latch> +#include <thread> + +namespace crowdb::tree::detail +{ +namespace +{ +struct TestBudget : AllocationBudget +{ + std::atomic<size_t> claimed{0}, physical{0}; + std::atomic<bool> reject{false}; + + bool claim(size_t bytes) noexcept override + { + if (reject.load()) { + return false; + } + claimed.fetch_add(bytes); + return true; + } + + void release(size_t bytes) noexcept override + { + claimed.fetch_sub(bytes); + } + + void record_allocation(AllocationKind /*kind*/, size_t bytes, bool acquired) noexcept override + { + if (acquired) { + physical.fetch_add(bytes); + } + else { + physical.fetch_sub(bytes); + } + } +}; + +struct Chain +{ + std::shared_ptr<Chain> next; + std::atomic<size_t> *destroyed; + + ~Chain() + { + destroyed->fetch_add(1); + } +}; + +TEST(RetirementQueue, LastOwnerSchedulesBoundedChainDisposalAndKeepsExactStorageCharged) +{ + auto budget = std::make_shared<TestBudget>(); + { + FlushRuntime runtime(1, 2, budget); + auto lane = std::make_shared<FlushRuntime::Lane>(2); + std::shared_ptr<RetirementQueue> queue; + ASSERT_TRUE(RetirementQueue::create(runtime, lane, budget, &queue).ok()); + std::atomic<size_t> destroyed{0}; + std::shared_ptr<Chain> chain; + for (size_t i = 0; i < 10000; ++i) { + chain = retired_shared<Chain>(budget, queue, std::move(chain), &destroyed); + } + + struct Gate + { + std::latch entered{1}, release{1}; + } gate; + + ASSERT_TRUE(runtime + .try_submit( + lane, + [](void *context) noexcept { + auto &gate = *static_cast<Gate *>(context); + gate.entered.count_down(); + gate.release.wait(); + }, + &gate) + .ok()); + gate.entered.wait(); + const auto before = budget->physical.load(); + chain.reset(); + EXPECT_EQ(destroyed.load(), 0U); + EXPECT_LT(budget->physical.load(), before); // independent shared control was freed + EXPECT_GT(budget->physical.load(), 10000 * sizeof(RetiredObject<Chain>)); + gate.release.count_down(); + lane->drain(); + EXPECT_EQ(destroyed.load(), 10000U); + const auto work = queue->snapshot(); + EXPECT_EQ(work.enqueued, 10000U); + EXPECT_EQ(work.freed, 10000U); + EXPECT_GT(work.cpu_steps, 1U); + EXPECT_LE(work.max_step_nodes, RetirementQueue::kNodesPerStep); + queue->close_and_drain(); + } + EXPECT_EQ(budget->physical.load(), 0U); + EXPECT_EQ(budget->claimed.load(), 0U); +} + +TEST(RetirementQueue, LateExternalOwnerFreesAfterExecutorDestructionWithoutRecursion) +{ + auto budget = std::make_shared<TestBudget>(); + std::shared_ptr<RetirementQueue> queue; + std::shared_ptr<Chain> chain; + std::atomic<size_t> destroyed{0}; + { + FlushRuntime runtime(1, 1, budget); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + ASSERT_TRUE(RetirementQueue::create(runtime, lane, budget, &queue).ok()); + for (size_t i = 0; i < 50000; ++i) { + chain = retired_shared<Chain>(budget, queue, std::move(chain), &destroyed); + } + queue->close_and_drain(); + } + EXPECT_EQ(destroyed.load(), 0U); + queue.reset(); + EXPECT_GT(budget->physical.load(), 50000 * sizeof(RetiredObject<Chain>)); + chain.reset(); + EXPECT_EQ(destroyed.load(), 50000U); + EXPECT_EQ(budget->physical.load(), 0U); + EXPECT_EQ(budget->claimed.load(), 0U); +} + +TEST(RetirementQueue, ClosedQueuesUseOneThreadTrampolineForCrossDomainChains) +{ + auto budget = std::make_shared<TestBudget>(); + std::shared_ptr<Chain> chain; + std::atomic<size_t> destroyed{0}; + { + FlushRuntime runtime(1, 1, budget); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + for (size_t i = 0; i < 3000; ++i) { + std::shared_ptr<RetirementQueue> queue; + ASSERT_TRUE(RetirementQueue::create(runtime, lane, budget, &queue).ok()); + chain = retired_shared<Chain>(budget, queue, std::move(chain), &destroyed); + queue->close_and_drain(); + } + } + chain.reset(); + EXPECT_EQ(destroyed.load(), 3000U); + EXPECT_EQ(budget->physical.load(), 0U); + EXPECT_EQ(budget->claimed.load(), 0U); +} + +TEST(RetirementQueue, RealMemTableLastOwnerSlicesNodesAndPreservesDetachedGuard) +{ + auto queue_budget = std::make_shared<TestBudget>(); + FlushRuntime runtime(1, 2, queue_budget); + auto lane = std::make_shared<FlushRuntime::Lane>(2); + std::shared_ptr<RetirementQueue> queue; + ASSERT_TRUE(RetirementQueue::create(runtime, lane, queue_budget, &queue).ok()); + ResourceBudget::Limits limits{}; + constexpr auto category = static_cast<size_t>(BudgetClass::kRouting); + limits[category] = 16 * 1024 * 1024; + ResourceBudget host(limits, 1); + host.set_retirement_queue(queue); + auto account = host.make_account(limits); + auto admitted = ResourceBudget::try_reserve(account, BudgetClass::kRouting, limits[category]); + ASSERT_TRUE(admitted); + if (!admitted) { + return; + } + auto credit = AllocationReservation::create(std::move(*admitted)); + const auto baseline = account->snapshot().allocated[category]; + auto table = reserved_shared<MemTable>(credit, 1, nullptr, credit); + for (size_t i = 0; i < 3000; ++i) { + std::string key = std::to_string(i); + buffer cell = buffer::alloc(64); + std::fill_n(cell.data(), cell.size(), uint8_t{17}); + ASSERT_TRUE(table->upsert(key, i + 1, std::move(cell))); + } + auto guard = table->read_guard(credit); + const auto *cell = table->find("1024"); + ASSERT_NE(cell, nullptr); + table.reset(); + lane->drain(); + EXPECT_GT(account->snapshot().allocated[category], baseline); + EXPECT_EQ(cell->cell.size(), 64U); + guard = {}; + lane->drain(); + EXPECT_EQ(account->snapshot().allocated[category], baseline); + EXPECT_LE(queue->snapshot().max_step_nodes, RetirementQueue::kNodesPerStep); + credit->seal(); + credit.reset(); + EXPECT_EQ(account->snapshot().allocated[category], 0U); + queue->close_and_drain(); +} + +TEST(RetirementQueue, ConcurrentLateOwnersAndNestedReleasesDrainExactlyOnce) +{ + auto budget = std::make_shared<TestBudget>(); + std::shared_ptr<RetirementQueue> queue; + { + FlushRuntime runtime(1, 1, budget); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + ASSERT_TRUE(RetirementQueue::create(runtime, lane, budget, &queue).ok()); + queue->close_and_drain(); + } + std::atomic<size_t> destroyed{0}; + std::array<std::shared_ptr<Chain>, 8> chains; + for (auto &chain : chains) { + for (size_t i = 0; i < 2000; ++i) { + chain = retired_shared<Chain>(budget, queue, std::move(chain), &destroyed); + } + } + std::array<std::thread, 8> threads; + std::latch start{8}; + for (size_t i = 0; i < threads.size(); ++i) { + threads[i] = std::thread([&, i] { + start.count_down(); + start.wait(); + chains[i].reset(); + }); + } + for (auto &thread : threads) { + thread.join(); + } + EXPECT_EQ(destroyed.load(), 16000U); + EXPECT_EQ(queue->snapshot().freed, 16000U); + queue.reset(); + EXPECT_EQ(budget->physical.load(), 0U); + EXPECT_EQ(budget->claimed.load(), 0U); +} + +class EpochRetirement : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_TRUE(RetirementQueue::create(runtime, lane, queue_budget, &queue).ok()); + host.set_retirement_queue(queue); + account = host.make_account(limits()); + auto admitted = ResourceBudget::try_reserve(account, BudgetClass::kRouting, limits()[1]); + ASSERT_TRUE(admitted); + if (!admitted) { + return; + } + credit = AllocationReservation::create(std::move(*admitted)); + } + + void TearDown() override + { + account->drain_retirements(); + credit->seal(); + credit.reset(); + EXPECT_EQ(account->snapshot().allocated[1], 0U); + queue->close_and_drain(); + } + + static ResourceBudget::Limits limits() + { + ResourceBudget::Limits result{}; + result[1] = 16 * 1024 * 1024; + return result; + } + + struct Entry : EpochManager::Deferred + { + AllocationLease credit; + std::atomic<size_t> *freed; + }; + + void enqueue(EpochManager &epoch, std::atomic<size_t> &freed) + { + AllocationLease lease(credit, sizeof(Entry), AllocationKind::kMetadata); + auto *entry = new Entry; + entry->credit = std::move(lease); + entry->freed = &freed; + entry->destroy = [](EpochManager::Deferred *node) noexcept { + auto *entry = static_cast<Entry *>(node); + auto lease = std::move(entry->credit); + entry->freed->fetch_add(1); + delete entry; + }; + epoch.defer(entry); + } + + std::shared_ptr<TestBudget> queue_budget = std::make_shared<TestBudget>(); + FlushRuntime runtime{1, 2, queue_budget}; + std::shared_ptr<FlushRuntime::Lane> lane = std::make_shared<FlushRuntime::Lane>(2); + std::shared_ptr<RetirementQueue> queue; + ResourceBudget host{limits(), 1}; + std::shared_ptr<ResourceBudget::Account> account; + std::shared_ptr<AllocationReservation> credit; +}; + +TEST_F(EpochRetirement, PeriodicCollectionStopsAtReaderAndResumesOnItsExit) +{ + EpochManager epoch(credit); + auto guard = epoch.enter(); + std::atomic<size_t> freed{0}; + for (size_t i = 0; i < 3000; ++i) { + enqueue(epoch, freed); + } + lane->drain(); + account->drain_retirements(); + EXPECT_EQ(freed.load(), 0U); + EXPECT_EQ(epoch.pending_retired(), 3000U); + const auto stopped = queue->snapshot().cpu_steps; + lane->drain(); + EXPECT_EQ(queue->snapshot().cpu_steps, stopped); + guard = {}; + lane->drain(); + account->drain_retirements(); + EXPECT_EQ(freed.load(), 3000U); + EXPECT_EQ(epoch.pending_retired(), 0U); + EXPECT_LE(queue->snapshot().max_step_nodes, 64U); +} + +TEST_F(EpochRetirement, ConcurrentDefersAreCollectedWithoutAnExplicitSweep) +{ + EpochManager epoch(credit); + std::atomic<size_t> freed{0}; + std::array<std::thread, 4> producers; + for (auto &thread : producers) { + thread = std::thread([&] { + for (size_t i = 0; i < 3000; ++i) { + enqueue(epoch, freed); + } + }); + } + for (auto &thread : producers) { + thread.join(); + } + lane->drain(); + account->drain_retirements(); + EXPECT_EQ(freed.load(), 12000U); + EXPECT_EQ(epoch.pending_retired(), 0U); +} + +struct Observed +{ + static constexpr bool observe_retirement = true; + std::shared_ptr<TreeWorkMetrics> metrics; + bool *destroyed; + + ~Observed() + { + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)].inflight, 1U); + *destroyed = true; + } +}; + +TEST_F(EpochRetirement, AggregateWaitMetricsEndAfterPhysicalObjectRelease) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + account->set_work_metrics(metrics); + bool destroyed = false; + auto value = retired_shared<Observed>(credit, queue, metrics, &destroyed); + value.reset(); + account->drain_retirements(); + const auto totals = metrics->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)]; + EXPECT_TRUE(destroyed); + EXPECT_EQ(totals.attempts, 1U); + EXPECT_EQ(totals.inflight, 0U); + EXPECT_EQ(totals.count, 1U); + EXPECT_EQ(totals.outcomes[static_cast<size_t>(TreeOutcome::Success)], 1U); +} + +struct LogicalObserved +{ + static constexpr bool observe_retirement = true; + RetirementObserver retirement; +}; + +TEST_F(EpochRetirement, LogicalRetirementIncludesOldOwnerHoldAndDoesNotDoubleCount) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto resources = std::make_shared<TreeResourceMetrics>(); + account->set_work_metrics(metrics); + account->set_resource_metrics(resources); + auto value = retired_shared<LogicalObserved>(credit, queue); + auto old_reader = value; + value->retirement.mark(); + value->retirement.mark(); + value.reset(); + account->drain_retirements(); + const auto held = metrics->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)]; + EXPECT_EQ(held.attempts, 1U); + EXPECT_EQ(held.inflight, 1U); + EXPECT_EQ(held.count, 0U); + EXPECT_EQ(resources->snapshot().retired_bytes, sizeof(RetiredObject<LogicalObserved>)); + old_reader.reset(); + account->drain_retirements(); + const auto released = metrics->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)]; + EXPECT_EQ(released.attempts, 1U); + EXPECT_EQ(released.inflight, 0U); + EXPECT_EQ(released.count, 1U); + EXPECT_EQ(resources->snapshot().retired_bytes, 0U); +} + +TEST_F(EpochRetirement, CopiedImmutableDescriptorsReceiveIndependentObservers) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto resources = std::make_shared<TreeResourceMetrics>(); + account->set_work_metrics(metrics); + account->set_resource_metrics(resources); + auto first = retired_shared<LogicalObserved>(credit, queue); + auto second = retired_shared<LogicalObserved>(credit, queue, *first); + first->retirement.mark(); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)].attempts, 1U); + second->retirement.mark(); + EXPECT_EQ(metrics->totals()[static_cast<size_t>(TreeStage::MemoryRetireWait)].attempts, 2U); + EXPECT_EQ(resources->snapshot().retired_bytes, 2 * sizeof(RetiredObject<LogicalObserved>)); + first.reset(); + account->drain_retirements(); + EXPECT_EQ(resources->snapshot().retired_bytes, sizeof(RetiredObject<LogicalObserved>)); + second.reset(); + account->drain_retirements(); + EXPECT_EQ(resources->snapshot().retired_bytes, 0U); +} + +TEST_F(EpochRetirement, EveryDeferralDuringCollectorCompletionHasItsOwnProgress) +{ + EpochManager epoch(credit); + std::atomic<size_t> freed{0}; + constexpr size_t entries = 50000; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + size_t acknowledged = 0; + for (size_t index = 1; index <= entries; ++index) { + enqueue(epoch, freed); + while (freed.load(std::memory_order_acquire) < index && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + if (freed.load(std::memory_order_acquire) != index) { + break; + } + acknowledged = index; + } + EXPECT_EQ(acknowledged, entries) << "pending=" << epoch.pending_retired() + << " queue=" << (queue->snapshot().enqueued - queue->snapshot().freed) + << " account=" << account->retirements_pending_for_tests(); + // Cleanup must remain possible after recording a missing autonomous wake. + if (acknowledged != entries) { + (void)epoch.try_reclaim(); + } + lane->drain(); + account->drain_retirements(); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/runtime_bootstrap_accounting_test.cpp b/lib/crowdb-tree/tests/unit/runtime_bootstrap_accounting_test.cpp new file mode 100644 index 000000000..b834c961d --- /dev/null +++ b/lib/crowdb-tree/tests/unit/runtime_bootstrap_accounting_test.cpp @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "runtime/tree_runtime_internal.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(RuntimeBootstrapAccounting, WaiterCapacityConsumesExactAdditionalHostStorage) +{ + TreeRuntimeOptions first_options; + first_options.workers = 1; + first_options.backend_workers = 1; + first_options.max_waiters_per_class = 8; + auto second_options = first_options; + second_options.max_waiters_per_class = 24; + std::shared_ptr<TreeRuntime> first; + std::shared_ptr<TreeRuntime> second; + ASSERT_TRUE(TreeRuntime::create(first_options, &first).ok()); + ASSERT_TRUE(TreeRuntime::create(second_options, &second).ok()); + const auto a = first->snapshot(); + const auto b = second->snapshot(); + constexpr auto additional = + 16 * detail::ResourceBudget::kClasses * sizeof(std::atomic<detail::ResourceBudget::Request *>); + EXPECT_EQ(b.allocated.routing_bytes - a.allocated.routing_bytes, additional); + EXPECT_EQ(a.reserved.routing_bytes - b.reserved.routing_bytes, additional); + EXPECT_EQ(a.allocated.routing_bytes + a.reserved.routing_bytes, first_options.lifecycle_bytes); + EXPECT_EQ(b.allocated.routing_bytes + b.reserved.routing_bytes, second_options.lifecycle_bytes); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/runtime_bootstrap_budget_test.cpp b/lib/crowdb-tree/tests/unit/runtime_bootstrap_budget_test.cpp new file mode 100644 index 000000000..341cd66a1 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/runtime_bootstrap_budget_test.cpp @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/resource_budget.h" +#include "runtime/bootstrap_budget.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +namespace +{ +TEST(RuntimeBootstrapBudget, ActualRootAndControlStorageConsumeTheConfiguredPartition) +{ + const auto capacity = size_t{65536}; + auto budget = BootstrapBudget::create(capacity); + const auto initial = budget->snapshot(); + EXPECT_GT(initial.allocated, sizeof(BootstrapBudget)); + EXPECT_EQ(initial.allocated + initial.reserved, capacity); + EXPECT_EQ(initial.physical[static_cast<size_t>(AllocationKind::kMetadata)], initial.allocated); + MetadataAllocator<uint64_t> allocator(budget); + auto *value = allocator.allocate(17); + EXPECT_EQ(budget->snapshot().allocated, initial.allocated + (17 * sizeof(uint64_t))); + budget->seal(); + EXPECT_EQ(budget->available(), 0U); + EXPECT_EQ(budget->snapshot().reserved, 0U); + EXPECT_FALSE(budget->claim(1)); + allocator.deallocate(value, 17); + EXPECT_EQ(budget->snapshot().allocated, initial.allocated); + EXPECT_EQ(budget->snapshot().physical, initial.physical); +} + +TEST(RuntimeBootstrapBudget, LedgerWaitersAndAccountControlsReleaseAtTheirActualOwners) +{ + auto bootstrap = BootstrapBudget::create(1024 * 1024); + const auto initial = bootstrap->snapshot(); + ResourceBudget::Limits limits{}; + limits[1] = 65536; + std::shared_ptr<ResourceBudget::Account> account; + { + ResourceBudget ledger(limits, 32, bootstrap); + const auto ledger_bytes = bootstrap->snapshot().allocated - initial.allocated; + EXPECT_GT(ledger_bytes, 32 * ResourceBudget::kClasses * sizeof(std::atomic<ResourceBudget::Request *>)); + account = ledger.make_account(limits, bootstrap); + EXPECT_GT(bootstrap->snapshot().allocated, initial.allocated + ledger_bytes); + } + EXPECT_GT(bootstrap->snapshot().allocated, initial.allocated); + account.reset(); + EXPECT_EQ(bootstrap->snapshot().allocated, initial.allocated); + EXPECT_EQ(bootstrap->snapshot().physical, initial.physical); +} + +TEST(RuntimeBootstrapBudget, InsufficientControlAndWaiterCreditFailWithoutPartialOwnership) +{ + EXPECT_THROW((void)BootstrapBudget::create(sizeof(BootstrapBudget)), std::bad_alloc); + auto bootstrap = BootstrapBudget::create(4096); + const auto initial = bootstrap->snapshot(); + ResourceBudget::Limits limits{}; + EXPECT_THROW((void)ResourceBudget(limits, 1024, bootstrap), std::bad_alloc); + EXPECT_EQ(bootstrap->snapshot().allocated, initial.allocated); + EXPECT_EQ(bootstrap->snapshot().physical, initial.physical); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/runtime_pool_storage_test.cpp b/lib/crowdb-tree/tests/unit/runtime_pool_storage_test.cpp new file mode 100644 index 000000000..319e11066 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/runtime_pool_storage_test.cpp @@ -0,0 +1,146 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/flush_runtime.h" +#include "runtime/bootstrap_budget.h" +#include "runtime/pool_storage.h" + +#include <gtest/gtest.h> + +#include <algorithm> +#include <atomic> +#include <exec/static_thread_pool.hpp> +#include <thread> + +namespace crowdb::tree::detail +{ +namespace +{ +class TestPoolBudget final : public AllocationBudget +{ + public: + explicit TestPoolBudget(size_t limit, size_t fail_at = SIZE_MAX) : limit(limit), fail_at(fail_at) + { + } + + bool claim(size_t bytes) noexcept override + { + if (calls.fetch_add(1) == fail_at || bytes > limit - used) { + return false; + } + used += bytes; + peak = std::max(peak, used); + return true; + } + + void release(size_t bytes) noexcept override + { + used -= bytes; + } + + void record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept override + { + if (kind != AllocationKind::kMetadata) { + wrong_kind = true; + } + if (acquired) { + physical += bytes; + } + else { + physical -= bytes; + } + } + + size_t limit; + size_t fail_at; + size_t used = 0; + size_t physical = 0; + size_t peak = 0; + std::atomic<size_t> calls{0}; + bool wrong_kind = false; +}; + +TEST(RuntimePoolStorage, FixedPoolAndNativeWorkerControlsUseActualMetadataBudget) +{ + const auto bound = FlushRuntime::allocation_bound(2, 4); + if (!bound) { + FAIL() << "pool storage bound overflow"; + } + auto budget = std::make_shared<TestPoolBudget>(*bound); + { + FlushRuntime pool(2, 4, budget); + EXPECT_GT(budget->used, 2 * PoolStorage::worker_bytes()); + EXPECT_EQ(budget->used, budget->physical); + EXPECT_LE(budget->peak, *bound); + EXPECT_FALSE(budget->wrong_kind); + auto lane = std::make_shared<FlushRuntime::Lane>(1); + std::atomic<size_t> ran{0}; + const auto calls = budget->calls.load(); + const auto used = budget->used; + for (size_t index = 0; index < 64; ++index) { + std::thread submitter([&] { + EXPECT_TRUE( + pool.try_submit( + lane, [](void *p) noexcept { static_cast<std::atomic<size_t> *>(p)->fetch_add(1); }, &ran) + .ok()); + }); + submitter.join(); + lane->drain(); + } + EXPECT_EQ(ran.load(), 64U); + EXPECT_EQ(budget->calls.load(), calls); + EXPECT_EQ(budget->used, used); + } + EXPECT_EQ(budget->used, 0U); + EXPECT_EQ(budget->physical, 0U); +} + +TEST(RuntimePoolStorage, FailureAtEveryStartupAllocationReturnsActualCapacity) +{ + const auto bound = FlushRuntime::allocation_bound(2, 4); + if (!bound) { + FAIL() << "pool storage bound overflow"; + } + auto successful = std::make_shared<TestPoolBudget>(*bound); + { + FlushRuntime pool(2, 4, successful); + } + ASSERT_EQ(successful->used, 0U); + const auto allocations = successful->calls.load(); + ASSERT_GT(allocations, 64U); + for (size_t fail = 0; fail < allocations; ++fail) { + SCOPED_TRACE(fail); + auto budget = std::make_shared<TestPoolBudget>(*bound, fail); + bool rejected = false; + try { + FlushRuntime pool(2, 4, budget); + } + catch (const std::bad_alloc &) { + rejected = true; + } + EXPECT_TRUE(rejected); + EXPECT_EQ(budget->used, 0U); + EXPECT_EQ(budget->physical, 0U); + } +} + +TEST(RuntimePoolStorage, OveralignedAllocationsFreeBeforeReturningBudget) +{ + auto budget = BootstrapBudget::create(65536); + const auto before = budget->snapshot(); + PoolStorage storage(budget); + auto *value = storage.allocate(1024, 256); + EXPECT_EQ(reinterpret_cast<uintptr_t>(value) % 256, 0U); + EXPECT_EQ(budget->snapshot().allocated, before.allocated + 1024); + storage.deallocate(value, 1024, 256); + EXPECT_EQ(budget->snapshot().allocated, before.allocated); + EXPECT_EQ(budget->snapshot().physical, before.physical); +} + +TEST(RuntimePoolStorage, UnrepresentableWorkerGeometryRejectsWithoutStartingThreads) +{ + EXPECT_FALSE(FlushRuntime::allocation_bound(size_t{UINT16_MAX} + 1, 1)); + EXPECT_FALSE(exec::static_thread_pool::allocation_bound(UINT32_MAX, {SIZE_MAX, 8})); + EXPECT_FALSE(exec::static_thread_pool::allocation_bound(UINT32_MAX, {32, SIZE_MAX})); +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/runtime_registry_test.cpp b/lib/crowdb-tree/tests/unit/runtime_registry_test.cpp new file mode 100644 index 000000000..1874c1856 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/runtime_registry_test.cpp @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +auto reserve_remaining_routing(Crowdbtree &tree) +{ + auto *core = TreeCoreAccess::get(tree); + auto account = core->binding()->account(); + const auto stats = account->snapshot(); + const auto index = static_cast<size_t>(detail::BudgetClass::kRouting); + const auto available = + core->binding()->options().limits.routing_bytes - stats.allocated[index] - stats.reserved[index]; + return detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, available); +} +} // namespace + +TEST(RuntimeRegistry, TreeRegistryAndNamesUseActualRoutingQuotaAndFailedReinitPreservesTheOldRegistry) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + Crowdbtree tree(config); + const auto before = tree.tree_state_stats().resources; + tree.init_metrics("runtime", "mem"); + const auto after = tree.tree_state_stats().resources; + EXPECT_GT(after.metadata_bytes, before.metadata_bytes); + EXPECT_GT(after.allocated_bytes, before.allocated_bytes); + EXPECT_GT(after.allocated_bytes + after.reserved_bytes, before.allocated_bytes + before.reserved_bytes); + const auto width = tree.max_name_len(); + { + auto held = reserve_remaining_routing(tree); + ASSERT_TRUE(held); + EXPECT_THROW(tree.init_metrics("replacement", "mem"), std::bad_alloc); + EXPECT_EQ(tree.max_name_len(), width); + } + const auto output = tree.flush_metrics_str(1.0, "2026-11-02T00:00:00Z", 0); + for (const auto *name : + {"runtime.frontier.published.g", "runtime.frontier.durable.g", "runtime.flush.repair.pending.g", + "runtime.checkpoint.inflight.g", "runtime.cache.frames.g", "runtime.io.read.inflight.bytes.g", + "runtime.io.write.inflight.bytes.g", "runtime.candidate.oldest.age.ns.g"}) { + EXPECT_NE(output.find(name), std::string::npos) << name; + } +} + +TEST(RuntimeRegistry, ExhaustedFlushScratchNeverThrowsAcrossEitherCEntryAndCanRetry) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_tree *tree = nullptr; + ct_options options{}; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + c_api_tree(tree)->init_metrics("runtime", "mem"); + { + auto held = reserve_remaining_routing(*c_api_tree(tree)); + ASSERT_TRUE(held); + EXPECT_EQ(ct_flush_metrics_str(tree, 1.0, "now", 0), nullptr); + EXPECT_EQ(ct_flush_metrics_str_ext(tree, 1.0, "now", 0, 7, 7), nullptr); + } + auto *output = ct_flush_metrics_str(tree, 1.0, "now", 0); + ASSERT_NE(output, nullptr); + ct_free_string(output); + ct_close(tree); + ct_runtime_free(runtime); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/runtime_snapshot_test.cpp b/lib/crowdb-tree/tests/unit/runtime_snapshot_test.cpp new file mode 100644 index 000000000..313ff1fdb --- /dev/null +++ b/lib/crowdb-tree/tests/unit/runtime_snapshot_test.cpp @@ -0,0 +1,104 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/runtime_c_api.h" +#include "runtime/tree_runtime_binding.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct WaitReceiver +{ + using receiver_concept = stdexec::receiver_tag; + std::atomic<bool> *stopped; + + void set_value() const noexcept + { + ADD_FAILURE() << "fully occupied budget granted a waiter"; + } + + void set_error(Status status) const noexcept + { + ADD_FAILURE() << status.to_string(); + } + + void set_stopped() const noexcept + { + stopped->store(true); + } +}; + +TEST(RuntimeSnapshot, SeparatesExecutorDurationsAndIncludesCancelledBudgetWaits) +{ + TreeRuntimeOptions options; + options.limits.handoff_bytes = 64; + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create(options, &runtime).ok()); + TreeBindingOptions tree; + tree.limits.handoff_bytes = 64; + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, tree, &binding).ok()); + const auto before = runtime->snapshot(); + std::atomic<size_t> completed{0}; + const auto run = [](void *context) noexcept { static_cast<std::atomic<size_t> *>(context)->fetch_add(1); }; + ASSERT_TRUE(binding->cpu()->try_submit(binding->lane(), run, &completed).ok()); + ASSERT_TRUE(binding->backend_cpu()->try_submit(binding->backend_lane(), run, &completed).ok()); + binding->drain(); + EXPECT_EQ(completed.load(), 2U); + const auto after = runtime->snapshot(); + EXPECT_GT(after.cpu.completed, before.cpu.completed); + EXPECT_GT(after.backend.completed, before.backend.completed); + EXPECT_GT(after.cpu.execute_ns, before.cpu.execute_ns); + EXPECT_GT(after.backend.execute_ns, before.backend.execute_ns); + EXPECT_GT(after.cpu.queue_ns, before.cpu.queue_ns); + EXPECT_GT(after.backend.queue_ns, before.backend.queue_ns); + EXPECT_EQ(after.queued, 0U); + EXPECT_EQ(after.inflight, 0U); + + auto occupied = detail::ResourceBudget::try_reserve(binding->account(), detail::BudgetClass::kHandoff, 64); + ASSERT_TRUE(occupied); + detail::ResourceBudget::Request request(binding->account(), detail::BudgetClass::kHandoff, 1); + std::atomic<bool> stopped{false}; + auto operation = stdexec::connect(request.sender(), WaitReceiver{&stopped}); + stdexec::start(operation); + EXPECT_EQ(runtime->snapshot().budget_waiters, 1U); + runtime->stop_admission(); + EXPECT_TRUE(stopped.load()); + const auto terminal = runtime->snapshot(); + EXPECT_EQ(terminal.budget_waiters, 0U); + EXPECT_EQ(terminal.budget_cancelled, before.budget_cancelled + 1); + EXPECT_GT(terminal.budget_wait_ns, before.budget_wait_ns); + EXPECT_GT(terminal.budget_grants, before.budget_grants); + EXPECT_EQ(runtime->snapshot().budget_wait_ns, terminal.budget_wait_ns); +} + +TEST(RuntimeSnapshot, CAbiRejectsUnknownLayoutsWithoutChangingOutput) +{ + ct_runtime *raw = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &raw), 0); + std::unique_ptr<ct_runtime, decltype(&ct_runtime_free)> runtime(raw, &ct_runtime_free); + ct_runtime_snapshot_v1 sample{}; + sample.struct_size = sizeof(sample); + sample.version = 2; + sample.bindings = 123; + EXPECT_EQ(ct_runtime_snapshot(runtime.get(), &sample), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(sample.bindings, 123U); + sample.version = 1; + --sample.struct_size; + EXPECT_EQ(ct_runtime_snapshot(runtime.get(), &sample), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(sample.bindings, 123U); + sample.struct_size = sizeof(sample); + ASSERT_EQ(ct_runtime_snapshot(runtime.get(), &sample), 0); + EXPECT_EQ(sample.bindings, 0U); + EXPECT_GT(sample.allocated.routing_bytes, 0U); + const auto allocated = sample.allocated.routing_bytes; + ASSERT_EQ(ct_runtime_snapshot(runtime.get(), &sample), 0); + EXPECT_EQ(sample.allocated.routing_bytes, allocated); + ct_runtime_stop_admission(runtime.get()); + ASSERT_EQ(ct_runtime_snapshot(runtime.get(), &sample), 0); + EXPECT_EQ(sample.stopped, 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/scan_output_test.cpp b/lib/crowdb-tree/tests/unit/scan_output_test.cpp new file mode 100644 index 000000000..609dbb408 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/scan_output_test.cpp @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_account.h" +#include "btree/scan_output.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::detail +{ +TEST(ScanOutput, GrowthClaimsBothBuffersAndFailurePreservesThePreviousAllocation) +{ + constexpr size_t capacity = 16384; + ResourceBudget budget({capacity, capacity, capacity, capacity, capacity, 1}, 1); + auto account = budget.make_account({capacity, capacity, capacity, capacity, capacity, 1}); + std::shared_ptr<AllocationAccount> output; + ASSERT_TRUE(AllocationAccount::create(account, BudgetClass::kRouting, &output).ok()); + constexpr auto category = static_cast<size_t>(BudgetClass::kRouting); + const auto baseline = account->snapshot().allocated[category]; + ScanPackedBuf packed(output); + reserve_scan_output(packed, 100, 128); + packed.append(std::string(100, 'v')); + EXPECT_EQ(packed.capacity(), 128U); + auto pressure = ResourceBudget::try_reserve(account, BudgetClass::kRouting, capacity - baseline - 128); + ASSERT_TRUE(pressure); + bool rejected = false; + try { + reserve_scan_output(packed, 100, 128); + } + catch (const std::bad_alloc &) { + rejected = true; + } + EXPECT_TRUE(rejected); + EXPECT_EQ(packed.capacity(), 128U); + EXPECT_EQ(packed.size(), 100U); + EXPECT_EQ(std::string(reinterpret_cast<const char *>(packed.data()), packed.size()), std::string(100, 'v')); + EXPECT_EQ(account->snapshot().allocated[category], baseline + 128); + pressure.reset(); + reserve_scan_output(packed, 100, 128); + EXPECT_EQ(packed.capacity(), 256U); + EXPECT_EQ(account->snapshot().allocated[category], baseline + 256); + output.reset(); + EXPECT_EQ(account->snapshot().allocated[category], baseline + 256); + packed = {}; + EXPECT_EQ(account->snapshot().allocated[category], 0U); + EXPECT_EQ(account->snapshot().reserved[category], 0U); +} +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/scheduled_page_store_test.cpp b/lib/crowdb-tree/tests/unit/scheduled_page_store_test.cpp new file mode 100644 index 000000000..07aa74fc9 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/scheduled_page_store_test.cpp @@ -0,0 +1,293 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "backend/local/scheduled_page_store.h" +#include "btree/tree_metrics.h" + +#include <gtest/gtest.h> + +#include <array> +#include <atomic> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +struct TestCompletion +{ + std::atomic<size_t> calls{0}; + Status status; + + AsyncCompletion callback(std::shared_ptr<const void> owner = {}) + { + return {.context = this, + .complete_fn = + [](void *context, Status result) { + auto *self = static_cast<TestCompletion *>(context); + self->status = std::move(result); + self->calls.fetch_add(1, std::memory_order_release); + self->calls.notify_all(); + }, + .owner = std::move(owner)}; + } + + void wait() + { + while (calls.load(std::memory_order_acquire) == 0) { + calls.wait(0, std::memory_order_acquire); + } + } +}; + +struct TestBlockingStore : MemPageStore +{ + mutable std::atomic<bool> entered{false}; + mutable std::atomic<bool> released{false}; + + Status read_at(uint64_t address, uint8_t *data, size_t length) const override + { + entered.store(true, std::memory_order_release); + entered.notify_all(); + while (!released.load(std::memory_order_acquire)) { + released.wait(false, std::memory_order_acquire); + } + return MemPageStore::read_at(address, data, length); + } + + void wait() const + { + while (!entered.load(std::memory_order_acquire)) { + entered.wait(false, std::memory_order_acquire); + } + } + + void release() const + { + released.store(true, std::memory_order_release); + released.notify_all(); + } +}; + +struct TestScheduledStore +{ + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<AsyncPageStore> scheduled; + + explicit TestScheduledStore(size_t io = 16, size_t queue = 64) + { + TreeRuntimeOptions options; + options.workers = 1; + options.backend_workers = 1; + options.backend_queue_limit = queue; + EXPECT_TRUE(TreeRuntime::create(options, &runtime).ok()); + TreeBindingOptions tree; + tree.limits.io_operations = io; + tree.backend_queue_limit = queue; + EXPECT_TRUE(TreeRuntimeBinding::create(runtime, tree, &binding).ok()); + } + + Status create(PageStore *sync, AsyncPageStore *async = nullptr) + { + return ScheduledPageStore::create(sync, async, {}, binding, binding->lifecycle_budget(), + binding->backend_lane(), &scheduled); + } + + ~TestScheduledStore() + { + binding->drain(); + } +}; + +TEST(ScheduledPageStore, BlockingReadDoesNotOccupyTheOnlyTreeCpuWorker) +{ + TestBlockingStore device; + TestScheduledStore test; + ASSERT_TRUE(test.create(&device).ok()); + const std::array<uint8_t, 4> source{1, 2, 3, 4}; + ASSERT_TRUE(device.write_at(0, source.data(), source.size()).ok()); + std::array<uint8_t, 4> output{}; + TestCompletion read; + test.scheduled->submit_read(0, output.data(), output.size(), read.callback()); + device.wait(); + TestCompletion cpu; + const auto status = test.binding->cpu()->try_submit( + test.binding->lane(), + [](void *context) noexcept { static_cast<TestCompletion *>(context)->callback().complete(Status::Ok()); }, + &cpu); + EXPECT_TRUE(status.ok()); + if (status.ok()) { + cpu.wait(); + } + EXPECT_EQ(read.calls.load(), 0U); + device.release(); + read.wait(); + EXPECT_TRUE(read.status.ok()); + EXPECT_EQ(output, source); +} + +TEST(ScheduledPageStore, FullCpuQueueUsesThePrepaidBoundedIoWindow) +{ + TestBlockingStore device; + TestScheduledStore test(8, 1); + ASSERT_TRUE(test.create(&device).ok()); + std::array<uint8_t, 4> bytes{}; + ASSERT_TRUE(device.write_at(0, bytes.data(), bytes.size()).ok()); + std::array<TestCompletion, 8> accepted; + TestCompletion rejected; + test.scheduled->submit_read(0, bytes.data(), bytes.size(), accepted[0].callback()); + device.wait(); + for (size_t index = 1; index < accepted.size(); ++index) { + test.scheduled->submit_read(0, bytes.data(), bytes.size(), accepted[index].callback()); + EXPECT_EQ(accepted[index].calls.load(), 0U); + } + test.scheduled->submit_read(0, bytes.data(), bytes.size(), rejected.callback()); + EXPECT_EQ(rejected.calls.load(), 1U); + EXPECT_EQ(rejected.status.code(), Code::kResourceExhausted); + device.release(); + for (auto &completion : accepted) { + completion.wait(); + EXPECT_TRUE(completion.status.ok()); + EXPECT_EQ(completion.calls.load(), 1U); + } +} + +struct TestUncertainStore : AsyncPageStore +{ + AsyncCompletion retained; + bool throws = false; + + uint64_t submit_read(PageAddr /*address*/, void * /*buffer*/, size_t /*length*/, AsyncCompletion complete) override + { + retained = std::move(complete); + if (throws) { + throw std::runtime_error("uncertain submission"); + } + return 1; + } + + uint64_t submit_write(PageAddr /*address*/, const void * /*buffer*/, size_t /*length*/, + AsyncCompletion complete) override + { + return submit_read(0, nullptr, 0, std::move(complete)); + } + + Status submit_fsync(AsyncCompletion complete) override + { + submit_read(0, nullptr, 0, std::move(complete)); + return Status::Ok(); + } + + void cancel(uint64_t /*operation*/) override + { + } +}; + +TEST(ScheduledPageStore, UncertainThrowKeepsBufferUntilLateTransportDisposalAndCompletesOnce) +{ + TestUncertainStore device; + device.throws = true; + TestScheduledStore test; + ASSERT_TRUE(test.create(nullptr, &device).ok()); + auto owner = std::make_shared<std::array<uint8_t, 4>>(); + std::weak_ptr<const void> weak = owner; + TestCompletion done; + test.scheduled->submit_read(0, owner->data(), owner->size(), done.callback(owner)); + owner.reset(); + done.wait(); + test.binding->drain(); + EXPECT_EQ(done.status.code(), Code::kIoError); + EXPECT_FALSE(weak.expired()); + device.retained.complete(Status::Ok()); + EXPECT_EQ(done.calls.load(), 1U); + device.retained = {}; + EXPECT_TRUE(weak.expired()); +} + +TEST(ScheduledPageStore, DroppedAcceptedCallbackReturnsTerminalAndInFlightAdmission) +{ + TestUncertainStore device; + TestScheduledStore test(1); + ASSERT_TRUE(test.create(nullptr, &device).ok()); + std::array<uint8_t, 4> bytes{}; + TestCompletion first; + TestCompletion rejected; + TestCompletion retry; + test.scheduled->submit_read(0, bytes.data(), bytes.size(), first.callback()); + test.binding->drain(); + test.scheduled->submit_read(0, bytes.data(), bytes.size(), rejected.callback()); + EXPECT_EQ(rejected.status.code(), Code::kResourceExhausted); + device.retained = {}; + first.wait(); + EXPECT_EQ(first.status.code(), Code::kUnavailable); + test.scheduled->submit_read(0, bytes.data(), bytes.size(), retry.callback()); + test.binding->drain(); + device.retained.complete(Status::Ok()); + device.retained = {}; + retry.wait(); + EXPECT_TRUE(retry.status.ok()); +} + +TEST(ScheduledPageStore, CountersIncludeMetadataAndCountOnlyFirstRealTerminal) +{ + MemPageStore device; + TestScheduledStore test; + ASSERT_TRUE(test.create(&device).ok()); + std::array<uint8_t, 8> bytes{}; + TestCompletion write; + auto callback = write.callback(); + callback.cause = TreeWorkCause::Checkpoint; + test.scheduled->submit_typed_write(PagePurpose::kPageIndex, 0, bytes.data(), bytes.size(), callback); + write.wait(); + ASSERT_TRUE(write.status.ok()); + TestCompletion read; + callback = read.callback(); + callback.cause = TreeWorkCause::Snapshot; + test.scheduled->submit_read(0, bytes.data(), bytes.size(), callback); + read.wait(); + ASSERT_TRUE(read.status.ok()); + TestCompletion missing; + callback = missing.callback(); + callback.cause = TreeWorkCause::Read; + test.scheduled->submit_read(100, bytes.data(), bytes.size(), callback); + missing.wait(); + EXPECT_FALSE(missing.status.ok()); + test.binding->drain(); + const auto counters = test.binding->work_metrics()->counters.snapshot(); + auto value = [&](TreeWorkCounter counter, TreeWorkCause cause) { + return counters[static_cast<size_t>(cause)][static_cast<size_t>(counter)]; + }; + EXPECT_EQ(value(TreeWorkCounter::WriteOperations, TreeWorkCause::Checkpoint), 1U); + EXPECT_EQ(value(TreeWorkCounter::WriteBytes, TreeWorkCause::Checkpoint), 8U); + EXPECT_EQ(value(TreeWorkCounter::ReadOperations, TreeWorkCause::Snapshot), 1U); + EXPECT_EQ(value(TreeWorkCounter::ReadBytes, TreeWorkCause::Snapshot), 8U); + EXPECT_EQ(value(TreeWorkCounter::ReadOperations, TreeWorkCause::Read), 1U); + EXPECT_EQ(value(TreeWorkCounter::ReadErrors, TreeWorkCause::Read), 1U); + EXPECT_EQ(value(TreeWorkCounter::ReadBytes, TreeWorkCause::Read), 0U); +} + +TEST(ScheduledPageStore, LateSuccessAfterTransportThrowDoesNotInventSuccessfulIoBytes) +{ + TestUncertainStore device; + device.throws = true; + TestScheduledStore test; + ASSERT_TRUE(test.create(nullptr, &device).ok()); + std::array<uint8_t, 4> bytes{}; + TestCompletion done; + auto callback = done.callback(); + callback.cause = TreeWorkCause::Read; + test.scheduled->submit_read(0, bytes.data(), bytes.size(), callback); + done.wait(); + test.binding->drain(); + device.retained.complete(Status::Ok()); + device.retained.complete(Status::Ok()); + device.retained = {}; + const auto counters = test.binding->work_metrics()->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Read)]; + EXPECT_EQ(done.calls.load(), 1U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::ReadOperations)], 1U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::ReadErrors)], 1U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::ReadBytes)], 0U); + EXPECT_EQ(counters[static_cast<size_t>(TreeWorkCounter::StaleCallbacks)], 2U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/snapshot_changes_test.cpp b/lib/crowdb-tree/tests/unit/snapshot_changes_test.cpp new file mode 100644 index 000000000..70dd55c6c --- /dev/null +++ b/lib/crowdb-tree/tests/unit/snapshot_changes_test.cpp @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/snapshot_changes.h" + +#include <gtest/gtest.h> + +#include <map> + +namespace crowdb::tree +{ +namespace +{ +SnapshotPageChange change(uint64_t pid, uint64_t version) +{ + return { + .page_id = pid, + .publication_seq = version, + .before = {.content = version - 1, .present = true}, + .after = {.content = version, .present = true} + }; +} + +SnapshotChanges populated() +{ + SnapshotChanges result; + for (uint64_t i = 0; i < 1024; ++i) { + result = result.replace(change(i * 1024, 1)); + } + return result; +} + +TEST(SnapshotChanges, SparseUpdatesMatchOrderedReferenceAndKeepCapturedRoots) +{ + SnapshotChanges selected; + std::map<uint64_t, uint64_t> expected; + for (size_t i = 0; i < 3000; ++i) { + const uint64_t pid = i % 3 == 0 ? (i * 2654435761ULL) % SnapshotChanges::kPageLimit : (i * 37) % 64; + selected = selected.replace(change(pid, i + 1)); + expected[pid] = i + 1; + } + selected = selected.replace(change(SnapshotChanges::kPageLimit - 1, 3001)); + expected[SnapshotChanges::kPageLimit - 1] = 3001; + EXPECT_EQ(selected.size(), expected.size()); + std::map<uint64_t, uint64_t> observed; + uint64_t next = 0; + auto collect = [](void *raw, const SnapshotPageChange &entry) { + static_cast<std::map<uint64_t, uint64_t> *>(raw)->emplace(entry.page_id, entry.after.content); + }; + while (!selected.visit_from(next, 7, &observed, collect)) { + } + EXPECT_EQ(observed, expected); + const auto captured = selected; + ASSERT_NE(captured.get(0), nullptr); + selected = selected.replace(change(0, 4000)); + EXPECT_FALSE(selected.same_root(captured)); + EXPECT_NE(captured.get(0)->after.content, selected.get(0)->after.content); + EXPECT_EQ(selected.get(0)->after.content, 4000U); + EXPECT_EQ(selected.get(SnapshotChanges::kPageLimit), nullptr); + EXPECT_THROW((void)selected.replace(change(SnapshotChanges::kPageLimit, 1)), std::invalid_argument); +} + +TEST(SnapshotChanges, DeletionRetainsOriginalRangeWithoutChangingCapturedVersion) +{ + auto initial = change(7, 2); + initial.before.range = {.lower = OwnedKey(""), .upper = OwnedKey("old")}; + initial.after.range = {.lower = OwnedKey("new"), .upper = std::nullopt}; + auto selected = SnapshotChanges{}.replace(initial); + const auto captured = selected; + auto removed = change(7, 3); + removed.after = {}; + selected = selected.replace(removed); + ASSERT_TRUE(selected.get(7)); + EXPECT_FALSE(selected.get(7)->after.present); + EXPECT_EQ(selected.get(7)->before.content, 1U); + EXPECT_EQ(selected.get(7)->before.range.lower, OwnedKey("")); + EXPECT_EQ(selected.get(7)->before.range.upper, OwnedKey("old")); + EXPECT_TRUE(captured.get(7)->after.present); + EXPECT_EQ(captured.get(7)->after.range.lower, OwnedKey("new")); +} + +TEST(SnapshotChanges, EffectsAccumulateWithoutLosingEarlierContentOrCapturedClassification) +{ + auto initial = change(7, 1); + initial.effect = SnapshotChangeEffect::SourceStructure; + auto selected = SnapshotChanges{}.replace(initial); + const auto captured = selected; + auto accepted = change(7, 2); + accepted.effect = SnapshotChangeEffect::AcceptedPrefix; + selected = selected.replace(accepted); + auto structure = change(7, 3); + structure.effect = SnapshotChangeEffect::SourceStructure; + selected = selected.replace(structure); + ASSERT_TRUE(selected.get(7)); + EXPECT_EQ(selected.get(7)->effect, SnapshotChangeEffect::SourceStructure | SnapshotChangeEffect::AcceptedPrefix); + EXPECT_EQ(captured.get(7)->effect, SnapshotChangeEffect::SourceStructure); + auto full = change(7, 4); + structure.publication_seq = 5; + structure.after.content = 5; + selected = selected.replace(full).replace(structure); + EXPECT_EQ(selected.get(7)->effect, SnapshotChangeEffect::FullContent | SnapshotChangeEffect::SourceStructure | + SnapshotChangeEffect::AcceptedPrefix); + EXPECT_EQ(selected.get(7)->before.content, 0); +} + +TEST(SnapshotChanges, SingleUpdateUsesBoundedCreditAndFailedCopyLeavesInputIntact) +{ + const auto original = populated(); + using detail::BudgetClass; + using detail::ResourceBudget; + constexpr auto category = static_cast<size_t>(BudgetClass::kRouting); + const auto bound = SnapshotChanges::change_allocation_bound(); + ResourceBudget::Limits limits{}; + limits[category] = bound; + ResourceBudget budget(limits, 1); + auto account = budget.make_account(limits); + auto grant = ResourceBudget::try_reserve(account, BudgetClass::kRouting, bound); + if (!grant.has_value()) { + ADD_FAILURE() << "snapshot update credit unavailable"; + return; + } + auto reservation = std::make_shared<detail::AllocationReservation>(std::move(*grant)); + { + auto updated = original.replace(change(513 * 1024, 2), reservation); + EXPECT_EQ(updated.size(), original.size()); + EXPECT_LE(budget.snapshot().allocated[category], bound); + EXPECT_EQ(updated.get(513 * 1024)->after.content, 2U); + EXPECT_EQ(original.get(513 * 1024)->after.content, 1U); + } + EXPECT_EQ(reservation->available(), bound); + reservation->seal(); + auto small = ResourceBudget::try_reserve(account, BudgetClass::kRouting, 1); + if (!small.has_value()) { + ADD_FAILURE() << "failure injection credit unavailable"; + return; + } + auto exhausted = std::make_shared<detail::AllocationReservation>(std::move(*small)); + EXPECT_THROW((void)original.replace(change(1, 3), exhausted), std::bad_alloc); + EXPECT_EQ(exhausted->available(), 1U); + EXPECT_EQ(original.get(1), nullptr); + EXPECT_EQ(original.size(), 1024U); +} + +TEST(SnapshotChanges, BootstrapUsesOneTreeWithinItsLinearCreditBound) +{ + std::vector<SnapshotPageChange> initial; + initial.reserve(1024); + for (uint64_t i = 0; i < 1024; ++i) { + initial.push_back(change(i * 4096, i + 1)); + } + const auto bound = SnapshotChanges::initial_allocation_bound(initial.size()); + if (!bound.has_value()) { + ADD_FAILURE() << "snapshot bootstrap bound overflowed"; + return; + } + detail::ResourceBudget::Limits limits{}; + const auto category = static_cast<size_t>(detail::BudgetClass::kRouting); + limits[category] = *bound; + detail::ResourceBudget budget(limits, 1); + auto grant = + detail::ResourceBudget::try_reserve(budget.make_account(limits), detail::BudgetClass::kRouting, *bound); + if (!grant.has_value()) { + ADD_FAILURE() << "snapshot bootstrap credit unavailable"; + return; + } + auto reservation = std::make_shared<detail::AllocationReservation>(std::move(*grant)); + { + auto selected = SnapshotChanges::from_sorted(initial, reservation); + EXPECT_EQ(selected.size(), initial.size()); + for (const auto &entry : initial) { + ASSERT_TRUE(selected.get(entry.page_id)); + EXPECT_EQ(selected.get(entry.page_id)->after.content, entry.after.content); + } + } + EXPECT_EQ(reservation->available(), *bound); + initial[1].page_id = initial[0].page_id; + EXPECT_THROW((void)SnapshotChanges::from_sorted(initial), std::invalid_argument); + EXPECT_FALSE(SnapshotChanges::initial_allocation_bound(SIZE_MAX)); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/snapshot_crc_test.cpp b/lib/crowdb-tree/tests/unit/snapshot_crc_test.cpp new file mode 100644 index 000000000..532193774 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/snapshot_crc_test.cpp @@ -0,0 +1,27 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/crc32c.h" +#include "snapshot/snapshot_crc.h" + +#include <gtest/gtest.h> + +#include <string> + +namespace crowdb::tree +{ +TEST(SnapshotCrc, ConcatenationMatchesOnePassAcrossEmptyAndLongPieces) +{ + std::string bytes(70001, '\0'); + for (size_t i = 0; i < bytes.size(); ++i) { + bytes[i] = static_cast<char>((i * 37) & 255U); + } + const auto *data = reinterpret_cast<const uint8_t *>(bytes.data()); + const auto whole = crowdb::common::crc32c(data, bytes.size()); + for (const size_t split : + {size_t{0}, size_t{1}, size_t{25}, size_t{33}, size_t{4096}, size_t{65536}, bytes.size()}) { + const auto prefix = crowdb::common::crc32c(data, split); + const auto suffix = crowdb::common::crc32c(data + split, bytes.size() - split); + EXPECT_EQ(snapshot_detail::crc_concatenate(prefix, suffix, bytes.size() - split), whole) << split; + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/snapshot_metadata_codec_test.cpp b/lib/crowdb-tree/tests/unit/snapshot_metadata_codec_test.cpp new file mode 100644 index 000000000..bd5c38ece --- /dev/null +++ b/lib/crowdb-tree/tests/unit/snapshot_metadata_codec_test.cpp @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/maptable/mapping_slot.h" +#include "snapshot/metadata_codec.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree::snapshot +{ +TEST(SnapshotMetadataCodec, SegmentMatchesExistingFormatsWithoutTouchingPadding) +{ + for (const auto word : {slot_word::pack_unloaded(9, 2), slot_word::pack_page_ref(17, 3)}) { + const SegmentImageHeader header{.seg_idx = 4, .generation = 9, .slot_count = 4, .live_count = 1}; + const std::vector<uint64_t> words{slot_word::kEmpty, word, 0, 0}; + std::vector<uint8_t> expected; + uint32_t expected_crc = 0; + encode_segment_image(header, words, &expected, &expected_crc); + std::vector<uint8_t> output(expected.size() + 11, 0xAA); + uint32_t crc = 0; + ASSERT_TRUE(encode_segment_into(header, words, output, &crc).ok()); + EXPECT_TRUE(std::equal(expected.begin(), expected.end(), output.begin())); + EXPECT_EQ(crc, expected_crc); + EXPECT_EQ(output.back(), 0xAA); + } +} + +TEST(SnapshotMetadataCodec, RejectsInvalidSegmentAndInsufficientOutput) +{ + SegmentImageHeader header{.slot_count = 1, .live_count = 1}; + std::vector<uint64_t> words{slot_word::pack_unloaded(1, 1)}; + std::vector<uint8_t> output(segment_image_encoded_size(1) - 1); + uint32_t crc = 0; + EXPECT_EQ(encode_segment_into(header, words, output, &crc).code(), Code::kInvalidArgument); + output.resize(output.size() + 1); + header.live_count = 2; + EXPECT_EQ(encode_segment_into(header, words, output, &crc).code(), Code::kInvalidArgument); + header.live_count = 1; + EXPECT_EQ(encode_segment_into(header, {}, output, &crc).code(), Code::kInvalidArgument); + EXPECT_EQ(encode_segment_into(header, words, output, nullptr).code(), Code::kInvalidArgument); +} + +TEST(SnapshotMetadataCodec, DirectoryMatchesExistingEncodingAcrossEveryPartialPiece) +{ + const std::vector<DirEntry> entries{ + {.seg_idx = 0, .generation = 2, .image_addr = 8192, .image_len = 8228, .image_crc = 42 }, + {.seg_idx = 7, .generation = 5, .image_addr = 20480, .image_len = 8228, .image_crc = 123} + }; + for (const auto count : {size_t{0}, entries.size()}) { + const std::vector<DirEntry> input(entries.begin(), entries.begin() + static_cast<ptrdiff_t>(count)); + std::vector<uint8_t> expected; + encode_segment_directory(input, &expected); + for (size_t window = 1; window <= 85; ++window) { + DirectoryEncoder encoder(input); + std::vector<uint8_t> output; + std::vector<uint8_t> piece(window, 0xAA); + while (!encoder.complete()) { + std::ranges::fill(piece, uint8_t{0xAA}); + size_t written = 0; + ASSERT_TRUE(encoder.next(piece, &written).ok()); + ASSERT_GT(written, 0U); + output.insert(output.end(), piece.begin(), piece.begin() + static_cast<ptrdiff_t>(written)); + if (written < window) { + EXPECT_EQ(piece[written], 0xAA); + } + } + EXPECT_EQ(output, expected); + EXPECT_EQ(encoder.crc(), common::crc32c(expected.data(), expected.size())); + size_t written = 1; + ASSERT_TRUE(encoder.next(piece, &written).ok()); + EXPECT_EQ(written, 0U); + } + } +} + +TEST(SnapshotMetadataCodec, AnchorMatchesExistingEncoderAndReaderChecksActualCrcField) +{ + CommitAnchor anchor{.magic = kAnchorMagic, + .format_version = kFormatVersion, + .snapshot_seq = 3, + .root_page_id = 17, + .last_applied_slot = 50, + .next_page_id = 23, + .segment_slots = 1024, + .segdir_addr = 8192, + .segdir_len = 52, + .segdir_crc = 42, + .leaf_count = 7, + .inner_count = 2}; + std::vector<uint8_t> expected; + encode_anchor(anchor, 4096, &expected); + std::vector<uint8_t> output(4096, 0xAA); + ASSERT_TRUE(encode_anchor_into(anchor, output).ok()); + EXPECT_EQ(output, expected); + MemPageStore store(4096); + // Padding is outside the CRC field. Previously the reader mistook offset + // 80 for the CRC rather than checking the encoded CRC at offset 76. + output[80] = 0xAA; + ASSERT_TRUE(store.write_at(0, output.data(), output.size()).ok()); + std::vector<CommitAnchor> anchors; + ASSERT_TRUE(read_valid_anchors(store, 4096, &anchors).ok()); + ASSERT_EQ(anchors.size(), 1U); + EXPECT_EQ(anchors[0].snapshot_seq, 3U); + EXPECT_EQ(anchors[0].last_applied_slot, 50U); + output[76] ^= 1; + ASSERT_TRUE(store.write_at(0, output.data(), output.size()).ok()); + ASSERT_TRUE(read_valid_anchors(store, 4096, &anchors).ok()); + EXPECT_TRUE(anchors.empty()); + EXPECT_EQ(encode_anchor_into(anchor, std::span(output).first(4095)).code(), Code::kInvalidArgument); +} +} // namespace crowdb::tree::snapshot diff --git a/lib/crowdb-tree/tests/unit/snapshot_metadata_decode_test.cpp b/lib/crowdb-tree/tests/unit/snapshot_metadata_decode_test.cpp new file mode 100644 index 000000000..b9e1cd19c --- /dev/null +++ b/lib/crowdb-tree/tests/unit/snapshot_metadata_decode_test.cpp @@ -0,0 +1,273 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-common/crc32c.h" +#include "crowdb-tree/maptable/mapping_slot.h" +#include "snapshot/commit_anchor.h" +#include "snapshot/metadata_decode.h" + +#include <gtest/gtest.h> + +#include <algorithm> + +namespace crowdb::tree::snapshot +{ +namespace +{ +void put_u32(std::span<uint8_t> bytes, size_t offset, uint32_t value) +{ + for (size_t i = 0; i < sizeof(value); ++i) { + bytes[offset + i] = static_cast<uint8_t>(value >> (8 * i)); + } +} + +void header_crc(std::span<uint8_t> bytes, size_t offset) +{ + put_u32(bytes, offset, common::crc32c(bytes.data(), offset)); +} + +std::vector<uint8_t> segment(std::vector<uint64_t> words, uint32_t live) +{ + std::vector<uint8_t> output; + encode_segment_image( + {.seg_idx = 3, .generation = 7, .slot_count = static_cast<uint32_t>(words.size()), .live_count = live}, words, + &output); + return output; +} + +const std::vector<DirEntry> kEntries{ + {.seg_idx = 1, .generation = 2, .image_addr = 8192, .image_len = 8228, .image_crc = 42 }, + {.seg_idx = 7, .generation = 5, .image_addr = 20480, .image_len = 8228, .image_crc = 123} +}; + +std::vector<uint8_t> directory(const std::vector<DirEntry> &entries = kEntries) +{ + std::vector<uint8_t> output; + encode_segment_directory(entries, &output); + return output; +} + +void expect_entries(std::span<const DirEntry> actual, std::span<const DirEntry> expected) +{ + ASSERT_EQ(actual.size(), expected.size()); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQ(actual[i].seg_idx, expected[i].seg_idx); + EXPECT_EQ(actual[i].generation, expected[i].generation); + EXPECT_EQ(actual[i].image_addr, expected[i].image_addr); + EXPECT_EQ(actual[i].image_len, expected[i].image_len); + EXPECT_EQ(actual[i].image_crc, expected[i].image_crc); + } +} +} // namespace + +TEST(SnapshotMetadataDecode, SegmentMatchesExistingFormatsAndIgnoresIuPadding) +{ + for (const auto word : {slot_word::pack_unloaded(0, 2), slot_word::pack_page_ref(17, 3)}) { + const std::vector<uint64_t> expected{0, word, slot_word::pack_unloaded(9, 1), 0}; + auto input = segment(expected, 2); + input.resize(4096, 0xAA); + std::array<uint64_t, 5> words{42, 42, 42, 42, 42}; + SegmentImageHeader header; + ASSERT_TRUE(decode_segment_into(input, &header, words).ok()); + EXPECT_TRUE(std::equal(expected.begin(), expected.end(), words.begin())); + EXPECT_EQ(words.back(), 42U); + EXPECT_EQ(header.seg_idx, 3U); + EXPECT_EQ(header.generation, 7U); + EXPECT_EQ(header.slot_count, expected.size()); + EXPECT_EQ(header.live_count, 2U); + EXPECT_EQ(header.format_version, slot_word::is_page_ref(word) ? 2 : 1); + EXPECT_EQ(header.body_crc, common::crc32c(input.data() + 32, expected.size() * sizeof(uint64_t))); + } + SegmentImageHeader header; + EXPECT_TRUE(decode_segment_into(segment({}, 0), &header, {}).ok()); +} + +TEST(SnapshotMetadataDecode, SegmentTruncationAndOutputCapacityAreCheckedWithoutPartialWrites) +{ + const auto input = segment({0, slot_word::pack_unloaded(1, 1)}, 1); + std::array<uint64_t, 2> words{42, 42}; + SegmentImageHeader header{.seg_idx = 99}; + for (size_t length = 0; length < input.size(); ++length) { + EXPECT_EQ(decode_segment_into(std::span(input).first(length), &header, words).code(), Code::kCorruption); + EXPECT_EQ(header.seg_idx, 99U); + EXPECT_EQ(words[0], 42U); + EXPECT_EQ(words[1], 42U); + } + EXPECT_EQ(decode_segment_into(input, &header, std::span(words).first(1)).code(), Code::kInvalidArgument); + EXPECT_EQ(decode_segment_into(input, nullptr, words).code(), Code::kInvalidArgument); +} + +TEST(SnapshotMetadataDecode, SegmentRejectsMagicCrcVersionFlagsCountsAndIllegalWords) +{ + const auto valid = segment({slot_word::pack_unloaded(1, 1)}, 1); + std::array<uint64_t, 1> words{42}; + SegmentImageHeader header{.seg_idx = 99}; + for (const size_t offset : {0U, 28U, 32U, 40U}) { + auto input = valid; + input[offset] ^= 1; + EXPECT_EQ(decode_segment_into(input, &header, words).code(), Code::kCorruption); + } + for (const auto [offset, value] : std::array<std::pair<size_t, uint32_t>, 5>{ + {{4, 3}, {4, 0x10001}, {20, UINT32_MAX}, {24, 0}, {24, 2}} + }) { + auto input = valid; + put_u32(input, offset, value); + header_crc(input, 28); + EXPECT_EQ(decode_segment_into(input, &header, words).code(), Code::kCorruption); + } + for (const uint64_t word : + {uint64_t{4}, uint64_t{6}, uint64_t{8}, slot_word::pack_unloaded(1, 0), slot_word::pack_page_ref(1, 0)}) { + EXPECT_EQ(decode_segment_into(segment({word}, 1), &header, words).code(), Code::kCorruption); + } + auto legacy_ref = segment({slot_word::pack_page_ref(1, 1)}, 1); + put_u32(legacy_ref, 4, 1); + header_crc(legacy_ref, 28); + EXPECT_EQ(decode_segment_into(legacy_ref, &header, words).code(), Code::kCorruption); + EXPECT_EQ(header.seg_idx, 99U); + EXPECT_EQ(words[0], 42U); +} + +TEST(SnapshotMetadataDecode, DirectoryRoundTripsEveryWindowSizeIncludingEmptyDirectory) +{ + for (const auto &entries : {std::vector<DirEntry>{}, kEntries}) { + const auto input = directory(entries); + const auto bytes = static_cast<uint32_t>(input.size()); + uint32_t count = UINT32_MAX; + ASSERT_TRUE(directory_entry_count(std::span(input).first(16), bytes, &count).ok()); + ASSERT_EQ(count, entries.size()); + for (size_t window = 1; window <= input.size() + 1; ++window) { + std::vector<DirEntry> output(entries.size()); + DirectoryDecoder decoder(output, bytes); + for (size_t offset = 0; offset < input.size(); offset += window) { + ASSERT_TRUE( + decoder.next(std::span(input).subspan(offset, std::min(window, input.size() - offset))).ok()); + } + ASSERT_TRUE(decoder.complete()); + EXPECT_TRUE(decoder.finish().ok()); + EXPECT_TRUE(decoder.next({}).ok()); + EXPECT_EQ(decoder.crc(), common::crc32c(input.data(), input.size())); + expect_entries(output, entries); + } + } +} + +TEST(SnapshotMetadataDecode, DirectoryRoundTripsEveryTwoWindowSplit) +{ + const auto input = directory(); + for (size_t split = 0; split <= input.size(); ++split) { + std::array<DirEntry, 2> output{}; + DirectoryDecoder decoder(output, static_cast<uint32_t>(input.size())); + ASSERT_TRUE(decoder.next(std::span(input).first(split)).ok()); + ASSERT_TRUE(decoder.next(std::span(input).subspan(split)).ok()); + ASSERT_TRUE(decoder.finish().ok()); + expect_entries(output, kEntries); + } +} + +TEST(SnapshotMetadataDecode, DirectoryHeaderValidatesFlagsCountsAndExactLength) +{ + const auto valid = directory(); + const auto bytes = static_cast<uint32_t>(valid.size()); + uint32_t count = 99; + for (size_t length = 0; length < 16; ++length) { + EXPECT_EQ(directory_entry_count(std::span(valid).first(length), bytes, &count).code(), Code::kCorruption); + } + for (const auto [offset, value] : std::array<std::pair<size_t, uint32_t>, 5>{ + {{0, 0}, {4, 2}, {4, 0x10001}, {8, UINT32_MAX}, {8, 1}} + }) { + auto input = valid; + put_u32(input, offset, value); + header_crc(input, 12); + EXPECT_EQ(directory_entry_count(input, bytes, &count).code(), Code::kCorruption); + std::array<DirEntry, 2> output{}; + DirectoryDecoder decoder(output, bytes); + EXPECT_EQ(decoder.next(input).code(), Code::kCorruption); + EXPECT_FALSE(decoder.complete()); + } + EXPECT_EQ(directory_entry_count(valid, bytes - 1, &count).code(), Code::kCorruption); + EXPECT_EQ(directory_entry_count(valid, bytes + 1, &count).code(), Code::kCorruption); + EXPECT_EQ(directory_entry_count(valid, bytes, nullptr).code(), Code::kInvalidArgument); + EXPECT_EQ(count, 99U); +} + +TEST(SnapshotMetadataDecode, DirectoryRejectsTruncatedOversizedInputAndInsufficientOutput) +{ + const auto input = directory(); + for (size_t length = 0; length < input.size(); ++length) { + std::array<DirEntry, 2> output{}; + DirectoryDecoder decoder(output, static_cast<uint32_t>(input.size())); + EXPECT_TRUE(decoder.next(std::span(input).first(length)).ok()); + EXPECT_FALSE(decoder.complete()); + EXPECT_EQ(decoder.finish().code(), Code::kCorruption); + } + std::array<DirEntry, 2> output{}; + DirectoryDecoder short_output{std::span(output).first(1), static_cast<uint32_t>(input.size())}; + EXPECT_EQ(short_output.next(input).code(), Code::kInvalidArgument); + EXPECT_EQ(short_output.finish().code(), Code::kInvalidArgument); + DirectoryDecoder decoder(output, static_cast<uint32_t>(input.size())); + ASSERT_TRUE(decoder.next(input).ok()); + EXPECT_EQ(decoder.next(std::span(input).first(1)).code(), Code::kCorruption); + EXPECT_EQ(decoder.next({}).code(), Code::kCorruption); + EXPECT_FALSE(decoder.complete()); + for (const uint32_t bytes : {0U, 19U, 21U, UINT32_MAX}) { + EXPECT_EQ(DirectoryDecoder(output, bytes).finish().code(), Code::kCorruption); + } +} + +TEST(SnapshotMetadataDecode, DirectoryRejectsHeaderBodyTrailerCrcAndReservedEntryBits) +{ + const auto valid = directory(); + for (const size_t offset : {12U, 16U, 80U}) { + auto input = valid; + input[offset] ^= 1; + std::array<DirEntry, 2> output{}; + DirectoryDecoder decoder(output, static_cast<uint32_t>(input.size())); + EXPECT_EQ(decoder.next(input).code(), Code::kCorruption); + EXPECT_EQ(decoder.finish().code(), Code::kCorruption); + } + auto input = valid; + input[20] = 1; + put_u32(input, input.size() - 4, common::crc32c(input.data() + 16, input.size() - 20)); + std::array<DirEntry, 2> output{}; + DirectoryDecoder decoder(output, static_cast<uint32_t>(input.size())); + EXPECT_EQ(decoder.next(input).code(), Code::kCorruption); +} + +TEST(SnapshotMetadataDecode, AnchorSpanDecoderChecksLengthMagicFormatAndCrcWithoutChangingFailedOutput) +{ + const CommitAnchor anchor{.magic = kAnchorMagic, + .format_version = kFormatVersion, + .snapshot_seq = 7, + .root_page_id = 2, + .last_applied_slot = 99, + .next_page_id = 9, + .segment_slots = 1024, + .segdir_addr = 8192, + .segdir_len = 84, + .segdir_crc = 42, + .leaf_count = 4, + .inner_count = 1}; + std::vector<uint8_t> input; + encode_anchor(anchor, 4096, &input); + CommitAnchor output; + ASSERT_TRUE(decode_anchor(std::span(input).first(80), &output)); + EXPECT_EQ(output.snapshot_seq, 7U); + EXPECT_EQ(output.last_applied_slot, 99U); + EXPECT_EQ(output.leaf_count, 4U); + EXPECT_EQ(output.inner_count, 1U); + input[80] = 0xAA; + EXPECT_TRUE(decode_anchor(input, &output)); + for (size_t length = 0; length < 80; ++length) { + EXPECT_FALSE(decode_anchor(std::span(input).first(length), &output)); + } + EXPECT_FALSE(decode_anchor(input, nullptr)); + for (const size_t offset : {0U, 4U, 8U, 76U}) { + auto corrupt = input; + corrupt[offset] ^= 1; + if (offset < 8) { + header_crc(corrupt, 76); + } + EXPECT_FALSE(decode_anchor(corrupt, &output)); + EXPECT_EQ(output.snapshot_seq, 7U); + } +} +} // namespace crowdb::tree::snapshot diff --git a/lib/crowdb-tree/tests/unit/snapshot_publication_test.cpp b/lib/crowdb-tree/tests/unit/snapshot_publication_test.cpp new file mode 100644 index 000000000..86deb7633 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/snapshot_publication_test.cpp @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "snapshot/captured_write.h" +#include "snapshot/snapshot_baseline.h" +#include "snapshot_publication_test_support.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_snapshot_publication; + +TEST(SnapshotPublication, CapturedDeletionSurvivesCommitOfTheOlderView) +{ + PublicationCoordinator publisher; + auto first = leaf(1); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .replacement = first}}, .root_page_id = 0}).status.ok()); + auto captured = checkpoint(publisher, CheckpointAction::Capture, 1); + ASSERT_TRUE(captured.status.ok()); + ASSERT_TRUE(publish(publisher, + {.pages = {{.page_id = 0, .expected = first, .replacement = {}}}, .root_page_id = UINT64_MAX}) + .status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + const auto &changes = publisher.capture()->snapshot_changes; + ASSERT_EQ(changes.size(), 1U); + EXPECT_TRUE(changes.get(0)->before.present); + EXPECT_EQ(changes.get(0)->before.content, 1U); + EXPECT_FALSE(changes.get(0)->after.present); + EXPECT_TRUE(captured.checkpoint_view->snapshot_changes.get(0)->after.present); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); + auto second = checkpoint(publisher, CheckpointAction::Capture, 2); + ASSERT_TRUE(second.status.ok()); + EXPECT_FALSE(second.checkpoint_view->snapshot_changes.get(0)->after.present); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 2).status.ok()); + EXPECT_TRUE(publisher.capture()->snapshot_changes.empty()); +} + +TEST(SnapshotPublication, AbortRetainsLatestChangesAndTheNextCaptureSharesTheirRoot) +{ + PublicationCoordinator publisher; + auto first = leaf(1); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .replacement = first}}, .root_page_id = 0}).status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + auto newer = leaf(2); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .expected = first, .replacement = newer}}}).status.ok()); + auto before_abort = publisher.capture(); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Abort, 1).status.ok()); + auto after_abort = publisher.capture(); + EXPECT_TRUE(after_abort->snapshot_changes.same_root(before_abort->snapshot_changes)); + EXPECT_TRUE(after_abort->snapshot_changes_since_capture.empty()); + EXPECT_EQ(after_abort->snapshot_changes.get(0)->after.content, 2U); + EXPECT_FALSE(after_abort->snapshot_changes.get(0)->before.present); + auto captured = checkpoint(publisher, CheckpointAction::Capture, 2); + ASSERT_TRUE(captured.status.ok()); + EXPECT_TRUE(captured.checkpoint_view->snapshot_changes.same_root(after_abort->snapshot_changes)); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 2).status.ok()); + EXPECT_TRUE(publisher.capture()->snapshot_changes.empty()); +} + +TEST(SnapshotPublication, CompactChangesDoNotRetainOldPagesAndRepresentationChangesStayClean) +{ + PublicationCoordinator publisher; + auto first = leaf(1); + std::weak_ptr<PublishedPage> old = first; + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .replacement = first}}, .root_page_id = 0}).status.ok()); + auto second = leaf(2); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .expected = first, .replacement = second}}}).status.ok()); + first.reset(); + publisher.reclaim(); + EXPECT_TRUE(old.expired()); + auto selected = publisher.capture()->snapshot_changes; + auto represented = leaf(2); + ASSERT_TRUE( + publish(publisher, {.pages = {{.page_id = 0, .expected = second, .replacement = represented}}}).status.ok()); + EXPECT_TRUE(publisher.capture()->snapshot_changes.same_root(selected)); +} + +TEST(SnapshotPublication, BaselineMustBelongToTheExactCapturedFrontierAndToken) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + auto baseline = std::make_shared<SnapshotBaseline>(); + auto pages = std::make_shared<CapturedDurablePages>(nullptr); + pages->generation = 1; + baseline->generation = 1; + baseline->capture_token = 1; + baseline->snapshot_sequence = 1; + baseline->backend_id = 17; + baseline->pages = pages; + auto wrong = std::make_shared<SnapshotBaseline>(*baseline); + wrong->capture_token = 2; + EXPECT_EQ(publish(publisher, + { + .checkpoint = + CheckpointPublication{.action = CheckpointAction::Commit, .token = 1, .baseline = wrong} + }) + .status.code(), + Code::kInvalidArgument); + EXPECT_FALSE(publisher.capture()->snapshot_baseline); + ASSERT_TRUE( + publish(publisher, + { + .checkpoint = + CheckpointPublication{.action = CheckpointAction::Commit, .token = 1, .baseline = baseline} + }) + .status.ok()); + EXPECT_EQ(publisher.capture()->snapshot_baseline, baseline); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 2).status.ok()); + EXPECT_EQ( + publish(publisher, + { + .checkpoint = + CheckpointPublication{.action = CheckpointAction::Abort, .token = 2, .baseline = baseline} + }) + .status.code(), + Code::kInvalidArgument); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Abort, 2).status.ok()); + EXPECT_EQ(publisher.capture()->snapshot_baseline, baseline); +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(SnapshotPublication, FailureAfterPreparingChangesPublishesNeitherSelectionNorPendingState) +{ + PublicationCoordinator publisher; + auto first = leaf(1); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .replacement = first}}, .root_page_id = 0}).status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + const auto before = publisher.capture(); + publisher.set_before_root_for_tests([] { throw std::bad_alloc(); }); + EXPECT_EQ(publish(publisher, {.pages = {{.page_id = 0, .expected = first, .replacement = leaf(2)}}}).status.code(), + Code::kResourceExhausted); + publisher.set_before_root_for_tests({}); + EXPECT_EQ(publisher.capture(), before); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + EXPECT_TRUE(publisher.capture()->snapshot_changes.empty()); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, 0U); +} +#endif + +TEST(SnapshotPublication, PersistentMaintenanceSelectionUsesRoutingInsteadOfReclaimWorkingCredit) +{ + using detail::BudgetClass; + using detail::ResourceBudget; + constexpr auto routing = static_cast<size_t>(BudgetClass::kRouting); + constexpr auto reclaim = static_cast<size_t>(BudgetClass::kReclaim); + ResourceBudget::Limits limits{}; + limits[routing] = PublicationCoordinator::snapshot_change_reserve_bytes(); + limits[reclaim] = 1024 * 1024; + ResourceBudget budget(limits, 1); + auto account = budget.make_account(limits); + auto routing_grant = ResourceBudget::try_reserve(account, BudgetClass::kRouting, limits[routing]); + auto reclaim_grant = ResourceBudget::try_reserve(account, BudgetClass::kReclaim, limits[reclaim]); + if (!routing_grant.has_value() || !reclaim_grant.has_value()) { + ADD_FAILURE() << "separate maintenance partitions unavailable"; + return; + } + auto snapshot = std::make_shared<detail::AllocationReservation>(std::move(*routing_grant)); + auto working = std::make_shared<detail::AllocationReservation>(std::move(*reclaim_grant)); + PublicationCoordinator publisher; + auto page = leaf(1); + ASSERT_TRUE(publish(publisher, {.pages = {{.page_id = 0, .replacement = page}}, + .root_page_id = 0, + .metadata = working, + .snapshot_metadata = snapshot}) + .status.ok()); + snapshot->seal(); + working->seal(); + EXPECT_GT(budget.snapshot().allocated[routing], 0U); + const auto selected = checkpoint(publisher, CheckpointAction::Capture, 1); + ASSERT_TRUE(selected.status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Commit, 1).status.ok()); + publisher.reclaim(); + // The independent captured selection, rather than a live page or its + // Reclaim reservation, now owns the remaining Routing allocation. + EXPECT_TRUE(publisher.capture()->snapshot_changes.empty()); + EXPECT_EQ(selected.checkpoint_view->snapshot_changes.size(), 1U); + EXPECT_GT(budget.snapshot().allocated[routing], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_admission_test.cpp b/lib/crowdb-tree/tests/unit/source_admission_test.cpp new file mode 100644 index 000000000..50e1a6f4b --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_admission_test.cpp @@ -0,0 +1,289 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_admission.h" + +#include <gtest/gtest.h> + +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct Credits +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 2}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> reserve(size_t bytes = 65536) const + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, bytes); + if (!credit.has_value()) { + ADD_FAILURE() << "test metadata reservation failed"; + return {}; + } + return std::make_shared<detail::AllocationReservation>(std::move(credit.value())); + } +}; + +struct DrainResult +{ + std::atomic<size_t> values{0}; + std::atomic<size_t> errors{0}; + std::atomic<size_t> stopped{0}; +}; + +struct DrainReceiver +{ + using receiver_concept = stdexec::receiver_tag; + DrainResult *result; + + void set_value() const noexcept + { + result->values.fetch_add(1); + } + + void set_error(Status /*unused*/) const noexcept + { + result->errors.fetch_add(1); + } + + void set_stopped() const noexcept + { + result->stopped.fetch_add(1); + } +}; + +void count_notification(void *context) noexcept +{ + static_cast<std::atomic<size_t> *>(context)->fetch_add(1); +} + +TEST(MemTableFrozenNotification, CloseAndLastWriterNotifyEachSubscriberOnce) +{ + MemTable table; + ASSERT_TRUE(table.try_enter()); + ASSERT_TRUE(table.try_enter()); + std::atomic<size_t> notifications{0}; + MemTable::FrozenWaiter first(&count_notification, ¬ifications); + MemTable::FrozenWaiter second(&count_notification, ¬ifications); + ASSERT_TRUE(table.notify_when_frozen(first)); + table.close(); + ASSERT_TRUE(table.notify_when_frozen(second)); + EXPECT_EQ(notifications.load(), 0U); + table.leave(); + EXPECT_EQ(notifications.load(), 0U); + table.leave(); + EXPECT_EQ(notifications.load(), 2U); + EXPECT_FALSE(table.notify_when_frozen(first)); + table.close(); + EXPECT_EQ(notifications.load(), 2U); + MemTable::FrozenWaiter late(&count_notification, ¬ifications); + EXPECT_TRUE(table.notify_when_frozen(late)); + EXPECT_EQ(notifications.load(), 3U); +} + +TEST(MemTableFrozenNotification, ConcurrentRegistrationAndLastWriterCannotLoseNotification) +{ + for (size_t attempt = 0; attempt < 100; ++attempt) { + MemTable table; + ASSERT_TRUE(table.try_enter()); + table.close(); + std::atomic<size_t> notifications{0}; + MemTable::FrozenWaiter waiter(&count_notification, ¬ifications); + std::thread subscriber([&] { EXPECT_TRUE(table.notify_when_frozen(waiter)); }); + table.leave(); + subscriber.join(); + EXPECT_EQ(notifications.load(), 1U); + } +} + +TEST(SourceAdmission, NoProgressOrMissingCreditLeavesActiveOpen) +{ + SourceAdmission admission(5); + Credits credits; + auto initial = admission.catalog(); + std::atomic<uint64_t> contiguous{5}; + std::shared_ptr<SourceCapture> capture; + EXPECT_EQ(admission.try_capture(contiguous, {}, &capture).code(), Code::kInvalidArgument); + EXPECT_EQ(admission.try_capture(contiguous, credits.reserve(), &capture).code(), Code::kUnavailable); + contiguous.store(4); + EXPECT_EQ(admission.try_capture(contiguous, credits.reserve(), &capture).code(), Code::kUnavailable); + EXPECT_EQ(admission.catalog(), initial); + EXPECT_FALSE(initial->active->closed()); + EXPECT_FALSE(capture); + SourceAdmission::Writer writer; + EXPECT_TRUE(admission.try_admit(5, &writer).ok()); + EXPECT_EQ(writer.table(), initial->active); +} + +TEST(SourceAdmission, MetadataFailurePrecedesClosure) +{ + SourceAdmission admission; + Credits credits; + auto initial = admission.catalog(); + std::atomic<uint64_t> contiguous{1}; + std::shared_ptr<SourceCapture> capture; + EXPECT_EQ(admission.try_capture(contiguous, credits.reserve(1), &capture).code(), Code::kResourceExhausted); + EXPECT_FALSE(capture); + EXPECT_EQ(admission.catalog(), initial); + EXPECT_FALSE(initial->active->closed()); +} + +TEST(SourceAdmission, SuccessorNeedsPublicationAndCannotWidenCapturedFrontier) +{ + SourceAdmission admission; + Credits credits; + SourceAdmission::Writer old_writer; + ASSERT_TRUE(admission.try_admit(1, &old_writer).ok()); + EXPECT_TRUE(old_writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "old"))); + std::atomic<uint64_t> contiguous{1}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(admission.try_capture(contiguous, credits.reserve(), &capture).ok()); + EXPECT_EQ(capture->floor(), 0U); + EXPECT_EQ(capture->frontier(), 1U); + EXPECT_EQ(capture->expected_catalog()->active, old_writer.table()); + ASSERT_EQ(capture->next_catalog()->tables.size(), 2U); + SourceAdmission::Writer successor_writer; + EXPECT_EQ(admission.try_admit(1, &successor_writer).code(), Code::kUnavailable); + EXPECT_FALSE(admission.activate_successor(capture, capture->batch_id() + 1).ok()); + ASSERT_TRUE(admission.activate_successor(capture, capture->batch_id()).ok()); + ASSERT_TRUE(admission.try_admit(1, &successor_writer).ok()); + EXPECT_EQ(successor_writer.table(), capture->successor()); + EXPECT_TRUE(successor_writer.table()->upsert("b", 2, encode_cell_buf(2, OpKind::kPut, "successor"))); + successor_writer.release(); + contiguous.store(2); + DrainResult result; + auto operation = stdexec::connect(capture->drain_sender(), DrainReceiver{&result}); + stdexec::start(operation); + EXPECT_EQ(result.values.load(), 0U); + EXPECT_FALSE(admission.finish_covered(capture, capture->batch_id(), 1).ok()); + EXPECT_TRUE(old_writer.table()->upsert("future", 7, encode_cell_buf(7, OpKind::kPut, "future"))); + old_writer.release(); + EXPECT_EQ(result.values.load(), 1U); + EXPECT_TRUE(capture->drained()); + EXPECT_EQ(capture->frontier(), 1U); + ASSERT_TRUE(admission.finish_covered(capture, capture->batch_id(), 1).ok()); + EXPECT_EQ(admission.catalog()->floor, 1U); + EXPECT_EQ(admission.catalog()->tables.size(), 2U); + EXPECT_EQ(capture->phase(), SourceCapturePhase::kCovered); +} + +TEST(SourceAdmission, RepairReturnsSameCaptureAndRejectsUnrelatedCoverage) +{ + SourceAdmission admission; + Credits credits; + std::atomic<uint64_t> contiguous{3}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(admission.try_capture(contiguous, credits.reserve(), &capture).ok()); + ASSERT_TRUE(stdexec::sync_wait(capture->drain_sender()).has_value()); + ASSERT_TRUE(admission.activate_successor(capture, capture->batch_id()).ok()); + ASSERT_TRUE(admission.mark_repair(capture, capture->batch_id(), 0).ok()); + contiguous.store(99); + std::shared_ptr<SourceCapture> retried; + ASSERT_TRUE(admission.try_capture(contiguous, credits.reserve(), &retried).ok()); + EXPECT_EQ(retried, capture); + EXPECT_EQ(retried->frontier(), 3U); + EXPECT_FALSE(admission.mark_repair(capture, capture->batch_id() + 1, 0).ok()); + EXPECT_FALSE(admission.mark_repair(capture, capture->batch_id(), 1).ok()); + EXPECT_FALSE(admission.finish_covered(capture, capture->batch_id(), 99).ok()); + EXPECT_EQ(admission.catalog()->floor, 0U); + ASSERT_TRUE(admission.finish_covered(capture, capture->batch_id(), 3).ok()); + EXPECT_FALSE(admission.finish_covered(capture, capture->batch_id(), 3).ok()); + EXPECT_FALSE(admission.mark_repair(capture, capture->batch_id(), 0).ok()); +} + +TEST(SourceAdmission, CoveredAcknowledgementUsesPreallocatedStateAfterCreditSeals) +{ + SourceAdmission admission; + Credits credits; + auto reservation = credits.reserve(); + auto before = admission.catalog(); + std::atomic<uint64_t> contiguous{2}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(admission.try_capture(contiguous, reservation, &capture).ok()); + ASSERT_TRUE(stdexec::sync_wait(capture->drain_sender()).has_value()); + ASSERT_TRUE(admission.activate_successor(capture, capture->batch_id()).ok()); + reservation->seal(); + EXPECT_EQ(reservation->available(), 0U); + ASSERT_TRUE(admission.finish_covered(capture, capture->batch_id(), 2).ok()); + EXPECT_EQ(admission.catalog()->floor, 2U); + ASSERT_EQ(admission.catalog()->tables.size(), 1U); + EXPECT_EQ(admission.catalog()->active->durable_floor(), 2U); + EXPECT_EQ(before->floor, 0U); + EXPECT_EQ(before->tables.size(), 1U); + EXPECT_TRUE(before->active->closed()); +} + +TEST(SourceAdmission, CancellationDrainsWritersAndRetryKeepsSources) +{ + SourceAdmission admission; + Credits credits; + SourceAdmission::Writer writer; + ASSERT_TRUE(admission.try_admit(1, &writer).ok()); + std::atomic<uint64_t> contiguous{1}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(admission.try_capture(contiguous, credits.reserve(), &capture).ok()); + capture->request_stop(); + DrainResult result; + auto operation = stdexec::connect(capture->drain_sender(), DrainReceiver{&result}); + stdexec::start(operation); + EXPECT_EQ(result.stopped.load(), 0U); + writer.release(); + EXPECT_EQ(result.stopped.load(), 1U); + EXPECT_EQ(admission.catalog()->floor, 0U); + EXPECT_EQ(admission.catalog()->tables.size(), 1U); + ASSERT_TRUE(admission.mark_repair(capture, capture->batch_id(), 0).ok()); + ASSERT_TRUE(stdexec::sync_wait(capture->drain_sender()).has_value()); + EXPECT_EQ(capture->phase(), SourceCapturePhase::kRepairRequired); +} + +TEST(SourceAdmission, AcceptedCaptureActivatesAndAcknowledgesAfterIngressCloses) +{ + SourceAdmission admission; + Credits credits; + SourceAdmission::Writer writer; + ASSERT_TRUE(admission.try_admit(0, &writer).ok()); + std::atomic<uint64_t> contiguous{1}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(admission.try_capture(contiguous, credits.reserve(), &capture).ok()); + admission.close(); + SourceAdmission::Writer rejected; + EXPECT_EQ(admission.try_admit(1, &rejected).code(), Code::kUnavailable); + ASSERT_TRUE(admission.activate_successor(capture, capture->batch_id()).ok()); + writer.release(); + ASSERT_TRUE(stdexec::sync_wait(capture->drain_sender()).has_value()); + ASSERT_TRUE(admission.finish_covered(capture, capture->batch_id(), 1).ok()); + EXPECT_EQ(admission.catalog()->floor, 1U); + EXPECT_EQ(admission.try_admit(1, &rejected).code(), Code::kUnavailable); +} + +TEST(SourceAdmission, AsyncDrainOwnsSourcesAfterManagerAndCaptureAreReleased) +{ + auto admission = std::make_unique<SourceAdmission>(); + Credits credits; + SourceAdmission::Writer writer; + ASSERT_TRUE(admission->try_admit(1, &writer).ok()); + std::atomic<uint64_t> contiguous{1}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(admission->try_capture(contiguous, credits.reserve(), &capture).ok()); + DrainResult result; + auto operation = stdexec::connect(capture->drain_sender(), DrainReceiver{&result}); + stdexec::start(operation); + std::weak_ptr<MemTable> source = writer.table(); + capture.reset(); + admission.reset(); + EXPECT_FALSE(source.expired()); + writer.release(); + EXPECT_EQ(result.values.load(), 1U); + EXPECT_TRUE(source.expired()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_ledger_test.cpp b/lib/crowdb-tree/tests/unit/source_ledger_test.cpp new file mode 100644 index 000000000..fbe0faa1d --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_ledger_test.cpp @@ -0,0 +1,204 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_state.h" +#include "btree/source_admission.h" + +#include <gtest/gtest.h> + +#include <latch> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct TestSourceCredit +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> reserve(size_t bytes = 128 * 1024) const + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, bytes); + if (!credit) { + throw std::bad_alloc(); + } + return std::make_shared<detail::AllocationReservation>(std::move(*credit)); + } +}; + +TEST(SourceLedger, ActiveAndClosedWritersFreezeExactlyOnceThenDetachWhileReaderRetainsTable) +{ + auto ledger = std::make_shared<SourceMemoryLedger>(); + auto table = std::make_shared<MemTable>(); + table->attach_source_stats(ledger); + EXPECT_EQ(ledger->snapshot(), table->source_stats()); + ASSERT_TRUE(table->try_enter()); + ASSERT_TRUE(table->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "one"))); + ASSERT_TRUE(table->upsert("a", 2, encode_cell_buf(2, OpKind::kPut, "two"))); + EXPECT_EQ(ledger->snapshot(), table->source_stats()); + table->close(); + table->close(); + EXPECT_EQ(ledger->snapshot().freezing_writers, 1U); + EXPECT_EQ(ledger->snapshot().frozen_retained_bytes, 0U); + ASSERT_TRUE(table->upsert("b", 3, encode_cell_buf(3, OpKind::kPut, "three"))); + table->leave(); + EXPECT_EQ(ledger->snapshot(), table->source_stats()); + EXPECT_GT(ledger->snapshot().frozen_retained_bytes, 0U); + table->detach_source_stats(); + EXPECT_EQ(ledger->snapshot(), (SourceMemoryStats{.known = true})); + auto guard = table->read_guard(); + ASSERT_NE(table->find("a"), nullptr); + EXPECT_EQ(table->find("a")->slot, 2U); + guard = {}; + table.reset(); + EXPECT_EQ(ledger->snapshot(), (SourceMemoryStats{.known = true})); +} + +TEST(SourceLedger, SuccessorAndCoveredMembershipFollowAdmissionRatherThanOldCatalogOwners) +{ + TestSourceCredit credits; + SourceAdmission sources; + auto old = sources.catalog(); + auto ledger = old->memory; + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, &writer).ok()); + ASSERT_TRUE(writer.table()->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "one"))); + writer.release(); + std::atomic<uint64_t> frontier{1}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(sources.try_capture(frontier, credits.reserve(), &capture).ok()); + EXPECT_EQ(ledger->snapshot().tables, 1U); + EXPECT_EQ(ledger->snapshot().frozen_tables, 1U); + ASSERT_TRUE(sources.activate_successor(capture, capture->batch_id()).ok()); + EXPECT_EQ(ledger->snapshot().tables, 2U); + ASSERT_TRUE(stdexec::sync_wait(capture->drain_sender())); + ASSERT_TRUE(sources.finish_covered(capture, capture->batch_id(), 1).ok()); + EXPECT_EQ(ledger->snapshot().tables, 1U); + EXPECT_EQ(ledger->snapshot().records, 0U); + EXPECT_EQ(ledger->snapshot().memory, VersionMemory{}); + EXPECT_EQ(old->active->count(), 1U); + EXPECT_EQ(old->memory, ledger); +} + +TEST(SourceLedger, FailedCaptureDoesNotPublishCandidateAndOldGenerationWriterCannotTouchNewLedger) +{ + TestSourceCredit credits; + auto old = std::make_unique<SourceAdmission>(); + SourceAdmission next(0, 2); + const auto old_ledger = old->catalog()->memory; + const auto next_ledger = next.catalog()->memory; + SourceAdmission::Writer writer; + ASSERT_TRUE(old->try_admit(0, &writer).ok()); + std::atomic<uint64_t> frontier{1}; + std::shared_ptr<SourceCapture> rejected; + EXPECT_EQ(old->try_capture(frontier, credits.reserve(1), &rejected).code(), Code::kResourceExhausted); + EXPECT_EQ(old_ledger->snapshot().tables, 1U); + EXPECT_EQ(old_ledger->snapshot().frozen_tables, 0U); + old.reset(); + ASSERT_TRUE(writer.table()->upsert("late", 1, encode_cell_buf(1, OpKind::kPut, "old"))); + EXPECT_EQ(old_ledger->snapshot().records, 1U); + EXPECT_EQ(next_ledger->snapshot().records, 0U); + writer.release(); + EXPECT_EQ(old_ledger->snapshot(), (SourceMemoryStats{.known = true})); + EXPECT_EQ(next_ledger->snapshot().tables, 1U); +} + +TEST(SourceLedger, DetachCanPrecedeTheLastWritersBookkeepingWithoutLeakingTheLedger) +{ + auto ledger = std::make_shared<SourceMemoryLedger>(); + MemTable table; + table.attach_source_stats(ledger); + ASSERT_TRUE(table.try_enter()); + ASSERT_TRUE(table.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "one"))); + table.close(); + + struct Pause + { + std::latch entered{1}; + std::latch proceed{1}; + } pause; + + table.set_hook_for_tests(&pause, [](void *context, ConcurrentSkipList::PausePoint point, Slice) { + if (point == ConcurrentSkipList::PausePoint::kBeforeLeaveAccounting) { + auto &pause = *static_cast<Pause *>(context); + pause.entered.count_down(); + pause.proceed.wait(); + } + }); + std::thread writer([&] { table.leave(); }); + pause.entered.wait(); + std::atomic<bool> notified{false}; + MemTable::FrozenWaiter waiter( + [](void *context) noexcept { static_cast<std::atomic<bool> *>(context)->store(true); }, ¬ified); + ASSERT_TRUE(table.notify_when_frozen(waiter)); + EXPECT_TRUE(notified.load()); + table.detach_source_stats(); + pause.proceed.count_down(); + writer.join(); + table.set_hook_for_tests(nullptr, nullptr); + EXPECT_EQ(ledger->snapshot(), (SourceMemoryStats{.known = true})); +} + +TEST(SourceLedger, LateCloseDeltaAndEarlyFreezeRemovalConvergeWithoutRebindingAWriter) +{ + auto ledger = std::make_shared<SourceMemoryLedger>(); + MemTable table; + table.attach_source_stats(ledger); + ASSERT_TRUE(table.try_enter()); + + struct Pause + { + std::latch entered{1}; + std::latch proceed{1}; + } pause; + + table.set_hook_for_tests(&pause, [](void *context, ConcurrentSkipList::PausePoint point, Slice) { + if (point == ConcurrentSkipList::PausePoint::kBeforeCloseAccounting) { + auto &pause = *static_cast<Pause *>(context); + pause.entered.count_down(); + pause.proceed.wait(); + } + }); + std::thread close([&] { table.close(); }); + pause.entered.wait(); + table.leave(); + table.detach_source_stats(); + pause.proceed.count_down(); + close.join(); + table.set_hook_for_tests(nullptr, nullptr); + EXPECT_EQ(ledger->snapshot(), (SourceMemoryStats{.known = true})); +} + +TEST(SourceLedger, PublishedInheritedSummaryIsFixedWhenParentsAdvanceTheirCatalog) +{ + auto parent = std::make_shared<MemTable>(); + ASSERT_TRUE(parent->upsert("parent", 1, encode_cell_buf(1, OpKind::kPut, "v"))); + parent->close(); + auto local = std::make_shared<SourceMemoryLedger>(); + MemTable child; + child.attach_source_stats(local); + PublishedSources sources; + sources.memory = local; + sources.inheritance = PublishedSources::Inheritance{ + .owner_identity = 5, .session = 1, .frontier = 1, .local_count = 1, .memory = parent->source_stats()}; + parent.reset(); + EXPECT_TRUE(sources.source_stats().known); + EXPECT_EQ(sources.source_stats().tables, 2U); + EXPECT_EQ(sources.source_stats().frozen_tables, 1U); + EXPECT_EQ(sources.source_stats().records, 1U); + ASSERT_TRUE(child.try_enter()); + ASSERT_TRUE(child.upsert("child", 2, encode_cell_buf(2, OpKind::kPut, "v"))); + child.leave(); + EXPECT_EQ(sources.source_stats().records, 2U); + EXPECT_EQ(sources.inheritance->memory.records, 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_memory_test.cpp b/lib/crowdb-tree/tests/unit/source_memory_test.cpp new file mode 100644 index 000000000..a51dd770e --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_memory_test.cpp @@ -0,0 +1,127 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/memtable/skip_list.h" + +#include <gtest/gtest.h> + +#include <latch> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +std::shared_ptr<CellVersion> value(uint64_t slot, Slice bytes) +{ + return std::make_shared<CellVersion>(encode_cell_buf(slot, OpKind::kPut, bytes), slot, 0); +} + +void expect_same(const VersionMemory &actual, const VersionMemory &expected) +{ + EXPECT_EQ(actual.history_count, expected.history_count); + EXPECT_EQ(actual.history_bytes, expected.history_bytes); + EXPECT_EQ(actual.descriptor_bytes, expected.descriptor_bytes); + EXPECT_EQ(actual.node_bytes, expected.node_bytes); + EXPECT_EQ(actual.payload_bytes, expected.payload_bytes); +} + +TEST(SourceMemory, LegacyInsertOverwriteAndPruneMatchIndependentLinkedVersionScan) +{ + ConcurrentSkipList source; + expect_same(source.memory(), {}); + ASSERT_TRUE(source.upsert("a", value(1, "one"), 0)); + ASSERT_TRUE(source.upsert("a", value(2, "two-two"), 0)); + ASSERT_TRUE(source.upsert("b", value(4, "four"), 0)); + expect_same(source.memory(), source.scan_memory_for_tests()); + EXPECT_EQ(source.memory().history_count, 1U); + ASSERT_TRUE(source.upsert("a", value(3, "three"), 2)); + expect_same(source.memory(), source.scan_memory_for_tests()); + EXPECT_EQ(source.memory().history_bytes, encode_cell_buf(2, OpKind::kPut, "two-two").size()); + source.prune("a", 3); + expect_same(source.memory(), source.scan_memory_for_tests()); + EXPECT_EQ(source.memory().history_count, 0U); + const auto before = source.memory(); + EXPECT_FALSE(source.upsert("a", value(3, "ignored"), 3)); + expect_same(source.memory(), before); +} + +TEST(SourceMemory, PreparedInsertionLoserAndReplacementUsePublishedCapacities) +{ + ConcurrentSkipList source; + auto older = source.prepare_write("a", value(0, "one"), 0, 4, {}); + auto newer = source.prepare_write("a", value(0, "longer-two"), 0, 4, {}); + EXPECT_TRUE(source.commit_write(std::move(newer), 2)); + EXPECT_TRUE(source.commit_write(std::move(older), 1)); + expect_same(source.memory(), source.scan_memory_for_tests()); + EXPECT_EQ(source.memory().history_count, 1U); + { + auto abandoned = source.prepare_write("b", value(0, "unused"), 0, 16, {}); + expect_same(source.memory(), source.scan_memory_for_tests()); + } + expect_same(source.memory(), source.scan_memory_for_tests()); +} + +TEST(SourceMemory, LaterReplaceAccountsBeforePreparedInsertionWithoutCountingItTwice) +{ + ConcurrentSkipList source; + + struct Pause + { + std::latch published{1}; + std::latch proceed{1}; + } pause; + + source.set_hook_for_tests(&pause, [](void *context, ConcurrentSkipList::PausePoint point, Slice) { + if (point == ConcurrentSkipList::PausePoint::kBeforeInsertAccounting) { + auto &pause = *static_cast<Pause *>(context); + pause.published.count_down(); + pause.proceed.wait(); + } + }); + std::thread insertion([&] { + auto prepared = source.prepare_write("a", value(0, "longer-initial-value"), 0, 4, {}); + EXPECT_TRUE(source.commit_write(std::move(prepared), 1)); + }); + pause.published.wait(); + EXPECT_TRUE(source.upsert("a", value(2, "two"), 2)); + EXPECT_LE(source.memory().payload_bytes, sizeof(CellVersion) + encode_cell_buf(2, OpKind::kPut, "two").size()); + pause.proceed.count_down(); + insertion.join(); + source.set_hook_for_tests(nullptr, nullptr); + EXPECT_EQ(source.approx_bytes(), 1 + encode_cell_buf(2, OpKind::kPut, "two").size()); + expect_same(source.memory(), source.scan_memory_for_tests()); + EXPECT_EQ(source.memory().history_count, 0U); +} + +TEST(SourceMemory, LosingVersionCasCannotDoubleCountAbandonedDescriptor) +{ + ConcurrentSkipList source; + ASSERT_TRUE(source.upsert("a", value(1, "one"), 0)); + + struct Pause + { + std::atomic<bool> once{true}; + std::latch entered{1}; + std::latch proceed{1}; + } pause; + + source.set_hook_for_tests(&pause, [](void *context, ConcurrentSkipList::PausePoint point, Slice) { + auto &pause = *static_cast<Pause *>(context); + if (point == ConcurrentSkipList::PausePoint::kBeforeVersionCas && pause.once.exchange(false)) { + pause.entered.count_down(); + pause.proceed.wait(); + } + }); + MutationStats retries; + std::thread older([&] { EXPECT_TRUE(source.upsert("a", value(2, "two"), 0, &retries)); }); + pause.entered.wait(); + EXPECT_TRUE(source.upsert("a", value(3, "three-three"), 0)); + pause.proceed.count_down(); + older.join(); + source.set_hook_for_tests(nullptr, nullptr); + EXPECT_GT(retries.cas_retries, 0U); + expect_same(source.memory(), source.scan_memory_for_tests()); + EXPECT_EQ(source.memory().history_count, 2U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_pause_test.cpp b/lib/crowdb-tree/tests/unit/source_pause_test.cpp new file mode 100644 index 000000000..c813797d5 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_pause_test.cpp @@ -0,0 +1,143 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_admission.h" +#include "btree/source_pause.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestPausedSource +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(16 * 1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 8}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + SourceWriteOptions options{ + .account = account, .max_entries = 8, .max_key_bytes = 1024, .max_gap_entries = 2, .max_gap_key_bytes = 128}; + SourceAdmission source{0, 1, options}; + + static void complete(void *context) noexcept + { + *static_cast<bool *>(context) = true; + } +}; + +TEST(SourcePause, BoundIncludesUnappliedWritersAndGapRemainsIndependent) +{ + TestPausedSource test; + SourceAdmission::Writer admitted; + ASSERT_TRUE(test.source.try_admit(0, {2, 12, 0}, &admitted).ok()); + const auto catalog = test.source.catalog(); + bool paused = false; + ASSERT_TRUE(catalog->active_budget->pause(&TestPausedSource::complete, &paused)); + ASSERT_TRUE(paused); + EXPECT_EQ(catalog->active->count(), 0U); + const auto bound = catalog->active_budget->paused_bound(); + EXPECT_EQ(bound.entries, 4U); + EXPECT_EQ(bound.key_bytes, 140U); + SourceWriteReservation ordinary; + ASSERT_TRUE( + SourceWriteReservation::create(test.options, {1, 1, 0}, detail::BudgetClass::kAdmittedWriter, &ordinary).ok()); + SourceAdmission::Writer next; + EXPECT_EQ(test.source.try_admit_reserved(0, &ordinary, &next).code(), Code::kUnavailable); + EXPECT_TRUE(ordinary.valid()); + SourceAdmission::Writer gap; + ASSERT_TRUE(test.source.try_admit(0, {1, 1, 0}, &gap, detail::BudgetClass::kGapRepair).ok()); + ASSERT_TRUE(gap.upsert("g", 1, encode_cell_buf(1, OpKind::kPut, "gap")).ok()); + ASSERT_TRUE(admitted.upsert("old", 2, encode_cell_buf(2, OpKind::kPut, "accepted")).ok()); + admitted.release(); + gap.release(); + catalog->active_budget->resume(); + ASSERT_TRUE(test.source.try_admit_reserved(2, &ordinary, &next).ok()); + EXPECT_FALSE(ordinary.valid()); +} + +TEST(SourcePause, FailedRotationRestoresOrdinaryAdmissionWithoutChangingMembership) +{ + TestPausedSource test; + const auto catalog = test.source.catalog(); + bool paused = false; + ASSERT_TRUE(catalog->active_budget->pause(&TestPausedSource::complete, &paused)); + ASSERT_TRUE(paused); + auto raw = + detail::ResourceBudget::try_reserve(test.account, detail::BudgetClass::kHandoff, + detail::shared_allocation_bound<detail::AllocationReservation>() + 1); + if (!raw) { + FAIL() << "missing pause credit"; + } + auto credit = detail::AllocationReservation::create(std::move(*raw)); + std::shared_ptr<SourceAdmission::Rotation> rotation; + EXPECT_EQ(test.source.prepare_rotation(credit, catalog, &rotation).code(), Code::kResourceExhausted); + EXPECT_FALSE(catalog->active->closed()); + EXPECT_EQ(test.source.catalog(), catalog); + catalog->active_budget->resume(); + SourceAdmission::Writer writer; + ASSERT_TRUE(test.source.try_admit(0, {1, 1, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("k", 1, encode_cell_buf(1, OpKind::kPut, "value")).ok()); +} + +TEST(SourcePause, PlanOwnerKeepsPauseAndReleasesItOnAbandonment) +{ + TestPausedSource test; + auto credit_raw = detail::ResourceBudget::try_reserve(test.account, detail::BudgetClass::kHandoff, 128 * 1024); + if (!credit_raw) { + FAIL() << "missing pause credit"; + } + auto metadata = detail::AllocationReservation::create(std::move(*credit_raw)); + std::shared_ptr<SourcePause> pause; + ASSERT_TRUE(SourcePause::create(*test.source.catalog(), metadata, {}, &pause).ok()); + ASSERT_TRUE(pause->ready()); + std::shared_ptr<SourcePause> competing; + EXPECT_EQ(SourcePause::create(*test.source.catalog(), metadata, {}, &competing).code(), Code::kUnavailable); + SourceWriteReservation permit; + ASSERT_TRUE( + SourceWriteReservation::create(test.options, {1, 1, 0}, detail::BudgetClass::kAdmittedWriter, &permit).ok()); + SourceAdmission::Writer writer; + EXPECT_EQ(test.source.try_admit_reserved(0, &permit, &writer).code(), Code::kUnavailable); + EXPECT_TRUE(permit.valid()); + auto retained_plan = pause; + pause.reset(); + EXPECT_EQ(test.source.try_admit_reserved(0, &permit, &writer).code(), Code::kUnavailable); + retained_plan.reset(); + ASSERT_TRUE(test.source.try_admit_reserved(0, &permit, &writer).ok()); +} + +TEST(SourcePause, CaptureClosureRejectsAnOldLateOrdinaryBindAfterPlanRelease) +{ + TestPausedSource test; + const auto old = test.source.catalog(); + ASSERT_TRUE(old->active->try_enter()); + auto raw = detail::ResourceBudget::try_reserve(test.account, detail::BudgetClass::kHandoff, 128 * 1024); + if (!raw) { + FAIL() << "missing pause credit"; + } + auto metadata = detail::AllocationReservation::create(std::move(*raw)); + std::shared_ptr<SourcePause> pause; + ASSERT_TRUE(SourcePause::create(*old, metadata, {}, &pause).ok()); + std::atomic<uint64_t> frontier{1}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(test.source.try_capture(frontier, metadata, &capture, old).ok()); + ASSERT_TRUE(test.source.activate_successor(capture, capture->batch_id()).ok()); + pause.reset(); + SourceWriteReservation late; + ASSERT_TRUE( + SourceWriteReservation::create(test.options, {1, 1, 0}, detail::BudgetClass::kAdmittedWriter, &late).ok()); + std::shared_ptr<detail::AllocationReservation> bound; + EXPECT_EQ(old->active_budget->bind(&late, &bound).code(), Code::kUnavailable); + EXPECT_TRUE(late.valid()); + old->active->leave(); + SourceAdmission::Writer next; + ASSERT_TRUE(test.source.try_admit_reserved(1, &late, &next).ok()); + EXPECT_NE(next.table(), old->active); +} + +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_prepared_write_test.cpp b/lib/crowdb-tree/tests/unit/source_prepared_write_test.cpp new file mode 100644 index 000000000..014f49ebe --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_prepared_write_test.cpp @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_admission.h" + +#include <gtest/gtest.h> + +#include <semaphore> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +struct TestBudget +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] SourceWriteOptions options(size_t entries = 4) const + { + return {.account = account, + .max_entries = entries, + .max_key_bytes = 1024, + .max_gap_entries = 0, + .max_gap_key_bytes = 0}; + } +}; + +TEST(SourcePreparedWrite, AbandonedBatchReturnsAllSourceCapacityWithoutChangingKeys) +{ + TestBudget budget; + SourceAdmission sources(0, 1, budget.options(2)); + { + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, {.entries = 2, .key_bytes = 2, .payload_bytes = 0}, &writer).ok()); + SourceAdmission::Writer::Prepared first; + SourceAdmission::Writer::Prepared rejected; + ASSERT_TRUE(writer.prepare_write("a", 0, encode_cell_buf(0, OpKind::kPut, "one"), &first).ok()); + EXPECT_EQ(writer.prepare_write("too-long", 0, encode_cell_buf(0, OpKind::kPut, "two"), &rejected).code(), + Code::kInvalidArgument); + EXPECT_TRUE(writer.table()->empty()); + EXPECT_EQ(writer.upsert("b", 1, encode_cell_buf(1, OpKind::kPut, "legacy")).code(), Code::kInvalidArgument); + } + SourceAdmission::Writer retry; + ASSERT_TRUE(sources.try_admit(0, {.entries = 2, .key_bytes = 2, .payload_bytes = 0}, &retry).ok()); + SourceAdmission::Writer::Prepared first; + SourceAdmission::Writer::Prepared second; + ASSERT_TRUE(retry.prepare_write("a", 0, encode_cell_buf(0, OpKind::kPut, "one"), &first).ok()); + ASSERT_TRUE(retry.prepare_write("b", 0, encode_cell_buf(0, OpKind::kPut, "two"), &second).ok()); + retry.allocation_budget()->seal(); + EXPECT_TRUE(retry.commit_write(std::move(first), 1)); + EXPECT_TRUE(retry.commit_write(std::move(second), 1)); + EXPECT_EQ(retry.table()->count(), 2U); + EXPECT_EQ(retry.table()->find("a")->slot, 1U); + EXPECT_EQ(CellView(retry.table()->find("b")->cell.slice()).slot(), 1U); +} + +TEST(SourcePreparedWrite, NewKeyPreparationCanLoseInsertionWithoutAnotherAllocation) +{ + TestBudget budget; + SourceAdmission sources(0, 1, budget.options(2)); + SourceAdmission::Writer older; + SourceAdmission::Writer newer; + ASSERT_TRUE(sources.try_admit(0, {.entries = 1, .key_bytes = 1, .payload_bytes = 0}, &older).ok()); + ASSERT_TRUE(sources.try_admit(0, {.entries = 1, .key_bytes = 1, .payload_bytes = 0}, &newer).ok()); + SourceAdmission::Writer::Prepared first; + SourceAdmission::Writer::Prepared second; + ASSERT_TRUE(older.prepare_write("k", 0, encode_cell_buf(0, OpKind::kPut, "old"), &first).ok()); + ASSERT_TRUE(newer.prepare_write("k", 0, encode_cell_buf(0, OpKind::kPut, "new"), &second).ok()); + older.allocation_budget()->seal(); + newer.allocation_budget()->seal(); + EXPECT_TRUE(newer.commit_write(std::move(second), 2)); + EXPECT_TRUE(older.commit_write(std::move(first), 1)); + auto table = older.table(); + EXPECT_EQ(table->count(), 1U); + EXPECT_EQ(table->find("k")->slot, 2U); + EXPECT_EQ(table->prefix_cursor(1).cell_version()->slot, 1U); +} + +#ifdef CROWDB_TREE_TEST_UTIL +TEST(SourcePreparedWrite, SealedCommitRetriesVersionCasWithTheOriginalCapacity) +{ + TestBudget budget; + SourceAdmission sources(0, 1, budget.options(3)); + SourceAdmission::Writer initial; + ASSERT_TRUE(sources.try_admit(0, {.entries = 1, .key_bytes = 1, .payload_bytes = 0}, &initial).ok()); + ASSERT_TRUE(initial.upsert("k", 1, encode_cell_buf(1, OpKind::kPut, "one")).ok()); + auto table = initial.table(); + initial.release(); + + struct Pause + { + std::atomic<bool> once{true}; + std::atomic<bool> prepared{false}; + std::binary_semaphore entered{0}; + std::binary_semaphore resume{0}; + } pause; + + table->set_hook_for_tests(&pause, [](void *context, ConcurrentSkipList::PausePoint point, Slice) { + auto &state = *static_cast<Pause *>(context); + if (point == ConcurrentSkipList::PausePoint::kBeforeVersionCas && state.once.exchange(false)) { + state.entered.release(); + state.resume.acquire(); + } + }); + std::jthread old([&] { + SourceAdmission::Writer writer; + SourceAdmission::Writer::Prepared prepared; + auto status = sources.try_admit(0, {.entries = 1, .key_bytes = 1, .payload_bytes = 0}, &writer); + if (status.ok()) { + status = writer.prepare_write("k", 0, encode_cell_buf(0, OpKind::kPut, "two"), &prepared); + } + EXPECT_TRUE(status.ok()) << status.to_string(); + if (!status.ok()) { + pause.entered.release(); + return; + } + pause.prepared.store(true); + writer.allocation_budget()->seal(); + EXPECT_TRUE(writer.commit_write(std::move(prepared), 2)); + }); + pause.entered.acquire(); + if (!pause.prepared.load()) { + return; + } + const auto resume = [](std::binary_semaphore *value) { value->release(); }; + std::unique_ptr<std::binary_semaphore, decltype(resume)> release(&pause.resume, resume); + SourceAdmission::Writer newer; + ASSERT_TRUE(sources.try_admit(0, {.entries = 1, .key_bytes = 1, .payload_bytes = 0}, &newer).ok()); + SourceAdmission::Writer::Prepared prepared; + ASSERT_TRUE(newer.prepare_write("k", 0, encode_cell_buf(0, OpKind::kPut, "three"), &prepared).ok()); + newer.allocation_budget()->seal(); + EXPECT_TRUE(newer.commit_write(std::move(prepared), 3)); + release.reset(); + old.join(); + table->set_hook_for_tests(nullptr, nullptr); + EXPECT_EQ(table->find("k")->slot, 3U); + EXPECT_EQ(table->prefix_cursor(2).cell_version()->slot, 2U); + EXPECT_EQ(table->prefix_cursor(1).cell_version()->slot, 1U); + EXPECT_EQ(table->memory().history_count, 2U); +} +#endif +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_rotation_test.cpp b/lib/crowdb-tree/tests/unit/source_rotation_test.cpp new file mode 100644 index 000000000..0ad1e5cca --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_rotation_test.cpp @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/published_state.h" +#include "btree/source_admission.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestRotationCredit +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits out; + out.fill(1024 * 1024); + return out; + } + + detail::ResourceBudget budget{limits(), 2}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> reserve(size_t bytes) const + { + auto credit = detail::ResourceBudget::try_reserve( + account, detail::BudgetClass::kHandoff, + bytes + detail::shared_allocation_bound<detail::AllocationReservation>()); + return credit ? detail::AllocationReservation::create(std::move(*credit)) : nullptr; + } +}; + +TEST(SourceRotation, PreparesBeforeClosureAndKeepsAdmittedOldWriterReadable) +{ + TestRotationCredit credits; + SourceAdmission source; + const auto old = source.catalog(); + auto bound = SourceAdmission::rotation_allocation_bound(1); + ASSERT_TRUE(bound); + auto credit = credits.reserve(bound.value_or(0)); + ASSERT_TRUE(credit); + SourceAdmission::Writer writer; + ASSERT_TRUE(source.try_admit(0, &writer).ok()); + std::shared_ptr<SourceAdmission::Rotation> rotation; + ASSERT_TRUE(source.prepare_rotation(credit, old, &rotation).ok()); + EXPECT_EQ(source.catalog(), old); + EXPECT_FALSE(old->active->closed()); + ASSERT_EQ(rotation->next()->tables.size(), 2U); + std::atomic<uint64_t> contiguous{1}; + std::shared_ptr<SourceCapture> capture; + EXPECT_EQ(source.try_capture(contiguous, credit, &capture).code(), Code::kUnavailable); + credit->seal(); + ASSERT_TRUE(source.activate_rotation(rotation).ok()); + EXPECT_TRUE(old->active->closed()); + ASSERT_TRUE(writer.upsert("old", 1, encode_cell_buf(1, OpKind::kPut, "value")).ok()); + auto guard = old->active->read_guard(); + const auto *value = old->active->find("old"); + ASSERT_NE(value, nullptr); + EXPECT_EQ(value->slot, 1U); + SourceAdmission::Writer next; + ASSERT_TRUE(source.try_admit(1, &next).ok()); + EXPECT_NE(next.table(), writer.table()); + source.close(); + EXPECT_TRUE(source.activate_rotation(rotation).ok()); +} + +TEST(SourceRotation, AbandonmentAndInsufficientCreditLeaveOriginalSourceWritable) +{ + TestRotationCredit credits; + SourceAdmission source; + const auto old = source.catalog(); + std::shared_ptr<SourceAdmission::Rotation> rotation; + auto tiny = credits.reserve(1); + EXPECT_EQ(source.prepare_rotation(tiny, old, &rotation).code(), Code::kResourceExhausted); + EXPECT_FALSE(old->active->closed()); + auto bound = SourceAdmission::rotation_allocation_bound(1); + ASSERT_TRUE(bound); + auto credit = credits.reserve(bound.value_or(0)); + ASSERT_TRUE(source.prepare_rotation(credit, old, &rotation).ok()); + ASSERT_TRUE(source.abort_rotation(rotation).ok()); + EXPECT_EQ(source.catalog(), old); + SourceAdmission::Writer writer; + ASSERT_TRUE(source.try_admit(0, &writer).ok()); + EXPECT_EQ(writer.table(), old->active); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_tail_test.cpp b/lib/crowdb-tree/tests/unit/source_tail_test.cpp new file mode 100644 index 000000000..dd2b182fb --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_tail_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_admission.h" +#include "btree/tree_core_access.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(SourceTail, RegistrationBarrierWaitsForThePreviouslyAdmittedWriter) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + auto account = TreeCoreAccess::get(tree)->binding()->account(); + SourceAdmission source(0, 1, SourceWriteOptions{.account = account}); + SourceAdmission::Writer before; + ASSERT_TRUE(source.try_admit(0, {.entries = 1, .key_bytes = 1}, &before).ok()); + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, 1024 * 1024); + if (!credit.has_value()) { + FAIL() << "tail test metadata credit unavailable"; + return; + } + auto metadata = detail::AllocationReservation::create(std::move(credit.value())); + auto tail = detail::reserved_shared<SourceTail>(metadata, metadata, 8, 128); + std::shared_ptr<SourceTailDomain::Barrier> barrier; + auto status = source.tail_domain()->arm(tail, metadata, &barrier); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_FALSE(barrier->ready.load()); + SourceAdmission::Writer::Prepared old; + ASSERT_TRUE(before.prepare_write("k", 0, encode_cell_buf(1, OpKind::kPut, "old"), &old).ok()); + EXPECT_TRUE(before.commit_write(std::move(old), 1)); + EXPECT_FALSE(barrier->ready.load()); + before.release(); // real Core callers acknowledge frontier before this + barrier->wait(); + EXPECT_TRUE(barrier->ready.load()); + EXPECT_TRUE(tail->table()->empty()); + SourceAdmission::Writer after; + ASSERT_TRUE(source.try_admit(1, {.entries = 1, .key_bytes = 1}, &after).ok()); + SourceAdmission::Writer::Prepared fresh; + ASSERT_TRUE(after.prepare_write("k", 0, encode_cell_buf(2, OpKind::kPut, "new"), &fresh).ok()); + EXPECT_TRUE(after.commit_write(std::move(fresh), 2)); + after.release(); + ASSERT_TRUE(tail->freeze()); + EXPECT_TRUE(source.tail_domain()->detach(barrier)); + auto guard = tail->table()->read_guard(metadata); + ASSERT_NE(tail->table()->find("k"), nullptr); + EXPECT_EQ(tail->table()->find("k")->slot, 2U); +} + +TEST(SourceTail, MovingAndOverwritingAnEnteredWriteReleasesExactlyOneTicket) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + auto account = TreeCoreAccess::get(tree)->binding()->account(); + SourceAdmission source(0, 1, SourceWriteOptions{.account = account}); + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, 1024 * 1024); + if (!credit.has_value()) { + FAIL() << "tail test metadata credit unavailable"; + return; + } + auto metadata = detail::AllocationReservation::create(std::move(credit.value())); + auto tail = detail::reserved_shared<SourceTail>(metadata, metadata, 8, 128); + MemTable parent(1, nullptr, metadata); + auto prepared = parent.prepare_write("k", encode_cell_buf(1, OpKind::kPut, "v"), 0, {}, 0, 8, metadata); + auto first = SourceTail::prepare(tail, prepared); + auto second = SourceTail::prepare(tail, prepared); + EXPECT_FALSE(tail->freeze()); + SourceTail::Write moved(std::move(first)); + second = std::move(moved); + EXPECT_FALSE(tail->freeze()); + EXPECT_TRUE(parent.commit_write(std::move(prepared), 1)); + second.complete(true, 1); + ASSERT_TRUE(tail->freeze()); + auto guard = tail->table()->read_guard(metadata); + EXPECT_EQ(tail->table()->find("k")->slot, 1U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_write_partition_test.cpp b/lib/crowdb-tree/tests/unit/source_write_partition_test.cpp new file mode 100644 index 000000000..c3d091046 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_write_partition_test.cpp @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_admission.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestPartitionBudget +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + SourceWriteOptions options{ + .account = account, .max_entries = 2, .max_key_bytes = 128, .max_gap_entries = 1, .max_gap_key_bytes = 32}; +}; + +TEST(SourceWritePartition, ChosenGapUsesItsPrepaidMemoryWithoutOrdinarySourceCapacity) +{ + constexpr auto ordinary = detail::BudgetClass::kAdmittedWriter; + constexpr auto gap = detail::BudgetClass::kGapRepair; + TestPartitionBudget budget; + SourceAdmission sources(0, 1, budget.options); + SourceWriteReservation client; + ASSERT_TRUE(SourceWriteReservation::create(budget.options, {1, 1, 0}, ordinary, &client).ok()); + SourceAdmission::Writer ahead; + ASSERT_TRUE(sources.try_admit(0, {2, 2, 0}, &ahead).ok()); + ASSERT_TRUE(ahead.upsert("a", 2, encode_cell_buf(2, OpKind::kPut, "v")).ok()); + ASSERT_TRUE(ahead.upsert("b", 3, encode_cell_buf(3, OpKind::kPut, "v")).ok()); + ahead.release(); + const auto before = budget.budget.snapshot(); + SourceAdmission::Writer repair; + EXPECT_EQ(sources.try_admit_reserved(0, &client, &repair).code(), Code::kUnavailable); + EXPECT_TRUE(client.valid()); + ASSERT_TRUE(sources.try_admit_reserved(0, &client, &repair, gap).ok()); + EXPECT_EQ(repair.allocation_budget()->category(), ordinary); + const auto after = budget.budget.snapshot(); + for (const auto category : {ordinary, gap}) { + const auto index = static_cast<size_t>(category); + EXPECT_EQ(after.reserved[index] + after.allocated[index], before.reserved[index] + before.allocated[index]); + } + ASSERT_TRUE(repair.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "old")).ok()); + repair.release(); + EXPECT_FALSE(client.valid()); +} + +TEST(SourceWritePartition, AbandoningReclassifiedWriterReturnsOnlyGapCounters) +{ + constexpr auto ordinary = detail::BudgetClass::kAdmittedWriter; + constexpr auto gap = detail::BudgetClass::kGapRepair; + TestPartitionBudget budget; + SourceAdmission sources(0, 1, budget.options); + SourceWriteReservation first; + ASSERT_TRUE(SourceWriteReservation::create(budget.options, {1, 1, 0}, ordinary, &first).ok()); + SourceAdmission::Writer abandoned; + ASSERT_TRUE(sources.try_admit_reserved(0, &first, &abandoned, gap).ok()); + abandoned.release(); + SourceWriteReservation second; + ASSERT_TRUE(SourceWriteReservation::create(budget.options, {1, 1, 0}, ordinary, &second).ok()); + SourceAdmission::Writer replacement; + ASSERT_TRUE(sources.try_admit_reserved(0, &second, &replacement, gap).ok()); + ASSERT_TRUE(replacement.upsert("k", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + replacement.release(); + SourceAdmission::Writer full_ordinary; + EXPECT_TRUE(sources.try_admit(1, {2, 2, 0}, &full_ordinary).ok()); +} + +TEST(SourceWritePartition, ImpossibleGapDoesNotConsumeTheOriginalReservation) +{ + TestPartitionBudget budget; + SourceAdmission sources(0, 1, budget.options); + SourceWriteReservation client; + ASSERT_TRUE( + SourceWriteReservation::create(budget.options, {2, 2, 0}, detail::BudgetClass::kAdmittedWriter, &client).ok()); + SourceAdmission::Writer writer; + EXPECT_EQ(sources.try_admit_reserved(0, &client, &writer, detail::BudgetClass::kGapRepair).code(), + Code::kResourceExhausted); + EXPECT_TRUE(client.valid()); + EXPECT_EQ(sources.try_admit_reserved(0, &client, &writer, detail::BudgetClass::kRouting).code(), + Code::kInvalidArgument); + EXPECT_TRUE(client.valid()); + EXPECT_TRUE(sources.try_admit_reserved(0, &client, &writer).ok()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_write_reservation_test.cpp b/lib/crowdb-tree/tests/unit/source_write_reservation_test.cpp new file mode 100644 index 000000000..b4c6be5fd --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_write_reservation_test.cpp @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_admission.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +constexpr auto kWriter = detail::BudgetClass::kAdmittedWriter; +constexpr size_t kWriterIndex = static_cast<size_t>(kWriter); + +struct TestAdmissionBudget +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] SourceWriteOptions options() const + { + return { + .account = account, .max_entries = 1, .max_key_bytes = 128, .max_gap_entries = 1, .max_gap_key_bytes = 128}; + } + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> handoff() const + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, 128 * 1024); + if (!credit) { + throw std::bad_alloc(); + } + return detail::AllocationReservation::create(std::move(*credit)); + } + + [[nodiscard]] size_t writer_usage() const + { + auto snapshot = budget.snapshot(); + return snapshot.allocated[kWriterIndex] + snapshot.reserved[kWriterIndex]; + } +}; + +TEST(SourceWriteReservation, CancellationReturnsCreditWithoutEnteringASource) +{ + TestAdmissionBudget budget; + const auto baseline = budget.writer_usage(); + { + SourceWriteReservation reservation; + ASSERT_TRUE(SourceWriteReservation::create(budget.options(), {1, 1, 0}, kWriter, &reservation).ok()); + EXPECT_GT(budget.writer_usage(), baseline); + SourceWriteReservation moved(std::move(reservation)); + EXPECT_FALSE(reservation.valid()); + EXPECT_TRUE(moved.valid()); + } + EXPECT_EQ(budget.writer_usage(), baseline); +} + +TEST(SourceWriteReservation, BoundPermitDoesNotReserveTheSameMemoryTwice) +{ + TestAdmissionBudget budget; + SourceAdmission sources(0, 1, budget.options()); + SourceWriteReservation reservation; + ASSERT_TRUE(SourceWriteReservation::create(budget.options(), {1, 1, 0}, kWriter, &reservation).ok()); + const auto admitted = budget.writer_usage(); + auto blocker = detail::ResourceBudget::try_reserve(budget.account, kWriter, + TestAdmissionBudget::limits()[kWriterIndex] - admitted); + ASSERT_TRUE(blocker); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit_reserved(0, &reservation, &writer).ok()); + EXPECT_FALSE(reservation.valid()); + EXPECT_EQ(budget.writer_usage(), TestAdmissionBudget::limits()[kWriterIndex]); + ASSERT_TRUE(writer.upsert("k", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + writer.release(); + EXPECT_EQ(sources.try_admit_reserved(0, &reservation, &writer).code(), Code::kInvalidArgument); +} + +TEST(SourceWriteReservation, SourceRotationDoesNotWaitForConsensusReservation) +{ + TestAdmissionBudget budget; + SourceAdmission sources(0, 1, budget.options()); + SourceAdmission::Writer first; + ASSERT_TRUE(sources.try_admit(0, {1, 1, 0}, &first).ok()); + ASSERT_TRUE(first.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "old")).ok()); + first.release(); + SourceWriteReservation reservation; + ASSERT_TRUE(SourceWriteReservation::create(budget.options(), {1, 1, 0}, kWriter, &reservation).ok()); + SourceAdmission::Writer next; + EXPECT_EQ(sources.try_admit_reserved(1, &reservation, &next).code(), Code::kUnavailable); + EXPECT_TRUE(reservation.valid()); + std::atomic<uint64_t> frontier{1}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(sources.try_capture(frontier, budget.handoff(), &capture).ok()); + ASSERT_TRUE(stdexec::sync_wait(capture->drain_sender()).has_value()); + EXPECT_TRUE(capture->drained()); + ASSERT_TRUE(sources.activate_successor(capture, capture->batch_id()).ok()); + ASSERT_TRUE(sources.try_admit_reserved(1, &reservation, &next).ok()); + ASSERT_TRUE(next.upsert("b", 2, encode_cell_buf(2, OpKind::kPut, "new")).ok()); + EXPECT_EQ(next.table(), capture->successor()); + EXPECT_EQ(capture->frontier(), 1U); +} + +TEST(SourceWriteReservation, ForeignAccountIsRejectedWithoutConsumingReservation) +{ + TestAdmissionBudget first; + TestAdmissionBudget second; + SourceAdmission sources(0, 1, second.options()); + SourceWriteReservation reservation; + ASSERT_TRUE(SourceWriteReservation::create(first.options(), {1, 1, 0}, kWriter, &reservation).ok()); + SourceAdmission::Writer writer; + EXPECT_EQ(sources.try_admit_reserved(0, &reservation, &writer).code(), Code::kInvalidArgument); + EXPECT_TRUE(reservation.valid()); + EXPECT_FALSE(writer.table()); +} + +TEST(SourceWriteReservation, GapReservationRemainsIndependentOfOrdinaryAdmission) +{ + TestAdmissionBudget budget; + SourceAdmission sources(0, 1, budget.options()); + SourceAdmission::Writer first; + ASSERT_TRUE(sources.try_admit(0, {1, 1, 0}, &first).ok()); + ASSERT_TRUE(first.upsert("k", 2, encode_cell_buf(2, OpKind::kPut, "new")).ok()); + first.release(); + auto blocker = detail::ResourceBudget::try_reserve( + budget.account, kWriter, TestAdmissionBudget::limits()[kWriterIndex] - budget.writer_usage()); + ASSERT_TRUE(blocker); + SourceWriteReservation gap; + ASSERT_TRUE( + SourceWriteReservation::create(budget.options(), {1, 1, 0}, detail::BudgetClass::kGapRepair, &gap).ok()); + SourceAdmission::Writer repair; + ASSERT_TRUE(sources.try_admit_reserved(0, &gap, &repair).ok()); + ASSERT_TRUE(repair.upsert("k", 1, encode_cell_buf(1, OpKind::kPut, "old")).ok()); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/source_writer_budget_test.cpp b/lib/crowdb-tree/tests/unit/source_writer_budget_test.cpp new file mode 100644 index 000000000..6e8cf20c6 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/source_writer_budget_test.cpp @@ -0,0 +1,287 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/source_admission.h" + +#include <gtest/gtest.h> + +#include <barrier> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kWriter = static_cast<size_t>(detail::BudgetClass::kAdmittedWriter); + +struct TestBudget +{ + static detail::ResourceBudget::Limits limits() + { + detail::ResourceBudget::Limits result; + result.fill(1024 * 1024); + return result; + } + + detail::ResourceBudget budget{limits(), 4}; + std::shared_ptr<detail::ResourceBudget::Account> account = budget.make_account(limits()); + + [[nodiscard]] SourceWriteOptions options(size_t entries = 8) const + { + return {.account = account, + .max_entries = entries, + .max_key_bytes = 1024, + .max_gap_entries = 2, + .max_gap_key_bytes = 128}; + } + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> lifecycle() const + { + const auto bytes = SourceAdmission::allocation_bound(); + if (!bytes) { + throw std::bad_alloc(); + } + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kAdmittedWriter, *bytes); + if (!credit) { + throw std::bad_alloc(); + } + return detail::AllocationReservation::create(std::move(*credit)); + } + + [[nodiscard]] std::shared_ptr<detail::AllocationReservation> handoff() const + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kHandoff, 128 * 1024); + if (!credit) { + throw std::bad_alloc(); + } + return std::make_shared<detail::AllocationReservation>(std::move(*credit)); + } +}; + +TEST(SourceWriterBudget, PartialCapacityRejectionRequestsOrdinaryRolloverButGapPressureDoesNot) +{ + TestBudget budget; + for (bool key_limited : {false, true}) { + auto options = budget.options(key_limited ? 8 : 5); + options.max_key_bytes = key_limited ? 128 : 1024; + SourceWriteBudget source(options); + std::shared_ptr<detail::AllocationReservation> first; + const WriteFootprint footprint{.entries = 3, .key_bytes = key_limited ? 96U : 3U}; + ASSERT_TRUE(source.admit(footprint, detail::BudgetClass::kAdmittedWriter, &first).ok()); + EXPECT_FALSE(source.full()); + std::shared_ptr<detail::AllocationReservation> second; + EXPECT_EQ(source.admit(footprint, detail::BudgetClass::kAdmittedWriter, &second).code(), Code::kUnavailable); + EXPECT_TRUE(source.full()); + } + SourceWriteBudget source(budget.options()); + std::shared_ptr<detail::AllocationReservation> first; + ASSERT_TRUE(source.admit({.entries = 2, .key_bytes = 2}, detail::BudgetClass::kGapRepair, &first).ok()); + std::shared_ptr<detail::AllocationReservation> second; + EXPECT_EQ(source.admit({.entries = 1, .key_bytes = 1}, detail::BudgetClass::kGapRepair, &second).code(), + Code::kUnavailable); + EXPECT_FALSE(source.full()); +} + +TEST(SourceWriterBudget, AdmittedWriterCompletesWhenUnreservedHostBudgetIsExhausted) +{ + TestBudget budget; + { + SourceAdmission sources(0, 1, budget.options()); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, {4, 12, 4096}, &writer).ok()); + auto snapshot = budget.budget.snapshot(); + auto remaining = TestBudget::limits()[kWriter] - snapshot.allocated[kWriter] - snapshot.reserved[kWriter]; + auto blocker = + detail::ResourceBudget::try_reserve(budget.account, detail::BudgetClass::kAdmittedWriter, remaining); + ASSERT_TRUE(blocker); + SourceAdmission::Writer rejected; + EXPECT_EQ(sources.try_admit(0, {1, 3, 0}, &rejected).code(), Code::kUnavailable); + for (uint64_t slot = 1; slot <= 4; ++slot) { + bool changed = false; + ASSERT_TRUE(writer.upsert("key", slot, encode_cell_buf(slot, OpKind::kPut, "v"), &changed).ok()); + EXPECT_TRUE(changed); + } + writer.release(); + auto view = sources.catalog(); + auto guard = view->active->read_guard(); + ASSERT_NE(view->active->find("key"), nullptr); + EXPECT_EQ(view->active->find("key")->slot, 4U); + } + EXPECT_EQ(budget.budget.snapshot().allocated[kWriter], 0U); + EXPECT_EQ(budget.budget.snapshot().reserved[kWriter], 0U); +} + +TEST(SourceWriterBudget, FootprintViolationCannotModifySourceAndUnusedPermitCanBeReadmitted) +{ + TestBudget budget; + SourceAdmission sources(0, 1, budget.options(1)); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, {1, 1, 0}, &writer).ok()); + EXPECT_EQ(writer.upsert("too long", 1, encode_cell_buf(1, OpKind::kPut, "v")).code(), Code::kInvalidArgument); + EXPECT_EQ(writer.upsert("k", 1, encode_cell_buf(1, OpKind::kPut, std::string(1024, 'v'))).code(), + Code::kInvalidArgument); + EXPECT_TRUE(writer.table()->empty()); + SourceAdmission::Writer moved = std::move(writer); + writer = {}; + EXPECT_EQ(sources.try_admit(0, {1, 1, 0}, &writer).code(), Code::kUnavailable); + moved.release(); + ASSERT_TRUE(sources.try_admit(0, {1, 1, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("k", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + EXPECT_EQ(writer.upsert("k", 2, encode_cell_buf(2, OpKind::kPut, "v")).code(), Code::kInvalidArgument); +} + +TEST(SourceWriterBudget, GapRepairHasIndependentEntryAndMemoryAdmission) +{ + TestBudget budget; + SourceAdmission sources(0, 1, budget.options(1)); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, {1, 1, 0}, &writer).ok()); + ASSERT_TRUE(writer.upsert("k", 2, encode_cell_buf(2, OpKind::kPut, "new")).ok()); + writer.release(); + EXPECT_EQ(sources.try_admit(0, {1, 1, 0}, &writer).code(), Code::kUnavailable); + auto snapshot = budget.budget.snapshot(); + auto rest = TestBudget::limits()[kWriter] - snapshot.allocated[kWriter] - snapshot.reserved[kWriter]; + auto blocker = detail::ResourceBudget::try_reserve(budget.account, detail::BudgetClass::kAdmittedWriter, rest); + ASSERT_TRUE(blocker); + ASSERT_TRUE(sources.try_admit(0, {1, 1, 0}, &writer, detail::BudgetClass::kGapRepair).ok()); + ASSERT_TRUE(writer.upsert("k", 1, encode_cell_buf(1, OpKind::kPut, "old")).ok()); + auto table = writer.table(); + writer.release(); + auto guard = table->read_guard(); + auto cursor = table->prefix_cursor(1); + ASSERT_TRUE(cursor.valid()); + EXPECT_EQ(cursor.cell_owner()->slot, 1U); +} + +TEST(SourceWriterBudget, SeparatePermitsDoNotReuseSealedCreditOfOverwrittenVersion) +{ + TestBudget budget; + { + auto lifecycle = budget.lifecycle(); + SourceAdmission sources(0, 1, budget.options(16), lifecycle); + auto table = sources.catalog()->active; + auto read_credit = budget.handoff(); + auto guard = table->read_guard(read_credit); + read_credit->seal(); + const auto before_observer = lifecycle->available(); + const auto before_allocated = budget.budget.snapshot().allocated[kWriter]; + { + // Register this thread's prepaid observer participant once. Each + // later write must reuse it without consuming lifecycle credit. + auto observer = sources.tail_domain()->observe(); + } + const auto lifecycle_reserved = lifecycle->available(); + const auto observer_bytes = before_observer - lifecycle_reserved; + EXPECT_GT(observer_bytes, 0U); + EXPECT_LE(observer_bytes, EpochManager::participant_allocation_bytes()); + EXPECT_EQ(budget.budget.snapshot().allocated[kWriter] - before_allocated, observer_bytes); + EXPECT_GT(lifecycle_reserved, 0U); + for (uint64_t slot = 1; slot <= 16; ++slot) { + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit(0, {1, 1, 0}, &writer).ok()); + EXPECT_GT(budget.budget.snapshot().reserved[kWriter], lifecycle_reserved); + ASSERT_TRUE(writer.upsert("k", slot, encode_cell_buf(slot, OpKind::kPut, "v")).ok()); + writer.release(); + EXPECT_EQ(lifecycle->available(), lifecycle_reserved); + EXPECT_EQ(budget.budget.snapshot().reserved[kWriter], lifecycle_reserved); + } + EXPECT_EQ(table->find("k")->slot, 16U); + EXPECT_EQ(table->memory().history_count, 15U); + } + EXPECT_EQ(budget.budget.snapshot().allocated[kWriter], 0U); + EXPECT_EQ(budget.budget.snapshot().reserved[kWriter], 0U); +} + +TEST(SourceWriterBudget, CapturedWriterRetainsPermitAndSuccessorRequiresItsOwn) +{ + TestBudget budget; + SourceAdmission sources(0, 1, budget.options()); + SourceAdmission::Writer old; + ASSERT_TRUE(sources.try_admit(0, {2, 2, 0}, &old).ok()); + ASSERT_TRUE(old.upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + std::atomic<uint64_t> frontier{1}; + std::shared_ptr<SourceCapture> capture; + ASSERT_TRUE(sources.try_capture(frontier, budget.handoff(), &capture).ok()); + ASSERT_TRUE(old.table()->closed()); + ASSERT_TRUE(old.upsert("b", 2, encode_cell_buf(2, OpKind::kPut, "v")).ok()); + ASSERT_TRUE(sources.activate_successor(capture, capture->batch_id()).ok()); + SourceAdmission::Writer next; + EXPECT_EQ(sources.try_admit(1, &next).code(), Code::kInvalidArgument); + ASSERT_TRUE(sources.try_admit(1, {1, 1, 0}, &next).ok()); + EXPECT_NE(old.table(), next.table()); + ASSERT_TRUE(next.upsert("c", 3, encode_cell_buf(3, OpKind::kPut, "v")).ok()); + EXPECT_EQ(capture->frontier(), 1U); +} + +TEST(SourceWriterBudget, ConcurrentPermitsKeepAllVersionsWithinTheirOwnCredit) +{ + TestBudget budget; + { + auto lifecycle = budget.lifecycle(); + SourceAdmission sources(0, 1, budget.options(16), lifecycle); + auto table = sources.catalog()->active; + std::array<SourceWriteReservation, 8> reservations; + for (auto &reservation : reservations) { + ASSERT_TRUE(SourceWriteReservation::create(budget.options(16), {2, 2, 0}, + detail::BudgetClass::kAdmittedWriter, &reservation) + .ok()); + } + std::barrier registered(9); + std::barrier start(9); + std::vector<std::jthread> workers; + workers.reserve(reservations.size()); + for (size_t index = 0; index < reservations.size(); ++index) { + workers.emplace_back([&, index] { + { + // Prepay lifecycle participants on their owning thread, + // separately from each transferred write reservation. + auto observer = sources.tail_domain()->observe(); + (void)sources.catalog(); + } + registered.arrive_and_wait(); + start.arrive_and_wait(); + SourceAdmission::Writer writer; + ASSERT_TRUE(sources.try_admit_reserved(0, &reservations[index], &writer).ok()); + EXPECT_FALSE(reservations[index].valid()); + for (uint64_t slot = (2 * index) + 1; slot <= (2 * index) + 2; ++slot) { + auto status = writer.upsert("k", slot, encode_cell_buf(slot, OpKind::kPut, "v")); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + writer.release(); + }); + } + registered.arrive_and_wait(); + const auto lifecycle_reserved = lifecycle->available(); + EXPECT_GT(lifecycle_reserved, 0U); + EXPECT_GT(budget.budget.snapshot().reserved[kWriter], lifecycle_reserved); + start.arrive_and_wait(); + workers.clear(); + auto read_credit = budget.handoff(); + auto guard = table->read_guard(read_credit); + read_credit->seal(); + EXPECT_EQ(table->find("k")->slot, 16U); + EXPECT_EQ(table->memory().history_count, 15U); + EXPECT_EQ(lifecycle->available(), lifecycle_reserved); + EXPECT_EQ(budget.budget.snapshot().reserved[kWriter], lifecycle_reserved); + } + EXPECT_EQ(budget.budget.snapshot().allocated[kWriter], 0U); + EXPECT_EQ(budget.budget.snapshot().reserved[kWriter], 0U); +} + +TEST(SourceWriterBudget, LargestKeyBoundIsCheckedAtAdmissionAndBeforeMutation) +{ + TestBudget budget; + auto options = budget.options(); + options.max_key_size = 4; + SourceAdmission sources(0, 1, options); + SourceAdmission::Writer writer; + EXPECT_EQ(sources.try_admit(0, {2, 8, 0}, &writer).code(), Code::kInvalidArgument); + EXPECT_EQ(sources.try_admit(0, {2, 8, 0, 5}, &writer).code(), Code::kInvalidArgument); + ASSERT_TRUE(sources.try_admit(0, {2, 8, 0, 4}, &writer).ok()); + EXPECT_EQ(writer.upsert("abcde", 1, encode_cell_buf(1, OpKind::kPut, "v")).code(), Code::kInvalidArgument); + EXPECT_TRUE(writer.table()->empty()); + ASSERT_TRUE(writer.upsert("abcd", 1, encode_cell_buf(1, OpKind::kPut, "v")).ok()); + ASSERT_TRUE(writer.upsert("efgh", 2, encode_cell_buf(2, OpKind::kPut, "v")).ok()); + EXPECT_EQ(writer.table()->count(), 2U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_core_geometry_test.cpp b/lib/crowdb-tree/tests/unit/tree_core_geometry_test.cpp new file mode 100644 index 000000000..b6b0187c4 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_core_geometry_test.cpp @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_maintenance.h" +#include "btree/tree_core_geometry.h" +#include "crowdb-protocol/frame/base.h" +#include "crowdb-tree/maptable/frame_page.h" + +#include <gtest/gtest.h> + +#include <limits> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeCoreGeometry, SmallRecordThresholdsIncludeOneFrameOverhead) +{ + Config config; + config.frame_bytes = 4096; + for (const size_t target : {80U, 128U, 160U}) { + config.leaf_split_bytes = target; + config.leaf_merge_bytes = target / 4; + PageMaintenanceLimits pages; + ASSERT_TRUE(configure_leaf_geometry(config, &pages).ok()); + EXPECT_EQ(pages.target_page_bytes, target + kFrameHeaderSize + kFrameTrailerSize); + EXPECT_EQ(pages.merge_trigger_bytes, (target / 4) + kFrameHeaderSize + kFrameTrailerSize); + EXPECT_LT(pages.merge_target_bytes, pages.target_page_bytes); + EXPECT_EQ(pages.primary_page_bytes, 4096U); + } +} + +TEST(TreeCoreGeometry, ExplicitSplitCapsAtFrameAndCurrentWirePayload) +{ + Config config; + PageMaintenanceLimits pages; + config.leaf_split_bytes = 65536; + ASSERT_TRUE(configure_leaf_geometry(config, &pages).ok()); + EXPECT_EQ(pages.target_page_bytes, crowdb::protocol::kMaxFrameV2PayloadBytes); + config.frame_bytes = 4096; + ASSERT_TRUE(configure_leaf_geometry(config, &pages).ok()); + EXPECT_EQ(pages.target_page_bytes, 4096U); + config.leaf_merge_bytes = 0; + ASSERT_TRUE(configure_leaf_geometry(config, &pages).ok()); + EXPECT_EQ(pages.merge_trigger_bytes, kFrameHeaderSize + kFrameTrailerSize); +} + +TEST(TreeCoreGeometry, InvalidAndOverflowingGeometryFailsBeforePlanning) +{ + Config config; + PageMaintenanceLimits pages; + for (const size_t target : {size_t{0}, std::numeric_limits<size_t>::max()}) { + config.leaf_split_bytes = target; + EXPECT_EQ(configure_leaf_geometry(config, &pages).code(), Code::kInvalidArgument); + } + config.leaf_split_bytes = 128; + config.leaf_merge_bytes = std::numeric_limits<size_t>::max(); + EXPECT_EQ(configure_leaf_geometry(config, &pages).code(), Code::kInvalidArgument); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_core_read_test.cpp b/lib/crowdb-tree/tests/unit/tree_core_read_test.cpp new file mode 100644 index 000000000..43f46cdf0 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_core_read_test.cpp @@ -0,0 +1,282 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core.h" +#include "btree/tree_core_read_internal.h" + +#include <gtest/gtest.h> + +#include <chrono> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +class TreeCoreRead : public testing::Test +{ + protected: + void SetUp() override + { + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + Config config; + config.runtime = runtime; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, config.runtime_binding, &binding).ok()); + auto status = TreeCore::create(config, binding, {}, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + + void use_single_io() + { + core->close_and_drain(); + core.reset(); + binding.reset(); + Config config; + config.runtime = runtime; + config.runtime_binding.limits.io_operations = 1; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, config.runtime_binding, &binding).ok()); + ASSERT_TRUE(TreeCore::create(config, binding, {}, &core).ok()); + } + + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<TreeRuntimeBinding> binding; + std::shared_ptr<TreeCore> core; +}; + +TEST_F(TreeCoreRead, DefaultAdmissionGetsAndRetainsImmutableValueAfterClose) +{ + Batch batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "original"}}}; + ASSERT_TRUE(core->apply(1, batch).ok()); + GetView value; + auto status = core->get("key", &value); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.value().to_string(), "original"); + batch.ops[0].value = "replacement"; + ASSERT_TRUE(core->apply(2, batch).ok()); + core->close_and_drain(); + core.reset(); + binding.reset(); + EXPECT_EQ(value.value().to_string(), "original"); + EXPECT_EQ(value.slot(), 1U); + EXPECT_GT(runtime->snapshot().allocated.writer_bytes, 0U); + value = {}; + // Late value owners enqueue paid reference cleanup on the host queue. + runtime->drain_cpu(); + EXPECT_EQ(runtime->snapshot().allocated.writer_bytes, 0U); +} + +TEST(TreeCoreReadProgress, SynchronousResidentReadDoesNotDependOnAnOccupiedCpuWorker) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({.workers = 1}, &runtime).ok()); + Config config; + config.runtime = runtime; + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, config.runtime_binding, &binding).ok()); + std::shared_ptr<TreeCore> core; + ASSERT_TRUE(TreeCore::create(config, binding, {}, &core).ok()); + ASSERT_TRUE(core->apply(1, {.ops = {{.key = "key", .kind = OpKind::kPut, .value = "value"}}}).ok()); + + struct Gate + { + std::atomic<bool> entered{false}; + std::atomic<bool> released{false}; + } gate; + + ASSERT_TRUE(core->binding() + ->cpu() + ->try_submit( + core->binding()->lane(), + [](void *context) noexcept { + auto &pending = *static_cast<Gate *>(context); + pending.entered.store(true, std::memory_order_release); + pending.entered.notify_all(); + pending.released.wait(false, std::memory_order_acquire); + }, + &gate) + .ok()); + gate.entered.wait(false, std::memory_order_acquire); + Status asynchronous_status; + GetView asynchronous_value; + std::atomic<bool> asynchronous_done{false}; + core->get_async("key", [&](Status outcome, GetView value) { + asynchronous_status = std::move(outcome); + asynchronous_value = std::move(value); + asynchronous_done.store(true, std::memory_order_release); + }); + EXPECT_FALSE(asynchronous_done.load(std::memory_order_acquire)); + Status status; + Status scan_status; + std::vector<scan_entry> entries; + bool truncated = false; + GetView result; + std::atomic<bool> done{false}; + std::thread reader([&] { + status = core->get("key", &result); + scan_status = core->scan({}, &entries, &truncated); + done.store(true, std::memory_order_release); + }); + const auto until = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!done.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < until) { + std::this_thread::yield(); + } + const bool completed_inline = done.load(std::memory_order_acquire); + gate.released.store(true, std::memory_order_release); + gate.released.notify_all(); + reader.join(); + core->binding()->drain(); + EXPECT_TRUE(completed_inline); + EXPECT_TRUE(asynchronous_done.load(std::memory_order_acquire)); + EXPECT_TRUE(asynchronous_status.ok()) << asynchronous_status.to_string(); + EXPECT_TRUE(asynchronous_value.found()); + EXPECT_TRUE(scan_status.ok()) << scan_status.to_string(); + EXPECT_FALSE(truncated); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries.front().key, "key"); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(result.found()); + EXPECT_EQ(result.value().to_string(), "value"); +} + +TEST_F(TreeCoreRead, ScanDefaultsAdmitAndRespectEmptyCursorAndDirection) +{ + Batch batch{ + .ops = {{.key = "", .kind = OpKind::kPut, .value = "empty"}, + {.key = "a", .kind = OpKind::kPut, .value = "one"}, + {.key = "b", .kind = OpKind::kPut, .value = "two"}} + }; + ASSERT_TRUE(core->apply(1, batch).ok()); + std::vector<scan_entry> entries; + bool truncated = false; + CoreScanOptions options{}; + options.has_start_bound = true; + auto status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), 2U); + EXPECT_EQ(entries[0].key, "a"); + EXPECT_EQ(entries[1].key, "b"); + options.reverse = true; + options.start_inclusive = true; + status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries[0].key, ""); +} + +TEST_F(TreeCoreRead, KeysOnlyTombstonesAndStrictBudgetUseTheSameCapturedRead) +{ + Batch batch{ + .ops = {{.key = "a", .kind = OpKind::kPut, .value = std::string(8192, 'x')}, + {.key = "b", .kind = OpKind::kDelete, .value = ""}} + }; + ASSERT_TRUE(core->apply(1, batch).ok()); + std::vector<scan_entry> entries; + bool truncated = false; + CoreScanOptions options{}; + options.byte_budget = 2; + options.keys_only = true; + options.include_tombstones = true; + options.strict_byte_budget = true; + auto status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), 2U); + EXPECT_TRUE(entries[0].value.empty()); + EXPECT_TRUE(entries[1].tombstone); + options.keys_only = false; + options.include_tombstones = false; + status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_TRUE(entries.empty()); + EXPECT_TRUE(truncated); + options.strict_byte_budget = false; + status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries[0].value.size(), 8192U); + // The trailing tombstone is excluded, so the bounded probe proves the end. + EXPECT_FALSE(truncated); + ASSERT_TRUE(core->apply(2, Batch{.ops = {{.key = "c", .kind = OpKind::kPut, .value = "next"}}}).ok()); + status = core->scan(options, &entries, &truncated); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries[0].key, "a"); + EXPECT_EQ(entries[0].value.size(), 8192U); + EXPECT_TRUE(truncated); +} + +TEST_F(TreeCoreRead, PackedResultKeepsCapacityChargedAfterTreeClose) +{ + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "value"}}}).ok()); + ScanPackedBuf packed; + bool truncated = false; + size_t count = 0; + auto status = core->scan({}, nullptr, &truncated, &packed, &count); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(count, 1U); + const auto size = packed.size(); + core->close_and_drain(); + core.reset(); + binding.reset(); + const auto held = runtime->snapshot().allocated.routing_bytes; + EXPECT_GT(held, size); + packed = {}; + EXPECT_LT(runtime->snapshot().allocated.routing_bytes, held); +} + +TEST_F(TreeCoreRead, TerminalGetReturnsIoCreditBeforeCallingConsumer) +{ + use_single_io(); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "value"}}}).ok()); + std::atomic<bool> done{false}; + Status first_status; + Status second_status; + bool first_found = false; + bool second_found = false; + core->get_async("key", [&](Status status, GetView value) { + first_status = std::move(status); + first_found = value.found(); + core->get_async("key", [&](Status next_status, GetView next_value) { + second_status = std::move(next_status); + second_found = next_value.found(); + done.store(true, std::memory_order_release); + done.notify_one(); + }); + }); + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + EXPECT_TRUE(first_status.ok()) << first_status.to_string(); + EXPECT_TRUE(first_found); + EXPECT_TRUE(second_status.ok()) << second_status.to_string(); + EXPECT_TRUE(second_found); +} + +TEST_F(TreeCoreRead, TerminalScanReturnsIoCreditBeforeCallingConsumer) +{ + use_single_io(); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "value"}}}).ok()); + std::atomic<bool> done{false}; + Status first_status; + Status second_status; + size_t first_bytes = 0; + size_t second_bytes = 0; + core->scan_async({}, [&](Status status, ScanPackedBuf packed, bool) { + first_status = std::move(status); + first_bytes = packed.size(); + core->scan_async({}, [&](Status next_status, ScanPackedBuf next_packed, bool) { + second_status = std::move(next_status); + second_bytes = next_packed.size(); + done.store(true, std::memory_order_release); + done.notify_one(); + }); + }); + while (!done.load(std::memory_order_acquire)) { + done.wait(false, std::memory_order_acquire); + } + EXPECT_TRUE(first_status.ok()) << first_status.to_string(); + EXPECT_GT(first_bytes, 0U); + EXPECT_TRUE(second_status.ok()) << second_status.to_string(); + EXPECT_EQ(second_bytes, first_bytes); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_operation_lifetime_test.cpp b/lib/crowdb-tree/tests/unit/tree_operation_lifetime_test.cpp new file mode 100644 index 000000000..3e04ba55b --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_operation_lifetime_test.cpp @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "load_budget_test_support.h" +#include "runtime/tree_operation_lifetime.h" + +#include <future> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +using test_read::LoadCredits; + +TEST(TreeOperationLifetime, CloseRejectsIngressAndWaitsForEveryAcceptedOwner) +{ + LoadCredits credits; + TreeOperationLifetime lifetime(2, credits.credits.bytes); + std::shared_ptr<TreeOperationLifetime::Ticket> first; + std::shared_ptr<TreeOperationLifetime::Ticket> second; + std::shared_ptr<TreeOperationLifetime::Ticket> rejected; + ASSERT_TRUE(lifetime.acquire(credits.credits.bytes, &first).ok()); + ASSERT_TRUE(lifetime.acquire(credits.credits.bytes, &second).ok()); + EXPECT_EQ(lifetime.acquire(credits.credits.bytes, &rejected).code(), Code::kResourceExhausted); + lifetime.close(); + EXPECT_EQ(lifetime.acquire(credits.credits.bytes, &rejected).code(), Code::kUnavailable); + std::atomic<bool> drained{false}; + std::promise<void> started; + auto ready = started.get_future(); + std::jthread closer([&] { + started.set_value(); + lifetime.close_and_drain(); + drained.store(true, std::memory_order_release); + }); + ready.wait(); + EXPECT_FALSE(drained.load(std::memory_order_acquire)); + first.reset(); + EXPECT_EQ(lifetime.outstanding(), 1U); + EXPECT_FALSE(drained.load(std::memory_order_acquire)); + second.reset(); + closer.join(); + EXPECT_TRUE(drained.load(std::memory_order_acquire)); + EXPECT_EQ(lifetime.outstanding(), 0U); +} + +TEST(TreeOperationLifetime, ObjectAndControlBlockAllocationFailuresReturnCapacityExactlyOnce) +{ + LoadCredits credits; + TreeOperationLifetime lifetime(1); + // Exercise both object allocation and the later shared control allocation. + // The latter destroys the newly constructed Ticket during rollback. + const auto bound = TreeOperationLifetime::ticket_allocation_bound(); + size_t rejected = 0; + size_t admitted = 0; + for (size_t available = 0; available <= bound; ++available) { + AllocationLease pressure(credits.credits.bytes, credits.credits.bytes->available() - available); + const auto baseline = credits.budget.snapshot().allocated[LoadCredits::kBytes]; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + const auto status = lifetime.acquire(credits.credits.bytes, &ticket); + if (status.ok()) { + ++admitted; + EXPECT_EQ(lifetime.outstanding(), 1U); + ticket.reset(); + } + else { + ++rejected; + EXPECT_EQ(status.code(), Code::kResourceExhausted); + EXPECT_FALSE(ticket); + } + EXPECT_EQ(lifetime.outstanding(), 0U); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], baseline); + } + EXPECT_GT(rejected, 0U); + EXPECT_GT(admitted, 0U); + lifetime.close_and_drain(); +} + +TEST(TreeOperationLifetime, SoftAdmissionReportsOccupiedCapacityWithoutChangingTheHardContract) +{ + TreeOperationLifetime lifetime(1); + std::shared_ptr<TreeOperationLifetime::Ticket> first; + std::shared_ptr<TreeOperationLifetime::Ticket> next; + ASSERT_TRUE(lifetime.acquire({}, &first).ok()); + EXPECT_EQ(lifetime.try_acquire({}, &next).code(), Code::kUnavailable); + EXPECT_EQ(lifetime.acquire({}, &next).code(), Code::kResourceExhausted); + const auto usage = lifetime.resource_usage(); + EXPECT_EQ(usage.limit, 1U); + EXPECT_EQ(usage.used, 1U); + EXPECT_EQ(usage.denied_tree, 2U); + first.reset(); + ASSERT_TRUE(lifetime.try_acquire({}, &next).ok()); + EXPECT_EQ(lifetime.resource_usage().used, 1U); + next.reset(); + EXPECT_EQ(lifetime.resource_usage().used, 0U); + EXPECT_EQ(lifetime.resource_usage().denied_tree, 2U); +} + +TEST(TreeOperationLifetime, TicketRetainsChargedStateAndHostAfterIngressOwnerIsDropped) +{ + LoadCredits credits; + auto host = std::make_shared<int>(17); + const std::weak_ptr<int> weak = host; + std::shared_ptr<TreeOperationLifetime::Ticket> ticket; + { + TreeOperationLifetime lifetime(1, credits.credits.bytes, host); + ASSERT_TRUE(lifetime.acquire(credits.credits.bytes, &ticket).ok()); + host.reset(); + EXPECT_FALSE(weak.expired()); + } + credits.seal(); + EXPECT_FALSE(weak.expired()); + EXPECT_GT(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); + ticket.reset(); + EXPECT_TRUE(weak.expired()); + EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_physical_stats_test.cpp b/lib/crowdb-tree/tests/unit/tree_physical_stats_test.cpp new file mode 100644 index 000000000..770916c2c --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_physical_stats_test.cpp @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "crowdb-tree/btree/tree.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(TreePhysicalStats, ContainerCapacitiesRemainClassifiedAcrossHandoffAndSplit) +{ + Config config; + config.leaf_split_bytes = 1024; + config.frame_bytes = 4096; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + ASSERT_NE(core, nullptr); + for (size_t i = 0; i < 32; ++i) { + ASSERT_TRUE(tree.put(std::string(64, 'k') + std::to_string(i), std::string(256, 'v')).ok()); + } + for (size_t phase = 0; phase < 2; ++phase) { + if (phase != 0) { + ASSERT_TRUE(tree.flush().ok()); + ASSERT_GT(tree.leaf_count(), 1U); + } + core->binding()->drain(); + const auto stats = tree.tree_state_stats(); + const auto &r = stats.resources; + EXPECT_GE(r.payload_bytes, 32U * 256); + EXPECT_GT(r.metadata_bytes, 0U); + EXPECT_EQ(r.allocated_bytes, r.payload_bytes + r.frame_bytes + r.metadata_bytes + r.io_bytes + r.scratch_bytes); + } +} + +TEST(TreePhysicalStats, RetainedValueStaysChargedAcrossClearUntilItsActualOwnerDrops) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + auto *core = TreeCoreAccess::get(tree); + ASSERT_NE(core, nullptr); + ASSERT_TRUE(tree.put("key", std::string(8192, 'v')).ok()); + auto retained = tree.get_view("key"); + ASSERT_TRUE(retained.found()); + const auto before = tree.tree_state_stats(); + EXPECT_GE(before.resources.payload_bytes, 8192U); + ASSERT_TRUE(tree.clear().ok()); + core->binding()->drain(); + const auto held = tree.tree_state_stats(); + EXPECT_GE(held.resources.payload_bytes, 8192U); + EXPECT_EQ(retained.value().size(), 8192U); + const auto &r = held.resources; + EXPECT_EQ(r.allocated_bytes, r.payload_bytes + r.frame_bytes + r.metadata_bytes + r.io_bytes + r.scratch_bytes); + retained = {}; + core->binding()->drain(); + const auto released = tree.tree_state_stats(); + EXPECT_LT(released.resources.payload_bytes, held.resources.payload_bytes); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_progress_metrics_test.cpp b/lib/crowdb-tree/tests/unit/tree_progress_metrics_test.cpp new file mode 100644 index 000000000..f6862a94f --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_progress_metrics_test.cpp @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/logical_page.h" +#include "btree/published_state.h" +#include "c_api_internal.h" +#include "crowdb-tree/tree_stats_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(TreeProgressMetrics, MaintenanceSlotsBoundAdmissionAndTrackOldestUntilItsActualTerminal) +{ + PublicationCoordinator metrics; + std::array<size_t, PublicationCoordinator::kMaintenanceSlots> slots{}; + for (size_t i = 0; i < slots.size(); ++i) { + slots[i] = metrics.start_maintenance(100 + i); + ASSERT_LT(slots[i], slots.size()); + } + EXPECT_EQ(metrics.start_maintenance(1), slots.size()); + EXPECT_EQ(metrics.oldest_maintenance_since_ns(), 100U); + metrics.finish_maintenance(slots[0]); + EXPECT_EQ(metrics.oldest_maintenance_since_ns(), 101U); + slots[0] = metrics.start_maintenance(500); + EXPECT_LT(slots[0], slots.size()); + EXPECT_EQ(metrics.oldest_maintenance_since_ns(), 101U); + for (const auto slot : slots) { + metrics.finish_maintenance(slot); + } + EXPECT_EQ(metrics.oldest_maintenance_since_ns(), 0U); +} + +TEST(TreeProgressMetrics, ChronologicalMembershipSurvivesEraseSwapAndRepeatedChanges) +{ + PendingPageTimes pages(std::pmr::get_default_resource()); + PendingPageTimes other(std::pmr::get_default_resource()); + EXPECT_TRUE(pages.insert(900, 100)); + EXPECT_TRUE(pages.insert(1, 200)); + EXPECT_TRUE(pages.insert(700, 300)); + EXPECT_FALSE(pages.insert(900, 400)); + EXPECT_EQ(pages.oldest_since_ns(), 100U); + pages.erase(1); + EXPECT_EQ(pages.oldest_since_ns(), 100U); + other.swap(pages); + EXPECT_EQ(pages.oldest_since_ns(), 0U); + other.erase(900); + EXPECT_EQ(other.oldest_since_ns(), 300U); + other.erase_first(); + EXPECT_TRUE(other.empty()); + EXPECT_TRUE(other.insert(1, 500)); + EXPECT_EQ(other.oldest_since_ns(), 500U); +} + +TEST(TreeProgressMetrics, CaptureCommitRetainsNewChangesAndFailureKeepsTheOriginalAge) +{ + CheckpointPendingMetrics pending; + pending.change_page(10); + const auto original = pending.oldest_since_ns(); + ASSERT_NE(original, 0U); + ASSERT_TRUE(pending.capture(1)); + pending.change_page(10); + pending.change_page(20); + EXPECT_EQ(pending.oldest_since_ns(), original); + ASSERT_TRUE(pending.finish(1, false)); + EXPECT_EQ(pending.oldest_since_ns(), original); + ASSERT_TRUE(pending.capture(2)); + pending.change_page(20); + ASSERT_TRUE(pending.finish(2, true)); + EXPECT_FALSE(pending.contains(10)); + EXPECT_TRUE(pending.contains(20)); + EXPECT_GE(pending.oldest_since_ns(), original); + ASSERT_TRUE(pending.capture(3)); + ASSERT_TRUE(pending.finish(3, true)); + EXPECT_EQ(pending.oldest_since_ns(), 0U); + pending.remove_page(20); + EXPECT_NE(pending.oldest_since_ns(), 0U); + ASSERT_TRUE(pending.capture(4)); + ASSERT_TRUE(pending.finish(4, true)); + EXPECT_EQ(pending.oldest_since_ns(), 0U); +} + +TEST(TreeProgressMetrics, CandidateFirstEntrySurvivesReplacementAndOldViews) +{ + MappingView mapping; + MappingView::CopyStats copied; + auto candidate = std::make_shared<PublishedPage>(); + candidate->leaf = std::make_shared<PageState>(); + candidate->metrics.consolidate_candidate = true; + const std::array<PageReplacement, 1> first{ + {{.page_id = MappingView::kPageLimit - 1, .expected = nullptr, .replacement = candidate}}}; + mapping = mapping.replace(first, copied); + const auto oldest = mapping.copy_bounds().candidate_since_ns; + ASSERT_NE(oldest, 0U); + const auto old_view = mapping; + auto changed = std::make_shared<PublishedPage>(*candidate); + changed->revision = 20; + changed->metrics.consolidate_candidate = false; + changed->metrics.writeback_candidate = true; + const std::array<PageReplacement, 1> second{ + {{.page_id = MappingView::kPageLimit - 1, .expected = candidate, .replacement = changed}}}; + mapping = mapping.replace(second, copied); + EXPECT_EQ(mapping.copy_bounds().candidate_since_ns, oldest); + const std::array<PageReplacement, 1> remove{ + {{.page_id = MappingView::kPageLimit - 1, .expected = changed, .replacement = nullptr}}}; + mapping = mapping.replace(remove, copied); + EXPECT_EQ(mapping.copy_bounds().candidate_since_ns, 0U); + EXPECT_EQ(old_view.copy_bounds().candidate_since_ns, oldest); +} + +TEST(TreeProgressMetrics, VersionedCAbiSamplesWithoutChangingTheExistingSnapshotContract) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + ct_tree_progress_snapshot_v1 sample{}; + sample.struct_size = sizeof(sample); + sample.version = 1; + ASSERT_EQ(ct_get_tree_progress_stats(tree, &sample), 0); + EXPECT_NE(sample.instance_generation, 0U); + EXPECT_NE(sample.sampled_at_ns, 0U); + const auto sampled_at = sample.sampled_at_ns; + sample.version = 2; + EXPECT_EQ(ct_get_tree_progress_stats(tree, &sample), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(sample.sampled_at_ns, sampled_at); + ct_close(tree); + ct_runtime_free(runtime); +} + +TEST(TreeProgressMetrics, OldestWritebackUsesItsOwnMembershipAgeAcrossSparseImmutableViews) +{ + MappingView mapping; + MappingView::CopyStats copied; + const auto make = [] { + auto page = std::make_shared<PublishedPage>(); + page->leaf = std::make_shared<PageState>(); + page->metrics.dirty = true; + page->metrics.writeback_candidate = true; + return page; + }; + const auto older = make(); + const auto newer = make(); + const uint64_t older_pid = MappingView::kPageLimit - 1; + const std::array<PageReplacement, 1> first{{{.page_id = older_pid, .expected = nullptr, .replacement = older}}}; + mapping = mapping.replace(first, copied); + const auto age = mapping.copy_bounds().writeback_since_ns; + ASSERT_NE(age, 0U); + const auto old_view = mapping; + const std::array<PageReplacement, 1> second{{{.page_id = 1, .expected = nullptr, .replacement = newer}}}; + mapping = mapping.replace(second, copied); + EXPECT_EQ(mapping.oldest_writeback_leaf(), older_pid); + auto changed = std::make_shared<PublishedPage>(*older); + ++changed->revision; + const std::array<PageReplacement, 1> replace{{{.page_id = older_pid, .expected = older, .replacement = changed}}}; + mapping = mapping.replace(replace, copied); + EXPECT_EQ(mapping.copy_bounds().writeback_since_ns, age); + changed = std::make_shared<PublishedPage>(*changed); + changed->metrics.dirty = false; + changed->metrics.writeback_candidate = false; + changed->metrics.evict_candidate = true; + const std::array<PageReplacement, 1> clean{{{.page_id = older_pid, .expected = nullptr, .replacement = changed}}}; + mapping = mapping.replace(clean, copied); + EXPECT_EQ(mapping.oldest_writeback_leaf(), 1U); + EXPECT_EQ(old_view.oldest_writeback_leaf(), older_pid); + const std::array<PageReplacement, 1> remove{{{.page_id = 1, .expected = nullptr, .replacement = nullptr}}}; + mapping = mapping.replace(remove, copied); + EXPECT_FALSE(mapping.oldest_writeback_leaf()); + EXPECT_EQ(mapping.copy_bounds().writeback_since_ns, 0U); + EXPECT_NE(mapping.copy_bounds().candidate_since_ns, 0U); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_runtime_c_api_test.cpp b/lib/crowdb-tree/tests/unit/tree_runtime_c_api_test.cpp new file mode 100644 index 000000000..cfca73b59 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_runtime_c_api_test.cpp @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/c_api.h" +#include "runtime/tree_runtime_binding.h" +#include "runtime/tree_runtime_c_api.h" + +#include <gtest/gtest.h> + +#include <vector> + +namespace crowdb::tree +{ +namespace +{ +TEST(TreeRuntimeCAbi, NullRuntimeUsesTheProcessHostAndEnforcesWriterCredit) +{ + ct_tree_runtime_options binding{}; + ct_tree_runtime_options_default(&binding); + binding.limits.writer_bytes = 1024 * 1024; + Config first; + Config second; + ASSERT_TRUE(configure_tree_runtime(nullptr, &binding, &first).ok()); + ASSERT_TRUE(configure_tree_runtime(nullptr, nullptr, &second).ok()); + ASSERT_EQ(first.runtime, second.runtime); + EXPECT_EQ(first.runtime_binding.limits.writer_bytes, 1024U * 1024U); + + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, nullptr, &binding, &tree), 0); + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + EXPECT_EQ(core->config().runtime, first.runtime); + EXPECT_EQ(core->binding()->options().limits.writer_bytes, 1024U * 1024U); + const std::vector<uint8_t> value(2 * 1024 * 1024); + EXPECT_EQ(ct_apply_put(tree, 1, reinterpret_cast<const uint8_t *>("key"), 3, value.data(), value.size()), + static_cast<int32_t>(Code::kUnavailable)); + EXPECT_EQ(core->contiguous_slot(), 0U); + ct_close(tree); +} + +TEST(TreeRuntimeCAbi, NullRuntimeDoesNotBypassUnfundedOpenOrRebuildBinding) +{ + ct_tree_runtime_options binding{}; + ct_tree_runtime_options_default(&binding); + binding.lifecycle_bytes = 1; + ct_options options{}; + ct_tree *tree = nullptr; + EXPECT_EQ(ct_open_with_runtime(&options, nullptr, &binding, &tree), static_cast<int32_t>(Code::kResourceExhausted)); + EXPECT_EQ(tree, nullptr); + + ct_tree *source = nullptr; + ASSERT_EQ(ct_open(&options, &source), 0); + ct_page_store *store = nullptr; + ASSERT_EQ(ct_page_store_open_mem(1, &store), 0); + options.page_store = store; + EXPECT_EQ(ct_rebuild_range_with_runtime(source, &options, nullptr, &binding, &tree, nullptr), + static_cast<int32_t>(Code::kResourceExhausted)); + EXPECT_EQ(tree, nullptr); + ct_page_store_free(store); + ct_close(source); +} + +TEST(TreeRuntimeCAbi, NullRuntimeKeepsTheDefaultReloadableMemoryBackend) +{ + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, nullptr, nullptr, &tree), 0); + const uint8_t key = 'k'; + const uint8_t value = 'v'; + ASSERT_EQ(ct_apply_put(tree, 1, &key, 1, &value, 1), 0); + uint64_t generation = 0; + uint64_t coverage = 0; + EXPECT_EQ(ct_snapshot_info(tree, &generation, &coverage), 0); + EXPECT_GT(generation, 0U); + EXPECT_EQ(coverage, 1U); + ct_close(tree); +} + +TEST(TreeRuntimeCAbi, VersionedOptionsRejectWrongSizeAndUnfundedStorage) +{ + ct_runtime_options options{}; + ct_runtime_options_default(&options); + EXPECT_EQ(options.struct_size, sizeof(options)); + ct_runtime *runtime = nullptr; + --options.struct_size; + EXPECT_EQ(ct_runtime_create(&options, &runtime), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(runtime, nullptr); + ++options.struct_size; + options.lifecycle_bytes = 1; + EXPECT_EQ(ct_runtime_create(&options, &runtime), static_cast<int32_t>(Code::kResourceExhausted)); + EXPECT_EQ(runtime, nullptr); + ct_runtime_free(nullptr); +} + +TEST(TreeRuntimeCAbi, FreeHandleRetainsIndependentConfigAndBindingOwners) +{ + ct_runtime *handle = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &handle), 0); + Config config; + ASSERT_TRUE(configure_tree_runtime(handle, nullptr, &config).ok()); + auto owner = config.runtime; + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(owner, config.runtime_binding, &binding).ok()); + const auto before = owner->snapshot().allocated.routing_bytes; + ct_runtime_free(handle); + EXPECT_FALSE(owner->stopped()); + EXPECT_LT(owner->snapshot().allocated.routing_bytes, before); + config.runtime.reset(); + owner.reset(); + EXPECT_TRUE(binding->cpu()->try_submit(binding->lane(), [](void *) noexcept {}, nullptr).ok()); + binding->cpu()->drain(); +} + +TEST(TreeRuntimeCAbi, OpenRuntimeUsesOwnedCoreAndRejectsInvalidBinding) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + ASSERT_NE(tree, nullptr); + uint64_t checkpoint = 0; + EXPECT_EQ(ct_snapshot(tree, &checkpoint), static_cast<int32_t>(Code::kUnavailable)); + ct_close(tree); + tree = nullptr; + ct_tree_runtime_options binding{}; + ct_tree_runtime_options_default(&binding); + binding.cpu_queue_limit = 0; + EXPECT_EQ(ct_open_with_runtime(&options, runtime, &binding, &tree), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(tree, nullptr); + ct_runtime_stop_admission(runtime); + Config config; + EXPECT_EQ(configure_tree_runtime(runtime, nullptr, &config).code(), Code::kUnavailable); + ct_runtime_drain_cpu(runtime); + ct_runtime_free(runtime); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_runtime_test.cpp b/lib/crowdb-tree/tests/unit/tree_runtime_test.cpp new file mode 100644 index 000000000..7d5382e1d --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_runtime_test.cpp @@ -0,0 +1,195 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/btree/tree.h" +#include "runtime/tree_runtime_binding.h" +#include "runtime/tree_runtime_internal.h" + +#include <gtest/gtest.h> + +#include <atomic> +#include <limits> +#include <stdexcept> + +namespace crowdb::tree +{ +namespace +{ +using detail::BudgetClass; +using detail::ResourceBudget; + +TEST(TreeRuntime, BindingsShareCpuAndHostBudgetButKeepIndependentQuotas) +{ + TreeRuntimeOptions options; + options.workers = 1; + options.limits.writer_bytes = 100; + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create(options, &runtime).ok()); + const auto baseline = runtime->snapshot(); + EXPECT_GT(baseline.allocated.routing_bytes, 0U); + EXPECT_EQ(baseline.reserved.routing_bytes + baseline.allocated.routing_bytes, options.lifecycle_bytes); + + TreeBindingOptions binding; + binding.limits.writer_bytes = 80; + std::shared_ptr<TreeRuntimeBinding> first; + std::shared_ptr<TreeRuntimeBinding> second; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, binding, &first).ok()); + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, binding, &second).ok()); + EXPECT_EQ(first->cpu(), second->cpu()); + EXPECT_NE(first->lane(), second->lane()); + EXPECT_NE(first->account(), second->account()); + EXPECT_EQ(first->lifecycle_budget()->domain(), second->lifecycle_budget()->domain()); + EXPECT_EQ(runtime->snapshot().bindings, 2U); + + auto a = ResourceBudget::try_reserve(first->account(), BudgetClass::kAdmittedWriter, 60); + ASSERT_TRUE(a); + EXPECT_FALSE(ResourceBudget::try_reserve(first->account(), BudgetClass::kAdmittedWriter, 21)); + EXPECT_FALSE(ResourceBudget::try_reserve(second->account(), BudgetClass::kAdmittedWriter, 41)); + auto b = ResourceBudget::try_reserve(second->account(), BudgetClass::kAdmittedWriter, 40); + ASSERT_TRUE(b); + EXPECT_EQ(runtime->snapshot().reserved.writer_bytes, 100U); + a.reset(); + b.reset(); + first.reset(); + second.reset(); + const auto after = runtime->snapshot(); + EXPECT_EQ(after.bindings, 0U); + EXPECT_EQ(after.reserved.writer_bytes, 0U); + EXPECT_EQ(after.allocated.routing_bytes, baseline.allocated.routing_bytes); + EXPECT_EQ(after.reserved.routing_bytes, baseline.reserved.routing_bytes); +} + +TEST(TreeRuntime, WeakMetricsControlDoesNotRetainItsTreeAccount) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + const auto baseline = runtime->snapshot(); + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + auto account = binding->account(); + auto metrics = binding->work_metrics(); + std::weak_ptr<detail::ResourceBudget::Account> weak_account = account; + binding.reset(); + EXPECT_GT(account->snapshot().allocated[1], 0U); + metrics.reset(); + EXPECT_EQ(account->snapshot().allocated[1], 0U); + EXPECT_EQ(account->snapshot().reserved[1], 0U); + EXPECT_EQ(account->snapshot().physical[static_cast<size_t>(AllocationKind::kMetadata)], 0U); + account.reset(); + EXPECT_TRUE(weak_account.expired()); + weak_account.reset(); + EXPECT_EQ(runtime->snapshot().allocated.routing_bytes, baseline.allocated.routing_bytes); + EXPECT_EQ(runtime->snapshot().reserved.routing_bytes, baseline.reserved.routing_bytes); +} + +TEST(TreeRuntime, LateMetricsReleaseReturnsHostLifecycleAfterExecutorIsGone) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + std::weak_ptr<AllocationBudget> lifecycle = TreeRuntimeAccess::lifecycle(*runtime); + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + auto metrics = binding->work_metrics(); + binding.reset(); + runtime.reset(); + EXPECT_FALSE(lifecycle.expired()); + metrics.reset(); + EXPECT_TRUE(lifecycle.expired()); +} + +TEST(TreeRuntime, StoppingAdmissionKeepsAcceptedCpuContinuationsUsable) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + auto credit = ResourceBudget::try_reserve(binding->account(), BudgetClass::kHandoff, 1024); + ASSERT_TRUE(credit); + runtime->stop_admission(); + runtime->stop_admission(); + EXPECT_TRUE(runtime->snapshot().stopped); + EXPECT_FALSE(ResourceBudget::try_reserve(binding->account(), BudgetClass::kHandoff, 1)); + std::shared_ptr<TreeRuntimeBinding> rejected; + EXPECT_EQ(TreeRuntimeBinding::create(runtime, {}, &rejected).code(), Code::kUnavailable); + std::atomic<size_t> completed{0}; + const auto run = [](void *value) noexcept { static_cast<std::atomic<size_t> *>(value)->fetch_add(1); }; + ASSERT_TRUE(binding->cpu()->try_submit(binding->lane(), run, &completed).ok()); + binding->cpu()->drain(); + EXPECT_EQ(completed.load(), 1U); + EXPECT_EQ(runtime->snapshot().reserved.handoff_bytes, 1024U); + credit.reset(); + EXPECT_EQ(runtime->snapshot().reserved.handoff_bytes, 0U); + runtime->drain_cpu(); + EXPECT_EQ(binding->cpu()->try_submit(binding->lane(), run, &completed).code(), Code::kUnavailable); +} + +TEST(TreeRuntime, BindingRetainsHostAfterCallerDropsItsReference) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + std::weak_ptr<TreeRuntime> weak = runtime; + std::shared_ptr<TreeRuntimeBinding> binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + runtime.reset(); + EXPECT_FALSE(weak.expired()); + std::atomic<size_t> completed{0}; + ASSERT_TRUE(binding->cpu() + ->try_submit( + binding->lane(), + [](void *value) noexcept { static_cast<std::atomic<size_t> *>(value)->fetch_add(1); }, + &completed) + .ok()); + binding->cpu()->drain(); + EXPECT_EQ(completed.load(), 1U); + binding.reset(); + EXPECT_TRUE(weak.expired()); +} + +TEST(TreeRuntime, InvalidAndUnfundedRuntimeStorageAreRejectedBeforePoolConstruction) +{ + std::shared_ptr<TreeRuntime> runtime; + TreeRuntimeOptions options; + options.workers = 0; + EXPECT_EQ(TreeRuntime::create(options, &runtime).code(), Code::kInvalidArgument); + options.workers = std::numeric_limits<size_t>::max(); + EXPECT_EQ(TreeRuntime::create(options, &runtime).code(), Code::kInvalidArgument); + options.workers = 1; + options.lifecycle_bytes = 256; + EXPECT_EQ(TreeRuntime::create(options, &runtime).code(), Code::kResourceExhausted); + EXPECT_FALSE(runtime); +} + +TEST(TreeRuntime, FailedBindingCreationReturnsAccountAndReservationStorage) +{ + std::shared_ptr<TreeRuntime> runtime; + ASSERT_TRUE(TreeRuntime::create({}, &runtime).ok()); + const auto baseline = runtime->snapshot(); + std::shared_ptr<TreeRuntimeBinding> binding; + TreeBindingOptions options; + options.lifecycle_bytes = 128; + EXPECT_EQ(TreeRuntimeBinding::create(runtime, options, &binding).code(), Code::kResourceExhausted); + options = {}; + options.limits.writer_bytes = runtime->options().limits.writer_bytes + 1; + EXPECT_EQ(TreeRuntimeBinding::create(runtime, options, &binding).code(), Code::kInvalidArgument); + const auto after = runtime->snapshot(); + EXPECT_FALSE(binding); + EXPECT_EQ(after.bindings, 0U); + EXPECT_EQ(after.allocated.routing_bytes, baseline.allocated.routing_bytes); + EXPECT_EQ(after.reserved.routing_bytes, baseline.reserved.routing_bytes); +} + +TEST(TreeRuntime, RuntimeCppConstructionOwnsAnIndependentImmutableNamespace) +{ + Config config; + ASSERT_TRUE(TreeRuntime::create({}, &config.runtime).ok()); + Crowdbtree tree(config); + EXPECT_EQ(tree.last_applied_slot(), 0U); + EXPECT_EQ(tree.leaf_count_atomic(), 1U); + EXPECT_EQ(tree.tree_state_stats().resources.resident_frames, 0U); + EXPECT_EQ(tree.tree_state_stats().logical.leaf_pages, 1U); + auto output = std::make_unique<Crowdbtree>(); + auto *original = output.get(); + EXPECT_EQ(Crowdbtree::open(config, &output).code(), Code::kInvalidArgument); + EXPECT_EQ(output.get(), original); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_state_metrics_test.cpp b/lib/crowdb-tree/tests/unit/tree_state_metrics_test.cpp new file mode 100644 index 000000000..400aa61f6 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_state_metrics_test.cpp @@ -0,0 +1,352 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "btree/tree_metrics.h" + +#include <gtest/gtest.h> + +#include <array> +#include <limits> +#include <memory> +#include <memory_resource> +#include <new> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +PageMetricsContribution known_leaf(uint64_t size, uint64_t count, uint64_t bytes) +{ + PageMetricsContribution page; + page.page_size_known = true; + page.page_size_bytes = size; + page.page_size_estimate_bytes = size; + page.kv_known = true; + page.kv_count = count; + page.kv_bytes = bytes; + return page; +} + +TEST(TreeStateMetrics, BlindOverlayInvalidatesExactSummariesWithoutInventingZeros) +{ + TreeLogicalStats stats; + auto old_page = known_leaf(4096, 8, 2048); + ASSERT_TRUE(apply_page_metrics(stats, nullptr, &old_page)); + auto overlay = old_page; + overlay.dirty = true; + overlay.overlay_runs = 2; + overlay.overlay_entries = 3; + overlay.page_size_known = false; + overlay.kv_known = false; + overlay.page_size_estimate_bytes = 8192; + overlay.consolidate_candidate = true; + overlay.split_candidate = true; + ASSERT_TRUE(apply_page_metrics(stats, &old_page, &overlay)); + EXPECT_EQ(stats.page_count(), 1U); + EXPECT_EQ(stats.dirty_pages, 1U); + EXPECT_EQ(stats.overlay_pages, 1U); + EXPECT_EQ(stats.overlay_runs, 2U); + EXPECT_EQ(stats.overlay_entries, 3U); + EXPECT_EQ(stats.page_size_known_bytes, 0U); + EXPECT_EQ(stats.page_size_unknown_pages, 1U); + EXPECT_EQ(stats.kv_known_count, 0U); + EXPECT_EQ(stats.kv_known_bytes, 0U); + EXPECT_EQ(stats.kv_unknown_pages, 1U); + EXPECT_EQ(stats.consolidate_candidate_pages, 1U); + EXPECT_EQ(stats.split_candidate_pages, 1U); + EXPECT_EQ(stats.split_ready_pages, 0U); +} + +TEST(TreeStateMetrics, MultiwaySplitAndInnerRootContributeOncePerLivePage) +{ + TreeLogicalStats stats; + auto original = known_leaf(12000, 12, 10000); + ASSERT_TRUE(apply_page_metrics(stats, nullptr, &original)); + ASSERT_TRUE(apply_page_metrics(stats, &original, nullptr)); + auto child = known_leaf(4000, 4, 3000); + child.dirty = true; + for (int i = 0; i < 3; ++i) { + ASSERT_TRUE(apply_page_metrics(stats, nullptr, &child)); + } + PageMetricsContribution root; + root.leaf = false; + root.dirty = true; + root.page_size_known = true; + root.page_size_bytes = 256; + ASSERT_TRUE(apply_page_metrics(stats, nullptr, &root)); + stats.height = 2; + EXPECT_EQ(stats.page_count(), 4U); + EXPECT_EQ(stats.leaf_pages, 3U); + EXPECT_EQ(stats.inner_pages, 1U); + EXPECT_EQ(stats.dirty_pages, 4U); + EXPECT_EQ(stats.kv_known_count, 12U); + EXPECT_EQ(stats.kv_unknown_pages, 0U); + EXPECT_EQ(stats.page_size_known_bytes, 12256U); + EXPECT_EQ(stats.page_size_max_observed_bytes, 12000U); +} + +TEST(TreeStateMetrics, InvalidDeltaDoesNotPartiallyChangeThePublishedStats) +{ + TreeLogicalStats stats; + auto page = known_leaf(4096, 1, 100); + EXPECT_FALSE(apply_page_metrics(stats, &page, nullptr)); + EXPECT_EQ(stats.page_count(), 0U); + page.split_ready = true; + EXPECT_FALSE(apply_page_metrics(stats, nullptr, &page)); + EXPECT_EQ(stats.page_count(), 0U); + page.split_ready = false; + page.overlay_entries = 1; + EXPECT_FALSE(apply_page_metrics(stats, nullptr, &page)); + page.overlay_entries = 0; + stats.page_size_known_bytes = std::numeric_limits<uint64_t>::max(); + EXPECT_FALSE(apply_page_metrics(stats, nullptr, &page)); + EXPECT_EQ(stats.page_count(), 0U); + EXPECT_EQ(stats.page_size_known_bytes, std::numeric_limits<uint64_t>::max()); +} + +TEST(TreeStateMetrics, MergeMembershipAndPairReadinessAreSeparate) +{ + TreeLogicalStats stats; + PageMetricsContribution page; + page.merge_candidate = true; + for (int i = 0; i < 3; ++i) { + ASSERT_TRUE(apply_page_metrics(stats, nullptr, &page)); + } + ASSERT_TRUE(apply_merge_pair_metrics(stats, 0, 0, 2, 1)); + EXPECT_EQ(stats.merge_candidate_pages, 3U); + EXPECT_EQ(stats.merge_candidate_pairs, 2U); + EXPECT_EQ(stats.merge_ready_pairs, 1U); + EXPECT_FALSE(apply_merge_pair_metrics(stats, 2, 0, 0, 0)); + EXPECT_EQ(stats.merge_candidate_pairs, 2U); + ASSERT_TRUE(apply_merge_pair_metrics(stats, 2, 1, 1, 1)); + EXPECT_EQ(stats.merge_candidate_pairs, 1U); + EXPECT_EQ(stats.merge_ready_pairs, 1U); +} + +TEST(TreeStateMetrics, SnapshotCommitKeepsContentChangedAfterCapturePending) +{ + CheckpointPendingMetrics pending; + pending.change_page(1); + pending.change_page(1); + pending.change_page(2); + ASSERT_TRUE(pending.capture(10)); + EXPECT_FALSE(pending.capture(11)); + pending.change_page(2); + pending.change_page(3); + EXPECT_FALSE(pending.finish(9, true)); + ASSERT_TRUE(pending.finish(10, true)); + EXPECT_FALSE(pending.finish(10, true)); + EXPECT_FALSE(pending.contains(1)); + EXPECT_TRUE(pending.contains(2)); + EXPECT_TRUE(pending.contains(3)); + TreeLogicalStats stats; + pending.copy_to(stats); + EXPECT_EQ(stats.checkpoint_pending_pages, 2U); + EXPECT_FALSE(pending.capture(10)); + ASSERT_TRUE(pending.capture(11)); + ASSERT_TRUE(pending.finish(11, true)); + pending.copy_to(stats); + EXPECT_EQ(stats.checkpoint_pending_pages, 0U); +} + +TEST(TreeStateMetrics, FailedCheckpointPreservesOriginalAndNewPendingPages) +{ + CheckpointPendingMetrics pending; + pending.change_page(1); + ASSERT_TRUE(pending.capture(1)); + pending.change_page(2); + ASSERT_TRUE(pending.finish(1, false)); + EXPECT_TRUE(pending.contains(1)); + EXPECT_TRUE(pending.contains(2)); + ASSERT_TRUE(pending.capture(2)); + ASSERT_TRUE(pending.finish(2, true)); + EXPECT_FALSE(pending.contains(1)); + EXPECT_FALSE(pending.contains(2)); +} + +class FailingMetricsResource : public std::pmr::memory_resource +{ + public: + size_t allocations = 0; + size_t fail_at = 0; + + private: + void *do_allocate(size_t bytes, size_t alignment) override + { + if (++allocations == fail_at) { + throw std::bad_alloc(); + } + return std::pmr::new_delete_resource()->allocate(bytes, alignment); + } + + void do_deallocate(void *ptr, size_t bytes, size_t alignment) override + { + std::pmr::new_delete_resource()->deallocate(ptr, bytes, alignment); + } + + [[nodiscard]] bool do_is_equal(const std::pmr::memory_resource &other) const noexcept override + { + return this == &other; + } +}; + +TEST(TreeStateMetrics, AllocationFailureRollsBackBothCheckpointMembershipSets) +{ + size_t failures = 0; + for (size_t offset = 1; offset <= 4; ++offset) { + FailingMetricsResource resource; + CheckpointPendingMetrics pending(&resource); + pending.change_page(1); + ASSERT_TRUE(pending.capture(1)); + resource.fail_at = resource.allocations + offset; + bool inserted = false; + try { + pending.change_page(2); + inserted = true; + } + catch (const std::bad_alloc &) { + ++failures; + } + EXPECT_TRUE(pending.contains(1)); + EXPECT_EQ(pending.contains(2), inserted); + ASSERT_TRUE(pending.finish(1, true)); + EXPECT_FALSE(pending.contains(1)); + EXPECT_EQ(pending.contains(2), inserted); + } + EXPECT_GE(failures, 2U); +} + +TEST(TreeStateMetrics, DeletionRemainsPendingWithoutALivePage) +{ + CheckpointPendingMetrics pending; + pending.change_page(7); + ASSERT_TRUE(pending.capture(1)); + pending.remove_page(7); + ASSERT_TRUE(pending.finish(1, true)); + TreeLogicalStats stats; + pending.copy_to(stats); + EXPECT_EQ(stats.checkpoint_pending_pages, 0U); + EXPECT_TRUE(stats.checkpoint_mapping_pending); + ASSERT_TRUE(pending.capture(2)); + ASSERT_TRUE(pending.finish(2, true)); + pending.copy_to(stats); + EXPECT_FALSE(stats.checkpoint_mapping_pending); +} + +TEST(TreeStateMetrics, ReloadableWritebackCanClearDirtyWithoutClearingCheckpointPending) +{ + TreeLogicalStats stats; + CheckpointPendingMetrics pending; + auto page = known_leaf(4000, 8, 3000); + page.dirty = true; + page.writeback_candidate = true; + pending.change_page(1); + ASSERT_TRUE(apply_page_metrics(stats, nullptr, &page)); + auto clean = page; + clean.dirty = false; + clean.writeback_candidate = false; + ASSERT_TRUE(apply_page_metrics(stats, &page, &clean)); + pending.copy_to(stats); + EXPECT_EQ(stats.dirty_pages, 0U); + EXPECT_EQ(stats.writeback_candidate_pages, 0U); + EXPECT_EQ(stats.checkpoint_pending_pages, 1U); +} + +TEST(TreeResourceMetrics, PhysicalOccupancyRejectsUnderflowAndOverflow) +{ + TreeResourceMetrics metrics; + ASSERT_TRUE(metrics.add(TreeResourceMetric::AllocatedBytes, 8192)); + auto snapshot = metrics.snapshot(); + EXPECT_EQ(snapshot.allocated_bytes, 8192U); + EXPECT_FALSE(metrics.remove(TreeResourceMetric::AllocatedBytes, 8193)); + EXPECT_FALSE(metrics.add(TreeResourceMetric::AllocatedBytes, std::numeric_limits<uint64_t>::max())); + EXPECT_EQ(metrics.snapshot().allocated_bytes, 8192U); +} + +TEST(TreeResourceMetrics, ConcurrentOwnerAcquireAndReleaseReconcileAtQuiescence) +{ + TreeResourceMetrics metrics; + std::array<std::thread, 4> workers; + for (auto &worker : workers) { + worker = std::thread([&] { + for (int i = 0; i < 1000; ++i) { + EXPECT_TRUE(metrics.add(TreeResourceMetric::AllocatedBytes, 128)); + EXPECT_TRUE(metrics.remove(TreeResourceMetric::AllocatedBytes, 128)); + } + }); + } + for (auto &worker : workers) { + worker.join(); + } + EXPECT_EQ(metrics.snapshot().allocated_bytes, 0U); +} + +TEST(TreeWorkMetrics, OutcomeAndLatencyRecordOnlyFirstTerminal) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + TreeMetricsOperation operation(metrics, TreeStage::FlushBatch, 100); + constexpr auto stage = static_cast<size_t>(TreeStage::FlushBatch); + EXPECT_EQ(metrics->totals()[stage].inflight, 1U); + std::thread success([&] { (void)operation.finish(TreeOutcome::Success, 200); }); + std::thread error([&] { (void)operation.finish(TreeOutcome::Error, 200); }); + success.join(); + error.join(); + EXPECT_FALSE(operation.finish(TreeOutcome::Cancelled, 300)); + const auto totals = metrics->totals()[stage]; + EXPECT_EQ(totals.attempts, 1U); + EXPECT_EQ(totals.inflight, 0U); + EXPECT_EQ(totals.count, 1U); + EXPECT_EQ(totals.sum_ns, 100U); + EXPECT_EQ(totals.max_ns, 100U); + EXPECT_EQ(totals.outcomes[static_cast<size_t>(TreeOutcome::Success)] + + totals.outcomes[static_cast<size_t>(TreeOutcome::Error)], + 1U); +} + +TEST(TreeWorkMetrics, OneCollectorDrainsAndReadersKeepCumulativeTotals) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto collector = metrics->claim_collector(); + ASSERT_NE(collector, nullptr); + EXPECT_EQ(metrics->claim_collector(), nullptr); + TreeMetricsOperation first(metrics, TreeStage::PageSplit, 10); + ASSERT_TRUE(first.finish(TreeOutcome::Success, 30)); + TreeMetricsOperation second(metrics, TreeStage::PageSplit, 40); + ASSERT_TRUE(second.finish(TreeOutcome::Retry, 70)); + constexpr auto stage = static_cast<size_t>(TreeStage::PageSplit); + const auto totals = metrics->totals()[stage]; + EXPECT_EQ(totals.count, 2U); + const auto window = collector->capture_window()[stage]; + EXPECT_EQ(window.count, 2U); + EXPECT_EQ(window.sum_ns, 50U); + EXPECT_EQ(window.max_ns, 30U); + EXPECT_EQ(window.total_count, 2U); + const auto empty = collector->capture_window()[stage]; + EXPECT_EQ(empty.count, 0U); + EXPECT_EQ(empty.sum_ns, 0U); + EXPECT_EQ(empty.total_count, 2U); + EXPECT_EQ(metrics->totals()[stage].sum_ns, 50U); + collector.reset(); + EXPECT_NE(metrics->claim_collector(), nullptr); +} + +TEST(TreeWorkMetrics, AbandonedOperationCancelsAndKeepsOwnerAlive) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + const std::weak_ptr<TreeWorkMetrics> weak = metrics; + auto operation = + std::make_unique<TreeMetricsOperation>(metrics, TreeStage::PageLoadPipeline, tree_metrics_now_ns()); + metrics.reset(); + ASSERT_FALSE(weak.expired()); + const auto observer = weak.lock(); + operation.reset(); + constexpr auto stage = static_cast<size_t>(TreeStage::PageLoadPipeline); + const auto totals = observer->totals()[stage]; + EXPECT_EQ(totals.count, 1U); + EXPECT_EQ(totals.inflight, 0U); + EXPECT_EQ(totals.outcomes[static_cast<size_t>(TreeOutcome::Cancelled)], 1U); +} + +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_stats_c_api_test.cpp b/lib/crowdb-tree/tests/unit/tree_stats_c_api_test.cpp new file mode 100644 index 000000000..0621f0037 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_stats_c_api_test.cpp @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "crowdb-tree/c_api.h" +#include "crowdb-tree/status.h" +#include "crowdb-tree/tree_stats_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(TreeStatsCAbi, VersionedSnapshotPreservesUnknownFieldsAndDoesNotMutateOnError) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + ct_tree_stats_snapshot_v1 result{}; + result.struct_size = sizeof(result); + result.version = 1; + ASSERT_EQ(ct_get_tree_state_stats(tree, &result), 0); + EXPECT_NE(result.instance_generation, 0U); + EXPECT_EQ(result.logical.leaf_pages, 1U); + EXPECT_EQ(result.logical.kv_unknown_pages, 0U); + EXPECT_GT(result.resources.allocated_bytes, 0U); + EXPECT_GT(result.resources.reserved_bytes, 0U); + const auto generation = result.instance_generation; + const auto sequence = result.commit_seq; + ASSERT_EQ(ct_get_tree_state_stats(tree, &result), 0); + EXPECT_EQ(result.instance_generation, generation); + EXPECT_EQ(result.commit_seq, sequence); + result.version = 2; + EXPECT_EQ(ct_get_tree_state_stats(tree, &result), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(result.instance_generation, generation); + EXPECT_EQ(result.version, 2U); + result.version = 1; + --result.struct_size; + EXPECT_EQ(ct_get_tree_state_stats(tree, &result), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(result.commit_seq, sequence); + ct_close(tree); + ct_runtime_free(runtime); +} + +TEST(TreeStatsCAbi, StandaloneTreeReportsItsRuntimePublicationStats) +{ + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + ct_tree_stats_snapshot_v1 result{}; + result.struct_size = sizeof(result); + result.version = 1; + result.commit_seq = 123; + EXPECT_EQ(ct_get_tree_state_stats(tree, &result), 0); + EXPECT_NE(result.instance_generation, 0U); + EXPECT_EQ(result.logical.leaf_pages, 1U); + EXPECT_GT(result.resources.allocated_bytes, 0U); + EXPECT_NE(result.commit_seq, 123U); + ct_close(tree); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_tuning_c_api_test.cpp b/lib/crowdb-tree/tests/unit/tree_tuning_c_api_test.cpp new file mode 100644 index 000000000..f328f8db5 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_tuning_c_api_test.cpp @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. + +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/c_api.h" +#include "runtime/tree_runtime_c_api.h" +#include "runtime/tree_tuning_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(TreeTuningCAbi, ExplicitValuesReachTheEngineAndKeyAdmission) +{ + ct_tree_tuning tuning{}; + ct_tree_tuning_default(&tuning); + tuning.max_key_size = 4; + tuning.source_max_entries = 128; + tuning.source_max_key_bytes = 64 * 1024; + tuning.source_max_gap_entries = 32; + tuning.source_max_gap_key_bytes = 16 * 1024; + tuning.flush_parallelism = 2; + tuning.run_max_entries = 64; + tuning.run_max_key_bytes = 64 * 1024; + tuning.run_key_block_bytes = 512; + tuning.run_max_bytes = 512 * 1024; + tuning.install_run_window = 16; + tuning.maintenance_visits_per_step = 8; + tuning.maintenance_max_visits = 128; + tuning.maintenance_max_candidates = 16; + tuning.maintenance_max_steps = 128; + tuning.maintenance_metadata_bytes = 512 * 1024; + tuning.cache_max_pages = 512; + ct_options options{}; + options.tuning = &tuning; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + const auto &e = core->config().execution; + EXPECT_EQ(e.source_max_entries, 128U); + EXPECT_EQ(e.source_max_key_bytes, 64U * 1024); + EXPECT_EQ(e.source_max_gap_entries, 32U); + EXPECT_EQ(e.source_max_gap_key_bytes, 16U * 1024); + EXPECT_EQ(e.flush_parallelism, 2U); + EXPECT_EQ(e.run_max_entries, 64U); + EXPECT_EQ(e.run_max_key_bytes, 64U * 1024); + EXPECT_EQ(e.run_key_block_bytes, 512U); + EXPECT_EQ(e.run_max_bytes, 512U * 1024); + EXPECT_EQ(e.install_run_window, 16U); + EXPECT_EQ(e.maintenance_visits_per_step, 8U); + EXPECT_EQ(e.maintenance_max_visits, 128U); + EXPECT_EQ(e.maintenance_max_candidates, 16U); + EXPECT_EQ(e.maintenance_max_steps, 128U); + EXPECT_EQ(e.maintenance_metadata_bytes, 512U * 1024); + EXPECT_EQ(e.cache_max_pages, 512U); + const uint8_t key[] = {'a', 'b', 'c', 'd', 'e'}; + const uint8_t value = 'v'; + EXPECT_EQ(ct_apply_put(tree, 1, key, sizeof(key), &value, 1), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_apply_put(tree, 1, key, 4, &value, 1), 0); + ct_close(tree); +} + +TEST(TreeTuningCAbi, InvalidWindowsAndStructSizeFailBeforeOpening) +{ + ct_tree_tuning tuning{}; + ct_tree_tuning_default(&tuning); + ct_options options{}; + options.tuning = &tuning; + ct_tree *tree = nullptr; + --tuning.struct_size; + EXPECT_EQ(ct_open(&options, &tree), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(tree, nullptr); + ++tuning.struct_size; + tuning.run_key_block_bytes = tuning.run_max_key_bytes + 1; + EXPECT_EQ(ct_open(&options, &tree), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(tree, nullptr); + ct_tree_tuning_default(&tuning); + tuning.maintenance_max_candidates = 0; + EXPECT_EQ(ct_open(&options, &tree), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(tree, nullptr); +} + +TEST(TreeRuntimeCAbi, BackendPoolSettingsCrossTheAbi) +{ + ct_runtime_options options{}; + ct_runtime_options_default(&options); + options.backend_workers = 3; + options.backend_queue_limit = 80; + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(&options, &runtime), 0); + ct_tree_runtime_options binding{}; + ct_tree_runtime_options_default(&binding); + binding.backend_queue_limit = 40; + Config config; + ASSERT_TRUE(configure_tree_runtime(runtime, &binding, &config).ok()); + EXPECT_EQ(config.runtime->options().backend_workers, 3U); + EXPECT_EQ(config.runtime->options().backend_queue_limit, 80U); + EXPECT_EQ(config.runtime_binding.backend_queue_limit, 40U); + ct_runtime_free(runtime); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_wait_metrics_c_api_test.cpp b/lib/crowdb-tree/tests/unit/tree_wait_metrics_c_api_test.cpp new file mode 100644 index 000000000..5188bb25b --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_wait_metrics_c_api_test.cpp @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/tree_wait_metrics_c_api.h" +#include "crowdb-tree/tree_work_metrics_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(TreeWaitMetricsCAbi, InvalidSchemaNeverMutatesOutputAndCollectorRetainsLateWait) +{ + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open(&options, &tree), 0); + ct_work_metrics_collector *collector = nullptr; + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + auto metrics = TreeCoreAccess::get(*c_api_tree(tree))->publisher().work_metrics(); + TreeWaitSpan wait; + wait.begin(metrics, TreeWaitClass::Routing, 100); + ct_tree_wait_snapshot_v1 totals{}; + totals.struct_size = sizeof(totals); + totals.version = 2; + totals.classes[1].attempts = 99; + EXPECT_EQ(ct_tree_wait_totals(tree, &totals), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(totals.classes[1].attempts, 99U); + totals.version = 1; + --totals.struct_size; + EXPECT_EQ(ct_work_metrics_collector_wait_totals(collector, &totals), static_cast<int32_t>(Code::kInvalidArgument)); + ++totals.struct_size; + ASSERT_EQ(ct_tree_wait_totals(tree, &totals), 0); + EXPECT_EQ(totals.classes[1].inflight, 1U); + ct_close(tree); + metrics.reset(); + ASSERT_EQ(ct_work_metrics_collector_wait_totals(collector, &totals), 0); + EXPECT_EQ(totals.classes[1].inflight, 1U); + wait.finish(TreeWaitOutcome::Granted, 175); + ASSERT_EQ(ct_work_metrics_collector_wait_totals(collector, &totals), 0); + EXPECT_EQ(totals.classes[1].inflight, 0U); + EXPECT_EQ(totals.classes[1].total_count, 1U); + EXPECT_EQ(totals.classes[1].total_sum_ns, 75U); + EXPECT_EQ(totals.classes[1].outcomes[0], 1U); + ct_work_metrics_collector_free(collector); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_wait_metrics_test.cpp b/lib/crowdb-tree/tests/unit/tree_wait_metrics_test.cpp new file mode 100644 index 000000000..1d54bec8d --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_wait_metrics_test.cpp @@ -0,0 +1,261 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/page_cache.h" +#include "btree/resource_budget.h" +#include "btree/tree_metrics.h" +#include "btree/tree_work_registry.h" +#include "load_budget_test_support.h" +#include "published_read_test_support.h" +#include "runtime/allocation_memory_resource.h" + +#include <cstdio> +#include <sstream> + +namespace crowdb::tree +{ +namespace +{ +using detail::BudgetClass; +using detail::ResourceBudget; + +ResourceBudget::Limits limits(size_t amount) +{ + ResourceBudget::Limits result; + result.fill(amount); + return result; +} + +struct TestResult +{ + size_t values = 0, errors = 0, stopped = 0; +}; + +struct TestReceiver +{ + using receiver_concept = stdexec::receiver_tag; + TestResult *result; + + void set_value() const noexcept + { + ++result->values; + } + + void set_error(Status /*status*/) const noexcept + { + ++result->errors; + } + + void set_stopped() const noexcept + { + ++result->stopped; + } +}; + +TEST(TreeWaitMetrics, CumulativeClassesAndExactDurationsDoNotDrainStages) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto collector = metrics->claim_collector(); + TreeMetricsOperation work(metrics, TreeStage::BudgetWait, 100); + TreeWaitSpan wait; + wait.begin(metrics, TreeWaitClass::Routing, 100); + auto totals = metrics->waits.totals(130); + const auto index = static_cast<size_t>(TreeWaitClass::Routing); + EXPECT_EQ(totals[index].inflight, 1U); + EXPECT_EQ(totals[index].oldest_age_ns, 30U); + EXPECT_TRUE(totals[index].complete); + wait.finish(TreeWaitOutcome::Granted, 150); + wait.finish(TreeWaitOutcome::Error, 200); + ASSERT_TRUE(work.finish(TreeOutcome::Success, 150)); + totals = metrics->waits.totals(200); + EXPECT_EQ(totals[index].attempts, 1U); + EXPECT_EQ(totals[index].count, 1U); + EXPECT_EQ(totals[index].sum_ns, 50U); + EXPECT_EQ(totals[index].max_ns, 50U); + EXPECT_EQ(totals[index].outcomes[0], 1U); + EXPECT_EQ(totals[index].inflight, 0U); + EXPECT_EQ(collector->capture_window()[static_cast<size_t>(TreeStage::BudgetWait)].count, 1U); + EXPECT_EQ(metrics->waits.totals(300)[index].count, 1U); +} + +TEST(TreeWaitMetrics, AgeCapacityNeverRejectsWorkAndLateOwnerFinishesAfterMetricsCallerDrops) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + std::weak_ptr<TreeWorkMetrics> weak = metrics; + std::array<TreeWaitSpan, kTreeWaitAgeCapacity + 1> waits; + for (auto &wait : waits) { + wait.begin(metrics, TreeWaitClass::Io, 100); + } + const auto index = static_cast<size_t>(TreeWaitClass::Io); + const auto totals = metrics->waits.totals(150)[index]; + EXPECT_EQ(totals.inflight, waits.size()); + EXPECT_EQ(totals.tracked, kTreeWaitAgeCapacity); + EXPECT_EQ(totals.untracked, 1U); + EXPECT_FALSE(totals.complete); + metrics.reset(); + for (auto &wait : waits) { + wait.finish(TreeWaitOutcome::Cancelled, 200); + } + EXPECT_TRUE(weak.expired()); +} + +TEST(TreeWaitMetrics, BudgetGrantAndCancellationAreSeparatedForEveryPartition) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + ResourceBudget budget(limits(1), 1); + auto account = budget.make_account(limits(1)); + account->set_work_metrics(metrics); + for (size_t i = 0; i < ResourceBudget::kClasses; ++i) { + const auto kind = static_cast<BudgetClass>(i); + auto held = ResourceBudget::try_reserve(account, kind, 1); + ASSERT_TRUE(held); + ResourceBudget::Request request(account, kind, 1); + TestResult result; + auto operation = stdexec::connect(request.sender(), TestReceiver{&result}); + stdexec::start(operation); + EXPECT_EQ(metrics->waits.totals(tree_metrics_now_ns())[i].inflight, 1U); + if (i % 2 == 0) { + held.reset(); + } + else { + request.request_stop(); + request.request_stop(); + } + EXPECT_EQ(result.values, i % 2 == 0 ? 1U : 0U); + EXPECT_EQ(result.stopped, i % 2 == 0 ? 0U : 1U); + const auto total = metrics->waits.totals(tree_metrics_now_ns())[i]; + EXPECT_EQ(total.count, 1U); + EXPECT_EQ(total.inflight, 0U); + EXPECT_EQ(total.outcomes[i % 2 == 0 ? 0 : 2], 1U); + } +} + +TEST(TreeWaitMetrics, ImpossibleAndFullWaitersRecordErrorWhileCloseCancelsOnce) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + ResourceBudget budget(limits(1), 1); + auto account = budget.make_account(limits(1)); + account->set_work_metrics(metrics); + auto held = ResourceBudget::try_reserve(account, BudgetClass::kHandoff, 1); + ASSERT_TRUE(held); + ResourceBudget::Request first(account, BudgetClass::kHandoff, 1); + ResourceBudget::Request full(account, BudgetClass::kHandoff, 1); + ResourceBudget::Request impossible(account, BudgetClass::kHandoff, 2); + TestResult result; + auto a = stdexec::connect(first.sender(), TestReceiver{&result}); + auto b = stdexec::connect(full.sender(), TestReceiver{&result}); + auto c = stdexec::connect(impossible.sender(), TestReceiver{&result}); + stdexec::start(a); + stdexec::start(b); + stdexec::start(c); + EXPECT_EQ(result.errors, 2U); + budget.close(); + first.request_stop(); + EXPECT_EQ(result.stopped, 1U); + auto duplicate = stdexec::connect(first.sender(), TestReceiver{&result}); + stdexec::start(duplicate); + EXPECT_EQ(result.errors, 3U); + const auto total = metrics->waits.totals(tree_metrics_now_ns())[0]; + EXPECT_EQ(total.attempts, 3U); + EXPECT_EQ(total.count, 3U); + EXPECT_EQ(total.outcomes[1], 2U); + EXPECT_EQ(total.outcomes[2], 1U); + EXPECT_EQ(total.inflight, 0U); + EXPECT_TRUE(total.complete); +} + +TEST(TreeWaitMetrics, SharedCacheWaitersHaveIndependentProofOutcomesAndNoHitOrLoaderDoubleCount) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + test_read::LoadCredits credits; + credits.account->set_work_metrics(metrics); + SharedPageCache cache({.max_pages = 1, .max_waiters = 8}); + auto upstream = std::make_shared<test_read::TestLoader>(); + upstream->deferred = true; + upstream->frames[1] = test_read::base({ + {.key = "key", .slot = 7, .value = "value"} + }); + auto loader = cache.bind(upstream, credits.credits.bytes); + auto input = test_read::cold(1, 10); + size_t success = 0; + size_t failure = 0; + auto ready = [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++success; + }; + loader->load_leaf(input, ready); + auto invalid = input; + invalid.max_slot = 2; + loader->load_leaf(invalid, [&](Status status, auto leaf) { + EXPECT_EQ(status.code(), Code::kCorruption); + EXPECT_FALSE(leaf); + ++failure; + }); + loader->load_leaf(input, ready); + constexpr auto index = static_cast<size_t>(TreeWaitClass::SharedCache); + EXPECT_EQ(metrics->waits.totals(tree_metrics_now_ns())[index].inflight, 2U); + ASSERT_EQ(upstream->pending.size(), 1U); + auto duplicate = upstream->pending.front(); + upstream->finish(); + duplicate(Status::io_error(), {}); + loader->load_leaf(input, ready); + EXPECT_EQ(success, 3U); + EXPECT_EQ(failure, 1U); + EXPECT_EQ(upstream->calls, 1U); + const auto total = metrics->waits.totals(tree_metrics_now_ns())[index]; + EXPECT_EQ(total.attempts, 2U); + EXPECT_EQ(total.count, 2U); + EXPECT_EQ(total.outcomes[0], 1U); + EXPECT_EQ(total.outcomes[1], 1U); + EXPECT_EQ(total.inflight, 0U); +} + +TEST(TreeWaitMetrics, NativeRegistrySamplesOnceWithoutDrainingAndReleasesEveryPaidByte) +{ + test_read::LoadCredits credits; + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto collector = metrics->claim_collector(); + TreeWaitSpan wait; + wait.begin(metrics, TreeWaitClass::SharedCache, 100); + wait.finish(TreeWaitOutcome::Granted, 150); + TreeMetricsOperation stage(metrics, TreeStage::PageLoadPipeline, 100); + ASSERT_TRUE(stage.finish(TreeOutcome::Success, 150)); + const auto available = credits.credits.bytes->available(); + { + auto resource = std::make_shared<detail::AllocationMemoryResource>(credits.credits.bytes); + auto registry = create_tree_work_registry(resource, "tree", metrics, credits.credits.bytes); + EXPECT_LT(credits.credits.bytes->available(), available); + std::unique_ptr<FILE, decltype(&std::fclose)> file(std::tmpfile(), &std::fclose); + ASSERT_NE(file, nullptr); + registry->flush_to(file.get(), 1, "test"); + EXPECT_EQ(std::fseek(file.get(), 0, SEEK_SET), 0); + std::array<char, 2048> bytes{}; + std::string rendered; + while (const auto count = std::fread(bytes.data(), 1, bytes.size(), file.get())) { + rendered.append(bytes.data(), count); + } + std::istringstream lines(rendered); + std::string line; + size_t fields_seen = 0; + bool found_sum = false; + while (std::getline(lines, line)) { + std::istringstream fields(line); + std::string name; + uint64_t value = 0; + if (fields >> name >> value; name.starts_with("tree.work.wait.")) { + ++fields_seen; + if (name == "tree.work.wait.shared_cache.latency.sum.ns.c") { + found_sum = true; + EXPECT_EQ(value, 50U); + } + } + } + EXPECT_TRUE(found_sum); + EXPECT_EQ(fields_seen, kTreeWaitClasses * 12); + EXPECT_EQ(collector->capture_window()[static_cast<size_t>(TreeStage::PageLoadPipeline)].count, 1U); + EXPECT_EQ(metrics->waits.totals(300)[static_cast<size_t>(TreeWaitClass::SharedCache)].count, 1U); + } + EXPECT_EQ(credits.credits.bytes->available(), available); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_work_age_c_api_test.cpp b/lib/crowdb-tree/tests/unit/tree_work_age_c_api_test.cpp new file mode 100644 index 000000000..4d2c429ab --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_work_age_c_api_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/tree_work_metrics_c_api.h" + +#include <gtest/gtest.h> + +using namespace crowdb::tree; + +TEST(TreeWorkMetricsCAbi, AgeSchemaValidatesWithoutMutationAndRetainsLedgerAfterClose) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + auto metrics = TreeCoreAccess::get(*c_api_tree(tree))->publisher().work_metrics(); + ct_work_metrics_collector *collector = nullptr; + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + constexpr auto index = static_cast<size_t>(TreeStage::PageEncode); + const auto started = tree_metrics_now_ns(); + TreeMetricsOperation operation(metrics, TreeStage::PageEncode, started); + ct_tree_work_ages_v1 ages{}; + ages.struct_size = sizeof(ages); + ages.version = 2; + ages.sampled_at_ns = 99; + EXPECT_EQ(ct_work_metrics_ages(tree, &ages), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_work_metrics_collector_ages(collector, &ages), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ages.sampled_at_ns, 99U); + ages.version = 1; + --ages.struct_size; + EXPECT_EQ(ct_work_metrics_ages(tree, &ages), static_cast<int32_t>(Code::kInvalidArgument)); + ++ages.struct_size; + EXPECT_EQ(ct_work_metrics_ages(nullptr, &ages), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_work_metrics_ages(tree, nullptr), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_work_metrics_collector_ages(nullptr, &ages), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_work_metrics_collector_ages(collector, nullptr), static_cast<int32_t>(Code::kInvalidArgument)); + ASSERT_EQ(ct_work_metrics_ages(tree, &ages), 0); + EXPECT_EQ(ages.stages[index].inflight, 1U); + EXPECT_EQ(ages.stages[index].tracked, 1U); + EXPECT_EQ(ages.stages[index].untracked, 0U); + EXPECT_EQ(ages.stages[index].complete, 1U); + EXPECT_EQ(ages.stages[index].observed_oldest_age_ns, ages.sampled_at_ns - started); + ct_close(tree); + ASSERT_EQ(ct_work_metrics_collector_ages(collector, &ages), 0); + EXPECT_EQ(ages.stages[index].inflight, 1U); + EXPECT_EQ(ages.stages[index].observed_oldest_age_ns, ages.sampled_at_ns - started); + ASSERT_TRUE(operation.finish(TreeOutcome::Success, started + 30)); + ASSERT_EQ(ct_work_metrics_collector_ages(collector, &ages), 0); + EXPECT_EQ(ages.stages[index].inflight, 0U); + EXPECT_EQ(ages.stages[index].tracked, 0U); + EXPECT_EQ(ages.stages[index].complete, 1U); + ct_tree_work_snapshot_v1 window{}; + window.struct_size = sizeof(window); + window.version = 1; + ASSERT_EQ(ct_work_metrics_capture(collector, &window), 0); + EXPECT_EQ(window.stages[index].window_count, 1U); + EXPECT_EQ(window.stages[index].window_sum_ns, 30U); + ct_work_metrics_collector_free(collector); + ct_runtime_free(runtime); +} diff --git a/lib/crowdb-tree/tests/unit/tree_work_age_test.cpp b/lib/crowdb-tree/tests/unit/tree_work_age_test.cpp new file mode 100644 index 000000000..fe8b97e79 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_work_age_test.cpp @@ -0,0 +1,141 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_metrics.h" + +#include <gtest/gtest.h> + +#include <limits> +#include <thread> +#include <vector> + +using namespace crowdb::tree; + +namespace +{ +constexpr size_t kFlush = static_cast<size_t>(TreeStage::FlushBatch); +constexpr size_t kLoad = static_cast<size_t>(TreeStage::PageLoadPipeline); +} // namespace + +TEST(TreeWorkAge, TracksOldestAcrossOutOfOrderStartsAndSingleTerminalNotifications) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + TreeMetricsOperation newer(metrics, TreeStage::FlushBatch, 40); + TreeMetricsOperation oldest(metrics, TreeStage::FlushBatch, 10); + TreeMetricsOperation load(metrics, TreeStage::PageLoadPipeline, 30); + auto ages = metrics->ages(100); + EXPECT_EQ(ages[kFlush].inflight, 2U); + EXPECT_EQ(ages[kFlush].tracked, 2U); + EXPECT_EQ(ages[kFlush].oldest_age_ns, 90U); + EXPECT_TRUE(ages[kFlush].complete); + EXPECT_EQ(ages[kLoad].oldest_age_ns, 70U); + EXPECT_TRUE(oldest.finish(TreeOutcome::Success, 110)); + EXPECT_FALSE(oldest.finish(TreeOutcome::Error, 120)); + ages = metrics->ages(130); + EXPECT_EQ(ages[kFlush].inflight, 1U); + EXPECT_EQ(ages[kFlush].oldest_age_ns, 90U); + EXPECT_TRUE(newer.finish(TreeOutcome::Cancelled, 140)); + EXPECT_TRUE(load.finish(TreeOutcome::Error, 150)); + for (const auto &age : metrics->ages(160)) { + EXPECT_EQ(age.inflight, 0U); + EXPECT_EQ(age.tracked, 0U); + EXPECT_EQ(age.untracked, 0U); + EXPECT_EQ(age.oldest_age_ns, 0U); + EXPECT_TRUE(age.complete); + } +} + +TEST(TreeWorkAge, SaturationReportsIncompleteCoverageWithoutRejectingWork) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + std::vector<std::unique_ptr<TreeMetricsOperation>> operations; + operations.reserve(kTreeWorkAgeCapacity + 2); + for (size_t i = 0; i < kTreeWorkAgeCapacity + 2; ++i) { + operations.push_back(std::make_unique<TreeMetricsOperation>(metrics, TreeStage::FlushBatch, i + 1)); + } + auto age = metrics->ages(1000)[kFlush]; + EXPECT_EQ(age.inflight, kTreeWorkAgeCapacity + 2); + EXPECT_EQ(age.tracked, kTreeWorkAgeCapacity); + EXPECT_EQ(age.untracked, 2U); + EXPECT_EQ(age.oldest_age_ns, 999U); + EXPECT_FALSE(age.complete); + for (size_t i = 0; i < kTreeWorkAgeCapacity; ++i) { + ASSERT_TRUE(operations[i]->finish(TreeOutcome::Success, 1000)); + } + age = metrics->ages(2000)[kFlush]; + EXPECT_EQ(age.inflight, 2U); + EXPECT_EQ(age.tracked, 0U); + EXPECT_EQ(age.untracked, 2U); + EXPECT_EQ(age.oldest_age_ns, 0U); + EXPECT_FALSE(age.complete); + // Reused diagnostic slots cannot be released by an older terminal callback. + TreeMetricsOperation reused(metrics, TreeStage::FlushBatch, 1500); + for (size_t i = 0; i < kTreeWorkAgeCapacity; ++i) { + ASSERT_FALSE(operations[i]->finish(TreeOutcome::Success, 2000)); + } + EXPECT_EQ(metrics->ages(2000)[kFlush].oldest_age_ns, 500U); + operations.clear(); + age = metrics->ages(2000)[kFlush]; + EXPECT_EQ(age.inflight, 1U); + EXPECT_EQ(age.untracked, 0U); + EXPECT_TRUE(age.complete); + ASSERT_TRUE(reused.finish(TreeOutcome::Success, 2100)); + EXPECT_EQ(metrics->totals()[kFlush].count, kTreeWorkAgeCapacity + 3); +} + +TEST(TreeWorkAge, HandlesUnencodableTimestampsAndClockRegression) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + TreeMetricsOperation invalid(metrics, TreeStage::FlushBatch, std::numeric_limits<uint64_t>::max()); + TreeMetricsOperation future(metrics, TreeStage::PageLoadPipeline, 200); + auto ages = metrics->ages(100); + EXPECT_EQ(ages[kFlush].untracked, 1U); + EXPECT_FALSE(ages[kFlush].complete); + EXPECT_EQ(ages[kLoad].oldest_age_ns, 0U); + EXPECT_TRUE(ages[kLoad].complete); + EXPECT_TRUE(invalid.finish(TreeOutcome::Cancelled, 0)); + EXPECT_TRUE(future.finish(TreeOutcome::Cancelled, 100)); + EXPECT_TRUE(metrics->ages(100)[kFlush].complete); + EXPECT_EQ(metrics->totals()[kFlush].sum_ns, 0U); +} + +TEST(TreeWorkAge, ConcurrentSamplingDoesNotRetainOperationsAndQuiescentTotalsMatch) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + constexpr size_t kThreads = 4; + constexpr size_t kOperations = 2000; + std::atomic<size_t> running{kThreads}; + std::vector<std::thread> workers; + workers.reserve(kThreads); + for (size_t i = 0; i < kThreads; ++i) { + workers.emplace_back([&, i] { + for (size_t j = 0; j < kOperations; ++j) { + const auto stage = static_cast<TreeStage>((i + j) % kTreeStageCount); + TreeMetricsOperation operation(metrics, stage, j); + EXPECT_TRUE(operation.finish(TreeOutcome::Success, j + 1)); + } + running.fetch_sub(1); + }); + } + while (running.load() != 0) { + const auto ages = metrics->ages(kOperations); + uint64_t tracked = 0; + for (const auto &age : ages) { + tracked += age.tracked; + } + EXPECT_LE(tracked, kTreeWorkAgeCapacity); + } + for (auto &worker : workers) { + worker.join(); + } + uint64_t count = 0; + for (const auto &stage : metrics->totals()) { + count += stage.count; + } + EXPECT_EQ(count, kThreads * kOperations); + for (const auto &age : metrics->ages(kOperations)) { + EXPECT_EQ(age.inflight, 0U); + EXPECT_EQ(age.tracked, 0U); + EXPECT_EQ(age.untracked, 0U); + EXPECT_TRUE(age.complete); + } +} diff --git a/lib/crowdb-tree/tests/unit/tree_work_counters_test.cpp b/lib/crowdb-tree/tests/unit/tree_work_counters_test.cpp new file mode 100644 index 000000000..9413767ed --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_work_counters_test.cpp @@ -0,0 +1,259 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_account.h" +#include "btree/page_cache.h" +#include "btree/tree_work_registry.h" +#include "captured_write_test_support.h" +#include "common/test_chunk_physical.h" +#include "runtime/allocation_memory_resource.h" +#include "runtime/tree_runtime_binding.h" + +#include <gtest/gtest.h> + +#include <cstdio> +#include <sstream> + +namespace crowdb::tree +{ +namespace +{ +uint64_t quantity(const std::shared_ptr<TreeWorkMetrics> &work, TreeWorkCounter counter, + TreeWorkCause cause = TreeWorkCause::Other) +{ + return work->counters.snapshot()[static_cast<size_t>(cause)][static_cast<size_t>(counter)]; +} + +struct TestBinding +{ + std::shared_ptr<TreeRuntime> runtime; + std::shared_ptr<TreeRuntimeBinding> binding; + + TestBinding() + { + EXPECT_TRUE(TreeRuntime::create({}, &runtime).ok()); + EXPECT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + } + + ~TestBinding() + { + binding->drain(); + } +}; + +std::string rendered(common::metrics::MetricsRegistry ®istry) +{ + std::unique_ptr<FILE, decltype(&std::fclose)> file(std::tmpfile(), &std::fclose); + EXPECT_NE(file, nullptr); + if (!file) { + return {}; + } + registry.flush_to(file.get(), 1, "work"); + EXPECT_EQ(std::fseek(file.get(), 0, SEEK_SET), 0); + std::array<char, 2048> buffer{}; + std::string result; + while (const auto bytes = std::fread(buffer.data(), 1, buffer.size(), file.get())) { + result.append(buffer.data(), bytes); + } + return result; +} + +uint64_t metric_value(const std::string &text, const std::string &name) +{ + std::istringstream lines(text); + std::string line; + while (std::getline(lines, line)) { + std::istringstream fields(line); + std::string key; + uint64_t value = 0; + if (fields >> key >> value; key == name) { + return value; + } + } + ADD_FAILURE() << "missing metric " << name; + return UINT64_MAX; +} + +TEST(TreeWorkCounters, ActualFreeIsCountedOnceAndSameHostTransferDoesNotCount) +{ + TestBinding test; + std::shared_ptr<TreeRuntimeBinding> next; + ASSERT_TRUE(TreeRuntimeBinding::create(test.runtime, {}, &next).ok()); + std::shared_ptr<detail::AllocationAccount> owner; + ASSERT_TRUE(detail::AllocationAccount::create(test.binding->account(), detail::BudgetClass::kRouting, &owner).ok()); + BudgetAllocator<uint8_t, AllocationBudget, AllocationKind::kPayload> allocator(owner); + auto *bytes = allocator.allocate(137); + auto old_work = test.binding->work_metrics(); + auto new_work = next->work_metrics(); + const auto old_before = quantity(old_work, TreeWorkCounter::MemoryReclaimedBytes, TreeWorkCause::Allocation); + const auto new_before = quantity(new_work, TreeWorkCounter::MemoryReclaimedBytes, TreeWorkCause::Allocation); + ASSERT_TRUE(owner->rebind(next->account()).ok()); + EXPECT_EQ(quantity(old_work, TreeWorkCounter::MemoryReclaimedBytes, TreeWorkCause::Allocation), old_before); + EXPECT_EQ(quantity(new_work, TreeWorkCounter::MemoryReclaimedBytes, TreeWorkCause::Allocation), new_before); + allocator.deallocate(bytes, 137); + EXPECT_EQ(quantity(old_work, TreeWorkCounter::MemoryReclaimedBytes, TreeWorkCause::Allocation), old_before); + EXPECT_EQ(quantity(new_work, TreeWorkCounter::MemoryReclaimedBytes, TreeWorkCause::Allocation), new_before + 137); + next->drain(); +} + +TEST(TreeWorkCounters, EncoderCountsOnlyEmittedFramesAndActualCopiesAcrossFailuresAndCompletion) +{ + auto work = std::make_shared<TreeWorkMetrics>(); + auto capture = test_write::captured_leaf(test_read::page({ + {.key = "key", .slot = 3, .value = std::string(1000, 'v'), .tombstone = false} + })); + auto prepared = prepare_final_page( + capture, 0, {}, + {.frame_bytes = 256, .inline_value_bytes = 16, .work_metrics = work, .cause = TreeWorkCause::Checkpoint}); + ASSERT_TRUE(prepared.status.ok()) << prepared.status.to_string(); + ASSERT_TRUE(prepared.encoder); + const auto frames = prepared.encoder->summary().new_overflow_frames; + std::array<uint8_t, 256> frame{}; + EXPECT_EQ(prepared.encoder->next(frame).status.code(), Code::kUnavailable); + EXPECT_EQ(quantity(work, TreeWorkCounter::EncodedFrameBytes, TreeWorkCause::Checkpoint), 0U); + ASSERT_TRUE(prepared.encoder->bind_overflow({.first = 1, .count = frames, .owner = capture}).ok()); + EXPECT_EQ(prepared.encoder->next(std::span(frame).first(255)).status.code(), Code::kInvalidArgument); + for (uint64_t i = 0; i <= frames; ++i) { + ASSERT_TRUE(prepared.encoder->next(frame).status.ok()); + } + EXPECT_FALSE(prepared.encoder->next(frame).frame.has_value()); + EXPECT_EQ(quantity(work, TreeWorkCounter::EncodedFrameBytes, TreeWorkCause::Checkpoint), (frames + 1) * 256); + EXPECT_EQ(quantity(work, TreeWorkCounter::EncodedPrimaryFrames, TreeWorkCause::Checkpoint), 1U); + EXPECT_EQ(quantity(work, TreeWorkCounter::EncodedOverflowFrames, TreeWorkCause::Checkpoint), frames); + EXPECT_EQ(quantity(work, TreeWorkCounter::EncodingKeyCopyBytes, TreeWorkCause::Checkpoint), 3U); + EXPECT_EQ(quantity(work, TreeWorkCounter::EncodingValueCopyBytes, TreeWorkCause::Checkpoint), 1000U); + EXPECT_EQ(quantity(work, TreeWorkCounter::HandoffValueCopyBytes, TreeWorkCause::Flush), 0U); +} + +TEST(TreeWorkCounters, CacheDeduplicatesLoadsAndSeparatesHitMissJoinAndEviction) +{ + TestBinding test; + auto credit = test.binding->lifecycle_budget(); + SharedPageCache cache({.max_pages = 2, .max_waiters = 8}, credit); + auto upstream = std::make_shared<test_read::TestLoader>(); + upstream->deferred = true; + upstream->frames[1] = test_read::base({ + {.key = "a", .value = "v"} + }); + auto input = test_read::cold(1); + auto loader = cache.bind(upstream, credit); + size_t completions = 0; + auto complete = [&](Status status, auto leaf) { + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(leaf); + ++completions; + }; + loader->load_leaf(input, complete); + loader->load_leaf(input, complete); + EXPECT_EQ(upstream->calls, 1U); + upstream->finish(); + loader->load_leaf(input, complete); + EXPECT_EQ(completions, 3U); + auto work = test.binding->work_metrics(); + EXPECT_EQ(quantity(work, TreeWorkCounter::CacheMisses, TreeWorkCause::Read), 1U); + EXPECT_EQ(quantity(work, TreeWorkCounter::CacheJoinedLoads, TreeWorkCause::Read), 1U); + EXPECT_EQ(quantity(work, TreeWorkCounter::CacheHits, TreeWorkCause::Read), 1U); + cache.evict(); + EXPECT_EQ(quantity(work, TreeWorkCounter::CacheEvictions, TreeWorkCause::CachePressure), 1U); +} + +TEST(TreeWorkCounters, PhysicalDeletionCountsTheObservedMirrorCapacityOnlyOnce) +{ + TestBinding test; + auto credit = test.binding->lifecycle_budget(); + std::shared_ptr<ChunkPhysicalRegistry> registry; + ASSERT_TRUE(ChunkPhysicalRegistry::create(credit, ®istry).ok()); + using Host = test_chunk::TestPhysicalHost; + std::shared_ptr<ChunkPhysicalPack> pack; + ASSERT_TRUE(registry->prepare(Host::stage, Host::chunk, credit, &pack).ok()); + ASSERT_TRUE(registry->observe(pack, Host::receipt, credit).ok()); + auto wrong = Host::receipt; + ++wrong.created_ms; + EXPECT_EQ(registry->deleted(Host::stage, Host::chunk, wrong, credit).code(), Code::kCorruption); + auto work = test.binding->work_metrics(); + EXPECT_EQ(quantity(work, TreeWorkCounter::BackingDeletedBytes, TreeWorkCause::GarbageCollection), 0U); + ASSERT_TRUE(registry->deleted(Host::stage, Host::chunk, Host::receipt, credit).ok()); + ASSERT_TRUE(registry->deleted(Host::stage, Host::chunk, Host::receipt, credit).ok()); + EXPECT_EQ(quantity(work, TreeWorkCounter::BackingDeletedBytes, TreeWorkCause::GarbageCollection), + Host::receipt.physical_bytes); +} + +TEST(TreeWorkCounters, ProxySlotProofCountsOnlyTheColdLoadItAvoided) +{ + TestBinding test; + auto logical = std::make_shared<PageState>(*test_read::page({ + {.key = "a", .slot = 7, .value = "new"} + })); + logical->bases.push_back(test_read::cold(1, 2)); + auto loader = std::make_shared<test_read::TestLoader>(); + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(test_read::view(logical), loader, "a", + {.metadata = test.binding->lifecycle_budget()}, &get) + .ok()); + test_read::Completion done; + auto connected = stdexec::connect(get->sender(), test_read::Receiver{.done = &done}); + stdexec::start(connected); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(get->result().cell.value.to_string(), "new"); + EXPECT_EQ(loader->calls, 0U); + EXPECT_EQ(quantity(test.binding->work_metrics(), TreeWorkCounter::ProxyAvoidedLoads, TreeWorkCause::Read), 1U); +} + +TEST(TreeWorkCounters, AdmissionRejectAndOneQueuedWaitAreDifferentEvents) +{ + detail::ResourceBudget::Limits limits; + limits.fill(128); + detail::ResourceBudget budget(limits, 2); + auto account = budget.make_account(limits); + auto work = std::make_shared<TreeWorkMetrics>(); + account->set_work_metrics(work); + auto full = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, 128); + ASSERT_TRUE(full); + EXPECT_FALSE(detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, 1)); + EXPECT_EQ(quantity(work, TreeWorkCounter::AdmissionRejected, TreeWorkCause::Allocation), 1U); + detail::ResourceBudget::Request request(account, detail::BudgetClass::kRouting, 1); + test_read::Completion done; + auto connected = stdexec::connect(request.sender(), test_read::Receiver{.done = &done}); + stdexec::start(connected); + EXPECT_EQ(done.values.load(), 0U); + EXPECT_EQ(quantity(work, TreeWorkCounter::AdmissionWaits, TreeWorkCause::Allocation), 1U); + full.reset(); + EXPECT_EQ(done.values.load(), 1U); + EXPECT_EQ(quantity(work, TreeWorkCounter::AdmissionWaits, TreeWorkCause::Allocation), 1U); +} + +TEST(TreeWorkCounters, RegistrySamplesOnceBeforeCallbacksAndNeverDrainsTheCollector) +{ + TestBinding test; + auto work = test.binding->work_metrics(); + auto collector = work->claim_collector(test.binding->lifecycle_budget()); + ASSERT_TRUE(collector); + (void)collector->capture_window(); + const auto &credit = test.binding->lifecycle_budget(); + const auto baseline = test.binding->account()->snapshot().physical[static_cast<size_t>(AllocationKind::kMetadata)]; + auto resource = detail::reserved_shared<detail::AllocationMemoryResource>(credit, credit); + auto registry = create_tree_work_registry(resource, "test", work, credit); + TreeMetricsOperation operation(work, TreeStage::PageSplit, 1); + ASSERT_TRUE(operation.finish(TreeOutcome::Success, 12)); + work->counters.add(TreeWorkCounter::SplitOutputPages, 4, TreeWorkCause::Flush); + // Executed last, even though its name sorts first. Every quantity gauge + // must keep the first snapshot until the next registry flush. + registry->register_callback_gauge_inline("a.mutator", [target = work.get()] { + target->counters.add(TreeWorkCounter::SplitOutputPages, 4, TreeWorkCause::Flush); + return 0; + }); + auto first = rendered(*registry); + EXPECT_EQ(metric_value(first, "test.work.flush.split.output.pages"), 4U); + EXPECT_EQ(metric_value(first, "test.split.output.pages.c"), 4U); + EXPECT_NE(first.find("test.work.page.split.age.complete.g"), std::string::npos); + EXPECT_EQ(quantity(work, TreeWorkCounter::SplitOutputPages, TreeWorkCause::Flush), 8U); + const auto window = collector->capture_window(); + EXPECT_EQ(window[static_cast<size_t>(TreeStage::PageSplit)].count, 1U); + EXPECT_EQ(window[static_cast<size_t>(TreeStage::PageSplit)].sum_ns, 11U); + EXPECT_EQ(collector->capture_window()[static_cast<size_t>(TreeStage::PageSplit)].count, 0U); + registry.reset(); + resource.reset(); + test.binding->drain(); + EXPECT_EQ(test.binding->account()->snapshot().physical[static_cast<size_t>(AllocationKind::kMetadata)], baseline); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_work_histograms_c_api_test.cpp b/lib/crowdb-tree/tests/unit/tree_work_histograms_c_api_test.cpp new file mode 100644 index 000000000..78e5d3df3 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_work_histograms_c_api_test.cpp @@ -0,0 +1,150 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "btree/tree_work_histograms.h" +#include "c_api_internal.h" +#include "crowdb-tree/tree_work_histograms_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +struct TestWorkCapture +{ + ct_tree_work_snapshot_v1 work{}; + ct_tree_work_ages_v1 ages{}; + ct_tree_work_counters_v1 counters{}; + ct_tree_work_histograms_v1 histograms{}; + + TestWorkCapture() + { + work.struct_size = sizeof(work); + work.version = 1; + ages.struct_size = sizeof(ages); + ages.version = 1; + counters.struct_size = sizeof(counters); + counters.version = 1; + histograms.struct_size = sizeof(histograms); + histograms.version = 1; + } + + int32_t capture(ct_work_metrics_collector *collector) + { + return ct_work_metrics_capture_v3(collector, &work, &ages, &counters, &histograms); + } +}; + +TEST(TreeWorkHistogramsCAbi, InvalidCaptureDoesNotDrainAndOneImmutableWindowSurvivesTreeClose) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + auto metrics = TreeCoreAccess::get(*c_api_tree(tree))->publisher().work_metrics(); + ct_work_metrics_collector *collector = nullptr; + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + TestWorkCapture sample; + ASSERT_EQ(sample.capture(collector), 0); + EXPECT_EQ(sample.histograms.enabled_stages, 0U); + constexpr auto stage = static_cast<uint32_t>(TreeStage::FlushBatch); + EXPECT_EQ(ct_work_histogram_enable_v1(nullptr, stage), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_work_histogram_enable_v1(tree, CT_TREE_WORK_STAGES_V1), static_cast<int32_t>(Code::kInvalidArgument)); + ASSERT_EQ(ct_work_histogram_enable_v1(tree, stage), 0); + ASSERT_EQ(ct_work_histogram_enable_v1(tree, stage), 0); + TreeMetricsOperation operation(metrics, TreeStage::FlushBatch, 10); + ASSERT_TRUE(operation.finish(TreeOutcome::Error, 110)); + sample.histograms.version = 2; + sample.work.window_sequence = 77; + EXPECT_EQ(sample.capture(collector), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(sample.work.window_sequence, 77U); + sample.histograms.version = 1; + ASSERT_EQ(ct_work_histograms_totals_v1(tree, &sample.histograms), 0); + EXPECT_EQ(sample.histograms.stages[stage].count, 1U); + ASSERT_EQ(sample.capture(collector), 0); + EXPECT_EQ(sample.work.window_sequence, 2U); + EXPECT_EQ(sample.histograms.window_sequence, sample.work.window_sequence); + EXPECT_EQ(sample.histograms.enabled_stages, uint64_t{1} << stage); + EXPECT_EQ(sample.histograms.stages[stage].count, 1U); + EXPECT_EQ(sample.histograms.stages[stage].p95_ns, 100U); + EXPECT_EQ(sample.work.stages[stage].window_count, 1U); + const auto fanout = sample.histograms; + ASSERT_EQ(ct_work_histograms_totals_v1(tree, &sample.histograms), 0); + EXPECT_EQ(sample.histograms.stages[stage].count, 1U); + ct_close(tree); + ct_runtime_free(runtime); + metrics.reset(); + ASSERT_EQ(sample.capture(collector), 0); + EXPECT_EQ(sample.histograms.stages[stage].count, 0U); + EXPECT_EQ(sample.histograms.stages[stage].total_count, 1U); + EXPECT_EQ(fanout.stages[stage].count, 1U); + EXPECT_EQ(fanout.stages[stage].p95_ns, 100U); + ct_work_metrics_collector_free(collector); +} + +TEST(TreeWorkHistogramsCAbi, OlderCaptureLayoutsAdvanceTheSameHistogramWindow) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + constexpr auto stage = static_cast<uint32_t>(TreeStage::FlushBatch); + ASSERT_EQ(ct_work_histogram_enable_v1(tree, stage), 0); + auto metrics = TreeCoreAccess::get(*c_api_tree(tree))->publisher().work_metrics(); + ct_work_metrics_collector *collector = nullptr; + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + TestWorkCapture sample; + for (int version : {1, 2}) { + TreeMetricsOperation previous(metrics, TreeStage::FlushBatch, 10); + ASSERT_TRUE(previous.finish(TreeOutcome::Success, 110)); + const auto status = version == 1 + ? ct_work_metrics_capture(collector, &sample.work) + : ct_work_metrics_capture_v2(collector, &sample.work, &sample.ages, &sample.counters); + ASSERT_EQ(status, 0); + TreeMetricsOperation current(metrics, TreeStage::FlushBatch, 20); + ASSERT_TRUE(current.finish(TreeOutcome::Success, 50)); + ASSERT_EQ(sample.capture(collector), 0); + EXPECT_EQ(sample.work.window_sequence, 2U * version); + EXPECT_EQ(sample.histograms.window_sequence, sample.work.window_sequence); + EXPECT_EQ(sample.work.stages[stage].window_count, 1U); + EXPECT_EQ(sample.histograms.stages[stage].count, 1U); + EXPECT_EQ(sample.histograms.stages[stage].sum_low, 30U); + EXPECT_EQ(sample.histograms.stages[stage].total_count, 2U * version); + } + ct_work_metrics_collector_free(collector); + ct_close(tree); + ct_runtime_free(runtime); +} + +TEST(TreeWorkHistogramsCAbi, EnableBudgetFailurePreservesThePublishedViewAndSubsequentBusinessProgress) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + const auto before = core->capture(); + auto account = core->binding()->account(); + const auto stats = account->snapshot(); + const auto index = static_cast<size_t>(detail::BudgetClass::kRouting); + const auto available = + core->binding()->options().limits.routing_bytes - stats.allocated[index] - stats.reserved[index]; + auto held = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, available); + ASSERT_TRUE(held); + EXPECT_EQ(ct_work_histogram_enable_v1(tree, 0), static_cast<int32_t>(Code::kResourceExhausted)); + EXPECT_EQ(core->capture(), before); + EXPECT_EQ(core->publisher().work_metrics()->histogram_totals().enabled_stages, 0U); + held.reset(); + ASSERT_TRUE(core->apply(1, Batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "value"}}}).ok()); + ASSERT_TRUE(core->flush().ok()); + EXPECT_EQ(core->capture()->coverage, 1U); + ASSERT_EQ(ct_work_histogram_enable_v1(tree, 0), 0); + ct_close(tree); + ct_runtime_free(runtime); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_work_histograms_test.cpp b/lib/crowdb-tree/tests/unit/tree_work_histograms_test.cpp new file mode 100644 index 000000000..2b42deac2 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_work_histograms_test.cpp @@ -0,0 +1,222 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/allocation_reservation.h" +#include "btree/tree_work_histograms.h" +#include "crowdb-common/metrics/latency_histogram.h" + +#include <gtest/gtest.h> + +#include <barrier> +#include <thread> + +namespace crowdb::tree +{ +namespace +{ +constexpr size_t kFlush = static_cast<size_t>(TreeStage::FlushBatch); +constexpr size_t kEncode = static_cast<size_t>(TreeStage::PageEncode); +constexpr detail::ResourceBudget::Limits kLimits{4 << 20, 4 << 20, 4 << 20, 4 << 20, 4 << 20, 1}; + +class TreeWorkHistogramsTest : public testing::Test +{ + protected: + detail::ResourceBudget host{kLimits, 8}; + std::shared_ptr<detail::ResourceBudget::Account> account = host.make_account(kLimits); + + std::shared_ptr<detail::AllocationReservation> reserve(size_t bytes = 2 << 20) + { + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, bytes); + if (!credit) { + ADD_FAILURE() << "fixture credit must fit"; + return {}; + } + return detail::AllocationReservation::create(std::move(*credit)); + } + + [[nodiscard]] size_t metadata() const + { + return account->snapshot().physical[static_cast<size_t>(AllocationKind::kMetadata)]; + } +}; + +TEST_F(TreeWorkHistogramsTest, DefaultSummaryDoesNotAllocateBucketsAndEnableFailureLeavesItUsable) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto credit = reserve(); + ASSERT_TRUE(credit); + const auto bytes = TreeWorkMetrics::histogram_allocation_bytes(); + ASSERT_GE(credit->available(), bytes); + auto unused = credit->take_reserved(credit->available() - bytes + 1); + ASSERT_TRUE(unused); + unused.reset(); + const auto initial = metadata(); + EXPECT_EQ(metrics->histogram_totals().enabled_stages, 0U); + EXPECT_EQ(metrics->enable_histogram(TreeStage::FlushBatch, credit).code(), Code::kResourceExhausted); + EXPECT_EQ(metrics->enable_histogram(TreeStage::Count, credit).code(), Code::kInvalidArgument); + EXPECT_EQ(metadata(), initial); + EXPECT_EQ(metrics->histogram_totals().enabled_stages, 0U); + TreeMetricsOperation operation(metrics, TreeStage::FlushBatch, 10); + ASSERT_TRUE(operation.finish(TreeOutcome::Error, 110)); + EXPECT_EQ(metrics->totals()[kFlush].count, 1U); + EXPECT_EQ(metrics->totals()[kFlush].sum_ns, 100U); + EXPECT_EQ(metadata(), initial); +} + +TEST_F(TreeWorkHistogramsTest, ExactActualStorageSurvivesItsCollectorAndIsFreedWithTheLastMetricsOwner) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto credit = reserve(); + ASSERT_TRUE(credit); + const auto initial = metadata(); + const auto bytes = TreeWorkMetrics::histogram_allocation_bytes(); + EXPECT_GT(bytes, 64 * 1024U); + ASSERT_TRUE(metrics->enable_histogram(TreeStage::FlushBatch, credit).ok()); + EXPECT_EQ(metadata(), initial + bytes); + credit->seal(); + ASSERT_TRUE(metrics->enable_histogram(TreeStage::FlushBatch, credit).ok()); + EXPECT_EQ(metadata(), initial + bytes); + auto collector = metrics->claim_collector(); + ASSERT_TRUE(collector); + credit.reset(); + metrics.reset(); + EXPECT_EQ(metadata(), initial + bytes); + EXPECT_EQ(collector->capture_histograms().enabled_stages, uint64_t{1} << kFlush); + collector.reset(); + EXPECT_EQ(metadata(), 0U); + EXPECT_EQ(account->snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kRouting)], 0U); +} + +TEST_F(TreeWorkHistogramsTest, SelectedStageMatchesCommonQuantilesIncludingErrorsOverflowAndSingleTerminal) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto credit = reserve(); + ASSERT_TRUE(credit); + ASSERT_TRUE(metrics->enable_histogram(TreeStage::FlushBatch, credit).ok()); + common::metrics::LatencyHistogram::Data expected; + constexpr std::array<uint64_t, 8> durations{0, 1, 127, 128, 5000, uint64_t{1} << 40, UINT64_MAX, UINT64_MAX}; + for (size_t i = 0; i < durations.size(); ++i) { + TreeMetricsOperation operation(metrics, TreeStage::FlushBatch, 0); + ASSERT_TRUE(operation.finish(static_cast<TreeOutcome>(i % kTreeOutcomeCount), durations[i])); + EXPECT_FALSE(operation.finish(TreeOutcome::Success, durations[i])); + expected.observe(durations[i]); + } + TreeMetricsOperation unselected(metrics, TreeStage::PageEncode, 0); + ASSERT_TRUE(unselected.finish(TreeOutcome::Success, 999)); + const auto result = metrics->histogram_totals(); + const auto golden = expected.snapshot(); + const auto &actual = result.stages[kFlush]; + EXPECT_EQ(result.enabled_stages, uint64_t{1} << kFlush); + EXPECT_EQ(result.stages[kEncode].count, 0U); + EXPECT_EQ(actual.count, golden.count); + EXPECT_EQ(actual.sum_low, golden.sum_low); + EXPECT_EQ(actual.sum_high, golden.sum_high); + EXPECT_EQ(actual.min_ns, golden.min); + EXPECT_EQ(actual.max_ns, golden.max); + EXPECT_EQ(actual.p50_ns, golden.p50); + EXPECT_EQ(actual.p95_ns, golden.p95); + EXPECT_EQ(actual.p99_ns, golden.p99); + EXPECT_EQ(actual.overflow_count, 3U); + EXPECT_EQ(actual.zero_count, 1U); + auto collector = metrics->claim_collector(); + ASSERT_TRUE(collector); + EXPECT_FALSE(metrics->claim_collector()); + const auto window = collector->capture_histograms(); + const auto reader = window; + EXPECT_EQ(window.stages[kFlush].count, durations.size()); + EXPECT_EQ(collector->capture_histograms().stages[kFlush].count, 0U); + EXPECT_EQ(reader.stages[kFlush].count, durations.size()); + EXPECT_EQ(metrics->histogram_totals().stages[kFlush].count, durations.size()); +} + +class TestEnableRaceBudget final : public AllocationBudget +{ + public: + TestEnableRaceBudget(std::shared_ptr<AllocationBudget> budget, std::barrier<> &allocated) + : budget_(std::move(budget)), + allocated_(allocated) + { + } + + bool claim(size_t bytes) noexcept override + { + return budget_->claim(bytes); + } + + void release(size_t bytes) noexcept override + { + budget_->release(bytes); + } + + void record_allocation(AllocationKind kind, size_t bytes, bool acquired) noexcept override + { + budget_->record_allocation(kind, bytes, acquired); + if (acquired) { + allocated_.arrive_and_wait(); + } + } + + private: + std::shared_ptr<AllocationBudget> budget_; + std::barrier<> &allocated_; +}; + +TEST_F(TreeWorkHistogramsTest, ConcurrentEnablePublishesOnePaidInstanceAndReturnsEveryLosingAllocation) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto credit = reserve(); + ASSERT_TRUE(credit); + const auto initial = metadata(); + std::barrier allocated(8); + auto gated = std::make_shared<TestEnableRaceBudget>(credit, allocated); + std::array<std::thread, 8> workers; + for (auto &worker : workers) { + worker = std::thread([&] { EXPECT_TRUE(metrics->enable_histogram(TreeStage::FlushBatch, gated).ok()); }); + } + for (auto &worker : workers) { + worker.join(); + } + EXPECT_EQ(metadata(), initial + TreeWorkMetrics::histogram_allocation_bytes()); + EXPECT_EQ(metrics->histogram_totals().enabled_stages, uint64_t{1} << kFlush); + metrics.reset(); + EXPECT_EQ(metadata(), initial); + gated.reset(); + credit.reset(); + EXPECT_EQ(metadata(), 0U); +} + +TEST_F(TreeWorkHistogramsTest, ConcurrentObserveAndSingleDrainKeepExactFinalCumulativeCounts) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + auto credit = reserve(); + ASSERT_TRUE(credit); + ASSERT_TRUE(metrics->enable_histogram(TreeStage::PageEncode, credit).ok()); + auto collector = metrics->claim_collector(); + ASSERT_TRUE(collector); + std::atomic<size_t> active{4}; + std::array<std::thread, 4> workers; + for (auto &worker : workers) { + worker = std::thread([&] { + for (size_t i = 0; i < 1000; ++i) { + TreeMetricsOperation operation(metrics, TreeStage::PageEncode, 0); + EXPECT_TRUE(operation.finish(TreeOutcome::Success, 10)); + } + active.fetch_sub(1); + }); + } + uint64_t drained = 0; + while (active.load() != 0) { + drained += collector->capture_histograms().stages[kEncode].count; + } + for (auto &worker : workers) { + worker.join(); + } + drained += collector->capture_histograms().stages[kEncode].count; + const auto total = metrics->histogram_totals().stages[kEncode]; + EXPECT_EQ(drained, 4000U); + EXPECT_EQ(total.count, 4000U); + EXPECT_EQ(total.sum_low, 40000U); + EXPECT_EQ(total.p50_ns, 10U); + EXPECT_EQ(total.p99_ns, 10U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/tree_work_metrics_c_api_test.cpp b/lib/crowdb-tree/tests/unit/tree_work_metrics_c_api_test.cpp new file mode 100644 index 000000000..896f8c1b9 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/tree_work_metrics_c_api_test.cpp @@ -0,0 +1,134 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/tree_core_access.h" +#include "c_api_internal.h" +#include "crowdb-tree/tree_work_metrics_c_api.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +TEST(TreeWorkMetricsCAbi, DispatchSchemaRejectsInvalidOutputsAndPreservesCumulativeCounters) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + auto &dispatch = TreeCoreAccess::get(*c_api_tree(tree))->publisher().work_metrics()->dispatch; + dispatch.inline_probe(); + dispatch.inline_range(); + dispatch.queued_step(); + dispatch.run(5, 19); + dispatch.installed(37); + dispatch.begin(); + dispatch.begin(); + dispatch.finish(); + ct_tree_flush_dispatch_v1 result{}; + result.struct_size = sizeof(result); + result.version = 2; + result.cow_bytes = 99; + EXPECT_EQ(ct_flush_dispatch_totals(tree, &result), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(result.cow_bytes, 99U); + result.version = 1; + --result.struct_size; + EXPECT_EQ(ct_flush_dispatch_totals(tree, &result), static_cast<int32_t>(Code::kInvalidArgument)); + ++result.struct_size; + EXPECT_EQ(ct_flush_dispatch_totals(nullptr, &result), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(ct_flush_dispatch_totals(tree, nullptr), static_cast<int32_t>(Code::kInvalidArgument)); + ASSERT_EQ(ct_flush_dispatch_totals(tree, &result), 0); + EXPECT_EQ(result.inline_probes, 1U); + EXPECT_EQ(result.inline_ranges, 1U); + EXPECT_EQ(result.queued_steps, 1U); + EXPECT_EQ(result.prepared_runs, 1U); + EXPECT_EQ(result.prepared_entries, 5U); + EXPECT_EQ(result.prepared_bytes, 19U); + EXPECT_EQ(result.cow_bytes, 37U); + EXPECT_EQ(result.active_steps, 1U); + EXPECT_EQ(result.peak_steps, 2U); + dispatch.finish(); + ASSERT_EQ(ct_flush_dispatch_totals(tree, &result), 0); + EXPECT_EQ(result.active_steps, 0U); + EXPECT_EQ(result.peak_steps, 2U); + EXPECT_EQ(result.prepared_entries, 5U); + ct_close(tree); + ct_runtime_free(runtime); +} + +TEST(TreeWorkMetricsCAbi, SingleCollectorSharesCopiesAndTotalsNeverResetWindow) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + auto *core = TreeCoreAccess::get(*c_api_tree(tree)); + auto metrics = core->publisher().work_metrics(); + constexpr auto index = static_cast<size_t>(TreeStage::MemoryRetireWait); + core->publisher().reclaim(); + core->binding()->drain(); + ct_work_metrics_collector *collector = nullptr; + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + ct_work_metrics_collector *other = nullptr; + EXPECT_EQ(ct_work_metrics_collector_create(tree, &other), static_cast<int32_t>(Code::kUnavailable)); + EXPECT_EQ(other, nullptr); + ct_tree_work_snapshot_v1 totals{}; + totals.struct_size = sizeof(totals); + totals.version = 1; + ASSERT_EQ(ct_work_metrics_totals(tree, &totals), 0); + const auto initial_count = totals.stages[index].total_count; + EXPECT_EQ(totals.stages[index].window_count, 0U); + ct_tree_work_snapshot_v1 window = totals; + ASSERT_EQ(ct_work_metrics_capture(collector, &window), 0); + EXPECT_EQ(window.stages[index].window_count, initial_count); + TreeMetricsOperation operation(metrics, TreeStage::MemoryRetireWait, 100); + ASSERT_TRUE(operation.finish(TreeOutcome::Success, 150)); + ASSERT_EQ(ct_work_metrics_capture(collector, &window), 0); + const auto consumer = window; + EXPECT_EQ(window.window_sequence, 2U); + EXPECT_EQ(window.stages[index].window_count, 1U); + EXPECT_EQ(window.stages[index].window_sum_ns, 50U); + ASSERT_EQ(ct_work_metrics_totals(tree, &totals), 0); + EXPECT_EQ(totals.stages[index].total_count, initial_count + 1); + ct_close(tree); + ct_runtime_free(runtime); + // Closing the Tree also retires its real roots. The retained collector + // must include those completions while preserving the earlier window. + const auto final_count = metrics->totals()[index].count; + EXPECT_GT(final_count, initial_count + 1); + ASSERT_EQ(ct_work_metrics_capture(collector, &window), 0); + EXPECT_EQ(window.window_sequence, 3U); + EXPECT_EQ(window.stages[index].window_count, final_count - initial_count - 1); + EXPECT_EQ(window.stages[index].total_count, final_count); + EXPECT_EQ(consumer.stages[index].window_count, 1U); + ASSERT_EQ(ct_work_metrics_capture(collector, &window), 0); + EXPECT_EQ(window.window_sequence, 4U); + EXPECT_EQ(window.stages[index].window_count, 0U); + EXPECT_EQ(window.stages[index].total_count, final_count); + ct_work_metrics_collector_free(collector); +} + +TEST(TreeWorkMetricsCAbi, InvalidCaptureDoesNotDrainAndReleasedClaimCanBeReused) +{ + ct_runtime *runtime = nullptr; + ASSERT_EQ(ct_runtime_create(nullptr, &runtime), 0); + ct_options options{}; + ct_tree *tree = nullptr; + ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); + ct_work_metrics_collector *collector = nullptr; + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + ct_tree_work_snapshot_v1 window{}; + window.struct_size = sizeof(window); + window.version = 2; + EXPECT_EQ(ct_work_metrics_capture(collector, &window), static_cast<int32_t>(Code::kInvalidArgument)); + EXPECT_EQ(window.version, 2U); + window.version = 1; + ASSERT_EQ(ct_work_metrics_capture(collector, &window), 0); + EXPECT_EQ(window.window_sequence, 1U); + ct_work_metrics_collector_free(collector); + ASSERT_EQ(ct_work_metrics_collector_create(tree, &collector), 0); + ct_work_metrics_collector_free(collector); + ct_close(tree); + ct_runtime_free(runtime); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/version_test.cpp b/lib/crowdb-tree/tests/unit/version_test.cpp index 8cafab3ae..083789f7d 100644 --- a/lib/crowdb-tree/tests/unit/version_test.cpp +++ b/lib/crowdb-tree/tests/unit/version_test.cpp @@ -25,10 +25,13 @@ TEST(Version, FlushBumpsVersionAndTagsSlot) EXPECT_EQ(t.version(), 0U); ASSERT_TRUE(t.apply(1, put_one("a", "A1")).ok()); ASSERT_TRUE(t.flush().ok()); - EXPECT_EQ(t.version(), 1U); + const auto first = t.version(); + EXPECT_GT(first, 0U); + EXPECT_EQ(t.snapshot_view()->at_slot(), 1U); ASSERT_TRUE(t.apply(2, put_one("b", "B2")).ok()); ASSERT_TRUE(t.flush().ok()); - EXPECT_EQ(t.version(), 2U); + EXPECT_GT(t.version(), first); + EXPECT_EQ(t.snapshot_view()->at_slot(), 2U); } TEST(Version, SnapshotTagEqualsFlushedSlot) diff --git a/lib/crowdb-tree/tests/unit/writeback_metrics_test.cpp b/lib/crowdb-tree/tests/unit/writeback_metrics_test.cpp new file mode 100644 index 000000000..1d062acac --- /dev/null +++ b/lib/crowdb-tree/tests/unit/writeback_metrics_test.cpp @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/writeback_metrics.h" + +#include <gtest/gtest.h> + +namespace crowdb::tree +{ +namespace +{ +TEST(WritebackMetrics, TerminalCleanupYieldsAndNeverCompletesOnePageTwice) +{ + auto metrics = std::make_shared<TreeWorkMetrics>(); + WritebackMetrics writes(metrics, {}, 130); + for (uint64_t pid = 0; pid < 130; ++pid) { + ASSERT_TRUE(writes.begin(pid).ok()); + ASSERT_TRUE(writes.begin(pid).ok()); + } + EXPECT_EQ(writes.begin(130).code(), Code::kResourceExhausted); + writes.finish(0, TreeOutcome::Success); + writes.finish(0, TreeOutcome::Error); + EXPECT_FALSE(writes.finish_some(TreeOutcome::Error)); + auto sample = metrics->totals()[static_cast<size_t>(TreeStage::PageWritebackPipeline)]; + EXPECT_EQ(sample.attempts, 130U); + EXPECT_EQ(sample.count, 65U); + EXPECT_EQ(sample.inflight, 65U); + EXPECT_EQ(sample.outcomes[static_cast<size_t>(TreeOutcome::Success)], 1U); + EXPECT_EQ(sample.outcomes[static_cast<size_t>(TreeOutcome::Error)], 64U); + EXPECT_FALSE(writes.retire_step()); + EXPECT_TRUE(writes.retire_step()); + sample = metrics->totals()[static_cast<size_t>(TreeStage::PageWritebackPipeline)]; + EXPECT_EQ(sample.count, 130U); + EXPECT_EQ(sample.inflight, 0U); + EXPECT_EQ(sample.outcomes[static_cast<size_t>(TreeOutcome::Cancelled)], 65U); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/writeback_rebase_test.cpp b/lib/crowdb-tree/tests/unit/writeback_rebase_test.cpp new file mode 100644 index 000000000..15a69b9d8 --- /dev/null +++ b/lib/crowdb-tree/tests/unit/writeback_rebase_test.cpp @@ -0,0 +1,236 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "btree/writeback_rebase.h" +#include "load_budget_test_support.h" +#include "published_read_test_support.h" +#include "snapshot/captured_rebase.h" + +namespace crowdb::tree +{ +namespace +{ +using namespace test_read; + +DurableCapturedFrame receipt() +{ + auto backing = std::make_shared<BackingRef>(); + backing->backend_id = 1; + backing->content_id = 33; + backing->generation = 7; + backing->length = 4096; + backing->owner = std::make_shared<int>(1); + DurableCapturedFrame result{.page_id = 0, .backing = std::move(backing)}; + result.summary.emplace(); + result.summary->primary_used_bytes = 128; + result.summary->live_counts_known = true; + result.summary->live_keys = 2; + result.summary->live_key_bytes = 2; + result.summary->live_value_bytes = 6; + return result; +} + +std::shared_ptr<PublishedPage> dirty_page(std::shared_ptr<const PageState> leaf) +{ + auto result = published(std::move(leaf)); + result->metrics.overlay_runs = result->leaf->overlays.size(); + result->metrics.overlay_entries = 2; + result->metrics.writeback_candidate = true; + return result; +} + +TEST(WritebackRebase, ReplacesCapturedSuffixAndKeepsNewValuesAndTombstones) +{ + LoadCredits credits; + auto captured = dirty_page(page({ + {.key = "a", .value = "old"}, + {.key = "b", .value = "old"} + })); + auto current_leaf = std::make_shared<PageState>(*captured->leaf); + auto later = page({ + {.key = "a", .slot = 11, .value = "new"}, + {.key = "b", .slot = 11, .value = "", .tombstone = true} + }); + current_leaf->overlays.insert(current_leaf->overlays.begin(), later->overlays.front()); + current_leaf->content_id = 9; + auto current = dirty_page(current_leaf); + std::shared_ptr<const PublishedPage> replacement; + ASSERT_TRUE(build_writeback_rebase(captured, current, receipt(), 10, credits.credits.bytes, &replacement).ok()); + ASSERT_EQ(replacement->leaf->overlays.size(), 1U); + EXPECT_EQ(replacement->leaf->overlays.front(), later->overlays.front()); + EXPECT_EQ(replacement->leaf->content_id, 9U); + EXPECT_TRUE(replacement->metrics.dirty); + EXPECT_EQ(replacement->metrics.overlay_entries, 2U); + EXPECT_EQ(captured->leaf->overlays.size(), 1U); + EXPECT_EQ(current->leaf->overlays.size(), 2U); + auto loader = std::make_shared<TestLoader>(); + loader->frames[33] = base({ + {.key = "a", .value = "old"}, + {.key = "b", .value = "old"} + }); + auto fixed = view(replacement->leaf, {}, 10); + std::shared_ptr<PublishedGet> get; + ASSERT_TRUE(PublishedGet::create(fixed, loader, Slice("a"), {}, &get).ok()); + complete_inline(get->sender()); + ASSERT_TRUE(get->result().found); + EXPECT_EQ(get->result().cell.value.to_string(), "new"); + get.reset(); + ASSERT_TRUE(PublishedGet::create(fixed, loader, Slice("b"), {}, &get).ok()); + complete_inline(get->sender()); + EXPECT_FALSE(get->result().found); +} + +TEST(WritebackRebase, CleanReceiptMakesColdPageExactWithoutRetainingCapturedRun) +{ + auto captured = dirty_page(page({ + {.key = "a", .value = "old"}, + {.key = "b", .value = "old"} + })); + std::weak_ptr<const OverlayBatch> old = captured->leaf->overlays.front(); + std::shared_ptr<const PublishedPage> replacement; + ASSERT_TRUE(build_writeback_rebase(captured, captured, receipt(), 10, {}, &replacement).ok()); + EXPECT_FALSE(replacement->metrics.dirty); + EXPECT_FALSE(replacement->metrics.writeback_candidate); + EXPECT_TRUE(replacement->metrics.page_size_known); + EXPECT_EQ(replacement->metrics.page_size_bytes, 128U); + EXPECT_EQ(replacement->metrics.kv_count, 2U); + EXPECT_EQ(replacement->metrics.kv_bytes, 8U); + EXPECT_TRUE(replacement->leaf->overlays.empty()); + ASSERT_EQ(replacement->leaf->bases.size(), 1U); + EXPECT_FALSE(replacement->leaf->bases.front().resident); + captured.reset(); + EXPECT_TRUE(old.expired()); +} + +TEST(WritebackRebase, ChangedRouteBaseAndRunRejectWithoutPublishingOutput) +{ + auto captured = dirty_page(page({ + {.key = "a", .value = "old"} + })); + for (int change = 0; change < 4; ++change) { + auto leaf = std::make_shared<PageState>(*captured->leaf); + if (change == 0) { + ++leaf->route_generation; + } + if (change == 1) { + leaf->right_sibling = 5; + } + if (change == 2) { + leaf->bases.push_back(cold(2)); + } + if (change == 3) { + leaf->overlays.front() = page({ + {.key = "a", .value = "different"} + }) + ->overlays.front(); + } + auto current = dirty_page(leaf); + std::shared_ptr<const PublishedPage> out; + EXPECT_EQ(build_writeback_rebase(captured, current, receipt(), 10, {}, &out).code(), Code::kUnavailable); + EXPECT_FALSE(out); + } +} + +TEST(WritebackRebase, CapturedSourcesDoNotConsumeTheRetainedSuffixWindow) +{ + auto leaf = std::make_shared<PageState>(*page({ + {.key = "a", .value = "old"} + })); + leaf->overlays.assign(129, leaf->overlays.front()); + auto captured = dirty_page(leaf); + auto current_leaf = std::make_shared<PageState>(*leaf); + auto later = page({ + {.key = "b", .slot = 11, .value = "new"} + }) + ->overlays.front(); + current_leaf->overlays.insert(current_leaf->overlays.begin(), later); + auto current = dirty_page(current_leaf); + std::shared_ptr<const PublishedPage> replacement; + auto status = build_writeback_rebase(captured, current, receipt(), 10, {}, &replacement, 1); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(replacement->leaf->overlays.size(), 1U); + EXPECT_EQ(replacement->leaf->overlays.front(), later); + EXPECT_EQ(replacement->leaf->bases.size(), 1U); +} + +TEST(WritebackRebase, BudgetFailurePreservesCurrentAndOutput) +{ + LoadCredits credits; + auto metadata = credits.credits.bytes; + const auto remaining = metadata->available(); + ASSERT_TRUE(metadata->claim(remaining)); + auto captured = dirty_page(page({ + {.key = "a", .value = "old"} + })); + std::shared_ptr<const PublishedPage> out; + EXPECT_EQ(build_writeback_rebase(captured, captured, receipt(), 10, metadata, &out).code(), + Code::kResourceExhausted); + EXPECT_FALSE(out); + EXPECT_EQ(captured->leaf->overlays.size(), 1U); + metadata->release(remaining); +} + +TEST(WritebackRebase, BoundedOperationPublishesOnlyRepresentationAndDefersChangedPage) +{ + for (bool conflict : {false, true}) { + LoadCredits credits; + auto initial = view(page({ + {.key = "a", .value = "old"} + }), + {}, 10); + auto current = std::make_shared<PublishedTreeState>(*initial); + ASSERT_TRUE(apply_page_metrics(current->stats, nullptr, ¤t->mapping.get(0)->metrics)); + if (conflict) { + auto changed = std::make_shared<PageState>(*initial->mapping.get(0)->leaf); + ++changed->route_generation; + MappingView::CopyStats copied; + std::array<PageReplacement, 1> changes{ + {{.page_id = 0, .expected = initial->mapping.get(0), .replacement = published(changed)}}}; + current->mapping = current->mapping.replace(changes, copied); + } + auto runtime = std::make_shared<detail::FlushRuntime>(1, 8); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(8); + PublicationCoordinator publisher(4, 1, {}, std::pmr::get_default_resource(), {}, + {.metadata = {}, .initial = current, .source_catalog = {}}); + auto pages = std::make_shared<CapturedDurablePages>(nullptr); + pages->coverage = 10; + pages->generation = 1; + pages->frames.emplace(0, receipt()); + std::shared_ptr<CapturedRebaseOperation> operation; + ASSERT_TRUE( + CapturedRebaseOperation::create(publisher, initial, pages, runtime, lane, credits.credits.bytes, &operation) + .ok()); + Completion done; + auto connected = stdexec::connect(operation->sender(), Receiver{.done = &done}); + stdexec::start(connected); + runtime->drain(); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + EXPECT_EQ(operation->result().installed, conflict ? 0U : 1U); + EXPECT_EQ(operation->result().deferred, conflict ? 1U : 0U); + EXPECT_EQ(publisher.capture()->stats.checkpoint_pending_pages, current->stats.checkpoint_pending_pages); + EXPECT_EQ(publisher.capture()->stats.checkpoint_mapping_pending, current->stats.checkpoint_mapping_pending); + if (!conflict) { + EXPECT_FALSE(publisher.capture()->mapping.get(0)->metrics.dirty); + } + } +} + +TEST(WritebackRebase, InnerKeepsRoutingOwnerAndInstallsDurableBacking) +{ + auto inner = std::make_shared<InnerPageState>(); + inner->children.push_back(1); + auto captured = std::make_shared<PublishedPage>(); + captured->level = 1; + captured->inner = inner; + captured->metrics.leaf = false; + captured->metrics.dirty = true; + auto written = receipt(); + written.type = page_type::kInnerBase; + std::shared_ptr<const PublishedPage> out; + ASSERT_TRUE(build_writeback_rebase(captured, captured, written, 10, {}, &out).ok()); + EXPECT_EQ(out->inner, inner); + EXPECT_EQ(out->inner_backing, written.backing); + EXPECT_FALSE(out->metrics.dirty); + EXPECT_TRUE(out->metrics.evict_candidate); +} +} // namespace +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/zero_copy_apply_test.cpp b/lib/crowdb-tree/tests/unit/zero_copy_apply_test.cpp index eee71e077..b4456bc63 100644 --- a/lib/crowdb-tree/tests/unit/zero_copy_apply_test.cpp +++ b/lib/crowdb-tree/tests/unit/zero_copy_apply_test.cpp @@ -243,7 +243,7 @@ TEST(ApplyExternal, RoundTripReadBeforeFlush) EXPECT_EQ(slot, 1U); EXPECT_EQ(value.size(), 4096U); EXPECT_EQ(std::memcmp(value.data(), big.data(), 4096), 0); - EXPECT_EQ(drops.load(), 0); // still in memtable + EXPECT_EQ(drops.load(), 1); // Unknown backing capacity is copied during admission. } TEST(ApplyExternal, RoundTripReadAfterFlush) @@ -257,7 +257,7 @@ TEST(ApplyExternal, RoundTripReadAfterFlush) ops.push_back( {.key = "k2", .flags = 0, .value = buffer::wrap_external(big.data(), big.size(), &drops, count_drop)}); EXPECT_TRUE(t->apply_external(1, std::move(ops)).ok()); - EXPECT_TRUE(t->flush().ok()); // drain -> materialize -> drop_fn fires + EXPECT_TRUE(t->flush().ok()); EXPECT_EQ(drops.load(), 1); uint64_t slot; std::string value; @@ -288,7 +288,7 @@ TEST(ApplyExternal, MultiKeyBatchAtomicity) EXPECT_EQ(slot, 1U); EXPECT_EQ(value.size(), 256U); } - EXPECT_EQ(drops.load(), 0); // all still in memtable + EXPECT_EQ(drops.load(), 3); // Every unknown backing owner was released after copying. } TEST(ApplyExternal, IntraBatchLastKeyWins) @@ -303,7 +303,7 @@ TEST(ApplyExternal, IntraBatchLastKeyWins) ops.push_back({.key = "k", .flags = 0, .value = buffer::wrap_external(v1.data(), v1.size(), &drops, count_drop)}); ops.push_back({.key = "k", .flags = 0, .value = buffer::wrap_external(v2.data(), v2.size(), &drops, count_drop)}); EXPECT_TRUE(t->apply_external(1, std::move(ops)).ok()); - EXPECT_EQ(drops.load(), 1); // v1 freed (last-key-wins), v2 retained + EXPECT_EQ(drops.load(), 2); // Duplicate and selected unknown owners are both released. uint64_t slot; std::string value; EXPECT_TRUE(t->get("k", &slot, &value)); diff --git a/pixi.toml b/pixi.toml index dc8234321..3f19898bc 100644 --- a/pixi.toml +++ b/pixi.toml @@ -1,5 +1,5 @@ [workspace] -name = "crowdb-kv" +name = "crowdb-paxos-kv" version = "0.3.0" description = "CROWDB — distributed key-value store with multi-paxos groups" channels = ["conda-forge"] @@ -38,7 +38,7 @@ isa-l = "*" # crowdb-protocol's .fbs codegen. Provides `flatc` and flatbuffers/cpp headers. flatbuffers = "*" tomlplusplus = ">=3.3.0,<4" -# Rust toolchain (crowdb-kv) +# Rust toolchain (crowdb-paxos-kv) rust = ">=1.96" openssl = "*" pkg-config = "*" diff --git a/third-party/bytes/.github/workflows/ci.yml b/third-party/bytes/.github/workflows/ci.yml new file mode 100644 index 000000000..79b57dafd --- /dev/null +++ b/third-party/bytes/.github/workflows/ci.yml @@ -0,0 +1,211 @@ +name: CI + +on: + pull_request: + branches: + - master + push: + branches: + - master + schedule: + - cron: '0 2 * * 0' + +env: + RUSTFLAGS: -Dwarnings + RUST_BACKTRACE: 1 + nightly: nightly-2024-09-15 + +defaults: + run: + shell: bash + +jobs: + # Check formatting + rustfmt: + name: rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update stable + - name: Check formatting + run: cargo fmt --all --check + + # TODO + # # Apply clippy lints + # clippy: + # name: clippy + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - name: Apply clippy lints + # run: cargo clippy --all-features + + # This represents the minimum Rust version supported by + # Bytes. Updating this should be done in a dedicated PR. + # + # Tests are not run as tests may require newer versions of + # rust. + minrust: + name: minrust + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install cargo-hack + uses: taiki-e/install-action@cargo-hack + - name: Check + run: cargo hack check --feature-powerset --optional-deps --rust-version + + # Stable + stable: + name: stable + strategy: + matrix: + os: + - ubuntu-latest + - ubuntu-22.04-arm # TODO: update to 24.04 when https://github.com/rust-lang/rust/issues/135867 solved + - macos-latest + - windows-latest + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Install Rust + # --no-self-update is necessary because the windows environment cannot self-update rustup.exe. + run: rustup update stable --no-self-update + - name: Install cargo-hack + uses: taiki-e/install-action@cargo-hack + - name: Test + run: ci/test-stable.sh test + + # Nightly + nightly: + name: nightly + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update $nightly && rustup default $nightly + - name: Install cargo-hack + uses: taiki-e/install-action@cargo-hack + - name: Test + run: ci/test-stable.sh test + + panic-abort: + name: panic=abort tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update $nightly && rustup default $nightly + - name: Run tests with -Cpanic=abort + run: ci/panic-abort.sh + + # Run tests on some extra platforms + cross: + name: cross + strategy: + matrix: + include: + - target: i686-unknown-linux-gnu + os: ubuntu-latest + - target: armv7-unknown-linux-gnueabihf + os: ubuntu-22.04-arm # TODO: update to 24.04 when https://github.com/rust-lang/rust/issues/135867 solved + - target: powerpc-unknown-linux-gnu + os: ubuntu-latest + - target: powerpc64-unknown-linux-gnu + os: ubuntu-latest + - target: wasm32-wasip1 + os: ubuntu-latest + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update stable + - uses: taiki-e/setup-cross-toolchain-action@v1 + with: + target: ${{ matrix.target }} + - name: Test + run: cargo test --target ${{ matrix.target }} + + # Build for no_std environment. + no-std: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update stable + - name: Install cargo-hack + uses: taiki-e/install-action@cargo-hack + # thumbv6m-none-eabi supports atomic, but not atomic CAS. + # thumbv7m-none-eabi supports atomic CAS. + - run: rustup target add thumbv6m-none-eabi thumbv7m-none-eabi + # * --optional-deps is needed for serde feature + # * --no-dev-deps is needed to avoid https://github.com/rust-lang/cargo/issues/4866 + - run: cargo hack build --target thumbv7m-none-eabi --feature-powerset --skip std,default --optional-deps --no-dev-deps + # A sound way to provide atomic CAS on platforms without native atomic CAS is system-dependent. + # portable-atomic provides major ways via cfgs and accepts user-defined implementations via critical-section feature. + - run: cargo hack build --target thumbv6m-none-eabi --feature-powerset --skip std,default --optional-deps --no-dev-deps --features extra-platforms,extra-platforms/critical-section + + # Sanitizers + tsan: + name: tsan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update $nightly && rustup default $nightly + - name: Install rust-src + run: rustup component add rust-src + - name: ASAN / TSAN + run: ci/tsan.sh + + miri: + name: miri + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update $nightly && rustup default $nightly + - name: Miri + run: ci/miri.sh + + # Loom + loom: + name: loom + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update $nightly && rustup default $nightly + - name: Loom tests + run: RUSTFLAGS="--cfg loom -Dwarnings" cargo test --lib + + publish_docs: + name: Publish Documentation + needs: + - rustfmt + # - clippy + - stable + - nightly + - minrust + - cross + - tsan + - miri + - loom + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update $nightly && rustup default $nightly + - name: Build documentation + run: cargo doc --no-deps --all-features + env: + RUSTDOCFLAGS: --cfg docsrs + - name: Publish documentation + run: | + cd target/doc + git init + git add . + git -c user.name='ci' -c user.email='ci' commit -m 'Deploy Bytes API documentation' + git push -f -q https://git:${{ secrets.github_token }}@github.com/${{ github.repository }} HEAD:gh-pages + if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && github.repository == 'tokio-rs/bytes' diff --git a/third-party/bytes/.gitignore b/third-party/bytes/.gitignore new file mode 100644 index 000000000..4fffb2f89 --- /dev/null +++ b/third-party/bytes/.gitignore @@ -0,0 +1,2 @@ +/target +/Cargo.lock diff --git a/third-party/bytes/CHANGELOG.md b/third-party/bytes/CHANGELOG.md new file mode 100644 index 000000000..48e2df99b --- /dev/null +++ b/third-party/bytes/CHANGELOG.md @@ -0,0 +1,449 @@ +# 1.11.1 (February 3rd, 2026) + +- Fix integer overflow in `BytesMut::reserve` + +# 1.11.0 (November 14th, 2025) + +- Bump MSRV to 1.57 (#788) + +### Fixed + +- fix: `BytesMut` only reuse if src has remaining (#803) +- Specialize `BytesMut::put::<Bytes>` (#793) +- Reserve capacity in `BytesMut::put` (#794) +- Change `BytesMut::remaining_mut` to use `isize::MAX` instead of `usize::MAX` (#795) + +### Internal changes + +- Guarantee address in `slice()` for empty slices. (#780) +- Rename `Vtable::to_*` -> `Vtable::into_*` (#776) +- Fix latest clippy warnings (#787) +- Ignore `BytesMut::freeze` doctest on wasm (#790) +- Move `drop_fn` of `from_owner` into vtable (#801) + +# 1.10.1 (March 5th, 2025) + +### Fixed + +- Fix memory leak when using `to_vec` with `Bytes::from_owner` (#773) + +# 1.10.0 (February 3rd, 2025) + +### Added + +- Add feature to support platforms without atomic CAS (#467) +- `try_get_*` methods for `Buf` trait (#753) +- Implement `Buf::chunks_vectored` for `Take` (#617) +- Implement `Buf::chunks_vectored` for `VecDeque<u8>` (#708) + +### Fixed + +- Remove incorrect guarantee for `chunks_vectored` (#754) +- Ensure that tests pass under `panic=abort` (#749) + +# 1.9.0 (November 27, 2024) + +### Added + +- Add `Bytes::from_owner` to enable externally-allocated memory (#742) + +### Documented + +- Fix typo in Buf::chunk() comment (#744) + +### Internal changes + +- Replace BufMut::put with BufMut::put_slice in Writer impl (#745) +- Rename hex_impl! to fmt_impl! and reuse it for fmt::Debug (#743) + +# 1.8.0 (October 21, 2024) + +- Guarantee address in `split_off`/`split_to` for empty slices (#740) + +# 1.7.2 (September 17, 2024) + +### Fixed + +- Fix default impl of `Buf::{get_int, get_int_le}` (#732) + +### Documented + +- Fix double spaces in comments and doc comments (#731) + +### Internal changes + +- Ensure BytesMut::advance reduces capacity (#728) + +# 1.7.1 (August 1, 2024) + +This release reverts the following change due to a regression: + +- Reuse capacity when possible in `<BytesMut as Buf>::advance` impl (#698) + +The revert can be found at #726. + +# 1.7.0 (July 31, 2024) + +### Added + +- Add conversion from `Bytes` to `BytesMut` (#695, #710) +- Add reclaim method without additional allocation (#686) + +### Documented + +- Clarify how `BytesMut::zeroed` works (#714) +- Clarify the behavior of `Buf::chunk` (#717) + +### Changed + +- Change length condition of `BytesMut::truncate` +- Reuse capacity when possible in `<BytesMut as Buf>::advance` impl (#698) +- Improve `must_use` suggestion of `BytesMut::split` (#699) + +### Internal changes + +- Use `ManuallyDrop` instead of `mem::forget` (#678) +- Don't set `len` in `BytesMut::reserve` (#682) +- Optimize `Bytes::copy_to_bytes` (#688) +- Refactor `BytesMut::truncate` (#694) +- Refactor `BytesMut::resize` (#696) +- Reorder assertion in `Bytes::split_to`, `Bytes::split_off` (#689, #693) +- Use `offset_from` in more places (#705) +- Correct the wrong usage of `IntoIter` (#707) + +# 1.6.1 (July 13, 2024) + +This release fixes a bug where `Bytes::is_unique` returns incorrect values when +the `Bytes` originates from a shared `BytesMut`. (#718) + +# 1.6.0 (March 22, 2024) + +### Added + +- Add `Bytes::is_unique` (#643) + +### Documented + +- Fix changelog typo (#628) +- Fix some spelling mistakes (#633) +- Typo fix (#637) +- Fix broken links (#639) +- Add security policy (#649) + +### Internal changes + +- Move comment to correct constant (#629) +- Various cleanup (#635) +- Simplify `UninitSlice::as_uninit_slice_mut()` logic (#644) +- Use `self.` instead of `Self::` (#642) +- `BytesMut`: Assert alignment of `Shared` (#652) +- Remove unnecessary namespace qualifier (#660) +- Remove an unnecessary else branch (#662) +- Remove unreachable else branch (#661) +- make parameter mut in `From<Vec>` (#667) +- Restore commented tests (#665) +- Use `sub` instead of `offset` (#668) +- Calculate original capacity only if necessary (#666) +- `set_vec_pos` does not need a second parameter (#672) +- `get_vec_pos`: use `&self` instead of `&mut self` (#670) +- Refactor `split_at`/`split_to` (#663) +- Use `Iterator` from the prelude (#673) +- `copy_to_bytes`: Add panic section to docs (#676) +- Remove redundant reserve call (#674) +- Use `ManuallyDrop` instead of `mem::forget` (#675) + + +# 1.5.0 (September 7, 2023) + +### Added + +- Add `UninitSlice::{new,uninit}` (#598, #599) +- Implement `BufMut` for `&mut [MaybeUninit<u8>]` (#597) + +### Changed + +- Mark `BytesMut::extend_from_slice` as inline (#595) + +# 1.4.0 (January 31, 2023) + +### Added + +- Make `IntoIter` constructor public (#581) + +### Fixed + +- Avoid large reallocations when freezing `BytesMut` (#592) + +### Documented + +- Document which functions require `std` (#591) +- Fix duplicate "the the" typos (#585) + +# 1.3.0 (November 20, 2022) + +### Added + +- Rename and expose `BytesMut::spare_capacity_mut` (#572) +- Implement native-endian get and put functions for `Buf` and `BufMut` (#576) + +### Fixed + +- Don't have important data in unused capacity when calling reserve (#563) + +### Documented + +- `Bytes::new` etc should return `Self` not `Bytes` (#568) + +# 1.2.1 (July 30, 2022) + +### Fixed + +- Fix unbounded memory growth when using `reserve` (#560) + +# 1.2.0 (July 19, 2022) + +### Added + +- Add `BytesMut::zeroed` (#517) +- Implement `Extend<Bytes>` for `BytesMut` (#527) +- Add conversion from `BytesMut` to `Vec<u8>` (#543, #554) +- Add conversion from `Bytes` to `Vec<u8>` (#547) +- Add `UninitSlice::as_uninit_slice_mut()` (#548) +- Add const to `Bytes::{len,is_empty}` (#514) + +### Changed + +- Reuse vector in `BytesMut::reserve` (#539, #544) + +### Fixed + +- Make miri happy (#515, #523, #542, #545, #553) +- Make tsan happy (#541) +- Fix `remaining_mut()` on chain (#488) +- Fix amortized asymptotics of `BytesMut` (#555) + +### Documented + +- Redraw layout diagram with box drawing characters (#539) +- Clarify `BytesMut::unsplit` docs (#535) + +# 1.1.0 (August 25, 2021) + +### Added + +- `BufMut::put_bytes(self, val, cnt)` (#487) +- Implement `From<Box<[u8]>>` for `Bytes` (#504) + +### Changed + +- Override `put_slice` for `&mut [u8]` (#483) +- Panic on integer overflow in `Chain::remaining` (#482) +- Add inline tags to `UninitSlice` methods (#443) +- Override `copy_to_bytes` for Chain and Take (#481) +- Keep capacity when unsplit on empty other buf (#502) + +### Documented + +- Clarify `BufMut` allocation guarantees (#501) +- Clarify `BufMut::put_int` behavior (#486) +- Clarify actions of `clear` and `truncate`. (#508) + +# 1.0.1 (January 11, 2021) + +### Changed +- mark `Vec::put_slice` with `#[inline]` (#459) + +### Fixed +- Fix deprecation warning (#457) +- use `Box::into_raw` instead of `mem::forget`-in-disguise (#458) + +# 1.0.0 (December 22, 2020) + +### Changed +- Rename `Buf`/`BufMut` methods `bytes()` and `bytes_mut()` to `chunk()` and `chunk_mut()` (#450) + +### Removed +- remove unused Buf implementation. (#449) + +# 0.6.0 (October 21, 2020) + +API polish in preparation for a 1.0 release. + +### Changed +- `BufMut` is now an `unsafe` trait (#432). +- `BufMut::bytes_mut()` returns `&mut UninitSlice`, a type owned by `bytes` to + avoid undefined behavior (#433). +- `Buf::copy_to_bytes(len)` replaces `Buf::into_bytes()` (#439). +- `Buf`/`BufMut` utility methods are moved onto the trait and `*Ext` traits are + removed (#431). + +### Removed +- `BufMut::bytes_vectored_mut()` (#430). +- `new` methods on combinator types (#434). + +# 0.5.6 (July 13, 2020) + +- Improve `BytesMut` to reuse buffer when fully `advance`d. +- Mark `BytesMut::{as_mut, set_len}` with `#[inline]`. +- Relax synchronization when cloning in shared vtable of `Bytes`. +- Move `loom` to `dev-dependencies`. + +# 0.5.5 (June 18, 2020) + +### Added +- Allow using the `serde` feature in `no_std` environments (#385). + +### Fix +- Fix `BufMut::advance_mut` to panic if advanced passed the capacity (#354).. +- Fix `BytesMut::freeze` ignoring amount previously `advance`d (#352). + +# 0.5.4 (January 23, 2020) + +### Added +- Make `Bytes::new` a `const fn`. +- Add `From<BytesMut>` for `Bytes`. + +### Fix +- Fix reversed arguments in `PartialOrd` for `Bytes`. +- Fix `Bytes::truncate` losing original capacity when repr is an unshared `Vec`. +- Fix `Bytes::from(Vec)` when allocator gave `Vec` a pointer with LSB set. +- Fix panic in `Bytes::slice_ref` if argument is an empty slice. + +# 0.5.3 (December 12, 2019) + +### Added +- `must_use` attributes to `split`, `split_off`, and `split_to` methods (#337). + +### Fix +- Potential freeing of a null pointer in `Bytes` when constructed with an empty `Vec<u8>` (#341, #342). +- Calling `Bytes::truncate` with a size large than the length will no longer clear the `Bytes` (#333). + +# 0.5.2 (November 27, 2019) + +### Added +- `Limit` methods `into_inner`, `get_ref`, `get_mut`, `limit`, and `set_limit` (#325). + +# 0.5.1 (November 25, 2019) + +### Fix +- Growth documentation for `BytesMut` (#321) + +# 0.5.0 (November 25, 2019) + +### Fix +- Potential overflow in `copy_to_slice` + +### Changed +- Increased minimum supported Rust version to 1.39. +- `Bytes` is now a "trait object", allowing for custom allocation strategies (#298) +- `BytesMut` implicitly grows internal storage. `remaining_mut()` returns + `usize::MAX` (#316). +- `BufMut::bytes_mut` returns `&mut [MaybeUninit<u8>]` to reflect the unknown + initialization state (#305). +- `Buf` / `BufMut` implementations for `&[u8]` and `&mut [u8]` + respectively (#261). +- Move `Buf` / `BufMut` "extra" functions to an extension trait (#306). +- `BufMutExt::limit` (#309). +- `Bytes::slice` takes a `RangeBounds` argument (#265). +- `Bytes::from_static` is now a `const fn` (#311). +- A multitude of smaller performance optimizations. + +### Added +- `no_std` support (#281). +- `get_*`, `put_*`, `get_*_le`, and `put_*le` accessors for handling byte order. +- `BorrowMut` implementation for `BytesMut` (#185). + +### Removed +- `IntoBuf` (#288). +- `Buf` implementation for `&str` (#301). +- `byteorder` dependency (#280). +- `iovec` dependency, use `std::IoSlice` instead (#263). +- optional `either` dependency (#315). +- optional `i128` feature -- now available on stable. (#276). + +# 0.4.12 (March 6, 2019) + +### Added +- Implement `FromIterator<&'a u8>` for `BytesMut`/`Bytes` (#244). +- Implement `Buf` for `VecDeque` (#249). + +# 0.4.11 (November 17, 2018) + +* Use raw pointers for potentially racy loads (#233). +* Implement `BufRead` for `buf::Reader` (#232). +* Documentation tweaks (#234). + +# 0.4.10 (September 4, 2018) + +* impl `Buf` and `BufMut` for `Either` (#225). +* Add `Bytes::slice_ref` (#208). + +# 0.4.9 (July 12, 2018) + +* Add 128 bit number support behind a feature flag (#209). +* Implement `IntoBuf` for `&mut [u8]` + +# 0.4.8 (May 25, 2018) + +* Fix panic in `BytesMut` `FromIterator` implementation. +* Bytes: Recycle space when reserving space in vec mode (#197). +* Bytes: Add resize fn (#203). + +# 0.4.7 (April 27, 2018) + +* Make `Buf` and `BufMut` usable as trait objects (#186). +* impl BorrowMut for BytesMut (#185). +* Improve accessor performance (#195). + +# 0.4.6 (Janary 8, 2018) + +* Implement FromIterator for Bytes/BytesMut (#148). +* Add `advance` fn to Bytes/BytesMut (#166). +* Add `unsplit` fn to `BytesMut` (#162, #173). +* Improvements to Bytes split fns (#92). + +# 0.4.5 (August 12, 2017) + +* Fix range bug in `Take::bytes` +* Misc performance improvements +* Add extra `PartialEq` implementations. +* Add `Bytes::with_capacity` +* Implement `AsMut[u8]` for `BytesMut` + +# 0.4.4 (May 26, 2017) + +* Add serde support behind feature flag +* Add `extend_from_slice` on `Bytes` and `BytesMut` +* Add `truncate` and `clear` on `Bytes` +* Misc additional std trait implementations +* Misc performance improvements + +# 0.4.3 (April 30, 2017) + +* Fix Vec::advance_mut bug +* Bump minimum Rust version to 1.15 +* Misc performance tweaks + +# 0.4.2 (April 5, 2017) + +* Misc performance tweaks +* Improved `Debug` implementation for `Bytes` +* Avoid some incorrect assert panics + +# 0.4.1 (March 15, 2017) + +* Expose `buf` module and have most types available from there vs. root. +* Implement `IntoBuf` for `T: Buf`. +* Add `FromBuf` and `Buf::collect`. +* Add iterator adapter for `Buf`. +* Add scatter/gather support to `Buf` and `BufMut`. +* Add `Buf::chain`. +* Reduce allocations on repeated calls to `BytesMut::reserve`. +* Implement `Debug` for more types. +* Remove `Source` in favor of `IntoBuf`. +* Implement `Extend` for `BytesMut`. + + +# 0.4.0 (February 24, 2017) + +* Initial release diff --git a/third-party/bytes/CROWDB.md b/third-party/bytes/CROWDB.md new file mode 100644 index 000000000..74ef811da --- /dev/null +++ b/third-party/bytes/CROWDB.md @@ -0,0 +1,29 @@ +# Accounted owner control allocations + +This directory starts from bytes 1.11.1. `crowdb-upstream.json` records SHA-256 +hashes of the original files. Upstream licenses and tests are retained. + +The additive `Bytes::try_from_owner_accounted` constructor accepts an +`OwnerAllocation` implementation. It claims the exact `Layout` of the allocation +containing both the immutable owner and accounting value. Clones and slices +reuse that allocation. The accounting value is moved to a stack guard at final +release; the immutable owner is destroyed at its pinned address, its box is +deallocated, and only then does the guard refund the charge. The guard preserves +that order if owner destruction unwinds. An `AsRef` panic also destroys the +already-owned control allocation. + +The original `Bytes::from_owner` behavior is unchanged. The internal vtable gains +an optional accounting callback; existing constructors leave it absent. Conversion from an +accounted owner into mutable storage or a `Vec` performs the same explicit copy +as the upstream owner constructor. Accounting methods must not panic. + +`test_accounted_owner` observes the actual global allocator deallocation. It +covers admission refusal, clones, slices, cross-thread final release, conversion +to `Vec`, and panics in owner `AsRef` or destruction. One test owns the allocator +observation slot so independent test execution cannot overwrite its evidence. + +The additive owner allocation lease/context APIs let a downstream native or Rust +control allocation use the same admitted budget. A lease retains the immutable +owner without allocating its own control block. Consumers mark successful +allocation and release the lease only after freeing its associated storage. +Borrowed contexts require the immutable byte owner to outlive all their claims. diff --git a/third-party/bytes/Cargo.lock b/third-party/bytes/Cargo.lock new file mode 100644 index 000000000..9f0d7576e --- /dev/null +++ b/third-party/bytes/Cargo.lock @@ -0,0 +1,444 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bytes" +version = "1.11.1" +dependencies = [ + "loom", + "portable-atomic", + "serde", + "serde_test", +] + +[[package]] +name = "cc" +version = "1.2.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "find-msvc-tools" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" + +[[package]] +name = "generator" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_test" +version = "1.0.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" +dependencies = [ + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "syn" +version = "2.0.110" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link 0.1.3", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] diff --git a/third-party/bytes/Cargo.toml b/third-party/bytes/Cargo.toml new file mode 100644 index 000000000..8f8c73edb --- /dev/null +++ b/third-party/bytes/Cargo.toml @@ -0,0 +1,141 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.57" +name = "bytes" +version = "1.11.1" +authors = [ + "Carl Lerche <me@carllerche.com>", + "Sean McArthur <sean@seanmonstar.com>", +] +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Types and traits for working with bytes" +readme = "README.md" +keywords = [ + "buffers", + "zero-copy", + "io", +] +categories = [ + "network-programming", + "data-structures", +] +license = "MIT" +repository = "https://github.com/tokio-rs/bytes" + +[package.metadata.docs.rs] +rustdoc-args = [ + "--cfg", + "docsrs", +] + +[features] +default = ["std"] +std = [] + +[lib] +name = "bytes" +path = "src/lib.rs" + +[[test]] +name = "test_buf" +path = "tests/test_buf.rs" + +[[test]] +name = "test_buf_mut" +path = "tests/test_buf_mut.rs" + +[[test]] +name = "test_bytes" +path = "tests/test_bytes.rs" + +[[test]] +name = "test_bytes_odd_alloc" +path = "tests/test_bytes_odd_alloc.rs" + +[[test]] +name = "test_bytes_vec_alloc" +path = "tests/test_bytes_vec_alloc.rs" + +[[test]] +name = "test_chain" +path = "tests/test_chain.rs" + +[[test]] +name = "test_debug" +path = "tests/test_debug.rs" + +[[test]] +name = "test_iter" +path = "tests/test_iter.rs" + +[[test]] +name = "test_limit" +path = "tests/test_limit.rs" + +[[test]] +name = "test_reader" +path = "tests/test_reader.rs" + +[[test]] +name = "test_serde" +path = "tests/test_serde.rs" + +[[test]] +name = "test_take" +path = "tests/test_take.rs" + +[[bench]] +name = "buf" +path = "benches/buf.rs" + +[[bench]] +name = "bytes" +path = "benches/bytes.rs" + +[[bench]] +name = "bytes_mut" +path = "benches/bytes_mut.rs" + +[dependencies.extra-platforms] +version = "1.3" +features = ["require-cas"] +optional = true +default-features = false +package = "portable-atomic" + +[dependencies.serde] +version = "1.0.60" +features = ["alloc"] +optional = true +default-features = false + +[dev-dependencies.serde_test] +version = "1.0" + +[target."cfg(loom)".dev-dependencies.loom] +version = "0.7" + +[lints.rust.unexpected_cfgs] +level = "warn" +priority = 0 +check-cfg = ["cfg(loom)"] + +[[test]] +name = "test_accounted_owner" +path = "tests/test_accounted_owner.rs" diff --git a/third-party/bytes/Cargo.toml.orig b/third-party/bytes/Cargo.toml.orig new file mode 100644 index 000000000..57df5910d --- /dev/null +++ b/third-party/bytes/Cargo.toml.orig @@ -0,0 +1,45 @@ +[package] + +name = "bytes" +# When releasing to crates.io: +# - Update CHANGELOG.md. +# - Create "v1.x.y" git tag. +version = "1.11.1" +edition = "2021" +rust-version = "1.57" +license = "MIT" +authors = [ + "Carl Lerche <me@carllerche.com>", + "Sean McArthur <sean@seanmonstar.com>", +] +description = "Types and traits for working with bytes" +repository = "https://github.com/tokio-rs/bytes" +readme = "README.md" +keywords = ["buffers", "zero-copy", "io"] +categories = ["network-programming", "data-structures"] + +[features] +default = ["std"] +std = [] + +[dependencies] +serde = { version = "1.0.60", optional = true, default-features = false, features = ["alloc"] } +# Use portable-atomic crate to support platforms without atomic CAS. +# See "no_std support" section in readme for more information. +# +# Enable require-cas feature to provide a better error message if the end user forgets to use the cfg or feature. +extra-platforms = { package = "portable-atomic", version = "1.3", optional = true, default-features = false, features = ["require-cas"] } + +[dev-dependencies] +serde_test = "1.0" + +[target.'cfg(loom)'.dev-dependencies] +loom = "0.7" + +[package.metadata.docs.rs] +rustdoc-args = ["--cfg", "docsrs"] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = [ + 'cfg(loom)', +] } diff --git a/third-party/bytes/LICENSE b/third-party/bytes/LICENSE new file mode 100644 index 000000000..58fb29a12 --- /dev/null +++ b/third-party/bytes/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2018 Carl Lerche + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third-party/bytes/README.md b/third-party/bytes/README.md new file mode 100644 index 000000000..90d631c97 --- /dev/null +++ b/third-party/bytes/README.md @@ -0,0 +1,73 @@ +# Bytes + +A utility library for working with bytes. + +[![Crates.io][crates-badge]][crates-url] +[![Build Status][ci-badge]][ci-url] + +[crates-badge]: https://img.shields.io/crates/v/bytes.svg +[crates-url]: https://crates.io/crates/bytes +[ci-badge]: https://github.com/tokio-rs/bytes/workflows/CI/badge.svg +[ci-url]: https://github.com/tokio-rs/bytes/actions + +[Documentation](https://docs.rs/bytes) + +## Usage + +To use `bytes`, first add this to your `Cargo.toml`: + +```toml +[dependencies] +bytes = "1" +``` + +Next, add this to your crate: + +```rust +use bytes::{Bytes, BytesMut, Buf, BufMut}; +``` + +## no_std support + +To use `bytes` with no_std environment, disable the (enabled by default) `std` feature. + +```toml +[dependencies] +bytes = { version = "1", default-features = false } +``` + +To use `bytes` with no_std environment without atomic CAS, such as thumbv6m, you also need to enable +the `extra-platforms` feature. See the [documentation for the `portable-atomic` +crate](https://docs.rs/portable-atomic) for more information. + +The MSRV when `extra-platforms` feature is enabled depends on the MSRV of `portable-atomic`. + +## Serde support + +Serde support is optional and disabled by default. To enable use the feature `serde`. + +```toml +[dependencies] +bytes = { version = "1", features = ["serde"] } +``` + +The MSRV when `serde` feature is enabled depends on the MSRV of `serde`. + +## Building documentation + +When building the `bytes` documentation the `docsrs` option should be used, otherwise +feature gates will not be shown. This requires a nightly toolchain: + +``` +RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc +``` + +## License + +This project is licensed under the [MIT license](LICENSE). + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in `bytes` by you, shall be licensed as MIT, without any additional +terms or conditions. diff --git a/third-party/bytes/SECURITY.md b/third-party/bytes/SECURITY.md new file mode 100644 index 000000000..b74a831cb --- /dev/null +++ b/third-party/bytes/SECURITY.md @@ -0,0 +1,9 @@ +# Security Policy + +Bytes is part of the Tokio project and uses the same security policy as [Tokio][tokio-security]. + +## Report a security issue + +The process for reporting an issue is the same as for [Tokio][tokio-security]. This includes private reporting via security@tokio.rs. + +[tokio-security]: https://github.com/tokio-rs/tokio/security/policy diff --git a/third-party/bytes/benches/buf.rs b/third-party/bytes/benches/buf.rs new file mode 100644 index 000000000..616d18748 --- /dev/null +++ b/third-party/bytes/benches/buf.rs @@ -0,0 +1,186 @@ +#![feature(test)] +#![warn(rust_2018_idioms)] + +extern crate test; + +use bytes::Buf; +use test::Bencher; + +/// Dummy Buf implementation +struct TestBuf { + buf: &'static [u8], + readlens: &'static [usize], + init_pos: usize, + pos: usize, + readlen_pos: usize, + readlen: usize, +} +impl TestBuf { + fn new(buf: &'static [u8], readlens: &'static [usize], init_pos: usize) -> TestBuf { + let mut buf = TestBuf { + buf, + readlens, + init_pos, + pos: 0, + readlen_pos: 0, + readlen: 0, + }; + buf.reset(); + buf + } + fn reset(&mut self) { + self.pos = self.init_pos; + self.readlen_pos = 0; + self.next_readlen(); + } + /// Compute the length of the next read : + /// - use the next value specified in readlens (capped by remaining) if any + /// - else the remaining + fn next_readlen(&mut self) { + self.readlen = self.buf.len() - self.pos; + if let Some(readlen) = self.readlens.get(self.readlen_pos) { + self.readlen = std::cmp::min(self.readlen, *readlen); + self.readlen_pos += 1; + } + } +} +impl Buf for TestBuf { + fn remaining(&self) -> usize { + self.buf.len() - self.pos + } + fn advance(&mut self, cnt: usize) { + self.pos += cnt; + assert!(self.pos <= self.buf.len()); + self.next_readlen(); + } + fn chunk(&self) -> &[u8] { + if self.readlen == 0 { + Default::default() + } else { + &self.buf[self.pos..self.pos + self.readlen] + } + } +} + +/// Dummy Buf implementation +/// version with methods forced to not be inlined (to simulate costly calls) +struct TestBufC { + inner: TestBuf, +} +impl TestBufC { + fn new(buf: &'static [u8], readlens: &'static [usize], init_pos: usize) -> TestBufC { + TestBufC { + inner: TestBuf::new(buf, readlens, init_pos), + } + } + fn reset(&mut self) { + self.inner.reset() + } +} +impl Buf for TestBufC { + #[inline(never)] + fn remaining(&self) -> usize { + self.inner.remaining() + } + #[inline(never)] + fn advance(&mut self, cnt: usize) { + self.inner.advance(cnt) + } + #[inline(never)] + fn chunk(&self) -> &[u8] { + self.inner.chunk() + } +} + +macro_rules! bench { + ($fname:ident, testbuf $testbuf:ident $readlens:expr, $method:ident $(,$arg:expr)*) => ( + #[bench] + fn $fname(b: &mut Bencher) { + let mut bufs = [ + $testbuf::new(&[1u8; 8+0], $readlens, 0), + $testbuf::new(&[1u8; 8+1], $readlens, 1), + $testbuf::new(&[1u8; 8+2], $readlens, 2), + $testbuf::new(&[1u8; 8+3], $readlens, 3), + $testbuf::new(&[1u8; 8+4], $readlens, 4), + $testbuf::new(&[1u8; 8+5], $readlens, 5), + $testbuf::new(&[1u8; 8+6], $readlens, 6), + $testbuf::new(&[1u8; 8+7], $readlens, 7), + ]; + b.iter(|| { + for i in 0..8 { + bufs[i].reset(); + let buf: &mut dyn Buf = &mut bufs[i]; // type erasure + test::black_box(buf.$method($($arg,)*)); + } + }) + } + ); + ($fname:ident, slice, $method:ident $(,$arg:expr)*) => ( + #[bench] + fn $fname(b: &mut Bencher) { + // buf must be long enough for one read of 8 bytes starting at pos 7 + let arr = [1u8; 8+7]; + b.iter(|| { + for i in 0..8 { + let mut buf = &arr[i..]; + let buf = &mut buf as &mut dyn Buf; // type erasure + test::black_box(buf.$method($($arg,)*)); + } + }) + } + ); + ($fname:ident, option) => ( + #[bench] + fn $fname(b: &mut Bencher) { + let data = [1u8; 1]; + b.iter(|| { + for _ in 0..8 { + let mut buf = Some(data); + let buf = &mut buf as &mut dyn Buf; // type erasure + test::black_box(buf.get_u8()); + } + }) + } + ); +} + +macro_rules! bench_group { + ($method:ident $(,$arg:expr)*) => ( + bench!(slice, slice, $method $(,$arg)*); + bench!(tbuf_1, testbuf TestBuf &[], $method $(,$arg)*); + bench!(tbuf_1_costly, testbuf TestBufC &[], $method $(,$arg)*); + bench!(tbuf_2, testbuf TestBuf &[1], $method $(,$arg)*); + bench!(tbuf_2_costly, testbuf TestBufC &[1], $method $(,$arg)*); + // bench!(tbuf_onebyone, testbuf TestBuf &[1,1,1,1,1,1,1,1], $method $(,$arg)*); + // bench!(tbuf_onebyone_costly, testbuf TestBufC &[1,1,1,1,1,1,1,1], $method $(,$arg)*); + ); +} + +mod get_u8 { + use super::*; + bench_group!(get_u8); +} +mod get_u16 { + use super::*; + bench_group!(get_u16); +} +mod get_u32 { + use super::*; + bench_group!(get_u32); +} +mod get_u64 { + use super::*; + bench_group!(get_u64); +} +mod get_f32 { + use super::*; + bench_group!(get_f32); +} +mod get_f64 { + use super::*; + bench_group!(get_f64); +} +mod get_uint24 { + use super::*; + bench_group!(get_uint, 3); +} diff --git a/third-party/bytes/benches/bytes.rs b/third-party/bytes/benches/bytes.rs new file mode 100644 index 000000000..8782d0066 --- /dev/null +++ b/third-party/bytes/benches/bytes.rs @@ -0,0 +1,120 @@ +#![feature(test)] +#![warn(rust_2018_idioms)] + +extern crate test; + +use bytes::Bytes; +use test::Bencher; + +#[bench] +fn deref_unique(b: &mut Bencher) { + let buf = Bytes::from(vec![0; 1024]); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn deref_shared(b: &mut Bencher) { + let buf = Bytes::from(vec![0; 1024]); + let _b2 = buf.clone(); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn deref_static(b: &mut Bencher) { + let buf = Bytes::from_static(b"hello world"); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn clone_static(b: &mut Bencher) { + let bytes = + Bytes::from_static("hello world 1234567890 and have a good byte 0987654321".as_bytes()); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(test::black_box(&bytes).clone()); + } + }) +} + +#[bench] +fn clone_shared(b: &mut Bencher) { + let bytes = Bytes::from(b"hello world 1234567890 and have a good byte 0987654321".to_vec()); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(test::black_box(&bytes).clone()); + } + }) +} + +#[bench] +fn clone_arc_vec(b: &mut Bencher) { + use std::sync::Arc; + let bytes = Arc::new(b"hello world 1234567890 and have a good byte 0987654321".to_vec()); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(test::black_box(&bytes).clone()); + } + }) +} + +#[bench] +fn from_long_slice(b: &mut Bencher) { + let data = [0u8; 128]; + b.bytes = data.len() as u64; + b.iter(|| { + let buf = Bytes::copy_from_slice(&data[..]); + test::black_box(buf); + }) +} + +#[bench] +fn slice_empty(b: &mut Bencher) { + b.iter(|| { + // `clone` is to convert to ARC + let b = Bytes::from(vec![17; 1024]).clone(); + for i in 0..1000 { + test::black_box(b.slice(i % 100..i % 100)); + } + }) +} + +#[bench] +fn slice_short_from_arc(b: &mut Bencher) { + b.iter(|| { + // `clone` is to convert to ARC + let b = Bytes::from(vec![17; 1024]).clone(); + for i in 0..1000 { + test::black_box(b.slice(1..2 + i % 10)); + } + }) +} + +#[bench] +fn split_off_and_drop(b: &mut Bencher) { + b.iter(|| { + for _ in 0..1024 { + let v = vec![10; 200]; + let mut b = Bytes::from(v); + test::black_box(b.split_off(100)); + test::black_box(b); + } + }) +} diff --git a/third-party/bytes/benches/bytes_mut.rs b/third-party/bytes/benches/bytes_mut.rs new file mode 100644 index 000000000..b06943621 --- /dev/null +++ b/third-party/bytes/benches/bytes_mut.rs @@ -0,0 +1,266 @@ +#![feature(test)] +#![warn(rust_2018_idioms)] + +extern crate test; + +use bytes::{BufMut, BytesMut}; +use test::Bencher; + +#[bench] +fn alloc_small(b: &mut Bencher) { + b.iter(|| { + for _ in 0..1024 { + test::black_box(BytesMut::with_capacity(12)); + } + }) +} + +#[bench] +fn alloc_mid(b: &mut Bencher) { + b.iter(|| { + test::black_box(BytesMut::with_capacity(128)); + }) +} + +#[bench] +fn alloc_big(b: &mut Bencher) { + b.iter(|| { + test::black_box(BytesMut::with_capacity(4096)); + }) +} + +#[bench] +fn deref_unique(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(4096); + buf.put(&[0u8; 1024][..]); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn deref_unique_unroll(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(4096); + buf.put(&[0u8; 1024][..]); + + b.iter(|| { + for _ in 0..128 { + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn deref_shared(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(4096); + buf.put(&[0u8; 1024][..]); + let _b2 = buf.split_off(1024); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&buf[..]); + } + }) +} + +#[bench] +fn deref_two(b: &mut Bencher) { + let mut buf1 = BytesMut::with_capacity(8); + buf1.put(&[0u8; 8][..]); + + let mut buf2 = BytesMut::with_capacity(4096); + buf2.put(&[0u8; 1024][..]); + + b.iter(|| { + for _ in 0..512 { + test::black_box(&buf1[..]); + test::black_box(&buf2[..]); + } + }) +} + +#[bench] +fn clone_frozen(b: &mut Bencher) { + let bytes = BytesMut::from(&b"hello world 1234567890 and have a good byte 0987654321"[..]) + .split() + .freeze(); + + b.iter(|| { + for _ in 0..1024 { + test::black_box(&bytes.clone()); + } + }) +} + +#[bench] +fn alloc_write_split_to_mid(b: &mut Bencher) { + b.iter(|| { + let mut buf = BytesMut::with_capacity(128); + buf.put_slice(&[0u8; 64]); + test::black_box(buf.split_to(64)); + }) +} + +#[bench] +fn drain_write_drain(b: &mut Bencher) { + let data = [0u8; 128]; + + b.iter(|| { + let mut buf = BytesMut::with_capacity(1024); + let mut parts = Vec::with_capacity(8); + + for _ in 0..8 { + buf.put(&data[..]); + parts.push(buf.split_to(128)); + } + + test::black_box(parts); + }) +} + +#[bench] +fn fmt_write(b: &mut Bencher) { + use std::fmt::Write; + let mut buf = BytesMut::with_capacity(128); + let s = "foo bar baz quux lorem ipsum dolor et"; + + b.bytes = s.len() as u64; + b.iter(|| { + let _ = write!(buf, "{}", s); + test::black_box(&buf); + unsafe { + buf.set_len(0); + } + }) +} + +#[bench] +fn bytes_mut_extend(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(256); + let data = [33u8; 32]; + + b.bytes = data.len() as u64 * 4; + b.iter(|| { + for _ in 0..4 { + buf.extend(&data); + } + test::black_box(&buf); + unsafe { + buf.set_len(0); + } + }); +} + +// BufMut for BytesMut vs Vec<u8> + +#[bench] +fn put_slice_bytes_mut(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(256); + let data = [33u8; 32]; + + b.bytes = data.len() as u64 * 4; + b.iter(|| { + for _ in 0..4 { + buf.put_slice(&data); + } + test::black_box(&buf); + unsafe { + buf.set_len(0); + } + }); +} + +#[bench] +fn put_u8_bytes_mut(b: &mut Bencher) { + let mut buf = BytesMut::with_capacity(256); + let cnt = 128; + + b.bytes = cnt as u64; + b.iter(|| { + for _ in 0..cnt { + buf.put_u8(b'x'); + } + test::black_box(&buf); + unsafe { + buf.set_len(0); + } + }); +} + +#[bench] +fn put_slice_vec(b: &mut Bencher) { + let mut buf = Vec::<u8>::with_capacity(256); + let data = [33u8; 32]; + + b.bytes = data.len() as u64 * 4; + b.iter(|| { + for _ in 0..4 { + buf.put_slice(&data); + } + test::black_box(&buf); + unsafe { + buf.set_len(0); + } + }); +} + +#[bench] +fn put_u8_vec(b: &mut Bencher) { + let mut buf = Vec::<u8>::with_capacity(256); + let cnt = 128; + + b.bytes = cnt as u64; + b.iter(|| { + for _ in 0..cnt { + buf.put_u8(b'x'); + } + test::black_box(&buf); + unsafe { + buf.set_len(0); + } + }); +} + +#[bench] +fn put_slice_vec_extend(b: &mut Bencher) { + let mut buf = Vec::<u8>::with_capacity(256); + let data = [33u8; 32]; + + b.bytes = data.len() as u64 * 4; + b.iter(|| { + for _ in 0..4 { + buf.extend_from_slice(&data); + } + test::black_box(&buf); + unsafe { + buf.set_len(0); + } + }); +} + +#[bench] +fn put_u8_vec_push(b: &mut Bencher) { + let mut buf = Vec::<u8>::with_capacity(256); + let cnt = 128; + + b.bytes = cnt as u64; + b.iter(|| { + for _ in 0..cnt { + buf.push(b'x'); + } + test::black_box(&buf); + unsafe { + buf.set_len(0); + } + }); +} diff --git a/third-party/bytes/ci/miri.sh b/third-party/bytes/ci/miri.sh new file mode 100644 index 000000000..ca7f41df5 --- /dev/null +++ b/third-party/bytes/ci/miri.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +rustup component add miri +cargo miri setup + +export MIRIFLAGS="-Zmiri-strict-provenance" + +cargo miri test +cargo miri test --target mips64-unknown-linux-gnuabi64 + +# run with wrapping integer overflow instead of panic +cargo miri test --release diff --git a/third-party/bytes/ci/panic-abort.sh b/third-party/bytes/ci/panic-abort.sh new file mode 100644 index 000000000..4284791f1 --- /dev/null +++ b/third-party/bytes/ci/panic-abort.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +set -ex +RUSTFLAGS="$RUSTFLAGS -Cpanic=abort -Zpanic-abort-tests" cargo test --all-features --test '*' diff --git a/third-party/bytes/ci/test-stable.sh b/third-party/bytes/ci/test-stable.sh new file mode 100644 index 000000000..ad9757449 --- /dev/null +++ b/third-party/bytes/ci/test-stable.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -ex + +cmd="${1:-test}" + +# Run with each feature +# * --each-feature includes both default/no-default features +# * --optional-deps is needed for serde feature +cargo hack "${cmd}" --each-feature --optional-deps +# Run with all features +cargo "${cmd}" --all-features + +if [[ "${RUST_VERSION}" == "nightly"* ]]; then + # Check benchmarks + cargo check --benches + + # Check minimal versions + # Remove dev-dependencies from Cargo.toml to prevent the next `cargo update` + # from determining minimal versions based on dev-dependencies. + cargo hack --remove-dev-deps --workspace + # Update Cargo.lock to minimal version dependencies. + cargo update -Z minimal-versions + cargo check --all-features +fi diff --git a/third-party/bytes/ci/tsan.sh b/third-party/bytes/ci/tsan.sh new file mode 100644 index 000000000..ca520bd7f --- /dev/null +++ b/third-party/bytes/ci/tsan.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -ex + +export ASAN_OPTIONS="detect_odr_violation=0 detect_leaks=0" + +# Run address sanitizer +RUSTFLAGS="-Z sanitizer=address" \ +cargo test --target x86_64-unknown-linux-gnu --test test_bytes --test test_buf --test test_buf_mut + +# Run thread sanitizer +RUSTFLAGS="-Z sanitizer=thread" \ +cargo -Zbuild-std test --target x86_64-unknown-linux-gnu --test test_bytes --test test_buf --test test_buf_mut diff --git a/third-party/bytes/clippy.toml b/third-party/bytes/clippy.toml new file mode 100644 index 000000000..5cccb362c --- /dev/null +++ b/third-party/bytes/clippy.toml @@ -0,0 +1 @@ +msrv = "1.57" diff --git a/third-party/bytes/crowdb-upstream.json b/third-party/bytes/crowdb-upstream.json new file mode 100644 index 000000000..c3c3d5248 --- /dev/null +++ b/third-party/bytes/crowdb-upstream.json @@ -0,0 +1,55 @@ +{ + "crate": "bytes", + "version": "1.11.1", + "source": "crates.io", + "files": { + ".gitignore": "c1e953ee360e77de57f7b02f1b7880bd6a3dc22d1a69e953c2ac2c52cc52d247", + "CHANGELOG.md": "98b59d530d4ee459b9183c8601aad783bdc5f5ce01c68f2df7f29438f47fe871", + "Cargo.lock": "3db4c23f448167dc361d799583456431062a6e3fbedf2ceb075adfa263da9c6c", + "Cargo.toml": "ebdcb7f4b82227b25a075ca774a6021037502d6305d77fee8e01b157c84ffdaa", + "Cargo.toml.orig": "3a4815f3f9939b85f08ea5785f083f5153234de58e4125c64be33fe91459075f", + "LICENSE": "45f522cacecb1023856e46df79ca625dfc550c94910078bd8aec6e02880b3d42", + "README.md": "275ac7bc295e3cc659d28f05261645ac5bd6286476afe2ee3c1ed719f7b6df02", + "SECURITY.md": "a3335079977c2f13bad59e323fdc1056bdae5adfe55f18d15ac2c930d741828c", + "clippy.toml": "583b79f56f20c85fb7820fcfa97985e03a4b41da31c8bbe6c03b721d3df5e800", + "benches/buf.rs": "72e6b6120b52d568da068f17c66a793d65602e400c595778581b63092e41d8dc", + "benches/bytes.rs": "7084e564f8568f52c9fdb76a06e58701aa7f0c776209d29a0f60c38a19748203", + "benches/bytes_mut.rs": "1326fe6224b26826228e02b4133151e756f38152c2d9cfe66adf83af76c3ec98", + "ci/miri.sh": "e74475e7aec647a490a1f491b5dac6a60e3c03926a7c7e1898fa3e7c626415c7", + "ci/panic-abort.sh": "aec2e5427117e533d7a9721a926eb6a9d72f264510066baf2c501357df2d936f", + "ci/test-stable.sh": "fb751f06f314f728c489b7a7854701f632e24de8d0540ea8cf3449c6a853ac3c", + "ci/tsan.sh": "466b86b19225dd26c756cf2252cb1973f87a145642c99364b462ed7ceb55c7dd", + "src/bytes.rs": "54568a31252a7cc4f1dcee26d2d34dc6e0af9c9d8bf09522b694c3bd6a2a04bc", + "src/bytes_mut.rs": "aa9ce99a8c58100341f3fd33ee33c821fa89ebc3b7e98190ebe66f7632b4f6c2", + "src/lib.rs": "037b377f32c52ea6cf5ebd0135ff80a6f8c4b25e663aa5a9786c6310c2f91e46", + "src/loom.rs": "7c89f9f5f900d5b9b7c43d08d467957933eba90d5dcff1b38cf9a7e48494a577", + "src/serde.rs": "3ecd7e828cd4c2b7db93c807cb1548fad209e674df493edf7cda69a7b04d405d", + "tests/test_buf.rs": "c55dc7ab1a7392c2467a079dc77223fbec35ff54e9eda9b05cc1d4e22cb36626", + "tests/test_buf_mut.rs": "1f28f3d125e98b81a813a414a89a8c9b5d45369b10dda187a68614ff285c820e", + "tests/test_bytes.rs": "57f461b6a05cfe58dff0c5d5eb46277557274eb4bedfecb9aa5d295963e0c9f4", + "tests/test_bytes_odd_alloc.rs": "ad5df84a35038359413199762c65017989d905e6e965a7b863beaa9962a7e1e8", + "tests/test_bytes_vec_alloc.rs": "5a0e77cab12b32374b2822df8d9a6479afe68b55c1cba1511ebf4199be532caa", + "tests/test_chain.rs": "1784287560d45e0225b2cd9d24498264031c36f42cebb7e92108e11641d1cce0", + "tests/test_debug.rs": "13299107172809e8cbbd823964ac9450cd0d6b6de79f2e6a2e0f44b9225a0593", + "tests/test_iter.rs": "665e1837c1294d7695ffa352a666ce8673fb46fa603164556524e87d5517a6cb", + "tests/test_limit.rs": "6c5801f1e94f0abb2b4994aa1ade8e36464f26bf9e13e3dcac02d0f2f6f08ed5", + "tests/test_reader.rs": "050b872d89376b3b83841839b4334066d8357b01aaf3783a180aac25a5550f3d", + "tests/test_serde.rs": "2691f891796ba259de0ecf926de05c514f4912cc5fcd3e6a1591efbcd23ed4d0", + "tests/test_take.rs": "fb34700a8ed75b3c6937a4dee0bffadb64f7c7f735bd9faa8730eb96a9e9b651", + "src/buf/buf_impl.rs": "a14aa7e5e7fac1d4d1a3617d616af0b478a9a31235b773bcec11255c8c78446e", + "src/buf/buf_mut.rs": "c76d0a846cf6e2c42af7acf0bbedc49a30ce3b9185392835c6a6f8b06a128b9e", + "src/buf/chain.rs": "27ff0a6fce049c51a24d45003b9741b655c28330b48af23ad1514587760b0c93", + "src/buf/iter.rs": "d4dca5b7f9b1cb441f22ac1862e28b10086721879163a810955aefb5cd7f3e58", + "src/buf/limit.rs": "e005ba140b70f68654877c96b981a220477e415ff5c92438c1b0cb9bc866d872", + "src/buf/mod.rs": "3f60295316d44b510b942abb31a0d975ae488bd4b52c87f5252d73f88f82715a", + "src/buf/reader.rs": "cda8bc221a1de06c7395d5c6e80f8a5924198eafbc2decc0909082ce8781d789", + "src/buf/take.rs": "443766677fae38b398271693ed6134c7cc14949cdb9fce299c44dd55ecf84fb3", + "src/buf/uninit_slice.rs": "52629b93ff7a08db45fc69395580b34fa3609fd9309ea01f86e58488b02b08a3", + "src/buf/vec_deque.rs": "baac165527d1bdfc3bb259380bb4722e70c3f11e8a901e8bab6e4f951b060711", + "src/buf/writer.rs": "b137608cb40944a5c3e061a73087c5a48e50048dae002d0334a23cf4d3547065", + "src/fmt/debug.rs": "4d217b8f43e489925bb6904010df617efc63b50a3cc1ab126eec54b43b66eb3e", + "src/fmt/hex.rs": "54c05ab7d95d7381e499079d63773fabd2672f3c4929b56b02ab87d4495902b5", + "src/fmt/mod.rs": "b38222dcfff70eb3ffb6ce305b95f3fcfd9fa68cca85693eb2ed01d3da7ddb88", + ".github/workflows/ci.yml": "1e635153dbcbee2afb3fd51a83340439f07f223b179e7cde28a68a8ee7385008" + } +} diff --git a/third-party/bytes/src/buf/buf_impl.rs b/third-party/bytes/src/buf/buf_impl.rs new file mode 100644 index 000000000..192034fbe --- /dev/null +++ b/third-party/bytes/src/buf/buf_impl.rs @@ -0,0 +1,2962 @@ +#[cfg(feature = "std")] +use crate::buf::{reader, Reader}; +use crate::buf::{take, Chain, Take}; +#[cfg(feature = "std")] +use crate::{min_u64_usize, saturating_sub_usize_u64}; +use crate::{panic_advance, panic_does_not_fit, TryGetError}; + +#[cfg(feature = "std")] +use std::io::IoSlice; + +use alloc::boxed::Box; + +macro_rules! buf_try_get_impl { + ($this:ident, $typ:tt::$conv:tt) => {{ + const SIZE: usize = core::mem::size_of::<$typ>(); + + if $this.remaining() < SIZE { + return Err(TryGetError { + requested: SIZE, + available: $this.remaining(), + }); + } + + // try to convert directly from the bytes + // this Option<ret> trick is to avoid keeping a borrow on self + // when advance() is called (mut borrow) and to call bytes() only once + let ret = $this + .chunk() + .get(..SIZE) + .map(|src| unsafe { $typ::$conv(*(src as *const _ as *const [_; SIZE])) }); + + if let Some(ret) = ret { + // if the direct conversion was possible, advance and return + $this.advance(SIZE); + return Ok(ret); + } else { + // if not we copy the bytes in a temp buffer then convert + let mut buf = [0; SIZE]; + $this.copy_to_slice(&mut buf); // (do the advance) + return Ok($typ::$conv(buf)); + } + }}; + (le => $this:ident, $typ:tt, $len_to_read:expr) => {{ + const SIZE: usize = core::mem::size_of::<$typ>(); + + // The same trick as above does not improve the best case speed. + // It seems to be linked to the way the method is optimised by the compiler + let mut buf = [0; SIZE]; + + let subslice = match buf.get_mut(..$len_to_read) { + Some(subslice) => subslice, + None => panic_does_not_fit(SIZE, $len_to_read), + }; + + $this.try_copy_to_slice(subslice)?; + return Ok($typ::from_le_bytes(buf)); + }}; + (be => $this:ident, $typ:tt, $len_to_read:expr) => {{ + const SIZE: usize = core::mem::size_of::<$typ>(); + + let slice_at = match SIZE.checked_sub($len_to_read) { + Some(slice_at) => slice_at, + None => panic_does_not_fit(SIZE, $len_to_read), + }; + + let mut buf = [0; SIZE]; + $this.try_copy_to_slice(&mut buf[slice_at..])?; + return Ok($typ::from_be_bytes(buf)); + }}; +} + +macro_rules! buf_get_impl { + ($this:ident, $typ:tt::$conv:tt) => {{ + return (|| buf_try_get_impl!($this, $typ::$conv))() + .unwrap_or_else(|error| panic_advance(&error)); + }}; + (le => $this:ident, $typ:tt, $len_to_read:expr) => {{ + return (|| buf_try_get_impl!(le => $this, $typ, $len_to_read))() + .unwrap_or_else(|error| panic_advance(&error)); + }}; + (be => $this:ident, $typ:tt, $len_to_read:expr) => {{ + return (|| buf_try_get_impl!(be => $this, $typ, $len_to_read))() + .unwrap_or_else(|error| panic_advance(&error)); + }}; +} + +// https://en.wikipedia.org/wiki/Sign_extension +fn sign_extend(val: u64, nbytes: usize) -> i64 { + let shift = (8 - nbytes) * 8; + (val << shift) as i64 >> shift +} + +/// Read bytes from a buffer. +/// +/// A buffer stores bytes in memory such that read operations are infallible. +/// The underlying storage may or may not be in contiguous memory. A `Buf` value +/// is a cursor into the buffer. Reading from `Buf` advances the cursor +/// position. It can be thought of as an efficient `Iterator` for collections of +/// bytes. +/// +/// The simplest `Buf` is a `&[u8]`. +/// +/// ``` +/// use bytes::Buf; +/// +/// let mut buf = &b"hello world"[..]; +/// +/// assert_eq!(b'h', buf.get_u8()); +/// assert_eq!(b'e', buf.get_u8()); +/// assert_eq!(b'l', buf.get_u8()); +/// +/// let mut rest = [0; 8]; +/// buf.copy_to_slice(&mut rest); +/// +/// assert_eq!(&rest[..], &b"lo world"[..]); +/// ``` +pub trait Buf { + /// Returns the number of bytes between the current position and the end of + /// the buffer. + /// + /// This value is greater than or equal to the length of the slice returned + /// by `chunk()`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"hello world"[..]; + /// + /// assert_eq!(buf.remaining(), 11); + /// + /// buf.get_u8(); + /// + /// assert_eq!(buf.remaining(), 10); + /// ``` + /// + /// # Implementer notes + /// + /// Implementations of `remaining` should ensure that the return value does + /// not change unless a call is made to `advance` or any other function that + /// is documented to change the `Buf`'s current position. + fn remaining(&self) -> usize; + + /// Returns a slice starting at the current position and of length between 0 + /// and `Buf::remaining()`. Note that this *can* return a shorter slice (this + /// allows non-continuous internal representation). + /// + /// This is a lower level function. Most operations are done with other + /// functions. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"hello world"[..]; + /// + /// assert_eq!(buf.chunk(), &b"hello world"[..]); + /// + /// buf.advance(6); + /// + /// assert_eq!(buf.chunk(), &b"world"[..]); + /// ``` + /// + /// # Implementer notes + /// + /// This function should never panic. `chunk()` should return an empty + /// slice **if and only if** `remaining()` returns 0. In other words, + /// `chunk()` returning an empty slice implies that `remaining()` will + /// return 0 and `remaining()` returning 0 implies that `chunk()` will + /// return an empty slice. + // The `chunk` method was previously called `bytes`. This alias makes the rename + // more easily discoverable. + #[cfg_attr(docsrs, doc(alias = "bytes"))] + fn chunk(&self) -> &[u8]; + + /// Fills `dst` with potentially multiple slices starting at `self`'s + /// current position. + /// + /// If the `Buf` is backed by disjoint slices of bytes, `chunk_vectored` enables + /// fetching more than one slice at once. `dst` is a slice of `IoSlice` + /// references, enabling the slice to be directly used with [`writev`] + /// without any further conversion. The sum of the lengths of all the + /// buffers written to `dst` will be less than or equal to `Buf::remaining()`. + /// + /// The entries in `dst` will be overwritten, but the data **contained** by + /// the slices **will not** be modified. The return value is the number of + /// slices written to `dst`. If `Buf::remaining()` is non-zero, then this + /// writes at least one non-empty slice to `dst`. + /// + /// This is a lower level function. Most operations are done with other + /// functions. + /// + /// # Implementer notes + /// + /// This function should never panic. Once the end of the buffer is reached, + /// i.e., `Buf::remaining` returns 0, calls to `chunk_vectored` must return 0 + /// without mutating `dst`. + /// + /// Implementations should also take care to properly handle being called + /// with `dst` being a zero length slice. + /// + /// [`writev`]: http://man7.org/linux/man-pages/man2/readv.2.html + #[cfg(feature = "std")] + #[cfg_attr(docsrs, doc(cfg(feature = "std")))] + fn chunks_vectored<'a>(&'a self, dst: &mut [IoSlice<'a>]) -> usize { + if dst.is_empty() { + return 0; + } + + if self.has_remaining() { + dst[0] = IoSlice::new(self.chunk()); + 1 + } else { + 0 + } + } + + /// Advance the internal cursor of the Buf + /// + /// The next call to `chunk()` will return a slice starting `cnt` bytes + /// further into the underlying buffer. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"hello world"[..]; + /// + /// assert_eq!(buf.chunk(), &b"hello world"[..]); + /// + /// buf.advance(6); + /// + /// assert_eq!(buf.chunk(), &b"world"[..]); + /// ``` + /// + /// # Panics + /// + /// This function **may** panic if `cnt > self.remaining()`. + /// + /// # Implementer notes + /// + /// It is recommended for implementations of `advance` to panic if `cnt > + /// self.remaining()`. If the implementation does not panic, the call must + /// behave as if `cnt == self.remaining()`. + /// + /// A call with `cnt == 0` should never panic and be a no-op. + fn advance(&mut self, cnt: usize); + + /// Returns true if there are any more bytes to consume + /// + /// This is equivalent to `self.remaining() != 0`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"a"[..]; + /// + /// assert!(buf.has_remaining()); + /// + /// buf.get_u8(); + /// + /// assert!(!buf.has_remaining()); + /// ``` + fn has_remaining(&self) -> bool { + self.remaining() > 0 + } + + /// Copies bytes from `self` into `dst`. + /// + /// The cursor is advanced by the number of bytes copied. `self` must have + /// enough remaining bytes to fill `dst`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"hello world"[..]; + /// let mut dst = [0; 5]; + /// + /// buf.copy_to_slice(&mut dst); + /// assert_eq!(&b"hello"[..], &dst); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// # Panics + /// + /// This function panics if `self.remaining() < dst.len()`. + fn copy_to_slice(&mut self, dst: &mut [u8]) { + self.try_copy_to_slice(dst) + .unwrap_or_else(|error| panic_advance(&error)); + } + + /// Gets an unsigned 8 bit integer from `self`. + /// + /// The current position is advanced by 1. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08 hello"[..]; + /// assert_eq!(8, buf.get_u8()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is no more remaining data in `self`. + fn get_u8(&mut self) -> u8 { + if self.remaining() < 1 { + panic_advance(&TryGetError { + requested: 1, + available: 0, + }) + } + let ret = self.chunk()[0]; + self.advance(1); + ret + } + + /// Gets a signed 8 bit integer from `self`. + /// + /// The current position is advanced by 1. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08 hello"[..]; + /// assert_eq!(8, buf.get_i8()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is no more remaining data in `self`. + fn get_i8(&mut self) -> i8 { + if self.remaining() < 1 { + panic_advance(&TryGetError { + requested: 1, + available: 0, + }); + } + let ret = self.chunk()[0] as i8; + self.advance(1); + ret + } + + /// Gets an unsigned 16 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x09 hello"[..]; + /// assert_eq!(0x0809, buf.get_u16()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u16(&mut self) -> u16 { + buf_get_impl!(self, u16::from_be_bytes); + } + + /// Gets an unsigned 16 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x09\x08 hello"[..]; + /// assert_eq!(0x0809, buf.get_u16_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u16_le(&mut self) -> u16 { + buf_get_impl!(self, u16::from_le_bytes); + } + + /// Gets an unsigned 16 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x08\x09 hello", + /// false => b"\x09\x08 hello", + /// }; + /// assert_eq!(0x0809, buf.get_u16_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u16_ne(&mut self) -> u16 { + buf_get_impl!(self, u16::from_ne_bytes); + } + + /// Gets a signed 16 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x09 hello"[..]; + /// assert_eq!(0x0809, buf.get_i16()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i16(&mut self) -> i16 { + buf_get_impl!(self, i16::from_be_bytes); + } + + /// Gets a signed 16 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x09\x08 hello"[..]; + /// assert_eq!(0x0809, buf.get_i16_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i16_le(&mut self) -> i16 { + buf_get_impl!(self, i16::from_le_bytes); + } + + /// Gets a signed 16 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x08\x09 hello", + /// false => b"\x09\x08 hello", + /// }; + /// assert_eq!(0x0809, buf.get_i16_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i16_ne(&mut self) -> i16 { + buf_get_impl!(self, i16::from_ne_bytes); + } + + /// Gets an unsigned 32 bit integer from `self` in the big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x09\xA0\xA1 hello"[..]; + /// assert_eq!(0x0809A0A1, buf.get_u32()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u32(&mut self) -> u32 { + buf_get_impl!(self, u32::from_be_bytes); + } + + /// Gets an unsigned 32 bit integer from `self` in the little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\xA1\xA0\x09\x08 hello"[..]; + /// assert_eq!(0x0809A0A1, buf.get_u32_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u32_le(&mut self) -> u32 { + buf_get_impl!(self, u32::from_le_bytes); + } + + /// Gets an unsigned 32 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x08\x09\xA0\xA1 hello", + /// false => b"\xA1\xA0\x09\x08 hello", + /// }; + /// assert_eq!(0x0809A0A1, buf.get_u32_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u32_ne(&mut self) -> u32 { + buf_get_impl!(self, u32::from_ne_bytes); + } + + /// Gets a signed 32 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x09\xA0\xA1 hello"[..]; + /// assert_eq!(0x0809A0A1, buf.get_i32()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i32(&mut self) -> i32 { + buf_get_impl!(self, i32::from_be_bytes); + } + + /// Gets a signed 32 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\xA1\xA0\x09\x08 hello"[..]; + /// assert_eq!(0x0809A0A1, buf.get_i32_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i32_le(&mut self) -> i32 { + buf_get_impl!(self, i32::from_le_bytes); + } + + /// Gets a signed 32 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x08\x09\xA0\xA1 hello", + /// false => b"\xA1\xA0\x09\x08 hello", + /// }; + /// assert_eq!(0x0809A0A1, buf.get_i32_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i32_ne(&mut self) -> i32 { + buf_get_impl!(self, i32::from_ne_bytes); + } + + /// Gets an unsigned 64 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08 hello"[..]; + /// assert_eq!(0x0102030405060708, buf.get_u64()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u64(&mut self) -> u64 { + buf_get_impl!(self, u64::from_be_bytes); + } + + /// Gets an unsigned 64 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x07\x06\x05\x04\x03\x02\x01 hello"[..]; + /// assert_eq!(0x0102030405060708, buf.get_u64_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u64_le(&mut self) -> u64 { + buf_get_impl!(self, u64::from_le_bytes); + } + + /// Gets an unsigned 64 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03\x04\x05\x06\x07\x08 hello", + /// false => b"\x08\x07\x06\x05\x04\x03\x02\x01 hello", + /// }; + /// assert_eq!(0x0102030405060708, buf.get_u64_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u64_ne(&mut self) -> u64 { + buf_get_impl!(self, u64::from_ne_bytes); + } + + /// Gets a signed 64 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08 hello"[..]; + /// assert_eq!(0x0102030405060708, buf.get_i64()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i64(&mut self) -> i64 { + buf_get_impl!(self, i64::from_be_bytes); + } + + /// Gets a signed 64 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x07\x06\x05\x04\x03\x02\x01 hello"[..]; + /// assert_eq!(0x0102030405060708, buf.get_i64_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i64_le(&mut self) -> i64 { + buf_get_impl!(self, i64::from_le_bytes); + } + + /// Gets a signed 64 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03\x04\x05\x06\x07\x08 hello", + /// false => b"\x08\x07\x06\x05\x04\x03\x02\x01 hello", + /// }; + /// assert_eq!(0x0102030405060708, buf.get_i64_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i64_ne(&mut self) -> i64 { + buf_get_impl!(self, i64::from_ne_bytes); + } + + /// Gets an unsigned 128 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16 hello"[..]; + /// assert_eq!(0x01020304050607080910111213141516, buf.get_u128()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u128(&mut self) -> u128 { + buf_get_impl!(self, u128::from_be_bytes); + } + + /// Gets an unsigned 128 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01 hello"[..]; + /// assert_eq!(0x01020304050607080910111213141516, buf.get_u128_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u128_le(&mut self) -> u128 { + buf_get_impl!(self, u128::from_le_bytes); + } + + /// Gets an unsigned 128 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16 hello", + /// false => b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01 hello", + /// }; + /// assert_eq!(0x01020304050607080910111213141516, buf.get_u128_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_u128_ne(&mut self) -> u128 { + buf_get_impl!(self, u128::from_ne_bytes); + } + + /// Gets a signed 128 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16 hello"[..]; + /// assert_eq!(0x01020304050607080910111213141516, buf.get_i128()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i128(&mut self) -> i128 { + buf_get_impl!(self, i128::from_be_bytes); + } + + /// Gets a signed 128 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01 hello"[..]; + /// assert_eq!(0x01020304050607080910111213141516, buf.get_i128_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i128_le(&mut self) -> i128 { + buf_get_impl!(self, i128::from_le_bytes); + } + + /// Gets a signed 128 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16 hello", + /// false => b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01 hello", + /// }; + /// assert_eq!(0x01020304050607080910111213141516, buf.get_i128_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_i128_ne(&mut self) -> i128 { + buf_get_impl!(self, i128::from_ne_bytes); + } + + /// Gets an unsigned n-byte integer from `self` in big-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03 hello"[..]; + /// assert_eq!(0x010203, buf.get_uint(3)); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`, or + /// if `nbytes` is greater than 8. + fn get_uint(&mut self, nbytes: usize) -> u64 { + buf_get_impl!(be => self, u64, nbytes); + } + + /// Gets an unsigned n-byte integer from `self` in little-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x03\x02\x01 hello"[..]; + /// assert_eq!(0x010203, buf.get_uint_le(3)); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`, or + /// if `nbytes` is greater than 8. + fn get_uint_le(&mut self, nbytes: usize) -> u64 { + buf_get_impl!(le => self, u64, nbytes); + } + + /// Gets an unsigned n-byte integer from `self` in native-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03 hello", + /// false => b"\x03\x02\x01 hello", + /// }; + /// assert_eq!(0x010203, buf.get_uint_ne(3)); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`, or + /// if `nbytes` is greater than 8. + fn get_uint_ne(&mut self, nbytes: usize) -> u64 { + if cfg!(target_endian = "big") { + self.get_uint(nbytes) + } else { + self.get_uint_le(nbytes) + } + } + + /// Gets a signed n-byte integer from `self` in big-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03 hello"[..]; + /// assert_eq!(0x010203, buf.get_int(3)); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`, or + /// if `nbytes` is greater than 8. + fn get_int(&mut self, nbytes: usize) -> i64 { + sign_extend(self.get_uint(nbytes), nbytes) + } + + /// Gets a signed n-byte integer from `self` in little-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x03\x02\x01 hello"[..]; + /// assert_eq!(0x010203, buf.get_int_le(3)); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`, or + /// if `nbytes` is greater than 8. + fn get_int_le(&mut self, nbytes: usize) -> i64 { + sign_extend(self.get_uint_le(nbytes), nbytes) + } + + /// Gets a signed n-byte integer from `self` in native-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03 hello", + /// false => b"\x03\x02\x01 hello", + /// }; + /// assert_eq!(0x010203, buf.get_int_ne(3)); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`, or + /// if `nbytes` is greater than 8. + fn get_int_ne(&mut self, nbytes: usize) -> i64 { + if cfg!(target_endian = "big") { + self.get_int(nbytes) + } else { + self.get_int_le(nbytes) + } + } + + /// Gets an IEEE754 single-precision (4 bytes) floating point number from + /// `self` in big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x3F\x99\x99\x9A hello"[..]; + /// assert_eq!(1.2f32, buf.get_f32()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_f32(&mut self) -> f32 { + f32::from_bits(self.get_u32()) + } + + /// Gets an IEEE754 single-precision (4 bytes) floating point number from + /// `self` in little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x9A\x99\x99\x3F hello"[..]; + /// assert_eq!(1.2f32, buf.get_f32_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_f32_le(&mut self) -> f32 { + f32::from_bits(self.get_u32_le()) + } + + /// Gets an IEEE754 single-precision (4 bytes) floating point number from + /// `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x3F\x99\x99\x9A hello", + /// false => b"\x9A\x99\x99\x3F hello", + /// }; + /// assert_eq!(1.2f32, buf.get_f32_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_f32_ne(&mut self) -> f32 { + f32::from_bits(self.get_u32_ne()) + } + + /// Gets an IEEE754 double-precision (8 bytes) floating point number from + /// `self` in big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x3F\xF3\x33\x33\x33\x33\x33\x33 hello"[..]; + /// assert_eq!(1.2f64, buf.get_f64()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_f64(&mut self) -> f64 { + f64::from_bits(self.get_u64()) + } + + /// Gets an IEEE754 double-precision (8 bytes) floating point number from + /// `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x33\x33\x33\x33\x33\x33\xF3\x3F hello"[..]; + /// assert_eq!(1.2f64, buf.get_f64_le()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_f64_le(&mut self) -> f64 { + f64::from_bits(self.get_u64_le()) + } + + /// Gets an IEEE754 double-precision (8 bytes) floating point number from + /// `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x3F\xF3\x33\x33\x33\x33\x33\x33 hello", + /// false => b"\x33\x33\x33\x33\x33\x33\xF3\x3F hello", + /// }; + /// assert_eq!(1.2f64, buf.get_f64_ne()); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining data in `self`. + fn get_f64_ne(&mut self) -> f64 { + f64::from_bits(self.get_u64_ne()) + } + + /// Copies bytes from `self` into `dst`. + /// + /// The cursor is advanced by the number of bytes copied. `self` must have + /// enough remaining bytes to fill `dst`. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"hello world"[..]; + /// let mut dst = [0; 5]; + /// + /// assert_eq!(Ok(()), buf.try_copy_to_slice(&mut dst)); + /// assert_eq!(&b"hello"[..], &dst); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"hello world"[..]; + /// let mut dst = [0; 12]; + /// + /// assert_eq!(Err(TryGetError{requested: 12, available: 11}), buf.try_copy_to_slice(&mut dst)); + /// assert_eq!(11, buf.remaining()); + /// ``` + fn try_copy_to_slice(&mut self, mut dst: &mut [u8]) -> Result<(), TryGetError> { + if self.remaining() < dst.len() { + return Err(TryGetError { + requested: dst.len(), + available: self.remaining(), + }); + } + + while !dst.is_empty() { + let src = self.chunk(); + let cnt = usize::min(src.len(), dst.len()); + + dst[..cnt].copy_from_slice(&src[..cnt]); + dst = &mut dst[cnt..]; + + self.advance(cnt); + } + Ok(()) + } + + /// Gets an unsigned 8 bit integer from `self`. + /// + /// The current position is advanced by 1. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08 hello"[..]; + /// assert_eq!(Ok(0x08_u8), buf.try_get_u8()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b""[..]; + /// assert_eq!(Err(TryGetError{requested: 1, available: 0}), buf.try_get_u8()); + /// ``` + fn try_get_u8(&mut self) -> Result<u8, TryGetError> { + if self.remaining() < 1 { + return Err(TryGetError { + requested: 1, + available: self.remaining(), + }); + } + let ret = self.chunk()[0]; + self.advance(1); + Ok(ret) + } + + /// Gets a signed 8 bit integer from `self`. + /// + /// The current position is advanced by 1. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08 hello"[..]; + /// assert_eq!(Ok(0x08_i8), buf.try_get_i8()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b""[..]; + /// assert_eq!(Err(TryGetError{requested: 1, available: 0}), buf.try_get_i8()); + /// ``` + fn try_get_i8(&mut self) -> Result<i8, TryGetError> { + if self.remaining() < 1 { + return Err(TryGetError { + requested: 1, + available: self.remaining(), + }); + } + let ret = self.chunk()[0] as i8; + self.advance(1); + Ok(ret) + } + + /// Gets an unsigned 16 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x09 hello"[..]; + /// assert_eq!(Ok(0x0809_u16), buf.try_get_u16()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08"[..]; + /// assert_eq!(Err(TryGetError{requested: 2, available: 1}), buf.try_get_u16()); + /// assert_eq!(1, buf.remaining()); + /// ``` + fn try_get_u16(&mut self) -> Result<u16, TryGetError> { + buf_try_get_impl!(self, u16::from_be_bytes) + } + + /// Gets an unsigned 16 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x09\x08 hello"[..]; + /// assert_eq!(Ok(0x0809_u16), buf.try_get_u16_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08"[..]; + /// assert_eq!(Err(TryGetError{requested: 2, available: 1}), buf.try_get_u16_le()); + /// assert_eq!(1, buf.remaining()); + /// ``` + fn try_get_u16_le(&mut self) -> Result<u16, TryGetError> { + buf_try_get_impl!(self, u16::from_le_bytes) + } + + /// Gets an unsigned 16 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x08\x09 hello", + /// false => b"\x09\x08 hello", + /// }; + /// assert_eq!(Ok(0x0809_u16), buf.try_get_u16_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08"[..]; + /// assert_eq!(Err(TryGetError{requested: 2, available: 1}), buf.try_get_u16_ne()); + /// assert_eq!(1, buf.remaining()); + /// ``` + fn try_get_u16_ne(&mut self) -> Result<u16, TryGetError> { + buf_try_get_impl!(self, u16::from_ne_bytes) + } + + /// Gets a signed 16 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x09 hello"[..]; + /// assert_eq!(Ok(0x0809_i16), buf.try_get_i16()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08"[..]; + /// assert_eq!(Err(TryGetError{requested: 2, available: 1}), buf.try_get_i16()); + /// assert_eq!(1, buf.remaining()); + /// ``` + fn try_get_i16(&mut self) -> Result<i16, TryGetError> { + buf_try_get_impl!(self, i16::from_be_bytes) + } + + /// Gets an signed 16 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x09\x08 hello"[..]; + /// assert_eq!(Ok(0x0809_i16), buf.try_get_i16_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08"[..]; + /// assert_eq!(Err(TryGetError{requested: 2, available: 1}), buf.try_get_i16_le()); + /// assert_eq!(1, buf.remaining()); + /// ``` + fn try_get_i16_le(&mut self) -> Result<i16, TryGetError> { + buf_try_get_impl!(self, i16::from_le_bytes) + } + + /// Gets a signed 16 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x08\x09 hello", + /// false => b"\x09\x08 hello", + /// }; + /// assert_eq!(Ok(0x0809_i16), buf.try_get_i16_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08"[..]; + /// assert_eq!(Err(TryGetError{requested: 2, available: 1}), buf.try_get_i16_ne()); + /// assert_eq!(1, buf.remaining()); + /// ``` + fn try_get_i16_ne(&mut self) -> Result<i16, TryGetError> { + buf_try_get_impl!(self, i16::from_ne_bytes) + } + + /// Gets an unsigned 32 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x09\xA0\xA1 hello"[..]; + /// assert_eq!(Ok(0x0809A0A1), buf.try_get_u32()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_u32()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_u32(&mut self) -> Result<u32, TryGetError> { + buf_try_get_impl!(self, u32::from_be_bytes) + } + + /// Gets an unsigned 32 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\xA1\xA0\x09\x08 hello"[..]; + /// assert_eq!(Ok(0x0809A0A1_u32), buf.try_get_u32_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08\x09\xA0"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_u32_le()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_u32_le(&mut self) -> Result<u32, TryGetError> { + buf_try_get_impl!(self, u32::from_le_bytes) + } + + /// Gets an unsigned 32 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x08\x09\xA0\xA1 hello", + /// false => b"\xA1\xA0\x09\x08 hello", + /// }; + /// assert_eq!(Ok(0x0809A0A1_u32), buf.try_get_u32_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08\x09\xA0"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_u32_ne()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_u32_ne(&mut self) -> Result<u32, TryGetError> { + buf_try_get_impl!(self, u32::from_ne_bytes) + } + + /// Gets a signed 32 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x09\xA0\xA1 hello"[..]; + /// assert_eq!(Ok(0x0809A0A1_i32), buf.try_get_i32()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_i32()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_i32(&mut self) -> Result<i32, TryGetError> { + buf_try_get_impl!(self, i32::from_be_bytes) + } + + /// Gets a signed 32 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\xA1\xA0\x09\x08 hello"[..]; + /// assert_eq!(Ok(0x0809A0A1_i32), buf.try_get_i32_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08\x09\xA0"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_i32_le()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_i32_le(&mut self) -> Result<i32, TryGetError> { + buf_try_get_impl!(self, i32::from_le_bytes) + } + + /// Gets a signed 32 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x08\x09\xA0\xA1 hello", + /// false => b"\xA1\xA0\x09\x08 hello", + /// }; + /// assert_eq!(Ok(0x0809A0A1_i32), buf.try_get_i32_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08\x09\xA0"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_i32_ne()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_i32_ne(&mut self) -> Result<i32, TryGetError> { + buf_try_get_impl!(self, i32::from_ne_bytes) + } + + /// Gets an unsigned 64 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08 hello"[..]; + /// assert_eq!(Ok(0x0102030405060708_u64), buf.try_get_u64()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_u64()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_u64(&mut self) -> Result<u64, TryGetError> { + buf_try_get_impl!(self, u64::from_be_bytes) + } + + /// Gets an unsigned 64 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x07\x06\x05\x04\x03\x02\x01 hello"[..]; + /// assert_eq!(Ok(0x0102030405060708_u64), buf.try_get_u64_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08\x07\x06\x05\x04\x03\x02"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_u64_le()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_u64_le(&mut self) -> Result<u64, TryGetError> { + buf_try_get_impl!(self, u64::from_le_bytes) + } + + /// Gets an unsigned 64 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03\x04\x05\x06\x07\x08 hello", + /// false => b"\x08\x07\x06\x05\x04\x03\x02\x01 hello", + /// }; + /// assert_eq!(Ok(0x0102030405060708_u64), buf.try_get_u64_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_u64_ne()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_u64_ne(&mut self) -> Result<u64, TryGetError> { + buf_try_get_impl!(self, u64::from_ne_bytes) + } + + /// Gets a signed 64 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08 hello"[..]; + /// assert_eq!(Ok(0x0102030405060708_i64), buf.try_get_i64()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_i64()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_i64(&mut self) -> Result<i64, TryGetError> { + buf_try_get_impl!(self, i64::from_be_bytes) + } + + /// Gets a signed 64 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x08\x07\x06\x05\x04\x03\x02\x01 hello"[..]; + /// assert_eq!(Ok(0x0102030405060708_i64), buf.try_get_i64_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x08\x07\x06\x05\x04\x03\x02"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_i64_le()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_i64_le(&mut self) -> Result<i64, TryGetError> { + buf_try_get_impl!(self, i64::from_le_bytes) + } + + /// Gets a signed 64 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03\x04\x05\x06\x07\x08 hello", + /// false => b"\x08\x07\x06\x05\x04\x03\x02\x01 hello", + /// }; + /// assert_eq!(Ok(0x0102030405060708_i64), buf.try_get_i64_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_i64_ne()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_i64_ne(&mut self) -> Result<i64, TryGetError> { + buf_try_get_impl!(self, i64::from_ne_bytes) + } + + /// Gets an unsigned 128 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16 hello"[..]; + /// assert_eq!(Ok(0x01020304050607080910111213141516_u128), buf.try_get_u128()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15"[..]; + /// assert_eq!(Err(TryGetError{requested: 16, available: 15}), buf.try_get_u128()); + /// assert_eq!(15, buf.remaining()); + /// ``` + fn try_get_u128(&mut self) -> Result<u128, TryGetError> { + buf_try_get_impl!(self, u128::from_be_bytes) + } + + /// Gets an unsigned 128 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01 hello"[..]; + /// assert_eq!(Ok(0x01020304050607080910111213141516_u128), buf.try_get_u128_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02"[..]; + /// assert_eq!(Err(TryGetError{requested: 16, available: 15}), buf.try_get_u128_le()); + /// assert_eq!(15, buf.remaining()); + /// ``` + fn try_get_u128_le(&mut self) -> Result<u128, TryGetError> { + buf_try_get_impl!(self, u128::from_le_bytes) + } + + /// Gets an unsigned 128 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16 hello", + /// false => b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01 hello", + /// }; + /// assert_eq!(Ok(0x01020304050607080910111213141516_u128), buf.try_get_u128_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15"[..]; + /// assert_eq!(Err(TryGetError{requested: 16, available: 15}), buf.try_get_u128_ne()); + /// assert_eq!(15, buf.remaining()); + /// ``` + fn try_get_u128_ne(&mut self) -> Result<u128, TryGetError> { + buf_try_get_impl!(self, u128::from_ne_bytes) + } + + /// Gets a signed 128 bit integer from `self` in big-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16 hello"[..]; + /// assert_eq!(Ok(0x01020304050607080910111213141516_i128), buf.try_get_i128()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15"[..]; + /// assert_eq!(Err(TryGetError{requested: 16, available: 15}), buf.try_get_i128()); + /// assert_eq!(15, buf.remaining()); + /// ``` + fn try_get_i128(&mut self) -> Result<i128, TryGetError> { + buf_try_get_impl!(self, i128::from_be_bytes) + } + + /// Gets a signed 128 bit integer from `self` in little-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01 hello"[..]; + /// assert_eq!(Ok(0x01020304050607080910111213141516_i128), buf.try_get_i128_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02"[..]; + /// assert_eq!(Err(TryGetError{requested: 16, available: 15}), buf.try_get_i128_le()); + /// assert_eq!(15, buf.remaining()); + /// ``` + fn try_get_i128_le(&mut self) -> Result<i128, TryGetError> { + buf_try_get_impl!(self, i128::from_le_bytes) + } + + /// Gets a signed 128 bit integer from `self` in native-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16 hello", + /// false => b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01 hello", + /// }; + /// assert_eq!(Ok(0x01020304050607080910111213141516_i128), buf.try_get_i128_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15"[..]; + /// assert_eq!(Err(TryGetError{requested: 16, available: 15}), buf.try_get_i128_ne()); + /// assert_eq!(15, buf.remaining()); + /// ``` + fn try_get_i128_ne(&mut self) -> Result<i128, TryGetError> { + buf_try_get_impl!(self, i128::from_ne_bytes) + } + + /// Gets an unsigned n-byte integer from `self` in big-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03 hello"[..]; + /// assert_eq!(Ok(0x010203_u64), buf.try_get_uint(3)); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_uint(4)); + /// assert_eq!(3, buf.remaining()); + /// ``` + /// + /// # Panics + /// + /// This function panics if `nbytes` > 8. + fn try_get_uint(&mut self, nbytes: usize) -> Result<u64, TryGetError> { + buf_try_get_impl!(be => self, u64, nbytes); + } + + /// Gets an unsigned n-byte integer from `self` in little-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x03\x02\x01 hello"[..]; + /// assert_eq!(Ok(0x010203_u64), buf.try_get_uint_le(3)); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_uint_le(4)); + /// assert_eq!(3, buf.remaining()); + /// ``` + /// + /// # Panics + /// + /// This function panics if `nbytes` > 8. + fn try_get_uint_le(&mut self, nbytes: usize) -> Result<u64, TryGetError> { + buf_try_get_impl!(le => self, u64, nbytes); + } + + /// Gets an unsigned n-byte integer from `self` in native-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03 hello", + /// false => b"\x03\x02\x01 hello", + /// }; + /// assert_eq!(Ok(0x010203_u64), buf.try_get_uint_ne(3)); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03", + /// false => b"\x03\x02\x01", + /// }; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_uint_ne(4)); + /// assert_eq!(3, buf.remaining()); + /// ``` + /// + /// # Panics + /// + /// This function panics if `nbytes` is greater than 8. + fn try_get_uint_ne(&mut self, nbytes: usize) -> Result<u64, TryGetError> { + if cfg!(target_endian = "big") { + self.try_get_uint(nbytes) + } else { + self.try_get_uint_le(nbytes) + } + } + + /// Gets a signed n-byte integer from `self` in big-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x01\x02\x03 hello"[..]; + /// assert_eq!(Ok(0x010203_i64), buf.try_get_int(3)); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_int(4)); + /// assert_eq!(3, buf.remaining()); + /// ``` + /// + /// # Panics + /// + /// This function panics if `nbytes` is greater than 8. + fn try_get_int(&mut self, nbytes: usize) -> Result<i64, TryGetError> { + buf_try_get_impl!(be => self, i64, nbytes); + } + + /// Gets a signed n-byte integer from `self` in little-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x03\x02\x01 hello"[..]; + /// assert_eq!(Ok(0x010203_i64), buf.try_get_int_le(3)); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x01\x02\x03"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_int_le(4)); + /// assert_eq!(3, buf.remaining()); + /// ``` + /// + /// # Panics + /// + /// This function panics if `nbytes` is greater than 8. + fn try_get_int_le(&mut self, nbytes: usize) -> Result<i64, TryGetError> { + buf_try_get_impl!(le => self, i64, nbytes); + } + + /// Gets a signed n-byte integer from `self` in native-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03 hello", + /// false => b"\x03\x02\x01 hello", + /// }; + /// assert_eq!(Ok(0x010203_i64), buf.try_get_int_ne(3)); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x01\x02\x03", + /// false => b"\x03\x02\x01", + /// }; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_int_ne(4)); + /// assert_eq!(3, buf.remaining()); + /// ``` + /// + /// # Panics + /// + /// This function panics if `nbytes` is greater than 8. + fn try_get_int_ne(&mut self, nbytes: usize) -> Result<i64, TryGetError> { + if cfg!(target_endian = "big") { + self.try_get_int(nbytes) + } else { + self.try_get_int_le(nbytes) + } + } + + /// Gets an IEEE754 single-precision (4 bytes) floating point number from + /// `self` in big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x3F\x99\x99\x9A hello"[..]; + /// assert_eq!(1.2f32, buf.get_f32()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x3F\x99\x99"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_f32()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_f32(&mut self) -> Result<f32, TryGetError> { + Ok(f32::from_bits(self.try_get_u32()?)) + } + + /// Gets an IEEE754 single-precision (4 bytes) floating point number from + /// `self` in little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x9A\x99\x99\x3F hello"[..]; + /// assert_eq!(1.2f32, buf.get_f32_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x3F\x99\x99"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_f32_le()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_f32_le(&mut self) -> Result<f32, TryGetError> { + Ok(f32::from_bits(self.try_get_u32_le()?)) + } + + /// Gets an IEEE754 single-precision (4 bytes) floating point number from + /// `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x3F\x99\x99\x9A hello", + /// false => b"\x9A\x99\x99\x3F hello", + /// }; + /// assert_eq!(1.2f32, buf.get_f32_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x3F\x99\x99"[..]; + /// assert_eq!(Err(TryGetError{requested: 4, available: 3}), buf.try_get_f32_ne()); + /// assert_eq!(3, buf.remaining()); + /// ``` + fn try_get_f32_ne(&mut self) -> Result<f32, TryGetError> { + Ok(f32::from_bits(self.try_get_u32_ne()?)) + } + + /// Gets an IEEE754 double-precision (8 bytes) floating point number from + /// `self` in big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x3F\xF3\x33\x33\x33\x33\x33\x33 hello"[..]; + /// assert_eq!(1.2f64, buf.get_f64()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x3F\xF3\x33\x33\x33\x33\x33"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_f64()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_f64(&mut self) -> Result<f64, TryGetError> { + Ok(f64::from_bits(self.try_get_u64()?)) + } + + /// Gets an IEEE754 double-precision (8 bytes) floating point number from + /// `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = &b"\x33\x33\x33\x33\x33\x33\xF3\x3F hello"[..]; + /// assert_eq!(1.2f64, buf.get_f64_le()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x3F\xF3\x33\x33\x33\x33\x33"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_f64_le()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_f64_le(&mut self) -> Result<f64, TryGetError> { + Ok(f64::from_bits(self.try_get_u64_le()?)) + } + + /// Gets an IEEE754 double-precision (8 bytes) floating point number from + /// `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// Returns `Err(TryGetError)` when there are not enough + /// remaining bytes to read the value. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf: &[u8] = match cfg!(target_endian = "big") { + /// true => b"\x3F\xF3\x33\x33\x33\x33\x33\x33 hello", + /// false => b"\x33\x33\x33\x33\x33\x33\xF3\x3F hello", + /// }; + /// assert_eq!(1.2f64, buf.get_f64_ne()); + /// assert_eq!(6, buf.remaining()); + /// ``` + /// + /// ``` + /// use bytes::{Buf, TryGetError}; + /// + /// let mut buf = &b"\x3F\xF3\x33\x33\x33\x33\x33"[..]; + /// assert_eq!(Err(TryGetError{requested: 8, available: 7}), buf.try_get_f64_ne()); + /// assert_eq!(7, buf.remaining()); + /// ``` + fn try_get_f64_ne(&mut self) -> Result<f64, TryGetError> { + Ok(f64::from_bits(self.try_get_u64_ne()?)) + } + + /// Consumes `len` bytes inside self and returns new instance of `Bytes` + /// with this data. + /// + /// This function may be optimized by the underlying type to avoid actual + /// copies. For example, `Bytes` implementation will do a shallow copy + /// (ref-count increment). + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let bytes = (&b"hello world"[..]).copy_to_bytes(5); + /// assert_eq!(&bytes[..], &b"hello"[..]); + /// ``` + /// + /// # Panics + /// + /// This function panics if `len > self.remaining()`. + fn copy_to_bytes(&mut self, len: usize) -> crate::Bytes { + use super::BufMut; + + if self.remaining() < len { + panic_advance(&TryGetError { + requested: len, + available: self.remaining(), + }); + } + + let mut ret = crate::BytesMut::with_capacity(len); + ret.put(self.take(len)); + ret.freeze() + } + + /// Creates an adaptor which will read at most `limit` bytes from `self`. + /// + /// This function returns a new instance of `Buf` which will read at most + /// `limit` bytes. + /// + /// # Examples + /// + /// ``` + /// use bytes::{Buf, BufMut}; + /// + /// let mut buf = b"hello world"[..].take(5); + /// let mut dst = vec![]; + /// + /// dst.put(&mut buf); + /// assert_eq!(dst, b"hello"); + /// + /// let mut buf = buf.into_inner(); + /// dst.clear(); + /// dst.put(&mut buf); + /// assert_eq!(dst, b" world"); + /// ``` + fn take(self, limit: usize) -> Take<Self> + where + Self: Sized, + { + take::new(self, limit) + } + + /// Creates an adaptor which will chain this buffer with another. + /// + /// The returned `Buf` instance will first consume all bytes from `self`. + /// Afterwards the output is equivalent to the output of next. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut chain = b"hello "[..].chain(&b"world"[..]); + /// + /// let full = chain.copy_to_bytes(11); + /// assert_eq!(full.chunk(), b"hello world"); + /// ``` + fn chain<U: Buf>(self, next: U) -> Chain<Self, U> + where + Self: Sized, + { + Chain::new(self, next) + } + + /// Creates an adaptor which implements the `Read` trait for `self`. + /// + /// This function returns a new value which implements `Read` by adapting + /// the `Read` trait functions to the `Buf` trait functions. Given that + /// `Buf` operations are infallible, none of the `Read` functions will + /// return with `Err`. + /// + /// # Examples + /// + /// ``` + /// use bytes::{Bytes, Buf}; + /// use std::io::Read; + /// + /// let buf = Bytes::from("hello world"); + /// + /// let mut reader = buf.reader(); + /// let mut dst = [0; 1024]; + /// + /// let num = reader.read(&mut dst).unwrap(); + /// + /// assert_eq!(11, num); + /// assert_eq!(&dst[..11], &b"hello world"[..]); + /// ``` + #[cfg(feature = "std")] + #[cfg_attr(docsrs, doc(cfg(feature = "std")))] + fn reader(self) -> Reader<Self> + where + Self: Sized, + { + reader::new(self) + } +} + +macro_rules! deref_forward_buf { + () => { + #[inline] + fn remaining(&self) -> usize { + (**self).remaining() + } + + #[inline] + fn chunk(&self) -> &[u8] { + (**self).chunk() + } + + #[cfg(feature = "std")] + #[inline] + fn chunks_vectored<'b>(&'b self, dst: &mut [IoSlice<'b>]) -> usize { + (**self).chunks_vectored(dst) + } + + #[inline] + fn advance(&mut self, cnt: usize) { + (**self).advance(cnt) + } + + #[inline] + fn has_remaining(&self) -> bool { + (**self).has_remaining() + } + + #[inline] + fn copy_to_slice(&mut self, dst: &mut [u8]) { + (**self).copy_to_slice(dst) + } + + #[inline] + fn get_u8(&mut self) -> u8 { + (**self).get_u8() + } + + #[inline] + fn get_i8(&mut self) -> i8 { + (**self).get_i8() + } + + #[inline] + fn get_u16(&mut self) -> u16 { + (**self).get_u16() + } + + #[inline] + fn get_u16_le(&mut self) -> u16 { + (**self).get_u16_le() + } + + #[inline] + fn get_u16_ne(&mut self) -> u16 { + (**self).get_u16_ne() + } + + #[inline] + fn get_i16(&mut self) -> i16 { + (**self).get_i16() + } + + #[inline] + fn get_i16_le(&mut self) -> i16 { + (**self).get_i16_le() + } + + #[inline] + fn get_i16_ne(&mut self) -> i16 { + (**self).get_i16_ne() + } + + #[inline] + fn get_u32(&mut self) -> u32 { + (**self).get_u32() + } + + #[inline] + fn get_u32_le(&mut self) -> u32 { + (**self).get_u32_le() + } + + #[inline] + fn get_u32_ne(&mut self) -> u32 { + (**self).get_u32_ne() + } + + #[inline] + fn get_i32(&mut self) -> i32 { + (**self).get_i32() + } + + #[inline] + fn get_i32_le(&mut self) -> i32 { + (**self).get_i32_le() + } + + #[inline] + fn get_i32_ne(&mut self) -> i32 { + (**self).get_i32_ne() + } + + #[inline] + fn get_u64(&mut self) -> u64 { + (**self).get_u64() + } + + #[inline] + fn get_u64_le(&mut self) -> u64 { + (**self).get_u64_le() + } + + #[inline] + fn get_u64_ne(&mut self) -> u64 { + (**self).get_u64_ne() + } + + #[inline] + fn get_i64(&mut self) -> i64 { + (**self).get_i64() + } + + #[inline] + fn get_i64_le(&mut self) -> i64 { + (**self).get_i64_le() + } + + #[inline] + fn get_i64_ne(&mut self) -> i64 { + (**self).get_i64_ne() + } + + #[inline] + fn get_u128(&mut self) -> u128 { + (**self).get_u128() + } + + #[inline] + fn get_u128_le(&mut self) -> u128 { + (**self).get_u128_le() + } + + #[inline] + fn get_u128_ne(&mut self) -> u128 { + (**self).get_u128_ne() + } + + #[inline] + fn get_i128(&mut self) -> i128 { + (**self).get_i128() + } + + #[inline] + fn get_i128_le(&mut self) -> i128 { + (**self).get_i128_le() + } + + #[inline] + fn get_i128_ne(&mut self) -> i128 { + (**self).get_i128_ne() + } + + #[inline] + fn get_uint(&mut self, nbytes: usize) -> u64 { + (**self).get_uint(nbytes) + } + + #[inline] + fn get_uint_le(&mut self, nbytes: usize) -> u64 { + (**self).get_uint_le(nbytes) + } + + #[inline] + fn get_uint_ne(&mut self, nbytes: usize) -> u64 { + (**self).get_uint_ne(nbytes) + } + + #[inline] + fn get_int(&mut self, nbytes: usize) -> i64 { + (**self).get_int(nbytes) + } + + #[inline] + fn get_int_le(&mut self, nbytes: usize) -> i64 { + (**self).get_int_le(nbytes) + } + + #[inline] + fn get_int_ne(&mut self, nbytes: usize) -> i64 { + (**self).get_int_ne(nbytes) + } + + #[inline] + fn get_f32(&mut self) -> f32 { + (**self).get_f32() + } + + #[inline] + fn get_f32_le(&mut self) -> f32 { + (**self).get_f32_le() + } + + #[inline] + fn get_f32_ne(&mut self) -> f32 { + (**self).get_f32_ne() + } + + #[inline] + fn get_f64(&mut self) -> f64 { + (**self).get_f64() + } + + #[inline] + fn get_f64_le(&mut self) -> f64 { + (**self).get_f64_le() + } + + #[inline] + fn get_f64_ne(&mut self) -> f64 { + (**self).get_f64_ne() + } + + #[inline] + fn try_copy_to_slice(&mut self, dst: &mut [u8]) -> Result<(), TryGetError> { + (**self).try_copy_to_slice(dst) + } + + #[inline] + fn try_get_u8(&mut self) -> Result<u8, TryGetError> { + (**self).try_get_u8() + } + + #[inline] + fn try_get_i8(&mut self) -> Result<i8, TryGetError> { + (**self).try_get_i8() + } + + #[inline] + fn try_get_u16(&mut self) -> Result<u16, TryGetError> { + (**self).try_get_u16() + } + + #[inline] + fn try_get_u16_le(&mut self) -> Result<u16, TryGetError> { + (**self).try_get_u16_le() + } + + #[inline] + fn try_get_u16_ne(&mut self) -> Result<u16, TryGetError> { + (**self).try_get_u16_ne() + } + + #[inline] + fn try_get_i16(&mut self) -> Result<i16, TryGetError> { + (**self).try_get_i16() + } + + #[inline] + fn try_get_i16_le(&mut self) -> Result<i16, TryGetError> { + (**self).try_get_i16_le() + } + + #[inline] + fn try_get_i16_ne(&mut self) -> Result<i16, TryGetError> { + (**self).try_get_i16_ne() + } + + #[inline] + fn try_get_u32(&mut self) -> Result<u32, TryGetError> { + (**self).try_get_u32() + } + + #[inline] + fn try_get_u32_le(&mut self) -> Result<u32, TryGetError> { + (**self).try_get_u32_le() + } + + #[inline] + fn try_get_u32_ne(&mut self) -> Result<u32, TryGetError> { + (**self).try_get_u32_ne() + } + + #[inline] + fn try_get_i32(&mut self) -> Result<i32, TryGetError> { + (**self).try_get_i32() + } + + #[inline] + fn try_get_i32_le(&mut self) -> Result<i32, TryGetError> { + (**self).try_get_i32_le() + } + + #[inline] + fn try_get_i32_ne(&mut self) -> Result<i32, TryGetError> { + (**self).try_get_i32_ne() + } + + #[inline] + fn try_get_u64(&mut self) -> Result<u64, TryGetError> { + (**self).try_get_u64() + } + + #[inline] + fn try_get_u64_le(&mut self) -> Result<u64, TryGetError> { + (**self).try_get_u64_le() + } + + #[inline] + fn try_get_u64_ne(&mut self) -> Result<u64, TryGetError> { + (**self).try_get_u64_ne() + } + + #[inline] + fn try_get_i64(&mut self) -> Result<i64, TryGetError> { + (**self).try_get_i64() + } + + #[inline] + fn try_get_i64_le(&mut self) -> Result<i64, TryGetError> { + (**self).try_get_i64_le() + } + + #[inline] + fn try_get_i64_ne(&mut self) -> Result<i64, TryGetError> { + (**self).try_get_i64_ne() + } + + #[inline] + fn try_get_u128(&mut self) -> Result<u128, TryGetError> { + (**self).try_get_u128() + } + + #[inline] + fn try_get_u128_le(&mut self) -> Result<u128, TryGetError> { + (**self).try_get_u128_le() + } + + #[inline] + fn try_get_u128_ne(&mut self) -> Result<u128, TryGetError> { + (**self).try_get_u128_ne() + } + + #[inline] + fn try_get_i128(&mut self) -> Result<i128, TryGetError> { + (**self).try_get_i128() + } + + #[inline] + fn try_get_i128_le(&mut self) -> Result<i128, TryGetError> { + (**self).try_get_i128_le() + } + + #[inline] + fn try_get_i128_ne(&mut self) -> Result<i128, TryGetError> { + (**self).try_get_i128_ne() + } + + #[inline] + fn try_get_uint(&mut self, nbytes: usize) -> Result<u64, TryGetError> { + (**self).try_get_uint(nbytes) + } + + #[inline] + fn try_get_uint_le(&mut self, nbytes: usize) -> Result<u64, TryGetError> { + (**self).try_get_uint_le(nbytes) + } + + #[inline] + fn try_get_uint_ne(&mut self, nbytes: usize) -> Result<u64, TryGetError> { + (**self).try_get_uint_ne(nbytes) + } + + #[inline] + fn try_get_int(&mut self, nbytes: usize) -> Result<i64, TryGetError> { + (**self).try_get_int(nbytes) + } + + #[inline] + fn try_get_int_le(&mut self, nbytes: usize) -> Result<i64, TryGetError> { + (**self).try_get_int_le(nbytes) + } + + #[inline] + fn try_get_int_ne(&mut self, nbytes: usize) -> Result<i64, TryGetError> { + (**self).try_get_int_ne(nbytes) + } + + #[inline] + fn try_get_f32(&mut self) -> Result<f32, TryGetError> { + (**self).try_get_f32() + } + + #[inline] + fn try_get_f32_le(&mut self) -> Result<f32, TryGetError> { + (**self).try_get_f32_le() + } + + #[inline] + fn try_get_f32_ne(&mut self) -> Result<f32, TryGetError> { + (**self).try_get_f32_ne() + } + + #[inline] + fn try_get_f64(&mut self) -> Result<f64, TryGetError> { + (**self).try_get_f64() + } + + #[inline] + fn try_get_f64_le(&mut self) -> Result<f64, TryGetError> { + (**self).try_get_f64_le() + } + + #[inline] + fn try_get_f64_ne(&mut self) -> Result<f64, TryGetError> { + (**self).try_get_f64_ne() + } + + #[inline] + fn copy_to_bytes(&mut self, len: usize) -> crate::Bytes { + (**self).copy_to_bytes(len) + } + }; +} + +impl<T: Buf + ?Sized> Buf for &mut T { + deref_forward_buf!(); +} + +impl<T: Buf + ?Sized> Buf for Box<T> { + deref_forward_buf!(); +} + +impl Buf for &[u8] { + #[inline] + fn remaining(&self) -> usize { + self.len() + } + + #[inline] + fn chunk(&self) -> &[u8] { + self + } + + #[inline] + fn advance(&mut self, cnt: usize) { + if self.len() < cnt { + panic_advance(&TryGetError { + requested: cnt, + available: self.len(), + }); + } + + *self = &self[cnt..]; + } + + #[inline] + fn copy_to_slice(&mut self, dst: &mut [u8]) { + if self.len() < dst.len() { + panic_advance(&TryGetError { + requested: dst.len(), + available: self.len(), + }); + } + + dst.copy_from_slice(&self[..dst.len()]); + self.advance(dst.len()); + } +} + +#[cfg(feature = "std")] +impl<T: AsRef<[u8]>> Buf for std::io::Cursor<T> { + #[inline] + fn remaining(&self) -> usize { + saturating_sub_usize_u64(self.get_ref().as_ref().len(), self.position()) + } + + #[inline] + fn chunk(&self) -> &[u8] { + let slice = self.get_ref().as_ref(); + let pos = min_u64_usize(self.position(), slice.len()); + &slice[pos..] + } + + #[inline] + fn advance(&mut self, cnt: usize) { + let len = self.get_ref().as_ref().len(); + let pos = self.position(); + + // We intentionally allow `cnt == 0` here even if `pos > len`. + let max_cnt = saturating_sub_usize_u64(len, pos); + if cnt > max_cnt { + panic_advance(&TryGetError { + requested: cnt, + available: max_cnt, + }); + } + + // This will not overflow because either `cnt == 0` or the sum is not + // greater than `len`. + self.set_position(pos + cnt as u64); + } +} + +// The existence of this function makes the compiler catch if the Buf +// trait is "object-safe" or not. +fn _assert_trait_object(_b: &dyn Buf) {} diff --git a/third-party/bytes/src/buf/buf_mut.rs b/third-party/bytes/src/buf/buf_mut.rs new file mode 100644 index 000000000..3154dfeca --- /dev/null +++ b/third-party/bytes/src/buf/buf_mut.rs @@ -0,0 +1,1671 @@ +use crate::buf::{limit, Chain, Limit, UninitSlice}; +#[cfg(feature = "std")] +use crate::buf::{writer, Writer}; +use crate::{panic_advance, panic_does_not_fit, TryGetError}; + +use core::{mem, ptr}; + +use alloc::{boxed::Box, vec::Vec}; + +/// A trait for values that provide sequential write access to bytes. +/// +/// Write bytes to a buffer +/// +/// A buffer stores bytes in memory such that write operations are infallible. +/// The underlying storage may or may not be in contiguous memory. A `BufMut` +/// value is a cursor into the buffer. Writing to `BufMut` advances the cursor +/// position. +/// +/// The simplest `BufMut` is a `Vec<u8>`. +/// +/// ``` +/// use bytes::BufMut; +/// +/// let mut buf = vec![]; +/// +/// buf.put(&b"hello world"[..]); +/// +/// assert_eq!(buf, b"hello world"); +/// ``` +pub unsafe trait BufMut { + /// Returns the number of bytes that can be written from the current + /// position until the end of the buffer is reached. + /// + /// This value is greater than or equal to the length of the slice returned + /// by `chunk_mut()`. + /// + /// Writing to a `BufMut` may involve allocating more memory on the fly. + /// Implementations may fail before reaching the number of bytes indicated + /// by this method if they encounter an allocation failure. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut dst = [0; 10]; + /// let mut buf = &mut dst[..]; + /// + /// let original_remaining = buf.remaining_mut(); + /// buf.put(&b"hello"[..]); + /// + /// assert_eq!(original_remaining - 5, buf.remaining_mut()); + /// ``` + /// + /// # Implementer notes + /// + /// Implementations of `remaining_mut` should ensure that the return value + /// does not change unless a call is made to `advance_mut` or any other + /// function that is documented to change the `BufMut`'s current position. + /// + /// # Note + /// + /// `remaining_mut` may return value smaller than actual available space. + fn remaining_mut(&self) -> usize; + + /// Advance the internal cursor of the BufMut + /// + /// The next call to `chunk_mut` will return a slice starting `cnt` bytes + /// further into the underlying buffer. + /// + /// # Safety + /// + /// The caller must ensure that the next `cnt` bytes of `chunk` are + /// initialized. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = Vec::with_capacity(16); + /// + /// // Write some data + /// buf.chunk_mut()[0..2].copy_from_slice(b"he"); + /// unsafe { buf.advance_mut(2) }; + /// + /// // write more bytes + /// buf.chunk_mut()[0..3].copy_from_slice(b"llo"); + /// + /// unsafe { buf.advance_mut(3); } + /// + /// assert_eq!(5, buf.len()); + /// assert_eq!(buf, b"hello"); + /// ``` + /// + /// # Panics + /// + /// This function **may** panic if `cnt > self.remaining_mut()`. + /// + /// # Implementer notes + /// + /// It is recommended for implementations of `advance_mut` to panic if + /// `cnt > self.remaining_mut()`. If the implementation does not panic, + /// the call must behave as if `cnt == self.remaining_mut()`. + /// + /// A call with `cnt == 0` should never panic and be a no-op. + unsafe fn advance_mut(&mut self, cnt: usize); + + /// Returns true if there is space in `self` for more bytes. + /// + /// This is equivalent to `self.remaining_mut() != 0`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut dst = [0; 5]; + /// let mut buf = &mut dst[..]; + /// + /// assert!(buf.has_remaining_mut()); + /// + /// buf.put(&b"hello"[..]); + /// + /// assert!(!buf.has_remaining_mut()); + /// ``` + #[inline] + fn has_remaining_mut(&self) -> bool { + self.remaining_mut() > 0 + } + + /// Returns a mutable slice starting at the current BufMut position and of + /// length between 0 and `BufMut::remaining_mut()`. Note that this *can* be shorter than the + /// whole remainder of the buffer (this allows non-continuous implementation). + /// + /// This is a lower level function. Most operations are done with other + /// functions. + /// + /// The returned byte slice may represent uninitialized memory. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = Vec::with_capacity(16); + /// + /// unsafe { + /// // MaybeUninit::as_mut_ptr + /// buf.chunk_mut()[0..].as_mut_ptr().write(b'h'); + /// buf.chunk_mut()[1..].as_mut_ptr().write(b'e'); + /// + /// buf.advance_mut(2); + /// + /// buf.chunk_mut()[0..].as_mut_ptr().write(b'l'); + /// buf.chunk_mut()[1..].as_mut_ptr().write(b'l'); + /// buf.chunk_mut()[2..].as_mut_ptr().write(b'o'); + /// + /// buf.advance_mut(3); + /// } + /// + /// assert_eq!(5, buf.len()); + /// assert_eq!(buf, b"hello"); + /// ``` + /// + /// # Implementer notes + /// + /// This function should never panic. `chunk_mut()` should return an empty + /// slice **if and only if** `remaining_mut()` returns 0. In other words, + /// `chunk_mut()` returning an empty slice implies that `remaining_mut()` will + /// return 0 and `remaining_mut()` returning 0 implies that `chunk_mut()` will + /// return an empty slice. + /// + /// This function may trigger an out-of-memory abort if it tries to allocate + /// memory and fails to do so. + // The `chunk_mut` method was previously called `bytes_mut`. This alias makes the + // rename more easily discoverable. + #[cfg_attr(docsrs, doc(alias = "bytes_mut"))] + fn chunk_mut(&mut self) -> &mut UninitSlice; + + /// Transfer bytes into `self` from `src` and advance the cursor by the + /// number of bytes written. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// + /// buf.put_u8(b'h'); + /// buf.put(&b"ello"[..]); + /// buf.put(&b" world"[..]); + /// + /// assert_eq!(buf, b"hello world"); + /// ``` + /// + /// # Panics + /// + /// Panics if `self` does not have enough capacity to contain `src`. + #[inline] + fn put<T: super::Buf>(&mut self, mut src: T) + where + Self: Sized, + { + if self.remaining_mut() < src.remaining() { + panic_advance(&TryGetError { + requested: src.remaining(), + available: self.remaining_mut(), + }); + } + + while src.has_remaining() { + let s = src.chunk(); + let d = self.chunk_mut(); + let cnt = usize::min(s.len(), d.len()); + + d[..cnt].copy_from_slice(&s[..cnt]); + + // SAFETY: We just initialized `cnt` bytes in `self`. + unsafe { self.advance_mut(cnt) }; + src.advance(cnt); + } + } + + /// Transfer bytes into `self` from `src` and advance the cursor by the + /// number of bytes written. + /// + /// `self` must have enough remaining capacity to contain all of `src`. + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut dst = [0; 6]; + /// + /// { + /// let mut buf = &mut dst[..]; + /// buf.put_slice(b"hello"); + /// + /// assert_eq!(1, buf.remaining_mut()); + /// } + /// + /// assert_eq!(b"hello\0", &dst); + /// ``` + #[inline] + fn put_slice(&mut self, mut src: &[u8]) { + if self.remaining_mut() < src.len() { + panic_advance(&TryGetError { + requested: src.len(), + available: self.remaining_mut(), + }); + } + + while !src.is_empty() { + let dst = self.chunk_mut(); + let cnt = usize::min(src.len(), dst.len()); + + dst[..cnt].copy_from_slice(&src[..cnt]); + src = &src[cnt..]; + + // SAFETY: We just initialized `cnt` bytes in `self`. + unsafe { self.advance_mut(cnt) }; + } + } + + /// Put `cnt` bytes `val` into `self`. + /// + /// Logically equivalent to calling `self.put_u8(val)` `cnt` times, but may work faster. + /// + /// `self` must have at least `cnt` remaining capacity. + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut dst = [0; 6]; + /// + /// { + /// let mut buf = &mut dst[..]; + /// buf.put_bytes(b'a', 4); + /// + /// assert_eq!(2, buf.remaining_mut()); + /// } + /// + /// assert_eq!(b"aaaa\0\0", &dst); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_bytes(&mut self, val: u8, mut cnt: usize) { + if self.remaining_mut() < cnt { + panic_advance(&TryGetError { + requested: cnt, + available: self.remaining_mut(), + }) + } + + while cnt > 0 { + let dst = self.chunk_mut(); + let dst_len = usize::min(dst.len(), cnt); + // SAFETY: The pointer is valid for `dst_len <= dst.len()` bytes. + unsafe { core::ptr::write_bytes(dst.as_mut_ptr(), val, dst_len) }; + // SAFETY: We just initialized `dst_len` bytes in `self`. + unsafe { self.advance_mut(dst_len) }; + cnt -= dst_len; + } + } + + /// Writes an unsigned 8 bit integer to `self`. + /// + /// The current position is advanced by 1. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u8(0x01); + /// assert_eq!(buf, b"\x01"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u8(&mut self, n: u8) { + let src = [n]; + self.put_slice(&src); + } + + /// Writes a signed 8 bit integer to `self`. + /// + /// The current position is advanced by 1. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i8(0x01); + /// assert_eq!(buf, b"\x01"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i8(&mut self, n: i8) { + let src = [n as u8]; + self.put_slice(&src) + } + + /// Writes an unsigned 16 bit integer to `self` in big-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u16(0x0809); + /// assert_eq!(buf, b"\x08\x09"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u16(&mut self, n: u16) { + self.put_slice(&n.to_be_bytes()) + } + + /// Writes an unsigned 16 bit integer to `self` in little-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u16_le(0x0809); + /// assert_eq!(buf, b"\x09\x08"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u16_le(&mut self, n: u16) { + self.put_slice(&n.to_le_bytes()) + } + + /// Writes an unsigned 16 bit integer to `self` in native-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u16_ne(0x0809); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x08\x09"); + /// } else { + /// assert_eq!(buf, b"\x09\x08"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u16_ne(&mut self, n: u16) { + self.put_slice(&n.to_ne_bytes()) + } + + /// Writes a signed 16 bit integer to `self` in big-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i16(0x0809); + /// assert_eq!(buf, b"\x08\x09"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i16(&mut self, n: i16) { + self.put_slice(&n.to_be_bytes()) + } + + /// Writes a signed 16 bit integer to `self` in little-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i16_le(0x0809); + /// assert_eq!(buf, b"\x09\x08"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i16_le(&mut self, n: i16) { + self.put_slice(&n.to_le_bytes()) + } + + /// Writes a signed 16 bit integer to `self` in native-endian byte order. + /// + /// The current position is advanced by 2. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i16_ne(0x0809); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x08\x09"); + /// } else { + /// assert_eq!(buf, b"\x09\x08"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i16_ne(&mut self, n: i16) { + self.put_slice(&n.to_ne_bytes()) + } + + /// Writes an unsigned 32 bit integer to `self` in big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u32(0x0809A0A1); + /// assert_eq!(buf, b"\x08\x09\xA0\xA1"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u32(&mut self, n: u32) { + self.put_slice(&n.to_be_bytes()) + } + + /// Writes an unsigned 32 bit integer to `self` in little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u32_le(0x0809A0A1); + /// assert_eq!(buf, b"\xA1\xA0\x09\x08"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u32_le(&mut self, n: u32) { + self.put_slice(&n.to_le_bytes()) + } + + /// Writes an unsigned 32 bit integer to `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u32_ne(0x0809A0A1); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x08\x09\xA0\xA1"); + /// } else { + /// assert_eq!(buf, b"\xA1\xA0\x09\x08"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u32_ne(&mut self, n: u32) { + self.put_slice(&n.to_ne_bytes()) + } + + /// Writes a signed 32 bit integer to `self` in big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i32(0x0809A0A1); + /// assert_eq!(buf, b"\x08\x09\xA0\xA1"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i32(&mut self, n: i32) { + self.put_slice(&n.to_be_bytes()) + } + + /// Writes a signed 32 bit integer to `self` in little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i32_le(0x0809A0A1); + /// assert_eq!(buf, b"\xA1\xA0\x09\x08"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i32_le(&mut self, n: i32) { + self.put_slice(&n.to_le_bytes()) + } + + /// Writes a signed 32 bit integer to `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i32_ne(0x0809A0A1); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x08\x09\xA0\xA1"); + /// } else { + /// assert_eq!(buf, b"\xA1\xA0\x09\x08"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i32_ne(&mut self, n: i32) { + self.put_slice(&n.to_ne_bytes()) + } + + /// Writes an unsigned 64 bit integer to `self` in the big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u64(0x0102030405060708); + /// assert_eq!(buf, b"\x01\x02\x03\x04\x05\x06\x07\x08"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u64(&mut self, n: u64) { + self.put_slice(&n.to_be_bytes()) + } + + /// Writes an unsigned 64 bit integer to `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u64_le(0x0102030405060708); + /// assert_eq!(buf, b"\x08\x07\x06\x05\x04\x03\x02\x01"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u64_le(&mut self, n: u64) { + self.put_slice(&n.to_le_bytes()) + } + + /// Writes an unsigned 64 bit integer to `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u64_ne(0x0102030405060708); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x01\x02\x03\x04\x05\x06\x07\x08"); + /// } else { + /// assert_eq!(buf, b"\x08\x07\x06\x05\x04\x03\x02\x01"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u64_ne(&mut self, n: u64) { + self.put_slice(&n.to_ne_bytes()) + } + + /// Writes a signed 64 bit integer to `self` in the big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i64(0x0102030405060708); + /// assert_eq!(buf, b"\x01\x02\x03\x04\x05\x06\x07\x08"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i64(&mut self, n: i64) { + self.put_slice(&n.to_be_bytes()) + } + + /// Writes a signed 64 bit integer to `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i64_le(0x0102030405060708); + /// assert_eq!(buf, b"\x08\x07\x06\x05\x04\x03\x02\x01"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i64_le(&mut self, n: i64) { + self.put_slice(&n.to_le_bytes()) + } + + /// Writes a signed 64 bit integer to `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i64_ne(0x0102030405060708); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x01\x02\x03\x04\x05\x06\x07\x08"); + /// } else { + /// assert_eq!(buf, b"\x08\x07\x06\x05\x04\x03\x02\x01"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i64_ne(&mut self, n: i64) { + self.put_slice(&n.to_ne_bytes()) + } + + /// Writes an unsigned 128 bit integer to `self` in the big-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u128(0x01020304050607080910111213141516); + /// assert_eq!(buf, b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u128(&mut self, n: u128) { + self.put_slice(&n.to_be_bytes()) + } + + /// Writes an unsigned 128 bit integer to `self` in little-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u128_le(0x01020304050607080910111213141516); + /// assert_eq!(buf, b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u128_le(&mut self, n: u128) { + self.put_slice(&n.to_le_bytes()) + } + + /// Writes an unsigned 128 bit integer to `self` in native-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_u128_ne(0x01020304050607080910111213141516); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16"); + /// } else { + /// assert_eq!(buf, b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_u128_ne(&mut self, n: u128) { + self.put_slice(&n.to_ne_bytes()) + } + + /// Writes a signed 128 bit integer to `self` in the big-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i128(0x01020304050607080910111213141516); + /// assert_eq!(buf, b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i128(&mut self, n: i128) { + self.put_slice(&n.to_be_bytes()) + } + + /// Writes a signed 128 bit integer to `self` in little-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i128_le(0x01020304050607080910111213141516); + /// assert_eq!(buf, b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i128_le(&mut self, n: i128) { + self.put_slice(&n.to_le_bytes()) + } + + /// Writes a signed 128 bit integer to `self` in native-endian byte order. + /// + /// The current position is advanced by 16. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_i128_ne(0x01020304050607080910111213141516); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16"); + /// } else { + /// assert_eq!(buf, b"\x16\x15\x14\x13\x12\x11\x10\x09\x08\x07\x06\x05\x04\x03\x02\x01"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_i128_ne(&mut self, n: i128) { + self.put_slice(&n.to_ne_bytes()) + } + + /// Writes an unsigned n-byte integer to `self` in big-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_uint(0x010203, 3); + /// assert_eq!(buf, b"\x01\x02\x03"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self` or if `nbytes` is greater than 8. + #[inline] + fn put_uint(&mut self, n: u64, nbytes: usize) { + let start = match mem::size_of_val(&n).checked_sub(nbytes) { + Some(start) => start, + None => panic_does_not_fit(nbytes, mem::size_of_val(&n)), + }; + + self.put_slice(&n.to_be_bytes()[start..]); + } + + /// Writes an unsigned n-byte integer to `self` in the little-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_uint_le(0x010203, 3); + /// assert_eq!(buf, b"\x03\x02\x01"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self` or if `nbytes` is greater than 8. + #[inline] + fn put_uint_le(&mut self, n: u64, nbytes: usize) { + let slice = n.to_le_bytes(); + let slice = match slice.get(..nbytes) { + Some(slice) => slice, + None => panic_does_not_fit(nbytes, slice.len()), + }; + + self.put_slice(slice); + } + + /// Writes an unsigned n-byte integer to `self` in the native-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_uint_ne(0x010203, 3); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x01\x02\x03"); + /// } else { + /// assert_eq!(buf, b"\x03\x02\x01"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self` or if `nbytes` is greater than 8. + #[inline] + fn put_uint_ne(&mut self, n: u64, nbytes: usize) { + if cfg!(target_endian = "big") { + self.put_uint(n, nbytes) + } else { + self.put_uint_le(n, nbytes) + } + } + + /// Writes low `nbytes` of a signed integer to `self` in big-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_int(0x0504010203, 3); + /// assert_eq!(buf, b"\x01\x02\x03"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self` or if `nbytes` is greater than 8. + #[inline] + fn put_int(&mut self, n: i64, nbytes: usize) { + let start = match mem::size_of_val(&n).checked_sub(nbytes) { + Some(start) => start, + None => panic_does_not_fit(nbytes, mem::size_of_val(&n)), + }; + + self.put_slice(&n.to_be_bytes()[start..]); + } + + /// Writes low `nbytes` of a signed integer to `self` in little-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_int_le(0x0504010203, 3); + /// assert_eq!(buf, b"\x03\x02\x01"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self` or if `nbytes` is greater than 8. + #[inline] + fn put_int_le(&mut self, n: i64, nbytes: usize) { + let slice = n.to_le_bytes(); + let slice = match slice.get(..nbytes) { + Some(slice) => slice, + None => panic_does_not_fit(nbytes, slice.len()), + }; + + self.put_slice(slice); + } + + /// Writes low `nbytes` of a signed integer to `self` in native-endian byte order. + /// + /// The current position is advanced by `nbytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_int_ne(0x010203, 3); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x01\x02\x03"); + /// } else { + /// assert_eq!(buf, b"\x03\x02\x01"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self` or if `nbytes` is greater than 8. + #[inline] + fn put_int_ne(&mut self, n: i64, nbytes: usize) { + if cfg!(target_endian = "big") { + self.put_int(n, nbytes) + } else { + self.put_int_le(n, nbytes) + } + } + + /// Writes an IEEE754 single-precision (4 bytes) floating point number to + /// `self` in big-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_f32(1.2f32); + /// assert_eq!(buf, b"\x3F\x99\x99\x9A"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_f32(&mut self, n: f32) { + self.put_u32(n.to_bits()); + } + + /// Writes an IEEE754 single-precision (4 bytes) floating point number to + /// `self` in little-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_f32_le(1.2f32); + /// assert_eq!(buf, b"\x9A\x99\x99\x3F"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_f32_le(&mut self, n: f32) { + self.put_u32_le(n.to_bits()); + } + + /// Writes an IEEE754 single-precision (4 bytes) floating point number to + /// `self` in native-endian byte order. + /// + /// The current position is advanced by 4. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_f32_ne(1.2f32); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x3F\x99\x99\x9A"); + /// } else { + /// assert_eq!(buf, b"\x9A\x99\x99\x3F"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_f32_ne(&mut self, n: f32) { + self.put_u32_ne(n.to_bits()); + } + + /// Writes an IEEE754 double-precision (8 bytes) floating point number to + /// `self` in big-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_f64(1.2f64); + /// assert_eq!(buf, b"\x3F\xF3\x33\x33\x33\x33\x33\x33"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_f64(&mut self, n: f64) { + self.put_u64(n.to_bits()); + } + + /// Writes an IEEE754 double-precision (8 bytes) floating point number to + /// `self` in little-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_f64_le(1.2f64); + /// assert_eq!(buf, b"\x33\x33\x33\x33\x33\x33\xF3\x3F"); + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_f64_le(&mut self, n: f64) { + self.put_u64_le(n.to_bits()); + } + + /// Writes an IEEE754 double-precision (8 bytes) floating point number to + /// `self` in native-endian byte order. + /// + /// The current position is advanced by 8. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut buf = vec![]; + /// buf.put_f64_ne(1.2f64); + /// if cfg!(target_endian = "big") { + /// assert_eq!(buf, b"\x3F\xF3\x33\x33\x33\x33\x33\x33"); + /// } else { + /// assert_eq!(buf, b"\x33\x33\x33\x33\x33\x33\xF3\x3F"); + /// } + /// ``` + /// + /// # Panics + /// + /// This function panics if there is not enough remaining capacity in + /// `self`. + #[inline] + fn put_f64_ne(&mut self, n: f64) { + self.put_u64_ne(n.to_bits()); + } + + /// Creates an adaptor which can write at most `limit` bytes to `self`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let arr = &mut [0u8; 128][..]; + /// assert_eq!(arr.remaining_mut(), 128); + /// + /// let dst = arr.limit(10); + /// assert_eq!(dst.remaining_mut(), 10); + /// ``` + #[inline] + fn limit(self, limit: usize) -> Limit<Self> + where + Self: Sized, + { + limit::new(self, limit) + } + + /// Creates an adaptor which implements the `Write` trait for `self`. + /// + /// This function returns a new value which implements `Write` by adapting + /// the `Write` trait functions to the `BufMut` trait functions. Given that + /// `BufMut` operations are infallible, none of the `Write` functions will + /// return with `Err`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// use std::io::Write; + /// + /// let mut buf = vec![].writer(); + /// + /// let num = buf.write(&b"hello world"[..]).unwrap(); + /// assert_eq!(11, num); + /// + /// let buf = buf.into_inner(); + /// + /// assert_eq!(*buf, b"hello world"[..]); + /// ``` + #[cfg(feature = "std")] + #[cfg_attr(docsrs, doc(cfg(feature = "std")))] + #[inline] + fn writer(self) -> Writer<Self> + where + Self: Sized, + { + writer::new(self) + } + + /// Creates an adapter which will chain this buffer with another. + /// + /// The returned `BufMut` instance will first write to all bytes from + /// `self`. Afterwards, it will write to `next`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut a = [0u8; 5]; + /// let mut b = [0u8; 6]; + /// + /// let mut chain = (&mut a[..]).chain_mut(&mut b[..]); + /// + /// chain.put_slice(b"hello world"); + /// + /// assert_eq!(&a[..], b"hello"); + /// assert_eq!(&b[..], b" world"); + /// ``` + #[inline] + fn chain_mut<U: BufMut>(self, next: U) -> Chain<Self, U> + where + Self: Sized, + { + Chain::new(self, next) + } +} + +macro_rules! deref_forward_bufmut { + () => { + #[inline] + fn remaining_mut(&self) -> usize { + (**self).remaining_mut() + } + + #[inline] + fn chunk_mut(&mut self) -> &mut UninitSlice { + (**self).chunk_mut() + } + + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + (**self).advance_mut(cnt) + } + + #[inline] + fn put_slice(&mut self, src: &[u8]) { + (**self).put_slice(src) + } + + #[inline] + fn put_u8(&mut self, n: u8) { + (**self).put_u8(n) + } + + #[inline] + fn put_i8(&mut self, n: i8) { + (**self).put_i8(n) + } + + #[inline] + fn put_u16(&mut self, n: u16) { + (**self).put_u16(n) + } + + #[inline] + fn put_u16_le(&mut self, n: u16) { + (**self).put_u16_le(n) + } + + #[inline] + fn put_u16_ne(&mut self, n: u16) { + (**self).put_u16_ne(n) + } + + #[inline] + fn put_i16(&mut self, n: i16) { + (**self).put_i16(n) + } + + #[inline] + fn put_i16_le(&mut self, n: i16) { + (**self).put_i16_le(n) + } + + #[inline] + fn put_i16_ne(&mut self, n: i16) { + (**self).put_i16_ne(n) + } + + #[inline] + fn put_u32(&mut self, n: u32) { + (**self).put_u32(n) + } + + #[inline] + fn put_u32_le(&mut self, n: u32) { + (**self).put_u32_le(n) + } + + #[inline] + fn put_u32_ne(&mut self, n: u32) { + (**self).put_u32_ne(n) + } + + #[inline] + fn put_i32(&mut self, n: i32) { + (**self).put_i32(n) + } + + #[inline] + fn put_i32_le(&mut self, n: i32) { + (**self).put_i32_le(n) + } + + #[inline] + fn put_i32_ne(&mut self, n: i32) { + (**self).put_i32_ne(n) + } + + #[inline] + fn put_u64(&mut self, n: u64) { + (**self).put_u64(n) + } + + #[inline] + fn put_u64_le(&mut self, n: u64) { + (**self).put_u64_le(n) + } + + #[inline] + fn put_u64_ne(&mut self, n: u64) { + (**self).put_u64_ne(n) + } + + #[inline] + fn put_i64(&mut self, n: i64) { + (**self).put_i64(n) + } + + #[inline] + fn put_i64_le(&mut self, n: i64) { + (**self).put_i64_le(n) + } + + #[inline] + fn put_i64_ne(&mut self, n: i64) { + (**self).put_i64_ne(n) + } + }; +} + +unsafe impl<T: BufMut + ?Sized> BufMut for &mut T { + deref_forward_bufmut!(); +} + +unsafe impl<T: BufMut + ?Sized> BufMut for Box<T> { + deref_forward_bufmut!(); +} + +unsafe impl BufMut for &mut [u8] { + #[inline] + fn remaining_mut(&self) -> usize { + self.len() + } + + #[inline] + fn chunk_mut(&mut self) -> &mut UninitSlice { + UninitSlice::new(self) + } + + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + if self.len() < cnt { + panic_advance(&TryGetError { + requested: cnt, + available: self.len(), + }); + } + + // Lifetime dance taken from `impl Write for &mut [u8]`. + let (_, b) = core::mem::take(self).split_at_mut(cnt); + *self = b; + } + + #[inline] + fn put_slice(&mut self, src: &[u8]) { + if self.len() < src.len() { + panic_advance(&TryGetError { + requested: src.len(), + available: self.len(), + }); + } + + self[..src.len()].copy_from_slice(src); + // SAFETY: We just initialized `src.len()` bytes. + unsafe { self.advance_mut(src.len()) }; + } + + #[inline] + fn put_bytes(&mut self, val: u8, cnt: usize) { + if self.len() < cnt { + panic_advance(&TryGetError { + requested: cnt, + available: self.len(), + }); + } + + // SAFETY: We just checked that the pointer is valid for `cnt` bytes. + unsafe { + ptr::write_bytes(self.as_mut_ptr(), val, cnt); + self.advance_mut(cnt); + } + } +} + +unsafe impl BufMut for &mut [core::mem::MaybeUninit<u8>] { + #[inline] + fn remaining_mut(&self) -> usize { + self.len() + } + + #[inline] + fn chunk_mut(&mut self) -> &mut UninitSlice { + UninitSlice::uninit(self) + } + + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + if self.len() < cnt { + panic_advance(&TryGetError { + requested: cnt, + available: self.len(), + }); + } + + // Lifetime dance taken from `impl Write for &mut [u8]`. + let (_, b) = core::mem::take(self).split_at_mut(cnt); + *self = b; + } + + #[inline] + fn put_slice(&mut self, src: &[u8]) { + if self.len() < src.len() { + panic_advance(&TryGetError { + requested: src.len(), + available: self.len(), + }); + } + + // SAFETY: We just checked that the pointer is valid for `src.len()` bytes. + unsafe { + ptr::copy_nonoverlapping(src.as_ptr(), self.as_mut_ptr().cast(), src.len()); + self.advance_mut(src.len()); + } + } + + #[inline] + fn put_bytes(&mut self, val: u8, cnt: usize) { + if self.len() < cnt { + panic_advance(&TryGetError { + requested: cnt, + available: self.len(), + }); + } + + // SAFETY: We just checked that the pointer is valid for `cnt` bytes. + unsafe { + ptr::write_bytes(self.as_mut_ptr() as *mut u8, val, cnt); + self.advance_mut(cnt); + } + } +} + +unsafe impl BufMut for Vec<u8> { + #[inline] + fn remaining_mut(&self) -> usize { + // A vector can never have more than isize::MAX bytes + isize::MAX as usize - self.len() + } + + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + let len = self.len(); + let remaining = self.capacity() - len; + + if remaining < cnt { + panic_advance(&TryGetError { + requested: cnt, + available: remaining, + }); + } + + // Addition will not overflow since the sum is at most the capacity. + self.set_len(len + cnt); + } + + #[inline] + fn chunk_mut(&mut self) -> &mut UninitSlice { + if self.capacity() == self.len() { + self.reserve(64); // Grow the vec + } + + let cap = self.capacity(); + let len = self.len(); + + let ptr = self.as_mut_ptr(); + // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be + // valid for `cap - len` bytes. The subtraction will not underflow since + // `len <= cap`. + unsafe { UninitSlice::from_raw_parts_mut(ptr.add(len), cap - len) } + } + + // Specialize these methods so they can skip checking `remaining_mut` + // and `advance_mut`. + #[inline] + fn put<T: super::Buf>(&mut self, mut src: T) + where + Self: Sized, + { + // In case the src isn't contiguous, reserve upfront. + self.reserve(src.remaining()); + + while src.has_remaining() { + let s = src.chunk(); + let l = s.len(); + self.extend_from_slice(s); + src.advance(l); + } + } + + #[inline] + fn put_slice(&mut self, src: &[u8]) { + self.extend_from_slice(src); + } + + #[inline] + fn put_bytes(&mut self, val: u8, cnt: usize) { + // If the addition overflows, then the `resize` will fail. + let new_len = self.len().saturating_add(cnt); + self.resize(new_len, val); + } +} + +// The existence of this function makes the compiler catch if the BufMut +// trait is "object-safe" or not. +fn _assert_trait_object(_b: &dyn BufMut) {} diff --git a/third-party/bytes/src/buf/chain.rs b/third-party/bytes/src/buf/chain.rs new file mode 100644 index 000000000..c8bc36de9 --- /dev/null +++ b/third-party/bytes/src/buf/chain.rs @@ -0,0 +1,240 @@ +use crate::buf::{IntoIter, UninitSlice}; +use crate::{Buf, BufMut}; + +#[cfg(feature = "std")] +use std::io::IoSlice; + +/// A `Chain` sequences two buffers. +/// +/// `Chain` is an adapter that links two underlying buffers and provides a +/// continuous view across both buffers. It is able to sequence either immutable +/// buffers ([`Buf`] values) or mutable buffers ([`BufMut`] values). +/// +/// This struct is generally created by calling [`Buf::chain`]. Please see that +/// function's documentation for more detail. +/// +/// # Examples +/// +/// ``` +/// use bytes::{Bytes, Buf}; +/// +/// let mut buf = (&b"hello "[..]) +/// .chain(&b"world"[..]); +/// +/// let full: Bytes = buf.copy_to_bytes(11); +/// assert_eq!(full[..], b"hello world"[..]); +/// ``` +/// +/// [`Buf::chain`]: Buf::chain +#[derive(Debug)] +pub struct Chain<T, U> { + a: T, + b: U, +} + +impl<T, U> Chain<T, U> { + /// Creates a new `Chain` sequencing the provided values. + pub(crate) fn new(a: T, b: U) -> Chain<T, U> { + Chain { a, b } + } + + /// Gets a reference to the first underlying `Buf`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let buf = (&b"hello"[..]) + /// .chain(&b"world"[..]); + /// + /// assert_eq!(buf.first_ref()[..], b"hello"[..]); + /// ``` + pub fn first_ref(&self) -> &T { + &self.a + } + + /// Gets a mutable reference to the first underlying `Buf`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = (&b"hello"[..]) + /// .chain(&b"world"[..]); + /// + /// buf.first_mut().advance(1); + /// + /// let full = buf.copy_to_bytes(9); + /// assert_eq!(full, b"elloworld"[..]); + /// ``` + pub fn first_mut(&mut self) -> &mut T { + &mut self.a + } + + /// Gets a reference to the last underlying `Buf`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let buf = (&b"hello"[..]) + /// .chain(&b"world"[..]); + /// + /// assert_eq!(buf.last_ref()[..], b"world"[..]); + /// ``` + pub fn last_ref(&self) -> &U { + &self.b + } + + /// Gets a mutable reference to the last underlying `Buf`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let mut buf = (&b"hello "[..]) + /// .chain(&b"world"[..]); + /// + /// buf.last_mut().advance(1); + /// + /// let full = buf.copy_to_bytes(10); + /// assert_eq!(full, b"hello orld"[..]); + /// ``` + pub fn last_mut(&mut self) -> &mut U { + &mut self.b + } + + /// Consumes this `Chain`, returning the underlying values. + /// + /// # Examples + /// + /// ``` + /// use bytes::Buf; + /// + /// let chain = (&b"hello"[..]) + /// .chain(&b"world"[..]); + /// + /// let (first, last) = chain.into_inner(); + /// assert_eq!(first[..], b"hello"[..]); + /// assert_eq!(last[..], b"world"[..]); + /// ``` + pub fn into_inner(self) -> (T, U) { + (self.a, self.b) + } +} + +impl<T, U> Buf for Chain<T, U> +where + T: Buf, + U: Buf, +{ + fn remaining(&self) -> usize { + self.a.remaining().saturating_add(self.b.remaining()) + } + + fn chunk(&self) -> &[u8] { + if self.a.has_remaining() { + self.a.chunk() + } else { + self.b.chunk() + } + } + + fn advance(&mut self, mut cnt: usize) { + let a_rem = self.a.remaining(); + + if a_rem != 0 { + if a_rem >= cnt { + self.a.advance(cnt); + return; + } + + // Consume what is left of a + self.a.advance(a_rem); + + cnt -= a_rem; + } + + self.b.advance(cnt); + } + + #[cfg(feature = "std")] + fn chunks_vectored<'a>(&'a self, dst: &mut [IoSlice<'a>]) -> usize { + let mut n = self.a.chunks_vectored(dst); + n += self.b.chunks_vectored(&mut dst[n..]); + n + } + + fn copy_to_bytes(&mut self, len: usize) -> crate::Bytes { + let a_rem = self.a.remaining(); + if a_rem >= len { + self.a.copy_to_bytes(len) + } else if a_rem == 0 { + self.b.copy_to_bytes(len) + } else { + assert!( + len - a_rem <= self.b.remaining(), + "`len` greater than remaining" + ); + let mut ret = crate::BytesMut::with_capacity(len); + ret.put(&mut self.a); + ret.put((&mut self.b).take(len - a_rem)); + ret.freeze() + } + } +} + +unsafe impl<T, U> BufMut for Chain<T, U> +where + T: BufMut, + U: BufMut, +{ + fn remaining_mut(&self) -> usize { + self.a + .remaining_mut() + .saturating_add(self.b.remaining_mut()) + } + + fn chunk_mut(&mut self) -> &mut UninitSlice { + if self.a.has_remaining_mut() { + self.a.chunk_mut() + } else { + self.b.chunk_mut() + } + } + + unsafe fn advance_mut(&mut self, mut cnt: usize) { + let a_rem = self.a.remaining_mut(); + + if a_rem != 0 { + if a_rem >= cnt { + self.a.advance_mut(cnt); + return; + } + + // Consume what is left of a + self.a.advance_mut(a_rem); + + cnt -= a_rem; + } + + self.b.advance_mut(cnt); + } +} + +impl<T, U> IntoIterator for Chain<T, U> +where + T: Buf, + U: Buf, +{ + type Item = u8; + type IntoIter = IntoIter<Chain<T, U>>; + + fn into_iter(self) -> Self::IntoIter { + IntoIter::new(self) + } +} diff --git a/third-party/bytes/src/buf/iter.rs b/third-party/bytes/src/buf/iter.rs new file mode 100644 index 000000000..74f9b991e --- /dev/null +++ b/third-party/bytes/src/buf/iter.rs @@ -0,0 +1,127 @@ +use crate::Buf; + +/// Iterator over the bytes contained by the buffer. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` +/// use bytes::Bytes; +/// +/// let buf = Bytes::from(&b"abc"[..]); +/// let mut iter = buf.into_iter(); +/// +/// assert_eq!(iter.next(), Some(b'a')); +/// assert_eq!(iter.next(), Some(b'b')); +/// assert_eq!(iter.next(), Some(b'c')); +/// assert_eq!(iter.next(), None); +/// ``` +#[derive(Debug)] +pub struct IntoIter<T> { + inner: T, +} + +impl<T> IntoIter<T> { + /// Creates an iterator over the bytes contained by the buffer. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let buf = Bytes::from_static(b"abc"); + /// let mut iter = buf.into_iter(); + /// + /// assert_eq!(iter.next(), Some(b'a')); + /// assert_eq!(iter.next(), Some(b'b')); + /// assert_eq!(iter.next(), Some(b'c')); + /// assert_eq!(iter.next(), None); + /// ``` + pub fn new(inner: T) -> IntoIter<T> { + IntoIter { inner } + } + + /// Consumes this `IntoIter`, returning the underlying value. + /// + /// # Examples + /// + /// ```rust + /// use bytes::{Buf, Bytes}; + /// + /// let buf = Bytes::from(&b"abc"[..]); + /// let mut iter = buf.into_iter(); + /// + /// assert_eq!(iter.next(), Some(b'a')); + /// + /// let buf = iter.into_inner(); + /// assert_eq!(2, buf.remaining()); + /// ``` + pub fn into_inner(self) -> T { + self.inner + } + + /// Gets a reference to the underlying `Buf`. + /// + /// It is inadvisable to directly read from the underlying `Buf`. + /// + /// # Examples + /// + /// ```rust + /// use bytes::{Buf, Bytes}; + /// + /// let buf = Bytes::from(&b"abc"[..]); + /// let mut iter = buf.into_iter(); + /// + /// assert_eq!(iter.next(), Some(b'a')); + /// + /// assert_eq!(2, iter.get_ref().remaining()); + /// ``` + pub fn get_ref(&self) -> &T { + &self.inner + } + + /// Gets a mutable reference to the underlying `Buf`. + /// + /// It is inadvisable to directly read from the underlying `Buf`. + /// + /// # Examples + /// + /// ```rust + /// use bytes::{Buf, BytesMut}; + /// + /// let buf = BytesMut::from(&b"abc"[..]); + /// let mut iter = buf.into_iter(); + /// + /// assert_eq!(iter.next(), Some(b'a')); + /// + /// iter.get_mut().advance(1); + /// + /// assert_eq!(iter.next(), Some(b'c')); + /// ``` + pub fn get_mut(&mut self) -> &mut T { + &mut self.inner + } +} + +impl<T: Buf> Iterator for IntoIter<T> { + type Item = u8; + + fn next(&mut self) -> Option<u8> { + if !self.inner.has_remaining() { + return None; + } + + let b = self.inner.chunk()[0]; + self.inner.advance(1); + + Some(b) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + let rem = self.inner.remaining(); + (rem, Some(rem)) + } +} + +impl<T: Buf> ExactSizeIterator for IntoIter<T> {} diff --git a/third-party/bytes/src/buf/limit.rs b/third-party/bytes/src/buf/limit.rs new file mode 100644 index 000000000..b422be538 --- /dev/null +++ b/third-party/bytes/src/buf/limit.rs @@ -0,0 +1,75 @@ +use crate::buf::UninitSlice; +use crate::BufMut; + +use core::cmp; + +/// A `BufMut` adapter which limits the amount of bytes that can be written +/// to an underlying buffer. +#[derive(Debug)] +pub struct Limit<T> { + inner: T, + limit: usize, +} + +pub(super) fn new<T>(inner: T, limit: usize) -> Limit<T> { + Limit { inner, limit } +} + +impl<T> Limit<T> { + /// Consumes this `Limit`, returning the underlying value. + pub fn into_inner(self) -> T { + self.inner + } + + /// Gets a reference to the underlying `BufMut`. + /// + /// It is inadvisable to directly write to the underlying `BufMut`. + pub fn get_ref(&self) -> &T { + &self.inner + } + + /// Gets a mutable reference to the underlying `BufMut`. + /// + /// It is inadvisable to directly write to the underlying `BufMut`. + pub fn get_mut(&mut self) -> &mut T { + &mut self.inner + } + + /// Returns the maximum number of bytes that can be written + /// + /// # Note + /// + /// If the inner `BufMut` has fewer bytes than indicated by this method then + /// that is the actual number of available bytes. + pub fn limit(&self) -> usize { + self.limit + } + + /// Sets the maximum number of bytes that can be written. + /// + /// # Note + /// + /// If the inner `BufMut` has fewer bytes than `lim` then that is the actual + /// number of available bytes. + pub fn set_limit(&mut self, lim: usize) { + self.limit = lim + } +} + +unsafe impl<T: BufMut> BufMut for Limit<T> { + fn remaining_mut(&self) -> usize { + cmp::min(self.inner.remaining_mut(), self.limit) + } + + fn chunk_mut(&mut self) -> &mut UninitSlice { + let bytes = self.inner.chunk_mut(); + let end = cmp::min(bytes.len(), self.limit); + &mut bytes[..end] + } + + unsafe fn advance_mut(&mut self, cnt: usize) { + assert!(cnt <= self.limit); + self.inner.advance_mut(cnt); + self.limit -= cnt; + } +} diff --git a/third-party/bytes/src/buf/mod.rs b/third-party/bytes/src/buf/mod.rs new file mode 100644 index 000000000..1bf0a47e8 --- /dev/null +++ b/third-party/bytes/src/buf/mod.rs @@ -0,0 +1,39 @@ +//! Utilities for working with buffers. +//! +//! A buffer is any structure that contains a sequence of bytes. The bytes may +//! or may not be stored in contiguous memory. This module contains traits used +//! to abstract over buffers as well as utilities for working with buffer types. +//! +//! # `Buf`, `BufMut` +//! +//! These are the two foundational traits for abstractly working with buffers. +//! They can be thought as iterators for byte structures. They offer additional +//! performance over `Iterator` by providing an API optimized for byte slices. +//! +//! See [`Buf`] and [`BufMut`] for more details. +//! +//! [rope]: https://en.wikipedia.org/wiki/Rope_(data_structure) + +mod buf_impl; +mod buf_mut; +mod chain; +mod iter; +mod limit; +#[cfg(feature = "std")] +mod reader; +mod take; +mod uninit_slice; +mod vec_deque; +#[cfg(feature = "std")] +mod writer; + +pub use self::buf_impl::Buf; +pub use self::buf_mut::BufMut; +pub use self::chain::Chain; +pub use self::iter::IntoIter; +pub use self::limit::Limit; +pub use self::take::Take; +pub use self::uninit_slice::UninitSlice; + +#[cfg(feature = "std")] +pub use self::{reader::Reader, writer::Writer}; diff --git a/third-party/bytes/src/buf/reader.rs b/third-party/bytes/src/buf/reader.rs new file mode 100644 index 000000000..521494958 --- /dev/null +++ b/third-party/bytes/src/buf/reader.rs @@ -0,0 +1,81 @@ +use crate::Buf; + +use std::{cmp, io}; + +/// A `Buf` adapter which implements `io::Read` for the inner value. +/// +/// This struct is generally created by calling `reader()` on `Buf`. See +/// documentation of [`reader()`](Buf::reader) for more +/// details. +#[derive(Debug)] +pub struct Reader<B> { + buf: B, +} + +pub fn new<B>(buf: B) -> Reader<B> { + Reader { buf } +} + +impl<B: Buf> Reader<B> { + /// Gets a reference to the underlying `Buf`. + /// + /// It is inadvisable to directly read from the underlying `Buf`. + /// + /// # Examples + /// + /// ```rust + /// use bytes::Buf; + /// + /// let buf = b"hello world".reader(); + /// + /// assert_eq!(b"hello world", buf.get_ref()); + /// ``` + pub fn get_ref(&self) -> &B { + &self.buf + } + + /// Gets a mutable reference to the underlying `Buf`. + /// + /// It is inadvisable to directly read from the underlying `Buf`. + pub fn get_mut(&mut self) -> &mut B { + &mut self.buf + } + + /// Consumes this `Reader`, returning the underlying value. + /// + /// # Examples + /// + /// ```rust + /// use bytes::Buf; + /// use std::io; + /// + /// let mut buf = b"hello world".reader(); + /// let mut dst = vec![]; + /// + /// io::copy(&mut buf, &mut dst).unwrap(); + /// + /// let buf = buf.into_inner(); + /// assert_eq!(0, buf.remaining()); + /// ``` + pub fn into_inner(self) -> B { + self.buf + } +} + +impl<B: Buf + Sized> io::Read for Reader<B> { + fn read(&mut self, dst: &mut [u8]) -> io::Result<usize> { + let len = cmp::min(self.buf.remaining(), dst.len()); + + Buf::copy_to_slice(&mut self.buf, &mut dst[0..len]); + Ok(len) + } +} + +impl<B: Buf + Sized> io::BufRead for Reader<B> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + Ok(self.buf.chunk()) + } + fn consume(&mut self, amt: usize) { + self.buf.advance(amt) + } +} diff --git a/third-party/bytes/src/buf/take.rs b/third-party/bytes/src/buf/take.rs new file mode 100644 index 000000000..d8621f3f3 --- /dev/null +++ b/third-party/bytes/src/buf/take.rs @@ -0,0 +1,187 @@ +use crate::Buf; + +use core::cmp; + +#[cfg(feature = "std")] +use std::io::IoSlice; + +/// A `Buf` adapter which limits the bytes read from an underlying buffer. +/// +/// This struct is generally created by calling `take()` on `Buf`. See +/// documentation of [`take()`](Buf::take) for more details. +#[derive(Debug)] +pub struct Take<T> { + inner: T, + limit: usize, +} + +pub fn new<T>(inner: T, limit: usize) -> Take<T> { + Take { inner, limit } +} + +impl<T> Take<T> { + /// Consumes this `Take`, returning the underlying value. + /// + /// # Examples + /// + /// ```rust + /// use bytes::{Buf, BufMut}; + /// + /// let mut buf = b"hello world".take(2); + /// let mut dst = vec![]; + /// + /// dst.put(&mut buf); + /// assert_eq!(*dst, b"he"[..]); + /// + /// let mut buf = buf.into_inner(); + /// + /// dst.clear(); + /// dst.put(&mut buf); + /// assert_eq!(*dst, b"llo world"[..]); + /// ``` + pub fn into_inner(self) -> T { + self.inner + } + + /// Gets a reference to the underlying `Buf`. + /// + /// It is inadvisable to directly read from the underlying `Buf`. + /// + /// # Examples + /// + /// ```rust + /// use bytes::Buf; + /// + /// let buf = b"hello world".take(2); + /// + /// assert_eq!(11, buf.get_ref().remaining()); + /// ``` + pub fn get_ref(&self) -> &T { + &self.inner + } + + /// Gets a mutable reference to the underlying `Buf`. + /// + /// It is inadvisable to directly read from the underlying `Buf`. + /// + /// # Examples + /// + /// ```rust + /// use bytes::{Buf, BufMut}; + /// + /// let mut buf = b"hello world".take(2); + /// let mut dst = vec![]; + /// + /// buf.get_mut().advance(2); + /// + /// dst.put(&mut buf); + /// assert_eq!(*dst, b"ll"[..]); + /// ``` + pub fn get_mut(&mut self) -> &mut T { + &mut self.inner + } + + /// Returns the maximum number of bytes that can be read. + /// + /// # Note + /// + /// If the inner `Buf` has fewer bytes than indicated by this method then + /// that is the actual number of available bytes. + /// + /// # Examples + /// + /// ```rust + /// use bytes::Buf; + /// + /// let mut buf = b"hello world".take(2); + /// + /// assert_eq!(2, buf.limit()); + /// assert_eq!(b'h', buf.get_u8()); + /// assert_eq!(1, buf.limit()); + /// ``` + pub fn limit(&self) -> usize { + self.limit + } + + /// Sets the maximum number of bytes that can be read. + /// + /// # Note + /// + /// If the inner `Buf` has fewer bytes than `lim` then that is the actual + /// number of available bytes. + /// + /// # Examples + /// + /// ```rust + /// use bytes::{Buf, BufMut}; + /// + /// let mut buf = b"hello world".take(2); + /// let mut dst = vec![]; + /// + /// dst.put(&mut buf); + /// assert_eq!(*dst, b"he"[..]); + /// + /// dst.clear(); + /// + /// buf.set_limit(3); + /// dst.put(&mut buf); + /// assert_eq!(*dst, b"llo"[..]); + /// ``` + pub fn set_limit(&mut self, lim: usize) { + self.limit = lim + } +} + +impl<T: Buf> Buf for Take<T> { + fn remaining(&self) -> usize { + cmp::min(self.inner.remaining(), self.limit) + } + + fn chunk(&self) -> &[u8] { + let bytes = self.inner.chunk(); + &bytes[..cmp::min(bytes.len(), self.limit)] + } + + fn advance(&mut self, cnt: usize) { + assert!(cnt <= self.limit); + self.inner.advance(cnt); + self.limit -= cnt; + } + + fn copy_to_bytes(&mut self, len: usize) -> crate::Bytes { + assert!(len <= self.remaining(), "`len` greater than remaining"); + + let r = self.inner.copy_to_bytes(len); + self.limit -= len; + r + } + + #[cfg(feature = "std")] + fn chunks_vectored<'a>(&'a self, dst: &mut [IoSlice<'a>]) -> usize { + if self.limit == 0 { + return 0; + } + + const LEN: usize = 16; + let mut slices: [IoSlice<'a>; LEN] = [IoSlice::new(&[]); LEN]; + + let cnt = self + .inner + .chunks_vectored(&mut slices[..dst.len().min(LEN)]); + let mut limit = self.limit; + for (i, (dst, slice)) in dst[..cnt].iter_mut().zip(slices.iter()).enumerate() { + if let Some(buf) = slice.get(..limit) { + // SAFETY: We could do this safely with `IoSlice::advance` if we had a larger MSRV. + let buf = unsafe { std::mem::transmute::<&[u8], &'a [u8]>(buf) }; + *dst = IoSlice::new(buf); + return i + 1; + } else { + // SAFETY: We could do this safely with `IoSlice::advance` if we had a larger MSRV. + let buf = unsafe { std::mem::transmute::<&[u8], &'a [u8]>(slice) }; + *dst = IoSlice::new(buf); + limit -= slice.len(); + } + } + cnt + } +} diff --git a/third-party/bytes/src/buf/uninit_slice.rs b/third-party/bytes/src/buf/uninit_slice.rs new file mode 100644 index 000000000..aea096ae6 --- /dev/null +++ b/third-party/bytes/src/buf/uninit_slice.rs @@ -0,0 +1,257 @@ +use core::fmt; +use core::mem::MaybeUninit; +use core::ops::{ + Index, IndexMut, Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive, +}; + +/// Uninitialized byte slice. +/// +/// Returned by `BufMut::chunk_mut()`, the referenced byte slice may be +/// uninitialized. The wrapper provides safe access without introducing +/// undefined behavior. +/// +/// The safety invariants of this wrapper are: +/// +/// 1. Reading from an `UninitSlice` is undefined behavior. +/// 2. Writing uninitialized bytes to an `UninitSlice` is undefined behavior. +/// +/// The difference between `&mut UninitSlice` and `&mut [MaybeUninit<u8>]` is +/// that it is possible in safe code to write uninitialized bytes to an +/// `&mut [MaybeUninit<u8>]`, which this type prohibits. +#[repr(transparent)] +pub struct UninitSlice([MaybeUninit<u8>]); + +impl UninitSlice { + /// Creates a `&mut UninitSlice` wrapping a slice of initialised memory. + /// + /// # Examples + /// + /// ``` + /// use bytes::buf::UninitSlice; + /// + /// let mut buffer = [0u8; 64]; + /// let slice = UninitSlice::new(&mut buffer[..]); + /// ``` + #[inline] + pub fn new(slice: &mut [u8]) -> &mut UninitSlice { + unsafe { &mut *(slice as *mut [u8] as *mut [MaybeUninit<u8>] as *mut UninitSlice) } + } + + /// Creates a `&mut UninitSlice` wrapping a slice of uninitialised memory. + /// + /// # Examples + /// + /// ``` + /// use bytes::buf::UninitSlice; + /// use core::mem::MaybeUninit; + /// + /// let mut buffer = [MaybeUninit::uninit(); 64]; + /// let slice = UninitSlice::uninit(&mut buffer[..]); + /// + /// let mut vec = Vec::with_capacity(1024); + /// let spare: &mut UninitSlice = vec.spare_capacity_mut().into(); + /// ``` + #[inline] + pub fn uninit(slice: &mut [MaybeUninit<u8>]) -> &mut UninitSlice { + unsafe { &mut *(slice as *mut [MaybeUninit<u8>] as *mut UninitSlice) } + } + + fn uninit_ref(slice: &[MaybeUninit<u8>]) -> &UninitSlice { + unsafe { &*(slice as *const [MaybeUninit<u8>] as *const UninitSlice) } + } + + /// Create a `&mut UninitSlice` from a pointer and a length. + /// + /// # Safety + /// + /// The caller must ensure that `ptr` references a valid memory region owned + /// by the caller representing a byte slice for the duration of `'a`. + /// + /// # Examples + /// + /// ``` + /// use bytes::buf::UninitSlice; + /// + /// let bytes = b"hello world".to_vec(); + /// let ptr = bytes.as_ptr() as *mut _; + /// let len = bytes.len(); + /// + /// let slice = unsafe { UninitSlice::from_raw_parts_mut(ptr, len) }; + /// ``` + #[inline] + pub unsafe fn from_raw_parts_mut<'a>(ptr: *mut u8, len: usize) -> &'a mut UninitSlice { + let maybe_init: &mut [MaybeUninit<u8>] = + core::slice::from_raw_parts_mut(ptr as *mut _, len); + Self::uninit(maybe_init) + } + + /// Write a single byte at the specified offset. + /// + /// # Panics + /// + /// The function panics if `index` is out of bounds. + /// + /// # Examples + /// + /// ``` + /// use bytes::buf::UninitSlice; + /// + /// let mut data = [b'f', b'o', b'o']; + /// let slice = unsafe { UninitSlice::from_raw_parts_mut(data.as_mut_ptr(), 3) }; + /// + /// slice.write_byte(0, b'b'); + /// + /// assert_eq!(b"boo", &data[..]); + /// ``` + #[inline] + pub fn write_byte(&mut self, index: usize, byte: u8) { + assert!(index < self.len()); + + unsafe { self[index..].as_mut_ptr().write(byte) } + } + + /// Copies bytes from `src` into `self`. + /// + /// The length of `src` must be the same as `self`. + /// + /// # Panics + /// + /// The function panics if `src` has a different length than `self`. + /// + /// # Examples + /// + /// ``` + /// use bytes::buf::UninitSlice; + /// + /// let mut data = [b'f', b'o', b'o']; + /// let slice = unsafe { UninitSlice::from_raw_parts_mut(data.as_mut_ptr(), 3) }; + /// + /// slice.copy_from_slice(b"bar"); + /// + /// assert_eq!(b"bar", &data[..]); + /// ``` + #[inline] + pub fn copy_from_slice(&mut self, src: &[u8]) { + use core::ptr; + + assert_eq!(self.len(), src.len()); + + unsafe { + ptr::copy_nonoverlapping(src.as_ptr(), self.as_mut_ptr(), self.len()); + } + } + + /// Return a raw pointer to the slice's buffer. + /// + /// # Safety + /// + /// The caller **must not** read from the referenced memory and **must not** + /// write **uninitialized** bytes to the slice either. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut data = [0, 1, 2]; + /// let mut slice = &mut data[..]; + /// let ptr = BufMut::chunk_mut(&mut slice).as_mut_ptr(); + /// ``` + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut u8 { + self.0.as_mut_ptr() as *mut _ + } + + /// Return a `&mut [MaybeUninit<u8>]` to this slice's buffer. + /// + /// # Safety + /// + /// The caller **must not** read from the referenced memory and **must not** write + /// **uninitialized** bytes to the slice either. This is because `BufMut` implementation + /// that created the `UninitSlice` knows which parts are initialized. Writing uninitialized + /// bytes to the slice may cause the `BufMut` to read those bytes and trigger undefined + /// behavior. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut data = [0, 1, 2]; + /// let mut slice = &mut data[..]; + /// unsafe { + /// let uninit_slice = BufMut::chunk_mut(&mut slice).as_uninit_slice_mut(); + /// }; + /// ``` + #[inline] + pub unsafe fn as_uninit_slice_mut(&mut self) -> &mut [MaybeUninit<u8>] { + &mut self.0 + } + + /// Returns the number of bytes in the slice. + /// + /// # Examples + /// + /// ``` + /// use bytes::BufMut; + /// + /// let mut data = [0, 1, 2]; + /// let mut slice = &mut data[..]; + /// let len = BufMut::chunk_mut(&mut slice).len(); + /// + /// assert_eq!(len, 3); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } +} + +impl fmt::Debug for UninitSlice { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("UninitSlice[...]").finish() + } +} + +impl<'a> From<&'a mut [u8]> for &'a mut UninitSlice { + fn from(slice: &'a mut [u8]) -> Self { + UninitSlice::new(slice) + } +} + +impl<'a> From<&'a mut [MaybeUninit<u8>]> for &'a mut UninitSlice { + fn from(slice: &'a mut [MaybeUninit<u8>]) -> Self { + UninitSlice::uninit(slice) + } +} + +macro_rules! impl_index { + ($($t:ty),*) => { + $( + impl Index<$t> for UninitSlice { + type Output = UninitSlice; + + #[inline] + fn index(&self, index: $t) -> &UninitSlice { + UninitSlice::uninit_ref(&self.0[index]) + } + } + + impl IndexMut<$t> for UninitSlice { + #[inline] + fn index_mut(&mut self, index: $t) -> &mut UninitSlice { + UninitSlice::uninit(&mut self.0[index]) + } + } + )* + }; +} + +impl_index!( + Range<usize>, + RangeFrom<usize>, + RangeFull, + RangeInclusive<usize>, + RangeTo<usize>, + RangeToInclusive<usize> +); diff --git a/third-party/bytes/src/buf/vec_deque.rs b/third-party/bytes/src/buf/vec_deque.rs new file mode 100644 index 000000000..55d5636b1 --- /dev/null +++ b/third-party/bytes/src/buf/vec_deque.rs @@ -0,0 +1,40 @@ +use alloc::collections::VecDeque; +#[cfg(feature = "std")] +use std::io; + +use super::Buf; + +impl Buf for VecDeque<u8> { + fn remaining(&self) -> usize { + self.len() + } + + fn chunk(&self) -> &[u8] { + let (s1, s2) = self.as_slices(); + if s1.is_empty() { + s2 + } else { + s1 + } + } + + #[cfg(feature = "std")] + fn chunks_vectored<'a>(&'a self, dst: &mut [io::IoSlice<'a>]) -> usize { + if self.is_empty() || dst.is_empty() { + return 0; + } + + let (s1, s2) = self.as_slices(); + dst[0] = io::IoSlice::new(s1); + if s2.is_empty() || dst.len() == 1 { + return 1; + } + + dst[1] = io::IoSlice::new(s2); + 2 + } + + fn advance(&mut self, cnt: usize) { + self.drain(..cnt); + } +} diff --git a/third-party/bytes/src/buf/writer.rs b/third-party/bytes/src/buf/writer.rs new file mode 100644 index 000000000..e72348f40 --- /dev/null +++ b/third-party/bytes/src/buf/writer.rs @@ -0,0 +1,88 @@ +use crate::BufMut; + +use std::{cmp, io}; + +/// A `BufMut` adapter which implements `io::Write` for the inner value. +/// +/// This struct is generally created by calling `writer()` on `BufMut`. See +/// documentation of [`writer()`](BufMut::writer) for more +/// details. +#[derive(Debug)] +pub struct Writer<B> { + buf: B, +} + +pub fn new<B>(buf: B) -> Writer<B> { + Writer { buf } +} + +impl<B: BufMut> Writer<B> { + /// Gets a reference to the underlying `BufMut`. + /// + /// It is inadvisable to directly write to the underlying `BufMut`. + /// + /// # Examples + /// + /// ```rust + /// use bytes::BufMut; + /// + /// let buf = Vec::with_capacity(1024).writer(); + /// + /// assert_eq!(1024, buf.get_ref().capacity()); + /// ``` + pub fn get_ref(&self) -> &B { + &self.buf + } + + /// Gets a mutable reference to the underlying `BufMut`. + /// + /// It is inadvisable to directly write to the underlying `BufMut`. + /// + /// # Examples + /// + /// ```rust + /// use bytes::BufMut; + /// + /// let mut buf = vec![].writer(); + /// + /// buf.get_mut().reserve(1024); + /// + /// assert_eq!(1024, buf.get_ref().capacity()); + /// ``` + pub fn get_mut(&mut self) -> &mut B { + &mut self.buf + } + + /// Consumes this `Writer`, returning the underlying value. + /// + /// # Examples + /// + /// ```rust + /// use bytes::BufMut; + /// use std::io; + /// + /// let mut buf = vec![].writer(); + /// let mut src = &b"hello world"[..]; + /// + /// io::copy(&mut src, &mut buf).unwrap(); + /// + /// let buf = buf.into_inner(); + /// assert_eq!(*buf, b"hello world"[..]); + /// ``` + pub fn into_inner(self) -> B { + self.buf + } +} + +impl<B: BufMut + Sized> io::Write for Writer<B> { + fn write(&mut self, src: &[u8]) -> io::Result<usize> { + let n = cmp::min(self.buf.remaining_mut(), src.len()); + + self.buf.put_slice(&src[..n]); + Ok(n) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/third-party/bytes/src/bytes.rs b/third-party/bytes/src/bytes.rs new file mode 100644 index 000000000..c8ea71c55 --- /dev/null +++ b/third-party/bytes/src/bytes.rs @@ -0,0 +1,1677 @@ +mod accounted_owner; +mod owner_lease; +pub use accounted_owner::OwnerAllocation; +pub use owner_lease::{OwnerAllocationContext, OwnerAllocationError, OwnerAllocationLease}; + +use core::mem::{self, ManuallyDrop}; +use core::ops::{Deref, RangeBounds}; +use core::ptr::NonNull; +use core::{cmp, fmt, hash, ptr, slice}; + +use alloc::{ + alloc::{dealloc, Layout}, + borrow::Borrow, + boxed::Box, + string::String, + vec::Vec, +}; + +use crate::buf::IntoIter; +#[allow(unused)] +use crate::loom::sync::atomic::AtomicMut; +use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; +use crate::{Buf, BytesMut}; + +/// A cheaply cloneable and sliceable chunk of contiguous memory. +/// +/// `Bytes` is an efficient container for storing and operating on contiguous +/// slices of memory. It is intended for use primarily in networking code, but +/// could have applications elsewhere as well. +/// +/// `Bytes` values facilitate zero-copy network programming by allowing multiple +/// `Bytes` objects to point to the same underlying memory. +/// +/// `Bytes` does not have a single implementation. It is an interface, whose +/// exact behavior is implemented through dynamic dispatch in several underlying +/// implementations of `Bytes`. +/// +/// All `Bytes` implementations must fulfill the following requirements: +/// - They are cheaply cloneable and thereby shareable between an unlimited amount +/// of components, for example by modifying a reference count. +/// - Instances can be sliced to refer to a subset of the original buffer. +/// +/// ``` +/// use bytes::Bytes; +/// +/// let mut mem = Bytes::from("Hello world"); +/// let a = mem.slice(0..5); +/// +/// assert_eq!(a, "Hello"); +/// +/// let b = mem.split_to(6); +/// +/// assert_eq!(mem, "world"); +/// assert_eq!(b, "Hello "); +/// ``` +/// +/// # Memory layout +/// +/// The `Bytes` struct itself is fairly small, limited to 4 `usize` fields used +/// to track information about which segment of the underlying memory the +/// `Bytes` handle has access to. +/// +/// `Bytes` keeps both a pointer to the shared state containing the full memory +/// slice and a pointer to the start of the region visible by the handle. +/// `Bytes` also tracks the length of its view into the memory. +/// +/// # Sharing +/// +/// `Bytes` contains a vtable, which allows implementations of `Bytes` to define +/// how sharing/cloning is implemented in detail. +/// When `Bytes::clone()` is called, `Bytes` will call the vtable function for +/// cloning the backing storage in order to share it behind multiple `Bytes` +/// instances. +/// +/// For `Bytes` implementations which refer to constant memory (e.g. created +/// via `Bytes::from_static()`) the cloning implementation will be a no-op. +/// +/// For `Bytes` implementations which point to a reference counted shared storage +/// (e.g. an `Arc<[u8]>`), sharing will be implemented by increasing the +/// reference count. +/// +/// Due to this mechanism, multiple `Bytes` instances may point to the same +/// shared memory region. +/// Each `Bytes` instance can point to different sections within that +/// memory region, and `Bytes` instances may or may not have overlapping views +/// into the memory. +/// +/// The following diagram visualizes a scenario where 2 `Bytes` instances make +/// use of an `Arc`-based backing storage, and provide access to different views: +/// +/// ```text +/// +/// Arc ptrs ┌─────────┐ +/// ________________________ / │ Bytes 2 │ +/// / └─────────┘ +/// / ┌───────────┐ | | +/// |_________/ │ Bytes 1 │ | | +/// | └───────────┘ | | +/// | | | ___/ data | tail +/// | data | tail |/ | +/// v v v v +/// ┌─────┬─────┬───────────┬───────────────┬─────┐ +/// │ Arc │ │ │ │ │ +/// └─────┴─────┴───────────┴───────────────┴─────┘ +/// ``` +pub struct Bytes { + ptr: *const u8, + len: usize, + // inlined "trait object" + data: AtomicPtr<()>, + vtable: &'static Vtable, +} + +pub(crate) struct Vtable { + /// fn(data, ptr, len) + pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes, + /// fn(data, ptr, len) + /// + /// `into_*` consumes the `Bytes`, returning the respective value. + pub into_vec: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Vec<u8>, + pub into_mut: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> BytesMut, + /// fn(data) + pub is_unique: unsafe fn(&AtomicPtr<()>) -> bool, + /// fn(data, ptr, len) + pub drop: unsafe fn(&mut AtomicPtr<()>, *const u8, usize), + pub owner_allocation: Option<unsafe fn(&AtomicPtr<()>, owner_lease::Action, core::alloc::Layout) -> bool>, +} + +impl Bytes { + /// Creates a new empty `Bytes`. + /// + /// This will not allocate and the returned `Bytes` handle will be empty. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let b = Bytes::new(); + /// assert_eq!(&b[..], b""); + /// ``` + #[inline] + #[cfg(not(all(loom, test)))] + pub const fn new() -> Self { + // Make it a named const to work around + // "unsizing casts are not allowed in const fn" + const EMPTY: &[u8] = &[]; + Bytes::from_static(EMPTY) + } + + /// Creates a new empty `Bytes`. + #[cfg(all(loom, test))] + pub fn new() -> Self { + const EMPTY: &[u8] = &[]; + Bytes::from_static(EMPTY) + } + + /// Creates a new `Bytes` from a static slice. + /// + /// The returned `Bytes` will point directly to the static slice. There is + /// no allocating or copying. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let b = Bytes::from_static(b"hello"); + /// assert_eq!(&b[..], b"hello"); + /// ``` + #[inline] + #[cfg(not(all(loom, test)))] + pub const fn from_static(bytes: &'static [u8]) -> Self { + Bytes { + ptr: bytes.as_ptr(), + len: bytes.len(), + data: AtomicPtr::new(ptr::null_mut()), + vtable: &STATIC_VTABLE, + } + } + + /// Creates a new `Bytes` from a static slice. + #[cfg(all(loom, test))] + pub fn from_static(bytes: &'static [u8]) -> Self { + Bytes { + ptr: bytes.as_ptr(), + len: bytes.len(), + data: AtomicPtr::new(ptr::null_mut()), + vtable: &STATIC_VTABLE, + } + } + + /// Creates a new `Bytes` with length zero and the given pointer as the address. + fn new_empty_with_ptr(ptr: *const u8) -> Self { + debug_assert!(!ptr.is_null()); + + // Detach this pointer's provenance from whichever allocation it came from, and reattach it + // to the provenance of the fake ZST [u8;0] at the same address. + let ptr = without_provenance(ptr as usize); + + Bytes { + ptr, + len: 0, + data: AtomicPtr::new(ptr::null_mut()), + vtable: &STATIC_VTABLE, + } + } + + /// Create [Bytes] with a buffer whose lifetime is controlled + /// via an explicit owner. + /// + /// A common use case is to zero-copy construct from mapped memory. + /// + /// ``` + /// # struct File; + /// # + /// # impl File { + /// # pub fn open(_: &str) -> Result<Self, ()> { + /// # Ok(Self) + /// # } + /// # } + /// # + /// # mod memmap2 { + /// # pub struct Mmap; + /// # + /// # impl Mmap { + /// # pub unsafe fn map(_file: &super::File) -> Result<Self, ()> { + /// # Ok(Self) + /// # } + /// # } + /// # + /// # impl AsRef<[u8]> for Mmap { + /// # fn as_ref(&self) -> &[u8] { + /// # b"buf" + /// # } + /// # } + /// # } + /// use bytes::Bytes; + /// use memmap2::Mmap; + /// + /// # fn main() -> Result<(), ()> { + /// let file = File::open("upload_bundle.tar.gz")?; + /// let mmap = unsafe { Mmap::map(&file) }?; + /// let b = Bytes::from_owner(mmap); + /// # Ok(()) + /// # } + /// ``` + /// + /// The `owner` will be transferred to the constructed [Bytes] object, which + /// will ensure it is dropped once all remaining clones of the constructed + /// object are dropped. The owner will then be responsible for dropping the + /// specified region of memory as part of its [Drop] implementation. + /// + /// Note that converting [Bytes] constructed from an owner into a [BytesMut] + /// will always create a deep copy of the buffer into newly allocated memory. + pub fn from_owner<T>(owner: T) -> Self + where + T: AsRef<[u8]> + Send + 'static, + { + // Safety & Miri: + // The ownership of `owner` is first transferred to the `Owned` wrapper and `Bytes` object. + // This ensures that the owner is pinned in memory, allowing us to call `.as_ref()` safely + // since the lifetime of the owner is controlled by the lifetime of the new `Bytes` object, + // and the lifetime of the resulting borrowed `&[u8]` matches that of the owner. + // Note that this remains safe so long as we only call `.as_ref()` once. + // + // There are some additional special considerations here: + // * We rely on Bytes's Drop impl to clean up memory should `.as_ref()` panic. + // * Setting the `ptr` and `len` on the bytes object last (after moving the owner to + // Bytes) allows Miri checks to pass since it avoids obtaining the `&[u8]` slice + // from a stack-owned Box. + // More details on this: https://github.com/tokio-rs/bytes/pull/742/#discussion_r1813375863 + // and: https://github.com/tokio-rs/bytes/pull/742/#discussion_r1813316032 + + let owned = Box::into_raw(Box::new(Owned { + ref_cnt: AtomicUsize::new(1), + owner, + })); + + let mut ret = Bytes { + ptr: NonNull::dangling().as_ptr(), + len: 0, + data: AtomicPtr::new(owned.cast()), + vtable: &Owned::<T>::VTABLE, + }; + + let buf = unsafe { &*owned }.owner.as_ref(); + ret.ptr = buf.as_ptr(); + ret.len = buf.len(); + + ret + } + + /// Returns the number of bytes contained in this `Bytes`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let b = Bytes::from(&b"hello"[..]); + /// assert_eq!(b.len(), 5); + /// ``` + #[inline] + pub const fn len(&self) -> usize { + self.len + } + + /// Returns true if the `Bytes` has a length of 0. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let b = Bytes::new(); + /// assert!(b.is_empty()); + /// ``` + #[inline] + pub const fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Returns true if this is the only reference to the data and + /// `Into<BytesMut>` would avoid cloning the underlying buffer. + /// + /// Always returns false if the data is backed by a [static slice](Bytes::from_static), + /// or an [owner](Bytes::from_owner). + /// + /// The result of this method may be invalidated immediately if another + /// thread clones this value while this is being called. Ensure you have + /// unique access to this value (`&mut Bytes`) first if you need to be + /// certain the result is valid (i.e. for safety reasons). + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let a = Bytes::from(vec![1, 2, 3]); + /// assert!(a.is_unique()); + /// let b = a.clone(); + /// assert!(!a.is_unique()); + /// ``` + pub fn is_unique(&self) -> bool { + unsafe { (self.vtable.is_unique)(&self.data) } + } + + /// Creates `Bytes` instance from slice, by copying it. + pub fn copy_from_slice(data: &[u8]) -> Self { + data.to_vec().into() + } + + /// Returns a slice of self for the provided range. + /// + /// This will increment the reference count for the underlying memory and + /// return a new `Bytes` handle set to the slice. + /// + /// This operation is `O(1)`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let a = Bytes::from(&b"hello world"[..]); + /// let b = a.slice(2..5); + /// + /// assert_eq!(&b[..], b"llo"); + /// ``` + /// + /// # Panics + /// + /// Requires that `begin <= end` and `end <= self.len()`, otherwise slicing + /// will panic. + pub fn slice(&self, range: impl RangeBounds<usize>) -> Self { + use core::ops::Bound; + + let len = self.len(); + + let begin = match range.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n.checked_add(1).expect("out of range"), + Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + Bound::Included(&n) => n.checked_add(1).expect("out of range"), + Bound::Excluded(&n) => n, + Bound::Unbounded => len, + }; + + assert!( + begin <= end, + "range start must not be greater than end: {:?} <= {:?}", + begin, + end, + ); + assert!( + end <= len, + "range end out of bounds: {:?} <= {:?}", + end, + len, + ); + + if end == begin { + return Bytes::new_empty_with_ptr(self.ptr.wrapping_add(begin)); + } + + let mut ret = self.clone(); + + ret.len = end - begin; + ret.ptr = unsafe { ret.ptr.add(begin) }; + + ret + } + + /// Returns a slice of self that is equivalent to the given `subset`. + /// + /// When processing a `Bytes` buffer with other tools, one often gets a + /// `&[u8]` which is in fact a slice of the `Bytes`, i.e. a subset of it. + /// This function turns that `&[u8]` into another `Bytes`, as if one had + /// called `self.slice()` with the offsets that correspond to `subset`. + /// + /// This operation is `O(1)`. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let bytes = Bytes::from(&b"012345678"[..]); + /// let as_slice = bytes.as_ref(); + /// let subset = &as_slice[2..6]; + /// let subslice = bytes.slice_ref(&subset); + /// assert_eq!(&subslice[..], b"2345"); + /// ``` + /// + /// # Panics + /// + /// Requires that the given `sub` slice is in fact contained within the + /// `Bytes` buffer; otherwise this function will panic. + pub fn slice_ref(&self, subset: &[u8]) -> Self { + // Empty slice and empty Bytes may have their pointers reset + // so explicitly allow empty slice to be a subslice of any slice. + if subset.is_empty() { + return Bytes::new(); + } + + let bytes_p = self.as_ptr() as usize; + let bytes_len = self.len(); + + let sub_p = subset.as_ptr() as usize; + let sub_len = subset.len(); + + assert!( + sub_p >= bytes_p, + "subset pointer ({:p}) is smaller than self pointer ({:p})", + subset.as_ptr(), + self.as_ptr(), + ); + assert!( + sub_p + sub_len <= bytes_p + bytes_len, + "subset is out of bounds: self = ({:p}, {}), subset = ({:p}, {})", + self.as_ptr(), + bytes_len, + subset.as_ptr(), + sub_len, + ); + + let sub_offset = sub_p - bytes_p; + + self.slice(sub_offset..(sub_offset + sub_len)) + } + + /// Splits the bytes into two at the given index. + /// + /// Afterwards `self` contains elements `[0, at)`, and the returned `Bytes` + /// contains elements `[at, len)`. It's guaranteed that the memory does not + /// move, that is, the address of `self` does not change, and the address of + /// the returned slice is `at` bytes after that. + /// + /// This is an `O(1)` operation that just increases the reference count and + /// sets a few indices. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let mut a = Bytes::from(&b"hello world"[..]); + /// let b = a.split_off(5); + /// + /// assert_eq!(&a[..], b"hello"); + /// assert_eq!(&b[..], b" world"); + /// ``` + /// + /// # Panics + /// + /// Panics if `at > len`. + #[must_use = "consider Bytes::truncate if you don't need the other half"] + pub fn split_off(&mut self, at: usize) -> Self { + if at == self.len() { + return Bytes::new_empty_with_ptr(self.ptr.wrapping_add(at)); + } + + if at == 0 { + return mem::replace(self, Bytes::new_empty_with_ptr(self.ptr)); + } + + assert!( + at <= self.len(), + "split_off out of bounds: {:?} <= {:?}", + at, + self.len(), + ); + + let mut ret = self.clone(); + + self.len = at; + + unsafe { ret.inc_start(at) }; + + ret + } + + /// Splits the bytes into two at the given index. + /// + /// Afterwards `self` contains elements `[at, len)`, and the returned + /// `Bytes` contains elements `[0, at)`. + /// + /// This is an `O(1)` operation that just increases the reference count and + /// sets a few indices. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let mut a = Bytes::from(&b"hello world"[..]); + /// let b = a.split_to(5); + /// + /// assert_eq!(&a[..], b" world"); + /// assert_eq!(&b[..], b"hello"); + /// ``` + /// + /// # Panics + /// + /// Panics if `at > len`. + #[must_use = "consider Bytes::advance if you don't need the other half"] + pub fn split_to(&mut self, at: usize) -> Self { + if at == self.len() { + let end_ptr = self.ptr.wrapping_add(at); + return mem::replace(self, Bytes::new_empty_with_ptr(end_ptr)); + } + + if at == 0 { + return Bytes::new_empty_with_ptr(self.ptr); + } + + assert!( + at <= self.len(), + "split_to out of bounds: {:?} <= {:?}", + at, + self.len(), + ); + + let mut ret = self.clone(); + + unsafe { self.inc_start(at) }; + + ret.len = at; + ret + } + + /// Shortens the buffer, keeping the first `len` bytes and dropping the + /// rest. + /// + /// If `len` is greater than the buffer's current length, this has no + /// effect. + /// + /// The [split_off](`Self::split_off()`) method can emulate `truncate`, but this causes the + /// excess bytes to be returned instead of dropped. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let mut buf = Bytes::from(&b"hello world"[..]); + /// buf.truncate(5); + /// assert_eq!(buf, b"hello"[..]); + /// ``` + #[inline] + pub fn truncate(&mut self, len: usize) { + if len < self.len { + // The Vec "promotable" vtables do not store the capacity, + // so we cannot truncate while using this repr. We *have* to + // promote using `split_off` so the capacity can be stored. + if self.vtable as *const Vtable == &PROMOTABLE_EVEN_VTABLE + || self.vtable as *const Vtable == &PROMOTABLE_ODD_VTABLE + { + drop(self.split_off(len)); + } else { + self.len = len; + } + } + } + + /// Clears the buffer, removing all data. + /// + /// # Examples + /// + /// ``` + /// use bytes::Bytes; + /// + /// let mut buf = Bytes::from(&b"hello world"[..]); + /// buf.clear(); + /// assert!(buf.is_empty()); + /// ``` + #[inline] + pub fn clear(&mut self) { + self.truncate(0); + } + + /// Try to convert self into `BytesMut`. + /// + /// If `self` is unique for the entire original buffer, this will succeed + /// and return a `BytesMut` with the contents of `self` without copying. + /// If `self` is not unique for the entire original buffer, this will fail + /// and return self. + /// + /// This will also always fail if the buffer was constructed via either + /// [from_owner](Bytes::from_owner) or [from_static](Bytes::from_static). + /// + /// # Examples + /// + /// ``` + /// use bytes::{Bytes, BytesMut}; + /// + /// let bytes = Bytes::from(b"hello".to_vec()); + /// assert_eq!(bytes.try_into_mut(), Ok(BytesMut::from(&b"hello"[..]))); + /// ``` + pub fn try_into_mut(self) -> Result<BytesMut, Bytes> { + if self.is_unique() { + Ok(self.into()) + } else { + Err(self) + } + } + + #[inline] + pub(crate) unsafe fn with_vtable( + ptr: *const u8, + len: usize, + data: AtomicPtr<()>, + vtable: &'static Vtable, + ) -> Bytes { + Bytes { + ptr, + len, + data, + vtable, + } + } + + // private + + #[inline] + fn as_slice(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self.ptr, self.len) } + } + + #[inline] + unsafe fn inc_start(&mut self, by: usize) { + // should already be asserted, but debug assert for tests + debug_assert!(self.len >= by, "internal: inc_start out of bounds"); + self.len -= by; + self.ptr = self.ptr.add(by); + } +} + +// Vtable must enforce this behavior +unsafe impl Send for Bytes {} +unsafe impl Sync for Bytes {} + +impl Drop for Bytes { + #[inline] + fn drop(&mut self) { + unsafe { (self.vtable.drop)(&mut self.data, self.ptr, self.len) } + } +} + +impl Clone for Bytes { + #[inline] + fn clone(&self) -> Bytes { + unsafe { (self.vtable.clone)(&self.data, self.ptr, self.len) } + } +} + +impl Buf for Bytes { + #[inline] + fn remaining(&self) -> usize { + self.len() + } + + #[inline] + fn chunk(&self) -> &[u8] { + self.as_slice() + } + + #[inline] + fn advance(&mut self, cnt: usize) { + assert!( + cnt <= self.len(), + "cannot advance past `remaining`: {:?} <= {:?}", + cnt, + self.len(), + ); + + unsafe { + self.inc_start(cnt); + } + } + + fn copy_to_bytes(&mut self, len: usize) -> Self { + self.split_to(len) + } +} + +impl Deref for Bytes { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + self.as_slice() + } +} + +impl AsRef<[u8]> for Bytes { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + +impl hash::Hash for Bytes { + fn hash<H>(&self, state: &mut H) + where + H: hash::Hasher, + { + self.as_slice().hash(state); + } +} + +impl Borrow<[u8]> for Bytes { + fn borrow(&self) -> &[u8] { + self.as_slice() + } +} + +impl IntoIterator for Bytes { + type Item = u8; + type IntoIter = IntoIter<Bytes>; + + fn into_iter(self) -> Self::IntoIter { + IntoIter::new(self) + } +} + +impl<'a> IntoIterator for &'a Bytes { + type Item = &'a u8; + type IntoIter = core::slice::Iter<'a, u8>; + + fn into_iter(self) -> Self::IntoIter { + self.as_slice().iter() + } +} + +impl FromIterator<u8> for Bytes { + fn from_iter<T: IntoIterator<Item = u8>>(into_iter: T) -> Self { + Vec::from_iter(into_iter).into() + } +} + +// impl Eq + +impl PartialEq for Bytes { + fn eq(&self, other: &Bytes) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl PartialOrd for Bytes { + fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for Bytes { + fn cmp(&self, other: &Bytes) -> cmp::Ordering { + self.as_slice().cmp(other.as_slice()) + } +} + +impl Eq for Bytes {} + +impl PartialEq<[u8]> for Bytes { + fn eq(&self, other: &[u8]) -> bool { + self.as_slice() == other + } +} + +impl PartialOrd<[u8]> for Bytes { + fn partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering> { + self.as_slice().partial_cmp(other) + } +} + +impl PartialEq<Bytes> for [u8] { + fn eq(&self, other: &Bytes) -> bool { + *other == *self + } +} + +impl PartialOrd<Bytes> for [u8] { + fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other) + } +} + +impl PartialEq<str> for Bytes { + fn eq(&self, other: &str) -> bool { + self.as_slice() == other.as_bytes() + } +} + +impl PartialOrd<str> for Bytes { + fn partial_cmp(&self, other: &str) -> Option<cmp::Ordering> { + self.as_slice().partial_cmp(other.as_bytes()) + } +} + +impl PartialEq<Bytes> for str { + fn eq(&self, other: &Bytes) -> bool { + *other == *self + } +} + +impl PartialOrd<Bytes> for str { + fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other) + } +} + +impl PartialEq<Vec<u8>> for Bytes { + fn eq(&self, other: &Vec<u8>) -> bool { + *self == other[..] + } +} + +impl PartialOrd<Vec<u8>> for Bytes { + fn partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering> { + self.as_slice().partial_cmp(&other[..]) + } +} + +impl PartialEq<Bytes> for Vec<u8> { + fn eq(&self, other: &Bytes) -> bool { + *other == *self + } +} + +impl PartialOrd<Bytes> for Vec<u8> { + fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other) + } +} + +impl PartialEq<String> for Bytes { + fn eq(&self, other: &String) -> bool { + *self == other[..] + } +} + +impl PartialOrd<String> for Bytes { + fn partial_cmp(&self, other: &String) -> Option<cmp::Ordering> { + self.as_slice().partial_cmp(other.as_bytes()) + } +} + +impl PartialEq<Bytes> for String { + fn eq(&self, other: &Bytes) -> bool { + *other == *self + } +} + +impl PartialOrd<Bytes> for String { + fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other) + } +} + +impl PartialEq<Bytes> for &[u8] { + fn eq(&self, other: &Bytes) -> bool { + *other == *self + } +} + +impl PartialOrd<Bytes> for &[u8] { + fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other) + } +} + +impl PartialEq<Bytes> for &str { + fn eq(&self, other: &Bytes) -> bool { + *other == *self + } +} + +impl PartialOrd<Bytes> for &str { + fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other) + } +} + +impl<'a, T: ?Sized> PartialEq<&'a T> for Bytes +where + Bytes: PartialEq<T>, +{ + fn eq(&self, other: &&'a T) -> bool { + *self == **other + } +} + +impl<'a, T: ?Sized> PartialOrd<&'a T> for Bytes +where + Bytes: PartialOrd<T>, +{ + fn partial_cmp(&self, other: &&'a T) -> Option<cmp::Ordering> { + self.partial_cmp(&**other) + } +} + +// impl From + +impl Default for Bytes { + #[inline] + fn default() -> Bytes { + Bytes::new() + } +} + +impl From<&'static [u8]> for Bytes { + fn from(slice: &'static [u8]) -> Bytes { + Bytes::from_static(slice) + } +} + +impl From<&'static str> for Bytes { + fn from(slice: &'static str) -> Bytes { + Bytes::from_static(slice.as_bytes()) + } +} + +impl From<Vec<u8>> for Bytes { + fn from(vec: Vec<u8>) -> Bytes { + let mut vec = ManuallyDrop::new(vec); + let ptr = vec.as_mut_ptr(); + let len = vec.len(); + let cap = vec.capacity(); + + // Avoid an extra allocation if possible. + if len == cap { + let vec = ManuallyDrop::into_inner(vec); + return Bytes::from(vec.into_boxed_slice()); + } + + let shared = Box::new(Shared { + buf: ptr, + cap, + ref_cnt: AtomicUsize::new(1), + }); + + let shared = Box::into_raw(shared); + // The pointer should be aligned, so this assert should + // always succeed. + debug_assert!( + 0 == (shared as usize & KIND_MASK), + "internal: Box<Shared> should have an aligned pointer", + ); + Bytes { + ptr, + len, + data: AtomicPtr::new(shared as _), + vtable: &SHARED_VTABLE, + } + } +} + +impl From<Box<[u8]>> for Bytes { + fn from(slice: Box<[u8]>) -> Bytes { + // Box<[u8]> doesn't contain a heap allocation for empty slices, + // so the pointer isn't aligned enough for the KIND_VEC stashing to + // work. + if slice.is_empty() { + return Bytes::new(); + } + + let len = slice.len(); + let ptr = Box::into_raw(slice) as *mut u8; + + if ptr as usize & 0x1 == 0 { + let data = ptr_map(ptr, |addr| addr | KIND_VEC); + Bytes { + ptr, + len, + data: AtomicPtr::new(data.cast()), + vtable: &PROMOTABLE_EVEN_VTABLE, + } + } else { + Bytes { + ptr, + len, + data: AtomicPtr::new(ptr.cast()), + vtable: &PROMOTABLE_ODD_VTABLE, + } + } + } +} + +impl From<Bytes> for BytesMut { + /// Convert self into `BytesMut`. + /// + /// If `bytes` is unique for the entire original buffer, this will return a + /// `BytesMut` with the contents of `bytes` without copying. + /// If `bytes` is not unique for the entire original buffer, this will make + /// a copy of `bytes` subset of the original buffer in a new `BytesMut`. + /// + /// # Examples + /// + /// ``` + /// use bytes::{Bytes, BytesMut}; + /// + /// let bytes = Bytes::from(b"hello".to_vec()); + /// assert_eq!(BytesMut::from(bytes), BytesMut::from(&b"hello"[..])); + /// ``` + fn from(bytes: Bytes) -> Self { + let bytes = ManuallyDrop::new(bytes); + unsafe { (bytes.vtable.into_mut)(&bytes.data, bytes.ptr, bytes.len) } + } +} + +impl From<String> for Bytes { + fn from(s: String) -> Bytes { + Bytes::from(s.into_bytes()) + } +} + +impl From<Bytes> for Vec<u8> { + fn from(bytes: Bytes) -> Vec<u8> { + let bytes = ManuallyDrop::new(bytes); + unsafe { (bytes.vtable.into_vec)(&bytes.data, bytes.ptr, bytes.len) } + } +} + +// ===== impl Vtable ===== + +impl fmt::Debug for Vtable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Vtable") + .field("clone", &(self.clone as *const ())) + .field("drop", &(self.drop as *const ())) + .finish() + } +} + +// ===== impl StaticVtable ===== + +const STATIC_VTABLE: Vtable = Vtable { + owner_allocation: None, + clone: static_clone, + into_vec: static_to_vec, + into_mut: static_to_mut, + is_unique: static_is_unique, + drop: static_drop, +}; + +unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let slice = slice::from_raw_parts(ptr, len); + Bytes::from_static(slice) +} + +unsafe fn static_to_vec(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { + let slice = slice::from_raw_parts(ptr, len); + slice.to_vec() +} + +unsafe fn static_to_mut(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { + let slice = slice::from_raw_parts(ptr, len); + BytesMut::from(slice) +} + +fn static_is_unique(_: &AtomicPtr<()>) -> bool { + false +} + +unsafe fn static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize) { + // nothing to drop for &'static [u8] +} + +// ===== impl OwnedVtable ===== + +#[repr(C)] +struct Owned<T> { + ref_cnt: AtomicUsize, + owner: T, +} + +impl<T> Owned<T> { + const VTABLE: Vtable = Vtable { + owner_allocation: None, + clone: owned_clone::<T>, + into_vec: owned_to_vec::<T>, + into_mut: owned_to_mut::<T>, + is_unique: owned_is_unique, + drop: owned_drop::<T>, + }; +} + +unsafe fn owned_clone<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let owned = data.load(Ordering::Relaxed); + let old_cnt = (*owned.cast::<AtomicUsize>()).fetch_add(1, Ordering::Relaxed); + if old_cnt > usize::MAX >> 1 { + crate::abort(); + } + + Bytes { + ptr, + len, + data: AtomicPtr::new(owned as _), + vtable: &Owned::<T>::VTABLE, + } +} + +unsafe fn owned_to_vec<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { + let slice = slice::from_raw_parts(ptr, len); + let vec = slice.to_vec(); + owned_drop_impl::<T>(data.load(Ordering::Relaxed)); + vec +} + +unsafe fn owned_to_mut<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { + BytesMut::from_vec(owned_to_vec::<T>(data, ptr, len)) +} + +unsafe fn owned_is_unique(_data: &AtomicPtr<()>) -> bool { + false +} + +unsafe fn owned_drop_impl<T>(owned: *mut ()) { + { + let ref_cnt = &*owned.cast::<AtomicUsize>(); + + let old_cnt = ref_cnt.fetch_sub(1, Ordering::Release); + debug_assert!( + old_cnt > 0 && old_cnt <= usize::MAX >> 1, + "expected non-zero refcount and no underflow" + ); + if old_cnt != 1 { + return; + } + ref_cnt.load(Ordering::Acquire); + } + + drop(Box::<Owned<T>>::from_raw(owned.cast())); +} + +unsafe fn owned_drop<T>(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { + let owned = data.load(Ordering::Relaxed); + owned_drop_impl::<T>(owned); +} + +// ===== impl PromotableVtable ===== + +static PROMOTABLE_EVEN_VTABLE: Vtable = Vtable { + owner_allocation: None, + clone: promotable_even_clone, + into_vec: promotable_even_to_vec, + into_mut: promotable_even_to_mut, + is_unique: promotable_is_unique, + drop: promotable_even_drop, +}; + +static PROMOTABLE_ODD_VTABLE: Vtable = Vtable { + owner_allocation: None, + clone: promotable_odd_clone, + into_vec: promotable_odd_to_vec, + into_mut: promotable_odd_to_mut, + is_unique: promotable_is_unique, + drop: promotable_odd_drop, +}; + +unsafe fn promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let shared = data.load(Ordering::Acquire); + let kind = shared as usize & KIND_MASK; + + if kind == KIND_ARC { + shallow_clone_arc(shared.cast(), ptr, len) + } else { + debug_assert_eq!(kind, KIND_VEC); + let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK); + shallow_clone_vec(data, shared, buf, ptr, len) + } +} + +unsafe fn promotable_to_vec( + data: &AtomicPtr<()>, + ptr: *const u8, + len: usize, + f: fn(*mut ()) -> *mut u8, +) -> Vec<u8> { + let shared = data.load(Ordering::Acquire); + let kind = shared as usize & KIND_MASK; + + if kind == KIND_ARC { + shared_to_vec_impl(shared.cast(), ptr, len) + } else { + // If Bytes holds a Vec, then the offset must be 0. + debug_assert_eq!(kind, KIND_VEC); + + let buf = f(shared); + + let cap = ptr.offset_from(buf) as usize + len; + + // Copy back buffer + ptr::copy(ptr, buf, len); + + Vec::from_raw_parts(buf, len, cap) + } +} + +unsafe fn promotable_to_mut( + data: &AtomicPtr<()>, + ptr: *const u8, + len: usize, + f: fn(*mut ()) -> *mut u8, +) -> BytesMut { + let shared = data.load(Ordering::Acquire); + let kind = shared as usize & KIND_MASK; + + if kind == KIND_ARC { + shared_to_mut_impl(shared.cast(), ptr, len) + } else { + // KIND_VEC is a view of an underlying buffer at a certain offset. + // The ptr + len always represents the end of that buffer. + // Before truncating it, it is first promoted to KIND_ARC. + // Thus, we can safely reconstruct a Vec from it without leaking memory. + debug_assert_eq!(kind, KIND_VEC); + + let buf = f(shared); + let off = ptr.offset_from(buf) as usize; + let cap = off + len; + let v = Vec::from_raw_parts(buf, cap, cap); + + let mut b = BytesMut::from_vec(v); + b.advance_unchecked(off); + b + } +} + +unsafe fn promotable_even_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { + promotable_to_vec(data, ptr, len, |shared| { + ptr_map(shared.cast(), |addr| addr & !KIND_MASK) + }) +} + +unsafe fn promotable_even_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { + promotable_to_mut(data, ptr, len, |shared| { + ptr_map(shared.cast(), |addr| addr & !KIND_MASK) + }) +} + +unsafe fn promotable_even_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) { + data.with_mut(|shared| { + let shared = *shared; + let kind = shared as usize & KIND_MASK; + + if kind == KIND_ARC { + release_shared(shared.cast()); + } else { + debug_assert_eq!(kind, KIND_VEC); + let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK); + free_boxed_slice(buf, ptr, len); + } + }); +} + +unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let shared = data.load(Ordering::Acquire); + let kind = shared as usize & KIND_MASK; + + if kind == KIND_ARC { + shallow_clone_arc(shared as _, ptr, len) + } else { + debug_assert_eq!(kind, KIND_VEC); + shallow_clone_vec(data, shared, shared.cast(), ptr, len) + } +} + +unsafe fn promotable_odd_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { + promotable_to_vec(data, ptr, len, |shared| shared.cast()) +} + +unsafe fn promotable_odd_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { + promotable_to_mut(data, ptr, len, |shared| shared.cast()) +} + +unsafe fn promotable_odd_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) { + data.with_mut(|shared| { + let shared = *shared; + let kind = shared as usize & KIND_MASK; + + if kind == KIND_ARC { + release_shared(shared.cast()); + } else { + debug_assert_eq!(kind, KIND_VEC); + + free_boxed_slice(shared.cast(), ptr, len); + } + }); +} + +unsafe fn promotable_is_unique(data: &AtomicPtr<()>) -> bool { + let shared = data.load(Ordering::Acquire); + let kind = shared as usize & KIND_MASK; + + if kind == KIND_ARC { + let ref_cnt = (*shared.cast::<Shared>()).ref_cnt.load(Ordering::Relaxed); + ref_cnt == 1 + } else { + true + } +} + +unsafe fn free_boxed_slice(buf: *mut u8, offset: *const u8, len: usize) { + let cap = offset.offset_from(buf) as usize + len; + dealloc(buf, Layout::from_size_align(cap, 1).unwrap()) +} + +// ===== impl SharedVtable ===== + +struct Shared { + // Holds arguments to dealloc upon Drop, but otherwise doesn't use them + buf: *mut u8, + cap: usize, + ref_cnt: AtomicUsize, +} + +impl Drop for Shared { + fn drop(&mut self) { + unsafe { dealloc(self.buf, Layout::from_size_align(self.cap, 1).unwrap()) } + } +} + +// Assert that the alignment of `Shared` is divisible by 2. +// This is a necessary invariant since we depend on allocating `Shared` a +// shared object to implicitly carry the `KIND_ARC` flag in its pointer. +// This flag is set when the LSB is 0. +const _: [(); 0 - mem::align_of::<Shared>() % 2] = []; // Assert that the alignment of `Shared` is divisible by 2. + +static SHARED_VTABLE: Vtable = Vtable { + owner_allocation: None, + clone: shared_clone, + into_vec: shared_to_vec, + into_mut: shared_to_mut, + is_unique: shared_is_unique, + drop: shared_drop, +}; + +const KIND_ARC: usize = 0b0; +const KIND_VEC: usize = 0b1; +const KIND_MASK: usize = 0b1; + +unsafe fn shared_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let shared = data.load(Ordering::Relaxed); + shallow_clone_arc(shared as _, ptr, len) +} + +unsafe fn shared_to_vec_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> Vec<u8> { + // Check that the ref_cnt is 1 (unique). + // + // If it is unique, then it is set to 0 with AcqRel fence for the same + // reason in release_shared. + // + // Otherwise, we take the other branch and call release_shared. + if (*shared) + .ref_cnt + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + // Deallocate the `Shared` instance without running its destructor. + let shared = *Box::from_raw(shared); + let shared = ManuallyDrop::new(shared); + let buf = shared.buf; + let cap = shared.cap; + + // Copy back buffer + ptr::copy(ptr, buf, len); + + Vec::from_raw_parts(buf, len, cap) + } else { + let v = slice::from_raw_parts(ptr, len).to_vec(); + release_shared(shared); + v + } +} + +unsafe fn shared_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { + shared_to_vec_impl(data.load(Ordering::Relaxed).cast(), ptr, len) +} + +unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> BytesMut { + // The goal is to check if the current handle is the only handle + // that currently has access to the buffer. This is done by + // checking if the `ref_cnt` is currently 1. + // + // The `Acquire` ordering synchronizes with the `Release` as + // part of the `fetch_sub` in `release_shared`. The `fetch_sub` + // operation guarantees that any mutations done in other threads + // are ordered before the `ref_cnt` is decremented. As such, + // this `Acquire` will guarantee that those mutations are + // visible to the current thread. + // + // Otherwise, we take the other branch, copy the data and call `release_shared`. + if (*shared).ref_cnt.load(Ordering::Acquire) == 1 { + // Deallocate the `Shared` instance without running its destructor. + let shared = *Box::from_raw(shared); + let shared = ManuallyDrop::new(shared); + let buf = shared.buf; + let cap = shared.cap; + + // Rebuild Vec + let off = ptr.offset_from(buf) as usize; + let v = Vec::from_raw_parts(buf, len + off, cap); + + let mut b = BytesMut::from_vec(v); + b.advance_unchecked(off); + b + } else { + // Copy the data from Shared in a new Vec, then release it + let v = slice::from_raw_parts(ptr, len).to_vec(); + release_shared(shared); + BytesMut::from_vec(v) + } +} + +unsafe fn shared_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { + shared_to_mut_impl(data.load(Ordering::Relaxed).cast(), ptr, len) +} + +pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool { + let shared = data.load(Ordering::Acquire); + let ref_cnt = (*shared.cast::<Shared>()).ref_cnt.load(Ordering::Relaxed); + ref_cnt == 1 +} + +unsafe fn shared_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { + data.with_mut(|shared| { + release_shared(shared.cast()); + }); +} + +unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes { + let old_size = (*shared).ref_cnt.fetch_add(1, Ordering::Relaxed); + + if old_size > usize::MAX >> 1 { + crate::abort(); + } + + Bytes { + ptr, + len, + data: AtomicPtr::new(shared as _), + vtable: &SHARED_VTABLE, + } +} + +#[cold] +unsafe fn shallow_clone_vec( + atom: &AtomicPtr<()>, + ptr: *const (), + buf: *mut u8, + offset: *const u8, + len: usize, +) -> Bytes { + // If the buffer is still tracked in a `Vec<u8>`. It is time to + // promote the vec to an `Arc`. This could potentially be called + // concurrently, so some care must be taken. + + // First, allocate a new `Shared` instance containing the + // `Vec` fields. It's important to note that `ptr`, `len`, + // and `cap` cannot be mutated without having `&mut self`. + // This means that these fields will not be concurrently + // updated and since the buffer hasn't been promoted to an + // `Arc`, those three fields still are the components of the + // vector. + let shared = Box::new(Shared { + buf, + cap: offset.offset_from(buf) as usize + len, + // Initialize refcount to 2. One for this reference, and one + // for the new clone that will be returned from + // `shallow_clone`. + ref_cnt: AtomicUsize::new(2), + }); + + let shared = Box::into_raw(shared); + + // The pointer should be aligned, so this assert should + // always succeed. + debug_assert!( + 0 == (shared as usize & KIND_MASK), + "internal: Box<Shared> should have an aligned pointer", + ); + + // Try compare & swapping the pointer into the `arc` field. + // `Release` is used synchronize with other threads that + // will load the `arc` field. + // + // If the `compare_exchange` fails, then the thread lost the + // race to promote the buffer to shared. The `Acquire` + // ordering will synchronize with the `compare_exchange` + // that happened in the other thread and the `Shared` + // pointed to by `actual` will be visible. + match atom.compare_exchange(ptr as _, shared as _, Ordering::AcqRel, Ordering::Acquire) { + Ok(actual) => { + debug_assert!(core::ptr::eq(actual, ptr)); + // The upgrade was successful, the new handle can be + // returned. + Bytes { + ptr: offset, + len, + data: AtomicPtr::new(shared as _), + vtable: &SHARED_VTABLE, + } + } + Err(actual) => { + // The upgrade failed, a concurrent clone happened. Release + // the allocation that was made in this thread, it will not + // be needed. + let shared = Box::from_raw(shared); + mem::forget(*shared); + + // Buffer already promoted to shared storage, so increment ref + // count. + shallow_clone_arc(actual as _, offset, len) + } + } +} + +unsafe fn release_shared(ptr: *mut Shared) { + // `Shared` storage... follow the drop steps from Arc. + if (*ptr).ref_cnt.fetch_sub(1, Ordering::Release) != 1 { + return; + } + + // This fence is needed to prevent reordering of use of the data and + // deletion of the data. Because it is marked `Release`, the decreasing + // of the reference count synchronizes with this `Acquire` fence. This + // means that use of the data happens before decreasing the reference + // count, which happens before this fence, which happens before the + // deletion of the data. + // + // As explained in the [Boost documentation][1], + // + // > It is important to enforce any possible access to the object in one + // > thread (through an existing reference) to *happen before* deleting + // > the object in a different thread. This is achieved by a "release" + // > operation after dropping a reference (any access to the object + // > through this reference must obviously happened before), and an + // > "acquire" operation before deleting the object. + // + // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) + // + // Thread sanitizer does not support atomic fences. Use an atomic load + // instead. + (*ptr).ref_cnt.load(Ordering::Acquire); + + // Drop the data + drop(Box::from_raw(ptr)); +} + +// Ideally we would always use this version of `ptr_map` since it is strict +// provenance compatible, but it results in worse codegen. We will however still +// use it on miri because it gives better diagnostics for people who test bytes +// code with miri. +// +// See https://github.com/tokio-rs/bytes/pull/545 for more info. +#[cfg(miri)] +fn ptr_map<F>(ptr: *mut u8, f: F) -> *mut u8 +where + F: FnOnce(usize) -> usize, +{ + let old_addr = ptr as usize; + let new_addr = f(old_addr); + let diff = new_addr.wrapping_sub(old_addr); + ptr.wrapping_add(diff) +} + +#[cfg(not(miri))] +fn ptr_map<F>(ptr: *mut u8, f: F) -> *mut u8 +where + F: FnOnce(usize) -> usize, +{ + let old_addr = ptr as usize; + let new_addr = f(old_addr); + new_addr as *mut u8 +} + +fn without_provenance(ptr: usize) -> *const u8 { + core::ptr::null::<u8>().wrapping_add(ptr) +} + +// compile-fails + +/// ```compile_fail +/// use bytes::Bytes; +/// #[deny(unused_must_use)] +/// { +/// let mut b1 = Bytes::from("hello world"); +/// b1.split_to(6); +/// } +/// ``` +fn _split_to_must_use() {} + +/// ```compile_fail +/// use bytes::Bytes; +/// #[deny(unused_must_use)] +/// { +/// let mut b1 = Bytes::from("hello world"); +/// b1.split_off(6); +/// } +/// ``` +fn _split_off_must_use() {} + +// fuzz tests +#[cfg(all(test, loom))] +mod fuzz { + use loom::sync::Arc; + use loom::thread; + + use super::Bytes; + #[test] + fn bytes_cloning_vec() { + loom::model(|| { + let a = Bytes::from(b"abcdefgh".to_vec()); + let addr = a.as_ptr() as usize; + + // test the Bytes::clone is Sync by putting it in an Arc + let a1 = Arc::new(a); + let a2 = a1.clone(); + + let t1 = thread::spawn(move || { + let b: Bytes = (*a1).clone(); + assert_eq!(b.as_ptr() as usize, addr); + }); + + let t2 = thread::spawn(move || { + let b: Bytes = (*a2).clone(); + assert_eq!(b.as_ptr() as usize, addr); + }); + + t1.join().unwrap(); + t2.join().unwrap(); + }); + } +} diff --git a/third-party/bytes/src/bytes/accounted_owner.rs b/third-party/bytes/src/bytes/accounted_owner.rs new file mode 100644 index 000000000..cad13f4c9 --- /dev/null +++ b/third-party/bytes/src/bytes/accounted_owner.rs @@ -0,0 +1,171 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the MIT license, like the bytes crate. + +use super::{Bytes, Vtable}; +use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; +use crate::BytesMut; +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::alloc::Layout; +use core::mem::ManuallyDrop; +use core::ptr::NonNull; + +/// Allocation admission for the control block created by +/// [`Bytes::try_from_owner_accounted`]. The exact allocator layout is supplied; +/// cloning and slicing never create another control block or another claim. +/// Methods must not panic. Claims must fail without acquiring any charge. +pub trait OwnerAllocation: Send + Sync + 'static { + /// Admission refusal returned before allocating the control block. + type Error; + /// Claim the exact control-block size before its allocation. + fn claim(&self, layout: Layout) -> Result<(), Self::Error>; + /// Observe the exact allocation after creation or after deallocation. + fn record(&self, _layout: Layout, _acquired: bool) {} + /// Release only after the owner has dropped and the block was deallocated. + fn release(&self, layout: Layout); +} + +#[repr(C)] +struct Accounted<T, A> { + references: AtomicUsize, + owner: ManuallyDrop<T>, + allocation: ManuallyDrop<A>, +} + +impl<T, A: OwnerAllocation> Accounted<T, A> { + const VTABLE: Vtable = Vtable { + owner_allocation: Some(owner_allocation::<T, A>), + clone: clone::<T, A>, + into_vec: into_vec::<T, A>, + into_mut: into_mut::<T, A>, + is_unique: is_unique, + drop: drop_bytes::<T, A>, + }; +} + +impl Bytes { + /// Transfer a pinned immutable owner and admit the exact control allocation. + /// The accounting object is stored inline in that block. Admission failure + /// drops both inputs. Final release runs after the block is deallocated, + /// including when owner destruction or `AsRef` unwinds. + /// + /// Like [`Bytes::from_owner`], conversion into mutable bytes copies the data. + pub fn try_from_owner_accounted<T, A>(owner: T, allocation: A) -> Result<Self, A::Error> + where + T: AsRef<[u8]> + Send + 'static, + A: OwnerAllocation, + { + allocation.claim(Layout::new::<Accounted<T, A>>())?; + let storage = Box::into_raw(Box::new(Accounted { + references: AtomicUsize::new(1), + owner: ManuallyDrop::new(owner), + allocation: ManuallyDrop::new(allocation), + })); + unsafe { &*storage } + .allocation + .record(Layout::new::<Accounted<T, A>>(), true); + let mut bytes = Bytes { + ptr: NonNull::dangling().as_ptr(), + len: 0, + data: AtomicPtr::new(storage.cast()), + vtable: &Accounted::<T, A>::VTABLE, + }; + // Store ownership in Bytes before calling AsRef so an unwind releases + // both the owner and the fully constructed control block. + let value = unsafe { &*storage }.owner.as_ref(); + bytes.ptr = value.as_ptr(); + bytes.len = value.len(); + Ok(bytes) + } +} + +unsafe fn clone<T, A: OwnerAllocation>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let storage = data.load(Ordering::Relaxed); + let references = &(*storage.cast::<Accounted<T, A>>()).references; + if references.fetch_add(1, Ordering::Relaxed) > usize::MAX >> 1 { + crate::abort(); + } + Bytes { + ptr, + len, + data: AtomicPtr::new(storage), + vtable: &Accounted::<T, A>::VTABLE, + } +} + +// Dropping this stack guard refunds even if the owner's destructor unwinds. +// It is declared before Box so unwinding deallocates Box before the refund. +struct Refund<A: OwnerAllocation> { + allocation: A, + layout: Layout, +} +impl<A: OwnerAllocation> Drop for Refund<A> { + fn drop(&mut self) { + self.allocation.record(self.layout, false); + self.allocation.release(self.layout); + } +} + +unsafe fn release<T, A: OwnerAllocation>(storage: *mut ()) { + let storage = storage.cast::<Accounted<T, A>>(); + let references = &(*storage).references; + let previous = references.fetch_sub(1, Ordering::Release); + debug_assert!(previous > 0 && previous <= usize::MAX >> 1); + if previous != 1 { + return; + } + references.load(Ordering::Acquire); + let refund = Refund { + allocation: ManuallyDrop::take(&mut (*storage).allocation), + layout: Layout::new::<Accounted<T, A>>(), + }; + let mut storage = Box::from_raw(storage); + // The owner stays at its pinned address through destruction. Moving only + // the accounting value is safe: it was never exposed through AsRef. + ManuallyDrop::drop(&mut storage.owner); + drop(storage); + drop(refund); +} + +unsafe fn drop_bytes<T, A: OwnerAllocation>( + data: &mut AtomicPtr<()>, + _ptr: *const u8, + _len: usize, +) { + release::<T, A>(data.load(Ordering::Relaxed)); +} +unsafe fn into_vec<T, A: OwnerAllocation>( + data: &AtomicPtr<()>, + ptr: *const u8, + len: usize, +) -> Vec<u8> { + let result = core::slice::from_raw_parts(ptr, len).to_vec(); + release::<T, A>(data.load(Ordering::Relaxed)); + result +} +unsafe fn into_mut<T, A: OwnerAllocation>( + data: &AtomicPtr<()>, + ptr: *const u8, + len: usize, +) -> BytesMut { + BytesMut::from_vec(into_vec::<T, A>(data, ptr, len)) +} +unsafe fn is_unique(_data: &AtomicPtr<()>) -> bool { + false +} + +unsafe fn owner_allocation<T, A: OwnerAllocation>( + data: &AtomicPtr<()>, + action: super::owner_lease::Action, + layout: Layout, +) -> bool { + let storage = data.load(Ordering::Relaxed).cast::<Accounted<T, A>>(); + let allocation = &(*storage).allocation; + match action { + super::owner_lease::Action::Claim => return allocation.claim(layout).is_ok(), + super::owner_lease::Action::Allocated => allocation.record(layout, true), + super::owner_lease::Action::Freed => allocation.record(layout, false), + super::owner_lease::Action::Release => allocation.release(layout), + } + true +} diff --git a/third-party/bytes/src/bytes/owner_lease.rs b/third-party/bytes/src/bytes/owner_lease.rs new file mode 100644 index 000000000..ce9cf9b3b --- /dev/null +++ b/third-party/bytes/src/bytes/owner_lease.rs @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the MIT license, like the bytes crate. + +use super::Bytes; +use core::alloc::Layout; +use core::fmt; + +pub(crate) enum Action { + Claim, + Allocated, + Freed, + Release, +} + +/// The immutable owner's budget could not admit another control allocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OwnerAllocationError; +impl fmt::Display for OwnerAllocationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("owner allocation budget exhausted") + } +} + +/// An admitted control allocation sharing the byte owner's accounting domain. +/// The lease retains the owner without allocating another control block. Drop +/// it only after deallocating the associated storage. Clones and slices of the +/// original bytes do not create leases implicitly. +pub struct OwnerAllocationLease { + owner: Bytes, + layout: Layout, + allocated: bool, +} +impl fmt::Debug for OwnerAllocationLease { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OwnerAllocationLease") + .field("layout", &self.layout) + .field("allocated", &self.allocated) + .finish() + } +} +impl OwnerAllocationLease { + /// Mark the capacity as physically allocated after allocation succeeds. + /// This operation is idempotent. Dropping an unmarked lease refunds only + /// its admission claim, such as when the external allocator refuses it. + pub fn allocated(&mut self) { + if !self.allocated { + self.invoke(Action::Allocated); + self.allocated = true; + } + } + + fn invoke(&self, action: Action) { + let operation = self.owner.vtable.owner_allocation.unwrap(); + unsafe { operation(&self.owner.data, action, self.layout) }; + } +} +impl Drop for OwnerAllocationLease { + fn drop(&mut self) { + if self.allocated { + self.invoke(Action::Freed); + } + self.invoke(Action::Release); + } +} + +impl Bytes { + /// Admit an external control allocation using this immutable owner's + /// budget. `None` identifies bytes created through ordinary constructors; + /// their external allocation policy remains the caller's responsibility. + /// `Err` means an accounted owner refused the claim before allocation. + /// + /// The returned lease must be marked with `allocated` after allocation and + /// destroyed only after that storage is deallocated. It retains the byte + /// owner and works after other byte clones have been dropped. + pub fn try_owner_allocation( + &self, + layout: Layout, + ) -> Result<Option<OwnerAllocationLease>, OwnerAllocationError> { + let operation = match self.vtable.owner_allocation { + Some(operation) => operation, + None => return Ok(None), + }; + if !unsafe { operation(&self.data, Action::Claim, layout) } { + return Err(OwnerAllocationError); + } + Ok(Some(OwnerAllocationLease { + owner: self.clone(), + layout, + allocated: false, + })) + } +} + +/// Borrowed accounting callbacks for an external allocator. The backing Bytes +/// must remain live until every successful claim has been released. Claims and +/// physical records must pair exactly, with free records after deallocation. +#[derive(Debug, Clone, Copy)] +pub struct OwnerAllocationContext<'a> { + owner: &'a Bytes, +} +impl OwnerAllocationContext<'_> { + /// Claim one exact allocation layout before invoking its allocator. + pub fn claim(self, layout: Layout) -> Result<(), OwnerAllocationError> { + if self.invoke(Action::Claim, layout) { + Ok(()) + } else { + Err(OwnerAllocationError) + } + } + /// Record actual creation or completed deallocation of the claimed layout. + pub fn record(self, layout: Layout, acquired: bool) { + self.invoke( + if acquired { + Action::Allocated + } else { + Action::Freed + }, + layout, + ); + } + /// Refund the corresponding claim after actual deallocation or refusal by + /// the underlying allocator. Does not emit a physical allocation record. + pub fn release(self, layout: Layout) { + self.invoke(Action::Release, layout); + } + fn invoke(self, action: Action, layout: Layout) -> bool { + let operation = self.owner.vtable.owner_allocation.unwrap(); + unsafe { operation(&self.owner.data, action, layout) } + } +} +impl Bytes { + /// Borrow this immutable owner's allocator callbacks without allocating. + /// Ordinary Bytes constructors return None; the caller owns their policy. + pub fn owner_allocation(&self) -> Option<OwnerAllocationContext<'_>> { + self.vtable + .owner_allocation + .map(|_| OwnerAllocationContext { owner: self }) + } +} diff --git a/third-party/bytes/src/bytes_mut.rs b/third-party/bytes/src/bytes_mut.rs new file mode 100644 index 000000000..5fa2d3496 --- /dev/null +++ b/third-party/bytes/src/bytes_mut.rs @@ -0,0 +1,1942 @@ +use core::mem::{self, ManuallyDrop, MaybeUninit}; +use core::ops::{Deref, DerefMut}; +use core::ptr::{self, NonNull}; +use core::{cmp, fmt, hash, slice}; + +use alloc::{ + borrow::{Borrow, BorrowMut}, + boxed::Box, + string::String, + vec, + vec::Vec, +}; + +use crate::buf::{IntoIter, UninitSlice}; +use crate::bytes::Vtable; +#[allow(unused)] +use crate::loom::sync::atomic::AtomicMut; +use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; +use crate::{Buf, BufMut, Bytes, TryGetError}; + +/// A unique reference to a contiguous slice of memory. +/// +/// `BytesMut` represents a unique view into a potentially shared memory region. +/// Given the uniqueness guarantee, owners of `BytesMut` handles are able to +/// mutate the memory. +/// +/// `BytesMut` can be thought of as containing a `buf: Arc<Vec<u8>>`, an offset +/// into `buf`, a slice length, and a guarantee that no other `BytesMut` for the +/// same `buf` overlaps with its slice. That guarantee means that a write lock +/// is not required. +/// +/// # Growth +/// +/// `BytesMut`'s `BufMut` implementation will implicitly grow its buffer as +/// necessary. However, explicitly reserving the required space up-front before +/// a series of inserts will be more efficient. +/// +/// # Examples +/// +/// ``` +/// use bytes::{BytesMut, BufMut}; +/// +/// let mut buf = BytesMut::with_capacity(64); +/// +/// buf.put_u8(b'h'); +/// buf.put_u8(b'e'); +/// buf.put(&b"llo"[..]); +/// +/// assert_eq!(&buf[..], b"hello"); +/// +/// // Freeze the buffer so that it can be shared +/// let a = buf.freeze(); +/// +/// // This does not allocate, instead `b` points to the same memory. +/// let b = a.clone(); +/// +/// assert_eq!(&a[..], b"hello"); +/// assert_eq!(&b[..], b"hello"); +/// ``` +pub struct BytesMut { + ptr: NonNull<u8>, + len: usize, + cap: usize, + data: *mut Shared, +} + +// Thread-safe reference-counted container for the shared storage. This mostly +// the same as `core::sync::Arc` but without the weak counter. The ref counting +// fns are based on the ones found in `std`. +// +// The main reason to use `Shared` instead of `core::sync::Arc` is that it ends +// up making the overall code simpler and easier to reason about. This is due to +// some of the logic around setting `Inner::arc` and other ways the `arc` field +// is used. Using `Arc` ended up requiring a number of funky transmutes and +// other shenanigans to make it work. +struct Shared { + vec: Vec<u8>, + original_capacity_repr: usize, + ref_count: AtomicUsize, +} + +// Assert that the alignment of `Shared` is divisible by 2. +// This is a necessary invariant since we depend on allocating `Shared` a +// shared object to implicitly carry the `KIND_ARC` flag in its pointer. +// This flag is set when the LSB is 0. +const _: [(); 0 - mem::align_of::<Shared>() % 2] = []; // Assert that the alignment of `Shared` is divisible by 2. + +// Buffer storage strategy flags. +const KIND_ARC: usize = 0b0; +const KIND_VEC: usize = 0b1; +const KIND_MASK: usize = 0b1; + +// The max original capacity value. Any `Bytes` allocated with a greater initial +// capacity will default to this. +const MAX_ORIGINAL_CAPACITY_WIDTH: usize = 17; +// The original capacity algorithm will not take effect unless the originally +// allocated capacity was at least 1kb in size. +const MIN_ORIGINAL_CAPACITY_WIDTH: usize = 10; +// The original capacity is stored in powers of 2 starting at 1kb to a max of +// 64kb. Representing it as such requires only 3 bits of storage. +const ORIGINAL_CAPACITY_MASK: usize = 0b11100; +const ORIGINAL_CAPACITY_OFFSET: usize = 2; + +const VEC_POS_OFFSET: usize = 5; +// When the storage is in the `Vec` representation, the pointer can be advanced +// at most this value. This is due to the amount of storage available to track +// the offset is usize - number of KIND bits and number of ORIGINAL_CAPACITY +// bits. +const MAX_VEC_POS: usize = usize::MAX >> VEC_POS_OFFSET; +const NOT_VEC_POS_MASK: usize = 0b11111; + +#[cfg(target_pointer_width = "64")] +const PTR_WIDTH: usize = 64; +#[cfg(target_pointer_width = "32")] +const PTR_WIDTH: usize = 32; + +/* + * + * ===== BytesMut ===== + * + */ + +impl BytesMut { + /// Creates a new `BytesMut` with the specified capacity. + /// + /// The returned `BytesMut` will be able to hold at least `capacity` bytes + /// without reallocating. + /// + /// It is important to note that this function does not specify the length + /// of the returned `BytesMut`, but only the capacity. + /// + /// # Examples + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// + /// let mut bytes = BytesMut::with_capacity(64); + /// + /// // `bytes` contains no data, even though there is capacity + /// assert_eq!(bytes.len(), 0); + /// + /// bytes.put(&b"hello world"[..]); + /// + /// assert_eq!(&bytes[..], b"hello world"); + /// ``` + #[inline] + pub fn with_capacity(capacity: usize) -> BytesMut { + BytesMut::from_vec(Vec::with_capacity(capacity)) + } + + /// Creates a new `BytesMut` with default capacity. + /// + /// Resulting object has length 0 and unspecified capacity. + /// This function does not allocate. + /// + /// # Examples + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// + /// let mut bytes = BytesMut::new(); + /// + /// assert_eq!(0, bytes.len()); + /// + /// bytes.reserve(2); + /// bytes.put_slice(b"xy"); + /// + /// assert_eq!(&b"xy"[..], &bytes[..]); + /// ``` + #[inline] + pub fn new() -> BytesMut { + BytesMut::with_capacity(0) + } + + /// Returns the number of bytes contained in this `BytesMut`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let b = BytesMut::from(&b"hello"[..]); + /// assert_eq!(b.len(), 5); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Returns true if the `BytesMut` has a length of 0. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let b = BytesMut::with_capacity(64); + /// assert!(b.is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Returns the number of bytes the `BytesMut` can hold without reallocating. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let b = BytesMut::with_capacity(64); + /// assert_eq!(b.capacity(), 64); + /// ``` + #[inline] + pub fn capacity(&self) -> usize { + self.cap + } + + /// Converts `self` into an immutable `Bytes`. + /// + /// The conversion is zero cost and is used to indicate that the slice + /// referenced by the handle will no longer be mutated. Once the conversion + /// is done, the handle can be cloned and shared across threads. + /// + /// # Examples + /// + /// ```ignore-wasm + /// use bytes::{BytesMut, BufMut}; + /// use std::thread; + /// + /// let mut b = BytesMut::with_capacity(64); + /// b.put(&b"hello world"[..]); + /// let b1 = b.freeze(); + /// let b2 = b1.clone(); + /// + /// let th = thread::spawn(move || { + /// assert_eq!(&b1[..], b"hello world"); + /// }); + /// + /// assert_eq!(&b2[..], b"hello world"); + /// th.join().unwrap(); + /// ``` + #[inline] + pub fn freeze(self) -> Bytes { + let bytes = ManuallyDrop::new(self); + if bytes.kind() == KIND_VEC { + // Just re-use `Bytes` internal Vec vtable + unsafe { + let off = bytes.get_vec_pos(); + let vec = rebuild_vec(bytes.ptr.as_ptr(), bytes.len, bytes.cap, off); + let mut b: Bytes = vec.into(); + b.advance(off); + b + } + } else { + debug_assert_eq!(bytes.kind(), KIND_ARC); + + let ptr = bytes.ptr.as_ptr(); + let len = bytes.len; + let data = AtomicPtr::new(bytes.data.cast()); + unsafe { Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE) } + } + } + + /// Creates a new `BytesMut` containing `len` zeros. + /// + /// The resulting object has a length of `len` and a capacity greater + /// than or equal to `len`. The entire length of the object will be filled + /// with zeros. + /// + /// On some platforms or allocators this function may be faster than + /// a manual implementation. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let zeros = BytesMut::zeroed(42); + /// + /// assert!(zeros.capacity() >= 42); + /// assert_eq!(zeros.len(), 42); + /// zeros.into_iter().for_each(|x| assert_eq!(x, 0)); + /// ``` + pub fn zeroed(len: usize) -> BytesMut { + BytesMut::from_vec(vec![0; len]) + } + + /// Splits the bytes into two at the given index. + /// + /// Afterwards `self` contains elements `[0, at)`, and the returned + /// `BytesMut` contains elements `[at, capacity)`. It's guaranteed that the + /// memory does not move, that is, the address of `self` does not change, + /// and the address of the returned slice is `at` bytes after that. + /// + /// This is an `O(1)` operation that just increases the reference count + /// and sets a few indices. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut a = BytesMut::from(&b"hello world"[..]); + /// let mut b = a.split_off(5); + /// + /// a[0] = b'j'; + /// b[0] = b'!'; + /// + /// assert_eq!(&a[..], b"jello"); + /// assert_eq!(&b[..], b"!world"); + /// ``` + /// + /// # Panics + /// + /// Panics if `at > capacity`. + #[must_use = "consider BytesMut::truncate if you don't need the other half"] + pub fn split_off(&mut self, at: usize) -> BytesMut { + assert!( + at <= self.capacity(), + "split_off out of bounds: {:?} <= {:?}", + at, + self.capacity(), + ); + unsafe { + let mut other = self.shallow_clone(); + // SAFETY: We've checked that `at` <= `self.capacity()` above. + other.advance_unchecked(at); + self.cap = at; + self.len = cmp::min(self.len, at); + other + } + } + + /// Removes the bytes from the current view, returning them in a new + /// `BytesMut` handle. + /// + /// Afterwards, `self` will be empty, but will retain any additional + /// capacity that it had before the operation. This is identical to + /// `self.split_to(self.len())`. + /// + /// This is an `O(1)` operation that just increases the reference count and + /// sets a few indices. + /// + /// # Examples + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// + /// let mut buf = BytesMut::with_capacity(1024); + /// buf.put(&b"hello world"[..]); + /// + /// let other = buf.split(); + /// + /// assert!(buf.is_empty()); + /// assert_eq!(1013, buf.capacity()); + /// + /// assert_eq!(other, b"hello world"[..]); + /// ``` + #[must_use = "consider BytesMut::clear if you don't need the other half"] + pub fn split(&mut self) -> BytesMut { + let len = self.len(); + self.split_to(len) + } + + /// Splits the buffer into two at the given index. + /// + /// Afterwards `self` contains elements `[at, len)`, and the returned `BytesMut` + /// contains elements `[0, at)`. + /// + /// This is an `O(1)` operation that just increases the reference count and + /// sets a few indices. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut a = BytesMut::from(&b"hello world"[..]); + /// let mut b = a.split_to(5); + /// + /// a[0] = b'!'; + /// b[0] = b'j'; + /// + /// assert_eq!(&a[..], b"!world"); + /// assert_eq!(&b[..], b"jello"); + /// ``` + /// + /// # Panics + /// + /// Panics if `at > len`. + #[must_use = "consider BytesMut::advance if you don't need the other half"] + pub fn split_to(&mut self, at: usize) -> BytesMut { + assert!( + at <= self.len(), + "split_to out of bounds: {:?} <= {:?}", + at, + self.len(), + ); + + unsafe { + let mut other = self.shallow_clone(); + // SAFETY: We've checked that `at` <= `self.len()` and we know that `self.len()` <= + // `self.capacity()`. + self.advance_unchecked(at); + other.cap = at; + other.len = at; + other + } + } + + /// Shortens the buffer, keeping the first `len` bytes and dropping the + /// rest. + /// + /// If `len` is greater than the buffer's current length, this has no + /// effect. + /// + /// Existing underlying capacity is preserved. + /// + /// The [split_off](`Self::split_off()`) method can emulate `truncate`, but this causes the + /// excess bytes to be returned instead of dropped. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::from(&b"hello world"[..]); + /// buf.truncate(5); + /// assert_eq!(buf, b"hello"[..]); + /// ``` + pub fn truncate(&mut self, len: usize) { + if len <= self.len() { + // SAFETY: Shrinking the buffer cannot expose uninitialized bytes. + unsafe { self.set_len(len) }; + } + } + + /// Clears the buffer, removing all data. Existing capacity is preserved. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::from(&b"hello world"[..]); + /// buf.clear(); + /// assert!(buf.is_empty()); + /// ``` + pub fn clear(&mut self) { + // SAFETY: Setting the length to zero cannot expose uninitialized bytes. + unsafe { self.set_len(0) }; + } + + /// Resizes the buffer so that `len` is equal to `new_len`. + /// + /// If `new_len` is greater than `len`, the buffer is extended by the + /// difference with each additional byte set to `value`. If `new_len` is + /// less than `len`, the buffer is simply truncated. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::new(); + /// + /// buf.resize(3, 0x1); + /// assert_eq!(&buf[..], &[0x1, 0x1, 0x1]); + /// + /// buf.resize(2, 0x2); + /// assert_eq!(&buf[..], &[0x1, 0x1]); + /// + /// buf.resize(4, 0x3); + /// assert_eq!(&buf[..], &[0x1, 0x1, 0x3, 0x3]); + /// ``` + pub fn resize(&mut self, new_len: usize, value: u8) { + let additional = if let Some(additional) = new_len.checked_sub(self.len()) { + additional + } else { + self.truncate(new_len); + return; + }; + + if additional == 0 { + return; + } + + self.reserve(additional); + let dst = self.spare_capacity_mut().as_mut_ptr(); + // SAFETY: `spare_capacity_mut` returns a valid, properly aligned pointer and we've + // reserved enough space to write `additional` bytes. + unsafe { ptr::write_bytes(dst, value, additional) }; + + // SAFETY: There are at least `new_len` initialized bytes in the buffer so no + // uninitialized bytes are being exposed. + unsafe { self.set_len(new_len) }; + } + + /// Sets the length of the buffer. + /// + /// This will explicitly set the size of the buffer without actually + /// modifying the data, so it is up to the caller to ensure that the data + /// has been initialized. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut b = BytesMut::from(&b"hello world"[..]); + /// + /// unsafe { + /// b.set_len(5); + /// } + /// + /// assert_eq!(&b[..], b"hello"); + /// + /// unsafe { + /// b.set_len(11); + /// } + /// + /// assert_eq!(&b[..], b"hello world"); + /// ``` + #[inline] + pub unsafe fn set_len(&mut self, len: usize) { + debug_assert!(len <= self.cap, "set_len out of bounds"); + self.len = len; + } + + /// Reserves capacity for at least `additional` more bytes to be inserted + /// into the given `BytesMut`. + /// + /// More than `additional` bytes may be reserved in order to avoid frequent + /// reallocations. A call to `reserve` may result in an allocation. + /// + /// Before allocating new buffer space, the function will attempt to reclaim + /// space in the existing buffer. If the current handle references a view + /// into a larger original buffer, and all other handles referencing part + /// of the same original buffer have been dropped, then the current view + /// can be copied/shifted to the front of the buffer and the handle can take + /// ownership of the full buffer, provided that the full buffer is large + /// enough to fit the requested additional capacity. + /// + /// This optimization will only happen if shifting the data from the current + /// view to the front of the buffer is not too expensive in terms of the + /// (amortized) time required. The precise condition is subject to change; + /// as of now, the length of the data being shifted needs to be at least as + /// large as the distance that it's shifted by. If the current view is empty + /// and the original buffer is large enough to fit the requested additional + /// capacity, then reallocations will never happen. + /// + /// # Examples + /// + /// In the following example, a new buffer is allocated. + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::from(&b"hello"[..]); + /// buf.reserve(64); + /// assert!(buf.capacity() >= 69); + /// ``` + /// + /// In the following example, the existing buffer is reclaimed. + /// + /// ``` + /// use bytes::{BytesMut, BufMut}; + /// + /// let mut buf = BytesMut::with_capacity(128); + /// buf.put(&[0; 64][..]); + /// + /// let ptr = buf.as_ptr(); + /// let other = buf.split(); + /// + /// assert!(buf.is_empty()); + /// assert_eq!(buf.capacity(), 64); + /// + /// drop(other); + /// buf.reserve(128); + /// + /// assert_eq!(buf.capacity(), 128); + /// assert_eq!(buf.as_ptr(), ptr); + /// ``` + /// + /// # Panics + /// + /// Panics if the new capacity overflows `usize`. + #[inline] + pub fn reserve(&mut self, additional: usize) { + let len = self.len(); + let rem = self.capacity() - len; + + if additional <= rem { + // The handle can already store at least `additional` more bytes, so + // there is no further work needed to be done. + return; + } + + // will always succeed + let _ = self.reserve_inner(additional, true); + } + + // In separate function to allow the short-circuits in `reserve` and `try_reclaim` to + // be inline-able. Significantly helps performance. Returns false if it did not succeed. + fn reserve_inner(&mut self, additional: usize, allocate: bool) -> bool { + let len = self.len(); + let kind = self.kind(); + + if kind == KIND_VEC { + // If there's enough free space before the start of the buffer, then + // just copy the data backwards and reuse the already-allocated + // space. + // + // Otherwise, since backed by a vector, use `Vec::reserve` + // + // We need to make sure that this optimization does not kill the + // amortized runtimes of BytesMut's operations. + unsafe { + let off = self.get_vec_pos(); + + // Only reuse space if we can satisfy the requested additional space. + // + // Also check if the value of `off` suggests that enough bytes + // have been read to account for the overhead of shifting all + // the data (in an amortized analysis). + // Hence the condition `off >= self.len()`. + // + // This condition also already implies that the buffer is going + // to be (at least) half-empty in the end; so we do not break + // the (amortized) runtime with future resizes of the underlying + // `Vec`. + // + // [For more details check issue #524, and PR #525.] + if self.capacity() - self.len() + off >= additional && off >= self.len() { + // There's enough space, and it's not too much overhead: + // reuse the space! + // + // Just move the pointer back to the start after copying + // data back. + let base_ptr = self.ptr.as_ptr().sub(off); + // Since `off >= self.len()`, the two regions don't overlap. + ptr::copy_nonoverlapping(self.ptr.as_ptr(), base_ptr, self.len); + self.ptr = vptr(base_ptr); + self.set_vec_pos(0); + + // Length stays constant, but since we moved backwards we + // can gain capacity back. + self.cap += off; + } else { + if !allocate { + return false; + } + // Not enough space, or reusing might be too much overhead: + // allocate more space! + let mut v = + ManuallyDrop::new(rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off)); + v.reserve(additional); + + // Update the info + self.ptr = vptr(v.as_mut_ptr().add(off)); + self.cap = v.capacity() - off; + debug_assert_eq!(self.len, v.len() - off); + } + + return true; + } + } + + debug_assert_eq!(kind, KIND_ARC); + let shared: *mut Shared = self.data; + + // Reserving involves abandoning the currently shared buffer and + // allocating a new vector with the requested capacity. + // + // Compute the new capacity + let mut new_cap = match len.checked_add(additional) { + Some(new_cap) => new_cap, + None if !allocate => return false, + None => panic!("overflow"), + }; + + unsafe { + // First, try to reclaim the buffer. This is possible if the current + // handle is the only outstanding handle pointing to the buffer. + if (*shared).is_unique() { + // This is the only handle to the buffer. It can be reclaimed. + // However, before doing the work of copying data, check to make + // sure that the vector has enough capacity. + let v = &mut (*shared).vec; + + let v_capacity = v.capacity(); + let ptr = v.as_mut_ptr(); + + let offset = self.ptr.as_ptr().offset_from(ptr) as usize; + + let new_cap_plus_offset = match new_cap.checked_add(offset) { + Some(new_cap_plus_offset) => new_cap_plus_offset, + None if !allocate => return false, + None => panic!("overflow"), + }; + + // Compare the condition in the `kind == KIND_VEC` case above + // for more details. + if v_capacity >= new_cap_plus_offset { + self.cap = new_cap; + // no copy is necessary + } else if v_capacity >= new_cap && offset >= len { + // The capacity is sufficient, and copying is not too much + // overhead: reclaim the buffer! + + // `offset >= len` means: no overlap + ptr::copy_nonoverlapping(self.ptr.as_ptr(), ptr, len); + + self.ptr = vptr(ptr); + self.cap = v.capacity(); + } else { + if !allocate { + return false; + } + + // new_cap is calculated in terms of `BytesMut`, not the underlying + // `Vec`, so it does not take the offset into account. + // + // Thus we have to manually add it here. + new_cap = new_cap_plus_offset; + + // The vector capacity is not sufficient. The reserve request is + // asking for more than the initial buffer capacity. Allocate more + // than requested if `new_cap` is not much bigger than the current + // capacity. + // + // There are some situations, using `reserve_exact` that the + // buffer capacity could be below `original_capacity`, so do a + // check. + let double = v.capacity().checked_shl(1).unwrap_or(new_cap); + + new_cap = cmp::max(double, new_cap); + + // No space - allocate more + // + // The length field of `Shared::vec` is not used by the `BytesMut`; + // instead we use the `len` field in the `BytesMut` itself. However, + // when calling `reserve`, it doesn't guarantee that data stored in + // the unused capacity of the vector is copied over to the new + // allocation, so we need to ensure that we don't have any data we + // care about in the unused capacity before calling `reserve`. + debug_assert!(offset + len <= v.capacity()); + v.set_len(offset + len); + v.reserve(new_cap - v.len()); + + // Update the info + self.ptr = vptr(v.as_mut_ptr().add(offset)); + self.cap = v.capacity() - offset; + } + + return true; + } + } + if !allocate { + return false; + } + + let original_capacity_repr = unsafe { (*shared).original_capacity_repr }; + let original_capacity = original_capacity_from_repr(original_capacity_repr); + + new_cap = cmp::max(new_cap, original_capacity); + + // Create a new vector to store the data + let mut v = ManuallyDrop::new(Vec::with_capacity(new_cap)); + + // Copy the bytes + v.extend_from_slice(self.as_ref()); + + // Release the shared handle. This must be done *after* the bytes are + // copied. + unsafe { release_shared(shared) }; + + // Update self + let data = (original_capacity_repr << ORIGINAL_CAPACITY_OFFSET) | KIND_VEC; + self.data = invalid_ptr(data); + self.ptr = vptr(v.as_mut_ptr()); + self.cap = v.capacity(); + debug_assert_eq!(self.len, v.len()); + true + } + + /// Attempts to cheaply reclaim already allocated capacity for at least `additional` more + /// bytes to be inserted into the given `BytesMut` and returns `true` if it succeeded. + /// + /// `try_reclaim` behaves exactly like `reserve`, except that it never allocates new storage + /// and returns a `bool` indicating whether it was successful in doing so: + /// + /// `try_reclaim` returns false under these conditions: + /// - The spare capacity left is less than `additional` bytes AND + /// - The existing allocation cannot be reclaimed cheaply or it was less than + /// `additional` bytes in size + /// + /// Reclaiming the allocation cheaply is possible if the `BytesMut` has no outstanding + /// references through other `BytesMut`s or `Bytes` which point to the same underlying + /// storage. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::with_capacity(64); + /// assert_eq!(true, buf.try_reclaim(64)); + /// assert_eq!(64, buf.capacity()); + /// + /// buf.extend_from_slice(b"abcd"); + /// let mut split = buf.split(); + /// assert_eq!(60, buf.capacity()); + /// assert_eq!(4, split.capacity()); + /// assert_eq!(false, split.try_reclaim(64)); + /// assert_eq!(false, buf.try_reclaim(64)); + /// // The split buffer is filled with "abcd" + /// assert_eq!(false, split.try_reclaim(4)); + /// // buf is empty and has capacity for 60 bytes + /// assert_eq!(true, buf.try_reclaim(60)); + /// + /// drop(buf); + /// assert_eq!(false, split.try_reclaim(64)); + /// + /// split.clear(); + /// assert_eq!(4, split.capacity()); + /// assert_eq!(true, split.try_reclaim(64)); + /// assert_eq!(64, split.capacity()); + /// ``` + // I tried splitting out try_reclaim_inner after the short circuits, but it was inlined + // regardless with Rust 1.78.0 so probably not worth it + #[inline] + #[must_use = "consider BytesMut::reserve if you need an infallible reservation"] + pub fn try_reclaim(&mut self, additional: usize) -> bool { + let len = self.len(); + let rem = self.capacity() - len; + + if additional <= rem { + // The handle can already store at least `additional` more bytes, so + // there is no further work needed to be done. + return true; + } + + self.reserve_inner(additional, false) + } + + /// Appends given bytes to this `BytesMut`. + /// + /// If this `BytesMut` object does not have enough capacity, it is resized + /// first. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::with_capacity(0); + /// buf.extend_from_slice(b"aaabbb"); + /// buf.extend_from_slice(b"cccddd"); + /// + /// assert_eq!(b"aaabbbcccddd", &buf[..]); + /// ``` + #[inline] + pub fn extend_from_slice(&mut self, extend: &[u8]) { + let cnt = extend.len(); + self.reserve(cnt); + + unsafe { + let dst = self.spare_capacity_mut(); + // Reserved above + debug_assert!(dst.len() >= cnt); + + ptr::copy_nonoverlapping(extend.as_ptr(), dst.as_mut_ptr().cast(), cnt); + } + + unsafe { + self.advance_mut(cnt); + } + } + + /// Absorbs a `BytesMut` that was previously split off. + /// + /// If the two `BytesMut` objects were previously contiguous and not mutated + /// in a way that causes re-allocation i.e., if `other` was created by + /// calling `split_off` on this `BytesMut`, then this is an `O(1)` operation + /// that just decreases a reference count and sets a few indices. + /// Otherwise this method degenerates to + /// `self.extend_from_slice(other.as_ref())`. + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// let mut buf = BytesMut::with_capacity(64); + /// buf.extend_from_slice(b"aaabbbcccddd"); + /// + /// let split = buf.split_off(6); + /// assert_eq!(b"aaabbb", &buf[..]); + /// assert_eq!(b"cccddd", &split[..]); + /// + /// buf.unsplit(split); + /// assert_eq!(b"aaabbbcccddd", &buf[..]); + /// ``` + pub fn unsplit(&mut self, other: BytesMut) { + if self.is_empty() { + *self = other; + return; + } + + if let Err(other) = self.try_unsplit(other) { + self.extend_from_slice(other.as_ref()); + } + } + + // private + + // For now, use a `Vec` to manage the memory for us, but we may want to + // change that in the future to some alternate allocator strategy. + // + // Thus, we don't expose an easy way to construct from a `Vec` since an + // internal change could make a simple pattern (`BytesMut::from(vec)`) + // suddenly a lot more expensive. + #[inline] + pub(crate) fn from_vec(vec: Vec<u8>) -> BytesMut { + let mut vec = ManuallyDrop::new(vec); + let ptr = vptr(vec.as_mut_ptr()); + let len = vec.len(); + let cap = vec.capacity(); + + let original_capacity_repr = original_capacity_to_repr(cap); + let data = (original_capacity_repr << ORIGINAL_CAPACITY_OFFSET) | KIND_VEC; + + BytesMut { + ptr, + len, + cap, + data: invalid_ptr(data), + } + } + + #[inline] + fn as_slice(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } + + #[inline] + fn as_slice_mut(&mut self) -> &mut [u8] { + unsafe { slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + } + + /// Advance the buffer without bounds checking. + /// + /// # SAFETY + /// + /// The caller must ensure that `count` <= `self.cap`. + pub(crate) unsafe fn advance_unchecked(&mut self, count: usize) { + // Setting the start to 0 is a no-op, so return early if this is the + // case. + if count == 0 { + return; + } + + debug_assert!(count <= self.cap, "internal: set_start out of bounds"); + + let kind = self.kind(); + + if kind == KIND_VEC { + // Setting the start when in vec representation is a little more + // complicated. First, we have to track how far ahead the + // "start" of the byte buffer from the beginning of the vec. We + // also have to ensure that we don't exceed the maximum shift. + let pos = self.get_vec_pos() + count; + + if pos <= MAX_VEC_POS { + self.set_vec_pos(pos); + } else { + // The repr must be upgraded to ARC. This will never happen + // on 64 bit systems and will only happen on 32 bit systems + // when shifting past 134,217,727 bytes. As such, we don't + // worry too much about performance here. + self.promote_to_shared(/*ref_count = */ 1); + } + } + + // Updating the start of the view is setting `ptr` to point to the + // new start and updating the `len` field to reflect the new length + // of the view. + self.ptr = vptr(self.ptr.as_ptr().add(count)); + self.len = self.len.saturating_sub(count); + self.cap -= count; + } + + fn try_unsplit(&mut self, other: BytesMut) -> Result<(), BytesMut> { + if other.capacity() == 0 { + return Ok(()); + } + + let ptr = unsafe { self.ptr.as_ptr().add(self.len) }; + if ptr == other.ptr.as_ptr() + && self.kind() == KIND_ARC + && other.kind() == KIND_ARC + && self.data == other.data + { + // Contiguous blocks, just combine directly + self.len += other.len; + self.cap += other.cap; + Ok(()) + } else { + Err(other) + } + } + + #[inline] + fn kind(&self) -> usize { + self.data as usize & KIND_MASK + } + + unsafe fn promote_to_shared(&mut self, ref_cnt: usize) { + debug_assert_eq!(self.kind(), KIND_VEC); + debug_assert!(ref_cnt == 1 || ref_cnt == 2); + + let original_capacity_repr = + (self.data as usize & ORIGINAL_CAPACITY_MASK) >> ORIGINAL_CAPACITY_OFFSET; + + // The vec offset cannot be concurrently mutated, so there + // should be no danger reading it. + let off = (self.data as usize) >> VEC_POS_OFFSET; + + // First, allocate a new `Shared` instance containing the + // `Vec` fields. It's important to note that `ptr`, `len`, + // and `cap` cannot be mutated without having `&mut self`. + // This means that these fields will not be concurrently + // updated and since the buffer hasn't been promoted to an + // `Arc`, those three fields still are the components of the + // vector. + let shared = Box::new(Shared { + vec: rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off), + original_capacity_repr, + ref_count: AtomicUsize::new(ref_cnt), + }); + + let shared = Box::into_raw(shared); + + // The pointer should be aligned, so this assert should + // always succeed. + debug_assert_eq!(shared as usize & KIND_MASK, KIND_ARC); + + self.data = shared; + } + + /// Makes an exact shallow clone of `self`. + /// + /// The kind of `self` doesn't matter, but this is unsafe + /// because the clone will have the same offsets. You must + /// be sure the returned value to the user doesn't allow + /// two views into the same range. + #[inline] + unsafe fn shallow_clone(&mut self) -> BytesMut { + if self.kind() == KIND_ARC { + increment_shared(self.data); + ptr::read(self) + } else { + self.promote_to_shared(/*ref_count = */ 2); + ptr::read(self) + } + } + + #[inline] + unsafe fn get_vec_pos(&self) -> usize { + debug_assert_eq!(self.kind(), KIND_VEC); + + self.data as usize >> VEC_POS_OFFSET + } + + #[inline] + unsafe fn set_vec_pos(&mut self, pos: usize) { + debug_assert_eq!(self.kind(), KIND_VEC); + debug_assert!(pos <= MAX_VEC_POS); + + self.data = invalid_ptr((pos << VEC_POS_OFFSET) | (self.data as usize & NOT_VEC_POS_MASK)); + } + + /// Returns the remaining spare capacity of the buffer as a slice of `MaybeUninit<u8>`. + /// + /// The returned slice can be used to fill the buffer with data (e.g. by + /// reading from a file) before marking the data as initialized using the + /// [`set_len`] method. + /// + /// [`set_len`]: BytesMut::set_len + /// + /// # Examples + /// + /// ``` + /// use bytes::BytesMut; + /// + /// // Allocate buffer big enough for 10 bytes. + /// let mut buf = BytesMut::with_capacity(10); + /// + /// // Fill in the first 3 elements. + /// let uninit = buf.spare_capacity_mut(); + /// uninit[0].write(0); + /// uninit[1].write(1); + /// uninit[2].write(2); + /// + /// // Mark the first 3 bytes of the buffer as being initialized. + /// unsafe { + /// buf.set_len(3); + /// } + /// + /// assert_eq!(&buf[..], &[0, 1, 2]); + /// ``` + #[inline] + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit<u8>] { + unsafe { + let ptr = self.ptr.as_ptr().add(self.len); + let len = self.cap - self.len; + + slice::from_raw_parts_mut(ptr.cast(), len) + } + } +} + +impl Drop for BytesMut { + fn drop(&mut self) { + let kind = self.kind(); + + if kind == KIND_VEC { + unsafe { + let off = self.get_vec_pos(); + + // Vector storage, free the vector + let _ = rebuild_vec(self.ptr.as_ptr(), self.len, self.cap, off); + } + } else if kind == KIND_ARC { + unsafe { release_shared(self.data) }; + } + } +} + +impl Buf for BytesMut { + #[inline] + fn remaining(&self) -> usize { + self.len() + } + + #[inline] + fn chunk(&self) -> &[u8] { + self.as_slice() + } + + #[inline] + fn advance(&mut self, cnt: usize) { + assert!( + cnt <= self.remaining(), + "cannot advance past `remaining`: {:?} <= {:?}", + cnt, + self.remaining(), + ); + unsafe { + // SAFETY: We've checked that `cnt` <= `self.remaining()` and we know that + // `self.remaining()` <= `self.cap`. + self.advance_unchecked(cnt); + } + } + + fn copy_to_bytes(&mut self, len: usize) -> Bytes { + self.split_to(len).freeze() + } +} + +unsafe impl BufMut for BytesMut { + #[inline] + fn remaining_mut(&self) -> usize { + // Max allocation size is isize::MAX. + isize::MAX as usize - self.len() + } + + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + let remaining = self.cap - self.len(); + if cnt > remaining { + super::panic_advance(&TryGetError { + requested: cnt, + available: remaining, + }); + } + // Addition won't overflow since it is at most `self.cap`. + self.len = self.len() + cnt; + } + + #[inline] + fn chunk_mut(&mut self) -> &mut UninitSlice { + if self.capacity() == self.len() { + self.reserve(64); + } + self.spare_capacity_mut().into() + } + + // Specialize these methods so they can skip checking `remaining_mut` + // and `advance_mut`. + + fn put<T: Buf>(&mut self, mut src: T) + where + Self: Sized, + { + if !src.has_remaining() { + // prevent calling `copy_to_bytes`->`put`->`copy_to_bytes` infintely when src is empty + return; + } else if self.capacity() == 0 { + // When capacity is zero, try reusing allocation of `src`. + let src_copy = src.copy_to_bytes(src.remaining()); + drop(src); + match src_copy.try_into_mut() { + Ok(bytes_mut) => *self = bytes_mut, + Err(bytes) => self.extend_from_slice(&bytes), + } + } else { + // In case the src isn't contiguous, reserve upfront. + self.reserve(src.remaining()); + + while src.has_remaining() { + let s = src.chunk(); + let l = s.len(); + self.extend_from_slice(s); + src.advance(l); + } + } + } + + fn put_slice(&mut self, src: &[u8]) { + self.extend_from_slice(src); + } + + fn put_bytes(&mut self, val: u8, cnt: usize) { + self.reserve(cnt); + unsafe { + let dst = self.spare_capacity_mut(); + // Reserved above + debug_assert!(dst.len() >= cnt); + + ptr::write_bytes(dst.as_mut_ptr(), val, cnt); + + self.advance_mut(cnt); + } + } +} + +impl AsRef<[u8]> for BytesMut { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + +impl Deref for BytesMut { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + self.as_ref() + } +} + +impl AsMut<[u8]> for BytesMut { + #[inline] + fn as_mut(&mut self) -> &mut [u8] { + self.as_slice_mut() + } +} + +impl DerefMut for BytesMut { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + self.as_mut() + } +} + +impl<'a> From<&'a [u8]> for BytesMut { + fn from(src: &'a [u8]) -> BytesMut { + BytesMut::from_vec(src.to_vec()) + } +} + +impl<'a> From<&'a str> for BytesMut { + fn from(src: &'a str) -> BytesMut { + BytesMut::from(src.as_bytes()) + } +} + +impl From<BytesMut> for Bytes { + fn from(src: BytesMut) -> Bytes { + src.freeze() + } +} + +impl PartialEq for BytesMut { + fn eq(&self, other: &BytesMut) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl PartialOrd for BytesMut { + fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for BytesMut { + fn cmp(&self, other: &BytesMut) -> cmp::Ordering { + self.as_slice().cmp(other.as_slice()) + } +} + +impl Eq for BytesMut {} + +impl Default for BytesMut { + #[inline] + fn default() -> BytesMut { + BytesMut::new() + } +} + +impl hash::Hash for BytesMut { + fn hash<H>(&self, state: &mut H) + where + H: hash::Hasher, + { + let s: &[u8] = self.as_ref(); + s.hash(state); + } +} + +impl Borrow<[u8]> for BytesMut { + fn borrow(&self) -> &[u8] { + self.as_ref() + } +} + +impl BorrowMut<[u8]> for BytesMut { + fn borrow_mut(&mut self) -> &mut [u8] { + self.as_mut() + } +} + +impl fmt::Write for BytesMut { + #[inline] + fn write_str(&mut self, s: &str) -> fmt::Result { + if self.remaining_mut() >= s.len() { + self.put_slice(s.as_bytes()); + Ok(()) + } else { + Err(fmt::Error) + } + } + + #[inline] + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> fmt::Result { + fmt::write(self, args) + } +} + +impl Clone for BytesMut { + fn clone(&self) -> BytesMut { + BytesMut::from(&self[..]) + } +} + +impl IntoIterator for BytesMut { + type Item = u8; + type IntoIter = IntoIter<BytesMut>; + + fn into_iter(self) -> Self::IntoIter { + IntoIter::new(self) + } +} + +impl<'a> IntoIterator for &'a BytesMut { + type Item = &'a u8; + type IntoIter = core::slice::Iter<'a, u8>; + + fn into_iter(self) -> Self::IntoIter { + self.as_ref().iter() + } +} + +impl Extend<u8> for BytesMut { + fn extend<T>(&mut self, iter: T) + where + T: IntoIterator<Item = u8>, + { + let iter = iter.into_iter(); + + let (lower, _) = iter.size_hint(); + self.reserve(lower); + + // TODO: optimize + // 1. If self.kind() == KIND_VEC, use Vec::extend + for b in iter { + self.put_u8(b); + } + } +} + +impl<'a> Extend<&'a u8> for BytesMut { + fn extend<T>(&mut self, iter: T) + where + T: IntoIterator<Item = &'a u8>, + { + self.extend(iter.into_iter().copied()) + } +} + +impl Extend<Bytes> for BytesMut { + fn extend<T>(&mut self, iter: T) + where + T: IntoIterator<Item = Bytes>, + { + for bytes in iter { + self.extend_from_slice(&bytes) + } + } +} + +impl FromIterator<u8> for BytesMut { + fn from_iter<T: IntoIterator<Item = u8>>(into_iter: T) -> Self { + BytesMut::from_vec(Vec::from_iter(into_iter)) + } +} + +impl<'a> FromIterator<&'a u8> for BytesMut { + fn from_iter<T: IntoIterator<Item = &'a u8>>(into_iter: T) -> Self { + BytesMut::from_iter(into_iter.into_iter().copied()) + } +} + +/* + * + * ===== Inner ===== + * + */ + +unsafe fn increment_shared(ptr: *mut Shared) { + let old_size = (*ptr).ref_count.fetch_add(1, Ordering::Relaxed); + + if old_size > isize::MAX as usize { + crate::abort(); + } +} + +unsafe fn release_shared(ptr: *mut Shared) { + // `Shared` storage... follow the drop steps from Arc. + if (*ptr).ref_count.fetch_sub(1, Ordering::Release) != 1 { + return; + } + + // This fence is needed to prevent reordering of use of the data and + // deletion of the data. Because it is marked `Release`, the decreasing + // of the reference count synchronizes with this `Acquire` fence. This + // means that use of the data happens before decreasing the reference + // count, which happens before this fence, which happens before the + // deletion of the data. + // + // As explained in the [Boost documentation][1], + // + // > It is important to enforce any possible access to the object in one + // > thread (through an existing reference) to *happen before* deleting + // > the object in a different thread. This is achieved by a "release" + // > operation after dropping a reference (any access to the object + // > through this reference must obviously happened before), and an + // > "acquire" operation before deleting the object. + // + // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html) + // + // Thread sanitizer does not support atomic fences. Use an atomic load + // instead. + (*ptr).ref_count.load(Ordering::Acquire); + + // Drop the data + drop(Box::from_raw(ptr)); +} + +impl Shared { + fn is_unique(&self) -> bool { + // The goal is to check if the current handle is the only handle + // that currently has access to the buffer. This is done by + // checking if the `ref_count` is currently 1. + // + // The `Acquire` ordering synchronizes with the `Release` as + // part of the `fetch_sub` in `release_shared`. The `fetch_sub` + // operation guarantees that any mutations done in other threads + // are ordered before the `ref_count` is decremented. As such, + // this `Acquire` will guarantee that those mutations are + // visible to the current thread. + self.ref_count.load(Ordering::Acquire) == 1 + } +} + +#[inline] +fn original_capacity_to_repr(cap: usize) -> usize { + let width = PTR_WIDTH - ((cap >> MIN_ORIGINAL_CAPACITY_WIDTH).leading_zeros() as usize); + cmp::min( + width, + MAX_ORIGINAL_CAPACITY_WIDTH - MIN_ORIGINAL_CAPACITY_WIDTH, + ) +} + +fn original_capacity_from_repr(repr: usize) -> usize { + if repr == 0 { + return 0; + } + + 1 << (repr + (MIN_ORIGINAL_CAPACITY_WIDTH - 1)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_original_capacity_to_repr() { + assert_eq!(original_capacity_to_repr(0), 0); + + let max_width = 32; + + for width in 1..(max_width + 1) { + let cap = 1 << width - 1; + + let expected = if width < MIN_ORIGINAL_CAPACITY_WIDTH { + 0 + } else if width < MAX_ORIGINAL_CAPACITY_WIDTH { + width - MIN_ORIGINAL_CAPACITY_WIDTH + } else { + MAX_ORIGINAL_CAPACITY_WIDTH - MIN_ORIGINAL_CAPACITY_WIDTH + }; + + assert_eq!(original_capacity_to_repr(cap), expected); + + if width > 1 { + assert_eq!(original_capacity_to_repr(cap + 1), expected); + } + + // MIN_ORIGINAL_CAPACITY_WIDTH must be bigger than 7 to pass tests below + if width == MIN_ORIGINAL_CAPACITY_WIDTH + 1 { + assert_eq!(original_capacity_to_repr(cap - 24), expected - 1); + assert_eq!(original_capacity_to_repr(cap + 76), expected); + } else if width == MIN_ORIGINAL_CAPACITY_WIDTH + 2 { + assert_eq!(original_capacity_to_repr(cap - 1), expected - 1); + assert_eq!(original_capacity_to_repr(cap - 48), expected - 1); + } + } + } + + #[test] + fn test_original_capacity_from_repr() { + assert_eq!(0, original_capacity_from_repr(0)); + + let min_cap = 1 << MIN_ORIGINAL_CAPACITY_WIDTH; + + assert_eq!(min_cap, original_capacity_from_repr(1)); + assert_eq!(min_cap * 2, original_capacity_from_repr(2)); + assert_eq!(min_cap * 4, original_capacity_from_repr(3)); + assert_eq!(min_cap * 8, original_capacity_from_repr(4)); + assert_eq!(min_cap * 16, original_capacity_from_repr(5)); + assert_eq!(min_cap * 32, original_capacity_from_repr(6)); + assert_eq!(min_cap * 64, original_capacity_from_repr(7)); + } +} + +unsafe impl Send for BytesMut {} +unsafe impl Sync for BytesMut {} + +/* + * + * ===== PartialEq / PartialOrd ===== + * + */ + +impl PartialEq<[u8]> for BytesMut { + fn eq(&self, other: &[u8]) -> bool { + &**self == other + } +} + +impl PartialOrd<[u8]> for BytesMut { + fn partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering> { + (**self).partial_cmp(other) + } +} + +impl PartialEq<BytesMut> for [u8] { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd<BytesMut> for [u8] { + fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other) + } +} + +impl PartialEq<str> for BytesMut { + fn eq(&self, other: &str) -> bool { + &**self == other.as_bytes() + } +} + +impl PartialOrd<str> for BytesMut { + fn partial_cmp(&self, other: &str) -> Option<cmp::Ordering> { + (**self).partial_cmp(other.as_bytes()) + } +} + +impl PartialEq<BytesMut> for str { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd<BytesMut> for str { + fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other) + } +} + +impl PartialEq<Vec<u8>> for BytesMut { + fn eq(&self, other: &Vec<u8>) -> bool { + *self == other[..] + } +} + +impl PartialOrd<Vec<u8>> for BytesMut { + fn partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering> { + (**self).partial_cmp(&other[..]) + } +} + +impl PartialEq<BytesMut> for Vec<u8> { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd<BytesMut> for Vec<u8> { + fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> { + other.partial_cmp(self) + } +} + +impl PartialEq<String> for BytesMut { + fn eq(&self, other: &String) -> bool { + *self == other[..] + } +} + +impl PartialOrd<String> for BytesMut { + fn partial_cmp(&self, other: &String) -> Option<cmp::Ordering> { + (**self).partial_cmp(other.as_bytes()) + } +} + +impl PartialEq<BytesMut> for String { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd<BytesMut> for String { + fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other) + } +} + +impl<'a, T: ?Sized> PartialEq<&'a T> for BytesMut +where + BytesMut: PartialEq<T>, +{ + fn eq(&self, other: &&'a T) -> bool { + *self == **other + } +} + +impl<'a, T: ?Sized> PartialOrd<&'a T> for BytesMut +where + BytesMut: PartialOrd<T>, +{ + fn partial_cmp(&self, other: &&'a T) -> Option<cmp::Ordering> { + self.partial_cmp(*other) + } +} + +impl PartialEq<BytesMut> for &[u8] { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd<BytesMut> for &[u8] { + fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> { + <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other) + } +} + +impl PartialEq<BytesMut> for &str { + fn eq(&self, other: &BytesMut) -> bool { + *other == *self + } +} + +impl PartialOrd<BytesMut> for &str { + fn partial_cmp(&self, other: &BytesMut) -> Option<cmp::Ordering> { + other.partial_cmp(self) + } +} + +impl PartialEq<BytesMut> for Bytes { + fn eq(&self, other: &BytesMut) -> bool { + other[..] == self[..] + } +} + +impl PartialEq<Bytes> for BytesMut { + fn eq(&self, other: &Bytes) -> bool { + other[..] == self[..] + } +} + +impl From<BytesMut> for Vec<u8> { + fn from(bytes: BytesMut) -> Self { + let kind = bytes.kind(); + let bytes = ManuallyDrop::new(bytes); + + let mut vec = if kind == KIND_VEC { + unsafe { + let off = bytes.get_vec_pos(); + rebuild_vec(bytes.ptr.as_ptr(), bytes.len, bytes.cap, off) + } + } else { + let shared = bytes.data; + + if unsafe { (*shared).is_unique() } { + let vec = core::mem::take(unsafe { &mut (*shared).vec }); + + unsafe { release_shared(shared) }; + + vec + } else { + return ManuallyDrop::into_inner(bytes).deref().to_vec(); + } + }; + + let len = bytes.len; + + unsafe { + ptr::copy(bytes.ptr.as_ptr(), vec.as_mut_ptr(), len); + vec.set_len(len); + } + + vec + } +} + +#[inline] +fn vptr(ptr: *mut u8) -> NonNull<u8> { + if cfg!(debug_assertions) { + NonNull::new(ptr).expect("Vec pointer should be non-null") + } else { + unsafe { NonNull::new_unchecked(ptr) } + } +} + +/// Returns a dangling pointer with the given address. This is used to store +/// integer data in pointer fields. +/// +/// It is equivalent to `addr as *mut T`, but this fails on miri when strict +/// provenance checking is enabled. +#[inline] +fn invalid_ptr<T>(addr: usize) -> *mut T { + let ptr = core::ptr::null_mut::<u8>().wrapping_add(addr); + debug_assert_eq!(ptr as usize, addr); + ptr.cast::<T>() +} + +unsafe fn rebuild_vec(ptr: *mut u8, mut len: usize, mut cap: usize, off: usize) -> Vec<u8> { + let ptr = ptr.sub(off); + len += off; + cap += off; + + Vec::from_raw_parts(ptr, len, cap) +} + +// ===== impl SharedVtable ===== + +static SHARED_VTABLE: Vtable = Vtable { + owner_allocation: None, + clone: shared_v_clone, + into_vec: shared_v_to_vec, + into_mut: shared_v_to_mut, + is_unique: shared_v_is_unique, + drop: shared_v_drop, +}; + +unsafe fn shared_v_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { + let shared = data.load(Ordering::Relaxed) as *mut Shared; + increment_shared(shared); + + let data = AtomicPtr::new(shared as *mut ()); + Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE) +} + +unsafe fn shared_v_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { + let shared: *mut Shared = data.load(Ordering::Relaxed).cast(); + + if (*shared).is_unique() { + let shared = &mut *shared; + + // Drop shared + let mut vec = core::mem::take(&mut shared.vec); + release_shared(shared); + + // Copy back buffer + ptr::copy(ptr, vec.as_mut_ptr(), len); + vec.set_len(len); + + vec + } else { + let v = slice::from_raw_parts(ptr, len).to_vec(); + release_shared(shared); + v + } +} + +unsafe fn shared_v_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { + let shared: *mut Shared = data.load(Ordering::Relaxed).cast(); + + if (*shared).is_unique() { + let shared = &mut *shared; + + // The capacity is always the original capacity of the buffer + // minus the offset from the start of the buffer + let v = &mut shared.vec; + let v_capacity = v.capacity(); + let v_ptr = v.as_mut_ptr(); + let offset = ptr.offset_from(v_ptr) as usize; + let cap = v_capacity - offset; + + let ptr = vptr(ptr as *mut u8); + + BytesMut { + ptr, + len, + cap, + data: shared, + } + } else { + let v = slice::from_raw_parts(ptr, len).to_vec(); + release_shared(shared); + BytesMut::from_vec(v) + } +} + +unsafe fn shared_v_is_unique(data: &AtomicPtr<()>) -> bool { + let shared = data.load(Ordering::Acquire); + let ref_count = (*shared.cast::<Shared>()).ref_count.load(Ordering::Relaxed); + ref_count == 1 +} + +unsafe fn shared_v_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { + data.with_mut(|shared| { + release_shared(*shared as *mut Shared); + }); +} + +// compile-fails + +/// ```compile_fail +/// use bytes::BytesMut; +/// #[deny(unused_must_use)] +/// { +/// let mut b1 = BytesMut::from("hello world"); +/// b1.split_to(6); +/// } +/// ``` +fn _split_to_must_use() {} + +/// ```compile_fail +/// use bytes::BytesMut; +/// #[deny(unused_must_use)] +/// { +/// let mut b1 = BytesMut::from("hello world"); +/// b1.split_off(6); +/// } +/// ``` +fn _split_off_must_use() {} + +/// ```compile_fail +/// use bytes::BytesMut; +/// #[deny(unused_must_use)] +/// { +/// let mut b1 = BytesMut::from("hello world"); +/// b1.split(); +/// } +/// ``` +fn _split_must_use() {} + +// fuzz tests +#[cfg(all(test, loom))] +mod fuzz { + use loom::sync::Arc; + use loom::thread; + + use super::BytesMut; + use crate::Bytes; + + #[test] + fn bytes_mut_cloning_frozen() { + loom::model(|| { + let a = BytesMut::from(&b"abcdefgh"[..]).split().freeze(); + let addr = a.as_ptr() as usize; + + // test the Bytes::clone is Sync by putting it in an Arc + let a1 = Arc::new(a); + let a2 = a1.clone(); + + let t1 = thread::spawn(move || { + let b: Bytes = (*a1).clone(); + assert_eq!(b.as_ptr() as usize, addr); + }); + + let t2 = thread::spawn(move || { + let b: Bytes = (*a2).clone(); + assert_eq!(b.as_ptr() as usize, addr); + }); + + t1.join().unwrap(); + t2.join().unwrap(); + }); + } +} diff --git a/third-party/bytes/src/fmt/debug.rs b/third-party/bytes/src/fmt/debug.rs new file mode 100644 index 000000000..82d0aa5e3 --- /dev/null +++ b/third-party/bytes/src/fmt/debug.rs @@ -0,0 +1,40 @@ +use core::fmt::{Debug, Formatter, Result}; + +use super::BytesRef; +use crate::{Bytes, BytesMut}; + +/// Alternative implementation of `std::fmt::Debug` for byte slice. +/// +/// Standard `Debug` implementation for `[u8]` is comma separated +/// list of numbers. Since large amount of byte strings are in fact +/// ASCII strings or contain a lot of ASCII strings (e. g. HTTP), +/// it is convenient to print strings as ASCII when possible. +impl Debug for BytesRef<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + write!(f, "b\"")?; + for &b in self.0 { + // https://doc.rust-lang.org/reference/tokens.html#byte-escapes + if b == b'\n' { + write!(f, "\\n")?; + } else if b == b'\r' { + write!(f, "\\r")?; + } else if b == b'\t' { + write!(f, "\\t")?; + } else if b == b'\\' || b == b'"' { + write!(f, "\\{}", b as char)?; + } else if b == b'\0' { + write!(f, "\\0")?; + // ASCII printable + } else if (0x20..0x7f).contains(&b) { + write!(f, "{}", b as char)?; + } else { + write!(f, "\\x{:02x}", b)?; + } + } + write!(f, "\"")?; + Ok(()) + } +} + +fmt_impl!(Debug, Bytes); +fmt_impl!(Debug, BytesMut); diff --git a/third-party/bytes/src/fmt/hex.rs b/third-party/bytes/src/fmt/hex.rs new file mode 100644 index 000000000..1203b4198 --- /dev/null +++ b/third-party/bytes/src/fmt/hex.rs @@ -0,0 +1,27 @@ +use core::fmt::{Formatter, LowerHex, Result, UpperHex}; + +use super::BytesRef; +use crate::{Bytes, BytesMut}; + +impl LowerHex for BytesRef<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + for &b in self.0 { + write!(f, "{:02x}", b)?; + } + Ok(()) + } +} + +impl UpperHex for BytesRef<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + for &b in self.0 { + write!(f, "{:02X}", b)?; + } + Ok(()) + } +} + +fmt_impl!(LowerHex, Bytes); +fmt_impl!(LowerHex, BytesMut); +fmt_impl!(UpperHex, Bytes); +fmt_impl!(UpperHex, BytesMut); diff --git a/third-party/bytes/src/fmt/mod.rs b/third-party/bytes/src/fmt/mod.rs new file mode 100644 index 000000000..b8a0eafaf --- /dev/null +++ b/third-party/bytes/src/fmt/mod.rs @@ -0,0 +1,15 @@ +macro_rules! fmt_impl { + ($tr:ident, $ty:ty) => { + impl $tr for $ty { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + $tr::fmt(&BytesRef(self.as_ref()), f) + } + } + }; +} + +mod debug; +mod hex; + +/// `BytesRef` is not a part of public API of bytes crate. +struct BytesRef<'a>(&'a [u8]); diff --git a/third-party/bytes/src/lib.rs b/third-party/bytes/src/lib.rs new file mode 100644 index 000000000..e5e71a134 --- /dev/null +++ b/third-party/bytes/src/lib.rs @@ -0,0 +1,182 @@ +#![warn(missing_docs, missing_debug_implementations, rust_2018_idioms)] +#![doc(test( + no_crate_inject, + attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) +))] +#![no_std] +#![cfg_attr(docsrs, feature(doc_cfg))] + +//! Provides abstractions for working with bytes. +//! +//! The `bytes` crate provides an efficient byte buffer structure +//! ([`Bytes`]) and traits for working with buffer +//! implementations ([`Buf`], [`BufMut`]). +//! +//! # `Bytes` +//! +//! `Bytes` is an efficient container for storing and operating on contiguous +//! slices of memory. It is intended for use primarily in networking code, but +//! could have applications elsewhere as well. +//! +//! `Bytes` values facilitate zero-copy network programming by allowing multiple +//! `Bytes` objects to point to the same underlying memory. This is managed by +//! using a reference count to track when the memory is no longer needed and can +//! be freed. +//! +//! A `Bytes` handle can be created directly from an existing byte store (such as `&[u8]` +//! or `Vec<u8>`), but usually a `BytesMut` is used first and written to. For +//! example: +//! +//! ```rust +//! use bytes::{BytesMut, BufMut}; +//! +//! let mut buf = BytesMut::with_capacity(1024); +//! buf.put(&b"hello world"[..]); +//! buf.put_u16(1234); +//! +//! let a = buf.split(); +//! assert_eq!(a, b"hello world\x04\xD2"[..]); +//! +//! buf.put(&b"goodbye world"[..]); +//! +//! let b = buf.split(); +//! assert_eq!(b, b"goodbye world"[..]); +//! +//! assert_eq!(buf.capacity(), 998); +//! ``` +//! +//! In the above example, only a single buffer of 1024 is allocated. The handles +//! `a` and `b` will share the underlying buffer and maintain indices tracking +//! the view into the buffer represented by the handle. +//! +//! See the [struct docs](`Bytes`) for more details. +//! +//! # `Buf`, `BufMut` +//! +//! These two traits provide read and write access to buffers. The underlying +//! storage may or may not be in contiguous memory. For example, `Bytes` is a +//! buffer that guarantees contiguous memory, but a [rope] stores the bytes in +//! disjoint chunks. `Buf` and `BufMut` maintain cursors tracking the current +//! position in the underlying byte storage. When bytes are read or written, the +//! cursor is advanced. +//! +//! [rope]: https://en.wikipedia.org/wiki/Rope_(data_structure) +//! +//! ## Relation with `Read` and `Write` +//! +//! At first glance, it may seem that `Buf` and `BufMut` overlap in +//! functionality with [`std::io::Read`] and [`std::io::Write`]. However, they +//! serve different purposes. A buffer is the value that is provided as an +//! argument to `Read::read` and `Write::write`. `Read` and `Write` may then +//! perform a syscall, which has the potential of failing. Operations on `Buf` +//! and `BufMut` are infallible. + +extern crate alloc; + +#[cfg(feature = "std")] +extern crate std; + +pub mod buf; +pub use crate::buf::{Buf, BufMut}; + +mod bytes; +mod bytes_mut; +mod fmt; +mod loom; +pub use crate::bytes::{Bytes, OwnerAllocation, OwnerAllocationContext, OwnerAllocationError, OwnerAllocationLease}; +pub use crate::bytes_mut::BytesMut; + +// Optional Serde support +#[cfg(feature = "serde")] +mod serde; + +#[inline(never)] +#[cold] +fn abort() -> ! { + #[cfg(feature = "std")] + { + std::process::abort(); + } + + #[cfg(not(feature = "std"))] + { + struct Abort; + impl Drop for Abort { + fn drop(&mut self) { + panic!(); + } + } + let _a = Abort; + panic!("abort"); + } +} + +#[inline(always)] +#[cfg(feature = "std")] +fn saturating_sub_usize_u64(a: usize, b: u64) -> usize { + match usize::try_from(b) { + Ok(b) => a.saturating_sub(b), + Err(_) => 0, + } +} + +#[inline(always)] +#[cfg(feature = "std")] +fn min_u64_usize(a: u64, b: usize) -> usize { + match usize::try_from(a) { + Ok(a) => usize::min(a, b), + Err(_) => b, + } +} + +/// Error type for the `try_get_` methods of [`Buf`]. +/// Indicates that there were not enough remaining +/// bytes in the buffer while attempting +/// to get a value from a [`Buf`] with one +/// of the `try_get_` methods. +#[derive(Debug, PartialEq, Eq)] +pub struct TryGetError { + /// The number of bytes necessary to get the value + pub requested: usize, + + /// The number of bytes available in the buffer + pub available: usize, +} + +impl core::fmt::Display for TryGetError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + write!( + f, + "Not enough bytes remaining in buffer to read value (requested {} but only {} available)", + self.requested, + self.available + ) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for TryGetError {} + +#[cfg(feature = "std")] +impl From<TryGetError> for std::io::Error { + fn from(error: TryGetError) -> Self { + std::io::Error::new(std::io::ErrorKind::Other, error) + } +} + +/// Panic with a nice error message. +#[cold] +fn panic_advance(error_info: &TryGetError) -> ! { + panic!( + "advance out of bounds: the len is {} but advancing by {}", + error_info.available, error_info.requested + ); +} + +#[cold] +fn panic_does_not_fit(size: usize, nbytes: usize) -> ! { + panic!( + "size too large: the integer type can fit {} bytes, but nbytes is {}", + size, nbytes + ); +} diff --git a/third-party/bytes/src/loom.rs b/third-party/bytes/src/loom.rs new file mode 100644 index 000000000..c80929092 --- /dev/null +++ b/third-party/bytes/src/loom.rs @@ -0,0 +1,33 @@ +#[cfg(not(all(test, loom)))] +pub(crate) mod sync { + pub(crate) mod atomic { + #[cfg(not(feature = "extra-platforms"))] + pub(crate) use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + #[cfg(feature = "extra-platforms")] + pub(crate) use extra_platforms::{AtomicPtr, AtomicUsize, Ordering}; + + pub(crate) trait AtomicMut<T> { + fn with_mut<F, R>(&mut self, f: F) -> R + where + F: FnOnce(&mut *mut T) -> R; + } + + impl<T> AtomicMut<T> for AtomicPtr<T> { + fn with_mut<F, R>(&mut self, f: F) -> R + where + F: FnOnce(&mut *mut T) -> R, + { + f(self.get_mut()) + } + } + } +} + +#[cfg(all(test, loom))] +pub(crate) mod sync { + pub(crate) mod atomic { + pub(crate) use loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + + pub(crate) trait AtomicMut<T> {} + } +} diff --git a/third-party/bytes/src/serde.rs b/third-party/bytes/src/serde.rs new file mode 100644 index 000000000..0a5bd144a --- /dev/null +++ b/third-party/bytes/src/serde.rs @@ -0,0 +1,89 @@ +use super::{Bytes, BytesMut}; +use alloc::string::String; +use alloc::vec::Vec; +use core::{cmp, fmt}; +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; + +macro_rules! serde_impl { + ($ty:ident, $visitor_ty:ident, $from_slice:ident, $from_vec:ident) => { + impl Serialize for $ty { + #[inline] + fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + serializer.serialize_bytes(&self) + } + } + + struct $visitor_ty; + + impl<'de> de::Visitor<'de> for $visitor_ty { + type Value = $ty; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("byte array") + } + + #[inline] + fn visit_seq<V>(self, mut seq: V) -> Result<Self::Value, V::Error> + where + V: de::SeqAccess<'de>, + { + let len = cmp::min(seq.size_hint().unwrap_or(0), 4096); + let mut values: Vec<u8> = Vec::with_capacity(len); + + while let Some(value) = seq.next_element()? { + values.push(value); + } + + Ok($ty::$from_vec(values)) + } + + #[inline] + fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E> + where + E: de::Error, + { + Ok($ty::$from_slice(v)) + } + + #[inline] + fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E> + where + E: de::Error, + { + Ok($ty::$from_vec(v)) + } + + #[inline] + fn visit_str<E>(self, v: &str) -> Result<Self::Value, E> + where + E: de::Error, + { + Ok($ty::$from_slice(v.as_bytes())) + } + + #[inline] + fn visit_string<E>(self, v: String) -> Result<Self::Value, E> + where + E: de::Error, + { + Ok($ty::$from_vec(v.into_bytes())) + } + } + + impl<'de> Deserialize<'de> for $ty { + #[inline] + fn deserialize<D>(deserializer: D) -> Result<$ty, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_byte_buf($visitor_ty) + } + } + }; +} + +serde_impl!(Bytes, BytesVisitor, copy_from_slice, from); +serde_impl!(BytesMut, BytesMutVisitor, from, from_vec); diff --git a/third-party/bytes/tests/test_accounted_owner.rs b/third-party/bytes/tests/test_accounted_owner.rs new file mode 100644 index 000000000..1c24f61a5 --- /dev/null +++ b/third-party/bytes/tests/test_accounted_owner.rs @@ -0,0 +1,214 @@ +use bytes::{Bytes, OwnerAllocation}; +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; +use std::ptr; +use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; +use std::sync::Arc; + +thread_local! { static CLAIMED: Cell<*const State> = const { Cell::new(ptr::null()) }; } +#[global_allocator] +static TRACKED: Tracked = Tracked; +static ALLOCATION: AtomicPtr<u8> = AtomicPtr::new(ptr::null_mut()); +static OBSERVER: AtomicPtr<State> = AtomicPtr::new(ptr::null_mut()); +struct Tracked; +unsafe impl GlobalAlloc for Tracked { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let result = System.alloc(layout); + let _ = CLAIMED.try_with(|state| { + let state = state.replace(ptr::null()); + if !state.is_null() { + assert_eq!((*state).bytes.load(Ordering::SeqCst), layout.size()); + OBSERVER.store(state.cast_mut(), Ordering::SeqCst); + ALLOCATION.store(result, Ordering::SeqCst); + } + }); + result + } + unsafe fn dealloc(&self, data: *mut u8, layout: Layout) { + let tracked = ALLOCATION + .compare_exchange(data, ptr::null_mut(), Ordering::SeqCst, Ordering::SeqCst) + .is_ok(); + System.dealloc(data, layout); + if tracked { + (*OBSERVER.swap(ptr::null_mut(), Ordering::SeqCst)) + .freed + .store(true, Ordering::SeqCst); + } + } +} + +#[derive(Default)] +struct State { + bytes: AtomicUsize, + claims: AtomicUsize, + releases: AtomicUsize, + owner_drops: AtomicUsize, + freed: AtomicBool, + refuse: bool, +} +struct Account(Arc<State>); +impl OwnerAllocation for Account { + type Error = (); + fn claim(&self, layout: Layout) -> Result<(), ()> { + if self.0.refuse { + return Err(()); + } + self.0.claims.fetch_add(1, Ordering::SeqCst); + self.0.bytes.store(layout.size(), Ordering::SeqCst); + CLAIMED.with(|state| state.set(Arc::as_ptr(&self.0))); + Ok(()) + } + fn release(&self, layout: Layout) { + assert!( + self.0.freed.load(Ordering::SeqCst), + "refund ran before actual deallocation" + ); + assert_eq!(self.0.owner_drops.load(Ordering::SeqCst), 1); + assert_eq!(self.0.bytes.swap(0, Ordering::SeqCst), layout.size()); + self.0.releases.fetch_add(1, Ordering::SeqCst); + } +} +struct Owner { + data: [u8; 64], + state: Arc<State>, + panic_read: bool, + panic_drop: bool, +} +impl AsRef<[u8]> for Owner { + fn as_ref(&self) -> &[u8] { + assert!(!self.panic_read, "expected AsRef panic"); + &self.data + } +} +impl Drop for Owner { + fn drop(&mut self) { + self.state.owner_drops.fetch_add(1, Ordering::SeqCst); + assert!(!self.panic_drop, "expected owner Drop panic"); + } +} +fn make(state: &Arc<State>, panic_read: bool, panic_drop: bool) -> Result<Bytes, ()> { + Bytes::try_from_owner_accounted( + Owner { + data: [7; 64], + state: state.clone(), + panic_read, + panic_drop, + }, + Account(state.clone()), + ) +} +fn released(state: &State) { + assert_eq!(state.claims.load(Ordering::SeqCst), 1); + assert_eq!(state.releases.load(Ordering::SeqCst), 1); + assert_eq!(state.bytes.load(Ordering::SeqCst), 0); +} + +// One test owns the allocator observation slot; all paths, including a final +// clone on another thread, verify the free-before-refund ordering directly. +#[test] +fn exact_layout_and_free_before_refund_cover_every_owner_exit() { + let state = Arc::new(State::default()); + let bytes = make(&state, false, false).unwrap(); + let slice = bytes.slice(8..32); + drop(bytes); + assert_eq!(state.releases.load(Ordering::SeqCst), 0); + std::thread::spawn(move || assert_eq!(slice.as_ref(), &[7; 24])) + .join() + .unwrap(); + released(&state); + + let state = Arc::new(State::default()); + let bytes = make(&state, false, false).unwrap(); + assert_eq!(Vec::from(bytes), vec![7; 64]); + released(&state); + + let state = Arc::new(State::default()); + let result = std::panic::catch_unwind(|| make(&state, true, false)); + assert!(result.is_err()); + released(&state); + + let state = Arc::new(State::default()); + let bytes = make(&state, false, true).unwrap(); + let result = std::panic::catch_unwind(|| drop(bytes)); + assert!(result.is_err()); + released(&state); + + let state = Arc::new(State { + refuse: true, + ..Default::default() + }); + assert!(make(&state, false, false).is_err()); + assert_eq!(state.owner_drops.load(Ordering::SeqCst), 1); + assert_eq!(state.claims.load(Ordering::SeqCst), 0); + assert_eq!(state.releases.load(Ordering::SeqCst), 0); + external_control_leases_retain_the_same_account_until_last_refund(); +} + +#[derive(Default)] +struct Ledger { + admitted: AtomicUsize, + allocated: AtomicUsize, + refuse: AtomicBool, +} +struct SharedAccount(Arc<Ledger>); +impl OwnerAllocation for SharedAccount { + type Error = (); + fn claim(&self, layout: Layout) -> Result<(), ()> { + if self.0.refuse.load(Ordering::SeqCst) { + return Err(()); + } + self.0.admitted.fetch_add(layout.size(), Ordering::SeqCst); + Ok(()) + } + fn record(&self, layout: Layout, acquired: bool) { + if acquired { + self.0.allocated.fetch_add(layout.size(), Ordering::SeqCst); + } else { + assert!(self.0.allocated.fetch_sub(layout.size(), Ordering::SeqCst) >= layout.size()); + } + } + fn release(&self, layout: Layout) { + assert!(self.0.admitted.fetch_sub(layout.size(), Ordering::SeqCst) >= layout.size()); + } +} +fn external_control_leases_retain_the_same_account_until_last_refund() { + let ledger = Arc::new(Ledger::default()); + let bytes = Bytes::try_from_owner_accounted([1u8; 32], SharedAccount(ledger.clone())).unwrap(); + let owner_bytes = ledger.allocated.load(Ordering::SeqCst); + assert_eq!(ledger.admitted.load(Ordering::SeqCst), owner_bytes); + let layout = Layout::new::<[u8; 17]>(); + let refused_storage = bytes.try_owner_allocation(layout).unwrap().unwrap(); + assert_eq!(ledger.admitted.load(Ordering::SeqCst), owner_bytes + 17); + assert_eq!(ledger.allocated.load(Ordering::SeqCst), owner_bytes); + drop(refused_storage); + assert_eq!(ledger.admitted.load(Ordering::SeqCst), owner_bytes); + let mut lease = bytes + .slice(1..) + .try_owner_allocation(layout) + .unwrap() + .unwrap(); + let storage = Box::new([0u8; 17]); + lease.allocated(); + lease.allocated(); + assert_eq!(ledger.allocated.load(Ordering::SeqCst), owner_bytes + 17); + let callbacks = bytes.owner_allocation().unwrap(); + callbacks.claim(Layout::new::<u64>()).unwrap(); + let native = Box::new(3u64); + callbacks.record(Layout::new::<u64>(), true); + drop(native); + callbacks.record(Layout::new::<u64>(), false); + callbacks.release(Layout::new::<u64>()); + ledger.refuse.store(true, Ordering::SeqCst); + assert!(bytes.try_owner_allocation(layout).is_err()); + assert_eq!(ledger.admitted.load(Ordering::SeqCst), owner_bytes + 17); + drop(bytes); + assert_eq!(ledger.allocated.load(Ordering::SeqCst), owner_bytes + 17); + drop(storage); + drop(lease); + assert_eq!(ledger.allocated.load(Ordering::SeqCst), 0); + assert_eq!(ledger.admitted.load(Ordering::SeqCst), 0); + assert!(Bytes::from_static(b"ordinary") + .try_owner_allocation(layout) + .unwrap() + .is_none()); +} diff --git a/third-party/bytes/tests/test_buf.rs b/third-party/bytes/tests/test_buf.rs new file mode 100644 index 000000000..099016e24 --- /dev/null +++ b/third-party/bytes/tests/test_buf.rs @@ -0,0 +1,439 @@ +#![warn(rust_2018_idioms)] + +use ::bytes::{Buf, Bytes, BytesMut}; +use core::{cmp, mem}; +use std::collections::VecDeque; +#[cfg(feature = "std")] +use std::io::IoSlice; + +// A random 64-byte ascii string, with the first 8 bytes altered to +// give valid representations of f32 and f64 (making them easier to compare) +// and negative signed numbers when interpreting as big endian +// (testing Sign Extension for `Buf::get_int' and `Buf::get_int_ne`). +const INPUT: &[u8] = b"\xffFqrjrDqPhvTc45vvq33f6bJrUtyHESuTeklWKgYd64xgzxJwvAkpYYnpNJyZSRn"; + +macro_rules! e { + ($big_endian_val:expr, $little_endian_val:expr) => { + if cfg!(target_endian = "big") { + $big_endian_val + } else { + $little_endian_val + } + }; +} + +macro_rules! buf_tests { + ($make_input:ident) => { + buf_tests!($make_input, true); + }; + ($make_input:ident, $checks_vectored_is_complete:expr) => { + use super::*; + + #[test] + fn empty_state() { + let buf = $make_input(&[]); + assert_eq!(buf.remaining(), 0); + assert!(!buf.has_remaining()); + assert!(buf.chunk().is_empty()); + } + + #[test] + fn fresh_state() { + let buf = $make_input(INPUT); + assert_eq!(buf.remaining(), 64); + assert!(buf.has_remaining()); + + let chunk = buf.chunk(); + assert!(chunk.len() <= 64); + assert!(INPUT.starts_with(chunk)); + } + + #[test] + fn advance() { + let mut buf = $make_input(INPUT); + buf.advance(8); + assert_eq!(buf.remaining(), 64 - 8); + assert!(buf.has_remaining()); + + let chunk = buf.chunk(); + assert!(chunk.len() <= 64 - 8); + assert!(INPUT[8..].starts_with(chunk)); + } + + #[test] + fn advance_to_end() { + let mut buf = $make_input(INPUT); + buf.advance(64); + assert_eq!(buf.remaining(), 0); + assert!(!buf.has_remaining()); + + let chunk = buf.chunk(); + assert!(chunk.is_empty()); + } + + #[test] + #[should_panic] + fn advance_past_end() { + let mut buf = $make_input(INPUT); + buf.advance(65); + } + + #[test] + #[cfg(feature = "std")] + fn chunks_vectored_empty() { + let buf = $make_input(&[]); + let mut bufs = [IoSlice::new(&[]); 16]; + + let n = buf.chunks_vectored(&mut bufs); + assert_eq!(n, 0); + assert!(bufs.iter().all(|buf| buf.is_empty())); + } + + #[test] + #[cfg(feature = "std")] + fn chunks_vectored_is_complete() { + let buf = $make_input(INPUT); + let mut bufs = [IoSlice::new(&[]); 16]; + + let n = buf.chunks_vectored(&mut bufs); + assert!(n > 0); + assert!(n <= 16); + + let bufs_concat = bufs[..n] + .iter() + .flat_map(|b| b.iter().copied()) + .collect::<Vec<u8>>(); + if $checks_vectored_is_complete { + assert_eq!(bufs_concat, INPUT); + } else { + // If this panics then `buf` implements `chunks_vectored`. + // Remove the `false` argument from `buf_tests!` for that type. + assert!(bufs_concat.len() < INPUT.len()); + assert!(INPUT.starts_with(&bufs_concat)); + } + + for i in n..16 { + assert!(bufs[i].is_empty()); + } + } + + #[test] + fn copy_to_slice() { + let mut buf = $make_input(INPUT); + + let mut chunk = [0u8; 8]; + buf.copy_to_slice(&mut chunk); + assert_eq!(buf.remaining(), 64 - 8); + assert!(buf.has_remaining()); + assert_eq!(chunk, INPUT[..8]); + + let chunk = buf.chunk(); + assert!(chunk.len() <= 64 - 8); + assert!(INPUT[8..].starts_with(chunk)); + } + + #[test] + fn copy_to_slice_big() { + let mut buf = $make_input(INPUT); + + let mut chunk = [0u8; 56]; + buf.copy_to_slice(&mut chunk); + assert_eq!(buf.remaining(), 64 - 56); + assert!(buf.has_remaining()); + assert_eq!(chunk, INPUT[..56]); + + let chunk = buf.chunk(); + assert!(chunk.len() <= 64 - 56); + assert!(INPUT[56..].starts_with(chunk)); + } + + #[test] + fn copy_to_slice_to_end() { + let mut buf = $make_input(INPUT); + + let mut chunk = [0u8; 64]; + buf.copy_to_slice(&mut chunk); + assert_eq!(buf.remaining(), 0); + assert!(!buf.has_remaining()); + assert_eq!(chunk, INPUT); + + assert!(buf.chunk().is_empty()); + } + + #[test] + #[should_panic] + fn copy_to_slice_overflow() { + let mut buf = $make_input(INPUT); + + let mut chunk = [0u8; 65]; + buf.copy_to_slice(&mut chunk); + } + + #[test] + fn copy_to_bytes() { + let mut buf = $make_input(INPUT); + + let chunk = buf.copy_to_bytes(8); + assert_eq!(buf.remaining(), 64 - 8); + assert!(buf.has_remaining()); + assert_eq!(chunk, INPUT[..8]); + + let chunk = buf.chunk(); + assert!(chunk.len() <= 64 - 8); + assert!(INPUT[8..].starts_with(chunk)); + } + + #[test] + fn copy_to_bytes_big() { + let mut buf = $make_input(INPUT); + + let chunk = buf.copy_to_bytes(56); + assert_eq!(buf.remaining(), 64 - 56); + assert!(buf.has_remaining()); + assert_eq!(chunk, INPUT[..56]); + + let chunk = buf.chunk(); + assert!(chunk.len() <= 64 - 56); + assert!(INPUT[56..].starts_with(chunk)); + } + + #[test] + fn copy_to_bytes_to_end() { + let mut buf = $make_input(INPUT); + + let chunk = buf.copy_to_bytes(64); + assert_eq!(buf.remaining(), 0); + assert!(!buf.has_remaining()); + assert_eq!(chunk, INPUT); + + assert!(buf.chunk().is_empty()); + } + + #[test] + #[should_panic] + fn copy_to_bytes_overflow() { + let mut buf = $make_input(INPUT); + + let _ = buf.copy_to_bytes(65); + } + + buf_tests!(number $make_input, get_u8, get_u8_overflow, u8, get_u8, 0xff); + buf_tests!(number $make_input, get_i8, get_i8_overflow, i8, get_i8, 0xffu8 as i8); + buf_tests!(number $make_input, get_u16_be, get_u16_be_overflow, u16, get_u16, 0xff46); + buf_tests!(number $make_input, get_u16_le, get_u16_le_overflow, u16, get_u16_le, 0x46ff); + buf_tests!(number $make_input, get_u16_ne, get_u16_ne_overflow, u16, get_u16_ne, e!(0xff46, 0x46ff)); + buf_tests!(number $make_input, get_i16_be, get_i16_be_overflow, i16, get_i16, 0xff46u16 as i16); + buf_tests!(number $make_input, get_i16_le, get_i16_le_overflow, i16, get_i16_le, 0x46ff); + buf_tests!(number $make_input, get_i16_ne, get_i16_ne_overflow, i16, get_i16_ne, e!(0xff46u16 as i16, 0x46ff)); + buf_tests!(number $make_input, get_u32_be, get_u32_be_overflow, u32, get_u32, 0xff467172); + buf_tests!(number $make_input, get_u32_le, get_u32_le_overflow, u32, get_u32_le, 0x727146ff); + buf_tests!(number $make_input, get_u32_ne, get_u32_ne_overflow, u32, get_u32_ne, e!(0xff467172, 0x727146ff)); + buf_tests!(number $make_input, get_i32_be, get_i32_be_overflow, i32, get_i32, 0xff467172u32 as i32); + buf_tests!(number $make_input, get_i32_le, get_i32_le_overflow, i32, get_i32_le, 0x727146ff); + buf_tests!(number $make_input, get_i32_ne, get_i32_ne_overflow, i32, get_i32_ne, e!(0xff467172u32 as i32, 0x727146ff)); + buf_tests!(number $make_input, get_u64_be, get_u64_be_overflow, u64, get_u64, 0xff4671726a724471); + buf_tests!(number $make_input, get_u64_le, get_u64_le_overflow, u64, get_u64_le, 0x7144726a727146ff); + buf_tests!(number $make_input, get_u64_ne, get_u64_ne_overflow, u64, get_u64_ne, e!(0xff4671726a724471, 0x7144726a727146ff)); + buf_tests!(number $make_input, get_i64_be, get_i64_be_overflow, i64, get_i64, 0xff4671726a724471u64 as i64); + buf_tests!(number $make_input, get_i64_le, get_i64_le_overflow, i64, get_i64_le, 0x7144726a727146ff); + buf_tests!(number $make_input, get_i64_ne, get_i64_ne_overflow, i64, get_i64_ne, e!(0xff4671726a724471u64 as i64, 0x7144726a727146ff)); + buf_tests!(number $make_input, get_u128_be, get_u128_be_overflow, u128, get_u128, 0xff4671726a7244715068765463343576); + buf_tests!(number $make_input, get_u128_le, get_u128_le_overflow, u128, get_u128_le, 0x76353463547668507144726a727146ff); + buf_tests!(number $make_input, get_u128_ne, get_u128_ne_overflow, u128, get_u128_ne, e!(0xff4671726a7244715068765463343576, 0x76353463547668507144726a727146ff)); + buf_tests!(number $make_input, get_i128_be, get_i128_be_overflow, i128, get_i128, 0xff4671726a7244715068765463343576u128 as i128); + buf_tests!(number $make_input, get_i128_le, get_i128_le_overflow, i128, get_i128_le, 0x76353463547668507144726a727146ff); + buf_tests!(number $make_input, get_i128_ne, get_i128_ne_overflow, i128, get_i128_ne, e!(0xff4671726a7244715068765463343576u128 as i128, 0x76353463547668507144726a727146ff)); + buf_tests!(number $make_input, get_f32_be, get_f32_be_overflow, f32, get_f32, f32::from_bits(0xff467172)); + buf_tests!(number $make_input, get_f32_le, get_f32_le_overflow, f32, get_f32_le, f32::from_bits(0x727146ff)); + buf_tests!(number $make_input, get_f32_ne, get_f32_ne_overflow, f32, get_f32_ne, f32::from_bits(e!(0xff467172, 0x727146ff))); + buf_tests!(number $make_input, get_f64_be, get_f64_be_overflow, f64, get_f64, f64::from_bits(0xff4671726a724471)); + buf_tests!(number $make_input, get_f64_le, get_f64_le_overflow, f64, get_f64_le, f64::from_bits(0x7144726a727146ff)); + buf_tests!(number $make_input, get_f64_ne, get_f64_ne_overflow, f64, get_f64_ne, f64::from_bits(e!(0xff4671726a724471, 0x7144726a727146ff))); + + buf_tests!(var_number $make_input, get_uint_be, get_uint_be_overflow, u64, get_uint, 3, 0xff4671); + buf_tests!(var_number $make_input, get_uint_le, get_uint_le_overflow, u64, get_uint_le, 3, 0x7146ff); + buf_tests!(var_number $make_input, get_uint_ne, get_uint_ne_overflow, u64, get_uint_ne, 3, e!(0xff4671, 0x7146ff)); + buf_tests!(var_number $make_input, get_int_be, get_int_be_overflow, i64, get_int, 3, 0xffffffffffff4671u64 as i64); + buf_tests!(var_number $make_input, get_int_le, get_int_le_overflow, i64, get_int_le, 3, 0x7146ff); + buf_tests!(var_number $make_input, get_int_ne, get_int_ne_overflow, i64, get_int_ne, 3, e!(0xffffffffffff4671u64 as i64, 0x7146ff)); + }; + (number $make_input:ident, $ok_name:ident, $panic_name:ident, $number:ty, $method:ident, $value:expr) => { + #[test] + fn $ok_name() { + let mut buf = $make_input(INPUT); + + let value = buf.$method(); + assert_eq!(buf.remaining(), 64 - mem::size_of::<$number>()); + assert!(buf.has_remaining()); + assert_eq!(value, $value); + } + + #[test] + #[should_panic] + fn $panic_name() { + let mut buf = $make_input(&[]); + + let _ = buf.$method(); + } + }; + (var_number $make_input:ident, $ok_name:ident, $panic_name:ident, $number:ty, $method:ident, $len:expr, $value:expr) => { + #[test] + fn $ok_name() { + let mut buf = $make_input(INPUT); + + let value = buf.$method($len); + assert_eq!(buf.remaining(), 64 - $len); + assert!(buf.has_remaining()); + assert_eq!(value, $value); + } + + #[test] + #[should_panic] + fn $panic_name() { + let mut buf = $make_input(&[]); + + let _ = buf.$method($len); + } + }; +} + +mod u8_slice { + fn make_input(buf: &'static [u8]) -> &'static [u8] { + buf + } + + buf_tests!(make_input); +} + +mod bytes { + fn make_input(buf: &'static [u8]) -> impl Buf { + Bytes::from_static(buf) + } + + buf_tests!(make_input); +} + +mod bytes_mut { + fn make_input(buf: &'static [u8]) -> impl Buf { + BytesMut::from(buf) + } + + buf_tests!(make_input); +} + +mod vec_deque { + fn make_input(buf: &'static [u8]) -> impl Buf { + let mut deque = VecDeque::new(); + + if !buf.is_empty() { + // Construct |b|some bytes|a| `VecDeque` + let mid = buf.len() / 2; + let (a, b) = buf.split_at(mid); + + deque.reserve_exact(buf.len() + 1); + + let extra_space = deque.capacity() - b.len() - 1; + deque.resize(extra_space, 0); + + deque.extend(a); + deque.drain(..extra_space); + deque.extend(b); + + let (a, b) = deque.as_slices(); + assert!( + !a.is_empty(), + "could not setup test - attempt to create discontiguous VecDeque failed" + ); + assert!( + !b.is_empty(), + "could not setup test - attempt to create discontiguous VecDeque failed" + ); + } + + deque + } + + buf_tests!(make_input, true); +} + +#[cfg(feature = "std")] +mod cursor { + use std::io::Cursor; + + fn make_input(buf: &'static [u8]) -> impl Buf { + Cursor::new(buf) + } + + buf_tests!(make_input); +} + +mod box_bytes { + fn make_input(buf: &'static [u8]) -> impl Buf { + Box::new(Bytes::from_static(buf)) + } + + buf_tests!(make_input); +} + +mod chain_u8_slice { + fn make_input(buf: &'static [u8]) -> impl Buf { + let (a, b) = buf.split_at(buf.len() / 2); + Buf::chain(a, b) + } + + buf_tests!(make_input); +} + +mod chain_small_big_u8_slice { + fn make_input(buf: &'static [u8]) -> impl Buf { + let mid = cmp::min(1, buf.len()); + let (a, b) = buf.split_at(mid); + Buf::chain(a, b) + } + + buf_tests!(make_input); +} + +mod chain_limited_slices { + fn make_input(buf: &'static [u8]) -> impl Buf { + let buf3 = &buf[cmp::min(buf.len(), 3)..]; + let a = Buf::take(buf3, 0); + let b = Buf::take(buf, 3); + let c = Buf::take(buf3, usize::MAX); + let d = buf; + Buf::take(Buf::chain(Buf::chain(a, b), Buf::chain(c, d)), buf.len()) + } + + buf_tests!(make_input, true); +} + +#[allow(unused_allocation)] // This is intentional. +#[test] +fn test_deref_buf_forwards() { + struct Special; + + impl Buf for Special { + fn remaining(&self) -> usize { + unreachable!("remaining"); + } + + fn chunk(&self) -> &[u8] { + unreachable!("chunk"); + } + + fn advance(&mut self, _: usize) { + unreachable!("advance"); + } + + fn get_u8(&mut self) -> u8 { + // specialized! + b'x' + } + } + + // these should all use the specialized method + assert_eq!(Special.get_u8(), b'x'); + assert_eq!((&mut Special as &mut dyn Buf).get_u8(), b'x'); + assert_eq!((Box::new(Special) as Box<dyn Buf>).get_u8(), b'x'); + assert_eq!(Box::new(Special).get_u8(), b'x'); +} diff --git a/third-party/bytes/tests/test_buf_mut.rs b/third-party/bytes/tests/test_buf_mut.rs new file mode 100644 index 000000000..f1bd3b0a0 --- /dev/null +++ b/third-party/bytes/tests/test_buf_mut.rs @@ -0,0 +1,284 @@ +#![warn(rust_2018_idioms)] + +use bytes::buf::UninitSlice; +use bytes::{BufMut, BytesMut}; +use core::fmt::Write; +use core::mem::MaybeUninit; + +#[test] +fn test_vec_as_mut_buf() { + let mut buf = Vec::with_capacity(64); + + assert_eq!(buf.remaining_mut(), isize::MAX as usize); + + assert!(buf.chunk_mut().len() >= 64); + + buf.put(&b"zomg"[..]); + + assert_eq!(&buf, b"zomg"); + + assert_eq!(buf.remaining_mut(), isize::MAX as usize - 4); + assert_eq!(buf.capacity(), 64); + + for _ in 0..16 { + buf.put(&b"zomg"[..]); + } + + assert_eq!(buf.len(), 68); +} + +#[test] +fn test_vec_put_bytes() { + let mut buf = Vec::new(); + buf.push(17); + buf.put_bytes(19, 2); + assert_eq!([17, 19, 19], &buf[..]); +} + +#[test] +fn test_put_u8() { + let mut buf = Vec::with_capacity(8); + buf.put_u8(33); + assert_eq!(b"\x21", &buf[..]); +} + +#[test] +fn test_put_u16() { + let mut buf = Vec::with_capacity(8); + buf.put_u16(8532); + assert_eq!(b"\x21\x54", &buf[..]); + + buf.clear(); + buf.put_u16_le(8532); + assert_eq!(b"\x54\x21", &buf[..]); +} + +#[test] +fn test_put_int() { + let mut buf = Vec::with_capacity(8); + buf.put_int(0x1020304050607080, 3); + assert_eq!(b"\x60\x70\x80", &buf[..]); +} + +#[test] +#[should_panic] +fn test_put_int_nbytes_overflow() { + let mut buf = Vec::with_capacity(8); + buf.put_int(0x1020304050607080, 9); +} + +#[test] +fn test_put_int_le() { + let mut buf = Vec::with_capacity(8); + buf.put_int_le(0x1020304050607080, 3); + assert_eq!(b"\x80\x70\x60", &buf[..]); +} + +#[test] +#[should_panic] +fn test_put_int_le_nbytes_overflow() { + let mut buf = Vec::with_capacity(8); + buf.put_int_le(0x1020304050607080, 9); +} + +#[test] +#[should_panic(expected = "advance out of bounds: the len is 8 but advancing by 12")] +fn test_vec_advance_mut() { + // Verify fix for #354 + let mut buf = Vec::with_capacity(8); + unsafe { + buf.advance_mut(12); + } +} + +#[test] +fn test_clone() { + let mut buf = BytesMut::with_capacity(100); + buf.write_str("this is a test").unwrap(); + let buf2 = buf.clone(); + + buf.write_str(" of our emergency broadcast system").unwrap(); + assert!(buf != buf2); +} + +fn do_test_slice_small<T: ?Sized>(make: impl Fn(&mut [u8]) -> &mut T) +where + for<'r> &'r mut T: BufMut, +{ + let mut buf = [b'X'; 8]; + + let mut slice = make(&mut buf[..]); + slice.put_bytes(b'A', 2); + slice.put_u8(b'B'); + slice.put_slice(b"BCC"); + assert_eq!(2, slice.remaining_mut()); + assert_eq!(b"AABBCCXX", &buf[..]); + + let mut slice = make(&mut buf[..]); + slice.put_u32(0x61626364); + assert_eq!(4, slice.remaining_mut()); + assert_eq!(b"abcdCCXX", &buf[..]); + + let mut slice = make(&mut buf[..]); + slice.put_u32_le(0x30313233); + assert_eq!(4, slice.remaining_mut()); + assert_eq!(b"3210CCXX", &buf[..]); +} + +fn do_test_slice_large<T: ?Sized>(make: impl Fn(&mut [u8]) -> &mut T) +where + for<'r> &'r mut T: BufMut, +{ + const LEN: usize = 100; + const FILL: [u8; LEN] = [b'Y'; LEN]; + + let test = |fill: &dyn Fn(&mut &mut T, usize)| { + for buf_len in 0..LEN { + let mut buf = [b'X'; LEN]; + for fill_len in 0..=buf_len { + let mut slice = make(&mut buf[..buf_len]); + fill(&mut slice, fill_len); + assert_eq!(buf_len - fill_len, slice.remaining_mut()); + let (head, tail) = buf.split_at(fill_len); + assert_eq!(&FILL[..fill_len], head); + assert!(tail.iter().all(|b| *b == b'X')); + } + } + }; + + test(&|slice, fill_len| slice.put_slice(&FILL[..fill_len])); + test(&|slice, fill_len| slice.put_bytes(FILL[0], fill_len)); +} + +fn do_test_slice_put_slice_panics<T: ?Sized>(make: impl Fn(&mut [u8]) -> &mut T) +where + for<'r> &'r mut T: BufMut, +{ + let mut buf = [b'X'; 4]; + let mut slice = make(&mut buf[..]); + slice.put_slice(b"12345"); +} + +fn do_test_slice_put_bytes_panics<T: ?Sized>(make: impl Fn(&mut [u8]) -> &mut T) +where + for<'r> &'r mut T: BufMut, +{ + let mut buf = [b'X'; 4]; + let mut slice = make(&mut buf[..]); + slice.put_bytes(b'1', 5); +} + +#[test] +fn test_slice_buf_mut_small() { + do_test_slice_small(|x| x); +} + +#[test] +fn test_slice_buf_mut_large() { + do_test_slice_large(|x| x); +} + +#[test] +#[should_panic] +fn test_slice_buf_mut_put_slice_overflow() { + do_test_slice_put_slice_panics(|x| x); +} + +#[test] +#[should_panic] +fn test_slice_buf_mut_put_bytes_overflow() { + do_test_slice_put_bytes_panics(|x| x); +} + +fn make_maybe_uninit_slice(slice: &mut [u8]) -> &mut [MaybeUninit<u8>] { + // SAFETY: [u8] has the same layout as [MaybeUninit<u8>]. + unsafe { core::mem::transmute(slice) } +} + +#[test] +fn test_maybe_uninit_buf_mut_small() { + do_test_slice_small(make_maybe_uninit_slice); +} + +#[test] +fn test_maybe_uninit_buf_mut_large() { + do_test_slice_large(make_maybe_uninit_slice); +} + +#[test] +#[should_panic] +fn test_maybe_uninit_buf_mut_put_slice_overflow() { + do_test_slice_put_slice_panics(make_maybe_uninit_slice); +} + +#[test] +#[should_panic] +fn test_maybe_uninit_buf_mut_put_bytes_overflow() { + do_test_slice_put_bytes_panics(make_maybe_uninit_slice); +} + +#[allow(unused_allocation)] // This is intentional. +#[test] +fn test_deref_bufmut_forwards() { + struct Special; + + unsafe impl BufMut for Special { + fn remaining_mut(&self) -> usize { + unreachable!("remaining_mut"); + } + + fn chunk_mut(&mut self) -> &mut UninitSlice { + unreachable!("chunk_mut"); + } + + unsafe fn advance_mut(&mut self, _: usize) { + unreachable!("advance"); + } + + fn put_u8(&mut self, _: u8) { + // specialized! + } + } + + // these should all use the specialized method + Special.put_u8(b'x'); + (&mut Special as &mut dyn BufMut).put_u8(b'x'); + (Box::new(Special) as Box<dyn BufMut>).put_u8(b'x'); + Box::new(Special).put_u8(b'x'); +} + +#[test] +#[should_panic] +fn write_byte_panics_if_out_of_bounds() { + let mut data = [b'b', b'a', b'r']; + + let slice = unsafe { UninitSlice::from_raw_parts_mut(data.as_mut_ptr(), 3) }; + slice.write_byte(4, b'f'); +} + +#[test] +#[should_panic] +fn copy_from_slice_panics_if_different_length_1() { + let mut data = [b'b', b'a', b'r']; + + let slice = unsafe { UninitSlice::from_raw_parts_mut(data.as_mut_ptr(), 3) }; + slice.copy_from_slice(b"a"); +} + +#[test] +#[should_panic] +fn copy_from_slice_panics_if_different_length_2() { + let mut data = [b'b', b'a', b'r']; + + let slice = unsafe { UninitSlice::from_raw_parts_mut(data.as_mut_ptr(), 3) }; + slice.copy_from_slice(b"abcd"); +} + +/// Test if with zero capacity BytesMut does not infinitely recurse in put from Buf +#[test] +fn test_bytes_mut_reuse() { + let mut buf = BytesMut::new(); + buf.put(&[] as &[u8]); + let mut buf = BytesMut::new(); + buf.put(&[1u8, 2, 3] as &[u8]); +} diff --git a/third-party/bytes/tests/test_bytes.rs b/third-party/bytes/tests/test_bytes.rs new file mode 100644 index 000000000..b9bd5e12b --- /dev/null +++ b/third-party/bytes/tests/test_bytes.rs @@ -0,0 +1,1722 @@ +#![warn(rust_2018_idioms)] + +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use std::panic::{self, AssertUnwindSafe}; + +const LONG: &[u8] = b"mary had a little lamb, little lamb, little lamb"; +const SHORT: &[u8] = b"hello world"; + +fn is_sync<T: Sync>() {} +fn is_send<T: Send>() {} + +#[test] +fn test_bounds() { + is_sync::<Bytes>(); + is_sync::<BytesMut>(); + is_send::<Bytes>(); + is_send::<BytesMut>(); +} + +#[test] +fn test_layout() { + use std::mem; + + assert_eq!( + mem::size_of::<Bytes>(), + mem::size_of::<usize>() * 4, + "Bytes size should be 4 words", + ); + assert_eq!( + mem::size_of::<BytesMut>(), + mem::size_of::<usize>() * 4, + "BytesMut should be 4 words", + ); + + assert_eq!( + mem::size_of::<Bytes>(), + mem::size_of::<Option<Bytes>>(), + "Bytes should be same size as Option<Bytes>", + ); + + assert_eq!( + mem::size_of::<BytesMut>(), + mem::size_of::<Option<BytesMut>>(), + "BytesMut should be same size as Option<BytesMut>", + ); +} + +#[test] +fn from_slice() { + let a = Bytes::from(&b"abcdefgh"[..]); + assert_eq!(a, b"abcdefgh"[..]); + assert_eq!(a, &b"abcdefgh"[..]); + assert_eq!(a, Vec::from(&b"abcdefgh"[..])); + assert_eq!(b"abcdefgh"[..], a); + assert_eq!(&b"abcdefgh"[..], a); + assert_eq!(Vec::from(&b"abcdefgh"[..]), a); + + let a = BytesMut::from(&b"abcdefgh"[..]); + assert_eq!(a, b"abcdefgh"[..]); + assert_eq!(a, &b"abcdefgh"[..]); + assert_eq!(a, Vec::from(&b"abcdefgh"[..])); + assert_eq!(b"abcdefgh"[..], a); + assert_eq!(&b"abcdefgh"[..], a); + assert_eq!(Vec::from(&b"abcdefgh"[..]), a); +} + +#[test] +fn fmt() { + let a = format!("{:?}", Bytes::from(&b"abcdefg"[..])); + let b = "b\"abcdefg\""; + + assert_eq!(a, b); + + let a = format!("{:?}", BytesMut::from(&b"abcdefg"[..])); + assert_eq!(a, b); +} + +#[test] +fn fmt_write() { + use std::fmt::Write; + let s = String::from_iter((0..10).map(|_| "abcdefg")); + + let mut a = BytesMut::with_capacity(64); + write!(a, "{}", &s[..64]).unwrap(); + assert_eq!(a, s[..64].as_bytes()); + + let mut b = BytesMut::with_capacity(64); + write!(b, "{}", &s[..32]).unwrap(); + write!(b, "{}", &s[32..64]).unwrap(); + assert_eq!(b, s[..64].as_bytes()); + + let mut c = BytesMut::with_capacity(64); + write!(c, "{}", s).unwrap(); + assert_eq!(c, s[..].as_bytes()); +} + +#[test] +fn len() { + let a = Bytes::from(&b"abcdefg"[..]); + assert_eq!(a.len(), 7); + + let a = BytesMut::from(&b"abcdefg"[..]); + assert_eq!(a.len(), 7); + + let a = Bytes::from(&b""[..]); + assert!(a.is_empty()); + + let a = BytesMut::from(&b""[..]); + assert!(a.is_empty()); +} + +#[test] +fn index() { + let a = Bytes::from(&b"hello world"[..]); + assert_eq!(a[0..5], *b"hello"); +} + +#[test] +fn slice() { + let a = Bytes::from(&b"hello world"[..]); + + let b = a.slice(3..5); + assert_eq!(b, b"lo"[..]); + + let b = a.slice(0..0); + assert_eq!(b, b""[..]); + + let b = a.slice(3..3); + assert_eq!(b, b""[..]); + + let b = a.slice(a.len()..a.len()); + assert_eq!(b, b""[..]); + + let b = a.slice(..5); + assert_eq!(b, b"hello"[..]); + + let b = a.slice(3..); + assert_eq!(b, b"lo world"[..]); +} + +#[test] +#[should_panic] +fn slice_oob_1() { + let a = Bytes::from(&b"hello world"[..]); + a.slice(5..44); +} + +#[test] +#[should_panic] +fn slice_oob_2() { + let a = Bytes::from(&b"hello world"[..]); + a.slice(44..49); +} + +#[test] +#[should_panic] +fn slice_start_greater_than_end() { + let a = Bytes::from(&b"hello world"[..]); + a.slice(5..3); +} + +#[test] +fn split_off() { + let mut hello = Bytes::from(&b"helloworld"[..]); + let world = hello.split_off(5); + + assert_eq!(hello, &b"hello"[..]); + assert_eq!(world, &b"world"[..]); + + let mut hello = BytesMut::from(&b"helloworld"[..]); + let world = hello.split_off(5); + + assert_eq!(hello, &b"hello"[..]); + assert_eq!(world, &b"world"[..]); +} + +#[test] +#[should_panic] +fn split_off_oob() { + let mut hello = Bytes::from(&b"helloworld"[..]); + let _ = hello.split_off(44); +} + +#[test] +#[should_panic = "split_off out of bounds"] +fn bytes_mut_split_off_oob() { + let mut hello = BytesMut::from(&b"helloworld"[..]); + let _ = hello.split_off(44); +} + +#[test] +fn split_off_uninitialized() { + let mut bytes = BytesMut::with_capacity(1024); + let other = bytes.split_off(128); + + assert_eq!(bytes.len(), 0); + assert_eq!(bytes.capacity(), 128); + + assert_eq!(other.len(), 0); + assert_eq!(other.capacity(), 896); +} + +#[test] +fn split_off_to_loop() { + let s = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + for i in 0..(s.len() + 1) { + { + let mut bytes = Bytes::from(&s[..]); + let off = bytes.split_off(i); + assert_eq!(i, bytes.len()); + let mut sum = Vec::new(); + sum.extend(bytes.iter()); + sum.extend(off.iter()); + assert_eq!(&s[..], &sum[..]); + } + { + let mut bytes = BytesMut::from(&s[..]); + let off = bytes.split_off(i); + assert_eq!(i, bytes.len()); + let mut sum = Vec::new(); + sum.extend(&bytes); + sum.extend(&off); + assert_eq!(&s[..], &sum[..]); + } + { + let mut bytes = Bytes::from(&s[..]); + let off = bytes.split_to(i); + assert_eq!(i, off.len()); + let mut sum = Vec::new(); + sum.extend(off.iter()); + sum.extend(bytes.iter()); + assert_eq!(&s[..], &sum[..]); + } + { + let mut bytes = BytesMut::from(&s[..]); + let off = bytes.split_to(i); + assert_eq!(i, off.len()); + let mut sum = Vec::new(); + sum.extend(&off); + sum.extend(&bytes); + assert_eq!(&s[..], &sum[..]); + } + } +} + +#[test] +fn split_to_1() { + // Static + let mut a = Bytes::from_static(SHORT); + let b = a.split_to(4); + + assert_eq!(SHORT[4..], a); + assert_eq!(SHORT[..4], b); + + // Allocated + let mut a = Bytes::copy_from_slice(LONG); + let b = a.split_to(4); + + assert_eq!(LONG[4..], a); + assert_eq!(LONG[..4], b); + + let mut a = Bytes::copy_from_slice(LONG); + let b = a.split_to(30); + + assert_eq!(LONG[30..], a); + assert_eq!(LONG[..30], b); +} + +#[test] +fn split_to_2() { + let mut a = Bytes::from(LONG); + assert_eq!(LONG, a); + + let b = a.split_to(1); + + assert_eq!(LONG[1..], a); + drop(b); +} + +#[test] +#[should_panic] +fn split_to_oob() { + let mut hello = Bytes::from(&b"helloworld"[..]); + let _ = hello.split_to(33); +} + +#[test] +#[should_panic] +fn split_to_oob_mut() { + let mut hello = BytesMut::from(&b"helloworld"[..]); + let _ = hello.split_to(33); +} + +#[test] +#[should_panic] +fn split_to_uninitialized() { + let mut bytes = BytesMut::with_capacity(1024); + let _other = bytes.split_to(128); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn split_off_to_at_gt_len() { + fn make_bytes() -> Bytes { + let mut bytes = BytesMut::with_capacity(100); + bytes.put_slice(&[10, 20, 30, 40]); + bytes.freeze() + } + + use std::panic; + + let _ = make_bytes().split_to(4); + let _ = make_bytes().split_off(4); + + assert!(panic::catch_unwind(move || { + let _ = make_bytes().split_to(5); + }) + .is_err()); + + assert!(panic::catch_unwind(move || { + let _ = make_bytes().split_off(5); + }) + .is_err()); +} + +#[test] +fn truncate() { + let s = &b"helloworld"[..]; + let mut hello = Bytes::from(s); + hello.truncate(15); + assert_eq!(hello, s); + hello.truncate(10); + assert_eq!(hello, s); + hello.truncate(5); + assert_eq!(hello, "hello"); +} + +#[test] +fn freeze_clone_shared() { + let s = &b"abcdefgh"[..]; + let b = BytesMut::from(s).split().freeze(); + assert_eq!(b, s); + let c = b.clone(); + assert_eq!(c, s); +} + +#[test] +fn freeze_clone_unique() { + let s = &b"abcdefgh"[..]; + let b = BytesMut::from(s).freeze(); + assert_eq!(b, s); + let c = b.clone(); + assert_eq!(c, s); +} + +#[test] +fn freeze_after_advance() { + let s = &b"abcdefgh"[..]; + let mut b = BytesMut::from(s); + b.advance(1); + assert_eq!(b, s[1..]); + let b = b.freeze(); + // Verify fix for #352. Previously, freeze would ignore the start offset + // for BytesMuts in Vec mode. + assert_eq!(b, s[1..]); +} + +#[test] +fn freeze_after_advance_arc() { + let s = &b"abcdefgh"[..]; + let mut b = BytesMut::from(s); + // Make b Arc + let _ = b.split_to(0); + b.advance(1); + assert_eq!(b, s[1..]); + let b = b.freeze(); + assert_eq!(b, s[1..]); +} + +#[test] +fn freeze_after_split_to() { + let s = &b"abcdefgh"[..]; + let mut b = BytesMut::from(s); + let _ = b.split_to(1); + assert_eq!(b, s[1..]); + let b = b.freeze(); + assert_eq!(b, s[1..]); +} + +#[test] +fn freeze_after_truncate() { + let s = &b"abcdefgh"[..]; + let mut b = BytesMut::from(s); + b.truncate(7); + assert_eq!(b, s[..7]); + let b = b.freeze(); + assert_eq!(b, s[..7]); +} + +#[test] +fn freeze_after_truncate_arc() { + let s = &b"abcdefgh"[..]; + let mut b = BytesMut::from(s); + // Make b Arc + let _ = b.split_to(0); + b.truncate(7); + assert_eq!(b, s[..7]); + let b = b.freeze(); + assert_eq!(b, s[..7]); +} + +#[test] +fn freeze_after_split_off() { + let s = &b"abcdefgh"[..]; + let mut b = BytesMut::from(s); + let _ = b.split_off(7); + assert_eq!(b, s[..7]); + let b = b.freeze(); + assert_eq!(b, s[..7]); +} + +#[test] +fn fns_defined_for_bytes_mut() { + let mut bytes = BytesMut::from(&b"hello world"[..]); + + let _ = bytes.as_ptr(); + let _ = bytes.as_mut_ptr(); + + // Iterator + let v: Vec<u8> = bytes.as_ref().iter().cloned().collect(); + assert_eq!(&v[..], bytes); +} + +#[test] +fn reserve_convert() { + // Vec -> Vec + let mut bytes = BytesMut::from(LONG); + bytes.reserve(64); + assert_eq!(bytes.capacity(), LONG.len() + 64); + + // Arc -> Vec + let mut bytes = BytesMut::from(LONG); + let a = bytes.split_to(30); + + bytes.reserve(128); + assert!(bytes.capacity() >= bytes.len() + 128); + + drop(a); +} + +#[test] +fn reserve_growth() { + let mut bytes = BytesMut::with_capacity(64); + bytes.put("hello world".as_bytes()); + let _ = bytes.split(); + + bytes.reserve(65); + assert_eq!(bytes.capacity(), 117); +} + +#[test] +fn reserve_allocates_at_least_original_capacity() { + let mut bytes = BytesMut::with_capacity(1024); + + for i in 0..1020 { + bytes.put_u8(i as u8); + } + + let _other = bytes.split(); + + bytes.reserve(16); + assert_eq!(bytes.capacity(), 1024); +} + +#[test] +#[cfg_attr(miri, ignore)] // Miri is too slow +fn reserve_max_original_capacity_value() { + const SIZE: usize = 128 * 1024; + + let mut bytes = BytesMut::with_capacity(SIZE); + + for _ in 0..SIZE { + bytes.put_u8(0u8); + } + + let _other = bytes.split(); + + bytes.reserve(16); + assert_eq!(bytes.capacity(), 64 * 1024); +} + +#[test] +fn reserve_vec_recycling() { + let mut bytes = BytesMut::with_capacity(16); + assert_eq!(bytes.capacity(), 16); + let addr = bytes.as_ptr() as usize; + bytes.put("0123456789012345".as_bytes()); + assert_eq!(bytes.as_ptr() as usize, addr); + bytes.advance(10); + assert_eq!(bytes.capacity(), 6); + bytes.reserve(8); + assert_eq!(bytes.capacity(), 16); + assert_eq!(bytes.as_ptr() as usize, addr); +} + +#[test] +fn reserve_in_arc_unique_does_not_overallocate() { + let mut bytes = BytesMut::with_capacity(1000); + let _ = bytes.split(); + + // now bytes is Arc and refcount == 1 + + assert_eq!(1000, bytes.capacity()); + bytes.reserve(2001); + assert_eq!(2001, bytes.capacity()); +} + +#[test] +fn reserve_in_arc_unique_doubles() { + let mut bytes = BytesMut::with_capacity(1000); + let _ = bytes.split(); + + // now bytes is Arc and refcount == 1 + + assert_eq!(1000, bytes.capacity()); + bytes.reserve(1001); + assert_eq!(2000, bytes.capacity()); +} + +#[test] +fn reserve_in_arc_unique_does_not_overallocate_after_split() { + let mut bytes = BytesMut::from(LONG); + let orig_capacity = bytes.capacity(); + drop(bytes.split_off(LONG.len() / 2)); + + // now bytes is Arc and refcount == 1 + + let new_capacity = bytes.capacity(); + bytes.reserve(orig_capacity - new_capacity); + assert_eq!(bytes.capacity(), orig_capacity); +} + +#[test] +fn reserve_in_arc_unique_does_not_overallocate_after_multiple_splits() { + let mut bytes = BytesMut::from(LONG); + let orig_capacity = bytes.capacity(); + for _ in 0..10 { + drop(bytes.split_off(LONG.len() / 2)); + + // now bytes is Arc and refcount == 1 + + let new_capacity = bytes.capacity(); + bytes.reserve(orig_capacity - new_capacity); + } + assert_eq!(bytes.capacity(), orig_capacity); +} + +#[test] +fn reserve_in_arc_nonunique_does_not_overallocate() { + let mut bytes = BytesMut::with_capacity(1000); + let _copy = bytes.split(); + + // now bytes is Arc and refcount == 2 + + assert_eq!(1000, bytes.capacity()); + bytes.reserve(2001); + assert_eq!(2001, bytes.capacity()); +} + +/// This function tests `BytesMut::reserve_inner`, where `BytesMut` holds +/// a unique reference to the shared vector and decide to reuse it +/// by reallocating the `Vec`. +#[test] +fn reserve_shared_reuse() { + let mut bytes = BytesMut::with_capacity(1000); + bytes.put_slice(b"Hello, World!"); + drop(bytes.split()); + + bytes.put_slice(b"!123ex123,sadchELLO,_wORLD!"); + // Use split_off so that v.capacity() - self.cap != off + drop(bytes.split_off(9)); + assert_eq!(&*bytes, b"!123ex123"); + + bytes.reserve(2000); + assert_eq!(&*bytes, b"!123ex123"); + assert_eq!(bytes.capacity(), 2009); +} + +#[test] +fn extend_mut() { + let mut bytes = BytesMut::with_capacity(0); + bytes.extend(LONG); + assert_eq!(*bytes, LONG[..]); +} + +#[test] +fn extend_from_slice_mut() { + for &i in &[3, 34] { + let mut bytes = BytesMut::new(); + bytes.extend_from_slice(&LONG[..i]); + bytes.extend_from_slice(&LONG[i..]); + assert_eq!(LONG[..], *bytes); + } +} + +#[test] +fn extend_mut_from_bytes() { + let mut bytes = BytesMut::with_capacity(0); + bytes.extend([Bytes::from(LONG)]); + assert_eq!(*bytes, LONG[..]); +} + +#[test] +fn extend_past_lower_limit_of_size_hint() { + // See https://github.com/tokio-rs/bytes/pull/674#pullrequestreview-1913035700 + struct Iter<I>(I); + + impl<I: Iterator<Item = u8>> Iterator for Iter<I> { + type Item = u8; + + fn next(&mut self) -> Option<Self::Item> { + self.0.next() + } + + fn size_hint(&self) -> (usize, Option<usize>) { + (5, None) + } + } + + let mut bytes = BytesMut::with_capacity(5); + bytes.extend(Iter(std::iter::repeat(0).take(10))); + assert_eq!(bytes.len(), 10); +} + +#[test] +fn extend_mut_without_size_hint() { + let mut bytes = BytesMut::with_capacity(0); + let mut long_iter = LONG.iter(); + + // Use iter::from_fn since it doesn't know a size_hint + bytes.extend(std::iter::from_fn(|| long_iter.next())); + assert_eq!(*bytes, LONG[..]); +} + +#[test] +fn from_static() { + let mut a = Bytes::from_static(b"ab"); + let b = a.split_off(1); + + assert_eq!(a, b"a"[..]); + assert_eq!(b, b"b"[..]); +} + +#[test] +fn advance_static() { + let mut a = Bytes::from_static(b"hello world"); + a.advance(6); + assert_eq!(a, &b"world"[..]); +} + +#[test] +fn advance_vec() { + let mut a = Bytes::from(b"hello world boooo yah world zomg wat wat".to_vec()); + a.advance(16); + assert_eq!(a, b"o yah world zomg wat wat"[..]); + + a.advance(4); + assert_eq!(a, b"h world zomg wat wat"[..]); + + a.advance(6); + assert_eq!(a, b"d zomg wat wat"[..]); +} + +#[test] +fn advance_bytes_mut() { + let mut a = BytesMut::from("hello world boooo yah world zomg wat wat"); + a.advance(16); + assert_eq!(a, b"o yah world zomg wat wat"[..]); + + a.advance(4); + assert_eq!(a, b"h world zomg wat wat"[..]); + + // Reserve some space. + a.reserve(1024); + assert_eq!(a, b"h world zomg wat wat"[..]); + + a.advance(6); + assert_eq!(a, b"d zomg wat wat"[..]); +} + +// Ensures BytesMut::advance reduces always capacity +// +// See https://github.com/tokio-rs/bytes/issues/725 +#[test] +fn advance_bytes_mut_remaining_capacity() { + // reduce the search space under miri + let max_capacity = if cfg!(miri) { 16 } else { 256 }; + for capacity in 0..=max_capacity { + for len in 0..=capacity { + for advance in 0..=len { + eprintln!("testing capacity={capacity}, len={len}, advance={advance}"); + let mut buf = BytesMut::with_capacity(capacity); + + buf.resize(len, 42); + assert_eq!(buf.len(), len, "resize should write `len` bytes"); + assert_eq!( + buf.remaining(), + len, + "Buf::remaining() should equal BytesMut::len" + ); + + buf.advance(advance); + assert_eq!( + buf.remaining(), + len - advance, + "Buf::advance should reduce the remaining len" + ); + assert_eq!( + buf.capacity(), + capacity - advance, + "Buf::advance should reduce the remaining capacity" + ); + } + } + } +} + +#[test] +#[should_panic] +fn advance_past_len() { + let mut a = BytesMut::from("hello world"); + a.advance(20); +} + +#[test] +#[should_panic] +fn mut_advance_past_len() { + let mut a = BytesMut::from("hello world"); + unsafe { + a.advance_mut(20); + } +} + +#[test] +// Only run these tests on little endian systems. CI uses qemu for testing +// big endian... and qemu doesn't really support threading all that well. +#[cfg(any(miri, target_endian = "little"))] +#[cfg(not(target_family = "wasm"))] // wasm without experimental threads proposal doesn't support threads +fn stress() { + // Tests promoting a buffer from a vec -> shared in a concurrent situation + use std::sync::{Arc, Barrier}; + use std::thread; + + const THREADS: usize = 8; + const ITERS: usize = if cfg!(miri) { 100 } else { 1_000 }; + + for i in 0..ITERS { + let data = [i as u8; 256]; + let buf = Arc::new(Bytes::copy_from_slice(&data[..])); + + let barrier = Arc::new(Barrier::new(THREADS)); + let mut joins = Vec::with_capacity(THREADS); + + for _ in 0..THREADS { + let c = barrier.clone(); + let buf = buf.clone(); + + joins.push(thread::spawn(move || { + c.wait(); + let buf: Bytes = (*buf).clone(); + drop(buf); + })); + } + + for th in joins { + th.join().unwrap(); + } + + assert_eq!(*buf, data[..]); + } +} + +#[test] +fn partial_eq_bytesmut() { + let bytes = Bytes::from(&b"The quick red fox"[..]); + let bytesmut = BytesMut::from(&b"The quick red fox"[..]); + assert!(bytes == bytesmut); + assert!(bytesmut == bytes); + let bytes2 = Bytes::from(&b"Jumped over the lazy brown dog"[..]); + assert!(bytes2 != bytesmut); + assert!(bytesmut != bytes2); +} + +#[test] +fn bytes_mut_unsplit_basic() { + let mut buf = BytesMut::with_capacity(64); + buf.extend_from_slice(b"aaabbbcccddd"); + + let splitted = buf.split_off(6); + assert_eq!(b"aaabbb", &buf[..]); + assert_eq!(b"cccddd", &splitted[..]); + + buf.unsplit(splitted); + assert_eq!(b"aaabbbcccddd", &buf[..]); +} + +#[test] +fn bytes_mut_unsplit_empty_other() { + let mut buf = BytesMut::with_capacity(64); + buf.extend_from_slice(b"aaabbbcccddd"); + + // empty other + let other = BytesMut::new(); + + buf.unsplit(other); + assert_eq!(b"aaabbbcccddd", &buf[..]); +} + +#[test] +fn bytes_mut_unsplit_empty_self() { + // empty self + let mut buf = BytesMut::new(); + + let mut other = BytesMut::with_capacity(64); + other.extend_from_slice(b"aaabbbcccddd"); + + buf.unsplit(other); + assert_eq!(b"aaabbbcccddd", &buf[..]); +} + +#[test] +fn bytes_mut_unsplit_other_keeps_capacity() { + let mut buf = BytesMut::with_capacity(64); + buf.extend_from_slice(b"aabb"); + + // non empty other created "from" buf + let mut other = buf.split_off(buf.len()); + other.extend_from_slice(b"ccddee"); + buf.unsplit(other); + + assert_eq!(buf.capacity(), 64); +} + +#[test] +fn bytes_mut_unsplit_empty_other_keeps_capacity() { + let mut buf = BytesMut::with_capacity(64); + buf.extend_from_slice(b"aabbccddee"); + + // empty other created "from" buf + let other = buf.split_off(buf.len()); + buf.unsplit(other); + + assert_eq!(buf.capacity(), 64); +} + +#[test] +fn bytes_mut_unsplit_arc_different() { + let mut buf = BytesMut::with_capacity(64); + buf.extend_from_slice(b"aaaabbbbeeee"); + + let _ = buf.split_off(8); //arc + + let mut buf2 = BytesMut::with_capacity(64); + buf2.extend_from_slice(b"ccccddddeeee"); + + let _ = buf2.split_off(8); //arc + + buf.unsplit(buf2); + assert_eq!(b"aaaabbbbccccdddd", &buf[..]); +} + +#[test] +fn bytes_mut_unsplit_arc_non_contiguous() { + let mut buf = BytesMut::with_capacity(64); + buf.extend_from_slice(b"aaaabbbbeeeeccccdddd"); + + let mut buf2 = buf.split_off(8); //arc + + let buf3 = buf2.split_off(4); //arc + + buf.unsplit(buf3); + assert_eq!(b"aaaabbbbccccdddd", &buf[..]); +} + +#[test] +fn bytes_mut_unsplit_two_split_offs() { + let mut buf = BytesMut::with_capacity(64); + buf.extend_from_slice(b"aaaabbbbccccdddd"); + + let mut buf2 = buf.split_off(8); //arc + let buf3 = buf2.split_off(4); //arc + + buf2.unsplit(buf3); + buf.unsplit(buf2); + assert_eq!(b"aaaabbbbccccdddd", &buf[..]); +} + +#[test] +fn from_iter_no_size_hint() { + use std::iter; + + let mut expect = vec![]; + + let actual: Bytes = iter::repeat(b'x') + .scan(100, |cnt, item| { + if *cnt >= 1 { + *cnt -= 1; + expect.push(item); + Some(item) + } else { + None + } + }) + .collect(); + + assert_eq!(&actual[..], &expect[..]); +} + +fn test_slice_ref(bytes: &Bytes, start: usize, end: usize, expected: &[u8]) { + let slice = &(bytes.as_ref()[start..end]); + let sub = bytes.slice_ref(slice); + assert_eq!(&sub[..], expected); +} + +#[test] +fn slice_ref_works() { + let bytes = Bytes::from(&b"012345678"[..]); + + test_slice_ref(&bytes, 0, 0, b""); + test_slice_ref(&bytes, 0, 3, b"012"); + test_slice_ref(&bytes, 2, 6, b"2345"); + test_slice_ref(&bytes, 7, 9, b"78"); + test_slice_ref(&bytes, 9, 9, b""); +} + +#[test] +fn slice_ref_empty() { + let bytes = Bytes::from(&b""[..]); + let slice = &(bytes.as_ref()[0..0]); + + let sub = bytes.slice_ref(slice); + assert_eq!(&sub[..], b""); +} + +#[test] +fn slice_ref_empty_subslice() { + let bytes = Bytes::from(&b"abcde"[..]); + let subbytes = bytes.slice(0..0); + let slice = &subbytes[..]; + // The `slice` object is derived from the original `bytes` object + // so `slice_ref` should work. + assert_eq!(Bytes::new(), bytes.slice_ref(slice)); +} + +#[test] +#[should_panic] +fn slice_ref_catches_not_a_subset() { + let bytes = Bytes::from(&b"012345678"[..]); + let slice = &b"012345"[0..4]; + + bytes.slice_ref(slice); +} + +#[test] +fn slice_ref_not_an_empty_subset() { + let bytes = Bytes::from(&b"012345678"[..]); + let slice = &b""[0..0]; + + assert_eq!(Bytes::new(), bytes.slice_ref(slice)); +} + +#[test] +fn empty_slice_ref_not_an_empty_subset() { + let bytes = Bytes::new(); + let slice = &b"some other slice"[0..0]; + + assert_eq!(Bytes::new(), bytes.slice_ref(slice)); +} + +#[test] +fn bytes_buf_mut_advance() { + let mut bytes = BytesMut::with_capacity(1024); + + unsafe { + let ptr = bytes.chunk_mut().as_mut_ptr(); + assert_eq!(1024, bytes.chunk_mut().len()); + + bytes.advance_mut(10); + + let next = bytes.chunk_mut().as_mut_ptr(); + assert_eq!(1024 - 10, bytes.chunk_mut().len()); + assert_eq!(ptr.offset(10), next); + + // advance to the end + bytes.advance_mut(1024 - 10); + + // The buffer size is doubled + assert_eq!(1024, bytes.chunk_mut().len()); + } +} + +#[test] +fn bytes_buf_mut_reuse_when_fully_consumed() { + use bytes::{Buf, BytesMut}; + let mut buf = BytesMut::new(); + buf.reserve(8192); + buf.extend_from_slice(&[0u8; 100][..]); + + let p = &buf[0] as *const u8; + buf.advance(100); + + buf.reserve(8192); + buf.extend_from_slice(b" "); + + assert_eq!(&buf[0] as *const u8, p); +} + +#[test] +#[should_panic] +fn bytes_reserve_overflow() { + let mut bytes = BytesMut::with_capacity(1024); + bytes.put_slice(b"hello world"); + + bytes.reserve(usize::MAX); +} + +#[test] +fn bytes_with_capacity_but_empty() { + // See https://github.com/tokio-rs/bytes/issues/340 + let vec = Vec::with_capacity(1); + let _ = Bytes::from(vec); +} + +#[test] +fn bytes_put_bytes() { + let mut bytes = BytesMut::new(); + bytes.put_u8(17); + bytes.put_bytes(19, 2); + assert_eq!([17, 19, 19], bytes.as_ref()); +} + +#[test] +fn box_slice_empty() { + // See https://github.com/tokio-rs/bytes/issues/340 + let empty: Box<[u8]> = Default::default(); + let b = Bytes::from(empty); + assert!(b.is_empty()); +} + +#[test] +fn bytes_into_vec() { + // Test kind == KIND_VEC + let content = b"helloworld"; + + let mut bytes = BytesMut::new(); + bytes.put_slice(content); + + let vec: Vec<u8> = bytes.into(); + assert_eq!(&vec, content); + + // Test kind == KIND_ARC, shared.is_unique() == True + let mut bytes = BytesMut::new(); + bytes.put_slice(b"abcdewe23"); + bytes.put_slice(content); + + // Overwrite the bytes to make sure only one reference to the underlying + // Vec exists. + bytes = bytes.split_off(9); + + let vec: Vec<u8> = bytes.into(); + assert_eq!(&vec, content); + + // Test kind == KIND_ARC, shared.is_unique() == False + let prefix = b"abcdewe23"; + + let mut bytes = BytesMut::new(); + bytes.put_slice(prefix); + bytes.put_slice(content); + + let vec: Vec<u8> = bytes.split_off(prefix.len()).into(); + assert_eq!(&vec, content); + + let vec: Vec<u8> = bytes.into(); + assert_eq!(&vec, prefix); +} + +#[test] +fn test_bytes_into_vec() { + // Test STATIC_VTABLE.to_vec + let bs = b"1b23exfcz3r"; + let vec: Vec<u8> = Bytes::from_static(bs).into(); + assert_eq!(&*vec, bs); + + // Test bytes_mut.SHARED_VTABLE.to_vec impl + eprintln!("1"); + let mut bytes_mut: BytesMut = bs[..].into(); + + // Set kind to KIND_ARC so that after freeze, Bytes will use bytes_mut.SHARED_VTABLE + eprintln!("2"); + drop(bytes_mut.split_off(bs.len())); + + eprintln!("3"); + let b1 = bytes_mut.freeze(); + eprintln!("4"); + let b2 = b1.clone(); + + eprintln!("{:#?}", (&*b1).as_ptr()); + + // shared.is_unique() = False + eprintln!("5"); + assert_eq!(&*Vec::from(b2), bs); + + // shared.is_unique() = True + eprintln!("6"); + assert_eq!(&*Vec::from(b1), bs); + + // Test bytes_mut.SHARED_VTABLE.to_vec impl where offset != 0 + let mut bytes_mut1: BytesMut = bs[..].into(); + let bytes_mut2 = bytes_mut1.split_off(9); + + let b1 = bytes_mut1.freeze(); + let b2 = bytes_mut2.freeze(); + + assert_eq!(Vec::from(b2), bs[9..]); + assert_eq!(Vec::from(b1), bs[..9]); +} + +#[test] +fn test_bytes_into_vec_promotable_even() { + let vec = vec![33u8; 1024]; + + // Test cases where kind == KIND_VEC + let b1 = Bytes::from(vec.clone()); + assert_eq!(Vec::from(b1), vec); + + // Test cases where kind == KIND_ARC, ref_cnt == 1 + let b1 = Bytes::from(vec.clone()); + drop(b1.clone()); + assert_eq!(Vec::from(b1), vec); + + // Test cases where kind == KIND_ARC, ref_cnt == 2 + let b1 = Bytes::from(vec.clone()); + let b2 = b1.clone(); + assert_eq!(Vec::from(b1), vec); + + // Test cases where vtable = SHARED_VTABLE, kind == KIND_ARC, ref_cnt == 1 + assert_eq!(Vec::from(b2), vec); + + // Test cases where offset != 0 + let mut b1 = Bytes::from(vec.clone()); + let b2 = b1.split_off(20); + + assert_eq!(Vec::from(b2), vec[20..]); + assert_eq!(Vec::from(b1), vec[..20]); +} + +#[test] +fn test_bytes_vec_conversion() { + let mut vec = Vec::with_capacity(10); + vec.extend(b"abcdefg"); + let b = Bytes::from(vec); + let v = Vec::from(b); + assert_eq!(v.len(), 7); + assert_eq!(v.capacity(), 10); + + let mut b = Bytes::from(v); + b.advance(1); + let v = Vec::from(b); + assert_eq!(v.len(), 6); + assert_eq!(v.capacity(), 10); + assert_eq!(v.as_slice(), b"bcdefg"); +} + +#[test] +fn test_bytes_mut_conversion() { + let mut b1 = BytesMut::with_capacity(10); + b1.extend(b"abcdefg"); + let b2 = Bytes::from(b1); + let v = Vec::from(b2); + assert_eq!(v.len(), 7); + assert_eq!(v.capacity(), 10); + + let mut b = Bytes::from(v); + b.advance(1); + let v = Vec::from(b); + assert_eq!(v.len(), 6); + assert_eq!(v.capacity(), 10); + assert_eq!(v.as_slice(), b"bcdefg"); +} + +#[test] +fn test_bytes_capacity_len() { + for cap in 0..100 { + for len in 0..=cap { + let mut v = Vec::with_capacity(cap); + v.resize(len, 0); + let _ = Bytes::from(v); + } + } +} + +#[test] +fn static_is_unique() { + let b = Bytes::from_static(LONG); + assert!(!b.is_unique()); +} + +#[test] +fn vec_is_unique() { + let v: Vec<u8> = LONG.to_vec(); + let b = Bytes::from(v); + assert!(b.is_unique()); +} + +#[test] +fn arc_is_unique() { + let v: Vec<u8> = LONG.to_vec(); + let b = Bytes::from(v); + let c = b.clone(); + assert!(!b.is_unique()); + drop(c); + assert!(b.is_unique()); +} + +#[test] +fn shared_is_unique() { + let v: Vec<u8> = LONG.to_vec(); + let b = Bytes::from(v); + let c = b.clone(); + assert!(!c.is_unique()); + drop(b); + assert!(c.is_unique()); +} + +#[test] +fn mut_shared_is_unique() { + let mut b = BytesMut::from(LONG); + let c = b.split().freeze(); + assert!(!c.is_unique()); + drop(b); + assert!(c.is_unique()); +} + +#[test] +fn test_bytesmut_from_bytes_static() { + let bs = b"1b23exfcz3r"; + + // Test STATIC_VTABLE.to_mut + let bytes_mut = BytesMut::from(Bytes::from_static(bs)); + assert_eq!(bytes_mut, bs[..]); +} + +#[test] +fn test_bytesmut_from_bytes_bytes_mut_vec() { + let bs = b"1b23exfcz3r"; + let bs_long = b"1b23exfcz3r1b23exfcz3r"; + + // Test case where kind == KIND_VEC + let mut bytes_mut: BytesMut = bs[..].into(); + bytes_mut = BytesMut::from(bytes_mut.freeze()); + assert_eq!(bytes_mut, bs[..]); + bytes_mut.extend_from_slice(&bs[..]); + assert_eq!(bytes_mut, bs_long[..]); +} + +#[test] +fn test_bytesmut_from_bytes_bytes_mut_shared() { + let bs = b"1b23exfcz3r"; + + // Set kind to KIND_ARC so that after freeze, Bytes will use bytes_mut.SHARED_VTABLE + let mut bytes_mut: BytesMut = bs[..].into(); + drop(bytes_mut.split_off(bs.len())); + + let b1 = bytes_mut.freeze(); + let b2 = b1.clone(); + + // shared.is_unique() = False + let mut b1m = BytesMut::from(b1); + assert_eq!(b1m, bs[..]); + b1m[0] = b'9'; + + // shared.is_unique() = True + let b2m = BytesMut::from(b2); + assert_eq!(b2m, bs[..]); +} + +#[test] +fn test_bytesmut_from_bytes_bytes_mut_offset() { + let bs = b"1b23exfcz3r"; + + // Test bytes_mut.SHARED_VTABLE.to_mut impl where offset != 0 + let mut bytes_mut1: BytesMut = bs[..].into(); + let bytes_mut2 = bytes_mut1.split_off(9); + + let b1 = bytes_mut1.freeze(); + let b2 = bytes_mut2.freeze(); + + let b1m = BytesMut::from(b1); + let b2m = BytesMut::from(b2); + + assert_eq!(b2m, bs[9..]); + assert_eq!(b1m, bs[..9]); +} + +#[test] +fn test_bytesmut_from_bytes_promotable_even_vec() { + let vec = vec![33u8; 1024]; + + // Test case where kind == KIND_VEC + let b1 = Bytes::from(vec.clone()); + let b1m = BytesMut::from(b1); + assert_eq!(b1m, vec); +} + +#[test] +fn test_bytesmut_from_bytes_promotable_even_arc_1() { + let vec = vec![33u8; 1024]; + + // Test case where kind == KIND_ARC, ref_cnt == 1 + let b1 = Bytes::from(vec.clone()); + drop(b1.clone()); + let b1m = BytesMut::from(b1); + assert_eq!(b1m, vec); +} + +#[test] +fn test_bytesmut_from_bytes_promotable_even_arc_2() { + let vec = vec![33u8; 1024]; + + // Test case where kind == KIND_ARC, ref_cnt == 2 + let b1 = Bytes::from(vec.clone()); + let b2 = b1.clone(); + let b1m = BytesMut::from(b1); + assert_eq!(b1m, vec); + + // Test case where vtable = SHARED_VTABLE, kind == KIND_ARC, ref_cnt == 1 + let b2m = BytesMut::from(b2); + assert_eq!(b2m, vec); +} + +#[test] +fn test_bytesmut_from_bytes_promotable_even_arc_offset() { + let vec = vec![33u8; 1024]; + + // Test case where offset != 0 + let mut b1 = Bytes::from(vec.clone()); + let b2 = b1.split_off(20); + let b1m = BytesMut::from(b1); + let b2m = BytesMut::from(b2); + + assert_eq!(b2m, vec[20..]); + assert_eq!(b1m, vec[..20]); +} + +#[test] +fn try_reclaim_empty() { + let mut buf = BytesMut::new(); + assert_eq!(false, buf.try_reclaim(6)); + buf.reserve(6); + assert_eq!(true, buf.try_reclaim(6)); + let cap = buf.capacity(); + assert!(cap >= 6); + assert_eq!(false, buf.try_reclaim(cap + 1)); + + let mut buf = BytesMut::new(); + buf.reserve(6); + let cap = buf.capacity(); + assert!(cap >= 6); + let mut split = buf.split(); + drop(buf); + assert_eq!(0, split.capacity()); + assert_eq!(true, split.try_reclaim(6)); + assert_eq!(false, split.try_reclaim(cap + 1)); +} + +#[test] +fn try_reclaim_vec() { + let mut buf = BytesMut::with_capacity(6); + buf.put_slice(b"abc"); + // Reclaiming a ludicrous amount of space should calmly return false + assert_eq!(false, buf.try_reclaim(usize::MAX)); + + assert_eq!(false, buf.try_reclaim(6)); + buf.advance(2); + assert_eq!(4, buf.capacity()); + // We can reclaim 5 bytes, because the byte in the buffer can be moved to the front. 6 bytes + // cannot be reclaimed because there is already one byte stored + assert_eq!(false, buf.try_reclaim(6)); + assert_eq!(true, buf.try_reclaim(5)); + buf.advance(1); + assert_eq!(true, buf.try_reclaim(6)); + assert_eq!(6, buf.capacity()); +} + +#[test] +fn try_reclaim_arc() { + let mut buf = BytesMut::with_capacity(6); + buf.put_slice(b"abc"); + let x = buf.split().freeze(); + buf.put_slice(b"def"); + // Reclaiming a ludicrous amount of space should calmly return false + assert_eq!(false, buf.try_reclaim(usize::MAX)); + + let y = buf.split().freeze(); + let z = y.clone(); + assert_eq!(false, buf.try_reclaim(6)); + drop(x); + drop(z); + assert_eq!(false, buf.try_reclaim(6)); + drop(y); + assert_eq!(true, buf.try_reclaim(6)); + assert_eq!(6, buf.capacity()); + assert_eq!(0, buf.len()); + buf.put_slice(b"abc"); + buf.put_slice(b"def"); + assert_eq!(6, buf.capacity()); + assert_eq!(6, buf.len()); + assert_eq!(false, buf.try_reclaim(6)); + buf.advance(4); + assert_eq!(true, buf.try_reclaim(4)); + buf.advance(2); + assert_eq!(true, buf.try_reclaim(6)); +} + +#[test] +fn slice_empty_addr() { + let buf = Bytes::from(vec![0; 1024]); + + let ptr_start = buf.as_ptr(); + let ptr_end = ptr_start.wrapping_add(1024); + + let empty_end = buf.slice(1024..); + assert_eq!(empty_end.len(), 0); + assert_eq!(empty_end.as_ptr(), ptr_end); + + let empty_start = buf.slice(..0); + assert_eq!(empty_start.len(), 0); + assert_eq!(empty_start.as_ptr(), ptr_start); + + // Is miri happy about the provenance? + let _ = &empty_end[..]; + let _ = &empty_start[..]; +} + +#[test] +fn split_off_empty_addr() { + let mut buf = Bytes::from(vec![0; 1024]); + + let ptr_start = buf.as_ptr(); + let ptr_end = ptr_start.wrapping_add(1024); + + let empty_end = buf.split_off(1024); + assert_eq!(empty_end.len(), 0); + assert_eq!(empty_end.as_ptr(), ptr_end); + + let _ = buf.split_off(0); + assert_eq!(buf.len(), 0); + assert_eq!(buf.as_ptr(), ptr_start); + + // Is miri happy about the provenance? + let _ = &empty_end[..]; + let _ = &buf[..]; +} + +#[test] +fn split_to_empty_addr() { + let mut buf = Bytes::from(vec![0; 1024]); + + let ptr_start = buf.as_ptr(); + let ptr_end = ptr_start.wrapping_add(1024); + + let empty_start = buf.split_to(0); + assert_eq!(empty_start.len(), 0); + assert_eq!(empty_start.as_ptr(), ptr_start); + + let _ = buf.split_to(1024); + assert_eq!(buf.len(), 0); + assert_eq!(buf.as_ptr(), ptr_end); + + // Is miri happy about the provenance? + let _ = &empty_start[..]; + let _ = &buf[..]; +} + +#[test] +fn split_off_empty_addr_mut() { + let mut buf = BytesMut::from([0; 1024].as_slice()); + + let ptr_start = buf.as_ptr(); + let ptr_end = ptr_start.wrapping_add(1024); + + let empty_end = buf.split_off(1024); + assert_eq!(empty_end.len(), 0); + assert_eq!(empty_end.as_ptr(), ptr_end); + + let _ = buf.split_off(0); + assert_eq!(buf.len(), 0); + assert_eq!(buf.as_ptr(), ptr_start); + + // Is miri happy about the provenance? + let _ = &empty_end[..]; + let _ = &buf[..]; +} + +#[test] +fn split_to_empty_addr_mut() { + let mut buf = BytesMut::from([0; 1024].as_slice()); + + let ptr_start = buf.as_ptr(); + let ptr_end = ptr_start.wrapping_add(1024); + + let empty_start = buf.split_to(0); + assert_eq!(empty_start.len(), 0); + assert_eq!(empty_start.as_ptr(), ptr_start); + + let _ = buf.split_to(1024); + assert_eq!(buf.len(), 0); + assert_eq!(buf.as_ptr(), ptr_end); + + // Is miri happy about the provenance? + let _ = &empty_start[..]; + let _ = &buf[..]; +} + +#[derive(Clone)] +struct SharedAtomicCounter(Arc<AtomicUsize>); + +impl SharedAtomicCounter { + pub fn new() -> Self { + SharedAtomicCounter(Arc::new(AtomicUsize::new(0))) + } + + pub fn increment(&self) { + self.0.fetch_add(1, Ordering::AcqRel); + } + + pub fn get(&self) -> usize { + self.0.load(Ordering::Acquire) + } +} + +#[derive(Clone)] +struct OwnedTester<const L: usize> { + buf: [u8; L], + drop_count: SharedAtomicCounter, + pub panic_as_ref: bool, +} + +impl<const L: usize> OwnedTester<L> { + fn new(buf: [u8; L], drop_count: SharedAtomicCounter) -> Self { + Self { + buf, + drop_count, + panic_as_ref: false, + } + } +} + +impl<const L: usize> AsRef<[u8]> for OwnedTester<L> { + fn as_ref(&self) -> &[u8] { + if self.panic_as_ref { + panic!("test-triggered panic in `AsRef<[u8]> for OwnedTester`"); + } + self.buf.as_slice() + } +} + +impl<const L: usize> Drop for OwnedTester<L> { + fn drop(&mut self) { + self.drop_count.increment(); + } +} + +#[test] +fn owned_is_unique_always_false() { + let b1 = Bytes::from_owner([1, 2, 3, 4, 5, 6, 7]); + assert!(!b1.is_unique()); // even if ref_cnt == 1 + let b2 = b1.clone(); + assert!(!b1.is_unique()); + assert!(!b2.is_unique()); + drop(b1); + assert!(!b2.is_unique()); // even if ref_cnt == 1 +} + +#[test] +fn owned_buf_sharing() { + let buf = [1, 2, 3, 4, 5, 6, 7]; + let b1 = Bytes::from_owner(buf); + let b2 = b1.clone(); + assert_eq!(&buf[..], &b1[..]); + assert_eq!(&buf[..], &b2[..]); + assert_eq!(b1.as_ptr(), b2.as_ptr()); + assert_eq!(b1.len(), b2.len()); + assert_eq!(b1.len(), buf.len()); +} + +#[test] +fn owned_buf_slicing() { + let b1 = Bytes::from_owner(SHORT); + assert_eq!(SHORT, &b1[..]); + let b2 = b1.slice(1..(b1.len() - 1)); + assert_eq!(&SHORT[1..(SHORT.len() - 1)], b2); + assert_eq!(unsafe { SHORT.as_ptr().add(1) }, b2.as_ptr()); + assert_eq!(SHORT.len() - 2, b2.len()); +} + +#[test] +fn owned_dropped_exactly_once() { + let buf: [u8; 5] = [1, 2, 3, 4, 5]; + let drop_counter = SharedAtomicCounter::new(); + let owner = OwnedTester::new(buf, drop_counter.clone()); + let b1 = Bytes::from_owner(owner); + let b2 = b1.clone(); + assert_eq!(drop_counter.get(), 0); + drop(b1); + assert_eq!(drop_counter.get(), 0); + let b3 = b2.slice(1..b2.len() - 1); + drop(b2); + assert_eq!(drop_counter.get(), 0); + drop(b3); + assert_eq!(drop_counter.get(), 1); +} + +#[test] +fn owned_to_mut() { + let buf: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + let drop_counter = SharedAtomicCounter::new(); + let owner = OwnedTester::new(buf, drop_counter.clone()); + let b1 = Bytes::from_owner(owner); + + // Holding an owner will fail converting to a BytesMut, + // even when the bytes instance has a ref_cnt == 1. + let b1 = b1.try_into_mut().unwrap_err(); + + // That said, it's still possible, just not cheap. + let bm1: BytesMut = b1.into(); + let new_buf = &bm1[..]; + assert_eq!(new_buf, &buf[..]); + + // `.into::<BytesMut>()` has correctly dropped the owner + assert_eq!(drop_counter.get(), 1); +} + +#[test] +fn owned_to_vec() { + let buf: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + let drop_counter = SharedAtomicCounter::new(); + let owner = OwnedTester::new(buf, drop_counter.clone()); + let b1 = Bytes::from_owner(owner); + + let v1 = b1.to_vec(); + assert_eq!(&v1[..], &buf[..]); + assert_eq!(&v1[..], &b1[..]); + + drop(b1); + assert_eq!(drop_counter.get(), 1); +} + +#[test] +fn owned_into_vec() { + let drop_counter = SharedAtomicCounter::new(); + let buf: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + let owner = OwnedTester::new(buf, drop_counter.clone()); + let b1 = Bytes::from_owner(owner); + + let v1: Vec<u8> = b1.into(); + assert_eq!(&v1[..], &buf[..]); + // into() vec will copy out of the owner and drop it + assert_eq!(drop_counter.get(), 1); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore)] +fn owned_safe_drop_on_as_ref_panic() { + let buf: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + let drop_counter = SharedAtomicCounter::new(); + let mut owner = OwnedTester::new(buf, drop_counter.clone()); + owner.panic_as_ref = true; + + let result = panic::catch_unwind(AssertUnwindSafe(|| { + let _ = Bytes::from_owner(owner); + })); + + assert!(result.is_err()); + assert_eq!(drop_counter.get(), 1); +} + +/// Test `BytesMut::put` reuses allocation of `Bytes`. +#[test] +fn bytes_mut_put_bytes_specialization() { + let mut vec = Vec::with_capacity(1234); + vec.push(10); + let capacity = vec.capacity(); + assert!(capacity >= 1234); + + // Make `Bytes` backed by `Vec`. + let bytes = Bytes::from(vec); + let mut bytes_mut = BytesMut::new(); + bytes_mut.put(bytes); + + // Check contents is correct. + assert_eq!(&[10], bytes_mut.as_ref()); + // If allocation is reused, capacity should be equal to original vec capacity. + assert_eq!(bytes_mut.capacity(), capacity); +} + +#[test] +#[should_panic] +fn bytes_mut_reserve_overflow() { + let mut a = BytesMut::from(&b"hello world"[..]); + let mut b = a.split_off(5); + // Ensure b becomes the unique owner of the backing storage + drop(a); + // Trigger overflow in new_cap + offset inside reserve + b.reserve(usize::MAX - 6); + // This call relies on the corrupted cap and may cause UB & HBO + b.put_u8(b'h'); +} diff --git a/third-party/bytes/tests/test_bytes_odd_alloc.rs b/third-party/bytes/tests/test_bytes_odd_alloc.rs new file mode 100644 index 000000000..4758dc2f9 --- /dev/null +++ b/third-party/bytes/tests/test_bytes_odd_alloc.rs @@ -0,0 +1,147 @@ +//! Test using `Bytes` with an allocator that hands out "odd" pointers for +//! vectors (pointers where the LSB is set). + +#![cfg(not(miri))] // Miri does not support custom allocators (also, Miri is "odd" by default with 50% chance) + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::ptr; + +use bytes::{Bytes, BytesMut}; + +#[global_allocator] +static ODD: Odd = Odd; + +struct Odd; + +unsafe impl GlobalAlloc for Odd { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if layout.align() == 1 && layout.size() > 0 { + // Allocate slightly bigger so that we can offset the pointer by 1 + let size = layout.size() + 1; + let new_layout = match Layout::from_size_align(size, 1) { + Ok(layout) => layout, + Err(_err) => return ptr::null_mut(), + }; + let ptr = System.alloc(new_layout); + if !ptr.is_null() { + ptr.offset(1) + } else { + ptr + } + } else { + System.alloc(layout) + } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + if layout.align() == 1 && layout.size() > 0 { + let size = layout.size() + 1; + let new_layout = match Layout::from_size_align(size, 1) { + Ok(layout) => layout, + Err(_err) => std::process::abort(), + }; + System.dealloc(ptr.offset(-1), new_layout); + } else { + System.dealloc(ptr, layout); + } + } +} + +#[test] +fn sanity_check_odd_allocator() { + let vec = vec![33u8; 1024]; + let p = vec.as_ptr() as usize; + assert!(p & 0x1 == 0x1, "{:#b}", p); +} + +#[test] +fn test_bytes_from_vec_drop() { + let vec = vec![33u8; 1024]; + let _b = Bytes::from(vec); +} + +#[test] +fn test_bytes_clone_drop() { + let vec = vec![33u8; 1024]; + let b1 = Bytes::from(vec); + let _b2 = b1.clone(); +} + +#[test] +fn test_bytes_into_vec() { + let vec = vec![33u8; 1024]; + + // Test cases where kind == KIND_VEC + let b1 = Bytes::from(vec.clone()); + assert_eq!(Vec::from(b1), vec); + + // Test cases where kind == KIND_ARC, ref_cnt == 1 + let b1 = Bytes::from(vec.clone()); + drop(b1.clone()); + assert_eq!(Vec::from(b1), vec); + + // Test cases where kind == KIND_ARC, ref_cnt == 2 + let b1 = Bytes::from(vec.clone()); + let b2 = b1.clone(); + assert_eq!(Vec::from(b1), vec); + + // Test cases where vtable = SHARED_VTABLE, kind == KIND_ARC, ref_cnt == 1 + assert_eq!(Vec::from(b2), vec); + + // Test cases where offset != 0 + let mut b1 = Bytes::from(vec.clone()); + let b2 = b1.split_off(20); + + assert_eq!(Vec::from(b2), vec[20..]); + assert_eq!(Vec::from(b1), vec[..20]); +} + +#[test] +fn test_bytesmut_from_bytes_vec() { + let vec = vec![33u8; 1024]; + + // Test case where kind == KIND_VEC + let b1 = Bytes::from(vec.clone()); + let b1m = BytesMut::from(b1); + assert_eq!(b1m, vec); +} + +#[test] +fn test_bytesmut_from_bytes_arc_1() { + let vec = vec![33u8; 1024]; + + // Test case where kind == KIND_ARC, ref_cnt == 1 + let b1 = Bytes::from(vec.clone()); + drop(b1.clone()); + let b1m = BytesMut::from(b1); + assert_eq!(b1m, vec); +} + +#[test] +fn test_bytesmut_from_bytes_arc_2() { + let vec = vec![33u8; 1024]; + + // Test case where kind == KIND_ARC, ref_cnt == 2 + let b1 = Bytes::from(vec.clone()); + let b2 = b1.clone(); + let b1m = BytesMut::from(b1); + assert_eq!(b1m, vec); + + // Test case where vtable = SHARED_VTABLE, kind == KIND_ARC, ref_cnt == 1 + let b2m = BytesMut::from(b2); + assert_eq!(b2m, vec); +} + +#[test] +fn test_bytesmut_from_bytes_arc_offset() { + let vec = vec![33u8; 1024]; + + // Test case where offset != 0 + let mut b1 = Bytes::from(vec.clone()); + let b2 = b1.split_off(20); + let b1m = BytesMut::from(b1); + let b2m = BytesMut::from(b2); + + assert_eq!(b2m, vec[20..]); + assert_eq!(b1m, vec[..20]); +} diff --git a/third-party/bytes/tests/test_bytes_vec_alloc.rs b/third-party/bytes/tests/test_bytes_vec_alloc.rs new file mode 100644 index 000000000..9c7601717 --- /dev/null +++ b/third-party/bytes/tests/test_bytes_vec_alloc.rs @@ -0,0 +1,146 @@ +#![cfg(not(miri))] +use std::alloc::{GlobalAlloc, Layout, System}; +use std::ptr::null_mut; +use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + +use bytes::{Buf, Bytes}; + +#[global_allocator] +static LEDGER: Ledger = Ledger::new(); + +const LEDGER_LENGTH: usize = 1024 * 1024; + +struct Ledger { + alloc_table: [(AtomicPtr<u8>, AtomicUsize); LEDGER_LENGTH], +} + +impl Ledger { + const fn new() -> Self { + const ELEM: (AtomicPtr<u8>, AtomicUsize) = + (AtomicPtr::new(null_mut()), AtomicUsize::new(0)); + let alloc_table = [ELEM; LEDGER_LENGTH]; + + Self { alloc_table } + } + + /// Iterate over our table until we find an open entry, then insert into said entry + fn insert(&self, ptr: *mut u8, size: usize) { + for (entry_ptr, entry_size) in self.alloc_table.iter() { + // SeqCst is good enough here, we don't care about perf, i just want to be correct! + if entry_ptr + .compare_exchange(null_mut(), ptr, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + entry_size.store(size, Ordering::SeqCst); + return; + } + } + + panic!("Ledger ran out of space."); + } + + fn remove(&self, ptr: *mut u8) -> usize { + for (entry_ptr, entry_size) in self.alloc_table.iter() { + // set the value to be something that will never try and be deallocated, so that we + // don't have any chance of a race condition + // + // dont worry, LEDGER_LENGTH is really long to compensate for us not reclaiming space + if entry_ptr + .compare_exchange( + ptr, + invalid_ptr(usize::MAX), + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + return entry_size.load(Ordering::SeqCst); + } + } + + panic!("Couldn't find a matching entry for {:x?}", ptr); + } +} + +unsafe impl GlobalAlloc for Ledger { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let size = layout.size(); + let ptr = System.alloc(layout); + self.insert(ptr, size); + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + let orig_size = self.remove(ptr); + + if orig_size != layout.size() { + panic!( + "bad dealloc: alloc size was {}, dealloc size is {}", + orig_size, + layout.size() + ); + } else { + System.dealloc(ptr, layout); + } + } +} + +#[test] +fn test_bytes_advance() { + let mut bytes = Bytes::from(vec![10, 20, 30]); + bytes.advance(1); + drop(bytes); +} + +#[test] +fn test_bytes_truncate() { + let mut bytes = Bytes::from(vec![10, 20, 30]); + bytes.truncate(2); + drop(bytes); +} + +#[test] +fn test_bytes_truncate_and_advance() { + let mut bytes = Bytes::from(vec![10, 20, 30]); + bytes.truncate(2); + bytes.advance(1); + drop(bytes); +} + +/// Returns a dangling pointer with the given address. This is used to store +/// integer data in pointer fields. +#[inline] +fn invalid_ptr<T>(addr: usize) -> *mut T { + let ptr = std::ptr::null_mut::<u8>().wrapping_add(addr); + debug_assert_eq!(ptr as usize, addr); + ptr.cast::<T>() +} + +#[test] +fn test_bytes_into_vec() { + let vec = vec![33u8; 1024]; + + // Test cases where kind == KIND_VEC + let b1 = Bytes::from(vec.clone()); + assert_eq!(Vec::from(b1), vec); + + // Test cases where kind == KIND_ARC, ref_cnt == 1 + let b1 = Bytes::from(vec.clone()); + drop(b1.clone()); + assert_eq!(Vec::from(b1), vec); + + // Test cases where kind == KIND_ARC, ref_cnt == 2 + let b1 = Bytes::from(vec.clone()); + let b2 = b1.clone(); + assert_eq!(Vec::from(b1), vec); + + // Test cases where vtable = SHARED_VTABLE, kind == KIND_ARC, ref_cnt == 1 + assert_eq!(Vec::from(b2), vec); + + // Test cases where offset != 0 + let mut b1 = Bytes::from(vec.clone()); + let b2 = b1.split_off(20); + + assert_eq!(Vec::from(b2), vec[20..]); + assert_eq!(Vec::from(b1), vec[..20]); +} diff --git a/third-party/bytes/tests/test_chain.rs b/third-party/bytes/tests/test_chain.rs new file mode 100644 index 000000000..e809f44f1 --- /dev/null +++ b/third-party/bytes/tests/test_chain.rs @@ -0,0 +1,177 @@ +#![warn(rust_2018_idioms)] + +use bytes::{Buf, BufMut, Bytes}; +#[cfg(feature = "std")] +use std::io::IoSlice; + +#[test] +fn collect_two_bufs() { + let a = Bytes::from(&b"hello"[..]); + let b = Bytes::from(&b"world"[..]); + + let res = a.chain(b).copy_to_bytes(10); + assert_eq!(res, &b"helloworld"[..]); +} + +#[test] +fn writing_chained() { + let mut a = [0u8; 64]; + let mut b = [0u8; 64]; + + { + let mut buf = (&mut a[..]).chain_mut(&mut b[..]); + + for i in 0u8..128 { + buf.put_u8(i); + } + } + + for i in 0..64 { + let expect = i as u8; + assert_eq!(expect, a[i]); + assert_eq!(expect + 64, b[i]); + } +} + +#[test] +fn iterating_two_bufs() { + let a = Bytes::from(&b"hello"[..]); + let b = Bytes::from(&b"world"[..]); + + let res: Vec<u8> = a.chain(b).into_iter().collect(); + assert_eq!(res, &b"helloworld"[..]); +} + +#[cfg(feature = "std")] +#[test] +fn vectored_read() { + let a = Bytes::from(&b"hello"[..]); + let b = Bytes::from(&b"world"[..]); + + let mut buf = a.chain(b); + + { + let b1: &[u8] = &mut []; + let b2: &[u8] = &mut []; + let b3: &[u8] = &mut []; + let b4: &[u8] = &mut []; + let mut iovecs = [ + IoSlice::new(b1), + IoSlice::new(b2), + IoSlice::new(b3), + IoSlice::new(b4), + ]; + + assert_eq!(2, buf.chunks_vectored(&mut iovecs)); + assert_eq!(iovecs[0][..], b"hello"[..]); + assert_eq!(iovecs[1][..], b"world"[..]); + assert_eq!(iovecs[2][..], b""[..]); + assert_eq!(iovecs[3][..], b""[..]); + } + + buf.advance(2); + + { + let b1: &[u8] = &mut []; + let b2: &[u8] = &mut []; + let b3: &[u8] = &mut []; + let b4: &[u8] = &mut []; + let mut iovecs = [ + IoSlice::new(b1), + IoSlice::new(b2), + IoSlice::new(b3), + IoSlice::new(b4), + ]; + + assert_eq!(2, buf.chunks_vectored(&mut iovecs)); + assert_eq!(iovecs[0][..], b"llo"[..]); + assert_eq!(iovecs[1][..], b"world"[..]); + assert_eq!(iovecs[2][..], b""[..]); + assert_eq!(iovecs[3][..], b""[..]); + } + + buf.advance(3); + + { + let b1: &[u8] = &mut []; + let b2: &[u8] = &mut []; + let b3: &[u8] = &mut []; + let b4: &[u8] = &mut []; + let mut iovecs = [ + IoSlice::new(b1), + IoSlice::new(b2), + IoSlice::new(b3), + IoSlice::new(b4), + ]; + + assert_eq!(1, buf.chunks_vectored(&mut iovecs)); + assert_eq!(iovecs[0][..], b"world"[..]); + assert_eq!(iovecs[1][..], b""[..]); + assert_eq!(iovecs[2][..], b""[..]); + assert_eq!(iovecs[3][..], b""[..]); + } + + buf.advance(3); + + { + let b1: &[u8] = &mut []; + let b2: &[u8] = &mut []; + let b3: &[u8] = &mut []; + let b4: &[u8] = &mut []; + let mut iovecs = [ + IoSlice::new(b1), + IoSlice::new(b2), + IoSlice::new(b3), + IoSlice::new(b4), + ]; + + assert_eq!(1, buf.chunks_vectored(&mut iovecs)); + assert_eq!(iovecs[0][..], b"ld"[..]); + assert_eq!(iovecs[1][..], b""[..]); + assert_eq!(iovecs[2][..], b""[..]); + assert_eq!(iovecs[3][..], b""[..]); + } +} + +#[test] +fn chain_growing_buffer() { + let mut buff = [b' '; 10]; + let mut vec = b"wassup".to_vec(); + + let mut chained = (&mut buff[..]).chain_mut(&mut vec).chain_mut(Vec::new()); // Required for potential overflow because remaining_mut for Vec is isize::MAX - vec.len(), but for chain_mut is usize::MAX + + chained.put_slice(b"hey there123123"); + + assert_eq!(&buff, b"hey there1"); + assert_eq!(&vec, b"wassup23123"); +} + +#[test] +fn chain_overflow_remaining_mut() { + let mut chained = Vec::<u8>::new().chain_mut(Vec::new()).chain_mut(Vec::new()); + + assert_eq!(chained.remaining_mut(), usize::MAX); + chained.put_slice(&[0; 256]); + assert_eq!(chained.remaining_mut(), usize::MAX); +} + +#[test] +fn chain_get_bytes() { + let mut ab = Bytes::copy_from_slice(b"ab"); + let mut cd = Bytes::copy_from_slice(b"cd"); + let ab_ptr = ab.as_ptr(); + let cd_ptr = cd.as_ptr(); + let mut chain = (&mut ab).chain(&mut cd); + let a = chain.copy_to_bytes(1); + let bc = chain.copy_to_bytes(2); + let d = chain.copy_to_bytes(1); + + assert_eq!(Bytes::copy_from_slice(b"a"), a); + assert_eq!(Bytes::copy_from_slice(b"bc"), bc); + assert_eq!(Bytes::copy_from_slice(b"d"), d); + + // assert `get_bytes` did not allocate + assert_eq!(ab_ptr, a.as_ptr()); + // assert `get_bytes` did not allocate + assert_eq!(cd_ptr.wrapping_offset(1), d.as_ptr()); +} diff --git a/third-party/bytes/tests/test_debug.rs b/third-party/bytes/tests/test_debug.rs new file mode 100644 index 000000000..08d2f254e --- /dev/null +++ b/third-party/bytes/tests/test_debug.rs @@ -0,0 +1,35 @@ +#![warn(rust_2018_idioms)] + +use bytes::Bytes; + +#[test] +fn fmt() { + let vec: Vec<_> = (0..0x100).map(|b| b as u8).collect(); + + let expected = "b\"\ + \\0\\x01\\x02\\x03\\x04\\x05\\x06\\x07\ + \\x08\\t\\n\\x0b\\x0c\\r\\x0e\\x0f\ + \\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\ + \\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f\ + \x20!\\\"#$%&'()*+,-./0123456789:;<=>?\ + @ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_\ + `abcdefghijklmnopqrstuvwxyz{|}~\\x7f\ + \\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\ + \\x88\\x89\\x8a\\x8b\\x8c\\x8d\\x8e\\x8f\ + \\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\ + \\x98\\x99\\x9a\\x9b\\x9c\\x9d\\x9e\\x9f\ + \\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\ + \\xa8\\xa9\\xaa\\xab\\xac\\xad\\xae\\xaf\ + \\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7\ + \\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\ + \\xc0\\xc1\\xc2\\xc3\\xc4\\xc5\\xc6\\xc7\ + \\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\ + \\xd0\\xd1\\xd2\\xd3\\xd4\\xd5\\xd6\\xd7\ + \\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\ + \\xe0\\xe1\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\ + \\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef\ + \\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\ + \\xf8\\xf9\\xfa\\xfb\\xfc\\xfd\\xfe\\xff\""; + + assert_eq!(expected, format!("{:?}", Bytes::from(vec))); +} diff --git a/third-party/bytes/tests/test_iter.rs b/third-party/bytes/tests/test_iter.rs new file mode 100644 index 000000000..bad901860 --- /dev/null +++ b/third-party/bytes/tests/test_iter.rs @@ -0,0 +1,21 @@ +#![warn(rust_2018_idioms)] + +use bytes::{buf::IntoIter, Bytes}; + +#[test] +fn iter_len() { + let buf = Bytes::from_static(b"hello world"); + let iter = IntoIter::new(buf); + + assert_eq!(iter.size_hint(), (11, Some(11))); + assert_eq!(iter.len(), 11); +} + +#[test] +fn empty_iter_len() { + let buf = Bytes::new(); + let iter = IntoIter::new(buf); + + assert_eq!(iter.size_hint(), (0, Some(0))); + assert_eq!(iter.len(), 0); +} diff --git a/third-party/bytes/tests/test_limit.rs b/third-party/bytes/tests/test_limit.rs new file mode 100644 index 000000000..dc40cc5f0 --- /dev/null +++ b/third-party/bytes/tests/test_limit.rs @@ -0,0 +1,86 @@ +#![warn(rust_2018_idioms)] + +use bytes::{buf::Limit, BufMut}; + +#[test] +fn long_limit() { + let buf = &mut [0u8; 10]; + let limit = buf.limit(100); + assert_eq!(10, limit.remaining_mut()); + assert_eq!(&[0u8; 10], &limit.get_ref()[..]); +} + +#[test] +fn limit_get_mut() { + let buf = &mut [0u8; 128]; + let mut limit = buf.limit(10); + assert_eq!(10, limit.remaining_mut()); + assert_eq!(&mut [0u8; 128], &limit.get_mut()[..]); +} + +#[test] +fn limit_set_limit() { + let buf = &mut [0u8; 128]; + let mut limit = buf.limit(10); + assert_eq!(10, Limit::limit(&limit)); + limit.set_limit(5); + assert_eq!(5, Limit::limit(&limit)); +} + +#[test] +fn limit_chunk_mut() { + let buf = &mut [0u8; 20]; + let mut limit = buf.limit(10); + assert_eq!(10, limit.chunk_mut().len()); + + let buf = &mut [0u8; 10]; + let mut limit = buf.limit(20); + assert_eq!(10, limit.chunk_mut().len()); +} + +#[test] +#[should_panic = "advance out of bounds"] +fn limit_advance_mut_panic_1() { + let buf = &mut [0u8; 10]; + let mut limit = buf.limit(100); + unsafe { + limit.advance_mut(50); + } +} + +#[test] +#[should_panic = "cnt <= self.limit"] +fn limit_advance_mut_panic_2() { + let buf = &mut [0u8; 100]; + let mut limit = buf.limit(10); + unsafe { + limit.advance_mut(50); + } +} + +#[test] +fn limit_advance_mut() { + let buf = &mut [0u8; 100]; + let mut limit = buf.limit(10); + unsafe { + limit.advance_mut(5); + } + assert_eq!(5, limit.remaining_mut()); + assert_eq!(5, limit.chunk_mut().len()); +} + +#[test] +fn limit_into_inner() { + let buf_arr = *b"hello world"; + let buf: &mut [u8] = &mut buf_arr.clone(); + let mut limit = buf.limit(4); + let mut dst = vec![]; + + unsafe { + limit.advance_mut(2); + } + + let buf = limit.into_inner(); + dst.put(&buf[..]); + assert_eq!(*dst, b"llo world"[..]); +} diff --git a/third-party/bytes/tests/test_reader.rs b/third-party/bytes/tests/test_reader.rs new file mode 100644 index 000000000..19a16324e --- /dev/null +++ b/third-party/bytes/tests/test_reader.rs @@ -0,0 +1,38 @@ +#![warn(rust_2018_idioms)] +#![cfg(feature = "std")] + +use std::io::{BufRead, Read}; + +use bytes::Buf; + +#[test] +fn read() { + let buf1 = &b"hello "[..]; + let buf2 = &b"world"[..]; + let buf = Buf::chain(buf1, buf2); // Disambiguate with Read::chain + let mut buffer = Vec::new(); + buf.reader().read_to_end(&mut buffer).unwrap(); + assert_eq!(b"hello world", &buffer[..]); +} + +#[test] +fn buf_read() { + let buf1 = &b"hell"[..]; + let buf2 = &b"o\nworld"[..]; + let mut reader = Buf::chain(buf1, buf2).reader(); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + assert_eq!("hello\n", &line); + line.clear(); + reader.read_line(&mut line).unwrap(); + assert_eq!("world", &line); +} + +#[test] +fn get_mut() { + let buf = &b"hello world"[..]; + let mut reader = buf.reader(); + let buf_mut = reader.get_mut(); + assert_eq!(11, buf_mut.remaining()); + assert_eq!(b"hello world", buf_mut); +} diff --git a/third-party/bytes/tests/test_serde.rs b/third-party/bytes/tests/test_serde.rs new file mode 100644 index 000000000..cf4aeffa7 --- /dev/null +++ b/third-party/bytes/tests/test_serde.rs @@ -0,0 +1,20 @@ +#![cfg(feature = "serde")] +#![warn(rust_2018_idioms)] + +use serde_test::{assert_tokens, Token}; + +#[test] +fn test_ser_de_empty() { + let b = bytes::Bytes::new(); + assert_tokens(&b, &[Token::Bytes(b"")]); + let b = bytes::BytesMut::with_capacity(0); + assert_tokens(&b, &[Token::Bytes(b"")]); +} + +#[test] +fn test_ser_de() { + let b = bytes::Bytes::from(&b"bytes"[..]); + assert_tokens(&b, &[Token::Bytes(b"bytes")]); + let b = bytes::BytesMut::from(&b"bytes"[..]); + assert_tokens(&b, &[Token::Bytes(b"bytes")]); +} diff --git a/third-party/bytes/tests/test_take.rs b/third-party/bytes/tests/test_take.rs new file mode 100644 index 000000000..0c0159be1 --- /dev/null +++ b/third-party/bytes/tests/test_take.rs @@ -0,0 +1,84 @@ +#![warn(rust_2018_idioms)] + +use bytes::buf::Buf; +use bytes::Bytes; + +#[test] +fn long_take() { + // Tests that get a take with a size greater than the buffer length will not + // overrun the buffer. Regression test for #138. + let buf = b"hello world".take(100); + assert_eq!(11, buf.remaining()); + assert_eq!(b"hello world", buf.chunk()); +} + +#[test] +fn take_copy_to_bytes() { + let mut abcd = Bytes::copy_from_slice(b"abcd"); + let abcd_ptr = abcd.as_ptr(); + let mut take = (&mut abcd).take(2); + let a = take.copy_to_bytes(1); + assert_eq!(Bytes::copy_from_slice(b"a"), a); + // assert `to_bytes` did not allocate + assert_eq!(abcd_ptr, a.as_ptr()); + assert_eq!(Bytes::copy_from_slice(b"bcd"), abcd); +} + +#[test] +#[should_panic] +fn take_copy_to_bytes_panics() { + let abcd = Bytes::copy_from_slice(b"abcd"); + abcd.take(2).copy_to_bytes(3); +} + +#[cfg(feature = "std")] +#[test] +fn take_chunks_vectored() { + fn chain() -> impl Buf { + Bytes::from([1, 2, 3].to_vec()).chain(Bytes::from([4, 5, 6].to_vec())) + } + + { + let mut dst = [std::io::IoSlice::new(&[]); 2]; + let take = chain().take(0); + assert_eq!(take.chunks_vectored(&mut dst), 0); + } + + { + let mut dst = [std::io::IoSlice::new(&[]); 2]; + let take = chain().take(1); + assert_eq!(take.chunks_vectored(&mut dst), 1); + assert_eq!(&*dst[0], &[1]); + } + + { + let mut dst = [std::io::IoSlice::new(&[]); 2]; + let take = chain().take(3); + assert_eq!(take.chunks_vectored(&mut dst), 1); + assert_eq!(&*dst[0], &[1, 2, 3]); + } + + { + let mut dst = [std::io::IoSlice::new(&[]); 2]; + let take = chain().take(4); + assert_eq!(take.chunks_vectored(&mut dst), 2); + assert_eq!(&*dst[0], &[1, 2, 3]); + assert_eq!(&*dst[1], &[4]); + } + + { + let mut dst = [std::io::IoSlice::new(&[]); 2]; + let take = chain().take(6); + assert_eq!(take.chunks_vectored(&mut dst), 2); + assert_eq!(&*dst[0], &[1, 2, 3]); + assert_eq!(&*dst[1], &[4, 5, 6]); + } + + { + let mut dst = [std::io::IoSlice::new(&[]); 2]; + let take = chain().take(7); + assert_eq!(take.chunks_vectored(&mut dst), 2); + assert_eq!(&*dst[0], &[1, 2, 3]); + assert_eq!(&*dst[1], &[4, 5, 6]); + } +} diff --git a/tools/README.md b/tools/README.md index 4621f1926..7fdaa281b 100644 --- a/tools/README.md +++ b/tools/README.md @@ -50,8 +50,9 @@ pixi run clean --runtime-only pixi run bash tools/test-metrics/measure.sh test-access test-console ``` -For test ownership and timings, read -[`doc/working/test.md`](../doc/working/test.md). CI calls component tasks; +For test ownership, inspect the component task and its checked CI map. +Measure timings with `test-metrics/measure.sh`; retained run results are +under `.crowdb-runtime/artifacts/measure-tests/`. CI calls component tasks; update the package groups in `pixi-tasks/test.py` and the checked CI map, then run `check-ci-test-tasks`. Feature-gated or ignored client tests need explicit task selection. diff --git a/tools/benchmark/bench-chunk-kv-regression.sh b/tools/benchmark/bench-chunk-kv-regression.sh index c3b06d510..c47af1b6d 100755 --- a/tools/benchmark/bench-chunk-kv-regression.sh +++ b/tools/benchmark/bench-chunk-kv-regression.sh @@ -63,7 +63,7 @@ if ! [[ "$READ_PERCENT" =~ ^[0-9]+$ ]] || [ "$READ_PERCENT" -gt 100 ]; then fi if [ "$SKIP_BUILD" -eq 0 ]; then pixi run build - pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server \ + pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server \ -p crowdb-diskdb -p crowdb-chunkdb -p crowdb-chunk-kv-server \ -p crowdb-chunk-kv-client fi diff --git a/tools/benchmark/bench-chunk-stream-regression.sh b/tools/benchmark/bench-chunk-stream-regression.sh index 1335ac967..d0175943a 100755 --- a/tools/benchmark/bench-chunk-stream-regression.sh +++ b/tools/benchmark/bench-chunk-stream-regression.sh @@ -199,7 +199,7 @@ def initialize(args): root = Path.cwd() binaries = [args.binary, os.environ["CROWDB_REGRESSION_FIXTURE_BIN"], *(root / "target/release" / name for name in - ("crowdb-cli", "crowdb-kv-server", "crowdb-diskdb", "crowdb-chunkdb")), + ("crowdb-cli", "crowdb-paxos-kv-server", "crowdb-diskdb", "crowdb-chunkdb")), root / "app/crowdb-diskio/build/crowdb-diskio"] value = dict(commit=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), dirty_paths=subprocess.check_output(["git", "status", "--short"], text=True).splitlines(), @@ -436,7 +436,7 @@ trap cleanup EXIT # identified executable directly, so a case cannot trigger a Cargo rebuild. if [ "$SKIP_BUILD" -eq 0 ]; then pixi run build - pixi run cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb + pixi run cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server -p crowdb-diskdb -p crowdb-chunkdb if [ -z "$BENCH_BIN" ]; then pixi run cargo bench -p crowdb-chunk-stream --bench production_bounds --no-run \ --message-format=json >"$LOG_ROOT/build.jsonl" diff --git a/tools/benchmark/bench-chunkdb-regression.sh b/tools/benchmark/bench-chunkdb-regression.sh index e6c88d788..b0ebea6c3 100755 --- a/tools/benchmark/bench-chunkdb-regression.sh +++ b/tools/benchmark/bench-chunkdb-regression.sh @@ -100,11 +100,11 @@ any_case_selected() { verify_logs() { local label="$1" kv_metrics diskdb_metrics chunkdb_metrics cli_metrics local kv_rpc diskdb_rpc chunkdb_rpc cli_rpc expected_servers expected_clients - kv_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-kv-server-metrics-*.log' -type f | wc -l) + kv_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-paxos-kv-server-metrics-*.log' -type f | wc -l) diskdb_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/diskdb-*/log/crowdb-diskdb-metrics-*.log' -type f | wc -l) chunkdb_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/chunkdb-*/log/crowdb-chunkdb-metrics-*.log' -type f | wc -l) cli_metrics=$(find "$LOG_ROOT" -path '*/measurements/bench-chunkdb-*/crowdb-cli-metrics-*.log' -type f | wc -l) - kv_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-kv-server-rpc-*.log' -type f | wc -l) + kv_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-paxos-kv-server-rpc-*.log' -type f | wc -l) diskdb_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/diskdb-*/log/crowdb-diskdb-rpc-*.log' -type f | wc -l) chunkdb_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/chunkdb-*/log/crowdb-chunkdb-rpc-*.log' -type f | wc -l) cli_rpc=$(find "$LOG_ROOT" -path '*/measurements/bench-chunkdb-*/crowdb-cli-rpc-*.log' -type f | wc -l) @@ -117,7 +117,7 @@ verify_logs() { echo "ERROR: incomplete logs for $label (kv=$kv_metrics/$kv_rpc diskdb=$diskdb_metrics/$diskdb_rpc chunkdb=$chunkdb_metrics/$chunkdb_rpc cli=$cli_metrics/$cli_rpc)" >&2 return 1 fi - regression_require_metric_files 'crowdb-kv-server-metrics-*.log' rust cpp-rpc cpp-tree || return 1 + regression_require_metric_files 'crowdb-paxos-kv-server-metrics-*.log' rust cpp-rpc cpp-tree || return 1 regression_require_metric_files 'crowdb-diskdb-metrics-*.log' rust cpp-rpc || return 1 regression_require_metric_files 'crowdb-chunkdb-metrics-*.log' rust cpp-rpc || return 1 echo " logs: kv=$kv_metrics/$kv_rpc diskdb=$diskdb_metrics/$diskdb_rpc chunkdb=$chunkdb_metrics/$chunkdb_rpc cli=$cli_metrics/$cli_rpc" @@ -194,7 +194,7 @@ run_case() { } echo "=== building release binaries ===" -pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb +pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server -p crowdb-diskdb -p crowdb-chunkdb mkdir -p "$LOG_ROOT" "$(dirname "$RESULTS_FILE")" regression_init printf 'Wl\tGrp\tThr\tStrip\tEC\tCli\tCdb\tDdb\tKv\tWkr\tWin\tCoal\tchunk/s\tblock/s\tavg\tp50\tp99\tDur\tErr\tStop\tSpc\n' >"$RESULTS_FILE" diff --git a/tools/benchmark/bench-chunkio-read-regression.sh b/tools/benchmark/bench-chunkio-read-regression.sh index a0411625c..6cbb3c98b 100755 --- a/tools/benchmark/bench-chunkio-read-regression.sh +++ b/tools/benchmark/bench-chunkio-read-regression.sh @@ -149,7 +149,7 @@ run_case() { if [ "$SKIP_BUILD" != 1 ]; then echo "=== building release binaries ===" - pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb + pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server -p crowdb-diskdb -p crowdb-chunkdb pixi run build fi mkdir -p "$LOG_ROOT" diff --git a/tools/benchmark/bench-chunkio-small-write-regression.sh b/tools/benchmark/bench-chunkio-small-write-regression.sh index a19663b47..4e48a34ff 100755 --- a/tools/benchmark/bench-chunkio-small-write-regression.sh +++ b/tools/benchmark/bench-chunkio-small-write-regression.sh @@ -176,7 +176,7 @@ run_case() { if [ "$SKIP_BUILD" != 1 ]; then echo "=== building release binaries ===" - pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb + pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server -p crowdb-diskdb -p crowdb-chunkdb pixi run build fi mkdir -p "$LOG_ROOT" diff --git a/tools/benchmark/bench-chunkio-write-regression.sh b/tools/benchmark/bench-chunkio-write-regression.sh index 3512d1827..e4c5f51d6 100755 --- a/tools/benchmark/bench-chunkio-write-regression.sh +++ b/tools/benchmark/bench-chunkio-write-regression.sh @@ -78,7 +78,7 @@ memory_bandwidth() { verify_logs() { local kv diskdb chunkdb diskio cli_metrics cli_metrics_file - kv=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-kv-server-metrics-*.log' | wc -l) + kv=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-paxos-kv-server-metrics-*.log' | wc -l) diskdb=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-diskdb-metrics-*.log' | wc -l) chunkdb=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-chunkdb-metrics-*.log' | wc -l) diskio=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-diskio-metrics-*.log' | wc -l) @@ -90,7 +90,7 @@ verify_logs() { && grep -Eq -- 'chunkio\.object\.write\.e2e\.lh' "$cli_metrics_file" \ && grep -Eq -- 'chunkio\.chunk\.allocate\.e2e\.lh' "$cli_metrics_file" \ && grep -Eq -- 'chunkio\.diskio\.write\.e2e\.lh' "$cli_metrics_file" \ - && regression_require_metric_files 'crowdb-kv-server-metrics-*.log' rust cpp-rpc cpp-tree \ + && regression_require_metric_files 'crowdb-paxos-kv-server-metrics-*.log' rust cpp-rpc cpp-tree \ && regression_require_metric_files 'crowdb-diskdb-metrics-*.log' rust cpp-rpc \ && regression_require_metric_files 'crowdb-chunkdb-metrics-*.log' rust cpp-rpc \ && regression_require_metric_files 'crowdb-diskio-metrics-*.log' cpp-rpc \ @@ -186,7 +186,7 @@ run_case() { if [ "$SKIP_BUILD" != 1 ]; then echo "=== building release binaries ===" - pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb + pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server -p crowdb-diskdb -p crowdb-chunkdb pixi run build fi mkdir -p "$LOG_ROOT" "$(dirname "$RESULTS_FILE")" diff --git a/tools/benchmark/bench-diskdb-regression.sh b/tools/benchmark/bench-diskdb-regression.sh index 0d4db3b2f..879d2c511 100755 --- a/tools/benchmark/bench-diskdb-regression.sh +++ b/tools/benchmark/bench-diskdb-regression.sh @@ -149,10 +149,10 @@ trap destroy_cluster EXIT verify_logs() { local label="$1" kv_metrics diskdb_metrics cli_metrics kv_rpc diskdb_rpc cli_rpc - kv_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-kv-server-metrics-*.log' -type f | wc -l) + kv_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-paxos-kv-server-metrics-*.log' -type f | wc -l) diskdb_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/diskdb-*/log/crowdb-diskdb-metrics-*.log' -type f | wc -l) cli_metrics=$(find "$LOG_ROOT" -path '*/measurements/bench-diskdb-*/crowdb-cli-metrics-*.log' -type f | wc -l) - kv_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-kv-server-rpc-*.log' -type f | wc -l) + kv_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-paxos-kv-server-rpc-*.log' -type f | wc -l) diskdb_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/diskdb-*/log/crowdb-diskdb-rpc-*.log' -type f | wc -l) cli_rpc=$(find "$LOG_ROOT" -path '*/measurements/bench-diskdb-*/crowdb-cli-rpc-*.log' -type f | wc -l) local expected_servers=$((CASE_NUMBER * 3)) expected_clients="$CASE_NUMBER" @@ -162,7 +162,7 @@ verify_logs() { echo "ERROR: incomplete logs for $label (kv=$kv_metrics/$kv_rpc diskdb=$diskdb_metrics/$diskdb_rpc cli=$cli_metrics/$cli_rpc)" >&2 return 1 fi - regression_require_metric_files 'crowdb-kv-server-metrics-*.log' rust cpp-rpc cpp-tree || return 1 + regression_require_metric_files 'crowdb-paxos-kv-server-metrics-*.log' rust cpp-rpc cpp-tree || return 1 regression_require_metric_files 'crowdb-diskdb-metrics-*.log' rust cpp-rpc || return 1 echo " logs: kv=$kv_metrics/$kv_rpc diskdb=$diskdb_metrics/$diskdb_rpc cli=$cli_metrics/$cli_rpc root=$LOG_ROOT" } @@ -213,7 +213,7 @@ deploy_group() { regression_fixture setup -- --nodes 3 --kind kv "${backend_args[@]}" "${kv_args[@]}" local groups=() for group in $(seq 1 "$DATA_GROUP_COUNT"); do - cli kv group add -s 0 -g "$group" -n 1,2,3 + cli paxos-kv group add -s 0 -g "$group" -n 1,2,3 groups+=("$group") done local group_csv @@ -316,7 +316,7 @@ run_case() { } echo "=== building release binaries ===" -pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb +pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server -p crowdb-diskdb regression_init mkdir -p "$LOG_ROOT" "$(dirname "$RESULTS_FILE")" printf 'Wl\tGrp\tThr\tBlk\tCli\tDdb\tKv\tWkr\tWin\tCoal\tops/s\tavg\tp50\tp99\tDur\tErr\tSpc\tFB\tFReq\tFKV\tFCo\n' >"$RESULTS_FILE" diff --git a/tools/benchmark/bench-kv-common.sh b/tools/benchmark/bench-kv-common.sh new file mode 100644 index 000000000..c934d66e1 --- /dev/null +++ b/tools/benchmark/bench-kv-common.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# Workload driver only: deployment is external, all accounting belongs to CLI. +set -euo pipefail +# Match the baseline runtime to available logical CPUs, independently of RPC workers. +export TOKIO_WORKER_THREADS="$(python -c 'import os; print(len(os.sched_getaffinity(0)))')" +CLI="${KV_BENCH_CLI:-$PWD/target/release/crowdb-cli}" +: "${KV_BENCH_DEPLOYMENT:?set KV_BENCH_DEPLOYMENT to the prestarted deployment state JSON}" +LOG_ROOT="${KV_BENCH_LOG_ROOT:-$PWD/.crowdb-runtime/artifacts/bench/kv-$(date +%Y%m%d-%H%M%S)-$$}" +mkdir -p "$LOG_ROOT" +DURATION="${KV_BENCH_DURATION:-20}" +WARMUP="${KV_BENCH_WARMUP:-5}" +TAIL="${KV_BENCH_TAIL:-5}" +KEYSPACE="${KV_BENCH_KEYSPACE:-1000000}" +CASES="${KV_BENCH_CASES:-}" +selected_cases=0 +known_cases="" +last_dataset="" +# Optional observer (perf/samply); it wraps the client, never the deployment. +BENCH_PREFIX=() + +selected() { + known_cases+=" $1" + if [[ -n "$CASES" && " $CASES " != *" $1 "* ]]; then return 1; fi + selected_cases=$((selected_cases + 1)) +} + +load_deployment() { + local state="$1" + mapfile -t endpoints < <(jq -er '.endpoints[]' "$state") + deployment=() + if [[ $(jq -r '.kind // "container"' "$state") == native-process ]]; then + mapfile -t pids < <(jq -er '.pids[]' "$state") + [[ ${#endpoints[@]} == 3 && ${#pids[@]} == 3 ]] || { echo 'Expected three native PIDs/endpoints' >&2; exit 2; } + for i in 0 1 2; do deployment+=(--endpoint "${endpoints[$i]}" --server-pid "${pids[$i]}"); done + else + mapfile -t containers < <(jq -er '.containers[]' "$state") + [[ ${#endpoints[@]} == 3 && ${#containers[@]} == 3 ]] || { echo 'Expected three containers/endpoints' >&2; exit 2; } + for i in 0 1 2; do deployment+=(--endpoint "${endpoints[$i]}" --container "${containers[$i]}"); done + fi +} + +invoke() { + local label="$1" operation="$2"; shift 2 + local output="$LOG_ROOT/$label.json" + echo ">>> $label ($operation): warmup=$WARMUP measurement=$DURATION tail=$TAIL" + if ! CROWDB_RUNTIME_ROOT="$LOG_ROOT/$label-runtime" "${BENCH_PREFIX[@]}" "$CLI" bench paxos-kv "$operation" \ + "${deployment[@]}" --label "$label" --warmup-secs "$WARMUP" \ + --duration-secs "$DURATION" --tail-secs "$TAIL" \ + --max-dataset-bytes "${KV_BENCH_MAX_DATASET_BYTES:-2147483648}" --output "$output" "$@" >"$LOG_ROOT/$label.stdout" 2>"$LOG_ROOT/$label.stderr"; then + cat "$LOG_ROOT/$label.stdout" "$LOG_ROOT/$label.stderr" >&2 + [[ ! -f "$output" ]] || cat "$output" >&2 + return 1 + fi + if ! jq -e '.passed == true' "$output" >/dev/null; then + cat "$LOG_ROOT/$label.stdout" "$LOG_ROOT/$label.stderr" >&2 + echo "Missing or failed benchmark report: $output" >&2 + return 1 + fi + jq -r '.history_line // "dataset prepared and verified"' "$output" | tee -a "$LOG_ROOT/history-comments.txt" +} + +prepare_dataset() { + local signature="$KV_BENCH_DEPLOYMENT/$KEYSPACE/$*" + if [[ "$signature" != "$last_dataset" ]]; then + invoke "prepare-$selected_cases" prepare --key-space "$KEYSPACE" --loader-num 32 "$@" + last_dataset="$signature" + fi +} + +finish_cases() { + for name in $CASES; do + [[ " $known_cases " == *" $name "* ]] || { echo "Unknown case: $name" >&2; exit 2; } + done + [[ "$selected_cases" -gt 0 ]] || { echo 'No selected cases' >&2; exit 2; } + echo "Reports: $LOG_ROOT; append dated results and conditions to the calling .sh history." +} +load_deployment "$KV_BENCH_DEPLOYMENT" diff --git a/tools/benchmark/bench-kv-read-regression.sh b/tools/benchmark/bench-kv-read-regression.sh index ab8933859..cbd0bbe5a 100755 --- a/tools/benchmark/bench-kv-read-regression.sh +++ b/tools/benchmark/bench-kv-read-regression.sh @@ -1,104 +1,35 @@ #!/usr/bin/env bash -# --- CrowDB read regression benchmark (lifecycle) --- -# Usage: bash tools/benchmark/bench-kv-read-regression.sh -# -# Regression sentinel for point-read (get) throughput and latency. Uses -# the `bench deploy` / `bench prepare` / `bench run` / `bench teardown` -# lifecycle: deploy once, prepare once, run N sub-tests, teardown once. -# This amortizes deploy + pre-pop overhead across all sub-tests. -# -# Configurations (all --workload read, mem mode, 3-node cluster): -# Single-thread (1T:1C) — isolate per-read engine cost: -# - lin_1t: linearizable (lease barrier + engine get) -# - minslot_1t: minslot (no barrier, local serve) -# Multi-thread — max throughput + read-mode split: -# - lin_6t: linearizable mid-concurrency -# - minslot_6t: minslot (reads distributed across replicas) -# - lin_16t: linearizable high concurrency -# - minslot_16t: minslot high concurrency -# - lin_32t: linearizable saturation -# - minslot_32t: minslot saturation -# Correctness verification (--verify-bytes 8): -# - lin_16t_verify: linearizable correctness -# - minslot_16t_verify: minslot correctness -# -# 10 runs × 20s ≈ 200s (no deploy/pre-pop overhead per sub-test). -# -# Reference platform: see doc/design/kv/kv-read-flow-analysis.md. After -# a run, update the "Latest Benchmark Results" section there with the -# results and CPU model — absolute read throughput is platform-dependent. -# -# Prerequisites: -# - pixi installed, project dependencies resolved -# - jq installed -# - release binary built (pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server) +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# Paxos-KV read: externally started 3-node host-network containers, mem-block. +# Run with pixi run bash tools/benchmark/bench-kv-read-regression.sh. +# Set KV_BENCH_DEPLOYMENT; optional KV_BENCH_CASES, DURATION, WARMUP, TAIL, +# KEYSPACE, CLI and LOG_ROOT use the KV_BENCH_ prefix. +# Defaults: 5s warmup / 20s measurement / 5s tail. <20s is a smoke run. +# Keep every dated result, including failures. Old native process data below +# remains a useful reference; compare code, resource and dataset conditions. set -euo pipefail cd "$(dirname "$0")/../.." +source tools/benchmark/bench-kv-common.sh -RUN_STAMP=$(date +%Y%m%d-%H%M%S) -LOG_ROOT="${KV_READ_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/kv-read-regression-$RUN_STAMP}" -RESULTS_FILE="${KV_READ_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" -REGRESSION_LOG_ROOT="$LOG_ROOT" -source tools/benchmark/bench-regression-common.sh -export CROWDB_LOG_ROOT="$LOG_ROOT" -regression_init -DURATION="${KV_READ_BENCH_DURATION:-20}" -KEYSPACE="${KV_READ_BENCH_KEYSPACE:-100000}" -CASES="${KV_READ_BENCH_CASES:-}" -DEPLOY_NAME="kv-read-regression-$$" - -run_subtest() { - local label="$1" read_mode="$2" min_slot="$3" threads="$4" connections="$5" verify_bytes="${6:-0}" - if [ -n "$CASES" ] && [[ " $CASES " != *" $label "* ]]; then - return - fi - local read_endpoint - if [ "$read_mode" = "minslot" ]; then - read_endpoint="any-replica" - else - read_endpoint="leader" - fi - local display="$label (${threads}T:${connections}C)" - if [ "$verify_bytes" -gt 0 ]; then - display="$display verify" - fi - echo ">>> $display ..." - local output - if ! output=$(regression_client_for "$CONFIG_FILE" \ - bench kv read --store 0 --group 1 --duration-secs "$DURATION" \ - --loader-num "$threads" --connections "$connections" \ - --read-mode "$read_mode" --min-slot "$min_slot" \ - --read-endpoint-policy "$read_endpoint" \ - --key-space "$KEYSPACE" \ - --value-size 64 --verify-bytes "$verify_bytes" 2>&1); then - echo "$output" >&2 - return 1 - fi - local json; json=$(regression_report_for "$CONFIG_FILE") - if [ -z "$json" ]; then - echo " ERROR: no JSON output"; echo "$output" - echo -e "$label\t$read_mode\t${threads}T${connections}C\t$verify_bytes\t0\t0\t0\t0\t0\t0\t1\t1" >> "$RESULTS_FILE" - return - fi - local ops_s avg_us p50_us p99_us p999_us errors corr_err - ops_s=$(echo "$json" | jq -r '.total_ops * 1000 / .duration_ms' | awk '{printf "%.0f", $1}') - avg_us=$(echo "$json" | jq -r '.by_op.read.latency_us.avg_us') - p50_us=$(echo "$json" | jq -r '.by_op.read.latency_us.p50_us') - p99_us=$(echo "$json" | jq -r '.by_op.read.latency_us.p99_us') - p999_us=$(echo "$json" | jq -r '.by_op.read.latency_us.p999_us // "unavailable"') - errors=$(echo "$json" | jq -r '.total_errors') - corr_err=$(echo "$json" | jq -r '.correctness_errors') - echo " ops/s=$ops_s avg=${avg_us}us p50=${p50_us}us p99=${p99_us}us p999=${p999_us}us err=$errors corr=$corr_err" - echo -e "$label\t$read_mode\t${threads}T${connections}C\t$verify_bytes\t$ops_s\t$avg_us\t$p50_us\t$p99_us\t$p999_us\t$errors\t$corr_err" >> "$RESULTS_FILE" +run_case() { + local label="$1" threads="$2" mode="$3" policy=leader + selected "$label" || return 0 + [[ "$mode" != minslot ]] || policy=any-replica + prepare_dataset --value-size 64 + invoke "$label" read --key-space "$KEYSPACE" --value-size 64 --loader-num "$threads" \ + --connections "$threads" --read-mode "$mode" --read-endpoint-policy "$policy" } +for threads in 1 6 16 32; do + run_case "lin_${threads}t" "$threads" linearizable + run_case "minslot_${threads}t" "$threads" minslot +done +# All cases now verify complete dataset bytes outside timing; keep these names. +run_case lin_16t_verify 16 linearizable +run_case minslot_16t_verify 16 minslot +finish_cases -# --- regression sentinel configs --- -# -# Regression policy: only update the reference table below when a new -# run is strictly better (higher ops/s, lower latency, fewer errors). -# If a run is worse, do NOT update — investigate and fix the regression -# first, otherwise silent performance regressions slip in. -# +# ---- Native process history (no containers) ---- # Reference results (2026-08-19, Apple M5 Pro, 18c, arm64, macOS 26.5): # 10s mem mode, 3-node cluster, 100k pre-populated keys, 64B values. # @@ -156,45 +87,55 @@ run_subtest() { # lin_16t_verify linearizable 16:16 168670 90 85 175 0 corr=0 # minslot_16t_verify minslot 16:16 207835 73 70 129 0 corr=0 -echo -e "label\tread_mode\tT:C\tverify\tops_s\tavg_us\tp50_us\tp99_us\tp999_us\terrors\tcorrectness_errors" > "$RESULTS_FILE" - -# Phase 1: deploy the cluster once via the explicit test fixture. -echo "=== Deploying 3-node KV cluster via the explicit test fixture ===" -CONFIG_FILE="$REGRESSION_CONFIG" -regression_fixture_for "$CONFIG_FILE" setup -- --nodes 3 --kind kv \ - --kv-backend mem-block --wal-backend mem-block - -# Create bench group 1 (group 0 sysdata preserved). -echo "=== Creating bench group 0/1 (group 0 sysdata preserved) ===" -regression_client_for "$CONFIG_FILE" \ - kv group add -s 0 -g 1 -n 1,2,3 2>&1 - -# Phase 2: pre-populate keys once (prepare data before measuring). -echo "=== Pre-populating $KEYSPACE keys ===" -regression_client_for "$CONFIG_FILE" \ - bench kv prepare --store 0 --group 1 --keys "$KEYSPACE" --value-size 64 2>&1 +# ---- Host-network container history ---- +# 2026-10-27: AMD Ryzen 9 5950X, 32 logical CPUs, ~62GiB host RAM. +# 3 containers, 8GiB/4 CPU quota each, no cpuset; mem-block Tree + WAL; +# server workers=2, inflight=32/coalesce=16, peer pool=4, event-write. +# Release host CLI, 5/60/5s warmup/measurement/tail, 100k keys x 64B. +# Image: crowdb-node:ci253-shared, sha256:89532eddadf085c527116a1986bb67149017eaeca381abbcae89b501d1c9abb1 +# Revision: ed0447797d137512f0bdf2fd3db1c46e22804bdc; source tree: ea1c3256e20ae90140bd95e727ab87c748b3a033. +# Before port consolidation: mgmt/RPC 10000/10100, offsets 0/1000/2000. +# Raw reports: .crowdb-runtime/artifacts/bench/kv-host-read/ (includes CLI SHA256). +# label ops/s avg_us p50_us p99_us p999_us errors/incomplete +# lin_1t 15673.817 63.700 60.159 109.567 135.167 0/0 +# minslot_1t 10799.117 92.483 89.599 150.527 187.391 0/0 +# lin_16t 108878.550 146.824 144.383 251.903 299.007 0/0 +# minslot_16t 180776.800 88.296 86.527 139.263 185.343 0/0 +# lin_16t is ~53% below the Sep-02 native reference. Conditions/code changed; +# no cgroup CPU throttling observed. Cause remains open; do not blame host networking. -# Phase 3: run all sub-tests against the same cluster. -echo "=== Single-thread (1T:1C) — per-read engine cost ===" -run_subtest "lin_1t" linearizable auto 1 1 -run_subtest "minslot_1t" minslot zero 1 1 +# 2026-10-27: 1M x 64B read baseline, 5/20/5s (new standard). +# Same host, three 8GiB/4-CPU containers, no cpuset; workers=2, inflight/coalesce=32/16, +# peer pool=4, event-write, mem-block Tree/WAL. Default ports + offsets 0/1/2. +# Image: sha256:eb0dd67818e3f3ec2f81ba6969db8320f2dcd8a9c92c1bf727d1d1f7cce9d09d +# Source tree: b984ab3e0838bee528d880e6bb83fc34c3c54663. +# Raw reports + CLI checksum: .crowdb-runtime/artifacts/bench/kv-review-read/ +# All zero errors/incomplete. Dataset differs from preceding 100k-key read history. +# 2026-10-27 lin_1t host-container ops/s=15305.9 latency_ns={"avg":65233.4343096453,"count":306118,"max":678089,"min":48201,"p50":64255,"p99":87039,"p999":105471} passed=true +# 2026-10-27 minslot_1t host-container ops/s=12233.05 latency_ns={"avg":81621.3496348008,"count":244661,"max":1754642,"min":55010,"p50":80895,"p99":113663,"p999":151551} passed=true +# 2026-10-27 lin_16t host-container ops/s=76390.9 latency_ns={"avg":209298.92435879144,"count":1527818,"max":18931996,"min":41271,"p50":167935,"p99":428031,"p999":15925247} passed=true +# 2026-10-27 minslot_16t host-container ops/s=137165.5 latency_ns={"avg":116380.78411116498,"count":2743310,"max":14660221,"min":41411,"p50":99839,"p99":352255,"p999":1318911} passed=true -echo "=== Multi-thread — max throughput + read-mode split ===" -run_subtest "lin_6t" linearizable auto 6 6 -run_subtest "minslot_6t" minslot zero 6 6 -run_subtest "lin_16t" linearizable auto 16 16 -run_subtest "minslot_16t" minslot zero 16 16 -run_subtest "lin_32t" linearizable auto 32 32 -run_subtest "minslot_32t" minslot zero 32 32 +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/read/; full verification passed. +# 2026-10-27 lin_16t host-container ops/s=103176.05 latency_ns={"avg":154955.89959830794,"count":2063521,"max":2919434,"min":45220,"p50":153599,"p99":247807,"p999":319487} passed=true -echo "=== Correctness verification ===" -run_subtest "lin_16t_verify" linearizable auto 16 16 8 -run_subtest "minslot_16t_verify" minslot zero 16 16 8 +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/read/; full verification passed. +# 2026-10-27 lin_1t host-container ops/s=13340.1 latency_ns={"avg":74842.10373235583,"count":266802,"max":2234769,"min":48370,"p50":74751,"p99":106495,"p999":144383} passed=true -# Phase 4: teardown via owned fixture shutdown. -echo "=== Tearing down cluster ===" -regression_fixture_for "$CONFIG_FILE" stop +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/read/; full verification passed. +# 2026-10-27 minslot_16t host-container ops/s=134152.2 latency_ns={"avg":118881.20723253141,"count":2683044,"max":2860484,"min":46610,"p50":114175,"p99":223231,"p999":397311} passed=true -echo "=== DONE ===" -echo "Results in $RESULTS_FILE" -column -t -s$'\t' "$RESULTS_FILE" +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/read/; full verification passed. +# 2026-10-27 minslot_1t host-container ops/s=11314.6 latency_ns={"avg":88249.34428525975,"count":226292,"max":1078029,"min":51920,"p50":84479,"p99":154623,"p999":199679} passed=true diff --git a/tools/benchmark/bench-kv-scan-regression.sh b/tools/benchmark/bench-kv-scan-regression.sh index 49d83c167..369d482e8 100755 --- a/tools/benchmark/bench-kv-scan-regression.sh +++ b/tools/benchmark/bench-kv-scan-regression.sh @@ -1,112 +1,53 @@ #!/usr/bin/env bash -# --- CrowDB scan regression benchmark (lifecycle) --- -# Usage: bash tools/benchmark/bench-kv-scan-regression.sh -# -# Regression sentinel for scan (list) throughput and latency. Uses the -# `bench deploy` / `bench prepare` / `bench run` / `bench teardown` -# lifecycle: deploy once, prepare once, run N sub-tests, teardown once. -# -# Configurations (all --workload list, mem mode, 3-node cluster): -# Single-thread (1T:1C) — isolate per-scan engine cost: -# - bounded_10: O(limit) fast path (limit=10) -# - bounded_1k: typical scan width (limit=1000, 64B, linearizable) -# - bounded_10k: large bounded scan -# - full_100k: full keyspace + byte-budget pagination -# - deep_pag_10: O(limit) pushdown proof (start_after near end) -# - mixed_1k: mixed value sizes (64B:70%,1KiB:20%,16KiB:10%) -# - minslot_1k: MinSlot routing (vs bounded_1k's linearizable) -# - largeval_16k: 16KiB values (R67 regression: snapshot stall) -# Multi-thread (4T:4C) — max throughput + read-mode split: -# - lin_4t: linearizable (all scans serialize on leader) -# - minslot_4t: minslot (scans distributed across replicas) -# -# 14 runs × 20s ≈ 280s (no deploy/pre-pop overhead per sub-test). -# -# Reference platform: see doc/design/kv/kv-scan-flow-analysis.md. After -# a run, update the "Latest Benchmark Results" section there with the -# results and CPU model — absolute scan throughput is platform-dependent. -# -# Prerequisites: -# - pixi installed, project dependencies resolved -# - jq installed -# - release binary built (pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server) +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# Paxos-KV scan: externally started 3-node host-network containers, mem-block. +# Run with pixi run bash tools/benchmark/bench-kv-scan-regression.sh. +# Set KV_BENCH_DEPLOYMENT; optional KV_BENCH_CASES, DURATION, WARMUP, TAIL, +# KEYSPACE, CLI and LOG_ROOT use the KV_BENCH_ prefix. +# Defaults: 5s warmup / 20s measurement / 5s tail. <20s is a smoke run. +# Keep every dated result, including failures. Old native process data below +# remains a useful reference; compare code, resource and dataset conditions. set -euo pipefail cd "$(dirname "$0")/../.." +source tools/benchmark/bench-kv-common.sh -RUN_STAMP=$(date +%Y%m%d-%H%M%S) -LOG_ROOT="${KV_SCAN_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/kv-scan-regression-$RUN_STAMP}" -RESULTS_FILE="${KV_SCAN_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" -REGRESSION_LOG_ROOT="$LOG_ROOT" -source tools/benchmark/bench-regression-common.sh -export CROWDB_LOG_ROOT="$LOG_ROOT" -regression_init -DURATION="${KV_SCAN_BENCH_DURATION:-20}" -KEYSPACE="${KV_SCAN_BENCH_KEYSPACE:-100000}" -CASES="${KV_SCAN_BENCH_CASES:-}" -DEPLOY_NAME="kv-scan-regression-$$" - -# Keys are k{id:020} (zero-padded to 20 digits after 'k'), per -# workload.rs format_key. start_after is an exclusive lower bound, so -# to return the last N keys of a keyspace [0, KEYSPACE), set -# start_after = k{KEYSPACE - N - 1:020}. -pad_key() { - local id="$1" - printf "k%020d" "$id" -} - -run_subtest() { - local label="$1" limit="$2" prefix="$3" start_after="$4" value_size="$5" read_mode="$6" min_slot="$7" threads="$8" connections="$9" direction="${10}" mix="${11:-}" - if [ -n "$CASES" ] && [[ " $CASES " != *" $label "* ]]; then - return +run_case() { + local label="$1" limit="$2" threads="$3" mode="$4" size="$5"; shift 5 + selected "$label" || return 0 + local policy=leader mix=() KEYSPACE="$KEYSPACE" + if [[ "$label" == largeval_16k || "$label" == mixed_1k ]]; then + KEYSPACE="${KV_BENCH_LARGE_KEYSPACE:-100000}" fi - local read_endpoint - if [ "$read_mode" = "minslot" ]; then - read_endpoint="any-replica" - else - read_endpoint="leader" - fi - local mix_arg="" - if [ -n "$mix" ]; then - mix_arg="--value-size-mix $mix" - fi - echo ">>> $label (${threads}T:${connections}C) ..." - local output - if ! output=$(regression_client_for "$CONFIG_FILE" \ - bench kv scan --store 0 --group 1 --duration-secs "$DURATION" \ - --loader-num "$threads" --connections "$connections" \ - --read-mode "$read_mode" --min-slot "$min_slot" \ - --read-endpoint-policy "$read_endpoint" \ - --scan-limit "$limit" --scan-prefix "$prefix" --scan-start-after "$start_after" \ - --scan-direction "$direction" \ - --value-size "$value_size" \ - --key-space "$KEYSPACE" --verify-bytes 0 $mix_arg 2>&1); then - echo "$output" >&2 - return 1 - fi - local json; json=$(regression_report_for "$CONFIG_FILE") - if [ -z "$json" ]; then - echo " ERROR: no JSON output"; echo "$output" - echo -e "$label\t$limit\t$prefix\t$start_after\t$value_size\t$read_mode\t$direction\t${threads}T${connections}C\t0\t0\t0\t0\t0\t1" >> "$RESULTS_FILE" - return - fi - local ops_s avg_us p50_us p99_us p999_us errors - ops_s=$(echo "$json" | jq -r '.total_ops * 1000 / .duration_ms' | awk '{printf "%.0f", $1}') - avg_us=$(echo "$json" | jq -r '.by_op.list.latency_us.avg_us') - p50_us=$(echo "$json" | jq -r '.by_op.list.latency_us.p50_us') - p99_us=$(echo "$json" | jq -r '.by_op.list.latency_us.p99_us') - p999_us=$(echo "$json" | jq -r '.by_op.list.latency_us.p999_us // "unavailable"') - errors=$(echo "$json" | jq -r '.total_errors') - echo " scans/s=$ops_s avg=${avg_us}us p50=${p50_us}us p99=${p99_us}us p999=${p999_us}us err=$errors" - echo -e "$label\t$limit\t$prefix\t$start_after\t$value_size\t$read_mode\t$direction\t${threads}T${connections}C\t$ops_s\t$avg_us\t$p50_us\t$p99_us\t$p999_us\t$errors" >> "$RESULTS_FILE" + [[ "$mode" != minslot ]] || policy=any-replica + [[ "$label" != mixed_1k ]] || mix=(--mixed-values) + prepare_dataset --value-size "$size" "${mix[@]}" + invoke "$label" scan --key-space "$KEYSPACE" --scan-limit "$limit" --value-size "$size" \ + --loader-num "$threads" --connections "$threads" --read-mode "$mode" \ + --read-endpoint-policy "$policy" "${mix[@]}" "$@" } +run_case bounded_10 10 1 linearizable 64 +run_case bounded_1k 1000 1 linearizable 64 +run_case bounded_10k 10000 1 linearizable 64 +run_case full_keyspace "$KEYSPACE" 1 linearizable 64 +[[ "$KEYSPACE" -ge 11 ]] || { echo 'scan requires KEYSPACE >=11' >&2; exit 2; } +printf -v deep_key 'k%020d' "$((KEYSPACE - 11))" +printf -v reverse_key 'k%020d' 10 +run_case deep_pag_10 10 1 linearizable 64 --scan-start-after "$deep_key" +run_case mixed_1k 1000 1 linearizable 64 +run_case minslot_1k 1000 1 minslot 64 +run_case reverse_1k 1000 1 linearizable 64 --reverse +run_case reverse_deep_10 10 1 linearizable 64 --reverse --scan-start-after "$reverse_key" +# Actually populate 16KiB values; historical runs below used 64B prepopulation. +run_case largeval_16k 1000 1 linearizable 16384 +for threads in 4 16 32; do + run_case "lin_${threads}t" 1000 "$threads" linearizable 64 + run_case "minslot_${threads}t" 1000 "$threads" minslot 64 +done +run_case reverse_4t 1000 4 minslot 64 --reverse +finish_cases -# --- regression sentinel configs --- -# -# Regression policy: only update the reference table below when a new -# run is strictly better (higher scans/s, lower latency, fewer errors). -# If a run is worse, do NOT update — investigate and fix the regression -# first, otherwise silent performance regressions slip in. -# +# ---- Native process history (no containers) ---- # Reference results (2026-08-19, Apple M5 Pro, 18c, arm64, macOS 26.5): # 10s mem mode, 3-node cluster, 100k pre-populated keys, 64B values # unless noted. Post-R67 (spawn_blocking maintenance). @@ -173,60 +114,81 @@ run_subtest() { # lin_32t 1000 32:32 22820 1342 1327 2072 0 # minslot_32t 1000 32:32 22823 1341 1318 2088 0 +0.0% vs lin -echo -e "label\tlimit\tprefix\tstart_after\tvalue_size\tread_mode\tdirection\tT:C\tscans_s\tavg_us\tp50_us\tp99_us\tp999_us\terrors" > "$RESULTS_FILE" +# ---- Host-network container history ---- +# 2026-10-27: AMD 5950X 32 logical CPUs; 3 x 8GiB/4-CPU containers, no cpuset; +# workers=2, window/coalesce=32/16, peer pool=4, event-write, default election profile. +# mem-block Tree/WAL, host network, management/RPC 10135/11135 + offsets 0/1/2. +# 1M x 64B, 5/20/5s; same deployment after read matrix and a new dataset prefill. +# Image: sha256:eb0dd67818e3f3ec2f81ba6969db8320f2dcd8a9c92c1bf727d1d1f7cce9d09d +# Source tree: b984ab3e0838bee528d880e6bb83fc34c3c54663. +# Raw reports: .crowdb-runtime/artifacts/bench/kv-review-scan/; server logs in kv-review-read/. +# All zero errors/incomplete; full_keyspace now means 1M keys, unlike historical full_100k. +# 2026-10-27 bounded_10 host-container ops/s=11496.8 latency_ns={"avg":86853.30406286966,"count":229936,"max":5065726,"min":56421,"p50":83967,"p99":139263,"p999":241663} passed=true +# 2026-10-27 bounded_1k host-container ops/s=1361.15 latency_ns={"avg":734435.0102119531,"count":27223,"max":2934998,"min":601728,"p50":708607,"p99":983039,"p999":1327103} passed=true +# 2026-10-27 full_keyspace host-container ops/s=0.55 latency_ns={"avg":1659944006.909091,"count":11,"max":1877296928,"min":1406636784,"p50":1635778559,"p99":1877296928,"p999":1877296928} passed=true +# 2026-10-27 deep_pag_10 host-container ops/s=12500.65 latency_ns={"avg":79858.59522104851,"count":250013,"max":1251827,"min":54341,"p50":77823,"p99":127999,"p999":202751} passed=true +# 2026-10-27 minslot_1k host-container ops/s=778.65 latency_ns={"avg":1283793.9630771207,"count":15573,"max":3995902,"min":568707,"p50":1236991,"p99":2752511,"p999":3407871} passed=true +# 2026-10-27 reverse_1k host-container ops/s=370.05 latency_ns={"avg":2701686.825564113,"count":7401,"max":321203070,"min":2276460,"p50":2490367,"p99":4390911,"p999":4947967} passed=true +# 2026-10-27 reverse_deep_10 host-container ops/s=6156.7 latency_ns={"avg":162273.8791560414,"count":123134,"max":1255176,"min":123632,"p50":160767,"p99":216063,"p999":245759} passed=true + +# 2026-10-27: mixed + large-value scan, fresh 3 x 12GiB/4-CPU containers; +# otherwise same image/server config above; 5/20/5s. 100k keys for these cases. +# mixed: actual 70% 64B / 20% 1KiB / 10% 16KiB; largeval: actual 16KiB throughout. +# Historical largeval rows with 64B prepopulation are not comparable as 16KiB workloads. +# Raw reports + logs + CLI checksum: .crowdb-runtime/artifacts/bench/kv-review-scan-large/ +# 2026-10-27 mixed_1k host-container ops/s=224.3 latency_ns={"avg":4456280.788898796,"count":4486,"max":13092660,"min":1418798,"p50":4063231,"p99":11206655,"p999":12451839} passed=true +# 2026-10-27 largeval_16k host-container ops/s=31.55 latency_ns={"avg":31667539.259904914,"count":631,"max":40593476,"min":24946544,"p50":31850495,"p99":37486591,"p999":40593476} passed=true + +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan/; full verification passed. +# 2026-10-27 bounded_10 host-container ops/s=13687.45 latency_ns={"avg":72957.76122287205,"count":273749,"max":1142009,"min":52030,"p50":71679,"p99":102399,"p999":130559} passed=true -# Phase 1: deploy the cluster once via the explicit test fixture. -echo "=== Deploying 3-node KV cluster via the explicit test fixture ===" -CONFIG_FILE="$REGRESSION_CONFIG" -regression_fixture_for "$CONFIG_FILE" setup -- --nodes 3 --kind kv \ - --kv-backend mem-block --wal-backend mem-block +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan/; full verification passed. +# 2026-10-27 bounded_1k host-container ops/s=1404.55 latency_ns={"avg":711771.9990744366,"count":28091,"max":1835865,"min":501384,"p50":708607,"p99":1081343,"p999":1261567} passed=true -# Create bench group 1 (group 0 sysdata preserved). -echo "=== Creating bench group 0/1 (group 0 sysdata preserved) ===" -regression_client_for "$CONFIG_FILE" \ - kv group add -s 0 -g 1 -n 1,2,3 2>&1 +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan/; full verification passed. +# 2026-10-27 deep_pag_10 host-container ops/s=13140.55 latency_ns={"avg":75991.47226333753,"count":262811,"max":1056270,"min":51360,"p50":73215,"p99":134143,"p999":186367} passed=true -# Phase 2: pre-populate keys once (prepare data before measuring). -echo "=== Pre-populating $KEYSPACE keys ===" -regression_client_for "$CONFIG_FILE" \ - bench kv prepare --store 0 --group 1 --keys "$KEYSPACE" --value-size 64 2>&1 +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan/; full verification passed. +# 2026-10-27 full_keyspace host-container ops/s=1.0 latency_ns={"avg":994163950.65,"count":20,"max":1068703819,"min":900398625,"p50":994050047,"p99":1068703819,"p999":1068703819} passed=true -# Phase 3: run all sub-tests against the same cluster. -echo "=== Single-thread (1T:1C) — per-scan engine cost ===" -run_subtest "bounded_10" 10 "" "" 64 linearizable auto 1 1 forward -run_subtest "bounded_1k" 1000 "" "" 64 linearizable auto 1 1 forward -run_subtest "bounded_10k" 10000 "" "" 64 linearizable auto 1 1 forward -run_subtest "full_100k" 100000 "" "" 64 linearizable auto 1 1 forward -run_subtest "deep_pag_10" 10 "" "$(pad_key 99989)" 64 linearizable auto 1 1 forward -run_subtest "mixed_1k" 1000 "" "" 64 linearizable auto 1 1 forward "64:70,1024:20,16384:10" -run_subtest "minslot_1k" 1000 "" "" 64 minslot zero 1 1 forward -run_subtest "reverse_1k" 1000 "" "" 64 linearizable auto 1 1 reverse -run_subtest "reverse_deep_10" 10 "" "$(pad_key 10)" 64 linearizable auto 1 1 reverse +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan/; full verification passed. +# 2026-10-27 minslot_1k host-container ops/s=1272.2 latency_ns={"avg":785748.5947178117,"count":25444,"max":2498383,"min":519425,"p50":770047,"p99":1245183,"p999":1646591} passed=true -echo "=== R67 regression — large value scan (snapshot stall) ===" -# R67: 100k × 16KiB = 1.6 GB. Before the spawn_blocking fix, -# persist_snapshot took 0.6-2.2s on Linux, blocking the async runtime -# and causing leader-election churn (300-600ms timeout) → scan_errors. -# This config must show 0 errors. Reference: doc/design/kv/kv-scan-flow-analysis.md (R67) -# Note: this sub-test uses 16KiB values but the cluster was pre-populated -# with 64B values. The scan still exercises the large-value code path -# via --value-size 16384 (affects only the scan, not pre-populated data). -# For a true 16KiB regression, deploy a separate cluster with 16KiB pre-pop. -run_subtest "largeval_16k" 1000 "" "" 16384 linearizable auto 1 1 forward +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan/; full verification passed. +# 2026-10-27 reverse_1k host-container ops/s=399.35 latency_ns={"avg":2503342.7186678355,"count":7987,"max":4551340,"min":1948547,"p50":2473983,"p99":3506175,"p999":4079615} passed=true -echo "=== Multi-thread — max throughput + read-mode split ===" -run_subtest "lin_4t" 1000 "" "" 64 linearizable auto 4 4 forward -run_subtest "minslot_4t" 1000 "" "" 64 minslot zero 4 4 forward -run_subtest "reverse_4t" 1000 "" "" 64 minslot zero 4 4 reverse -run_subtest "lin_16t" 1000 "" "" 64 linearizable auto 16 16 forward -run_subtest "minslot_16t" 1000 "" "" 64 minslot zero 16 16 forward -run_subtest "lin_32t" 1000 "" "" 64 linearizable auto 32 32 forward -run_subtest "minslot_32t" 1000 "" "" 64 minslot zero 32 32 forward +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 1000000 keys, value_size=64, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan/; full verification passed. +# 2026-10-27 reverse_deep_10 host-container ops/s=11787.55 latency_ns={"avg":84681.64017119758,"count":235751,"max":1082029,"min":57210,"p50":82431,"p99":134143,"p999":200703} passed=true -# Phase 4: teardown via owned fixture shutdown. -echo "=== Tearing down cluster ===" -regression_fixture_for "$CONFIG_FILE" stop +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 100000 keys, value_size=64, mixed=True. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan-mixed/; full verification passed. +# 2026-10-27 mixed_1k host-container ops/s=181.1 latency_ns={"avg":5518864.853948095,"count":3622,"max":10438412,"min":1641084,"p50":5439487,"p99":8781823,"p999":10223615} passed=true -echo "=== DONE ===" -echo "Results in $RESULTS_FILE" -column -t -s$'\t' "$RESULTS_FILE" +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 100000 keys, value_size=16384, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/scan-large/; full verification passed. +# 2026-10-27 largeval_16k host-container ops/s=36.4 latency_ns={"avg":27400801.240384616,"count":728,"max":41199855,"min":18380413,"p50":26083327,"p99":39845887,"p999":41199855} passed=true diff --git a/tools/benchmark/bench-kv-write-regression.sh b/tools/benchmark/bench-kv-write-regression.sh index 7b407a1f9..1f2575a16 100644 --- a/tools/benchmark/bench-kv-write-regression.sh +++ b/tools/benchmark/bench-kv-write-regression.sh @@ -1,241 +1,48 @@ #!/usr/bin/env bash -# --- CrowDB write regression benchmark --- -# Usage: bash tools/benchmark/bench-kv-write-regression.sh -# -# Regression sentinel for write throughput with coalescing enabled. -# WAL append count tracks coalescing efficiency. Results are appended -# to doc/working/bench-write-regression.tsv and documented (with the -# CPU type) in the "Regression sentinel" section of -# doc/design/kv/kv-write-flow-analysis.md. After a run, update that -# section with the results and CPU model. -# -# Flow (R125): deploy once per server-tunable group, then -# (clean → run) per sub-test, teardown once per group. The clean -# verb wipes user data on every node (keep group0) so each write -# sub-test starts from a data-empty cluster without a full redeploy. -# Server tunables (max-inflight, coalesce) are deploy-time, so the -# sweep groups sub-tests by shared tunables: -# Group A: win=32, coalesce=16 (5 scaling sub-tests) -# Large: win=32, coalesce=16 (3 clean 16 KiB repetitions) -# Group B: win=64, coalesce=64 (2 sub-tests) -# This cuts deploys from 7 to 2; clean is much cheaper than deploy. -# -# Configurations: -# - Scaling: 1T:1C → 1000T:16C, coalesce_max_keys=16/64, -# max_inflight=32/64 -# - RPC tunables: --event-write --peer-pool-size 4 -# (event-write coalesces frames via I/O worker; peer-pool=4 spreads -# consensus send pressure across 4 connections per peer. send-queue -# uses the default 4096) -# -# Reference platform: AMD Ryzen 9 5950X (16c/32t, x86_64, Linux). -# Peak ~264K ops/s at 512T with zero-copy crowdb-rpc + event-write -# (+ page-count metrics + flush re-check loop, 2026-09-02). -# -# Prerequisites: -# - pixi installed, project dependencies resolved -# - jq installed -# - release binary built (pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server) +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# Paxos-KV write: externally started 3-node host-network containers, mem-block. +# Run with pixi run bash tools/benchmark/bench-kv-write-regression.sh. +# Set KV_BENCH_DEPLOYMENT; optional KV_BENCH_CASES, DURATION, WARMUP, TAIL, +# KEYSPACE, CLI and LOG_ROOT use the KV_BENCH_ prefix. +# Defaults: 5s warmup / 20s measurement / 5s tail. <20s is a smoke run. +# Keep every dated result, including failures. Old native process data below +# remains a useful reference; compare code, resource and dataset conditions. set -euo pipefail cd "$(dirname "$0")/../.." +source tools/benchmark/bench-kv-common.sh -# Defensive: ensure ASan/LSan is off. A stale CROWDB_ASAN=1 from a prior -# sanitize-regression.sh run (same shell, or exported in the env) would -# silently instrument the C++ libraries and tank throughput by ~50-100x. -# This is a release throughput sentinel — never run under ASan. -unset CROWDB_ASAN - -RUN_STAMP=$(date +%Y%m%d-%H%M%S) -LOG_ROOT="${KV_WRITE_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/kv-write-regression-$RUN_STAMP}" -RESULTS_FILE="${KV_WRITE_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" -REGRESSION_LOG_ROOT="$LOG_ROOT" -source tools/benchmark/bench-regression-common.sh -source tools/benchmark/bench-kv-write-sentinel.sh -export CROWDB_LOG_ROOT="$LOG_ROOT" -regression_init -DURATION="${KV_WRITE_BENCH_DURATION:-20}" -KEYSPACE="${KV_WRITE_BENCH_KEYSPACE:-1000000}" -VALUE_SIZE="${KV_WRITE_BENCH_VALUE_SIZE:-512}" -CASES="${KV_WRITE_BENCH_CASES:-}" -SENTINEL_FAILED=0 - -append_failed_row() { - local label="$1" duration="$2" keyspace="$3" value_size="$4" - echo -e "$label\t${WIN:-0}\t${COALESCE:-0}\t${RPC_WORKERS:-0}\t0\t0\t0\t0\t0\t1\t0\t0\t0\t0\t0\t0\t0\t0\t0\t0\t$duration\t$keyspace\t$value_size\t0\t0\t0\t0\t0" >> "$RESULTS_FILE" - if [[ "$label" == largeval_16k_run* ]]; then - SENTINEL_FAILED=1 - fi -} - -# sample_rss <config_file> <label> -# Reads server PIDs from the config file and reports total RSS (MB) -# across all server processes. Used to track RSS growth across sub-tests. -# Prints the human-readable line to stderr, the numeric value to stdout -# (so callers can capture it via $(...)). -sample_rss() { - local config_file="$1" label="$2" - local total=0 alive=0 - for pid in $(grep '^pid' "$config_file" | awk '{print $3}'); do - if [ -r "/proc/$pid/status" ]; then - local rss; rss=$(grep VmRSS "/proc/$pid/status" | awk '{print $2}') - if [ -n "$rss" ]; then - total=$((total + rss)) - alive=$((alive + 1)) - fi - fi - done - local total_mb=$((total / 1024)) - echo " rss[$label]: ${total_mb}MB across ${alive} servers" >&2 - echo "$total_mb" -} - -# run_bench <deploy_name> <threads> <conn> <label> [duration] [keyspace] [value_size] -# Assumes the deploy was already created with the right server tunables. -# Cleans user data, then runs the write workload, parses JSON output. -run_bench() { - local deploy="$1" threads="$2" conn="$3" label="$4" - local duration="${5:-$DURATION}" keyspace="${6:-$KEYSPACE}" value_size="${7:-$VALUE_SIZE}" - if [ -n "$CASES" ] \ - && [[ " $CASES " != *" $label "* ]] \ - && ! { [[ "$label" == largeval_16k_run* ]] && [[ "$CASES" == *"largeval_16k"* ]]; }; then - return - fi - echo ">>> $label ..." - local config_file - config_file=$(cat "$LOG_ROOT/${deploy}.cfgpath" 2>/dev/null || echo "") - if [ -z "$config_file" ] || [ ! -f "$config_file" ]; then - echo " ERROR: no config for deploy '$deploy'" - append_failed_row "$label" "$duration" "$keyspace" "$value_size" - return - fi - # RSS before clean (measures leftover from prior sub-test). - local rss_pre_clean; rss_pre_clean=$(sample_rss "$config_file" "pre-clean") - # Clean: wipe user data on bench group (group 0 sysdata preserved). - local clean_out - clean_out=$(regression_fixture_for "$config_file" reset --store 0 --group 1 2>&1) - local clean_json; clean_json=$(echo "$clean_out" | sed -n '/^{/,/^}/p') - if [ -z "$clean_json" ] || ! echo "$clean_json" | jq -e '.new_leader' >/dev/null 2>&1; then - echo " ERROR: clean failed"; echo "$clean_out" - append_failed_row "$label" "$duration" "$keyspace" "$value_size" - return - fi - # RSS after clean (measures how much memory clean actually freed). - local rss_post_clean; rss_post_clean=$(sample_rss "$config_file" "post-clean") - local clean_delta=$((rss_pre_clean - rss_post_clean)) - echo " rss clean delta: ${clean_delta}MB freed" - local output bench_status=0 - output=$(regression_client_for "$config_file" \ - bench kv write --store 0 --group 1 --duration-secs "$duration" \ - --loader-num "$threads" --connections "$conn" \ - --key-space "$keyspace" --value-size "$value_size" \ - --event-write --rpc-workers "${RPC_WORKERS:-2}" \ - --verify-bytes 0 2>&1 | tee "$LOG_ROOT/$label-cli.log") || bench_status=$? - if [ "$bench_status" -ne 0 ]; then - echo " ERROR: benchmark command exited with status $bench_status" - echo "$output" >&2 - append_failed_row "$label" "$duration" "$keyspace" "$value_size" - return - fi - local json; json=$(regression_report_for "$config_file") - if [ -z "$json" ]; then - echo " ERROR: no JSON output"; echo "$output" - append_failed_row "$label" "$duration" "$keyspace" "$value_size" - return - fi - # RSS after workload (measures how much the workload grew RSS). - local rss_post_bench; rss_post_bench=$(sample_rss "$config_file" "post-bench") - local bench_delta=$((rss_post_bench - rss_post_clean)) - echo " rss bench delta: +${bench_delta}MB (post-bench - post-clean)" - local ops_s avg_us p50_us p99_us errors correctness_errors wal total_ops - total_ops=$(echo "$json" | jq -r '.total_ops') - ops_s=$(echo "$json" | jq -r '.total_ops * 1000 / .duration_ms' | awk '{printf "%.0f", $1}') - avg_us=$(echo "$json" | jq -r '.by_op.write.latency_us.avg_us') - p50_us=$(echo "$json" | jq -r '.by_op.write.latency_us.p50_us') - p99_us=$(echo "$json" | jq -r '.by_op.write.latency_us.p99_us') - errors=$(echo "$json" | jq -r '.total_errors') - correctness_errors=$(echo "$json" | jq -r '.correctness_errors // 0') - local snapshot_completed snapshot_failed snapshot_max_us election_delta - snapshot_completed=$(echo "$json" | jq -r '.server_metrics.snapshot.completed // 0') - snapshot_failed=$(echo "$json" | jq -r '.server_metrics.snapshot.failed // 0') - snapshot_max_us=$(echo "$json" | jq -r '.server_metrics.snapshot.max_latency_us // 0') - election_delta=$(echo "$json" | jq -r '.server_metrics.election_count // 0') - # WAL append: aggregated across 3 nodes; per-node = wal/3 = accept rounds/s - wal=$(echo "$json" | jq -r '.server_metrics.wal_append_count // 0') - wal_per_node=$((wal / 3)) - # RPC aggregation ratios: frames per syscall (sagg = frames_sent/writev_calls, ragg = frames_parsed/read_calls) - local srv_sa srv_ra cli_sa cli_ra srv_s2w cli_s2w - srv_sa=$(echo "$json" | jq -r 'if .server_metrics.rpc.writev_calls > 0 then (.server_metrics.rpc.frames_sent / .server_metrics.rpc.writev_calls) else 0 end | . * 10 | floor / 10') - srv_ra=$(echo "$json" | jq -r 'if .server_metrics.rpc.read_calls > 0 then (.server_metrics.rpc.frames_parsed / .server_metrics.rpc.read_calls) else 0 end | . * 10 | floor / 10') - cli_sa=$(echo "$json" | jq -r 'if .client_transport_stats.writev_calls > 0 then (.client_transport_stats.frames_sent / .client_transport_stats.writev_calls) else 0 end | . * 10 | floor / 10') - cli_ra=$(echo "$json" | jq -r 'if .client_transport_stats.read_calls > 0 then (.client_transport_stats.frames_parsed / .client_transport_stats.read_calls) else 0 end | . * 10 | floor / 10') - srv_s2w=$(echo "$json" | jq -r '.server_metrics.rpc.submit_to_writev_avg_us // 0') - cli_s2w=$(echo "$json" | jq -r '.client_transport_stats.submit_to_writev_avg_us // 0') - # Inter-replica consensus RPC: latency (avg us) + tps (round-trips/s) - local r2_avg r2_tps r3_avg r3_tps - r2_avg=$(echo "$json" | jq -r '.server_metrics.replica.r2 // 0') - r2_tps=$(echo "$json" | jq -r '.server_metrics.replica.r2_tps // 0') - r3_avg=$(echo "$json" | jq -r '.server_metrics.replica.r3 // 0') - r3_tps=$(echo "$json" | jq -r '.server_metrics.replica.r3_tps // 0') - # Inflight window pressure: enqueued = window-full hits, wait = avg queue time - local inflight_enq inflight_wait - inflight_enq=$(echo "$json" | jq -r '.server_metrics.inflight_enqueued // 0') - inflight_wait=$(echo "$json" | jq -r '.server_metrics.inflight_wait_avg_us // 0') - local co_factor - co_factor=$(awk "BEGIN { if ($wal_per_node > 0) printf \"%.1f\", $total_ops / $wal_per_node; else printf \"0.0\" }") - echo " ops/s=$ops_s wal/node=$wal_per_node co=${co_factor}/${COALESCE} avg=${avg_us}us p50=${p50_us}us p99=${p99_us}us err=$errors" - echo " rpc_agg: srv sagg=${srv_sa} ragg=${srv_ra} s2w=${srv_s2w}us | cli sagg=${cli_sa} ragg=${cli_ra} s2w=${cli_s2w}us" - echo " replica: r2=${r2_avg}us/${r2_tps}tps r3=${r3_avg}us/${r3_tps}tps" - echo " inflight: enq=${inflight_enq} wait_avg=${inflight_wait}us" - echo " maintenance: snapshot=${snapshot_completed} failed=${snapshot_failed} max=${snapshot_max_us}us elections=${election_delta}" - echo -e "$label\t$WIN\t$COALESCE\t${RPC_WORKERS:-2}\t$ops_s\t$wal_per_node\t$avg_us\t$p50_us\t$p99_us\t$errors\t$srv_sa\t$srv_ra\t$cli_sa\t$cli_ra\t$r2_avg\t$r2_tps\t$r3_avg\t$r3_tps\t$inflight_enq\t$inflight_wait\t$duration\t$keyspace\t$value_size\t$correctness_errors\t$snapshot_completed\t$snapshot_failed\t$snapshot_max_us\t$election_delta" >> "$RESULTS_FILE" - if [[ "$label" == largeval_16k_run* ]] && ! validate_largeval_result \ - "$errors" "$correctness_errors" "$election_delta" "$snapshot_completed" "$snapshot_failed"; then - echo " ERROR: large-value snapshot sentinel contract failed" - SENTINEL_FAILED=1 - fi -} - -# deploy_group <name> <win> <coalesce> <rpc_workers> -# Deploy a 3-node cluster with the given server tunables via the explicit test fixture, -# then create a bench group (store 0, group 1) so benchmarks don't touch -# group 0 sysdata. -deploy_group() { - local name="$1" win="$2" coalesce="$3" workers="$4" - local config_file="$LOG_ROOT/${name}-console.toml" - echo "=== deploying cluster '$name' (win=$win, coalesce=$coalesce, workers=$workers) ===" - rm -f "$config_file" - regression_fixture_for "$config_file" setup -- --nodes 3 --kind kv \ - --event-write --peer-pool-size 4 \ - --max-inflight "$win" --coalesce-max-keys "$coalesce" \ - --rpc-workers "$workers" \ - --kv-backend mem-block --wal-backend mem-block 2>&1 - echo "=== creating bench group 0/1 (group 0 sysdata preserved) ===" - regression_client_for "$config_file" \ - kv group add -s 0 -g 1 -n 1,2,3 2>&1 - # Store config path for run_bench/teardown_group. - echo "$config_file" > "$LOG_ROOT/${name}.cfgpath" - # Baseline RSS right after deploy (before any sub-test). - local _baseline; _baseline=$(sample_rss "$config_file" "post-deploy-baseline") -} - -# teardown_group <name> -teardown_group() { - local name="$1" - local config_file - config_file=$(cat "$LOG_ROOT/${name}.cfgpath" 2>/dev/null || echo "") - if [ -n "$config_file" ] && [ -f "$config_file" ]; then - regression_fixture_for "$config_file" stop 2>&1 - rm -f "$LOG_ROOT/${name}.cfgpath" +KEYSPACE="${KV_BENCH_KEYSPACE:-100000}" +run_case() { + local label="$1" threads="$2" connections="$3" window="$4" coalesce="$5" workers="$6"; shift 6 + selected "$label" || return 0 + if [[ "$window" == 64 ]]; then + : "${KV_BENCH_DEPLOYMENT_B:?profile B requires a separately prestarted win64/coalesce64 deployment}" + load_deployment "$KV_BENCH_DEPLOYMENT_B" + else + load_deployment "$KV_BENCH_DEPLOYMENT" fi + invoke "$label" write --key-space "$KEYSPACE" --value-size 512 --loader-num "$threads" \ + --connections "$connections" --expect-window "$window" --expect-coalesce "$coalesce" \ + --rpc-workers "$workers" --event-write "$@" } +run_case write_1t_1c_win32_coales16 1 1 32 16 2 +run_case write_16t_2c_win32_coales16 16 2 32 16 2 +run_case write_64t_4c_win32_coales16 64 4 32 16 2 +run_case write_128t_4c_win32_coales16 128 4 32 16 4 +run_case write_256t_8c_win32_coales16 256 8 32 16 4 +for repetition in 1 2 3; do + selected "largeval_16k_run${repetition}" || continue + load_deployment "$KV_BENCH_DEPLOYMENT" + invoke "largeval_16k_run${repetition}" write --key-space "${KV_BENCH_LARGE_KEYSPACE:-100000}" \ + --value-size 16384 --loader-num 1 --connections 1 --expect-window 32 --expect-coalesce 16 \ + --event-write --require-snapshot +done +run_case write_512t_16c_win64_coales64 512 16 64 64 4 +run_case write_1000t_16c_win64_coales64 1000 16 64 64 4 +finish_cases -# --- regression sentinel configs --- -# -# Regression policy: only update the reference table below when a new -# run is strictly better (higher ops/s, lower latency, fewer errors). -# If a run is worse, do NOT update — investigate and fix the regression -# first, otherwise silent performance regressions slip in. -# +# ---- Native process history (no containers) ---- # 2026-09-02 (same hw, +page-count metrics +flush re-check loop): # sagg/ragg columns are 0 (moved to crowdb-common histograms). p50/p99 # use coarser histogram buckets (500us increments) — not directly @@ -318,53 +125,100 @@ teardown_group() { # due to lower per-op overhead, but saturates earlier (non-SMT 18-core vs # 32-thread SMT AMD). Zero-copy comparison on M5 Pro pending. -# Build release binaries with ASan explicitly off. Rebuilds are cheap -# (cargo skips unchanged crates); this only recompiles if a prior -# sanitize run left an ASan-instrumented artifact in target/release. -echo "=== building release (CROWDB_ASAN unset) ===" -pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server 2>&1 +# ---- Host-network container history ---- +# 2026-10-27: AMD Ryzen 9 5950X, 32 logical CPUs; three 8GiB/4-CPU containers, +# no cpuset, mem-block Tree/WAL, server workers=2, window/coalesce=32/16, +# peer pool=4, event-write; management/RPC 10135/11135 + offsets 0/1/2. +# Release CLI, 5/20/5s; 100k x 512B (old native: 1M keys and clean-start writes). +# Each case prefills/verifies then overwrites; same containers retained between cases. +# Image: sha256:eb0dd67818e3f3ec2f81ba6969db8320f2dcd8a9c92c1bf727d1d1f7cce9d09d +# Source tree: b984ab3e0838bee528d880e6bb83fc34c3c54663; HEAD: 9380e341d634cca89d8ac678aa0340e56cd7a3de. +# Raw reports + CLI checksum: .crowdb-runtime/artifacts/bench/kv-review-write/ +# All zero errors/incomplete. Measurement-end memory MiB per node: 1T=689/680/680, +# 16T=2533/2726/2725, 64T=3576/5214/5230; retained state grows between cases. +# 2026-10-27 write_1t_1c_win32_coales16 host-container ops/s=4835.45 latency_ns={"avg":206663.62079020566,"count":96709,"max":1445419,"min":139992,"p50":196607,"p99":321535,"p999":536575} passed=true +# 2026-10-27 write_16t_2c_win32_coales16 host-container ops/s=47325.25 latency_ns={"avg":337976.5844395962,"count":946505,"max":17705262,"min":159192,"p50":327679,"p99":540671,"p999":983039} passed=true +# 2026-10-27 write_64t_4c_win32_coales16 host-container ops/s=71047.35 latency_ns={"avg":900637.8576456405,"count":1420947,"max":45301604,"min":162352,"p50":892927,"p99":1327103,"p999":2637823} passed=true + +# 2026-10-27: unrestricted CPU follow-up, fresh 3-node host-network cluster. +# No CPU quota/cpuset; Docker NanoCpus/CpuQuota=0, container and ancestor cpu.max=max. +# 20GiB/node, CLI memory guard=90%; same image/source/server profile above. +# 100k x 512B, 64T/4C, workers=2, window/coalesce=32/16, peer pool=4, +# default election profile; prefill + overwrite, 5/20/5s; zero errors/incomplete. +# CLI SHA256: 43a7ad7904bde40d11c1408b44b0bf2b246133867bdde4af600facc1a4aa0129. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-write-unlimited-cpu/ +# 45.1% below historical 151431 TPS. Prior 71047.35 run reused containers after +# 1T/16T cases and used 8GiB/4-CPU; do not attribute all change to CPU quota. +# 2026-10-27 write_64t_4c_win32_coales16 host-container ops/s=83187.05 latency_ns={"avg":769169.0450635045,"count":1663741,"max":18326773,"min":214733,"p50":770047,"p99":1212415,"p999":1564671} passed=true + +# 2026-10-27: same unrestricted CPU fresh-cluster case; Tokio server workers +# now match available logical CPUs (32), independently of RPC workers=2. +# CLI Tokio also explicitly set to available CPUs (32); prior CLI used default. +# Other conditions: 20GiB, 90% guard, 100k x 512B, 64T/4C, win32/coalesce16, +# peer pool4, event-write, default election profile, 5/20/5s, same image/CLI. +# Zero errors/incomplete/elections; sampled throttling=0. +# Mean coalescing estimate: 2314913 server puts / 151039 proposal rounds = +# 15.3266 keys/round (95.79% of 16), inflight_enqueued=0. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-write-unlimited-tokio32/ +# 2026-10-27 write_64t_4c_win32_coales16 host-container ops/s=115746.6 latency_ns={"avg":552682.1046825566,"count":2314932,"max":3584362,"min":222993,"p50":536575,"p99":905215,"p999":1310719} passed=true -echo -e "label\twin\tcoalesce\tworkers\tops_s\twal_per_node\tavg_us\tp50_us\tp99_us\terrors\tsrv_sagg\tsrv_ragg\tcli_sagg\tcli_ragg\tr2_avg\tr2_tps\tr3_avg\tr3_tps\tinflight_enq\tinflight_wait_us\tduration_s\tkeyspace\tvalue_size\tcorrectness_errors\tsnapshot_completed\tsnapshot_failed\tsnapshot_max_us\telection_delta" > "$RESULTS_FILE" +# 2026-10-27: Chosen/BatchChosen notification now use send_one_way (no reply completion). +# Same fresh 3-node host-network, release CLI, 20GiB/node, no CPU quota/cpuset, +# Tokio32/RPC2, 64T/4C, 100k x 512B, window32/coalesce16, peer pool4, 5/20/5s. +# Docker image ID: sha256:f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff +# Source tree: 2e5e29a36a45cecbba35ced31fae980d0e5cb509; same CLI SHA256 as prior run. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-write-oneway/ +# WAL append/s per node=7585.27/7585.22/7585.32; no reaped counters in any node log. +# Zero errors/incomplete/elections; full verification passed. Leader node3 (prior node2). +# TPS +0.35% versus 115746.6: no demonstrated material throughput improvement. +# 2026-10-27 write_64t_4c_win32_coales16 host-container ops/s=116152.15 latency_ns={"avg":550749.4989313586,"count":2323043,"max":3217648,"min":226702,"p50":536575,"p99":892927,"p999":1253375} passed=true -# Group A: win=32, coalesce=16 (5 sub-tests, workers=2 except 128T+) -if [ -z "$CASES" ] || [[ "$CASES" == *"win32_coales16"* ]]; then - DEPLOY_A="write-reg-A-$$-$(date +%s)" - WIN=32 COALESCE=16 RPC_WORKERS=2 deploy_group "$DEPLOY_A" 32 16 2 - echo "=== write (win=32, coalesce=16) ===" - WIN=32 COALESCE=16 RPC_WORKERS=2 run_bench "$DEPLOY_A" 1 1 "write_1t_1c_win32_coales16" - WIN=32 COALESCE=16 RPC_WORKERS=2 run_bench "$DEPLOY_A" 16 2 "write_16t_2c_win32_coales16" - WIN=32 COALESCE=16 RPC_WORKERS=2 run_bench "$DEPLOY_A" 64 4 "write_64t_4c_win32_coales16" - WIN=32 COALESCE=16 RPC_WORKERS=4 run_bench "$DEPLOY_A" 128 4 "write_128t_4c_win32_coales16" - WIN=32 COALESCE=16 RPC_WORKERS=4 run_bench "$DEPLOY_A" 256 8 "write_256t_8c_win32_coales16" - teardown_group "$DEPLOY_A" -fi +# 2026-10-27: user-requested native diagnostic, three standalone release KV servers. +# Same staged server binary as notification-fix image, now directly launched via +# external systemd user services; no Docker. 20GiB/node, swap0, no CPU quota, CPUs0-31. +# Same workload/knobs: 100k x 512B, 64T/4C, Tokio32/RPC2, win32/coalesce16, +# peer pool4, event-write, mem-block Tree/WAL, 5/20/5s. Fresh data; leader=node2. +# CLI rebuilt only to add read-only native PID evidence; workload code unchanged. +# Zero errors/incomplete/elections, full verification passed, no reaped counters. +# +9.78% TPS versus latest host-container; one run, not an isolated Docker-overhead proof. +# Leader WAL=8269.5 append/s over complete internal 15s log windows; approx8553.6/s +# from HTTP deltas has 0.557s end-sampling skew, so do not mix those intervals. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-write-native/ (attempt-1: preflight failure). +# 2026-10-27 write_64t_4c_win32_coales16 native-process ops/s=127516.45 latency_ns={"avg":501723.8486289416,"count":2550329,"max":2439695,"min":225471,"p50":489471,"p99":790527,"p999":1011711} passed=true -# Large-value sentinel: one deployment, three clean group repetitions. -if [ -z "$CASES" ] || [[ "$CASES" == *"largeval_16k"* ]]; then - DEPLOY_LARGE="write-largeval-$$-$(date +%s)" - WIN=32 COALESCE=16 RPC_WORKERS=2 deploy_group "$DEPLOY_LARGE" 32 16 2 - echo "=== large-value snapshot sentinel (16 KiB, 100k keys, 15s) ===" - for repetition in 1 2 3; do - WIN=32 COALESCE=16 RPC_WORKERS=2 run_bench \ - "$DEPLOY_LARGE" 1 1 "largeval_16k_run${repetition}" 15 100000 16384 - done - teardown_group "$DEPLOY_LARGE" -fi +# 2026-10-27: unrestricted memory + CPU container baseline, fresh deployment per case. +# AMD 5950X/32 logical CPUs; no --memory/--memory-swap/--cpus/--cpuset; +# Docker Memory/MemorySwap/NanoCpus/CpuQuota=0, cpuset empty; cgroup memory.max=max, +# cpu.max=max. Tokio32, server RPC workers2, peer pool4, event-write, mem-block. +# 100k x 512B, prefill + verify + overwrite + verify; 5/20/5s. +# CLI workers2 for 1/16/64T; workers4 for 128/256/512/1000T. +# Profile A: window32/coalesce16; profile B (512/1000T): window64/coalesce64. +# Same notification-fix image sha256:f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff. +# CLI now accepts memory.max=max; finite-limit runs retain the 90% guard. +# Raw reports, image inspect, cgroup timeline, logs and CLI hash: +# .crowdb-runtime/artifacts/bench/kv-review-unlimited/write-*/ +# External observer stops owned nodes only if host MemAvailable falls below4GiB; +# no intervention occurred in these write cases. This is not a container memory quota. +# 2026-10-27 write_1t_1c_win32_coales16 host-container ops/s=4761.2 latency_ns={"avg":209877.27959337982,"count":95224,"max":1770434,"min":153041,"p50":207871,"p99":274431,"p999":350207} passed=true +# 2026-10-27 write_16t_2c_win32_coales16 host-container ops/s=45774.45 latency_ns={"avg":349418.2195482414,"count":915489,"max":1675024,"min":167951,"p50":339967,"p99":536575,"p999":618495} passed=true +# 2026-10-27 write_64t_4c_win32_coales16 host-container ops/s=129753.35 latency_ns={"avg":493052.27683639765,"count":2595067,"max":4463586,"min":199662,"p50":483327,"p99":761855,"p999":1007615} passed=true +# 2026-10-27 write_128t_4c_win32_coales16 host-container ops/s=158281.3 latency_ns={"avg":808343.0829668444,"count":3165626,"max":2721382,"min":264362,"p50":782335,"p99":1368063,"p999":1630207} passed=true +# 2026-10-27 write_256t_8c_win32_coales16 host-container ops/s=177685.1 latency_ns={"avg":1440245.5994630388,"count":3553702,"max":5921808,"min":281762,"p50":1392639,"p99":2605055,"p999":3096575} passed=true +# 2026-10-27 write_512t_16c_win64_coales64 host-container ops/s=194868.8 latency_ns={"avg":2626473.476040033,"count":3897376,"max":8206807,"min":491744,"p50":2523135,"p99":4849663,"p999":5734399} passed=true +# 2026-10-27 write_1000t_16c_win64_coales64 host-container ops/s=209807.5 latency_ns={"avg":4764418.7959129205,"count":4196150,"max":24693742,"min":625525,"p50":4521983,"p99":9764863,"p999":12582911} passed=true -# Group B: win=64, coalesce=64 (2 sub-tests, workers=4) -if [ -z "$CASES" ] || [[ "$CASES" == *"win64_coales64"* ]]; then - DEPLOY_B="write-reg-B-$$-$(date +%s)" - WIN=64 COALESCE=64 RPC_WORKERS=4 deploy_group "$DEPLOY_B" 64 64 4 - echo "=== write (win=64, coalesce=64) ===" - WIN=64 COALESCE=64 RPC_WORKERS=4 run_bench "$DEPLOY_B" 512 16 "write_512t_16c_win64_coales64" - WIN=64 COALESCE=64 RPC_WORKERS=4 run_bench "$DEPLOY_B" 1000 16 "write_1000t_16c_win64_coales64" - teardown_group "$DEPLOY_B" -fi +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 100000 keys, value_size=16384, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/sentinel/; full verification passed. +# Single fresh deployment, e2e election profile; not a continuous three-run sentinel pass. +# 2026-10-27 largeval_16k_run1 host-container ops/s=3678.0 latency_ns={"avg":271747.8872349103,"count":73560,"max":58775148,"min":186252,"p50":266239,"p99":364543,"p999":602111} passed=true -echo "=== DONE ===" -echo "Results in $RESULTS_FILE" -column -t -s$'\t' "$RESULTS_FILE" -if [ "$SENTINEL_FAILED" -ne 0 ]; then - echo "Large-value sentinel failed; evidence retained in $LOG_ROOT" >&2 - exit 1 -fi +# 2026-10-27: Paxos-KV naming migration comparison, no CPU/memory quota/cpuset. +# 3-node host-network mem-block; Tokio32, server/CLI RPC2, peer pool4, event-write. +# 100k x 512B, 64T/4C, window32/coalesce16; warmup/measurement/tail=5/20/5s. +# Command is now bench paxos-kv. Image e1fc2fee3decd3e0b4693aa6d9518a9bcf7ac097b1e2663b62292161563282a4. +# Clean run: 128782.6 TPS, p99=765951ns, p999=970751ns, avg=496774ns; +# zero errors/incomplete, full verification passed. Before rename:129753.35 TPS (-0.748%). +# One comparison only, not a statistical equivalence claim. +# Earlier attempt:122869.55 TPS, p99=843775ns; clippy overlapped, excluded from perf comparison. diff --git a/tools/benchmark/bench-kv-write-sentinel.sh b/tools/benchmark/bench-kv-write-sentinel.sh index f12c85d3a..c08e7a57c 100644 --- a/tools/benchmark/bench-kv-write-sentinel.sh +++ b/tools/benchmark/bench-kv-write-sentinel.sh @@ -1,12 +1,51 @@ #!/usr/bin/env bash -# Shared validation contract for the large-value snapshot sentinel. +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# Snapshot sentinel: zero workload/correctness errors, zero elections, +# at least one completed snapshot, zero failed snapshots during full load. +# Three repetitions of 100k x 16KiB; reuse the already running containers. +# History 2026-10-27: FAILED run1; run2/run3 not run. AMD 5950X, host network, +# 3 containers x 12GiB/4 CPU, mem-block Tree/WAL, e2e election profile, +# 100k x 16KiB, 1T/1C, planned warmup/measurement/tail 5/20/5s. +# Memory guard stopped load at 7.64s (80%); 1 incomplete, 0 completed load +# snapshots, 2 elections. All 3 containers subsequently OOMKilled (exit 137). +# Partial throughput is NOT a valid perf result. Full-read verification still +# ran after load cancellation; this was not an idle reclamation observation. +# Node 2 flush output entries/time: 6081/1.102s, 11532/2.538s, +# 19481/8.325s, 42188/34.295s. Frozen queue reached 10; active could keep growing. +# Evidence: .crowdb-runtime/artifacts/bench/kv-review-sentinel/ +# Image: sha256:eb0dd67818e3f3ec2f81ba6969db8320f2dcd8a9c92c1bf727d1d1f7cce9d09d +# 2026-10-27 follow-up requested larger memory, same image/dataset/CPU/profile: +# 16GiB/node, old 80% guard: run1 stopped at 23.521s, 1 incomplete, +# 0 elections / 0 completed snapshots. Final inspect: all running, no OOM. +# Sampled node memory peaks GiB: 12.488 / 13.829 / 13.517. +# 20GiB/node, old 80% guard: run1 completed 5/20/5s and verification, +# 3725.7 ops/s, p99=358399ns, zero request errors/incomplete/elections. +# Sentinel still FAILED: 0 completed snapshots in its window. No OOM. +# Sampled node memory peaks GiB: 15.518 / 15.248 / 14.496. +# run2/run3 not dispatched after each failed sentinel; no three-run pass. +# Evidence: .crowdb-runtime/artifacts/bench/kv-review-sentinel-{16g,20g}/ +# Both follow-ups used CLI SHA256 1537fe8ad93a31974956b5c5019d6df900c870f923ca0957f5a0b5a2cd7dbe55. +# Guard subsequently changed to 90% at user request; above results remain 80%. +# 2026-10-27: 20GiB/node, 90% guard, same server image and 4-CPU quota. +# CLI SHA256: 43a7ad7904bde40d11c1408b44b0bf2b246133867bdde4af600facc1a4aa0129. +# Three INDEPENDENT fresh-cluster repetitions passed: 3794.15 / 3864.7 / +# 3720.3 ops/s, p99 430079 / 352255 / 434175ns. Each completed 5/20/5s, +# verified bytes, 0 errors/incomplete/elections, 3 completed snapshots total. +# This is NOT a pass for three consecutive repetitions on one deployment: +# the original reused run2 was stopped during preparation by the external +# host observer (MemAvailable=2003020KiB, below its 4GiB floor); run3 unrun. +# Docker OOMKilled=false; exit137 there resulted from external docker stop. +# Independent run2/run3 used newly deployed containers after preserving failure. +# Evidence: kv-review-sentinel-20g-90pct{,-fresh2,-fresh3} under bench artifacts. +set -euo pipefail +cd "$(dirname "$0")/../.." +export KV_BENCH_CASES="${KV_BENCH_CASES:-largeval_16k_run1 largeval_16k_run2 largeval_16k_run3}" +exec bash tools/benchmark/bench-kv-write-regression.sh -validate_largeval_result() { - local errors="$1" correctness_errors="$2" election_delta="$3" - local snapshot_completed="$4" snapshot_failed="$5" - [ "$errors" -eq 0 ] \ - && [ "$correctness_errors" -eq 0 ] \ - && [ "$election_delta" -eq 0 ] \ - && [ "$snapshot_completed" -gt 0 ] \ - && [ "$snapshot_failed" -eq 0 ] -} +# 2026-10-27: no memory/CPU quota/cpuset, Tokio32/RPC2, 3-node host-network mem-block. +# Image f576b3d9387c58c81df9c0e99bf1ca85659fac77ae05e285b539fb7840e7d9ff; 5/20/5s. +# Dataset: 100000 keys, value_size=16384, mixed=False. +# Raw: .crowdb-runtime/artifacts/bench/kv-review-unlimited/sentinel/; full verification passed. +# Single fresh deployment, e2e election profile; not a continuous three-run sentinel pass. +# 2026-10-27 largeval_16k_run1 host-container ops/s=3678.0 latency_ns={"avg":271747.8872349103,"count":73560,"max":58775148,"min":186252,"p50":266239,"p99":364543,"p999":602111} passed=true diff --git a/tools/benchmark/bench-leak-inspect.sh b/tools/benchmark/bench-leak-inspect.sh index 78c9f4117..a55c980d4 100644 --- a/tools/benchmark/bench-leak-inspect.sh +++ b/tools/benchmark/bench-leak-inspect.sh @@ -1,94 +1,41 @@ #!/usr/bin/env bash -# --- Leak inspection: run 1T..128T, keep servers alive for heap dump --- -# Usage: bash tools/benchmark/bench-leak-inspect.sh +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# Inspect cumulative memory against an externally owned container deployment. +# KV_BENCH_DEPLOYMENT is required. Allocator settings belong to deployment time. +# No reset between cases: growth is intentionally retained for inspection. +# Exit leaves the containers running; their owner handles shutdown. set -euo pipefail cd "$(dirname "$0")/../.." - -unset CROWDB_ASAN -DURATION=20 -KEYSPACE=1000000 -VALUE_SIZE=512 -DEPLOY="leak-inspect-$$-$(date +%s)" -RUNTIME_ROOT="${CROWDB_RUNTIME_ROOT:-${PIXI_PROJECT_ROOT:-$(pwd)}/.crowdb-runtime}" -REGRESSION_LOG_ROOT="$RUNTIME_ROOT/artifacts/bench/leak-inspect-$DEPLOY" -CFG="$REGRESSION_LOG_ROOT/console.toml" -BENCH_STORE=0 -BENCH_GROUP=1 -REGRESSION_CONFIG="$CFG" -source tools/benchmark/bench-regression-common.sh -regression_init -trap regression_destroy EXIT - -echo "=== deploying cluster (win=32, coalesce=16, workers=2, MALLOC_ARENA_MAX=1 + TRIM_THRESHOLD=0) ===" -MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 regression_fixture_for "$CFG" setup -- --nodes 3 --kind kv \ - --event-write --peer-pool-size 4 \ - --max-inflight 32 --coalesce-max-keys 16 \ - --rpc-workers 2 \ - --kv-backend mem-block --wal-backend mem-block 2>&1 - -echo "=== creating bench group $BENCH_STORE/$BENCH_GROUP (group 0 sysdata preserved) ===" -regression_client_for "$CFG" \ - kv group add -s "$BENCH_STORE" -g "$BENCH_GROUP" -n 1,2,3 2>&1 - -sample_rss() { +source tools/benchmark/bench-kv-common.sh +[[ $(jq -r '.kind // "container"' "$KV_BENCH_DEPLOYMENT") == container ]] || { + echo 'Leak inspection requires a container deployment' >&2; exit 2; +} +sample_memory() { local label="$1" - local total=0 alive=0 - for pid in $(grep '^pid' "$CFG" | awk '{print $3}'); do - if [ -r "/proc/$pid/status" ]; then - local rss; rss=$(grep VmRSS "/proc/$pid/status" | awk '{print $2}') - if [ -n "$rss" ]; then - total=$((total + rss)) - alive=$((alive + 1)) - fi - fi - done - local total_mb=$((total / 1024)) - echo " rss[$label]: ${total_mb}MB across ${alive} servers" >&2 - for pid in $(grep '^pid' "$CFG" | awk '{print $3}'); do - if [ -r "/proc/$pid/status" ]; then - local rss; rss=$(grep VmRSS "/proc/$pid/status" | awk '{print $2}') - local mb=$((rss / 1024)) - echo " pid=$pid RSS=${mb}MB" >&2 - fi - done + docker inspect "${containers[@]}" > "$LOG_ROOT/$label-inspect.json" + docker stats --no-stream --format '{{json .}}' "${containers[@]}" | tee "$LOG_ROOT/$label-memory.jsonl" } - -run_sub() { - local threads="$1" conn="$2" label="$3" workers="${4:-2}" - echo ">>> $label ..." - sample_rss "pre-clean" - MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 regression_fixture_for "$CFG" reset --store "$BENCH_STORE" --group "$BENCH_GROUP" 2>&1 - sample_rss "post-clean" - MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 regression_client_for "$CFG" \ - bench kv write --store "$BENCH_STORE" --group "$BENCH_GROUP" \ - --duration-secs "$DURATION" \ - --loader-num "$threads" --connections "$conn" \ - --key-space "$KEYSPACE" --value-size "$VALUE_SIZE" \ - --event-write --rpc-workers "$workers" \ - --verify-bytes 0 2>&1 - sample_rss "post-bench" +run_case() { + local threads="$1" connections="$2" workers="$3" label="write_${1}t" + selected "$label" || return 0 + sample_memory "$label-before" + invoke "$label" write --key-space "$KEYSPACE" --value-size "${KV_BENCH_VALUE_SIZE:-512}" \ + --loader-num "$threads" --connections "$connections" --rpc-workers "$workers" --event-write + sample_memory "$label-after" } +run_case 1 1 2 +run_case 16 2 2 +run_case 64 4 2 +run_case 128 4 4 +finish_cases +echo "Containers remain running for inspection; their owner must stop them." -sample_rss "post-deploy" -run_sub 1 1 "write_1t" 2 -run_sub 16 2 "write_16t" 2 -run_sub 64 4 "write_64t" 2 -run_sub 128 4 "write_128t" 4 - -echo "" -echo "=== Measurements complete — servers remain owned while inspecting ===" -echo "Config: $CFG" -echo "Server PIDs:" -grep '^pid' "$CFG" | awk '{print " pid=" $3}' -sample_rss "final" -echo "" -echo "Per-node RSS + heap details:" -for pid in $(grep '^pid' "$CFG" | awk '{print $3}'); do - if [ -r "/proc/$pid/status" ]; then - local_rss=$(grep VmRSS "/proc/$pid/status" | awk '{print $2}') - local_mb=$((local_rss / 1024)) - echo " pid=$pid RSS=${local_mb}MB" - fi -done -echo "Inspect the services now; Ctrl-C stops this fixture and its owned services." -while sleep 60; do :; done +# Diagnostic history (2026-10-28 Asia/Shanghai; raw CLI timestamps are UTC 2026-10-27) +# AMD Ryzen 9 5950X, Linux x86_64, 32 logical CPUs; 3-node host network. +# Container cumulative-memory smoke; 1/2/1; 1k keys x 512B; no CPU/memory quota; not a leak-freedom proof +# Artifacts: .crowdb-runtime/artifacts/tools-migration-20261028/leak +# 2026-10-27 write_128t host-container ops/s=148813.0 latency_ns={"avg":859382.5470557008,"count":297626,"max":2702898,"min":344565,"p50":835583,"p99":1449983,"p999":1744895} passed=true +# 2026-10-27 write_16t host-container ops/s=44567.5 latency_ns={"avg":358787.00964828633,"count":89135,"max":3372416,"min":199503,"p50":342015,"p99":618495,"p999":1458175} passed=true +# 2026-10-27 write_1t host-container ops/s=4365.5 latency_ns={"avg":228884.86095521704,"count":8731,"max":800821,"min":161343,"p50":227327,"p99":288767,"p999":356351} passed=true +# 2026-10-27 write_64t host-container ops/s=108350.0 latency_ns={"avg":590196.4161744347,"count":216700,"max":2162980,"min":270383,"p50":573439,"p99":974847,"p999":1662975} passed=true diff --git a/tools/benchmark/bench-regression-common.sh b/tools/benchmark/bench-regression-common.sh index 1169f1cc7..8c623f514 100644 --- a/tools/benchmark/bench-regression-common.sh +++ b/tools/benchmark/bench-regression-common.sh @@ -46,10 +46,6 @@ regression_fixture_log_root() { pixi run python tools/benchmark/fixture.py --state "$REGRESSION_CONFIG" root } -regression_report_for() { - pixi run python tools/benchmark/fixture.py --state "$1" report -} - regression_destroy() { if [ -f "$REGRESSION_CONFIG" ]; then regression_fixture stop diff --git a/tools/benchmark/fixture-smoke.py b/tools/benchmark/fixture-smoke.py index 5e9a95f7a..aa61c5883 100644 --- a/tools/benchmark/fixture-smoke.py +++ b/tools/benchmark/fixture-smoke.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright 2026-present Gian <crow.db@outlook.com> # Licensed under the Apache License, Version 2.0. -"""Exercise native measurement ownership and real attached KV workloads.""" +"""Exercise native fixture ownership and ordinary Paxos-KV data commands.""" import argparse import json @@ -75,17 +75,19 @@ def setup(state, owner): else: raise AssertionError("foreign control token was accepted") command(state, "reset", "--group", "0", succeeds=False) - command(state, "client", "--", "kv", "group", "add", "-s", "0", "-g", "1", "-n", "1,2,3") - command(state, "client", "--", "bench", "kv", "prepare", "--store", "0", "--group", "1", - "--keys", "16", "--value-size", "64") - for workload in ("read", "scan", "write"): - command(state, "client", "--", "bench", "kv", workload, "--store", "0", "--group", "1", - "--duration-secs", "1", "--loader-num", "1", "--connections", "1", - "--key-space", "16", "--value-size", "64") - report = json.loads(command(state, "report")) - assert report["total_ops"] > 0 and report["total_errors"] == 0, report - command(state, "client", "--", "bench", "kv", "invalid", succeeds=False) - command(state, "report", succeeds=False) + command(state, "client", "--", "paxos-kv", "group", "add", "-s", "0", "-g", "1", "-n", "1,2,3") + command(state, "client", "--", "paxos-kv", "data", "put", "-s", "0", "-g", "1", + "-k", "fixture-smoke-key", "-v", "fixture-smoke-value") + fetched = command(state, "client", "--", "paxos-kv", "data", "get", "-s", "0", "-g", "1", + "-k", "fixture-smoke-key") + assert "fixture-smoke-value" in fetched, fetched + scanned = command(state, "client", "--", "paxos-kv", "data", "scan", "-s", "0", "-g", "1", + "--prefix", "fixture-smoke-") + assert "fixture-smoke-key" in scanned and "fixture-smoke-value" in scanned, scanned + # A management fixture is not an implicit benchmark deployment. + # Container load/report coverage belongs to the container benchmark tests. + command(state, "client", "--", "bench", "paxos-kv", "write", succeeds=False) + command(state, "client", "--", "bench", "paxos-kv", "invalid", succeeds=False) result = json.loads(command(state, "reset", "--group", "1")) assert result["new_leader"] and result["wiped_nodes"] == 3, result command(state, "stop") @@ -101,7 +103,7 @@ def setup(state, owner): stopped = json.loads(abandoned.with_suffix(".fixture-stopped.json").read_text()) assert stopped["ok"], stopped assert unrelated.poll() is None, "owner-exit cleanup stopped an unrelated process" - print(f"PASS: native measurement ownership and KV workloads; artifacts: {args.artifacts}") + print(f"PASS: native fixture ownership and Paxos-KV data commands; artifacts: {args.artifacts}") finally: command(state, "stop") if owner is not None and owner.poll() is None: diff --git a/tools/benchmark/fixture.py b/tools/benchmark/fixture.py index 15a643852..64b17207e 100644 --- a/tools/benchmark/fixture.py +++ b/tools/benchmark/fixture.py @@ -19,7 +19,7 @@ ROOT = Path(__file__).resolve().parents[2] HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) -BENCHMARKS = {"kv", "diskdb", "chunkdb", "chunkio", "s3"} +BENCHMARKS = {"paxos-kv", "diskdb", "chunkdb", "chunkio", "s3"} class FixtureStopped(ValueError): @@ -168,12 +168,6 @@ def run(args): if not isinstance(seeds, list) or not seeds or not all(isinstance(seed, str) and seed for seed in seeds): raise ValueError("live measurement target has no Group-0 endpoints") print(seeds[0]) - elif args.action == "report": - path = args.state.parent / "client-runtime/artifacts/cli/bench-kv/report.json" - report = json.loads(path.read_text()) - if not all(key in report for key in ("total_ops", "total_errors", "duration_ms", "by_op")): - raise ValueError("latest benchmark report is incomplete") - print(json.dumps(report, indent=2)) return 0 @@ -187,7 +181,6 @@ def main(): subcommands.add_parser("target") subcommands.add_parser("root") subcommands.add_parser("seed") - subcommands.add_parser("report") subcommands.add_parser("inspect") reset = subcommands.add_parser("reset") reset.add_argument("--store", type=int, default=0) diff --git a/tools/benchmark/test-bench-kv-write-sentinel.sh b/tools/benchmark/test-bench-kv-write-sentinel.sh index 71bf7158b..7e56e6c27 100644 --- a/tools/benchmark/test-bench-kv-write-sentinel.sh +++ b/tools/benchmark/test-bench-kv-write-sentinel.sh @@ -1,19 +1,4 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "$0")/../.." -source tools/benchmark/bench-kv-write-sentinel.sh - -validate_largeval_result 0 0 0 1 0 -for fixture in \ - "1 0 0 1 0" \ - "0 1 0 1 0" \ - "0 0 1 1 0" \ - "0 0 0 0 0" \ - "0 0 0 1 1" -do - if validate_largeval_result $fixture; then - echo "invalid fixture passed: $fixture" >&2 - exit 1 - fi -done -echo "large-value sentinel fixtures passed" +exec pixi run cargo test -p crowdb-cli --test kv_bench_test snapshot_sentinel diff --git a/tools/ci-checks/check-ci-test-tasks.py b/tools/ci-checks/check-ci-test-tasks.py index 702439f73..eb5b83bb3 100644 --- a/tools/ci-checks/check-ci-test-tasks.py +++ b/tools/ci-checks/check-ci-test-tasks.py @@ -11,11 +11,11 @@ "test-cpp": {"crowdb-tree-ffi", "crowdb-rpc-ffi", "crowdb-rpc-test"}, "test-core": { "crowdb-common", "crowdb-test-harness", "crowdb-protocol", - "crowdb-kv", "crowdb-kv-client", + "crowdb-paxos-kv", "crowdb-paxos-kv-client", }, "test-storage": { "crowdb-chunkdb-client", "crowdb-chunk-kv", "crowdb-chunk-kv-client", - "crowdb-chunk-kv-server", "crowdb-chunk-stream", "crowdb-kv-server", + "crowdb-chunk-kv-server", "crowdb-chunk-stream", "crowdb-paxos-kv-server", "crowdb-diskdb", "crowdb-diskdb-client", "crowdb-chunkdb", "crowdb-chunk-client", "crowdb-diskio-client", }, diff --git a/tools/ci-checks/check-production-dashmap.py b/tools/ci-checks/check-production-dashmap.py index ee9498867..7a5c0af2e 100644 --- a/tools/ci-checks/check-production-dashmap.py +++ b/tools/ci-checks/check-production-dashmap.py @@ -23,23 +23,23 @@ # lookup guard. The reason also documents the maximum guard lifetime. ALLOWED: dict[tuple[str, str], str] = { ( - "app/crowdb-kv-server/src/mgmt/operation_registry.rs", + "app/crowdb-paxos-kv-server/src/mgmt/operation_registry.rs", "operations", ): "low-frequency management operations; guards are dropped before polling or await", ( - "lib/crowdb-kv/src/cluster/group.rs", + "lib/crowdb-paxos-kv/src/cluster/group.rs", "snapshots", ): "per-group snapshot lifecycle handles; guards only clone or remove one handle", ( - "lib/crowdb-kv-client/src/service/discovery.rs", + "lib/crowdb-paxos-kv-client/src/service/discovery.rs", "services", ): "service-name cache; lookup clones Arc<ServiceState> before refresh I/O", ( - "lib/crowdb-kv/src/rpc/snapshot_registry.rs", + "lib/crowdb-paxos-kv/src/rpc/snapshot_registry.rs", "sessions", ): "snapshot transfer lifecycle; guards clone Arc<RegistryEntry> or remove before any await", ( - "lib/crowdb-kv/src/rpc/snapshot_registry.rs", + "lib/crowdb-paxos-kv/src/rpc/snapshot_registry.rs", "expired", ): "expired-session tombstones; lookups only check contains_key, insert, or remove", } diff --git a/tools/cpp-checks/sanitize-paxos-image.py b/tools/cpp-checks/sanitize-paxos-image.py new file mode 100644 index 000000000..2e225a90d --- /dev/null +++ b/tools/cpp-checks/sanitize-paxos-image.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +"""Package the actual instrumented server and libraries into a diagnostic image.""" +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import uuid + + +def run(*args): + return subprocess.check_output(args, text=True, stderr=subprocess.STDOUT).strip() + + +def stage(binary, destination): + symbols = run('readelf', '--dyn-syms', '--wide', str(binary)) + if '__asan_init' not in symbols: + raise ValueError('server does not link ASan; build with CROWDB_ASAN=1') + (destination / 'lib').mkdir(parents=True) + shutil.copy2(binary, destination / 'server') + dependencies = run('ldd', str(binary)) + (destination / 'dependencies.txt').write_text(dependencies + '\n') + if 'not found' in dependencies: + raise ValueError(dependencies) + # Package non-system runtime dependencies, including the exact ASan runtime. + roots = [Path(os.environ['CONDA_PREFIX']).resolve(), binary.parent.resolve()] + for line in dependencies.splitlines(): + parts = line.split() + if len(parts) >= 3 and parts[1] == '=>' and parts[2].startswith('/'): + path = Path(parts[2]).resolve() + if any(path.is_relative_to(root) for root in roots): + shutil.copy2(path, destination / 'lib' / parts[0]) + asan = [p for p in (destination / 'lib').iterdir() if p.name.startswith('libasan.so')] + if len(asan) != 1: + raise ValueError('expected one linked ASan library') + return asan[0].name + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--binary', type=Path, required=True) + p.add_argument('--base-image', required=True) + p.add_argument('--output', type=Path, required=True) + args = p.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + asan = stage(args.binary.resolve(), args.output) + base_metadata = run('docker', 'image', 'inspect', args.base_image) + (args.output / 'base-image.json').write_text(base_metadata + '\n') + base = json.loads(base_metadata)[0]['Id'] + # BuildKit interprets a bare sha256 ID as a registry repository. A unique + # local tag pins this build's base without racing a user-managed tag. + local_base = 'crowdb-sanitizer-base:' + uuid.uuid4().hex + (args.output / 'Dockerfile').write_text(f'''FROM {local_base} +COPY server /opt/crowdb/asan/bin/crowdb-paxos-kv-server +COPY lib /opt/crowdb/asan/lib +RUN mkdir -p /opt/crowdb/data/asan +LABEL crowdb.sanitizer="address" +ENV LD_LIBRARY_PATH="/opt/crowdb/asan/lib:/opt/crowdb/lib" \\ + LD_PRELOAD="/opt/crowdb/asan/lib/{asan}" \\ + ASAN_OPTIONS="detect_leaks=1:abort_on_error=0:log_path=/opt/crowdb/data/asan/report" +ENTRYPOINT ["/opt/crowdb/asan/bin/crowdb-paxos-kv-server"] +''') + run('docker', 'tag', base, local_base) + try: + subprocess.run(['docker', 'build', '--iidfile', str(args.output / 'image-id'), str(args.output)], check=True) + finally: + run('docker', 'image', 'rm', local_base) + + +if __name__ == '__main__': + main() diff --git a/tools/cpp-checks/sanitize-paxos-kv.sh b/tools/cpp-checks/sanitize-paxos-kv.sh new file mode 100644 index 000000000..49a6b4265 --- /dev/null +++ b/tools/cpp-checks/sanitize-paxos-kv.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# ASan/LSan correctness gate: three owned containers, external host load client. +# Requires a local CROWDB_SANITIZER_BASE_IMAGE. No CPU or memory quota. +# --built reuses binaries built by sanitize-regression.sh; symbol checks still run. +set -euo pipefail +cd "$(dirname "$0")/../.." +: "${CROWDB_SANITIZER_BASE_IMAGE:?set a local CrowDB OCI base image}" +export CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-8}" +export CARGO_PROFILE_DEV_DEBUG=line-tables-only +export KV_BENCH_LOG_ROOT="${KV_BENCH_LOG_ROOT:-$PWD/.crowdb-runtime/artifacts/sanitize-paxos-kv-$(date +%Y%m%d-%H%M%S)-$$}" +mkdir -p "$KV_BENCH_LOG_ROOT" +if [[ "${1:-}" != --built ]]; then + export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PWD/target/sanitize-paxos-kv}" + CROWDB_ASAN=1 pixi run cargo build --locked -p crowdb-cli -p crowdb-paxos-kv-server +fi +BUILD_DIR="${CARGO_TARGET_DIR:-$PWD/target}/debug" +export KV_BENCH_CLI="$BUILD_DIR/crowdb-cli" +pixi run python - "$KV_BENCH_CLI" <<'PY' +import subprocess +import sys +symbols = subprocess.check_output(['readelf', '--dyn-syms', '--wide', sys.argv[1]], text=True) +if '__asan_init' not in symbols: + raise SystemExit('client does not link ASan; build with CROWDB_ASAN=1') +PY +export KV_BENCH_DEPLOYMENT="$KV_BENCH_LOG_ROOT/deployment.json" +export KV_BENCH_KEYSPACE="${KV_BENCH_KEYSPACE:-1000}" +# Diagnostic smoke duration; the same load model, not a release TPS baseline. +export KV_BENCH_DURATION="${KV_BENCH_DURATION:-5}" +export KV_BENCH_WARMUP="${KV_BENCH_WARMUP:-1}" +export KV_BENCH_TAIL="${KV_BENCH_TAIL:-1}" +cleanup() { + local code=$? + trap - EXIT + if [[ -f "$KV_BENCH_DEPLOYMENT" ]]; then + pixi run python container/oci-image/paxos-kv-cluster.py down --state "$KV_BENCH_DEPLOYMENT" \ + --graceful --artifacts "$KV_BENCH_LOG_ROOT/servers" || code=1 + fi + exit "$code" +} +trap cleanup EXIT +pixi run python tools/cpp-checks/sanitize-paxos-image.py --binary "$BUILD_DIR/crowdb-paxos-kv-server" \ + --base-image "$CROWDB_SANITIZER_BASE_IMAGE" --output "$KV_BENCH_LOG_ROOT/image" +pixi run python container/oci-image/paxos-kv-cluster.py up --state "$KV_BENCH_DEPLOYMENT" \ + --image "$(cat "$KV_BENCH_LOG_ROOT/image/image-id")" --sanitizer +source tools/benchmark/bench-kv-common.sh +ASAN_LIBRARY=$(pixi run python -c 'import os; print(os.path.join(os.environ["CONDA_PREFIX"], "lib/libasan.so"))') +BENCH_PREFIX=(env "LD_PRELOAD=$ASAN_LIBRARY" "ASAN_OPTIONS=detect_leaks=1:abort_on_error=0:log_path=$LOG_ROOT/client-asan") +run_case() { + local label="$1" operation="$2" threads="$3" connections="$4"; shift 4 + invoke "$label" "$operation" --key-space "$KEYSPACE" --value-size 128 \ + --loader-num "$threads" --connections "$connections" "$@" + local reports=("$LOG_ROOT"/client-asan.*) + if [[ -f "${reports[0]}" ]]; then cat "${reports[@]}" >&2; return 1; fi +} +run_case prepare prepare 4 4 +run_case write_1t write 1 1 +run_case write_16t write 16 2 +run_case write_64t write 64 4 +run_case read_1t read 1 1 --read-mode linearizable +run_case read_16t read 16 2 --read-mode linearizable +run_case read_32t read 32 32 --read-mode linearizable +run_case read_minslot read 1 1 --read-mode minslot +run_case scan_1t scan 1 1 +run_case scan_16t scan 16 2 +run_case scan_32t scan 32 32 +# Server exit/leak checks run in cleanup, and can fail this gate. +echo "Client load cases passed; checking container shutdown: $LOG_ROOT" + +# Diagnostic history (2026-10-28 Asia/Shanghai; raw CLI timestamps are UTC 2026-10-27) +# AMD Ryzen 9 5950X, Linux x86_64, 32 logical CPUs; 3-node host network. +# ASan/LSan diagnostic; 1/5/1; 1k keys x 128B; no CPU/memory quota; all 3 servers exit 0; no sanitizer reports +# Artifacts: .crowdb-runtime/artifacts/sanitize-paxos-kv-20261028-run3 +# 2026-10-27 read_16t host-container ops/s=30377.0 latency_ns={"avg":525800.1293676137,"count":151885,"max":25600828,"min":234074,"p50":479231,"p99":1032191,"p999":2179071} passed=true +# 2026-10-27 read_1t host-container ops/s=2029.2 latency_ns={"avg":491675.2288586635,"count":10146,"max":948873,"min":271194,"p50":518143,"p99":667647,"p999":741375} passed=true +# 2026-10-27 read_32t host-container ops/s=29203.2 latency_ns={"avg":1094542.8122808458,"count":146016,"max":19420729,"min":259994,"p50":1056767,"p99":1884159,"p999":2457599} passed=true +# 2026-10-27 read_minslot host-container ops/s=2221.4 latency_ns={"avg":449080.81120014406,"count":11107,"max":936864,"min":250033,"p50":471039,"p99":655359,"p999":745471} passed=true +# 2026-10-27 scan_16t host-container ops/s=7433.6 latency_ns={"avg":2150358.186262376,"count":37168,"max":21788473,"min":1365190,"p50":2162687,"p99":5144575,"p999":7766015} passed=true +# 2026-10-27 scan_1t host-container ops/s=329.0 latency_ns={"avg":3035564.5708206687,"count":1645,"max":47784866,"min":1685694,"p50":2981887,"p99":3735551,"p999":23855103} passed=true +# 2026-10-27 scan_32t host-container ops/s=9528.6 latency_ns={"avg":3354508.0670402786,"count":47643,"max":17309319,"min":1527312,"p50":3276799,"p99":6619135,"p999":11141119} passed=true +# 2026-10-27 write_16t host-container ops/s=9336.6 latency_ns={"avg":1712016.3134331557,"count":46683,"max":10740314,"min":1070495,"p50":1622015,"p99":2850815,"p999":9109503} passed=true +# 2026-10-27 write_1t host-container ops/s=615.2 latency_ns={"avg":1623120.0812743823,"count":3076,"max":2441336,"min":833972,"p50":1646591,"p99":1867775,"p999":1998847} passed=true +# 2026-10-27 write_64t host-container ops/s=28933.8 latency_ns={"avg":2209824.375146023,"count":144669,"max":23462877,"min":1225008,"p50":2113535,"p99":3342335,"p999":15728639} passed=true diff --git a/tools/cpp-checks/sanitize-regression.sh b/tools/cpp-checks/sanitize-regression.sh index 327f90250..697adc55d 100755 --- a/tools/cpp-checks/sanitize-regression.sh +++ b/tools/cpp-checks/sanitize-regression.sh @@ -71,6 +71,10 @@ set -euo pipefail cd "$(dirname "$0")/../.." unset CROWDB_ASAN +: "${CROWDB_SANITIZER_BASE_IMAGE:?set a local CrowDB OCI base image for Paxos-KV sanitizer containers}" +if [[ "${1:-}" == --paxos-kv-only ]]; then + exec pixi run bash tools/cpp-checks/sanitize-paxos-kv.sh +fi RUNTIME_ROOT="${CROWDB_RUNTIME_ROOT:-${PIXI_PROJECT_ROOT:-$(pwd)}/.crowdb-runtime}" RUNTIME_DIR="$RUNTIME_ROOT/artifacts/sanitize-regression-$(date +%Y%m%d-%H%M%S)-$$" @@ -79,7 +83,6 @@ RESULTS_FILE="$RUNTIME_DIR/sanitize-regression.tsv" DURATION=5 KEYSPACE=1000 VALUE_SIZE=128 -PREPARE_KEYS=500 GATE_FAILED=0 # Path to the debug binary (built with CROWDB_ASAN=1). @@ -92,7 +95,7 @@ CROWDB_CLI="$(cd "$(dirname "$0")/../.." && pwd)/target/debug/crowdb-cli" LIBASAN="$(cd "$(dirname "$0")/../.." && pixi run -- pwd)/.pixi/envs/default/lib/libasan.so" restore_binaries() { - pixi run cargo build -p crowdb-cli -p crowdb-rpc-test -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb + pixi run cargo build -p crowdb-cli -p crowdb-rpc-test -p crowdb-paxos-kv-server -p crowdb-diskdb -p crowdb-chunkdb pixi run cargo build -p crowdb-console-shared --example regression-fixture --features test-util } @@ -112,7 +115,7 @@ trap cleanup_sanitize EXIT # --- Phase 1: Build with ASan + LSan enabled --- echo "=== building with CROWDB_ASAN=1 (debug) ===" -CROWDB_ASAN=1 pixi run -- cargo build -p crowdb-cli -p crowdb-rpc-test -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb +CROWDB_ASAN=1 pixi run -- cargo build -p crowdb-cli -p crowdb-rpc-test -p crowdb-paxos-kv-server -p crowdb-diskdb -p crowdb-chunkdb CROWDB_ASAN=1 pixi run cargo build -p crowdb-console-shared --example regression-fixture --features test-util pixi run -- cmake --build app/crowdb-diskio/build -j RPC_ASAN_BUILD="$RUNTIME_DIR/rpc-server-build" @@ -137,7 +140,6 @@ asan_client() { local log_prefix="$1" state="$2"; shift 2 local endpoint endpoint=$(pixi run python tools/benchmark/fixture.py --state "$state" target) - rm -f "$(dirname "$state")/client-runtime/artifacts/cli/bench-kv/report.json" CROWDB_RUNTIME_ROOT="$(dirname "$state")/client-runtime" pixi run env -u CROWDB_CLUSTER ASAN_OPTIONS="detect_leaks=1:abort_on_error=0:log_path=${log_prefix}" \ LD_PRELOAD="$LIBASAN" "$CROWDB_CLI" --management-endpoint "$endpoint" "$@" } @@ -193,115 +195,6 @@ bench_status() { printf '%s' "$status" } -# run_bench <deploy_name> <subcmd> <threads> <conn> <label> <extra_args...> -# Cleans user data (except for read/scan which need prepare first), -# runs the workload under ASan, checks for leaks. -run_bench() { - local deploy="$1" subcmd="$2" threads="$3" conn="$4" label="$5"; shift 5 - local extra_args=("$@") - echo ">>> $label ..." - local config_file - config_file=$(cat "${RUNTIME_DIR}/sanitize-reg-${deploy}.cfgpath" 2>/dev/null || echo "") - if [ -z "$config_file" ] || [ ! -f "$config_file" ]; then - echo " ERROR: no config for deploy '$deploy'" - echo -e "$label\t0\t0\tFAIL(config)" >> "$RESULTS_FILE" - return - fi - local log_prefix="${RUNTIME_DIR}/asan-sanitize-${label}" - local output rc - if output=$(asan_client "$log_prefix" "$config_file" \ - bench kv "$subcmd" --duration-secs "$DURATION" \ - --loader-num "$threads" --connections "$conn" \ - --key-space "$KEYSPACE" --value-size "$VALUE_SIZE" \ - "${extra_args[@]}" 2>&1); then - rc=0 - else - rc=$? - echo "$output" >&2 - fi - local json; json=$(pixi run python tools/benchmark/fixture.py --state "$config_file" report) || json="" - if [ -z "$json" ]; then - echo " ERROR: no JSON output"; echo "$output" - echo -e "$label\t0\t0\tFAIL(output)" >> "$RESULTS_FILE" - return - fi - local total_ops ops_s errors - total_ops=$(echo "$json" | jq -r '.total_ops') - ops_s=$(echo "$json" | jq -r '.total_ops * 1000 / .duration_ms' | awk '{printf "%.2f", $1}') - errors=$(echo "$json" | jq -r '.total_errors') - # Check for ASan leak logs from the client process. - local leak_status="PASS" - local logs - logs=$(ls "${log_prefix}."* 2>/dev/null || true) - if [ -n "$logs" ]; then - leak_status="FAIL(leaks)" - for log in $logs; do - local summary - summary=$(grep "SUMMARY:" "$log" 2>/dev/null || echo "(no summary)") - echo " LEAK: $summary" - cat "$log" - done - fi - leak_status=$(bench_status "$leak_status" "$rc" "$errors" "$ops_s") - echo " ops/s=$ops_s err=$errors leak_check=$leak_status" - echo -e "$label\t$ops_s\t$errors\t$leak_status" >> "$RESULTS_FILE" -} - -# run_prepare <deploy_name> <label> -# Pre-populates keys so read/scan have data to work with. -run_prepare() { - local deploy="$1" label="$2" - echo ">>> $label ..." - local config_file - config_file=$(cat "${RUNTIME_DIR}/sanitize-reg-${deploy}.cfgpath" 2>/dev/null || echo "") - if [ -z "$config_file" ] || [ ! -f "$config_file" ]; then - echo " ERROR: no config for deploy '$deploy'" - GATE_FAILED=1 - return - fi - local log_prefix="${RUNTIME_DIR}/asan-sanitize-${label}" - local output rc - if output=$(asan_client "$log_prefix" "$config_file" \ - bench kv prepare --keys "$PREPARE_KEYS" \ - --value-size "$VALUE_SIZE" --concurrency 4 \ - 2>&1); then - rc=0 - else - rc=$? - echo "$output" >&2 - fi - echo " exit=$rc" - if [ "$rc" -ne 0 ]; then - GATE_FAILED=1 - fi - local line written errors - line=$(echo "$output" | sed -n '/^bench kv prepare:/p' | tail -n 1) - written=$(echo "$line" | sed -n 's/^bench kv prepare: \([0-9][0-9]*\) keys written.*/\1/p') - errors=$(echo "$line" | sed -n 's/.*written, \([0-9][0-9]*\) errors.*/\1/p') - if [[ ! "$written" =~ ^[0-9]+$ ]] || [ "$written" -ne "$PREPARE_KEYS" ] \ - || [[ ! "$errors" =~ ^[0-9]+$ ]] || [ "$errors" -ne 0 ]; then - echo " ERROR: prepare produced invalid output (written=$written errors=$errors)" - GATE_FAILED=1 - fi - if ! check_asan_logs "$log_prefix" "$label"; then - GATE_FAILED=1 - fi -} - -# deploy_cluster <name> -# Deploy a 3-node cluster with default tunables for sanitize testing. -deploy_cluster() { - local name="$1" - local config_file="${RUNTIME_DIR}/sanitize-reg-${name}.toml" - echo "=== deploying cluster '$name' ===" - rm -f "$config_file" - local log_prefix="${RUNTIME_DIR}/asan-sanitize-deploy-${name}" - asan_fixture "$log_prefix" "$config_file" --nodes 3 --kind kv \ - --kv-backend mem-block --wal-backend mem-block 2>&1 - echo "$config_file" > "${RUNTIME_DIR}/sanitize-reg-${name}.cfgpath" - -} - # teardown_cluster <name> # Destroy the cluster — servers get SIGTERM → graceful shutdown → ASan # leak check on each server process. No ASan logs = no leaks. @@ -678,36 +571,9 @@ run_chunkio_small_write_bench() { echo -e "label\tops_s\terrors\tleak_check" > "$RESULTS_FILE" -# --- Phase 2a: KV-only cluster (write/read/scan) --- - -# Single deploy for all KV sub-tests. -DEPLOY="sanitize-reg-$$-$(date +%s)" -deploy_cluster "$DEPLOY" - -# Prepare: pre-populate keys so read/scan have data. -run_prepare "$DEPLOY" "prepare_${PREPARE_KEYS}keys" - -# Write: put workload (consensus + WAL + storage). -echo "=== write ===" -run_bench "$DEPLOY" write 1 1 "write_1t_1c" -run_bench "$DEPLOY" write 16 2 "write_16t_2c" -run_bench "$DEPLOY" write 64 4 "write_64t_4c" - -# Read: point-get workload (linearizable + minslot). -echo "=== read ===" -run_bench "$DEPLOY" read 1 1 "read_1t_1c_linearizable" --read-mode linearizable -run_bench "$DEPLOY" read 16 2 "read_16t_2c_linearizable" --read-mode linearizable -run_bench "$DEPLOY" read 32 32 "read_32t_32c_linearizable" --read-mode linearizable -run_bench "$DEPLOY" read 1 1 "read_1t_1c_minslot" --read-mode minslot - -# Scan: range scan workload. -echo "=== scan ===" -run_bench "$DEPLOY" scan 1 1 "scan_1t_1c" -run_bench "$DEPLOY" scan 16 2 "scan_16t_2c" -run_bench "$DEPLOY" scan 32 32 "scan_32t_32c" - -# Teardown: destroy KV cluster, check server shutdown leaks. -teardown_cluster "$DEPLOY" +# --- Phase 2a: Paxos-KV containers (same write/read/scan matrix) --- +# The diagnostic fixture owns containers; product bench only injects load. +KV_BENCH_LOG_ROOT="$RUNTIME_DIR/paxos-kv" pixi run bash tools/cpp-checks/sanitize-paxos-kv.sh --built # --- Phase 2b: Combined cluster (diskdb + chunkdb + chunkio) --- diff --git a/tools/cpp-checks/tree-link-isolation.sh b/tools/cpp-checks/tree-link-isolation.sh index cea975e73..0b3c6a272 100755 --- a/tools/cpp-checks/tree-link-isolation.sh +++ b/tools/cpp-checks/tree-link-isolation.sh @@ -25,11 +25,25 @@ link_libraries=( "$build_dir/crowdb-common-build/libcrowdbcommon.a" -luring -llz4 -lspdlog -lfmt -lz -lisal -lfolly -lglog -lgflags -levent -ldl -pthread ) +if [[ $(sed -n 's/^CROWDB_RPC_ENABLE_RDMA:BOOL=//p' "$build_dir/CMakeCache.txt") == ON ]]; then + ibverbs_library=$(sed -n 's/^IBVERBS_LIB:FILEPATH=//p' "$build_dir/CMakeCache.txt") + rdmacm_library=$(sed -n 's/^RDMACM_LIB:FILEPATH=//p' "$build_dir/CMakeCache.txt") + if [[ -f "$ibverbs_library" && -f "$rdmacm_library" ]]; then + link_libraries+=("$ibverbs_library" "$rdmacm_library") + fi +fi -for compiler in g++ clang++; do - command -v "$compiler" >/dev/null - "$compiler" -std=c++20 -fsyntax-only "${include_flags[@]}" "$fixture_dir/public_header.cpp" -done +g++ -std=c++20 -fsyntax-only "${include_flags[@]}" "$fixture_dir/public_header.cpp" +if command -v clang++ >/dev/null; then + clang_cxx=(clang++) +elif command -v clang-23 >/dev/null; then + # The locked Pixi environment provides the versioned Clang driver. + clang_cxx=(clang-23 --driver-mode=g++) +else + echo "tree link isolation requires clang++ or the Pixi clang-23 driver" >&2 + exit 1 +fi +"${clang_cxx[@]}" -std=c++20 -fsyntax-only "${include_flags[@]}" "$fixture_dir/public_header.cpp" c++ -std=c++20 "${include_flags[@]}" "$fixture_dir/local.cpp" "${link_libraries[@]}" \ -Wl,-Map="$probe_dir/tree-link-local.map" -o "$probe_dir/tree-link-local" diff --git a/tools/cpp-checks/tree-link-isolation/public_header.cpp b/tools/cpp-checks/tree-link-isolation/public_header.cpp index 19b431c04..a8bc14108 100644 --- a/tools/cpp-checks/tree-link-isolation/public_header.cpp +++ b/tools/cpp-checks/tree-link-isolation/public_header.cpp @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. #include "crowdb-tree/crowdb-tree.h" +#include "crowdb-tree/split_overlay_c_api.h" int main() { diff --git a/tools/load-tpch.py b/tools/load-tpch.py index 90d49d213..0d4d63ce5 100644 --- a/tools/load-tpch.py +++ b/tools/load-tpch.py @@ -21,11 +21,13 @@ REPO = Path(__file__).resolve().parents[1] ARTIFACTS = REPO / ".crowdb-runtime/artifacts/tpc-loader" +sys.path.insert(0, str(REPO / "tools/ports")) +from default_ports import WEB_BASE def arguments(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--console-url", default="http://127.0.0.1:9090") + parser.add_argument("--console-url", default=f"http://127.0.0.1:{WEB_BASE}") parser.add_argument("--sf", default="1", help="TPC-H scale factor (default: 1)") parser.add_argument("--namespace", help="new Iceberg namespace; default is a unique tpch_sf1_* name") parser.add_argument("--bucket", help="S3 bucket; default is the namespace with underscores replaced by hyphens") diff --git a/tools/pixi-tasks/build-release.sh b/tools/pixi-tasks/build-release.sh index ea287e9ba..aa753370e 100644 --- a/tools/pixi-tasks/build-release.sh +++ b/tools/pixi-tasks/build-release.sh @@ -5,7 +5,7 @@ set -euo pipefail cd "${PIXI_PROJECT_ROOT:?}" bash tools/pixi-tasks/build-cpp.sh -cargo build --release --workspace --exclude crowdb-kv-client -cargo build --release -p crowdb-kv-client --features ffi +cargo build --release --workspace --exclude crowdb-paxos-kv-client +cargo build --release -p crowdb-paxos-kv-client --features ffi (cd app/crowdb-web/ui && npm run build) cargo test --workspace --all-targets --no-run diff --git a/tools/pixi-tasks/test.py b/tools/pixi-tasks/test.py index ba3469d28..3b9426b74 100644 --- a/tools/pixi-tasks/test.py +++ b/tools/pixi-tasks/test.py @@ -13,9 +13,9 @@ ROOT = Path(__file__).resolve().parents[2] RUST_GROUPS = { - "core": ("common", "test-harness", "protocol", "kv", "kv-client"), + "core": ("common", "test-harness", "protocol", "paxos-kv", "paxos-kv-client"), "storage": ("chunkdb-client", "chunk-kv", "chunk-kv-client", "chunk-kv-server", - "chunk-stream", "kv-server", "diskdb", "diskdb-client", "chunkdb", + "chunk-stream", "paxos-kv-server", "diskdb", "diskdb-client", "chunkdb", "chunk-client", "diskio-client"), "access": ("access-multipart", "access-s3", "sdk-native", "sdk-native-python", "access-iceberg", "access-server", "access-dataset", "monitor"), @@ -111,7 +111,7 @@ def unit(group): if group == "console": # Test-only native measurement ownership is separate from product OCI acceptance. measurement.run("console", ["cargo", "build", "--locked", "-p", "crowdb-cli", - "-p", "crowdb-kv-server", "-p", "crowdb-console-shared", + "-p", "crowdb-paxos-kv-server", "-p", "crowdb-console-shared", "--features", "crowdb-console-shared/test-util", "--bins", "--example", "regression-fixture"], env=rust_test_environment()) measurement.run("console", ["python", "tools/benchmark/fixture-smoke.py", "--worker", diff --git a/tools/pixi-tasks/tree-asan.sh b/tools/pixi-tasks/tree-asan.sh index 6a5bb935f..91c738f19 100644 --- a/tools/pixi-tasks/tree-asan.sh +++ b/tools/pixi-tasks/tree-asan.sh @@ -4,10 +4,13 @@ set -euo pipefail cd "${PIXI_PROJECT_ROOT:?}" -cmake -S lib/crowdb-tree -B lib/crowdb-tree/build-asan -DCMAKE_BUILD_TYPE=Debug -DCROWDB_TREE_SANITIZER=address -cmake --build lib/crowdb-tree/build-asan -j +cmake -S lib/crowdb-tree -B lib/crowdb-tree/build-asan \ + -DCMAKE_BUILD_TYPE=Debug -DCROWDB_TREE_SANITIZER=address \ + -DCROWDB_TREE_TEST_UTIL=ON -DCROWDB_RPC_TEST_UTIL=ON +# RPC is excluded from the default build, but its tests are registered in CTest. +cmake --build lib/crowdb-tree/build-asan --parallel --target crowdb_tree_tests crowdbcommon_tests crowdb_rpc_tests if command -v setarch > /dev/null 2>&1; then - setarch -R ctest --test-dir lib/crowdb-tree/build-asan --output-on-failure + setarch -R ctest --test-dir lib/crowdb-tree/build-asan --output-on-failure --stop-on-failure else - ctest --test-dir lib/crowdb-tree/build-asan --output-on-failure + ctest --test-dir lib/crowdb-tree/build-asan --output-on-failure --stop-on-failure fi diff --git a/tools/pixi-tasks/tree-tsan.sh b/tools/pixi-tasks/tree-tsan.sh index 3729a9204..f0b515800 100644 --- a/tools/pixi-tasks/tree-tsan.sh +++ b/tools/pixi-tasks/tree-tsan.sh @@ -4,10 +4,20 @@ set -euo pipefail cd "${PIXI_PROJECT_ROOT:?}" -cmake -S lib/crowdb-tree -B lib/crowdb-tree/build-tsan -DCMAKE_BUILD_TYPE=Debug -DCROWDB_TREE_SANITIZER=thread -cmake --build lib/crowdb-tree/build-tsan -j +# GCC cannot instrument the atomic fences in Folly. Use the pinned pixi Clang +# and a separate cache so an earlier GCC build cannot retain its toolchain. +# Optimize sanitizer instrumentation while retaining Debug assertions. +build_dir=lib/crowdb-tree/build-tsan-clang23 +cmake -S lib/crowdb-tree -B "$build_dir" \ + -DCMAKE_C_COMPILER="${CONDA_PREFIX:?}/bin/clang-23" \ + -DCMAKE_CXX_COMPILER="$CONDA_PREFIX/bin/clang++-23" \ + -DCMAKE_BUILD_TYPE=Debug -DCROWDB_TREE_SANITIZER=thread \ + -DCMAKE_C_FLAGS_DEBUG='-O1 -g' -DCMAKE_CXX_FLAGS_DEBUG='-O1 -g' \ + -DCROWDB_TREE_TEST_UTIL=ON -DCROWDB_RPC_TEST_UTIL=ON +# RPC is excluded from the default build, but its tests are registered in CTest. +cmake --build "$build_dir" --parallel --target crowdb_tree_tests crowdbcommon_tests crowdb_rpc_tests if command -v setarch > /dev/null 2>&1; then - setarch -R ctest --test-dir lib/crowdb-tree/build-tsan --output-on-failure + setarch -R ctest --test-dir "$build_dir" --output-on-failure --stop-on-failure else - ctest --test-dir lib/crowdb-tree/build-tsan --output-on-failure + ctest --test-dir "$build_dir" --output-on-failure --stop-on-failure fi diff --git a/tools/pixi-tasks/tree-ubsan.sh b/tools/pixi-tasks/tree-ubsan.sh index a6e239a2b..2c0018a8b 100644 --- a/tools/pixi-tasks/tree-ubsan.sh +++ b/tools/pixi-tasks/tree-ubsan.sh @@ -4,11 +4,16 @@ set -euo pipefail cd "${PIXI_PROJECT_ROOT:?}" -cmake -S lib/crowdb-tree -B lib/crowdb-tree/build-ubsan -DCMAKE_BUILD_TYPE=Debug -DCROWDB_TREE_SANITIZER=undefined -cmake --build lib/crowdb-tree/build-ubsan -j +cmake -S lib/crowdb-tree -B lib/crowdb-tree/build-ubsan \ + -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS_DEBUG="-O1 -g" \ + -DCROWDB_TREE_SANITIZER=undefined \ + -DCROWDB_TREE_TEST_UTIL=ON -DCROWDB_RPC_TEST_UTIL=ON +# RPC tests are registered with CTest but excluded from the default build. +cmake --build lib/crowdb-tree/build-ubsan --parallel 4 \ + --target crowdb_tree_tests crowdbcommon_tests crowdb_rpc_tests export UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 if command -v setarch > /dev/null 2>&1; then - setarch -R ctest --test-dir lib/crowdb-tree/build-ubsan --output-on-failure + setarch -R ctest --test-dir lib/crowdb-tree/build-ubsan --output-on-failure --stop-on-failure else - ctest --test-dir lib/crowdb-tree/build-ubsan --output-on-failure + ctest --test-dir lib/crowdb-tree/build-ubsan --output-on-failure --stop-on-failure fi diff --git a/tools/ports/README.md b/tools/ports/README.md new file mode 100644 index 000000000..9b8dfa55c --- /dev/null +++ b/tools/ports/README.md @@ -0,0 +1,26 @@ +<!-- Copyright 2026-present Gian <crow.db@outlook.com> --> +<!-- Licensed under the Apache License, Version 2.0. --> + +# Default ports + +All service port values are defined in +[ports.rs](../../lib/crowdb-protocol/src/port/ports.rs). Rust callers import +its constants or `ServicePort`. Explicit deployment inputs override defaults. + +After editing the defaults, run: + +```sh +pixi run python tools/ports/generate.py +pixi run python tools/ports/generate.py --check +``` + +The generator produces Python, TypeScript and C++ constants, plus container +profiles, Compose examples, Dockerfile and sample configuration. Edit the +adjacent `*.ports.in` source for those files; `@WEB_BASE@` references a constant +and `@WEB_BASE+1@` adds a node offset. Generated files are checked in so they +can be used directly. OCI staging and the packaging tests reject stale outputs. + +Each active service listener owns a 1000-port range. Three host-network nodes +with one listener of each kind use offsets 0, 1 and 2. Multiple Paxos-KV stores +need distinct RPC ports from the node's configured pool; offsets must account +for that pool. No range is reserved for a nonexistent listener. diff --git a/tools/ports/default_ports.py b/tools/ports/default_ports.py new file mode 100644 index 000000000..a8a2f56e5 --- /dev/null +++ b/tools/ports/default_ports.py @@ -0,0 +1,22 @@ +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. + +"""Generated from ports.rs by tools/ports/generate.py; do not edit.""" + +PAXOS_KV_SERVER_MGMT_BASE = 10135 +PAXOS_KV_SERVER_RPC_BASE = 11135 +DISKDB_HTTP_BASE = 13135 +DISKDB_RPC_BASE = 14135 +CHUNKDB_HTTP_BASE = 16135 +CHUNKDB_RPC_BASE = 17135 +DISKIO_RPC_BASE = 18135 +WEB_BASE = 6135 +CHUNK_KV_HTTP_BASE = 20135 +CHUNK_KV_RPC_BASE = 21135 +ACCESS_SERVER_HTTP_BASE = 7135 +ACCESS_SERVER_ICEBERG_HTTP_BASE = 8135 +ACCESS_SERVER_DATASET_HTTP_BASE = 9135 +ACCESS_SERVER_HEALTH_HTTP_BASE = 25135 +MONITOR_HTTP_BASE = 26135 +SSH_PORT = 2222 +HOST_SSH_PORT = 22 diff --git a/tools/ports/generate.py b/tools/ports/generate.py new file mode 100644 index 000000000..ef85e76d6 --- /dev/null +++ b/tools/ports/generate.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +"""Generate non-Rust defaults from protocol's ports.rs; --check never writes.""" + +import argparse +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "lib/crowdb-protocol/src/port/ports.rs" +TOKEN = re.compile(r"@([A-Z][A-Z_]+)(?:\+([0-9]+))?@") +NOTICE = "Generated by tools/ports/generate.py from ports.rs; edit the .in template." + + +def outputs(): + ports = {name: int(value) for name, value in re.findall( + r"^pub const ([A-Z_]+): u16 = ([0-9]+);$", SOURCE.read_text(), re.MULTILINE)} + if not ports: + raise ValueError(f"no port definitions in {SOURCE}") + copyright = "Copyright 2026-present Gian <crow.db@outlook.com>\nLicensed under the Apache License, Version 2.0." + python = "\n".join("# " + line for line in copyright.splitlines()) + python += '\n\n"""Generated from ports.rs by tools/ports/generate.py; do not edit."""\n\n' + python += "".join(f"{name} = {value}\n" for name, value in ports.items()) + yield ROOT / "container/oci-image/ports.py", python + yield ROOT / "e2e-test/default_ports.py", python + yield ROOT / "tools/ports/default_ports.py", python + typescript = "\n".join("// " + line for line in copyright.splitlines()) + typescript += "\n\n// Generated from ports.rs by tools/ports/generate.py; do not edit.\n" + typescript += "".join(f"export const {name} = {value};\n" for name, value in ports.items()) + yield ROOT / "app/crowdb-web/ui/src/ports.ts", typescript + header = "\n".join("// " + line for line in copyright.splitlines()) + header += "\n\n// Generated from ports.rs by tools/ports/generate.py; do not edit.\n#pragma once\n\n#include <cstdint>\n\nnamespace crowdb::protocol\n{\n" + header += "".join(f"inline constexpr std::uint16_t {name:<{max(map(len, ports))}} = {value};\n" for name, value in ports.items()) + header += "} // namespace crowdb::protocol\n" + yield ROOT / "lib/crowdb-protocol/cpp/include/crowdb-protocol/ports.h", header + templates = sorted((ROOT / "container").rglob("*.ports.in")) + templates += sorted((ROOT / "app").glob("*/conf/*.ports.in")) + for template in templates: + def substitute(match): + value = ports[match[1]] + int(match[2] or 0) + if not 1 <= value <= 65535: + raise ValueError(f"invalid port {match[0]} in {template}") + return str(value) + text = TOKEN.sub(substitute, template.read_text()) + lines = text.splitlines(keepends=True) + index = 1 if lines and lines[0].startswith("#!") else 0 + lines.insert(index, "# " + NOTICE + "\n") + yield template.with_name(template.name.removesuffix(".ports.in")), "".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + stale = [] + for path, text in outputs(): + if path.exists() and path.read_text() == text: + continue + if args.check: + stale.append(str(path.relative_to(ROOT))) + else: + path.write_text(text) + if stale: + parser.exit(1, "Stale port defaults; run pixi run python tools/ports/generate.py:\n" + "\n".join(stale) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tools/profiling/container-server-pids.py b/tools/profiling/container-server-pids.py new file mode 100644 index 000000000..ac7b39808 --- /dev/null +++ b/tools/profiling/container-server-pids.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +"""Resolve host PIDs for Paxos-KV servers in an external Docker deployment.""" +import argparse +import hashlib +import json +from pathlib import Path +import shlex +import subprocess + + +def server_pid(process_table): + matches = [] + for row in process_table.splitlines()[1:]: + fields = row.split(None, 1) + if len(fields) != 2: + continue + args = shlex.split(fields[1]) + if args and Path(args[0]).name == 'crowdb-paxos-kv-server': + matches.append(int(fields[0])) + if len(matches) != 1 or matches[0] <= 0: + raise ValueError(f'expected exactly one live Paxos-KV server; found {matches}') + return matches[0] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--state', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + state = json.loads(args.state.read_text()) + names = state['containers'] + if len(names) != 3 or len(set(names)) != 3: + raise ValueError('expected three distinct external containers') + records = [] + mapped_files = {} + for name in names: + found = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True, timeout=60))[0] + labels = found['Config'].get('Labels') or {} + if not found['State']['Running'] or labels.get('crowdb.paxos-kv.deployment') != state['id']: + raise ValueError(f'container not running or deployment identity differs: {name}') + table = subprocess.check_output(['docker', 'top', name, '-eo', 'pid,args'], text=True, timeout=60) + pid = server_pid(table) + # Container executable paths do not exist in the host mount namespace. + # Keep the matching binary and mapped libraries for stack symbolization. + executable = next(shlex.split(row.split(None, 1)[1])[0] + for row in table.splitlines()[1:] + if row.split(None, 1)[0] == str(pid)) + symbols = args.output.parent / 'symbols' + symbols.mkdir(exist_ok=True) + binary = symbols / f'{name}-crowdb-paxos-kv-server' + subprocess.run(['docker', 'cp', f'{name}:{executable}', str(binary)], check=True, timeout=60) + for row in Path(f'/proc/{pid}/maps').read_text().splitlines(): + fields = row.split(None, 5) + if len(fields) == 6 and 'x' in fields[1] and fields[5].startswith('/'): + mapped_files.setdefault(fields[5], name) + records.append({'name': name, 'pid': pid, 'host_uid': Path(f'/proc/{pid}').stat().st_uid, + 'executable': executable, 'symbol_binary': str(binary), + 'binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), + 'inspect': found, 'processes': table}) + if len({record['inspect']['Image'] for record in records}) != 1 or len({record['binary_sha256'] for record in records}) != 1: + raise ValueError('combined server profile requires matching server images and binaries') + # Some release binaries have symbols but no GNU build-id. A private symfs + # also resolves those binaries and the exact container DSOs, without using + # possibly different host libraries or modifying the user's perf cache. + rootfs = args.output.parent / 'symbols' / 'rootfs' + for path, name in mapped_files.items(): + if path.endswith(' (deleted)'): + raise ValueError(f'cannot archive a replaced executable mapping: {path}') + target = rootfs / path.lstrip('/') + target.parent.mkdir(parents=True, exist_ok=True) + subprocess.run(['docker', 'cp', '-L', f'{name}:{path}', str(target)], check=True, timeout=60) + args.output.write_text(json.dumps(records, indent=2) + '\n') + print(','.join(str(record['pid']) for record in records)) + + +if __name__ == '__main__': + main() diff --git a/tools/profiling/profile-write.sh b/tools/profiling/profile-write.sh index 0d70da2ba..ef5bf90e6 100755 --- a/tools/profiling/profile-write.sh +++ b/tools/profiling/profile-write.sh @@ -1,171 +1,105 @@ #!/usr/bin/env bash -# --- CrowDB write profiling with flamechart output --- -# -# Builds crowdb-cli + crowdb-kv-server with debug symbols and frame -# pointers (Rust + C++ via cc crate), then profiles the attached write client -# under perf record (primary, generates flamegraph SVG) or -# samply (alternative, Firefox Profiler UI). -# -# Usage: -# bash tools/profiling/profile-write.sh [sampler] [duration] -# -# sampler - perf (default) | samply -# duration - benchmark duration in seconds (default 15) -# -# Output: -# samply -> opens Firefox Profiler tab (also saves to -# .crowdb-runtime/artifacts/profile/profile-write-<timestamp>.zip) -# perf -> perf.data + folded stacks + flamegraph SVG under the same -# runtime artifact directory. -# -# Prerequisites: -# - pixi installed, project deps resolved -# - samply + inferno: installed via `pixi run install-deps` -# - perf: linux-tools for running kernel +# Copyright 2026-present Gian <crow.db@outlook.com> +# Licensed under the Apache License, Version 2.0. +# Profile servers in three prestarted host-network containers (or the host client). +# Usage: KV_BENCH_DEPLOYMENT=state.json pixi run bash tools/profiling/profile-write.sh [perf|samply] [20] +# KV_PROFILE_TARGET=client selects host-client sampling; the default is servers. +# Server sampling requires host permission to attach to each server UID. The +# script archives container binaries/DSOs for symbolization; it changes no permissions. +# For client sampling, supply KV_BENCH_CLI with symbols. Build separately with +# CARGO_PROFILE_RELEASE_DEBUG=line-tables-only pixi run cargo build --release -p crowdb-cli. +# The sample includes preparation/verification; report.json measures only the middle window. +# This script never builds, starts, stops, resets or changes the server deployment. set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." -unset CROWDB_ASAN - -if [ "$(uname -s)" = "Darwin" ]; then - DEFAULT_SAMPLER="samply" -else - DEFAULT_SAMPLER="perf" -fi -SAMPLER="${1:-$DEFAULT_SAMPLER}" -DURATION="${2:-15}" -RUNTIME_ROOT="${CROWDB_RUNTIME_ROOT:-${PIXI_PROJECT_ROOT:-$(pwd)}/.crowdb-runtime}" -RESULTS_DIR="$RUNTIME_ROOT/artifacts/profile" -TIMESTAMP=$(date +%Y%m%d-%H%M%S) - -# Write config: 24T:24C max-inflight=32 -THREADS=24 -CONNECTIONS=24 -MAX_INFLIGHT=32 -KEYSPACE=1000000 -VALUE_SIZE=512 - -echo "=== CrowDB write profiling ($SAMPLER) ===" -echo "Config: ${THREADS}T:${CONNECTIONS}C max-inflight=${MAX_INFLIGHT} ${DURATION}s" -echo "" - -# ── Step 1: Build with debug symbols + frame pointers ── -echo ">>> Building with debug symbols + frame pointers..." -export CARGO_PROFILE_RELEASE_DEBUG="line-tables-only" -export RUSTFLAGS="-Cforce-frame-pointers=yes" -export CXXFLAGS="-g -fno-omit-frame-pointer" -export CFLAGS="-g -fno-omit-frame-pointer" - -pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -echo " Done." -echo "" - -# Verify debug symbols -if ! readelf -S target/release/crowdb-kv-server 2>/dev/null | grep -q debug; then - echo "WARNING: crowdb-kv-server binary has no debug sections." - echo " Flamecharts will show addresses, not function names." -fi - -# ── Step 2: Profile ── -# Measurement services have an explicit owner independent of the product CLI. -REGRESSION_LOG_ROOT="${RESULTS_DIR}/profile-write-${TIMESTAMP}-fixture-$$" -source tools/benchmark/bench-regression-common.sh -regression_init -trap regression_destroy EXIT -regression_fixture setup -- --kind kv --nodes 3 --kv-backend mem-block --wal-backend mem-block \ - --max-inflight "$MAX_INFLIGHT" --metrics-interval 1 -regression_cli kv group add -s 0 -g 1 -n 1,2,3 -TARGET=$(pixi run python tools/benchmark/fixture.py --state "$REGRESSION_CONFIG" target) -CLI_BIN="$(pwd)/target/release/crowdb-cli" -export CROWDB_RUNTIME_ROOT="$REGRESSION_LOG_ROOT/client-runtime" -unset CROWDB_CLUSTER -mkdir -p "$RESULTS_DIR" - -if [ "$SAMPLER" = "samply" ]; then - if ! command -v samply &>/dev/null; then - echo "ERROR: samply not found. Install with: cargo install samply" - exit 1 +SAMPLER="${1:-perf}" +PROFILE_TARGET="${KV_PROFILE_TARGET:-servers}" +PROFILE_FREQUENCY="${KV_PROFILE_FREQUENCY:-99}" +export KV_BENCH_DURATION="${2:-${KV_BENCH_DURATION:-20}}" +export KV_BENCH_LOG_ROOT="${KV_BENCH_LOG_ROOT:-$PWD/.crowdb-runtime/artifacts/profile/write-$(date +%Y%m%d-%H%M%S)-$$}" +source tools/benchmark/bench-kv-common.sh +PERF=(perf --buildid-dir "$LOG_ROOT/buildid-cache") +[[ $(jq -r '.kind // "container"' "$KV_BENCH_DEPLOYMENT") == container ]] || { + echo 'Profiling requires a container deployment' >&2; exit 2; +} +case "$SAMPLER" in + perf) + command -v perf >/dev/null + if [[ "$PROFILE_TARGET" == client ]]; then + BENCH_PREFIX=("${PERF[@]}" record -F "$PROFILE_FREQUENCY" --call-graph dwarf -o "$LOG_ROOT/perf.data" --) + elif [[ "$PROFILE_TARGET" != servers ]]; then + echo 'KV_PROFILE_TARGET must be servers or client' >&2; exit 2 + fi + ;; + samply) + [[ "$PROFILE_TARGET" == client ]] || { + echo 'Container server sampling uses perf; select KV_PROFILE_TARGET=client for samply' >&2; exit 2; + } + command -v samply >/dev/null + BENCH_PREFIX=(samply record --save-only -o "$LOG_ROOT/profile.json.gz" --) + ;; + *) echo 'Expected perf or samply' >&2; exit 2 ;; +esac +sampler_pid="" +stop_sampler() { + [[ -n "$sampler_pid" ]] || return 0 + local status=0 interrupted=0 + if kill -INT "$sampler_pid" 2>/dev/null; then interrupted=1; fi + wait "$sampler_pid" || status=$? + sampler_pid="" + cat "$LOG_ROOT/perf-record.stderr" >&2 + # perf may return 128+SIGINT after successfully finalizing the recording. + # Accept this only when we requested its normal stop, not for other errors. + if [[ "$interrupted" == 1 && "$status" == 130 ]]; then status=0; fi + return "$status" +} +cleanup() { + local status=$? + trap - EXIT + stop_sampler || status=1 + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +if [[ "$PROFILE_TARGET" == servers ]]; then + # Docker's init PID may be a launcher. Resolve the actual server executable + # in each container, and retain the process/image identity with the profile. + server_pids=$(python tools/profiling/container-server-pids.py \ + --state "$KV_BENCH_DEPLOYMENT" --output "$LOG_ROOT/server-processes.json") + "${PERF[@]}" record -F "$PROFILE_FREQUENCY" --call-graph dwarf --pid "$server_pids" -o "$LOG_ROOT/perf.data" \ + > "$LOG_ROOT/perf-record.stdout" 2> "$LOG_ROOT/perf-record.stderr" & + sampler_pid=$! + sleep .2 + if ! kill -0 "$sampler_pid" 2>/dev/null; then + stop_sampler || true + echo 'Server profiler exited before load; inspect perf-record.stderr' >&2; exit 1 fi - - OUTPUT="${RESULTS_DIR}/profile-write-${TIMESTAMP}.json.gz" - echo ">>> Running samply record (${DURATION}s bench)..." - echo " Profile will be saved to: ${OUTPUT}" - echo " A browser tab should open automatically when done." - echo "" - - pixi run samply record --save-only -o "$OUTPUT" -- \ - "$CLI_BIN" --management-endpoint "$TARGET" bench kv write --store 0 --group 1 --duration-secs "$DURATION" \ - --loader-num "$THREADS" --connections "$CONNECTIONS" \ - --metrics-interval 1 \ - --key-space "$KEYSPACE" --value-size "$VALUE_SIZE" \ - --verify-bytes 0 2>&1 - - echo "" - echo ">>> Profile saved: ${OUTPUT}" - echo "" - echo ">>> To view the profile:" - echo " samply load ${OUTPUT}" - echo "" - echo " This starts a local web server and opens Firefox Profiler" - echo " in your browser. Press Ctrl+C to stop the server when done." - echo "" - echo " Or upload directly to https://profiler.firefox.com" - echo " (Load a profile from file → select the .json.gz)" - -elif [ "$SAMPLER" = "perf" ]; then - if ! command -v perf &>/dev/null; then - echo "ERROR: perf not found." - exit 1 - fi - - PERFDIR="${RESULTS_DIR}/profile-write-${TIMESTAMP}" - mkdir -p "$PERFDIR" - - echo ">>> Running perf record (${DURATION}s bench)..." - echo " Data will be saved to: ${PERFDIR}/" - echo "" - - # -F 999: 999Hz sampling -g: call graphs --call-graph fp: frame-pointer - # unwinding (fast, low overhead; works because we build with - # -Cforce-frame-pointers=yes and -fno-omit-frame-pointer). - # If you see truncated stacks through pre-built libs, switch to: - # --call-graph dwarf - pixi run perf record -F 999 -g --call-graph fp -o "$PERFDIR/perf.data" -- \ - "$CLI_BIN" --management-endpoint "$TARGET" bench kv write --store 0 --group 1 --duration-secs "$DURATION" \ - --loader-num "$THREADS" --connections "$CONNECTIONS" \ - --metrics-interval 1 \ - --key-space "$KEYSPACE" --value-size "$VALUE_SIZE" \ - --verify-bytes 0 2>&1 - - echo "" - echo ">>> Processing perf data..." - - # Generate perf script for flamegraph conversion - pixi run perf script -i "$PERFDIR/perf.data" > "$PERFDIR/perf-script.txt" - - # Generate flamegraph SVG via inferno (Rust port of FlameGraph). - # Installed through `pixi run install-deps` (cargo install inferno). - if command -v inferno-collapse-perf &>/dev/null && command -v inferno-flamegraph &>/dev/null; then - echo " Generating flamegraph SVG..." - pixi run perf script -i "$PERFDIR/perf.data" | \ - pixi run inferno-collapse-perf --all | \ - pixi run inferno-flamegraph --title "CrowDB write ${THREADS}T:${CONNECTIONS}C MI=${MAX_INFLIGHT}" \ - > "$PERFDIR/flamegraph.svg" - echo " Flamegraph SVG: ${PERFDIR}/flamegraph.svg" - else - echo " (Install inferno: pixi run install-deps or cargo install inferno)" +fi +invoke profile-write write --key-space "$KEYSPACE" --value-size "${KV_BENCH_VALUE_SIZE:-512}" \ + --loader-num "${KV_BENCH_LOADERS:-24}" --connections "${KV_BENCH_CONNECTIONS:-24}" --event-write +stop_sampler +if [[ "$SAMPLER" == perf ]]; then + symbol_options=() + if [[ "$PROFILE_TARGET" == servers ]]; then symbol_options=(--symfs "$LOG_ROOT/symbols/rootfs"); fi + "${PERF[@]}" script "${symbol_options[@]}" -i "$LOG_ROOT/perf.data" > "$LOG_ROOT/perf-script.txt" + [[ -s "$LOG_ROOT/perf-script.txt" ]] || { echo 'Profile contains no samples' >&2; exit 1; } + if command -v inferno-collapse-perf >/dev/null && command -v inferno-flamegraph >/dev/null; then + inferno-collapse-perf --all < "$LOG_ROOT/perf-script.txt" | \ + inferno-flamegraph --title "Paxos-KV $PROFILE_TARGET (whole invocation)" > "$LOG_ROOT/flamegraph.svg" fi - - echo "" - echo ">>> View interactively with hotspot:" - echo " hotspot ${PERFDIR}/perf.data" - echo "" - echo ">>> Or generate a flamegraph manually:" - echo " perf script -i ${PERFDIR}/perf.data | inferno-collapse-perf --all | inferno-flamegraph > ${PERFDIR}/flamegraph.svg" - -else - echo "ERROR: unknown sampler '$SAMPLER' (expected: samply | perf)" - exit 1 fi - -echo "" -echo "=== Profiling complete ===" +echo "$PROFILE_TARGET profile and workload report: $LOG_ROOT" + +# Diagnostic history (2026-10-28 Asia/Shanghai; raw CLI timestamps are UTC 2026-10-27) +# AMD Ryzen 9 5950X, Linux x86_64, 32 logical CPUs; 3-node host network. +# perf host-client smoke; 1/2/1; 1k keys x 512B; 4T/2conn; not a formal perf baseline +# Artifacts: .crowdb-runtime/artifacts/tools-migration-20261028/profile +# 2026-10-27 profile-write host-container ops/s=12240.0 latency_ns={"avg":326643.42691993463,"count":24480,"max":1008324,"min":170362,"p50":342015,"p99":438271,"p999":507903} passed=true +# Server smoke (2026-10-28 Asia/Shanghai): 3 host-network containers, no resource quotas. +# Same release binary; diagnostic image changes crowdb UID/GID to host 1000 for perf attach. +# 1/2/5 seconds, 1k keys x 512B, 4T/2conn, 99Hz; whole invocation, not a formal baseline. +# 5829 samples / 48.459 MB; server functions resolved with archived container symfs. +# Kernel symbols remain restricted. All three servers stopped gracefully with exit 0. +# Artifacts: .crowdb-runtime/artifacts/server-profile-final-20261028/profile-final +# profile-write host-container ops/s=11412.5 passed=true; p99=589823ns, p999=839679ns. diff --git a/tools/profiling/setup-perf.sh b/tools/profiling/setup-perf.sh index 2a604e497..311212ac2 100755 --- a/tools/profiling/setup-perf.sh +++ b/tools/profiling/setup-perf.sh @@ -227,4 +227,4 @@ fi echo echo "== all checks passed ==" -echo "next: pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server" +echo "next: pixi run -- cargo build --release -p crowdb-cli -p crowdb-paxos-kv-server" diff --git a/tools/rdma/run-chunk-kv.py b/tools/rdma/run-chunk-kv.py index 7901ebdee..b87d9aa4e 100644 --- a/tools/rdma/run-chunk-kv.py +++ b/tools/rdma/run-chunk-kv.py @@ -93,6 +93,7 @@ def main(): parser.add_argument("--address", default="10.200.1.1") parser.add_argument("--device", default="rocep35s0f0") parser.add_argument("--image", default="crowdb-node:dev") + parser.add_argument("--binary", type=Path, help="Reuse an already built Chunk-KV fixture executable") add_posting_arguments(parser) args = parser.parse_args() posting = posting_settings(args) @@ -102,7 +103,9 @@ def main(): output = (Path(os.environ.get("CROWDB_RUNTIME_ROOT", ROOT / ".crowdb-runtime")) / "artifacts/rdma-chunk-kv" / uuid.uuid4().hex).resolve() output.mkdir(parents=True) - binary = build(pixi, output) + binary = args.binary.resolve() if args.binary is not None else build(pixi, output) + if not binary.is_file() or not os.access(binary, os.X_OK): + raise RuntimeError(f"Chunk-KV fixture is not executable: {binary}") if args.topology == "containers": args.image = subprocess.check_output([pixi, "run", "--", "docker", "image", "inspect", args.image, "--format", "{{.Id}}"], text=True).strip() From a346e57883fea876f1420b2946d6f526b2451e25 Mon Sep 17 00:00:00 2001 From: Gian <crow.db@outlook.com> Date: Thu, 5 Nov 2026 16:05:32 +0800 Subject: [PATCH 02/17] Run tree flush with subtree fork join --- lib/crowdb-tree/src/btree/flush_batch.cpp | 8 +- lib/crowdb-tree/src/btree/flush_batch.h | 21 +- .../src/btree/flush_batch_budget.cpp | 19 +- .../src/btree/flush_batch_handoff.cpp | 90 +++-- .../src/btree/flush_batch_internal.h | 57 ++- .../src/btree/flush_batch_route.cpp | 4 + lib/crowdb-tree/src/btree/flush_operation.cpp | 2 + .../src/btree/flush_operation_stages.cpp | 1 + .../src/btree/flush_reservation.cpp | 22 +- .../src/btree/flush_reserve_bound.cpp | 9 +- lib/crowdb-tree/src/btree/flush_subtree.cpp | 358 ++++++++++++++++++ lib/crowdb-tree/src/btree/flush_subtree.h | 63 +++ .../src/btree/flush_subtree_join.cpp | 280 ++++++++++++++ .../src/btree/flush_subtree_leaf.cpp | 291 ++++++++++++++ .../btree/maintenance_publication_internal.h | 19 +- .../btree/maintenance_publication_state.cpp | 7 +- .../src/btree/maintenance_split_layout.cpp | 26 +- .../src/btree/maintenance_split_layout.h | 3 + .../src/btree/maintenance_split_step.cpp | 27 ++ .../src/btree/publication_commit.cpp | 3 +- .../src/btree/tree_core_split_handoff.cpp | 7 +- .../checkpoint_split_progress_test.cpp | 6 +- .../integration/flush_reservation_test.cpp | 20 +- .../incremental_checkpoint_test.cpp | 2 +- .../tests/integration/parallel_flush_test.cpp | 240 ++++++++++++ lib/crowdb-tree/tests/unit/gc_test.cpp | 14 +- 26 files changed, 1493 insertions(+), 106 deletions(-) create mode 100644 lib/crowdb-tree/src/btree/flush_subtree.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_subtree.h create mode 100644 lib/crowdb-tree/src/btree/flush_subtree_join.cpp create mode 100644 lib/crowdb-tree/src/btree/flush_subtree_leaf.cpp diff --git a/lib/crowdb-tree/src/btree/flush_batch.cpp b/lib/crowdb-tree/src/btree/flush_batch.cpp index e0479341d..46f39a127 100644 --- a/lib/crowdb-tree/src/btree/flush_batch.cpp +++ b/lib/crowdb-tree/src/btree/flush_batch.cpp @@ -56,8 +56,12 @@ try { return Status::invalid_argument("every closed source must participate in the captured batch"); } } - state->lane = std::move(lane); - state->options = std::move(options); + state->lane = std::move(lane); + state->options = std::move(options); + if (state->options.consolidation) { + state->options.consolidation->pages.gc_floor = + std::max(state->captured->gc_floor, std::min(publisher.requested_gc_floor(), frontier)); + } state->maintenance = std::move(maintenance); state->report.batch_id = batch_id; state->report.floor = state->captured->coverage; diff --git a/lib/crowdb-tree/src/btree/flush_batch.h b/lib/crowdb-tree/src/btree/flush_batch.h index 925465298..1826fce19 100644 --- a/lib/crowdb-tree/src/btree/flush_batch.h +++ b/lib/crowdb-tree/src/btree/flush_batch.h @@ -4,6 +4,7 @@ #include "flush_runtime.h" #include "logical_page.h" +#include "maintenance_publication.h" #include "published_state.h" namespace crowdb::tree @@ -32,19 +33,21 @@ struct FlushResult struct FlushBatchOptions { - RunLimits run_limits; - size_t max_sources = 64; - size_t parallelism = 4; - size_t inline_entries = 32; - size_t inline_bytes = 64 * 1024; - size_t max_run_bytes = 1024 * 1024; - size_t inline_ranges_per_step = 8; + RunLimits run_limits; + std::optional<MaintenancePublicationOptions> consolidation; + size_t max_sources = 64; + size_t parallelism = 4; + size_t inline_entries = 32; + size_t inline_bytes = 64 * 1024; + size_t max_run_bytes = 1024 * 1024; + size_t inline_ranges_per_step = 8; // Install the first bounded run window early; later windows double so // immutable prefix-array copies remain linear in the batch input. size_t install_run_window = 64; size_t routing_leaves_per_step = 64; size_t max_route_depth = 64; std::shared_ptr<PageLoader> loader = nullptr; + std::shared_ptr<PageLoader> consolidation_loader = nullptr; PageKeyRange selection; std::shared_ptr<ApplyDispatcher> cpu_progress = nullptr; std::shared_ptr<PublicationLane> publication = nullptr; @@ -54,7 +57,9 @@ struct FlushBatchOptions // Runs after publication admission and before source closure. std::function<void()> before_capture_for_tests = nullptr; // Called after a bounded run is owned and all source guards are released. - std::function<void(size_t, size_t)> after_run_for_tests = nullptr; + std::function<void(size_t, size_t)> after_run_for_tests = nullptr; + std::function<void(uint64_t)> before_handoff_for_tests = nullptr; + std::function<void(uint64_t, uint32_t)> after_page_for_tests = nullptr; #endif }; diff --git a/lib/crowdb-tree/src/btree/flush_batch_budget.cpp b/lib/crowdb-tree/src/btree/flush_batch_budget.cpp index fc40c83c3..17ab5ad57 100644 --- a/lib/crowdb-tree/src/btree/flush_batch_budget.cpp +++ b/lib/crowdb-tree/src/btree/flush_batch_budget.cpp @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. #include "allocation_bound.h" -#include "flush_batch_internal.h" +#include "flush_subtree.h" #include "page_cache_residency.h" #include <algorithm> @@ -40,6 +40,23 @@ std::optional<size_t> FlushBatch::metadata_reserve_bound(const MappingView &mapp // controller admission. Retain its complete source-free base once. bytes.add(PublicationCoordinator::complete_view_reserve_bytes()); bytes.add(PublishedRouteCursor::allocation_bound(options.max_route_depth)); + const auto paths = AllocationBound(ranges).add_product(ranges, mapping_bounds.max_level).add(1).value(); + const auto slots = + AllocationBound().add_product(ranges, (mapping_bounds.max_inner_bytes / sizeof(uint64_t)) + 1).value(); + if (!paths || !slots) { + return {}; + } + bytes.add_product(*paths, shared_allocation_bound<State::Subtree>() + sizeof(detail::MaintenanceBuilder) + + State::Subtree::fork_bytes()); + const auto child_slots = + AllocationBound() + .add_product(std::min(mapping_bounds.leaf_pages, *slots), + sizeof(std::shared_ptr<State::Subtree>) + sizeof(uint64_t) + sizeof(OwnedKey)) + .value(); + if (!child_slots) { + return {}; + } + bytes.add_product(mapping_bounds.max_level, *child_slots); bytes.add(detail::BoundedRangeJoin::allocation_bound(options.parallelism)); // Every accepted range can leave a retired COW root alive. Count all paths, diff --git a/lib/crowdb-tree/src/btree/flush_batch_handoff.cpp b/lib/crowdb-tree/src/btree/flush_batch_handoff.cpp index 5ecef6121..3a0e712d8 100644 --- a/lib/crowdb-tree/src/btree/flush_batch_handoff.cpp +++ b/lib/crowdb-tree/src/btree/flush_batch_handoff.cpp @@ -38,23 +38,38 @@ std::shared_ptr<const PublishedPage> append_runs(const PublishedPage void FlushBatch::State::start_range(size_t index, detail::BoundedRangeJoin::Completion done) { - if (ranges[index].record->committed.load(std::memory_order_acquire)) { - done(); + auto &range = ranges[index]; + range.index = index; + range.join_completion.emplace(std::move(done)); + start_range(range, {.context = &range, .complete = +[](void *raw, Status status) noexcept { + auto &target = *static_cast<Range *>(raw); + if (target.join_completion) { + auto completion = std::move(*target.join_completion); + target.join_completion.reset(); + completion(std::move(status)); + } + }}); +} + +void FlushBatch::State::start_range(Range &range, RangeComplete done) +{ + if (range.record->committed.load(std::memory_order_acquire)) { + finish_range(range, done); return; } if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { - finish_range(index, done, Status::unavailable("flush stopped before range dispatch")); + finish_range(range, done, Status::unavailable("flush stopped before range dispatch")); return; } - if (!ranges[index].probed && DispatchScope::take(*this)) { - ranges[index].probed = true; + if (!range.probed && DispatchScope::take(*this)) { + range.probed = true; publisher.work_metrics()->dispatch.inline_probe(); - execute_range(index, done, true); + execute_range(range, done, true); return; } - ranges[index].probed = true; - auto work = detail::reserved_unique<Work>(metadata, shared_from_this(), index, done); - auto status = runtime.try_submit(lane, &Work::execute, work.get()); + range.probed = true; + auto work = detail::reserved_unique<Work>(metadata, shared_from_this(), range, done); + auto status = runtime.try_submit(lane, &Work::execute, work.get()); if (status.ok()) { publisher.work_metrics()->dispatch.queued_step(); [[maybe_unused]] auto *scheduled = work.release(); @@ -65,7 +80,7 @@ void FlushBatch::State::start_range(size_t index, detail::BoundedRangeJoin::Comp options.cpu_progress->request(ready->node, ready->owner); } else { - finish_range(index, done, std::move(status)); + finish_range(range, done, std::move(status)); } } @@ -74,57 +89,55 @@ void FlushBatch::State::Work::execute(void *context) noexcept auto *raw = static_cast<Work *>(context); detail::ReservedUnique<Work> work(raw, detail::ReservationDeleter<Work>{raw->owner->metadata}); DispatchScope scope(*work->owner); - work->owner->execute_range(work->index, work->done, false); + work->owner->execute_range(work->range, work->done, false); } -void FlushBatch::State::execute_range(size_t index, detail::BoundedRangeJoin::Completion done, - bool inline_probe) noexcept +void FlushBatch::State::execute_range(Range &range, RangeComplete done, bool inline_probe) noexcept { try { - auto &metrics = ranges[index].metrics; + auto &metrics = range.metrics; if (!metrics) { record_work(publisher.work_metrics(), TreeWorkCounter::FlushRangeJobs, 1, TreeWorkCause::Flush); metrics = detail::reserved_shared<TreeMetricsOperation>(metadata, publisher.work_metrics(), TreeStage::FlushRange, tree_metrics_now_ns()); } - install(index, done, inline_probe); + install(range, done, inline_probe); } catch (const std::bad_alloc &) { - finish_range(index, done, Status::resource_exhausted()); + finish_range(range, done, Status::resource_exhausted()); } catch (const std::exception &) { - finish_range(index, done, Status::internal_error()); + finish_range(range, done, Status::internal_error()); } } -void FlushBatch::State::install(size_t index, detail::BoundedRangeJoin::Completion done, bool inline_probe) +void FlushBatch::State::install(Range &range, RangeComplete done, bool inline_probe) { if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { - finish_range(index, done, Status::unavailable("flush stopped before range execution")); + finish_range(range, done, Status::unavailable("flush stopped before range execution")); return; } - auto &range = ranges[index]; if (!range.ready && range.runs.size() < range.next_install_at) { auto status = prepare_run(range, inline_probe); if (!status.ok()) { - finish_range(index, done, std::move(status)); + finish_range(range, done, std::move(status)); return; } #ifdef CROWDB_TREE_TEST_UTIL if (options.after_run_for_tests) { - options.after_run_for_tests(index, range.runs.size()); + options.after_run_for_tests(range.index, range.runs.size()); } #endif if (inline_probe && range.ready) { publisher.work_metrics()->dispatch.inline_range(); } if (!range.ready && range.runs.size() < range.next_install_at) { - start_range(index, std::move(done)); + start_range(range, done); return; } } if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { - finish_range(index, done, Status::unavailable("flush stopped before range publication")); + finish_range(range, done, Status::unavailable("flush stopped before range publication")); return; } if (!range.window_prepared) { @@ -160,13 +173,13 @@ void FlushBatch::State::install(size_t index, detail::BoundedRangeJoin::Completi request->transaction.pages.push_back( {.page_id = range.page_id, .expected = range.expected, .replacement = range.prepared_page}); } - request->on_done = [self = shared_from_this(), index, done](PublicationResult result) noexcept { + request->on_done = [self = shared_from_this(), &range, done](PublicationResult result) noexcept { if (!result.status.ok()) { - self->finish_range(index, done, std::move(result.status)); + self->finish_range(range, done, std::move(result.status)); return; } self->publisher.work_metrics()->dispatch.installed(result.copied.bytes); - auto &range = self->ranges[index]; + record_work(self->publisher.work_metrics(), TreeWorkCounter::FlushHandoffBytes, range.bytes - range.installed_bytes, TreeWorkCause::Flush); range.installed_runs = range.runs.size(); @@ -177,29 +190,29 @@ void FlushBatch::State::install(size_t index, detail::BoundedRangeJoin::Completi } range.window_prepared = false; if (!range.prepared_record) { - self->finish_range(index, done); + self->finish_range(range, done); return; } range.record = std::move(range.prepared_record); range.next_install_at = range.next_install_at > SIZE_MAX / 2 ? SIZE_MAX : range.next_install_at * 2; try { - self->start_range(index, done); + self->start_range(range, done); } catch (const std::bad_alloc &) { - self->finish_range(index, done, Status::resource_exhausted()); + self->finish_range(range, done, Status::resource_exhausted()); } catch (const std::exception &) { - self->finish_range(index, done, Status::internal_error()); + self->finish_range(range, done, Status::internal_error()); } }; if (!publisher.try_submit(request, options.publication)) { - finish_range(index, done, Status::resource_exhausted("publication queue is full or closed")); + finish_range(range, done, Status::resource_exhausted("publication queue is full or closed")); } } -void FlushBatch::State::finish_range(size_t index, detail::BoundedRangeJoin::Completion done, Status status) noexcept +void FlushBatch::State::finish_range(Range &range, RangeComplete done, Status status) noexcept { - auto metrics = std::move(ranges[index].metrics); + auto metrics = std::move(range.metrics); if (metrics) { auto outcome = status.ok() ? TreeOutcome::Success : TreeOutcome::Error; if ((lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { @@ -245,6 +258,13 @@ void FlushBatch::State::cover() for (const auto &range : ranges) { transaction.required_installs.push_back(range.record); } + if (subtree) { + auto status = collect_subtree(transaction); + if (!status.ok()) { + joined(std::move(status)); + return; + } + } auto self = shared_from_this(); request->on_done = [self, detached](PublicationResult result) { self->report.handoff_status = std::move(result.status); @@ -260,7 +280,9 @@ void FlushBatch::State::cover() self->captured.reset(); self->inputs.reset(); self->options.loader.reset(); + self->options.consolidation_loader.reset(); self->ranges.clear(); + self->subtree.reset(); self->maintain(); }; if (!publisher.try_submit(request, options.publication)) { diff --git a/lib/crowdb-tree/src/btree/flush_batch_internal.h b/lib/crowdb-tree/src/btree/flush_batch_internal.h index 687f3f2ed..c6805cdcc 100644 --- a/lib/crowdb-tree/src/btree/flush_batch_internal.h +++ b/lib/crowdb-tree/src/btree/flush_batch_internal.h @@ -9,8 +9,34 @@ namespace crowdb::tree { struct FlushBatch::State : std::enable_shared_from_this<State> { + struct RangeComplete + { + void *context; + void (*complete)(void *, Status) noexcept; + + void operator()(Status status) const noexcept + { + complete(context, std::move(status)); + } + }; + + struct Subtree; + void start_subtree(); + Status collect_subtree(PublicationTransaction &transaction); + void start_load() noexcept; + void start_leaves() noexcept; + std::atomic<Subtree *> leaf_waiters{nullptr}; + std::atomic<size_t> active_leaves{0}; + std::atomic<Subtree *> load_waiters{nullptr}; + std::atomic<bool> load_active{false}; + std::shared_ptr<Subtree> subtree; + std::atomic<uint64_t> next_pid{0}; + std::atomic<size_t> next_range{0}; + struct Range { + size_t index = 0; + std::optional<detail::BoundedRangeJoin::Completion> join_completion = std::nullopt; uint64_t page_id; PageKeyRange bounds; std::shared_ptr<const PublishedPage> expected; @@ -66,19 +92,19 @@ struct FlushBatch::State : std::enable_shared_from_this<State> struct Work { - Work(std::shared_ptr<State> state, size_t range, detail::BoundedRangeJoin::Completion completion) + Work(std::shared_ptr<State> state, Range &range, RangeComplete completion) : owner(std::move(state)), - index(range), + range(range), done(std::move(completion)), node(&Work::execute, this) { } - std::shared_ptr<State> owner; - size_t index; - detail::BoundedRangeJoin::Completion done; - ApplyDispatcher::Node node; - static void execute(void *context) noexcept; + std::shared_ptr<State> owner; + Range ⦥ + RangeComplete done; + ApplyDispatcher::Node node; + static void execute(void *context) noexcept; }; struct PlanWork @@ -134,15 +160,16 @@ struct FlushBatch::State : std::enable_shared_from_this<State> void acquire_route(); void start(detail::CallbackComplete callback, void *argument); void start_range(size_t index, detail::BoundedRangeJoin::Completion done); + void start_range(Range &range, RangeComplete done); Status prepare_run(Range &range, bool inline_probe); - void finish_range(size_t index, detail::BoundedRangeJoin::Completion done, Status status = Status::Ok()) noexcept; - void execute_range(size_t index, detail::BoundedRangeJoin::Completion done, bool inline_probe) noexcept; - void install(size_t index, detail::BoundedRangeJoin::Completion done, bool inline_probe); - void joined(Status status) noexcept; - void cover(); - void maintain() noexcept; - void do_maintain(); - void finish() noexcept; + void finish_range(Range &range, RangeComplete done, Status status = Status::Ok()) noexcept; + void execute_range(Range &range, RangeComplete done, bool inline_probe) noexcept; + void install(Range &range, RangeComplete done, bool inline_probe); + void joined(Status status) noexcept; + void cover(); + void maintain() noexcept; + void do_maintain(); + void finish() noexcept; PublicationCoordinator &publisher; detail::FlushRuntime &runtime; diff --git a/lib/crowdb-tree/src/btree/flush_batch_route.cpp b/lib/crowdb-tree/src/btree/flush_batch_route.cpp index a5f48d037..46972720a 100644 --- a/lib/crowdb-tree/src/btree/flush_batch_route.cpp +++ b/lib/crowdb-tree/src/btree/flush_batch_route.cpp @@ -103,6 +103,10 @@ void FlushBatch::State::plan_step(std::shared_ptr<const InnerPageState> loaded, acquire_route(); return; } + if (options.consolidation) { + start_subtree(); + return; + } if (!route) { route = detail::reserved_unique<PublishedRouteCursor>(metadata, captured->root_page_id, captured->mapping, options.max_route_depth, false, metadata); diff --git a/lib/crowdb-tree/src/btree/flush_operation.cpp b/lib/crowdb-tree/src/btree/flush_operation.cpp index 167c306c8..7f86f7a6d 100644 --- a/lib/crowdb-tree/src/btree/flush_operation.cpp +++ b/lib/crowdb-tree/src/btree/flush_operation.cpp @@ -323,6 +323,7 @@ void FlushOperation::State::finish() noexcept if (!admitted && !capture && !batch) { source_pause.reset(); options.loader.reset(); + options.consolidation_loader.reset(); metadata->seal(); if (release_metadata) { release_metadata->seal(); @@ -331,6 +332,7 @@ void FlushOperation::State::finish() noexcept } else if (report.phase == FlushPhase::kComplete) { options.loader.reset(); + options.consolidation_loader.reset(); reservations.seal(); } auto outcome = report.handoff_status.ok() ? TreeOutcome::Success : TreeOutcome::Error; diff --git a/lib/crowdb-tree/src/btree/flush_operation_stages.cpp b/lib/crowdb-tree/src/btree/flush_operation_stages.cpp index e7517c122..8f90f97e1 100644 --- a/lib/crowdb-tree/src/btree/flush_operation_stages.cpp +++ b/lib/crowdb-tree/src/btree/flush_operation_stages.cpp @@ -206,6 +206,7 @@ Status FlushOperation::State::acknowledge(const PublishedTreeState &view) expected_sources.reset(); published_sources.reset(); options.loader.reset(); + options.consolidation_loader.reset(); metadata->seal(); // Automatic maintenance still owns the prepaid cold-page load window. // The terminal flush path seals Routing and I/O after its last callback. diff --git a/lib/crowdb-tree/src/btree/flush_reservation.cpp b/lib/crowdb-tree/src/btree/flush_reservation.cpp index b68d0cb25..967e86c9c 100644 --- a/lib/crowdb-tree/src/btree/flush_reservation.cpp +++ b/lib/crowdb-tree/src/btree/flush_reservation.cpp @@ -96,9 +96,7 @@ Status bind_flush_reservations(PublicationCoordinator &publisher, const FlushPre .backend = reserved.backend, .cpu_progress = preparation.cpu_progress, .cause = TreeWorkCause::Flush}); - // Maintenance runs after handoff metadata is sealed. Its bounded decoded - // bases use Routing directly instead of allocating new cache waiters there. - auto maintenance_loader = batch->loader; + batch->consolidation_loader = batch->loader; if (preparation.cache) { std::shared_ptr<PageLoader> binding; auto status = @@ -108,19 +106,11 @@ Status bind_flush_reservations(PublicationCoordinator &publisher, const FlushPre } batch->loader = std::move(binding); } - auto maintenance_options = preparation.maintenance; - maintenance_options.publication.pages.max_output_pages = bounds.maintenance_output_pages; - auto fallback = reserved_shared<ReclaimAdmission>( - reserved.handoff, - ReclaimAdmission{.publisher = &publisher, .account = account, .options = preparation.maintenance}); - maintenance_options.prepaid = reserved.reclaim; - maintenance_options.publication.snapshot_metadata = reserved.snapshot; - maintenance_options.acquire_window = [fallback](const auto &mapping, auto *publication, auto *credit) { - return fallback->acquire_window(mapping, publication, credit); - }; - *maintenance = make_page_maintenance(publisher, *preparation.runtime, lane, std::move(maintenance_options), - std::move(maintenance_loader), preparation.cpu_progress); - *out = std::move(reserved); + batch->consolidation = preparation.maintenance.publication; + batch->consolidation->pages.max_output_pages = bounds.maintenance_output_pages; + batch->consolidation->metadata = reserved.reclaim; + batch->consolidation->snapshot_metadata = reserved.snapshot; + *out = std::move(reserved); return Status::Ok(); } diff --git a/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp b/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp index 1300defb2..076c27ff3 100644 --- a/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp +++ b/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp @@ -1,7 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. #include "allocation_bound.h" -#include "flush_reservation_internal.h" +#include "flush_reservation.h" #include "page_cache.h" #include <algorithm> @@ -86,7 +86,6 @@ Status flush_reserve_bounds(PublicationCoordinator &publisher, const PublishedTr .add_product(source_count, 2 * sizeof(std::shared_ptr<MemTable>)) .add(2 * sizeof(std::shared_ptr<MemTable>)) .add_product(4, shared_allocation_bound<AllocationReservation>()) - .add(shared_allocation_bound<ReclaimAdmission>()) .add(preparation.cache ? SharedPageCache::binding_allocation_bound() : 0) .add(preparation.cache ? preparation.cache->load_reserve_bound(root->level, workers) : std::optional<size_t>{0}) @@ -104,6 +103,10 @@ Status flush_reserve_bounds(PublicationCoordinator &publisher, const PublishedTr const auto cold_depth = path_depth + (root->level < 2 ? 0 : static_cast<size_t>(std::min<uint64_t>(view.stats.inner_pages, 3 * path_depth))); + const auto leaf_window = AllocationBound().add_product(cold_bases, batch.parallelism).value(); + if (!leaf_window) { + return Status::resource_exhausted("flush cold leaf window bound overflow"); + } // Only captured inner pages can become cold cache entries during this // operation. Maintenance-created inner pages already own resident routes. const auto routing = @@ -112,7 +115,7 @@ Status flush_reserve_bounds(PublicationCoordinator &publisher, const PublishedTr preparation.loads, root->level, workers, preparation.cache ? std::min<uint64_t>(preparation.cache->retained_page_limit(), view.stats.inner_pages) : 0)) - .add(AsyncStorePageLoader::primary_reserve_bound(preparation.loads, cold_depth, 1, cold_bases)) + .add(AsyncStorePageLoader::primary_reserve_bound(preparation.loads, cold_depth, workers, *leaf_window)) .value(); const auto backend = backend_operation_bound( preparation.store.get(), diff --git a/lib/crowdb-tree/src/btree/flush_subtree.cpp b/lib/crowdb-tree/src/btree/flush_subtree.cpp new file mode 100644 index 000000000..c7c6cae7a --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_subtree.cpp @@ -0,0 +1,358 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_subtree.h" + +#include "page_load.h" + +#include <algorithm> +#include <limits> + +namespace crowdb::tree +{ +// stdexec joins a runtime-sized child set as a balanced tree of when_all +// operations. The storage belongs to the parent until every child terminates. +struct FlushBatch::State::Subtree::Children +{ + struct Receiver + { + using receiver_concept = stdexec::receiver_tag; + Children *owner; + + void set_value() noexcept + { + owner->finish(Status::Ok()); + } + + void set_error(Status error) noexcept + { + owner->finish(std::move(error)); + } + + void set_error(std::exception_ptr /*error*/) noexcept + { + owner->finish(Status::internal_error()); + } + + void set_stopped() noexcept + { + owner->finish(Status::unavailable("flush child stopped")); + } + }; + + using Sender = + decltype(stdexec::when_all(std::declval<detail::CallbackSender>(), std::declval<detail::CallbackSender>())); + using Operation = stdexec::connect_result_t<Sender, Receiver>; + + Children(Subtree &parent, size_t first, size_t last) : parent(parent), first(first), last(last) + { + } + + detail::CallbackSender sender() + { + return {this, +[](void *raw, detail::CallbackComplete done, void *context) { + static_cast<Children *>(raw)->start(done, context); + }}; + } + + void start(detail::CallbackComplete callback, void *argument) noexcept + { + try { + done = callback; + context = argument; + while (first < last && !parent.children[first]) { + ++first; + } + while (first < last && !parent.children[last - 1]) { + --last; + } + if (first == last) { + finish(Status::Ok()); + return; + } + if (last - first == 1) { + parent.children[first]->start(done, context); + return; + } + const auto middle = first + ((last - first) / 2); + left = detail::reserved_unique<Children>(parent.owner.metadata, parent, first, middle); + right = detail::reserved_unique<Children>(parent.owner.metadata, parent, middle, last); + operation = detail::reserved_construct<Operation>(parent.owner.metadata, [&](Operation *storage) { + ::new (static_cast<void *>(storage)) + Operation(stdexec::connect(stdexec::when_all(left->sender(), right->sender()), Receiver{this})); + }); + stdexec::start(*operation); + } + catch (const std::bad_alloc &) { + finish(Status::resource_exhausted("flush child join allocation failed")); + } + catch (const std::exception &) { + finish(Status::internal_error("flush child join failed")); + } + } + + void finish(Status status) noexcept + { + done(context, status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError, std::move(status)); + } + + Subtree &parent; + size_t first; + size_t last; + detail::CallbackComplete done = nullptr; + void *context = nullptr; + detail::ReservedUnique<Children> left; + detail::ReservedUnique<Children> right; + std::shared_ptr<Operation> operation; +}; + +FlushBatch::State::Subtree::Subtree(State &state, uint64_t page_id, PageKeyRange range, bool selected) + : owner(state), + pid(page_id), + bounds(std::move(range)), + active(selected), + expected(state.captured->mapping.get(page_id)), + output(state.metadata), + children(detail::ReservationAllocator<std::shared_ptr<Subtree>>(state.metadata)) +{ +} + +FlushBatch::State::Subtree::~Subtree() = default; + +detail::CallbackSender FlushBatch::State::Subtree::sender() noexcept +{ + return {this, +[](void *raw, detail::CallbackComplete done, void *context) { + static_cast<Subtree *>(raw)->start(done, context); + }}; +} + +void FlushBatch::State::Subtree::start(detail::CallbackComplete callback, void *argument) +{ + done = callback; + context = argument; + try { + if ((finished && status.ok()) || !active) { + if (output.pids.empty()) { + output.pids.push_back(pid); + } + complete(Status::Ok()); + return; + } + finished = false; + status = Status::Ok(); + if (expected && expected->level == 0) { + auto *head = owner.leaf_waiters.load(std::memory_order_relaxed); + do { + leaf_next = head; + } while (!owner.leaf_waiters.compare_exchange_weak(head, this, std::memory_order_release, + std::memory_order_relaxed)); + owner.start_leaves(); + } + else { + schedule(); + } + } + catch (const std::bad_alloc &) { + complete(Status::resource_exhausted("flush task allocation failed")); + } + catch (const std::exception &) { + complete(Status::internal_error("flush task failed")); + } +} + +void FlushBatch::State::Subtree::schedule() noexcept +{ + const auto result = owner.runtime.try_submit(owner.lane, &Subtree::run, this); + if (result.ok()) { + return; + } + if (owner.options.cpu_progress) { + owner.options.cpu_progress->request(cpu_node, owner.weak_from_this().lock()); + } + else { + if (std::exchange(load_completed, false)) { + owner.load_active.store(false, std::memory_order_release); + owner.start_load(); + } + complete(result); + } +} + +void FlushBatch::State::Subtree::run(void *raw) noexcept +{ + auto &self = *static_cast<Subtree *>(raw); + // Terminal callbacks may release the connected operation, including us. + auto lifetime = self.owner.weak_from_this().lock(); + DispatchScope scope(self.owner); + try { + self.step(); + } + catch (const std::bad_alloc &) { + self.complete(Status::resource_exhausted("flush subtree allocation failed")); + } + catch (const std::exception &) { + self.complete(Status::internal_error("flush subtree failed")); + } +} + +void FlushBatch::State::Subtree::complete(Status result) noexcept +{ + auto lifetime = owner.weak_from_this().lock(); + auto &state = owner; + const bool release = std::exchange(leaf_permit, false); + status = std::move(result); + if (layout_metrics) { + (void)layout_metrics->finish(status.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + } + finished = true; + const auto signal = status.ok() ? detail::CallbackSignal::kValue : detail::CallbackSignal::kError; + done(context, signal, status); + if (release) { + state.active_leaves.fetch_sub(1, std::memory_order_acq_rel); + state.start_leaves(); + } +} + +size_t FlushBatch::State::Subtree::fork_bytes() noexcept +{ + return (2 * sizeof(Children)) + detail::shared_allocation_bound<Children::Operation>(); +} + +void FlushBatch::State::start_leaves() noexcept +{ + auto count = active_leaves.load(std::memory_order_relaxed); + while (count < options.parallelism) { + if (!active_leaves.compare_exchange_weak(count, count + 1, std::memory_order_acq_rel)) { + continue; + } + auto *next = leaf_waiters.load(std::memory_order_acquire); + while (next != nullptr && + !leaf_waiters.compare_exchange_weak(next, next->leaf_next, std::memory_order_acq_rel)) { + } + if (next != nullptr) { + next->leaf_permit = true; + next->schedule(); + } + else { + active_leaves.fetch_sub(1, std::memory_order_acq_rel); + // An enqueuer either takes this permit or observes our idle edge. + if (leaf_waiters.load(std::memory_order_acquire) == nullptr) { + return; + } + } + count = active_leaves.load(std::memory_order_relaxed); + } +} + +void FlushBatch::State::Subtree::step() +{ + if (load_completed) { + load_completed = false; + owner.load_active.store(false, std::memory_order_release); + owner.start_load(); + } + if (!status.ok()) { + complete(status); + return; + } + if ((owner.lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { + complete(Status::unavailable("flush stopped during subtree work")); + return; + } + if (!expected) { + complete(Status::corruption("flush child is absent from mapping")); + return; + } + if (!active) { + if (output.pids.empty()) { + output.pids.push_back(pid); + } + complete(Status::Ok()); + return; + } + if (expected->level == 0) { + leaf(); + return; + } + if (children_ready) { + join(); + return; + } + fork(); +} + +void FlushBatch::State::Subtree::fork() +{ + if (!inner) { + inner = expected->inner; + } + if (!inner) { + if (!expected->inner_backing || !owner.options.loader) { + complete(Status::unavailable("flush index requires an asynchronous loader")); + return; + } + request_load(); + return; + } + if (inner->children.empty() || inner->children.size() != inner->separators.size() + 1) { + complete(Status::corruption("invalid flush index shape")); + return; + } + if (children.empty()) { + children.reserve(inner->children.size()); + for (size_t i = 0; i < inner->children.size(); ++i) { + auto selected = bounds; + if (i != 0) { + selected.lower = inner->separators[i - 1]; + } + if (i < inner->separators.size()) { + selected.upper = inner->separators[i]; + } + auto first = owner.first_input(selected.lower_slice()); + const bool touched = first && selected.contains(Slice(*first)); + if (!touched) { + children.push_back(nullptr); + continue; + } + auto child = detail::reserved_shared<Subtree>(owner.metadata, owner, inner->children[i], + std::move(selected), touched); + if (!child->expected || child->expected->level + 1 != expected->level) { + complete(Status::corruption("flush child level disagrees with parent")); + return; + } + children.push_back(std::move(child)); + } + } + child_join = detail::reserved_unique<Children>(owner.metadata, *this, 0, children.size()); + // Callback transports feed stdexec's when_all. The continuation executes + // only after all direct children, including accepted failures, have joined. + child_join->start( + +[](void *raw, detail::CallbackSignal signal, Status result) { + auto &self = *static_cast<Subtree *>(raw); + self.status = signal == detail::CallbackSignal::kValue ? Status::Ok() : std::move(result); + self.children_ready = true; + self.schedule(); + }, + this); +} + +void FlushBatch::State::Subtree::reset_work() +{ + finished = false; + status = Status::Ok(); + children_ready = false; + child_join.reset(); + layout.reset(); + layout_metrics.reset(); + builder.reset(); + loaded_leaf.reset(); + next_base = 0; + output.pids.clear(); + output.separators.clear(); + for (auto &child : children) { + if (child) { + child->reset_work(); + } + } +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_subtree.h b/lib/crowdb-tree/src/btree/flush_subtree.h new file mode 100644 index 000000000..ce9979951 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_subtree.h @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "flush_batch_internal.h" +#include "maintenance_publication_internal.h" +#include "maintenance_split_layout.h" + +namespace crowdb::tree +{ +// One page task owns its leaf work or joins its direct children. Only the +// parent consumes sibling outputs; no child publishes an ancestor change. +struct FlushBatch::State::Subtree +{ + struct Children; + Subtree(State &state, uint64_t page_id, PageKeyRange range, bool selected); + ~Subtree(); + detail::CallbackSender sender() noexcept; + void start(detail::CallbackComplete callback, void *argument); + void schedule() noexcept; + static void run(void *raw) noexcept; + void step(); + void fork(); + void leaf(); + Status coalesce_leaf(const PageMaintenanceLimits &limits); + void join(); + void complete(Status result) noexcept; + void prepare_builder(); + static size_t fork_bytes() noexcept; + Subtree *leaf_next = nullptr; + bool leaf_permit = false; + void reset_work(); + void request_load(); + void load() noexcept; + Subtree *load_next = nullptr; + bool load_completed = false; + Status collect(PublicationTransaction &transaction); + [[nodiscard]] std::shared_ptr<const PublishedPage> lookup(uint64_t page_id) const; + + State &owner; + uint64_t pid; + PageKeyRange bounds; + bool active; + bool finished = false; + bool handed_off = false; + bool children_ready = false; + std::optional<TreeMetricsOperation> layout_metrics; + Status status; + std::shared_ptr<const PublishedPage> expected; + std::shared_ptr<const InnerPageState> inner; + std::shared_ptr<PageState> loaded_leaf; + size_t next_base = 0; + std::shared_ptr<Range> range; + detail::ReservedUnique<detail::SplitWindowCursor> layout; + detail::ReservedUnique<detail::MaintenanceBuilder> builder; + detail::MaintenanceChildren output; + detail::ReservedVector<std::shared_ptr<Subtree>> children; + detail::ReservedUnique<Children> child_join; + ApplyDispatcher::Node cpu_node{&Subtree::run, this}; + detail::CallbackComplete done = nullptr; + void *context = nullptr; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_subtree_join.cpp b/lib/crowdb-tree/src/btree/flush_subtree_join.cpp new file mode 100644 index 000000000..389a5ffb7 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_subtree_join.cpp @@ -0,0 +1,280 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_subtree.h" +#include "page_load.h" + +#include <algorithm> +#include <limits> +#include <stdexcept> + +namespace crowdb::tree +{ +void FlushBatch::State::Subtree::join() +{ + prepare_builder(); + + struct Child + { + Subtree *task; + uint64_t pid; + std::optional<OwnedKey> lower; + bool touched; + }; + + detail::ReservedVector<Child> ordered{detail::ReservationAllocator<Child>(owner.metadata)}; + for (size_t i = 0; i < children.size(); ++i) { + auto lower = i != 0 ? std::optional(inner->separators[i - 1]) : bounds.lower; + if (!children[i]) { + ordered.push_back({.task = this, .pid = inner->children[i], .lower = std::move(lower), .touched = false}); + continue; + } + auto &child = *children[i]; + for (size_t j = 0; j < child.output.pids.size(); ++j) { + if (j != 0) { + lower = child.output.separators[j - 1]; + } + ordered.push_back({.task = &child, .pid = child.output.pids[j], .lower = lower, .touched = true}); + } + } + for (size_t i = 0; i + 1 < ordered.size();) { + auto &a = ordered[i]; + auto &b = ordered[i + 1]; + auto left = a.task->lookup(a.pid); + auto right = b.task->lookup(b.pid); + bool merged = false; + if (a.touched || b.touched) { + a.task->prepare_builder(); + b.task->prepare_builder(); + if (left->leaf && right->leaf && left->metrics.page_size_known && right->metrics.page_size_known && + left->metrics.kv_known && right->metrics.kv_known) { + PageVersion version; + auto result = detail::next_page_version(*left->leaf, right->leaf.get(), &version); + if (!result.ok()) { + complete(std::move(result)); + return; + } + auto plan = detail::plan_published_leaf_merge(*left, *right, + {.left = left->leaf, + .right = right->leaf, + .left_page_id = a.pid, + .right_page_id = b.pid, + .left_parent = pid, + .right_parent = pid}, + builder->options.pages, version, 1); + if (!plan.logical.status.ok()) { + complete(std::move(plan.logical.status)); + return; + } + if (plan.logical.page) { + result = a.task->builder->make_leaf(a.pid, left, std::move(plan.logical.page), plan.primary_bytes, + right->allocation_owner, + plan.statistics ? &plan.statistics.value() : nullptr); + if (!result.ok()) { + complete(std::move(result)); + return; + } + merged = true; + } + } + else if (left->inner && right->inner && b.lower && + std::min(left->inner->separators.size(), right->inner->separators.size()) < + builder->options.inner_merge_keys && + left->inner->children.size() + right->inner->children.size() <= + builder->options.max_inner_children) { + InnerPageState combined(*left->inner, builder->options.metadata); + combined.children.insert(combined.children.end(), right->inner->children.begin(), + right->inner->children.end()); + combined.separators.push_back(*b.lower); + combined.separators.insert(combined.separators.end(), right->inner->separators.begin(), + right->inner->separators.end()); + size_t bytes = kFrameHeaderSize + kFrameTrailerSize + (combined.children.size() * sizeof(uint64_t)) + + (combined.separators.size() * kInnerSlotSize); + for (const auto &key : combined.separators) { + bytes += key.size(); + } + if (bytes <= builder->options.max_inner_bytes) { + detail::MaintenanceChildren outputs(owner.metadata); + auto result = a.task->builder->partition_inner(a.pid, left->level, combined, &outputs); + if (!result.ok()) { + complete(std::move(result)); + return; + } + if (outputs.pids.size() != 1) { + complete(Status::internal_error("accepted index merge split again")); + return; + } + merged = true; + } + } + } + if (merged) { + record_work(owner.publisher.work_metrics(), TreeWorkCounter::MergeOperations, 1, TreeWorkCause::Flush); + record_work(owner.publisher.work_metrics(), TreeWorkCounter::MergeInputPages, 2, TreeWorkCause::Flush); + record_work(owner.publisher.work_metrics(), TreeWorkCounter::MergeOutputPages, 1, TreeWorkCause::Flush); + auto result = b.task->builder->replace(b.pid, {}); + if (!result.ok()) { + complete(std::move(result)); + return; + } + a.touched = true; + ordered.erase(ordered.begin() + static_cast<ptrdiff_t>(i + 1)); + } + else { + ++i; + } + } + InnerPageState combined(builder->options.metadata); + for (size_t i = 0; i < ordered.size(); ++i) { + combined.children.push_back(ordered[i].pid); + if (i != 0) { + combined.separators.push_back(ordered[i].lower.value()); + } + } + output.pids.clear(); + output.separators.clear(); + if (combined.children == inner->children && combined.separators == inner->separators) { + output.pids.push_back(pid); + } + else { + auto result = builder->partition_inner(pid, expected->level, combined, &output); + if (!result.ok()) { + complete(std::move(result)); + return; + } + } +#ifdef CROWDB_TREE_TEST_UTIL + if (owner.options.after_page_for_tests) { + owner.options.after_page_for_tests(pid, expected->level); + } +#endif + complete(Status::Ok()); +} + +std::shared_ptr<const PublishedPage> FlushBatch::State::Subtree::lookup(uint64_t page_id) const +{ + return builder ? builder->lookup(page_id) : owner.captured->mapping.get(page_id); +} + +Status FlushBatch::State::Subtree::collect(PublicationTransaction &transaction) +{ + if (range) { + transaction.required_installs.push_back(range->record); + } + for (auto &child : children) { + if (child) { + auto status = child->collect(transaction); + if (!status.ok()) { + return status; + } + } + } + if (!builder) { + return Status::Ok(); + } + MaintenancePublicationResult result(builder->options.metadata); + auto status = builder->finish(&result); + if (!status.ok()) { + return status; + } + for (auto &page : result.transaction.pages) { + transaction.pages.push_back(std::move(page)); + } + return Status::Ok(); +} + +void FlushBatch::State::start_subtree() +{ + if (!subtree) { + next_pid.store(captured->next_page_id, std::memory_order_relaxed); + const auto first = first_input({}); + subtree = detail::reserved_shared<Subtree>(metadata, *this, captured->root_page_id, options.selection, + first.has_value()); + } + else { + subtree->reset_work(); + } + next_pid.store(captured->next_page_id, std::memory_order_relaxed); + operation = detail::reserved_unique<Attempt>(metadata, subtree->sender(), shared_from_this()); + stdexec::start(operation->connected); +} + +Status FlushBatch::State::collect_subtree(PublicationTransaction &transaction) +{ + if (!options.consolidation) { + return Status::internal_error("flush consolidation options are absent"); + } + const auto snapshot_metadata = options.consolidation->snapshot_metadata; + subtree->prepare_builder(); + auto &outputs = subtree->output; + uint64_t root = outputs.pids.front(); + if (outputs.pids.size() > 1) { + const auto status = subtree->builder->grow_root(std::move(outputs), subtree->expected->level); + if (!status.ok()) { + return status; + } + root = subtree->builder->root_page_id(); + } + // Only the root may contract its level. Descendant joins retain their + // level until their parent has made its own sibling decision. + auto find = [&](auto &&self, Subtree &task, uint64_t pid) -> std::shared_ptr<const PublishedPage> { + if (task.builder) { + auto page = task.builder->lookup(pid); + if (page != task.builder->view->mapping.get(pid)) { + return page; + } + } + for (auto &child : task.children) { + if (!child) { + continue; + } + auto page = self(self, *child, pid); + if (page != captured->mapping.get(pid)) { + return page; + } + } + return captured->mapping.get(pid); + }; + for (size_t depth = 0; subtree->active && depth < options.max_route_depth; ++depth) { + const auto page = find(find, *subtree, root); + if (!page || !page->inner || page->inner->children.size() != 1) { + break; + } + const auto child = page->inner->children.front(); + // Root deletions belong to the original task that owns that PID. + auto remove = [&](auto &&self, Subtree &task) -> Status { + if (task.pid == root || + (task.builder && task.builder->lookup(root) != task.builder->view->mapping.get(root))) { + task.prepare_builder(); + return task.builder->replace(root, {}); + } + for (auto &next : task.children) { + if (next != nullptr) { + auto status = self(self, *next); + if (!status.ok()) { + return status; + } + } + } + return Status::Ok(); + }; + auto removed = remove(remove, *subtree); + if (!removed.ok()) { + return removed; + } + root = child; + } + transaction.root_page_id = root; + transaction.expected_root_page_id = captured->root_page_id; + transaction.expected_next_page_id = captured->next_page_id; + transaction.next_page_id = next_pid.load(std::memory_order_relaxed); + transaction.snapshot_metadata = snapshot_metadata; + auto status = subtree->collect(transaction); + if (!status.ok()) { + return status; + } + std::sort(transaction.pages.begin(), transaction.pages.end(), + [](const auto &a, const auto &b) { return a.page_id < b.page_id; }); + return Status::Ok(); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/flush_subtree_leaf.cpp b/lib/crowdb-tree/src/btree/flush_subtree_leaf.cpp new file mode 100644 index 000000000..048743f74 --- /dev/null +++ b/lib/crowdb-tree/src/btree/flush_subtree_leaf.cpp @@ -0,0 +1,291 @@ +// Copyright 2026-present Gian <crow.db@outlook.com> +// Licensed under the Apache License, Version 2.0. +#include "flush_subtree.h" +#include "page_load.h" + +#include <algorithm> +#include <limits> +#include <stdexcept> + +namespace crowdb::tree +{ +void FlushBatch::State::Subtree::prepare_builder() +{ + if (builder) { + return; + } + if (!owner.options.consolidation) { + throw std::logic_error("flush consolidation options are absent"); + } + auto limits = *owner.options.consolidation; + // This is a whole-operation result, not one bounded idle transaction. + limits.max_changed_pages = std::numeric_limits<size_t>::max(); + builder = detail::reserved_unique<detail::MaintenanceBuilder>( + owner.metadata, owner.publisher.capture(), owner.coverage_record, limits, [this](uint64_t *output) { + auto next = owner.next_pid.load(std::memory_order_relaxed); + do { + if (next >= MappingView::kPageLimit) { + return Status::resource_exhausted("flush PID space exhausted"); + } + } while (!owner.next_pid.compare_exchange_weak(next, next + 1, std::memory_order_relaxed)); + *output = next; + return Status::Ok(); + }); +} + +void FlushBatch::State::Subtree::leaf() +{ + if (!range) { +#ifdef CROWDB_TREE_TEST_UTIL + if (owner.options.before_handoff_for_tests) { + owner.options.before_handoff_for_tests(pid); + } +#endif + const auto index = owner.next_range.fetch_add(1, std::memory_order_relaxed); + auto record = detail::reserved_shared<InstallRecord>(owner.metadata, + InstallToken{.generation = owner.captured->generation, + .batch = owner.report.batch_id, + .range = index, + .sequence = 0}); + range = detail::reserved_construct<Range>(owner.metadata, [&](Range *storage) { + ::new (static_cast<void *>(storage)) + Range{.index = index, + .page_id = pid, + .bounds = bounds, + .expected = expected, + .record = std::move(record), + .runs = OverlayStorage{OverlayAllocator(owner.metadata)}, + .installs = + detail::ReservedVector<std::shared_ptr<InstallRecord>>{ + detail::ReservationAllocator<std::shared_ptr<InstallRecord>>(owner.metadata)}, + .next_key = {}, + .next_install_at = owner.options.install_run_window}; + }); + } + if (!handed_off) { + owner.start_range(*range, {.context = this, .complete = +[](void *raw, Status result) noexcept { + auto &self = *static_cast<Subtree *>(raw); + self.status = std::move(result); + self.handed_off = self.status.ok(); + self.schedule(); + }}); + return; + } + prepare_builder(); + if (!loaded_leaf) { + loaded_leaf = detail::reserved_shared<PageState>(builder->options.metadata, *range->expected->leaf, + builder->options.metadata); + } + while (next_base < loaded_leaf->bases.size()) { + auto &base = loaded_leaf->bases[next_base]; + if (base.resident) { + ++next_base; + continue; + } + if (!owner.options.loader) { + complete(Status::unavailable("flush leaf requires an asynchronous loader")); + return; + } + request_load(); + return; + } + prepare_builder(); + auto limits = builder->options.pages; + limits.max_sources = std::max(limits.max_sources, loaded_leaf->bases.size() + loaded_leaf->overlays.size()); + if (!layout) { + layout_metrics.emplace(owner.publisher.work_metrics(), TreeStage::PageLayout, tree_metrics_now_ns()); + layout = detail::reserved_unique<detail::SplitWindowCursor>(owner.metadata, loaded_leaf, limits); + } + auto result = layout->advance_complete(); + if (!result.ok()) { + complete(std::move(result)); + return; + } + if (!layout->done()) { + schedule(); + return; + } + if (layout_metrics) { + (void)layout_metrics->finish(TreeOutcome::Success, tree_metrics_now_ns()); + } + std::optional<TreeMetricsOperation> split_metrics; + if (!layout->layout().separators.empty()) { + split_metrics.emplace(owner.publisher.work_metrics(), TreeStage::PageSplit, tree_metrics_now_ns()); + } + output.pids.clear(); + output.separators.clear(); + result = detail::build_flush_leaf_outputs(*builder, pid, range->expected, layout->layout(), &output); + if (split_metrics) { + (void)split_metrics->finish(result.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + if (result.ok()) { + record_work(owner.publisher.work_metrics(), TreeWorkCounter::SplitOperations, 1, TreeWorkCause::Flush); + record_work(owner.publisher.work_metrics(), TreeWorkCounter::SplitOutputPages, output.pids.size(), + TreeWorkCause::Flush); + } + } + if (!result.ok()) { + complete(std::move(result)); + return; + } + result = coalesce_leaf(limits); + if (!result.ok()) { + complete(std::move(result)); + return; + } + layout.reset(); + loaded_leaf.reset(); +#ifdef CROWDB_TREE_TEST_UTIL + if (owner.options.after_page_for_tests) { + owner.options.after_page_for_tests(pid, 0); + } +#endif + complete(Status::Ok()); +} + +void FlushBatch::State::Subtree::request_load() +{ + auto *head = owner.load_waiters.load(std::memory_order_relaxed); + do { + load_next = head; + } while ( + !owner.load_waiters.compare_exchange_weak(head, this, std::memory_order_release, std::memory_order_relaxed)); + owner.start_load(); +} + +void FlushBatch::State::start_load() noexcept +{ + if (load_active.exchange(true, std::memory_order_acq_rel)) { + return; + } + auto *next = load_waiters.load(std::memory_order_acquire); + while (next != nullptr && !load_waiters.compare_exchange_weak(next, next->load_next, std::memory_order_acq_rel)) { + } + if (next != nullptr) { + next->load(); + return; + } + load_active.exchange(false, std::memory_order_acq_rel); + // Pair with producer publication so an arrival cannot miss the idle edge. + if (load_waiters.load(std::memory_order_acquire) != nullptr) { + start_load(); + } +} + +void FlushBatch::State::Subtree::load() noexcept +{ + try { + if ((owner.lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { + status = Status::unavailable("flush stopped before cold load"); + load_completed = true; + schedule(); + return; + } + if (expected->level != 0) { + owner.options.loader->load_inner(expected->inner_backing, [this, keep = owner.weak_from_this().lock()]( + Status result, auto page) noexcept { + status = std::move(result); + inner = std::move(page); + if (status.ok() && !inner) { + status = Status::corruption("flush loader returned no index"); + } + load_completed = true; + schedule(); + }); + } + else { + auto loader = + owner.options.consolidation_loader ? owner.options.consolidation_loader : owner.options.loader; + loader->load_leaf(loaded_leaf->bases[next_base], + [this, keep = owner.weak_from_this().lock()](Status result, auto page) noexcept { + status = std::move(result); + if (status.ok() && !page) { + status = Status::corruption("flush loader returned no leaf"); + } + if (status.ok()) { + loaded_leaf->bases[next_base++].resident = std::move(page); + } + load_completed = true; + schedule(); + }); + } + } + catch (const std::bad_alloc &) { + status = Status::resource_exhausted("flush cold load allocation failed"); + load_completed = true; + schedule(); + } + catch (const std::exception &) { + status = Status::internal_error("flush cold load failed"); + load_completed = true; + schedule(); + } +} + +Status FlushBatch::State::Subtree::coalesce_leaf(const PageMaintenanceLimits &limits) +{ + for (auto output_pid : output.pids) { + const auto page = builder->lookup(output_pid); + const bool clipped = std::ranges::any_of(page->leaf->overlays, [&](const auto &run) { + return !run->entries().empty() && (!page->leaf->range.contains(run->entries().front().key()) || + !page->leaf->range.contains(run->entries().back().key())); + }); + const bool collect_tombstone = page->leaf->overlays.size() == 1 && !clipped && page->leaf->bases.empty() && + std::ranges::any_of(page->leaf->overlays, [&](const auto &run) { + if (run->min_slot() > limits.gc_floor) { + return false; + } + return std::ranges::any_of(run->entries(), [&](const auto &entry) { + return page->leaf->range.contains(entry.key()) && + entry.slot() <= limits.gc_floor && + (entry.flags() & kFlagTombstone) != 0; + }); + }); + if (page->leaf->overlays.size() <= 1 && !clipped && !collect_tombstone) { + continue; + } + if (page->leaf->representation_id == UINT64_MAX) { + return Status::resource_exhausted("flush representation identity exhausted"); + } + auto compact_limits = limits; + compact_limits.max_work_entries = + std::max(compact_limits.max_work_entries, static_cast<size_t>(page->metrics.overlay_entries)); + compact_limits.max_metadata_bytes = + std::max(compact_limits.max_metadata_bytes, compact_limits.max_work_entries * sizeof(EntryRef)); + uint64_t run = 1; + for (const auto &input : page->leaf->overlays) { + if (input->id() >= UINT64_MAX - compact_limits.max_sources) { + return Status::resource_exhausted("flush run identities exhausted"); + } + run = std::max(run, input->id() + 1); + } + TreeMetricsOperation coalesce_metrics(owner.publisher.work_metrics(), TreeStage::OverlayCoalesce, + tree_metrics_now_ns()); + auto compacted = coalesce_overlays(page->leaf, compact_limits, page->leaf->representation_id + 1, run); + (void)coalesce_metrics.finish(compacted.status.ok() ? TreeOutcome::Success : TreeOutcome::Error, + tree_metrics_now_ns()); + if (!compacted.status.ok()) { + return std::move(compacted.status); + } + uint64_t output_entries = 0; + for (const auto &run : compacted.page->overlays) { + output_entries += run->entries().size(); + } + record_work(owner.publisher.work_metrics(), TreeWorkCounter::CoalesceOperations, 1, TreeWorkCause::Flush); + record_work(owner.publisher.work_metrics(), TreeWorkCounter::CoalesceInputRuns, page->leaf->overlays.size(), + TreeWorkCause::Flush); + record_work(owner.publisher.work_metrics(), TreeWorkCounter::CoalesceOutputRuns, + compacted.page->overlays.size(), TreeWorkCause::Flush); + record_work(owner.publisher.work_metrics(), TreeWorkCounter::CoalesceInputEntries, + page->metrics.overlay_entries, TreeWorkCause::Flush); + record_work(owner.publisher.work_metrics(), TreeWorkCounter::CoalesceOutputEntries, output_entries, + TreeWorkCause::Flush); + LeafLayoutStats stats{.kv_count = page->metrics.kv_count, .kv_bytes = page->metrics.kv_bytes}; + auto result = builder->make_leaf(output_pid, page, std::move(compacted.page), + page->metrics.page_size_bytes - compacted.removed_primary_bytes, {}, &stats); + if (!result.ok()) { + return result; + } + } + return Status::Ok(); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/maintenance_publication_internal.h b/lib/crowdb-tree/src/btree/maintenance_publication_internal.h index cf93195fc..d8f85b15d 100644 --- a/lib/crowdb-tree/src/btree/maintenance_publication_internal.h +++ b/lib/crowdb-tree/src/btree/maintenance_publication_internal.h @@ -34,7 +34,7 @@ class MaintenanceBuilder { public: MaintenanceBuilder(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<InstallRecord> record, - const MaintenancePublicationOptions &limits); + const MaintenancePublicationOptions &limits, InlineFunction<Status(uint64_t *)> allocate = {}); [[nodiscard]] Status validate() const; Status path_to(uint64_t pid, MaintenancePath *path) const; Status guard_path(const MaintenancePath &path); @@ -54,11 +54,18 @@ class MaintenanceBuilder const std::shared_ptr<const PublishedTreeState> view; MaintenancePublicationOptions options; - private: - Status merge_inner_pair(const MaintenancePathEntry &ancestor, uint64_t target, bool *changed); Status partition_inner(uint64_t pid, uint32_t level, const InnerPageState &inner, MaintenanceChildren *output); Status grow_root(MaintenanceChildren children, uint32_t child_level); - std::shared_ptr<InstallRecord> record_; + + [[nodiscard]] uint64_t root_page_id() const noexcept + { + return root_pid_; + } + + private: + Status merge_inner_pair(const MaintenancePathEntry &ancestor, uint64_t target, bool *changed); + InlineFunction<Status(uint64_t *)> allocate_; + std::shared_ptr<InstallRecord> record_; using Change = std::pair<const uint64_t, PageReplacement>; std::map<uint64_t, PageReplacement, std::less<>, ReservationAllocator<Change>> changes_; uint64_t next_pid_; @@ -66,6 +73,10 @@ class MaintenanceBuilder size_t inner_merges_ = 0; }; +Status build_flush_leaf_outputs(MaintenanceBuilder &builder, uint64_t pid, + const std::shared_ptr<const PublishedPage> &old, const struct SplitWindowLayout &layout, + MaintenanceChildren *children); + Status next_page_version(const PageState &left, const PageState *right, PageVersion *version); Status summarize_leaf(const std::shared_ptr<const PageState> &page, const PageMaintenanceLimits &limits, PageMetricsContribution *metrics); diff --git a/lib/crowdb-tree/src/btree/maintenance_publication_state.cpp b/lib/crowdb-tree/src/btree/maintenance_publication_state.cpp index 2ae68beb0..fdf202a45 100644 --- a/lib/crowdb-tree/src/btree/maintenance_publication_state.cpp +++ b/lib/crowdb-tree/src/btree/maintenance_publication_state.cpp @@ -9,9 +9,11 @@ namespace crowdb::tree::detail { MaintenanceBuilder::MaintenanceBuilder(std::shared_ptr<const PublishedTreeState> captured, std::shared_ptr<InstallRecord> record, - const MaintenancePublicationOptions &limits) + const MaintenancePublicationOptions &limits, + InlineFunction<Status(uint64_t *)> allocate) : view(std::move(captured)), options(limits), + allocate_(std::move(allocate)), record_(std::move(record)), changes_(ReservationAllocator<Change>(limits.metadata)), next_pid_(view ? view->next_page_id : 0), @@ -40,6 +42,9 @@ std::shared_ptr<const PublishedPage> MaintenanceBuilder::lookup(uint64_t pid) co Status MaintenanceBuilder::allocate_pid(uint64_t *pid) { + if (allocate_) { + return allocate_(pid); + } if (next_pid_ >= MappingView::kPageLimit) { return Status::resource_exhausted("maintenance PID space exhausted"); } diff --git a/lib/crowdb-tree/src/btree/maintenance_split_layout.cpp b/lib/crowdb-tree/src/btree/maintenance_split_layout.cpp index a0c65edd4..277827f18 100644 --- a/lib/crowdb-tree/src/btree/maintenance_split_layout.cpp +++ b/lib/crowdb-tree/src/btree/maintenance_split_layout.cpp @@ -44,6 +44,16 @@ std::optional<BaseLoadRequest> SplitWindowCursor::needs_base() const } Status SplitWindowCursor::advance() +{ + return advance(false); +} + +Status SplitWindowCursor::advance_complete() +{ + return advance(true); +} + +Status SplitWindowCursor::advance(bool complete) { const auto &limits = limits_; auto *layout = &layout_; @@ -61,9 +71,14 @@ Status SplitWindowCursor::advance() if (cursor_.needs_base()) { return Status::Ok(); } - const auto outputs = limits.max_output_pages; + const auto outputs = limits.max_output_pages; + size_t step_entries = 0; + size_t step_outputs = 0; while (cursor_.valid()) { - const bool work_limit = entries_ == limits.max_work_entries; + if (complete && step_entries == limits.max_work_entries) { + return Status::Ok(); + } + const bool work_limit = !complete && entries_ == limits.max_work_entries; size_t bytes = 0; if (!work_limit) { const auto status = primary_bytes(cursor_.key(), cursor_.cell(), limits, &bytes); @@ -80,12 +95,16 @@ Status SplitWindowCursor::advance() layout->statistics.push_back(current_stats_); layout->separators.emplace_back(cursor_.key(), limits.metadata); separator_bytes_ += cursor_.key().size(); - if (work_limit || layout->separators.size() + 1 == outputs) { + ++step_outputs; + if (!complete && (work_limit || layout->separators.size() + 1 == outputs)) { done_ = true; return Status::Ok(); } current_bytes_ = kPageOverhead; current_stats_ = {}; + if (complete && step_outputs == outputs - 1) { + return Status::Ok(); + } } auto status = accumulate_leaf_stats(cursor_.key(), cursor_.cell(), ¤t_stats_); if (!status.ok()) { @@ -93,6 +112,7 @@ Status SplitWindowCursor::advance() } current_bytes_ += bytes; ++entries_; + ++step_entries; cursor_.next(); if (!cursor_.status().ok()) { return cursor_.status(); diff --git a/lib/crowdb-tree/src/btree/maintenance_split_layout.h b/lib/crowdb-tree/src/btree/maintenance_split_layout.h index f4aee5e86..100c80502 100644 --- a/lib/crowdb-tree/src/btree/maintenance_split_layout.h +++ b/lib/crowdb-tree/src/btree/maintenance_split_layout.h @@ -28,6 +28,8 @@ class SplitWindowCursor public: SplitWindowCursor(std::shared_ptr<const PageState> page, const PageMaintenanceLimits &limits); Status advance(); + // A work/output window yields without ending the complete flush layout. + Status advance_complete(); [[nodiscard]] bool done() const noexcept { @@ -47,6 +49,7 @@ class SplitWindowCursor } private: + Status advance(bool complete); std::shared_ptr<const PageState> page_; PageMaintenanceLimits limits_; SplitWindowLayout layout_; diff --git a/lib/crowdb-tree/src/btree/maintenance_split_step.cpp b/lib/crowdb-tree/src/btree/maintenance_split_step.cpp index 0703b0bdb..02fd4b4b2 100644 --- a/lib/crowdb-tree/src/btree/maintenance_split_step.cpp +++ b/lib/crowdb-tree/src/btree/maintenance_split_step.cpp @@ -184,6 +184,33 @@ MaintenancePublicationResult build_split_step_publication(std::shared_ptr<const namespace crowdb::tree::detail { +Status build_flush_leaf_outputs(MaintenanceBuilder &builder, uint64_t pid, + const std::shared_ptr<const PublishedPage> &old, const SplitWindowLayout &layout, + MaintenanceChildren *children) +{ + if (!layout.source || layout.source->content_id != old->leaf->content_id || + layout.source->representation_id != old->leaf->representation_id || + layout.source->route_generation != old->leaf->route_generation || + layout.source->range.lower != old->leaf->range.lower || layout.source->range.upper != old->leaf->range.upper || + layout.primary_bytes.size() != layout.separators.size() + 1 || + layout.statistics.size() != layout.primary_bytes.size()) { + return Status::invalid_argument("flush requires a complete leaf layout"); + } + children->pids.push_back(pid); + children->separators = layout.separators; + for (size_t i = 0; i < layout.separators.size(); ++i) { + uint64_t output = 0; + auto status = builder.allocate_pid(&output); + if (!status.ok()) { + return status; + } + children->pids.push_back(output); + } + PageVersion version; + auto status = next_page_version(*old->leaf, nullptr, &version); + return status.ok() ? prepare_outputs(builder, old, *children, layout, version) : status; +} + MaintenancePublicationResult build_split_layout_publication(std::shared_ptr<const PublishedTreeState> view, uint64_t leaf_pid, std::shared_ptr<InstallRecord> record, const MaintenancePublicationOptions &options, diff --git a/lib/crowdb-tree/src/btree/publication_commit.cpp b/lib/crowdb-tree/src/btree/publication_commit.cpp index 1967691c5..dfbb8144d 100644 --- a/lib/crowdb-tree/src/btree/publication_commit.cpp +++ b/lib/crowdb-tree/src/btree/publication_commit.cpp @@ -154,7 +154,8 @@ Status validate_transaction(const PublishedTreeState ¤t, const Publication if (transaction.handoff) { const auto &handoff = *transaction.handoff; if (handoff.batch != transaction.record->token.batch || handoff.frontier <= current.coverage || - (transaction.coverage && *transaction.coverage != handoff.frontier) || transaction.root_page_id) { + (transaction.coverage && *transaction.coverage != handoff.frontier) || + (transaction.root_page_id && !transaction.coverage)) { return Status::invalid_argument("invalid handoff identity or frontier"); } if (current.pending_handoff && current.pending_handoff != transaction.handoff) { diff --git a/lib/crowdb-tree/src/btree/tree_core_split_handoff.cpp b/lib/crowdb-tree/src/btree/tree_core_split_handoff.cpp index c5ec17d64..c82996a7e 100644 --- a/lib/crowdb-tree/src/btree/tree_core_split_handoff.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_split_handoff.cpp @@ -117,7 +117,12 @@ Status TreeCore::State::prepare_split_handoff(const SplitSourceSession &session, done(std::move(result), published->stats.dirty_pages); return; } - current->maintenance(std::move(published), std::move(done)); + if (current->maintenance) { + current->maintenance(std::move(published), std::move(done)); + } + else { + done(Status::Ok(), published->stats.dirty_pages); + } }; status = FlushBatch::prepare(*publisher, *binding->cpu(), binding->lane(), session.session, end, std::move(options), std::move(acknowledge), &handoff->operation, std::move(inputs), metadata); diff --git a/lib/crowdb-tree/tests/integration/checkpoint_split_progress_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_split_progress_test.cpp index e37ee3d4a..701fd718d 100644 --- a/lib/crowdb-tree/tests/integration/checkpoint_split_progress_test.cpp +++ b/lib/crowdb-tree/tests/integration/checkpoint_split_progress_test.cpp @@ -39,6 +39,8 @@ uint64_t checkpoint_jobs(size_t entries) auto *core = TreeCoreAccess::get(tree); core->binding()->account()->drain_retirements(); core->binding()->lane()->drain(); + EXPECT_EQ(core->capture()->stats.split_candidate_pages, 0U); + EXPECT_GT(core->capture()->stats.leaf_pages, 1U); const auto before = config.runtime->snapshot().cpu.completed; std::shared_ptr<CheckpointOperation> operation; status = core->prepare_checkpoint(checkpoint_detail::defaults(*core), &operation); @@ -49,7 +51,7 @@ uint64_t checkpoint_jobs(size_t entries) status = detail::wait_snapshot_operation(operation); EXPECT_TRUE(status.ok()) << status.to_string(); EXPECT_EQ(operation->result().durable_coverage, entries); - EXPECT_GT(operation->result().structural_windows, 1U); + EXPECT_EQ(operation->result().structural_windows, 0U); operation.reset(); core->binding()->account()->drain_retirements(); core->binding()->lane()->drain(); @@ -63,7 +65,7 @@ uint64_t checkpoint_jobs(size_t entries) } } // namespace -TEST(CheckpointSplitProgress, DoublingOneOversizedLeafKeepsCheckpointWorkBelowQuadraticGrowth) +TEST(CheckpointSplitProgress, DoublingFlushOutputKeepsCheckpointWorkBelowQuadraticGrowth) { const auto small = checkpoint_jobs(512); const auto large = checkpoint_jobs(1024); diff --git a/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp b/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp index 3695f93ef..2ec9897c2 100644 --- a/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp +++ b/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp @@ -339,8 +339,8 @@ TEST(FlushReservation, ColdRoutingUsesHeldBytesAndIoUnderCompleteHostPressure) EXPECT_EQ(operation->result().published_coverage, 2U); EXPECT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.to_string(); EXPECT_GT(operation->result().deferred_pages, 0U); - // Handoff caches the root once; cold maintenance has its own raw loader. - EXPECT_EQ(tree.store->reads.load(), 2U); + // One root load serves both routing and the parent join. + EXPECT_EQ(tree.store->reads.load(), 1U); EXPECT_EQ(tree.budget.snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kIo)], 0U); EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); EXPECT_EQ(tree.preparation.cache->snapshot().misses, 1U); @@ -380,9 +380,8 @@ TEST(FlushReservation, DescendingColdRoutePidsUseHeldCreditForEveryLoadedLevel) EXPECT_TRUE(operation->result().handoff_status.ok()) << operation->result().handoff_status.to_string(); EXPECT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.to_string(); EXPECT_EQ(operation->result().published_coverage, 2U); - // Handoff and the separate inner-repair/layout windows each read the two - // descending route PIDs. Both maintenance windows use their held credit. - EXPECT_EQ(tree.store->reads.load(), 6U); + // Each descending routing page is loaded once for the complete flow. + EXPECT_EQ(tree.store->reads.load(), 2U); EXPECT_EQ(tree.preparation.cache->snapshot().misses, 2U); EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); EXPECT_EQ(tree.budget.snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kIo)], 0U); @@ -408,22 +407,23 @@ TEST(FlushReservation, LiveResidencyAndColdProxyPublicationUseHeldCreditUnderCom EXPECT_TRUE(operation->result().handoff_status.ok()) << operation->result().handoff_status.to_string(); EXPECT_TRUE(operation->result().maintenance_status.ok()) << operation->result().maintenance_status.to_string(); EXPECT_EQ(operation->result().published_coverage, 2U); - // Handoff caches the inner. Maintenance reads the inner and first leaf - // directly, then publishes that decoded leaf with its exact summary. + // Both leaves finish using temporary raw loads and retain their cold backing. EXPECT_EQ(tree.store->reads.load(), 3U); EXPECT_EQ(tree.cache->snapshot().misses, 1U); const auto view = tree.publisher.capture(); - ASSERT_TRUE(view->mapping.get(0)->leaf->bases.front().resident); + EXPECT_FALSE(view->mapping.get(0)->leaf->bases.front().resident); EXPECT_FALSE(view->mapping.get(1)->leaf->bases.front().resident); + EXPECT_TRUE(view->mapping.get(0)->metrics.kv_known); + EXPECT_TRUE(view->mapping.get(1)->metrics.kv_known); EXPECT_FALSE(view->mapping.get(2)->inner); - EXPECT_EQ(tree.resources->snapshot().proxy_pages, 1U); + EXPECT_EQ(tree.resources->snapshot().proxy_pages, 2U); EXPECT_EQ(tree.resources->snapshot().resident_frames, 1U); EXPECT_EQ(tree.resources->snapshot().cache_pinned_bytes, 0U); EXPECT_EQ(tree.cache->snapshot().reserved_waiters, 0U); EXPECT_EQ(tree.budget.snapshot().allocated[static_cast<size_t>(detail::BudgetClass::kIo)], 0U); tree.cache->evict(); tree.runtime->drain(); - EXPECT_EQ(tree.resources->snapshot().proxy_pages, 2U); + EXPECT_EQ(tree.resources->snapshot().proxy_pages, 3U); EXPECT_EQ(tree.resources->snapshot().resident_frames, 0U); EXPECT_EQ(tree.resources->snapshot().cache_bytes, 0U); } diff --git a/lib/crowdb-tree/tests/integration/incremental_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/incremental_checkpoint_test.cpp index 48b710c5c..a21278617 100644 --- a/lib/crowdb-tree/tests/integration/incremental_checkpoint_test.cpp +++ b/lib/crowdb-tree/tests/integration/incremental_checkpoint_test.cpp @@ -180,7 +180,7 @@ TEST(IncrementalCheckpoint, LargeTreeEditPreservesUntouchedPhysicalPages) const auto windows = work->counters.snapshot()[static_cast<size_t>(TreeWorkCause::Checkpoint)] [static_cast<size_t>(TreeWorkCounter::SplitOperations)] - before_splits; - EXPECT_GT(windows, 0U); + EXPECT_EQ(windows, 0U); RecordProperty("checkpoint_split_transactions", std::to_string(windows)); RecordProperty("initial_checkpoint_ms", std::to_string(elapsed.count())); const auto before = durable_locations(tree); diff --git a/lib/crowdb-tree/tests/integration/parallel_flush_test.cpp b/lib/crowdb-tree/tests/integration/parallel_flush_test.cpp index 9e09b4430..b33f39d74 100644 --- a/lib/crowdb-tree/tests/integration/parallel_flush_test.cpp +++ b/lib/crowdb-tree/tests/integration/parallel_flush_test.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian <crow.db@outlook.com> // Licensed under the Apache License, Version 2.0. #include "btree/flush_batch.h" +#include "btree/published_route.h" #include <gtest/gtest.h> @@ -400,5 +401,244 @@ TEST(ParallelFlush, CannotDropAnotherClosedTableFromBatchInputs) Code::kInvalidArgument); EXPECT_EQ(publisher.capture()->coverage, 0U); } + +TEST(ParallelFlush, SubtreeConsolidatesWhileAnotherLeafHandoffIsPending) +{ + detail::FlushRuntime runtime(4, 64); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(64); + PublicationCoordinator publisher; + initialize(publisher, { + table({{"a", 1}, {"z", 1}} + ), table({{"a", 2}, {"z", 2}} + ) + }); + std::promise<void> right_entered; + auto right_ready = right_entered.get_future().share(); + std::promise<void> left_finished; + auto left_ready = left_finished.get_future().share(); + std::atomic<size_t> leaves{0}; + FlushBatchOptions options; + options.consolidation.emplace(); + // Keep both pages to inspect their distinct task completion order. + options.consolidation->pages.merge_trigger_bytes = 0; + options.before_handoff_for_tests = [&](uint64_t pid) { + if (pid == 0) { + EXPECT_EQ(right_ready.wait_for(std::chrono::seconds(5)), std::future_status::ready); + } + else { + right_entered.set_value(); + EXPECT_EQ(left_ready.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(publisher.capture()->coverage, 0U); + } + }; + options.after_page_for_tests = [&](uint64_t pid, uint32_t level) { + if (level == 0) { + EXPECT_EQ(publisher.capture()->mapping.get(pid)->leaf->lookup(pid == 0 ? "a" : "z").cell.slot, 2U); + leaves.fetch_add(1); + if (pid == 0) { + left_finished.set_value(); + } + } + else { + EXPECT_EQ(leaves.load(), 2U); + } + }; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 2, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + EXPECT_EQ(publisher.capture()->coverage, 2U); + EXPECT_EQ(publisher.capture()->stats.leaf_pages, 2U); + EXPECT_EQ(publisher.capture()->mapping.get(0)->metrics.overlay_runs, 1U); + EXPECT_EQ(publisher.capture()->mapping.get(1)->metrics.overlay_runs, 1U); +} + +TEST(ParallelFlush, CompleteSubtreeSplitContinuesAcrossEveryWorkAndOutputWindow) +{ + for (const size_t workers : {1U, 4U}) { + detail::FlushRuntime runtime(workers, 128); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(128); + PublicationCoordinator publisher; + auto source = std::make_shared<MemTable>(); + for (const auto *prefix : {"a", "z"}) { + for (size_t i = 0; i < 24; ++i) { + const auto key = std::string(prefix) + (i < 10 ? "0" : "") + std::to_string(i); + ASSERT_TRUE(source->upsert(key, 1, encode_cell_buf(1, OpKind::kPut, std::string(32, 'v')))); + } + } + source->close(); + initialize(publisher, {source}); + auto before = publisher.capture(); + FlushBatchOptions options; + options.run_limits.max_entries = 2; + options.consolidation.emplace(); + auto &limits = options.consolidation->pages; + limits.max_work_entries = 1; + limits.max_output_pages = 2; + limits.primary_page_bytes = 512; + limits.target_page_bytes = 256; + limits.merge_target_bytes = 192; + limits.merge_trigger_bytes = 0; + options.consolidation->max_inner_children = 2; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender()).has_value()); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + auto after = publisher.capture(); + EXPECT_EQ(after->coverage, 1U); + EXPECT_GT(after->stats.leaf_pages, 2U); + EXPECT_GT(after->stats.height, 2U); + EXPECT_EQ(after->stats.kv_known_count, 48U); + for (const auto *prefix : {"a", "z"}) { + for (size_t i = 0; i < 24; ++i) { + const auto key = std::string(prefix) + (i < 10 ? "0" : "") + std::to_string(i); + PublishedRouteCursor route(after->root_page_id, after->mapping, 64, false); + ASSERT_TRUE(route.seek(Slice(key), false).ok()); + ASSERT_TRUE(route.leaf()); + const auto found = route.leaf()->lookup(key); + ASSERT_EQ(found.state, LogicalLookupState::kValue); + EXPECT_EQ(found.cell.value.to_string(), std::string(32, 'v')); + } + } + after->mapping.visit(nullptr, [](void *, uint64_t, const auto &page) { + if (page->leaf) { + EXPECT_TRUE(page->metrics.page_size_known); + EXPECT_TRUE(page->metrics.kv_known); + EXPECT_FALSE(page->metrics.split_candidate); + } + }); + EXPECT_EQ(before->stats.leaf_pages, 2U); + EXPECT_EQ(before->coverage, 0U); + EXPECT_EQ(before->mapping.get(0)->leaf->lookup("a00").state, LogicalLookupState::kMissing); + } +} + +TEST(ParallelFlush, ParentMergesFinalLeavesAndContractsRootWithoutChangingOldView) +{ + detail::FlushRuntime runtime(4, 32); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(32); + PublicationCoordinator publisher; + initialize(publisher, { + table({{"a", 1}, {"z", 1}} + ), table({{"a", 2}, {"z", 2}} + ) + }); + auto before = publisher.capture(); + FlushBatchOptions options; + options.consolidation.emplace(); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 2, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + auto after = publisher.capture(); + EXPECT_EQ(after->stats.height, 1U); + EXPECT_EQ(after->stats.leaf_pages, 1U); + EXPECT_EQ(after->stats.inner_pages, 0U); + const auto root = after->mapping.get(after->root_page_id); + ASSERT_TRUE(root->leaf); + EXPECT_EQ(root->leaf->lookup("a").cell.slot, 2U); + EXPECT_EQ(root->leaf->lookup("z").cell.slot, 2U); + EXPECT_EQ(before->root_page_id, 2U); + ASSERT_TRUE(before->mapping.get(2)->inner); + EXPECT_EQ(before->mapping.get(2)->inner->children.size(), 2U); + EXPECT_EQ(before->coverage, 0U); +} + +TEST(ParallelFlush, FailedChildDrainsAcceptedSiblingsAndRetryReusesHandoff) +{ + detail::FlushRuntime runtime(4, 64); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(64); + PublicationCoordinator publisher; + initialize(publisher, { + table({{"a", 1}, {"z", 1}} + ) + }); + std::promise<void> failed; + auto saw_failure = failed.get_future(); + std::promise<void> release; + auto released = release.get_future().share(); + std::atomic<bool> inject{true}; + FlushBatchOptions options; + options.consolidation.emplace(); + options.consolidation->pages.merge_trigger_bytes = 0; + options.before_handoff_for_tests = [&](uint64_t pid) { + if (pid == 1) { + EXPECT_EQ(released.wait_for(std::chrono::seconds(5)), std::future_status::ready); + } + }; + options.after_page_for_tests = [&](uint64_t pid, uint32_t level) { + if (level == 0 && pid == 0 && inject.exchange(false)) { + failed.set_value(); + throw std::runtime_error("injected leaf failure"); + } + }; + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + std::promise<void> terminal; + auto finished = terminal.get_future(); + auto operation = stdexec::connect(batch->sender(), Receiver{&terminal}); + stdexec::start(operation); + EXPECT_EQ(saw_failure.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(finished.wait_for(std::chrono::milliseconds(0)), std::future_status::timeout); + EXPECT_EQ(publisher.capture()->coverage, 0U); + release.set_value(); + finished.get(); + runtime.drain(); + EXPECT_FALSE(batch->result().handoff_status.ok()); + EXPECT_EQ(publisher.capture()->coverage, 0U); + const auto left_runs = publisher.capture()->mapping.get(0)->leaf->overlays.size(); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + EXPECT_EQ(publisher.capture()->coverage, 1U); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->overlays.size(), left_runs); +} + +TEST(ParallelFlush, NoOpFrontierPreservesUnchangedRouting) +{ + detail::FlushRuntime runtime(2, 32); + auto lane = std::make_shared<detail::FlushRuntime::Lane>(32); + PublicationCoordinator publisher; + initialize(publisher, {table({})}); + auto lower = std::make_shared<PublishedPage>(); + auto lower_route = std::make_shared<InnerPageState>(); + lower_route->children = {2}; + lower->inner = std::move(lower_route); + lower->level = 2; + lower->metrics.leaf = false; + auto upper = std::make_shared<PublishedPage>(); + auto upper_route = std::make_shared<InnerPageState>(); + upper_route->children = {3}; + upper->inner = std::move(upper_route); + upper->level = 3; + upper->metrics.leaf = false; + auto request = std::make_shared<PublicationRequest>(); + request->transaction.record = std::make_shared<InstallRecord>(InstallToken{.generation = 1, .sequence = 1}); + request->transaction.pages = { + {.page_id = 3, .expected = {}, .replacement = lower}, + {.page_id = 4, .expected = {}, .replacement = upper} + }; + request->transaction.root_page_id = 4; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(request)); + auto before = publisher.capture(); + FlushBatchOptions options; + options.consolidation.emplace(); + std::shared_ptr<FlushBatch> batch; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, options, {}, &batch).ok()); + ASSERT_TRUE(stdexec::sync_wait(batch->sender())); + runtime.drain(); + ASSERT_TRUE(batch->result().handoff_status.ok()) << batch->result().handoff_status.to_string(); + const auto after = publisher.capture(); + EXPECT_EQ(after->coverage, 1U); + EXPECT_EQ(after->root_page_id, before->root_page_id); + EXPECT_EQ(after->stats.inner_pages, before->stats.inner_pages); + for (uint64_t pid = 0; pid < before->next_page_id; ++pid) { + EXPECT_EQ(after->mapping.get(pid), before->mapping.get(pid)); + } +} } // namespace } // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/gc_test.cpp b/lib/crowdb-tree/tests/unit/gc_test.cpp index 25f972acc..807107827 100644 --- a/lib/crowdb-tree/tests/unit/gc_test.cpp +++ b/lib/crowdb-tree/tests/unit/gc_test.cpp @@ -269,8 +269,10 @@ TEST(Gc, CompactSparseBlocksRespectsByteBudget) ASSERT_TRUE(BlockPageStore::open_blocks(tmp.path, 0, 0, blk, 1, &store).ok()); store->set_sync_mode(SyncMode::kSkip); Config opt; - opt.page_store = store.get(); - opt.leaf_split_bytes = 256; + opt.page_store = store.get(); + opt.leaf_split_bytes = 256; + // Keep surviving pages in their original extents to create sparse blocks. + opt.leaf_merge_bytes = 0; opt.merge_gc_block_free_threshold = 0.01; // Very small budget: only the first block should be selected. opt.merge_gc_max_relocation_bytes = blk / 4; @@ -289,6 +291,8 @@ TEST(Gc, CompactSparseBlocksRespectsByteBudget) ASSERT_TRUE(flush_1.ok()) << flush_1.to_string(); const auto snapshot_1 = t.snapshot(); ASSERT_TRUE(snapshot_1.ok()) << snapshot_1.to_string(); + ASSERT_GE(local->ledger->stats().address_high_water, 3 * blk) + << "fixture must populate multiple complete non-anchor blocks"; // Delete most keys to make blocks sparse. for (int i = 0; i < 200; ++i) { @@ -336,8 +340,10 @@ TEST(Gc, CompactSparseBlocksMaintainsDataIntegrity) ASSERT_TRUE(BlockPageStore::open_blocks(tmp.path, 0, 0, blk, 1, &store).ok()); store->set_sync_mode(SyncMode::kSkip); Config opt; - opt.page_store = store.get(); - opt.leaf_split_bytes = 256; + opt.page_store = store.get(); + opt.leaf_split_bytes = 256; + // Keep surviving pages in their original extents to create sparse blocks. + opt.leaf_merge_bytes = 0; opt.merge_gc_block_free_threshold = 0.30; opt.merge_gc_max_relocation_bytes = 8 * 1024 * 1024; Crowdbtree t(opt); From 6ce4220d3b6b75c97edaafc9dcf8ea75a6a319e4 Mon Sep 17 00:00:00 2001 From: Gian <crow.db@outlook.com> Date: Thu, 5 Nov 2026 18:08:11 +0800 Subject: [PATCH 03/17] Reuse unchanged local checkpoint segments --- lib/crowdb-tree/src/btree/backing_ref.h | 10 +- .../src/btree/local_compaction_internal.h | 2 +- .../src/btree/local_compaction_steps.cpp | 23 +++- lib/crowdb-tree/src/btree/page_encode.cpp | 59 +++++++-- lib/crowdb-tree/src/btree/page_encode.h | 14 +++ .../src/btree/page_encode_internal.h | 5 +- .../src/btree/page_encode_leaf.cpp | 7 +- .../src/btree/page_encode_plan.cpp | 13 +- .../src/btree/publication_snapshot.cpp | 16 ++- .../src/btree/tree_core_block_gc_async.cpp | 14 +++ .../src/snapshot/captured_frames.cpp | 35 +++++- .../src/snapshot/captured_frames.h | 4 + .../src/snapshot/captured_frames_internal.h | 1 + .../src/snapshot/captured_frames_reuse.cpp | 7 +- .../src/snapshot/captured_frames_step.cpp | 33 ++--- .../src/snapshot/captured_write.cpp | 81 ++++++++---- lib/crowdb-tree/src/snapshot/captured_write.h | 6 + .../snapshot/checkpoint_operation_steps.cpp | 10 +- .../src/snapshot/local_checkpoint.cpp | 15 ++- .../src/snapshot/local_checkpoint.h | 31 ++++- .../src/snapshot/local_checkpoint_internal.h | 4 + .../snapshot/local_checkpoint_metadata.cpp | 59 ++++++++- .../src/snapshot/local_checkpoint_pump.cpp | 2 +- .../snapshot/local_checkpoint_registry.cpp | 17 ++- .../src/snapshot/local_recovery_metadata.cpp | 6 + .../src/snapshot/local_recovery_pump.cpp | 2 +- .../integration/captured_frames_test.cpp | 67 ++++++++++ .../tests/integration/captured_write_test.cpp | 40 ++++++ .../local_checkpoint_directory_test.cpp | 117 +++++++++++++++++- .../integration/local_checkpoint_test.cpp | 99 +++++++++++++++ .../tests/integration/persist_test.cpp | 1 + .../tree_core_primary_reuse_test.cpp | 39 +++++- .../tests/unit/snapshot_publication_test.cpp | 54 ++++++++ 33 files changed, 789 insertions(+), 104 deletions(-) diff --git a/lib/crowdb-tree/src/btree/backing_ref.h b/lib/crowdb-tree/src/btree/backing_ref.h index 8484cc7e9..4412693e7 100644 --- a/lib/crowdb-tree/src/btree/backing_ref.h +++ b/lib/crowdb-tree/src/btree/backing_ref.h @@ -86,11 +86,15 @@ struct BackingExclusion uint64_t address = 0; uint64_t length = 0; + [[nodiscard]] bool overlaps(uint64_t begin, uint64_t bytes) const noexcept + { + return length != 0 && bytes != 0 && (begin >= address ? begin - address < length : address - begin < bytes); + } + [[nodiscard]] bool overlaps(const BackingRef &backing) const noexcept { - return length != 0 && backing.storage == BackingStorage::kDurableBlob && backing.reloadable() && - (backing.address >= address ? backing.address - address < length - : address - backing.address < backing.length); + return backing.storage == BackingStorage::kDurableBlob && backing.reloadable() && + overlaps(backing.address, backing.length); } }; } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/local_compaction_internal.h b/lib/crowdb-tree/src/btree/local_compaction_internal.h index 7ca58c588..62188dbdc 100644 --- a/lib/crowdb-tree/src/btree/local_compaction_internal.h +++ b/lib/crowdb-tree/src/btree/local_compaction_internal.h @@ -9,7 +9,7 @@ namespace crowdb::tree { struct LocalCompactionOperation::State : std::enable_shared_from_this<State> { - enum class Phase : uint8_t { kBegin, kSelected, kCheckpoint, kCleaned, kDone }; + enum class Phase : uint8_t { kBegin, kRetireAnchor, kSelected, kCheckpoint, kCleaned, kDone }; struct Receiver { diff --git a/lib/crowdb-tree/src/btree/local_compaction_steps.cpp b/lib/crowdb-tree/src/btree/local_compaction_steps.cpp index 83117fd40..b542d5c09 100644 --- a/lib/crowdb-tree/src/btree/local_compaction_steps.cpp +++ b/lib/crowdb-tree/src/btree/local_compaction_steps.cpp @@ -30,15 +30,16 @@ void LocalCompactionOperation::State::child_done(Status status) noexcept void LocalCompactionOperation::State::checkpoint() { - auto options = checkpoint_detail::defaults(*core); - options.cause = TreeWorkCause::GarbageCollection; + auto options = checkpoint_detail::defaults(*core); + options.cause = TreeWorkCause::GarbageCollection; + options.metadata.retire_previous_anchor = true; options.write.frames.exclusion = {.address = candidate.exclusion.address, .length = candidate.exclusion.length}; const auto status = core->prepare_checkpoint(options, &child, ticket); if (!status.ok()) { finish(status.without_message()); return; } - phase = Phase::kCheckpoint; + phase = candidate.exclusion.owner ? Phase::kCheckpoint : Phase::kRetireAnchor; child_ready.store(false, std::memory_order_release); attempt.emplace(child->sender(), weak_from_this().lock()); stdexec::start(attempt->connected); @@ -58,11 +59,15 @@ void LocalCompactionOperation::State::pump() finish(Status::Ok()); return; } + if (local->checkpoints->needs_anchor_retirement()) { + checkpoint(); + return; + } phase = Phase::kSelected; backend_progress->request(backend_node, weak_from_this().lock()); return; } - if (phase == Phase::kCheckpoint && stopped.load(std::memory_order_acquire)) { + if ((phase == Phase::kCheckpoint || phase == Phase::kRetireAnchor) && stopped.load(std::memory_order_acquire)) { child->request_stop(); } if (!child_ready.load(std::memory_order_acquire)) { @@ -75,6 +80,10 @@ void LocalCompactionOperation::State::pump() child.reset(); ++rotations; } + else if (phase == Phase::kRetireAnchor) { + attempt.reset(); + child.reset(); + } if (!child_status.ok()) { finish(child_status.without_message()); return; @@ -83,6 +92,12 @@ void LocalCompactionOperation::State::pump() finish(Status::unavailable()); return; } + if (phase == Phase::kRetireAnchor) { + phase = Phase::kSelected; + child_ready.store(false, std::memory_order_release); + backend_progress->request(backend_node, weak_from_this().lock()); + return; + } if (phase == Phase::kCleaned || !candidate.exclusion.owner) { finish(Status::Ok()); return; diff --git a/lib/crowdb-tree/src/btree/page_encode.cpp b/lib/crowdb-tree/src/btree/page_encode.cpp index 7a983569a..13600560a 100644 --- a/lib/crowdb-tree/src/btree/page_encode.cpp +++ b/lib/crowdb-tree/src/btree/page_encode.cpp @@ -138,29 +138,72 @@ PageEncodeStep FinalPageEncoder::next_frame(std::span<uint8_t> output) PageEncodePreparation prepare_final_page(const std::shared_ptr<const PublishedTreeState> &captured, uint64_t pid, const CapturedEncodeContext &context, const PageEncodeLimits &limits) +{ + if (!captured) { + return {.status = Status::invalid_argument("page encoding requires a fixed view"), + .needs_base = {}, + .needs_inner = {}, + .encoder = {}}; + } + const auto page = captured->mapping.get(pid); + if (page && page->inner) { + for (const auto child_id : page->inner->children) { + const auto child = captured->mapping.get(child_id); + if (!child || child->level + uint64_t{1} != page->level) { + return {.status = + Status::corruption("inner child is absent or has the wrong level in captured mapping"), + .needs_base = {}, + .needs_inner = {}, + .encoder = {}}; + } + } + if (context.inner_fences && context.inner_fences->lower_leaf != kInvalidPageId) { + for (const auto leaf_id : {context.inner_fences->lower_leaf, context.inner_fences->upper_leaf}) { + const auto leaf = captured->mapping.get(leaf_id); + if (!leaf || !leaf->leaf || leaf->level != 0 || !leaf->leaf->range.overlaps(context.range)) { + return {.status = Status::invalid_argument("inner fence leaf is absent from captured range"), + .needs_base = {}, + .needs_inner = {}, + .encoder = {}}; + } + } + } + } + return prepare_final_page({.page = page, + .page_id = pid, + .coverage = captured->coverage, + .gc_floor = captured->gc_floor, + .next_page_id = captured->next_page_id, + .context = context}, + limits); +} + +PageEncodePreparation prepare_final_page(const PageEncodeInput &input, const PageEncodeLimits &limits) { PageEncodePreparation result; + const auto &context = input.context; const size_t lower_bytes = context.range.lower.has_value() ? context.range.lower->size() : 0; const size_t upper_bytes = context.range.upper.has_value() ? context.range.upper->size() : 0; - if (!captured || pid >= MappingView::kPageLimit || + if (!input.page || input.page_id >= input.next_page_id || input.next_page_id > MappingView::kPageLimit || (context.range.lower && context.range.upper && *context.range.lower > *context.range.upper) || limits.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || limits.max_sources == 0 || - limits.max_entries == 0 || limits.max_work_entries == 0 || captured->gc_floor > captured->coverage || + limits.max_entries == 0 || limits.max_work_entries == 0 || input.gc_floor > input.coverage || lower_bytes > limits.max_metadata_bytes || upper_bytes > limits.max_metadata_bytes - lower_bytes) { - result.status = Status::invalid_argument("invalid captured page encoding bounds"); + result.status = Status::invalid_argument("invalid final page encoding bounds"); return result; } try { auto state = detail::reserved_shared<detail::FinalPageEncodeState>(limits.metadata, limits); - state->page = captured->mapping.get(pid); - state->pid = pid; - state->next_page_id = captured->next_page_id; + state->page = input.page; + state->pid = input.page_id; + state->next_page_id = input.next_page_id; if (!state->page) { result.status = Status::invalid_argument("encoded PID is absent from the captured mapping"); return result; } - result.status = state->page->level == 0 ? detail::prepare_leaf(*state, *captured, context, &result) - : detail::prepare_inner(*state, *captured, context, &result); + result.status = state->page->level == 0 + ? detail::prepare_leaf(*state, input.coverage, input.gc_floor, context, &result) + : detail::prepare_inner(*state, context, &result); if (result.status.ok() && !result.needs_base && !result.needs_inner) { detail::ReservationAllocator<FinalPageEncoder> allocator(limits.metadata); auto *storage = allocator.allocate(1); diff --git a/lib/crowdb-tree/src/btree/page_encode.h b/lib/crowdb-tree/src/btree/page_encode.h index 1148ca967..6ecefad07 100644 --- a/lib/crowdb-tree/src/btree/page_encode.h +++ b/lib/crowdb-tree/src/btree/page_encode.h @@ -41,6 +41,18 @@ struct CapturedEncodeContext std::shared_ptr<const void> overflow_dependencies = nullptr; }; +// The page owner supplies the final page and its completed child fences. +// Encoding borrows this exact output; it does not build another mapping. +struct PageEncodeInput +{ + std::shared_ptr<const PublishedPage> page; + uint64_t page_id; + uint64_t coverage; + uint64_t gc_floor; + uint64_t next_page_id; + CapturedEncodeContext context; +}; + struct ReservedOverflowIds { uint64_t first = kInvalidPageId; @@ -139,6 +151,7 @@ class FinalPageEncoder [[nodiscard]] PageEncodeStep next(std::span<uint8_t> output); private: + friend PageEncodePreparation prepare_final_page(const PageEncodeInput &, const PageEncodeLimits &); friend PageEncodePreparation prepare_final_page(const std::shared_ptr<const PublishedTreeState> &, uint64_t, const CapturedEncodeContext &, const PageEncodeLimits &); explicit FinalPageEncoder(std::shared_ptr<detail::FinalPageEncodeState> state) noexcept; @@ -153,4 +166,5 @@ class FinalPageEncoder // Oversized primary layouts require structural maintenance first. PageEncodePreparation prepare_final_page(const std::shared_ptr<const PublishedTreeState> &captured, uint64_t pid, const CapturedEncodeContext &context, const PageEncodeLimits &limits = {}); +PageEncodePreparation prepare_final_page(const PageEncodeInput &input, const PageEncodeLimits &limits = {}); } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_encode_internal.h b/lib/crowdb-tree/src/btree/page_encode_internal.h index 15c12abe1..b629a7d5b 100644 --- a/lib/crowdb-tree/src/btree/page_encode_internal.h +++ b/lib/crowdb-tree/src/btree/page_encode_internal.h @@ -44,10 +44,9 @@ struct FinalPageEncodeState bool prepared = false; }; -Status prepare_leaf(FinalPageEncodeState &state, const PublishedTreeState &captured, +Status prepare_leaf(FinalPageEncodeState &state, uint64_t coverage, uint64_t gc_floor, const CapturedEncodeContext &context, PageEncodePreparation *result); Status advance_leaf(FinalPageEncodeState &state, PageEncodePreparation *result); -Status prepare_inner(FinalPageEncodeState &state, const PublishedTreeState &captured, - const CapturedEncodeContext &context, PageEncodePreparation *result); +Status prepare_inner(FinalPageEncodeState &state, const CapturedEncodeContext &context, PageEncodePreparation *result); void encode_primary(FinalPageEncodeState &state, uint8_t *output); } // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/btree/page_encode_leaf.cpp b/lib/crowdb-tree/src/btree/page_encode_leaf.cpp index 18c4346e2..ef8d4e1cd 100644 --- a/lib/crowdb-tree/src/btree/page_encode_leaf.cpp +++ b/lib/crowdb-tree/src/btree/page_encode_leaf.cpp @@ -75,7 +75,7 @@ Status append_leaf_entry(FinalPageEncodeState &state, Slice key, LogicalCell cel } } // namespace -Status prepare_leaf(FinalPageEncodeState &state, const PublishedTreeState &captured, +Status prepare_leaf(FinalPageEncodeState &state, uint64_t coverage, uint64_t gc_floor, const CapturedEncodeContext &context, PageEncodePreparation *result) { const auto &page = state.page->leaf; @@ -84,9 +84,8 @@ Status prepare_leaf(FinalPageEncodeState &state, const PublishedTreeState &captu return Status::invalid_argument("final leaf must match its captured routing range"); } state.context = context; - state.gc_floor = captured.gc_floor; - state.cursor.emplace(page, context.range, captured.coverage, state.limits.max_sources, false, - state.limits.metadata); + state.gc_floor = gc_floor; + state.cursor.emplace(page, context.range, coverage, state.limits.max_sources, false, state.limits.metadata); const size_t capacity = std::min({state.limits.max_entries, static_cast<size_t>((state.limits.frame_bytes - kPageOverhead) / (kLeafSlotSize + kCellHeaderSize)), diff --git a/lib/crowdb-tree/src/btree/page_encode_plan.cpp b/lib/crowdb-tree/src/btree/page_encode_plan.cpp index 3fe4e8113..911d23104 100644 --- a/lib/crowdb-tree/src/btree/page_encode_plan.cpp +++ b/lib/crowdb-tree/src/btree/page_encode_plan.cpp @@ -6,8 +6,7 @@ namespace crowdb::tree::detail { -Status prepare_inner(FinalPageEncodeState &state, const PublishedTreeState &captured, - const CapturedEncodeContext &context, PageEncodePreparation *result) +Status prepare_inner(FinalPageEncodeState &state, const CapturedEncodeContext &context, PageEncodePreparation *result) { constexpr uint64_t kPageOverhead = kFrameHeaderSize + kFrameTrailerSize; if (state.page->leaf) { @@ -43,9 +42,8 @@ Status prepare_inner(FinalPageEncodeState &state, const PublishedTreeState &capt used += key.size() + kInnerSlotSize; } for (const auto pid : inner.children) { - const auto child = captured.mapping.get(pid); - if (!child || child->level + uint64_t{1} != state.page->level) { - return Status::corruption("inner child is absent or has the wrong level in captured mapping"); + if (pid >= state.next_page_id || pid == state.pid) { + return Status::corruption("final inner child has an invalid page identity"); } } const auto fences = context.inner_fences.value(); @@ -54,9 +52,8 @@ Status prepare_inner(FinalPageEncodeState &state, const PublishedTreeState &capt } if (fences.lower_leaf != kInvalidPageId) { for (const auto pid : {fences.lower_leaf, fences.upper_leaf}) { - const auto leaf = captured.mapping.get(pid); - if (!leaf || !leaf->leaf || leaf->level != 0 || !leaf->leaf->range.overlaps(context.range)) { - return Status::invalid_argument("inner fence leaf is absent from captured range"); + if (pid >= state.next_page_id || pid == state.pid) { + return Status::invalid_argument("final inner fence has an invalid page identity"); } } } diff --git a/lib/crowdb-tree/src/btree/publication_snapshot.cpp b/lib/crowdb-tree/src/btree/publication_snapshot.cpp index 5fc462f08..2e0252fc1 100644 --- a/lib/crowdb-tree/src/btree/publication_snapshot.cpp +++ b/lib/crowdb-tree/src/btree/publication_snapshot.cpp @@ -50,17 +50,23 @@ Status validate_snapshot_publication(const PublicationTransaction &transaction, if (!transaction.checkpoint || !transaction.checkpoint->baseline) { return Status::Ok(); } - const auto &action = *transaction.checkpoint; - const auto &base = *action.baseline; + const auto &action = *transaction.checkpoint; + const auto &base = *action.baseline; + const auto &previous = current.snapshot_baseline; + const bool reused = previous && base.pages == previous->pages && base.coverage == previous->coverage && + base.generation == previous->generation && base.wal_offset == previous->wal_offset && + base.gc_floor >= previous->gc_floor && base.snapshot_sequence == previous->snapshot_sequence; if (action.action != CheckpointAction::Commit || base.capture_token != action.token || base.generation != current.generation || !base.pages || base.pages->generation != base.generation || base.pages->coverage != base.coverage || base.coverage != current.checkpoint_capture_coverage || base.gc_floor != current.checkpoint_capture_floor || base.gc_floor > base.coverage || base.snapshot_sequence == 0 || base.backend_id == 0 || (!base.pages->frames.empty() && base.pages->backend_id != base.backend_id) || - (current.snapshot_baseline && (base.coverage < current.snapshot_baseline->coverage || - base.snapshot_sequence <= current.snapshot_baseline->snapshot_sequence || - base.backend_id != current.snapshot_baseline->backend_id))) { + (current.snapshot_baseline && + (base.coverage < current.snapshot_baseline->coverage || + base.snapshot_sequence < current.snapshot_baseline->snapshot_sequence || + (base.snapshot_sequence == current.snapshot_baseline->snapshot_sequence && !reused) || + base.backend_id != current.snapshot_baseline->backend_id))) { return Status::invalid_argument("snapshot baseline differs from its committed capture"); } return Status::Ok(); diff --git a/lib/crowdb-tree/src/btree/tree_core_block_gc_async.cpp b/lib/crowdb-tree/src/btree/tree_core_block_gc_async.cpp index b812fb7c0..2d34fb162 100644 --- a/lib/crowdb-tree/src/btree/tree_core_block_gc_async.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_block_gc_async.cpp @@ -102,6 +102,20 @@ struct Cleanup : std::enable_shared_from_this<Cleanup> attempt.reset(); operation.reset(); if (!exclusion.owner) { + if (local->checkpoints->needs_anchor_retirement()) { + auto options = defaults(*core); + options.cause = TreeWorkCause::GarbageCollection; + options.metadata.retire_previous_anchor = true; + const auto status = core->prepare_checkpoint(options, &operation); + if (!status.ok()) { + return; + } + attempt.emplace(operation->sender(), shared_from_this()); + stdexec::start(attempt->connected); + return; + } + core->publisher().reclaim(); + core->binding()->account()->drain_retirements(); reclaim_free_blocks(*core, metadata, backend); auto candidate = select_sparse_local_block( *store, *local->ledger, local->gc_cursor.load(std::memory_order_relaxed), metadata, 0.5, backend); diff --git a/lib/crowdb-tree/src/snapshot/captured_frames.cpp b/lib/crowdb-tree/src/snapshot/captured_frames.cpp index 7118398fc..bd306d097 100644 --- a/lib/crowdb-tree/src/snapshot/captured_frames.cpp +++ b/lib/crowdb-tree/src/snapshot/captured_frames.cpp @@ -25,7 +25,7 @@ CapturedFrameStream::State::State(std::shared_ptr<const PublishedTreeState> capt : detail::reserved_shared<CapturedFrameBuffer>(limits.encode.metadata, limits.encode)) { stack.reserve(limits.max_depth); - overflow_end = view->next_page_id; + overflow_end = view ? view->next_page_id : 0; } Status CapturedFrameStream::create(std::shared_ptr<const PublishedTreeState> captured, @@ -60,6 +60,36 @@ Status CapturedFrameStream::create(std::shared_ptr<const PublishedTreeState> cap return Status::Ok(); } +Status CapturedFrameStream::create(PageEncodeInput input, std::shared_ptr<PageLoader> loader, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedFrameLimits limits, + ReserveOverflow reserve_overflow, std::shared_ptr<CapturedFrameStream> *out) +{ + if (!input.page || input.page_id >= input.next_page_id || input.next_page_id > MappingView::kPageLimit || + input.gc_floor > input.coverage || !runtime || !lane || out == nullptr || limits.max_depth == 0 || + limits.max_depth > 64 || limits.max_pages == 0 || limits.max_pages > MappingView::kPageLimit || + limits.encode.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || limits.encode.max_sources == 0 || + limits.encode.max_entries == 0 || limits.encode.max_work_entries == 0 || limits.selected_page || + (input.page->level != 0 && !input.context.inner_fences)) { + return Status::invalid_argument("final page stream requires one owned output and bounded work"); + } + try { + const auto metadata = limits.encode.metadata; + limits.max_depth = 1; + auto state = detail::reserved_shared<State>(metadata, nullptr, std::move(loader), std::move(runtime), + std::move(lane), std::move(limits), std::move(reserve_overflow)); + state->overflow_end = input.next_page_id; + state->single_page = std::move(input); + *out = detail::reserved_construct<CapturedFrameStream>(metadata, [&](auto *storage) { + ::new (static_cast<void *>(storage)) CapturedFrameStream(std::move(state)); + }); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } + return Status::Ok(); +} + std::optional<size_t> CapturedFrameStream::allocation_bound(const CapturedFrameLimits &limits) noexcept { using namespace detail; @@ -72,8 +102,7 @@ std::optional<size_t> CapturedFrameStream::allocation_bound(const CapturedFrameL } return AllocationBound(shared_allocation_bound<State>() + shared_allocation_bound<CapturedFrameStream>() + shared_allocation_bound<State::Load>() + shared_allocation_bound<FinalPageEncodeState>() + - shared_allocation_bound<FinalPageEncoder>() + shared_allocation_bound<PublishedTreeState>() + - MappingView::path_copy_bound()) + shared_allocation_bound<FinalPageEncoder>()) .add_product(limits.max_depth, sizeof(State::Path) + shared_allocation_bound<PublishedPage>()) .add_product(2, std::max(*page, limits.page_metadata_bytes)) .add(limits.encode.max_metadata_bytes) diff --git a/lib/crowdb-tree/src/snapshot/captured_frames.h b/lib/crowdb-tree/src/snapshot/captured_frames.h index a00f6607a..e49322d23 100644 --- a/lib/crowdb-tree/src/snapshot/captured_frames.h +++ b/lib/crowdb-tree/src/snapshot/captured_frames.h @@ -103,6 +103,10 @@ class CapturedFrameStream std::shared_ptr<detail::FlushRuntime> runtime, std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedFrameLimits limits, ReserveOverflow reserve_overflow, std::shared_ptr<CapturedFrameStream> *out); + static Status create(PageEncodeInput input, std::shared_ptr<PageLoader> loader, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedFrameLimits limits, + ReserveOverflow reserve_overflow, std::shared_ptr<CapturedFrameStream> *out); [[nodiscard]] static std::optional<size_t> allocation_bound(const CapturedFrameLimits &limits) noexcept; [[nodiscard]] detail::CallbackSender sender() noexcept; [[nodiscard]] const CapturedFrameResult &result() const noexcept; diff --git a/lib/crowdb-tree/src/snapshot/captured_frames_internal.h b/lib/crowdb-tree/src/snapshot/captured_frames_internal.h index f217a8122..1f8fdced3 100644 --- a/lib/crowdb-tree/src/snapshot/captured_frames_internal.h +++ b/lib/crowdb-tree/src/snapshot/captured_frames_internal.h @@ -53,6 +53,7 @@ struct CapturedFrameStream::State : std::enable_shared_from_this<State> void finish(Status status, bool fatal = false) noexcept; std::shared_ptr<const PublishedTreeState> view; + std::optional<PageEncodeInput> single_page; std::shared_ptr<PageLoader> loader; std::shared_ptr<detail::FlushRuntime> runtime; std::shared_ptr<detail::FlushRuntime::Lane> lane; diff --git a/lib/crowdb-tree/src/snapshot/captured_frames_reuse.cpp b/lib/crowdb-tree/src/snapshot/captured_frames_reuse.cpp index cdf48fa73..15bec72bc 100644 --- a/lib/crowdb-tree/src/snapshot/captured_frames_reuse.cpp +++ b/lib/crowdb-tree/src/snapshot/captured_frames_reuse.cpp @@ -7,9 +7,10 @@ namespace crowdb::tree { bool CapturedFrameStream::State::reuse_primary() { - const auto &path = stack.back(); - const bool leaf = path.page->level == 0; - if (leaf ? !can_reuse_primary(*path.page, path.pid, path.range, view->gc_floor, limits.reuse_backend_id, + const auto &path = stack.back(); + const bool leaf = path.page->level == 0; + const auto gc_floor = single_page ? single_page->gc_floor : view->gc_floor; + if (leaf ? !can_reuse_primary(*path.page, path.pid, path.range, gc_floor, limits.reuse_backend_id, limits.encode.frame_bytes) : !can_reuse_inner_primary(*path.page, path.pid, path.fences, limits.reuse_backend_id, limits.encode.frame_bytes)) { diff --git a/lib/crowdb-tree/src/snapshot/captured_frames_step.cpp b/lib/crowdb-tree/src/snapshot/captured_frames_step.cpp index 78fe66393..88bf787d5 100644 --- a/lib/crowdb-tree/src/snapshot/captured_frames_step.cpp +++ b/lib/crowdb-tree/src/snapshot/captured_frames_step.cpp @@ -56,7 +56,14 @@ void CapturedFrameStream::State::validate(Path &path) bool CapturedFrameStream::State::initialize() { initialized = true; - if (limits.selected_page) { + if (single_page) { + stack.push_back({.pid = single_page->page_id, + .page = single_page->page, + .range = single_page->context.range, + .fences = single_page->context.inner_fences.value_or(InnerEncodeFences{})}); + ++counters.pages; + } + else if (limits.selected_page) { const auto selected = view->mapping.get(*limits.selected_page); if (!selected || (selected->level == 0 && !selected->leaf)) { finish(Status::unavailable("selected writeback page was replaced"), true); @@ -120,7 +127,8 @@ void CapturedFrameStream::State::step() return; } } - if (!limits.selected_page && path.page->level != 0 && path.child < path.page->inner->children.size()) { + if (!single_page && !limits.selected_page && path.page->level != 0 && + path.child < path.page->inner->children.size()) { const auto &inner = *path.page->inner; const auto index = path.child++; auto range = path.range; @@ -151,21 +159,18 @@ void CapturedFrameStream::State::prepare() return; } } - auto input = view; - if (path.page != view->mapping.get(path.pid)) { - // Only this page's private loaded descriptor is substituted. Unvisited - // PIDs and inner child validation still use the original captured root. - auto resolved = detail::reserved_shared<PublishedTreeState>(limits.encode.metadata, *view); - MappingView::CopyStats copied; - const PageReplacement replacement{.page_id = path.pid, .expected = {}, .replacement = path.page}; - resolved->mapping = view->mapping.replace({&replacement, 1}, copied, limits.encode.metadata); - input = std::move(resolved); - } - CapturedEncodeContext context{.range = path.range, .overflow_dependencies = view}; + CapturedEncodeContext context{ + .range = path.range, .overflow_dependencies = single_page ? single_page->context.overflow_dependencies : view}; if (path.page->level != 0) { context.inner_fences = path.fences; } - auto prepared = prepare_final_page(input, path.pid, context, limits.encode); + auto prepared = prepare_final_page({.page = path.page, + .page_id = path.pid, + .coverage = single_page ? single_page->coverage : view->coverage, + .gc_floor = single_page ? single_page->gc_floor : view->gc_floor, + .next_page_id = single_page ? single_page->next_page_id : view->next_page_id, + .context = std::move(context)}, + limits.encode); if (!prepared.status.ok()) { if (prepared.needs_split) { report.needs_split = path.pid; diff --git a/lib/crowdb-tree/src/snapshot/captured_write.cpp b/lib/crowdb-tree/src/snapshot/captured_write.cpp index 59905ac7a..cf8934833 100644 --- a/lib/crowdb-tree/src/snapshot/captured_write.cpp +++ b/lib/crowdb-tree/src/snapshot/captured_write.cpp @@ -7,6 +7,22 @@ namespace crowdb::tree { +namespace +{ +bool valid_write_limits(const CapturedWriteLimits &limits, const PageLoadBudget &budget) +{ + return limits.max_frames != 0 && !limits.frames.plan_only && limits.max_dependency_steps != 0 && + limits.frames.encode.frame_bytes <= limits.write.max_frame_bytes && + PageWriteOperation::allocation_bound(limits.write).has_value() && + static_cast<bool>(budget.bytes) == static_cast<bool>(budget.io) && + (!limits.frames.encode.metadata || limits.frames.encode.metadata == budget.bytes) && + (!budget.bytes || + ((budget.bytes->category() == detail::BudgetClass::kRouting || + budget.bytes->category() == detail::BudgetClass::kReclaim) && + budget.io->category() == detail::BudgetClass::kIo && budget.bytes->account() == budget.io->account())); +} +} // namespace + std::optional<size_t> CapturedWriteOperation::allocation_bound(CapturedWriteLimits limits, size_t frame_count_upper, size_t dependency_entries, size_t reference_heads) noexcept @@ -46,16 +62,47 @@ Status CapturedWriteOperation::create(std::shared_ptr<const PublishedTreeState> std::shared_ptr<const CapturedDurablePages> inherited, std::shared_ptr<CapturedWriteOperation> *out) { - if (!captured || !store || !runtime || !lane || !reserve_extent || out == nullptr || limits.max_frames == 0 || - limits.frames.plan_only || limits.max_dependency_steps == 0 || - limits.frames.encode.frame_bytes > limits.write.max_frame_bytes || - !PageWriteOperation::allocation_bound(limits.write) || - static_cast<bool>(budget.bytes) != static_cast<bool>(budget.io) || - (limits.frames.encode.metadata && limits.frames.encode.metadata != budget.bytes) || - (budget.bytes && - ((budget.bytes->category() != detail::BudgetClass::kRouting && - budget.bytes->category() != detail::BudgetClass::kReclaim) || - budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + if (!captured || !store || !runtime || !lane || !reserve_extent || out == nullptr || + !valid_write_limits(limits, budget)) { + return Status::invalid_argument(); + } + try { + limits.frames.encode.metadata = budget.bytes; + limits.frames.cpu_progress = budget.cpu_progress; + if (budget.work_metrics) { + limits.frames.encode.work_metrics = budget.work_metrics; + limits.frames.encode.cause = budget.cause; + } + std::shared_ptr<CapturedFrameStream> stream; + auto status = CapturedFrameStream::create(captured, std::move(loader), runtime, lane, limits.frames, + std::move(reserve_overflow), &stream); + if (!status.ok()) { + return status; + } + auto pages = detail::reserved_shared<CapturedDurablePages>(budget.bytes, budget.bytes); + pages->root_page_id = captured->root_page_id; + pages->next_page_id = captured->next_page_id; + pages->coverage = captured->coverage; + pages->generation = captured->generation; + return create(std::move(stream), std::move(pages), std::move(store), std::move(runtime), std::move(lane), + std::move(limits), std::move(budget), std::move(reserve_extent), std::move(inherited), out); + } + catch (const std::bad_alloc &) { + return Status::resource_exhausted(); + } +} + +Status CapturedWriteOperation::create(std::shared_ptr<CapturedFrameStream> stream, + std::shared_ptr<CapturedDurablePages> pages, + std::shared_ptr<AsyncPageStore> store, + std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedWriteLimits limits, + PageLoadBudget budget, PageWriteOperation::ReserveExtent reserve_extent, + std::shared_ptr<const CapturedDurablePages> inherited, + std::shared_ptr<CapturedWriteOperation> *out) +{ + if (!stream || !pages || !pages->frames.empty() || !store || !runtime || !lane || !reserve_extent || + out == nullptr || !valid_write_limits(limits, budget)) { return Status::invalid_argument(); } if (limits.write.chunk_backed ? dynamic_cast<ScheduledPageStore *>(store.get()) == nullptr @@ -77,17 +124,9 @@ Status CapturedWriteOperation::create(std::shared_ptr<const PublishedTreeState> state->budget = std::move(budget); state->reserve = std::move(reserve_extent); state->inherited = std::move(inherited); - state->pages = detail::reserved_shared<CapturedDurablePages>(state->budget.bytes, state->budget.bytes); - state->pages->root_page_id = captured->root_page_id; - state->pages->next_page_id = captured->next_page_id; - state->pages->coverage = captured->coverage; - state->pages->generation = captured->generation; - auto status = CapturedFrameStream::create(std::move(captured), std::move(loader), state->runtime, state->lane, - state->limits.frames, std::move(reserve_overflow), &state->stream); - if (!status.ok()) { - return status; - } - *out = detail::reserved_construct<CapturedWriteOperation>(state->budget.bytes, [&](auto *storage) { + state->pages = std::move(pages); + state->stream = std::move(stream); + *out = detail::reserved_construct<CapturedWriteOperation>(state->budget.bytes, [&](auto *storage) { ::new (static_cast<void *>(storage)) CapturedWriteOperation(std::move(state)); }); } diff --git a/lib/crowdb-tree/src/snapshot/captured_write.h b/lib/crowdb-tree/src/snapshot/captured_write.h index 6b6a1d411..40fc7be79 100644 --- a/lib/crowdb-tree/src/snapshot/captured_write.h +++ b/lib/crowdb-tree/src/snapshot/captured_write.h @@ -56,6 +56,12 @@ class CapturedWriteOperation PageWriteOperation::ReserveExtent reserve_extent, std::shared_ptr<const CapturedDurablePages> inherited, std::shared_ptr<CapturedWriteOperation> *out); + static Status create(std::shared_ptr<CapturedFrameStream> stream, std::shared_ptr<CapturedDurablePages> pages, + std::shared_ptr<AsyncPageStore> store, std::shared_ptr<detail::FlushRuntime> runtime, + std::shared_ptr<detail::FlushRuntime::Lane> lane, CapturedWriteLimits limits, + PageLoadBudget budget, PageWriteOperation::ReserveExtent reserve_extent, + std::shared_ptr<const CapturedDurablePages> inherited, + std::shared_ptr<CapturedWriteOperation> *out); [[nodiscard]] detail::CallbackSender sender() noexcept; [[nodiscard]] const CapturedWriteResult &result() const noexcept; void request_stop() noexcept; diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp index 4938a2292..5b5c94fb2 100644 --- a/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp @@ -276,8 +276,11 @@ void CheckpointOperation::State::written() begin_chunk_checkpoint(); return; } - auto status = LocalCheckpointOperation::create(written_pages, backend->checkpoints, store, runtime, lane, - options.metadata, budget, &checkpoint); + auto metadata = options.metadata; + metadata.exclusion = options.write.frames.exclusion; + metadata.retire_previous_anchor = metadata.retire_previous_anchor || metadata.exclusion.length != 0; + auto status = LocalCheckpointOperation::create(written_pages, backend->checkpoints, store, runtime, lane, metadata, + budget, &checkpoint); if (!status.ok()) { reconcile(std::move(status)); return; @@ -295,6 +298,9 @@ void CheckpointOperation::State::checkpointed() return; } } + else { + written_pages = checkpoint->result().checkpoint->pages; + } report.durable_committed = true; if (commit_metrics) { (void)commit_metrics->finish(TreeOutcome::Success, tree_metrics_now_ns()); diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp index 4fa870572..c359ba4b8 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp @@ -75,7 +75,7 @@ std::optional<size_t> LocalCheckpointOperation::allocation_bound(size_t segments shared_allocation_bound<State::IoStep>()) .add(std::max<uint64_t>(bytes, snapshot::superblock_slot_bytes(iu))) .add(std::bit_ceil(std::max<size_t>(4096, iu)) - 1) - .add_product(3 * (segments + 1), sizeof(DirEntry) + sizeof(ProtectedExtent)) + .add_product(3 * (segments + 1), sizeof(DirEntry) + sizeof(CheckpointMetadata::Segment)) .add_product(segments + 1, ExtentLedger::reservation_metadata_bound()) .value(); } @@ -129,9 +129,11 @@ void LocalCheckpointOperation::State::begin() finish(std::move(status)); return; } - admitted = true; - iu = registry->ledger_->limits().iu_size; - segment_bytes = round_up_to_iu(segment_image_encoded_size(kMappingSegmentSize), iu); + admitted = true; + const auto sequence = registry->sequence_.load(std::memory_order_relaxed); + previous = sequence == 0 ? nullptr : registry->anchors_[sequence % 2 == 0 ? 1 : 0]; + iu = registry->ledger_->limits().iu_size; + segment_bytes = round_up_to_iu(segment_image_encoded_size(kMappingSegmentSize), iu); window = detail::reserved_shared<Window>(budget.bytes, std::max<uint64_t>(segment_bytes, snapshot::superblock_slot_bytes(iu)), std::bit_ceil(std::max<size_t>(4096, iu)), budget.bytes); @@ -155,12 +157,13 @@ void LocalCheckpointOperation::State::finish(Status status) noexcept const bool cancelled = !anchor_touched && stopped.load(std::memory_order_acquire); report.status = cancelled ? Status::unavailable() : std::move(status); if (admitted) { - registry->finish(image, report.status.ok()); + registry->finish(image, report.status.ok() && !unchanged); } if (report.status.ok()) { - report.checkpoint = std::move(image); + report.checkpoint = unchanged ? std::move(previous) : std::move(image); } image.reset(); + previous.reset(); window.reset(); reserved.owner.reset(); store.reset(); diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint.h b/lib/crowdb-tree/src/snapshot/local_checkpoint.h index 07c85913e..a6070c640 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint.h +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint.h @@ -3,6 +3,7 @@ #pragma once #include "backend/local/extent_ledger.h" +#include "crowdb-tree/maptable/mapping_persist.h" #include "snapshot/captured_write.h" #include "snapshot/commit_anchor.h" @@ -12,20 +13,32 @@ namespace crowdb::tree { struct CheckpointMetadata { + using Segment = std::pair<DirEntry, ProtectedExtent>; + explicit CheckpointMetadata(const std::shared_ptr<detail::AllocationReservation> &budget) - : extents(detail::ReservationAllocator<ProtectedExtent>(budget)) + : extents(detail::ReservationAllocator<ProtectedExtent>(budget)), + segments(detail::ReservationAllocator<Segment>(budget)) { } bool retire_step() noexcept { - for (size_t i = 0; i < 64 && !extents.empty(); ++i) { - extents.pop_back(); + for (size_t i = 0; i < 64; ++i) { + if (!segments.empty()) { + segments.pop_back(); + } + else if (!extents.empty()) { + extents.pop_back(); + } + else { + return true; + } } - return extents.empty(); + return segments.empty() && extents.empty(); } detail::ReservedVector<ProtectedExtent> extents; + detail::ReservedVector<Segment> segments; }; struct ProtectedCheckpoint @@ -75,6 +88,11 @@ class LocalCheckpointRegistry return uncertain_.load(std::memory_order_acquire); } + [[nodiscard]] bool needs_anchor_retirement() const noexcept + { + return obsolete_anchor_.load(std::memory_order_acquire); + } + private: friend class LocalCheckpointOperation; LocalCheckpointRegistry() = default; @@ -87,6 +105,7 @@ class LocalCheckpointRegistry std::shared_ptr<const ProtectedCheckpoint> candidate_; std::atomic<bool> busy_{false}; std::atomic<bool> uncertain_{false}; + std::atomic<bool> obsolete_anchor_{false}; std::atomic<uint64_t> coverage_{0}; std::atomic<uint64_t> sequence_{0}; uint64_t generation_ = 0; @@ -94,7 +113,9 @@ class LocalCheckpointRegistry struct LocalCheckpointLimits { - size_t max_segments = 1U << 16; + size_t max_segments = 1U << 16; + BackingExclusion exclusion{}; + bool retire_previous_anchor = false; }; struct LocalCheckpointResult diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_internal.h b/lib/crowdb-tree/src/snapshot/local_checkpoint_internal.h index 9318a3fd6..c597bd68f 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint_internal.h +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_internal.h @@ -60,6 +60,7 @@ struct LocalCheckpointOperation::State : std::enable_shared_from_this<State> void pump(); void begin(); void segment(); + void finish_segments(); Status pack_frame(const DurableCapturedFrame &frame, uint64_t *word); void directory_window(); void anchor(); @@ -75,6 +76,7 @@ struct LocalCheckpointOperation::State : std::enable_shared_from_this<State> std::shared_ptr<detail::FlushRuntime> runtime; std::shared_ptr<detail::FlushRuntime::Lane> lane; std::shared_ptr<ProtectedCheckpoint> image; + std::shared_ptr<const ProtectedCheckpoint> previous; std::shared_ptr<Window> window; detail::ReservedVector<DirEntry> directory; std::optional<snapshot::DirectoryEncoder> encoder; @@ -88,6 +90,8 @@ struct LocalCheckpointOperation::State : std::enable_shared_from_this<State> uint32_t iu = 0; Stage stage = Stage::Begin; bool admitted = false; + bool reused_segments = true; + bool unchanged = false; bool anchor_touched = false; bool waiting = false; Status event_status; diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp index 4f52a862c..a017debee 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp @@ -35,9 +35,7 @@ Status LocalCheckpointOperation::State::pack_frame(const DurableCapturedFrame &f void LocalCheckpointOperation::State::segment() { if (position == image->pages->frames.end()) { - encoder.emplace(directory); - image->anchor.segdir_len = encoder->logical_bytes(); - reserve(round_up_to_iu(encoder->logical_bytes(), iu), Stage::DirectoryReserved); + finish_segments(); return; } if (directory.size() == limits.max_segments) { @@ -50,6 +48,9 @@ void LocalCheckpointOperation::State::segment() return; } words.fill(0); + auto old = previous ? previous->pages->frames.lower_bound(index * kMappingSegmentSize) + : CapturedDurablePages::Frames::const_iterator{}; + bool same = previous != nullptr; uint32_t live = 0; while (position != image->pages->frames.end() && position->first / kMappingSegmentSize == index) { const auto &[pid, frame] = *position; @@ -59,9 +60,33 @@ void LocalCheckpointOperation::State::segment() finish(std::move(status)); return; } + if (same) { + same = old != previous->pages->frames.end() && old->first == pid && old->second.backing && + words[pid % kMappingSegmentSize] == + slot_word::pack_unloaded(old->second.backing->address / iu, + static_cast<uint32_t>(old->second.backing->length / iu)); + if (same) { + ++old; + } + } ++live; ++position; } + if (same && (old == previous->pages->frames.end() || old->first / kMappingSegmentSize != index)) { + const auto &segments = previous->metadata->segments; + const auto found = + std::lower_bound(segments.begin(), segments.end(), index, + [](const auto &segment, uint64_t pid) { return segment.first.seg_idx < pid; }); + if (found != segments.end() && found->first.seg_idx == index && + !limits.exclusion.overlaps(found->second.address, found->second.length)) { + directory.push_back(found->first); + image->metadata->segments.push_back(*found); + stage = Stage::Segment; + wake(); + return; + } + } + reused_segments = false; const SegmentImageHeader header{.seg_idx = static_cast<uint32_t>(index), .generation = image->anchor.snapshot_seq, .slot_count = kMappingSegmentSize, @@ -78,6 +103,34 @@ void LocalCheckpointOperation::State::segment() reserve(segment_bytes, Stage::SegmentReserved); } +void LocalCheckpointOperation::State::finish_segments() +{ + if (previous && reused_segments && directory.size() == previous->metadata->segments.size() && + snapshot::DirectoryEncoder(directory).logical_bytes() == previous->anchor.segdir_len && + !limits.exclusion.overlaps(previous->anchor.segdir_addr, round_up_to_iu(previous->anchor.segdir_len, iu))) { + const auto &pages = *image->pages; + const auto &old = *previous->pages; + unchanged = !(limits.retire_previous_anchor && registry->needs_anchor_retirement()) && + pages.coverage == old.coverage && pages.generation == old.generation && + pages.root_page_id == old.root_page_id && pages.next_page_id == old.next_page_id && + image->anchor.leaf_count == previous->anchor.leaf_count && + image->anchor.inner_count == previous->anchor.inner_count; + if (unchanged) { + finish(Status::Ok()); + return; + } + image->metadata = previous->metadata; + image->anchor.segdir_addr = previous->anchor.segdir_addr; + image->anchor.segdir_len = previous->anchor.segdir_len; + image->anchor.segdir_crc = previous->anchor.segdir_crc; + io(true, 0, 0, Stage::BeforeAnchor); + return; + } + encoder.emplace(directory); + image->anchor.segdir_len = encoder->logical_bytes(); + reserve(round_up_to_iu(encoder->logical_bytes(), iu), Stage::DirectoryReserved); +} + void LocalCheckpointOperation::State::directory_window() { if (!encoder) { diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_pump.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_pump.cpp index 544e184d6..2f22b6940 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint_pump.cpp +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_pump.cpp @@ -90,8 +90,8 @@ void LocalCheckpointOperation::State::pump() segment(); break; case Stage::SegmentReserved: - image->metadata->extents.push_back(reserved); segment_entry.image_addr = reserved.address; + image->metadata->segments.emplace_back(segment_entry, reserved); directory.push_back(segment_entry); io(false, reserved.address, segment_bytes, Stage::SegmentWritten); break; diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_registry.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_registry.cpp index fbf962619..5f1413796 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint_registry.cpp +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_registry.cpp @@ -4,6 +4,14 @@ namespace crowdb::tree { +namespace +{ +bool anchors_differ(const std::array<std::shared_ptr<const ProtectedCheckpoint>, 2> &anchors) +{ + return anchors[0] && anchors[1] && anchors[0]->anchor.segdir_addr != anchors[1]->anchor.segdir_addr; +} +} // namespace + Status LocalCheckpointRegistry::create(std::shared_ptr<ExtentLedger> ledger, std::shared_ptr<detail::AllocationReservation> metadata, std::array<std::shared_ptr<const ProtectedCheckpoint>, 2> recovered, @@ -24,8 +32,9 @@ Status LocalCheckpointRegistry::create(std::shared_ptr<ExtentLedger> continue; } const auto &anchor = image->anchor; - if (!image->pages || anchor.snapshot_seq == 0 || (anchor.snapshot_seq % 2 == 0 ? 1U : 0U) != slot || - anchor.magic != snapshot::kAnchorMagic || anchor.format_version != snapshot::kFormatVersion || + if (!image->pages || !image->metadata || anchor.snapshot_seq == 0 || + (anchor.snapshot_seq % 2 == 0 ? 1U : 0U) != slot || anchor.magic != snapshot::kAnchorMagic || + anchor.format_version != snapshot::kFormatVersion || anchor.last_applied_slot != image->pages->coverage || (!image->pages->frames.empty() && image->pages->backend_id != registry->ledger_->limits().backend_id)) { return Status::invalid_argument(); @@ -37,7 +46,8 @@ Status LocalCheckpointRegistry::create(std::shared_ptr<ExtentLedger> } } registry->anchors_ = std::move(recovered); - *out = std::move(registry); + registry->obsolete_anchor_.store(anchors_differ(registry->anchors_)); + *out = std::move(registry); } catch (const std::bad_alloc &) { return Status::resource_exhausted(); @@ -71,6 +81,7 @@ void LocalCheckpointRegistry::finish(std::shared_ptr<const ProtectedCheckpoint> const auto &anchor = candidate->anchor; generation_ = candidate->pages->generation; anchors_[anchor.snapshot_seq % 2 == 0 ? 1 : 0] = std::move(candidate); + obsolete_anchor_.store(anchors_differ(anchors_), std::memory_order_release); sequence_.store(anchor.snapshot_seq, std::memory_order_release); coverage_.store(anchor.last_applied_slot, std::memory_order_release); candidate_.reset(); diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp index 676c96abc..e312a12a1 100644 --- a/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp +++ b/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp @@ -131,6 +131,12 @@ void LocalRecoveryOperation::State::segment_read() finish(Status::corruption()); return; } + if (limits.accept_legacy_chunk_refs && std::ranges::any_of(words, slot_word::is_page_ref)) { + // Preserve the old anchor's extent, but do not reuse its legacy slot + // encoding as an ordinary byte-location segment. + image->metadata->extents.push_back(std::move(image->metadata->segments.back().second)); + image->metadata->segments.pop_back(); + } word_index = 0; stage = Stage::PageExtent; wake(); diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp index 8cfe8f685..60a548b46 100644 --- a/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp +++ b/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp @@ -101,7 +101,7 @@ void LocalRecoveryOperation::State::pump() segment(); break; case Stage::SegmentProtected: - image->metadata->extents.push_back(recovered); + image->metadata->segments.emplace_back(directory[segment_index], recovered); read(recovered.address, recovered.length, Stage::SegmentRead); break; case Stage::SegmentRead: diff --git a/lib/crowdb-tree/tests/integration/captured_frames_test.cpp b/lib/crowdb-tree/tests/integration/captured_frames_test.cpp index f3c5c4bdc..42b944d52 100644 --- a/lib/crowdb-tree/tests/integration/captured_frames_test.cpp +++ b/lib/crowdb-tree/tests/integration/captured_frames_test.cpp @@ -87,6 +87,73 @@ std::shared_ptr<PublishedTreeState> snapshot_tree(size_t leaves, size_t value_by return result; } +TEST(CapturedFrames, FinalLeafOutputEncodesWithoutBuildingAnotherMapping) +{ + auto output = published(page( + { + {.key = "a", .value = "final"} + }, + {.upper = "m"})); + TestStream test; + ASSERT_TRUE(CapturedFrameStream::create({.page = output, + .page_id = 17, + .coverage = 10, + .gc_floor = 0, + .next_page_id = 24, + .context = {.range = {.upper = "m"}}}, + {}, test.runtime, test.lane, {.encode = {.frame_bytes = 512}}, {}, + &test.stream) + .ok()); + output.reset(); + Completion done; + test.next(&done); + ASSERT_EQ(done.values.load(), 1U); + ASSERT_TRUE(test.stream->result().complete); + EXPECT_EQ(test.stream->stats().pages, 1U); + EXPECT_EQ(test.stream->stats().loads, 0U); + EXPECT_EQ(test.frame().info.page_id, 17U); + const auto &buffer = test.frame().buffer->bytes; + ASSERT_TRUE(frame_validate(buffer.data(), buffer.size())); + LeafFrameView leaf(buffer.data(), buffer.size()); + ASSERT_EQ(leaf.count(), 1U); + EXPECT_EQ(leaf.key(0).to_string(), "a"); + EXPECT_EQ(CellView(leaf.cell(0)).value().to_string(), "final"); +} + +TEST(CapturedFrames, FinalIndexOutputUsesJoinedFencesWithoutTraversingChildren) +{ + auto inner = std::make_shared<InnerPageState>(); + inner->children = {7, 12}; + inner->separators = {"m"}; + auto output = std::make_shared<PublishedPage>(); + output->level = 2; + output->inner = inner; + TestStream test; + ASSERT_TRUE( + CapturedFrameStream::create({.page = output, + .page_id = 20, + .coverage = 10, + .gc_floor = 0, + .next_page_id = 24, + .context = {.inner_fences = InnerEncodeFences{.lower_leaf = 3, .upper_leaf = 9}}}, + {}, test.runtime, test.lane, {.encode = {.frame_bytes = 512}}, {}, &test.stream) + .ok()); + Completion done; + test.next(&done); + ASSERT_EQ(done.values.load(), 1U); + ASSERT_TRUE(test.stream->result().complete); + EXPECT_EQ(test.stream->stats().pages, 1U); + EXPECT_EQ(test.stream->stats().loads, 0U); + EXPECT_EQ(test.frame().info.page_id, 20U); + const auto &buffer = test.frame().buffer->bytes; + ASSERT_TRUE(frame_validate(buffer.data(), buffer.size())); + InnerFrameView index(buffer.data(), buffer.size()); + EXPECT_EQ(index.child_at(0), 7U); + EXPECT_EQ(index.child_at(1), 12U); + EXPECT_EQ(frame_lower_fence_page_id(buffer.data()), 3U); + EXPECT_EQ(frame_upper_fence_page_id(buffer.data()), 9U); +} + TEST(CapturedFrames, MaterializationBytesFollowTheLastOutputOwnerAfterStreamDestruction) { LoadCredits credits; diff --git a/lib/crowdb-tree/tests/integration/captured_write_test.cpp b/lib/crowdb-tree/tests/integration/captured_write_test.cpp index 514bfbab2..78f053fd7 100644 --- a/lib/crowdb-tree/tests/integration/captured_write_test.cpp +++ b/lib/crowdb-tree/tests/integration/captured_write_test.cpp @@ -7,6 +7,46 @@ namespace crowdb::tree { using namespace test_write; +TEST(CapturedWrite, FinalPageStreamWritesAndReadsWithoutASnapshotMapping) +{ + TestCapturedWrite test; + auto output = published(page({ + {.key = "a", .value = "final"} + })); + CapturedWriteLimits limits{ + .frames = {.encode = {.frame_bytes = 512, .inline_value_bytes = 16, .metadata = test.credits.credits.bytes}}, + .write = {.max_frame_bytes = 512}}; + std::shared_ptr<CapturedFrameStream> stream; + ASSERT_TRUE(CapturedFrameStream::create( + {.page = output, .page_id = 17, .coverage = 10, .gc_floor = 0, .next_page_id = 24, .context = {}}, + {}, test.runtime, test.lane, limits.frames, {}, &stream) + .ok()); + auto pages = std::make_shared<CapturedDurablePages>(test.credits.credits.bytes); + pages->root_page_id = 17; + pages->next_page_id = 24; + pages->coverage = 10; + pages->generation = 1; + ASSERT_TRUE(CapturedWriteOperation::create(std::move(stream), std::move(pages), test.store, test.runtime, test.lane, + limits, test.credits.credits, + extent_write_reservation(test.ledger, test.credits.credits.bytes), {}, + &test.operation) + .ok()); + output.reset(); + Completion done; + test.run(&done); + ASSERT_EQ(done.values.load(), 1U) << done.status.to_string(); + const auto &result = test.operation->result(); + ASSERT_TRUE(result.pages); + ASSERT_EQ(result.pages->frames.size(), 1U); + EXPECT_EQ(result.written_frames, 1U); + EXPECT_EQ(result.pages->root_page_id, 17U); + EXPECT_EQ(result.pages->next_page_id, 24U); + const auto raw = test.read(result.pages->frames.at(17)); + LeafFrameView leaf(raw.data(), raw.size()); + ASSERT_EQ(leaf.count(), 1U); + EXPECT_EQ(CellView(leaf.cell(0)).value().to_string(), "final"); +} + TEST(CapturedWrite, OverflowAndPrimaryUseOneWindowAndKeepProtectedReferencesAfterInputsEnd) { TestCapturedWrite test; diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp index 7260c0a70..22d8085c3 100644 --- a/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp @@ -10,10 +10,13 @@ using namespace test_write; namespace { -std::shared_ptr<PublishedTreeState> sparse_captured_tree(size_t leaves) +std::shared_ptr<PublishedTreeState> sparse_captured_tree(size_t leaves, bool first_pair_shared = false) { - auto captured = std::make_shared<PublishedTreeState>(); - auto index = std::make_shared<InnerPageState>(); + auto captured = std::make_shared<PublishedTreeState>(); + auto index = std::make_shared<InnerPageState>(); + const auto page_id = [first_pair_shared](size_t i) { + return first_pair_shared && i == 1 ? 1 : i * kMappingSegmentSize; + }; std::vector<PageReplacement> replacements; for (size_t i = 0; i < leaves; ++i) { auto key = std::to_string(1000 + i); @@ -25,14 +28,14 @@ std::shared_ptr<PublishedTreeState> sparse_captured_tree(size_t leaves) if (i + 1 < leaves) { range.upper = std::to_string(1001 + i); } - const auto pid = i * kMappingSegmentSize; + const auto pid = page_id(i); index->children.push_back(pid); auto leaf = std::make_shared<PageState>(*page( { {.key = key, .value = "v"} }, range)); - leaf->right_sibling = i + 1 == leaves ? kInvalidPageId : (i + 1) * kMappingSegmentSize; + leaf->right_sibling = i + 1 == leaves ? kInvalidPageId : page_id(i + 1); replacements.push_back({.page_id = pid, .expected = {}, .replacement = published(std::move(leaf))}); } auto root = std::make_shared<PublishedPage>(); @@ -47,6 +50,45 @@ std::shared_ptr<PublishedTreeState> sparse_captured_tree(size_t leaves) captured->mapping = captured->mapping.replace(replacements, copied); return captured; } + +DurableCapturedFrame rewrite_leaf(TestCheckpoint &test, const PublishedTreeState &tree, uint64_t pid, std::string key, + std::string value) +{ + const auto old = tree.mapping.get(pid)->leaf; + auto leaf = std::make_shared<PageState>(*page( + { + {.key = std::move(key), .value = std::move(value)} + }, + old->range)); + leaf->right_sibling = old->right_sibling; + CapturedWriteLimits limits{ + .frames = {.encode = {.frame_bytes = 4096, .metadata = test.source.credits.credits.bytes}}, + .write = {.max_frame_bytes = 4096}}; + std::shared_ptr<CapturedFrameStream> stream; + EXPECT_TRUE(CapturedFrameStream::create({.page = published(std::move(leaf)), + .page_id = pid, + .coverage = 11, + .gc_floor = 0, + .next_page_id = tree.next_page_id, + .context = {.range = old->range}}, + {}, test.source.runtime, test.source.lane, limits.frames, {}, &stream) + .ok()); + auto pages = std::make_shared<CapturedDurablePages>(test.source.credits.credits.bytes); + pages->root_page_id = pid; + pages->next_page_id = tree.next_page_id; + pages->coverage = 11; + pages->generation = tree.generation; + EXPECT_TRUE( + CapturedWriteOperation::create(std::move(stream), std::move(pages), test.source.store, test.source.runtime, + test.source.lane, limits, test.source.credits.credits, + extent_write_reservation(test.source.ledger, test.source.credits.credits.bytes), + {}, &test.source.operation) + .ok()); + Completion done; + test.source.run(&done); + EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); + return test.source.operation->result().pages->frames.at(pid); +} } // namespace TEST(LocalCheckpoint, SparseMappingStreamsDirectoryAcrossWindowsAndEnforcesSegmentCap) @@ -97,4 +139,69 @@ TEST(LocalCheckpoint, SparseMappingStreamsDirectoryAcrossWindowsAndEnforcesSegme EXPECT_FALSE(test.registry->needs_recovery()); EXPECT_EQ(test.source.ledger->stats().protected_ranges, protected_before); } + +TEST(LocalCheckpoint, MultiplePageUpdatesWriteOneSegmentAndRetainUnchangedImages) +{ + TestCheckpoint test; + auto tree = sparse_captured_tree(3, true); + ASSERT_TRUE( + test.source.prepare(tree, {}, {.frames = {.encode = {.frame_bytes = 4096}}, .write = {.max_frame_bytes = 4096}}) + .ok()); + Completion materialized; + test.source.run(&materialized); + ASSERT_EQ(materialized.values.load(), 1U) << materialized.status.to_string(); + auto original = test.source.operation->result().pages; + ASSERT_TRUE(test.prepare(original).ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U) << first.status.to_string(); + auto reader = test.operation->result().checkpoint; + ASSERT_EQ(reader->metadata->segments.size(), 3U); + std::weak_ptr<const void> changed_owner = reader->metadata->segments[0].second.owner; + auto updated = std::make_shared<CapturedDurablePages>(*original); + updated->frames.at(0) = rewrite_leaf(test, *tree, 0, "1000", "left"); + updated->frames.at(1) = rewrite_leaf(test, *tree, 1, "1001", "right"); + updated->coverage = 11; + const auto before = test.store->events.size(); + ASSERT_TRUE(test.prepare(updated).ok()); + Completion second; + test.run(&second); + ASSERT_EQ(second.values.load(), 1U) << second.status.to_string(); + const auto current = test.operation->result().checkpoint; + EXPECT_EQ(test.store->events.size() - before, 5U); + ASSERT_EQ(current->metadata->segments.size(), 3U); + EXPECT_NE(current->metadata->segments[0].first.image_addr, reader->metadata->segments[0].first.image_addr); + for (size_t i = 1; i < current->metadata->segments.size(); ++i) { + const auto &old = reader->metadata->segments[i]; + const auto &now = current->metadata->segments[i]; + EXPECT_EQ(now.first.image_addr, old.first.image_addr); + EXPECT_EQ(now.first.image_crc, old.first.image_crc); + EXPECT_EQ(now.first.generation, old.first.generation); + EXPECT_EQ(now.second.owner, old.second.owner); + } + auto advanced = std::make_shared<CapturedDurablePages>(*updated); + advanced->coverage = 12; + ASSERT_TRUE(test.prepare(advanced).ok()); + Completion third; + test.run(&third); + ASSERT_EQ(third.values.load(), 1U) << third.status.to_string(); + EXPECT_FALSE(changed_owner.expired()); + reader.reset(); + EXPECT_TRUE(changed_owner.expired()); + Config config; + config.page_store = test.store->device.get(); + config.frame_bytes = 4096; + std::unique_ptr<Crowdbtree> reopened; + const auto opened = Crowdbtree::open(config, &reopened); + ASSERT_TRUE(opened.ok()) << opened.to_string(); + for (const auto &[key, expected] : std::array<std::pair<std::string, std::string>, 3>{ + {{"1000", "left"}, {"1001", "right"}, {"1002", "v"}} + }) { + GetView value; + ASSERT_TRUE(RuntimeTreeAccess_for_tests::core(*reopened).get(key, &value).ok()); + ASSERT_TRUE(value.found()); + EXPECT_EQ(value.value().to_string(), expected); + } +} + } // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp index 5c81871dd..3decb7991 100644 --- a/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp @@ -28,6 +28,105 @@ TEST(LocalCheckpoint, PersistsExistingFormatAndAdvancesOnlyAfterSecondBarrier) test.expect_read(std::string(3000, 'v')); } +TEST(LocalCheckpoint, UnchangedCheckpointDoesNotWriteOrAdvanceSequence) +{ + TestCheckpoint test; + auto pages = test.materialize(); + ASSERT_TRUE(test.prepare(pages).ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U) << first.status.to_string(); + auto original = test.operation->result().checkpoint; + const auto events = test.store->events.size(); + const auto high_water = test.source.ledger->stats().address_high_water; + ASSERT_TRUE(test.prepare(std::make_shared<CapturedDurablePages>(*pages)).ok()); + Completion unchanged; + test.run(&unchanged); + ASSERT_EQ(unchanged.values.load(), 1U) << unchanged.status.to_string(); + EXPECT_EQ(test.operation->result().checkpoint, original); + EXPECT_EQ(test.store->events.size(), events); + EXPECT_EQ(test.registry->snapshot_sequence(), 1U); + EXPECT_EQ(test.source.ledger->stats().address_high_water, high_water); + auto advanced = std::make_shared<CapturedDurablePages>(*pages); + ++advanced->coverage; + ASSERT_TRUE(test.prepare(advanced).ok()); + Completion coverage; + test.run(&coverage); + ASSERT_EQ(coverage.values.load(), 1U) << coverage.status.to_string(); + EXPECT_EQ(test.registry->snapshot_sequence(), 2U); + EXPECT_EQ(test.registry->durable_coverage(), advanced->coverage); + EXPECT_EQ(test.store->events.size(), events + 3); + EXPECT_EQ(test.operation->result().checkpoint->metadata, original->metadata); + EXPECT_EQ(test.source.ledger->stats().address_high_water, high_water); +} + +TEST(LocalCheckpoint, GcRetiresTheObsoleteAnchorWithoutRewritingUnchangedSegments) +{ + TestCheckpoint test; + ASSERT_TRUE(test.prepare().ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U); + auto reader = test.operation->result().checkpoint; + std::weak_ptr<const void> old_segment = reader->metadata->segments.front().second.owner; + auto next = test.materialize("next"); + ASSERT_TRUE(test.prepare(next).ok()); + Completion second; + test.run(&second); + ASSERT_EQ(second.values.load(), 1U); + ASSERT_TRUE(test.registry->needs_anchor_retirement()); + auto latest = test.operation->result().checkpoint; + const auto before = test.store->events.size(); + ASSERT_TRUE(test.prepare(next).ok()); + Completion unchanged; + test.run(&unchanged); + ASSERT_EQ(unchanged.values.load(), 1U); + EXPECT_EQ(test.store->events.size(), before); + EXPECT_TRUE(test.registry->needs_anchor_retirement()); + ASSERT_TRUE(test.prepare(next, {.retire_previous_anchor = true}).ok()); + Completion retired; + test.run(&retired); + ASSERT_EQ(retired.values.load(), 1U) << retired.status.to_string(); + EXPECT_EQ(test.store->events.size(), before + 3); + EXPECT_EQ(test.operation->result().checkpoint->metadata, latest->metadata); + EXPECT_EQ(test.registry->snapshot_sequence(), 3U); + EXPECT_FALSE(test.registry->needs_anchor_retirement()); + EXPECT_FALSE(old_segment.expired()); + reader.reset(); + EXPECT_TRUE(old_segment.expired()); + test.expect_read("next"); +} + +TEST(LocalCheckpoint, ExcludedMetadataSegmentIsRelocatedDespiteUnchangedPageSlots) +{ + TestCheckpoint test; + auto pages = test.materialize(); + ASSERT_TRUE(test.prepare(pages).ok()); + Completion first; + test.run(&first); + ASSERT_EQ(first.values.load(), 1U); + auto original = test.operation->result().checkpoint; + const auto &segment = original->metadata->segments.front().second; + auto fence = await_ledger([&](auto done) { + return test.source.ledger->exclude_allocations(segment.address, segment.length, + test.source.credits.credits.bytes, std::move(done)); + }); + ASSERT_TRUE(fence.status.ok()); + const BackingExclusion exclusion{.address = segment.address, .length = segment.length}; + const auto before = test.store->events.size(); + ASSERT_TRUE(test.prepare(pages, {.exclusion = exclusion}).ok()); + Completion relocated; + test.run(&relocated); + ASSERT_EQ(relocated.values.load(), 1U) << relocated.status.to_string(); + EXPECT_EQ(test.store->events.size(), before + 5); + const auto ¤t = test.operation->result().checkpoint->metadata->segments.front(); + EXPECT_NE(current.first.image_addr, segment.address); + EXPECT_FALSE(exclusion.overlaps(current.second.address, current.second.length)); + EXPECT_EQ(test.operation->result().checkpoint->pages, pages); + EXPECT_EQ(test.registry->snapshot_sequence(), 2U); + test.expect_read("v"); +} + TEST(LocalCheckpoint, RotationRetainsBothAnchorsAndReaderOwners) { TestCheckpoint test; diff --git a/lib/crowdb-tree/tests/integration/persist_test.cpp b/lib/crowdb-tree/tests/integration/persist_test.cpp index 4853d8ab6..0b84fa8a5 100644 --- a/lib/crowdb-tree/tests/integration/persist_test.cpp +++ b/lib/crowdb-tree/tests/integration/persist_test.cpp @@ -1083,6 +1083,7 @@ void prepare_sparse_compaction(const std::string &path, const std::set<std::stri // Keep clean old leaves in their original blocks and rotate both // anchors; automatic GC must not consume the intended candidate. ASSERT_TRUE(t.snapshot().ok()); + ASSERT_TRUE(t.apply(++slot, Batch{}).ok()); ASSERT_TRUE(t.snapshot().ok()); } diff --git a/lib/crowdb-tree/tests/integration/tree_core_primary_reuse_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_primary_reuse_test.cpp index 54764b6f8..1fd9b966c 100644 --- a/lib/crowdb-tree/tests/integration/tree_core_primary_reuse_test.cpp +++ b/lib/crowdb-tree/tests/integration/tree_core_primary_reuse_test.cpp @@ -21,6 +21,20 @@ struct TestCountedStore : MemPageStore mutable std::atomic<size_t> reads{0}; std::atomic<size_t> primary_writes{0}; + std::atomic<size_t> total_writes{0}; + std::atomic<size_t> barriers{0}; + + Status write_at(uint64_t address, const uint8_t *data, size_t bytes) override + { + total_writes.fetch_add(1); + return MemPageStore::write_at(address, data, bytes); + } + + Status sync() override + { + barriers.fetch_add(1); + return MemPageStore::sync(); + } Status read_at(uint64_t address, uint8_t *data, size_t bytes) const override { @@ -33,7 +47,7 @@ struct TestCountedStore : MemPageStore if (purpose == PagePurpose::kBtreePage) { primary_writes.fetch_add(1); } - return MemPageStore::write_at(address, data, bytes); + return write_at(address, data, bytes); } }; @@ -97,6 +111,29 @@ TEST(TreeCorePrimaryReuse, ColdInlineAndOverflowCheckpointReusesWithoutPrimaryIo EXPECT_EQ(test.store->primary_writes.load(), writes); } +TEST(TreeCorePrimaryReuse, UnchangedCheckpointIsZeroIoAndKeepsSequenceAfterReopen) +{ + TestPrimaryReuse test; + ASSERT_TRUE( + test.core->apply(1, Batch{.ops = {{.key = "a", .kind = OpKind::kPut, .value = std::string(12000, 'v')}}}).ok()); + uint64_t sequence = 0; + ASSERT_TRUE(test.core->checkpoint(nullptr, &sequence).ok()); + ASSERT_GT(sequence, 0U); + const auto writes = test.store->total_writes.load(); + const auto barriers = test.store->barriers.load(); + uint64_t unchanged = 0; + ASSERT_TRUE(test.core->checkpoint(nullptr, &unchanged).ok()); + EXPECT_EQ(unchanged, sequence); + EXPECT_EQ(test.store->total_writes.load(), writes); + EXPECT_EQ(test.store->barriers.load(), barriers); + test.core.reset(); + ASSERT_TRUE(TreeCore::open(test.config, &test.core).ok()); + ASSERT_TRUE(test.core->checkpoint(nullptr, &unchanged).ok()); + EXPECT_EQ(unchanged, sequence); + EXPECT_EQ(test.store->total_writes.load(), writes); + EXPECT_EQ(test.store->barriers.load(), barriers); +} + TEST(TreeCorePrimaryReuse, ExactReceiptReuseIsNotReportedAsDeferredRepresentationWork) { TestPrimaryReuse test; diff --git a/lib/crowdb-tree/tests/unit/snapshot_publication_test.cpp b/lib/crowdb-tree/tests/unit/snapshot_publication_test.cpp index 86deb7633..00fc428f1 100644 --- a/lib/crowdb-tree/tests/unit/snapshot_publication_test.cpp +++ b/lib/crowdb-tree/tests/unit/snapshot_publication_test.cpp @@ -118,6 +118,60 @@ TEST(SnapshotPublication, BaselineMustBelongToTheExactCapturedFrontierAndToken) EXPECT_EQ(publisher.capture()->snapshot_baseline, baseline); } +TEST(SnapshotPublication, UnchangedSequenceRequiresTheSameProtectedBaseline) +{ + PublicationCoordinator publisher; + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 1).status.ok()); + auto pages = std::make_shared<CapturedDurablePages>(nullptr); + pages->generation = 1; + auto baseline = std::make_shared<SnapshotBaseline>(); + baseline->generation = 1; + baseline->capture_token = 1; + baseline->snapshot_sequence = 1; + baseline->backend_id = 17; + baseline->pages = pages; + ASSERT_TRUE( + publish(publisher, + { + .checkpoint = + CheckpointPublication{.action = CheckpointAction::Commit, .token = 1, .baseline = baseline} + }) + .status.ok()); + ASSERT_TRUE(checkpoint(publisher, CheckpointAction::Capture, 2).status.ok()); + auto reused = std::make_shared<SnapshotBaseline>(*baseline); + reused->capture_token = 2; + for (const auto variant : {0, 1, 2}) { + auto wrong = std::make_shared<SnapshotBaseline>(*reused); + if (variant == 0) { + wrong->pages = std::make_shared<CapturedDurablePages>(*pages); + } + else if (variant == 1) { + ++wrong->wal_offset; + } + else { + ++wrong->backend_id; + } + EXPECT_EQ( + publish(publisher, + { + .checkpoint = + CheckpointPublication{.action = CheckpointAction::Commit, .token = 2, .baseline = wrong} + }) + .status.code(), + Code::kInvalidArgument); + EXPECT_EQ(publisher.capture()->snapshot_baseline, baseline); + } + ASSERT_TRUE( + publish(publisher, + { + .checkpoint = + CheckpointPublication{.action = CheckpointAction::Commit, .token = 2, .baseline = reused} + }) + .status.ok()); + EXPECT_EQ(publisher.capture()->snapshot_baseline, reused); + EXPECT_EQ(reused->snapshot_sequence, 1U); +} + #ifdef CROWDB_TREE_TEST_UTIL TEST(SnapshotPublication, FailureAfterPreparingChangesPublishesNeitherSelectionNorPendingState) { From a09ca2a190b91788e6f46efa0681579732e7fc67 Mon Sep 17 00:00:00 2001 From: Gian <crow.db@outlook.com> Date: Thu, 5 Nov 2026 19:29:10 +0800 Subject: [PATCH 04/17] Refresh public architecture documentation and documentation structure --- AGENTS.md | 63 +- CONTRIBUTING.md | 13 +- README.md | 82 +- app/crowdb-access-server/tests/README.md | 4 +- doc/assets/chunk-io.svg | 156 + doc/assets/chunk-kv.svg | 115 + doc/assets/chunk-placement.svg | 103 + doc/assets/chunk-read.svg | 64 + doc/assets/chunk-stream.svg | 48 + doc/assets/crowdb-architecture.svg | 97 + doc/assets/crowdb-chunk.png | Bin 0 -> 240517 bytes doc/assets/crowdb-chunkkv.png | Bin 0 -> 377100 bytes doc/assets/crowdb-cluster.png | Bin 0 -> 228747 bytes doc/assets/crowdb-dataset.png | Bin 0 -> 279626 bytes doc/assets/crowdb-iceberg.png | Bin 0 -> 293183 bytes doc/assets/crowdb-overview.svg | 54 + doc/assets/crowdb-paxoskv.png | Bin 0 -> 369391 bytes doc/assets/crowdb-s3.png | Bin 0 -> 148149 bytes doc/assets/dataset-read.svg | 78 + doc/assets/disk-allocation.svg | 109 + doc/assets/disk-io.svg | 74 + doc/assets/iceberg-commit.svg | 54 + doc/assets/iceberg-read.svg | 55 + doc/assets/paxos-kv.svg | 136 + doc/assets/s3-publication.svg | 62 + doc/assets/tree-flush.svg | 83 + doc/assets/tree.svg | 93 + doc/design/README.md | 59 + .../dataset/design-crowdb-access-dataset.md | 589 -- .../dataset/design-dataset-control.md | 438 -- .../dataset/openapi-crowdb-dataset-v1.yaml | 2535 -------- .../dataset/openapi-crowdb-dataset-v2.yaml | 2002 ------- .../dataset/spec-crowdb-dataset.md | 537 -- .../design-crowdb-access-server.md | 320 - .../access-server/design-crowdb-native-sdk.md | 185 - .../design-crowdb-iceberg-upload-flow.md | 395 -- .../iceberge/design-crowdb-iceberg.md | 1291 ---- .../iceberg-rest-catalog-open-api-1.11.0.yaml | 5285 ----------------- .../iceberge/iceberg-table-spec-1.11.0.md | 1858 ------ .../s3/design-crowdb-access-s3-rdma.md | 67 - .../s3/design-crowdb-access-s3.md | 345 -- .../chunkdb/chunkdb-allocate-flow-analysis.md | 189 - .../design-crowdb-chunkdb-mirror-to-ec.md | 278 - .../design-crowdb-chunkdb-range-binding.md | 318 - .../chunkdb/design-crowdb-chunkdb-rpc.md | 111 - doc/design/chunkdb/design-crowdb-chunkdb.md | 1294 ---- .../chunkds/design-crowdb-chunk-kv-client.md | 178 - .../chunkds/design-crowdb-chunk-kv-server.md | 804 --- doc/design/chunkds/design-crowdb-chunk-kv.md | 376 -- .../chunkds/design-crowdb-chunk-stream.md | 627 -- .../chunkio/chunkio-small-io-flow-analysis.md | 372 -- .../chunkio/chunkio-write-flow-analysis.md | 204 - .../chunkio/design-crowdb-chunkio-reader.md | 164 - ...sign-crowdb-chunkio-small-object-writer.md | 343 -- doc/design/chunkio/design-crowdb-chunkio.md | 625 -- doc/design/config/design-crowdb-config.md | 195 - .../console/design-crowdb-console-ui.md | 1865 ------ doc/design/console/design-crowdb-console.md | 542 -- doc/design/deploy/design-crowdb-deploy.md | 850 --- doc/design/deploy/design-crowdb-oci-image.md | 374 -- doc/design/design-crowdb-access-dataset.md | 64 + doc/design/design-crowdb-access-iceberg.md | 58 + doc/design/design-crowdb-access-server.md | 46 + doc/design/design-crowdb-architecture.md | 91 + doc/design/design-crowdb-chunk-kv.md | 72 + doc/design/design-crowdb-chunk-stream.md | 44 + doc/design/design-crowdb-chunkdb.md | 79 + doc/design/design-crowdb-deployment.md | 44 + doc/design/design-crowdb-diskdb.md | 36 + doc/design/design-crowdb-diskio.md | 36 + doc/design/design-crowdb-monitor.md | 38 + doc/design/design-crowdb-paxos-kv.md | 44 + doc/design/design-crowdb-rpc.md | 52 + doc/design/design-crowdb-sdk-native.md | 64 + doc/design/design-crowdb-tree.md | 68 + doc/design/design-crowdb.md | 58 + .../design-crowdb-diskdb-space-metrics.md | 818 --- .../design-crowdb-diskdb-zone-management.md | 930 --- doc/design/diskdb/design-crowdb-diskdb.md | 1066 ---- .../diskdb/diskdb-allocate-flow-analysis.md | 203 - doc/design/diskio/design-crowdb-diskio.md | 885 --- .../kv/design-crowdb-paxos-kv-group0.md | 551 -- .../design-crowdb-paxos-kv-leader-election.md | 373 -- .../design-crowdb-paxos-kv-observability.md | 505 -- .../design-crowdb-paxos-kv-reconfiguration.md | 260 - .../kv/design-crowdb-paxos-kv-rpc-client.md | 294 - doc/design/kv/design-crowdb-paxos-kv-rpc.md | 550 -- .../kv/design-crowdb-paxos-kv-server.md | 234 - doc/design/kv/design-crowdb-paxos-kv-slot.md | 671 --- .../design-crowdb-paxos-kv-state-machine.md | 455 -- doc/design/kv/design-crowdb-paxos-kv-test.md | 272 - doc/design/kv/design-crowdb-paxos-kv-wal.md | 626 -- .../kv/design-crowdb-paxos-kv-watch-notify.md | 894 --- doc/design/kv/design-crowdb-paxos-kv.md | 523 -- doc/design/kv/kv-read-flow-analysis.md | 204 - doc/design/kv/kv-scan-flow-analysis.md | 281 - doc/design/kv/kv-write-flow-analysis.md | 303 - .../protocol/design-crowdb-protocol-key.md | 488 -- .../protocol/design-crowdb-protocol-types.md | 201 - doc/design/protocol/design-crowdb-protocol.md | 91 - .../rpc/design-crowdb-rpc-diskdb-migration.md | 169 - doc/design/rpc/design-crowdb-rpc-rdma.md | 652 -- doc/design/rpc/design-crowdb-rpc-tcp.md | 201 - doc/design/rpc/design-crowdb-rpc.md | 457 -- doc/design/rpc/rpc-flow-analysis.md | 766 --- .../tree/design-crowdb-tree-chunk-storage.md | 283 - .../design-crowdb-tree-engine-flush-flow.md | 163 - ...design-crowdb-tree-engine-snapshot-flow.md | 131 - doc/design/tree/design-crowdb-tree-engine.md | 365 -- doc/design/tree/design-crowdb-tree-storage.md | 331 -- doc/design/tree/design-crowdb-tree.md | 161 - doc/dev/crash_debugging.md | 141 - doc/dev/env_setup.md | 186 - doc/dev/hyper_fork.md | 424 -- doc/dev/physical-node-test-plan.md | 720 --- doc/doc_index.md | 127 - doc/working/plan-ci-stability-review.md | 42 - e2e-test/README.md | 3 +- .../src/hardware/hierarchy.rs | 2 - .../src/hardware/sysmd.rs | 5 +- .../src/service/discovery.rs | 2 - .../src/service/registry.rs | 2 - .../src/transport/cluster.rs | 2 - lib/crowdb-protocol/src/key.rs | 2 - lib/crowdb-protocol/src/key/chunkdb.rs | 3 - lib/crowdb-protocol/src/key/common.rs | 2 +- lib/crowdb-protocol/src/key/diskdb.rs | 4 - lib/crowdb-protocol/src/key/kv_cluster.rs | 3 - lib/crowdb-protocol/src/mgmt.rs | 4 - lib/crowdb-protocol/src/sysdata.rs | 2 - lib/crowdb-rpc/ffi/src/client.rs | 1 - lib/crowdb-rpc/include/crowdb-rpc/c_api.h | 1 - lib/crowdb-rpc/src/client/client.cpp | 1 - lib/crowdb-sdk-native/README.md | 8 +- lib/crowdb-tree/bench/README.md | 5 +- sdk/README.md | 9 +- sdk/contract-fixtures/dataset-v1/README.md | 5 +- tools/benchmark/bench-kv-read-regression.sh | 2 - tools/benchmark/bench-kv-scan-regression.sh | 2 - tools/benchmark/bench-kv-write-regression.sh | 2 - tools/profiling/setup-perf.sh | 3 +- 141 files changed, 2416 insertions(+), 42708 deletions(-) create mode 100644 doc/assets/chunk-io.svg create mode 100644 doc/assets/chunk-kv.svg create mode 100644 doc/assets/chunk-placement.svg create mode 100644 doc/assets/chunk-read.svg create mode 100644 doc/assets/chunk-stream.svg create mode 100644 doc/assets/crowdb-architecture.svg create mode 100644 doc/assets/crowdb-chunk.png create mode 100644 doc/assets/crowdb-chunkkv.png create mode 100644 doc/assets/crowdb-cluster.png create mode 100644 doc/assets/crowdb-dataset.png create mode 100644 doc/assets/crowdb-iceberg.png create mode 100644 doc/assets/crowdb-overview.svg create mode 100644 doc/assets/crowdb-paxoskv.png create mode 100644 doc/assets/crowdb-s3.png create mode 100644 doc/assets/dataset-read.svg create mode 100644 doc/assets/disk-allocation.svg create mode 100644 doc/assets/disk-io.svg create mode 100644 doc/assets/iceberg-commit.svg create mode 100644 doc/assets/iceberg-read.svg create mode 100644 doc/assets/paxos-kv.svg create mode 100644 doc/assets/s3-publication.svg create mode 100644 doc/assets/tree-flush.svg create mode 100644 doc/assets/tree.svg create mode 100644 doc/design/README.md delete mode 100644 doc/design/access-server/dataset/design-crowdb-access-dataset.md delete mode 100644 doc/design/access-server/dataset/design-dataset-control.md delete mode 100644 doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml delete mode 100644 doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml delete mode 100644 doc/design/access-server/dataset/spec-crowdb-dataset.md delete mode 100644 doc/design/access-server/design-crowdb-access-server.md delete mode 100644 doc/design/access-server/design-crowdb-native-sdk.md delete mode 100644 doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md delete mode 100644 doc/design/access-server/iceberge/design-crowdb-iceberg.md delete mode 100644 doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml delete mode 100644 doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md delete mode 100644 doc/design/access-server/s3/design-crowdb-access-s3-rdma.md delete mode 100644 doc/design/access-server/s3/design-crowdb-access-s3.md delete mode 100644 doc/design/chunkdb/chunkdb-allocate-flow-analysis.md delete mode 100644 doc/design/chunkdb/design-crowdb-chunkdb-mirror-to-ec.md delete mode 100644 doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md delete mode 100644 doc/design/chunkdb/design-crowdb-chunkdb-rpc.md delete mode 100644 doc/design/chunkdb/design-crowdb-chunkdb.md delete mode 100644 doc/design/chunkds/design-crowdb-chunk-kv-client.md delete mode 100644 doc/design/chunkds/design-crowdb-chunk-kv-server.md delete mode 100644 doc/design/chunkds/design-crowdb-chunk-kv.md delete mode 100644 doc/design/chunkds/design-crowdb-chunk-stream.md delete mode 100644 doc/design/chunkio/chunkio-small-io-flow-analysis.md delete mode 100644 doc/design/chunkio/chunkio-write-flow-analysis.md delete mode 100644 doc/design/chunkio/design-crowdb-chunkio-reader.md delete mode 100644 doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md delete mode 100644 doc/design/chunkio/design-crowdb-chunkio.md delete mode 100644 doc/design/config/design-crowdb-config.md delete mode 100644 doc/design/console/design-crowdb-console-ui.md delete mode 100644 doc/design/console/design-crowdb-console.md delete mode 100644 doc/design/deploy/design-crowdb-deploy.md delete mode 100644 doc/design/deploy/design-crowdb-oci-image.md create mode 100644 doc/design/design-crowdb-access-dataset.md create mode 100644 doc/design/design-crowdb-access-iceberg.md create mode 100644 doc/design/design-crowdb-access-server.md create mode 100644 doc/design/design-crowdb-architecture.md create mode 100644 doc/design/design-crowdb-chunk-kv.md create mode 100644 doc/design/design-crowdb-chunk-stream.md create mode 100644 doc/design/design-crowdb-chunkdb.md create mode 100644 doc/design/design-crowdb-deployment.md create mode 100644 doc/design/design-crowdb-diskdb.md create mode 100644 doc/design/design-crowdb-diskio.md create mode 100644 doc/design/design-crowdb-monitor.md create mode 100644 doc/design/design-crowdb-paxos-kv.md create mode 100644 doc/design/design-crowdb-rpc.md create mode 100644 doc/design/design-crowdb-sdk-native.md create mode 100644 doc/design/design-crowdb-tree.md create mode 100644 doc/design/design-crowdb.md delete mode 100644 doc/design/diskdb/design-crowdb-diskdb-space-metrics.md delete mode 100644 doc/design/diskdb/design-crowdb-diskdb-zone-management.md delete mode 100644 doc/design/diskdb/design-crowdb-diskdb.md delete mode 100644 doc/design/diskdb/diskdb-allocate-flow-analysis.md delete mode 100644 doc/design/diskio/design-crowdb-diskio.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-group0.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-leader-election.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-observability.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-reconfiguration.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-rpc-client.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-rpc.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-server.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-slot.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-state-machine.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-test.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-wal.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv-watch-notify.md delete mode 100644 doc/design/kv/design-crowdb-paxos-kv.md delete mode 100644 doc/design/kv/kv-read-flow-analysis.md delete mode 100644 doc/design/kv/kv-scan-flow-analysis.md delete mode 100644 doc/design/kv/kv-write-flow-analysis.md delete mode 100644 doc/design/protocol/design-crowdb-protocol-key.md delete mode 100644 doc/design/protocol/design-crowdb-protocol-types.md delete mode 100644 doc/design/protocol/design-crowdb-protocol.md delete mode 100644 doc/design/rpc/design-crowdb-rpc-diskdb-migration.md delete mode 100644 doc/design/rpc/design-crowdb-rpc-rdma.md delete mode 100644 doc/design/rpc/design-crowdb-rpc-tcp.md delete mode 100644 doc/design/rpc/design-crowdb-rpc.md delete mode 100644 doc/design/rpc/rpc-flow-analysis.md delete mode 100644 doc/design/tree/design-crowdb-tree-chunk-storage.md delete mode 100644 doc/design/tree/design-crowdb-tree-engine-flush-flow.md delete mode 100644 doc/design/tree/design-crowdb-tree-engine-snapshot-flow.md delete mode 100644 doc/design/tree/design-crowdb-tree-engine.md delete mode 100644 doc/design/tree/design-crowdb-tree-storage.md delete mode 100644 doc/design/tree/design-crowdb-tree.md delete mode 100644 doc/dev/crash_debugging.md delete mode 100644 doc/dev/env_setup.md delete mode 100644 doc/dev/hyper_fork.md delete mode 100644 doc/dev/physical-node-test-plan.md delete mode 100644 doc/doc_index.md delete mode 100644 doc/working/plan-ci-stability-review.md diff --git a/AGENTS.md b/AGENTS.md index 1bf9045e3..3c15b8991 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,55 +3,36 @@ # CROWDB -Distributed storage platform: Paxos consensus, per-key slots, WAL durability, -crowdb-tree engine, and a disk-block allocator. Rust workspace with C++ storage -and transport exposed through FFI. +Distributed storage platform: Paxos consensus, per-key slots, WAL durability, crowdb-tree engine, and a disk-block allocator. Rust workspace with C++ storage and transport exposed through FFI. ## Terminology -- **Repo chunk** is the discussion term for any user-data chunk, including - S3, IcebergTable, and future Dataset chunks. It excludes system chunks - such as WAL and tree storage chunks. -- Implementations distinguish these user-data uses with specific chunk types; - do not retain or introduce a generic `Repo` chunk type. The discussion term - does not imply a shared implementation type. +- **Repo chunk** is the discussion term for any user-data chunk, including IcebergTable, future Dataset, and S3 chunks. It excludes system chunks such as WAL and tree storage chunks. +- Implementations distinguish these user-data uses with specific chunk types; do not retain or introduce a generic `Repo` chunk type. The discussion term does not imply a shared implementation type. ## Project rules - Run every build, test, lint, and executable through `pixi run`. -- `unsafe_code = deny` by default. Keep unsafe confined to existing scoped FFI - or low-level modules; raise any new exception before adding it. -- Keep hot paths lock-free. Before adding a lock, stop and ask the user with - the contention, ordering, and complexity trade-off. -- Add Rust tests under each crate's `tests/`; crates with `test-util` enable it - for their own tests through self dev-dependencies. -- Preserve existing work. Never hard-reset, revert, or restore files from an - older commit without explicit approval. Use stash or a temporary branch. -- Ordinary interactive work is committed only when asked, as one coherent - commit. -- Commit messages are single-line subjects without bodies, trailers, doc - references, or requirement numbers. Code comments also omit those references. -- Before committing, run the relevant gate through pixi: - Rust fmt and clippy, changed C++ format/tree-lint, and affected tests. Fix - ordinary failures up to three times; report confirmed pre-existing failures. +- `unsafe_code = deny` by default. Keep unsafe confined to existing scoped FFI or low-level modules; raise any new exception before adding it. +- Keep hot paths lock-free. Before adding a lock, stop and ask the user with the contention, ordering, and complexity trade-off. +- Add Rust tests under each crate's `tests/`; crates with `test-util` enable it for their own tests through self dev-dependencies. +- Preserve existing work. Never hard-reset, revert, or restore files from an older commit without explicit approval. Use stash or a temporary branch. +- Ordinary interactive work is committed only when asked, as one coherent commit. +- Commit messages are single-line subjects without bodies, trailers, doc references, or requirement numbers. Code comments also omit those references. +- Before committing, run the relevant gate through pixi: Rust fmt and clippy, changed C++ format/tree-lint, and affected tests. Fix ordinary failures up to three times; report confirmed pre-existing failures. - Playwright uses an installed system browser. Do not install one locally. -- Use 60-second shell timeouts by default. Start hang-prone commands in the - background and poll. Show complete output; do not hide errors with filters. -- On test or CI failure, print the failing case and test error to the console. - Enable `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` in CI or when debugging to also print - container service file errors. This extra output defaults off locally; - artifacts retain complete logs in either mode. -- Markdown is primarily read raw. Prefer bullets; use tables for real - comparisons only. `doc/doc_index.md` always uses tables. When a table is - used, pad columns with spaces so the raw markdown aligns visually. -- Console UI changes follow `doc/design/console/design-crowdb-console-ui.md`. - Before changing visible behavior, update the UI spec and its ASCII layout - after agreement with the user; only then implement the UI change. Do not - unilaterally change the spec or introduce UI behavior that is absent from - the agreed spec. Each visible UI change needs focused test or E2E coverage. +- Use 60-second shell timeouts by default. Start hang-prone commands in the background and poll. Show complete output; do not hide errors with filters. +- On test or CI failure, print the failing case and test error to the console. Enable `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` in CI or when debugging to also print container service file errors. This extra output defaults off locally; artifacts retain complete logs in either mode. +- Markdown is primarily read raw. Prefer bullets; use tables for real comparisons only. `doc/design/README.md` always uses tables. When a table is used, pad columns with spaces so the raw markdown aligns visually. +- Console UI changes follow the agreed UI behavior spec. Before changing visible behavior, update the UI spec and its ASCII layout after agreement with the user; only then implement the UI change. Do not unilaterally change the spec or introduce UI behavior that is absent from the agreed spec. Each visible UI change needs focused test or E2E coverage. ## Documentation -- Use `doc/doc_index.md` to locate the relevant permanent design. -- Keep architecture and operational contracts under `doc/design/` or the - owning component README. Documentation must be usable from this repository. +- Present CROWDB access models in the order Iceberg, Dataset, S3 in prose, lists, navigation, and diagrams. When only a subset is relevant, preserve that relative order. +- Keep each prose paragraph and list item on one source line; do not hard-wrap at a fixed column width. Let the editor or renderer wrap to the available width. Preserve blank lines between paragraphs and structural line breaks in tables, code blocks, and nested lists. +- Use `doc/design/README.md` to locate public documentation. +- `doc/design/` explains overall architecture, capabilities, and design advantages for users and AI readers. Update it only when explicitly requested; routine development does not automatically refresh these overviews. +- Public overviews do not replace detailed behavior specs. Keep component usage and operational instructions in the owning README. Public documentation must be independently usable from this repository. +- Keep a single component article directly under `doc/design/`; create a subdirectory only when several related articles need it. +- Store shared diagrams, screenshots, and other documentation media in `doc/assets/`. README and design articles reference the same assets. Prefer editable SVG for technical diagrams; use real UI screenshots for product views. Public design diagrams do not need ASCII equivalents. +- Write public design articles in English. Explain concrete mechanisms and user benefits with established terminology; keep detailed implementation flows and development progress out of these overviews. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0eda9d6a4..2b12bb34e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,10 +76,12 @@ pixi run test-single-node-container - Add dependencies through the owning package manager and avoid newly published versions until they have had time for ecosystem review. -Start documentation work at `doc/doc_index.md`. Permanent architecture and -operational behavior belong under `doc/design/` or the relevant component -README. Keep documentation independently readable with links to files in this -repository or public sources. +Start public documentation work at `doc/design/README.md`. The English documents +under `doc/design/` introduce architecture, capabilities, and design advantages +to users and AI readers. Refresh them only as explicitly requested documentation +work; they do not replace detailed behavior specs. Keep component usage and +operational instructions in the relevant README. Public documentation must be +independently readable with links to repository files or public sources. ## Code and tests @@ -115,7 +117,8 @@ security boundaries to make a test pass. 1. Create a focused branch from `main`. 2. Keep the change aligned with one requirement or one coherent maintenance purpose. -3. Add or update tests and documentation with the implementation. +3. Add or update tests and affected behavior specs or usage instructions with the + implementation. Public architecture overviews are a separate documentation task. 4. Run the relevant Pixi gates; run the full suite for cross-component changes. 5. Review generated files and the complete diff for secrets and unrelated edits. 6. Open a pull request explaining the problem, design choice, verification, and diff --git a/README.md b/README.md index c1730bc13..1051fe2d7 100644 --- a/README.md +++ b/README.md @@ -6,77 +6,53 @@ [![CI](https://github.com/buzzcrow/crowdb/actions/workflows/ci.yml/badge.svg)](https://github.com/buzzcrow/crowdb/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) -**A distributed storage platform where ownership, data movement, and recovery -share one durable control model.** +**Distributed storage for Iceberg tables, AI datasets, and S3 objects.** -CROWDB provides a shared storage core for **Iceberg tables, S3 objects, and AI -datasets**. Each access model keeps its own semantics while sharing one storage -core for distributed state, placement, protection, streaming, and recovery. +CROWDB brings native table and dataset semantics to a shared storage core. It owns the path from metadata publication to physical I/O, so catalog operations, data placement, buffer movement, and recovery can be designed together. -- **Iceberg:** native catalog and FileIO, implemented. -- **S3:** core HTTP object operations, implemented. -- **Dataset:** native access and direct GPU delivery, in design. +Access Server and OCI container deployment are implemented, with deployments tested on physical Linux hosts. A shared Web console and CLI manage the cluster, services, and storage. -## Why CROWDB? +[Quick start](https://crowdb.dev/docs/quickstart/) · [Design](doc/design/README.md) · [Deployment](container/oci-image/README.md) · [Website](https://crowdb.dev/) -The hard part of distributed storage is keeping authority correct while data -moves and workers fail. CROWDB treats ownership and these transitions as -durable state, so physical preparation does not implicitly grant serving -authority. +[![CrowDB Console showing cluster topology, service health, and capacity](doc/assets/crowdb-cluster.png)](doc/assets/crowdb-cluster.png) -Storage bottlenecks move—from disks to CPUs, networks, and data movement—but -system boundaries tend to stay. A change that crosses a metadata service, an -object gateway, and a separate storage engine can become a negotiation between -systems rather than an improvement to one data path. +## What you can build on it -CROWDB owns enough of that path to change it when workloads and hardware change. -S3 objects, Iceberg tables, and planned AI datasets are native access models -built over shared infrastructure, not conventions layered on top of one -another. The goal is not to claim novelty for Paxos, WALs, trees, or erasure -coding; it is to make their contracts agree on durability, placement, bounded -buffers, and recovery. +- **An Iceberg lakehouse.** Use a native REST catalog and file storage integration with compatible analytical clients. Tables, snapshots, manifests, and files belong to the same storage system, connecting table publication with file cleanup. +- **Dataset readers and training pipelines.** Organize datasets into immutable Snapshots, use Splits to select samples, and read the fields each workload needs. Rust and Python clients support portable HTTP access or trusted native access with retained host buffers. +- **S3 applications.** Use familiar bucket, object, and multipart operations. Large transfers stream through bounded buffers; small writes share physical storage work. -Read the full motivation in -[Why we’re building CROWDB](https://buzzcrow.github.io/blog/why-we-are-building-crowdb/). +## Why the architecture matters -## Development preview +CROWDB separates replicated control state from range metadata on shared storage. Paxos-KV makes durable decisions; Chunk-KV lets metadata serving move independently of its stored pages and journal. The Tree engine understands distributed versions, and the Chunk layer supplies placement, protection, and streaming for all access models. -Linux amd64 containers are available for public evaluation: +That control extends to the data path. The native SDK and TCP/RDMA transport preserve buffer ownership across asynchronous work, creating opportunities to avoid staging copies and bound memory across the stack. Mirror and erasure-coded layouts provide different choices for write cost, recovery, and capacity. -- [Iceberg container](https://hub.docker.com/r/crowdb/crowdb-iceberg) provides - an Apache Iceberg REST catalog and native CROWDB storage. -- [S3 container](https://hub.docker.com/r/crowdb/crowdb-s3) provides - S3-compatible object storage with native CROWDB storage. +![Iceberg, Dataset, and S3 share CROWDB's metadata, Chunk storage, physical I/O, and management.](doc/assets/crowdb-overview.svg) -Both images use the same CROWDB runtime and can serve both interfaces from one -container. Use disposable data. Production use and on-disk upgrade -compatibility are not supported, and Dataset and direct GPU delivery are not -available yet. The container's PyIceberg write and pandas example is in the -[quick start](https://crowdb.dev/docs/quickstart/). See the -[S3 container example](container/single-node-container/README.md#s3-container-usage) -for port mapping and boto3 setup. +Read the [design overview](doc/design/design-crowdb.md) for CROWDB's strengths and the [architecture](doc/design/design-crowdb-architecture.md) for the mechanisms behind them. -- [Project homepage](https://crowdb.dev/) -- [Quick start](https://crowdb.dev/docs/quickstart/) -- [Documentation](https://crowdb.dev/docs/) -- [Architecture](https://crowdb.dev/docs/architecture/) -- [Demos](https://crowdb.dev/demo/) +## Run CROWDB -## Repository +- **Try Iceberg or S3:** follow the [quick start](https://crowdb.dev/docs/quickstart/) or the [local node and client guide](container/single-node-container/README.md). +- **Build and deploy containers:** the [OCI guide](container/oci-image/README.md) covers shared images, local clusters, persistent storage, and lifecycle. +- **Deploy on physical Linux hosts:** use the [host deployment guide](container/oci-image/README.md#linux-host-deployment). +- **Use Dataset from Python or Rust:** start with the [HTTP SDK](sdk/README.md), or use the [native SDK](lib/crowdb-sdk-native/README.md) inside a trusted deployment. -The repository contains the storage implementation, tests, and source design -documents. Start with: +The [access acceptance guide](app/crowdb-access-server/tests/README.md) and [E2E guide](e2e-test/README.md) describe tested clients, deployment scenarios, and restart checks. Compatibility depends on the interface and operation; the native SDK exposes host buffers, not direct GPU delivery. -- [Contributing](CONTRIBUTING.md) for the development environment and workflow. -- [Documentation index](doc/doc_index.md) for subsystem designs. -- [Single-node container](container/single-node-container/README.md) for image - development and packaging. +CROWDB is a development release. Use disposable data for evaluation; production use and on-disk upgrade compatibility are not supported. + +## Explore and contribute + +- [Product design](doc/design/design-crowdb.md): workloads, scope, and advantages. +- [Design library](doc/design/README.md): architecture and component articles. +- [Contributing](CONTRIBUTING.md): environment, validation, and development workflow. +- [Demos](https://crowdb.dev/demo/): see CROWDB in use. ## AI-assisted development -The code in this project was written with AI assistance. The architecture, -naming, module boundaries, and trade-offs remain human choices. AI is the -compiler. The intent is mine. +The code in this project was written with AI assistance. The architecture, naming, module boundaries, and trade-offs remain human choices. AI is the compiler. The intent is mine. ## License diff --git a/app/crowdb-access-server/tests/README.md b/app/crowdb-access-server/tests/README.md index b4ef82773..f61e8751d 100644 --- a/app/crowdb-access-server/tests/README.md +++ b/app/crowdb-access-server/tests/README.md @@ -3,7 +3,9 @@ # Access server acceptance -Protocol authority: [native Iceberg design](../../../doc/design/access-server/iceberge/design-crowdb-iceberg.md). +Architecture: [native Iceberg storage](../../../doc/design/design-crowdb-access-iceberg.md). +Protocol implementation: [Iceberg wire types](../../../lib/crowdb-access-iceberg/src/wire.rs) +and [request handlers](../../../lib/crowdb-access-iceberg/src). Client recipes: [container development](../../../container/single-node-container/README.md). - REST/official-SDK conformance, native durability and ecosystem compatibility diff --git a/doc/assets/chunk-io.svg b/doc/assets/chunk-io.svg new file mode 100644 index 000000000..bec78a03d --- /dev/null +++ b/doc/assets/chunk-io.svg @@ -0,0 +1,156 @@ +<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="1500" viewBox="0 0 1400 1500" role="img" aria-labelledby="title desc"> +<!-- Copyright 2026-present Gian; Licensed under the Apache License, Version 2.0. --> +<title id="title">Chunk write pathsLarge write: one dedicated pipeline and active chunk per object. Small write: N objects across M pipelines and shared chunks. Direct EC and mirror-to-EC conversion have different completion paths. + + + + + + + + + + +Chunk write paths +Large writes own their pipeline and chunks; small writes share a bounded pool. + +Large write · 1:1 · dedicated pipeline and active chunk + +Large object A +Streaming input + +Dedicated pipeline A +Bounded framing and buffers + +Chunk A +Object A only · mirror or EC + +Seal → location(s) +Complete this object + + + + +Large object B +Streaming input + +Dedicated pipeline B +Bounded framing and buffers + +Chunk B +Object B only · mirror or EC + +Seal → location(s) +Complete this object + + + +A larger object seals and rotates through additional dedicated chunks; it never shares them with another object. + +Small write · N:M · M pipelines, M active shared chunks +N objects + +Object 1 + + +Object 2 + + +… + + +Object N + + +Shared writer pool +Bounded admission +Choose a pipeline + + +Pipeline 1 +Batch → frame → mirror writes + + +Shared chunk 1 + +obj 1 + +obj 4 + +… + + +Pipeline 2 +Batch → frame → mirror writes + + +Shared chunk 2 + +obj 2 + +obj 5 + +… + + +Pipeline M +Batch → frame → mirror writes + + +Shared chunk M + +obj 3 + +… + +obj N +Each pipeline owns one active chunk. Objects keep independent locations and completion results. + +Large write: direct EC +k data shards + m parity shards + +Data shards +Arrive incrementally + +DiskIO data writes +Submit each data shard + +EC parity + DiskIO +Accumulate parity as data arrives +Write parity at strip end + + + + + +Join data + parity → seal chunk +Next-strip data can overlap the preceding strip’s parity I/O. +Partial tails retain their exact valid byte length. + +Small write: mirror → EC +Optional conversion · 8 data + 4 parity + +Mirror strips +Write every replica + +Parity accumulators +Fold each +closed strip + +Parity DiskIO +Write + fsync + + + + + +ChunkDB publishes the 8+4 layout +One healthy replica per mirror set becomes a data shard +Data shards reuse the stored mirror bytes. +Mirrors remain authoritative until EC publication. +Incomplete or failed conversion stays mirrored. + +Payload completion → per-object locations → access-library publication +Iceberg, Dataset, and S3 own visibility. Shared writers confirm the durable cursor for immediately readable publication. +CROWDB + diff --git a/doc/assets/chunk-kv.svg b/doc/assets/chunk-kv.svg new file mode 100644 index 000000000..70df36b45 --- /dev/null +++ b/doc/assets/chunk-kv.svg @@ -0,0 +1,115 @@ + + +A range-based tree distributed across nodesDynamic splits create partitions; balancing places those partitions on different serving nodes. + + + + + + + + + + +A range-based tree distributed across nodes +Dynamic splits create partitions; balancing places those partitions on different serving nodes. + +Range-based tree +Ordered key space [a, z) + +[a, m) + +[m, z) + + +split at m + + + +split at t; then balance + +Node 1 + +Partition 1 · [a, m) +Own range, Tree, and WAL + + +WAL +Ordered mutations + + +ChunkStream + + + +crowdb-tree +Ordered key/value state + + +Chunk backend + + +Node 2 + +Partition 2 · [m, t) +Own range, Tree, and WAL + + +WAL +Ordered mutations + + +ChunkStream + + + +crowdb-tree +Ordered key/value state + + +Chunk backend + + +Node 3 + +Partition 3 · [t, z) +Own range, Tree, and WAL + + +WAL +Ordered mutations + + +ChunkStream + + + +crowdb-tree +Ordered key/value state + + +Chunk backend + + +Shared Chunk layer + +WAL chunks + +Tree page chunks + +WAL chunks + +Tree page chunks + +WAL chunks + +Tree page chunks + + + + + + +Split changes range boundaries; balance changes serving placement. Both storage paths retain their shared backing. +CROWDB + diff --git a/doc/assets/chunk-placement.svg b/doc/assets/chunk-placement.svg new file mode 100644 index 000000000..7ca6c78f3 --- /dev/null +++ b/doc/assets/chunk-placement.svg @@ -0,0 +1,103 @@ + + +Chunk placement across nodes and disksThe client obtains a persisted layout, then writes payload directly through DiskIO. + + + + + + + + + + +Chunk placement across nodes and disks +The client obtains a persisted layout, then writes payload directly through DiskIO. + +Chunk client +Allocate or query a chunk; use its locations + +How a chunk is allocated + +ChunkDB +Choose replica / EC placement +Build the chunk layout + +DiskDB +Allocate physical blocks + +Paxos-KV +Persisted Chunk metadata +Nonzero store groups + +1. Allocate / query + +4. Return Chunk ID, strips, and segment locations + +2. Allocate blocks + + +3. Persist the complete layout + +Payload · TCP / RDMA +One logical chunk maps to physical segments across nodes and disks. + + +Node 1 + +DiskIO +Local byte I/O + + +Disk 1 +Segment + + +Disk 2 +Blocks + + +Disk 3 +Blocks + + + +Node 2 + +DiskIO +Local byte I/O + + +Disk 1 +Segment + + +Disk 2 +Blocks + + +Disk 3 +Blocks + + + +Node 3 + +DiskIO +Local byte I/O + + +Disk 1 +Segment + + +Disk 2 +Blocks + + +Disk 3 +Blocks + +ChunkDB owns the layout. DiskDB owns block allocation. DiskIO reads and writes the disks. +CROWDB + diff --git a/doc/assets/chunk-read.svg b/doc/assets/chunk-read.svg new file mode 100644 index 000000000..84f231775 --- /dev/null +++ b/doc/assets/chunk-read.svg @@ -0,0 +1,64 @@ + + +Chunk read flowStored locations resolve to the current layout; reads follow mirror or EC geometry. + + + + + + + + + + +Chunk read flow +Stored locations resolve to the current layout; reads follow mirror or EC geometry. + +Object locations + range +Logical offset and length + +Resolve ChunkDB metadata +Current strips, segments, and valid bytes + +Map requested intervals +Preserve logical byte order + + + +Mirror layout + +EC layout + + + +Read a usable replica +Fetch only the requested interval + +Try another replica +Keep the same logical range + +on read failure + +success + +Read the needed data shards +Healthy data needs no parity decode + +Recover a missing interval +Read enough surviving shards → bounded decode + +if required data is unavailable + +success + +Verify frames → assemble the requested logical range +Checksums, exact lengths, and current-layout validity + + + +Deliver bounded buffers +Consumer backpressure and cancellation + +Repair is a separate ChunkDB-owned operation; reading a replacement does not publish a new layout. +CROWDB + diff --git a/doc/assets/chunk-stream.svg b/doc/assets/chunk-stream.svg new file mode 100644 index 000000000..508a11a84 --- /dev/null +++ b/doc/assets/chunk-stream.svg @@ -0,0 +1,48 @@ + + +One continuous logical range across chunksChunk rollover changes physical storage, while logical offsets keep increasing. + + + + + + + + + + +One continuous logical range across chunks +Chunk rollover changes physical storage, while logical offsets keep increasing. +LOGICAL BYTE RANGE + +[0, L1) + +[L1, L2) + +[L2, tail) + +0 + +L1 + +L2 + +tail + +append + + +Chunk A +Sealed + + +Chunk B +Sealed + + +Chunk C +Active tail +end(A) = start(B) = L1 · end(B) = start(C) = L2 +No logical gaps at rollover. The journal owner decides when an old prefix can be trimmed. +CROWDB + diff --git a/doc/assets/crowdb-architecture.svg b/doc/assets/crowdb-architecture.svg new file mode 100644 index 000000000..c6496c090 --- /dev/null +++ b/doc/assets/crowdb-architecture.svg @@ -0,0 +1,97 @@ + + +CROWDB architecture +Iceberg, Dataset, and S3 form one access layer. Range metadata sits above the shared Chunk client; access libraries also reach the Chunk client directly. Paxos-KV forms the metadata foundation with Group 0, the system group, and Groups 1 through X. + + + +CROWDB architecture +Access models, shared storage, and the replicated metadata foundation. + +Access libraries + +Range metadata + +crowdb-paxos-kv + + + + + + + + + + + + + + + + +HTTP applications +Iceberg, Dataset, and S3 clients + +Trusted applications +Dataset / S3 native clients + +crowdb-access-server +Listeners, authentication, and runtime + +crowdb-sdk-native +Shared topology, budgets, and buffer leases + +Iceberg +crowdb-access-iceberg + +Dataset +crowdb-access-dataset + +S3 +crowdb-access-s3 + +crowdb-chunk-kv +Partition state, catalog, and serving authority + +crowdb-tree +Ordered page state + +crowdb-chunk-stream +Ordered journal bytes +metadata +Shared services + +crowdb-rpc +Internal transport: TCP / RDMA + +crowdb-monitor +Discovery and process supervision + +crowdb-chunk-client +Payload pipelines for access libraries, pages, and journals +page I/O +journal I/O +Direct payload I/O + +crowdb-chunkdb +Layouts and protection + +crowdb-diskdb +Physical allocation + +crowdb-diskio +Physical byte I/O +placement +byte I/O +Replicated metadata + +Group 0 (system group) +System metadata and service registry + +Group 1 +Replicated KV state +... + +Group X +Replicated KV state +metadataCROWDB diff --git a/doc/assets/crowdb-chunk.png b/doc/assets/crowdb-chunk.png new file mode 100644 index 0000000000000000000000000000000000000000..050b6c3a096f2c8818492e02d60762c966853c6a GIT binary patch literal 240517 zcmaI7cU%))v^I(_3eu6@dl99DW*CQaf z??^y!7k;1Uw#EL7JoRmH*Goym;Qs9qeBU9iks({ zOC3OePh}W86?}LMlkW(t_Cbq$CawJZIce_@vxfTJI^u;^!0dINnRPXtDQtRiUSR`x z>@yUv#2-)9Zj|;~#oW0l@OksyO!Ea%DsM8>i=lzLKl(Ug6zuE=hGiP>oa?#HY0Z zM<(g@1Vl+QH`u*-m&C-2HmUY|sv5eqFbVBWdEa)BtY3J5)p#2FK#zwIvGc~I^Zp}Z1RGk5 z_oZ z2!zy5egZpHttgW8c_!qc0HAZsTog>QFYuo=FcW0Xff~Yxf%2y|bThr1B1}^E>_k%U zvJ_z>v*3m*Xw!TWE?v%Q$s`FO2UuM_I_V=!+085II-PvVv{h8Xr;BL!9gRO1O?5X9gx#SY`Qun6pI9hW&k2?6#o-_?bWbVCtC$V0gLJOO0U?8P z*lMpnroLR8@KHMBx(_(Di(wx4`_v0+n_3}DIMJCuXQr!omwa^FJNgwe3$A*tEl^hN3s`#$ zfBfr)4ZrjWYUvyUu&Md7<6S z5}pOmHjTFbCZtJD$+j-F5$(eHl*oY&9?gZdg5tF)t>qz`*-VmXrfRIsPurfpb@IRt z7!T5vJR?am-A^K?VW=rvtK7NTXC0<4^czf11Yb5RQ+^Rc{j~scAN1#KYa$?(>Z3OgI0c!Z8lPT5`qw^tvNt3<2qQAP&2&k> zoLYG$NL<6F1IVh6`%ILZe-~}`O}r&u=`5p+Nu%{~|9Hd>?rSzAEFMK&l0s-~>uY&7 zp2vD}=Ql1T8Vhgb*?ZQ~G$(4hP~L`s0L1C-uMu;gKA(}$PS zz~$~Bu}~7z{HQZl@6d*w@g$nsaqcp$*gMX4)iSQWQ+J4rP1zMQzI+iXrrVwlu^H+M z<#Z5EVhaj*%BeE-XbLE{8E08M>B1=c#NJ+ogmXVX%B+!G=Sz4o zxm|84X|h``=qt5Ug=CXRw9jMY6Ofc3+IYb)`bWZj@HoOi4eA9H92o~5mE?xH)D$hj z38yQV@@19EaVJ3`N6(=BCTf(=k0rp}hHRJ$n{6;OR#+@>40Yu_w)4UAA) zE$N;J@M6veQ`ISCdT~I=r&x~!8yz1YW5nF69T`A^)h$W z2eZY8P+BZYF#Y|}?8gwAF&2m?rr05My>?I)K$tk{%Sipt&(oS|EHgevoS z05x!)JKyM*s~?Vn207P{^b+ zS87FY1U$0M3IgRW5thwuDJ~Cv^oEtQCx|9{;_!~@X*JY?e(7hY^wb-Jk)NtkcMWo% zH*v3tFlkc8XMC2;KmL8UWP$_Ce{@`3HxZV6K8GW9W_tz+++_o)Dn%Pu9u-{T^W_ea zQToFVQSUuo|7Px_WU-nu)BBOcQ(HL~YDPh>u9dqHsAO3+V!XXDcfN1>l}*sYUr!z> zdsJaL$&EDgetLKxpF>BBm`^r&qM-$w=<4|1TIm-Wf0TI&fq{mPNX!cQ+mKM9THz-(E zN+5EM|H?f6Zn$g5l}y4(m@qt{5N~Y|_bLorm66tn7Eb!<=?{}Rm+d3oqrYE=h`GWtmgG;MK@>YPlP^H z0JV;F^(z$p@VoB=4jnKpFz=~|Pc^48UmPCYQp--GU5e2)M}0Wd0E98ee>a?qOq=wLOW2eu zNKNS@0z~VqFXC#~5 zm2`7crEDdku2WYG(ygk%z4K(XO;t7vnS*{-bn#X$N1tHLAhp&beFoKb06Cqtn+##I z7hHU?79tp!mL;S~oSF9m~}_9yn1GnvJm;>Geqgr zz?2Nop(HQo@C7JYzjS=R8Ey}$dv(KIqCU}DlLxNmpZGEfv*VC(tro^lTPG>J)hdGU zg5)M0I`unQKyKwY69_zA@ZOj(`Bzz2B(;+K#e!asUz;rr;Q0lcIy}Q493y5An|skI zD5Nq7k!GRz+o>9UXN*K?(KuuwMXAq_!^&_1uV6AbIJq-#5+hEA#kegWO(dc?GTH7|I&zeT} zDaS`F*mCb0)a8L-{$OKL#Kag*?!Cq)pu9z_myp5+E8j*b0QfIE5nmUhU(Rm(0(3ac zHrNRX!@Y?f$r?uNdY`n@c$dhk_x$1$;v3kZD}f$a_LWT(cdGhJnA%n!|E4RO@I8QW zVe(R-0uXN+?V5Mh{SR|YQn>CsXq>zQF#1Rip+E-Fdl915SE0uV)c=6g=(&TPZTm;E zyo=|L1q1R`BlZ_Q^$9skpk)w^awvhLQo@dG|GsDt>~IA}H^;PlAMO?&0&%4H`V89n zKsu<*hJX-c1It`9T}RlGAhimmKTl~qd+zl|;D}Yg2@{LK7OY-ENxdI2C)GtFpB?`!+d7;*Heo!7XfXSiQB@KJJLNVRFgE(Ur8(I>1yfB0A=CxtJksWi`||E26cu@lqewwEf5+EI$c!->yBU|$=Q zYU3uTRYbxFCr0?B>BatfP9XrLExYGj@{rp>DNIRVqnJcC5lT^IlQaoPEv7P8 zuP_=h{Ww@T{vg+p0#13ZllqOu0l7Ztk?x7)%l_>3Vx@hZPU^mHPMXSPeEN~36tM66 z3W1>^KwPSCW##i%Rv`)IWkme`e9QRmX%^G+vlq$KDL|oULf$)(Gp+UwO}@XFHFvZy zs!(VOmm4yBMxR1rGD8YGa`B>%VYg3oy6JWJUD`+i>+zk3#aRYWn&du{#3P2(lJ7&Z z%S14qH!s>@nIM#@hj^pxSjzgx1z;v}rIHjjzMcYncUdA6;~t_$N@(t&l@fyXDNbB3 zzQ@Kq&R00_yMwjtj{aJ-9m)ILkHt)bri8TPGejaT-ZnU>E9~bdA09q?pS$r4-oNiAj2697b=^Cb=qD(0JVLI3%Xi9HC+~Rx zI)0HcO3$UE=J`8gpNy=Wj}Z$*J=%9ZH&Ghnz7QeK)tQq zM*0GzY+e6b3HxgLc6ueN@Wj%GYMYQ$)*wDR9SN&g$EDcn9VS$gxM38V^J&vW{;wIU zD2+$d{$e9sl_sA7$eP52!=5L>cZyq7 z3~I7mZSx(G#xmdBj9s)ki`WlM`!%!moh>Hc;MH0}k& zya+O({@?*KZF~ank!!=C>rv@_XVpH%oTG;kDMqP3ztmPJ^Qm0q`6X1&^Y;tfPyVe_ zwkEu}8TiRQ_z4mq?W=OSRz328ZT^UHmkTm+VU(8HYDGpUJuv);NF%wlH$bo_7oaDX zm&!08wW)TO8S>o3ck1UtHNH&fpj%o&(Lm=p$AebEkiz2Y*M00=D%WNM_MFSuN)Y+!*hrf6Z5N$XoJ?8+oib3eV%ta7 z@eLLQm*i!060!JE!D&6r%cTH*62xO_WNpYsB#KTJ5r&KWFY5|y3OiCddtA#}F zH{=#ag#AiDmi4wZzhsp{@Wp+CJcrg+zQBb~+B#D!Dlrl@4R)tJCqC9&Ns{_PH=)*G zlrn~V`yiN|kGM`@qu70=UK^0YkR~%os9|$4^(3{z`1D?JJ)*LM;c<-nqie61Ap&t? zPNVP?kMzu=Uwm>&ci&HjPV+I(BR)*pd?-I^4!I%e=Gz4?!LcfBuyfW_E7Fo-frKBIt^OdAtUO6fam)}G?@e8Ti+y^>qfL;_1 z$8yZ{?IJ%9;;Tn2eg_G;JiVG+5H|kVwu1ahBF3D`@bxfDU@*Bm%ien1HfU}){*{!? z~pEAHn9Is!TqU~ zPNxPIs8d+yC-Mu=!z=jyi$$!0N>-J6^Xp4=%+8P|86le&cGv1K|rX9cfIOz+IR z{dyo{Zx})`92KMGZhO=)w_7^gYHDC@2ZFEbS%4#LpViVm8~m24@Hj`Z=7w6|X>Cb> zllPuzfJAq`tp#7xGaa0tn4bfV!~^V$zSK)nJsZJbrq zJi%{KDY?1Aqwsi&&x1a;lb{U~?GKTLoZl;noc3+e!sfhj4g`mmxv33Cr2H&6uDIPH zTb!V_PL=MjGnBv$mzQQ`_z$EJ6sR(3*Jcd53mbs}Qb zLx>w${L7G6PWSs|m?3i!nWO8n-Ur6!5FZ%xgxdImPO4_ILJ5mn(eOdqxbp%8lgIE= zTXezcio&^uVPk9)RsA}1L9r>u^;y)(bm@Inv}==Y>l6y08uz7fs6rbD+k8^oW&EBS znNvC%=ox-b>dEgpk6FhISKK^C^z3nE%`d%IB6^wEqci;zP{Afm&jN}AY2uNaiJhjZ zVjh0AQ0vIIuz6K+x^NbC9}&jP;&)tX53-6MwN6z|DhLs4Np`>02oO&t`S9Y(wD{Z9u2rvcY zRbd63RP1u#m+s<^Nc2dVYSN_5n#{4qG(*hFh5?af8*F)QjLyf!ghtcFCX(V3 z@or_BVK!>>XH3QC&)J#>NduzpSXydGn2i~wDo+G)g=@hYHWdW#f2mP-fj0ReYI5RV zAu?^ODzoBCW4wAkSN**V5eklOqI9BPlJ0=|h$IA)nwbixf?4G3EGgW)GSVxxO<2PD zVw(yVvKFpIQCz9&U1HBF9+b_PwJvKI@5W%0Y*or#VOnwANjnOl7lsWBmm0hUlil0` zoo?DYcEvejr~O|AZZSGaot&0J?SNJHl6e{yM}6VE=_vZMl&$#X=S*(U@FEX4* zxzsmJTN8{{YMo83lTrje@~XypHyDMNG0hn!NnP2)q-jDqNyVsENzM7}yGl~Bo#5;cf&W=cKd5imdlb@&kzx-q|AbJYEuMMH4-AsjAwq%a97kQ{l}5M2L5`p0r{#X z$dwk-7^o0u`3~LO?TUU@g;FUM2gwBHET}z~BJIqdLCJswHH1@>&s5=puZ#;Z!c}D! zypBjSW0(HsMD+&kN3ZRSNr{E1FXfpfzs%06`b`eVDwb$f(}Jc071ufZ=lKvD*|Z<| zq#I|Z?a|cjG=i9vhp5jQ6K@NRVm%s+8tk)q4Hm?s|77^wVM(f=s4}8G?k9x&?JM|l=mN?9{K`* zWIc%vcdn`KaVD*>*=oUx?06lkFKaZ>`Id~12dIX-`aDgowE;AzI56M>ms1&u zJWcNpp!)4@yU+R8G)ak00%T8;;ysxa^9&!~CQ=Kyyoyd{!>zMT-0aZeG}Z(}Dk+i| za=Fg>mp%o4giRC-N+g0Gs{o6iqm4@cyj+_oBLS?qENK8=T*7D>9$d_03V*AiV=Sdmh@SiE;P0T zm2{huJH@5U5XJy}jD=1WA&W(e>5m5v=Z?s}J|^`1m7f21!Om%0*Qkv6fm-}CKap`n z>9I>fD!4pbm1_*tX6o_rvW$oF?#}DFQ+Q|Q?{wHd@_4xjF7Uux>Kx_K=Y~@~QN?S| zYx9`CzN2h%D39ND<{m3foZcHq^X_+|lomSeS496d4#L<%%ZHUC?)#|QDkvMvOFfQq zYAOjV5O2^^zruYEjl0}6#ov2*N5j~A7q5!I+?*>zJW)MU^+R zwVtsx{pr{)W2lIpT2GE8o8nS2G>x^ejc_xtR=+Gh-%3U3B|Jl{RUbr_|MuS+r&}B} z`UJjr{c3JJx_y|^Mc~QfpA)(=s+;a2G(1IBEV)2lt;-4!M58aXKpYXiG^Oog-k1}J z`VYlh&3#_R*Ja^p3HSlaVZUJfD~z%!JL}{~G+>mIG3X`MNy*BWNK0t&heZ-l4}WSh z{ptD>Iz(c`w~|sQngTt5)^+9f8*N>ddaTi}TOK6efTV;|W~o9aKUKa0=NOPD;t>bI z4}S}SR+EY~bzWWdA^Q9b8(&B{JaQQ1u5gX_Tzc5fWfu>qQ3?62PTsT?1ZdI_OzpPM zdt%o0nIR1106#SQlyo_K^=2K|bSVMSfBTDnQhMvh{6rnUfCr5-xajRr)>AS;o3MXa z;MS~ct%=7Rp7L^$|tDTC3yYp9~-`tqAfj3S8x0%UIKif3P3okkl6}*P+JRbIdeW07C#~{~ziEHVc^tNKc7B4j_evwz_TAjo7TPwB{A z>@qC!YOzHrOFuQb*e%I_ZDlqPB%H}=(~g%-Y>L||jN&wYUBx@qJok*w=@suA?fnuL z|I{XlPQ7AEMiaIrr8{oX@+u^X&zC162j=Ld$?U3JZRij^hm9AN8@2UNHR^R9O5Z_j zY6`4Xlv=$>q5D#nBjC=vZS0!Ju!|`UfU1rga~8!bOH-DKXgOY%zQuJ(YkRcr^bd$N8eX&S7 zi2gc0BV7pwO$(ch!Y!u@9meQ%vjm8Bi#}*22;%)1z-b1Af3?vi+hRG2fE{zGM%lpYbk%HU}@vY$)+e@SEQ}mV< z%dubJZD)uH`H_>TZFZGrX#UGaY0@vZuhY@3&^c9rakI(m8R@>8bt-BKQmoP0b)Z2R z4>Rb>n@G+9sb9%5GXkPR-C37k*Gcz}qDcUeMs>;T9t}>FAvr2ofimAcnG52vPlb-s z@$%}!!3=_@ijxy8UEN6@sYI{6kWM&_V#Ipy^A=}BS==v}=s`>qgZvhcxQJWoKQwuV zlDZE~vV?6NeH*eZCYFR#{u-9BQ0?0==J3P{t5DU>wyG$WZ5IS~CJDxc+UTF8&cUo2 z2@%J7Pp5^+-MSq0uXX zohP;uuleCqj5(A(uL*}6_FocneE##>;bb)m>qko((Q`b|wt?M06rdxG zQ2O%n;lIVIDb1zyuCZK0SMIC-X!u|4jLf0Cn0(RsuZ+5eL_l5weYVFn8HE;0$JD>+l z-m9$ae^@Vbcz8zlnoBslLIJR6J*tdif-NrBJo-Ow=Ra5Z_vXN~v?pHna^IT?%s=v5 zpZPVCMmR(?F*uV?nGMywe1c}ydr=J25R%_rJ9nLL0fb#YGgy*n3n8nwTW)IjQwaGk zVDBR#*%mo$-rV4N?5iq>+{QX9-N&^D{NKCxe=ZlHWG&c%y(OFBgyC%BauLduSw9@b z;9uOc)a~y1uJ$K3W&8j?I`k0xU`c4BHNpKd>f3Zb3u29Zyi^;4JF7X@!2~$jtoDEJ z(6Ln&gopML9Wc)ukV^ay+s+8$zN>V9Zw7!yj=k!@4rM9C_c{#Qd@V3t^ux8sMD&OP zkfZws-r&h=w;u}qpg1pfqPEf9>61%o(BbU-i>s9tu0;9fl$aO}R<$PzBibT+R75T| z-7oEfe+nD~*;)z>{SOZ%RE31Q-VINrijo2Np+5-c%f_qur7cR4iivi3kyCoZ&y2U^ z1x7x)l@vqfVwaec;c=b!k?+_`%*CO6p zuimr%{=x%UX_yFqd%THXg#Et__ka1zm-@t6pzI51a>Oy!X7_%@UNM~0MBIU)%i}L# zs6NC<@fx_xVW9!N1K}k~1W-B)j}x_ZV13jia%6*Xmtd=u2$q|tFY{%ylqh*vK}AHm+T9?b0`GKuj?NXg6qa z(2nJhiHNR~TMB}C5ik*=nLT|!&0xL|bAD$*MWKNitQ<=N^iLL=3+mo1bJt1F`glY# zXZ67R@M@6>z#JHN*;U^SF|sxY1`jihqbNG}7U{D>)HBvoF|lnW>CNr$>g+89c*Ovc z2cf!J9p8r^%UKQWK1a5=JgSox`u}#pYx;T*@3bS4+^p$YADeIWhoHkWW5BZ8YGlFn zC!H3XH|24D3=R}a=D{29YG?Vll_GV@!m;^52Z&T&s`3CFbKO$ibbXyTQXDwCxW7z1 zf)jHo@C5F1E_Yv|{OYZTfxE_sZ`S5KWUAR6@PxU{oo>?Xso4?|xvQoa>%H$}{`YVR z-5*4eS(ews=H&A&?qxgPTfRhZaAXT@VI{(*%}_FhGZF&P>lseC$9_(T@7|r_A(H?@-V6YVTo~SOM<$^4_669jTbp)gNJd?-sZlL`Y(RKYSeY?wh?gUZ#i4S8rsX z;~v|>Vjuo7>i+gUu*d&1GThcWY=OaJJd}zq+{;~OQ9)O^($3RI4gLDeWcCGJJL{7U zdBeG)->Baggxox;{kHyiL9(>TsQW!q2e4-Q%TAOsZR1 z&zvPAfBm^FvfWEOS8!k;fteun=KZ_1Kq4aHaOH%9H!26da4@By+oB(SEuzOvxkVo^ zwySvR?zfC5^qLw9@%_dBGvlZ%)1qx80c+tuevk2Qa-2f1LmeVRv2$Hd z^y*yDZ0%0R%DzQ|3o~cymZXKppz*igDRqC6JDW zFfAqS{}~V1FjxG}*|87Y+(S5?1^^H5t94wr_@hj3-@SCc$B6tl1I@f2C1%zarT8a^ zEo=cEl-@9?z0= z5Flpv=JLX@4W#fgNRHO{Zg9~?R@lThJCoB=2y-`;a6Tp`No0m;-A<32o+MJ{G5)zc|QO^slI_f;`2c``%{lb za34QoNMHfmVQcf6!*W+_7-IeK+d0Hf+zV~NAZ9+rkYwRGR+bI&J0M#sjZ-{(m9gwW zOn|*TF$Q1Jc97(Im1Ofv7CC=&a2bDnbLJO%{Dk2Ait#363=Vg=y3QSu&!tTW;xEAW zWSX?Q;q|Yt-IuT&NEGVghuuxR@jmckQV<{~7g1Gv(ft}<`-l9T(B#j-$Wb@d@akol zABXYMw~LrCPP3sgPCOYJjJdms-jTSTv0T6wb*+NKe1x2RcPPVAh*ZgC>}wtWQ$72H z@3)ulfi*1H02ka4v9=t9`$~CJ*&Z+-u1h;05_&U)t&I*n5m&3ao>uL%TAAI`9f>Un zOz`V~D?OqXVGM#5(cM&VBaE%t+uvNbEX}`owRG){UbBm%vP<%*Zp8~;T{#!5&Q%?h zEg5Q$=QsP+g<*x~4yoRB;}An?*MIJETphZvlZ6E_o*tl=^ZFl;1d=%jTYiUGkDSjq z8D0HVt-b25sSf<5qY+NbiDGtu4aGzaM-C_c{h3?07Nls7c03G_Thf6Jg@M#`7LjY$ zCQC=|VR8LlkwFwso$kw1V4+~lcB(sqyK?Pw_H2fk2Y+DW4H zm#=Q8L_XOYKT9C@omChU+`v4E(GO6oyXkX9;)mXj#~^?0Q`WzoH}aUzFw z<)a__$Qu(xoS1huZa!?NJ=B?`s_Rk%aTQbL-mqCn;_LFy&QC*40H+vHHKFi(9VaL}F(@PE>c|pt9qWUA}i5sv$F-ymmRK%cqJzfbOT>O*&Ie)%`?Wb~iz~QdAX3Ya&A%=Op%xh}9?2fps%zt>raLW=xt=-XO zwk`c%rd#j89$)~9!*;jD_-$gU0Y?o z03h)NtDLf$7w2LAoN88{6i*iB!)yFJhP%MoyT~WIZnVblaP-EFM`X*tMFcw&Ov4p7 z#uoPq;cEK?OlCtQ1sz!}h+zKZixuB4tkIK$pUF0#2Lzu(Q%wSUJ!{R` zzf;zqpVOKyv}bB|U5XsAkbM_fXgb&rK8K}(wwr^_4+v>-*ZyG1;QN}4iS-AY?ca_! zC$$ik`y3oIGVu}a`KT*X!i6Tg<-I>AdN%J?2d~a@GxQd5uGfEabwxvyEA6hlDHl28 zAOAOg($iQtI4>R0G6o0KUUw&+G-?N>brH@qyMlWq)n-oqae38w%UBst;_u;d8^iWZ zlolP{IYz@IL3WjH{Xa`Nz7L{0Va&>ee3Ao`KaKzFgAZ3%gu;cv?*#monqrBcxD>xI zzi29A{0lw=#;qnlvr+f1jC~yjcF6yhCKik{u6o*Q7U$I-Mui`WU|(2qnw1WpVQ~1W z!onvb_wC~~gHT7-TG>&U*!2^wWvQoOF2p_@+~ghEQY&xnib`6o5*HaOLfRQdat8;+ ztOoW7OT}LXOt~*1TS9%QoS(?!`fE59FnkL5Va~%NG%MP)x~W*IH7>G1Iz)EL&!pX& z(7DAe!eKW5U;DKm%ElEwME^@MJ#X^-Vh-qCN}FC zRT1r4&}r^kBADU*NXY6>f_+@|nJr^{XxC;#9iEr-s?+ayrI*Ui`qjE4GxxZUQSI36Nje)~vOE>vsOq>NPzrxNhY0I%+}x9Icyv2~|3(7@ zcyzu7e0L_S8W}a)90I<%vRccG?-MFc7d;KI1wqU(?}4Eq2bWzm1kx@^|_hraO<@3vJ2V=QR64686V> za|w^Rnw*yLIAF~;_zbYSQ+9K3=(#h{8+?4&)o-bV`JViJE3azym+b2gV}_5JHztK) z;2xhU)Ms$6GNX>e&mYdFaFAB$;TK#$wL)IRaoHND357E5%s&rTc9Oti~?$?bF|8tQkbXks#Wi7n{<43O)`gf&k#*WCxDcK^&4~ zS$b$lS zIqY}qmV%uB8!%cIc>S9{-R3d3GEiPGmHEKd3=;&B+q#`GD4)X_?09dyOnXf+15%~B^MCp*CG!o6lLef7t6>ZDYgem~tagzx2sDkT zLO=6q)F(mAka?xo`5j%W&j4rJ=;iY!<9FluM32dupQiH7>B-%qgp*!ObR#s*Zlk4qv*v+oH^+r)5-0fB$y5l|C(b{pic=-DuD0EOU z?8XV^#nx&II1Izyv$!%Y2vfR)6g$B4TOrmP%H-Y4##de4185%`)vmMNoLB~Sa;S-# z_H2s$75x;B;p`(`^5VuWwZKNgwto)gfKpo!+idnD`Tdx#hy4$uI2zt#U!Uu6gxc~e zb{v#J6XI_Zs1kW({@;dvHE}z~=(^B-)@sxCgnPsz4*1CUpovIK9=!eAA@Sdl#e+6# zfpX?I-Cv?s@Rn_0q6zaUIKMA5Od*|g#%?6XhrG^2Ha`Bza$jUw6=*iq_Z>l_jzV}V zKKFbx{YlflKWS&WTp;R&{|Ix(-vQ0m&8guPsy7E!uTi6rX-}MM`{O--8s#|9Wl>l+ zGx73BqC${Qdk`|e#gB1s@bIh>f16Jozd~DjDlX%~!)zgkP)m-6zl7wz?DS7b@5=Eu zmP$GrH#Wg9*XC!E<-?T;^)w}=azVTwYPx2Aoa;y`&%hJLJVs!I8`w46x&Qtl{x)|H zelVG{`_0cA6Sbdx^j9BrIK2r!JG)5)Kl!n&njn|w*RgL%Uel4fTi2F9CN*>wA-0yq zee($c@}CjvIQG-AUr#na-^SKCJnj{i+cE6e@Czfm@w3`!LkWDR^jt|XhM&CO2*!eq z8*GM+N;!_q(4bwqrL$PTLFY9Q;;eJS1=jFMe?&G{GL&$t7O&`yr=@QOr0yrB=r?f`BaJ#y$k^6K0aFSbB{*a?F zYWm3bYAEwKqb08giW9>ezVJ}}Z^F`7n9t?geV#4iu5Y`>S+|yRgm(@bz}DfpB};XQ zZSS0Ay90e%?y=xQwJ~pLFuJH)FFzkpDF{FY@1mE302}rN?v!9hhI~ZF5}!(S6IA!4 zb@KpcDhFZ7T)f~?4B2{sz%8{Lw<%Mt3SyU0#A00uhDFYRR(r#`%>)#O-DITn5VNoF zYhOBijDirDG7sD_G6%ZV8f0oMkE<2tS*sWE2h7Cm(gP<4PSp&KQa6X{z+4YLj<9zq z+EA}wE98+wlv*#C{#lxuXysWQ_#Um@t}v?uyU8iKSk0sII3`SNDsW{#g!_1w$a2bJ z`AWUP)hvrc@mTpE@`*1(4~8I#D_hDKy{LiG{k35HW&8Gi+Sg$0b{!99DAosZwN)E+ zws+k(NI$Qz(U(wTt^qi5L2jvuUXp8Oh^)odU9r2jt@LzW?jxI=FJ`fIbGTFm!nj0J zB)!#ayYZ)g!%2rDsaF0^nWyLz&jP&1adei9qjdL(eYwR~famc>gUE>GR}JlgNrkK_#g`9l}_Zc*3& z1a3&6`;anVmvzK$ZObtTfikYMwz2t0g}^Qj!2sd$MMba>B&+{|&K z#dKmtme*uD9-0IR`YnUHNy<7CQ*}FHoZUkg1E>SM!tCdiPgQ9#E4q{Fqk^p+E-jHtqY0Vt?jU)J(8cY_-lO~xD|HP zUAHhS)N;k!;tGtU&2-(&xw*nDwCu8c(NImm!0N6f-Q`JF9=Kgj1ht+;$_wwz2=1xi zFa_PMoPi?X+Dm!2wl3oKxg-d#$l~HcarMV$7k+I4XYyt{SNo(vAsMEsv;s(z0WnB~s- zmd>Wu{$MT8cGJR{9}^Db5`u;$N)>beNm-wN#RFq6sMgf z)zqS=h;s|opD)X1J-@;uTm62@^jx-|Z`!eDneG2ZJ?LqkgcAKBhKfT-T>=~zIq1O? zxty5HE(>%eNBwmLX3;!Q3qh+Jqjc?l+Jjl z!YuH!E1u2)NP}H?V%j0TtZV(YMOn-pH5WbdfkkL^z{E3>J%EwS&dA}G6j~z3~k)X?)T)vA|8$Ot;R8U zoLiIm-C%neJbS8Mx6c1)7d(!456e`%3a2_oHjb?SJ(NEU%wY2XF1P;vQ5GDE%kw+B zOtor6?Yqs?d!+p<=z}p`@rNNdEC&e8(wRGwiGic-eHdaVW`15cY~@*Cr$QV9N6#%e zyVah*{gl}!J1eFT%lRjM3BM3JCfIuG8UPG_c=7*Y>n-4-+S>ME5CrM&6r{TwM3D}W z?(XgeK~hRUdSFCKx*2-t4w3E#fuUo7A-?hbf6qD3Iq$oF!0egXd&OPrx??W@#Waak z3#03}^tFV4_GInP8f5+kf-F$&^<+)k9lNB)fN~I!S@Wo_gbXlm&gf{ zdx7JyScV&pJ+~dKY{VbC)Dtx>x{5)2Qtl5&64q2=srHKL&*eCZT6T6KJ=Xofv?=D23Qh_N?)$2X?Gfy4X#X4=`V2B>9byWN!h;pSB00u+syb1ky&5%IB zC+@r8Bg*lh6DDCDDe5)eE>jX&Chx4K+;I_ygYxUEGlbJo;msY&(XGIfJe8E^SSxyz z(8!d`i}{AYvvY$b9iN+$DA!&}-K(tt-79_P=kw;-AU88B@|T@wM95^{H)W-r zDjMw#4)21>9vT~Zu9rl`rdel0DWGrCkdQ;Gg8)8F?QA7CdWweyj1w0JSuUA3BG0L;Dw0Gdntbg(NpbN(|C7R z+BU5#ZC4`7S8LhQhLF4tBL>LAV&D#)(#QP%IVV6`bzE(|2}W2t%fjmK{QSm*4qnM7 zP^>}39h4U9BZs)TF_BWt=(^3SEn5>O1d&}u%ohywN+%iG#% zZ@xu-L{mn|7(>BJO1h~KbR=ezu1kE){nJw5^Q>^2j7p%QqESR#nzMrvnS`QS)`PCH zblh73?$Ib#y9^vBu_nH-9s*_hoXK#q8PYC<-ETBXKVzAG_fO;@W2MAU3Ba<&t($rx zN4ENCY(-FYo=^@W?=!{VXUSu+=FlF-|kiNvY zVD*Tg%brH8HQ980z?S9YG1Ny=Yq57s%;ELHSiLI>7&u<tf}=BcQ6nxj@vI|ePK?i9K%3N&#A>2M#Mj4tVWnE;i5mkvb3Q#m9FTFhvB&XyzRu^!|>8b4e@}njKpr3OtX;&~0_&X`XR8?=*tCuih&Y zmvcBwdDwkvk2qs8?EF0vPD#_bIulI2R=pJCj-MP;#X=-Gm7}!&dV>oN~980A# zT|flZH_TOzZKCg-T3>XJVl`ZBl>R`dOR1JkC%M?Tx<#;gkV!$fdA43pc}GDulgQ@M zk@ZrR*d*kE@dQ%(s`lhXkus&KgAV7#bNkcVE_;WSH4|WQn7a^woIh!kmIUh(b^L& zTr*V0pE$FH(qpDvtT!(g*WU8zPyzv?aj4YWGvKF+uDKIG2-@ch_a#rxrt(FVj6UUf z3f?-Sz~+*$?)D{S(WVq9yP;)Qf3C-~{Wpy})a#eb7z_e4n!ffgP#$hnh#}fyOG?8F zUrh!(9hl4oDNK1Ru*o&2DKAp@w?2GLA9-5!v}nIQhpSITQIU^YJae}ps2r_92|%eh z&ixz-GFJ291G&~*u(>iOcK+Ra_edOvpOF%_eI)z@y>AeN*O%rj#Zx(ju*;sOlaNm`$0=yK_H@w-1IB^(KXT~5a_4yXnzDAt*Dakcrj$bXp0JB)Z>c*TS zyJJY}iFL+%C9o5Ri_xrEeL)va`hj>Cx9+##xH#;JXZUb})cSecI?h5zsta&NZBvl3 z)VQO^tepgt4YdG_}U2*d-<758|)z%;%yLXE~2mHWKeabJlD086XQp1|*v%)1-X}4QVmY`%0pTf6KmvZa7 z-m2$=5RPZ6Ij=Ar7THb)INtADj*Eh5fPtB7-A&7cP9|!3xAq@CK%Bf7RUFLj)4Ebf zm!AkXL%`uS!6BM7Pjz34tNbIu$}kBSc`1rpbX94%!~(-Qr2n^q#7h?ugL8W}08?^q!DyAQVV5=v$~LB-(;*_Q*d(Ptj`E8>jo6)PI}4-D{X+wb9&W7xvKfAZ4$t`!mY!|iXK3l z-8zeOeb(*hDNQI%&V8RKNA%tjom_9vt@D$8-c-z%?}4vAiwAcY0_CKGIad#OotynJ$9r?j#|$T&=qJ`2Mma{;(qr*X zO?90Nf0i-LAWCJ&vbj>A5OC4-{0x0KYZ9Ly)qa7zVDx@55P@U*Cm=`()F4ay)0gpm zTydOyF-cv_q70Tjfq?Q24}s-=Ww`&*L_AK6) zLvU^X+vX7!;I^>$cB7-fr=E-HAdSu6Dfkny7knjWH^Lkopr=a^f zuge=!>mFJbmYA>Gui93($|Q{8+HF&3Yy~I9^2Pj?SK<3N3=S)zzfi5~Z5NV*$;jZE z^dp_!jo0J0bKkb5_i@9kB*9yFa>Evj!GnLN9Y-q9z@67_rDbNR!)FbV7X{#@42wgw zsj*puW~$E2RBG#_Gjp4***={ANhRL9?E?QJ~Y~J-zK5{1w zO+#gUd!bQlkFrgXSE&%8NzWp z#ITjb0chr_BuuAAcR+}5{tePeZ5Z6SSG2fC%g#eKf=940+nd`f7X~WbZzw*fBbce9 znS2yS7qfqeAnrfOdSZKoUcNdOKerjJU|-3e=m#fkq^Aqy0MrL^JJ*$}tly}S7^GJ{ z2JhqeAO?7+twa16+rQ(Wn!l`^AoCO=`qEHbu;F({a6BQc51x{DKe2f@&QI779n9)e zns?Rg4z0SUZN+%$WQx*GM=qshvXkfiFr^@Qu1&fcANoM>(uV|h z(-DT%_SA3Cc5aO%Rn^`{fzhY|L3);IS*53zdxhC=?2Dqnlx0BpBd=*XR-l7vC_bH3 zgmcGW?|c|G!lEjc=rDlus?w>I1n)n`KJVT2eqTaP>rw$%anHiR!HAswenF5rMFqZV z8#W06^YEBeKN8NYlN10jNv$Qp3xd$u)_%z%RZx?|YYwQZp4rC7+pt_Zem1+aeLi#i z07%-TWnm3*>@+>98F8WWn#UH6ls^56x>x#C3Dp_H{}aXSd$ zr76blC-ssZh)42*eHMPxrO(jO+C^P)|rAGl9Dt_%N&t^xsv%`B>1Lk>nM9sXK zUn6g4xf~f>&mGA9&978fAE8JiBNwirx$G-R=JGic-Ha4Oo7}F&D+;&Pl>%?2JUJ-;tjjWTXINFoE)!^c{qqQ8a*#}Eo*Ai^ zQyQm9*OiXiPD@#(^$LP9G-th?A2H zoGxN=)F)!fYY}qgFkWkUeytN_E+(3Q!~*c1(%i%V(dbL|N#B1PitD(3)@Jg6qXcSn z_e-KF>d}K`NZK3=HGzE;crIBjEU$Q^pr0rSNa6|aS_yK@6w6MWqe{ZX z`|;A@3f&urbiyiqXCoV&d0v3YNil`$9Xy-`pvN~_td+8%dr(vBZSX*AO(b`>0rD(+ zKfIuXhrs$0XXhB{5t=Y`>%?R6<^Ut{QRc$pbm_Q`mIRuJVwh2;}iki6Q1{R|}ZUzU;UC?u~@64WZOBjqebNbaLG9v!cyYUe#JG;Ojf1t{QgUlwK@Ya;u2bU53vM0vWm zR=Hj2z}a{Y}P3p=OzSsE`kWFcb3K+*fy_;eQcxITr{qHx||Oz2i?Tf2!n zTeGLFy-QfJjNG;Ur2l$0qfVyTR=b%9T7i3={>rxdu{x>_33 z-!o=UB;{#IO^2iTq7yNZzI|)^oT>*KVsmPnhBhKaU_xf%mj-YKwk`-Ljx9$XPYB+0 zELNvDjElu~pW7?48UBYdp@hU-My`m%)VlUgXcdB zn+l^`J!lbUCFsu^%L{y&0zjUuKe~b+~j5hI*)VN1OXE#e>d2%o5XSQ?D&^yrQ*44cCB^e+iZH5O7;yKAcj{p6GIT9!l zURIR>Hy_`N`p0%FmTt@+-yqa_L$(`0)_iXF41kbvd4%ENk%ekI4;LY`!oh!Y0m==)1*r*bZdkubrTi7ovOEOAX)ogWN_~Wa<mGWl(u4>ZDEwDZqK|u5L0KI zv^(`d>@TZKxnN3$Ov28R=W`tEQ^z3QqwZMJYUq2qq`x#h8ezK3dJW{` zQrUWkj-aSG+_b%kk@>VU=M_U96aaO0&m@TLd*i7|2pdj#ioHbuw%;kXTd5`}`!~UH z>wz{Od?k0fu#(X~bWKK?9`9@1Bf0c}hz+*wWRpr%(JNq-q z;+DI9(6BJ=j>3Dfes;q>%4qJmes;^u!Ll_{MSO83q(u6a=i^~zN7&ecA7q&(>)8vY z;lm3vK_Nr+&DL_#j*-#|(KFD*#?%NqfM#RQ#?ClQl`GerM&uTEW+j1T^wZE5lWoJt zIofqTTGRe*A?cr{&c@QQ7K=?xiSYsFPnmd(EtUK0-U6R|d;6kpDY&Bap_EcP8af}{IQ%TKCDO@;9U}*TA)Ye`0 zHD;itn1lq&{AS8?gzac)X=-b2)#^5H zwE}P>n4J2|kbhI`+8Ahif70UL-iAlO_T9hVxg8}|F*+S)iwpUXROYkU{w=oA_XvG` z{l7qJT;>+$8KBcLlqF2D0N0eNw@z1aQ@_-#ue4et{tyryNi;k_fY>?(oB(2~; zD@pX&PdCZ*@)A-p-{$Tg5+@Ksj?{j-Z0!>xRFUi6G>@-1wem-nbtY1T7xRu_xdUlSF!kLJ=Eic-c2pcdtZ?TTr|Bj+<0`S753Yw1)5LC!smpG@oS9 zX@PoWcEtLN{+@7e=Yysp<-z;^L}>r#wScN`pbsw{@J73EH$d!{N($#DNU8%paVcWx zWWdu`fdEX?Iwt#*ke2k60Fx9M0XP>wanf@fw>?o6JWOE{NU%s?`oD}z>B-lb-ovOSRE@Qru}UnhT%JIiFs;k%?RAwKcblM zJ@_mNbg_a^>w3l$mIJA!T+aa+{9}J45*rm8Qe0ooKE3f@q8A-4FBTn>z{pV{CRS>D zQK(Yj7Zl*Rm1~#IW44VDD3H4t9%qUGmO{9B`I`wBIb|rU&)H^mnB)H9=zr8`Iu9h0 zJ@a|W48x1`)HPE%t&h}0EW$b1a42zh z8<>Z!^bcmKe(W=U$smGeqZ!9)i$Pp@=O*ry44Hi z!c?ksbloutpTEyjS;{piXz>+jV4!@0Yp8FVWH~XPcI@-I+ikG*!y|$*5T|NFjTYB- zrX%g+(6`cl6)!CTJU6%ZE48E8A|f-$nAdZ!RaL$)E4{)yOX}6@wC0_BE{XA98cRoK z^jQia*AzD$XE?CBYE%Y61u`^554XS9$Z)ej8X0QFYetGvlw#|aXj*+O{;bQiEQL40 zlEi;c7f6he|NbyurgA=oVRCBjE>-^aI_XYdz?2^X8OoGdwW*xY%EYKpMjSM%%{hI$Tg3-PzoK(L&SU~de()zJwS0ibYOLl^V$7IKjO|7ox z9{wErqp}>CGB2UTsE?_b+pZrIp0P0jRWJ*sTV3p)FO=)NC0)SDUQJK4n{e~Lk>aZM z9x|5w@)-E|#-4g6SYlxWeg)O3r6*p=icgU8 z!@D)+g-@86uOStcAEMx&VBP0)_rya|C$l>woDYEXk(~6nk|kmwpj#qQLN!nk{ay1w zKYZ6SlJllk(RMD5sobr4F1Y^$_~b zh|M!_#5q`sMogZ0xslm-%Q4|v`S?-wKw-aQ;D zwO=(7?d-tw4RVI=Jv15c50$edKTTM-55wd7Vyx3NU#|mAidHVzWD{r-`^*bVzXRa2 z*oLh3&CvvRc0eE&iv~yDoUOOSD7gWgH8BdBX{m0m6V%%5fQiq~F|wRYw2a3SR44`w zZx{~22L6=!lTFu@#X3)qAi?+s3D4fWIs3Y^C>@&`mjU9-ERz4!0$}I*-Q-yqgAXS{ zUb;LD{{(hVmk5@kow{O$;m)akNQW-h?O{`yEu7iV@vzmwDX!LXjb`r&o zmh!)43#Hax#HM#yUDFWvFUeu zQWu&XE)Osbo}C8JOR zj;WcN-_ z`!l#+r`$Wa{4rc`^E@rxl?U9~v=1=Yj%SnvZP3_XL0qv-`(cgELs0*T8&@8@-s2 z(`KdI)9_uP(jlvhp{Nu2_$l|J|J=ljn+LCfA*pI`Lvg_JqCMa^T#0Tj2hG^3g*!<>iHSVNmpp2HcE`)@^nGNo_Qg{*kgKsCB$txZEK!_ztCbY037$v zY-@)h2t$9NDYx#y@%4Om&KL+7+?^}}oV#-H5Y7m^7z6NllA^7&)PxMEQktnYRv86s zGOfL#a>H?Pyw7R{s)C;oDaW~uZ?@>Y2AbiLHQ(s()t*5ETYqbO(?bv6>SPyL_Pf=h zeH-IgMpt$00B-tM<;TaMz?)5FK~58F$^a=l$NL0lC@kG%<6_QwW(48!D3*)o;L^|01og*Z?<1ple8`kcK(;9X02HBljAIkzI@3+pKJQd&t6Lk}_k4Ka5Jb{Q(-^Nq zKoelbku3zxaQvpg=ZvioCL2KIaXtq6Lu$#4Tk&k|@=k8da%~EP^c8ou{9@&eo=mP< zD=o%7wJdn9_KAkbs?duj{AIpuU5JgKnv|~;-XxEqAMD8RHf1RjdY#<{TdtFV$B9Os z#CtQ!Vh?%q6KsHK{k0ZTveZvAT+s^hroZ2jTx>;!^uRN+1T(2bN4&vQK^*K4n;!i} zJ{)_hP4&)>D2e2K=_gvQxv@R{7So{CFy|4RZ)9@zB^W2 z^8q>B1a~oUfqT!NU!3`i--AOQvZ;gAoIuF&*WyoI`LgO;?n&)#NIQjv%h2AGwx|0G zUwiw-qYOERstbz1uIG;W&0$ZiFjdUDW@oJGvXdK@6*ZN;qYz1zu24$Z1n^fFb;xI_ z)nE!D1_npt%%LhypmHb8x3$jD<-5Zu=HU^m!-E+F<%c1vOB#Pcm#0G@WT7Qq0~OQs zfJ(|K-9wn25PI6*U)2oTPl(UXaNVkJu^_ePlbD)v(+UP+F7QcE#m*>RN2cuYP4P+O zxXl?)V1$T&)imeDqb6Ihg~4xaPYn3Jc&P7i{Mh0&M$XxKHR}(9=e1}99wNPC{tG$i z7q9Gk=!@u}o!${7(xeC@Ry&`8+Mc>YTqC1|geT^fDbHfu=`wt!y zod&$=D_MR~k^RLb9*HXQwsXxdkb1)Uj=wd12Yfyt0EUYLqTMdl+@jO?_L&)b)&Xbf z=Utk)f;qI({kE#~&^!RR=< zn^(TfEQ5PxZx>WNj6-rZ+r+({7P?ETz~ATW9ufn9v%MeG_0eD3nZ1NtmQQPk^g-*L z-elnSB27LtED{0FaEb90WIXxBgoGp&TDStlV(eBz7q5Vu8Z9**7e&}H0)$yDD9Vvt zg;Pam#~u^sJFpXVE#rjZA{0>BRx;V*;*TMXo9j9?#KARiqQnx&<7~R@Ev42{Nl7UR z?K)ILTQhoWV|_U$CF5aKYN5uCV5g8_GytZ;Ix-^SaP#8Zbvv|WV5^!TCmuRANTzr_$!63*J)3j}g za0hT>EPVbP;I0pD35H>jk2cOYklzC<#)s|8R16my{4k6YQLC0z?98SPhp8ljsl1lQ zOG{vfMX8xGeSJxbWjaPMjS2F>*_pC8c%^^iU(b*5rAn}P-4xq7y{v2RCr2iI*j7m6 z+VuSg4LFPW(u3M+1bmK}Co_Cv^ zbku7;&zNhuf+$!zvcp%OQc#eFHRIIrtyX>z@g9k3104UxR|HIrC@DsNb`nSW(HxW@ zMEr3xEKHk&$^F-hEQC{k2?XT8c%2^M+7FRKmZJ3>KNGsVIOU9yR{XDDiL-uAH3=fy z{IMwe2JUPD7t^2aAAP)*wAnKztU#;7Bra{l-Ghs2AN0w+qdrCD25e-^(S5f-5Dxie z$(g~QSwJnp%*EMHy=&R?s}R9ctdME^)OD8X?!j_)TB41IqZz23mIgOEjFEBVV_^#? zu6GM>2k+07A`OMJsEc!lJ0OnZl!Abn*bcjqDLWrAK_~Ouu-j$mwnjP3nI#j0sXEUO9ggUw)*)yRx<+ArN~dS>!vx z>({ z$l&jjC9AK&Msi1%LEOF&HtX2J#)$nNI~HNm%8@0L)Im{_;gM#QOS@1QH`Q5y15{*H z7a@=uC$jm>0vHEpnDlNQG7?;R^weKU++EQJSoBzJYXm^eplM?WaI5!&oh{$z>Cw_f zcg>6^>Q~ZS=Z3@3-c{+iI4b6pqKfIE_c|RFfW~Kw1iA=(ECKZmZ zX)gTD?aAk!?n&Anw+RA;pcn|wPb(*h3v1>LoBN$s!HaTRa7$Nz8Qc=Q|KSdYY8Xy? zZqr_odni{(_inYvbp*+WFCJesR1XJY6Cd}(C%)ykpG~K{@>JUS&Q`e|8uFN}W_;(e z(&9b9LE7us=y~rZ7MlOmZynt5g-4@mw@$Ch{rY$8d_7%C)o!nV#Tg<>Q|J3ho z;>Ugvw?EO&vBy7nT`%z=K545zrJ>_cb5Xd-h6}z-Spou%d8&4f))LD~lco27CN8i{ zdf6{^4nC;w=5Ni>8ZH=B>EIfE(ve{Ra@v_Q;d#6}j7f-fSYhKLAM6!`7oYjdxE)cZtcS@r0-Y=!M zROYRN3++)^2dpS@;>BHat#FO`T>GmeW8dHP7QUL=8|4$RSr$&?velz2|6Ak%ZTez5`_Q|Cv~An?-Q8$a z8IKF4pErD`jkzPvwt@Qo$43GrW($oLs2fQGSwnYp zg5ovI&a#lFB{!-nzH?aHA7EeE4s!24g7ZNa(db`OZUx7UcofyVnFwxQa_L7rLA$mIlQ9m;=uUnrZ8RD z>F2`z=1GV}mss6Gtv$op&=7#fQ8?YuhYAS3W`)((9WI7ku91iYotg%pg@o*lKv%7n z76s-hRg-^xfV&)yM(v`6#S1UR1q9f0_@efqQk?-$ehFSmX|8aoPMVhBn9gfh{B|)h zV98tKG?ryyY|w6t=kv5vCDY&kLHz@M*zd)}wLPnifD`xI#9$e2QcI0pbo^GY9x}YH z-BD$@ZK2~8O{jK)hR_sCwti7Ntcyv64nn41zj0>TWn2DH-n&t;S5|- zG#YMb)^%7>Gycm|aNn5sArv)6O|{aXwW!Ox8D6|(GN1un3s@dFDBtb4MKQzM_ycY$ zANO;%kC%FqMLhnj)m2jW2y(ilU_UcP?dazb4 zo08HkY$9Y_z-O&b1-fRUlO^SI0$<`sA-~Yb5l$qb>{>)S${B(=OTiZg?$D#IlPkOz z_2xOzs;0g3O1iDvODHBw9o`&P5%1#KEGaM6{8~TJJA}YfTdabuId;%{^s+Ho6)%Cq zbWSS|Yev5S=-Fh5jr&$RbjHg5uFZJ2<4Shr@lQVt&z~E*Em2)6OTm3P)yL3>a`Af4 zpF;&D0lpS+|J_Y{1y(cj8rstNDAx_#q-u}(# zB;pDdzXV-W)N%Y(@T=XfV0hUn5Ma-0|H@ zPb}ISaHmuDi4$hB&8YZD23+XJcrE}Bd%a75A`!x4N0+G71tzkxW!G%OV>|Gg%Y0@o z(9FQs-TffD%yB_gb)K59>9d@$4NM%HWv@Ap@o=>JGdZd6rQeUzn|?d)t7p&@vF864sPfk=;3xZ<{9yK+(EYV3XUy3q3FKT&>+YBKWw z3REYHp^Ek5d2F1;jno_;_Z0#AFH~J(v_8 zZ>UoKw&nkGCZBGO{498wm5tlLvA2I9N|gG)2Ma$>;v8_V{`T=fq`8+-dma8y{HPND zR}&Vdb?P6CBL@!QdJ^$}wcrX1dd0w$l93^==co6726(HM-Wq;*rX3YOT>AO_0R4Zt z%zsXQ?dY--^Ov zNia%?-A(F8sNaA5E654sFO`UH(q--#6(L6%}8geCqA*rCc)LmMSH(sxsS%qQR<{;c@oE>O5au#Iz0Jpjc`)G@=5Cde{ZC%@50o>?TXw!Uy3d!YW#23r)W${ zQX+CinoT>^0>lDekxJ81sw#fQK-5@?Rc(FF`W5iMqTuAKwRDtjC7P`g2&9uq8#fO2 z+^`^F4%zm86h=%RE#^(ae{~|_r@w>9yF%Xz%CQR2Yf1DOKaWb$|35+u%WmIOInhg5a_0jSK%0-K9u|09gpLt7H z1av9A70c=Y|IR_k8`MD-8z?k7mF(@&Q;q+35+X_a@W%{K7?mfWav(X~h17(0)=)%El~&uuwZ$D5JrR(NC96~3L~dlpP%T$TNr z_NTPm5jlH%3)Fnz*yyX~^n3js6a#ps>Yr5Frz{r~J3Cup=He_4i8CtnaAQIeT-%I=oIz!u%uB6m7OTa&-R3k|6!MAKS6T}{XlcB~{lp4VLxU$a z@bQc0AM>y`3JBR3@c)rmXsM5mCLJD$EvaNZS*J%mOd!Lv`c@t-ZV71C`4U&xA~$%?)FCCxpGKF+D%(zny4 zc@*)PT-k~f{&QN^`w#mwIMn57E1i;{KU?s!>#dy7Ol%L~ciV(%GIQZf-BDFkG&8fW zw@8{KB$50X@NJ@)BC)CZT|!_K+7+XicI>|-!xYq!HR#Y}MI2IySXW3HwAQbpMC%)E z1dwp+^=W4nXgN?cnlUt-pBLRHaEAO>b;TbY%d;%G_T%pUUM+g`HC-!g0WbMA^=%I@ zgLelVzNuJXOrq4|mnU&3rih5h%NV)Ej49b~0qk7`^2$4$agh7>a?_943^^HX!9Tn$ zYO~P#!|~+Rx(SbZ=S`kf{!FF(rwk+|SU|W~8LJAAfiy3|t@k77;UzoS@Scn_Zt;DV z-E8E|lN$-*>Ivl4!kf6`sBSaR-+&}Ftd&`Knk7D!C1|5NzoQMrdp z4Zo&pn?5U#Z0XBcxmt79ABfwnIIBs_Ja1{j?>?dW|K#dR^2 z13tf@QrTO-vS6)kuBS(Wd5)c9%TPsbH@n`FBvcME&Siv zt27^}^bD9;I5FL1n1J(5$eWu4G4V?D+$dP3)RdQM6iMd>Jq#dbTf7;Lt;x2xT)7&VFvRoAESNTcxi z5nA?KOz-{ZGg>YEV9111K%NvPx5qD!mVjWCb`AQx?_Z1-9B#c=1KJ)Y&9OB>6L|)G z)owfC9=@;IgJ+(rX%UH?v+DX(XAmvn)P93RD^u7|CK~z(>h$yf1TOVdhM5yPEsn7* zu`)aMAm^C6^LI_###J;s&DrT}UReyYF#O~THPyo^YO8%OC*x@N4Ci5EsAZ_!jiC4( zG6mncSsBBK_e`;01%UoqLH#I$h=c@}BwgMy@EOCq2*}`K_fnMv%`kT>J3P@&@9bZH zyjEEoI2&?K*cTuaDO@|QFy~;~*U!N1x&1sYTe!jb9+5Yl%lvVVCY-xAC9!hnnusF3 zb0YZG^$3g;zXxxus`u{#ZM2e}i3P;XP5i?S-{6}CE1L)R0)}xsSv?hz4Pqp^?r+Oe z$EZ%_?)^+OA@Zu@h(*fHfc0-@$R2g@C)(%QAM@3bmKdaa()M0^|u>p9}~OHNr-f#6S0d!fUb8?g^fEi3E#tu8S%!Y9-fn; zwVqIXA!Ta9~b z&Pi37LlQY(J~hbQ@E=A)1eEA{Tpk4Sl}t8F>gF~MJ__edDioNwBH1Agth0{n9fqvI z1wW=f)PmG)DxXpp{H46lrLp|G?EGtg<4gKGz$$9-%BNOargzzr7O+|u*U6k?oyUaf zpu{@&g;@eFVXJ-BiLzk1UkTu9|DEv!cGH>uIR_4<0}OBHuM@LH4^Z zT-uPXtYAJLLkp{8uyt)AR^^)P34M&k?c@qc^2IOuz?vnD@FefaoKfMmk#0SYn6^9T zJC_@=iLsSWRnAOob`~_71D^!XHl@~fKEisp9EK-j$Sx#9fVaSq_d>gG<%~P4Ih;kM#{q+f8l!D5Fe3W8^YcJj1-F=UnZ4i97aY9gNUH6SLYWWs!W{Q{ZAh0$Or&lHFs_w%} zuEP^)ozH#lcj1qyWgA^Q^IeU_$<)5rKDv`K7L2_P_=b;eN ztozh~>TZ6zzA6B8n^ZvtTmxRssR^ebxm0bXZr*7GM=TgLIXO$BVmqnIA9tA)1K`0_ zj8Cd+v`%k{cAQRMi<8o*a&uB@plVLMz$Q&_i6a@oaNyRWTC_!Da-Cv+769(Kw_+*M zhT1dwo-xCPn|neMVc+h(FMjrb+ijQfq1y@mHhI)m&#pllqq%;X7KT2<}a||u} z$ClluXwN(RGDCvWBiNjAdj^chx0mqGBp>)GR0}+m=pH73GMmX>Ih@iK^U>uU^}8W3GlJdBCgUtScWu0yT2y4S`PKJdoC%$b)%WNs1@=DjM3AKGk?Moa>t zFEO0gyIZYmLmuALl7NRouP_er1+prN%*1k5bsn=i5>g$5pfrSpOVI~XF~NI}fR~M| z+Uy6SGanc4{2N{+STE*G$**3$ba`}Qf$jHoUR0WndVzcQgTvL{eh3{_T>fd0;cX_& zX8a(=<{hA*AMo)41&iKklKPsm)kh*^I7ZUIm98dKjwZB9a)Zn@GBunt^2LH zHm8X)+{75djecZ2X(6lZnC(S?OLZJ%A@O`uNJMu_EpLvDgx9|NVboRfM2jo`KeYg3 zWor;SI)xs^#ZD#R3H{;7^4eLZ5q<{b5E$GL?iGUrqu~pv@A1A|v*l*0_~l@V7m=|x zr@`pYB$CrQ6Uk*|0W3C!y_N35Vv^IovczhJVnT;=?tbO$e*OTKr>vZw&XKB@?dF19 zkHH+3Hy)YdM@(^CfnRZiA|&0k{&J&5fxyS}mw(m_e4Ye;Pnjcu#-wX7Ih}U+`_3 z@LRr$8dzi}Ht2rQAE%6T^8{H5Fx2#Vyg8d&-r*gzdT*ipyrJ`4n=Td9k!LimQfH_G z`o&(+5NqV=P|@Hh1QG<*#ui$Y>1ExHRJTq0>z$>&U@%4RQ~633L00)`-H%DaS*pPeZI`%Br7sHl$XeOy$BeHz8YdYE3ny> zrFkGNTgy=qUMPQBV$HZA^)a+F8(Yq7%dthONZNpk_xJcDyrNT1Bo=E-*@qZDF`fjb zeVoa&=>*{DNr<5jOcIUy|5o|nBpOFxY_{@!@^>h}E5!T`d~M#-c?u5fHt->@>Ep6I z0164+8kZM)_}S?ou(=#YO52$JN(~M?tEgB=Aca@;P(A02j4Ych<^P@f+JMm3^IG~W z7Tr!$6MBGh{EU#PO&qW4SjJ1)rIbJ|kJT|MU^z!iQ)e-+E^!Wnzn1qGGizExl; zL_=vnWF^kq0nG32PrS2ElDnDTxZ?H>0T$cX`%!gW8&f|od?qen-#FT!L!kN60|{fG z0+fP@uQr5K5}heEEuV6}HmBC1m|&@jNz&{hZ;E#lTkQ8m2p_3bIg_Mfb)m~jPaus`+;tqe-*wi7&kvTmXewNkSqsX=wyY|gY`5D>%lU1+ zILHykF}gdnd^1)eGuNSB@$FoO$Y_Y|3nPBwU*1&=&Ib7JW^yRN9oi8*jr zh?4p;-R$nA+5P1Z=Fhha`-BM&Fg!rZ*Tlyqo`L%I%J$TV=7J-mv9l?S=8$vY@4>qL zNuP5usl+lV?jknwQ4D6iI~JSbzvoUh(Kph_?z)2Q1sr_7;(ColxHa#Pw_xUWR?ws3 zl|R>M&ZK{O2*0`fV0-DoJ7sn0w-%(3CMXdcPsVqR{cU%iNZ;3Ge-q`&LSrfB6?1QA zMvc#xhWhzrM?=Q}r{YJ(F>-I|->=Y;O?LKM5x zw<;7QEiYb%-^k>Hb8M^%k}|5JerF`*YCS_Ys8f7B@V<3<2JVd$ zEcU(LS*W8*=HGAaMM*sLnLv4Dt#=xI-|gr08L(%k1ryN$ybE zZY_wcnj=RHvLO9Q)Tw%2DPai&hbz1jTuKukZ}eB&AJ6SOMyb!k3^A4LAL;ZjjZ-A~ zCg8qQM^oz?hF6`G()3B(i;(9NDzDorFI}s|c)kmSfk}XjK%F!`Nru$3K>Hx3jS)Ra z1$*mbS2zs9Kt%tE4(tD8?Ja}jSh8(l$s&twG0PUSEM~HpS+bbPVy12}Sj@~|F*7q* zOcpaUGqt`xb7$@x-#hP%coFYUMMZa3W=3V@UVH7#T-zt(=6t4UAtH{8Qs#vIaDdQP zzl|>XgX%R59%OfS_(3jgms3f=Nka>*-a5Q!|3tFrkOB5gN_bL?NLvImf!C5 zFkV+@`R2iycSSD6afwR7H$Ua|;SYxf@m=KYG$eR%P<`YP_XdBj+UW1-b6%`D5>QWl z;5Ejph7w*R=nOf3hTEz=Ro(KkL*sQqiK}$Ix~(bX(q2r`GUMwaB#o?P9)fRC2)EMT z9pU*>??IiAMM+B25Zj+d1|)MY6$x4CeR95`NqVRagrUl3X`fAf2tI z)b;+%M$CK5RtSxqZ}6n_)8zNn8CqzKj82uqtIuEwb%^@lc)!(hhKK7pKLr%DVyWDb z7IcG7e+uuJegei4aLQX#o?~R6p-6eZ^ME#Z^57Tv#;_k+%yNj^kzq8}pf;U*x+T6@ zI1#IaBht9eH-YI77u~sStP`+T6Qb=btS;x>-Xt8#l8ro}p7rJF=89C{@C5|}6C=M* z8RFpvMpj?1g;=Y9cwe{Q{Z#vm*c~c_IxW@RfRYO85H0rCuxnyWDekX6@d6f_D+v7W zpaQldg{P=Ap|>qrK;0=$f>7p`T)jO~ocOxnV&qRNIWCTr)0%?xb@RRJ zxKqpI7R&VHer?77@+iMISDFF1U7H~&J{f%As{PT<9vF4kU(NE9xccXL?NzDvyPz>^ z-ZOMZqSaRX;%JiIz~yeznCzXlL?P-Y2ngE1 zy&5cfJqH_*WYkoBjJs=F-+1a7UXWtcrJbz(z?!-#v#kyuN`2;m0PxrHz4%A2!_O3e zxY5)TW2Ngt!;_`7Uql{t4QnE(U%ra6A2z%v>4jtdA|u|C$Dxknydg!(#&*87?Whu~ zvlk>tu9=YvVbVJm#IFeWfaVKDz9h#T4*K3DD&>GD-BE97od^4I7L=iYfvL+|RSj&t zAK>eO+f7=}p|kQ2{h0Nk&fxJ3P0 zK&}c*J40T}4MvGySSMWIOyF|oHKTWB=U-s-dsRpKd7awT6M-!>ln%S@{rC7e)T|Pg z1sd;75wirSm*)?BFCL*Tlm=3oW3UieUi*eB9XBWQal@p<(Khz)a0$4L3*MrIpefV3 zbb2n&QJlWnivjEP5UgA1gaieGbk`B*LryW$>B;Z&C@1K#jmADzUcvgk0yo_dE|>F~ z-|o0%l)nz_m;R!gpCoS~?hXv*j6F_SVIe=Kd}}dqf}d2_2-pB|pv9?5eH~!hnkR%3 zT-Wh082K@0Sjv0x97V8d%1>&vtQ7)-wW%S3=}FQly+e zL2{dWwLBuh;0`6zKo-JCGday#DkMNXOqo15!NO5q^_RGXevh?u;=^U9MyfCI>m9$Y z5LIQ%(6foWbMmsk-#wxQ4!uq=g>suOW^a+u9t-Fys)t`ruE03c&-r~t`fILH(0Zrs z&VOW%f`LxSrdK6Qk zDdRl&KF{nMher^HJ#f_(!cO64+bc4e_ER=7uthDpc^`*e9i~Cpo&fKOlW+O1s$Qa1 z+Y_>6+^zw^?-di{RIexO*)mg|kn*_BUstF@@U@4=jKv;B8yXFhFNo^cW%|nHLnF}z z@h@aqpq#t)vx{a))8w$H7QV}PkuwM{a;%zAXK;8H^Tp-`X8`qBvZ^!Ak5ucC+Xh|y zdno9)LF46QR^gPgj+EvKKEZHnT_O|uzg!-$^W&yxy(#J_ie29RxPt#X=^m+2v-s2Y zgME+3O>RZ7Cl7PlSn?P}PfvLA*dAgijf|-yx+nYTK89Ic;LT~4V^IMoRDI^fvEi3@ z3n;b6`VEEIJtm~Ud)=Dz5Rz$**qC`Iv#}OdC1$IYxmiJ#EQNQ&hB)TX;=?FgquXw# zUj<;r@D@$*MMOT?F{kXru?s$rz?MxLvZlY`eb4pK893M$%?D>_D3aLCt)sC6g)cII zr?l;{=xvTL6^qv8Hei0ODf1ZEJ%fO*5*)O3M=h41^4>-v-z=wTQ*$NBQsL)oz|+sH z@42c}aw=UAonddd$&q=y+CO@Jz8ldv*b}bjLJb7IpZ(}%2i-J{?ZqowAC~A0Ips2n zCj1X?GuKFLPG@qwv+DK?U(`PQc55Vqb6lbLO87hv(hIZ`diA=#^L*Jko#w#3gdg(JeQ^$9e4Kjadp~5ka06_ z$pJ5~*){Lf>nR6zxrNE`ZdwqwwiBxi`k=++yfU?%up&>Ey5^txMDgOyE^gsUsc~3? zhm&uhC4ZH9$pEb`4E-~hY zi$jA;77b-ZZT8e5yw%au61<(TiS_5q0aRk-eSRo2mFnB@$zm)@&=bVaUpI0OEUo8 z64k}3I_tj)Nx+A#>A6;03`sbx4r57a1nxGWa#RGelIz)``tOkh&ma9N&F}N)Q9>x_ zM&Eb!1Y*l&1C!!^ELu^}(UlF&e@T-CG+UsLBr#et7myE^xqpM?gAF%;G_P*R73>CY zAuXR*6c+bcn~CyAz;6h3)H>P4vSJV{6(%O|8IJ+W?yyI0LcxcHlf?Rpn1AMB^A+Y2 zD?=an=q`&9Ge;Hfa~o{zkg8RhpQsr&wa#*^>QWF}LsOK}4%78ygm4c~_8CGh`YnQs z98UN<_z;}k;?3rR)kCIt#@R+zZvtM4EkXEljK7jM+gS~g3Y@fNW1XZI*3ms`UQvdz z((`7AC)pW|!o!0C;S%9|)_F;zFy`%9s z^s!iULd@KWNKPpyw(_G0N#&{N%caqNB9~$?8YT|{aB0p_2P}2&UnaC&TFj$EE8kb^ zZo-@9Y=D%>n#5DUIrR1^Z>__Oinczw)gAX7n6Ye!Mrrq6IH#A=1&sVi%{I_r-%H0; zoY(Cv6mDYmm#AoB3kvy|qf&fbc;h@E?DS=(w-A*>CR5PvN(t1$iVY?U6#UGDvSWK2 zpzWrDxD>!H^2+2n1U^cVf`%7t%X4l8*XP;q0)V1J7sJNrhHa>67oU!E=z`T(h{i;^ zEz5LH+R@}PQ|88Y`4^Mx6wXJ^z|w8%BCe(c_mKP~QODbCp@7@j=;!swW0A?^;pulY z(uiIInV2fca=8TVd^tjwJD8cz6qDNQkr$bwI}}+~9<6R^j< z$m&RFuC~a;Ub!lNkP(OL{mL|Vb}O^Toq>sw!LDovpx1NCny9mwq59U5N*Xxc^6K@= z=MqeMyH>Hy1~elZcb@B!jZN||!OQ^9Y5ZYgF2Xqwru|uLwmfV8NDl7~VK^O+`*D_6 z?Qwed>WlU+W_-dMMGdLn^ifU$Ht+v#06IyQs&EV|X^hsrXLzss zlD;HJiAH7e4GoRo-6^ErWR>F|CpOuHEa6iyHouENcQ3rWpZQ?gmR=TD$W3N(dPOc7 z5&}qyUoa0}H2z*AxZUoW^t#m(uGxS-l)o?RrcMZDiznN+hnc^81Y;ht(r|%Hgu+pM zTnkQ}lZb)m^jMjzD-@i;tTm>VLDXIuxdfx=6#D^V%0H&!X|OFQ<>FLJ zNJ=~fg1+;x)D)tgXEj`g?UxX!W?L9wBXfMf@#ATMJKN&@O)H;m-W#UkY(gITf?fmy zzVCPE$WcSZlTwW*oL$z-rczZTbmXgm6yS(M7uAFel@u#^Zq44o7{;QrediXLkhg92 z+0PG|&h64U6tNr{F1Q99aHaDCg}^eQj_D|H6yFv$Py9tUQ!~Gmjc(ZW*ayj__qIpM zFSK!l06eMYWzt6;O6jcRT>Q?P?b7O@o%3V#=@(Sm4`xU|&~13nIu+>X_3TBl87|MN^es_xh}vcL_#GiaUa>2Y1H1iWNTfE)Hj>?0bT*CvSlcHS2?3Uf%~VIoj5 zwWfRv>a?597gs!c@KdI|R?r1`K1C4+Go0$}u!5y$lJ>WMqjXJ7MD^5ypaaCS)f$X} zgm8DUaTtYOGAXeVK=vmd^c>-RLs$z&j}6YRG%Q6Nea6_i!LvuisL^RMZR$WyG5Tp7l6j*_Q})hC2^Q&>ISuJ zL*h)*;Z#;zuaoaQtgc7KM&r|z|5NS+SCIT$8t#g$Fye6Bl8mR0=j> zQcH5(m7pO~N4yDf;V8_}*%VR-$%ibANd;w$qS;(UIJ6ZN?O(SqptYQx3cUd@N0`Uv z!Ro~bl_lmhQ2Nyo@ZLE(mn5tZ@6|A)wn^f)`K$E&XY{7q7QwlSWhv7Tbb!b7+mhD< z#yQ10?(pQ4cmBKyX4zV*-&iF$_nkQvw_$-&D9gV?n|+nZpkYbg$5EAs7kd}ot$thx zRaMmhysuNw$B&!Aq)lmhsB5&LVP)l-TijHAf^&1yrJPWLItuzfzm?|3srg@C*>SFd zW-x?PPo*=b@kSlfTO0DvJ}^_7))tS2vOrKtY=p^L^o>V;#ec=IZO0vk zqvfb!AxE}Mh}m=%7>V1?`|_c?tK!WDln6+Ix}GvF!(oC(*(>-g*05*1t#BODW)B^s z*=0hqYfgmH6%!{T1sR}6By9DA=;}T?*}7jbR%Y0qhxBRiOIs-s0X5l+zC}lE@Ides z)YHZ9DjOV#VZ?=G;NI4%VElJjQK!prWDXM)4(n=~fZP|lSvhc)2?w`rYNX<_z~3mI2M$n zO->4+Z8emef@Vq;j${3M{qZH=Sp#fd#J?ofrBHz^kD6@4H^Ztd*Usr8!3W8XL~GhscpeMv+sy)r=~ z?co4O*(irMG77`C8am9f*l7TthN?ArWBq-ipc0CHyDzc9x6)g)>tQ{DWP9YAeI5fV za4e;}Zh4gMZMarM!9!fU?u1k60uDXA?TPi&*d%yCa+E*0s#VUON$S~esph2z<_$Nuy0tQ}5OoeCy) z)E@?SQU@6j#n~B6dNZ}PRq>`#h)6w#-|gvHcB>_K$N1yk;PGc%rr3V)t_j!}@;30g z7zCudg&(xyg?R=)yu$R!B@fPme4(Ftz%h{Le9Q#)S1ztPnMv?$GSywd2K9|@;8XAWy&w0kZK%ZfPF&B5NBJ)tosb(Z?xr9OdxuFB5k6tTeB?4_0dVpNXX?1R2yI>AagKzhDRzM{`IFK zTZ?Sz+*aYd5iKq0zMI@2a!KI#B-lz-ZS(exfon}+LotY?wJQpU&}v|1@!T=6h*v9Q zX)r3nY`ot5{oQ%MQu;zt5I3l zF0t`$=&P69ABVOgVHr2b@+{}GgvkUxbHDnM1U4VjcwGTYTiDD%keCC3NAA}`=lp-| z1;}~`z?>oFtxRbST?9wBs-YFWH*Yr_Yy4J6Y^uUiHFPY6hrj!reV138bamKZoJip6 zhQJE&(_r1zJbMpaLXaGKnz^6w{p-8qQ+2P@eF2^xZ0j%GN0DCyvr!QopDWXJSG}n! zzL{-Hr>8$)%~qhjKlSxABs&}T(G1%j)SsJD6)GZ zt}205GJ#ms;mE4$uR_(a7#8ZQ;ym%S{tg>I*mj()^d}RQBd3>i;f++I7N4AzRV2u*Tecbb;qbB<`Yqv)}bB zgC3Yt4cwLMIbFPG1cqi8Wz}cN?=#@Q^-Ti>>D-HXE>Blrd_t$+Jw+xg z>x$yzyLZZ-6=H+oWPgD@M|my@B1crljs~9fSm+(C7{1CJ|OGtns>@L5&OLmCK# z9Sc_Y(mv0-BQVZYUXGfv-#r_jBD|~Tka7uNY1rHEb59{F0O$5P=r!a$-$M7vtG2Wr9BDBV@5NqNn-H)!SEC3`4*@)5o_=Loeok z*G9RT%v|e!EXbg>d~oI}YC(QB;-@ePg24+V4W3Nzbnrb!8dhB%FMn^u<$LsszRxk> z^Fp8^ji=^AegAXLMnosg=+!j-*pd{ab%fYos@JW6clo*KD%z? zGIMHZ|+-R8d; ztt2EZJo`#IvY)|?aMR3EKVxrhoTQ{e+xg;6O-%AX5)cfJ8Bwx5wc1)IMQnXNCVx(= z+OqA;(l~YEq@tlFe>G;{VRb%6t&@g}=pDkAi%%J&kV_dG!?G3Rah~{aI>6WUqel^j zoN%6hZMmcsys*lKwsoJ!4cOb_fdFG_xoVCKn+|USAv6p=gL&op@4Et;DG_0zZdgCz zuV9uCFF#Afk}~>zZQ?XrF-u+ zW^9rgTB#2mdv>&V6TQZZb!>k9O{8o-xZC<~zc8T5_%$2_?=-ue$NWW`a8U?~cZ7NV zTt&xoM)wq8Dh`e?&!(c_g_^Jsm)zaL3l`5-lY4O-(=OSD#669qfdj)GD|I3H=V*@% z)G=VKoCd2``MBrQF^YkR_Mc+|BujeE^W#m=PC_XO=chSN5SX(ZNQ1Yn`WWk!c~8u6 zf$b)qEuOBTDH=W`ov*DC@9n8A!`?-_iFVcr{(UJ6yhuM5^Dbh~={L@I+v(DF>dHHc zYw#Cz6v`Tla&|i`YYO&T`)p|YH54VAS3~6TazSuQa}2GczdX+wj2^E{u&r3q#{S~L z;9Q7-h*_46_91sUzZgZQn$c(ZgcWwMJUls1%Mki~-B{r?8`Z(%g^|ciJ^r_^RL+*~?GFFneLAvT}*X#iALT1thTZd%f7INvuqANE}Od=`@(7f>3~%)y4DAHAOUjDrPrNhhPkZ}1FcuQMVR)Q+Jrm&7f0U9OE@7d^-IK&5cPnrMJ%kIM}a5V_t zZ-VdM`SDco_Rh5DRb61n1;6^8+BD)nFS3}ka)Z-%rJ5)#>9J*=5#H6B*|n|`#@{vy zm8*Q`%+T}Ur+V)VG^0wyBKzMBh|m}I%GoP+GzQy4HZHFoTv2XK=l_0wP*(aTb(LeR zZT9nzwd;S)hTL*$o?@P43GcxiUU4$at&X_8v4DwG}~T>t zQ}a8{EpLDPTc7{c!H3z&LjXLpQ~w|bgwJp&b7=YF!YL;5*6+PsW+~|~MGKzItJ~Xw zZ-M`z8(z*%+$t@05TlgLczq=kEvoU{rHV}%=?y!O!?2L6Jv}^z zqyM4#51lk-!_xQ|X1|qaz=?)hl(f&eoQvB-nLRNRA`xojvtad%{4;92WI&?+hrr!W za%qlUQ5bF3&q^>qZaz{>B}i5_);En>jIkV^WgEeG;Gi4Z2#-R_J4e8IrA}(7+51( z9b(%)m%ETS@VfNp1C{^X*8jPgWfpJCm-Kaa@8UhLK~TS=AEY|VWsX4CUeb>d<}X({ zX1GB2u?8B--N42C6V1W#{Fx>%4T`DAx}>PR80;*gVST9dmEKTn>>B2&qs5{*mJL+z z?LauO4@5=1F|{$+M4lnml&AX*e4MFv8415Db$0{3tv?sOyNK9m5Z`CAltJ2rmC$zJ zhytVsZ5#S4g5x4a{PJ@6!cI*@$9IJ(mnT><*sGU@w$r~ZaIf7T?n!?jL0c}N@IUOY zYk2qVEmlMsZnf>ZJv8{@)aftXqDApHqA33&!>V+TS?CI4G+VmiWC1lm(aI7+@j(&2 zM$g`hv#nH<1kRtO4W$yFjuAjXReQe%kfR^XLhjfatO4a~XONyi%I;U2#YmkLem*s< zcV2fCm6c0jl7!uS(i|7YbrqY(CKZNvP|^}n>4TwWt+ig`GkqCW$$FMQ42Jz@aCw-6 zetq~txY(1*v2Ofoa4b4Fb7Iix0DJ!CNd$F-1zR20VntD_malEcVBSkvy)#z)a>@BGK8)^^{SpPJd z;3=2e+g$#FanGK0#uGvD;K0!7fYUp`+e6Ilo*73HAssy@agzSHqSBC9Sa2fx*1GR@ zb%~oluJbilHTZIF-_0O36DVzZr4kCYJu5a$!`FKBygsNr6HcIcC7wA2 zo!s~tAI_yr-1S0k3nfV435Py&%7v>6o2>S>obQBCAdA}ih8?-r>|&?xz0z7rub+{} zncE8&k()kJ@SZK%ptb%(I<|YZJRULco%$VSFd8l-N|(I@>9x&@+nHUUyy0aUOVz3L zz|p(cje;nlBds{x7q!z&k6hx{=zgdzNmdT!bVXN!lN~FM^hNJw-+B;nB)i`N91)pX zeGH89oZ%67CM6dCVDDZ^TNPItTWy7jtaX_!#(moztP#t7tgcrKK$CAN)R^A`M6VPU zvBGV}FW>QiGY5xNQ(KjCS?AME7T9PuX1qIv)pm|D{p~)ypn-Z-3Wi%i0Pq5|Z(0J$ zs;H~ynVSlJeSfKytw2*$QE_+o<&WX`v7SGi8~1W^S_pX3AmL{+Y9?7&CPj;w!ZZ5O z-VzSPXRkRQWOb={OON-7>C-iV&?D`&B@DQ|=5Y6PMBt*FS!lF|D^}ylro0;X7+`E= z{CbDsfZ{d&q+=q|7uk1!sFEzP(iW3ZjCG>APs#Ry?^RUorPEeY0_4l51!nHWP*bIl zCp&$%WHx$ubBF~!&H?K;IAL$E$eyuYpA7Cf%G_MgHrZBjZ`32VhtvfOu6>C)0S8*_ z9ehy>BhEgVRi&^w0?jG6dX&+QPH@D)M4$F^30RleoOZgP>Zz*Za&-;wFZeT2@Fy#Y zKE+!-F9XM7=oa$&`v%ATQ9{5Vp*|E;a!N18#rm<&b+EaGojomt>-L<7;1=%b8g+_o zK~T!a@8MyUzw=UasW~X2ORyfkk2)E52eM4w6LlXDsokCP29-AyeZ}uy2*l?dJTEm_ zA-0C-GFa;#o>#9r)yFF!cD+~})cSQ7LPB#wvJELunNWPYHfUKJa^?Xt$1`iPVYl_V zB@9LdjuaveOzC*qq}^Rj@jik$$jE?$cit05`8g()i*xbc%VG8Oya9lhgE_XcF?1ER zg^NpzZeOggKlGT0iTM&R8^Yia{(!VCvr^uS$mSL}bsVTRK8&gN8K2+fH6Vr21c`${ zB+4YiRcg>5?A@uBy57U$`Tl}?bUH^R;J-jFedA7m^z~tf*xrTYlp6jb=a$$nA1(**YN<|Hhxd1k@hxz!eg@^vkQMP2;TN@ zL^8RB8%O%;h_&j8sJ#D{c9Q9O)ffH|=QkTq7R(WL*h3H+T9K$3fk;!Oe^oJY&Hf}R z*~voJ)*%y4-9Gt!#705dlL1eYe|+p5Ayn@M6VfcP(LU%Tb(iZ#L=G6(?|QiSR+%Ex zVnk7S%9>M3SgILpKf1o)w!w1;s1FynVfI)v5@mmGq!514?|~BXe*6TBH!LBLD(>%r zRiF85%IN(_l$rQm^Wztt=#T#HnDVciQEgR~EZ%NULP4(W7uQTTp?w=)VoaP_1&Evi zqMq@?&Z9olr@rd7Fl{b8HH+?SrFM3xj(s)iXr8$zB|c#3v#TyKBV;3Tj;G7@B2G8G`jsxjc1h zF}ti!KKg*yclmKaAHZHvj>^U{wh#ZOWRejJThu>Yr3dR$2g6FG?Q>UB7!LcFE0t^AeB%H-m>AMS5X6CA>2*y2m(<4W%swj`}LN9?2?c5OUx^O68wAo@e z=WoPpU43jJJhH+tG&yBXv>z#sU4KT$#r4dwSuUy*NQpFt$#yMEuDAImWSxU4HhV~# z+Q)l;LL*c@?k*Q&Lfgs}3B!Vrn+In=9X)>d>XfB*a#Vosu}HE4TEk1HsKkM1O@@=#)G+9;b14XBK?l*hAR zRzjUODgMR7$#!%(vDQ#a)-@Kn^asy(sHBLK2i2EtMwS$AYn1asr~S-8M&?}_KK;S? zmSCstyt1`dD9^bG(yixRK-G+j@`EC&-G}PluS1I(l}2#-jB@*76AjVHDLvy8{FJiT zy%!zG>xJw9*4ndEak-VwMOSbFLAN@wpf%R_;x7*T3p^gZpn;m z+P_HeSI5gbtq0hxh{A{oBZLrxI+&1t6PmuJ(&vMkk|xw_DcVOJ0d+w_NKpYmOZDng zb3C4TP+A6MyW@8KlA5NUW=_v$iJy!4gl(g`{_xl*&N_UZmOGK0XcAJ_v7r8{;rMf>SXkXeCG`Ka%hqFD2o z*wbBonLu)ou(iQ!kv*@Ysx8v6u@z?a?9$?W4}D*=Q-Hh|;fWeSB;p)}g@olNk0Ybe zRL)r6&7f2{B(*ISZ2WSR?IJk$Enj=22ylJ}?q z;H9Q;LZM`L9?xk@1Q_i*EJCPqrl4imuSZ(&$=~0QoFE^b1&e=H=R)ytwa&#TUz?A` zCcQs>Mlcc={owkpK!7Fd+n^ZM*ph_Hygl#L0U&dH$aHPc!dVnAsM`qV%rl$~t`;B5 zMkQgZD22#zVT4Lt+(#GgcX;oI-V>N>+~|&U_hY5+iDDLByg92#0Z|_Qr>=7&$}xgI zjQVn)ZG@%9;BK+rE+X{%U$;ZpPsrK6V3eDbT7DcVz;m`&6O6Dl<_Ep(#a5u>+#p#zi3_219KLb5g^PfX2Y zj`g>ef0*wB&b5b(#^&{E`83DWl{u0>oL+}J{~OMifA~;{5XeGPEGbM&LmVUgJKqi& zr74#(8zxlNyv$$=)Iq;;b=2xMl{K!`kcQWjhx}|?uXnE0ac~5ZG)9!{ija}sMuK^b zzV+u#oOD0z1oN(2l17c|h_(pJ-)v`9xowzcrq=rXUTT5reV^%mZY$FF8Y2E3;r)$2 z@%Cwwcyxmm{PA`i(_`db;4Z46H;^LCMZ*%-bf-a-_^RqjmI&YyR#=sDSAW+zy!>X1 zdKj3qayj9el3X9Xc)r5~mh0qz!QVoHn`QjoWkLXdAPq?Yl(ABLc4e1 z_O=;#f|NS^EKM16*CV65W(M;rNaSuyJ=F^8D@$M#MGgAM`?j|J#p{-mNJB@f&OrZR z2Ac<#o8(+~K+84N?J+0tWV903$LmuRoz`&Gd6wzjF+A%sey%BRFUt6COw-zn$)i8# zTI!IWHH8BHkJk}O-Wg}i+AMAH$X>@2ijJGls6~m_8!Y(8pdjTYYeWY+i% zuK{~YnH1B^r^$(i&|H}~nQ(J7+rzvkr*)xx5#s{HMwh#~rm?jun*X>J^ znuG-1nDZ+VClN96FZ*m>v0LHNvD%|P zf4hnG{j8mLB{K?!)sGzMQL*TM$|6k0j?`aB$cnQj60xd_tZO37*Rvu=U{f$4;bTil zM|)4q9}D@J8;z0uTLOxXDsDkMw37VNBsg+n5CiL^kvYa)pV2dlp;S}G`C^3Q4upoR zNd4&yh!DAhRaV0dvOF`MgCeayeWMM5aUsC8AlnOeIBtpw@a}{eblz?*E)4{VUmqbr zhEHvW%n=)nfjkJQwbhguT##;Jkit$QPB&Vk=Q7;qwtj zKGrtjr*!|BU6>N|Ss zYAzd*aUfqfom8c(4`n0%&|f(40z7Rq^@v4Kw2}BmF@Fzb!D{P?FLB1kGJ6nXmwyh;QMZtMV<)U2=rC z9j(iO?1}cRneIzlfloJwpJ$@Y#*ldi{jPCT(ooc2BOf=W{Vc1s=JeUFKs z?}B8fu-b7JjsK~mKuXAx?5RNLAd+E!G9+lEaUqol)`#B#BagAM%GRBE4%Qo*eR`g6 z?A_k=4o|E5S~isByGp)l!a5IF8MjFBAbj2!E_f^8_IaE4Bmx6%R{IT21~D+&*;7}j z5_Wki;>BooEp>kWBa;=YVkES#9IMWxMUSpk5Br8_0|9f-5MP;_J=gP<5HVj6wGqjC zb%Buw**u;lrvA>Mwg(Qz6_uE=RE>G|>Z}Os33*d3 z^j=9F_LS}sy!FJ6Gy8!`OT)oF%u%A&+)Har_%^`&Y}-m-MMECGhSP)?7F)xV0f`NW zzK^ZNjAam_io9d!_f&)}SP9Agc1A`A2nxbb<98sXO~bR?!_rM_4IFsBUy#%T3K%iC z$yO0+_x#pMWaS!{6i&sVoHHHt9}znW3Y1x|y6CZ9#$tBXBAuA;xa!8_by45WO4;5) z(e*SJM?bIKrYc=geRzb;Rd{%tC5`xX;!Ev zEnojrj5UdMjxUdV8C-{UIydJHv$hSV1$SU^Hgvp)ZItA{Vwr!Nr&}DN#Y&c3$A6tH z_XdpLf(9=K8%AzZ>~*e<@Ka_}B)taWjqs@R{guryDI>Fr58x6$P|8J01xxfu7N+~i+-0T2d_6y}Lw4i9m6cJ>^Hd;NDQ9VbxcijE>I&neJH&_6yJ z!8iXDtu})9Y)o>msL53R@3Z_O6`5BL>5cgl#f-bZa{8FdL_~I;6k6r%Lbl#D`;49Ih+H+(}3r+P|~iJSz2DXEN1~8jTo@Uxl*qbn;Z2?z>XXoxJQ~ zTDo|-8ji5M6)h7nsGj7Phe=~FfdmSbL|Dt()P{ef8*-F$l)45dMbW$#{0}2$K^9(ADx9IZ~Af~o4=b;>*muRItGVQo;n&$bkAZuNAZr>BhdTqd*Ash8f-(V zS2WgOCfWY^;N17YZGYk(?`7ksb#F#tewS&Nh4(~F{Z!3FSz;ncz@SJvGr-$3?$w#i z5kG5ku-4Kz7peb7C(@p;{;w$;3?*1zN4dS2T|0YTHmI<|{u7UXe`9~$?Rc0W=g;z$ zSD$H)@9p%MBJHou(!D;x-4Q!>HzT9E%ei>5XQUvi*Rt3Pq$L4=ZH=KEv04RmVp^4F zZeG)CX>jJU`Q}ViT?{VN(rSJKz7&6Vt}@%ExFe7?Wo|;`J5!dieWvN-?IqpeV+EEfkRaJwoo%37j@B_8ECN%56D=TbtOYTc*FD{zS z{|~{!c`i$ROK$eyjM9d<^zFUwyB5~AsV?_Vkm4wu9P<0!%eHy*X=xIXnR1mpshO)= zWc!V>Wh=>I`f(p#%3wjVS_&NA9G)ng6KOTSJgzp|IuOHr>AO&S5xzlL%$q^@@?h6E zMWwRjTe%VOxWu=$5mC>g2@xN%JbVWZ+1N^C(oJiI`ORA zz4&3&_|7W4_u&~1@I0f|;q{a4dB1%G`+3~Dq^T)0{pR&Vg57*}SI!TJ^rFcp(Wqs$ z06bX$=cT&O)th+*E55@`ukb*{7`o#nv|rY_ZQ2=cXqBff-u^i;e%+3$MWGPlZLhr0 z^GvUDO@3-x{swyF_{L?;9V3~f+uY+Z;*W$6QZKWl{8Hpf>by}411gurJO7MFW8 zv@$l0JF|Ggkt`};v%8FqU+XQ-94Q0G)v)(c{+3}(_ZsYog=`)xPu>C)z9OSQ zPSi zDhp4#bva;km!>Q8Q&=U=>YSPiF3HcWL*5k8+f>kRMEK3VA@u+Kq0oj!>=ANrx<0_E zoj32mk{9AW)Dw<2<>Z_s-=WpcCwVD<9Q0j1nW&U5o%C3s+nBFn8?zJ=Yq^fSUGPhr zsY>*%S95GZBI@p~cdaFHKs^jjyf!Q>2xy>sah$l$ihEh+rr@^|K%Y;#OBZ11u-Y=nWoX|+c6X`39shrZ zL062Ov8Tg(d$7nCDDQo4;?(m79F%7mePEH(Fu58GfIF%cVj}s5xX#PHKSR^%ob6Kv z69?1(iA4Vdlo8s$r(m!j^uo@ydDQPhHT#lTc zJ`j%$YLt5P-W2bR8p5cS>rmSGLFp~=(}}Jr_MgZ_qQvCRogqg6R<-W@xiLJ>-kNC{ zYp;wg=6yrTt->3@BdpU1@M9=&b-#$6#GKmJWpMH}tX)3w@R z{2O;L&c;lvNlRgIGW_K@EO0V?gpXs${K&@z$a z8PqYX?D}YwpPAA`-A*a9ITZB#t-+kx5yf@X0pG73}4z@SvMm?=MmsY+y%mC0`|QJTYP?QcQV59l6b5#)yEn^=@$+aQ@_u0-}?NB7H3*N{>I zxh%!Et)E~5>}3o+66r&WYL&x>`;9tORe9jz6;*P6VeWr;L?rXD(xx^+Zj*R&!qKcgHuHFMfy3n1JRo)L_1{wYnMl z^!amrmE>j`@S!1n1ZW@&|6oXrZ8GHe;q~ql5eOl6E6#2YGEo8rHt<@^ob&TtLVio? zWV1S4TVrScHmT`{eF#9V#5PP1lJ^PdVxC{;{gd;@DEZ`(0hLb%%;=UJCLk8R=FUg# zHuvl=8OIw4YndHC4vf0G1L>Am;*xAEq_QWJVy_sB#tbQMd?D+K(MHlgSJqm!LVfG| zEy;n#-73Jxy*63`Q>d<0O?Ek3N}7zBH;42mA&J@g7Re)L*d2$*R?=Inyc=uW`y}!G zL2o7BzJSXJ-hhV{jlj2VB;1D$U&EEUlzwc}Bx%Z55{x}R6BpD;Gtaq-tp83pY%_%eu zdIhcsyXTk-sbX7{fHzwHBq*BLkvm}WcGf(;L|$F{HStnxpda=6DC>lKNdW2W6Y)xVkbD~7=)H6)EOUvT+O#X!F zWCJY%qm6|u3Ry}os1mYf86TTM7|h}3u(XEhs_j5uqyFHUJMCZ<=rHYcTa}_;7y#Ot zyu8mECU#@c^}t=VjY2t)4bn=Nu0k>Kv2V>8mV zCtM-XRGrxVUc+=n@a|40zFo^S$*SsVp@R&Tk6<1GLmxtn?Dlu0vcxS`)1JS`il??X zl!YA&0FuA^o*XICgZ>0WFF+bHlKohVxgLuU1d?A>NaI0!%4>_&mPXR^>%7tu3SNYd z{D9jzRr&5f22x>8qY`yV0%tCyptgaw4I3jZyjB97y*9fEHElj)VSL(nJj3Q6q^`b% zf^H+<(8ygo8;UC>I`YeTPR9y5CWHtKw=b`p`4a}dyn?osE0|X}I;lzLVI<+lO@fDO z%&!E?RcJY$$##gRk2~Mx9P>Cf=z$)AjaDM~ax$AR6{tb)O@xw7AA5s-N2Q_Z(Cme* zR5CLWN~C(Z{6gCH(c%x72;RpJr;u8AwG^hkv)3F^d3kzHE?C!KrGB-W1HjyOuDhH1 zuMuMI*Lw|y2}KfJlN*>DKu(;E)BJ;P9Wr>Uc{qz&q5;DxE}qxRn$s{pEJZ{6eod9z zNQ8|VdO*xRPqIKE{-Ubs#TAK*5Hi1gP?z(Oh3qc}4OU+E>)GTh_eLescD?;8b5^k|U8&I& zB0}#6FKSP&@duZ{VZl=UcN zS^j%r)>QHI8P*+zk0&{=y)&WEKt=h!1^P^&E<^x1rI;!R7S!GIk>Pm(JKvrgp7Odu z@WHN`pd6$Swbp9=*uDElic za6;AwB5Ipcls{~Y9QMUN*2300(gQFWEc(;aI%n*l0Nx#^p^&Hd|6OePKe>(n zFGM-^vG#q#dxk3I%%#I00Txp(dqQ>L(Rt0?=hzsVqF&UBNiDJt0}6|V34uE&7WBh= z2b_LKw4#zT0@!Gr_({k4$Tr- z8t7}bDeYs?5lc3*f}OYLRIlKzT{o`rNUOnji+X7~Q4W820l99|J{R-Vbu|h@?ME)8z-Z>srN@$U4U7+uH1|kqNPk9Jzv~*z;oY)%bVNyC;+4czyvXJy>dDY~ zPy%e{DlB>Giaj(8UZS-(0CvDLXg0oT_orni9$Ba!H=b1*8Ih1a&iL=zng96WVPhfn zcb`$Q5~|Ng>I~nqmGxN{$nCmWB>1G}v4Q96O?7##^;Bh->1@r8oRo`48R6KtY zlCjmc#&lXB+oumDf@gB|C~TfTRBJU}u9%!=r>sVftWO+-Eu#3dGs3r9E0!`r?}u51 zX0+1e zQ6qP0-JKHf=ER18!e|1sTZ1HP^91}X8PQ9abxo8{)C-zDMz{ieK+$x4x=Bl@lq&m9 zgw$JmoOy~e9QYayT=$ZmiH2V6IPediqnAc$@gYik&47mrC_QnXg3E z&?I{ps^wvwoAird$YqVEA#6qm%M{j9jSE$)vDBL}5Tyngks?2b{vdPLSO*9*HbuTO z;L^18HRkAj=O(0n+TD^^uD!q62HUoD6oe*iPh200KkS@^ikr{u zjWck$h_qIiYrmDQu)$OWY$xJm!-fIhU?K22lRTT+?TSf)9HElxvK?+j7lBKcC-T9A zY}V2Q&q`%{`*#h#1Q`ai8;{`_!9obUZ1K`~ZmcGDD5zTxs8grvmxXHrdN0a(T>FU9 zEiJ5sLT6qr=KaWu&oETpfH|-;aaBJ)(4V^{@ZNtoJ5j$p+mwC>zI+*Onw}*5uIvT1 z@7yQW3iY!elZ3U?6s>8{^u+1_z9O&H*`~fO)TFr8dSh>`x~tW(xM!YOTmpamA8;F6 zoG&U4XWMewRupn(uC)CMa>OK5w9_G`>?pL#sBc`#G`i}yW!S18!O02j1DQ&GvCXipRC!whwD`NbOVnT65J^4H*>+6C#QAYbG z1{Kct?ozTo!z4KleToGk0I@HpWs?LcTw$%0H^~B)<&Y6aG$=8S*{LEFAyHFZA+ZV^ z+e~dpPjLYD_77oKE^MmIma_{-rFEJnBpq**I5;hcCyz+ZIa$eMDe_vSYs`}}O}-XB z^TlcHs=fY(Gjw-*59;2x(bBHM1sI7A?aSgWlc%I<<`0?>D)qGW;3&xw>j~v?L1rwj ztaUCngO1YA%lgn3lZDZ49k_XtsOULYFf)lFO;@T(w92Iw-m3r4Ji$Qhc(&jxOSKvM zd@m)SBXtvM6Le^=Yr#P8=uM~3`r4j3wJxxzNhsr44gHuTwCZFK+~QOKYcRhS9SE_)tUE7=hX8@(ik<}Ehd!tcJ?MOA(`Bf#c+A|A( zhbe0$cMolL(biPfKt%XHeFbu&L2BQ%9ZFMmGOYc{PCfc!L}wlS=&;SnbQ_57N*0IBp$;M4~%0qXBvP(`d8&HI;rX z8Y3VBt$W+!3;?E(n^x-QJ=ti7+)a%CnP-2!h#hlwg2={uo$K@BMwmDRJV3f)$}|1Y zp*AxY>1Ah*7=#YdLstUosfw@yM{l~_H~of1!}xqEmYzd%crGN5YfOtRmV`6E9(Tyx z?+gao0sQAwR_8*7J;hu|+_J`iQ0Q~`3IB5;c5EijiaCN^16M@xL@J!jVSQesV2xKK zIH&`QFe%`Kx96hR1UX&l-5`jH7x5D)v6-s$H2pFs$8^)B=H-u#03~QSl_!IyNFOrg zd=gFS*F@~HL^6cWf-%49*L9k7?;nI1#-k73`n3sRDAN|tpz~Sjaj^QK-|oV)K2PZ2 zIu9&8`<;l(Ay_xxW!RS^(+R0-E7o8j@^SxM;!|LO9$rtJAd0uz{PM!6Z(u2qE}A3+ zxRQ1be~@Hszvn#5f<3&-9s!u>hT>Vn-TKx<`s9=>;Im}pM`uytvxK&OpC@SUiZt4S zCVFT99AsA+?0N=;ISlM@fJ$puza)d`!Eo!+&Rgs#9}&L^6#+pcQnk9ssXO1==7>1> zI{RT0jJg{*Y0os1Pjctyet3Xx@K?fTx)ep@b?-X_MDP$kdu?Ex6g;t$u5;Ap#2UGF zO2d5;6ckh(-Rs)NV|>T4w!Wb@RFC!Li|hYuQSw~Rb2A@w7c3sK4F*mVq7lkkq<#?+wt1M!j=Y= zO73RlXUir^03-gHcUH3BiH*2$2==6kjda7-*K+nOZm2=0nIgDns83r$G``sLTCFw6 z0kHhqzaVv?o(uQP7^^MeZjYiRjRr&-YBw5=pWd%)4ABsHs)eC5#x-tq4%ws6(iWrR zN$8?`;NhZgo5<56D1W`IOSEwE{AQoU=b+-!$>z$9w?yMIuG=S5sLb}IHtU+1WxO*o z^bbrCs<)=~h8O$jWqb=DHHYVh!m~jze(bOn?ima|yhr7GUUk+)uADv(jaJlfEa}T8 zxPiA)oV^9u_W@oP(M^rJ6dYd9a(QX9rh&;Nn{>WNy}ilYq#^rtp4fqf35U2-AZ9;) z6mR^jG-oMxGL**zlJICyHobzhiU0EsMr5x4Q`Gn16 z%O)c^?%H?HV(P+DIKcfI-z~L3?1`Ifx)(kj0rOskkf8LNl4S7|zPfhSOQ%=(*yOwN zpY&F~yF{y4+)Tk%wz(tAfnOOe#~#)cVGZjWR$EyPZD1c;F5&;TsWqkXXZ zLhMeiU>f|se2HEoW1hz-jmv?3Ty@ry1E8D20!dP z)A|X#%oMn+fgJRc-NY34zHic?5@miawk|-FOX>eq!eZ+8K0d$?QCf*$YH!3&89};i zyyL#Y^PTjhW95DG)1MdVAS3+n=5JYA;yX-Ww#Pc*wd`lS-z^c62XG$G&?IY*FF6!bzBNO@SQ_Xu915#N3Y5{Dp-(}|kWJllsnMPc5SGWH$$J1%G*OD}W52j<8 z+x5>e;jDmNKVQ8aBLtw0B8?%;;xl`^4eW)ztUXaQodemQ0hq8r6kgs8X8aAEPkoK| z*1pYR`Z`@ipVV1wXc>&lATNy?%5~;_ZK3|%H*3UQbkkSC_wbHzOGxCNG!erun;hsIqLe?eKB(k5+cE z?-AFhOXZYenaZdKTsE&)y~7m*EJf|!Jgo~iC|BkAj9lRQixJlFsf@vjOdUay?-cxA z_fRKEbUtcIsbCC42IAjJ-Sw_mOIicR7@nH2y1ls?dd|%r){E1fOPj=iBjscqEtya2KatQ&<qT zaSsSr4oI>c?@b5b?97LYtB4JZPyIMpyXB9hWcI4>b)q6nZLe6q>WkE& zJ!oaxIKpFd?V#uPyCf}?A(@TXl+s$5>FRD^kK)-ilU1?&RRh};brD|}sDJr~J+Lq( z>iE?PeoIuI?0d`1%-NJldoe-qT~~_Zb9gH6Y|Wk=tS4)ID9p5sIb<(&jv`*#g)oMll~w9k|EBUzaB=JTdb%TK5Fp1 zFyd><3AfXtkte(hPvaftwHyQ{f2d%Cn()#9zPZ} zL-J04a%eY09TPNA9b%&sXsI!{1tHAJ6BPn-Eu0g)FqwB34A*s%lq&OnQD4LC z#%g?U#tl`UbopYY%d&N@y!5jxGK6#AwWoPcs9G#{sM*`2PRW67$RVHoxG4bXLZ z`s+pCYksL+CgYp5H`q^MsWHkNz7Y|ZZjFPEGCSj0ph0spBcUFBsaLr5z}a#~8^D$m z>OQobozzqD9D@G3BfrzrBcP>1f)3W+?onFwyUVA2uc}p&sNnJVv#bQVt9^k|Np{29X%upzoe; z98ZTbO&?D0*otD6jr(@xCJtEt@yt=yycql?Yaz*B!C=j6EuhobO(_X=s_c@#90$s( zx}hAOYib^!?S-`zT>O;|TiuWuIxMDmtu;q9?l66F^h}C!4r$ddGJkqo9YIYg4P#NA zt^32#3-uusqK^wd zYj}HYgfG$3bMy%WjODt`U?NG0k@6W-2(GOl8F!)9J!gE_YR{eF09twU$GO&CR(?1= zqRqN@-GCw$76eP9YTIGKfvmSx6XQke=1u$Z(-oRnnF^A`t7 zYDVUm_V$jh|33K&9SQusoBQTbItPrnBvJWtT))L304@tuTZcM*9BdxJB@3G)&;*PZmSPy=wjzAoC&Cefdt0$E8reg0g-z9wMq=d@ha9uKb{{36jn_*Wv?1K@ zidm)$!@=*8gSDfo^M@B2?lWN@;y>$BG{qV#(;x~HS!u&Vte;|q>iJu?w7o8(&^S^_ zrw8y1{2_8%BWC6>C3fm_>!9^|L_Pa$N!X3`LOWf~JtoOBuoNZ}oBU|tZYZm>&Qv1s zu9jzlgzRWI2Qhf}@TvQzRLuGcsEHb!B^-9_VL9p{u9qf%e03{>(KKVJ%heF)UL8V; z>OI{tyJef@^mtEJ+BEK*L!wK(KS+eao#sAa@~J4cJmZS;+R zSXTx5-eJ@D_Ol5cp3U{jPrY{$h?@h;-rf7D0RCX`5B`FQuzXpuHw$(oo3L}t#HKem zW%l1-&z?KuCi<&gXo%Wx4-&A3gIjI$Isy_!93X5_BCaGx99V=8$u&F*n#k8364xK7W1Q#Gv{e&GvxeG zXKwPoUqe<`a?oIekk|+~XwYUfbIQInd`95i@a8hpzJMuy@aZxC| ztaQ7Qwv2Bm(ux#nI6Umj{s``;lpkDL-tYb5{o{d=)t(A1R*w%yIYli+|BTKcYIIyy zhjQ{6j^~yQSp;4CcjMOXQD<=H-S~y}X)aXo~QWOP|o3(l$Tkx9<^{+qHZd`Qhdfui_ z#gFYyMb65oDQ*3K*q^=3`Ud=~@nz#yhfVfs7bC^Me8`52p-7_q9A zI+H8Mg{Pb$?Co20HL)V+Y!d6#m^j6+-}Cov|79Wy-faI!W7f8-@AKw5?GgAbCP-Lx$DywMm?qO1F{iMA%WVvub4Im$!Prc1si++9-NSK!uD5ix%rs31fdM zenwOtlsTs4)Gw{YNvs2ui;by?OMCxM4VLvV1RNP3yW+o3IU2-45xM zFh>!t7YBFJIZ-G=UHV=@cjytQk^=w{Jx{* zw5g*(*_imGC-*<|!8f63{*6ah_9vWDC3B9?3Li?zlvi~;DTTjD7)Y?c!wtv&P&ENb z%z}IwXFaS^AGU#qg0PUz?Ne`z=C(Y5Mqu|!`@}24jxcbqz+;aW&`n2>0+8-5`>4J_=-%& z#>hmenC{!E5jS;TUafg5zb6)~#Nwl^I6C0S-_seoBw^dUh<3YwiAEaG_Di`2uP*lN zn9&XNbmWJxn5_ojc7BWcsYdH~DkbfDPt|#nVINc^sD629qy)7Dp>(xVoGL2BY!iMN z%!00XDk{+?J+5r!#Enr;Ldwj$f7oa{>22%$1Il_Qk`j2GYX%%F$AO1j_ zRMK|#E1Q52n^inLR9h~BZfbX|H2LWJ#{u-WTY!6LmMe@JgO_A?kqQB#q3Qg1J4;*< zS0^bQc5k`QtB;0MuPZzSVzYddrVwMuW|x=NvP5y|DD|G8JKzzSH#mKmOv%7vpBQSS zq}Tghma#&}j)ce1CqaTvrpj@F*Os(bs3vO0l)qBQiSI7seRPt{;L0)jl4$zdqRt2b znVguO@a1z-!3?HD{+BL?Rxj~isf&ncWfuvQwUF(Pqo|E_hP6b}?kI-c1%r)-`HW-d zsE5bjKDB&eE2sG%!?x-5_d?ryS|)nil=qK}|AieL3oTHzcISSQpx=XlY^EyoLjr=s z{Gxh^jj&ZS^9<0_)8Ce7l!w0l;h|Bl{zGS?-z>{E`7hq)CZ3ji4@JVXnT&UxaGfIeyLe#Dsx6<+${J48j9^*d{|{r-|}P! z13&@EpZ3VSN{i=eu`ldo(H80E2n z&)M|P|I;%#b|(d1Y1&h;hK|lxBxzsvm)DlVRD17&^;yLn-#uu8#q$we=oxg#Vjbe! z#YF$G`c`D4h3ngyz!PyJqOZwAlPJBQ%3WC4B4m_8(wfHO61}E{p)>JDoI0x_x*#E( ze5#3|ZbhLQNh0jXAIsI0JXJ0(4d@5oM;%pV*kTKH)E-+?M9yvCB^lAo({r+bzg;HqF=bF4$uI9L z;z$vq{dsuJ>Ncx_L|nTV-ceP{%BG6}v)hd}GOljB>`d2XdB{C3Z(h;Tz0fx_IxEue zI#`fhlPw?3u3Wf@KX;HnkT-nvBwlI|k}&9VWPcsmaS_l6?q4~*!h>2Ow1i=1kHL#K z*9QYAnPg&=RoKwYu6a?;1_2sPc)Px9nxQkbZgu8ct!p~n)wobtQ+RK#3&m753%{c3 zlr>pw){v3Wv%6}5BH8Uc#Ni^go5w)#=2m|Y^dkL>gOB`LB1(GcXNCy82Yb`;Lx|30Ka<+aY#{$HUY!S^5Rl? z&{#=n{I?T*fNThNbC&&b`@1sM;`8e1D;q@XE2xn5n?~sKmm4CG+SUl~rhg|#ulmN; zCkvxmokE=%>?4)G740858Tc%}y@af&Ts^E#nawACmVcT*kl}uetVs||{fn8ve&}`S zIfw|9Ca``c74R!jRCLXC^5}Va%kj$$sHoJSiur06J1;)&Pwi<#52v8;VHNR#o71lz zF=mw{{AO6$o!ryDoB_K7f)1QGo=b)i``ae%F-^EYE5T=deSm@{?O^GwtKkLBr%zIg{jueAjfHr-Nf_SV_^8T0~IFiX=H(FQhX)quo z%R4~)>?ypsb%$?0>lq@*&*;D21`P)xh^I611?q<_G`Z(Ne+omy9XX=f- z_`=?7ba}R&TQb{6?@$+QZ5jc&@id(*4y4lW1<-&4q84N;j5PU#qf>@jo9-J2!OGQE z89(`W;JDPyY;j%qW(vd6cV@EYAWECau1{wZ2ha+1_s0m%X=ST}O3|l}I%{P2pI`sb zoc0*o%F0)SnfSqJhR4c9C3_P!LqR)4)M@c=;14+*fuUc3bCwj7Pr%d@LF{)P8Tv|C zJVoBV1RM?oozf=}txmeVDK?(irW(18u^^vs7^kRTX#QgdtOvCG_dZN*FFsE%_mnMaZ7J47KHiZ~<4^~PA`-mhesH~i~u zbhi6z`n!EUfrlK@D=-&IU!Cdp!*0j!Dx-G;)qr?Hdz!8|Lw$SAcata|!>st`Dhp3g?IyXH7GRh@62KJ8+okn!!JXgi2_gqDu zZJ1?{ttc$?KzeE9Hy=uTUO;2_NnWG%oX`p$Jk7fjSq?O?*o!jMJ z$gh`2?x+cXSoR!3&HHjVLX{ocWvi`SrnL4OC$j83H6ogA^@KJ6oUFQ75>#2=uLyOM zFQ;)xyT~#PM@#f==P9M(jTDaUwV!hLO;@WZq#P7iYR|F9^6ltJfI=MiEZ;odp*5ut zM9S<+ivk9PNTF5rrZb?IiS%S<ACb zL-^@%F-OAFK#uy#{$;0H2DF!LiUqCAAs zmzd~bzv{ob2n5a{Eq=@hFd1QR2TG` zmr%Im(wkjD@j+!QSgsB&0Ws;!llDS)FS2%~m=jsGjdPzgm^>XaB`r~?igIurf{{mG4tPxkeWlyBgSSUru zo-bWS<`6i%u`dc*IP7+l6jT|Skr`a{m10X9ir>58a0pMZ3F}tQ)=)w$R>!%*Z(tMb z@9SAj+DGlW!LYKLQrmgBqE=eohn9*LJ^A9sd-jY>>|bxB^%SPcVU=rR%yD{XZrl9! z!KZSpxIObAP}zr{nye=3ddBy%z0^|YmrP)bk9UEM4@cu1GgJ1h9u+1&D>Up0R_#vu z{HuNN){O??gwi=(VG1|3M0fJ_PJZV90j9jZ6E|We#hMazK zc#>%IO;sShLZ`J7aa9^LL}4j-R{61=c9QWiLJ|`WGYr=nZ*SFRdb5YHop$@)yFuWy z?Q-`W2Wi0%qt_pftV3SyAc_PYnLuB#F=TV{LZ!EtDR*BG;#6Fj6dPourPUxSvm+zp9 zEj`lO8+SB777PsL0OkN}DYb}izn=8hREWmFYYV!&CTjSIC%#&LY$z_ap2@RLY}EK* zpX%3bsFBQIPOT_f!pIOR|Ff9xyZ;LeB2;{Gg)vC-vW}U0QH4hI+f`%v&h5Ml;S`$- zaDT^SdwZb^{6xAK;{Y+ajqzmHrYrjg&Bm(|^5qdP95tr1jm=R{AE&M-Dk^kA@MaD? zN=&1Zt~CSyQxCjCT$^NZ5dKYm*PH3^h%lVSEs}h_kEl}SAtajd6(9EN5fhkgp@!!I zT3LFNNeD$5quJ(Uj&t=-bMv8hbkH%V4PCg2oBl1NaU^hNDgseeT$2s}ZwPssaBrv? z0bG>bcF%nDM8%v~hF-Mv-CHk0#K8# zjp~VLnVw%;H$AurzDl80Mn4%4=riRjke8>4^U8JpX?a&SN%c%ZM`z08$5SD@^otZD zSq*c$9`{At=x#DiewECuMWqI=#}Xr_emxMah^w%=-)YekZ^QWz37;AC?C+YTAknHR z^`9Eqls%V!MXq*@7t7JDPM$SVk8cVe`rZkIna=N2YAL53*6RGQzKB%9CfASf5Z0Ej zHIF^K#kp(+=AEIF`jPibs{Y^==r*}9Bnc=|`;ws7^ongT^YeO9QR?<&PZvwcx>{7a zLgcc8Bx42tSfF9kqSUn~4v)nK&uw?)eW*^y^qECO5VjWo25w#{G;r4=@04&kUOD>s z#0AP@`JRX_uPZYv-PG?h8R<~F`xc25TQTz&jKOnRE=$O&79gdq=861thJUXRA*1^g z?99p4wc85Xo*6$_*q84+_jxX8BNe99dhvdAX$6A=iJLaxj4U5*4Rut18zE~}$ z5J{{h@%Z?bfRqW~-`PQNG7d-t?@@TuIP%TF|lg`_oiwT+R! znJ;5+W(k;*?NQ?ujEVndhIbox@uJcn`I6i)lHIL@NI=gfONfaMqGSJfV=cy=Im$X< zb$HTHl=yCVWFD4>3(=R?JjI*0@;k3RYDOd!gEchvPn(ZwkA5O1hI7jk8is4WM*Q@) zCs>`NY%>@*aBnvG*TvFKWar`Mj`K}Grskeyk2Ex{Hf$j;`OJ|8Nsip)%%y8IOaWh( zvkf|?4%}HmY_!h0l`boAeM-Kd(d-tOr^@0}-=@ReuGmjUEnUZ;4Q~(n!RSG|rb@jS zTVv>3lfAfl?bIdtkP2_+dH(QrF`DNkHe|Ftz90J5BC3#4_E8qIyX0pYA9}3vuBJ69 zx=eoFa;HqNnAaEMDO0xV3=jidJa=oa4|(SLL=yuPUo%tb!0sug>h&5kxI#I8<&jab zYi)F{F1~mhu^n{BU?F;lv)N7eMQC?1@y()VwZn9u(oAnv(1>jv_DE?4lXrH=!^J)H zj>c274=3!-Tq}1yK+f7L+zld{j``E1k==+{N|$H4~yXTfy$I_$7A-=5{z3kn#O64~(fdoY=AW?VgV4eA?@B z7yP&PWtw}ZL-7)SqH|u3EXS55-!>%#qIB-qAx{SC$Vp|&U+hGMZ2ejyON{jXwy2L9 zA2>;K!-BHWf!t*sUw~NrS}n@8h9E#S4M`hh;w27BEAdlIl~#(~zI%VU*dCZ-M-3DC zxw6m9sBm{r-(Za?xHF;;&8Ta>29v#xj*{(^PLTKh9ZyfgR#Kj%_Pazcc*BA1$DlP@*p{ z2qG4h6t>Y<^4z)syXq?|2DxLHAp>I96fXHtlf8C4ucJq+#S0%_j9H)(jMcW3d|soV zLJIFh%rwmhRQnEJde{WoG*)zG3RdF9-F=2yjCPwwYq@JR%nhNh^j^K9G~{q871d7j z7KVib&o#r8YGSH#mX_!mc{4p}Z%?XP9KDfyK$k>AQGF6D&NWJrtPVgIiY$ z58q=d7_-3Px#dgyVy9%pJN5&%I`1nSyAVP+`KE$@k>&Wf0A*Qk#7_C*4`_q~!0vYt_#WYX6FSbgI z%@FHO4=;X-h6r_O)=F-yxDSsDsf`-PRJ|K3z~1aQI)w~2E|9uK&6h&mvuHjWw*a5- zs&V-=0KeXR)K5n>NFHgie=vr*It;yFACfDUH85{agO2_9@v!+#=%za{dtZ~F){wrn zi{6Ep-;D2WitU{VA(M_Ym-j~aCx@Mokfs!(PD>u(d%;L5p~snNa;BUs?hb-bFzL{M zzKhn)GOGTI1jW#OcY^M#Pk4nrd(%7vK*lldK39(w8vfLeQAo-5~{mdfdc z$#s};eqq^_VbruVq|=9ZQZBOfYgrlT@W@(2q=91HH^rD?Z-DvD%?-}sIbB;TW=o#k zbJrU)uhZ1Q4SZl%HF1TnooFH1&7hu_4}(Z7$0SWVphV!75#`8~V5G803w2sgS4q$P zNQUNXE&K>9x?5vOk1m0DJMX9u-rL_gRl^Xc+*f)CofP0Lg@(OPOGoXf-o?~$+*H3} zRC!W^hP?q)1o%<)mimb9d#6z%17oGF6;43X2T_}-|4?C(9q$GT8L45IKicPtzS)S=nHOc>}(J=(D&S*>6sQ64<1-F6h^s^OQj4bj~!lL?JNS6 zhyFG6gYDSaCt-K1z$V>p3X?h??6kUk!$$NubPeUFGRET{AaBEdz&@dgB%Y2mbMDza zs%j@LSf%OnXNzrBAGmSi8tWM}*)tYNMq}ckVuZvBR+h|X#&*pF2OLpgDQ@hN;?lRFE8yQ0`vW z*670JV(61Ll;Wnv6Yf4YWZ9qZof1klPzKdT&d!m+KqXj-$O8nv?}Gr{ZZ&boLZyn0==&!Wy*WnS7QKqz?>Eibk8M!|0%WzXM{uemChNS4w_kcSJG|A#WX7Bb{ zl|RqaZ%ncr7~qLt)xLlTicfnk>+8|^QF*%nU*lw$o^vDWU&ndCp*Ja4rVKm`+lz|P z!xHx|Wk>*ryBmWOs#4&`0|OilM;%@2w!VaCWtlxN%PtckLMHL_?rj0fYBR;h)7N-%QM`#Maj8c%<0| zj=|mN8)FytO{V8`nbPr$oagBR$DV+CqF03?;_!vmArm_YCU%XkR@32=Hx@${eP?6? z@th{f;&j2|3Q!LW>M#16i~Klu-U6fU8xE1xzB($w9G*c=5V)sxyS+h~>*sO+c@Ip3 zvTgr!uETBGQp!jP4bIshhOO=e9*?HRUF6bD+E_!e8lB$!D{4c0JEAiOFlyyV2$b$b z7W?9npm@b6k^_GGyDUzF8JuC}eQwwH$~eJKBlJ#OY-(dF+_>TwFRy2tVdb1)`>4#e z9m%_P8=G?z@wzX$<;q4Q(HgtK>w}TtkQp`pI6(|&wbhH2QhCd&fszgtHyWHdeD@AVtJWb#nLpi0DErnmH2>&qMhl9xt&MQ zuXs?0F^#3#iD=z~R|Fg@ctsWQcJe>K z^Pd;a7)9FebRtSYp%I2ahVv73))n9QE573{Pauo{PYqzXEZby7`LO%@^D!X?nR{9K z(k4Yg-#*~8^c8#5xpwdLhyUxu{`uc~YT4`ZShdrF?p05}{M8;;0OW?M)595g>~noP zXFFE~M)@&3w>VAk&?q$xkJSm6`LkAw!$Yp|67A~ut_lhYNWayoT}1_?e$U|iDK-97AQw~BE66TkYcb)U2lXRh zvb$J|jU2Fk#@|FmFhMn1%XfY%e_h2vk^svicbtSB=hdx{ltQ~pT%B|^;G2S8Z9I_Q993ecVf_VgM1$v z?)5cd!Eah!5cVo}v67oN@4R&;18Ta;ksgbp8uZ_KY!1IGt-Wd%-IYk4;Ck<|$zwn* z79fbvyMwA#t#b0i)W$F(pXlhO$n$!q)0MIceSc`{318;&!tB`KGq+*tu7cbvX3>L9 z8J{Mll@_2lQI_b?=M<6N(hn)Ur>2gloT@rH{aa)FU%x8aZ{3e9O66O1zD~UyTa5q|H*ft{-=>WcdJg_`oY{Th7H|7XSCg-UQT{&7#E!k8VUg8RCSZY~pDt{3bC7rdVCF9K z(5#CSXsB&%_x-sk3D*I{v7jww_B3GYvC!vC;(hoH$2rN}o*$OjXn0#EO}x*-BaJfu zaBTKE8f|V@aqlYMt&92l2Fc>7CB|x6!1+)IXkYWSC2`##bw4Nz4?%d;ok~oucjy>$ zQtQC^@lqn<1%}6q?I*Vuil*ZsSIk|?lZ&p`HgI8KnMg?ePg8H0>|HytF>8%sx{p%> zckE7YOj>-1cc~6<5e!Gb`^B2NyIeidoOMewt+P$vu&{DAqx-#>wBzDTA~59y9_Sbvc*a4JbD$@`+em+;uyEMK;jerS z3z~UCIdc_eSR1#nbz#K$@bJ*+Qj|O}WZ_N!q`FUo6M%XF@<`R;70j$IdrW`G$?k#l z(B&0k-;u0P5q^}Bi{nVQq{1=MS@9KE{`Pq}=vpfMEw9bgqaH8=WzD@)KM~1~*q@kw zCIPTDIhL9vFVH}>UqwX%APpRHP7gh|!suysm0sJa=w&>IRt1Q8LLWSu{`}3Z$5AiD zuiwY`wYEVs<}}uIyl#FM6dnM%CKGmx!|w|uDkJ;TmR&;B`KIOrQe`}ns!q5U z7ntM=Hp@EIjBD0kr|yE*9Y0QRSfiNUfQmW09GVMEj)P0ogUPiRdOATBr}sT4=bKMj z!U*yQ-O9-2T4|-)m>lC_wFR zbIQ}D-+sma@-Q1`XEgO&^4$_8r9^a-^Yrzi8Ie$ZuF4&baPE+$09sH^Px(r|Mu5|m zn|QIBs5Q`1WOjb9M`J*OvNqMxQmw)#&5K_M<9i+83s_pKE;H@*a)fU7aG<-j>^=n3 zCtj;>MnLQa_b?zLBk{;Io&Ge5a^Gg+QFKO6FL;ETci6P+Dk{-~EH5@y9mh5G7~AkU zke&+$kmAYbJ?rxAO1~JGaoP4S%7fI{n$Wu|zMSm%IxC(fTMR&$F^Sk1zzCpX23x3! zG4yW9acOtujhh>PLpRZP4c8u&u0?>eX#rAMBRN48`~e+>BP4D-cHSBO1uOn*$!-nd zQQqGdoQsb2PRR+_-NR3=ZMlAyPlu{eV`b1og%+|(FZ(_^hByYC)-tnaWl|PodL_J= z1>Ub;)grQ#&?PCuB8j}0`mW+m@{F>tqg^-={Qq$F)-ixhJd&EmPfflG#vZDQJ&z4{URnfeO zQpZXiZ0FNEJvg7~YUBn;#3VWRyO`ea4PGK_7Qk7Z4 zcj7HIUgGL(ML5iDYDdq)P;UtE-CdKIKUqIM4Cz`|xB~f;9|wFO$jEx-K=h1n)wrcL z{I%zk%Z3UwzOkL`>vUay=LUPhM_9KkRyr2Zs?f_n6*tf3)o84nwp|F+_q}5!^w^_ROLDIx+R9+bj;QpabSbzn^}`vpndPy%6)#U zNyxUh!5WJNurnH>o@3sX@|y`(6{zX#b7t2Ev4_uEnLib4x8hEmyVWbTfftix*;A%U zQNk#Z<%%T{`VH)Oj)=@;kf2J&6Z+?o`O>tl-q>SlSF-JJ+J{=}nABe!AzoeAc$X-; znN1tC6~RhEJeM)VWOJ`V9>{8&YDl0d?LaUgUYzz*1T3(PTxc7o{A!a^HXhSg{c+Z0 zSDebjv9q>M7Y>g7_zWpYC^}~-{bO%JLhddGir}YK=T2H1BVGk2I(x=x63N!CV=RZp zX-VDh?DttmVPR{~6>r372Xp9bJ6rU;=QeLBl=3idjaK(i$y#PzTS(xClag*op)%)f z@icOnr)`AD(#`bbV?VtJCqDNLr@PrX!GF}F2T*BR&K|@H&JTtS6QfT&3S%C4f0AL~ z++1;_RL%beJ}9RXS7+ygJwDc)kG-`Gd}~K2ehwRZz6)YRWAe`s9`hNQyJGoLxQ$Xm z5`sw+yLF(6ph}ER^l$w;Tpu(Vd&xv!q1ea2AU}D8#fndF3`rZ?8stTV6G(guocE8x2?r-PvwY= zZ@2s!WqvH= z%^gW%D${R=D(|bTl9@cR5LHBz#nQuIAU_FH%S4B^a|S+jRqIm0Ww~Z=`@61MRLwH~ zJyK2rp7ii40P9kGKM1&cv0DUC+252K7(f`WNJltXeGZ`+w#K0zF&@;<2dD1}f^TVB z$14Q$O(8VLc)9%%Dj2QpR1H6r#qFfI`sWMGU1}Koy3saVS*)19J1g!# z7Zz-9rji9r5cz-*Yzy2?&C-cGhE@l7}9AM zaEeZSRpN^48zRxU$OE?0f9~5+q+S$n?-Uvk-rl;wDRudQEC#!X?iV6(*CsboZ~SGc zJ`BPPrHHiHfK{R(8R{?}M>}_!Ser7X<;L^vMr$xTTXO+iQIF6#3!LXZSP6WY zxyZy^JYk;X>5WfNm9UK%bp5jB`63{9Wrel1y&J>T@f9!d%syBPdLCl?p*cq`i?f~L zYZ>MN*8F8@t?plbThp!1iuKmXR9OCtLdm3?{UQ~V(Ox?Ag~@TURRUSZdahe9EjBSR zrvL||Nq1XQX+yI}@1KnhG|%>e{^N@~9@KuqwyL-A#0??fs$9Bs z%+igTzPKzoRsKtm+w{;UT?>7KCHhLA-D&@L9c8HFcyAk6RIFZvMg?Lv_;nih$A!Us zDQ;=-?FwwoP4Ui$l(#;FKYvoq5U2cYI^;=4XW*?YUzDk$o8wXpOroi~OG7`Hj3V>s z*ROKixzdK?dE0#$1HE0lfuR_yjm(*IYRC87x~ql84jul&&X&DIh(0QSP`05@F(Q zJj{6l(k(kD;PSu39`-q4iCzTOZk00n4*+VAu$71_p$;)|AIh)-HU*_`B z)cio0>lj8^-yEyPzxv2GW?R=ggbziovw%jaQ#3xEqczJi7mnN#W5(L$10Hq;v?S`8 zTE?sxg0vA~%`Fjo8Bw^c~1#U~@#lNWc>joXJEh^sh+_U0qJzbC&Y+<#(FFuD2ub zZ#iUj*hvdH`@%qvV*~74x(s-Mr(dd0&30OMt~FQ__SiAM7ueJYUBL{c;~raI=Re-_ zL2hC^`cbn+7JFnz|E@UkgvlIr+CuVUpcBj0##eP!36A&Xw|3e+v%SW=70aD{f`Q?9 zv(cEjBRee=63H?78$D1+rvnP2&IfY-GA=kDH*Xd6 zf_KERn6;+t$=w$lm~F>XBHn)da&?7#bRzGu*_b6vqVNeUd^L~<6Tp|-LE&% z)@;uG?FDbz*Y3FjLu0?2js7mjQ~mU*`wuKn|5NuFTXdHGf6oH&o9$NIX*5zt@x|-1 znb|k?>PbSnB{{guzAE*PescQ9g0ws@?|$L1#_Vyd3HtH!m_GG0=Y9oO#G}_BLJ34A z+*XZ`BQ3dAve!_b8VBaBHp4YhOIyQi6lS7|dlgG)cCd5`& zUrx=Q!pi4HjJ2FZPw*ItO}bE$BLI7oa66KVl(lhv1MR>X8hW@6#1n%*7oBE+jHdJ# zqu;57WtX)b-OqtsU7`r0&pZdNsg9cvU7%Siw;udsrPhX8S$FAf1p%4an!)o*kiiV_ z+Qkzo&3H$K9!@+5RII39n}AB2slMS~7I!aWajGHt!ML`hw{y6!?$`BeO0PZUITnbX zxUB_7i|otgj*VF-N>%Zm%{ZRr+&He6t9v5xSN+*DS+bzCzvU9g+PH}iy- zWs!f#O^PP1X6;J>AG{E~&E#Pk_MKh&HTYu)zNdBRO_#ZEYuvX?FH*9{(O zM~qgaauX6%-C9G0#z1+F%u*saDZP}Fq>ZtQryvLRNPV}W_lrVU9Va_Aj>|zvQ*##^ z%N%PbTQ5wg+Z+DE(!V50%Eq{+s?4>kg?;1U^qPH|^Opi4VRlc$sMMZC09x2@HK%3*c zW4zm&azMM~5ch!{NoVagxL;PYQNtJ3hf1-4XT+~&Oj@taF}d2om#U1Rs=$h_k!blN zY^Mpc#|@2s*C^BFpND+xnvb8}zGBf`*%05p{ajRmUZ>#d;OLkS-PmV{{|q`p`P_N6 zwMpIGK9^-hyf;VThiH2l4%-lUr^Ox@q}T!vxz`5+u?8{?3GVEqFk(H*4?zg=!~&5& zAIb9Uv3#FBqM}SM#<~9wG{+g_yp&q;LYl=de*Tp&e8;|myvKudgCe(=>0}dch2YJ^ zGGFqIC3w%qwZG^D=G410oixRN@DwvS{h%kE|CReja^~;dk?NufZI(;WD&OE$s zQxIUU8i90Tk}W-uW1nAJThQF|{?8pG9(lT01v5WWAjkD^;ywl@<|lE#ic2JuqdlNM zCJk=;F6=|?l0mnRJYE^XLYpDgnoW)JFyO88En2%`l_(n#WDHFa6{&j+Yb@tK08T6M ze*v7Qtp4m0=aL84TVo5Y*j9bD1V;j*B1|yZ4y9iOF8sbc(;79NB2}CxNbbz0J(i>` zn{ED9)uv-)lai|q@vVP4-Nw$EKuneQPLZkHcbNH@`1mdPJRnWTG$hkKJ<*AIl*H_Q z{UchR@K_3p05ht>^zG#iI+LGCSCHSa%UM6NKj~=W)=*T$bNDg`HY!Q2w-Kiy^V)^k z@tApo{UWmn-z&FM6Ow}a~wWOv4NMSb1m$jZ`BX@^+ z(1_1QUZ+`)qslk<#u+DgZ8V9Wdp3f;aW|0w>b6noha2ypYk&XWWY%zdY6%ju$FGg* ziF#Ciy&7F~DRiETzpfnJ`(?tAgp=S+pkv*nt_-H$TFJri&;6L0yJifz%pG)v30Avw zat4E!UX1&0eX$7GH782YG0$!XmUd{IkY7b{9&x012q8HFM~&Nde#=O^WhU!DHr9DWrSTu%EZvWqEFKo3JY$_R3j>l$c-*rI zKXlw98?|;G7!xhlBbl$I?oTu$+6~DpBSmJwtWQKd`{#=9UwDd-LZZaX3p*?K5=?8h zyg?Oip-)+qNEGe5`5s_8d+ydtLF9UqFNY4dWt>h2lFs_*^Y#OMzxbjBk4>^Gt!`18 zHm1u?VW5OiuW9ZXvc2h{L$tqb1%9Dfsx&3GrK&52pXv6r+%X8AIduaG*LUBn%b)Ci ze>3YIm*baEHceUGcD9vMNDZ?LAuSvuJ?j~+!x6q&9u#m3(m}vG3?1JEdEude_6k(7 z91lO;SZ~M0+x7kW@)lLMOSyr{|7~$-%~T)TZf<=*3k`^ip7&KtFmV+_6{km17--b7 zr5j9eSicRThOeHgrhQ$!@lA?}me=-?7`Tes{r?+U$vYo(FI#~1-=8MY$E6q*BQ5Zp zg+54EyV;|;1Wck3kdrsi;o|ZfgLy9e|8}3LoiWtAV`f&Debv~sP5SUnX|}WHB$tf- z$E!N)?L=S7cv7P^(<5O9w9W9R$;9(>T>PUqpnjiE?N4#b-xxo+QDa;d;f7gdO6a-K z4+-2&Z9l(q5vspvxFyCL2kzj@$6cHR?03 zSjg74|2_R|Y!Nxa)K+iHF>gay zMEm7IK`pm@|At*o4tXQb2ALi0nlN9oZR` zr)eZ*a#B*<5#i;3!Ij3(Pz0dF=>HL1`C>@PQ>+C&pMFal)5rs+{%M6va6K`ECl5dvE1c->&ze0jxa{Q;y_KC0Pu0~);@+xk2y21c4X>D|UG@I-h z3xX9Z0rjpM7^y7&bbtO+_$-Tof5EV@wVEC(*4ku&owU}64_ueBuwU+)TVmolxuhm{ zGX26~_RqJL(U2H>Su(x^qvFK3Ib)oim331~Y=;!!bNP5W~Md|5iCAYod54 zJ8MLC%cI{=R!+85pg>zII)m%tyi5)n6-@Bk#-)PZ&a%8&+xrhfF3kFlZf>HcrAAFn zQkEleQc9A+C}`6}jX-bp=9d7`A1zGE`)fUgiXEo0DRXcXT-V&wc(T0`q6=WOxTEr#z18=#T$LC!nZC^hXX0NUYCO2D{I+94|gflGwCi?!b@vSchCj413rpvkgHT>n@*h-P^gQYM9ig0_CcEdL!ODt%UGN-T~ zXR*5HO@DcCk9Svp7fP>R&|-4@`>Bo3v$kA;k{^dZh zGU)d$-$?Ar`xDJE)!?l*;O66ycKQ$6n4u$X(q~{`@a1-i8>|h916N1Uhj2c9VuGCO z5>2Rvj(6R`lj@>Sf1{wTS)VH3sUlquJ=@e^r-?XK7&hj4m~el#d#~OhU0LGQ^Q7A< zy3ZpN(#Yg3BQN9~gWP}JJ=WG7_)GLw-p&7+WEh9Zd`ZQ0_rKHmGmUYlbP8pPuiE{R zdbe822%cyq7%3T*`dhwHm{R?_7ez|Hmo2HN{=yAL4Z?xm@wK}RJtQEe}mH_^NA z9wdW|+PeVnu`%KahoOD%SI=cZfSybGdez=BfxXDF(C&o0TRDM(8fpGOfUMq*ZQ6^C zY`qt2yi*;maop*@*8kE~A-F>Qmelyu0yjqdFGDCaBA9{RAG-PKYw`$+0QK4jessTN zkl}ZE0-QeF86va3sQbIzF@&j>O~WxtPj8}qQx^TlgyMI_7N_=C^hz1&aMN0 zYLRJmqKCP-|E^ulx2`b#Ulro5*XVkL&fmIJw+3Bwe&j@=cXcW7K7jI3aynYx-vR7U z3fEr=J`G}gn(Ix}T_RT}sa(wx*(2pq@2}CrGS`bh0M2}{5 z(`pxAw|~DzZOL3Tr{|4IBlXFF`{kaP)|xusAUWA(^Cs$za7#q*XXFbGZ$s|$r4ekp z5-3lo5l#}QOkGhIm(UtfB>nI_#j`)x{O#D4hGJJSF+#)Wm8<6Q8A|@kI7U=tM8V5AqE9tn&L{%%dGnrxQAc#4z1RoV2y8^{^u?jiDQWM) z^m4N)W3us$nT*5-PV)(Z7O(lK=C!yMp51|8H+ai3!cAREc^fpu;xExg}P>spQBV zx?%t1<%0bChW9P|Ge{4!;)CPhvinn*@D}d(LWOb*s#t~Z`YY<=60}yO^uEhS)x=f% zQ*iWmbVZz@#U)sg3A+iyF0DqXe*dq!E%y?+F@_V~Qww9N4$r3P!sMqbszioA+A$n; zEd7p17kkT@d2qe_OK_ztcAd=G!{3OsxwST;@#j^zUQ6l?bU@vJc{#Z2r&4!pSDNn$ zrH&7IQ$}*-*S>P(A~sJp-q~!rCHtP%ntTFAn(sE+JaI(bhj#o?=X_&CbMZT(Ad?#Qi?|)7q|aiIQ!K6nZ%FWjqlV3n zL_MqzTRFgxSQHNevbzL*W(&Q%F4jwPjxly+$C!72ZA@{-oa?)V6meCx^vd{cAa!(M z;IJbjH^pv!ygbn(-ZL)aoZ+>9R0%MOG7gq8^Ad?6kp=G{0-4v55rOqR-YxG6>~Fip zr9|U2PR;}sRh*imb;Wi~ZDC2RM2!{`zNHw^b^RY;gZ_8?_dOG~g!6vWVJ}bd^1jya zzL^By8{b8|TGK;0{E}OHa&atlAR3C{oJN2janQjw;_DAfli_B%Z*9+7R*tx=&FXN8 z0>63`I^p#-C`oitQu6&VT2k!fuHuV97lf9dhZnTyZdS#J!H?n-2*!RAJ($P<5o^P^ z)w5&_RNJQSl+VlPK9V@AlfcL?KYhfU#Poj^h(4P6Xy9Tg>Ok00%}uZ@76wGjS4Tdz zvkJ-&rzEP=t0JwyDr(DvVZ|U~3##vHnXjGt)^=b#lJ-ddbjsU}>rBIO(nHExPuLx$ znSWp)d{_J2o#v-dX`TZE#Bqtq=pIMI9*NYnyET_omdjNMJUF?D;XBl$a3uizO4n;b_!N6&7BG@4{ z_~cB24RUkFXX@TnB!P+6enY#F$Zgu;Q$kT?e!^Mj?Hlq-$Fh4Qcbm2_ z%TI6Zyn>nn#oqc-GJ-5)-_O?3By!J~kIUsZ0q&e} zn4<@(uXOvPJu|PpCCA5F<1L@OF;@Mt7*4(mI(6ixKu^oYM^3xAx$Tamb~!wjd0blW zhF?9m<6{;2fx2VeCV0ssL`3#hE%_fQy=1KxR6%3)!p;i~&IUKXqB7YdO^;&7H(9RI z0o=96mafpfXM3emnEC5dad3Xo7)y=0cBP449F}2ajqj}iJo@2Sv3o+Lae>4U%%RSV zJ+&JeBZP4u^R8jX(Xd{2H@n%-0$0_Ee~fgI5`nM_J!w(C8$+m zY$FuJ=jaX1;l-QMdP6^Uiwa$=y%!}@{N9hLe9b2WJDmE=n$_nGQ0&x%jQA(e|H&qnN{(_LQc;V zel_D}<+dQ!Aao*rLMH{hK#0|1NO(n=*9~vCug7FGST{&LXoOLradM>emd&d1AT?Vb zRxZTuHDvZg_xKF|V83GTVBR)RIOUFRKRQWwV|%fAO0*} zsF_mTP0pe?G}R8%m?(ZutU&wv*M4u{XFEn%8r|CYWaa+5fDHn-BA70xJf` zV{^;(nMO9aFSP{yD@Mno*oA_A4-aDcI+TRyhFVhb>VB?cy{1r9fcaJ1!JP)JpCZm| zJ4xSBvQ~v2^H3chy42{7{T8{U+R^Wp7C7f>VWjv*9{>M!4|%)&pgxb!^Qxmn~!j| z-i{Jwvq!_;kb3IHP9drEd~O=h-MvNW z8oT`TJ5Nop`>#~Ci`Y>|KFq|mp4Rg-h2wj+aN=LMmo6=r9w@6p3Xg?lqa6BcBa~$m z>CRwSCB;W*c4Tjj&!%(t7xsK3k}mKn@aVi5Xr3QgMVL@flWHlM7=5SbK5?L%T4jVn ze##VHXIN_8YG^$Kg{u1$h%zy~PfjTKT**tvo)+a{plqaU5*zsR41WhAdmwPZ#X3rpA zL_G*-Nf}!}O+LrwI%ZPC~~py>-j{H@;!`eYVw^`^!*4q?ow0 z9QE72VZ**GH-eKD!;vT%3XUgja?$Ir-)bT`BDyrx7B8;-5~|C;%IewSq(scuRdC*2 z!(g@2*zJB@5dA$j{JT)!kz*WE%95nh+_k|Nd<7ux)PW&6X|*_6!B5nqvK?b)xAEe? ze>7-*v=ijyh7t;@bf3=(LbjCG9C{o{loZK_)o$&z)n11Wd^}(ej2^;4vx(hact@Ji zq85%b17R8hlo*U{-CtBA(aH*)B(rr$l9)MbW?6~gG^u@b{OL&nHemcSeYc|=cEROp zBC&-r`y@C#m{J>=?JSz24W@W2q9qhzwI`p~VP3S5FY;ugjg{Lr;CsfSl`N8N@z1)m zx%GP~`Rr%31EztC(|XE!V3xv5#DOz$um-ZVQmnG7)dDFD(?dvwvW^B0@>ebCaIr}1 zZz0z7_I5|rHEe(5PIydI7T%T z>a>3-(yEaL`V3SVS&RSw; zco|qN;K3awpKu|KwrS(edVd{;6{x9FdF!n#)71yRb!6-^r8Q5bXUkjsT)J3HOah$wjcHr|rSDlvI`U}+81dp#`|J)4W=)*QY z##*d+umu@X2$_*#8sYp%dti5Vd;^Rf-O$Hqv1(11pDtvU)W^hS=XO}fb%(#tDE8+~ zcm4^Z*y!q$6c7t%?9%>~iYkSr{`5n~d>7E10>EavnyzO4$&3a#{r*ZS9z4^zo65|B*0f%m{wDDrF2KyCTEi?6L4=wxH+;UFgM4R9>Aa?S5d*BZa8oJ__y_R)zKvm!u zIr<#a^EKC1!TSy&i#2&Eo%n!Lba(QXE&SHU1@!rV{FJC_1ulMGp2E zUFU3x_)m~1`q)=51YKYKqZ`e@&_~eAka0~4s9y?&%OBM%eYDB^V7dDFp zbxtix=i^2JB;xu#QM0vo#JBn)nq)nl05#uUjG4+sQe04uJH6cs)lFxXxi}t53TDuR zth&vP9ELxlejZe|1sEe-M{6g2Nf`Iu@SU?C4q}$o7@Sqm!u;=9fZn~mswJetfJD#g zsk#%|9ACs(84A-EvqG0&NiGJXU0U&6$^+ovXc^U?-`9OWxT3@6B&ls*%4ZDd?vqAf z5i;nxoH`hmLHP3EeAm^&XRtChs|28OQ=+!|>8LWM@%zCQNE=?fKJj2DNH46kgv^(t z&5fEm8!u&a>}Ireo2tO1YE-m+IObkY^u9wzOoGf{o08k%Bpa0v1q2AZ%wu{Fh zhTmjhDP+9Y2c(;D19$OuATmBWgpAN}U<-yK@dvuOe%A%t4rb6WY{7t7)%y+$#=yrr z%JYB$jcR?kK=f?+4jPJ8SXDW7tdEXY-BU2dYB9O-=)Hewu|M97tk1So0T=34x-ttE zS8PEL#eNGq)qKHM$=J1Guz1v^l={-mj@GU9oV_ri`q+m~)JkHq69=?<(MQy^ZDW+S z)o1z?U&L)5vf?-6K%3R_ zg`Wv6F;e}zrx`<8=5PI@Z|SVT1{*qR{;Kksc6Wx#Sq@jEvK?sPUZ&j9gof1}f#yr; zT(vAdwXhm<}kG>r&3Ku?W*V>vwoak){QKg zPAs9#+g8!a3Nr$Ct7|?g2-`&;Iwn%fH%VYEpIL>r2;n-n6BE?#k|^b^@cw9*p#m?l zg!`D%5);D2kfWVIqEPm3m#5@Kmh!lWmkx)NhCg&bq&&(m9C1z*slb^EL3hr3`ygV- z`Bn02DFmr(W~w+o=i#*cVtTQ1wka0zO5fPQnh@OQrA3Hp%9CV;CFni2m1vKCelxox zzrUloBdIzAO77U<&ujGjF-6}6TUQR>pPWPS6>TXZ&S94zq zV_IK)R8r>RZb)oO%|`yt93$xy1O-MIa@U=?NLreFpKOmwSR3`s1+x93_Q_iG09hWN zEtG2vKkq~2Hp?E{$;yfJX3D$G2IbfjlbOTpH)DsvYrVAX9)gjrS4a=Lk|@j{Pu*rE zeC7NsGmq-T_1p+>>Tp+AsM(wg84g0)1s5Uiy=Kozce4P+hM{LcK0jK0MGXTvOQ+tV1}ii99g&O@E5{nv@l$%^L411C}a41`X+5Ga*J=rkvn*cYZ zXto9RTCy&K2q;&la^lx|IYN42#Bk_|7{mtA+Jl$2n#E{?3bK9t%gBL*8k=Ya%+(^Y z^qvvul5l0-cZs%vR&rjS-P(B3cR$x8ezRO>e}2?rnI|36K6+ny;CpfP`F8@{<>$tu zhk;fvBw|TsGsjSQ3qsjSlIbUbu(9?46HKzdY|^RXh1_bd;deN(crqU$Hk*L#zSm-J zH~{l~CRvWu!LBuoSbRFx)^N_KkHoT%VXnd)cpWwz9EA9+(|p~{O}P&HqU)%A<_lLC zNBpzx{u5o&yUh0T9`|PN3A_x76Z;QSx#M*5evB&S7mte1iW8=DcyqPn;`MKGjTy1x zP2}7M@vJWU+xDS5iY=es_MAJ@UjX!byU5tfyF3~k_Xc`J32N;^m(tI`f_>e5p~1v2 zIjA33xbI#oavt5jAtt1}Sa>+2bNHysyvb`%#vwq8;2_MB`<_Oqxw!&wk&uoLg1rz7 zQ#1%+%goCeZ8Y}hM$@MV@d|agbm@99Mgm1gPdoFAsVd=?9Td?vZL8uOXM082YAYwS zhg@^u2H1Q;(`Xz3GDRyjPSz@NU!;S1WjD5C*PIVmKX!(DF=?y5xUi>t;yUtv0zV&p zZVj}u2X>*GTdeeoC`L7vWk}!N6=XNZ;`l?znhh7#(Amuzd8CZ>48~-zZB>PCRs@>{ zjy{|zlJ*LxuD0G^QWoYYns@&uRgQh`Y*$r!N8viiqQ-8NR$FZE5Z$qwlZ^uh1>l(8~6)R4pOjj6=}s7I>ukmdwh8!uLkw*%bm; zD333a%dNK$!^M-lGNDqYhcB@ygT3Pg#{Z}?>90{a5_xX^@SfF%yzm~|r%~5XHVwfd z3%J%fBWp`3#X{BmLJ$HMy_G)Bs}}<*LGHTUqF8?b488Q~&8$csBLAGo5s(h|qICd;$$M>y;~ zXGiVF6fIDGdeJxktXFt1mb()NEaxsqjIYf%x(>zadQwB+?&=dO z>#@EN=j_O=UukA7*W{&Iy1sh=#Jfkl$4eJ}5-li~1CXR}XWZaeQ5+_w|3pfN$x&%t zW`p7tluj()I5*mNm98nj?~(Q8#U5S&2|E6cL>x5ctjrK+CkDMzF)ALgGxyw4Z?(+U zVD|g2oc!FYGt=^P_FLcfoFDnLwg?EZUYj~)KX8_4yWcFtM|n6n-V-8GW+@n*PD*Nk(d1LJhj6JquPk?hwQ!s_WAtbv z*ftjca%V*U@#!<8`%vWK6I4N;E&BGxR!-jqz1zf5W*cBO5w#he378(iM!7402p(y6 zB@fW1QPUIM^fo@w-q9ZRS^bsdSzpi*PnA4dx+@*>D50n6V8fWr;NQUC6xrk(HB!#T ziX47qp4iylKmMH;dNVMI@ljT3q_v#KuhW~&%J@c`x7onwO*~vzKH)wE)2G&r*4*rf zKKKw}`$;8OtFD7;dXnO_1}2l^dVQ7fk`Lhu&LNeSiYJ*R>FUu`omF_$Z**=`?eiI?dezhq$drL@L%$x-pC z{57k?ItZe>5bWo~dLd^>37{kvXwpttDfr3REpN1326!dhtW(;x_V4k9vSe^DduUfX z{+JH_$A~0{R@7c+B^d(E+j;wf7CJi2ETvn=L6SmZ6JvZDWr`jRmXU=ZI<(v>hYcUr z@}=6o@?QDLg-G@HoXptHU2gEhxWDel(BzUeZ4WbYI#mX+6(q_Ca1X71Q&T$X&`NGO z?zr;KIZHT88J3@XzuU_BM3WTpNC>q%Y5gY;I9*QgA(xD=l{$KQd4xACBt#JF!XQT2 zW3rj0Xm!Vua^(SPdR|T)eu;xXzq~=mFm%E-sk;gE@d;mqCYlOapK<#A46n)3`0gB^ z@RafO4;4*xnGi~sR@#`hM4X#X-2~U!T)jM zcv6hbpeCQ_AHU>gO5{ej1;bSr|pTJ~Az~q2{tK9OV%3?P4s*sJDN0 z%-xW=-4F9GCqwk@#ob0w*AjX|t}36FhNYt+cRIMj!<%PQ=avk_b=lLPsq4V)wALAZ zeRPcWRcw^z>@)p^e!(V=U|ftsw$w@voV4T#gq>B#T4gXU}=F-{Si(UrLQ9)*qdRlJ)KJbt)d6nFz~=kI($ zIYcFSWL7lToKTZ7Z0S31vI0=42TwTE@X7q$1nEaA=&CPg{3$-x;co*O=8t7gLEK8H zQn{wXO)MEa>BERmgNS%OP-#xcrr*Tj&?`#f;@WKpjwrUTrnB|22lwOhxqA>gO6hIK zw+Cdcs;Zjf(#>}Sun?lqY=gl;GwuW65xXD8XwCtVIwM?L4qPfp&It*Vkx4m&Qd2E- zaDeRX*UAl_xh}d%ve6RSD(<63lkZUa( zUj`fHZVP^@}s|HQC7!dZ{E_h5bMuX@C14{ z6egSD5PGR6sex#A`vOK*(v_CIf#WuZHfH@q2JsZBzPTMe+N3XES6+FmjchQaGBo4Re{HqMQ>ouMNlS9@>i58#u+lvND)vUi(^ZzaNlJ>@@ig)`_ zpbN7Ff(U4bR-~3^KZBB2PX~kEHDpj){QC_UX5EKifz#GReP};dzC-et9rv)MBERE0o9S=M%Xq8{IRSx zr2QXtAzdm>4X$9eWeT`2)1}7S)j=!%h*Un6~`;KUDai0`f<5J)A5(!|D$LZ>-wA9c$waQoHf%d%lVCaJNxd z;qUl<=4I>YaEwB7u>VZxh9eqwl=ueq-=FWz6JE6PjU z2kiGD5X1NF{PHz9z87v!sE^2Gy{;B^?lAvqi0E4pVxjWp`kwfz5F0{Xu2-iHeF8Q( zfj-f^xnsMXw_LqD53DW)fL&L)`$Z>=V!2Ejz0D1;gDeYzu&VEYB0X96Hd|<*I<%em zc3n2w^+6@-(=Dcg+$QN2vjD+yfw)=JtWFha#Yt{AzuZx{VRwkfsV1Z?Gp^|lD`rWE zS~RQ!8@kT@TR@FZhneTMtzlGGvPMD9Vwfc64OO?J>fwqZXuhJBwn|h_Q91R*QKs(s zbJkreOk^}*{$K(R&ZViI0Lh^{i(N#Hn$S1P6v9yi zd#UqPsipgcsaQ2=>m1>);zgGi8^mZDnvu!u*k#85JzAp6ZW$p=4iveV!MH3e&RHDJ z>q{nzIeZekt7m)x0;kN}?5c_?FLwGUAmxT|bf#0*M86TW5-u8oG#^$hp8@xDZrZx| zrzdyF@;riZTO==$A|C&nN$72|yFT2A6QqfLyW|e>S4E1O76!fRm!aqZ?>6lfFWdhu zm3IxrPL21V+|T9xhaKEIo?oSCJp$k|BoLMUT>Os2_^eXqyB4!$g9caIw=#o>#3D*e%V%7e zx8p!9ydRUeMG5e!f*l4|pCFG6Z9@qa8QD=k1p)c{E>P}|Q2#ydO1K_|#E5xCQ1GaM zEV08U<~Z;SeT1yy#a6|c;~8I2k$2;#wK`_1PyjXmTiJZH_P#ZlVJh%+7Ln_0st>E+|M$ z&v+a;ZLS##1svyD{aJ%R)D*Vzl8g5%W9MObWyJdH<-x`mv+5n0%ntmKYj`48wGHXx z9W0n?g_Az>LJ#Wj{5)D`pU4lmTg-I!2$%g;2zxFU4jOJOXKeD?(Xg$2pZhAA-KD{ zOXK=_zx(bvd!KjKuho2XuUe(E#uzn+4_?Kzzs(FGdETSU^h|YCLH=Z34uRt-Z>1K7 zv;!nS*I!ZyF0sS=N-^Q<&+^OvTys!-g|SCh7UOT}dOEynY~=Q|&DrthEJw%^OYCw% z`Yx3l!Nt3j&brm?{v9h01_u@_2)5rdMc|Z%^$G$)Jjsv=;qm^EL51knS83g&XOdU7p~}}#mA37bkFuD%8c9oV(Lw<8P*B*V&#?B^$rnL2TS7JxgwjLLAXxN+++hn zxeG4wb*8wl{1p;fBb%F?)~qY3NImK+3=#2p{`wCsYk>Mg__UVn=930m8=7oeoWOvn zuT2Io>@Y79zsMOVqS^bZ-#j0@lTt+8I9*tkTXin50#^$6A24L~>or=~!;|_n@D~Y( zlTZvB43&4d#hFo2=E~OLR_NnC&o|Ru$j(gvDkzQplV?$+w}d%&axXC$rA!!m_Qw z&0#rv2W0>5s7m5T&@nc8y}kw(QsWR@zG)Nh_Zs^_V70pJ%Gk-X7YDB@!!9W4C368B zJgQu$N=+`dQN>Ica%d+ODaB{i-UT*sC)k53biLk=VE!T!mXX)1uGQ-Rs?l?NHc8VZ zj61ofA3|N?y2JX22Cg}Q|Z3fr0U=ZDWk57n3AQgE3rJYh?? zH22?Sz>}4#W=bJE8m(uHEo0eBA_3sOj{1qP#9%wrCp^C&C#688u~1_)%97JHQxz^O z9kQMz(;R$)u;4B|d;u~$T4BfdvIW%Cgo|N^UQhpqTp*vv37}-&v**785mDAG08-vC zhIW_8u3R8RhY)*3Njw}BdUeBk3E5((+VnDUzzSjBY!q8E>vmog4rgfK3ZL9VkuDBK zo=S1qVzS@ncrqIEjT7AG{=A=y6icdqpI+bW-&pR?evpgtp{b_i+0SRVa&zxA$y+;s zF=KZK4mHCG`HV;|D0xqVKl3f3(K9b2=Wi;B>MRd>?Wx~= zFoy;r-)68`>$_4GQ-NsN)M^Lds5;yu_(lhQ16Kx{iLy(@^cIWwhyNUE_!o!Ah%q80X9)|K(?*_=`E!u9?WmRFigcct{r)or)!5sOpxIK|Wh+fFFV^eXgh0G*2@gOPJq zFm1JpLicF2n41KP!?=$Jqi&yQBUG8|`4lNB&gELIAEBDwgYwBJk1r8Kqm+YD&YB*&Gpz8q?@7tKodNSJmp{pem zPB;V8`(Cd0<=%8mQKzD|f+^#pvarN;`;LLbmB@xy&pX8Y$bt83YzzMG&wM8I%RWC0 zLS<}DttiEKSTA3NUVH`=b?#pq$UiwgdJbYVMxBj#>QAdnMP&e2DOQFpmDip*AZZ(- zK+l$LWlWgGmTI5SSj?6vGg}Hx&~2d+{eyfjDMQM(WTR(TtyDkuz3BLhuyBl;o*#d# z{29RP#1m;E5aT&)e(hVA$7MOPQ^`g%P;R9m5W8a!pN1eLE!%xP3MWhWNk&>vSC1De zg>P{7fycpj>VLHWGwfN)H$@Fs!mg+2s^S!Jx`-)5<9uw@*pPCKTLJ+3i@U|1w9*v) zM{~T24H_d?FDz9|wEH51i(_%eKvZQSqRL!(Q)g`~8Zd6$w*3Sv+7M6pj5-3I)EpE| zWf|aw;N8tjdjvoFHt87e?CpPRlGhwfcUMWh{`ag4zT3VyV^k6@>^;b}Y1WqBiIp06 zGZNAeRET!>AAY}Z{?g+4GMMtGls26!1s#nqce99p{4(;DkU*~;&4vcKbGazsvF z33}bWlMY4{erGxa16sWrbdfN@G>d^O^Nzc5+h4RL`{Nine9>!=c?P`cp%#|~?&esv zQ_SS4RH}y;*ZJI7(<2qIxE}_uWP!5tzg8rpgAds}mYPd7Z%Ybyj2aR@Q=C14pM$Ggd(?a11*doadaRNW^|CEQ` zns0e>S&RNCmori5EU-23#_#?{hKDSh(NlOPI|t;Qgr(18|1I?D^dV4`$fP#};U=9e zW^>R(7K${DRmk}EFb7P8WE)l62MHdnHCzd8FwCqf7Z_Ndq7RRD;SW-Q8GBOFM8@~pH9f4-(!2eA_@T-%Gda+ZLUWD(kOLrZ~qzGfdgs;eNj z6l;r8DPUeq{y1+>;$1m@GaSCmLDY@&)9*&iIki_nx>z|OkTh+N3a%tV@2?+eiu+N` zy0)CuA?$4Jv3-yr^U{MBK*?y5CJK%}={^>HwCZVQ4 zVR@om@@9*O#fxElZlEmtUE^9Rm4pvuMEsH_BvA{=SV;R+RUf{TD{kU}hDR2%>D@Pj z3h7MaoSP|BSIFAqV|;|ebCL6rivVyDt)lQx^tcI>Dl_RvEB&R32b|GSpA7lV2Oao; z8vB9C_9~PvcD464-o2TLNzsL&*I&1G)efcRfjrN+FA3D zF~nXbUx5o9eOUFMLh58Nft;1ZcPjK}C(D}~SksAhHZOc-%>`-Rjn{nSjN?c{m3dO2ILvG*=-?T}pYWlhSyV7`Xz#~FZo_nu4hZs1Zp0_pa_r<3PY zMhIEY=?g!B$HxNvb?I$ry#Vk@{L&RE21V*Wi-RRbrY`|2lQq`FlyWpzo^)kDlXn-b zi!kSJ0x)I!NS6rk#NB)!ES6TtH6iC@JVbigF`a(qC|jvdE7y7kvy^2*4vghW<|6j( zmJIx;P&AS!weJcph$v5DgLz_H|F$~FiEAPSc>Hsw&Q{gjhRcjgSLsO-Rb#GHCd`_~ z-F>iZXy?3}bI*EcuXlO`%cPct5Lv(2QH|1-$AE*uPY}>nKuHA21uIKMH}iLt;-D;Ey->84ZfbKMfpb2U>gvjDJ8E z>*|~a2ODDwv3azIGVZKNOKg*s)O%DTX?wXjy`W?nZkQ-b(0g-N_IEOewA503oLC+` zU&{=~nW>lg9xy+E8Tegf>hW4_U$GW5*>B_r_B#Ta`gN}ghejKd@vW|ljn1&6ce}+y zSAK!7;)Gte?omrY3r%rehzs74w*4=BwShxz#&wW823BgdG6J=|+0@BRv!~l8KVJk? zD7{fe{UYHyg5eC7b~aM|v2fuA0uFs^X0%m!6NdVr}+VpX#Ss`(G{I-f}yN z$U23CIBlcEluIHdOTGTFrke*#ACP0~m}#hg=clOkD`vo4uTGzl7lC)0?N$)wk$c+1cU+9TBUTJiFlsFHd-GOX<=4;gV+4e0SefV4d|e&5G+(?XYO+K*RL8Dq9rU*c)N zBTe9gYjNWdzCB%>&|1W62sjK7!TrUAUV#M}b6*Ut+7?4RGPQ+?lNyD$Eb0OD9Dr*l@t%;M&C~70kn9`16UCoO ze5z5_sbVP%DbXw$F3QW1g&}o6-Ykv!534O-KmS3fe#ae62ux1ku+r#f$do^=Tfzy{ zB_<^e7%t7x*Lk$?v?o!W!`11yu4w%N-7${qDNHC{99=Q!VTE0BaBAQj1lgsJN|yop z)R;S1Cjh$FIjnA6!zEHy=Vlo2Yx2dL4ba@~_&Dnbo4tu&bsUKP~T0>3LwtmNy88>2&eiz44RU+4X<*}v0iXZo=+B_1)JHHbzu zun13ErNROs6uaS_oMVCbk)JbD@PU>A84_9wwr0lGDnc+1?26yf|H(^&o z3-FV4n}o+4bTd>w4=3*t>y?^JU>jK)QL(rkOJH`@1 zZO*8{>$i6U8|^vdudT#Hr(ckhWG;xj*(qd_QvCUre|(#6)Jcn_s4;(p&T#ckwIR@| z@$j`szL#q_z<`+hAEgPcrv+U41niq6(+bVSFTmk+Dj=4wGfZy4fhlC~oA%YS9o(Co zK%W7$aFqd!12`0#mE2=ot%;mzeO z0h7%pxWjW>{dsF8l?ZeB_x)b^6r+m!Z}W?hC02LN`5I^J`X{Hni-4dC2JIy&A6vL= z?`Pf2EJ$Nl)2>|wyu_DNr@4?jy$rYZz~I&#{(yQH?ZL{p0v$Owe@J;p-%m=@LvNf0 z2i8bfkNi9X>5WkCE|g47>hAh-@{&MbGdN0YMa|GlGc{;XCmBYvE<|)dBy?|X z6op=xioc9r)5Lw^BhFm0^vh~>EWd;cs`eKHj(I}|Y@;ahZdu2~dX5J-CS zAv9~yEK~kaocDG4=TT(?zFKBrraXU&kd+tB0)}6+ErFXjg2DSiX7~N%}x9lJC`IfFr6rUKeXaZnG0>WKqQ*VZ%^HMOdc{jBlXvlM^+Xt zk)Lx19FU2^U8;Z^gQcvEHihWB@AHO{(w8ewgeyFh57N9tKiL!i0b%!Vq^nGp4K$`9 zW$`CUg33 z{l*5ZdVXX(61nvHox>!*!xqk)=MivR7x`wZ^1CQOy;w$q{>u{zfR5ZyN1lC!CM`q~ z4V}=mXu|{L7ysI-J{2)xZ*cu7*TsP-*>BUc^z0Vr#<{6&Gc6da9_* zUPncR(QLm<(69Ivrs%W@SWz8j9Brj8jiFio$3xx}Tf-)lHZzW~gHe+`Y*w>L^@)jE zP#R9AxK9UN9X>1E`vXF|u3Lz6rk9(E2RZT)-p=`FTw1C+-urviiA7cKHp#KY)bWzX ztj8D5Jvk@!r49g_*5}OH1(a%jh*K)%nkVrZmoU`8`rt)ZQfZ#@?>9$uSU99gZ$|9h?Ef?@J>f3hvXGjR|{xzlfH>yNZ@xy(+M(a-V@6MF0%U zn)ug8%4}bUt4H)yT!#!`Z3{{RCQlpsgZ7o2`1~0KR}?~f9Mf7Z+&Kt%qJg9AUQSh8>y~c zY5uxeBATOo1zr47!@=NXAIO~g7GclO7}tCN#~AWl=lVm$eLE?6vAEt$fHuH5HySF7 z|l7%4_4PlQt<0@LqttKy7S$ z170<0P@j+07KY9}N^Rn<@-eIK3O&VlyDurVYg0%x`Q{A4-Q5{Y{OiLlvZ;M6L6o}% z?vd7pkvY(KG+detAM>Q=tTN;IOKl6s>>;Q7dllHEh!i2PPsCtU1gn3wyX7b4XG~De z#~9>aBj;-xUB|Yf|5*4pM!UJm3f7|J8S0Ags zjBEW{c>reZU)cYP)WANIx^qXpg+GE?7zSe?OwSJ+U!J1r!9|ktpZu6T2uuO~wOTa) znjKY8?-&TBuIHGUz>WB9Z8(g#Qv1@Wq5IEoki4mo3;*9&Zm1iSFSR|TKNV_7cvu&M z>2gJQ{tw=^dCVx{Z;I=GfjR%3k_VT=hWWp6zkd~P;Qu}JA1m8m(!T#f%7Y6Get=^e z)PIx){P7Dk=q6IstXM@;HDO}vb>(q8@}qppO9+R~h4D}`KwgY~>m)03&gT1HCzJvl< zKIO!A^n|n^7qqqi;IPak7}N?E&jZ^74;}HnR}=oWgm}c|eE=)yI7%>Hr>YghHpKs% zI^h%cglWB@xHwBN`fX<8K-W7;FCoKjVVVlVi|^ezAH$!YuR9Nt>9{V6IHJ&WlT>VD z6rzsO^kDbGC|S^Ou{b?_!vIzT-C3=i*6-UcaFD=d!Kvm9}Q?{?U;q)uB5%sH+~YmHrNOGj0VT3%TUv8 zR^PJXKJUr9P>A#q7P{-NO3{%4>-R5*ag>R(Vfmz312H3HI44%0xhMDpf`@jBV>4F- zhi2Z9j2A-taN8y_J#)O`dji|)&XxwqF|-qC(ObT%{4+}t>AMg|owC7F(>ok}HS*w0 zO=vs1z{ic%SVHRn`wEyPSk=1SP=beBx11HEjjN^JuXPCEWM&9ZFgg>R!Cj~h6JrCk zX86b@x7M!HY?IicN(?|Wj#`gf4hIpL%Dngs(II-M7P58@8= z%i%M&s|AOJBv=IrXevtZ{b12}D0iXC!7-N6-KH!^!I6i@zmmp{j}9v=S9SsqD3S;~ zDZg^Xu$P=rQJ8J7qW$G{S994XJ(~6g&wHxRhI~aGKPfea_cT>l8_YhVx4Y&l0HED+ z>MDfRW$>#=*L2TJLymF(wj-PZPFNp5<@&V0_PI1xz2;B{Y8rARbEP9E(S=-<1^XhQlcu8o`p-6m zlbBF!jq=K6DE*9~$yH-c0`iF=#UsiHnIrSvz&_l`GqSL%FnYf@u1=vZS`x~%i@oIQ-~r5 zVxF9srjN=qD9Cq+XzyXPy3Ax`@rLtE?Q4jBX9;5y#w46MEuxxrY(?zZZO$Zzld7Nh zAYp7XBoK?e_ZDZ{tzEo*CflpfrRm*zuXFGw%Rd zzUiDBK^MbeSVY3+=LhA|0mrBdm4c~w#P+k5MOPG__fkS#;K^m~7pylfyN|r4pV(qv z1!7yEvk#p>eNl&y zew~_Us<*ZA-#Z_RIlahyKW*7HCL}omDm+*@quteSciasKA5dy%Du-qO@9Z|qkYs+< z#D!9LP#_el8C+V59Kga%pQK>y1GD*bjHqbA6+~F7)!dBBtR5TpLvaB6*>-O`qFlne zs{9~uAzo1gk8NjvNy}~5Nr-IUTQ9d33Rr)AsWFG;GLF%Wfp4&LL$$=!@lf1yB4#pI zkCs}3N0yp;JWlwJL~UV1+@C;F%QX@G8E0m$9k`!Zr6c6}f7$bjyhi!O^H`skdS(I; zg)#=WOm-_C(aObs>jwBPoBE=IIq@_+P z;tjE|%w~fs^VY)^V_6&`bJf{Nq<|Pp3?8d#AI#^Ph<<4!qoU^yPemslPPN0+XK?0= zgE$bM$YDm#&2QRH+*RMK3-q5EoI1-<{nx9~w8SQx3Xwy}3Tt*h5c;#V3!vB{+lnlM zXVgI6xCV3Fc;A!3p}hf!EM0CWC*sGDQAkrTU%CW@h+I zS{v^PACeAps9^s{V8#{_hPuIG>x*x@fGxHL1flSa$0Pa^2Um;ppVUIP*EStf%&o8& z&=1hn6*uHx<0>K|VhFzT>PA|Sk+ZT^E1RQvL_G4+Wff&T<#>*!iCB6noYY0khLXlX z88cQ^jDhpEFC*2Hfx+cKe8Rc*8RK7_H2zyDjn9PE)J6FO)vaBxI6gQt>t4RgTHL1z zs+~Vmr(O_0Q?>cQ^~c4)^k5Ofcg4p514Tvi7#rEs9Y>Wz1^%NcvYuGKu!L8JV1HQ? zHnaCQDP`v+irfp$%y{DFcwsu1C1D>gv`Up=>X)E3fehvuEN1U{>?of@`2hZil{Bq8 z(?eQ4rrR0uc7A)rwJ!^H4?F=EIinBy1ef;ZU+KbJ`bt=DXWXNX(Ske`@p|!+<41~N z@;Mvw8-XpQG=fFMJOZ4VTRH(us%+;uiMWDwwEHZB)^xUB*f?q$G52c2kAf>2Ag|i$ zosHuckP$iCc1PVsD05iMgsk9%qkA#4fz7f%CFB6>cKgBQhYiW(W6gV=|IPCwRz!o7 z#qam+NHDqzci~2nRc!vIQ76w*!JWZ(QA{wfQ8^o6`0lP!fUzj zv|~||2fWDBMqdy}h>78$39g8k=(|Vr!FPDFp$eOm7n$tnmcRJ`XFZs_SJX8g$I1Hl z=L&V+QExg}V~~B}^Gr8lETA728}!~WjAte9D5P4&ET>g znK#R)*V~@j@MknJ`e8tUH3cY(A0If|p1jp}zST4Z;?|=d~a1-?KF5YbMC=)LL}^-L49)?(-amT+e6DSx2#;TG#c^m#O|}&By!B6BCVp zhtDOS8Ry})Y!IgXLY-1(MyE<_f>*vqR(56?fIv6NG=C)2pZ9F8Jp%VQZ1w0tL;HSH zXxJ5MPKe?6y2b1!K}b^eO*sH}%q}(d!D{Pny8vTBk>`#`$jw`@HwPyg2vpWcMU_c? z>pMi*hF_FgOTA+ghL(J$-AV{a)ol^+ug#XB5ZGE*n57*sQLgentVAzwofs$(v`C6RqX^Xf% zcKCP-(A3d_^4#|5eD{JmdHVx-QR8@EtHGjAvA<;N&A_R0$3sZ;%}tlD#ksePXfwk2 znS~S~18u#Cuv6T=GLtnqnHfm?-rekmrn&xlbn^w7U8?VU1;xI%a$5T(4(H7mq0kcH zW;_y5w*!`MN7CpCkm)+c(Ccrq5cg%e+EyNJis_wiEo`P;eP!oWcK@vN&o6k zL1eMt16@3e^@d9?XTZvsVUpkXMo-U-`yH_hi5g~nuTP1;7Lr>9L5+|0-zHYzZdn7) zaIU(>4xSsXW?MX2UZJo+>y@{=>AWachY*@>OG7o}_WdWLUp^Uks@>(VyADERtTOwJ z(+6CZ*^qHsW1nC#0!%ZI?kn2Rt|lQg=Ro8;-I_ib_tcr6_W_8GgO^^F8v9TUQgl=8HB=(D$|yYj;=FbL+;t?_6r|z&b=F7>{FeITpv)cRP?Sp@qyb=&}qEU#2D-hN>6*x-Tuvb{K1#Fg+Vn zR+=xUf>z%KRp$?e3vP=%TmXlT_l8n3eTAwJCVEnCc|MfmKCbKPE+ZjvrMKc-GNJWZ=fh*;x5lAJAoojmk|#*6I9#$%KgFe)nBxv2J>( zf>4McHv+4tDLy?#BIj3k{u|5tvdHfG@9i!r&L<_t{>0H;F_yZ7CD01i_g?_E(5Wgd zg#ZUvdt!=%UWO5}DBci6;)ia@_P15L68Z^}m##qbbF7J0cgLl4{E_1&^t84cR#h7X zO7k&{62`AeFMw!MkZVPl!eh2Lk*vN>1l5>lY=QNZK}j@TZ-wm`6*&9c)4JE4DkMjD zBo8D|{dJzn-cvE2#Ba(RByaKOk3RMnMWEomIQe zi`~VPCe?zETfeqs({RH*D?FR;E=y%`ITIdhA%7pu&gRpT(zFZu*(xF>>@_g=JzfG0XRFSBcukCWRzKvhkGB#W;eS36a7$Rw!yQpm z(q*&3jcAipZoRn(w7o&p^*z_{;nL1sd3Gsov{m*HScTpArR;XwLL&RZ<@{p{%;;Mi3!h$+(gO_$(bc@`WfKotvTd{?R9sYwW{07m*}8R&$g>4 z`pYfW+x2Q8pr_HXhu?TYZ6475)lVVT;loy$&z~Hzy;YyM+wF0XooZ^I&bFd__q+{G z?(%I96r&E>HV>zDYeQ~0z@_WECtbp@ z8tOaJID1znoah-cm6DszZUT4l>Xt>>0Siu~rHScF*aib=q}%FK3t`nCH1Xo)7fFT5 z!3k`gG3}1Vf4tKe6uhU4RL(%1YC<0fC=%|nHOj}z=@X*|g^4^bnDngJFUVL(xjW52p z2j2s0a>$xHB*+CT!}&Xc{T}yI-FME0`6jVQvFWLGgEy@hBv!ro%MW?kMbEYw2b(Hm zNvbD9j1MMGYlEk`iUy&z(KmnO4k8h|jJVKo)MzwrIsD(pw~@`MvPlUoV~Vx;NRd1w}GK z67XJWYb7BqS(H|`^X0uqtAz5AzMWQWCkw_%fS%W_ypCa#>#a{B8MPELE6%=NNBz>S|wb%A zPPn^8rO~zv#wFf84{wv=r8I!uZ(0_@Woz|EV#qiKBC5Nn@FOW{dZg4rAf7dsTFl}$tP`{DEa|!bM@YngPJ*>eD6bxc@ZJB zs~*|;-k9U7W!VO?-za}PxiMxJC&z_!;iiheR{De-h0#F$fzTBwBR)YkC!ey@20|i= z-KJ;-DtqW*%UhMG%xzUbcyARbgeUD=g4q??ag2JJDlvOu{tzHQbc1uLIJCal zGu!V%y2EnT&lnWBbpLqZY>!_)%G7AO=fWVQa(Ok0nYI45s4)O`ZTsIVhK0)$V%*Nc zGx5CRDcR#SRyA#KSz~7Ldg&XYFMxt~eMKc)qDp(=p8IKGtsjrEn)r2mq?A}(>@ED$ z(AWeRLWyF)uq6ZB^zRywaUPNk@-3gQb}=xFQi8p(O{(P?cE0spKaY$G;u;|4xpy&( zeY`Ne8f6TpuWbl1?)#{_h$jKkx+p#9D;&<`@SlU7c#`2gBqSul9UpfE*73BQR?{Qp zuwRwWB*3og_CuA%E{Cyu>dQKLr7%5hnbQ&P&9W={(KkVeA~B9r8&U_YuGTfaU4F2< zJ7UyztgElhWiaqxUBy2YH`R3{8NgL+{}Rf%_{OJ9Y%nr5Xmj1$2M3_wJeh6zV8sos zSRZv#=HyEOhJg;UT0U!aVzvz<6t+Qu)Q;LYtRA6DK?d~(_1Im%p_ zKJNcuWU&<0ELR=`trB?-Kkuw0G2A2PeH&r&lE*49siAx$Gqb6Ptgf3X zBlEjB_wJrnE#zRtIxW87D`67_qw)MKeyejVVCbZ;hDd$ML<2px_tde8_^K?t zcK52_!}WE~;g)~?q=ZfSL^xHak1(6>F~V~4A&sLJ(y9)Z^3A|s!{THE?p)1vugb?e zB2q(sT#p>H>^!)-ny82KHMZ%3&Yq9{7uW*o^xl(X@p=M7R|&2;jV8@hkB$&9LVcAp zO5c6HSU_@6vm`UlZz`am-454b4Ur!#T{-ec0EDYM-aOJ3OX*;^Q?s@h=i%%KM;Xq} zjIzp|#N*t~bO5Z~E6d)zexj%5Od%3X6EV5>JbYZ`znNGZKOS(@a9u>#Q-(aCD^l4y zsKt!oprc$@!@Eh;8@-Oq;L?d^hqW48b*86~?K6c+UbiM&A_4mLTq@XE!;4Yu$H%3Z@!K#6ZEC@jFp?Pmk=Gao^oJwQYckL(~K5!h^E>W0C z8xlaayZ?j9SDc;kl83>MJ!T9f(3|FRCkojpvq7Ygv3kp7UBPJjXJ^8I`lm3cC-KPT z+REgV63s2`QWbc4C1cS zfu9}#Tg(nRGL_^zs6u80WI#}?KCOFuE4(Y`5N}Rohl~4!^k-bj9Su%He1tpI_70`s zT&O+%&_O1i+l#>Wi)z-VXg8~MxufS(L(KP3MGe!+NfmxKZ<5#&Sz{`L>J>w_qY1C+ot{dYACr{ z>)xXzpbMOY!=8p{gi9&CZ!(|ouMWj_Bq*hHC1eOcsu@T@G!{-(X5x*tR`bK3vn7xO z5|Ul?5e(B>=mmP5JAiV#rX%l8Q9n3Ka<>rTo1rYiiUTxlfjZR^vV;lBXK zA@Lut{o&{kSx@d1W;30vP$5wU=cnaXOU;8zefECI-Uy}Fdp##df^|fs&aZ}p zo*AzPXELF20arRgnzA|p94SQ2)I zYW$hY=zo2A+Opx%Aju>U-OZhR89#q0eU}<& zDQ#Dwbvd!F@S=p5egtItI7_=eJxYC%FLuH6UASu7^uL=oaM0r8QP7CYl z6K**wM=G!uS(AcvWe=HK6T=42!@WVB4BmFI`9>)z+)?}7RhpAil*DHy%K2EZUA6T5 zHbae=QU^z=hBzL2TC&tr;5tY15&bIBO?AqV@lb-Asji)^xbNt|7t`S@q#oKh>jE7m z9CS}0oGM{#CFhhg;m`6+<+r>(ptZTkWMJck7j5aO&jN|z>U8etmKCIP<@UQddeFr1 zivRcp$D21dt6GM}HOJBx|UI zyZru{K1!25IM04Z@Mwf;$fcP~|HJ!K@%ZpUv_Zcg!aixo{nULFg(*0b4wGS(?Oz-#l5}nLA z6B#+XVm`e$Wxwm{QmGkTaA{qm3l0$UCdV=)PRC#IuecOYM>Tey0nQNCybFDu)|k>Q zdqBB#{y^Io2~=hqj#tZ%n>no!6P;0%u@GNJDJcdbKNptFl!D==zmUYn`o@ZKC*NY+ zT6=O5+^*;Ko#@so40X~+^Klmz1cyb9@s}l8ii1Z?4mFx;z2bw8IvQ_ZaEQK!BDdn7 z=yp`X#xiqX?R^R(eFYuRwivN{luElaW%ZSHqjwS^m0SnU<)2v zI+oif2h!(NEnYsvu(>sf(x<~>zm{ZyRJ!|fszC09n)F4%*sTkZfs14`mU{+$okIu@ zfh$2iFQ`jwFf4*bPzNx3!MT+XmD& zJG7Mq>ol$9At~9u>IL$_h1aj=$GSkiaOJl5Lug|95qXA#~!rW|l zT(=@5mGCP}sS-og_JAtqdpl;j*m0MAY;VvBoUheb1@VcO%MHzu`HFQ&(lK zg1LYDM(w_rXH-{TgezB=Vqjv%6U<)!i?U*GjI8>pI(~S+X}9d$9(_DV`%vGD1}3%^ zf1jJ~dkeh2^fcIP?J=%XUXBPvJseMLcP%;n8nE8!+B$p2+w?fa9=s*^(`1kN;?4z8 z^&963iO6ZaA029gRHgx!^j+$oj91dntr-V41s8?A5za31H!q_5Qk=lU#cEo~&gUq8 zTSc>&S>0c0p}CagHNxxw{}b?>)zQ?-{zt&u8qkb&KKOu&XLisJu)doUfSbJR*kx{7dNK98;S@s)@@5RlY z!KTdzj%JaG$`vNs@ZkFVBn`X6@RstQe%}rntpBr){M{(|FBVX-l~3iEY`+u!VG|t26|D8ON$?fxbtR2Ci)Vi$ z=51$#-g$Hrd}9c-_YCta-#?E&H|?LfN8g5bgK^ZE0W$^+TMvv$wg?`AtT7CxIum;n z$=y}x8h6ixDf}B>E)pfjwvexvS98_79Fdg-Ws5pm#Lxpd<(|8M+en^a%Z(}AjLua3 zJC!y_>p_CI^jfY}dkr@GGc6*mF-FS?^kQeLeNRN4+rfnSY4QzMHe%KAdbJiSgat0= z`b)g8<3|g#!~xV5_cgZ_`D`~_OQucn?RQHGmG9ojT57%%NvJyGzd>-tE1(&Tb`M@9 z-X_)x-$K?$3$~s3a=}{X1Mm;om@b*3AFXVU2CK5cQaW0~yj;+x{92Qr-ESC8za*9$ zt1=xAb=o>wDt%h-t(45zvgT5dS0+9qN*}5Wwp?m;L_SIN>P2wnq_I((YDsjowDC2u zDdg}SnJIip*eJS=<+THhi;d5DYz0k(ms7Y^|SO(4nPb2 zl|F2x#gqZNXM8{H}eyW_h zh&NVnE>^SaHCDKTXfu05cJ=Wva1|c{XY@CbKzt9<_JiB5&duR7+(lIcBRi$6zZ2ml z@ff1&S049KK<^6F=Xbtzs`Iu;DibS;%XKweaLxnN0L+o~Ij@Ih@)Sy=fRQJC&FT&O{T zZ=NH3IY*wnsK_1t35#@$*pjAa*QlIm*T(TxM(oDV|QHo z?-2~X8!Ao)ZZ%^xvnOfVtPG9Ym#Rt;TqB>tbfE!7PK1v?#msl)>IZsD4}ad*yqD^H zoXgJ90qiwhu^lbN0DO7x`2j{t0dq#7>(ci&TeA+(Z4?pgk)5U+sAV&I)E;`)Bx~;; z76abBz{oqhlG8q^v6DaMu0ZA0_guv9r=7Ev`h)bIoF`cHM!+q;OT=R5YMZP^;W@kUzXu6V zX#}J1|Iqf8{LqaS3J}wDn|UksD#-&N34f`6F=h zzz6NtiTOaO`^6%@6t#OoJbBmT=5p#1L+GNorai|@150XSGIyKhEF8~ihTLGW=jE{= zH*w9Q0GYgZL$2zI>Gu2zZSxTGiDk7{+)vw1b>tB@qRgl3kzRXy4sno_GBcOR1c`7g z2Ca+ssV&SP{M7^ZDuqxjR12Toh5pbE_#rA25=I{BP7|JpjKLW}OrjKqj^@=`N0iWi ziV3C|RG*h#9915G+^~EaVc`4 zjK{@qs_O^N8<0oC3ib`+*iuS|lJ``WQFvG<1BL52^|(kZ7m0A^4hMyu%}8w5eV3fT z6>g+UmIwauNMBItRHSw32a0G+pH|_Qgb>QZeY>XfcSyS7Ina+7REMW=6yf%uizn;% zDe_vVY?_d5$kGy0>GyR%*O$t*OQ52qnt7=hF9jF-C!Up<^9xXrbgGFM&wIA^p;HU2TweB?V3*}=zKnx#P`UCAOBq|$959_M5YJar&Kdoe`whoS(;W&@BEkd2rhlV#66Q7~cSqjAXkuCI~PhCRORKwkc z6DbJ?;9N*)z5++eKg4a|_t2wzyBukv~igJd`t3A^_z$2j%Bw z1u5z)#lzA>L*C%Sq4gt&Qztj#O-eg8sl`@LlXtEspQ19$Le)(KVw*}DKB0nYhgJ* zPjqKtVVNK57oFUP;0w4A;52@HrergKE_Rf7BJxGPY}o8OFR}MWO12xA8oI37p0Al; z#Nx2|QrdfnAu=uqg8$Lrm<CoyWc>``^(k;)GL85->EvA?mWq_%;8!0MW5ly`+*CKGOBD?(Nyk~KBK-jyrF7oarEfAsRxbs`_2W)M~((22`*dkrl#3D=2HD`XbgkCTC z_k-CP!^!Fozz7qV%bX95b4bFWWZY=Kh>u*_p_sIpYn$5}ok@)F=b8%Q;)(tYJu13# zW+S-hDm!X)yNy~Guk3p5I&ttXHEX}*E=%opTWrKW3;VjY=G7wSBqg7%f~b0%x;o>6 z?7~Bs$?(~RmRsBV+zAS(>o;@u^bogCUf~Lc3TMPkhaYesDIk6@Hi>j{6_Ud#qP~U{ zGhUhCBj+k7O>o3+=gz0(it=No_k5q#xjMY-%t2-B<2S91icR!4?l#0JnwMrEVM-a3Nc{1&WzZ?nS`9|Bilg}ceLLp3`c82=MUHSv1n8PGeq=1wTM3t>@W!8 zz$-oA7G#5~mQ|`!@_uXIRo-JAe>*)*q1}L5gI*DDC6LEJA1QE&y5-i;>Zkw*gr9$s zj%w*bgf}n9ns)nfqI1)yHSM*&KJyi#lrC%stXH-ttcH+YQh|Q?mSChWs(|i>#C(TX zALvXhIIiEvTZYyrl;G%I>28P+?ZI!2EE%SWix0~$!L z&lN|Dg*j1>$yalEv?ig099H@dUn3#{@Hv6F$P6G>`?Z&F@?_7OxS3+wWfMP`YW2{& zJ)oS>@m|foV~81kGFaz!jzi2xG#u!*G-bN`{isL~&8If(M=a9d?eR9PFl`oZ_FG+D zwuw9l*O@1)q;YJ$P=7D{^U?1C;yW}PrW4E+yEj+78C-ivE*bf52qWDbZ)~83EE)6m zk(Cawnvh8ZBSA^KtDz&RZ*Y@hTuNx8i=gU}y88InoTS&9G}`L=9&y z*{0Hf4F3)OtT4I-7|Lb}PfhVerM+u93%oNE3_xCMP)6ADrnFy7B}cKs3z7QXbyO!r z$Wi-kZ9*8XZt6hng19)(JTK4pq?%)eX!`Tcc7t!LyxEDejNFQ*6gg59S=G$hIOnI< z2Y2mBo`@ImZzh)-WqS&x$96s%v0gqKT~m4DX!CgkynV1yETIGrpM=GZ-`udYtD1vd z$eAbyJpA^<)s$1<-`AGyY)lZA06F&Bn%J&*Kn`^bYz*p^4>E$N#1^XarYF=@`+R9! z0hs0X1AC{*Ey0g`d0_MVd~-R7uLu=EN>HXMf6&$oC%= zYIVeqxD(Btbu`56jizHYmF*N^5DlrhHz6$G()PB#6MrqO()2l);)z8&v7?P;*Xlx` z&VgB&MVsPExSAkRZo}Z_{FzA`K?0jwz~~kDIp;vI0hAGx zLhM<>W8Jd9Dh)B+7e>`&ved2$nmh1FJGy8PS7a;{S@P>#JX{%=Gj5x=BTQG?lm5{= zbx^u5J!wOuF%_87J53jUJYGklxd#m7N*C$wkUYZ1<$`;6k!cRbZ<=gJ>o(FnS&*0V zY)bKC{8T*PX2pKb67u1G{c8kHdlCe6*co>65+z}vDtuGgHIQdn)!XoUZ#$yO7~bh6 z<2hE}r~IX3$2yCnuOmwpPN!rzf+NY8;jLVH-A6scUq8Wc6uKve-75zd(^MA>?jb@* zKzR7NzyBQa=@Z7rgqv+aXQF`5&=c1?D-NSC-hJ02)Qsf>2@i|%+=MkN+LfPE;PvV@ z7BT$B{iot24rhCtkR6B(hNl+S(mh-lkQr!Bq<-+Gr8UJ#H&w`7T0K9I2k7fsLRIbw zo1(pl7No**9m*J{D|xFtuq$a4joOz{HGxn-m$LoeyySCGxz6-auj($XTM=tC5Fpfj z`6A$W1&0l~_oCG3=Nn1AxO@zWLe^`E(fCV8Yb^&uvauEGOp7coq}&x3u@gg@f&(d< z>X39f=VdJr2Xy9if|5IS#AtC@fibn=Yfes>RQcgv-U}n4owGH@pKpoqv(AvI^qI?&Z!`A(+ZGW|7{C5|Y)+Jz1438RUaf3wN+yL+Ic z(B1VjxpZrX$!2=GpD&!mHPmpdvt6gTt<@CSVxSmm@^3wli*uWO)7D3aCR86Qmv7?1 z(34n`gzUmicX7?YFWpMUtw=J|wPl>rF9abxW%R}<`7lwa#-Pu$F|PEN1q4>8ucM#W zlcfhqx(Zl7wUImfIKhAoaQqfX6q)=rh_TRF@!zb|Z%6l51?`Cw992PIppSe@O?t2( zP(xPRZ&43*cxY&DD{iop&gX}}uxAny*`t=(MP9)d;92MWa~RdMVF1T#xAK_E`(Eb` zG;)@3S8CYb#f4i8DcpVx&uTHdgHfF-Ak4hnyQzBb3Z!>71?M}AlSNBp(gr%j{dZxeiJ)8dB$=R8Z zT>)zrOgp)o5eHJbcB?lCUDz%v*K>qPCZ#X*?=)K+iTmRR8fk$wW)op#;UbX3*DieH z*4HTqiFeD%5>^5bZW<5@XUUgSYzVvUsi)$EVM?iJ)}&Q*q6k}p5)#HrAp(8(vwFgD zObh+}`nu^w1QpiRU3=&QsZ%4+pqa-Jf514flrx#2QVUVso@ri~} zb99Q>#*POQrQelCs^inftjSy@)IV3J(BDNYh$rN z$qjd@!NfY)K=uC=_x}|X{NE#t{|PM7iM^7PSY0ZFy|HGS&jer(Uz6K_K~Rx~r7_4% zdw$hzvqM75lYP7+TAqn~c~lK~;gUMH$73fGIhM2W9X@%31MZmwXF>(;ReyEQ(Bqz zhP|C`Ibe)L;h}LH(*OGP{EJ-g#n2gX>0$3PYn(J;XrEJii$Uuqfh}U${*Q$gU;-P| zx+GoYo3Am$bn*j{jmy3iREhg&u3Qd?__wAtav3T_vlMQI(D2xU>MKDN#?V)$nBDN| z=S(Sm@wJeghaN8pCO!v2>t&A9g%wrZ43z}gWB8r=!M#Mvhgg}{-GzB;P?YYzz`xmR zWbl3v!*JbnC;nA^JzUktIpoF#{p}R*VmP?4Y)~w}H7nkmYA2a>3>?Me?mZI0zQfZ6 z<#(RbN%u52qjJSx;Fa3>sgz9A!J3k>;i9W>Mgd?>CXzRB4oZHyhz1tcwr zq{SDp8ub!;@`tVDH+s%o^uBR6^MLWszy3tZtq0K$!?|9B+TA{{y!m#{cW|;baN~K+ z*NybgtiG1@GdnqHj@d&(B_YADi0z(bxx337=HhP9V1*Ae{T4KTy9?Yo*j?C_s}JVM z98R3oH+MI~`Ei;(O`#jVG@l`l!sz0gG`KF6-*j&03n)f{arHMjf~qPy9@Hu)fDb=z zaf$lr{@&um&ldC7);|b%JpJ3XV@&12si_N?et`y#mY$wmAa89e+DqSB)2No;!jha^ zBjws)uwMulxAElLM#D4^wiWvCKRr3B*x-%g&5QAkXm{Ra@-PXB)nnyGYZ?EIXuCBt zXWJbnb~Mrs%)DB<$#Mh%4828&Rq@)~j~}EbOv;b?MDI z>Sb;&^v<{>pOYtPSg29t?zI$vI(83Psl0*@dXqG>DeUg>KN)mn8@4JVQtBeYAnl=f z^D)h5#_-=8?H-4VT+zDJJni5TNGlG`Jz5`_Af&fFu9IYb{`ZW<&<2)Nd|K^~)I*~5 zma~_o;42J^6`{KP3sLPUuFo3c+O;;X^}^X_O)QP>b!_r^+hJb{zO=ZL@4~1LE~#DW zCLkC@!{dn>a)J++I(@|!HNQ0|FMor!x{esT7aic``%kp8a!*N-AIZw#vHL8MyzcZjjxi7gK)NZyrPo`8FV0%_6O*O| zNJ@30C}pE4>DC`1n^&Q|tGI4nduLsYv`u^8O)(F^>ppLoMC09%w+~mKy7?@J+|&@W z%ONf5CN$GMaWZ*&<#@T=Ta@fYgPNQ=7!Hv!1sW4tY3D`|g1@=|Jc4HY2?G0jjD8#n zR_JuRc4e@@0rw@NzR9x)Je%+qe^>hc+=1a0I9w@<{&8y+v!i+Hz??$m){JmRo5ur5 z;voy=2DTpsQ2Q$VcZMH zn>yN^-zKP5MRe?GOLpgSYW!$e4*Ekd5b*fz@w#JhmGzMXUcGUx73`^}KpHi;XLxrZ z5IiP86@9FN(kp6QG{r>pNa3xjs>|MNHR`8S0hhN9@=HojbhwmAs|rZfEuyn+Qd1P0 z&-rWIC!&1QoXR$0e(QS_B@zEf9i*}JX8fkoKyT}{F7h*(%=V1(kb5tj93`doH3c_a zTQ3isJ9B#22y?7=gUZ*g=mzMEeaG)pTsnVrNvBYz40dL%B;0{$=t>@GpKUZTe5rK1 zeyIU1X>;VQ`y$eHzZ5<4} z#V*uGBrW~BT;`4?O#uwnUjFtj!^6)D^rQlSVWm_((p+=!F2fO&l0Qs=G=$0v3&FXB zr?X}Q_{FoD8`5tHc5Y=p8l{r_r;@By(MgutAOHEYM3Eorv-k&AP>hhs*^*^rt zLn)hk@@k|2R`0iKKTDD0QXCI+jKehodUIB5cNcwm7I1B9F)e3=4+Hga^ELU8xg}XO zSig!!FD&uMl8Xv)WwQFg;{FrLPKOM)AsMN354!dmMU$?I0yEvHq$gi(BHLYNLC*l1 z3TGY=hkh!E1FGm-T;{qP=GmCyoW6B5pd9SeBZxA88p>PUO2yUCmJqV$xyb34q+#nb zZjM6yux+b!kPixwR)o!)rsG9V2<;ndD26#<&zyU7xe@wKO|{8)$tezg9`sL1=v414 z3$vI{{R2Ro3~%W#KY7)>Q*u?0S@VhC+{I!0V(d9WV`eNCCO zMl2oDmvt(_WhNru&mjBLFzD8|-#m12l(v<@i!3Lx?U^lRhYec9m78_>-a1HPFPjid z_uxq>)kShEL1s5}=W0tbucym(z3eTblC4)3rc6|7s-w^!G(~4GAke62Rr9o@o^v(G zk)`YcSwowRPwHXnGLQXCovdhJ-^Jp3Y4rYbuknOvMAd;_t2hq^rJFRBoauvzb>qiZ zCw*e&`qhp-)4z0Yi&a@^O#s4nm%7p7={jexpLmO>UWE(^0&*X9{A`g*8^&3p6e0JviwDtfmnauy=+uA zZRKn6Kc@%gfBQT;zW#1?u&Xv94J%uMtW=YQ5!o#ENWVCgf2e5*d+Pt)bWjC#etv$V z^WH4t_n}mQe7@WR8sFk1nrGT-&kCz$M{5-oLHZsf{mV>XrKq3QTKJpM{B!B2C7VtBaqVArot4NUd%b@AoaUDil~pnE z{qoDWyp5UpWrAg+UK(?o{o6pIC?ykrU05jpUss=`r}8#ju)N_Ik2+YuuvLs8EB6Z&M4&L-F?s9L>ZEy=l5>P-+$-n2aUvh?N32G zJtYi6l#Ob^_nLHRgP&MvHtTy`8rqfeSBLYTN|EFLlQI9Vqx|FpM133h%W^s8&bqN=D}*rD-D!O_>wegHuEsHCPQ5_eDmBap{Uk`kl~QK z^oA^ZM2`KssB%*@lW`~2{M`psFQFRLBRF%K6X;jd$m?|$e2@L=rb*TmWDWG2>>0z- zHi4&Bx^0CQ_i=PKaagk)rMqs_82gF``k?>>FITL}{x_PuHpsEd2z8T846dtDSa_@} z((-PoU-U1QZaGW2o_%vi{Xnu5%zgB_BAi5YXZEAmG7%DMe#_SR*bT+MJqlagaM-+R zSHx635i)!pU1GaM>1s6ZN_@!ErXC~6%eP~2KBuj^wstl_patpHlKT*T3tpnPkA{^c zx8@+n*PmZ(I>L(xSEt2;6hV#zl+IS3LMMqY8qy=B)puAoZfV2f)6-;U&XwYy$v|$7 zui%ohN1aYXQ^qKQ{j8`Mldc|@f-8OMLIj^+Kqwj|3H@p-OpXeh>NEWK-}dmkF=i?D zkY^%~zl5y%#~uE9RIy+8y6`H5C;xSjNDdEqMTe6WcSJ z?y)0Zq{P$jbB)PlVlgb9X&@IRERv_MJyGzN26Gi|QN-%ox0b3t1*Cf=$L@%|yRZI4 zHib@ev8fdk&sC#Gc~O^;Op_wN2Cy;g8?b*h2WzC|OfcstF@$hP_@1Ab^5oLM5d+&+ z$G})3z6sjyl6Z6pJsn7vk5}9pKq^>dd2h8YYA;YkMZx0}D>pUD$M?uHW{;I$%;{&#kEr8O!c&+WmLU_gYW@L(; z7!G_!8s+G$CDp^Sj+>z~{ElzxR!rJcQPO6lZp10&+XEs;!8>+EdF}7f-nZx%iv2WX zHjD`1Xc@yBRlEt2Fhtn5sGA=y9v>i$$_jr1`4Do-&gdQ14PIX*#O^!EVNrSP7{GyZ61YBo=g>}xCNIYHPGIu~A&epaG2_{|%q!i(YE ztp)s!{Y~^FA-5mYY>X+qB}a`>TTkr1l4_XKe_nBKM*sA&Wb(Gpi#R6wEYkzUss!-k z>DD+*9ObGOjHEc_DzY&6)KTP^xX8tPET3b(m|Ypu;d15cSwnj9?*YMD%Ipc5dKIp< z)X9FK{x5Bc0uG-nGUyI|6~~yQv4i_4=X!kHGO)?|DU%|=PTmcE&B-IAG`SOyL;Q`*w|gtv{iMwG6$p&rz_b_q*Ny;Pr1-X`Kg|Aa(lmnfPE{k zmo}3$o&V4R9L2_^%$xHf%xOeCC_kS-I(EhSgrirtTkdU#4@dR}&=gB^%o907VwPvP z*7jy7l{aV`*ITZ)rUYQFcfDTzOK>&1%tVz84+=F<4!2A?SVaG3h*4&RE=e-Z)}TW( zIw{&6r$fiQC=rcA_xRPvnHGzSEA%rzv1jqmGPX3#h6Cm1yFEpDPq#E(HJFz5>?ogX z(|KJ9o20NtHH}PhP-^xD)kD~KQng>GYDX{LxLdz!lJ3bZ4i(Vb{k-Q?-RMA1r&FHx zdTB=@`Q2tjbjP)KNp`NRX_V^R$k}M`>5NfOQckg~dDP>x>hS}8rS|h;KNq_mwRoJv z?GR2cmkkMKZ3m_cZ1sZ&%1^Uz;#!VgI@YBF04lGyvbEl6?0vd)EzoZXTK4|3N({x5 z3f!I@o&uPWCJu=!Q5!7~4CuLivUogQ@mbo-Zfm22AyFPq>>V@Emk0OB%=ILSSTp#Z z{cLdf_fRGDL1KTtK03+h+KYX$~o>rg%$wPu^ZBu9xI58IeX}WYMq|%n;t+ItnMd3{+`Z_Flusr;ml*=BsKoU_Uisvz^EOaJ$2-KhGUP z67Kh}#e);l9Mc~##SU%VvfQ%c+Sn*r&6hS+sqa=q@xi!BDE){hA}kO)bD@-D#_(7h zbG^y#jxN=i=4wZ{Ic%Me0v~8i%vCBVDtpn7f4EE`$X#eB9j4?vy6LbyamwfxL9=0L zMre`R?S^|{x-kF3UeQ{il_LD@%l(cmjS#Zi9yze=Q_aC0~G-(d`h-^|MdQA!Sk%Y7S*d|%(&wX9XaYlbR zh7JB8r4?AUTzhia@gePY%SrIA6`s?)!<79vPWsCH2XRT=5q95&uFP!j{Ou3#=&R9K zT_LMZ_+mC#L_&Ps-n;58>)R#EbMFtRPH2!;jF`6zKNh)Le7^!AHZ|UZ2P5-~RrPO{ zL5`{)U*+WbBf$KBgwNUBNmth| zd`zmWQe_6gQ_4hNM5RdV!z~ovYZ2Os>ba2}1Spapth#d#cttW9JS_#t%Ib;OdsJna zEk1csak3fL|B{v?PW^3VreBa1Fk_#hPrcYcS)X_&I6do z8ucea1+m*Gn1$~K*Sod#zPmr8&K(duG>I7S%G!+A?TEgjdd?Xk0|@;U<6xlL4OZZD z-|K$?+GmG-1J4iapvI4F>>}%q6-Ffu7Sd_g1)bA`7|Zkmn<^ZI3e_#wI-0bD7MI5B zb~?`027uPo6keBDC0Abby@NY+iC3g@Q&U(NjLQqhts&Q2gQs*X%Xgz(mNR!Q_uH6L z%F#y%B}oxom8$94W;^@FyGFv=-_I2aXFPZPbvYPA>ki!)fmS%iIt(+1;fUG52SH{& zzS#BruaAXtKYnLnR%Wr3npuR+QJiyXo1lNeVct(_RG@scGoJY-H2ysrvHZ{ zu_pf=hx9Sv=CYA@yXIX*y(3>CgD@~#W}(5;9yEo_xa=G4W9l39 zO_Y|^0!<1wmRhy$4lZw1;ugIznh%!xWd-^YkAj~MOkV_n6VzNWjRF=zaW%n0J9Hn38zkY_hY?0p$ON908z?)5R1=!&f^?dW)BJXtMnB^}nF`Yzk#Ua~i!t zwXDjS+abp^ZFT$|MA=c{UlIP+(|1a1O#+FWER#DC*$7Yy#s405{sXC!l%1QqKU-~q z3n$>RpS_;Yl8#pMF-_D=?RC*4wd@iXBVJA_JId1zxmM)OXlTS$;M=SoN#OV^2lAgv|G(15zz*4lCBLUd=Ei+U2AAeHsE3IrmTYcZARGEj2aVpJ|ZGN`G<2CpzbAi6bSM zutb{w#l~RV+xA}&yKK;3D2h(~omO4@9bfqd=vXP>;r8Q~wHgW-`zILAK$9XJU{bDc zM!8}(lOsZ;)7|M#79bUzrt5sM^O8|ZaXCt|>2{iOXfM?nj^@c0>#!6S+iE|wAS61-@t%ClPxps{}|W&1GpBx=iZh?3ZS8Ss)>Yly&`H+A5$EAz6b zvJMJu5^B+JF~&3Z0V*ZWdk88upKS3t0NnimsbkTrqU$cr_@+&@<*3g7= z$l6Om?yw=*n>39j&tDlYe#UVd27^hVDciA*bk;4f%~~w^=83dDAlSZ$zA=*HBF6?Wl|(cmsJmxo>A4 zaBpGzG_1gnDz0~9LCDVr0qp1V7H-7`yjUf#&Sfh*_5EF*&DQy89NaOfE;7<|E;hdcKe;`mHi?fdvt4{a-;QKktfPL|Lq)^-z9T&07zeF+AADu5Z$Te?PqZQGT^?<)Kpt zV>GwaxyWaqlA`#U&PN&hYF?94gsW`aewq)FD0pc8RI8sdwBmI3c=p`1V+J%*ZVV?z zpm{oqHNaFt{*#CDlzO$p5Y@xaow~j}!8pJ|QEX~6Iu@0ubxzLG6r*HJ8y2Uh1U-$s zGsE(;H72V}zafxJCIwDAh448G?IhRGG&|AzB*osI)Z=PSIWhY38%GcCzj`X%$KW5! z<{^UlbsKxY%UTk8ab*6^pEP08pSd_XvALw-$aNO3%x1NjB|K6W>aae7uW~JVO?!lUL_U^KCbAh30@_}bvW^h z&cEGJkWbW7G!l>q9IZ_QaSBIAw$F3}x_v(oi-ZhiKC%uz4*W6|Mu~z$-1nlo|kUW$W(u%gd z5GE?d?d#2(vR;_><9^O&?JRT;pY!80^TA4d&;7&?F~Y6BwM%VKn~Bkr5RzdwM6*w# z{hZrNZV^b-N(TG6go?ZIJuBw5_6Or&to;P(sT#Y-qbIyTM;C5L%0OzZ()w9wDn46* z2$Rdh2-r=E-k3#UeX%z>@lJ*NPN^r`elfVI(HcR{bpB@0X6RD$EjO-UQ(zmSLjt!Q zX=`lh&wO5bST{}dNIi`nEV*(!4=5b_BEPZ*GgsGT=Aps-_lwDLdDG7+nHZv1XHlFT zTLP19@RcB6s^4{+Mn%_jq z!KGfqxUb$^1Dr=0$EtM9beLkw=M(uYLF@e7l?RmxG@GTkqEU2umf!|HrbTL`}v z$f3GANxcF$I*{WQ*T>zU6kd405VfL`TXIF=pNEu|=4CjJG8Pm%i+T#Gu>d*`Ma1pH z?a(MBoUAr?SU4l(H!^9W%lFaOO9a1jJ+jnlr2yBRI!wjNQ^S$b$G5a5qxxC`Ef6yo z0p5w%CclP@qXwB?I>?o|i@oz)X8^pLsu{9K4Dqf}@D?F)iyQkN5d2Zw-1hV$=g)Lw z4zhW6ExK}Lb5PP5s_UIn_}i+p$`k5!!XL8*wQp5oAnEA|?dy=;if7W-c9FVkwK8L* zM@;sYiZ?G;Pz(7Ek$@domT_Y1c6w$+DV@2P3{R;@#7*oY{B^`0&#etNWPUy#L|Psp zV{*5fJ+h(&Zds9UL8jA*)+!U&4RD@OPCY^2`xz3@k+;5o!88^|%|_>+751^>0n4#t zw%!5PsanmlP9+{a^80cDrf@zLgHm+mi#XcL8J(BLVYJZ7fY3>FCgY+TzBD z`I0?aeNDN^G6r6n!cWrax?Kf(xT7FboKA=Sryyt3A7zBxw@|e{4BDzRro#BjtB(6q zA!oKqb&bdbju0}vg3ApH{~N(d_@|tq2;1jc7YY&oEovLY{|jofqHh=Vq!wr4O%565 zNno=#9&hJ9qe0(l3-eb^tfj7!KJT0tT>e1+ z;Ettg5=UaoA!7noIhA8x>Aj75`bs9dvyL=KaXqQ$ z!Lu7;so&a_n63%Z3g1xGQtcK=cZ*%EWPv6Ll14}jR)93s)i9YbS)GBx9QumJr<*(V zC3GE#JG5TV=LPK%Wwp`U3^SUE9p8hk$=udorlsgfx!P1OJr-CNXt!Cdu#5imEcT#z z5J0og9^HRFokT927&0WZRYBqZLxqF{?e!Za<`ls&Wcc@Te5gxxz67}~VpkYH+O_2C z*?QKd7CVd5pJ?$p(DTfLJ$8-$o%M-@zs6$f^+{fU z`@=5fuhF+851aEDH65N2$CK|#{~kbc38R_+A0tRfe$&PQjPXfz)$01=j1tL!+L$=l z_eiKJGn+UkY{j#9e>*bE| z=Ci`Z>*rr(>B{y0%tnqrAN*}X1-A33g(p{a%%+A?37uej`m+3)65a~@Uq;G*&tBpL zo;{+JU9!~Eae-zVDN?gVoaR8CmbKT5fyv(#4v1D_e?ePe0PfLB#p?+kGn+T+lwv80|ldo$C` zwI`l|-pm3FoatYLenkcMgUoO0O5Wr6mUsr@pNM3*&KRb(+>yI`-$!Y$+buX8ly8m% z_sn!t{Tb%b@&63oB_~>vI56_49iD33Z41u6TpXOpx{h$!_Mg8}B#PmWp?~I#q=@#lw&vp2GF*Tdry1h!TACdG+?N_IA7KnRos5&1&kM!@8+{>tOsEaQ6EPnaqo;Db`%0LykYv@1v1qVW$RD z4uj#>V@>8oQ^&>_7iY5kx<6F|qXtX~vuWHun291g3Mdb%i0=+Bkh;pRC*E=2gSJm5 zs`hwdN|MtyHk39h8&D>Nj|F`nOlg~E2eWrYG9i=Bcf?jaE#fMjrs1aO^?q!7C-r;X z;N`@XuJWLut@*bwC$Cylaeh(`mx<}NGemNN20H5Y+5@jle4%}~v8i*{eq=;EkaYpC zH>Q(F~9)k^p zlU<;wV|LvFaJa5RN3&}j_h!HJtzb@1!%+g`Il4o?OnF0g=DPmm3AR{qr2FSN@PQv~ zv^o846_A4gz_{B9r_lC6em2Yb#8oev(7hcT)}1SmW|MGOm@g~~%9ZMuwLPO>A9OpV zpW~`3%6_z6z)>0^iBLYZAxV%+dI#y|xeqUc4C>xLa{AM)U>69#y* znFEv|5ibOY@GbpZ<$r+*+hk@L=dReSt>!G%4GR&vhk`36ud#NC2179S=2xS<~M_r5ZrAY7H#a*oA!Bpg8ITlm$BZaY^ z?Wm#ii}><-V`5PF{?K}2bp6SBZ&OP-2Zhub$e1*&3fgUJn7k&)wZ|#!}V2o zQDW5Vx2N$|jcFq>dC2IL4!Ai2m0(V9@z>Y5^8uue`a@}2dcJ*Q4+>s{Zdo7=R~4W{ zy_KE5V`*!>B+XYS%qzItwZ`l1KF_o$OX#FZx6m=09-5>Nv~l8ON_ym{c^4nNnH(+I z4P{wnkt8F%s;tL4kr5uYKsK@K3svH{;}JJ=$hgt;C#wSQyt!sJm3I$JzGi}!03LX3 ziZP!HI#X)g)?U=*gnAL4OFFks|V$mB%nydB|r(s&TIIO1CS* zpmsh-A?h49YaM(^ISm!85~I7mHA|bR)S`E!T*+ZpAGOq~eAf9E%q#0ZeO{qA%lr1> zHM~yPzM^{YyIGRCq@)YnN-qLG5$&#TD+=R09fy0-juMA35l(iyw*6FO%4~TMAIC}l zQ}}A*rn6y^%l->uLkSp@FLG)mB@zqw{d~JPJH<};h^C5$>8UYUpv_n$w-3{;1vCx) z)$7@Huke_P#G=bh2cKnnbrxW{AA>YI>xTaVI)1u_!m_D7&s$&y?#tPpsBW7NFZ%OQ zNDgJ+6)jaOfa=EHijT@9XbIM@{vjNZl1Fr_Ee48;@X5g3^m^I;&WTv9mKjm+=bVh0yHbwQ&<*DtXsvl-6Q4 z^>nYl=b8S7qh2p(%@HbO8qSA_Wal01$_RHYWs?!1#89kL8B|T0EK(699e3j30Kp8 zkt)T_lOSmEGnV3ma4RgrY;+En3VyHCaBkk;)wX7NEA}86SnZI@vGEPIVCh{%*>?-U zwgUjx)ZP-C>N_)mvNeu2!o~aw3TSWMK+up?YF0^NBwdGQoAq&FYnpmVzuo|o$W;|= zYv`2;iyn`&q;Qs`6jYLxmgvRRZHwIU>;CIMxK_F^+cgtnQZMWjUICCE$x^%6aA$;X$7&QZ>fr&>5l$x|92d@#E?m^OYv)3~z9Vrk* z8vvwWR9nhvOR-zjqo&l|H|SYa-r^fc6=82LLyY8$Tg?5W+!30`K9_Y1NRKEfsjdPv zWN+4fvc4)@HhXh7mf!PGr`Taf9z>-*-~fXCKF}Z^=8?7merF-h$)(zo@%`SE?c5T$ z4pCxLgyV97CH?aWU^V@U#a?)~qT`bR&x>nB(J-kFX ztdKZC)@vDQz-`xE(JTp~H>Eq6b>IneMfFo>1GMx;GfUo(A4WH}YYqo0O&_VJfGoxU z8;EK3l`s!jOX{d1QTL*m@yXIHW?b?8CmQBG439?UicJp6@RS3+cC3RKl_vL#sbsLJ zaxy}A-0whWy1=v_segL3MntIMg^$Fj!_C;YTtBYl9UtB(4v#Skisz5aKV0;E!R((t zS~#g&9gWPUfy-2giPdn0t*bE&^ys_A>)zls<_T4xR2!C)MQ!Y&SS@_y{#sP;jW(T;1%m| zI{3r%cRM8V`ewIe4Y(NRy1o5F3-B*Cy)=>d-)ps84wdmh>OEC%Ap5qsTfFqv|6!rfu?OYhqyDAflJ!`m~or_DX>m_r|cd3HjBjcHw*Fh+0e<;rKL zm*AtPERZF*;u{Y4`J7emQlpdH*q5=OQ8w0q?p&6P++~Xv^r$e}U*(fO7o7lX0#3)e z52cuSj+xPR5uDg2Sd;bVaFQXwF=Q$VTKz+dVd8W z;0oC&3{5>N?B4uh!)7!hL)hQE!s4Ja_CI5{FteYWF~;SYzhinBP!-Np zwknGMQ)T8T@ly}`GEzA&4t*`>mu^NmO4xb3{~N3PL#pO~FcwjNcJRLnZ~s56{!#}P zGU;<9_6L|ezZ?)H-104jURy{-5#Lr|L1jpsg{E%vLT7+Am{Sq4#pVP(F^=yfz`E{Wo=&_v$X`xmJ$g98{g-99zp`s9ooIbS}aNso` ziSFO`O8pFDCv__V<4KY`>YK-eiIn#HJ_{j)?a3r~N^mb|gRtC;Gf{b8-8+91 zk2ucRF_Js?{@PjoMohe0IOp7CI^w*@9eNOB6};L3G0*#UIjAu+eT0N|tGF_iu*o`F z=Iaef?L&`G?>&6JVLR8v$y{eAD9!=**-_YnTx5u8`N7Mb8h4@5ZwUAH%fogQnkY;| zb$Rs=-}_m>t3wM)0{RAVEfEk(cKuWpeRb4PDFZx-4-vZ1*0;bL@+E|DbSn<&L>3J zQEu`3N^Es@x-Gn~;-*CM57 z#)VE2&=EPw6<)NB*rcW|ZP-j-zYkBLcz1?kB2xS9=;o5MKE4-JCoSYuq;&b6i}Z~xp2eK18b8pI zU}|yz2IWcYivov$F|av2rpO&P!<2s>OqM2ILZG-mQknM@?Y-Fr{rbYa;EB503&0Jb zIFjDLuyuIlVR7Gd3(4q4-B0q5T|sAfD+F+|BajZgH}{6_3mj~iirlmCR|g@i^zcJ5 z=Q+K2&5?`+YIrNW4w0-I>`CcH78jZ7v6`=?8$##dOG?dAE5b#5oMmtF_&!rQUm=5A z&R#Wuyu2vGb}RRTc29x?zPuIzrx$)95}()5Q+~_shC4q{`f&O3Lg|LUJLMLsOV9Sz zOg@QpiYS61-s4M}ei`;M7%s*_Z$~SD8csXhVMq7?0 zi5Ler9b4PyMfPkwJcYWgz&BhJ4&nFP2Ult7I)y$VjeD3Wkyt0eR^uN@@KKUQY=RKotA$S4>^%fEa*weebRgqV|j(l6!X%7L0El)*{PO z(OzPpRlGqLRxhuvHhpgzFaIU7fK;J-x7eS)4M?@D&`(FNv|5+8XFjh)9$7@ZAWO@V zO<5uW!hjf@SVY&m9P2ez^98(KI(U83eEZPjcMf7L@FqkhuJ`wNI6z*+%#OaTm(I5I zctBs&uJJj$QaT)WsMM^X^H*GO7BS4x@(2^a=?hZ#t}!Q%cyq&SZh8NnYQi7`{t?qy zp7+LU;sine`eb|FasYWSXrZrJ^+B=Dh*)o~Youbw_8P>vG4s$Ei?r|U*SV~Cx*hRo zT($fo_8dp!Sj$zbw~`S#T{0@M{`h4=)K>)L5>J1vhmtR=@)ctlh7Kqog{D+MYMPW2 zRn>lk`2WM)TSmndWNo9tLkJ$+A-KD11PH-B1b26*A-KD16Ep;OcWE4gySqaJjrZ4? zd1v04`~AE>?ya@DSFf}BoU^NH*REZ=c0JD*@H?(rm4)iUh*64_LFjk>xQ)Fx0hWs< zbV#xz!|$6L0?4x2xAt@frfUsZXD`!t3i|oy`ZSgQxV+MY`r_=HX)0@HO1Wud4y^JQ z^oIS7eNj(FDH!{Ln%casir2e(&uys+rzu0SUD>t=x^>+TF9;J^DaesD|0F8_=)GG|98_Wbel zcP~Z)6n6%bzpO zhozl84>X-Ihris=Zp9QW$JJW!n`4iNFG;U9ZaV+cAJIu<-!Ipy{GdMp)1rYb_d&^q zeep^P^0IY-ztiM3>$H;W#g5S#8dLR?98?kChj9&HJAaXrU zcS4A-(Bv3=Ph}&-V_q!KGY>z0F8vtMFmP+PC8?~_lb8xI{TvGSmP|0pco>bLGVl(c z@jZ$m3Zu&zf80yBvnN07aN)bHK*sm?`>R}|C^=e=dr2y#a973XXN)%kF;m+2 zcdt#F2m43ho}L|*v73JtE?)VHk&BRbiSB!M?0a{jt-wg0IyVS*f{_^7XLhMng5hB& zes{8n&|Ci;6SFsr0lz-Y!M}3iZ=YCsUZ3V1=(>J|a8c%sG%w1PW?k{R=sf^|c`aLB z%8}r6^m*Xv7)bjv5S6N3@uh8UpM2njH|n~#gD3p;`&9i5|l32g-&>lfkJzQBHi=z`^J%s-Gf z6r}e=3lV@WID-ue?l_$X_hRi|?4)w$pc;PdhSf;6-m?N@HI6@+QJ{xg<}!X^hv)LZ zFk9j<`hF9e?}J;WI@OxMk;}-vCByu~rBM`)p{44yJ9WBta#E?GyzstrR@#Ec?I*BC zpklG8ycFF>46hWbZs3*Unf%+BFQ7f_PakDCa-Q zYpzTzP#Kz#p%+K<`p>U_rOrE-5aL7c`NbiaP?MmTTAqj1MU$5HlbdjkllVOE-JIWs zS0^>{n1Pm-2>XN_&8>g&fl<-s>$mxON*uJ0;>JUH5LsrCd^5v#A4-~H_hNY&Mr|09 z_0Z$`AEy})>N(Ld9cyk5|2Q`2&QCGzC+~;Wg6ymrFg0ag=oCWcLVCvL8wnZR{>SOW z*S`h>)4U0t06Nr!6NIt+o{(*|jB7xeq~oRsJS`C_i%w0DuBV}k_kp7jdA(s@-PYh1 z>gejGs4kVrW)w9AwLOW9$t{jvPGeHpuqcPE4#DNzi#np(j&gux-`Iqc zJJ`nQ!!mA-Hy!ty9PQCS2H5emso0|V#&YdjqQ7Mj@X1qoJUZ&xT0mPH|0oz4$KYL+ z=nLK!zgd?4MmSAo-RZGx`-?AiuaC?igG2c%E*}QYpU9*m2EBfIhM$}`;{$Vc?QO3H z8PM}3t4hfMf_{AAp``@`E6K{SxAfq0kao)1hs+mD?z@LuMA8vT&oN<^7aL|bx%Z9; zX9d>F4sGaF=g(PpP!`(qgKMu1CVKvxhN9bxKSDKG$Sr1s(xLXyBogM1gAC8kZ9^ejSqYl7|RHf{0x7b00=)Ml<$>DOj6W$ zya}}%NKE)K{NPIo^pfwnfCQHH1+#q1k+-Smyi|)zliV5Nd>>2)I4jV!5ttLAi#a)$ z3DfV1v695(ZT$1@i^KFSpoME#x4i@=owvNP-(fo7F7*C8;4`q{jhBXHrK@?3N$s>e zh0DM)%RfN|)Jf%6`}}hnr&=suhlbN8yzBT{A87uXwRK&plFioQ;QU*A)yW3LZo1^y z8fn_!bLGbQzFKNHX5P&-F|U1j=+~}_jmO33!KIb}_PO^7+HaEt83gV6RUl8p zNAnd2T~MtKo}eN2XmgJTEj?kI;<}Xp%|i@5(y4TnN~{M3_~KrLl4;p^gtoRUAK`nU zloV|Xw`IAF(ZagAFlui`4^Km=&D(YY|MlejCVcu&81xF5!k<1$F8PG$3lqRY`;0&2 z4(dg`pr`7S`~%EypCxxuoreQ*vAQ-*&p9M#1m@S#RqUDLd~LDzhHq?V2xZ_x*^Otl zMOV3*G@As^05NpC-j$AV;|VV>(pDubS?mAuM`-1fP0rVI0_I`wD2C0WRHuKbl*i^;cl|ZlQ0HhUDYCJ zx~7mleYwG*U`hXh{>nj0!h^D*-`LYm5=TsTWj8;?O$8&T+wh$qcl178%;pWRf)NTj z?Os&5nUy)>$LVT_L>}wnf>UHS@z@Q``Fo@DOcBTdq26;_T8v+NsvDev5f;|gr>j<6 z53?=A4iOxAsQ>Eg63)JP%7SSN?Pqd^oxd_kp#{qyvL>OztU`=2*&s(00KFS@%{O0`9p(BC; z4$CVR6=HNVnM#?r*32#6*Zq5TaXsy>wDj&B_-~fn1=}&a5ph<%DX~%3&GvSxMIc8Y zQp>48KB|yM-*K9%lv`V=RSxRAjCp!dNh!U8Whe2%v^w?kT1L@8c{KW36V-~+_PV!5 z=RMUTTy}JfhV9LKbSE>6u2w_t>T?a(6r5>4pkpS(`84VGz*U}RN1pi*lWK?!V6+1v zX$CdVPzGOrl(e^Cs4~utnxcND7`O^j(~q6P`7Cn=Y7_H(t2m*jC;{RE&n~GI6duy(W`p`~=Q@2bC%$?Io8+Uyxmqs} z_v$V~Dj$4XI%ED^BO_x*P}>f{M&`{MMyf!<8DV03e&f8---L)kL?;>I{*0=7@z`^j z6`PqJ_QQl`z6SZ)^W;3JWu_t}W#{OmKu*XZrwC-pt1R1|lCR6{A??5niDKgO>cK2F zjvh?!6DjuFPdobRVnv?qQ5$V%jyg`~RqUf2+>JYror-TZd2cNx7k)R)M|;-qfA}Y+ z%SvtWmF{oZkIQ^@QQ>(kuEKo6cY2<<^ZoB?_L5Q3d0r&-9xPI=BeD61zkKB=BTQzt z@t8sHsvVf+`<|`UBbBst$#8Ql zIQqQcXym$9Tf{$)>xP%}mb?}C2KYts33@*cM+Z{U6%8NkreZVFJS|P|9yiXnKeRIo zO;VbRq7ES`=}8Q?+>lz%1Rq%8uIzO~((fk;Q%(j4c)(Ss6F+_+YegI6WrP~8*AwYr zr#0(ncQR*Az3Gbo_*BrEMg;sQ+dLFMqjvI{kZLv%e;93n)BJsGd)2$l-hv&_c{QFl z_Q}5UgJXwsA5Ul!y3P=mnkLbwH#3`&5*@ zL^n!I4#x4Pm>nD3;nFrt-{ivzUAYw8(Wv^e_2W-2%M9~D%F4vc{IrJr@(ir~wU(GI zE)fHr3UIAk=G81EPCc~;-Z&a|T^edWdylSHdh$-H9nPYRS*d*~JNb9~Z+ct3bQ$=E zmg=HzytxKfGbMV)eV#~Ww+G#Is`!s2T+-m+$}Mp9z6{^aa5tb;)3=WtJ8LTA01yH{ zPZ&&DN2c?*5uOrjV0xhzX)nu=%IYU3pa^g;bTFb5e^&2MRB&?l8;glDs+v<#m~;5X z2eB^#}i+-_e8D4X_T)O}9h98!a=jmUAKl6Egf}YQeke z)jaOQu`GR5h`)K}5%=pu9#}3$^VRimDL3I#!w*;c36mR1+5LTF_L zcPaoGAaX#=SUTO_A2k)OY{VIG2^BW~&D^@Ycaum!?gg{OR{X&2j?L(mkxga?UHdOC zam`OEHGo3vQvWTp}A_Cp%Qn;-cmXNyr|aB4%Lkj1lO_*>4RbonE4 zO`wvKgQ&0*I&lz^q$j6#38M*~Bc1zm!n7`hUX<;!_5(Eq8EiHNhh}d3Xa}MU^kNmlN=O`RDYx7)vol?+`>W`mCCZKSWG^?*1>($&_6m(g zCTOhfiGzz|e~yK$(!92Z)<)5riCMOGfb=kE?+-}@_aZxQNZkRf>`1UITbJ`MK@>Z7 zK8cW&;;c1jt95>}-q~_T@uSza+ezh9?7ay^jHewSL)sQF*^C$X5A3c%3Qbh)_0uh_ z#IvW8^gzIRp-OOhFVKvJ8!ET>;gzC0nk&*QwXSP)J z^7w+F;!X92`zQOB5YjDhvM_^C%7{C&rgKxbepuvfZ>gTJmup>>`Mp&KyOWaz*xZKA zsvjuKJ20}8CJwRrbhtNh&$lXT`5=@+->ts-^3j+bE#A?Oip!mBgrA~irk~0Q`JbNX zU_sfhw5gT~S+2}h*KVe+O@^Yft(l1|RsjT#vHW7{ctv4QuHG`$b#YRiVoXb{2Aivj zhG3;>N~(&+wdpIa5uWs1C} z!4augW~mu*$WS;6N}fQTe``FMD^_QzB`CYSFdRZ$bF^B=&%Y|R!21(X0xq+Z!US>K zI4QyN3e#!6UIgLT$;OXqZ&q`%K$#PJL8}&p+UwI))lYURkji45c?+FoEMDtbyH&c^ z0~QvkLaQTMSgDiXW0AzsT!{|_(+H>gb7`A0UtC8!2t0!&qYo9k9MOnJO9Ndo2y(_B z+={wwG5l81=r6?D8A+dZRv_)4jjdIpcNQJ;1X#@T1?-$)@HqHNOMSk%L1YGZ?gQ4c zZ#znTdcB>e!?0Hb^O9)T%SBw3$O2!9b3lM!dR<5hM@I-s;gSM-l|2WL9h2FO@=jQc@2rtp{WlCaJA`u-!fTEoY}zUYw?c;h|=$ zes-1Z^1%0Hjxy@%_j3iaeaI}D^cN)D_5PQESI9ZBs70$K$1G-lw+$W;tc&McZnk6%_*w=cW!;l2nlS>W9ZI z?Rs7H->zTFe$*IC>%xV(Iv{;WY7*n%VnD*=up&}yJ5E#5i`X_vS2J4+99C?%(sZ+6 zbk=7qI!6yAykD^xzV#h1zNBb{KTvQ|oyP^QGy>#O9=(i%uyv;wFpD9Z4t9WV+20;= zTjE^xyj8>hVOpLplStP-M{IvSAX(CtP1M$%jF20t(2&4;x23M*g+UFsADa!oYu*|~MfH4+byZpBoZ|$V8 znoq`s=dXSHnKrK0aGT9%xPK(B?AUui(>1l5rf+!0Akb&=95Z;RhdnD564dyzliYOE zZQlA*=KjUq=p!M&BT?xyM)lr$?A55WWraFNjV5^OBN`2;WAvy~=gzmEl-zII86z2< z)abCT`T}yf-5dA&U%3FI+F-=OWwEV7db3O0Gg>Bu_-_mnFyC@qd-JE-Nk{u3A=zMs zyn0FuCQL2csbzJ*o~(%x#nLW>A^s3SG0AT=eyHcZo1dVaYeV&|K;utVV~N-1+YEzr z){o|Gmk&-(iX*=yW%^2hQcLSYxeGp9^q8ia|V4eQpM0cXZiSO5O2m=FS4rrh;x*m82WT2>j%0Ozr7zed&)LmhNAa< zO~>C%KY5Z44fhU2*F`b*XVj*Qeibk}Rrt!xK6n@P6(y*NIjF|y4R}f>O*Ec%s5mv> zy>xCCgN@F5ntE-u?lnz;=rc%Uw90pJRMsig!xg_wOnrD56v+kpL!6T2?3^_{P*Gqb z1aiyQb$hpW<~iHI4<`a2ZnKVk++ANmpX8qqrS8+h6LHYV_fv0{{*3wSo5-#U!JDL7 z!<&Av7Cr{3lH)hft%OyT`AgHq0zf1tvakju|*{_|GM@4t^&?gh%N<}Q0$ zheZip0g4q8Mjm%t;o+;Eh!1%(JRy+w2jzKC3GmbTt2kQ^fmSLsTszn{l@xELFFJ3`4+As{#S9Cxt%b-`*o z_ks6WbBGG$v|HKf-E=;A4*hWdMgKC##+~vTv|8Cn;1Dx#rtV4s2Nf+*<}MZ!dzHdy z=i-qu{e>l5T{FYMtk(LT#f=pOsu9@yxQ22FgbEz5?1D*`AmKBssgFoX#vSw*YjaG` z4Kx(fG+Q$fe95lm1-XQevEEka#$PD&2G9bvM*yC{KeR8SVns0o8bH-`64!2*ck{#* zI~AQJzsfPib@bKb{7!8aGP~`jc**EycomQ(2sQLYM6awzU88L~uV=EoLRu#}dwt(! zs9S=l;y=!?=GCQea^^iaC?$Ryzzq-vet@rMzBk0XtgNJ7!%+Zrij7Zh7MjTWL9e?) zGLF19)oxp?AG>uAw~E#fn@AofHa=u#j{j6F70{u8K?wgX^-fD#km0~$0ETKx`e5h; zZI16sODt2TteM8HJ2dkV5h{H7+$OE)>zBb^1QdBWXsoy zha4RM7x-gO^UtIc8#XarOb-iJqBxyzV(@t5szkvJ{caA;x}UUNAi0a(**B~8*4bAf zZx6p9^xj-I7H!zu|8T~pIBj3UA+b-Z; zTnn3j&!Axxsc4j&JOZc4+ow#o-3PdKOD*4>9o2LC*_7C={C9ilK|F4@NX9%CawcKt zL#m<(I@ly6;hTn&xw-@yX(bC+Utfn6>89SAuTmp*jmj8|6&Y2uZGNUMH!!oK>&KQ& zjrmjY%oeyj4XrS&Lv>6xmq^6u;buG0 z!=y(GpY_M~2eEf%1u7N+*lWSC4X}abV-f1$$koQswk)-pqmQ`Wp9n)jF|oG|XTWT3UV&NLS68m`F0;FosyTmvgUd4sVHag@x2}h}&fFc+xTH1rK6&gd$96-E= zuNJsF=u3a4)fm@*t^GD$4RF4767N@E&?Da`{Iln1ch5E?Lk+6~sFa5D74$%?4t0=@ zRut3sGli<#IOGQWNZ9+D=7Cr<`y%ET#P3yqR3xgepl7GQ*95n1vRey zkoWD#T>D3>3l+qG$uznJ3)k!iZPtQ9`D+KTSgY+L{cv8&G3jDsFwM{8(*@$OdKx*^5j#X|Q-EjV+&hHD$rerd%@lCz~>H1mLEkZwDp zllZMBGS{b z5?9LQ(qNgTHVVC`sPYQod6nmsUdY=a2%q9(1thbxhW&VyW`f6aw(yO_tYyC<16jHLU{l3zv3 z!gxLX%BD2(^RzuiLrfP4TFxS~=XoT&h1yM*k&U%Do#$lqil`w2oWG&RK48o{Qk8?`MxYU% zUHpq#5xXvhZ(z4%N74y2iZ+vL+$W;xiNO+u)dk=J*ZY*qLHjb}7}{7xv#mf~wjTd| z`j*a&FC1*X30O8jmM4|@7=~$Ph|HKNZhW?-ZbF9OJ4mxLULt~bMaR9KG4|Mv$!)Qv z^A_!Gudp@VIop@KAS>5Dx`F@_sn)7|(z~r04MN%yIS;^|@&hJW&o6Uig#Mi(U&Y}H zh$rCBx2cHVm3#&XnFn+fyL61YpsH znSJ;f?%phG=*8EKP{diH%#jl;>m%sLd>fHVJjFY-Dv)_M8A>FwO(Vq;cQf{NO=by& zT-K%8npn|ocpwKXD%`R7!DQl*DCvsl+;&(@C&bzwb3FgO&B?_2xp(Bg{Ro|?Cv%CQ z@(+S$sCOog2UY99Xhk{tj3_DXUz}#~C`RA% z0bY2~?V5n0-YD2uCxP3o!B5`yoNMuRSC)!K&nYjwjpu%T&JWcXkX>J5bGQZ@{R)5u z-^(R?4YM@fAP*( z&TKxLLdU1mAz{=@ZdvFa84hOp#Zwb&$qi=i`~> z@)zx@C$nsoyxAak$kIUWA_aJD3i;{OAL$yCj{}os zzg*p$VR(7l?W>K08UDE1vWLpZoSE6r)0wopsgGNbti?tGTEDM7yY5p^{}L?(*tXuF z!Kw~wapkqL9LZQYY|{?aS=F92DXE#{5mH@uL_uoCH?-b#IDOkbzJ_~x(;MDay&^9r z642bTpY1Z1^~uSKRfqVW>zhIL5&8@g7^fESsHp0py|Q}tR>El0O~a3D?6m#S!Qhg9 z!FMTdVq;TrBo)K*`O^3&`I*RqP;yWht0{iAICM zBD0^c@K(o9txFh%+3xDuTr#NXi8lZ4jvIA&r!;EJV_xlAY=23kE}+gTwblXF5tDZchYtL-3ER!8S~D`^pxd$h;#d?(`Oku*^6zsq-% zH~$y-d-LGCT&jOp>7PHG;u-(0<^HSeP^)TpHg)-LSYJ^ ze`EveZ_mlmEZ85_EG<#-?T-t`|ARR82@&6(xv-EnRfq)EeB^)Me1+mVqbDI@&@Pw# zA9cM)g@-?OoY8YAsf7u*FZs95eo9E5;zl0YuiyUmE2;h8*HiCPar_bsNevEYqhNQ{ zfTih(c<3Fyg-KGF=z2hjPR!rGbt-Szjh#h{5kpbgG_#2Fug1rYw_hXy_ylW*8hG$428G?tg6zdNG22- z4zMP&kN?{nK`?3R-{t&g2Mm5v+gCCFZ%}=hv=D{=FMISq1^hc$i~j$la^SWZqQ7hf z_vJNOWr)E+o3=N_3fUuUu5KUM!XqNKlLSt7_RY3kg#KRnck!+Nhe|u;@GwFjFP!O< zS@D^pGwL>E&Zug*F`e4-wy1YhcIMA(gj_GsUKc4R;*%<@7gPR+`tkqs0px!la0z=r zpj3}Qg7<50-?LA{KL_o5FC!unPbnGRN0og*85c%J>UeIkOn&z-t%^tc@Q)tdmrT6s zq!4#kSzJC+bldBWjOUGCFUac=ISS?xI|skJyFD&eSNXTtY*9j~LVt%iN$m@U8ui z3V>DVlRxeBC~Yj;gA~y%{+IQG;D0Q{AZ>nE<2nXX-9IM+m_|H_{J%F8Il+?{ABZ58 z_bl4gIpvIew?!xPm!*mxwok9Vg?>{Yv$E}NGKF1TT4=tO{HDV$>E?&-`M0zGT?7^X zN^wt98#HBk;?tqU75Jy|_I-mXqHoTR;^4_5&1v&%mK0x+)IZrD*j*L;paEMRR-d@! zKzh+Ya|VE(mf}{xW0=aT;q$EIId=u~;dO77i636sAWqXGKplF^ohBdg^^X63v9r$C zvE&n@{ay3IbL=mV%9R|JA7P0RY#wv}?i`qfe)G4vo(#3a4q%7g$)kEPDP57+qshLk zS7=UKcQNEZ9SAO}F>)Y}BJBX*D{tABL3z1VukEE7S9${W)vi zC$$#KJpEV)nzlK5vKA7?C9HsEAn4uVn@zE|7!E*1QXx<@*=n6GE!kw``hhSO9N=(?uzuJm)=nqNGP_@K31(pf0L&oi{o9z9OIcVPb0?H}hgMpIsRWWXEY`&6!D)yl&=t z$JKbbIV$bpTT;rx`cu>S_^z06`rJyYT)_sGNx7E9*w)yKG57i*$}7R!E`p|^IykG2 zwd~mi!+!knK~wza{lBjw^j285}~=PEFOj!vdOOSa;kQBAd`oWuau`fJGQ0t9{O$){J=U* z_73#lLWvhJZ43^*(H6a$qW)TjTVxc9HjdEu+!5lKZNU|sW2j)hAq6?35zR}VcJBQ; zZ>~H$!;>%#8&9xjM?acqL4p*66OU1N{)m#BL*89eZ- zX*VD1{$-~6US9*4x#MdaP^YpzL8%gN$0KOUcIGR2YdMmh#94@-h^VvR&)#Au#p2ge z%N<=hV%(mPI|lN0E;60xoR78?6O+Y_lMgvo`=a8&)$zX_nq;_|xqS|IOM!E5F!xqN z2<`}H&7O`~?@%tOe_)_rHBA6K$~jQE>OiOHnIy)_`qZ>-yeG;Zj0s)^`jySU#gB5k zdJ3bL8T6m@rT5K=0o+OD*rqBljfGp`glOtfpUY>?Z`CTNt+=#sAm*5D)zNm#ybX@}A#D%47G3-WUM~!5exFZPmH;W=Kde$u zmaC-{ocH>dmQvLEzM1l1z2`7gAs|oIf*9#Ud#$^2ojZoR>m6=vdu;>-$=~6a%$%hq zWjWt2>Iz|}H|#_u|5B{o4dJJ!bnF{8J;L!sRP&<;h9 z?Tf@tOncIC@U;HU{UJNoLCP|lA;v_7$UgI0Iv8*9R@8^X^*whAMIZN(+2@r-Jh##9 zU3&7}o7m3Y%45+L@_8%-4rY-{O#9%83U`mQU}#OTRhd7S64-f9tOGuI|NN*-?)}Vs zHT;SuNkcQ#VWYQ1MDT`}Ps3%S!C$w;Tx~3xe$%nhS$CSW!^c=8R#Y$<(~ZQX#r3sq zpyEVhdM#!qchX=;Ma1c96Oy4BhM9%)ktf_|EbK9DSBV(5>|F z*jfW)Mo#{9Q1qW&lN0eEY+ph2w(pSTrV@JTi0AOPm7~Vbb3xuFH4}SHarQr-rdVF; z&cOU<%_V{$-vv44=HOrc(`h`DLv7yPljkX@Huk<0is;b>%Or{yb`yzvb`<*|# z4nIBb$;)`(n<0OFdIQb060l#G<*^a`;ayQ%b+QTf_CWkcG49dSa|=e02mIV?qs$`k zl&&VOJ5=SVl0@PBsPujCz$JwKoQB6x{L+l6j8F_c+R-E6d`Ghtzy?T3+88^xS7dzCIsmXFsEEGZ+7+pPQ}z zZzpEw(^*z37*cwZdzOdF&(BtOp!x;=fycfetK0O6jVf*xsRqx6n6{0 z%b>J?=4d^4iB;rYW$pL~hk7zM7z#HX9%i^Lo+^@O+=f%SUWRNTup zqEq+cPAEXGWYoV%)F}KEtM(VFb{7^pY?wa0tOSNLJF|TZ|ISm9%xx3IA0t3i;7Bd7 zouQCDBw+FwtU1|)RmKx$tJvPA?$cq#Oyns@x= z=Y-`k7K-?Lmc8Tb^aadwvyt1tG|6aGG23Wf_BMnv!X!iXDpjKill9;89(yJ83mKZM z&07=;?hCRDyHp?cb6KfT=?lI!6(@oj-k<6{^+R(aynj^sk{nlK*D7~c7 zH8m1%cluhFIY5wIR!^X6s!_g}CaC@=gQ1vyfuQ4_&1RtDE%w#E0QLCmapv8H?pdD} zH}Oy8XA`CNNn6*}AE3*RILlZn;ywY=X0!=9$Vo4Q1JF^!-80e?BJUvqe2M4WX#|$1 z3`lO`I$H|h0m1=A0(rh_PJn5cRM1>LB~)zX6*kWkEE?S5H;2 z1ifH_taY4y!N-N{U6FK+0Yl+BG$ci|6d-rY&hnsOwXI}xA}qmsgKjkiL7*BVpq3$& z^sz+P!OX8;7>s);B8Jf!;<&=e86!*=!0k*xqQPpmwj8_r)e=$!UQ88c zWJ0)Q+aH}Jz^9?~ik+xaTwCVg!e=6e6WuK2uZ&<-4lH}|Z z{H}OgI)W6xhvcp`KrcY*@VfHm9 zBlGR)!j~yXtL$=X=t=(U6tjC=wK&1Vt?~?*E!R<%SsS|CEqTstiA^5lZs^=2f&o@< zA_NFgu3OoI{=Mog7PhU^yjTo|;O~^Sdk(Bu3uqXs-=u;DL zB^e9dYGRTM(BE{I8t{vgS}S@WFu$0g^^iU;RDnYAMRg{-=o69SUl#lpq}S3PCk88Z zU)aoIiF@QoPVVZx;~MozJ|2beo{XcE!3E7!DjxerE_!JuJ7YY{SfDe1CS|VMW9zP8n15ib6Q@2+UR&0ZZ5LK|=f@W#e|H+(=l6jkx??L|T;xUx$FKpWt^X9CICeZS6` zbu2VsaJT}OuwQkSi^~MWY4n(JYUv$7fx6c&t8?lR{|Ro))RW)Wdu}>xu#EC$$@YZ6 z4ylKCo~iTvMzMsM&L*C2qF1|g(hdE!*M6alr2;qB)A6#O8S4c0gN7OIeW|!%VBcn0B0u(&4T~%6FX@?>;KdcI5I1+Kgq#Dmy zFCU5Sp9`~IPYnu~w7S4FC@pE{#CT%-y&FC7Vac6xpR|z*Za8vign6J=Pqn8_@p!C`B?l?1#9uafj3{4a+);}&GeYWllb!blgnQ-%YstQHwL@E zMuj!>@nCl6xaE|pC#4!T5*Q=iZNnDhnsTqvkBJCh6$bB+1$*6mxDdlm+T2yt616A2 zZx)CV6sQwN$sba$Yh6ifqrfm^7y9~5`PCmyH_Cv(0#di9Huwa&iS!RWPkxCj4zD2B z)-nC}G|M=UP3ZpeA-Snzv?Waumu=td-Rzu+mEd^)H&1zC`umX{S$ zWG%xIl$V6e_F%OX$#+gkNR%g%VBnUrZ{!)~U48|f;Yz9@_*Kmm>w@?Yy|Q}4lpWtb zZa&+5=gG!A6(!Wo%qA+K-MKc!5~P&TUoI;W&M(&Iw1btX z^~>%uX7oK6A;0tjEflFKM)gN)cA2%b@9S-bIz8=S7%n7UlMh)m^-nNEo+B9~}ap!IdH zvyoq(K~ruQhyyWg@n#*bauE5gB^}6wJ#UGvq=u-bYH~k?&JhYe3Ty=>P+%tPug5e* zlRj0Q`)HOr#KpIvNNd^K9V|t7b<0~WAA5bf*IcUyW3{@Q7(I!5yOS%aXG8s-7#%)g zNn%T(^Yrsff1Q5F;9auPHjd;@Bj z$D4kxS)VL54J+ff+uy{*A$(bp_Tc~^D`LpfxLc0v6@hWc<20tF*A%px43gis?z5N6t+f0SNv#4#J0*uF;;k9MAyjt z=(8jD5^?ixw<+-f10%7WQWr@&O-Er9=}$-er2JR3bc)YmC9yZf8_A&bO9TbT|A5Jp z9c94??J%^dYUtL-$1eYf2yO8s7%w{y7rHyZJ||6cHL@#iBl=oOeT}MSGhZC=xW(?G zm}3NE4t8EdE|#Rf>v$4c{t;vueDgYx8#J`B@LVT^8v5T2(wfh-;)4=t|3*jULxWEvn&jKyL`b#i0b> zedaw2`jB#Kenj!-Y_1Eo9v)V_)ri${l!9+Aom0WU`Yx;BDWaRnLgjth5v!s6vYXj< z<%`oP^oW1ff;9)YKCQ^z;+v!BO^hUpD+X+T`4EEyHlUI_nN&Fg2y(v zbv(&#F=Erf_WKo+R-+SZ)?&m)rzVbzZ(^W=#Ul6jrTug=@v_%k?k;``#OAx;909nF0QBM6(zS={Hu zUsy=Y1l*8v0?*CD5*Eg#79$%1yNoBc-k6X{f>@4&dBv)Nw_ys#UBLC7KX5};UO4mU z#6#Eh5;nxo6&=9LkgU&MiIm#Fj^0xC4yhre32>L-P}K<$GI|RT3GZ>z?(%Ft`ivt@ z8WfwS&lQ9()?KJ6+H=L1>Uf;Ze^F|;EN`k(6-G2&%moYA7@v*%hkuhfUnp0o?0>PE z0^{fSm5IyQb%uGH+o9+12#2>qZ&_BMf=NGP0iR``Q||H`#OH57UhFbAL+4Y9zzWl=)$%a_g^J=y-0tv8`^z!0^>nQrhVl}?`tg^Myso#CL~ z>j>A6d)RF4m$Wqv=t>3>`Ew^~DHv}%<-`Gc|3DafIOK-i_h_aMPmC@8f2e!wxVV<5 zZ!ihLf(Hu(*Wm6Jg1d*{5Zv8mg1ftGfS|!0f(N(Yu7l3tGK1{o-rW2Ao_+SO-F-j1 z?;kKTeY*Sf>8k3k`gWZvanS^-Z_MI}!nf3B^1cBElAXQ1fSgM`-hbop!tS7QrT2wO zX2}4y0O(1^%5X6L?1YU(>Om z%()Z%u1O^`EjmI1bnTV#V4uA&#{~ zq3sWi-xa@?en&4$L>Gp>b$b;&{HgAcA%ptsq~L)A1qIytLoX9q+Ys|XC#3|qpeLq&l{w5cyay&TnI`>+Fdd{-k%E?))u* zs>G47Q2|bdA`6MkI|% zLj%eM>Ax;~xE(Fawk=GlV9-^lD@uYeOZ`oq>Da8_W1^Dj>l;v0ZSU+H;eD5kQ%y_5 z4g30s1uiMgVLS$-XW@en22|kR+lfW~-SiKgi}>+sOh@pZHQFrpG>ZJSvi#wJgCCAB z|66ipezU(j{B!+pK|YAXlNe5SKRxY8b#_^ak=ydtAN0jo5s&SAuzz@8U$dTZwJ{BS z>f!QDK>oF;*qGA=t=`+c2(FpR56U@JKn?n;ha&YOw2S<06EH7(AL4Xt9~rCb^v{Llb+lLw&0 zfZ#nNhbGwRJ(MZVjS%199L1}?s`FiUJ1V=dPh=C&L`5IDuRdR@tL|>mQO`1yNUqs} zT290>wy9)-y6gT+@>!+Bkap*Iu%U!r1J=lt73XDcZL;cqE&?jCb@uqr={?R|^#Q}3MIUePlax0|$G+Y)h#xN5Tn$YR({ioj+z ze$z#6eHFE)q@dUuh>SrzRGw-uW*y(Pcn1EV9GCT#l-=<%H;uRC3*3##pZJMIU~${U z+TCT+#-}=}15D-mu##$F)@&>9*Xs|Ts|4+FwZNmF2N$F`E#V|Dj&l`1U-d1wTt?r$ zR8!j4)YR`)FqXt_-Lfj4w=`{NBDa1GXt$x7kxS%*q|NYIpK=M_&d;rQpAgH7D3eDc ze7BoAYCnzrK>tEP=3p{%#_y_&N^)hQId=Yo6RnqsL$w63mdGsEn3 zA*kgg1BN`oa*}!HI7<4}`hb%DozoiN42wOZzUatP?4adNOdkMteib?Y5z!tN?9e%hlF0HS`aSA2|7G5UXP42>QlQLjuSQh5>&AzO zKU`iPO$ifyH{4|Tb*$}{qy2BXfK?90P2P}b-vl+hn4Pkkk$u5iJXpXyDs=nlu(5KH zjona?@m3F6U-1<9rtr3xn%41?1)npGP*3i9bn>txfp4TbuVvAM*DPuPC+;1kUw9p!wk2tK74>#HNc9neu%< zs0oS18EO{+TXL4NU`OMl&(*fW%=Y|`1bb;=h_q_v`?UopQcQv7 zz=gvnve)U5$N8!f%*)G-w!?ED9eEjKlh1aX`r{2FI_*NGt5nfvFM&~;-N*#Lyxzy%x+n@ieO5e5QM^cFJ*wT~aEIUs zLkhl}&S@KKXAgV@9E+9Ie;U>t+s*D$FdQ#qN1}4xnss+DyYnFNwiND`MJjs8TGv|- zo!tg)&a#Yz9q=Y)R)NGH@!_dRX2X9C7@c={q-xT1==JX7!UpO?J)bonVp<)&d8O1i zTZ5QvS}~$M-1-ASbsSYqr7^+Oz~sAK$q$9*TWh?DT&2@QjJt_|4|ewtT9*Q6#+?x0 za0ghd3~`D{X)9tGw2?XpOEmIv!XqSfQ2w(F7UzK*LDQNiT1SGv0{m$9B~q+HR)aCz zvQQ!Taf@i+p1ZGCbESLqg|x6a&UR?9$8CfZk{(|=V$1}2IYo4R3SD1yZvO(*-h8WF zTNp=7)T#3}EZkt!T;^0u(ts}x!0Zu1ZHfqq`m+`B|B@i`yWOm}nvM8asqMI66gulJ zlu$$(81Vv@ft#0f5MyuHPmkB@M?)6=4Y@JxcmR4}%fVsJP84w;-vjgKm8Hbytzl>B z$QYW39)XsFBACAJOOL-I7%T}~?ZK^l>Rm`RkH0uDXt^6W=^=RUawu~tJ7f-ux|`jM zvJDtX7j}HjQCB7~Pgs0ic*e!Sk#ZnABrzZRs7rs-@)g-qrNrK z8pSbF*rl`fT?kHcMYO9S)_FNiJA|WAGo?3R;#`LRjA{wYt}yD4Dd7XnAxg(FTODqg zRm(b7#0`=QOqy>D6|TY-c^Wh1Pi>#z0JH|vKGmf3p)-Yt-`gd_k~=v7Pe)puS9}86 zEO{m&Hq0cyB7Xq4-?-kawx=v7kQ|)7EH!S{BT+Rvki0_ZSo5WUJ%YYOTf59v5pzl+mo%g@yX^yz3Q!t(2OSFXEQ9K z!(yi78CmqL&86?C9!h?L5^L!^wvJ|und1OGFGD&FTj*4aCjJLle#N=0-K0~Papw6V zjOJl_72Ij0`(g^`LSZUr1}hT<$aeOX=hJs?ai?pifGOlYo&cfFw;g<;6OSCRS(xJ0 zT9Tk*HNJ|)IrOGd$)cx2#c``UteZ8#@QMc?Iezum0Sg`l7$ze6^8COxAH)ak=Rrf+ z7wp?2YY#}sY*Lq-^@W?|P3DO@BkE%E%la;|pL$oqNzXO(32@qnMB1)*hAFLd3~{~* zc_nHK$2x+?sCkflSflR-T??#jGkM={E#DLj8oj**UsRt?Z5_6e^MY+Z9)JIscJG;Z zw`?s(M;lQMXWI;F?v;rOacw}`RmX`t>uxjIN$Spsy#q%GXU&vRv&AzY1KwsK$1MQ~ z{T6A{O{=3euRLAx$4jf|q#ar1)wj+1#DDF^6g?y@1nW!nni~t3`VCZN^@{*;E<(po zEZf>qR=4rl93h`&=|3cmX7`M?&+NPG7;%moH%;vCbR07``;eeGi77W`#Xhx!F;wv- z{2DmExbVFgaQeVWEzC{G`+i3AkCHz9QPGh?CbYIdLeTe$Y+!Uz?-#lF_^4)@-_?3c z!MPUbK}pe7Q^Up^g4UIo>E{W{wlzBxG>KnFM(Zje_#~Alpu<&G)25)v<;Ip<{$%2mf$TOH136``c0o@Zd%b`zdU($_mfNk+huWDo zh8%c0=OWn1BNPMWGdE@4P0p4yMxwgex|f4sKkb^(jB2&@*6z`+_5IA(3xA#$bvN-C ze})qH9;aq3Ash`qQM0~bzP2{%%2Oh#WVW{R=cf!#Yk7$&?KxYN;GkDpZB@sJ=oaEw zi9WnF`3)x3t77Wra%mepZ%h5|Is?wp`h(sMyvEME_O0Gzj~HoOoh6qY5yK>vBYUwx zI2k5;+}&dSJfCr{n&H8}n8J3%2X>wO{(9p|@q#U2-p zrkRWNSg&7ik=p}urZN}RRo`12-8iagv1L+XhuJrST?c84F((iLs6#mZ=)uGuoz8SJ zWulx156mS7+toiKz!PyZOV|pW2f!6rfU+*5s!4~V zYs*!|0`JUQ!W+zCoxX&Gxl5r!#Gz#=1mooy@PI&b5m0|Ey((yIIm9m_f`-4*t@phx z%)m}Nl47MiA+#e}34S1aM%jW0goXd?&YJkSIDy6MRvqBV!2a`gV!0hMAL{Aq$f>6U zB~@)?V27!-sro!gZX>j(zxraen$!plSIt>i4Je)-$6{6S_PSkbEUC@9NAG>SNWl0>4c;8#qG5vu|na(OF}>B zjieYMmkgQLk-9?kXKaf%^u>9Nmff}mGyAS;h|D9 zJ89i3cen3CTog({D>32ZUO>mpSDhP`O)1U`)ww%UBB;gQnH~k1uA2$aob7xA5@b{F zS9n41e%^(IR4zGtFiuv$(Ygs$xN*4R0>nTG?srEET199CL)3%6h$t2%Y}lo4+CtiL zeUc0$0Y4Trfhwxd${VF#vsp@EW#aMXt)=dwinJdb)#IxVoW%KLM{^a_fHCd!c^u4p zx9nHL5wG5%P)!zS*<4A**HATDNXnZqS$|_JEGkVZHDj{uWD3H}79ZKGfx233vSK!f zh1tzC}M2Sm?E+hbu&g-)H^GIK`XGq>OYe(`0^ z3Z?HaTwn5)qxf`$%{Gh`OGqTnUAzbpx@Ek{`Z!vTU;KUN3j+8bdon{UOL{uLLJL2n zqdmk(TzHB}%|ous0N@1_N??rk(AK?w7mr^DJ~SfOjwU`4+@DNPqZn){@d)=^CLxb;rO>AW>%5}bi}O*FM=2%h9>m2 z%3~nsxXxPIJ`^3}!qKMHPpRi~=FW3$XTO9^RYgJfi#s_hxy}>%vZn?N;ZKXtyr|&H zZ?4jKu+r8dv6A0<}UTh(l?5aw@FOvQ&GrCtp(xAl86Mr*9PgVhFkB3c!(hw>M&1q|cG_O{!?p z;el~r^+NIL-1nRJ3GAzI0viVk`YG~1sD~Arc7s&|ydexwR##V7p1j}x7f$wKP)tDQ z`Ug(l#P}2rOL`TvYUTJ)AkB9t%Pdrl|IKkWHwoXQW~U2$J`uZ@Xn@X^&0@X-hD`bZ z6-GGTJpFjHS6!_q#8#=aeQ%IE5nj1Xo+crwC@4kJm~>XCZz9u-27CkyUCJ%Hw%y-= z&40bs3JRy#4T@~Jz0aL$)cSH>5v%oPDslXXhV}4k%gaKmnSqaEt>@*|)!RGuheiwm z&I6|#L8!yl?(Z;j$A^Fhqn4}FmL%9TX-yGPH#tU!wD3WhFytkZh1v}A5Cv`K-=DnS z$!x9xq-jct$VjSOU6<&)P@5CaC!|ek?*Z;{>1f@yjvQ}J4^vJ1T!@AhG#ic95-~V@ zHQ>buW?c7)@97_fNW39pbHRnud>0?>4t0LqN;QV~6zDjeYq&dzUbIAc+w_nGiahIc zH9f85as0fcm|N|csS?|%$b6vda6z24WVIZ??gUiWZkS&)?#~zWj=nB+o5%)tDLh1$ zK*Nv)7xr+gVBRU<{3w1Gha8c=)Eu{ll65t(!deC{;&)ieXx@5@S-02`=Ww%!fy2~D z*v2p&8}>Xpw>a0QM9+mEHmgrg$FO&JRjN(-3IZ=F4c~8vy8qkPZ`k#Etn(!&gM48GzfGBf>*_gZ*bLeJ?P{4RH5dDitqftP^=54 zV+8h}YQOoh=>HN}lSvTQn8hO$zg-V}m?EPHfT818dVlcQB2qp890RP2@bg+S7yy3M z78R(P7$gpZ#s9O;?-9(ht;?ofiY={A5&ahg?qfEq8W@JOl}SAX!>VEBNB4ISpI5yT)9(ozn@>|)`2T!dyuJTR_ z9N0`a{aMQ&e*Omezs2C&=At3Iw&a>6@LjE^KUfX`lD}yfo;$S z)1CiBy{Y{F0$j7~zsf<#c{2k4>9hHFp9hE=ItPEGm`WzJG3+-A{HF$NLJmsC|91kh z{}&_hk$-^I$#bHRb#Er(!Y(Ld`)&0K&mQrs-Yd=deCFbdh813<;wT71I`uMBn2qe9 zbJgb`bV3&MsONY}z{*quGo#1Y{m<~K2TZuYI2GFL_O&n#czE{CKp4n064=^ z=}DTP74fkncTvh#)?M|*O$4{6ri&DVv;J_# zLEdMW3KG|}^Qk*gYS+;oq!%XAMzGtG)JP8KXGcJN8izr^k?>AXnojYmIP*3Fo$ z{P==$+f~YdV^1~04CDwoili-R0lUg%awGC5!2?CQ=`y&2?PQ#^`^PVzk2|(hW>5*} zDI-d%$pt7^ik*o;o~(v)Rq^fUm&j`B<~@D05c2kKII7;$Y?cVY%K>l3TBkb{zMK}r zIWPUg@}bNyDR&8rh_I8-v!dMfK3TztbLqikCRoVa!&0tcOI)lLo~Rf`xm%6nUGu&~ z#G!F_(Tb73!;piB(z2OlAsi5Y<{n&5;g- zi}b@Qy>XJ^gH^TgXe70iCq9l6c{3xwv@~vW_S5tHtjSik{t>g&3C#gE>s_-g0fV0` z2w5MNcX$`-BZ6lhhwEEf#9cd6-U70F%D5cs6|H4I^daEMKcJ|Y0cW=E5nI|H=$$BL zWaMP2tc(#z+b=meG*dp<2td#ugi72kP?Nt_+-_LXJTN3W5zc_Juu%D!h-J^|MlrC& zK+K`aDz|jtM_c|#JfNyRa(ZNK>3)`^Dy18eg?j9d_z!CX>aBg_JKv;eu#dmDT@Z!5 zS8GLW_KzF~iS`uh;sH)yB(CKs#TC2U-Jzih(j~vk6LhDUp;0oMg5W?NAGu@J)5P+e zY3CFOha$Cz2|UJ>TL**bm`9EDbYbzUf~qcwJ;(=}EG6^kM_We^+PO}A&l`_iV!>Y2 zf#V=qLdrKJcY;rhp{0^ij&o!9jy)>jN!1y>Z7jlLqg-R1Z$-Cf#J#{@R}ZmBLB;O8 znYG2ROlv99cjry%_vh*7Y1kz2p*4zA3eBXF18uHU38mcinKVIXn>FKui z-q3o^G6qQ1Z$_5bZ5uH3&T{EO$lw?Z41e`8YUgrA`IppF(!36cd>kJEc;~ibzEp)A zKepOcTTw1@OzIs3 zWQr;py)l#jEP7+)Vz->8p@GkKUN+}*I@vKwzaqc6(siybW2^}A#o)Kmlsw5b6?j!W zu`}aN6F+DAl30h9odf0F2(`7?t89U~Qu`XQbOt+O3k1WoM z86WLrs8v%6))!@Z?e!-FbpGHX+Xv`3g`sp8N(_b3f~$1=gvN7E$#rK+453GaBkf$- z)S^|FH}hnZWh^D67#z9n@u8rW=x~&+{gpyuwi-P?&v=27}WNhIV=)LQSIUX~1w8Xntr?RJ)!VLC%tA?_CP@(>ABh}M_ISiPZK zl3SEWxJns|`lFiQ3*g1Uz0EKKTp#IFD98Bm%MoBrnN109Iayqmtl<>)3iQXw4%M7X z4jk1L;lPioulE-dzdJ6$o(0N6+!blQ-LN9KO_&e&cI{RPfK9o5TP!xh`=o;Wd9{mi z&Z?zC&W=kQdRRiI&yE}gZwEI8uk^8)wEM8g-d}+5r+GHj^!+v@NxU|mn}x=1-Chf< z8Fcq2dbs;B^N~iG9q&xMv&AIK_oclx;dIyM$(p}t%IN}EnN%*pld;DX(P+uvT|}?A zuFq7+LiymJ^^}-8GYx2yTjrV$Uekg(9oD>sG{CsIO3E56a&GV@WLDo01K7~IR)0Ok z=y!$y^744$`3;{vPf6gjRlB=@)3tqQLgR6@pv#B@$&(b_;1h(tm0BHayL3)i>^zFG$NN)k-yD@XuF9uIu6xY2ACT#gdW)&3 zh-C2TIS2M_!5cDpR*~HE(>*;UTc%vo2Gy2$T!K4J@LclVb29ee#=&CP5ngJN#tAI` z>z|Fq*7&atbCcrSsZNXYZxVr>>cSKmNHpLNzZb%agQ* zJ5Dh&#FKFZW0Jj+fNki+LIpyH@{x|o2MCt10MAfff9(7gl)mdoM( zsK4vn^0CUk4)Js5YtpvN4ar?Ii|P2al1uVRWSsE_2JH2rq5@M*e(s^Y83N~@45NE9 zXD|__R2yO#HyqYWW%rUr*iAN?Gd0CREsDeGXy{0=!DAYsl~3DhXi1XdqRhv7pEli) zTgNMcntTLvuz8+AHQg-3pADl`6v0j~S(ZTH;t1h%jkT0^y+}sd{UW0-c8D@F${zDq zUfr2Yh{A!>EQ+8H7-?Mk)3DCW9@fe#5<2xF<1JO7A+Pd^4yAoyJEMYUfaS81cMUW%;r#f(hm|4GaM2COvemZRVAsShZ?p&8 z3I1|LFsjoYfBTWHFUYF7kuAK)U7ozzf}4-pXgIZV%=_YEQ5U{%&gEg+9gRsGAxy@i zJ#zy4#z&u_cgL4KdQr}qPM13|-2ew`ojy5@iEFiOaE6ULm5-F=YdgFAX3qiRtENKd z%78x=@DTrZ#Xx6=uMWEDeC5TKr)ZQ89}Ms5O1$j9WZ7T7Hy4WdS}8i zLet#j&tP@MxNN&yzt8r?4*AZc3SwuMv~jMr!zGjxL-+gzOktv?&g~zW+rItdACyo8 z9-&i&9f@mnUYZUtKY)NemQNcCI*_R<&o841;8qIb%sx>|I$WJ~5{ytDRmCs2@983I zBY6vL1+=Gge(=rCMw@XzVnV$O&v7NKCLQ2l0TduEcs63JUU^640z>l_B1q}7k z6gTA_dHq5e9@ljFy2W225xl!Q!?JZrUdUUmBAnYH;BU#n;da~{Z6gcJhm|=I73SoM z-MK~fw5WMrvwXc2aVP8YwRnR&^4P7TwnkRx$nlwh2V|BJ@r+6FP0tVihy7`%qj$)r z7wL7C9j9)~>KddIjUt*h^p%e`P*as?8EN^i%L?0kmTes`MJ#2v5uPvLJi28%EK1T_ zWpNwBMdoxR=&XZ>WPqrqu;~9U2iv9HhGMLPd)F0il(#+YmZBUIpKTsb6Z6C%#ZA8K z8R>!w>NdtlggEvge35;(%)KN^7BhgAsnvD@`IxNQNhkj16A#pJ;G?ee6Ni+;4cR9y zC|ExIT9{8@wl$Fc9Rn1kzr49(rvY^nOG>s82pJ2K;}NUlC7g4yrF_xX_d)BR5sOfk zj)D1W0hV{WA6q=;8psP?m1W1hOC%@Q7^ZV9hZ0`9=_9LrY<2FK60fMbTZ^_|43s1B z&6ex5jvp=JEpWSi4AN@yAURSLO1Uf|z#+{YMYeE@yWwfcuexc2~TEIFY~YJRik-o5GXo-nH3l zEd)%9>6l0N4#g+i<*Mw73eCsi;}zB-n}_ebiSR9O2dfRl>C{4&IUPtd@DeA9^T&S4hl-xjWg&4ESU%qhc`yOwoMR$~9t}^#v!Osno zO{&bEdzTkLm#@a}Vq8(2=t}gWp)AXBN^1d41khgsIH1W;Lb8+n?2qWXb^F=0pDxk+ z+*`v&s#|lOOHiis0@{`sHuOaq~#0u zD+{HP5^jb@+HZ$OUM6_48s4Iz^3moF(wHA#Lw@PoE=SW`FB@A(=Q)be4|A)!-llWu zF0<}tl5rPrHZAwja8^Brsz%S-Yi_ou%M;x2bq8cu$!K`609|jc?27lidD5V4Y9z#C zJy*-%ckh?JWLphS7@k4=TtI$;{ReRmhp9=L@b8$%X}yOn9?#9LV~iq#un+XMU03}y z>0FMd;!mgfOz?q;e8w;84~MCxnkdcOKQKpa6UN#ABt!KTLiPGBImhr?S&oozmupEsms$vLG>HvHtp4Ggvh?4;I5~as z*fT%p8MI5=nKu-f(`QM3^=tJ($M%ijX4E>+2_9)p<0o?sVcoP( zNtpl6FxZ3^pIVn=XoMFWKIrUoC|RosOA9!!v;2_NgID788G1-Z@R?ulN1mv!MU69y z16gus7im8=t$NoQ$>>8udx9!G_X#W+S9dajN!5yLYvirnyi;#9emovZ(D{&7vXa(; zB=@dqoL-9L4d?{vWyZAQ57HKhW{2fk1(!_n-A=`#vag-j4Gk0bd-WZ=Vx@1fv4qPN8?nf~*ZBUgw4P)w_4-2vVA3=3lIadj+ShK2N?!&lp=Xf>7`s8go?*f~ z{tIPZRa2B$D6KAd^G;wLY1O<7U0{Z#mnDBsb|sJSec#Om zJR1{HfjS4us(3d--bNM^BNRg+{CZ2^`m3DPv#`xD{~PDtJd#;Tc^uRD?5_CJVoY2C zE|uq#tgR{GQ%g;9CSQuGbV^2Lj%6nbx8CW-i%BFBR-L50o~xp#{$_nO!6ua@1Fj0^ zj9QqvvTn5w&-batH&=bLbZ{|zxUJdn#Y_x@ zQta{aRp7g?4hJ^H59{kgj4<%!beJdYR+}PK_v{Vr0Cj>mH&__Ty=eO01aI+ZYo9n( z^tzSRW|+K-F&LY0jKmxkA;VB(y?X?XWi-*JH&15Wy9UQT$U8;M(@m!wYQ5@_H;xZL=NXQmTC*a3vnzUKE* zjKm{b4oNMID->fu!BFKa$6fEj6CFDT%BhxMzfqu4orr_x`9w0d&<~Gu&IwRz_xEoS z8H_8Y>)Yv=h-5KWR@@qouGrI;@CRL{araDUj=owdtFz*E!Lv0`R>{>cR}n(NS-C_)LIPi(sbr3 z@UjPE`|$+`TT5|xos-O=^j6?Y_2wC^+_})>Mm}~YnbRcdA+TkS8~xO%yxYk4PS==A z_IF>MAYf_&waqw3bqJ0YobjBl3^sVDoFqnT`Q&Jcd+jjRM*5qr7c%|Bw2fAiWg6v7 z&;Is!2?*F*k7meyGashB2RN$amEr$YSW z^uJ#QkmbQs(`X2W9o;0^F`E7L62PI|4!R6K&H@9iOlWCH^k+2lfBeiv>Dk#MU?oQ; zt%=H|QSGw~=pJneE~U;eFKm8sctXi}cjP~GF9LV3U&u~(_ro}bxPao&!tcW$Ntnjy z{f{ezoEFOK)PHV}(1{3`V30xl>y?6k1c&PX76%o<(bIor=_P5-6_pyC6FsVNY2EPy z5zM9T`SV-o{~n}^Lxkm^XVasI9j`!iDWpomwZOCbs2RA%u)kE_$q;{SpkB_=yy@@# z#$(6tevPq~@9AP$#2IS38XhZn+5kHgBMo^nh^IP9yuIwaBz2(4R!t0x4N-t!WK4e& z1_PSE+@C|L?7W{Zp*9Cw$Bgau59)80pTBHHZ;4*Nl)tr0b|5(F9NLS-STli`^VyVD zbXif$Obc5qWv@rIhg6+$F|ru+?S$s;bR7XE%ZkL*zwAj}7lR3CXSjr-Z}(Zd?}T_eu^)I|D2D92qk4=W|fHxakaqL+5{h}e6yl~f0 zxb}RC8V`LMnG9sL1vc>9qcEbj?5MM&W!9Bv zpYht@jP$-G_rtQb9;-fbb14yGmQL3X^RaX(cTZjDxVk{r2`OpZa|jN=L6knxy>T$)&YVmMVf~ zg%o;4a46txIjGa+qzO86WQ(+1V@>5OlS3{VP0i(H?9Tr+P1k-EXL0+xE z=s-0A<*3S?E*SFzl4D}xGc?$P*cP5h?3@Zr;o5y3l3E^rTXV_feeEl@^IhO<)|z+v z-V3@ciOIqLvQR(2?9{rkQcp@d91d6|=HcYUDwWdJzh{5+41OIkcWcNm)Dtj-pm>=+ zo=f5omjeI%QE$qtzT|Ndo!dsf(<3Pd8cV?}vxoE@B8h@p{?$gbLEs=$vgV)9g#jDtz8BK2@;5e4x^(AlA zN#0G?(`HO%zV!qU z>8^$&Uo&&y#*1s6$_jzBh)D>0zZM~<8<7p%V!ql7aoV*;=KRWgHxtJ%46w#+^q+X` z4M;`dQM=<}z+(ZvPY$Sx4Ug}oV1-!Z6-Tuvq4_*k6Cp!hS6YLk4a~n6*%)4c3X0tf z`t`YWI=`~+a}&{KGECwwMU3b8zTq4d-v_r9T5AqY-QzU+=Jff?UFz_;cVf@%Z=$lg ziLV+K@_Hk_<)-F=r{uF_UE>Llj~7)qnO#vIJ$VjgahDDH&52j5P8}#@FnaIhfqVoc zWBJdoi}Z=!sU$0%A8H)mH+tguPh&li5${bHYzJ$u(N?mBVNv!>(%CLdVQftEg$d}^ z-d>*If4V%R?*xCQI0U752I2CD3XU!a9Nr!#x{5IK7Mh-odEX5jD%w9Wmjcc|{&n9% zi+2;`7Xqhk+c)ORq~pZiSb5c3hI2-W*$BzPVTQ5xpPFv@@H*PU3lXpjDccGqyLo)* zVzDNpLmf-ph$uDOVepsTTOl4L+9JSTCX>&!pK}s#yj*)*G6(GdexJP zPS2$1h$@H8FBkw@(JT=TowlqP7$_S%IX9kH!T7fx_QV}h^8LT$#54w+)*cGNAX~PK zo^B+$q*e9}FRi+|QDLtQQtIe$2YX(G$=on7XdaXvhp_)f?0e}CkO?9oxeKX#TdMdyo0UBaQF9Pjus37WotK2UtwEsl~I*FSA zK3xz+t}TWwG6jSVhD^1Xs!dm76q0zSEh~XcX04q)uL`L;Z@AdbV%@j)ULNXGzVQUO z(>HeY7vQgd*73xLyj_jQ`}Uh%oD;cWpJCi8EzMJp^~pWcjqp~YSMSMBwF1~Fd1SP4Q691)a|4R5NH&>*iXmwyJ zfS75isd*z(@o!dF3buOu7~KrLtj&L8u=YF%Gf49y@)V07Jho<&c=#D^=_ z0yjK!cjND!_6S`*IQ|&Q_8q0M;^vP1f%WPOmnU|%-xH4e_=ZbZ@{6qbocj$KAx!S8kD?t)G02UXymVAbJNULnOb0)rIwUu8#{{Y zMukXJ$r?JfrB=TTQBgl=)+D+PY2oC)xTcQZnF*B&gyhLF6*i28uk%U~c+XW8Jp9B$ z1?*`^<_O00XUTycIXl+?&6Wsw5>jA>dRz>u_K^OTSJq$Q7UYK4$RNTFZ?l(642A67 z?fl%Q3w^QZbks5qJ!M`kC)jN(8H0+NywP#Hz`<=?*{B1;{dsWJR$pv3h?9|p3d!+U z*XBEylAG+nH!e540pPcxS4SF*I!&odDMu??A?w+N1U}}ZK0*@>7m*nEx2u}z?zJTj z^fwb3N*mn|CV&gHJ$npex~Fu}m913Q+OSbH`}jL0v{sOBGlvZ;oRp&Zr1V=v5rg5u zC+5~2D>SZM_3YLaNn&-HTm=vIii01lD)!vTRwmC&Ng4qM7*Up~ISmHHe z`LmMi=ED_bcj9i2>e%MfG2AteMk!cqk8;A{x`k^$yyc|>)>}k2vd^r=QWUJwe4{ZK zskJBj$;%N9#86DWOOhA%7ax50%&X|j`+fFaowyP^x*v`I^nB|5d4bQBviZ%qcXNuw z+mw#W98#`G3{0E#S2y~-dGFCYC;Wg!zEqSyAlopFB0|dZuJe^BmZlTL2?gg@cH2gl zX3&;Mn&G;So8JwGy~461kdfYJzu zN{}c&uEUfr*Nk+pv=F8DNf6su$KBO$!+p#sVLf+HF_!s_J;q*4Tk&T?TT`hO9UJIJ zlE!rx!lUmQU1phgFsp|On?@4tEsh^)($GV zB8_^X%3WaJgmYVbo5IH~YtQY#q~dhG;9ELHtr`<*;~`nILUYW*oHSXBPb}AVZuM&) zT~N02rKD!s7}H(t!JEpCxXk1;u+3WTq)$}L6fd{)Oc;?~htbK2)`Lv5ef%%AXwSK2 z8Rz#Dzd)1f>b9BPAob%xu#F6cX_R+BbI*6;?z=y`LW+H|fqwotD=RAwk9sc@+uy-?}Vxt;#rA2}_j|@w)FTHle591#t0O+=@OgxiI@R z(>nDy+I6xr<@t_N9naC>hDKNqf+rAjczSb_pd!PAV(X3&ojHsNxIr-xG2MlND>>_} zm7uwoM)b|zp4)3aOzw6-*qJM0BepB2bVp0geUf$fRzI;y?}1kG@E*&>#F*WzM}pOn zOqQ$Ofs4L@e_bZJp7$L&ex>tuM`V1WBz8sD_bN_xfK>jhzwCC%XZw!I{o}=C9gUCd ziS7;?0t{Etud-V`H{GWC`@3bz(Y)JYZoMqVJ66VtvqKRGiPbf;#dr5YAo8TzZ80v> zbkYiVunD!Z`+ME8RU|P3&Z$@Eg z1c)4w%2(`c_aM;z5?%AdTHoaY!HWGaT{nXd_cUB{DvqjR8}6K$$e_@%2x{5{p_yt|&d18+x~`bi7A_Z-gJ{ zXe3?YpAk)!g;>2^A?)s*p}``UjvTimbm{bM_Fmu(xm4waoEX_OXj(8VF?{Qd9v6ne z0dy`P`z_G8G9VIxXDm1$S0mWK`TIbme=p+9yi}mGH_L}X;r^)55Bn9rRFYDaY=NR_ zAW-ml9SZCjv23TTg|Q8zN3RZRNmU1B!lB6&mnI{ zL%8dZLY*ZpSbH2A_1R@dZIl>IqyLU*;Tl=Sll`Ov6W_^!(-oIg0K;6l%iH6f4UyeR zUS~lV<+=?Ss9b$C`NubTbfQ?p)5J(tzdJ%Oc*%beKorXq@l1*!9r>&R>c)yKrARyX zhzBkk6r^rW%?z>WPJrCVKr4DSN`^Emw( z^|hZ zX*WP=A+JE_^^I`<1>O#@OF|=aa`PZxIO$>H#pw~SL<*c_8_UyOE zX6SWyS4|6dwyzmdmI=5&^VUSth)Q6261=^;Gzi`8G^nql4=qyqICc6%Z{_T%*w{-P zuEd7ufjL(ww{sw;qPVkgZYcsy5xRJn+ve$ViPFwZcsRCv%WFiu9-GtmleT20d9Mrq z*nS&2!pGSv_i`?OYV-jw3SPE8=lEj{-u2&g!mxmq`z3dCkK@11$*wp;V$& zJBRt-X8m*FH|JeZ&?)|@_u;;^9y}f=A~Iq~`j<+RY9^yV@y|sh@+uJ-N7zpu%)dVV z>iVyLTZ3^pv@p%zSLVO&`CnH*guM-WY4Lj;|55hWPhI1`*53bfF_L4n%$Jb=Q5!Z^ zuD^Ab|IyQY+W%WO*z@_@)P*>3z0?I${I^lxrk5!mx35XSMm8nM9{B&!2>;hlfuQT5 zIs!UTSZFAsSzUTKib(sK>FfVNB$$q_(el-x%g<)vyMK-GAiFzgpXX6;&q!F}PyGVw9oXS49a5_4+g|uU3i@>di8e?D#%7-C`dEygICI>>t55G!tbNA12GqR(6oI(4sw}v+qYjAHJ_EbJcTzKmBx{Zkt~LZdWJ;F}QudR_`nbIGsARdUxiFHTKe< zPyZajvPM&SO2K^gETZ8|$Y@G_N))lOn>O||BdHbD`l_X=ng-yC(cS+QNmL-65Kg!R zp=guZxh%UAB2SJvr&~v7d&$z)Y`{GbPoDL@s#R0ML@&8l_B@8Dx^zFme{e}dex3U$ zC8#Wd2sdEpXLyX^rv^?xR8qFtR(eoKu9WT8@v>C_KJ8ny2D2RhK}~&m&athF`~6N) zx?oFIKIY$W$(=mGX6V4{hX~z#s&{aF54rR?$=>fT5fQMsk?^|wC6FW)pn(P~4 z%LDPr^spS?t=_+{syQ%CQ?JVuBa*8RO(FQaECMLk~UfV=-R4?5^Q58zp zw!Ec+deS?84t^VB1LRcG`Ft>_<+X=1+~R5O!wfC}1*jLE+Xph8T0LW?xM~i~&N(Co zJ^0BncdZWnFW|dvm+Oi9fiyInd4+qc=}`H;W_{?QWa%Y=#%9M1;R(tqXE52s{-%+f zDqoy{Y7-&rBF(%T6KCo#*ehri$5(f6nk|2_12gQZVwP-uUYepq#?!OdX#otJI5 zw%F39x+Q^c8{H@jQE;H7F?(0v)DtS^+vHN*{$59d_puNmbk3z@6H9p6n6v*wiNZU= z**+3`=YmEk1r^AK=kr+-NR#!EAbvz>bv%2hTUqQkCA!yI&+MXS&ab8dwU_rkevatR zRIVtE%t(>i`U47KMtVO9*fymKTH4`DPWY4Cu6V=6Q{bAe%xA_1nK>>@uAPT9biUy7 z^^4tYiM)4o^xuyRXHGc+QD+Zz+BxLCqEe}>X>_MZ9ggUsxt~!sHicBWJyumgXmgvF zZCxZ)DcG^|YuLGvMNCx3>yY#cwSQ)w!CU~0swAR$!z{~~{jhyKSpQwi#{aKi!4a&V zJ3oYV$KItM&oApf6C=vBAsyvw(M-AaA}?GU-^*AAaUs@hlzjo;Iyr3Wl2Ckz_AUNKGrfB@hhtzGYbUD z_aWSqxSgNUzx)WaoWY=sMj$zj#-J};hxOr357%WV^L!SaDG^}afOGBAOYV84p)qgF zy@@us(3hVxNh`MaYI`}F9k@2{x0^L@zA%TvZisa2X%^>FhUhjJjYO}{uzd`kMj4yZ z`A;Zn@YHx} z_hx9xcI1sMWRC*uS*dBpMNc5fvl5$pep`b|Mq=OhQEx4WBu~hbN{5UREqdHlcZPAH zJF<7|vXYsmE;l~Sn`f`Y=3wUQI8=U6QFP&^O!tfqe^LWP$$;@5AtT^h#j>^TBM>UBDuWe3L_rCl9OUwYRQSt=}icohg9Jp7tGTkD>~ZP9WTyuT3e-l5-*C{W z=#6M7uQ&s%yg4Io1K$xJRdub<;NvPZZB&tG}LUkIkOaGjQ(Y97u77!y+O0yo*?vyQwK; zv;r`=i0UKK12A?PZHSMITSW0-j3kLaT$~>LT%b)S8 zK$F)#F%749SB|iKQmukMVA`2jSe;xbM0rU8pd4q5{aEQ|`?*5kq7gH*z3TwuS)f&T%u;ru zrfU7l?+EIv12B6D0oqh$s=&`DDgFzg|JO{rYSsQs;#nNZb#xISY{Z9~FRr$F@@o%Y ztndglW>l-FJxM=4=}aA$7{Eju?NkAna|#K+>+&=g|JJyXhD&}qIHr|EGBa>MZp%z3 z^rQ!Fc2%%KUJ?p8Fs#e&WbxD2UoAhNKnczOKk3rue?{@&JJah3>4R_)g3Xs?bZNk7 zg8&)-+^Z&^TnKS?*eD^C=n3RCkmmWdKQ>ns!6#gB-G_kOeRR?l|9hk`yNv3&^@kuc z6=v3jQhI{8U3D~`HKC@YyqN23u1s@PZtJnchJ==gb0O!i9Z{1p5hJiQrUh_mN3`;9UyRQ3Ea9dsq7y=XK~tjhl{|B^Hq?-J*dmAVgOJQ0?x>-T@HKAW zh5Z8#O#cCt!6>SdMre-BG#cB=mc6k2v2ZA2pj}NknayVGkiQjXxgmZopA(S1m17r1PKV*m5^*yA`bVAL z8m#$O|DFpZE(1f9YIj8m70b!G*wc>EW~(~ykXm`8%r2r@i*#>m*t~pYiWz0L5BJG(W76eTfe`|76d5IkAm~c`Af=hj;mqVO(Kki$ zVK4IUnVkT{&%FgK_DvCc^OdznovvRla7FKOr{^^ENjh@S3ut!3ZL%lgd8GHOKr9@d z>>DRL`mbYQM1NX-<+(r}zPnu;cj@{!5l&JT&V6o4rkhiCEJNdeW&WuomB7x+)hY2! z^S2=BX)G;)4=QYbkyU(fQ$D!aV-XgWQ5R zi^zcmF*9c-$^BF{r%sl9->g{DmI+`TXRF1V)@T-W;cUk{q@v+pk34 zH4%Q5z6tqkq_7T<7}#y?I#ue6+b)+T2ZA%m-@j9w^FD3mckJjny3|x#pBoWo*Z*qz z2}8sOrK-L!L-$uDyC=RTzR(=pMF-ZyqIZCqW314!MN&NQZAbwN@WBr3tj3I-BsLR+ z-0`3s>YWJAb($k|Z=osT#bqh_cEvhwINI{v2Iv>wO!j(hqlsV71R>h0TnortJ0WsI z;(rr`nISY(H*>s=Zi31^hCNuXB(d!ul}o#aa|De{Mt}LKa)sm-l~kHiyT5K8hBVJb zVC=(U**d+9U7Qzs~rBPCe6}9!kt&`~z#sf_oKSnKXWHvFjtmMpQvt>)!8bco4|r zUfJiRDL35-aX6ECk<<57o-0aiw?9~&C%nj4eOQ_}6xWK6!ZVl&6KH?;`}xFCK&#=j zY3)(i?Q^e+U3X2%BQKzhoP4&1iZquqT6!H%9et>EUr@WiI7_jKlX0mfVhq0M*~;=itj_9FXX@xec9Gc7x6ZtZSJQq zp`favtw-Da)q#_4A9$YBOm3q<(%2MKN3bfzcu!0`Bm~Xm*ApGv>;Vy;7Oa6}T4iO6 zW`^wt^^z@py5AL_E_c(7CR2(TCH&r$d}%trU_);jd%y7n=ww=t;SqDek-ez99`QvE{C3ybK=;^BfRK zPXNc@*O;v|IT3Ald~={;C;KSG?C&5MqJg!i@2{YQl(2I<65nOdnKGoZ&3z{rGG);( z*FWeiyu_HN_ke{fxa!!=$#1m9Z_Lr*eOGS^XsjdErR+s>ct0bebz$i=+jxUT&tFhM za~N_RaI}vKO`cCIU9CYui`m8_J@TnOAm~>pR3gjijEQuyE>SChN`Owe)7aB9pG`0+#F6t4;3t9`J{mCp_d1jDnfK>W*X#2wxOlSBT7covD^%SmxAlzc+i6dSsG=YnDC@cn@D$2rX<(wSWpqVu-d=snz9A zR@gf8aPr~$Oufnz6_bnEZ;d~1kpjL*8XOh*la;%8b2Araa0T3zlE(+UvUpf4edY*_ z4+qS(l~Jb!tp=p7mZDe4K;=VXm0Ti{t^pureW81LA+8qYc1obWdqI|WK>NTg);j3% ze~lu3048`G4Sm@`%J3ShILGSwbcvo*?EGuX;VTeMsU< zce>T9;KxD^xL`L@)bB2-yeYB>nrwhXd66RiOn6M5%`G-_0<79g{o_SGtvbNgSRG%u z%hrV=^y>sA{I}{bgOkB>UCPykJdLy(p@c|lZ-QWxZU<0r1s`0223>L_h7|4&&qKEbcR>Jno1O?)UpLIwJ?rz(ox( zMvI&sF}ZC_Y{Pf2^MF@pSW@m{UzGy~s|}X8WmMH!KLYuGsy`uVVGu#ZMD$>+L0r}% ztB$hhW5AsIgYyG61U`BOy0IBcs*kU4j(#P}(C^oqcM>hsIF%5DSUc|R(AKIS#9!Ir z9|-y*z#x)5EDwTF9-*|PibjLTWRxVxM0P33Gi+vpFh$M{9NMJy0nfpi ziCeX#v~>?8v!IvV%;X4KYLv=d%|2`%E%{ulN(OSvd&-&;;r$s{V<8v_XuyDuB*A3m zXhP_EC)9uFv^0SVA`bmFNpGuJf%fNHNdBv5kIU7Bm}SB2W)c#dMywtNgGRH2Dwg0f zC+Ng7s*c}r9wh!aT!)$R(l@nBws}6bBvu+QKKwUBcZei&<*9iPgMSq1PUjcn>U|Ky zC9Q<7bF6`%dPGB+?Ijo{J2QixB_ERhG4?x!&QMP~kxWbNUR>bFY|d@x4DUboFz}4Z zy0{y=V{>5aXk6YyBUDIU9-=-*%gXAMx$uJcT8PB85S{$t1ERX|`EP5#+_NcrrVgEaXET;WxI|!3h~?!1YpGul`@qo5PMx(TkHdXVn-wMvGO^ z`*7Lum#wb&QV=Ls4GvixNVOK%v-3=&@zamkv&Y>GGlkhtc6-_2 zv*HJe$h@S~lD@a$r|x6CGpgT{(e2527$j}zgo zf}>it-6*s^Ff58ulubXs$zq+E>I?IVsWA9j5{!~(q z6*dxv3a4ZL8XdLYUj))BSJWK`rK8kf!X#+psA^J3^53Oq^i8S_G z@991#>bS&gUSKpKM{&JrhSkO#GUVGqj*V%w+@8$Sy2#n7+|BHc zWe=rpR32}=S{Ys<`?Y#9it5#bHd0@=x8lba*z@m$)a8)Il86BOlKsSXtk$>|caaplLfS#O6LIEqi!+8X? z`XrcE4Ed0tWtReh-|=!rCpa4!=)+50?esE9xCUUu9Hu{lN#yu1v^jOB)CT2-keOb9 z`=j}JE5F*fyr|!2X(CG$7`hQT*6 zWDWyqZ{T2|+`d1qBY8?aN#15}b~QSwgFuu2p}G8_KkQ^Rd94Q>+2U(KG^iCdIA! zO;_)1T64!IYh%&bmKdj*1qQ>BMCmm)D_X_N67lbT2&0bB54@|vWQGc>Oujg~R^&DM z(DcX96VZYe;K{}OHFSD5Yw)7&#(C9wPQ@ARXQqIb$I;wheubQ-mzjD|22gyw)Cdi^ zn<83xnycr-xg%#jOhndazpKz*+`M{ki(K~{YosBiVu2|ho}yv_w$e}{WuPMETuTO3zK4z+OLxt~6maXo-CcYlbEd#(k#rEyYT+O- z8&F)?yHW)1*6We>S9rCfKh5f#Tl>!qBH6(;%$a`2;3ikSEk@$WVaDT+ldQr@ZuZdnTQZiRvI!BG&oNpu<@GtU*|J0jJCb{;EU9*U!S!ue-_5gc zB@P0ATn2XUj)|)Umjqrm@6#e~T)-95(iIm`7w*)-Lv11*`bJ_RLg~MWtA2-YSc$10 zi5V$}SYe|px|e?*T@{*N#tv-B(A92;_pS;`pT$EE@Se;!f>7eWdf-(jRChzxDYm}O znGu$MW;wGWLO3jBbDTE5n$c<2B@xr6_UxhRV|I)B2zIHfMjCCTP~&RJnV)NTC-^yS zeE0eGqMSSrOGyi!HxRWqe}-g1UBwI{6PQ(t>!);eci*jw=(-APg{OOUr<5sQTX@U+ z(Y_;*cW$|pJp7;E6aTHQACuf(>5A!j9bw%ZQ z;2sdMz~;O?0w2JpruP<}#SvZIKKYhhvc|i8{)L0;gj-o9tbf|SqJz-96I8)V^mERz z+^#%^yNJ0k!R(g8$>HlCf{5jNvcQ+gDy|jb>7C_wsxo>zVz>}t)l>lu@taln)Xw9! zEd(mW&jpranP{#T?;FyAjo`SM55@dFCv8lk()o_8`g$|FepaTymI=iI7UrmL9gb`} z^Z?kdteR@slS!n=jEPfem4~`oGlFXdeO)66KX%1z)<1l!Qv=h~pRnr~8D}^#{@FU= z|4zp=T_!&i75ezph6yR^KpcX;Ozx+OhYfKDN1N>{ws9e{{KrV<)ON4H zOlgHBcFGP9jNN)b)c7lzj{C>xnZI6kQ*XyLO1f78v7dmPcNM%eMR-;;TsEZ0YFP? z!4X-c7i6eJ=MKHm-xWhO#K7(PFt7}IU=CoHTuSoLD^xK4HsK%XKc%E!Qad|b%zvq2 zCA(B#iKBb8b-=#?x8Xw;cJ}9UJ+iRj8|?XUAov!&C4k2SkNm<^@&s&cxI1>Y=A^A~0F0%eK=e-@-qfY>SOfct(f^hF~Ry)T5Y0i%& zs8_zl2wz zJvw7aNVN3Ld%Oq;2-o7JP3Y{zUvsr*3oR?@$xo|9ge-2&oxE`V=wSXRVkE* zEKmi=Oo@oxyVOrK+nW&XQ3bscLOXU0s%{Elag-zf?1{L-XxPX9sBQ+Zu#q8gf-kyz zOXa2w_@qxHLWJXAY|YRNO^?qoYP{%2fJ21epQl7rXXO?U(tkDLoQu2V67DLU8-0nC zTHL`A5MChErh6YGXhEvu*320NafX=lAhf6qguh?12NMSU(B*%-aZvkXwYx9&{C}*n z%Z80elWe^!Me`&H-&tD&kOh~Zxw@ivo0+VXFQ04c^2x{DD-zA73PP`Qi?M|>qutKpM*4Qw)CNm z_f*1g5>5>gbGHGYfT5FiQd613=M_ZZGMn1gogH%`60mRMa)P6SkvBYFU15z!}XfzJ@3 z%q|w6@P|pyWkBd$>`C@Gs<}~#wtC|wV_0l5?7T(AZX^a*?q<=5)>drk18goi@CS20 zol5_(GX<|p`aJD6{RwirCE40w5!*cXrywJcB>-~6a}69IhCy9V+9=)!`a9&e$KY2F8mm$k*wtX-glP z69EHGH7X0}jk%xr3?`*brX#4$n;8@GzMmC~!Pp}~~Q zjqZC~+~PF^6Pjp&3$RvjDjM;r>uo+@RSY9Q;6Xp!{3Nla_gz7L(y~mqFtp~aEJBui z9BxIBL@fB(z#)AM+A!hwrY+fVeKY-pbw2;E{ZeF2?@@Q7EIpYrJxkt(bGo$7%`cfC z7DtJ%5*$kU3U)6)^t4j~@(GJ3iG7=qYYX3lla!Tw%J5IqkoY8Fhf5a?QU$ZPHVJx* z(f8-nGP98mZU&2Q;EXT+;`9ilzawir38K@fYTmud(7%x}-(LyO8zX5G+UfoA1Vm>0 z72ebASi}3ic-WU;fI(i<-}FFjP5ep{vojMxoIgsA@*7x>R|*nOZ|Z+w{8&g&p#U9k=fjZ;A_ae0lK zRECo*2ZF5G3_zjr?s57k{a!a!LNc=rdf&(C$0ci7DTR8}_rjDgSO16rwH+{X()iHB z&h&7Du%q_@14^9k>##Ah^WkY;{uqpZB8WHF0f!d;v3%85?8SstYQ+W0{&BN_5bZe%g86L7N(-%buO3)i+6G zplLsulCp`}A^<=xm|KjPQD-F(GWa+3P3G$yR(zcRfipBjiR`#LnH@rsM_=bX;k%$}yYN@M|C;9In(^F|~HR!!+tA5U?+7LFkKd@lW5wyT$IIK=_BXqqevOD-ImeAex zxI4fU6`Q_fiC}QdO;xjgPe4Hctm4h}NEeD@W&(@peO&~G`;oc^Dik-mZ>KFN1wgbc zbB5B2CfgB-k+ZbAc}UI@Sjo`qG`7Y{og=wKwVS#VX9pkWNfdi#b_UxL&pz$VH142B zs-;l^TW7vAdP$P(_czQPRY8-uS)9M2@@B_Uh_2Qot!T+8DRca%W~PMKvg&`+1=E`= zktM%tPjs9U-6$B%wzr#8`HA#;7cJK&>GpHZEcBxuqKS^(R-z%!VUXAEWr{hiDgkNV ze=BHN;`|9shkntqq^|f6A(%Sxs&C&LpQ5nF7e2m826i??UR8V#cXw|gx{v{{LZYG& z`@A2u@B;N#3j2k+jXkhne{$Tm{YBSxx$zlPdD7txLTo8nMF%6LAe59p55_yMJYs|% zz_jaSG;b>KaQ}2%NyOvjtN7!h7Qj_=I8MKc9Gv24e09^@2a|Q{t;k1fmHs!Bq92@i z`EL;LF-%KP%x>V5dK$rY;D~fLA9sM%rq2rH_ zq;{pqw^`npv-IGMk19GxicwXH&c^>97%Y3*|CZmVu4+u6ncdl4G+F5;{9mH}JF_oZ zOeN#LB>BHC_HXb1YnJ6Z!0VBS=)1?E-^YFVat8ZFM%d_A8r-Xkpr(*MkzbO5#@~oJ zQ_rA8;W;*{rviTE2&2Fh=TPHCzM`;W$8j# zXmQokhXZ()r#AfLM9_!Eu*+9wnp$r~p-NW?LF`mN~!9|y0ih@swsQ+_|la>37 z_kT9O{a-sNssDQ5{?Cn(1=8Zbd&U3vw~i6q6w3b`54E{3fYA*~Ol`|5E_5J+yya!T;wr^#A`Y39~HrX8U)(LMk-+UpgI1 z{okda&BgtHc{oG$bVCX!1A$F$V}rP!&1|y()9>oG*w3D94rY71zEr?D z)Penps4yM)iYm~j=9{ApIq-!h3jh5ENCU)|kPj9%D^W(K`=`_ZZlfRF+36&xxHKYn@1@#H6G?Y!Z11>dJ#L%Pcy_oeKlZ z*letn+e(A6I>Te!$rzk74kmzQM&~V1uta=foWiRTL>0I1mPhjYD*cSCp&s0YfY%j|zou|W%jF8?>Q2lY$ z)i`d^DNu@J7k+w^QAd#<3rhp0%IksG2p(AdOwR~KWoeg z4;Qpn{CZX+H=4|URv|26hf|k5Fo^8(x_9#nOnerL8YgV${XzNU$pmzhgS>*h>THs($r$P(oR295E<(cxKg&xC=poriGwoPo0;`oSvaLo)fO@?-BFONNXb8I! znw0d$#T-S@7YljJ!LfU?V+gYk>$ixC82Y_ruap$7J5_?CGwI{FwDH(h*T+IQE$v$U z6Gpb9JnP?rYKMnC!pG);Dw7&6dt#2d#CqrWX7hf!Cp)jy!ykp#1A(Q%oe}lJNPjbQ z{i=sYX|CB-9CP9eYF*KiZOB2b@@G2y;eC6}ZSNkW%ae>P%%5^qzoEnpC9Zltlzl^G zEQqRx$ON!rb$D`NUV2To_uw*lVf8`YI7+etHor&muq5cy|8@<}=Gd?KhB4TqzsH>9 zL!ar(H|*T#_{ulBTt$xy66yd=NvWtG+zb_IWofs?3m{@WwX3DHUT2)NRMq3_Noth^ z7HLbw@jue|UII-e9%jdIKOn#0i-UDmME<||hl|rIJQF|oBz#N9pO`+Do6qI28S&h= z`WXTLBej6K!LQygYWUFN?UT5^T^7 zYl>Pk>-@@4(sYZrTW-*?L?Cjk3K^<0@`Sp%=GW0|)gj-#HiHBvEo3pV3JPJ$X-mc0 z+%iVyng-^SWk#<-mjb$`ih)^%oJ#FMy`g&~Gn+H~OrKbSW!+UoFMQm|4i~#QyJJ8l z0K3bw&#VTt{??Y7py=pA5Ink5h3mS*c1u%XmsjvN6&t&djqi~-Uwd})bcw}5ZZhA& zm^EhU`Hb5;c0Sw{!JJjL0ZUbj0Oz~r>kX|+?ie9xdAD}yO@WxXp{9r~=gzy~s6uF< zi+XA-CJj!kLqyn>GLPKI6K=Vt5XxBu?s}ap#*+tNJAJ!c?S*+wrbAC-ljq>b-{?Z` z$jWdm>`GLz(_d%TJr%g4p`x*f`Dty^gFzjqw|g!)634kFH3EHHw^HAJN=3EnzEq4bhA`bh-60?%64{Rs?JD47GIne*>pd|*^uaN7#4|(-dns`pOqr}?V9uKaIurV zi;q2c?z&yh)?r2VUed6SD*6)<=QpT#yeXX))pDVPvn;88C8&Z7o;Z0uPIH4H2z z6`%Nq+CwTUD&LELiV9DcZqp+&Bh8VZ7FEMt27XD^w|d$IY9h~$O$=XfD278fx723Z z(&Vs1+)`e>P;v$af;qg&mi}OxYw7#mfU4@)gP+sv%VR<~--qy;14{M|>C@&$-j2IE zl+7;N^Eh9$f11cb-o3>Z?DIli7XrEDms&D%7f!9m?@*tFDX(#F!#^g!M4T`oY&B`5 zhX}+s{njpCdV{0jh*oRsh;Gj!>GNa4(r(Q@32?DlQI@MOjEmrEtUgs@>-}pCbICup z6_`@;;#06p>VCc+IMr#$ys%!yOs3s?-5+FX_Wx zuYrO;P^C=@VF{HMm6j%ZZODW;5ADGs0Cn5)ekzmq4#WAQ_e*c)9V(#2*D{1#X6KN8pGB%JteZ_%Dseor6U@mgG6kT6+d^f z1^R)7rceLS?9hIssB&|1BGV2hq339#Wl>Q81+{r$c>vJGR;psK$vUHAUcE1cfGpVR zf2<7Uy6D7tkcX~P6Q3rXklH#dukixq#L5c+DO$Cj-jk}>;OKWkeqi^EVm@F@ukh0# zNP<=1aFO3>m8AiYD0c&7eNLgo29|lzS79tJb2e>sr*_)!>^HlCOV3IU5g;e9F=6Sy z>&#qePHg0`vr_WHSONffb3LFJ){9LoS=nArvzXRQs!c@WlN$UBc(>cLu~q4;+8A0; zJOA}lyRNLO?xjHwfg!#-YxEwGY0cxA2_d*`rtw58{5W0=e^3`}sYKQdKQ7BCr zQWWocCR-#+;3qPK!7%0(1rBtUH%m@4#et`?a1<|c)WydXD;Tp_U?bOCP+eZDrZ$`# z!-$cD+J}6F#L*#pM{|5OxxCQ@yxmwLeRAH%p>Ct(#Eoy|RyQ zi*4eIj7~$n&t3cOf^+WCSKXQK?Fbn<{Xv9&+{~h1DW}<-m)$F+JtlzC3N4}a_Fk~f z7j2UW1kx%h^bD6+sxBp``xB-{Rx&q3VOBet@iOUULb-1z;>OoD(p0wFoH3Gp2FS;2 zrP-cShmH?=7ym#~wqJ((B`0B;E1%^kxM%@y>V%0GQ{XQ`HjS^LhK-@RQ_L3Mutf8g zqaKfjg+z@iT_(E(d;C2P$QOz{_V|DIr`8T$LGy*3h2b365{8B8+{#JFD-2vO3pN~L ztLk5xEr#Ut$EimZu9Lq^h~Z1ZFD0tuz$mP}@oPVDu$)v#bMpn@l_{>+4*Zb~$Wia+ywayDP~n0bVV%wJ*R?heQ;4yY?0)zPo|#-pn4G zo#NJF`Sy_62R!nifCI{s2-bz@HhUz_v8(96(Ee^rn&uTuyL_AdJ61$4ld_T~2D7LZPbFGO>L8Jk}Te&}C zh>$r30grwSu4Q=Y;`3p!qI;{EYH`)7t}p61s{Ut6D*i&cyKKVT{`(d@5^TM3E~ejO z$@EEK|CiZUv#saHFSbvcbO$~ItoN<{#7MgbVg+kwxovfE$q$hib8*nnYz;p$wHs<{ zECjBzF{&&dw>usa=ra6Kv7pMwpH^EKm2BV5UU8OsJzL%2$edn0vqpbTdlvoL{P6k! z$Tc`>Vl4cm|E#UwFY7FaWaE()FUa&Cr=%WdDoh;KRP<11<-ZJnwH6NEIa_&@|SKl z)FjEvu*nL%PQ-=*7}$|Aa%g7nFWx zvi4**c$P^{z-f(PLH8REs06h%|tj=z^ zL9~yLgM}#eP0MZhzi!a6w)&lm$Ub}$h5IFw+=obAZm50T+TxUp0VZYZInxM#pxHuj z;V(@c?l}nsFsQFLPP6=<-39ZEnYqVsg#Ovr0bE0FJGk6?bf!T* zNmUToHY-C)kh6sZ8%K_txVe~3zA1=7n%(_P;?##2ZZGU#hY72dTAB=R)V+rWIeikf3f?!^iZP zBU^Xymv}*66fn_ep1<02IJU7pAx8pyyb(T@$9(PfH`l}bkpf;p+`-`_Nvx-a=ms^!@P2j`FUKja)ym2W5|ddNfG4vCiLmPUMh z&fNzT`zs2UG&d2tx4AoajYg0Us#yxtnj;&JANTR*Pc!KLP#O=24tHo#=Ivi;WTuP} z)cd>(YG!>|I7r^+vqo`MMYu*sZ?@i-t;crA>AYZOFehHB;;!0peW?4!&32Ph z4_jKEEve9}@H$41H6djZRqKrCLF6@&UB2Bz;7Bew77=VI7v}hrKKx5oh&@z;M1Hl& zvbQg@Ae7``(Q>a}*C+3NrmBW5mKv@f)dsP_g>_M$n84GOVPvsLgbr$fjg9yiTQWOsOkBBa{uF=1fABNyJXSCCQ zFnBFg_BLCg26LRU2v+>f8`8%bt%^xEUNAWwvLxe5*7j*f-t= z^+{SEIx+do2eSDP6;4|>|6a1ILw9OH-AvOP=}cj*;=Vqf zOQiVdpTg(iHru&u0Eb6x=3M02v~fWmo8CiN)9)k-Y^X52>548Jg}0a@0u9wx7W%X^ zeR`_*T>h-oXDDc^wjjkx=f3`&`YhHL!zPo;9QUL7hToEU61mYs<76$Z?0~ka zA0)|{h=ad6CMGvDiJEz)xU+HH`cO77dixB=)ohCU+&D7?C#{?RE_Hy|lF37rZk{+t ze7dS4+lH&qJBd@GYeQ7q_kFXx^GgOephze= zAzrgN9^1^^7B|H11-Wb$DbR`bDZFQ|Z-WW3?14H=Rznq)QC2+XQ zGb#ET8LpXi|D!b9k5Zz|<1FNoAFY@;x>tv0>4KJ&CQA-y-9oYcZwK2`PGy_-0k+iu zw)1N3wb@8-Yv%nY^dSjz4DXj^%+q1;7w(R=+j;wWw z?(HBqgk_Ej(-}t(+MQJQ~W@ap82hQmV?CAXb@nKaJx;DL}jw^hAVxlk&k}kGf za-{DQIrEEsem*<>Zk$n4hp`p;WeDhnwRA9bp|$1gJ_|BK(W1{xJiB1BMSq+@D~2fd{^fQ-j*Ea> zG8LPCl;TDmwgx>VG3vIKwV@()P&6&mFU^Mb9yKmjV(s21BRdzw!!+S7CGG)n@PyIf zcbF-bLA!sB{OZtP`RTA;2Um`CP?2bNnkDW~$LLk$o`3Vsv1 zKhcjfzWA{q!Qdmdar3R}@mEYoogY$=AsdsUAu1 zj^MJqpnk@x-bhnVhLhe@cHMzB%QFL4|H!Z%b3(+yLSykQ7aWt&ssB$91Hy=aA-4!` z2Tj)>swQZd8^B;}AB?SqSM-vTHq=KH&pg)k5;^V8c zmnK)S$B+!S!W2n-gZx?g#An`G2=hb4aqT-jEsJT`H zv}Q11udE7l~UHiS^l@Kwzn<}{LdIn`M5Xh z1-KFh^Z(7wWeRsSJg3E3wefeFkMwOHFO-<$EaYq3T`(@pEHohvem)6qF)yGBXgm}K z(l1ZNbaVsO*#kX1mSU34snz76XHncXw%IBlp(^}=1=Ux7G^23Mj3Q;oK_Uf8q=Je)9R5av?fwamrGOUqnsZs|!shd#(8k(~j;X!hjo+=?Ofl+((d= zKa(cHmL8WMkh|4(sH8S0ihe^frd<~{t1UE;OTg*kp4fP2n1v9D|8{>R&RIdx)d_Z8 zd3M-iB6sEXh+D(;geL5cuLkP%789G)t<%M`50Sn3gp7WFi(6}&j;%*^tr*=0URO3i z`E>%hZ=3h6){5(DcTfknSIm5vy#}ke&dLu@uMi4V1{Q@`GpF;#V6OnTQ~jc3Qp*DO zDL7v!`EK`bu)G=++_!MF#-1XCw|7+E+w8Fij7QADDU+*?1?6mhjouYxUx^}GQ!&*s zAtP5Q#@;>w%4}rvTf&JM2mhy>6ktV0lD$XX) zHee<13=P{860YEb@$|v84-XI%)JRH3>d4@5x}39^!>m8o?%c)DJ1Qy!Q?I4-8)5C5 z2rVWTD<23?jzNnkJ0gpy94{NKi8*>%z(&}^&(sLMh3bafHz-x;B9k>)p*qmn=jh9A zFjfeq=7|F;w+&nn6M}e{DevvoY3nGA%dxduZ1g8W+PfzX^m7*LflH#~9U%JK3w1C; z^>A8$>ryu13^ahXkdgSuz(`Nr_0icbj~Bz1bt)A^;Hk!z~Lo} zHhKjMhyxA%tn}pxkFf5i$l&L`F;Cd!G5z-?!-$YPEiZRT4#P4aCj*2&qA-@5dW(FQ z&z8=#)>l2XttSJxqvAjNqPM&6LZ?qiaqAqNv-O-wv@OQRKP3{sO7wh-yf&xC8<~W~ zbaVvF_(ag9{1Bs%a>l>JrWJ3>*9EpXM!0`LZo(g?Oe8NejskYWs{g1>0&w9PMt`B? zRkC=EGwOnZV1Bw@>3Oz*t|{gdxW8ewa}!)S~C&Zshr?uhxaF(W3K%3aOqL zc~d>RzZftp{E`z& z89D-{vh=tgI|FhZ-0_Gb2Wl*bO&ah?O---#)GfqEH}%Q9wiq9qY!H?zx1j;^FyxfZ z`0sv-3QM2Hu&{l3f`zZuMd$pzqYmnoaB8pe%NDdrMJa!AdG zEUK$?<;ljDx{xJ52tEO|%^$tddiymDI;=ooJU9<+^^DONoqxI~<87}buPY0FEBMVg z@7ICjadY09^lg#Cd`7bw_}KFGFV)0JYIIj)hUi>|Fq&gvM?ZahtcfVR7Ilg zhJBLx{Op$=e#am$Yh+KxPry?nN%>aSl|nE*bZC3rmLP1 zZk|E8Jl~`JtNHkVcHd@f_*k9ak#OyhcQXF^2_Ntmh+vfdivSO1!KZdewl!#bRkmsH z7X?sZOVNckk*eKBcspu@YyLvTY(wW(MO{(g`$qMnUVDl z?SBIgL-oviE|&1Y(4W_K>GVml$sx189-D{MzYf6<9!hXo{hj*XoBtlsm_W3Oa;H57 zA;MQUG5qKE|7r^e(QN$xQbK&$G@k{FUO1%BS=3h0tNHxzI{){^MdNPZG(KRmwcm%$L-fBCE>WTT zArjJ&C}wn`|6fWD)9>$aZsvj$6#vu9rT+gB?Bu^AY5sS)mf%Rw4XxeAMFS;)%nI>&i+AM6MDNf=k%IqB?Il zK~f72p-B9e^>r4f+$72i>R0J$fl9b^?yUaP6rjWOdXfx|R{egZTliNo zM;7EAq$S7?MVZPPQ=>_r-8Ho*i!mkDT zMNZqWpj>D4=Y7>@U=$J7@9$ zem2d>CCtO-Qb|FVgCFNBwEHLCi4mZy0?NGCE9Vwebkme46!5Pi=RE${{lUNDrT+yr z>0XC=3$l@QRH@}&ENyr7n*WPC(LyG3@BkNVq8{J;`a>&#)%4$5eOuxXDwXovd<`7)k0yY*J8i#r8x<5yfzCJ%aT)r961qiTP(Vb+kz^%|SQjy-rR{E=*B5rUd zmn;Duznsssw;bD^YbiC`TF_fF`hA#Ei3dQI$lWj~je~hnX94&0l=eac=^`B%l-x=! zn!TSdYIaL(2C_O3@yV+5R$P@|ru~A7~WwWBc zSTEOB+6t9$a`)`F{orCfRrMvzH^FRYkvixHXS)e0f{`qc zwXUdQ7q8RSc0^wtCj%NaENSDzN0&XBE&9Q$rgekUR+oX%97Jac0n)zp__0ZkITNL^ zZT!(9Kc3?^E;~g3OB&@s%8I6h3E*X>&N9hjIGlro0y8WfUGr3OGg#C`ucT&~|1isv zFOzeR(|J9}Nquj<8`Y_s9|@;X3WWX;cSBu&q%z?Fj0~gAyje3cmiC}~%`y6Q^!ce5 zWJC*djF_2qSqb^k>Ph4S3+N3~7;z73$hZO?iNYSl9Z;R0Y4e0;pEvItO?X8*B1FiZ zy3S`DjQvz)r$WqsvOuTra7l|qj767frBqXeBDeR`ar{n9HhXZgkl0;0QSjH~Z#H>W zxo+K0Ca)9Y9tdXg(Yh!EA2`SeW`+O&I_JrdUWpB0uOCNH=wZibV9fQHiSY>Latoi- zb-D%Q?q2EHqk3(0I(VHcy#6F-CtwQpGLyg(z1B~XD13&~t9?oGs)acr1f61BjBiG} zC0d$%Me4!J$SsTzN!rkK93K1iw<6BbBm14MG2=w^n@LB>q)~}};mZqQdi3-A>x`;r z%Phi`lf7>8gFYui1=|wW%6v{mtR~(x+Lrev&k&}h3E2Z-7X~Ltysf0ZJhOfmh!e>0 zo-X;V5O580LT?FuclOwvw&hXAaQrrK3A6^6M;S%hfOCxT{4?jysXNs)La}R-_l)H~ zzO(bynihy*9#9xN8%pPvq>gw*Eb%V{ui?o`8R8Za&%B7fg+m2Q*VnH;aZ43Iw}g(vpo#3&5RqG7Ejf5&?9cNN-q$nchEYbHyl zXxQY7!-7mK53c-2V=Sfm`_V#7Gs};1R7J&cEIu~u^aYq3?WQ6_G=j(0?n!fqQUmzA zoGa#ocp^a3R;{;UJ^cwiQg84F0eXX~FJ}_S%&s@z87dq$L`~+x@xS?O6hZeQvv@64 z{8toWtAk$DMvhYKEE<(nytcG#sk2)G>RnebKkjsnD58q>Gus|FB<|+hOHlgHrE1`v zH(#Y=K(IJGoGOjcY2{>`xifeUQIt?nFrLk=Sg=<~OmhLi^(_D8z{eE+YSD_q?HwLF z#>?(so2Z(}twl?Y_mr~K8H?vVC`q%P&?Y1EQAc#ZS6d=%$Ae;*KZgDvHs!$TW)Bda z=q^U)@x0Ub1oxnv{pf3N$K6+~KZGRdU)M$;0h}3IogWiweZx>$S!f2fRnH7pXg-D^ zD|~&W#o4IqAh&-0oG$+)tMB&{!)I{EJK>XsMV;v=fE?A^ye>jE++f5cbzs3UYytq6 z@mDX87|^*}G&|1m1h_zg7KK%(NqAMoFq8a4)GS#YyH&rq5o$vlbaJ!7j0*sl=HmC6 zp$Yg6DkEkeU}Pxyzvfd~hr3>(4dSgTthG8ED0`L!EJUZqOGu}Y=DSLrSx+&w`TvMN zXumzW9h&G1k6*lFLJ0gtU@<0}=VtUg8R#CId67;RNZ1?2LprDmfWuPwwn?H-LWF^a zXQtT<7Kh9F5v%o{hmlLSa+Ep02&B-2Ivc?~+~u)l4BqW=rLoIZ%RU_|;Prv!am1tr zPiRu-$dw4sZxZ=FMDjxl(SFI(&UV>;Xb@T%ch)d-mhqX=2SUa2wX4^hgj`&WrVUAg z5smJLPQ*CU0|&4QxE6n={ktZhoAbuR4!edybXsOp4nO8kxYN^T;>Ov4Aphjv?Q=?e z@BNbJi9Y7rc+qUc9;)U;pDJ4wqG4IFT`fgcGOz7*6pPxL=*+mlrt9BA8i5^@M!5xG09Ny)0AOFh=F&0*3)t#`+`=bztq9rDg5k zB>G=YzapDR(?jzG20I_JY!U<#3ZT=IyZm@mdql1Z;6ns=ii|6(I)W<-3%?bM4II73 zw;tl74v$3N^mL=eZ^a>^PDlqh;^c_jCIR<6#an@Q*EZIqyCw<&*(g*2i2pr3Vh19J z_e+dbjyj*sVDSWGzU~NWD@Hqcg@~Sh8g_D#H1+_VVxcQ*G54^>r{3T5;ij)4Hx!xY zN?24GB@7~{aQuHD%k`t2CMTCQ7J#q$wfx3=rGaOb+liI2Y8ppNS)(}_G0)7oLc&WD zn>Rsn@CiVC?W=s%kF*Ep{i1^?e0ymGt%*#(5DBRJMXl$+e94;xL>K$}_9iSj5{JS) zIBz5fai0WEtv&xIilll7cdUyHSN0}d_dxXErx@P)oo>cQsja9z&;b_rw{M7tI@ybT zZ1^muTI2XP$DBufif!2UjC{xknH{rl;ZE~V@`pdnI44E-X1v5pz8Rctli`6&GYMXh z5NQaYFD?fyH;-CYE~0Q}&7@T6xfDUnTjRQH8#N1I~$VQh(Hs$4vd`Uz66p09rBGjJ{R;)ig#O=6!u48h+HWM;^_`J@;wO zfMD>hj}C{U%L4Zp5Md4B)gOK&F6gp4ld2Iz+78FPmjc#M&A)70@-&2DjGndm`<^)p zeizme8CQzLKoZJt>Yg_uB;YS*H7T_hV|%*N9%KLpiq3E!(Xfot`oK8cMUoVj#cyx0 z$w1ZT7Q0Qqh4z`(egMx#You(@A>jVz3WpiDs~q6DT>7bFY|r{`Ar!`tZa7VXNlgO& zv3V*>J*W(9WQ0=|{GOdnSdNwnACTTM65p&LN2{mo)Y6o!Fv$0Q=sCKTPzTo*ciamz zAx+ew*=ZBVUheA9C>u$IJF}Ig(jphtq21C+oquRx>xk?lX92lw@SjZPD8ncw0P&oA zOkx*?CoG}!@xJmj*0yV&Ld8PI9euK4q=T1`>RZdiwB~*GsdG(0XOS50Tf#JVv1CV_ zPtHAGxC{>t+MN>f2is+a8sX5?fn{1hlTLG4N=l4RkY)g>j}g-V0O_Z)SV6~B4}3as zK2##vo;8q_-?f*2{Yo_0iiLIsumlj!y7{IpTIq#hq>VZyjuw^25p}nM!N@2ibz@J> zftDDys67^oq2|YVD&+W39hEH^qz#%||kDiMFbiZeXtj%#a0 zeo04v#k6%}rK=c~emT!J)OQY*>LhI-&2jeBr!h4J-H9@*+E&KL)Rgu`P{qxw>1q9gbr-s)o8#k3xp60sP-}D zN1kWh+39WfaBcLK_iCk#`o?IAFnRu!D*Z-8C`YUbdi6+wk_YEm%LN-R%*>~}x5OAf z&z$svE1q0fIN5u8a_b2L8cQTrVBdt8g{=z0UT_pQka*-hZ?+UVO$h`=M=Aa#b;ChiZ)Y>+6#%(Z3l8 zFCE2PV-S4x)&|Tf3Ybu0I=|?fnVR2s>|?+DU$8K<(dsQ!mleE;EskjK^bbY=qIC<@ zb>eA7-u37=;2jk|T2bRxI+{)*mrJbLVWHNw0Vj*f5!JapOW4LO@7JZAPzO=30I;vR zuy_wn^ET4nsJ}oF3oLgLc&Or`m51OO_jiS3f$LY4$t!%%_U@in=+w+FnR~)wfC|2@ z1m)$JSa?!ZQyzo_@UAv{GuL~NNgen^vLT&;N;a4zgobF}=+?zhuBT(&m1&ctS_vWS2PuuNU zf}~B7XmLrdY;?_-UZ{T7jNvF=J1W0IGPJn{F|G|{^k2uN@J1SGVpv{r9S(<~&A#=4 zG4HO<4^3p7U!)23!6{Pin1*Tm#WU~E4M|4eRAWhcPVcJcy_^0$v$rwygK2GC>)>aQGzu+KVgDHK0+%|lMAc3Rln{8 z%0D1USvpz#_&O_%a2Q=VzOn&uub2tgnn{@mU|yd1820u>o;wv%&?F#AEL(5pk;bqR zNF0I&CZ>N^A|#TfqO$pIgEJ-UsRHy~l6rpJ)Ono%9_n)UC0E}(Zkn&ZRo;T6I4~g% zNcwMo9j8pwzR6^p4isZZVGKw2r41$#>{L6C0$O~0g@Q)t*&L4gO(3CQ+qsVHByDr- zOjYHmh(|E^8v_?+m7f|xIncB9i=&?g&M`yomX#5y3Le?Dzi^(ZY*@W%T5tFt_QOD~ zT`cR=cyrN&Ysp}3s@ts~B9)V>dMO_X^A)R+^n9^HL* z_N?WLoq3B(6(mc=E$|q9g)#B@h@aDpm-MuBh8}El@fdvzC?YdnyXIc%VM-J@H|p`C zZWgDetSDc{x97mw^s6}nAv?oo^JXtbx^uMeDm zQDObwo{SeD4c=PPgcXakJHqI?e|doN!-h0U>+6$wQKRC{GpZ|5db!1a+coCZKD>SrlQ-B!yqG)RUAm9wE{ZcL zhF(D*+n9@dJ!s}V^cN4#yuhAJ8xXE+i#GBkRn=+eXPvaki46;AW~v1TIEGKV#W7I+ z>xjlckBvh}9Q)Ys2`SuHyuIwzR?XT@%w2@!lChJZDJ&UJUu5FcpO(W@=|5Q@?PT@Y zf>LhQ&V!d_aU!i}vTLV!%44#olR^3h3vh+B!e!%!2-+oSK+Ytahx;EXxpW-dA4_T} zLGiu2i6g@HD$a z=HuF;Nx(pY4u|I%rz|nP?9nwBqu}ytfW>hmmyeF%uFqkUm^Dsyb)siYo9WBwO$uvc zJh9K@wwzmG(PaH1%^KVf%F^^PJB7aI!U=-{VJ=dfwN-?mF%BZJ26k!_r-p?pOP}Ys zJQ_)DFyYQ?fiNIuxbq#&pz;Ta?bEI(ssT(Ys*hIp+cW`pK+%z0o~Tz~r!4E{q@vFJ-1J!* zhuKfBjBkmDX^osuJ(HwX8DNeqYcsKAr4?HY=x_NG6v3Z%`KoDHUI}LjBkMx76aF!7 zc(1;EUU8zLrcQ6FNh6BIzn*s`5vQ+hF%v5TJt@>q%DbJcMI9TuSMML$JhMD-Aq%^% z1@s#XywI5}#gObsGmky4YW1AUs(Dud821hYih@A16w+ z756&%-?y2+ACuW~;IKa9cJd7r@h&G%troB)i*<1PJlmsT7Dd)|y2M2<@JP6r-P@eU zg|LfB6`BFt@8f_ytw|VMa@b8J$~KK}-ylbG`{~QCm2CFWw}kDAuRnbWiu%PdgM_q; zIAcdF5knmNFzcgE-aZ5lS$>_LyqjV;7!cv>{J|a06*&8A4$R}8zsR5BLZJS_mwXo{ zBSU?1XVC`azSh?qilXGMz<5=GKZ)eu#m1W$ryqR8TH`@(=FxDt`V3`x#mXKz zDvqqyJ*V;*o6oV+lEeFe*`hD>)IU74sFUud`T;WlAZQQO3S91@FhQ2mQev+eUHv7bhKV|M4Ug`u5~z^8zjU$v?7$8 z!%W@lF(%;0rUSw1qS|rS0TAcDr#MeNG{@9{Mclfnll;=AG|)zd#K*dDWAf1?I9|~R zU;MP~iTH)QuR8ev;0@IP+^3*GtN1|qvXKu|?xN;5t6ViB`O)>JSt=#&kXh9J$M@3i zP+X$XK8U1C?}jV)?YGCLqctyAjk&c@D1F@a$KN6QMb8sHzh-!z)EyDXppL}!I}BGs za_XHrW)^&^n1jGxCe1RqCli} zdHb_pcNH+X1a7<**rdyHyi&^3Xob&-4dp)MO`1x*$o+grBRUcN>Gbj=X9I5@AXNOd zl=#O();hE5%%NQ% zZ*;Su4_Ri<2@ibrL;UJUz_-;Oaw)Ne0nUOef~yJi&5yhI!;8HH~iXSI@U>@L59p~}=*l%pYB;{9Q6_`9ZjoJy?cC!uODuY)AnBK0)J&{LZ4&B?1?viV-|gMP484>*wG`k$IcZTJUbl;`Dx~Sz3U7oiZuQ>G z{5ZEZawH$bVFo=|kN$8zGKby98d$jx#b*g`6Di3~Y<;x`CTL+%+IEAt=`Vj@%OW!C z-m%1gPiCH2{CId`5H_CE5570}t@L2nm=N`ah=%+u`?K6{J{t0rXG^0)OTAz@UpefL zKh2HIT z%6eOzc5y)jeiQL8<*Dt$B^ciHoo7lh7SQ0p9N~RZLkNwABl6uhdN`Jf_0hHm@q#j1ldigenzEXhan_wZSSJ+K@rZ_tYM1ro4z3xEAw!DJ<$ z&2lVm?-~@iceea_pEWobvw57b9@XJD)X@oi`9d}n&?B(G{!?0*aE3_~3}4urEkmqE zF5ya1@+z)7AX- zEqhwOUabB|S~xC73Md1xN$tqd0_dj>;ZVCVQNWut@TB6KOz*1&kpE_zY#$8v^4c~B zCv-aKtw8HKeW|deNerQP&xB@r(zce{~vQHuQs0vEF$|e*Qg*-+a02 zSwRZrCSOl@ANY!}a&2VXG3DAEIzvs+s~F|K1HhUe-&)FK+kq;hT!|q05MQ-tbG>lf z;Q7)OwnA{d%d<0DkHZS87m{Xw$p6`{pZM%`_R|5*#v>O z)OGC*=-H6QmI-VA^k>AvD7ai5{1Q?)0419qn#uxIION<_L`MM3TB_e!2uVc2eb9P*+ z-%+leU|DI(Zs~YQLvqoCP(>Tde0Tag%U0$U8>s%ia%)OJ)GYCWNi$*aLTD1S^)pZ3 zbA&&KF!|$Ai#c=7IbEZ^aFCXmNLT~-u2L4CL zpg5N6SQJ?tk*O7(5k4C%Pb6FJn?2s|yq`Fp32w=M?7rk1DNB7{}w#j1lhr^kD-fP9(-D=2A+me8(V;V6}!p7K< zt?8$u4&~DywrQ|_o)x3eKPARS|IQnHKux6CU!W*;1`U`$GdDL+N=k}b$ep|5J6x#v zV77nsj!7xO-#(+>>bf0b&YXRXtw|-DQ1GPjTEqdgpP6|+9e3{K3-uS4v6;xoBGv)#h6WfS13{H4xz z+lJ)R75R6@Ol20qs3gyIFGVAj3THgsM;_V4=>AGoKn{G4_OJ9xZ{T*7jeXNFs;Iv% z6x?O#jz-4wZ3_GtGF8S5`-cVa7yN|ES-$oM zigtOek>9+2_t!VF(;fU@L;fWH`>jgLl}q`b-30YvR4bW(f15y~!HGchKkUjTj|Kk! zKmS*S{GYBm=9W^FIQZHL?5#tri$3M3XiCXCbHvL~78cTK7)1ZwdL>wdzv2pvO5hm|D59Gy`lB~* z{LHmGoG)dy?v&Ihld4kkBjOWC(qQW3ETDWC(WoR-EMIn8tTD&VJUxXf6hvT)qN6eN-wzSg>OF)t2=`$tO zPA2@Q*39*BldE<}&=12@ya0lJx9iR^@mS&UA1HTmSdF0Lo0JDacgW}}XB`)G4&(d7 zXOtELS7JVqn|Bx@XUeR^vymar_o6@bQ5oqs#ucf3-U_AxASLZ;kliBr7BYP!bj{0_ z0`~*oXAh6geKa0L;-f7-{#Z#okPmU?o!? z`EAO*Y(LG+acL;QJXbofl(_xzSLV8(65YFZHunt~JT@)^w@ zb~Pe^cMt^#wGNaRz?F>Nj6x@g>)IxOcc5Xu z8xSnt5?_u2SOwdfF8ztS(qhZWz^0ZH|2D1%7np+ohEBpoCqs=19-GL^a>2Fn`0{|> z0OGZwd@Wc=zb*RCwcEPcjh9E_!mZ7K;d*ERQP+q^GbBaNONjkjz+*c-IeaO(-t#vU zab3fWPm#uYkFnuV*?uev|h|K&@oGnW~URjPb3fT-YeU|dwmK-IC_hveO?43yVCU}0g+=uM*=0Hnb>z+TW)depI z;+gWZeAt!PON2$VHC=jrvYeqU?Yl;8+L3yJ@k`GpwNV|FQF z{JV85bn6i1Sy)B<0C~m1sloNmsbNj4@7c>C;#G#|J{hl{?jh(eDAVAaOw_ z=SXb#gAXhjBy%ZS%H-eehrOl!4PC+L*@4=$!m*cPeJyl}xn&S2ScP6fyk=cVet z%Q8hvgKUunDvs|hQeqBJVK>FB1+6m&J(G)3L7~vNec8_NQ4kw=gP+b}(EPj9zsi>s zErSM&{M%e@&gPPq==Z>ux|?3wi+jI!JZ0kDJps_8b4Zoo3yyGueLh#rV_W=MO0h_u z`haCmS#=$_&dad7(A+bxAfV?2hZJz&f#ix{fFN3ApcZ*G=`F`Mvc^A~HFr zD*`2AC*sHiWG;7CjG7bT_dr90CR<5-R&qDIVJ`@DYUHJ$hr`EbxsNKYHi$swr#hEj z!e@3S=H7HPjfJu#eKd7>*3qcD6H3MG)f%@Ach=nH?KwN6E{y9tiv$NW3-sr=Wz9P+ zWnL_mh-)o{Dzy_TS-!g*IBo7cjWLA1woC^zKO9h}-j7ih118AzpgW*czzk37`O&I` zca9IfyIiqYtE02z)X{VG>1EeWhj!2BBbM*8L-;q&_m<`6P>U@YBmZ|p5?A)sNA;^sW>bvb3 zy|Bk2`#@)svGn+qu{Y>HR+48DIfI6!$!=M7kHL8mB^UbT>jVbH4m* zi=x#h(}j)`uePlYupk|#pf@IcBa_L&MhKk#c%on`A*Xj&pbIhiVHhiwH`m7FPr&!E z<7h^A_1*GNBA391X1eOsjjY*=z(e?k(mDHn`jOB1o1kHjfI(!s_awW+-6#UH{VMRe zh-!Ga?Dn1*`uF~YSirqo@6Em5q19s5m9i*;%x2NYn%Y`^SeScj`$EC6qxJgHr}fM` z^d~>tH46A#b{&AJ9`H;A2 zWRckfb2R-XinZF4ecSf*BA)daK_@cF6Ep}Y3TL1+eq7;T3yZhmX3#15#39G^Y^8cl zPWW?s2`#R9I{Ja%uGR!Qq<`xUH44dG?u&ngf|f8s`+O zC4V?^$ag-RAl#Yo`f&jzoBKbtq|BAgi&u%FCCpfj_}4mK z?jd0R>P}{$qM6?v*+w6CzVVSw$5?cpikyfIUBR@|49RZAPf3@2-C4UVsjn>iW)%-7 zw@m7l@{x(sr=cwXPy1!n4$Df20B=Z!n~mR-_-PZ~Me7ctL(7(aLaou}oN|qaeWy1| z23gL>%`Xc#G}NguTKFn1rN#>SRb&hnfZY{^%=)A+J)|HtTcZv8Z(KSz=B)iVwVRoC zywVH#6Af`66vUUay(n2yLgq-x4d08iCD0Rt9M{2>l`x^cjh3j zsbPSdQwGn$Or7GpV3c*!eO-!#@l|EcJQc2wL9gJ~vt()q$@&kxex`M^tr^D}?`v7& zD{*G$dJ~4Fw)_oRNn*yB62PD|sJh5}FV?l9$@M#g^T2fOEpg`cV}M0QW`B5tjjVJ4!CjEu8oS^=rY!;Hhe?Iwts zjdn$Kb>Ave{3JmIX3scNSn>-BzB#PL(ASuA8L>_NjF_hV+shWCW_6~L6=c6e2PatK zJInTu{!xxvAiRdUs_&yWo@KsStfGCh!u1D>lB^dWJcif&t z;ppX=jnXZ)fOzJD-rJkV@u{kEjDB#2O7Y^Sidn^=%!epuGAy;s`1E0t*M+x2_?vM_p|` zm6iuNWFh#h=F^G@w^eRz-WSAuanFgHAr>H8rvNW$}z&O=35o;kY&@7~64oiR@wD`D86il{$5@Sh&}A2LDqPz~fJ^ zn%gQ+c;S5Ff02J&t6-;g9%q=vSry=A6?EnfJ}UcF{;9^CT2hZOwkhQ!e6oayAcBZo z!9I!pPxP<)N8O{8oj0ZAOl+~W+L9W{NR&Tp%n00O%m{e&S}J5;pw8fvDtycj>tX4a zSYXDG#zC0k8G-bUx9ZPB2rG4LJD&vSt;Hv6k*3Z)Si&mPhRrTBaRV5Gw^&w`KA{%A zzg&#Cj$l~riMf9E-Bztw$xl^qZ9v4&T~OMA)$PmS4~_CL#|F|`H|)$t33O(s3xs4z z$+x(OdJPiVHD(;g%k^MTtmx{aKLuv4Y@)c;Tf<%r%*?*as`J>*=R0NX z`hNyt;1Re^u;2AlZ>U!s$q6Ej*xIgPnyU5o&HQ!(&x2%$zF<%kBm`~C5<1oRus5GC zBMf5Yt5MFTHZ3U9&!6#&zW96=P3D z%viA=xEJyHgMxy+%|Mda6&tGKH*uS{2GI9nbVa;8OEMoe>E@Y7bF2KAbo=O3tN0UP zs}_A1*5Z*e_VZ)o>n!T;FI(#P-CnMhc6N({HUxg8<40&I!`E>+XB@geF+J7gwgaG| z@%aw5;bv;Lf99<0=eC?JvA4##ev7ynvDScBOR2A*P;Srlt`8HKC70dvvmNLG#-B4O z(E;6GtxbSFhAVh)>cW@1#GDuy{O0Z~|qx`z)dcAF>XA^cUY z*y)ZUT@sonqA|`aHxJDZ)~EZVQ7v4FTc3%M^x>0cfHS|>nDBmEvi7B*tKgl|(^S(& z?I*@#e@I7E2?9xV2S25cgUC%VIKtakTH<+Z^OQgm--nCR2yA#13M@9Uc?`;HBI`jjswF)IO$Il9VD&?p~=R z$&C4TW>bYe`>tRM?I7H}(zG=DR0Y?1$P~DW5Ry;kH7#+}z3P9j|Blo@ zRp#pF{?8>Z29plUwm0WjzNDGFMcs}z)s#YC(fTnHeNm;0=ZYw!)3a^0I7U{jNuDqPQd^!i{;kfd-oP48dyY8&AUfaKIrJE4$G9}Z& z`V&`ujb4~aKZNoPzK?EtMJ~qu<>$x1SECkVRPYelm&)~?Eg&cGxt(G1gpE?MfhS=Z zlSL?lVQt{g>Bj;)-_WsomEA1xK0Zp=b39_T%C(nm)mf${MRs9}Yk{Y=8wkh?6v zrk`tt0T44?!E4)%6D8Xp#WXfW9}9GfWUN|R3}j32R( zc)4^ae0_Ak4N;ZjxFr`GEBH3L7jQlL?M0RMfG3%5T}}q$#>UmO4uFBoOJt#O#AIhn zd3eN7y7z&^u)%@qymOMhc<$95sTgo@!!tI-?`h1u{XnJIpXCi$tT)k!tcwJP81Hnm zm>$r5de85BOvlUo6c@YOmpsgB`x|hO;@?*!lRZVJert@{0kCMOsGzsscc9*$;IX3kzM$m zKU4XN-|%qQ5@W%+TkDhG^T8~>7-FFv)wtA&HbQxs<44OomU|&&E@_F!Bk6SQYpgqi zbIf8)3I|&xHFyuVyQMgmeMPy(N;+CDU1>RfhN079M~Zf3s*^R6kexW57h0bymFue; zo6My3Nk}{ML_~=KYw2VC51X+Joo;PTm6rTBvK|SYPR69g(96u_lra=RF9XnYzpY`1~W%M(uX;Apkj zv~RdEN!PU;y)%>C+86$}`Tt4$e_CJuKcIU~SI*9tn@ecidj+6MXp{}Wcm9%6H8rKw z)zvk$pkE~b4%oN(i{u-&!uP+a0Edj8obr@`gDQ9Bf@1T)N zT@oQl6+HVxC35k7G;mFUNJT}Z56R|Jz7z{pd`~H>^xtBEdieLUmS?i6b72y1bn4HT z4uY|)G3CDN)&nQS$IJCb7xlHfN|TLl->YH4C33svrXaW|P6Vz04aS*fV9GCON>e1} z`=qO@Iyd)Ec;j9CA88pk5}sjpZ5v#ME+GF2cbGq>3jbXu%>%~0*SIy?-*xfl=mpmw z8T&ttRFVK!;QzBLTWa=ap!c5y@6!Jc>HO`X)>1oPPELQE&#%3ml6nW#C}|Dg7-8crf6Ao|{I$Dy2seF@)>yGWIRM+u0HZXTEub z|Fh8lWA7ge_@@x~|AB`s%34tWG0qQ*Z2}78O`qYGC@c63Ui14}@7}!|N&in~rSQ!D zItREj`i$WJOHK8E#m+P+WHR5fLA#ku`FYjn>Pd1J)sbMwe(Se-VhJYKb>e}%;yIwN z3l4eGTgnC`n8aI)l-JOn05Q{(w&1aB{si9ccR&7Z{l`>)g|i;U`U|Ih770Rt%kc%D zzronne?uDA#?eTXuiv3;2Lche9M4@q66Bn0 zMfYcb+cD0{HJ?R<);qT*vlc0%3tFD{K-T-ZrV@u3m!CKTqIj?T5@=#b`4b|MT?-XK z6q2{;W1hw&9->^CFne&geBt0P6;#Mdd{N%Bq-`!xhdjag5mt|zkjqYfEXR-A6GgCc zb4s`A21T(IIjA0Je^f@^?&u%0;seRZTD-h@rs#uae)HR(WI^yHJZJwySu$@|to=g- zZV`}SPD?dY>&}aal+#5Z>vlKWl_|ft{XW;t3nj>WJTK!8s2N0p%Y*70hF^I+#3~T} z6Q97jo8Mv=a!+!QU- zOR_?j{K+mYi%s-*GCc&GzAtd3Z()~cnC0Ov-7|>+DzLOnRBM{?$!o|Gb*1_0?M74L z{w<4E&C9&!L70mx^!pyBhy#odq`W3>VsW2_WVP(a(EFtBzh;VZE*s$S!Xqlc_H!*%uyy3*O zu%`Nc;umyG2V>TLG*<`$nl4&W@Ua%=nyC%%8&IpO_#WP*uG9&i0E<*G7MZOXKJBwe z^U1*5Rewp|uY-y#7`!jH!lTW&&Uc!*PekHUkkt-irv}!Q-%r1$CKRfpmd|OAKQG>n z`{jd^AG+SZ2SaIo6idgE4CQz4^IP?o65fXg-Le6kLu_c6`s9=^s+>70SKVHpA0jn0 z60J1h<6|7l+##P2F_~c#AjichCyVGqL?Lu}9G@+d3^q1{*F%@4chuO~9PE*mFv zbI0FTknn~;c~A&^2i09HM7u7vl&C)W>DfZ-jstj#=hC{7jY5sD_XbX=m+Pq4R*;9& zZ(piu;l(?d87T=}o#@&W=nMl=7I*%rx{fpyX{_eJ`wmmbYFopB*d%S17X3H z=L+7^NSAj?#MPA3EpF62oUu;|oX03UhdZ1W+&Z$-B)b`r-+mh4g#=^Bs zV~_h)=5v?oY8vW)a{;JqTkQiss*v+0?_^NMcTkwcVN&dqn`r}1Rl4rGpSgdkp z_s%X(WNK%(p@S#lmX5P^iAXTBNfN*Nl-j+=nOrM@JW^w*w&*i?E=8v)L!4hd4(ZX* zp+AXSo+-11jaHu2{TN}B(Aka}J~>^5 zl+c!D`vmu-Dy!Juv{SH-@WgpiOX9r>Lq?3xbf+f5#WR39O|(9EHTO9!SXU;Gq^J z(G1a)T_LfZ5?>a{@maaI%KN1yc_z!>U--TYjL2zm*4JSsS;$rZoZ?#Su`O|Syzx8C z!Ke1y5-L3NCG9J6Ubd@pen==qq7@jsU07YYTKwqzl3I8lM<(}TDAvyq5T=xCYI_`8 z_&e*piv8W5oH2!>XhwHk`H=k$A<~Qbv%&L(phKtkxZU4<3EH>=eNI=s2Ey8}UWa2P zx*z@4<11vh!=?70woQGHYIGH=FG-H#viqa^(h7$r@m-$k=iq>7Xh&soaK^=)@)*2i z``PxKfqd({T5@l4%|3^DiL%4Q@P0yPo@yBHaNmTsq;Dgq@Lgk|tq|7 zAyoG>HLu*s_x?8L1*FQ!;Nn#jF5d$+Hhs}7MM_=C53VfT{95l^5Hw>9doqoLR49cL zC1}IA{HOVg{dA{QuJrd`hQFdppGz#H+M!&IO`Tp+DqwGXesGGlLGa4!9BcOw>8caC z;+~5YUDX)Qa}&E*NOln10540C_AyL9+=tyM-XN^`9ynL}^CgLY(e!Rmr6N-9XL$iH znIi++4gH{|)e${nc;*u*(2PSR`LuAjGZoh$Ta2T}l_!@JOO``I<#Mn0+3v|+7J-f& znAY%{@Ztb9qZLWxYXrHxE>CXR#1ougj2Wn!w8`C#p}=5dMWhSieZcn~;g{}57T=sy z?fp-#ry5MxfnxCT}PM$`f-^vE@pa+dhg15)W>i>FpKmLHq;$0=8#PTQ921>L)w2(}tZv zgUaDiO0ij{?b+cuQ@W{xg7lrfqSy2d$5Xi`l9VjT9|-H1+5+FxRH*1 zd>t(iPfsjP!0^OaBz=)iFncj&3X+zQNntT4OjOf1FrW^2H-darl$eQ+u8k9pHNE-O zhM)5u(;fhzuv>l#5LNg?e}$CNN=gCu`7*)i^S=Pl#$-EISiTIP{`*hq549HYunXnm z&jazHbkLDMmL++e_&8PfB25qSy91}~GCeO>hPc>=fUhrju;v54z)QMqqw$`MdaYV; zpM;>MlL%(cP7?K==%N!j-(Z*=58zVu3>=X+Lo!GYIySxRpH$%!4&cBK2FL|^#ta%1 zB%Rw=KTB>cO@st7judpZNLkjm$6mI+)p?gPc)AJ}+}}k*0=n*uV3Esv8>5Y)_p*52 z0SG{gK_v=_2aqmeSAL$Mx=+SjVuq3uX9gGtoDp}=!*uyBVkoXJBtOI(=?Ql)OCY%E zI^QpGoK&C*-`c+&%n?inIxug|Y~G}?qtzxAV!;@eDXX!Vl|c`Oj(-Zg+>RM;HFBXw#$Nzl5S7jLia_x$+79d zY0NNFlliU8&MG)MlfyL4oDJ?p@pQYYkCr1Tw~Ko_f%N8`k}n8(k0q+>Cz=xInx8Ol zDy5K*%Oh)?#&w4{`!{vhqwKm7pkFH_O?9^v>~MZXVztr+I~#jCdVrc3P7V;5WC^Jg z%YHxmxX^qQ$e59Y9Y4Hostehv_Y-IVld?Q}8~R};&bKp`Zelk*4fDxOhFn@cbx*HS z6c0AEENA!CL>($ae?GI6T-<@ET`Z!fo(rS;9o z=D3do?Nu=;(erXwB)nb^x9Qnlig~MlOds!|!WkC0P)s0bj6*y)`%+boog5b37lh2u zlJfex zYxz}S4y=+oG18x}7_Zohq>~q;|C5G8*-Zd(sYR=rVDO)v!1iVaYrO7`-FR8IQ4;};TLdcz&_buhM-_vN}k<7*Bd*g!2=1Hzqxy~|*V=|o9)HaI%H&dw|S#T{j5mno$K&;~bYeR8im zEOe&Chnc}YMOIw_p?eR}%j~G@?9vGh?aIV>N%UTjTU?zfcPJCXxdGFdKWyx`aV#sP zvSb7`LdcTY-e=d!9|2>#UcuO_&o<8DCQ@V%CA!VOq~P%!(!0i>P-0QuJ||1;-MYFL z+y)(a0%f=dQx9|~dOq(mn5)bozw|rZXIpaK3++J`w1p6;q@xTrVq3SGO@{Q_p5);o zozpqZQ3N7(qfq)8uFY<%4e;9mtg7RfwNUhb&H7KJi{i@jRnPA0LPw0XUM}fIzR;g+ zFf$+TPv)nfRl>M&k8};vmeCV!@4?NBK3?*n0Jz{@T_YR#%@`b}FZN-v-4nlbwgR0<#%_?p?09L-n z*>34(pdTfMct9Pu!)EU0xZE>LWs*Pz^U5)%wu|?hDy{q3B1D>^;DK*rMMzUO&gpldpR+?%qu6xXt0{4E~ri7+d%%H^gydI0c(f-~L8 z1C##l!YR7NX2zb|2E_7c$B2YAH%;zs`XTAwQ8Hk)7+<9Sk+{ zmrYxu1Mch_hJ1PhwA?rTN~T$tCgNvisbXH#1r*HK%}Tx=Xxq;Mq411_oOVl(xiefv6S}4yC zBWdky*l?0`fZHd%;$KhwQ{|5zOlj}#C^Z`DqyMp@{w#gj=ikV3iGYqs2)F!X*g!>A zf*P1ksKZNc(R|1)Piz9jKjr^vF<`}AfMSRK3OBM>e6Wn zV_?tlRM3M%hzs{Ys*<$g2aVRjFD)+)5b@H58I#WEwIsvn@P2gUkw;)9x>|9u_~mwr zULtU(g}ix5H+367b-l!G;-94Og(J^8Zf)FDV=qbLMPTT-Ra*&$+y~LoGz;8PV1lT= zdW6{otqb`MtKryBIbr!2ytnrxRHB?2RLDD3z{7ku*Z2m;OBz6P;OQ_IdB`MeQSum{R7b zf-f$@@n6R$eG>onfq4e^My$M6(J<=e&@emI)VHNrZH=$;g}fM_S4eRnE*BAf^hudo zPH}?+Ny|Ke8y;t!urm};+^qL!WoM?gXEaZ4$MwSe+|L2aOooY8stHMO-QM8VK6feE z>mnZoc<%LOzxgC*97q}~bqP;|sgE079f0COGxUwl9mQ$!_2OPkL(!(R|tI$cJg5qBGxV5XcwJBZ}PxYErfQujNmv!v$z)L%vfT|H}_Iuu>F-Ku**#OzRmc_k;Q_sg(DXZIFXc4oYpqH@Y{srhVmJmoYp^T zUBPDnQ(36Rzbtf+k<0Cuddx0qG|=!arosoICW>w%9`Gc?SI~~N-XOvpzXmU{fml-M z!E-u_>`&rf(sBzCY$s@;7k4}H3TNF=tL)a`Ds-Axa(jz`gi{P$Swib7jC#1nPdkhm zep^JLhOk=T!G!6}7&9v5BmFs$yM4?XZZUgGKjfM99ECl3L;(j!L+7Uy`C0BKr8xY> z22s_guL~I&z@)VxcB7}3g<&CIj(u{{uV*Og*UrjtUzLF~4b}+xbw~));Kv4KO>ZRF zNpX(2%PR>%jAyYYXK#dyyw*JWcfjcaLbaF3%$D4iOC7yRn-1LahIkuwFq*KsLs}b; z*gvR6o#jlPjJ_m$%4-*ji~6`BtH6T1R>h1&2+KeCz{ytX+GQM!|bIKV%=l`1u0^a9Z8Hkxq^ zb7dE7Diynqy}{l<6&6KPjE&TDQJ#r#v+<=*?_>-m!>~%~-{sJx4Fs=eaZ`*P+Q#}OEm&02HsUjhbp5hspS?|4$Phjj zVlbtJrX9AOZyqJ-<41S!ab3q)V?ZkS%b%?bas){pa#e}=Hqmh#{3qsAMeY_%*Q@X4 zNgLY8%OA~Y_X103+BZZ~JBH^h*de!rVi&J?$Iw1J{u0gU$Vs+z8I0iUA_|A#seRS<_@dge%|RwSb|kpB=r9%ywmV;=e|c#>Cp%WPjQ^=(Ugt7 z?O8wwT?Nz~5|Gu?+|QWv?t@UQzf4K3-t9%phNyyzw%CQdLG@!$*y%%fk!rKigs6BA zd!8v9vJbJR8+mV^(`(99ft(z%&f&gimWuFMnV#>O=~k4J!8p%G16GF7$cG@kXEgRP z14qcpY&9+P^Z>jyN(je2-R2Dm9~#E8p9Go z+N+vXm(VyUI?#B?d)}PjgdI1C*k!-B(I97#4nbqSp%BXzv3{e$&@J^$Gq_?)&N3fa92E2%cBL*FZ`2gz~axzgl<+c*eLVMxFu3Eh&5(lu*OO(Of?D+a?c!J7 z%VU?w*j(Pr0~Z8c+0;JoD21}HDfG% z_^=1P;7m=!67Vo^73678NQe_r>Ws}LQasrK>Gh0FrX`|p>~yoLvTQYXNO)32><+T8 z8673QjT9pZ<$!=?iQF|vePiNVUC=0f$s+cHC)3RyvtgXPQ-udA(ujC?rhv(1$YFy{=z25nW|KkO>d}#ynw8;(ux;|J<9-)Rl?0aAYK$!P+r$=}bVYdJLKe8W=xTw|bfc<^-xf z1$9uM`97@emkSY%J{Jp*w00IcjTH%0C>Go?SLZ3Hzv)i4+5GmEB*|xja`Xxmkshh`6W@n95sc<%eG}bi zV@X5yt?wR2XDfMA#cToS6|L)${`NdsQ7zzGmu;{%6r&deZc?pW+2q_<9$Lh z!*f^pvmKq^gTL3N*K14Is)m&QZ&m1t&808=XVAK=6Cw9nzf}(2lU^U2ha4S4^tQ;yp9lMY+uNm|)L=i&1v8bVa)z`&c>#U{b0bsp*gINi7)I2CL52 zf0O3>i4U%hZ>mJ@U)ucAw|Whxo*i6{h*^@TB=;5Z1-DmXPqG3>To^4E7gq{72WmY& z>I5&@B6bzcco`J>UvQ}|o$lu+XlR}D)4d-z9!OIhhO18J1lpFI%pqsLjayy3lsWZ* zypg%W>}jO=c8M%*AZ*hB^sJnWsawO$AD>ygy@NmEvx%0q#~sceQo&qmSx*XD3#cKw5NA+1&=^bVE40Z6)VL2HkZXmCZ%bnhMd zo+QQYSB(q2PGu|@vTB-ljoj^H!={m|1CwyrHJQe^rHH#C+{xLRn!HV1N1BJe!MN2(JV(fUEjIz$#j6P^tz5m-nwt_H{;_8C22&Bbai zJK3jOd2)k6F0Cfn`QH+~h@1P^@_KdRHruZ5_*nEIm~ju4=4*-M_=dJW=3VDy?fbUR z@@kBhJbb}&a5ZJ_8_t8XN)(GErV?Z}Pebq}uDnl9a6;mzf}(ytSpCp7<8JKE8rLuW z&T0PLBkjCn4+~&Z6H*31LwZr3n6g?o*33yjQq0N@(wH5E#r&f+MoZ<<77P_5;s3iP z%Iyw`eO^~uYcVHhSIU+8T4evd34gyR!@o=saCR&Cvn8(LSm}MG^!TIhr)CjMLP%5` z!qNX0i(kMX6c)rZ237Boh3kw43cj6y0Z!T^x#*+9fV?$1CP~9@_ldNwNut~d4K7_Z z2AzbtFla2t=nua?;2W?YuI1Q&JKSXL+vk5vIbTg^O3@kwx^t*uxEpgy6E%QcM&Yhr zAFs39j6)46q>MIQXxV)~XrXp*>3#@OJwtmglv)`ai>f>o#X!RF4dfgL?A2EB8C70q z=AyI-VbecWKDkTF-U)C?uF76I|NF*E~(&nPWT^uI*FpiXWSgkHnlvko!69j{ma2Y)@` z$SHrjDPt^WE+#wW3_LgadNd2hcyqX~tzeS5zPp)Iq7ftCNy+v%GD-a~cW_7V!*-1y zR_*>kf`2(Rs{@5>-vL_kUc0pGvN^6LeKz!g6LhLV_H_ELm@GKeKTF*6z1WPHu5xJ< z{6%H6FOScab_n4H4u1El2-&0PgSI<4CWG%B$32;x*R%|k~rfBj5}vFPDGxF#e?FKYRyH4}GNlys8WUpM>%!Lso+&!)Ve9@H0>JP z^8yEdsahn0tmG#Xwr_pbT}5@Z<rXN5rg;(>QCO z6sZ{0wfhNgwj*->Ms?|V`P9gh*;{N7?PdF-P{0bfHj$Y@Sz9}8G_c}As{Dr9$Db^g zfOREbXfk6Zc*IE_~0Y+qhgn@X!65*mUvr>q;$jE;)9mqGeVpmFmK35w3*7 zRpRtfDr;4)`Hj!FO_B-2aPzoiyO!flkng)!Crry6QR%@L3d2Tgz%pim07MrBt~yNW zE|F8u1zZg(2|G-)lGkKs6PqyJOCsl)$kzWdPA%Y$y3}|*(Cd1-R;HeO%e~GxP{UjR zI~@I{UvjcIhO@a9XK_rz*9;$Q9TI;>nh*)`etX)h`HbgOkGbKGYDjiGe;wJ;{fZ=Vw^`eaAlZ{;A6nUH#y`OCvb7*y!p@ zv`25K7Pd@{?et4^mKd}nQi(jar}QI%>OIir4drJtv8w;bc^c=XpN0$YFn0OWgRi93 zrzX6$f4-T#pq27Fv7@)gYHKZFL31P{rYH);}`y6g*bB>qLD_^8?Ka8iP zcOq9(az%d&MQQViX8fKObdWL0b+yVa|AjZX)d91Q7`2IqA%MuDQzRvQ%N>J^B+_fw zrtRw5UHg{nZ!W-~?N3TcA8OOZ=+EV>y$&~;*L|7p0bbAS-pwE>hCsjXhe^%8|#jaW#Ts}SU_t?M4w{v07 z9%b3t$5+?+Je^PTo%C+KePHfQ6;8%##JMrw_KEEet|z>ond^l&ME5*P-7AfZ7n}a3 z-{eh|R0usr6w$X9J48xN-e4iF1wIkQlF>~fOfc{XFJo+WYl`l##)3Adb~P}0(+8>dmL4j2BNbr?D zS~*L{x;nq_K+9_srQAxRH+|mo>XeII;Tbm+spp)*n3-K&#RmL@^fiYi(&ajN;idX~ znR^ZlT=}m?sGNE&AZOS`6`DL*Z)2X{FTApBW6oMqX~M)BYpx-Q6}a!$ zV<`Pt5(CB}mU_>=$itV}Tl7jTS;|`m9;oTBq^_-N-|=7&iU}*b9DG`-Fp@ZicRj9Q2&f%XTthXrVJ1)qpdbPVxcV z!V{!?L6`vOIHgiSQL;xNlTLMsLpS=Ocw4B6b`kc-YRKi|gHmLU#pS$m0Ax;Ph?v#_ zUhnU145V*i@C(74Gco|*6j4j$L(xWJXhiMajdmhuw^@1580bFk{=unlSn5uK|Z>Y4u{fD4!UydTxrrNIVAr_%l_$_ZGZf6bJ);=bfuKc z>B&UgPai0hrNTW=m}yixj;V$RF-vS67gfgJhyP@7FA ztUuGe+2w8W20Mu~^jBh1KVne>LV3d%SH*J-23sv8V`@9_kApQw&wB7JWvl?djyh-M zn8e`+jwT}hC8O^hy1U?a0lobt?Ur9;*9R$j9vVx>9%#c{7P%HXc0?N3;j>|RY90tU zo7kMy8?3xuynU^!BEryVk&=P5qWB$SIN4T(t;TG!}ZL_0zCmab2fzH7vtaf{rWipGiFb-yD29r5k_n&;hs+LvK!!0JohlS%mR zTCLp4t%y)gXzqvUm~B_AU)x)bk0Qwp%H&9Op@px9`Mt9l@dF@|i3IKapaTqrRVdog zTAnB(1?>uiZPbG>mve5(fdgCRV4!kxoknn5cp>^-#E^Adb*U+laS$yLUwQB(frBy7 zV|_#H?N@mROj;qf&_XD2dL+wWt4%hq$5k3P=MCjp03!b%$N$vP~_ zx$XEEaPH;(B{{TZh$Jl^*j?x^I;y7{pI;4h@Z`=wKa}5Z%DS6tO*Fy)Nm%x!^AAB+TqGY6jcqfa_VQEX{yr<;; zI$L0H4hMtHo^hXl!K0TL6(*7KU+=}^3M5T_F3Mh7{kDmD8LV-(q~OT^uE*7ck`0qn zAg{e|cu5b9gd;TfsvsQuQGQFHhAN$B9k(NU^>I z8n)HpB9v0;7RbuZ3Z3fj3=YnbAEZ1@BS@^X5VZk{NZ(i(@JBlHw=qX9)oLdB4@l^t zMgE|9dMV4CbvBotE8D%{Y_ddvh@#nh=g;(m2Eehg>WAUsA*Qqw-QSm|=iV0}dm%0y zr7p)VSa#n1VJLwHRaq~VvT9_qQquYw3s-?*stDB+xUH3X@?#B~qo261N*8O~kg82r zNU62a(VI4ACVTx3&`mM6&yu#Kkff5&!@4q7$%7ttQ*M^x!pwkm^j>3@An_Mp_2V`s z3##LF$qh7TXmUh=F>U56zX?GS7r@0ijGeLjM-*XGV`W*LV@Xe(hD=zrK5HEwDUYi9$;u@<E0N)mZ^j+Iy*80_v4h4#((1)}VlRQx2E z=B4GN``|u5vDLCb@?Naj%!P}5gktyHZbPI~CDMvuta0OK&^FKfqZ#D5u}JY@fM1A@ z;fJP!PD>tp&vp(vh6;+KeRyLPvvkHVe#+NcL#9=XGyoRPyw2c9L)hAOwjv>sQB-xbCzG6^H9Gog51$ocGX;SxfaBWM}?WM_Bq5lkU9nPHK zmqsEdkVT6;b~}VjW_t-eGkbJ6z(bAcL^v?C+tp!T#y|mOLSox(m$x^xPDyel(F4p0Ub7Wu}B zA}a_4gwr$=DKr9lxhBEEs@VhL(dn`Dcu+T#&m_5CD{WfykMlX<#t!Er=eRQ-{fT;0 zkd-j-wd9P;%8RUXEQPSSd1JxsBdC+7An~pUYw;FQUWCGQixfIM89@v1fBAX; z3qEGHSy!u+>he1W#VK?{PEa4P=uNrz*}M7ogPmG~1r5&zIS%t;rO!M!$9)A^Kq)n-M?8B_ZD_%L~ozQ$%X zjU}@5%d_PhX0wHnJEYj0uD-^zz!}LIkM}iJ(`P$CGJNQ-Y~Eg4&z&nLB8aKBi2uC6 zs_{@1yIm0U#~EG%W-jNa=dS~fSMwAZLX|@ zCZqdw~<8JzDjBYY^|C23M)}J zUw1fFF=O{|bK0~sI5NWTDSzXoeU7#rIB0Ki2D%9!=5IU>8hYKml(&ZWBp=okmqF-< z(@+1`^>9}H?{L3i%1yTPd$w<|Fw5JssfHil>8 zVQ}2Lj5^vsA(j8Mgwe26epvK}IqO}>NvA!7(G*f{+qihCK!h?gfNs}h{=ZEb9Gp_) zKau?e=Io$Hk*JJDODBC)8t21gi&qzUmJy}L{C}k>tlvqT#%26F<-h$qC1hBwiT`c? z|8F!j@5Y)d5n&&;EGVfvT(*Fr$=h?&)-|8v>PV38|6Z&FK}{8bB6m^(7wSFQus1G? z$A2!C(*Cm~ur$(q_B8-&H9mn_dRkDgJ?hA@Aj^NA9TgYBkfCMJ^gvj4hwV$GY<*p6 z!BTkt{wUy2PBgz%1{i#HS4%&=SCj;m*B&P%!|pwRzrL3d%wIXPN<>`h=ko= zDRFfy2nRE6odt-aJs;{fv4xI%2#L@o@H^=o)Xen=R`UhHS z55k`vfWPfy5^QvJtpM1IyOG13ssH=dqs@@t>Us;te{Y0|;Sc4hCVC=mM}iozSN~@Z zD?x00>ELUxebq6?ef*@B0_|R7`ibpM8yK!H*7*4fc>zi zUdHYAh^L#Nwg$nf`!L>mI$MxbG^r=vNjn)x7y8CQHO?%r`M&iW%O_R9A#~esBGrVq z?c&)J+-e4*ICb%@qhEj0->J^^sRf!8Mqp&uW zKF@!o$L{y&=wdl`l9+eW`Ck9~PB0vMb0buoR+rWCP!Fg z(C*FS=rPFGgosJc!<^z9(C;jWK>M#spF>&VT~M5AsN1=1}KW9P;Tf#&AqG|I9f zIrbdytD|ju=K{siS8XD`i#7f>Dce<{*_+E_x&ns$mj6{u&fx}fMi~K@WjAl6ld+T}jd6t?-F#=r zzQuFaxrO+#rq*1G$mE!d<;*Cw=9xrV=E30BA#+5wKknQ@8a3u{p7HLn!V%QTMlW!V zPpjr5$aB-@? zu%CVD0(g4;zyWfF?YO63T!8Iof6kzqz<9LCce2R0N^j&wZ$YQS4pA`P&~Yac;OW-_ zftcNI$hezl{AuRJH%g=Q-X#5?(Im7o!Z-BXtK_s;kXU@pGAkN?pk&7_HSx__+{IFx2hnfgq@iav zv$7&`>}L7c_15fn zV1}`nfeOsmf4f@gTV*?PqXZ+NV&6&Eie24K0JN;QwF_-`ckqW{Cj>~ZX*%I(w^so( z(*Cr8N>v`Lb$!h%g4JP68`n!nzGkbZ1OW{;dFq=o5*)0qZ{C0r{ zkokn+)4l>m&+6*s+pD=0Ltp%KK*e2wY?|ANxxI(2RV_fd9sd^lf6TWu>VFm;)42qy zk4LmH7}S0tX1>;-Tjv85le~;6m{5`y>TYr@wjU!uX<5nmbP@& zl$pufvu}PVXUAdOCNFjx&`Y9F=^WOT2M#~Iy!Lf}u8DaBK#4En_N~mVRoU}Q8YZ+A ze_d^HgAd6jdSP%01+Qp4fLsLj+9QeS21+22)VR*0{?HSjH@t4&G|${$2}Su?msChN zOfs4qGF$_NW^Ek#eX_*kNTFT1Lu*odF0o$DI$!W9+p?qMaJrE5j%B&Qrv#t}caqxq zqM!W#omsN;^4mY}^&x}P5OSR+D$LhF*pAW|^vi?`uRxpMhWbL2Q3HgOP= zmN&S0_j}Qr*z8n=;e1V#==Ye6!exnXwIgZ2rE6n#03E#pu z?Xlz~Ni?fB&e~S_ie3Y5vpXyLGnCE?TX^7BsIo+}(;aTdL3ilk%y0A*c9dfIW}Ah; z;Q6H~s^~-+B0OaA{z{kzFG@9b3A&4jA>3XNCXlqsnem!k?}hCmr4V3;{?x)_`Ea!U zKXxL_|Jhae$9Sp*dD-S%VBomCC5=~_d%%8m^;GIH{|wo*2F4MxJXU;%Ww#bbHgXqu zo-A2s&*yFF&6|A}gECI%Jq<4gJ|0-FxQ*8RMAc4;xW6R+{?jG9&CZw8n#l0+Fx<9~ z#V0gy^yw1ozv-k9Yp&`T?!SNe^M9rLzgyi4M2L2$>0@=qPw8>V8$TSn0#c-a0JC{) zf5##9J0;Vk%1is)(c(o6g^{b?_lXjQ?X9^X!~SM}Y1q z2g3?qqVC9N2bY2Zdeq@lIbG;V;Q_o}j1qI~{qQlE!1+BrA-DyzOxT_7TA)=kVfBc|> zL*PS%M;nP0$Wke{HHDV&;$~Bo%MF(CH`};vKGEwu7T|)&wA>#1`eLD`@te1ylKGBR%+V_Wt~*Tic~e)TpAP?VKxlmcec6xxNvxNG93w&!T4tB;~=b0 z&Ay2ZcwBG!;%D%$)KusDXCGNJc}q-`jt10=j*VZM81(4J>;)Ao<2OTWbVtm%?9wuD zcRL?>tT4;D$!&l?bLVrh1dfBk$G;Vl$U51Gg*%BTXhAm7AHUd)HigM@e+L3A?8mwR z5!^p@PIjUkum>>D;7i}6L)q+52us`{cJn7}G9LA{JG1+Dzj6Dd;^IvDFohxvlbl}X z8X_))tSC6#RQTSi@GMd26F;2$MPtR#KJCwyaja`xRJLxio!Y`*)26WmMtu~zBhkl* zZwU1e)tB_GAb68qkqWVM=iWYj$>26j+Lkz#zKI=UBJN1W-MFK-<;zyc3vsJ{UYJaO zwCNv}(f0!7ODys+{#I}r7-oWa{JMqS*^W2|$hz43II`SeOq$%U@JIf{kv&~*e)M78 z*pa^>eV7uV&pC;hM;E)i!7`bu{#?lR7*8yef8mGC<+%Z`zOQ-K`uax8sPA@HRM{4+X-kUxlqLVwY7c*hvMsizT4mqlwZv351|B-Hra1*-1CiYi>KZg zs%(Ex*5<#wKqzJx41bN-ArgAt)!y60>c{(G_%{;?Q|+&ZGLJ>fjpdZ^v=`o#-P-s8n^^ro9yB%k7rI~Cl z;)wcUveiB~memoM7}QQc8TnhKqbnh24C=}_V2{mJA)iy^5t})}XETUkAogM@jCB#S z6eI@v7E)%jO0Sb|0daF=+Upc1ba_mvNr2zxWY)e5sn^SPsw8R66?l5(|Ncjeq5oR< z$-;s%z_P5ym$XSjdlg_On3p-c5a5x;R>!nB_BB`%lqyL8fXL_|77!V)A-+zG_@?&}w8L z&wPIixG~-}_3?CcO0?pHtcd_4(@a6U1#?q+Ykaf-M3qA^Cn4zqUEP@+n1~6IZr3(k% z|KfieLr{>rk$YUOrG$0O_KbFSfMzz-5zF(VA7KX$b z5Jh&y*T$aV0l6Uq(NS$3NdSJ(p!ryt^n;|6`{CoswSr0R-EgAHkj5YqFlEtiR3W3w&PCXvN56iPFVs7yA ztUXP&?F*&)g2avwWD5kb!;-4 z#3s)xf7$)~)% z*lS<8t~v5Tr2Akt1zz6>>LA2?IPpPoC3A*zkyO^3AVh@$1G$DJ8v?`o773yp?_Qa2 zr7z|}b!XJns|A?cw?>6?ry-i-+d@;;ugz+F?tJ%H3?=~@LM@G)FyX_yTG{)HHg#_H zC3|#kB~dJaap};)2leoNtDbrZ)_ss!PnCv^UevE)o}et!!ph9SGS!ZOEJILtnJQ%q z56?yyvF5e}fhsimwNGqSsVfU#>JPVJvU^kF+Wqp)I93ZCy>$vo#Pr9Y!vjG4iO|J_ zC({{I4TWLl4;A8b+u1+@OvtdHEi#`aiw|HcXy0JFNHZ;A`6po`vX=>Jwg|axo=DK7 z|JXPD4K1_?u_|*-XPQocKo)%`-G&vVBq@7Wy+967qGxZ1yG4WlC1hyCBOobazVRyG z6dd~nwhtfiY5U$@4W%iADLQvNEYEJ^Hd2Vt-;YwvfH|}SiZ&}oB%3w&seDBTeRSzx z5jT`qjP*~<7jDpYksJ;S!mYv7eD#@v(x2JAgn`Em*`qh3_e7R!Sa3cJr=V4$nVYpu zd54cYE6#+^=L`ojQHM{6ZBZ7B9`zNkQ3 zZE-^yzd)Q~ZdKs1H^zp$EDv|1xzVwDRP)C{`pJdBlXEt}gBxMi{FBO0pI0~0Wc|T_ zH*Iaghcv0drPS{nMMz1B&h3x-^C>$mBm&tHp8%9`Lc0d_cNCW2tkbfr6CO}6)NTC9 zNrU{cDQ54uO=u#B#TX#pE-(W3cKwty;n{GAC;f3CG9$hTn(A zYNQVpi}(?H`!Ne+3(YobB-fcZZ@i;%Qs3m(%jmR|%h&L!3ByuP7mN``a(z$9bsk z;wRzI(;UA=!>I^@>-V2QVye_%yTc}E2OL#PSOAZcRb)SkI;RNz{1#AS_gt9jBkc)) zw|D$es{Q_88>3fHSxmF{>%JX2Mf58F+PrLcY`!gF7I?aAQ3(;K{9mk{WmFwYv#<#yK!RIv2p)pFLy+K- zpur)yyL)hVw-7A2ySuyV#vL~9Y`zUS=gm3y-XEX!TIme4cF*+m)O6KUJzZnaR$Q-n zwC*F3Ug9N;?=m`J#Mg4vw<>DQ%>V<&AH2I zCqHpVnkph@&uH+!1RJk8^G0C}Ep`TB`koZZyw7sbf*|lC6)&zngG0wu>6HmL=mFt{ zvA>y3k?*A$ax0m44M#GQw7m1vJ(SF%P21#RtUjH(JDane#)OxvAqv4?6&$2EyFQEv z99~zk8VL;5Td`#S;4!MEPi8kkfTtt4GmhbU_AQXJ;PbGCZrfPvh2u%b7kWqMRpXj( z9L|Ag+(`app@8Lw5MKiaW_sq#D>t2;)T)5pinUVKJ6tqXJ5H97)nDazz9c4u3vw1e z4Z`gmt5{2~a=i^~qkhThy+x!>96J95GxC9ye$;uzQ1~TqwIDpsX5}lDA1_;yJ_YN- zp@PHx`#$}CC1TTLsB6G z$dc2yz}XlW^J2vEaYjmxE%D?CWR+Sd=^hc>RD@bAcr2a}c0_{+Jzrrn^LLfIbHOR9 zqEe#m%Ri}>T;Z{rAa|F>M90(Q!zOnHW0Lx>tQBl-a8%D%Gq`Zh_TveU^$s>=Q@+^T zIM=}IOdfmrT;sGZrLV*lqZ1ByspTShK53Z`aGL3@KVW*8cG^xtZfp_m(BkM%hU#gD zf90K-{1|HO8+5dONfZjdLLgdGt$}WkqnWym2{Q+?-h29nc(w1Va$WgW1 zMyefRuA1zp$*dAs`y1s6l`I2UZ3%&eXwO!Jit z+I{^cX97%r`O#B2pKfF?ORuNH9lxYgHWHY)9jx+905-2pXv$GqVa+zurkOvGGi0Xw zPR=6SN4a6H3OZY9ZEM%+Fw=!vY2V%&M)`$n%@+vv$t(|+EQn)`@ue&v-4nJb-e2nW zfu;tRoC(p$o4UT@;mw!(B$ja~nKhf6K{FcZ63#oaA7UzBWn|>S*m`d%4|mkpHqc{rY^n;Oc%pdWPOaN{2keBN0Vo7#;ix z-f0DQ9$a8v@ndQ#4r7$ zC`iYi6WqlT9p)>!sO)U7Vd{44=&#KQoJM;O=5TxGYrG#syp77J!UPR8 zYoQ720bG7<>P;tRd)^fQQqYnI;gOSQ$mzGEW}MeI3= z>Jr;$OF3t1m{nQj`wbSx+}lI9Kfx1#zk4ug&iYq^n-%E7SH8MO_pz@EhL8SwuO*I; z{y$ql@Hoy3>Fpem7-btsjCy18-G8Ys$Dbq*_%wDl(oi&P11*_)v@!-u=zeF* zuwX;0e$%FxTH`>UNdOGi=~9b3psqi@Hm)zUDfZrff`5$THMLY_*J?o7Zp>#Ec6~s6 zO;Lrbvd29h18|)CAIs?0_lxqqpCcx6J!9NgBlPfdfrZIn_cNw1y>q*vbG0$7zmD?B zlt;k4ftUV6!uj4;NQaTF3%Z{&Rlp)Z#1UG~v0)AT+r~@YM?Mz^0s=iZ;NxSW9=R?f z(ZEYlNOG@IKOb0h45QbIR^Ij6;hubBnX>y<90K7ICh96C1!?(3-_$?DovQ|je5w5P z{(Z342hn%Pair8@u!$cT=kyuH^hLL>VECs$dg_u#HxkTjZnD_WrC(9m;2t*v&Ta}h zm)MdKnnY8%L-*;VNa5KwRe24@)}~@Oq~MzlvGw(8Z|;2o6{L?lDbs!%$xa)Q`>?8a zcq2tpy4~lbtV1rzc6VGeH4;y5>3Vcgt#T`@rY0?0A;X2L8i3!9kR?wXa2z?Sg9FKQWg;FxG=n`- z>r%4)eXXnUSpP*1!$gApz{P=!6AIT02XC~facrCmntjCTMbj{x?q5PAM}AD9-&(Ct znBCJNhWFo;yE|@bb1E9R1$qg>RC>!dCCkA8EF@9hFhZ+J zH|}@UK58AJ*X!Fk{q~y~w_J^nSAcs{Ykle(4ja2f--4=?5P(z?8iI|*C0Oe-XW&+x zpk!bOmSI+97Br^+73IP)O_y7@^_TA%RbIc%tH0{uq^I%gH2F?*BE+nqV^htnT5Iy& z0Vm)P->H^pJtBHZb)YOOTZ;k0PlGhKOKV`kx8g?9z(MxQ=YRs4H?L8k>ARSIcwlYb zy~#7f9#$wz<(~N{T+Qs`bJ-99LXU7cPS^^cS3lrJ?hc4?ocsw6qyH&~pAf(%{Td7S zlhq;YR`^EMhLAP5?d{>!0+UC}AC-kT1zm9Ig|Z{w%C${58qb%{iyp>}C@b8M5ICOT zFQ=Trvx#gR-^?_G5iEQaKcZ!Oz3UCjk`N$)3+rNC%8F-W@4#{=zE6K4@iUhbe3gGn zR_d>(RgsO|OJDt|dVNlM(|P%!C5b*OIY@d7UC@>nMP}uYeZ!8E9`qVIC^Q86s4NXFy}aYN7;PYC&rc*r}3;)#X{r|1Yr zVGTC~lDjnemQQjldN8TyIFsWM+5x0aqNsjB#D&9|4fxh^<#ww0f=uI9hl6eNme#4= zmT2bw6;cEFqT8!Rm&nM-EBlP|QXfA;r0L5PEX2;VPy$b4p6hzm<*hA!NXYu>B41C> zk4eeNs!PilOoCT~MT5V^V$Do_|B>;NP?n8Ne7QMjzc4(l8>rzF3*#Z*(PP7>6~6BC zaPUcYTR~2)(QAo<)?%C_t@1>P{sZg5^;L4}fyN1gYDdZTiPG|l$K&op`b-EcQ$^-V z8jm@RJ#x?WzPVEYoIHfj`|ylM$PK) z95Tg*PG6*s?t6N9$`4nJx9;`r27(U@)oW6T>?pt!3-2{gaN14UCkhMg?RE;s%iaOE z^t02B^v6;&nJtC|XGC4Ybk1sOsu_43_Z;ghjc>AkqbrsxCzT*6O%L!`e{{5E##lI0 zSrT!2h(Aq}e69-{HkYdS!Yi|E@h9Mk}$Eyt4_mYJF?hM{d>rp ze?WudQ73fJB=Q@jd9zMiZX=+L&ak=|A1Gz&-H0;@k$lQIneboloykbGN(#>hV02!* zcyUgj{5m|WTboBzst=@irJ%8Sco3uH6#jX+IJM?li)d{1E$Z6s7iIO$8l-Y4NJs{M zKNNiIE_c<}T`+K}OEm?$en*L_i8ATRzaLA^xZe|b;%$ibZJ}th@-=7rcDlj3cS8~n zzzx9bo7y`1_{N;zaAUxUf;N~KwAMPUhLeAbnz~3R;mQXI3FWFju6)AmV)!dB=bOv2 z5uqP`Mq~d%A|-H_vgFiW-yqrH>?l-z-VFYL%jcT+XW4<5N?ItZL5#p!yI{e7+syo` ztY7>pZH5EYKeSt=vLN*X>TfF;T<`L(p~A>k>a07ZlCU;=Gyu4GqbHVncmXcM#KfTY ztj&)`HSx#-*_kgO9!>84#!3Hfe!^t@gv!}$wLx9a#4P=pJex5`tvwXL0*A%|wzGRV zxHgPCwU}?~!9dgBrR5tv4#$S^xg9ommeu>~IF&QV#*BCW`WdP*<9g-H_0zr+#@|P) zP{Qv-M+t1hTHBZXGjHC1Z^QadNsQ_;#jhC;?utgvljv|5NSR~CPJjFP?e>@0pOD}N zJbX7Yc;9Cunw-AFQ#+2QPW*mI?vpd?At5BxAcvET_fioTX)#w$%(VYpTdDvAbY|q| zXlywl(%9)W!lQzq0MVgo2X^}P`DekjdY3Iy;mgj-rPyzyMIlZ2r-9Hx!-pVSFd$QC1 zRYYO_U-rRW+PwhxgXJZ5h>yrcH#z}-DT5CLq^o+=pOcI@9b5KJ1x#poH|@QXGLJ8j z01Fb~m#3bju-WLsk6FEDH_6W{i(R*`g9tAf_=lLigrvs`^+_5$$4?N?uEG+r^DPN} zYW(@RF($ZiB!ks#Az7_@Xfv(If37tv`eenPWzC*N7HEr%fd=z~905FJ??-1|fqV7% zh7l_;OyWoWnrG5KxV+)kL*uOcH3)ETz!&p;w8rN_wjOTJbY3saQ?&UrbfUO*>Y?CH zgYVeklQ4A-JS)NNLGS=HO|0%S{--NK_u@?xUXO zw4ydPObTHi><>pWueG$3!%nUSM|o_hY%)p)e)nQ``gs8GseN|7L|=E|voR8}yk)krMCbU36nvs&CN8u)`UFbA3YTm0kSPa95|-6uz}#Axu{6RwzWZ{cyc1 zWdS#j792M$6nOfT5zb3XU}rmVUYXWXwO^M-v#`GXf;F~I57Qmp2xDEk_moF%bWgnU zLP|E*PIO!lM={%Fzqlz(|FI)?Gwp8S%a<=lJ50&R^O8Z&KIk;)4R@M8ox;33x&tUc%OH(jQ@36G(JWVHA9rnJE(vWdKAdd2Ok&C(=l1+S(uQiK1Qa#?FMo6WipFB+0qA z1-D$`(Q=kMAHtbs&s)!QW1)PI+4NEQaLo;-);4?aB_a|lDIF~JS}F>f8Ul7^Fm2Q< z`h$ZXj`(L$2*UlH8&aYDy)|CHW|qG&yGS1B-}iV{of?msjYZp1JE=mN5_~~LWbQXB zx3J;4*bNh5nayE7weDc;RW!^zs=)YcG3Ac@rNQwfo+r4BK`2>l)ECowiG!*!q29>l z_HQE?enft6B|Mwul40U_i*-y2M8X;*fg2oj{zhCJJ3xug9d~`;#jQ*9jdfX zxHiNm-CcDm_y(;+_W%N_8UfDnNCsFoWqo;89n>*5n0JILPK9&4Rkzv*iKjrv95w7OmOvfHMFa&YD&+8*(a0fo!~e>FH^GQrNGV zw`j@Enl?=rs}i$;wAsI2<8dT%U+v`{*wnv!@6&0ONkkRxWYTX*=+aJ(`I3Jvfi!4~ zXK>vEAbd^6l*r@8zV$=r(njHPZ6>mkA*v3OaHJzYsu+sa5TBaN9es2RZ&K|K;nNnC zu4$5i#7WH4RkWIuGnX_2r<1aQJZ9N15#dvM@9jx>=1ji$8ChCbu=p=6vvyw%R^S=D_5qYLF@f4|;((57jTSMB*& zXL1<+9RE?vR#YNXCeYE#qnR;Ue!+T+?yF7v0X9+A@(Jfa+#{ikI?9Z3SglF=X7m)G z)ZT1>FCFC7_aFevkj${wt+;6@HzA~FPj2C+65zDx)>yXkD(AAI{2Z8QYKypr>G9}S zWukQ4jqgo=)xVNAh*7A1$1$RTFLM5=hW=21OOy6G;@+?Nexo7Xv(=OK#B7sXa-?r)^PLJe#TLQf20E~N z8d87J53%$k4@HL7WDC{=B>0$JK)$u`s>HQe&3R1xv|Ez49jru1MAt5<)-CC7RThe* zso6P=!!{lDGUw45vKxV}%utZ5TPzp_4CswL6KgYMPC*zV{=?UYeAgzEP27#YK1!thv9Q1v zf%R|Qve9bJ#&{hSs0%MaY^F}2*pAIj?#+%@NL#vQ9FU0;AWbkZ%ivpR;s$-VE)-rL zNE!X(+o_3pyxIc5*n8tGvBDqN)BD0x{9<06P zO{>Rg?e6xV)-v*1_jj`fG)EVNr;c@)lR2bTZGM6cj6}0S@h^H8zZg|v!TOI2L_q;Q zTi>?S@u8t-1gqO;DhnAAn{>;4ryCOx(%o}!1>#yRVFxr#I$oR2HL~nOFLQ1O;ZWZ0pcI34 zQe?oXV*V6fZJFCP zeGt_FqJ-=#wcV!3?j@-JfIFX2vSb=V6Zyoz-cuM>k9HFY&=7A)vE+JJ+Eu?St4BL)E6C;A-70? zLg(L8*b;a=NE8gt0394}@HrFbkTivTmi#K_K3XFE8hY>x$8;Ceuv>8V?+j^4z4K@c zchI=?OWHtzw=*+v<9EXiV@9?aBQ?Ks;(QB*H{TM8wB^Qc?gaCA6eOv=7wuqWg@nsu zhm5Dc7Qy1b`KZ=k&tsN{3%fsZf%KS?mfpm#Z^b7}E?;h^gMDFk%N5`|iR(T*DfTeS zboFXkUCBkmYQ9rz_2-?w)q*8kwK3Z{y|(fD)dn)~7m?Mm&29r>e50K2pwt$wQA+gV zklDyz34-39ei(7B%usqWv&@;u(8t5TI#uqnB>R?bPWls!$0TnPIB^Y=cE5reGka1R zdFkOjTlxcl&Ft3cuS8Q8&;G>wpmhM<*;z>ZNMmNNQC;5PV~T->W_PnNEUqm}#ZgKd z8abN`?V{)$CC~^rfWVfL54YB9h$DNlN;{^sZN~C$hx2XAJ+2}v;>M+>q)xwY!_9)j zX6-!3*sc==9Ybc_L(bS|H6Na0H?$Od;6`ZEF=tMwW!ZI=X2m#1NCPmrKZejuyGJQo z{rHT8fUa=s<29`Q{Tgdd=-hA^U2ucea%PYEapwNDGGuN3JWZ?eskDYp zzzgx_HQ$-H6&(dqZzxV>muDsF9_rD9Fdw}r94^;#8gi`_`t3oNZcbd=9B39Bb`>t0 zxPuyVr;NrgAMLJKYFQ6X#cjHg6y(iUy)iM%n+FkA9}+BQ;LV%!LX@^Ga;p*Sr= zCQLkkq=hs(a+3sKR^cHeZkL zs7+TX)vi`X>g-Kl(+@Cx@0$7C2|kbx=)w=gD0%qiBt38-Y7VMUJL) z&}+P59DBE#1Sc+Mo2!j6_+ZhkF=)qK6SkQ}&e(!AcxHn9?bfiVLG@FN27HlWV`A;m zk0D*@6?0pDImFYLwY3(8klKLw_0u&$2H?i4{A&8gDjF=h#^7z2JL+XI&JPxITc5Zq zFcI-=*BH?&?tUb{jP7&bSGASYV7r@sNbQ-%ch?JH)G{s3EiCUI7{aF5FWpbrD;-nRCX5^_jfmzN^^Z@Kh|C!kFVIn z5l2w%0_**1atghBQgeD2pxg4{8uT?~qK>4_tJy6``FLFRa*}^UBIl$v%bC|v`?V7d z0x}PpU&PA%0L)EQ?NTXMgO7;?mSl<%SCyGjlp)=bN>I&V8wUt-tsUX(9mogU;ofJ1 z-g0_Mw$YrDkeZH2dk<>Sm7iz$GnS&ifSi%7A>DLfgm_V?{A&e0x) zWvht$JuHtr-af$P&83pCLD|)ugMHt@wswpPnAL_fF*b~wh4<#7MAZ)O{pR`A?t*c# zW@bDG)Wd1mf{~UV{ccfeY#v9PW!DkA>p*DD`z*dxOPb|BZy3o#cEY*VXsM?~Q49E1 zr#wt^<39UIK9I2`H?K`O)Zm$W8s<}2^l=ToF3cW0@>}CUUyuucJ>x%MRRbi~`nV9_ z1!ug4-vk_&d~aK{w%S$J0Dyx_G;5m*{{AdNFgk|$x$TMtf7u3+($v2lIRV#c$^A_( z@6E$DAcF%f(GFBj$eBFO=&wdbh;GLRu$5c9x&TMtuC5ZfAKQfUL!t4w_*TaNJcAw} zBa{YI#_PG2fXrNYMlDucD~=QzB<>~l1L=@0(Lvrv+^;YjuMDl_W@fPoU)z3O?#8*q zFKqMm`jxR>SLi^wUmonbTso@l^g56Y;5oZ+oH ze}l8p5n7@;xNV+xL+myMqf+~+I7R*K zHt%_JPTFTDd@?fL8c1k3s3S=0yw4%f7{)^vUp#)1V2q?yCHNH!oldn?4(TDx05C7y zd3}=g3C9vGq^~v}33LukXu;K(BWv>saiE1#7tq4Dmg=N=0t6nChTw#BL1dgz!m|MN zCf+VsA#E<%UHHdSol-*aMEggKW=55JUm(8?v01=uJf8(#AIcGaS@fVr^SkJL_43NL zVmFne@kW`_A0lc0{k1iMDjeL`n@cWgkTbqF;`dbCH{r>t?doQkh2seP z8Sn}8X>dW@^y6sa31DM`JI2d}S+c@ip8@f;*^=QXVec92-GukBd}|x8sMTm~`uB zp&DO@g?}4B!-@L8$t*+j&P`}bIU|~%M-+Uk&0Ox^seZA!Nouj;wKNp;iz}Zw=14Xm zbGMNS`OD6A>;(6Zgq~K*X+UABamOmpbVWzsKx-ZVNfAEa$IjFR-*_bSQDBB>^-Imz zTwPmhT=y?Xq@)NmY_nLl?G)4wHYK&_#g8X!by1s*9XTyA6D;=U$Eqr~Y^eqfb=2O( zJd%`jp$p0sVFH9_a!@KbT>iaeR>whS zwk7=bbv*rHnuH}xm)2{j=oc%|S!Lk+qZy_dL^OyYPn%p=+y9^cyAG26_}X zXB54io|F}iPW`L#7kaM>-5%W(G3u@S@w0y3ppq}Ay%}ojR2InF)%GN3*QLs+U6eAZ z$=h@A7*Xu870ywaRp&{0sq}&n$omP*iwo?<>@IRQhKy)v_7TuMz_?A%IJ_JE+y_Q+ z2JOeD_Z!UV2>C!%isXnoYm)_u)=p$V+KIo{-KWMrYce4S!E{no_Flhytl}hR3MKq& z7#ZqW0d-)(joY`|Ko_QiQ?5D}Lqxj9w7#${uSL*ks+#2it&}-l7-t9yc?kdK8QoSq z^Y+bO-D3w`T1AFejj3uq8!X6onR(pKg2c&8ntEf30J15)tfGNG+Ywv zS(563^PP27xH{|#FS}V0?aN&IqByUT>PwcD9>u;YoW~U3JAA0S>Fp-BY*;GdGc?c7 zb$no5WEvO4eZf$T7neTzemmc8?MRz*@qQG-FAXI@?SdV5o@0fwHkGXK7QoiaZ6ypP zNWE^JJX~9T8&OpICMT|C)=(b337xD?4{uYUagIg|S&zLs2d=$Bo}tU0>Y zrVEnGRQ710&oX|C^r_$l{@1$0sMGPzKmop*{e{pwTMsMt^$W8f+s7zbuQ1BpgG1+e zUYUNVO~LlQMLtn@jj{Ka<9s*9mG_y%3$VZr5O4M|Qp)~ZeTg=`@NRG;r?698y&`bq z+!l54%4MK*Tk-*>-F3ROPIHh)2+Da^XVNyo(Uy2SZhQ zcmCot%%&MN`B`G3n#~OU(?3a%Ln?cq+Tg>;Z9PY3Ca)$`pVEl_u%Mi1IHM;=Bpi(J zO6d_WDV@$7@nA_>xszU&8jLMOj`Kk(DK({q)Auk?=Y*;2O{H#JO=>!3kgzQKRGpS4d(qaLZoPWz?wf0J0cUy;Rm8p+Y4{d z!sSg6+W7TDOQ2&>{($SFo6UR75CUm6c0SkYA_g#^2nAlior)_?AKDNU$?s&kw4}TR z&k_1V3w{+EF*f$gs!y8txL7Dl)oQC&2!ER=?!wm4lN}Q%n7U!kYbmfE=9M5pZuRh^ z>3)5L(6NeR-InwAHVn&>az^rjQqURu9e3Qu#^)5hSY8X}-^rySr;VQCW+=OU@Ipw< zegHlW0s$8klYIRp3gGE{|F^wGXjaz>Z@7|#t`w>i^ljaf3$OhIZX;@!YCIpR+>?b} zqQ-iAmIm{r_A?oE8NF3#Mg18Dy)|9hg|o-6bEk)W^-ELdr5o>GerVhzv?A{|?{BS}f%1_O48SirlRVSu#)S}Re1 zIp)cIi~Z-B&C~k1CVK`=_-20qNSoS@WeVNm@%xP8^*0V!pxQ{PzXLBgBJ4KY-#lXl z(DCRTJ=kiUrgJV*eD|dQ(Ai5kwZ6%lV@Y+GLlgINh4U0h(rINd~EuDL0C0&3Lx5 z!~WD1Hm*fMx#^n_KmEO-kwJp>pU+*#$yZ4 zrP!*tPeuA#Ga{1iA;;6TU;e(?jh1*rH)TYE!>ZsYq)iV1g~pL&v9s(fm_6#Ft(>W*S2Ggq5k zTwS94j|G=wWC)?YOaN3xqTPybXul#lGa5vyA@I9MoK z0#g+QaDLgPZhbA%by7|b>8AwQjUXAP@I7-Uk2Ho<{00Zdx}Q2&)M~af@U$OP+qop> zfo0PSEBaI%o zc6)=owas8xZD+&(Uc`O<*tsC-DagnvzcXsYeF?zELPS3vYM)WwBImT^F-GvV8MkZJDGH|nSzX_gw&yJF0+Vz6*6O}*U; zyES_9CWxx~BHmIm9X(K5k!e<}6qqxbc(%3osv6LnG3!3syr?^zO&N2-4oIa>Z2*zH z-tPBptSEX&V&beYcXIokl))D2w}W$=yVOet{Ef)P9CY(*;C`v8qi?Qa3@WEJ4w7W5 zSJ8ppG}I9^1!*=D^PcK7zG>YY9!toVK;cCM1A?AW7wU__bTcRmS(tR7$9J{*)^5SuyDsVuiKZU}rcfh%K8yIJru@1bpWJ;R6tyBS+}ZL~=H z_;7@VY6vTgT_~&XF&a-c(5UCtmM*Od-dxjSj=M)m%RKB-{_*P(6B~$yCxAu9#rRDh z6%q*p2Z7)ZuQVvx03Lbq6R$Y(06}Sc3!0v&xbIyMSk93tZ@p7b_m#AC~P5z|- z%0*YJdn6|PO1;HfC|`MGnQwFxdBXp4of-T~!h>DTz)->eK|Nku?wSE@8oK~OY%%@zqcWL#}d zN8v6B-G`#JlR8|Bf)WFy$p-n8hcbo0X$7>R@8nE7UCw2%J0Og6!fKN;6zc z*u!Wc_-x(y0>MlsV=(^YKPTC=bhIx!{|Tug(JyIFv4*O|LvDGU^0@}bM!|Rjm|~kv zRVShIn6-b1^=*Op_o9u{Oa(|0)!RETUAqPb591p}p^p(XpV zqsn7W2_Gyh>^pAGq@Y8mqNB`4kc5*=&T~Dzbb2(OE}|mK8FtIbP4M`r=>GBJyRpbG zrZj^_cIm}&&$Pf3atkh+&ysPC>5cNidRG2mQUXPGq7SO@s>*T}6IUrqykCB^5XDswDj?jm6+$!KR*`#% z0LM5#Z#K?n~(#Vx~2|UrUu+VM0u-R+SAvg`cq_hL!ohuNW1?V~x zyeoG}LjX253>xCTnXC09&Tgzk#5G zhyoPvv7LWGJV9-S8xaL#{{(^(Co}|q^&R}{XX3y5LMAkx`Jcampg|2gWH3GktrJip z@mo%w>JvC<9oU=z(E|k-H+As*;+XjKZo84fQl~Dj5*!$`n+e9B{w|SB^jLxcLA!$4 zRSD!Z12dc8PRGBxdt$avo-i~B>fbb$1E*z!nrBaq@1M_3*uHsz6(TGkr7wvR2|ln# zZOI3)y%hX>vPO=4s$V4qWZd~|C~+DuaRp)Dwa)1MX?vXlgXFIk%6`al0EjI}1anvQ z49@}D21DzHU%X!hF4|C^@w-1NWqkKHO8$4xpZ||6mVOH9|GVHnb6WqG`>1E1s4&qP zWLZ_`p~1FlF@s^m2?wnYV4e_f3Zt18oLkQI<8xz;r^el#+7)qNUhTFX??2ZX zDc$Y8Fcq*cCE7H>FE5GbnFPM2^%);EF~K1`5LK@hDJQL0eh`%LKT;NQK-CmPg7<-Y z1HPE&qy3FO!iDdJR^+G*fuYI2i`fR4kSok1=gzNr7ACsqKx7INcmOV>{YwV^@MR}d z7t$~}$FQ35Pn`E(cqDj`7HQ-E{WJ6UU@fp{J~jR8XcZ8UPek88j>VJ~cUJ%=27lA) z{~4#JjV$kHI_|&h`z@O&k(cOWC3Itb54$KRVjq8aNTKZ-U5G}vz(lY8TymTyb^zm`^@Jh=>NEy$xq$WZX-MK{w-72PU5ozl0`xD~J8 zkpt}bYBX7bYLRho_IY)WOn{LJ)3Z52KYJE?($mxXS8`0Z7SCW4UFPl8l*TD&Fu;r! zSUt=l#;l$=KC#5FVPSuOHK}Ajva`@9D=m{FqbnsaU zc4faJ5%)&Jc6pGJd{BVJ@C}bJ!XF;dA6)6@QH=9&%ZCmXe$(O_*yDgzc4(W@)k{RG zarfQ>lz!rqKAVf2)ESG!-5ESKH zC({Zq27B%+zAI`rGEW^DR%JyJF`aJfi4tLk*;ClZv#SH0(RV>jj~$w z`07sh-z-4?xe?P1OYGv=pll)SyI#q?EF(3Xm@b&~4Oypykq@rO(H6;rS|p!6rcvsB z0^HpgdMcqJwtfY8FRFb*9$NZ;@QSs-a1%FLB6I^mona_6R7z}0a~l|H&E3d zb*usrE#I%}s!{P}(VvKSBhqUvlzMK1&IxeMdK|v__{=Il#EZ-(u?HJiynbAD(`k`N zVDA?g6A3?V)hxF>#}SIki)#N_VKEE=b2G4YI&PpCmX1u~(^ey;V-1P!cFvB}Snnr} ziFGHXdpW^`y+{}6D93tpN%E;Hg83byj@t{(=0}3WDPK}`*0{dDz)t`1@gFE1D<^DL zauKO?ahtNr9=3&@1LoUX{PxyYAB`6|I(Hy{M&IdxY5WC$?VeLd?pZ3TK6N2-w1YdW?el-3n}a) z9hupU@9#wMqJ_p9{hjWC3$QJ^z>$+ zw|#G#K|K|Njr#0wb-19BLP_l8QM+>l$`uLJIlC0fQ?`#lNJp->9hpPe5XEoaIH0e+ zk@{xZ{T`K0l$a8-@W$W6Jy$2BDZxliL`SpEldfU4oLmGU^7}z!f`#a z4<4~9vZW_z=~eNmE9(}z?9{mAYVVNl@IoT>2<`f4*|y1zgWjbqC^mMW8)INj1s$0bU^KqfZVp}ToYuk~VD_v{IGY~6P~i{W=H_mc)tOxP_q ze9ann2qvnao7>ntQa>Def816DF{7%6D(~>scUPJ=eS=7$L z&^**Z2qrBcmX)D@sF{2-qcy@9x!Mj%7X)o0S}{zB76DZl@s-yHL>6) zj{D_H=*oI^r-$TI0-5K!q+t^Q9{zx_0uFEWgc%KM_VzHUIN_hRx5aamV7x2|54|X5 zpo40gAWeZ-hjAhYAkb}xWCdvECIHyx>$GZo>)PV%s*XDhWnEb94il_mSJPrnd~Gq| z)-8Ns4@3K7XM&^gT9~!_@a=e7V(&_ZRf5vO^(t3M8xfYEApMMg%B5=^;l1xCjw*jm zR_}tJ5Cv2({g)8B4cABzTCw%3>AM+Dym}HMmqn*t&icxc^iPY>5+8nH?8`|c-U+=T zOwz$}+xKALFyyd#tE+7zUk+f7geeZy-{%8$?lEFwo86#1*`)s|Gtc^!Ga|8VW@zK` zWOakaHT2b-qILYx<|h|s@APy@S6BG%lsI*7Y4*xIlq3}nGi0By)IPXvdO<|>FtsW; ztn=*(M%nGJ81&(~3A@cYH%&KY6R%6S56z%Y$d6d3Odd zX-)9iG-fOQLCjjLD)_|>Hjr&tE1>K9O@_nGZ3c;sYV9M?V!~#+sWEoRF6Zk%*O#0a z{C@!9D3}csU_!px{`BpG>*8I5HO`xCCkmB zJR0Cv_WA1hO!n{JGe?7Le6d@FRX)9ZDKErIL`{U;F8e~3=uR^>MJpP0k>f%$8bfw#%ckXd}=H5cd%x;Ot`^mB%!m%uVDCWWe z2r*yI##{U*(21g+Cb!0wm0QDxAi}|U2@dPsZjeYUI_dryQoMSCUWc1n`eQi5?fu#i z-d)v{)#WFxm_k8)3B5VaG|P_^Z;B}qA^t9np5LJ%S{oLX=Pyn?cG|)VR90)Jn^@%b zm+Oy*!vRB0hE9Q(9?WtMZt^D|&ZjFZa*HM;WLh%vsW5!!Z>L*!lq?jg?j@Q)JMqp< ztl0UpkHuFxyGp>R(j=XNIPMw+3FC?$sPz4jXucAsuKXB1qoo40%A+Y0z^lKz0MSU& z3V{00U*HYHg9Lu?4rT_wZ$Kwm3&{XQe(A_T2thwJ@rO+lx9h19aXZDP>3E}B4{}P#F#*y>qO$bIa9Hk)cu>ohk+v&=r+%GM)3`0T`4R1EiF#3k zl5RDw8F6F^HW8fL10yq^dsNByPOP9X^0+yo+(#)2SgE?=t=!^fJ|Co3oL{Fhyy3Z$ z7BZpK&Pco0jkhk|0(P*x32RW2;kC;pep9}2j`hZvb`$u>ujzW}jkq~|iDei~e<3)BgUeQo3nXm2igAC8F;}T5c#3t|C9$gavt-o5;Eh1{R1{%TS zh84c;c?VmSqXfvVD-VN2oQ;oHWWt;=Hr7OZ$gCVAeX?4ipwc;5unLUoK^M$~cudSj zc^EF2M;o)TYjm$RamJ{hO$+ZkhPv%&AcP^rS5a$Hsy0vdD+KnRpNbsLoN*Oa_{rd9 zCpURnuymu-lz+3s-qb=Pc*H3!&jZ*{DDX-;Y5myeF7WDbkF{;;L~6PEq1!~9utu@@Dt&(EAa5>Q zX1bH##m=&0SNQq{XK(fj5#jYxcAbT}qVrcmsmDWRDeRPA-hg>r?Pa(HJ3MBMt1i)_ z6iztHSnyY)=LaLC%}0}gMU(vLap<($nT=)^Ga)A@4%oL>%I3Nw5IE(tX9j57QJ7$>Fn7{eZ@%@w<4F539nQ2G+C9-uEd*iz~2fT6VZDt zwC`!6?b@Bc)P$=tRchAtDd?%qOJa2M8DGAY6@fD@a~orDJ6TJPg0I5i{o2uRs5nmR zZ`^EV@oRnOHO?HXgb04*4E}n43`gwkq3n>VyCf~PF^gr2J})zWhYZbl{d(1VgqXTY zkCWtiIx+2~$-Ck@FtQ}Fr13R zJ%%j=&0v=ELE5iXPEQr}e7j8UfrOK@3yAwWN)#qa0nc)AjrR%x0%CLngUn)7q+ot` z6UHykVp0RcCv-F=j|AN!`c0}6XSrE6FR!DwK7i{p56QO++p_aZhn#M1X64Lz><^T7 zln*QVZnehVvTsHXr*aa^)%B`M#?q+eK5%6m_L=h#U}i?7Vq4!G=OkF1iucbI$Sf~V znY1W7uS2(VHGHo*<-DYxqUy;yXOgZ%XpkLF2r8Sq$2_M7Zm`7GZ~m|Lo-?4SrCHRA zUXgN9Kmi3&YG_IaDbl1$3!w_AKWcECx; zB=0^nI^Ou%4!9+AZe%e;43vZ}YIA46AyGHn7h|&IY-YU};W78I!xu50w*3jE3zqfh5s1Bf8UWy`FgN-Ghym4k(;=z z4vPPas5;p>8Jivg4kVm@eBC^kJ?LBO6Jmy{kEb2V10}G;wPlUk{ZcjqRp^a-g)FUO zTS{xP+ETu6+WKPDe$zhFVInUucGryMdd~xeI@jdy=jM8iAL{ND)?98AzR|{1?dBaq z`h*JF-figag2%;iZx76^?e~e6RBay05I@(M`pSZ?etJotV*K3%Y^F@4KQU9Kou?0# zPQDyV!k{aqD>-aSR~&(_?km(#&M0k`lx(*NoR4XSM(j8FI};7H(rSD&rYx%Uux9F! zIu+z?J;O!ZF{CV^jeZ0=c~fjfV$jIB6tt!`KhBv`Om`u>CaT`3bQ-uFJOgiP98I=&AT4yccoljytPVv>ewzQ`1_w^dX&#!%5#VT zSzZwhfLp$CYA$}+P^nDiOF+eAPVXt-N@J*0dBgT>aSP~U|IAGu50=Z^LR)F-fzRJb z_*|-6A&zaY0Jq2x)#fNBtnoX?;xEr-#;>|ble~Ii7Di%a`q(BJ*O61vJ5~Go+*Ln} zh2Bl#Rb(tA-m^|r6=nxtbikE9eZ8x1V{CX3JHF}uW4?P-;I@k06IB0_(Bg_cCHzI>PTdBAqMD3H0^@i zBY3UPk(c=SR^>vvS6INF=~u^#S*}!yeTW92pBRdAhXXrZy+dThQ&}!nHuuF!1@f>bS+?E zBbtJhx6i+9q^Vw0XT<tIwVhiVTghh+@(r)z_BQ!jVZg0)_N{T_$d}69PkJvUK(f7}v-h&xqZxz-`C#%ee<>R=VEMLPcC&+r$}P2BriIFNN!6uM8M_0V zyk)Fb*pH}3%XvJ8E#)&Qyub{dI~Z{H*=wS=9J6&{Za+|&n+qt3GuA=?jT-h8Q$Wd4 zsRSYgAe#KjXJx#sZekyRJ~}n-wRB^4eOI>!EpSZvgMfGZcC`;-u`x{4{*>4YcN_5r zg1guqgFHvO*jsoFlL(r$!c|xvm=4lXz;WOfXw(s4-ea&ToLx?7DM(8RPJR5Pd;bP9 z<;DwUswY+AI((lI+VZe4VShn<89j_?hythm=fQs8$D`|bi zue|xA1^Y0AF^JmzxxwYL?GG%9@d8VddK`H7{HILAAO3L(RkqQIW1Azin z`Ss!a%Zuj`bxz}oqLcH-ozQIHZXg3Djr7P|rVO-p-6`cx-IOW< z21l}Mnc_by+v!9G8aU=Elqw0a_uTfy%xdR^jOK$1`>d;7-P!}Zcsi#X_GWEc1B2jf zIC4h4-Oh!ZZ>Ht$=UzdIlEj9J@ej$FLzb!iq9UpKBbEF2d^7ZRR?j&YkSeC2MP`=g ztpY5K`w^tH@h@h^A%f{~>wk`a5tEOlCi~{opV#r&pHLHRdUo$jAu38gtDGl+n7R9? z;DHBe_!7da@ioH5?9M^tqFm+Vjx}WKt!K=_1Vg&%}Fm3BzBL~%isgN%vaD{ zmx^6|a`jO;INb@ugEb9Y{nkX9<>cjPjsY+A8CkM1>Fq3`RaS3LD;0AmFrqlXt?(s- z08B^y=S*pd4`=wA>hpZHi?ZsNDi zwhI)mk_MN8r$?+fRf4R!`S5U`sNKzbQ*!#L+*eSoD0(3&s|{I#=~I3nZAinfH!w+o`Z$*XqHxN(KOIPTLV z9_JMC;xu1j@oK+q-xK88=vsC^hkQ+;M5ugaWv}vwX9qmvs7buA%!4RqMDe(V@k;9H+%f2paY)-g$2HuVwlCqN zEqyxNA%YzMkxN@ld*4I`Rn^y9xnJRRdzi!vv<%)6L>*n@V4ufRlosKv0rZ$gNcI}Z zUq6>Jz^W|jWZF3~AD{uBkOptsEDaHo-V*>aG8P6}_0zH7ldc?Aemndub*}q@Uelku z5T45oplvX-wno=i+E(t$5I_zbgpCiXAc_qGc#1MStHUjUu#;6J>Iarc*VAmdV{{mmr6Jw@fi$*&Xwx zqlmbBS+czTb=1>rN*~{}uJCxut#oks5V>Yijy+afm^FzJ-z2cydHU*Xr-q>a_gW4R zzYfypu*THua4Ro{!^RdUq=Lp1))JS-&Aouew&<0=-N;ixp^A*}W#@g5X0irkArl-P z?kcv4DqETKiDlbR>hp&9-7N_BbRLsz-KU#ODkXn2s{-_=@|ZaH_S`WTqoPLuWX>ct z&-S6tgy$Zn824EVY*170C11(48#mn5<^?htKbxb7M2L+ZU2<{-f4wM50odrAA8%~3 z$?@m@*yvLqzY=a$3%HlcomqG}Qx9!#jd@J;#vWRxix`Yi?svyqGZgw^F2a1uhK|KS1avue+8JPb!?Q8LWzER$| zv<6&yrxD6jXaHGZuWADwSNgFBaw>1y-kpj{Y`!doX=Y-~Di0Sr2URSHy<*%L_{nc_ z#R92Wv9#9VH}E;fs&%hc7rv)dTl6(+ZOESKp36wEaNV84wWV`pg#d?Ueidc4@0W9( ziBpG}V^zo?#5mKuEDg;%${*3V)b`G143k(tgjhZ#c;7E?N`SVEWuMC9apj!E5{Lq$ z`s0zT1J=Zoaj%LlA*%q+CW3bprx@EeP888(Jzk?AY^`4#8GaexPl3eOH@Nipl3XcT z+>00|u*=ik7?;EkUVAN5mWRpTbx%n!%{X7C6v=DM`37u`6E8V@tc_Oe( z8ZPhCH0ugZ`;1WOfaIO|aXhhgdhy*W>J}gtp&#F1nVN1t<_=g>sogE)iJEah-*CiG zgYaToki28hq6o;3)EYy0s+Lf@Wpgfoy+}-^;em9Y7T2I!ONf&7?&px<7f$m+;;u#i1g&+Y@7MRp4q+`PZM*FYTsY;n1YwnvE~!_>gUUNUtOop%;J zN7{auI=vTsrPUGavfJs4Ij|39YJ=;`%Rmco?wh-E&nk3srcYjD@Q+(m)Acr5-*L^N zAmF&w(tjSQ7snWm7Z$O!gJM8Ck|mhnap9nuN%FiNGZ5+Iqe>^I;w*d zFz8w=TYTf`$RC0tR*Wq>o1tqG_yiZ@^6OV`0>B-AI_KLKc9h!m1bGEB)C?|EZ%f6hAi$Q=($UuN z=C#^}%c=@k5IsHK86N=wlCG36Uh7Y_H2d&K$w1!UW0V4PO5`Ny+)g%-Hz0PWv0M#k zxYv<0Xuf=@N+cNn{S7!OCBZ`gQX|_AO1|S{z#gnM@6e`6VSht z*~6~MQlJunuO>$d31xQ%0NM#asArs+dzCYhk6&=sb)TJiBXhltfz0>vyokFexox#V=5BKoL%_$b`V+%#H|4>gd)Zqs`v!hf{uSjN~GCo(v(oY+N7cBnH;oU&mZQL?Rq z-a2~Uc)U?h2-(k0ms|e&=D)8KCPmb z{fzfg+f+DR(6(ngmt{W4Nz97B`{hyvPec9SP+yj#c(e2TDN(%JAw5HvWq!Z_Op>=- z5e2Cl_|)jT9NFPEj5AC&qkx1{vgvS0|4vv@9$1DJ#@1~K_@RR>dB7H{NlgPr;nGFC zwh#gAnj#{b88`#QHK(HDo8hjmfl6G7BIz_RcyRfHlKg6m({mf9h?3jj^h~g)#7`Z3 z7Js6C0cx|-i-rTv4imLiO*kY|F&0CrohQHB*MMT&Yl{t_wp%5%LdT!QpiS=aDRjv7 zSpS(C($Mpj2|UkjRm<+Mq`hSXhGdsBmcWYxmC6Ew5)+V(qgq zj>or%8SvHX;;MQ%nP@%?`?Ot7)BY?6iuLlC8vDT|Sm<8$jO4gbCmzrm9dqFV=i#wc zy@d!}Y@8|hTRvcn$LoXUlyGvx%ncGO(`xEw31iuAJQTD5aAm2bw-7lM<340^kRV`+ zHL&j$k?#BAK=qwyXnvfbM*i;3KKh&nuTvKaY*li9va;HU)U0Wpgjj7$9tx1=grvR3 zO;GlaO_LCl*S+E_McAa77(f;dMRX(mf~2`Jz!;x+;PRc#l-;YFGHcB{lAWyVY{*ys z*U}JKBM-eCtVK?pw30(?x&jPhLVaBEprP8~rwUNRO(#rzwPB%}1Y0LJNeke7-5^u0 z=wRjt+h=FBXP8%~%(h%104b_9Bc~lzL=yV+--WKzU1YwE#+Jb^RT^Ap2>?Wag zsp?3TWskn1>1y9=-!8qW{kZ1Qn^e|nEBcxQ9N{$=YVa5+lV-!;W_Ry~G2OFlP~=MV zq4TXHJyk<_+R8?KG#UAL|NAV3>*wl@luP=;!h(+uKwRA1#idN5;M;(B9K8oRMZL9k zgqG&MxqD&=NK8+@uB1PaC+!-pHzl%i3nP}ZS8SSVLR$ORPf>!W;G6q>K^zfTk z!1+H&T|(R2k))kyoL>^|1)%-+bfkg)kE(ggW=1LhwHws399K>(qnjDpM~A_!$MYvW z>d3FsJHA{&B^*KRHeWMgjd~6iC<@n4QBqN)1fxdFO#=SLMW3Qszx|WIcabjOYf=34 zhhG%J&hmX`130PX{0qk{$@gCUOE!thzC$IJUwF=c(Yr~G->@pZJ|BmRrjEJ)A!i0| zgdxD@y=ESQF26AEGEgxw?xXj{UHoAgo}oM_FK}=fVtaa_(*N}L6Ut#7`IVLWTmzBAni$&nVTnYRt7~M{e`~fBzgQx&Ai&#< zOv_-HA;R;&^&A}qWP=6u^yAR=dB!VZsmz4oq)g>A7U3TYl4rz#L0C{= zd$9wyE``~2c1ncpWEPlLq~G_Uz=44gWNF;mg9uD%(oT;hP;J?7xUcE$oor%=EVu0C zoV%S_HfV2L-DFDl=NpF|`Z5_>hO)D{%hXirfvqWHeNgWIBW@eOwT` zMH_F`7vbHlz$Ub=aQ3%Ku9rK*>(N8ipTSto14Nir7JBx zc>kYkEGK9j!dZRAm6erSReOmk)ZYvvn2^M#r%ifv9IC3Sa2Db6{jGMv$Nx|?lCKyA z#J2lT-YcHi!LX=hS6r`6D}_~(Dywu|J!*QBP-6Sk8gc&_#ntPW%iDL{3e4PTt* z?Rid`_12FY=YLTQ6U$IKQG$3o$n?=^S3+J*`~Wn6PVe9EaFsaS2eoI1~sT++BkQ z56mXyoaAJo_WZ_HI?go9ajw}OIHt57Yi&aM<)jhE-1vs!om@1?c|Ea zZk2dw#PzR6(k>RJt~O4NjG8tM7Fe3rZj8Kwj4Dp@0ug%PuTMzuo^0QU5fg)bTG4vYQbAM-#J4*InyT7*FS#C?qI-AP0oSRE z(vR}T$61%$ZTnJ_lt2iOW2QpAF_#W|iK3$lar>#E#Y*yvNC(A>h=V92Od?n|z6g>d zbj_aWKzI=(QVeKu&^Mq-bI!rkdctR9EEJ&=aEQzz(Fu2sBC~aoPw9h1GUWxH5ZUpW z_qQ!G0%c(pI_NJEzRmR|RNvVixh=31$Sk1azmT!y%Ha4;v`;H7px3|}jO=E6kVtH1 z^PVqHBg%QrT>@n~DWwU&Qj)eD9}_y8@PVYm@a>+5213cHq>HaFivOB~aJ*BYvWC!1pj{3{!r{BlW1QU}x{h z7HY-1!d9Cfsm3g zSd|#*+e`@^D@F@-q=>O;2Gdhe@cbs@EhiPc%Ei!p$N%}{N(58xmm0b^RMLoxFc@Eo z%!gDMGYAKc?(wS`eXo&lZC;vJ#sZ>&XV`~(jPkey!QyY0zVIjMV0sbDV4}*We3U&v zhzmkQj?NZdL1|~+mKWiiM9W+oha%niQbn};hJ%eRMf)HgII1cgSu+}~E<{PNs2%t* z9{QIO8~-=enivP05)O(BWMxLzWA5cR`OR5WigdBxSw7XAu^^&GHq$Udhv%H_DHZ#% z0bj0+)lcaUF)m5&2u6qY7f-!o=}7sbroZ|Tw{9)o%XXOUPLzlxR{PwO0*G3>t+ zgJJkgX7F}q2%Xh$MBmB#Ok9q5b=&S(D7}?9$5W{oXfADq3c?P4EduGC#lWGRl%01} z;U`f#m6#}2hd$r%zKEIB6}iG@(2Ju) z_WZzi2+gpUcDd+rCaGt9swSrrH2e4XC~*#e3(^y<01AIzDh2>ZDT!FQF2+f0!%5R) zNVK~RYqbwNMz68V0!kBoC`l4AfwYW(2&l;$3YPuoN9q8UTb>_ZFe+CqP#x5n*EhkW zyH1hGT{M5p#@9{CUVw%hPN9!@O--gh9_7!m?H4%_KuF|`5}*L?4qW4`;2gcDyY zFQhs>L{nh+ZpMU0$WF3DEnH)*&KF%0UUJVu?Fn<=7G6`@9kq~vQ%R7bd@fnI?lgQz`k8lyHn$zxm(JbwckZ7&AhbFe!#V z(St9oa-b$vsn8DYwhoi$SOt^gR4i!~NPh@@QJn?B1F2npxJf8h+_7?@W=DjSH>^{x z)4O+lim@^~1DU(1obIOec~uOyu+%mk=ZeN@Fs8rp6J?sR68)p{)cKnZZmQL5#p0Y= z@m-O4kSHJ(HzSH10-uhfb*2BD+aOm-BqKMmrB&>mPBeNiHZil<4!;04w z+}i5HqHCzZqLT5W1iu*hq7u z&E!FArB8R|Z{POeuW^Ci!;ef}GO$xcCC};~UCr6-9JdKxZZ04SO(1o1`gW87?yA+o&K>T)dIlje|6RtsArt08ug;A4 z0yVm3;>(0Wps_9K{0Q){GmjrLY1IHeN3ZzjmN>6BRuSaJ?0NtA*L?jrC9>4+!{d*y zn;CpgKKTgelP;ZP(yw0}xn$c@er5xm{tR}6o~ltU zOEj(eh}KV%%9+7q&ToWz<22eHp2tN3p28;d`(nY1Hu=uw$gUKnb2I4NTefhgm1d;A z?HEv0vM90-6qc36Tu7X(PLlO}`_&A!3t0eBCy&jI-*+aNqvTTlb=Ig46WfJ`{5rdZ z?CwH~B?wx+Xv>UOtN?yKoQ)4`+=qNly(d33_cNBM|tV~_N6(C^`uipl$oMZ zh1aJ(en;;jx^(1--id*7&Kbvd;`K9(X^|ka7vMC9Pu;o+C$e6dh>-mO92cchhHq}7 zPIdyL7wI@=l3#$LM5L46M@||~BBsmDS@V@^d=XyoliUW?fH{gRWX=4~tU)LUXrh?jHZIN0i zh|*pCrgx*iD-Jwm!$GFfY6bxLhD15VF#Po)NgYQ&l0Ab*IFm2-Hw#XhXb$zu3OvzFm#JY|y3G_rBZ4&s|LOkX zY_v+%Ohj!wsi)o-!(3K8>#z5mP+lp0!&}O*Wa!iO{OrZPZZ$2tv!}W^LiDJ}aDoY; z=lT|ncbi5=+A5uFQlJSst-@kf+mL9jOHsK!M5rYCBRxYSwTUbD@)C+C8;PJyU1#$ zaV$-(_Yn$f9^%P~LmLTaQqMWUEO%in<29Y@Dpx6PM3AmJiUOQSkB!C6gm&Ff{xpBH zH*T#wcU86l*$a9ugg^C4w>*j1gS)o=5Z+Rw#6k{{qZCQw zTC*)!&1{Ocm&rsKV96+})LkMJbsF%dA}agVR5M&H09=xkP=#R1mWu6K(o%SPdnz;V z675@ZriYdR6MW63exbYPMt5}#-Izcp?h4wdS1AK~st#4m05bah9JZ2VS+M!7C9^EG z4RBxso#Z-vh-+mW(aRNDdSGdNnxoW*kM1leg?uyyNvU`)srfBhGrUs9&4^a-8%4gk zimS+&h6Q{cy4j^YU788E7mai5*togBf zC!}yF62@Wo;($CzXufW$H$l_NfxfBX4 zF5j4SMmHy4h=i-8N?8l4Ui$sj%A_=x5R9Y`-xMBC{-|3qNGB7S_^vw;FSgX-Gx7si za4FG+xj?C4&H*xrFv_`;{t#OT)dzd%C>BbWfcnMEXLWS1$~zeSMpIXPLQVY|#_e z0zD||=QuL#cy7jL&Wg`oe)96wQo~%Eu(5wCae@fSI^9%@j*Davq%|3gHPf^*qvo&Z z-$7kum6O)p0Vi?mO07wkbP(Ph{_I;JJ1aalu31f`bxP-q;+jV7lO8oUGEIUil|Ai< zcij`_60fa}9AM=#kf@@4IRxckDUJ&XLw1dqa@2dj1VHOn#=0DTb`ATWAlPos;8k&! zupig16*X7sQFD8>pOD~{fb}Aax)%27wO>SUyy&a?eQ>0c6Je=xvQKteGm>WzHg$TB z0O-=ZkiYE;I%5ndc)Srp85iFjEjdJqy4 zj#3qFX_A+XjSPJlPTXO{SB%e%VJy)|;6r%4%uY?9Y9twPE z5hDr$HGeB8a;(i1zCDo{(8lf5!RLz*q(o~CJQrd4!H)z>;jPnZ1s;~hl1oBHV;zUG z*1f37YoVkh1@K;xxeMemZF*-q(V^w)OP0;gE#IN))(|*LXT$UJ$X|lPIKXq+@jb(^ zM}+Gk&PGvIvDa~Il>DKSU-a0Yz8q{%?{rf7vAHUD1Lc-DtHg*`!ZtQctVcw_RaetIC#PwX8POV{1c(`n)9&yF^+wb zOrGelwR6l9;TKh`T21BE)*s@mC)3=`$fdWd8467`3x=qirGAdEb{55UrdA7YD23H$ z@J0d*I_WPa)jn{1AQepERe@2!6SD(h__z3}3Gslf)B5W^yL15s6LY@A+;2F0Glgqq z&c1fu7E-0fAL1)sf9VY2j}O3<**;CE{rvu(%MYfVz)a&?&~bQLKVHkM8;rdZy3eNo z;<^2R&6X<9r-1(9rot}9zG^gT-cliBo?WveV)yEWek%KWdMX<2F2)iFK!Kq)M!ht% z>Pfs-0gKQGwHe)E2EdsO`Jl~fuQwtT_}-xs+B|wWtY2gbaD<=IaIRTde-8QD{;0Uv z%)~mRz5Shyu(~;43j_>y-}^bW3k0^rConUyI;+3Hm3>TD0Ub2A+t1}s8c#(=Qc|(K zn~&hHD4;ek=CV1+I!O%SnhZ58vunn|2M{X%&7V+{2NqGXF^5*ruWL84uwKJ<UDgfiEZ~*xg&-JFaI?t)*M3V7$W#k8T@Tj@A0z$hluVj|Gwj$M&>UPUWWye8wJ zQRX|%inrI9?Ic&?@-Nq#&smP7(_ERJ+DA`3_NdQu+@Mw3PzKHYQz*FpgO{_(Ua87?SHL%FY#IGZ2l( zhY0z-2T!ioy;jwj|8D2&UkD2kV|G!!l2N#AErJ9G^5>+bSc=9R60DhUC z_38DpfLaW*)<##0P*)!)^le#-N(4Tt)Rt`2pTnBM;U5c*pwE`-W4o>m%oM^c`A-_X zeMdyoQr-ko?-HsJJnzI2rcT`wRCvXoJ8k_>oV~-dj}gc$9Nnf7GDLw4xXzRR?UL^qW$dx{g?7RvIyEA4&O%s$;69fw zQZ?Ak;g^uLuL1>rcL=4SR^mBsN$(`1e;_xZZTxqRP@>JeHYOV>j>2g3Qr4WvG3EanNe`$iSu}UL4&GWnd*-|0`Y!N|8EJrGG(*mvN*Gzw z)Va(EzH-&Ykom+czgFGjH39DUK-c(zkDoN3X$TwzQGDI}r;9ogZRjSd9uR8*%w(0- zRK&7gE090whgGpM(mbykiGxQh0XQdz0Rlz#xiKTBi0qdoo9y^zJY6p% z0bh4Ul{4G*UK>(Q%*Qt@krZULrbIYULa`6h}Prol+VOnG+Tqi0cYZOt#9X;?9oE zwUrgrYQ9%^C7kzS_D8ZT#AQ4Ecn`jm_m;&BwPqF^tWkLjayDA$rc0~-#6FP7>2C6T|v6Jp8 zAl|FftXBsJpZICt8bwNQu#0A>*iB{F*^lPBXQVx;ON}T1u5t=^KkaFnUwk#O+3Arz z0fAveqX8o@!6(V673uG^QM3SwzYqecj>%!gN)Ytbu zrkk6_N;8nt&d`XOAhXsvdjz-rIjOQaP$^jHiF3GE1LCmD`j?1YiEK`|isR=84sf$}7;oZL zUeHJTcwoMvC%$r-G|In5P`f+zou9zqzCb^#?X6tc_Kxw?@14TmfW~hOZLO40`f%pjk5_k8xtWGENDwKCh&A z+j4b2i0w%<aZMsv%|z z1K5gIjZts2&V!E>FEp?xnjk5Q`}oByV*2!6qqbZdb<)}Cem4NR(!R#pxPlF@NWpWp z-HcuAtG|`Cd%%QSJ0Idk!fMlKvlp?c%vTcxF`##N4)4t%@a7YT3HylSda`TvE_(rG z7%7fn;uKnOMH=q*vE;C9#R6{Uysk0aXO=oYvdpn-`#HS|uIcc1l#*2qPQHOyKlcIQ zq{>|R;eUt}>6+}jRcX*uZ3<@PlyX=z*CYI@iRo1qKRCur9RU#E{o1|0vd^0C1fTlNTv;f7Kqru8eEk_Fya_iNk5w+VOFfi)vth ziof$EP_!qv5dU?&g$0&wv32^y?89jST`seF_;*H#;OFpWd~&r)gYP)+_=fu2svmn8 z_CNccAn=kFsFxdGLoL=dT;`o%n(#U8PY`Pt?Jr06D>7{0_^e!Z^?UY11cm&ubVebq zeu2p8b!P*y-dB5pkoW8za6hfbdLV9rr%$OWPs3$GwPJq#!oDcmeXKI zXfJx%X;FsGW-_+h5-L%p%Hn-=T2N|+uypE`@yIF@3Y&d__tfgX=i;jney8H+eCT!) zR+`W$1*Olf)em<*#j*7LA1dMv#96;tjB!xN}2Ewz*9GzQtdGt<4Wxx zbi@@n3pTl9MTbb>GG`DDZebIJyp8&sQ}z_frQVAR6>n0~Q+7nrwckq_!oY^$4wJ~~ z2n4>MZrB?sxPqJUUZ|)Ck_uK4NVzVtDX=Yu7{(ciq?65(hWlLkWI$kvccUWzaD-FJwm-}!EHx7(J&YgO+Q81 z)68$uR(l(mKZ}Rt6!iMZ$m9&~hWgpZL*qL6sBE0bd`R5bypfgOeS+UC;ci{p_SYe- zbGfTp?s=bM9{*a32a={`IJka_gr8{ib&AG9OEJ<1U+a`t^)o?C^6LEIeh!<26`am4 z+@UUnX)+lA+dN8kJ2tx{kW|<6?|4qSpBUAj$6u}OTnKjQ$c8VoVlmo9I`qB%=dpai zNQ#pC?;d8Eu9&t$!_dlaoWkT+YDj(6y#y`jRz-CiN08p0e}q|ZPK4KYkw24tubc8? zUI+R6EM-dwN1tYBDw=+JCZ1s>Fs+R%u>G0>!s%sMQH~L}F{Np!T#A)8?zC2HED3+& zm#T?zLw?kcOVcuII9@cLap>qH(&RW76p5*y*URbEa4<*aPA6Zoo~s$_!Z3_?vXggA ztpw^JQh@F^@8Zg+`GJM4YIR+LvRMqkI7k-k`<1bunWW8p?)w_1I3Lo^mg)45sxk3b z{JNXYR(W{A2|Y7?Wc0SMD*+}P*kB``oI)1#FEew%wSneM4=8rGB?1jtf%_|cP;e*> zOCEe(37fSV%}QbXApu~|KDb+PJIfx~?WYz9^UN<{f##Q8>zx*J76^!hzjII{@nHn! zAH2@Hvx56nH@ZB%Gy?m7b+>q{(lDx#*{H;I+q-khxLOF@DzliKa?KoccEJ@K!($bd z6XsO;c(2*{_3grLjYKU*y(v!PXF;nfQxA~~b>QX;0ky=ML#Az!SUYwcE|~N_mF?P< zp1i>Jnjn~u&9EX(TKR+DQFPVC#3G1_#rEVQlN23E>d>$j@nGLCJ%^%2_6Rr`$-PA3 zQudv-mPl)2CIY|n!P(?A`rY=`U>MZAs(UkbHHLQxiWD%E-|O8qW7<} z1-)`tzX{5#X(chg()~rFMHwf-A#s&ko&bp%5-Dk9Di)!?5_s=XNxs7UH6Yf0)T2%jSb$oS<)?QYpKelx<;?k@WA=CtADj7J(DD*0ZqMrj6!PYV8$`~}S#sLCjo zerEQazw$&wzlYBCKzQ(j`Vr}lQ8c_l8jhX+5+eA^c039|-T`SX2Q!3SD5!|AZpUem zNmlS28BV}S22=ZVUF2=`b6~=K-152*B|Eh&5fW*xDL)5il27$$B>~QS(k?eL}b@9 zJ@(hWXFc~0G*b1gVr|sZlhG*kA*HSjv;592Kw_-f>3B4h*NV`VPQ2)$VsVC-B@Md8 zfV5G|g^m58k_A^EptM$uUG81=!1{%+oIldYyjQb+c&jrY3tOiyvWs`~{y9pkru*JFu?DNO1CPvWpBdoBclI&7eK-NWNXS{&c%9_ z;^-3`7-&>ZXT^Hd-4y@rY4(8-HEWs8%@eDQDiXUQ3R2rk727M%IFZJY@l*Pr4VqEK z->uHF1gYqEBtSiX-_4lvEZ0OUwkK?<>UtEzuSs19#}b}~W~J@kIo=sRtDJ+8cc&pd zY+YN)Xj+X=qY=U%u9OH(T7H#$g8nP_m?%2LeBAm4UG^-`i{YbV*2)xsB{b5_sQdxrNu{zE{o8L!@&hS1I(pY?*c#A_t@ z8-_>4R1FA%1mE}bwb&>5vxA58+-Zfo)Rd)4FZFY*P&)ltT6LI)4%r-(ytC%04`{nC zC;T1ttvW^9h*5!!*;T?UZO{O&x1>T^t6FKIJdqldGKkV!#I@q3JYn2pj)_^zP|}P| zo3ve_VsuQWO76#aQU^zl9EXsa9YVG1mXRO0X>X7{ww{so>I+l8z2;&QrvyrI%%_@T z>kh29#rq>>w=jX|5$mfRNVRhzVx{^AzVecQ!+l*o2luDuKQk6V5ef<(RIm(Ahy?9Rd&I>)Fo_G0g-o8)3FMXif(O`t}1--=X!jK0tClqtrXX|qTQCh zRTm$-)%kHg-tzAtYpX@yId(^o!9v`J=lkLQ@{lTvm9=$|1cFhzSa+&MSvPi6+EW72 z+FiZVH)*T*W9!Q{0x`Wuxsp~D(h7o%OX~({ciESuIW6z`iK`nYP^>`Es-i%P`B=Uz zL%9#>EdAgRQ|(Wx?noZfwE4tEXaQ83Uv}7PYUkyk3h3R9biqls4MM!qE_S*AemR}n zrK(%Q#s$#yt5cW{-vl%W*|?Y*fnVhgA5aK+*^A4jjbHC(t`GMpoXsiF6{UXfERT17 z`F#shd$fP1r$M+6djYXNqn>%mUXnjspzG@7V%zN)AMir?pD^vQ>F?tMR?h188p@?d zG=x;bHp_xm@Y*by9YqNx*Tzj~!#{)DtC>1>Bj33nkF5%EFpgYGROz?>HD{8piXU9h zKDUfbD1SZSY5l5km`u%_!~63flIzMR?XGBwz^ihLUFf}J(cq;Zcp1#|)OGOhp)99u z7)j*FQ+0WNov@xWuU%$Wj^VU7l`oH!UWxZFB>0y!KGjVH=JN4;m}N}a>Z2fVM!yOR}cI3_#ojvD{g>3{Y8ui`JkGjxVk-!L!b7RDR5 zQT*RV`mc%pvoVjzO;g8ah1;AE4la&~Jz4EIZlm)5x5Gb89lT-!ffypYlXged70}?#l@rCm4A z+}2=Bc`|*e38P~YQA-ar>IsTLc7rPiT>Ds`*npE@2#jNIrp?ZuV>eytMl)BpXX0uG z^KS(Ne*T^Q>Ax-Nf6DLP?xR|u0TkiwVd~HjS&b=*I47aG4hq%OHhGnKuVYu#lygjv zc|0x1h>)Rs>=tru2wwp&0v%=r-1nw~(5{(z0E$fFfn6C(R}vZ_$MwvL!92#C|NX4~ zZE+txGfAZw1KW@pYxdN1T|Pq*+hZc$lW%0NQw9Azey8jFv-c80xS;aBPJ?bQpY|iH zN@U9PCqc8na=CH*NlSBSCovJx@m3CJYRXoHjq6y|{P^E?lQsYap#`GOakE8kGWl=B z|9xXuJ|(<(yen*P?rP5uMbY-H7LDoDRPE)fdHSzpKtBHruW6M+1HGtS!Q_dU2-Qq7Mv;MTw21 za&ZrwM*YHNXru1NMT&=}(=T==3NQG_lrOh2l?%>5or9=S{=z(N$&JU$uUr;U%ws1Ft)8tNa%KiT!pv$gqPDI z1=Cq&dWN@M4;P_B3mz_eOoC407z^ApY9cZOB@S+b&+YYc`W*o{pLQ^)+r4t7BE#MQ zj0_S4&s00YB6krphiw$H<;K$8=J^o=mY$R8yza}Orf=BY%Wq#n2I%iMxq|4x?jPMm zUBTGD*FcU7t!hbc^buVQmy}q^e(TZG)sFj*z~^6-2ew!be9Uhjfh|{-58E*<>8b$- zPQ>o~&wii;{XHr88|HLdi^!-BV%xWAMGg2HOu;K4aFLby;$tGTx!&Dia@@`PvE?UnK4Hq{mn!B0kac~-sy zIZ=|C!Tv|#R=%*H%?~*{DrpNU&9xmYLCN*%$vZtRTkeO_EH}4 zUSbEP3-j`tP|>=$;&Qswg)evsIM03EWf&r#q;ALv3pKhA4QRYRF#c`3@j=Lbx@|z0 zpLkj!`CXFZ3(E)w?;RtN2^wD8>;Rdw0e zFkbEJZ|!_&X53hVO64nwpZV>~!n9}n`e5sQxQO>iHuhxHgAS?p$LIx^qfYqmbq z6+z>5Hyzh`&Rr^|YKRA;xc)QR{g%?8cG{U@(6|NIo37CGHuE|=u9cu)5 zO4dI}zjmxB*>1sbnN#~>^%5}vf~J71kX8PfKZ5D{8hJz#b@}B)4yGX@SA{I80n@Y; zbiinhO{CR359*ASm-#~}h-w>_0hlu$zm<~JZxuiylaG$_omLK$>bH+19Ws_qO0J_bQ*2<)9gsxb=gKnn? zdzz|SBKVWDN$j^oc0jprfeRtgb0DACK-A(_UBCG~BInDzED^?TpZxucUQF4hG50<5 zgN74XB2)WUE&694$My5vKH*0p{g896Gjb9G$KV63ho+b&x%IYt=R6Qljq;10JJ#T%-0vj<9Eq` zDWS7!*>epFF8F*_v$Rx>CeM5vfJd-+X|s6$6*PZ=Uh32mcAB|GONWD2Is)3$xy2&u zs~RFCy@GRJ9`DJ!o9>%a&aCx@5_G|$hVocTtw(DsMqn0AY})SA z(iVJoC_sP1P`*(W;PL8g5xJDnrHE_|fhKdCAb(r=+CS&?_u~KLcQ2$Z=~$54-l4iV z4;@<1lN$`;D!JGwojO^_dSg|Mc|-5?N_Mj~1ybOM0tS#cU3;{4t%5o(1lqSC80ZP6 zeB){ZGPHsu#tlo$fY9hn&B^*L`zm{I4kdxBLYv>*V#_k=>GYz}+c|ei4O^igMWO zf^Hs8%K$2p>7Ji|{g+ot_C1B{wPS3A=E%L80`?ayM@<{-+Sp|8)XQm5;G zr1Iy-Y5>xeLQ)4kBxS7>nNQh6q=UYa60@W~gS^2ujOKqKmAgdTY5yC+Co*j@x2fE; zTT{NSDsf-)-OolfysxqOZjc`O`Ywih(!ML^H^4N6lK&D)J*f0V&ZC(9xz`Q{xoM%a z=z8MsJ8#M*&$n}6zG;hYnXH7Y1y+>}y5_qB{AZHTS=a0e*G6pn#g^xp?TkNqOC7jd zPf7Q}dmSS#(oLZ$ls`RLHt=`%r=p{G8L!s(Nx8^`yT_!6$$_m4AmV-cEJ4|s6S zAD!uZ2P!v1oNi+ zgYRSG-3{97B<>V;VoAM5E}d2*FurN^j&#X}-q>rZlV_Qgbba&C`a>S>wdd@S&?PNn zRDSCn8S#QQft`mJ>F}~wz~A7zQ(B?hh3@)2RYz0Q-k9e$`u-{Uz65kLc`qxn=H#T*&4?jZrzy4GG7b&I4Ba4SJ09$b>lsF z)p`f;a9F_1-8cQLi=At|#mwK?XLb z-9lYHnbm&_7RDTDpVED=paK04C>w;Hu!5duzZbhWY$3M^eDRQ!_sXLq*XXv9-;3uMtzC8}80?++W z3b}A)X=^-gZQ1fG2u5t}yajzTc5&+AWZBd@l{f6O9>alcKU0BDH_}`j^O*3JP!qHb{js~%zg+%d<#M%;Io!tu5ZxTPTU_8JanqaL zmU=j&O6^j78)wDEe)~}q7)wT9R^j}y+x_V-IRMzD5myhR9e{scR`gURlJ!Gw6IPZ->6!`8R z9O3za47(3Y9RH_@w2BINH3HB=qCyD4rTbrZjwccuXslYNrS`*tmqL=b`kGF9b>|>I zNPGL{H)C+m(XSKnB4aS=_BIX$JkGl(VO8> zy4fC?UN|b`zaRLcX~O}}RXiGoH;0o13NZfavXX+^y*STqH~2*Qt0M+K9H~!Q_j(EkV_p zvaS(;0=6x ztfHl(A(BS7&Vfd3@Z8{dBpL`o&ZFKtBX+r40)Y@&O|KD}Z46UR4=mSua`D@HASVUJDxE^+_JxE%~t%OWg3*EuNpS z-`)V_xK`nxV7=ciPZU5$4;k^FgJ8+k*u)a|>-nQ@y8gWTTh+OM`^&uKhm8Idzgxfd z^oh|~Uov!SZ)V(WFeh3ZA>87|MkDO-c)5p#;`2cXa<%6v!E^FFN z?)$^+uI|Gy{(#8K;K2^vZk7`5Bc4yj9ceYYp6EtyOJCn_oOeehjTsoy`~5ZFOHSi8 zl*i)JHpQfj9z|90RB5XrGjq?BeX#rCRMtIsvUKUUs)MIc(Wm>A-TTuT1OK(s&ztwl z?@{|jZ`b5Ek(nlgS{{6h052c==4%R}V|t@%b8g{<>yR7oxugZ-5@1VQO*H(+D-Tn2 zm-F=`(K`8SRLjl_?c0R_R72K&QP1O_(t>!o^ltMTQ_fE1LS7_h(JwMn+7_nHC7RLu zR@cEvF0%~2G4&qzpn$W)!KTfLXwKo3$}doVl!u7Vx@pTDW@(u_AG#%Pa*7nRcc3e8 z+uBKtIOd~3(yd$xLWVO2V+jhX+&0|c(;j>vvtu>(OsX6!bL=9y5f!&vL|qfH)gI#kto02n-I$?Isf@IYmxte7qwEM-)z z6C(XC`J(iB(_hwBUBzzPoXSnPtVz##fF@I6w{dK=ASWUIe9tWhtFz;*qcL*c`#)^7 zbtd#Ij^9S)7*=a($~IK(2POSn;pcRylUL_+RDh)*n6cqYKYMDlXng zEV}lnUJSen=-ly6J{imyj4&l$gUpug)_ZlG-}*#`(^;-mlBV)kz{-t%P^2CJFWT=X zMhLK^J2yYh%!yZHmB{Ws8nt(XK1?=lhPucl%oLm~VluieCbBldvLavZjkOXPYq<|S<%YPtA6`wsX9UI{n zGdyP7k9GVy#Nmsr92Ng9ch1y^GO7;IP?ZR!8Y_UFsu57p(Ln6&3F>a_U7|N}8T8su zZN@HC8E!jj->$U>Yab4WkvIA|enPM#Qjb`@(hJNsVSYS+bSmv<#w16|84Kv^UZ80` zl866D2NNC_F)SfH&@I8AxFo)z%Zvt-4{q2raAdbMquzG}9~!HI z63L_C-I;%0ES;G%=JixUHW~K%=bYd(*EFI*(J6>i8yb%#!8Pai)Hd0csJ>o9;G&o> zGqjG~J0fVvmMr*ji*+0hDD5sN60cyuOaYK z`vT@BF5q%ePy{97-fOmajSR2fD}SJ_A0c=SZrP0g-1Mt@FRB!D(C~QEacY;X+uh$d zcsJ8`bL;+1Dviqn``(#0p$(Hiqy73X2T-kT@Kk=Uj$Ll^^F3YPO{jW=%lbyoX0RCKT$SkesUp_6;Y@_#?gp-2&w*`bAur5s&eAC=`ww-A} z^u%EE+-YU^)52)&8vdzv2e5u%pSc_tqCfd>5+R48?f%^bzw;yJ=!^ zyf};)NJIWK#o`bqem~@%dA}I`B5J)?=Bum7+w2G z{srx-&cG7CoOybE4M~B~Io7&-nvZ z!bO~W(38Tr(W2pXts6?R^p{k^zG&H6;K1KCm6hzJORk$&_YZtc1{S#Im!gX4HdFuQ zd0m-V*Q|q*_~wRz=bdiT(FO3J`3TT*ys%-PRvB0|)cOS)fIV4y&F(l=%UcmiUS~Y$ z@X66+F5zj@>4w!&THOtf!5y(ds|zt$XwSQ)JRtabDw6-K&;oY0Q{}mq&M7N-t$eWX z>@MWfF)ms>`O@cz7{Tx-uJvYg5ROJ}rm)BsGBk&hLsSiCAGTqA6TN`Ij*e8PJL)_+ z;J@~yi|!us1-@!NdSf|%PuKECwTp`Pb0<~P$#b6X`dK%67rQP&Hq2c+^ zDY`-5ja29W$tEh%;Hisv15)yK_o(nw`0z zL3H8CbaDaNs(n!`C?ba6?noGa^20%jMCsB#@#KLYCZZ#X#@yTExi*17+C7*AX-ON} zO+Q~h=b^#?Ymb=;Rlt{Sn@bb@#7av=S)qDsbRwMP{8;kO36rDW^2oWz9Ky#w-Jl4G zYXlY5XrR-Mc3j!K)^;9x-8+=wF$#ozUN_izG1CNrl`nbGfWd`DaT9+IO*)$aP+C@^Tdlhkw0=^k*8 zF1WX>bI_ya7DGRLz^vCi=SX(%eZqA!5^&)G3qY1!1eg0eWX(P2^_f(9AdC)gAFy<> z+$ZckZQA3Ysg@w+oSgkjdi|=T&`|uv`vVU?D1uNZGRmdBzI^k+m(a(Ug=Va+i5b#$KiU#PuaryX&&kUQXTyJGTWl=bH zcgJ>A0r{!J2fyCvhz0kx9|hd-G+Az|=T~NGN!|nNd^h3^0w!+LLeLfG$Hd3N()a0@ zfNe+Q)r_R`HGge}$xPiCU z_cu_lL0lswvLql`{{Y>dDYQOa5mq8`QeZG~Gbf2?j>bu%_Yvh0w`H4>{GDNfn^JX% zXQ}zSG?o39@jK4WQMUu|1%ML$PEh2nh3Q<<1K!A28df##9A#M#4+{3yV^<zp&s8r>azR6s;Zc?9y0VkMcsIN`Y{<}$IOqB|G$`xNZ1Gfk z=)!PmonLZZ+BvAwVM2Vq=8`Olo~2~3=FLFcPys{Pwp#edLVnM?ED z*ZVNcPF0pm@>S4CMTPq#_bTvR=EDgTt5b%jI`>^q_n$o$2LHb7@F}Akq~Fy%L1d~% z@Jv0FlT!KvuO^k_($=;|aH?cY$Iz6+x`(Q6AO4u!#qrq^s>Q8jJ22yvz4l{jC&C-X zbHB$vK(oikZUw`5zEaT1CnBc%?S412WZa%#W9Q-1|3CQp3aGZWtzAm-Vuj*X+^rP1 zQVJ9=(BiJe-L1u);toZMySuv~_RiQWnd@6~ep6P` z*9o)p_oyWelnRY@*8E};u9|363$-Z_=!>Sw%zW2kBZ=>+?L|u^f!Jhm0U3xH{ z=X6{dW_1gFZv<7O}49YnBjhUb$;u!@}KOqA=j){*_BI(mI>}!zCSnXlUqIT1&B=^`u>rE)1KNyv2uYo9Y$h{NN572TN zpY{QamStZ+ayC7Qszk1$eC^!(1{L$m^_sGq+V7)%?z|VB?nLhz{doqfFNTq!WAknH z_I1$jsK+-CY}y`3aE%U6qij>Iy1WKIEw`P6S;inwyH(CPkn&U6_lUz75g5;q`jJ^M z-h7Y(ypyo4Xk0PU>uS7RT{yD`*mPB>I|g_wn+m){TK1;1L0c9U6olN#yk;{4I1#vbw6m$1jTRilsZ0!GSi}zMUtK9b@h3#SXxq`nr?JSP1+hOu{#J9DG z%fWe`yil%_yD5i0XrzwG?mQab}g-f`mV|gHbcj?!UEgHhfaq`v`llJ_Y%9&B|dkb9ZhslRI&C zNvey@R9SKtTK3C7lOp@o`4J3(OP1J!rDND=V{*nGpOjoSs`-Lk_13@l6i0oLecfpA z+Q<-t99+9k&pMtemKl$NN@&4rqv87FMn{fV)CreSs;a2D#EMfweBhn{EYqK`^+vEg zJF%KQiIV4W{1#obYxRgJjLH3Ea)Y(h{wrw6RVep5M$DMS|}X(vM4SsDA>Kx6>_ z>IH(hpE5G4Ts}#vV79!ooCgr#i&a|d|fq0%c_w~Xj1fIPx&w{g^uvc9=?B)b>G$TR14o_586t9o{a(n&jr{DH^z#Jv~sL z_l<>}wX3y)Phepa28GwO&YJwC@8V>Pt>iK|f5yN*HST+;nNOAqA0(#LF-?vEf1K97 z!&R8IQ6q@9u54|e=OZ%k5EFJgvzt|A+$SJtzkY6wFkDzsG9`ZE8u zD`HfRhL*NklPzc^&UO#v0q%Md8(4ClJP5JwlJS&6X4R-No3=2(-F7k}Mp0u_G(9N@ zkG_qLh@yK#gIG4dVyZwv{e~uQp12i%bCTcFdN8A%z#p$9kjEj(R3-afrpI z8q6rXI24`gxnTuY#O_-5w;kZ8CAXixX|%9_oA$>&uZ`F!#tw@&jircdzJ(Y0@_B-v zwgz!h{rJxi`kP}uvT$WJlW#dMZoP0*@)w9F5fQ+;Wus}^z@sq#K9j=@a9(&Wv@s-U!ZjH>;_}+5w^W9*7XvfIn zxzB}B#Fdk<3rBPLie!-GufgZ6wd+2uB>}x1t4u=83AV?~y6$oYu=^fHn=#RDW^M+0 z-ucfS)@Z?qNNfz)8G%6heVb8E)1&!>8f`4D3Qb8A0^aD(H@=@EE}iak$SUo*3^o>Q zf%7xhXWhj#<*sAtpW6CIa`!OhnT9vW$>@J^y8;=92Daap;?P@wYl2S&-t$4JS*aBQ zx}QCD67H6`csr~puq7v(lvS+&`Yg-zR1MdT9|ax@KGRU}v8?IKv6Sg7x0%Td8a}Mq z#d{ZQIjk@bAqoP?pn2`O$?Td4-BgF|!})h>>)UC(zy`Q%1{G454uV1@($e&_k?FYU zgUwzXTy1CCrx=Y8r!P~ZlWu%MYg=Ei3K$Fm5zSSoxGu%|dpPHRhzbIGNXHNl1}KYJ z&w>_hmxjZPB-4Z}jRQ?1PptU;y1GURf{u!rN=r)tE1~bH*=%*~iwXz(;Ln@vpEP7T zFQnKz5hGB`?|yXfZni1PlzflV6|R=Rq-Up=jhIXD8t zG)Wb%l25k{sWUS_w?V^&{css`PJTLUymta{XyHg1bdC&CDehm_p@LvV_qI`ElGC?0 zue7UZc;}~8EM{u?2JNwPdM|nyJl=;58rHE2xP<8<`tVv(fF!a!~QH8w~ishY73 zNmrDF{};|4yM?^I?Z69v*Jqvt49DTq&6p?2aa^=JvJUsmlbn{H8-C_L@FegMr=QSz zZ&adu!G`i8+rZ3uVF9$GYMv}5*s1;f0^GUx;dhciT@{%|9!Dt|$)7?!tD>H(8@RjiM8L@&dk{ZXbde9p^MEBO4~x|!Mzrm{_9oqj z%?mpkpfBfijXP(THCxVnfp*_xEBn`X_BaV3oITIb^1V=)qU0^4a4LaQs(@hsBT zDkU|&Hw#D1=kt(|`WPXb;dl_bB7-G)Oi9aLbbbTi9d+Ns-OpjNt=SvTS9^y$WgNSO zWrMqNXMHOKfEKgyl5xG7;9!UocYr=-M%LBCZLScpp|c}0WuM`t*)EBo3zvQqQQJI9 z4{qBFIUSsz(dp@R_JGD)G?a`fq=>8Jm%vM-hEEyA61I6E!3gT_+18Whem*XTzfASR zQ8&Ql`P5b)E^bQuO_s?Nx3^d0EV=4wl$3Md4(Cw~Zz*oIaUK(UN)6|u;!l4o8f8vh ziRi_tkz-lR^PF2?OUQfTkYK}~z?=%&p`mJ06j871*Sx<_#WttsZ#hTD_WJ}mtKl2* z%;zGvRh4NwzQK-yM3QH<-e%?6H#%m} zgWsR7bfhHkpzyURGrih*Hz^U4>1I`D8xavHs|*OAx3bj`35D3RhuE>w1RC9mO3p`6UZ%DLZpskm6MWuXmy%_DXABUJ$dGV6(=WY3K_gU&Xx#S-!&V(&WF`s;v&eW%US_tQ=?R>&pnf0v>5jE)Q zImKzh=k{UI$eATujGJE#{~;~u1KIuutugeF2(2%(US>~KeCI|pB#g7Hpw$!dI3;0b zM|Oob&y?E2k&8imkK4=@rMd2^x_bV$j(gzLX>Jt(J-4+^t5Lbg?~PwLzK(9wg$h&o zFqJ30h1<=vtF6s6&ne$=&x`YkjaMjU=1zx!IPMw0w1nROqB72>VP>%tBm24+W~69G z^}EU-5>np~Aq}T{S46&}*JFizOR)X;Uu!OXuDagyE_`+y0RzTH9tXHyg@qRzp;sp> z!1b=2INA_yuRXNVGHVOJs}~rI1iF$a4A9f>k&l}l*AhgL@J3|V zHOJViS5}T?TUiIk{&DRDjAhA1A~yOa&A@X264%zyhFEAAzxdjfEm!hW6sG-7swp#5PvcPS=n_Z3}@4#%k+4`jl_j|Ur&1LEuo1`OVf|*4!*ZM1aN?3}61uUhE8aJ9Sk|a^iX7#k zN(-0(x4l@ZwqiWgcNF8{XaWr!1U82ko?q0Fq^S1r=TONx2!xO$8T4@6e$MF$d%;Pw zsX3X9`0;2y<0yuydxyhlYGENI5z*mXzgH*ccCRp81GEc(rOB&E;Em9ifC^<_)eKB1JNOC5ownb;0)5(rTcT zPyjEvCJcLOxoS5VwCLzMPp+@)SG@uAi>OntGdpdFYHQ{{%c81%F!YYE|M2fuYiJM{ zj52?QdJT|y|6X?;kBR=*ul}K-#*(Vy;*X<3KYu$I%UP8 zHU>?Vvj^YAAFSxzea?+VD8*+vh^1mbq6QC0v2CaheivR6G(l$-!J9S;uj`Mr4i;U=+>#7JU;$0@aOjSUixd3nVsrp5kr)2E94vk#AoHt zUPyO;``J^Pp2s&(?{_B=ko$UeQY5MW2n_*av`jOtA*X$2)ckJM?dOkNUvDQs|Fp;B zvmIJJo8>3l_rn%sZkHX-N=4P}>G5*>_MFz9o&30He5F5)3Jd5=(ox*nC{o6d$yW;o zL!I@1GI7Sj6-4pCUf85MB)2UCeiK%H16b;`qtFU9mQ_%sHu=f`_ciyXz0XeKi20gOZ!L~5?TLB3 zOapDn`0b4*$EqAh26h4QADpf(J>4edP0yixoghHCVE( z;7Q(Xy}k*tXxUAc0N<8d9InWCocimVt|rr(PXTNC3kr?LnzOvSIv>NtF15Vwg2x}L zZyC&lj2)!}R;fne4(>keRpsMERrAd+1B9(HuLOl>9@e z^~%N@YHP*=cR&{;<@-_#a$G5zq7(~TnL;yt4KARoWlO*N3mTrbob=NHyLpk5MuQYY z+i#$+>BUZlOI%q>Hqm%Y@FzSnmODilqQz6>tH@RE zBHF%Ek=<%+y;x`*Gq}R83p{i_mO6B}`LPlMm0H!hJ{e+L`K|EfhucMp&B5d1p-G#K zh{iZ+55c##+Mpe1md)}eleHkk08?6-TuyE-_ljM7IN8-!VN#&K z!=1URlW2lhZR#MxdQz;uCa682Wb5FctbM$gDnBPwc0z+ zuDiPyFQ|a@sZXH&$+R zS9Nj&i+S}dFhu5}=NH4FG&0z+HTK6?Xw#Cil34$>lVSLUF$e{1S>b{N=@A8~LKBe? zjbwa^^{cZ2+1*}^qq2TNl_>N~1IsU`$;x~EsS9d$)$&|RFc|t+!MREvSQUQnH5R|Q zMu1_I+gcd%e4k(qx}VM9X4?gKQWdg2O_s{k=n4zsI|7TlI=8U6&3 z#ufrrGCn_I(1tUYjJMx)qVU)B$q$l8>rWzV%s{2lv7DgyAok5e(DGD}FlJng$p>za z(TVY+oPn!4afUXQwP)e~UYD?-P57dmbed97QLVnFHkl*oBN6oR(7TCZG#6joAqw}V zV>F5d^uS21LkCh)>nZEV~g`jO_RJg2Ty9H zMHKD6d=m!7LYspHArktvGQ=+4YPj^IxnRaF%0v&Xer?-dr`mi~dg)Q((l%02yw|iG zO*R55DiZzHP?-3Ln&?1Fvo3S}W9R z%qezoh<_*4FLi2fNsEmzk4BeR8Xynwai|Tg!{q$6PmLkZcHJh3o-w90tdo)Xy^BwR zc8~w{*Xo+#DP$@t@v$zXKx&q<^`Q5;(p6d;%{PTVRu6yG4o<|ZAySGI;{d{GXj$Gl z>T^g93~Xad{k4qnDHTnSa! z>B{q{8{}*}^X3mC>Bt4Db7pF59YSgUiuJ*-vR-rWC0}3bpBCr!rSQcqU>&U|2wUim z7H~^mEP7buzj-f#`298Rw@=>;(bx#_XSWS|M%kf&dC$`&f3CG$cY(HvkTFXfcVeU6 zO3=>DeY)@ZHF6IS%Mk0H%NF|`{h{(JUWvfx}E4+7kg&?(0U<#bt&6p4dAC=8Bni>xp zg(l#4M#_>_&|4s+K8p;edeQR(x!jy6m_hFsi(z#>%g6J?&$=9pEuQz_N0>t>{om$2 z)csabRn2QFB3x}h%FcMGGZ1>Uf7eGLz2xaD7SQvWqv&~Gb&znT5)o#I!=gvJ-a>)c zEE(GNEfV^|+$U8%^i1&cYdtmmx5K4D*^>fFlVQ0(tAFgotI}Kj=LEyZE#O9nS}bO- zfG(l~MJs8+A(U<=OdR)T?Mif?U7t}?Fuadl(8Gm?FC21w4g9R4esZj(#h|&ZXWi3j zAX#FHYi(wb&c_HqPYE+Kob9M-X$7|pMlS-?djBTNkNhU-3~p^$DYH_aKhxE4bHDO_ ze_GsFY!%{So%<<_G?u7pDt4mYm`AaEVfQH=3n>5$_49!t^cYs_LDD56CB8(5o|IbU z;o<#kwN0aN@+n0pJ+1nlnU2eC{#GS?nl9C_NmJmN*o(i(kumWdZhJ1Sop|ECkwIGU z>-43BT^TOo{#~pc&;u9d^FHGqE=;ze=^e`4Pe4y3$y_%^vpEZGTfX_cp4=4!~Zk`WSzq3)GlfW;&4H9N}`S({GencER0B5Dm;ucEb-%a)3 znh041GO)8_Wvs^xsp0>(2c-2UCE$TtCKi_7-7m;kfA8UCT#Pw9tE8f?J-XowVPayz zbpMac*1u%RNKVVlj4tzmu5dkks0f4fZ<2=JP)yXX0{cG<;o+0Xs4^jqPUHTgkK<1V zlUc&dB@SxpDpF3duZBx7-4*_$mGj|EqW@P5@IN0jzN#xO4x5P}jmN{g(1~^!`nz_Z zp>gWv_Mzmgso}l`&XOs+erevdtb-M05uV}xUB-I?Kkv)TVDyA4E)Lu(&M9x_!k<{CM(?LeKiIDa|DAZr_6n0QNt(x}T~0 z!L*7fhYlPZZh-C%D@Q!x@^Z%OGxX3q_b%X^Nn?TVlrXuu*ZLPa#=D;@yVNW*$l1SD zTt7b{+Q`CVsIT_-Dd)L`ddI!b^949lRyipRn%KGNlm7Tc{d1oqoXCdM6H|z8o0VO> z3|0zxhAa)-2!jPtg+&fxT>dp|tkLm;()M<>hyU-8&#s{!;H8?xy~8?hzyivuG8u1r z!$y_RwWear*5z^L8)GDAD!(5UWg7StM4QJ$HY0IY*b~vp0%?R@IZUB}Xho57%2=;7 zn#uzEh{)#jH7nBIIIC;ch5iZaKnC1-m*C@+hf_LqW<`HP_=j)I4%oypuMo!w$InYY zb*Twm`KCD@auV}6(iB?em*!>mW(Mw^ZVVirw@-AQo=O94u~AbtJn$u_#4^Bpl%JE6 z937@8JMT@YH`%o3tRSVAPHr|U-4V)SW{->VvBq!H7t?mHd&OZJxixMfm0;+~sT=H~ zm;4sE)Z>5l7)%}i3aG`8d<`kwrdKJAiYud&N zs@7%$p8W}W1Lc~D6VpcEq44oqpaU&d;+o0Ac?N;kn%?*dFxMjhYsF3i(>&A`t;V1+H8b@QMr=gG3S|>HRlgEtILP} zlY7js4vBtnzfHJ?JqlLCk3l1^DQ(`|HL<^y2Ihl=yhw%rCIL}#yK`N$W~qUGH|;Ms z_(>#GWRgt8{dVqK4g%jx-CTd`rX?T10~u$({g~nD?p&SnzJvxl+cHNujN?<&SB%Lr z)+hS`l_tY!7I7Q?2;aZuSUf!tE%-GM_gR_E+${23mpj9UZep)aXy7jryB|ro0QBF! zW+mZvBNv^St!c|VU2e?cjqEc0WX=#6f|Xy|9sq|V=vGZxMNQ9j4pPsXynPs*FXKhZ z7n-G~(|9qGVLxXk?iw89WpGl)#GIQ~5CnvO`+&u&n_V-+&dp}+qpXA&Ldw%tXfq8h z_;G?C{{nInQ)99#?Pj;qPvL)021wzkC~J2di6Z&)#43e212Nhh1>;jU>^A#M1yJnR zdN}Q0>o%gn=Jm+?6S&wdwb_==Guid6zSFmaiQUNH7MrjG#v;sTC(?*-ogdSwG^p4X zMMUKn_HuX7abvEe{c_6R?L(?eMgF8cG-=ah51Hu4N)e9w?B2ZpV*>q^Sg$d$Li??d zbQFb*PpoNUrS=4>b2hF0xfsG^rzsxKS^JAfaI(paIt{`vrQ z(=W6Lc7d?EP%an}_cbGiw2HaTocp8~M0fp>+W$#x!_$k8EQJ%5bW=Hf4TfXf3u^hmypz9nHJZsC@)p~GZ zBK{Y;ULJpf=XCA0u`ht}Ye6--%lMA@maYbuofBgBiqdxY*FB(y{;GEs;WhkQpCgzg?+M)Qi25YLt^Nj z0xx%mb9T7cEoQ#3h4g6Qb8Z8dm1`d&IWL;mo6Aa>IN5uQi8Dn(c>0jLuKUYJcjn=d zZD3Kh>2Z5(Gu)hFJur);Mos~K6}--w!Zzcd@CHdmTwN|Tn7t?ydy49WAJ94RUBQ=^ zvSdd16&(CWvmlCzlSqcTje>*6dO9Er7exjH2gO24ieGpUdfvfJ+WS)_MhoTaUcl2mkuaZaUVD8Ia#g2-nWtb;tQdZ= z@3}7}&a8NPi(CXS!VeZ1qFMBAaZSw@!x+sk@uiEizP|uCycP zEJjZ4=GIs5yY9!3cU216xzEo$y3URg^wZJ(SZK1>gM2T-F$0`@*+%>F7X@s6=Y7bc zqvpVc8oRo_=X}&S{<3FUeAV94ujH4-7BFp0Y-Q<>a>$P45)JvjX|?}Of|{Oy0QaS% z0>uJ{QEjD3o^VXT9YFrG4V#X7rOmbWl$dW=OsvJk4ktV*TW^faa|V^F^CdhaGOSw- z509<#o*k2y3k$awdtTh&8^cHVZ6_eWjh)eH4NVK-;=UaL>2X4%uh@XO&$xzsh48e0 zt2O0e@!g+Rqr#2$EpkExOJm~Zc%t>6qh4%oK2_SS@<$Rn9Z&FIf#jY0h_w!nj2ozaJsihHT=?08M?F=D-AHmJ5y5KaH+`95$Vr40~`zc{+Z_^s( zqum8N-KZr&uvf%NGBdj-6G=VA0<^ImIF$Nhk#{CT#ry>%Pq+>hvK35SFmbvd`uU4* zro=*Bmx#n*-XR#ksnH#y)GlaqqIP@3O4h2KJiy z3CUc+k384k__g{StD?k`kK`>H{NX;Jaf7Fz_2aFmtEr;bhp!p4M8b-mE{jaW0xNH4 z@?(F%G4O(;E7SYw#(u;9Hm%ha7^BS;G;ihh=-Bq1_Gz#Ze2PD)+!cXy*xpUD%V9Xn z&stnkX_brmmpv8CZg8)x?Klw+wX=;m(yZs}LYfeN-jptiGq4D2Ic>BWYjHo&4N}>@ z*{*q~D~A&hGW|d)19q5v0q1It@H{MNsD-offd|gg4@AK3@M5Bm3lI23GlM;%-X)dg zudhz)$U2gn{Kd~oyBuxMR7e>cD`Sbp6o~W%B{@R>DS<9i*gc%t6MgUUzx* z)nU*_h{sdQVKX#YXQ{@K7FiH=SOh{Yt?xkn)PA1L5k0G;Hfw8){y(lmwhNvKM|O9Z z<8CTsxbM~B;d}>dqi)LSvYH3HEDoE@ThzB0D@}sO-5pMfr*(ZMczxd9A>W!=EH?Px zr4?z@7GKK#Mf&@rr z+lJ>$e#%r`JA^LS^-YQrMGo{f@>-L2G(L=R!gDH5(k?zC1oZSqVU<@%P%$yjD=ZEO z><(VOVxtYh8WHv0iHI=1iXebHe^M?sB1Ji#3~;sy-&9S7A@o$f{SD%62`3TawI;I+ z<%Q8<#Yfmvji)y7gVPx^3lK{!$Cqu5c4iA5cQp@V0|=*MlZac;F%A+XexamvPL+o;07qu?SBd{9$pY}fBS^03hw@i=b+3O0uHxW62G>JgWQ^tH)>n8H8^h8Qx`zc{z5vFi@>o^~4$LPLE31@pUaBJ-Yn_Ly3r2$W<)zvHm;Y{x0)Zl5nl3k}& zWb*8lyavcB5J9yaF7!Pf~& z!bOvKIhp-)WcE9uU5}y&ZB2G&lYv9oE-e=~i(z3g2L}e5%BohRaB{VGwkosH*^%Y7 zc6Ks$x4GY5qV~s|aR~hnYo_-T?wFj5Ke643JSf8sk1dKznGPe}H!alyQ0hfwT-a9L z{EvP;A8t847%ckbe16)3Hb!x1@M=^rHF6c$M?y9xET$3ax{>yf<3=jXZ)X(-K%qRXO~TLD!kxLNUFcs_V(Oltsud zJo3y9MgjaF-WBtc+lnfBae8*?uPjNXrVxvSaD{hq-$xPq_mDC?;V^W81ik(!OD!mvR(o0d zUt53&Jkdqts|A6d)xQM&x2amDtJBcXULXfv3zgs`l7H=VWeS43u8PlRtp@5zi9@Dc zr~k@J|7!#;_&tWW{;%PY|84*d9wthUv1M%q2?1H4I8x_Ns@TOD3rBEy)zhaIa zRkdK=Ez3%S|0QH?Gf$GnVk8=TRrFSxk_*%l5I9|7hAUETbeNrGkYu}eQ=vB4s0J5U zYf{42qyFUr_(E>-A3@@{bEYDmfcp9k;t=EuqdLvv2p84uTRhBQ(t2Ka^&&iRPW=kw zB^A+PbJM4=uVks7%S!-8i+{*5%qMh!@fW$=WO0(|i|IZC#r8228HTTkaB6u8aa=wH zubqX!zeBP)(~a#6qa3M?#_gYT*$rT&DFpGgg^L~y9e+(NFiK2H#NBtnGa~(*lEk+( z!iAYhO?6w^tAwNdPZW5lc6mwrj+lpio$ZDfjoBV7=MQ~*Ni%w!SI~?LcC$$73&NoP zUn8y$r9B$5Y*GP&5DojKW$=nidPpyJ-$bKCW3drumpb$_{}YaDP9cbz0D>hhcm_-& zC-$3SWF zikO!tdApe&U1qpzG` zi`mz0?rf+z!J#;Xh5`lqPzLlE5>edr8{PuvQz+Wp;X{JUuvXa)j>e_ zv&NSoAb1}BTS(-U(yLfyB*}YGeNZvZka^5iP^=7tdRSCUe=!@a827W%UW)k4i*l4) zO(OXFgD|?(QnzbBvYn%M&1cBZJQVpfGWZ*j1@yPviIYz?x%TgZvSeOMxuf6H>{vV1 zLCon7t&*5S__HTY@vmM=XubaE>S&}H=(&aCZ@f3G0=;eOGMXPHFp8EEk|~zK%$@uC zp1X3zG*}KtT9yEexQ$)0yIhVCWW&(khE_Oa#O!V|Qqdj1IlmwK_f?_G9JlHsqNPE} z7?beSqSwqaLf`u?4h^@Q7eUg z$Rdh-e!oT*)nOSuD-o^myP2{|bN(@<&oj1P^SF6#HoCVCGXF3&S>xr7Z6%J7qqI!9 zf{7pwwmfd{+d4%C{UB9T<43ST^{okl7ovZrA7z3@mr>EeE*D)_v|d$&$}HoSs2S>4GK zL+);w#@Z+YRMy2nIGb64sDL)2w=n_2zhI&oG1+GwS@L)4{}w=;iG{RK7I@7M@yrTd zoSWNjjlK5SNm@w>n^mLIaHW0>4O;|%_fr7MyDBJ_g2TL2Oiwp61;LV3{YaCPyVFMHn^}B1GxUM^TKe1 znSO~e{F4?|a(k_V%m?_4yI_5^t_fQB49bU~Az1lHDqpL|C4oY!2*tPfycKR%4vyE< z)M?;;IypEgiE9YfI20q9sdm2rtM(&RVD3iI{?8UWq%RY z^cF`$7o&2CvU%*1d86+84V4Jt|NX}(a%W0-hgEF`2<4hE`WNC}!clP){x0W0-PWdm z8{&`dE?`07ckkajtrJb*h_{_q8pOQC&wQCUpNMZsic9cpe?v+9t8)_1hW|N*lM^YB{&Wx}Ay@OqbL z%nbpBAJ}B8sCr2enIXI;4)cs^WD8cJD_qK7h5iMZHLlM&IH;T=st6aqckfr zsQUbHBWGN_OLnHMGjAj)l$*#DwZ<_D7Sz8-B$grwQq60X3K)cgEb$e=Wxt94v#muO zkzV+Hxo&4OMaR;|;r6JpqAX=;Ikv*KZ_k$jn%qyW$)}ZruA|XwZh}4+{!x!Pib{nyMwc#?E!zv~3>dV;;0|%e)#g2W zeou_0I&4Z_ z7txdRF<9Bny~^84$MvXV(GhRg|CW(M@c8%nGb(~$t@7+A5i0-L2a7ocvXLLdYzwBY z*PB-tP#~b%y9_`lt^7!7k&`XqWsGzdU{lM_xgB{Qxuk||z9LYl&zF|VShQWb)^`#- z1?t)MZ{_`ewE!B+&Cm6mv;S>youGoyfsO}*l{WM73Ix}wb(w6fjP_|D-rFd{Z~aS5 zG;m($#`-pU<{VE@P*4Qa?~mXCH)1HSJw_-rYc;AtbvVuUpe7H`LE;%>-MvKjt;8JB z@GCpW1pJg9P8%UCrAS|=|3QxuFEnO9Z_I8d-SL<%x-D9V<38S7f0Lx}rPX`TGrzkB zH$p1Xn1ZZ%DX^G_{}KnW_**>Xt;v|N3ZqoTpbmZ)M-olmug2$RnuXAc$N; zWXU6}{(u~Wgl_$l`rT`i;ZB8kSvc;y#2L`haTnM=ZcYvL$z(cDW?rT7@ND8=K4wl? zkQFM+bLn2{D}wDw6crU13cm>c7N*2-;(7(QjJv?haLWNl%n238f;>`5O>d|QH8BP& zNBiTKNiheG%?=CPmXWHfokmRExIkh)KT@?W{6LElyj%T|$q8<{HId*{_O_~4r09a} z{O{fC1ESuHZ%I!9qrFYB7|`;uC1G(KsSX z7tK}c#;{tBE&_cOz)_|EcCljrk&|z}8$;s-cC??g8-+v9Bgv|=(U!mHH&KY5+aqt9 zlNT<}SCq|WAQ4iF|47dXp~p)i(}4wlvA_^*K~RrzEk#lg`RIPD7%kem=a1Uf)bhpY z8fCVmP*rC8;JqluY+tcY-?yTI0(SBcRh#^kPHv};mxaHhCs_P=>r-D!toN=iwgaY$ zU8UWBX^NU$TS6d?XyOjC}T=CToMaKIBq{`PfZ^O&} z+GsavT^Hyyv?eF`+@z#JA`XG3GZ)jZ2=@6)4SgH9P_@dDLW8y{K!$J9wD*59Gc#L! z`3>HBY`A$kcMyPAa6a$eWJUM?B1{>36e>P5njk8D*nyPO^IMJn&Yvv zyeO!tLr;4+>%Mfg?>0jO{pQ$6Tabx1ez26_*GP^&7_z^}fw8i9K?R97G3bq}08KEI(|cjE0Px z>2_5v?NElg$!oQZDOTBUOwGmV)sXafUWjh!WhrFaA~_Q(e7C{r=-f7+of)uktcM;l z@z8d_lx)!wSE@3I%e{3q+@9j+KF2pb(ef=ruGUpLjtw^ymG(XjZgK}d%j)A+lbxLW zQhqNGI4m2>Q*?!$%n1n-QG;)s_8-qeQ);2RY&L;i1LY;MA4q$298)RII~1LynHwif zQUxs8)@j(b*ta!Lw8fUaTD@Kl8t!zpd^>);yay){6r>>RC3FI~b{V1zbe8O9#t_&( z+{)7|qzy7ISIpvD_3n|p1Gcb_AMVpflIr;Y&XVn^2quC?c!qUnTmnKmHecCAh;+9S zZ5qk|>ApU6>F;YE_@UHSeOr+)__eEf2ANl+`yjc|dD9M=^xkuYV8<%mR2MeBi2^>f z4zW6{Ly??Y4}X}PNc((7I*54gb9=YAJj3Ka@qX`jsOORTi;=OETH2<4@hT}T@9-1& z{^w(Kl_hbF1!!ky8Xy5fm2!Qj+iadL^jOvqT1@SiBpnwtPYJ$}aE1KdP8NUj(;kPd zGSKu1U^a8NkH>f0^ncj;3ZS^QZCio{cY+h#-8~5&Ja}*j?(XivgKHDqHMm>j?%H_c z?)v-OTj&1s?t4{KZ5p<$y?WXjbI!4-LMsfPT7H!Jkc-a>zw#xdS}0;`_+aQiWy6Mr zC^7cret_`4&bDF1Z5_emMJKlNR0jjz+ z6S?8m(uzrv_CA|CVPiC@cs_31F52zukq^YEi)k4T$2SORW) zXj$Ts9buesLzBZwPv}!>6>Jf2s6JogfyL4Ln&Llic>DN_lZ1mkLr$onrdCGG>u@>bDAms#Bb$K(U3c6sXLOzP{b9k~-eh3=eom_!Yo{sY&BKNRHhg45 z4a)n>loz&jgdNhkJ%*7;n3{+Bv*+%UpSSR`q5Y15w;g(@VO|OQ$-s3iF^b6fz-^zK zof!&aN7sH+S44$oouY9SlfRn_BL5FoLC^t0mfJ}<(X6;HN5L~%Bg$}OzyY{Z5n-kelZQG z8JWviA$>*Tlq50sC`AY~3OBYEx z3%hmx{F%WrPWYy4U6&xcd^%SSbYZqg!>4RRmCNt9;~Bm8pIevAv&XAl>B}Fs*s{lG zn$BYE?C&rR$imOI+xc__Ii^#}Q+c~)#KoIPFa~B1pDh#v#2FZ?@%qb+*g?+Q znZ@|g)@06u6@Yhqf%${kClt3C6Q$$sLloIkM%Al>=dZbW6QLP9RB&06?00H@n4I%_ z5yx9=fik1eGRPX%f)wf^V*3!AFCAw#q@l3IQt{7SZ4)^Uf4ykJzSMC*^F_ge+$NP=Gr-V#Z|qH>|0Y^%wAmPs zf5}e8WdvGsg{RJYN(!f6ZmT2g)J*4=VtzM>pDyOU8fU(vI96G1%pzZpz5p#y?ow3dVN3vg>#NuEGc->YCwik$+ zdz>fH*4&BZ1ip|XNfQq%$H_f4-}13^KC#N$X@nRy#-X*8(4$kW(R6ESYRa(HW2VdD zOM2*>#LTr7=!|8feQs-xpuEeaq_nCXl>Y!S7WtLTSCC+r+rxPK*bB^ETIgM0&-;!I zoQ$KQ8W&qvilr?sUPgSKPgyjz-P!gGmW*{PJT*_uaVTtRa^Kq8Ix4g0Bj(R^=5v30 zsc(CG^-95!jT4SygF>QnPSjO>`ZaSE?^+B z8lvG^2?1G<;Z0}tkJR%q{0KKysnN1STbruxY8$dp5ivIJN4P>~vQ_BUBO^BNCYYwgTuv}xIxEa$Saaj@>1yzG>uiTk>QSu;1?G;G?BsWJGeBu~&a9*c9xPWeR<>6; zvVmZJR2zo{y%R&gOiPC8z=}CpUA7z|K%LXYNv~)AtSK5Ay&rma4}V$ub;$E-plGEZ z=SSR0OT^%$0bfJL4yufh^8oZ7KKV9Tl~dE|m#}?1YT=vzZQa*SpH8zQJ~@T!G<}s+T^9eQu3PZQQsT zR*4fiJea3;RkO6lqx$#aJ`we|%2UryFNZXoeiI|}(IRWiKfL2cH=n-_>JK|<`%ebQ zouP#-*icsy@--(Vj@CpSfl#kZ&mmmepq2nTwdQCHCVmXPfmISLN?Nj^T<+a!TPT#87c;v_|W+EmJaL z;ys4t8*jTOiz_3+LMzP|Sf9U4!(t)v#AUa~*E0r82CUA`J|)U$m?uM?aq1VgYp3e? zG>a%lvXnHagzn+BaCNbKx2T;FXBv~y_Y4^SUDljUM4pDEA-meOXU;tN-B_ z3(L*kRf9oCrIp*WBn-ho4CX=skwM#}Q)O!JX4mE*ISUJESg(WnH;I_usXopOSN}ox&E*i#*DYMgFxi5_h zQKIVNkL-66)FnD`bFkSX(v{T`n}Au=w-^$9bp0MOu(8+Hi^)VC9+6`g8FtWwdK7&< zhPMGrb4;pZN%L2@y<0gX-1UrTNV&iE1bB+zbKUDy-+y|wbfl%+tZY-4ma%buT}o&&35DW zvt9Hi8Yi4t2)xyI^F;(_ez9r<1^#f_=I@&rE*@c4+}m-RstwRfsXR`M)T7gZ`&x>e z>~w&3K;_mpa{v|UFz5aK9Pc2}AAc>k35qjTyV(ZLcg0lTGA^AvswqZ+=+)0b)q*^6 zdXqfZRxU~JS<0a0=k6%{NcS62RJN!NBo zd(S;oUJ|nF+~w!!m9_nvDg36;wB^_hBHlpo;J9Q85r8#Ch3K`*@D3+53ZQrwDDh)2 zakkP$C96KICxbNKSjTL*DS{5|SN5E}y#+r6He!3q_P9ynYQ*9FsKoU>9Z?E zI-&l`)V-GhVz=o#vzLWjtfCPgI1PW!uZxAD)wiQ3;x^e$i0BFX> z0kR%8L#x|!r~Cd$%0IW^;@uZ1DqFBHITY3E@~&?9L*3TMy!d6VE^!rZIy1CJHn6QtRM9jXEScTu2hvFF;NT#C z+PgWp`Es|u&FZJMyuMbe96x1tgnC-xLfBN6XI-hst0Hi)o5JyPY!~QFnQr zet7AfsnRVzkoPH*_EW?&LuD5vpfnu%AcL_p{-u^>!U%a^z5N?cYW6k&0Mc%9X z^ph#nUjllg@t%zlww=iO%{xi;wcra`WmzifWo&8oOj_Ay4p=6_;ENeQ_> zxpk%Lipl@r21et4xD#NZO)E(I6B|`%b_UDAIep|c46wKrzQXt1rh|N(5LDi09;Ybs z7VrskrhHfon6Xh9Wv7ZF3KI5tIVeT92g)=UAU4eU>F5q5Xf+C@xqNm&$!?A#s1x4l zNT1qh#!XDxgEY6$v-)Muxnt29(e>uc(Yiu{0RqSaOhJNGuQBfLS8>q;TH_;sULG+f zW&*-;XA9cy=Xo;Ce1)vrn=IzJC}xt+%c!;7#h%z-rkZC95XI3_dr(h^cL?pa3VpS; z>dRm?Jm8$sfI3eBTp423%GxLS0)Xr2s$U(L$B_MUz}aW^+C!Y)Q3#3!L7C79VWQt9 z9g7CiM2_%7?5Gknyc~6x=!uzRw;n$Sg&*0FAL%2|S+>S@;ky$EF z2|Jrv(>}-|!MG!SB8GK5LC78GZ`crQz>}QmwGwJ6v}3CaY?($JZSf&Qr~_@Q$gg9fu&h{lj_t@;bVP(-2J`Ren#r}X;lntsEHliE*x#~pw@WyqP|n_ju*Z5 z&4)`_OBhDQ0f$ksw_qxxXWFWT_C%VD^_&f9wm)ZpS`+0_!?b zM<0G27Q$N&nHAxIo7rsPYuimqO!`r?dm~XU><%QtWQUGI1sHd)q6t+}e1%wqF?Oy; zad(Du&n_KaD6PPnm2#brHYHlzXNz7m$nh_d({t{rE=#0XG~Im3=Grg;B3pzSgFJ%XJC5AV}JdR7oW%QYq5kc?A*S~t& z07OzK(-cJxCC}0>#6$ZGh$fLII#q@kM!@IEYLhi&bsbZ<+CR6Cj{tvHUJNdTbKCN5 zKc-ytcTU4x^=Yst42h>y>hgQ;NNa~p=lfOa>w>l(jb_)_Jm#p)i6=xP%)>*P?vDnA z9BS3jzAto8aZ`t6mC%D%f$<45KLUXxtjz6OeNh7A+ZM#5A*VH;QQAy|gy-9PKAQ!h zmS4zu9>#iCd7%M1o-~qunldESUaidICY9yW>RfH&NWAPZTu_w*DU7N!U{?4>hKmc+ z^SoT+w@IY$I`W$NbIH^l7uy*VGBdfRvsCb1dSPu@_EA)woD~HgY*{py$X{FBfl|=- ze4eWW(tYM%qf@T8iw2pv<0qTRddoMXPA!d>=xIMBAUo=v5?1T{z(pm)zzkt6LGIYH z9vz7_HV1wlK65l26plHl5@gGCz%KQhWV6I7Wp8hS7b`)0JI8d#X1$XpMLF=mHwA8! zZG=EN<5+Xs5i$5QPUit3=41q8*b^d(4uqqEQr_QmK36-IWn9a}w$7rPFr7)C(mvA2#PuDeyI4PFAA<<}vBxc+-~V;O0!?)5Lo7kQ@f zeRv{EUZ>~ZzeBuQLd%QOvJ^ZG9FZQ?A^cm*w+js_y3obRuaKP_FO0}*&or@&z6{nv zVTuvljq!2JGi`E?aFM3-MZc5MrxxCZlE@2V&#~gj;2`%sm!@F6bs8n;9SkF*VR4}^ zbKTt*O8acc5xTk@RM!=b&%o?b$l%fyc!S7!1Dq3vId zT!7jf-@Z^SzGI|9Z4jSzVWjkAPGBuG!a#>k_xw6?n!p8=pH^$@0TSsLo!42>&he5e zAWr!XJ7@o11ZoR=U`}J%{8v@*FJWzmv5rWhQ~#GCB08o@8s|yMRklKekRx8Mtn*L= zMGH*eX%=Gbr}`3P<*a3~sgUg|hY#DsAW6tpRVXBW=nk~6pIr0qii3Yuu-fzOSv)EZ z((JZ#t2V)mjGz^6qzMF6HbDqpiqy)4#U=Y^HO7l&8O-@;@~z*>JHC8g({xnU85kT5 zT!BZYx>RC7N$p)dz@%dJ$r4Lry8%)O5X;k4(ZqDC&|)|E;SN0*H1x%>xqk$ZEQglK zV!==_EM>R^mLJjp7i`RQq!~$mBuT_^74zQaAylI_C3TZ9f4IWq$w(&j8dGk^kK+ot z*nU+0F)(Ye!=3?=+1y>C(-ML7?1Am5CIf7H#elg z+W@`3&JLvK-JyPn?2Gs_5{adx-{q;;oMS;>7jn{GTO0Jx3Yujt=05!e5`|`#=W<3a zXLBs5m;|&?6#1=uO4{?L&sNw2Lr$QQWBVO0ZAW-0j}@&eJ%Q~#Y6JPlcYdDap&BVw zrZcC`NS0g8581R27qhUr2i;wZo!M{6-c>{cuUtssv;szKjVRG(FH@9FW3G=eBog|J z8)Jrduhy&nPv(TJS2<5Ey#yaj!d54w`M>$MQbn%~loD_o@bL~TceiJpJY7!cFZCK3 zdmFL(a@0~zLG5~!i+pp~wEQT^8RY5wzArSEqcL`GAIXres>hDqxT$y;gA~O=ioFkL zbZ^8O+Ozi(*+wo(mRE+%4MV}^qZ~-yM%iH}X)G-Q8&AF=glA4dh7};nj#Lh3FWHC* zv%#Crr+D$|pTLh_)~e#9f*Lx8z|zq!BaIPyseU@}4ieKI(!c=zN>-Y<98p5jZV5bl ztlnGD*gT|;BHF`_XH2$$T7NPH6s8+fJL5IRsyfPrpo*6Rn@BX*xj)Cy800&8j)+{z+Q-TCbssmVw6ZC-`NRc+uH;gzZqbT@OF%)S3jja?w>WG zGV*}fe}Nn~C~|idmL-Z?`E-UQwtn%US5uy-?+IOls1jAbB`5X7z_jCYo){LI)YHE1 z2qYJQx4GH|fB~@%EBU{Pz1)P$)?*FkGv7=#rD5ZL)Ercx{@|A#TB#06MRa;NU3}j| z5$Hgo+5fXQi7n^Cu>E1>-aw6)w8f4^#py$xrH07HZ*w~&32ZP{voY>KC12~&bh&ra zmq3EQYXOdMv)yk!KF0BT?D5+{Y)@cLDKw zQxvbLfPZ9R9Y=f7x5XskPy*3M5}Bni{>(IqsRHkUvbx2~ZumzLp~wh?*cw;Aqk695 zvh1nqpTL64O!tggKh;UMXI&JZ`^v5!2zK?h<0q{+(LY;XV%Fk7B17{AJmWOd!!lWF z9TN0_j=eEj@n|j?_fGM<^G$y>wkx>TH>(usxj?PL6!8*?;01hiabh4Pb9Zcv*WeD( zD>tz_O%wk7UEyhPUowevd|@m9)X8f~jmz@u(o;8)_;~gR`dbqeQNgk94YB?%-iB^t z)S%!Q_%}eY*XobaIB2v~QMUX2x+OwbS(pQ;szW}tAn>??adGo}W7NXZ8`Z*Frz%Jo zV4X8j9_I1f*`1cuvzdUqnDMlu8*ED8g}Qm2ek1s5rhZ~ad6ej<+*tO};;;T1MqsL=% z7YSf|rzrL5-ZG};gz9DAMlWrJvU;Moit|h`QilFrl)9-LVK--cg?~yQ#RmmITCp^d zp^Kfa3%%bPU3CAzY5**%Hr2!^X;0Mq9H6{{gNs4)l$@$~5I702b>MbaTS!2o=_8y= z-V@qAb(s+L`mXmp<|?zbf9S#{RfIVpmF2$huKxpCY6br7S#gGpgSb&al8dEQZ#r(@0{ooj(h;z#pv7@K1O1!ln^!3)|oH53G*!~m=x3<(CP*f z*%5bNY->gTWYCG)i`w)y1#!my=7q;%Rqwz}oe)|8(H;rBpIxI{A`}3+h@L%K2hx?c zYk`PKv3~Yj7dGXlPUmujhuckUg#N|4mGWCgCfYZ9XmilwbqB@&*BaB?MG)WV8_S(M zkpyTzr_0goofLY}MgOCl=EJ6|Bf~^zMwipH*29%ug45Zy_u2U7b(4e8T1nx?zAMkm z3Eb=R`ss|8(kV2WZ&Jil^8TkwpvFN!A8ID?T9zc5a5#eFZ4sn+KnH;no5TWLb+k+B zvofLu0spg)BB>xldhxYEEf$&2Ue=I^BO2MJqIR7oNY1U3=&SC%K>fg{5n6Nn+tV&b zQ(0s?IN*Y)z#1ma8(*Qxzywp&yJjhNX4}knrq4!-YhOoQ1~i+|wuf7oU)4H^Ee)J{ z2R_N*hHlfcmSBa0=FyDRt?+`;2S=sB^Fy3}1?Z9^hd8)_C7xU>ORcr9IGb-7;&yf> zvfT>wvXmr1O|P@$M&2Y>#`uIXFCyyaZ=W9@eM(O=EjY?{A!WDb-FTcDX(J^$Q*q2~ zVjG>|nj6l!l2aFTRSVOLyQ-6XDr@fV^g--B#Dak4|JgjGAxLeWsc6-&2f7}JzI3_Gn*YlOQCU2EEE9;$* z4;tle5-qO^xcmg~rX*jcbTSEBW%X_vm|1>JP~*z>JWOH6OZkH$W=kJ>?t0%UX*kGW zePsV#6E(DANV7q2x$apvk8LM~p8p@@dXPvvAMq~$<`*j@ik4n`ec%^_)fpS*iHyJe z>Db1?>rgYLlnydN5Y#Y|6g%teyv;-;1U-zy9LvdDlvL-#5h&tp*QK8iEVA!3+T701 z0m(j~;m+YJ>a4YA{vF2u`?xbq5dCTyHEU3(Rb5~A$AYZ{EZw8z2wcbhx8Zr@?(B+R-(e2aRZ~g=}P++ z>R?{+y|lw-O`q9ay?q3~C{siJfiiyB@ZVh--vijGxKam;*(iF%YSQIxNYLr&1A!Ld)iV(M1nAu{864 zc7k1!^;tKD`FD#aEhuVg(kZPMzAHGDQ+5P$kH%?){;nqWYeI$m9G{%Sg$0f+&;PDR zNg)y|K*&_Xclyef*pdTWm{~re?qd*TqG0; zTpN;z@WF-keA;!pBaBz$3coft?XZu3`w9vQ=JD8fP%s;IdH5*x>|7$g+7dI9a~Qn~ z5mVqBtRL)HmgwX{On)l=yBVh0QdOh%Jz7oU(nXNiG3590@26!KMh$;-i=gZJe^vok zOb6ZsI6mNfP*GNkZqAC%qb0WaS38*ww3@>|TBd8O zn11#2B$!p)xR)YYet`C0*tRW5g&xIY#FNFCQ#(wzryr%pWf4kcd)hpoV)~e0VT|=R zF#YdiGlTo;91dPK6&F`vb@iaS+{Am;Gdd=rhyh)nc~8KEl#3c1G5|+9mzs{I4s-(=%$ann~JQuXDtHpin=A{Cg_?w#0uO2e}6{^}fkH z#VOem^QETV#{ZwE>^rG_%LGJrgMWL0?;HMigI~IFOoSzbW(*FP&QLC?ul>LI$i~>i zLK`T#sNO>e1te9p|BKEL(yDmEz5s<+hRbPim*KBsL;UNXgZyip_5C_}CwYanD4j!c z4iakr=OYAv4%0C+1Ao|r@$xnOFWJFt-b*d>rui!Diz0$vbXoV#I1d#PR@h|>{`a(1 zV(v{J5)PMI**7!tJ09Y|4)892yi)p36hcFB2`DlHe3q}AacBCEG&A=buuWGiK*g9I z+a4+HU^$-&#~Fv+ZQVV6KS>2Y3Et8VOsPkTiRW``F?|kRrmTivg0k0WrE^Q(xKP#q zHihhDYeC`qjgkDv^ZqrZ-l(doe27+LbMuCS z5=d5L5)vVG6}=dcxd~aOB&0T*WyeQ>jF81bU$SoQi@iM^kM?^KD5tqOM49sEK5G8gz(-rT{kVTK%95!Pfg^B4am+u=G+qtyKhqbHW< zhF!L(Hvi`%22&N>ee5Z~RQw*)27ek|{>cycFF{YJw%m+d? z^d>`$kdT5VVeWer_;W75P{~%;?mw}o;tES{U0}o zFH3CpI@5E14jU3oiY+R9K%cDCWxLSzRQaynSCXKWgM*6MT!>#C5{v~Z;DtygM@_hLdE@rd9 zy@MJbyFreRGp&NDS%b@`|7b7;g+?D9@mEcEuHS~3P>Cr?O6uSJ%WB)ADX}phJUI*V zk0B9i)P2)9v!nmUhLR5MINnlY_~2YF#CzD=n`?*tV_c3BUyd>R`t&paneQ+N@jlB9 z9KElR$^@rDtQ@(iTgZnGbN6cnsnL6;Yt7vTNI4Kx&v6<6CuuB$3(y#IfO9@3HO zr3`A|v+7qNny!inA(qtW!zWt7>;!!ZsS7+TE&cB=dD*|)oQkekH{ocav2tnAi|zEc znTN?Pt#^Y9S?ly|jn5TrTiuR~DXoNYdwv$&)ZOpKrR;`SU8`&=dgySHiJ7Cj8b%u; z*3mDE;PIS*=5G@oD6 zsBR(#6yCu#D};v^ou#%uJp7^_m95oZVlU15K_&;@{|!UTSdPY2Ca&E|d!Rl&;f%B3l&Ukm3)^X8z4}PKqV>A1#bj4aC*Z&(A6=_^Km#s=yp)| zI--27kv^CdQoLTr+CK5?!|Pz1UG&G62ecI`SuiFQwN8#-UIC}EZ*3ynw%`mBc|D!% zHHN9k1Dfosn~3muN<67>fQO>6jis}gkE707${51?qyQj!B6rFo7O>|vPdPiMo2NW60{Dk*S)x??cN^z>M;>4`UBCu%JaS2zibiZ0jC7K$X|@LeMQW>tZXotD#c(sx^hJ zK(&Bfzn6D2;FtJ0ZMWJzx6Vzw42_HZjgKF*58muJTVK!Ee|Z90`y+h1)`VW>A!=w0 zQA5{S!(B3ueIR^=>8LAKkuVV1&acO;EKet2g8e!2aW2tz2T zu-7*|<@LXGPr3~8_e7LF=UIa^TE!!`qOd1#=i|#rzB=lUy`BMl$PS}37@HAM%`Qv} za4-^pi{S|}aFCqh)v=%_m)kh0bT@+H`D_(lov1rgV>FUnR~Twj$`WtO!*M^{v+0m1 zxdYEOiv`xPqh^!sAP(?4_9Zs;xkfy8ukviXYi}@r*5`Ft*HHHfOWn~wZ=5rkR-_AN zrbQ`Lt#7w5<-365X5|~9bob%F!$T}ytw0%VBS2>^Z;lrB+qQuKiLyH3Pu1mZ2?vXR9M?a zL}nm5;kkJ+GXIhhMV9obsU|7VMx8XK{FQv7pNqJab@(p{Y`%StFavm| ziH#%Mc|EY)ez)geh*B4rNO^vZ{I~z*+H*BcPlINyKUZThaDN(2c zPhui5z^tfyl{t6?Y;3MRwU>wkmL-O#EW`J^^8wGHXKVOAn=p?Ie2=@9JS3Gbf+pRz zBHwS&0Iz}oPGsjMt5S#7cgImr<8IAM>0`I;{7n^8RO=5^5Ip9iYu8C7P!|S6e%eRW z!1e9p@XsvOjAN=X?Veh|7}bTqpv*(m1Ui4jDR+9x@K-V{=)h=pqayvVZ)ewKJ|)a{Q|z~<*y-j3!dI> zdEXOFef+&Ocuu?SA{An6FECAGP6EbYO`g_w?HGCN8Gdk~KmVX1zMCwv<0gAC#BA*; zas;0kACO`rsSU_HZJ3BGn(P*Ool=*WQTdSiL~Xcr=1%w|@HHPJiVOzt8{VCtK>Vr+ ztla~7L#1a@^Z~98uMH;pwZJ%{iehDCOrK6in7mz^xTCs6uZF@v04U%1HjL@%83T^{ ze2NV39MhjZ@U{2p+v5^4)!774jHgPd{*^<)7Ip#D5!Y%<^u=9`=ZTjYe0Jpe4R1Cm ze&)=?*~rWqkM8L%tX!`qIq(#n{i#8`a+@75Rd28AD3ZYbX=#O|#L_vAW?W1r=drh4 z)ZiNkgF!ado=)(WNU~q;brLGhY^JYoY+7W~+apEoT8(oQN@kVfx_bn(7Otj|l>JJh zDMynPtSQ6b>m~E+Yq!F%H~DqyJ^r2CenhiCO5RiC{$gt;`s+FEZ4&QA2OvIZvqY$` zFva!T(k`Z_t1u`6<+loeMV0g=Zkp3q)WO7V*m$xv|-Bpas{x`cODaN zdvisa%04~u#d?3!%6=7G!JeiI28nFqohR9w z_m-8fd%yocevSY1u`yPLiUGFm!E5n)+^Qbz!Nu9>#Tq0goQRj1!*+e`Y^4GcmV=D_ z8UJo|8auNYXGh(b@~Pb2I^xR1Tfs~r3yFe(CeU6Mu~nVb@Frzxa`wr}?w;hPcAJr` z7t70_^>hK_lBY=0H}+NkB^#C8Db0EQ(oC~~ohY3T{o2NJl+7)xi)6;OKq?XQ(&#qXS3tDrE+)}OnNE2k!_8{@D< z!m4|xG#)o!ryYgCoylV{yQ&^$mFQ~&wxF9}+gEB?UNAIXu9Pqky~88WO~i7dd_}n2 z+m=76k;|u|9jibxI2D&X;uSrd~BtsMve= zHb+fm#_PTykEtW-Le44W3Pfpw%#gwE*IQh;m=CKq@aS*8Obb%2{7DU=Np@``u&}yMGVa1=;>p+b->9Q5_hM@%U2>lO6HX1CMWuEf-j>N%!w{X z!|S4=-9RSAkD*U^>~*I66YW_tDKwQ@V4{qF1Gf1gxW^lFAM}a_67CZ44U3+}I6e0F z(u82+S~HTWI=jl=78SY)mawJPAIY!umSZ@b`9ytP`0s8HyPH-*uMT6PKk(&b7?Zsu zB}J&SQjTBMaG1?Va%Y0Zn zeVP8cBERy7qi*`^^6c<%WufjL-CDWUV(N6wTtrxMJTWA?1jCot>J>6!N+v&Kd zvOsf+HIx=>HkzQU^1GDFvx#E^(xpp-7de!cOL!I}ctuI1#xeJ-b{87SQJ;iFhoCgR zNAGz&yh$$uQMpe;Pmq0su~+P8Ee?DQu~HXdp_>wv-7D_qn|uQ)z&UN>F;b(tVD!%!1W|JJuEE| zn1#Ab0KsU9vdHlL57F!39sOW6oG2R9=BhEH*O$8c9civ@S{kEEbQ;$ld6XN_xYbo* z&3Qkh^;E&o934(~R{U3}*z#71(B%(?N>Nm*9bD_qFVY%Kb zGA47iTi9urqDkMA<$>eRK)VFSREl2Z;f{frG`66xTKyM&q_j!93_!NJJ8Jf8hp0{q zEMV0)_nmcZ8IL1v&!~*yK_?ico`GM^5-0^F=;wnz=;z8F#3?ZZpW^06a~O*-UV6)z zQiY?zA}kPSxB7Sc@_|=Nu?&@deq6`B>7HmquucJj`cXG)UzF2|F;kl#;e?Ir zjOc--kysl*|NHgSQvRzOw;(!Bozt$6`k7D%sR^p6wayIi=%g`d!7EfGbzd1fV)2cG z{_50pbNY^NdfVvnPhCUYG9Ui(kj)v;-QN)`8$K&iMv^NgN}q- zHa&UMaUIo1x8kMr3KoTa8g2@DT*X`Dw`q&-R7V~d^4+uxjoy%tkYxjV%x0$a*^*y@ zxyg5@Frt_8hn(fhI+Fp!_Po?f9vJ*_qyebxp|_;c#Wo)v&?4{5l&^SBlLy1yrM+%# z@gOaJ8D7v4BnF?|O3k!V>=QH1-0$O49w-K6G_8qg^t*V2_xX_3_z4pKe04Ry(%QBv z1SP3d<8%xMKAoDXC8xcDT_i2C%=2T(H){h_ZJPigkU1Li>!5c21pRm>c+13hkj7pe zbhoZ@{I+A3)wqoUxdEp%U;6O$lihASE$BDLDc&yxkHw|uXI{nePu+=ZN%^ll-^e*9?I*Zc0MpV7TKpz)1RB>tjzoU;5)94nfj-1FELPvjQ=z|wBZ z3yycxpUdQ<)OuF`s+U>3XECeHP_H7Bj429w;eJ-D*7@}Jjv55)6~Wx!!q9p9E6A6e z$nVcxeHc$xGeR)FyeALF`0|XjnrJPz`G}~&ewEg^)K(OXjr~3S>DP0W2~*SWxl3vn zbqeq{vJ57nGyuuwCA+0CU=M3&&mr{T@$rj56*dZB5J>ak(3nHSM=aUOIY{%ofpRG1%KP9Karv8Gex1=O1H)AZUx^ouku)LND2 zFI>g7i7`JS8M`};a*rn)9`JJe0&5>xqT&$x(*p}DdX}W@B0o6u#Qm>VWx#p9erLWq z*F-3Fl5~zZ>iqzdTw#8nmW2~tX(crEXyj@=mRG3PpZ!43UmCJF2&6|QR(HHzQc^Av zQk9F=H;a1rPW-lBsSd2-XwATPxmeSVZ@l{H%ZvY`Ba&;r@(z^FQvUc0WZ#pe+1WB; zdb*si?8cXmvUOj~&-lKatSqVQ!)D@~JZ6f0#W5sD4H_92)7n{9EoRNTHh{zb^4?rz z@~{j82PeG5>f`*Z$EXiqh}>JGOuGHM&n@Gy1Y_c`D~oN;@K=Br3=I%HmF^<2<*EZo z!1xnU6#~W*Ewsk1fGgf&HMqiLNq9yx%Lo4{Ijs3?J>&=RH{~+qLD3x`KH+33bUn=O zvIW8nzqmw3$*@e(;O}ntj_3l)Bd*mC+q#Qt^_xxYS$6=w(gNoQ^659I5FT&| zve)*BQ2i=);X7VV5Bz*ck8VL0^c21Atf^mlF)1%BeC|apRVa&4jCp%fR8>XsCpBL} z{@r)t;;^}=`eMV^7UkoAewUiy_~@em*{%cd#-o*nKw0u?!m7J|@}gBfRmg>U z;Jmtm1P>)RH+HX%(Y$XqUiYn)J;NO@xLn@7!dWn>2bT7p{GHJYnpYy8>C09KIVWDB z&@QBGH=V)3_?$l}Gvuq01}%~6d&!1jTrO?{)N|IaG}ybu!xORaYjyC#XJb>B)# zRq=3@cUK6q-sn$vE&1m&+g0bpk+X)*Md{uMcnMb=dlrj{6>%&;#gu>yR z^7%qHwk%O_fHRT1eJE@YPGfk_MuB4Rb_ut>8mFj>kZt1SYN}Eoz@OM2nI0Y>?g}p; zg}qV{_*_0ea_8dgHq#M60+&uQ0|7~jhZe$=8bAG48m4u4yzN3j&i_n7@$kauwvC%}7p2o!Zh zr3v(nH{lajRp)vkKXFx6hRZ?Bl?CqsVw-AGNJZF*Ldh5X@w(I^@|IF2R%3u1gv>xZ zzefn=>f|Ss(40_SBRk{%2IAWvd(|$xHrMk|k1WCPJdyp%fooH*jx)4Vglt?5`gdj7 zW=EBfvrcJl^n<3L(l=P}|De^bfA3?|PlSS}z^>XHM*w`>EkIvvaV6j#a5G3FAb&M@ z4!lPCUVtq!aL5mQ+xfBzc|nDYpCh|{d1q0|FcP*(kqf^ZN4}q<7J`_O@wL#ueliPe zLEC64dL3~YYZ&8&%T4qzRp({?Y>#seCje)}6^g;Wqnu4RW1Yf7Bx!?LOlqmXH1`{W z$DBh(CS|h7YTGWma_!ih*0G%&BU9m!5F1UkKAjLJ@DCv$l1u$~sAl@Wtd8((QyKl3 z+-@H0RmnRZ8{)43cIDLPX=Pm=N8T#KClR9R?6FpC>6oZbVC_}5vcZeNURuY2Yj+Mn zc5mSx46jH4J}Odaee^pSV`9px?4H4QQ?T4LzY}fEDT(^6nhOP7)|b0Qp{_R^Ny(aA zw@RJAfkBeS^JPux3(7}@3fSSflhr#t+zEv6q3?=6!YH8xY*dQGpKq2|7FK58PPB?9 zX5TI*2eNa(Z*S@*ssm0N#w;&fv(ry#t?%V*iLgHbf8y}GYy`(yTBK+Z% zqf`6Bha#I+LBIvFiXXb)dDPY`iQKqZ0NaS~^pNOHAO9a+Ume%xvaJoJKueM0P^`GS zyB12(;uLpxcUmY=v=rAsad&rjg1ZNI50b!_eeUVr_q*rbf0Fn2CYi~SnR#ceXP#9% zdn%fjJ`b7#os}uMdWx>abeznm_2pyOh8^AUmow+|ug;L#HSk7(hx?QL0=)}MJ`MTx(O}E)7lUD@|CgtnS|V(`iGynlt5Gf!13&+{`pv= z2G>h?{OyZ)b1;igAq$3o*VncA_?ypc@5G?(2mCN*%%bU+zct}j8q){Xud)P$C$!~m zqVt%piy+}5>w84V*gNT$jfmuYu-a>SCmQUBd6Yj&&J-9lG*!HO@a^UL? zGV+pm)UyHd=`?hu|Aa60qp_0CaEEAZeR|&F=k7IzbIkTHFZvVXrMR&vvLh0d01-F@Joa!?M{MWB{_prhOYXKp(JwPTs(YO1PQoa6DN7=EmXAew3++Vi%KH zSD~pWwEgg^9}f)_4_ixy=nx2?Cr-@`5#?igv2=5YN$ z{FX;#aQ%ZG9fHk8a9w24N1 zDY-YIECSk5sS_ze>RcKhyfJ`Jdt8}j<;Koe!c+rj?=`A6hP7pi((2E$%e;y|t1CyZ z`zOGCyI$h4IR89SEvu8Fu%rVog$fuin_pK{7_K(A@+9%WbG$BC^;uqdj(fAPwl>O5 z8(o81uynlk9qB_w_W0+JE6ct52L57|gRI4xs*5#sop8DD%C-AStp(z*CzSV!`-89_ z$JJZwU2wy#;6MdqhL_2dt$w}I2bmb!w<}1;%n}-5d>yk@OVvZnEx`*<7i!X9Z&ur8 zh}Byg(k@EP4S<>0vW0sMr+G)SCU7`xWqYa4j+9C87*FGUJLP1gI?f z=Hg<{WZ?!kx|0s+#!f4-$0_S&%Nv3EVObNJ(2jhBojNIW!pNNyq1(BXeP|K^D%?&V z#1Nc_xz*p!8Ycl{+u~|#f*q5az4!9uTKz$^RN++gOa2mrKh^J`G6T90tz6vz2js(k zmx1Rq&clP>G$|ELQvQu}7?fZ26|D|~F9^)J1L;H0a*N5-($Lo1CHQB=F1cx7biwNLeL0}Mq(WVVEZwMeo;M1?w%E)_lVH1&;cV3x-4F7zY84c0VEamgWOYPqBZnMDeY;!q^@P@1O59+lMn51NJ`<>4S!=} zg-tNt-7_ekiGK*{Z@6z)i#{IP4fCRdQX;%NBz0xap7p6pbo0~-pDKls`X#aLufR=w z*$?rS>eCyodU5G0~py-C6Rf1(I+YA<^a`+ZNI#8?%!0FujeE?rZM| zY!34@9zzLdkG?Z{_c3JGRK06E*`xOH6jTbt5=oQ=jheR4GVvpHCdl-U zb98TEzfu)DNbSrzH8b^1nU8-p@kzmn2d34FfT}Dw@yVo#0ZYo~K!l9NNP|B3Pb3Tk zOfW8{{{0H^BIa&yzMuQ>ZLcrCXYETp1oP^itv!;W?Pck_s4_>a-Ml8l3Yu&<5jnso zmVAD~!!h_?>Hqp#`3;@(ci7Iuj_18tiT^&-?d)V?ys+7T@5eO0)9QOpIQ2dsdN;nJ z67?77O0Z=u*V=ry$6T3N#5$CvaQwuenP_#~|q0xi! z<@W(2b`OX{WO271jwLdiu_fQFLpA^JZ%bVkpOrh7Q=Gt56xV^v(v$E+EEe2#c?DNY zF3y%pc+y&q|tvn=qW@CBF~Mmf7~p}?ZR6;&Nqxawh{KpU;i|c3jNp| z@+a`Yv4Uf--<2wC-`V?NI&-h*dev4J3{O_R-hG*7GO=N}tm9C1^>KvRM@f4$d@rdA zQfDTAfH;#-dEZkO?@Gts8Ps#yiD3EU%Una23Czf0TC)g$i3&a2N}1O;I*4$n-UvSp zOlu=6$i&ffiH6*?UaSiVS^jt)C~3XwHcl+}jYG^)P}c00spuF}^pOFB`X3gTTa(U^ zrazjCKD~T{$70ea7jeH=dmvAUwPx=eGT@esg&g<;^O+Zb_}m=Qm^JtP)19#b!y-Bx zYEuN_8-QWgFHJA#I@o1TkC%<&w-H8u2Nk=KD;7AfRcts2?6TgYRJj2wH|}^oTVP=1 zR=3vjHo(>&JrqHv4m+kGQ}O9(;`BuN6?yMjh&822AMp%3ZyeYCXDo-8b)^JH?Y0~m zqb5G)J-2FX z`7y$o`#T^`-(6I7ml!>Y z#uI>Cu302xNNL(_GJ1md&A^NYP`aR?-X!r*ezD>?VEhozU&JJDA)~=XS&=?$pzj$P zQ=QNL)XqzAQXdOwN}}|>!P#r$N1Q(AA8mK){L$5u0#tgrjFc#J`QkphUuCZsQyiY` zc|m&Q5o!7?F>fXD?aE!ZEvZsqF{`Rpr7xp8Nve}1IhQ-#q;Sr`)k1)*h|@!9sgEfh zncpvo!$FU+yRA1Z~+u^4TTh89dX73(Au-jzno~k55JkW(iF`!j+dMw6+6(lxx07dM-mw1E1I@ znC7U|(tqDO%>T*dL{l@s+uj_&SR|Ke@kow8Y|nZkLLQ^p!ID+HXKF4o6~wFI%?Kv3 zi8-su02T8?$jQvcENhgwVj5<2gg}>`H#xx9M5~8)SGSw4^{Xu614+x3XS+E{$n5-H z>@fy9V_$_^suDjyOVy=QkM`SI`(iclxK~?}n7VkjSfm#KMw>c|BrP!-d$B%ZBNP6} zA0O-4#Y-}tCUn(LeR8$)*`CH_CW*#g-s9EWN} zJq%y0Ih8b~1#V;)WLL9gK(?$S%SIMGY7VaGq-jN^L?vHkBVl|$O8n)sAZsa;iO8Hu z$ojLol@h#$e*BD66PFGPqwKb@8aCh3hfzLp2G;i&$7Uoe?sJVK+hUV^)0I;R{ z+M!-^!K3dFo|2M@ZKXAvW9K!@RD%c)OR7%iIgZzOlWhZsA!-_DkZ6h>VduDhCs1h# zJ5t(Yg5P-B*|w^xmD!l!SNYV|u>1C`$>Qw}KsP3Cw<3yG9v5#~@$0zPOSdShfyep-NBtJw>) zW%%Sr{s#Yym73$H8(Qu40kRtJ;8T~;r1JIoquXz-PdgZ&<=PNl%^O9M=5^`E_wC*| zTwIPE(O*~|oG>R+8dMpbXhtUU5J*-SMHV(xZwB?y;<}hJ?TqW7B)i&SnS@50y8aCP zbyi>PeG(xA&cszB@xAUv{dP#U5$)thc+t?Vyq854z*qwCiA`Du#tm@>TjovYwLOx# zXtWd-Ipy%aTg-nga$){4OY{^{;qSeZ;Tr~-22_zeZ0^Y{At_W4dEQ2)^tQ}#?wz^Z z?{BbH|HN4xVBS5Ek?Br~ae}&P)5^Y@EFrOZc(Z(BV?@l*Y8DBo@!Pu$WTmb)UM z3j=ly9yG>v#J2o^nC7+>0Ax%qXS5&l`a6DhwbQN_>CBPZ*CZx*egl0j_30^}zVpe+ zOPX`|oSs~B(i?_R@Tf>y840fEi=chFBgh{Mh`(U(+8~mTx$5~Gfl>)5syKdgA-W6G zvL9E@nO?tLl620MI$Op(eq2f5Z*ck|%cxJMg(x@4nW-pli~hrVU!#g59U*S=e|9v(%zyW?kLR7ntX>EK{D z{DuW=DETJ`cAumT{z`HFe)r)(z=KvuL?K0ZOYl`#Q`k4j5r%_eDY@V(pxTtkrk zRumvCm{7L2+Ped0p?sY%HR|zk)yurSt&H^39_VB*&J{|rgYHX zXgmgweCi)KkdjaCI+nvZWVn(I0Jp7AG-op09+xdB>79eK+rL48^G^ECSkrj&FY3NQ zVgMFCJ|mQ`|3r;pKKP^zX-;i%+6W~pEGYvaANjv}CA`d@&F?|nAWs+x&%0Ri)R%6S zM&|_Y{RW%VH#1Zk!(86jp+1k>Ub&*rwd*;SgZ@VZMV%IJ$6yD*I}KIvy8T{VV`Vm} z7<{&yD#K$0J{aF0STpUSLm*Vr?aLr@+Z=~I_3i*WSoW`V28*vc?N#hG4d7St zG|Z-N_{C+Ld?|kZI$iIjSM{VkRi+kG} ztHnYf4^GfNc7w}i*S$+RRkqlPsC8I2Nc$8fO3b`;_1p!JF$fn&kU7yAl=Dj`i{WB4 zZVmV6U5_+4h_Muq$wGS}(=XA!2%$gac~I58#E#ci(r-TSbnZ`Ll{X(eDe-TT#M+wQ zsIG=aAup`ilPCNJTd+U79+g*gKFFfb10dV`y%o7MU_`ue;pNW|`1DzkY-6B)-=(dW zJyfGGwEkDKCu5O?;M+fX-@@p;Fhh|=Q1x#kS>m}kPZkk^_$MSS=lX+0zQZt!E+I+ZL4F+0QG4;cS0R#!KYdx>D*fw=_pWRK9d=ZD#ptR# zCRs-rJn4V10O0+WM$RJMka~{^6+dQDjI4u^T%V^H&l96hx?ki26t@7X1DHh)xO&@K zr)e$-ZB^CYT@-7}^R~0J3LZ40+M|9s&7eDTez4Fv1tD!a$NsKkzchz!){l3-#wmEy zfDlYF_)Lg`?b4%;WdOOqpl>(L)gEIA2x5r>5as{gO2+!$eMNmA@&$J$1Y~f_Kq$w$ z8^J3o&bbfJ@~&^wSGJrnY=68?!4~anQPH8Yh{yHqi~Sad&G+w=#-9l?`0du-Iio0q z@~Z%I{&;pU7?t+O@P0A+hq4QPZD2ueF6=~VY{Bo~UvorvO+%jM3|AthzrnSdUI+CA z6ec;ZrjCW9+M4NoAMNe^sAIC;obBCMRAjeMBSpYR&;LdXBljBDl6H0Zd2t;i)KK>l zLm{X$HlPI%U|k!SkzpfBHL`0@*aBtbu~;A!KJ{gJkn_;MC#G>8G;9XZ2r@@W3<2K; zaAh<*jt7IAY$ZzOOGFTBEpxEg;{(Lj+Kmg5efp6b%jI%iCTP`oSq(grFTbUS+?-(e ze=FNk5lD+vc081w)uFY3^S$Zb=5+D>N9=!JO%5tmOgbt`@nwsP6AL~pYmqIwG~2jq z`i_h<36ic5(=&A`pnmatn>x2%?T|!r1P;A`Q_s^tZ_a8Fc@-q>^E-GKC@T+F>0r_` z*M>F{%LT)OMYL50XAPc4Hrmh(`8@Xx@5<$R9-I2rC;W1fy(XQCIVE~Su+kyecw6lj ziu;#zN{0LoU*i0*UjXf<2jQOuy7+qg=GTE-mp&gX_}np`c!S>O+L^Jg>d> z`kf0FGBxt5;k%)ATD0yN2Ke0V)gs+1a1qmrW$1>{3jf}7m_yz0H(?)2!ViXFB>h=W zKYE{KBWOJ+m)}qcONJ$`6sn7EL2jKy08m<_SpB_&BiSTSM!brN{0Db(3P)8|MblQS-^v7~`=gxafTDGs^| zA2axJ0lEGee77p_r8(STE+ERNyG2*%$R$M2g}?n)cg3v?r8%Oglt^(!kIZG0J85gk zGkQx>y`W80Z_m+sn94}{` zfR$!O&@tFxqiqrx=evmp@!RG-wzTQ+(O`49xMvD*q{3H71KO=C?JTmMpq-tp_FEwG zI=XQ2wXgM#BiOMFQ}mvt`CuRdsnV~jn;p=tVt5&EPe8{Hw-<54>UOD8?*cCv_`ZBM znYZOoRq}g&um`F1Gd+}#fvtz-?oOxkNQ7VfS#$n2=6M@S&lzd`pqXjLby z-Ns`CYz{rJm=6#b)dS|8w>a>*AMY!bQ!+FI=D{eJoB_^le|P#<| z6ch`Ko;XNqH4Fmk{Mf(14qU>YmoSM+EE)-l3`DD(x&}_|$oJ9(zdd;{9t!4EjlC$w zeumUxbNq3|zs>hM2O?C*t){7|qV5!IB7o)^`x1@3_xb0r$B^&gY7NPMOg0j=BNZe3 zdxl&ul~q(mF*R=%qhc8f%{-Je+kj`n47cS3xvAE%fME!|MiO=eP`+Rn=?K_T41uy z=>vidpL&!37>TeR?({C$Go}#>3h>CHUJwf|U;HnDSB)Fa$`DOv|i)7hs9#@+p#E-PB}nh%Nd1ajm@ zp$mFkb{dH41*2L92<9C$e%m@VKD}_~l$v%><`P(s z@d=`wZ4`{|7nC$Wv92N?(Qew8B10#erUX$;Nilk*H zYDokAf>J=tW#t&(!`LG{eNRm(YSS0Sf3-%MF^X7%_U!WYOhpj7>am>RVG$tn#gT~R zHV^1bvxBcKI|wM)S2u?_oN{b|l9<~^!uO1l-)MBr{!HUj|L=h7zw8LHzsEgYR{~eM zyn1~iQ3M_MH?~y)Zc=j`RG-bTyq?k{*M$4>11$#!eaf2Q207 zgwaAz@8;0cfvLw9$$(WYgO$F>*24HH;rp2xUe-Lr!&!VN7{~^0idr~YEN=`9 z;KT86YMQ`-K~kMT)L)N1ok*9tPeJf9#oO+>LE+c>(u0&dv1f&w5dXQp~EbzI!SZxojT2HzDVur3sfW z&g;^{om2Ueuj*8h`lhUo*jm3yBz1Dy0_zdVDFoJcpEhhT&Ig*Q5=@E`x5b2yB!2jE zXw6~A@M+$NN-u!BN)M-y*?d4p?vAeE#lM0Dw)wRJ=^|hGO-W89HxiMx>idxBl37xR zt|KBmh0VCt_oz7!ilw?C+2e`h$c!4XRzd#?L9i0tvLLYerFH-2byr2ms{6_3g>vzk zTxy*k8#*Ef+r7q>13uP%{+ffk?ckd-!(4`Ze1TOFzBBB#GT|aPl31QMGtfOqSaXmB z=q~w=@3L378J}qV`UETTbCIFaA(l!9lk}_(fq)T8&sO&&y>gv{aFIhlE`8QPMWg}{W9BDyt1Dk^5|n)L zxF#0*ugb9fZ4IdS1h6!uR`nHJ&T(s?!e&z@J!0^>P&zK`aRg&h$mSn%Y;Q;;T#IE`1Zs+LG z<$P_CpRQn$({=Y>+or<25)YDGjps0AbET9 zD{7t)%Ck{pujVrC4okk>$ugKF{^|)>do^@K@7jAW*Vo8{GM5 z5gAP1L>2%p;0}Dk1=4*de*=!DJx^sFpxSoH_j^-CRA>!ZSKo`%xl6rmOlnNuDv$Pj zY^I3~b3}_Xw`6ax4bEt{yRl=;chWz_7%(~F56>kkU@Me?OL(k5A6l-cUY(qv$tRyq z5r)2?;No!>@vEzHsK*A7r^W^Vy7u8W&rNuiRSgVRGk2M@4c~DDS%GkQI?mDk<92^$ zeQ`oedwl|{9{?t+_MUxLngB~*T3>lK>sN&h?W*B*Sm!;@@b}JZIi)ZMyP@aipG#_V z)RR7W3+8~B_9=G12B4I5)c$|#=T_1l=wimXvNjm{2v&W5gUZybJG=klX@0i_Um|Qe z2ZL?HSsie~rWo-ToZp&M`|%6fT%uWAm0iQU!x5wK00%ui%F!Wgn`-&`KA-BB^asGl zU1c0jXgREq<|TLQlfQceyhol61?3j`$zqE(p98wX>1!(YK*S7@)Dim(iQa#~_qh_w zJ+J~!n6Eis$cP_cMgO`&#Sp#ZFxcjM9TC;D{pK#@Er~@o18tBbxk<`@fQZ?-k^X>) z7ktlW?Wm#9C@qYWtqx%RPYt}DyEGaj2*Y=o)3${8hW5OH#-d1}hFK0IeUO{+>K%Ft zd=i_t7@twlFWGY?i&YD>AISdX=IP6%CY#V{Py3p*Cs|Ew8#Okx)b2ZfQXb}%e#$F@ ztU@2GBs6pxal`;lqjC)Gr*kemk+kTiZ&lZ#GUE8$Kei)qrtsHCAp?m&ORhWj*F7tj z9GNzi;~fxK_vGeXDCYINL>~3q(we|)W5vSe(u(b=BKo9@{+lMU%@iS1GsvFX?Z|h9 zLeX*qsP6~y)&7?=037{49T&wIcvpbkULG~uqU%@!@4vG7vH6v!54~E&OB`!TkjY&B zX5XtO4cK+YvIu+LrAFM^JX-z@4HTo1y)9p%1zgY$|`~5!3=Z@#Ohv)resZ3rVwA=7mB^t8{4t&<#nVb|j66CYM3-!2kcj7Fb(YA?WYGq7rUf{_K>Slfqtl{L`$b^o>?_7P+? zIy%t`5UmfVsDNsBad|S#qp2=l&v5OI`-DL3<35s3I;Y^y)+fx&TH&0m52h&ZGhiL@W+jK-@b^Nv7lt0+I!Ghq=}>{34c z(+fbg?`;{=Q{y)gO~q^jOV#aWmjMndUp7L<#OgLR#C0+GkQXX|#h|ACn~g> ztbLGxV}(jr1E-!so^LuwRMS6A7$O!k13=xBH zql_OiSX%=(t_h=tPeR#{zGuC8^{xM&3l@0+=U_L(A&85U$1h`KG~YRdd;^ffDcJER zWKJ)$Tq_~4ldO%;t!f;Hy4Rdmb;SMh_D%UF2D8TaKFdL=eiNq8(Z!-S2ElJx0+Gm9 z2n!v)UwL`ARp+@gE&bR9>vun^Ru}^z@x-$dwx*6f?&ysF2V9qlsDBONwP3p97kk7R z&7U3I8I-G7VLUu)o3$qtl=E|^ybo@hPEto@roMI`5Q{IT2k9|jG#`~_jJ5aRGW`%aMs~9bablX zbr%ZKBAGu)gBkt7n>;P8vpozVyngzV8}f!9cxon;Rh2YoLzPJOJIKd>aEl7UsWXWkm0 zCoXK|L08L0==USL_FHQse%JlLskce(N5f+}w|8SNW^T3^~u8 zk=r*x4&&^n;;n@XYPS{BvdBC#tr5#UreJLP|qiG!n~bmS}w$3wEe1!{DRi+wguEy$M?=`trCZ z`UktRh+hcnlZ!LBCi9uq6F=T=dE{ZEaUv!V-$gHdVO3Rx&=k$^r+Y#KO|C81otI!f(Jc0iG$hgUDl&#!7v-9mNNeSKY(On*%%mCP6s6P~gE7-|@vGP;sP%X?C} zHY}3!LMD5BC-+nh`@;bY-mA$w_TV%lslZJ45c2XXD0F*Xzg0~+p(bsPbK7oBllB8FS-_IXo|lmprMF=QCPOiNj~7*rRk3$_!wMewO?zGE-n(yU@yd{rH`vdejc-YL+>W-AO?kT7q!@3pN(C9zD?@Ei9 z13zBj%eTM)B31JE>YyXfs@AMaJ#9m3CD59vw@EIo50T zfjDJ`!E?Nl5xGZ;hEGxx18fhOEo^qafji4f$qJi=QKJcO?gxbvm&;SS27_RQXtNIP zN-pc;MEz0xd{X$uUQC+CrFuNnaB!XNJs7e=`5QTl*@MH#V=iZs&54$6382|CI!KB0 z?PUWM8IM5*Viwn94K|pSamCns%@-a1UkTd!@YTXUgEp0QN?zR~88?}tBtB3~5zmf| z#}FmuEOLXc#rDLwPKaY1uDp-L%k1z>gkj5E3=9kV2dty9?>!aJs}+L2F9&AvhFCMu zFFrG`+j=F{ruWQbDC%(|%LZ7y_8~?tLU7&0Ki}Jc`ewMCZ+kS6$dZ*uTmgze|Mm^4aUGc>Z5OryFw z7Ia*ExQU`TApNH|umDW7Nk`|Da!2CuuNTEHWH8A{1f_yKAA1RGgQm^ow#(Dr$|gy+UhUs`l^s{i`r7%G)3iF_x$K^N7pns z+gyB(S5T98LaWhnLMD z%5;%;u20IUo3>8GZi;6YFOnZ#vnQBoOl5WeW}EDyG@h?R%|9fSI>dG_qBT+!=JY?l;|hF|F;ND$cLSlz zGdpFXk4V@CRz0by-M_$PC*xt$)e;VV%OCUv@g*~Ah&8WHuFaYBaM8U4+f8QGyd!{Y zf&Qdu51^z#2RNg>kgMn8RWDyv^{3|x%_**=dMccc@bp=t*Qq}$e*s)j z*S#K8mCH(%{#2+RB>PDne)JIa8=UZ{)nSVC#{H~|Z1&fYfcswJBk5Oq6YZ)qW0u6x z%_P4TxhE1uSi-Qj#6Tj=~j_^{7yXV(bmvLl0iclXFq zPf4NDtvhMo^Uj)vlM+D{F7Z7*1H+60@2czL$jZ|w1X=S+uT^~H@ks$dl)Ci_gSyt* z##@Br7mKFO1;A~VQ8dMeCzjBSnc4b-n5Un>+Oxw$ZP9nc^SwM5x)Z^~(RIbTG_s9{ zcLj+ImAudRF+$$R+&%CP0!7GELqj6cYuQm$yS-ufSG_t#+`Q{5LA^2hM*RgBGY9U-V~n_G7C&soBJkLQ zLxNSx?r&x;X=4vrly(>}i529w3dtmFbb)SF39o9LGqKP7bsysfEBu9{ibKAA4#F#Y zr%Xe#$>tTkUu$QZ!JWS z`y(}XkH=}__!sv4#^ylk8SkeZG8BJuGfj@$`~X&ck%6%64ij58zh?6=I#mWMVK%*iCXnYJ6n5z9t1?(?LTh#D)oKO*n`@R zQ<&*+Y%&W>4`QN7DcAIL>`!NJw|SzQ`8fbEhk(nn<3-w#cC@J6Cp${QnMZGibjJ0@ z^Zh2!h%5y=;-LYUmpD6{@5x;>C}c&}PlwJiGAC43R{w(3MreNes=fnPQ|a~VzR6nC zqWn(^O+58N3W)dHkn*f;Ny&eoyg~p zSR(-*-;eMmzX|5fU7l9;Y6OZl+4QRErwD{<>U(C|GyfikDBRp((cToT@0lSrwESYQ zlRn>CAm`#A0mkm_2oG=irD=&e2-RrrOyue~q(bs<;y-LVoAM_ohF4UErLX8ZJ6CtH zv9uYs9sqFxo6TV>vF ztt(S=|Iik*zxa$P?G7g~0$PKhuan$J&ZMw!=R-dl8(}Q%_>GPOB8b2R%)6kNVV416 zkQe*Q(hxu|koM<6=#`%Gsc7mj`RU$iQorg=)Fe|%U0^0Sa(l6Sr%1U=X{&&>X<_vb z7Qp0g^E!{)84E~yR`u%L`ef)AtE+opwCl=ljP_;2-u=Kynb)JtAe8z?j@y}qR6(&C zn?;q`6UDEX+_)xp!Fe!IPfgjBlLb|!o1GB*$@2%LJ^Z~$L0%{N64hq~0KVw&kytgX zfXF>%YC6q4dT!aRqn@X$=%l?QPZ3BhmdE-tP1%TMw>XDfJ}U)(Xia1bKvI;2I&Tab zNiZFiBcr<5WJO=p;FhNZ7G}DB;P-x?@o9D7T=(Xa7L2VZOD*OxyhLhj2w0 z)lAxl!|~^Vr04tOC>BxfH{+O^MtwE)$$o6F!3kitt-LR-&r6FXIe}#J56_oQc{2O_ z-bi364eH|w4j8M5V3V3aK%C+P0qX~C+XuP~);n%wN5gB_hF~8HciO{z7 zH>5#Uc&`=SJ^vb;jCZ)V*YOj0x+4GL=a*Q5q$`>Uy=Oh*htyoq)I0=v6tZuTpB`ln zrvaWm+QV<3J}363+q;^=lO2l&sK;0Htv@byY?ADL5Rn2t`$Pm7zSU3ey(F~SQ$iV9 z^dAS;=k$HQ)sX@|_Olfky!5QU+S%?S9(M2^*VEk#uXvwU2567US??gYAAi+7Xnd3& zaMfcZng_`AjMIi57{5d7NDAmx+Eeag_kU^<;uX2X^H5__I$irP`bdrzpt!{U=+ftA z=J~BnJ2yY60CfIr{Oro*;OB)o`|%GKpT816A@TS(oARSix?7>8>zXRVTppr=b7!ni z9?2M^xQ&-fyT^~~p8ffq>bz)K9u-f-ifO&OH9YAuQx@1-1l-|7oEGQ1D|Q{56gkAV zc1vM33bd&+(WY|d{T24IQY@nR=-eg|k>{^qA46&#MAL>fh1;uC?wvicd0(;%VaATg zWju@lo0#dHY~Uzk<(Nw+J3MPKM#-Leeqe=Y$ojQ=-$~8jYRHPu;5ewn-TKT2bMCK!FS#_9Ci?Cr#}25aC@xKLNi;nZC?3Y zu-faO(}$4;-Wn=sWau}^Y24jFT}!?q$$XLJEO;95oA>^W!A5}9{AER`1H}mxUn^N7 z!#+l62*z4_dFw3d^emwCEhJCCE<{8rG>0={8O`P+cz|@XxQtpZJsqO;ShsT1E5=Yp zGO|1Zwn@Q`?P#=~wRz5mPP5(ZXgM8t%bE6t2QQx_OOyp`jW>ih{eOI-gJ-m&CfEi_ z%f0i`IGQ}InMd}Az0A-hC%Ln>(qxg$ifK4VqaNXQqJHuYup5MyXWi$>WF?NOFSY%G zX)KMKavi7Je@|ROLgL%Mh?D;z|B!6b>Q3B2;-u|!eMbunx;j5Wi_1Rzlc+xemQ%zjWGfd1;f{a05d?q zcAsIWQ;4XwE``~)lF3akc`3R(@gG`Fc+;ZEX#10rnt-7K+abQHeVeCA=&{GW6&9X& zAw9yYJ`Lt~K7;!H^l%ZFKgP~MBEPO4Y2~$wu|2L7)P$=uKlnsHt;xXjM=;VA6w3E2 zPV%EA-4o9yZ_NB2iS?x1h2W4g^N;1*?+m||7%5>eEOmeXyP zRux-HYP6uYn#aQKuLYPXl5j_|QSr6gbz47tAinY<7g`|y?}WHvgJuJ5Q4Z!nis#+5 z#{477l_^%@RvCU@L*Apk*3Jf<+c<}^H{bdB8tkY`p>)!>Xw;%MtVg0LeY>clYN9gB zh}y@i)yNMwdqa$AR~8JOxZ4y9`C#B051;;2ee+m|W@{#P#ca-UY-b^bltrKGljb`j5OPfRHZMflD|x9p>!h@J=-? zsT;;(z3{s~6UU$TZK_1)kgq-ZUw==$h2+0*YzZQX`^WG8bv0-Z-lxwJ`L^aY6FKn% zHRZzpV9`z2JJrW)6X09;3|qPx5--Wgag?RLt}8Vz693Ed{&e-P_a|p2=~zJV&Ist{ z{(SfQOF>dh%=;j=6!|W6@xQ;gU9G7GbKAEtd9kVp5bveb%jLB^y{fTmg8mv9FMnX- zeI3(}J%2@%ogG30{o}M06ajEFaj!ep1F`2G4E@tbum9@I;y^+>4+(I z7yLubFBhs#3%M&b-?paa+aIO=;VO4yz@=@{QkV-CxCD zaqoQ6q;VzS3AJT7nxWQud9xJ8vCB`+HPd-^g(Ub-t#FuPAtJ2V+0vCgcQ%DvYF($g2>h;*?J&&zf(@(uahuwI-MIhvEIvE=~)_UML`R(uHLP7D>oFj#f zfdR#x;2&qXB*MYPHR4G5M$(^$ODdU!c~)(CqF8SWjWu)8&-H;u6yX>vw?G|fd>g~j zV05KB%j%!4wbHRhY{*T+@3A*KX7J zLMxZKP)d*|Wct^wQVTPq!uv*LqJfwzikkNOSmwV7P)j_#T2)8sf6&bpI|;EpVJG_O zI7h0VT{@O(eKUp>~ zo#Uk*SidHTSR)%iIQb3CWK+FC%6xLA)v5RInYL33zmo7ni^(1dYe;^a_2?V8Kl~t| z#%vQw+qZPkYvbB;$(k(mo3@I!O}cE>VPjL)pwfI=u^v(x-ULEUzj)_fyFaqPzm6kn zexx|(%a@8it0cEn-8W?-#`=ypS^7CTAgAvJgU!N$kaew2db<9Z&m1QHU#*^ttj_Ye4ANUk`K76~S}N@@Tv6%;1c# z7%Z(U9!eQqV??3s=N*FgCVDmY1${X#1UdmML8Q>I8Bq<#f6Nafm)5SFtUZ&9Jhn3F z>un*d=L{zWw)PIqjSPN)(xE^%!g&Tt5=tXmhl42;qnHjn&5hKsgBZVUgL|E&8b5_uSp2&-kjmnXW=G zk48c(kjxF8CTce{kVHKW2@OtgS=8uY6)sL7ubf`^f@*&J23l1H=r63YapFB00ZB@8 zl8=rBY9@lJ`zJs}NsSF_8u=pEbVuES9G8nmk$y~3@Ft0%OWcle0Fh{FV<|V&-I;tn ziK8T+X}UhYuu%B0J71$W@9Xji{@1o|SH1Xv7}1nZK^6xcy|QXam_ovy#7Zrv7BhQ8 zccqiAp2k=)4er&3!!Wk!A?X2RfYHPV>lTqfFn>#@{`j$I2Mg4AJkgsOtPPvjUB`>OQ%~O&SC*V21h-9Wy zd3?1MvF66iwt`b^fgEA>z9d!BFF4J|&GdfbW6_>}nhlZf$RV<;&>HDhb8x~9CeI=C z_931Oi3ldE-x7pc`$j}9=pb>Lit%n83cP1bv18ojxan~De{{WdKpac6J`5p15;O!@ zoDeLyyF;)*AOv@JcXxuj`y#>J-CY-Vx5XV6=eOs5?|sj?_q*TxLo+)wJ>6Z^Rn`5> zQ;Vlt-IgjAy=yOIq==4rriwNvOM*=G~G!w8q<2IOXm+Tw~^oDoI)% zc1x2vlp;ZBSt4;J7W{zaq#ycQ`1XsL@H_4coJU;jZzAcf?c58d6Oo1>(bcweR0C&~ zlMaKPoKy4Xvqr`jAty+6y0#OgQj|G0*eqsydKgnAu6xo;4Y)s|((tPpy=ZRdoF{Xg z@Xq*MUyDM}^SbpLou?>y*Q#0N3|OzcaZp@TE=C1#s|H0Kjxrqzj}11HA*v1YaEJv} zSv~I8XLB*h&e2eMUKg)?x)Xs5d{)~3WQ+NWRR z6WD+BT=g0F1ZriHg=JrgJarp?i{f4;!^;Mi+&bjh*kH$Qy`in}&Io5%@08<6-1mEH zG?v*Hs;RZW65(2AWnf?-QXE7#pjkrNpruc$yT`g`Dcx_JG7{dmn!Hp;X_-4;%{V*k zxcWG?!UB{N0qXau#=4fWwJ_g4+d6#JY&RGTvcP$>W3h4LC6ZhN>0NN}zDMfU`s6=$ zWNXMi|1 zh7rLld$rb!5k_?cqmtKJjS<%_b9nhhWA&AF&k7wS4&kWOz9)ZEXGXBH9GEd zkF4ebk%Sh+KG$*wm?@CFRq?D>km745WBO`S``X@_xWIgvtBLmP^cA56HObC>7C3X5b{ssft z41zNd?+wZ6?jPb^*`{zq%@|$Mm;3e`*U+qIZi_$DyY^G5M`SzS>E;a`mWkH7)2Y2_ zOjWJrG&(6w3YoDj*z(5%%SFUyFH!PH;md2**j*Pq)y~JN}U9|C*KQS{FbNq2K zxUlznVd9FI6{>OK_{l@&zw;-2TM@g|vE_0B4n-oFB_#n-w&bgAc6*|$BnnNv*cesA zgW&_DF-^I>21~iuLQ0}Q)E`=h>2>{TBV{OC9_;ZQ_( z3xct7#?Hgfx2f0*)%Df3QAfQpVDoKEgh|VmDOv*#nmIDy{$nOtD;7Qm;HWcH$RU?g zuS|yBN~ZsjQfeHAk(loR3EqpiuDtO3O=02s?q{geIbG+zZf|8hTMq~QM|(=2UW+qn zk%fZEy`QXIyJ72(O6r952C1QdmW$@g{nOi1>-aABD97x_j{NAE>yLpen7?(YT*KR# zoTE}YMapy`1>3lM54dZp?(s@!KX!_|XMX=tMLxb_R$tMom);kSGe^7#na-Mxz6d{Y zrdvDX--r%J(j8rmU}7v> zG(?uyYWy^?H@`oB@AeR$8r3ozCrvmFpO6KR#QTvBDL1XapE{qmg0uPSin-qEhAD zsRYofN~O%nViO@?U}A4C(1l~%;1;|rA*nG^s(K~K>6-CY9r@(-V^EbLUPUV#FoKFc zy(N(a95Gb-wW!nfh25gJ{!Aj=7^TLTo`QrtliYsINc0ewFW;%6L8Zo#QeDY&aQa7F zGK&>64XRsMiRj8o#8SCvE59MdkrR1br5nX-N)xP_!*25-zwqz8GU6euF?XUyTb%hd>xe7tx08$`IOV{s#KLK@VwkasXVaH#s9TGw!@yhqGYm zGb^Luia%+2mTO)c2XFmDXU)kUH7auZfS!_EIi6ew#J<`@fkcZN3*>^?`jH_64NpThDX4)Jf4g+4bD)(Nx!9Xk5PcS1f zq~eu9LGpROM1OhT?>h7RoG0pgj~50Q=i7ZzvgG+o7h5le3#Hsz#l<5x4;Lr|ZQ&z~ zfiCBF_1`0@ukr8psUR6)WxBF%4a^ICk8EqI&7WkUF{duZd~LcgcnYz}`En?(;mY$) zE;g1qbbWOFIIo+nrm;*>znFGm1ecBXa@E4s9Nn|sX9oByOyDgYjYP>}UHi(7~1(eOIu~?Ad{~;=W_XG}Sq2$xpA$KG&{Pv_;52jty z-qkQYV*nvSe-f>aGG?A$HLS+WKlYh$Cb)KMaj#d=&S~ELD+>4{@c@HigQ$D~bE8uK zHt$F%A;6-?jB-hLAT7l~&pRq^M}%l8b1E7DZZ-{fv*Ws{Xo`uAw%?#{#z?5%r>8uX^HWDOlF&$!bMk{cHK>&_F=}k%TJ^6{S7D~D5oy|nG$y1pe zBFk<1vxCc$+j%?C{jtXK%4+ZU{Nu9y+h;S&-J5}=?Q7p{jfnJ`3d7x=6X`vjA@!xC zgx+@AV8?h&n8Ni4#IuuNI$I`trM~Tw*^uQH~sZ_ z?Al<#2WBnrJ+8SXA5g=F`y-!~^?Ld~`Ou$uCi||mgkB-#p`*bhp@&_hu`K)R!Y;_V8Iu6C}s`#f*tB+8yZ|myUWallg)qzRF{}+_fv3!fNpGH z-!@k$DRbs$IHPSYlbf$FUfJl6#?%cSrLxG!!RSxk)J24` zHOIyShHwc?)t>KQL& zAdPtHwJ&33rj%3|_x*z_Gj%$T6jd?P3Bq%jcfHxSvBEg!0A^}*LpH|aA0~wQ(RDT~ zMS1H(T<-TfO+p^R7gJPaM>v1HYW!BkzdPT?HrullT^ES9QR5)`2s4}ly~v8Ldx}3t z<#M^|D0FI}4)W8{o71V)mcDaVT>$FyQ3ck%oQg#l=gmR4$c6$FPsnE4nTQy0}BJRCcOtj&xfzKRomZD7I?nUc}! z!t32?uhxETS*l(i;W)B)buqjt4V;PNFXPu={>8A-SF`PjfRZWGZS7&Ia^$!(XM~_$ zKu3A6w{*E@ON)q{MhI$tf35vKfMdhAOQ|&zxfR}ucc3*oPDstP{(C#(JPBaU1~Jmc zABiFs(=BZIU4#A)?QtXsJwjMSqAbqF%|?5&?iJ@^`_jQ-%ETMM3i%;0J(p3`>30)_)e#4veaFBOJ%(9qbwF>g$lpE zKUgGxZ7M18s=k_UBnaStnP`yDznJFglRFZDbFZ~R^u50~43rdD;y~L*Kczu*j4E!6 zF_<@~HfURI^^jYdc97*{$y2VtF-GJbrEah3V%Mr-e)C5*3GdHnEAZo4|2EuSS-5d` z_~i8g!ANM^^0`J<#5OHz`O#F!{xw@``|h7zIg2WXlVz;sJtp3Z>!+>WIN7<_TOeeA z{YcEH0w^^MJl3Dy8iB;^@`tV(k%GWMVbpHFobn`b^Ca<7%6kUlSY)*PcII77w|ESi z#|!0bW%8oRjths0#(`P74a-!@(Ls+pzI5kXT!($?Wp^E78AQB17r1J~_rc1BCy zcI=4%bM;Vfn~0JqEMnh{?em3q>&x8A)*pCvlw$1z&XLPcS{m-#B)ozA8Cy06bo$G% zXgiG(BbSr)My#kbE*?4xB|;9OMz79lpw5MVM)fe3idiX7M%(5fsI$5oS$OP zoh$|YO=zT0-Jp?lewCaH8vPgY5hM}-PTUpau~WD(+VTqolT>Lr^bRF>L(evTD_m(0swNJGNB%| z7E*sVHZoFOj`BA=%52Rs_|^v!rR15;pQg`u-52U@DDbUt$0iB_H#69i9NkH7Z@Z8! zcJ=yM;ue$P>zBq%lz2XfTVMwEjVu1J{^LLP+x}4r<9kwhA;Cb|iT>c77aqPhKa1|x zlV#&Vy#?;LOq+{MCqg=L!ABk)7~8}$Y0YYU*>=F0^Ob56S0qm$8ii@MM|#~O^EEc& zctuhbT@5YVS)b~Z7(@ti(Gw%;qK=X;PRbw&s-@H{mI$OidhWtc}fS#!>Ji z$ZF;1Cw8w?=|E5zgDo5lhAhFX5M|&)4rik~3O>XQ1_b}`K?i+xhg+{?vFY#woVf}s zikf4!XjBe%UZ+!s-|^+uOf zXxLnl1NAgFQ8J*}=_;wSJRt8&b=R^@m@#_dxh_8R=%r}GkM@4@&Q6VBL zXVZue{@j*s6V!MM0~fTVEIaNOM0|3DJ&nz991XSp4UIpn`?X?GT}>xzQ5<{bYx+=d z*r1DVE^hcpuDVlNPQ;hLf{^RWRfNqBz(s8oj>TP@&sOy80zDt1j#uJ9&9qbcOD=DF2n{?O)pWBt{XJ(*5Pmp26IXT@(`=3sO;7$hgq;HYVtV zn0JH*OIvTJhIoqb{6uq26dcNs0`zV3@l#^7)20}jm_2p zRYqKr3-!sb@d^RZt+PzJv#ahjR54Vym?lpYUibM*$BG~XN>txEFJ+<*LcV(VpUsqY|vL-);L~RZnMd*u`a7W^NmLzh%>O#E;z@&7lGf?s!483 zq%nR3`qA~U!r1S{3BxPHPLJ1vWL?HLaxp&cY{OO2p|a8+FN+9E&o1MgsN5kbTKZM)c(r*{`}BobFx$l-ynqS^;bwKk-pz$F|+n>Q5<<-(!h z4ZOME*9M0Lq?K;zr5TUR+t`%E);)82Fi~js@!^vQYn~5W`QF;Uw!V0BRYbXjjv}7r z3?Ko|E62iUY~il7Qu~h?MivkR4QL5@ID#}kskpl3;Xs4>e*QOb2-Dy-juf8mPJ6~o zVk5pj9{a?k` zjg-U5TGFtbBFx|@=jhgNd)dX8xl)EilpfyHguEnMjtcWT?LHBqc~5IEH7#ZZ&?%RArWf!TYREkfD& zRe=4h$_`Y!_!JlAa&vEbT)yzzov1OPaOFd-NqiUn^XWz#bhqpb*quIUIjHvhk=U#5 zz4a;)^1*yy?ZLCF%^`)^ZAy1|K?V6i55RRhj`(7(DWQcJXHQ@Q?-N)i z`+mvS!TaSE%h*XZXPkGosZ8$r7Cml&mXnoW6REPs`QpJ44L-hDg6$rp*?uKcz3nVl-$~ zZzoGrn9ZAizm#33S55odKkVR$8yZz~OpC4tS3s7HPTa;88k`42$xV7SR>R}@mZ_Qp z0J^|4=R+hehw}=>MQG}bEVY}Z4%1gwWG^lyjpHthyx{M*m*02@aK!fWg3r(jYV`&x z>0PUdKZLf(J)XXkrX6xsl-<8F92pE1Q&94}UJ2Ql|e!OxY!a0h+Bg-D00kavW!7%*T;^C@)RB8Pau9SLwAr?8!r2kExW~CBvqXDKuRL^9|_TK zI~Bvf>VA=b2i_*VC zLMFKb6sDX9tKYe5QQ-djHa?L{_6*7!FDbnokpv+svaeb|uNqM$rApZWl>NE+Ek9 zS+0VsZAGtTeM(NZ=}MxMOCZ){V^_V3xdTQ#ZiK<2Rw7}T_Xni`CiwApyWiL0?d-X{ zSp$bAltg=uM*MYtI}s`DGZ}0A%CSBs_SyVW|u3!ZofB|^aZZIZC~#1y*!#0F%iydruE*FuP9~T@i3cQ`K|aw z+854#rfucr8d&iK0={vd@9l21gzv$}Bos%-$@4ON_VJ`wrY8uS{mii}BGlPIT4Hys zg6(Ck)4?$OTNEAoFb*(tDj;B)F>S2R6ZL$3JSHK>kCWl+7X}W>EKl|{kxFqTisd-0 zbXOr=SXZt^KwW&dHJ0A{yjHqc3y|C;94|xVBFz>EvOO}A&lv5^qh1=YXwkPGq|=lOxU8|jta}Am*7(>Emy{@TIzXT< z3Mu7)h>#Ch7z$_y>p?%<_|l6ZtSd(X51WY|5-eeZl_HJa*dZRp5)=UA`p31S(klu;w; zB3Xru=d0`~pJ7lEy zBtBs5DCo9G@@MEk3sUGLF{erG!Na~TH@&M_l%DWn_)7b3m@{pZq{~${WegJ=8+|{9 zT~2`+@S*<<-e?f*N!&PQC}0uuPlxW0z0Ox9m}Kd`t2Lc#E;6!!s~hJRV`5-Ty6yCj z6Ihixv)zlw?=+_I3Ampq>}~e`-SrQ*nLJ^_J>5gY0cprQV-pAhi#$s{sR)6n&g}B4+Iz(=OtuW zchsvnIetFC88&E3MTGtA621kK$9Mbr`DmfQ^$Dz`N|V5LxM6(m)uTGF*BMx}VAf|& z^Pe)7GWiOBfA1@hV!;lr`i+`mw!%a^!zc+%*L`HI3Y-@?2_~c|xYJu)I2sK9TNL*7 z7@w0C6rb!*Y_ zXWQdH&`|84;6Y;1y2Co}=@(O+C%e+d@0GhM?ZomjsO7m<%x%S4Bwy^f-uI(P3;y)7 zfc3j1{URtQCx`w>a7WCW^#>$$R|WrWSQ6GJCr9|hrN50N@!K*7BU=pV=zcDVii++W z;jiz;IT4TS{}fg-;=qgeeD>0Qd+_-2-FG{Lw@n~utEJD|NyGeZo61~;Hs0E6p_jV; z)k%)7<4E*9{Mw1$Jmy;HU%R-iY{2kB50OuCqexcGMSRM0&7qMe)-XEv!*weT@{jV= zrtyl5VEBKjF@L&iK&k#>vPc;L*CFa((8qrqu!hhz)qf#InRR5o{~)I$g_G4H%1mGR z7ufOd3ol#L>deR=(T??@Rg|h-5n@q)oBCr~qZ0%LpnKaUj;3ow+P*b+Lgmt8Q1K&- z{m1%2@L=9-jj}b=%%K{q;lV)1A}P(v3{5a^I3Xmm99GaF0vtH87tRNCFONoLd173u%w~sKhyZ~3RNWq8WvHzV*093{ zP3i}TfBV^H=GfiJ&xw5;{m=Wq(U-9rQ_uI+A#mo!iWN-_<3h_uJ%ML_A9J8qurK8q z{`bz_o^F8nPx?}g%I*L71Ai5ob~0Z&27I@^I}8nZGzwI&>k8ej!j7!s%j;u42a$y z@sIu=P-ZAXSmmAy$>qqOp7pFXgpw$u`*gZ^T%7GhIIMfDwOJpn{DZ(nr@*UB#&4d* z6I7X2Z^>Q!sppWGS}s1WJr#sU=kmQHQSB;@5#sm>D7qJZ1cVzj#Ds;8fi0u!rThPp ztuIF_Lo<%fnReP!;XJt2O~^dzIVO`$@7K<>bQ+e!oJYPp@l61oltFrs{B_E;5v9LB z#e`GIfpWm#)E?RKJU=`pKip!Vnj7MNypOByohciUJuoF086zRvxugaDpB@ zOMX_q-s_K+P3FGh;Z?{wjc}eWo38t|EHX5bY2I^YNVA4GZfGitM^v}X~cjivU2sIC5njA3BtOm2p zmRt7M$U-Lw`vyYWZQng}tKREwsu&drI%rtCUE>O$##EIqB-ytMihdg(X?2(VlG0%y z$Hg4W*z?YbS4e(HZnqS2Cb`gcj2oBCr;kx)wJ+G=8+E=g2;FE|Yp_5Of6ic{k-w)p zs9jF9PJ!$(E@`^@f>-nkL_4TT7cz!JnY;&#ng%2PiOP~##Dvk+#i0a7y|0igxs9;3 zRs(54LRwhrOAVA=(*B*FS#>Xh_%(~OAy%Z&A5Y&Zi^3q}!5tZbb;$q4GW5~KWW+@N zAj=|?y1QUzYW7mWC~uFRD3e#gU^g4X@{=Z7ZHPi}YIupTIWqou7AG7WgmLug1xJRc zYo>{=<7~b}nVALR*jVBF$OE~71~QOUly+Ak}eQI_0%7v0J{p%ou zwKI@sW9x=_y@fP59w`gAjOPXnfji5U8Z4XHF=&*&LA6QJZAhAZ);I{rA2XTzgFAN9 zu)!@K+-77pb7B3#n_J4*2V1BX^+T3rfq@Z3paR9%-*ggl*t(nWeE|A=;A?^L_{N1x z=QYEM6$g&H>1sZb!~2djo}M!jOZ!7n;Mpfa``P8lrxAx>Pj}}xYlh8f|O{ZS+3iesS?sC;x9Qh<{Q1yt^p5-rlqAwP?Sy zPh==y+#fk4e!0TyIdwfr2hWO7g`XzbZTc4f1_58K`4M$UUkkllww{v;C0HX{Vy730 zYshQRlL~Mj&UKLwG>1;nlxsRFF?_H!126Xn*X3S7Xm`t@w4()SljL3$7eW!GxuznP zHaag`j)_)Wkw?YcqygyQhdztsj~pv4d4uB-b-%9PQSzPUWfTc z_p3k?l-zrM^A(yGw~DifJ~@JlT!(ot)xi5mkigqFr|l2T|FdBv0gaoAzq}4{nu1E= zNF&uvY?oQ}UO1Dt&pf(2M$)p;n&3Q5QME1B$2u$p!^WTTMbP8ff3o_tV6>DWzplTX z?1rXtWgqi<&;EAeSiDoU`+zyPkuDkgN0ffW8xyfGb|ITN;U)u%cO>QA3}1w{lMfh8 z(|$2XOchFwtF&yLl&JA-|GG6i-bg+`*X%yy0$-G9mca=&*`se0KQb`3$vAb1DuMo& zNTP2U6}+TXuv=4jY%#u|+XpDBM{ZC%Z+v*yg+tIuGC=pOxA!8AB7xQXV?aCZ0=;^c zf6j`2JS2oX|BQ*<(=gw$qw8C(DT2ToGm?C3@2uBo-6xUfmSjM&Qx7-?wxu4(;SZ_tz3EfY7ZoMsyCjAs5 zIWbS&9!TdXA;PdfJGRGS(5~i1VjKU%Eb?z@BC)b6_j9O!YC{b1C(e7FZ1S)*;L*n7 z?STX3p^K33t&ZI3Ja^zQKagZT*^_&#YdbAo9_L3bw$dc)`sXw2tB@Di*W2yi5w2__ z6!x|nd}u&c|c!4inOs={?H!1gG|z^c!~7k!=(!^5e!s4ob)S}DAgM*BHdW! zOX@WhRx0n7$Rf-W(wW?jS?YFYm1~3%T|NoVmAf1bE04c>vED1y76Rs4U(JCBr2yix4(bC0kVv;&oney66FGczO zqGYznJW7h+d28R-Tqz0&B+k}s&SfoYH8TA0&mLhYROF7&mnloL9eYmqDpDL96mW`8 zZE82)w1X7kU&`ygu$c$R-Um&1Pf4Hh-Lp$UKa+z5==)H z7OUAGOPRa{w-1fOige@COB1(#15kSIG*4U}sIXc7H7kPi$Kk!issp9^?$6E^>TYO` zVQ?wmBdZU{4|(q$6=>X{TKy;-EavL^FrvmG^k3ulFWtc#+bz}`KZ54& zm;uyU4YP5Wui;B)#D;fxPm2*>O&#CQSBZ9L(Q!Y=^isNC^$Z{_##&~Jy3j%b<1c#X zYsieDx=^C6;fJ8H+>7i}0XfyVUNAmV8mUY#Uj3EVzl3FdRiR-mECy_cyR%$RrqTY9 z8+gLArhdoa)o_WcCH)jChtv_P$P|E{>T+G>QkM2_Hc!^`sp}-D8rB$}Sw-&b#iqG` zJEHh^j)KXrUS2;MbFcV1hTYS$HNW6q-I?$bIp;>bZ(+IQuHT<8dFoYSx9sp2|AcM3 z9`{qrgJ60t!Nc}vJ>y4V4egG>w-M$iwKteh&23`YvhrEN;vQFy!})d9T%s#b8F*iE zs=s@elze_;vUk>Qb05C(s15Gv;O{-E>43^I74ivTS3d7 zCyDLm9&JA^Tvp~C7TfyI&9@MJ#Qpgn*-YOMMS&hy{)7+{vF_IZiM_m_@bSfp_e28F z)5l;i;vFpB4Q}~<&tqMdA@KPr6$bh+;=@V(p>=XOSkB(1jHh$?IdM z6%_BwL1RyjwJ7%}eWLIR3U~D&%O%IoqD1|MjDAhU-5NK?=Dk`|eBr7*^t$c_B$E2J zGE!7=DyL;rUaNJ_U}d!x;h#tK>Ghaf3N#){41$lS+3}ftKU7UaWUUDhS8@srb!cP& zuix^DA4;`+qL+E(?3Vm;uS;dESyyw=zHDIg8Ha`yIy}^a2)*nn`wB)mhmZNol!<3P zh|c@PP8SrGUY}(4H>`ggFUs+j*HSv1cC)s`l9qqX=Z`{4y<^Mnv2M009)#EmmCT)7 zY;-B<0-b0N+K@k=;5Pxa!njU@LUxlemIsuu*#>0IkQ0B~Y&>qkr?N@pE+IgHk&E zOBARghNQuH>-a!EK4HlN4^TUUILW(#ea1F5;MxAn;jG7}7@Rx0H|=&KR3_&z#ukFa z9pKb=35<*9@xTYXt^1HK8JC|q~!WKgNZ#pQ3L+6>Uux?%gdO_ zItiwEOiHNT`fbw>n|BH2-;a&MJl#98QX=b5c=Y%T#co_RYz@X|wn8n@>lyKf^kg!d zPgw9lcFXSm)CdWNNI;=(1>9txNFG&W4v%vfsh{<09)w5RrHksaxjJs!%LKo zg$k7IhH0c$q#h>Z&j^6ECdeyB1+H7b7M3a$*~H&FU?6m!IOQtE-+MqyyVtjiC)?J& zjjLAF)WaT`>99W!O3x!}DjqcyMMZ!2fa4fpWQ?0An4A+veyeSs-;*Kx$kRcfe_SeJ zu&@mOs-Y8YZ0n7(qsGRFBn}MkQXf2hQXFi@BjM}ICSv>jE&jUVvD3D~2y{%THL0<7 z4cnuTR;>QIB`?lSyhzN}S0t`6YpUqi=GR$z`iTuZ)q~P3jj+Z}P0xs#EKsq0iTNL- zx}0N*FeHuCUU)B`Vs6$jBx>sKP_cA@F&1qC#j$ka0~F$u?w`DT2Tp_(M?9hJV1q~u zd!(krt{wfBr<>pR@{PN^`~|ES+;D2K)dDwmDi%%g@1Pd1D1y_MzZZNLbu9uZ2idli z6n=eevh^kQEkk~bh+p;OunRF)uQQb$I>-Uao#kns@;i!rKOvv5Bo}^Xt8Y?d?Wj1g zd|t3}5Yb`)A2IT^KJ$yEeev8Fl$QOdfA{hkh2q@At_FT$hvJJ%68VfC2Ev2^25#qx ztvU01I1^?nm^HL5wG|6k){=S6-R^^<9iW+7TL*NEO=b+oULx;!V=9Qs$^mM(s$r5i zf^m|R=M)W+8XG(^b$;jNo5zbS%LO-PcmRc!a9dw?@I!tC5^IraPh;nNh{ufy0AOMt z@s@7;O<8%2T0i`f1_Q5E1H}+-)~c~_)Iix{-Eh&Yv2h4ed06NN#J#wIcV^R>8#<(T z$3Gp$p!0C~J3W3^>6%7pn!$iG3BUjFcG||hLq+Y^t73l={ctF(-2l&j z+n#&!XxG~O2ED1COz1yJ4jMyLO0oaj-1f4OOQQZo$WGj1B7RE}d< zKqIFn4(a+@-))1q&W!15GLY(PtXxGqDAC)^8(iPI0%+@)i!kF|Jt0-OgJ{x5p=~EI zzWqO5v$3l2I63h(#0$dTTP$+OC5!7uQEBr`&-_wq3mYKr2!DDD1CxS++W z1{YyR$EYCGc5jHdbY231)Bl8^5 zxMuHc-EFzb5lgZPEMvf-5J>-r;ghyEkglb8M8!duSy)70Z$G1R1q)c^`3mhFkC98J zkKiBqGy(l{8CRp%AeD4pzjL>f3Fin>Q&UIW*A>yjoS|L^rNhW5?cK7D7jK{}<n2Wm)@^D616Vem+C&9>2*J6s~(LEm}7}Q~XOCEg{th_bo|0M?f*$VZ~(j5gus9 zRG-m+Zu5T8z0xrRd=H4d4>QwUX>}+q4;<+vE`9u!T-OYv2#_u~6~TE}QLi<9csq2q zJ7-=Xyic6Zm>p%;^@kItIH9csaCN|x1~D#xrYSmJ^F|C{dit%5GrJ2T1@uY`_wkkX zbj7d-(tOsH;Z~x=d7+7E0u5w)ar@szeRwD9^1EodK>z+gb&jx_qZm0}(7_BA7r**F zE~acRR#b=|2KqbE6bq>_@`r3+K%^+>+uk^y|4RvG{@7|*nLDrKc0o<$?+7RuM#cYu zS@*>vkY`#Y{`w79^ViH&){v$eovnaAg9HU&c`U&kgKH9j`WVQy?iFF8L7|dpt(#jO zGgpt{dF`$#opRbm|2t9FO{^y}go1IigYI~c8Tk0Z`i+VGKu(x|S|P)kAJ3#<%bFCN zi|gSsg&?H+?bbWmwxV4u?2kYV>qsO9UY}2<+bHf?tk;yJO$BElAA1`4$!FaMvC%pd zxrqk0tc9yVrrpif1^4sV69q>8a}1;cw1BfIl@}ojfq0+xqPo+z6$O%?Rr*UQ-(}`q z0viM>h$F1P6+!=p=6k_nhIbGON#nDmfm@~eZJ~hKN~*nA92SGmy32|BC;TboSKEvX zuFc$Bg0gqX4JynZ=J=H};$hG+hmwabzVgT~V5f{$J~*y;GHiy1V3o)ZaTa$^{vI`V z28<9B313^fFFxfi3F7L2w8nCd-O!H5MF36VLTz8(y?kwO+ESf&d<)_;A~U~1!~uTa z@w*{_27z83L4+-3fOHhieL)1sl)hR<_d_Xf`b$x%1b+8a_EUc{pDWiLfKU@i>3Raaq)|G;qOlJatinEbZ;;tvn6Vq~Y?4JK zWLK^5P{%U*|4Pc*tc-5)X5jHsX!046nM=hrfDII~W@4RliGkqk3+Lc)c`=hditEjk zB%__|hSi0sw)3I=HREK zLv+ScqmH>*#;!*{HSPKIT4kiMYC^w)hv?(lGu;a+(GuOZJDCY$@h}jGU$v*tkk$nm zKBog<_$H^f=_Mvi8Z)xa*!`-a`aZ6=~=dL>+oc zqVvojk#MDL>^XM!i1MwT^FKKg|9d59_~Yw)BD!0+=pOG}p>R`L#u- zqTgN5YImWXx&TOijO`H7otKYpRL5Bl_hCGVXV4B(p^>nvcV~MJ=PPtKH18X76Z}P#HI*rC!^Eq@0CDGGve=!UKi*smA#z6 zz5eRc4&EO1V8?bE|Lk?APdnV6K%{x2QJnN31BdI;@O>^R1zf78dn}Oh#{}s}o;_OG z%kW35&$yt$YkL5DU9@wz-H@VzLgzuHjy&u!fyOO5pZk-*R??0RgEXn@`RettoOf8< zNQ1m_V;GOM|5KcAxY{+Yd;5JNACPu?$YdK{y-6d)e))YUKf0GRI0Fo2YCES?tz8w{ zDI5wvRkXIi;>&KcB6nW=6n;02T;HTGnVKSD-6L8u5&wToAXXkfFt6#bY!+++BXsZQ zG)q%3f9oj=O@1nbq31!A2UJgCcF7M7NlSe1jIK-O5GY$#>YZSS`J-~Dx7bog!!RSq zly<9M{DIU;1yd};IWIIuq&s4i87suEoTwfsm!+nth}OJDp@UBy=&{ zAwPzfugyv-#_6v{V1p9@Dsh%zJ?$H3FW_)1ye}_@6|DL#%?4|Y^Rm|H$rn6L1Q_U` z)^4cU@vQf?JtH!^2)@a!VYwd(k0DI?`To}?<`Z-;yrhRTgl%z?VB>Xsp{T9^C zB00E8;vf2}L&5QXxK3XNZcwW_ui7_>LGwz)nI7)tOW<-brN<-pGB-fI|rX7FNc+70`ru5^>kA8DF z>GFi2WuFx??RYrU)y6G*(PK%NMgI*hlGbmd}-Cx+!FUvx@z_>ImQIC>f5_|*QQ*3b?C>? zd-pBqCUnWR?f-};w)(+JRY}F@=)vV*GgNb@T505XD}2l>8dnblnrJ_M*w z{ndTa7re|66baW)8H4ubtOPk96`mCiD-zT97a2l85K5;^_c|bb(gn@Q*AtuU1RHSf z&GN!zI;IqEsTxweM@=$eB>}*jJW)QSkWLj%!N<8lNvc}{CE;d@L3-7NJiSnpYW^S2Mo*EMcu8xn@5#k zSN9Ma;+ct|_jDco@`3eYLvNir|A{b+gw3wrKro%eBFGGw{#=LQ4-=eQabXiN)OA&d zAJ~Y>b`>HbE+j4~<0VP1Rv;Sy&Pcu;+4OW&?2hT5Cd!|!- z9z!OK4H3dLb28|oSSev;aa&#)J2f_W8^KPl_hzE3hh?hN8fZ_VyZk~oy-NKb0q5Tl zX-L^8d;xGfFZv&s&?ZM1lBs+tZ-uy4 zFNzg9&Lq*BsXv%rc~Mzy&;h_mdZ{H5as9Ul2&|JUtxul}jX)VCzt48l|LizvN#Y*B zv7r_hC$l6nGnkV}DP8ooWK)X^azebFQ?xbPqde_lb8}=v>WlLqaX}MXhnZ)(uI3T8E{?EdqfD9; z6W(A!Eq>6zUYEQFJR%Hi>c063{rT~aEM~&+i#5>rVx(%@?TC{xn)v4p@(rJt%{V! zdzGOAMG&EmyUwAs6_^LWs9ee-2X^<_7J+`qL9WU}8@0)<>o=7Fxd`m&23)pVsf8ZP zPHgChS>bcg{)D}srj~}Pmu-sYScT9*oj?8^#Q)={^AqiU73XioHb4CxBGZo)hGK7Y z+;v&wj{v7@0zZL(*a=F9ixTBb-4yXC1G3kQg=A)Kv8moDge@+V!{8fS#HpF_;4yzP z;W~ptgU$T_2$+5ro-MEjBv^%3M13nQ9L#gl35(w6KseSnoZH=NM#=OoRrsxzza@vwhOCr?0O(1h zI0W~uMd5HnJneMHP@mV4)w6c>?LFe+O@NY=BOS2?x=Wv0ZAtXO_CCvO?CIH1tLG`6 z@rgVqQHD^+nDq3jsGQEAWh4ucOQ@yBs%GHi=(Qc_^sK--3|hmtd=~%mj2L8K%S%s+ zR9^=+Bbm^>_63g9==UD;^uv~0ZU83z0sGS8hrPI|`eOWs|5o%Mx>l1v;fAhu&*&Y= zeO>57sHlp!AsNHkO5U z+lKkDlccW)n~{9;){&%M-eufa(E0tyq6wuljo(`c@aj%}sxnVSQ!&#Hy~Mo@65LmM z_$Qo9w@H#9Jz%Ed;Kp!8Q9*(!mrFdaM>oedez1SG*_UcpP!36Q~I-2f?M zJXr}XoeAf?X=FUfVwxwINZ5&}hxOkAxIs3ug({L-)by9ZZ5aZNQL&B2H$K*b{+(AJ zfZ+h+pi?uTL_n;WH#5a{(6IbyebQ(j8QSqUu8(P`@Ae9jk_Q9lWz#5e0Dm$enLD{A z*}Uv~`%ZF3&mU|$9NuH$!b-fKM08IZ)~4_Am>k|!0dm0{iV?U;n}n=I8Lt-&%+~h? z4TKfyqr#+4lhSPf#$1=X;ABaJU)>`OCB8<>99XB^v*HjL*rVG5(tWG&hP-X4jN2^ujJeGgxe!~{?!lKJE6U&aLup?|_GH){rGXY#R7 zf(Ht2)*1?=j<=|OS9@uGc(XoS3>-q81O()(O3Y$ePZ+%)UX=^_I>%)71G^t|Omxar zlT+Kmgy$}%_Lw?Go>LtwS<(5o*mX|xssskQ;_7lOR%-IPT~e2x_sg8VWsa@N6G10Q zy`QtW?7EV)`6!j161*Nm#ZGigPzQ$Hy|8^Am)fGmCU0hZ4G0L}BZ`IvYg~z3TP(XD zo@YqQBO0GAweG~RMzlV{AQVm8Y={)tKWXRXGU4$e1ZgN9DUN6B2Pv|6?Qgr-%LXSV zq9+;GhF?^H>~#O^Bft76;Kr1oy5A;ZbBJ1;c;cNRLg0}SJyZ4?9a&@Aq{QU z15zzP8*7)ec-35dy)OJ*L;?X;AWX_}GP#YgS#6Yvrje&@r9gHNkkO9G%%xIwg zR(P-8j9JL|>rYofn*RWK|8apfV)N}|AH8d13oV2o3_fxg9uE%$yF+A0=4kJrSg`wD z$)gm_B#?iCaoW$}K2gWnKx;kII3@dco1MQ${FSv+_-zuzB68aIZZkc3WgX&xPWG(# zvo>y#h5iQBO7)uyi+tazT%IFaybx_IHtTt)`s*6~fA0ezTv5d)3<}AwY3mAGcKQ&5 zuopCQW|O&eN>^`qiD!PWloZnk4dO&j`@8}lcKB34o+`_gf8MbFW#{bOIll%B0A}>3 zISO=Q@)hIKYh3zQ2v`Z{L3o|QkG{&i@Tcd*t%#2q@58rxo0$Lg<9}Wl-brD~zWho$ zyqoGY7-(inWr2%HHKL=1?5Q_>l!+BYePQ--ZuD=cS+fFpb_+AaH3H2^t}hsOyz2x{ z%)b2OE2;k{ObQy9Ga!vp7?M$D`jLXd)UAZzY1exF<&B3ZjX?gsuw_u*>za?O6Zw|X zOnHg=G3hU!{=cq{Gc1i(7_vF+@}*<9 zWs|bO7bpRF;x+%E?f-fjIIeO5gXP*+E_oO{K+`A6U}H@~x1CsUYEzf@ za!^ZPh-?sZOJEG@HjgAuu@C{0Yg8aL>ho&K6wEZ1y{+b=KxDU#MXhLqDB`zNwwxa) zE{oPh-s=a6{uP%^Wz??mPk$z4-_q*S>brj&<@YFttnA#Yw}W`+ zRR>E{;|6qEI&XZ4bv&akb#;FYosgBYE6}G-@ZgQBQtC&zS_ujN?12$|a>MAg;_TsT^adJ+_HfY`)FZyS4ru<7*L={n_RFV}W3GeR%?f_wCItUZB> z#(oBm*SEAS>Uzb!UQ2SnoC>s+C4+Q}VEL2w0%H=8Cc=(O%9{Y2EvG(*u%XTrL`pYJ z;Ppexrm|5KJ8Q~rLwM`ab>T-peOL1Jg$P_$@MsVVZ0b@V#EyUFU^FY-`Rq+Ma%$>D z`~yh%caH$ySK|vZk&urMoLr&>TOqn+N2L4cZH1 z2x>H%SIKIjF2eH>$!-T|YBm-qgXl+gn?{76d6>~Ft_$CMKTHm7tz}DO11*gd@M$v= zl;dAG7%y!LC%+Ttt7~PK?iF(?YE72PYK{;V6k`dzpRV$O6$EqGcgxa%sE3_6T>fmT zEm9Ba)o<53tc!|Hu?r0+qv02?39%XDCdOCRSMc4s7v;b?Iu#VSOnkw0?g=SO_O@ zK;E=xk~+cz>m-M5-Tls)gSeeJPk6(TKzbuZ##I7uMtnj~QL{nXM3spT^E^S;{r&S3tGV?oa7MC6&}JeY>hOBctZ5o_jr zZY-@XHITyjj&tAZh-ZN1d5Zd!IKp(uGbNeuAbMr*xfB&IZ73)R?e@$B%#tWYx7=iD zBr;hFQ-s(nj-Ba2)b~Bqe!up3m`cr~_&gog_k}v|wBfw*s|r>|GVIju@aQ4tf$(*w2WG^U@Mz1x*SjE;jGkG0dnel> zJ*F|En>0tYmUhNr;RVJp#o=K-2VJNTux#?UA4~$H)7}T|5+K%W{84Zp8K9@}4rQ3Q z=Qi5(dt)K3Pc|jnzM{Zdu3}A~pkvA~3%pIUL`$hOcx`lJfAn4Yv6msqNB#1x>Sjxq zcmZ6Wb{gr zBq{r!=Zx#ex!H<-&lTB(m4}_vgR6JKtG)3yJfcQe`2?7q@~sW##=*RC&lSkc`=jwE zJex+)P>mQmU~YWt_zkK3&>C7yZQge`Pd}z7JR@(-uGU?4Tw1->4f!^&l+9Uyvu&MT ztLE}|7x5f_sdJcK`P^2QUj+g=KVR?l#U;VI{z#y-_y6mHrnepQVH+hzn}pIPl$+|q z5mrG@qz@72IW4CdGtkVbz4*y%>jWVLR^Gbp%+e+rwsG(Q#G$X_=$1WB>{L!uGfwXN z_^(8Y1CkR#Pf2l!NVdV?AJ&BQ$}2=$-_V7lh`xw48JU=j8`E-U?VXgXXQy1L`|6{r zvSDuXK(OlcEN0gd7OCiJ^X{2lC>m!bq)=lk1I$8x*lI4}E7+3`Op^*mp*iT8k{8qv zfrro5sRF{+p3PiSGyLyZg;A4rJCTBbBRl3^U60_BI_qQ%?B72G^}A5}7xN|H?28oy z$*(z4`HvX(1#o^Hjk1qT@c7qD3G3#5pTX$g5C=-IK3?V{1@!zn{UF)V@s9i~>1$%1 zNC=9Z-h1dGjvf38J~6tpX*qy?MUJYW&|@>k&j6arq=|SOX4f0&d1|^+wE1tnfGGy^eN7w&(5~B z6skFhH7o_yN>mWZ!eT2 z?(YsieYk^@<&-Mhx};hlRuUQdlU0bVI< zH{12+d&rse0b92=7 z8B>K5KzJ>=B1()ykPRyRhyp3HO}GcG})h4N)mZXZ=$_?4aZ#ua)%o$Mn!L8Q)8@s&%BmS}R-ldu>_i@9&zSl-`W>S#$e% zN2U3*J}`|iq0Sop&WVBms&&u%j?#!0jd4bmGgJ#4o!1H!ttTT7&?Aj|o zuY; z3Ctt6Xi(drN44AS!-!HWRlOBBcGy?lQuKPa%1fU~bRQ{`X4hGRT&4txtqQn>rpcWlCyU2|s596R^xh)$lk2%yx7*K8h_by>Or9 zP48B&aLRmB>a8J!8)bI%!FS%z0QOf#5siF9hLOKu znaYdCLBLS4fkr1`T)PS4do=sb7Yp4wJ4*ru+wi#mArC`r|C2ndJkk`x>D76QXm0PB zEHkLd&7&c6g5`k;b5+mV_h+M54FA{T23`Ke&RJeTwCLkn990(0_FH>dR>^MUFc+<85@(XR+5*xdj zftM9(2^@+3FD97JH6c#?y#w!=yv{%wb4?V)nBvhLbON6}%5t@AWbm{Hka9!kksj+y zI>(0rGo(=|$lwZApu+DdpGf=e^81k}h2qi}Hf8Z9w>9NAr*=&clpoadYylSbbWO?7&tN(^=SDoT=X`r_OMU&#H~;q3czbHk$uGgUpWP1d{!#O;*Ao#t zq_cJSCcFR$^X$dEuC1eK(@WxoHgqD0m5r`8lpl8?a6xAD{FTZ7PJAWpP>l`Ziymz} z?wis`t4`bPjDK3tZaS$|&*2+*w^H{I;=8|3edudRG3sA2U5RMnv>-&iH}7~nu~fQg zfou7V4ln&)u2=d**x^Uv>mH*CDjIJ8c!2}Z2Y5;c!r9~qA&{YEAQihe@BXOzwMG9g z3OFY?D&`ywDsyLTfz>__6M?|TM2Q#1(5pt_BX@L=@cwEeEuJTWq{Tr4+*v3#Q9 zki_5pOK@B-jW;nrCG?hKwyZD*U@nHs2`@R>GyoF#e_Kf(0g14)ZZ`TGg#L>xS9Vx)sBO)ju_wpx!<05nU`QYj7M|b;@1#T zl2QW+6*LZ~PSO{x$3xwpSQ~ubKliSbEGkc)q*n9m!^7PV5kHtlY$w`{ zU5DpRn;9x;EQpxN1TF9qBf<$XOGEgRPSrzuU%?3QIi{u&`*ttfNa6(zaQ!|$JPEa_ z&jKUGa%xjQ0h_+L3bjhfjaFppPby;@MRn)n{ilzL&;{HKkU|WqAm}HY;VA2B+fOLn zSPs*hSxO07RDtAql1uzyAug^Z2-J})dz`qC)%9A))*-4hb3N}pNkZdnD{i z{r}2|ww#8+4CbR5uTsCJv0JL7$*L77is$_oB5~L{0N5{)aL<+}A6QODqu;ZiTad$YJQIxeCWPt# zL>%LD+jfTe4^Oae*QF!e^99A|(ho}9)bu(D$^A}8*}uLR>_r}0ZFJUggpkjNWlg@4 z(%q0zZs9As@+-Kq!~FOr)5YGlnEt`AkcvG1w~b5JVG!vRePe`YD+9oos)v186AbFe z8;H4CLz^coZFCQ@=zfiZZXJ)%!197Vpo@M*cu;8$lD@^7bS3Oj%7e)jOdV*^Zsy`0 zU~!rxHxWn}&8J8?AU8j3?MHY085vQZl8mqY(0r>c5yR$SElaIhM8~{Atm1Wu1FYX3D{3F#;c?kl{v1X+SwPNbG$%7P zj{cPX%D|Z2r1rBQ|NR+zuFRS1=F#aLfquLp@Z)pTbLRy9LqB>mOU4@^vFWw~A@6=+ zjADK+g7s>1r%*!1cASaToS_x#f#P8VMIY)fWCXh>0Dx01Gna>nEb*WsRiGkscJcrJ^U_?0FEXc}laCD=^j0pzsi>rN~B-RU3^%Do2H2Kq~xil2jAj%~~$k zb<@KWQl={*NV{XcStVp4ms?5UfP|GAUT13Xm^v>$vP4h{Z)Pm~;u#hys*GAOn&H7w z{=1_`yGUozPzz3&zR?A`S!uzzLD+k2cgANTxdUY#i;GHTDjIzT#cO z)973n=Hr|E7YfYAvRM8Ld~2$7vhMvfqq$V`=&WM>Ajg5f?D|ElO?f^j|(t7e3)!+_wGK+UcD?K~FIu53ILP&5ozsxu50 z%nKu{Z@|QQvi^#%Sy!%xC_$~9#~cEsT7{mCx>yP6VWl>q<#U2U!Hik}_x~VDb>F=* zI^TxAvMnx9!apbaEBLbF{)-fKMme#$7|eb_-1Tk=PS1+X%DG7CN`H#`^kRb?x39y; zEedxm_Dk6R3zk!NkPD(1zhS>eGq}xZlTM}moN@9sKA@6^EU>Vxe^!l(xcwGFdO{vp zoyM(*IHgU-e9Q{S(6zs7qHQPpWC{KBHQN$fc-LC*?F))!GlmRR^&s?2cXlB0kIMx^ zGg~2hBLn9}W>b27jcM`t;|&aL1Nngf*x-9g|iCa*b z6ou7wvy|3{8eIA@)4zNldX{hJcs@CLSNHEo@FAIR!mlX((AIw-G9=3%GE`6cdZhVc7KWuuDr45E9a;;JESZ}sh5AK2(Z`tVw8K9LJYKcawKR_12_(3q>@$>Ff&`Jw zOR{@iX`alycTUlrBlgyPp1ntAdRlKtx0#oCzPLm(z?bl0>#4wGwQP7RD%SQ3MNcAK zuNdH^dZ-Mct>KZx3ZU8J@>1!ITB|+A`f2{y-aDoUodw{lEaWBvq4*A~)^hgAB`NIr zPMjqS`|@&EOpbo_7dKlb910GzZ%szJfnAHkF!e|UZ)c7_q637WNgXF$6%Szhy0a?r z*f8>q*wiU8SoVn!=iQCII~afQsvT=m*GGmxC@o~9spfwv_o?aV3Q{^(;s3`9BZtjA6leZWqj=_H{r-rOMa0@?(Z ze8==JO0m_N87p{7y8q_ep08gJA=3wEk~#T<%YTb=bjJ=BLNj6Tol#ISJ|~AV@;<%H zGoM9bA}4}@(?Z?t#sc7QUcr+HZ>Fr~+kf{gPdK)FSI`&^(2-X>MvyCy@g^~!jo&WM z!BxI{vCpbvfojkuAIj*R2F9u5*$Sx|?CTADm|m>!XmNH&#Ce_d-c)|p2rG!IB|xtaISRHSu~p1q!?Y=Uo+j1SQwW!EQXlbvU%cbB zrU4(n#O1Kb1|GLz7$n^5;YDRTrYkERxfq830C!ldi`$5h1WIlKbthB?Q_#@?LT7!H z=(wrY@lMJW@|M|vxWTlUO2Y+wMqY20o5Zuh{(k5UWNl{O4Cm1i%zx-O8mmGvC@;6e zJ~_$E(Qj?%GWTx| z$w!I2s^njO>Xq(s6ffq!m;@^oPhup#AQ-508opd6Uw>vk!|gGajDBm#mT=UHvoAt5y_B^3#yU%u?WNlq%q2%? zf9>?>V!br5GW1`6PHF>*{&${B?iAY`4ci0rq|9B{Tm?-RLk69?))}%Z3YbjCLrjA6 zYpF4esI-=r^PejKcb-Kj93x}YW?onr{JZxSRhTc&V88E5x)$Bv#)2yx>hIJSGfY=p zjhS3y%hP-5_}5mPai37o$)IOnA<#Z9G@h#>&*^b#U~^{qVk)s=nu*T&Kc;nRh+ODj={|i-mGx6eylZJ! zGfyInI&GV$$FGHBnXE}jC-Zh(j>HntCObPbAueZL?k;0D=;>3RMn3m835OwCZw+sf0)T{Paz zhknl`k4z}@6PfVadSg;XtBYaHE^n(vC%4NP`oSOHt*M^yc>zlL4If!|_msM_zpEAe z694k=Tk8WD=SO^%+X!$BenL;*5;Hb7_PT70PS-|KJSxhqTb-*#%G416g!i%jurRRBhBp9r@w#pOBYLY1NwCF zffR#CAwxUiBce4F%~H^GFxM`E$5ge@zfClB?T#Bi>u%8fhw35AKL2+Cb)jELGqj}X zTOm90-8dYP#>8hkcKZP24@Z8oWjeXi-N^r(rl8W!a7sqm$A{pobZsnB8(x>QFG-Br z`R+G~z{>x5;y#?6K$!xU_k;Of?h(DZr)2t)nA&`m>K(iME~hVHy(xp=+zp4Eh0&c8 z^*@n(@!${rVeD0#3FjSI4?|zPHc)+)HOQ6zC)sBm$z88}6Q3ir;Npyo{|4tslDN)l z8TGaFSpSp5EZ_#`r={jOT3>uL!D!`MW0APtH*M>RIN-O-;(n{uJ9lUPL;?Ju9Plap zW5~E~kK3spKK3V&_QzHy?ADS`M*KHls{VXE!vO|;J&}}o2A!8~Y^^gtD;fU0%Bt*b zT0>aSh&REya~ugqmCo4fA2A$!-rQkzsHFK^0o5sQ;q@B_g zKF)ZmynyQ|)eRjwHgfmE+w-)I8&Ol*CwKcFdjS4{EvAQ>)981$rh^}XFcL`pA1`we z@NdK|Z=I&0g3d4A`riuw&I-5+x3eug_^kq=%+*ROvYL3B^gezu3$sRu@lP$=Z2wx{ zCTa9Kl9QSeo< zwN&}XCjr0J>m>At)PS?lG?=cxjE*DB+sd(6Occ@Qr(4R@7ar>pX>o|NdZ!N+xc8DzKx(6z z#AMmj(y}%Gho&Hj3A7dc^MX-+s}yqj<&oUx8Qg< z9H;M;bK6J>swYXflUAz2+)otg$f2uQYu#S?f+p=Bxf-GDP^PMI#By$q2u*X!v&)rv z$F0+pcF4(?dIcHCOsnG1G1RA%P|_{i&s0tK{7Tr@nbjBjMNbm+XOWsyQUM<-iLlLYQ!o(EjFi=NXvRe;=|RI{lv+!B z`|>Dx)^0yzgnT1>M=2L{q8pZLmk3xP72u+fJB9s1$S#e&{DNh#K8r`OYtI4G7G?r+ zZUP@^xXUA@NzU?NsFG@j$}lWLkVu+jX;*+;5I9cj!eS9=Vq#ff5v3t}X0%y08Uv~P zi7kV~K8R6=k+p&$#fM__oMCd!!|vIwoDur^nMM^}A#nJ%sGAOTSFnf6AsDUt0Md{V z?16TPXRd7W3{LT=*0Wl(Axx^q6tp~Dz3Xxu%89TX3A;~(uS9>n(sB1D+zKSi(B}Ui zV_(dv%E>t*jsJK@Lh2(bHA-;sv#iX^_pi)d&}s!+bt*oO$!bU~AtN^II&8LT{6l6i zt>!V8jxNlO?(|^3UBoB6JBGCDj62#E+Rl8~b#pG~)~YcikaFO*sA)k4supYT;PQxj z!f7tzJ*wa|l2BA(3`RR}UpQh;i5cNd37}|-882(p>rh((wVQl!-HUodz-50#p?}tL zHgdaE1xGq*vC_IhM~s*f*P#}@C4cBA2RgLoABciy%7OR-WthCP>6U<1&*dWMs4BtKZjUY=NYccg5 zjxUcW)wv~P^TrA8)d0xGK4H-iS8OkBrYgOyzVc1F9)@*l5}EXbz8yMFcY+GC`KE~f zs`}pUt-UkhV?s3}!aH+TGi=3cxHD8Lh&o$;W6j$i3T3t`u7RfkZkM$I`D(^L3ao%T zT}|zZp0Ar5Eq%Aefu6`TboU+=d^$eF?GW$|srv3xXRu7dQ(vsre}iTqzzaN2;jbDy z*eNU;>_g$Y+_Txgvz2cZ9_6-sBpy30>GZ*7@L;)#QzB3k_|;I~-q>_>#Nog_Oa+V~ zxEV^|RQl)I>+@y6*6QkCsz=?v-}|n}{U#rrPW{i_4@|*gXDHFQ+>JAotcYF>`NC+Y zg&}#41cG<=Q)TtaH~C+NmLVzW?Qne;%saSTenL$AJSc?lc-UHj$>^QYgmeR&ni>)N zh1pfAULoIIak1s%5{$ooOrkzJD|gFM*y)jXuV=3o>ePSpgY4w$@LJ zT&nNi$Q>N&7O}7;EJBHMP|_5@SGJJoH4>sG9v{9vUE>@PGh~l)mZfy^+>Kyj*}#bG ztHQ8*{Yj2N2&MPnk6rzJ@?eZeAXH5n-9Ai3Ta;Zap}8h|>s5}SAx@CiU>fVmk`RfH zF)^vAFJN5 zZ}fthX*CF$*$G#uIw^R~9{-zw`Th`9*^*bacp=i9*v9+Xl|T0k9mCuz(bHJVpz`C}i2`3&0wBGbe8^JV1#_OA3SSHXeXt))N}0aF zZwp_&j*i&8XaIR_i=4h}Rp&gx^OUjP!Ec8*iyzejZ^W4MivxAig3piX#v9j=>CN}r zJ?;GRKuus)%VG_d0Anbp5k) zRrjmp+Y8F=9dXtxN;Q?5{n_B{fU_FEB=7=q6h~oWiqrmNM}!*Z;?g2CVkRuZ#{Scz z#>eA^3_!MOcFPlQ&Qqe5Hm?c2Bg(=q|c2^LxY z*xdGN!{LH??^8u(<3ip3$`NBzf))cARPKVf0v${9HId@*|SaW&T0{fDO^ zJ=WJFGB8;6bNj;<_<-z%JXKugo9@EmfNaiMc9NU-cUa^Oo>b_pbcb}kVBM?Xbz8T# z57d^5cd9IPS~9>Y-_d+0Q^mE*m85tip|aww7EfCQz!vNBFwD*8!x8!S--y6Jk%q^M zeXNoA0)#`|-+ifz3y)5|pZm`Sstx8YJp|@vtFf1+6wLNbK+ZMQ5+V)=IHZ}Qu6*OZ zouKX2iIID46RducB`f@lw|^r(P>%SOuED_aICOa7SB*Ct+($&EVngVnb~{|kjc4Im zyCjwtIA?;Na4b}8->Et4H%744h8P>qXsl=n^d|1OwI{r6ef|`=>tUzgja1c^v$XSk zgpwQ4-$Mo6_cZ5PkaKQP6o#}if|_|+2P(L9ghGS=u*XS;vu>W04SEzN`(b}beT1u6 zQ7liKEd1sh`-~qpLo(dxc<7?$((ttjY;Jr*p|1D==w&Q3c< zJbDvVJj%x(w1nQHAMBYKnt3j_^)y(RfMI*-cUH~~5sve9UKx&mgpErd?(!wl4DPEW zH!s<1Fj9>C0aM}4+GD-+=2EGXQU(C07g zWpwq;){t$X>Dg74`I;9FLy_@al-s-`52fjMu|>BYsD_vjdOGt^TK-s_f_&$vJSJ}0 zr~;qpC;aar?H~{l4JRP)a0UGO!9>YTAo?G1V%{-$T~GU4A!|-+@FV9(PbDIq9+Z$R z+tyhW*^8e9@A(~=rZA~SGsrNYXx%WkBigA>wQigW9h;w5w$@gpNpEx&NH$EP^{r;~F~B5i$myycGVoOy#5YAw8OuuRJmSdYu;I2GbYnok3 zCHTwC)Z~o=TSI?UlOTTe9C}{2w*zJji#lFN33(=1H!sM9gJItbF~Lyb}^9bo&R6j&CWGz((c%+^*q*wc(2gCvlcrs{Px`cAX_zku;`P z0PKvcYy92t;0bO!<=5h{)*i1dFNTX*Qug5_nRxT~f#<1IDm)3^{-Rp&eN;Y+|fR0-=s+SCyw~1cTS5 zO-?q`0~Iql<_2-?FpoudqNXcuOVpOwtwMF?+~mu>$z-*i7Xo@p!wQHVB;zDYSUrX> zgMx|Z_wglK!qtsY%J%2=L)Lx)k08nnwRR=-@r}q$@;rE?hZ8W!T$`40n$BnU&@EVv?M;j%!tjlpw{t+LHBtEX!ZP=A(}LyQ z`L5>A!kkKL3Z$=Kcd@-<1c6(sti)|lG+sa!a+EUGHdU-0AN6lkQI0Ew z?*vPP$ET`TojLLt%kU-W zgO?|KjZc2Med9;W6iif)V^eAC3@k^-c*{x?`C6nO2l|YYQ7wpz*V@C;0>tU3&#ZriDXY%omIvj~s|OEYum}_LWv|lrJiROy3YT7o zzQZf^X>C9Aex{tSs?W z@WT7K{Nf}FxdKjw%>J*Ulc4qfN{N+U%VNT=MA;#-ZJM2c`MbswmYcArQm1N2#v#!k z?ZRtJ#%lN7ko_j9$e~QBEsm1W7xoRZR#tSiVzLOyZJtmg8 zdQV*20V=M-{J+RGL$GcJ2J(yHK995b&&~}$$fy7zUn}x(Kejy)9TuvM^5c|d&q@tu zZ4_lv4!N@LU!CSB(nnJR=RNMr2vc#6cwG=DT6pix`BS7`$BfIDEM1JGsJ|H-pE|^w zE7Gs=!IUrMo2WDYsVwym*&~)t-d7oGG99el9yW8L0GY(kQC2igUNr5%28~6m7YMOP z3WGjPav7!SJ=2^10o?L%`5ui@^u=bn4j$+pNbvY659d0iVS@({vp$SU;H8kV| z5CuY$rz(nIaNn{{U5nG|xRh!WM(RW$!j}WAzV>W*lzd1hdZPGmT%Yl63lmT ztkjuj=BBf}5Nso1!!OU6!9RGNFUD@J(w@A5P%z^!gL9xduV7{iH77_(83HNpqxU=8 z+X1#e8NEE8xjjL;Q%ZQGiwn$u$R!B1@hB}b)s4!zu4vQ^}tBOMD@BH%?URUxEu6tpRWjsvCR zD4Uwt?3Yasu4_LsPXeCpmfs*$Rg3+?!Uo!PH$jtBmWFR^9Qo0~@PM%E)6G;OiGi)P z+mbz%e2zf!kVe+%)XCW@ubo7E?RcUi2`WNotl|0cF0pcubGZ{N4aU}th4sv^)Xd;7 zYq`-;(2e^8WOolY7gCUYGMXnYz&R?s!;=aHR0-3kKmi7SfGD1F;n}PpS8`F&gp&MJ zUZleu%U-5%pViX%iF~~)lyba<8yh;Z5)j4?8#sS&F-RZvI`E^RnJ)rm+ zlhDm)nAApQE}J&CFAJ2Ff37>Qzb-sl+t?>jc|<2RahR`iv^F1^&FvF8g?)b*X;ecH z#ZP5_$d-&7Xdg2#rFLuRNGp^xbCj01O)Vj6Vsl8t>L;8~5l=nF>m1M2v3|-}PN$UB z!gKC)ZQ{%`rn6m5$`x6oit#5b1}RBhxt$Uxa>ml3-l$+b8Zu-#7y{l5WWID7`&cWW9 zqU(d42(KsaOq`EpUxk){8VPcx&lOZ#aTAggd;?p)=VX9)29x(1><;#jg!6ZR)$2A6 zjV)rg+jvQZON7zUMzlSbITl$<1F74`SH+GcemD5 z=Ani6)UpZO_@Rx>_SY1+Ogjs+@`DJ~o&EVoUouV4v~)-9?9TAd44PZVK_>wm0%wyn zk-pm54`z~%R9e_gY~~`Cc}!s#%d4xsar+D!JOQ8aV#~_tbga+67PaOnJ|Ily8q>}0 zyK+M~;KgbxLCaIEF*S|WA&uWM;Jx|y%QZvFS2f5z;hUT7$<~>Sn{UUTK#(2TZ?gBKet9yg}h_gSa@T4wwGtxEd>Ty>DXvvn<8UN)!I?h9Yyh#nhWbmjX`5st_ zYiFS1`Txh)TZYB4ZPB6-0wic~hY(zXyGyX(?(Xic!JXjl?$WrsL*p*N-CY`P?{oG( z=e+m5cmLH_Rb5@Zddiq%jWrijeO_TM#zDnh%6`G6Vr`Dn-}9 z%mv|}EqTMoE=<@I-(3V;>*@^|8l(5?qjq;7xlTsgZ$)=B_lqC3hU(_TAui|p5MVH{E zeLF)R;FIZCebOU*xn$kdslfibx?iWjeE|0!L%J>wRg* z#b)W)p4si`l9rmCY6EA+qKMo*tG>oaV&T1KP*YJ27WM~?72db1b`DHpW^5@)?3kug zwp^lND6#y@Y+Z+tQ7Gb;KD z2m$#wj9Z(G38An;M?~$BrH%`~^*baOt@~21Em(*xFi+ljk{_jLEM`d^f}^*3YQ9Eh zlX2}Mb%y(!>S6!V`3>L;tI!xzy(DP%+q>yw#4G?45NMcFWte$DvOx@Ns1NSg$_C#o|;*DfTvp}xX_n*Jbj&YAM$1>UZ=O~H9g290vfpX zI{zxt-X0=)A20^H!K)bvvW*%1^&q@9P@G~F{9m}NRorjT`cHa>VLQtmCz#G=bXTGM zi?vzg6gxt}7m^gF@jLz#SwSQ90;%?5#~fj$ISTCn z5~#uUoo~?dCf~fxhLhQl!rtRn2uTITU`gh5`=WU0(x7;!{pEqadR$ov9w?T4*~2ZX z{TcG8HLN=811%8)u>n^$sQi~Ss?ZwKr28kut%P(7h~&&mJCvNfyk4pv96v?EZ%_%U zN*sGIXnzv$2>+yT4dH-K&)#e5s04RZ*xOnu+Ye6g_%Ts@m@cq8;9ku#hf>#No{2)n9A0UK8*x6g0zHID0; z?{^vJKYAYCi-ps^yQg~fCM_3QaltE!B??}uq(Z;>MoyaRj%Dj&ytG^ER)9L-5 z9&dE_HIYsundwaegcR)Lr%dr?z+P-foR?ZQF5rAV1<$RMMj+p3mVf}cVyfRn0}W|^ zZp};3t4s=~cb^LfRj6Z%40mt58ryz=cA$SrdbHNT;yAM1wY_c4kpidRC_vFC#|39W zaC6|BmD2*)G7OA88Bb)#4UoS>v3Hi^0?7aRn(1`K_5=W6_V+F0IBV?=^CbcE6#xFG z3#ds%=ikATs@-JXspU(y39U%miU<8od2DwHC5r{m9E@N=hy_PpL=)|XT}gg&b2D=ujLhrh;nR6C z1?Li6FuM4_;m>i5k)!YSJCr}zfD-(3gBCxULM+vRyEq8(Sa%R_>bz(}#6XC)*Qb~g z)x_d62Z*$NSK?0tq%OaxljpZOlNA()RTkHkJ}t?(x_$w*l`ln~>R%k;1)i_oh)3v# z7oIY-$>^45-zjmaL(3yrY>i|xL6-lAn(=`m71&_|n4f9RE2c-{gM{`hZK0G*XaBk3 z>g4xzQ&US*JW0Ud5Dmev6G&v_!iO3fK|DJ$al`F0^uJ8vBD#37+^fO+3vwg$Nmk|} ztf^96wuHezd%^K$bsEO%dVda1H6xiCvEjVYzHJ8=1QZJ`tm}0UYp70c2LqW&ydptJ zuHG@uPd^J1XIZ!;Ja6^dexa3nj+rWV^@FW|X(+3G9=g%fzbCH2tkNUm?~9T(lQMs1 z2ryAmf;8zzlgj!Jb@1u|#%?jk0tj{=jX8=~)qBAi-OcS8BKm7o8I{bjd`}1xc zuwY!-55$`UzfWY!yU$%+nO;;_xx%B{-Ypli`QaJ?B-#ov#X|FpGbz;u4r1`U<7hiE zd{64TI>#v{)BS1BwHv87(Ru%1ezX8mgEx^7H?{9rz+Dz#612~O=$HfWa1L2$Z-3fo@L|@-*2%l_c_Z#x22{PQe!F)(>K__6eru+Dq;T3z)6*j$s z<*P~KUYMDD!Q*Hyp;Yyc7T_;W>Gu^a9P3AxSjLsE*}}v;A^3S(1zditc%T1u%mvkB zxbbc4Lf4)HEHGl1%^O4X-w_d}bi%nIq{^oK+J6F5zE&dp1{K^1zf^l6OwMDM?vy76 zL4w&i6zxm+mv+jfeGY#9v!huFK?sB6*`~w@7w#3tZ>=GyYs9f@I}K9uU|I=q z^Dfn0ib$)I`D3eLOtR9b|NCy#{RIQF1CnD`6T^4n^l~)KrQzkS_zZp-l4e`g#ugZq zH5HPBOy1okYE`P*-?bF6kt@{4HU@IHmQsfb!~5#-?uJb`65vwdUjd~#dQ$w)6s0J_ zq4Z+3s1g{BK!sIkcKq5^Y&>@M2h#;ydFN^I!S{lk;#Eh63T1DotkiqYUR;e>mVO$hx8t8x}M2@qE$^_)w^#u<4O?=nAO) zMG~^q zG1yrX-%!o;Q;zT4EW$b3kL1Q<>6iJ(DLc{o=~~Z8vymD5wOZfi1H&qV$FjrMOLmU>O`9e+Z-ZYi-8#)Tpgo^!M~iNvv(x z2TcSIdq}iN9w46iLb*!qK~c@gl$r?)Qcl+-5fDDL|DqZ=!SUB*UCMQ+(Dn8K8OPsh zPJ6@eQh~Q;qXaFUOU->BTpzx9{lQLZR84&>#-4%uzp2D*Ck_y`hr~@L2!n_XITf)A zQshPL?qPkF287*Z-xCNRu_|D;ouNHAQ*jwMsvRmiIM_=km?EJQ`nC% zOfY%+PQRPj&D&o|q0$^=bZZn1wRwL5g@^#^$s#k858dpvvDl${&^Euhlw@T9*{&@H z%`1%IEMip%BK00OYx`vo#a8)}hr9N+uAttal>%_pIm^&@Pw0 zd^2ee8o9$EZ%>cf#{02pL!DHQNa}4n1n}etcmr1gI$tof0lFA-Dlq8jsTCQ)u%?G# zshq0>i=YUVl{ZCn#YIF90o)nlr%4F2yrDmZ#gA=P4i0#PNV#l^HH52zUNT=udaKmc zV)#md*qxc;|54L|dpb!xt*&~tJ8*&F#~v6=UAiqDlp){{X7M?>8&P^^dF6WiOaSLG zO6u|Q140&uCi*Sq_}Hakaf40dozi;r@dK86at|rwAk@Q3J!#FsG5b+k0%$b{5#1M;t3|a`YuQzcyb#C#n=dm9TSRLYz zB)n9q=W?VYvyon(1$nTk|2w%}ZI*vcR23^;a3hX|N>T>*-|Y|Rq&7c5fBn#w2iz2V zeUXT+fITr_aC)WLfxR89ACi&@Ba!S44U2axi{XdK4BoT*MuVkGG{tAOHPr9Io@;@r z@ZK0Wa1vjWK#9%GADSK4x6=G^M#jRT z_B_Mvd@O;#lQ0_xtiGNlAYQ8iSa+)9Su9?X|N<^HVcbB~4XX>zHRTN@0)LGE!4*Er->?QH4a{+N8t(ZwFj%LKO=l8}K+ z+1Wl^WlrmfaW>F&;c8TX7H%Sr%tjX(hKe4i5WBC~zhkFijxPCIpSGU{5kL($7!u69 z#2#}uo81z}yM-IvO<5!r@vR^q@=OX-*UJ<_Mt8oL{RA!3mJ*DAcl#?%9D{M%G(T9n zBI$`de({egwHj-PO^^L)bB9T-J;)oC`!z^AgjO{=syyo@>kXd!0~7-_o5O>x-t^uZ&9s;r0i2h8zfoi$b3%_Oq(W^6Qa)J>ta|)Weu7+vQ!83jG` zF~afoq4t-6OH1&W2o#Prch%WP`rc7Xx&}e%EnoA*Mkv%KF4z+`m-28>##J@`Fp0^Q zbP%izS&L2mW`PPsJPpcD_CJ&2GGE{h8zjf7!wd!d|&bUF=`_I7nf7^7bcOSsW(yhXV?n z>sy3IOlZcv`J+HL-DR0OxZY~g*wVgiz42o!saVGp6q_=Xxhmn}Hm-hcbNybUxvgPz z?EjYnZ@|vj`{WH9#_3Gzf`1@Z(}?^{7J0hT(zI2Cb|)0kZZ@}7*k{F}DB3F$GXJ1% zeI`0o!tiLX(k~q;UzZzQlWloZwQ)UbC^ka~{X*u^L(3ZhV$L(*)474b3mskrqCk)9 zr9qM`e4g8bkSMZ`|K|&%JfhB&C2?OTzfNJ**Y#@#G?`Pe?D)5VgVV<`79lU1v98%r zUr(qsPCQ3rHh51#Hud)p!k%a_1SQCk+jr~~TMynJ0Ajp?yA>ScXG5rXydQO4r*OFJ zb@7HlI>1nNKJvY>YVjvC3?K|1!O*I7h%1)w?E&D4$`0wXgs%N#wy)`qr0}I84c@_g z#1()7)r~K98AXs~`;8;iHVjGezzsD6^ z1)g~9_0XeGKd>M)BN24XCifXd`!TsDpp#rXp1uSx{CzW2ZZ&A83m`gZ%KnZ}nJaHa zGw2d9o{XzUPgS4>Rby{7AMbSX%>RQ9*qhW9Zw}&~1&=sP`!?9TZg{4irfLBc11;`r z+$Ox-Z|;$|1;}9_!>L4ASbW0e0uaFu|9hzHK}|yvf^4j~Mjsja`$t`4`jwa2CzM~*OUg>F z(84c}w|7i8J-xiz(tMS%Og+yOA>|gO98H+K_pPP6Cr^1Ep61!wr3dQDL`Uc1j>T z)qg1#@k@PrdIuFqc$1OIY65=N%`@IXkqsJIlTMfAPMoef)Dc=tl+C~`7_RBM$~_jtN%!rR?Dg=&-04IgCEDQJrUkP z?tvKj%_TSU^Gj=B zW*X|71~H|F``Uhao*|95XyU$e~)m?G2Y}b-w`PV zEyc*?*MJaFG)H!IS%TdvF0g&$+9 zDz7?~@9;co#h5Z<7!DPG=C2Q_EKizgfBEu_oLu8-&ifHMKMK^q9T;5P5e+fRBzuLn}i zWdc!=NghrdA6V-1W_LAdh>BL>M?js}WZmGU`z=FgYV~Ts2t!Q34m|WJrh;p+!Bts? zLhrgCcv~OMqydy$2s8d&eNHftgOpa{4rV@X+2YwC@uT^SU$Lu`9=5fKNqYb%?@Rf^ z(te^&0q_4y07y!EF6z~;6uFKGDBj#IZ{ju-sP>zfu(zt)KDfSaTV9f1dg;h&Ue3L} zUKJQDEd!aE9b9v`%82Pt7t-uCdl*=qgst$!u}Z`)&V9!NVsbTq{99hf@vPul^QG@t zBJiG}_9=aP)Q158n#ZZ&u!3qL$Z+zES@aA>siDJ&_**Ba%ob}f-V(ILrF33S?45^U z@Ra*SvlhDY6o$8m8RgaO%k$cU1Q(cweX;!>l&EOFGz@HzJ3S9;dQ1DJ0X^LnLy^QQ z!Tza2jv-juV8iijILh-^R7iF9ipUv9Ak}ofS!!)USlC@=XFlP9pmKsh58*}%b21A} z4T#~s>oxJ0cbmbQ^`HgAc0M51@$_~ck{U5*&7Y}ky+rmHw-44bn`ub>e{I1C%n7MN zgSS!ryFwN^!01m2ion_L40)POj?-eM5&ZuXtwrClSi(h!b`3XbRVK|?N+jA5M6Iol zrPO<}2H%q>d|0+Du` zvDhu<(})|qg=YRKLgi3~oKWGM56Z7vh(99DmZEU?{Lm&^X)rMxdkdmWe9RjKZ>$k5 zFF9L!#@H}zA_mZ`-BpFgH$%tT)z7C1#)LCVZkGEHUN)4}m>};7>FS$BP%aF>$jgPq zZOq5RW_Cg3BwM@?Ufc+lo|U_s?LL%4HolhYNcWnLfUu`mOyGzPS>p7)Fdg{3S`X( zZ|DiMAm)FvxIjbn)HZBhh`r#E&d`SZC*25zu1fb44NYy+^+_+9+b93x?_g+~y!#vL z+!fy(52UDW8l?Cj-oxB@E4V ztM6NtXtUEvqUL@!W+#UiR0f zL-JqWxOgYlo%H~}RrQ9Zrn^LgqsuY&v%c!i8s6Ub*J43;BqLJ~r~xhQriJdQXu?N` z6pinZGL!ist-C!&BkvR#lb8o6t+}=i3g=pp+vcTYAWOgsGiB+k%l<=~A(QW-zn0EoaRFCYTG<{3T-=6 z6~OP>U?q-f_-}*gM=XT|^SQXd=d|U9?`W?cD5#h}#W%mcmibfhe89f7LJxQjVm^$S z4b^V+^9i{XM&bJ0_yciirPBJ_JY5^hf5j0Sk14fMP$>cgAe7Iiboa zrhElT#H=+3@3o)47B;#gQWFY*ABwlNS`N9WChVF|P$RD!; z%%UeAT{`VLqnwV}FMNxssQ@i4_LJzNL#|!#XXksN>iO*r;=q~#$HUoEj}N~zPNXmX z2xze1=Qso9JlGXO-|Z=i@C7Q1ovXiixLyRmkM%@8bpf$wAq~1fNl-WfGPrHN;y-4& zQab6~ciz-I{rx?*)!$wbDh_7F0f%0z)O%8YJR1Nx=^(tYg4B*4EZ>- zq8-s%#SXcXTw~@$aZ6p36>cy_41K{Da5Os&|FFP|vsBI7I!Boo_g#-UzQ&n`d~xy? z`8>F|ja@q+z&tPE7Q?aWV!v-lTMtpFCQ3YW;7b*I?31Ph(U&1fo3FFzu&t7N-qKzL?!6@&4 znl3%Z12E|WXBL~3>by7&6nIpC$-+nLD^L$ezuIRf#P`n|`cr=L4p*-SR}t_IuQi1& z>DpfT=~fm>>oM?Zc>Y;t{LQz{I~=+G!)ox(xnY89wjovM^?}>AtUI@7$(?6b)YX5T zW;w~`PpPkKZ?z$Xmv^L@a*oYfe3+ux%wS#Wd>i1(Y;oj`@A7W&@VaedO(FFnV7qzY ze6ZWqbEbD4zUvQh`%1si=>jOFLXWpC@!gtfYub^tPuDW0O(&q!nE{ReQ$HXBL@A4> z;cDx}?b5qA>nB>6uODG*>jg|kiD!!0dKvdt%lK>C#DABVz4mn4EH}#(ia6{~h8|rw zy~ov9u^vpNM(*|wFx|&{>~w!GDqZ7kuq0{cZTNF6eKEapOQ&u??r=&I8Bx!lGXMOM z*{cS$)z0e8FF7csH_Zp|R+juQwn-H3-2KHvR;}-Ar9atv;L)zkH+t>OBr!AyqkLeoZ{;YKWrnLxQ$2>6Zu z5LnQPr#^82uTlP`NNq1JXW~9sS^cw^oLwVSyq2(C@=QsU+9EMV&ugAJG#KWK^e^2~ zr+LUJwY4SLY-UUL4BpSnL`Ir`8S&7Jv7s@N(8-#I+1puK(Xu+6Df)z`jzREx6~mwG z_K&Ml33;`Z=}(Zs_7}`%_qZ_2v>7W{lg%-h4jkoX7Z;kt%`r|=c#QF(0$*u<%`+F* zJ*RlvswploxntiFp^h&F^_bCUGQ4|4E53XSr;bzOc)f#_s^$!;B}x7h|Gx(e>}XOCyEdji86yW`>@skEjO z4?Jq#4WMgZUtP8_tW8yS?%>%~Qgb=$l=~;N>JYq>+Ln(UfKN9h->+PtSb1^Y^noWd zYWK|f7Z56&Rb^7`7d+EPv1dvyeA=Fgx2T@9UR{ktNBBMBUGy(nFjvbKB51+ zcB`4mN}&E+MP>0rIx)|35r|iE@;@X^KYz^s?%}mIh$w;UEd(L)KX#$L~B_5$;9ya*%|UV6W0 zTrt2<5M1bo2a)E(Pe=u49N<>#@ohU68fcA1nOBrz2Ef8_#iw6Eca!i`V^2JX?_-B!X<`AfE4B^tru< z^m_L>$^BWC?k!(O+6e^WovwO6u+)^({zle-v`^#sv|9D(@`yX;upjV47R0_X!)mB? zJZpUUZDFXu?i*{+dhAx@XSdfu=X3vidEa1fO!JGIE5jeYPr6*#)iH9bEPjj;KHCRB zUQ$$mI8q@ni^-O_Pt5x#ysuubv|{(5{=5;vM{LrU|4wTDb3|wqmviM;URf=?>AGR4 z8`)tncvi$nm04oX^z>=n@cfujP+u{SAG#1YjYZ+KnFVwc*TM7GF> zf+Sc$fX%?mANkKb8m~8MG#lz(p@0w{gO^k*J9DY(((=#nULPeJ9TZo&R*+JFw>i)F znzu-h^dD=#E+pzG?laa*5JZ=>%$Y?!k$>IPRHJ#1q z*z49qmWBg*J>q?)VbW|LIt{zuo3k47dxy_Yp2ZxzcMJB@mmInMY`kPSQLr8Q9}oCD zn+WlAD+L7wL7-z|D!kfW>2bz-1(Ih?9KB3>o~~m|jO^w&X;n9NB3XNb%uY^*!gfiT zv@c*-)27OZNvqQOPwA2MA;l3*6;6Tp9$TLi#{6;#(2^(m4yH5|(!pJ69HCx4LT!6R z84SHgqt527hWZmDE|<9itB+;$7ijK&iJYE)842A8Sa#d@_!cy{6pi(+9MF+1y)|}- zj;`={y`KGe&89uy=pF&NK5s<;;@HffzjTlKS-RRQJwC7TK8hq|E_Bz zAqc6E>@_e;P&`wAL}KPbB+_zc^7eZ_cb{inK>s4LrDDk&Q(RwWX{SDIIi4=2YG(`| zF^Qx+1z?0}iHh-u6du=U`$$K2rbLuuSsHX?Ahi|oKE5~9Z}m0fias!T#qSr=MKB|A z3CW#yoy2-=v^T5Yjq9}f|M=M7Ie=*Nm;M-yg(f@Fs`&r7BI;Ar{BXQ(yfb<6H4x=L{%0xc{0vjS-7KU zV%peLj^2s2EuvT;6D^eUYrnu}8SA~uTTRbmGC~%&#G?s>@sDHvd#E0Z z9Z&k}zk7bZI0#yq_Ab>se&+rBqR$=if%nbc5UUr(C_KI^%$sp~Je4mC91jotIrB-0 z^B;@DOdXFu8C(CO1xWh-R$zpiu=32cXUTCusVj>M44;i*wpu2P?myE9<*6fhRubH4 z|EwhYbwJZ&~t8PxX1P_1!-;uJjeq}mtn#re4T*u|S%5KJtU zjVd*pZbHmhkF-Al-ULj$Aqe0ar=1H9wqw!u-@pxtVdwN;B2m25Gyd3PCFmN4ap!Lx z@%^gzRtwnQa%o4o#^qymKD1bPTz?LB$wj!Fy#&pCLUm+0M1Vnv;+i?#rWXD9n!cf! zKi!TxC`NoXr7}M~*F@kvX}vX8?fZ@O?h78)#MTS5Yo^TKd%67M#rD4P=2u?kJP~Z? z#=fue+uOb-XCb`y^zL+pJsme+dR44U!Kd%Vyoc6ed9R$lYgE_*Q=-JeBIrRkHv|-4 zOV(GI592SE+%GBZVH?pHX;Lr;Rs{lHB;!B7=azs1JeP0hrJkJ`l6>(jNPI|=YmNlu z@jSsdT7L40l9cwi*#7lC4E@7u*;3bZ%;@!aqo||x4CiLzuQ$<)tiH#jF}p<2>YjYe zKYiu-^1!`H)L~as&4&iFLjQRhzU%lKzr*0-q+fw_g+rArncv?ZNG0J_Yb^F^+Q~oSg+Od zKZtpYsdeE!n1*Q%-XXBg1(l_kNs{H7=L&5I$A|HUBBeEZ35 z9x6UWI3rb#R$;Yi}X3ngoT0MqUS}kzTt#@`a zYj?$C+OsVSMQM|Xqlx88hc4wMN^?~^!SSuzJ6icgzbxi09up0=XgE_Vmo0IqR%#$FYBoB=+EC53Qe@ zaZgHzueOS1(x?n{E9L?&4Aw*pagB%+2ON8r*g~hm;8Xd0IRSiWz$rnfOV2gAdk!br znUjFjTaUFurAX>LAaYV#!^sK|o?7+NT~OK7;`oGXw>>B6O+P~$-(Kg>_Me*%$#DGi zc65C7a>}nMcSzjyQ}TTcY|8iyCVj$p1TH!`mGb4ie?6cou+zpnWQ~66*g{@gHGWi? z`hKOw0lW%?lnu5XtkI-c}#d5AfxZ$`J?!D7aLA9_5{Dt@qT{(1g z^G+B(Iyt4sQWGoD-1H^K3LU*&B&?J3c9qIQ#0Sv6{J4?8F4?8Wtmf3lIyNR&})LN;jKU^y}}O4Nyu)KDAGu3De-_+i$J2 zg|J`ous2Hg8lvsj--vY!worJ280pdZe=N>?vx77A#0|~7f#LOxA&;Ak%*6kBlJ_x! zX_Zn;>KqyrpDTqojd<7i`QyD?z;Z2&M+}kd#lGuvw{E}-d_MM)ilW0Q=KIEy@h3Fl zRXg`t*pGLI1I-PoZXr6G*<^2DtM>TbMmg9LDn2I3Qh8)Y;*S0qK_NxwwW<8w{9)x7 zcU+9U)=L;w<>VsUGl_!SxWS%Z9s?8egPfY0xS-&#fHsozIp`brzls4H-VcA_GN%)i z;ZE&oD;=Vqpp?V7yMGrPL=AV&9ANNf9C2XRRFgl(KI%hCBS)mF z&ecx(E&#&p{Ho=Zb@Cg%^abavBUzOb`6&<1;YC z$-fyqkpvrr!D9kfvs{6WuJoE>&GwQ)& zTEDj!Yde--;~#U@`})DEktd{|lb$~2^|c*t)5b>-GV%JF(n%oW#yZ@vlLX!heg8mF zP43G6O_2rB3~LI}aF2)Xjm$QRM~Yv{;29Z0X3l&i73r=iJ}x1;Fr$t7!RVtSd1A)~ z!5DD2yLSu!5kBqhghT)ImN54XM&TRQ%~enQl2-cmjTJ=Z25#8RwuIHs&YcJ@d$jcL zKp!2s>0)1K=%tG2r_SLf_1a}{$k5||ckE9&JAiHkhYd(wlV9`~v+H1PwmEeA3MYrImG~WM@7-?XFf}_!u1gI!o6!sL zgdN!;07L4YokfG^btq%R=fN!OoA}SVm@>DF&@yyJ&a0+4C(G#U2bALwYPA{RBqo+0 zk~FP{2B6*{cW@pR@FH=UM_{*Wld9(aY=l)+)|1j}4<9eFRFr+fv^}4Ewc)*DJvRVR z1?}4QiPOy+-Rjfm=myT@ne)!oO61rI$N2nmy0aFJc-E*;Cyz7y_DE?(6#{^;#EpgT z{n_wEs`PWZ7eDM5byZvMOmxk zGcIbV$*Rz|4lz`_2)0{mvASk33D^AVY00EBg)w4Yj5cRx`+ONa%A95_*>>L!6Mym< z?joAm9Mp_5VS@Ht?76MF+W%atixO|}>o%z3<|QuYJo7SC>qlKHRpu)%u?w}Xl~lJ; zknvbn;ZDJ%lXXYJQYB`jaeb^xbjqtanBRHJWHhlhS(}*v@0PGKQ|M&Ho<#^`HTG8c zc5XYqmCsjWwIr8BRO=T33mMO6*u1JD+Kwe!0fP7eQj-ntFei06`WXolrjj6&GrzK0 zN7a5NPpXcrBZAkJ5_w-rMmT+=CDfd)(Zzk2*Baw{t4Np^m6zm9B_&uAxxFBav>1&X z=?JxShz?r1{M`S+^SiC26IqM6y8A+@5eqsh)xa!M@;;5L+zqUiVCN!!st#Iu6g6QZ^?Bwo2htBS8E*49)x=HY^tqKo`y~q;{1ufSWnOx zUCsxXH>4T44(!P7xw+Y_|A-GpAtPs|e?)z5jU~9@@0l|o1!Xl$g%9}E${;!=(#NAW zBsjMLwSaw92Hg~Hv}a71mrq6LR;?6AtOgKKvUGZri?Ac}EZw%<5Rsl;q|4k1rZZR! zzGAE%2p$AyrOX`jmI9Ty^EH|4lST7*;-jO&$}bXGzBkSq^P?oIaU<>#@MOM=2Wx)C zItWF=v$!R2Umu#(@)m8s<{x?9-b9*?Y>Q7~;kuQ*huoHQWCHo4|A3dHM0rN#hgk65 zkRZirZv3A2LG8L1m8s4i9MwL$XEZ${d$#&O6$Z>1tG#fnG*$KZ=+s>T2}kEhZi0yC zO1|-2MD14wqm$|Her|3XS>>+9li5A#EV=B>HPrdO>6LI(TruY{oT$M?!Q`XUmxk!s zayaK#+`0+I(krqvNAo(5Ji8xv^;R0RHa&gz6G;c1h`&@p%4 zp@8uCmjvb$0~XFx?t)2%lkgRbr`ZnC9Q4DTyD5yiV^pm!RhO_2&`8|xVHL{;Xy!~^{GO|yahILMzywGQ9m%!;3W*kwEg08Sei znTc`9#T=*Ye=bBTZAbHw`%0m>p2y#!tE{8!JovX9i|Tj=12Ld)j*O&k``Z#JRti2Q zh^<^Egap?U*cTl5cSyBj0)JziB0!dNMu993m>kW~i}GMMvv`M?8;gG(#%$_c>JzNP z$mVA2Ol;-(U=x~$Yhs{H4*{2noK|Twe#(4nQx9JMWe}4 zu`IQbHd-GKij@3o(gpzm(Riv9grE_3`(gl7mG#(Y8BT7=U)lwfY608bThS(qD5W&WGx_=?*_6o_X5ID!i>cm<0=& z=T2`W>8Hsw^)i)2fo9v*a)0Q3aXr&&&6>H?c`yK_de9@Go^&QWt3#bRPW%x=gY90R zdIPw{s`#{h1w7;@{T(R3rS=*;)iuD`J48z>F)45&ovq)4jWlVn&qt8|(syxetJG3k z7v~RCISrqW8Rs~PvN&U7|0?*jGS%#CuripP8Vs$trji7tlNm2fX(m5NKxuO5@4xl~ z*?SHwT%-DiQ4x6yHu(vF>D#}!W^b-|?P-=Pj|_617{GtY-EJLe!^5T0y|nHAGk^jVUTFl zw7RU_`3!;ArOA75?d;TT;ycr8BKWzt9NI#nS3ezt47rgUWFWP0XZP0MuWgdIsbpE^ zY_a-ldLUW8XSEcX(0Cw%=xuq_S?1xDOBeTe$;T0P&gd!+r`RLS12khBJPBSVsmyTi z^wfsqQ<9{%XYt~gPbUYf3{F}G+c{tPoy=FTvsK^5YOs!fqjm%(NG_8Oo|2bOTejof z+TyZW48Yrl%q+-V6<+mkhLG4WKq9p)f4COtuF0Gk*4$r? zZI`Ie{XpW2F zquM$HvWc@2nS+_VEwDY|R;Ke0m`i*(!wQSIoy(!-ai;{52Sg%SDvmzlLLshYFl6#5 zzU?9iyo4enmS>RDF@gqr{hl~8II#y}bsgm9&;Ih6-!FYmMY6b)udEa$H(FnOvRG|4 zt4!%zUs~f{yQZa5IKaJ$qONXygEgpMI_=B0MCTQ^y!Z@R07<7krOx!PY4bTMH>>?q zg^*_(2u1D-9+)yXlNAGP@(sw_PeZnwAunM&lkjl(ZaQ*TiKKsgIOy%VUEX64prPN} zElpvwusE)L1Vt!J(XE$%T^ziIS7ZCH(U+JZwYj{BfiIqOUtfqSYDe^lNS>%dUA2B0 z*mmFQr{qI;wqxo@U(OCu+wL@Y_6N=3HT!46#?-y1kJNLt{FZ=s)o+9hPYslIOl5ST z+$owgUe^ywB6dt+&oiS&z(QW(T|_5yp&DZ&wL5;K>+M^N)4CRr3|LW)f(fEgUSaTh za$%c5?paTY%A3idxhv3KrwzP>5)0$5$rK^JO9hUfJEHhUx0}^D0d1Tsw(B{W&>%_2CG{t*6#|8Y&e`wR8WWjb&jWN!wr?Eu>f+o9KN@S9J3#Gn<)?|% z9!(4W5tx+`MOyYj{NM_FHh517eM=t=o73e3AlZtn!-jouJY6{3WW}L>SEv7ua9|fQ zpIVz$nA(gd<^C+ZGuL9U6Z-|7qN1x)#@ahG?ig3Ex=23jCvG@Z9L~Epk)_#JZJ3FJ z^-`|k5mMHxubYv+iPh|cz1uTL*8qvAji~(zyKfX0az4EM?94@%6Q7|qHma@wr!QAu zp^AKlpAiXgHn{^?OAGl36Qq1HQ2e(*>Ijdg6obE|Ep90yQyRnj$&bY-j!n1QT>K*oRANid5^p@oB$8ves8z zxm-3xhq!*FHoe-nUiDzA>3k%e*O%DXnlx+~E=?n5=BeRpnThRIdO#}WsJc%EbE*na z5aTHzI>#su`-yLdPYb9nBAmbQ#3&tvmdJiKpf=ZtUPjKpsd-;y{_PUi-V!lV}TCc>p-^ceDIHGu)~3Am5%I z=u_D)kMv_40YO6dCyPi~r{05C%;*dmqoKvHrDB?2YvYCmMoaW`QeqEwRfei-7Top) z_!zf02|Ok9am6cHq^`Oj4v>QZb@qfoSPPhZfJL1(2Md&Khj(IH^ukvHsIb0f7C?nPLi7AQ%Q(vzvy^}efTw~+EOjQ! z5>_Pwi?=Y8vv?(T4^uVQmK9raoPVvIW~7^#I`x*e!c(eiwVk~=3q6=g9`E;~Wm?Ia zH5Hr^)8!LPoV4U|d7h}MNBi_tJXANuIxuCnnY^CJ*W49Tvf}vw`(QdlzxNAUBI4^!i%%_Y%`90mWbL4_brL>yaaH z$KI)jSV*~wn#d|dl0r9XuFnrgf%lIlNw%`R&*G*!*F@v}ZxRSuO+)~)uwgO@vI!9z zHObRJj6mnZ=!+Tc3gEmVP&taMC$NQ*O8DsvzerLAf&D>br2kfvJjn{}=i5b+0mzt{ zD2i$8PEB&f`AFu4PmmPIeHQ20fdO@*ZIE{vCcVjGujE5R-}uf5t?o_lI`An z*LocI;W8bh)kqcPWNEr~%QdIQ;PR>H;^vOQI^ruealD6Ym(=2)5d@wmwprH%k4D$| zz#}2@l}#pjiwz6mMF!K1zUx)aaaG%R7daI?O8Y)VxN&7ATwDAhdE=?=mwp4n| zM*Xb{fi+htGPAzBa7vBrChXg|v_o_Fun{@Aws#wdj1{g331=}zyGz`AUkIa9Q_wkS zB%fSXbJFcj<4|@3zBOgagGh~$?$!HZozDyBKK-Xc%b}tQ{!|a|`S4In_0iCgfFH_o z3Dc&)?S#`#*y;Vd`%=l3=jZ9h4$y(oHOl>Y_a6-n;bqDdVzx5xj2mY*Dt z#G#e+pK7do!_4I!>RaNs#%H949>F4utTa;hOG1XXR>V{PwWq}pU-I)5Yp_XxP-aU+34WIG|Z0sYJqnkK^?e1sWs{6oVJ@o&1r((+2ykw5H!8x z?7UXVl-$;qa!|Pt`>=CONuLx-O324an18wt`O;hS;*_r@W%*%g*%Uto!U$op7imux ze{S4a7bk8`AiTPQP(^ma9~y6dSRhQdLAci>I7)f0h6t31BX9tiA?^1IRt$c%F{+$v^Gxw%; z`*Qg`C8pfW6_2+9M}C6i)cLerK8IKQ>akx#qk8$WwPBL-8dkZbVFt-HIWTcKoPN&N zk=#ab?%zHq3MvltKQLkBT9QvdcxW{&%)3RG^T(4#b*^yG(?L`N5rlG1uvt$DuL)RV zJ_3K{OSePXnxB5n`5)%tS;9eS;!w3>X8E&^hWZB`m9Xta9rarXoxjuJr;4~7hCzmf z^zzCQu2(0|5!|s}h9%#im`NE;L)^4eTu*gLe0{eiB^$5;bur?6zyC0}Z`Il*GZ>3X`=CXFcyv=#vNe9#quhRX)e2kg;3@5~spu5TM65$%7)U;J5IQs&xF`a)$)p@26u zh1FyCDNyX`4a1tt5W*echnV7fGhsv0Zw%^ZdieX26HoxeS0von{N>~eI=%^Pa0378uM4+40I@W z*Dp5HSIcd^0x()!G+#(u6F2x5{Id%94vXJtCY?^Pj+AmMT?~VfJk6A*ktw-UN{rbW zBeFVx3F6?%ww{h(*uWQJG^R;a3>bhhrR;*`Qq8s4<(t@#Bjbhglp!8GK!54v3|dv7aIk$gRp6}N z{bLEWU($cfyH)1gU5xOvRki_=C-K<2Slc3}##mp8AvQWDq!yskhM?W-dclUnm_|{l z%8GCJ_5S3J;NB5!w0OKM#4e}k+YqvY4{tZO`_@cPgs5OTwW@w^And7Lj{TiWr1r!Y z1=Uk`?JD)1p$^yhm`;=?;Y&99>N2o{z}hT7YjIDUeJWE9`i^WWXRrJlV?vemeniy< zr#IiSrpkz-#!(ozm37pE?;jlG84NfWv4eC0n8{TM=trj^7t1EAb~@R?Z_c* zjJ$lrL-qP&K8QcNOFZmMEfLv*UILY$N-D|GmymV8rtD#!Gly|GXfN9x*Rn(g{R@ka z)K~saWZkGaQKHyiSoCKZx(5!*QXH@L9KI5TqT6}k*5Jg1dc!9$XkYg~EW0^QWF~obhQ?s-z1X!?wk4(Y6PzhJ%ap(!W6HwB&NmC52CWDU)d)>@bt^nd?o7n*w*J6axk+y0 zgzpsg4(3AFlqq(f$VAD>xFlQlPlxI zc3UaMX+PbD_)kYHTdhABwCtrt`!#*6FBJB5d+p%7a`5J~vCo9+r<~zEY6vtk*aDL& zrsFNk)lyX8o3m>wBA8cEvf{qs>D5EYzu#)^YjOgm+klJHf#d^uzVl(en>Mp=^dU1E zSpZup-p;eSO4RC&yy#?0VUH`qI-8Kj)`XpHwN+l&=~HIpv^rbS)Af))iK#KkX}`Jk zBzsMOS0~r$7vIUNcOIH^j6Na{>?v$L{)r4iwN&IKxU^V-XE2KKq=J~ad=(?M2=$Ua$&_7*a!!cNkm=j1x znIOQKNJJ+pM5lbf;%6)>2!33g`1JU0=^81(QzoY0t^>7Cw_Fk`zmmXWvAKv zx-U}>1%?* z`Y?4Vq;Y?J**37`Cw(b04(dyQ2Ph%B@6ZZ{?e+rMGYosg+vFuF90n#I5`>WDd_SkP?m-nw zF2iY;Cu$10e(Eo;4y6~dRJ9Pb(vhOo$xK_PmJK0dC+9M2p! z0%o}5uJPhk2!v*~F!_MPs^IYT5vdv0mr%KZj)LcN(!!wAKywSkJ|geKMohquD?da^ zDd4eq>1k|CuH2~Sd0&t7UUOVkjE!t+1Z-P}H2 zL){kD$`VE8Wtr5vnwph@uQ1;{xecl73-T*lx#?97g$* zyV0%~$hK*wc~tm5Rofu>u=eSaF%OOvi=vI`*2m3Ni(`?_N_fJ(kr*ypgz4rLZsc+N zTmpvwWf&e2>-^f&;%ei?I?tv($W><|Ss6Q#LaY`v3c51|(MTa>}wj$08WlF2$Jsc-V4I0=)Xp^}KmVUu^2|A(%W8gK^;9CR(GVq^N! zh#CXsPziM?WNcZ0l16WvE$OzvG_!S|>=#JCGH5km+Q^aNU0x7zK=4JJF~)WW)w0d1 zx@TPt^7V#$raJ0-FuDEiO2Is3(pJE&xZwsch8bMm3DoY8-%Ip*i61TyI^}>4gwBPj z(&TbB!`P+(lGhxxtC`52iEB@7p1li@1iHr0BHR-2VF-?iMS2C=M~{#U=A~J$D$pD{ zwTSrk^oW2l56#NwAu)mVCF}mp2!=CHM4n!)Yl0{K@S`9VG7YiSL*T(%F|o@&D}Emu)w`qw(U{U}`$LHdE`jDX*iNmjv&lj9by) ztvMX%OqMW)5vjO0nN>MoF-YIif>&4HXODr|TcCK$4}Y1yPK-}q|3w!6qXocUx#Z;- z+enlaJ00lZW70^^L}{|VmY(UFEc~~B-~S+?SQKQTQv5lDYlx&a$3Y1A%5wh#?E5zK zi1%{~Sar@8UQTxlB$!NNay|ZsywxwHp`!Y5GRKC%W*Fb6(}iux<5dZxgQ((bfp~=A8Lh$#B{r2cC$Czn2A#cmX zLEX!wF=KKe9$vL=z`*f|YvPwVnEA1NAioAdPY+wEzu$8Ca746*bD3y&TL28EU98)u zvcHsEx)JuMqmcw0O-RA~xBAW}q*R^3uwJ)U{P+kwL;H>5BW0p`P0>5!+T7N^L8yIR z;^p@R_YdS&%ieKK0fM#WkFrmX_%h`DW_IU%zpAiSmPmZ!UX?bO0Y(aZVT}B@r+Bqf zZy)ZTfIvEg8!B_r)AaoBPRZ}DH*lO72<_|+nQ8h|%_rV$PL7=6hq#F66PKO|2huG; zb6(@PNv$8~NPo!fv)1d$YFIwju_hX;>))up6;k5!w~;lP5}1z=o=L3YI5PbUZM)w> z9u996(BWUJbGEZM`Sj~k2Y6AHST6yz@nbpDpvP%fZ0pt0x14R;zks;_Y6SxPRXtrr z^cezglrnuq$8xXqL<)>I(W!G=4E1^~kLOGK|6y?(+r%9c?Ntdb37UV&!G$KAF3IEF3z*~caQw;91(ZqL2Y*XQ>eYYTwfY_7WwHukcI|}$6Bt2A)ONF zevPc~_a1ob2EPpn&z_4yxw6!Ss9VZ9r=xcPQWE(A9=i{tQPgH$BFP-oA5YH$!&4i= zPO$xuNSHF1_O1L2CG}yesFNBdwr5Cy`Y=Q4L$Vx8@5otG*=iGL*VcqKZ+@<8$M%bK zU&{!=)DoIh3^c`;oB0sgvtdh84r+(->`k3ap?=ov0%>etcWU7Va_hyKy}z zSW^|3J6oa>@DfCG>_BN8us(P?a_zAYJ?N`Xn%rK>h9Z{cBfysUqcM)ydA5ejwg$C- zzi#4Pw(D-l0Bdpl*-P@v@Sc`lp;;;%OPhj99AQ&-Lr^KFm&us^pJ(ozgZ zOXKesCmX?emCEgeT%k8)XK-R&N#dZS|5<{RXO0_Q@cv^YP#p3&QTaKuD0H{_+A9>np#{Z4R0`?*g{O- zT17+b5BnBTVOgygx%~?z^cm_!-(|JuxTQP=E$O8+vYfAbI6wb3XRQmz<%&;2QfzfL z*xc#nyVc?%(_Q)nrWC{omerRXSP6gTKrWSzI^X1$_0W*KirO0cE!Uq`DV&;bx2;qp ztR)#W5F;VtfImO|e1$ULE~tp5J`P4q|Es;v>Pr)>7oY}LqM|XQ3og@K&_h9<{945mLu3I!3 z^z6qWxHD|KGfLdL{Zy0ke03pjQ#)?G;!mQoBix} zpovT>hyP(yjdxr;KIZMi+T_YybZSl9QwlC3B^mMEJgYM=3oc_r&ml{C*MJTlUo6!^ z!Uc9EF8`cOk`nk7Ri8)W3$FF^&WX~qmEX|kgVLE^Y@i2&jDtsy2Hjasqu_$j}Y4mHu@odjBgA! zJ8q_TI`P^550;~XyeM>p0wPDeMyS~{Ki7_%lk)^hNhES6(37p24@vOVoasaTUC%@3 zS9$%uMg^)$7c0@x!EiG~Fa%?9q^3%vOi-6+sxQD7wbM|O7PT%kMCU!Ve~ery$>D?Z zl6BniqT=!8DQ8S%;=)ybh&<9geZB0lRMa{!XNfs}^U{)eje&a;b@?y*F*e%CYwl_7 zf|KTJ<+^lDzL}2~frLZxUJk0(<^@|qAB!z7=6@Qlr+>_;hB3KVqm*IJxFr_her(2i zsh&Ty!Fu^miaU0|+j}kG++Jz!aeGkDAvs_N2^R`K*G7>dE~JI@4E9MYP}+V}X4jw; z(S2BQIQgAo6XJZm^<1e5tI(^rr25X zxb1fm_kJ4C&1Zs}iDd73@AXoDpP-Ue|K*2d@LllCvd(CPyPlOQhdj;>HxJ?8n%3Kk z27xK1>;O{aV~Z3-M22q$|73!3v|A5k(T@%mg8?B$;Dbd2`a2;QOz)q98fu0|CmBY)lpCx=U=c=sa!md`Ah5Tg4`mPMS& zr6l?XLxm5&&zxH4-3jrby6d$I#77OPA?L0;75P~7ZRZm0o|Rbu=e>wB8BL?HO%7w_!5$%~W9Bz9 zy}rkb{MCeBhiYzT`ic#tN=IZg?fPCU2n8k zdg3YgVCoepRz5tSnIyZ~RdI9RZ5|R!aeJ%2Q}aJ1+Q-*={RL>V3iGvd6M)BQ^xpP%r?pZVNYF zNvVPts+_zGbmsgq8@!_pHqMBL*6!CMQy%FmJK`gTVYM2*IshuC)UD-bicz8bopCt} zqt!FHwsOc_ciu0f8sL%y86(i|hcaEpwQp{GEZ{~-suH`Zh^2lWI))@wsC7-RWxkPp zxnz_!(0O8~c&-%LFh#FMyI5~iAEG-JY78Ps$Gpklsq>BwKS~w0=)qUMX~$);LXoi; zO@260&@d2@5i(+X<-c|SWqvTkJv;Kp1tsj|&WY{z4>d2h)^!e{?{HnalLQy#33Hqu(xxwcxuyK*5I>#;DF z`*nHbBu)QJ&KOv3dQpa+wEc-Xq#@Swc=f;rklo2wFPM^p`kpGK!c=RP-T^c!51h*wRT%Lwd8NsQsT@~W79;4q4jkaJ*V`}tPvbG-90jtva(l+35IS-* zPfwPh=@M012JI%H*A38pugp1=P;vfZcl1B5hZt?yu4z?5rttn&KeDC*AKz`A#qQ2Q zm-oVp%~>B&BOHsl$f|K==heuu?9+nTS@XVtOZ#m_K+f~FA}uYIph{?cTinqR%^#YA zzfA${bP0vU-%>~@ic%}hDPtnG{%}YJ58;$UqnX`ju{6sJ0l0!Jgr}EA4c`#SKZ|e{R=2{(UYH@Tf?yK77K+5SDhdM% zVJ#aT4X*s$TRvRl5v;49lzo~6d|Klk3yoqR7dP_=*B^nxS5EAU)y>PfY<(Mr5vq?r zSup){@Ag@^o$O!%K!qV;^yk`YWOvlk?B-Xb);B9V?v>1N%Jw4jam(dER&EMNt!bnN z!iFBKc#0z$qQxuM<77NAbrnqF*T`0@?dDdpkIP9S78pR)jSNx){Of@YfAj0yJ4eVS zs)t*=@n6xsAKQC<8w5}+VR;f|BukA8sp@9e47K13UL4{+^SdJ_S;<*$FXQxP`Z3QO zBy=V>^=VtpEhN1HAVwuiliRuQYLVICWF`j{$l4kOR}_lmEOR68B0qkyWuSbW3#Uzx zsotf$Ud1qhY<1c{^MFO;a9~@?Br!aWhUG0GPf|$yNza)(ukFs0dfrizG_#%jDu$rr zen-mpB|LZ#m25j~&M1M??9798ZyLR>P)2b@*SlPZ+~Zg~!oFIOW#O)AFPN`#Y> z6NC3a2)Mx;59fdaSYOBe37zWlnwpp^R&r10D>Iqk+u%urrVq2<)pg!-zK{QX15gTE z`WIY*EU1EU!;qD@FyzG!#>8F5as3B=h&=Cj$;nMBk#G7gt4Pn#)uZw(Wh*~C^f0?u zY8O==0-%vnv993}8#>m_DlL;^O+ZWO7alLvKy9(TeuR$*Px25C^@w>Z*CZ@$TW=VR z(Nh4Z+`&qULMg`?nuKkUgvWRkUrCcHNq|X!0-YxBZ1SN^F zXiNU#R8xV;sUjg%Vp z1578}-o@(8rz;SN@taR=wc1LfnTM>waL$w&s*QgC&R6By@dJ_{!wJJ9QJhO%K%3q* z0v|!BAg%pB7nf^f)wAR|*$Hn}?&sL0H#OL64hF-A&9jqXxn4CBQSs~u8Dh&!*WahU z-a;gC7V!?W-9c9p#X0cj8$b>kE2#IKDxuY<;$|iR;1Z*{>ml z28{P{ZM1#?hs7Uo^v;RqH%LS9Cpu;hpAQ1!I{1=KUOu!8bc8FYVLrzS=I=>HeYZR2 z$o#VvwJFMS`0f^G3w#^mv@^AB`;tNFD#T?oXj;ShX58_9iBb1)Un{d<;};XcJSzG9 zac#sIdNPy8wp$Zz$sZ{o(lhY5NJ4oe%Es7XnUao`}iG!9~Tz zPtn+YKnkgUqC;%S)0$ywxy>Wf1PIFwJd;#rD+f(hJ)QVP5UceE6$!rCQwA3c0V!d- z7Ng$U5p!e3${$bBe+#E7HTFqCVPPYTSp9Ng%UbZAgOz?p!CltHYwa-o`pT0K+tHy9 z1C8bQgl{rPk;AnRc!<|ibSOG?rq?^t2bDCz$vj^=b#P&~@PWn*3CYy}tM-dk+vN_@ zysm-+?5!&P)kQ^R)#kL{8%E*uM#50&!`JD~>Kx{b9{fW&xOUhMrEc-Cj&|bwFHkb# zGqJ3YLG(W#x78_3ef{6|KHOm8m1$uBZDL;uA$*Q}LD~*(v>A3&$r*>0yy5E(Ai_{5%SWKL$=UHeidP0x&hn{Hj%6u_jW1Xq8AeK?w$<0rsZuthrK zC`)jdb%cp%P*RwwMuj??kG)qzUy^IH4wVlEL6biYQkB7zR>{Dz9X2mcf^a_lt^i?5 z)MBOXCbm@~@F);{*J2`2$d-UbH=A6HEo37)z}}3|O!`$Uo8RXYF2W8SP|1l;x2QN- zf9#v!KxdkAcfvNmphtkS37V#;=^M6HIiGwBPr0DVPcVo9kZVW#Mz^hGlwHPRuxoAv zl6tb}xCep`PyMbR4MsXf@-}^B9$5Cp$fU3Z?F8Uvj^7KiR=v|K2PF?C?>;@MTMpJ^ z1a%^dn(B^tq|O_vLhQ=(AnQmhtOM+!`s0Y#x-WO)lMsRa4B=YOu!he+af{FMm2PFo zP3t-^dVMukiP3`!;l5)P*clza+mFol*{%!rES;M__4jWcu3o6dPxq0Yyk}Ma!v4p| z4>JNiCEm@dTrYllhvZzF%?QXeau}uemQ7;|y*=LMPK%FNh5CDJUu9rD7ph@#&!&DX zu6d|wriNrC+SIQDdN8qB5tf?lfC|%Rf2ApnxbrcQQ@-Eu(Qk^bzIc*b(6)}+jcVDvFBU5daC+j#qE%Fjd%P@J{ zh@En$DDhkE;3IamL-}><-?UOfTPuBO4pg(7!r*%pxM z3>ZYsiagVwUJnf?*W)4y(VQI2gk9XltSrF9!sU)1OVV&LEsUHUn*$HRwVb)GQZ1D; z>-9c=eHNRRVXiUGyu9K;UlxLulmt2RBvy_HoiC)fKBImqBQrW1%I$;YP(e8N7VuvL zOG8F&iqpLvg&027qZ#|536_=MdxynKPcqkui*)#4)$BpjW2C@MGh^C^L~RZt#;%~y z=8A}Jj6u5tS#*nYDWqo6-N_ovD;oShZU-7N#Dyy0GMIR@XEJ?RWCSZ&Nqxag&&QZS z3nWHk_HyTX=!MI^rG!#Ycy%NL)N|}qmbvT~ix?tR5<>|A$!w#^;}TFCA1^1%8=`-F zoPqD~yk`~=bpM5)Y4>s-l7HNUqd%wUAh)1`1)|$XRX%##NNw6O zAfkyP>CX7I8v)$JIb6E6j4ba=eK~*owe|M6ikGagw3qm}p3h$)Z%X|>@Gwely6Lyj zUiuM5iM!bln#2c2AV3YB&4m<;mUwrC!o)!*EH)KxV%OZW#Q?e>MsI@?>7V^%3p|E@ z&o4WnV}5_bxyBj$DkzvqnVB*gcFD-i!}j-}UrpLd2(%y*MILWx(Y?-L%yO^C$Wz2^ zw_75QY5&XdX4z_SR{ce=gK#(%mpm+`H+~`qZf}!;+B%85G_|kA$ zX({1PS?#guJI5qGFM@zI)v;te+);P!d7=nCXebYhVGl#bftpM^el}PU%J<;p_IhW= z13XSa=rTM$R+^w$i>Bs{?-G%v$CetzOCD;q4wn92k2nC^7H34E?VL}1?hxI`DIApH zWEL6{xrpD`0?U~Ku^bn&l6m9FS8}>7C|J#zuNMmSi%b2pogxg@ua$6*P~AUAlB3QJ zf;pio)-GqLJ~2nQp&>6*FzLH6#g|Te*vSfB*@nhKSlsch7XQ-@64oRrrfiD7?6{q_3!FlY}4fd0_ z0^q@b`&m|yt3m<>Nc>{x#O98ZwH*5Rpx?j{zCQ;zk%3Z*9;`4{*y0C}Rx`HC(b4RA zSthl!?1C1A4e?tm@v+!+zJ$E)i?Myc!j{dXtO9kWE@hav&kjI(5Lmj%Lem-9xQ3b@ zb!0FOa6d~Gl)1)I&#B46H8MRnyE>#ivCklRSxc2W@vOmAsRNtrwjlR;ULHXM1S?J1 z#6Y_rb~*OC?j#cOu9%Eoyz;nF7gCx>?6C~^3;|d9mmf>ij>ZkEWT_1Vcb##MEs zpPZ)4?QNUB1vU$GXFKESWG;g1Vb>lfzQ(-uydSU~Qm4fRUtA7SG9b<@0eG@&O}4fz z-T`wstV^xl1nhvmX~&Bn%g^pz%@}EK8|VV>pOZ&2HM`ZD!@361A5_*mrMy@JX+A_{ zud;hVq{NYWrg=l|-$i|5bI0af&xCP!2j96WNSt(yTFe88HaDY(e#?Ryfa|UQg>jFGptG}YSYOT_IYo0{H>2|^{EV0 z(R$wZh_A#RqOz9h9{=cBN#p9$hOEB8+wAr*StXl^yHUT*1w3xA5k9LD~ip{#kh1S9*8KnGpCC zH^Q1}u-+M9P-x?Haqd)}P?@TdD!8?0Psv&T9z$k%wzn2f%e{@!tM|^0a*73yM10^b zW+!^c5$XNso5mr$w4it7i9(pUhdS4pAp4f^Ysz$Nf73moOkpJP<}!R`NA-jAMOSkK z<%Dkc>MUC5dn55RC*{N214X2_>+h1~mTe%|q8t~Rb~(=~W6P{5OXN1E>p9(RJqw2*au(bF90^Ao&y@LsKvRQQe9U}`wE zeQ|=d2zr6y9nB6P=h4dcnYU<|&&AfB3mA|2 zw0TlyK6MRw;N@v=dAbfFp=fYf0NLkqGBD6*e%hYtAj5B*r}x&q=iqV5Sm$c3d-*uF zUm#B}buef7U7d^V3}~1%bULL01s})NZA0vM z-dfZ1%Sw4YZw6qk)k?#ft566|)Onb5*f>QIZ zRqI|tkc>pMsk?9PAB34WSivJ?Ww^N6gDtP7U5^>tM!RK@69jguCm;aepSOOx(|}!{4?J*A#u8>? zoQer zIKR8}p3d7tukn5)(aQ{z!eMW=@eO5n{v1-AIB|DsdakHhAKdxIjSloOkIQ(cW=6EP z-ZjquuFl!=Q?s_kE<52obo{%TTHS#gL`P#r_FWH*MTyT5kEEcSwKZ}+d04fA85scN7giU?)4S{X)|yHe zmDJd8iCAy*a^}*g?#h}ot2QK@UnjHne_x?#R$QssUXZSb)QXpORvl#&zf0fMva&D^ zi&~-u`b3r2_?MI<=v7tMHT{u>^CaPZZ;!vmQY$IdynBW0QZ_2Mdby|fUJpG7_5>o* z3osP5e@HN3ErA+<*&i7_+4jfwjw&lJE|>*n*$Y1c?r=39_-hIVz|#v{q#?Q_KFqTB zlLOBv8jSqz|GuQ@t^HGlmWYGl;jHyb2_X}MA}!Fcc8QRSf$JTP0EcUDe+c~N6WdAl z!o}b#HZp0qA#sY#x9lOb?KYb>k?QN~!9q5Br8C@!^3;|Umy&{SVX5h=t!Ui(P%9@x zOG}$w(2{~!Pya$}uJU-+YFn(V_4b-`(rQp*&0(SCuC1scrQ!U~@|=fOEor`)n%39& z)1*qrgC7YT|Mk|=6j;+?D^-?ZR^S)N0Q&}idFauiIb2Bz(qPRiJ;OMDBZ(531Nh+s z*1}-=vYt}LatmdL0pnSWIIo5n1wpN0{mSf{q$j?{p!EQAj}EVA#ijICJ1WEDk#7w$gw#KeMj^!EGNUu{iF(P6K(u&w-S$VsA-vPw zEfUZxDtG?o0QkQF{8g& z%hbj$`20AAGmW2T+hVlvijtCts_Bt`NRD`rdgv*RrfY68Gxf%lTEIMSPup~2!F6-K z;I{vj{gh{p9xgYlxtO(LUwQcLCtSX%su@v``k)I}Ce%e2>y5e1T%r9r!hMTAS|nt4!Jp2fe2aS#&_|$!OY%{ATgspd%A~{Dhy*A>m>? z*gEVB@5tO)gZFJu88EEH-p;=_uRBe-`f~8_vHRUSy6cZ!9r1dTb9kTa1*;RXUG7UC zDNX4I%vH$2sar<<)lDo3FwgnJA5mf5w|augz-1aNQytz_zBYU7GDyiK-XsT_%VtxK zMyJCuoG{GffJ8F*rhfccod z1xo26>d~DQk-*fm)W~c@Q=0>p$gbAbElnSOVCm*Mo`Nk=lgnPgMYSupx_>Kb1w;Jz zl8lEJYX|@`Vd#7*tF9N$bZsM?aq@87@D;_s$c}#zX&*0B$-yg%LMSWKUS=`S?evUJ zdD2lAmAJld;EY_QK4Yd#9#K1N)X5T?@=ToF`^q@!91d61|D$tcXJ?Nz?d3(qf(;CN zN~kGgOY7A$-4w*KnD87a<~%B~dC<*0P4NsB9R0-%wA;w+8GZobUR8dpGi&mJ-=K)x zy51`Nm~w^2&n=@dk07vKeOe-u^gUf$dKor8qt|50c8OKd{nt#gx6wdV*x@yEMA7SBqb)^ezsg9m~6M%Yc`#`uCBy zdn!0R;MN3l^-Jflsg&0n^K_aCn$tPUS=de1-Wb{yAcCBieq5hi>>m{G|FNtup5=Ue zdIaHjMZzfXd_VCG{z$Ccu3jd7O@wQ|j^g=PXbJrho?)c^b` zXyN>J0sTjlD`|7{AR*B|^=3JE4;$FO4#nha)rQ}zfyD%lEt9iW_F&XxrMQP)XQs0! z1j0soDfCeS7h0#+?PzM5CQmAO>#P>t>pzu|(}DL_d+$?h+i1s)R4t|He-VxUJm;m> z9>GKmKu~|QNr-KHHm_p?GDg5NmKtE;<86q~pN*j&eeq^WIKeZrbvkq4GG~I-BDXr= z2f&`}TmuT8R>u01euZbN2VAvdTfGgLmjTdl8C^O{U&8)MV`sl(K*Z$Ik! zDQaBS*jiM-vU<#F-(i;AEwqZUkeZ-Fy@8_EheIE~p&w&2ngWMqRwN}+^Y!ciFE>Bg zjW5p#2o@JV%Rfg#AcYW1{Cu;Jc&Cjf4Ab10lK7%>LC_FGuW-8+x{9sEkU9`yd>TEF z8A&mCm_u%6$0oXLdU11My5#txyt2%jn|!^@b5^3@ zlLg{HK!}w#iNcXoe1X!%F=xo|b`ZKtFtJ=9%yx%s0)s*o7(VgIfYZicr(B)be2F#S zz->55=LPQ*#$b^*|Hq3t5&bXrhI=uZPLH<_)z1i7=V;?&$y2|ar*f%B$xb6oHL94) zD=No76_#^XsEy$M8D_n212LX)SX<}A4et{* zdj*I^ox}y6NYuH+~I< z(mlL;;aATjW|!lRo-|%nH9xZ&4Twm%OcYN>JECF}qmw87{LRqfi6 zlkK8-0)|*`6NbrGnrHdPXJ-+KCg(Cl{U(Fo?(ONqB|fCpRweWgMmm-`3`k<2PJEWzVP^ojq92XM&j@lVOmm z$!ov3z%HeH8c;|diFsr)zOrCR{lk@eh~8helsjcwv2_UpM|DovzYU(NCcQpr8k256 zvf6T~bNKE%1JO1?_b(|@TvQ_^11LhMFi5(b~LuYry zD8Zy(%O3dG1@uNHQ$I}3#{%|#*Md1&Wn!TJi%b6RYu;`Ke5MkTaeJb*@}z~5VNb$n znPGJOB%FR@_1Y5-4z&KUR|3-sWot>&H?Xh{+(2h&*;GDcHFNt1leG=&awbq~IaKO8 z_pW6^Fgqr_2z`X@_rFxcZlFbATC7SziS6KgE>Y#QDlk&xOGY~LlRfl`fJQbYYK(A; z&fENq6pA(Trv9S^oIn&m>gRfj3NyTdx|1eRn}JyWQJaMPd=m2csl##c#S`-u(Ga@r z0lpU^()Q=g!`#jfVG=?)D{Z#d)&r%^;l0TP_szK14>_chLzEZoL_RPGRj zUlY()=X_(*;JmkxPKx`RxHC?Z-4?acU8&tZS&{{#TYNKYloIzCRHS%ejhWm$yW1|zqR$VvC$L@0wrvr0in9(rp=z(6Rr_>k+i-B(_l3IAE{_(KP zy~{;TqsH_MNx22ZUwgBG8paW3G1NhdaikqneefGB?)UJOdxJV%p0{OS(6U(?1&)t| zz&k~C_boh(<2ODwxT=_^oq#^_~(w*I*}q z96pakee1y?p`kS$85Q-*O#z#Wk70jYa?qi{C(V}raA7?sUw~kxj9ATHjn^fTkjz2a z5G^_zerKf`k`JD=4&R%#Nv39dLxWsq{`-eu(gcsMY(q{$a6^A}AYG zpl904ZsX*gsk+hY5OMUaAL(d4Uf_K>yqGSk{QU`$zXgy#Xy@Sod)kx4=RwnLik{5> z;$2`R5SU#EVeJ+8+k&ax_gPXR&UhW+af{u1(r-+@p4BA7^Cg=hqtJ$Dc3JL&IR zfkmwi@*}&9wC*pqw;g++uQW8f^A2V8Kc6X*n=UGDn(mpwFOS^xmFyR|f-$Ny($F%% znC|}q13dJJccj7$kbhw1cV8QQdBvs!p9Xd6>h4bO zi1t!@rG(`_5AY=Hc&XSja#mp*KzvstY!CM*Ha>fNukdFkLj2LfHqi*+4o4aTDNUb7 z?&%DA{yZmNiD0*&js8H#TFk1%XGI%L{{A|AU)7hj+WT&R5}ot6TnSm~6Rnu%1_F_ua9g(fRDNqL4iO zK~}=Qqw`77agM!c=)vSDU+p@((H;wI&N~O^KomwU5mCA~E3&L}#V?VJhJJstBI&Pn zxzB$r76kTV;r2>cePMd!M?Ai`EEfbbD7=S@&o68Un&~LJp=5Q? zViGeOtWozzG{tPMF{ZU%Dv-&P*F&=5rq4JCYv+fK#tso2zTQwCLh>w!dooyZ_+A_- z)bf0yzt`;JdNnem>%JfuzdIyQ%&J;aOB=3%T(A7G|2$kOq4+#ds35`pXtWCBXO=ux z==9@PoY{6Wab)2x;A7d)9GCfj==#dAxV9u(BtUR?3xNPZg1d(R!8N!ALgOxtyK8WF zcXxMd+}+*X;dSoJ+?jcA-Y@#==A7-dt7@&PdK!X%AMru3SdX2gOLa?jdmRml;E9ib zpBf@eG+_2+h}Pmg$+vZfCCQ z`MUziKb2Rst_xP@E>S5xD^H2V(V1U3&X~NqXx)aqU|ST)Azi6+MerPYLJAK@XuR|> zWCqGi$4Cxh2IhZrywawpIu(7`mRzl!P;TrXcUjn5jsO5U|sxo*#oqAZ7PC<%9;en)A24QNNw*S&_KOvf9ZY(NJ17^3dOa?)$s*RN8pE@f|ZGbJRRO%yzHEhyE4800(s>zPL(YN7eAsi)e%lBnytkTU&ik;KBqw_$PzH|)_Cqwu6bXCIg+F&_GG&H zz1r3@4WkP70%|iC=@Jl+Fv*pMLJ%q@iQSfHJ;>X|&6Gh@bn}nS*{!PvGVV26EG_`P zu4Dfoj1h4{!Cy)gi(|M~_n>c5?CT?}2(zuA{hw}*H`(vZ4A_;h78|xJZ-WBnQTE}k zg$k8F*FzCNLmLW=wWI;+EK@q4TaEE%PC?5JY1$B{tM^a%jLX~sz7d3gr2Fda75;0F zOQFpukZSw)?L_#XlB*jr9sRx%;-SSbbXCcr#+cKr^}7zDv}BU6+RxdbH%o`;VlGWw zT)tM=Z@Sdn9l_lKJBu8`DPRmY@pA&R;(WC&)xwyW{_MEW^PSI33@qx77thcF$rBze z%3?L*Es<8o?n5^oUd;DVcnIt%Q>IYuH0!W$B-m4FGg3>K5wFS~FrsG>f}{4L@Cb$U_qYi6@%G|iFGVazp%8;qIkP9ger40|8nP0FfQeqOzQ75A`C z48@pYLij-pt8>Gxz7vY@7thRyhv3T>m8_aG&&-pE60NMih-vEHJI<69 z*p?RT)BTyfb7P)mS-xF1av_)01>;+DH*u8ohBlwDB_m;GTA^Q04)x~t&+o28k2F2p zITjmypfuYg0$@c<*UBm0@ch_T-#HvuG0C)h3ff`kGbWX;uX=gPAmruDl(^8t=tl(RIO08}`n!wgj|aG$-+CpbN9H`FqsDn|Nyy8FDhVIX^Q6w)$7 zfb7Pp0Em(zXg{>4r=(;$u6mcZof?ljZq11YIQQ99euVYR(cz*s=;Hl3`V z;I-C{m5Bfoo@*wu8X(w^>t-OnAUKyoDYL+cuwQFIF=^)N4(=$?QQ0!h;k^%gFQgME z;^6KRgMv#hboUJ{15iP=0%B9KHSYOwW?G9}eSuZP{##J^g=Nvp0&=y$ub(lNO{(L* z(3*V%e9pfAI~ekBqoZ3bMLfnX44_m%au|&=gb0p6K|&1! zJa>OGdmHf1T%+(`E|iOM4eh!@E}N?mM;cR;ZgSzo{D>UmT9af~jew;KQo(l=LxX%o zVx~p%N2M{loivw;dA=eWy@##frS3#V%Q>2u+6WK4GsLKbQ!t-9N(D_0V7O zVd!qeKqY*pjZY6n65!kY>UfLoJauHbAqRbk|8!$!g5fJl*mJ&59eM$maYy05mIwoxpE#9wC zGxwR69Pr4fyNqW;8_n2cZ+M^Z9?67c>55+N=SJ(?9rLeO3V^ht+LJTJ0`@BskwBQ& z3_QsF3rt9K+B%&7rpKyOC=gE7GtF!DrX+e* zmh|^BPMaWepe7A{*LkvBTLK$%TDaX!zM&zpc&v|TqV(#05N<0%ZSo)u0>6`xnb5R~ z&NOoQI5CsDv`B`Q5vh`dSKP5@He-8p)j`cSyz1d}-RFTH6n&rPhd>GKgP$4;T8v|n zf+tn$e<4d~vJThFzLWf7ya2d9IjUO%wKQ+b2d4F$M;3n%RII?+wJ{n-Vz-d)R6tgm z(<*O7401a#*OO;D)95~q{cLZY%hDwDm|C9Aan+HWPjq@}Ha@>_#~`}S z4p4OH4ATz+;Nipg2uHBImk!o|{#*Yth^_KQ&G&d!|I$HHURW~I{4&Z!3duT`8p?xuOu6IkPuAby-$561>BY~Ol2A_KciIu&Orrub;)3R!utkyG21i z3lVlQ7#P}Ep3Hq>t54-ln=#w_gD|_NHb>qWI8@jEBQ>M@i?#>7mtP2ORI&?bBJ-bC zZj#rVxln@4ww%D+9FhC6?O|Wa7gn~mXz&aBYVK~H)cSEV&Q}^GAX1XZKm6-%`&(v$ zi5b@BQGW;5|L6A?h|&MUvjeUcsFU!Y#rghiY(IDX58daFw-1*R{_h~v|GZ|~8O6u% ziLWM~)^qQy?ePAy@qcu*hcqoD8yand!Nv*;?7`NN8l|qq`U4Mv0U?xdsed>9uTS^- z;Iv-tm|)J=M>w%c_u-{KyhmMinm*T3zeW1|t z{&HaN*vTAv3T|}>RgsdwfUnCGF^};rHCh$&togN*e9ol5l^mVie|%%u5dZnEuti^) z5HV&oJ8fqM_1Kz(t0AB_!mjmNw8(MlX^n8+_MAc|?a_Y>Jeh|xowApV= z#;tR)lQ`LZnjSjgVX@!U;h8X;=$VudCp0erI4nZ{pECw#1exZXFh$anlI~ayF~i<# z8=U57;jsXTcC2QfKuE1NUGwMh=%I(jKi8t~Fl|&+VCkj0)sevtCOq~4m(^E$J$3$g~hm@52z?wnFAo}wcSt( zCfc-Pz1N@-=P%t?9t2)|b5!D(%E~h>^fWpx=bcf0&6r!>?~Nx*HBOQI@%VopvmieZ zcG>Lp!WJ%163jlp>MTGfQw8=h?w4yFFuAgE&61w}((TB)<4U=~hi4MPrJAqR`j}W{ zhZ*!a-J0`=1?70L2ULJ|jgsu8@;AuKUIdvBZaC*7A*;pzGhc|WB z;rnCU#=6AW$F$BRF6W$~_ePbT*{&--UBK0iWWM#bQQwP5=|8Z$r2OS*vY>`co12*- zec>Jl10_WD!SrXYT^s4v;cFO!kWXRYv&CrAl}#iiyE7A?*1qe06GQ+vbs|mx#{2Il z9epwTvIX3s?lhdN7L<2a+@s71Hyr_Q^3xiR7vlwD4|i2;Rsc>~fzOuC9Wy&AkJ}!% zZ@`P)+b60E*&RAkSF5O~L(=1;tnO~;Aykr`WR@_tlJSeQH~kCD7p_`kl%hfU&8%_71*E`N_c>%KUs#dVwe@f`DWJu(@~bIGDC~(2yxlsW^De9%$rm_d-y#AhRRI6 z<(|_V@zEA^f;0OSsf{&m*U_@f^TrHEd3=4DNob-Yfm;U;%5>2x(P|FFZZGJ;l3WXx zpq=)Io%I7*xwW5Cn%AW2{jVXliOQe@(*)w=z)ejL-|&opblnOK0=TxKo_};Wo)e1ll3(|Z^|@$2duBRp|(a|8ny?sabh-DlsC?J<35+~ zXusgRde;C&(RnT%rV+|shgSu*(!kJGVv8&@?jUR}PVE8!IDeUfM znaP3viDMxx{^QiWDC19s_kGZ6F{~GQ#!cUt?iRRwV~BO<5R(T-8Q_4_l?xum+DrNv zlDiS2Px8Fg@j3FV^uhB9Ty&ONGS^_}Kp z(-xdm-BxsXqRp$xJFcMF_X*QpCFg4imTB0KWUhkt_mRn-Vdpso7B&s(VM$SoAoV13 z@8rNwm4%Mb8u$)VnO35``0qxoo?e)iS+VIH@{V6zd7p)MK@YC{d$yw~cPV2o{OQv` z@yAiGObsmPw4{;XI7VK6^y*@sRTv7PEu3@nS*As})#33-hVz-%?ITJp*17r&7gf%c z?yjTw6v*|ZGz(z&AS$5pI(5j`^B5T<@#Zg#`a*Jpp}XG+3Zvb3rj9(8nOOEHhint* z5ew1zwQ)P8a$2s23TSKhFLT*+tluHhcBw(G@P}>%UTv;hyI$(lIyZkl^XOn{hvs(g z;6OFs>lM04K?>L3joGl{@ZOo%=I8p62(?>C{vij>eS!3RmS>5}40;ySIeu}*dnJdu zNW?8uzR&OIaYIwgPWnN^@0hK8vh_}aB)lmeHG=qla1bRplQ2RiX>i|th4H>1g{vwI zxC{Du6{#MCCqYO;LM%4dvO+}Ho`2hHymXdpS`W`?7qacXWjxDYpE4%9=!x7TP_scb zs}x9jbu*C@2AAto0q5-O@} zQaz*1!6p}xN#*bm%(*!rqopKfINm%ae&tupJ&PloMd;cIedUgZRe?RobGjpm%~D8Q z96lcLi@dz{X_XD3DVA&UL8ILT?yz%c7B+;^7E=mV2T-3rmAyD*SS}9-lzBjtC z*z{R(@S4P!TVa4uUw-hIzof^))w&{lc>|eCyBU|rQ&(tZ)J)soA$sztuZ;8}?#YE{ zE$!wt0URBjxz8fYLJ@fhxWIIsJxx;Fd$s_C2F>-&hK{L_U4LbZ5JU)Gf<^~s>GHlX zV}~c=?Y;Euo*io;Nup0QelYBF{V?v%Uv`@LzcgrhP?=bhiGb&%`P@B|ntwU(suxWD z%&720!0r3GDs7P+6cGoNe6$7CO>ADV^d+l_^UHZ7I0M7VyXKRu;VtjD($@${+){Ag z9gG|6jomITC#@FTC-|&8A8ZSXOt+v)%kaZ^`5MgX8(F`c8*O{NhO=un?LRhj%$0Lt zW6Ob;(+0N>CM`{S*`gs@5|`d02ytjg;xk%PQCN->BCy?!cB8*N`lD-b(!WOVg%F>1hhP6@DV3%iF>bG>spput5 z2F&iVRUhw*SI)k7$qNmGpz(Q6^>X6_-bZ5g&?0db)n{WYh~3gL%xYbMTLqp2Lcnj7 zee(E=@4gpFH`6r;2Iu>nuy-hZ1wWWbm&ObUmAf+ ztz-7gp9;u~kKctGz7R4g5$ATayY=dNDw^W4X8~}Kr2~v8c?~GsUY_v8dK6gV10|HZ z9Q80vZX0*1y9>_RL>#}P^|c#D32T{`7RyR;9`b6(K6~Ti;QCYn*_wT*nAXKofm>Iy zQ$S#6l9mkfNV=$Yd=-r+TI;&3o9EF z-_X@YN4KS?9Aebkk-V@zwqWt|{py1RC*0a^mfbud%e1qAnZ;~YZ9ZqKBdYd^k9&gc z8ShOSm}X-6KHA(AiSX#)g*RL^UttYBNTqatXHsW()=TR}Y&=5&U);)X-g;rLE2|^C z9BpdyT49e$$<9)(NbNO(!L$d-5BV)T+z>>biHA+ArbF5(?-KX>@%D(pm(Kc<7P#sh zE^XP~S!NEuTxQwntFp@z&jXKGzM5we7q&Q@L8pPH(+>3|b)yDmk^h+};>JiZn#|#X zGu>=?U5bDjh8NlHcn*5$5∈>b>G!M|X7I)&5x-ORDUH$n7Y>P*=HG%7YOWUzg|% zg&~9{Z%0>W#Q7`};ldg3=vi~bmhG9lR021VexE5)>zk1F^Et-K(~Npo&s7mKQ9P&V z=f3h2zWtk%gL#6{G~T|2BE2suy9379!h$^N&(qk#u8SA@Ewxg|&%{T4q8MC8L_A|R zx|_vQjc=nxVOWdHr2Q-&_u+TSXRUg9Xt^gHgf-%s_Wdps3ztU3&3@2Ssl&r&iCms_ zP3fvSWMx_0!Jip`C(x7y84V2`JE)A15)#Cm(;tVl?mrQE1|tyl`hD^6gqPc-?%R;l zdR&LhXla2GvqsPTRL0-lu_0&88QkfIS_C;XJS-Q2@Y%jQ-pWt_SJ5kENmJRs2uD$Q z_t92^{!3O*W|N&DR7Ob&R)G4q#1PU^XmBGE63gSemkQIHABQBh?#AmRtS1a4md_W| zl-0e9a2l_L>|-}Kzt%64OG*-PDuHDMww$Kp0-v=|5OhHIjkS=7NF(7E)Lwih0$5^V z;p*gpkioxYCt%F{`O%51xA!y%ktqD5;q%_6&G4|K{LII@9vBn!PvM(uN}3rsxblbL z4H+dgU##4cLqfhZyXsq=u*%cZl!i=m-$kov>I!6Q2^E^gBvPi*%Bpq`h3`b*ZdAH3`jB9k3kGF&)V0|SGC zUcZ>811F(E!Zh5N_*ih`jjRt(vw1>Gg@~(9yUb?m(RB9`!ySXRco<4-DdoDy#Qi;t zh#Jb4y>?+O{L;Fvj zs4(Np^~@^#%vngY-=HIyj>s_yT5)CEaiqXr*K z4kA%OR`~1;wTNE?H*Yzw%m7oWN6z;XHk-uVGvV`D)C`?BQziPGW)yT-+WIQr#a)f= zP@SfJ9xRN$DpqG2mAFh{zZ6Pr$|ZFs5<2^IZ;@UFm*+LF-V z8v(ThCX8izv{GE7Q-qy12$-pj?rofz@=|*pxVb~16_$EM;pi4I9`P9M1ngwIfcW+A z1ELJXRMT=e+v@(|kzcT38adQ+3Wvf(?Bp37jBp(PrD zs68ewUS4+$D4C3ic)T9Y9v>^ezNyAjr1_=c9XL0rt+6qDu7#PclStz*X-RAxCpGP7 z>jSAzKM>4L8}lTTVikh59%+whU@RVwm54=Hq2gb=N`g-&o#ICwm3lLA6d=I|e|cui zA{dcoXaye^i}PrGQ?VI*(1%|(t5)dm2CAwQ;>UJ>?AUmXxLJFx>|UB8q_AvrMVGou zT6?+hixCw`kaP6HSUiy&P}^x}>eg#n$!f|Smo;RhK-KdxG#r~%6K+BQwZ=r^5tkoZ zI*9KEx`t01#o}U{SiV?|U3ph2!1=*^YA}DGtu`*E1Y<(31&k?_K~tgOVCaLu8eREypqN z*ix(nFnJtuxZ)?NOPfnMjxh`7E++4S4fNi*vR`RUUTI7{6eSrS?qh~k;<|mss$wq% z3=3hRfJ$BpsQCIh4QqRZg_e}Oz7mFe{wz^@dN-8T*kZ0CG?Arvs4}(~O7;RjmA+HG z9jy{h9Ovz{)?D&J2W@H$OSp!dJGT73iwB&uQ8*y85Jx{CKZv^~z`Ph|Yl%Loa|<`K zQlNX9nDRJa&7hNp+#|IjF$R8AE9veFVDj)qRZ?h62dHt>f>JgF=IjynY>^@0n*t*v zQSaZ4Z+V3m9F$jDi@|cC%vX>_lH4H|J55QGFaV?9h3PczKH`AXs^YF@ZI2h@I>2W? zjy^EIu?^+~`Yadx2Tn1%IQowP*gYP(WenCeu5rbtX~nzUZFmsV7e1E$&#2uJ^1bi$$Yq{AI0Qm3B-8!urNxg;}6iaNPzg zMQU~Axt14kdy5lJvp4S*jXx@KkalDRwFI$4re(Od$J-b4uV<*iOcl4RUZBz;zpo zpq3&wHujhF!kplzMa|PZoaV}rozp{uW#;$Lr}%b7pEWk2;$x5c8aiftpk_qgV9z2g z%&#nulcE|rYAL>0{cLO>Z0K-IFw)e6R^C>*jp_l8U$&S|rg-E5NFPh0(amSfAa7aiYDK+>*G}{XB>~>tcA5PzF9%$| z@bCjAPs}3p1#$wyjU-1O zap$e(wa4maXQ=GYgr}+)^hL$i=eZWtPG_narn6+C8X)1G{k2~e;xXz;OZfr;l97a71`;@tGOjwc!?7xjmlnX1?rnm6X zrLy}d(N=6RhPStNle;zPE}M5Pwh55?UyHby;3l^X^jNFBi}(ii8U#x9m3|j4tuDQ3 zL@_BUK0zDlx&O&3s{?B2k=o;TyRNyN(j8+|p|ULyNa=N*5V}EcP29RcJVz{vYQWzs z-Aj1ge4ho|r!bgcSpdmocZM%x>xS4i!rj+bA3tLW8>T)TlF!xP z1_M`Lu4}Rc3G#4Zir^<06;m^WaKEGj*9wNkmpR10Eye3?!;XEsW43y0A?7t2Bj?cB ziGo&N(>66nlhtvpUB%L=%6v^pC_P6am2vmIAEOs=7GK&jp#0{1^$nE|GD-jS2WyD`U$7EP;SKk@}+-$PI83ehlP3$!&~Z&J~wbJ#f46I zGiK;A4Alb)YFxi&g4S?)pVc*xv>&j4YNS&EI;H9K72=Oh@|XQv1^vDv8WxMjN}i~C zRAfEwPdrh32ETm!h$u#R2`GlRHUNMYjg-s6IxH$PXJ=-OBbaF|(XRGTgk7wlCHs#l zxVl4MBj*+vcZ*2rmqv(I40re2w9FPF`#`lJc)S^i%`B-bWDx7Qdu;>nL!Ca;@wtXA zu(>nCyl8pd*CS@aedm>=^wm>;fvV;oC^)4xpPPTI zy@7Sp*yF#3e7BsNe^p?g6I%2#>WI<4KRn5FrdLob3$iB)x0!LWxZfY&{={TD@MRB) zoPj1|v9ge3LRoOCjp4-`JG?}uOUW_p(xjM=1g!nb6>M6i^s3+R&kU&*6Q^oNWEsJe zpb1lpOQo_dB!1thjb(iTDD&e955$q@%-R#rTyaB;ukxfgh3FZ(8IeTee2P+3(J*NQ z?nUFOFtDVtk^=J<6?e&*diC?vw@kX{Fd)xO z$>uux2}fe!Z0$+9Q$;gJ0RMt?9db;ZM+PvE!7vKdnvx+R`c3{panHrLl<1=djD@9a zyaPg9M1=Jz)u`E4MM|f_Zb^W5?*n?)zR}WRyzN`-x8~mhQxoIi;g-akgNv4$m!HX{-|o+hmK5ODmO3N@GNmOOT!|#zaP~0%;YbdYXm#e^yE(zb`Ucn) z=a>`yC(KhOYlxH)i<&^3abY^V;;uVZow1udU$^hX{0y!|i+vjI8dSLgGIM^hbWOU| zDai*XXGot-9EUW5)=%Ipp%C0;sBcSFI-g!=}F5uUCZ;@}{GWr8rIu`|`wlJTK zR5NS39*Ur?ehu80N0_R>(Qa(zoAV{6PbEE&t%&fW&mY~SUhgxPuFpQh4s2r#<4`Ga znC+C5Tz{(-y?DNe2rkdtx1%Oudz-5{5c!45mq*MqXf*JQqy#FYP2gs#eC%H(XqWSs z2^4L%geeDCT2|aP(^=10pmV?IAH>hdXm2j7o1LJ#nGdkdL&x+l!pCb-$K*F5+6ey^ zSGFpCXpEUecpHR0)9;-t6>#wdnxKSJZoHmJufSPyr!i9$2L0D`1&<1hv?(_nft)Qc zDjM~Xg8bOQmyRsf6T1}7BCi2m|( zaTE(eY03x?ZMH8^8`=Sm8d%deFQvG!`j|r4P8XG7jhImV%lxJUzqyVFOeYBNo8sRm z=}1hdG?}abiNbc0|s$=*p^VbN9FYu<~*e9qV-#a(ISBAe00xHPTpv0%bMiGA;`mQ zepH-ryo4|U`6z_)MvMJz*xUIQ;*B$hd1o*lFO@H#wlIF5)r{#1c2V{AnSx&pXYed; z#>AH?>Qi!FHXSKw{=`sj(WaX{Rs7A9&v5R1x@T`dJ0URFmJXA|r1Jnd-KKRR@t@e; zraBVrUQe5l5V2S3X=&&`K^V<=Sn1~1P}j}(w<|bM=>#&e>2$NMKfAO|`qTdSq|9r5 z-}P_q+r@#d%Lj)P$r!@671>;-Fgx)Ayq2Rs=dAUt2hR#5=z2gu3PK0Rhd1-webhAV zb-jMCri^WS_~dF;gqyvYXI*E;#xpsV)h-a;=`K#E74QF0{7qquVUxkS@)v_l&oOCt z0^Ci4z>;vE69kO@86Jt*&kX?)?$gu7_+!rifL08Iwi6-U(KN(K#URrJeuI) zz*>jnuIwq|lX5l3T!)*-=@F4+%RXJVx{5~TD1V_XX)H^a(iQyXDODf3!$QwoA^xa= z&LQ_3>*r(Xh@DuWN+kZu6eVXSJu7MgRK*>a=|fxnj)MG$$4!~((@Lb{oXeXm3MoGt z3BE-H!7I7P8~6?zE4G6*9hcuT^H0Y9c^dO^Q%PR3V!8t>5|{9_F<<-A_!7wV@UEWG z&CU$;_DTD77MH}OzjtfuiVY6=54`tB5%4xYLZ7~U!NMD=KzHZZTkpP; z?Lu2N>V=Qk#n0Ot7S`!w^A`Ryl3iK%ps;jV;gQu5o@UwcCOB$sU4csuKl($;~dfxF1 z_#o^HXF1V4^2kt~?&e5~G`;}8@$w^CF6FykGa#C57_;|Bc#5+)0R{z}R;NU7D_hOd z*pg)}#^WCAg3eISABQifZM6Y|R9kXphWC0`6uTu{(^gZK|E#^>j_K{qtyihaxQ~o? z4NEtxv+INKPi}-|DzvFH!YqmHcf8!*$mG5{k*j$-P;9&AkMoF>z9@MiCFwjgehc&s z)rFU#Y^jw}Z$2OCi!JNWaQ*WVRQ%!}mAk(pAGJTytPe=*9fV>V0Ydd(cQ)Rx?b|qw zew#Tm4?KYzW!|)4b|)Ow4fSImOQTR)L=*BNS&`3bryc z%7DFnFEj6cOF*D@ud^Pmq+uy#tBpU_QRj@5R+j$m{el_Zwp7l;^$o7-W1=m4cnPW@ z<1AB3q|c@*ezsV&<*w18O@*P*4B?_UWd0&$gGlU)<)c7UWMZ z|KPC3EW%)HxT~kX96mPEA?e{lmb#}cT-S#)A=-Q{EWlMF z@yUNk>`j#~>%h+K=O^kP`fkV@qGN?em%9fb|(^!^OW(MAGd{&D; zc12MdT=) zv^w9yuKU4TL*(S-K#b;ufk4r!B{(twfVK-2SlwEdXagdW3AO=42o;Bywf^MN;}FJR zwO|6b#+iPoxtsGxF06qGMCy6{vTtf<^{RB>=Sy%9H8AUk7#RKT!@$Sl&)_mL!tI_D zgS5dfeFf6tAbx(ji>&e>1^?UZ^NI=Cr%x~;y%PNyAHZjcnfg(lV^#?qJA0!_H+r^^ z2(D6@m?%E<>Sj?EiPrT4Rs=QbY$Z!OZ$x|llm@K*HNgMH?0@!da1O$J@>@=?C2j3& zba$HZaZ9As2-HXyvRn@{7%ql;0S`>CcEFyYr&L$&`h)Y$;mB-H1jP2^Yi#pE0w^P1 z=hy2Z7o1g!bz`UQy5EKMX;~Cn7Nq7&>(gNl4_QxXp@~yd%*#`SY6)|4cmvtVN$Ia3 zOmMmpQqzG?GJjF4+=Vn#Gk!j@XLsImQ9J)GG5FF$Uq-`4{>gyUL*t4As3P`$!ZVHV zhN#u~1~A&fb$PB5sKe#6+EFu>)otcvEo9eS21a z_xwc_gB3j49wlU$th@wSU%|84Guu=83y?;^e#-?^5qWFVL4wGDwO6b}67a04 z%E^f-C@3szA(MTBE+*KH+(sg^AjT<%37PTk%m4y^_@W`xv0PEW`gnmC_p~e++^-Dq zp8EXx>CZFZE%uBm{)dGb2&-SK%BWNpLF?OagNFo*-+{e}LxJk$*T7}~NQ;5gB(o6u z^IDNOZ0t>CY^>?X=)}a+vcIy=zXtp@tKWR42*Do$HKb=4j_H-1@aX=$o*)GTis>;X zvyjIr_6rvh`t!`DhoC#&pT?IT|M_I$e1>c8o@cYn9Ku=@>*U{6~c-3&LhiJm)F4CYbGoAssUAe0Y^5Wgl_)BwHu)N zDg+hZrF?ef2l~wpX2QDlp%I+NaY=E2oZ{Aruk+wQGd$Zv>u`wOqGuZ$C zKpyRoboAmNU7Qx`wxzG|Y7Y+N_Ymqb?LIgL&K#AS1NLyiXvBcELL4|>uxF!zY@Vwf z`1*Y9jHyWePBM-MPgoa-obKMcLEbMe`Iyi<=AUIb;AO@@@8z?pR=tBb`DXy_`~b7> zC)_mmclf=ab~jKOLxklu zP#wvu9s34-dF=?d!>PdN2Kicr;glJy-8~znKJ~rWG*Nj?PJ|l&=cJCItz58g6j_H9 zND@H%>eii<;CZ-JW{f{nUGBDj98ccQ_1w$d=TN?PagJ!PDKO6>Cc_3 zE%BR!VcMyFCXX&P|1rs~LkKKlf*ZO*4b&+bX8)rvBza62{50WK0z8}+c^C+NPg@E2 zgc7?hQ}}gDp=34~ih8(Ay^lzWzyu2mAV-FIkuG0bw6t0#Qcy$Ft230#9i<>&SNIIL zdIsXh77r|a+;`*Keuk=v7GhIm{wjGW!s%_^xQ6+)TnK55CH>S2B9|s zl9vG?;v?K4MH3qhs$&vZDZiGo_6E<3#r4K(f^~nHQLB9xtC;3*bid?UM#BnS`M7T# zkXnwZQqh}7e(z-jE2}N)xyKUf#()Wi*7R{k0U7$Zl2kI3!k9yW)>{bzh<)K|GS2+_ zZ)n5MxY=?*(1+bKLwF+o0F}+H%4fzj>Hg!J=3Nh=zs~31>-N_&Zn*P9A~|gdQ~E0* z^Y%yI+wHunya>(ayNT*JD8J}oPC@z6X)1C zmsmg1dQrp2mQJu1Sh7_j;%ZjqHpjhx$5i)pgTb02j-&5AVz&GXU}V<8cEqE5+7CLz z@lRDpi2aB7;}wUseQdRZi;2I zL?O?yHDx`Z482P+Um)lj?OJ}qY$>wn&JD>Wr;4TUyI7JiFnZ1R#%!8M$ z-0Ny4kZ!Iyn#4y*bSqT{N=7n*4~clR_V(8idlZHJ`p7%$9_6(FNyu@VHCFQe(XWKc zEn=oIH9vff?}fv`|DC?l@%FJxJWL43?=+Pk0clA1xcytLsr|#QoRJ*3^oDl_Llh+7 z^CfQ$E@zys>p`!|ve0hKgort50(;O}^Nt^Hcz@#s-aJ3R6%&6DzQmQ*D_eQoAsDc& zDDUfM*QrykJ2t$1R64BI;4YvX;N7P6p|6UL-E3L#5+3`tHHm)3hH_AI3i-&GipouZYlhGi%s?@|cKYn< zTrk_H2o_!bFTvhzrmXs#{+RCN=%w956MV&Lm(Jc-DP=j$wBCcs+8=9tNNiZ<>rA*M z1z0wye+)|C$x3}4sng)+34J`(@C|tP^Vx-Rm&fP*B+qw^wlwoXN}>eSBhpScVQ=_x z+8kUX`0&tlxx5KDv;w%R!sEjQVzYdfLHm7p zGs~PNl^W5nXG4Igo`9gl%P)pJ{mizSvU@gokKilpCcnl2=tfhUv)bH%+?!{ zzV!+aFmGNVs!X4;mQEW(iS8No*SNh;gG4|P@I}YbL*}pC@7Jz(LW%FGvij)^J-i@= zF%2cLN>N5BE_QwU_XW^UqwjhRc*M7)}9sdOHnHjPf|OA z{Ji;H9*Rpsg37GE3GVRoz0tEr0<5sp$uKJGuxB?SkMSafTngxgX|JrF=O*>~{|U1Qpu&ql0}6_-j6GS`g0C-C9>=A6fn`XZN?7;Ev4- zy!ps16>-pwPxtZx+2!-^WI<(c4W;&USeRI-F*!<$B++!qGZxbT)5sYPL!J-Va|FO~prl92p(Su44+`W6HnVKG_~o+GD2O zEh!(QArzbTl;h`#_3fRH?|gxtFghM}*6YWcOg~pv=1=8{w|7ac)=B`by!rTvuE&lcfgYJ6|ZBsn%dcuA*zR9$EnAfvyFCYev?r>{(eXTq5CY3N}`u_NphKvl-2o<0!@cac)8UNwPcO*xu z;JC3hB(cW%Eh66MViIGyT;LT~(ecU3z&kR)dsQ}IRv9Ysv zjFv6yd_>GMTQ49UcbS@oo7TjBAs<#nCloxaa!O#}k*k5l!ZHdpUG@7PH|Rxi#(r4M z?ED~inidP58+{SX6)Mg#JBvx;20-Eiqceu^8F$xYkT}>)-mZ=Qu&}L8Ye;FOI`A;+ zk5M94E-$`zMNqBY){|gO8pTdLAgTZ4B^9b=x>^oa*DBCu&vox~P2Be_KZ)5!o0p=7 zrBxX2F*5b^#Tgfij}({O2ED_fn`HVq{n5d_)UzA(y7I(SU!s05lIfH@OC4^;ETdz{ z74A&imjuLlLN%<^ZtDw0De`+uppTWbn;#ru(!XLFm&gZO?rgB_x_uUp!Idg)LD#$7 zto)+t_0xLCO@VxoA!W;F_?0#lWAD^j12Z`rjbv zLiMThEEa~~yiex7%_`M_(oEkCSHRJ}3MzCTJ8aE~o@z9f?T%?~B3_d52iy*?5wnl= z6;)~v3(r)fP&cqx3?`}d|Al2XR7ttvl6c(0e-+n%YI;d%@!rS9?(G>gWRliYjP6h> z%1RR$ez+o_d+43x#}Rl)0@90*|1B;4r7tZ*DG@!OC}AZzC#pV7>YG7cN4CmI{4l#2 zWH_JHLoy0k6^MCQ%1Y2V%I&0W|7dn})kDXraOaLNl2e8+d+5yf`pR8mX!cIlF~D6! z`Ph%9=6JZLxB5UOp(p8~{BofU$;@1=(q;w|S-`s(tQG&nXL*{xf2zS%jW#L0$&wg7 z-RU{e4g*Uzi8Od;20b_5;1zA!piM)HzycA6V-)0I)L$j|($@sz_x+5jkj*dLJ|}aF z5o~@iwR7usfLmhq%N`?dh$84QwOdCe&aIWN7z{toH-`pm2`g`~ zv0pR_VfY%Jyiwl^V2nk|P1-f?z z5sgsiWsEmohw|Aob>Bkq3-J;b3)O1K5pE^{nNWc$g8#SHKz1lR!eTp{+547OXH*_J zp!;x2(P#kc&0W*Sh3<#**73p4Z{gPr7cr!q3Dwripi~lPnm7a3+lYF+f9v$n>yPNf zSgRaoE42$#Nm{p^(Dww;ufPL22K7P$AY8}KPkS-&GYm2Ad{A=IVvJhvq(dN}YLuhq z+WMMS@DmV)8nfxaiD%3Uzz}v86UM_NGeDI4{wZa!M5=bPVb`B#b}BJpIMi3fH)~Ab zYah@-@E?$vIlRT))NF{As`M|l%pZu$BP;*n0_4}>(HO!_)FNiZ_DRgPkFX751$#Az z`5(HQ?pS_PF(RAG(|h?sxDX>VDV6$ z>GPsZJ?Z=vT|q7c!XGPB&xT>N?dcNx58h^6Byv&2>;Gd2f&r#g+3O=qVty$#q|iIo zV!CVtv`(P+O$~f;He{I-?*9nG>nZo%0+GD7O-Tt}Q&94+VtTZC^Z(*x$?r$ohYnAn z;OEQ1;G~Z0B!j%a3;+2m27TD;o5Dye^brA{*UA&dDo>2^2$r4yhp$~v7_&2hvy#2e zG-2DV;uL4ruaYBGS=?sZX;NgZvo>f1=8Qf8{b-Y4Jb&-2!U7ezE}al|zjiPrK#rjp z3jX>NhSBr9QynyISj9)?M+C0?)9n!h z$4KyE(i*s@?Ry9RNv_;8KJ7PC`Q@A!Y?XuAr)qKBF7ii=)qDoBA3m1^d}I+}dRA}L zAK{P*{aXC%>H%?8RI>MXJ?(WpW{?E_1<61 zWf|jxJn!vG5|?pXadkN^7M{fCz4e_8>{!WO)oaY1FON$cKfYj7^GePFX&z-d-yK4d zlJmV`2>&0tzB($7XWN>D1QH~K;2wgzy9E!yg9Q)n?jGFTb#Qlgf;)rT;O;iKex2WY zcdhsCecxL7gG|qKRd=1LI zvEIGEA1gYEMSetyO|p?J(8Pf63~FnFJ9(&qK9;o4KF9he!+ZgGqzt&ho}mOA_DrE- zE3x3Ls%A}p{~JVl`hv~GTihHbS3zP9IZjK?)tY-`2%@#IMKmk7diBTUkP84A|81QL zf|D~PbC_|?$LwR$9bo=MK{XGR(m&Qbel%$B?x`1@Vj4;6?r8wPe(IeznxuQzA6Fxo zxE_?kyYydvXg7hCnMO1|`fm2v283$}EAu5nD|hpw4sb??BFVF=QWrpsiJ~o1Gb}j9 zEgy_FIE{!SmRM({lRHy<k72ep*#8msX+7?@Sje2E8gYX<{M%1cS7px+X z*riKKyOE3tj+zWdUSxD`L}4(X0@`{piQyCAzUqrAXI6jZktV|k%R0p|mn_{CmPCn+ zsxU&r{-LA=S9ql?7GjX3$7OTAMBMD*HYJR+c^{FQ|28s>CW+q-x2v9$lF@pUrh!m77;rN~*+) zUG9YkM^S{QLz;~K^&4zdjx-fz6!Aly{`)6Y+YjW^DP(=#Y^U(vx}w5LR8;gW%=Y#+dhN5vRcHgpl^9?A zPI+@Mh!RKxSau80j?Vn=q^ZUxL!qGN+T-!<<_ZH)aes>A)%5=iveXRWRWKNtZ7Bb^ zJ~cb@KqOu&Pyw$Xg%I{2qHQc`VgRUiDyBd>5b-dVUOG4_29R+%0P+Un7`wp*y8rbB zH_K@|m`97z)g=uyVi95HJ>=FZb1sIk$Zp-b#un}k71IAJMB_UHXi|AnofvPw)U0@6 z`Xj#W{SC~sVdHWu!Rg=F$`-dth2Bg+gmnZ+e`I{>MFswpn*M{mL#rw|4KpW@B;OpS zH{9vP%(4VC>dI($^S(avv`#RCrad$iV*s`kL?dN&ixNraZfoUa`FF_TQ*7pJX`HWNcIqpeu(h>hp9NWuQ zYV4S+g(`c*%8OHY+wz@}p{Hkn9;j7nuGyB{6O(}80eQ0%g7Q&dP0;YTX2w-IYm)`< zmi*xj4ScQ5^I!oosw`6ORYL{ST9%Fb4>tnR#2MAc1h!d9)(hjU3){BT+qL!e{R>+> zj@h-DpIP!Gb1TodY^HttoSZT8|eyca%)!-j=CHYAdzDM|4U}$aT%)oHeBivfp5V4+;q2V}Tr<_(Jq2(sa>M`t#E3p z>65W->ex2QD_%a_siSL5fVFruY5U5)yLP3L%dZMukZq*R3wN+yk+}`erehM+( z+6Y9!4d-ZYA6=dVw~rz@yFE0zGpcSbze(4I4hepqGj((e9r9+Ft6zXX;LEJ+c$X#& zg(X>W=fI3R%eho)=^%AD?-V>W;NFLjrJp(0>Iwq9yc!V0NU`w|X&XU>G6sI>I{HXN zKwbFEY-N%Tl&+ReB%NZa`JA}~*iOd|94?2Whk%(wM`z<7@LERWSNZWf_6q!uHgv0o zbaJg88lI8U&KyPNHHW8Io}=p7=hu!dsi~|@qv1B4>BdXe9mbP?`%y7`8V%#xoer=9 zQ&?x3cU@kE8yhxuTiAH>wDY(DMoC9+&Cg)%p|Q3;gWUl^J_+`Bx9W!=+jxuRG}6(J z-|YiT%gv3#%nKK9&A`$GUV_yI-~&wU=S{U`llj-10YMF{-;5;-xvrH3yZf~ zcjtxz+1mCfhy_6M;Hn|PCJ@&?Txqg$YhFgC5J>8sbaJ%gd_bZQSp8FKBkXcshr(&| zp7HY6&WW%&h$EZ~f?`AafYkrPiP7#4n_*v}+J?R^XXFT^6CzumbDU$w8PyyR=*bH8 z_sP29OfbKj%XIS8Tu;FGeald1K`%s1Qll_H_|M7q`P(ZW07)AEavR7uA)u>i2 zu`CPsWNJl>tR78Co!c@WFS{KqL_rp6ac%O!y*gmG8}hidz7fOeVPx|Fw_z>TlL>lN`S@DP#)vdSy`Bj)07205}ukM)`m3KlK{>^{M3 z9pB&IG4ZBGO=BPr5%c?Z{J$-B-4RjJ(;APn864&`?lIT9Bj%Bj{k`C`?FLY3TUwU@ z%4fag?((GJAs36OP1f)*olW7q1r{ZZgGK7_@VXDMn7md8y6v9W|91OtuRTu_+;X2L z-y$5{#U-||(^JyWtZ&@$0@RIND$2&KiwDtQo=u<7gdt#AxIJNRwz)v(nLN&Z1X=!WqhC99%g#tD^Yu5zWECK@r zAk8-O(idj>ASRqHLtFnOfDPREw*hYI(D!9bEcE#$G=Sm71=N0m|E@T<%3yxC+jJDr z2$1D~s~XM5S(SfI0va=uLIHLPeFiwE@T?kO-2y~6bF4C;0RZxNZ2socSFr@_R_tw0 zaKQd$-O~ORO)L3SmZz;jm*>ri6=F~6K1fci+eex+Xr5R9;LIz`D-t5LMJF){lYh5U$ihv zmlp!hSwK0P``cm8doN7Rvjs1pHlE1P;sYYi=T`wZf`;>ka<3#mBC?4}_t?_Ndo^A9 zrJ%+S#}*lwrd%7FUKtzQ>zkS;sGFD?$t&Jt(xRXg0Y3#(*3<+FjNX$ zVHD7>6|sd&;&HpMrD^be|CxiLUpJkOT+c91!0*+-;9&iLFm~nA)fWm#45;aXsOxtV zzsj{Rb`9@v9b&cgBz+-i6Q(7WBJdF-1=(DWqc<60*Kab&+l2%(EPcaoEtxS}`bO%c zT(Gor^aEo<)!$k^Gk|T6eM(an=_`BM>?|`xV_5$99Q$ZU)+2_o)oA_-2Ag*}dIRYQ z#iD&`#pEAOmd85VAT~426J7)ANZ1#cd~Sg-J8#0ba);epZtZ~Uo>SFs-vv5{sA8@0 zZ;1NbTR`tku(&XqA~kusRBtx<=`mL*a3z*nqAECu*0WLRE08u9POK)1lsr7c=(Ypz z-Qev4cmv6|zZg|u6MJ&efDStS$oU(oZgeOe!MSD}Ve8)1o_Cz+c3Qq6N0@zYm@!OTJ$0kP~giqETmwztqyk zh?5h2s)vNljH^nzCG1n+7Q#ueLJNe0_$-hykzJ!2kw zbXkKyCYu@lPw~sA&3A3OnE&D8$^&o{g_4Htq`?Z_18#v{B$kZcBnzb#Z-Jg?~D8AtN5KO z>eP!T^Jd2voyFXh!Vk}p#LMavr4lrK&-T%&aPJlNjd^rYL;LGUJlD;_(mTMs{y}Xh zFRpsn(qLMnFCd&nZ8xfn{KsPv7t`Vh=HOB$wd(`pwpfpnYjpZ?(ed5Z8X?r4{H;PddRV26Krqe$ zCTuceBdR=be;Ti|i9njj!FLM}!g}bB#uN7k|RUeK#jwkScXM zF(^7@(VH@G;?8GDegJ2E&|h(Tze|EGKQZ3G#(!;{ABVd7-8b@Oign^{d~2AbRl|P9 z=;5xdpXZJAeT{YWg%(rCJ^s{9{PhF+|9;v+;<%MJ9=w5T(@kNI_IMKcZl_=TKE?}* zjTgd&M5I(v@y6ZswfA-MK}@D^nQ+-W!=w)Z_4YGua|D#PnZU^AtUh6@ltK80*~zp5 z#I?7cz`IO4y#*#4Svfj~C$0Bm;z%fRHapg&i7YURg&^#nvo%Ln^ZAI~K$IM^{so8J z`5-LO?cWpnxd~&)ngj#~x(cD#$YRVh{I{&*nUlBU7H4187O~~I55|kW&Qx;f1=Ndm z*du78<7IR?6Q`kC974Z!`v~yA0dom$oV&bv5E#8|E5Z(==EdXpDQH8`1lHBMK)htG zI_qi9lmdmSn=A<4%X+xw6|_J_UcO`bYhM^rwnS$fbkh*K%l& z97vq|!*&58)&ldj1R+s#N+^e|7UMFFA&}nnzuu#WgVCwr98aUekfJwy z#qB0&7Q=!g8a1&&g=qN8cO-U!PG3uCtG)N}Aswr51TUkZo*bt%ejgJ^-{a2w0bcU{ z0#7cZJnSz8XG%N%%5Mt+{tZg2g5o4>(w!1Y%X4+CMCdjZnDX zpt5^GnVwQ|t1KrG!)`C$!TnBS9kW~wJc!>mU#U)cUxud2@gzKpf#a|_+XX0w4eRV( zqORi^UW6G@;xEgu=(O-TIZH0xk0YkX??Va#3UB8Z+`cscbshZjPQ~eE#oTRI3j@95$hqQ zb$_Ypt!c8t*LUZ?qd+;PzZ4o>4r1z$VND>NkGu9qHXn`_WWG;scbUk>#M0LQ?4I>+ zxMT)FYNK%+%BwE0ZcwkV-EL>6OMNkPZ6dbFwR-5qOUyvriL_N*n=9qnf4sAN`>-%W@?L}QVrTT)VYfiytyj}P?ifEuBGWDx zAxJ@gaA5|?LVXKXzo5!&GM3k~fblXF2jT(^?K1{@0>d&7~IDa%qCQQd#? z|Nb==e{sop(W0w96~uc2BMH{z#Ju!i;?8NbRWZUsFP^|aD`CX|O5Nw7)j#Oqs(BH` z^whQO+%N2WUiLTUuYI`;i(evJ6TjO02Y`1C4JMJJ4v5 z|IgNddDY`9Wl$hi#;WtJM0Cwf;KDK0j`q%(!$izlW$v{ZhABKoZh_7P;XpPSL)PLw zYACXpT+sx2Y9v1$6{@c^BCiAW*T4CI2^7Gr`k6;&sIlNQRZI52CM?imoEEtRP$K6M zO?J{@^O&QlTpW}ZbPo~<8*`#0hxnmlP|py|A`**`{U1RFzg`>`By6egiq{|8j5$3p zy(ek>eqmrEB}bX{SdYHD#*Is?XUcmamn%FcDtVpbU8~!KU%Y6lm>X@_C>#?ABnLB_ zNx}o@!oPJYKi95@+}mv`U@vaR02kEPND~{xSJoe8Aw=YcnbXV^&Ue8f=Q_Eh63DcU zZc`+R#-vz?=bR%gAb#{#An<7!JZ&`r)O0SFDU#X?M7tv_<)q-q(BnLZn>%98eJWy_Nd0zgN-M6NrnSRzMub! zqWVugU=$eGe&zOliZP;#6StCguceIG&j)k;Me+fWBr-t4c-n<>m1wZlLD-x03szhI zC1_0EJ`HJGm|%Zt`rB68a?cU>pC2gTEstMB3k-=iM}52RA~=C8CG1Z;q;4 z(6jp*)EQdQJb!IydANvE`hCnJS4jJh-xj0f!Hvdi(kA2eG5O~Cq6!7!>Hnv0qsW!c zWDSRXbY2eNqpAzj6g@& zzedgG{xMJe89XWfj+oj9D18dt@j%vf{jar!V)<&Xon`qVVV`}`5h&fMk$nmLo>y)E zQz(eJ*un~Y1U9VZH%EqrYyH;K`E=>lUXj=RNhr?OQnRH}4}#g(&KFaWNUwjM?cdk} zq~HNRvpsuui(R-s1Yhn=_$yFEFq!Xtlnf;Gql7Ob@=8NK>`r59~ z8|mCA=&|AsRccpn@@aq4s<*Po-bI`8&1}?69aTC^tx#(E)Agh;kBpZ!B8yv zpWGtEB2>~DticWat-T|%yEbQQr?+=}eMt@4;ssOZ*MOME2id>Iv^_F3F~P*Z0Olpv zvb3b1<0WFy*x!&t+$2Ix2$6nAY?>AjS>L$XT;nAY53wmNEj=@xPP6k8OO$kBqKK@q zJeD0vyAxfnhC?RQy~zMM?{7bw-@Di~;fueMZ*sl)pgWjdVZ2&S@&5e_I=z?EhH&(WqgdvlD`AbRpgWdu{?7qnE9+2J&jl z<4FHu9;cH(A@SN9L}zc9n5r1WTu&tdP zm-CsE8XG{bd3ui-QhGlOYa51>TaYY1TKQvMN2TE{H5Sn%0|Cj?DWz=<3oV}h+bSi> zD*5>sBPCCQ{tZJqwR+N%M{N!=Bbd?;n&8X*)6Kc;w}D;)x>{ASFKWi{`*A?BA*{L-&(yUaq8Zz);WazKtW^Id)Doycom|_nD zXf&jy%h?BUsg11g4;p%Bvj}07XW1JC7C#*e^BW)Vny>Gf!U>D9TRPM>(9X>ig-+}f zKQ<>Gm%-RrHF2rtYCOgc+jvE}pR>+>hsQq<2o~B7B7XkNm0XnYp-N-A*dWq3hvtK+ zxH;*0aTk?Yk&UfwtAWeYJt54rvRR&m*86HH@^Ved-l!B2<%B_?yV&8t_i>GmjtFQ8 zzK2^sxx#Um9nuZgmDB6%C*@kI#9Z3L&Xwo?`qrYmFGn3#R-Mq~SwbK`PfbT4Q)hI# z7E)G5=Wx(yvp+e?@RgI>RBq8E*HgtO?TEGpL~#)PR{`G(W+O_(GfmOKpTIMIMF$=dv~QRS41Nkjk&pbrBbn6k?E$^YM@zw zVhnw6tb#P*)@w?1gj&mtX=R@1WE+C+8T9Jvu&u4FU!GX1ihgHT0%x-8dy`D%TObzE9zCZWB1H5tRTSG=a% zwK`|l!7u`iA}(D-#%LjQo`uz=8HEb#<@3n^;baB5!f+W?l=z~5+m@lyvgxB zo>@fE(FuD2|75pZ6q=scgyB6GF%!}{==PI19rkc~TT@>zs8b@8S1qh^L!6ej{7q^) zR&`V>%6ev|dob?^DfOT~)0SLHu2;sMz0k5yyb={J?!=3Pwcf=T|64%nxQpay#1S)E;%ghAl~^R(`>4lZIxcVD}K79r|&U1 z@ur)1vXB^7n_y_x0zyZi75-zmoE=y&s-?++_pOvHPuoUxPI7WtWt$$bZ}~KRb0I6V z)W-2iD2O`0+-*TDHC$YDhz(kUEHO?AU#`Xh07E@WyUH6Lz=?8fe;y43JN{!fb#d`~ zTo`tCcE8r!*V3u1FZrChOw&DGY69{C+92!$nzFlm;kqoq3#V0fk z^fgikv^WJK8oaQwRU(C79?GCap+ge|H+H-$#Tfhh`$xCO93Sv`_4lJpOY=SQFXU1^ zMV_id%m!bg67{wYlC=k<`1SV3Pf~ZgBZ0XrYC2?KOLji=r4^TTO&ffjsnzD1#^)As zI`{l%rj}JQP>Ez=k=n-IhJ@t_H}MnTNu@E{jB_d!taRt*|FV7wMzJB495;e~$?wy@2i!{D{xCKunr_ z@>CU>8(OdEVL(a!hrcMQ4+)x(BAXJIpfInqhQ^_w7QIRZx!tIB-H)1Q3YuhJu`lH0?1V74g+qJ~$S++0tb2Bjd>s|zzr6Yir? z-z{Z&Td&H?Hobj)(Ih0ihEuPEwrc8>){E^Z&TV;3RAPqmMtnqqM=s6cV>r!n8`H&e2>!!SRK+{u|~$Sut5Y&2opO4=;f$j@inLD#+yk%cA4^SsWlF(a_8uBnoz zw4R+M<6`nYVy*>_BQoaZ=#9q9Ou1xoItc&tSwwVbq*dnriZ!k7ikeKYo43H7aDmIfEmmfGBOF+} zV8G;&c3g&uq(<-LrAF>wE~Z;}3Lk=qULJA6hkEhy`p^{A+?(P+$;mny;`|M|Vs&!G zrt5pTBv(TQ07jP`R7y({P)pO;INKeSqCt-$J6Tbn!)&O$SZFv#?=UzxmH?0|Lzr^& zGJh;=Zf%HtpM2{08I_vaePyP?gkv?=pkBqyGBr3n+S?{0jHIGK$|W51#nyIYj+erS zqg+dZoab=Gft80Q6xdm|;?7r80mHm9ve<_)%?2=Vm`2mXICqNOF(j`X4rC>=Rng@W z0{&E{kKkAk%f`Y<*iZY{;6x+l;~C|`Qx-$u#GxdcF!;@|kL}2`gvNtAWN8V1vLY|u z|T(Fc!8!#*XZ25Y;7c!V6Vc2(tK&2g2O#5`N z1RP-iV;jnA1twBFJF<6t9R@Ci%H-6x1rf0&1s@->IE6*g_2{ry6EXO;f{%frMvgL_ z8!*u&I_R&|gEp8aM=gxP^#;qA7gWWVnUS^<{`-}vkN{;0@A$>QnNd9DRuKLuvGzzM zU6Gr9eW|Ur=Fwa}rVtg_LxpO5WDJ=)>9l3$L@JiVn#1Y$j%8{C#RIIosjS_{{Ga%M zgS4V%a`OJPqjl{-)o@Ijj)6hAhXrzYi>BUk_2x<;fTh|a) zHudoEaCedmfm`53lVf6DnMcPF6itX_GS7k@Xh>h9f3Xja$1U*3v3BwmP$6WA_lSTM zn`L{Et|uKk9A`eUQhU_>l=h$c-X~>Zq{c9pM+CMI(r^HY#9`c`Llm+1VwR;h>U4BA z++tjur5Pj?qqn>x((7l~+KYb_y>>XWSBSm_Yv&Fc*8so+3E;T=R8`GVoT;{_yb z?KfjKRO9?aOsJu%O0HOUL`xtR47gN)Z~rEtcmq)!?4 zUTnj8w&j@)d05iU%G#zOaaq}Bq)|DcU}_P{27(ahzX8^P(bfW`IUWAs6L6}EN=y{D z#93X$%H|#(9PR7d@x=2_&@r$hkrlZLZE@!zaY?ycw!Q*RfLNM5~{$_o_iC?d)=6k&q{!p_38I2kM4aBszwo>0z zr^d&}=Lvt@@HUD!QgS%>9<7sFR+&BYMEPER@WR#(dWayWbl~FIMjxjB&ZdOg{jTKC zLbP8Y=0)j6X0ax0S;&*hJDbq1gJ=p2QDAMkD`I+&Yi$-Iru2P`k;T?@dAIbj1EZCh z+ArtOidLY-lC~3`6^Ny#v2Gy7Fs7-WAJpE3nOtLOM-p`UXktZQA_0rczhREqwljrq zkQpgVfLI+IW#U%YR8{}0Jsk8OKTp~kO_wbfZ`yMSbe5Ooo~@o411Jas0n|A>*oZU! zr@6Y4M|1c?7zf^w-qCREg00{WmVdjw2(tdJa7DvPmf$pN`~CLoc<#a2@Cfz_&#%8` zsdp+ke+wE$p=I|K+q}n>;}6Dp`(Kber3KF$OmeL_eUKHFm)3?~#KtES2K`R3xBIoO zk)n#@k`4{V6|6WDIU}IH@j~&>B>SHYXAj9eARCQDQOJof=#0&^e4{e^AuRU#LB(rK zURY?G9H{dxg387$nA2*3fI^qXpI7)wc;)qoLQx!z-j^0kkFf+9sB`@_LA|`O@e);3 z05<`02VYl-=$z)yv8c7dBWnJQ|3JwT40Cd9QprDhc_V=-OiGu*?UJenw5;iI(1IZ{ zUSLB6*d&E8e7-`~Gffi!I&i@q8(oX(DpCme$)BbZZX74F@-s^|ahgeLeK+Qv*j6 zB|?Ygq3`fRxMk0zIy`Gla=YmtsPFfrOHo!w{8|6>p@Qgz_m+E6I1IQGWskOM`K+(5 zL33j@XfHSBC@Bw(Ib+gWLgu6o>{e^WLd{L35e!o$W@fys_4=e48*+m9j{LkVx~%-? z(YX(f*@7F&5%U6~IgQ}tj%#!rnEB1AQ&>3GApQMEhdnl@2uR+jI?|z~H&321fnRxB4ZjFD`c`~CA(Sn#aJ znej3_@U0D$mp}V-P+Ec*U>X}rrW;xm#uAQRD-;YA$9_-Ud3HW(|KpGJb+e!4#WYZ} zM|X#TS2`e;$h(sPdZe&>XWB58K4;N!hfSCs0EpwzIs%2?J>JXI-p_r_DAgOjTY8i> zKXSP}7sJr>!Z2at{SFTc@i|5c_KO0CO-EwubQZ-H1bFkL;TKoDtf zU~x^zFE8{-$3x`I47UjvMC3EhSz)KYWvKAkkCmS}mY7}ep)9WJmgZXa44zKBLw{4 z4usXtk|54U8Y27IgoAmfT)9n%hvK>sFYXmji`drne6g(l3L9@mLR=1%5RCV8F377)t!n#;x~pGcsq zzE(|Mt%;MM0x}D$mDY5GRh5JkFY%Kz3f&ot6NBN@N#9H`inToi-{%0UiT%M|r>C^; z?nR~}6~f%zvL-)q9)w`9dnEYPQg5xD>hQG;#-Jw4kC3~wPl8~`QQN73pP`P!yW6PJcvFPFK2fK?!ur5QLYoa9)B9kJh~++p)SS)&g79?w^D_${ZWkiRP5KwI zyVv~<{&uA29~5fm=W}M~g_NlSO7?*jTX(f#Gnw0qCDEq8K@=8AynDfO4~Yuc{oAj2 zuF3X|XR=>8-ihjK%n!DsVKL^C^i z>cttRXEp-$#b|AtD?FrR{&Cqu^r7X&XC{3AWiYXISxb7U)56axVq*z|ZN>9dt(8E~ z`(vIBwsmB~<8i%aCzA%F5DI2@ocy#L((P477kf}=*E)4OHh;z_*06K?^2w4iJfrT8 z$1F4?q1?#omO0G9%7Zh;-EGB$Dm^X@E*1J@1)465);piK1dtx5bM2tAJ6GIr0!w z{EX`Z13sna_f}Y_yoQ(Qt!NL)iWLqXp8dq>_<`lYRgbeN7=tCJu^2Jasl$fWbSf!0 zXT``a4Rtdw=dqPKZf^s{qm+&HWIA>yz-|v&%o^d>UI8NZ(zM7?dH9HSR{zO5d+p(3 zV1s}~*vT#Gk0~MMF5MH(RhjS+t&8V0?X8*-VC% zZ|UwvbYF0dN1M3@bcfqvZ-|bgrRiK{MMOR_`{!^|)vhmSvJKG+oGx4``>CIu)=1Qx zTPxA?w;R#!%)xK6i>ho`eu-KZC>e;H>n#=RIQpP4v;q zu58B&(OHBSYky8Q=ff_Q%uD>Y>*ZMtzHfAPUjm7lqJ3(E!?vdOD(Z8TJK4EZ^);mF zG(hos&jRXZZbusS#;p8r+HTkcMTbWv{`B@n&=xG}YfJRVRoWST2_95kVi)aB`ZKY5 zJ#}Gw(?KsRdos%@A!QHKX5T2~cQ4%{E{ zmr%bLF;`V8*rK_;B#&NHF`VlbNcULMt_u_TUKe|FJr9>`Znz?vZ6gcg$C(gRV=Q#LBPz(<>XntTV8KNW(M{ShiBS6ch>UH8Y+ zK5Nf>=iin`FXi=qLdBn*97&@^?ch;wHN@MOG1RWN?By+S9oRZVtxh*wAZ86lhwXXa z#|z_{(=9$bLci!DK5I=jTVz35_l+r`k(k=W`FGwS>iblHLTwG;${ijX`#yTgQkO)T%x`s2)l`5`w zd3kF0a;2Wt$6)+%c{z5T|4xY<(_)t%KC)av_11)F#odG^nx#ITyQ$xv%7?8cw3h1o zM4;neA4r?X2ez)K9CvKqm36NXr9UmmQ;q2jXtbnFSpAo1xfD)oVAdB+>7+tDN=WeVXqspvgaZn_p5__{F~L zw)M{EP3$BVu$z;d#41S{&jk`ndws5V_R1)9HODtuTYZfMFP@Qn#s)^$RH-1~mzltq zZvQ?a3u1H#jR;Jw%~w4n^y7W_bbn?-`*hruK1a~nUuAb%)t*#3-R;f+wTr+yg{`#7 zHRyk!VhD}v9!|Eir=wr-WHp$(ANx|@b9h;Z)*Hg0Ys^8Oo92VL z-3uJCZ@YE$HG&J|RiJp+Fu>6n_^hTlR@e0Czc^Xmx6buwg`FL{HV>1q^vy6C(o zPnYoGGL>OJoi(v)RWK6Eb)r3w;{b^-8gJ)f&_}xAG|d6wR~LumzJ*%PpLKN;4kt2_ zlwugG{HIHvQ^Ln2?;IhkhjB5$uWO))$QGAGQ7qaGTEXf|`KAs8nW*YZwa+T2im?eAZ`jw=cl>wdw|Gw9?SWHCnQZi_ z%ccf@vdWg5vZ=}Q(Pu5UEUZ_^b(Bn*k-=+;rV|j^KKrI*%y2t9n|&(Z>UO= zcDzFNi7bWY4#_f14g&*+zAkyo8IJxOSb^~AQyQetvY&o`aD~(DfQ&YE9Twsi zac`_r!7L{*LSYl#G{I#K&K%={_gr5h(f5zod#X44-@!##3Bn7|r7|460=(J)CQti! zue40VuyPh!?poBj1BL@p^*(&9B0M)I$+wSff>OdOI<%hK!#rIt_vEk4t9X45BfxU| zv{a9#H4IqXPRC;8F5FG;=9W;tIU7ZOp)I3RvBB!&sR1i4lY#G#Pb}0*o!x0Z_ZqDc zEzXJgp1$7?gU1LbH)Nu%%jUleBdvD4AUvQ4r?nWDUf!4We7&4CNaI88%lNWQo*w#{ zoy+fQ83py*oufmy`Z;L)FCL4jh}sGin-X@L!q3Dskgr>~BSVogIe+JtGP zj|s|;G0`Q(ide8%rU1V3fr$)to+VAK{mUtLz61a8Wv87@(xb!R;^vJ_MnY|y>Y`)O zWC2!1+giA1w{aIZ0qO#Ho-Y^rE61y^7%0a3k95Ru#Xf~v#+r$A=n>b*kXJj+h1UmC zQ3dSHllm+kjAKe^%5|~31~;H#42|`o{nj(<+ni{Y)p;6g%-+46ZE~(Z2~DrreZn#? z>+D{B^pTd*n56Ugd9P`Qb7xDL-!Yba3M=W-QD)8_w=F52wQ>dr+W~Fu;Ad@g10@QD zTNOmy8iMrp#}qimBw{lL2wEf#7GwX^uapW9XMx1-NK_@>VZ;TIUBbQ+TA%j$lSyH+Aku&SRAiiHE+Hwi)ip7y!Py)- zzZQu--uj;Y0d3H+vAemuzzUy|21A&9o@=%ba{vxqhOm)qKE@GDSu^lWf>eDwA~2S( zUjgbRq8QOQZx9`m;!k|c9Repazp#a+B^fw2g9+KN_yl8KW#`~i5}cdH|EGKkp40Z4 zuJ6qq)Vi!LS0k|R%6@(!tux9FdX3{0BBl|GlS>Ki_HedgVXKo7Mal`+Jvl>)m?*_smy)i-ANAj@^2y3~3f|7PAt)})~wXcw{ zXW<>Tgc6Q|`hRZa&;IQpg_qPV4oR+9<)!aLSbANNbqJcn=0~y1EAo?vrcwbCSSo=+ zgaB#&B!S$E&I&E6WyT@@FjCFQ*npG|49MAmvgZ-X zJ-hiD>)3@{g&H!O-C4#QI_R9m6(O0Vzf{QkoXYSA?QLxvdCc1X6DCq z<|T1MTizJfu1=NIyJNUg%0iD*>QSM05E_n@V72721jssDTeve2XKp(Qc8Ioang>AJhC0w=>5TglN|B9BWu`7 zE2aFZy7M!Wr;i7Yr;wR^u=fC z+*Qzro55rVaFBIWHaCimPQ&tq9q&brirdXRZS-<4$Z^K zof@J}L1kZ*=E<+3HCJ;Gd4Gia3nJA@f9#gE!EudZm4CfE)JM+YLEj8Y7T*{pfVjFT zGu^G{il758@~A`{7K3){Ob`|Q-yTXj&Zi<~Se zh=WWJKdFf#W@$1}2wd%l5XUYdRLw~{i(*-2>|;ioPohz=l}mfcGU~?)D4UlhCR2{4 zXyh?6#S6+HDAolGGsWe_P*cB7Uj_>|eC3Kps(SYNt%8melOuh;bhghu%lYA=+#oCF zYvf-+SFjtQm&8Xi`M(o~>bg&PMe`Ndzl6- za<{9Y7-ke2{}PbvT?L2ZvcHVJ1AGPZG}=s)gez(AdF|TeN&j6_dN6f=$$A?VE5284 ztltCoL|hID*Xd4acX}u0?PbElmz1XjYRyvjb~763L13{Zoc@I%^BW4Pr_p96DHU#J z4LI{*h~+r%crnC+q{R%>*ujw&uprOeMN_5DALd5XINKA+6ywkpG-l!%NYADYP8E} zKoSTRI$l)i2F4_No@5K6rnParZ#{oFNxEe;Sy}@;wHS~I-E#$kkYNJwOkFv6_l2Tp z7XLhd^i6W)$@3M(K3M53PKzOv|BK7JBM@3ot>rDKYJg){WFwJwr#d}f@e6DC?Z)M# zl3hi&z`S~>|EVQ4p8R615yk9zjd7{Zi=+pCOLSXP?&bmFQmKRK0Y7`B{97F2NE2L$ zX!}gZyl8Vxr%w2HLfS@G&jR%Bavegb+(gxIttKX9)sp(fQ0V1xMEp~#e4*H#{=FaB zB*>Knbk;O;t_Q%M-{0$DEH{?Qjxr;IRUW2r9|h0;mc0^;lGRWy)9Xa2b?Ih}m3hQ{ zR&ZN`QSs2+{bF@DBI%VUVROc5GXFcMlGYAcFS62g0l0`bgUllhA+=HQtGcBwC#}qY zYnLY>b+tJ`pF%i*H2IdQHP=8<%qIb ziBykggN>NN(`QumF>NcS2x>xUKLCB-jx&%hvzy-Rd7v;ZHT8~yi6mLZdxt$=!q1`c z7L(v=9YwYbz~L|b_STxC#Yb^(;U|b0u@@BogG}!)89V3C2MUAj6PqM2kBv0M*l~kg zoW7@);Rf|Er}q45gzfMYP4V8H8-l>nyDjSi@}hj0ABFeMa7P zD5LS+wTD&><$Y7X3!aCPJHFtbo7qMs5<6HPywJ&neF0 zH+TPJrMm@2h@3FsL5=S?tAvFb0T5la2)*t|vC%-k(Y1rHU-p_)WSSdCfJRe0Xbj zud|*Pp?{E_0h<*8REm()hyXg0D>A!(P&#$pephWT)zLVp2}gA-3px>|l;qV8Hj=y} zU3hCZN7}vO)X|NAyC$`casPDOxvCNEzpJ~Hmz=4$S58{GxcgxwLQh^^c4_k18H*Pi zQvx6oKspMWDI~m9x;(#o@0jrMZGw|Ou?@FWQo83+-0^!}bbLCN9JKXRNVNUupqlB= zEMR)p4iBuPD;(5TH~Pe#xsMGm6+@=ihf+P4g0<{a&fE7)OVz7Gd$ymsvYT|=uAx01 zUfl0g<6!s4m2s(DEnmN88fb>hS1ilLWUzj0LX{RAx}9;HGBxcxd&{tA=Iuc;-0|Ft z5fEjPg%24Yt;nYr`r;g_)4DHW9lT1x?~FhNMSEyf>XHjpU(JLtpCcAj@Rw= z)9pUf36Wwe%z(BPzLY2_fpYm`_IW!pIKkn>4)19N4_GX5f_T^O&+7##Gwhe|*)fRu za)SznjAaC)L2xfbA5t#;+y6nT{sr|}$1=+60Nl{#L@NcZJ}{iMmpS&rBMmbH@yHvO zu)LMbeTtL=JvYc^0FJHt;L;X|yd3tXxNHz%A6eury6@KOYR6sc@3E(#T{6@3pxHNr zZ`|J`MdPGLH4~<0R$1 z4x#05P!giHy<_L&8G!2%MoOr7&lk9M*Hz>9Hao*5mk`Xq>5>ca%SC<^z9r;EHkMiM z-nfmUFri-8_e!VG31|CzW)wrf^~*_&$$q`d4$Z)?IZm%hs{U_DLi+a#=$Fr?H`j>i&?G;^f=nGr2yNqdC2j8%{#7M*ZzsH;+X5kjN1T z(RFti8{<<%#h3Y)`ATCFIu54(q)H_d8EadrlpQhOmY${S+BTFE1x}WOR8mCGiWL`u zq)BGM!F@g@wu0&LK||80(uS|M{$xWbcZYOfnM+buTD5T%E ztH|c@NMw-NnL%(Y_zB*W*pMv|`}f!4Ughwyc*^DKhOBDF)th!!{l0G;)eBtJjfCvP zgjxus*SbR~iY!=JAH8ICzh5qbJ~fJ&U!H`p93Sj{{NeBm@gaycV{k#60yst0AS^RH zcy{Y$*JNc+I$FeD#x9*PxWC9Rz1V?;g(>m(0DNx|6j!z>m%o)pwu&bl`mMG=Fw#c|s@0U;2*z7$fOSFWY(R zBIC+7p)ME++&6|Tbn&m&`l8tjO0jxhd*anx4Y@sLXJ%DtPBlaPA!b9|`3rLnV_$Mf zs;ItFX9f)ED1GF3iE7w_|Y79QKdC;xe29PG-%Yn;iH4)A1;B4=Gk@6zsB~2UmEVmZ5qh zKi#T5Hp^RY=&!d#j7D;KX+;g378&vf$7O2KZghevvSwYuYBW4q5%G@-mS>swsCxFo z1(O>C1T9-$)JV9srzLwIEZ{B)$9CdD=B?a;D0)xMc-5B1RzIBVSW6(2^@qOmgXRNu z=$b_e`o1^>jbLo?S>r&5;k@=h1UnW-bR`t$U%_Q{13% zXWfx2?wbMgt7&lQ%=)k1u?OZ0t15c;(^EB1=ZGla`1iWwMmG*QljJoPN%;q26Qn!x z;)>?8k7k3{C?N;No{z#ZjG1Q|t8TPvWP~9`J{)-`4xtrBPOE?T1AE)` z{nl=ATU~Ff=*9x0i;m-MBdX`UHmGPj&DYG<(`QJccIU;Md>_M*HfMycKOVyL;qrY? z0N!>Gi`$ZQoVxHvf*5;of!U2RCObERdk z#bdL3rvJEjpQvn&r0#lDiC9!{c9;70I@EgtkAL$?IUI`@LioiG{Zjjsc%V*4+WSFF z|C4a^tgrxJt%$Fg(d&~7NMK)Hckrk#8V*5=zuR=68=662{71W?1yAAKoe;~r`_yP$-VWoz;}kc{vALJGp-(~YJHtWX%e(lKIp&?jYh6IVPp6Dq%PA?B;)aF?*!12@#MQIrIo>+)4 z%DB{899fU8JCXN^UWD`$vhsu|(EK`a7h5j<2RDN)TrrWeHS8Ygsg*VQ+lRRPbib_| zWp^Vb-@F}qY?mvI;8GzOP}n)e)gydBdeZ_|x0 z z53`)e8d(VVfYB$rRu~2F&lSKJ*5b-6jaTJWqZTcW77f}&z(DPfeOu0(9#QAVF8!5g4&$-dp+YF8Y$yLguh`&9{&WT^z z3nI3E+mnZB#QKT%`whWxb2>hNG&2$+XLU;aS?>{pl?9{TSjKiVd9KOc`fFD>+g%ft zxI3q#=`h}R>XL-4GUPlJ0bYyW&+81;(icGXtO%II~613?BjvQ2@<|f36UF{*Nr#X zm@psI-)_TVaY|F}ZcFS&`J5eNByG@d4+|_70#_48t8tZTZK_fO;gsHM(cxf>4F<#_ zi+u~u(#%Zgd*6lP%8KH)d@C6|J?VF$ez206u4)0u`?&z5JS=5lqQoItXl&Fer6HCSRZGwe@D>CLS~y!C}pb-=v$nANuUM--#^a1E$K^nLZPBVVLo zS<*_whkM$%whEBhPBPue8U!BdP|MhD2PI`G;6CUpq=Bd_RzcjbGr~V6nRR04!3pc8 z$lUTy+l_h!$nC5P#z^V==Mg?DocJG~3*32)qgG2Cm*>7MqmsZ|98U>yYwo}U4puqg z0^B?N6g3b(63BX%!8gBT&(SxmHKnVL_tI=`$Br5uGpp* zOF3|%iwIQOxh-!@-jqY#f3-W%=y*`pdQM_}35zYA~TV8Lz^-3Wg5w?Nz z(BzQCb&IN8BvOydXPy{VR=H_0-zkb4@1}0n8-IiiM(L!B!r#;u58(@HYACWvEwbwM zz1+;2R!7|E30ZtipIAub=jmh$tq0^$i3^%#KUk4$y81=2qJ3vgWqZFWGgxCV-6!yX zgF>zy{~s?H6qXPqhN6O3dpb0%RjZ&0xdZj75eaDXJMxXcV1}B%I+wng9sm1?M) zekn5-fE?DO!QhN^D70{Kfyk`deMsYjIzFPCa)Fb_TNmmFNgv=dy9rx5%dVC~GTQGA zK_?9kVNgjMPmgq@4!E6SNxfQ|LzV^im{o)0vhiy^$~&5gb@BaMv4c#ai{!5AcLIS@DI zkd<70({h7XcSd=G3E?@GR00-p=yf|<3mfkiON++Zl}fOV7>)8(E0o_%-JJg>L>u9= zfBvb86f(yw61x2ZwVkhFzT)DPMA0xDzL|8w5~GDa1(k|k{RRrrAUfWZr*EsbH?v0X z$@Huu)zj&EuVo|&4?A3p@?Mw)cjh#kcrD8F#^Sr_{ta|^Wqsd<@@vPKc%LIWXdxKo zfT6%rKu?A44=gQWhd5bp0Oq`K;v9^Q3)aR_%o38#4R?Uw?roY5?4lfqdypB-QRiG( zW)P}6zQ&qyB&{O4kL}8y3o#0xJ1})G28j#Z@v(b@8@5+hJKL(Z71)x<$%5ZQHE+N*0$i-qA=zAPFkNJ-uQd1p=BHUtW$4=HG+2nEOJ2xR zk&7rzcV2y!;eZNTorl4i{VHjjs9M3i+VZbIXsPDI7(DQJU2c?mxRVM+={UIB9$8jc7;oW9CmO;(4cIgck!%WbJm3bDR_w^qOtLf=A0J zSDf4siAhuY>o24vq$`d%xjyfn?NP+7CO*?Oe9!m1D==`ZLiYqK+oLUs=(A~gak;hTUd2Qpn=Th_``Fc$?3R(e2Q5W(@_rH_uB{a{r|}YSi}#wI|bq! zga%bEOnm==Bp%B38K-J4hN&X|5I#~81g`ZyZ$QNkYrtnx^g1I29E=vVdh&0f!i-rn zcIbM^;wLlUIu6&j=bkA>g4d3CK1!tD8(IHJfYuM?kR)O*dtfx_p;>|3Rt#+XQb#63 zuZ)Su8j+A0tPvaI#+}f}9O}!?1$NmapT=x}=%fMzE}N`H&n=FAi}2JX<&-1+9-)DV z3;hm0z;|}yg)he;~}BKrrhyM7m#&VrgPFYPl1!8l_?XO4dM~9Qn0p zZ+IF!JKfc;(*54s6E%1PZ3QwJ?}lcZtPS4|E#v?vJm@?-d|+?O$RL=R>j)rgYywf* zN%K7W_U4Zli^N-+l~}l)_?(l^xii$z6+ii3>K#~L-9{mKt>0!i@omjJ2)KX+h8xtv zc65Y$Q?x}NLw74!61*88#`=~;_un5rn$4YfP2>ng^{$N$365lT#8BZ7)jj(HONTRN zNH#{dU9|0q1uM_)U>1(}`-3TIg9&NPSixlO-TJ3ZdeOpt*1KtX_pU`xC!|2~W0rD@ z-n(0)5HAe_%<^|$o{T8r0iQdw+7m5MeE!4%`M|r8ORY;9LtaLUSMRL&QG5r*G458ZRecEpb}wxaGqFroY8Gc!UL@@O-dN*_n;L7GAaVp z56Lo4u+o}{itHE&tY6P2e^L-U0&Ordt9fJY#C-VbnxUU8ds}%14s93~>#akv^)J?P zghwCWgS^#8;R1dH!iK#=hoVVZYza zNE-c7b~+k~KUuSY^Pr+}qecQ^4HYBboQxvm=@b$yGv|d=iXeG~vl{>4DZNLmkJ1Pc zcw#!nl<0>^YV?j=r7e%xb1?GSCZ`#|Ka{TPF>=)Dfo84GrTD?#c z(O$=Uk=^UOtV+Cp3Yi$iZ=wSQRBXdm9lRP z4<>rFB&fr+8XV<|gy#M<7PN$7I2qvlF<_HoDf`jZl_rhAE-?HSr5K^}gM9@@i78&oFz4STWH# zW$kcI48f1y>Y}szQdoskrUPrtGK|gL%?qb>l#-Ja z#*YltR}`U{pOGXbF_)w@YfH}=zSsOA>mc%;eoAnQ_pCq7K>kr%bkrfnAMrreHOw>o zWB>W|t^3}aq4JZBCkk}Q&C!?^&@i(*a=ArH$ouZRkryIVcLuWoOauZKY1xS&UNujX zP@2~V@?b^3bBhTP1Js?gOPS7U`xlMYo!zpa9e8KrjeNF(^*bt^`EWM-Ypm}!#*t87 z>;?lKio&oR8%3kjcE%-_H?!6-#CAnN1&?Tv@J22XRv&?PjuLMGg`Z%*$1$VaQsFdF zGy=Ss$0;k$?Q7VQK5-?Y_b8ksAiAGQe?CluPX_rNaGMeY!_GaFOf`>B>YEsr)ZJNM zgPK{pKMb3!x*FoRK&MTBQhqBJZt#2>pApWYjg>W}_3#IB=aJO8hVIXM=FIE%9D>6JOWt!az_=nKA)eoU03@2xs=uH zPzH-N7(Ca|p1I<7c~Np5$Y}uu5^1QpY6w;+LZ%F0m>7=u68#jo99ckz3tg5 zE)zXxTS@7O8(c255aACrZ2o9I*lHtm%UCNuECb=81ro#iEhf_R`#Wx~F3ann$xkb( zLL9-de)@320TVqtXQr0}%ymDSa`np11sq652g!-zD)sFdRcl#=nip|6RvZ;7ug&hi z<~7_;doKySV0^`{n$~cSHNGs1CUEuUO)9IQUi`6mdwG6!ez3ND%yo;sZ)C+2a^g}@TokMO{lG2Myx}d@7 z=ZXLvSjz0rnHu9&Z~1sIOWtU zP5=zAOkX`&P1ACC*1~Q-mD4&vPyz{O;3Ef~iNpJJ-wT7*jwiYpjMsj9O&l6-%Qcb7 z_O@(F`$9PdvZv~^UVY0Ns#pqmBHf}53g#ycW74}qFu`mxYYm|-`@@ADUU>HGu&12L z%j}y}GN$rV^t&%|d%SdRAv=HcMfs5?{jS@Gx^JM8+iL*8SiXw1)a2+BucO-VK8g|b zmce9gQ`PgLFY!GIPei1WdX0yzsE032UiS^A&CXF3Yfe)rC-KDl_E$pcyBi6yH&Q~; z=6x12&9T09IJE(C?>p>s6$cx^1g@66YP;B$Gea8Ly_xv>-yr?$;!`@JrcTt6mvx=Z zKSql+Ef#Zrwl;sPzDHWY0>8~y50V(pZ5o$p_Yz1d96*`<1EU-MPD(I0RBbR359EE81L>kuP;*=goo|kPbD5im~P>8ozo}q_@1VbYrVe zkO}9uHGI2U`I>h34@@VdKNB$Vm)en-Z5F0icf*EI)mv$NvL7g8=?S3K<~+EqG~eF& zah%*aiEFt+G@mXFB$%Ao>A5@zNtBPh4K+wwA;X?TCL7XV#5Xh%qRBgq-tlx>vUM(f zj%B5=V3m&LbO$q=4Rl9)Z8!>qg+@0r#O3zKq)8c%C>T$tmM{qL{~A)dt3C8<%DLfG zWS=rVOI8tA%(8fHP?6e6tEt`ZYW-ss_u*4yqf@_R}PQnJrD%M5=_UCFA3vkuZ*str#A4Z;IUiVNpXooz;A^w zU|$K%e!PaTN~o6=3(j%8n}8<>AFDpeYbYTK7w-2=rSrdwG-O{6`%q#Cd}#(~!j~DU zjCgVen-`C@AXeFKRUSIK9c`reI=kx${b69T0RgC2?kCRVNSDe@!TnCk<|@y;0KfKi z?=yQF-=5~=&~vWW)&&Wt6D{0H(M34{UULEVM+5-8YLCvO%~v7gv?_L=)j}%pGuL2? z?6H&~z6hr5;K5ATFgs;J6IC3=5K^2k>1526g{S+YG-)KMm7jysV7=q4v$LGpJ9`_t zO8O9cdiS7As@Okhbb>$oQHWax`LZMzJ-)eA@2@mJxMZ5`dilsrpD$`}s2j3nYCCNO zMpo7sn192kVphxoLT#%;Yfn*D0>)8IdJ$gS7?e9_=-=li~xGBC6<&5L2P9{4Ugh~n6s zo%h%u5K9E;ecAOgbzJ0ffw4Zu_l7`vE-Ji*XJ2lYF5t-oI@e7>Q=ryxGgZKFW!6yZ zc7Vwl0o5Zmimmsanb7^8qxQn+*h>|?{Mil8I=vNty~K9FU_Nx1y;Jd0uIjuC8_C_j ziOgkLL;yGceQVN4V3-sK*|dd?Eqpn@{540=iA*rKBK>K9H2=3b20Q%J{Sz`WGW<`U zq`PCfj#D+=uFltk=^EaQWTKiJVrdfoww+V#nLl|dX;a} z|CGA4YTsNRm2kxzSh3zY?)%rKM($N)JRLa4>GHqo7UsB~9E3Hz+STdMmrEzY9-3eL2SB-kh15n4(p zn}=`@Hiok1AYj^wX}j~%m5G92grGhn9Nd?vRoA|1=tmf;`nP~KO+ofYoxt0@QEln! z7TAMmvuC^?5_1w`hMaFcenjhJjj3#;P~SXm2$#E2;|8~EJi4HwkqF9= zq>q}?nr<^i`p-anrj*=8CsQ#F7%Owx#TivNf#{57Vq?EkY-eDUV^wryuB>7%dhCXW zLSCId*od**l8sOYJ9cj^Go!}$ULdjR@&hW!HtD$?LowXdaylx~DCq^W#9)C>=1NuE zap{i!JEt3KCVy;D00B&7M6rG_rzIM3obzAIAjXbIg?HPp(o}}>GY&jaU1G}8(AI7} z-Cw^ii5szYMRl_>$AM{4KIDUo6>axj>L>rzzE^jeuh9eH|KFA1fBG&|d%!aKVa5HI z{99XFGo{1FTQhG;qKO*HlcTP#xCC!GD|mvq$qq)dPJtiEk5qVgw+{|NR(*rUqM~l? zr&p5P z(kglsN0gnjXs6#w7g8&%dB;SXEE{P-UhPEk>64S&^3HEdsEb{b_Wb@Z&W`tfv9ly_ zRG~Tt)+d!i&U~hIOukYh!O29L85M1c1-hzvP8_YGdVB2lN zu?hUO>sHZs&<@AGh*jOF5_!o3RHrI_O;vd>30_=WwvH5x<@Sb9ujC8`_3z)CJ=8)l zH1i7!vkUtT=-4yXo>suuL!OqeEBnK^JKkpPhQ$-U-~RWDM5#~b{(Fxo)G3S>y0BIS zRvg^y^40G?VIS7ZFFvv4aCbjNzLE$|j>gPCoe$epX5-7mPidfEKC#!>3QsjZAp9=R z`QADgu-~)lV@S*#cc7^J7xk9(G4}yMmzcB=0=FNKGX=s;bk*P#@Sg#`4v$SD^JY3_ z7C|eAKU$k|!*rz@Jp~t6HJg0Ut*~(83yHj_F3lK&Io%Y-4xbU1Uc%0M@@fQFcac-o z>^FvV+%4lyp{@EF$!X(F4PCT;hukl`sR1vH!lXQaf;|7v=7$Vl+Q%n^VM-l5k?({H zldSP=L$r-QGXqa>_)@geQYXR>i^+&}oPiSy{AGc&BuZH3ha4HI%U|y{#0wsTz0IdW{D&)Kr9CsSX$1g+X|QoU~@5NGO63cC-sAV z4omGZ`yXrhZwuP-p65-7Aop{eTI*|PbAef78on(!*KNn{C+5c037>$~Uo`FY9`$**xJ!Y#)6JJ@~F)R&q9|-$nPACHd z&%k;O(^d($(3a_u%6j~s7ji%zFH()k*YxX^em;W|A{jmUzF;`0b{&pq3{<3Vj?IFV2v zDR#p3G{|bb+b2FW)5`ve0K^8L>wY;m3^7=d#ec!ZzrM5oS$af;49H$TtG#21%y_lJ zCvCBzzl~93AXk9!_G?IM^C%V#7ZEI`pr9D$zW-;V@|lmxdU16Q(@b-!@hU$Y!~A^Q zS;4DAoS{U9jB`90-i;E_g$IGB-uVl%!vwL& zRec7-W8ohjgx*5%J-3dGCi)h0i;qgzugEbExPUfb?DeS*uYa>YntV3sg-=PUrXJ4% z?swNEM77}c6#ts7qx%19Tw?hXrPnln@bfr344(b|ZBgTb$)H`7m2$C)UiRf^rU#SH z?fbyQ#N9*Kb|ul_-)$G%{K=%-rA;cGGM`&0OXs+uYq#t=EWBxMqa6f7kaS_!-b%~` ztC94udjPSrOTe#D#61VI-D;B$xv-+<*jH+?O-!*1HH6<6#5`f$V@19l$DWY!6k6Rw z=!HD#1y8R<-m|(z2G0K{?Ee<~A38rIR=RrhM?Zrrg-rQTrjXWKnV_)P9j@U8y1Cza z*)Z`|kAocWt2B#*5n!0mR|MKbMZ1ETQz6T1i~_XB&|aoqb=wPCbr7&38TX2#pMfu< zsnHcqpww{QgPiYy6CA9b|Iz$Huxo3^+}%_pez<4H5@4lhp1p5+r6~ZaSo*jCY&&mL z(&Al>J!KZ2#;2i5MEyVCGeWX8oR#U(>r2G0!T9dXC?c$R_Hkfg7qKH_S>!=N`I~GI4l=MkXW}?6IrFEnEcadj7wa8}{8j zP{li({dA8?;5VqR-y#*3Q?j9fC8|qSEK(E-6S(pZ!KA z9ufzXf`4FKqkPTt3~nP#^0f1*`Wf|vU>h$y(B&cFo#E`jG6i0yPLCH?=1&EZc$wHJu|MeGkkL^TkISD zy0F+2mz`%#U<0?V!OSkT{wk6maP=t6S!s6QiSzvRqW+(jOuHOL^1AnY@rb7NMvF%8 z2*t(UDcg&9{ADSB>zpU^&dDwbKDI(c`-t`B^18@%)|(Z&I`j`wlgg$387OQOftfoo zEe3wop+4MBkH_E)<1rfVDN}&Zi-5@+vD`(w`9!Gt0F9f z34Lo}ZNAXzOReY-kVCj`;P6QPunE}!a&!BZ+7qLNdsrAfe*l;sV4c^&s&>TybE2Pb=O@T9Bkma_Gc=fQ5`)U3k?l1qm!+=EpJL?f1VLx%0J30yj` zeU_Qcp38m5be~YVEZ9t5_Tw(AJa|px;pIkwqK#c#V?bE5Vwl$G8WfA{(yQrZzk$VZ zrLfiIPfd%yGs6E!0vp#@d?5aXuI#D~uB6Q0e7tAwVYEE`3tAx2<~KD0_G13bkA`%z z#^9c17h3eru&Lcd{RdP2Tk>KtEUB*p7d(y0P5cdYaHu`o0&H$@tFNnJQVY-jw1Kgt zC_|`^Oz%4($UoQ9)%gpv6$LT(vwY%P-Jg=|?)@xJxaQYwGTF|k1Hw9za455?iq+kg z3~eWZzXmgm;V2F_d&b91&VEd3N3-MT*_#~%n;b%sX?i2<@4UJE@?r{ht739k@zN@N zI|rWVcpRgILx@(NraFob>Oh}Kdjh2wmwgMOw}P^68*UBwjb;Q$AJk9|k(7M9HXKit zMaQP%R~xyBqx-|V@09jHEK0PKH2l0|H89Gm##c)69Ca4UE~R6+s?>7D_Lz3#J`R?4 z{YK$Cm!d!L!tpp_?;729^#Ye}$m~}~Xf1vU4e1dio6k4>#Mh>JiOR2!W>vU=WlwdC z%9%TNgUgK6Be_h4Xa>EdGLIF}6?{z-z52@tG5g}+6=yyir@?L9>qP*NcAs*df6UmX z6R`viDUGK2``kObU~5w?ls54az)TJNaIG;!n7K6@6Zt8|fu=Wkyx_Ik1YGBv{j;(G zbl?m;#8n@ARcxIVBOLyNzDq}stlQ3R4Y2h^@0?B~epU25>!{q&?q^iX-HW?2jpBpu zo}R_+=n@GJRg()1ygM`6fL@r|LlKfwO_P4@D%LPTwV>S-Q8ePwD+wJc4>NbnNrY?j z*l!J`*T+hkJqp>^b33>LJkz_?XjW!w%)34k_$Z<4j!mh#Ua^z_p~^%Z39-Xw`~Qxv zJJz!WUBsU=soxTcMtrvzrFq_>)3whV3%>hE%~8y+pOc->k@v+dhp!XKtO*Rhz%n5yI}dWZwVTLG3aCC zy+E?~O|FL9Cd*X+3v5Lrind9m4>{f#tnLI%&`tBhLh!2RToS17?DT5som17<$yFs; z8J4@ayotbWBiB2K_&36^Ctq2{z(2$Yq`9IsJCl{(R!nlsClaidYKKZ3$BS7aJStPYIuKU2lA)OyzAjtI!G`=^lu9!M&GogM);`p`T8*qj;zmW%DjBfI;J#7&MNRQw7Bp<~NU+Bji)Ad-NPY;t^V)(ms34oOEcjHU?#ufV`NKychwU+-%dhr6O!} zT-t~^f#Je^en7>#!`QaY_G^SWL5ixkI4ijwN{fH0TGe|pcfyQf+J?igr*n4~mqFVJ zYQo-DIcs}Wn6TJMeEvTy`tsP!P|v_xY`D`I@6QoB=Bo*t3Qu={sXfF4L5llnz&oF( zEjYL=R#{(k4Ma8qsX@P${-0cc1F?v*8~Eu$;i;EmTiNmP5q@`qU@0=uu56J#vAZlm zI=*b_+%}MroQsUYR4EQEBcDX>D2f6N4bB>=tH-R=`sx9Pm|ssHO@`OIdF zHGUvu32$Q2HG$?(X$lw^I6OOZJ>1Z9GttQFN~0)~t^IK&!}}{*&XGW$DgWZ&n-UoX zUHE*f@@p*&-7MWo2n}j)fw6yE(G3q)%F-`)==Gn61tft>AMmpsKOBY!SYZj1Obm$< zN=UW1jBJTb69efGSoWMbw&5&f3pn@_agxxM))2x8j4y-An z|J@jm3vC!W`Vd_HMR$(R>m!B_N`gok>yprE7c}GzVVB?4D_D+PlXn0tT z%kiWnf24qJ)yRm=RuC80(CDF)-o#tA-RHoGAUht-VA9}InORJ!b&Wb#r_ZqwnlTUV zSw(N?2@6gBkhRgUQ-_O_#rfxnnB|1%Hao?}xZxB%Hw1o;AOqNpDib+Oe{s7K6bRW92tc>}URXMuW41J3)BkNYS_$u^tv@s9QLfB>!+ld-GS&|s z72oA|5kFB6ODQvnHkj8!v9(}E#qEP$SCLwADG}j%1u|**LkMEU>=2Dwj}Stys&!@L zHrvF^HxyLPYDKf;OlJGpv}EP|)r$wwcf`71JZfS%ch^Q^XD%J;U45&-*>a*qNLel( z&oV5TN1F4lA!uN_icG~}V|sugz1Ub8j2Mfki2lby4xVR3qZb$fSxBK7CoW6ET37!2 zSD!d;!!6Vi#8zL(ZarbJi}d|CBa zSDIf)n_3@-OsaT@pNt=9`JibKr zgl8t>mYv2|b3J0otGtKLDLLa~$1V24qrsFbjuZBra)6B+em(x6z6G7=1A-Xl^#CTSJ&Dt;0?Zr0o78 zz~j*fXn2c5@4-P&L-%hS6nQmU{a_?)Ol~f(xAS}god5wR&ZeSsFdOG;gG45_-J>V+~3 z`aI@4s_!n|*?T|8GS_0H4GdYp-3f7|6bK4d&^zbMxON41f0MncZ(x0LPcI71y(XC% z;A3ou1$2q=&JI7YNje@m;><+uIwWVR7&1&U006S(^eE#g)3hxtXXKYNMcNG?7DEwq z4rgk5@o||myq(8I*7P?jN-Ve4q?Uie=4m&A57(FT>Nk@9zO40c=;bOs&f4nsOFCyM zq0mXy*YS2z!S+K_->8TXIp&<0n9OgbEt-%f98hPwWlTOYq_Hc#@05p$~28b%Gru0w?BM0M@koq0C1rx#2=RjWgFC?dzqA*)| zXQcFB)cR9yj-2uMFd6&xSw{v-0KER0;*aaLY`#YX;^h;{y-x{^xi{6=1dx$&aoAWJ zv+T9HutL=tj(WNA3`#|uII213j6B8tmMu}N85+#jQzOz(-GTULbOQ+~(%RoyU)U7d zF&NbW8F@>!`mCw^;lkr+@!mT-hQ+ir8DdTj*sndh412ZQ<|It$@`eBicCG6{L?KTi zV@LFxJjq+DzYBGyttPao=gV4BmyP*??a#!k#4H(g5Y6wgD@yw@6Tg`Lvn%apyf?VV z!e4hm9n>tlx%Qr|$tsz73fBF@m9fHs1=p%=9hb%0$#r=Gi`XIi0e!O@sWEve8;l{q zC%7>cIqpwTta;$CK5UY&`947 z8&Op@?IbnTqV!CG+LQ*}kMbMSH34bs6)E(20rRGIx_joXSnFQ_` zuG}(hKAEwh3keb^@BEDGB6dwq{Ynf3?}6=UC)Y-~q0fXjBRU%@PNaIOU#ukj{&S(T zL%+MVzyDoJ%j#6H;`i^Z?9DuDE^k+7fa{lnt~J=Xt%v^`_l=da4j^*Coj)!(7~xG{ z*C_NR8F;%&k0bQn-F4sV?a&Na&5`tA)BQ zS6UXC8NhO-{t7|$`V^RHqc$98&(q=hH?1=H)?T(j-5z-GznFXLpgO*7T@(rK?vUUF zcZc8(Atbmv!QCwcx8Uvsm!J!GcY?EUcY-Y(ZfC!9>i+gVuU@_X?(Hh7n$>H~o;`cc zVc$3A@VSD8x{mS)s5@LwXfcx$1j*L}wx=7r$BX+@dh%E`9fq2g9VM%gnKy5#ma;q^ z@0;)OuVAYY#vOwfFY4yUk_rpb6GTjA1Atzb`lcl4QQ!32 z0=l{{sfwznH)wx(wmFjb=l-N>Y7Tf!sU}SVKo^60>EA!$)g6j^^UBnRuz)xZ!{aiw zI#caAwAoOKN&t8!@8*(9kcud>e`Rgl>eEKFk?Bveya)XpVcoKv7XHp%^QB5MaYV<4 zyC9V=JYi7;JgNPPeO(Uze}SE$Wh6lq20c-aJyY}_E<)M9wGa-!{0uJe493(TQ<@we z%YMZL-IsPxqrH;9&|m%@N)VxZR|Nznf}8CA3J0&BMPOgR%G@rfRA6yAlD2f`mP#w0 zQUC&83tYkbhO|r`X+s^^^bS2#C9G?^#X}P*JIsH5Zza=k9KZ+Cz)<5s0KEn*lpu}W z4VwN+{PEH|*q0NJBKNM!HKd;GL_X_79~CRkba#s#BzZB1K3=d%To(+8e8<)J73RL1 zk+8Gkand%{c!V6HsQmn>{GIB>dKL2y>@97JVHIkTwH)s=&twbAZsVan!;^iEt0X`a zI3~rx{Pn;z@FWI!cslAT+cSX32Co{63@_{U!5+je46|qL?>(~q-othDQ)*vTY{KH) z8XA`PVj9-!nyXI}s2%(_^c1ELNAo|69s=TDJMI4g2>(5>i78&s@n7n}866${boD!a zc}0cIIW3~&TzSRAdPDB-%`Gz-Gc!1JGE#62x47;zsQ}+6EiI$x-Spvbbk`RkS*MTN zLtp=t*scZBwr?|?pSN0dMiZtWUdczOkNW!a6U5U}3JP@H zQKl1*?&v|fIv2pJ-a4X%`&%}#&sFhoN6}0Q&U_{qSYMH;EQ!El1tyF%1K?oa7lqBj zzRzB^D@~PbB&AFsKMW#d|AH$t-{od0aou)-$JcQ580AdH5ri(|G5dY|rK0?N(z<&X zWqH7jkmd*kUpy#cj*pGNI(un=NQos8*n+;2ouliGZk0|mLK~J9M!+R4V0m3 zE{l~n|Er~qME{leB(6J{IhVzj#<IwN zIUcPm*tvfP*m-=T$_iqcjk%B%qwU|adcoHDlL8_4w^Ja8MAa(9;Onf~S0NcQ#w-HD z>tH^9{_Xr^VDil_0CISD*NpJm;dkxsW4OaHtq}&zq+FDeOJVk#fl#v6J&}@Qz0^2Tn3a6(2k+ zuGz-c97=E`LMN$cSI)~Ny_kQt^1yBR$?PWp&f<-|r?YUi${ZGg=Q9+@2-LFggh5j) zjg!x9P(}q%_=w1fjibH5B_FZ*BmDe`PrKrDN){7O>9!zW;>nbcTGMX(gBB?M65Go} z(K-aEG{LrRy6>vos?PLEN=kB@=+A4j4OF!PFWNekwwu#SuZ5C6sEV;a+VKkM^m*tO^yz%=go+|HVP5`NE*PKeRiOkfI^s;Y z$w+Z$+Be7xK$vQOP^j&1Vo{9UX43D4JJ0Q>ebhwow>y*B!NON{1FdaQcg-FbKB z14K}nIGqrw=+92aPr>^GBe;w8m)JTD9;jKjECk)|W83d#f*9?f1km}!_T%}dY0zna z$27c9y}zXscuD+G$^LX>zBicsFC(1SklQ_E0BV8^I^rjvJ~l;@kK&np_je222qX? z{dP^^L$*dUyuO|zLw+RHDS|NrsRnKBpATl&w|Af)$GhtNW}FS{jLYfJj+mkE-e$`G zv*F&Pz*6Pg-ZZ3asre=^51%x~SYDXiAk;`*%No@{vj3+pCM}_1)x=-@5Hi`}y91*O#&I@dqGe zg1%(GX&5Q<;f*%qNQLVOEZ|sypHUc$amz=rrzw8d`MTjk?6D%Sd4S@2Ji9sl;(Ktj z@yuoIMaIwHY-(NttmK>##eIu z1Xpj9HPpml|NC-JGMPDhvP<8+&Su!hsg@I{^kljee;?&ni@{)>aJa}H?x?m_Xi{8= zT)l2%s|>=b)BN=y(^&Mg&ia^Ey^S~0hz$F#x2|*vbh&mF%jtGVeiCpKLcZe9atw_a zy$yL3Q1f0Gb!oB2RN#Euu{txRSsPwyb!elc)g85&71kL*<&aKtW6?eD;`a0T(jLRv zJKLmFl0{s@Krd&GF=eJ*hc^Nlt*Rv-J(9q$%LMk}QJ`KShMzC7q(uAf{@(H0a9PTr z;rzmb&&h$+@8xAStcA2r%X!-iJk=dr=g=**_0>RZnmy<-!RyzN|EO$O_3`l(3%j4B zW|>;07pf<}4C5_KECHT{UGESWtt)O-M;81m!df6U&vD@*wP)g7CgzEKuI!7+{qKHM zFB>T3UWuEp+^JU(o9x$p=MMDKc(npM6g%R!u!9HQy64D#+$0tMk1o`BROeevsr;EW z7%8J-pRaenf0w)y$W%~{^=`dfx>T`rQo=cb>#uJ;<(c=r=YMrqQ!og18%9E@mX1Cc zCtP?;*wQVPsX=iB6mG zC7Y+AV;hoeIAZ@EqVsn&`lSf`mb+0-fvO3(6m>XwzWjNc=@UcX7cXm)4*k}29^UnD ztLkGn<5hH?yG{b`E#QSbJtuK-suCUmtf^;?3`}1|NMOf({7Ua%3il)i;#)&8TxWQ~&;NO+|nDC0}=$@;dHyNT11Tmx1 z%e)4O&w^9(1+9-)0nGtPjJH2-WMn$g>M@Cs8WU{`_r)a3i&Hrmkdiz5wy8QpliLY- zEIDv=5e_)FdomH(8#~2rX^YWEZE34|QO1bLdo4(2*h@UcmhnoNUZA0qYP0=g$onA_ z?$hN9CUoSjuP;??*O#LAUst{SRyqnkEyo#LU0qRiiT-$-K$O7$rjGxOjvvpG3@3d* z?A^UU-_@QAvFXzjjw-jcns6B{x(vItBfD~OAow(Y_85erFOaV9&iv|O&et0vJnB!t z#D8(uc57g9T}S0v5*zW0h1y@#9|ksvE$tf;dU$UAZ2z=`sG8;Ew(%-RIL3rj@J`t8 z#auB?!@+X!78iU8P<^ejW!yJ6id2SPtk56@t&T~!4+1OiPnV`uum~;lUCU!|2~t+! zR(US)*L>Y+*KOCj_YJD99KU}_$;qK8`eRwv&z)z!Ef`kij+17kv-s5M$ro`FKp`c= zqvpalw~&`z_bE110s-=|S3B}Zl^Y2yGPkf0vAEz3XwY?<;KxgvRsoO-I5(`A&J}G{ zJuYx@m#8toz8So41+HqxC@6n;dVWGeMC~(09w5b*4lsaPlEL`&Bd5RL>*VF9xHoy# zGrTH$LwHPhK~oc^j2t^l3M@#*ebqoV7VSW#&RpppOm2`xb}R8L|7_CN2S<)1Unu2fm2X_m8*Jy`m+8Sq^!&ZuB)C8f~SJVbIYqTGNF#4oO{=-%c&Es~MS ze+1IkPpEcwNsG!(zy!>(2`jT0_t#*FCclfTyn`V>AK^6Pi)rD{E|)D~cRJ0LuC&cM zjV8CQ8TjnM zQM*YXULyqAjNV1G^}55VV|HXHYsP`GEaf^Sia0kVf@SWSo=qjD%qMD`lqX}m>!V7- zs~4tpVD|<8uQA)ZRgPUu^B}G6gjIwZm)FwXpBdrje{fBtuZzr>Ol|eXf9ELfIyM;h zU1NAOiA){9c63@Z;1m@lG&&KVD`iW3CI10|XJ^JCS}s9i->Zd*-uT7WqICbw&K5h& zp*VrVp0uE-@@;#(^f<4Ez3$sAdz*3 zUld_#U!~Gk7~5|-ErICpYQHXsP?ycm@ql5Wg>q)cJf7e4OtduELlv1GQpR8P9oLRa z)S2TO{cU^KTdzeW~uhHehW2#S3jrs3Bz2Cx8q%K@Ny?> z;qTv{#E zS@v~|8FON1kBFon8tzm75j3dA9Z{rmSx{J*8U@>mh!ROnSu22*Oq`_t#A@Moo8?}+ z5kx~5!3ou9{~G@Ka%*zda;2Q?+c*$9tw+3K$J;OE7E^e>tEme8g#cgCHg!)GECx=hOP6d>v&w8YTA#eQN=^E)kAMn@ zxA%Xpn5SG2Rc<;0F*hz!USn^ng-7F8(+9M6J{7}6N9roO(VkEEMSG#8PhIXcqCO6O zubt+=Pl-;3HVY;Kl+bjo>sGkgC=jsHgoQgi8UE(xE-DW~+fD_EkiovSCsEi>Ch+)- z08_zrG!-#h<9qM}S;;$gGpARk&ca(1|7G4w*t3EhoDg0^;p{));1#8PQI5qmc9BKI zQ_EXZ$~tqEXKJ~WgGIHB6W4%-JkMG4C`{oJB$3J_}zW73PdObNcQUxewB6#?2jCSKjmh@!NGhHr%AtKGR3SI z>_8}!7p-$o%1_0}m%VHeY%#cTy<$(#u!x9=Z#v7C4c46#ag9N*56W5FkKT~Y=X9J~ znD@*#8&1h85!(F_+ur0DZv*x`?hH(?rG^ax*}XMe(=+B#*6}?2C)gk#QnNXHAA5fr zp2uo()}c|FUoNa9DL#kKxh!};e>RcQ@I|YsjNEU&pZIxjfPqV~BwkuvwxYfpI%@VK z!!>oKqSwrw9v*(P*yI+@d2Bw>WYAAfC@n*W$5q)6Vzj$Q@Tn9bWmJntUJ*~w?KPcZ zFb2<|;ujMuDk5snx`7li!KaUgP>^*#H-cppB-B4M9ln8{LUt=G7E7<6FP_1tZV4$m zJeB!5IsQv7(tha-&*IIs>S($wKu(v2pj!o^w_h^k9Hpeb9yfM0ytv*cG4*|izXm^a zp01~_aN|UJSA0%Z-;*m}tou>li!bLAu>-9Cv7;cjTl-{`l}_%Ou*;bn&AWTEnW`ecZrQ1+&80lxp??-apSvx}=^iUgfH*jt)abdoirqif+G4kTHvw_6=1L~R#` ztP{bW-%6%8UQ6!RhsRR=!*cJcuFt(G!bQ)`hX>Ny9G3_mcwgc``Wcfi#^;?%l$ZMj zfS#f>)8k+`jt`HB+3fC+I9PH=Boj>evMMb#_#+`}cx9*-4Fc??!o@qx`;nYdim0yD@FN za-X1V2uayucqzvP|MB+RQWKSBE>7WUTD3)7(6v4ZMob3^g2`C z^VnEUPXg{>WkW{C#Qu`AQR1~>pq^-i`y9w{y^~q-=!fCHIuo}$Dk^%RX<8V}16mof zzUnAiOy2H2V8Jtc1hrD<#INWjnx6J~R=me=&k}M=+3I$!iABAn16@3#>3W)xhJ#kc zFZT_yJlZU($#Dq?>KJakGnH+FP^yqAI#t}AUfo)=npTz9L)7{!^;(nNPVLP0#y~3G zPmN$Zqc4~5pU)s63g+t}+mG%H*_YFAcEpzwAA|z~p|6itgZR%g+aCW+w4UXfD>0IE zt^+#;k+76LKT>W|$e)mzcwEF72%RnW7PBfDD&4EhubdiOu1=Z}l${{>(efS;8#9SX zezunu-|(CF=?Hn|R~~x(!M(t>yQW=SF{;)#X`NDc_yM1#_Rj9q?d9@=8nLwR@Z|Vw zY^sR?=WU(+5d9k-7P@AA6fYh-dBw}J8F43=ssa+t&O=O=uB3*Hs4bmRqe|(GiLyHq z70HPdWG)sb)oJ#qjLSW%0&_)QM_k~)zgAhv0U_X$?C!tu?#PIeZ#nVwH z2W2>7bIJ<*Z;>%hY{z%L^KGNYjYdnW8~YuP3j&}>K#&|Vl#mi!%D#-)XjoY<9qWaZ*npCYB>W%OUN7d#ZT@2dInEY zTKaOT_cj4?WwhpZ+u4C$9l`i>Rx1i@H25VksMifK_Ucm_i~QqtaHH#S?9=0V-Q8^` z-Bq=Jd;bJktCi=>&Gqp@?Ua#P%+yARR*2p zF8PL)o`0xx3V$!ZP8=k=CmnMUhuU!ar2Av?0$bI>v&Qm=KmEHoN+hn@A!a6nBNoal zOOXjR6t0y)zVr-#V`MxHN7{Z}Ie_n*{MD$bmUBp0mEwz}vO4sHwQ)`A{aiicgvmWF z1x6pNGRB)jSt>HQse9cs3;ECqt8H@c8_|0(TCgf{_J`#Y2invS3lq6 zY6+Z+cXa(}txGkwllBIbrG>|;ox;s!;azX_KH$Zn$gHz|>AE;n00LSnqx0b|C%WoU zakY&VAGYs4(+$cj-cbb;VJUgDZ^e1pWQ)XLxMwj5v7y_W%!mc)l*>)DS2*0c_vdM=I00spfG~b|d8Z zT3501Iw~G+s5t*H8}t2CDquP?c}H_2|U>ksphH z|F194SUg2eCUhEJmrSqReu%H`#fp=!^}e+u_bSAQg?QrzELgOkZHGw$2-xGA(93OS zQX7veu|xQVTq#nN9$IaNqh3zFM9Mn*hgm`rr$qA~WopXh~#Q^s3zI&tXMy9x6p3GS>rl`AR7($c$XA_0>xrL7^ z_hgb3oE!-2HN_+n$a9O4XZ;MrQw>(YoVZ>i?X%;h50`zr`5whw0p^taP^u0JR;>`v zIw1P=cs_w*JqWt}Mc&89`OK?Hl(}-xpy4g$KMq6DA0w05UMo&ABvnzhPG*tH_40RS zT2h!bAea0vLE5t=zBu8hRa83%=59NXG4bDLmK>zw6j*;)vLwbMlItDP z?CvYh>7N!2cZ;1g?D9atHH&NXr9zQe9rae=HNvokvm>z@K4uqX3&k@TBMjxOaoFO+ zKjX+fOjFNR9vO*7C4Gfa5V0FAIeWfiJ55v-_>F&cbT^>rvB$7i<%3UU3uj+_7q?v5 zMLKj?m-9y3a*;O9qde)ZIm6>_)(tY5cOGO9MPuq)mzg*UUuS4Wiu*Hx!-7;zBUx!$ zT$9wc<)+|1{iY36Yx6bn0AJ4|cN7}pQ~&Lh2i7_IVP&m=pu2IGX(iQ_^7{s{IwcGl zF7C_L5UC}MJ3d%cZXok|^mZU5jyEzm%YxnTj27PTm+zW`bj<3=l$=EBvpMwYqssO@ zKfBO^Jqu%q_(M9c=FK?X3Lc@)ye-{7-jUPv1qKZyhP};Wu$rLRPk*TV_2uH$Ag3-) zMYJu#PVa2{qNd$}>^WqT_pC@dAvdAGl2Gepaj$TRDxrz<&B^1!pTp4dA+CT`6i@4_ z1ZDTz-BLku$0hWd?W;5<<+Y&5AyI(pC*$rXGhll=?W_p1-Y$}i0m6Fb zq@}H`jT|tzo%aiJLx(?)h%s)&+&6P3(M2lLB&U%f*sL($G{FT5uz^F;B} zJZ%o4T*2?p_Wh~17Sktn2%Zi{v?oceZgC3oPjeP!mfz}gcr>&H{rvo9OWm2h-EB+j zUCE0XlqJiGbnNU3{+`e)IZ}S&=C)asRm%+fteU^-^tS8XTM|KoB6Mk1+nWR-LFw*!)KI#B^W{A=+`;^>II5EMLQ#;Tr94# zsO_1I!U}~V{UF-I1kpF0K2(IY=M+Yy9F<0|>am@XD%FY|(TJF8gZ@z{d&eEkCTdi7SJ16@~9PQcq zJl@kO`t3;PkLH+0bi^q47IRn}4)4x{xaWyAh+;|_zCn8tiO@pn*a44&WmoTzVnJbd zt`+0)sejHad>R}{N;5zgb8V{B^BgIx+-m!fZpCbk#u(>ts(d+nS|<8qnZGw_xMLom z&N{J{+7{~UNH3jfq%hHc{r<(kp2~Hb*XJQePUtr+WXFwvl&AE^R7*XDNW%T@v8>*U zP&l`W7P&7cRkEGng?{3*?_@<`%dX?+x5{0gkIl*bPZMfwA*(54{>{7#>W}@GDf<_P zDeEci`_(S*AhFjHJ4`VD`Fask*+aEA4nP&Dok8=fc8_zHi*7+gGq2KaZM^CFj2yoj zDJH^R5(wj-*{DRuvGH23lVK+(RSG?~xr|cU>t~Bel!;N`@2|$4P$A7+!-g zoS(JDA=tm(IL*w-3Ef`O+l{?O^uPLrG9%bvB5R=+oZO5*a2naLQqFgF4?MW3ZX9gO{4UM0Bc@^RLW0FGy3w z2FebftarO>U0cv<-OLl8A1fS}-_^5ZL`D=qp{$_LgPfCkIV8X>IXgW9$hNvo9l@5AGKip&Clgj}V20INYj;Rq7 zp3rX;e9_oK`X_ZcofH7*DlZ@5(3+`lP4JeF4;SNaw8ul}r@r;V!ojqjNVNlO1k!P>cg-H=lAmcyA;qETj+Pyql*m}eR)qYK6218@H1WI-p9_Y z@oqwz5tJQWLeq-0fgx_gvINN>Z##ZpM_|QjbM-qNsIB?FI{OVFtdXi*&)HusW`RDs z&*{6_rS@+;T4ec*=YKgdBR8pPj)R{QrQEiJ>v33GV%z8Efto%ocA7k`>5SNB1zei1 zuVHbq!ZMfD^H#J1dC)aySz~E}p*s|4i_h>V+G{EvnPqdRyI2}%O1{fzg;sKfb+YJ2#SI+|glUQl(*{25dW!>qR(A*<{(_+)GT4}e z2RElS9BD{MNWGJz>se5nSaEOOcyt!6*r3z$@y*h$gN?A}E^ftgU}1vT5D>QaPtVyz z5izRwGE7Vz=!-TC|6IA{z>ZwOocqV72KA1_#;vX9JU>4Jh~2doFs)1>PO;shZv&>- zP@p@8P{gSak&!RGfY!btB1LwsgCB8sJ;gr$SOBB3r6D5Se9<>#m_%u7{TZZ4`z@%~ zH-X^0#+97DAx9`s^<^*N{Pa(lY|ajxxf){8#>YPo_uno>9ydI&NJ+O6+PWs!*F!t0 zdfC!~F5L7nHw@gOuKR&Uzfsc1#3YoJ)%>n<`K?wYqpdv+2^E!+ntFvU=XC0iER1h5 z!S{P=)+ST9Ouv`OE=5Tj3FAq&)=+4^h}vI7NwXcLJQ}=>#$fj!a)OyPE+^W9A=hG*e$b7pUJu2`q1kE_o}Y9I)UMEioTQUjQ43JAUcM;oYS6Kh0Db-ATh>2`@fpsGbl~O^Hj3Ed+2g^X zPqE;A&p^;QlzZ#y?z#~$D=r;c_0}d)GSQJQeG&?E00(2!om{Fr$1UOJCSXaAvP$MP*@%P zMwLq#9lxb}82gxb7(+gJ2RuGsU?xEhh&I4&8}Zw+@8G3g3vf@o^^mqh}0i^p5B!*LZRCu50Sl94f_>yHTmM-Q#1>LJ^)kx^S56zYZW` zuVHJa^gRlEx;-d6EWx1kQquuWtG9N9TS#-B#ZT!=s>ppYpa=@E(s48*DTk7K# zzaELhS@e~5j1U)oWeDe7L;?KmmguntjcO`N>V{~G=KWMj58&294Gm9RJ5p0#TI`-i zvJKvE^}*ileJB24WmP))a8gbY{PQPyj_!A8A175PdeOA~j9OQN%$>l?)03brRA6%k z*4V+*x;%N`8{w_f{6@9=6OYY;ZK{ogNVboVrHxFQxHRQ1fCw2IsYPYdd!F&x#N&s5xQ&yuwWGf#LDhe!vr% zDl&>nYtfd7(C)ZdlFDh~bFM0!>q(;IXaY9aFg94*oA&#K;Pw((pNHJEh=X=XE|el)91I$31e9wVH7rUBr0i4kST>h zb=-J2ua*@N4{PnsM)%mGyS#B)ZXj%SpoM`zEFLNxjMgXEvm^ z6_S|(O({ewt|ri!t1tLtEU+1qA=GgS(X_o~l)&UWSY!xKa~8=n z*{23vK+cC@k^jN_qOKk^w-X41V=!Q&okpJ1VK?uOWg=S66kU}St2s@p^}=zcup4V0 z^P>5uyQ04}WWKKA&0e-D8>;Tg4zV-0*I&vHGjAsn^~7`qwT&g zW}43HY=4T1;9RVsBgwbU@u}obU!KNV1oC#|$0fh(@Y#OFAlQHxr~37j-yW9kTB=5u z9u~2tZ67{xiNI=}MM?_$%Yxz83DS4BfRR&&ghvV*1F(akMiSyn!;BZq=& z%s098vl_nDk5oM2{=@6MfL3b&gqWUQzR3|b+1GC^yzw61OiPa)X1(NspIc*;{q5mH zYu}c=s}!V(BWw?H&v}jTMQ5XNF&ewuT(Nu&BRaO7wRI(Ng^s9OY_>unqN^%SwQ^Yh zY<^ol4%$AqxC<>-A;MPO7pN*Xp=k-kbIuJeuWAqO3$x+yTO6A>v1zQ(mKr#f6XG%C zY`k?dYE~txR2WzD^Ot6USi0a!QBWGBcKZifKFLLjI%qo2m8)?sW67diHgn+qmRs~2er$W6 zsG!aghdr^NczQgrjW;|vFv2!L?Qz+6SxA~obSO%OoNsC2lF%gMDHuLQtLeyzv*?E= zabp{r(@0~6qmsQ>Bimw)APRzcK8bsw%A+L*KC^MJDlAOFByq9H`aaXT3xyXdNa z>OjjB_yEkFTyZm_nW9P_3Ak(H8Q41QJPlUjRHqrNBQIo@Kc`opJ-c+fMv}aT9(D<|Z}w1q z9EIFl_pVGA_zSfJ*iQP3Q-z+OgtK+T=FinbfY9bG*LIFP^=YN*wQdW-v7QRM`JD@? z%$b>ulhk{m2{GXIgG&2YGYC=bofxHX+*V88La#oD{5BY97C)Og3+T|^`Sp$pu?wVC z+>u8Sac3jJ0R%l)9LAh`(IZjkkq4%6A~q9v_d=SwRrq;z70%ryb(?gdF!bS8wz>13 zUj%fVfd$uTEl6{a@c%80hx*4>_`=z%X|TMX0=+{xH_3-mUlHXJI{KBD5Q$V!BjCFn z4|LXuFG%Y?jTF`X(w$B`{u&t0B6K01lEB#LKiZcKtM#jz5=5R+I`g7cbZxCE@*9R4 zJzbJ=sp3vXL%ND>NZB(ePK_yw=t0Q&1wE!fQ1j&oQ~%xo9^G(1bn>F9fYbQG0gVkV z-D>*SQ#cFO*%<)T}-gZ)N1XcNpSQw`+htXS-VMytJ#H zSn*o=Dzbtx&+|;k&K9nd(T8PJm{99NjAQT|aO#HQs#Bk@y^QemcuG^a9H6!Ri|2tA zV|^~7$yGrX#I?Vvs^(GQ3xg$fSjc3cG<<)C6?bFll9U!i!D2BEVM49FyBtL`byMC+ zUpD8_0Xw0Qr13wK&7W`>K626O&wV2zYnk#{qn*!G=QS=;y)E|aYRRs7y-R26RV;I9=P*jl~izU1n*ug${fd6TEw>>z*sY zDW7*{!1k23xsVql=R}ha5tY5pz))R<%#QB;qW~1YD1c732|jZFg~y@~2t%GtV780e z_>p)#{@u z{6$`x#2z5Frtk|?9gRdR_M=nmvA-=RxgJ(-{9PzoNu5IpQl~ndh`Fzo}q&_gChn;cMPcK0x)Y-yiY3r zKd3X=*Em-$M5dMAK_Cqq%GH3eTam3Qh=k>gP>hOOwnFS!Diu|pDF)Br@Ca;ZB@~Hy zMVvWbTY9PQd3St% z`LawYG*fLic&1fE6FQ8bI5V5=@g*h8nf!(3H{S7a6C-J?(@q1X_=Q5gTf&*QJ6!Xo zi*KETMLE(D$Sp@Ta5MlkDow^4)I;nQFeJyHg_-{0g&gwUBrxZ`|+ll7OVaJbj zRHDv|d~i?+3#&cC_qU}vZ4O=+1=gZ!{JC=q&)&UT3&@qtXEsz3m6!7zi;+O!(}ddR z4w1gu6B05K0^$31EoM{vaAJz#!9d@u**Bnlb`V`#dw@`a!2MUPN0Fpo_RgGSqg)koG+DOk~DY!igD;S z{Xa1TCvxP|`_v!_L}UTq9sA)p%yqG#X={T05DhpPZpaUOA5HLFb>FS>8R&A`jjPXQ`tPl^+bggwG&>* z3iGH?42HHU<6{y6hVo%E2+Fp>4|l1ow-DAa|XYpyX~F?Urv`B&pVGA2RUc3hC@A z+jCQGNb2jQjgh`H5u|?A6FZgVTL^a=vgR_Etiy9 z5|g(WZti&KvEtr${U1Q$^ZtRvht0^0^khF^G2RY3+~NdTEzrl@0cOC1AxzrvmE15J z+X*}k7813tS+|W`?kZKgCBC6yM7=RL?3|9n>z`poqP3u8UKiuV!eN)Xv&}j)ofA`)m(zQ%G ziJZL`k_t2S;!3(*LuwG@Kum$ru<>_=4u*ER+r>CW8ezA(-O&w9b(DWZtDU732y4c( zDmv|4l!1|w!J0N?`KHpr0fR`YxX;A2*7sRjzUr{M_Y}}9dkKt`bqyZ z6Da9C<%KQCzwbWg8RmbWsK4|vWXNPVd1S51}=8W4Swu? ztJu+EIuVM5cIIV=3OcNPMwdj_c9*`OSs2oP_| z7mmU zZUm!_aUVH*$LhZUC)YWc;7hN0y>L6>q8?2q&Z(Gi6BV9oj`!>vytlu#Hje=kP#J(VE}&B^9hag zxTaEa;6D}waQY;7FC{IV3>+{U7asH#zR%&o+&w|;eh>2+HNzARZBPY(F$8t8)|_NN zPn(dD<(bb$1xQ?gra~wEyMzmfgzP5Z=G>Abhi8f=FgapC*chgHv4m0r6>hMliCIg`As?Vh6&@hmRng~`%*|C6tS zi&ZL#M#5wK-&`FDfU5(s?397_@SQ&viIJ3ipTNC5Vy#xpZhBx?TU+Nt7NaY9G~XHH z+v>k@*C1kl*M7AqgMNCYO-f0*=A>0Auz&Lqlse4J1kLqLI?2(;DNa}#?Gl*(`yhqM2A6y&hyZc|sV(%Dxaa`wOh^7t75`V;ud4=1|Na?PTp7Ti{HGBAd;WhnVophr^bxzIv*$`;83*U!SW~hv zzrHo%p940OBQ5FU{o;0e%2i=Y(}UX2A3C%MN}HXdxLOSaxD9laP7_?;LICKAX~c^^ zP)tnUc4l%iXIVo}$-y!e0G~?8za$V(1H{XN)rY=-fDgvi6*3O8CBT)i+WCC#XhIkO zOxo{y%h|An&4%FJIXTU3ZBmLK{V(W>8zIy|>$C+h8QPXvK8$N?7>7soOs=<<>N&^~ zrp&b!dg+xB`%6+p?WyYH;dkHT94wdB*YFpC3f2j0enS4HxOVP*f`{}y7(g|`N~B%J`Cx*4eh}z-a_1Rg z>Dw=SVopCGT;G9nbS1_<32Ms)M(L~M28GS8_Q8CdNR^ zsozS!fMq8b8_4#2>{$yt!_RE=qF1xnM%`70YCjo z6iNWE`YbFq+hp`M7NaA!QxpB7cZ9!9^j1j({3|CHp6r0fl-fV?_Qf6Gh?q_P3oBx4 zWyVOE+1*uF7#`*Srh4-Hem~!YM~EDQCB^*_V38-}IO7%+{(UQ?b^F_HtpMf?wg7?U z{OoAadmnvS>UTGt_yGW1-*(~9<h)X^+mqYwgw z;O-DygS%{ky9al7cZZ-sgS!Vvg6qcJ-QC^Y^-j)vzxRIUIp@#)1$$=Co|*3I>grmn zYpE@{@DOhAnQb`Z=-ObnCnbx1SB&gaKAdqSj5f4>u+^3e1#G8(Z0KkGuOE|2XSWN~ z44nr|-jjRH$N3o8OD(Im4in6Oq4>rOT{~W?rwO zE>(4aPt6!ZX`K0>Cu2gh?$Vv_{16PPiGTSEOCr>^3jE;@`?1%Lbl$2za3x!G^0{%Tk4Ud5T z`%H=?6g|OQnE*KDSdDCS|A~>B$27mDX8LIlUdksK8)E_9-fI~+yKI|%(=eVCv$XTl z-2U=*yyBsqF!S1efy~rZF;amtyn&n_u$x~-)n1z*f1Z!$C z8_Xsxn{Qi~qV-#yk~Xp7XD&@jNIg>tt96%ko4^-GGR_R~^C#by)1yVcE~i#;k6rKV zEAKsm;^1fRg3E7wB|D>I@Fhu(J-hIc!3_z?i5vD@9<4dek)6*3qL<$~y`fABdbl$9 z^%PXYM=Vvy8GkuBe-1nH#Lfy$X0}?Ds`>+B;HYLdp}P; zMEfH9rTVnn&C6dbS69I~<3#`f5Td-8G+N9T?+`bk?tYZn3e&X-dxNfgZm}{5>u9$b zh)!oBc<~M{YOvyJ%@FD0W*w$K{5e16r^4wEVJX^6Rl)`Ea*2KW>&ZRsC9$u7Rz#ffh4oh^OWY5T!d@ zPw@hK#fV-^+;>}Y9~{0LLjx6O>H}1kad_@bdp7J9l#HDpMM(V{N@Qa3g+o^aB`YOk zLCLAomDJ}7GX`K~%{+zdA*l*lCJ#%n1`QpJ>aJT;4Nm$yhxR4=;t?&fOkgZ)XCTh=_83o+lUF+Dbko8zbxIjFlahM2AjVG7}7PZ zh@}eoa3m+`s~ARp^U+cDC&yF@m_F#Ad?VYD$ch-WYn!YVa&#MRFizf-1Ne*uo{Mn|%q=iqO22D2M$p z>wj1{$~SQb**uBGa+J82T78T?U5f6$89>dai0-ZSuI2Q`=q)+k!>Lzj(k1gCdpxyb zP;`W^YYD|?O>Wi8;3kC@kjLF$(~3%A6XHiBoXgGvH5P-ep*5Z!0(A{pMW zT?y|B1?!1_J%6xTkGL2U-cE)}LW{OulW0V(!201b4e@I5-rXH_wh9rlLEwBPgwS}0 zXwUpm&HKrUnG4S*>k-o*;5RDKd2(sZtf)D zmwMWxiY=}w<3@i{?eX)F|G_ko?`oWw#Y++!;x$Fd%*uOgfbMWs7q6oK$*NB?&J7D? zvV@{TlV4VgvokKKdv`55?rzSt^~D=h3?|-iehFvy;~cQ!aAKQlpWxS$WW1#Oa^@EG z8Vpo*lCIZW@duHmUGzKag<>a%QR4o@S%^RhVX?dFrNJ5W zeHFzV1De5J_m0f)X?K|YiG#eb?0dKiGomU&s&;wA0{_w!We;c-pXwx2Whtl{O?hYE zRduJT{$d_KxP}lRdT?;=)WgOZR3RHwL1Hk;tTONY1BO2~q<_Juh~=ppde@41!O}nO zAC8~dor6%Y$)NFE6q4^-Yix+!R0G5glZr4)j%;V~g4Q4u9fn&N5)m5`t&C~Z z0KpFD?`K2HqHE&?;(AJ8a#n}oFO<=^62OBtUhfYj;1W+!w3bg9P1O*LCjVYzrIAB^ zt&{<>rjq%Zxh|gM|JGAL^(apjPixMnC||a=JKavcG45Yh9V>)dIXoAvPt~6lJcA;) zP-cHZ$F@Qf@ay^y*rVi|k)^H7;*1>!8fWTlIdFI}7d9JC2N(pozr0jQ6;ZWcby8rO zRV$>Kwf~tw-*xMSV{3h%@vvqoAoT4#e`=(`166$t+eHzp z_uIS6h%fjA;YC_IV=#3&UBY)$?R#jQ=tA0VfGPo!`4U4k4J)Y7=80>zM8eoyh!g0T zxb%2gTR9G^IJELV3c~qzqU9G?_l{F%$0PmQ>#Lb1S}ZXWnfc?|wiSgf^d3Gq@oEJ| z@2*Rq&bI|J#Fwg&w;>#s*q|3y zt7gtQm$-CSeM6 z<;)J6D`nQ2k)4diJ#UByet=DdeeFzbIdwhTRmpId+RE>%rR1H#W(P$y#xIY~2zv-| zY5nf2&+CY3L@9k%Ey+&wdFW7LZS9eg!PmsE(73zOcDF)oBiW|CT4TA=Swe4uS|qy~ zy)ig_@fJ@htqb-pROMwOz5FTnvl66!9-(_wlty94T|eQJQ&sP=q0CN#=CM;QRQ1%K zIJ$ISF&8PBJnx%is#OU%b>e^h!=aTSdjN3egkQD$PcO60#}*Ooouk6Y5$~Iuz2|2& z$sIbZNO5F?OH0A_0O9RKC=Mu%zM%<@pv+K2^@jVd@yWy0^DXgm=)wj^*z(5x_RrzjV2|IQ_vz;tPj&ycP$aN704ZC#P z{CA__6DEmBDIkvF(%u2(WikDe`vo@2_$g`IGJ*kc;ugTTsyo?a08o{7Flwo7NDU+9 z)hbmY^gG;$l;@%HU}AzT^cUeXMf<`5T9GUKT0S03jXOpXu_`iRK-f{_Jq9w zsrSP{D6q9lzZdyrsAYs~JdIUS=LT}h%vVqolrxEX)h!}+7e!X+AS!bt;E{eH4K`#t z3iGATk`N{}GWF?TfU>{F^rX_65a`fgi_OI0aeW_CVQv;{{wGlETRyKN zK5q)I7fCvD8p#P%;o5QtcP4Sc%;XyPv5s8tBeMGCDXHrY6fQC9GDo!H#0cE%akudl z>PtUYZ_K{n5)2-K*Iea#=A1_3OuNc#@xPC4A#GUt?U&ZLp0@JlecsYORTi`9)H6Dg zm*PE-jbheSCrdKomgb9+D>FOa_&Lt(g;f@;x7Oo#pMgS*J6Fw~m0WT*W;cG|>c&Mk zb*9(4A3b_wE$fxd{6qJ~W5J8CB`~*sPYh3O{BL`s5vAtHlFA`E0`TXV`-S*-@k`ueJrrRT*|D8DE1sXP~jL|hB28)lNo`$`*sbd5uEY0M z1!S^mK7W53)%>cM0clQZbb#fC4&7Ae$c6lCEmko;i;U+53dXf1eKmeA@8B6`Dl1B~ z_}xh+Sqd8|)|Oh=_p|x$%tIf|c^?VIdJY89Wt_KA1`%4XAJ5NDL`<*kf^RJclh_o2 z2T1ho(DEpLTXDua=it5Z0cyjvig@YlpLI&e?b3{MQYj&)9<7-UM-Z!Em_fJ)qY#yJ zxL-0xa*Wq`L|c?QhtwKD^6rR}UUe|?B?i2Ur{g2dV4)j8*SPK|A zYs1-`)BQ`9fqu@eN>Ta-FzZ%+{s-4JsorR0oY!vb1VY(9{gB84_=e&u0z*9U99bGF zE_HE{t=Z!&?vT|j1d-d+K`X{#gXe)$8Fjy4t&EhZhuxdLg@Y{FUPZEV=xpzq?_RL4 z5fX2~B?9${YpD`#zJw1BkCY!tz0Q}XD)(DnJLlA(%*q0DCsC*DxcH%5ncN0A5L5|m zcwI?J?LLn&YKo^pPOcILUflKK7(XEVGq?+?FOv)q`;3i)F$}~S@#*m+fb{{h0PE2R z+MHvVn|qpK7f0@%>O7&-HlKzIsL=Kjre8!k&N zvas&*fS)*`@vfc!NF>S77eSL;{|0h=w~@1ic(uviSHNlq>2Vqu91`rbx@ z0))+~i$#Rp(~EF3$5T^!AOFno4=?f4zUIIc?&PE6o=ebA+`Q=;kjg@@BiZUNR{m&^ zZllV7KWQpY`Det84{Z+brO*YZL=CMJMyHD_3pcL4tCVy+%a+5BkcgJ4$frX~natf` zzlr>8Mo^OHXzuhAD<2NMSw=5icp2lczPFm&Y#U#S3G7G?9A#b>qX`u&x2pC7s7p@u z^D2NB{!=u{YS`18&`DRf$Qe*lISR6NP#W6VHU1mBAra0eA{0gbA1;7aZImPseQxg> z$EU>&r~MRhbgDW)TT2EaAJc2;ss0h}o?%YC)*4KKF@SK;d14+oeHWm_$;a|9qV%72`-jzy(7e?yA0ipiMt$Bic8kwDEr-NB`2h`O4t|KQAoJdls;5TMJbheP>zaMAMw-PertMl$x zYbw7!)BKM%>#n7wGoOYuD@&`Sq=^N_T3jPVs+es0#4iih|1Vg6H%`&Yn6H~ArLS-< zI;LaWAENbie)v?4yRrw$>$(=HJMCs=Z0;E6W&e40AAGC?fw;J(%{ zoFCfFEK;kfhrmx68gbtboCw{2TSa;g!MXr@YdGz;-B20mQQ?6RTL~A5x172jueIPi z9=2dyD1$FLJ;KA*NKHiW<4a^ZuA3gerweXyFN^cWNEpIo4qQ`=8BBK`9ZV_cJ#yr8 zt0{Ne)1ChHxVs@~`h?3q`J<@m)z%{+=2E)5=*9XE2LunMJjW*tL{#8LL`3p6Wh^pp zs@Hhz(NtPTbg!-9ADNq=(HFQrmWqixKTCLu1!!P?Q-yWkZ=l;iTupy`r=!(3Y4vQa zdy=y;XEz&bFpYxB`EA0%#QUaq$qd^JLVx>C_NagQPJiFuJr5uF?)BGRfrXEyC(C#L z7}!8h)Mxd7#B=#_n6zE3I>Oab3n%HgOI>zLNp+IRtkmeSUX9wXju9Ct zz3azs9%|+vv2}qa=tO1%e=<{=4LeN9$vo)Nd?VC3B?BM2<&tExJF5as%{5jAfQYCq zFjsl)lZA;mUS4`9~+;W2jvjRawH`A>Qn*6>QV$dh&q2qEVyH z;w%|LfkeiqBS84tcwu{^8N8r?>hJcR&=aw6X1A37_p&MZuaW#~uWQE1VJ-zhK4YgI1=b|en0p^SbBVY~X zjpgLalfH*wdRgT>45a#ayHS77p*Ofwjb%Kzxj3T+dbV}$>VdX*26%d7Qlut7A%@h* z(x);77kCx?)iVEQcy7kf+S-Z~+6C#qVp~N0tfvAC<;}zKHD~QcF`(Z539f)+Uj8P4{L*`1<}-ROe>G1P^L9|f1&AYS90spCstEWYAbN}gYVKH_3+Y$s<%bZAzsO#j(VKy`nSk8Fo2%Nf>=%CvRda0nmEJpYRW zMj4EK7sWGd5sdqwmAZVuu~$R!j}`|$P79Su!G8MB;~x~~8h8khAPA6Pi{9w}%Xt#Z z=n_CH?jQLSg)A=|2P5o?$336p#g-VICErgMnW){O7Kza^A4n{Nqz)kdc?rrpw)*7b zy(PD#>Nbd)Tv-e3?Dm=i&E97-TAHGSXn8;(BKp6S2v1P+O>uEr^#Ms)O1d2~UEd9rU2@xPh|eD0 z>qa#nlz13`3q1c+R8;&)WF0nT6;Fr~|Mx3_I^*-ntIF28EZ#-kUZ}v)H(Q0) zjStxB7#`l-PMr8|E4T=Zg^X+$eECjb=ki^YkVk;(@8)(e3!}Yl6Kx`%9WihwvyIdv zK=Y#cOJMcYJGZD%Lyv$_&T|-GA0#F&BOhNg<`8^M>)x&(2b?Ku@a4)^VW< zh2f`Yo;;^xLPWho7S73QqpfYadOF@zx`2=fFi<-w>ySN(?m9p=)V?^sc6{8%@^9)z zPm~Id=xXqBb#N$khuD6W{^*807A}d59jd(K;-Z}EdguHo%m@(`noJtMO`E8-vz;`R zURGbIt&{oI4*cBWig4_h4YIslk$jd=kFZ#6)4MI`^@9SLY@Ow01v(b7Z@p8}sm<*= zfl>vozS%lDbnwC%Z%-<**uOHevFjW16qy+9l|M1sySNln^nv|{temB?43|XF7JE7lWWd`^T0|T`Z zAKZG`?JsHs2Up;J7v~Dpj0$SI&~gmdjxG+r`{6NC$QAe={LBjf8id zE=y7tFGLO`70YD9J633ZR?)iQj#$tR=~0Rvv2+3M5$pkD0-!}g4GjqCGY@a0&>jon zk=O+VJ@pBV`qjY5;AY&zNV;Sx0$b5>szu_Cj=~=!T{=})YGU-mhhq-HAJ>Ji*-%^p zd+%910%k&(8DS%CU;s48wCD1jkpTloIwln|w)(JXKx!gIHu%!PJbBvA;1MvjoZ=Ax;AAT>gU&!Rd})BS z318u*n>7pFhL$rNUwO?vvnK-Ue7Nl?w%LI{#`z%6SzRa$&tCrNw&c~6g0c0U&&hOb z`$dBl?!yNK9(ocHM0bN>bye>dfks!Kuxw@U#M^IQsZjEuUy$GxtdTuw&=ycG)$DM3 zfL`;SMp02wDwF~^2bJ}}x#rUpQl>+^`6P>v!|oXL_aT5JGv41xn;uwdx98{=bfE|> zLlB+K#1~|LHaxR~b^o)5VmkW#BP)h~aFQY-5lI%%=xCLB&al;Kx;Slm>r;LSIq!Sz z@{X+!BI4it*Llpv+6a@AG`s#0_DknZCQ7*sx%62Z^-AwlZ!7sQP21f_H(YK^+2lh(T5RI|znslmAPLb!Y=)90N6hm%qF4C|0vr)@h!w->V zJUHuH0S__R81Qo1sSFoB7E7SAKU`!$9&GyaK#8Q(S1uNk?jDiV#i~UkT)*S8N>^=d z?fQa=69Q0uk=W?n(y2~~PE#yr(a0aqQptERayw1IG2NDj+ol`#$!L_$hDH~kpq>cO zPOH}zxA(oo%u%@kCjevxJ<7!6YOk(cIdaueKo=s2XjU2W3 z8YaD6cF}z36cijhS+OzV-C{kXQBKTdAC~R6Dw)pVR&fyeS#2fyS2R@m$X^x%4@!}f z@^5=5Wynj}GphpGbY6*-)~ynt{O*ok`No1-HYNQtB|?Z3thS;u5LT*2Ec`~NpM%E6 z{z~hntz%NzX7yV*^p@}LIHg*bj)n#(#QJq{bsCGQK{1pA9uHj>>s4!^=*cxQ^9Jv49C;GM6h;^CUW= zIDOJ!+M@F0)O10uc+~6VNtUMO%w8>Oc`f3Q8vy)g9SgebdDPT6m%q|#Ym;-fsTvw; z8dR!}(T+mg-!2);WwMO_nQ7y4yiO6CSouKPT5kbIH%0?{J$k+fAz#m8?6z zrbt^zjwF@o$uF}LVH$yB7%gj?CO4)%vQ~Y~Yxn%3LxSEzt{)MT&VkWWMoci$KQdwM z8dlG)7}67G3TS2uWa(-y5Dmc%4h{m3VIAKs|-;3c4Ch zzgn_uu6^;Y@vU$2q9&BfEV`M>S1ta4a81R@`Bq{C0rr^$~x<-HWQq32{u27`y0V=Po8=EW=;kS{r+O8KoZ_5%&VCe@m-}U!GcE$l|2|gtSMJ3*-nL&uW&sP z`}px=K)6W)v3m~G3tCnzFk~GaI%aw>fkgve6jd&2ECx3I0KW@DM1idjE2jc+h@O8I zpuRLH6~Tm+U;x%|_+)ffTjlA3W+&tOeXkE`r^RhgwFP8)>^I+Z=T=wfU9RuU2?2Nz zTT8&i_PwFbkr7B3_ud16o0Pa}J|ABT+($U!{ENTOXb=EdTtLvI(|c-VvBO!6!*Q*l zO0U-y0EM|kSYy2q*+(2>HaLC2;;LuFP-gpU=G2_8`4_e)E5Ov+Ha0V0DAtPicn#=y z|2`Nf^a!-0lJQ`lY3-){bml|wk?iF_zOnK1K*W|f%W3f3A6-9>PIGGxI;+`|pFe-v zzGUghYuon#A^FK^0C*O;%^&5XWA(rYcbT*K-`IqSX8acv64AVLKIO=PfI0aRm|RWE z*})S(;lDw8T5-nohL?};GsF;8kK%6`!uTtp7s|a8r6xUVU7Sa98@`WHP3qZ$F5P_0u@gR9VHVmz0ptIJ z1Va6ZAi&;aS+LrNeEm6BRB>G$Y`xSPO~`wzz$hYb-Nc)Ezj*ZXL_mSeChTy*hJzl9 zm7+(CjVSBh`~}X=o{p;ewFF(|?Gz9BFnHx1QvVRYWP`@7tXmSlm+#gM=j!%}n{K`p zIFt#;v0*D(H6bza^10rD{T7$>j}cI)QdK4r@9KQw5ncO@l8uOH+juT&fV8b$|6QOE z4FgM@T7p`U+J}bK=7-rOdY$GU#NZNAiXEyN(5iD}Qz*BSfMHj!tW>Dl-a@58ttRzn z=3Iu!_53TUl8(|0KHrq8xWzWKq~i2&PXq9J*{s5kY*|3C?@ZYyHa>o+f5Ir0*9(jA zv>}~LoW)|c1nx#ycz^B<7jU6D%cl9w&5@+akh-AO_lLp`cjx3!?kn!@Wit=x+HH^p z6qHr+CWCS3a%^{9F=5{=IvVZTjJrZcfCb*=xbR)&@ew%|!>i1}`5~x@FnvZy!-Bb| z_no)1ZxE;X@W7V%;jkqhNiOHs$yP+=r_eZ_=Ek`j^JQuD`{!zT3*4#57mHBy`3lHV z&6QFA4q<9qS|y%lp3&&iWhoC2T)>ICdp_h`A%EH07#JuKsFZey1DvVV7mk?X9l&`$ zSie4%Xv9idM{Dz1qEM3rD{~JI=TU zYo^K*m~23sJ_VVQ*=Lw8F&M#LO5D93>5`QhBapHRHeb&-t%Cn-{H-Gt_ezDthL=-V zO9LVIj44_TmW^sF{?+;L8N;bMZK||33;y$Pk=-5JSPhNw&7M`cEhc=q>9wZ^=}L2) z$4li?Sat>0T79&LyN+gdja#(-WSDxS9wJq%5mhfeUCznA8xCd>EQ7E1hXP}Q*Z3#m z!{NSchHgvFtEIEH3yHhF6wWvF0^z2Olp(1B_KbD|uoC03skYSvXx!G8_Ar>E+qO0? z-AgE(7319R?Ir9q1~hv1G*z)U?9KINbn17H8cp;feK2mzC0gkH@lt-{h*;)}0zWVT zTJpSo`fIQB!yU%MiJJ&itf}Gqv-wn0tDd2gRfey8k5oQS6igcY$HwSF`gkAQXV?uzM-!qJQ>v;!id^=Gf43yn5(NG0@ovEBT4p!Tu0V+EcX zd^w=(RRee5#qA`T_8H~hd#?iaIcN7JmN};L8^PbpbId|n46ot0v9}HTxqGUGLs-41 zucSgfp-OAmkbCE*<40975hH)!A;Hdht|w-e?l2 zKWy|7o4cL8l;~E={e9~K3eoM|43SdTBHjDXQlVr)D0$Iwm~4k((+S1^3Jjz{cYh0# z{dJ!J)C5iYkqt0J4Zb%Y;M|z!NuJbgrSau^@4jfG>rIR$t$`9+ zQzj#chcNls6Y?p91?i+IK**e@CVV{|@A0x31Sm=OTHL!@l3Iv;IGvBhiCNRA`V>2j zKt}N7UvRSNexuqhEAHH+{w=1|q%-bppg%OyquPAQ0Cbn-Wmry4bi9JQQU-o7-7~SC zbA(Jg_X7k@=RsFwtqYmasWQ8it9?3TqibZ>LOv{K_x>C5-hvAg&IqRc>Wx2|c*aNy z9o{e2!pLCr`l?L8JGg|0ho^G-1xy-zfB^C%_#Zx$#DK^%l&LJwz&GE7Ivw<>1lnvr z!vm5Iuh*=<1zx-_IK!>VN@{|Hu%TW9Zm8*Ka(`|gI${^pd%5Q!BlUev5x1x`W|GpZ zVvD7aMo&Z%h5&LC+Ak~Pp&~zS=171}HWchI094&B9p%4Ti&A35*1h?c7`tUwB%}|F zaiZXS&90g>1c=U3q0XY(E6d|^5jg;@JTFfOxHPv^LUSPeXQEUn2}JyF2<*9VtF3Y} zi`u&N`8mc9rK8!}T@LhE8hL6oebFWtt53w#gdq^ts?t64rRDNp6np^7peFq5TzJI( z-$<=+`2KJ18;G#~X3RD0}-Lj*8>}w}?9C{jxXQtd~9;!K2qvOs5P> zbov#g@QB55fw$eL*m`1(*TnFe#$L%gB=bUQv`8V8m4UJ9ET1NWSYJvh$j{@ zrW;M??QBMTQcmwTHZMaEN+Pg45;GzS5WHc=+Evn2dP?fPzMS36zlrw|2<8&EkVHMT zM1H;@NgycAgvNsmxMBJm*3l!o2XVy@B#8e25tQ0N6n63f&2Me#hS>V*`SG&4a)9$F z_!l69*kwFS^+m6I7R&to`yhSr5)-b~npr%`TwEnna?ZN_^RsaO82@i)KpT3dCm6!V zN2b@CW)x);fg?Sb)Clw%K$7y&u~K~~5eeYOi4c+kbVkPp?ggNwe*gkOsy0TG06YmC z_)94RP)wkw8oTdI1|wr|UnYSeDaU*@lFVoW9mSe*H6h(Iq}@FQ8L&TmBqgm zW?#-#)__&KZ*^Tm$Px2kWW;?t@uhcoA$+66qcvigOWRpaLD+cdDQsANw<$BEIO0Z0 zWr>QA8IINcWFYP?tOsyuehb;ok zhDna8~aB&eXhF1VC+1=dyHYy-d$B&YS2gD`bFIIc=CtLlA><3!KF%4|UT#?#;kO^( zoBV74!=SH6wtSMWd?Enq?&tvJ-NBLDW&lpLl4e@#Kfcr2>XHV`|Z-u0IV=gX-Y8iGJW|i;+MY1vhOxid_=ZOWF;f z@)GlmhxTBQ8Q#<@e4xM( z(bPl*K&A;=g8&*DcBJY7yb&^EL07#sUKF;f2CjIRrN-dMx_!>Y`Bz{VYyz;7W@(2K zgAo{I28b}s*=p^NZ_od(G&d^f2*Xk4D4Iq>W;CP4l$&~O0WgGfCoUac@pD80f}3m) z!C$puMdiQzV2PDD8DP;3EWaTDVlWFpM*_mccK|c^FU+~P`%^lmm0+>$Ebf2tcy(9u zAlT3ml7qg?k|$@{mb@r>AO+U-+JuhoKO!M^smk_(RcZTz9cgEyzi>@M&wGyy*UB9~ zeZ!jrFuO{?tAM;v7kaUL|Z|HC_8)6WbSOstHgr%a?J{agKm^QbQ=lu`p zzNEvc*bo6PXhT$>SI$LBS1ic}8Rd~9{|!(Q9@jyvX`W zb23~ozf~eTEp;KQKJ26sH2lDv*@9n)b~Cip%;$#goHeqEjbDA|_h#=cM6XFRi6WG- zb^Ur~OIzC+t4+k2h>t`tZ^Ka_jJnxq0$#9Jo@wpSD^M~@hUF7_CQBbth3`8JcQ37r zIAx{jMjSG&-Yjgydg{{E0${f$oiZDe*Y7P@(JO!j)HgI3R1f+5?N&|svErFlFFPQo zJI9zjKCS@k-?h>HpDEo&AfK{W7x7QL-p1=M&l_M^{7Gc9jP#?A2W5*7yUAZnmkfSG zBuLW3T#Kl$ zc!>T0XIyXVzh_ERFTS#LTftkVEI)3~z|Y+ne>Wgi{omDGO}J5wjioRn=8YNZ)+ zrGA$s+WzPu2w1T%WCxH>7XNGPAaVk?C@ScOm-pFn5ddsOX|h={f*|L_-sH)TYR(RAaw)W^Cn|-JLOdY2cj3p62gmDp(wHfrFXus#gxDPO z>pGTpwHRh?xJ@jdQS;sr_IXxv7GI0hMt4@(@rqMmqUOy4ivpmHAZ(W}=e2#8U+AvT z(l&i(nGwOxo|!wmw-;MxWaj931uGG26;^l1%})~Z+-EQ8Ugp?^KHwM~I?8c|{bQrW z6jbul`M$Ch?1wev>adt`?Q{k}=)A%urh_Nv1v*9pwx$n@IPfzF<|rbwm@DrwpC8MpS$l_lS%*8)%D~)+9HO%o9YiE@pDB?c2e}ZtBniQwByry!E=olFY&dF z`|6%(yf6Iyf#KZksn0Xpn}zK~#;9p8K`k12>|OjZ zEFIeLUDC&TX^tUY^Gc|2f-ycR_3^lC0$3MZ$wiv8ZHJZVIkz&JE^vT|(OJs$#WFDf zj8(dn<0KdJy3QDZi<9aH1S+ihI1#LzM*WY8M4Bq~E~ZZPG(a1^wzi~z7iS;19Nn7H z1)D{$%wqHE*+;geNE0=sTfrS&W8u4=8QlckzE`=!n;U;ciC*k4cT1p}l-LIARNoVtYA)HMaZyqF9mYXm+Hq1Zv#K`SKvjf&<88@$jiLZ1H1;37Dna=w? zq7^g+#hVLRtX8rzZ>-MothML%~&*S&S|sVZWP$C6~GIp)s`VJH11QPQsnll+_@!Rm-*E>e^BBnmB)*%<8h+h@Y)z;`~%*dLfBDqzNF7`oa;uN zk?=O{B`O< zU~le+*Jj#!R&}uKZ;cF}hxCC5AF^x;qTV8wq3QQ)l6y#qBroE#r-Pf8Qt>%j<D+W?z^7yQO?PRZyx3_{b;4qN{ILSGBtR~?*-fGK zSek=(B?d;l+;eR!{n_csS0)_?lkkXw6UZoT=}9HiJ3o!HS{p}0vDmql_DB=GEPQOn3M5N z+{AiWI7(AzVL=@TlpFUPjuZeCMAPFYw(!q zD?j%M(PHr2;;OLvDHbT}0ur`t4RbH3p5t2$&_oc7IY$Td<^=u$0i zNhW6^<6FXaJqII)_YWEDu6pJ&u38Q7$;+g`8(?aY)mR1j9Ocr{w@=zj-_gk6`%?Y% zZ68ujzA}UBpZ_c}QVq8HT00}@Vgx=ct=)@RQPa#(9GRCrwVkyE$#}PUg;}BZCe7?& zsau%OpF(Py%Rbk zD`hmH3dCHBwK}Q5v;)ixYTA4{E6s}=rB?SA>zT_77m#Vsb^>jgyz?+%^3yUV3BCTs z%sstgmU7BNp0OWTY?pKfZ;c~+c3TF^!k2OUn0~Xh>l;*?aG?4hF-bqWWbdvd%HL;E zj~I>~I=8;qGj!x<2eHR%JXb@G^=Dn5Eq*&@uXZ=HgClVR36|gIs$WvZUq3C3{IWd+ zfPZ3tJuDV0S00fS`IuEWKE;e4@!}Y?aRU_oG>!S^g9vN>Q}JS3xMjk;-LTB71I=@L zusWY#+E_*mlH%dqxBAt$vDKD7QJtL_Vn0aMafra5IRolBogoRizM>=eW%{~WjQ9_U zrPi!APC88sW3o${P5ZYqQbY9YTD3YWlNg-kkQB67>mgyCHS58SOeq0B1jS5YUS2vx z8CZ!Kbk8q+K}7sS+}T&DW1$b;qaN8#pgpJ2Gk5!W8qlL5oWtuCj%H0-0X}QReGvNA zwVjmyizQ(-5j?5OgHYV1jf&qCSr;{bMcNMCm11z@ddHd!Rd3agn<$uYfTD<9)%_ia zjzX6MsV*k9oWec@?_9kJEhD?|IE_%s)i~ zsyM;i^{F6M&dkzP_T>9cm3y>~8suu;Z0A1xq0w;t%kzfkQ*DO$3`WIoFL!oRv@ymf z&Zk?u%?C)8hq>}Q4BZ!J>FIJ|bzjrLXRdsho-G;#yqPFA z`-bLwi_bpHHPu~g>>f+(5010dcV9RSk?h8zdf(D2r?JOeraZ$87YP}VEf8TCntM+Yo+s|{9A9t zQo>*kOt~x^!o*gyap#qD#QH7F|Kqh@Uq0hT74=OqJEyC3TU!0>*Hv(SgFgA$9wo~c z3#=tIdi|HSwjKrJZhK=Y*c*!bW!#{cz`3cz>Z?ubTDJ{dtUMudosN`kal`tYuN})`-rf1@@Zx|n`A^q%_Vgoo%NYmB+Sua~ zuHNbcx*u<_n!tIez%T^^_#F@8#={-R*smNACJ)4UzdBNEEF{MA1EF3>N zli{}AqktSg=9@8y?y<+?o)ziWO&K>*tx7vEi58&DB~2#mAPV`~R@@mO*WWUE3%W zD@BVGhf=g?afbpe6nA%bhu{Q?7kBsK?(XjH?(Px@5Kj7h=bL%o=bS%#CNtT4-?{IV zrPo^5fE)Qr^uPO!KM1p+{09yf_Gxly38&M^si2Y=N=`bwdOL=`MY!7;8as(L1JyN z!yCEf#S3SyyeKAlE$;M4rr+7X^JR!LJ0Jsmfza(3gEA?(<~kg|R=xW%fX4#OF*A^w z-t5&fBk1bz;4X>dzAd`IpZF|+KDtN#2ukV9Dk!9GeV1y#4t(P zK6=X-DG`liAL%yXQAR(Wn{@QU$s^e6)eQ=!$%(rV!qSrhP zi5e(e3+~)n!Rz6I-s~ZQWc*&p9J^e;;}3peR+H$j)Fzf&9$Zd zYGS1K3g#KQ>6|@FLcQA`=gVw~wwpQB3yMJ314h%Ny#0U`Ucj#K(VXdAbDTLVV6*es z6H9hAom5wQhU&`J=POfo+Zh8Mv!tI}eRJ{%^eF~eoPFDX#-;%&96C$Wvw|0RBtXG8 zZy&o{L>Hse;S*tEI>znZJY*MvZnNuPB1W4z-UuVEjNBArKAfnv`nIcJs=2a`n)_1g zL@)L%mTIG?vsRTOfnfF&+vN?F_+JmUKk`*)H5N&Xy5$zz&eyv2TxFtM+a0AQRKnL8 zN%+^-d^l!fNQk+4LWg6f11e>T$GI+C(M;UPgvK^2_qr0)#ik58HQ9_@+d`+P!SmE> zo&IB5*?Q$XZFL>y^&L2P8Qq%Xa3jpY-|F8F@19ri0n>>1R?{MB*620TrEQL2Ro zw5r&w`zi&GyprW&T-MYqF_x@v?Ez>6JUA;N_@Fg5crMA)NXhjcq@u_~&SC>0sfVO6 zw{v>nyI$~2?InHzKYkdwExFahB1R zRqxiD^yuPLj>{E0bU65Q3`X8>``P<1uB@T78#nLeY`_C@?|S-AN00SlLm6Yr^WLkg z-2l+>Z>u(8Xy+nZlf1jlrb^AKH4g5`R`_VXW-XNMV}_(7VR47U9xm4%!V^_;9N79` z_N$1@0*Sa&!b-3sE5NAp@F}GF#}U@I^fHgU*N*MiYc?`d?;TvXutHS?nhu#}cJ;jn z(tu#Ffk1O$osvtW;X+)S+GO~K!5zv1mfD7Miyng8jg4O$oqaRI{$Wv3)zDTrebK@A zGkt6$?0I(KW9eo8otDX?;9Bdz{2|{$u>3C;_g>K7S1ywcm7~qqA4% zBW6`Id$qmLU6LaoEK3jto7`u%BLPLDkqVlczn}QcSKMe3_$@9= z3+je`>4KSVX5yygeEbJ5Q1V_I30$yBrV;D5>`HhhJzbn{0(MUPOTAvMs}tm{V`5@N zv@8FQK>$lXt3y_v(+OV)QOHty^$Ih#0x^_`J7SDRer=s9DY-ezv*%dL3;YqOU+++;4%8wRu8|ix_x# z-@zP}MPYc%_8r=YX4=S5H?mvq)HyUDk-OT02?YSY+S_^J=e;zM9Rt)ze}6(oKa(}{ z_+b|WnyW;$T&>S`{=%aug$mf0Bn=QgoHFi=bU-ti}+xB064PcYp9qYU}ca{iddi3{X$ zceV}tQokULGkm=$Rqb%0;TEcyJuTW{39_e&6P;>gjnayAZ?C_vcf~)=N;f``%|#ir3mLdK#sQ;G-mHyorN!Tn=%*0@$hYMzqq8 zr&mG}9j|+SGVI&b3I^1RoFLq5=Pa$i_;9d+D|e^mQpUH2Adgh%U4KsovWs-hZ?TsW z4%`kR7D^T?7si&(bH3WJ>U5uzRBUN%ES+%@E$#U|;u;Kpu^1Hg;AhFy0HTxMepuR$pb@ug*h|GEl?+6PqkMwnyLHHlRfy3w9sTWIBir)5bl%xat zoFcNw({KaWPol^Yg>!cVg3+>cgYELE*)05=jPH6AO{@jtqtanBj}kW*Eu@iafX|S1njw;)!ls}%Q~oBm1VcZPc)9Xa^Z~2!k3&A zpdaR5@Gr_)uR19Evb}_R@vTmEE{m=Lt7{CR61JSUn|)#>=-094|I7rlPv8Z9*c>)b zF2j;UL3a{HGz&S6x4AH9QkAPcVeQ~5X#_NM)Y1Zx$bfBd-hN1z9rF>kVFgz+FNNRQ z9!V`CbX!WhqFemue&^hI>eH?R@Qd&?4nLl!;BhI9@2D&bsS*?YtX!t4E zb)8I?4m;D-RBo*Xh-eGriCh*nwuNeGd+gRY`q%>qlVRh`WS%`wRw7}E9HjCU+x^weZ+-Kqd- zu9FEsjh|+B{TGw@T?K<)n!Q$NI~yF%Y{fEK=QwfXBcWXDx(B`-|nHcjNe zoiqaK!L6D&t>@IN|NLaW&?AHO#}_1QXx~6?E%8AuYiZg_vO3%p%eE@K>r-#qZ~I0)oAKIwe~wQcpUk#!Jsm7YElVic>0odN z$AF+=Tu`BZ-64vj>Fab&ndK9tFj-Fdfhg!NuJz+tXSgRrkOi=++N)o8rdHBzr&}!r zuf-3vyEEKOlSuPP%zynCxHGZ(_)*=EMxeLs55`l;#5E&Nnnvx$K)|N-VSh~KlGN?6 zta|rCUx_e=JM!t$xllrpP}@)CeT^E>IaRA0J{@7O;^=jT^r#a$2tE=gnWlZyt z5@$cg+nkXI%h)~$bCK;og5Ja8#F^E8k0`L=88+>td*}usQ;%DyqF4$Sv_JxjAAECj zW8PTw(8*uBXlUfz;dbuvEkK(>1P9}Gk0(mEt>@LjjwgxEc+^j$!Y7&zajS*cXd9H2VSxjuaKbwQ{$!iBQZRl`ZNxtqJYNMWO zU-~govz2f>YjNR^+J3P)Cu+lfAqQsOuvWg9>rV`vA-X^CMy6?w+Fx>+@Chs;I>vLaaK++R%@+w zS?<(9oe7ww!^mPT{k(bq>Qvm;ZfADC$|yO*8)&M>CqqU#fb8IMbN7^D4K6tekuTfSHqK%K4X8QA)YH2FH%M>F}O? z9G}KWxelinmHy&UH^%x2B$*eP)+GM!3*J?RLC#u8a@;MBh<1|pDj5D>i&1A8jqBBf zmesUp0NZH-9)WjZus{3r(o;KuP9OI`IQ}2tvH^fv=REi;E`H=+`<{~gwbp%d0~ZV$4`=4O`d%!7sh<;D^POX#OQ4)u5+PQ@_rY1+a-)IW zi4AG$?r)nII>FXe(>KARzfToa#jxVPjj0`4(S5k z;VBV7BSmHO4;AC9Xp8j#mV|AO&xMu%;@6atm-E8%b4*)YCr+)THsLi1p&)(`VJ#w; zuG&jW_I0e~RRi zwxFAg9aRTBCv#(F{l2D)GGx!(JL zh0ZC}ZU08w=diwJ48U(?Mn!KfTUvMS)I~C{YG?>{%dY-G@c`^mNiO1jfdJ!huP0MX> zvUret7fw|j;6V5v>Nya6mUlZAJ?@cfrOB!PYs$-zqUw{av6uKFYwCKY(a( zRH;vi2(9CER`>n|pJkykf;^u(sKK;ErHLKk6q?IgV0KsZ`CWe5A-q^2mGW|T%Kx@I z%U}teWOsZ7=n&)ywO~=|^7t*+*)d{w4Wo`m>C@sD3TMh@&>d`kxh|4A9XthVM8Kvl z-tUL)rMleaY-NlLKRvJ(>r_m<`8*Rp3}!sQ-K?Jf+?*Yk^ZXD?T`5*&9t-&tmrVlGk;1tKk0KTnt?bK6m|EPXpPiLNt%+r%4+)fEXT z7>0pPmBp0#miP7|^C{5TZ2OVt#pS`ml%b-P$rd39I!`l^Z2SlY+v57Gnooq4co`yq z(TmV(nm99!HB{r{eAZ#%)v?}&v3qRs$tJ?HyV(PKxYGff%NlAXnc98zS0@w{NJFxY zn!BVrd8*=UvuAY+cG0$PSeVa7(reWMftAlYvbHBufsM5BfTvrYX?vMxECFP_WB0oY z`$vQ4d0<#!DvviV^UL(ut&=I6#|d$6iQvypE9A?z_K)7~w=)@u&@)Ru^&^Jnkme2y z#Gh7UR5990dgB{ne_jH1U!SJi?oO2OGd-i&mBtrN`2k#9(8T&*fH!zA-6Lz>9>4Xq zPW$6N%~JCnT8oR;NM7VZon7A2`WmmRQuXuRla*782=}fbo#^XAg;@1-ZZRFRPPqpU zJ8r9$X_v>JkY9{Y#U;`H?>y!Gd}g&x<0lXDQo0&-5~JX%m5B3pyS$y?cuu}%qMt~ek*N5@Kat)d z$2SgFUqknfqICOfu|(EFS~k~5j~jM{h&v1i9UL5TtcJXC(s^9^JD>m}UKjM8v14rY zkxB)mSKO`rbJ0TIipCcK%QU`~<5f3~(aaF7q$D*y+gZ2k-Qlwa!Gur8US{9$n3WBg z7;R6LEZCs2-D$bjru_Ka$j-w4Vl@iYPa&s zlPu034h5*7O`s#%q?ooHKR^Gq&1MYt+qgOJwM$>865g5sd;E*y(n8^kY3iT#$c;X{ z#>7=V=N`w3@GnL_T3M2{%}p&+jFjm+xoqn6w(B1qIoOnW^L(yv=~;a>*OH+qBp;I2 zXjgue)>kk|_{_Q5cKR4kPk-2=Eql76(8J^tP35vXbWmEoqxkp5(UmuLQk*1}`DSCp z2>Px2MxlZs%P#3ac7< z(-`JY6$x$6M~AwuO^_`&Lg&zR5omqekc-;sT+sx2PZU%<=fG@=P~!r9D+dJ}JJYOh zw+gmngK^crQ)`1f38|X+EXjzVGu|d#JOChG%QgR60CnL3U_PjUc>F!61Cp!@DZTl^% zQ7^5}1_=^%j1g(N(4yTUr_&Q#44U~}37xR>Kx= z=qb<2ZG!sn`ko=zG*<}(f@if6bhW$41G3Ccm{X>_U8y#|2#KqFq@6N7^4$piC2}fr$|VkA7)T$noGW0 zPOp4#U9Mxh@B<9S3k5aHjT$cT^8;-ly6TVrzeUeGNTE&*O_P}nE<{8iTFjwNsCRl0 z@PPAd+=9mCHM%xbF0acMf0#mSF*Q}3T<@|$wXH+^nNl}2O60bJl|TRvLDv6wy&~Ft z<33}+yw#75AB$2Y;IfC5)%m!0Tj4@&+%3mV1v(ONt;B0R2D02J$>YZBwfNtAf}I4@ zW(zH!;{IH^yWaGAC2PPcM?dcqwdtSrzQ!0mj$dvbdKa*IOR$Q0YQ0kZ*9Jj9_Lc7^ z5@f?hAX%*a@@Feij;?nnc*B4C8a0+F0$z~HBHMg<7uR`TOqi{pF>?H-n@Z8^#h+3c zNpm}IcDlp|BpM}4{=SrI92;(PMe{!o_WwPN#%wK1UqofJ8Cq7{ciJ)C&`pwt8IVm@ zd~TOH-de2Q=wMU?3#4tX4h`+gvCJkTpee26 z%^;EY8)S6d|2Ko+e{P>SYyz1A5c<~yJpU$;D;Lp&dM=a!P1EbKQOA{gWduJ9L;%+7 zL%!q+^>ECowUgla(Nux#Jo4R~gt8@q{?#`=;wZfiKzdPb{H{je8s|0W#eG1ot@WPv z?nTd3(2XW=cyx%#c0T97U)*H51_x$)DkhMQJPVw_lWq={kXq>m*K19sRo*Wv>%XuN zwXZARW7R>O=>jcB-+_b%3R|P|o3AbcoUa7gmR}R-OL~6Il$dzoJ_%xY@W}LCLz%u* zo=_)cyaVs$o_m>_|6aOh@Qk`%D+SiR{^m0IJM!GW`Y9}==)9&FTlG87T@H=>}E0aFnXZDeq z3CSwUk!Y5NQe4x!Kd(m+e=U~HMLT?o5;AOp2naGAPwrJu2Y~$yp0!QOEA$6~EAzsh z;bzrLlu1PMsdfT^HxIrz2Qt7oIBG`!L%V}ld=i+2F;Fs;cNUuwH2xB>zbnO)4I=-N zi{yh3C4o);Nsq;?p?jZ^v#Yn4PhiwsiH$!4hz6{_N5o@3xW;SLuY%e=gyx2O`OMIZxumyy1lF2>VA;bOh4#)s5Ba6{QdhxXogmfW7bdg^)` zz+XIX7@zcsyK$BBDG1cNX{YE(zIX7CC19_p`l6K6yD$6SGJrXufdZ8ySW`f18K|WfPIfiuY0G^p1TWm0p0iK) zM1M{@pi<4Rjw8yxSex39`h0{2JYxT;*ZglCQbA@4Cm!2iPicWt`n#}dFK=0z3|DSo zKr#EB)4^uS^&H*ZzNDUEzouh~rhumW|MXAM#*sKc7SJLY0a9P)fsFv#Pk&pdyV z?b5dqpA_HRtL``_^h$}@GhxPdoe3k&GbG^oMx23%bJ$n=T03q`XtT8t+11^-+nv*G zuA=|O6A@sOCNbIQmqz0UXXI|b_x)Wx=ZAaG<#rRc6a>yL?DgouP=a;542ZC*%|E%Pm}@D zRM>5F|B%fbE~=_M^YJ8?cv1nS?g*^Sc@wgyCX3Td&C1DK7mE_FLF8f`Cl>h1N=D5K z1hD;0H$@VGDPirQHj$UFC6~%81vi&ccYpB_gAlSg4`WiA{Fp%|1{LsLy#)6|%3nlZ z6f|yz*J@K`Hunr$t@vwo-QH(ty=IL#T9Zhr(dZ+Gn6CA)+?^dnfwf6fJQr&2zcYQ3 z!N=Bh3l)Z-`RvZvX}kgHhsx{~7t~>*U_2rtP3E%gd0R56m<1x4cd>>_SJeSl;*-I8 zR1IdNS#O)HGg7G=3wQPMvp!wYrmuFNmrM0bXRB{YPWe6uIpk3wKh!IvJ4LT+amHj; zE@6MJaN=;!arGr+N~f~hEGTK)!uq~fYxj-mmkx~!&#@JRjA&+(cbw*vJ*g0*%4Y5{ z)^L+}!OS2>X4J;|ugsZ4+oX?9Dv*grzgtK2z?(d1E{8iJBk93aS=XX9>YzK+=-Azi z&l-V%c&$1zApE51YCfh9X`aR+ylSP^s zJ9L7Gc9g{x8$9>M4Fs6RiV0TrPhK35p?DWNX=d)+bT``P2aT+-W1OiGMO_|(| zmW>ii@u;JezVHt_cTfWscNt#anR5F&_s3>klX1-RkAqqsS6V*U821Ze2J&_ow!-e$ z$h=1?Go|>c>e}<_ebr=7*ejRswuUWcPJ?krqqQ7en^uy@lY%F4rw{zXOTc6Q`E=y| zQd5d0S@OE6kdIdn<8B`}dw%*HospELVu}Uo`BVNU9ozUVLW}P=@Rm{`qh<*LVRf9} z4*wIny{$*|Xby!18WmvgCyjwS&wIXqj)@WwJ+ljE*i08bSkY)}dYEyw=*==-fx8m* zi+Fe~#A2HYo0CCN*V~estX4{SjE?y|kF4$ii3|=dkb88PU+1#Grs-yGF! zTIWKKJ`QCg8%;N_&~_|sr4!rSsy2bnKdjRI4)vi->`F&t*h~&w7m9U=;ve>>3R|-8 zlf2yQo0-L$OyxpXAjBN+lZe80@gQGyYN)pEps zJDtB~;oNL2D8av)QcnwdF5dy^Y3^bIayt#wd+6ThgUGjMhFS2&)gT>lzGmErbz4e-O43X`4Xp+j091KLLBgkqXK2W_q zYWyBR54mwk%o)6%hJq*cb}xv2qt1+%u!^ zjtSPb{NmDt1jM6qW~2Ci2`Zpp=oT@@E7`hU`3}d5gCo%>3l0sK_`-yNsJ#flm)ALY zjYx7AY`p=p9(hgKYdL~F2kmLWJ zPR4MOivTBaFOHe%3tv+fy{X=ZL%FGsQ&aRClsKc2C?B%;D><) zPrrLGx9)oqmvi~2hg`EHEJYAyv=zHza&?5aIC+!YFaKvPnL!^G)H~j>jynD$Op$2n zthoSqN0jirS2LPnn)!eLt~CsHVI%zcK}pLO;XevYR+3J;%aF?qW4!c!!!dHULSdf0 z)uf`2ypA0bj*nL**PJf1iz54SS=AI%#OuQ#7gP8uJQ8YyQ^Z)Of4j_+aIeR(@|oAY z{zH?`mGv|(!^NBCP&BjaJKYu4%$d?6ntpMK$bAn;&MI=x_OLtBRh z#D%hrJ`Ab`@P5IJYYY7xkU!qX%fYCGtn!SVqCvS7uk!4}Q(dSRM%@?cVg(ESZUDMg zo<&UB$5G2UMmEWW+r|~<5Olm7?o6}3$cP=XCUI(L#xa0wy@-*_*_;%8%2g`6kK^YP*g647c>QIruShhBS+EZPGBqI__M z^=8bEife|0-wb!nX4B5--%qC_w>sKC8!4%HPA31{8EIyyeAeGo$aGP%JZ-Qt(fZC) z`3xVF%_u|dpobVZz|Y#bEb$Pi?}60lA9gsjS{okKK&fP1%Lx?9Z?LWZbSRMDRH}|n zt4))j=w9BI)pY%}|F0?t_Ul@22ldAGkZ8C+E zL8Y=?J{vA}1u9j4JY8Mn4un5Ir{%Fi; zt8>J7?pPk{85Y!_(YIRx)ds z-L7blgF!X+0eZC^8|e_&Pi^gRr&CX5t1Y2w&*=wUENr+nccGY#?8P-7$om+}me8A8 z7N7cp+ItWQ^}N<3xidTy{d9Vd{toxrllx|Wmiyxr$&q{@BeT_od{;$T4UrcF5VIGQ z9agov=iCSN#PO(Kv<7dXOZ-ipb3P-57a}{D(;zy?wUelX$lA#3A-?hw&mY_DX|%oX z$RkP2lC-=~;Qaf4u>hOj>$D`lse!TVTWNS)AM@u$QMs zv-|*z(^&izo4-r2WM&s6{~>sdj@|m0$27eET=_Qq0N1Skr{))P$%@=QnOK%V@B6xa zOpp@K(&%tGpHh(8XFrJkFRqa^UaqhY6oI(eAIzJ&4_-)_Zodp2dza*FO~9}>785+J zIf}F>%e^+ckRZ=1)s?jTV5<@OYs3BPxGKcM4vBZ8Ky3GJWD654x_1pn3#$>pTj)e`!f6rvUD8%zQ4Lv zSW@TM@lIK=!+fpiRYo3R%IxmDGL(M;oQEb)XEjnyER$RH5PU^NTSbyMTUate-UztE z=ka55cd&jG7Ps!?rP+2)OMUpWC?fOR!Gdb-nq1B&JKeh4wNhD^F8Bj%!Q?%uBWym# zgA+fxn?41r^)Tc0$L|46wtxBXzMk@+gw~}+1`T0igeSiypQJHEB_tzZVwBZWg6X(8 zDKhsOL-^2=a$LQ_i{#7<*RTTZ!%g?}W%gahQTif~-<*F*YHf+IkDj(r{%gaW&&IcZ zUWO<3{ef!#U#vMoSN4pckIyseyiY33qL__|KibPf)y2+1i_C2Cw~gmpET{=a$D~Jc z014#O){7h?6H*DYk^(`slF~VI^-WcGq$vfI6WGa{8H(;YR{^Pw9PWkzD*dXZD(=x< znAC4y2Jd>(>wZf|se|EFfx0o5b)-f}=Bn;=w!|Mmp^6)&w1crR0kGfa(bolfLr!KR zckF6u2a+;*nYp=w3OADdWl{QhB+fSK{O;bRIg-@oq%J|fBwKEYLc4!o(xzrdYroIK z!0EbT)h@4cprkF2PB`d3wCqD$?~>!3`XnKE^Ryj%e2v$J@zbadg; zVUc}J*$qD#;|;!GGrU37{7EVOSF`4OB=X0A>DI#3vkcEtGV^#PdsedhA%nF=p0K?y zKrkv!cQ8<)2&emR#Nic&1OkB>j-ZjJ;{iViL zeDTIw?tH{zrG-ZRh1e=pzUN>^sMVh<<>zCa%sDMh^IFrm6p7U`SqRL$5Nn2<+t}Os z9jY^wLJ1CJ!QhOVw&94e%t=t|PP6BMR2yf_NnX#TYLyu9)0|ih#z34aZ)iMOnRAO7 zJOxevjeW6SJz~Ds%$R2jhbETOI>Vo9^-#s-om?C~IWQmd=!p+7$=uM8f_A+%Jl=~a z+n(gaQesS{r2$W&=#3Uw-Zae~sdmpD!pK25NKZBL)ZxNn!BHri?|5a1oH86td-Xo& z4X!#o{e|zt<{J&|JiTRsldZBT?V+8^G&xqs#bDj><+t~~wZ^uZ(xx#o)q;-4mloQa zV!MHItuS6b8v_d2Ut}zF3h=c(MMeTWfx+4z8Zp2qo^a!4NHJHp2=# zJ;S$6wz~(vojaE@q2%Co4%G4j5?*Ga)wg=z`+Qi`m-AhI?qOd8mv@QsH0}+G5Uq=tM5LzdoO>e0gmJ>am5M?V1M(n`^lxxAi(Eo2%YoJLzFLbaadNqI?|@5alwCq z$eu@R@0@W$I;Y{jKIgMDkdGPXTz&>uvVBC_V1=_%t9p3;JL=)9wUqpklaf0RbFiR9 z>pPeE0UOnX&bS6!Y@5BY0`Rq^&ti~^7x3^rXuX_n721XD4lugg-Mt!>FjT7{jd~tr zZR)i^r!?VmLsoq6Kp*ojiR*x3b?KORMnc5>D33b3f$vXsiFr)pM5~&h-<{=yAwzX{ z=)&;kNf-!^N%+c*O&wKWe$Movl+V4slkmq-zmbVzQ(%TzHR_L~GITL2B&lUT!%@DbUE8Drv!q4qZ z_Yeip=fOBxJ>geyyziSG58C(Q0feFP^DE2?!UM$fU{{%w3Otpc0%xBmuvoE~T$Yw1 zf`6^(*2?iLdvZ++I+K3vhAp~lUaB@&dSzwKN0_~ZAcv>kFg&<~EoUgbe#&`!l}>-) zb9PTtZ=^bI@lhJ=&);rgS4%2Q`(Rhjyni@h*tjlTi&B)a*<0Gk+bK6XX;_`!`?BeV z6<4}(LMI(^xKYL8+?hwxpJ8-#=o38|cUz@rt@h>ymtiA%y&+_0!>gz6xKMr6@Amuo z2iAR;bb&I{!EKwGhhn7yhOj;vzU!xR=L1H+ z-($vigSQrX?&w-JgDEt^AKRzW!zmgMwqr^NOd`Z#{0sPdr&ep1IskCXRK9Y0xnjWz zkK-rGEh=k^L}^*LIv*Vx?yb7i=*z8BTJq`*d9h)ro)%j;o1pTR)3s3)9pJ>e zJnNr}#;lWk>dSY(uI14DXNQ!aq>8Z;iTR*5dV$y0xF^=H?;irUwDa`OKuFXh*N3V$P>a?PuxdO zC0TujLIJ0R9d{ugSG?J8>+P~q!w~Cwp6z)A-{Pw5Fn8<&#Um^C7YvvLIYIy^1e3iK z{1UI`OvQtfImevg4siu+6Ls-*krSsB^+NvsU{2h_s`&l5w5qrhyF)xA8X%GwB}2!{nT8 ztYHCDem^2pp&(7`vNdNYKD>6%WFW?i9fQxEvL94%Xb1dR$R~Mp!7xeJuM?J^U;`W2 z<`Ybl0(NV-xH8+Z+t|665OajYiCr9juluymTl6|u@ZhwzAk#86(&jp}X|bl0v=Q#u zHSGcxcWG0jt4w%k6>e&3$+dCi+c(LF_Su{r@6S5ta^@=hk@$~>;P#=dGaJ24Z$Ror|PqSuaaSauQ-hWB@+tZE)dnHKpawa3r%js7uTrWr) za~`JbA?Z&CR(xR#PO~$wkYedH7XL%oTe+`&?E*>rn<*>cu>ee)U;oFNGfWZ9grQe^ z?vn%HvcMbQTLKOa-LN%)7K;(j!V9x+_KJo#TWG3;^Svon?_lG>itV5 z6!tVIO6gN9#^w2LollEL(^_R3oM|enUErAHmN{P85Y2|wvf+k01;!2+Jo}!Qh9?A zy;>&oKq!DxF^MxUaI!^#_vMec{@z-gAH?uKlavS0MDWa1#7kDbc2dnj4@ z9zjov(LNy}Qi9FHY1Q?e!NoZSkH!SmO&g*#dUCX1k+|emkm`DMjr$bWL6tY1!K52c ze-Gw>mL?O+1K>`ZQi&g(o1k27=C_#}>eSCINX0z$r4yP9nSSu}2^{cKb~nx!jLUr^ zK;n{f>-n|iFj7C*I^YY5hmg-P@w-j8;0rxS5+=>+yCmkV>{_esjfkl!n0M`mS1XmL zUXVM<-vDO)rw1?R&dgzd;lic63sXm;axw=eN9YW<{ey*tLI)M|7t#0EoUzW7VLPnZ z4E}e!*hkpJzg4wl2O}*%$nER21jB0OM~o;2dO2!ggPg69%KCQ+#@|fh+4Y@xF z2kXBh>7PL9aCTKqL?}197DrzCP4)%gEI#wr;k(Cc*Fvw0q{7qyV2o;(@ixn;1y9Nd zQbeXn>f8?;Gxlh-_4GPe7w^;RepLuD(eZbY-Ayo*E>X@q#|ny~LK0P2=emJCn>fYg zI6SQylONeUG+%_yUPvOLR?3Q6Mn52%dE8F=_yZoviSri^5O0-avGM#9MgM;Gb&(#k zatF%1Giuo9Z=Zd;+_S#rJlxvrdY!A4=~ctEa90f0ZKf@{eVdSha*G7t9-Hr_sLSj6P`F zoOQVc3hjZ5uOsTSkm(a7suXfOo0P24PRp+|^Ml{HBdbM956o?Sy3r5R(M$sIW<9tN zx^jGdniz#m5bADE*QU9o{KF_aa$4|UHL=xl8W0vnZN(}l)LDGzmF9(0QH4;(^B~SS2NJ#F$?|)X)%)@JdEA84@nyiLr0(?Y8}YZ`eH(95 zD8b*zy(S7a)S*Db(d0+=x%xTaB5gVw1plG!8S6=?BYnjod54u5_2dibeX2wC`!2Xo%@|x&A8} zEBEeBzt!rhQ%+6nOl?7$^#f{`PvpBoP}PjLrtEQGFAgeEU)Pj>pULX`aD1^O6!*Wc zUcbjo#h_qBsXtC@+eK3ij>Bf+zQEX4*SUnZ` zUQ7I({IgcD;3Wzl*f0?IYaUkcWFP zAkIVbtzW3|vzrLAvg=z3Psa~dMkrQn?#cWgHo%8 zd>a!R+ky(!#01xr`9xc!cD}{-mIRkm=C4XpdE+v%jI)wdO0;O#T`)bLgL&Q{TU+Ga zqP$66&&zp(q6ev`U>~L+=%vC+a1k{65GY6Dy!#N6C2Fq58kIL`$W~>nl1IT(!9gY!g&c&LdAGz!ne>lxa&R+QX7AeK*zXF(w<}_SXFBM#zeh_u3!!5^L@Q{n@0nm^bZXo<3JNlU;Cv|7CV^{Bz z1L(DgK?QvDB05y%{3N1!@Y9(|dm^)F!}uXGpU#ec!W6?CV>M}QbY1EJn0SpRx0Se& zmJHf)9KmMVO*Fkpeqfh&-rr#L62v$M+@YHDM3sf+EK)7b()W&AU6$pnnH6e@aeW!- z{QGxB_?&As6rVj>*L_vv@~n?1<9 zfK8!j;qWBU#$8E(7^O95Mzn-L#ji!BGT8y>z#xve-A3n0G*6%y*(IKoS6>?v zlCEALo572a4GANvTnwWi&$BS_Qs%C>d^1wF|MKNr=A?4M2B5JG(#xs>K*`6<{$}-N z>a2bnd?)}eyl3@vp<1bpLnCYN=>g7T(2!1_6N3dMA)-p1uVI)>scwjB3PgRi# z7&_QO>I5eSXrjk5hKN0d@Nfir*D%nNPkGscGDWFbPiok|_^t>%c~##qI^HxWSvAa_ zraFW_>h!XT!x9O)$6*LEcg;Q#UZnGI{ji42#u2mDY0$#q`$RteX z5Z9wCaK`dwbVGLo)j|mTg1qKE`xHE8zx~kA67D`>Pa%^H2&{SWT+5CTKd+Vq=o~>IH2_eBPxH|-QC%8)>5Zv9}Ex1c?CxHOL z-Q9iT8+Uhid7blp_dEHWd+!_L{qbrH#^~Ln4TUlz%=g2G>yp1y#+=6x}(|?HvRj>P5 zKb3q6NJ=8uHnm@qD;~x4SmuhK*d^jAX%1L2B_j5;JhLTt-a`>_6NUA8IB(9Wf0l8Q zl_Fca{k%=b$%D}$thA%nrv->)vZRMBi|4B=&(#XE>)o#kA2xdqm&RPPO)oo#Y7;nR zTc?)nC%sw9szx@f<@|*D@Ka&rRT`b68}i+wROB&P(ouMa?}&VDlo*%>ve(;m+Zf^xgC6 zh+?MKx<*hE=4H0Bkbx2w?U0MI`ezd-YKc)alkF>3kBvF)K#(sjxyb0%*UvZ-N18~h zx?&uTPqRb?FqIMm>3I0c*%&jic(>t@x`^9?UW4s|YLYhk#>0m~m%9iW1IWN`7|%XX zWDF;8ACN38@iDyWVgWnh+WWoMy+b>?y$Df<@Sy9-{|q{ouIU%*#5%3K3s6b8n`&@S za_m3*#(mwXip5XT4~2uEe;4@Zx7{^L=h?OKi;1$-_F_P8P@&s zN#1P1G&@&|$#Yyq)$+a<%cIaJT&N50y<9aANriR1FkW&1VNaI8zS@N2iTlXdnDA>f z%<}K43FaG?|H=s80YTTYy>b(*Yvd5u1%bd2K>B=I^)z>+aw_&FCWNM^Cw0G-hH)`y zQN`4W3zkFL@^1`I>5%UYA-SZ7CQ;nRxw(5U|E_d~*c)I;){c*Jg@ z$?4_{%BoUydzJr{NjYAcV11^=hzWd{l=hrXN(}grgD1lzbH0^R(B_$=5ZSEOiOBPI zj3@vzBvi5@V|G7Mx8?<@B;m9e)c7Tbgwz3!`R6B*1%VG{p*3=}<0$6Mz}ZQT-=9SC zj+MRzOD^U?MeP8eL>JQ0)pJlCNTJYB|5@Vx^|k0f$`-lux8Tr!mI0}JD8zr>iW(O$ z3xiAgAB8Fvj!mBVeVCNoDppe+mdX?RN%hxDM#!dUtecKjBMX-V1)HE65T`!YL{7|`Bj#@9e#57l(`fP1s z++Pn&)srLQu*9&<=YIZ?C;T3tRW-VL9!KIns;_Hn7SM4l#Hla8 z&P*5eCt;h81T8)|XqJOy+(KMPCgO)fp*E**rxXsMdj5^95=n$x9dqs z9X8C~L@7&d?%^zZUj1yT3H%j-RpE3FH6$aGmPEkS)i+UX0BXsce?4_LP7> zxOLc(A0#C!Tc^KSWp?bPOi~ynZ?b2vj%wRnrF1=Z$k$C2x0uCu0KZ23POZ!4L`@!? zrhwTR8zk-Z7+|T6iX=e>-bUtZMNMbWeXu=}$rYL~usQbu6_uM(Uw{1#l+RKbPp$|z zrHw27w@-ddG7S?yLNltmy1MLVRPYG;m>p+k^xpHb>0`u+V@7<49}FxmbG}~oX-AG( z45E`udUFWZ#|DN~F4+E=@>KfL1XV7Nycpr>kof7+zZyX8vn zRL@P*Ua4R8Go((rYpj|?tFu}8By50osp}qWO4yza7&}4DC~!W@{mBOnEuQ-0a`)^g zUX0D=O+E5y#!9=1#g>;5XxTPrI9sa-lFbnj@g62kDS{W`>QiZ| z)38dusyqlRzs-+HUR8;CGG)IME+Bt>FyZwsYf8u?7*@OZwHXUh5~$o0&K;l1acF&U z!C7@MYO3A<*_7ePAyqVvg34{)Rk9u^;gnNV1&oB~i0YpSX9Lu9n9i7_AqOZQ0*cFm zo6@|+LmWo<2(e}!Y4^w1nR2VbY8-$3Fd^j;-7GQt>KVw4;QTbek6pD|A+aW2#>ENMgGBBx<>NRPBbokl-JzLfkiWCv2*6RG23D{t>V z)55}+9Q%Z~$av0kq)~0pZ<#!f0?{YMfhhrKMoD$^_wL1xDNX}sIL5yytma$RQ|fS* z`QzINGD|cn{H{Z^s2akPFRyyeTVg#s{1qi}?ue@kjJ^w6^2N|uP>(^J@yuVXv%rzE z;&|vQP3>XLj-?{BG6nm87@n)S9e<~VQR)^}p!@o@djMDU2uo}D!?L!OMs*w-_{S+0 zY8*NR4^Onku8xJ~I{J;GpuhNaoAgiXbMX*T2kG}Qp`^eAUQB8OZMAfhblinYD3uq^ zCW?e+9nrsC)Tyox`0)PTO_$G&U?C?bz8@rMAf3F=DeM`Adt5h3Cu}?kn|1YG$Rs2t zKNHd;@Ow-mV4=PwoiYRGH<0gW7?;Ir=;fzS!tsfxtw@G26~yid`E=?h)`*SUxb6*q(Q7uVERIT`J4R2QaVh_~%x-y~!H47|;9acag z0$;VIlO-WOYCj<6D$KnV7@8No)P>a8XT>8%_xxX40DEpcIQRi?Zv-O+!#-ZY)wa;k zCkIMm4u}as8X4YM+5FD8W6vyFcY8&tsF*f1tk$qH5QiW-bX{0v zw5M;u9z3hKW7di+9vV9ETLuPK!rc7EKOme9iEeB_Gt+TtQBG2p=cpYL@C2cPTeVe_ z{MS<%T+-1NkIpBCG~X%=+RwZ)vKdSz>eExWDIyX?;0r9n2Rwc0GaTukl}k}lEG#mwas};^4cd$KLCppamk|U72B%0#9!aXD?DQqxe59pKYPd6^9VSRf zJ^^5e?uhzl=~P23=E^Y9MPQo%Cv(!{_+A4F27y#NOIVi^>YihZXXB&h2;ZL5OVsSS zxHCIBSJLCJAxIn>=)6kW+G2xc*^x-QSc~-_sll;hppuH-h4+Xvas=kt!lNbzhUj78 z;l`jb>(VB>riqixMZ1|96rdi-4~jCpn1rZe!Uhs8I>^l~FJ?|$q@D z(+aLoPa7Kuh1qpRfG3&s%#!l}Rg-wWR$URc;x4%7n+HY%))`yRQ$UkM_It4zwtzdn z+JjvGdBHsp^`jO>n>$s#zy-HdH{gPPq>e1zp!>QT6R4~}uf#W}C}dV+^Q_S&1)4aB zi4(XZ?<#FM=xej3##vfjPi5a_9L-g=_ZD?&sbcI@ZDhK8@f`E-AdQf2)@rHNj?v(X zExd^j&a%0dee({H-LO*aJ@V-%HMch^iQ5S$^WOy5a@hsHEe;%)8TQ}MlT*5q2%NKN zIkW%qC2m@GUIH5D0n()OEU6s^NMs|p@@}FSdR>*4yz-{p*6VX`3#nJ5z}hmxN?4?% zxxY)7Lz>Gl%fV5CIk}n(WxxjXbUREX>v4bP**cT=JD zuH-)hqM8;g_)d6QbYdQC%&TQ7-yt%in;`ZBGn<+k*N1ISYR99zQv5{#H648Z*2k=o z*6LR^`7rU($K1Z}$DeuS)Sd$Zz)>6f{ilLv2MGW?nEK-xE$WNbbhnNh!q&ms600f- zl@{O^|M!+)I>Lv?<3L7y&@;Cm7>3>EtchBSFX~Zwm(_L2>P3SqjY<{cnDNA}#JINz zt*m8B-(xSL{PWy=r@U>=+-kg*-NvA)C)K~152}Ek>$Nu$uoA%1bC3>#gF>ZX$ zHKm4{k?0~IynI$t4;)9``xJ-MI50}wjnJwUeh9jDO5Dn(6&l*1t!NA|xyFNE-yUccLlRY}ce5{~T3QOpIRsF<>|NXMx z6C3;RJabMDT=e(?zk)6!yo?L))Seusw=(%=>q5pxpQg zpL}%j1nVGWkBVtcv#VeFqUQ}L*lOA&Mf&S&DC8O&K{5TCJZrsmo?f1-1r7xV zDeEgH_s)<&R#eANY%(nNk zxR)rGrxo2RP_w>l0W^-u%F_=8Uop*RmdZ48MPku=r*#+z<>bJFI_0Lm`r(r4#ul#q zt`N9cYG~%03-gtTYnSe4_tUO2yyHkQw&mcUueyAanqZq~YcsyKXCES)deEMpsC=7? zvsXRRM62yKi1U`}H)-}#;>UR%jVOvbJo8;0;n@bKpSoU=$n~?>1UsOVO^-|6WZh5e z3YPP2Ad%xNs5(4?1OO|_X7V`0eG*A*|4(}Fcp`GX97<8ODdfMRNn!Q_c{}T|$t?^WQ+WIU#03p^{Uz7R1z|Bdf;7e7MgmEmpIb8d4eI%By zR?iJ|Hgn{j^o~lh-b%t~(dc!qSxP+P)(|S$om^AqJAm6*r8Bm_aj4@ZdEi8ExP(L2 zEeXluZ$={ke_NoJQExHhUc80;!JPLDCTe`RxZJm@Ry-Wojkw)CUw_DSBb#iv7z@ig zzWbg=_`F~DNT21tZOEq$%e-|Uvu7_(MaqW+*X9<}OH}f6$pxo=i8Ywb(z~#ih_7>~ zNAu2m3WExB!&6sec6@k06+g^1JHYyI)zJQiR$43W3ZFf*WRCIvCuydR>0G6l$6VzD zJ*LkSq6%%`2j(U7%&|MM!_yAw`?iOWNjCpJ8k3igAT~K`765?Wd z!}jLVujX($yecl%fR`dK9B;j}yFdEYWr8XefsKJoIe;xBl2|ltA55dx9xOkmJc8KQ zLPEDLxy!pRTW+o{6C`d-dIIec_LD&oL?`6;?S0G-7ffvfK9To=!n>={dUFHz(vnXH{ZCwA3!%Z&ce--$Kps z&bKX!I4Mz)mSpbP+i(`mn#Ny>L2gGwag=yPEGhBsW-dmB26I@{`%0hm9e?4r>_`Q{ zf&@pNZo!ILJ2`iY#$IiWP@2-5R`ddBL{ht#45Yt-73oiAh`k?>!odt=rrI!rrr9z| zZ`>eGvls`p>(Ln%59g%qtzkEgD?Ju;@}ee~z2TIF{y`Mg;Q4?+I^kTN_^z+_!awnJ z?Yu^0s#m*kZ2L`Q$PPQfgFJRHtMVIoaBOxeYfkJoQ&IiMm@?R+J!nLDQc4FHg!{gB zp0$ZCq{0!MyFco>gi6cp!-DRnU$pGiDfcrC(%Ou!Z3Mamf;1|6T{<*1WZktlBiIwT zY0ooeaMC|mKq@%H)JFYy_#LCNx)ol-%~K-wx7GF7)h1GKiLmS~&|tc79q2A5jkaJp zg!KqN&8uroc)vdKf3S47oUOJhyt}Xa39`KlIo%zGI0Gv_tt*c_=RlnP7%nltUKG?@ z4!yE$@Y$l6_Ibf%Y`dW7-Nu}hkTd;q6n8_5P%WlA6fVgxXa7%XEco)|K5aBw(&MH* zl1;Na;?bqbdy3Pmejl$c(f#ZJf^RyB_e|s1DLQJY9nuov9QI`i-xX$iM(@y!inQij z3?B8p3n`MOls-w5GUv}*@9v_+;<=N;PCSzW%VaWDQ@`@Gq6-VUu*@)9?74t(Ni_M0 z&Z=^3)8jZ!5>o3BIJUoALbS)N7G{e?rNnq1P1bRsnRiHKdo#yD;(vOc|$yZ9ivA z>X#eKAw$NEq2Vzqnpt_UuBpA)DH4Y7Gkb3t`t2cloN~7K)Zaq}HBMQ0Cf~i~PV2o$ z*(j-F4+e{bU%dgQXeZiehGZ?YT<34v7x-ymd875|x+YWAVauiu*up}xv{gh9Gd>s<5M zwpIrfOT?@t33{eH9&lPww`M@PzGT+KWNqrgmS{FRqn3}}#f9z-_ov7YNhB)+bdD8o6nb#LV1D}+%(?O$v)o0>?w?e6{9BCYb3P@#X7&u_j zB~|3qnZq$pTt^}RR18e5wx63P71|L2vnHK{&*ggzi#j`xdRGGO2f)F*HGgXu&d0v8qz;8jck;(^_%smfs3_KoM7F;jyG8FF&+s{vbq zZ$SMM%oHCi-9!}^VfHe?osG(%|1pUNlDBPqw;axY*i zI24qrb@{i03bH2xkUrOuwV%I<2$Spa(SS3FVBfQLmEn&FMWPqnbY$tFAdgR3!inH+ za2E=t37i5Q0fEWcrT-gGiH+k-CfLRpw)D|Fr$c>7qSTb7pt~L;!wY|#WyJa2E%{~K z3;(s7xb^k7J22W~J($UUG~^BSUO;@cpW=|u?B2P*f9)w^;Kz>4Wpz+Y`+YR9=z_OkaDcz_FyNMoM@eN#ZygjHQJvW7FgH($k)1Ui zi8OjlC4?rII;sMc$dU3vEWz7CgCvpE48d}5f&ch1Wo2_i;Pta$h2aQM>NK(IRFP^(CQCT%hQ0R3Xm;I>o(6w8H z0b*@u=NUKttThgNh~#2`xI0*@gLiASv9K&YcMoU*y?lBW0MEXu*X}s9SN^ITwYrOF z(frMi2nck_n$}LM`!|16PycT#8@}ce9z1=J-?mI4m z2m0DZfuMHg(f>gd!Eg2SS6SWLm_E^D%A5jp|(*4cuo72k$of4dD++jmsmFJ>*r?xTD5 zD4BZ!Ro?8A4ejHrxQNJhTn*{|t$8fArQ`H=qo|mflF@8#*Ci3F!SXlr8qegZ<(#R= zg+EXCg@04p+HD*qKqJXy0*#FKRgPW9HnyDX{q$+Ikh_q!HDVjFF7G2W@R+sJGdOWr z)2n36J?Ty3D^MoW8e6r(f=0`Kp52 zBPlBCPe#jgf>nCjBU{=ma0+MRUIMU~O8z<-ke@nYHxN2v(?b3ipWb{Fb6<;>%PPQZE-e*p!cm>u(Q^nc9J ze~``pB|r2JM+AJ3X8{se|4}lGfPB_}RLDK}C+YRi;_&@Xe(Rs*PXbFEFDW)fRM_DA zg#LfjT;pky3muDYYd1^%pu?B*n=2Q18E^~8K`pJdV{PD{HHj`zeK4y*1(FzlSkTUH zamYW}93XcA!PdHRy8#)DVW9>h06k%qQ}`|rxw&b(=C@z7cndVe;bp5!s3?;C#}hCI z=X04u17TSQ5N}KZ6#y?$_p>()u440q1UTu0#H0$~2wA=@5%`C&`K&NlP1=V54pySo zyZD(;T-H0)Rbaj+xQn#TO0EBbQ^#ad#au0heXa$YTerAr0sQi_C#IQch`}_OSdQK$=L|hm??0R|0@rvC8Tlwa86OJC%6U39eo!o zLafN_pWYU;qItT1Z3G#`IgIeS(3Q-w_FrBckaEy9h6vOJB(vkO`pt{?#lpI|yLVsA z45#wiqtL`rvFiNwym7?L5>uOL<}EEoh_Ck^WQXl9;jTojNvueB4CHk8^-CAi!m2)P zao4)GM$ZN1_NJ?5n+BMu+|sy0xrs88vIbXk@3%_F^npNv1nPc|lU z_hE;G8uH8cuB?x?xq9>QOYW~<$6XQP`#>AarwU6CBRpBy30>&0b(7CLCZJXVaDndU z4W0*W9!e8DZ*kB@WxA4`>hserX6C%Z$a?Zh_mMv!3!STX1xH zMAoWczxBs(pU*!6*qu=}^Ot&Oyg`(`2YF1kkRZWxe3?2flIif7`}fGJDbSqHM6Zr7 zLY8QY1|#Wq|H@XWU+uJ0J3uaGQCx4xG;U{9OI6vF_{#m&zuR+c;vNmZq z4812Jt9rjKgTSmukpwL+x1%G^RYRfFVxrOMz=+)j@|JhIA3;^X;NL)xPjOl`yq?Ou z!W+NJ>if)rl<{^uwZDKs8#+-8gNxXCFB>N=waS@zeSuV8 z-Vlva9r5!6^U=O|&V!J@W>HC$Knj!bR!FTS3PyNytIsNelSG7y$>KkgOvdAqzG$9sN}~SRJMiCmKknf zXp@jIhvD_h>j48Dfsg~PkZ~LckN=GK@AW*pIpF9*6Ea|F~mf=_-^gUl4xKmq!m*?=dHHAn#jInTuoysRQwfND+ez!#T(1%^p)JrFCR(7s~0vfhvb>neODcX0ii>TA0=B2a4=0}KpIaD)B ze2w8)S0=6~|8h~2P3pNnD<#CIHj#5He5w+@?tB7$%Wb%vkg$6-#(!P1pkSUAzrqQ? zS$-AVqD$%S4{O(-J1sa*Ssk=i@Rr>Ni7x|bR%iS#IdR-(@7#;wqQb?ZMF3I{@C?#G zMibx_h;5ybKzssG04Y1^c|{5|D$u-gz1aKwV#)#NO!0x3#&vWG8D|Oxxuq-jBN&o zf#Xsh-<6ls{ycdiE1xVp=^z4TUx!<(e5Ch=z0dWPX2YGo5N4+#wcVd-YXTcT>ymfVvYRx6?4s)aEw05u&r!dpQ$R9r@F* zf|RB)mFp3qfKFp9Yx+tAd`0*+n6&?oiwEe@As|(rmd}in!5$8z&;d98$J+va7hw#y z7jVcd`)&Py66#YitG-TTX80Yo^?of!+_Epv?pL^6SrncigS#`xyuca8`rxbq{_5|{ zPlPrVyB8b7!+7~uT-3N)iHRtL0rXa{W;ptF*j5dNMOJDgCww+p{Mr2s<;D|Ri*5#D z6C`AEYEs9Kyh^t_XA?aAj4@4@#Q6d~oXL09_oox>n_wJs1P?%Q?<}EHKZt4={3qGe z!W|OQAGK9(`yq~pDI&5#7az!p{=4XF)vA1H)cSHv*E`RI7Tnw=WG{z35ynyCL#6Vl zrMRglv8jl&LM!c>yb&%-&OEvB)F&n?54rOq;tOgnZ`E_czkp|x*ZniU{Q)Oy`nh|a z=MWbH#o1$d_C@i1s2tJ1n8+TMm}OfjGRt>I^wuGBxf)!ojh2zHl6!hlFg4!%Q2pMN zA7?cBYY1j`kNhZ(f!391H-RLM=IYp2;HFH~q-?-o2Qng`VYTN&b=94W4OJ9L9iu@Fs+40bH+deH%f8&jPcqD!#|U0#0=M~ehEJJ}1GaZFO@9(NX%zt0 z{-2`3?7EL^wE`W$Lty|PK+74}yReVHpI%p*%8L$ma*N#3%q5{r+(7E2i9ew|wbWS*F!M`}IZP3{4)vK>V?( z|I#Em1=0sBeuipqiVn%j?9My%kq|fK{k@C4vC~XsykuL7QS$u#y9UI^39OZ#hPc$c zd=nP0rQ<7o=;GNA6GvZk1?(EN0~g~fs1|P(V4)P1I}SPXCivNZ?Z4{A_=I(7H}Wy> z(TTTJ#7$nGRvCDriM)DXL9;4J9{b{<+sfvWn?kzx#XfAWrw#R~B>j5h`q{d7HMck$3y_mS2Y z-=I)gO-z8Od@hpxlHuhdWYs}@p*OPSnbIR3)L-n|5f!n)ep=OJ19zJFzq9~HvM+Yv zV+5W<*|Ek8mlp?P(%#08_xU4x1SqKHsq;oGI0OMt!tUAI_-q#9o70p=r@jip27o*R zS5YvnZz2+4sdUEBZ~6QlZ$2ZmI5N17)CffdbUoG5rQN`;s8vN54alBx(~J!EdUW$8 zDb#N))JBUbjOa~ZVr48Y^+gP5RwmiA5>Y(1%A|5!aA0B*iWuiQ5`b1&5f_hAN3x(wsScf7o!}aa{*8JQ{#i+lH_-V#y@?G3K7aEy|CU2ef+fp!HU`!fl5xoaK0O-a@+ZyGetJh9Gcxll z67Lbc-m?U23d^nCRL8&P3xG7BRPlQ33*aN<40z0M=_A%k@Oi~yRPau4Z7Y8z?Pskk&I#a9{67*nI932i4Ls5nn3G4LQ!>Qw+qTMP$JlK3nacc5S5gDC0Css>08@kj`T$^F`z70STKPPT$;fpcg!7lH z{&VQ?Wu)16!Nv)}0V**uNwKcUCoMj(CodoTKaNBKHWm=GbQh{^y@A)z z?_1F#By>`>Tlf!+rFoEWB+{qJ>yhA_9gr$)*<#}xFYei#{%9UU(_uon$;+6D5bwY{iQC}|JI zk~ff526!cdIyL_V20wbrZ)c%MDZ;Ws)?hRf3QCU76z^y`a=sR5N@crS?h)K*;0ncR zZh!>@^aJ{cG4CD(LP8_}VG`Li{u78BAe5La-NqLYV~##0&_|DtulbIhhY?f;!uV#B zk>-YlM>7*MzBBK47Xs**?hB~u-nkZHmhf2OY~E25=u7`O(SQqoJFBK`Cc=XWZ$+!tdS?>%m>@;xn<=5?Ebx5A`q$0B4rpk@*oTg;DDe0uxjor-Y42my5QyF0`^Mq$}EOgITx z*q67)Mml~p2@roSI1t7h_lgi8Ta;yQj$(0^-^Wc9#yp(H!G;| zQpSfh1R4SA353S^&GYH?Yd3(~1MPNhy1+imPXSG))%t|Ow<6K7m#rmbw z`;q7}0Jeqgpp3}0^%Ld5Rsh?+8ir|WgXkdQgO`qBG!2=}#iwWVDglIt48(`W3s@+< z@zs>iyK$d76gqOI3O+xJ0mR*w`$wjV7`8Q;0`X&d#&^_=?`XGSI+b5xbUlaRpVnCS z(uOLY%Ac>}j_tD(vEC?n>x{2FNCKDSwGg{Sj=+nG<~dLN$t3)<{Ly~@$2|A{3%B== zoFm}W{;%S6Yc(KH0?0e@kpG|!Cr?s2Rl%=hC=lgXp=@k~+>mcgqkf+DLZ^lQw~Gq- zH4l#vE&I>M$@89khieV6if94V;^$8-w>SVr2SATAAinyeWZv>l>ISfL3iG21P#>I! z)vROxrTj{Jia-aFeX6W(Ti8+9E-Jy5WF%_lJ;d;366?NKI-p|xs|-wpN^TzfX0+c; z)Mh3cW7F`-gr62pvU0=swvC@57k$Z51)KURBMbFwv-`(2 zUq;Tpr5ZWa>X+Ip-i>Ogwp^2g7(aG6({-v^uSH3mY(0O6DuQ^t2QxF7(?=lHXGRC4 z0IN=Z9*>911mUz2>?g0egyrsRJjCS#B!v9g7@q2l2 z*H&=&p(xW0(hBEl(9gg2{tH2qtmqc??@#{+y0B{Ud%kcKK-e<7%~97`t#km9pQa5l zOVqmWBh?fuoZIC#4PT`NS=qD<4zmZ8>ytIo5y9Y;WrhS%1z2i5&3EQ?(Td7&?|4bC z)TlQc3P0pZZ?*#4fZCw9CvK@RZ_hNH2UwqN!uxF77P9{Plh39l$3t}b@FqCueBIHo z(YG{R0KD9-YdQki&(0U;iSX!z5RL7qMY)-k(fjTc>I9D?rYl9J@&08o)oJD^@N_%@-G_7BkJKxjx*Al!7ORYpd z{`mdN-Eq_9W{fOndT7sieJ~H^Xxa$+JPfQY<9|cCMA!57K~^^lH@GvF#^+>K_bs<1 zE5V$e4vSvQjh}`2Cufxe=I#M|@Gm_LLhB`iG;1Y!!B`@tirVtd#;pRPY|d%m;en*lj>WOU%qy@MUEjj>{9d!v6D@M?1UNRM5_%%8|?9=vy*IzH@ot7eOk z;>=WdW4Eh}75BJt=vGdsP+v~8*^6Ai$=)B!u|`>TvYI}|0bgqjWA(Bnhgr^$Gn>(u z=qg8k9Flk>+2m&vRyijeu73N%C!kf1$_1?XK>_j{y7>V0;oEjW_lGDFU=EYIxej9-)!OPp>F?9kUb@^K zToyUlxx^s}7 z_{RQ-1aF&78;SoXy{@ zJauaPvO9A%g4_Rr-{1gzt{Hl2k2)wgYItJkyZ!t18EugQ%t}m;X4Ts0U`C0Gx5Iqf zx#)gW`zyh9*kFLd9F~7ur}oBO=fRG*mN$v{_N4v0w%b}j1LlS=Z_**-q3HQ!zuVI{ zPxZ53=`}w#8-ALKApD4PdUodVOcv~8Wu-jdXkX`4SHFrxoII5imB#@xB2j>fTm)Fi z0O(>H?2q1v$7KFtY5X3bq5vjqr~QHsSlWQ(aU5#FanaX36`JaR9KYi`_Qe$SUU#SJ zAl?eIHCyKMR45_?ezESx_4XmPgU}$}DT~qgn}_V=uE+Ek+w3l$9=(W~Z>-5+qA|*c z5*ZwxC&HJPhIG7BudttFbRbuaL2P$qPN$*mu>D~>E%t-mD7C4(&5x(d{8MkoqUZpG zRbI~jvM*Pn?K4WjuQxiyc>V&0X`tS?hiTvJYBq>&Q9#Git6r=u$m&(s9n*7AWhlF} zF@2~}A1QOO+Z^@`jjmJrsuA|+$D!@{u@Jr*72O6#&f#y6VVwjk%zvcb{2ft{`#>Ul zt}YU6Qr@L`k~0gt{i@T~q^pplnXERFj{o1XrLTe-LEy^7jRl#|`hGMWKyvcN&Eg}q z>+G6y8Oh^s_G>w@JOXV=6W&&zc%<}~&JcH)FZVforX;!<-h$p9&YQWVGY|>8zC&1x z2kmOb59WB=N!5~7>N#DGKYgWu1$aQF@ndpqN)Irml>W;(10B^3s4d9=9u|l*eB zqUG$h+V$4{^Njmzzb-2Rf91`ai`1JbmAc7UX--CV?NT~+ z@JN*$=WyA=k;#-z$*9WEbXB1&jfNb3qO1@&R=R^%at*=p^3kxPnT6&&LuCcj-`x)l z;&yCX1&uJRf-)}GAFq-1m*jRUulH$67OZ1m45P4vysp4uvM=NTCtt6BtOme$@}@j~ z#d!It6cQR}w1lBtv8y<`i74%Y_Efu@ zsMWHk0zv94)c!XN8+-rKeI#giw>L>#XExS;b$fN>@Sf)Vx&+e4hmAeZObD`)x+W~D z-vLA*Z-=WM0nxUT{Te-s^Wh6{0@K5i}we% z@aypbd+eu~FN8dj)>uW)KeD@y*Zlxci;MQbEMxm75tG%qtMrN>*4=0}a*2JO4y4i3 zmNw4){J#FDi8ln1C+1k2Fg)-@&>% zU3cy6105^vJGS#d7-=jxz0BV7c5$9-C_1Mkk)Px?+gPOGZiVZOmXHE29H2K0L%DVu zUHW>_)X&*NXxTjf2F)3pP@gL53P~~&K8iCJdUB<&W{iH!KFw)pLIg*)2k?yguJiUJ z4-;;@s1R!8x#oNC3K;1;dWKgKZZ$6P-1HO5H$3;(fzQuLCvyrvZQ?67dktRvir8By zrSgUi3}Cik5+!~r^^jGXWD+mN=$?cpFdmOiGF2l0}$XDn4;_%(8lZ9YEz5b8zKr>&Rn zZ){DjQoWn0!n1ue{Ta3B<)w$CmzzX^`cwN2=khk0nIetIXYRi*Ajc$0ee|3ayH#s-oQFVo#Px>5Fk{E6!~n;w}*-VT{CP>B5rB zFZiUBD1|PE_)Lal)mO*aI2t?TmZ-{9OVTH4Rts9w(HvN{hWC?Vk+`!l+v*X1<5swv zC#h1x29q?PV|tok_z=5Q!%J+zE$K;RzC7oX(KB8mz3E@_7vS<-;5~L>hgIn~>|6mg zH3)B`ait_Usy#$VR^l_4kb}X*lktI6^8tr}QVc^wEqNx9Yi*wWVlb%s}2XllK z%n`0GNzYp^ABY$YoWNfgB#WQHX`POoidF7}0nJ*2B}dj)&zyP4EfIo?-HNTdd#z2bJ=7W!2?LEuC2x~ijPh*ZeH++acDcd*FqIw(E9D&E<3Bh#Z)faLv z--L`O(~eXsb?C!de`Og{9L@Zp%rnvTwq~bK@HtLrzjUALjG<3AUDW3W?qV46qAK}J zT@<-523NQr^S2X$7gl$(YFx7sj3@2EOV+BU8JYCGK5X_y zZT;wVT`!_G+6>2u%xd;)VOjg~=*vaa!=N?uiS|^(U}gG4AheH@yq^`mH=Ls$8J&5XSrd`P!9f zA*DY&+s#R(6HdpSHO;atV<0k}pUZE^x4r-HqhP_(>P8A#@qF{KE{68CXRy5GX!}ds zSxQ%gMYur5a92e-&iJVgiw#17!|AS9iyg7KFi8=}~Hp0f`J++ji zU=?edJ$c{PY>ip&-^~k9k!8uZZ}j@>M!l*i)1QC!O2DxKP9VOS@^L^G_nML+z`T5< zV3U{AergFZ0em_yhMQ0Wub?{d21eH!cr~;Y2Oj<6Q+Vd0V0pb`_`l(2qlUKK`(_s} zAttvCCuGwx1?~QqlXUI;YZ1{NI=Qr3zS92F&cT<{Vu;Zk`>{gp6>?W=4(o-w0>H;$ z`+@=_Zt1nZy#8&f;mNIRALYnf_ zwUXET8(jg%rX-1U_b@Ut-hu~!mpRvugN2nQ@f-a!Y7gcohvflEDbM&O(9blV_jR4%d%V%I;0!e7h zU2}i3rKOoND;K;`&6>`6>LGyMHrR*{m6GgRXX4zcV1gZ0(%g)+G6IB#=UMqZJ)ZC2 zSvgHY$o%t1HBr~u1b2G2FqW9Z=DNFNtvS1Jq#TCsMY+i2tg(n22i+T%&VNK~Q#x>j zQ#P6dTy#c$9eixYq`%qM$bBq~+@E;y30{QJXYZ63Va(Ff#GMS2_pv;G;`PU-Bp67I z(CpU!_?$3&<+6O@hQ>eo4jGxV^>}AjSJEigY--WD-y>8;mV=+MV(}=irlI3|3o!>t zYRa8a*T@(=K2lm{axm-UQ_5z1+D5b-3qRvK&MzA^vR+uT{o11&>LZiyOam#V2#&Pf4?jejLu+%0y9QZ>^)}^RNDS@7Sp2TlTB~ zEZrs@ax^l{eFyvEEv9-UBbp}^u3+Ub=?#11{nhq(&WZy_R?xhda0=);w`SJM`*R@J zQmA|g8MVebdn{(AMc%x;m$l&z1sCQNRdsE-{8-riRaAx6zPgnXAk-pGA)a{8{8-fb zMMV|m>`~5|6Xo4>kCr*m+cPt7fok5rh03qPdw0iNM0E%HCGDBrjCjFP7P*v8qa-400W zi{QLwD0g8vm?Gh1E}*i@adQute5x<~_4b_0iU5nU_r0SJ!Lfy~i`g zNXN@QE4u~gZ;+m%wt|+W!e_?>!PZ6aTT@$Y1@+S=ctA5b56(DqH$U_HRCwlBpzvhk zHkkP^qGk#7bwIAs@rT<&N?R8{Rz)4h{_~XH#5oL+u8KNH{LW)a<+GQyn>J==1Gp7( zt~51z=9iQ(G>lRN1}L>#e$hjkZrR!<{M1KRA!c1MZ3Us~8K4urPO`Ci)>dA4$lsq` zi6>(#2Z-&q>A_E);-2`mjB%p=#zBS%T&&3F_qli`P?%Ml)Tp|B0m?qQ%%)Pd2!hpm zvV*?5J|9pRYoOvhZUaZd@abMotBFqTPm?r}lm`l~<5$$e=4P18)i1eTfB05OZCcok zG7*9kuGbYt(;Ag5rvMUXuLHl;?UThXV`nN`r$+q}kN4e5pQ8%g8aR_BG(HY)jf!I! zM4>09U>HF0m=2Bfq3R;Q{BThrywh0u+11x^g`)W9Tm{Q`Ke)BvU;mbE*IU&A7dCPL ztt?+v=YLLdy#)VDt`gAyqD3`H_`iAeF!TSSMb-OHE$V=nCa)j-^L%@^Gik1K0jP@Q z_CqeieJGW)&d-7Qqu`(v_MVfqj{1kQf#9$(qn>b*#hX@;>b2Bg=A)=6VZr7KbB4yN zxe%xI>INoq^2^?_OcBI4OxlA5p9_UG2ve=i0{`(BIlio@PKWQajY`c;@259BPW@q1 zavX@dZ&+F&XDHagoORJ3t$ByDId z!d53F0zsP?*O4SnxV2&6A90FMa$nJ15?ryFep_t8F0yBkWv9=7zrV&f*WDHqFqIe0 z;h!r$@(H@`=mY(?7U1u-Op@VvSzf)NO{%szejS;|DIz*9-Z8KESy|h|4Qv0*`K+rV z&(+wGi2Dz z?hW(tmDkOU6AHF1kM9|XG&6`z0a(WoFAvkDW(M3z)iNR*sgnV7pZ+NEuP+AD4B*Fj zKD{}PeF`XSsLgtRFd4xa-m<-doYIrxJtLis&wj5wr5_ZOe*hJ)?M9oQpD!vQp|M1E z8vrn*odtnt8a%Mn3Unf1sv7To*GS}EQ?;8hoNyMT25s|7uRJB@UJuRbp_XoWwN|B( z1`X)W_ZJ(6?RRopdYQupP{4oCf>^q^X2J)_fp}pOD>MK-*Z+Z_G0nqn6V60zDZTh z{4p^REEK(cK7X$W3;HAM8~cHE=;+H7c~gy8+kp!9Ww@m+;SKeRmL-J(MLIj3^H z(-A_AwO8yK9x>nilpfN=8?ROq9o!1fjUMpTiDX0WWTx7YoXl7C-YY+{r-8)#bPd$C z{)G1aQn>?r{?f4Bi*p^6zeY&p^A`j$u7O*>5L1GE$}HoB5_upu+kui5VWL$0bn(qp zHAQ`KV@OstS^1$ZiJV4xbghnSd_{v=LzM$oqzCdNjX=*sV=1}fK1NV4!6uZ; z{b8+or7;?RI>zhq;-lyXRKU7JZ8}7^n=!6DN$inx)3nSKKblvF6bG8rVx=aLWTEE4 z0ame?;N~PmMtTIYb&H)??S>-$6$1gS$$xG9y?*RnMw z2RBCfx6k}k&Cz_lQr}9;L-)6!4erN^%^qEMP7&)&7d)PLU69?C-0cT$coIQ}m?Ayk zPi^wJQgN2Ls>dqEBn$oUA_?c6lA#y^*>&jfajHR~;qTS~b2AWqW9#raGxgtLvshU9 zh|B8{U&`HRnm`&PbE?g*(AUeb8$FAI*da8pZOeijrH?i;^JWzGvU7iJ*a$^ z!lfNL6%G4_g=`CpLH$l)4?%gsNfn}w$v3j6l*@xLi2jkGF~hQ-Z*&#EGpptJkb|sm z)X;I|L7{;Xd|u#D?Z|es9-m~`hiu(TB|8J3_0c0q6I7Rh_sULyuHk$W(Eo!63 z(->N_-8BDVL^uYSdyLT7{H=^8&vR#B;ILv{O0Z`fne#)5Sgp-vmI3>1|6Gk7yMfwp zNzZafP)gF$9(wQYoZ8pHkP+T)0Q$V-b9&&z_z}mmJAK~EQeA05+=$x07)U)+>?;zm z887&?MS%4^#pMnwA!1FL#Ttz2R;(`u()q7t8vhDBtKaew`*qh^I~=wjV3|1k+KGH& z!2+J|EBR@&ZBCI`8^UhJ)=`N`8_>z5nGBHS83%gY2TRVt^l3?rZ&gyDIVV_mX3lxh ztp@cVUN&0Xab^af_wJskkM0-7?d$#yB+p12Zf;(qp?M2KsK<6ewc6@JLSjp4Nzt$D zqo(=%3#bfsR?Lwq`7fDm)>Bczt-&BmsEy;M#F!B>J=4->z4raUo%uYHk|ZTs`Rm5I z0sz@rt6=QNkp40|+p6#w(u|8){) zA4M9UTOa|DA~Jx?fW76*UTqc=Ez+8VNK;bl~d z7ho=MrYt)iSH!74C&Gio&obN~sP9UBa6 zCAwPiB0%QB(bMrdBdwX|2^&P+viA1x1$J3k{`t!EsTgeS;$C7wx>HPE`W(>TfR{4_ z^sZ4XTD7>t%7-<^O0+;dAW(@>5f>%Gb^Ljs3vQP>r?$HJ4NqvBT4%e6`P%*nayu2E z;`nsh0oDPxZrbwH6=`6&T{eU~yW?1qe2fUA99z4~U$m1&*p-bK-i&l^Z(L5Cj$X+n z)1Rucc(rG+i`gP{dgSrbJUs+fr!o&Fi9@Q>aaPu_OZ#fFq+FkvyhWF3ne8q84@!AT zBPyP7#x}b@zx=F2lVH&N!FeUwkIFq&m(#CmRGZpT(wgdT>WS?zCPkkAUS0tq#xjsP zjJ~$CDj^@+>i4nFAVsC|%ZlzgO@1{s7zaIC>rJi2bPO=U*9*0l#%mz^eRltDmQdJR z083MFn3G=TUP9VTj6pWK@^7r^;93b((<8a0$XxI^-0TwEk3YRavw2i$iQ9^9w15to z8Qz$>zqjdvmtL*ldbma4uoiG5x-NN0fYC9bW=N6m!Je1ecLO|N6|Qz7Ql1Dl;0@y% zPBGlxkwo|oweB=;@330;*QB-B=#vrw|6#XJasZ4S_FFuYdb0zTR^w&Pv5x|a_)(IA zulS)7O*P2kA*CHvEl^LU;Wt|H_R#9(p@j7l(TbH`2dO{Zx_%H6-FIdhDTc4nn}eB) zPs2>*LRNi%&gIC#N$jC^!vmJ+I#hewe z7G&|d==+ej5{T?q8Ilugw0Z}vM|~jojty1z{GI}JPcdvndG4*OK#)aSx$m~m<@n7eOpEO z2{6s7>i>{1@t@DqhI|B5b=j<`9J$j6Z=<|99Zq7vm*tsgI%c-2pOku1-hW?W0mo^p z4(l%sc+qw2u|Fc zJSY{tbg=8(ET*%cMT;JJm`Y1ieK$5ox=5ezEiU+4(P~#@mKQB{xTAbYmK30G26{D; zJKCT{>O5z;NwsZ*F5Xe>?&Z^dI3nZbbHUGDUjdP3t>7#zHyA>5t75JZh zt=F6WrLO_zE*XdUi`7()pe_*s%q85cjR@a=vhT4*|4^)oQg@bZ<6(`~&Q4#wO0@d( zKJ%?LUkJ@}czm2?2JZ*h2O}Iq$ST+tnl6uv<;74?2YmrA`JmJAxW20|oCbZ{hBa&& z^Y8PQAWlTiX6)w}GFdle@&}MZyoD zPv%N;=Xi*ma~?L(>tgoe03lkdu+l0jd{({eK*&(RfU|47YL)iEg!gCba|ZF zWKx&XPV@ym!IY4)hW1wA(rfGZtp!1OBfDA~Zwx-H+3S_XNSM@Yv^5?Zt`*SY!_&EX zZK)=i_)I`k6_P=o;H4Hh^xx6-2(9hqxn4(~*G*#~{18J;2y&jJ^2(=~-KAp;obClaFse#G{*d_FA zxg{vQsL$Bj>@>1H)Dr-z#`AyFYzbuCjrd7c>VuQFtumTb0U>bEI3uq*AO!ZP&R?D{ zpR=g4IHz;6dIm2K)Bbyj*#73&Ms+ZM3g4zXTE*_Y$9?6za==;MFvuU z<%DGJ?|7bq@ZyB@4J##Y5?L&g;E z@(I@D^70R1L>u*koqF^zP(amM$xCwJJc2c>NPtmg&Tux^qB=F!vjlRr2TPj*jPqe; zjwD%9r3wXD^4Mq=v4k{*S=ZLEfDdD0LoyYsP#TwvzcsF5$?0!I+i zCv;5mKtB^WF~V(7^yMr*85**&C)bzS0}N7M9~;R&L1!U#)$9kTB69bT{ioAfcyKKx zz(lu2IfuxYvWObZ4#GRE+_ZHuw&1beY{DOLW@(k@MnpQf1fnUP{!tp=;0l#f0Am~Ra!l%HF3brtO!ru)IC8uT|YX`sW zj{>)nB%8`=|1wn6#@uSh#kRh4Ht_+FY{;1(K4GK&&nas z*%1`taGQ1PWK65BOTRj}tgNM62z9vbkff2w4BYh8kH?nh3ux;Ya76?ZuwK&Ly}D4X@}n0C{&29+()_i^a7b{l z)osZi@T*M08SG5V_t$gcZ*^t0F_FIW*UCP_q3F*+klNltx7==sF`KSkqlEBF_`!)- zYW%{(ME%}vGNH3Gx^E$-jd_B2a_^B_OGJr(7p~Xlf^~Of0X#cYTLxOuofUT3+ik&7 zfZt+P8H14!iu+LfCcT_%$RTK+5o`k3uRL2$G|sqZcxQkYDFkF^6dZCksU#s_D(Za} z72Pb-|Cet>7&dnVl1=&Uv3K~UI}%^V^P4&ZKvzjsC_E_+I&Esnd&MNlL*fJME&4Oz zLvRdrH<;TtU+naTtT0ft%GmjgHDm{;sB1N_LC`SNI)c@67PE(XHw<+NrWhy`_((}l z%Qe7%9q7er3RVYlA$_&+T{f;Xs9d10Pb7Y`jkH9}%(XqLL$Sb6pMbxORC;^B#$}vL zV!*0$@Y9F4;UvB(3YxPDe2d1+RF=^@XiPybME(PUd3p z5&*>QO88A%7e!x6@zYL|F}m~GU5MiYSDo=ML@%fL>T&+%#Es{ZL4~K$+K}Mi_HqJcZfugdT`EE! zGMD7!-v%WV;#ExV?hWtf^HE6Sit9CM!gJ5n$09-|zc<|$T>vHB&Q&f(h{uDA#Y@P4 zw+*k#H`6BAS`Z<(1y8M{t;uVQA^_C=eX!Yt8yh~HLf_S*3G_h+&fe7afh)E~2@XU#8A?*S z8AzPUPR8Ijqvf8FuYav^xqUA^!@R$?Sl_7Crpa5Cxe^l{7W+@;YJU6ahTNR}-k{yr zX5IT^kU3+%?q$^00ly}iN~LuzJUo`~tDnoR-)Z8uHtf59O;({-6;|F# z9C6$X9>)fy{Ql1$@TM@=L!MOU2pWlocK2Js0%?}%y<0OcK6m?A5eEg0me4I^<=2_c zu8Ofh`^xfiSmTo8#j*28%OtV3>y?wOz|yjdrV-j}Xg!vyry(>=dq)$c#Wj_<&C^m2 zypyYL9fdD?#922h`?OEar0!wv^;=0o9`zei>_>$qGbX~K!&**-Jb&-efu{WA)w-7H zciqVpY*WKwyuJTdA!D@cZ}9{kUQ<*r(;hGDG^$pKjv&4XM!h_|AFm!4Y{Yb}h$Y zNBE%`!xgf0iF0`WW2wIM{K_Pdt3?@aog#UxrsFoodXW5H=k|c9&*Z$Z^q2BprgGr9 znFAr;X)JDe3puf~MNAw$W+e9nq>GFG9SgD#u*AQ){)efMLwbWz`P{lTdPb=?R<~|o zHo4B-Jyd^mvvK7r85BXq&;B!~DtK|_@n$JDkeBV1|ECsp51RTYYg!o?K@@2M(3%@IFI_|3W0 z&w^Z+l{PoN(x8-us?I*Hm#Eq1N06&sTQaa0mjdJ#xpH-}z?O>2S(e=s(UA4O1Q}*- zD=8%1Ctr^cW%hS{>riux7e7tc$<@gm_aOrotj98<$JJ+k^$qT|ss;gpRm#I3@S7fN z>E;>_kW0D09<@{HsUzlCK*mM*RiB~+2hSquI#GbX?XSncwk0+;4RJ}%u-Ck7+^GTv z&TJdwE80S5zQj+}8>40^lA?r!FSaW{v-(20M#eAC(&%*>f)qRM%6ZMP?mvb6Ynxmo zmiPJ3+J%GuXwEJ8>PeLy52wQO96`02E&bH<>+?g_kFz1L!?S5F(?A~ai34dnnt&~- znwvNE(6i6TnebsVyEE!i1f8nQdLMqrLv?_Kwi6!75)GH)1YKgy-i-loDr5~hiuQ&Ei-rkXmMSzqzrzP+gI5OHgXT-@R9EckpIlfwKQ4f@Ryy*<4B za~&_iaRC2pqhvEK*!g6}%;>@D0_GLiiW@z3$e$xmN%aZa)5c|o|A7%97ur>VL!v=H zh^?8rG^396C#co64N4sC*f+%Bex0)T*}^!&tS8qMsJ%1%ZX7{&?TCeOF*Q(no%1Oy$5MYy%lCQ85sBKaC_jA z>7?I1lS=j{zgrZf&KFuueltx-p_wLvNtM52L1UluJb>?K%!=Gb<_hW3Cf#ky)Ckk6 zxE-_`P#gW0Wl0Ofs6)cL^x5Lg3amW#kXT$YmPFPD>!4LEi&(BU^q3griUMg zZyf{TaYNk?%;_mdbcxI^r~^)9${(-}b3D!q2R)Clv`0WbB)oI$9~g-Ml~w{g&Vlnf zy?gpDhf&AS^XfDAvkOuQqqnZgu_S-rFl3z)ipET2`#d6%&juQ#X1VplwS}iioh>E* zg6LyhxVu^uk*TwTyUK^9d&eK2qZ409ECS9TNqa&4N6*G060;+HyA}8JFfk*A3zlOK zrQWc_RH+d6SQx0rWVzZwlf~+Pao5$y7M91S+f6SGG9#*g?@`FfpUnzrpH|-;-LKvh zH#Z%hUNjv)uFgJc;8oh}uagPHK4UftxJ1>HJ=`2&k5s$Q;T&nG-aqqJT<0HY2f34xkI0@Rh5*^_@mvmWS%F!u6Mq;4^WREJ~Ht=KAnPmJFC~$w-r8N zY!@x|U-=31YjCA6fOt85&mM@`y@*752E4<1kUO1)Z zE$HS)AW-)dr9n;27FI@2JuK@nM7Ves|1&%@JqG=+E>#Y`=65$}>m6Yvg}>-AaJ` zPcO3Q9uRglYzO#*1{2Qdffe9|pVgaJCf<)40*KjkqJ|H+2D})4rO>flOP(Lg?BT$O z1=hD}IWfX5SM@={w61L1;@#OM4u7jV*xC`|_Csgu)qLC9vmHF2m5iJ}-~B&MDB$CE z#i((bEq9{jX=JR~MDx3Rzz9Ee&BZ4aILaZB#i^9d3al#!0|iI!|tlK1E2x& zrnm)w6i*`8jynr(N2-GW%2WBWG=4K4fO>Iz0+KFDpo<(Xumh~U-Z5Ed1HaqX!>)2W zGWlbK`^4S(DyWixvy3LB78hm}wQy!sIEA!fnLmQ7W7uFLx|a3vm?&IFc+(I-cIMlme#!`1fa%jRpx#QDx>*5n{`^G_TPaiz>k}I(QEzDYBmI9pALu9(ao^J zUHJ9p!tjLE(y5cfWvBM9K$;{!Sa##sEs{9=XKKk(i`xKaGSL;jCKjHE=$Cijxum|0 zB~hB7Zy|7MNx5X9qI;_4RZGgXOU?Tv4ib+C8EuppOz%ZC2#imJA89F_y5#<SAd zH@(16%p-RT(v3h(DYu`2Rl`jLUJ%&O1^|ZMIpV0n{2vx-@+&T`tyPg?x7S$J`aLhJ zeJ93E+Xd}y_v==~gT zM4MKLq3Uo1PWa#QR@;BeTe+x9u>HH2y3kVj1PO>BSNyqQDvUQnNO6aZhEw04w`VA{ zf0r2|7RGI3&OHKfmD#`Qf{uXy6rJ16-o-@g9sEAOuDKxsEa*8|#boH#dPiiumdm4} zF*jm(TGRdhi(9p|tXru7>)yP|PkZ~>uTPa}{}Fs%{_Fp5HR_*KT)>sW*i>H>fH;%G zabq!F91>l;aFqTr4_@)sX9b3|t< z{?G~ax>0i^Y{pymDJpg|mVk4EX?5>iM-aV6+xjI{iW@qLzu)iZL`;rovkg4v_voM4 zOAV)I`{)-ep_rH-myyK7QlzHw^6S7>norXsVWJ|O-g?AP=8(-1tZ=^GD3oW|Pnp!Rgcb+n2QXy@ z3&w!`GxRjRP7y!=OuIl=^)#dobHk2rt5VyZI89BiGXn~Lm;3iJ#XGI_eMrh!EtI{S z4j(ekHcHkhx4(y6I?4UpSYuHuL}ApM31YzRBPJG$bhI&p{QjxC zO~v#5hZmL!`WDa82RIP{2iUHgD@OOq`RbUbG~SfA-rp+!m#J|w7+4!vUbY3b>{Wv= zO7TcbATRMN1b;w-cc6#v2i=lCH3fw(0p~0D(buQ`aK1yhBW;&u!Ge<*{lSY&DB1uW z#U7JVk{OfvqcOmF5;}ZWj%o0b5~dji6DU%2-vY7nP=ld!a$mlDdB2@8$HIez;0F{P zQeLQDumrFU+ynH4ffHx@DdM2F_nf7Zg`N|P7YaX5M+Y)M)MUI$ES-1&tO(sQ;&)1( z;Wx#LN0Yw z$4mf8NggtGOC~2j8<;cq-dS$q0rbSZ?jYEUvVL@M}H2?Uvf)oEq|W0wcG^k^~q1bzXq2gHd0L#vhWXGu%|;h~RJ^3bfM2i z0lSq_a4^t-G*vqrP$R!nvHASjb@rkKJwL2|p7~i=(6QQA@stlb`dOF~6G>O)v(jPp zTVv~?eG8{SFP-;5A*~w~_rgB~Wx8Obn-f)4pe%r_^@6#I(#u_(VEM^@DEgK*%uhP` zvl9W|5S8FLm3dvWVq!H2mrp=Z+>25GXkC#KQt{wmUBW(uoWU@CjtwOGqP<#7zyv6Z zAMnKkfaby#2UMlccf2@&uu=&3^XF{{{FC6$y_jRQn1Nxy?MA@LamSycPLnGaUq>L( zfes0v$_}4HnBIbK-un@cr=q581c*yP;$NfiRNNWrfs6-mCxo~_oAEP!#(lE|{}~t1 zZ~hSaJarA+*~|P4##x5kt0m%RhfyAX9qHqw!?UeqaRy1DXMZpaRF&)gN?bDEdm^j zkTf7hbwTs@Sy>EWP;hVSdI({_YzQ)QvPJmLSoa0DQt`8y+spZ=ntO3@eL+Ec6s`0l zf4Y%(7&al(V8oo7mTWaVu+^dBpnU7FHhr*;3gss>pnvt5XnW;5v|k%=L9Z3?SC|@U zD;V7YXqJ=dPeBzmklB@icGw5#i<`;J3dwk1*u=jk;eGii{`Gurx3Hup#}u1>5T)RS zwga#4^IEDD;FBXxZ(+c^M;wf-S$y;$ z_HJ*_-Yf~LZME_q_9)?6J&a1-E*gu)s(^?%zMiTQ%b zvf|FRwnah4@u0iNc-Q@o5n;&9{$&oas|Pt>p0|q!nfWtAvP^Af(ig^qfpk*$Xy@nZ zfUtdmwc-}}T<3j*IA89}^z`Oy>PqWd*GE(mdjXW?{9VXT$j!u5aX4=}rS@=+5zW;( zXxoI;eh^D;-k85nBDv;dY=zZ~z?Ln141=#H>YCqRBX{J+bk|;n`|7r6VBf94ev)ch zxv7NHY|enbQm2Hgr-!U!$9HgJg^1iF0d}>Yu5_~lL!hdTj+|V-X1vMWG->{oe6-4^ zX1b~do`3J!?Op+Imb1O%w#Ft8NQLf=MSGEQg74i>6!iPt>YaIGNTF3~dl2GL&wWd_ zi>>*P43#*oxwcCRhtnke@j8PZq-8)vK%!c|bqQ3LKSZ5Y=SggyU9z5*Z{T7B^1$lnMajzLL`{bSejTp#7CR1XwZ^waS2wDf9N#F1X1qFU`~?rFP?K2$p01p4 zTgo?X;-8g%Eb~PQ$TQ_#c$1kd3DM9tb+cM-{`R;XSNTy)-#+0^9xcv4m^0pR4JtpE zBuyE`+X%bwedr;bU*`39r5J{X6+2<=BU2v718icd~>IJ|4giM`bJE7+MVN8 zyUbC&Gk);?Hp|*m#Cvm~@L(g&nMw9{EQ}_J{M%@0DcDWZ`6^m9O zHHvdo!eYw#*Wb!cHe`V<0;Y2~-~ZiiXCiIE)!F)V)~M|hYxCS4)?A^qx6RL>IJL37=nzXL-XZgSr*%jA z^~9~26Va9Z-MES$MZ$Ot3%MKgZt*}Zk6}E-wGe-ANE9i?#BZ%v!&^M3cm3+MtNhc@ z!A%gmr72ng)1F3iF7MO|;w(lWVh;Ni8}s|eJ>`lP?gpyzd&4J=Z#2g(Ex}4;qB#yZ z1Gi;nM-%iz4DADQPux5K(xki6>2jrIM--+DzhRH8m)E=0mN!EDTPs0Q;1k!U>!g*@ zy`@x(LQXLOl3>m8`)^9F$oOM!^1FqZ||RJT8!jdWI7dv=%xF#BWrx0 zi}KvGLL@D&b*db#HWKmPlwZ-oHYW{<^z9BlFMV_{ZvfuR5>X^*gvIsD@akZ$W4Xz_ z_y{nnzvMU^lct1a+178E&VzB+*5st8L2v%YhmQXp+ zTPU$ZsA`IlPAYllUK&!9%Dv8A7wpIrPTL*yt!uT1-;h~B`-WCseq=kbwk2l~G3|WZ z?2W4EmD>O<*%1jh`^pD0eTmF5!A;+?K9B84dpsFqI>eJ9tBH$RVd^&aM7zX%%U^u^qu7#S56~-Vb z`2I*}9jmd_0P0bHa}*F2?bH|FTHAQ-4%n-D*8e77aHe0pf#_-du566ay=K}$?KpyY z)h8+@Irs0sZ!?o2-Pf)~E>6@a&arD2PFT;5Qy&*EAbF;5*i+nB-jkEF!_XkC1ZwrC zYo`cDh?FuvhOwdsV%MUy>MlejK3oE!f0{jK^f`7pO;m0^fJ{!P4H{%A?kqms0y@*upJpb_}h`(7V<7gM-N^P-p$68E%)JTRE%Mh;Zx5*69M%R1E?>-Nsqi_#q|yv5ru8}UNjBxjo8;+ z;`Cf5g$;p!vgJ^BNIspTk-d(E$X_?}nXMXbmNoKp3!}uPmK!?nfBGT`=iXeoy2Kb$ zri+b}z+8_{OJb-%ZQ!`5+3Ga-gnnLk0)&I(#Q+WC~7ik;3~RSt5(f--0I z#AZDcd(^?d+L0s=I!LYlg3`KZ#yR2?K&E}4@<&F;@5A7vsBT2tcbBg>C5F|gf52LF z^4YAhwdQtixVi`&&AGUkd>X>lXV? z;&B|9#s2%4NsmrkAS0DK79=8SNtK&qvG!}cq&&Di{SwYUawfw53cb{*?VysQNdMNE zsC+n8>NqX+M5hwS)8oewwi&}_R}o!~>l)wRDd|7#Pv8>4)F;58f0)MRv9Kh6-DZ8Z z+4fGReou%{9ERsQ2fp!U#CYJg3RKHteMY&MMmik&OcaXEj{aQr^Uow{LI_>!+Q;M5 zA0sYl#Frlzw4s%1%RTJrFTZLVW4*GR9gmijcP3{zzX*WX>8n4Q9n_PldE*I97V5lS zuBRbsneDxXuRY)FEx{@BNONtp8)kS=)YjT}BpEV~;MhFDGx%8=_@UbTx0y=8t3$%d zAll=)^T9IbFZD}E@O>7VO_7G_-LW)GcCnn=$1wk7dYn_opT2PDpnJ1?w@vR>f}+2r zjXUAOMr!2wiXa})k-_vY9X@CS~R)=`)Ta=WA- z40~Y7{hD#k<1scRE*Dxn}lGzVH9r==$C zPUAnTp~l0HAYT-T2oZ1At7zk3WY~}_Lfy4tZ2Pz%HIK~gVz-=nY_0}&bq$WJEA#%1 z4;6E4_C$SK^h1&MgTm<4engM>z0UT*QT^~+huix%nh(hP^9c6agjd8UKC-iomp|M_FD}01J!$-m0oF86u2oTpro* zuZ;w=1-G{On{Cu36BJ%YD{nHi>~QLZ zU12zb*McKmN>HzWY=_UAtu*PNl}E(5e=lmHn@YHkBs^QQYjkoeIHUeT5dwpJmdVN^ zh8=`0VN=p3EZ$wDMeyP!!|&sIGjKJf!|dGlVVxB^6fR7&H$^YZwjsko zRkh^?im8}UduaXg_IPy2L+=(vKNn=WW3hhLN~_BZG7f96azN@Z)&}#1M7k|?V?BKm zfYE}o3%a-BR$kU|C`oZ)H)++FaH%n{rH1dRDS!L$+4wm|Q)#AuAUHd?^}ZYAYGHZ` z8d&Uza@JlGC=5xD&);IEmL(iMbPVfq&FQWj%j6f*c($ z0m*Ay+8E=kWpjLdBx~KN_!Hr0)9C*BsWF)ztciq&FZMD}u5YG2xr1x*Ea==nVnx2L z-e<4Nx{qd#vfX3LfC9uy0=V#||DIrb)90$ufkGR%mGw!`gi|HrzJImEzfN;JUIEwt zdDM5~{NIH*!nlB``+;B)XBO4R-}au7Q+q-)q=rP~&#qR;MHP0@*XA8J9Qpsa{nI+J z4E4mmi8rVHHmWP*GDMD>hf66Y&4aK1GH`D$kIakXrqTpbgAy)q)4RcDJr#$#k9>4@ z-2^qscE5VC{^`eM=jHo?pobyXMDuv}BSW|TYM?y2k2ce=?#WYEe;&Y4jPh#zuSu1fAT_gYyj;E zC=*-=3bs5-8FE?l0<>+2HEnJw=#kb zcLbelNt{K&wC3-5qZurV7A=1}E|+!hYXt;!x1Y_hq`xJ458-ZR)(3JwWxox@~$4!1CwJTF;2ktI7*E4tKgmHeXf zTGG*9JU6(l)nQ$5ehO!A9EZ43fw@n22ILhXy>95KMf$^AvDa?X8*VpR$JZa}25(13 z{2IKo?EXvQ+Pk7C=H0^|8-4pSWH%$$j0LkXfzkOdB%&SiaF{VH}ChY)G zkY$}SEdQ#B=(RRJeLC2pG*oM{A}7bSZO-I-niHrgEk$O0a(#hcS+wpKlyds(Z z^l6z)1>YY-_W0HBt2>>!c(*mm^0gnN-vij!T216ZfbQ_fJyqiF`mNEOozhc<5F7$3 z0>GT})5COYwN&2Ga7fOz9$>W3rP_x5R7w!tY!nHGJK4MFseN3DZJM-y#J07X4J>mf z)W3Cmt#c%s)mr?HK}6zo)Zo|C7352aVj~YnT{L5D-meO}w1=!$j1I^d3mX(jl4%2j z$6j4~>!J@$BBrjD7>(Xie9npPf`~9d#%2e%2DgnJu6hcbx*1N_Gx!8m{)x(^Yn*yE z9nZ!(;}mJe+t+WNDmo%Hn*}EBJ=-9V^go*aZ8?(LLBpHI)N!4>F&@5Wg4&(N2D4tB zq6|M*n31$td_`<+%`^apos*zM;;Txj1W;(RwKX+j>rzCG^L6%fFUtc&3!nKyICuHI^Ry}zd8@L$x)?L6@TG8@u<<0l| zWi&&cyNUW|C_f@MtdmJj&22df*4=}uY+zhapPLGP2)-H5XvUxvt9aKPmfHqBd0K`2 zykbVLO(fs!Pc!5e{;tZODO}J?iIOgkUQg!--p~<&pro{pX2D#a^pc2S=ckGVexcg8 z-YJjTfqQH8v>&ndHM6lu^RBVeV^G9w*w}p^ro(`A#I)+a3pTMtA*1|oD+;h31U$Q^ z0%t3@=8}CNT`Mf{6+|B8MsA#3tF@N%Hx)dw>vwFQy7rb{9cL%^HJ>6}d0o6VkC<(1 zi#CR(!Tx$dd+eDg_~;$+_5$5Ih2*XKiqade>K`~Wa=hH^IN|fiNL4`H!=9X#@#CyJ z2EA)BA(~;Kh*LvrSaTDvcB?|{r==b&r*l&N=biRGLy2sBF$%*mjk4I9qluOFh6F#_*Lt zaY(?R&GRtZe%&W=%m`*?Zp9Id_chp=?#Gi8t>jwAR<)3aNq1U27ktlPebO1J8Rezk zx{lkT-*Stb<~19b-Lv@T7!oW|E#t(H-t8T+++8+C5=2`l+rrgX+=Mtv+lyxcYe@D* ztW#~Ed=vlA5_eI%8_&wad-*Hpjlg`eisKj>2GaE5Y)3CjkAC6TBRAu&7t(MCX|f)L zG-eIwXm3c5`=X6vnDnJzUqzWrGK|K8qopXf8-%s8kJSx#z zXM4T$06HAY-MHQ;=R9t);jkatmqS5SHXi*0-f?cE^HTGl5nMNzn8bp5gEW+ev5|7> zs9lGzJw9NGOmyckk&YZJl^9)(JdiU_?uzbY1e)$y`bjD{x6|l6I;bhwe`G|TSQ~Rg&wS(b z`#6Ii&INx}hWgZe`*r!Ua4J7gC&CzxY;Z}?DH8e_@srn6yYK)O6IP1#yOEys`Hzde z0^lgBbf&H-gVrW7Z&J@2xo>HGZy&#WK!D9r5Ncd#U7ZLXU4u2AK|8#t;bUy#PeC{u;CMW?DF1yoL^wrbiGrk{DF5Hs9IsT zrbLgswZHoFxus{U#ax8gGM_lSFXKu!?+eSN$kp5cX)PlXJ?qW*ZQ_h6(Dd5}7kSUK z3Y;EMNtd2B976ru@bsEBu@e=@qLFxb20(9qmXjc07xk4^Ar^2(Je7;kMN?Dx*DdqT$bk*axdOMF6=pIE0d=1?Mfwt4ju#Sa8^7IFOU_g4`Z z53RPgSZPqfMFLA|Ory_FHj<3peeS&reX%z| zVDBiO#K#S2i9N+6hJH)|CY5g7FZLhfFMS0N&| ztAR4W-}t<#2Aj>hBaswXZ=cUduo_LjS>!>@5b+eTb<=L(pO}W-nac91o)Bj}9g>$U!t3X=^kAuDE2&z(%^xEcP)YP3DFy5TQfy$%^&_hqRPz;3%h7D>E0 z(y#C%r71EZMb4L-EYX4W_or({`}WJRA8nz7+ zdRIMLa3=U*)7w@~x7&yQ@p@?~UT0ZGiCVEpZxQ#TIB*q$J)NI*uhB{k=h=W{frV3R z7&kwS`u!1-v2|taihveOoyiW=1G-0h3`CsO^D|=`hJaiM%5w7mL)Tk|#knkNpjaTd z1=kQH!QCYU0>Oh5+}+*XH9&9}k^mvNyA19ySa5e87#wDRJL~Lw_FC)Q=k~AuG1Jf2 zR#jc~)?50l>p?xO3-mXSUfgb-BJ%I^xtEST*A)h06b3ZW1YqQ3G57vhRh=Eduhzw{ zj&_o|8KMoP9=2-Oa$`W)46w4t*)yR~c7awceU&KkS+3%9sk={x0RiO1`IYWAN66Go zQ^(ym=2fJiK6FAIm%cb*04gPmtydUs1r6!@u4i$xi?}!U2T`fBm8UXUugQ1k3b+h< zeAOgc*mg-3OXQJa}gc5tdjZ{9cj5T>PWKD|DUWDN* z`U`}%=Q#FO2`@Zrb~kHI@l=7OPTSMQ0?p{vE$zFD3~+VlTw7o@i)EoOgV2L%AP4mn z4bN^1q+D?BYfzL2ty4e=2y{Ak*sh6?ye)!@D=H-iadEuK}LC;Xf36 zpl^%1CUaWPCOu&OL6s3g(fHM{Neq!8wW6g&Bsx~E-%-V49iU;$kNNZ04{yxOhmTVy z;uzT3&{W{@;MS?OtLb8^5PH)0<==-dU**x8*wM!by;mhV*8Vo!Syc=P5;{dc-X83X zn28om(UaL#Nl7s`S&x-d^oL+;Q9_Coo}*rJY8HX}*Kat+>YX<}G4`(?toNw;|K9$x zCa|GUXEGQUj3Z{PHAnr1zC)CYG6ztR5E~^j;nM|`}aLhBqC%3yvoxxL>;es7c0aA4NFdzl4E7V z9t3k>EmN2D>AkyWQD#k!#wK@zCp;)K4qb8Wf*$zl>;B1(lH#P4`YhmM1k#b9V%z5F z?^!fHQ!ERBud#fdLOg?s=5p>9&?NHnm*npcsGdkCa(e>zdf=HhjnyKf8N~2^bAG-| zznK+p2>V^TmW8oc4SvpAm1|*mEI)ZV)Mv^>$e?d@jAj51C3@T@ozy{R>bv{9Rg0R_ zW()-$HuV!Q2V>%h7UdQ@L)mKNba15!T5jpV9swn)643}Gn5rUiO)A5QN#^Tv+A!D( zbIJa@#6cM%w3eNsloZ4XYSfiuRJHZ)p2|#(laDR*j?Y;ILhle#ZQwb2Nu@s$FT^RB{CYO%#L6M#W0oq|SoFu;}<@0)BYu43n^ zV|B?+)UgC47YX8zV>S9GWO`q2dvVG=MFh3E9n%H_2}U~wYvX92cT}fU+BU%#RiiXH zc4!)t9APgxzeMW-(_WQD{JvMYJZjr@+T1?TlY4$=r&wwzH~(MYUh?`WM@|25cXL3w zZeuicKrAw^7Ml@DF78C8gcUj3cIAhzzK?;6^@(9Fel73L(=ii^_0+2bCAP;`Nf)2_ zqy3T$?S>*3s+ zZ$H#!K6@JzJtyc4fG_q_T{?y0Cp}wWUeli&X`&t+BJudW5#F5z^>nJmz0zq?w6Zjz zY4xBQv@>ybA628%Dj(c9jA(HQGR#|khV_3W#_@ht)&4m%Lt&E)LFAB2ptlpNvtYgbeJs@38@-?Wn5Fj@?T4^+0L^wQg?O0GUuj(n= zwz*FC(#?K29j9n#klozUzC^fE=i}u*cmVh-uA62J1T#C@y_v~cJ{qAO76&_@CeDI0D@WWSuQ?PAob5|kG zYH#Mp?C#>SScxI_$&G8HBUkIzrbZWpdv7hz>0qSe@nO2NMrGLChEbJa|Kvt}zll~5 zS@3;}LpHKE%GB-;M0XpaV87v}0rgIS4~Fi$*X$#-Q?4oO*sv0*H&{&kgl<3iZ|?*O zl^p{<84g4I&sj8YxJPd}kSExUL1KD~NGCooUbP}A8#a?;2nJ?jP6+vG?O**sIcY*w z13Iz?($}&R!#==D)CaLI4^bKoDJfm?+dNoUemPmZ2!0nlft$3sCYw(y;x9h2dws1& z*fg=*p$_1y&cRw>%b*Z6DLy~owgkoW1a+bGP>YX@f#f@I`7zr(3#ThB{A{0V!-ALn zKdnQ)WOaWaVh(L@Pm#vny;VsqK?Qs0LWX9Yd5styy_G?!`a;Y%Igf{$VMWd@tfycB+A^bgUBBTuOb)@iGX3A zcNrAayA5rJ^p$_){~Rl_-xoXV+J~tsi*bH#^9cGNb@uB^UP^FjyV2sv8EMlacx!4O zK)+z$<+$3lzZn;IIo={@)P!-#m$}lVb@2st2f*yZp16C9X@ypuLGso>HfYQOJxws>O6otu-VlM20+Zjs7l|E0} z-=Ydcu%p=b=6{e-PY6}#KDXCJKl;dI$LoxwxHOrfRG;oVR(JZhH<=G~7vXmdsO&}< zD#PxkIXi`%TX^u{SCjj$zMw^x*GQaXVsAEO@wFQhHC2GX_)#U@g*?yQxUv=wsXmAU z-O1Ud7vhfL<2m%aPEIGTD-}MKu@Pmo^+WPS;aD^aJaL^upxAOCsgJ%e#Rvt`=4IT` zhK4;>c{0i@hM%i2auea-8Solz;rS*|N;t2Eav4c1Fs%GBI2RKvh_1_#Q3r^ zy@HcLk^Ti z1QW4RGNI6?I zFv7H_59*dVTk>^&QInlZ%su&(N!RgkS4B{gqh@7O-jx*kPJ{i2;8fg2D4UKth3Nsc zbntk>m4V!jHY@;Y3S5k1QY_1_C?`ZEDc%w$GXHAl)w8Q_EDdSt$;Fyh=l)d4Bp|@n z#o!<|m?YIYO2zANf+EKvWH?7m_Z|YBO+@`TB~A7AP+bTj9oUf>OhVl=I3r`~6R^6W zaGLVGXO(`b4EHbOnsHo%DV%Sw2j6hiOjmqzok?_sNsFfYV4hPz1(V#BEcR|83>MX? zopzY-yjJAmXqjMQsS6z}HMz7Q%(WTo64=0gyXT|%uLp#zq;b=E{GnndmFdD&po&Yv&SO38f~9`?CwW<7Di3X zf6H|23RMB8b^Hy)x@d!0Gzx}n(I(Wf*ue0+$>8ZLoGwTDs5n%;eI;%!@2VHi6idw8 zQdE1rZD-%E-MP{TF?rRVkE8py2vAffpO5SGfPqzaMhgpmN1^f}S+v5kD1i&T2NMFC4S8v4dFxU#s8|69iv=Y-($ITNOw$EHaWF zu~{BHj^f9xY7|kowUzb`VV}a6>n@ASy|bH$QWU*;+wmxWEooQ!kY%*2E#X7gR2>(a z-h-JlT!Mb|J6b0E#Of3gG40?4?kgZPEUc?SjCGZh`$P+MXK%StIJ^EGM8S647TO)9 z7mB>$jWjogVv6(OeeumKKN6}@Xs^svk;{u^X@ngHydY8}-KHLUPGd*DaFaJQbX{fC z4N5JAluG@Hh;B1&HoD8$#UsSXaE~H3-|sJ@Y2}3|wI{~s4l2j*Idb#5YGSV&^P?I1 z-$ier@~aW>3^g(uUt=&X+`dYGqnZ1MV`SrQN`GuT4C00u)VgT^>a+Byfi}*+__JnBnQV zkLu<2y#3~2rMi%cQXhibNoYhMr>Io+!Kf+Ka7{#$Evy$x;9}#+gh(B^^}cXw`#@tI zJ+^CVL&IhvR?GuC4!9ILDrWZ5!KJfmK?ODfcmt?ES)xwrW9!Rp?s>P@egQ3h^nL~L zd8)@{FKNRAZhTIS)-*|S8vDNrqpgf9D$cH!kPBbXg>3R^M z8Er+0CrM)~$?|KSCxLAKLuNv7$!h`V{O_wKD6Aeut@OntceS?&1#lT$*oiKv-c6d3 z5zR235Udr_EE4SZ-YWY({gQH|MD|CC_})(qf+P##B(u}y?S0d(sqRa!_+`(FI_}|# z5qgm_`pK^D;QQK(7stsc7>}VgM`m3c_|neBw4zw3L(h5JT3Pz+LH6RrMv}nzfs}o1 z|D^6zc$kg*r2EEOg9UoYBT)i-Qpew?Si)cFc}@Wd{5NFS!X1=F#59|ti-Ga@fE|lK(Vvr8_$~7KFq^4qwBtSN zT6vHYG5qQZrx|*os$1EyzQfu~jyc=Bd_ti!N%oiSBlAb{OV1LN0DZ zIv!I_Pxpv6d0f)u3_0&!eI0}CMppxHeZ%!9fxSRt;4ue%OyA{6Z)MSsxUO)J|O=ORF1>%pQg{AOngg1fm1A9eub-y#Trv0^Hav`sZV`?Vh;D9eaI z)+16K_g7*Oibeqhgi4-gPt8uSdT$o2u2u(j1yNbNhZRL|fAnY;%G`t;9ZeM`cp~wO zmgggTwnC#-AE8kztWkp%jXfY`v5mNz?)>8euqB$)wPwK4q@P3~251}+esS!0+mQ;3 zN2w*pHqqCn5iPZ-Wie$)&#gVeiTDyzv=m5S7v%VksH#_D5MfJPr7(j4kI_GDubZ8M-lffpa3CtQQk<^-tSMaHbf+uv5wEZ8e4)Ug z7ypf2Z)@-6)^eiR3Yu8;FKI%zt`C6jXmwTiS3IFy_DYdH-fTC|IWq_xsat~OBj}K0 z(8;{(UEdr9l`Nb(!$@cC{bN1g@slF1Q$O7A#xA@NvMiFlradWQa#TESbKxHzqFf8y zax1qyEaJNI-~Qk*jUFHKO%$nR)|cjbjU+y1B*mOlORe8__S=VlOV}KXAEs{#L|PQM zvRdhT@PBgbXHhplGwe#?dLvdw^tb^kh%c)CL6ex)OSldm??KEC;Cb$o*|}Z#B#O*NN}&f0iYC* zYNAoxwH~XYFXThX9iiDiX*tntikML880eeJDx_Ka0Q-?T%W5YQ?|PJMaK)p?&B(cN zowbsaCb?C4Z4bgI(0gvnYOp5bTIFzBVP{$29$Y=(wPXyiYfcWbwqUrw>AqN$*E%DU zsv_B&8Bdpofd!`xPKa*fM$^6S&D<;*Z!jKqk9=6r1KbH_P+LDJ^RoP^qAR-Ex9W4h z0gQ^b=02H44eX{Z1`ECY1`Yf9DZb`g2X@+jp$ahe=Tf~UuoD9C@rEKIBBJn*C~&1Y zL(jt=vRtXR`Se!c1;5$GWU&ybHmrE;KY)Yi5UD`~&Y%h2@7^D=gBzRHKmO%p+a|i` zOqDa5*qFLyyv4^NI)FAf1N5Hrogu?{F@>1@Kcmy1Ng>^GyoJYev6FWGJ`kW#2^A97PH za_+Zd0XW$RIDC-^<^@y2pGB5fS-@0e^qBqU$1CEz(dPe<2bKW0d(3jSzT^$(`>KrlJcUNpF(&V98 z=w1SB+tU1P;(^gxG#PG2Cgc+0K`a6iMh*7=L601r3^twj?lFer_$C=6@6g1U_iKyP zV|@MZzUuQVZ51N%jW<}s9i7>04eWvdfM=)?e+7$I5ge|vuAuji9N(e*tVTin4j!>Pzt2Q9xCgt4Y1omibcRK=`h$h_V-Sx(dwO{#WV=fbG+^7$yu z9;Wqg*i7;(RVD5};jEc2kNMJ+Uxtmjn7xt=9sLP#3eCcv2} z;IudbJCsm!>YCw|kEP?N<`MYi-;d@tmts&ybt`&DTj6vu(83wFBUYx<=+7rxi>Uv5 zepB*kRVXvZUzGL|Ph>uDc!Vu#h{N@V3vmA6l&t?IH>Y{`7X^gNvexTb`^2>~?5m#B z_w=?rDZBmiYY_aWt(7W@(od%Uzi1u?x{IV8i%J7jcozo}r`*9GT?}G~7YMwgiEg zg>Ff0oD5GN^q0umb3BUz;+6`Z*0G2KACcw2O*0*Xr`}YL**^JfbR$QDr(z;qPi&OK zLT(lw#K{t30_q7Ytr8Tn7nrdE=KtS4BBD0C)OK^m3HS=wwwx4^NO*&GL)c~|w(Lbs zS6mI#yG$y#y6k+j^D3mo$Bta+ISHF(o*>6I%FOQG&4|lTY!YnN6$|WG>g1Hw6KvW z>z+-LWTl@T{x3)?2qVHq+EO)B-KzO#Vk7VBh8442w;_2l+$)0s?wXmJ>% z$g=3z7hIPi00t)8SwOcU+fBUGBRxn zW6QADlj9s96Y=aiuezL%i`IzvzkpOwfBHbO!Mcn2yccVk4@H1XVxdRWp1rtl;n?Wu zN0P_fmawXx;+oqO18uC?W9}3*5sC%qx^w4}+gHMyoP+x@QgVaLo$=i3ne zbvx)1JM#|3s%h9dIP|lAavr~wE%&6%GvI=i)oPsVSQV;TU;P3UkKXsn;I@kzbahAQ z3V5q~TZuT+l@5Q{!slDh{fblOmW}P((jld1CXl4^XNAg@U)!TiFGL=ZicR6tHmZR4RfN zh4Qq6-70+Vn#rBrojyddvpLs?)s?Ye%-|*4$^BE*!3E^dbw)?jbB;t(PtbaIh1h^I z=siibh0E4KAU}t97+YK7XC3p6$;vQJXS{7E>uznDO4c`dRIE^QockZR7aNzWF6^JS zht@uI8WIf0d=Md{n0Kx|?8VAjd4Vyr(BMsO#B{Z>Imsnz0sVURe3u^GgXS$l5R>uP#l<~`6*n{Gh?*EGgQ1}=53jBx!5&x<5LVUtuRL)aa zUY08L@5pE&Mvf8>z*c})TmDxIV$_w~#%MUzq0EGgEDVkQ;Hy?uObB!!|EBfT8 zotpIj8LFg9g6=)?ciH?v3sf8UV)nq%VwF-c?4ZL;pD}&3+jw&2W#J5$0%*lm(8cQ& z0nk>?M4K193)3!v0^3b0eb0ENk*1UMU ze<3pCVCjl^q1=MoRaIM7RvnN@sm$uH4gW{eC(+E_dlu1wzY#yTr=Y_1dSA;%JlqOs z1AHv24Gp&#gZ-p%i%j_Wywu_E298f=e3M>lzEaV@oWJ0C5vXG%&f*(gj5f<;IR`J= z&?l36tmXReX(tCY1}O%k>XR2jv@s^$TADoz%;?;my||)TO)?mrpi#NAWHr%SjYH;* zt(B9%*j^Kenz@=3jjR+5ubA|HjQ!g*`ky0nfqR>TF&PZ^Ot~MU2bU4#x^OyjAc*0*j zUz-x1u_%W_r5ek!s@@pZ!hYdl8?j~2yUa@?dLtlj<}rL-)u~_ zr*FO8z%p>CH~Gr5_@&`t_~8~khUxuP&FirRz4u{I#`A}v&!PUoJ(MZ8j>d~5Ghgts z6cK}4uvI#^Lpt*wxe#AyVJ-*Wg=V?Ei5_i_CGvz6FAt%M`WR6hW!dZ8R&EtEt(p12 z?uXAWj&;+p;&~4QQu{$oo1wDV&C#k!hOhdJF$zH&P#b@D;V?in?UHR9jF)-a`lruX z_f<(p!hG+VnMa8)u|%8;R(lykh=-ChYp6?7=Sb`9WrpCNjyw}kQn3?SkfK#NJYv`V zP0k3<;17R^e4Fr}0yE$g7WT39X9Gf^m}Q;YKlb7eo)mLPoU7c~i}^Tjs;_zl=bI|k z@T7=F4$U2v^EV6NDG#Bj02x<;rkblq?W{+2Ihs9JKt^&`;296}Uio~K9yP(RZ?hCN zT`>FQ30Jli&6%8=1flYRV7n`U8I!EDV3pu0szKD{&c!_HHQ3UpUr|Kbh~e3 zDaF(qr%FQox9fb-Apw5VTthT<>SSlfMq?7(@3(FB8vLhPPSxcp;BK1F?P(*L77A}% zZ+HVwV6Qv$s~o5QB}M0@x~DW1shql@D?3g2ZpYoCK33$%KVzB9@{Zs{7y`0W!WPtK zG&(UAr`CoPV#y?GQRmZ5#3=%Bs6I%>*j+FuQe+&OSnPG3V1H)>39p*s$`MmAvVct4 zpI;UG>_f?J=Gp`$peQ^)U7cN>=_Fj(XgbnnFt)qv}hi_IHRQ%ykM7hox ziu=JhrxXyb3T;2UypT5Z=gHaG+?2M^GMq0Z?oDrrVxS3}0G!lG=uJF6Q<6B@w)WNx zJ@$R!FyJeYx6$~OEn-RIRh!gD;B^OwI-tD)4^VCe>h=MG1FuQ~T<&3qmoRD6mHHeq zIAk|ffyB_?;#F$CoauRCgC>SfNxMM;%1vI^4pUCGU$-9C!&Gcg)s!f7OeFIn@_gDW zJ!M31@vh&!Q$*(i0b{xvc6L<3scItx%Cr&lrvt5J^`_iLaqn4IfZnF-?7TI?K(38xn9of zeQD?Qd9x__=74E&~EIH`OO}Nm)^VGQNFJOmnQ<;E% zxENOa>uy`r@+HsDCtnu7x9_*f;dnY4z79JV^$oTzB2?SL$1#dk$Bv!q(;?RZPyLAn zW+rp9`Ka5e&E8Rt(0_<`_$w3@vK~pO&AjX}^@0~5 zlKY*pQ}AQI+Sk@(m0ke=u*(H`aEvX}-wu;}Hkwu_?UDrz3oLhu%2+*6>mn)W#KCN6 zgD6WFa+|Q#PyB3YGgp0$d!W&Ph%p8Lux%cMIeAsY4P%64* zr)!8~kv~OEFINMAO3KxQwL3f&hTc*{{R@d@%FdGO`tf6b8DH}qkMYvHD22c0Ll6@F zGSmA*Hf4B*9rvM6m-HaI7(4o;v+ZOAxG6gtYD%o|ll7G#yXizJVC(>2e$bcVOW>V< z{^8?XP?mq01fw1|MMmeKNPZ=QP}0aU->P(27b9uve6?{m^G}}F!k2E@>(@Bl<3}d? zngrifzm3;qVwcCU2HoS`J{b4}c6%?OWNt1{!vRAP&oJX3-|=$C91bpZBwD@frM~9J zgpQeS@w1uQdP9zg)6!?=zRs~7CpX^v>{Ts=HQ?by0cE#a2oD3bG1CX_d$kTPnQdf2 zdsyVFxZ5gqI;;Ma9UPU>VIl$&59lmm@N}Nc-c5K_00Qx(-x_+5QWlduP8ns>^YB`v zjQC877iv{+6MZiCYuE=+(X46;9heNU>^~UnH|V&W3U&)Sd`!(@t9C^ zE#S?|GKHdtfNyDwG}v^4zjp%Inuwlq{{n)-R==(7?HI2Z!flfmv|^{X4seQzf3yn~ zbK8Xo@VLG7V{?V~#r#x^ZwNNmULS#9y}529ZOlP4Z@fp&WY^`^*)v7Sd=1(-)lHqF zln{%uV3y6=y(hU~p^4llmP_)P1cw7s^8ru#^qmy+UDSZJ&0ZIWA;U*vDd@_mI?dU@ zoiK$K^#PU<33T*}b%!yPDB?~8^ zx4M|3W=Yw-C&2gFfSUWv66}Uc@NjU+I%?lr=aj@$2)?&6DPN8lW(l;I2UnECU`joX zLO?`C$t~sq4;z`$Ac^sdKFXg~;vaH|$E_EixQHohHQ!;+u!UtlJXs?pv2PVg-cFzO zb5!$^Pwb%o;B`j4U}Ys;icPP09;ZmwDo;**>987~&bCB^W~CA0^~gBu@rjXNyrek+ z!%dKqM4?eZ3h6kFIZlNF>5`5VdftJRLlhST%<$=qlE^s8R@Z7H-gCY)MrtHivL>`1 zK!0Dpi`RJBk<)@en(qURm;<} z;L~R)AW3VVcK@?w8uqv*d*|?jLlG9AM!`dERlL5U`3Dx)YF|Bht_ufw6PrSR&J+lQ$OmNP-sk%Yh*B#p|_Al?-*Q;)!my2_!?MD08s4SA2 zQq_8~LPm(5hh+*D;CRQO?jYZb<50gHXvv_}oNOMtX}JZswh`hdD;SihjV!jQ5xs(`PQU(J z{^sT&uz2Xz{~UQrQ&i#5?*k5W`Gz4ziQat>=02?mGGz_OIy5B2VreA2p~QRngqObg8i9FON zi&h+NRWYxslz7p)K}$QID+4tl^A`mYyYd^|3Trj43|u2XU#^m&F-zSq!QUbsgsL8= zg<#L^)tY!RGm?v5@!uK^)VI=2=I_<;(u<*-c5aZyC_yudW!jSvEe#irjm5%Soqg1y zU2FyHwd3`aHfA<+*_#OVLJFM`T~5yX-6ff;9KK9OGzsGcCh9Q+|n2$)7 z$0FkiU?g|oY_j6M1BGqudcBQjRa>!kFUj10cj^i{pW%+!HQov@jAd356h{@f*z>bV zhTAO)tw0SMY`YYT!_gsEWcR4|ol@=`V!>hjfHpcLjL>q{A#1)vo4rQT0~B!RYV#v= zQV}GsyDTWtIkT1qMkc(JU-P{Y3=pzk*npQzqxk7$G+bUWd7i98#}ZNlip4j6&$Txi z=;Zal7xs)5kl}&G>^4~PYi-}c;lj_qot=bcc&zkK{BVWtRP~m2d24s3c(rS?;^(;u z4;_MyR8ohWM{39yoJ!&g>GckB%vU&`IPlB5>97k^C6 zF0?lsT~y~C<7t9)#Cu zlFd2RNfq`_&^u?yt<-d zy0H;k6a&F7dk8lNeq0wyqWvr=uJsi&YxAon_3;id9*>^@RGj%#;>KX@6W?trVR2s$ zo|zO(7Kc|Z)+8;;95P?2%3e(9OmFHUgjEkl@qiCM*FK4*S)s9R=peY(X1%AyJS{`}0BB+?o*gnHq*Qqm{R1^}zk( zKbsF18Jn&`kQ1Q)bF`cb5oHI+VsyPvWnzUOQTD%XrX4IKsc8Rd@qt@)xzTsE4}i1Y zthBJjT2#JJR@OC#&?0WHa3M}F4&%WPGdYa(4ZoPg>Qv}3692192j^bPUcnj14Ppul zB(|^n=JV7vs!j8>L#^rMgoG&m)VYqhG8LcU@zhk}a>?)&^ZvK~eslX145Qdn4IfkVzuCJ z*Eno9;4%<<{E)JxO2)?VB}y3KCEWJXLYuQwad9xV_z+sobZvr(@k5RFsg%G%s-=R} z`?i_0hCri&+a4MWQBT3lPZ-f?{WB3Ga+S+JK}RI#kCtmrh2Uq-l&ulAI@+{#foPS* zk?;3<8!+1MQs^r4-s0yLVf4kg6>RoJ`I18qTkz&aBP|+n`(3qKi9BATl(MH^*Zw$z z9j>%?tyx|0JX(4!PAF1<_bT<69ag!(LSkHo|ArUsWbF7|#4PSZYIBcnW;@bcwa$YF z7K7OmsU;goFgq9jaYfqDdT)C@49auQtvrFiU>zN;-%tr$)724@sw4(HLY)Oe<%O~S z)(c_TAKY)Z-x^4{@m8^*^!`V{gUbR zvz~6ppA)-WY+c5Ta=izkDcFdWi*Q`=XTiFW_$e`de4JmaA5yX0XtpqCGm>#E)7cVk z=(|7DFBN@wrP|$<3<6Ot^wctZ>2p7lo{w_P%sTw;QEVxAA#OS){ET$9IB=wQiBIF? zDC2+DePYU-U%nX8AjzsCp_+&fX6C;4(Lu(47T|MB=)4k+6;u$9zWiO0mbJq|Ni<#6N! zL&s{M`eDmYS?_nMTKwy5duvW*ShdLeJ%3(L+%6b9$VX|tZk|)JSfwWPmj!IDi9X^A z%*^XVpRC2S+M8UI>idf?B&Q)yjYl7U3)wHg8ndl9nBzwAtfHSk?G7+zD_KsAFexTe zfl{8V)uwnX8osQaIX|5DE@=qEu4AgXZuOPsLk>g(O*A~YvGe?onS+tW?13HhD!7ra zW$D$qY{>Hy$zS$$_do1w)-L5$G(CbXgV8akT&7df&E<}GNN;L&d*=J;cJIH8L@Cni zFW?%t(a5QKAF&0|^j#(`_q)~b+H0Hn8p0a{+Vwa$J>fbTMT7h>om^o z(1H&F_S}L6GQ`~?-!p!5k{-NM)e@`zX|KpkV~?GR9_oRtALlR0vrf-w%f$?Y{jxeS!;-oJs!@bYVjBKeVIriJL&XL zeAV06A{rAym`*hx<#j8sWw0DMd`VE_o#&qh<1QfCBfjhu{&F?KGg%_yAfiYwGzGVi zS=Pr6KzeoQn-c=nMugI-V2C^Vl>{2t!HCPQi~biI#|3P&x5JSE9F0#us0TSU933o> z<+e+e$fC7-`efK)gQo)Q42P4O)}`0_r=gC5wWb0H%!T` zZRefT%xnI6e?s)t1Qrg@_lmK)MiV)HakCzl_3+1v_Qp678VJ(Nptg_Y_E^2K1IJxm zc>G*0=*Xh98+7d&9?(oBz4LU15O}fTfScc7DFjRQCidFs?0n^E!K^7zsp+3&sMIKu zPlGAAzj(2;zrW3Fba~KC;q*44#la!w1rwcX1A0ftnoZKK{b;lG<2U%hYtpRfO*|9O zujk*r_*|rG5ID3}2v?*3+k`l*d}*M?tEEe>+NkXE|L}AC zPuCUyt>nLa@qdMKRDS(q!2b-KERjXOyeHZY)ilM1>>|Gl^De#QWs-09+ZmPQBR3Xd=XcV`(*8ZUm5+2ryGGV zigJq@(s|WVjgk}`S!s5u7cXAijjiAH`vpHgp1H@VFi4@pO+i=38>{+h_O z;og9O$toj}X#s|0@V#3ruVd^6gekk4E`aM0%26Rv`1hP$U0|n9r~*q)1IPDS$Y?ie z=dzM=QS@{fKh);GZM7bkuUG@K@>eAc)#S+ky6VZ7=3C=(W)`-Ld2Q>V;|N%2o2U83 z``h>^Ulg|8eqB73y~)hm4Cyk?zNMCWzQNbvto!n~X&3)VUZnD)T?aUxf+oyP4XIbX zmJ89o!A9b@!1%Go_oqJkaw&o=1v^x%L*-^q(@+kxgms!s}jCA{32Wiz)N8U4lF0T?squA2aXPvsB*7-L3&ieva(E zXt_8cN}!O<`tj&yBaIro)Aj@hZc8InP$>R{wh%;nCXx?73QRF$uSWd5OcB zU^-XLw|4~_2WQFftzLY-1P~-1WZKCKBfuk?@h`L(U;x{@#=_)dM7spfs{~+b`#T4l zDHu)rr5Q?ITRrUp=Sv5%@J@X8wHRQ2rq#as!{)vZ^GIO*yHTav%d4-RUqz!;F>8|; zguJ=4L>MBja(NkVvSu6 z8aCs7^8-Gj2@hY+>;UTBh|KvF3rO8#)veynQv$-HsCyj**pnO>zI^pLa%8uWXLA@D zJ#L$L zEXi-Gu|Moe{wZOvqs`5zU;XOuxVCFnE$KXD*=a2W2aaFI13vEc!;>+mO(n)N~DG!JF`mfL=|nTZHAa; zzf=T@YrD|?=7_x*suCcCkHD%@!c6m}-Sx|(W$vsFbDlMS!n5rA^1BjrzEaIDol~Js z-;Ym{a~xSJ4@r#YFM5AcP>Q*GA6vG|urZ^N+avlzdA`)K~YA+ghHB{6$!PEu(TS3vY_M(cVmab;s>y>lch|V zn?S<#M#WsHT`B;OaQjtmqEXD*h^!pA5BZY+yj}NZ-rh*IQ2@T$SVL&*PdWgKyZ^-k zoPDXY;IQl*PA4+@O1Kw|PJJ?_dxGIOSBzhm9F zZY;Xut;=ES{>CFyQBiDET!u5HgH{3#xru7_Xm-9=adN$0AKWzhu9!v0VvTT*2BBsZ z<>-aG@cURetN zp-@ZaTaG`sqJfW0!t_fi)6#?j2;CO8&z7wcQUi^+Sjj$!&gVs)VGT(5v5U7651|FF zj&@5$%x9trgmJZo`SVb~%`-jWk9O_GR zKYgt=O=W)7%%fc3Zry}dD3Y-vkfO}TeSfOZLJ-aQ1+h2ft|yb9k1YOF|90!VTg)4I zzHV`x5zmR zT4FTNKerP0t+RrTSesE#pHUFJ6M}a1`SrPQyu?jypeEzoxiuJvyL!%M@T;7e_ZAMt zMsJ-LlO8a4b`@E2$uP<7Q1X#y8Z!)pVKG zCUV?{WAf|CFS$wo&$of)r!wbPW~}CF5Z#;hp_)*0EOeT=Ds#id;MIKBh0&tHz?AF1;D>b9BSo zGsrzVF7WoNxEHg6tptc7PQ-N~Z6M`J^aCPJA!Oid2^OhvB%Ir$YuX!LjqWgMUsEFN z`SIJp$}E$Q#mato+s(#ObM%X{rwPK2s7c0q0e=a{9?c;7x}_J}Z}SFI1;0Jsu=gQ?R^6faF zD}E)01^k=IswZw|7F%V=i~DPSiEI>tbi!QUjby=P)bV_H?GuVxLBeV8oQkV0BsQA! zlk1pfjD19(Nz)=pXuD**lWgLTbs`x!7|%O*$t-ay#;rQ2O*EF!^Ox93x`VTLybq$J zml%G1^=iEy&N;I>h|dzWWW=9;;+wwSbV}Y1Uo$+bA9QWB(60}aPiu`_E}b(Oyrp2* z&C!lPpx6xC5ZMOfe!&SYAnZ*5(9V_suA6U`BY^{$pvdl8MAEBKX z2n2OxP@EEF3O#ys8S$BI4h~=WY^D|LkLQXfpVe{4O|?p%OL1JTEqJ1w$(ezdd4?7| z@a%S%c)gA3>*&r3jgs)o1#Ob2WX6LMEzAdF{e{XY3WzPt)C0$c>buRmG1IDfiJO_UGxzMB;?yV;ptG<<*M zCmN*c`Y*OfeX`1ThY0Lwe|l_ZxEXKi2YcTX{3h0dwlA;muZ`Y*$$9Hk)iriVn00sb zzBKBOQ04fWKRocsR)acc50uriNre zrf%a6lQpZNoH+RUeCX{~t5|QiHkzR`jh$8zwNkrI9dlYIM|+R>?OMSfZnz|<6|2epd<+mlg%&V4=-zrGZ+e*)3f z()guBomF_;1_g(Cj_9)P1AU#P!GFufCJNsXXknyK|TaY^G%q_-N&ee#8{=>9)s zy=7Qj&$2xX!QFxd3&GvpA;C4cySqCC4{pKT-QC^Y-DPlRaQIKoeb2eSbKm|j%;p)G z-Mzc2tJkVtl`cG%tkz4Cx7>NJhR3l=Q_W|Iv1Mxgtx}T+)WN^r9rT#Q-93MRt`y4~ z$GB~J61zesR$!*y%`d?qG$Y*xBQL~@EzukmS>##=+q{{wZ2X?-Qsuc&yR#jPEKfAq zuiZ~f+Ka4~vrX0j?zum|C>GN77M$Sz#~A+g#vwrgbeG&ex%R}#BE2s<#Nt^%9)R!C zWo(Tm{bmbHpzXn753cIm@$z5Jp&5u(_SVS|vHGn__bL7KVeGhQ;u@%cgZmM=a z(MxY)4pc8|TZlH*iJ(|etOOL+aHrEq3JHPp-*Wg|6K?x{w7qgg^8c*mA{;|Pvj!A3(e@@`<-{}>6U($La zfoazsK8M|Y9U!PhewJ8UK~h^W?6k7ZA@9w^$K}Iqg~A-PzT?wLOPFM>>2~abdL*l~ zsS|6V-vC}rs1gEKh0JSn`Ui9g7ahS}>Bk$S>0iiPYOc4_uDN+lJPJpA+r zr52y0y$sdI+dtC#Vi8;e8fbN3 zq7Bune;4bopy>{>SkUC3vme1dIL9mOb8&cl=3ZZ7$_yDTV6rkKB<~s0HEmUky~D?_$&lE>)+P5;{Z!ZV`TggJ zYmLv2!42Vw_>?BGEpV*dHDjIa*VEQ#k05L+1b?BoKwlOv>PQ{?s@Tdc#cyPXOS&D) zqX|QGTiUZZ!@B({mAqZ=Cv?Y*|9GgCI9Pk!$oQ0ltrc1^TRyi24I65+`dW-+phcr| zYSd`BPMgZMN-zss=dZ~$f{dD%ISkYrFpzDLcXA)$YjTXdSTSZuW52z)>|YZGxZ_{F zyYlJ!q>7J6d~Q60!7pl_j~_YH4}kTd2?ejZBQIJTE2om~zp_#92XI~ARkWl$2z+4C zJ%nocAfLU2W^0y$Z?ANrD9<*>CG>)B8CeWoXCo1>^llzK~VFp6nC+dD#-fkD@Js) zx1&EvCQvJb`da-H^phHll9KYPfB=!|Q>qxsrkjZQxV5DeZFAYoZ+lyC&~g%TqVK#6 zT66dxYMAK&nV?-rQvCGVX$DYjvN0ljQx`;<{h{;4Skkrq@o{N)&|3JX%V5;cTfqMD zD(5eL^6y4(_&=Bs4}I}D2od6{VyEqqIer+0U0noKUI-Lv$&WalDkHKUa1}z+NhwG2 zWZ8?1O0MI^$f^gg8jziz%RBy}-`C(p8&Fs=PzD(}$GrtmncR39zaMB8;7~4g5L&wI z-3I6)LzHIQkEC#U@6J@n{Pnw!tEXUMW#*uL6>SI#(JqB4Jy-$AZhL2jvkE|?5r-`F z?_2!7KZS-PeA-Vkf=jNRMS495BQ?UUCr`$lr~_BCuD~9cY>F{5aWYaqQJQa`4o0IN zD^p^?G|YXIIwf-wNS`lDInw>TQd%Rjtsh@v5=`yU{$^$$@t3#Fxi;xH-%L;#bLSl7 z^S>GDS|zkS53{*BT@Hg$EpHMy*Mwz_T=thE#v5WJ$M@^3wO)dHe5sT5>vxQ z2aoKK>R?i+GgUqk9-`#Ox?M6MYe-<_u>oo^u@cX@JG^#cQiJqkqyrOA>!q#c#Tr%^sLhxE(s2APx5=iIL3cpR4!p^h~;Z=!8$;4o~FHbQc(n zyH_R^U(Y=;7>j4O+2I?Fw;{`w;j8L`!;;PE_g)d%jZElOuP7&KMp}k@FJ? zqQ>%xT{(#aXbZGnLqX3}@4@PrDMBC!uC(??X9!oBuvL3D)!Iy`@E%yAZhSZgHzM}< zqhuZT!Kjt87FZS)LO!j#$u+Pz^FJa#aTvjcoyj)Ai0E}8)HOp1-t<^LSD}~#@kMy% zfAO_l=@m%sZ6cnp{mJ&%_NsH^$3yu%xzK2wPs*T|_p*OD?uDVtFW<;dfZnFKbhQgt zjXqzE1R>fnvW#t55_dUxA0i~*2P#UiGM@#8p+~{IBQxmIl0?nU0m?=)9=a4__4490 zd|@51p*nN}VKjfy#ecNeu^2y4_7HS8F*eV7XS^KK#@Xtg?}ml*Rab`Q2sfA#B5`@H zA6qlOGmXFNkjlCmTp6A}-Lz~%^;gYteRZ|oWQ;DEF)q7YwH($L2=O(Wh@O0P~ zd#P8siVS>IW?sJ4pm!|kc{+?Z zr~L}$p+P7@q3yYct;3cO<&2rkJy3U(D}J;eZRTJwo_SXM(?CgndDvjJtd1j>iGeMc zjp@0qPS2a*MS;hDxI4jdeL-bKBaq~L`c~EA+I9`5cCH3t4Y|-c)|D`l1LDo@igMh1 z*GCt<1y@q%D{{SM!}@+UV9dccO>jNmF94+R(8oaOf?D``2Y(s&XBl#-XsdHa;u^>% zqq8B$W9&*#SN>1`!bAT#JseGUHvn?5)zI9<22FxMIL}`=F4U@bRpMzj96Fcuwd~wE zKB)Q)7N{;#lvuj-cuN%rdUWx{8qOtnhs5RR)f@pxJmHItSN79}gl|pesqg2Kl?~ku z&?;pUzQroMES+Hj#40ccwv@HZ7tXstEZZ*hDY;1lGKX}VgC*X5E{&YNJA&gnRxM3X z0jh_Pkse(+voXM)-i{0Ht55z-h#xo($K!{T9trgLOBDDJn@NG!4?L{(^b(Vh&)n4s zOe&qkm0OOH5 zl7MmuYglSLVWT?HkHd?Wr&}>Is201UC;PelzKg6cn?Ghx216$Y#(Zc_!0Nhhr`~O zD}m9D|3)S~t;=8~mBM|$Gj*A6%r2=yD#R3KuCWs7D&~GhP1HL^L@i0Ee*-hH0|-m5 zE0FSlz&%^ENbj<8NoKT%$}MA$wgCt{HK`aNK>e;%v{@es+w%o%GsE)9%>uoI{FhLH z2}!a*DU~vcVHKM9E=Vt=ynO_K%Nl!#wB+isibeDseJl}~5^Ve=kwfzpI zmhR|7Jih`4(O*B3n;qXY+x2RCHXHKkwM9?3DH0`#m8fmE=ltKFYSiE?(WSJUU=MvA zuXGG4N{3)qHL#hlvQRMAe2E*Azqh^X0m+^GVZO$2 zfYh=NG}*ShaE)3F;cD>&>G9Yvxsx=|_UYvg_d-r)fWX3bMhn)5vwHc0osPhjo+k-k zl>!s(K=8D)&2+!;FOUfxN#OCD#J%PPgLKe3-SFL9w0{7e%K#uav~n&6?w967Wk-Py zSo>%9COW;zxI{#2%;FV^)DzQx`+O?81c&s*c&k!PNrzq4KowimGFiJE zVqPH>K9m~w(!#UK5tQNb;F)jZXb3miW4;-Xmk!lTsV=U{s{DV~t3=xe z=0Smt`0FgIZug_%GV;y#ihS~DvYfX)L`^Ltn#e7e8oD`kH+y#mL^!j+^TY+zeKzGW_A<70X;K@7y`XYopmhMG zAM!QS`67H7RK=%TW%I&?h1{(isxki#&cUJ#Ohs)|55WpLyL{iIQBlva=C2IZnP~` zI`CgZeJXpyoT@0WpdgpC7#YmTNNJx$z~WJO+tL>tX3jk43yt)X!R0RDIQ%mip;#YEeR=E&by>4-enkpBAhoi{| z-Msf|vIxCw@n7ri&?=gi@{P`!>eMr=-4L|Hn4|vbtV2oszO9mII0XY$!do z0s+b?jz2mL8cn=cwo0+N|pIFQCZZEDqaZUsGLqihlRj=)x_ z{hu)BFUj}lqt;zMKzK4Z=%C8QQDWOUKJ;Xb!vk={8h4McROWk)IX#HrIenD%-9WL7 zdt#&rv@tY^=e<;nBMb_4_%;DXR!^;>0X1jRThPUipmEGXGA9)yk(|3ycc$rGtq_zR3r)fT-H zMq^CdnDQ8J$mE-qE+mIPr;>Wqa*LWyxvrzbh^UsQknf6yQQgASHYy4TNwh#;zw za`gmBH>1yVnlELR=~6$C`euHGwDArYPLLG4mm%%Ngr_5I)$Ru@CD&5o-D>&Y+w8vW zbUZrQvg>r)oARWpJ|?FF%uFI_S8nSpcQ_f4FKf*D4MmUO z0MB9dx1f(LaAXVq^$d!L09y69NSdyrP;&R%-KUrKZwyeJX6cVu42!aRV;>&di!83n z&(F+iz#=BvN|L&!taN+*`Q!fP8999^0wJv?*5a)U@Q$paw?t{t3Oh_tdA6S7&D6|{ zG`$1-!~Z%5JsHl%8;cvyqi_&iA$@TJX>0pDCs6T{)b?f2jjfIYcd|ayr4|^1Zmv0c zWrW9PC^IO)7pH7f&4D*rUrD}~l^I9@)NOYfqIZBJCLQ)O{?Is-P8MI>2z3fC_d~nk zhKw>W9}Y(>|07++A>+}p2-oR~_E{x9+kvq2$qf8rEe#D<1@{K)YVfLtc2nP=qXX#; z+NntY$F@B3hs2kWdQ4d_GCrXsI1p_FwG0|k3w`I`r$9syN}dI52ca{8VxnjuHmx1B z>tk!H9S6vDHW{5&L@W12O{@{ZnSq^ zM-`JgK8ol|eyLnU+Fyy9-4GDB$cTLy`K2yM~Uum5z{6 zEOKeX#|hzj-MhSvfb?C*Iy#yp6suY>sizw{9H< z(ms0D9{EPP>rb3X>~gM^JJ~)jNhvM;A3T;XjT8BF?=kk?2F2AG0%tMeiC@%~(+eO{#7`Yc3nmJOubeC%+miv#&YUN@%_ki>z^QzCSqW zF-~9<#nw91bxn~;@Df_2D+&yx)B5ANny#l;y`3Kz-ENEyjN_*W?oH3tn?c@A_g6(v zs-6tL)L=;*unFfKGhJZ(?7H8Y8gML};V1;!1MrV|>Id`=y)-#5EoEQvZ1b+KdVOXE z^52ekxxRrI?c^S5DK6$j-cHS1C`WSsDG}$W4_qpjh{jUhd5V(Zt@#0Iyy=TT(>c+J zAsrKq9j>)5?eXbDVgOd#&e!(358DS<&=c!4<_q7F%e4(J9U4+vXTr+kBdI5to1MCI zj2YS$LVJfd1~4}L+h;h0mMs6t31?OZ9T8EEYOgwWj%#2`>a@T4PmKVAw5Hn2CiFWh-g0J_T(z(IB#b9<3I5+Ze&wRi%z+cihnd6l%uswo4(@38~Z4n6a>Ga1hc3iD?eL&aq7fW{0Z z2?WH$oE%4-2eqr3CD6B)ie1Uy!}f|uVI(~ga{(AB_ZDA{xeW06zTuk+Jmog1zIs|K z*$y=~4NmhOEiXi!pw$!chi6-^#WAvgu#5o%Us+8^=fsBa3Bt|&7ZehnQOM*4+Nlfn(39IX zctddTrrmdGp;_r~SX>>gFHYHm<%jgSk)(;IAW4ehw9$DSboGaHri8|~rikxAlrS7i z%4XitmIq9Z>(KPxg<0>`nfOwux!Y|x*PvFO+Bv3xRL7$us95sz_^Okv9upW~Lj1{y^b2dy`*aj+8$SeZ45LG`_a!NK(TG zSx(Sj!ZNf9*#N{7fJpVfR38r0MMa+OLhlKqC=SMRU=xZ_Z0gDxI4ClYuHOf zoZ|Z3KhG)^>ydE2%05oj)msxzbWeG$Kdc);vvPe52;@0HqOj5Sxn6cnwlA#YGk9d^ zi|88Kw1)TnvUkMDJi!ez)j_{y&r;*t8{fP3;Ry}&9b~d{0VxAb7>%4gsNERiS}3D* ziihhtD6jD+H1hRUi`YDd1q?idH^w$78P4jz@QP_P1~ZI zv%CikjE^r;>Aw^bhFV|Yl%?N|7M}@@arbnO9!$pdrZHyTxu{=1_{EG^cM_)b<33r= zxxYGjeCNEf(Va?sE8?GMp!EAK|50auR)%FwQTf0y&NKJuP*=2kj`^a4hy2C$_*@Q= z^Uuf`|IGIvL$|WQaLdh?p)P?uV2Goe_|ocNH_2;zHo1h}QBzt?TJxr1X=&31mii8Z z(dO+d05pS4x2P4|APK@8I=Ae1xC9ED$U?a;9RHntqij$P8`0$vk**J+l<2?=Fcgs7 zF#?IdzEor91Hxd zWE4chQzjFs=WgLVO#si!e{&=6lHe{EzX>KIrc~p!>DuVsg1Qn z*R`Yivona4@EN6%o99!xX6+s&h$dw=2Lj4p*+GAxQl|&}4BgM(tCP=UnV}Xu`JA4x zWfYfrO&|r3&X;Q5*wX#~(+wt27%+1B#(&9^muv7AIpFc77 zF=s|dtI(&8(P&3S4*e`4GfaAVAzD2lG(az6^j&ey_kH7nGx|?ZVXl~BU3a_Q^TK?dx%)^!Q+r{M zz=Y-T%zU00lj5kb$PX$c*3UB7nAoi9?bVvyV3|I?`%$Ip|{ z=F4*iDKnS28ugEOXfu`)RcNEz=BK~q)_aFh#TfgPF)b4lXriVd*ck9UGyRA99`saZ z1_T8|T#n@WZ_G!Xm%+c= zP7X4#Mu?)c2$NM4y(R})y2K{>1XeK3l*>MVm=>sDo^iT8jW@J*?b?yiVY+igk5|b> zD*=B32q6XA=|(pUge6jkzQ_7B!st(7922cFMj(reW%j}Je!Yu>5nFdWF=mm#+cG^2 z`+f%Iwg-6l%`Q=d0~?lHTG02=<4_SH^1w_6Ir>LdA|f<@Zkw#XKuvt(nZB-z5rd;5 z150UtE*#2}z?iJPl1B5!&T%u$(P3kjAs9Y5F;-f$E4p7vjQV3fum6K0*r&&s9Cz%# z3L8AfcvKR9eIO3ax9z~F*}yIC*Yhg8`RS(V3ef21F2B)^iM9U{>aw4Sj^xN_6D|Xk ztpLVqQBA^>D!s`mF^l=Nlv=Y20=0Z9uavliJFF|l`Id&*s}Ku?52xhoGmdB@$F9p> z=I)XYB0^&#m3_^vC>aO!bYc8yYl(@AS8W^{jmb4InaXO}H}Fq0ih5^XZpC)4U;*>n z8DlV+W;}tR(N5cj{WyVIVPJeXjg3H-(aXKcW%Tp(CS%$>|MFyuw#K^3* zBe3w55>H7WAr~RK-zN+XD^`AtzB|)u)vCW}9xg>S${}ffnyk`7G@675QY|VvUB;HWxON4+>1^^Exz_IfbA} z7sg3WIvbn?0kPaKFU4Ey)x zb*AMAHo_yTz_4v*9#$R3r49hYqf^L!=WB~*8}jrgi_^tBg1&DvJ7IG47=c>JGol(z z@v-ZxnBMm!s`1DRsZ$R-?rWOeu8HsN?MQq>llw$D8k&{S4<;JpZoO%b-N&}e@u>mt z*QU7Kp5%OZ$JsdWxAj^@{1wnlS3yJD4(u%yF1Oam+nc`@Wu%@9@`R@6cuzhhyJGl~ z!c@8BrSP00^?thB&p$hS8RPc))OH<>yLT+#6r(7S_M3x;{J;Sjqe-8RM^0@lsm$&Q z9#>}E(-w7g7-qJMnEx1O-bQ@6I;g*W<_iNhWqavQHP~&DGGb5%1cSzJxjwhPpyF@^ zHMOz*8F>)MEdyJ+AfUPM(4yHec=Hr&0N7pJsYaaMk%CWr;d$qim>)I79KLv2B3Eql zb-;G4O#wM0f;J$z!L}W@%el;Krd#7cHY;lG6Xp z0sZjF!BH9BM+GE1qgtsk!dtbgvVrGKW~1$JkoX6$v(5>0HfWfU!==408IorE_SD%% zR!neT>wQIR3UCX}Om9PUJV57Wxq$xZqCQP{J-DpQ;T%EL-VgZMTM6;QHjup~bRukH zAUnZ6xdl?Sfg-+1L%!-TNmS`1>u5~#Tn0x=@a=#aTNb~^!s;wpT&1Q^Q(VA(XZ5Ap z$u~&Nhtq#yDbyKUp3xLoMEXn$)^puGm0Z^eVx64sV5pPIWY;n=>&K71Q zyEcduZe0wYoovQq1+Lb{XchSsZ2Uv(biS_`raCsoe4*3sbD7%Ql)+G7r3c@a1?6QA zA$x=N+8y#Nw$gL>pkKK-Ot|-U(qu=GIETpXDx29FZ;a4WRv`^n&s6GAT{=wVsVYBr zg$F9BkGukoS)R5alY^P;MDZ8PJOFD>_CDIDJw=M?Bq&$}e`Rx8P}wgcJw9sR{@CQ54xSe=AXsYSx~+juC;kYT1Rb?E9$BsJrlI9Ij{k^EKMl|N2}&}St(At-S~w)DQcl2}+EvZxSd zD~{dlWh-&$mzLZ=g&okc4>eh(9nhvw<9x2LN=Ek$%wJ)M{Zo|Nieq*;&h?WpXNbq~ z>Q<>?8x5Opa9e0RBSEUa8|SB$NnB^sY~RwapAQnRmQ?wsC)X%mKM|~cf(ku3V3jsG zRz{nPrYMNa<&B^45kGqBi?~H;C9>bcQ$|Ae1^*9%3X*zCu zdVX*jIn5#{_^EY=^JcNzy3%5(lwloxD$sTZ+0Gp7BlEeMtk&dSww!uM{MnzmaVNNyxnYVk(-}U36n*R9CcV>ErNFhiiUp@ z?j08qLTo$Ds^T+DUmxW`&$ARgzF!#(POJyt$8OR-n(_KAd@@@S9|F=5)pVMR3Cr+* z&U}w6xaB`|B0Tc1#KYgfaKfFA6w@X)1@L^AS?)jP4TA0f)EHUeeQ`RLT3)n$T1v1? zalTj2LVb!R!&mi&&8mr&ICo+NNe*Pm3~|17O`6f!NHmxjxca7wS)MFg&dt@ki`*sH z+-_^x6K7{PQ?7r_y0Lz9y1_&KxTDoSPpfe~;3#g$B(o4?nH7j4zNlZ9aSAq?Zbs1> zAEBqyjjN}~jMDaGF}`5t4t^*!bhE{(e644-Gu$l(ANO_TrHCf7FSXXL6E7T)E)aXW ziMKs@A|W*`ZB~Am{#HXx<@S<#Men|?42bN}CVLPAWM+046P>`!t#1hb3_DmN!U*GV z_OjGTqrEE+CB$24fd@ycOX1;vU5&q@_c)pNhFog5vQlN1;7N?s+!%cPL;56-IA2mA zZ^wnF`NLv7Nx|)jc4dI@@yUbpIbRPEw}04=!2K|#YsZ#x##EseX!LPG=5)h0>vO^1 zsW}_YUD?D**oH^$O9BPMZ!(aoou8q)vvc^Un*aS5DrR=sRsykiK!c?MpTy{hm68bK zh9_qx@0LKq)%GFpN+Gu8EO>;|#f%^K1?PLS`gTb^o;tX!V*LO-n(1W{}@%^I%O|H?bKYj<}akccmY6#C3RjKUN zB+QntHt}wEq+7H;{5LTfilE%sWX|ITsd1X7GD>|k5+++!HtSyGgh?J)_W(pd$S2vz zq%gX^GdGQmZiJ1tohq1tCW1E|s*(Ea?6M2L$s&TPKYuue!;F<)M0s(S)2~us?Ato4 z8dhg)e;c9;7!u^Ivh9xiGR0{fMWlS};thVExTnN4tNV7ExDxgY%^qKRbzpWR0QROn zrrzWegs6{ns*I)M@Yjm0lOA%jN{Uuct8WK_OQ~fVtlAJs~_RS=h;1MmeMUVcFxggO8#><@9zu| zCuob~FU8|H2?Vexh{_E_2)FQ5lyHc{bveduiqoDR+wjHi76YBLG@@EvxH4+tprQCY z7~yu#S56#ppF<7gVlamK}?bU&cO%Gw=`iL6|Rx`(@2O z0jm>#E|z5yHE1h2&|5rw_wpDwI9R{F`*^8imG9cwvRp#7hZ0|Q|HQO!mQMBJitqoeM=6nAg^<|#kP7j`_wZq#OWjC&yr4t%X$H!ct}q( zky=CNLI>LG!C7ERb_7R)FE^^c@R^Y*{B>J8eS@!-$1hD2xe8_z!9x@`lP{a|>Me;3 zKzyN!1^<|+*gId1p7tZKl_zB|DDAOM#A9<4{cn&+xZlA_#>E@u-RfL{%JK-x>xi2; z7GEfbYYaY|A)!SDFrIrcjCAO`rZMHIsU9DE;(aJg>h^L&Rdajo#@w><71^!GQBvF#+Rh)`has679Q56Gcog4MkZSGWF4C z!MY$>@w4jeTs_2#9}kBuUNy{#ta$PrTzucM2AnU&%DK#bOh_A@f+rkTMdVDO)QOjV z8H{Pt8M6H_Q;|c?dp~1_*~S=L_m%nnluaRGNOzQq)b@0pUZY=)nT=u=k(DG&J~dtC z%lMqtvmlSbfnCplF1%jy9H8cGH;sg~EUwO+U<_j=k3aOtE;^W`=>sm+2=4x0iTzjCPJt|F&$R(tlrI9SG+YV6Lev*XzfO zmlO^A3G!u)~qBOCt}s;5-HbjLkm+(BHG_FzB}6(7hG_L89u|+q5bD# zjlC~}Xrt0}kn~q}WB)upx;{wCX5~-b8(6GPN(qu51_ZjM#q#BGSWQt=FO7L(DdoJu z&pnV_Iz^Yi7ZwM4qfnsHYnlk3C85-&N-!-z-fi>GZlvxU*5iu}#el^RTlm5h6p$4! zwbf54Tj9T1V4I^wUOyle8hb&2-pGTn&lQ$~6ncY*}-9R%&nDM}2hT z9OLuh72K^Q#QwU1qrJb?+qSF|Z&kPwfdcNGxp!~{aliYqKKKS!$>I0tj0g^AGu>SU z+$q*rSSBtjjXASQRa^B|entEMA91FaXLytf!ikG94lbzwmf^8zEG>L+bBxT0Nvw`~P zh@)@c)MHQj7aY<$t}Vi&{0%oH@8$MqGlX=o`8~dmd!Y@ zCksj0C*21bsf0J>`9%aqdyh$K9c;L5o{U-zv4mN_Z$>3| z$*GC~@hhB$@OW~?(`P1;n&PYWF~oC)=xqfAd!C{R{4<`B{+nwM9QUw|{-=1w{uV5C z&YtX3Q=x|WgykhQ?HtBrRP2}rVl1o1h$JsEhHrX&pxEo|@oN}vS^bC_GM)Y_k8(0_ zk#$k*NTDun1&1%<0UaO1PdPI1?su-c@V7EoGYn51zYVv7HTChdUGcgS%FKKBN>x&u z#&auPvmeTIXUY5#+Ntro=Qx1U8P!sli~w`S?00C7kgs|jZ)Dc$}X9rYA7 zMM%b4H>cz7EWyg|vF)sM3gim?9mX;yeXrN!%WBL*Mcde4esnFh&H&*spiO(JD=3(@ z-gmn!7fL-9k-#v^$dY>F9)xScCLF*qG2jVtN49MvjFWo0 zL#ZmmgAv#qWGT7YJHBRH96+ZNFsz}eL=eE*M(rd_53YKs^QhOer(ugd^!^T`lY{b3 zG33=f#PRJEoFp4fka3{lAq3ECz07*ANdIhaapDtE1gw=0zgH<`5WI68mRNg^hS6-h5{})jq@O}esMn!yIgojbZ@%Okjz<{c*HMFKZc8!7UK6dn0 zevu3NWM@Gz!-5rm=ZK)=?)l9-^;jV_@YZWCT3oW(o+5oCx(*cBc=kdHznShV7R3<(_Xwl!Sq9KBPE_GGYX1x5gva&R3 z(_@UVXu#M^dBj(?ePWY)yKgLQVf*WrN1fG$9dClLJ~X{sv*QZ}tKApFsr3DGh{^k! zD~tCAO9>g1iGB5Q+2HJVMfb8cxF1B)fN4=HahXo^L!1^FD=olL$YRRQoI5kyPhY)XhN7|wVO#+ZG&Pt1 z31J>d6n$rFTU%Q_IsMUCCD{U8rYhyh9!#@I#rK2=2)R`S7cQU#_F?7am6b5ztIqU$ zrUB~eZNzerjfKkMuoa)Ox{F-!zUjJ0WhlxeK>P2LdUbndWl;MvjARbjYo=%xstUbb zHVgz|Bco;ICUC(adZa3{z1c%WUB1Qbss6H%LuK>ke@mlWM)8uqD->DYNSkr|uKS4w zr#$CyrTPO~Feg}t#!?4%yHe~DSzi3YxaEbU|0C|Nt&f^az*@MnpU-)?4&ehwR?_+f#dOZBf@lt7-rvIjZUz5q_K9I3)}f) zPLIS~6i05dS)t2hhQ%xqXfT-8rIWKPI@${Ny*$k>zEPQ$#{{j$a4{03>ciBTQL_F_ zb=!8e|H|AYp(~~&{{$gFMNw;}h7)e%A&snVA7P>T#nNSMuZBoNT4{Rs)Ccd!QR&*b zI4niFnson|qV02_z^}?5pXUJ4#N7n*cNyi&s6#VEnEt`iugbXj=%uOQ&oIdNKS5~C zWfsOkBs{%#_hE9N^7_VfX+2f*@x2CC;I3AA(gJ*BQffo^4-%JzmvzQqh8!NS7PX3& z&nHsk*Ovuh^&shIRcoxc#=)2$-u{OnhF8aefNm6VhPtSGJzCw57|nM9W>o~ntnrFR zWK0t4HkavIaHE0S0S?TmOz>@+x72^@0!(IF;;~yr)zsAZ9vHBbCo3@3*|{^G8`ej_ zYh;hed}2*+FD!P4fdS866_?W*5-t-K94morNpq`wQ^cpX2+2a&UXnt1Z;Pr~qV7C3 z)}L5@s5$&}v0b{PJGPqNoMYx3Q*8{`B@4eME8_;XQFKTmg@MW??4Bw6;0l){>0 zW(@{HLxA_t6*mg3weTOV3_ItiRbM>D1mR)SV2K>Y4Fm>G7Fs(qKJp2(k)^DK)NHn1 zg6*fQFj2%2I9AgAJQ1s9l{}4q8`6UJ%HR>boG_u4tv984lUBU4#FqUHPRSmR`k>wN z6@Or)=6=(v)83rwKX}G(0my`Oj1W%k<>10fBr}3LoKs&;(VQ|AnSit%D7zh0f%GPk zkw4MlD1LtLN60<_Z^Qzb9>NF()meN$WNT3T_}*nh4t4N9jMz_`X!S1>6L?n*32BC! zgl1?cf282^WI&KJd@hKY3)QvfmHQXg;ik2-K4Z9k<6hW{-zoPPoDSgL#h6X23`2&G zj%|MU?9$!PDj;)eVJv~{-I;i9uYRP-H}R3Mo?fBk;Jac!H#m`yhRH|OZWm#wK7FS^ zfE185XnIO6i%rdowLiS7kURocmmo~ow#V`{uowq!x>@coj({{@yp#4$VgEOH7F^T@~~P)%bgEJ zt~F+#yb>CN(}A+_P3s(%kku@{e@Q9vO{!C3U*YF zc8`cEP=k)kAi-brHDr9{O{=Hn_Blw3^zz5U;1&jO>ZrF`ISLV0l?n;puB;S5e*LX0 z(KRYhojSt;gsbrS_h6O}WLH`CN`ViPpU(&JjMZjWg_cYBeFSTGgzD0`rwX!-K*DiNJ0ji-+WHok8cK2Yi+=YP-9phpD)S{}@{r$NsRYoE@|5Yc8|Zm%<`*jIMU-frteXUAZz0(2LJfN5h{cB-aB zDi-1YAqsX;Cg1zmRo(zxg%393+ryaUmQP423u!aY_6(Gh-!AqVywV^fk6Bvak#sVq zMhmI--S}j-^T5>meH+uth0Uhh@pklerq`ppJIwc+aooP;BXOUdtz07&RT{ok9!PCZgTUrSSE(b8{uET=xoo|75rwrX zXaD*M^2CXE!5PfK57yO7CZ9BR722tDhH=f+-A@Z}ZnebC@}-^g<-@HYIk z+BF$-vFcM!#l!W-(9x%T{dC5?A5t_bxjSE<;Hu1~`(pqUPFO*9$5p3Pu_&LJc9Kje z_>EjX>D1pVPiMwlO0J!bhA&Tqk{WX^S`$@Plz~XZ^zLE5*t^i9Us*o|Wq+(Xxoj9< zrYRR_I78h{;4(fVakH`PV6TT0d#{-bQK)eyWXeQw#fktLCSG~E1`+NX*ducc)v7%R zqzrE6wRXI9N? zS2ip=Cs1DT?XGg=_ggc^Q|_p1hs;VcM6&w?#fr#pX0Ii?uybeL`7*O5I4enBM}8>ep;bTMZSMYJ1ivqZHcDuA8oyD4#rJ_AAf~lUadC!d~|{8g~Z$F^qIq zg@6cl&HZ1S8{&`O}4D&(sArcp={J z+bqiA$=)3*HchZ=COiJ-8{vvlr5<`9wQDDB{szEPkx3J^vOu1<5Y_7tMyAE@sUBhK z&r-2x`)jN402iNL;2f9^t9K>Rec6VY9CcAsdtWvh5h@65I=E)9N4^6e(>E{$8ZMtz z4adC*2*1kl$+x+j>w@__9bkurhu17#F=!t>#o(QwY`xlKU*N zu{$j^2qG!5Sc-}<8n!_#bTE-D>2~pD;~cX(A~^EkezhbJ@$4Lly5%555IKTO97OE> zcwPoSmCb$mQ9~}{?9)yz8vlMKoQQ1me#+{kUK8PK&5AdxY{!Jk{JupSNeOqWQe$GY zr?L@AgTr{@SC7-{Ir)uFuBS>4v5Y)~>(e)-Gk19Ip}CvHW&Q7!CDbpS;Jr0&2&0(=rZz-YM zd{)$d#G6~J$_=qyVS*aqNIxbTm>zjh6&qSq;KRkX6jF}3Oa9=RTitqWw|t4pG2IV2 z67!mz6Dn75-tR{kUoor8L%({}y3k8;b?BDZe*3rm{l2krCVCli zEV?W`53v;RSh9w_?jm10|Cr2(xbNT;Kw0v+t8AS5zB4PQv1mP#4GQ9Vsercpdh7en z`9{lvGkZWsYD;qDy>qU*N8h!HW$+B47V`?8NXgYAqU(}8(q9IQ1~*P^5j6^ zwQT7^u;3v`aDuzLlVHK!-Q8Uqf_rdxhv4o`L(t$7+?_@ne|_%En>qK){l#vec2%!h zzP?(8H}9Z%WLh%2<2Elp)1Qx$jVU7?ii)i`MY46`rh*5qAm@R}TaxZS4UE=KjE0mC z`Zk>t=659PsD^%{f6|b|!A!p490fi2Uip zn=Y=ThpV}t))>fseOtke$3>14RdLIHmPkV;!9YbTb5hy|4EP??Sk-{bV~j}a@#9A9 zLTL!30^Q~X*P?sxp2fQewGh0FLZstNm{gq!sT&lQrVw7NodZX&E3~E3mJ@TxjhEuB z`}Et}k{i!hQHVI7hWqeZ^2`U>suh^cZQYV&Xq)4^g;TtO$faROvP$!kd=)i4*Z^}_ z&BPOof)X;mC~vq=tX5o&C-(BNFr`)Gv;(2WEZYYbVj8yOg8r;K1|3{NlXprJ;d{9Q z8>Rz&^+9<}+}3jpAMHP;scm%^8ZYxkDHWP5IgJ#+NiH!I4b@PzzsW3qr_Bu{R-7w_OeRLzP&*D+bvRvi2mQft)k-M zbg9gG;I0nRSJc{qGy{-?R0w{zp0ngJYxuAo0iF7x}Ld|N08&CY+s}ra4n?&4*Z1?n?n)gq~XG} zh|}Q2%B?-Q!QL&3vlxr{nR|!!^`WfO8Ic%BS$P&h3wc6?y+;jpF5%_<0H~c9iiKh)cizi3A3tlc9LKJUQ{0-MG$iBcfAyJm8mAUlc zK@s0t>VKM95Wj{1s+4CKi_SEqZff64Oy~oD>-K&aFKRV4E|oL1w`D6o!co4rH&Sjz z+^YbD_NOCQzPm@8K#>JD#^}YKoqC4Rw}mrQXyRs)Q3EA7 zT2sn5G6YkYkdmo{nJVt8szZ%)#^ju-Tc*L0%#EaS9cjQbl~Msq)a7yf&sXD$5X>X< z?Kk!g*<5TC(t)%N|J>%OB)^{-e~{mfV_#{A6)gwc8J*lUmqji_w#lB>ip3n^^pMPQ_f(`l$b;?{)ND1 z-NzN>f;Q8M>S}rB5 zoMt3-GUn{{8kcVARH{FHJ~far;mJs_@vA#*^QTe;b8v5aYe<$Dzuv&850a3+C=$}E zQ!8ICHNYmDueOd}0a3s$NW=Ut{PgMKe`e1Pd-ah){(i^T`4Od)A0 zlvk=wy)VBtL@&;vH)>3WTM-#ntcsPyP+Y=b3tfKCmg(UR%dBd3v&A+xiY<)e4GE_X z#xu<;{zI>y|3E-B@QU+*7mhk0n#5R(lsRJwmCu#(6ufs9hH0I%m5Td~oF)jMVqWMN z#Cn;7M4w0p$>A9E3r4 zjODo?r4~j=LJ6W&Ad>vokC}B(rhQWw#hNcBWbmssR2jUniYzYxA6HtKu7 z9vYUa7Y0^U0{`eR++UFaH$x}9KiWuLeyt9t8Ut zS{#kej%s-F&|~=T=I7*Y=4~*f#jyyHGE8poo*)H(80g!YWGbcm4rn^7sP<`7rSu7* z)yd!A8OKpk3xD@a8v8B3o%c#?lkVOL1b|={9k+HsmNMd%aK*Mj;w~( zMg@l>{%rp3EqY9oLL>Wo)T6?w2&yIKB9a-ZawhYQ#YNgt_8nc05@rG?A=CTI2bR_q zG4izi{#%6Uz@Gi6)|!!a;K9w*a7uGv@&Z}?na8@s!-=M9Fs$YCsNVRfIRqZ$db*Ms z9yoBc)Wj|2v%L(FiYT8QnZ*;_^~}75{JG1k{XvkbI%D>Qs-uL~e?~g$Kq=C#&5Cw# z)8U#bc{1OnTNn*2F-30}z08w|j;NNzT}0aLV8NrsQB%UKO#SH>?y{dzuZtOHx1dD< z4d*R7$G5JTs{Ol@5#E#Z?2LcTPG(S)>RVk5Zb=B&gA@NRtQD}s*Mok24pbHK9UHrBUkg)^ ztDd=B@d#(k#d)L|XLF%(zX@{7lf<(%y{42Ef8}g<)*Ue%e`9`Njn{~d7|(?S zsq_9CrRU_W`)7mhLy@;YUC%$?vMYY%AVBgiwDmCdK0>Ag`5iA5*#B;v!0e+t{dF9+ z3iAW1vqabF0*>jkzeAuQKgk(bB%m-@%| zrK?C$e)s&j<;p}C=1v7n>Xyri zER6-jf_GG4PVvH|mL6FVfkX#ok<5L9@K?`?-Kw;W&Zny|mxiOwRQjGKuq7%62}eZV zL^oDcM-|AogbkGc2xJ9F+U$(t-P!4m97wu=Q1BR z^N8j$Z?9rmE=_;ZyRF#GT^inxak>`}oyhGAYB3Oh{w9DNE4u_C^+@+8C$^n(`p`LrNVABpIE z1!Nc~YjZLKic0XkS0d?|+~FTo*xM+h(E?UT8dvd*4vr7Aq}>~ou``D;g zQEDjV;7f!41&fm=q@QoN2k!E6P~({sg+a5Es|6vBs7B}wIyLOqgWe*NNKuI$xy|#DdAm|R-7YzHLtYMS+ z_Psk+SZyYS&1W4*=u+V-CInoNH+1@*{CuO3CTPr%t9MyyqiT?`3m9EVe>1$B(X6)) zyLN6#Iiw35BPa4Kw_x#Y__6$Z_Ca7Q^}vAO+qD5&d;+vv?hSAjv{@gOkB5q?!IR>g zP>^YRpOo+Xz;vYZ$N3F_Ccdjvwd*6$$4<(4sapNA?#~uDh{b+asW?=Fj8W4Va*)Q8={TL2VoL<#B+=$>v9-YMRc7a5=FmAh z@ap_D#MkEfU}(|X(EB}j{x-!)4L`_jief2Oj|WT+E#_sR8)Ofxp0^ zD7)fsWOs~2W4pZcqn9WpMloy$H3|IPawpFADjXLQ3eH2(L8V8eQ_|imU$|lc5%W5UbsY!; zY6M512%4Io6!r<z+@<;!$Ahp`V+P|2VZdRQC_AQPkgPNk=I}8r{66ti9er?it9ZNDpP8{@e%B73U zslm=-=L@}cPCDlfLhqr}O=d?_U$Sjc>A9)gjcIeqYHxFJj9w~>ry5(3l9Ud6hSTrz z=rrD};xXqVUPq2Xym7ki8U( z-_v0`ZEdC&3LA%BYg_Qroa|hgE1DUY;+G@Z8)k2~^u(P$=_@6R_gIXBl_YlKIC&NycLUwbj zVi<+PWHz!X8duple%8eeB5y9|;pGSygS_Cu0)WJ`NV@vBAYUKQlv3!Bzb#DeGXh@- z>cQDTRBbMQ1n(IX?bE%}=~v>-ScL`=0cQV+P-b*%JXR+AO&1aXSq8#(I(V;U$_fM> z`|>9m{uYJ8Jg2bLS};T4SbBewgNfFf1^fSF0gj%%KuSEXxu_I5<8YJlmsXs;eB{yuD1N|IayTC4cFBcuz%s0CCCyBu9jai=l##QJaw zRJQG$ac7K@?tR)jz%NriLV*$&HZ!4b&4ekJ#a4jv?uxjKpnuQYad^xtAXxdspo(rN zwQ5Pmo^;>hM@Cxyi!T{Q1K*F~*QSSkX$({XyxL;B&y}@oB|5eTN2jlqYN5hX0q`0w zqe~6Za6ak9B7ePanMQ3nrTFq4xpaWjks$e)? zkpwb6&IqNE@4q~!B}(=G45au|+1#~qd7q>3qrr{PCYBbq`-+dbQE0rJQCP8fwDJap z&h-8}DIuMubjwIia1PJ#;|2W(8mC1%6W@Ar(qv|*!=+@Ok3WjV(dcs)NNh987{F-;aK`FOgLJDi)=#!~PVX>;GjBTg!3Wt)C0X#I?k?}S`esaCc03P=ZLUi|F9NFsW(RQ?ublLlyJMNV`cj9oX zew8P6$;j1M9G~DGlA6gmaNtUcDp2F-JBD{v?i&?DSNkShOm=#Jc5QcR)E(mnykFjW z2Hc$sRquBIHo-Q>S+g-N1$^R>N@B)o%_Av@I`+-2xJ=Qk(In;s<|g)`G02%QV_fW* zT(%OK@yK-_FhY_Fa7z_Sl@AX0U9J5ASYIhkT<-mah>6h;e1$ir&hA%_e&748=jI1y zTedw>Xr+xRfCEDeLQ+iKS_4su9cMH4K|qk_m4V%_MENilE}QojEk9gGQSjmRzY1b( z$dgi#UhMy>b-k;2_HLM)shFbCuW*a3qA&T7?V93yM&HT1((`WCOuHUdB$G^iz$G=g#IE->pqs7WEu0!g` znY7kVn3vHBhnq*XJi1qz>cf2`S{X?J3Mz2s$h{@d{$F6cy4&a%kXW+*jE9mmi-vI{ z+O9*{v>X|7=DGoG$7nzG9}(ybg3_UHE|zh^tQB2(+l$&6Las z91*m4o^;K1rbq5UfyWOGAdQ?kd&9+1`dLU%}myAplpy?2;bu8g>4VEb1CtNeVI-aIqbgfuORxS{s~C+ zzAZBA!q-fYk=~_Su={m|}?@epd>9r#Y8*)DSbe`~tV-M4KjMbTr54 z;SHB`Gr*2YR6SR7f8e1u^LZV>3$dU*?IrzC2|IHunI_ZEnNr;&f7z2u~2S;8s z>lSF;J6FSy(_}&J=n4;P{Ao``Eh=g5Hq_*AnQ^yiu(qmVphJW->0~;q5u9TsvH|cY z2vi{*kfhVCFd z((tup8ff@Rfk{H^sY?;f1BAmYS`RAsc5vMU+VIFVT-Z3xp5bCdQ}jYyQ3`%rj25R`4;xwxIJn9Kjde zI@l|f{^dq{B49%n=&c9+a#UC@#|6fE{-^wF`iAEHT3lB3Ois$*;w6EM$QvgLGmpN3 zu`n_>E-4G?=6D)ue({PnxR}hAJozS7_uL0$C4>^&dgpv8JH`bZsp3^unp*cVXkPCt zhs^JV72~@WN3_aqilw2bb<4%*k818qI+>%txJ*VShi0nK?vSLvBdIcR_)H><1)mKT z?Z)H#!49Yh3Cs>h``uyqcl1r>1EP^f4hxX@`vQ( z@-^<8FH%#%j|Qzan$D9=TNtg-gpVosjQB2GS`T{~*?_zErLsYn_IFqk=y2t$J2LyT zj>MvMjFU|^hUq^vJ`S1~1^J=Eg^p*f-Y@@x`$P>_9#1MZm|gt7*2C)~HT^ee77LSc z1RFV2tR%vgxw*b|d=w9M4MlUmDQDv;EqTIdGMiHbjyq=~$w*>3IH13X>)+69MmR?1+9w-5#Qvm-)A3`!=d}Y!H0_$*0H}C&lwsN5WqxdA!S@oP7#tX= zepQ)c%iTO2&GaXo;FFlk8PFY9*#wV^w71Zh0uKt)9)4>a9^dR1Ti6t=zLFzM6+1f4 zyDo^Et0_+2hPLX8cM;!Q>e)huGMl~xvbr6^b3aW{3*35_1(=2GFycjJR|=K_(t0H?6buvBF#&JGnj0+o zl~ zChyKL(<}e2AJ+vx^sSCci}z?Jq@7TV=nCn_7-BU7gT7l%CtLo69LCut77JVz;jBKP zXRjs;0*?7{L#4!(*iXxR04$YKz@*iIysPc8Zz(eK|CTid3n||O5J#7Jgy(Te3sfkI zli@LUb0}#iaL_kOYbPTQ;wtQaE9t(Uw^g*U&Qm{>;(10$i3vJfpG5Xae&@EMfy;Nt z_)g&L{D>vm0?m259@);oE5w|MOH;e!QZ^SI_jf6ye2Q0*#0B|UGb1#S&Q8&7o>`00 zjpaqiL@wpdFNI}E+h*h`aJyl29N7e{%@a3ATw z;uRdTz}T$*$X{urU_baoe`Y_cc-d_@%s{keKz`P{u;B@(k?_y^vQjDU*YfprJu)BY>IFG`QR(kLkeeILsRjB3hGFlmTJ4)7~-8|(ez9UCt628G$tOXgC{|G zW`o=SeP?98yNEh(%r35L1&1kFxCc9SQU42?xid4QuseY=q@^p6i&}lRr0wWOkB$t7 zF9-2L51dVJUDb7K4<4Y&kMNSlkKEcUPqUm<2zP+Y-DJxZjDiTLkVE!}-H08ug~1{3 zPL8D`ot?*)zYofuKNMz-vYAlftrascDCAH9ZQ--gf{y0X2mo=q+?nZAJ-CrdGag5eYUyF^Ei5NkB84!BxFzy3HuDBKa^Mh|^E z3-%G&4&7+8e5dShu`xOc%jF?iM?}*tcJ=D$DzP~(lz=Ba>)lK!tc0oH;KM{LC}u7% zJwNQB)|4^lupZE9axFFf?i=a;rq#Rl)}1nGSSFV(F|2k0k@(>uhBC8K5$Jp0D7^pm zuStoCy^690WV(Bhvtr24tl@*VZIu(1l^Soz=#Q|puZ=+G_#|C!VvA)lzntV92_Q;N&iuwRHkmT;3D;L0^}2Jr$a@L< zo_-VVpjm!6am~p0yW{Nh9Noy2z!(V-uzixv|LqriJ-}$s`S4EKK?3j$Z<@0=n2-Hj zOCC{xw8#-Rx8yngN~AOx{3A(^??T)g?Ue=nJ3Js9i51CfOSMD2i zW5Q#7*5 zYA~7aeUh8yvSiqUOfl8^8lg2cIP{Z0z6KY3|8w%RsCTBKHT1)weSCk zYAkHsl1l|?XKQRxrS9|uSkn+g=*ZOx6)kK%gvjG1b^6`@c?Jx?eu9A^9V(5Y;t3>B zI9ib~%aW7B8Zuk#`0@NsacRj=%eJKKGy^M*Mp<3bNZ#~h`vJH8 zF7IiO7-fWaJQlU`J>*{E$@OF$hnk|(h1k#_&f02oco(TPC|&t|%a5-2?vjan(kf3J z+uJiN8eYD|If}-m@GdjAXZGV>skF-KKnZB9ptj9pz6_bW(}uF`SO34esQuBBsh7VG zDY?6Bs05!e*U@`GYXyx&zAt^TY!|zZ#<%>wu2mueItS=RHt8QNJQ$6(s;{&s$*tD- zWvhxD#1BNZp1)WgRo1_S73JCsw9wcI_xQEtZv3hmmGvIS?~c_{@fXG6TIyIb`nK}* z6g3^0PKcS_S%^ejz3C-PObHFr!y&%0u>Pxg{q{lwo4p?|kWPR0IoMDLk4Seiuz^sQ zJt%08VWMhLhu4p{HCO>{-Sy>_HDt+6!(jJA-L@c^dJ^yJ^=-F9Hh01JZ3b{TccUZ! z?E#p2If_us%r;itma`<6i`h}$>96K|`W&<9^!o4;?Ax9D!#NTk-EUGd>{q8!mgsm( zVFYc>C@|nJl-ZyO9W^JO=&d~1b72$Q$F`ca#93-00wW)02i)HyZLDfee|X~#j25<; z-j4BtT~|(ITvww>wdh2-wXI`7g;k`fFqtqzd1%}H6cBSeA2u#+duluO5WuFPSxs~Q zF*Iq-R+A-{T#Gx$Y^~@!Vykm{Ru+f6L)z3qPgXbUQ$T;Zwwb}I=Zt@EB@f|aFvvyT zM4h{IDwv!8<-2oSVfNA8qJIrX~x^Qr^j|ve_6H_t&!|3b3cS#A0+G<8BErZ7=vU z48V6uy#iVi65TQ`nW-C?nC^$&PgkML_Pcp9U0A6VJO8AZOIQ%zDp8oHDYd$nYxmSl zc#!f+C@$$wi4@kB$C~wtUyW}ng3|xSnBl`cd>N#plzRLg+`I5CI3L@0(0qGL|YjZntbza5k;a2e29%sj+Jd( z(?N&m*54#-Q}cjL%By7HT@-h7`X_qLZh8SS}{e&(Kv zt?u8^IJ^e4S*U4N9~xstc%fRJ>_l(|yv(`+?4Z_TlTXjovnJ9DOY5wb(@SC_6NKP8 zrE&vi_}K8^WXH3e{KTIci}-u4fzVj1(ji&WA< zb}lLcwso1Ps7+`_C>PVd{h__|?PhFJ4d}Au!A7j)wy{Db4`&@A;EG zdVDB?%K}UxP}gu^GH*=yMTrV7M=+pE2>Fd0YTT4iA2_o~@_UbNWWy!8xn^Qtbb|xs z5jOUe_e&6(!_4&@fHz$|JJzH}tid1ugpMxNTM&{5K9%S;a8P(7MYd8|BbJoMY`@`_ zKS)Q+INsLPowY)xBN@ysxb&54J8OTh1aRB*4<(0*fdgDF0jOSH( zBC(+{nCQjC-(hY(ttPO{V*u3Li@W^~t^V zSF?3Z-@jN)4m?Crnx^C^t8RJrRaMB`0L>TdoXmT5=#R9zend%fU7L)Mq70cYJuznL zt{1#@8aOe??|?7Hz)k?^;$@E43Z#Sb)*sQr^^Sx*QC6`VmuKB3jb{dvZa;W}!X#vFu)geBH$eHPw807zS5`FFH3_&!{lgJSKil{2Nt55<+)?f#d%< zG6rE6V($kj+ZWGGRGoUr2U&&I)>d8UR_8q=adGiOqRwif#v1z&T2QawN!Ew20``9d zbyPhWctIn=ONmdTDe!*g>>^E(Ui9cHPBQ@&DVTp_!1oYn&Yj0;)WOsrwY4VQ;|({W zukV!>y1;G9W5us}*}2~%e~!Tg*g8Q92HVh^hx|GBrQK4~L6Gtg!RG4KOCsMvx#uy6FKw677q&+@RL~u#CZh(Go;Ip|r6>Z;v}qbP2$IAJhmu+I6ivY@(0dhFW(GH%BY+!?8aeSAC*ihX*g2EtIKubntyYQ@6J~5$~TT>05=f{zA=jZNr@aLY6tzCP)g~3A>A0p z|Kb%ehPH7nZ^(`Bz$2}yj~V*v!5fssN}=fuN-8DK67FpmZ_pz2eq4FvI_FFAfcMxb zf3zOq>B~{OPH)(U!VAfBJ^2M3E1CX$4uyxByGP5-Su!}`%WgqjY0P$hPZ5rw+4Qsq z{kO>JzjzQVbA^H{E3J#qogXXrAU-z%U$@y%$05`biVoi;+jFnsxMBiJqHr$tQh%UH zQ$B0OB3MDPIm>NObWnS*`6;n)Z=5_>oS^LOPXmW~YD(3)ctBiA^vD|F1UyT<$6ZvT z?OG0~r^vV>uY>LYXHLOj0HNmAd_2aTC&4(+<8)dV>)8k!6 zxzbu@y0-ugk=1l7nkiMHXFi99M1Bq?h<5W(+$5~Bpn8KPBXu5oy+=!FitekS`UL|5fs`6Le#>2z5K}2g@+e z;a^8^%kIB3D^9;e;E@r0iT&&`1Z}yTY@ut90-%|*GY1uLicuCjQ(3cD`U{B+@1ewo zlJsuw$4w;lt9d`Wmg(hft68B~sb(pi$DhCDW2;A5tS`omflEumPm;z~O%oLtIHwTV!Pwe7 z40h;%YQLj-FkJ3*LXGzsfU54gc+&0lL!47=!X5Zk%HxG_{^#k7b;EbnNPtU1aekw;mv)a%GFE!5+Tr=MZ=gr-#6mFo5Pex}<>k<*du zWC^_UU4>(D>0|TFMK-)1=8VR_VS{Y!kMwwXa|u4k)8$r&!-qEN=xV`7R>;4#!{2uJ z4Hc`dsNdu?Cb{wNgv%{Bv2kTg&&zmqL;X(@qd&alD^V8yD9n8PFUNj?5@NI%6&)Kn5K4!+h;$%86 zMVlDB5z&t#PhIp~6jQ(^^_;U{Zf0-{n{8Waq%&jrYeb1WjIOP;!TisGL{6kpS$4zL zBxz}F(%2liGV;~ra39ENCt_!a9_i#dWhA=~1wrVinnqM;n>Sn@tl55dC^;~^-r~?Z z9Hj4}!OB094xZofs&di!zW$Q+O`F~MoJ^vrZ9pV~3}*Dn10&HEN`_INInFQuAj;7K z`#Ej8iez&3+@7QTajF4(pgTh1Q&-f+6#n_owM;GROCrv=@->?eLVyo}fk9x01nxI( z54yFJwFzJr(tX~;ILS4~s0;$eQa-xNz2za7Ic)YxxgzaZtthbB$*v}DY%QiV3^vhL zd27LotbrOO#tm(Et?8(dq$k|t`QpoF_`f?kgX9uNGw~kgeuIbWdWeT}@d5IJ zczo8ddRfsF6h($~Y|b2}a0s5UZ$1tvfs7}4>Wh6Ay^;)i0yw$&Y7X?|_WVi@F^Kpp zF??1X%ZZN6E<*%TOKm~3oP1ZqCwf-;J&4e6{)td6K1+ipePaES8$3T21G>resmoL( z-yy^rZn8^%{0bn?ACKjbp0Y=I_K?J11Vj=CAzc476W}NNNHfkqu|U#nV#)XY9(XrU z+Kpd)tpY3eno@k>HmKCC$$0)jiDuA#XY5mo!sY??Dv70IkRQ-RqHd_v}3kqbMcZn*i& zvqw5+f~$!i=ZkbR&$3X;MV19yO)5cc()%~X^)vQRIScyrIH{M@lj!3T(QR^ z5T8^;*-Q@H4Zgg%wb2qWvvhXVRu8YJ`(f0Am=1U*$~wRqAgzr8E4vuM&i6Q>-i+f7 zHKJD&me%D}bXHB}==@^1Iavx|XF)^2{cK>5x=O`<@bvzM>YM&jsvWY7&)m0=k_<#W z{l2OLh@5TiNXG0o`L+oJH_XW9a9SUv;$GJCRO4}ypJ%;PeH#9#7 zO(S-J4WG_(cX3U+O@`k|u9FC*Z?4lt*JUmfEJBeiKkmk2QP~-uyE4uS5l%LGK}p0` zrT@B_mmIUzQ2r*dYmfADP;cHE*?*0o$?d^)vMs$p>$WH`tb>os=?WX=qacx<`sMgR zHTH#JYIkrA;sAu{LNs1w_d`}_Oy<^;F2cJ1+fITl9>L;=Hf|Ie#PP$GLXz&Ychy9d zG_%8u&aMQP37RzV`*&Z#jfH_a&bUCXmc;$&nfx=a!DKAXimQKMc*n3N{Rfq zfWD6)imxP#mGWn6KdGKZe@so54(iYg>g$!^oibkp$kDg!^lZqXRleTsVJg{Sm!jQ@ z!~NwCF=onpFXi7RLOI+*qLGO-Kk$)kc;iP?i)wM%u#J)57#4mpGn1Tsje4Jb64ntD zxO-WX#%@EqWz}*Q(NCy+12<=R9#cpcd|Kjg1xjMVU5KT&j0#K%G6zlmc}NRIIe1FE za{BxaW{|y8QkOfLs#wJu3t#KaG9pM*QG(u45;~g=RiLjdq6z9^Hz?X0*Fndosy0+y z#ZJE4T1r?K=qa)-y<;hr%w)SIkOF(r1^MS_U$qZ6bY93dQHJbMNwUnwlunfsJ&p4t^&e{HD7*qXPrugoM+bON!?s=tzcn z^YvQ00KJ%C;g>l?HG+h#&SY-oNA}3*s=de`4q$wHfYLpxW6y6dM?xfY2I3^`$1m z=+KxhFS{ikLc(nDkkUu51yywQEL;~SpXuDUF{3Gd-^iC`k- z4pnYVQ{j=)u4Y*G%N?tV13OTsYv*R_We_;1gC01C6(38Gn$&~T^zmsm3NdAQDJV;o z4$(Auwp4+LZVjwIqwriz{K{h-)s7Gi*g~dksb&Cbgo(ByJpjgdY7{U|a>L6vH7!kO zs%>uH$$B|ToqmRNgVdBU$>~vGgu&t@*|g091hUOlFKC5fL+#PmvyRk_@`6s%<~Ops z)BKTNC*g;UH zM7D0SBwX7IX=)w6#~glvoXUT;*>>PX|NMRLqO1SvUfN*c#$*OvkHz$G^tIB*D}9D# zH{)~5k4l}Cbf#?4)Y5)Y8-zpqBNO*y-kYIy9$u@@>XP7stzc$4LEG};>r(^I(X>#Z zPgzivJxiRV_6HHv6>Aqv0*ckTZ5_T&=wrS@Ogo)AYzpY59eUhp!5mQ+ z9>T*5MK}?a=}eRusYqS;KAS~+v6HYB8S<^m zY10NHeZL)03ca-(>|O!yd!X_PB(mEr8J$^7GZZ^|dS$xR#T<|*m!aqm7QjzS`N&Zc z9=r$>Lxh*E1M?{fhS)LOYY1+A)>9hKuDj<4+5-MRf7+I9DI)9Wa02w(HoClORec~0 z{9{Zl^o#QH{?OFu2t;2$v_@asj|vE8y+T^Y{f*F&KEa^?>ctw*HJx!}XzuJbG6@2#@oT=aG^? zs{X9E2u=m`o?w6=3O)Z0Wjnhi+e*y^sgk+bcsB{7RsT5yPw6oM-d5mHpp5!_)JC&g zE({Ud-rw(No-XVSF6@H?ly!5#eO@fhy+80i?8cC1s5u4$1@{fd;cliuKtk#OY<0}6R{N?7p(PXj)u91X0bA5$E zXq7OkZbanIq@rsVJgfB~=?%dR#~F9LQ{aQ|laE=tSTId)ax$vhF_&14!k+Olo=DE; zjYC8ZPNFiE7Si$q`L(;P^8x^a)W38<3arfs z=dS3W!r7)%e8rEAO3AYLC3A}VvRPlZ!u?%|yVBwtIDeE<^S*vKq3-#0EcYAN?m2Hs z#90%z0=h3)Ud5>W!^9&myB1&umXWSPS8}&mG&(~t>W2XzOWTyAzjQb6v`EPCC>s5& z6AB!6c1~yfl)29}$Olui%?_z#S1~1Qcwylmkw(V4Ge zU`h3(UU0C4p(zTZbP%sMc{%yy-isQD=E0>ZScvxa`3G#W(M8*}b}K6z3$BGd;LfSA zkYMA2YU7obVvA?MSx8~S>n1-0UF4vvttG}z?asNn zf-njRIeu%iHH&-rUi{E6+xyK4lomD%&AyS)F(yt>>0Oso1^>S@bWjW# z8?u6?q;>y|TD8hbBsvfEV154wbIEz2q|~7V zwfa;uL@W#_Z?8fVu=SCtTm6pi$q{jW_r4)^{%no%4L0L<6O4}~(??{-t|!)>0I>@j zHViicn7q4DoKBCrMSA9JE$d`#wrwSwiH`3%9dAo|NwK<@gzaSV$q zOBdKX^^WJZ&)5D<6ERTJydt=50CV9?YxI{1)tru3YNri>e)|IwI`-J=XPc6iXWC~y z^oot4ZCc&yry?&Jukn25K7Z(k_nyEY>Fq*%<1Xv!nH@f|l&N*nyUKK~Z}g@J&-Cj9 zBY;Zz&&LZWP+?&v_SGk6fq`Sax*pkNm+)&0W{S~ozf_o2g^ZPHEQDJZyDp}YCE?&MWj)me4Y^DFU1t=Wu{ zFm{^8o7Q)G^r6;fn@A=2l*r1KIai8&M^_WP#O{PU82rQ;sfjDCW~x#8j`INFEQh4i0+5o#it-Ty!Zlf6RIXee|7UL@8*cxKuYfRc~1Da(j8q&`<6`1vB2X zUm}K=m6dIP08#L}B%?F+{*hnuf~-Mx4$}!Ii(P*!(OeFj2wwr^XcY(^44Hm(zah-I zs31@aWx*osrN=1XwP`401b=?^YNz~}ip}#4#~^Y|#Nhm{tK}hOd=8&1sr%}S_B)Fo zW(<)MKVtGie#Yq zn`;oXIq&q()%gaaE+RbOXfuasbGpd_uc|xvr50r*l~y^p?o66BpROJjwEN!Rr?N3mH{>JB_0=FBY&G}tqHwRQ9#u{=59 zbNS!fR3+*X90#`?RvCNBYwDyS`X3AkI60{1+VJ>G@F-XK)bo7yB10*4=mT2hh`2p$ z_^SM5%ep0zk5t>V1wsfn4D6k%of@CA| zf}xq;*kRoEFCPH93WWr*je|C12@x7q6w6*=?qnxWJ{C$S>2uj?tcMf%^g!j6Z4r2S z@MBY{)N$6gnUkmO^058IsF102X_64|`qe#|mvV&NsA70<8cW_GJG0|%w)eNM@S8+OLWSH z3xdeC+a6#FjIG6W_<~$8L~`icUepq1FOkEj zRM4>6uYy@R86L?SRayA3toC;CJEphJ%a7& zaZ2ch!#B1II#;)&O9x4EPT8S8fdBc>W-R$uUrW4t#_}vD}Ti2gxM>Kk5}bPF2aQlPnerC zqw3-Z2# z^vayF4I|~zl-u6eEeVFW^h|IQ$h`MSL*o;%4a0JXm*Fwr2O-7cFc3TC;=NO^!F@1pKM- z)F_%xFjSNfLdKmBixND>?6hr(#rtd-HwrN{2XAyAMAGGRp==Teq^{V0SID)-6Zd!_ zX*%bQo>mMS$@BH^_#(|=j!AoBQ2gUFS_<{L%P~+HgcM=D-t-G8~e*uuAprb1kGi>_*dK_9=(8~Xj3h)Nw!*k4g7#dAk zSVVYO(O6r(HJ;gdx?RBCDW^+DX8jqzeieZIC`!^ncH!)-$g({h?+~s$+L({M{;4(%m3u)xi z$&#O|aoS{Z<;ZZ!T8MI94L`a5Joi0tLLF}Ae)7UhnG!iCns0>ZTs2eptnmRD%UZyo zzai|6LEFNa1E+_}xhi^SaErWMyw0)O%e^}8X(?55C9Xm1TGt)gWC@w|DTudc^v4Wf z%j9Geu`cbc)Rucan?%AY?)&8pA5e*N-pHKekQWrg#zJ#$C5%mu&>(UcD#fU$*g9Qw z0^@4_W6}RBz7bU=#C-WLVN@_vxT_z}w)8695t63D577 z*1mW8F_;kLn^Y9~0EoqGP{w$>w<1On1M-`rEtuRMCkb>rD$JM0-98|qVU$LR+3gV1U# zvMT!2j09bX;NzQEf?G&tWFVkrkLBTQg&mxbgF|j}s{qd6Oz0M=tnV02(h$@*D6vw5 zIim|G-FA#s@uu}h%}Bmz8LPk7jDsPKs4-B6(|He9okvp|E-^%PK*jrN)2lN=uoUMj z;B9s4`?i4~#PW6l3sSu+Y`Sa;1C`5_<<5z;Mr+~*c?=4^BW7k8*Yh|jF=wDIJ)OaG zn|5Sd0XeTH%<8Qzqtl4p#K&th+#}szfnrR}qg!FMOn=Cj(QU<6Tm_|D;+oL8yelEP z*XVM^@k#K4d~t;@{=H-4McO5NfYhmG-iIzP^7i zy2>lWAi)d&cxgUta_YGlSO3aF2W29@2?&@JwCUsxd@SbK-q~>g-x^7zMnyx*b-8SF zp6Y5tE9wV(sB{E@lUnQ_tQ59u_xgz+Dl*BlN(STc$Jrf^K?)T&3M$>(O0Th~qlwf$ zNgl5Xsgpy-87?OR8RXW7ZWXv-*>KsQ%;rO_9wBkHG99#T*+f#Z{)q+ne_2#X+Q@sf z*VSukds%yu)EigzF6(FPot|79j&L@xnWwAn6OQx`m*yh=vlYZe4LEa@TWI(YUhm5< z%oRLG_X}UMxUT`a);u1oUFz)*hW%4T{oX7AHI4^94fIKLejOZbJd~T`-}mPF&}eRx zy{le>SgGRw2NU)eLh)~Y2LwIKJM|i3;$(=4(zVH-_zlJUvg?Tcfhs5A=?jP7S$wq; zui)A4%2S%g+A-J8Hpx6^pq(IeJUwm&3$=ebE|K-wQq@Q2bR*~hpRVL%&XJWcJakGo zlp`Aw-Px6UGutLpLrE2V%;!*j}eE#)YwDoQltb%~ucK14Ai5^4(G))0ul+lrBM=1TOm(83&eC zhr83eCtZBmlP_MKd>0GNV%aBMbc_xH%Vrb-D|?O<=oa&3fpwnh4sy2oeQgYN7VySD zvV&7T2NZRS?PXj|J_-2nw#~8c)aM5C5|rmPN72|hT?lzJy9al+CW!Y=Bw8&ci&v$h zWC=kiUQIcnpEHn9rYPU&PnjliF#ABqHtgN_jZfoA4OPXWv~JI;r_M*ypng$>}|j-u;%h3`WP zqvxmyuNS#4_Wn4}H&DkxSdeJzp<4mtvM=8ps1`ZWKIOJYkwLA=?@RVO0b5*X<(ipx1jq1rQ04ZR7$UUJs5HM90C8I*@kSO(S50k`F z^-5fI^5{5|wOD9#sUCxjib~Cy7>(D?D9i&fwq3fC zR^DEmPWF9+e=yq-AZFSV)B2|Kw`VOE0*KRR}tdzUj1ftL%S)CH0(=tXko1m#zH7- z(u>U0^x<3_KAe)vPwg_IZf)eTtJ0J9RzO-w&nY_!R9{0bn$!^BzI!YQA!o%9RA&ZK zn^cDfL-5|(xiJ3#Z-FvF!`{)lM=UpqI%qGI60d(K5UQ8l!`Jzpw;4Yyvp({wpxc>4 z1v{w5dM+1})Kp-d+Uv3_hJu_KVmRd2P-?7jz0dZEm?avmei;DbLPZ)z#WcLh zLde7swEE*p1~CT8PiP6eRze_D24k{D|0E6Sc$rzDj#n#n8$8p8FjTD~;@l|G-F2zW z*3wWWEk@OrYZAcI=(YKL!a*K0Un_*1IQB4LQJgIqqF7OtG@R_6_GI(=9MT}$@G%B6 z#3qTgNiB`c{60`oVO>+#0CrA=Dy<&)^2q?PQpVs{D<@Tl zKlA-aSlnr0H+_G8C#Bs{=SRSizRDA+rwPki>53Q?VIMvxQjYZek=c;h8#Z!Uk&1~t z5wNDO+i`Y0s>c0Ju>REj2gDpJY8+Xo2!@L`e7cD)a9v1UFmS#lFZmvzNrkzrSRajj zR!b|4Db%f%`%n6PxH24KR_RKG5PwE$xI)!Cb zQDt@s0UO4Udwk;ZPc2EZ*Fl~MtW7EF>3i2in5CtwCD<9z`wu0;b#=302)d($G8`83 zot)Zg)O_LppcH8 z(f{=n9hl%g1f+!J94wa0b-UG0Ru+qc(kL>$A`Fi5`HCDn9QqttE;>%cc%u%3cED&V z)MqCHm$PD<#dlbo#f-p|{Qm4PVgLXu(~W~>G}UmIAui0uWis1CBuYJ%xg}Q4rm5&r z|D|tt$3x)V2cw2X{I^9rZ;)?@|krpktJ8^o05nL<6Gk3E&M}kSt ztXcTbNT*Y919er(hP}r7JZ!^agbtP=a7#2jsp?04BZc>r)-Ssu@Mo)r=RBmFKiyD+ zgJcNbw!j%^#h%!{Q}ez#=Lv0~)zR(ismMm9PJ?Ne8rwsJ2G1cQt3-|@2@S9y(`!?Ek`t^CI62&uCz=!@L^kfJT~BL^3u)eZrIr&? zFQ!26j{_t-c`8yRqY`Jnq)#5sZdF0uNHozzAgSV~sFA>u2<+x$lDqtdW21|1snhE}$Z-noix5NG9MzbX*)>Nkf8uKHuXdVd>?loVp>k;ZL261JLl9}as z)k6YO@Y>Nw%kD}}PDjMvJoIpz6P`}$w@u*9yX8#Z5BgsSeo*7W5rZh*w~_358Z}9? z6;BWHH<`*8!tmGA;@f@DV3LL7lZ>TVY<)T|C~Npj%ei0sbl3(T1>Q*U=N)%8gu~N+ zS3)*Kx=p1;&9S@S{1LOw+I~RW4U>>2Q(X_EhiK*5+B@8uP|G%_8QeQG57lDLnhM02 zE<4yKqm_LJPh)m8of#`+cH7JZjS%M1Y%FaqA!xmBDVjil)@uqw9)W^KA7}*AC76sDGgbA5@(0?vD-a^A13@cC? zduZ@ko9F%|cqTm=Ru09N(F^30L+Uc&s6_zmF|aldb-WaAm}!6QiV$0Ui<|60m_*vwF$T2CP^BxA4Rg`b57IOZ9l zQpLWW4)x5a6mv_7?Yjc;ylYpVa60;ivM{9(IAE5w$PY88S6Jq_ zOL>WD#`NA0JWoG`4BWX|j?N$W_I@eMt1%YAa-`X$8Ujl<-!si`hc`htj)=iYtnWLB z3TfP65H;7t+MhQRwu|2OYF(E zzn*b=C{>zd#*s6)6r401@$a6O(lg5#JyZzpE z{7RT*xG1#BU-ZbCPa{GRE@_QbZv^nNJcy0Gi5xMfzy-3~QyQk7EaOf=O=j&bd5&iS zX1j8Kl8UPzj<&1Eq>DRn9l9^sWoV|Bm-?qpW^78~1YSK;^diPrnN~)KQ+hb&=nH zGRJIo?l0+>@CYju^m)Bis9(2nzf(4|#L_Z4HkCH2Eoz{8)njSx1y^ccINk5vbE7M3 zJD^F_f{JU&AR%8MIa9Vbwws|0C=p^Jt40;6uyERcC=|S-)3;N*>yFztm;WllvbNHu zU(7gEmjr1uPjEe@RZVy$&6-4)X;*F-v5b>wwzk7V#Z44e@SQE>i=*c0+<{&}QIZE2 zRcBUAtvW|Sp@cz87DD)S$wtlZ(^@`o9Z9yH!BSE@ZXUaaO5sYA$`bP`!mEtnFJ7Eb zVzYpS!VZ5sQq|eg>L((lczzo zF5`%iZ(|!aVK40#4z?*d?)j(0^t3IwL)pN=IqO}CfN%?%^494(bK`K%cvP+VUxoBy z`(+l9pX$SHm@A9DUHyei4FH1`MUdw*xU~%q{jHUd^N$gMjt>b5t++$OOiw>-Ho%%! zno_V*m4E?$k8r0FwB_XECT>5)kw?3K0|QZhl&dFiyoEUE*|M7=%ko}!gapN0Eea%1;bke51RXe=uCZ2~IN z0sf`AVJ87kCLv3Y1098N<9#@#mtl*0G zJNvKR;te=^3IgFQB^V3G6!04rM0*Vcd&g9b<)9~gZ95$-p+J?WRbMR6-Ptc5jz$`6YkRccN?>n_Nx}bzdPE*o23X#U;*moAwl;-uC~htAMXOY*A{@+o`(y zOo8Jw`1_vBilFn?(kQ80)Vu*CU(^I6MwG0PhmiQUu9UMv?YL)hE zE|qQ2y~9BS{Ju|GSqIu;RpQz&pmBG0^$2Hn(pqt{lKI2GArU$Xbl8TfqWCr&TW`j3 zcZ3E}+-*;wycnU#>z_(GX~R6CW0ikk^LwPg!6g4$9e}Bx#!uNd7w1LlSz$pkTGJd6 z^oZp1hA;Gb!H3OeMK1jECz_a>B%y!JbE)3f__n~YJoa@6#;MdE7eTZ4qPWUe@NJ8Q zo2F_gcX{&QE)lrB9H6h{Ng>X|FKk7WH1Em}6O9|^^7X2H1@eKuXxn=i`pUx@C3_y1 z*j4W?x}xPwxCnkfe>=jCJi8$jyh6h$)|?F0>(u=jiO>Pj1lX z3Ldw8DD)>Y@U%B89bLF=8||y|#YBIpFkDz|LE*SuG_jTQ<*&&=p~GlH`?tB2wTPi5 z0`34jC=PyjK>C8g$-dtvJHq#ny%s|wwNik4=8)}rT>4mUW{CNe$xP8B~rO zIWHa3`YIVhRf?hk4dI?c(4NpVW_#2=-^L)SmL!RTH6kF*v!?Te%+YM#>GCwdBir25 zC_`wl)$IPYBOE0epRe{v`6jKh@{HD3v)M*LUfy?br1UXoAE)T(gZr`9AEnUH*jlJD z?Wg&*H%!@Lg=I-nEvS{`wLLJj`t!a>g2{!>*aWeW?KL-%_1sV=5#-Du=&7y>l+WrRZX;@OEyp$EDxmT4U1i4igQ= z{iWu*fso2$a-afV*Uz|VzX|13NIQjfJ_cQ?YRdY-VRxu zWH1AZKj@rXHgFP|z|d(54Obqy@`4q{=C-*{?QOmW*>32`mve2!`h_+ofxTcu^j zYdzYrv1PDVMw4ATM4hVr7v&S11uL6~&t9>rbbXu%8j~NqyQoe)H5{^aWMi6b{}t`$ z8O1zv-Gg}H|e#t2)dz_hW~Y6n_1@_;X0hr z9>)rr!~|};2?=i~%#eXU)mZo)*?<+)ojl^OC9ILQO(F?O?}=QodRwH?J_GS-p+Wi+ z;FL3k&BXm{(q&-8%&9?7zdD!Unr1cy6nF)_!$SKhY6^8VF)+w56yY;hOnfNvqSzYsjzX0s$hL z9@2kH`H9<-D;8fw*v>G`pB?W;WR_~18v)kM*9kzJpr&8 zL$qH%EId$YqLIUs4>|GuH4xeXdAaG1ffCJnD;OH5ky2wEi&LPE!agqY zeuRY0UC?#J;dh|Cx5#VU#vc~ zBRr$|RNZ-BAT#%#X|vw;w6xd}`LD3XJS30A+&1t8oFrBfkL&cJTVV&{CkN1m;}(vPy(zgL3F8ac0(7!UGMfeR4& zo{l_?L6P-yYX^-Sfib&Mb^o86_g@Pu5b@)yb}uJ-l(uSH{2dS%8h<=@>kua%afel6 z>Jfv&LbuLX=o=``n60WO#3Ht+CKSjonU_!qiW-s_lV5nyA3L09=d?w(yfB;2{dge( zl)9%SH~%h>hE7n>QyNCX5h3B^p-5mY8NbtNm47})d{D%sWY|C7AXtoek3~qR7P5@t zJbJKZAhAnX-CfJ0D;nlQO3xLjY>ftTp~<6pC2KD2>XrMB%Hb-OQk*;xUZ&HTCEx9g zk^&GYu)s%-tJu?vYgk!>W}Mj+N=~*83}Q5RvZj`xPT z^D2v=wE2ym!TdmR#myaOB_lAHL0_0J?><@71t za|qzr1%#!c-4JkIA&zctQ_8I6Rps+L%D)F@B^+sIF#aqofACnM_Pw5Py}SK=a3Ky) zV%hUUMk#yv6_Gt09uBmVm||?#P;sAVBMWF~?O%w7osT9yWh2wonu6ii`@KEMN;06n^#2*( z88u*H4b8?(0y;+c${bPu9s1U1sK(})WFd1w=q&^;OK`5K4;D5<`e+b6^5g)dV!R3k z(}qI*4b!rhznfn$W;K6o_7`erVyWRFa^dHtN&nvi?Blp&V=UfB^yJ8Z&1B!7VMm{3 z`nAX)mM^1h8T9+UlNuncB6P?=5-ahG1jJgZG3890xtg3G-bznd;{+Yn%oijy4vmju z9EOT}FPoG&>7UI6$+c&Y+k0W!Yw$pi9yc7H2b4NfpTJw@%!D+UmkjCvG8tlkE_6vH zzA;Ax8KL;#R5&owcjVJLhjhmav+8e#priI1B?M!!=lm-c5Fn1Srf#V<#=mFkf2Hdm z0Xp@Kjh*0jCkwRD(9lzA9B2d$o_(lq*^s(wc6qnq$P6I60{=TT5CwM$M^?o|)ZSpQ za10i-N-ie6!EgTz9YecocBaQKB5^F5ogImY^%+kM!7 zt_Z?F3N(&&lJ4_q2jQVei_cD7%Q#W@g3|}DY=x;=ZKyu^J zi@HvqcAh}@UQLpiYDJUeSX*-!cK%<5wMX*QABC&EVEyaUHaCz2+xRuX9Y5%A|w znCi36*950e!^vp}la|Qf${px6UPBzAUXH-@>vup+;SDr0u| zz(m+IUrNLZy@-LFM(a2_0!bV4BmbwE8$Fvf@-z~c_#eNTO#}z?Z3lN2bLm7hNRBG7 zcx*4Vx)Ox6LdU1_`ynTX>X62;=+0SpM#epwr7pSZ9-*Ui6&Ont9`o9hr;CQxKvMLD z7`ECX^=gG%+s%|Tg)*@ z7-zex-|&;ACzrMgKkf~`5x#_c`qk>jys>FRSdSV;Mz#dQrLB z#1c4_THNFv?x4zw_c082R8DwizJKS;G#_>Tl?0ZPBT3EIb-Erc%5o}yP+U4J$yTwR21#p!YjPfRm`jfI3<7zL{-cIg(E!q9IQlE-|rjL@0N(1FDs_n9_y z({PsZQzR90p;vVlbCuQSWMmH4?@=QJKgjg2P@UJT1~7CxhIB{2EM+8sVH!Tf@_FCM z)q+~Q!>xmA3&22- zeG*;p$hkQSpwP>K>{7~pJg5Ml{$WcUw){joNUMWC^B0H3|C+deTwPt58jQbqb=1l{ zhV>9M_n6LU5zXG|z3~Q^c*(h(cBOGbyJ!l*AR2e#doGvrrhVYZ zJg|-fdF{ck+`-9=p@--OFh7Mu-<8H8^)7Ab!0N;bC(`8~9T>p5nFo5$CqTfZGdVkA zw=K20cB_U!+?`BWcmm^?U@d=Ki|XLw6x|hehF}n**vDvRhnH7>)6Wil1w^o6< zTlOs+*p1t}w$F1qM)+ak9j27d;gt7KJih&ieDD%uqHl&BU{6lz`amTKeBMcXvQ+ zTpEFwXfd`O>(Tc^hYQ|{FtvUJY8Omod__Tac(rD1!3bzmbsjf5OW7k|S8tu1qK!D- zmc@E9ZXP%h4z2{VFh2C5)(Fm)mLOw==AlNLsjfgkLof;xLLHea%rvF@A9cBgLy|HT z6+`4$i47(-Z=GlOr17>l&a)5nT+d>FZY%R39Z8lWha6u>orzT1^2s+)>OQr^{5|TU z%0f85r}FVPAaJ)fsP-tj)yVtsvYg8BptzR%H-NJ<&XOrOUNelw_Ycw-H-~n9l|ZX()brUHsrd`N`Ifl zB#4;smuhwg0=$ez0wSpYkfRJuPLnuWr4{(Zn6G-d`N0)_J~4vA=9Ce|6>(o>feKx* zSryu*#RKu?=_}w8RYtxd83bi+CTC5q!Imx%v2Z1DtFVItYSG`A+-J@vI=~9~2lKMU zk9G8-NH3Vfa<5dd@K(&D_!{TQk&IW*H=8zH$!(2p)I<~KGDxr<_sQ-q0^ zor-s(i;~W$X;k?Xd7ON*N7z>hi{EU`x#<*lwK@9QY!|4962 zn}_Y6Z65B66JrI9q`nG8^5F4Pr-xc8i;6MAc{`=4Q8pR_YT z?YZE5a0~>;BchzLhd(XjBl4b6kDKKGZCo{UM$ixFmt@;?V(woRqDha>eWrYbCg*X_ zz)(Kjrg}V9eb@<30tvVUp5ePr4npt92uy$pQ^*9h-G>S3N9lFPfu|scsX9{^`4!z=>ER4?OpNE%t%X=8 zl-Cp8sd*gp^)M%H|LIbrno6aZwy7%b@g~@AOW~MqHXNIy_=^|k4I8G)^?z^jhqHzqh2jNXtV!Jl#4+JZ${jqB zG#tRsIsV|i9_xs>I)#DJe}Bw!eW4{2*bbCZJ{iK!XTG+%G$N>HQ#4SmBzkQPCqNIZ zp;r*g5{$AYJ$WfJ-P=do$Q{ZOi@8_CPBlKYD=DG!l=J(E##J%BR+aLZVuS-#W$u&o z+z&U+pmztdRFU-f_`$1C+E;Ev%#C%(8nOXwjKeaDtg?PzfQgF5n9eBS$=GM2g)u$v zKl&v*f~2!nbya@Hv_kV??GtQYs*LkH)zRm11_y=`(5wK|?&|WGk0;I5%IYbGubg%_ z*liV-n3~H9J+IEpqEswN17O^}cUE+|;Du4v_Ew@QcXpPuagw5>{j@q%asMeQ>uPK) z`bK~wTY8Dfm^G<6->Xqt5S~uYlr&bPYAxjN_cNZ?lVu4fnMyK28GCp8iP!m=Bsz1F z<_@z^F^BlflT3|6VfG%79*24)&J_>MdRM9{0woGv)^J&*9py1Jo!^-fKJjO}ij|_U zP->~C3zhWT>-Ew~K9NeMT6|-o7GS3WKk0e!;Qb>PvteP#^Tt<=)idMqwFH#z>zRku(ziUFP;QZ-`KPf_$JJKvnk`^ zc)LCNNDKFw+*Fh#Xu|{+uB)eCxh;z|Ia?06K1Dc6%~9lVRAsdl2cn}zi2&-hPE{5o z;HW}t88-3niaZ>dlTsK*EeCNf>~WDq80&xZ7zV}Hf8Be(-`bV`aKM-L3x4b~Whh^T ze{iAJSFa_c<-6+XwBh*Uw&tH~$G$O+Lruiy;u*_)X~aVnQ2`;O+!8FUVXL zTH5b*v`LX~z0-5|;)S1zcea2Oh(D-1Na(sXObE}aUsXZca3*>`iQYmmvcZpfXx98g z1F#p6>wV_$@^Hk}omdv-yFEf~s&~}I&Mvu7zrLca2(uYT0s3q~+0lL7oct0tK_?QL z3KHiqVUEP?RgLf#k{~<02L~9aAP(6cn&wDt_H{98r*zwpkhL-22TPl*B-QVICS$u* zn|s9ElndEHI8p@B8JdCk6u4ec&tyKG0p1@u67J7D7(7?R6QI9yMPL4sM|Ql?pZtHacrC?)9^s3goY^vBFXll%T6xBbeBEkQx% zlNaZeeb~_kYY*lhnHBL%2eZxL2mz(%o+XY_wJZ(n_gByN2zW7&Wbe-52&g)V@5YS= zuC7`Q_Fv*YQ1G|Y8vpe_at2Uz7+-9;{-U&8Ykz}s~}*b$%M zvEM)`|9?$qYHY(pi{R#_!+Qpo%?Qx*Quwy+J3q;@-5DOxw}(bDRj{=9;GMHuu@~nO z+(axmxMD>z&tcWU=O?4A(f7mMHn*q* znN3E+_3Sq{jXAqB!J+D6B^C-_-TNYfh?ywuRJHRx|E(?u?6)jy?`Igm#M?e>&mbAOHiA;XqRcs0k%cj zgnu2dp$n+nmN${T5oz^+Cnh@;T>Tva=p$#Ee1%?r>zn#SRrqE6>W(JHv9NjwOpv9K zKh`CSr#!#*WGQ6D1#d1M8<1HIQ6Z}Es#yR&!9%uX+!V2H+6d}s4sWGKnpYa{&&gpr zt$k_U_Jhr4%%t{tsd@dn)clr4DWt{GXhpKlT9|XrfdA2RbGe9l9)u5GbbxLLh?D6d zw?A3Ff9JATOw!+s$uoa8fN0c?=~`a=8l4gL5hRCM%ZbFp@0XQOIh%^-+;w^Plet(C z_MyyNi0({Rk>2JM zSaWlOaAnp6y#`Fi>d)fh4+GrDwrQWIVa3YE0EM`_*>pR}atL)zi zCs?=l2trDimlx81%d|dlCH`W-R8{0%+1t(d`a6Yf(?U!j(@GH;-i~q9F4$f~E?Cwo zij@rf4NTh)S!>3#U5PeE{9GlCF1qBY zacF19uV?Fn={-CTX+aniW*aTSGhz1CUY3U#g)Ta4&V^vGUG)~iU{oF&%>{pzS?;4L zCnN>U#-+KED6byMBs$HpCKPvep5(;wwWlIyZK|}8+-Zw24aRT^{&jtTHqYrrx@b1) z$Q)9xo=(X2__@r~r?*Y=u*VmpD@zelInJ)i!?{V#u-PyB<+F1GS&^YVG%!&DN}nX+ z$rZSaAn;b1jjUhIEm*uTl#kt$AcK%RupaHXEsAxYMrQFsD#QBsk0hWzvXHiO>@Iss zEh9YggfkWdkF^#W)rGnztnDAI@$G}nrb$KWJuG3{Le6B|OrN7} z1+Ip0Svll>pF|6_f!$o*Ww^XVv1m#d5c}O}l^VCEp~?)uU}Ky{m|mU{s@Q}dJg4dL zS}2BuQ_IB_(7RQ}Y%FG)ZMwy*ge$$eTtO4<+#<4t`8FS4J5Ek8H?Iz_?0G0D{w>$; z`1TJw+#j(1DUw=#_iQv!24`^R%oI?z5%vD2(HGk~h6(t^VHut8Krm8lg9aGHd`Tj{ zvR3uuQ;peAuNDJ|Fft$W#3jKSXA}Lh%!pc%3pc05X4l{?pNG1dq?(FLuuJ0}ZDana zn9uL$rLAg_UVw;}DXyas1m-B_P8UsSlL#2Bhac&6(g<2XB4;V+&m_ygbZft#j4uFM zmuRo!le#lbWD$XTV7JVPntu(KtYWtR@e37C#$w_|EyX|FCe!@#8R~t>?VxG|wFnwK zfzs0+g+IvarNF+j0+Oi-gPl-jkDr0X>H>my~lbn?f45ht| zJ%K(=IB0e&3YXFXhOvV7{=g+VzbGu5?<`6vwQ=X0swCj{1{%g?idX$1RP5HL`)q61A-=Y>BSWrO^R!x0%svwn1#Ufo%*YSTa1WBHG zAg{{cCQlaAozLocTOF~dl!Ohp*~jUK({07UACABD!jyix+0%UGp(cV~Br612B|Q1v z>LYvaOR8qljY=lJ7B*)M7m&SG4pYM089Lb^vbK&U0wA7_a)msD(rm)5ban_JV-NoP z$t*ODEoE6hh?dQX{PAq)ibkrD9&2uiOYu3Ni%*gCT-=Ipca4LU3ZJN^#!!RvN zzb7!$tU4ew2JI@uZ5{ub3C81Efp{%)HVW@R; z0|UmRS#1p+TL1XA6=6rhCnLM<&1*P>|dR2~iW-?O``Zb4+`EI$w~&4H3BfC-9R z@@H3JBV6YTf+46)H^~7iCc8?qRmHgh%Lobc#&Ak1Mh#W=QQqqx zuT6gXm^5{~*x5LPk0kl6DSByWh58VK!_&SVt!`T@Hi2J>jfqfoUC4-S0%5tLDD!xZ)b60$lSQ!Onm$1@3y%>D(By+;^u{R%G5yZXDB5Ez z%ymBHkei>740f^jZ4>vgBYk_t8(vu_uXSU8|oPLvI;> z{t;O;^OL)q?xAnHN_zo?>jlMqGJ+CVUn=-b-%QHxX2KoqX)jA-!lhntxT85TXrO(r z-99QI3KYS-WluJY1l2Zz*`^&j5;94ZnT96Em$8C`W<=3=vNrlu;tL>E&WY}dS#D1@ z{P)`f#M_#|B_`bB6dO{O$`vTT(~Cu@RP!sK8yr4OIpb5{dOpsff*{a6;z9MQ1Zl{F zp#lNR_iTEoy`RrSlNug!hbKdp6YgO@HaEp(W~1?cbkxL`PN%iQ13#jTo{nTIGAv>O z_J79Jonk#IX;$gZ9@gI*rnZm(0p7PgnXu zPL&SaER1Q+0JnOV4S;`n95}`qZ)b+9C5x)(LY>j=wBTQ(n(A~oD%&})IaSnpUBFfOYjpIE#Fz;@2Rfo%%_e7swGHd!O%`rhKep9J}XCYE>SYF|KtSM z{>vJH+le^~ezpUnC>K<|$D%KdJsiu5z+E7uVTAimQ_ZxcOzxwJ`0e&*k2)2C?<+S!1&=|4MfDFBnC5lo89# z%ge{U#hBIt^Kzh2f_z~tIkH?bVkHGm4}l{w?07W&f1JH_R9sE4FNy>R!GZ;c;K5x7 z3GNUexVyW%1$TFs;O;WG1$TFc!QJ0{=iYO^ob&EoZ>`sV_3X8$*WTULRllnG^)9SN zOL_JigZ&qI&OT}V$zOAQYibX9Kc7pxN}dtFJh#QynojPJn|#*;Bu}4}5sn?j7ho)2 z`vub+j|7K4mfa+B|7+;`W)yz(TR^V3(00wdp`zSgvG!$0P*Q1RE%Ms@fdzgz<(B|O znbkM2fkGv$bo`Ms$?gt2bi`CaT5~@v7y6=%`DQ*YyZbKYQ!dq+;k>O!)&TR-s%k|6 zKybW$1tpWr`Q2L38{5AvzaIZf^qu6-$ztU??G9PTOTh)Cc?`jCV$l2er}yti9vPupT)Uq!6)@?h_>&+z?kc5%>u2-pL?Vrd-{ND@?m5Z}lV1FBv%29F-WyV^(mY#bJZg zR0ui0YAVn#D=G$8Hm-21mW+nF!MgeBQ1 zPAT{uixQ;OTT9J#%Aq;17BaSc_Y*>pOQV*pxrtU>eaoon$k|zehve0y1$H zyMt!gh=sK#1NZ_LFIFNA7^?z=vcU-j=b+*cY=xVg;`PjPZ)`cmHFM)rpuf;{BPWDr zo4BVVA{F^r8I(u#@aRhs-LDBf4|SNdMr#C^oQYO1ekpZ}(C5-0GX)kd_?3k@Y!$u^ z7DVo?0<#gHkzZatI~xN3>!4fT&Iq6>uq2?d{gCnPwH$Bey^L^i{}74Dl{&%28-YbE zQ$#DnGPkQ$o(TBzL&((;*X>*J<1lBD=TYBOHswwoU3@7)(8b(^**AwLX6GHXtMZM5 zkrMQyjHW;$Anq7b{J;$!PYM+z1gZIm()dsdhqAqkTOS#rYTI? zwZA+g&!o<5-Dx~xMOPcV39?JvB3?4j{JvP>c5i!1r666s175sGb&F@ix#L+c1Z6!OV7jntAkim7yA3 zVQ=pIu&;ZNM=TZy=$-yK zU^$o)&~o+9MDeThn#_L4vgX}KRe?yTl1X<)+Ug@~aDOYQigi;qkI+eierljm)uvfd-o*MaWvQz5V!K zjNCPf}HN2^vGXF2zNg>nEo6=?k_a3 z)Z$4cVjtkw-kbLUpj)JQ@x^^nnXkKXlmePN4>D=t4w6n=cd}PWHp!ab)DVLJRdA#h zrL&HLv5lG8KI=V~mYi9y=(p<9y@gQ(VRR;273#CK<2#$Hn3jt!Nj&4V^^!EWx20lA z7BEu^1R^|r7LARKO%PzRmxzD>BreVG?SLekl)U%rO$3B9HS>cbnYQtD%cai>83pnQ z8K~`s$Y8!(w&`jf6YP_tXZgvO53kCtZgWK2L_FwiI(&MCi)o<|%2w3#np z7Dpr!f|sY7$3`zIj4ZUb(U>k9T+)P*Vjk4secNP%l@;rV*C87~!CM^Rl7A>y^fj1J z>goEBAN#+v7wkI4UJ0HBpIUz7KFh37NAm}T5?FPrjr(N8KRC&M){xcuZL9%KryF*k z?{w;VCv1_@{)1a~f5*;_hEl#9hzXPI<+9l-THrp0A83G4=AFu7rM04H{bO+;2<-p{HGK2Hyl3 zqS{PDd8{Yqc#-rRexu_j z28VA5!*-xgG`)(V-^Dy?^L_%UM;feuwqTDyBFi0%9<{-oXv$ub?P`gytz@`9@RC;7 z)fuqn0Qtyd!_dBsCg=xE}L>xJmDCy<4Wy<2!i%x5dy_{&l9V{fdlMU~KA{RY?bJ z?L~b3znkxW_1ZI}LAW!Gq5k>d#?`K1t?lbDvah#d%5}PWg6`;gu0^CiFHE}~oCp}A zh4?3zoU=NT<&ZtK)WA2tWoD=^mTxQ@Qyn4{h;qY{uY3WX_-`f>8hV0b6dOC+3YbQ- zH6}lXe+ncPSyVjeE|)0di71CxvOP>Yldu9Jb~BlynkP6yKeqA zavQyqt-~DQOG^@7y9@P+nwmaEN(|Jz8&{a5mLyfEmL=P$8=gi0Xi0FkFVkGE7mP=p-=H zbgMD8VnCN0R`6x8Ge%z3Iix}Z!5s;2xXaoF_0WvR#azRu z@PSf8qR~vcYQ4klXwETCNkE9&PJ`U^yIS19C5YA+FD}E5Q;+D!yx<u@H22pdz-Ls?Wf#|J`_gw{9hHpGpkQ7Fe0?o#AB z05xdA)W7buqWD`)mlSFiRzqcDLTrKTTox(nry!AO^F>%_e2^#SF%z> zMY|-aNLAk!Ei9(CFHr{+5_SIRL(-6_{&2lKNHl#{r1=9=lEERnYMF>=AUm!DfOPD! zbIKppFOSLMjLYa|McEqR6Hd^b$Hgzdt2)Xx;N9(^}pcTcA6Z3RQs z-&I>t;B!GGLopP=Luulho>`0RjLCDLNG<_wQ( zYk*Yk^3J{zQRWMx=CmtLhjS**CDAo;6Z@sI0EQglp4J_&epz9o1G0OBp=Gp-Qu z6oZ(m6*=ClH$I>l*O2D(p!Tc>cL5>`JUPNqfG>eiU5!8vZEIbROP>CI-_wSTKsu_f zj98@Hbyiyzv`h;~4UYB`4!9OlpM$scn)RdyfxA?sd70eIYuPx<~ zl`))p8@_EG^?k0cgs~EslG@m1ioJvpw(`u;bfRJ|_+>a$#akg$u-_^eH9sw4kCbMH zXm(qr=Gf&|!0UKcdP#~1iAKQwQ7>8j@r4ET+T=>;(DI2>^YUw4Pw&S%w?M4yjJ|1; z%<7lcY$9Xcv00JW&SwTn)W2FRfB1Gil4s4E{QeEmpwzw@t$JqwNL>qusgys8UsnH5 z>F&tS(dLLA!#p2Z{k|bRL>T>zaPl2cXy6eUe#2y$83NRrH*xD^dMEG8H!r{%>=r|r5grqOM*r>6Ipf&`3_F4;Y&jm&0oSoa%G~T z@cky3fsx$@u@`PaS0p8Ndliv+dRxt)RTB_TQxntfMFUTIPEz2Cz<*X5qbZ-!y?nnh zKnZfAU?BQH{zZXgzM85(xGUV4YOyV-aB9ufYWe;KMqF)lMoyaKsQ^!F{UPiK#Wfqy z_*g%7^C9ehO#g7`kqo3Uw79U8;o!ep>V(y+>7>f(ZB z-lDU3w%g2gQme!+EKGfc@DaGJI^Dg<&xrP~Sj9>y-8p~^QmIt%+i)8?j&<(wNk}kX zQGF_B?@D!!T)9Q{vMPiT|NY~C^janfnz$kU$HmxAZgSJV6x)AvR}Cd6 zKmU&#TZR*Ii2q(VaNj3@tDn=TEH`#;7b;YdYe z1rHHzMD+c==?nVZcW6WoTlD}rogQPap{@aB4qN4x7zBY{EXLbC?PeTbbL0;mKJocz72!OY0qdAFe|`NA-t73h8iNg-=Z z@y_bfUsJ&`K)I;~D+kZDulz=oc*$OU1JKI;L8F_Yu&ACFXUSyAg9o$R5!Jc$mToR2 zcry(24Z2;xBVAFHb5Y-Pzn28M@;4AEiQ)*gA1B-KNWg0w`tEr^=3HNQv5o4}1`)Q1FQ!2>7u1e71onZnYW*%c*RLojhcdp=&=>^{y1kUNYHcTZ<|xP? z5L3Gk>9e}h+ZXH+t~b;o&qM4!6o|@W&mwh>%B3({(79Qwtg}?Qchs^4WTg!BR?1tB zAdofJ5H&c z76YGq^ho*j@TUubPN6oBbsxu#nKqJ#|0Km3wj5cl+hbchiv@v1pl=7i+_+XhKR(Hl z=;zY_&l${l@z9_c;TnsgydIl7h-9EG;t0@X z(sFrGqP>$g6)hetdp@qGwfxFohsI$qMwgw-c|crjeVE!DMe)18API7@NVI0dF_jN{ju+ls zEgm{M%DCs(5h#c4mFQecxuUCu%_|FE{B)Fj)Ya;PB44JqW0@2LD=A_ z{3n#rY{oFE@a>W{15Sby?9xTJO+?OFGdb}{alUNUFnF8MUDGwcn|F@@gn9bmk0wve zE%>7>sE=>??Jw)FpfmWLqxCF&P39!{s)()d(@AG>Y^_)Pu1$$D?*KFRC%FA+uY}}x zw)Dfe&xH81NY7d`sWsH8d9#VqKF)nQtX%ohXWb2!OdPv!+A?DdmV(SC>ky+cPS|_5U-t&9E$=M{M<6R6j=_wUs>z!n zL2dU#!Y3eObf52(w9g6Z3+OMk9;+e#k4kT%f-_OU50fcKr*~mlmDCnRbK~6HH6^#2 zkgX-qn5^fK@2Rx5@W-DCU5k&%^bvys#V^+DBR$W^-~+0Y=E|G}@;ywIuR(OL`1+k2 z-)pIoNlpS!@K<;;sT57n?7n7t&S>Y&+fY=bF#yjRG04SGFeG-bAC$ zmPcwvc1?T}b((0&x4&~Qm1rzBdo#>)6MtyN`)b2LQxvV&eMMJblrFx~5ElZ?#kp|b zyQCZ55sm;`j*-bIpO8G>-GH8Y@dqQXj-a_t25z<1NB1_(ssK!_YrjQMw&4%m&>|>y zN~GfoQ}F&u!k+rluE?p>YC^xzo`qR$EQ;E&@vDI~Ie*w}7pn*zuO<0Kfo7>hVt32` zcbOID^AA29J!W}+@OjBk8PW&hMVP;r`4*Mh98Vn(fgc(!Ng3SXPx6rCthmKNX>y@M zS7Zf|M9wbYN!0g^!@s(gl{Z|L>T8#8zq5Y8%o^$ZCPlJQ#T}uM{8nG8Rx=Tb`rheO zC}BpHJ^q$)8jKdI#ubjjvA2W197w@7f&zesc}}_*j>`@KU<}vp@yBVeB^POnR@>#7 zoxeS#t1vq~@sQk+qcxckvh0fqit8*t7FUJ+Yte@)3cgT+SGi1~_y*{D^vF8cauPpXGSd!oSM=@f{nH&}7AF9X(e~H}$F=+MsJV4H$;<2M!xAM!XIl4`tp77| zk)kbx^#R|O4aq5TSDZ9x+>zVm%s2a5J8Ir0Ae%T;=CWvHP&1>qiejeaq;?;l6h$Wk ze`MR%Vm6YmdAQwfLBqZWun^LWbH5=6%!3kz2kHl1J9hOnx4COFD z3y5B*T<&{H5JY{KyLyyyeW0J)jA3b2+T3EJgRY_iXU^dsf8VN&p;1YROnxa`75(+$ za;lxL1n}$vGW2=TMkbU5O0Oh!HWq{=_VK-ax^L1MysJZ5=G;2lLFRfEXV&;bSZ5fU zl>qDVyuD@k>q@h9hRY2j!^>xX&!8Je%!Bl&!$4CdTvTLGt?Bb50v6p;v=|+!Zp$Oq zx4&Rhav>O5#jebj@SyiMR$N{>`sz3^;JDb9MS2}kT_J>Z$Bp{EaycTJ3!IKk*4NN1 z0i5kep&ESSGORrB&Z+wLHCv+;0Xg;9A(6UWiox`W+@Vdng!hfHgK zP^#@|_bKCAZ%N6zu+rEEDBR$mYiU~YeY&|1qU$VP808R8WViO^Qn_ft>Mfxm<)nAu z7iur_W*a=&t)XT4#?0nhp{)qL;@^_SXkG8|x+JIkFRJe9f!p%;q`(z&k7??xrHeXm;#r(j}W(s68v$455$KZYTm#Vxep4L6|gwOBCgE zyq6V6wr`&K2-cOzKTBFs_!^K3th6Ni)(x-dHbJLina2kxHO8mDVHKQD z$vba8T`#pN{Ej+kDg6vPxV&f=_Ts-`dw18{qKk#MVEJ{)@l)AU{F6)8gd`&;SWa!o z`Mt1W%Z$O*U$SE?H!{BPnG8+vg+`_`G|+$)mo8P7Hg=(uKx#+BcnN zwZG=Pzl}l7$6?mLX*pk6Tia9qMMmPI1k<)y?}_(2jc@$q-94Vq$3oVs3HF1NS5UQ2 zr^LdtnWc7(pTSBWBjE*OHy<)K$m@{Pl6e9J zzR+3e@k1dc6>Vex*dUw<^y1t}f5#~=m(xH$8uTqeT6U#p5*Ir* z=y$8VthOH*e~fg(H=GTTQ!*sy1M)fdr~UX)9D5g*e7oy-Uv)ADs;!ZlchM$PsiC8ay{ zs~OL|MVHZU6(jRZuG2ZO3_*hEkAYogG(W2XuhHKEPuDokb)d`u(zHIbdyJ?RFAGXq#+lse12T#+K# zK*6>&j_a_$fLsB!x!n~dOS7P$XnvP5{>1op-~y%m$QZisDZfptQf}yw&X2pO)KPA) zgg5BtDTjxZ8(JTp_eogb)3GAhTjKVqCQp6!us@vHNDjbrv462d32(RZynd4ogGDmF z(0Hw`OA(J+>rx5-69>qkM#y1IwK{1+W#)-!S`HK#r%xJB?e~kCU~zp|;c<3H)al=k z^I?qhqFhG*OeAqNOf#y4#F;6(dl&zK|DO2ul$hTLng*!Z%Y;yEgM?aAHDj`>@2*d7 zwd2UAfi3qQ8}K-I-7l@dm>l9AeesasH|xy$y(Dz8aAqmPyfCDYHU;_>4w=Goo}U`;ov^Tup6XVjugg*TQ zI*6tEKE0g{`%3X14V^E1&cne^%d<@v{*}lxtcIvPG#+(mX{2=`$+mhjZ0kFjmzPRN(L6`O>F1l`KH z-PIh6$YWLHH~SI)=Q=ZNY+>-amV={Miu+Z2``w$U`DPf45#Yt$ZsrQ#SyE#)#}{sE zYqRA6gySd<*z*8Ra81~^*X^*YI#?6NBR%cvgZ(33EuWnUiM-LY?@Bc9JyLGw@YP=* zSW)C63n01%LY*hZ4?1$kUl_NtJ;8}G5gXW=rxv&r+*F?}G&aE1a$evK7uo=+-d2H9 zGyLwN9nXd*H(Dpcv>$EXsA~#wolE|+2+6@C%m;7<^CMjl5%zF)c{1mT+B?&FRs zs!!=L`oAK(TT~1>SR7)bcZ6I~uiT2(zIM*TJ*JJ>b*j>%f7cJ3VwwK_=_scuc!e(|*Hut792-ZM8A z0FIJE87Z;Mmurq9RLSwVtmeV6aAs7$QyX(^uN+wMWxss{7KtZ$)y6E|@Zfv-n>wA7 z0h`;97uh2x*t(>>7KZ7ld63twj1_g2&I$!00T z$LdHK#K$3u&o(1YL&YhOasH?TCqxmGEDVhQM&}K_>80C4;FJ4l|5w08Gs_@czMV6V zm)(QUc;GkcZo>Kln`EtX?Nxj)hS8hNS^@BM(RJ!Fwdn{?b$$h4u_L^WB3I4~FSu*m zwxbMp(Yb5e;|+c@Y4FQWx1Hy6SgwQR248kjjE)243tdWg1eL)*G8N<}fQ8Ru@JH3W z#m$yy#W8Uh$JYZc>&1kh=WfDyC?J~AdVnX1PN(NL%fR*31tXAWTRI*Vi|uPQ)As!& zUi-bLP@^0{w?-NINU3IuOU=_@udCykoRpkF;Le)a(#g4NL6wnH{bStcQoPJu&TP*; zDH(y15nM>K+*#U<-GE)s9dLFg9v7XvzWWi;-z5Ov_ksGJP`dlE}C-_=D z{p%_n*d#vLtfw|ak&$DCiIdmnQ(W24RzwN@yAj?%mOLwywbo3R8$pUU_j>h+ciYM5 zBN}XrY+Jyo{Nm)=RQ!!`*0KY{LG)0>_n2EI;|n)m6H%uw49s8)@o#SpIbYerZgsrI zqQa?-G^dZdE9tGCk@KbSd^D}fckx5@XKJgKVzobF+akG#Kti`BPPD`m90k#==QLcy z@2&H7n4X;n?%=SLz)uNRYKTZ16N3?I=ZwMXLIu*gENs}Q9q7TC_{4yDq^gFV04`#4 zecq{th8?|=H*7|Rd#BlmIU8iZ1w0Vjsx!cK;7B?R)#}aVxfkYj9!-@NF2cl{xVyn- z^le01tDzYtHJHFv4yqz$Bf?*a+Hdiw0EPGsM4oJ`&iLgCS*W&n5|mIv;<@0u5d87V zOyATqAvZWz2R@&xqd8P44SLO}Qi(^C@?k;IDw}6|KxJN@GeeSzx z2*p@p%e5hY#*k3S!ktU?=S_8OvJ#vsGV@HFNjITh!OO>@^FSu4H$rc>5D+*fBvJN} z6&BP^RMFj8%6f+7AJE#XT3BN;9xozTT786=;F<9w&6eJ6E2v#abBX4_!SVE!Gp47= zH=dBv&wN%S9xDy$FEal4+A}a&n#Hn9aV7Nt1zbsWGqX(uo)t2e=3D~)QQMVqqG0u` zdGFcroHXX}?afmOQH28qnH;oZ+rt+W;p#_BD#~KmXA0hQ3L|6j;9oj(tMq3s+duWU zzv~k!eW?a-uh|E)bq|wo-un%-EA{Vj4xf-Kc2+n|!F%dq5gPllrHzQr>i+Vz2pU7v zSUo=;LH2IE+J#tgFzO(mXpEJBIsIM~T>q9z@GrznVy}hh%5D*EvPG!By}ju26)Sq5 z*mWh7fFTHwQL;jj@**>n!sE@OGx&W4E_nLS8iYw$_n}DHKCC{UTVm52Z~Rbv_bNvY z9^rswQiI*^(;TK%6^;Np2OXn7^(afR8^4~juzh`ya+LAhd8UyT3aV1C_seKk)unaQ zTfx5Zi{_X+N{-#7@fTUb;a@KN6a%=@*>Ab7i`W-utYO4!s}hLWqY0ttYK;i$NcbYA zydGWO*CtU{7y2Y;{Z-s7o&*)vxM!wk2*)-1@3cx@cauY9pJM z)+`I#Lxx@-^VeGmmiBNUp+N_=l)dNF-VedlC|$NDOHv8!xPa4nN^fmPt@|b5Jm*ps zqKyI@V&kEu;pS!QI4EM#&4S6}T&d_revdQ)z z+LzXwf|4D!U*N#Nzs;VhEvVk|NaPJD9+g2$W+2C^#wKK{*0jB5Ndp5n|G9=E(SP;l zoz>cCEZ`Vbb)(SoL7G^svC`;Xo&~0e7qo~gCZDN`tKv-BVT|s7EYx#962g-qA^Rjt z0@NBoH9$~*;@UZ!LGwxfna3^~8qGpKv=hZxD}W= zl{sLAs~?F#sc%EgCDv5)b^_zsbox<*U8&DYN$w2Ea7VVT5ozl?=SdS~TztC3OOX+X zv2C(>X|B&8&|Fy>z)^i(oa&!fyVkM2{aXgd*e&-9e$N+)1YnyUx zj(i-cs`(qe!8OM-A4Xx~ozRMxmwd%pIqF6lJYPZxF%;DW-lI z$cZ6Q@zsALcWq4zn^~VV{L3iMLfr2Cf24sg`dKaU!BtuXK6`tY>*=!iI?%~N1Vvzv z-DPuVIawsmP)hHvI2M@YzzrrSglC2Tst{+G)`hvPM+1n^aDf0iFD7I>Nd(?Y)XoxUk5@Z{hjMW z;HC0m;BU;m7J~!bt|{bCqX$)0WE=q*d1Wm)mR;j>1eBCMPryqmFB;@T<8Ae<96$18 z&ygj^X#H}NF5T+DWBS>0NEkcJdG=GoV0c?|dAHTM=Ad5zcsV@t84sh^CtnU?9ojzw zuv|cv^fP~U^b3CvaoHXBkfqa{BIz0&$kb&)A5M-EEjZwOHJ)Zr5h;YigTu<;6xqJ4 zW8kBKDUuE=N2=PFWWckSt5K!mOcHooI;a_vmX!~@7jOBabC0}e8%Rb5nUG^QMonP? zhCDM!nQckwlt|D45T;;gcmF%0<11u=p6=({wK&q&MEsLJf1VW;fSNH0<}Mc*Cj5Lje-1eI>o#HEr0o# zS<0ozK!&}5I}R=>uIT;~zibB8Cz-boBgxC+q4Ur0(!8Q>qlxM->7g%%JX`RA$q{b9 zwC@UNs^jXwM*{Sdww7~X#?+k(215j}xA~06!!$V+#Rs0uMjz=fr_(ps)HNM;H50z- zj;#$Z%$U?N+)oP;tR(ozt15ulk0%ike3oLkgttIV6d%M1Ztv3{- zVIRWH+5NX*v^%Ys46qNrv$|7hO_*FS)IxGC%lB$at`;d)tGmY`{RwZ_myh@8dWc3k z3y=x9t_E69cQ!LZ3QrALGH~@tXF-v1O%G1v%p{^=s`D7*p7wlVI?e@1tkg%X?T1rx zdXxvgfJ<5trFP-HgQTSReFTH0H~XJ6Lp9o>)+j68)BgpT;2yp14z9E~sRwf-z2TJD z%wzq_LS8@KaPj=FB$?F_{u=A9nnn)UJ&skY2zpN3irwsQv{Lwt&6`JC-e)tX>})*M z`7!>AD%NNU3E2|fCD2(;n&Upyz%<^UPI zZwQ@^OP>&ERQPRBI&_M_CixScUc`8_^2?i~?1B9@*&ZG5pM%Od|4ENFg@vN9W*aEH zi63Wp%n3! z#S6QjWVLTEioG!pvi^|2M?lN6jni9qP0gHm(vtUsznI##V$(Ie^mYaRO@%-T^E}o* zvGc5u8cd9UgVh~Kup3Qb zjsiO!DqL-W^>aHocN0r5wmsR96HzA95sxn?M^P7E*j^sUdu7cuKWm0ICRES|Yua;2 zt7N>hoX-w|KW)!ndBvf+qzePz-z?3FN2;DcTd^m*76Wv==@gs}P((i?xTTO=TYwV7>l(;(_tsv- zNTV7WBhyKKL&K1&+weo#mQ8(T;Kvw+O&!<&4t(eh>Yg@4s7>~5LwvCdTvAyzNrR3i`oiHp-Cs|A2iG@A+mpHqRb$Q~mKx9z8vPfhfqRLJ^My($ z?il2>U-=I5gu@hALOc5uSXqA=XGs`Uu;2CR-{04?zZLt%tqPMQ(-}4`ebV;rmwxf4 zq04p;eg=U(OaJpEtFI#p!$@;zEC65@tXJ@q(IVi65hE@UZ7hiaXuM%ZQyqVvG-vSr zU4Vzf?i+cLmF21%_JVPcRGI1S`PMT-Ps=$_p#4D>J4*B`N0dx>uLNyvPZG-Guff9w zx`+k1?*i`l9t(83uYNob3LIYJF4m;YE`0#U5@qr2`LVtOXWHv;l3aiZFCKZ+X;O^1Nw#()GFiSq?=S1FEc)^ z@h3$^<9%a3B{rXOWT(J@3QRm#G@{d{|NSkIfd8yu&miBmW9-||n$W`_f_SsV2WUc{ zOh_Htyu&c8xM2nw;DE&>1Z>BAgJq?RyrH9c$QCm6?zx`VgYnZH;aG~ZFM0e!v+$?0 z`VP;)Bx(aSX;im@vyg$$>TiTx-cugCFBB?Z2jI_3c5?E94MgYb9UeJ3d5Q%a7ssJ` zK6g?;=W6YywHVm?j5n}@BlWBhVA%KJ&vJ&Q*B&D3qs}hYXV3ncf`A$yU9!KVBDwT` z#N9?}|Bj}$Go%zRk9pq%aFa;Zbj<6E8Cj)`eFrdJ?t2Mu^MaNO%t)k;y9g1 z44*HS{x5Qp+<~P%Y!{ey$RIxSqtYAv_NH3$MjG=(w2Y@+|(teO9Bbf;MUBrouxa&_rN zXe7qfV^wPn`VUAtU9-s^TSb%te~q&10&v)GDkJ=d=QLzBULvnUh}Yy>a^1Pt8@O?` z?m@)-v_yHRI^!nGcPc!no82i&651W_+v$|&NBW&!!lH=9#|^`6UV~uKD$iI< zTO)ZOx8U14q7#G1D40GJ?Oez3lFeGv*{~J*FCrc~SNxgcGiUG*H|zhKK-ALg}ziYqCCavu+JfD zm`?8ssYO6h8PuLk9f5tk=OHYJVm-8_zssab+Zc<^>0M$gLJWCDFrEa-pnpw~t?YS#iZE z#5dAeEit*g1xi4Q5(z%>i7YgZT{;E4BxG7@-l5{5sKVni2T%#~9nbgO^0V z*2RRNYfGFnSa)+i536GWEwTi+)Olu*^pY^189k=!h$Xivvu@E6`1qbCYdIdMVR``) ze&6an+C2EK-2)4JeM-xN%*#D;bkP?)6L3nB7hk&U|gDNFh(Ri0gUTG@5Szqfz z0R!lvmfCOh=cDRk3dFkS6Y&lzmngyFji|!UUv(_U+8&+9Ig98wQ=GXdW+4Jc()|k3Pvd7yQipbsiPDMY9x0UW}V;2!+!fj zn0)2@Be2v=MSn;2oZLR3+wDNh=RR8{oh?>kKVPr|FPww(Z z6RAKH1u}?j3x6?<+hm1t@p+21nqrA*ssLyC`r^<8p06Qjf+AwGB2|j>m^Vf*fP3pF zQrLuVMK}@C_)8a@O~l?xLEa|>oZ_`d96~d=sqCoz1{I3hGZmdDM4zP3P5Nc=^$fM2aSxs?#t;{i=Ww4B36BOnLcE-`mJq}>@ZN!7zU$O6mZ9wb};ta{@tL*uSay2(*R8Kdz@N5GBi&!|ngYVQgQ2_WkL- z1M(lL6Vm^YI`pUV78%YH?>Xk~7MsEwU!c*;;)#bVW;s?i};)x64Xov?V zx@(ilJ=Jvq~PT)z}HqS`08X zPms0~?To)p@XDFk?6h$0k9q|biG8`67P_Cj4|F4gjWB-Fh~E6oPW_+O=}N`qL}sF? zlaj(2D_pFh>r2-_Bt1F@I(pmtv`*QeU6aOANfcy_$WW<=Bj?G#`0zH{aa6n=>dXw=hG669qxSy$tdgHHoD)YkDCy< z0IdzpCO~*$i~$Z+0Y12X=N2#ihP(&1?@^1S6jumVmmOs+PO_ra?zdm2?WGH>bcUY@xMq=&bED!Zy9uE6&e4=@ zc*UJ%xt|w&1a_HJ(i(4QG^&^@$IR3h{`lQ)e9kl{0hqC*J7+K`?m1#U>(fb791mt1 zYWT$;;Sbku|6G9sYHRX-9%9x()qA7#!m z)S2q^MiRTvO2Kv~)7i>wH2b5tm+u-XX$hiNN1%x%?{BdwiFFUQihO*zTBZthFV!qR zs%_x2X&*zIg<@$n-bjz;yv&Y`KHyQN-JL~ux{)jXo*>oNA+bx}XEN|6`*l;88U0lH zpGJ8|CpGrTYv3ZVt+L!j1fFsa8kVQ&|l4Wj7|EwT8lb=9k;mRi|i%;Ngc%;hMwmE(i7;uDv@4M0> z_mD2$23<{t-o}zSo(b4IXiv_+UD-4d*H>&ms&$Zej|1&cj#QSVK~gT@s*Z_qc|U<5 ze-E+mDNL_oQDUi>a|zzPIfjjEmysuDfD}2K-M4ie0@=m~w0bcQ0J8Rs&$~-nBn;%C zJNpe+7GQ5Rt+|!^N81#b)eDyXaC=_6VrPtT48;@!ZX!Tq zGuyFf_WJ$j%^iDsyB_zP>5*dh>{21r9}2eqJ$1QResL_ilbs~q$^SQsR`NSTSDc8u z0`BT*cyK)GABPgzgI#&sJGY6EL4TI0JJI6(2pcUFd^{whsk>Y(di7eOXm!y|?)TvT zV#;4p8sSZ5V-ej`JY82^67r!9I*@*;MS1Hf3OI;%LLeCA2gN^L4yx8lbv4Qw;I?62 zkCacSJvuP=lU>`rTGtb-`omg_=E-p@@K?hRL?C%y_T%2|_pGl@m|otye1yLmXr@Tr zjvM(v-V_wc@}?DJr(=rHo{5M-AV;<}Sf%>tKygdxolXg8g$}@G$;T@hTr1H)D1N3; zT(&frg5%#@HiEE9XYK1c|4CgP# zVyub>@;xYW`f9)bAhn#dIE$QHcuuwL!zqnwVl#_Zje_4bG$?dgWt8*P3GW2UXZ(yM zm*-Z~CQMn{a7xx;>BB&)BMw_q+f^RtR6Pw+#H1_ecm0bskHfslwqxMpC&{jH$^?(+ zo4l4ztSFPw0t#X9*QhX=QUharQc{`~5zW#4Pa2gBj~YA`kLJU`@cZ~@R;d2U{DUiZ zS5-@L1fY3t0CvLrH)pN>t^l>~jMJK&DXq`K%`zeWdK|n`pqqA3Iw?L<|X>tY=~$mRd~q z3Y?8&=}N6L5JiNo)ZE&7M_kmbjG9F%r&HPkcJ1B6V$(ho{Eq)`{+jG5 zl*s>fL3W+D7X7;!kbAujSl5@%`pSAsVsh+IHEaak9ee2ZIxVHYLF91Kr0!~J2&%*E zbWo&Xtib+ay&tfDu-KOQ^35-yS3>p7x<|{;8nmSC>oL>zv3~5Uy9@J|waGVhrvH`7 zasn4CW)4Tj*mk@B0lNxLWkt)8d0x8iC$^7$&PEzd?T9KjcRIRyQB^0%8hG9+aa{)_ zdAk@_yPpk`?c!co141BCkVy^}QpuWmx?%F9pYT)he({t zM9<6EMn-=9{lRiIzzmcnwc2XrNdz;2R5BRF6(2fOuYYhqnP?DRKV5zgfs0k8x3&AR z_N-W}EKBHUhCex?CZK|>}A;LMtX968|KF^V2WS9?Co{1YMpZpCCstQp=>Y&G*#POoR0e*)dUrw@aRMBQw8h<>;)E6p5L|*=irY=k{m%J%&bhzt{K-z9oxNvf&8#)+U9;YK zEkV`50zSaa9^!NT@+D9^cO?Hv9VgplOw*~eYZwz?wCsD{_mVWLMk^f3YYyLxUU)0O zhAj%bUNNqWjkgz0au5%n+ETfd_&PF%)P3ADs7mEppEDM|PJdX)yyv32JcuBo$Kh?);M@;!%>&SlUXAab9#%u zXLmJSsgZD}%`~Wq-+8VRacO!lCKW$x`(PUD3=;pdair=FcA0k8dd4D>e4#N6weZY$ zt?_+bD{a|vNssR2 z`<@h~q?y_67Pe}uX1$>*EHu!wqpCA#$#lwcYBHJRo-2AQQWs#?WU9(l^_)mW9-ps^ z_!Tq01-=V8cZW=4RE)V%$@cZ7_fPLmG5EDqt~>5dZz~+(QQ+~8e6hdbXyaQBwOBxk z+`Pk;%uS4v{GXc((y%;q)7Qp=BGu|VA?~f6B!#`W?#yC4A zs{+)MWP8{ihI0@)SY0FDh&qj4wa12S-f2?s=+vu!c~c8X0Lp*aes>{scIbVn5~F6q zdwp%iAGCYg(^%KQqd=mlPCNQFv8oaPq~tb zHhSk8)R34`fPuV$esYOs#vbTyf1fu;=GC#&4&_(5lcte_^TB-7^hXrY!So0#A`ZG{1$eWi|Lb1J--?zS&VKEp%-QbR{oIjYV(YNv=H5^(PX(O!<|& zQ@^Su`Qp6Hr{VGFOHP&*x@NTAew=M{4F%UFgX)x9DtaR}XW+&lmEqF8tM~Nt7wEbG zNd<2Rilij^)@kQ3r^$_Ao1x$a$=_g%F(vIn|1p$?+X1+Q2VV=r9wLZWMf+U4%cT^NsEWT)bRP@v88dFpNk`}vd##|aTlsYcK|%gm2^3$ z+n#SDni0gzh8-WBb=Qs3sNc?Dwze*#o0?W8MK{!TPYp+=0s`^R95gYEO>k03$aM7# z04-8QFnq68S~e#g3(k698Ov=Fevf8zj=aP zw&=9b0ZNoMRNvF`m}7zRD=+3KdCQ~NW_h}`7WKUD64qClInHE*E}65lwaZ z&h~X0J8aUbBmJ4A!(gJeN1E|W zYwO`=W-2?8txe-&TP@52QZO2zEGgUWX0?OdR5g7D)qNL3u~Jm)uEJHKh_)MNFxL}6 zHGLr#E0d!V`w9`AjjuVmnoXOs-$LQ*J&N9d6I4_@#p7|A2w4kCJ~Ngism}YEC9*wh zr_5ohI2N}HY#W8(Q*FW)J&2XYOKEa{2nDDGz{M9?L~sXV!7iiiSigP6z^r(@MO4_@ z`ZU-?CeeIr;t;>p2G`8=$iF!#f$n6OgwA~TOU2-}O!XNcpVyV(yY33lcOoCatr;J9 z!1}uSoz2*NL^gk8(*1DFzS%c-yq-^6(UKkS8E7^2+!CmnZ(Gg>b)brWDdcid#gQ40 zyfSB{=JxjRf#w$4tEj_kp4Gc!#FrLx6w`ccO2Q~)=dC7Y--R3OR$?7sRaM_Wbba0e zr@D5N0mq9Y5){*pOe!r#0yhUM-Zw{hU_TosOtoOG4yN7sK3E_?CiPoDw=9>KpO1^6 z`31>vR+_cr>Kd8V(YvYW1e2bv0DvH)gDYpwy_k8i^-_Q9*_rRj*mk#L-f5tb`$DJU zf&!!rI9n%@3w$AG>tC6IyLh>GeZ!)f2iAOwGlj9|mpkL>ALC^%(hM6?k#MRSy2|$31W12W$(^ZzL-q9T>^V4gDc&wfX94PG zX0@}w4Q+eP1Y}W(q;0#pL#R`vKlT#DEgSZ$6Iww!TBC2F;-4I_9?Sv})kho7ZK?oUZM16k@&n#UY);OmG;O+GmHV`gky2&w zA$&M(0&d-j>CBJ)={d`5KJy+kDW3Ul$Mv`ortu*b-;Ugr*^&JPz%@C{ik>X^fo~D2 zaZ@HqCaq4Z?JNvOQ5lf091U)jnhmH3oSsX$ppS=(H(breDQTfh7l@zG#*O*Bu;RuIDUFtvYOYs;cgZ zHoWhd%^&j&C2OC#Sk88f;_qy_6l#2_ZlN=Ef=~)j?w#X`qXNz!k7=#T^zW#keI(O? zDm#{QwdFmHY^7@;K&%M|DCc$)G4{N(ilILg6{T9>6(t4fTgrF^TxzhPGcquw3F+Sl zge{3YMs!s`ee+6=QORPv(%KW-d$L)Sh_+D4(YK|aVxPla;lg(x#{K2aM-B98+o^1{ zRESjhSe*t+=nk@lcaNC(QRT{7j8dI&mt8Iq+#ZQFgGO82cSp<_&x`%3>67yD60F{( zJ)etE;0=J88_p+GUb}F=seyV`;wLUzQOI1`h}Z59bhfbYe~ATUqXWg}pjimFV`xG( z&oS$mi6TF5Db1p(;}-uGu9HKC+)g>2?U9by<@iRqt>oD z9Kkt_9zU>YslS)m84v;J%JJ;N-Cc3Rr+n~2z3f{#PQ-kT51j>tJdn`{&~R!K1iMIM zFBp;U!FlP;*pxQ0i`EiOGfcL254HY-4EE&AuR6KIV)oj8aPlSLUpO98P9$mdp%k8> zm=qks)jan||1DSJ~$+C6QPL~tc7bq2}=tN$Y^}Q}5c2JrHY0IZ!-Vb8wvq}>81WGRq zK!gqj{**9&sGkJ{Trhc4^3i(VM(e;FL;CFTetY(4w-C;x9)?*!WhMdR!Af+X;+n;S zlk@7!|NxjJr{VcnMpC8edGN=xIiQ*-RRd_|lixc0Br z^s=J{rkGhpf3{x#m>}e_WcuIUWq{jlokM4p@b11e-5S>2PPs=J*Jm4j_v)_sIp`nx zF!#WzF-r-Dg~;SiFqEKt_szut^GPNH_WEo*C{d&HdY;VSYJVGl;Nz<0P20;?Z^oAc ztd2C+b0i>s7cGQB17?ZF9`UvBzHIhPr)^N-PV6K;>!4F7)107-Y@)Kl7=Q&AB^{gZV%XLx=f z0AKX%P?Cr?il5T$^M9(ZkHT06RN}4NAkgb*2|L8K#_w>W|N1fxCTfWv?3C6l-8A=~ z|8xY7an~N;hgI=@oa`xlJ+E4$JeL+)^_peQmx9O9-)-ub=ONF{-C)jOI}j8AP)X*kiyWvzZ0pg`0g3^Ub|H<=I?UTN+f$I8~OB%n3m7|Ak;6<-{$m z#7H|DukBDMcoaZdHs5`nxD-}3j-iMr`4N(&Hgfrr49VM&mI}BAg)IqXb36JS3%*cs zB8|lBt#QA7Y#GSjJh`EQ{`iBmIjAtj>S0e1$nZzlR{E}n2A;w5s5$|PwK)O3o zltE2Q+?prsrkUU@Qs?F~P^1+u-Jf~;*w(mw?;(XXYxc1OQD>5lzd@_NP)1ra@o!Jp z2#@bh7Is1O_Ekq`=-7A7hE}q~n{)KqwM;%7n$rApN%QeDOT*$6-uW#ZKXCCezr9Rb z2PmCu3IB4v20Mye{l@5XnAEZ4!UOPV3F36 zrEW~l*m>Kx#!m7@?`=8$FT0360*#-yrVjqL%vsfA>5q7qNeTN>jzb>_Qz8E|l*!~w z04!m*UY!ki=B0?8?=BH&yifXLRj=~qUibrYZ}MeE+MxI`Mj|y@rTt8CTP<&YAD#7wAjt|*lltld?FkS5PP-Opm>8g& zD|ZK0y0H`)yr))Ox}`9)2jp0dYj2CrprcgosYyDp@ixz5L}MDoK23jpD{fdlqHo;> zew6BUS-EMD-ps!R#w4F)#Mf#GU+0Q7sB)uf^6cbFY6m5{*dLmYjCs0EAaRArSo=Wf zOmKYnt&H1Fgw9X<8gL?RPTP|5^ZUpPwV$F_UMF4Nt0+d&!05m+BQ0OP<>>^>#k*{! z1*=&_fUg&BClegA{ml`zyR=$SWvLx*F4!vlYEs_m2~KB$V5G^eDPS!7eAZ)?!KCPK zZ8lK&Qf~<4zYUG>`ebEKHceL-94liYabYfTMTfh|kx!qNx>m^j965DyW`w04omC$7 zL{#jA34GaLB0y-c*KVr;@Sd=#kgC)8x|A861~fcpcW8o;WG=x_mQ-*1*4>D@5d9xkhhppQH@@r zsEVDFp+}eu#}dtPGvJKQJtkNDGu}0)C*OjQfrK@R$?5JZedgo!WUVU6!JNf&$>${$ z#`8-27B~3H>HRPkmgB3FXOftjD~oq`C!+xThpHC=0MQ^ z3>2d&kDmuBT@mohc_LkdYV@ncTV9bX%Dh(M%u9DGJah@T)@la8^jQ&aTof>JZDxGV zyC@j(iN|;+Yj}AWr{E7Pu~U**a9b8jZmyP}0lC+r z0J%(PaFFbGNkLdz?OO={?RGIzGB9s5!O##`*Rob_iPF{@b${dp%h7pY|LQal1=UGg z&vdTy;VavprfF2Xkw5znm*+?PozMi9VtNEGWFzXmFj;tB^h=Iu-@l(80r!@M^Nri{ z6EQTe+8q@hHo~Ld0nDHD zBh#FWnb(WYFA;Rk7VDGY)L7tdn1L0R+y9c+tUm^<4b*@1^J*1hq@(P=uYdB4(~_a! zg~Q$jq-YK=G{%_ymz^{&glQ}`rWp=YPtHhnYIcF%KX&P#=+WFf$XWM&&-h84b!Dyy zGe@dg8I}`l-_rh|rLe|l!|+25r*(0u)hPVY_#J6;ZAmoGI_P3Uk)QK21qzYWI!kRD z0VBlu*bLrqsq{KRUli%0ph|P6Soc=#!b#t^e?~7y8I>UXfjrqM>{<2hARs8m$|#K} z`Gdkiewo8uQrPVrZ-_s(c2e(9lh*MVz-Vx!E-9ljCOt(`-7%Dx!q^3I(jAoZfrwP~ zaka9MNb?vEBl(ne%gsqKqNNKS;l-6nGCxyiG*@k1c$huar?*2I&-mz z4Zm=@A{BO?ja+D;32-Xjtz|B=)_s97-HJ^p5~$Fui%&1RIOR?ZIzDChys_@`HBB8w=!%!Wwfrp*nX|#CIQ#LHPSpVq# zAV1QrB8m86SN$l@9rEvcf7G0WwbpR5u%32LMqhd^gwmM!}Eu5Im2C2IBc@6 zbSXturO!vLqTm?lh`HCl8uj^LTE|jl!^PFz@T#tJoFPt{Uh#)W`BP!3*uCD3(l@o~ zWwt)c3m={KpI*w);U{MzJJHeM%7JA28|_brLl64CTVV&xixUJLA70#Y1gQs+h~nIW zWR&Jc6({(fcK)2seCCT!Ytw8@;C=A>F9vi~Xphf1*B0I`ReA+-<_4Sz8H}P~*}i=M z(0UP}G|mkmao^;AW&=Szb21&UkF}(Cc-D49rF!wVGK?~&P)&4N{b+!gUGb7Z$Xtgg ze9x!$KXe{3NS0lrV&bEQ^d>xfjs_2Kr zV53SNW?f#8`(5IIz&F6kzR5!F%H$(At?5AF&})ydgt$M<-tlhEp@fF3D|M#Iu8whH zU~kDKu|@Yd0tu~^0q$Q4?=sa8qBd2NrfE_?vOOey!X$JK1uJ#2q0iL6@4w{L4*0N_O;lV_Aeu#pW?xYN zZi&ij3Wl3|7St-Z@A;%os8wfIC*>F!-pA?~F&|S|Vzqfap19)+Tc?pw(bQmY48+iD zAoT{etjz>q+*7{ato-0-rVd_A7gaGl7|h&3rY0!p*hV5IO(S*J&gXDbNWgiGo8s(w zrD)h=%P<&O@|wyI&SFSTRx`-Bv+fpUgqi1t)%=0G; zG4I+BTsZHuZx3u_U_-H$q^TjubIsNzuRISTmPoO-w=@nCa;)d`n6y(H;+X<-H7pmK z;KzFkQ`I!59@@MYZfpME!x~iIjvDFCf>LFpm);dg3N>D#NX)jh)B7K>c88!p{lqv% zF$_(f?=;#5WW#^DmeVK+IPKxbG;nU7`nXzeQex4b^RIo$()$ykf(x&1FMZ^ex*A6u zP{>2;kH*wEP$ZbxVl!@c>?p;BP>L4i)qRt=mGqCho=~X_kMy)bYUC3ocYjcATbwT4 zp!+Yvm)8}b&&RM91V=Ov&Kfdk?ol_-q_A*W!jIVST;Kc{IOZ1Uv~AI2gl9v=geY%?IsFN@pE<}=MO_^sPWA79Y3rGlUG}+ z)I)=J-fF5_gn4#^kI_b|s1Nlf&_Jl|{>j#e4! zv_se3eOm75d4+!sd9){u<#>&quT>cw`bgysgiO%ocyrF)-+n1i#QS1Jm0-GCRlO{f zmxQ^}b{}aE6rl0#wQXZ9o+VxDv9qHILL9+R2pqAgcKBgs`lHhw$s2dARR`RX8hMq` zW4IzJ_U6-a0Yhk&6G^m;0V|tVe?^WA7(l%f@_EW5J^M?)AmZEPNW!T%!CM^D{>#h? zp%dz%{WoPSeBM55h6tWO0kfMC?#ZR(o}jO@WcvLKZn_QCw%h8P+>03dSL4&(A+5>g zC+B@r+ofwbor98(*)y$9O3G*_>h2n6vP(*@DDC>2&H0*TX(Os_Z2oh+*P=sIBc;rn)0o~H`nSsKIz`#|H>5=`vGp;r@$zgP2pQ|%0={ZtK(;IVqHDsu!YfF z6f-a0iyxQuSZFz_uAYc*H5okl=k3pbS`|F{iNZTxZNg$GzL~6_1X;;(wFaKj-z0QB zNXI7k0}>tg4#&|@dmm(7*C!KCpmj{QlOv-qyFxJmWqG~dX_Du+$3yJlIxt^%UK@8_ zkpyZ5-tAAH;8O>9CFkN}OYH?KaP8ie1r_2t$`==%<>{lrVT1dV@A!E!jwnY(d*xBC&=f%KLMH&vKe5!P>Q23 zjpx}+2&I_g!J66mydO@QkZUJv-tq+{#y+d-=dJ;fBHPtlE$eDLZn2Z7e0HUMM@6jmESc|!X8l*kFIYWho$rQ%7=E}?p|7UL~F_l$v&pS zY+vX#EpC@Qx_IMT=d5(Dlbj(qo?Mzfx+mp|nw)Add%FFi{ur}p%sb`?jOy);F}@p| zX3%_L>s$A3h~sw_%WuEXBhc432MIG7UOT)5paOF7yN3~a&J71k14?vU%xA^oYO?=~ z88LHcSH_c6v}+DZ>Bx^#hJPJ>p}49jZD>exaO6-Dw2EEt2vvk$yt6rtiD-S0%9F07 zx-=*s9~dl?;~SGmh^Q*+t45yE8=`A9DCz@l$_~;s=yj4&*|KsbT)~(Qr82l&F|2m% z1lRRaXDVK}=kgLuWMpkt{}s+ze`y*3ceGTDUs;?5UHFja!CtJMG&zV_7l&=M)}Syj z6hwi~Mb~YTs`t4Tjl2#B>CUW-pf|TD%<^sDc6GErG>0f0c4%z2=|=fo84Q_TmfA%p zg?YoAfgpo5Y$)PK)a8+*=bOD{JfuPW@Q$Svyvv(=XexX{t!(g3$=?4`_u|P&Oc>0B zBTDOxskFVsMi$MiwI(h~f^L0Q#Aey{!O4d=9o7`KeV$OkG!b&G^&^Ur7_{B|XP#x? z{>)E)Pjed+kznj|;x!wb&{b?p5tgL=4Kt~8&1ug2nC}D=7xSa9A)HKXIQB-27*=lVx;e{p?MFg3siheL`EP*az`iy@|wOUS&LS$=ZUr1ym2023q{mQh=n@d z^Mirp%nDlTmxV;fF4Z60*Lk;>+O)_) zT@eU&)$Ht9mifS@g-4GlD}Hka?3lJI-|y!;t#W$SenPw(*!;i(>uASNLdY+NGuBl2O&8kj2KN7t8 ztqsT#pO`9&L`we+mZw=VJ^uS`$P3U?$glgF;{E}$Hf7r)XU(P5_h0eWe-?d;|6c@> z0K{Ar^c#7|p9^4RqCauI-)yqH{HJIk^6`?Al26qqmLLE5b|kmxTO=+$WHR}VH%~{; z6tx_zVVd&4OX)wa21xp*YS_JW%2t9D6~u54{ta+OMcokM9xxBF57~D(^X9pJsil@f zkp4fO^Rah%F+{hJjC>r0wd;}JoS?1FzpJDFIMP0Z)m2pB!*`eGJzc0qwiF4YMlV<1 zISkGHqfy8&0q5-0CNA$0y?TZixVUxmRQq|sp^Pk!Z$&ZR2a%R#RV)5383GW;2d|2E zMUZTHvI;em=qSO@q-8F43#ebqu*Y`tR$yD^n3(Jw9Bj28Cq@dGxXVY`c&c9-7^4&_ zh)${i|Jf(T4&^ilfE}yQBMi}|tVA>LaQRwV^OvQl8oYhq`W|p=yc`RU2Ooo5J#x^y zxb}CAh9zgQ6k;$It8#RHKaM+~K>0H$P)hMiPft;ESQDN_!p`C1;?mhikWEd)D=NnR zA52@nFec{l5IZ|tMM+-mA74;0aN!XG%P+*=O}eF=R;{Si6O{cLiCNIUo>crxpox0Ax@*e?#?Ozsm;z;he6k?taeu zGao;GgkzUD8~JnpL=N@83W(!lRu)x%B3;F;DP77hH1*Q|%Gs=cipERvdC%~E>TpsS|F#^vQ@?dw>d|2qu=Is&%sn`#neibEqpL*?4?V0od{PlG3p-n?&lWbOZ$6tlQE~84so4Fb;^g4urQ+Zch77%f9M4?SE*@=_bh0blo& zQNz!E1OLTJ=JX{R(j_3LfC>#kxVZ!Q)Btt@TN&;Gf#}7cE`Ys%XCh+X_{@sx`?qE+ zwco`hmzOUsJB)+G4Q}K?>RJRifz;wKx^yNXKvI5Da37^y2HiqrK!kx1oR!LC10e;tU5i4w*WPZiZR zqhf$4)Ms`HN?o24j}IBbwYas-wp!&UzkHZpgSE?r*O0TIN_F7V0~t6X0eC=U_0!o* z+9ylXKseQtNl8vA0+6_v+)i=LL_Zr5D0A2Ls@RNDdH5$sGCFiRjtI$)Y(M+3`5A2} zXu8)g1a*U@55*cd(lbM0Mn0_rfeC_2E~7!5DxRS=oM8}70J=ZxSt~|Xm!k&hs@lPNKacA{m7*NiM<@seTSQ6f3NQp~Gj1tMT(e(}_pB9%BmKDeoE8hVD zD##fbQ4Xco_=`3H3R+(E5a<^HASKNy$Bw3dyA(z%ZK983YIFKKljS`K5+Oh!^^b&D zs0ItWr5F!zWR|rdwjS&QpaIecXHwa{`cKjq8L$LZ`FicEtiZr^xij^9%m?tI$ zvt~Z3Lr@*Vh5QAGmc5nON8Rw`vq~I=U*k`Y4)?_1!h1!CS>VLYSokTJ4D=ob zLG6k`*dj7>8BB^kSdcU{xywis4C(MPW>^ir4DctXY_bzwcACVrCh>V#Jnpn$!%8Ev zo^jCas{Qb~HP|5ht3J?5?O_m)nl2eMn~AN*fX=``KYEY@;24IK-^MVZGgblnbd%(7 z8{xWg3X!i?kk>_Z-E`;}2wHCmbHEUl>CeO$%Q?g?k`B)D!)EGd1G?;WmfoVBD;0|d zNGn7ZmI4i^Ya4b0S=rGcL#4$OkTR{X9Po+5DDbt{nPliG@_h)IJjO%wk`a zonVlrHTb*{lOM7~t1AcM532V>2Tsmr(sfhKIAfBC^=N-I`f3fH0|wUgG(?HhA%$1a z`~RK6o>C2jYR|M27(ntZ7_2U9?zuVH44gk5Mp1V9+r{_vUdp@CwB_Crow~RmtasU@~bfNdX>J_E@}@0wX2V360mM< zxpPBdfvoF)GyBnc21aDV0zO1SiPl;>IuRou&V@s977{SBvGv3d4;+`K&9l$=KKS+f zf&{T>rwP|r;BNpGMq~0yq*kt>%;lw2(*#q~L)Nb6V9ta=X5XNyZ=Bo?g#P#X!vp|m zxj&9KXAt4n8#Ab!9n#? z#n2BFv`RazTPSF5ZC>E{Pe9Szo3f<>RP$Tq@MEJbeT9AP_eNQ{GlKnKLAoGx%QHJ0 zHEa)h(02v8vip!oC%AxI-9nDml+B`|FPG6ufYB-&s z-q#?Br_cX*MWxlS!Ci$YfT=26_+Nbg%aRJ@EH^{WOzQM`y21|$9X=IwE9R($8x4}l z>3{{R9wx6;R)s4gx>mMeTq`m|sM`S<5V096#^5Ziu#j=2>1^#YmqI&KpZW~8j1tOj z#R_5we5)0E#LxP0PnWCwL#(xQYKPg$#^&*wPB$!?U$~G0nsPf?dWbE_DrmfR`b8Nd zGIJqPn@>TBY^trJo|ht;pQt75mK!LmKyK7ZNY45vJ+sbY#_@oxRV%2>A18&q&jpLx zoMrdT`&ML(F)Qn%WXoM#cH5O!10G<=6ktIxD^ZvF{@bUt6-hK?tI+E9nKv<};P+}9 z&QC1NWs0D^P!b6C(6m56nN?6<3Ji2W1`oDlRrx@2qOg)1|4%1~E72ap;`AlxrFcHg zye<}BIA4=lFtxMK6eP&*`DH_2f-aweafC zbF*cplR4#8H}BE95Swv$QJBb5d>VuV*b3r7Yar!P;i@vS7jEC~Bkx?@$}ofw2958}oMA z6$|@23zG2f2DAuDQ01NUu}5wSxJPKHnmM{Bx(ynE}l&t(UBuSX&y;o2l;2Gy)M6*UAZh;iQFG6$3B81?7I}tF)TH+f@N_nvF{>+-oUHLM znoXL3ZOw$)!3v*iicbK(YH2v+bzl_VGOMv%Njm>u2Qei}HHmVVg<0vk@lKFB$y32H z_=OrU$ZjK8oqHR5!QKQmwphq0b4IP3G_$EaDQ9sW;vfBJE?Iq@-|)@MCh0)(>f&25 zy{B?k4o@pz{LewSY#Q>AJ7Rxjcwr$2L+F8i_RmA^))r7Tufe@EkGp|UB2O9dOQ>?C z;7RvL?2bOZe=T#3eefo5io&I@Y@uT>Mr)_sAfRie5+1rBlfLVi?_}6dvKa&lQpJvj znI_)ST8Prr*l@uXrsL_b=BwujR8Bw^VI>h{72B~(qobociS__!w(w*9H2>Gl_9zo$ zQnNQR$kK4S4<^+JU-A{TVMutieIz(}k{yIdbnG%niOH%3K}wnYO6#b-QcS@qw%7kW zTJ2@H5W35eL2SqgyKn)ecSf!dv?wyNq5?^oz{o-^@IvVOzbUpv8=Dz_{j5x)ZG_J0AGCH_<(Yv6y}c1w=zAlBjsUV|lM0yB9k*Uex;<3~GkuBo#A zN0WYCAPdf6-qtGN9)`1Ayk;Gju0R3(v33DqHE4QGY6==~Jn zw&Qe(HS3QIIBw2AbxkrZ0m`*anO(C!nUfuf+paiW9J9O+%zx_7_|55D5qQCw>g49t z{PI4e@4-3LU+K>V|0&ZhfS#syl4ZR6w zbW5GsMA(17yFB)T9e$2{2hdOqP~~bp|cQR;xsHx+(`&2o^psat~_+? zV$hY!7m>*W>j!Z`$8Vd1fnPIOMuDuO|iI)=a1>t?ME4bL6~Pb=aK=|vJG*tN?k%~AX7da~B!abM4T%|t_#p&p^i z*G`9y7@vK?g8akx52HPkN(B9>>1veI03@nouf0Q{E*L=p22(+&z;+?|x}b$#!Lh+I zlkZEVv+-_nLMze1ah18#^_3I%QvL)@WYsU3aRP_M^aUk90_!O}s{X|Ky&tyxL4(+@ zk_OWcdYLSI{gO2SIx^u;tajRk$?km zVpCX6Y^Kj9@NZU0(kK-eX~F{+H$>GwdzZ1GH@=Ql-~+Y>#%xDn%d}r2PDV1fVR>vO zUFzsEn8aIJ>{$nS0(pVvr{d`{n!LHax?17sA3{7?=@OMj(LVj(*}S@FeYlljjM3%u z$Uxi43PN9e7GvF&CU87=Eq}K-rzTo(ix#!KhbT~qecp-8M5H`11`L8kUiFd?r%fw> z3~6n(p}>J3%`eei0w8*&_ar@m*uotA>CZ;lrM__deLFp0W%I=h(5H*V60`$4{X7I| z4Bh(TTJT{5zM1wxd8oF*&+fcnijYeO4Pg_54AH8^kg;wdWcF(?UO81a)1TRSDTwjI zeU0@>`EMnLi~fu{8`5cjARh8(G9||*^570N{K{>odqGD*`h0U0g{-K^yLib;Bwq&HfSip4@#F>@jWZNfI@{0ou_ z9unwNtpDsxuC{;-PVdwVD)l;A5s81b5n7@QX)rjLI+=LpQZS`7usgI2*j{RAR2{~^ zaM21Elp2*psAvzBWpGoT9Krx`lLXW0CB4JXI>?jYwd3|b1C5v3Xa^NUMM}$P1lqpg zMopW{vmo%v4zhn^5CHdo%I@?1mKAZ2Qu?NtA(=Y}*SGWr)&oDPP0IK89uI@(bvD+1 zBIRzS3-W_X!`KGYdwi!%iipG{`H*Y++5s{s%-R2_EhYPk!c9u(vO*_lZs!#-4L$qq zSN?rXg3z%t*+ij4b$!%bQHnL3B8up^uVks#2773nrM2vHfj!iGhyVI9qbIN{cOZb@ zXp=iDM^%uMYO@l)Cn+P9jDmdBlUJI#l+*b1DMp1)x|f0XtnTujdiNw-K7*-RM8Z*W z!upvDTyrfw$J)*xIzUIgM8`Er@V5hBxj9279VlNRyWt*}5;kc>HwR-3!kNN-Gk*&l zedNrcL+zFAw63OOzNiB$$X(ry`6a5o;7ln%-XosaPvdGtNGB19ln2d(N<=?^(;B*N z)MtO)78T5u#yWO}THqOSGP3T@f|R3Ze}jhL<7d~^H>5u1U4?gGpZbXPK-p+1kQu76$OKv{s!dN>I3zP;jEz z*zQUES;h2YE0|&6;ELGFn=i>+cZmo5J9r8-I_NvO&o4wDm~wkHl-ttf0{W!E&m+oc z&Qq7`LJsE}jewa)kYrVpE^&Wly2EY>1o;V8Lp3^C=9;w$&x zLh`)wSbH`9P|>tzObOdZ0g5<~AJN(He?i{(~Is zq|%_Nk^XfGrM&RBu23qM;&kseaXOQJQgo#z8a>GaRnozr%R`T{N|EY#SP#Psm6c*~ zOFvwbFFnwUDyn>z_KcBR3};Jpv^f(-nu5@1%iXwp`xZs^0aW8Jo8QXB;|EECfsWvc zubnks7nM{9j$rI(S=i?F<@Cm@hh~j^bMD{cIj&$%CL9R0TeN)qU{yl6HMZSTyc~C# zHs4j7E(N#JK%`BxaZ+lS?g7b$WJDJiC_f}`^O-cq{i60u%pG-m{Ds8Lw?6MWXC7aO1Js75tT82980i)u+v;Y?q=HILt6i zbvXDl(#CZac6J?`w3FcE<&6w~5?eFCjzR{7734MxzH#e#_Oo#2O>$&MXZkU789n85 zi3f3p1b$OxF}c&$5_xNw(^_z?B`>0@0mk3YAW~(#yg&)(tuKG^Mg4RcYtT+e4F5C| z%89y_u*7zge@KlwMy><)sZ24jd@#iA)>X4rvRsMtiId4Lq+{Y)$Jb+iB6q4wb-_~m z={FDWs|E;Hl*&sKM&%szgnFo>1Ylpa@)X2HShi{hS%l_alwd{*zL8-j>o{2V1BI`hvq(R8 zFu$_CR15^us1Dr|BX!k^oK|8!6Ws-hPIs4~@)@l}~rtWRx^VmXQMEAm_unbb?o zTyDjYWW6EsVfaG|68~wkkex{wG)hX_tw83H(gtCsuNd{flvpU~=AZvPPm! zSD%WX#O%Jl%VPZXYY-(G&YT%!m?BX$GxwhJu0nuco$1yMpr_UNv~czB@srn^W* zxXllhC~XGnqYn^OpJQoN@K4nS*-?y|<5{#LK%`01=jQ$be9X)qEs+GSR)INL)hYQ# z(QfH{THw??RX@z>V_3K!fKRmpIeQ&n;*=q6J3n+(H0=> zqS%NK`6EI*v~XhvA4<&X6aD5_Hcu_6ApJenh76Qbpq(ZNo4^Ax>nD9LX6fYTU6i{; zH*1ux+#dqt`20vlAxl^ihu8Kw17MDtl~gMqd1>h~lm?PrLchlp(O{s^5;Ps(2!G6)?#vtoUj{r<%3gaPjn z9hr>hFiuF|wFC9|{# z95INEwQLq#3gNfJaI;JZ<1sMP0EkHpalZ>H9UBV6n-n9k|EF=|RuXQ2uihM$nU~iV zzC3{XgT#;Fgh%af^(pnfaHKn<3v05}5wYdQM^-wLo3li`gv?`Y=P@O01b&9hG3#0HGdNG&MJ0f7=}z3i0Nr;rQf{5V(K3 zvGky4M1TWK&DiA03ZC8atvq;gtdRhV0F74^4JNoT8_=a%OxMbsWRM9|6I47`v$1+D*8f0L9|uh6 zPBLO58ilkHL6RIfQLV?FvaVjs;&B?d52Z5iD=eo`XlDwh;=n=b(QFl^63)x;!r1sO zFvbxq7bJzqn*4qWz)g|>?<71kz^J^QghDg3_DPIKX?%9|C`G5;I>5nQg^bSd)-2(ccCsQTC zM-s&T5mI-%!)Yj0l%e3U35(>{bRNi^A0R2R`aU|b)s?2tP7JsPm>xA#pB%eIVR!xH6@ubo2(y!+sc>gPr4y0 zo~Hq``vz81xyawIT@>DYU4Za^*8&(LP8%sl5C2r52Bf zW?MD5Rt1C`R<5enaF-bVrm@LttWO3Xk*3vwXSO04JQHo7Ig5Va)pnsry?o~4j1vHyXe5Itp`g=UK4Np*x@Nr(H^E+IA$ z6(P;8zZAVJ-gj zOh1;4RcYjJjT9~rxpl~9?a14Io@^z%Sr{9l3iSe#u9}}raDzNyHkeY=2uG@!+jgI= z=p#Hv$^KB^QeZAwBjWl>tHNq(b=IZxTobojL-&QGEAYlW2;skT-k`zshtwErEX?nhyj*c_T|R3oBWoR+x_B2hN-gz;id@h8`EWDqCi zlD|xIcW)IE@#PIhaDevr2r6p$Kr#u=LDO|X`&IH{@SplTc|YT`D|BP-W_G6K1PQr3x<8gR1CFYxItyqMY{c~8pJ;)qI*@)pv ziX@4Zl(J_hV=*E%uzF&Vr%*^>##(!{!C)rSYz$vDo&iNQ>o;tK@ktJf%0$; zF?*sIXkU-}K-^u|GZ;6TwpDsVk=MrZ@~n|B_h{^q41%8Mx_FD5+h<^rCS*XGDBvU7 zq#16m(toy2DQG~X%ykl}5z-|}Khz`3>ZMS~Y5~gFZ&F@2ES(Rz43$-h|A@Q}F@x%5gn&}PV2lGq?f179l zSEMv4KTB|DQ0Xc|u^?5c{`OC*JT-7T4!Xtq-!sV-^IkL+Q6BNTKZgz+H4-O=L^7r` zQI-&MyKU6PGY>r1Es)SR{4s{WycM|Fm zC&D_qZ}H9C>&+D@ZjhvEwk4YWJVdcN;v&JwzVhq-mzr3Y06Vi1lRiT42X*=GKiJ`FYf z-hWYmxy^2W4XKkJ{B5hraN$!v4GC@64Y}hXfdeIAk@2`7IFUz+e)<`yY+9(&0bQ=+ zhom}d(}nFLjPOwPFFeZ+2uscXi_BGcGHRoHF_$-24x_7xm?Gvhoh`laeh$J5%)`L} zN-I{=HKvevH)#AkRDSYDA~jg<2E|{$jijc$QCDy|nd7+HeH!l_{B5Sm$dQ9LS8h++ zNvpc$1N(pXwmk#OWQyl>#hu7ghSpaW7cBm^RcEMDgn=A|=$BMn(~~M!cE9kn6f)#g z%ax5Fk+i+NoaR6mvc(CJPBPy{$pp+I$Ci+&h%&=P{iH~x3`+qZ;r+?YL zn2_OQDHju=JGxMWe~ZKuE)Nzxpl;3L*ST?7IXTpJsP(^3#;{Hj#k#VA;<+l~F!EOS z36Zj1$_aO--jcjXx9pu!+FAF~lTHL9hF0?Mhe2P{ z?c%(-Jn`bsT3>qq;q*k|X2+*KdmEd~MZ+E!1dAzi1Np>VR&!c@dv;xuLz|u&qhC!I z%~X?d6`l&2+GoUa)c9X~bWrghwZk{OdxnbKei;&Wq!ZyMX$Gw%?Y5ufRo_NpGGXzX zFUSSM@~ERUelZ+yNlw_6Y=XdpvF42L1EzXio0|E%cNJV zKb=TJT8&bD0VPeBESb$5S49F-dVRyoR?KDw$pE5fkyvLriZ9pVe6-Q7IvN-%Pz7hw z1HPww)=_rUr)U9}@^3DkypO+a6}Pu!C`kWDOd1?W>?C4YJ?-4S$RhQs>^aeCvK1-Q z<&VU%yS~#(Y2H3`;D6}|d2@RgUb?kE!0tXaBYYSSb-C1Pa~qqna2cB0c{ASnG^A(b z{Z8@QWtGzaHi~MJ(%jxEtUF114VR4=5)(}gK|6Qw!+r;n<%phZ^<A%^@X(kBPwh9E!Osk#&$nwrnm97 zdGkoW+5|PN`4(N@^Un0Y-Q#@y<*xd=Sv%JWmz3AG>!C%lj;z)AZ2-G=`N(|aYm{{F z@6ZC%jgUn`a)M*z4#!ulh?zYf`reHSFoJCMvE|5J5d$cB!v(6$ z%2!8qlKCAl9h#zdF)%@{{d5e0nCi*FWkxobNlQ3B!|Hl- z5dL+!c<-TQ=>gq}bxEaRp{)nHBAM^*vO;@L0lcKTrV9Hy&ctBTV@sA(HZn%e6y!_3 zALKa;lxy{KuCR;r`D?~lc0GVfGPH*sXY_dYWPjIU+cVlW+U51Qz|u zY*woOM#JT_Js8)l9OrSb!P53%`-Y{;&pQJbiMHQ z_B_YC=d`!?qV>6CC#AEGu{n|5YpbYOuWe1o`^A~^`At?K`qT)02zcID8PK78#%65$2GR&HcARQ?JfCMjZ(;XNQr5eX5F%+r;d0n>cU{u4 zTP>FfiJyLWYxK~Sc53WCs}%P<;~8J)xoXgSpWS_Fr!(x!=bc2_`Y1!8KesRx;GEzx zo#%4X=Fa%)S>ayORo6r>{Us}p@A&i|u&+uj6R)dV+@D&BMxS2=_qw(6I1g9Zj(mA0 zY(00rQ9n7pXqs@jnj^+ zIRwo8Yh2dc_dG!m3x%A&Q}J9XNdDfHF_-{)zaXdm)CCp3EE1~U(^|NGLYWs0b2&;# z9(}EIv}Tv58z`>kk4UO5jQSpdf+IbV{A z0<7ZF2|fAk3Bs2x)IQ_ToVK=s={(-E-*wHAVo&>ZSeGR&RQ-?}qXK9+f6#h5+q88% zb_E0EJ%H=kflLMFD(4K5AY;mBjJf@dd6v%Cac|1eXrET=oZmKZT@hR#Qbsiw7vrC$ z)t&cbn?_{LEw`ZWb+)sB>P9u&AaAns(^|JKR#^;}w&G$2){K8xYR6lmH?Z}xaq(XW zLT#?62@R4lxri-s_jH5LnY9K1qZtvtkE8aSoT-8F7%)y-onP3zueQoQeHjtW-109s zUt4?F6ii5oZ;s!1c{2L3_Mnov!oJdt-Z=QUB(~+*Ru#x&^n%xVeWdsG%+8^m(jzSV zgK#&!w;PY1+rLqW%0t#m@7Y4xs(aeQyQ@+A=KVwG^F}Q1pafg;IDHS=tP!5e)H?G1 z!JOoWxVY2W4LR4-#G?8;Z|_JsySC68Ay2$dFFM}+j4y{uyiZFz^_!}8(4Na^fi0}( zsPLyNjOXnvA)+6`DL;0pv>j+?o=$P6ODUkM*;p5)FF)xg8c)_JOWk(;Yi^Jhcb4ng z2D)-yB5yo5P3~P=%g&kN9mX4)FE;mzTena=cCP<^@5NQ|zG!G}3V5+JBj z*%Bd2>3qzKbvSUUtAvifQ*qg7y1Q)`TOB`H>bqZlu3uJ}mB17@=U}jP`$ei54l=v7 zKlbR?P)QhCk5*Z0UMbXhd9{NUz3b)&Bf?AGA2T;EzniXi%w|QmlRe&^2snfA>V)B) z;fw7eeBrAP3=XH%0Tth$O>CFgkw|<8`5ysc5Lz*$>uffwa3DjK%ie;HpgX~g_Xf)1 z3D#CI`tT;XS?SuqiwC9N#)Z*47mIsuyV*3)*#C*X4ehy&Y|dZvckJtDcbHBZSv^HU zciF5EVIDtSu5uk9WFV9h-f~Xk_ZIyxE8O{34KHr3u-#qe&8OuJ$M-R^Kc}^UkVw$C z2Un}1I~A@bcyL|GK#WW0`mjQUjo%qq0zefnV9b50Ze2pF+UaOP(l;GGnx-iBe=7I zoWFxvFKfjlPLU8=sF(S_<4!OR(zQN|K{Xe~z}8WRKm)j-Gbs7%y@tmpjaB z7D`5&Hq%i(t(TuYTpR8CGt+NiBeQu(Bs)efIAsD4WY1;jD^2X|4?eSB?BTgMT4S<4p z?T~Aw>&W>T60R~fTj|97pZ|oR58D;paB@gt9El-sidWjoss?&Vj8r7u`{71oYbsl4_)ZT0tp zyq;S^`rr3SgA5!0JMAiynmqD3Axd)Jj|lhZ^vGZcquzmlnLIHD7ilt zc^vf@=`;~&d-4oCADgH=dk^p2cB5RZ&Kop`QzVt%<2`Q|rg%9`Y$ypo3nicBYQ(!- zN&Pr+pI9dPk;b0>w54{kwUp+?oYeLtGtK77{xmWbi0UA)=GS?8^YY8i&^-@dwO9`x z3n5I?cKaA&XENz{}tNhOd(AO+! ze*RN=*~Z1h@4)L(<7r95J*Ev|w$?_-S0lh>pPqlrd7`m~eUFcj%zW7O=eXpNaESd0M_hW-0T2ekWcuDj1?A_zC z`|!8#`Dv@PDV5n;Hao={gcpBLp4|w{wXN)JZ&swloV-wG+GJIRsH(e(Z6c8)L&J;H z=V|NIvN5;yLU=r=I=aB z+y4wL+zyYNK$kSwJsb{RRs46v=&DLY)U{I}q$U>7S8Fl2Yt(h6v^{qo)CrFJ(+9&cfW_XHHJ)%xLPAqwC=Mn!k zOYKx%D(7+89qZSg%(i!i3->#TJp=V>!?bXkE!S|G*Q#eY&8}IJorIse@E!+hW-7iN z35uToMkgjG-w3wzIIu{2xO=%5D1F?nyXzLYd3U-`-(%koX8eA7X@T(pv(o0_=ktx$ zOPJFE-=P}mHRIL3x9MD3i~oRKBe}}MN%MZO0fG4-bo&xCt?z)9W%I7OC)Q{gE-N}% zL^!m${XiLKWViHqSvCbQ=$r4E=@?tusb+GY#N1^59NW4_m3rIs0{`HASlIh6U1|=i zi^6GkZ)Oe2#7+w9BrWA}W#YQE{N~8?0fo@voYKoDq3sd<`94Y2emN;kEgkfg&FuO4 z!~s4E)xW}8Su=di0ug4nsXQ=mW%85Wb{eJTxr(wuAqEwo{(gWK?=VRdmM9cL%?Cd%lxw(vA2d9-hT4#Z==9jP3*3K`z1^8VSXPgS~huh<6XVTNzc zk^lU-lVbl)ZYWLVC5)%oWtSrM>5}kxh0b5t!O#;~QB}Y9k$8f5AEj0Vj`gy3u*h*( z0F`(#mO#DDOap2>8?R`d;T6KXVLn1HA#Cy zW_iyu(Z)X)N+U(X+zwD&Uwq^mE{n7-$}yuM@Pg)-&b*BUFXETSl@zxgm6_#ev+G6A zT1M#4srIsprMLAf{>aG! zi01}voMCA=kG)yU@xm6kFMC|xlZ58N=euYW1{LcTWeW*o6>7Q7fl8;7#O{XQE6n?% zh@K_d{vG9Hyeg zkMc?9IkEtR17CiugNt1}M6T=a@RxiLy8zlhx{?t*-Q7cZnqEj!+NBsluH*2$w8S$Q zO(?%+6?E&i0B}Mg9G0*^BO)7EktX=_%rOu@JElHeIF9mS5s75t532FHqR~57oW(TZj#Pn6u*;b!dE=Z{zg@-e_NdMl`10KUws_e+b*S@sQT0 zS~j!MD-$x?EYpg<79mh|w~NBI{)`fLu~p_3YN3&LUVAZOdVgu*aio}T7j+zS zcRf7&`)D#3iV+Uu;OEb1fv27q`pa>AHU3rReX&q%P?y>eF!;3-uIxVf=V`dNMfeta zXJ41cb_=q+j~6=m6ufRILXNP{dN7CEc&VByk9zpSL|G^dN~j8%II62SYp_(g&L$`8 zs-B4DtZED`Ds3pS>8aW+*$isZ>OhG--={5Gy%*w>vd899^*}%o(w}D|dO^z_60#e)mLB;n(gN zK8A9VQkwzSa}9RZ!fYhkJ<5I8EtZlQJi_dttd_HJqJP!7*vzHc7~X#VJm%@qa{bhe zEl1Y!>&Fr2G3D~zS0W{WdyM6~d7a{iZm}OD40j8ok!I(q&dI(1`rYgI=92(nzJm!b zAHztQQh9kFu!K@>o|i2!M7&V=o_85)IeQ^3WF@_8Z5k1QKM0NvL^n`VHB?u?t6;1pc4Z(%1({A@5`<`r>T8m z>_%r?nln#i9$ogY|Ggv^wLUvhEF(%qKg>1SV(c@zu8x%?yRGjfpU*18*o1v^G-k>V zlLjpI9Rs1odJTWZXm>d6ezbQkTwyJ}G9DAoEY;?zK!%C})y)LKuf@c@kPT+aQxIW=$O+Hp`t!0|1Y+s$dp z!0~E^Q`RNr{YMqgqrJyt|0=-!{Y=B!ScOR2GTZFL%?I*|n46bp;HB74=2!BgC)`|M z(x0Kof5OUG?lPUSc)p9QDc1NZq#2i;B_$SLPm2>Hmzk`Xo@R+>W6RM0C`Ehu3Oo$* z)g+};HaN~0edQG9>pI( z+4$&KbtK;NvsiSKRk`l#&nzoA_PlF>!gsx4c`ZAW&p?cL@^}Eq1x-&QFDj7stD`J$<9=;a}&&2=8c`e7&4iQzTCK2=XJJs-s`+-|&VtT_BvvpHC~ zzow`oHNLMdz`#^l?HPFf%cxm`?4oPmyRiI&BI!@S!if8iAze5eemfhmJTeK?nB^88 z6i{3lgoc+bGIrw%$;!Fjs_F2bcz?g6s9Q5_Z*;%i5`iW^=3+U_1MBRch7NL!SF@wz z1>Ph%-&~m>pbOk_dAA=3@||PF8IA0Q4BDOi}qe??hZ-lbxXbNZu_yo z*Y>Z{ieg;aBfKWg+oz5A*2v?q(&)!M`D;8N4c~~NPilA;dqKiFZj?1o&WPr}sq|j6 zk$WpVY?JnY&9~SbAVdD(*JA#XoRfTWHdD1RkM(0gN@RBGH`zkkY9jj!@eKD1Tk7WX zm(*^#)P7(&O&#qU*x} zvp+^rpgF5Q|NpfB0yVA<&LoV@XP+?*Z#3ARrbp&ymfS}0U|zd70vjLX98T(vvlCpe zuD@!sc*CYdv|IK__Q6#yFJo5qSGjui-st~WJ<|FHAD*(b`Ln{Cc>93wyf#6{misfj zuPxKViywxJwD+Yxsd~bheLmu z4!IP}dh@J@_mlVSQgY2D{{P3^TSvvQbnC+q0wlqLI|K+2g1bvda0?LJo#5^;xNC4v zf)m``86e2uI=F@4ZX@61-h0k@-?P^D|F0LbsP463ktDLpsIrzAzLbf}S%@<2-WAU#H zQ}-q~V8I~gdXGErHsv9FGbzaf#luwyR&};^NwF?;o}+(v0CE{A5x?$`B6e&&gZf^z zX_2WjBCPfh5l1+Q$yXk5W9`@9*j!eo5k?g&!{{pdW3|ZC{h=*vO&^)U+|7(+t)%DIcC=_eoi8J zincun8r{wvI#@kwC%iAF9B4Q6WU-ZQ;;OSx$v!n2aqsrZ=+g~&Y%ud)O^UckhUa%a zVkeZ3J&MEf(nbok@bHA5qJ|<(6QHItNPRj&oi)(%oLhCb2Q^vcUpwC}L66H1H4O$E zjZbsgo&-z$VojOE!<)vRW|#p0UVnJPgwK`y6w*-@W6RHNHF2NktE}9y2OGN;V~RZg zd7%`*?);=AK}AByI%Ox{`+M<^U%%#s&*es9vxNws8GbC{g+AUC0lLM;#n-EG3fL*8 z>@H3|odT|y#2?QSGC1vDR>gQorvGTv7Y@92|FUW%il*;S5q3^<)2l-^b%6`ZS*M*#pBY$ z4oK&weJJxXKX63%`ls$1x(>lk4QcL`SI5aT-27(KRtkfl^c|B)?|8B+;-+k2IxQks_v;I~v zTm~MrGrS`$TQD5CIu!+8JwzN6HZE3t)b*XFu1bS#9}PUU`Fe{UnAw6Ha`27{11AkY zZe7E`aS?s~n$lnb0^;LmSv86OJs9|U*r2g}&;oZ%wNpSG)kW*nvBTh+v&kxr({fhA zN5-L~q0E8ZkiRG_wM7PqXT|MmnzruVk*EferKNk%soqTx#T)c(o6Q7r7IYTyXmPuAcJuhlXL@n3S$JQ3Q{ zLK>w=lGGI;s92YB=%1!@cpFdkdVxD}w(!~5Qr@1S5_n?-?0hUcCV05?k}EW+w)TV0 zcp&+AG?%%t-jpiCfvoR#To}d|Ujts-i%3`XD}G7{D$KgjPBVya7h{-AYg^12-!$b; zzfz#FQRgg5`+#=EliY5>{|)Lp#Rwe36?QjZ(Z8=?Dt{sz&}qaPuj5;#6BupBcgkjZ z6xJyl$D!Vnc??TTIo0|g%J13Id4rs z(`Ss3smLko0b`z*xHjhFV*L51XQnuD^zG#t0@m5l?5izEfMyV(}{7T$BH492vtEb2w30d0c*Iq~I z|FkaG{|UZW8-kmYVN7Nf!!V{DGPOUrb7_3j>hwV2Rf=Oxf=N*-y1d2N|NecAuz^|)vn7@};Uzs^ewxf_?z#xo3rCfR4 zhcmX>pzncqJFmOhze>Q>9$Y&zUH5q@T=&41$_T)h{V(xL<#+{!vj*DQf^_ecB zzgHXveS!}(8aw7+mq2`xoZr!6DLOTD$&MsW+EOVWSaC3;>}&tQOJLK)Btx;7!S+Sy zSxt?m?Js$K^!5D^J;Y~m?KO^kzh+Z@^p*J8FEcrFt5p6q0hYMUZm+Zz#`nN&ar4de zpkp!B+Hb_Q~_SbqjnfljLlsfRjQKj1>ZtQ_9YjmTEya3L{c?2Gh= zbbE#WsM|k#XUjWBpjPWjs^?k7(7mTb)d{BovX2`t&2J-%rZeb3E5tX|7z|gFa~9w+ zovro?@hve;mz$R>_dFtLvnNs7=6Wi@Q@FA+?TW{hINk4-@T1K5C!V%?>N8)Kq+`xwW zgz^==m=%vQ|kG^grxYqiG<4bM^XTDBTG=h3yxv#Q>BaLic6hgP1RgZunHEjOtK?@OITnoyN}O1Y8gh2n)h3xDO$CN z0}ib{+SNzj(%BBt*jVD2f86qUaY=WuiUZ!>oCqQ{j+m2bQ4SbM(p9?y35@v`RMvvHJ77TzUs*>yAUzRasBSmIA|C0hTEc% z_mhzA`Mro2rey^Y^W)FAiyGQvHRM(Zz3;W_+u>HzS(%PFgE2D=?{_dlWt=~0 zJ;@#v(v_s^c!Hjqtb2pwF6^Oh^$P2rpfu}0z&laH1r_3PC6g&7U4H<>ZgMc|mtUww}2 zBjV-t06Tale+wOsd{+L$maOUa-j4*LkN=}-zW2*6^J0rP^ENx#=*}pkDF=ymogb=U zb^5J1EajRsJIk-K*SKf<{3X#mgnvzx3P;m6(x@?~ysfx+j;yV)j&7$~^hwhD%)84e z?Hg!>!K2CxR_%7@H7JPlR$lutTkn2j8(QMIAfP^>Ya^fVUYjSb_{V6xQw7O{La82X zfJ!*=5>1cEuq6c&t}y>@r2Ka2zF$Y*>0Ck#S~m%5qewYgQLa7k`I1~2B$S+XuhZ>4 zpeT*`TM6R0a8|vZSC^2_zF)lnjS&Yw=(*@-yz$UVcYgf?({43Wpq`kZg0cQ<)pl5W zab&POciDN;Vk{4fpfPp~{ls2zLk(s-zrZ6mV=*VolGJ?P`?$0Mrv;)~-+oK5R7pdL z3!>Ve6Q7V!-d{Ydc+Sx(M!;Q|yJ=YVEJbx@qSZvNnFnoPaS3uFnRZ%+L7s9VF8gkL zm_h67XAIZs=a9RTkwU)9g{8b}K>)yAH0*}9u8x`1cTM79z@emO7?gZ5+t|px!jhrX zz>(-a>T0KRhv&<8Ww8@FqVGM(3%0D>X)Q{1@77Za^P3^cQARdcP0Hq6utY)C4N@rg!p7tagV;9WR^V%>*NGe zzKs5ICPayyX-maG!en0T#SpgH#{`nMTN89u7q4`@hu)02a_)D)UQ;%3m$t`%*4o^CB0t@B{rFJWcH%f~4j-dV(tQ~86$r79s+aH$oGcadSc}p>DNY0G=hL+zyk8A1{5W}or z2^!fQJSx*$9yqSw=#2$g4MYT0AF~?4H5!%Q9$`Pw4?4eE@1Da{9#g+ni7zm65n*Q= z|2luL*D?E4`by#cM&Dm7RvN`H0p6`w60aiRQ;5+2%llw09fZxp5xf8No;hWr4zR+9 zQ1Zt{yOEwNH$;p7g+k8wkwZ;b6WA0!(DtcB2-i(OL)|1ktph$);-(@`+vs~MonW=U z^F{0`c&15(+6P(2rAtX@Zgz16Z1BCIB37kadmzvM@nDILFdsDZt$!&eo$u??!kwR( z%Vj*L+wg{EGZPalJ#UdPLvoqYb>@{B2auliJzhN zGQks-F9&&JcyD9gjs+?`Bf42w^Wl>tp6Q0y3GNY9&+%6vrM$K&-d-|;G;366WWrM| z_fcCFUQ7D&1&QyXs073wlMf65^NvOj+^|Q81K9JTTh6nVuRSprkTalCZ6la7)sxi| z$y{ya!!GPQ?+EkIc_M==gPmq_$Qvp)#rT2~f5DMM6jSO*Ty6ep$F}{JSpIj<<*z+~ zpEvy_6#W6QtUnfNwo`h7+P6p|Vrbm8h5Tu=#5FuMpdKO9vO@QoBx|`QA$TD7dFD);+c_enjD?HNBii3)L!}aDyTzUD z$AtxLoyqM7q#^>ct4k+LcH2)cG~{Sed?e=ML)Mi~v?$GG(sB#Gg$TkSy@ks-wRFt~ zCls)G3-PAxV^kuv!V+$mk1T`{*lOD9(kFQZEnm>NUG6y4I%SspYVi2&y}NbE>OBnW zn@%vvJ~ppzzV%$W)dJYxJqmd$(X?tkkwU;r;MHSY*z&~J$BSl2ZUg)C$+yT>zdvvXeS!Gxi9TreZ!`9h_Wt^v zl;r&^zgJP-^mV7C!Z2t}Mb=&n7nkpbip^BdP>dOy+QP}@&8q45=Z#^U#aej&+qAX9 z1|P!1E*{rKddzOug9gokrkrLxWE>G0OAU;|8`o&r&0()5Wfc15cvF*XSm+0}<;%-r z3DlpL?Qw|^daM}`uGbvfbAB86;KdOx%(-k+t8M$Ne)gqF745Y4yU?;H)zv9Vjg_7^ zaGzV;Vk0jq(XH;|30x{NNi{``WJL+FH1Gah|g;$?iX@qEVzD)1UNEAq8 zJ6@c_{2!a{$N=rupFU~Uc`c=nwfbll3~5#BjOI44_l+-rO!iL{G0{-yrX!|B1*25N zj23GQZl(%i$pmva3#8T5)%h>=v_E|?QFShWhJ`JW2$^#raQncfQ{uE}W}c|pQkN?+ z(x|52AkInF5U}sT56IfOMc@a7YV-) zkLOwd2BHAeL008{WwSq@tPRta=R8Du#`TUqHp88~fZeI)nq7Pc3QF*6k834|GOEyu z!9|H!>}hgvg`-D69b;Dmb1YsP@0zHt`0qAe0j6S(axO9Xp>N4AA-BV|m}XbEfQe^m z=kt%GEno>}G~0EZ@r?FUtDg;aL{-B;J~yIdzzqf)ht=6l^uw}pB^`g}r9=LEzw@q# zU3JMQYO$Coe#2h#Q%ffkvS3fy3LjBGq_M}zsAqhMan%S z3NCkYfS*TFM~iNW3RRu4deyZxF1MUVAEvDa1{D>{&HJ?;vrcLz(W<_`rzmSpOwv*; zo@~h*qm4YAN*X<6t9I|L_T#Z4O$I(Uim0>slIL6ijZhPfO)}j6S^i%wy(C zSZfBV4L8g25JP=j%o9#KNOm+<9+sa!VjBQ&i%4Tr0&6m8xp2a{+?LI6i2XG^<9uxe zJ8bz0>4vx&r<69cQg`^8O;cAB+B#-gbL;35Mzoa7E9fGMjAlh0#OvZ%7RD7Ra8SwD zA3UP40qg9Jqh|AIw)1H`p(E-v-0dXof@xwX5u6OQu~?4z+C;-3DCu#DK6yabL~j3H z96BjC6mL(lY%7_)Iaz`B{Sc;^L}4b7S-p*Eu`{#6d4U7!^2!nSQGo#T@-Kqmv$DRO zU4g?bf)}7?V=Fd!o>F(=Vro_ZUDo>WK_E!mUOnFVk6zy zuMjl|Zm@T)4MX`AdpDi9A1|idFq+l$b^CsgZSPPChR$gyYX20OUbJ-V}|cdnbd@%TB6<+XiehWGrh51KEPYuYu_Ke&vHsA8O{H z)1qStgf$2|URgxEA^CV{i=aAa?X`uri zPBNTQ%^2TfNEiHE<%>33bdkmFKD}!P?znEC|Cm+p0YwAGkfG}>Ivpr#bK?}9_ zvah?pCZ-d0yqoWJ?8iVb3-n@a@e1r+kvoMB1cd2l_uuJDP;n%+e^5_d{mSo29Iohf!2k=jNQFjVc0H)tpC!$ov8XwKz)AF4p=zxgaj znK(>JssiG}>OFM9#KyE4=ecG!F<0NR-IN{Fb_Ps3qUZ+pA*>%7^GI{|0Mrv_z};LJ zy%`qP4ZA%&8W*v+5qX_ex?|4r`S$!5|7~tvg5u02jQEjo6cF zuUFd0=Wx?xFT&RMi<#Tzphs_Bi@OT@!o7(x*7c`-352ANb8(*d_hQY-uG%yQv_v>XrQl+sb&!DI3t``gK(<{})?iUk%ct zF!T|TWeD|k;N`rF`L&*izd6Y^ds`ysOn4H3DxiySXc(NPPMkHO-M_b{gl zryVtRK;>TZkGEymI<~bq7~OEah^`aVwH(4)G*u%JF%fBA{gnUuS~IDAn|V#AEkV9I z`)6g$9ghvI*^queWvwNcP_V+tE4H8BBIV>R_TG}%l|cou)y1K$dy2F9s7u`qFZEJTT@f(cV-p@;ScxX_(P0s8qWu2x9`i4omc)_QKnX4uk1PVDAS0r;`+cdzf?XO+c**@vnwBGp;Zzk zG*ii*0}pjWj~sTbioGAHwj_CCDcM!s;j22R6e7!rt(TTZcRkFwcvn@ovZ0)Q_1#s6 z1NmKnpeim`xzGseqPDXoeF25l9YV=ZILzT3jA$k3`Gc5jbcW5i9SoRMD0i_eY;P{w zi*sdOEP5Ay9of%=-5pAcvx6I|H{so_<@ZNDuntq$RqxwwTPI#lBS%>|x$}3cE|?C@ z+28N>(tWLCf({PGGJIu9JWC=~FH07V z9blyAzMUE(TOCklA$4W&3H5PGBhDkTPOJwdoWQ6+6L1x)bQ(MX;4`%Gju^eif$?Vv~=%Xo_ccX4CoE1^7J3&K{)2HNEAvDG1$TS{W}hkJ~U`t8E6UpQ0Nw8nNP zf&@EY(Er=&bd3AE{UZ zZU8Xnt^9dw3>V)ZaX5LeK7 zhSiRECdFxjrt2;k+N~oa-LfJH!m%R@-9Uye*^ZvCXbsXbhs8oj(|#3iSo1~1*woVF zm_v4=3N`JlmU`cEgyW^-VOXRNVC4_S5lbar4@zS*=g|DcAx>8y+&rJdlBAs?rovt| zg4zZavzps2_x-|;|>zg_&+3%pcLQvsi~ECjv;xO_8!J* zGFi)gO&1YNbd6;+^vgF|_2zX*#|Y__aA|&{hF{<;q(7F&iKc`_=gYBYn zU zTJDrw-yb;di?d1baTa41>vlxN7S+HfyJ3GxijDbqe(}V+gNy>uO1Tf=k^6Lqj!F8uW!dQLuU>Kwr@7CeUO~mD8@1b<3b0# z>afjmO`9e~i-b)!`M!Vu9u*Y@v_|AyC8w?4d)xw_x`@ z%w2%e1Ct6T9A6>JOGNv<<&MjBc^VUNl>pKzYu8DIlM>)Zv+^$zz)Bi4EgF3+Z04LSJUDYF0Kh`0T3^%bh`g~$agRCDV1z&aIin&r<{;Z*TF>FFE=W+ zIQ_vN!HpAb65*hjp>0%=jmwSepk8C3U1OcO83O>|yQ$MzA?GT|LS*-iCf7i^q;^5b zxh?A?5BY_4&SjsH;X~-V02rtH^xx3{z420UU?Yza9j`o{#K8k1Y(t~;MgLDDgSB~RYMDO~{+g2_AOXS_ZKq%WH zXdId~gNXlfQ~%R{cILR{m*h5+m}r_3;MWP#>7ooJ2D@*XvC{9Y((&93t8siQ3zjUD zl#2s_p0c+uahz!kqoZS(Sedmx{>?Z15zZYJGrsJom^d0vnjp23Br|1otMu2?uMznv z-hp7=f$629r*R0?+H~8zyoW3sBFCL(d@M*0eM#-Ci893ewzRw)XlOfh;Ko$*w~_cW z?bCx9>1OLtg`t)bqi(@ZMIh?Q|Y?1$Y!F8Ah~|Ua3p`*IXf{>-8|fR!MU$C-%~Sk6FqWQyCdl4);p`NCukS z6Ea4lK-Q&hdvbcz`ye5^>~a4qPI-)rR>z-(;X??g?Tx@(kt)i?ui;nAh~$+}=ZS0X z8ZrE-YYqa8s^@sJZ1AE_w+7_qFz@qoi01x|j{bV|s83oDKdrO9dkQUNUiWmXjQU%& zWYycI_lM{h6u(x~B=-B>@c8%*|KhZB51^bhG%O1jfCny!am?_KKZlvmbF8(roo zJ-LpyVxp9er034JDTG}@u}0JDOqS|2o2#ozH+rfl%~{}Ij@K(hL#C_^Iff4A#NkX| z8`0j&Kw!F(l6UJ`#EGsONbrX_b`@V3Y#kmHTp-wSd(=Gv`H=PPQT>od+J5H$XHJ5l z_s)8IZ>=VBhoW)Bel;LTuJ?v4-OXtoO$;M#c*!}J%sTg#U1H~oiYa~elrRI&mf641 zM~rm8moeXR^$UIO4c5$`y!nG5tG@seBFpwsJsV{mO_gU8Po&q!XseCB_II)hVh;uY zE-AUftzl=&S>34jFLuE0bxFSIb~HPAQ)5`fWqk=OcxBi0K=`!AYKFlS51+(sGYsqn zM|oXb$YT@bYU-OlGxDW*%@Tf7NrEXo9gaoE#V7E*_VnSJ{OFQe+>D_pOsWt|fw9p@ z+y#fq&ZyV^$^Za>e&e(uk6Xx`S^Q5y2);%PdjGAk-iJ?alWpbsKbF_u6=d?UjxROa z>p>5{knp?Pq-#f5!J8^8=T#9a?C0VVv!X32nc*?Yib?5oFH<^-&A#o{^z zd${^m*Abgjfx_Z@Yma>dUh2ZXRR3C?993Vv_h;dD2BP$ZY>6D9$Bsd7E@M%DLl_J^ zQ3}|)b9M|nIWosPpts$%mYc*RvC_q>ei(F&im#jWXnpzY_v%sE)f013dIq(W_vg{6 z`-}sRDpz;$#~X;31+(Aza^vtLt)OuA}-^Gw{a_&>%E_j##=e*k#CRX%Wq$|nQyAql+A zgAyABoQWYNJ9Z@50-SKL+jHM%wyR|C+uFyMwt{X{EIE2|d`~PU5brhBoLw*6UG1jp z<`~%=b2|a{>Y%xv)((EU!nei~QVqgioy)xw#H9774~Xa6=wnnAP{wmSWlEm%1iO#v zc!EcUpX1*s1UhDQD(sp{OPs)F{4vUoo#t@)nleBa(`x!mtIpS~`_8Y$)Wj_YouSZqu# z3An*G#ls_UzbEvAYjy7-lKHt=!69?- zVqEETrfWsV;dJ;p8=k#}cut*qCzApv=^FQ0fqPdh#kLbPON8kuVecMzuuyOo%!Mq7 z!{k6PlwhLCXM6Vv85}{n?YR}X!liH6_cu%6edr+Zg!(2v>md=1nhD2&qVzi3Z8X?A z%B=}cCfPhvJ8X@5+DwAX5e}^;*xf_e-Bc0CEVqa=X!TQCYOSvsns~bpkzqo5P=C1d z0(*cQB7X7Zb3`+qz&Wv)EXDACqz3E{=XGL|2&D+#!Z_aSJ4srB`?JZr=TAxyie>s> z-^@qRZ_PEu%!=i!;ofi7IX%sx$6|gg;2U7&Q&hxx4b1l5jC%MO)hpp;ZeG8_&mYh$ z=HQEZsB3!+Yx5i>s%kr|lCG8`Pg9=1p?p^+^N-73X0W~#w9onR)C+o0qKu)8WTyw^ zrS3R$wAcMWbQwa=fw%Z=ecvs1ZN ze#)r~)moMt&2%FOnHrXwsoLqk2i1G;%2Rj<-~6_D`*W0oBu-!CSbpt(UtH(|b;F14R-IJe4?^Rc|yUeJlR*iQuiAVxC5P5SUb#?{I5GPzJt=TTPi7!|)t3RU_p+MstFOruiReU-DJ|5Z6L2k>7lWy2VkU={_PRvGD9Mv>bdr{4R zPux#CM>l)6J(60z@qL>eEQT{Ukut`Jq*by8WG;=2J*CkQKG{yMJ-pwSOZb=q^hx3N z3}2a29B@*yvQSCSS4-|JCx>)JE)sh2oz^Fxzs95+B_hH2bdOaDHl(R#@w^P9iefo8 zYV=@W*Xg4n_@d3FB;H7dviXP20j$l|3{BP9x$x>>rUn{_5ngL`Sv>JZ^ovFjK!V$E^4smZ`JB=*WTu!cRX-i2mcA zgI(!a#+FIaHx3(_~B~5O1 z$iNC$Cri8YBZ&>Sy!}-Lr{^J4!bb@l;9VX-J9jdwcY-fllJfF z*>!U2KpSz~#WfRN&*J3jApC_dE4wFS zU!GV(Vo5_~q|dL0@^fzV9|pIVnmm*kKN0FPN=hdfda}+D5|)VOXMFiNE)IKM4d4CT zrPJ{&&=Q7E|Zr^=$$X>0drIq{vnk~@a)KM_33K|_PVGX=KKS9giEW+TFGEg^4@d>Y-F&bck|vKQkViC7)H z?6$R-GdwQGKMW1#H$f40un=^=U+-|xY*L5_N}QH>7x1VE=V*96=}~18w%lg9jDMjb z!o_ydSZSbQ*4Q5}`B!gUHN5x3)%#^n8>e2G(Chh*}L-%u$gz)3*gZd~hi~=o!^a1HRDZsOmB(u(PImf6P&YF#*Vzy{kH}q-bA`qf zD_cOh;K4Qjh_=g(&7pqL(5{2_mjYJZ|CnYJ(e%~XyF1^L-mI8s!+4AceEt$q!9Kvq z*w|*Tuf~Po?T~wZ0AT8`u-ECmz-(zmDE5~Da7d?y?7#2i~=%k zPr9;Eggw<#(iISbNQyuLkmFhzp$26ud;9dVBJK}B;B=>_BTy;x=WrCh_c6PQx~}%* z8HAC&bHKZweTNYM!yjGqP9IYu|5J2#_KQ=o^?;$v>sSL33XwIz@5I?%V3D|x9XK?T zeI1a{sn`4x&!#}n=7mFgjA%B$C-Ae!Gb?*mTVgiz6S{;8Sa-hJN*1ra*V3-rVbbza zQR5Oq7@o90r*(yH*`S>7acw`C%y*4ZY-?R}8+G3D2dSE^iUEOj3qc*G6&4y4C{0%ab zTon*%gkxdJWqCn>fVaV{Tu*FE$B-8ilTxf4VPO5b~u#;p5=)I>V%fnC`b;AdhNuJJXN zo014;dp0LSjYmcVtTxI>pnn}2eZgh;(0?kJlDV|$?op&GB>C9{8Q&7L7cIDeyy4M0 z3~otc1U#6!tVis*689cMWi2M+lQy#kX%rf4~LM2-b&yujKYd8io`30En4` z*G7N?U?(Yj9&*35WbQB~P=Z2+bSiqwg7C_U-R^9j@CW{#eTv>~MBR9kNp2*Y@zD;G zox!DhGU#}vEp>a4J~o>xwdk~CwKo-YeZRU(K6377B@PTzK^CE(ymknCw@OM%V7`BL z4%l;Dd5^>Yns=3rIA-0aVDtLVeJrG;^j&W|ujI)C2W^TtZr_j}reh0bH@M+A#&_S& zAY7gEV?WL%$Ou}#S&LXkuKRHEkvkXf$q6QHQ+)gk_n}yi&q`=+i1~g z*L!XA*PhS|t=F$hJ{syP1}I~r3NF19b*o&YX(Z+_;IA@+a`uAF1Y2 zgN(Mm|HuV=%{|;FenwW%H06~m7?=AuMKjz={}CC>KY-=(E!R^MELp zfo@14dSoqsaI1N3^0fHxZy*|OOn*qOjS4L5|GfIAox>WNYY_OatHZ6z|IZM2T&~$o zH}@Q5aiJM;2zGajflj)Bi|Rk*;!pEpb^F@K*|(!blqv`Dx=S@NW;jFhwfNV~OI({q z<9;k&3lf_joZb}U$B4p=|2lSZG&P(l6h2UFI*)kSZ+>d)oE6en3g>~mutspHwDvS4 zn~@3JN(W zL?dTK<584Gkxrmn`rjUN)GMT^i<8qNIGY=4r>uL7t$jNGvC{w7uUD<>k!$^V@(XXtb zA;?xzFk+9O!---D3r4`-&M~Wp;zoB7w7AElDhTVvzi-Rak7r6R)+J0xSd@}VH0@0J z8Z$IZ29J6@_sbWvk}w`bPxF&b`fTXf5uG76nVNik+{ zd2=(dKC`wS<6nz^P+p-)&O4Wy>jBC7EnlIz@;QNAX;1mNRnCToAZOMr{&!X$-RW~h zX}>8x#9t)lAh9K~o~)G?A%zEHI2rM$c~b$0Atx|4RpZ@%t*M|e$!}twgHj8ntgRp7 zZQoQW!ORz zpxO#D{k#b_)-x6cBtIhVf#tJepmAx9*>Ls1$Hl4Rf6B1`M>-3+-7^D$+wezWOHdbD z;Qgi1RdJE84=qtVzG|_qI^#~tCP#WNSsUQo)^XQIb!bqZp9Q})xcQ1_Jln-PWJo#qX@j2>_wldcs^<-?~xZ(_dJL<_Hs zk>g#ypK4Usfxz1J1#H|HHQpSXV)QNuQ;kVu%E$|~XI1s?_I@-(5&I1U5=}1={ zCc6K|V?=OAv*u@g=^{uo-@E(grI6gv^D8%eG;H!XHSSD=gts)wCO+rnpWUgBVD}# z_colH>2z57JH3b|9Vpb{ZZ+R|H4UOxTu+fLW!d{Uui0gIA?66^$O30=P_!w<(&!6A zX{l6f!S1~}2IE1&@=cV?7X4jjEW#(y8#z6SgmpKX1t3qYctrAhsiKNy_YAEYO|>I? z(`@b?g=kt^sb-F9&@OxSxE&5z?#UX+3TD zGCSe)*-Q%Q0+&0G0_msxZ-yFqg_y_~h1YyyL3LkDv5!DI#|wF8eDXO)YyY9mWuiyoYHY=>xt<4>jxVnR9Z zpnasX?3vjzw>y>We3v^Wj1%fcTh3qf_Ijd{5H|J;$tb*!WgC_vvY= zkdfL`AVlK$m;xckEM)`f1XJ`+$ecSPOvMd{9~KT7OxfB2Vu+{rt&X3>@1>a?JemRZk$u-7!u0moqG85|;z^OyjRdJ0&DVqPcrAd9 zS1RNKD;RlJ+YfUkqSWN+67Yt1in*K-*$&+$sZo6#Xy7y9liZBESb=*3!>C+mhZKVEQ@aO~aWbB#2mEp(S!UfdRcyj$M(Yl0W@Pv;D0vKp9-5U4& z041`%wUk9bQWLn(qT7_DhDVCnw$Z6KsQY58j;>`zuSGEIT>Oq{{}{$xT{UJ4hb_OV zS+J#a{V6rpwuvTvqcmVy`i;kmUs%(B;5(K~P2p;9C%?8fB;o!lo-g&US^#sjjfciu z0Q#*VNj!F6HilOq*jAVE@=zCP5UpwD7dlQDcqZ9Ew-ehJ=6UKhtG zO9qVGMx1U9c6ZBE6+o{WcV>-&U)aTr4~VJxV(&|x)43kbpPN@04I&1o*X5uT6??zN z-qK<%2EViRT&ez+5z}(rRYuTp4qy>KvH^W|&k8ggonKRY7&&dJ^Mrz^h*Jk2mpbkU ziR`VRp0hbz(cd;7m+HhP0U*`^EK(`8O=+$ep%ecXi=A`x*_`zb8(SFDvT->5hAC3> zC%hH<9kFzT#>$z)6+f&vjyeeJG-MV#t}MhJg)rJn%qGLqjT%=q)=E6?}aiWux;V08@wbNdKJFQ}hwpaZM;bbB`D(~BkgKrTNYFVT3 zj`oDP>;9xfaJIOia3bfttV5nyD7d-#T>kTR8p)teCb+XD5&RH=|HbEGXlb9D0X%kg zi7&gVsw9OZ_VRWxVmk(7XcBpq+wc9*M&3>zq>l)baqnn+)5|eGkpGcf(J0N#`qEJ8 zt|}vt@f@kx)0KvmFxr(f1u0#rXLTE?n7@wm>Gy5fMwIuTSi0_Pkx)l{fZ{OpMKn-1Q%32tM{=VIY35UR5)Qj$)%nuv=XFsn#bV`!&!| zflQt+qA<&$sz2n=AHa4LFs0As^&(xx_vH{HfbrB1`*b;MgPCC~IY)LaY0t*>GBS!1 zdsf7Nf{Jvmw7!}dV_mFf5AGh-Ia9Bm(GYk~4G$$`7rLV8jXY#w{*I2b-6g1!dmrU^%OMTECMgiX*SVi)FWk%g#bu1S4IJOY zPDuCm-SuAWBu?4uGBOB};qbq46S8F6t_RpLWIJ3@a)qh#gL>MnxV&4R{)EDstd&v#% zbaHz9cvPf=8PCBg1e)49Ne#3vhmNIRF9JhCH_I8W zI#a^E+8KEt!+MoVAyeqii2=`wWj%Y#pp-w@0%>JgnYroJ_#1e&p8c7-Cr^S8LrAcs zhUP_Hk>U&I^!0T}6IQSj*yBygDk_S63kGU2r_I4vOlx1_;XMiDrbV-AMNhcD?IYa) zs*8p@!Cl$;_4L5Orlh!+)DRQG%=_-;UC zIKpx5z30v4J^T@=SY&v1^y1EIB6m!sAz1Itdq+Uxqa%rO!b@s!Y0tx<*t|bX$kDGh z6(UdlFcFZCa~k#C1tdJycI)n;yZSsALYprtOh*0cvi>pS{HJuqQ_ia>mHP#(Z`#}W zduig8BD=mL;yHtt>yLF#evHB3kPMGZeg!~XUK6c!K!Di#t|n(`JF|l{z8IT?>r)$5sNjH>GjO)9rpyX`jBDVj3#nGo;}s%o3Z7vN)T^s z2fXva!b(H5zY^V`>PX=_9*|T=^ITYBY5`)Z{B7gnjZ+qx-BBETc>mzAJ*AR)yUdBl z3py^gEVV9uJFMX3iR^3gB9lDLq#L44+vjWiA^;C;Xr=O76@o50yo;+m&8;K{089;7&4!$IqM3n>SBHLJ)!8so2 zZ~W4XL6P2Utqg@stdqe-qsLHl0-hMU{BClVdTm<*NcGUbj%qgUQLIeYx(%qNef!MC zvwAD$|6}egqvG1Oby1u^aDqz_+})kvPVnIF?pC;K2<{=cySqCC2o~Jk9SXR$&bfPM zt-arm_x`=v8m&f~MNJuFbnpG^Q^^U-r*_1tCgzs5;_y>zu2ndO1W)9%Z5b>CZRq$1 zan^@=oSiY{Y*N-pA^RKy(iqkfcCaKFs-{4Mw?V+)^tkw)s~L< zmRl*6qZw;aJ@lr5&ay86n@Jw^X1OV6~+O&!yN8$s0Fq-VRZ8eLlNvig))6m3)4IE~n@lQ!{ z9k1ynZjl2(9pvSpul8=5#aBqiQY-YKVcr3G(dEd1Y8O#2`S4MTAI2v(#BE&ZD24pW z3rt2!N92fEzG6B83DTmy7cP*;ox46Y4n>7z{0G7Qak(fJNbb@Zdr1~bvzPfx-l zn;%e`Z%GaI`30YHT`MhRZ#x6N*BhXeYW95hzMez-9iXnes9qU?86t3u<~?Mbgm zu1YR1WVYc|k-Q~MTv{;&eHCBFLgzT^Kdp^TKVUXsT|hZk^e0dyYN$RAQ;b;G00&(% z>0NLIfkjZpo?MTr%6n&>gagm% zC!^qou}8{BDF5+z7}pEVb6bWDUiaVnz|Id14K>fRS=5{x`+Ypi5^}p$VYw5BUa~wX zTJD-FCvEEle~FpKeu*jq^(hD`8JV4@W2%L&v(+}#4)1$N5~v5S3tW^h^yHM3-|89S zZKY%4+&O*7j1QO5zX$G|b5*}R(8!J6)jjQoX|5W?dwcR=p>#8(=DG_>SeOqBsRmXc zI=nXpshqR9J5d`@uwC-i>gI(L{fwiDWBhJj6!y?(O)#>O7M(`@Nz{9WY&a{66X2zX z9-?^4^XVDod)vN*8-~nVwumt7b7PTHAv_fuYWkAquZWUYwBJiP!rOGi;>PtEl$KhstL&AoTK~_ z9ncfTfp>6wj*8{Yp@yl-*Kf%FB|ywrOzyx7YAzJtODYuYp6jc1OvfdP6Iq>53RZ~i zIOnAV^-%M3p_#6nEd5uYP;z8;@cIQjV;5;qzCe<}Q8+;VYyE|_%&1n;nY@G+t#pn} zxt|puNwpbsHJDOfWq;WDAFrf!&&3sA&f@JfCWpK|#P-HwHXq+RBK>}$Ki*2IInF+e zKFjHKuT;U^i65Q;%+|Q4jT0^C%REu}8S26X$9PBMIjRh2bs;UpV;)=`w~hO8WtF;h z;5x3NdBK^)>v?y^LYq6yF-tuhtV;Ai9y1~2$K_1=I|wPee+cq^2YiB8No_A4phN#v zA}a;P@^oC!EIP_Lo0O8>1uGzPLXfYgr4+Y{zjHEibiE|Hdv^1&InUZ`;QdH9NjC72 z7}tWqb>tKKew%Ev@3x6kd&09bzUcV2s;6Hd+)`QhfQj_dy!%v<*-MIixr?83_-M(M zU1jj8=H=S60%OwOtX*F~H2h#sUSZ}Ei+XfeK7tD8A{r*ak6ic8x4XBiJs7w_c!uwb z6+ttm=xo5({4o2GIX=Xmo@DG-9F6Njo0}G9f;BUcax#A^e%Ghoi>dLBV^d+_Crf`! zXXLl@y&BftMc2wvMIQA>3Nly%z2Lj^IP*I#Mgn6~A zF;H{+;ZSicovZ1vU?JL9Og?jjWd3@3{advz{?N_wHnX{(%p*q~-dRg_h_i+YE`K)P zS^7xvNlW=Sw8lN!Cm`mRyC+haRt2-CMDEJpDT8}=U^Y23$@Ttq97UUaVX`%&qz|Rq z0f@IDABG}MpY=xNw&THOU`DuFi+y=C)A4|kHlaHU8j-fE8S@BCKJ&H>yZd&UDw4w} zlDa(LXlu`>-*Id1w&cZPc}FjSEVr2lNU32gz{!hTH{I0u0{*%4)hPxDkG)igqHOk5lMzH8%*XDdXZvQAnv$4&!*B;EjlH=?6@Rk`I`{f7wMf2?&H62@fvN2np zAGY2L3&JLUqm1RDEV4>=drk!Xb34vrkc^JQwF6as7M+r~j1w)>TcV;b8&)Nks|D(|&gU6#QME7hLnJ`c(ubeyO zCMGBAk;HB-wKPsy&o>)7YtD^OPvC%cL#A9>=rb7l+YPc=60kg2AmFkvdw)e7%tDlDxpbS}1{)@|p)f|7yISL%tY*3=%xbpmZ<8!yk9#$F zNwny{c$B|RX_w&(t?KSScn&Iz9K?j-s1h;!II0(_E!eSnOP7DXDt6+CcZcD9N@dRG z{BAhp=7Cz$;#SN^dhfsz{IqR34lS4J^UQJ-`cbnhOVUT;cna2O?Y?Z901zhwP70az zJCmGUtQSPM{>+!ngKbm!`q0{Uxb8^PVmXp(>o0J=cq@nqrp;V7T(qylGpJiM+NlXU7f>?Dq?&t9Djny$*{1F;mn^ze2VHVWj0+kE8?Q4Z<260{)+f6EySN=fLh zAurr2nNE?pUTY~cm?~8#pNe{;W6wlXpBXN?bqsa#^L>^06c{2n{etk|$fhK{IhC=S!zPjr@Ia6| zSqb78rE|Tui&E*cspw~rTnj>crAE6 zz9@OR*#X=`?tN)r{birNfE+uGxz+p-gbZtXkR~?_2(u-bDY5*ak)BzZbvV$5C@7;# zx;CCbObyxuEQItyoogFTy}9^ofNoPs$WU6cog`5z#mrjcvn@=ep%g>9W}Q^u6u_Hv{BUr%bS~qf!ECmgpc|O zS2=ku0RE&@@3VpU7ly8Wzn;y|T?3tVN9zUR^|80X<{so^08d~T4ScCgjYi`cgyQP# zlC-uWeTpgl3zUH%4MxKPK*}7_TexSDuy2it z&dccQEzC22$IV`6)~0`2pgFes?tHU(y8XmGmCn|Y}pZA4G~b>>P%LU@8H zj%(db5x5)OP?r3y-qiSoDakcQ{MkRdUMQ+U_o%;@Najg0` zt>uK?chBdw1m;J~4aoT|nTykba2m(x1Dt?3?}?+8bA_w6rIN;07t)6 zVI4=r%QEb`5uetnl82S$HKfTh`wCNe?a3b+*1n4ONMH;1a5f^tuE?RsX&g(*^veWD zhzcEDAmFpHIi~xKDYmWUYIV-P12Z~{f7utmXKrX@(=iS#-h=%mxVS{3Qc~1laC6|+ z2gK!X7&V&o!NI|qSGd?wX;@<^*LTbB6n%ya<4RXl ziuwCkpKr$FNIwj(XUXjy?$MTPA4yoWrGKvarc6LF$NrRf?WeGL$I+5XJr70f244pVU_Y+IPUgoQ#KdP&M1ynkA5w#vf!5H|J`QR)DKKKkHWF8s0gI z*>2qAF74Skj9_biujbcU(}ObCTz?OvK{mPhQqoz=HtE&?84=Wi%gveW1qF|5v&#V; z;@=}OA#7}k&iH()s=Ef6a$;2qP3!PjS#Np1tZFf#6~eS>xA?W+<{IT8ovcn;dlW%A)G7shUUR=@N{2= z+jt|!b8Z8u)`g*~`S$zlf#jj9EzLZ|Gr@_D%-7SiZsPT*IVK=TWe>eg zj{PXrWh&%n4r)ym7S4&#*);q(v`brg)Q~p77g#TiShILf-F2Tv-7=7wBLw=QS%EM$F3&j}(EMPD}ds%+57IR4hL-hRh z1tEex{#Nu@4NUN4A&)VYz}cmdk?}run$Ym?6ETH&mnXdG=qY>g7u%M-K|`*%m6Y~6 zx{;GUzpeOqGS*E^FLAdjGsZhlrlf?tP$q=nC)~Q!cjH})zm+`={d9DmFtA3(B>F)$ zTja&2t$3e}ugix?$)qyt%R{p?HY%R+uJdE8w8oj~^k90APFqKGZy&Z3D!q|q%228x zkhjI&Z@3c5`^C3y)?mip_r_JsNKv~#pwOp_Gbp-VbuIJWCvOyQy;4gVK>sc-x->`V zyqwNbHSx0%*XE|~^X~)kFxgMVyjl{{pRqFpQ^#t7LvwVOT^Ks}XU4sA&?k+bA|tPd zwOag}I!CPdrOxo>hC-`O3sjeS8Y-b#JvX-x2l0$#CrZKOzn%8(fFqbW#YNq0IRc+6 zON7xn?QP$j#^q^xf8HF|wO3HDXtlD`f+++0+;kR$=~Gqu01?+j_uVUM(Q78Zca~^d z#sbfMg@jP=)7=oAH(N==_01CfW%-xYm|@FY7!UaEB`T$Q# zN-YJ@E9SrCb4z%0NIy6~5W^SrvB50!0m_UW{}3NrV(<0Ee62agr_Dyf=lkw_ zD10R$ptQ0PDVQ}Vw#E$D2$~9SS8(BEx-;j+-WsqzJAL<-bbr!iODCpVH_U11fwRkv z_Iu(uJGa|=W_pct?^4M9ob+}S>Iy|mT{BzdF3VO(y?M`P)qp}ESAaH zNK~T6{VA@*kTJAZW*amjOp|f`t%x z7ItH*-TEcw9m4(xKLp%Z?{{-cerR)_v2gd929#|2@w_S2fK#_}Zb&69xP>8GCn~Vw zVGfqPH*S9L!{cD^l6*>H# zM_(nx;r4X9W^U7kSic_u!O$h+)1$F2__x#@y{L8P`>Yqm%Zsrc&1flSx(-|-AvJUX zS<9jyo5ew>FHcFe_5)OjswOHv56o(7ZiK2^-N7RpD6wQ8%t4VfW=H{T_X!#%z4wvJ z?>=Vh&uE>cXDQ4i$B^5uN`Xa}NKdFfhHI9eZpp*-7O~uro9T|({^yVg|7@}d#5>9{ zmN9p&P z?PRFOx$I7{gTf(1`~;bGq*3y3kBL9M?RN{OJ?TMaOpU*Hm)&<+Hbz-se$Q%EV2EUa zz4Gb(JJQ*iLTiUz{-+<#bQO=9dE`2*u?s227PGU%tovcM>@xh%V?FzG3iV5AC-eF{ zaEO5N`OEDG^K3AR#+Xp}iOS#XH@lO?ZDKR~@*TZBy}yi>(0Q=ufmMRCD4vU1GequvW}5FF8c_yy#YH{uhPz}DgW4uBM*63GE$8*C|p0#P4$ped1Kb4AVg zTjo9@rLse=3q(asL2IZme};al+1$^mc8$r6^TTVa^`$5yjhZHtz({Hi*M+706gGg~ zM>!92S;4VP3n|C&GkTKTN@&5E5;Vs9IU;80tW!>DRnob@!5k>af{7+UTQ^uk=qgScl?n7f5+i-FhCxUld~nQbqFY z3$*uH1$B}F=h_I+b>%;t)z-@hQU5EwYew0JF=LP{ZXHTru)p;^eZm0xyHQa=0+qGwz$!HrGcW^XaFP&R$X7rG;dM8s$G;D{3%xCc(o`N)KdqpeZ; z{9p{Kc^9NlMW_L982*L_)!yUv-R#K2-Y*jf-NOKg&9Lo<;MR@q#^cQsHPfMORROAT zgx-2*2zEp?@rCA1IPOj<2x|fm>)dK?XVYak$GIc}`zZxqUI@A2Ch^){j& zQTTZOvubg5rI`PM1DM8wL#_SCD`3 zK!tk5_sM-gS6{+Z2jVYss<^JbQ|LfkK12NJps=hKdOv9l+4b}=*HITrfeTUivBc=%?9BwnTzo^MS3?yesc`e~eR7VahnNkTj z$yxx=eib_n9u{(!wAp9qi08GY>goffx+mB?4jPs`l*cOvm}&&5a!QlVF&z5j&b)kB z)fnxX_bSXU?*}%Z)UdE;=-?cO-qs?kN)k^dqNJ)9Ytn^GOU~u~djEy3(4uXKOMrIu z=ITSc8n#>ImoZbvI*${3g9#b(7bP0O3E*Y=_b*R+K>&0|rmIgnr^luIZWWnBTgYEj z)iR`3Yg1A)a&~G0JT>3XEH+TY*om4m)4XtO`7w}^oFNdue+X8@xW{nb9%=LrdXR64 zP6ttUEI(p`9SieK=ijt6eU7F|W{{n;xya03pmVDBY8~X&qZD7JZGhHl#0WWEVf(iU zX*U-zpP)O6X=}bV8Z!VGqo@|F%vV#aTTBP@_yth7tvc0OGZ9Vjt&8&p*0g_F4)zl|^EzZ84(GXb_~Z*316`(} zE`EWr1Ds8t$3~L~QMuukT!12SU)LX*Yg5B@(yEQCcZBSl<%5 zK9F5Etb|$lly-Hm=((4kP9<1zsmQinLt&!+gm!&V)XtVCIl(e3w2|GWPg(QM!O{5W zfzFe22DzBD>u2cPri_64 z0k6n+d|s*Mor?oAX1E5fH}9bQ6p@mX0VXbYcw%$Z&m%?LJP(+I=TRihbg?u7jt`KP zcDQ#Z-Oay|0#+SY)^MmetCKfsi;#wf&#G%SNSI27meddbItliKp`(ASpF_5OGfVU`5hq$5V~_-v{mamnB7!F33yXI?5T13v zj4&D%_4~d)nc`{q_{t`vx(-9c=T5e-gpIt9a({J|TR!e}lr`DA(WBvXVW6J(15tiD z)0R6c!@e!-pogO;YD&mAEV}+z!Yi;AD2%2kcyIL*^sORSMe2UGa%wQE_jlN>c1Pe~ z_wOS-Doi5G*@L&CIKx8t+0541DIBT)rP%qGwmbnS`ju+Bc#bEzt*{&4<;;&%$nAWWA zZpf@gMWe6vrhZIq(;%IRH@lb~w!J!B>NzjOYB-%HM%*A=29_GlbiMf}Wz<$TIPv}SEw9#33280r z{gW%dJ63Ad1;YK38(XPXc3lUnhs$~~hGt$@aXK+VBJ)f?!u( z#RRL~)mclEBXr_2l{05mZ?ak#$&+FN<3B159vOI*Q`f0|9P^oT@+%0Wf49~kYrEcQ zaeQ~IHINXcQ>W*%6y82-^P6hS?@zW$t|v*+kK1ry+d{hQ`MS_}Sn)oFtshl}&f)R4 zH+xL_-^l|1GnmYg)#m!1o1YNO{9&?rc!}1!)Ew8UR9p(#T$4nV$Eq@>qP8OApQuHo z3@3qz7Qf`3l_fW$*WrR1pi=7jsp3^{xs3 zylXy|4!CN@zrR0SIJ6h9i=|OUXU54vdJGnto0NA0>9X%|7lqPTyQ$JN72J(u>&B@H z!A#5*4!Ze7AX-){me%}VJn?@&K{$@K$8dy9mfAnSS2GM)QZJ6h4<+S`E_x5(eWZ~R2CKD|R+5om`juGMH%&ai4uUtvyL zDWP5{M=%%t&(fiIm1+%jB_)S&w^PvNEwaUchlKoXY+#`1A|0KA=D7nmxE_7L#A#?} zSA`E*X3SYxr6cc0`-iZ;X&LhuTc!W3_1IxQNx~we8Miz(%k18C$aHzaY+-JxcZL4v zii?T)9{)E=k)`2(m3@TQGygYokya`c{@-W)uQ9{_1DX+bp?~P#Ck;H7djH>mN%>-#6{xQA7us_71 z@QI1_>QRrPQ#AN8862U|i`spZ;F2T9v&JFT43EcJQcNAkQ#@d{mq=e8B4Rp_$1$wFqz^q z&_{IQ*~XoC0q5PN9XL0ixqsEy_J;hh?;EkCypjoz{by;=(OZf+yoK|CO6HNWh&4&c z?Dx6iVvMYlPo34^b10_f^&$i-mj`d|5|gc1IX_}d%6)S{fkx2*zyszBafw^}{=KH` z-0w>YIMNH#b(gEnpZ`fw@o?z^-#2&XDjS`^dkZ;S!-V2d&5e^}eab~M>}lEamxf5kgv zcv5#m%%9i|fjY{7klNG^E+vdSc)g(Y(uWeW`EmWdtM;w}HW7YJ<~##X_*|gNqPDC2 zwf$=%#d~UMxVXi)?R04a!JqMGz!yzUs@5c?zZK@_P=9I*c3c1%O( z(&c~de@RRteim(M3xi)+cOWoa?RDhN+97Vdn9ztCnHsbFl^nYEy0CuuGZ)kO6-0Qg z=Y#L;Hqm^*VN7M%w7k^rNOV8piRiz9LR6ESsHA7No$@OS>?)>lUweKB8PEgXK??vj z(`R+g*9-YP*GfuEtleFo$E8nB&L|vj1K5u-N9cIs9wr~GH5hmfpHdeMA;3v*i|G@k z3+0f=K|KX-pD^XENlBfeF{v7W`Lold=A5qhGW&M}gilYi5}Qa6kzJIxc~y|ip_v0L z-s5|bo4O;fTiBd$nNc)b3YFpB$$DW$)xe&<2f-(pK`fP}b`E3^rPDx%UqON=BpA>n zV!|FNMczAgk-+)!HR#Ph4=V3m35uwy_R^j%<2)wn-vl!(eh8lKy{;>Ih`+M6UDpAg zyCFr?g39mwrw?SN^j=EJ%fB1%Gwu+M?hV~qKRCYN^M>g3$~Ud)cKKy>YVx zgQ`@eLl1nl};KX<^RxgXQ7J?uNnx44i(!GMt`-Fj91 zF+bsh_0s0UJ1`K#+DfAZJ_Mt!n;AN~rQ@KRVJaaHAYz*$QYrjeGQ7`>y9C3d- zNMkDpzxPhSlgj3J`h}O*^Njqq40cCrrn->P`-^D6L1)Ll;+ek`^h20`;m2i#`1}+-tbwXOqMmQvSe29D}uA+)w|jmGcN~*&Z~tO8YX@(oTy}mhYyG3{r)WC>UvFF2im`X^n5G3 z@!f%MJ^3O#KN<5Ozao$1`~fYTudut)<3SwrWkKJP-Ba2Xzw=aJ7?ye}_r`+fFZuL; zkD=u^Sz3))T-~>)$kdPIR4@HK(1+u#Z&T%3uLG9%T}D6A?oQ+J1pj^^s=H{NB|FkB<< zJ9b2C{!49!ns&Gh)x3W07p3c8QsfcYSrgTm3%LL&1^;l%UuwZ#-sIv!)Cak*3H=&-k4c$Zqp0uI2W<2!*ET}hd# zux~jiV!f0k45p$IA+tE=Q>Uzn>1JoB)@Tg$m!2zpri5JVQalP8)3Rcp3)2G*dR$F z23nk-JJNsu+|^ZT_HUXEA3k(NKK`sfXCyqHH!ZSRVHWQDF!cT;XSA zp{AwPF&T@f<3(6g_ z~4gO9>98jxHjo;ISEMX>-Yf zm1HsBWm;-Q-^|-<=R+KKs?CL$B!_Na?uZix=Lbq(1a$&jQT!9-tWeLT_b{s&8s|jK*)c_@t={@tsZ>ex7Q&#h?B3KciuBCT5a@M9ApYQ9Rg?$BNZ*um3U&hQ-h2l30E_d0|4Na0;~j z=N0@Q|6dvnCqED^n#IR=vL@P=YWBnj!`okv)Tc_tt0jBkacNmvkCq)>!z^ z`uBwO!zUPhA4$KS@I%LDf{Y>_4WX871f~L(%>wpHUnk%-q3SfCOCze)jw6LVp^FC+kJ&L^0%T|9+7=!3vm zs)G4{H}P-^6|(s1fJ2NDam35>@+IzlF^(rvYcUxagCE3Sm$QsgYdKmleLU%3UIO$} zF3#ZB4H?UH)f#iunAW7X=pIwA=^UbP9o)h=pU|A|qd=|_sN-cqJLk)?2ZC;tTvM;` zr;+EIs|<9~?M9R=L_O`M25n-~agR`%Pyyn}ScN{4qn*M%R~5T1YfZ`rPp`pSwJ#(xN@r8;2k9ZE`+t zLG*qoqU99bE8m`qgX}HX7%p_E_O=;KqR&CXa8MnQwXt!d4dc)1w`g)Nw?a~J*qt=s zB84{ENs#r0qS%8CxnbBXsCO2^215ob0Wx)u!PTOs@p#uQ#ET56#3`;lF+R$iJ|i=8 zW}0mQrbHj#7G8EVl4|Y7k^5bOfpqI;ucJPnsNS-3KN_rFaryyw*;o%n^frq41wFX? zNZ0nRpF&SJV!)W6*Ef;QzU!n;_@M^8a$hDJRARODt{66Ay&vBB#FmrXrq~pRg8H!% zS9t^PI<2{XJ3N>2$C{iBq8NM;RhZ`f4gc694TV3%CzuJDiMH`K9ESKkHfG+?A%I_C zb+ui7QBj?vs88PQ*%>ka(_#*u$AB={$oZC>JlK1`wiyNh*Mq*CdQ!@#BX!F%y(PpD zY>eokQRrRB{IPuA@ zB65z(UsgZCH>OETugFs5%6S?hYjag?t`-$eSCyesF_G*@7%V9HK;-l&)|gz|x!LIT zY>NWBIa0rptQ*@-%25vda0r!(a@>n=vteE5#~vyD{Wjm(ma5F>&qukA;t?t|9*#4U zdZCQJAP8a(2uRsjnyk2VER`*$Eh~R$42*uCoJ*_y3_OCi^A!hUaUlDEowFPdCdRdujU+sr;UZs6tngv~+YHr*nSFYHB01ibAqBCrJ>Cr*05> zkNf;5cQQ7O%5V}x7u^72C1H(_QU`I8KnKN9w*_)mBt~2g!yd0DU zvh$%AXmiUwPffcaVoLx5)&>OZ_p@r%lG+3}LgK#F%rg{MlGy7D>U{RzL8QY{Gk>!a z&*xD!U>YfKIXA)()C;eep8CPmnZoH|=`u7ocDmRl&}~Q8;~2`eh*5f+H;Ue=nXHJ}&v;E6wm)9zzBc!z!J0#@@Z8Wx>oRJb7xL^(qaHmMLJ4 za~8pG(R8|n|79NWw7DlRh(llY)e;>=D9&tVI)UDSK|xC`&hYN2&6xg0*}lnqX9pTi zlksF>X?GI~<7sSBS$yt4+dZA_nsezUGMWt5+Id_$F3qgwN(Vrm&X3!1uwoJtS7XsR z(lRnz%l2J{w?}HAn`^mrPHV(&BKMm<6V^JLC1qk_7Q%#)@%}g}bPW8^j0}RNRbO}r zrY`nS+vUa@$8ESm;IqJAJLC4K%OSJ5#0C%M{$IHOLOw!S_TX+AoEo|l`O-Pl`Hq+n zf*Gu7zAtCJV`CWz+e3zRUml+_4zZ4|o8Bh2`%u@b+6fp*72MIuErj`YBP z!#)}>^JL-dcLr5-%f-BTrzz^6imVRC6dUr=i1fXqw8yyKu^T1vO2oKLk2&Fs; zc?eG1-w^&2X1lM?aLGFQ9bV)R>vGSXs$J-FWhq{r#zSrpoc6zi`!>55rgE-S>!;@Y zsu-J4(M!@9BVmx2dkW-Uyl74VZLxczg2sbyr<)#?cwNKz!Cuf2eitu_e$FezDo?HX zL?`@k$a!`fJyeotp4H$uECPd-3p`Y%*@a{d3yBx%`y!taP=`9{@-GUcVg6*fWR8UG zLv@lw6zFWz`gILC-ABH$xK-gMIB3sH=?97$Mj+gZa18XUr^}+%9q;2>xYQD2oNr%p ziy`~*3t84m_k<0_%!;sK>gMb{ve4~Q=r7iTP6tvgw;rRrSgHxXU|vL} z{wQjUife1W!5VRc6BFUk%&J?Of`XkCaWN9cC5=sMzA44U7HgjS@))RqLjA7S`j(?3 zr{~t)`DKrb9(0@4+7QR{4pCtPgONK5D(4T`op*P#OoECM5)vcVCz+Ybr<~f;ZpZ3n zA-!xjYSV(wZ?`&PrJYfvBG9#qj}z2zlqkDBn=c2-RgAv=436+)zF9pB`jZ9|EOdol zZ>=Yb>vDKp5nn$;AO3tL4n27Zs`Aqu^$XL{s6Uk+v?}bp!S_`ZkUQUGnuUd+3)29O zNanosZ1U61>vx$NCgi%5xD%}|Gj2cQ&Eh>Xo5Xshp6P}%w{9?7n{^E?O0#J?6Y6x0 z1Pq&Yw*~~GRtoGoyq=kBUJXhY+4==FEU$C#wvP+Fj$!!wFXu8Rls72WIS!^Jux|PK z<(%0MB`qbVxTQA3(vngBkt8Wp;C;HX5^XMc$xq1V1>@i9dYnvXy;%HtWZU0g7~D%_ zWOCB*Hh1`1cwHHOVv#Dcbwo@sI@=W zgG0iV^76vZ&CPW=a3B%_T@bx>Y@^tcT-f-^`mPfiGZUmqpYIc*0I zBmqN0B>t7U_CLgHPid z=k8Mq)G?XiziVib4ysD4uV{3YJ#9 z5QHf3u&}V?VeE{SNO!mWt=6ssd~D61i+tH)iI&`Rw?Ckl%#^BCO{_IDa1)xZxic1u zjIqX1o&5I5?faV|UIGE{dfdca;3eTHO(2PR&r3lIj+KarnPkm20(#}F8et*0%wfp? zkOp|9HXX7)3ikgv6~O$nbq7y{p9Z+|VSHWDe|!N}KxGR5;~Y&;sVR7mN&U52e>>t4 zm3qO>v_}VbSBQVyH3?|>`Vp+_DFyx`s;0vVC0e?KNCY%`|(KjE; zJ$Y2i^xB{_inT(+rA+_hNmVTadNvnNA9r?4T6q(rwCwb--Mj?@dJN6o4Zl3%k8Yx& z^H~3LR^n+Vri4=4S8O!GK2vBq)m`4ccXP!hB~2-KpZm&uAV!xa@MjNxh3 z{69nTj61%$eD#`6JU%3`-+vy#s8$ug@S?Za;8B;-x;zLIV#`*R_5E$>H$Np2I3DN< zgEx>H_)g!crMHK3VIYt~-|OXzisj2Va3ksx1^#(E@JNT8mO^CP#4G3@u!f8h@$vCD zCvzEk9j|>wIdUfR<@&M>_eZyYP!xj6-woCeHwh77#iEs3+1xSn&a!3bm4dW(5;j6Y zY)4t;@Shum6ekZ%#n6dqnUK|Pr}z&hq#BKOeX>klgB#E10>(KUulD@2`W=v6endjA z&k4~_FIIZQPyhk$P}4hX7t8&T^v5zDJH7^hN0!%IS;MYO8$92p$Bu zsTGB7MOZ_LLsy!&`Nd2%3qAkcd2-82?e~Mv7W>*QZOb3;UiH)iLjlNDwg_~Eb!@HB zwmo+Qk0%N`1#7qTi;drt7eNqVN%0lKPByQc5KDvmLZyW zIw-W_eD}5nb(Zo*p9+q%P?mr#qM(RUB zdo>iDWfGmZoGDNi@xDxg_oEYBDL7)T90C(2fkucO(H0h_GQ@$u8d186rpN>+ihYl! zRRJgRagMo_&wY}ZQjyepVC(P@#1p1bB`&OlYbTMXGIV=$Q|P6tJD#nlK!ii`!%P@`uLM+0D^XR&jB$3N5%O$1h9UW521wX7OQaA^Nk1O76MQ zy^@yBI_~lBf|K!DCy#4ZH#i;nHZPow{p@XeR{F$fFlX#30dqzvtZ!LKv3b>B13^AT zvD%uN!%c z2cXC+r)SqVJ2WvdLCVFI0RpLTI#=kRz|#z&#HVb7a>%rpG9`~goGm*Xi2o0BUmX_J z-i13zNsCCQv~)?Qlt>9EE!`c`F$mHn-JpPkv~+iOcMdSp-2-#C=bZ0+=X}p|@85U- z!3;CAXZHTZs`p)Ml@KZcgHs5ML^LoG6Mu}vA8&q$+LM+BTDUs5@H!u1>XGmU1@^w7 z#5q`kM9ijkl;t_!5mXgkVii%`La#@nB5w=_FP)=yh6YCa9O6BThp=#CoPTSk7^W`# z)=dOJYV;2}Djo=HH1j(@BQSDoXZHjp!8Q=Sj_LW)m9^0s>o|SjhWh(ILPyu-HrMb` z-Xff9d#Of0``)=m>jM{?xRPt0e9!cs*C)RhcxKV;zy?fX&i_th`ks?6`@m0|8`ayD z`7EtU==$NIlDIfkWE!vO{ONb4hz^BFYrC)OpRVemm;E-PC$>*KA@WUPUq!GGYatC; zQoxdKiT^hVB!t<+1L8iqmCBk)Q6tXgYW2`TosE8U%n5Z7pK%=ZTDfHSiYz%m5!lVPKyqrb+s)SV#O z-wSn_wMs%}pmwT1KITC9l4Vcx+cFvb?JlM~|1Uk|9U9PMSMWl|VZU#svc>f@p8d6g z@K8gw^&0?BRM!=N4B*L;u}63u`giSfMP;*uAMaMr6Sg7;LpF=U!KMcMZ^VjqT5m#o z;XFie6%RPQAS=XPoJ>i?V`(kHxSVA!gRcTFAo%te1 zd9!?#=NWd%-mffNX75-iynMPD++t2oFSq(R?JJz`WLnY$cOsyqm?#UnlQFFt4{`?$ z7xm6-4tik2clA)W`Wvy&!5ZebQ_2=oxpcD86x6I+*U0IgLeA0^kt2VCt-?ZHe?P;( zEOhwID*5|hu4_n^anBj4+J{q+wvsDOTuy`%tw$AJ6da7b-yShFIxS9SI5yyPR5s>C}tG+yi z{^E%Y0edna2N4d3}XaWtuZZZT(R|*r@Zp8=+zZ%HNv(P?jmpJ7;ka-$ENCr1UTJ$@pI{HwM{q z>*bEP1UwF{M;~o_yg{I__A3#&%hf0OxaR{$YEwUdUCOYSu;RC6bydJxX)$yOX}(br z_V#a6XFxR%~unSaoN?s$n>{*knOvV9*sddH2RD)B)PI-o$rFtml1ZtmkuR__LA=p{{zB z3f+eHr}NLpIAuy3>xf-b_OA^Lp2!o7Kx%xiPHvM`Ow95S$L=m0IU5?!&dx_i&q_md z$(D4<{X+V-7+c$bP}lfx%kl4H?N*T36vKXsSiWamjmbQn1sE?DVp%l%srrt4`Xg-_ zi+gK?f|*_*@x;g=57Y z_~N<(ylyro9qFLjEFSxjt8!OR|QqCN==uOI+>m=29_Hy_QZBY7@1dg_fOBdnm|HxLqtv*9~TjX zW`RLkDa1VX^Fdg94;vOng=51zDX|NhO;VKYY;MnVys zI0oLL>1o6CO22|_5bXi%EdA~nGosCKlwmbBu%9ovPrbPS4C$mhsDFW~T;H?2FBvDw z{$%DW4gIPC=qZ5`i(3S_^U7D(#zxDjGfUy7fVnd$Eq3wkVY;y^0nq2RhwJ;L8wTr>?<;fwr|}l&K}+0>xKb>AyWiVrx87Dk?R8sIb#7NA-K%$b zZ0Ca%NtM7YuixkV+xaY<_{pVqa(SZWFM|xogxG|H-CYhQT4qN+7r_;;p0h{R)dokg z>`Y7cCDupO9#>or{N-D@70!o?`!~)LmSu&jE=X%Vl@9h{GPNi(Y>wOPQ~gOo=d(^ zAC_BN0s$>8cMq?mD(RUJdOn86r3a^hyX^OA zu3fxVJ^T7CC2yCg-v_1@A^yvW7NFcIlp~6yxW2w_1YFG2+FCK098}`M{!;AkuK)?x z!A<3V+>;?6WlFZB(9=FlH`+<+*27Je{c17ode%Y#qBp8~Hn6gvdPM;WC566)48Ygc zwa(obe%R_;>E2L+$dGxLmWDyspJ!0_+1 z(UT8Xw>>%LSce{st_5e(^7BN%yh&w@=c^HT?R6y=FEbpuiTK?OMqeBNb}WGR(2eqy z=tvjw9vtIfn%Kiiotv?!3F+SdE3gp5bY!uiM@_xAc4$Zq)Wn?TQ8GH^fb1NEkN;|~ zpt;i}AJLiz)Gg~>Yj2OeY+RL(N@0m&2dP`$R>o0<F++Lm& z1zPgH_l~jinyLAqYWre!eS>Vs<+b5ZCzv{*df7FQTGkHYEeb3yo|=3&;t+pVb#1tOJM0MCYdU43 zCsSO~8R3W0|J(Wrt9f~X8Dh{|T!H^1^ZEP1IQFK_b9S-qwg)7QU}`5B+!z<7Mc>EV zh4a#K0ZH$WBLN}CwYxJrXnQoCPpQTb&0q(eIV0^OQoLw{Ix3b3uy~xkP(2+f`)@Z6 zz`CpFm{5y~$#lX51+=eiSm=&A)@U(pr9T{PVw%5Ms0hg=4}T0Oxz;{S&?Vc=t5D)3 zs+_c#%M7~?qkc7~Al+EDLZnD*!H#G{iay)3)Avd1U@JE6k*=H&(7Ffb=Zg^M-H5~pscGT3+&Ehz`&207bg6B!44EpY< zIl)5uK7uRL-$LGn$umhHQeC(|1&MfG(a^Gx`NeP2ZjYoZ?VjYu8Y`Jc(J=~>LD*(7 z!P-3_SWfndgA3U|znj`=-t#_Cr4(BtSM`Gv8i}c#Kune6h`;C9`C(E4At_e>ApbFx zXJYA1mJ#Nzgv}9ZA^sv}D;4+63myI{6BZpp_@{30lY`zS2^VgMyH|EQf=LPXm=Fh7 zmY1FpfC0~oB=gyLNK|~HVRWA7j(eiAguxS=s{B5!YIi2!%Motj?THpULG1_1v#TlSTy>|CJ2W^Hdu6yi zQl7uYlQSx>cD?n=al4hvKR~XYs1wC;nx;&(^Hv0`Og7Y$;Kf-@^lH+TGuIxLVIvQE zD7jT@)H8mh;5A>nM)TCwrp;{nfsq;?pNzy&9h(20tR(jzu$7b&J7ey`i{7QBr2&4t zpOq8LLS#VvIjBs3%yN<2kzrIF#1HsL(W3XDWFcqILWu36+xlU9G>6?jH{r*o80A(O zn@%%v734@LWpisTD?d+qBWg5@OR;cI-(taJTim%=&rh#Kf_?FhAvd(h`_I2N;!bPJ zj|LCPT$C9Y=<0vXZSyN&wa_&xkULvw#qY&~2!SteRIzuPF|k14=-cIH5<9by!zsk9O-q*46M>z1@c~UC}|1NwAuNYmkrq0>q}xE?^< z##8g~MDe;$qKhY*&hjR-XbJnt)jbU@t+w*czg%{}CJW=Ilc2GKs&7AV-N(~LZfr&r z#xca}e8{1G^{Ufyu?6GznzVxwfLbzzKix7sY1|;*f4Bg}#B&(TE7gCyQ-3_^ z+v>AKBXO{b?FoE6Vgbw@XjIHo%nipkck|k>Jv!dT@u$_zt30Jl2c`SPGp>q8j}1rF zPZ&!>$xXv_{WzJ`_EcSbp@@^c6>hVivI7vNk567oUbcF%7-8;<4_k0+4fgyrzWABF zOV%SU9Pl9bqa12(z|I5NI(YADGlRLW-Z#&msA%n~W+7IH#axr)I49sDR?f}zy4S4Q zNEsH2Rr%0%T82(qGsLQSI&7-X{4ec7O`Y#x<&W}=&hnW?IsfWr#3ZrUwMh&A&M(nPLy{$4 zo@*lBCZHe)XVeOgmOunlfxN0?Y$zvabj@LZ2en)cPA2$Yhy!r_e{`(>MswIWm6XT) z1{8-FXL8)cUUhUJyp~g9DqL}_*$-`XW_09NY>dirnNhP@M#B}M_};({?4#mMh6DQi za@3clKr`aPK@MIDsT}K~m_!S@B{32lzP|oigW5DSD{p38SwuO3pK0fq+?*O0`8`pX z6OSIEQ9i(NF3H>U8c9|}#e2*y*|lUcd^>j>^g)d?Fh$=kTOM05equeO!sKCi1yH&^ zy^~dknY}X!))lfaN|#3Le5JOd;>_b`vP?hkUw^~*LU5N^7XLVCOctsyGx5_hmxolb z$QM;p?8=-gvLy8I90)*4!ZurMBE|9-U;KOCoV1+e{DUg-HTL-#W>~x6*AxXqrpZA&AEL5eC8+_Q2%tJcON>f@&iD2^UmR3eqxwSE!^gqTriLW<>LN(vFQmvGWWblFI z8US}pDKz|h=GFXz`nuTx_{nk+$B_Qz;v?Nh_&gh~cBa#(fla4}@v+8$&J#Fz3H*J& zu&tNu~*3QW9l-eaajLdO9zgUm5tx z4KD_75pr;+hw~*z3a6K6Oj24vb6Y)l}N_V0%V_XFX6>_AuOg8(X0 zP;6S7bj$GWl%t4u4z}Lz}%y?;U*!p*A~H8uKhh z&iL!hneaFH7q9lDO(#9SU1W1vHjjC#q*UoyG#A)BJhW>T*DGnU4|r8N-Pe^S2p)d_ zrC<*Wdp)hX1t5e4Mm!N4zWo;xK6QU?*-f1jPlX2nXPy zinK2;e_qZ!n!0&ZN8Gk-;;+kqgIU+;#eWYpSV9RPJ*3CMWJ|6L4i0_90QX008#d@# zUPjvF_pxz{BMz9p;aeR36*az0PUtNJKtjNs9-KQaN617R9@ZI6P|$Oi5^4Sy8E~#I zCA0&Oj#qyp9sj|6ej^khZ!bU;o*<9L8K16Ug&%opvgMJ>c(BOm{(#fXfXTR!k&+poi@oUy|xWoA%aJ+Xr` zx9ZxaGPC%2KHWa@2;A9~yUvhyv<73O=gzOU?w z0G!zl0_G5&-EZnV1YK=J@@lZWC~CH9#_P{j(41IzWDsu2T93wyNaLb%*|N z!!|nhcWY7`*%*45h)V2G&EoPg3BzyJJ+9NuY#*P!`sA$tnb6tQW5y_|&!KD}CxKk~nL&|entI*Wg+*N~8R zT>n44=B9!AuZ(T#BtFVjkZ`j%fQd8;KEth#*o5i;8|X`1#(Mn8|6oGYY=%6ZPl(8Z znyIpihu+z!Gp{+gXi__A0VVy@PA)G2|1kD^%4Gi%_J&;L_uTyRkVirZHtZjMF2YI- zMDhS>(&a%mS4=Q58C$Wr4ScEk4h_@4Lw-`}BfWeeF9}frfsuJ_aj9Hk0Rg~-IXyYS zDwuNo-)mEX?rcfg*(DZrKEXwx;^V_m26|qSOa2?arr!8SL1AX2Ui8PO+RUmq)(uOi zIRE#%*ftz~PcFl|hfx~0;2eu5!JQH(|HyFu6QZeiSo%QT*8SDDGYDQmZrD)QIXnzso?*Ol=H^h$B+n5p$C3`eME(V2nf&u$GkY_}|D2zv zX$aydRLkLViUcSP@kQ(pbza9b5uUd$p67=e6;YjnCkxpD`%2fdvq1^oV!lY^qvt(M zjba*0_XrrZlqT!e8OK7B!*S(N%E#}LX!DvyLa{7CW*rMyTgzD*Kkekl=k^jupq;&* zlh@RWYmcp$lOC4$9NAuA_?BcDY?rDXT_xflpfKjAnhaDWSQ8uUrcvlKAD+0XM)NbY@C4uigfL0QQ|GLMd zoC|+HNci-ry6Y?16m%}RgQ0mJt>-xLu6k0y;=c^TxSCG?Kl%_q$JL}am zrB;idTqWidy^SwKj&>gnwisVxGaJzVfEIaQ-8i3P3bnySy%Nl`ywju2iUZinPA&+` zCMs^F6YwRlms7df7E<=#Ul_ar2g_!|2t%EGOOHVK2R z_+_(TP+U1(Xeyrz`-gd2zD&3eUvx3^?RZfN^H14-P8C3}ApRlsM+7Oe#;7}R&eh~f zE`h5j+Go4zRxoEMp!nAE5kyo6p@MI$#!Nu&sTr7sbj7^yYtr?>aS9#}%&5^>%?3q& zyM(G9^cZLhPILRDNcU~-B)E3m4o7XcV?m(8n(vWqxO^B zGnmPt^>;vhjr*{Cy@#kkQtY2zv%K42VgDLKGY;ADvw(!+WS;U|qtEr~_M*BCnR1Aw z*}c}ST=ifPBfGS&ur2AukmOG7arm&lyphiJ+2Xmq)`M zGXX~TqQ91S8b*~Bxi6Ju7jU!v+6Sc>CfsZ^WPti;*U`Di*xUPPc*z^OXcJD{*T`OF ze3#}n*1QPvW%||8_bmgB8`1T+cszx)li+Nhe0IXN?~ay-s0Tw*8)7az{0v>~2J*ep z15&1d{lN3t^&pr-NFZ!cFRkO(HtO~MH@lCGE=bO?3!Jwl&>ijON44LR$MGur@rrqi zUB7yo%d~Jr^qd?ygGa6FA>Tcc8(RnVB*U3(uB(ZA_C{TTJ)*}Oq2lz1Rsh$irPH?J zJ2?5KnxX?E`h#3Y7j9GS8dZMoxV!9lg@!fD9(es~>+ISY44QSZSDfn)C8|GxRf`53 zGcUXyy<^$_Cn3-&<;ERWB>XT2q5xa`#-F}tLbuEs1%+{w<{DfuJ~Zf#q8=%)^^n?c z#Kh$2lE=p9yz$l(`!jHnLE#hjln01pV#DInIRQzz;i~8NITWl_?fo2(oo`zWD^+{d zq9}!qPKGL$iF`Gz=WcjOvELiq#d16Hsa3l2WLh%w$E$o~v~51o0|(W?5oDDxQu)DHWi%g;!3-X^aoG#O3srb~6e!`C1hVv*pl`1o}p>VSAK9 zaaX12O2>zbU0gHWcRcEc>f7GD8Sfjp4M#t=XiX$VFSup*$Cr&*X>X?<)D;TAE=F_| zbex_r>!k3rr8DPYZ{^LXuq6AGQ#U`EcBH!caHQO!!5lJ*%tgrQdkj8S7J2GC^%j=-Z2eQn2vnM`Zpf z=H9s9U2|<$-NFBK-2XTP9jwmXjAzI%udMinP1FLj{c5LFHAY=K?#7pFXvp zlV5`}_4GhP$q&e=mkf1IOw>?XZtj3pFdWMb$1T&t4D+bvj36*DaA_V4T5fP*;N?w} zM8!F|;M3ef_ z*4O=z!)OU%iQ`_n4<&|??>)u-dDk_Ycth#X>>_&9ERbGdN6QlQggwdapn0@(( zRJ}1_Xi9+vSsX#rc1CqaG%{xYoZqIVv)shVuVU%_3x#KTnMS-GrR``z*)-duWsR$H};#kskrEgvekp!Tr$+Jo)@oI;A_SN ztlF=SRzz;Bx6oI7y>ttSBqckExUTgm$nPKL$cTl_h9VVCc2Cwj6Usz+$K!~3XZiZr z_u6os%o!^u?#_00N+~pZMtE;NU#M#FPhd{c=pYvrJ0uAIPRp57UOv{np04M(6|wP# zBq1Z+@$~YA$nClD?&b+pQ=oI89T|^EOo)ORu0!w@O=U|IIMS%-H+iR{Ryd?CV8QTam&U++^f;YO;P zA?Tw`v!7NA%-)qEh>yQcXDE+}nNYB|a)iETXn1UWg;XZC7(A4vXLwA5F<(yE&~n4f z9+b%wJc+qZJQbgO&;0uF-pE(G)e6Ji8MPz(xrFomg=ekWiYZN228Hz2(RN)!Fu#Ko z{Iu9VJd3{XJ;R&f0#xWYtB;qO{W(2R*3HgA_tfv$B%)pk1zrlHJ54h7DfT3~AnDxt zY6Q~(E}B#Se3c#kLD|v3#;UoR2p75=4$fuQ!B^J|BJ9KPUaJaFx#94ehoeLxG;FjS zZ7EuL!VyG+-We=SC+FYmXyy7tX?j?nfV?8>oA}3N1nm#^cPZyPy>zyezWkpS?OHRm ztF7V^6I~a4dF@xawujR|{0;_UXWPSh(+Re378>35+WipRemX;=@O)Zv#Ai$S9M{@J zft)Nfzfe4}%}e--1f)lCuEmEG)cSCX0mwO`JDe_1sd>Bz z2VHN*g?e9)Gc>_YG3cOm_4Oy)q{afiO}`Y^S#@}`*6jrJ6XrHHO`Oe@7`EJ>wjnEy zJjTWn+tRej$t%`w`2xi49j?}7i7w9fKtM2Cu+b#)5p&J`-i`AL`cF_IWn1gv>=JyA z37 z+$XgN2;z>aA<8d1U_&PH zusoUtEkso>zpuWb)fxWWJwqdGi;I^J>-f>B{@6-34rTqzqd6{qcOFC%DV0gor%nj) zymPWS@=;U$a*A>?t*&bZnevh|T6!FC)I(bE@l8)$CF_t>bxYu0CpOnzWG!HDi(QF+ zy%EoB$Pwb}_U*o`PVplKgTcbKDHd*c^##P34!9cHPYa#cdXHWV-<&!KjA-R@eehpZ zv!f7oe{b!tp((l4+@U1W4${?D6XhPhi8Ti;9kbpFQ9i}+d&!#Fv`MZRb>wR6;AB~c zG#k!6knlwEYsdl`#vfJ^(S)cwfS!GJfYpeNTA~7va?UbLL9hIHk9)|b` z;4GZ<-03IoGVoNTKIO?BK*wggmAXCI%)XksyVz5XhVRMU-(GO0gbmyf!r>o(NJ|}e z;VDn%%HjQZ9}Y4&NNq&`SXx0q*`j`i9)7LYuOCN}^1YbP&egowotA&f^OFnip+9X` z3)%5#h#nyXh@g3pi;Q2hW#B>xz=L@9Y!mT|{@b^)>5dgY0F>9k5RV)5pk4FjHzrn? z7!#ZRPPHUwZmJ(|t~uuyyUHSVMtz=Sl5=q$f=MYf%}sRu_C@sZ)EGXdyScPLD2n;F zKaGmnE>&(&4fIjJ^^NxiyYAJBacdK)e=Gyc{o+@J~}rC6$rJY-}qxN7HdBPQa^}yUd&;n^O@ts zrC7yO^ZV{|BnkuEZ0!qeO_q9~{*|glt&pv5WfI@>Z!x*FbVk+Ng0{MTyDFTUCg0-{ zVuCu?o5BFT`NK7GP;6>?$gq{;LLKO(^)!RX%artD%>0r0Q86NWeXwdYCEe=wl7Q^2 zt<*q;DCB@1`Q(CRA)9< z248l`@lg&$lv9jie)Np1a8DqpnGaM!wa!!n`yv2|q`&FGr?yJVyR&O%ZTonizqdug zUg3a-BrZ6RrHoI*X|H~np`WTUo7(&lHjdNN|8*g4IV#~$ z$-HOJo;e(^Nm$K97y-Vf@Wm#iSf@5>Xb3ZrMN4HDcB)4zaz57ijDX*L^W#)3s+A!n_A_m+f(3%at7lJ;-XS4Nh^vI5KMoJ^3;X($2VW}e zRFW`3TH`Iji(T}vw(KWKc~pvmpGgGKM5=gRQ}vrH97T&$P5BuHoqC8wM{{3go;JsQ ziE}<{hCZvA+Jp>ca;A7Oy+`QN`38Aqjwd;ic*o4etF$(0zjv%yD*^K{l@mV2ww?NIBF*vs2$|hP`2`^YngY zS1OGUbsVK{t~Ik84)~d9oHp`^AyDWik@9m=DX2=g`)K+|d!+MqIaGE?q1>?0`(QT7 z$3)=lsr9Mc&>m8ZQb$t^CULDlW^g2y+CzV94$Amlu=F`f>@2(a)`QIT&7+!t;#5NU z+HY{G?b#qqo5ch871DrB_R@=&^3f#ruiP#nL_#Q*t&ufIZ5Tl@o2q zyPA+8*Y#WcE9rHad=(Y8s+1_0p*dIOdJ*t#THD45!?rI5D|-U*$!r9%HLRPxpC~zB zjL7WVjam!(U`)0U4MHGZr+WDpO}tEe^F2?eL(D14%MSJ9~F(Z;0I892cBG&0Dl_>-vC}q9|YSnmtdvBLbs??~wZh zEZf!stzA8LQnCEFmr=RbtdBc)z=W=FE+X$CqA_+6U z-vmM}eAfk1_1$MO`1JtmrORxv#Ws$6WC1YnhP9jgqQ@ClnTSjW6M9Kh{5NxRrV4Ht z@2=>+qNCvJ~IEt}2%0R1JQDf<(xgTDB6CBa4QH_T8jkl}x~i|EO->hW@!PU!&^*P!CL{ z)~>l}6;+pw&>5W8$z1l4-tNrI$;qbaxFcL4Wc^(8WIZ1X&;SCvKaj8)h&`p=kK!UqG^E}PW62u-c&~> znqMhvGK$zWu4-nBrG>1-WS;abB>G;xyK_|VUhc^!_vCPHMJX#n^E7!PhFR7hyFz|VJyS^x-}s<%rQVu% ziYCOj^7T&|jU7vCNEp)c1yM_)0Z1Vq!fAfgd*~4B!GxvbO3#*L4kx18z@m>s2DpKb z!ai@DavI|N@Sl8!F7CQ(r*xq2+NnPAn>t$0bv3Q$VD{Q{9$Jh@`v;YlwpR>*tc z$c*1K23l*4mrjx<4-up*F#xnh;}gotG9h|g<-5C8uVd|`S;x5pG6B!eff#Uu{p7O# zfq_SB%@2r$8fBurQRK#}Ek08vI!T`p0Sfr=k0xHxJ8z^&5)p*9ub=KyM7-#)R&-)w z^rNT+UWBUUD+7$}plPe`@<|#9<(jZxleLu5Vk@Ac=>fp-|24(wb{BJZ*cjb%beZq8 zHIS@eW@K#aaTYo2z)v=mA(Ry@dMl=0-1ng$GWwkvc%98=xnV2p1jJ+iKvw4J8~M|8 zaKlM3MbHI3%ZC@!uK*or^vIL4P)%_wp9Q^u=`yM89x`W&eGZ&bs}J>fGgjyCV9jX- zXjylu3h!RE*r-aYdH=!Es&z%vRBUtIEF;)lGjm*x2aUihG!nqL3gYd|I}JKY`gp({ z_=p{yx)%9Qtg0`gO_SFDO92BXN+MEQdJA!!NN5~GT=VI(^1k7=HDjs*3nYXG>2hTC zcEcCg;@j&kxkozt%WFX7XA|4Sn%+WJskJ?e^X9Hhpj-^0PjQSi`3?& zQw|yU|ku%jY2+ciVhOnM*ck&CHLS=ro|?SqibQIqvabwa#pYqu8)zBtlTE3z=LKg4HRhmlK ztOnUV=dDrPeS_zt{`)j)r!+3qB>J!Jhia}Z0{zL(QC|{WexCZEqIJh=HIAW zkYlzGz^IvC{DEO@3SN-Hh+F7s@Oq+-U4bqMNx6R$!Rw#1eA{a4Me^G@x8thhsxUS8 zE}=mfj#F2&AjStT_)@!3HYcNo^*covhWed3dikX;DN0|4(=~ z0cjF{9!r@+?WUk(mt4r+04=xVokN&z-NC@A`j2Sg8lYajGiJ}zI#)BgZGxnsoM_an`Y^9}+T zD;P4H1Zz$K?Ji~s#FXJ~0e|9YF{Hq%{PQxYRexdHUDeN`Ag;A|8{&RCgAZR5H52{k zh8h2zrTqK!`u6_?@ql>?_;WV?8IasWnG8z&>qq{4*ubZIvwU#yL`q6n=I`JC&s(xM zM;ES-si{5eK29Y6hMoU?{PS0xN`_*pi0Wgpz!6`iiNkAi{`zQ)jvsJhj8ItgG|pA9N7qyHR*K2`8EP8Mwi!Q50poOUxqNkk>(U zmVbP=CyF83+9B%wZE;Ysni>wlu@wW79;)YNX6z~wEaxTec;!9AT& z{FXnRy{11TV`z!_1a21j<8JB&h)bsSdHi?4#m18Rf@b#|{(O5{^71$oP=QiX!BCYDCa@lqOMFRBu#BtS?HWV23!;o*W#fkl z)_O;lhYU2y1;^HlCG#1JJ|kWjr$6?n%T{G!xR%IS=Odq`Z-z5Pgn!j3s7JO$j(qjR zj#cKMA@XrFqxza4a}t}uRTL{92NQ08?}?|E8rQs>36W2o(`4 z30T$;QEiZvJY6PG6DA{A%1Fs80o+Ytw5}l;`it{RjoZOmFfQD1xb%Q_8_5w2Et>j} zfPk=NdD1?M>B|*D?#yspHassiW2rPa(#c3^le1SY_WCk3x8JYqrp|*lbda9g>V~3j zk5gN8HL4vR<;Qo6i)W9hgPjaatMHe;>eVxkTJRU#8`=*j*tyPPq-c#|2JH(+H|IrE zD|;$9i?Gq_tEGeexmd(*O2`3wYo2YO2`^kQ%DyHS@5X_7L6q-{%6I=?510JB=AVWQ z1J-t{yai%K23w)}jd9^zw+@U^`dNjZ5%w)nE2e{FCE zMv0s!-sd-bzg5IlRNbQloJxWmeN zGf+>TxGpcM7WdOW?PT})1`TVSKeoP|N}?){%>&U)#I(0yIi;H00w}qXD^$hU>#0YL zt~k<`{7p>VzSqP3i3W?14I&H6lJ)u>SXgElbpWr*3eRiS-o#!T`5;0rY?45AaWkl!--t*HlH!WoWi~97$QO$m&Bl$rg$_~RrDc8+^|L<&h&>02 zwCWBU2H8s2WhZodh4@ar@{(#vA2#%9NTkbqDl^%;+BTPRQBr0E4=E5uza{R`boCPt zeJtuoVwMj-ZxuGA?UoCdDGixG1Wdfe(jLxS7>V}68YrZpdvKG36Lp>v^~l#}R$L=w zs}-(@UR)!IgU@MRW{^2Qo1wz^;7O@%TQA2b(TSlN8;uAEh7mjTx?wpB z$%f|n*!Mnp4)wYb;Yjbk4zHEDLmq(Y9JYiH#A6cXB30M7w^x{K| zl?9(8T5kwD?S?V`DWyZ8*TFJRDc3qCl2|dZ8MkVA+D{iYNBx*9 zOR>3c-DCjS`D^8@d%ciMk3mO?+2bGGUSegCX8C7w*cC>c86I(QN)!G5D}DUX4%*0q zUvJ+0)p#RDXN5fGEAEF1^s9Hp1N+JSNy7S#b~L~MeDHaR!QNj%SFsD#q$au8oR%59 zaTt+6|7hL$RN*p+eMfMEC{p%be2O%C?=V=YrJB{RUvG&C*>>TM7Oy1F~?gGU{0dE%aH!q<_ z_9Bj*yLw${U)m=eJ8xXvZo8q}m*q5grRABu)AfEsim>Aw@Z3j3LZ#PYH8CH7%0p3P z4qFZ)Yzh-^|4Jk7n-;mXAYTE);T98{a`wU0Ne;5K79bw$dF-`p+Q9K?v{dKDtxavN&&Xc(e3`*Sr*9{`&q| z&G%qOdefL@#+WwOE2B~}nUR`zII@3M4TOgK{zL_(_Puz6@PzBvbJ%V|M@oEx+enN%`s&jMipW1J zp|cTCybq86JWUpsp=8yuEC1le=qjjX8LF|2vh{|HIFy#GN}|x}9g}IlkX^|oEmqEH zQndjM4J+wl3KGVvA1Au`+fsUS1!o^UPBB>PJ;Khrv%XWCXaj#HteZh92;nd#?UPDY*RO**A>%FzzOfEW+ira-Q zXO%zEYGn>sJ$jSV$eLsa73?n6`vLeAz)_!VDdr=;dPsil{mY9*pDUUGyeD;mYV9S& z0;HHMae}wS8l8IP6L^3<1f?@C)~;zw`-ZQn)PX4jUj2OX!RvT}(h-*K*7ItM4?Z|0 zierFKUIGoT0YmoctXCoLH&oIqjE}UpRxA|{O49uPurl}>F=~bS-^jBx*(%xu zYw=sZU~hsWmAQD4FWRGN9+pZ^ZeMU@Q31=zbA#g2OtWBq6JL-lt-B@t@eh&Zs302y zSu>FVjZ)dkhJF^!B&pNTkLBw5NtPsodB7qTZskvBZ0mp~cJT zi5DfbBloeUUbZI37;d8_D|^rtTF1%`6RkY07I|+Hw~0+bZb8 za^zx7n42Et_q=|Rp$87H!?Jd4V2gRq>Xc?V{u3V{0`yw&RCrDgctiDURzCr?O`S$X zt{i@?J@>tKe8kC}hxnw?4zH!9p7@58cbmY;Hkv7SQ^2UVx0J9DEgaSka!ljn zwip9VOe#MzsJI@_w-=)THzAnCg$7~}a+vg?f~E>iKt8vZG)zok%Z+Yl?RSWCjc)89 zXXuEz?LNty1SZ9Lr!68OGoHyj#bzsEP)Vl!@F08Z8~3^deC9;%>wr%tW@e9`ynmi5 z7e{|~4{m|n%z!(ss=2oJ#v{muZhO3c&F(~LN3F6k#E zdDhSjI32ooj2?|$-awt;)_2%qN2*RiuO;}S%?u+jg|9?kgvUu5SJ*2znm7ebXbtV2n<+}qpBPXN6`er<5hUbg{t$qW1MgDLXATg%`7*nURB34= z_R!2k%n^X=x}v!BkvEhOz;=$f-t!O1wQCVf@pNH5(}YVR&`ZSPNMhv5;=vqsJAvA@LiEk&>9 zj(!%acyBFz<%{TX_no02xiXb<UGc&3Acw^DX>je|+~VECs9V(!%0M%O&m|HIo`M#Z%??V?S9 z5Ind;aCi44xLbe_0t9z=x8N2axJz(%Z`>tFaM$4Ob{3KM-TT~o&iH=Zan~5ozPi_( zHEUKqPszsZNeVIog5Laesq@8*!4o_P0V(*3jSbx^fUrn(gm3|<5Rxr=PBW3GNzRBv zQYQWi7Pis3eq0LS=Tby(@7P_lYebU+3YthH@JdRAqLq}}(9jV#o@6SVK zq|*fMPpVB>5`qfky%(Dsluk2nfeLy>F9o;Tt$T{nttSDS0kGp?sJ^a;0+1xgXfjhn zvr!ezf92xSRY_6acm~laIqp|RrCYOUKT^wA95C~NSNRDHQs=dj7wW8C>vV^u-+fHokDE7jfOuY;TRNLES33tk zwWr&<@36*!-MU|2=&nn)(!3Q^$LHBsuVwg=5Uhh>!Q?duzch#K zX4Tjh{ex#3SpLm!MM`|e zeh`j}pA+k4*>Z>l*2Dm#wTk7=P{UDP7=fTe&g3=iC%pshgR+Z30*5!F9fj8y`-ayS zHUwNO?=MD`Ca!+MSqF<5H659a06_dq?IsrC?&To6smBT|0Ev&K@M=2UX;n@o_j3iY zXF>%9T)6V0N8%M|cZixTwWMr=*AKu<`X^*O_AcaycXj1jTg`;*CS=8PRT*Y8HbD7A zjq-(|B+hc@Qwk6g{`5#PH>>kD;t0$;r0?r*(Bhs%GPO2X#=0RLPy8mB4Pv>e{UbK-^S} z$C)a3=aE<%;~?Ss{gO;Jhp#dB0TUUAyfc6|yUf&nTNBectc1f(Vo> zYE947OP0I@z(&rGM&j5K5?iV3a(v10hi(8RVLW+@V=L5+`9FIi7G7~!aD?zGIFmO*o#pZq1OPLR{{a+b>+nBTG-im&*S+Z6VHh!3>v zd;RwLKBFv7P+fF3xbpW zTex~TJ^*5LmYm!!K4gr;I~v~FKN`D?o7-=PNB#)mv>{63HMikM2|IE$$^Ms1I&QaZ z;@12aag_G~7W|096!Ttc5Fff5iHk4pj(r>Wep_7YT&krtS*$MRaw7p~+tkvB4b&eEj)y@58<~a? zU^F6dlB(vu^!|qk5FqPlJ^)_AwjqjA=qQ<-XTbH7d=O^3)qYGia(s)jnRKx z&fcQeL#8BFp>0JfK~)9!YsmZ;G|A1~J9GK%7WJkK3$jK1?B=JwmoIedLGF^>0>p4& zazIjwj9r^OUjZ2nu*%EwK1|jA_+2+^e!?fjUvb|%z`4NL)IjzTHg_s&P?*o~c(tqP z#6&fdM!3>*Ni&h%lj1PBPmhdut~(fmwrr*%|IrE3Cj@gy^|5y}3*c=OEA*(t@wt;7 z=gl~xUxJe~8Xa`l67$q+H8tDw1~zmKENz`OVum}-7o}yx!>}}+BB_-;bQYVIob!b! zaXHL7+x_5=j*l~rmE=x)?wp||(Hs9Sd>Z%$#pNq18{EwGT=L!`O57#S2I>INOatcn zdbOspRS)X+o9C4w{h$VM$471iz$*di5e|_Us_(?iap2g_{YvTtH0niCG)P?Hz5A#n zwl-Fi>a$-Js(MqhhC)(|Pjw7IEUa>@lm;|~af~N6??_T3zq}kVOmpuKMA2GxXas2e z5#CzIL(R?Ml%df0N9(<5=8OF~$5G5zu*Z#R)h5ukn?U^8_tuBORQ~A0mV1AId&;y} z=;5-zK^Kd_Z!@Xh{DF595%EMlRc_sd|1i-%-IRIa7{$Wza~w;-qGOFj20MZS>fn|jB%v$!+$PC$FB+mwd!$q`wViOtl9Z7yX8 zc~I*136!UWo>lEyl%5F{*Maw$IOO(wGN!lL2wIW=$F7L*6jfWHI`6(a;<~>S_Y^fB zw$PaA6Ei48z#|zD!9b4+!$t1MB#Cm=jWQTjfNt_0k9{-NV7&+wMimx5pgJgQUO)I* zfc4|#y!oth8Nlo5utfto{-m46)9BpF@hOgjFJ(^IOgJHNFh~=Zvu3$;ru0P>68@ z8~^x#14n-5hdFKh;CpB=&I5(?o6#^=`J0R_+qs7BAMBwkKk}9`5zh-_yS~<^ z)sc_K*MCC82{$##`Q%3GboJwb&&7@{%b?p<0jv9kN!7MUO0uzZ;EZs$$jEH`3)eb$ z0wM|BNlyy2G!ICfKUiAiZYI6#pC7%R6BiTPdvL9#2z+IbM+GGa*ae+SODS&pQuV%( zSM1=b%z|3q`S9%XC~Qx&b$gTr6_trOHwaOtO5A=^U%_P5eP4*+Q} zEnV-YupVN&*bgRWUO>v_MDeVmJ|7TX22%tTHTh5k^N$7$D@RWss;30g8SJToqm=7` zK)wPYyFSofU!$6!=d(%`f^P4i^xM=dnL^5ZJ--QlQ?;W3xgzc;u&eGih27pA*Nm;$ z$xulfDALF(6W9z{m60W?I+(v~Ttv z_$@OZu`~b10_5t3B$hd`bxcv9ylX@Q_J>#6Jw`R^JL5U_G8I@6>(TVN)VIQ%sm9}9 zJDH_U5?THDI+YmtkUYaT!KO$;^^@cCbwHj_t^4W#@2T0F*-vt{w6{zREu z+gc0>wf4@oEmVdv$32Sk@}1AnP%rV7Q&tm9uZ@3jD4KuasV2e$xse7!y`bvJGEd9C z3Wt@xvlT>={v1Os_I=yEp4J)UkVMd4aB8)`pjSNGc{~k!XIH~&hb2MlWkdO-O~kMVjZLmH&WDZ892vvH*!5|Er!N%-FqW|0HT zG3mogvoCAx+0aUfxgn*Y;sRSA&qPs`_CC=`+7%}XKW}%)N;%WOajyqpPjOK5QYwTo;PBn71}6)&zC?5K+kBF~SqSH(LbT5WD;>D4=2)ftH&UNN4P*3)RPJM92 z3Z}me4JwQ(c|Y;Kgx%7Ps>k1&RLw(rE@d7rIEE^KzYY#b3O2b0A!;)0)Il7WP8%_6 zt$lJwG1cb0*wVvqY0!*63$TGbviNB?%&`IX$da0!ZeqIZ)c)tKTMeXWPblWoG150X zlBgd*{UGk%f1{O=Y^gxQuZXr5eqE&cw-9Hq%s*0WYoU1a%LR|;MtAs5-?96+F)DWp zPy6m0zcVAP^jON?e+p%IIxI-yG#$m8DCR$eeqCP!+3k(Ndj+kDJ(CBB1r>VrU#|FYU)tus0S$quW*Y_>gqlhe{j&&Bij~=yMrJhHJ!wbNSSWj{1 zCc{_$I~sp}THp)7EYEo&jt?J*wwmQ61Z3l@-!D#57)GUA)1%qMy+y9=Orqy!(=-JH=WcfKDDy%*&??V<;4mU{53X^%{RFUlQtJc zjjeO73W$Y=Yldp22cgSYnV$mJ*wwe8tB;9aiK!}J55rv2TyUbdml`1GT|&t-+VR{3 z(#iGd+TK~*GB7Ihw>t9U%I7Mmx4bvVa9(nB=8&ZR6D-5YTT8n&Z@L7MrWo1~X6Tvf z!SaRpDDVK{s+4o#Cix}t^~z677vN+Z?lY~kd`(vXH9DGUUxfByksBw=3+szH++jDzy&b}!yx}p7?hzQ~Nh*o9Wr`8^+slBzekO-_) zrlDp7twW8<2&@(`g22_4({`eJ7=>Pc+z9j{Dsq(8)ecms3nnov3@HzAWr&Q%2K6YAS%jdJ;HVQv4m_m zb=1Nf=}!h`Jh5Yr3UrxBheM%;s_hBI$^w>e623Q5p+yeg7^A&^D<$xF&Pm}R5U8ro zmTvRe;%8q9JsF+P_cDMn7LB;r-8IeDOE`Id_w0#oh_8kc5h4Y0@y?CFqR|OaksD0v zcl@)AUe)i(yot4e3A#w6j|BzC(1rA^ZjE|C1jjUmka!alf>D-<Fsa|>ou!H*ghTT!!v?VvvOE>Eh9gG z8GVXT-Wa+Tc`pnL;Qg-wy#Ei)A^DX~D$?s!DOmt|4SrIXq_U}e#k+(;EKHR8Ag2q? z0`+-dW?vsTP_}Bx4tvd;OBwV?Q#2@IWYQ{o$J=fJVH*77BY{mAJUkBDT?P8^Cpz_* zHC0Y3U2a_k@uk==fA~c7c#7^$hQ;#|^z!vqd5kbyX^x4U)5Yw1K~{{+JxtVCa)+#G z<;pD3pz~__^^gJz>Wn7}|D{|0Fym=TY&2UO^4!Iz*$1iOVCL8Z88Bt0q7bg&kFSQe zpJzT7vD(a7A~&>I1 zHE1yK#qNKR*?-eC6YHeYW-|ay1DAxy+0xQ7n8fP4HEX9g*qaeYH&v%bOQY%zs+?)h zxOa``G3JvE@I@BH>luT#&42feM?~oDwfhC*yL=TiQ(K)Tad+KZTDP>6{nDTn{%-w*Xy#*fxNaWq>6Tx;t8f z(lL;RdF8N>)AYF$5zqFKkpKV6X4=;3RsA@756dDYu)RdP5Wev^#J`B-lgO2C>3q%do;6MCtP)NXFXX_52I_XlGP;M}#n zW?{AM8Tu2LeE4`uFjD(~r&1WhtQBFfH$BR^DFl}N@MaBnC_G^huzBFgx!2wTCYrEn z(ObJNnkTyrttVpn-cfTvYWT^70YWY`l#<^3;AK85^!&b=dPgfa9d4$5PmJf0A;O%E zPu8dQ6;7e6k5(U+f~ElBO`Oz9-~3PkdE<+cnXfK#d~dd^pzm6^?|t5KM1@mWljI>_ z441fRho+0(Iy#>wuuLNnylNC@Q}{L#LSKiV0GM^4H<1{DaDLf!_0l&*Fa(WAhC;6i zc2pbn-~f=PO}iGr?;U^?3Wvt06pFH~X#n(pZdI?LnScPfK3%~tfa5=pN3rl@-l6&j6rm;M1|d){*r) z147Ml(`hBew1pVqxeB9}FreD6g;{94rFnmue96}L8XuH0yX>R3`E^{0+$T$-{wxaE@2i% z^zLsRoNR0_dA`!^4&6L2Ny&eA-QueAd10FBUV{U4q0(z@MSg*Z2)&#EQ#e{F z#)*oCmaotZ$6lb?C*}Ww4psbZyM^Y#87kz>B`He34wUSMkC&@1AP)aNS7_GFr?tsVk}&F zZWf=sfW#w{CC~50v3^(6)%d|$!-u=(?<|($EW&7CqQ88zuL{hNfMiU_l~brOt1ql+1lYxv`wP4$g0Vk_HlEdx;+s_3oJyW?v_a_As@79J zTt@&htI0$hh|S^-9R_J{ZO*dW`m7fP=v*vP$DZ83MIL^4vx@*^+9Yw9ckcTuej_Qx z9jP3t`7~Md$b5LY^pt)5B(cZO5l{aTjAiz_nD|J%*49a-vn9F4A<;>#2L$R)k110x zYB94MA0z5>b(8^d$Z}W^bq>M?qL~1P=_qlwhJ1Kq+z}@7)P)z?UwFFs2QMQg&dDp++tw|~(PtJ=M~B{l%I&(_)4L5CgG$Tw>{1aGE{a92M2I=9hZ z%Mvwb-umf?xX$PWKsU&N?*z1#R7|Zex(}Na;wWr1-U_vgOG@^;ra5sjFI@xmGscUq zS%t;KbUC&Jl07dTRTZ0rVHe$N1uqwX;SGrcy#g9vfdH;3d{mEwdizS`yuHIVnt9kl z!A7MoihQPq{*pL6^vu@t(4Hsd>iSyao~NTd4qo-ZK@bsXNgH_H z-Q8JfcdC%V1!!&0pUUFpA{SH88aFyZVmwU=^iVQ$(Bh#-@BE|80Pls~cr5mjw)i7> z&7skFZspw6gq8oE42aru`<`p69YCx2^M6D|w#@Cf=rt}hUIL8IXl2h*P{h&@ZK*?! zL$Mf0TvoPSyH@Fc#zvmBW9m-=oPb|RPc z7AhCHl=Sqc=;8AxYQ-i*QcA*Ls9!MuQJ6Jqq$pFN$%%~Dv2(?tks24bn?I(YOnX20 zS#84qG22hD^2Wb+Naz1Ej*<|`^*|DXAugmIJhT#{%7a$x%*xHpq3N_Dk|TW`>gbZf=xBfs~m&2eX^bnao}VY$)h+Y2(Xm)O|7q!AO^ z4|hwCJ7Pb9K-s!Jz)=SQ2@W=kCb+L#rhdy9NiAybJ{it!+;XoZdAbuus7UbI1#Ihq zD&}2(RAaYS@BZ+YLk)+1*@|$BZif@ExYZ{-pq_s)jny}RC_nDfV>@%nXlh%gKu|sj z2!VLAY;#5)!ai$2Yn7X|*P&GHo;|PLW$WCwtFS-iAvf%mp(Z9y9TB+wibC4bx$l5R zM3m-eak$tFB(x>rFh8zMm+S7$)gETjzeZhktTg!nw>g*s^EZ2L$^90Q6pSyj&=6t0 zb4DFb$nEpPWT=qyjELwd+Rk9)n&5ZxTs+GLB|rsBA8L96qjG<3H*E!!d@e692XZns z+e)@rj_EX;0$W=@3EY}N=?}zqZP>Bc?|fBfc+(+YWu$O9qyHAz6LKX5;~iP(z9Y{K z|5yx`5nivsw6{DrrAaGOl2dxZT6_1~uwRCAXX68uW%=ht=qndqGL#7G8J4n?`H1~a zO*7+LD_+|Y%_p*P-QV`#Fsh)7lcl!{J&=U=EDi(Z`19^{{kzPG1yw%J7x4MH6%fV$ zX{klH*(==^neZ0Kl)Jj|`-D)oYI@eQoHd7Tb8Ew0X}yu$o$8Dq$3Hi>@&`f0`Z@eZ zn+H0D0kS{5hqj3Hw;{^9|LL*AznfG7!cfcm-$Ik~+M*uED|Wl^bpnMu0kDyT9~vdH ztU(r633xSGY48G@E&(=P$t*ro)-o>l3!`}1giv~!Px^qnzrpXUt-sfG9totottV^O z^nBuHOpyPEywrWae43^iprAr`VN^7Pdh8qZHy`ddc)Ok_d$_g+bnZftGI*9DN#wl{ zlbexYxKcZN~%mBPGltq=9?ka{^_JsJ6lT(u4nMS1Ani^{`ePnLo+;9I7(2 zPvwBYY}ScAwvPyUU!!M{okM}Db^yxB)7{GMy67dVvBWe;!tm6F22Bn^`49>!v!3K9 zsya>BMoiWIYlM^M&cb#0ZSq`13-=tp)&#|fDNau7aqYGTD!QoN%h>i=Cma&1o zFf1=9cXPX*nIZ8k-LYAEoRHIUV3rxL@oajq_%RDH`{+27W#b}ne6JpDWxFdbr}T$H_A%lSmuX1r%zIR>qoldkdJ}fCSrvFy& z{xg-`_l|F*%~!&%ZkL_{$+%zjFk;KQXR~~KEcO8MVK9x*B_`(~37gHDKJ`O`ydy}R;CMD`{Iyi~rwKAjSf~+AvK|o-ZQ%FSnOcmN8U~payNvC+aSkjd zwPv2EoHb-31_9OltBj`54^NKo%=cXBJ}SL_*x_zE?R@=k_H*f4FZ$M~8CBPwAGB!m@~+d+p`pg$-~QnDAI{*8gxpPuoW14UnOE-djO#JJY01 z-&H~qOP?@sPnx=tAGZicX@cGuJYA(dM17VlY%#J}HNzPZfyJ;FD`)}tN;pJ~50zEG|>^^!;oFV9}p)n=M_=lTT zc11d_kPjvnPErtnq2a&*SfEGO+E@ve}(pV(dSzEqmnQpfFF!C2j6=YH0IUY$7e z@9gZxC1eEl6ee<`2}spToYz|x{4-S#R?4AluFrw~Qrf`m1B*M<+t;7P>bpF!D8*|z zKLIPAQot*VVCur;OfiJ5?(^xLHIzdNi9J6zIg^gMC{XzgzxjZl_-N9<^|NV&o6Lzt*C@w zy?piQ>=5ha3nIN)8Onzl^BuYOX#3~f!kf7t0^@`V;6 zeg>f>{E!L>LFD}+w9rZWRTsPvwI;UMar^toV5}dWBf(hy@=Z)%SAUHj#^^3Qc-4;> zoe{zQpgSae<@l|zFcPakwc2o=ovUc^5+tB+#BclDyGoj?`Yn1`3`b!jR(?(dND=T} z9U7WaT|y=qoT@z(&s)i;CNd;Qc*9KBM&-+$8SY@i)Tg-G>?79NtO=D`oeIffHaSru zw`{HVTc@$)7%JN5CQ&dQAoE#;Sd%(i%sU*35>KvJT0u0lcHfhp!B%C4+T(25R+ z!ghsYM}?m0=4magd6mZ_FB|wOBfZ=d={e-rxP4M=Z+4c`{Z>I<30Lb8)3Yd+*_iS8O|r68EL` zpq;?%{U!fn66@O6ZvNC{j{TWR!!7GAavO$$;o+5 zyT#BNLO8w*`QD>fWVxl&%AG|C(Z&hmv9a9d+M6r7^p$fEOusa5UP%cvAnQ*`1$c{` z8Ax4;p|OBcMu`C7`V)bWhnty}77u5F(3^({4yI$SK6Ng?CO;)73q5rU4{Ov8OZ--7 zqVUymX3p}BYG3_+8u(sX!E^hNWNL4I<@c;;NJ|bX<(`8!t2g=z0^0j1`^5>c?+)^L zRz0#l`?S5_w?`6L*TD|#^@GC?4cS^xBxQ9V({%C@e~>|G1LwbtY& z*0E8eqoy0oX)VG0kpS=EAA#L>v_k59HX1;}UrqWRhy}}#e-Yi6&7V7LvwiWrYS04y zHLwYXx?hxA={$cuuvY_6O40gKo2gc5HJkib=DB1iIiaij-$d{wyUJ!7PdqO#Hxdu% zuj^P6c>T{QomWmnQ#4*$2+be?-8@%^WGJYS`fE$|e_XIuszkDOC8@sszDBuuB0YZ! z@4W+L%tf*4FpQA>JYE~3Wp+vYYZw8yHt55d{6$)o!CD&A+-^%4s+Xj-3(&%1bJO+i zv_wsUkzPx34b2dc76j5AK>54l4$K@mXXH3rxv&QpEl0Ej8ILygQhu#9$OAR{O5@o( zzEr&XR|f?<$hIEUdW5`=cdr#qXyorb7G!D>NN8sLXUq&n=edin3}CmajfM^%xdOKy z6z~0yBP_Kpl9ooZXBY#+!YGNUVM}{DCN3$X#%myf6H7=)Y@E)Ft5v!2__F#{@ZneQ?L~DtR+4Nc3 zpvIdPFiwi=#Y%*y8T^tQ@yIs!%_|ZV6?W3J&k)7Omh>Iq=8G|g$;}`#1fwBuYCOc$ zlz-S>_+fo|B!&7&1iV^r+ro>1M8?!E5iG6lfsyde-E)?+6|mKto{(#Y86e)~hPANe zKB`vBGl~3c?INhHLF&;a^n!i2LnvV6{yTp8(@rwS##>ok5go27nzn8b~_Ixo6k(_IqmX;e1 z>%%FbdhO49*hsXPl;{CuL9>uEHWPj5_88W8Z`SszoI5@cdLGG+?c*Te9U~N1%#|B= z;PN=`N;L38zDiDJKA4H5{NY%tNl&H05hXI`8TDyproBQgR*@%%v)SXCP&chUqSN&- zbBbM#=~xqngO970!LZHslKEzdwXd}3+i4x<{PAWk!l4fQ61&?lrvIAujIH~V$tJn` zlJwmWZRZ%3H(-(F#Wyj3lvw)`Kfo%1{T=tR{cXV}2H`iXC+p|W!Q8K#)#DGobmpf^ zeZ)x{NknQ5-@oQ#KN(qB^%-xpeI@Y!+}@*{x-8l0aM7w*_?!$T9Z z1p;C1Z6LFf;y5+qoaY4h3<08qMD^2fB8T5XHp0UjU4+o}UmWsoy1+S@Wfc{ndo46N zM9oxPAA3NND280@^L{T>Wjwe8&yn)Sd{9Hy_waj5?9Y7D_LGh z^-*E%UFiO}f4La93e&BC%dmi*DfMCK$9`VQapw)p#cSP=XkM}G@OMq%$kqcmKrJe( zj#CwZS`!mqe_z{b{TlW+2l&^OUne+i_TA2??vYS)$E6)V z=JIK*(|V;?<~b6s%TdcQXvGAVWGfpRo>=Q;X3_HL)NHniroCV2&v`aFL|uryc_ly2 z()DR~F}1?mfA--KirHW4Q4QG|+q#6j?irASIGS5@XVx}23lv0w^Q+sAZA7l2C+DDG zaI>(h3dHsRYv=susb*c4?|(QO^g3JT#*{FKkJ{#O`*+n~>k zn}gYJvM8#!AD@A6*?P0nKjCG_2d+=Beb}h#G6~Dfp6+WwrQGLrfJiqiHN7@JY;`BABijc*z3M@3o#g7J;#Q3fgQM>#cdJb*on2k0y9^8U?nl1yg9aam`S%#< zGuA{Ffc!1KkmrcF<@#$b!5SiD6qEt9F2DubzBPr1x_e)&kbBRHnm;l&Hs++;E==(V zzVw{z!?7y!=_UqTFfbLdPZtC%GPd9jbSyv5-O67g8KBj z*Q_gz7V!nr*a^HG7o8FwAP+aA2vtYZwRHs&Z%VDI!|azz*B?;iDjTgRotH{q)tU#a zm6+crD$ny6VcS0N#*m*17W$^8_Rbsz)0Sk5gsAZG+%XdpJYa(eBa()(>_d+R!}TO3 zCE?!KtGs8zUVlB_5iKnh*j9PG8z>_yye`c;rwdd#{v=zj9n8~A@|IZZ%cuMRV4U8> z-Cj}N39Ia0Q2r3`T-%8z{EY%-bFB3|q=@vGqUX?blsyhW~<1e>b2BW?QOv&8`y}rT^IXf8Dh)P?Moh%HH~S>k$$X z#nOZot8)b9(SLs>3JUkB$-D3J?Em(R0soVA1Q+4C_s$N^Q~UIPA2yo_z{LnY{K4Ky zNY0ZwR$ft3a*mDvIXidmdt~JydFa1qjl4M=lWKrmF5@NP`-@Q9hR_)Oy`E%Upnw=s zfRMyriV=a`g`N zt=xEU9(8}_d|%~6jC4zK(BjIJB`PpnwLK1eN0C+<*7o*OIAAZg!AOaDWU5b~_2>?iWR`Dinhqn3eHyM>WJJP#&?OFj$bt4MAKQvVeB-m11oE`t&!SJZ}ZGFy&_7o!VNlp?&3c*cm_r&=PZEG_^w`xTwbEo5V3zySnkeYn=_pf!k}^Tm_{v4dgL^TH%F-9EsmTACkQCy8$5Xd7e0zy2hL0Vqw~ z8sxnavI6rPoP-?Uvg;4zZMq`_SLjJdx~&?JD7r;lfQv9J7aGwfa1e9CqyAkmsh3zK zx=)x2ef7I4cjJN$6E$SIvI8%m>=5vn{q0;KLF7ig^#g#fr!Qa-Jx!RxI$p3eOn zvW$7PbX%fufOjbqXQ@}cE(7JL3*5jeiV};BRB{9J*1_^2_Fz?>J1hnzGrfIIWn~9lRIjM z$yJ)^S_8)LRH2 zZ?BBC#6bf(16uF&h%YOTBOE>ug20DNIg;q(=?$JefQWq`Q?oL4k8Ybk`*eq!*1Ts< zdiCi@m-6+yD*Cq(DTankfCA7S%~X;amT!`8%99Gl+5((PlowYRj63lqjOdIziN64m z@W;nJ8y;&e{Pp$q3J>>1g^>D zQ7%nZU}e$6UKW6DrwFpHi;i+<@JMnQ0W=vSnsYKS**ehEl^eK=I>$NCnm|C!OdXTG zZa0Vt;CG3Gm@K#)9FP2$MFc7wq|Z5 z)W*QmVT@uMMiU6?(#b1p6l7%4Uw866yoWv^;&M{*JfVFfn@-jRuC}zlK`8(~Q2^Zg z62{PPzy1Fd`s4O(n^n8iS=;%v-=6;y&Zj@pOUA)4eB=b|Qvqra9v?-7hBX0wg-EGM zr_K_0E5;}EWTMk&0eDCbqN!}bNT-U%_IzHbpQWt954jRjK0jM*UNhPAFd^j|y6%097jt@@}Dv2;s;+8U|Zi;9MLU`w5s{JdJw}xet3F~v$S(LL4{XdQ<4dt zY*{eVOeO^m8nQ0fVZ~Y%mxtlCvl?vMYp+kO)9f&in!y^4#|?!D1X$@+6B4b4ZWN7E zTLxOknoWeY8Een>Xct44cFB#7a+UcHXzm0U1xk-$y~s-zocNXfJ>O1g*n>(NQPW0D zfk`EIg|}`*#ul*9*T;G1v$VzPW;V6>c3!!=^x7;8i<`TCT_(*za*O8wH1;x2V;_|i zZ99v>_8VM%f@Jn-7sIC3R7BiSB44mTuE6NNG|UXU5qvA&Q+~eU$Do$;8S(vS6eK``+b#)@7U>&xn!;o**won34V#P zm7img70|ZR*CBJ*r}9LJ4^=d3`Va#Gnn+uhS+22&s~^UiRe9gDM?Y=|9ejy+AFb7z zbN7s6!@cv1Lw6i|v~5xsiquCi-B`DV&kUhdUs+dV>hA^)Pj=GNF#$PAq&gx$IX{#8 z{|XO&uH}LNCW;DJKl+n{eUuYm*CoyVA`Ro{6mN6;8S4Tgr(VxT&4at$IKBBJxad^u zj(8%6qud}`>dVKxp)njfD%VI;*P0bHDOHz^1~m>BYG^{_8f2pNoYJT=2&n$)dTJK6 zY@34@d-?5&307A%mBIJAv1n6_At$h=5k{6n(&;vGdKLd{HZp84>p*P8>-J$sEk7+}&YZ3-m{8Lrj%E|YIdhn89Pvb?LPsM8vCzKFXm zTb{NC$^9&{GpvkYKy3XhOQ%KREcmZX`uK!;;%vpt%;-o7@w)u59LR`?Bb`q-PufpM z6xGx;Rthv4PjT+2nDq=e49<6y`L0^!X=uW=j%lf>6XG-pN3OSObq(8OMZ<8d+tu4N z>=dvFJ+2fIIZXS3vJ6*Sf=io~a$Dbb`wN{8>ajpaF8pqf3yz_9m#Djw#k}W4uL+om z3KBDRa16nKx%0uG_8Wk9ebBGf1w3!Qhvk%&_0ZTEa{%Ajk~mju(H9tSpcrlT^yJ|z zP#bEWY(mz}8Cv=FKwp4Hb4QqW=CdU95E{vM@qolLe~OBHW+WNLnlf+=)b*5M-0cC7 zz#KnYAhebi)VenRc5x@B=tFaqvT4ooI}iT}W=l=jHxX$79Mf?Lovd6<9&l?-0eZ}6c8`zY=fp-;ceD9-CR}|EeE?_O z+Ex%!YcXS9V7R3R*H?@fpSxBNx-QFJU^Ig2P7?7D2-Jt^?@Pdu3>plZD*=S*k20KJ zX6l%IVV2MY1U3`4uI?(i&*~3wn>2|ommZH`&u@=NBdpRhlXpvQ2~M#Fw@T&0xHC=@ z*-eM9z~J`Bhf8kNiyK5wBEi_u(B~^_-@e`c{N6$jzL!C6(RiRA0Uw?8(NDi(r!nfC z$^Qi`{s6lHS}AL~I!*>}@-rH1y31|!f04z4T5juXJX7S>-qRgZfAhmeFDE8Cs_bn$<$oe9H=DjHHjZzF}ghS4>GzpsJHJP zL-o8q$ZIKLWNQOFPO0>+D&(OSiGr9qVeYUc%~KW|0JJ}Le&a9gGzigud_Xk-LN);) zSZtrOP@h94DXqsgi>OmWSwsJ}ko-x_KcF99kFz30C3xd7ZYRH zZbjg4keiZ#vMx~vy{!fz+7~@aed?8{K&PeyF^fj{K!A0yy;Tu#@+5%(0~!tT`8-d& ze^^r4S^&Lt+6f`UR;m?rSYhd|oY}``mjYL0C7JwCJ`g!AkC>qHa%;Zp+9{`<@>?!> zh1**~lAiHbq`p88mFPHC4ZqdKLzVFscNh@E-gMB_71W~B8awFWdRV=Q&w)?3`!41a zlhcGi@gx;P;~wWc`%j09y~QECQn2LXgJ-%-QcaP%<9a}@r`dAr-1}-1ax{u}5!V}W zQr9N~-^8kDY3D}f>$QOLQfXy#1a%(;OF1)@-((R^0~Bk7pc5rP{vvp(z|?X8>x~j% z3R{1;KR=p~`hncLmP*i9XDF8X-sU-o5KaK6%lxvred`u5e!nP$;#^XA))&&cUA%NJ zX^6Qz4OiUEG^0O^AnXg+(7*v$>+L-*w>&;ejGlm1)Dn?cxkDu9|!UJSG z(q*k&rtzah(e(S)91%t2{=LjkP-%177XT_Xc{j#I20;Icq7LoAGX>6mDDfG; z_4%*=_)<1DG>{hH0MEx$qEOEV*h!CCpCrpiw1WbgmMc4oM!1^e5m?=z;NYA?nc9=S zv%aRY!50AXnY$G;MOb+V5Z=>G)*4*gYhQc_)YUE*;6N^y+at)=Nlw;-FIqnE#qE}W zalGWI0!g!owdhlNXc@x|j(tYzwOa||r705GM^-aK7*295+QCcwdQ}`d(cO zrwG7!4np{()3pTxhc&C;94X_em8>17=nmoJHQY_+<@xZXdk7c?;9&l(n;h`Od0pMIS*Y>is*erfDbQ3mTV9wtF4yL$yO0A*Sw1cX!Nq8WKoSAw&47HxbZ+qfeml-YR}pw5vt8cK7F6g-Kbu2~dwct7h(Tz1m9 zwp!Epp0}`7a9xn5TlH0$-#zm2!&Z)Uj4`683`*XF?^jn%y~U`*d3ZKP){Mad3Jq5p z6O^?PvdXZ`&qrMP8XFsLOjom9U#`*?%&KSS=2E(eZES1+$@12l{Y5X({!|p(8?Wl~uz4 z124_S&|hTUe|CJrVt`i;4|Z=hIKT3JG*!P7EIxTXJ0u1ac>gDEX2mNM0|i9=Zx_mW zwkAr}n=}Ny_xFMsU0QmS?@@Wv+C2|iAu!}TU(D$)3|HT3*S>O|2p1(l55;;PD(idsClT%a_mc43< zh6;bEerSz*S1jI5Rqv=13|+DAk@J4F&+rvN&W6sqn^1!Y=u;s6QOF3h9q&$HOW)Mi zD<{nFMWituy`LhtG<_Rmw`UE9PcCqnXUoG2`a)JpJ7^r z+Pue0^gC92_kI=6^WJzHQ5U)EbVl{PJflmojp6=bIp>v z$ILY8ID3XF#x*-j>-G+jFA53@o^M@$$?oi;zYRk=8%G@7%-;Q%V4hpY#A;bS7WZ@z z4Tx(Bj<0WGA4~kJS^$t<=v2l_wIvS^b)tvA;G^ktT(aGosa-B9Xonpdm#ejM03|~; zb)BO>nx1939~o4f1Yw)EAE!DoGnf{oXGO z&HK$1>djta|1jrySa}Q0IyMdm&n|1E{E)sb)hoY9;vH`&E;&LZg#YzZ5OuIow-_Qs<#34n|7NLU$8A;H*K56)yXB}%DYu=CZ%VizrTM) zb4U2|=g;r|bWQcY&_jd6!qexiWK(tZQZhebCNn_Ir;51 zq9r5*!yA5Pak1+*al>JtdO}63E)?`*pl}njS4@4@9g+6?=h6+dWL|sU(VCs`zc)cKeXv6q1QS( z?W>R$=z!a7=op!#xA%>`COG#iee^7l-LI4nn*b{bk|SSBHGBHgaQ{q%RsZPL<+VG% zXlYZ#E6!Ke3;$2v16VyQ4ep3MJrKmDV6r%?=z6d-)n)5Py?ChoYqGWU6G&$EVS&~Q>6So`24mLcjlKXH|Qp}6I7Isga1;k>l3Pr)EEYdSoa z=UrcQ)huU&&_X<}?#+9y*>6v#51kgatnq6e8;#W(j_=^<4r19iU6FO{Lf&uih(VWn z-t47|ioiRW&_1?3dWng*+;*FKZAqGHEk;?)9Mp{C9gK`r{_;8ts>K#hb{a`_Np~u~ z{|L~{Ybo}vMm?XOGd|4#k(|dE(Gf;=OsAN_pZB#Xb;1zB>n%c0AE#be=$5j7cIho} zi~*`*c?+S!g`AQSF6Hzz9m!pl;nnv2wyIAgBIK|KHEFX+_cT@oAHGxU{$lIrAnt zCDRNdbb5&Cv^F2V(t7@LP3$Gl-}Qb_$0_Lc6!=jx%FiV_Ie|X_%!4T^zvCv;7ja_V z?dOr3Oj@$gl1#?9LWj(B;w{ z9Nsr+OEoT38We~CuCcL~DZvbF%nQAOdSot3Z00y3?oNnqUPwfTM5JB3}4C^X8`3Gvd0E#E4X#}J?a9xfh>Rw(#-5Bd{2n6d?0%dn zwwg<2vs{C*-NOYRRQHtfIImiwW&ZOQ?h!Tgh`(DqgG|4LVT7jJ|K46u_;M;l21oL1KX$_~?uJ%4VoOMF7ta`-s zG<@UdpTE50b{CR-Ffak1)ikRf>>NH8Gw`%RlO3cHx%!3p5PTu=|L$Atof2UC-wC(@ zh)nFI6Sd{jvoK0xF!kKL0g}isnJbunk41K$+z?x#c4#ud6DuWu@%Ux;JxmPuS#`uE zJ_2%52S00);sDG?sa7tpjEDWz^w~| zk8Nm~U*OiRjVy4<#d6iZsKWn%-u?mOV5#B6fJl4}E(grGY2ULDw`6KjGBijgk>Kg( z->yU^fY$+^#wDjGDRMlb3y?K(rFImGzB|?JzL`^tB8ssXEYKoC>sMM@OpAQ{Klo)> zBpvQ&p60Qyr3ZS&co`7j@Q;ZMsA@c$*Sxjr)0leEcfG&qgXg?c(U{n9$asnD?=cke zhfp~DJj~NAlvv_>|>NWJo5nU@rwY%r=LE8^jMy@V!aQ;>^`^e z08k$=h>%#f!hPWAi_auUGhhAO=SBJ$T!>{Xanzxy#Y(Q9_QT+dr|x&8-VCgn(ci>? zmQHQu-zYHr&8_p_VBz0}!#^PB-^?N7S2!R{C45^^3^AbiBT=rx0QtBI0LQ1I>u1Hm z2>yzx#%Kx1wDs(B1)H(B!SFsU0T1rNRNhDrY#68Oq)$`Js5Y_}l<8v;PgY7VrkoQt zePtcRo#KbXj=UG)Bd`5g^IdF&$OG2YcE)}jhm28di3m0!{(F@JAzr>avc%xzlnE5b zOEQc7-PO;#Fu-RnVGbB+tuk@lZ3C$@#5%w+G zgJtIXQN#q9=l;KBfS8T1SGnwO1SIUgtQEg*jV4YzJs?;^0bLCU;14Un2oSCTTDA#r zW7hS;h=Bv0_~hRJb%$;;8UXLWkhurqC$Ng4>wdwc0E9#X>$xf^WR|SsQ`+9v1Q*>C z=Ym@p&4V{w<7eKE^q{LTI@7s&e@e2Z5}f_!grh1Q2|M&&=_lx`9~j|$2r&Y^_KewW z!@P#s0W=MK{2V7wUqoVS*$w#)tsWlF-2XAvIPs}|gle;ccGp{{Xy9{<_&OPuTXUeG zwjUl ztFvC^pA;Bb{lo4yYH5oSm0sx!nt&nlBU$k5)4AK;jC@ixi){r@SuwkC&J(iX0i9vG zsgA*qmrKYdq-&YgF2%tUbaU?G%hS5`u1G}x^|8_B&K?}PxP{ZlDUuBhmY}NBTSXQ=af3p?v|HY0FpvIGM+l-Qu!vGT6 zI5_5KNjzu^TsrjB_fW4ZEp63q>kQuenX8!cOUFbX<(dC)oRlFo5q)eXLvPLv4QvN^ zJdpnU&!!J78|P)cLE+MY#S!8?^W`ulg-Zv%LjK+FlnW94cFm#IDhb@@$QvS0i#-fd z{bt#l>rd7j>+63R%v1BF`zw88i|L4;ir;OvTT4%uU2PT5a9PWPcib1e^`D}z_4TbP zK4m(iXV$n1Q*ZaTLGn3B2|$gG4JRqjzi1eX(XW!BMp8_Gkr7m%5-94 z=}WXy+};QoDxafFlk{5SE^MJHd-4{aC4XXWy9;Yp(8{pRgwJ|B6%K*;Ls|}Tug``| z0V2|CHO&2wnjkCNdqll|I~}rQRV)U zfUOs8H-vVF(IU*tbyfCcH*+!O_3XOi!S72z^M8IldneZ3u=ism_17UUp5B?;0%+C* z+J3oZ#y=wm*#QoTC-%eI+1lX!-yr~NfqkJR`j-7dxAh)tPE*Z}0^kX{Q!==l`j_rT zE;-k5QsDKcZ*_bLwpO-865+oG(x(;~{b%k9Lf~UBryFXmxc^v70)%e3m(gPs2r?21 zhE{3`ptb(&I|I?eA~hWr5=6q0^_;fkVlKRso(|CeTGh)eEoT+yaKs3ersHr|0ss*w zlUt)mshi}P9ssbH@A7=%L*phISlQ##Jy=o?y72Zv4orB1e@inwxymc?A8OOyiBA|z zB>3aNmf&@fGp_$=&P|x#6Aanc`Z*G)hD(ZU#GPTjZJ!@Q+U?vH&y(o;o&d>NPz`_O z=LFbt0%j3x%~1gtpejSsy~H6`zZFvtNvE=yyA`kZAME$~5QBe=g8qGMO^4kNo1CNb zf>XbRobm%_>$zTQ(WVudKM@*0?(cYB0Nda-=#}^HhsB58*81d+lrxFft;%JIY_m?V zj5FUge$(o{rRaVy_(5gXzUS<$^EDHD%MrhK{cjx1S-wxC*GZiXdm0W?_9YBLPlQGQ zuVQTqTRrH&n^D@NNMQ@067}uRiWG5pvEw{bWi;PZ&p3MWAL1(LC^et-)KSmfItCmc zRg=BQv%2Jf9zka4iyq=nOe_d1mR~CRvRPIJkN=2!8hiSh|5tSn=GbV7EW|+=Z86lW z*{{y=ci-4A#3hFBXyW7-6kUB@U%r)rx_(iu9-_lFTH`*oH{R#qTz8N8oJ-qW(OAU% z2gIwq*;EKQ6$UH*(rRoe+mTgK z3RGf+cf6JR`ON~wY5`72P)`x#8@#zku$%Ic)@GY=Btu!_$<6SWO&;{1<&nPb_V7*l z>fVTdi}N!2IFz#> z6k9qmGc%*3XT;ctl9!gd!)fMyP7VK(u_(QE2{U&W3C8lf=71zP4JX`709sIjD_uXd^KAz6%*xT;l!e|ww4g+m}HpOcq&*jHG}Wz@cGU=W}$b#zyDGe z^=OCvrSkE0+tyl~+8IDDsr5Y6?b$b6nN{>WhBXl1j$H9&KS$ex`3T;AA3z1NbcUhYe~+MpP`V*3D_b9hNko< zCwm;ZHW@4gWq)naW)AmBaNosAI9LWg)RV&SUA8r5he9*w&L1 zoglWTV^Cm8zBDU+iT^KDi7x;ef=Jn*OhKC_-l1H>TdDgAExFNF$x5ao_CVmycvg0 zM0Be&$GGr)P7sa`o0X4RnN{&uWox!Y_HU8o#p*heayVLrRij^+wA7x^kcybYu!AmS=#h>mW=r_SiyrJQjbv&7;DgE)dc3yDRSm%UU}+^JuR zmz!f95*+*(?DGWVSD&k|bXw(;>Bqqoc{G7uvKdX!Z#VawT3u=H>kHRVzb>8gIuTs1 zp`noyNM4XfeY1K+P^4B*M)gSZFPL{@guBAiNM7b^ffF%$>o{Y42@3)4Qc?aR0^F#v zO1!qOZ0^mwZv*MU;E`sY8}`eNwNq2yZQnbT5T=Ow7A!$CI;KH@>)*5Ocq8-8t2osD zZaOQqgg?rBL(!*fwP@8}j^nW`f-g-Yg)DffYH_4Bl&Vm;Z&a3Pq%sY!Xz28&6TL;y z_UpZZ%$E7!tj6_v@MZ*}f9P5KD0UV$Q5?5s!VFFRV#$*=Ck;Z2a7Jc&wrP~v+VlE% ze7b+PK3KX_gSIgdEbJCU463a8bKF^!sI>Oo-yPPX*n1yl$fS-%%;b#J(A>uM_jBJ& z1Lss)7^Q4k<_Hwt^YW(IJ)eFXNaHjMj;d};pDyPKzBV|VuYXr~9ii{)W?VY;!V0)C z47SSEtsd8xPZ0+3CjIqRRUbvgg5NxP_UH@!g+%ho;W#0&MTbqf)Sq|Q`1p7A0sZJP zb6S!kY2mw3D;nPy_A{0AQtk9We)NvcQDeYa5|{q|K4I{uavfXn9;h+$=r7P*CgB0O zH3J2<gh#&04|VXb4gb5rA?h8#BWh3v6g(I)>lCs{RF-?ZjC+4l59h;TIpt4d=xR`h z_X~pOe@N_qP02DWE_+jq(~pCKNKj2^N6Zo{2!rb|KDjo`tiS*8C^PN3g6J}r8RL^b z?S=mw7g$Ecx$aC-p*q(lSGkQ$mXDn#tSB|mR&E@xGOH$um|6Xr=m?06kdmS4zMT?0 zskVhrZ_#;Aro;C+-w94dF!^WK$&<`9J7d*iTl_U6ZC1xbHVtFru*+jgLPEcR_-8gn zy+3`?SsgW4;{|}jdUuDPkKthZ|C9_+8;s+o-o9m1#%%-^^nBw8=Z70%Pr)AP z6@N!ED2}cg6sJ&BOvu?5>{gYU0Gp-(KG`h?ehL1j->TBO={|lixBAcDrX@3skF)Gs zpQ~xaa_wA^W(!xKX*F|;>tqH?B=Adwt^Fi4K;D{~>|P_V<|SlxL4Px-M7NEztgRPh z;b9rNCpud`Bj^2syR&l<+)vp4_cjP0G4RE!Nk3F45{7$;1{~vjPK=dfdeE+}sYv&L zyT}l1F}L!>9E)p&Bg2X#qGO{2hPE>2)p4SNj4pMSmt~X&zT_5!yh`aB;y>|PG7BanU{p(wf^H29o_}6BaE;!BHa7^C;C8EhzZD;S! z#`%j0;vLx)$s=fJL~${!UZWz=aOis0iM|7A5hFoKH(hh$izoD5#TOAl6!o3f zN^N{J%?nB*47!vK!Il9~XNT1jBB$(gw*=bi#3HcE1KlhGbk?#|7 zB8*AC{Fx<&_Ct+I^QbZ=6gl5+2il^kuL;8?mrjFbcIcq@+TOsLkUHg8`WkjlBZ+cG zia*w~efktP0q zO7p3h4aOapkjc1sqt zk04N6Z;}1lA5GupFcNSTR{YCB*kMdwk$f7>^K61x3Y2gFKY52~%Q>Vd>K8nrFXcKQ zE#e8&XPw-}Sg$+^bu#cc!IUy|6g$-TVbMqsn?p(lcIE^1JIx+Mt_}Av!$rAiJu^vC zUH)Rmxv1Wz`pP8HE%fsfatVXh56Eip%?DAUr>@i@j<#l2lD^kbq zX1#Q4(Qj$INEN$AXV!dcDvN_SO)j64epc@-7XzOoTn0lA9vu0Dy7j?AlFzf=;*J_j zzL(|<3#~yT=ULe5t3gxUYs<;4^;4^1Qf*&_^bYw=s1vN;n1Y#5x?g(3g1WpMvq&2^ z8aF9^C(EwCQN#xpeqrut)F}RZl5JpkP!u&=$-u@!JW5(W#X&ry^D?}XPO(pY@938A zsE+RhWZRL_B~6PP|Cs@OfZ;}{-OQa)oWl1?`!#ek?rhfOT7k-A^HedI42(dVsj3!R zA)NTbu9;A#GF;=%MFm`iVHvSwy+McA!=;Rbvh(*eq%B|9n#vvSU9}K%e?4vMr9^Yl zjX!{Tkvm8+2CFrEu0yx%)ET(VW>&>2fSNP{ES=w!| z4m;Gj9f<{UWsQv?;UX^R5(rL1!~P_+p>AtH^+|`#(A}_Lt9p7hN(K_mvTfizOi7_vDk*6p;db_~49Fgv4dKv911U zOrBCdymvPAWAR#JJD-4P?~8J2SaaOa3#BKnu`0s>)^?hj>YRs=z2y%JVjV+yO0X_L zo^K@ulc4ojjv&uW$)9a>8`;xF>Ey6CzwfoKU9nKuWI(8|YrifBW#yL?EP^K>&PF!| z2AvC8SQ5{+)<2?(>)}sj5IhyC>kHwY*XPb?y29w)62Bk6J(nH(-+t zTsyPlK5y_R*NMmo;YkqL$CkTglVpS8$&iwkMSFgOk3m2Q7^wA2cDQT+< z`a_yig0E%m>VmKZnWw(4-sMtyn{eCNb$PzDUvzX(>*WPPZ_{Aj;gy%OE6uKcx?c@Q zhb8(?YBNlC`=SG4wVg3vmNo|l`|!a78|~K4s$CR!C^~jmVgf+{NFzAJC}nTJi~^rs^_|pw=&azd3=VBapyjDnf7lkhgJYo-HLcbEVv^ zzfD#DMK3ILUoD~L^^n+;mb2Ikh`mjLWlaft%HB}Rhm6_%m-BNX89QF-6z_a5`Kf&N z<)an6$O8JNs)~Ka#isaWU;d>(QflTU!gA9G6B6%SN^LZSbnfuJkU0>kd|Ow;-bphggr{`ID{jd;n*sn8r!bQy3ZhGxqktbE&Fzs7c|fuBa`|L``F%=Pe2GT6Z2_&aIO*O zp|2JDzs&7r_;>ANc~<2#THO`N=bPpYyDScqPTTO#jwSgy%{rwo%q^LHzqfi(o?UBA zu>0`kamkwWlpP{Yy+ZG8p%iq9`vJ`1PN?CtsZ5Icny9ZWk5?{L?ST7G-7z)4jGV8LV zx*QON-liLoZ9Y2C8x-2wzO1pGsz?eJuaDNXZw}YBYqo9mS#n5xO*_=FQ6#$RnBnP( zODh#dcYnXqZZ6@6@46+Q?Xmnrw3)Hml8V3yWg>F70UWj$I{)sJ z>eZUAh{%@Xywh>FJy(t_+$O6+7F@u2q8#`5+Mz&85T(nf8qI3!I7ClbMaFW9eBor`2X&{$OuMDy zaBhCU6ZBa&QAB2m4^@xYUx;29$HrZB`_41;l7>GWBElCG`*Crsucj%rksByO?6AeA zdV)O9Nq|)tU7iO_a5R4hTYJlDcZzGdOnZKI0E^xGY)xl<2;$dnt6Ps)rrtl|aVBxz z=nC{%z+yZyAFLdLcWWB1aX^xkudgtALvEaOeOE|){)Q#)v^$=A{?EKhDe;xDC6xJC#CNjQv+aY5!o z>o(P{^jDvWdLFI*;0Zj~xA&dfzM{VRjeUjaUZ5T>zk0e?ojU>QJ5Y&wMn^@-dYoly zLJ~#JFvm+SKPDL3$=p3#jZybL=q>P@7EG&Ng|!Y1L9WCFC`bWhqxSTX5D4?VKT?Dn zEiGd}a#6SlQYnbtTzq zc9gA4(!9(bZ)2T#Q`-jfQd-e?c|Y5SPe*WP1fGhycojR66N)H!LH`34n;O1i!0Or9 zOqTQ6*M23G+oec>80-8Su8&h0N>kZF-RYr@s7`O3~3Ohz&Nk!@k2v_+0Hw-9?`@ zDL>rv>IlhKghoFi8%EBbUfQD5J(wnpJI#NDw1`Sihl)W^AbzrLz81O>jy`*S+?CA??iWrl7)iMby_wY;LMoyjh(^K9b zrD!3~i5^g#WhzDpv^OLY<8K_)Tp_1&C(;W>%wu?_mEP|_8p98-$DMGlPb6oZm%esT z_g&w^u@g_ZWlNm;CtagK4my9}D-Lojt4V>4`&_ZE1d4b~x`OcHD7wGTgY;=O`{lVV zw(x4~vNgjuY-o#~P1LVwSM-eOS`>}XPAha`XT5W_#nk*bI~zgXf^;QE>rc2$+lc(Z zz`zh7Lub6*8$YdOUKz7E=PxXDzh`bSPw-Mg+v|{tcwF-|J^N14+eQ$Mjbx9wg zDw$YWll5iHd2ZRxI&Im4{nG)sly{$5S@DniPVH;gPi!sCd-*zc!bw0sv$RA4Ziy4( zCoV=}%(qmN(_ZO+>5?NWHrm8jR%r5Hz=2)eU|T51-2 z18GA$vrU~2G50$UA=CDTa&r7uR?}6Ou`+>tHj{$cd3ojUHpFqKEvlzU1f8CqUT>yr zP}gQ>CxX_Z>8FR9Wv)9AW@ctVQ?I;VvQBc?XdKN)H>e>O_@&J?r?A@G++;ADWqCzX zg^>%VYe$Ope!?=Ds&=+0G_h7jIXdjLdQ0?ofRtB$6vhZ8@KWdwz1jZB+WTf}KE0mf zs`dwP-Kt|tv^y;csc9{G5ay?v4}GTxl~D_qprESe5#G?AM#SJlSbh5DNN&z0IQXw! z8%13!$NR0Pv)(^jDgz9^LrW*%o1w~5kGZ2FRu1}07qcQY7M;eZtM;VD8&c2Dg)~Yo z5Z@wf`BH$WdRM?@(TMPANJt0?=i+DwmGi`4S`%XV8ty0G?J&Q8s9m>@D4;9?66l6X zJCVVk4wad5ZiHi7A@HMQjLf`v372o@$7u&{FSUmk5A$hvpZD6cbG=UN_wA{GTvkr` ztdw4Gi6f^5X5Ckg71dcW?+V04(MrP;T3Nq*{hB&+1;b^1NO;T8UciR6H3P}b(Ry$j z6Yq-3dcwC2tJ$k|()H(;U%%&@g@xUA1wK|jjR%ETQIF}QW?jCMDhd&G`T=%7R5+&@ z!RH-xMpySI#E)7T-zxoKK5|^ky*-O_$Nz=*7Dx`Yj7)0IFU@r&tjS~b*qd@N-(UtmNqIEXq z+FjDfyoOki=g@91hr9R1ft~V=YQU<*)L*JzU?OT*Rh6 zULlX?v@|*+a8t(u&p?HT^i2ZrB9)egp1aybx(*q31}M;?$FWP*1(9=x%R4kc%H7hn zN&e@;>&N{)C?cM(rIQcl?J@7ozC8||polFS-v6}Z9P8b3=dH}3fbFtLr(3_2zYEHK zSao+|J=F8k!R@fSto-B4(9qm|^{>wb7A2@ty?U2Yd0N(j*VZ~eS!%`i&xc>pc0Qbi z?gV<`J*9h#x|@S~+WxjcoWjU5NIt=IY+`s!FP_hgI1O3l=KT2!`18LHz2@(>o>8t< z#|uNzAW_e*b%>c()tBo{mMj)45j43&9Fg%|5W~|rpT*ecDy8Z0HM&ctNqF6@-j{Wr zn>S}esmyyPb~~QdeD;{P)f3&C0`Zz6MCpeaGUcyps-g5 zc>|(d?RSE$i@Mo_82+vaRFws-uG4LB^m*D#I_c@SRIT~TYb!-Wy!HJX=pA48ogu{Q z0oHjvUEeT#YGLO=|1fgWp=t4!i3>&RM_LDB*NAKmetp3Jd&6_<&>hq(uOiec%<<#= zGir83CZfCQV@mR_cm10#TDSJI(>m}<8q=xOhn5G1grFkCn&>R?-h+zb&B>KDLq=l} z?=!hVg5D``J)@g`9u)L*ANdUkQ-5kGWv5Zv#)UofZ+=kh8FPHgn#mb`m-j)ue&^5a z2S2R)Pd(&@UaWufs@wRXQ!{4vIW6ghlMNgB0wlbvFyv11@`mUICNx}xRQP@o$8sp7 zr8nf>+Rb1_WD|L!184%fOIFa=U!Ir=PioI@VjD}#&xJrJ;WzIuAR=Q*IUz~>3H`@PXB!~!^Fh>?v>7UUH-CL3ij8PAs6|? zD!e$|x4!dPlqr%frhMTA>aG*S^|zWjnSwACOsTriWb!xje5v}@G({H}zh1vygs+h3 zj++h{Fdpw{T|D#G?P*%|lsc8?GEXNjFwCUa=QB?CnH-yC5A|nK~@7m>$KE z&594-Yx*8ypJK=(taZjqE>HGEZFGmJi!>#CAiX`J=hfvsrIZN>RKAsJx+vlYr+@nNK*1b`cWkGzDjkH_?EWBNlV-%e=%jp9 zR{!wEfT+nV_Cp>9X^M5i=sO=#70^2!(u+%Zw&o$u2d7e;=gTvR4?I75>=9GyNz(o{ zZ}n;~e-o_uNK7!|nXIT{nJKEL1Dvxa2z* zYpc|zkrX1j4c)O8uqwOoS#N#SAxNy(S=Y$;rj-57*oc48Jh#pLjZuXZd@ z!v&qwB6XlHrQ~tr+sCCGI?vVRYaNnPU^h-;)mYm~Pp4qe6-+PV8|I;0hqGUX6wz>G zU2#y3an#HF{3ndOY#8!-6n1Wuz8QGHTV}=IATS2^QgEcQmCWEbtA*9OsCnYu=5;kR!2S4dNz2j7olS$)iR7lhjJjliZRTvdG4WpbxOr&xv7U z=`#9?w?4dIDDy>cfTc<(j@dQsp0MG&ed%1a(_(cT6)e1iDbyQ2{K?=hS!~yCwKdb> z_M4f4RstE}xGm(pM>`!feG#h@W_46~zaqQR-`^sO7AnG;G@jeuzj19K$VXd;4H--` zqWvj5oaP}4*$4|3>Yb9D2zy5ZCrm%&MaoDqx@U$5J0nB8=M4dVAEk|+Pyvgd*bhsY z852E)#Osdkx^h5bj(5pK%0+`~`9zrC?(qkwuz!GDIf~7s+R4=>sKCk>r=0Qn(i20b zbVC|rRtMkR0w!4w*hDf+!LhkzwSN}QL7NTgCjUNf*COK(;*@^~yE46DY1sO`DNAAS zJ=!3fVFmJID9NnbEy=AuSs^nzzma}zLj>j6LzDG-b8C_KNi-Lo`Rtb?U>{02G1RFs z6|7!YT${eRjj8O7aD(t(53H$}dOKwm%xA#B_*TSZ0u)0TH&b-LmNl^32a(8cKTetC z=z)r{CukKWz4@2?bbo01g-E}=uJ_(j#GHn=0mpqV?;Di3(%(u8c`t6<`@WSQcn6x{ z%=D?T1p0IPOv)fKROZ2;-~;R>M{;E6v#bC{qVK;){M+8Vf5Y^w^EZnlnV2`W^peGs z-=~B}x4#}7>cNU4m@M`(bNb zUtRgB^Ii^m8RH_eRu3^R`p0;CW_a4mpIO4Or7Rp^Zy1A z@>w~uD!zubT;ArkLA(VhLL#tJbFs;#=}aE*Jp5tuV$X;FoN1vV#H@5^$-m3+l@f>V z(dx_v!(pV}=`R?Q1BO;Fi4TV}b;_r`A^yYAQ)Tn=gPfE^3)Hfm&+PmG5?@(8r8&LhUpi49Qlm%ePM~v zbmlwu+z02^2m@b|#!m@X6#)$|MIGYZg-q;QlM#@C)q!fy7whh^W6k`HZocx(@`Mgx z@f7jZ%as~HI}YPiEg|2kURbZ!=RUwRJVm43oL^mQEw)xCs3z%%ihJpDrj%M8Sdz*z zQZ0dd8ZGAGhwGWpJ9<&M*yw6wpH&C+7Ra1e3wLzSzgU2OGjw;k)}!n#H8&#uxViu> zVAzOY1t(s{+4NvH`V3e`Lm)b$RC#pHovS2~gVDX-E&lx;;X6Xisa`}D>nC6BcY9NX zPjC$oB5|R4;zL!TUc<9;Z%8c`eyH*=EFrgcX0Mi?$knpL(uRCey?aS8>Gl@JlYQ6| zGSXfwRpfX(F2ZQnO)(b`6o=;Fk8_mx{d0U!cP<}IN-}8%bu4J60;mG;r3rad_~plK zCJB^d`r(t+RD%`Kpct(nTfb9Kz{3$GYT5KXXAN7Gzgl@yz#`n!an`A5&wNXr#IBXM zp3MF+hGBWY9r^-BS#ipE3~(CeXQzIoAESY;M$;TvS>>L8aW8Yg`CVf2I(*&Y%WN-l zD6ki63WnsfXhmeL4B0zT9;(K;)tpm@As*^iyBOYG-gQqilIIhgdcXMaJS|#H6^&?v z&mbm{TX+UMbEaLJdV?Jkit9;ZX5S8~t8OFf^ni0u$XZ}su2}XB7f>$%b0Q;20=9Mc z16GfWF6!GPo?RrKGb#2agY}=O(@mBtTbre$c6CsTW0~6ybg1>4!_s8tC>tb3+3l^z zpW>%ehKDYJ*)MgPxm%gx#=zy=ZGbQU-5yBADStco!1$Ws#kAg4zgW-DH}!SLa}#NZ zUG^F0X1?~uqTV6vkTe-w!^>dL5xLWOYTTLlczCvXaL$gET7n_iW5(elx{ z+qM8wn*tiBE#0XxJadl-dU5xrwA_%e*8$LC4_csKKYXtO%k2t^Wb?b5+z;_>Kv z3MT44z{JVAnF4#r_#if4dLdIVsZG2kiFz%z+;a^x)oWZgtY09Zab2V>k~`#V;;T9m zu=TH6(p#^$gjI&fJIaaz{9i|(8}!j*co+HD3)1gq;mNMPO=_6Iif=fXF$cyx8YuAD z=IX2bV$F+s>t1{VnEN;Iu~3=jKaZ$&F!lJ!yP5JK$xtF%6-JG^#=-jBP(~ZbrG!#3;ow(W#K7%UZL$iPfQTlhaRv(-9)~1Jty&b zD1Dal7uIXHi#&==E5r4Z;Jv83@2M?@=U4HfP6(#g+EUZ%PPd%!s(Hrm18$%pfuCY@ zJ~b`KD*^Up2i(eViIcNd)FAllA>##h5HJFBmIhe3Xn?Hg@O8tka%>+{3r^)hzs1OfOLym9ULJB~hzKu_F zpOA2eQzF8ii%Wt=&J=L@8!|LeoaN;WJ1kC2<(>C&t^a@|P;h?Jo&gXDnDA(cB4%WN zo6V>4gCIBIvsHFBrK+Nkn>l#hbcCPVS`i*&@B4A#!eJpt&ig^BL8tWKb3Ly^lIYr| z?pEy79g@(VC69;1(E=L}el*AE*qH35Ni1ip?z6H#<>W@`=IE_>mAwS0Po~Qdd7@8u3$Hk?{||F- z9ah!ay^StJR8U~kN-N#c9ZD{`kyenF?nV@mhDAwtmozL&ML;?i9nuZbx!^qP{qA@B z{=WB|@A|HD{y6ixbgemAbIzxqG464XF?6oTppSkXd7fRcymg?o+AVVPQDE1_Xfd5# ziiQ%~-Mv3bSZQ)_ciP#-q1zVU0_Vs{(1V}oNPnCG+7yfR9MC2U&R5o^ZW=}$N36ypi%&H2#=U3Qt z5eC3++54KqaiJ5npn$bw(_Ru#$QKn8vjoXqg$Wwu{>5f~E-(LpE>-9cEq%-#4~VnG1gEVPBP_~XG78bU)Mis2UDuG3Z8>FnOujmkr z8ZjkFo#{#VB&@i`?ai^9Y>Qg#4SXAf1!Osnf7!!Y=2U67n~Q^f?Ddd5(OYa?`bbq( zd3HPOX8O?c?TaUo+P}A|GR^rYBdxAo9*u{q29<^8mDX@f&VM8&0_w&|qK9<$B-4bYcqxTz+;ew0XQ!<7cTaw*cEN^^N*6TyjEyNN8m&H=0@{(cOC zICA}NM$8>?S%m2C(Z^2j`KZa&eVgKW1}r=CLPxAFU}8=jNDM6$k2F~Bxz7w@$$eKY zXD_Fw&WVit6kJpjJR=xFDFqn+09WNm8*2?;uzr0c6m=MyIx>b>I2x4(fP;A3_wRqC zMLt(mWjs1kbcwf-{*&l}tMI4oiAHZ;gChWNtG;iSW-U#HW#8CgbqK%uQMB}n)8h{% z&J0;d`i;N9i>u>%bK{@_S#Ain8`b>NzMc&ruNi#bhDXv;*s8-E=y#AHewP1Z(f3q? zSW4OY5`HhD(L?yLida(Z_#`CV8LANkgIEeTyYs*D{6W6HjI2nRQ_-&vdH-I03q5Fj z3;p|dOAWF0WFtT2ryu(_b8{;~J+v@A62TnJK0pr?kJ)C;0Q6dPXaRuUUe(fArZT1I z@Zo=7Jpw~5yr`7;hI9fD`Ha5^`cc48<70VgGNK+o_-iXx#*E1EOWKWr=MmbwXO=}w z$LctMyavDZ*IlPs|GJBCT`Kivt8kJ^1Rj{+ka)K-ek0rcepGS(ui@?gAFutef7Qp? z?{MW#l}QEytOGeK$;0kkT#x^%%RKPUh-)1v|A&wKtvvtz)s9uDf8emI7Po$25R4jE zXJU^t?pFEB!1!KP|2%MX%Key^6^EyU+%*1pFe9oNBgX^ zE%6bg_zHU8`r(KKk0hjb)~@f>yZ#ANIZAvi+y|#Y5}_BK;oJ#9m6N|bh2OV6zPYK5 z?hlOH_wJy6NUiLA({5D0aN>?BYUcSI7o{csCFQ+`u6=}#q_mZ2X>-PF{rFHh%-qFHmELTa+=ovBURQB9DV8AHuRq3dH~j?-2W+>R*U)vkO>Wyoqyc8t}vqxPf_=@vY!`3N=3;~GZM9msoU&BB)&!6 zkor~XDbWBN_C9OGwW@h+-tNb*B$lvDm%a>26@lgaX_?!Z4vSSPRejojIhfp(^IydU zAOEEe3pc^MJ^w$zj{Y<5kA4y^*MpcbTS_2nEX>dxOf8=RjXM1?yYZh~Q=v4#N_9~w+#rRg0Tlby1&AnaWbnQV#ZISyRUt^RoPfx_|VX5QrmMci*9bEfc?=0Pqte( zu_l{G0PvX3y!-=wVsu)r-fAU!n(w7>gn_$onT<+KURBut#&_aIe){JS;1J%o{@mTJ zumIT3lZT)FUnu@Y13w)2v;JWi^H33~23J=>~_w%j&Et`}eM0 zu#CQy`E)3VN9#VITAUXABzW@pgUSas0ga>V=nBKgFlz}OZcY(VdVbU|7ta{L*ACzc z0(^f|KK_rg3(%N9G~dj{Y;17zNusj=o@9=<_$qcW&iNe#6G5kGD1{r5^NUCyazp*_!qK^rT{= z&hj9jAgrpo431bJu-)rRG-|YVxb*~c;Z8P=6*sRgKPu&arN{rX$;i;X1Ws=HgAH;k zsF>X;T(oMWAwZ%x0t8CXKzSCriB!=nNuz2SD(y=Q!{cx|cB+{uv@bWZrIjz&%wh)|$OK(lc$G`sqDHv6u$pp3WOy>cwU9mxQN3;#d) z?8+~lvj1t8o6S6m;Te@IXo};@`ioED0CM?n{BzZu;L-B-Tc^4+eOjl45hAt8`;oCR zJE)I%9JoY3i4e7T0uL0jAJ?pI0q7H$*4@`0oW^-zGaMS0B0BM%u1((0`Y=xLD(b(+#c+P{crO^Wf2pqy6LruVK!!^iV1zxnCYm}l zBJWVlBIft4Y~~X)8r+R1Ni|~}?NGwvCbh9evbjAesaWe%r=;lAPTTK#>5rX%mIx0p zvws{Et@!vfEc&I&!8PQ6(4>Ld@%OR*ASV4)dSP&dOF=T2=6A^97iNG5oZxaTe}f5t zVm$&T8?b))SxF0&GSM|_v$ebT(W;Fq$4fW z+W!%N19-uY{BKF1jwU+9j23Qj@wmK{m};(qkR5&UOEir@k(#z?yElI~^Pitlgdn&^ zw1s>h#Lac@GS`y$i%*#(oYeNCT^M#WRro;P3LY2n+5N)wpFgmz0T;4#@QL7m%&?(~ zar3*B+tTQdo?P6Ig?|jHeJNYXt3G6onoEA&Kbz7atub>=;oA>O=;Y7_$;zJuKyW2W zG+PGP?lH>VSC>?dVl>yVWB5qVN>Gxy(EJ!5ZKTW(-x|jb#Ya+Qf)O8svFN$-`~26m zf50MN(^g(y-9tz5f88Vw8#~xDS6E5)>|4dM>u*~1+2LpBXp>-YbA#do&$oXa`52t$ z5ObdAv$$x~+CvNY2mz_weE4#HfSfqS@)`cPv+`qGDDpP?-@k{XL!UfD6|2PEkBgK? z`S;Z?cFgv-{(T0441AFp`0W&8{*v1K_j9hNxLm)V6F)>J{{0*?^fu;i=YDsj{O|mB zE{00=4E4A3^tVyz{}kr`=JjG!uusW_)LQqoOEkr&xOIzV<#s6psYVA}Xc#6CvglA7 zZRU3JO75;wO>2(VuHPR{mQvTNP0@?6OPW@;4Y+KFKdgR(WSZE*5}%6ACM=m^WEFv$ zMwiFa4wftC)Cef3@HAv>%c>UfY!i^gQ$}~bHg72JlK_wSL2Po#o>#q{z4jt+e!HD6 zn|@+ON3*DR{DT5*^_d18e`c7d>yY8Yz^iXB4vE?=nvs%B=l4+u3?GJEed7aHGFzTv zAp`8!*g_Gc7a?&L+xR3zwr;*C5Wa5&zRMmbkL+6AD%D@tS(etKM;A_rj$}&xBh?p_{s{DD|_E|;YxZ!9A zancn!BplW`qAjSa!Pn|w=KNg=tM9+8%E5M&oK>d#>V^#(c{3w@qZ&hP& zqvBXJ)X{UyIj3`_zog{o&~u{axX!(W(&CTeD={ge9%*~(S56Je3&YPa7I_$D3`mXv zQ`?%RrbN(Pgqz{{{l$MeaQ?5J&1iY5X}rO<;gF*H=&$}1_XYX1k;q#R(q~|Vb3KV_ zbPHFXB6U147j)<#hWc88yTiCtf2xUfjRoq^F#VxM zmm+blzLKAu6B$shz}w+y<|wqX;zB>K4JKFyM2#BhIM$o zdzAthKp$@#6OG=?TtCLTnNljzvDTm5;LMJZzNJQ(Bjje%C32b$@C7*SPDr;(hw{%K z>)D2n7VG6o-d`ntNC5sm@R@_N!TeWx5p^u4o+#JVD`ECMd-ulFd(t;|j~IXknQ_^) z_OPC`a%j+pTQ6Dbt%5glIZ~UCh;{VoBMJ@@FkzH~<&IjI6zozVggqLiym?(545jMM zE$I=>GtE%J8rxEr8JClK_~pu!lg1qR#t@Ie@so&(W}UJeUtdi%qTLuJ*hQlig4ele zh3tw9yx3gECSI4dR#%^cliO`T!^3M~@#ft+6E%zV4A`UZjMamSy(H+w++07qGGOYvP-4_l(n8x>TLFtEGR_9eFR|iB z`)Z3tTK4jH^^bbq2oIU$t;kd4<;B~^u`L?u3wwpy_?lKYz5HmPW2P0~ZHqWMey0O> z$`g9)!AA)ZtDJYKM}T~%P!rp`?RlYEZc&|M!;{5 zHV|_991ZF*sD9CVZo>N>LX&XoBv4ZFbvpZ8&nFUp_8>iQ%DzwCTWXWuk@wEmh!5R9RE8OH-gM0+3|cvIiKU39 z__fr>S=A%Hsih6qk7uUyil*5`+-pil8#}QrHim?U60*2A5zqFSub(|a_D9?HJ;n5N z&l9-Zdlg1DDZT3CASY@)FvOdN76!r7?L?aK(S1#&7#D>(Cc# z?Bk;(8ZZznr>Wps*Yyq^r&X)nXdVZ99o(zkmvHZTh~FLOsUK6I$!y()Oic2YUmjflTBz zWiRUWQe(T9=bQ5<76gd62WdDfpS)UaIs;MNkYHpeJOU#_JvBxp4NAORX{Js$Z`5(z zd+u_PyV0L%Zx`X>k=-0Wms>O@T*_Z`62tP!;bo;rvU6P%dh3zX4pa?cl@l~SAU(hY+Z#`sglHkWa<%zJl)Rck$uexwLj zxo((a>bWRC(wZ%jZWqaG@M(U&NOa-U*Nb-~rqr@BJ*VW@Y`=+Zd|g;HHCauQU!g5R z+g#c(thH&I)oyk6)k)uHvlV_Z5x87GZ@_{qV4Y)yq-Vs?nW_66kU=(*IyM-xsjMj?F+RK3pyyz18Oe=aVKw-DNVkK*57>*@KDw ze3?4z&xJxqa}(ZW)_>}9K?`Pg?&O8@hL`D)=^*`NaV=e_*0;I~E|<}kBe5%TYyDcC zZc+VwcZ%ZC)XYWK^4j%`ZaAI!;-MCo*@WU*m71+$YSDc&Bs4xXt@Qe&n))oJ+B{d} zMN1{~Jk>Qp*wvyS<3y7hd0GtjDhsma8b4*2LBCI9C_q?At#9An!?@Gb{npTw?a9D$ zQ@4$twYA5Zs~OZ#=(>1?d5$WL$7(~aK)@#M-TL2L0JG7O^BMPbo%8urOJ6n!#P|@N z&I_DiAU|U?+=4V#cd`9$YJ>g52xNT2;vptPsg$u5fx<^z_C> zPfIm}RYY^|WR`+ls>XGN}zQe*l(qb@^5xqS_TQ8cu#+ zcB$+^_w!hFX@Kh7RW1WLp$dC?SYa<#ZbSTXgFzAV9UP?e<9G@{8gFqXTsDwrJ}W$> zVYFUxwaj+ld#BRchN5)8`22-MCC}DTXH*R36np1GpMG+8fAlVM&)3Hoc@6I-`MkR( zlGv`x!^ZJOGUVX0qfI%yt6sS!qG#E0tA%n&t)3K$+SKZj->gtzO-aVL&AerP@6WxR+7~5XOSjhAcm<5T zL@?DX(~Vb4G2@aSL#;Gfit=3t{p*i71Ow-ysZxAOt^_P9+6_FT;Mw)D_6DLePP=&U ztR#KGu`)Hqw@U{XOLlo(Gv8Q}%&M=Bs+CsteA7xdXP&L=pRdOWUT3qSZ63|Tg*)iX zZ_7D7ZIRfshBLOB#0zgfnQ?N&Zli!D?+`kix^9Fx^IHyIv(8M{zMC%TLm+su)jw?8 zg?egj8j^TzILtJ`vuyA38&7Ze48Kn$oexy|J}zY~K7YW@Cvto6a5I>8gGqL%4l@$n=tk z>KtEC4E3ss45QX~ZJ#-mlQ$gLdw*JvZt7+ayDfBXv?K35c=1?4+rDIJDhrz~eCQpC z&G5mA;%IRui4&3p+rn~+&Z&E;-zi#1htsOLd7M`UD^^BQklXB%WvEWtY50EHx{F+M z1&?hPisQ%}>D_VILH zlJq$G)qfmn#eJsQxwzL!2TjFHbi5`=bo8yp`bTAh_(=ua4paM*8Q@rLK0hrJIOw+} zuzH-3$iZc(wo97FKLt*^GS0)CqYZir4Pd|B2?+yx+*3a#l2xIMf;^>B)LrXAs{PK& zBef>vcx7U3ERFo%wFI9E6yV2FSEjx7n@@V{nTFo}_P3Wpj@iGfY!LH}|39jT3wGbT zaLA~lJI9A>JKv}#bE;?-A%KS1iG#<~KqX~b**W0SYZ_vd5>)Xb>w&cvEYmo-fLfTG6SI6tzBqz2x5<-!V;T4nkc$+c&xE)*aG2C6OkGy+$0yB+MI6zd- z8YZay9yGXO@`3G`sp}6MJi6$KQ#nI3uZm)o_B3gr?g90H`9Bm1pg|A<%^}gs_f=lY zQ;D-VCdb?12|!E4*=Ae7eKKSSfrL{6-IzFvK1pANCs1B$j)z#DO__CFs znwvMw+sU|JII3@)JOWBN0l^BxLktE|j7O6;PyZPCisWZxIU>3F!xDuu{U7S|Z?@R) zUivJrx)a=sA^!6^_;v@M=k$~Jl@S`&IRQQQ)hBGyU-S=uF_B_{JIm^1d<=v9aZVjY zB9pRIeS_w=35m#HP}QdopEhnIXVa)0dO!{S)%c(WlD@A<|D#_l!jY2n%A*F=VR!+ z9p4k)Rv7-hhoV0p$70@+F#4q)+@nxO9q{c{z>^V|?B-3NZ~J|H$Fh7YC+h4b#CT3* z`8&(yC+v~*(2RQzlH)=~*oXg(BK%X>e+E%r{V`G0bKh6VE93tfC}NebT=-vGr={De z6v(#l)5nhPNilUXhIG?zS}3AXx59pdX}Cn%*<$WOaEzcKYIXVQU(e@Qyh@9Zh4NvWMQ}5RxM_pN+pXtN7&nXJV?(jF`xWY!7RTr( z!rC*xJ(U+cH6O?%Jfcb2v!vA zI(n>@6?klah&nfmD+^B9?phO+GTyDzZ0dKCOYPMWu}w5cY*yAjLtqh^6E2hr>W9x- zFeYDw$%+UH;I!JSBrbeNWtA)9Bj`J$eU9@FW@g^wwk&wdp@|=sjjt85u^Z!5zEg*k zH5o5z390O8@|aH`88Pjq4ucn`Nh!<~^Nb0X(b)=U^%1u};4&E`%Ih2yRB*=|BZ~(; zLX&2~WV;~1?~%uFzEE{f;@-dO3-UA*q7wt%4gq8zrMf0i@s08NqPIAR(7DyYM0dHE z>nTBJwKOLUHtE*V?=M;^;+iwj`};@sOOmR+`nYdICpCkq z@(1P}bEpZY#iwn}2{<*$qbL8apdW3zHnsWIc(XRZ5rkQxSwj3b66j>Px8rT9;txMX zyhV@%-Z-8)J9VXj$)m(OwqaxZV-ceootsC;Q`d2H1C%Pa!oqV*s%}jLNPDlWA#`9d zrvvT|hn6Mjd38p*lnu&paf2V)e{0jdqIKgdDXYP6^{_6}p~!4KTSZBS_C$^DCy#Wh z&ngMcH*}F)#!Ke<*PXPzMdL_*s@eSv(@}H6Z32AuO-6T2@?TEwac;gCUQj=D+_O*nr#ru}S!kjYt7^lfN_nMW8#MgJ z9mfi=Y!uW$%@I!k*^FczmqcA+CZ2TZA#J)!;H{NVj9BljgRK$Ges+`X^$o!tG~}69 zk#oN@w#B-QDyN_ES?!M252DD$Os-h3IFG5c(|7iO8Kh@pqc@1m zsmsV?`pjeq2Chb2d5trp&Dy%aVIS`u>AU|PN$Cvcap=TH)}UJny7Sa|yuLq#n#dK5 zFr`U%=P3;h%|<(GZ=rX+Sd5)$h~SvEG3l91Phzy45>})7Ae6n_(N~AUrPh{a+C9Xm z7_`{0H9u|3XgWl{T1T&7FRrB3x&Ma1H__W4Xoi0uKflWim_tzSU}*#kYLE%0F%*Oc z5vTTZWz5xgcyCQLDW4qb<$Eet8GgyiIY}#0JQ-_*rt>o5ZqRsTLL!)}LpXu2mvlTGnzTqbW8%s9|OHiB9+?JQ)1D`a!tGKS#$ z>hOqvD3PmRX33&~N{SM7w>V$3jggtK6p{S>7zjks4?HMtv$w#Z9uI<}iK|ZEPZ!di z`%_7S&OGM8c81Nm#0NnbuaF^RNQQd*-}eo~Xp>@fWG%P6Z+&&PsH^oRv__kt$3)f_ zq9E~Q1m^vQ0?mN;d%UtBpi+~M@fXFX*E{SEqUexysV-JIL#-Db%NH6@HZnHS zPzW_)AL9_jG|@qg;#>+qXnOlntQlq$ z;Qv-8aQF=$X?ewD=4&49CuA4{l;1Fz@bCTDWm2*DLa}a_h?z8#L7yZ}Sg8$T7@CJ* zQWn3Y&jymvrAS;nmuw7zHfiQ@C4v!em5@s%!%i8+w?I}FOHv@l6hy)8+5vAhGU-86 z(BvakrBp72fp?03l?gk05k*q|79-ho%o=lgwl1%ndGO_=Bo zX51oJa@Tm__Vk+yw*rr66>ju&aEMY&=WETU`09K;^=JywP}+oE!UTERGw(J5x2IZX zcN?Ryil7|8iX-pPoe;@f#4QWDE92?A7ISzHZAg!4<*m1>1fc@h~fm>fDf zL&ZcE0ahRood`7f03wL_?KMZYQ4KvQR>4zICHK2R`xn7fWSY_VMDdhnnR1}eHzMHA zX@A;o>PqIg3L?&6tmVz0C=tO}1QL9V5HYnIyHvUgdqR(_wR_^i0-lDRdOj*}Dcxn# zFw`yYd*WWk_kk-=F2o*wuvgur%wtF-q%pJ6U*^2%x=9!oIGI^HABcEEuTE( z>kL0$y(&fpk}at2t8$&28^-Hy6HqV?yq!*4rb55d3sP0Vwtsz+E8=;?*we4p*Dw0( z8>sBFNMLF+=K>l+wdJq@ZKd{wqpwldYc4?%!*Z`?BQWoNJW4CB8Wvd3YtnwuKK72V zWRh_-A^qK5Ax2i#P$xdW3|XWIu$wS4_h@-}!G;t!^SwoV^1j_N-6$x^j)?=4UH#Qp z6avB}NdeL?%D@D5E8wnk{;%|f-KornuUfh)9UJG67PLYItY%uO z)P~TgV88~^o2=T-PdnauQcny~-hIJ`1sO1}lCh$Lg3z4_OAolLa&knr`;pMEVg)+6 z)$s|{ew*KCp6CzS?@gv%aCSa2AN`h*j-OXGr-ak21H2^OLohIRMCy~3QOyUB&zei1k5FH}sZNx`gH z#(K*Jv$Rt&ut>s+Bokq(IzM6)bMxsN_VY*g0XBlnA^3D zB^|1hQyV{nV{P|ZwHw$z(;}Zle+{w#ljkpUDMHXyN^)XJ);jCkC6d|)C6IuKsPL(8 zi(Y~CYgY;B%d2lj+{dYmT`S6)H7QGWBio_M zz5ey(&+O&auaRC?PhzEB5+)9*rUg^kkprqZBLhR}KFjC0lBq5*e>26=K6IcZcir-7 zNk(TyyXA8hq3tS!k5x3H#cUX~w;6T!mJb-!#Hq!PumYD3Vo6W*lVxNrtrB`o$$AcW z>|Tp4E}UF|#_Ps4QJa06{Bg!GCWcgX;OBZ*>wKgv5|mU#eN!MAy)#(Jj=}a?1UC9s z*8i>2x>LSSe+o~7IH=L^SD~``RjHrqgkX)tx*Pi` zrqKUgJx0Jr*>1@?Qqmja_9Y{~Q~u}}?!OJL6Ymmkqj1sL_hlPSr?M3VBTW`hvVWk6 zQ7xRr=2wl_y4|SmZ8#oo>r4!qPN8bR2rWUrbt_K*OuEY}T>eGhvs<6_EWWV$LSM(- z14dY&^1pFw$ni;&MRrhBQqd~%c!EG1Vg&vfV94!*!R0kwqx>`-wvc&Gf+iN9OqKrD z@Y&s4C}*;TuAqL`RxB)tb0o+IhufS6<|_nE7{yWv zDAEXtp%voai|g8DQ;`qdlVjv@VJ}C=NqB^5&J0ZQINGHIkO8L~+5_M6#P%Q~=I1JO zyC-k`lt5R~Ml%d$a4~{!D7nAT6!X2b*^_$7=VgeAl7!(~4Iy7bpNVDt0fyUdbT2vC zx(spbz9YMw$Spd;dof+6t`Y*FQf;p}$eP&}rf((7SbU)q9Akm_c}Rf0?q`fy%?m&c z5KZA9c<~pMu12>)77#?nw+H9RE zHvC7kgFw)U^CI^rI0c+V)Bxl@42-3*o4qP|J*5Pl#K1ni?BQssUN+-h%Gz?qqmQT! zgja>dH~q*eE*rI4pG!lGvZoA^D>3hrX}rV@xVA*%i*)^dU2>+3p$ia7CCUa@gm_ zy_wEDN%NLJ7_}XUdAnW_UmNXI+Ri$iY0|G(dJxz?)3pcMFhO;QiCqep3Fo!z}+3g&2_|x#`sK?Ih5tgFs*BI%ZSa*dsD1lNtE|M#|K7 zlBMSQSK-O^C=S*Z1+A0(7-jn#6E3%!TwuiC0=n;vK4#o1}z)ZhL zadQbZ8dCKQyl*`@i&t7NLq(0ya?hoKgY7a*6tA@ zt9CMdJ)F;UVKFl8y&+-~Ouik@YNjJ9{B-%%5#y(#7Cq;vJK?Fh#aOlL0e8#P;lc*` zAExs*`!Ao_hBwOypFbFsE8045Z8}-sf1vssW(Ntok4}vEYM~+gz>>cK6Q{ZWUzKBd?ad17EwFJ3(z?QVvz;b8zKf^d#HekqN>MIn1-)frERP9 zu#A7s6)~=TJ(?SKJ^F3E-(ss;IEGwJ^ZRUWppR?ZVH-)mW#4{p*SSMbZZnsH?tuL@~>5sf(NzS9M=P8p1t%N1o&Cu}5b9zo~*a9NM3_EA{T3wFUBiwSnf6}Un!~DX&-ZA*ed%fVY%qY6%+EI5Hvo<5Si%EEj+F*Twbmv?!VZShIfd(rXSpY;jQi&+sHG9omG!%~LuQ1RQN1tIdO^{AkM{g|55N4Z4Xn)3vNg{vW!VkamJ+HOn`)eC9k@aw=Gs@xgSNGxuMVnB<7&!jx!6Zr7h7hJ;9_doW?j>0kbL`y zRI%`vd(}!x0&Hu;;le`JIj!if$aKM7MqZNf0w>pIk<(Rq7SU^ire9a?lXYGRB8K%k zR+`tky25}(Xr7!rV>`m)t+n>`#HG`4NYVPJ#{<0}Fofa39?OV6(Fg_O@a8q&r8ZJp}%1PfrXr~DH zuBE^$#AgcUL1?zi4WShy*Djm*!$lpIGo$!@rheS*1M?YdZc#IPC4iz7KKGZxBu&az z8C!Z#z&vOSV5yBy6B>YpBzr&N&F1)7PwRH09=mwQFC614Q$$7Z3(Ie=FY`v3lZ(0U z(`|kYee2=SegNMn6w%=@R2?Ko9$YbT`4p_yA-D8BWKasShrT52ULelBR!iV;Gp(Ez zCq#h7-D=Yf-(_@l#ew{#+rGn>sWcvMewZU2T{WOY!(4^ex(;Rr#mz07W!)XU1LJ~3 zYpJiWEi8Bd3xpUiO+BV$_tIj+aWM@3GbBjo7=&Ll){MVejd0!QiV5@BOFeUl@X>DF zc&I&h^)61uHd^YbyS}jZ_1KZv>RM#xa>Z|IuQAas@iGK%e&F-S+DV}KeSOglTu1f zIqz6~CkUzXUg1HUZW&pXrHTh`FmLkof-s>?b-7z#SN)Z0*9R^7^XDk;tozRD7MWZW z8k(q9$ccXlQ&$ZRalAi3f3`FiUve2^L+o>qqjNNHNvvHEw`-9^kcqxuufIsZTZ?XjKDu?sGm_Kkgn{p1OD^Yb3~+rI0he zFdRF`W8ebqtHSD%fe9tWnfD!JaeFN}PCgtKZ|X1QK)Vf5v__k%dU;H@pNl=fBJQK# zZPDc2WpETMHN&A<$xE81tVJS6)E`fcS?`n0Az(TxmjZc!-!%qzHM2}%Q$w{S*ISNh zrdysX(x^-u3+Y!HUXYdRbayR9XkAd7!E_bndJB6d%N4*gw7_U_vbbil!om0ZvESKn z&cZToGZ|Ind8}hIMS|xpk(})%JG=^`hg!SC0oF=fFJ0Uc@XRQH^mx}@ z(>J%!H23gLav4EtID8|VWTZh(>xkSZjr@_Ive|e}GG@q8J)OSRoO>Rt;4q7)s+KwFG9vcro&Q;ShK#2*|6JXj-yYbdWCS ztYFHLnt2DK3b`n$qm?wltN0Tz;tf)`Q~>!)EbC}dI+6W^X`;VHrS{PJx_gsRD~DdU zXx3M*QuKTyKj+SyAnTYodVmKfm!y=jk z;&4edro@C33u#%eZE>q*eCrnos$!-DrQ~pv zx}wtUF~-ZiD}gDIu<3H-9V}ZhFrH_9wld<*1JXiZ-MA48xxY}Omd!IL zwR!h(??rRKd3a)G027aS=e5EJLVusz_L7_UXyXn6{`N@nUhUdCBIvvoutZumMaQU! zR;>EocO1SN4Z7RoHCHyeI@iIJB^f@-Jg0s%`Bl_Ax8Ov$v>)#(o8f+(d6z;;4EK?N z!l_*zMn(MEy2|3=GYel~0o!*x-U9mZRvF|;zv5~&<;-dZ8+-q0ZkaRaXVSYys33NxmXxe{U@8!`&|@nlQa zE6N$<_eEmZ4x+22AK`2Hxe;mtc!FU`w*7P-$P$A66cKf*Qa;`41W3>54sO_~8ZQB} zw`7^`yoPu7=jdz{E(d2)$MTNv-KMe0DS?&?zsXcw7msG9IZIZ8eg#I|^i;S;5&G!h z)2?XNu4_SHk^wytvpJIoD7l5sO=yPAS?IMb2i*ZeBwVXte_Op;4xdv7 zBCNsOGh1?f9c@P&+=L}lC7G=D`@s%lLf&noyp^oOxKn(ty z^9IvK-ynm!b4|?MCQT)lPLa~k-q_r#m{Y25#WUf_>JBDf;ij)DvHK|RraD5$A~&Tg zjyVQ?oSq=1E9YKhr$&GU(N{0~y0P*2)S}r~&oX{T#K8uArTfCTrm>?GY2&zlNkTDf zSZcT4P=B&%+ilgN>EI5Na;VVtzQ7&=vqRhSk-2d9tZj)XavCpTSW7|E4gkvtC>gQ5F<;#64Q9mt`BqM8w>c) zc0mN)X%M-$RZgQ1g0VA;adWIRh8pt`k81pJi|WmjmAFx#KG$#8dktimXgxF2u5$Az zQ?H*g~t`N5I?mSUy81Kd?dR)T8giDu@DQ+(bdTOTZ$>dTN(;28OW#z zpX4%o-}9n4#6HXLM%UV>f^7WwI1sA*GbV~TM8t$GEq9`*Vf`)B&ItE4T%j@|!LAp9 zR8;V_k7S%3!`K?yQW;xs141;8AIMO z*&dg2xHE9?tmRCm>>7!^;nI=j)V(V@Sa%Z9H}JU{u6}YoXAGw-zr;&xPW5V%YAi2V zK#}@-eCDySV{JM2Sj{YLO3vHHO)A=WXKZa%1gP23#4@rj>y0}^YhtXc1nDY_r?R_B zNku4Zv-!Fo^fR6DMEN~Dug)I3L2X%YIPw;{^a60RgE&Vw&$;8P=R zb3*+F-pC2hu7j--N75?_Ez5S2w6)8?K+BQzh@+oU8i}x4u_+?ayt}Ss?%gw5kvU~p zBlQ!8P6oC2Z#uoiuo71L-A{+_;@ekQ8EdtogKn*~{^1=Kt4(r=K~@$YZS9 zH$~OCdj!5v1;1~%r^k5)+j14@73JFCaY}4zC@lzpl;)<81;41;&q?C)>C@6+d!>c? zh>X*D>eJoWYGV@HAe3t5c&qps-O;t8{<)$+dW^%PORYnjv;yMeJY9qta!4_lBySVSSyG^(ReM-g4gLsadOKbsAc%`b7+zmLd^SKrH-Q z&fA$9b{{|)QP$rL(Y!QPZTdcs?AJY?g2^3uwdis$EwWYD?Mq!7lz1$bG0GIgr0*^2 zxEsMOMt+F?sGlc5ZZGOrdqgASr+HSMnK<=~SVgBUP2U<(Hl}dgyKuFb+1Kwttj;v6 zZu-=zqzq3o^%m&Rm_GngRNtl#iZLtbE?fXoap7`6(tW~G;hBmTC(RzArosYM z~OJOc$Fg(aGKopJ;RM zdzs=3S1aXgx4LSHFqC&4w=y&zk2r9?_|;;mfJrz0m>)Gi2_9ab>^=X|`&X?e74M3X zV}ZV4?WL%Vn#=N~N|6FjLw34S^Gkw@6p7`f2km;mU*-l4O;4cr{h|_|gnWf}We}HL zz8v9>Il$h$~6*Xo~YMBjT&F7a*IxPwwXK85~Q=D*w zlGxi@-bW`EI@6mbn3uzllFDQ+qR!I3ckSytAzM1fJ=@e~ki3tTY^23E_6j8+k{ zA8!8H!%$etx_?cRt@6+qmm>t=K7-y=VR@6zgrCHH@>GXKD?4Ko6hMs;hQ6cl5a4)SvYEY-$fiCN&rW zWH&>A8e!dIayUYO12%V{{Pvr#)Y(iP9d8pOySjpN$o0+i3scu&uATActSzRm^d_%m z0V5dl#r{u=@qJJhR;Y90#>LCN*6+C>X4?>p*fci!bhdZ~%pyCwh)b4JV-|*n+71$ohQ;Q2pfJRf;jcyvEiW)WJR!r~ zNcUO^EfY!8+FW;QnS%rR)|!DWwhHn>({fZdEr0dBoKthQe>8KNjTNt0?SoAkvpt@I zP34)H^-s6XrmEbIy}VPjkPl%n{Rhv9nYWziw;a0gA9NmyQ>$Jnr*%cwG|$0nMpxrN zkI867sDN&FDI$+k+20*n7@JeQ-dS|IN`UzYLC=z_b$$)<9MzxU3h(>cZUV(KyR30< z!)uQcQ)n*7-?_~ED?G7jHZ^8Tbu`@i+->g`c{(w5GAR>(vFcOiXg11agpq-3vg%~& zpngMbalsL{^z5pq^Z&)#TgKJ3H2s1&1Pv}B5L|-0LvWV>L4&)yTOhbga0^Z#xVyUq zcXyYKvw>NhbD#G)=id3v%!gTDXx83)t?KUT>Z*TLbpgy(GIgvk0WyDloVt!~Q+9T| zm8G!*UqVUH6j{81YF=!F8^oxmtU)gOlnApCvl6q*^p{ipEW1c zQZJ*i*$HfE`jf-$yOeNfR!Mrf^Cq)p1!vbOaUdX|U!4|0c`z-S+?J_{S03U|nku<1 zta7->L>{sWve%8I)yMs|h*yJ;>@t7!Vmi2fzMN$1)w>|NT9Jq)wO*P&pV_25SIZL;{X}xhiK{Ly3L;f z#ijJqNYQV_y)TF_93q`IC*+<9ScrpQ?=o3S*TKts&bamh5GDlG?zo47@~+M=e6sPx z6Qe+ASWV6ohLx>mHP@B?f%ZV2EIM_O5-_&a%%dr9 z&slh9nF;Pq4rc}iJMg72LwK)VdH>i+Dz3Of6oAkNTU?Sxj-tLYAwl>J0Li-$enD90 z77Jv!hv}GBH)k4o@W3)h`%+j&EI1L+ex%u2#g+fv!$swM@kNR2W}jw5q}}}EQ^9Yn zR?Wm6ReFDni)odLQ*0ETvmX!0sBmwbt$(jqyr^=q zKMw?(R12T1eD_eHK_gLjeS+y6Ockm@DpN!dgu2{xZ0CAW2RoU>%AxnkF5dy z$6G*o<}8_8UW9`u`<_dOsB>W%$iTm~j{=M>!)d7HHu-(~ zqo^(4W$0>A`JGJ2^S@G5bT0(WPlXLs)#6rvf!>M+^zSEPeVJ zUXpD82Eymlem#CakY0r$Kri-_=sj`-f{^mtF-g^Zf5W+h(i(p5FP&>yO3Y+<5D zf8|FLOqOdS7Cv1H2^Fm4SVALUQ4`)~1YuJ+Yk zq9FY&5VK-kCRu~qr5g^D1u?s7v>@OXDV_?Z|kkI-amW+8T&0Cb+u%kjqer|9E zj?lE%g@ND-d<_PPlys6!e+_BJ2iK)87<~akZGhzF*R*QN#COze4xNfmi6vw}e381T z{|HVA$%$%W%>L(ulOCB?HmqQ>FU*w}uhH=JC~V1!4Vt}OD0ViLK$JNU^%6dF2a?qG ziaU-n144g9mLjVbuyM1f!`X-axIlIXmblt5!}`Y+dl)P9m_$d-KLrB*Vw3pJTNRv= zP0Y_mk(+$}CcoI!TODRv`a;QpS5=}nwM$Ame54(^bELBS#i9yYHbBH@-TnMEwxvBJ zS|M@tlnI=P2w?{UGM$FE+ymCt^BqS_ zH_)SJLC;$vHPwtWUvRCh6^HA$NL)5*65w>vKAr+O^)HDUxpFFtc^N6`+*8wLg}al+ zCQ+q>7n$s@H9*l7UTzVaxDLtszDRg7DJ%S zjTI7v;vhgy4tQHc>rlYtQmCsk!Vfg$9^w#Po!!7gaGr`oiD!=XinRy{^Ic%M?V&dbQ+Fypo z4QT|>nPpQN3f^DY1#H)nGU$zm56oziehoK0^P-y%D{2N@bVGkVQ&x)B*OkgDJh^Tt z`u(e8%l`JhQX#*t07ESo)0cDY^6cYjfw+Zb)CS5Q?)n7y!YLc>jPW6jkrRI71^?m5 zi}--TjD+C>)TH+Q zdH;KC&%8vH=aKFsxuC-h6Jc@RoeS!Bjf;$TzL%&HznELwiKAxV$pwLP`x!e^7FqGo z^!Dkb^JSvvW2hnelVbZXm{VcEnp+znF z%X1C65cxp7rL)TOej(bZG z>7rk3J;RY?UOtm%s^+Og89pR~1nOy0R;%++IYWAOafc31aHnP{5${n_0KB&I6vK-@ zf(-PsEi2g94C{IRJidPKeQ(1B67r94_>Kotg7-rSv_<5_&ga|~lT=K8)-(m`iS z>Rq01B}E_@7&_HHS7K=N`|B7k@jm3QtMs2m7@(4@bw=u5UVZW&3E{6~eO-vix`che z?CO~+$#yZ9}1FsPT3euoj;UF8c;IC=wo?#Wj{6vEzHW zHrw@DoPFZoHy6uMlR_kY@%VTBiB5d&Udnui2Jy%A^seqPi%Yqr`2 zWBH&#N7A!^$*odBMJ?#)$KK^>(^bTa3J1Mato>r>dcP1(D)Rq8#2;hJ!UQH2cGuv zDDL?bS(7+rO2rN8)42t1%|7i_aCz281QEFOwjo^uI^b}PvqDV=AyE&CK7qKriyjCKs(gNv|-XFnjEH3wv{F7q>+Sfuu z+gAAi-*Q{Nl8}WP+1?rH>C4`r0HSvc$5-}3*mWem@bhPb%LZQ6g(B!(wK7gA|Dux? zT;zS#*UX)2ueK_-%RY$Jdc(wMYnh@sA>6f{mh64v)Q5u7tjIEaMQaX>%7`KJDJo#Qm2oQPGj`OB0C$8@o#~R}n98F_=GT+O1lW?CshdL8u zuOqKcwv%RrRYq@TViVUrY5;@wJ0RAZGc71^UW$|XB~e54RAa14@GN_0_UJH^xnGXl zP@WS1KuVnead1#`IEcQv_@v10U^A1#f^=@VE+(SVWqGI3>-kx8b(EnaG;rXeI+^!l zKo3H8Sms8scy<7lsW55apZE8l?n)sdWr9nmK0r6y1MCLY}%^elgt z+nk(_pr>%?ty5?vl(Jy`oHffO@?)7z1@JNBL-JC)PAu=&t~C-GA&w9x%QME=rzqp} zN(=1vJUB|%Wc9yF&^Y;w8`Q-M+%oLSOLgwjlK4J)ta|KW4r#B|NKi&r_?4%2q0QA< zM%Sz^-qxKTOZif<;!n&_r|l8#6l0!J@g)z===0Xt{} z!w?P~xvW+X_=qyz%sQLN^rIF>OJ^G*=X_iPVC%9_&NFWnu-x*mdN_hOPhvUxLkM}p z8y(9Gmd}~eC0ycL_1QH%qwDQG6ziNk=$mFFs^9YNFr2orhkAb`HaXFuMP~gTPp>9M$jWzu!&xi6@9@U0srTK$Qow?cPlI9; z-7eLaj_6oh@AJ%yhGSeNYLVbe4hjbDDf;z6Ojn6pm_BS(bzhd>F`G~1mBN84!a$@!z-Dm7_GWd)kYN#2vo8X zv!`$iwnP=>KuKrZ+$t)LaTiwFVj{Yg^9KR=5lc9Y#-( z8lUI4N>_clM7}eeYK)LI+$9F)@vd#OL5Ex!)>((A`b2eA1nnzt8tjg1kzM}-GU9`2GlE(?>W=8eNrTJq;~0XmF{oi`4ZKD z!Is@!Yl-Q@9v&YRMM{c)g)~ z*7=)>#Oxh8@*-h5Jj2q4Ta|Q&xtJPt*={Qcw`d;oq2I?2zGty;beNkD54a<;(a<5F;X-W#~-OF_f}@Gd~}3< z!&`t{l=vJ4q;8n1WPU9bZqROTsVaOnA6{RhOlO+sb|DyK*C>6~<1}WV)@o1i?ysky zi0aU8`{B>R=~hCs>I*S0Hi(Rp`=s}+{9D)Y%=Y$ab}6dad}n7>Ndq-A0Z1Mwgxg>0 z{fv@ak!=4J)(j9j5UzWBkHySCjI`b?aoQGSPowO-8(kd6Y2MYGQNMRVPH|H_L@Mx}0`LmAMkEi7LcVyDepiQ*(D zCAn&@QR;{tAT+GT+CqGr_o0G(6outFJzK8tO^-iXLl6uMmYZmLu*3$%pB2iVX=u^L zPa0C+JtEc-|AOPFeyAKBT{8GA8R4Lu;97KVJsGBrIMg;gjqWm1 zMbJ3X&F)r8jI2LOm}RqBw9~Do(B7VHTc-F)VA zM7?oCN^otrI;eVa`6ZCLFcvuXHPCuoTtFdXaZ5Kq@&jz1)tioFnZ1*5LJ^HB&N_9@ zkD7iI&nn(*NTJHCDcg(EUrxoo!utbDVZdD3ukVB7V-CO&e7;sZ*gM8_?(0gMo54`j zLLy^2kvhmYhDpuyWs*G{u598Z_zLzQ?UCIc1TO}(F=u%DHr?d|Qw3)$?r`^aMRe8> zT28@f^d~pJ`wvh_I9LdOABsB}+SInKzz2gA&p2*`c(G0#j}U zBn}6*rd-NA14lgx6TE*!gS57!LWd1J8w0;j4B9#22LDpv^Lg`Fmqa)|fY2wG(ks?Y zF!1Op1?rAT#i0Fp^l3m@8%aTJ&vWx+$2w;McDD%6in0dlTY5epmyC>G)NrE9;wtO& z3RaEx6dzwGIWN|kQ#xVHpI+cVAu0?@6w_m{M6&m4$nMUkRo&4bG>2mTrX+(i=Lq?# z>hfQROWa#h=QwfBo5){Q;lf-^m5}wKDDA;cCfay%J5-G-9uNwq#!I2g8M;W(xxfyk z@1|@VjsTHacIa#Md>^IZBMa_r4A$pO0}?yfbgog0{P z-%X*dFBp>O^^qiOWU)=`kLKWP?+@bpJE5W@?*h!-V4TZO(LSRZpW7=ta%BxABNyb` z7Tkd-F$%RM9sr$gw!+18Q5@K$z~My;P3a1}>LoSGlt^{wcBH`~CutwuDC}iUNn}N(M(1j!i+8^DHF<%Ve1&YdeS2&0G!FGB2yg zpMsP?YZ)QurgS%d?y@o>Lu9uWDMhI!;o6;d9{k)3wwrj-CxDpQF7=K}50f7%PaYh3 z={2Af`?NErrhaX&|3U|8*LCk9tXdaxC8^9DC?==vB?Twtxth)8#ggxq63sNXG7uA+-@ho=w&HXFJ3(B|80fGgD=KgjD3pNj9)I8RHAH*b zLW}1@9d=8C>!xy4qSt&%-9Gwm1X{OpT<=R6xPkkwq&&l|O>SbPS)O!qWJ%~ijY8c- zeow30aj>kyaRYlwDxyYl`gHJD;QS!cU^-FPnCd$P_0?^xhz-%j94Ta1(5$75hAOLn z@mh1hku6<;+l~pMpr*{&_eYJ(Z7&GEz%-5PB!BrKMx@X=a3V!@e20e6Y}Ra5c|kpu zFN-OzH(Rwg?VN0V{Oz2e(!Q+8=QWCtEC}W!`59ph4`#FE1z!cHv+UZ{+WPYh>7+`%cXPeHm*%n7MWXQm=Qh1tlNtklRcQe!1&aR~csVg)( zi(PtO6pTjSYLhIXTSH-v7NuBZ3G=ZYNJno0T>RP|_m&f4#+^Gd zyDHvyNxQh5f0#*mL5zL@^M7U-(d(OWWgdSdFX6a737AG5uV+Z2x8%gQZWZIZ!%EmS zig&H}u5Qb3^E;f@p~_~N)#Q<4ftP-r&UyS<;T!V~LM5JpSe&9LI$i z?bU}ViyPHzCKNUPu_g>e4qNiQBe-?`x;*WkF30%Lu;`5{t%5E0TYI9HDJIm1Is{nV_-e>I}vet&fPXywVBhATzuyy4ceTf15US$nJkG@$-K>b`DcX&se#>6neR*?lM5oMb&PW}<-to#_C zLBg_>dYJK#3RN`ij6`{wk}ak20*f+7723Sv^Wd4%$0|rG@0j#~4~5K0w)3Ccm!ycT z@v|t59P9`B!L}|WM82FRhi`$(CQ}6vk)N`2S|Y$Lgaz6>0j5O0F+-OuJ}+0{c~KdE z2)-);Qo5?2c>7kNec)S5dM2zX#4N~^38WQirF8?vj^x`tQqo_|aqDOSS^6d~FFMD( z-{JXg`nH8bIAXt)c~BpwR(*JSOWf{t`mJx@ zv!R;(guBa#XGy!%xE0loKil1*AtLQBnk5jb?LB7W<1Uh9*g#HIA({*C#LWFKEA#_= zgCvkg|+r4rX!EYk~Y|4a3Lx!xH)=nwV+3GVd{vNVRq0A z5y>!CKYT<^O6B;Y0#qJd4Jsd9Tl~Ctv31d#pwje>DUP8%{3sr+@+M`DW`EQQI}=Vi znw^;=4*>LFz6+UQ&`9A7&7S2{2ijMooP#p$5|uFPF8vW7-C?bD5J& z62~6RXlPWZ3S(njw2fZr%Ri1%T;iPSUs8{0b;m_n$d7c zLaX5x5Y5o++1!#_LOSw{QG{ZP5gHMKHv+aM#-LM){JfFm-aC?hPS0U3ZFRYUJgxgq z6kLOPqL;?u&bm2WJ?k%@`iFlIE_Ii*HwFt@ad~+)|MFD&p|++gQmThe^fZSo)OW7BYS*ZVFk9mm2|1& zRl&;By!D9FMEZ5Q1+h%lQ1hgkJbUc;pjpTLb@V7~+^q;H!{ve9;sERJNxb)J3Kw&z ze7k0Gl*UXpqIj=njzCu9oE+sWV)6-jA%$THpXfp{kwG0zuDmln4?lnylJaAT4_&t# z52BTmPr#Wx8nLv!@BVqxTyzx_>u^oVLZ_wiI6K&hzhaH17(bpF5W`}9c)>==Vr_t2 ztGl~pm!LR=@VPI_j3Jja4v)7fpMSCrT=h1IBfK%i5f686xlamjWx0=rhF8o^e2L{y zWiDq5baKd(u~|WCT09WyTDASmo5D5?HA~Kr?gN?G63wjw9gDfg>gs^4^*s5>PTX|~ zS_eL`8L9HC9s9w}E?XP>2^5_KnxeLju)P>UsG)KwJ2cSa{sMzCK2eRYkC(%IRs^gp zXL^V3qqfOQ;<83gR+zoP?+eX%ovI^uiqBf2Z&W!-e4Kd&+aK?I+da~cyD)(LBU+Ezotmfg*Qf@*=qpWa>-VEDBJ`tUQ8nHvm5PWqj>aT+e5ihW)?UQTZA zlaDWmuuUKM>LDTkqyq%1vH@rgfuZys{@uC z@B0%UlV(!%BUg-K&8DZcCx&;Qwt^{o`1tS*TOx-jUd&c-Kuh8FPq}w_8f?Op3+BsS z4HEbG|KoNcIuSI(Lyi#eCH;XdVeWv^GKV@UQobGBt}Yo?`!pU73`S$UG#(E{{DPVs z1;wiH$xpi3-BRK2<5+8X#qrL#WXn!kRWJUU>bSiTvovz4Ld zqcy#(Zn^F4vEjOUg0hE#2%fke_S@&WZp(|2rHv8Y$-(=wV=&JkXV8BPINpz~Z-s~GefRTpbjJiJ-St`wGn)E)S-ed&)xdW&y} zb9XVn@K;uS^~~F&_1zn3$B5=Fed@g7RKNX#V=RyKW(ueByiZ2b zPEGlsw9sk%;zm-L?ZA3bymrHUq~V0244)XKp_s2GAX;mY>uc?3T@X`qZNx9qMj*A( zx3zzt7nl$14BbMV=Pz)%B$eHE9Mg&$To}it$aov*O33dsWe+9*CuY7{)33QXUu=lrEN63fbO zN=az!arA`PRqdp}m2l;({YRe5qJ$+CX(YFx2Ex0fDj!>2x^fmSpvqvUL7n22vrk1e zi(c}7oPT{lY=Mi5N2zX~wgBRpj^}#l&bErigX>7|oN6--62rk*^6=-(C6&>Ju?I72 zXi2$=ytSJT_&+|n>vZb=7B1y5*b1^g@31*XySZk1Dtw4@RGw0JNEn>9PxvztSN=e) z3+%dW`eb7#4xZ>=Xpd6WGG1K|xVTvz2{NX3fD~$JwL!5h*M(+-H&aPWLVuSfqz(Qv z3lTYg2Qx;TwraDcTsiI$H|N9ot^WB0Cv!mwS_$)RaH!pGDmJ)bE9r=Pnd7<3g35Xb z*`~o9fY&MCh6emrSQ+##=V~xnpd`vbpJ`e9hO5r>f?oIDm&l-fyC3XwFplTCFW9*E zj5{!MSAG?63S*Xh%#lN!y4=)iZ3jm!L`Dtg<`*RSfm$fn70wB0`2npxI2WL$_gBl~ ze|@?K_rF>qkOZWpD--b$k;-yRw5APVmn^F~?LpJ_n3C@~A!mK7e;e0DsD4a}&8mm@ zgAK>2KFOtDK^^Ph2|Jh@)0=+FD|kPw<1^{9RM{wte~%iBRu}rmj(cF#BXYTP?yg$| zv*Dyz6-?_$gxmJalt$DV=#LGg(6AFy*5h;x6=?|WEv$`pH|^XmgZ3>E$71|0iJT2= zSc!klZR9;ny#Kz!mAZ&~lU~!#&FmkPi--ZJ@c+GHGnD|i4+B>)VY(6%f(ror#`&K0 zEcM`o4(H#*g0qhbttj`AY>%X+7?zi(esZ517(oxQ zPJ9eL0NxBu=7bwhVs}HdwPF@m|XS zsH*@#fY-4F$W3-QKn6}dg3t_LP2_)8jE>@WG(l8dBHHZk!i1vJhP7IXk(2 z$OoF|7hQcKGtqf@_hr?(KMbiqiFf{Ch@b8WY6_V61=-w12L4SOi=LxvgW90UKLEDuB045X zLh=Quyy>423E4lRt`NeGRjABJ9O~4Afgx=e@E4h`05DBzimmIO>rVrh-Lyy>6wQaG z90^;l77$BnjgKNOY>mzpg%@Xh7z;evfT76A(G1%IKuKd5$7KPcB45Y4j5_lP$e_sa;omYZu>!%_#*`$DM2+i zIq)p*+~iS`?-HWQ-lf2!@nbB04OMDd&s(8)j)eha@sXw;(!%SMQ@xV)8f$0IiB5!# zd0ZNjZr??gYtPQ{N3ndJ5hxeU`x-D93;DB4NmgB*Tz^C9uLpGF;Cym(WBakw+c`Gm zr>fxtliMj|1qax0{|Bivli)AW8-%lHrz3#utqWDDXScr3*C)$&vOcRri643>Hi`=fTXeYl@I?!z@}j#kCY0K=hz$BM4E_%FEbjIuAh4j}sfnJyLq`sMAJ)Gjs;o?=6Aj<4Yr8qP zz6~ECj+q$qR`|ER8y^v&1X6La!0b+T#te?QHW-0+NZPKE04hme@2+TKdhUyomN#AY zt%;i`RxB_(Jkn+k&h5ueo-0_myO{3;Ba-J~NOKJA*f};Y*fws9tztWe_1FwJg7r{P zdi9t-t7^cWIk^bGw$mr6*&ncC+Yb>zUK=6_(9H>DTZ6bH62x!e*+S7l5f z|7F9O#S9-=T-JmYV7ON?*lI&NggW)-56EZYU|;?vM;8y7JcBc{DmSq85Xq&#GK_`b zT?R0p-I+{rlu7Xg1(aSok&OfECM;SAmFku1BK929O5kfqf{drRxdZE$97giYcwy)K zfQKf6NLRqS5c(RAcy{{;d4BIT9SL?h>YEQy)P591`LZf13RgQ}$gn^041aspg@F;a zc7|eR2j|zR*I|K#bhl0;1W4WTxuuQuVqo;#M3H~i+0pIB2zXD-*@0htz}~KZYznRY zKI9P|JEE@EqV~<4Zg4+7x$)TmV&1&yci%y2ZoTihNjPjkC)W8oGK7{J%-Dkm!uhq0 zPVS1|(#hfKGYQG}V6;vgqNrYjUcKWdtzx)QReccK7=g5w9BGy@97b7MmW@T@kJ zzN?Ecl#Chq~{@qgTTUL;Cgg^c-=wRo%Lk*}CkR?iGcW zn^8NgY0X~0N%#jkmefCg>_!q}vY#VSZrV?74me(XKin{SWZ8gT0&Z@1ZntjOCiY7U z+sY8t1mR3-_QRjWYy%g&0LJ7!P`+2Vrr0Ye$hxA5FWz$Pj862LX_gHOK^E#3PhZ-( zAuUyzWlwrc{Ujo41o5U1S{0&wjX0rGIeR!NYG5gGqX%)4c zDzP!tj3gRUiNB{ zr4c=ae+S_iyZ&aGP{n(qegF4!IzFHzHc7n7M-~J#3|;gCMp|9bJi#I>^w{U@DlyTA zfFT?}2a9c+e!Nfpw@wKlQd+M&{J*cs@&CJfBb{S@A(9{P{~Jk~t`m{EcG9i3)1{>y zT;5U{?EpmQ{^JUW{X5Vt01D4Z~4Mi{X<@=8o|JPsq8o=rp095!IpH{sI13+V4 zes}`xR~RxGQqv$s%sxCzN4q|n?|Hoj0I0Qu9SXEE|B48TV4iIo$OoK*|GkhuYi=6F zG`7kK_1zmz6$2vM-Xz9JZ0L(=bBsXLbIyEe?*o+@rM+&`iu+qlPG<=WD1 zYnspGF;;DK2KLjH3+LeuC7HUL34*H1%IGeNrzEXlc{tQPUw#-LLj(Icx0`Qu9A+pl^4`&1Xil)D4@_2{N+~l0d9~9+A45H*%(h>N1qoK+8C>tb6Dzn z1b!8p5sgak2xKMR@ny&uX@A+iHFB2+Yh#&%vNk6IMHZTz(0%9C{Vea+<_{YZBU09O z^A!X;CU#kGY`@C%4Y1S+q>bvUJ(7vZKXNO)*b`ZK`8N}Tsbn<p@m0xv&7Jj!91f>lTt45G6`;M79~H; zv)MF|e@mQ4wA_4crTVwq8l6>QI<9Y?uVt;b>9%!@C0gr?;HT*2@9xh{yLmeCkW#&o z3hy;zpSt_hUu^(j9R%?!aBSmw`ZcM*2Pj53|B^>WZvhDEgWXEHI{<9}PIzzDkFT5t znB@cG!}^^$6m;q}$pAuw3E&|OcRzvu8npVODziiq3KLpkXBy;ds~4oF8;CEih;dmK zD`RW5Zqlz(7NO71O&DZTongtkAxa}!oEYMT9o$@pSCtvn45ADsZ_hRg=HG+w$BFB; zD8=$Pf!IN}u%pE%x~nR52mts_`7SpZpDa($Uea{VQ@NMaq_gHUY>i+A{@h?mP9R2;nM&LX)Y>g|oy&{RY|mE9 zc+73?>u3OjB9TG$r2Q}U&inW(h(g1OrGd2xKusNhnAQM7H`?3I#%9+e0Ei5LZz71^ zSJ$mW*I|pr+8_Ov#b5Q|LAIPEBzijH<@2_bb`%QiheeTFx3{-j5#hT#oRe)jGJk#w zQ>uB)GlcgYr!}0ude`{Sq>;z9x;JVa3~9%WIIwcf&FkttHkJ-W$N=DiC1<{=S(K)Z zYt~5a;+^=D!2teo(a85*Cv*OR^?%37QpEl_CFuAtYyo{AVt8Dj>@P$hfuVncy^{~D zN^_JEzug^$vNYakXm@TPK820Z@$+h=u{>{>*%H!eT(Klm|AD!sa{eiK1uvBn~lJ7 zE;xe7@l*7TV1ut(^bPKR8R3BFrMPW-dHO0|z?j7u%y9RA2gwsaj%G9O1J1xwXY+VK z>=C~#IQRkJndq4GPfD;2vHHmRjZ;H7=xG<9<<*ysQVsdXZdyv&#(1v3E;%h4z?uGW zO}=>zqNWGLxox?Yh?u*^(XDkm4)+n8$AxMmQ*ZD(|D?U2czD6G^uWpUPq_;c0GlgJ z`Sf3!ffoKq1fT_85&u5|X#dh3Bqi7af=qqAe0SY64CYX=n8RjcU>RPRU6N} zG$H}T(UyhgAB2*U2E_fPIBtHlRbdMJ*W+N)6S@9H5%AA}=_}sy@9Xb-_(EM>xd_oA zeP(R1Y+nO92Vxoki*VMj0O>6ezKFUTIh>(h2^|0($XY?+$VduJ!G=VH_2K~dlOBRu z6OIhR1oK5Qed6o)*~z06wea9X^nAxjW)*V`Hks0(+YhyXAAYgK3xasb6cOnT1%7OV z3O?iJ@WCUQ{o_|?i4^{U)B-_VdEIyX6@T~+^PH}#3lnu1-tpNRo$D7Pl+3l*dQlq! z;SBD#{0CAymH&R&S<09bJP!a+kmZa(64krcKH>|dz8mua-$AD{_4CrUbri^exE~<{ z1UvR3hyyzP#)-c>auA$9KHWhtD_wHF!&bkDc|ew7eJdic3mTmZHf2ECgKf z+~P*a_verZ&?eRR(f#k&oy>j6diQ#9f%Fi-E;eJ?xORg|WdKg=@7ufGM6+WY)nhVX zBe?Jgup2W$5-6uy&#=#(fPlkn`;cC}VAXeY4F^s~fg^*kfYe-o)PIaF&>e|uF}>oyhaTuh4DjJtik9x(|Ja7gRIG^a>F|}vw7V|jLOmXkrNhu%G&UL+{{o}!vVlfnSO@r`mmt<{x!kzzt@<}-!7W* zS7*}y5fc5k88O>Urd6rX9H9I3C|r;w7&gFEM8Db%6xQqrOPY=MM6P(X_cX*zfXcxR zGhEa>6@j%c_gO8e-gvlv&RBDT*Ki8sRrM>e-S93IC$=x6RWl1*JR0s@7VR-UMF3vCS6|uGIl`gN3UMA` z*gvOpKf;;bXm0gr@;cw7?6GF}%+R z_0!PX6Nvn1zTu$mFtdRfBtjgiju=V#0-w-F zy9L<25`m9B-bD%W)$pG^Yrr`f|C_m)n)`pT_SRu_bji0c5<>70+zA?jyGw$5fM7um z?(TjF1PksC2@XMmySsaECpaA3!qI}>Jp_ulV&p4

r;pIRdso$}TB~-a-R=vO zmIuP#*uAJ0`Du1VhCS#Wxg}tGx`W|9lpuPdZL#&u_Lij`&-?6#FNwuJaGsLl&H$zy zr?I;(d`zv>1eKKp9EbLRE;V4efy7`RJKH-~nut{%eK}m2iyL)Jh&CO9wCNnL7!li5 z@&K{!wfEDj{K5ee{{VF#MyJxuAK=WnM`dly0|6g0i~Hzp&ow5xiWUnk;}ZNfKj$PE zGFcbU6uU~gdlpveTFUYyteP@nj+ls-RU6yrfEYT5$bdbjH9t-l%at41ccFU5=vOy( z#U?`$ThCVOW20(~P}~VO)9kUVHd41~=mzcBz|7urtq1Y~RKtP#63s7?LKSgUu9xf? ze_^Ow^qcG6#A4JIdU5+*5#U!zwUTDc6(uZm)9`#hz_UBM)o(jw_F$a;m8CGEMnGjw zy1bpWqJOzcg0V69>PZrrMKppcTi;2nd%ns*C%CEBvQYiu?5cT-%@|4`PUx)wg9*_BcKB~~ z+oeB9L$9Q(R`b|j?Bh${^0!}&?*0C#O2G`4r(-P9>Z*|#R_hj5l1OR93z$2Z=Gj8f zPP{>`eprjoqPr^GuIV|`DSQ5k?U&ey+OP^0<@m#F7uJ*LA>sbby9=dwTD_y|l>M}h zKTf;4`~7vIedBcp^yAb--RpZhfTT-FT-xpiP|YR^&mX9ukcj-ty!PD)%iV{ z6H!eVdUZv*Sk{|*`Pk=WT6cyo>k1Qj21-M2hN%N@kLhy?=VcSC*durI>^z}cG2GV# zK7PHiu^03PmQ(q%9Ms62TD8(?*XD zCA$(LdSV1x*<;uv*f=C%fEP?>PPA3pWI}zz+mLJtc8lwX-Cj?HD)cVF)3C#@j~cG4 zV+zJ%Obdpy+oX_Zf z5H6wZcvw&cLWj&?8L&Z3Oin5+-WBIW1J7fsIe(DXn50r&sq}-uegN(ZFonLRiemDHbEY;Q11h2$ zbQ^2TNq0i}G&I1=0}fKapYfp04$6bvH{ZTCUlU|AYU2+Uq*Vr&f)x0x7M-<6FMaOZ zOH3O6ZSg(7^P4AfJ*rjb;Zzf41|R4zOTl+{Jd>1U$NN)faS-j(TXbZpz8_GIBPPzk z1fHm*6zo*dgDW=N{lJ zX5RQYZ+orqbMD@e4a_QEATf^o`$GPCUGBhj6(=8gjaF))@jUmdkOY*7B78E?gsWb^ z{?-COLpPgvd;o#uE3e{mqCej^OM9$Pu8+%t{!P>a?MV8wV>eepiW3OG`QgFIADivjtg~lJetK) z1wY<4)-edl%Cg5|<>r@^ob1#&Iy!!=eVb=n+wk~CWpH2L_{7wdJMax2&f)?ovx3Hl zlpk8LAC+U=Jr_Pz6R0Y~u^*U~l=(RyJZcDDBYe?r1%WPovqrbQH4Ha;{{FPNn z4N@^s|AiFW?}5P0a*jcP9ygASzw2@M!Owi7j^7Gt02-F3TLll?e_%(zC=+1-mhb6S zBm)h|I-YEiDmnk15z;CNJx#y>?k|?+zq!9YkGVQ1bad!a?PE+nQ_5@q;`u6GcwPNJAd$ zdozO@mDVRF?=i*JYQCuvYu&J`OTP*-<1u*|d-x3L;S#Rij&JH<_8!kXzG)~6`d4s? zxd?iC!@(p2eruU&K~ao6Gc>XK4?`y)2a2UU>MPs@MM`(cl6wWWy=z%PZLngW5Oi=s}R5J0_YtVW{^I0)4u037hT zS_wI+k=kG1pC0%+BYC?ytAs4+8U}CC1-BN`6UlcbLze2)BswNf#pFKfUMQ{6`}K+j zNYxN`!%p?aSTO>bIm-h}@K~LyiacVkXgQ+e<2&mNn^%UzUg#ZcwfX%B(oAk)=IEQ+ zv*t({?eVM$IcC2>Ak;t6GkjvIxInkcl?MPEAMTOec(K}o2cw?D~HGXsN z&W|?}YPVPkQHr=jhN@p-IUokyO`PR}vOa@lZ~>sBoX)<4MQ&1v63zbd&|wTnzK+5X zPK<({o*OD~Zm9F=T#=Wp?Y$2hGO8heC0krS83VG`w<##)$!T2iSK=7HXw*f-@a-o} z6**0FD;nwS8}UOzKF{F4qmEB4LHe3hJLbp#Kq{&h^uoN@ugld(&y~rLrOMa*Qs+lw z;{HdG;va75jSkKl-PY+_d_zO(T2pC4l@>H`vQNoH8P68*I57^|!PC$KNcX zL(5&5^;t}yX)~e8A4sUicF)x$1nrAe)48A?GnCJsX$Sa{z2iNLX{|uoHPuk{P&Dd* z=4jjDh4_MhHzZgwAT?Ty(9;qg>op4>n(oJ?;q-dBI6QBWcmL2Ie5j2v*R zP;O^IlmNm9Qa{Zby$!W4O8FKi0}P14jBP5$2$Z-s_DmSUt%(QO(TsGDQwq1S28`*+qDw#64icbm(c8aw)2%gOU-k3GXsp zbjC1b@Lf_S$8n;!p?ro2oE>Os7Z+r1lgri4x}ghQAkMcmH<$|nvg@VPR3pPZhTGAm z>+CoUhwK=FuGpEOvID8OhF9qpsPKkQgck(-13-g$b@ z`GH751Wft(cq0M18cLsS#)fx5GVz$&3A%6r&fPo5rYpm>yu&up7mh6Xo69ZroS50GtB_v4>m4RyORlx)ryvkq$&Rn+u_4|walB7 zAH&os_I;(V{SXA@yNeg()_Ds4Oo#4s0{^YJIPCbqnLQO53ALs20X$p4QHxkY??K|Nw`rSk)1k0Omoe?yuzjlxK#|!X zZ}-f+#c{>F7yp}>r^%MGvNS9;3EETr7+!Rds>;@+^RP&Z|Cbj0AJfjxb%*hcW@#VG zxmB4;=>&}A^}oSGK&Veu?#E%bw2X@3D_WV>^UkrtC*#b9!Z>4hT;4=tKQIdTXm#e$ za~O`$4wPBEzv{;V>9?kbl^|$%{^=B_e~ja%$i`3>n$7-t#tP$W<$^f1+aD(tfvlu= zRepbT0~SBD;6#cPa-7Sdew)vm@9F&X1{(@kYI#=>!?B1ADQl?71&|8L=zdOj88+D{Rr zAUt@l^kJ>33gOO!S^f5?k^LFxo2)-j7w~QRF<=Po#+gK)D*>P1AN>j}hXFNFlh6O@ zou?yxPS3J=y8Nef^v;@WP&7z~x_Oy3O`fURQPff%^aP2Pe1H#iqIxXYl)v!KS~rCP zFa)mmV*p%C8#Ss7s>p}J5J*KfvkPYAo)Dx$R+^``K7jT{%39aq4feB9B&EFEDnNAY-|UQbrZ(lhv@AP&_F#~L4#i!KTP zC?3e5dB5LKmp~zI(@v?j$@{UwqV1kv+q?i&^7y)aIMcp?Hc#XeOV0y4XUXF(Bgxe- z>{v>?93lwLiyY!p9hGj%4ty}4D+s5p=jW4`Zm5>-XA-b|8|99ZEMZuSRZzf5Vd2pz zbtLxRAJPrME&Y8dk)-#1)1xtb(BU zT0>qLqZ3S(=&oI}$8ZHF4YA+5&lCzvE|^)HcrIPqMidUxx*EO*7e3b!VRq}aaw?L2 zO_g`_9(MPTI}37~R5^|j7?L_ruvena%Z|Nv%kEJW;&OsySJg1|a@UXYO9#dAZQ1a2 ztTDBqF{e3BHd9tNTzQ~oG=P-1xxRB^k>D(+l*LBdKXmohg4?+bX@Mgnw(qTdsL!l+ zSv366FG#YQTvfO-0kkU(GCtVbOlW2c-D~sb@*j=1Ee9DZYVyv*vz6=)2S^71n-PppMKtpJ0>1TRY@i!D;l7Qaq3X~R2=|S*e_{3( zt*SQ}mi&?uGAj#=yv%y%B?$s#J8dm+Iz()Ke)$--o~LZ`jDs16Eh-!y+ZmglnStDp zMlEud&+yOv6n`4B*2|}Wttf^WKecRqL*5YNh&mE{pbz8CIRb+cy=ATUdL3T@Gt2vR zWQxpPRKYOAUX;R_`fF;XW(pk%iqGw2bKlxj*3AujzVvo*-fGt!FOs{i8W=IUgio59 z7OEW*N(x@1myhqKL2o?k3T;F(7k;2@pG~(N1}N!`g}qq_g3bQ52W!8PJlCc(zd_Hg z0o#{`vPGG!r9ezYRSPQeZn^t=UxRF*XW#n-M3@nn&K3Z=J3=Z0vZsx`TVTOEBjz@z z720rJg5Hqq9B5 z$eSi^G@!fvmF-uNcH4YRNcqJ4s#~iw3nzxga zVcdU24P9S5g&CfnR;yR8Aj#(yx5&wz8y9M^1lT&etup#sviz_^jk@>n+2qL-WF13` zO20r*TEjPDDeMru<}j~5pfYYwy}biPA9crO$)yW5mn%~KD)%a8(UvukIDR$eJlnez z-IW#0$N5J*h^+cXFE{%cNQB|f{{d_u1MZQpj1`n#thF@F=P(CvH1+`V4M1pe<43fc zZIR`Plj2fA_%%@^s9~Kcwg;N8ZKEYUPb3GD(|l&WFWcdf~icNAN|Y%WgQ+dc2yJ;*M|d#Ql?&XeGfR+-`(= z!-Edy>ul7%TWAohxh7djgM!~$uKl#)7EF6z#HMN~U%s&tUNIsQkNBwdN5tkG%ir5w zrQW*I>!H7;aTnGK>rhw~oWl&d#&qY)9azE{`!aq*Z0rlTZ?CA%Z{5MGgF3+^l1mYE zWduG3ud6AUI(~i77@`swWVCcGP%+ja;48={_cQNyZ@5u7fGf!lE9~Usy-V+K9-wxg zvl`}Ln4;}|IT0;sUA!8*g5oKmnaw4Kt5(Fn%~?QwcOd!gCR(z=eec!DWl~W3wLrhp zHD}C5M&X3aL~46rojtA~zRvHX7|K0vXypS|LDkfQOB%*-h`+bD3G{;vwcCC2#`goq3cz zE67M$yfBV2Vg`GFR-b#M=L(u_t?LqW`nD7BT;eK(NVwM0q^$IK1h0<@;|ba$%!orH zU(2R-wwDe%*VtGjv|R*b|I9wh-)t5i^L~>d-ZTWdF#5UD??08Z_3XgZ3`uX~1pZ>x zbZl8~I5%BkB{LC#QSk>|a9@z!qVh8d>YOH0uWAv4cI7)(1`8!T!*X+*J2%CLk?{ZIekzy& z4&n0+2K6f-g}GWv*)~8o3NqP>i4;~mb^yw60GApHa9VBjS`t!ePG?BhN!u68m}1V1 zWbW<^*_CPS_rIPQ(Rn40s_>?Jr8vrQ!9t0ng0D~09j;s7tk%0MiXjIP_j)ojoLkKH zMfSUtk0c#FlSkpaJc0a*$Zd{@+id}Gy|eBWcQ?#I6P&l4{qvVL7xWW6ZS;FgVQhRi}RYlXFyNXUj5gQNR zcdH1fo%rR}hmi+|3rvSl6X?D?u|{w}J0=7j?O~>+@%XK;Z>lv`V`@4z%|y-;yzjtd^NPG*8nZJBu~5g=(!dzq zhP=G){a3as1MLT>Kl#y~N&xC|*4Rjkl&8Z2VCV%eM6L9MYt(bK#|cyGo~+5pqStg! zO5o<3uO*VAT196V((w5UvG8XWykj$65@~Mxc*i*nvK!psR!ynJIESXR!)F(Fq+jVt zutqNy6;9~}mwoAY?|nBS?(3<4Ub%HyCEw{Qj-k3O8l19Yu-;QHCNQUrp(&|V%q(_U z`%AeS{)>lR#QKYeuB>`%GEKHB6^pe;b&>}p?g*y@M-&$#3CuFjfAJ2vEig08ziW5o z3V`WOTw2ZmuiJIR22Mh)#l5L>{sPneu8ehGq!X3A2~5U%0P)dU^YojYe%YBJ~@(aBrTEGC`wcfeR*ch9)%G$77`^IlhEp<4s1$*K0 zMBiv+;&+zd{NaefOZ4q4wk%`1>O|cO_4qneacmmDbeZoF&@5YLi(4YR!pnki!!&4<|+rujA%d?x|qvK%s#N(3+`}dCx4s{5_uS*4b~?9G$qn?M=Az@q1yUp*$;~;h%rm##GoLuhrJpTV1xf z(4BZQx4<=f87g5a_~thg$A&qFDUMkJ`Rq5L#lH#djN9K@04wxAMVn4I}AwwddI6ssuF z_8oe{-Eipr$M}>Bd&)mj%47OW0Ng|Z9u0uP*{bPkK>ec>C}wLXUAZW6Wda8PrlI*b zXn~;s4PrExGiWhWmN-y2&$<#)Ui;Rz_PYfHlEClGj|>jt%%O0pf;GX{r*;>{Rvsm< zg*1t*@{$k5Ax|KNrSZi>)7c}++YMG=q7DEpo8psa>2)MY>dGpB%4y-n@lsk?y+ zaBUXd?gNB^G4*!h!)hm)t*V74T0`4LV@ru)Zx zGnstloo7o%DsQlbryOIS`*c*biy*aBJ&cM9a#iW9cD9A{QNh zWRFSJi^m>3mj304W&jj9RFzWxo-DFG;b;}3!`bRwF?{x|lsu>QijI~PIKxr_$E!nL zSNMCK;U5;%6DkVSg+6P0>LFA+K$FtA@^lWZ0a650L?dtGo9LTv84556RX6ShP}j%g zEzwDird@4ox?(q)%eXp=slMtIjB;Ua)cx^FbGlq`<75nUbQr@`a18Fvb)|o(ZPF!Z z521m5bGvNomFgPVT!&rzJa!l9McOpM#dpQ|fc_++wLgUtm1P*U7IR(^lKgHk3VabP zp#vo#ec&k*UN=cW;5{rOnXT}5WEZFYSy#DRb4A~M-)+nNDv}}RU4j#n&mbdD(kh4? zB}Yx3B!&E>{(?2?45fggR2v2q$1&bL%vo9D?(59W##&69!Xsz6&G%jLvloP~>j^?v z4GCQ|omSxQ(VPtsvF}hS=2GdL(NjX&XBsX?^QFPEGC2NdeaD<9+e6gb@j6{`O#Y+L zX+?^NEF89_{x6GtVktU{8cvg@209~ClV_BYgc|Td-pMn0zQ<-&{hd}`6bOvNaJsy% z2t^2SqL9h4%i_$;ZHV$DLxK0pZ_$-V=u~|6a0^m0CD>J8nFmD90+$Wscxat+%3+x5 zC=Vy*`V)A1ke7leyUZdyv;1-uG&&;5bD>yua;zONUndi?J!t$X_O{bk0IDgE)COQ_ zTtEu(Ldc>07uAt(fIWCs88}ht?@0T|JzMVB=U|f<^7fXv@%e~IU=`aWo6Lcg^lc2sI81B7=eRk_o@GrJxqTp@ye-MGNDDIZ*~SMv$f|P}fR>mb&o` zx;P9F1?BNJw#7aNH#8gBW{rs|i|Pw?$T-gxeZd9FA~{T+iVs~{u`iecUg^M0MbdFt zSi8WaVkEFxLH#0SWzNhw8as&&3f~Ue=U@#3-UMgDvWk1MR zq&>XG^N#V#23X~zb_@uzSu1DU2FoI19f_?|q(%eKGEJbrwE(Pq93x!T6`w7~Sjvu@ zsPs=r65?ZQa(Klf@W_Aa-fk-r=6?QU!|oLx`sVsP^wqUK%oIf1=^6Z14rE;>S zUl*y^UkCn7U%Ocp)#yN*_Ij|l@$gWptbmZA8)wROnbD(HYby7-RScQ>c=)mt&49)xK5D1wS_iC!Z4-lR$gg)!D92<+DN=R zavlR=xvhycl=I<)ods6xxa*iQCLNm#&Bqa$%)(oekcJ19H>Ery#(g*%yk;=h1}8-Qf|Tg^u~k@!}CD-DE{Qw zm!Dhky=|4x>88tdHk-WAkjb|BShQrYz)I{2P^8g>pDC-2c^!1eY!jK9Bys#z^C#u; zF>os8*Tgk8DNyE{tg-7w2`%K-MhgCkm&Q|NZ@JG4gZ2y4j`cT;>H^d2Lj1-tlA0b2 z#rJhW$vSW}U1<}qvU2;Sd%mgPt~jTq|0o;a=3{fDgqywkSCP(&bR{k@XMelB2z?zK zEnRwY$vrm(yXhcnO>`E29RoT4TEF}@I{&IUi|4$g8I)TGJm%=GZt-{_I8D&!oun() zW69wkOu#}`bW|=op$4*x)iD`o(Z7g*!D_y|`JL^E6%q`8fFgv#>V6B5OtqKuUQKDW z);eK@loa7ZtF6YMl~MG&ATtm7MXj%yhotTtMJv9OyEFC`D+|`uzI&9ciT!lM#OzQS z#;}>^q#D?beaR%m>*vN9Y6+OZx@rgrJe+SoSUx1+Y+A3jt|QEkH&bs!X*yZR8lUwo zMY>}0%~3RV2LF1Smc;edi+=$39mDb1lF_{nJC}dW&p}y=zA4g-l|iuTJ08aSiPi(# zAY{Ch7OiS=bVVa-$PX=p`)&Tkr4;Ezz>@nSZ-~T1_Kw<($cN37{dv%sc%*O$CAF* zMys^*eI8ejc!^AFUP8NmyMF0&Xlkc&w*wggwn4oQ8Jqp4Y*zA05ko)L9(osh1%J0H zv~Nwdhg90PYg)n2G^Y;EgN)bM=^YThcjFsH*fr2E{ z7JS;F*e*Pvcj+K_)Jq~&i&WgJP?2%IY<|%HJj-JiUR6^vq|EGy1QU(M{;=DSvD+j| z#P&&}e#^neDLxcKBk=~Jq?QWKFVN9zZB2tQ&}&TrKEnn!r$^`33L(RHsnT#_$9yM`ce~AV3-eK2qm<=Q55Tyy zi|v7$lfp&?)ZC*3NK=b#KK=W@hf_r4vFN>g&y3+}yckS>|9zi4T+m*QiTr`U(i&JN z1&|B?>_{w+acmo=oueiyVi{-YYP@yn%Fnt#&(|DTn{ax{ZcA_UmV5CnAOLLmhnUFJ z(U5mh&j3!qGgh-6j zHC`yXae@8B55V&RxM~Ddfe%DFwdEEeJ)xOu@c;s%_>Bd+RS@X|fgh`g$SjiMQzq)j z2Pj0?JtPFlh6ksLHn5!>8FhD|5u>(z_T<*imj#qj>HndO(tc^hU*6bT@H=q1HfKGR)=~AB%YB9V9Mns*5KO#f*PI_kdgpWQwu41C1Amgwt35cXG*x4@MTFb{du|LCH9c{C0V$my+$ycsmrtM51D4a+AR)NAWPAw7 zr4ndc!nyaKH^0OG`|2P6_Q;Ap(x|2hWWJB*aBk9hmiV(vc3*)r$Lbw$(0ahzA0*CW zm%<3XB5Yo8JdSENmm^voaeP6Q;2y5CKBA8aG28X_Cs9Di+}1jR>u=)m3MyqMyg6>3 z21D2mhz~(~!(8t*jHUma4eMRs0VxGwmfxn;p9v`*vpuRkkc9vlI$$X#<+CEQAH5L( za+AHbGoEngpV6xFFz@pH|3#!H*DcKGm{H6B!{5|&jBY|+a<2F zy-DvVXZ)1p44!1-!Pg&4iD^m-Is&m3xb4vjGk1atV+orGPA7|5#>& z%Dvb29)~=P4DNN`%L#80xP2Lpw={Sd#eegu57O5ZUW+@xkFiDcQK+k%pqKv>EYJk9 z5{>I7$QM&H#M={8gC)I>z#`3evO0c5=IH*BXFn#%D!0Cq@lmFN;{|p z+K1!7?;j>8jEaZOo>4G36Mm>Vp(noDesx?h2r&JaYiS9cnZwO;GoMLV!Q$I~?6~}{ z%aa$bP1dNi8}|wF;fsBq@{I#0*3Um&_mo?36$biiHa^-7a?rZLPJ9hoyp~PVPWk)! z0IA1xw^F>HaaM9oOiAhR^$GFA;lO9D^zzL>5=H^Om}!QG!BX=07o;C!Q!4qt2Jd`2 zykKuJ-0T-Lw%cqUcX{pjtQI^cdB6!O6?}M^Jt(t1b)<42fOm}rQn($f=V_AG^QR@ZD{@~>m*x+^s=@;>9 zxKtKBngQ==i|gGTc}G3(`IAa0kQSYdZiPDFD}Bd+_?P>+A^~ zi4F*u(t#3D@_Srh+CMheO<_0511&fGXvQhs%4XEo0jUj)&Waz7;JspjZaD*aU9D}y zFkZXGF9(WxMr)^6KZs1Ep)c8`G7Eplx zuOz@RDKr^@fd(M4e^HMb^@t{HxnsJUmEHgicWnu>pX_fgd zHF1PDQsQVE*8Y~-xwn?jed#?LH}yoAVj4D_9nf=D0pJxSiM9KVpee@#08tQ5$4dj` zKd?980!+nk&t2Z*KQ`(O>si6q3tlDgk%zs$BV#HyOwrAL;J?u0kL;{!z^(#uNA~Z; zDb;dI#0|U{xXYvFPU3DKe>S#oZz07JJYgnWNE!#mQ*P~c!~VX^i~&hy&pT~tv;O-g3d)w; zMX}fPLtWfJ397=5&xi!U0t|z3D!v>J$!5i9s?|nU7&cMRb*u2ayDCmWYIi=u*87i} zVa5#1Jh}+GFD5={+2Y^wyqcasUK?|~fph2>{wUXJ`Ur*p*y81(Lpf}mB*?uE56#mc z_aei5kG$5`{}Zn@xU0guM@Yl^)2Uqhs3rosTw=KNQ5+;_&Dl@FTE4%~X+cxd@Y8Qp zpCVjNvZZu|pJSc&+iTU~e{)yo70Qo&fzZxN?}1|Ob_$>q2NXdizowJsu_nJPGN&D( zQNNmNMACw|1R|_GZ@H)6yP9Hu_Hw}au%+Dkq`cHw_Z;@N&PKxl(+jJcrU=4c}3L(!c6Stm3*k?+#hEll)#e z5~S!W@sILVUyl*B9j*zeDR&pY^{IQ`MtV?oe>Iz}KZLGmVqTM;c8=lTmzlbHT+)VBQjwyVk?V}&9Hz!GL-ALfmS+C`#=LAk7n!gKe%;-Hev zsU*xmi3gGg;CXPuB)Ee=NE*Yc5x8o@9Urcvgz?l}2B!U%-DR#Mc0ap;!rF$cn8&THIuZ$%{odLV(tXg%XaKI@odTC*U@EO~85s+n zUDTD?FG3a)*O2ac{mzBKNt<9|B5lk13z;0WZnS6w?g!7Gzfp0M`*QB&oq5prJC zk^T|xiTXVs=qcHGcai3a$9=B;9%pF9X76*Ke4bFj-7|9)zfp!f@_iqkwt`j9eHdQo zmvuHyqmE)>o^kuh?ds)k3&p13n6#bw={-^i<@w-ZK%$wawpW_XcO>p@Q|PJCKfnH# zj<WdaeLggfSRV0~edsUSQgv(NR@K?M5(&rPMMYpbpzi|xN+ z$MCJJkrI}ZqRP#Iv;pHzNRs~0Ebm*1j@N9IR(+=!hu0iu2q`pF#|o>oPU`FTON`yO zJ~=*r#cMwncxyrt>6gAXRd%{iX;>BQXn|!<6>}*_XLLOKD={1GYkJ4xh)*-Q@MYEd z)lvG%Icqu&I6!|_oyU`3y1(uhqhRUkUAmfria3j2;4O%y+Tn(L%HleuBjk1|v%zZ( zLt=^``4CJ{9aZ4*WQ~Q7zA*SW>b*8y=YuPivrbCV_OozK&z7%SNnY37JI z6%9Yy^sBecv-6*TQ9YHkOPyhBXz!aLu@v%>$SkxGYv(Oo=P9=pO>dZ9h~bGAU5&Hm zcGzv7Uj+05&MF@#dwk8Uh&xcYHLlF2Y-5K1Otd3+J_xOmnc+;A%C+GEcX(`KjLRIVF8c8fEyYRHQoe zX68OibulA98xOc1nsIXQa}2n0V{(fW?N3fZ?bDs{GBh3Wg6*SfufGc6~$>Jlw!r()tc^0}XF2y~kf}N=vhr%rk z?Qt}Uu4PFZ2(7XkR{paf!Jze6Ezr!{dB#@J(Q8QMu(FqtEelwEmChWYcZL~0nQE5* z!MfpZp8)8xK5cSp`^?aRP>v9yq-C#AYAC=)SHB9V+s+aD-&Q?|5h8uxy}Jx^m{OpB z8X4l|pJRxoOW)Jmb#f;QTP0yQ2S=uW`cO?U?L^bRY!z_sTQNr^=ZHG}nNW*ld7$jn z!zwys_PnH_?{>oHWJH@!cG*xJ%Y!?Za$5l&|0vfQ^>oQeGD8Kx<<;t~&kjeS)MyZ_ ztZe!vix0AxxvJN$=WzGRafue+S?oI~KKVxs(!e0}U{Owasr*M1YVFqlu;2>O|IkVo z_TQ~^irr6hg-7(7um9i0nF&B?^W=1Z7dr|6QO^QqzT^MU>BsXwbo%}OI=y^&;^*YG zwcKY5Pa*|+t;FNW=fkVdFtD)7O*wNuIm2}_1z-|orE!fZ+NKTWL9G5PzXbM(>f}LC zfmxM5lp+b=GC%E;JEaU|V3e{_AhvimL;pAs|Jw3l5lwEx93V6Ck^1TM&R`%G&2jpb z8!1nV?!^X9WV0HNaobZ7p`npOsjD;0RzWm}8=o$6H=E-v#$JK6*K_tcb%%l!Fn@CL zZh`)!Slz5y`50FjC27{Wi=F=A(i?6<_iufc$sX3LO{3t|)_S**@3EXYA(*R~PDkxH z5Eif{p21KUg)*KKKqk1NlOIk%YDO_u$qGWf2whVc zr}4Uc4d9SZm+zna3o+E>FQ20#@M*l#)$+LGUXVP+bpWMIDpwTF;!TuqF8w^aCp(h(vyt;wIE$GE zZ+DGECapPNYU2|9{(BLxuyFnUYH62);NMhhmFaG2Lm(@S-88JYbvn;?e2#RjH@LyX&2XyH#rYKlSfL4PUcPbXKrA?3h4RFoSgIX^E>r8j}tjZpqo3i6sQ2zj&soh*ZTODkbY`l52Y*? zTS5NMHc?BJu{VWj3bBULsNKZNKy*?r$*_GJS%X=#+a$$>y9|&B)-(6@%yf$%b;7{ZaCG-Ea_zi zqi~{ubN!%^eNk<24t)2fyn)0e&Fta>S&85UakLlFKuo1vT}awcKfe~8XTG2`+7-)I z?G)~>zK&MH;~yR}4_`JltX47+lmfXXqi(wHLN}N02xv03*8@Oi$Gcx89BT-Wap(C> zbZm0p!CqV+garmlr1tj19^3Wy_EPcE2B)Od(UWf;ShXy#e6FvDp2y#EO#+PO7uyh^ z<(f#HpC_~{x9=J1C8O4$92FH|U`I63^p1KR`tb&#${MwKxeD=z@hxF}I$$?N$F6?L zY;RE<4G5P{@|6w~Z*kI)7W(?PgG9uwUhA#BZMwg}e#JT@*g25_f8lhYJM4p|j!zsA z%_WdZ&_yDgzC2V+f8|Bu9ev3zeR)w_bK4?*9U$YeE+R}L+eEoB zc-WmbWlN^V68wvZl#)Q;p#lElYzVCuSL1Z~fIxR{z*!wXGB6r;Abm1~;rm+6%|_hq zp`~6d7ze$U2?d|6MTbq7$;G4!cp8Hy`usdp^>bW&Tw6|;h}3tjqN7f|IIDe{(s!j$b+c zh-#wY_LW6fN&y7JE%RHMlW_Eh%Vw|gKpl@aL*otWUMaPU+jTqN-I&!qvng^pnKX&D zg%i;ra_+P}9k-wdG?#pr?YwS=C6aZ>jdzMt*6pVT5+zn1v^#=K__J^Al{pzz87M8v z0%{GdUTle{K1{j0m(Z*7KAbwomO+roCWrLW*gICK^N!v$Y=Hy`x~8ze37+cbRxY@* zxO^dJ3s}I?oIw2A@DB(Gdd}9e&7)z?7(NdSje$jAeSmmT1<>kaM>F=9Z5(&2kz0B9 zJH|E4H(Vb?1qOcn3}DZ1?!!$Xgg12Gkv2b?sz(*P(BjIS^~__~w6?VXvB&TK*f=(u zd)-iJLxm|JG9nmOQ6Vg`>y6Tr{@4VVh;g^Io7W8Dbpzd(u%OXt*gcUKnQt z&fDKGO}_W$C`2T#nJs}vFze%udys@|%}S*rJL%tA09}hb8vqZ#SUydmh2n@u1qTFu4`#GR+L)+~E&O8NUkAk?2XHPQa6H#&UOAuCdn6qMZoKo9L3KH!O4LP+J& z7~SKxsqW~}XwXET>^SV0(;4m}w?HX2gR%}L3O47727BAf`@6o*?b^0; zO(B`gHEg*hG?>m+Q5uN`itNPHw-TW}l1(CCagn=j&vwmw78bl)zcD|aJVj{vqn~PT zZx5{hUY1x*Ms^G=mGh4%qh`E<>45 zq2`B&ErWwa*lxfVfLZ z4n4Ug|)% zgS$fr?hq`&El6;85AN>n{xxLp^FDi@bDi(U=^t}72~77|t7@&PyQ=PTKi>~wwUK%` z?|RONr2UzC+{1^HzHKKvq^m>U?3M)*e+SnU)>04bXzosuYfd?qw}XaOk;0YL$|b9S zMsF&G+dvdqUjrKc?d0UusSKb0<@@ni;m$8v!3}^+ZT{=TI{}UX0mzL{muh#xp8Vac z1K(lciBEJBHA!u=2e9XTDLy_ev3azWNfB_z{RkJn=+9ZustS1Fjdh;_W$azPAfCP( zO?>P(i0e$)vh#fD$+EZUdj&e5=Z&IQ*49pIB>)~~XamWDN3TTcno{5U3QUHf*xyO8 zftY{HdJ=RdGyuk=uqij*PVToBj*oBR?cU{iAH2}LIkvFAYr?1pe91qrcwyl8SEPiS zN8dY@+AI85nIu3{xuR(K-0#bqWoH`Ay%XIB zOeBH}T!*QgSws(RDfg3Zb-QI+aPh>~JmV_XP2imu`%8_x$D^E z;@*jO(Nb8V$`}D&kUxN$Xp!39_``^4zyvy*-e>cw!Q`I7&*({WEMBfR^AgPt9tphu z`%)aY+(i@=bf~+6%xw9qLk!n%!oZ1esLt>U z2=}VXpkr6@b^px+9jQy%nc%BL+V*|T0jEWA-9Cm#x0DeE&s|IX;sWQ#!9Oz(63>EX z_Y;T)0&gwjh=s|XP6dPX4!%uy{c{}Io0l{-w-_88!j+|vaScBS8t>N$vj?6jQLAo< zsZD+fu&0S!dO}Mg6&}$9?rqw*?v)Xs>5lkgi;-C6h{y8i-R-HFz#7=R{Bc=dYim-1 z$Cm?uO#ci$RL#@gV91&n*rB$KjS?-yGa?}}St@}7eRLNs<$ z^$7);K-zIfL(D>Z@IVe+lh>t;Ysn+QMar=&6vkwwq1vdni~{7Oq!>meR%Y7sG<~&h zQn>V5@}4e%twtf~C>1evs~5;$IfXuTz&U)TE{>v)e`4Q_;?pznAgV@aJIJZHwlXHB zkvFzE+2U%{;7BJq(5kIdDWJ=K;cha2{e>;7y8QGlAtRqt^PYC9w)4nMB&eE%EzL7HSWqk0@HQnxEjab|!n|Lcv1;N_)cV(zCeX&?a{Gj2uA2z=Lj+@)>xr z2i0u3E0c%*Yn;;8EL!yTu5$Li_y+P*5%=zX|Nt<+!pm=04LRgVo z_gp(8h5D3uL*dX2dsxLid}q($iZ%GtIm#mew#?MI+9ce`5WR}9Itv`ip%0GcA*hRw zp0p>=uqBYrkLenV^qNqKs%QRYVBAAOkP;N4J2axm*%Z+d?zw4t&D z#lm4J`F52<|B{qZ5Sf?XKVcuUU8#D<{}i8p((-j(AIldfvSvX~FTq`kQeJvtB|v=~ zNI4Mep{=`y+H*cz-gV1`ZZzAHeG%XYS>Q$h4YwkQ%E8kp%0;C{dxotpnQ;-HW;s1B z*zXFzFkhi-MOQ9Sk9|5m2IfPJxX-ejxap^9x!6v3>O}wQD6QfKK!m#E2e|o50x}ye zhhNLPWrn96qu0 z;#HA~rMj)P=vMizF&2(pNl9y>*@zUMJrsTYp`(v^jtOALDARhRp%iwW!ADFc)s2yJ zKeDafXmVJd<5gmc6?U^COH9n`RNGT>-6@m>QspCF{@_#{hORioO=dG4cw;P$cgEjX zw=ln4lJyA=xB~^eWv9Y@*B%g#2;x`#{`bO#IeqL|_}_>+@_FlcewNDs=F#+N^H>Sl z>cX~)S;4-rVq!Ik0O7CihimfN8a0%gK+9`J!=4!}9=m4E!Ry3ZVZ93)OarCzM*}$$ z56sspxEGmA3%7AKH&GlUb46-8Sq^F39kCnDciYA^lwc9r~{}~t`**|97{0H9} zo#)ezcqBqvczwpV;TP~aFb^0q1w>ZsvB_Q7mV=Fa2}bRJCMUbw`Bj3LW72{Zz}S;*~+6SgTqMQp3Xg%4{5t*z5_Ok z#S#X#XfBMKo36Wd_Fq14_>$qJTA{QfQBc?mlU4jQ%s8zJ3dgQjO3NwXHf zo}%a_V5LP-EZw^Mx+!*tX}j~6&px8V9^g_2?DiY@z9Y;rh)Q#giC)m!p6U#&`Ia-< zFgsG_4wgF*lKBGn_!4EnDmP1>qc3mJ*jSJDGPnAqdrBC~egIzXv!Pn#9(Kh2%1YIs zmT9AdLvYSp0TahlUA}b-LuoskoJrbin_}!CC_aZH-OE-)apqcEQxFC-}v!;H`zrfcHS6V+@cXU<vQME zf?sbk?kbQKxF`WSF;Vy3b*|d_kYS8f*7f1V$8tp;;$(@_CEX+g5~HDrERI@t0m{P$ z9CI)_N}9(Un*|$rgSxL`MU1#u_6~~d>_}rQ!q4FM28J}4)sq2E@%-C=S_^bNS*kD1 zBsZ2DrsRAjv(U~+x#C1TP5krty%09MX1APWal0b(lr(#uDA!KbQ&x^VC+#ZVTYo{g zop7>#Ml9E`@{kCBWdKotdDn}apVk$0=@piWJb_^JKYAr%*)ahJ93~bNyKl{M0txv! zTJDqX{O2Ja1MN8O<*lm_kDlf%@R<7chT&^qFUdS?@sg}DZr?p~99nc^b7a)%*BD5P zL25-zE86#u^yjH*%ONW=9vPk6;* z3ay?Bqi}lVf~hD@BExpry}Y(wQ>}XE=?Hf=j2)&>%9m}XD_&K7yVoHe<^hP`fqgRP z{~DEH+hD%t20{SD5%W4ZtI4WKBY#Fyni^Pg0ajCr*PMHkSd(cnc9*UfWh*Y88iWZb z2D2x4!JMi@QM>)r5JQ1PBFptO^Szb6Y1H6fqeDRSB)(SgVR%8^v?;rwU+C8Y(66X_ zeJL!m{^jlcn8r_5nNGi|-nCy)J2)^Lf_voLmO{M&-q^JEpCJ_-4zohn zp*xq^aYPLuPP7_+DVzyltGxv#R|_p?hM6d*3*CRBoSbzhE$6?3~MXU8q1WXrPb{U1sLC>8gM60E9|`L=9IA+n1)h{G8!5n?0&R04Nn|=kX*x9< zVoYW(c=o@tv7a@A$w5WzZI+gG$uXILmD83WT4%)o8=unnMqG1X*Ot{ z%IxXb`~Dw+2u=9nAg|8lDcC|a0!MABES40^-!^DZx(`M#ctw-=$aMA9Qe}SV3R94U zDF4g%V5aDaGGUakLkOZt^EG+6G;K=M(EGDV$kpA!kn>q0 z#zi5+0_o#9GRopcD%o)N`^~GN4o)`WV~s0v4`g8mjaJXRLZbyO(XCPwWK7!gBOEag zV>@6+)V2J4$NM^3am;C%vyw3)a;T&bT*^hUI@y0(75X33{&_6~$pna3Q;vJUWGJ^j zLrE+)Qc|DlVol9Vm3QXM-Y*VJk_Fga*4u{3PovjfC#TZdip8U7Cb-p_e|f|FOS7*X z5fUuN%)>;);<`H&-r`Z| zEsjiVs$xDAoQqPrmCM@;xJx3#Gp7V z5763j&?q3|Z%*+=>Dj~I%|JH#MDc2!{we1acDywdu!O%X2l7k5uPc5a1d=UYdD6zJ zugj3@w-9@53C6pY&|SV-X-ez&OEGq_`eW`Vw2$1>QMM^;&Em4ahn2dXN@KU)GiTCv zKO@1BHX1vo7Z^fkiVYBC#7dCw^AhP%Lb8fd008HoSMTP=>MFo87?1GQ%8DsZ<1W+0 zI9dYLmUQNmj(eQ}DlyhO#YH0^Q1$|w*-&9u1QqjVT)@3Lkdmix;4vhm4Ct&hY_zw< ztSJ=AganXf(4$G9qsQ|%Y>j1}lWsZd!~}7@%WU{H>hWGB(cP>;7BKMoS_ry+5~h|s z1E+3;#(*m9NDOpN)%QNMH)-Xa4%#^}yE0vsyq7c^23R$v`#3!wa1Mkcf}`_>VDd9Q z?S5c_DB$-)Ny*lhuFHsJ$8p%xFw^4+pnQ$h+8|FPI;A(je>#~7<)PS~4rZ;Mq__Xa zXtM;PG!O)U0KL9{8XH5pWg#m-Q2A7P8an)4@m4g3)_LwcqlDd9r4;~${o{%$0U~X0 z#n$wD%2GQA`d}}q)2?GV%)D=ZY1g>8d{r!^`e$i9fFY`V20B9)ChVV7u*HKOUTq9mvDanSKEJNUYq!Y(^^^(T*km@KT!=R%7BZB#K5Lgk2H-lW<; zBj~7!H=zW0zIzF!_JrLDwvl5MS%4V^T58v;hEUm|W&*u2o#$#-{0 zy!iD&0>1`s3~41aM{(A}V47eqOT5#DGxAq=J1A8(52lm?b?F2+gNk^^i?AoF-! zu|4a`$}AupKd=c$Om9-9__9PMS;q5`%_mU;WYom<;^r55>FwJ96xIF$#f@^_pEt`P z-!9wT5$1cotE3q&Z0tlHKkr&9SWoE(+3+$i(9-(TLrJN5d9liTIA=jeD8%iezP_iv zr~|jhcWVt|f;t#qS=6`qZPkNW{G%zO_<0V-y|ua5hV;tM(z;JYomlkV%d9h}nAHhF zWyL-)-}_dRnkas8tpZ8rljOz2h1RyWG5N+mC>n_#jNLWNsVSgc6zRyzs4x36gBAv6 zu$)C-J;b$EF`RwTCwBMz5}_o?aVsUZ_QB`?yA|k-2{4-Q*d+8PAWO?S!yQkp83%== z$~lWOTa~9~Q@7i-SEMAC9#Hoy)5rNZ76`zo{bpVEy%un6<7M{JMB8(up$^TdDS?Wc zO*~rQ&;;qMRNSxUP;+XB0AT^U+zt?g#T$1WC+_5=W3j}!-`)_GV4GMFUlduIm!>wk z^hP7JK*UgmO^Bf1-rbFGC>?m4Y+e>_rpBt{fG!u8J8jbRMrcO#4KW9DJ`9I}Zh%u$qq6!`yXnxje#o>)t(NUAJfY-f>Xo$$^cjpjZ(nTSa zYMNNYs9W@|z5ghIE9hreYT6ksgBIFiSUG%|BcFKvmZ({5l3cXbaKN4}*-LSD-y8Aq zf%(L($425rzTXwhmu^{ws3 zyDw~SZ?dEj}Z( z(=<9&tnBsuv-)Gv)=3o>s~RlT^P88-u@$pZLVk&4@|p3NHXTRzPW*H_3+*PEhht+O z{Mm_JaoHVay8Os!-PpmcGY)Se!#x9*wU8QRR)GbK$N1jKtgEBOY{T)iek2evrlmwo z$bmHy88NpKV%g7dP}QBWt=kHl-7^xPc@&gBjl0A^x!NUYxN)XF*^)%GB=I#WWUU zJzS4ht*kwRj8!Kx>XAZp%IAFH^cyi|OlZWOSuXBw{7(P3{GpJrzIr zg@kali1ybuR7?Tg)**WITk8c)+KI0n+L zm|*$_B2PD4OFd@f-O5c+t3J@F;NO2?HJWSo$yec;U{viNT>CI+x))B%N$y!FdkF&-=2&hwzp!#EZ>__iz0Rf7?DuQ)BYgZzxtT`S@E-OLe(X@1ju%B~J$|gr zWptR$-i&srMCf@cj^3v^2ZpDl8pp;?AnMBP&+he8su*T10QFe($h}dWoS2<~ z361h0sHSEZV06YKP1ftWD1b#VD)hDtHAW-}ZBbs~{kkd!Q|9>c3>HNh23B2nPP?U? z>eoD}E?RxG(0(YH8Hw7xd?Q^6iffgxd#DbDL27i2nB;hzJZ~f-conNDwojt}7}=Qz zaE#*7tu}ZiU;@Z?W64Kq^Ng#{BdpY2LwH^S79q zF|L)MMrT(3J`JPSl-{vp5~bIS{9FKsz0xYs$M_GeG`EBt3ibLE3(8v^*^B&sc-p}eYZteFV1ilx`H_5y2^FJa^C-?;#v1)q84vnleQ zpl>`TN^b|9q9v&oX<~4^Z9SXEEAIhWe2okJ?yrO5=rI<%4;{ZLek@##NGi|7iT)Q`AhA~)bkGeb*=|q21 zw3pnZ2d>B62eLox@EPaW3|@KrgOwy;qDEqyUeZ<20-e*Ze|o?%Lx?SqwEAFQNk z)nB3OX2l!nUPXt>Mcb4Uvyai#(l! zg~Q2V`+jwzvuL~a4i+6M;iw5h40`7iQZ?18Qd(P&VUSrci$SdFX1&K(AN}J)r@f>d zYAXReyZ2=Tc#p(C3WFx9Kp8`6``h=wXDyc!kX08&SZ1Bl(~d`6f3~A|W(HI@NE%~Y zm;}R)V!eD^5j1jdr1iW8MPrhIZN*w$4#F0R@e-M6u-o7C8?jTvt|L&*nb!)2B_zkZ z0j2BFv?xn6dxnX~TEv<~`;aS!#H>bs-i$~Kpe)({MC<_TCOl3bp?w$7s62_k_y7T6#FBp-6*q?2f zO=tud3QtPCc@tXXQF3fPR~6mj6`*TP-Dl+takL8It7}5THXDM+j1O7OM*>#X#rk%e zM&ff{$BDdu&!@f3tF%GZ>hl;7)3k`$mVa%1(sB|~p!CyA`6bC(AhJiol`5d!YBBZW}gTX~KZJFnAVG93<=ky)T*mx-0lI^vrK1F4uRn^YEHvmbe;g6%M7kgGa<^d{eS~6d)?q9Pjn4pt7i&?hBT=e5D zmnzH+>s_q}3K?Eo`6SJ8%MCKU3P9W(pV`?Wa4|!icBm*&GEsIfs(rN!L&2OKQ#0~m zXc^rG+=5Iv`M`X|_WjH1;LorXBqfPxZi!3eEy={EgSorI&`S7g;4L=%(Y~MOG3qTH zGNU>lyW7=1HWQuQT3ttBc+=5R#+YqNf-m9=wIWr*u2?_e z^3)UuX#Zv8lHPntWl;&3BTYRX*a$z_m7kZ~dPDFp!RHq}3<>u<5AdZIE&;3>jz>FuDnIe~ zsH1<{XB|igNHeL~@sSu?!KHTpT)(hc=1KGk<+>h!ThCBgmXx^$n`?nrwZB-(Fj=kZ zi`c{jyN#num(}o1mTWU`c$;BHr?ZU?f)UwpPdjwU;j4ntmQEzO);Rf6zPRQ+aOS|M z{${<|pO)EDuP?5*oVTZL!jag68&u%bcH+Nx#cK?*!}#SWdOp)D*L9p6+Ioy9jXcA_ zRaRmkj+~d;u*hUI-RJ`*yuxx*5r!Gq{H2`q>>ebb;QAn-&>=C}83>wFTE?arJlRd| zut-9`^{4O~t0nYaO0L1A5tKneflqbe!#sT_5vrkO=6H6*C9D?v5Te%6Czw?DeiFPD z!NJgJ!c`5!o=e9gj;;zkDrLc{M{X1cAox8p%`a-P+LF^3fVHVVMuBK%s z6xteFAy$=gm5K2)5hV+!_iVHdX^VuR?TIg;#=an=|Ndmrm}#uOy1B4WJUi~bNT<^| z=`+dPO`Q+%K`m}ezM?I@n7FWcEm?L^!2*Al_)djg*F~r556i$!nTq#**VfFcV@(5i|dq|fd)}Mxc$9x5?)oFJ7F#gdz!MZ+`h|#?C z_0N#!k`yU5!aq(3z${}KaeO+aj~hidkdXl8Y9ds%bue4^Tl$zl9FAExj4VI+3+b;Y z(s}5`X!Y$wM@*$pxn*BfJT3_z9}UC~(&pDHWR)XY5B+Y+4hp)90NflB_V7(tya*9T zt^tyvb~R2!V<-vwmU;e<3U0AFdZH|orQQ&JW<2w}v;A^C{>tr>5+O4X z)xk}K{f#+SeiWTL<&GGu(pvh$why)AGN{VkmFJsUYC+NwnLv2yqXoEY5O#06GG=5G z9v_mD!srt4Q?Enzx5ZxAlp^sGi_>b^2<-7?{#Lg7u-9l`N@7@w#?;^R{gCIJd%XTc zyOeA!W+NDDq*jIRT+!1_L@fg$!n4s4(Fb<=vm~GH-gOs|k+P?8gryj__rv-A z+ODe)tJq4^CVAy&Ic*v1xZBT|A=@Klq3bF!#49fhZBJ{q``L71)Jfvu$Pv%v&L2si zhu%KhbX=ebqTe6wx#KbAKjRz_3`{yumzplqARr?2p*!H;Bd{$G+}XNp%qrq9bfEB* z(d2>6%sNe8^ta4CngBj}%R8G#gpdNQ<)y1*Y&z286k@e%KP80F$nuNy*4vT*n^l)n zR2kK}z@y$uTl*qlP%L`2>zJemS}QP)L{*C%T+hI6jHEQso~RoRLOkCdFfVTdzPzL1 zdmyh+8+L;;uX?>?HWCgkdb_N{Gg_lRRW3;G$t8aWuWae+6YrV~&E7!m*&E+*X){*< z+Yn+z&O3FchM$Zy8Dp&r>`oZ&u6P21d@;@Oq1?I<@6~rR1z)2n7aP0yNoH71y?qp_ z=sxB!ho&qsjyPxMHEyK3rM2pLRTr>H51o;@R4I4y|KGA#VohO^02+ywN4RzXzf$mxy|;F3gd8VoVQTnX{0z zGYqIo?JkmEXtRft@5_1-JQi24>a2uO7AzV}>?HJ(y&V^Kg<>LFW=}IZ**2{g%5RsM z?^M1cig)kmB<=I#W6cYaepe5HJnhs@zlDzR?0%2+vvEVp zlLGYXrD)Ff+}Oz<&*z>L%EHyJH>72nrnEeCB-r7qg|9p3!>|>`m~^Qe+Dk1u7fWvj zb)9d0SepuOWdXN~Y{+nbBws1VW>2Rd8I)SjX32JckNl|B>X&0k)T>!DM%s}dxwxJ+ z{%Kuh`@~FivBi?E7RYTV$UD6p%A(Vuka)t~*N}3(IuD=9OMoO+E>4JkVG5aZ@K0W>W+EV966@y3_Tj2BUcWXa z>2Z95M*pOdh)EP0kBQv+H!ZaW-w&frxSz9TE-D}C$^giogiJ#J0(RuYHjfr3&{^rj zLd$pKLoj#Urrw;Jn;XDVb^_HsFtBbPDvZ5vG5UjiP?UeCz*;5yeTfY7%`H%L2?Tk* zn#t$>%;AOdbk&n_cPI*xsU#`$8n%^NaIv6RBppkNrWhZua^GaD?joDf*q!{{yQjEo z3Fw3-+8~eI_?FvKJqpG8zp``9xH>of_fkCHM&Q@g|2r`^HaE`btp^MEcEe_(5Cw>N z`Rw}g;reraP@eih7kwx@$9*F&;H-kqS9#JI6ZOvR50l5$iq=L+c$On2wp_%c6B;wI z1LkAFbH?tom-w#_pf~H`J3zXq8%~r*Hi4>-B-JDwMoR_!iG;IW4e>62rj!97wi(NP zfYa>DF0^`%X3L<(lITQ%wq!9Is7IN;zD)pGu3qk+iPJ7v>@uAlaMUQ``X~7!@F2VQ z&(UV=r~0X8a`N?q!-WcFpg2_9=Js#EULJS@K&*`}W=rhcb@LP~b7$mPZueD?=}f3p z7*n8^{%^%caO1@?johsIon}>g@@tz{JgrSGk(j9sr%-Q@$qY}Pke6Z^c~J5G7W z=f(5=M5E=-`@qt!)G8q$rrVNfk!iHXrQ_OI7u+>8rb@EjC(rK*$8<|%O)Uy z7ui5VKfT%1xL(WzN|_c0@@qlc9+nR-x#rKM-d)2bNja`&A!QwkgtPTyb62z(0#42^~TOgXX0f2VVVHPDd^jTahV_(`MvZ8>+!^q z`mN!A#zliTP?1bhGFwC?{t%1??c;R)U|nVI795?RuAqdl!&SKfckIVMw*h{u)T9gv zZ^rcf#BF2l?M&kGLJY@QUzA-n`j57{?CFf^-T7hPcbJIA7M7V+4)j{GU*?2&<^;dj1VZQg`|&X0`$Q16S0RK$7y=X(l&8CkjLvsL@ket(9o|3`{tL#udrVfop(E9}H!d;>ota41t# zLrJy~gl;t1ZjdnUu<&@iC~H_TtKxuuS^ABk(7?xGJ;E;|ex= z9qdc>z?Vih*pa6f54A=lR>s&L5MfGgdWqRxTw^X2(Cj0ru$yP$+>kZ6f2__tLz`+V z??Haw^TucIr;H#=P=@>Z3mw5A)kAySp~S9y6#lPm7K1 znYtKbWeeaie;`=Q;-QREIg~X}?z8!QPS9Oicd(pk+UBy=gL(&Py{})jko?&9N?+$T z?QGIa$J7|C;VDoKJxN~JbkOB-nq_$BSYvUcdUBt=6tS>&G z-qc?kGqJtgYUnJ5DjY4>BWI|BVl%VCdJYq4CV%X{J&qAiP-)jgh^3t-aegQCW%(}9 z{D27GpyRh&MsKhmm*EI_OtKgDDoH(qlK&GPPXa0(21kJxMY0&5ryt}_v8{hA4b&cH zL2uy`v=yqP2^sCi1a+I}yLP0GySvv#81=eiS|pufNmrAha!y@Z8ntQjyKCI6Du4AI zUzc}QrSDfQs53~oe`&re|8PJ|>>#W8+qW`ie~#N8qal=U^x`dBg$EN%|1bFIvIn#E z3~Qcc5u+(f1Te<@L4DxV_w~Y=y1Cu7P*SiZ0Yie42v2NYBBz{6 zOZ1T8p&Z82ga|ZaxwSgJp;;~65Y1Z_xa=gRMr-o^G5o6aH~QKFjrPW}`G7%@PS--= z=f)Xb8nZQ!z??9#&fzQl!_>BVPq|!c*VFe#LiGc|*2JETjKt&Voa7hko=32(zX!8Feon!7ZxJvhJ02)o@b3g!Gw6$Z53JZ%SeqA#QFM4 ziI5)M`R4rU^ri8k@;6n+$2(ckHUff)O-!y{i8;I$n#tXv%=PjC19BGyil+RgFk?bQ z(sgl0HXBC+do)+NiX&=`t^XL+(`|U)z~W+4ettfZS8wl!4>Q_|5Vqzn&~4Qtm$GGi zcDzv|>|1Y3VPLFpO4k*2XLTSC7C>45=$W6xj{4^Do@FxARRh!MdD9XEFkUzinsCVY zN_KxeCB3DjOsE0z?;BeWbBpn9(ou4X4Zi}EfZt8X59SMlr#}Fn+|-n0bU9}8Qa`se zn;2+g6*~C+y28AQi-6PUZW(M1m2zD5pE+BAfnfUyV2yxK0d@Sq77kZ+m$dnd?7z-6 zsj+215W9P5zy8; zB><_nT=~T11Fj*|#>H9ft(AK2Ahqr3^&Y<)n!09JI_bgC1A1O`eZhCVL!jHW9gvn| zUuVd{-6esk%h!JJr^Bsg1)+B)a%rRs9@FB7U`K&s+sKkQ+%Z90f%!{swjfpKJ1zdsmU&OJw(TOD7 zhUOVQykD|j1(`_@YRq@1A8fZA%r{}Wjq?^a36c8+e*Sv1Atp4$WSNovx0yDR^1t~3 zG%fb(rfW__kw)3yRKoIel|SaTRGyrFpIVW%EshOQ03pz6VC)&Unrz8wI)R9U)#Wn_ z4rm*`A8w9l(5W6zyet0@;hr}89UT__V%>>RW=WvC_&nI{+#T2O3gpo?KQ~$-sD1D; z1VgqMCi}<^un|cx7Yu%g4KMV{i}4gg2;?Bgtr!7QqG zHF!8ty3cGxoyba@D^qWlKg>8q-yAE9o|a2pU!q{KBQ+C?z4^)De2$}lUb)#>7cj!* zN?xJk_N`VY-tK%js{P8FbxmR=&>`n&49|JJ1MkYs6VEF65f=L9ck0uqrK0~$@X#$z z_sS7#sNqPU%>$iC_|d1Z90}X- zAdqu_ZSV&h<+pETbc(}YwyJ&;`Ox}~3pdGxY&@;}%(}Z$`E3uH#~mwr6O)!H2utLL z9`|;la!Rp+9e3_G4TYu>|L0-zNgY_xduZK`@ecd3{@`x_XC0l^?``qtzt@pdPaL=R;Q8(HBK-ng6b&Kb-Si&yYYtZz#A3@%haa)|2$32yw?5Lk7;2j6Siu~(A`SV9*?fYSSHDospt&UQ> z1w+-Mn{t?j<70D?x9ubXthW(alfNCK*Uf#{)fb=BMA!5I&cJ_-v2%(dWpJ5v&8j!! zNEeqVIt~j%D6JGtB>P|hygP|J9TMy{#fn0I6OfaLaEw%R9857~W15`+ix z!#qL?{@$QNd+mY{3I7_V4&+o699}R9{8`4E#SmJSm&}K+eB3{jk@8&3{=x|?#7-17 zSvQ996>bVeY3YN)7v19Ba0vjIpEH~n(Jta+6h0|8AB4?2cbw%wG7yib&u&KAbpX1r zx^5?Qnp`l3gfDldmMhlIm{&<8AVs|Fm`AsPi7ciZ^Y7|e;T<;7RkS50ZcswBpcbwI zUWkY=i95hi4%*!d;%q+18pQs}ldQI>hf>$<7;st;*?jwEuFT(ODebON^gNAP8xuA- zvMEB>eO~U#0Ly+qR&w_De8(f$TzD*95pDwi*=joNb^mBIpxdT$`hJi}P@pyWX!myI z{4~&l=Wc&trQdpL&JaKA(D!?oywtSk9)j4Q&SaUbHCcd+TR5p}MR)IltRC-DKe)|P z0r9sf2l|<%w+^y(FI(J+o-(b!QmERp6L+=14I%fcx5_$`!b7U0nh5Dn;q%SLPoA5# zvxRF$50P4MO|UXvGk`pIcNF@xp2fN?KFH4r*fM$Atw*JCqz^d}YlBg#@7!dLX30~E zS4#wM&o5(o%dc+Q#HH60d8+ql$4Y%4+{zlv$OX(IcZ?s1j?*$t-<5e0MdZ8`{&o->hvs^ z){(2$#;#|}CJ?4GMf0E+ve%w84+mHgczKJS3h&Q>p14!_V}jYDV*Q9J1Sk%kcz2Wa zshghw^&A`J%Wk6`I_!|qd(R87!~d5}EVh&Szcc5Y{~wrhz9?UIex#rnh^gxUiT*%8 zVC=6+9AwT?R{HdQU4*}Q4vWrm9q$KF!Ss*IG~%f+Njd=dlU1CD&#}rM%l0uo^9)(P zi^lZrfL&d0_*U9xVD%HYlijTUDrpmNkO{M>qC$Q1pf7PZc5FILoyFjm#+k88JTig1 z6vukt>wpp9egE)}M+0-4TiNlR9ime7R-&sLAIahxFrx{*@>dT-(j)wkkcTi7C@pDFnthR+g6(F{O5aNIP()Dg)(B0aV{9|QFMw>mv$5^& zr|&CECePYcbd-7sRw(;jt;?u}3=xiODgo-Di9{n;pk(n}+X@Ykdes;&rH7&TAe8u! zIm7ghFhZs2qrk17oy`ni?_|o<>nt}}CWO4AwhH<4s{d-^Yb*^NZj{j+Hx$ut4>&pu zNy5p~B?7Wt-3j1#_y8tk)O5n(9jA+AaXTGVdtG>-cYxpJMc?;cgD2iPl%Flb6VJ_D z4jx20hSqh%?QA*mPGJ<%y%5u0Hx#}SMYFh8vA<4}_{Vnz_3(IS?Pqp%re7#e8m5uS-9w5wSOQxvHd3Pni!q+Tu!a zwXRFbpqxgki>)+-s3m>#ZzwiJx6s|*<7JR#=5tFnxmlQr#J%;T?}{jQ?!y zt$TpNB3A@DoSF&n?l-|+0AdE+MM7V?U$U^#p5)ni`EmzS{xCx@+bveE3=~rB*`hMt za@P2ZWB&{A~%nay*onZNUA+uA+`d*cqe-N)Tld7zJysS1_#hK`nVX4|Du_ zJP))b?7N{US(zc<1q}@Lx-i!x)48z^7y3KrB{whjZ=fI#Ld3gak6nM53&#<(##NHX z--;3jQD%MpPzM?!dPl1hjr=)~V+YCDc>&`?&U`RsR%vEmAvaTAXb4i^yC)v@K;F~{ zdY#j?gz#@kEo<<~0M@L+f^3=Y36#EUZn~I4^fJ{u`00jZPDcv`AHM(J4 z4tp!j;u{#I>uSUY9k;Y&DQA(lxIx8W^=9&G?;_B<%XrrOyn^Mh`SiDl41pW$Cmkxi z(}SKi=D{{@3#Rnfk78A)cKWL;*Qe=O%bStirD;>hSv*ecO?ML!QLEI^$FuJmf5F?1 zO(E0_rgY2-?|r&NF=YWxq+@~+pqg78*s3OZJEs=#(lJx7_OmQeHMQIezPEmgAkzK( zg#0nt7Kb3w5l0|%>G-#K^B>^LhTJ?I5PIM5u? zKOk^>bq|~!iAR6zfu$=2!00J$Ti_p{%~}a6bP~TPL(uUt23snBy4Bs*)c6U8Rt_of z@>pWP@-Y}poY`Ip@x?(`3w7N|`m%t#Zu-77s_{hN2}x1aRYvp_{p?R-i4a{SU0axb zFK}o{W4vp&kb1*+Xhp}@r6Rhv7h5#gG29u5tLXaiTWL{lbSGvSx8Dl;chEX*ZA{Ui zU=M?=ijpA?4jR9b*C$4OaUkL2vyzhk*mg$iaz=jnN$Y@yTJZd*h~N)l66i%hG|TS& z=->6spPWJhiVu}ypPH3$;8;C8JOEbOxTmd(va-VHMN(2Uv{2eq^?h}jSv2HmwEzEO z@2#V%?B1$1O{|~-5xPrOF(0j3xJ!q*uUOgBj=G+Tr#8NS!c}conL5mb zEQC_!L>T2&L>t$xVDW{d|9p`3GsiSsL(byW#4l)IgI4n=R&8edhHerUUtmns9rycl zC9ksnePF$D{b&7h^_PE#YoRH`fEYK=M%+(6c{V7nF~agC3444J*i1%^O<2J;-UOS0 z%UaJMx0E zc<7}EL*0tQHjE-B(Go8Tc^jXewEQHAS8=QDxpy!dbV6(Y+rhW%BZP#ApD(tRK>=^K zs;;YfDpm;C-zx@xpS9wi0DpFuJNRRW>^`eV=yC@`L-(XNd3|_4!JKO#r)01EsY@66@qaUb|DQ8}l`h0%XZJWVw_cfb9^>Ag`AoA&d!zUK zatv5lsFfau*`;6D(`jnO7}+kc`F1GZ1HtLlH}l4Y)i0G681jx6N%ld}0(Z>R532=3 zJ7-GxR5H0T!KUgoj5NsVms-w%GHUd+q*I=&=M*&aY=t;67`r0t2gb&_N>+P5WISh$ zVPmbi8P*P*;014xMTDq>0`b2McDaq;vjniyAH}bNZ4RCuMBcYZveO*8%Ylg0{2Vm~ zQve#j3%St^xCVDdVjSFB%_p{H^w&Loj3=B>W=s1&3TP4D?hGf@@gF^+?ODt_q}-5U zi{-3*JX_7|t5nz4IAg#wtBgDVspp_S+a=5?$wjTSd@6PeR4I9u)@!#eW_<-mNBVfCX%0w+j1grw6q(OhWhBd`!iqen({#;m79Iq3j zWQwZ@r6*5Fh(S7bA?_<(s)0yBGszh{VmEIhg(?rV!#K0C>T2En_%{b^2-9uIJG zynR$UOwhyKSWy)V@(z0pRHOP1!tewlQ&ZCrb$~biF^E@IR*)QLKLDgf9(=QBCMK5e z_5g>T>$cPkv|+Y_-}e&m6v;`O-~K36*Fs5@x@XrP9r-cGZjfYM8{D9!ZIcR&k1l zQ&O^FDd=mTAf1y5?Mj#EJTlLjPPfj>IZoVTL+*XpIJS*dTi0)5%tzkuWD2zrb z-8?`53tq!e)fG0kvx7Aaq7`LwiDLckocOOphw=5!4pJ~FX{_h$B4!Ks}co5_3r?a*1z zGw;dLAXq+OH?bbpVKfGZlJ@1z&rM_a_Dkkx#FMI6#V9#)T}oL-HU{Z$Q! z#Q|yKZd_AhfB@=yEc}Xk^j8}8SQwOAC&%S#)z1_6)OBMNcBiJMnH^$T;`oby{gu{= zV)X)z+)$+2CNZd1c>#%5ov`RGPZzc{@i7M zr`KQwA@B;5P(GAGSEf<~Nq0uUIzIHVZ6Tge$2F6y%pQrA^3fD~#`Z)L;F4_e>-lT=Watr^R{0{IB&H!;<&`R&hIBYi$6>=Br zPtxt=tkCzU3QuSVU7(Xm8z<;aYR68(*tQp+8DL`NVJdKM%WlABCT%^IcW~S%trXyo ztUEG-u_&pI7QLW?+Pe4+xa~b{{FQyq&VUS8#Ss+zj%cjh5P@j0DF2bjt)av-CPJ~V zv;DMN()B0?Z-STjc~~F(DCXO*CNExz?Nt2$;chzgm3^@-O-$f}p`ug6ijNVCd&jc< zYZw+vQDdQtny4}S=M)8>Uk`DTZ4l&KvNhN3;fo+W!6_Byq1|+xadEnTo~-`9du`q}N{> zfATWfI=G;QNsy9Pu9Twc9$mH@QtFQa$-ll#_+i?q;~SB9q+_yANVa;WcQZlL_o&M| zGviJe0wh2nam+^50ik7=tNw;ck15S+5u|+5bh$ksmEfe04ILrS&TKA0h6cNiIXoEQ zr7TIz6BC1dGS($bozp);T+v6Sf8`#7BKeJL-VFcF!m)b=ZhM$V4WB~q&yFfq=e*5CQ5 zVhV>VXyVchS@FOA{7d&@xN)SB2T3~-rrBg*FF zL1yZ9Ms|mvbOmXatX5}ceQDjw)i`JX8IRRVG&w(JUuetl8@@@d4K`b_F~KqWUdS5S zy)UJ>{5p>wX8P)@TawkQ4fH<|a`At#~GkuLAh%DSwYt6SOThw+Mt#3zrClSn(bG8sIO?6_ys_sRB2H&nf-q#Ym; zSY%EJb6sT|APwNhP*7CxY)5(A!|V6;SMJW2dF;pTDJ6Jt>2=ZbZQR`&ywtkKQ}3h{ z6{G$x=IeG(NVs(H23E6aDIlNzZ*zoUl0NRna5)qLh+s*>!mH(1+hxBMEB<-N(`4pJ zbLl?N^C#|%Qu72EfUvug4NVAIeP1gPml;lOliEKmQ2dAPnieia$QjbXQ1M^8F#Fgq z8t~Qyt17;s0*MK?8;?l(ud7Dff64;8BS3_2DedE#S({wouf^Rq>RJ~2tW*XJJ29S=_S zCvS)o^;<&JfsHo%!(ZS3ydmJi5X0)@H%4aVDy|<0IhynPs%GfKQZSE{d>kOwgtO zYl6cTXjKN_{+XVH>wbJPR9YZeo@Rgl1v(`#Z~o1-^3PQ$@)^G?IoA1#s47@8uzwPn z%TD+=OzA%o6F=q)V-^jVlShfYoYbv6epayKch4v&RuWbPM_O9h!PfJ+?*9C{ne$({ z;Cr_pezFs=n|9;!(oJ3K>%ei9~f`#urQ zrTY4Z>m3}<@#@anJN@X2iXU|BzUm$^bJ>D*^Dh3k#bQxruY+>jP~$_uHF(Z}{qtv5 z+QByxB~#R>QnaHmuZ&hpfv}#~ZuO)t+}$gtb_q^?RlDU0)0zr4YILgxaSde?qpcSp zwf=pi-BTZ|K!8&cv*kW?M{>`OJ(iq-EZ{kCGcQcO@Fp^?P1?8Pqm3;IJjQz}G(WlS zn7l^Tb+IgYHrZQ&ds)1?ayIF(rP1beEk8oWd+WrmR2N0uXnfN6>EP2As2+)@2YKUC z#JuT#v%QZHU}hAI`4Atkj!!h`HEZqH+gC72mp>CGtR<=Jr|d$%R61y+l!-`0#%DMp zyhKm^zJ-h5We*qnBq_j6@6qY(P*mR&t;qutG6wrdx4fJC$OgPqM{r@mgk8IQJ`0;& z%d5kv)f_Y)HkaG>)&!kbr=*J=0;v4gt}GUI7|R!!2CfpxowE??{GD%17H;aT!D?O3 zT#G92C&T7)6RI95R0mL5W$b?#VB&_GmG*MbTR=dsym;VDYp@zdb{H0CB|d{n;@hn= zKQuF@)JxfJPPwmsauCLXqf#i(aQVj_73mon4p-V+xY7?Mcm(4MXkX~VLgZFD)SFz@+HDqrkunj=Kfi%eR1{iWbpviSfSK#>wcDNq5TIx2} z`oI#zPc{q)W|dEZkmrgEPPS2lU5+GXE@T)ncj?Lxee4`u$4a~p&}P&1bi&HR#WY<) z#B>?#t?|2>f9M8m>aD`u>e?er405aW9}Ut>iDx0tS-az)kafr}nl|scM^V^mQXq#e z_NRoO!P{1m>@81RLg#azw-tGawUtb9%->sh-DO`))Foe}vDl+5Wb4Wqc3+}SYu$#6 zO@l-XD~Loa4gpSb3uPhi`3h`E%2JYOTOTl>zYoKK31!tn@*p>b9Bc%W!ZNtIRp#GQK=5{z{3MmX zQN;paA1<{u7+I_TiVF@9En*WBR_}fWvRfo0O{vBH82e~Gk@7FlZSJB3T}*VI7FKl< zr4xu*{ZyZmBUBxF*yA)h=%8b^K+X7c2b+kmOmD9aV>bNid^KlUC0ML#27O~5E%X)B$;#5X`pj-FyBtVJwi#AgOhSHBB6$y= zODA+{f+sVQE+2`};H|M)E*7*iBA)CME!wR_oV^tSQkE!sVi2|P8{R#|v4b6w#{$LFVYd#+nM z5ii4X&VO(g-=tLA`|Xz4^;3uLWS+DRd^IzI0`9@#iolYhyO_41!{a2BBewU~-9;;z zg+CwtB&>P>s3ZwMS*+wXrJ%cqhO+>+6rdvP@C?7e8TUoA8S#Yh}~$AGeD z>Jc1BLlh>GzW!>W(goh2nLcb};>+1(4Qj_9d!td8sr}TIreXctl;R zX11{F` z(p&lJGbx{~>4KZX-`aC0X7g`)#GAZZ$rCr|`oc}>xQqvmL$U62-X;RLT%8dn2WWui z()`|TGfw1A2 zk^9F!3)A|62FII&Ikf=zhC2e1s{->C4jFbx3z zZUAs5z{p!~5v${jWP!e{U2&)z4qWm#tnsxZlk=MxEfopp^Me)DHBaw?`aTRPwL*XG z?Jx~}rouuyCsxT>=g5{HJi~Qev6B{?!7{Nt&$CWmT9)R~n4I$)xWxt@Kl6BRm5>~q z$Q&1botKeR;lFiEcQ&_cmTw<}j&gvEYLc-37) zG|)M+PB+hT!>h}hnDxPxJ+ZVNio#W_)%bkdX6s7?J3;=a^s4U;dN(v(!~`x~$Uj;( zHE9GhNJ#i>wC7@D&Sd1!n=-1qTm-H*a^U}KwZTed9yX8goql5rr=!7eQyK*V_cgD5 zd9K*T{sgrpdlt38Lo${rJau;EaMx|&P|0_o;c`z~4Euu`HXP|rvci~UJKcR%?*Z~Y zbTFGIglIAJ`mJ6rC}9tp@XqwNVb0-4BtpAFg7y}*#JF4S>G*ql=6ju$c@i3jByn{U z4a?uk6-e&dX#74lngrEs%7Q7U*?PDW@3}{+3%PV=`jDe<&IY{1;l_OO-q=*a4^D>p z{LvsSqNeb)I_ri(bD1y}>D8f@ugysdLj6)_l)(3Oz5GIxyeKHw>ilE{+BCzkF4X=k z3?0qvm3Tg0%jM76&7&Ir_=F~xeP)7Vi3A{iehkt!BrsK5ZBjI)O2kDP4)L!+$pwn8 z*vd*gOOd8SJkVA*opc?=FJ?YILf#?a@}BFq!attdOdJ0Ep}^ko1(pzte-_=9__*tq z{>gV@R@XLi-D$@{$=k59R)Gc+LtT~1)AIAAH+q?_%|fnd6el%KD1~PTZ)irePrlhV zIaSA3e@S0|Q*ZFZUd(FHXtZzHk}ccv_+EA*Umm&k*Js8}6;)?xx`|amv=UiCcfUwl0Q+1@XD*=W4u=Mits2SH{Q!2$svtXxL%{XRZBLydslL z>Y0Q~63rbtn1vRV8Pat)ETcxmpJwarbu7*3wr7X2DUk(MkgKLh0Y?IdVb%JO-9xO~ zi%8ZxtA4t~{251uFUa8E!=w2WO z7_;Dx#r|Cwuoy<*#>(Ky2Ml0HK~%`(cdnA5W^p;_f0eF6 z?=p|^K_7a-xYB2z;l}T$=2f4Z)SMkwC*4tNMK6WOTzt|Pr)Js3(mO-`)h$6R2vT_` zFtK*9{R4~s(%%=QQ5{(3((MVUHE8&OG=2k%h!HJZO6Tp_{@6Y!{Ndn;1-XveVWuB4 zdpm!vn)6^bIL-y6qBV2f%|jvWRcLqFi&tGV$xSz!(nTM>^n zwJtqleS7bbN~Jw{SRhF*ZoQ zp6T}EhRjJWsFfd9Me%e2j563BfTGXs-ES38K(GPffLM$W4xC){L(a@Wp-~v%Fvxm~!y;pO z_=yiCa$!vI+Pc0trKf1o87qktB666FdwVCaURe?8RwPc-pEr@=YTjm}$gyRP7=uXn zNotxE1rpBFxbS!c#6-z~G7{)AMu@FcIycb&rmi8E{EJdIG_O-R<+MeO`C16Sy~;>& za64XgTK>HA>UNPSj==YoG#@-+s`kA=#O&x&>Wmd_x}pKsg7lrwvu59`On;d!mu8sE zmJX>J4R+hu4Nep*(y6r^$P#3j#QxYM$CaypiWqrs;aL+i!qv@7Y+U~DxYf`wzp$i= zoz0sm=qs|V=`N?-Y=*uEguhPWKDCZk>!_S68x8noYIx-{Jnhs zS!$x@z2)t}n@)+#gHQ5@F+-?$x=KGGO$&O^3SYM=-_Rnz?M;@CFM?9iR5I`M&xTv- z&_c((@C6N7%VGyrj#{`>1%w%cbl)Kf>Q*iazW3zI4tH~RXBonSvP|>rpQL0@{Z{_) zTU6LKtYo`|qH(W589vW&{ex;v;O+sp1%FlgB(T!T2Hj9UJY}#Ek`WRN$Mk5BN?`Fb z4QC;iewQ)po??xHj+BK4>4NedQe^YB-)KF-;@RdupG0lG{`|z^rQwKaY4t5)YBcqi z1Cob`_}3^wZV9)oqkMcc0bHcep6?<~UO#JHT*Y1=>~18-7;GDKZiW@Eg*HEVxUOC9b@oM6TLaZ$LwtM=w3xP5hf zZS?u`XSl4~+>D&VcQ(ks1U0y^T*lvn2i|^XnF;65A@R24X zWxN-sV+s7G1UgNuyy#d(*mlwcaH0uV!_r)#90aRi{hnZcUx-|h;P$iG9bU6-zp4_X}s6t1A_Mti}1IGb!k z|6&OZq|YCRwOBo~3-nb#D-D<%_cX9|1g1pZcRw&AQOLket}1O;Rs%go{x#B~epr5x zO^kbOVekN{2v4ymFxqq-IWNi2gYfe=)m^$E|6FG57%OQQ~@*QCK^PB|OPzdiqVAWnGeCtxpV$oBJ)&Kv_5XF6})>;Kq>cH@uIia+usps{@X zpXD|Ghy3nivY)!*><4scVS-YE^t^ z6CMdB3(P?Lk0vtGm?C1yq31~I(8kK*gJ@GR)kM_EFh3ll7Zg0d@a-i3L9`qQq<|Wo zVggbR4g9>*%#10YO<9(5#=}(INNliJT8&&RgeNf!zh)LJehDCL@-13TC|Tm}I6t^<$(alfnw*n4Gr?kE<-`p_{+f}yJUy(|ZUrAL3zV4z_$wI0%X%&+ z)JB4me*UOZwM_nri`xZ0XN#+HT}nhUBiXlX!HM-ta<7)6d>$H(x7IyEC_l|)9HIaC z){#K@JIz6xIG)jy+}2c!;D>`Yk{E2Oa%mz0xUzXkb{@Vw&h&r7ku#Da*(1iDwZ zU%VxrBCLZ+2-s0r%G=~5#&^6xFXgO^X_0W?8;#f+hNjeb^@!iZ<)66thqKs@FxUrI zFjY58U$AZD`l@}`+M{vaL*a!mLFTJfx-_djc6nis43AqUR>?(Ap403P!(vZ+8Su>r zI{VsQ=5V!Ih!qF@$`fd>j)}2kUt;u~!S%$(i}gaWHP%_LrirW5qnUB>JzRV_I3D7J zj7|ErNi0ul1uM@qI{{K{z->26(6Db~6?Z!IQSh8GFr~?HVRKvo3K=c6UV97kMAU!^ z1U;^rYpoLPr>yrr&hYcEYH1mHBg1Mp^Jsu?x^qoUn1%3a-J%y4Nj1XWrAN2RNdPs^ z1B>Oz(JTvoFzhw+wpw%JYVy*q+KY9Td?Au<$j9ObQY5dLGZ?jJ=yt=^9dhe9aB558 z3O@*SR>ROVLsv&eE2^~Uwbh;dI zI8LD9DFqLIyqi1jKwj0W+L6>xLG4|HjGAx506qqjtgseHV_C0Vbt7i z_moyIa2(;QdoabA--^CMgZ&D>^Zi(*C9Ttgciy7DbKGYvRczb187501kmP6cF9m%h znD${1y!_R2w9^17*4kgqyJ9mX$5U_gGBV^;?}vUTevq`iU&8*V&-}54c_%d{DU%s< zuZuMU8q_hu``UJDhr+V9=l-o0{qVaZht|(e;dUa}ZLc7BJSX&+)KuZZ4;${?c#HZh zh|j$KIelN>aBD0)XcnU;|H3x?(&kHYaR@(6d6+dFO&wB?%Yf+pyII zW~8~%F`=qL`iRpsS;ral2sBt3GI#8%=TC}*S_UL29R0Bun<<~+rR9FemQOc)w2@=~ zFlETKN^XXx-^j}Eg?9hq2)+u68tS{sF@5Q0VDA5MuU9dmU2F!1GS`h*^<0|Tl zpE9*88#vrQ-zoE7?;T>*k3f*zh(DMWPKRnbnO(uZ_CH^KZJKMBWgRB;1fl$6=BC`6 zNNmZNO~l8L)i*sK5KJLFLy4}gw(szKemL<68Z)ybl|>MbKd#m2l@oI^HQul(-p>L3 zbN?LT@&ImlBmFD+?gnl{X$oDt-#$1vTpOXRWX$O&XO8u}szzfXv?eN=RT2HTwoVQS zNJ{m?Q8|vO%jc&hqH(ulzrF0Wv*$JgO?Jaymg3Y)#f5}aeXC~q!Hi= zIMvjwKTto($lILufeK!+%ah{29^%jvN^C~HjfH8BZr8b+`2>CBhgwYU6Y8ZJFO$Cz zG{JA)tL*mejorb|-A^$-PkFMjhg_f3xf52}@6=dxByIbOLw50d8oJbW&1MDx!#2x@ zB{^3E|9ZpKIT}gsGt@KDsm?K|%u;BCPg+dHMv_@0*E8HYOFq{om}L=+`ibBR~?dI3D>SRl85@@~4X2)Wff6zeS zqql+EdcV|~XN9`HH{`KFlST0K!;e;oAH63+dQXo%&A9C|#P0l=y#%%#8IbtS@N1>vg$@2yKt-|XpKLGKOjJusZ=!ZTqzG#<_tsQ<8?vqRk>^u_ zfEMH^D$x9hyZJHRhfj8OrhpqX=$fyHiMhjG=kd&Az7(9jGJUZ5T59)(pIM(en>F?P zc%h`4pE16*j^2cN>n6tDYMg!t^THQP(=9oU4YEIKbD6S>n53wr5C|vy;*4?}1xLc8b!xZQj*! zPekHZdd8a}Kej6>G!dU*ef#O3Geqy~)cj~3Iac_5|6NKsn|zzq5MMv2Te)IuKF+NlOx4)y&7$`(buqI~s2FGDrP?eq3=XwI+KU33v$lc}Xvs4uLOYxI2jnLJ}-WrLA$NPtzrc$9>+wx9~F zin0#UV5%4DUaRdUqUbbXXiIo|3cCN=XpENCc5m8gRsee7Ufq(?!i!RQ9n6$eC@T7m zbqKm)xlg-~IbL`z6JDJlWqChMQk?qU7WfH$2z*=BX*1><=S<;)fvK z<&>uTR16O$EG?aS(5zloBkmH{=iI@DJh*u1+Qw?7*ClXOL!)tiXsL2PXqmp9dO67W zz3mP~>XrHvLEa>%QW>4$0^h@~Qogu@1Uba53!FtVhp26vAbooM^A3zF?fvZF%Wkh~ z=+z5aD#*uXE#K;A(O23WX2#+$RK4mO8B+q*gsdzh7Mkrm^%CKY8!yjyCc9`B~h z3PC;iVx|)!PzVJ*+Px6=(Dqux3j-gox8NR7Z^ADU9Cr~ZcYi5!!OkP&h9{*xa+#ye znAOWHej2j0ZD>ux>=6zVtn9*@UpP}zYrAMyCXM4sjEMO7{?wFw^<+MnG`aPlTLKY- zr>K{$REx#EDOq&h28-qT_UeprpSXl z%FudoOS&*WI~z0!1b;4Q*_?R7Fi9t>9W%Sa)wQRhwo91iSkwIlPI-n_EK9w{e~Dr3~&2SmjpBas^0Nr7-(_1~V#Gz_CvpyQI4 zm#;!{w6WpPPcr+-_vv`Q(P6DKdgC2q2Mc*GPt3Jp);1|_?O_G8MpLD1 zMdthnZ?fEntBhj`w}BPW-u+r8^i?;@ABCf>sPLLZuiLL`Knls3Gsx#3DARLQ=4#kj zdHUI!f3a`I;D@HB)2yUs(yNJw(B$9?MPlPvu||)D!`|N-p8l4{W$%crFIU?sS=IW5 zJP6yGUOOwR(9{$KHikmJT?@zG83vEizKzMYK&5)`C^Oge_+kL#&F2r204@riwbPOwVKCAe_jo2ZTI>5-0Q4 z2PhO{j-d>aUGu?ew(d=|$o7!8%xW6;iCnlnUknV^^cs@nN3^<(v{2yg6(mIp@%X1NWw6uxllF@o`TKeg!YC+o;RuZ_09No~Qb60ystXl#i4b_QpU>D2O!zbvp5HpKMQ&jMEFZdjMY*}ZV5|L~7vN1Fi zqRfx-i*lt;WmL5eK^-1;E%I{SHV@Q|Ky}+6w|S zs(>-^4x|LnMZ&6%YkzZ#koZ_XM$X%gm9A=U0jhv)TOlMN=0I1-mb*?V#{4Rd&VJ{a z_b(Oc#1rfHCp!kcGt*lV(y$%8&fVGZR=&EhmQw}qHgyRfB4(%gasIrQi0&OYd9k09GYZaPVy54TFCS~>YqY)A}J9ND{=`53=HC!U3Dl0JL{6h;cYyK3cb^4Z% zwDSt*c0xsJY6D$|?TAV-Juvb*?eWg2rBSRbC2jHvL()s-|cFJ3F*E0J2iwP zz1+oRDjTDtGo!bxbg}s}%$|I{$23mGUdJx7nXzE>1G~7fmktLr#x(JgsfbG+O+i5FHZ1B-DOCm5b3R zJsxT4d~YjQUM6;}n_!+>Y$(F-iOk*d+>OCWZ;7$6?LH^ve8FF6m%2E+;_5wnu#Jx(Ki~T#-N*?K6V}p$z}(NWfizYYkRx4#TDGGTHf>WWy+cEP z+4+8;6E2l&WD|+ypdnz&MTe12P_tzxHed?7Dr8isy)h(U&eh_`G^Y7y@8iE|*uj^U zTEs`5HWIF4K;V2|%KQwP zSXto{m4y3{nsk(R5-zV0n))opE8$80-tze~604-TdWjuR_Rvca?7@C(%jSe!Iduz& z{(M&DnS=7{fB^RJS|hr4hgZ$kXrFn**BCSR+8E#0`ONh_4c4bMzUIDPH^yy>&Z@oD z=ckPTFMew;Z1U~zxjra51rp{!VDvR0t0+ydUHuL}Cwd|Z)UV7FngLWatHUq>I#m_) ze#~tMItG2GB@0dVwmX3-D@v!-;M)zK_vS5jk#lrL?q&5&JoV8;oaTc*y%xvbk&K0` zPsQ~8#C;#O5)qf^rD-jb^X@t-!3$-Z%QHb(fVQh(5$P>f&T?eVL!@aQf9Mp?12tRt z0Axmp!^=O{5x6kiyn{-M35L%fm*4a1Mwa*I+Lnrmk;}GFlviJBc1erfxC&e`Jn(Cc zlN>0Fc}~U?{mQP@N=(cxm}-6N?9*d1*JR82Ne_jX$jC&Au0D;i(hPZf6HVS2vbm<+ zT*{U2s}6%DN?ad*m|0Gt?sO#)lL&f}CJB=7W}z`_qY|_(;rS-zszvmKh8(Jt4QgCY znb4dsVjSIcBZ?~SQffCaJ|LPU0CgCDjU~^!mSrYyZCu|>V#t4_?fH0ebxCz{MZ|o8 zBgmdy9*$Fyx~4LM3&&pgE4TsYCmj(wPw39GLCR3lq_S`v-PC9L2tGZVJ-o-&hgaX% zrr8on@Aa~s-X@XutUrD`&~IJuLwSl5eXUQsu1}qFoxyx=i}!;TVw`E|Ov5KH+E?I+ zrn#(nW(#c9{x(ATpMb5`vOSo@X`srE`$<##Jv)~50J<}JG+x^WaAMfYNN%3LW-?`1 z?FS1Yg>doavv>ryEcGHTIJ{5SWu!7GSTQFJ>7Q8CI*ha>6u4ZH9+KYPA0+#6edt%` z0w3;DpwjbGbiVd@iwC=Va6%N+@Q(azr~TQrRT5*GDP<_Ynv|2%r2XU_Hd*AXnB5si#=hSTjb zf8B{`p8qXCk*9M{1i5y&$_Pgo{5sQ_KKiwj%xh~LJQoMJwy<_BB+Y&7CA-8 zvpVlI{OX1x;$*BhH>N6PX!imqqA=_=*BPhHH1!Gr{AC|sTC5k@@5X{KPunjri*!bMRJF8&Vx3TDR z*cZbvoZv5>fICv7l)6aZcfxsDGI4pq<@D~aH>kLf{`y3-b(~|>X)J56Z6+ktE#dN5 zANUv@G!4)$Ra%h>1;#ZNK6v?`b^aeM{wa;W15!R}|L^!+YqTlo(D?ZP<)~5Xuz|8( zif?dPe+Y0!3V1v}I%rdQ>%PbF{xjeixij`NWaEdgVb<7{c~^lpnxbP~!Ibx{-JiS< zXR)NV{6AXt7)F$GgEY2D>o6r_>dkP$K_?aju6#;?gf26q@PSwN>cQq;m1E$mi)W|1 zi_#5(Kze5%aRjdISUZ5_f5p#iZ33BnyaIm#YB`!|~#sKeN6ktxv_VcRnPF z?$@@D4Jo(kfMcF;&a$nv$Z4-H;K^*a>Q(xSBD-6%p#AgrvA{%CfmFZ^f>1w4vnb-& z=)?P1sFGdI%>yF5XKS1WW<4wBqSr#}&rCjcuwl)Je<5JDo8M8gJDjy6rRumG3NY~W z?V->fy*Cs0B#6NgzO$5UE=lu^)d(*_>r>Z@1z22mcSco2rgC*-Zbod&FgvUHee8#JpHP_h_4M=9Ct8GQ0$h^2VO6;PWap2 z0~RB~-OcR-Alw~jP)uO8_HZm5fSe$u%8^lSKk3C$DYd@`!fb4J02S>`bxQjC2iNYQ zl}olMUp?jWu}M)=<}OGj)dxt9E3Xy#pzFl)R{i~l)}7BZk_C`fox=9oW+U1pCcBXN zlDuaHJ)hV4T6LEy?^LuvgTuHAFRR!>eL{40G?O7(EuuHO9+UQSSab!aNj;st!PHU& z7>3l*CH>&~5@9Gx-``)S(0RuYXIus3-O_p(^6zzdEg z^dw_NA6#4-+WX9kKsKpr5yRR34LZ`fmY+QqclT8PIf|bE8kGVhU(W!=4n8TsmcwV0 z3sgm4xsv+HN9KLPTFFN~uZziiRK1K!>Y`0K-ZKc>;=?)Hn$+?Q^QUUFx9qDz^AbJs z$op`(KOa%F5tZ?5uk2iM+|({PhpbaTI4}K^%pi`n0Qy*=V zLO4LNf~9URN=pJfA^tY-VD>k(2QGG_Su96H%wP3`@PemO)l8=sMhU~on4P^=lAQc3d5LANG%7G_Y00L`%N`el7^lFv7$iSsnCV5 z(+<(Ay3mC4r?3FCGQJ9dj`n0cPOIYTb;%1%$7F9V>c0DJ(pNZan&RA5wQW?))`cML zHez1kVD=8|wM2~_Mh<(4ww%qj5n5r@0bYICF}!nh+5U*f!!Qhev8yeR&xZE!<_6Vg zM^pv}P8X$MUw%Eg&|P26sz2Gmp)O_)}}T?v3Edg9JYY+Jfno=488hJeCXWC%2rp-ECm=;Pk^HnM<05nQQf1 zuec?4|48)aVUNzALM(N*Ly%9t#PG^xfy!lhe+|Roo+IeoF0*YgK3eqErbJ-3<7vy!EP*ye{8Ams@yww*$rpv3$8#=NbyTm~e1v z6$q|&BQ&BzIv69P8RYuL!#MZu%zr(Y?9t1M%fr~q%l0k@8(ODCE@sOCTvJR|4%i#Q zeD7A)EKS5(M=}lYc4-(t=1aBp`G=>y(#@{@J={(f)ge@DyLRS5N|L8{H_hwrrqKZ) z%%$-yg`)^gKPBNKY2cR|d`}BNV`=F-$e)#Z#%>%aHgP^^G%F_RxmCNLlcg}bH{yu{?0W@EaNKRXp`n}lxpz2Nq!wPhcwx6D`<-gZ$s96XE5Qf+;|h& zvn`xQ#kB8Gn=fp*FJ!;ufibkg_=2{bm*mFWHKykTYhWBS%bmHrQk=P2L_xAs>qu6I zm<3T-viz*I(JDSqbTze;<#)mZ&zsyu*fpe^Yr^^Y?T7`3+SNR>-(KY9PD^x2w41YB z6~%8DH5vY^awzK0Ac@(+M4QP0avk`{T{MbSdnaS^n==R8!{OH7nB~hME<9b~0{fSS z8>%zvHBr?+pE{z4=xRx!MB6eFpCA~k)kW($HJLqD<;7Y!;X@wes7KtPV6~&#>o2t? z-UjTZ7=NniW&o5e#tZsOt2@KxtD~>wp+EP9n2#3AYr|G*LFJlo`bCD|BuhMJdG8Xd zhqZ#{SDlJE6ZvST=|{VWx_u16i4S36lU2`Y+tT|o&u7=}Fg?Q%elNGitg)x>XI&!7 zN=Z{9DCu2HI0_DTbecoV3azYe^oO}@w|eUZscrLOVU^b{C0_cpTlRLC>mP8ad>Hrc zGiz_-^cJ59j!D^KnyNJX#Hf1yi&cBuMaY8kQs)@%G{D+6N4;mHctLbm24~Q-{Id3y zH&SKno=|f2WnPKi@NF-;S3syxT;q}E3HjNQ%_Gj8Mr@vNG0{rpB6AR-YXt~i^V56_ z-*hHv81pluzJ^+fU$x%1A6)c+;2Wy8%qs1CFW%UK{^pbLk;gz2PiRxF#iQ3n`Jm(; z4U&fBkO)9V28HM_~I#3qf;qw142o+Op{-lIHP(vAywZX{D^o#~>2Iqq>5-}4^ zpHE*>vq9AngrNpKY0A!WM0H_zkVbIF?0=xY+F=};wf{%Y{}#5o@?h4kvVj-oOUq}}R@)w- zj()y_@5EGFg~zfx4pT0_U@n~`(~-sdFQ{(n94-NoDF((;M#qSX!{{$F>l&i7gn>$# zH|Sd!lhIu|H`P2j6SK)}$tlVkW!nF%h<~bMGj=44MF9&Dy}Y@oRp`>OOwbzs5a}Dx z4h-wUCJWPtJBS= zY&`Gg(ZN#^{zU5}`+Ly1*EJ8RjixNac~6@M&agMKQfin0pz4GG;qL(q&VCC}0l57y zHK9!dde-#K(oi`E^I6WeYMFINtv8BhAn_l~r)zRF^Hc$PHrQCMlljtwc8yQr##$$Ylp1+$F{T$9kUV2hO z6Qys#JRM4emkzVCz(?3Cvd1BO97ySa_QMwMYkGmM7^5rKQR74|@X252n|op`C|hns zn=L4j+2Ay;IA`m@VZbaD0L+u48%Lrf$K^j!u(Kk<8p6ht3^SQcD9!i}pP?cj&`LQ5 z2nklX3&7ONW%MHUO{iRjc+XeLs{Ff#Cb;rQ-tge^PA(d;sH@)Arg;07i%-V9Hhlaw zgC^Wr$|>+p+42#k$nI$^H*Gc)Y99T515r~buhW@w(z|~l3L~L3(PXR*ugO=bw|vJ`>6lhc?flarS8z~W^lgufasK^| zxVNXZpjuJkpi}dm){awMQ`vxYD<0kXVz!%h_Vq?J?*GHxTZcu}zHOs`2+{~j!-xn- zcbABil+q2-NQX2G-Hmi9-QC?o4kaKRLw9$4i|2WM@%+Abzx&(A-haHuzW;E|%%RSj zd#!cFd7amFn%@22b1Gvw2TQ3%4v5ip7+)W~YvyucfKMop{q))W*f|zW{AmLH0s&ln z;XTV&3Lmn)qkqy24u~B178V%{r_I*TNj>mk#sa>4`naHy#do%rcRH{CTk?a)@}($? zi6V)Qi$BcuxYhGm@Th2~K4axqSBzB#v8-TY@BN)yaG6uxTLGog0upgi%3xpvfh}am z#}wybl!Ei1>b3FabEH7awLtU{Vu7XUMc(}SW&2Ssn|pg6r0zE3lS$$zvllt*j(|+% zHne7Lv3t)>vp2_})R@O@b8Hif=J+&WFh`*_Bg|w)kekM@<5x2Yh%+Gpsv#M^i>gz) zu^B{#_E+-S`x2mp)5%AsPc4A~zD_ofmU!R70K#lbRmM3W6+TW36@VlTAxA9qA{&Rt z=T?4I2P>!Y>@$A3L3142^FuRjlXMspN=Q7#Dv_`HFq2oY7LdoQ?0@0@1%SZA`Et?W zGKT`b<|gA>f#kU}S&+bb`$Pj_vB^$%G2i_Q)wqML3C*2S)eHM{J64{CN82W8%G6&Y zQg%C3Ofsw-fF?_tM&C^%cx#j4{5py}gnIr|x&%RU@T4}_NI7p&(S z04`=gzGcWLy(h(R0=~rF z>8w4O$M04T9$ywl;ka~}AZDLjvhrQQ-DA9I0p@$QJUk)+k166~h2vxcY5*VrS*e;F zj}KE*wUe+^E%lgf6Z)^Zg7#nbj6>x-0UhE(nFjaTDGz<^SLq}l*AKD}M~9)LP-a3i zvsxa&BZlc4WTkc~5%==wFPe*6qHcwMa=9p)KbYZiV~%QZU=7FP(5h%iDEdxM&*HN+ z+K%X?54hk=OAP9_E(kHyiy=b@+Gx&8ZB?qCLAvKksg?GEW z4jUZ%(Q_b|y7uX*m;%g8E0ZRWtl3qM!+UL}r3>+UOstk~;cwX>9^Y^eD(1^^4pzhv zubcy)RL1{cKL7)g9(`im3O!5o=g%^y924JW6EDA6+laVgor6vuM?GX=JXR*8f$Y5j z0uYXiG-RHb9I9pw<^P4_#}w!r7Y~fQ`**&v#o^>M_B)f|G*vz)w(Wc_0;LDlX-4Iy zGS9ABE@oCs+VOx|$ncct4D!iz>W+6{&H&!MBahiFkp59H*&?T@7O+iUD>I&F691m$ z2%iO|dDZG*xA<$;G*^dpT!T~hH65%zzBai;q{6~%V`?K7@26GH!dK``E5gc z%OSFGF52X=^vo%1?`n!JB9djlHJZ%EiMFk;<#X*&RXw(>FD{6NB@q;46IV+36D~ z$y{d-E2||^xqh40cPS$mRM&+>NcdfSox4bq;N2#Yax>`N-0zM0dUrVG3JEteT(#`( zRSp-hdU&QCRd(vZMie2Za#KU#stb=%EnB?(#!x8Og~THVmt?-io3x&BZr5OVR5z|7dkJiH!53e4ilQb1os`MpSRy zt$ZaT*v=ZTK?C=iPL1sa1@g_OXPXs+IpI9rwE85&D&10(LW>L%I9Sk5P z%_pm62ly|>t(iXR1mfBX^+Aw{MM52fhl4|3U@CV~i9xRF;qca~@#{D`e80L1DSe~yccs%ZyKoeb=)Duo(kW%+slC~W=ZZmKN)1rS$> z8Qw@Zo#Vf$vIqSI<{-4K6(0axiodUo?7tKe9{1qS^*^85Ze+fN^x;DZ)ql%F7XhA{ ze=XCnKN$a=6eDP`v-@A)8?8xck>YQTQIkRm5CmAXcS49?jQtzQ3V7nEOha%1TpPHU zU;DKHY`XQ<8p(4&W}@8oD{Ue3Z#Tx)w0~^bAIB>7$3_SMUECpyh)C}Tfa-6O^7dF`#wJetJb3~w_&cVq{81yNinTQlzjqUl zPy~PgftW8xL{}WH2AuX=zBs6=D42F85t=GliC+GNkkmh5I`GeM053RW=pS4FAT%I` zB%wdn$@wqbs{dZ;u~2viqYVb8k^zzABSI2&ovST7*9_lE4!ehok2y{13Ic{-@{Jy))O18FQm+zoj-iK3o-yLbe~U@ zvVcW#&Z!*In00Um>P%bp=j5!iLh5}(wahN3((%jOO7A*!A|!C#dcHvbzTY49{i-k6 zojaVwK&O&^T~7cI?WpH<9?${GykgdnvU^`0VsEDIRkzgl$6%b_4NUk7tB6z)-rvyT zqqQBb=?5@yomVaK&7cIWICs);~Tg7LxDe$ei@>a05XrXrvNwrNfU^5X$Gf1fF_#ow*@m>%+;xWxkAtRPS0j zP;=qv6$6->#(r#Yw1PuFP0nSTkR~i$U!Pr`8Em?D9u28>?+nYVV*xWtGws28kS^TW z>0hg+;ZLLzE;?c{jZ-2}G8_=MIwQ}KzRaF6t5Yy5e>BshGv9V3^$W(O8ZFOV*5%xz~Cv#2AK z5$$Dxt0Pig<9~v-%b=-V>O3NC($CMRG&%@*O;H8H>AyG|fvFbO^oBZK)*%UX|L%Ql zt0`=c{;bXEwO?25qJg-YT#pbhs(uNnl^M1&n7hIKX`$epc>b&}$F;WD{CIk}k@c+O z@iN~*l)0l9<@eupPW}!jJ)gfzK~b!%*~gmXUbwqfQ@hU#q87qQz2%As%mgbw!=w0a zWWM4vLEK0CL~27glmr(NmfB3iQxVn$E?ssyMf4Z*(?dzb)GjU09WT97TcX0D6J!2W z7u;GpCu;hVfN!y{6YE0nVI}^uq5z8mL?Vh+h>}vjtJ<8<^-J%qr2eq*r5zrH}2BIy4vq7|luTY_{{9mZxKEN*R~$fKhDjl8TROLr2+N9|9m*`g#|^&PB62 zntT8x5$?DVX<6J{+l}A$w|1bXg`B$?RsFuz7k6D@yCG)}m5v8rK2KgfK+^lsVV>x| zm~pzQo-}<=gZRl(C0GkAOFxb;SIF~>2%0*Kp-WDWyansU$%Y2e_k;~%L&-S~*mPK-l zGC@RdbG{b8_(|vz4V9S&LeQ2n(j2aci5*wx!{$)yqK+g$oevn9C0ko}P(}~|8LP&0 zyQ#TzfuB{|{cuK3+DjZrSyMw?JjTAu6k*}=AFGPJy~%uos-Xc~zHDlLc4*IuI?B`d-z!2Gx-5B3&aX`7UF{WodEGR2C z+g2A$p3FscylOMgin+^VAgA7cRi6NXp2vc|u!%k&rwNQXFXN7ZNh|YSG+^ws8?;oV z8eGNt%&KwPIaqK%DXYqz^3*{2eSik_aI z+r0X^T{tFhEc)K$$fSK(X}7*U-4Xr)&#Y$Zu%v(|?HX?SFcX{&!PYA|Y9Be~Tv_3h zKRtSr)50G#@pNijs07&yi9&c(q>g7(1Fz-$1>1cry0G$~Hua=sO94eOhMK{;GMpF! zA7nhYIXTODm}PIsZ?$Gud>4a(C4nL+2o$)u@0fUW{AP0BOvYT3E2WQaIxTtl+;Cjt zx5Fxi*T?gjvE}V>-%EKm%MaoRXswjZ?B8d}LaE=_k+bP|go8b_g*RONaSY+}epp|S zrI4qZ>^XYfmpvxHj9+q=n7!S1ckcz``)4e&g{CFws;w=Ym=`v8*tXP1j@9J)eTX@-No}nPMO|HxQRs@CQz;)x zpW0xGDQ*JKg94`i2VCvym}+A47dg~rhdSawL+DEba7$!q%dqiVEQjmV{0@mc_?dC`#u^Om?txPEXc4$9gYB zmJ}Idm$sCOKks_=0MIS#wesgRVfUm+WxJjixHL&v(L4iCki=5Hm{KE*`2aj4mWD5k$$>Wl%s12`h^TBy&rBRZvO{ZfO^y*G$ zsTB?If1^fMuqIKsXG?}xwl*wTuFhS2WqQCf@2?4X26k8g#7)O+O z8`iu}Ee*RHC0{tVxy>;{Z?z0a?R_B7{XN32b4;p{GUSL^S!EYxnJ1xh#aS=TUa{x# zc=?A{y`AabuENRlK9$j+xz<sa}11R*J0X>0a6oAXeDK|un#6YB=E8Upq6HMwC-@~U&C zn5OEOgv#gE_?0&UXS|hF z;I|IbFN3!@qvK}aD7-A7P49E>G-`6EkDRV>i3o$))-1+$EnlTFXdDM^k460eh2^GL zJ&tCS=wwSQEFTFRYlu;iqxi_bbl^&W`@>`T_3Kt3ZDNmna)V|DX5h}XN#ejNxVV55 zy+K3ax19C5ds<~m+--CNjGaiKRM6*+iEj@Vg2rCKmkup>Huw8J#O`e^(8USDHWbuY zLZ=+6HnOoco`W>)irlaCyTWvQ=AV?@_0c<7zG$=z&7LB4PrRJ8rv;I9R|q6k^j<9+ za=`552m~*D;Xkpjw_<3HQntkohd)RgL%?uAbXiUjc<}JnJ(#+D=Bbz&{`DLZRBKUw zyJW*Hf;hJlVR(6liH`1wz@3H2>nm6GS~Wze{)hx2b6P^d4$kYj4*{Ru$S_RNFy;#j z>Y&z)QmyW)F`fSl9XD0Kw z)tUwzC!cG1;6k1sYcBQabf&PFwGE`_{e%S`k?>S`k5SNIeg&WLvep){<(_?qbp@v9 zduRD>(KnjD;)2L!)%MBu`x_R?$O@etWBTM~IP8kGsNOz#wttMkw5Ys6(3t882g~0` zZav%rzU*=JUS)cUJ=t7fJ2ycNIj#C-N3_#+|6Wj1_2Y-e^AA;KSSUgVAtuC6+cK`$ zBD9Tv2acLABsf)tKDX<0{AC zh_3S82(&9mI#NYbjd<}un%7+~og^b)O%|fLBnth#33nyFz<_7kbGIgHrF{7-(Smax z7DTSQE3?L!=)PINH31a&w9AWe?78p3?oiwsof32gUhIjAl&S+(Yi0N}99Eq8L!~Y@ z`vz9HQuDSi`e)GwMc}~PhcM2#Tds#X;je<_CV5)gqf2{X?e3ANTqD0Fa2ID_O`!xy zp0jGgiL?Ql-yeh=mIFuw6o+0S?UpBgIE9>-=0n8J+hZ zFj6JxmuQ%Jwq8odP=t6V7W>vP%R5yVLJL3k%1AxS(4L2Qa3`^CyJIZJDPuX&p@4+0Jz&;z6mq6LwdSnO*(& z`2LeV?0>j`t5`c8V%XAQIcSWPq-PP1axLssFd(qk@W;^>R@6zpB{xhno|8wra-AX2 zWHv3ZX3un-GZ@e#IbIo$Aj2^JP-P(pc!Z4&y}0wPh`RK$IXShs6uV=|&O|Z@c^-ll z2u!byX=^?^ePl5NuHi@aAQfk|>cp3Bsg%XeDllS?!844p`(#R&F|+owkD*<>R%tr* z%%3f34oA9rKqR|(^D)RUGAELhzI}GRh#fqQ))uJagf*kH1$c)9+P1w4g)&FtphAnP zx^K@~txWm9kVr|0XS{c7)6Le}u>Kt{8bypQM)y3G7uy8xWI`(lHSZOlv5@FiBRBR7 znOd`3cxLfaO2&OLZ=#n9kt)KvH{poM?IT4`=4aGdTvAmVf&Cq9As~kLheJrdS^Ayq z*ZIW1uwg7;2F{3rowh7$&0JDZ@huWyNSejj+;mlLi6-oyvbv-?31thpOT{@N>orub zdJc(2A@e_7j9y01Hp@LsqSQv<9oc9Rt};`0ZD|Lk_gH?!N(~Sqcy!8(1S;B8LdvS9 z_=M6HaiG>hzOl(+=sx-l551Lqr$SvAgKN{)d7J7=1_g@wSH#NL!?dHWnzQcDE(MgE zb1$-I-=c1dDdu44)zMR!wPlCjXr8ItD@HM=t`itrMy31lu*1km% zfN1p^z9VI4%m8i)CJAne3=7vJ4lDS!vgi8TWv!k9rZf=y4UxEkhYJ2eu8zNg-{!cr+wVW;;bEQVA?E}`SaTw20Hr!)WL`}Gy3-%2hk zEZNTax1Sb*e`cn2D+8g>13h?S2$D{PSZy_#Oj*6u1auaH^!Gc40B&& z1uo~7?x3&Oz;-sm!h|Q`bu%+llooxTN|vOdU!sJe7Fnu_b-|KbxQL-N7#{@ zYzL~&$_n+dH$H)kCF|s1b}!8T!To&?M#M>1VXcBeZ8;wsaqjXZe#`v#{UONR&197c zf~cW`gH_GwS&L0*bGz)8m%Th^?s&t7^|i6I@fq^YmXH&P!oi6bE3w%|X>M6ZJbxn- z3&ld4)dX=-DW97&uW9{)eMmoSN8mh3j+2+|(oGWK?f}%DkPM&6l+A}ZqTqzdYpCPfPo*qGKhzHS}^(9LU;)q?}@ z<>7LYsC;2zAq`sn@+Pz`N5=)tG{poo0@Gngi)tc9O{TMj*VOPUUZ&(p|N=CVLMc*iaRkHJGBV`(P3hUwaT zlZ4qEPQ}vxo#8G`L}!)Mb%CLsoz#sqAo}>Nh9>x3XJbsPNcu)fJa1>oj_Ser;6bYZ z6Kra2X9K(d;pY?cz6sUujorC~xF)|XE~uni67%QxKz=Hf6PfT0UR%*}#;Z;U0#oDg zn=r*74Q9rfF($b{Du#hVB3v;aVKn*9Qql{73tqXiRy?a8_EYjXvI)>6&>8N}M;EU7 zzNwI4vNhBubv^-3JRFPZes&OoJS_0zv$o6mz}emI;g*L+>0wRk_B_yM0p@D z4F}9*>xJ-fMGZL^eoY9&Uh|fd+t7+h%+j!)R87j^@%#oqw6*$WZrHb! zEt?V&<*@C+g3rb}!E|;QR$*$L8phy)Uej2H`?P{)!^=5b<#8E`Qgk>GYlQ+WvVNgU zvJL;H@zB@t-&u(@TIr};G>{R-xjz4j{P|4kO=^a&V~4EYA7dG1bL_c5vUMR?)a>?u zcHM3K7z+^vT~%&o`ZEG1i^v;+sFelKfZ%Va>MMJmX;urA`(|WBRx5sWg>>$JWhQFd z%fG!|y!1d%^5XCG3~PnQ`L@x1h%oeXu@ zN!t!Bi0FzSzd#G;Z_x6GEJ1ak>dQzxaf7uC-{hUFXTBvu0joFsh1pn`&*-px_0xCz zE&wVQ^>|e=SB{20+^N@5|Qff7i15|Ji37Z4ygoPY7AB zziGRi2;?;L1hB~Q%bEwo5y08aE{8gMa4*Y52n@L0Wrh|T-}vP*)eV2>5wxf9&_3Jc z48UQteJ#>3^~+V#gh*B~*LqNUMqh_Rz;Iw~nU&NoC(hHoxKIuy34P(iGwy#_3fm~A@ zgqns{ipcn9hKGWAq^vK9AdxlQtM$cI&z`k?vILy9qqWW4~-%nN$ox0Y^IEDt^;4dW2}+ zQxMtk!EIZ)6}8$z6ru-Lh^S6hr9Ri3t3GnH$L|lO6Wy03kCukVbwRk5nsMgiOlvx< zISZ7xii`)n3EVWjtE)U|G@mbfKv2hW%u>pnUu<-^V|1J@+9v4lqF|4567ZhM4$zD+rL4 zDh!(KK!>F&Bo~R_`7TT0HN`8kVf_*8geKh1&CQksYY`_OpGRod9qMW8;ksZrKF8NH zQV;_i1u&eEk@?%m<;$%FwzzsUA;B$hY10+w-3k(yed6oa;?BVqS?kxh?g!R-4fe12 zyFemJPesph_`hkJa}X7Mm_Nh*jI2^&5Edpf@vWyVka9#~CR$QDnLnt@XZdt%%IDn^ z-HNf@wp{vzbCU7<>X%XxJe=EaE;R$CK6~do6=T-H7e8(-By!hIji0> z&qlVnYgpP?LyCDRbp(9#H9cDNd{xMv|HcW8w5tUkU#Qj=k!i8y`?Oubl7>`xcZ#>F z00L))Nx$Z%hK3Y#mQz03XgVp12B7#DKLV4@7~(w zgShT_1_o=eWlvfd^J2lAw2Edxn^Wz32OE9^!=qr%ZKkv!yT#%;%jUj zUh934cS=ocnvGY2fz@-9p=c9*6~_~uaOriq=MhiHB&WC6GTGJBjlGI5nNI%XXjU-Z zG$u_CD(?Dh{|=j_*zE^usEVr1gB2B8*5$c!~@B%H6mx79eOIQNeu*e=c`hOWk%4-XmN zRG7Xor=vRb4+v}>l^L7cNZ{SGU@al(o#*n?E3^MGtffVBXO1(TBmKof7ETzCB6qvd zroh?a(*^n5-0mY3O=5?0{LB;-K3#}&A$rC3$gjEj#-dhWV)#}1tG2#o^(@IhkX0fD zA7!8s5#NW`B*pKqY+4Q{PCK$ZAdFy$nl26i@V_)e*v-g3TqNsF$HcvNQQszvu&8iY zFA=}*`oWhGgkG&%P!>~fN4yi))IKN2W(_~dcwr|#!WlQ^!SLEv?0*O!14FdC9+wrjV2v?3f2l;GZMC{*|6RqiTP3VmuHB>+*)I2hk-$?87P-FcZ1**TH@g zCf){L*66{0JAml(?A1Nrn;8e*45skj4Nt$d z>Ww2CoYwo-abwqlR2w!hd*vk_%^N^fi!75#jN;Q?PF2%+9W7Uyh%P&^=~v8G%zD`R z#+cX6DFx=N^hW%GON>G|?AEXnYZwpt5BT+*mL+a02B))huoH)zPb3qtQu5j@vPZu|6N)3`P0jy)0inc z+JJUgfoG=>SVq5XGi17Vcz8ieIKL7~jVC~zZe{L%W8ho|nDsw&fdKZj+OqJE#Nx@-Qn=%Y^;@T%Xjef9N4$U+0M>zJJH z{PfeQ6=Y{?Bcx>!?bW{aa{I6Pe$BIL7f|A+8g9Zyw7Lv75o+1>e7+fxeWW;sInTwz z(JFP$(B~%#?K6qO!_uoqe0=(6Hc=;gs0+?(FY&-T+_eF8ukpYC6h+QZkBd3iLW0t^ zDZCe3!D$J0@|RUXT8hQsHZRhS4n4yb7sZn2>qz^fx4T{6XLGcUb0N*yzQ^$*sD-9uF{* zAe48uUH81(I-|1jC;~ zAU{m-GO<~1Iz#<2WS;6#Do>Y}W9)yxN#g& zJq_^<8G7zxY`5TIwl|P|3S9!O#nKd z@BA6{N~XqLlgD(sqB4VeZ^L(xDwioisfBH1qapOjEqiZ_4NL^_J08~p0fI=pFRJt`gLjthIeKiR_H+?22{C#x@A z^s;PZE2@uwu`=3l_>6Kt$dP923fuj?3`aWZF<`$|Yn`_ph02$LT2Aje-zwvao%|wZ zvV|zM9{vpO75(ym3@P-T%znp>#P5#`rr+}0uMg+n-@Jv) z%>}ndi3=wZC;5-%+acmvQHaUP%Sr6 zg(Wo=PK5&7TLICG4h3C`f$8YHyie#z;11Uaqsz7uf{W>1Yv3G2-XgUzHQ`&_?Ha7( zr7M{tNqrdltu3Zrb?`BeFQkc8=f42Zc_jrr!oH?Oj}AOjqBNMTU=csd&|;%l5;>IV z3~Hny39fi+P9m#FOozk-^5zr6ojX;0NhBv&Dxk?H6tVlUY&!62o1j4(^y%PM^YKZu zU9u(UFLtppN!fk1a4UZPQ6pY^r;++ zUy0vF^CZ^pBwm8~eec2U16x?C-R7^Hmmb7F zRfY?JCL8~!X7c5(OLrHU^>jH@1}mgEG6#$8Ia^j`+%(Ae`Gcv?Kjey1Y|UrC+)-Xj zIkI~m6*BNY0nvWvlRVaZRR>;7!jt)tHz+e&JkNIrt-m%LH&%0fuANvh)| z#pc86VqmQ*%WRBl#hiZ(rtxhG5YnNhq%rl~-(jZ_fcY@R1x*({B()D_>l>|j(I$VX z$e}g)K8?G_vCGC)clYX1RqtI_8^CLFd= z{~gTj{;-A2O&XEP0D}5z+^{TdfpJ`(zPPW8`5v(m-P8%~yD^57B@O}= z>vjm{oZWO;nCh!?uEPt1R=SeIrEei*w+; zT$3!QiABXY>Qq}0`sC^j*bX@Y{f3xn-jJHff?5q(eM->hdZxFY=5l}aWX7)##oS83 zSDb1gJok5xBVp?gxb85PbM&gy$O`f_;XTaB?#O`tU=E_N2sv8dqd?&!XI#~ToO1g&#}@p`(^T&G zCystFb$oUrbfBL`O407#!?N1{?H1N71My&G`F9kZb0;X}XAWuLjU9iQl7ecg1AHe6 z{7+eiq|Ro}OuZ)s%aJzk5u=W$KA`nKW75|& z=Avqgf^~*GaGo5s+hLh}kB>{~oDPt?n04{1-BUa`cfS%!69ER-#K+NTXxsJQ)hE2A zEi^SO!6a&Ji~7_JXt+aZQa(>T9vl2C*LV}!*L|nWWfO+#%D8g|)1g0v%jq91XUmZ} z&wK0}95Ttc?d_w~(Ng_e`yMc=%N%;1^%P$vNz37<>99v6f#Z^Yxe88vYbojdGHv&S zuodbtc|rO5IOO~E?`tidqj_HJgqEO;#jo7$1L+Pn`kQ+Lf_ zy-#6O2Ze^a%!|V5B@E!kpzZ>kF|HDdPB*@Y&>Ld?hu>jiS(N6}J0gk)-0E2j6i3Jm zR*%-_HNMzriQFY`E~R^#Q!0KMne4zE;^h8Fz+Z||tH-Cw?n2w|flB(lH$dX$w31#0 zSQK!AYUQV!2g9P-&DbTbQSnAkpI2DZq~CG9oKn&X+~P_R_po{mD=vN!=Uq4^ife{F ziKmP?9dotIhd(TINH4;+sM{>yEeyV+h{~QNsO1RwIBtWV&-)<=h8-unhhk>=I-ZRG zR)Rgw_OPbrhom8Eat+?`;k>?+fgL8AgDSzH1j(0$vL}EMU$5L_Oat-8LD+Pb_rfSO zh1_^!4jhSqT|2zA((C&!gMWSuMUTGc+3L`S>CHh*olrT; zuhWxN-RI7~8^%bOMg{{u`NOI-_c$KwNwt5;3JFi;^zpJ{4ZA(*9~%aM;|Qn#5|VM} zw;_-cm8N_$soGlCPV!qv;zO$Y^Rx8#Kv4yf;!0c6#L;xVi%2VuxgJivez?(xmsb{J zI@@!6ba8xNu=l+|+FZLf3_{)dv7=vu5WJ{Q4-!Hhx5ITZfa3EX?7iBUm2tS8s@vVZlu}nYdf(|1z;kO;CF$Ck^6D7Jb?2tlN&b?%&d%S=qYuW!GH$ z^m#Js{!f&EAFqA7VIV(as^i0CokL?LRjFb^A_Ef0m=(WUM!;HUNiu#a{mBN=`O&8~+LlUv-_ z6!P(DL5e@4_^Jx!ZbSH)nDE;_jtUPYJLCbv&di9R<~u6hJ7;n`B)){3DPz^9 z*$=sm$aX&DeyxUSH>`|*U73FzS{+bLdPSG?AS)_~uJh(;55tU9D;<`zZ8uDWXz%h0`DdJ=yrkAFciWIAh?b=@v%Z0QZl-{?{>Tkr&Xg{g1`S z{Qu@N5fOZ#st{iSn9}hP%72}}E!FP8L-|iYv^)w;`Y~y~>OOt)h?D$t^(Kzu03hAp zSL?fPXUl90x&M1F57SbSb;=R&tN-KlKX`cg+@NjmWqj6hPXJzpPyZJh7fegGN|6r= zylaM(iYlO-hHGZv)#}6d@Bt;8!E%43f`91<{dIFiZ`dl8F&#k0$#O9t&FSA3egYt0 z{2%N8{Ij1w(Jnav>qO=WP`vbOxE@i+v~u=JB%3zvUTPF6;tc-(xNh}bG(X_yoD}G- z_1fJsWdYjjsA8>bDca*mhQPHssGp%Ho$vGRHnmdq~Z+d%}N&8}? zLNcNlwOy@W6jo?$aBI>bH8lmY#5oe4oxV4Jxsm%ZDx_!3?{ZW`xBE?8HZu0jRM4D} z=CF4w6J=)6I=09_3qR{FQHuwkS6hH~HrNan1ao3l6*zH6KY9>PAHj9RcV~$;eJEdU zm(acO>YASj2?R)@w0o;}k;47!h%F0I5x9pt>`CPj_U)REmrn@I7t2Dci7meCWzTjp zjdb2~+TEKO{yvEqgSxG7^yxs3zng8j&y(TZw`LpgnbjtDZk@!l_~t?l+Go&H&QfUq zedBHUpR#{_%86Cr=n8BIP>|Le-d6;Ov^qksJ{$_J5r*o!ZGj{VFit21=Z56(#)mgpzxyY>Nv8}dP z*<0HGh09Wky?nlvYUjk9|J#N0v1HOcv*ah@-M~^5hpb7e80nu~WQ^qc(=YJetbtrD z8Co$romaiWNhI9jd6F$&e98Wn%e~^yFQR+)CT@=&evbP@$ar)Sf715@x*(!9uE@g; z$nf2jV%%;X+mhFt1;S)RPabn-3eYnuQ>eWIN#6V-JC50`_!jOR2b$n;50KgJAK8EX zK3EaVUKp1kO#MTh!mjsljyjBML|P7$mUj7dT1hRCU44otwPT=%~xHg`b5f zHuC3q;<2^y?KWR~uD}zrK)(l%wt{)awhx8yrN`}jxCFX)LDgv5hPQV5S74C)lkymV zc2kOD7n|P_MHJrtaaD`L`4tty08<#2(rVf6aH?6_H&wa}OS0a&Y(ig*bZByB#!+K$ zdDh1pq@G10+2k^7O9E!^`s8K*FQu$UE$0bvDho(>;9J};7jtrsJKe=Ht4)QpR20s1 zJ(f21HIw5sFwKPbTMlO8DLO-jTQe4`FIAT4p+*<@c8OOrij(`a`49YZPep~yDJ(#U ze(J-FiK<2vrl}ZG_WY^EV(3E+)*8cbtds68RU-eii;pjQ6YJg}5Iwh;9cQvUl)P{} zAe8zrd%j^GCQzNqiZ_(rJ(v%Jnih@Qu9i7p(HqfA_;c2d3S_lJ`Ub%*eq^1xOO+hQ zk2a8^1P1)5HBAUh7`zZZOIev(mvyP7A>SG|*d=S5@d*&0m-)19?ti!J(KCEYJrDzaQL43BqPwp5en>d2^ zl@*4f$toZ9_N+U;9<;w8f~1tzupP8po+dX%Ux{?05VJ?DhMf>#f9){X8yuNZAuDD$ z5*jC%IeL>H=TdiEQ~C9NU{F;>LUmZ9ZlQGjN?l*QsN?4W(qFvdi3K_3nhLk&~Mah>iY{%j-bGob!%DGX(j9zCj47jTs zJ(*afReMBuGo9&Zih=nJu362|Gt%i=_t-F4?&Npf~*;_hX&@8l> z&4c&qB#&|BM6%O~!HH4?C~DjvnU%Q)u-UKA@=wqvK3sG;MSoPcnd-%N-VVbr#{vyB zG#g(QH7DsFTOyQv)KqFys%14=Ne`SmMU4oT^3Pl57g|=zQCvM-IyKn5rp}SB!v&aC z*cbUevYk3>`BS+t7pE7=JLo%8Yb;cPc25%RKrpKvTd1xx4(=DNo3Xy=P8DK>h?bwb z{dfMP&N6j{X55n;JRI=EQBtvu3G>90N9m^@FxHz@i(FfpIug~bHb3Jnxt%64wZazVg zlPcq@x~%oppk2%|?XYfzwvy0j1ter?@=^57{o!zDu(6FE5+kdV@wvPU!!<>%qeb`3 z0bG2lm7r{!!-!wuS305HZ#H*nFNg1TpUfi8 zanCXR4(@N4bGi{Q+lO{9E|Hc^tAMyJZVzn+GmM7$fvekUW-^mkdJ?Y83eO^}N9EB-T1e*BICvFb(|dVR$z6lt>K^UJga0^ zSuO5-g?JXo(!OU!^*x?8FGzJDq=+$VUrP==sB0I6Qnwh9 zBcMga5Lm~u_2`>H&;aF0t^BIu+$v?Mf#WZ0y$tgW!w(S@spiHc7JM~SGy^?}9P8E! zRVTO(rz9e~GS3a(q-;4u@)6{!iREH}RuIzAzoiDU2hlEcmzR+}ewfB2TO&5BNkDjb z>zB%=xmuLmJ*Y%j2aIJCjLzLJTD^;hRB~q<@k9@<$=NMZNy&Wr(J#Coh28Tq)!C_^ z1Wb*jB zjD_yX;a*~LGI{u2SWtQjr3Ua1j2NXKA9p#MmyiNMCA~t=VwnCFT_{0nZ@76K>@4$e-W<~h1KWCKgOxDJ-cbtSc9kVeM4yh(g}RzTan>`BGIAl`|WO{i1Gb3u+Y z4;GsLUzL4#R8!lta6kdY!nGh5M6gfHQUX$gBH#^869Uqa-Xk>- z+C?cQ^iU!}0tlhSfRum`fiJ%I{N4Mlx7N4TH-DV6&+NVD%*>vdJ?HFm5G8LK7*7Zd z-}$%gpx;Pa%fgPQa~~5DzXAYP^MMaEEMnSmdGAf22A}RHO z_WDpGtx`3J1qTecKXNmw&aNp@%INAx69q%>V+GE=bglpC*!c1k6x+8 zPALdmTNb^{-neLOb#>(-?w&tYNn-0~9-`LJA!f$)15qeDKj!LN`YyZda?2ddqo(uj zsq+A&;Nmy2=Maw3g` zcpKW#pq8g?Mo3Ls$h1Ppo_l;X^=Wz*I>Zx#`rKu?F>Y3WajaF@rRLeg?&uen&2=tt zzjb;$w!v}o4I=+K^Kc&|)IQQUZ#eudrC?a5ZRv2d8sTfYLO3o2!}Xxnl?48*nwgko z&!h2-%=g<-?XB%)U21%2f_O&Z#QkqSP8V8F-mBM3NGX3sH5`e%?BiH+e04$@^Ij@+~aMZ3_B3L`pyMc+jVv%vX5fR*4tU1i0>u>({gD#kAP8yTsy6hLLC0ck^$^fbVI+Wg~Mc{zAB01oz878Ag4ZG`-<^ z{xHWNQu z(5J;q31BH{Ds*?GLtuuYE^yTdxLX|-vTk-R%oVmIK}GfGTYAz-c_1E=Ex{P$h|Wc8 z)CaF4oR3)}JI1Po*CPxr)j8tRcWW?*U9**?&Xu_bzInI-nz6lDV1aMsoCa9Trs&YV z*K4iXKJ|E&m8xn})Xm$fN#8R!p1$eIp9tFeT)XCplm)_35v{2vwpBhkKBA3&Bg@lY z!k&C6%*;HjA{{Ngy_>fK%jE|Z5%`}y$dk>dzof@&A523zEOD2h_{70kN(N{@K9x9hNA}Qp@W9ht@f98$9HNTV z56+Hg!kWA`>upo`dCk|nQ}QBa!qu-X8H~n-qV`=isK>)cwE?Cd=27 z8(P9{R1<+4OyD0Io=AvK#_{_mi`E|G;X8*s5TUHEVY$7pv6MvoEptFLhNUrxY5B#A zkNY+)I}ffG7HH{0vO0kiOMYe`lRjGSQy<5CuGy)IyO-S7I8K0N{O;LI+zNel(WgDg z7$4P7 z_<13zEfh{N4$&XFBOV%tahRUc-W%H2E7x}%ef8Z!W&80*=06%m4qan9K2cCyX_3XQ zeu)TL>l9-lO(CxX8h@p|=$M#R#(k#KY%QIC)q!FO#zDlSAHl=(WTCYny(pvpw9W#j z`}CcovAxxvbE7<)YgPw@NytGzO*;%aDV0<&NMcm{4rwf9?a+Q=>CS;e zf@8S;)L+Ln!b0aPT*Tj${xSVnrBk-@XS@D%S!$_n|G`Xu*pJmvJvCC$$A~VZ9!v~$ z5Mo_7rg7pwLI4nXwJa27rA&l#rf$78 zh@lWF$u@(#HB}NCd2YOg`)h+Kc=K&#XG^ra;cSD38?Nya`ew_=-3 zAY&qMi+PYgr?_-Iqny^+5TxSad3xqxGE6mmV^=6oOj)u5+jsJR$jOcrOeV@A{L0j; zkFERXffrkaC&G^Ao--b#Tmg948`0>5FH|qi%jtRrF*@zCP(;O`Bf+PsbDuJfQ1oT| zuVj|euU(Fpai(SU8@W4PY&f~i5+k$Jl#qJ3c@r#@&GgT0gU!URO6Grf^Go->@t%D$ z?(DdxA65&69sTS+)Pt^^E4~I^g_%eMavi$$I`cFfiyK0Z!$@|0!b}3?}{y2LMo)x|RPg zVgVxqr#$|RE-m%{QN6+Yzq@4lf7FS#8MQqD$V{#+J_7(W1az6|@rI#w>>%oTQeL7M94hjS`XYyiNV zPr`bk5?@9+opsNo?;iU6hE4f#*t)CuG{qxx)0!X~4fOw*i~q6RIg)io4(mTO&H@08 zz@+yuR{&3q3S(FSb7xU+@3ww&72uH3IaRCC47xdlZ58?50XJv<1E9i*kb<=*sn+tny~?_30I9 zI;QD#?Rl1HHx0IP04l7D=jMW-B}?B+y)O6W4?8T2Vz-fdr8ik%xXhBjEqgz&+MLsq z^d#s?c=seb!;4-tkoERO%z-n8TPK0+$z~~6g}u8cxNx_zkjI6pj!kyq%7@ZXYfn8p zb^l67ia_QGuw%57aPsW2^=BOg+&4LVgCsr|)zu@h8 z)4I%L&t=@cP0m%6X(57hcC--K$#}JR*0#pZO1oB;_NSyyOqk>CgsxlZk!aZ(yBd2= z$WObnw?nepIuGMJ?5lbzn{e;$^*Aayd}Q6VNYviCQsjMj9F;FlSAp?wruK$NSmgGV z7>$lfrWJ`nM%5a~kDp!Jeh3HNKNRxrU5tWBO*OjDu#vZa7{GuSduN>;D}&dI;#hnd zdBa=FX&B)+yk!943rK(~>h}Nnz3jSojwQ6iVZVkr%MSoVH{BH2^-ALPFz;34Cw!o( zRT!vwb+x6)d?(Li$S~+iYh(Pd$4|?~nxu}U#r>s@0>8*Z=_lVF(q7#MuAd-oZAKKk z@giKz4DipAj?peXw%NFuTC=>UpBVuO-j2|k;h?mTPtDymx31C4K5_)gEF(1S@4MB- z*|$%7Ymu(s`|yn_5{bK^C- zpNiSrlUuX04PLPwOTy5WYKX^OXTpTF)>Mvw=dKcU!>{-vt%R(d4fGyRq>@r{#Io9{#(GF!p;!qR=rS3K3Go&LyPO@zTl*)HDn{N{VD!QW<3Fc5lSImlJM zcNO0)A6unTC1st2E`d~AfK?p};trh>72Vsa3Ck|H0J6)BQrL^F<$(!4ib)c$e{De8 zAfAn00GarK#n286+S?De6bX6S@S6(-4c?;$#SxBWZ{MYOn0yNsf9C(!)&OB8WK5{P zt}~ZL%e?c3pJ7eQ>;qAZk-V(RFu^0fe@fKi+(U4C3EKl7x#5eOj3e`-!eU8>Co0>U zHrEh|^G=e;D-~rm5?B^pv7@bLExSG@Ztn#4n5YK*$f3?h_l1B*n&h9}g zK2m|5D^*PhTc|Y)Ej_Fgne?ilGeI*hzbQNF?9lPQVAeIsVq>>CUyC+T;LZs6Az5^*6?<5rqeDkH_WcT1Km#%w;&m_6_q`*PL^}@zxKI0yO0m-O~?gXZ)J0pZogQ| z=rb=

&MrY7L2QzZ5C)Htyk_DzI-c0veIQv-w&j37l8$X|!hoDZ0IPPe4GO543rb zJ%?fF)9wsu1b6XHn!X9BY+l3nQD1))*S67uxGBAEsDpqTHL`MVEZDn0?8Yg1J>e_j~(OzncsC?m7O3%TV>ue2}J2*6^5V9#Kt037V zhI+KcH}y`OQ5FG@pjCx4Os7fZl0AVLU~_`;C6edouQ|)i5wym;u1wFl{|ej(5;wAH zMFA5GLt5dpTyEPPyOU`8eg6s&zLBr*y2?T$ZP0tlUpqo9=-!KzH|TB=V~>U^H<4lc zesNYr)dl#nt~*%-Rr$cdy!nFxT$!&$ukyS#wYg(zMo?7cNNevve-R115;()}63!7( zk{W7suEoDuXsFGW^i85Mj4VnG(Ax^Nuu<69)QEw!fw!N{NSCF2O@=;PAH*uF=Ih=~ zHz)nQ`LRwi)Nr0%(QH_muax($aZ>1=3-e|ZwQYg{*rHEfsM5#%5$y{t+FGc*3SC{j z&{=T;Y?vUo5(8y8;qhWfwRdHTAA5?WOQ|Z46vdwL`Q*$x#g`&CCGC18OE1gsZ-x=_ zn6MP(sKLi@L4&PSd9gm4yvml1z=i~*Dx5=};p!ORrK?_eTA98!tr#Hh$Ox-=f}jR{ zj|{VwwV($Jk%M;J(TI~**V=u!d4>DSPO@YqZSfr^pzq(+j6z%06i)?I4=Pb>rz;9h zC;VD{lHnWCXe9VqeN-^5b+MKqgG@Z@+`>1X=cOQVD>)G zXsd~sLuP?Wtl0pB!xP8J$&$fn8~f~MwOriGPb&(drKOwO%`p9&v)G3+W zh+HZ@%I2x>@@+)-W0ab;a~zR2Sr^n%eu@zUNu#&l^Gb@{#NEM~M&WtOAJ#82-rpcf zawd%uCmTV1d#RUk{*Mus138uUv7n86BC;(H*34XUrfxX+FI^1f!5p1UWz#TNdXKwr z=v0E^KzP~jb=0;I)9x_XTQ>5ALX>hhNebE!K>+|fG2%u;Bhii=Zv+lkQ}#q$kG`=U z+R8(l6oV2$g!OhKmFHAemCoS2sLvSnOJTWRzTkGV(vb&)%t=5G^|8t`z9WtupG)oY zR7;9>n=hSs%6{PzMkK;VPC;#Ga-L48cshB7bBI{*aSw&uUhs)cp_aTlJ!69z2u$9a z(*a}8n&N-x$^@tz_dGI?Bp!HnA98p;)O^)eqe*sK6s_x1hcxO*_ED^?di%2wH9yTG z3r8XiSc*;8f}$D(5LM}RHAbKgphrV6XoIvV$(l7@ZUu$xF5X6sSa+nLMIKCa2L3ta z4KFd+ul{-m{w0`9Mfa2Mly7ProEEZHB)Z(ejlFx4a`^(PNDPDHc>)({{(0Vexnpfc zp{i}Hm-8=t7cUY5{-(^w=~7a8eOHv0%z?OE(p>Z$CW5sX8m&MH8*JXn9Z!7|>2f-) zEmi?HDDei}P<1WXk6{(4NlFs7Y6{@BQ7NMveZ$%g<5n}bs6>RT+Cn78b_~;`Lyd0| z>*qur0spFdXcokOu5}@pwsM`*#m`EkZ^7GAzX0B+QekSB-6@O3HCo?N6HU#Db zp!iUp#KP+^1Q46u92q`lC^W9DtW5p72`Tl{|DKt;V1hS_ z`!RcyI9LA3u*knr@hhL^wQvI-j~^8f>n{nb6_q1=y~hIsxgrWu_v;(?v&Ikn8fNl4 zi`pt)=GeMk#p_QWMa1V?HyZmQLq;C$EI(>jSd9{|5pKK?I_MlmCuc$K*khD;r3|xv z!0|91(A^QCm*=Skb<*KU+xPv>!Wp9+83lrVzqu19ADd)k(zSD3=`SvTEy2ssGmC~^ z`bK)IwW#CM#<$LC+JDWDJxE#q?QC>W0N+SeH=tt2zTSO7qtlId@6`&B-8Ez3y6 zWZ4O9n=KG*TmeaqVTD`A#_q5am z?J*A;BI6A63YfU0ziLo8G20x@t~1}$(uxo`K*gC=?!xQ!wL?#JKChf)nZ5(ZDS4jE zNMc{*4wTjZH3hy7OjbGYSOQ}(eiDK-YO%(pT)&C=vy3YZ=8P#;zP&dWQxYQ@Xbf15 zm(k?moKIxPz_>EJ*P5(U1lc~SF`O@yhI~-hmh}kasShJg-KF64nuP=33A@0g*5_wb=6*6V=wrtxP%8qpoG!A+IDxU;hOS(pvgl003A$=4fE{ ziiMA#WQ0Yi`ogFApSY*JB4};}-BeeJi7(JERr`C!gdFc+Pav7IK1}tyapcY?PDy=u zc$P!!LP!`v&QfX}^RRU`vs@xzW^L*T@)g=3EDgzTJeJv3f~tmb|h; zqFLJ}p;1uYEBEvg-$XX9!cx_qFTr} z)*+{kn7iByH;s6w?G<-_S~7I)^+I)K4kXe4LI7XTggnQ8Gve14#KKH&Nnoe3O1G(Y zy430wYY#eno~jF&ivJa4z%vE1-(=M4!=)n@n{oce#aexz5xL;h0HMeph-$locdT}DMUu}(U%Q(v4f{ULFI1#YqNPJ7{%BVWem|f~m<&r%3dtxr z_sEi*Xbl`YE?3Z!^&obqr%pDJ%;plC)7-?K*vW-L@4)2gd8PQBEK;0^X4ilc}nYiXK`Y0ip3sEU>1Km^vBirTL<0 z?s|+tm&Qry-wiSL>aB}=bTW-?ahncWlJJYZp*8q4Zv6Fm9?r9@Kn?bX0d&Fr6J}Ex zP&)6wYUNL`iw>JV6dkr4A6+Ss%pc7+`crtu`Spw8eD!~!MF-z3`<2G~2RtZeO-)Uo z!%pN4TP?&fJnDj-R`z<}KL`N8GMtsF7XVvM1WBA^VhWu~N|H5jpY9tH2;0ZAA34Ot^$p1iNrG%~Nt$fo>^~e~8i~A4E ey)<_fKT>=DW40TPQFi`S4%9Y!P^09H literal 0 HcmV?d00001 diff --git a/doc/assets/crowdb-dataset.png b/doc/assets/crowdb-dataset.png new file mode 100644 index 0000000000000000000000000000000000000000..d250502c4f02dd1be4332fc96bd04db7a5e14c01 GIT binary patch literal 279626 zcmaI61yoyKw>?Tr3#E9W5ZvL16Wl4KxV2EcXmEFzQfP6f6bVq=CAd4m1C$cnU4lcf zz~j5`|GoQ-d*6E*8DpR9Y&m=Fb=H}4?iKM-O&<3(`D-*ZG+af64;pA_udLC~UO--9 zJk?m{NDw>~FWlZMetz}z@qc9={`8y7T~^m!)5+4^%hc5Z&C1cq!GhBbUyWp3Elk~QoE&LC+c;RDeYW3{i<>3?Md-@8{epIEE z(fkZk*#M!T(V{7SkpAoqK5X}CGtx@ixijnyTyax#eJ__t@Z4141>K7TY1x;IDr#D0 zT4LWcZR|XB8ZF9ZI#7G#^`*C-9&>eLh>il!2#cSumTM?WKPwS@{Pf!*UN4^K`@2DJ z*pNT$?nq!!D$DAah$r~Pb7|WEDH=&I_`JifsvW66?whr8US2;&Ds|>`_F=T#dy`PR z(qMp7zU?{sS+<8sm%g$RMiN6pv>QncU4Qs_EKZ;3Lw`%ZUE2_gc-yGO>pf> zf4#v0d|3_zfUt7nkXe-7fa>K>NIO?0;hjMdQ29#GjyhI(zp(sk0TD)(EeJGTqsJ#q zfUL&C#5o?o0Mjk^dMDShyf_`*o%proC_yv#2&wokl2C&ly<^|RzVHZnn#meZ1iXSk zpAqt!cRh#d0C+{-)0;)p{dgCRW$fegLzpKGOxGV;|KmeXb7VB#bb!B>DNkQAElYHL z9{~^44E*ydp`;WKD91w*l5>Qd-+33UYiwG|Fp)t5%|0|%`HNoR zb~$6*cuo4h645|myNLNPFkheV^9?ZoXx@qd#JCg9lvL9VaE7DDE4a2p2@c5fkUvr( zTCy;`EG$}ensUB)UREn`RW&UB8(7BCE&kyr4qUI9i`SzQk8pwTstml90}@|RU3 zQ=uy=GCwyGcd2mVxN(hWs_@mAdpSyTo z`X?~C8k#t$%hBPAgM%~Su77YjSMY)Ac@Ew?dgy3hpYZbg;S?NqvDno3K-t7*x)1(a z5v->1z@KJ)dt5{W0TGNWvoT*?^q`M9 zKXlsVSH;h}%nQk{v&ME?)IUCKNzh@2V{K3ZqWTr?Kwp!G6dsl7R@;+0jw#_j0a+$7 z1yaD+%iB8~bD!TLay3JrGoS4nsXJ5@(LBf4br0asJV;6T}VQh zHdVGVqYF9Q9GHpr$?d2_STrK9)xJv~dQI=eYtX+y>;T@YeeQT8Rj{9Un+AqP0N2a7 zL@=7e3{T%~rs)RYrGbf1ErqT|_O-|1oK@l_v75w$%*xqngO_P|m73CETDbIf+Hm52 zvBS71%se($aa+0w)DK{9V8qPCq%1i~DDPFzRGaVkPH9(OHOM-Ani&Ee2c`t>g6w6d zPxD1rzsNEsfVyU1J8khXUK0Iz0rzn3P>vKBGf~aa&X&6^I8b}rO3YkM+o6Qx`t3{dLZzv0E(f z19-*2rnX;$iwN|DV11XgBE)J#K7%q&>QpWO8Tu2k*?Gl z^w6(vMt5|)S#BV=gpLe8sMv%HcK|(6?I;%Okc4wbq7R2v^jl+=f`0<*gFns)uXOuy z2V&`EgM?7C69zfOLoxa}6QTtv{N(6<`XpQ>_Y=C<&TDavWB z9cExV1r(Jki5V7&%+97S#{Z@9CL4E;bdKDWI0&zg+vYwXgzm%XuL{0R##p84y<$T? z-GgElPyXdHC?#*4WhN;jGaoO;mJ$=!39r-8`S;KA(Qd*l<5R+?1sQ?%6?w}L^qG=W zQHRuz$UWrkdI%s25=;5b+f@!BT|CXwDB39ucI%_{w=1e?>a3KwMGgV*e z#ew@W+&ceM2)8c~4dhPATz~qa0#$ zz5^6@k~9k7+7X8W%7xa|Ehp)tiHw(ZCplczbe-swDZ%eQX1w<3EEMB;SA4hLV4!R` z(--edb!o_lgNqxHnS^hu!u%t&J|&P!K(OvF-8$I9+4<{oU+DZh7SFetjWx5v2br)* zMgQ+D>QuFTFYB)x;eKYj$~a&?k>Uq4^DGzGB*Js=@<8y2DKGHHE7lwIGV}lKuX^);Qc&jWTcLw(?H&q&b(vR zym<}6_2`5GWR&#W1)UT>p+0vBN`@jCc}rH%qxbd5HrU#J`Pr3s7iay7v3^x77-1^| zHQ-61Bug_1Wll`JqkE$aF#HR|5T!yx2XIK4cu0LIKr>K79Ct0T{*dAQu zeHC{HiJW@}2TB534Ec>t`-7Y(|2L%Gtu#luttsHnb2^_x8)SbFZEI_EF;*b1{aYpwu>0byZ|0hLL;3x)o{s=hugHpeMNPVp-3q{b-h8?g_Nf#JE)!=yNEOV@fC4NQ zLm52>{H8r~`zl2eWxCQ;M&CuJ#_Bjw?mg<7QOVx*7L(*XcxrR9TE)nG_)Z_yS(f6wSYZ|4527 zPj>c8<_&wj@(>Fg)XO&LDTzrkcw?|MHnA(8mQ1u9p>8nnyTdgUeA1D3+cqi5m5wJ$ zQ;=McsdvEyQY{-WINps@505!?-xOg8D1k(o)#gUa>eP3-!5VLa%j$4h%Mc_~V>Xg3 zOcA(d1eskzH*^dx(n_Dq2fkZJ&+EOdNd%Yep?mRI9nIUrLQ{MbVXwVX7!woh_oX66@e(gx$WMe&YmpJmp;W61>U2@CvS+b43+B3vc%MAGYUu z1JU)dy7JRQF?dX5Q*<93^5{Z;kE_;s?q!q)4Ojf}St{jX_GD*~_#u#5-ptQFz(D3N z?@KuS960E2OD;h&=2Xm`Q)8AP7EaC{5{pGh zZ?Lcm{^@Dk!M^oDElsU)0h%7uE{6(BrvJ3l;CWP?@-a-?EyZWB4pMVr943964~&V8 z**y82eW`Y7yV;a;1d8d`&LaL$%QLZGM<(;?a??#2Yh^D{v5#O2`7W})Z$Z%?xbate zUqX>piMnjkI(}S$94B0jV83I3BZZj0^6a|w3Hx>{sa0yPKy9O!uWdYvT|`OweVPmX zgaTz@&6g72wl|~GIff7_+e|}gft=@JP?Nt_AMPCBFoJ#kx{}mE;JT=moma(nLf-yA z#Uz%QvlW1l*s>x2&#?6cVdDrWrRG9hG%(!&cau@_gWY*LWz_9Is)3VYLvfZL?9|ik zJmC$^T%VE`9Z)h z(>PtEYVR>dwMO))e>`R9uzkjJOHeIAykI_r_QvBN6VJy3Z^1{wpVMwJ-|&Thkxa#g z#`f=bN_zg^k_)(}`e^jd37Z`O{g=$A=v=Ew<)%%HybA_|{^nRnbA>5~}o zlDwav5wxxBnIXq-V$R225%-e!Hv!ClQ|$1Ny_br2P2ODnpel70!8&<-S%8oW9ad|M zaygq|I|cv%`FtNIOW3{?8mvZ z2KGQ`KEqO$)X+IUoBa+mqncilOT%gAmWW`45Y_5fAA{WJnm^`F{sdH4KHP)ikV3}p zH?G-XO!1u9TXeu6;YJ(fI6aa{naA=?#j|>~*YxQjLJ=#e@9#$Q&%d4~(*Al;-s{Dt z!Py;midO}TLmHYLDzA*Cf2h?4uW-CShN(rVBtOd=^4wvMzPwh#woBiUCGIs@IIIG! zLpOdWY~s#xL@0Uuo?NHJa-_9XEcG_FnM`J!EEhoZvHkdCBlWSjSS_1xJ@j3kVz?$l z1fPx5a2m0jm?^nxQ@F%QMP9N-GP5m@DOi|Omt7W1gy@!tmz70zV$Tivb4Ji6v1`zr zP4Ok}Z0)B3!;NeB#c-&;)=y@T7Fqc&9M&Qq3~q%h{*e{DRtqcP+y1l= zX-c)`c=})hNfq@; zFkKBRQ6V$)zYF<{+33eSrj$1f)2;jB zj8W0j_XRH^7$hmmb=pO?5;1*OA@ zy+o<3fI>42{2Q&n*b)ePDUDSC-sWs>Uw70t*@5o{r*@De$DWG$c;>e!ai!2v)hlIHm6^)Lt_n>lUXHPKhq;H5%OHm zp^iAw!kl`X8B~v}u~;0yDa)$cRoCH_#A9RcqLyG^PD~v3`+fYizeeiOf)Ay+WOaJ- z5UXjz%^>++He3m4#Nen4Dk-%+d$5Xd(}1M{1Y!y4ysh`+Z-XLQV#zg&I~{NZ^IXE9 z@5bC3*~*h#GV$Vhtk-JhnIStGVAG0TK{Jo(x7lnjIb5?d*69w~Zbw}?GEZt+Ams8DlG5P0CP zsbV_+T_a}&vnb*laS0puRXwXVO!0;5rH7e{u#;9^{FSnFt+foRGBA`cDcz`O>Mf+{ zRDfxAdLs%x*=J7oSJx_B-#yGm10XOUB8@6g1MdMS2Onq{=xN_LLNz``DR>p zpq5}_vGwao+^@`QwYU5g@8bo4c)!ml@~g82d?q-EH6FWsx@>zClm21d_a8U6U*u|4 z)W2pySo@2X+hNyBZz$I&2+Q^DmX_8@>8dthoHDqZe$1d$QFQ0weu<|@q$C}{Xvak6 z04C|4k#;@V*HpX3z*`_@u#vV&5|3w0aSGr|^VKqb_xhl0*bb6-UAQiVIc-&Qor^)m zPNw5S#;w|eu85|K@KQ~LASiS>Ny5piD9lpK*bbNq?T+L%Kd!PDVbhxvW?UQi-UBYZ zH1b#58(1RMtke60n9JZf~b0dW6Uh&Jb!ma&}OQ#ZUqd1~Y z8G*U_&)-o<$)+1`gPESqS*OONm|NTnD(>Y=N~M3~+n37KSBy*mJcsFP#3owT=3)w- zwg9bNr4IOhT^qfoAP5o=8kw%)Y63d+C`0pKBE$+6UCnt4^>jt&vaWnV#4)-8Qi=qA ztg+VoSf776p5eeX_Iir9v;Ly8W@*!BXX_&@^JG<6G3*6Pv4n=Wr}l;AE(eH1jF^&c zoA#lJCU3`=N(Fb?po$`m5nnPt^Gh@P=Ihce>=)!5jYxc4n6i0|J3)^(ac-?qrI;u- zIqfhly}mJfmAkFSe=BBLQzg*plsI(e69Dz?%#h`;I7%;g*KbQ>4V8BE0B64GQ@g*l zoHVkm0Ti=>(qP_d7&bkF)nC}17zyab zglSHAhnD+3e?x-~RD=yON|XvDMX6jEveY17wu=V&fv{xeNK#rPKi&d@{P?2;^v-3FQ*Y~2R zE$#TR-TR?XsK#3l;W4$n$WdBd1J-qAt)!Bvq1B1}ZN8|O{dO!39u&@6@8Xmcm+<_T zc$$eiFIVC`Z4s(9?YMRyX?vk4GP&oj-+J@~l*C+r@RzY^%>0Sd8|O^2zuVrr>L?au z=^(10-e{Zg;g8rZr)v@q6EFGXTQoC6jHZBB!V0r~{Y&}+Q5gOX+AQt{sn2C-&a49s zjj59C&O$LJHbrap!LH?n?|Mt|O|;PKJ$bECDiZqnz9K8f6!qA@zljNm@At5*K_xL4 zO|#hCfs%sao0+DiEkI@1A3sPDDC!<2Toa{dE4Wfw8-2QsiG7wSL~iKr8iJmO1U7z` zEAIG5Ak-`)8i?powsxfwMV3`SL}57lc{YwkLj?-i8Zmj3mR`eq0!NT@wWuv-Qb&P@ zANAT%XV&K+?JDJ??D@Wi-!iv$nlBi38s>|>9Bcsc`X}_Z{}u+Qom2c-VecXUscKqF zGF5K@27v6g9#ghYl; zUQHYDf*cQV2)Tj>C+V+zXIgo})O@yPcZ;0z^beX;Qy@y-K7q>aswKA&tF%*gIWSfD_ zD5F?u=7d1wZyF40kb2k7q)Ir^G2Kw)#|7au5*ga*0N3PF^XAJWImxmR85Q59^uHqt z0>=AMHmb$v6?h4vXdQ2{&HEiRbJ%`}cTYx$<*E!ALYXnrtP%PsX% zLf(<0^fyu396VlD+i%0>dSvfX*yb(kJ!6AEW0r*fU8fz?#a%RWdM}&IG4oqGezew= zs-nwqrlyCMN%e(%G|8TM1%^7Xu8i6OzyxNlq*DaS7cWFe_P@iaze@PWVG^qi@-p%M zBMaLAoD~lx2{{VUrlr6&n;hb?GyeHiI~H42TKWeEDwnXPP5VbAAX}Q~m3A4zyuR_` z0E(9P(`#A(tz2#ci$XOyy5^4}qUV1N1V1^gyn>`0M13OajRDZI*Enc@ic&dLjn_1$ z^rz}HV`qX)fWwLra5KvG;CX7Y4H4P&pTru|snd7`S2uZLU%y6UCb6*RDoSTcmrc9R z|0>rq9gr!P_9EYNXKQoAxsJ@~e#TbyQX`!h8->LT_2QB22wCoXfr@Z>Qd(XRqURi~ z@#r!-W@iX_ zyC6wFBv|78(~y-i|J^Rqd^FV3iegEN;ov@*7oxWLLC?^!TDa{Ds$boaMWke}kUvom*Db=A0rbH{@@+-7wJ zOVhI7I#f#UM6HTB701|shCWU;hYh9{n5D#bP@lWnN!B9Jvnem}& zG0UJLDvjg3sa7LYMrqUD;x)d2T0r{Kk->ImXkPxfh$&%0X8;#l;3jy`!x-w7qPw^!-K!}{0p?%=_Cq7V{T z&J|9*O8_Th`o(EFRa#=D?pO|ULC5ug-o5QHIGV8DrtjX3=VKx{ z?bi`OG=H5w1xb{3qOsBLMLM>x6bEs+{2(qhzqA1ZLj&Vxs|+$oGjb7t=@f>UtUQt| zA=F>`x0C7O8ZO=B`NeF?2~p(}F>E-B*~Byfe{_a{gI3__B?2u<{DAaavAIa+qAn>V zKXRAGx!|!)ON+GIA?V@gA4Yhs}#!ohCl<1wJ<<3DW| zyABJ`*o2tREkrq|M*e>Q9+Jt^kWZ*+06NMAg6UDkd|_eLm7G{U`Z9uH+BnWbbyt0N zk>%XJXg~hj>}vGk{a+GS%*;F0+QiTIp)E5Ec83H9Uv2tyRg1~%^A!6}W{(eGVf2BC zM0koVo5cOdFPS~uC+`y5 z_Vkw<$z7%FtCE#be7RW7GVUb4pUj3Dh9Q-0swjQ4H!Q$9zI13bR-H5b70d6*L4N9n z@ME+1=Ax`r0xGQ?v=9=){E6F>5;&EbEB&S4It?SO+@vv8v_kDV3~-am3(A1~?nLpeaE05NoNRB`lLRmx%ho^@@Nll3=@Z~s`Jqj=w! zRjG}c6Z-r6IB>rQYtQYIP7HaMZO%tqPp-kViHwP!7)6@jQ>RH$daf@B-^~0w7a(y- z{MKWkfik^G_WrFMPR@+o0GZ-F@HpQjxVSY2Bf^ecs$1`-yidQhKUBXXDP1y)5$4D* z(2iNm^)qjORS#vQ`Xj7G9k0guqvtHPH)2DBhv`-mp2>Fb5K7t0Zy~*2wB7Q^_{jZo z&H)w)Qr!>}XfckAS>e8w-=JU$VE&ChVC%BhWA-T$cb*v@JM2mW)b{U+O( zG+vK|%E}MwZKs3Nb1le>1(iFKsCG5PVl&o@nRDRx6_d4YW_|xm1HD?;}2~{hFcbaX&-W?d?xufsORbx z4mIKa_J7uk%I6bFh)O#a7>{J#hzv9Mn|Z1-u{Er{urtIPs2NoEQ6B29)|>gai2T^U zqtfd$Lrvk*7k&%ZSH2ajKRngKfpxwyhv7+DnusZm79o*(P~(tPI5V+qna!cJ?RKqj z$zw?&{2KR1bO!%PI-mA}I{hUR4Red(xrkx%w!4dZdHt?5X|Rn;266nZf3}ZCcaJ%p zOl{KGX6}_`LUI+`w6`^B>jd#9b5>vHF&O8qU;0DI47N|rE{mY7RLR(lfY_=S_e=M& zo)bBC+sHRDlP@(C0(Bz?FMST^-B~BnIcf+`RY)a9mXZU@T3_lfjp2;p{|=O4-tdqw zE$k}bZ1^yL7_de6LKMcUt7Y$fBK?{>P>#mOJV+?#<3HkY<;E_L503Aju8CiN|F0s8 zDaW|hbSv+!mF)kn_^*2JgtbP;<@c|G6Fd%PDbaZxf*MQyyU+jW`rpMG(|06{&%&Aw zu#5D)YnAwqk^Y~F{@t2e=O zs|P!jiEoUh&=+tq0mQIEdX@cqrC*oG1_tEx&5411xMR1pgLkMb9A)Q^gN})exWUM zGQ#B+l}^y};Ar$8_WRGtXR(eYGS&K5yJ#_0GJEJSRX}yWW@m*|@@t;Ao~#e(D{oQ# zL^l2Es(c1y-X|!QSv)YqT{(^2n<}aexYPD8Iu{p!oS2yevhda>%~rcGAY&caUjFB{ z{D(1nqQC^t>0Wpq0<6s4iH{}ttWA+P z=_mia&b7v5g69W9_U7*PMCaSogX=a^KKAl+s(B~tJz@i^??~-}qLx8t9J`)uY{z8T zyM$)`(DA!eq6~Q>k~P^`(?XbH|-_=o`1JNMEa|73Po@Bvvxy$xgPS?TA!%qnEB-DldQ6^@r?0Ln(w6YH;wCW+Bcq3@OOQcN^2Sm!@4xF3|ny zxLTX@CqMiDyqh>OGAbtMVD3xI^NFj^o_}WXIly8WD-xH&mh+N!S>E3+@hhHkCLn|% z1As7rls$Aq$?KLpfGX9)+K*PF#P8&fi;;Yuh0RAr?0-($BV2wIsaiN&6F`Se0$3r= z|I~US<~6{d{$r>A!z^u~b9Fv^FcA`1<^cM_m6zRvkIND3fXjTM&tlL@yZviIW1lvp z@D3Ue2AFU2@D*gd7hfV=AU2LmDvN@KY%P+)X=Ple`Z#_j{jf#27e~eP$nQ}?MxI(3 z7Oxq2_81H~$TB$$s_Omm$-jkUy~=nNdlnN1&s~8Papl3{xqR=_mpW(;Wnng2-F}icxaqx;& zzIrW!%HG=ylVT>e)^rR{4jxnMvLsAFy*Rq-VYiz)n!F~Wk{KWg#cTWev{V4)u37Zl z)=FD{_0s`;salw$6#=hkC3Q?I8p;1N|SUl1Mr+$R^Ouqp5~P`t^cW#*a1-ojsnH+%Zw#vZSZ?R->6xqrJ#};j#|F2w3HBM5Ln=lHa6~J)Qjg%(KKl z-=4|p!msQ^_&A-PD>iDHA}XWa==!s5EH$T&5t*A0;CvmTp;H+;*M{1&vE|=EerEq2rmr4@GzyJhH)ctnaSlNZYKkDUX{^ z3bDD&Pir>xYV64v>}Jlo483@3JRR)#9}fSF7)^L`=`}>h8DIExTp3GYb`5afI^B(i zKN~$V7qnkgcx77E;nimv_UvSWA?`5A^T}pC)W9vC<+E-tm(AXlRb?sv(9=576$EUj z4jK4W6CM!t_z=f_b<{e}vUCH!jrFSW-<9OLyZtGy+mOWmzdWwT$oXmbQwV<0*GUMF z+ShxFdgu84EBo1|gt;mF&C`Ik%F`b8fb=M-8!S0^<;DgnsxcsCVe!{!x)}1M$aroD z-e$-ScrcxXdSf+YJKl}Cu+L;GO`Afl(H@kij^it@LOXcf&W2#ZbkWtUd4PwqVXtv`T=~W21p7HT@r?Mea#d5oKWcg zDujALYQM4O9eTb=fiWg4bPj8!gfc`%21h+}_-eU>tU%QTT0k1$_%EY3Y>-N|L}K%YGKx>lKc<4=_2K)KJk(TSkp{g8U~I!x@D_bN=Rc~vueTlEBt zo5sXfYuDsuKV=kzr0PVrqWSM7cAC1|ACgHuZlA-}@`;SPNjvV}!YdwCL8shO7lF`$ zft&fQphg{p@TFD}kHyrk$aiigXaFFsS-ERO-!z=6PZ zZsEtpq*S5Sk?RdFw@3BI^kdYPMfp;R7!LI#HSBf*Q)Ev|IZXg(BW`6(LMI^jIACuf z`cN@ohOa&^$B*_rr88^>d?|`&ToXFiPIEFH7x3e5Va&1xr(;{D?Goj*^Ol9n!g=KN zQRnHxm%9RP=f~1zN3~^(dA;n_%*Qi-qlW8`7<%LLVu7%+m3#3Sqf4~QO560Sx^_QQ zyVYc15E{1Bg_h*Q0L$q_kPn0LLq=gJxr<(P#ia2=Z-m`P{oL6JT`W8k^n_!*YMxc4v%|jM`D_ zFT#?`_Wqg?=#M*70RSBjj~U_oJp?RtwJdAz-;ZW5`(X%Q)Onpg)Z&43v48ho3(CRE z*JnCgVls|&yKhh1#q+6F2X(TCutM-pdb2A{4zL5JkjNYC{{d^>ugUh!S!H#v1l+YE z#*N#|$4PbLVjb4E+;-0TJ}^TvsBo{o{wvs!2JTNGI&~I(tTI+8^%`TjzXVj=w|OEY z4Wl0xy265X+?hS?E#aeko5@4WL}r##66zCk;8!TD^?1|OrjWtH9>B=EnGKv!f;}WU zkfI!dgXcNdViSt&JiWXh(&=dLU#G8^18jR{QMTq_gwHKAV!ZyqO@SU}*UsUpVv+0>sM;-I5QYHgHKrucL?q1K2rzKJXl)f@IDotE1)(TuI(Ivt zjaMju($(#kYdosF#)a=j`BFtd7-%e6&iB+#Klv?&362AHI#+`N_eL)6I-6xjYE6gM z8Loz0UW)-itOJi_ovYuFS%W6JkSKijG$i*!DDvUq4cvc&&;N2ksQ1sx24~QxPT3YX#ujfyb}ZNqzRZs$B5KOn)!k&4vVU1CFh<}wRHHQcrZ zz9S_)hi?otKMo%|Z#>>U$6xWqw)65Onpo8f>KS;vP?3AsuKjY)(pU|GDT43Ag1Y4& zT)BcS*D|q1vQz~8(aM{%%7VOsb~@@j?(1RRcR?NCxh1dj%VY7A!}&XoO1I;M6^r?^ zpmD#)RYnY-LNn3vZ$#r;_etB;w}mt(O9ndK-6O&X+?}%y*bC#^0Y9?ooz0!>o>p%0 zFxd=k=ksZyTF|X2_~h2>Y!W@Fi)p1b=CXZf!&}U>$o1q?7@x!D%$HW*Z!h!WTo(%8 z37pfWWO?)Mpwz~0A20D2+bzbO9;3c3%Q-$i%rv8X#AO8>*h0j#oyW8qAuY#bV8P6^ zTDdk_DeATK<9i96%Gsz2vBS^X)Gj#kwF%kJ?<}P%+D(G8JO3BSFkEpmGb^N49K@{C4%({vh3&DntU`?RK$r-+9?F6#CxsP#H2+d3#qz-u|m zQ$A)Vq?crL>n{l3;(YcMRNqVm2iLRdq3}-6W2*Ol@0i0X_Xz0rI|`U(j)NZ^bJ{N! zq-s^V5iaYND&@p1v-}41!tNF9Wca(AuKUCnZApk@kny^arcdWWr>exdiRhAmemo&d z6#s>kviZR*@vVJMBMNACf^+%92Q?8fn#LPp^v_-Y(ENrTQ-BzEXyN~jwg zmLEoG79-O5ES;~?tk8Hw1)=|n9ut4j-gScuT|~n9Ew^5oiGYQN`@$)W#0~w<~vp zcg<&omz_6hM>%bWi$Z*NlT|0vBUL5{ZIjElUywp9E`J`+nPZK8X^~Y+{m5P5S?2hO z&dS`Lj*$3p-A3ogo)vhsj-5K*ZYASNGULFX1W^?mh<|*)nK5^d;I8((@dNm`$f@WJ zVzE%FH63I6pO-G=dz&qJ*q@%?%${lL`fj|gG$882tG@0_(pzke8|gSTJfg<#;%@ds ziQUXL7r722kY^}NmMhe$r{z8!hoSiBt-pEU_=NdCz|z3HT8--#Q+TU? ziHu;QInNTwHuvAM--NXuZLP9~aELjRcrCT3hY7}hEU0L^ot16dShC{t@Qw$($t=yC zS%K-XaBqz~o-Annexk)9tqYyw`hUA|D-#U)!W~@X4DzMTHe}fMoDzDXngc~2k?TQ2 z(W>Yd1D+puOnbUA6w&d1tht^G1d97!@-^6<-s^#H&tykMOkSh^wCde<@JEs&YaFL) zgSb%xFp2Nu0XtTHZRbDSNbC?vMsDj?Z1?4HRMEKW(~XW%c`e~~v8PZC46$T&1-n`DiF>duG&t=9 z1#CI#HU{=m*RS2=8=>4P@1}%Y_i!6~N*=%e%B(MY9wU<`Wc5uzXSv_IwO2Zh6S>$U zC)qcWGP(i1qyA~^v~<5YV;fy%hRCy9+cKODa9PZ%NB8+Ye8z{j3hBHV|8h5C_*M_i zy%;0-Z(ghzO7!Sxnps`ukg&&z`4L&yK7QoqH}{ZJ0KSl?0WC`e!Hfglh<4i7SYMKB zlDKZSF8A;C#+_tV1^sGG>kjS>o@|9z7kIQi&nwdb4wvcjJEb1Lqouv@!no4AQ?zDk zT-{Rd7klwfyaSk{NJhJ)nlOG`Ax@bCq{l~lW7h_c$Q^YdYphkkQ=D-?VQFJcz0b^d zDwFP-DZRj9vsiI!ybu?P18u49J{A0ftox-jWuxA+8Lo}p?`{A~iZU9*?lGy>9~9Kf z4O0-ctqUEFwTT|~}IpOgY`2uLB zJPij_*&7kHGfknK>`mKuGc~&*JI?!9*v)sP@&I+e2}`sCGNUtQNmP+jVo;I57q=mu zX4kDgYX23tRd(3g{ZyRkB)H_7r@KYV%XJA^^@pBt{A8`jd3le;`65>Y=Sa7A81rbo zC40FqfMJbn-1vXV0^G8(!iPL+;%nZPu+(nCFm5w<`)CMcZ1HSlNFtra*PDs2P<`j| z{x41AOLBaPw!|=64&*~s1sPQmpQM9 z)%u6kD+3y{{*X96_byuF>l?)VNtT}o=#=i>`O>z-bEYbdmq(?@l~V8$?V(C1Yj9Y8 z1{7xydhE*^uVFryfPIDBLI}VQA8@HO@zY?@dlcDic2Lbo{y!haVzrW5&DsL?T}BsD zc7bwAA%9Dhf6MeFxovY!H{YigaZjX5i1!_L+yWeZR$VGv9A4T%8+gTMEq?prg{F z+dH?j1KkB+BC+~4sZ!o6FTdRXGWOMY2CX`}6EJSSI4hEHDw>sFk||ms0nIwA>6hoVunu?fKVK*}Y|Q$O$!*dLO^~qdJ%`OMEk1&7 zTpC-sLa}TybiH@!yn`B#VxL{6T-rw%VpyrTamX(FFE3Y)qBNkz&A@uu`xQ>1WryDMOy_;tkt+B6Z zUp~)To8tr2+0D-DqD~zS2eZ0!o|P1krS~qbb-|4tEc}bo=ni$qcblw^g3nN=0sd`Q zcY&!4#({w8`GY1W_nk0A+RfRN$m8_6p4Y(RfMI$YjpV+{GO9s`w{D-yv1uPeZt61$ zhcDRx?hVW3Sa+}bhQzV(&o%j=p-ZWj;0hIs$;qls&g0vF`<1|u1L|xvvB6+vqoF_O z+9#-E!=d4NgEY@gI}zmJl=lJzVcv2x2GTwiSs3>nk zu{pMyp&Jx&Y;MkEL4A&|4Vx|~zBJmB_P-FzJ{FT&4ffbj5%VW2YG31DzPgH>%t*mz`?U+gf#;>2T+og1GqsM~{&4PCkO zYExX6eRfWc$YQw-ulTfQ#+@Jjmq6u><@$sG>!NYUk}H&f6ZYZ(KJAK;x^_?6 zuG^Rdezo_uJ6GocU1iYqr;f*Nud@~z{IwnY#c}ySXR0#!8B^D>eWS6=I4S6Uj9r@8 zG{k#u@`B|y|4`U{`ELLmrsuW;tZ_ut-YzYhAda5++wQKkrTdIY{ky!IPKGsmXoB~8 z4wd>Xj5UOhj8}|@)%2COf#yzRCeH`Rc~zYVnnKczNc(J4Kj zJXbSSYV6&g99wjM1TEgBZtlQD9~5FG+^Idn?dK(7D=W_U?U{)w$71I{H^F+%fzjE^ zsFSRvY~g0Jg(C1Q#DkrcIcp4nx|Yqh8mh7Qa$4WHrAqpxNoR%XiBB8f@0PYp?eSG# zH~s6cc0bv{C(-LZs<3Ob*&cnfyiLbe-DhB7<*F{|?uoy&%MUc8V6DOrO&}j+=e5Mc z0O{kt;zAui*bkmo?rVRUm2b+Ew8EzHcm#rdBEjHZhuB}{&T={;aj`?x=4Ri(@VC1D}> zBYi$ADE6x+T<+KKrG5!7Rp&osH0ytpoYvAQW|X_TB3O8xUI`u_U7E{8iR@_x!R$n5 z&M%d#Oz)eSKeuAmIG=uD!yNG6&K7ps(

w4IEvIH2R;PfAQKvPY*waT>XhV)scD3 zjPplOtKDriHU(_BfZOc(X70nJb{-68Ua9i>)9(Z#vb_%`(c)Z3tfU6(g^o-=4M>`D z&v%-L>;zoYA2#iyFP^=__pGNIxEg3WQw|#bIo63HT10xA?0qV7K3gkmI_h?KA=b4h z8^%-ntzt}k50i}lW1N)5OeRmaH-70;b`Q3J(DA~CfW@~!s40g8G4kE^Z1C=c$Pqt4D4%gyNpdA;U!LT#kU<>1f1Q#db_8{ zj=X27vpg~uWF&i$W$1Y8UBSSk)>B_c<7V91LF?Muq1E?lMuBYDP{-}WVL zvaf=Rd=C1iT)SDM#`?4G`}jx<+fKhB5tBPcwxj8SKdHnTz5?@K$oyQpipdJ}V#6-< zXaX(CNuCTBP>O+4N2>0C$SMO>mc!ptkhEb&LHNd(hY@3*pz4uDu*>p>A){%UpoevK z{rPxD;JP` z+eScH(`_u~Dg4Z@hRx~vG@N@WSx`M&%|V9i<+nNy(BtE#14pBveY>lsC0q)5IlWd#iDU1(h5w7KuYhW6 z+t!AbBE^ckd!e{%h2m05DehLB;7*G>#ogWA-5m-PcXtWFlaN1s@4xq)d*0n+Fvd=@ zS$nOy=KQ{CJGtMAuH4G>ys(0qMHo)!pg$CqWwv)rgrQL7E}S?n=6)cC<9A3O;aROB z*iX!&FTC(=-W`Ig&NWHpgR*b$gReLC!AaH7f~}c7>b&jRRa%=l2FG6h)Dz!@m@)bS zyx8+|`^SNt`S+ZZ=hepOR=&3cKDU^}qSy3MVs~fhFe3R&?a6~tGJ)rxCPZR(Hmc(v zEA0S%6k=vn5|Ng2NrBQFg^zm`Z5!Ut4NfX_9x{zvfR{UhkQ=*$c3&{X(^+pcFbTGk znxO~0g4CLyU2vb2V*{+=AvpA^Xh~(aBp`D2wvVcM)8@co9Yq2Pt8+a7s(rbG4e7g{ zuf74xy?-`Lh)?h=Bm_O%FZ+8)b6$4iSM7!hXRx<6_^hdMgMDMZAzW*~a-iT`i^xR) zDTW%X)T=by$t2(kgW`lB)xM7<+Xg?l?dGHiIqo}7W{7NY9q=63FXc>}hiU0xF_(yK zW2-yD1Vz%#=hkl!4PrSjojhX3+A~Zy7*Paci2=Ze{8RnjbZm81`$G_?@gpba3oNgi zl;;!W+a`n8TlKtx)lSlV4s$4)T?O$fbSMD1YY#cnv@?doqWo-JScn_04rB0wBQSyJ z6E5*Pr+U1&5Areafw>=_O;YiPbdD%_sWmJbrIUwP&w8;oqho*G9HXas`m}cb`PSNI z?eT|=ODV^`$xQg9X9t}W1fzkPB0^!EAV&5pE0b3##;=Z|#V$f;&PZWsup&Hnpi zM5bWL0APMT-U2<`(0~}3uHfI+eNhwiESZLC{bw*jFfK0}p@Zd}faj!4 z5k=F2H1(nvZr~IZ^(%~;qU?x{7pB)rVR`kViVZ{P-zA7mX4k!t@6M;$y#dLZsw%Ra zNGRwpQI|+PO`F^uVGt%M<#ZXT{c3jJ=O4vGOIRv8zb}L3mr_BBDST~D>8fTEh4d0t zy0v4^-f=3{snvbcalN@F9Mf?C2JW7&u9X%jy?)K=e&HXT;(n%9uG^K<(;0$FRA}*8 zP1k!Ikf5d}DJ5l4qk-B$vogE`ylnCn9a;oU?G&C=<>c1Js+BF&-p@{j+P83G_?ILp z8XcR)MG5i0=8hc>K`sAkmB-Z(JRA~2XnZK=C9<)fkzxl@g0SRcO?Iob`iG56Tz31j zoKCP?X$aE#8%p@5rZK*q?~^cDXF3xBOAeGCXhN=5f)mNz)5_g7FK6%~T6e9*^;?LW z_~2K&C#%fCc0=2qx;sB3pn%Lod&DGs{l@MlH!~8AN?Edvc8un2b>#FWtl;UEj`S|( z@Byo_m4-Kw4)xqm`CJigxdLFl2VZNEUyb89zFQva-Y#NPl7n&UJS5uAH0R=~_ge+i z1Cd+NSLU3Kb zH3d7f3Q8^S-j_beDsv@&zTjmVg@s=tP|E^Aif0YImN3gJF28 zpOc~8{_WQX5`3+&OliuzKWa7PH)bU&nf%pK>}SH+ew<UU@5M_VFqS zJE{831wzq4a{T=Kr=to6{Fkq)lLf7@8k4?Omt=?JEGP|H@^#l6VUm2Arv!~&16U@f zcDn3ZYkJ(=t(qFt9Q)dy=Y`eGb6CA!!(|GNmF`9GtkMg@VH@oOqJx7Pqb+zXTjY))vl-UuGF5I-_D32X+xcIa zL@`FkPk(E^#Z~>RQDcutH34-YNsP!zIg8ES!VIilv3bMzO@Z831xl(G3_V0?qKpfP zc||ofjMJylygU0f!*?wxGJ=>t9&Rj_Kdn_%cLp!+1XAN)Q4oA|c2@L$_{1DgkZ^ML zd)@0kOu?OyTrH+VPTnM_j zMDFZW6cpx@(;8WD>gcux`+u~z{JeI>bbkLX-J{h%*FAi=tjk0U-xtVF<|~-MGey>q zl7(N-&5k{=vY=|xk^1|Kv}Z%Udqu7Cq;|NAwdIgF>9%FB?+9Je-qZa?^!|ur4^O;N^%(uaPfBf0}A7Ku@W3VQs_?h6Hs9O z_-v_U&qAplje$Jaq6ys>O|-7izREWlQo6E|;%$AVsGaORc(t>__;X!yK0pLa_w+ck zfM5R$ndiYtDt(*Y_l6&- zx>5frRry;#^OU;sV&K%2g5?$DUlvA7yQVuYV0F0B#DRte*yWA!GbFo2bvsTwz1}H_~%! z_L6>MVuem4QMzEP!IN8~re$hkBUeMsF~s`M#>Tf5Yx9IX2D2$|0qYLAy5i-8$GCjO z^1rKJb<2h9wZ)RgOK@;Ko4x2cMmIt}PUHs$TO)Cz1!BC{xSv^r+~}*F6!f0u_{u#K za<9;y*t`_H`TiT}v54^slFMe<;iIIeY~%*3w_727X-G`y_ha6-De9c6&yx81aN7TO zJmtODkU*0!z=x7*!aEecV*4q&=y~NpTy~N93OeI&W@U+QNa9mu2Y-ztkKi)#G*RhH z?iZCww6o7mV=)PnSJaq9?YBeMQLYtaX*$fOkXP=`Mc(fJ*rF2=hNk82XLHDC2$i6Vy;dvL#rq-c6#=2=-E~PBR(D1xzWicw zQ8I}kLV@{$%o5cot_YW&#Xylz8KoXkBkGmJ)!_bbmudoCw_iSB89idq^8Zv;dB;>? zH#2&_7Vb0Xn>Ft*5)C2oWgWXCtOiE!JRI#!Fq!{$r00Unl(m#cr&Z`zJ-HlVE7N$o zByJx&B6}NSqZq~hf;o9;zFev)st5kK#;cr?eau!p+FF`=T$9e|^Zv;vpH!1`S^U89 z8mqPqbtc$sMxgYl@UaXlfh_&qXiQV@a3%&tI);0j;3s!K64GEi)F>WMv(Qs7W;^J( z>tVqv64DN_)6;E-rx9X#jF7put^n7V*5*sc-p;aI6`?{J;VWl-5h1+2MYR-lkDhZ-AK zniB`mz0;kKlCtv*9XnwWtDlPZ=fk8|kt-*3!o4FGKF$Kz>&`s;H>G;Sy@hdInuYa@fVRo>AQbaKFFCdavLJLFMh>j zwg#hiqNWA^D7ra<36DFTJiA*YFq>;CRa(%LJ%2ko^F%Ry6 zy$F{u93)UBvxXThIoP%wAy;;JAd?B$^%&J~iaMf-sjA*jNrR5Z&epoJV(5anEO${Y zEsv#q&|jjnFtb^H(w7Vfr!h~TT)rU#x{2oBC=|F>InCRxB6V*oOYERDO`1LK#f^y5 z&i&$yc;p`I(i^!^pW19Kt27yFJi8%LU??cqr5rbY2xF7V^5$<^b$SlZ6jx4VTApE0 zRfOh@+4si^-dlTMw|(DIwc9=A5i9qG1h+${GwsHb#fUi#MMK@~xLj3uMo55kDVP1> zI*IM^Ev_4V%NZ4lUY9=wVD9{rZhj|}%i%%?Z*G@@9B%Jm;ahC`8+PVfpywYo*?f&z z@@(6}m?@(HS>x={-^n{(}{aB+s)oX-FDDbTRF?M|53iQ7S)XB-NE%Z4#vfv`Iw); z!&ECBF0Eg~sYXZa+6P9!!`{O^kxgN)G-Fcq1-6KUiQ+jpp9UIUc4zhD` z=t&^rc5IED5}@i==vb6+_;m-W)E*XU?^~jfPLGUsv!7*W-q< zRgBR9kW_1KH6|}N=O_snEh#egKd0P2Lb`(+d9JjS8|O3u~R zo+@j8j6tDe>cc@5C;$P+r_*gbh0=dnHx7;v8h!teK=PT-+b$n^`u;hNAD^|pN_lx% zmmP>eWt#&9hHMozd+%7}2ZhJx_orhG zpuI2?vyY@mnAk=0$Rrtr!2^4UJP2P|@5^7I8enR0K0`*r>@0a>tEO<`N<~lnlKU)u zRyHCEA-g&^RZZkJPXFuNE?@-g;4Vef2d6%zN+Fm+*lSDX z3J1`67`)}|`HxL5o^Q!4EF;NZ36V_TwIv0n( zP;n{OAG(Eis{@!5;ok|t9b^V~59r~(>AF%Va5zmJ9{w563&|-c+I+foQ@h7}>>o5p zHOm`I=jHQ6{%yL)XS#hKuF>Fe#wDN4Yx=OjN+M=y;%@^-3Bz+MJNqMsos&4pt1EOj z!}Bte3kbiv;oNZpJtIBme!HWr;?X-p6)vg?eK!R^DQg9cyh+lYx!EGPJ6c?EQ|*4j zzt5rdtUD?_qaAV~kRp17%P#23i?ByPV7*x1A}(YEA4;R*D?6!3P~8-1)WS(#cJe~=@zlC^#c}w2|1Um4nU13u=dCucI>ktU$my_?57| zsTSy57UJd#kiTjJrf+g4n2w;Lv~G2GiG9+ia|2?~d1Ss1YpipeV{M20MmMQ|@}DCE zBh0w<(>wJYNp4)MncFb|tu1y(!!lO#CU_>C!0vG+&zn4+T$h3JeuyraFGC z3^J;!rpupqPCYENZB%b<*4(+mdaB%SOsVh~)2GJU7&o~g@z(l-WT0r7sBZ_4I!ET5 zf}OE%!tdre{69}s8wzJEM#*Mhs@87l?%!XvSVLau_{@boik)wJTQBN|a%iYm8ccYO zqv;Lr;Y}Va(>N16gg4dQ-#oCb|LQNNw%O!Z8%MEY?{*n*t4tU3V?}gTV_|q9vQ*o3 zByoH;ad7+LH#MUUUoktoE5QKDU9B^4ti9Gn-c#5i22 zS~68qn5E}u4D#L^f*FZ-OLIWWdbpXS8O8pP;7u6`{=r@Yn48wdSy1-*@t~2&)eC64 zH;U%{4dWo@neylNY_o+r7=-@0zHf2_T?L@Ad8lfw#T9(jn?tC`Ko2PKS9E~O!*=M3J;6O=4)kKAx4*&|41WCV@q_e!9jXT?i<8~$ z3bbC-+;v%Bg4^vH7ho_eeclX_YPzNni4M@mGUH1=Zaq8A+cD1%G;F_r&pTx2#%gyh z&JO3;2r%=+jnvsC^60y;YW2rM+Ft>WQrg%I|A_b2? zQ{c`FrmazMQiKp?M>=A=>5CzH-r!q5}xes$oby6Z7=wg zZa5B!4)#DRN%~up@`G9S9ndL3|BSTO8Tp9{o@`fTai1%iRlYf-4}@%l=%%J>@d*id zWP2tz0w-?3ffi!I=O;NS5B@#W>u;zH&Lbo_-x1%TdzL{X zv}zjswP!kCm#qzrXB%pIZA8B(n49~ewy?=>vkFwRP411m-RAYf2tlp3k#@0IP#gRynD2it$Qya z;#7p;FUscp_)L0EfQC&=Zg&~5%lR}rJX9pKKtJN3gr5WNO|0=%xK#QF2Dpvf7(WDK z?F}$&TrNZAr^bH~2vN9@yJ&~+8QD~uM-Y`hzaN9*Cj?v;NyfOm!+tM*(j?*Y{F$>v z&i6p7s*!_8XhO!X*xN1OWATk!^pdSROu8wcR7|gQ)%NOj*}d9rP2leC>?~!smiFM& zbGc@mm`Xa2VJ9Ea3Z^oB_a6^!%GR=P{^E5Ui(+*TQbcodl#Vk2&mi8jBAz0^K?KmP zLyzq{12Hu|5-QHi-qD1z;mMIf+vUTQ`M3}nA}+2P_*&rJcdn7K8(>TB>1r9p&~_xW zPJ>PIV>4$&V$%G>VVVr_@w40X3O>iA+b+B73rDasmuFB>KudbpMFuJ-g{F* z5GAnGdAmH1KImn@+}+52MJbde45ONd?|H1e7o|}Dpv5WKpMsMg6v-djD>wYy=vO`CnuocC11|foFuVj9dY?gzz@a62D^Hgf@Sx>_TExx#*f5QU&L`EPEE-_0>WtRnJAvzt0TZ-Kw z*d5z*k2{mm7j!}g5g5pYOmVL$Uc3c-WDvPMsB>_{qVjEN)n+s!cM%lYYD>HqY2_Y6*#HP+XS64_k zn>?ibb7PJw?^oxXy^|*(91LWD5Cw4%!|Ik(vjJy7+x+ZqEu%Dd$B$RG%0CTk-W2eu zy}i50zDR{HE$a2*QN+&g2;!IFVazOVRCE7hfy1(cyo$KnJ2Fagx&2nvd84G~KP98K zu#t*O0Xx+z_KP-KSVetq?c$<(R5_oD)%GS{M`zWJT3U)VxFOg(nRww!p>%wegA&GW zd)Rx(y@Y!_U)0ogDxG(39^LGi%b^D=(>NY_eHTQ>@kUNt9 zJJEG&$|~$WPEIyLjJ|FM+&AYqFXtNQh+j4cvFykc&Y95#AQIxeCwz-Wy7A&2Tl_;s zyu}R0e{2!X5mJxAPwKbSug*X9_9DPbB($`2)x=CHVPSt)A^7>QY;TBz5Sp7CS8{_`xt#FRMM(oC$e4hPWBXWBG{K(1sVS%=7J^pVte_+BV_gQOGM;lgsA3VwV2! z_pN}R=j*13xSr?N&nUW^@{Ndnrfzz@$`$&LVVqda001u2ga&#B`pqWNt*HM51o$W` zTwatMH8ot*Sd6K<^7L-n#-$soS-}sRC!=+APY)A;)sv#`tKx<5uX}T2vh7m~LSFi!LLDNHVGPU3L`lDFwSrr^L^G zffmWJDmy^zlf=$ga9A$a?u_t-OjQYAr%xP>?qCI{=(+NZjEEHj-9o~gelR)d5Ky=y z@|zE}I^!Q4P%H>7_m!hi=F>|=i*S(WcfH|aNbsa|kMIBeB@7f5Ss^GPaSCvCebmt^ zq61s#-(rP1U^jY>pOv)fGp!YP`*ZV@R2CopY;BXLa!$0rQMbBa(tYU6v--<5l_Z@_ z$fUxilIdC%_*XB0NaBp&GZ(wHWdksYd9+c*-I$A*x-omIxc{tF#OB@*rQyl(oaE4e z>%CB#<8|`~RVbcP;lKsc=oWSFQSPWJIs>0g>E`7CvNO?F`NHv3iuu!=KM5W8kq*p8 zf`bE<^epA~ay$CT@HZ7w_|l^HJNx0eU}4tZ7x7UL9kX>1kCbw1fW>lgWqCI99d%2a zD{#s4qLxDhoP5O}>_2AL8JKgBA?@XWT{=1@)NGE;n#5+bw>|JCv8u9|+I$wdHqop1 zu_}>D+K#t^4t!J~0V~lpK10m_LK~LOAo~R^r(+{qCBZ&`@uxPQJCv)ti=B$1Ea9gh z-gkFJi6zm?j5^S{Ww5Mz=f$BG-oNcBX3*&xw9^=&rUak0@`gK;H5w{ZNg_in%X@IY zoDE?#JiHfo$WS_3B%sZg)cE!!Ln?kPNZ?hkWlepHA6(}eS+!e{93hgn^7-(=^Q*_YM@@D~HCEdAq zutDxlPBpp+!w0C14?pGZh%kOQZS#S;IDx0KL<3{}-ou0H*!3<66qj?+n(r%zZBGx4 zL65~Yzx9Wdy_x=|;YY;>LpBs;7)`~=i&$xD9c|n@#>S{G{1?x#$g!9{trfmZO)V^7 zSy`wMangC2$E0+MdvbZI(bPB~2roy=z+?2rhzVQs8B77r-E>FU3yJSme7=M*WL`YZ>LCbB9{<6_F^{E1!Z~vpl1gQ;&DV3ma-MsCc3%k1Q6+U0iIIdSjb5N~e5|G0tvU$V+PL z<40#cH&$s6W!kK?9Cd<+mzrZ{dY>T_`od6}Y3c57Dw=RF!N*fEbpSdM0h{g04Oq<0 zSkl5&JK$(M<)rUL%rx{8^%BEY( zjr+1n!`+WWG0R2RUuSo3x}E|}LE?i(#0srX$Wjp$(q7RfjXrg$*aw|5PwU8?kn zv~z zIo~&H%I0e@1SNH(Dd_FiBWM+Tk7upj(fV#Yx*be81>HAntk86PCN?d6Hm9#@Hno2m zJ`@MnN%&~35*leDcG6Z= zomNLr!r&h$TsHhq58#bCD#ynqbDonYl=}zgg-o}OeG-m`>bq#7yi7HyZf7S$9;2`#Omi0FK z4J4mk)^Omr*`3aU1~li>OX)drZTKE0LGBa8`4h0& z{bCJV4k9-K?Q}MJ4PnnkT)7@&LvI8+;Pb4pUT;SpA)%DTYqvcCpm)q!G z9d0<{RY|bBrmk7f?rtpkUx&lD*i<5kT1{!Ekz~cptUH}Ulm{8+Sp(YID~1Q6TG9=6 z_sLw=S`D$VHn0*#S%VK|b=k!|O$V~s<)eO9{^eWtJSN>y?`ZKJJG-t(xGnKljc%K( zD=-kFa(_GgWD9-ZV4j>ZhLZ%s_J37inFqH*5M$6`Fl}P3y~>$?!_FN!7+Ln!kB4GAmAtZc1cBuMy$) zHf@Z{gfRaG8ta+~MyVVgmKu7{BBirDtAn~`LZ$r!LRvM_DD^FmNZhOjYgf3>e`&yY z7jEzK^RXzxIXM*+g5zRGU%s|d@_Kb;Pdhj*!CCHjb4xkMH zceYf%h7E8E+RYTS1Mj*(11J}&$!TM4Q(|LR?o^J`*Cd$7EW*jbQzr%W)!&t|O$8=A z_Xy;!KuZm#=o?0J`zHVoRJJK3PpbouFZR{Y-b%Gz>4kKNV%Y(=?VQ(HQ94rOiQx9d5$*#@~Wvs(J-w&a1U>Lypz_*{M0>R*Q^2O!Nn z-d1o(T#n~h>p~R;-xK^r*h;_VYep0CDaQwPTm&A;TltJ7?IBaQX=M7A=Ok|0gSF8D zjfuwV2%%XZU-6lZ5z#vGg zW3ACi{ynqkx`%*dr1e7={Ar1{V1WX@yV=rOZaC)rp;0y;5T=rd6_RF_S8oXV(6r~~ z+gnp}$HApM4d}qZ*-|=IQk?}JoO%1bqa}J;s}kcHOt|2FnClA2C5Y~#O3l<1P82~~ z#)GkKo&snrEctgAV**)OItzdE>Ff2hhhDE741vU%H}QX;f8A{jh}2?NoT*vnAaQIPR{D0=p>25{Bk(vkLZKa9ob=!KA4fKu;58A zI?K4&FiR2E_QMA30<9^#uH&(PhyF%kC--R9eY48O6c6OB1zfp!(=+$Dyikl) z%Ue-={gom3F7Vl04A-FM&7AZZ|7ebzlUQL3-_W`^aW3P#$Yi4^YUb8L* zJe)-=2EY}Wcg;%S$2wn#=l{Rpxl9Zgy&gpyFX0 z9rv^9q3<9`x(I+)pTky{w*0RsHobjW_$$ff!%5QaxA4Tl5G6_Z9r<6^JeRkWW47hK zS}V6vq~V-Rd%LelD5RYf*rIlH)^|g}7Yy*YuK#zU`G$;)tm##G8lv0Ll&^3l$oV5; z*Z(;74XzuJtgpl1Ee)|VZA7LY2k=ifhEO;CAVeUOHm`gsW#n`=Qa()h0B9 zw5X}EajYK~3yxLSqI>vK`rI)%JA=ThooJQMKlQs{5*+Q_C4H0IOukQe;TvvoZ#K$l zxflXxrzi&)7IcS(IJ){fPfs7OE?^*cB{K2!EXYRI!|qwrVxtSDOHnx7wZ8WjZ>-d# zh%$3|X=n3h>|nxp9*TW=E0JlqR%@E>y!G+=FoE*?*Ru8E3nF*}C4i!|s^SgHHMcx1&6FmTr?&5UoPIxiRNdD~AwFaQsz;QLD}%%90KB@`T~ z@9RAdgj;K(H-u{ewo7UG+E;$EP*FpX49 zxe}3fr-i%F-9NX#)Hl`2Q#$%%^23W48ggQ~9RD8vXti*%kX+sGLS;dFaq@JMg2^;l zm8*jZ#YJ9{Uhla*Cnz(^5I*IK5<}?K9^6Bq^}P^ZhaHgHw%>I=DkyeM4f*+YY579_ zxXl)ZG_!VX{^BIhYlO)0$rqz&@a z7b`0{2WO>!&}C2|V|0jP3e%POUqA}aV4}ed`=O^xZ}W-isn=Hmrd-O(dJlo#X(ePM zPDJBmgv>{9yqlfsUU6h=D|xc0JfjdJG;BHPMaxpeqc-ZbyN3EtY`d#T==Vv#WaZFc{A5)&7Qv*Uz&g>v~tJtoVQ^S>o6gZgZlX!xVx4c$)$ z)c@^*Lc_(7uv)|DD6?wAz{r-ZDB1r^BueIMbNTfk?5QPaD2f0>L;q_^5}S~)Rze{9 zepjxY-Tc3$UolQy7g}Kk+6!AZUw%+zNnExpM^sk5`DcRL z$x4l2Z!s4)L_vn(Ff4zDDt~^*idVLejr|Q%RWp)#0J<;U2w~8``}(n{f>L~K=dC7@ z`1Jp%4#0PRLq*9P$-<(83j=+#&#dMX#Zmsu83XZ4K4vTYBz#pZEjea+%ZRqV%?*&qKQOC?6>$xJ}L36EsGg#ob?lpoZtlGlTZv*)Cu^%=>_vwF<%eL*bG zKmJQ{xfkbpr+x*V{YB(7%A-U3J#U104@i;n??ooWP<9sF{)=n=SLuZ;ggzjaODj|x z!b@L%;#35C1Cq@;m@l<4(WSa)M`-@JJ?X3DQm7|5^4Y|b#i*Y>3OBKl>^(^tjfsO^ zgcksBKL2w<`GBuw1Uth>ku#$XmZA;GN+_mtT9g0hs*aR4m$O0K>zKquxr8aI|D&<_ z6T()`5ucbciojeT^p^l(n|?RQ7mxP;V?O!vHA@td9|yMVn}y$q^*nPEwJj|jy>u6g zOyt6No;LBrkWYk9jE?wn2jmXd)(ofl^Pba{G7h0OR~0QydCdlt(=Duj*lfO4^%4a= z<86L!Zoo|JGhsVqG<2Mp=|Ar@-*0lsk_H)W+DmPN+%Mf$Gw@X@X3d7SBKlReE9fKn zIX!O_@>&}cUm?#?cKz?uHuz0k$k(S{YVp{;f0V*SWdd#l_6<%2aOL6nUwH z`Ij%`1q=_Ex69U8UgFq3`Uh3)}^r-(CSDoD<5 zS<2ey@|1ckM~=0?wSrkjlDHeu2NCB8n!{LmP?lo+8WIuJU;LU$oC~qEk0RE|^Q-@- z^6*dP=(!mIwAP!)g{n=>9U7KipESb@RwD~mF%cDk|4}S*aHuBN`juao$Y-H&_-pA6 z=5}=eODdgnAh8l-;oKfQ3%pkH_GfBrZZ$IK((WlKhHh$Kz3Y)-qkQzG726Egdod%_ z0x7&Oz#IeX7k5s%vK}H-*KcQ2W-q8scxcu?a*AQe2a?I_F(mp;W0pqq=c0$-6cmDx{9Ed#u$5nUt_k$QHEzH?Pdh~z+9+Ji- z_Qobe0V26kuFOaM+vsm2QNkzWnA!adq_Ej)*dw3?n^+t1AO6lEa29cofDSP1CM9lT zBlKLdb=aFY3>wZBDju{5ao7)0xCSvWT1e!4L+ObykqPM@X6gP++l64D0^v!{~S1@o4LwGRqt!)i)xa2C2WJqbfPHP$!ZgGL(R7DPJI*6mFdHN zO&h!AG3Krw)-zZ-F7J!z{VWWYRLRf3wIb^y$ZKK-6rYeKiLK)NEmVeUl?)1x)tas4 znz#pJ5v~PK4j$8VGS5Mv6duR>lIJy(FxK)6?6=CBO+15~?rL8TUND`}E{agnQpimT zF6Yoke$pcESqBoYfchv@I_?ALkxej=sm+H7<>bS9$)`t{#A!kU`c1a+ViAtPoaRG` z_CYrp0H-c_kW)v{M$Da#kWb|zrLzQp1)+KD_^C1Rbd7kjx)pOx`!A?dN+&sT+I9q^ zHCu&5h6RL83IU8qNLpH2%C!;(>Mhh?k-K3H<{)8y*Ly*h?DQnWrg7^$goH1U3{)+$ z?+NKK$#fBh&gACQhUhL?JveNwv_Cz4#`^6vUb50(?YL)paAhymr%BX4H@Ca1UVCnU z#@a+>0(ihl;jcuZ>^x_yjIhcVk51AJWbmr75P&Uzl~oa8JPi13gjO(MWIbWkCD(=r z7qBf!kGO0A{O4zOlY&KP;VZr$u%oP^qE8oeM^=`25WyU*pJ9lrGTGiGL5L?@xY3lA zr9nd*>1Se8(bAEejC#}lbXB*7QMVGNEVC^w;az+V!@jvY;gBeMcfok*fmsEYu2d9d zU;hZtF_?<_i%xcckBus2;92%>3~~AxoK^Rno}PZ8%gyJ0$yD82Xhy+a>XV-KyL^kcUy#1Ii9qrdC(?mE zzUy=XcL=kEy6?U%(!Y0K(zvQUx%0I<{e>m_tGXz1>s&&jr<{iz(^Zmb+U_3d5i_gX zG>>YlQ@?ZganwZuigu~`WxEhJu76Pm`7YYT9`>bDqD=tyKQ%Vs~>;`lQ2OYu?<<@c%ZVqXE4vRBrnj5wD@(N`q3n5Yw=!n0Qv%eBf(3)hWQs>$7 zO!{%&8nS?z}3Gp1rX6m=2z(SF2 z3jt5qKBFaRS-S;Oj>_$w{NFZC8lqBM*j1}qf^eX*08LBgrKS{2{q`Y9f@ihq>Nv&1 zqQ+&DkhMkdNz({e<_f9P^)7I?^O6(f3jU5d!_gYIM0Bsg?TR?9zA>?h>ARouLfXfN ze})O-cY(dq5AyX6f1ZgKT_q&VWX1-$ry%r(*m`G0lZyq_QBlgzFX&fW4$SOVX2BnvZnWKtgo!zS}~u)Ypg;s5Ov?F*frgg%w?_TL#uC%vUoJ%DoU==wVMouV*!0S zcR&P}0H@Qja+>l?b=Kf_GFX3hX0Q*o2Qc22!F5SusS{ux9yd-hvLh3@D0iNDn@KPh zWz{X1E0?IMp|v*YEKsp_UQww^%y&aNDZ=A_-H*B&xMX6dH+X`?r$01SBxGI7uBy9unRr@tN+mfa=gQE6I|SW|~Es&t^E9VG|jK$2n5*yvie=*702;A$RkpfW5X5O#c+$5x7E#Z@>9#KR$}o)}KM7Q?Z{Dk4LCZ9aCF;X_{q(Ph)moB1(p=&qr=czSo`$}{?*f8bC zs8sK;cCx!6J*)mB534Kq5C~gp16?woIKp-=mQv{P-pinxEuy7nTgxGQ?9QToX{5>4 zuS zfsJE-RwHAgZ67%6>IqJHObIAw(t|p(z>^MK4)(&64T1RlI@yhx`B+N%^sN4|{4$en z_ao-C1gKH6Q6tTX|{&hrF$r8chfd!%jX}XK`Iq+<3B|T@p2#?5ESp(xoPcR~DtU`1uDiRIG6Q zT8-?NuNsO;_|cZjhwbH1;cD0L%XF7QC+T=U&NY*|tr=JTv#_!GxIs`wED;eQkNuU? zqdOt@okZ=WOQ)YK&-wmxZINbs85B?b-^9PW>UaUg#-qwwOb(g;{c9@*g?=_Ty0Cq9 zt=#XXrPsPnBQz|EQG=-4gu7uxmWP`=sw^8g9UM*u>uPRJ-rY4p=w&$yh<4h{X?_aW zsSuqpr|)ueP%0ZRQY4;S#8E;JCE>WdjI)bT)o19tdGLbQ24YT^x{S)bu}Q#mJOWYI z`oP<)@`!|jcR@G&#(bLg#6RX`ejR=r zy3D|}PzdmJ4*bLtAsjQI4<1M$#2ZSn5A~JEkcQ>XyBr9D^x4XgFee!JU$4q;$wVwPKu^O_TxVnHY z-|C>x;Jxu=Z!!EAGA+NRN|W}hJMT~Rr5{?BBGpi+XZUjsW81czKcI*5d{&n~s@yAg z&>0uYdz>C<$5e-W8iXEtAUmCeq-g1+5!D{X>1;EJ_D9&JC4(Ow&)Tdb|pY_Pynz8TCVoH}08Ag1eNuJr5 zV5*v+Ys2kUyqy;vR3etGTC}-Q!(r>cQ)U{88z}l{!$apg(O>cwx=85&8g*0_0B@~) zddN{dK>A}dkOQU6?5#szG4=inv!igkx)O0jW^k5WwSX_tbK$wS&}bR?g|cSoMiuoN zL^)UXoeNOXr{xoKh{NaaA8!aKPLp`sGlTC&gHxtg0>^%%n;!^{^R;P$pR~*E61&W3 zd=A%E`PO?=TR3V;cEL^^Qqw1;q1O%^#^)P}3+ma6gO7nNyFPJIbuX(C3I*9yuLNf~ z6nlIGqa&$0CubCG)0sN3@wqRfz$Dvm@*CxqW4CjGgp~%<5Qcbuu@14onmV(%#skcK zGhVj(O__}02r`dYa1Z&{J?YA!D+&_)$ks5#kum0SeX|SxWJD|?5|QP?Fxi0|Zf^cc zG1J;l4>Yd3Cp=t{_miP-lz)ZD8jdTv^0*jq$Df(a&NMxkSaH4cRF+ceN}Kl1qy8el zwcMmp&(Gly&95m#DxMa6P9aAWo_0D1(r6cryRsj`+t5FuP0$bejc=@UCjGF@l&t=< zH$P>ToRC)UhHi8Pj*R8OE@)Ivz&$0#|L%}CDU z&#auCq8Ww@4)W_;8u#DIhf7T&{CM|aUw?KukgTKIZf$Rs^aH4pNH#ZtU7vm~(hIRZuw!ShTjxJdn zf_rd>BzSOlNJ6mS1b2tv5}d&$cyM=jcef$9ySohxIyiju-u?F7yStYkbPt?4-CcdE z>Xbc2x}KL$kE?D9gK3vny-lKwo6G)vt}r3!Z`*WtbwTm{)6t+epm$>Y%BgN>u{jYc zk*RdwU}i`s`8FD22KUCRtNmW2fd$7<**H#*juDUP@m6HQ{!U5hiT37dxwxT8Lf?#) z`JfrfVRO?L*xK9kyq11 z&=FoAxti8-Z-S!N9X6i47)T0VE>A0TNvojX@`eJHLrvL4-uUkKXIb}i8s#>TZoouk zo`EL;%e1|~_oFxikl&;lkep}f{)SbOqf69p^b1n{A64rE21Efu+_=sPcm@5NfcM()GFMQ^Au zvYa%{>=3*+R$)O3(N&c1)EB;IoG*Q?IJE&exp>f8RHCr7ehv@%oNxi$GY^_Y4l1Ym z#__6?0;|Mo(@$iOJs-1+ahQufGtTErn5_0|?M9DBhqX0PthU6R?*Wwg@ws9@bl7z9hO_G`kSi)6+Unw(1 ziVxdHqv~?`D=!P~7dJxw?9aqbeNxIhXhUV9`qQSa_5<{b*W)K+I7mVp)*tM8(Sxr~ zr6BVkt7$k5>_6;jUX6Ru_#nVJO4wfLoT*6=9As%lN1lT?1k>?a@?a25` zvwD2e#if%1yZ1JixA!YmOjQriIdX_(uu;@~ucb&^&tq&hrv3zMKvd>_fR*CLnm!&u zDL1HkO(9vCIcD)_xi8X@e%X(*>b*c7!R(P*V9T}=>i{&XzPoo=)E2Asb=oNF`ibL4%Yd6?dOBzJXa-EQ}J#m zGWx=$HB+W&O&T2gx|tR+fOi`{cs(!!8m_QTU>D4oZ)=)YuH{4nkIkfrOnsB643EwI z@~vzw9!@dHY9%Q~?ld$XJ?+o$aofiIwuJxXWgp|JH}!@RQQ^U0N3LniQjPdY$I9-F z0ZU1FcKc}LffO^OxtWLm$j?ZGO`p(2#s=Tt=iSUs$*<89mE> z9~R-&R!C7Qh(B`-TZRe7lmd^0Zr?6Xg?8jZ5`b1Uua4MpIMH zgAtBlt&&e)`3m8JFU2-&H*5hYk+sH8_MAYmir?%;VA;DtLfr9{KvP?qt$8zE{pUTRHM^6tDhv3FARg*&c3RU19rxJ5i73>)WQ zxh4r5?%|QO!31Q1_f}kxRuZ9gFyEBsK}a&*bX9x$tm$LO#FEz!IY>K%ZcwAMf+I5;`SS;-=bBN?#At+ z8p9bz3jTN^T(B!uyOLU|c28GMX3 zEgkM$*AV#;o2!^l1;{v(D!R6|xhP^xXBt*0h6j(Ojw7_!WQ{?iEn)_UO3PmIZ+jC< zMj%@xJBhq0+#C*?hM84W_NYcpcUY9O?e?c}d24J(7J5*3pb@tL>G;%nbz>O@oa%{Q z`0jLE^|+-oUp3zE%;(#YPgKPpr9qnhK)#@unie#I2EQr^wWmLE=~ku>{k}sd=x7MO ze2CMw7AEFrCax<2F|>cz(zD z;2X1^1-Wy>!`asImP2hT0?Z17k%Z@?l+}dH4h2V_XCt`HqNkG^5PfCP;!8gDTx^pI zZ>9lrrd;?E7YNp)@vOQ*N{mCIudAYPT;U!3l+xyyN9+V?tU`omG1JuyKM$gX7VL;m z*y$7eVIVa9g{<_6$aUz03*gOHU3peIGOad0k}k?sQu_5k)Hpf{ZLIh0TlNjy+uJ*_A9V%> z=$5W_JLMcDto~?TVDZe+nH#uFdiL9=!(>)?z074weG#7&8$WgiwJ=A{7OP0;FGso_ znAb5c^abX_Pb=euR#1$;qq}RWFmZtPpRjfGK`aosU*${|ap-R~}?He(9kzhs^s@XMvhDXv-1;F>N29&-|Eg~YogB{$0UW7K5A#!Rr0KRn@ zg^H9qP`qaW2?gngCjMKim3kt97%N(N8o=ElkaW3*x3Tkix|m4M<-vvS zg`~aEg3JZ&{FF}JA1U!_+^Qj>W8cHp=Ni-hp7CyX#}rg-Q5*9GM0^CY!J~+?K^$_PGw0$q~XgaX>lEUz7rAgTdob?boVG&8NmMonEBt0v~Lo zZPQ23y+glRZmQHYcq=H3qq;=+q&sZUFpWlDCshxA4QNU<;H#Bm(+D6$!_;QBBA77T zDjtf+eBoPIUIpEGz0WF-rv72im!z=nmNwl}m*Rc(^P4`Ptuq7V6LTfArkkA{9nJgj z*q#6fye2ssf7~C%U(YK(cF)e)=kxgT^I*22w9~Z#g1T3deerzX8q}sgEE*y|SibkV zcHR*N;>lH&70xiKj6U=gN!wePR=9K^TAq{D;DGs@W&C}sVyQ4taGmQx3z2W(qte~l znM1`|Y{}eQO}Bn{3vMPGoZ4p!8;g8ZE}HK0zK$5RD^qrfrLp47g#oWCj5H>8Jy0M&D^0;*SYC=mW7 zm(Ei>uk$y4*tjkBKovx?b@`rUR|clYp5sG_g`=vOiXlUNfS@q_f0 z2Z#*SRas&a(+}w{lAdgW0s}Gg7dWG8oPsVs7#D<@H*tDATp7D2bTHAvkUtJ5yyj+# zIy$nBj+fHWN;;tJJ{lRfo4F+URyi4!Bzz@AKB?{$vhajjPASFnO~(Z8l43)udF+zG z&lN}tr&HXWeYQfMyDj^$DZ|II#32yjyDGA;j+P`P=(*AO5bKRUg>UFg+BgCC&2Oilf%YQ6A-ti1ERpfv7dIQ~f$7H3gEq;H|;9V04&!Lilegq7M z_8N%4uk8CS)lp7p1vwL0-tSMBT3%%5t4$)Wcz_v1uSJlFFSu~LIn(Z&Zn4s^lrj5L z|Cr+O-8YukF%6_G=A4k$TfllTOcyjp=dc~jfxhfzRMfsYHi@0reAK8Jsn9RBTjKUh zC{jo7%(h(7(Gqr;Q-6EFwX5F=v}dTW6?POCI^hxJ<5T(DK-Q=5$&nW0$D0ed%7bal zfq0^-&9E0zjpha%@D|A-K~T$!KUZ2_&3FCN{sUUiJyMe@c*(Az(dHX-9rb(othetJ zwIj;PliT;I>h}FZk~sjZA?tc#Kb|)|!a4qw$R9xYU;4~E=RC_sJ#_Qy$KjFGNWcgJ zJ%hb*O1%8H-e5Bu36L<5eg6w@S|^|Q1y}QNZaY6| zbvx6bO)=wI{5~=7A%!!n2}X;rQ5XRuh~qmScmy#KNN!hB(Bgv8jSPw@{Z`#UHbWB& zYQMA6#zrQD=gJ*{<|85&vZIG9tQ9pr>%{8Q=X1*B)sydAUm6&~Sk~fDC@KqDYk%L^ z36E{38y_e1>tX^6A2rxqeSMqKhvU}t$3m&O0p!K+iC~=iF#g~K_1W%m{g9~`*4F&e z1gp+zKm9ybYR+ZltgPuT%yMKB!iT^J=`=pHUKJbv%$UL26{i=B?{yCvbQ6>QnQ@I2 zrGOPFxaaa-f%Z|&q}EQAtR}!`ykKEH)NV@j!ZVCs**(*-kupzDe1}(?Rr%-}Df)^%ditP2_Qk=iA=dxgY+=KNJQsn8fcJ7wnkQ2*iWBsYm zM!C$?f^BhKubvI6oW)g^LoSgE_ay28))9{h%dfrEZbY@)Vppg$n@E&waL_cr% z9+&`;WD0Q@e|an*OkIbk+gDw$2@9g@5_KP6ow=kB&y`L~^sSsWf7MP+z8xS)`qf9a z1pKZWiA9O9(zK0=nd^FNQ5kU(jZe7JxWD6Q8gtr`sg$JGmteXE107Q5@W_9PJ2-wO z4;+9Cq+dJ;r8Y{J)%^_uAPrS<#LQQS_LY+pug#2~APIXNw0!g-*U-`sr-JWyrj1_@ zP!z}BZ+T>^-yZ+ua!KgcByfy#ywJISNY^sd@~loi#?!1DGd#=#8@~-WiL@YI2#>Nb+Em7(seo zVcfWf$eP3U`FP`K)qG5g136UH;jYU0NO7$`k$xNSp=%u!Sze=nFc!yG0 zYwXKNC;W82VHO3j!a!&buJLPMUf28F!OTx4>1i!lvviwxs7tkdW4}kLa-7~ZjX{k7 zdRA9S4q65EH7LlJ6e9gbM=Hn)Y<{fkYJ6|Nu6?w^d9=LN-s-k8#OPL^tJbD>Qo8uKhd6Bh zn!I8cGC)~H+r;JTw7WK%JR8^UZpAmZ!oLLA{P{+Nmj;Prsq@Jn)ZwF{WY=?S1Pq_1O-yPVsNX88_#_}4>cZ>vAb{oS}6s4NjLeteu&1Wgvp-%kEqzvsLVwnBuX^xNOv;rE; zd~LVE0p>obMu(F>VZyNRoR%hkV&b)Zs7kBnVGyz3rb`G9*R4oQhy8RULBKp@1}f(& zKkC{JI>2=|K0_*BPL$Kkh-0(^1a8QFqw*6$|MqZFLOF!>z_6YmPAreJ9c+K2NH>BC z+2j?J>afHsQG0i2v{L!u$+zoWxJ|)i+oL_u#pTnG76&ALt7_C}MUOHM5&dA88tJa7 z#-ZqOHbI!l1&ATLr7!711G{O@KnW`02XMd5OX&f$el?={+93MX>SquenaGz#Bj9B2 z@LO1PO$4IYST?U3xW1hE=dbIVF2>i8Y}-#{g{&Xhe}cXpzS0Yh^rUD_>u`gwRg$pogq9QzCJxjaww6d)fAS%|4bW9B{2a`nRyc6t6c?JZQRE# zFDp~VI->J;J}$~ZI*t5!pupZ7JlDzN4X)3b#fx9b3XOQ$+x#ws(EvUiYlxc zU#X$6&CS_zP|9btlY4%q_ouyGi)iwx{_|~;SA8|1>zYWMG>wx+^Be{|{YK2^`v%U? zjr!`pWRS-BnU&DfF6Sp|r^`ilFkV*oUFZyvfYI#H$fde@?P0>fxok>LzWz%n1B|ok zZiQEdYa$ppGE~PAfW*0|#T8uA^@4jO?!LQ6sO(L%4szS{o^TzmiY@G*z3*Pk}-L5p*Ns9rH-dOKMSvT@$eRlUw z!_^mX(_5>O9E(079(<(pW_cH%NL%l91NIeB-5&87@P3Y{ZxLHSI~+Hkmgl)vc8-U{ z%tsRQ(Z$gIiwVcmyN87yoUnx#!ZuDP6+>kau^E6PrLn%RHm0+?S9;NN|0aLpx>RLu;-6O@viBv$zIxqR_5}qe$)`txbFOrIUjid*__xw11jJe{PeE--^1Q zzb{0uQBFMP0B<lAMNs)}m(yT~Xd!J}Ap6A&iVLGg$*a#}Q#cqG?Br~O2ldA&KG%6e zLKnN=>2#7(uuBSS+?@~!sRcF z%#G^)&fQGwnU9Rh6%=a155^Qa#{#W6RFP?!ZYdkO88B}2ca~J*Q$NrtH~Yh)(}TM9 zdPJ-G_mIlv3!g58DY^tM1Ix|Fj7Pl=;ZUETj@4(1sxc5QzuCI>=))2j$4A(_gc4kq zgHUg^cT$=ygbIIT58MQ6F!TlV@xiPUM&o)u)Svq(CUALmfwO$sw#)3 z#=9SNn-Sm zbrJOUxH{aa25}2pR1%S$rJ_Y~7N%ibiF3(!9*ITS}kL|Ty!Hf^1};nIHTFghDb)*mfA zNOE#BpGi@jW&in2aA&3=KHyfF|1j7K^%o`mo}EyM~J`4o2|K>sH`>si~@3QzB{gob(TFNOf8!9yY@GfsSNe3b-svGl8QL|09~O$07+YUVm28g=+1B%y{wiO(2o-RO zl+qM|RP(0k;C_Q02DWx&KXa(0d(sPd7uc_aj&h7|(Ydek5mMt>0L?fNa>H$&G zBr)=G+Xs?jA%DXSKh9EkoJQAfjm1QSEvieWI$}`=lH2uH`#MvMVXPJgR#)Ci@Cxhc zBfsGyB1c!4tYdT*Uj@5oRl5)eKQU#NRM9xfpd9eqNijXERry!@8JjwiGHeyHk2?tm znId#`R4Zwf3y_&>51ZsCc}kIoN5TEVsC+qhU#mQIPP-nGVO(ezzt-5N?Zo=K`!i^K z{2{Wnr6wq}sTLl_QwIkUrF*WHpj;1_!{XrIX`Ad&{d@Z)cy3a6rx0CJA?b-U%^{H& zlb%lYk&zKDwB(;4Oh*k0|P9Wc!XIT?r`JEH|YFUU*Ix`o}3kzT=sTWwE0Sqa?-vVj(Nyx zTgi99ebT`*EJN9GSvYDEMuTc1GIhOIlM{u+P}(u)Fok!WP9LC_#?f~S={H=Tz?q`d8;$-F-&>phS*!2|Yo)0;+kqhc6qRH{OI&x65 zK7#Te2?*Poy|)`G2*T3=TlrCe%B=>YtUtht%f6xK`06>z_6RBOs|J^ME{qOsUt4v(;29G9_@H&sG&p~g~(=_+zB7LB7b z8=7}PZ1F0Fhl_sp6XQgvSPiax=9=We7*^HRe03eY!h**3>Wu$UX zY5=aG2*Rh2jJkx!^l^uvcsx_n(R@mHEYARa4JDQX+-+^+(po%$%wK!=M$Ol8+7-i%jsN- zj==k#%?ra(XEAr?oK9V2I)P=ud>35#pKrgNqV6>iIjs>1bwd8Lnq%3QU4S<1ZoH=7 zIikmsr@>Xo#N;auM|`?N=EVd1exkuBGXfm+?Y@;ilj0jjOoViujiZL6FwyoZHA)#t zEF&1e?3@J{hc5CkCOxRtO*&F`nmOb62xj34in`Tn=o725?kLV`k7z~|&8@9|I@rZgw8%<`ssdt#NzG6b-94w3Oce*TYV zpN7t&q@*{!Zb(Llhni9S9GadMf$52!5K{VQIbQty*MC=CX{FT!w7@-kT<6GmeMmSH zJN++IwmA>)L|lRW@)!T-5TeU@(;x2ThCX>}d{1-r%%PR1240mfl`38+j9em-|cN~`WVljSAnvOFSywEhR2>$n{kyh!kYi$ zh;g1Y<}i5_>BT%FXwk456cELKLfvzCq8f{UuwM1ohYtr4oD3d@EG4D6xj->-Reb!+ zR zaB<7hMS=NdL&O*n5^xzfvCHp2V8>jAOj$R)#m5g=qmWz3-~I2eh-t@h>O%3Zip(po zkm#$*yA9YrFRrftFE|;fsFM=pSGtKvEXT!;nc?&({m4!#JBx7gj?5PPU!|Mlg22FW zhp&<=$RTf%BmYZM{`zx2Xq}DORh1a!z}~-v)4gW{PW~4K|8X|X2wYyaY?rkBb!E)4W)Ctab@|r!I`2YG90Fr z?EifUI=xS9@Q=Dk_6chKQ&a2g|FuHYBWUXqs9prxC69}XMgEg)P|Bsdj;3e|BE$D- zjFSIy=!Z)4|6u122jn|3pMJkHs9&!Bc{uqWg#X73a4h7bIby1JaL$3~h{*6-&(u8H zco(6%HIn>nMCcXXUf3}ZJj{@bc@KJ5%Kn2g8?gPqk#v{L0AcB>&n~6H9+OfZaW@k5VXD>>g$&GGFpKA2e9e*XJACcn2MU(j_jF>n!G`@ z{G`&X@c}tNf zpl5R3@L&2j9s6&lV<)*GK4@XVL*1#UD9kPY%G@OWwbOi5T3WZ{Vpe&lq{np^nJ91h z-{tiWI4CiPgdZG=NB^=-dUF&#L#T@k*+?9IM$kd|Fzj|OQ(c;5{%qmuTK~0wG0=G7 zj(#PpuQB>%nA1l)^yKj=F%Ecfr=Ml|tv1A2VXZd+?RL`+oUAqC03H+1YFpnueMB^5 zPM#?DaSw|ahF~h^1Be+v4x=DG^9cxC?>7DGS$_S&k}3+M#!P4_uB89&2)zBuhnL%$ z8zE-OB0xj^*M>=iEvX&<-8!NpM8P;rK zOhvH7`Iqp#I?_5qyiQ5=NzW5@R)7 z_G9(Bf8^=x7+$%mE#}%iO+(M?Y~qJjMqkUrwDGW?iml0+T_I-^;EScP2x_7TaI`>< zAcZ_%k3VW%jg;C5UE;~l;6HfMrr(0A7)lcrKD=IwCqI?JTa&AS z15G!W?euH8+E})I>_oFulat}3=2M{l#d7VKwU;YFiIOkBpC7O2&UvV6m*Psf}83R~~-P^R&(368PcAND`|Mb)mh}6#qpXz z*K+dBe8=-)UfvZ1l_}TXOuQ0bwlKZ#4}!`U-x7xxR8iuZbMh#IJy;z_#iM_(JMne7 zzsl&PEg|{NwyhhV7Tz|WY{#9lI=--bShoK>PM6B+1n}!PHUFo|{!xkP1@rzLvPc$| z&)Z)o6SG+p)WYKdZNpf!NJn*5-R?W172ud$e3JrB=$ z!A>izv@!PfjKl~czfSrBeV!G(**_@X_j*i?3ElKk$r5h{`bUYgRXoEvCJ=P7MvpdB89h)|^8rr(0kT_2YhJJ*$Z5FQML_HPP-r6VZNA!46 zHbFT9FXsVWK=n4&PLSvQLd(jzMux20ws2itbIEcmV^@C=suL)QDNGs@FI3@00lAe zniS!bn)U zNoVtlxr(nTEJRdS!e&WA17q9$gN<#r{85A&Ttqi^!3SxqdInGTv!&DX8Aq((u=b`W zpUbwGnp5(azeQm>HYuE)XX5QESWHCYnt@Rikg|ff2tzoc4+wXrk!y0k7?{St8A9U0 zVWk+~z}1DY6D} zlY>N9lzhEu!jLYKTSLm<-HU_kyA3|C_`H#n=+K7QYxk6O9sRDo#&lRI{WeYOqc*&e zz$3mbTKnKZ2x$2aPZHMo)^ZC2$Jx9{{>|(`^#^7{ms955Utr{CmiFJ6x~<(gsgS6@ z^l%t+khY$F?+#sZH-zjDm7M~M`-3M(GW`o9d!77In$y%A(#k%LK$!jQw z^UBOpL>2Sk{o@CSvUigKimTpY@2SH@mMp<39I+CJg z-ow4NnP7Qm*OY{GrHY;9WswSOlf^hUsJf+kb?#p^Q#rpY=Mj+L<+uGKmOwY18<>r? zvGrigkMk?iX|vdFZ7cnS!>Wjq)Yxdp16n&#Ddj&vTrDxM&Gv?x4bmlIdg8;a4?c|b zWp0k4ndtho+;_{~*c5k7Y98ZlY|>$VqAm3F2hB_AiCI*zVoc<@CT$n$R z?t83m9J>mUt_`k^fU*?cU8GT!zjQg-Y_~jbTdg^Wz(G^h{v4hOIVLJlnQ!a7iu|qR zelUtOcG)w!PK*IJ{3|--8ZXHqTX5w{*>WvUw#EDDbG7fv)6HA-Dg zq|R8T(&x&FrRR@v?sfmhC%q}4H2Bo&+aD2oz34wJuYWJ9>37hD z_q>kdz1}rcYruVfa&R@BesFK$dL!+MAVu~_&-!vi5v)3*Bwu zo*Z0O(vKCfTmHb!)BT)T+fND38zYuuwg_VOg#`Ebs>?gh@;<*nc|kTL0Jeg#(9k<0j) zf8Z%y>f(&6nrA+EUDxhcYpN2zXt?FedI5a8Wnevj(Wm{5?4~B%uW;u1dPteKuK^UQ z4P3BOfc5ga8Dv=Ps27Z8c8ffc-|RN>cAo)w?U!zdKyV5-Zr{4f6Mbt!GpgC@(`^o) z^Lez>bGH+IOnZr0^k)~zfv!CLm$;`qq+GjR1yX%-hf{+a^AQJLMkI^8$3@L2NM~A0 zyPYeUlE~_P+hVIX4JC_L+JyX7$ZJY}8s9!v`|G}JE8OEoONM3{s|f^@qQl)RsCFyg zsq^sB$HSYwfbb=L<7uA)h$Z*F##U5Y)GL#2zkT8?j8pKSqzVdJG^W$-|HuNn>UI66 ztJbs|ZZy+_0U9TBZ!;38tBi;&_cY*meVBU5D?0PRs$zAE$5#6Eva$*o$t!pGCvwT< z+FZ;1adeZ@?TO%pVc*o1QDxf+gZFKg_p0b4_`6pnflL$Z5w*2@JW7#tF(2v2tBr$O zPRi}N)UOX%L?aEN9U(>fM?JW=BE#lIQq$GWTKue^&3@)ZN9$9FL*2BvP`X!Qf9g?3NH%Y z%y-m$(7mEvR=z7xdHwD>IdZsGquPL3U4$=nR`4aQ>*qvn>&fQp#QO8#oEHR(%%^0Y z&OtsGH<6QfpxAoOk#!`|u%&LvD0Ou#I*r+w16UsE!0D1kxueGu=V0u#W5(5(+W7m& zr8s;4v|;*}j6#0%>4WmS7l$y70$}|Tv~TvcMv)Vs@&urpHBlpwb?dG#1OodRU%(Pj z+{I>yTDEjOe(G({E=%HN(4@84mRpcq%f5rrZZbE#PdG6x-~p&FceY6@=sg+`uSI_z z%zm7LN$en%60(LpuncJ_ogKvrYLv5F4wN$}y|Q(Aab-u@cwQz(z8Ktw0s&S({e&%R zh(MH=f7b?w@;*05LIKt|rKyDz!uf_*Xi^V8_i*R-B+`Qnn>CaW?4fXkUfCK>!?*_OlZCn-AQxzR56$ z$;pHb40bkJt|n45ojV_NAm3(YVD8aE<@5dC=G{I=N9MC|>8Zqcs*vn$&|{P z$A>$-h)(YlvJwd}x2>Z|T8%wX6j!Y%_Vp(KcYa64xc&k=D|4{aM1%2tf$irZ^6`E+ z%aTUsZ2siO^Y>oEuk;-Gv@s4NxKV*`zaYQ=pIpp|HgyAcy?ZK1y0-sDei z)$v(PvmRHQy2IGoQ+htPJ3^j+gXu#t;jfBh^PcaK!}^FFk$340p9ka7QEIu$i`= zrjgm%*SL~O5sf`ih%|i49z1+I8^=>XaEa+N02Ft+voNPW{2qY zr1}K*-!Zgb6;xz6PqamNk9TwWfCqPagi&U+h6Eb}8c@RB?RFf>T`1;)jrgDVtz3mZ zTf6cK`BM~Gyu0{Y4(M);%0eSjTfk8%v66HPy5;2Orr>4z8U(l$#i&0AirgDXU+NYnSm` z=D47c%nP~L-Ua4mgiPm1OM3ilax{6a!qWdGO463aDz>zTh?mK@-HWhvsDu2z-+3Hl zJTjktiKHd4eh`!g2Sxc0c-b2wKJFP1+L#OJu8d(~?D|}J!X7vY(UMKeb1(KpAHw%h zF<~{c&(4Gxa!NH>po9^5(D3(F(*{$#=n0))TThQQUig1_z|Mvv`+0YJW+O9BFK?_y z$x4BYnjPmu6o+Gt5BIZU@Z8#5+nl6}@KkF@-sHCLM^C+_R;oN=_JRjw`%|h%q4O$rJDp zh%F+Jx$4<>IIjgu=FlbjwrE%Vko=mLzzZz%b;IE2oHc_(q!CpL@qM#13Q*?tZ6EL% zZesN4tH^w0vh&AibjmM^d1B?BqJFYsBxpcjHPb}Xu?sW4mN0=_pe^YPBJ=_VTqkn| znV{Ukis74I`Ko6U>#uwmJDFHSqH3|6<-m!Q$r(bQE+nGbYpFrH2{yB6BL%MD)e^;vHK6v!9JWh=C}hKX5Wyg8G>7-2@wEmrakz`a@>eny1fQBE9n z#5Jq#=du06jTg*o*I$E`MzvhKA>L`+(Sb>=o8W^H<2ccx>Pt`iM5=N?3+W4V(#QMD z{@ID4k>{`<9e|75lipBfBo0N+|-mD!X<4>126J6f6A0?9{mc~DCq=4?H z=F8cfS>Px?Ql4zlCsM>86K_xSRo)#alkf>_mCsF=-U#22W5Gmaj33j#94k4jLCC}6yjyjbg-gOSQ&3>)~;%;NfUkq~YlfDV)s42VNX-bQNfFpsm63 z*Yk%z78229qgOi{G&d1pT0ej5ZV)Vze7xPvb1i8(QBK&&k~hEpYfOa6lRtIDM~$eW z7)>SR>;l3H9sFw7)>g`l4zx4oH5=aPv5-YBV>USG`-lA5NKUYPQ-61j&RA7%Xtt2L z*C*Fbo z@|E1X$8XQa*YW}d6FAc|KiIj;z;v>cAubKr_ALG z8TwL@eQO)J)*i3k&dK4E=hd&GQ_q?C*YgWo+-xIR&C%M~%iS1X0|B0j%8;~QysR;| zgca{O1q0+p)Z;WAV!t^pKCL5Bo0ETVK1J;;Ui=`j6V(Ro+GRZdNtkFNa4Jr-QN(Yy zLxRbCMoOMR;p_*oY!!^QGhTt}yZ5(CIL@P`p_#e*c4X zruPkURs%+yCD3h`jbI={ZNW!sL`6)nEX??E{p&S`gqvi0)+}r}G?7wdcyK&qAi1-hApooR>~|FW1U&40e)EzH zmapKP+E6MWUzBm#y0hB&BPls{ZVpdG0q$7_JH3*wfOaTk6bfPs$1<_cM+qhI@qPop zKOYm5uj~3>U;+GW&$RG75!L`_>MI{tvpxL79A0@)f9l;KA>iWN&|}MGbgz?x6Y*_G za`|8^VKsjs)m)!NNnDs}Jn?=_A2rb5r{Hich)I3GK>U86;WR)#0l{~$IO1?bL77Mt ztyi~a740Bl)nB&UI!-tjR4!CD(rVT8eOg9 zm5h3(v$d)obN?72B^LA0LFsX>tyPNBk>ZUkX3K#Mv`Rz5al+3>Ci{>mL?NMS(w(Y> z7GDTYF*Vzv@-exg0~3v7=1)nwm69dx&J}enV*h5v;rR)4OV$EYB)@jyfS@?^;qulAXDEFS@o9wd(v2+peN}{bQ&!4vf0=al05Z$r|mi8Z%$IKuDgV; zN-W9z`nmI7I-Hzk>)sT>5t~cWS(mv^nEK@!-AH8+SVhP62racKis~wx?EwEqaoqH< zTO+jww-5zt7ZrVfp^*(Ij-uzww@A$E3MzB*1J&Md*lAeVEGNlufIVCN1Ec!!zm!is zr2K>5etAmatGD*%E$V#DoQ_rQ6!pd-;VuIwS3WrKf+I;J{gn{pw0kAmheWeFvc~)9 zE@dCkn1rnh#*C@l9L{&1ZuGFY3f#TgoVn3<)ar^rRsApK*l^b{SeKU%c8BDm5$|6E zI;jTPvdbuKQUhqrc~M4oG%g3QnF=1Eq3JYHB>ya)fWZG5@aC7?}6B64iY zf~BJ4puhGO;`$i00T@ed=J)zp&&;aMz5a+~+gkqnt&c5g*e#zx(9M?}1=R&1+X~8^-gY7uV#lC?!fEK zj+*TAt*8;C&cpt+BLCEMjvXOJV>Jg4{C{-4bwC_Vwmu9YxQE~tg1ftgV8MdBLvVM8 z;K76Y;O_1&6Wrb1WpEkbn|JqjclX_UzrUturn{@^^pWSBda7$ey89>AvV{6uwW5~i z)>3UaQ|8(f-|P2Rs8Y+!6TTR5NB%k6hbevZiom03k&&&e07Of>lfHV$Rupw}g=X12 z&DpffO1-e``^V`4KDrCdh2yqjA{P0ORTp`4J0%sRsUw`MYdY=fn#>70k2`uD`ASN$ z6I$LRbqz*(l*PDJBzdd5DRas-8Ep>{8;lH^Vb;8OQEu4*uMpJI#PeR)dkXqvL5;~l zO*$XUzug@tn@=B*{_cY?RT?$ceJ!2L09aILaF$anMlW(JExb|-99VWzT(|=@N&)Qn z>bUyjhK!ljX{&ncH$H^ z91tAf?d>fWf5Vw1^Z1<;B@;jRP@1cl@_wHVtpnZY587v;3E(Y;Hw+NtbJM1UX2!%{ zO|^Z517||Z%Wg7yVTqa%8_G|sI}Lq+|F^5ykqil{=O6u6l%7xt;r`i7loXKW_01U`Pv`)Ab{y_GFNCtNLOLc z#~raoY7@#;Ozb>abA)<{>DnTk?PzgNUU7s$yQ-=8meBU`#-$O+Nikp#Xm7}+0%`G%*YVe2CD3I^1T z$?ucxY79RMdXNqhoKKe(9MmKv2%G1NYb%d`(OjE<*e@=|9aV`m)xa5IXP+r{RsSmP zHv_V?9jLT(A*yR!Fib3}e!v?_pLS=e=1)uNxjDXq+JCk_F<6T4kB*N6{4LKw!J3n_gFEywV#nj+ zWr9Z$$@YxT(JgQ%pTc{vN&HKVe^n9?GEJD2HGu}I)!UWj z)f6%kuh3lP`PYw?X+DE=sG`CR3}$7#o%;jwUNJr0=D%zmwBOI`Bn^&j>X)G#xSV&B z-zQ$(-tH6}NZq24+2;GB-CxQ-u4EYBFui%6J|YmT!DeT>hh)<(_m`aN(2Nw=Pf&%g ziQ0F(jysfueeNUkh4U5a2nLg46STC<(!IY&i_weEfNo@zf1iDb7zL1hyW5MAFRZqx z!qmBWP&thL8Iz?3g3kuu)#) zOP=!~DGUtDKY*rj7kr@U_{z6b%DY=oZBkXU9~coyG*15(IR8k}-cgxUqOdH<=5)pt z7Z<0&D@fGQCs*!}S@O}>FFc%>Z}sAKV67!u7aZQuZo$oNnpq{9xSC10Wb$qj^F~ZR zKno^i5s;#0Y!v~CBkmj{pheR?VZs?=dyNc%Uyrh zH4qfs+^5T>`1_Iw-$GO6l)zs7xyija3pv|9c|J1)u!Hcp&|C&gz^Ac3{Ue|FOv|pU zr33h&TBxM;?hdtKu_w{DYFsFZoSwBcvsu2WNp1LDwYqWw&oz1825j(f1%U!SbG%u*o@ zo_KTO1C995DNSpYKCq(T`4hVk;arv8zXF}_smuC`4(~_q_4%=d~JR>({XvU@%pJt@qR%K?c6tWKfzX}HJg?6St?e^|5q3c!pN;0 z#UDjSe;A078>+%c^1iJhbASuPG6U!8?ynXhl)3+AORY{Ao+zCoF_Y3B&95KmiZts{ zzG)^PN?j1>98SR)gW)`zQb-ug?SW<}1=qExmoXNFs+E>Ldl)sGb&_Cuir`RWZ}04!zoqNu0P2#bjVK^s{|T{545TN{8`(?D5Vg`B8&3Ge#9*91pk?%;DWe0!?<<{fI&$;x zhUm9vb6VW0mYy>4mCk8VFIadpVA%$L&jjumCzTo0N1`QHxxGkXZblLE0}PctC#q@O zsDra8-o`WRaI_j9Gc?7Dj_#g_V)2bheOPg5W6KXA*(?Is!P$uPnkVe(pf|q$tqaws zdX+DPvevzZMsiY-q@9tor{?Yaw3BQZ9z%x3G&CvSIc#P<(p+w8zRwLf{iO02ZRY7)KJ^r(V=^BcLGd8gn+jth^25tE&996>AdQCfm z0ufuZmuUTHIjls`**t){&h=u~N=xGo$nNI@cU%>AMpf*gt;u4;TCpG9tzVY6vY6;h zXw?6sO0o!c6raKCKp67+(iO#gB$u#zHPwaU83I)=;_uxG={?eNr_!t8WZnpXy(&ae z>Fp>ir+-z*{t?+7v=rbOYgTiREvJI93~*1z8j!N+Y%ogLVrA`wzxjX&nK*2RwWVb8AJU}F=jZLP3@8E#%!cU z3NLUt*%LovcKwipXRpTNcVng02@Zc7smadG#VJc{WUq;$4r@U0E9rGr%wsY!M42@loo{#ZqvHWZHY*|F0yh{SBhh|%r#nV2(bvv+J034&D=?#0U`SK@qCh)B=Fwvi673iNXobG?0p%kxff;N#dd+#oQ2DY+dz^cA8WKyE z%t0SFRW-vNMxhZDEG2W4iE-A}z){_!yPd}^R3^Vx8?twsIl_e4gn{ZXnJgVmrFuL& zgC?RZp`gFZ(jIw2np6zo*N30#8c8YsIp4%*ririEzUgtL_DtEu>C|LOx_~t87CJHJ z%AIr%6vq=1`=f{5Zcw$Cu51`Drqj`DEPL4_0WB$G7JHP`s_Y>z+M3+lX=bmi={Mn( zzT(zhfN)k5xWGR0)CNoXRO+$KQpvJ?$SV({L4okB!e?+!?G>Qca8|n>7xPE1u-Vh_ z8mDh^>uxe1e*D+ty(YN1LTdJIN+)z{nsjL0WO(CsyVV63MrA5=t@tRzxUZi8mu9Gs zoge9tu#*VLbS`t1VVomnFKStM-GviQkwk(t!9M_d_MIf_p2RQai-^ z>XaqzvrOP+guxl)!WKv1Q9QVZz6JKYluO>5%#QHIN&*IWZpdCy?Dyz+!+mmS)t3Xf z8lJuBi*Gu_b~_;h-u0YSO4}hA+XUglfT@eEG{svS>J3KE)Vc!^5Q$__wVwHD_jg5B zpGUS(*MsTigV2YzD)JZPLmqD)JQSLW4hP%!|4c@&=FZ;H9SNL@-Z6Mp4*%vu*Om=? zgh#9Ndr_Aw8CbJn@osukC)Vz{UOz4aty*t<{l%ZYjX%_`>Vz=o-|EH{m=!~Q-Lzcs zZ-4Ige!}Zz*9RzEy|E3RyCJ!an!N>NV|-mj)LLCD96WGRMC*rVXxlUIFz|-xc>|qsPiz}fo3SM=dj?Vjp>j}nfPLK!3goV`0aqJ_d6kX z`4`GHOl6puM{Z6*Fi zY`OeH7KJ&2q97-h=&ESf188EYDxR-Qsjgh}bS&*Q!gaq2K!)8-`Hm^I=F}(Zh0qpk zr|~*v<4kf#)*4OL<<8_b0rx$>qIiTM2xHq#1cXrdw#XWE3?)n9m+@u(;%I^IU&+N^ zF>zbIvIluBtJV88^qtn>D(|x~S(#sJtKAnL)kz43;uT=|fjQ>mweBn&#;ymvXhC2& z0qm@&_$&zBOSh$;cQHX$F=1@GDhrT@X$yziE|&?jQ_7oOK49a8pD;81t0^$+H`pj) zu99rXFSk|Su{p+ZU%Gjf$y}3}6`gzDxgFU`Mt();>BJ_tlapk^RPXFqL&X1z0BTeE zLE$rkw65PQ5y$>FEBm#94{tRj?)!8kuutl@=_o2WDfL%Z(D%W7cN5y`8;S^IVjT*o zR=@7_l{XCk75}%!i)Q;ccr=HgoJ5)%@#1xpdBfnKHrk&dg>Rhtdo8zjYUq zC)p^eq{y>#qt{p3LYgmw5V#p3pQ0zn5|AImk*-hpi6GK(*)$Mv!jTg-OKY+oU-J~y zIOeDLml*#!3J3mTOV%QJ;=Z{G-mz7I>N+ch4YDx8X6GX|NbfR)Tu@XQD=+LJ?a%u6 zY0mI6pvssR&uuTu!YAcEMN9ol6#pFCDpjf0=u$h@Eit4XhTs5nvhQlcHc_Jo$^&|r zH`xsIH&`Jv+W$%h{^w{_@pN_7MW;H+;J_W%zNK>q^S?y(*VpEqMf_+#`J)LF4hM_y zP|32V=zm-8zt7)}-Z4;>v3GRt*HQ}D%P9E&y;*~~FL!0-7!BV&dC_%=au@3~I# zbZCyYBiZPU7{d8GRnc9I96~Wz>vKEYq>=~@PDH-1EhyB~@C1vT-KARpXPa{|k{VCb zEI1@kF|NMOsDyuh;=c`=9Lq4sGM78KsqMf~-K^Qq7;4IwQ$20XL74|oVF(v6TImx> z_9EUEK6iQ$@HxNn`w$~ZEv1I8g0Q#mN4ENh;@#`*i!9iQ&5)6g|F4l|&V60qs?zjw z7nU4Fc0XC7emGv)BVu&PJDEIcCpr%!zw86-flt<$L3nkM4tEjwtS?Si<=ICc$9kJc zeyo5Q9|pxPU$px)nt{^k)Kr%fDZZ-*mRsh>ejaA$NO!xU=|CfQn_1)43MdbL_lvU@ zioIF3$KIOol>MsxLzmPKX$SyP)}C_<=Zw^cCUpoI8i zz1=|Pg1&)aAuQ4AA!v|IuU=nP_E8Iw%$Bq*mo$PWJxl&*Pv&rS&F&E@nOVQkUpG*D zy$a&3);X%Jy7xwsPxw*(rWo63Mq6V0&fu8&pS1-->HdccQ1c}44IWY|is=&iR@Yv&l&tMpKrL59*3Q+NPJV%&w~>v1Y5{W7 zdu=0SydeBzwL?O7@nU#`OgacwM!zzW6<+39pb{!LP64!JS#haaTz#65#}{3Rq(wpX zi%XePd*yOP5juH(?Hz9low{h@C7KlezQNV@c(6&0U(1t>=0kGuN;MZX1MPI| zsCF(eJ|$no;(P8Yt0uX+T)+K%pZCk#m{;3r4dMLQCZb4Fvh}HZs{SP|gG@}Yx=u@m z$A>E>o?2eX^>Lb!bknFNI0V(sR-T$H=z-jPQK4_ZNX94hZ({nosyMs-p}cugVe2%Sp(lkZ}eL24KYq^Awd z78jMi8Tu4)8j4XR6oteOtDWKq4l=wPHo1FDlH6$=7QOh&LaWAR{s_wet76p64+Cw33NHcFq7X zoPCH+K>ksf+kQg(yIXE{;_;k(w2R5+ZAgPLLpoAO0XE4%X4_YxE2}KdVtwE5ElD0~ z&EhmcEW=nOM{}S~*qGNW#H)b@?I$*e&3TciF+1gf?C#m>m(l*cjN+3OaDl0BF%#86 zV?FJgTEj=jr+)Qz_I=g3$i#SZj#chboSlR@mgqcqLrx+~&U3202L(8Rd93ALe@KZW z;Y=>57#EFr#dB-5nH9H!P8jTtHf;Oy2@<$cS9h8^jsLF3kXV}u6zR!RmVdpwQ3wf2 zk#av{kA5{5krN)@^p&5vt;?gpCvIVOMDclybekvYb~b>Q42?1Tg7>jQX zYjV2p{02Pq*f1lS0R8xLVuzJ+O~*{_Gj?}HtFLQm z_zg2RP=9pXBTT$9=AY&H0p}Pvj@_oHeMj-cD~vLd9Iu{!`|ucg3|#sRZ<0+Dy{Rxz zGvEQ%FZDkdd%8mNoD-e{PYCh3<;96l-Yl+G( zmyP+Zj>X}Rc<{RNmG$|DASKQ6dIPF*OByeFa{|mqTTIpE|tdvJDsB?i2!~e@^`qWx&A4{`m5~8{B z8eJ&N6^i|>3;zHm!*@Mj6xI6+QeS^JwzJdU z0R68A7SxP=)n%k*r(ZbdM)VZ{>NKs5v#4hePN!V~C7Bsgoc!O71!1@mh0YXBD5>Z~ zzUaDvoGkEp=H0&ipC{t9)fIlditydsE@1PmExjdyUdI|tzbu(T1z7H$86kr&-dLrT3LXiGy0Z^d4CW_Qph5FQQjc?J3#!I*-)Qwos?Cr^MLr= zM)Xhc(v&*=q$ttZ5t5%7I>d!HB$OgDO--LdfcTTfNm zuDUI77_2U^Qb&yxVLsZzCi~UP`V(n)hLI66pU<*mPs{uAy#+!Ids?rqeF7Ka&Y?et zh;Luy#HBU|$a%?X*!3j*4~+hYp8QJR+dQ<#MP(8V97%3zGeJg6g=mB<GVW z%2HR$Me!KEC!$U6EGhL^GWyrH4%{P$111-PM^CFH$3_giuEcLddchgmM@SXRLccAVmJS@<))AXms?aS_@QFtG zWP1ZJ->Z64Y=}ho?V_krkv6KL@Ib5s==2L|HsD;bkjkdcW?_>^)_U6T!yuf_N>eU% zv74K7alX7)2=s)I=*=vJ>>eTHSB5SuydHTPoJVzI|5{o&Y>;^ap``f)fw5^x%g#=cS{)b|q_vN1vjW_^#q z(g;v3ICr_BYZ?(CA8n|@f`_IS@0@>#NQ`Tz-3O6x?euL}88hb0!tdGzuDALicZm>B z-I7Cvms|Y^5Ozle-#5n)n-^EA8hdLzPqURfdi{RMT=+wd*r31-mbx~pu>&mFdoLIe zhs}@W#ie{)eQtSWg6q6K1Z1z-yIW@zQnQkuvCQ*!lugsq>T6v#LQf>DjSIOw-|p6J z$}>p9y2f2eV9sY?B)H_fq7P$5-F-CS=(N1g@4ud?T;G;k_@Xn3L+EewaTD7JbI{{GvU20GjNkue{9xNfi%f!CCx;&0lYo~A?&nx3 z2%Pbwm?yoWt%hO@^r&@vyn`vAQoL7!BtfKf(YTo7$Z220yWCBuOld2 zb@T-?ML?$`Wi!SIXT`NvOV(;8yj!F~sUB4!IqE9ql#TqIX}@!Ve)oos#IY=%9C0~P z8H?W1Xz7QH>GU?U5Qj9dr`+9HqEs59cv+G3f$0(nCt=yWo$*?)A5+d6-0wYt9b$#d z6)s;T1xUwQEDT0~%En(WJJ7#P(nmhGP8|+w6J1A-;MiPIYduYmh937(ZD2rcig2u6PU09kZ;9xZR}_SJX+gKs=&Yg2^n z&YIF=M?wj8f4E{u6LV}+JajT-)L^WZ?6AWdCVWjhXu$q%p^gHy!5#lka)Sn|1E!5M{u6G>Mxi&|lRqN0+ zc$%D6AL^9k4m(|PcWTK5Fg|=+5oHos5=Sn4kJApKO27U%EpuUL5{bUny4qJOK9b5c zlKhYw%OD$FA7vqJ{95lOC0vnRVR0&vy`Nf=WK~39Y?YYdvqxk)h`Rr15}N&ju{}SY ztqqh!ywcFWUL~YW>6AB`lm4@O34KzlOJhwqI!basIWGTe&;yXvvZ54@;;waSq z#k)5{_OLTAbiRpw9zez86L)Tzp*L(1GOu@wBkaLFgPCS6=EU^4b1KHqi5E`-up6oj zw@|lnj$ZQtRju>)J;v{)yjeEFpPahzv5SoY!^r0aya0kDko6BLkDebm4bf)1LGn`R1}7ucrXTF8nB}=x7wiS~ z!x-b5<@YDYuzBJKb7>(qBO_iAvM{a44&k>RPE+G`_;6{jXOU}X{3mJ%dRVhpV0x3i z9*k0~P@Dbm{EzBo|Cx# zCJW4+#Veu5dYB23u!_P2QJ;Mdb6yCup@wyVc5gUxc;8%T(yH;U9+b4Iqtkc89^?vb zr;Ko=Z(|&o1k0HvQoQ|XXf^b-{$oS5c=BpbH(paB)9lC!5UML)&2~gy9dcCqT3`_x z4QE^nKfDaBf_49IV>n^rr!Ed4D8$OpXrb_Z?Nk3IFzY%83bAK|Ypt$ChmiQ?34yR%Qm>U;w5IbBdv z`%GtfxVUNT3(T#en&N%;DB(Cl|M}zYo&f`_-Ot*z1)TZ^NjZwrj@Vn{H@ViK;8$)w{lxiv z2V|cps|_lOI0v8Ap>V(8nENbXnbn-Gnwt;TEzO1S1qS}c$_He&07E z)=K6%mQHDraWW*psod4f{@Db2K4ny+bWHb_Ub;pQKi&Zk;A8dq6^FbqE^9;^UkbpVE=OIp zZlo3w?I6uQd)Hl`DDJdi-`xD!l;r@BZ?i$`>;$`0denY*$Hfr6LSvEI-Q4XfK-eAt zLgb9p)mcG;QUxYCDatg3lk(`?Z!C{?fJ7tH1OpM|u~v5U41KuXqh$54bfp%)DC$MdmR&tNcq|2o2KDfOxeCECWG5Ko-+c znf`OHl4mDDSdkC((UZL%;6wCzr?XcZ}rDM`}$K;`->Nol~9jJx@j@M&7O zH3T#kh6huGu_6=daBV$EO*jSD)j#{KD$Te=VeGLSTQ|GnO43;?olcU;lM%wVG zUxQVh#%eUHD6pG%)*5+{vYS1(a$sh!cVE9mIdStnY_B1)oS8t}DaFO8M1`|%u`nL1 zCbYU>g^rDrqum19J^hK!^)jIOFdzbwv>amb1{MFY2Wu}!U!SrDQS+A)jj#sR;fmZJ zDH7G!-`a-4VH>M)wG>%ucwrM*VcSn)eI$`mQK87mwC=hv=L^QXRe_QladTHb2ldVb zTUAZ;dVfE_AEIv@n@~1O&vH{Cvf`$$xhw$GUsr5Bx!Ly@6oRr&!?TLeSmHW8P}AdQ z{xH&asS(-!W>GqOEZ6-SGtM6gT!!(mI32E}GtNzWy+$4q)P$mO2aAW!Lz_R+>?Bdf zUV?}`y}0_Mw$dn=s3nuH>>9WqxWz>_%+L_MoEDq7;QSbWdQ!5=ILvKp>{yjF&WZjl z9r(2}ky8;0Pf1C4FrH3J->LDBT2iykDBiudC}24=Tgd6#F<-z>SfF-1*4HeDKX(_6O}LSSF$Y7vHs@JS`X8Vi^-%HbapGb+%7+U${G^5BY zwStsUajsL|4^v{`2|JUi+$bWhw#k7NVisk4DpLAR?v!CeE}!BYupS(#fh~_gAEqB>!h7n)O5z#a+c9-z5 zprNtrwXYC~APN70lWx}i^OMd(>WM?;{iF9bd$l7ecylqBV0HcR*Y{fv4ZijOobCca z7zuu(2Om=UuT4oA+D}pBD)vVte>jqN6zT*SISrPZVw;^|((WjUo-e9?J3z0}aqo_e zXHG3XOitw#3m-@TnlA*q?*gd=(%i3xb~Sm5QtX|}BAzgTTrb&Z8?8M}TxvRptahlz zf=(LCZ;XTD`1UszgQ1Mx=8t=yaa=gJmb)>TY)!;}upP(~NE1Cvtw+nY&hgp?_Gx}p z2(8z&n{twPx99UUJk7cT4}&KH2bXhTp$O1i{PKe&7+dkxz5Qs28p~`eqi}P*l5>2X zPi>U%&(VQ&^Xo}!>dwv|iW!*g3C>9i0JmKW&-QvJvTml1dy$DEYhYs`_1K#6(rY*J zo5;aTP-7UW&+^c*R6{On)x$~hia^ad7xsb04NWpIF*ueJY3fj$oT^jv91lf zC%gho*f2=PAz3vvwfy*|gc)a?3csaSqdQKSRj19QLQ_M~iCF*TZvS$;+nRK@4)IC) zf^KI7kEZOnm3pRq9y?=$Y)N!T6J|nvH;_}higM48(XHFuU+<|2M!?U$jr~NF3$34B znP&_2I<$DqM{?zthp$q}Q|iH=;8ac0+Sf8Av|yyHk}!ORgG!BH_&QbyLQhJpUp$Ce zOL*HKw7hhzII~Ual6l&!IoP*VW?k;{6&vVJWg2o!MhH@NlB@ZeV10>?4dT;>?^?3h z9rhHN0Q}O@ro4LI73`V(4Mi`66(#BeU$!By>}hw0Uos(_9%|x6dz1_V^TzEAejT2- zmT%&=%=vNp$$E`sIxbHN+(8uWB5>(`hOJ{WMzqoR(?Q=?##w@FcQ3okRK5eQ!M;Ql zrVNX43B&EKF_jakCB^r>Pm5>!-SG%dmj-_NYF{}5QGb3z$*vwd$jgXQ|>)Vt_=pn2&yDVR^P<8>!eVr)S%qlK}2=a^UAfcMYeO|2Jb47l~rRsGh1hB zOT+SA48^io40vi{63RHuAGBaB)k=Ozla8!T2va*>z#?7CkUY2X71c^|^Km!sY_GGZ z0d>F3WV7pblp06tcJnD)f%!OSI)x`{eQT*!!0lCdlQ=vnu)D=y(%0K8d=H@Qf7)P3^5tQfDgxtl=v2tnC-^K!n1`bXY;Efg{ z3zEMMS!8WrVNR%*)b{B_Ulfz>-?LNyzs4@VCLU8Znq}K>Ig&t?>L54k z8Eo?c`hI&oR{eoYBmFI${=+K>4{XQL#%wU}{dRa2;8k}d?F{`-p!SbTz#|DITfj|4 zpOOlXj1Fnzp*Qyb;Q}aw_NZp(lYF%-5XoM}Re;|Cxz(ElYT?)r|Kc(K2)XH6GA-yj~;-6O6q5Am!WM=MWPR{vs}2o{_5gY^%WtL*fEjae>bi-DSC7@!B+&# zvk~9C5_hg}JzeQ*j=Z40o#5lN!7BG)5#R z_kf&iSx!Bhjl>0=G6f0P!TK?X0rk+TVtK+oWb@?#TXU1bxqf5b3#)-WG4gI4W*m~*B5N#eD8Kn?cHkqztZUDAQhmG8{RFDKNz;9E|2Nw#VMtQpX++$eHWS{BS%QXk(NkAbMH`6 z5%bU8d%PxmWHo;;B0P}iaVDsx(&S-JloA|5Jqhb)!*|X_C49o%VjZGWkl@1E$M-9C=~3i5)Eiyo`k)1;w91)&eD%}3QuFs+WuAwcz4AL z;O|pa_27^e!Zj*l^IdhL0E`rE4ogk@a_5y8pnehQ1jKh`?$X(Fk^}OQ3u@}JJW{j(lni3 z@q=>`ca6__G@8)bg?9c_ehIGN^9$+HK^?$)@|_@}m0E1>Y(a23vGm?N3du8OWeNMy z^S;$X^vbVk-*;Fqf{F)JEH8OmVt~0iA7^@kPEuL5HIYc--7rIOD4~juN9R3CfspsC zi!#XbOwI&1e(pMmJIbVP0zRR2L%Rc;m}?FV^P6{u7#nvbV%sRf?HwCNN)+{kSYrCz zSg|{vtQJksYs0VZTxaq>M68}r&H_|t7akg&i(WPrZf;*=Vtr8+M^Fg=+kAz$d)Ga{ z4>D|NR4{l#TiW0aTYsOH+(s~%epi=%y&9vM=G88Wp3yUEZ^R-3@yV8}gTRtj&S9>< z!8=mj_B38zgd~AUnLt#^K{ol%$V3-=w73Pw-lsR|^D!OdO1+$HXU+0G4*^Kw8(Hqeef}m#8j%bv;08tah zjJ4rNU%Q-^Z1S^QM7FP*)3t)+y1LbMk(4`?K2gu6A1?LqJc^X2(m&Z8eNC&+idvD0 zE;jD2%zf8NbYpgV=yc0~w;B!$(dv-;l92*AnAAEp93SZXvD5(3yn%Wv_2kE)gL9N0 z%7_&W@6??S$B$V7sK)j3i&&({|I6;z9fzcKhTcA?=X!Pu*?CXKRNCAGQ3;!SSy^_y zl2mC~o8J9$&}p_6T_Z7W;kD}$wBe*w5hhZl+BT@OqP@-({{9(Mx&YIHkSN_@b&MgWB|@BG8y#rb?*THSF^_EjP%T3vr|Q%M;h)e+&p*De2^Du8?t&BzVr36w7*UtR2Sp`^L3EseRhpi zoBSOc@zoxtTWolE>!Cb7pez)ZaryCbDv}i?8vlD0EwA|5HTaO17H4NoSG#R;`);9y z?=#HCXU}M4TgG+N?>*fKSx)wXB`HoF02*FCG#I z?*KzvrCY2!zbAqPkrAsVW1>Yn3(6#9@O?Pmv{QRkojllXO)RGDkd9rj9vxy3m+)p(J}!#O0gsDQi$d($_L7-+c$1q7-(MEVNiWoa$CuKkJnf(X1L zW%vf9k2b`s`i)!Dn#6;>MeP3v2Po*lO0!*KF1FTW#D2&&0yCvjeQzOkHQ|& zG~{bMSTdo8cmR=cPC|id#CYSIKXQYvEMCxicFeE8{6cU`RhVl8VSq zFY|o)LhwGJx*|Zko8V3oZfbi=_wN)Y2u}YkUJ#D$P(7nms6tCuJR*4F4e0}T;9~{9 zu4G8Hw-uX|1z$8ZB}NTJ5ZRfo0>H&lK6U$a?9bj1kv!5)U-Ike6rH*DdyEg79MR9@ zeF)%Y?CAS5wLmK`;q_v?5%VnWhpS{3swlj^xc7ZoqkqeJ{%T2AbN4_mr1#yhhNj}H z$BJ1dgwge@E3N_gLRL>UTb@!W2_;XxZQ=6qhJV}q4=~=0H9Kry)9hi^To5LO`vO_u zPp``5!L#!5o$2y!Om)OtnqGx|-bIFrPOGm3dd#z|NzOYtSi(tCYTmy0aKvF#HTk+6 zFOPo*ec+B_YY?D1Y)ts3u=Q3SV`3U{rZuGJ>RGQL_&9AlrZL{mA>_US)eVT8)!IL zpkZJU{-Bzcyc+XG7g1mc(L==yi&seN`XvFa z$?Y%Sf9-9;Oiz$Ly+Uk1+LPRjL&)8Z3wxpb#celgn^PjXP?Hk`%y$%EOk8v!^FG~k zzmO%RV?3j}F!FmaYhD{w?g3?D7~Kbh40Tb}=mhD7t|<+M3{ispTt9I(cb*5<`Qpc$ z2F1EdOBy}kd24&I(`xzI+4`@fE{tKi1Wq8xFS|^9pbnqUB#B?ZNYCGK*P>JGdOwxd z^Ac|IF=Hjha4LEmgpf6Gg4T52m^cjZ7O+9Zd{*KG#L4zwVT80lUS#2m|3}gXk}ojW zs29eC_EDTKe?R<%CV!gKY?3&5L)4UT%a0!4Y{z|a5CvQ~n-_IxkRgQgZ3`G{ z3turq&pjeW&&0MCd}4Q+F}!0S73-jGzPs-;enU)ZAx|lEo`mf2J znZJGW(onOEoY1z-(4P)v>QjrqrJ2B-Z2&zQ!FrvC8*A2a=d_{4yKkO%y#zVCZOhM1 z7SGNe^F*f7H{oWk$kC_pWshZMrNZyJ?C{w6Qr(@K&Y|alD!fkE4Yiw)$^A66Z#-68 z!X^l=%1+lWi9Rv22Jqd#p&f)Q>NHFkBGDgHmYzA`GVrCFPV0KtPhBoHLH z1(y&Y1W#~x7~I|6T?dC?!DVon!6mrM;O=gNew-`sIq$vS?O%JX*}K*FzdC(-@WX6I)3u9@?8k zSs7njCF`&CRWViemFj4!BjC~$>M2^%j9k$Ua|!nP)?$tl-o{M@@DXv?U>EW!!V`Mc zk>BLmUHdaLBKe8#jir`r*CVNeuxkMKGAiZ#e2kf)pk(Tyw8u$S8RMxzm5)VjxW7dV zTcl-am0204&hq2!nGqRcFP2PYMk6%l-L@aALxI!40prl)zwLIm%jk^)Sol>G9Ul)k6;rHow$}6h98(LH4 zyq4JxLo0Vlqd;=scyxw400|KBZJ{nGgBi*k0Kt*_QI8$(-RnF8@)ABNyfLPK}9JnlZLo_7|(~xXW0KZI6@(>q*E{=Z~tVwLs$%zV&oj3-34pOx>VZ^ zE{d{uH2KHfri|Q`=esV!OlHtMQLym#5bZ|D#?f!zW}%blr!<$N7*-!s+U84CN*!os zbzU>B;Gs%!@y(ObSFmM7)cEb2=xZ*_h{qzjIS+IL%P$V+T=F@(I~9={1x0%~1D@+{ zvytfgcdbwe|CM!2aZSMp=W zbHO03NcUKBE)LR~vUtZtHM}S-M$uE49-_0kBovGj+Hf@7O)_iz=`$BGYnH&*?E+!D zch7A2DKCCwV!E{R2=&NLe?RhzGbMGbpBzatJdxl;SYt{c8P@U^6WQQ9w>)WtO*<}< z{IaD{*Aw+hwY>{{7zxC1aqd&u_o)>|>#;s04zCWbi3#jmCo5bcdI_ys6zAikvnn}* zy|QEu8Y}K*Wr=xEmPYvuEJ^S^4f={S5oR8 zeDZ38(QP4;i*;eKAHj5TDZSC7~owH*sP9)QHe2S{CRkTF1upt3B{&EsarlTO7}wfIWW&1fyj77ys=Td zwL1z@Bh8dE*_Zn&suHJO%yhIAfg&*VxrNnU=d6s@qXESDTR4h-2_`C12lcqLD6OL9 zBkR&0j21T4$}tTN5fWjT^W10*amsPGD+n^a`lT*mJ@UlrLAcE9lBBP;FbR$kz5><1 z^XawigSn3ZHNcD6ZqFp&^3Jbak*olHH<{H8%Jx~|uT!5%WS;E~crRiB?~;?!#%CeC z^9+FMySXR^$&UJ$BZV7Z_oW!-hGxU9nv%J>6RB)}^O86RkTFw69#fO?XI1%Lc=_2| zHa3s`p^WRG00AJm5xu&Px z*YjL#cV)Qd`>t!Kg>ZU`U&$$xa9kD@ToxA>-rWn=7)CJKo&RR68PVOG*kVvitlu<< zUR(2cSwFX`Y=wawdtI}pb5(e4y-ZX`ZY)9=XjJsh=%N~?)nz~M_?7Wfi|oT~cx3a% zgK8W%^dk|z_#t~)}BdXYf}(tj>Qn;arV z{VZ*--+TWJd?IzX9cuEhb@C%YvZH z^R82~CwoG`LIcdR8#+}3o^EJM5pG-LczbV#G~d9YsL%=W0OBW2zL;C(wpxj1y zw-d4E&ZXkyZ`jTB7w|>*aSim@+;0o)-7-|+McN{ z%V(Wn&i0prQ7&|A!ChrBdX5$ZI`+=3SE?Yq1Oo~X-Pl5qOYexumo|qo(now9Yv$JT zYg}v=A-((8f??zLtgUk>+yH)aZZCw=#PV+AhiZ~H3LxtW zA!q zqx858Ca75x`!g*t6CeW|rMCZGbXsx(#cAu z4#0{2__{JFU!2?!ORm_iW$}(WN$CY|SmKhS$xqA3G_7Pao$k$a_*2tjObtF+ zuJ-0c_UO`hXvK3Uo+P}E$y`Qjr0G$Kh%N0)dG#-;$9hisgwFHTAHU)!d5PO1PY90N zH)nziuTgS-ln%acZmSA$L*>qcSX@WO_x9AA8@4WSlHT$Hr_c6C7f6S=aRAuftRxA% znNbPC-8M?BKL=99LBfGWdNnp*uFl0*{4`a}zpV%jvsEDL50AHGNhWd@Cf{tUDP*Pf z!QxDkwjD3(ceYjr7@yoQ3kfvGewQ%SY1DQ+t()L8K|99{ zJ=BB_P{HbrL5?_t3CV0nJ7{U%pk%?n+yI!X`D1;ojgL@&9&@*$dI3C611XK&4vJ8v zQ{YM8=*Ml+*tP^1Xht-p>E+feZW^{&sSMFZ)RyUWIRsA{;mxgT4I?55(%_P6UT$Zm zM1$P)SCWxjk`7*%j0 zgsW>qbv7Y0lf+__4cFWH=CjRonkVjs9+!G2%>HkP_1P3Krrh9;(qSQi5i3nTgwir-&`zv zB?BBOigPQzebf?6g^>LGWAT%uK9N+@SQ78%80~O5`Hs1s8=%3}$M8cc;OGJ+_|UP& zM$Rry^6tbWO#zpi!=5Ig1Z^iFMzv}O6#-<8M1onM*w&1U@&a(Y^xx3o((m$K`z<2< zC?RB6@JY4jRk@W{F!Rdj7&IMaqPo&3tiV_T@s`hav*^n&M-%g<$-aQT0s{mljijsq zKG8Fm4H`KymW?-XDD~6nlcn-5t^3eCFdJP;eNN(QrTW$dXm(-H0nyC`c?}bEk*GeO zt*sJ!BpbmX1yM&CdlfH6We^lNyJ?pS+wzArm0fMZwKZLr+p*PHD_LB>1tHxk63 zi_{OeA|I|TWU?dm42zjPVLec_Z}QZ7^B6S@oV`Uc!lHc$v9JN9b&8GokVbLif*IS# zJEsb*0cyx;8m{RJqUaNwwIF5C@y6YAF+Gv!=jz%hzcq0`0;9I8;SeQaX&21# z)}?z7FJnokLnlepAX$ck<{v=jEkC;*r2%~aWg$L9<@eLI<5z*JU6w6)n#i+`%V$Us zY1@6DYtq7MP*`3r{I0Oqs2~XG_rZ0MtP?!jMu0kb3OebBi`K7RH*B01yY1(jRD=nt zY{e_|G~^~$E&Mm zr^F43qgFNtIr%5K!>QeYXf!mNDndHCj6DZ6S){TYch}ZKu+LPtuuv?o@WC$C;fzt$D z!jP-VQ`@ERi8b_^yzcua%WqcKr(Q3YKS!GPnJ}KS{lH$%id>qfT|+W|Iure&rlGp8 zzZHs6lCl~+{*%sun#xWEyqqUg$tn|T@dF{R#C_?O_j7dmTxEaRf|il)JDyKvZvf{I zPqxQlL$Ws6oEHX5lP~&~rf)Q8k>q+$M@Yes7KY?<0IV6ugV=p{&@G9w~1W-g4W~JPIcc2}dcsaE~mBH%PB`a_+pI z!05!cc`x*t1XRaSXMwAVjZ54wWb?H+N8BqfR>Uf1618Dw?}w^Qu>35S5=*p|*{pBj zF)=$PdH1b>A#u~3LgjPJ;hYPwyUl`VzB75>X4JTE;jGm}YD26dd7!cVB;(ofl@ofqeLm>gQzrB;8z8isU@!$Nb?*yl0SMEN4?bn|lG zaq7H*9(^R|ky-C0a^+vx_^gk^5B4CKO_3s;yr*A#&}M+0H+D?azp9-1!IS3trfUgL zIJtgKb$Xf9P!@s7JaYGnpkDysD|Ow&Q4P1h;Ce$5D4$uAL*ERx78GP9c*FxrX=Rar z(Qe@8m@PN9ghHcRT+zoT1bw^qBFmUSxBdp@#+X85-m&-^VR~N71y3Xv$y?4@^iWyy3 zf{@D9&ijC4;&QpFn~5Pd8hY>DFD)9P584?$1x=RBXKAY(yEPff?6T=xtkUbP(ZC12 zi`B#BXVBhDU#o^T{C_OaPzYO3^0$+yLN9(=a4HC=;E`46Mrn@24RtRi(n#Xu{bTv@ z<&n>u$Cv|N{iTdtzb?CQ@{aacTxUzcKP_GlL=)PhH>KAG&X{`Zq1&XXcOMe4uis_Q zlbtYYn>x}3x&|;^c{Pvqvqx2Rxeu9E*&L1TuyAj{iaSV4FUdt84-7*|MLf4fz01|~ zHccg)m}CMJ^wbM@z3!bwo+0q28uu5;AivvZvJWj^a(^OMGqI$z7H8U4l=M!R%7e0& zigo`){;KY&PmIfJxzXDld$~&Bq=M(als9-x?!uttcs@Mx8fbl2Doh^91B`)&*n_>mvHDhAi{Adlvp%mTH4fJ%V@&M(jUrt7 zkYWZ?C*Q@Ht~0EN+!l^}Ty9493i0%vTxL;oYWHnQ#HR(&PI*jL-(2r7(Xy7-{{H+8 zV|XtdngRW)YSz`gx~7Kc(_eKpujJFR70@?TW2$_X&gj|?^XMLIiRYLe+zk`<+KFUD z(JNN?edP=%s0gUO^2Y(UL34Kf!-O-3!*-dBz|)!Df|#*w%;_37nCl7kZj9mD{T+?) zKtz{Z)dMn5&+o=CsPY&WnuSqCRtU`@Fm0iVk*=m;>>K zR^lIbsKJHc`}EQUUr3G;l@Y(5FGdQz+d~Df#Gxb150MDJdhp<|J9=-)Bx^(t9Y`DC zP|q-W6Dc-~{Oc+34DF%bZM+6E&d>9+6MU*kmE&P&chANdy${v=Q{eF-Cc@sstsSOG z3NT$zkCNY;QD!{_;mk{k2Kb4MYdp7Xk0(G#*s=(dCc5G_-Y!9rkaZ+` z9)xy_j$tyvuKDdcYp~>1EDHy=FWI!IHPyE5YY;1u)6a0C24=sYd zg~=RNJ}}CA{m_pJ!@(q)1dF(&{78(YHQz^Id|xYnq}d`fC$|2E)%r31m`XLy46(qU z$vi1FW3i=Zw5rqcJFS5OmlaK!UJh|h0{7`-qOrJP#x2eUO)1g7aImp=vJnMTa&A_H zQJzch$?A39AGBY(*#(CgIl*U@jj@?8Fi6(yVH{QeOGiUNh=uZ-b~WR0(qPI!n@a!9Y!+l* ztvUm*(9k1G-~A;LwX&fF>VSt%M3dd3zJ$#GiCya|PPj~AUfieR$o~~*zXiDFN?lKQQI0|}on}2FjMr_HkK(_7 z`p4^`K4cMP7dwsdEO3aS0SwMHrh-L4eSPYl4gjy^*^NlFO_#s{^BbgrMzkSDwR~dtwydoi<0J{KXMLjL3+PW)H&yqPx1>Ssz=Zse0I z12LYScFIl2RiBVhC=pf>8=mi8r@o_43-V9gLt@z3&)LacE-Al%LP-DlS3;jLP-zJW z%Kjwv7=fjV74S#-vhPcNwlI_Nx%lXLBoyj!n(beS8Kk1|DL6}1={d3lGmqwqx7+gZ zGQj=TZeepCV7{i9v=gBAJ5_(>s-%+RPfImRje|9ZQeM(Xm6VL{*D?pOMjK9BYU&QF z_oMKGe2jhJd`?7vesgUrq20OyZ@8FQ^u%fxhb3km`JSB%S-S$+P(uL9{p}qQk%%L9 z;|=Nu0p_F)GrsGWB$YDx%yc{JBGaWkHaGNY$(MM00rr*jJ3(U8@P++5kbnt^UjXep zZ14*wRmmVlepQx5_;rfdJG>N`Ll%)%*hCa5UIx_X+jpbCxD@f%R#Zkn?&ukm=#wEz z&ou66r$_Z#8v;YE~L)mZ-;r-qoLLVHpPh zYpK3?FMu_LQks&Ttgb|lq=^#9&K*ttD0IA|=-w-WGmZ1=7Xb21b?-TofUc24t3OPy zkl0*n3lgD9jVzzdRTBu<#Zf%y$2sDhIJEpyLv!+ z^1qvM(}aSfI9;sGk2?ANE{HTvIE67Td+(3nW&m#Nj(-ShOVMy&b1}EBc}F)`^PWMXT!ZG4XgLKlG@2X_iOM* zf*mjo1;}M8gDcsrEu4SWvv7;GIdFoPLO`sPN$0~08!E-Oz6?_Z1tDb3yH+1{A4CBP z|1dxQrBc^zMg?#FUKnbME7;TR4_%Bh!fDDp8BR~|sqqc2*&^MDn!pLaH-G$P_eRNzve?(}!NRGtxqpI}se2XE@4V3_xpqz+rTlySZJ`y0Y#RhH~{}Gl#u4J(bN$ z&U?H*0^Gxu`3c!LmYNB2RFMCP#GRIXKgskKnMCN?${QCg0N`P&3q&<~o?FatljpVH78Rs2Ck5^`-!Q=-Y z#M+B){f%$6OoHPX3;l%~!4mw=|DR20^Sy3+UA#gUf*}#(aJ2Bb%Ef1fS8!}eoLIhv zr>kaPnr9B!KSq?p4!pvOyg}FNDZa;(`dBS^Olre*-o1rjIzJX1+0T?O@VnxR({qD# z^cjy#{?J9ro+(1zL0>nGh4wB!#t^ctGRD!Q-J@y+Ha9St~1C26)zm<>JOgtiYy~v6hkT= zo7Xa`@kAR=`uCFW9R&ariP@ENL6M7qz_3&3SL0fAkT*ViRbnGw{hp zFl8#b(KO=zr|a2!NwJa=);$rBYtKiot`-merqUJWxZQ zjnU-N|1HQ1TT8g&3w(GuXO@Ota2>uXi}1IRxBB1n92hh2$j06KrRMsb+o-gW(AS5X zH4;&rcCi{WVylKI<@^v=5$rz3E}8jUl#;+3O> z{M{ibNfHrf-_eTyFP5c?rTIvxv7$fnEO{5v*a$*h9exZZ0E_*GLqo-*tmYjZYitd5 zcr8jM5$hNxw+H%uiYFngYgFcN{9WCjH#8iH<$?7zR2svN_uQG#z6BuZVl!+NNGx(G z`RVTtQ~z=w&PK0e7{^@Km54L$gQ|sP`_A&;bc)i9b!~B?3LTAJ{{CewER%{QL(CId z2Rnk98^4y+6x|i$+2~_xt&1G!QF1;J3dsWn*X~;7FM+y|Y@JCQgznRhxF}dXoZf(h zJ;Q2wyNDW?@iy|hT|fR`X8OV4us#Z&ts*UQK$O$!kq+o*}*!K8BfI7?Gxef5-hfq=67BlKm=8E{grTg z_IdW1qK^VnIS)|T$#E{sV5%-i9XuK&u%aU<4U6ypNPxxg`9rl9lOZJm@+X`v`b06K zCZLw3rGA$Q^TKV7?7X@`QLnn@!L>2h+D zJWdOE^FdT49;WHDh-T9~d>gwQ{oP%A6viF@8Tbh`uZnSBWW}u*VpIB&mIK@ovz7!y zF1>jujBI+2IurZXkkUHf*st+)Sl^`}3=8dl4(2aWftYyUBNl;$k$(FJKP>U2o|)fM zu04{(Uk!Lzs@MmIK9DuKU+7i-AiuPGb||rXnBNEs{d@GI7rjA;-P_i@!>vmN66veq z=m3TuJ|6lw$xw0=$3K|6Jh2k#A4-@+6o}h}(@H z^$rO)=H9}fl=RE({>NnTK&9WB$AQ_@3!Ysre zyX^%U(*E?7#;z$yruVmRuV=lJPp#|N^a~|N=@1dN&&Ff*G@MAJ&oix>Ry5+(6jUt5 z0UZO8y0&nS!qc!-B_W{6`v=P!{AiQe#_;r2>F`p& zyb`>OCheh0uIa!-|u}+phNLQNXSti_IQ*s+DQ%!odU)t+c}}f@aul z#tDp!K34O&6K7|yg!dl~Kb9ZZyDxE?FvZYr*kV%*yJv3o9aGZ%wwC;iid5e?KNKNJ zd7*}VKv1sNps1~J>#@GJl1x+ai7^Xn_%6|L!e9lka44H$fh*$>vtEO1i9_A6y^cN&a)!#BobEm`TG#Y*3E><(>vl zQin6Mr+Uwb{K?;qBRSTDGhdcUKI3_egr4%hhdqc+zRr2WwMkV`W~;*fdqLqPAzp{5 zOUkn%6%N)bTl)KVSXfQPxS1U$`~e#eH#;NtBLnTM^a|taU$c%FEO$sojpFS(J09UG zluxUGXO>mIx3SVKwLS-7_Yn5nem^E??_T@`8H{)C0$t$PwrjHy)g?lBt#lHZ#miow zFUuwIqvV5a+9Ik41uq@lYl1EWErxXyM2~IBk)%s)+jSmwKs%bPG*NXDKUxsRx!jgr zMOFS<(QDF1y@%!pw2{YH(HR%&fE9y|JodSIGKg*YX7)!8ju@$#mc}-7@x{!MSkJ@F zAUEd>DcV8n)h&G$|KNXFp?)V=oT0@x_C1Y|jO7mhYW}A(UokP}+&AbiLMyGEn7PDq z711uyb{j%ki|ygydZ%lW{a$OL#c6lm?gBSz@Zm@0W{Fn{j2!72v*w1LdY+k@-7Ssj z8S&B+P?=Tlgb!gme|U8r$p2+X#jL;KdAegSbbKATQ9diK%536L)xU91#E4mxc*)JT z;)i+8+hZi!$Yz6|*H#zLEC=qe{wU|q;B=YYOoo)*t+A0i-AB44=Wp2g0Mcg5rGlRC z<3RO~md}=WpM<&cCDh*WqV`?DeAc)ia*o!sq0lFUAmRLrt&$!~*h>>y(Jr?dcdq%Z z<6d_U>J0Dih6fWjVWSgSQxbnGa*9ofM>_L35g{}ftq0`%T<{} z(F7~vAz^ddFTG$rXfC64IY^=>{+i;YIfzgG#fICGlbwJAUh4^`1K&N*N>+cjiV{|( zIUb8v`oMZr$bg8P%JS!UnZLF7)DfQZaHMRrHI z?o4*v1St<46;zIRWJ_g zi`Oy)7A9bEY56{^@w2^=n_al5OnfdmGv#nwZ0ZblmOX?tQ6ybEP0Nb}wxiTL5upz% z|6xpYi|Ar%8(rl*f=;zYkxtLgi@Z*Lq^U_Mpq)?5w^|D`%ObIs^})NQq6=uY4GrCF zj|q<{OoaIro3Y%vAh!=P_{u$so>2ucNv0Z6mm7nI6Gq?>$G{Z`dG=(@;!IS0zZi73 z*$oEe0TP{dixi~avsd|Cr_r}livrI0AgcY9;YY+MqieSVIqPJ~whbDre@cXtQbxAt z9JOBSZqCcSa#g-4@`B_Ftj7(^%xN1@L;#kAki6y53p7z}1iS*=Oxh=J7hI~{0W=5@ zCxL%i%Q&C#lbE>pjNrB^frh0?U-}~Fn#WZlI=(JderNmr=x?qo1{>|(Bn_}z`*a+o zQ_H}sB2I;vq!R0&N^lMClXO*|W;EBPDqx~F$84^{J0enWMWFn^V0cx8CrX96rD%>cOY*( z_%?<4Dk_`|Enk87owoK>^A-Ou@jU z1haElw1XJl98w5#y@l=OvB^?2$88Xk@zK<~)<0#NvLb3->auio2I4$@h+tiSD!a951*Q+pFd1f)W* zqTA}maed1SHI>uhLuHUlC$t=!1}B5BHbl|7uIc_deDTPG+vxDa{e8qEt~Xhi|E}_; zk{ghm3UF?}A~#j;z86xCXWfP{vaNXQFu)rmY@2)zj66T(z~wv7&5z0+|5FnBVEwFG ztWc2c^VD>kxmA?y-D!OV8FPh}GxK0jKPgv-g)Szx9$VQ?*GGzaJavD6(o2F5y?sv% zaa!6FVnV*xC5!pjpAUWLtP{#(9^|HteyXTuSe#548`)ClzR?~t1|J+t&FA=8r1(j`$7%t}J?$=7Ikn`QfA!|;#Wb(HKYFpyvWHy5DnCmtl4 zW#TFNs1?K-5|V{V30V6WUjq3~ku5lsi6biafk)X)HXA@t%XIEJ#6ChIYzH0+5cF+ zjk^S@Q%`exZ*F#Q9(H81QhVACT+_g8|)(If=GCr@ymSnlfwIlziMW@Snc5l!v>ze1&xEWLF^-Nu;QM@ z7uNqapa1!^X(RU6_WJk14aDaBpD$XoQk(u$n!lbG_5Lz_|2~>J5_3NM`*f7{0iQ%f z|Hp)zh>HHfTB!*^*NXKxkoUMOvk*SQe@+!Q$TOQWn4YfAv8P`);^V|>4|1r0#=uB~ zsau1fQG3tJ(PE6tGUqAT*yN_o9}N4Wirz%647Y>C2^EEw|I?+HG*A}{5oRpzs6rg` zSf@8g;NbN%6|Rn;886S>`r1-#!Wys~{XwW8-)s4YhFX;LI_OOx7FmkqpZ(n1)MsF$ z_tG4t5BxV#b98Cy)2HUdd#0VGg?9&#^q)`MnU_deEPo4Xk7IxB4LrS>iECs?+GODd z87Z*rF9ENPbw`yVF!F>b?M5+Z8bCS=n@ec~h2Gu4N5vmc7CO>W;op|I+qDffNf|cB zs1K!C6@T<;3cx#w%#qgSZg0Qa<*YxJq=udMrBIxG)yCmTJw+z(yxdBcTuhK!Z;k;j zLO#!29K=?rXk4ZD#yho0E`!q12LWu2+DtNX)P^n7MT4PDW%F9tP>(oAv&~)>d-aU+ zH<0?&%_E1sV{Ouw;(>Q6hs^@@OG78B6g#@(as?WWgj^ALBoT(tW>4ZJ67ANw5oPJ& z!3wgnR2)eH$OQL7d%2`=VWl*+-8|fW(ujmD``YAj%7wxb9_uvV^qgU6FH4E5^_taX zcCY8*NY#8r)^V0NWjM-jR1q}i9(3>BmQmVZa+={}xs)We-YcIf^o!wYF><~7j*;e! zz|QN>Dh*A0p!^S24P`l_p@huTcK{YDId}^_;a(O-at611lWk0oM%QR>^jYHCeV?S+ zWKa@@&GGFPX+`??(h{G3&d0n@A{#F3=0d%{!&oL(yFC$wui?moD}7h#dT%avTy~h6 z+$YTr(eBhXQs;)VeUr6;&N(yzx)iea=7_*YH}gZItn=^2Ga$@&i&l)t$yt#UBnlbt z*Nu8BZ_w~tXOhFOTZLE6r9W9%#y#6@l{#%K-xIIfXP(4EZan{9#c}C!1$UPbgp!(E ztgSl%j-;nT^L1T>6f?vPkl$Z}K>!Ox`Rs2oX$W)YWGP%}`xo7z)dKp0?JWTmxrQr} zhqaE>2CjYULAJ9Rjzk=B;5v{R{&;His61sf7aF;+uZr>qkLfRI+`j`>U@xZ*aylgC zunjtwX4Y2@gX_h#36bt})Y+X#O%dYx8dYIPv{HvZB1c1f<(3&rTTbff^Ip5uZy4va3lsufV*wPIxJ8!$@ioM*9O=lLY+fEeSJs*;uRRNYXgOgGwKAMi_Iv^mXA zD6>CM?ioy|8{jj^;m(3!T*sekdMWd<@{Q_8>u)}M&ikVod3C$|+s6Ka=(gn@hF!E1 zdFGTh0uI7feVr@k<*n-XIct_66J2OUb8zG!KQ0lJTxjXN#ltKZ{ zhHiDLUNYBhGB1@uQ@ydGDNYCEM_A+nBbh%2Y3=@Oux|`y&`m+{y&%~Nw-TwKhp?2lM zOBWqHS$Iut)s$wn91GkUb|?7`?!5N znNPnPtti-HsIDbFfAz$aroU(hgZZ2}h5voNJYd?EPDUqZajA@C$sRsgM$<5^Eg5^T za+^u$;lXDty*(5@_&Obgm(Z-x=6L0*%23{EQf+W?ynbB_&rCz51xVNba*YJP$8Ca6 zd(ILy+Z4_cB}nGkWXm;(9c|0$hF~+S7O+%A#?&UWdKl@of+Q0~*w-nG#MA76fSPRC zZc-v%&Tu$jV1E12i^ASya`9%)Wug6jx-3Y7&EyWnnVj#-q67$LB&<}fjT#a0swr$J zhPc;4y58>e2W2+ntuMjAx6rZ z3Mc`CAu^kG6(Z2FAD$B{fRVyO-Le!uRkL1>+~EOjp#XV|B;tX2DX38H)aZQw2u{k= zhgQv{3NgkPZbO9<_AE2aFNa4|W$E_z@^QHB$|c$JETwwp^zSLnJ)f|wuQVsmN2rFf z`O@b^=Si2E0@<|5^K~zE4(0T?Nc2X#bh-;N7EM2!f94=>Feu^orH0OGFRpLW+i7^F zDMkn~X0!!1ic0Vha-@HvAkQ@)be?j_<3TR8_&82b63b}vpcoagGdeGWZo?2=`6AC2 zmc0XLo}eKqz!|mfxNco*L`+=Oc5_Q)ElHQc-ice>KNxy+EU=h+fS16ThFJg0V})Sg zk!uH&m?>s)C_%3{BBiGt`;=Yr*H>G0VSjQF%(0s&SNdQXni5Siw4vLptnMixM9##*dN4 zPIq;#COH-JY1R!-cNPYKILO{d67Q`d(`yRzXxJwCH7TTwJpRiy`BF&7=`LVasR9ceBP} z8pYW}$1G}6#CZZgns1hJ4B{${t$2m_X2v&fz-&_|*>7#8mO=30{Pv*>opcz|%7P;l zBP{z~Q~ea?Ctb0!K%Yqdq|WGN!_ZW4;+13~fWzzitw?CKz&g`y&yXYvv2}#Os;mc^ z*Qf!L$KlYV6%y||?o!j7%h(BVF31|l*>3{_54i-gvQz~CQ0aOyHoO?K5s$mG1=zuH+NVdx{fS=|o4dpKswnmx$VaC~7WtC4KI_|Cy zrGCZMZ%40D+Sm7GdZ3G^R!btcB9q73 zX7)vXb^y4gyP#Fo`N$^w9t)&nBe1h2^?Y#jqU$!eC2=#-v7Rx?8$zlNIguZDa`9qYWJ= zLN+tU+Y3w>hMER>?N@nl$!G!h_TG(opBq-ram0oFn{>F3!8hc8ydY6s5uF3|+SHj! zfA2c6icTT`h=|l}4wcz^ysDsrnE5#PHpDo^4D`{IeeJ4?5_@<)I# zlGgzAObFHR8ebJe$Q|DB(pk&9&B|Cz%a%RWD;4p1@0=&9gpfzoOeRH#tXOwIsg}Q88;cm?`EcJ{iMCOZfH2!MHJ{6x5u|Q#G^~_Q?iDuJdxh1SMCv5UU+h zs%}~a_mzMQ2F{;fa-?OVA`Om{walF|`)`G!*m#r1pA5M?;w{xEcBMizXl(1S=RPNf zw8{`B4HE{`Lwg9 zzHnqDiP<5&wGNFy76!R?wF?b?dsSHzk*~~zG-#?g_&Zu$BjkNRA_g8kPFl~-?|e!S z1^duM=gBGd+rtEExR)c2M`RxufWL8TS=H6Fmw_Og`7WFkl)1s+UR^5g)7o5CP;rLW$1Ow6tlmW z9?^U{aXut`nugZeEDm}<)e(5CUj~+X7yNeH%8LsrkI7*thQLO~^1Q9c^cy=bgR{oJgs1 zUfIenC?l0LR8DnSJo@=gWKJ%H)f*Ms9@!2xc7(X?0 zPxEB40d5F5IeOixqMFFZ*a~YKK+{~3p`OP%Z9TTC2EyT+Q~l#??sx>^Si>9le6QRO z1DIn!*7{=b^~on-nN}n8aq|!ze$U%gIs(nihVN?4!b&TjOFY99iJuQ}6$~ed2E5bc zDx8lLCmc&RL-|EBan|s%I-8FlngFDW&2|vz!k{hfBma4?iHf`n6P_Gs-zYj{rK#HB%i#H9MH#^|(WzW4&nDAtxcN2Z~ z&X9dGI`rtVeQS7oMOTN?8e9pSH_AYo#Hr(B3*YN(EX__PsXONtWLR3Pu({S#<%v3Z z`Sa3(qJMAgH+t$0LlN~UCz&jPk`wL-{_yv)jxJ!{%(_wh@>8^3agzM;19>Piq*HT< zi~IGKcqOL|*%MvVmaW)(Egtx18MWoLZ5F(ud}CFuk+h-|{hJqThoRLJzXHbHb~Tu0 z^r(60QBNuzWDdv+tz&F42u5zO;((3bOg;wDw6_GWly&<{OKhn9h!I)PLqBz{%3rIT z?H+;GQ}D=t^*o<%O+dB|?5n^!V-5VZHyeYaOQ5r>J8|!J5t39dr0J@`SBF`rBLSj& zR8Mb0`*UUQgcOUu!FiXav!C0V9C?P#qgO6T-)ME=VU0BHTQWG)z+-1`3nb+7Go=B~_+EqUI~av7$s4TOO75H?F!aa+-fYnpXF=K$*)m+V zVA5Gh9tNX*vrQx_Q)Vp=cJzXOc|=HZW5_65amHWzXh~Hqixcr^itNbBY_2dG2GL5B z(`E4wI!=l)EwVA)39|+m%?b(;&A^SO*wmn+mD%>B32iz?Gba*ocpxC1z#)pnVNmoj z0~@J$6ZGH;Ygp=jsa!*tNCNPdl@PPsbZgrwIDsKpE1CpqQ~Jg1L2I@_!n~=uWieKI z!f{PQ3rvbb2pI8?<>EfiKv?Qpcg-VGMMoF-SO%8e#pYKZ?2d1{Z7Aa4!V~*GCY^AZ zBzJ^Zu_TdT#&SOTu&9y4!u}NZzL@+2s_YbvTTUB*xiD1q+?zOP9sQ>!RL zZdJnC&YtL1i4R+#=Z8DXtN8+0ZFzT-KQ5E17Hu&Dt#aqR_8p1%6zoQ3PMvZi;ASH%l;nVr5Z^JtV zShjPRNRiZUwvIXJ`Z!Gt1@lzzuc*rL+4OE`CoV!;o)@)nZ*iK|H}ymOvgIkkUP#M# zC%u&l1u_&x1>w zr;51~NI;{R3?AxvX2w>lf9J)WsZG9Bn%0i0JAzkTt_lm(_}o1Q}El|;s0mn96Y=A(5y zcQRJyzFqeZG=Rw0eu8W#m#BSwc;_NEd3#&?u@i_N8|1}z_sH7n;AY+BcgoI}=DjYl z^Oj9*rIS#^j(eZOm2=qZqrARgjG-!H`9QhJQ6}crr7epG9izS-E&l1YO#1}cq%NE)g{^8nGtpXmP$ zk6DU5Xhn>WTE?G*Yav^fqFr~m3={)hSkJ2;@6h)N_%|7v?^K5NNhEOg zdXMK*gW&On0`*NV`-U$u&WohuhNCer_FNw5X>>a@-J5$jjGYT_##a^-ol9fv9*Uuf&o}PGn9UXwD?BwSG*1JgP zx*u;oGimRP2Rn_0X+ndqh)Iv<*;IbPu_(Oe%kXe{W3w5#!ZIZ)TI$54rY}-1O%ha6 zJ8)FS*R@U#jASv|{aJRa0M|%55tfYW-(xp!1oD@MC;22JG_)+C`$IWg%@>}#gVFXM zUe}}Yuv_nbOo%f}cz^*gv|`h4&-(?+UlG_5<|X??scp_23M*W?DF*CCHq(aj;&~!cF@6tK?CNEZ^75($!%?0YdO2uBwsVIxZxgyV%umj#>Gfza# z^~>%fYv-i;85=`e!VyLjs)N-UkpMkv5AHe<59Hix&ZU6IBlrj2HHD>LFINhXzRpsofmOD#)C6oO&zoRuEKosJ9SJJr<_u~UhAC% z%%j4&UOzjKi%aR3$C_Bh#F;8Y^ti|72Rzx&kE<(EzC8Lb7obFEqYq+pUh%=T|L3Ok z6$+0E3G0`GSAzHEt{tZO4~rNybex&pn%N=(;r!RiG=wBA+TbnFbRh+aU2iY zE{Mmz}#|!|adH%mC-#<}Q9E?1kQJ&01{dm~T>(!npUX7H4^IJ7siS3M5JS z(j5G8ijF^&qxWK-2i2#IPYu^uXBH)S=FdMzY$Z!_7!Xro+)D) z4)0|J4pPIPvXU#}I^c#408uqOvN;dw71#tZEdF49Q`>0`ACn7%!*dernS@b$ZkRW|Wt z>u!m(TabKBh$X3?j1f(~jFpN@y+37-?`F-IL&~3YmUG~eOX7FH8fM1+e$&SuccOrm zcI)%7b1ca`e_3Wq3z%qBFcz_2J8TQyFnyLVL#3p}f`YX8J#BUpkfZbJ?Fei?T z_D(Ds5n2D+30P>sB|}$rsgHGNxh6@e>pV*MM@5onysX`>eAP_<=u4eh&~D7fK^-F3 z?d}YWYmfr%p29OCo7WcDES{a0ROg_Xv99{5rn#~C?h`SmzHbgw&P)K7{2;%MJKQJ4 zSMl78(HPYhXP(DqREWyP>%4Is#wTX>zd|r{OX_KUp1;xGuSqBy>5kTikq7J`F1abU zM{(I%vF^F=df-ttg_zR%!;I6A+nRNxPH!v2UtizP z7)Hyjk6D(Wu;al=X~J4PvQ@tfJ8pt5)J+GiJN4`QAkk&dU{97k^PU${U)JuL436_) zHs%pj6agO{>(zK8XpMa}D1F{Dv)exmf0PGsJ-6uB z5!lp5u^X1=*QomN8+T$ykAHaJ=W*z8 zu|MZ64BTGd=2;~bCN!~p_X!Wv-yVv6e`;AcK(aCDrT1<~P+}}pS3B-L9A7EE^_g?k z(+Bzf*k2f!jnJ{P5_O@tn$3}`y>-X_)UFQ;tn@VO2PkQ}1_w&qINf4n!-ukY*vh^eqBEcR zpuYm;+Uljl&R)ySnUqY|exne_%ZqcWe!i&97ujeW3(J@d-rtbqUwYQc@pz=}$v!rv zFW>krJ{`edaz|)(+o_QJ{_)`gBnp2vttx8NcxPbQcA}0?5{6yrRsmxc!RbSJn!f4` zd>8h+0nn{s2#K1$ThRs>Z<{4FaON;)s@FVB12`@iDk7_GM8v~F-*Dv9ycjE`=fvxL zJ)N^&5Y*jvuOd4oBh;OipCRS_CT6YW)Fo~r&)KE;RZPbBF)cPYbN7u~{^I%3=wl+? z_5;%PNV{h0eeN4%zWqBTgcA;8$L{Cd^OrxLKNkoP`x|1}Kf_^q=RGiJBDeUqYW*f3rh zy8GvH-|=fEG!-9%HnTD1=0D_NT~k$^;ow#1WYq@zB|&X)^pXxlcjlwAZD200L=pFi_m@khRf`o>f6VgysNa&B0y`6&uJoYG-&+8y%Oj>(;aKQ2XBzuE zxKY;Kp{{8Vc_a8b@p`;c$?Ba;e5x00xSOWGZN1y=D|F_gPV7ZS-QP9q@U$%_S4s4u zEFBOR$7&+bXolAioUNy_Gaq^c52MoVpwxV1RTux869l6jBEhI{;o)%WH!Rt~$((=# zN54bFkm|eEMVG@F6xF>=x4dOA3596lm0H{s$2z6Hb`$Yo==tI%$fLAURO9`a83bkh zI}-evQ+v-fg>?UYo}-R*mSVrz=ZauOUowjcJkbMx>m*Cgit5R^s|LNO|3H!Wx|tdJ zIWuMVZ|rG;6pb+W*YuQXsx10yeIaNAm*AdtOb!Ujp}+mkXbZ=3u^yNOidtVvec98AI{%4 zo%z>ai9ZRq_H{3&XkHJ_71NmUI~40Cz^P4iwP*02(uNWKZ^HPefk0oYPy|+1)vS{P znzF3%v?tZGk#o+O4pBJOxlv5FcplbZKVCE=#{t~;QhZKws-t~7r=fssRed&CdqIiL zTE(VWLh^QGOy{HDm?MYa1~XaPx<&ma3Lk?z)eMtGaIatZylKHzSnn)z8>hQ z#kyhN>?rGgs&hnCF>>YZ5~g{+reB7RYy^=yFmCwVUw;v1JnRFA>R$`f6!x&PY{UB-@#(2Bo_%RA%Dm-Ou7e4ug# z?yeC^V__kL#Ip3V&&3_CKBiJmAnBrXch$vXReF+tVmrI{SPhAl7ryrnuKoU%;nNGH zl=n@#8X@rpl%mF7^+&9J>`mYA&D}MQOfcn&FMpduM|2PJ?9W4q`O62$s*~H>h$_ z&=_+x@vw42AD(Xca#ZEZ{L41}Xj!|oyVH-agYiLANg@E}B2wgorf_4pUM)0$?# zpND3IZ$fz7n9@NQ&aXKe3lnQ7QNY=ioZho^mV!YD)Tdi1#1LW$BeLNO84ku_r9DD} zH{?(js9$XgDLrXdGcnt@|AOSO&!M%!=PxCn5B6Y0aYFxG-#g^i-_H_Twv8?8d`UMN z{d81LpUztvyT2@PV4U3IP&26U`-fxxfauI+pH3@a^GH2WglfFz+{Nq{fvD+gXWwds zQU7-bahm>r*+Cw{N}k^vbO`N{tEljn6gK~9Q^1% zO#5hKUv~=pgO3gyFyj6LywmHFkz#J@^i~K#NT1`k=uZ=;eyQ^Juq=oVg8q2QPJHPJ zDk{Z$(o%M>Xe}Sm3Gn>L4lbCo17Dt%n-2^%0;}Eek3<34TGrE!4=EGJ3`~&^e%Gf$ z6eUj2TQ@oc1Y(4*NI$%o6jNMoBo5tC&(KLwG%mcZohsc?9v`bnVb)I3a${C*UHE-o za=<+vQlpDgpo`Owu0*;_fJ^w64{a2T*~N_id29r9M821`#}B#?wn(v@aPI&)dnYEy z2^iYGRknUs9_w6n{O1yp@>Q+S4;JE!A3UvmX>puYXR$R5?BOBr(H4>q8+s$8N8tmd z>d>yj0^zA%h@mt3Qop9S|oH z8e7f&SsJ9YvS87vT>6YJy+1ya{Bzaa!e5k4MaWVw-`plrMnA8Ax=wppowfqU`#>Zj z-$uN4p?<@^5@~Fkx-M7^? zHeU?EX*g?|Jd=hhhvRp2Ov@792{AXWQ3Ry8p|?{IZ)!r)+Qbk^ok^=>Dv=XH8Ou4V zzKTql!8fQIYI$i50AYxiOq)znaw!u}g2GhODr#GXsK_9-O_}cNZSA;J^lD?^-?Tp?|iNyNlL z<1#-;CCG0DSEyjjZQcA+L&z8*ld6Ll#`;&5=-E(}#=iC=$}p}+ zahZ2J?-mbV)i_wzblR5*WHtOrQgEw!Q@MrLUaYm7vue&G2g`?Oqa20!zjpi2rxS7d zNRxO24oa!vwrLJ6J*ERgOn(tm0A|M+F}Y9oO4vFIwVo4d6VQZyM+pzPI}N{krFmcr$V zv^top)oO9LZdM?|r?arpY}_myP*K$!YU+Sa{!~Nt>=en&cPb}fbHiXX#%Iq@umAR` zBH*zqds?Qu5D1j;3?vhfKL84)JbYpkvUgH(>fYX+kVZ|~ z&fZ-=4Fb%C=Vu_Wg?%V8dd-i7EjQfK^MQ)0|C`Rm-1nbOGHS4z<#H2E@xTnZe#ws6;7*=z`czm1fkk?u|)9umYzFA|QzZxH;vLJCc zo*VeD_M#6f>5kt~vtnOOeb&wvk=VTUA%P7^&&f!DcF606&Y$z$W7A61LoA71q0deCZ z5!qui;%T{u4N1Buhl3|HiBK1}-N2p^fir6AEDjBx53q9J-oZ#iJoaqoyE(=Wqi zy%tt<%ALq(M$M3(<4*{tQ{|}gqDnNOG&b}*!~r!i^VGCgumM;I77)qQ{mNdbEB(Ib z+&ED`_O^esg?CD^q#cFG?ZfBOCF5xqk)80bp1@A$639f2>5hD%j=i$Ll}@Dc-23ns zCd~0h^bAHg7qTJ)sZovbr?|+?7zyfq#=gdg?0Mq^vW;wh%b8u>)R2(-3tQ$W z=h5Sb9u0TttF03RnUa`%X2x{rZP?hF;a(iD*No1)OJj_57S4GJ&C{J+TKDiIwm5&~ zhTl^*cC+;05qW&28!1kf-E4_`OY1LH0HA1UsgJA=^I+_Qb|4UzL&yL5=jW@Eli@Ue zS2|R<)HP}f*ttE318LfETSyC)Ve*LfgLm?>z<|8!+r8B}Z z^J@*e43F{$0e5-K`Fn7)c+qJhi5(m0`-|hpPj#595|3G{*|e+v0OeWg%=B$V1C~Hv1R?cBTwdWR?p-0 z7jK~^oE57F zmSO*}cq_wU3{f!`!9f(e@t3wy(d_dd*tmSGG0X|c%R3OH3>`<-N&7X4l7?(69FFWf zSL+cM(~c~t1=#`I$sm4u6On%QA}U<9rrQ0MfXcg@grHy6k>{GO+`{HFYD)~0(A>?{ z7|G=IGB->wLVAph*bB#)-~$Lv-{>-1i(qi5f$K;(2i868P78YWX~apUvxnwPMi~yn z#rHhmYOGE>EOPYNA6r8`d*`E1d%OS4i3AeEv`w&pf@vFk!Q&W)bjE$Ucq-(e_*FHj zV4Y!+rjT;vh6Z!gIFLFF!5?_F-AmNQyipQR-1c zR9E`W&Df8}&gg$^QP8#B3tic}AmDGkA?`@;ot+6BzYHeW9oOxeeapsy!l50Q`50s& z_&Mm);4nwm;vKF?23J;WjJMoEQ$l$W(<|Nx^S&?>K>poMqI9$nJT);z+O8x$8(&ZK zi33Wg1{)TrZE$~ITlacMR}S<(oVxCWvF*x*na}Mt5}t$(`d&GfEiwT)&Ra zX$>eB1(^Uk8@1S;(X|nD&2%eU_pgTCm5x5aLlrYd8oh%MoGTsM!iUV>{eiDk%nd;F zU}a_{j;c3{Oc^io=-Hhic$*rm#xCHYoNVZ!?hb=C;F!2<7GFOr)}SU7Lbs~v?|-#= zkr#pyl#J?px4lX5IW)jypVW-}V?2TcC)@40;)EYrbT}!Rl)1)(#sEk>6hB%BNvhkP zy~%TkXW~{@ydXVI?ze_+Ot2(*K^8jHt5yt)zI`>c@DkTs6f#1d*|D_l!|gzt*Gn0y z1iW^Lj)POfEXSLAlrbKU8NaHmbYv>HUZY$M&gTxz@>3r`Ka+bxNu{&k&usf7T_0F; zEnmDXkkiEQncx)8JAmHYT5HyNNCvX#Y-~PcaG6hI(20;rgICprF7%lY4Q-cWJ$J^_ zB$v+Ra2VGq>}>2?peK@GeQK?2x9$(k8n6GBamaraL&Ti{Q&Bn#Y9HnJRKFF4W^2m^ z?Ux1ogfiNcUDB{ZTcEsaN7|<)vU+p%Xi5SSf4@LH(qmsvaP z)-2VaA0lxUA9>-Q{!IX?HvQ6+glfYd-v6>!P_9byF@A9dwf;;wAv+1}H8jTb^0Lyh z%7L&H=Q^RyhA0ZtKxmV{;wiZCNOH{_Zz`3(hQ?f0TxrhBbt@GzF5|V)%s($sK(6+Q zWj{VA@R7-oeQU$D943K?ZL9&@hLh`ct6C@Wyhk)-7a#c&tF%Toni7Pt_f^!A8sRmI zZMXJf^z%wR^2N(&DIS|;4TqRGTRN6imBGll3sH%`9eLg&#W}k=@Gqchp_2JP^g+J> z-QzFlSXNte<WcVY ze^Jj*DLP2BXXASR)_r|@;Q8ztsdgdTr<>+<$LisdIQo^z4 z3GiPoz~tkzB{rO9&pw|G5$YUW4e#*4YVen6JJ*ma@ftJ|EjUCO6(eROYIIp6kzmJ$ zD!Sk!+QA(GImW{z6T^Ibk?U$>hQ}qVvDMwcDJOkkbSy(OqaB^FBpj0_<4sq$nbLzH z1~SGsV|W1!g8pE6&h%C{delJ~EYF2Q9NJ#T8JRnT-}GaY%mi#kSd0yeD8_ocF|oK& zf*Il&)JMxE)QqL;Eg3A#6;K^5LMy&VGGb|Yh8)9}?yQR4+*~ToKOFdponS>l3JR3} zvRK>Dy^W3TbxO(Gud8!)Hop&v+R;^PbVqu)?@`|qdubIrx_hY5o$g7k-s&$zpaT5P zp${z<4CH7&5z?}e9ZlS@x8|LU%b5ADDA{*su*U691--{@bqj*1Dbwe|?nJZ`!`&lF z(t|TUllgrz%@0=#%<+XQULU9-2A4;xF=V~w@wzAC8R{LgnCCE)SAYT&1aN1wK55{x zCy}|$B4tYNMA1PcD2}kvOOfNKw-jMODOEKKdzARGOCq)d?IPX?;mg9J^ngZi` zhNiBQXH?*MGnse6B@))UZ_I015u^^|vrVDbyr``X(lRNkp}Y!=B;ePAB~`LHp<$c1 z%gsFSBHO-^moimc7|@zZJ)H zwd%kAPGj4URIj)RP!+jN^SQs)dxV3EWnB9u>ib#~LVq`Gevx)#R9y)t%CGcF#zjJ_ zac+QI)zD}+bfk7YKYrk1`*Jihc>fEASrkYP_^SJM#+<6Y^gvud{QPw>{b#uw-oRs@ zMV?BQ!z8+AV#u#EHYPZ$FgX#Ol(V~T zxOtv2qo*E%SsT^fO`UkL4I`HCnYV>(R2efYn#8|a1(b>2j-~r{g$i#?O&~Hv2v!sN zFCfMC*LZ88jq=X3A%$X@UUdy2g3lVz2!tb$gQ$jAv3tv)`uXl)@iEnxXWgwX!!&7W zieCEdE0sJ~U=DZ;9@g}3-YAoK&lz}YHK))6HySB|ED`VUB$6%?8L3CZG5=->Nvf}~ zq6vQiLCudNcXuCW=fH>AZKpp@h9aUZU+i(*p5>?{0!WD0SajuUUVKlz zR94mODocAeAcHV{aph-X>rSEwpuzghOo05!!`uT6@Vn4s?%knRJ#E|FoDDxBnY;9q zz}xxO=pD#4F(wah3=RW=Vt23`N&k|5b=fWWnPus>15JVh8Wj^>#w9UGCk9=DSR*J1 zF*vPw?SvdnhzIzEmuwteeyus)L-fi}DnDd5uhZD>0uWs#cZlzGfcw`t0~6dggR@md z(#M{6L9L#$(DOPXz648iPMh_YZbb%zH2fGUIecT+ApMPI`A&z)@A~d)_1|!0WOx8t zD5quSne1LV4(`llV_6-!xYPP({dLu^cuqk38ZrG zC&OVDgWmfwD*P^)LGO@mi`|XwC`ffaK_fkb(mXm(Fpal`d@E(4$)>|uhU!B9Kn1K( zRP)f3Oh>x-aZNC!sXq~5bOzWwj_){-e7c?byon)cS!&sG6);${nK3~27*X-_( z_gR}^Wn^# z>E;;_Vnz#V)5}Y3L*IJ3w=*`-o-AAo%ADUrVB&Qe)Au+TtwES%amFJ&kC3}ob{*t6 zOR0GNsO_688WQfVDSygbzkYvD^8}HY-D;?DRHJ_0Ux)pN1HBW$Lu-Dw?YYbhub$ut z@AF~P>yss&Huz;Q)z=3m8QzNyK&LHF(33sWkxJIumyR7`r6Wx8=dCZk#kYvlh&_X# zj*-Zjb1omJw~~UhU8J1m*l8C7rQQwG5|z**hv(34p~kN4Q+vOmUxlk#d^NcS#+c3; zq)6n?CQO~5Pr7S`Pw9VwByOGXqS2o>b8%A6kL*)U@PM5qD9I)I^}>XlMH8kk{s_t3 z_NoPjVsrUw=-2Px59`WEX;Zi&jcOcsRq_0Atz{dF)f~7w9lQh#p2U9qYu0?}8#(`Y z>Y#m3Y_7UKn2{gKW->vl)%UG1EGAmht~(`C9yIUfYhX5A?h;p+_~+S-fwBY^k3C0n zrdR)mjtx*0@jpgK4;UIrq!mb<~Fc=R7|zKQ_jGS?1oVD))4z%eeJ? zwkLGi5M!5{|LgWdGvk^puoG-MBK3YcyekyCwd?6si2s)@g53Z`@nFH2g%^SM+l^AI{? z41;G`msazw7~y!zbR1VqbdFpKUFA?Na-Q)=w@ADE(jKHyJN7T$jWd>+uCk+!Beo5e z!eUm(I}}#qbA7jP3?c9U7*erL5Go~$eP|nB=Y&WeeILJr6hmz1%o}7YBkD=dW7;#e z#~L}O`<+Kw0*xp79>3PuR*VW~tyDV5U0}N9Zcd4RU*#?ZMA&GIc(F;^g&}fL(k}_- z$~QOT9K|$9IHP>oq`of`6DmTIv661kK>B{Z`PSBVu`_mrn~6`)`yGD0|3C+Mt>^B` z8YkBIr-8&4&^1D4fZvr8wz;?D?VxAwTFxaO(s|3r#=$r5laQj@9|Y`QwYrh^ln?M_ zxR;I%=b16!@5hB%qlBfl^~FBA{+upn+$qzN658obA+&eqxMHTONXoxlN97xSwm!z`aZ0>%5K~nI#Gzd$d~E zo&xkr_g5zClt*=%x1Cq+os{HqOBKlb+cZbNno3iG!)Y5xVjhljtnd|-jC zp@7_a3VC=w4f(ux{993~qNZyVH^f9s$6N4P5qvcujg%p`{Ee>Y-<3ieF)$!Se_)7J z1DJ#&KTU@{dHg8@9`Gek>wYmypEuzks?m_M;p~agie`4WXX8uEEbo+C3?L?tc)|0G zTxhZs2}{&xNS$aBhAmwAFxy;Ry|6*p!zJj~dCruQWh9VikM`Qon|+XWWz2SEC)yty&%kr5edkl6H-Ht2HpePg z2nY7@Q+%dtMW!QF3P_)*Z-TDeaJF-Nbg;t0G7|(f~m>?xjjlTU(F(1)PSeuKnd-+roctwpmA$a&s}o ze0@cv<9247xa9L_5;&vlxX#YHH|+}dC+b@Jkx&s2Vta@u_<6adBNXFudyiDdHm-VX zolnHaY`z*c7G-xf+Re*r*2V{Oqr!qWn2F{XMJCfg{g@2r^yWN24_~!Qlf#^m3(z9=+XPup_{7*lTbdv305D+dK3~F9fc7;q>q;GtYR+Licy2TA^Ylqy%~YGe|(NW`&74iFeSceFsMY;TP5iJ zw5YC#P{Z2d@W}_IzSnzbT%p(FoR8({_n8CI$a0V$u3u#0cc7OIsVry7r~2+Sl=mh# z4Mj3>b3ZjtiCBk(92_s+1=Rk1eE74KbOvrd`0FanPgPo^&ezZ#=-^j9d=rgBdqgVl z4MMh^jq7gjk<((&Y?5@tp6z!M;L3R6qy2_x(LdJb9><7T;n)0NMgauCbpl!=w-vK& ze}@?B8&Z^sBz8DKY(3hd?t_XMs4WUA1-Hqr&7xb*-V3i1&G=7O8{nkc#QOq}?m>tb zL*F(oZPcUI&+sluW^(CvrPS^D@(&4}JSKcd?#|_Xwe8=`U-;GCcggJWisl2 zi4>s+)wb?Bk6kcmcMvI#!Zh54t1w})*!sbV2VlE_3SHg=Qk{o0&PiyeWPjTUABlz+ zH<2l63_T;@HD=61aAd~_3-&)vZS!>UIRwJrSRt>kPj(Q1KEZvL4C$J`n6^b$iUg4m z^@k&y9B3`QB%opTPQ=&G6%T}GLz=evMWBB&onXjXRh~WqD_`qN?|6{@fe50ceHV#^vzljN>)id3 zj-CAcE{)JenOPvW;ga;yy@khGtSU9Yc1|Hh$t!hu6g&AFrF-zN)+4=YW>_=^I~Qbg z-P7m|m!prTtNU4A{#41s+e?!tjx|Yv(Vrgd7QE50O{Kk_5GV^~qvqaQ=J) z`(f_bp2rKA?3!XjtvF2BC|GS{Ce??G+0KxfYEBR~@{68>{URrA9#*t*F^Y zsGW&K)bq?OjPg7TnBx`+>pW+Ukl!~|aB1hyB zY_oF)nR=yu-o2SHaGj>Ab$)^6BqVvG(`vVjuaB&XlTYsh+C*R{;o{za1e~yw3d9tA z#eGoBrFFdesuCjgk$OIIxFdtsV1oWBALo-8j)0V`S}zDan);z_{ge8!N-K;HN9Fz| z^~VU|ALrO<&(V>gz#t<^mqL^k6F=4XLHKh?(tC>s71#iOC-BQVyf~hHfBn}aZ<}ui zX4=y(`=*b{+IRHPm(~_Tx_n-xt(;l^iV3sk*Wx^(9Z*dZcPhB9`>KhIqr#?Bouxlu zVUM|2t4YtT{TP!)zH0o?gt>4yn>Mhau9RaWG+#XfFit%tSd;^Ja>I(6}1ZoE@#7XMI%lpw#@9faC6W5k$Uke}q9%)Srr+EG|+s@kUH z29G5s(>Vj)3nM-~vAcMinolK%76L#tCEEo=CbE&UD!sUReIe8XBEANuAJK67a<^t& zZRCuI^npL%qGR3qq$%J&z5I;(ZxzjLHzxs5!k5d;bKS0jvufHbbF~#qbzU@k=kAIm z@@3R}v-mBh<5VPmehbH4Je%FOr##6lQSM@TE$t%Nc6%1o)w)(n`PoPGR8EZPcJ6Th z>J*j6u5=7f+y!qc_;$Zwjz1YJd6JW7tku0hrg<}# zg39Sa=}#2*?IYvA8UkClCKejHFJoGt6Y+@`0NwCxdQ2W*DCYnB<%CfKV@=u9rokQJK8<3X7cBFWCo224aGUHT6|hGdMfGP z7XtgH4Ys=G%h0kq#hw^1we&OT)S}9shO{fa5@HLdT}rfc1x`?0pT_inDZlzPh7nXlS$u<2`0RT1;da4R1=j zoZyYSuQ5}vIHvENpnUyZ64XD~b2snWY1Mk4L-ZM=A{s`}(b7uK5^#;wYlW>s?yJIy zE$yTU96 zg~2eigAKkpl&bXwz5oLCW12_o(ev9+x6lOA{I!wBUkDK43oE0szn8>$nAynaDEtJg zBkVgT2F3R0t>z*!3JP1nqmIXvM|#dPCj6ci^gE7x8G>Bp?9qmBmW*Tv{V1 zd?SbYONp<)etyBf!>sI`8V1)M#>03PQ&R;|teL^=eyds7nZ0v!aRRP~*o`81_YrT$ z%vF1=XMe~i7WOge=o()cK z1j_}b-H7}FV^CJG53~HaWzZWC-Ji#p4rUI?!9+a zcU3)4RpU_NUj^K}imh4?DO5mx?d!`{Xa|Dux-1)x!pJ3+Ek;zv)l;^8#hu`AN3+%H zX{%avUOOXyJXdnPUYFCgrgb}|kL%|M126H8J^S;csn99~hbix*dfa<1i)5vI(RMu+ zA8jGibdF7^J_#5zs7+5)aJ(!NjVh50Z>C9d-bIOpN?p%=lvFh?3sQH5Ek#QkPm+Wh z)Ax+Nma5`f*F<6(-Rnm>Oqp59*0WFJtgi*Ki8ed4Nv8cODzF{MG?`r zfWV%1`IFwsNpuR1s5*T7RN}JeCT{$6{G|jt7*jKR+l_h*bj3Iqzk`Je#SocfRzOQa zhIeFcKqLhXP1uAU35C<9KD}yrcySmFKc@RTJUpp%_6<{W(`x@GzS`Q_-zydT(vZY6 zhC|tXb~B#p0O3_oz`Z=<_D37#LTg&^o{6xL#pOUQoy@!EeAckWyz~Hp%i{7M-M^pf zOND~J$@j~avqDq{toany&>5D!r(P{-2^Eq`kNQgS{)VYb4$tYuUt&u5Mw7&SKrxHAbV<+WutAzYOjL85Lp zhDlMWkt5+q&_co$|7_Wci!azr97NLfGg;0TB#(3Y5o+y+{e

    CEFdEv|Oi=cJy9 z?5|AtYw|#Kw3FfQ`c9q-!v7NHxGC3IrR5{yEI~HNoDy)(Tb97yr5uJP$p7B=k+7)a zSpa151c&-C)2c>m591Z6@dnjqr`r9%rtSIxFQsT9f1Ls&$pyA)1bz1KJjIjRxBD)B zlH0=mFEf`a}Z^(Vn4HnGp@+2WU-$Opj|Qx5VoTyHeK%9 zZoAICo+@GS{Rhyl6A4sq8Vc>l^;@xJ`SY z*p0lvws>@~?F%o4#TVjV$sEaDfy%n~_glxuF)g_iVc@DOuPq;jUch%5woaNM%wY*K7u~&%4j=r@F;hZsmA2)Ga{UM;FYpa#jZ-unq&8Qs-)K5v!R(!n zKQ`afDs)+GhA~~fdi^L1azFdZ50Mc2�}Scv@Q%ctIz>^D^hqoKJ~?$46vSLZ!3 zL`22JrhG(4kY|dGJCd%bUrdC`Cv&&Z*_&e+q1uQ|7WQza)kyEr{{cQKj%Mep_15$C zUi8Z&=d0UW;5qZtusrc#TQ*wt+)w#cZ`1M@T&Knv*Op3)yAgH;VzSu=<{iJa?X(nL zI=bUkk2dI5J?AFn+5ljl9{>PYvJHdb$Fg8^OH3RdAE)BrPRy5#Gvs=? zk?=8Br)CJyROlTD$T!b!xl3V+TDPY$+Cz&a5&S%Ij)lXd?O6G;KU+Qce!KIv4-VYNsG`*Y&{$NA*@$dSs^EXcp#cqY;%XfKl1=8p^tWJ~;$qyC z>J+htjyUUN#=T$sJRSyT99M01St(4$n#;CN=?W){Fq4A z^XYu|hv^+$5pt<~i+mA>yQ=mQ5!x-TZ2PfgEV=QCF3|yRBS&Nko9- zUY6c;@v*C~bTAulF-8L}b+>8crz37*%hmatc)D$QN)R1Ryk!`xC%HowZIO6y&D6Xg z@pumxaa3nzy7wAOph^orWbrC*GT*j#<`^$TQCwYuV7XFAb3)IayKNNy-I^uO+a@$& zs7{bO?GTN5@8SQR&83lG25MD_b!4-}E1FmIxW}~d{}bt<)A|>rq>=2h3y0R^c=GG_ zD1GKr2sNaK)}G)& z#pPgdp)|R@`XhT5PmFDrR|v#enK#`Vr-|L{ylf%mCdTkGzyvwvN6p$J0E^YA6z?n5 z;sl+fba^^C6&e3(mnmOob4=^iMfh ziobo4gvty}>Ze%iEe?Df-8N>B6XpZgHOsWox+M{Y1 z82oO-10ium*Wt&BrzgbFooVUf?4o;Jsyhc&9VV+FS0L7IoajN2JGo$!QlCdtXkR=< zkwDzBZV#W|oiRAhZ7-z1@euW|mj{lkIBJi+yA?aAeB^(ke);SzeT|JlN5(9k{8xv) zYzdKQ1WTunUj0e|6ihdvt?T27Uv1I@HQ3-L>sGZo1Y!-2@p;xOR8i@pR?Q^a#ZI({ ziYI_t%}|>CY6ue7 z%e&qiQTHss37l$0cp=^}8T?u`9Go#-oAiM>b5tB}*wvh^K$X+8Jr2W;fXk3TZ4~OL zki|g(FVWdA`D__4sFR1I0(kdYfF*6%xF2ZsdQS^&^{81(lvcx)WeqILyE(%{yzi-6 zCa<>nPFny%Myl|fg+7S-QyQKDHTU_u(-ZkbzjyoE0;OMSfMyUtbMsX7^yb<1HGdPadIg_?1y`|RF%V=&_VF4E>cS_T-%x+} z5U551cKJkUN6@t} z&ySqI0pm<`^yow}=O8K8wS#HM3^uZpn#3@{d$eR~8;3FfnpvMOj~(3CPS>oC)V1Cb z4A=TXHhAVJwn&81eDR^ZLoFk=f4$hIh~Z41m75=ScBh2=&vLI_ zHt{D`*&tQfj!uGp;$kvTXb6fs&zn)7gAzzfA*zT7okSy{OY%qj&R^wjN9L@q-^h>H zT1=4f?TT(5K7=B%TG5o{E*; z`YMeKFm@)7{ikH$KxRtUz`Di$dK*U!8jE44r09z4zSPu7sjZY86(;Fx`iHHQv>1b@ z*fj$TcD**_1hPu6&KXqMm@u!BVWSSkp`YH6u+o87<}cPXbq8t_o3@1`CM@rF8dCOW zQ_l|ZG5yubw)cRBRTMEMI@~S-wN%HeRNI=3IiTGZF~Rxlz+;v47RCs0E>NnYi6Was z!Sp7BBB`~LpNXLmqsbFpy43)5Z-)ELvQ+eHi^kD_Jv3dX(tn<1bXI0Q=+gM5((7xIO_tPJ9LRpy{m4K^faWNJ8AB8NQ z?u_}eq0v#3&5m}?=hb#Nc=$v+(>ewtng;8|Z84bw_4ke)!#Q2cR>$8{x?%&$PVO%i!Ss`sfP317kTGe+1lu! zz}IR2SObPch9`Yj#)+aL<K{;!sn; zWwo;BiW9nJ9qjas@#5!z!F)9ycCiNQZr=IAi)8WN;Rl655%-FW(YEAPN1pZ)oF{^I zSao)%izfKdz@$=chU+{<_GfudTR&Crcl3nx-QN{cB^71+Oc&X|NfF?Oa4{Ng z%s>MkI&V(dlRCyl9bwsT8Mo`aASD~eblMsMU1M>bKYK7mOVv3iLU6k{l1gx&MUwj( zvXE^=du8l-6scrSu{pB77b79rk3lh3-0bEG%$4k2Sjo7KEak=b)f7tbyK4plzqw)4lu?QgvB zJbwnmWk*n9R)nYY;j}92YK!m36so00mHm!OQCCu(p3#;$uFp%UuUC>`p|X}HG%mvu z|6E7=UZ*c1f+{gGOX}_^eu;j8VdlbLh{VfuH|zA)N63QRWny^DLdXCk-e&zk@L-kK z>1eCnVM#gWl>yU_I*;A;`s2jk0;B0abL;y$?rF_3=DQwGGq3LWVyN36raGL6>9NB4 z+J);Z2gEv-@f24YW>fA4FUP{(;wk&0j$$$>cYc;$L(V+X5{fx@S$KcyE5GUhEopM& z`76Y4G-pQMYoIqYMDw$GoYBY#Y{9FP3p+g4a+Yyu^}H~aolH`B?M}6@<5Lfl5%>A6 zcV0BK${!~9& zCrj*H#ruq7rwC2`oXD3>vaLkvN4!%BP)_e}JvVfdO(MCi^aE~6OF8RTWQh^IdHzLw``c-RaEj~g z`sB+%x5Fl|#N_1|lli2m;@{WvukMW%(W4bvSuY2J*4bv<@0MxG=P%?|`|o%hu*aYD ze|!ytpD>+;Oi_2dT@iKWb9BT-Ow>^xm<_!=AzY=l*BiPx8)HH4mpmq8GaznT%W~rS zb!iKoRQ(O=_ylq|Q)+1osO0sbQbUXzANE}(N5HGub^kqdj1&X3vZc7*Mo;&9KC)9E zB3uv`cR%SsE!=wsi2rba3;#FCZ^38METGI)qrc+?r=TE4&>W`^;|IEIvvZz2Q&(}8 zgof=n=X?5hN>_eMx254d&VbCtrM%@n)%lW21AMTVS%GaMSI+!l=@=PKk^R$^^+Z7w{cZqL0Q zw2*Ow=X!0H33;;od}1H1RuX!#d_IP2i>tqPDMF?i6_EbfP?Rcj** zHsV;HAQa>$aftA#0XBnIN8`K5#7@S_tK%mZBCPc`_ey6f3r=*pawnLrR*1EPyW3D~Ix7cK5N?o)n%i z`iHq)aPMVi9kBO>;z10X-UB~Evc`7iEys^9At_kAoL^(|tpPC z*%+oUxJRqJE*tPs5D<8@=onA}AQG@dlv75Nbet%~QZIXpRT(2Go%pSRgXg`+x4GcA zORX*e3MUQ!efPmdD2eqbu!}%0TdFn3=gTlzbAsU_4k{U+ZKbKM3JJm%&IY3*LQO_> zXLe^$Jd=R zOZ=`V6--NRh9E9ch|1Xf2ZN2HRs%U9>4W5Zz?N~SNXHTNEFf(JJ`7|n5{-iCNM@iv zbgSuv43_P2q9?N{ls?$rlma;B-uzDr`LpuOdk>7-b&+`z7frq*)Iz=IxveAS6eiuM zWlJpCfaT!~r%C6n!)ZCB+RL#iK(r+=g(HaSbh}ff2;;L#D9ch;(elgJicFrg1ddG1 z_^{O3hWS6TxsXw%tWD6 z(eCV_yXviBuV`fuN4Ll+zabP8%Ocn6Eo@9iT5m9fsCB!Z5xaMYgLX7*& z1F#PLWg*oVO+yDnvk(>D`!a{7EbmCCl=+!ATlie^^&W>V9r(bT_gxk#OO)(0(MIJE z#V)X47Ch{e=*&YN_d~$z()k=nOj)kqQeabzNU4}3;D{*Q$CtBH&hKycS0)6akyCy? zSrBS+lp!`m5q@bV=wMmw_V(rWQOE!gXNNz=fZH3th6;RT&Y^*YO(l;50j)ukDqitM z3GUtSxN#TFVD6z8Q<@QBm8|F}W^h2cV{AyW(~ZS@1w4W2a$yoGTW~-d(+;7k_&q-P z?czNaBFAhf@Mc29+<$u8A@`}KXc+VQ@~&?%IEvVG%{weOiycXHM6mt}hK zMTWPlR*|N5c?R<(*Zn<-K$M1NZ<3S2xsJ0vA`N)}&95KlB@h$D^LS}2nZK@^CHPGAxsiPhmK|{BbdY^MSHh}5 zbpN{fSxR9%eLO?Ccu;EdC_<*&jF8{`k9B2 zxg~Uj%JJeAAE;b zc!Yuh;P=wi(__)A%yBvgia*odUnM7tle%=TwK6psV*(VOf7My8Ob_JHy=?!`+sOlO z@2GV=$S*=;fo_{k7I^Seyn?ydap_@21||s$ z(@KM$TU{fr-r};3W3jW(oR&nzTHJ27w{iV@8UvpmtDh=uPDGd>o|;gSlLw&3rPBJ1 zUQ4p^JV}~Uy`~v@3jm)Fg@BA^S0cwG-8 z;d5Exo{B*W&g?58T`AyE>^=G0FtJu&5Xbd#KX)_k|o_S;4S;wO z(^h|mgMZ0FZ6_~cEGLFaYrMZcEtF*QWjJ&76;wwA4v?|x6S7?SntI>ek4h=<$p?L1 zSVQ~E&4Hc<+@bu(pps%9&E$?N`iKD-D2&1#&g53K5Bn>9pT?GkeGgwY8x_Bu(QB73K`YHPbXahRnH zT|M)P>G%^CVlsC(+8addf9UytzPOq7CwL&$$j%@Kr>9{=9!_w#OCks2>-wX(^U^VeO-HYt(#kf(tyw=5E;fE5>O;1Bt96SxQT zQG(vCIxSM_fTo=9@v$2p{zoA&h9?u5 zBri8?uB*YVUS+S7_4nGz^v?Z{y2YVYr5OI?0>tyXZ*Q@ikD*i9Ll`%Xxr0gu4NUI7 zQ}_9;`gg?f;6}auS~yxFdC8m!-nw91-SzjkLPCYCC?3A_fCR^tWSN~cHsltN z+JwxJL&RNu0BWuu=I^zU)P?pZUhR$4ymzFzuLy68x6UWg?-Z^0i`RZB+2#t5PPX@q z+LxtWkE4~GvA2liI;=P)J?f zoV-zV*0UoJloN^4nuheLbKlyFcc1ILw}MG4;7gF=(5=C#2F`lF6ZLNYtXEZTrB3uS zFv!R))J0m$#Nv7w<^llRJ&9Gk6hAt09(M;~diIRN%X+_;wld#k=RlHNkq62slJnBl zWk=bZA3_R?;~x;4TUs1B8Q))qXsa@MB(RIs81moMS43-gX~;Zd3XEIQFIXSuT^QRpD91r)*{eBc; znPZV%@@VCKG4xOoBQb)y1yC0uJ_0l+4djX)Xo{rSX{+@g;Wj=<0rL}*QzZAz8oIgl zmN87w(eR!w$CLS$y%XH^`sL|$;p!vYGnp0`;l+%K4EWq(^8kirPmq!er|{h! z6*zX2mz!)Ke7K{W$|G7;UQ=^L0 zwL(~e*9;h8DuGx)j}T|}7nNq*3G^;62y>5rlZ)N7Ic_F8$vqlo&7?w$S9ExT4Ua3y zR!{CgXV*QZ(vfsC48DalPdr>4cG0H}#5xaha_H73h=XTEi2-{T!szr0MdNS0B0VIIO~#R2N_o;e zfGAJ)!q{kKa8=1^&nX;*wAB=nlF<(YS$Qf~iBtHNBS;7(=1)VUe12h|j%w9C%=N4z zNy&a-bQNAeAokiAg`r{i*b-wd*Ob9S8b`TM%FeQI*~e--XSt9iQ`v;h=4C*;9XrBl zr;Zer7-SL5#mVUXtwPBut(amNTO(^PBbkbhJ{At*pO-i?Q!Y&x8!8T@)l9d49!+0c zAcRe9{(am>O})*ka{6@s4;#@-3fr^%U6vx*lk)D<)s;&YjW7;{uVTfMjoP)0?fe$hxlCR}pA0Fl(GdC(%0>;`sM>kINw<;Q6Q*R6e)%H@GH2~~U-dCs48 z_yieGCXD~lfAUnlt*d8C1>sv1wVX96DMrt^62)m3B_ckTOSqUhl`x*{CN8y+<)yl} z^b2K^DskQ0hJ+U`4KkDdzUS=qH32jeE;!Q<#j{E72t$N1jV}mWa=mo* z2fMnCUugLo3dakvUAsu5y~i;+Rz@{G*4;D<-?S0jIH7!*vz8kv3Lz>4{@r_$ z!H#w%xcm0d+a2YHiJbZiNXVC$N>4reXH%OZ2jt#P74w}ppooXI9b7R>S0YGHfYHX( zf+u#vwC$L4$8T_qhEBj|ZpiGxu+CyS>#r+t>-Zu5tOMtB&>>PxIN4R}biMUVGTUV^ z)QA-SeT!K&aBNyAks2}f7MX&aD>i%_s@WCK^D^Y()-(IFAKWDdYdClU$+BD6sd=Kk z98@QwSJe|0K&hAn@mdj-U`T}PFaJaUn(nCbC0)2aX9Ar|@SC_83O=5|c)ymjlRj}V zDzoF@RF)?T1T&$OX(hOEVu>>61)n8h*ed2%-|UeMhf2gXUbr|QU$|S9=|X8xH!o-$ zZBQUkRR-8&^y+8~BqT4GmTo}DykK!+bI0P#?SgmwYi+*`?Vra8^;;^g^2@Xyd&$~f zI5KVr<%@4G-;vaZ_U<`M?_b$}=IYzp@a80q@y-q4Io+FDe!G^p8yYTEd9Ooh*^KQ{ zEPfcfOl>u1Gxcr4p-mK6`Yr422ateae_1HaG^Wn1_u1!I9P7S7qza z)!DTs4%wN^sVWN-TLjI6D>Ki{2OgZ;pKc}w8+y(uEl)HvRS~Gw%J~qyP);e6b3+5| z+vkh|gsSsNnwYU;tgE8qG#30X*Kb_BAwWo#P`O;mWFQjh%u$4`v1}y80Gd8ayL9VA zjz%#jFXnKh*DE+l{;sp#!XOjRdBLP}z2_i@TcmHxeC)vd*gLE?$P)#ut^mR8Tw~79 zzXPqgw0uLeVpszodt0UUmc6nx&41~eVr zeHXdHp2!KK&ZP53SAODdH)?XJlLq)~H#IC1(8-Vvu81I-YFW0$Mwqzl(l(;QX+(3ae=}HLcNz96Z!#?nf0fs9tzf3Z4DQs)@V%&} z9DS9ykI5+FG=I_bo^X_uAW`u8w!i!RTKpV`FOlD3VACFApO6p&1T?mY@>TA6Yy2mOWCSuSQaiQJ;m3qoeIU+1HfN&5z+Eq+6?f3;S}CJ zw1kWBwcxNXKxG(?P*RDfOx8W<8e<_e;erdfcsx`Dsr7xhsl0K}+g1HY7hj%L_q_r_ zh<_=#*0a4;TC^>b;K%zXe(X5l&1eEiqh&cjw2PX$>{F)7duB927L#pb4s|#TwiCTOe%yL+skSD-5w}a@3H{LS}kaw ze+J+&K1yZKOrwM*_{UfAp7hIpw{+7`Xh_9%wyL8mdJ#xi_Q z`2Kr(^p`2f$t{Z@b}Pk4m+~vlF%--}E5y=$t@z?}ja;Nq{`pm3soRw6b1&dQJ?XT< zCo+w-^6#F%DPzk!tRyKS1X0zys=bUEjPp|m(3sr1;wL6g@)3Hh-2i%j23 z#oJ1!r&8UsU*ehuT(>H^jmC~{`Bb?1s#B~!H#N0fOlIV0GKaZRzeycs+Z=y>RVs}{ z)czPI&Fw_V^W%$?Eqrv@IX10=bSC!X!R4Fdv*I*GFQC6YzVNt#d@ou4{pm=o?HjG} zg^JbX7Ai)Ln*TUce_l9m-1mN~+Kdf5CA{(x65b#F<568#6pLeE3{wR_VXI~~X;25H z1&rZ)+(-P}%J6PE#7kWwMDiQVy`hKZjpmsn`h1wJ=IiSH7g;$8=0~^kx!-*07}@2I zJlCi~ZyrLHuC!6o_Y_eBkEkuB5t;Q}gsEctgAL3rtDh@f09abN7s4TnY zDPv}DzCn_*UKL~(5Tlv&9%TBy3xG`o=Sxa4fn%D+1AD0rx1LqC);Ny(vIQi*3Sy`J z>aWrF|E4E+PH)tOzhz8sPOyax9@RD|;o*uBd2g7*`}uPVgT7*mvEg7}SKYQvJyTO0 z)hMtwVo%7)y?=Euaqc*m&7<4fSf2IQTwk%Ndgursk&^F6(KvP)=F)8UeS~uoV?z2b z;}uY~MKkbNO@P6^s~V^_ll-mMiSmWcs9(9AL9`{+EOm2`al_*yq+ftUdPym9FREGj z_pq`jb~J_O1f%+qV_;&8x7w(&OJv*aQ+-O$p%bjJyd!%xKxcbR`e;wSJasioczAmD zOH(vW2&zyNLg$9)CPt?vk)S{K89&)cqB(0IGhTD0?>sV_=+ zdd(N8HRa4Tm{IW+@K=-8f-Y2PEMKx$2F;7O{yL3OtRq1o z*+?~41bnwPS6JRQb{SM(b9WnwgKQhqOUJ5~Q!#*c%ndy`^K@<>G$(`l)rnH;@fvLv zf0$nSyE^Q~97kl-A8rt^?3FpaV3`AF_XWK_UJfWvNvzxZ-%S$rT`F$iS!M{n8u8f5 zpM3ni`4dBkgsaW^e&)D+`s$a-yE+s8^Wp^7O8_*ILl|Y$UwqA9e~mC(%7b%=D!PH+ zo2l+_U8sfa;G1z>sKKE4zA^N@o1J?sMJTGU__lHtOK>Y^j*GFY%E;2OT2V2L%7 zyJ&Ol<)YuG!xIM%xkaU2LQ7W}DCJ1Y`+o#hD{q_RrtTP6(Gl5!$40iaa~Z><{$!Tk ziZah!@lA*U@~+5B+zKYDpWyTbcQLTqvi$Yq%a`9HDli91-{m3slI+TNIxJw;?((4tL|cJe1$vfO#4bP6I>pk#vP30#e%I@Co+a z8yQgyqZ(jmx%=PMn4@g0(wuciyG^adc?FkM3mf8b^pw|M>L~u+W$}9Mvu@1d3{sVf zFzTLK#K+Z6OG#X;a~VVk6+VU5vsWq~D$4tYPukZk(yQrQ!ZYrzNaLu2Ct?uuGdnJr zO7unkG~Fm`r744#A@+;Okh8ml=WTCYIC2M-`=;LgSzheV6nPrVSPIHI4rp)M!b%N4V>c z!oAN2&uXaS_>IB4oHnjS%0DBZRH5C$qmX+&j<&UuBNiI~SWw#d?m4Q8hnpw-G@Ue) znZZ@VWxm&$l~L=hhp!~a(MLceh0@Z}15VUzO`;h0veh_F`Gz8IX@KZZVBzm#7?-8f zF6qZl6T343;hiWn_;FhjXZ5lqv*Blgl3C|fHosg4f2v3zaX2HXb!Ud+mZ`UO*(6L3 z2FoKbX@*Xym|q;F_+OzdiLQJsTO)|;>7?=9xW#!A2gjd#M$9zn24Drx1{A!8QdK?r ze+=kEy;%n<79XfX9e>Y43Potz5>Dp#)$WM!N4sok0ckdF^m6evRcTQYE3b>)e(k;( zV|$V;>A8T$|BN@os5ca}Yu;KWG*P(9YOE@`h9+jB*y_b^lYXE&+7uB~vC7u{;EV3t#j5iQ% zq$&lXm$7RdfJf`0(A;w6mDTiT%)v|5FMLee7N+V`JRphSMG3Ya4Y(6srN;&w?gyX! zJGqv7mt$6{;*32BNqGn-gDAGLSa4uulVjVSyk&n0o}gH{`@hfyPQJ9p^yyGWg%9P+ zA`v)y5XCe8UJQqSa_0Z;WAwiD2g5Zk*qw-+oSS_dOMVXu+Wfo42|yW^57(sM(@WTF z_zpjRwqF)HK6>RZ{MJ6%VVc^UPQ_fc__N}jZWMB z>-SnU_K9j)A}wgwcm4~qNh_c#laH4@|pjR=3jjPo!MxL*#&qqP2gT-B!wxHgK+p%qM-#_dZX?8 zcTsAqh*r%;z7*r@gDs^kX9^1$jf$Y1p~-Ip)S?%!7AlrQ1c{q16U;wHAlO;1|3FDm zSxw{cjZWYY0~h^^AOBxw$8!QQF5TyCa&Z@PsOi?fwpKFbik8WzU-zgE z?bk3VEiZW@R*?FQDt#PHpCp3*sGu})PzU&E763h`a^t4-SwM^avqCu7 zv1Aiqd{?O`kAmX6p{*cxl)G4EA?0_N%!a~Z5bBHD-w`hE%5Yn3$EQ^~Js#Ip!FIGt zC_Z%ieC2R5o~*twQcqZyH3fw`#lKHHsKPCzxwnLjWreS1EmW>!Mw+#^7u-e5#Y(V*ovAa1xf@%M}cM$PUx zqy@{l|6Y!kqSe)`yF-|}-Wbby0ApM7lo|7l$J)}SdMA3K@JaZ5PGGvfsHtZ|PG5mg z^777NvPXbSPuV&v!tK7a+#IWVpN zTkHcNE5o=q#uI%?jBgr1#?7BIU?tt}(G?6ES^_DM22qD`VGx)V+;PyYCo5?XYAlLw36a|DmcxuTRFT%6Bty#d4sGp0=QScD> zD37Jcte-G{gO}UZUA`@yZ=q6$Wu`7SyKU;_wdWo_T}I8tv7=M(3w%q%+v-QH>n7CXq5h91Kwmk}Qn2+$=&*S}OQuOhY4B}SSLz2-yZVEO-z!qBt!^oD<}H2` znG?LmZZ|kDyzNPxBj(A*=*TG$M`XX84IBfbbe;X0l$Dop;u1)LE$&u^$P3{w5UHaV zxRq6oE$O5;4Jzi@@iI)P!=`@Csej=4KJ|NSU+FRI4yg97t4n*RYv)`7<(Vrwe2Isg zi^&2CS!-s&$N{;Sjr(JHlc zqOU#r%oKvyJ>HGyaKeBZYwlgshL+GyY>96jeEI#x)Nd=Jv!{Xy7LLAR3=7`Nw8=E= zPJXY=Llutk3<jX#;Q-b?~_ag{%pptdRs$i@V? z<$af>TO)8Qc3jisinZ%$(c-x8t%JA6XQ(`^0;Bw|I8wgnY_vr)QrqewOyFduvGBd@ zw>a9wbW$`~wXWX=Qv}9L$J=J{C;bFj)NS9*&%KfDo&MH0fPrx|=`v`+=BoR_ zsVG?{#_wF9J%jg^h?rzpt>H{-^cgz&0$i^)o{fmc9aEqaDT>K1AAfSo9UDNyk8y&z zSyQ(LD+D%5^1NFT^KSclOk@5d-qezNlqCl~^T|@=Qi;-J_E~B6x!rv5{h-aX%_E-v z1-m1iHgZ`7`=oF~E&#gs$EqEHPtHO2Qk+5Wpm*j{xtY~c9pQT6V zLj*W{bnwW4J#P6HbiWTZb&h>p3)~|herWH!hIWQN&iOSrV%`%%F1^N;3sKiwJ3d)G zRecD+yxnUV`BCqH-qKW_c3r3fU6dpTDO?U(1=^*f7SfRjijR>r5;#uWtTAC$L0rhi zeNI;wh_yx86;ZD%Z4z%*7g3Iza=lRrX3Lj92<0MI-M;O+bka~(gnmv}UDOAUVK>BV zBj=u7&0L>jXk=cjklDf+(|~>)e*j35tm!*`fL+!rm9wWfi)&+~gZoWzwN$Yd?D9%3 zFV%k_m7zxp!Ca1BJO&_WrM;S6t=bh03d66{YU@YqWAk5`Wh7Y0YEkqbU(@Nw!TT)z zmXegOXSqC#l9w+We%j-2)$hS zyc__dpC~mGd26pex^o_dXh7yw*=}4+$FcH61NI7v18AEha>`-Ahklf z@l1v)1`?m;l&w?Hl4fDN)IuIu8r(Vq%KxOy@Nw_HHtyAyb1|rxpU3<68!H?S+XY*8 z6_wa`K63?*=gOMd4jd_7?+1A?2LYTjm3^Mynz3LtG8syTr3*5o)2vciD&;0}qPt_U zZ;u-N>}*pWJz6Bm0b^Z-4~VK|-U&la8;t(5iSus*s6QT09xDSJZ{UVIV+`F$J#aeH zuP@#GZit{MLc)W=kUB3_Gykkx4*1O2eCI-CK#eFlIXbSa_)47qSP^l&t&U*oVwpC; zJHq5k%l@ZF4O=qbMTG@MJVgjSP8P%Uhpkkp#&K_^%T)^n90HW*pff!}yc3;t1P4XK znWYX+TKm8{!iLs>kr`Beq|KU;WP|8Dm*a8=#}>8xr{DeM_a76^4E%dKU*3AIU98$7 z6#9j+-PfJFmfF2S5g@qf54cw730UvRi_>=8jB+2MnCQy4GN5K!>S~0JP5XkbozUBH zz4H~7zTusR&QhZZ4MXwmk4Bx{#3|(tRMC!y2)kDD<(t{pq4kWt$IpK60N56jIC9#A z!$no0?566wE;h4YmFgXqQ?1H8-aj8I${p&{9A?HZlXkw!l5H{^;wJp_qc$D*QhrS~ zu@A}o?7kOrp)aF|quBy4RVH749-`p*-4WTRM7q9Er%i^*n;ch4rJ#*Pjw;7&ymf!y zVT54!GMyo9L$Tawh$Sin36j@ZZ6?}V&!rF}p?s?bF;&0L;PMoBC45ZzxQGge;`<8S zqIvlq#C94h2jNYy+r0YmHKg38B=^ncHD+%vi}MpT)JQ+(p?5jbgWdtQuvhXdxw7`> zP{gbudq{L>_;W~V7_gJ_a;?#7m?+evA~^H%<6eXNLEq&4AWqpxdtVJuz;Q%#l-1w~ zv>WwaTpyN*Q3&=dOGd}$zr(~jQsl2HI~^)EFGz@dE1{3*M3b=Pq$q}G0*e0WpB$JLf>jC~6{Nf3CllR8pYS$*vTH~%YepteRX}Oa(%J(RCR~$! z@37HHu-lv_3A$*3u@)kjkaJ3+F!G3xQ7T2zJ_~mUAyGaiy&_+Go84@SW1TqhZS@Oj zeZ6d13^ooHlUo#jCnDW5ICb~8^LQnaJ3jtjY`t|{TV2yG+(Lmuad%45;_gm?;>C)) zySoH;cXxM+YjAgW3GVLprT2ZF^PT6s=fC`tovg|1H8a;-v-XT`xBkShQX?e|4GpHK za4cf$w^1|Zl4qfZo_4Z=5d#_Gc%TQstl{uS4usN$n>}lNCSBSRofr2IfYi2&2OtfG zea-IB7|ZYm0wbZxTVF1pOF|(V7a*KAf=*5 z8KwaFTj4qc?z=o)8-DzTq_`;7glX{VbIu8IVL2Fiu2LG1Z$q>W$CY@#A}4;Ec{p=s z*TUh5zuR@ppnLa&{|QOMnks*}gDaF9Tcc>!6YVo`HPEm0k_hUmt-8%_PO_cZc2>G) zVmZGiJT1e1>7eo!0pcinP(APOT4ueoGjLWPn!hxXLtcK8enJKk`99oYgUi#xhY*_@ zxzI|IIHW-phGH)Pqo;~)h>g{ZH(u}Nh>0#fA0fHqa~3^0#DaH1J(bw&;^pWLaqufq)M|M8Fez3YLJu_5a6ZW{{G#PEy4qxmmr?1% z2Ub0@wsU0+gcpfvj?ijlHFh+hxngtjPsJ}hJLC(2 zC=E8a#1uZg(o*j(5XbVR$8OSZ-BxXtPj*Gj(tq0d>`1)UEcmpOg2l=@+TQHMw9YjU z?2j!9N0HuEVg3Je7GMWhn`Fm85ZePZ+Q~1#=gC@mCP#|9meXdk)%d^&*l_}9r%Ubz zg&fWEyN5DoFzyfXY)imw%c@F$X*DFr)@h)!XJn-MFld9sW^sLM%!l`9D$0-YdcHL` zmVwt^riPO+iX%N_Tg%qzvS-8-$y3E6j$p5j{ghf=%t&M48Q&)hLTc=MPnVIn`0GXS_}PuRkhvqHCFfWJUi%1Wa(S(}DAZCn=w>lWt5R!z)f z&>g}Nb-(x)GX<_f=4h4{<7(DmrVbB2u@$oC05dQkGGOyN65vO2n9PR)OI6V5H^Vvzf`AtA|I) z3#lGlm1!rmP4*B@xr^;YRz({}9Y8r()$FZza#hO7{)Bb^$F?JY#3}S#4OAZ*K)9d) zPgPSiJ|! z%j-p}NAxs=mVtfTXn59A(WcaIgPKhOtL9hBk#yw79HM+~EFQkb;7!TQVy0XK1fTt( z)>Hr3>d_;S9nCXn(bU$R!(rN=1!lm>di$Oins{aV(o!wRtbiu{s!}EZV(6vC7B2*= z)cAL5xVrFWsU_IlqhX4YX9pNJ+-u_1^Pjhu=18%~ryTWN%);C3M2r*PoV z1KG%y(~LQHOi;Y84r`};DFm6q{6(KPR!Xd6W;ILG>H);SJ<+BLlTL4~^RpE`M&SC5 z+i7dN`(%lz-AYzf8KQ_Z8T|n(hGU%Qcm?hEAxkou4|m zbgB#Uvtus)^lI??uM+C^mH15qUuKc8X1%ejUhA?Bo!v_v0HYM<2r($Y62D0ucX3wqu@|%9#Gi(hV*3H_40hvkBzuISPOL<`3xk zde2*RqrhaW6D3rhS*@J7OT>Kn~YLL-s)Usj`or9 zJJ_&<-)J}dTWY(gBMeW&lK2DL6}P?#IqC=;sDuLF<;s_rYj4 zW#+P=PkjuZ;wrY)uKTfKdg<_nTSfXA+5jI;L%n&w1>1TC{Q*_OF2WKnK(17W+@v;V z5_1G2Jcc{-tNw5Zh4pC+rX|S5Y9DwO5t{*Q0!)n=Ll86vg9Wo{ZbzkwL}JqQ&vYhK zobOf*ZX1yHwSZp@e$GcImAf0C%kVmwRMa*J~>t@&M}_yaz)4F8+f=E+yE29Mzp^YJ(`Gg$xDb{{EWBld5(>%l*{# zPRo33{alSY5Pk%K{sW8JjT>$8lEx_1FLB<6C`Ezod1M&cU;4;C?$EEzVUNBYHn=ZL zJSa+aTy?*7hKB`M{Wuo{0?Wy0hvVim`1Hvq2U=O_?nJUw!xsQ+ssdw!oi07MMKyEs z!bJmJaCK`~sMD{3FE?EAJ20*gAaMgW+`&Ow50xsl&U8beb~v+2I7%J$=H4gu^IrkO z4LUOHI!ut5Xn~SOl;i^2kqZD-6{cW>u&;AP%%{A#u~VG{mcKteM;X1Sj-^EucDJnL z>UCH^i-;AJDxEb)AcMLXVqy$?rM7rY8ocM%KlY7hRFZ_Y0u<%*oNPJrAjj0$in?m) zL3#-iCN7Mg#Z+p)5Yk7!w{1ow!6(yXJK6{+SNOlM7Tlcry9lMkH8KIj?ZNfU{^(`` zY+nh{8__-4tbsTAyBSl-!zyCw2$xSwM-rz*1>}Jj8_|Mw=coRUH+5$uw_&d7qxQY_ou6J>3hb;TZE;i;>UY0- ztl7hAvP6D5PJC#8CD$_DuzS%n=1H9=KWY3#lW;7z0Tt^fD`^vm4DyNo5Miei3~Mla z>?74|873j|z*Oi@Pi=&!!-$H-lJHSbA#4q^-HHlw9-|haGr2R-wP%s3D%r9b*5kPW zZv(UT&}|AH|H%y-j0CD*r)|Bk&wG%z!|0t$3`eAO_uG6{3t+PDgPyNW= z@qrz_V5XvP011H5&Xk85b_b-|&-z&NBHFJy7{bOz8c zs0kpv$3CjZ!4X|xd0#ei*kvByT*!#?ybWI=*JkfLalZ<&KJEGz{voV@F3uLwE@%=F z9=3T+lIssa{zmW6)2d0)mVrOW(ULT%h=C8+XzsAw-J(uQ6=t?$w^!h9MdI(mP{*-& zJQ?j%DWSVroSGo5R26$pWN-HAIgi<7qa1R*0YQ%L#PB^t@99^P)Gt}t`e9rF5oHnW zcmQF7#PEA*Xik1d2C^(wSja;?5$JhFLqp9Wd*vAH{hBFwxhzP()$FW=lM9Euvbt8# z5{fq$ya&$D-x~@TQ!td6s3Cwfv)Vmg6sB)qcM%dOefnGy{q2`NXmAh?H6vy!Had5f z%2{5?SZzCC3gC+6O-$z~5R|G>Y-A4AHq2UW{#+lj4whikAx^f2^!;dFx>7Ke<7l5Q zg!Z#)DV=K~(`X>&1jZ#$D7a{;7OC~aKlfc|wKsiu=zw%I)yUy^1}W?8{6aLy{5YP! zC&iDrcb|%5zA9i|LM+4S6??LlRR*^+zIR<%d~+4{{0Xik#yzCnMe^;@6dy6HUojr2 zeJ&SH#2ijYe>B9{TTRb|Vb-M3*y7vY!MO(0Cala3HO#SFnu;#y!NpN`Jm7t{b}$2^ z7z<5IomZqRI{(Qt$MRdGs?I{wbiorDQp@fGHGyL)TanWgsT*R2+jNmTyW0(Q+gog8 zuievw)*VF|$EWpSevp~jSbHZt;*rN7s2iZ zy6;bGlh^LT2S#b5emnoi`}{K5V-M!*cs;llg~9(x#zYnJVl2@NC}#?W59|U+h_y zb(@wl-_c0@h>^XSNM`e;QitwCDQi)NvN^tdK9HQyk^R05S7jh!mH&_Jfc1`zT20af zx!2$mGolL;PGn;(L@6wWjm@z@Ytrf$KZk%O!49_+YLSsdl}fFCUsg5@%Solcmyv~{ z@AW&H{d$_P>c@Iat_`Ry!}II~^urmW57=}jX*2nWnMVjmohjJknjPB4B>u?kaUT%C z&H?0AJw1JM``zPloA8@nBwD*Qj5(HUku2INBxhTH4PLDiJ_RU9mxc1uu&<)74)>Wq zS09n7suhh@J5%Q8sBLc5tMkKGb%4rXw37_evh=Dkl^EcOJxn;(<|?DJTzBjzw_@G; zA84roh)3m4iE$1DQk^@8mv$=DR#%eeX~oXo6yfKee?H`+;$RB*Thfmyj$6F>Uxqme zw!(h$SVT&yi+C|#W{NB1D1J4qV%}boW zdEa+vG1Cif>a?WC33xES;sgR{1t}CYT4Z%rkGUG?)yAhZV3v`N#IDU6bbOd&N@iV+ z5#zM0B#+KqN8jimW&%7tFc{*p!^UvfED=BUsKt3+P_PXc@tnk=%7&mGpVwYmsIeqv zqrshuNf~R1kv!{53-qZ&tVjUg)aNL-b`$!3yiD-d(2EF{5u@Qu4#P%}`8`yB9{*q< zQu}UlbZff(#haONrw&)7bI|Glm7U3bUA9-d?Ve327^dfkMO!3cmQ)Uagd_$$uUkn4 zx1?a5Unh83atDgC9;yB`D?F4owy*neTOI*|*9jT6vJ(Dk1&BH$Fm9<22ZzWNqEwuAfIF4xRa8Lnw^I10Dm8J3v zo7BAKk%IYB9TPkRF`#z;9m(^9S05V_e(Pmd>**BEx5c_sQt~9LaJ>0bx=hWGqJVn& zkWdk8hKkGze~icE${pR&kH-=V^w`vC5-OU_QdVA!DH>w~$#DQ-H}kqkD~=^~{bux$ zOBVH2wT8Z)vD8`pKW!o4i0|4zK7JO(|MmdsU8YZ{MUzdt@ z6&r26mLb6Y_~RNX)6-#Ae%lD&xCw{Ev9mxs6WU{;IEDo?=_s!Z8?IW^_pR&Twh2iB z6R_k}=X;wOmt>GZbQEv%0+)KGrmS}`iAb|(xRYTM(mkrIK#CS-_a#C@A@xSiQ{MDR zl~@)2L(jc)nS@*G0A%pmi_^12y`XdQO}hg$XzBHzrH@s2IsPt+lq>`_sn&squ&>2f zL@*n>Uo@ZR!_^!7-x^l1AI~UGfHnY$@BYX_!B@^piX9{N@ps`*GG4OW_Ob&FrVP=H z!s;=&EJ>^|)5TM|5W-p#`eRAvgBEPQTb8w-)i4Le>)A5>L}%ETqx*;(Zm_2UzL*=? zW={*eXpb4sfLIOK2)m-4R4p_+0%sE1UF=Piue?(G1Gk$MEtzrO*KezbLk&)`-na5E zGec?pU50u059E*T>?sZQG(H*x(El}hzRAtSI!+k_UlshT(1@G1*&Q7d4V((nunNHTMoCYNlQ)v~;5-YS?VFYdZr?UOG=N$Yn@uHr zNx+@8&c(|X$T0tcEk;NiHbj;+N7*Pkcw(Mz(_WJdEoV{0bplY-Hm)Uf ztwlhw=oLz%2=D-3KRuk5xBe^2CWox?aukAJNEL>*@j^v!J+Rn`z1d5xp(!m*q*Ns_ zcuP)yDBf<>XO3J0@+i*mH;H@Ey5fIaT)BSYKd}R&M9M@?L!u4(MdgNGrPLYYZuA#? zCD6k7CjT>8w+zoN3u2wyy+XgMDWeFz@!RreDZ{sjuV5{Hwk6uZ@*tZFuy?1usJW011H*sAv={#x&7f19+1hcK`GRm-c79biajS`1oi$GC=`}zv$0%4H;q!0CQRHIl;#-eXMM)>bg|IwZ zkO)0eU2}~`tq$QvmhHD9Ib(9*q7U?4YZI87FO3Ou73N@-qK3kT(SLzKB3IXK}H5(2eZp46vJtQZnUnZiy75Ilz?MA6i zXC>VB7@)gaKT(TsQ{C(}%2YEgCtc-JM*c^d39IR^p8tyY*X|kRn{m9hDK<^f#14j$ z$A@olz;+-7!SdH0u8h>NvWW98T=_!Az9gcgI>k<6*fTD+pmF45I%=fOY~Sj-EG&)~ zd-ng)WRBjY7Tt2NeD`hTNW`E-L_A>0UOuv|hO!UzNzA|MXYa1b<3GuL?X8)Bu zp1sxwo|;0<`Rs>zBecJ0LGJQ=K|&+sL|$0M^36wax$A;9Av z&N{a^zO}B&Y>KhEXa~~7I*%Z{;ketJ5vM?SR-mCB+ zq^jN8&IF6l(d;wX){Dn#G+{mmvMZqW9}X9eZI36$rfv50jxv2gGmCGI>nXwju>|JU zea7eK&r+#Y?4DV>_rS^M9F})!4na8&N%mdVG|9m&3*YBF;N;p>W-bV zESYa#(*Ij7q4+gsCnqPs%jE#k2aR^Ogmg8)v%UV@Foql;!bQap4V)Ron!oOmSxLUI z7HUK(iJL1|K8CeMI+08C{tT=wRSLOf-I`ubGchPRWHJN zBdQ~F?NjsxjUd00_WCJyMUq>Oe@Qy~c5QWs0TQe= z4LPUR0Ng2Mm&ZywR|qnmDkg>yO;x~I+% z`KaVKz2&op)Icq)434DC5+$V6`&>=?u_x> z*CMjwYJZW}h@_XdY!x1OiFPro{roj+bVnsuc)pr?-sT)>^%}JJbg0v?Zq6sfGoCMv z0(y&fm*g4Bk0K{@#-&)k%^HMxU7qQj@12S^^yo056=Sv%-&>7{+LP{^*u^)0j}u6v zd-S;NqVU(EB-UPOxygEGJw=M_;=kC#2_VqQ9KYJSm@AgQ!x9SC`{^C{kCnQ#sr=!* zdqw|EFx0>ID0Aa({k6)8OS-#dM)4N^M^xnPWQ<66rS)PRJ%VMLPut-_Vri0WoJRqo zI_2%e_!4%(*@#+j)=Em3tVed~xV;O@M3 zkT$hBKDTdk#A8S2@lA{#9y2+b5b?TM7M{ zX`9ucb*(6Rc&WV8Wbp1&|GC*`^{`PwIMGlf9))uAkVd!-1v#HkF$7rDjzZqee_-2X znR3#!Zk=GV#9Lo4ELZ!fwj$mpgdC>quRG@j@wVP6oHiA{t@5S-R!YLMMbq52L2F6TFvjdLPzrg~o) z9nC&SCeaa-Mw|J*UqP`g`UCYl7}&cZfKj#XvbKu1Xa1;81$UX>=^VKLMx?K-rV z>@q1iw%K#7Gq49s$baO;pAc9uP(Mzx9S*>pKTV%-A|xa9o`2pcdl!w??WoTOho65U z4>*e3DzNzVrz$vJ5b&lJx(?-Q4Gt7WWR|kIl_F(Kgnhs+-hIGVO3;YakFsIIFnh_A z{IW|InGh%CFl&01SrL@yN6W_Q#%Ls3)eUauA`wC3J{UqgmmKz;7DJ(9+ml20lI1&o zOmoT$8`=e+0yV`rmXTMXXygG(e!#q`mbhUD}EuKEIJ(AzeB}^-Wp7xsfMo*verQaic_}xFig)-D!;fl(s znS_olai}X~i(G%$pCs4*Nu5n;&6&98huZTryj>t9#fYD6Vd1s$L zDO07Etf9=};0xK9DFc27?mcWxMBxkqWx9_(Rh%~Kb(UOJ4R0Ce+n2)YDLmVUDz?{V zz%HgA_I_El=}B4n&q|T^CG8nC@lWR9cySoy-Ypdv5!tDj$0{u9kh^xx_=&7|_iYUv z=m7ts9q=njOnrQqyWXkpiKqrgeOPIB@BpU%!*k;rR&l?$JOc7KLhZTKQr4SAMc*Sg zbq0t;jc`nJvxUjLvUJ(g{gX|3Td^T}{Ixji&ghtk?h16rMT?)0?jR~exa zD;Z6@!iUEl;gI`CS|xKXUHhXzcd_6r14bK*2|^w*r@rR`WErvi*vyQu-A#?(ClbQy zWr*|r_(vSO`U-0?rKv2XTr0ZQ7QZ>?woCGrOGMXe$82U-^z_Sx5i%5aK;rh2X;%u2 z_o_xrzX|?q2B@!~GrPL#XpV%GCY(;q&tt6v`o4dFvm0vD`FgI0kJs@M7zY)}v9up& zRVuAus?duj_GAu%b&@2qG?Zst*f?))(vIazZF)bND;wC)0(6~CYSL&vVHC+`(Rso| zfG5)dB;HHiA5Vac?6lVM-^^iB>um?>-qD2OltvS=*(_yucIhf?pG+6(!);BEeC`^N zyV0bh(|XUMv7g`}Doc}oSUo znc!qR;7$WM&8a|cKO#jc{RM?rYgTvZd(=LI(@*DB3p>tM0OVGn+s(y$va$v-J+KbNMrbu4 zz54^{YU^~@mcU1o%=Ygg888WQ$@xwp|MGx$aF@ zjkM*1&N3E{@5j9(4@MsgHb{hCL=xkXyY=i#PItxP;A~&oi2YYF^^+ea0L=Mf%nr&E zXc%o72JZZU;htDXX&zUV5X!@OgcB6!3#Tt099Z{_kl52pAul|eDKO4&LOjUq4c{lo zHoUTU)3iO(?B;$7xqL3FSYa|aG2w|H+0mr6tW)MIaaaDcT}QZ4W1r07^hv9}VhBhH z_RB)zV87gDb&@wXvh4JHTU01cB+c{dPN`H!yTI%)#NrslY)BlZ-g`M$=XwJC)xSht z?x1LIlPw6uIN`NRKSd4N=N-ooRg+LKDv!Rg@%;c1;N@fIT$%Dv$!#M)`|k=3lw0MHD^zgWG*s_( zjoL`|hO1CHP?}#)z3>8d%`v~b5TgW{4|UOcy9-(eX=Xr8LOg_j!RmW^18G&d#4iuV znTao#KdK8`7Cyaj@Bh(V$HJ%9{R%_|t^aR0YzKzJayCoxnw5_U|37sU8Oiy!4`gP7 zr_pTj&+=OR^(2D}(+OVByKhOluIRxOlvGsf!QsJLY7-K-OR`zUI5HXemP!bG zMSB;iS*CGOqc;S@`fZ_E-VX5NSU%53s~B_LqGMeIWj+*Q)f;)XHK-n=?e zby!WbgE{UM$9Y&_cQg@aw2CdkR{Jdq>DRAd{k0oKDa^I35<1bh-oku( z?E6cpF z0vH?heV>J^F>fLLJmpZ?uW1eqd$NGmYa6(U zOzR35|0dd&-)c|D{s&u}2mI_01#XvwY|ldAb%Qdm!=L1c|M~NmlY{2Zt<{z@Y9Cr2 zZ(9$~-Sw2CFde^It+DI@)ayva^}f!00}vn00dtHX-DDwt({_`Xk%UCJO&5d^j*DGx zw_SQ*#@pC?De-(s;(*2Jb7ti;w4yiG;oQ>d3O{G|`4J@-jx(K*q-6u$V~cpEHPEPC zwLR&2yFMZEf9mglX|gXChvjU#LE3x?vo6uf>vhfZr>0AHDLg3r`!)Wp$j0xe8{qkfA!hPm)bDoq1r4JZTOPpxqvA*IUZFmCDucU>^4@mw(28pN_m2xTf-f&Kb`YX> z1(K4+DZKyY-2bKFrU_DkZF-lI4EEQc$h~Llj?K-tNxKo{_V~`6Y>{w4JCt-wxWPz) z=*`h?L~Wr>tNa!rQgmAPc*>gV(ltgu;Qb2=&iDzD*Iro;?c}=s;a|S}|1g2S%9BHd zootfaB}mV)bx)ju|9oPSB@xZwry3TE*E@K4_&nuPY z-J(M1>!tjXY!uy?)ntP(#B$Pinx2isx1@^#dJp?Ng8aMHlQyf%WLHr~E&e4#}A z&+$aE;IZBLqY4~#*Pls@#=n|ek4U-NL(E_*?Y#PwaWIHQK_o{)T~(2F(~FNw|sK~;?ZzBzn=!_D0aF3M~peW=uGmyoFdzBbpG?S>WuMS05qgqWAQ zpjI1k5^_Jy0Cdp{gER?=dp&p6fceE`SLY_m)5{6wUSLkob}9a~I_w{s_}^QbDPO>w z*AcfdGaj!m4vvs)S>mhJ3X8C`}%;V7h)X_73n#O=zd!|?A zaW(B!b2K7=3NFX1m$tZUt=wyBwezI0iox%y5B?>{Zj3>g^w3d98V@OzIP1LJc@bJe zp`9E|Rm%!Rrvh&Gc%i!>n(pxx(E7Iilk1KhhY4mGXu2~8$I=~MuhWLNEb)^aQ`v#W z6@lq%W9nW0+SVr0e&`ybl5 zN<;mv3c*l!p`fzXtC+f(a>XYzssShHtLyJ#Zs8Xs);%;0D>evG^0N4?ZFfk6s=XuydI&qh@AGi&;d!@OLDvvr1=3XeiRcF zR+cXOr6X9VpIJgPD0soaOFcA1L1M)khTEG{QQigb`sZ8QKPpRzt3-LdV^!|m(Nvwvuf?m z1JFUx5~m$B>o}u}Ne%`1(_4(X%w1tmAhjx*!;zRu#`gVB_9rR~`f1wSg}(MxyDzcH z-6eQ$Us5L_PH9`YpVwl~20zZzr~XJW{$+Eev@QceG{49VtX0o`i}XK=C7N@O6xU5dwx@bC>6Zd?nDwfKE50=!lq>@~obGW+Fn zBqn#)togK^mjFGT536|B2yfqTKfqPCW1@JSynnu+vha>i&4hpRV9pVf=WsmvaBPg2 zubZ%PAK&nUN2Z(=7Yt_%R|jfMR?BzSjI$3))(WZV=jWqS%sTZHkm!Nqiqh}~TQ@c3 zi(n>}E~BJJ6%~G7vC1$?2}(UZvvI(o88vcZ$v1(Q)l4le#il_@GGFBnh{J0fae^Hh zH1`L`s{W*!YLILuI-;NxH^ZF7Y^N6`!$Gn$;F4uvyLJ*h%=S!8Jq}H5kq^>Q>rBEj zZ3U6c-(~b4BMOdPVlNtq-nzLh4?B}HV;rv4xU0ZnaE0)qbxh~P7cBKO4-6+74@1nh z^~|s(Ld^9P}rNI+0TYG$Q4(jps>t`xpe|K^|JP)a4^xGfZQ9}LMX3$|8*Z-TC*8-YPI^DIa z{(>LqDwiBOL{4o>@OGN+nxjk7w(%VMsnI8#VD>j*C?6C&Hw9>$IVNNOT=oelH8o>6 zy5Cl@GVTJze275uUC(0Ze-4CsZm_z+WsDY0kQgfJ?97?)FaaG&#j#agTskGtok#7} zw_N?Y2}yEKx8ATC@wSf4TaUWA&y8|9&(r@Ez^!Uo>wGCT)t^4CVv2T7r-Z_jS zOQ?rW0cS>TcL2%r!X1FKuNJOnm(@=hN?66n>=Gdm_}I`l0icyQ#|b#~UynoLBM3N- zk-a=#OmqfH7mfIfx9yb<@{$uuMp*z5p!O@@4Z!m%^V22vha=J~)?6${DREz)tf3tq zx!=+@>qGJ`smMizpUh=z#F~M>f0Uf|Y!cJwJ=+gK5)fKT8wMXUVuFV*O)T*5{yN?I z1b_K=!=adQaWUH*m_T=Mr=~`Z7v$IV_t&8H@IPlZhXIYx_OCC`e+PZx#{5G({lV4N z;&n8uPknokdXVDUf{EOmSE-`*r!-Txy;E;tESk(6bYvg)B;UEe^zQO(!wBfa+EBwx zM4H57j;A5LW}wHIQkb0H=&t;On>*_d^^X=7)IO;qT5lEHdvV62;PJ3lYIk3m*@Al> z$1bfD)Dwk~L9o}9?N(^uk|$-!RERe#^}%`J<|cDcg*o}8V%g@jE5t}6J}K88mXKGn za29!Eq$dpJ_l(3s5_V);1DI>R^qytjfe^lGKl&iYyoox5zbSe4q>>TVOPvD?|&$Y>)??A#=iUf)e5YGF{o_nz>p!}-VgiSJ&s(j~KB{Apd1kbdJ6 zzb3LWIXb~1rLm8kF;RgME9 zy`4_EQr8olYTW=_W;m7hbJ3kSlaQoZ6*7()JQo1a8pF4l5_m`Fl$ni%7WJ;V5txNu zLmE{g8Boc)ewR0J##oD^aC%-au|*wy$~i^y;>?let>IvKtu@74E8>ebgpz5O`V1uO zvZwdOE6is#mRL4c$EG_?nS(@}1?|+|mj-r|9~1AJFT3h|%QoPDpVE{r_$6My)w{JX zdW4xBZ9(0?|G2e&EV|2VGBe*)6C)-& z`n(^mBf%#$%L&zpl|EI96|DAbjrmfY)_eWEmjIq(3MAE|#??H~tA99kT2*3@p)xRV zF0DJVWMKu*sd(hFt#r+SmuUTsi_xcgZUmP40goUpQ48fK;Pm@7<^4kCF7pAwY9#cZ z>yvt9(u7$*ADZ;m@QeX@hIF|7%}Wb>jSNX^qI_DEs1yk`+5{@Bd;F)=e*lw(9f14^ z$>{Eg#Gn)!v^^fH{6RZh6^LJ zO!{{J=+~_T(@>ICSTSWvs_2dEwaP;YKT*V`vFf7Xs>jA|Lch}vn0Da<>Av3EqB79q^77-*sMvQ7kt16 z=Dw<-jNOMpgw&t*HVvIm&c5ww)=NuB7zaXh4u{-XqU#CY#g^LUz6KxORAKhqwU?OS z-R}6YYxm}eng_~c31vM|<%|FgJli07$)$3l_|6*aUK7iF6MVyz%|B}oT~)X*5*Z39 z6tPy|V^o(&USlFgV|Klv;jkwtxsGGNL`aAZb1K$`gaHe3)i{4GG~=?GF9?}4#yYJB zFIjwL?fc9{p;5*SQc?fXuQ*MY%9u7DlMbauL6Ke_`&o-2?mUmMQwmB5eTq*xDl}*N zaOUpu@{0{;Im@;Rddf*9c7WBv@~}a%*a5{j6KDfmrYGgpHLifku_Z35D(c z^^0mrZcU$KL>QF=!`XLpH|_xBwjY!psvyb5CjT>>N2aBhmshj=#KvAmcW6@iVA&l` zZrtnUu3RQ0icI2XmEz@&pK6>h2Xj~b4Sawoc>n@K2CodLV8t3vQ`)#RZk$tp zitQHS`x1t2g=XH(S7b26nj_XDYcb>bOIH$QOq+J(PM|>(QX!isZ>oxNT*j2fEtsrhDNYYI6?uo<+`k-rr_fvi?ApI{OEz zb9fln{A#GyTPecG#idEto_2F6R$Ax)_bxL4cS7kKJ6Vd9rZDb8djV2unb`Kzok1pV z#AV{wwYL0I|JUDmRY`c?s53ra?nM7{Vz!Uj7&*%ZF%D;TLlzY$A@ogz9(OVI^Xr5Z zW|c}mprPMS(w(bm^&w8)Z$#2rq6qbQgFXeQgOVmwsuOAan(GRlGaoysF*#S>XMdJq zvR@jFdqsAgM8BU_Pn^HZmhWlL9cx9UjEYr-xo5`J5(_2y8`a}$sGoZtqbc5WX$_j1 z7?K(%bw1&$q|lg`R3=-HYVAHULpfT#B?fwDd8*t;B_+9!6`qDvOO)QBOS!&KieYdQ zN5$o3{t==dYeSm_0w9=OjW8^3Ojwz6MR{EIBnLRrQ+&Qh_Mos?@I?r7%cfgej|mE@&)~dTW+lnACku6#*!%|E!0`$5?f$Co@Zgx< zFLcbQs&PIY0lfwsg9umJ(--V~AJ|BDeWdZI2{D%Pe+1fhXXhGakxzZ#@-(HN36-s~ z+!V)KV|;5o{0yx`#>f)gZl=%GKxxECrOrFf>R7;Eyx`9CFeA>w^8_W?10O)L6bN55 z6;eDnGmSd$HfV|KahjOtlt(CTa|5J z=V!m=pYh9NYYnwMx2K|>0ui>A^Jb;K<^tEOeJ4a=*~#e#UZ)?}M#E(i};$W1QOy#okKHc@_$ylzg zIOYeEB{=Bw%_rc2=G-Ye0A-r!`MMGF6|@mqBm1k(G?n=Tn-@zGE_XzJ=V4Zg6&8|n zT=ZMq2A?t$WbGf4Sgg{1RD4Z~YZYc_aViYyu~e`IlNt*D)JStJs7xjg2`01q+?Nr* z>3Qj%4jP$QI4YLe^=yY5kHn7u5KTm3rrM(xuwo-;&Mwtp^vk7GY?#^Bfsz3~(qy;8 z;hu!pEt1zKsNF4${RLZLJVqD#mw3c--xfUW8Ex83_F-6)N&zYkVNijSd5hC|V3aKT zOB{qNLd@+_g&gx`Dwe@E-Hkg zi)2xodOTs(Vie<~5=Kku*W_;q))kK#c}nk_g^*Z`g;6nr!Xxxp2heF!gWiB~nF0_lwy7Oz!o@YY$Wj2g*94vUzmPqb=!(Z(o!) zTLiPrfGR%lJz3xoNn;Dfz3`T+w)yu2f|BHG@(I4OOncXZOF*A5^P3!Lyulp1EjM*` zXZVy(Z#d+o53{Ocn+a3j-^al?O>of*+SU3U8e7PCI4B0Zry$H8l14XD6(2hDKZFDO zbmdbu=@+D3lix=N(s8@q4&ayBkG&f$h2y`tF9E(q%zn{7sD6B+TPE*eNWs8QH_tJN zE#3q9`-X&n+2_;-$Ik?qNLEJ^md&d+L_0LjBv+JMXEYYO?wV9+=>WCa66n3_{Xxp( zx;r5rGq^uPKV^xZjxXD|>ERSiO(O=DE|$<8hMXCP`z5-XT_y3yiHN!CO{(B@IusBw zNxs7@)R2nFrd4rG6`tAU>XRZ%tP;!_Y-;E+WFZg?CDs;b@>A`1Co-Q}uu+7TP26U} zBVr3zWW64~Yoe2X@kU%=%XDFJ=+SQ^%6d<6VQ$qFo4)Vq{u-$lWyCQ)uB0uHH|BBh z%I?6ez_7}uAsQ^ddj9ue#-EHjPW5?uP*%rP32j-kYar-^lj<4taj76ZF%fF!Lrz4O zcK(C~s&ajkGy#yJu~tT7Cd!B6e6c`Oebx|wL%Vn7vS|apgkWDYWsiqc%F?D)ke%g| zNnd1W@=!-PHRlGkGJ|GGaD(2|3KcbiQ$iRKv?W(-4Ae{yBPoXc8gA}`ZW|K+hwL8x z(fc}a8RAw6{k9`*;uNuZ;D62nSgl!sqy;}@n~R~ z(bt0PqF`;Mn{O8+Y{*{O!xMOH=cxU06pMGlYqk0=+?`5JbHUcZOC9x=y63v=U=d<& zFY6mQ*7Ww+DA(k*9QJsKU?cOzP%_-7DPu&Xn8j!}IB|9kDn@>Klpc)r#8fr`9Zkoc zQd5%21fC;7i7ef~JBqf0iR!9X*jC%;iLQlr0=6_+t3?;5ksyz{f(Lz7c$lS&LqWFH z2C}T19&=Sk7t0~2G47GAEF=4eqFB_O*lJy>FV>GIC-Wu$A9-&T6xZ7I`-YH^gdjnJ zOYjif9Rk5Mf#B}$u8k918rKjkxJv_#HSX@vK;!N%yR+7MzxR9huCGp=i(Tj9%!}@M zF>BVW$HsWZ_&wwARa3ie74$9|_pji;Xy13NPb(|aelAQ?U3Qd#*IF-oZfi$1c79Kk zks^B3b?@Z2vzQDM(aSWBYYWyUJ@kD-yC7IIEB;wJlY($`Jzf*$AOl(~)Y59m<_YMn zcpVXyu?^esC9`0v#I;p0k&gOsUlSH^^-(b8l9m*Fi1voeDomb&!-DjLn&AMN(NSWa z#oG2oX7|xrl)Av4o!=iNk`Ix_yT3I*;ukm|^vBZyd0$Cap0Z~%I8|vt2O%;O$$B2P zl+QTd#0?-L;dbQxwpuV&yC)%`nr9a{yn;VSwD|nMdcu?N$J@7?UklOYZSx=xwfOXz zLaCe_?|+Z`>wuqvsSjUExoY&QkHN-zx% zxFx%xJhKoIUuX)M!*?N?C{0TJRWQ(7cn`bEyBG3wCB;=vc>>&DXse@z55*7~;>+iM zN&HQfOBfs^{_QJHO4)UF4STKZpeOB8FX5*n1`QdFr4Mos^VBANsv7J=-?ZM-$HXOb z689Qdp-P@rXtWBod486a9lSa_Sy>HmEOX4TjM~zg=}H-K)&?p@Bbh#nk4>13oJRB9 zP#7(Zkf)=xJVab%`L($cVK`U!nxdHl8=&JhuTf2I>a@If$coIxOKV98@g`G2v3%`% z@fvZ4Gfdf4hS;5Jr&k+eMxUkRMUX4R{;9uC@S2W_Ibr4X6>9+_4cRl`2!j39fr2`{x#|ZDm@xTy>{O5aWirl9db7 zd|>6(qEMlh|0Unep6(`A7IQ$2j&C{oMhei~_6Tl5?+qmm59gzP`WmHsN84VO6nQ5` zOb}^L@VA_*I@*%p4^T7f+h<-z8)-deq{>5WV{x=a++%?pN47cEM_WIjyCx1AWa=sM(!YM$ zmzHWQcV4+=Zty84vmZ%gVydj&8!A~vx>VWSV>*P%98V@6Dbag*a)3>!t8Z=4#cooJ z0E%G`CCd>h4b#oTzb;{JM1(%On9yb0cgm`ZQ(v-HCXDOd&kQtmCaN?XcQfrU z!vnTGS)=A!St2`jg9mSSO>n;g~8NeZNA+L ztw@FfMwc5x#~X%-RH0}IFQE^ICtI`E2SJQ6l1oG%nZToBf24ePOzV2_bGzHq+YLu< z!%1bf?C%dipO^7*dK{8kFyF(UxZ#+(jJlO=xw(a2s9|EnhC`H9>r0#g7)VbytM7hN z$?qm5MfR$v+2y0PuX^6L97|NO_Mzg-@e@bQpH}KSGhM#K@x+&J^~g zY5t1|C}sc)-$3C}-@Nt4MI&k#K5w}OHs5RN5e06z=!184l{&FEbx5XU#hQF)C%1Eq(Tok7JhX`uNI$LP1Xf z^KN$92b!QcnYzi5n%KWeygKNp^F~uJI8srh@uVa*<0v5dDF0LY)uC>fb$zA2KTzCn zk$5a|12h*;y0q^}ob6ik6x99-NsjMgD^$nRu3RVN?q?_sH1t*5-a7uL!8NO@u#vp` z);RdITE-j9(sfYOuZ5W{M2#k|@?UgIi(uZd+;Y{R<=9)lKu6#2-c(SH$-B4`WfXkh zl`F86Qu9_D?N7VK$|Tbm(tpD@b(Ou@+!1{#CnWR|^REWQ0JSlO;j@=nPGowL?km9_ zhqcMK7pc~8w{-OrS8q!KH??!j7uK&`WcF+py{&1>mW;nYlQ5mME0ZH<{rM@@_5-3~ zbzl1XfCIILa9q_c%2-Z1WY0UYT$=?^5u-GiOnQ3FeP5_dx>{vV{)$%GVDGh2VyT9N zi3+E=lzoq>gBhj>6trVC;#w^L*H3AAkQ(fzcK*tTqREBH=2At^xUTj!l)>QBjyj!t z)t3z%2u4@$%y{3Ry;4jo@H_}>XL;UZdS;N`5!lL;5rlQiVxdmEYHtXFesf2Yxl_YT zvNpJ58}1JEJb>EJ_EY}**>3&<-=gGeZJ+MA%(*{b$THdK942s z4xvk2lQnTU+OGH&OJ8oDX|?wPZyco$T_1&iqIC~5cHqd58zvUHTzqOVoi-@^LT>7<}4YVJ#k zRBTnAE4H?q|5nWUt4ny87{i{SKGCH**ve*E1hDhEhOWd4D^wpf#X0hZY$?%*J?cB5 z@)DHjQB~|^T;rBg?d?@(;I1)4b0b$zW==mcB`dT>)w7x(wD6KxtWB(;k*Aex&Mhzf z@G*PNw;(9&1+7!7z~pC+y=Ht*Pf98_SiwbmaQgA8hIvJj_>}WG2p~(2&gINn3M~~y zO_#4LMIRBG3HbnJBh~SC7ceUKO7hAPSZ$YMv0B38@cyJd#Gbk1PWLT9hx(~VEfZkF zEUZ%D2x|8Y1Te)vzV9ls4LU_9Z+ic;*nI2a>JfHd`=Hq8wE~BgXf_sn!Oz0o)fz}0s4qNaWzrfO)K^W+x-SZVcP#J z#Y|oc`k2T)*USPGn*S8Q{!CD<@&z|Z`pGmQ6TF}*Xv-<;%b~2`r-lF>SLmQW!``vl z^Bs>g)R`w9d)FxH^K_vu_4qt1nj;3O9_M%5r!Q}UsIwW$RvieP4jfRa_IfHl2%wMX ztJQprxPw?`YOhn8kR8cAvdt}1HMk@p$f!5)hm}P`BXUCh_HYyHJ7D?07k|355k6*k z*xf1X9uMp4&`{K$5}n~J3vVa$y|Jx~Pv`qp@np)8qd4Ao_4#pdEdIbsdyB((Xep&+ zUhRiDfmhwxTYh8Xk{dlNl8Ib;09@JmBe zPR0czd^uczYD=qBzG;|CISM{>fuG2^$5Ple)~lVXQ2e};iOKJ|WJGyEL$QOa!~FM} z@vmRdx)wG9fgS%xm|09WfS;S5`}e#iys?*(!yq-hll6t^m&tH9Yx zqKtce0I%Sd8x5YaW94jFmHiC@S5OZ*qpb9z?$_y$srjn9kq@7Q(7a*!+n=Z4gP>Tk z8X@2(Z+gKRAG)!LaZ8A78~>^XF~V_s&okRJVofZ<> z&GoC~m{lI@Bn)g72uGRDP1FraxDS?Q8sCp*zSO&(=JwuII=qKoQP!~cx&KW@`Ojz9 z-Yo?&w$0t@bzurhYATMj<0pJ-v9B^ImI}GMo1mKvA;y=NmcPM-vdbN_GXGBX=@^in zpq_R_=KWnA0Ie!VDd7|em##E6%C*7zr9j9hMm%uRw_Kgk^co4hH&R zd29EB9IcizD57@^*3=W3qx#Ac3|HxjCLF;KdQPlLGK7jkhZaISz;xZ5==+&St__$r zyq02rPRJiX1ZD29Em~Nl`p>>`FmKv^&i<2CGN&bg?HRMcx)FEbHnvPdWhBSKVzPw1 z^k=EqG*;-|>o+yek%Y&h?Nu!i3BKJpif9ZXn|V>ynYdV~1UR}>#0+W5n2`wh`^Ear zrUs2#>Y<<=<=nOA*_DSLbYHSxyv$LT@VI(fpXpmTMP5v+)l~Uu5gE=3J|th)8>0xa?@PXEAQ1I%?kc$5DhFTB@y9q-a!L)L_hTnBfyet zcTEzb@ua6gIEQxlJa1`O%e%glC}{LN<2TA{8Q8S-T!D}=zje1AcJil2qq%t?`%If< z?Xrek0fW8VG%Z%I_!O*uLN`Z(zOfL8cX5y1lFkjhZl!TFI*pniIWq2IF{530m2pfP zopJ^psK@2;gq3tqTa)qoTrL`l1ich3>|@sDvm8|rlasw@E(v$p(N~z@Oc~n?#VHu* z-!IHvwUj)HZ2ccxXnb}?R-|@?{9}tO)=gAW$l3^sKj}VXNC!0NlAyKS5!x#znPD6o zyz|+8Cy3GizxD4V+eP}2bM}9?!{2wq@UQ;+HUIuiH${dkc>m|cGlC7*Oa9NxA(QqqT%p54f0WgN8S%(6 zpsV=Oi3?48IFXq2L?j-7_(^r?`qdTL>c!RUaXEioiB4Mu$9j#&gloJifHGMabN1e`n@Ha4E4ndlFSC9?Og^+nsx$I2x@4um{D^saODt2 zWL)!OV=BT)+@)?hu3!2%IU$#(YptP-ac3{KmKdC$2TV&F?nRRrG1ogv8FWcX^T+ZT z?+(=C*|75*q}{2rHc%huCfCGkmA~2WuM!*nAmEJ@J7ghmSo33bDk33b->OvU_LffT zwRb^jXUZNm7hQ&G)_(9o{KB}IP~c=q20bB0hq-GBM__ERycZJOw`HQrwv`0en2xvI z)ETz@lFLQ2aJ>cF51uqGEa$>M>szfem#?3I2l)Uej$%xfoW?kJM{!}avg?5R7GbaN zq=y%^kGapZ%Xw;Z{25fXm&H71QKw9`G@{+6n_0K4q+&@L*=eB!Y zsb5;bnTwMnIN*H4a2`(jMOK3D>2U@pjHxxqFDK-f25{`c-}(C_+F>DqL~!nsO5Bcg z?G;qYw_bRlDHzCnEdRXU_4tGlNJxS=FnXu4daw|@jFr!Pj7E4@#dR#@{tN@Lfcex< zB|(^!{Gx3U9IqeyZE#LYrQrO%=P=^~PbIujYeXBBepmv!q?($&QKhF9sEWWD(^36d z$tiR3s=%SEvOJVMg97KTn&g>oK^<+65nksiU~KRUH`Z><_Z*24;4PX5CX?A8L@ePU zrN#KfSR#Yl?AsSa6y#Pt6Y$`ZqZweHvy3W>f}~0(>=zg!e>dvs=29!Q4peXi(1EcZ zhWL_uIznvopoH$pTNDqV-+70eMChzOqA$ey9zw(Lhdf$824C)~S$mf>YWm!v_^s?ctG`CYnro-0vUqn4mXy#CFa?unsmc5fQi(b$2-9viCtY+9pQJdutz=8GA4w-WpgK6g(0@q?XGby=!o z;jJ6xXq%7)!BML~Fd+jH!x6!k)1smE{n_D=@Pe_XheAUrf`zo{YInni)m&Y03ap@V zJ%`_cvvf6~{c7BE+=0_-z!0+5C#pQ8s1Is+bpp_ZBoez@saGdo!#i194rJ46%6eqG zFxznSo!v~n%TH2KP&M01%yi>Ej!DTom^6H{%7`o=Y#9}?2#gO#B{uzzQVlp_(j zawC@YO}{mR(Yl;^rCZ28z?O&f$(bJ_9=TPnzH#l^7*3d`+m4#St5;cqvj0Bu!jWXF@a4is9YXODT{@xUs=- z^Xgx8#Wj?iE)UBp=o*ZeZL^&bY zl$TaWOUdN-VWOc`ESscmQ@kp!%2E@dO5A9*_D0fYI61<{6I4^1W@ob~s-Z}fJygPN z0PNG^NI1^1N0saNp|Bw8aD+p2Bu~g~kRO-&O}V^)>TG??13d3lY_a=O4RfN(-bPJA z2o$TEctAvbJahq-h-7=xBo!ypD{%-y1Hv-SmZM z%+JJ^J&$^(gRqEWaoiU|`0)~Lr7j!jz8*$gvivn7&l)}R769mY>es|$cYh8p=jxO7 z2{O*QJ6lP9b6&TRrBWvF3hvCC;rqPhhEm=Z`<`EoBpgsom?!le)`0Z-89ff;G45ErM1-Y z+_=l@LHp(V4KX-+2)a+@Z9-eJagaiDz6vP?2ZUP;YQ_)qG3uJDyjG#bpp0 zM|N;d6N_12v{PgxSwmuyz*GU39e;AKBYG;mS2N)ldL$K%-Rn2#SuHQ5N@dZ@o-1$& z;~|L~3Z){Uliff>kWXFXGWnw%r6ZRI-!q& zsbj38vsKk5QcWp~$iQQi5d}JU5-Zb{oy${^?X=Wak)#qv97?@2z2`C29<=y32}~!E z)CXn@gSmD|gutR$EAKKg%XUR%^y!N;^*6}i_X~hJAoxx-c~zVsKK$3=WtBnGB|&qV z_=(7^JhIFCxD?*l!%}#rWXXM@tX+#GVtMR}*{UyE6pd75hOU&mQD|tij5<7I2|^4m zh;ckHL;)WWNY|BZRCnu(BB_cisM)-xn7w_uSAFGB!)Lw2oE*4%(3f)l48wI15*S^! zzIp3PbFwwQ6_h7UL1&NumETm+zm@^Z=(X)@xj1Wt}0eTQajgWUh zi{8nvTDReI*T!4Rq#C3t-5iIyJU=d+3bQelC}(d- ztl)13xMK!l{>vMiDxU4x3KEM~^}QJ~wHjjVzAUh_IcA_avkh%-)Dr}FJ~^7X6V})B zpD0M%YPbk0@d6bduil%CNEvjQQh)avuF6e;r!%NE(fw=H9m^~nX|#!ux=33(_ScP; z)lSK}T7(y;+oPEjKFuU~sdAv&-4xienl7j1+V-2A1rzl%2s>He`flv%@y?-!aoPT? zP>1gqns!Fh2u^L8yt4&MMxtRy(BBvH*@b ze|F@q{dgns>AJEbP-wQoO%=?4dRZ`_G~u4DO83cyaW(Rs$&v(2;Slxdax1oWD$;$n z^URpcIG9u=t6J99T3S+#uYacC7@(V)oM}z3d-L*9yQAVRGc~|3FNJu)e8JMxXDh#= zt?!rVA(_Prc2(IAV=t}AO6P0^ClE!VDwj%Gcv=%@${j$`a5dym*Lq&f{G+tJt#j3c zp+TaL-vFYDrnPRqa@MlL7rg9gt!2$h)M)dyep&s1xu`O->51y&T76rSQtz64h6n_u zZjAGtf##NFCAd7r%DeePuZ)+PBh2TZ*vQdqCkJgCWS0Y@xn#>;-Ntx&1!BgK`+O~I zXIBc+)cO00`iAs-_khqHzF^cyPiBMPyEcVQVGp`73A+C74H5Sjdv^AyenV{u^H~)O zud4Z5Q5YfpH7qYFSP+@O+R)}?u z*9Q4feGEfDcj>ga*83_x{j0UQ`T=F!acMn&r0rzK3nw!>qoQU90*s|TqAJfnZi}wf zkr6N)XXY4v)R05r%O2BD#!s*|t)0{benKs3H7QQFE;Q@&Vr?lj0RWjXRg8h0Tv41s zXDLlZ74w(E)^JJi-k^d%JDS2Y@ntpZM>j(g7xL<;%gFp5!79g%6?fA0RK@5o1 z-F**t{rl&xyz$>BeL?kVWxM8(hXl%SVxpdeIt-c{>`_&=Ur(GQba(^?gh6vWS zh2!p@r;A8mAH83`@?l0mv}HnUZad~&DPZ;a$oF|i7^@kznGk=pw~18`*2CIQCwq+c z!C>8Ta2J4;iEhfey+g_AQ!3=V!Otoujh^KeeKWZ{P|Wyn^l)rDAOF)wX0ByG&aXJ; zJdm$~t0{P5xqRNnc(NmbQV`D#he>Rp*WHjA{7TlSvijZ3X5OIYO5h&EOnHcJ#&B;< z2b%aJq^HUZz=c_WO5KoJb4uufH;5=if^Nr$608~tW3iOARkNDaSQHi!CwmEx(yPnt z5E}KkOeGX7LXe^|+0;|{%erIj#Xobc>1MSr-EFbm=CLnQ&o=T8wXKNY(5r=S$;zaq z*W|lnLri!pQ8U{e$4uc!mDW0AaN}JsnKU{{2l&x>Gmoban%Lb@pT4p!=?fpf#fVf1 z-los0;#J)bSLC<$6+%^5G&)p<~K2e#ulu#x^~C+^M5Det|AEYZ^zn+p`Bx zp7en8fPlQ7q&J2AGv#$&-d+{gM8EDoD~pHFeSBDbH&t2V?ys8k5Q%Ug@2w&0kSK4U zBmd#VobRTU79)PkD#z%l-pvPX{gFihYdeBs^YR@zNf-s49T8w>CMuDm7OUl;0boWi zh|g4gU(FUhI0N5FDI)HdPj%ZZ3SY^;H}a0xJaf?AE${n=n%BB%OvTL^)q9#pucqz2 z^71Z2c9%qC5U4b#4@gzVGAKg*t*i;(8#}CK@Zx-iXRe)e+G3IW`HjD?jBL?{=8|X^ zT>OwaDG*xmh8Ka&+D$(EUeJtpv-%3%IHcVPzT93Fn`Ci6H}*WG<+9<`DFdGV=1x=) zJHg@H+=tbr+0b-wy@oBIS}opibU~w5E)#mtD{Dw)-GlsE!92lNNm^s9&jK7Sy_=H2 z5@8>&kT-WtRGPur8yqU!+rPGjp!z;aXx4}gx5&25gdAJ*bUzJjG49Yb8nQ=>FmG$x zs~!05Cj?iZ&KbKp>mt00{VkYWJmry_C-AM+Oky2<^ahh$+wGKkMAZpvh3oc@r)BGP zY*pf^*Pn*tA*-y5X-TX{uL!dNVBn}q-}?U7Hefn&_2QRUx{M^Gu1X76>#Jy~v0EeR z14d(0XXkqOD*1-O>=6KQTR)mpX1XZl7J-lf z<;c=L$29O<*do-7Hb6wFm+QbO3lx$4EIHn4$S7on`!MPeXYajk#ixPDQ1BAJW0iL`cd|mc+%7MHKhydgd{GXBxNC#Mz5=;!Ib?@rKN2T164JctVRhtuJl)+B9-qi&2p=ryT18ImLDhH|s z2_hk};}E|WkFCptbKky<24}L|pcd6XYLud+Lh*tPTY)pkyLDQjx0VjWyE(0!-l*By zd~4h9#5E2lVGFn>u;g*z92J+|6E9GikAK~C(6nqUiYvq=H-Q7yRgSIoZ~s7#J>bs< zevrQB#XX{Jfy|bQTb+0B)+^MqmKGiyCb}7yoiD8<_3!M)SCWw*!63#R3)i1|h=i@I zC#{S>ioH?kXO_fTdk$gU)ZMfj77*j9AVQUW!@XOEcx`VQFy!IH$aV#IB(I?IIE9n0 zn|v*SvIUY-a@)-2{CZ7B=g^YReP<}hRByKc!*bIf$G&~3`EmIgL9_KXRUt!jc55m4 z5te^^AGo@>sXLr_^(Zx#;CgaRQLm>@s>{_Fv`j(sPDuYb#AC_-N!`mQR##i>&D=0k z1^la3L%#8dn6`b#d<75aSbNx_-|S8F65vqbtSI0;jl`Z89dEtjZPQ@uNW46s&VHjT~em*2l>Oc*l{G8*#s(!=t$cK4i?X%K=u^u`J>mbX5e~sYWx~S?8!Z*H6Z}0;3k++Y{qJq`$gx5rrW4 zLZA3jb@wDv=x|-})#gPf1R7|UmzP&wZVJ+nZhUro5`58+VT7CEhOYdzW~#5HI0wWx zi=@NppOa(1?-~a?DP)0$or<_5USl{&NbCpu3Dmq=ovJ~6w$DUK2MOJ)xm#?*NR#!L zNAakw*Ec5~f?zDeL-VwcF6dQ_-P2@)t63ar-sln1$8b^u*=%X2|uQlP~FjH zNInpl)xF(hV1YKwt%5CJc)Fm-^ia#jTA^(lQvP<<6{n}Dm3k{z@YeFxncG2j8_ZhD zObBOdv(oMJ>xV?I5CH@36GMYHwfHN9k|ComE06xa5KTX(&BN2-v2R7#;BiOE$WJZH z9bxE#uwb1lR*Q-drYz~Tn-_E=C`Sq%;YeOjZ)~Yr26}YU(z#j0-EE{Z#;10hqHiok z$i?sNkQ5rRY%fpjPj%I9`UF2U%QMa_PwuiZ!-uBcUi}rOvoVY2%#oh1Gm~P!5Sj6& zY*`ekh-fDRwypf;9O!JW5&inn+hoM5iwzD$VKhdU4)m~jF2BN7eP+3JrWhtf;khU! zdP7qpPt>#jUiVbU`mi|lbRkRr>f-0bm?fw-8C`HzvGzr{xXfU_E18)DKcb=>)m{?Fz)fu_kq+r}fK^!Om^@CDu86$- zn@oZ;@30`-^8A@WBNgaD_D3P$zJhdR)>Ik`)=cV*`lJt$Laqr=k_`V$oXr#b3cl?f zZk`I1m+6|D30P1HqjTg1;$-UAoU6lQoUP^;WB&ukV`r#WefzDlHU7v_Gg1D5qfix& z{1o$ea^aulzhm{LE>wpV`ST~T^4QasS(v1J``b)EL}z6|;q=Zu+cTkub1WhD)|=U3 zA)U)S6JilDfPs4X^$~^q6^ox@ z??LfA|Ku5nFFeY+6P0kA?ju3>IO0QDJrpgZ@2;*Lt?n;Ha3ynX(BhMHw#om5+0)rH z*HIK4;_^LiZs9T zk4CbA2bPF>ip}TXl+Ldz@#|P zkX(CNh7n_$Fm#wRO5xH}?`=4RdrsP-GVFC72~&~1G9K8&#R%%JcIXbTBfu$gH}rwS zaI3P4^0JgSi6ijZiR)$A1R&kfutIbHTzc94_AydQ6lwB^3WtYATw*?ABu8fElAYIa z7sj|J*)~*va#zj!nL%^hFm>r|ISfH$_h{HeJ#HNzd zgqlfIXora}l@)z;{zbvfa1#EsK$w1?JNpqI5Em&>uU-0hMZI|zaXHNnT|avXFHuqC zx|32oXrb-(`ePq}qNTX+4oLDL|LDcC&^zJ#nynvJ_D&Ctl#03$-Q~@xwbGGvmACPV z-@p3yuvbV#QP^9(2vJe^scVFtlax5>EIS42uX3l|z2^O0O6)Q(5lID6mYkMYpye^p zJK?ZrIP4jk$%-kHA;RyvaCH}BjEB~54_wr{bZ+5J$8C8ur&HJhcm)K5zM(I@;GrGV zsmk-okHr3yt%*JBA9p7*Pocw~8|-UduXV%D`WLfBW?DPG*>xE3UglW?SN@1pYgYKR zF&+LW!E25F%R_V7)Dy&525Gg&b2_8-La~*SJ zI_n2__OUcYz$hwwSC#cCC$MqE5RlqX1IlsInJJGBuyr z;4EnoystU>;sQS7_e5Ljkpp`;1RcD~M)dBLPZ{#t%gi<$^j1T?E?m#@I(2#ODL~y? zkN#nMIa6z_)Az||;7U{1aI4JH3^0Debh;FcdkfJq_!K7P*~n&&+eq}+AWULb=vMp1 zI((RXIy4cAoa*C){Et$)gP*p8VUv(rWgz@epdB11@<@?_3(lfv$6n{QbW2dCgCk7Fw_T!~tybI!tr)aLzG3&4lh&rOkLS7(b0#55jIe|aI=g7 zfsFa#1;3YAe-|u1TgiS2nZFLKY4H=ijOtDWc6iPy%M@Izo!D6A6@KeTRfWT_n~E+3 zt})D9u{vlu^Gadt>=KKPYJ7qb`wCZf>#d&Gj{{43*8N8cMfj**8WMU-S7^FEb+4lK zvyCSBh#Gd~%gpYvBp%;R6W%!-T+EDZheXq=VJlQ_uKZy*|LclX(o%#od)q>palpq1 z4kF$YTc$!H@!*Zpm~c|M zC!sN}2cRFa!h_LaSCtA(<6rTwo!k=BZ0-{kbdZgXe9VX*IakRF^7%Retxv(H3bRWGBDx87{MOGi*6TH2oCLJp}d%2jotYzC4Z?prOsmk76|X>hbe+pz6X z_PJNf5yPf19n$+Ov3FuESJrM5R29ei>7Dtu*Ri<=RV&?4-u^2r#yhN;z{MZY(F(>x z*I4(5SAxEM+5WOVW*(#rnsd*tA1)}~?2ILK1I(gfp;J!HH;w&HU7EB{+3`=QLYa2W z|JA>LZhEOW1G;Zehkt+1JY&RfB3)no2AOLXdYQx!j1m1pF^(sx#Gpa~&D5xqEHOG$ zzWY#VGx_6y;ls4zVW^M)-Au$Eh(uuRo5#}Ytf9f2PBRJUw=b_}c{AE+=i;+2s;CRh z#zR{SZzrxigmgTP-f2rvjt{8xrlUJVZ&}x3%@T#xH`4O0dYP!>YY`49z0n(%B`ao+ z(J>5u=Viw*!n>3z@P!}|%8{pTMJ-?uZOdEn$ih0)3USRIvFmX}AU88q40GqwEQR2Onr@3_P&@RrcI??E9?ORu`KBP$oUeRkHe2O%tcIFa@79V-@CF{X0+idh~7LL~g!ve0*$ z)Xho*bMp-vBTvljg00jFVbwPr=eE7g5#vINAZmwTkj)anv!QkVASOA}DUj?Q8Tr51 zo46PztsH~0w;1b_QC@}l+ePg;+M z&lKyYy%d8z+hC8RAJIn!v!=WG}A90x8doiUt(z1S#LpF6+ZmR`{o{FOGm!8S!Fcj;_z6T zLIw1&3&?Xwavk8qzi?3YdD0plY_G>y@jl$Z=IJ-X7%0Zy`Gd+(B_yn{&`BwTb=hlc zkIzt8@_ER~&~R~O#1cnkPP8qvr5wQH%{2G9k#E`^`d2TLmBy_XI;(Q+MeCS-GEN6x z8i%I~d)&gu<5YUL83v>nAMU@-X%w7as5ZR2mDdbqn~}3o;Vl=OLtUV)&l~f3U8v5d z&Pg6Paj0hbnnE?>8U=ezR##dC8slL;lIwS7-Qz2kCP_SbcHhB?=RrQn~HW3gxR z4AJvkG|`$q_k&PcO|E^QsuF4OVdB{^R}4bo#kK7H_$1_K4Fjm+odF<0-nsTFv9{hEn1C8>Ki%fZMb_Mh(<|~S^q%?N+>ug zg@ongg-YRhKS=^704k!V8&M(QTl?Y2b{it-;g1x_(;_Pe*Y{Nd0zt7DE*CTFM*k0V z&R++ja)tjIX8+f)9BWW$JFwtHvLRs&V7+ois)@nkgiO&?FTz5gs5`hMgO7vXl|E$b zEUjC)wLhMPWyyyGuLi~!)NrYqUNf*T^V8_z`>**0^Ml{_p(!bH?i_PoM;iSVj4m3l zYN^&jRX?KakMv5|PRWeP`2IN~@y)Pr*3)PF$sjng7o*y3wlQ^#g{(6|K0s%Lg}lRs zKhpMtK*(i*HB94_XN_zx@8dk*ehoS}#8fp+RPcuVs(xH%4B8yvPVaTCatn?PaTgrV z3fI`Rft-ARNr--pU(EWQJXUfQZeUg(H8tG>4-(gpeE5AOD(=MLZt8{2qA*1ugAF(exj)v@qt=e+w(5h1grAiO9}qx_jxri8buDZ-s=x+FO}h z!qeQNohWYo#;j~O195``ckqEorR|q9aD~~Ypp-8QFvgFoqoJM}{Jp)aW5+r~EZR*c z*S}cFY3_sWh^NlmCv}^gIMyFE-owo_BGR(o#qk(}ivOLw(NBsbQ1p`Y>=&h83zon@ zKMNW7QbSPm{|j;_DPT)iMlI%;JwO7*_&go`V|NzebL-YMGvu36K6@lo&Cc)R<4TfV z)>P*BX?A7WQU?2dIj+s+g@zswariO)9YW4nkUrKxzOvA6zRG5ON!A&+0>i9$-zv~m z{Q-sht8`avbzeEkRg^4imCW(w73@^pRGkF0O+=_a_bCS9Emx(L4Q8SFUhiVEoX075 zM;dcxkYoL7(8IK zXP1?h2|rlLwR&4fD_3ek<@0})+BvcPH?{M0*xQD|mM|GnPO- z#nwi+_%G_}3|U&KJo}tB>2<;_%hT?M+W3Cm7Fdhp)W-= z*U)sT&tw6$5!e8R*6%W8+%H4ll+0`QhmwVCw(~EiaRq^nxaqPW^fWg6+^N$(NN_Nb z1r^ll!k#5TN^v}a&FVsmD$Gi|oZhjBf7aLQ45PKH;wi@lgn%#qNe#cu`71TN{777q z{KWVqh_b^Kfr+D3YB`@A3)E$|W<@GzyTW;sm%IlT)ZPwXH zSzZl(21n0q2zgT^%4XYlQM=UUFINhmu4wMdn}dXbG^2MOz^9Srr5-Wu4FD+dI@c%?@xm0 zq3=7ua4vFE$4ea4!fznrD_rpC-FAakNb21Y{G2o~%6ofh+IN6&f{pjZrL_rUxL>K? zLhIl|r-9-QBjo(<`gXu!(Dxm0h_0Afh>Kb=cz@xj>*(t1;iW$Vio{-B&4J!aTb{d? zOW0MyNf(zM>RXEa1#9{&u5d0@YlziS$o0aVUO6L;t>>IVU8)G9z;tmr8S@=aF!f?T z)O*5;5$(=PRF1CNTo_^=eS^ zohLsk&Lk=-HUkGILs5z>yXJRPgD(s|b0MngV7xDqOiJ|CBkx!m1gyzp2FqrD;EyG% zEZu{~{?0rXFILO+Feb-v9|1bul}5<2fh@;Gk%BTHlubL?cFw%&Wmy6u|6&2UIQYi! zN*gB_CUc$Y34!woR_Jv!uKK)r5(IY-R-)aD5f4z!kF$4J`*svV{Ug5)&?@wgW=UlB z+;sXDrTPvPt8iO;S1hMoWMJz12P~>l`eq#&5$cx=0o{=Xp9u-0+7;M5eEY^(Tv_e5 zosjti;aQP+rP%tY5shxvk44lT9rIL)9~ZXPvG=#!v|?(vB;$H6e%kE{bfPu`X|rlp zSEXRvq!yFFBMv-a41Q)GqOqoeSf{&;2yHE4x26@+hll2nR;r2w;st=W)QJmvsqMB{ z(+B4H3v%9?ur&iNL0ek&Bjc~zg@q_}QA_3_X=`A?{8FyA&5qM)Ry+>xl2e@%^}^$bmPxB%bV63j z3bgAKEI#l*K?^vobxg@f?axXo&Ex+V=OYj>=O_)jJic({0NvKL zLiWy}WR8XiY16oC)#Bq#o_N!B-_FsvK0<3U$46hK@H-=w-|HjwQP}2uiSa|YmJ_1Z zOQojP1D_&m=-51?ezd?UYA@kjWhGRd)o=iHJ!=hpN`BLG@3qIZ>cFQWbI_l)43m7) zBOB>nm`xA_)>jSi3=kFf0tNvUKNk79Olm_-6=-!R@HAI=^6P7Q$kwQCK+gN2zlqZ|P zsgCxpB{Ca_U-0QQ_EdgOP!%A%W&$%vu&Q?*wMZ)F2tATjTs`zyn=@dA>V2FB*h?-gLmkaU!L5pK`B;E# zbE^(YRh~2hG&Kd>0vTEg6puXXzfP;onlxXG&y&cyh0xJv3dMDXP*=2Uct?-+3KqV|H8Dlo=q z5@W`l*L~c**sZgd1|}(7$`q4r)Zw9cvI-M|hlpe<{2E;!DLg;Hp$#u8L zht=}pbCXEJfF(|GuORd8wvz%#6K--A9K|Q80*N76Kk}^Ro>#RL!5&m9DFJGqyVEP3 z7_{5c(Pj4bNhg!wJ9sU(zd+J%bNzM`^mhM*hC<8PX6Ls9-OIjIz1_;C@*0GW>W1St zzwf)lNwJ>3tw_EX6w*s2x(Lw6)55T#p?ubHm;#xxW+wnbMMq50O zv@N%J1#oSemrw@w7~i-h+`;}TpIM!}i{>rau%+7B^Ay(EAgQ6^dGZ_w;x9a8O6Yq; zC=`O8;_FA%=KGA99?5HKBj$=(Swgk~PfY`Fv_lcA;-nNet94UB16me+4LMR&yZ(ls z5to0~pTo7yuVBubFQt6ZH$m%%8hyGe{RC|ftg(?>r`WNYJIL=;mNq&4-0@|>vHQO0 z!6Z|=CM@2QVpv~sQf4(vg4O9doZdNcZ9abZK1jZSF>%;=5cFurc<8+J!}Hobk*GN# z9-kt#h@w=hv#6c_{vh;Oy#@1gg4r{0=6Uc`6d3Y3u3DPb9z@$Gx0T&%x33Bpt|j5L z<7W;(pA5Yyi(u=pF+N>j1Ho3FXL@oV*1x%IQD7r4iVu1_v&yqDMqQ9@PBN0LF27G< zdaBp~A_b8PISPf;-OYY0iX!UDIO9BHOUK?at=QJ4;r77DhY#jGJF%fz{I-%cv)Y%M zGRz`|?hf+Q!v z{x9y{I;hR}Tl+1gMGBPS4#nNw+fv*Lw75G(g1Z*?;tr))DHJX4?!jGxr?^WXBq#K{ zfA87*-FwcQIrE;g=RMCK$;=%fWU`)H)@NPU$_Je}j=1(?#xb}f5--iB8PH`Prhs#w`W`=sl#m??Rg zLH=lpX-mIB{5-3BDtgrJf4-su_|t(+qcCgLvF$PLQ-7V9@y3n!O)5h@a48Ac`IsG3 zc_9d^;S~{nq?55Ynfq}8wA0f;$88hWHa_7NibB0{#Kq>*_)~)zMS=zU&`Nw`c28TfY9dmLy(b~|4y)j$?vw|FGbR#B8Jal4;CL60cO1g;wH=YtyxVm_ATpo zK^l$u9{ALX_58kuErX%1@FY%Wk7(E22>F5qOsmuKofwqPLK@DjUr2aOv>Q+A%Ot=(`hpA_btEuJ zUwFlriY{XI&qh37(0f&9P4pfhl;Tk**01L}mr6b%5q_!s;Y0EpaB|-&F@{*I37Q=b~seulR6`fe2e{N!4u-?lkZDmTOUtD-z=&{D(!g0mB@Jc-Ekjo z>K|89OoIL|!aoE)>rRmPxGXTStbrh+%iyKv^NQ$2DBCDM3`SWUIwT&IW z97a5E$jG`IpxCkoaGt(8(n&V@2mt*!%Y<5k%~~yOR7V31g}nO{tG9OWDmhs?xgU@$ zwOpxvjRdG1u0CYf1q#b}tdXon1M4wYu8yGJ+aGguUnXBZTE9q6hogtQUwH@064@Fu z8mXW~v9GiZ=kJsmXvaPu^z%a#wQ!2b3$XvJ)Eq{^p)(%Fsr0TQvJfFlyvkhm+OBSC zY)^60ZEALwHrNP*DV#eCY}xyeN!^F9Gi#66SB@*~-kb6Q$Ccl0_Mc77?6o=uXfYX# z40rn_6ZsBvt8S!}PJ7V}D?q|Jl5YAW>O}=LZ1#q$BBvn_$xfFTXVs8k_=eG^ zb>dqlGj&*R7N_dg4QK3kk5|cQ&7VDPUfC2w3Gjs^2j*_xq6o!g5{UE|F<5wBX0&vq zQ%1S)4B0zwUb+f2uRZVp5T#4$ACPsWFFL;(nKA>d#uww91rFa|#nNv(qE{eiawWQe zmv;&$Jg|cCyiKs46iA7Uk9sC_#qp*kXZYlp@8EBUK4|hDVv^p|c=%okXb=%)<2!=s z1$Q(n+(PD|@7G3MNQ$gAW!mmP$|9x0o%<{jB(41kulWCzF1qtroz zNdZhfz6KBxMuqP%VqZ+FqkFY^-Dq1yDnN4&irTV9??dG*Fc>gyt)OUZGd{}fT>mkr z%|?j!q;_hEF?1qeAvmNd2R}K-7aPJn;-bDN5G4Bj@Rczm3pLWfTgK4bPVbAWdxb?E z9Hq-n!w@o*4ep!CjyV4(fx$)FmcmXsPI+WPiRwe6=|f*B-OJ}0P93vDyV)3bP}_TV zFc@k;oYiE;A=m50gT^c#X{X8`*y20VLz?XFA`!~)0D=lc9DFy8w0lmx@xJeUU<=y3 zj>_NA9rGfoqLyTm6La6w$j3vqd6-e6n7`04@sC6%lkX)s5+62U< z%W5|lL?8%^m-*s+$Uv;wc?g~9A<8r93EpDSM#+-HQ=Q@Be?oTRZ^&}O+sDeG_g1bX z=d8IDdnV%=7q@;ju|)k?v=D3*-`ZFYjWzJJ9|I#GW@9;F?uRSZ-JQmof=XAqSX_bz zpywTZUNJ20(=y8$YPN30L-fILXRk(8ZYy#JO}nzNm#|)Gxh+j!L+o#zT5C}+d?zPA zCQ$M)0L=Ef68CnO)^dD*6r6N<;VZ^GA(tWhJu3qx;DEyK%7l76UAn4o5|(jOVkX zvp!xZ))CF`rnEkEQw=S_w>l6;MK49X4m>c@v&E8PsNZ~e-F|4}TLmX)pbnUf7DaJo zp3tPp9evDFvzG*QbFMKUj=6sV?8}d21 zg~!K!$EZ8k=6vjBT-Vj|X&sW6^}nqz=63TZOH9(*fJ)mQvb)zWnR#fz7-z(Yg9CrNQnm-+kC3dL~jdo83mrbsY|=rtz@w5H_gWO za`gG>XaP?sjehZMAR2RjkwUgIo9iy@g^rJo7qVla3^M?AG~$fmWq3bob{l1@<>)YK4oL#VRBZbh+- zVcp6Ao`sN9&gQjcl|!Z@VOwm3V3l&Lb2+QbyT`ga%Y%+D!NDDH*!>C8?QzSA(9Cg~ zWGf_^^(nUpp#6gd9!?XTaK(YniBH9<-uQu4-S1rgg&pbLVF|GoV~LKJccNxg0g+g! z9Iip{Nx@Yoc8Uy-V(m=H`)?U+lo*xqzOZa6s5T&T(8w@qy^Y&UXJd+14FP=;^3S7}L;3~=o;`CeM$oZ~{(16MdPrSdp z$i_cr!a7VWHNyVhRf1W=@!urS`eGdaXrzTt*r+f4eMQFC0+8{GxqTKbDLDULjIFp-Q!&G{ zLey&5iO_4*a<~;vmKLYE66h7%6C};%72cqx5lIsl7~veu?S)fLOlI^K5j;Wnmm_m_ z`KNhV+5%S+&DyFXf|gH5U^?$N`a_{2-M44Y2}&(ozj z@W;vrP%bv}n#(YVx|^RXjc?`Iyn?-9(bIQOP`MjQe>G+wu`GH|r+x{wW?nEy1J z%H}&bBSqPG7^v}-Vg?FFhU9&eE>DoB<6In!TXm>6FFI3t7t(L*{Bu{r5V^DStWzvo zRDz)?Ksnqbq@M>`_sJaG!n|0Wu*2LFVdu7+vj0dzGEpD>3gjLt4u{$hNk{x;5R;5ifTZkkiOND?Z0Vnng%yQN^I1ahp1J(O7i z?5dM;$5>Ie=_Gwmd!jmb(K{fiItMG3hf_W7)ZnDp%bFMZrhJj(7M)+yh^ee36BwAj zpvqtu=#8M93-40j%zAnYb?@Q0Fm@Q&^ZXeqX=Il?nF@izklwb$81u@SV~-0Vj@P)a%gF^dwKkAzY0vK4R8W8R0#U@mVE)8SyM^ z<3ctRP__u6rt?Rkda!>wuZJbFDiJh;EvVM$hGClqG`5bPMs&2ANdpdFnQ2uC2Vm7Q zj2SO7O5;1|_pBsznEsg|C?DrEeVDT8oj+DLX*-r+fV*L3H4s*Njj1z$W=Rw&cA9s9 zHonOnp*L`@*d|smRN6mJinANrz6 z;9QFFlQiEjNhsl+yRG8FmEaWp;6MpL*>tov$&+)t9}8Ju^jIL@lO zFvFaxYN{5_9`bzFAO5yLZhzCl$HpK&SK2>2$z!5wpiN1}l9V|>S6XIHCi_4$A<=ce zu)WBScACunE?$||ibFbo;XWRP`GnwCmhpferxaB{`3O zEt(U2G)70?9VP^8Q$iLYCZQH!#GSn~crJXQoqfi=NcLh-DMt?eVM|g05UWIE;WgHl z<99y1GU(>zNMNaP8u~V5%Tod+E~JEH%9%n@;G?m?ri7q4Vv7#wy)v-X zUFO>em=v{hUw~Y`)I8vp=OKQeqb|zA zxnD*-+bmKIrdbA%1}(|(>;-Rs=5#n7s~j@`{1mzm_b(0qWB%>H}R!;-XHwwO8B zXoq+!WfsLb0=4Xmmx`J`Xa_{mJxeIEvdA89B`ge2{Ss8)X1wjK+t7}~KkI_-je3no z+ID1Z>N7fYJB5VK<{5oJ-uXa*P^UxU5bE^B3?LwYHx&Cu(@9}4-z)HoMZoq|(ZT2a z4F<-FeMg#QekeL=rGaTCLJ8q15icRrd(BsKO^J+%30_wQa zuzS>haxyWGjx2=^s0}sFUH8y3Qs2-MF1NXRPx7AIhZ)&DW$F{4qM6==y_hJ3yQ1TK zM)8;Uz9Kl>?9ChE2UE=YUk0j8^fCSgC1(d^^)=pp!gCHcXEX5jgs%dAo~BFqVTM+T zIU;F0E$D+P zvs;1Y{h{qV?YAi9CMF@zgNp1WDZLu8+smJq!=NR)Qq*ZY2eb39(jUmB@Emp`EkHc4 znG=4XF!=1N0M;7?L(`7=buMpz(XGJ6LU9V@{BK8T~bWx^7KI50# zDl#*#$y@HfN13fNJ-TG=@#g-P6~tN@@#0gdqT(AaoDGLvNhd~hsp1c>atFUI%+0h2 ztBLQ$YF3L~4Zo*3&t=@Z)+5QVbe=Da#C#zdF7G}qwXyG76J_iEomfBjtVpCG$YLPjo ze?r&%&!g1-+J$Aa3i6(b5K)raLp_vOvc5n80rKEH%gZN8MP|q5>l1tU?!GaT9(5%& z0E)c27>f@dl`li;6xr13$A_2~#?KmG+m0vD#UH%gnDL1q^JLVJu<7YZ>YtRH`N0UX z<@!S7X$%$m?x9M%cP&cN()i`ny-V`$-U~ZG(U?`V9)6d&c|@rTnuvCPYyA5_+SQt% zug}3Zm!@WIr118&Hbu1<*Wp|8HFi5^il34lwI6wBxore8r#-PwkJ!82s^Szj*y#`2 zFkguCpDc>FmryuFjR4gV`Noz)MBbd#IG&_0;y_WmhAtk@qaboQ1NfqZY#LF=cq=Hj zB1t?S2Ccou{<~OnVE$dKSD`6~EbKjC)Qht5K~|;#f3tw*XOKad;tK$Wey5Sd(({}< zC&iK^c;WAfz2e#;-o@*HU7q_ZuX5<Xqq!Qxb>WpAWka(K4-|%Z@Ru267z@AKU*+*9uC4Y|)gw|ejc*~v zSIJ=#gWILNo;%pQa)_M(#TQJ>PQV>E(lj48(m{&W#JMvv*O^w<2Q14v^t8ps?zzMf zc9dIX=zx;YE?BPvJ#D$#kg^-=CY}bUJ)W0l*YCq2`aA*}4n>U~9CI<#%<{ZW-sJyM z{XEyqThI2pS`fJYV2fdy${-bKlJc)$CBfzP9 zT!|(9a)g@eGDHC5;wLFgf&H*o31}WZqC9k*!e!uX6Ie%4Y&XAB?lWZc-Qz4dqrpYwR-jiH^&kl&Wnn^o`^OJI@YMR^b~3Q z04IFCWn(6n#Mg}wBISrC*1Fx)twS9t>P;#w9l3^Um{M+-Q}~5Pb>Q- z`+o?6SasbPn+xRM*5ftUYy$oA$hE{hN=$kvEeC}BT0CmMOqpU|H`1ZFa#hLn^Fr4Q z7h>LEwr+DkSVetC>0H9IC^C4R?2J0UXK!J(25sf@Nen44dU2KK+@Lh=U~Ov+XkH-p zr(*omXS&Gc;$aD5!a+6kE|6~(?r*mEOJ~dbZLhT{{b^EbEHDP|e#YJC^(8`0kVstm zr|MjhTqM|j)dqYpLYx?#H~t!6k(Pu$&k}!zp}ARHVRA9Y&PL{gocfwft4_qme$*mX z9$auv#_6vFl?|f(bPEl%ah|~rGjWuz%K7>Pt;Oo4?M1XU{JOxkMjO3)o)=vM%g}=) z#s5jt`4{*`UW#V(^}c-C0iaOhjj$Kc^rz@0^+v7{4i%ggcZ9Vg3{)^$GIR&MM;5oHVJmo{tpVU8cr1CT}nll{@oWdT3y{F;#k{c+ub+_Dlqa?9cgXZfHf|H>l?5SDiMrs`zO29_Z$m>Q=Y4FgERn>nKyYmVz>5^vh%|4nTM)pCK^~97ZAyC46ELjQQG#1cday_w%^nhi9te6O0R*E zIjH6ioEpkJKqw??^$`Kf#@}-(?2p1~ z%=4mJN4M1=B63$AH%Gom)%MZ3(-JRx1=Ad)x_>6+R*}s!M9xM9?+N+E)|KZOUU$WB zd_tmGhhopsLf?C8HiOU8{_}&bC89^<7gFJ3o@HIGK#OZxpU(_{xS*R6Q{R3p#$h)F zN*LAoSsK7n%7p&Gp)s`an_LX?!D5vQ)edUE%g9f>G;N=lUa%kKeLmh%DeR(hF89SB!$APlRmjOOt0Trzh~J6OlE!w6Cwq zlQgTmrp{E+Hx@4mo=HMp0d^t4Baq>CiSxfFK@;ma`|PHDWg1m}PA|*so?>?UT+^5b zNQ4dqr|Aa5?Sw!2H~|!_VAP`AfK?eK@W&SoEVuH&*u4vcP;7gavHyHyE$o;osV4hl z@$oNHD}#aL1v8Rdv0QgJaZe>&H?WelGmMrY^~WIE_c{9WCG1G8Y9M!Np*;UvayS3X zl8Ev?z?+6oN>TUp^=yK<`_0o)Go_`=$Di@ymxDn#O2!yK;>y{8oFxZg`4QtU?EIqB zCeQGQ*Go1(pgrXICbZ}kUc6T#?XwgLwqm#9d%{5AwP=TxK1hL%T4iu0Otst0jb1r+ z3g5W*AaKR%w^qAP#8G-#C-M} z0picu{5W!po-9O=!a2B&8cRRV=D_ADiiG?q`gRPNj0Q4~3gjY>RSs;wG+m%zJTH^% zubUr=f`FzfZn9Qf{d@>_=5vgKkph02Ipc@`w(3x{eK2Xg-f_MtfpPAFJxOP&KDRPR zEQ>Pq-vS$D zktS7`ge%U~ZeS#RwWYBkGdFJAb*J=4)yksdk>{)r?pbfwnX}E%se0BWu*#H%_+w zs^gDp6~4DIeTQXD zForDUA6X=nCvGJ=yjX=dE13GVhraQpb+;Ta2auNZ4#bbC@+Xx4NzBVLG2Sg{6j7kQ zy5+H3Vv0t{$)(;{6|ePS^vc=nvXX(Xm=JQOqQmen{R6a0NMvgUm`7nK#|0aCV4f=b zOsby3bT5EC`GlpfY{*|&OL@&5(fwk*cbPq*<-0v`y|qIJC!szhx$G!arsns4-;vY2 zF~8mW>kfi}N!%!GLz$MiE>8ndN1v`G{-7?kU)9=dubx;IC8;84W)&*BizqXbD1`D{ zBNbd&WB=6S{UUXz%Exz4D(-2J3lyw5MPe=Yrp8PX@iq_;SiF?%nM$C=B_1dBEP z3DMX8gy={IfEQtYNMaBMjk1#UUbvVM^Ut%}3U%s{g9Rgj?6y{uZy>IDLqlX{x9)}f zEC#{tu;hkB4K9Xwv+bGXst-dYr5$Fo^$Jj=l2lvxW-~&sj$h7r(peb+&`>c1YO)+; z%AMKM+Rw2qsZdJm>r)L27M(8(V+@u(YllQE){0zo|GAHdE~NwX(OHheMR)v-SDzXx zV{ByHJJ13fVrQB0GTNN6!+)Cev@f+HzpJ{WLY<|&#&_5;iW>iFIo4a}`~uMeJsk>; zCsBPJbZ9(;e%@>=hhFFQc7{17(wC=a7OGpp z4`42INVMS!_czf5lTslnk+T|bPngG$P|q3f7~?$j^D!rlEd;-q?Uz-weF6%<-}1xV zT!YtD4t}B$=X!=FLq#^9<>=#@4F+`}tA2cO!VqeP?V7|d82d%sVR+4hXS>g1NbX0c z)~@-OEX0-U4masyMQMbIUT|4n+V^RNTirm`vz)={2L9O6i{-D`tuMfwx@LgtBDa!A+6So^HtlCbR+yUa1d9zdAcP42> z8LaoGhhHF2kv}Ifeo%S=^2UkhxHj26?LYq!>LgH&Nmw&P4|E^U=1GE4CFimj&ePIp zJOQKHnvIuoA0J75$;<@#)uM0Ri4nLOA%&I1)@nGKS--a+hTc!8y%9wuYX_x=UYE;C zqA*CvY`Rt1Zh#KN))2pmxH73|`7){F=}ng>t-m8exCn7%mo2+D|I-@eA1;$X)m7(n z@Yi;jFYz&R#9?A|bU*(%Ftbrrx&N2={Oc3H2l74=uK##V_5NoY-~an#9QS{*A>u(8 z2KtYG7;zYa{jcU_gr^TTAo%IX5S;ygelu{m`=h<%|9h+lNzp2-y*je{mo(#p9krYG ze5RN*5V=4K7hq{|KB`=gqP`fBSrs+vB`;KAqjYyMh~Oo>Jd4$|NmkyJaHZf=RP{x~ z$;Np~wLiqVyu^Sq*HgC{8jGm4PIyZZlUEFNe!GY|M#t;I5d7{y)y!3Hd#wDEm`lKk`QZpnY=lw#0bg% zb1{`C&r39uv!N(Za~pYVwMHf>H0d7R@EP%+{Q%aJb-otckWK|K6%{rH#(3fF8Oi#T z*Xw2Ke|XkIls6!9mbYtl0Q_pRXHL}2%*Gfs!Akd#1W4YNI{jf(ou}e^UjO}w!~6%v6_6iFnFG9^1fc= zLg8u({qj_^+X03FB@T{_%D-nnFZM+9cA9IjPSfG;Q*mi^D?Mz4`v0dZ;+D8a9vDwta{WCITbGZR0fH>@E+Nv?w>PT!_68dDEsMOvEn! zjAcHW64&H1O2E8ot_79v;OQn-6Swo83-y^`+yZ%D4&<=b-rL3`hU;KNn(vaUxz6r& zo*AXHQEW*#Z8jyh$NBOsLvXHdW{aq;`sH_z!U$<>&?s#HN4H(nrKAk9eZxj$n*7!l zlOlzT@21Q4vjJXjxk`~`v$|F1Gu{aDJDM%j^j1OU++Iea4}{`zLsBhf#0b6+kkKM< zwrwDoBxFg_|256_4&8@MRhcTBHb4YHfp*WN2{1Fc5)c-~{j%q^GyfR?J(qU8xpi_~ zevqXkab-2r>|Pz0RECR~@*(nuA^hMoU3NshQjaWVucjZAfJ_a$Z4hnX<*t}JZ}p8T zV?z&9qHHhX6oEkUN(Q@E-e(YB&o;xCR1&`VBeXO$W71j6aO6%g z9d3h@1)|WYEt3vRX-}H_(CCIWgsi%{(L?Hw+$Ku$u_9;4WjqdiqbSis<&WXlQb&J? zsO81o-kNS;;ND%7Z%sL%Exb9>2OYCCUmi!!!bxm226#=TlxncG_ZH!V-Q|Q|*gLz# z?SK%fnt2%Z;(>%k%YDh6h5^TO#*=R*@je+bqkQVH-%&B;u|o5|Xz2H5W<^u4V-beM zdy)dNqgP~)P^osCN^>8 z2olY%wH#?Xyv!oe>o(*R=dRK@u02}ZVKVhdj$ zbw*mrqQv#4>12OeuCOXR-#&A11Nz$P?Jvz<0I#T$5=0Wxn<+;Hdaack{#<*`71o#= z;PeTfRGHU>SNT^9mD@YI@;MT1YwSJ;5kc^iN#5&|#m$Pn>8g=DzngMvyl!D$+wT2D zp<*s7wM7fj`P{z6p9F9(zE}n+ZS=H#Dq&0H5<#Tre@76MvNF4-B-YZUWSe!EcV1x$ zKKp-wg_K-$u@P(D*ihH6a{W^Y_)iq04N;|>D+$U0E`P}@HE)f3LYfIpn-X#R5O&231KM9JJH`gxwnF${5aeBwKA_6L#e4XCxE5god6Tcv3EPr_rtgbF?ZB<`1p!x%iCg9t;mwNa7+}L z5o2B1SQKDTC*k?(Wq3SqdD z;GUM)b}NlyRQ9N_Fa^XnIb%xU9h;BV07&>8@R64m95MYfCy|v>C!q)Idv&bVbj zv$;{NZoYFosc+Tzv6Ub5`!#+lDqa3@1J8AQ;^@5ev0s#|Nv(8>W}M5DQRec+!SZFq zA%?*?_+7|b12}Cyr1A}*p@lwzkOmCcBxu~{R`03aV)es(8b07=;x5eaH^$Qf1J3VK zWlS9ITNkN`SB5B|xtYVp2Mamwjm#X6fex`%+2ZePXh56q;$;nG=gXe^_+j^#-?&&{ z3(G6nZ~-jTAnR6ld0S1yPf_wl;moI1nafi$9?Rf0@a7Px2o{?#I-deO8w0p1upm}P z|E8!Q{;Q~HhOjPv(hVEO-qp%W(E1Sf&<}G(KTmi_!(Y>H4D9gzjUjTim8!xBm$%8E zR<=^KsyTv>T3jDoXhSm4VFr(?wMB$3=mc%y-LjM0{PLdzE+!++&sqo46OYMnxV(WmKQo@U*~XE=_I zH~8amG?=w*5@};~KD7`Y(T1pU=Aguxok#eGShvX8(7=MFw|ASk%ap}@6wXMy7M_lC zxBcRutX@6aDLji8XDDXOBPf$#ECjG^ye$D6Ke8eJ{sS!D4tEXRP4giP4_rx8ktQvA zZW}u&Gx!~qM-gqLpx{h)bc>k*iwD)P@ucpJYQ23qAJM1Ai|oV8^z=`8TjzmeYZE=c zo2T_!dU4H`yHHs#&e9nw4*BM?R!Lsji}FUl7Tw6N4`bZ_{|y>~I%s~-J?x@P>kWK~ zGB@ZM23p3*!ifk+9{gGiSL+^dP`l!q)FWQ3HWRJOz@NCwJ#b-_=^6<7^?t_nvG zbtlZg*MnmqpYF%RTC53}Z)CE2 z3V)<^OU9f3IVIBBXX~qD&VgX$I2y3!9HhHEZ0N5XG~?WyUa$!xjQDwFa6ifFHbs02 z_PSlhTh`DD#`$pKJ?cG8rYU;;5?LO|Qq;RsOedHVPSaBj54b*-)13XD=NwgwaHxioFHlK!tYYcwm6G5O>lCpGMY#L zU>jA_M}r2L_cqQ3GDz8u{_q-$Z2?Jkf&v?d6t4UwD4`pp8N8@e^KQU!7eAn|R(6x= zRHqkQjAx2Lu#pFd))86?7~Q!Jj?TC>WuQc$sa*4qA4wEb?RtGG*!NXMr^Rv^cblr> zb4Ogz!2#P*zTft(!>(%TUx%g|^w$rS8}A%!e)go)Rr3bwj7^ea z)Xy3Jj^3$$uG6R>mYixw&EKWglDJS##|cKbCZPAaM!ha>!1AbwKF3Rg2bezSFp-x^ zcx8VyXv>eCB_ogFzh#2;xDm`g`yt_)e5c*=5T&K{5R@PeY}HeW;<2py!StmxqXjZj zE(Dc+{*h8F@FovuMSM@5Mt_KIja+%fZ)jUuX?Jl$T&4VLJt%$Dw!;T)%`cOrEQ*_e zY6$x+M$G)a%cUHDkY+w3UZpOHwqCFfh0qZb{+qMR>>2gVy4PfC@_zUq|>pwIwxf||3Pl*^!!3*OdNhsQ9`d1Ha@ztx;{B=Re+Gt&Dh!SaIs zM`_T30|N&g?p7y*z{#3yU(ne~>+ZH}xwz~WHgSlsh_Uvyvkur16Z(;2VH{?*a~u5K z8UzgMdzvGuUl-VqQBNug>EpN8>Zq-s($p7Wp`a+cgBk`dpHv@hJWtKQw`W#1ME zpZ=;8K$k8o646xtR22Swp7DIJ^&K=e<%;0qI(oXw&!4i)?h7V)n>Sv4N8tCUX4Sec zolT#AoNX*-c72y}H#H=Cqm;%MRc~t_pG`G=QnZrO7ah+~dmY_hL6vmA!e{BLHg&~QE5`Zn^DMICr^ImpVg3lXweaT zY@3dI<6^f+*~)*FMo_BdaKhk$eX;bnN3j14;8*zb&i_L57GVXQQU(@FU0g_ zC!gz1Tp*`K&*IctNrb*Q`0a%zH{p|{VJLY$s`Bo22=W5QeN3Gfr4qRt0gMXPp(WMu zAx6`UlO+Ec-nYA|UmM=>lzjnw#I}yB(?4HhIt65ws= zAR!3l835;^DSTf*{Ex?&ngWl2y81?NRBSm&IBez*rT%C9o>^MCjG3(OB=C6YrnRN} z4`&I3=tOLWww;s1oM&}$eKvxFSD6z_K|Kze4?mkHMODr+eXWRS3~OEKiUR=IZKgvj z{P)`jiU_xr<~&G>fz#oL$&ouJ{bIC^@#Xqhb=#ilNbmFnkdySWoRov~Z{ax>??d_< z?cTA=rfQ6}FIb1drW3Htm)gO0`ioY?J*c2Jw>HscPoeg<8pHg@j(ZFO0S>tCpXiRH zFw3=8`%lA9HkSnYNsjjW>92~&P<{@MuZC4VJPEtT=+SG8q%g`e7==Rw^z$-8`eL`5 zNNPQI+Z{gC2hB>~|6qbfi2JkpQDyXF@LF<<4J-8R-VDKgsYDHyf7G~|^=|8kN!pY9 z^+4@>ZUP{eWN7gu{XeXXz_XyyZ3ws@hsd%9Twwn(dtf_Zu*2Y6;=6gPg*(iGj_Dr; z`Mk~orL+arxpi#~X^AUr=cGA8;z_X%9^<>S5LMi1V4%R{isUv}Tj9lmPenmCmZ+Ma zDY~eB)O{dB%r+Eq)5vsg)b}l9O*^>r5B_+F@HEM`n2&r0BsBVP#T05~_&F>mXlT0W zp-eEn`q0xEB{05qMz3*c@5e)AYh+}G#Ws6mxZZ$ZcDB>^C6(&S7E+e$B(x}q% zq{T1U-hJaB_g_jQthnbMNCf*WMUiaw-Y$A8pR}>a5*{47ynZJq221Amq~zbOw4B~{ zF0lu_RkMEtPtPn5*MUE9K1277(kgtXZ%u`rGaWIQ6 zMr@VOv@G2h#OZV|6?u5?a)?5FUe6&-%cVa7G<_YxmtGgvl5UtzPJ$n*`D2YZ zwUzt{PRD#k^X5aX`d>0|6h}cDO1J_h?>{~ZjE(l<7LP#>Mtubabv4fnjT*Hf8Vx-k zAuOK0f=ucMEE`um|LJVBG3(*Z*RJqx`#Y~wWp01;^~Yiu47ndqiK|6hV$5YY_>W8N zb>^sEfC58}17#f?ez;qmX=)f?%oLppO|82CSKGPpke{h1UdUPk$A8`+xgRw(%GbdW zlj>T1reH3Feeu5I9>5~`4%MLb`p~qvc$D4+?GE=$DfoFlb7wb@v_=40m>mL5`)dyHyH1#xyhBqHKpeg$5gkZw&kBidS4Hs{1 zOe;B2${Bj2;2Z@2*SSUoe(h?>)|{Q3%-`rgN9a?w%9c};_&rLY6>IKZND=-!j{0Is z+oL7zJdyl`$Hy(9$_#G2n>#`6H5Z-^T}?R_3vM0%RThS}>RllQ8zXdu_3_PN;n**}eye##?wL9VCkJ zq-SIXwd97r`)BfOhg*Z%A0&}rknsR@4UOQNnn64O>@d-IRJFn2cYkCAlvucOZ+6}1 zShc}u;^ecv_(jzSRk|Wzy|fSEKRb9&CwWeug{^;x63I7a5^&Y$@C6}Y8T7EE5=yfe zIz9hWGJf9`f7L$(-DpCCLErNLTCV<~PN33i;e8j#XAp?P7%`8xQ&m`lnkH{XxNqo+$S-;gm9LDI=N&ETAt1L&A(L>xzKWG;aZl7dX(14)z+*DWsGL?D_Mh0)s|#hhsXbEl$b#E z>aU!r`+t@bRsWR}UARd6mvSP@;%EO!V)WlOM(A2i_9!PBCpZ!{I%IhKy#HK-9_ujI z`@dffUji-yJNfsvFWodmuRg;m9rzz#+rNHV-}n*0{l}{?gZbZE@cn)9f0Hp%lL=ve zzJlZZF#aEB&hv48S@z44+o88rgfkVz=+UGy!(=BldzF25dJQ3rcOfQ7Xf;z;x$>u=Ju9uRBi8hm*Tp z+b&xgG`58bG;k{|t++R2^k|Fh-Ye@$&2`gdYw;IBpPL_aeHGR@TuSK+F&yZa?sN(u zrD_pR&OJ{OW=52lPyoJ;nI>CtQwe9k1pqu;uo&M4ylxnUX-JNS$AZB_A(!|D3^jUoU*0^mq2s zMWMGw+|fN8M}2Jt{a^8y5grkw+#KdD9Bn}x8jJ}G%l#l&b8M6%z<8#Y8%l%82E6zD zJS=tDwlWg;rHxpvCtbw*$p&FCqUeIwV&+Nr!!qQLMpDI%*w{i#vvJpOU617%^y@)q zZ7c=M@$O|-zlMifneyGMrYEDEJf5b;Vw`|0{!-udkqcIM&Vo0WvkFTvQ;#2)cvNt{ z#rv!hXvb(3uOn1Z%vx`tx7Qk%(X#t1_)%1(!$9}tejx6f&wN zc5oCw>oJ0z_jY??FXjYr4rV^1lc-NW|Kp@Yv4q@HH4T8;9W&%idaSKhT%)YEf9O@uk_Yt3%pgyb?5y;C2?x+zx9e=XkxC8!(C)KEP-6Cv ziry;-io`y&gDeM`#`PyLc||B0egor+v_$|TK-T-ErC<2=@N_KJU9R8Y3|kNqDT~$4 z(ssk5W>v12#%|Slk7-nWIXGp~gU7Xh*^Y(Daylxbcm4>WAF|}qfH&saFGnxd`#;w> z+5b|M>%QtcptQz%o9^Myo1~=~Ft|^YYceU@XXj(zyG|!wzvc!;=#tR{fA6?#rSe8(N%5&}ZeG;^U~B1R+T_)p zp*C{fR|zw*B*cye9GGJG;K0yoK!=Rt9=d9yM;;*)zCB#6U;wxaE4eOwP0mGY&ta) z)mo=ucz``R-?*> zwdPzAp2(TeclucpuX*ixxUYHjd|%4wHOqSRwn)V8kVP{O`6cZfw1(Wo_Sv+NgHup) zyPppGt;1UbyPbn0$|&4gLiu@Aen;+f$(lJ?nSW4U4RQZNnG{{V({IZFIs~rIcVzZ2 zasDKOk*zp7-uwx-&dLMkMU$zTYPDX|PA6{FZC#07a^Z+h55YD)UnI)+&AAee8Yo?J zV6UkY=|z{JyNHldQT=l+;8S?4gSewu0q*ymoHmf2615G*c%7`f#jQg~u${!ETkE)T zD5#M=KkxVG`}Cht%;jRGV03c7X3B2oMiCXT``(SK0;=YP>{u-8G?YHIz3CEz8Fjoq zJSX#X-Q^22G)qjneo=NE;&f7IgISlBJzK#UpZ>~xq6vhx?w$1eM#9`BpK2sU(9zEa z*?%e2aMGj-K$hj9l{2Tg^@7oaFo{J~*s^nc-bm9JC3LdSiR4Jcty6c)4CGH06qxoR z9@}n1Xt}<`kCFA3W#PsxFt8(gyaUPBQz&OC)WyOF`CWzCW?sGeWmle;$>tSCPtU+g=GNWpV$J7zcZ~G7vxDZU4?9@dH^t{GZ48Q3O`#$8xy4MTn9SI@e$V6! zlsJ#o0pjwA2A2mPmSke7?WIVH=v#&qvPR#4n@+H3TLm9l$Qm!AOQ9a@iixSzOvz~B z;nMMIZ@a(R;EUjr;4as1P(+e40TAEo84&|s0-R62VYT~@Yf?+GlBKb@-k{*%rTTA< z;KRfvdEWwM?B)hu=KGmo`BjB_6N(|d#J7tH#V~&X8Z(D(*iAe$bQ4t76Z@M96x}X5 z81^t6r@hpF@ovsRL6V);YS9(N;8(j|CeZXGqUjDDVZYaf$J$}B-pGB~u@dGokrZ!%!UDt_K7xde~D+bZ80Qwnz~Y3GoMm#yxKJzAP_PKMbTL zA6`yfH?O&(Ar(jgzAt~$P~YrQrQLe}HTDN;(w+xb&JMVGIpdhyhFe#?73H>rso$^a zEUTN6_X{eGq*>p2tS6bG&@;p=?##|&Sm54Jq&A;gW-?X_ndbK$2(OZmAaSPkSXkR zjq=Y*T9#iA$V{IZb=C45Zqk~cPK1KaCaxcg1Z+M z+=DxXyF=kncara0d+)W{y64_Mcb~h@`~lTwLkjAhv)(cK=ugk_AL2|eOzjPad-`87 zF&VdWJZi->bKJk$V#qW&~eIPy%dLPf8XOT)&gHO zSP|;Nhd5nbM^X$eX$^OC-`-A&l^kM)S9`pg;zfT~WA&qGfjM_5u^p`?NvLlN(cu{j zv$r>X>tm_0j>km_+Mjka2)xBJyZi{4K~k4NVUiZbSNWaZ-r_($Z1*E~LgwW>r=m%^ zrvQJ1`{grufRd{K50oc7@4hghCM4Dvn@d$nyBVdjtD~_NJp(J4;gYX^ZuTYd9Fmt6 zkJk)H-wn+LJZRzv3+lwOGsZxkbb7vL(zM8R3c@tXVT>2FM>)33!1uQq>CNVR<_ailxg-^>@O-(@V)w%rfQI%BK^{38|sNG_%UJi5g1?QHw=v#Yw z4=W>TV{@Fwc@hc1g~0-GV%pxGH^Xy&@aI<_ue8q=ihayLn4Kfr3KBSBRL5V%EzJ27 zj}0cACC8pT_Y~0uoq{kc3lgJ;BlJO}&(<03oC!a7 z!@8`d1MA4*%PPXwJY3xwApXxeyIwY16YqvVeE!|#t%XC##e1CdoX7n>(19WC9leuU%?FX{)}9QomX*x=KyR?)>1!HULB%?c;i z={R=W&7n?u{%$V4eU8v>(9s!mh*>`>yh@yF?LUS^=OO`#k~xl*8}_fO0R*sp+5oI#g@@ z=A+Nibo?TXLHXAnhSaZd?_ot5C4|=OA4gtSptv}zCh!Oh*i&%-^l?`~Wlytuvz*Qw z6wwm%0F;}$JVJZ`%I!N7dc#u?{sPMNc)tOFa<}c>WKg|=h(Fp8Yi3*yhu*3NDPUGE zkbb&%P2Q5k8F9tLu#sh*DQU?0i$L=e#R+i|)dYKtI^i3m`+7iZm8^Y9*>W*XD<@klEnNF2=)|A{!X4(`Ag{=7=#=WU$>O3+Az+x z>9(Rd4{E@^SQ?`jbV19v7r|DNRl7ngL_9sJrLN5@K_vefHB|NWas6!m;hMbop3-C_yiMS^b@L#PQDh@ij5IXCQX+B&7&S+FBe&jn;z-xY*XN^-35 zPXS1|Gyo}weX2PvI=o)*cPeNn+`QS0>2Xn>^@Wr>nEC<($EbmZw#RT=gVE-kO4FnE z{ls$L>;qD6!!+!P28emK^JLnGUXzkULb4_@S>V|D+2P#*$jgx^<1^S;yE3veT3dn)0#uOi9W1JEJYy z5k{?T^Sd+j;|PTSN6LOUwX2!BeiLf--wU;P{WX9>L3hUoTEDNS8UkhcQxhp zKZBd67Nj)AD7wB%TP!w31S751yAJk7sDi`0NSUA7JG7SVtyc^!<6Y+Slx5s?_Z_1M zk2I$}*o2&`+ORm+z2JaNNYDW@@6HH^zL>{m-RQ-%&XCF?kbH?0FS(~G%iS`@w?b}H zQmiPxkZ292iR=gk+or>TYC z2uRb5@#%RCsW++-I~>HbOdBW9d#+gSwEGs+AeW}Bpcd1;YZlsJX@f4@iM^7;#Ft=t za>cH(WhU99enFf`>76;7u8x!py+mtOgKF$d{1WV#`C&}omY z%P{CaPX(t~Mk%s8L$wT6U@oX!La0`qYRBsB8H!MbJX2zcpE*tdCr-}C_t6)UXIW5# zvHC_OKju4*3eQbEg)Hz`r54g^ zhohJN;V^lf-V%WTiPs6;501|-j&+1wUBO7=gbm$mC|f?ApfJ`g8%z2jUq)>v%UN5> z;Hfp<#uvxQIpi#N;MKj?k{cy6)H*uqf%cPk1EqVx20G6{wP4y_?SV&GDAwWo(zqH_ z>EE0@M^u4Y&Tiz-M6p@ocK737Z>@5N!?ccYEOdvle9xyGS_uqS`vltY1TK+($!ZW` zFIj7(D3sU+NOG26*~gR>zBMD&*KOM2_$KI%J5|FnWU9i+n9sDpEGO)Fz~R1+!d#_; zxA;qX@A7(2|9-f3`FenY-il~=FtL;E!qK+^qlj@W}00+KuWwyFSxM4Ea2=8CKo11~SGC7b8uvKmz-IjL$zEBVZp9r=Q za^2hrtA~D|^e-7MgW-#K<;gdGGFh`w5EaSr;EUMKq2LUGh)~N68)nTdRJuCe$8=P& zWlF!lz`Wv)kA8HX4zd~tTtU?FW$ih2m=CU?46lfgw{RtNiLEg9r=c>(Qi*J7ABQRd z44h6~&k#eG*G7)xeWm^F7Nq{Bom9TP^Z;2Cefvso%zU?lkAlFZDxzJ3 zcA+*^|CNn4tZ1MfecP9U`SzBk3Ct3#`_dHgSmj2(Rt z{!p-1XAvI$;8C2;Y9L|vxVU)7G{)1QOcg22BX^OjW&0{=g9jM0@OR<|bi>E59&@U9 z_tXx=cGR+A=M@*n{(Kqwt4X%xVpm~^(7mPNapz0ZjNW{}59FM~nXj4Ed>=TapumzF zZg3WgsQPmytI9;JGAsDQ@9q8BmZIioQV{=?9zh%G;{=t+jK;KrYbu3XUa<_$h;zm6 z7r4$yuQkRlVLqK}Du_@>_pPgzi}9ZBihevJZj-Dq!gO6C?pS*wvkQVP07;ij z8@QbSqmH-gB=Gg$O6IZg9muSJ-c4T#+9$ymsc8g-)H$#(cFZ8AvPL_Hg1UTdk1Ipg z$M`U;?=&z$rM(bXp5fcw4fz}RuoX!m?FJ0xTZL(_wHmJ%Nk0d5I~Efqx0m6@ZIK&* zdrJp!Z#NMH08+Eibu|eZ-2`uh=bn%|{O$w|Ga8%sX|e_&khNXf8=oB9pCd*Nut~Q!z+Lzz3C0b@#8*}}z4v+PYfff7RaYCaiTpMvA zK%jfv9g~f%zo*|;9zVkyY_M=lMMk}QW-V*d^`l>0T+Mga#In40c5sxIj<(=u$DD3c zMCF<6E*>jWm?LnDV(R^FBg!6rnv4(VS6;sB{M4s~M*i6fBIiGX28Lmu!XgBj)I|qU zjp|5_R*scdhprgd;?oVP&s=}=Qwf>^3N;%MdUEBr1mu}rZB?CGrB4Gh@Ln(h?$Lf9vIQM7P+MGUpol;>&NW^tF-gD=`@?dDdR19N z?=ukKx0{0b`T1>NwY4BlFDG&4jl4NV^;MunADTLKW`4wfkvGS*8p?mE4WIgrD7 zG`=Ey%ev+fP3jsxh{EXP`#E`|tmRst6JUjXUgNFAOzp2Qba6c!+q_%qc3U;PKhk{c z<@q+LcEOF}9dJ)RIUJuRg4F|lA8Xt);Hf|{RjLl>w{F()Ztwo#_rZ+*=rY_qpM1$2 zG}W3;c{l}z09OofRPl*40vNZ{K+8>f0eTg?zcFsNHn(f~oDaF#K9zp9cjT2WJI%Y9 zG-+3KwOWams3-JcgHSqv2*q6+E*hM&0yDQ0~2KVG(fd^IEUzcKs_{Hde=$v-ptKjT~&^A?d{ zvb?inJHfb1J*c~Rux6k1@}6W?xEsL=q!v_@VN`^AKl{qX5`wa#x9UtSptyVuqV^qFk_ed z`rfR6-9lqc7e~V#^^NMUE!!I*G0`EUPTgxHwoF#Ie^gL=hz#`9rA#I^SS2K%^=fh@ zdBV=RfuV{S|If^JGfyDY_Y>C^{CKQvzNv39b140bB&6R~9K^HyRma$ISdy8n5@@@< zKkwkud*DJ{;W&BhEU)f4P*Hpy_p}c5{TegcJ749WDot8O@Ax>%a_C+i&}X8n{#l<1 zusv}!S#cPh#HrXFV-*;ZF!_`!1k8q&TZU?{e(oq z*-|g=&sZ3a^utOYEt3S>9{yyZ=yYnEj1~z+n<)X5n9Ff-iu0P|+7_#o$Q}>3(`=F7 zKO%e!$s4gR3RP%A@Y?(WtVXl9BD^_o%vM%2sK8#QvG4?ZC**LiAQa2p#6&#ll@2Mq z6Qb4CJ?ehH&o=OBEC=faG5&7IS`(hEY@TmAi}TU0BNY-UIJ&XXRN)NeP>e;A6cV)D z!qjCEegYLH_o!WlB+vp5A6deW?kk_|gjVr+vz?nB#I~ztz~RG_c6=v&&qtZw(T$@> z!XKV51l9!{K4BSy3)yVfB`IR=`jlOfGu5kp{PEo;VeM+nAszRe$33GG$TGPp?t9^59-g~T0oTbP|F1iq0yzz(omN-PS$%w<}j$M8+__lNm zPeH`&q2FhguRglU0elu`B>m8r5r5SG)VJ)C8*-5+t_fn_a-X9ZowUj^z3H}4WeNDs zKdgE4{I?u8-4s>JV2uxKAcZv`juc0+%80)-v^nNG+oG}JN0t=j0g7g;glD_#zDd{J zK$k^(paw>l=%#{853AbU;59zcZtI4erX)csu7#9aj?hB{-WktnWNn*2@hLh>s!0E_ zzbjr3J0lh-3g4}3uKYOhtzOFAoNt&3?&rN~eC=Q@ds*F*2@1@J&`$Yvfr!vQAX=QI9ss=q$fS53Iqh&DUR6qW70Z0pI< zP_5P_>t_{wV`5LHW)`zOHFTNKr$HVgrlkb7+4ZRRC|#|X4t@|l6XXoE zAt~){O**0wQU~C38xjLlnwz<`p^#LDI`~L@4Ttn4d?~HsEgJ;Y!#Sp(69lzf+QW?) zYf2O@)U1;fTv}hQWp+qM^O6>d``)Y}QwH1&C%e$KwbY&Wt<;73?)c3~A5{l!d`$vE z*5-Xcr#P%Ye|>D|8?f3i&SMh|OHTPT1NiuELtzBI^Mdok#V=Ww)OKe}v^ak_)`nYuZNZ_ikgHKgt zQHE)tFYVp6MARI8~&!PL?QKbzKyoIG4R{NDHu-B}?_rDQiY& zjWM1DKlk;Rn}#&*RMiluaG_QvesS$%LGoIL3#VZNF_Vyp5 zhi~+H-ajJ_LJzW(^c^nxzm;?@U8O2gw1X#&BDEA6vBO4kQ5NQsVm)@+vaKlR%EH$e zdNcRL6ukrqqgXig;5Xf`JddpbHk^9wp)i!#u-(&0l-_dHl?=q0lL2(jEqHuOl%WjL zCgsi@ns<)@`_EEP9F)~&9`9Pms-1gFyz87(?YM(@p2q=4SVdr9x{*gAsU=EQy_=P} zfCKz%mK_KfGFihp*mTj3te!Ge=z?XNcJX&CxS@;*KvGfiLbG&uv%b2O{(2g;bbq`W zc<(?32vi?WDkO-AKee15Zt~tTv{~*=->*taIJudDwXAvkyhpxsj_eoE(jG5e#6{h0 zN(DozIeqv0NE}*)dT~A9JquwWZra8%{*dzYM@hB3l7h%OcXXp;gm}nE^a^_`T$ts+ z7dHQDQNIG7D%=Ys3>nmWbugzkN6e@@h2u#96B+6u!BYs}0Xz{cjlPrht5gVqG+zn& zqLV8#25*$FXQ%JoCtGVquFcELa$py`wj%eiD<5Z)>rAh(yG3Mfx%RvL^=z+YOLXdm zAh(y;`UGXjRXZ4*X|H2-;J)*aFa&iev7_R>zj?jU0*Rc+NPIEfUf_S4*s>*k65(B} zg;KJ`;VmHJ8;nuPEXWWBEuH@$EJoo0_OH08a@t?t^%|etfnHHD0U}WmgTKYX7e6RL zFO2_Jc}>KMTfoQ8N$J>X!me?AdzWQSdog@|Zy7RV$>=gv&Z2EoZE-SwYntTJe(b7W8+%Nnbq*vzUL;E&9?1q`E~~NE z72Jh&CFy`^=p4&$lXwG3*+U@x(TUMSUVZhyskL^bRwf3kH&+g36T21ySk)TZPOSlE zoO-Z1-xZ@1SbAV*7@)TpTkq+q->nyG834?t>zny_%#*Id%EfhY&P}~Ei25Nj059|?C@Aa!^lm` zUtR&XneI&22g+?#)d4G84z!dZA~Bb(6dctg1DEB%R&@IjiXeGu_RGpHQSk$mx^1z~ zyf#dPANK~cTpK$A)Rd%pH$}^l-nCthyO+T6~^=A*nhea?;F~UatGSJ*?dC zP^0_$EieR0)2AK4dug7UK!`C37EdAhtYfB-xcbW=+2v%BY-*u(!fd)MV2)*I{$Qaf zyQm|CpE2k@4ZSGpI22OL$w7{XMC#fP?>R^ZW(d~xR3Uz|N(oV9;UtB6gb zbN}qH!NbK4=hVsQQK{9ttSVg$Y|r@iAFrRl4uHPsUrNwKf9ur;9#UhINYLDVjDh&4RH*{ZL<|_-6(%m^BE6Ab=hX> zYGpI+WxAABU2ZkNgi2iRlpboity+rWWY=vr?fUH(k5zI&O{B5OCh$UFmJUa5WkzR2 zt;WWAynI2gD+FwK7F_~|aOm{Tk*V~|TQ_O*-XZ8J$X~YHR`*HNE;ens?K5o=efij2 z=$979}P&oO`ZP?g!$Gjg!AjrMjg(`R`ML_@RU})~fr{ zybBT%PWU;%YIMctd$HZ>*|~0_*^WGPvl$h|QQsD4D{{#oJ-n2%-H-{VX(U6KndNPaMEgj^ zH?Kc3vHk^Umx{zCMPX@(1LonAs%6#4Sz7cF3_ZV;R=!smqUEE8&&G8aQ@V1K8U#ta z$yI#J(qvg@uN9k7cMHef9saZ|2J2nT-r2#hELvRBEou`zCM_xaN%aBtJ!`ysc3%b3 zu!9-8XS=bWuMo@PD=ds9H8?-0W8AsS^e*#c2@wJhT{;k#eYq?_b-ZD^av({?q|ZA1 z{Loh4zv`~bQ2S;<=;&ZAhA8ZHI1F*wn>J(dKe0pxH;M z>p~`(u`hzL&RW=m%nF;AYhwLF_q2vFvR$OZLA1x}fXtt5h97>o-M8X)80RMd15&KJH0$RPrc)Y?yL(Jgel&*r(ec4LaiP@kP5((u#xlb6Vf{aTkts7_AQMhI!H*N_-+*I(<4yDU2$7rCJ*dC zVC2s6L8p9)Ts!Y>1#i7dzgd09E8fv&{e%iEJUxwe7?3CgH)Wn&YRFUii1Udl-wVGA z9^&lcnu~V0cpWZIW7yf@qKxp6D5TNhe(=3P8uMDqJ~lcuEcVM~rat+ub2u4$14;rx zrw#0>0u?ORvIy;fAuuF>SUte_1(8UjE zRmwIuSH0LQx=GTRdEMPi1mKSqqMH(*h+r-f&80BbNbz9+i9&z27C-(_e@2d(>~BKQ3%s@y2zA1B# zpe&GPr#wK~l+yeHp@R1M2Tp~*I=%M5qK0n-{+}r;Ir4f`Rp?tA_>*fl3uK?6Uo|uT z8*h)|A^sLd`TwK0=k`Bsy`UH{BW?;=NcoEgS8usrwUSW#55U#yc^TJSt){L{m1|^P zYgI&GnVVm@{;p9yvBYI6Es>Ir5od#@9`?kyGNAcY6>5|qGt^8MnR7m06V!ud)K;y^^hz8H{>YGC((@|$#j()I+rn%u6TAka#mj8T_6Y+=-<`R<9!+h|uhvd3T$ zXVR%c@7!wcnHIHg9z@tplY3EtdUtU+=Oz+2v1)Lz>I^UtrN(x5EzinTHuchKc)B=e zvpTk|Q&1Lnxtj6r6+4w1t~@*G?>in>1EPwq=G2jgCG_?!NhCrldgjzV_PB;UL=`a@ zszspkmA{WLl<3hsqOYvvms9HHMVlNo=~76=bh81!voUJ=HBRaJ+u#`z_}l6hjLg?K zwTRASXP+IvR=PRg_d>W_4j}zmQ1HuMZ92#|L3^IS{$kP?lp5H>6t$Y1?nq+4 zzc}(UW1`D#!#DvqQPS;n;K!5OAvn@YQt(%4l;p;bZ!zove5R8PWb4dqE z{G{rkKGlpw_W_@6Zd{56wIB;<+&Bf%4&>kL1LMlH#9_|9$6l>>t#<5>~!|NL+M`Y_s4)%ll<_#cX0 zpcwYQk@5a#oEvY?wBLLB4LXBSsB^boLJZ^3aCYyIEdR>zQwswd?-)TG4Qj9|h&I#U z#FBOML?7{}aujxOJfLxfrD0!Nc^}r>;lNP{ z(S~pHbDJX;4HoNoxAx$xbF_A;BeY+qSlRHVrj%uA39(jHVps$byOl6#p@A05=;=`( zKYm1B{R6LyyiuG!uv2vJlfAwaD>8xbQ{0Ez)u*U&dp$zp#TS&m$x)j@8KNLJphEK9 z0bE7Fo7Dpop}QcE+*KDylLB){HLS8#`lAeTDZeZBIh%}RR<~IE#-9HAwAc`jfn`)E zklDj7=fg%GmW@JAxb8{Ckm|+N9&c&9y}E3Y@kh}Tn&rrLCoiZ56s@O0DZl!pDPQ5P zM+ta68ma!HX!TwNuv(arYh%OBJZfx^WhKe!pwhPTBy=&@PYde}Lo;d8Q&T=pb`2U@ zsty)Lh)&E?qZ1t~imAnjwjI2zG0A3Sq{_aw-_1Y#($JMnX*L2x6_Jeu(Jy@QvA5zQ z%hP;2S@t+O-ZEFWHf294KQshQ2||?z+CC;gKn}mzT&2H^%--4|{D$?avrI)LCT=&k zPM3`?CgA1d`JVHRHn3zzS!o_TZ#z!nvIqDA(SAP}r`1P^vsmUF#jOEcPx_@cNAu;T zav$UN;E{>w$=ruBu7lO2porgaJ4{~t3*FCrt=U+$FDzKL+;{P!zd=8hIirvsa%{U@ z*oQV$qT#@;(37~*pfG>0X6}&Mnvd}BiZ1tyRukf+Z^K&WQ@XakmJiG*%w(Q4mkyQQ zM(CrNOotjG%lqs`9##qAc=?{};Rfwv{hA2h$+%Q=bphpXXs8fm-T2!$sygeZCyI)1 z$#wDD`63H@A@U^+V-emECEcXt-g-S4jP?ro3^n>IkE1#SQ=a?k=vsu{8REm`4oKN! z4=i``HxmO=6AZSOpMDq3n7cZxrb5a9aCHr)a%Voy*^!i(3$5*5v>g_~CxRmUfS6Sm z5M*FLBaJU%aa(DK(!pwOQ>uPr0ALE1*`5-z`j;a~Ehgbf-o}K`gb}r8lP%+YOWP`E zvct^}tvf|t5iM=SyM1e5u!om+pf{h$<-!zoiO6|8T>+?B_x#S>7LHRY{dv0tG4Ve- zyMWZ9j=OZVEVm^_#?7t#fIWFWb`SLM`RxwOy06{wv4D5#@qyjAT{WOytv{$&7+H$+ z?cboG#XEp{^_C>{myIZ*J!#X|pEtQNV`dcM?OCr%fxA+b9K*IP@Fq}2kpTTP3dK2X zl=E$!#)rUbkN?P5yDkco7Gz%CPLoU-ZIL7xwQJJrkVYRn-ZEJ-5pg4xmPz&w3UE|KPIT>!FIH?be+U^GoX53wx{S$ z#yMIF2K$uWF=*X2x!+!Y77=8NnzDyH*7l;Ca;gh0nY!rMxrrUmH^3LPrBXvFIty)% zT5@W0*fsSp<)}C-<`lon2y>S=%CVo)wYWA=A?VGgGo3cjrr;-!TN5m$;ss8z zrqg6TTf(q!cYPrDsY>MP$x`3&+e1lTx$a{(bRyfQReY)cm<5n!s~bd)w|v)a(dugD zCehP78@dbKH#HgahCexs#ZPL(*lHxyY!<()WuSKu)fijUHl?IMewmXVXyC)f;hAce=~P5dssGGHWEYh(=#=Y zHpR8ey~Z33RgN{4kr=hiw=v}KOI)>NwXxE?yrZ>a&!$N?klPM$*BLvtvAT^si$y*A zTge*XuOLr%q@|ye6O%trq1V_YtMhvG98j`K-rH=hE2yI`GTHaiu|I9%dE6iMX;FhY zt^Sb9cHdbMn1hpk8ospDuv=*TL_o`6bU@t-Vtle+^8WrNsJCcFqs@#qU<2_)8S*Q2 zov@&^ok2!ZiOkROCmj1#>D{lYH6YB0KY~wUi#p%%fIr>GP@#r0uAU7yN^Pl2anDdv z@5$Hj$m>JQhKHo|_k@22+xMUE>rVV|zWVVDa%IixIN9hzBy`n2E?SrFzL=cRCj-}T zF&@p*1ov)RSDynH0cJR&XWQAKK^YKl)koiUM#>%r)vEEWChRYBYDvhuKeR@LKsY00 zQwAbIQWXjt%jb&pm~DkHF}35 zcRn8Nr>9$mnyJ+xrZAZxr&$#e8jFfvWZ5;FbclY?>3x7)V(A|X|1uVxZFGe8=6%PY zPq#I-)sf-JrxkpniIRiYK|6pUDQqP_f$BxWKKDzjrXZ|*DrcG0*4Cv{JVQ?YwNh#F z%AanO1X-3`3%*hkPK_-2NxkPEMD1|$pQO5WLRwsE#qhC>uBA!DvF#gFE(nP*dFoGV`m^K3ZolA*P=ymho41SM=66jXA zpY6}{8S-4I2=x8r`=Upf6_u23)<^DV)!i`+1)~UjIU5a=vK#uQe$EDGz0I?me-t z*nc|h!($^{tFg7oOrs$fF+)pxYt-r1ciX-NW%K6Gs|I00ZyOtw52>wLP1;fnErtyG zyWD}+Z3!l3*Q|igt`K@(jYT%a79y2&TAQR->c`PQjjc6nT?7FjK|3)w2vwD-K9Qq^ z0|8qn4pm84Imsp}EX30Cvbqx?Bj^zDC%L;HvMo% z3e&s5$Hq(!1K+=ee*a`w1Ay$L0Fd3!_m9e|1%^**oIZ>AuMWzrA3lT}FmWu~k&qGn_fC#pZSA;ZC4rW??z6@&H1PI-*)ZDFqju4AYUn zw4gbahD|rr8V`civ-!Eyy_3Ym*MCTM+R*l@*?x^ik&$lYR&b%+1Ie!F3rF%YdSF$W zmVZlHuu+sz=dkXog2c%1soHXE()qlA=|g+^8G zW^!ROvo#ljqhT`K*bHIlLIL~!9-^N^gWr!Qlc=&F z`o|uo?MT$`H#Q8!660ayE{#85p6Zeb>yUI#3Lr?N86L6*hHm>Wlwr;C2De6Ithqb# zCD;7^ZvCq5X0n12ilh{`W>V5QacxeNK?SMiML6-qQY7mNldG^4Dw`|>bHe4w3HD7UkSVTrnV9iwq6<=#Yh(HZt4)#Xb*@l zrww0XpJk&VNU=py3MAekKFCwol^Crz#S!h$B=4`C!w7;8hNZ^n!*`g9wjGZ{zUIUs z7Sm6l5<*p0FfJKnDJe&x*3@*H)TguXgN`lE^n|YYPnk7bQMvlbU%sUAq>eV+)0Ebn zz+ZpRZFKVEPwhHiunN|qG^ALN5|a}hBG9RZwz5eo$GFRGxuORvN5d1a(D6C^*N0we zYK*FuO`}eI9?6MknZmwV(H=`n3`jSa`JN({&f*iO-q-h}=-EI!G(*a!@KJl|&{d*i zJZF*9UYZpZZKa<+un1G%lc1xuJ?K=a?Lb9g5*5$-eJIIRy(OSirEipnrmuGh=uF_7ROM@-bK(=@gmOAf#To{d9jBS~uE=5=y9A|wvPE4Ik_Xi@MWy~m zy}e&PO2!|hDp?o8x3PZ|qvF|LEPjeV+5fY#x!t5zaw@S(jDcCZFrvx&Z7F;k%jJ+e zw}igv?t9e=J9kM=2MF-czI=(lC2ep#JB!W4wPv`JDYgOx=n#|1p6;w&s$-iwK4i+6 zdAfZ8@(4CJP>4?DxAQwh-(=)aRaTW5vve9z8D+Q!KMCV5QRvqMUzGDoxGJr7%hOqunKIQ(dr?9nTlwo!Yvz{oPZzqg zI>v3I1xPB|$jx^-MD<)gyF3IgwYN;xO)$pve(92s`tbXyZzK0p@TJTP!7I!cSExjV zOS<(bw}OKG29+m2FTNDhuk9yL8{C}m8P8ne`^k>04Uw##XDWZ9eVJe$VTl52l+|n&Y56r};)1meURu zJ#FMXIZ{TPwmA!W`<6a75aSr)_T$K$gX^VuqH0=){BE8rFj{&|z15r?#M06hzulF! zU$*A{-Vv&0v|+#tL8M8EMbm++(e!bc5HbgK$nd!J^q3%moe zj!J;6BZw%l7PU^R`i4KXc2>*!o*#H5aF3p|L~~Hi`qFYmOPA6x^(-PAX4qZX9t=COrc|r7^X*?y>>9I$4L+#A?wRU=JA8 zEwFJwlgsJ^!OG6@Bt|c_*JSSm>3h!KZ+ZT7k@l?W3Y?-R-u{zyB%h57aXEoex^m@0 zhxblO{Pz0w2cKgKCxs+}{7yH1YCDwYtd@Y*aTops7>6l8IQAQow58rKpNU+<_I;?G z`7?-FOueL0iaa?=_AU@Gp>ruqV^rOjm->=IeG`W^$08gaW-nco@b>R80ODL_k`yNRso6*ihqK`Q04!SLD_QK3Bq-tHGI0h>lg~RlT_G0G@UtPuk|;E;8k= zT3ky+1tj8X0F(nah;cOovW~*b6l%7SYCP&AWK%V`O}D##qmEs*ud$qAo>jn_yyR4s zJ8)6j+!lz@@f}cXk86MLzvQax?&Q4YRZ(Zv0V3&!%VA(|#OGOV--gO?#Yu32U`6Xq z{I85mA{7r?x05U!8h3@YQ|{h-_@J8-_5yc%-yE6*{>|Bkq@xk0;MVrey60P;qE9#P z2E>~$-XV9we(j)$+fd}B6-U#P1m%7x*_YlC-gb-KqXg*GNAE!Gr&`FgZ z=LqREGbZAM%xw$labwzAT)NxyE1HFkHQgI7tB^9^jI_WyE~0?cHL&`W8h}avRdRyx z*7`s=Q18X)ruGj|Jtl(9PrpXOPKdnBMD#NTGapKlL`?QoFYX?PClw7K6n1 z0$&B3kZy&xXQ+whGZfYP&86~2{xsgzOrHQys{9yhsIs!h5v>*qw2h6NZ_m|Sz)P&+ z_%`u7#odUvweyMcNuxVtcx~LD1@a^xcG}yU5zo-z1WFO=9tX|toIdLS`9Lg z1n6C}Gpxm+7v=VN(CbT!Y<4HBZE^BI|G6R6-MUcMX!7K%GK-3vSd@`X<9+V0wfFI2 zs)JlR-tdLa7$yJPt2Q!v)!Szv<6-^n+|OJq-)VL+atDWg!VsNc&2^{~Enk#YW*dV2 zhg&{v{E0il-HMEv@@feb7iSYEMZc`=N?{MXIM3*QvzF6TXZe9kuZ@P28LZk^ z{+wCtfE%VQ66Z^4*>H=C1D^QV=iovu`lLFa+X_2AAYk~Fa(QsSDU^h^Xbnt)k>Ki#Yt`= zVvkN%NCw=tg`R0Yv9~5Uq)#8%9a4Zy^6V7)m$1}lvWq~B{g_R=%(aOPOI`H@)O#MM8n`RX8|#YFPIIhs z$e?6_mjG5dt2#Wtck;+y+&W#Y${Tv+^~kqk<1 z#Q4|m!L17Mgca(r1E_|7IT%fG>orR@tbG6FhxI(!|1nXh!ufB%V}GHswtfx((*4t6 z_wke$mXO3|^2w>gKn(giX_}BjawVscs4u}TEl zbuoQ8{HdlyYhcUst17~6kwX)h?^M~dV{(4K$jgA0B`s`+^U9fw%9fFC(vljUp`pqG z({4~N#8d;%_h+{h85mfRU6D3*7h&C@R^47Y1rwP#NGw$kpm)KlClvccIxZ&+2V`4q zm4x~@-)aDfmWFb2k!^M9-GXCp*6v@`G|s=uIEfJuc)5#jjTt1ErFj4}b~9RAWx8a? zJk6BPzq0lAB=)vuHuM9-DXt9m?5o$a1Yequ32=LT*_2o5f!a5w4J5nhcNaP~y2TQm z!wr=59MP3Ssu~ollc{bPu}cGDhMy}U`TBn8|~?HR@98`?pu5O%bXFP zoFSu)mQ9UN`E+Mhp85d$U)7>cId=*{MFt-2_BRwfQ8_6RHbY_5Yq63Y1o0p2 z+(@1=7NC$SjkRpaooie9AFqdPlNTR3k!Su92GUolHz&+ zss{L+_pizZ|IadXY|N&9LBrDo1NP}I8BYsw?Q%E7nF6+ zuVi=)k2z_EJ)@>c$TatV-UnsG1=PlyH#_FAH0V5432>y|2y%_}du}vO&IxEixUDf) zMCIkfQS~imeeiB{UZn2kKB1awK0;FQ+sbX`%O_o3sjOv(q5M40aTA=693I&CE z#qW8mieMZlfxL>EH5lEI^@MVnoxFQ_u?>Q}?67j8fc+2!QF|FS|GDtN+c+%e#Sgbz zukn0!ikfAovs}!t@rdfjf=F$UA{b~*mM&4}*O_0D_6{pNKR1NV&7@eRN-3N}Ho+n^?JD|}$ z$l`qA2z9XI38OG`0n)>PI~Gm>CBGG*`^T@HXoy|U>3iauzrf@5y?}G1|KZpX%l0KR ztQ}4dU9Yp@jjj&VYauwsRoYpu%vlU-br%J(sUFLdU8PL1Sa%SkCtY(8+M~_Xxf7JL z7k*JKtxpiQE;85+2W|lxL)pBIy7@U;oQncs7xuNM2c&r{))S&dZriRn?wUQ5wIW|i z&6j~})DFkBLsiky!MxHVJ&gMBU0y;WO~DwG4E%o5c#mMFHIZvaYuTgfmYl+tgb1CC zo-Grq6Swe$)?MrMzc_a3sva59JdX6M)I$`z-f`(8oOVV5%q2~utt!kB9o^qp^k*Am zv3g+g3su42-X|-n0B$IVn48GEUN3EFV7eofP&p>m=6a7CCFK6 zC$e8@wPQr`20UcISIw1hQZR-&^9Y?w^XuXC)woI!SQID8bzSulQeF0g8_Ls3W&m(Q z^_5WqlD4yewVVA!=bh@7&y; z$&wsSfr@K$^GcHV59cq*wrR1q6z6j~0Vf5-Z)2O75(JYq)Hc{9jDAJ>1YF8924suAEc| z;CCH{lY7NW9NO&URQG;{^59*NlG$&Dtzj+C(F+> zT4G~yl~#<(NJAls*B&eBi4|X`OwB9h7h6yB;eWRpE9L+AKMsCBS7vZaU@B-$fm11( zG=+4t#0pAz`0e&?0!XdBvlf37D*iqIMt83aifHJ&TEbhxm4XW(^G;c354@Cu%a2_y z81bq$mZO6%z^*&+`}o#=@abh$FM53!1)UKIogms;`f&`tJ`+!&mJX1RKAdUqU~SVZ z-giSTSM@yL2b%0}96fi@moFimPqYeM1Nk0)zkw5-d2uonTD}7CZ!Zr%4)jCxqY@ z-09#UxVyUtcXxMbr28!Le&5V_=gh8}Q)i#4y=Tw!M^_8As(aPLTF-M|_jUa+I)@AI zolj|{As&lYM=mn36Rl!Yw9y4q{ql#WdT7Uqv@vKGN1@muZ|h8{-ba<_+Po+zOEq~v zCnwmr;)Y7yj;A=Judu(4MRG}f+A2gVLsl19eUnNhY)>tPozl?m*dU8tXsE!9O@tZ3 zgH#?&DAjw78x-sq5y9c@9cG7Z`?95h@b`RkMZxzI=x5Eya!gO8Z+_JpM_2ppSgEqQ z?*)A_W)#7VTBPftm5PGrBahxy2}!mV@Ul~muMGy`5fZwS`Rrx;-&TAw##bp4Gb+c* zar-XSW64X)%{fiB^?q=LOFE3}>}LA$t{w7)mj#{Gp+a#h5N0PF+&DY*SD0PL$;VEX zAg8$R74}d5IG?coIG?x^DQv1ZT{|u@NjM{s;%2QeT@w`n@KJbUN|L`U9imr~$N?J&#ETkcRO4G&|=@ z{1txt3vmOCCuyYX(W(<9oY+f+W}z9Q>*sXd^tw0V5XIw{`iGp6%aizJ=F>d#siCx% z;Fnec^YMNu#UctvAM76#%GGSv3vc~t2@(8r=dL& zYIw^-`T1f|cg|T@b9* z>m&XAeX43u`w#Oz3F)uEdSHyG{;Z3LI<(qX9*&B-#Od)5e^X@er&9x-LO@o)(*vi{$!4Rr(DFPc08;lj5hewm&0fgr3vQg)4i0 za`%7IGriH7&vCr>I&I`}^ThRit{o21i!fe+n5IK^cA=X3hHJZrH{v)eUiZ0nKb`@( zcDll+#Jn^e^07i*0wrxZQFa;+@$ifkW(lsVWk}HLD|21-=DuRRo;%m!mzk*ZDm5|6 z%KF*dFwnyi9;k85$AUezcMjy*H2}GG&ppgh>Z=$CfF!tHX_E}1!7o3c3ZDP$1$aL@ zt#UPsj$S&x2%gO#k9bz4WBQ;Ds+ zWYEY+_c7NJRFvg@+SjQ-W~PxJZ`-4bV@NzhEIIR3*2I-$j5*1)!JY!xE=4|56&mqr z@|AR|G?67u{^*~?l1Gk;G#f4Nf*J*b3OpV}^Y<*6JR=&#s577{cY5wkL);D9xzcvV zAib8rskdU@JpuYOr93w@wF`DLQ`Qj zre7;GP=S`>8X4{;W<}5-5|Q)WtSweMQ%Y zNZJm?T640)8rcp>TO_lz;~64Ntyu#h$LFWc*^}&e&`pZNq5#DJ_jGa&0uD9E2I zJl(#7lB20`-FcQia$+q4#2aBr6C7WcojLJOomAnEnV+qIX==|~y%?9%0cd0~2aB&b zsNY&>PXIuOkl>de2m4yA7>^BB1fG!OHt!Mf6N(a?VlDSv=QEL= zPzmb+_k5ts7GErBZQvRaRlv`zeC_66&4RagJ?pu7%}06Qrl0)1_ZTF3-l%AecO`vt zHu5jA{e1PCR{Dve{-eqfv!d63igF~akPJuoo{v0u^U=muAUDw!c_**R0g_)3z9OWp zq!<$(bQt!L7*a*&Y_YCKgzvdAZWsyWi9vy-0SlUr#hk9CfuXG1-$;VO-)twZa?l-U zalMSF*)M4#@%X(Lg-JXO{mTC)47Dq}Qxkl}CoCG@*ot`KUhq(!R=0wqxDu(X)oDhR z{tF-)<^Bgmqo})ol$JEjb%3tg0qi#$86!%8;cr1Qa;BdQ1`LC`WP3rdr!td?r3g!X zz{V6&e>Cqz#*o)+KWKDup=Qzf~L;9=4mTh~ z<0<_#3F>_svfepPw`jP}{;+816Z1hwHVJuhdQ!ny?!-eiP zEfoE>VY;Az;t!NO?Wa3|t$EW+@2L$89dIQJpc}8L?Y5ymZa&j?-lA${6Ng8_{1Nth8qy*!F<_E4wE@MgX}GvqyX*HJut zk@J6&MpT@m!t00Ni*f>oQ(SBaI5Qn1)g}t4Cut}n$d@Sx2GrzJPwmX_$n?Lo{Y*{c z+!K%V7dI4wxL9F}PZO<;30S+*63-vFMogLt%Ryfu&q&xJmKOD2_nHjVRNC_FiJb*g z9$B*&BEMw&Z2Xc~KGN-3$0VG>$b))Q zUF%YY86WG0-6S6Z-OR$$fM-MU3GCD(4X`L}owD^|JkbC#Mf$6`i@xtCqzzU05kCI_ z@xk>19qn>6nhanO{dJMV*ZkAiyf#}EqeAX3J=x-P)c+XUEj4k7JknfH*pyz%i)(2% zr9S4Ch5V3{&%prGO60Ro@Az&~xQozhBBK#P3E2h|3^drA^~?MPmuJw$`{=v!$|th- z+NI;tQ$to(Bdv?4PbYZ3ugpvBjpkBWHcf#9_?xJWmUZ8ZDEy7jA)HOTqIWGL}kt>;bOvCsgW3gYtt zE4AIaF%U^G);M%RZg$=1HW=nxe{0GiWp_e2Inf@I(i;N%)0lLFaU@24s`~En#e^lt zW>#cXm)v%pJgS0A@L9EOeU|)0161acf|c!4Qqrewnl)p-*WsaU6=|k!c!($m*Z*|) zP}bG^%>R*3C`4g1rDv%1iW=U?9NlY?+n;bmR|?V60g!_KtqJ|NTj*ZdS^KZ=7OH#v zil09*PS-WJX7+Eg7x2PXr1tR-$#Al@%}rh>t$TwU`~L);2hZa10k2=QgYVkmRX}Ev zf~S?t4II`oA2+1;So;tyg7|pduUAl zCMqN(0Tc6A9gv5YI?18NSy)scvQAuN6CwDC;SPf|s2O0>O zJl>q00hOT{)_(wr=7hGB?j8oNyS zWud{X%wo23!!j7~l5oYHt)(tgNd@e*BEW|WtLZGK2!wFuRsU><>F?0tXOg379D>yO zw7z~48pRs+M|2*a%gW1TWk<(UgU{b4<>e-=)7I4*)UI^#uxn}9u+vD|h!=}}G-M*~ zj!EN==UP0?+P_3|Ms1fPDBZksT*Y2EO@tHY-`b@ju^vw~?npPL?3({Y{q# zbl2tBt9fp4brms3h>5>>Cn&J3cwKRRVzA&D5`dNO(NhshL^F5&lD{bte5Y^uGtfHOt+Z4!R_>rsz1vH}-Ogf-*O=*# zE)5W$ST!DE_3{o;7uNYeQn-cD?M{xzBl^`2>?JN8tSy|?4598Zk;Q2tZfB-gLGZ`U zWvNsw2YjVZb(60U@dn_fDK+P*%CKYf2#o zjlC297MTNGlO~wsC>LB^SB5Ii>LQ7uadI>uc;f%AKNtt$!S%rjh{H?Z;)xv@OKYbI z0o#2b$w0mQRR&U>XO08;98o9{%`UF(OW9BhRs@W6v&)i=U;#oBrz5RXRA)e)C9T2P zlaLlp;(7V03<(Z~1tdfvVpOFwv&O9QjvKI~v9f`codi49Nd(qkKq0g?gHc}o_T*eK zavd(_H$QLp998cs$cr}S^zuyXLyV@5v`SRSn+&N|(sD7K4ou-jym%f6Oj`i}2=82e zeNR9d7&DR2i8AU6JDvtZxgB(YD+S*tH)23U>1S=j{JLs-ya1*9-S+wS_62%o;k-mk zVT!prD44=QUhpUDpmcbqiusV%P_k5+dP(m0uLZj<4(+Uj92PJ5TzuD`In-J*vQq0i6_5DI$!UQ%J@rk5<413j&%1=OFIq%5Vl`#<8nUzqUNcW zZxmPuGwQF;;><-*Npjl}`>;$kU$7dF25epZtB;uj%wfQFIHH(X%U=0Y@5gSn90XO zY}M-$u@ z7Eh>4IHrC#2|?q|HLj>bRkA+$o235G?fS~hi-iFV(G%qP&twlP!q(EZKLIwYAV{Ao z2R=2<3&XaAcNxYr~*D1hR7^ z_L1YnX1!iPVn-}}54>?j62fC2y7$G&gQ1TH|6@4}P>xPQ z%LGqkuv!Zp#^9vC-wF)#C;CGJI#W{R6-}s4(l}`4@?jT~vW+uQgGH&ytux3mSGyRT zBy&P5iZgrfSNB`}=t}aM0P<1t4Ww(-<&3=*Bu%y3e0;AJ&5pziyWwcm2{{Mrm^0>6 zPi>o3dwc+V*_0#3EWx3nCZpMMaw*tAr-$X!L&NiQr>pXp?kH9q7J~xLDWb6Ra>;Eu zBrE3kt8LA*&WJk$#h zL=PQNZ_7D_sC<@#_syr3Y&3O4^O7HF$ubCL#e0(LdFuKRJ(C4@#eB0j53i^VVtuN* zeChW3RN0&PIrzwFPba2zuIag=LnrbZ830mLfx4J>r%b|~d%fw|ctrfj>2vsitsXS@iiUov*Biah52LDK{qAmW8Em4aO_ub_$K{hVyTlhFuBOc ztI^=PDXlJQOymrcXBLzak+o^xR?>xxv*nF{oX+_qOPH}Y&iKE^5yU?#X%bu z!4tQ!4`fde-`4whRxHd~DSiTzdouqTGPYOARWAv+i}sh+=FL`YvdVhbTe$YVVY^29 z8-2C5>Jb3n{HO$0x|yG}dGwnfz~Proybd7Xbc?Ew9m`l;F7O-ir(UpVUz398Uyi!^su~HE;l_ewE$}7{$hT)U#`L~a%2+S}1 z$g;;tt@d%{qAhq23kTaEiPdGJ-RyP4!Xu83xZaK=6qSBzkeLi39wW+S^1Coz9yf?; zDHgfZq>xpya~EGMNY9e)vn_4iJXz4?zQOvpr_Zx;qINCXh5fZZdOmZwTxT{j!fqWd zc_EG=`TgkkBU{adt|`S|by>%^{=yVhfSaf`D1G2?H5=K0Vo8@jdP_^9_}9-TW$EA3 zhr$dX1dp;v8NIyl zS7bZyUq;`Ts@G}o_J{cIUoZyRb~!eh*;cv zh(_!3%gduh{hxCGHJ|I|z7CLupfHw<0;`Bgc~$E9FEqb9V+s^Bi0w3tRse^IiZ3}H zf_=%11|$r?k4}96_z{jV-J1vR4ikp_>fhNeI7iFlGqzl~q8j{4>s6BWC2!7TLLA;^ zwDz7ixQ=9Xh1x%%H}cO5*a`CpXxb!b@3{N=ZOU3xRZS5^9TNkU8*1^OvFMfrHuDIR z2?_Pl7X$XCMHP3COmAT=xhMu<{jqxgK8x~d5?Q?$aJzzj9GF4^ZOnMiLLBo_>1hZ8 z`s(X!ImK;+z2m{@hdmYwPM9hQ1?d}z`?x3P5Q~OrZ-c=mJH4iO3%LZF@sx6GowT); z40zd)K$}2rlfP42AvxGhN6nDjGt9&v?K=nL5spK9FHQc-eyYXfxzyU1Azx4{seaKd z6wj16)yJ&m;|2k9&2H3%e$M;uS0#832)9Gq_@2sNvH*nJZ9SlEVowry%4&GqN8~J9 zeCgr0j6=huTaR;C6UM{0FMA5$zikZ2*^>xXhY;N_VBR+rWAlq-87Eupf=P?|27bZm z^_a2sK)4<2I~6AxBhuG7CuaK%5LMg!nXd&jgcQSq+`@Qj5&kRb1)1%WWGIHQQArJW zGGc;?q-_B^CT$JV;@g3<2(Mx|)8jDJR6*KIp7|esBNoFCA;CshVVCQ9cgmg~A+_p7 zUXhck@CbwXK_+Hel1$(i{TW@p-ofMq?*wk>*tHf~Kyl`s_ z@LZN7EbV_4r99s2oX^1h6mc$~>fwG617k26Vd8FsYl>oS7GUn|(>aYET!-&|^a8@` zSnetw5aR?&VV-4`IqS>=R-rw^nIG?QQxkhwx;X1OXwnAJD?b0RWau{UjOQuboYvJMAB=}m&!^nz!j@Ck?bB^ zar4;S(??&@4YSX)aPN7LcX*H}8sC-0>&7pSXMCps(~uGxF^PlHwMqls^a`R=ro*7P za$nZ^hc_!%v?Klz>8~Fs-+Wh>CzV5kMlo_< zyfeV8U$Hk7Kd9am_O*BwKL3?>WDZQkNyl{ErqG{*xzG1TX72T#A(>o(s5t0*!+meR z13>Q?$dwiN(34sl_PRuplgWi?AHkET2Y=n<_+inHlOZwSh>enm_NX^-iQC+sHsVdL zDXwG+bc2~1vVHht_a$uyoJ1>jb8gl=IC;l#a(5OiBHnTx#2_*fGjZ{fGNO(0=@Mo2b!Uh(F zXn2wqUlG`gO$^J|uR{7DtVAN^%?j%{T6f!I0)B_;82QYfp{gkAk89HpOnx^aAF^51 zVp!?q?KZ%e$jQ5GADD}Ucm)3(5>&1{kmz^Fly80W&t8Bodg;aYiW)DXEsXCZsiXq; z#4o?Q&IA&}`{7XkVeryR$1oun4i0haFYs*J}_;|n8H}NK)oUTfi6vdS&wsJOUc-@Lk`Se-Sa8)CvgQ7R}iSpN^~|=b!I7WtH9R z$h>)c?$xa~Nig2TFPX`A6c=@P{YYyOZTpbxv+5RRl2mUo5H4eJxSwTaM_YsSG&!}W z^W14@t)gb=*$czByYI;e0l1*J%bxJ$)K3GtbSthF$sil@kuN}g-J&i5N+X2u`631Y z6d+wCtaDC654 z?9zI7>Kfo5y{|OlVxN3By@2~`OuiddyXD~Yf<{YQ*+Fip@Q+t$_AnULiOdAJdq*%8 zxY@YvaRXkVWp-q1GREw!Ba`wb{s4wcOF0fpe*2@XYXFd6z2#AbZfaSvtg1bUg9W83Os0D z#aN0$qgZeA3u{pBoAAqD zO|*-BzFS44%xjmI{wd}%>0}15gLkpm!D|sO^i~2$+t$zn#ip?yV`hMrd5_%RZAaLx z$^E&dNB!K5hz$sM`~c2cS$17SK5BBr>g@O}?oNSqX|jhWm7FtrZISS3+`m5N+a~IPxP`+b5lC>>=@v#P|$Xug*~>+||8>m(JWtwKdfq(k|SRIFekW z*jV&UmADYRaKOsE0#REGlFZ1FNtv6I_f}?|;;dG)W*`9)lJP1jFs@cAv+6Hh3>2j{@xGFvAf2(?%u z5na6-NSW@0vLv5b{FPM!o(-iHF~&o`-CAzc6f$pl+Lh2Z4I->=yE7yuwS zJ(a1VZc$_vpY}4mp6lla07%KfN$RIE4lfS&0RR$gG1B4a#!Cr?!@sw3PJqwO=fXqe z%RP# z8gOKSri0X~T`V9(&UeiP76P$YSxD~8NBOg&L~~jVeZO2FhIWBZR;nvU<<;ar;-rn4 zdW}zXynZFElkdeEUMQ+-eDV}~lvBEUz)_KmAl%Jp|CrXBZuD$l+GwV=lA5ll<-Ck% zigmFqQi_qDp0P{Is#c+FZD^NJkS@l-$@w*NFD;@?nq-rk!*oR#f!bHAu-20W%lz3ce$RJ@VL`V#?5Bo|jnG=hYC(I-omMt8U)SgD(;pnPapf4pdiMaN3O_Z}X$ULR zE4SYd>vtpHCx7=wL#|_fj98X#{Jf>kfBpx6bXRd5b@{!*8a1LRrEO#`up&xuBq232 ztS(H)bPSku0bC8xa>(R?z|U!+jN00lhcU}^y0_CWnAt8ZtV;U>18xcVigq;)AqO6H zEL-4;@_YOdO|}Yos!sWLJydv1W|y9EUt0Fy_xGzL$0|Gy`1b-sBkxV~G?Uy{7phc& zLp0%uB-)cMf;MM2d)&Oaf6$K>$9@x@*W8{4E3oPV@oS!hg`X#LTP9`LuO=tE9_-&E zYvZVWejnUbb1$>w7Wy;t4bA+gPI)*%a{rSLSo$S<5J<=4O39mX_`JjPuU4Gy%cX^?yxcP230?Q6gfiwfu3FX~gwuSTC63J}qqJezUC z%-zL!Jyj97rsXqaqqEk^&QE)ysY<(QS<3y)gvzk>B?kfZN_{r_F+evW_5quex;fGQ ziGoA!LeAZ@ccTZb(;2;^p5Rjiq#fp)UK-O4aaB`U;N~EJ^(jzn@UOQfhb@#dv;oUvM28unG?z&VrPMltuIsGpOhBtEo>TqRbQ^Iw0#Eg{%&bT|75&oP9w1C5u1%kQ-3~9p2tBu zAXRoyWUv$ZY<~_H#ftYA8OYv z((z?yBXPV-C%Wi;S$Zn?{BWUHy5di2Od_orup7S9JYQP?gq@IskOIC6LIMuGg6pI1 zHFWB;(&KI6N~MWk*lk_mYn}SVZw@!7rNT#w3gWO&FO0;1!V69n&S&;5`yqS$0aEF> zW+lA5!`g0N6SDWk+4?!3bURc|smWUyE;g_(HzOFgcRdgk5`wkmN*5Fdl-Oa}Sf` z<_Y%f1UrZ#^4l>H+m(1{=Zr$G z0~s?I-6Yiuy$<`9gRl6K&#yR3kOUj^gS?e=mn^}g9&G26p7n`#UInqk>|q9NRvoGyw?Ux+{@EOJHA%4WdD*i& z&hGs;k!#C;Z6NbD&Z^~rt8P?fRmAOP$1O+yt2dVThj%Uyh4N8TETM0AW-OlX&Gh)K zml2NK^;aA|IPcKdjwlb>ciBzg{DjZQ^WtN%gHTb}6^dwf0VQ40_vH1_z`S`N{B7Pt z7C!1)dB|XVbG%3oBt(7qFw3SLI@HTQx%(#e1OC8jZI zLa7*Xry-!JPRjecE=r`sZ#u-wk*P`dg59%=`D}kC>^B+j2+i*p-D;W z!`bcOiRNPE5}OJ7?1E{pYu=(wU~1X#JbigFm0)a~9dUBQZES4OQDKAiIxwebp~T+I zxZZ${_H}4~a)I%txygEdt=c?Sw_y8*OZe&8iePV^f>#f|adU+9?P98etBSLuNmPtp z7|oji)i@~Fa}8~?{Yl@T0j5@IOw1`kG1sa`Y1LX(0)IsOvsK}sDnGi&($bBE!9uH! zH9lvZ;P@(ilp26a^ws=CnDen6nL<}H5dzak%hGr{*0x__4xkc!o7*d>M$G&1@L*l( zoycoM5X}VC)(syMR2=XCeBNbyR>Ng!f`gUIgv;&+e*HGfd4;U(%1(Xg7v5z(bMb=^ zna_vw$=uY!zY8`fJ^)*g#8ipS&VuSna~dNNTty0=`_*_+id6qG%&H(GA-N=GjDnO8 zw%6U9z!sIIQE?8~l@v&eCU<44Dzl#|P}5M6s}0^5XUbFedYXm}wU~)cc0YBV%qcZD zq7YA7G@j;XO}k6C(VShWpJ|SQN`88A1?r<}8hf>;iKcq`494i$SHDhdLw@$5e89JE zM#Nc?@q!HvK6G+*1*P2pHVMKfh7CL+Jh7<%uF#Uqzy4cLv%3xh%!aT~ljG}mSZ?ty zlWT+ifX{@>+8q>;Dr^pSJ-(q2%}4Iad6f`tA!7vJ6F1(!SYmc8DJYKA&&gQ+M+q-# z_vZl-;{bm-|%{=0OgZAp)>K2yk`hmfxK5a~FuO6x_vP8d zMCk`h`Q|(yJzVV@>SIIxy1iE&`8{mEL{;s$qlMwtlhq`Zq1bl?6CT!(_c8t^j`Ekh z+XL`~ozB`ysjdSSb7>bRwdVtc{3}0rAi5YA+mw>iJ$vZ6ZvCWj&nj|26N2wat3>4O z(X`}uuI{rHmWv7Yd-f2KSW-YlrX_t^wHY=b1rn)G0_>qhqM2a04%VE&lh+6&fK85HZxhh8|-^DkvEH9Z?04&L>_G9#4`!zO+q zP7{4Y4A+if|HPB1mZu#UkU1UrE zipv+F&(v*h&jD-BQ~8BZS~4~MJZ&=BHuK?Dptq5dbgpKF>|Kv5J8Id^J`+2QKv~ZB%YEh!b{%&^0-zN6Cjz zJwoWJvc$K|`C65((oemcu^T_H8lL?0p>kPeBRl)V{{=BU24;@l(5nk*Y|))N>@ZQ; zgS?@q-q$L!2B=2!p-_Rp>lwAVoEqwWUXQ`Q9^XdOTuf8Ub`y@T0yY{IYmv z^f3FA;ll}4wcjT>sQN2KJZUfcKjX2K*@(*&Bt~p>cvXL&BP9dgI)vl?3VCJDJjl5) z;n%%odBu}+Ty;mA%^R))0Z~+2afOwI?Xeg8y(xxnDuOoR$fSdGnB3sQ zXGok`x_O<8lLMJ9U`k1m2%8RtM^u(YJuY8;T*6(iIZ8HAOoWGy*m9~bnbsvtny#+a8u_?Ll)oOUUMZr9-@egCKsFcaKV_F^0Brz7awlL zp%&!wPL91~cZ|MQ3K7c+-OLPK`Gx6br#IExZ`B`H$33OFj>XAm_|=+EpSe|LG&O@F zk{dkWhDmAUX25Iy?L3gS+L)>aN)82>LU(e*-#kmg&QWxK7N);~kRISqu9!o{k`7)J z)Zw^PNO@Lb0Pdk45C4G!*MobH%~+VthU1|rAC0)~UP<8Dp_x8bxY1NYLopFE+fPcT zY3S6b2Twws(B5Ej=xK&H)3m!Qiq$?SdadU;n#C=~XcA$?Z$;Ij3c)lByN0H z7B;CR8CA~%4S`8%>y-w;ATm4Va$5uG*v%PK8mUX{|EMv2O*P*=T5v+6iC zwOhm^oVjeQj=}W8?adxQaL3 zgpU2j!x~l#sKGHG0CH<`(xB+_(>=8cp;a;-YY?0N*;wFtVF>3tG&Wf1^qh?|&Z^#E zv(eXvLr%3TvhxMX2B_8fNF0sMk6MlyVP`W6*~ zQ~UY4N%0yp!5cT6U$t531c59z8kz;gvdAeW zI$k%r*Z}8rem|V?K0mjz;1Rqk1VaB#;~7bTpi8o{MXd-{)@(Y;1b){M>0zJRP44YA zp+ku-g1V^Sma45bMMG2e$w*E~@vD#?BDbsA5OsGHmuPa(x63-VgVZBTgFlmNgH+IrkmV*Lbfa(Z_^_6s=o$02gREZuH$Py!6TdK;s zdPr$TVAy8pEl+B0JtnuUwmD^`zUIF*E6va=^!(OSFnp+|Yx0A+3Pql7>ev0UQ*{$O z{iUtx;3XfI=h5ShyS7JAMxgX1I6Ihj? zN7_wSroA1*=u_p=oI)e%ytaVQdHAE= zCo6LXW(+Ev;zw4TIsJCPiZb*HWA5{*3VSh1d0so^;8YPfx!Q4vnclIUJg%3A@BnsD z&zXH5myYcLOdzq~2g2)mXn3b%$_I^tDRk*0^cdA}@AjuplxwNZ#)pms~3rqka-b{`P68_i1pKe6uhs^^0 zNe8WUO@&6dCmJA7*!qL10Kyx5MX1Y9^ZC&D-7v-I#)$M%2Kw3V>u`xk-04tN2?}k2 z=r!~P`uGb(iFH|3HVGTH0^xNE3AFDrcOB_{$lE);)P1ZJ8swli|yS}=mE6Svby87oP%!ZA_p}dmk>1E8w3LXMYKbe6-(I<|XKTZblHJkTR zvDTKG61d)lc*lRN?dD;S-%iu-a6{okYv6BK@w7edUU}Pzv@I=Tp0@qrS2>cc?|zd7zZ_9-(eB zkI&{|6PYwmaqSXD?}@N}(U5dgSI*H$Iguc)L6&)+-n62rA=a+w-hS2ZHUnhLcXbQz z%w@@I&wt*y=9~(tE3Y*T=*c(k;+cDpXEu%g)JWqrr+wW`mcVn*B)p`+BJ6>k?mH+h)%l4W1H9TV zz;@TBESsa-f^hW-DY z&##?P#mid^OWr<+f5F1@;%)zLH1zZ8ZdRdlvJ<7+HgWMf=YxP^(Jv_hSI)`)I?tW8 z#*zvb)ml?eUQt@@`?j%Ji_J^QlI}lv;1BfJ*)D^2LA=(q*^vqCNiFmH>P4ZO%jy^? z8Z5m}J;G)Q#GIR;M!e0Bw+H2xMU?j#H}aaz$e@VYly=t5car_*3MplV zcE9C{Lj>>V!*{#viEc4cNtaC!qZn5XrJyftu<-?j~im<%PyX-c$s$23iw>H-b!PI=8iM3x&MSuz}~1 zo9mo~6@hMcFU#jK@lFx?`Rpf$E^YgPI!Z0 z?NgVC9sxnwA$khlS#cL_!Gq9>uB2q!HGeC$m!tTVG<+^th0ux?PZ(LOtfq*F2H%3~ zs7E4=mu}Y*Uv7R~o6ILg9vHdm%A2AHcD&1LUiAqNN(GhyI$zvU{4nr=DXX&D@kOhH z*nnoxE(I%@70tokB8P<($R66;oH}oP;R+Qglh2M3*q2Y0zagzTRZ>s5p|*1zQGW)i(?dFa3$kKO zZQU*8uO#^3CCBm;V^l9s*acey1N;m&%f~d3qAt)wm9qWUj;Y`;3FgaT<4xGk&gXhH zKdAFDSgpKyuBxGjgO2jVXsx~w+)z|=xnKJOIzHmMVGtJ=uj=9KIQ-6&=t?b;%qxP~ zKapRQGSsf!sgULmpl1L8^q9xew$%Rx=-HLSgRKS|)C=(ks#yL5&|~%4EaX}n4(rGv zV56g?6vkAm*d1Hi%_6+r-8K4huXaSpM2eVQdd?*t5u#7FEOdAkoI2zE5e7bahJ1gI zDjfS|KZD4eVSMeMkR5yNi7}#Qd${m%98L)#ds%?~R>ugSCg^LUT~2=>h% z3{T5F1p;yZHf*KPx2DVJteWn4G&D50)jVAtqDk%5!E*o{^Qc!@KE%p6|1cr07NdUbU+%lg^ zy2t$a+W8p)3P@q*s%g2}S z>lXuA@`Z%brqsXrqqRuf2S-&+ccIvzt1rN8t>&oz_fAL6UsBxPPg3r`r;IV__s6r# zrJ()Gm;dWYgiPZ9XT2>G2{mO!g6+0&w*-}oWYTDiq4IF1UB;$mfL<>YkL37Ok5#?Q zpzxFFMoRz}x z<0Sgy>v===-Xv#M^!T+f5gxGEIChk5C9-kR0(wg+kp*6YJtMWh?jeJt68Mt5DbBkH zN%5SSfv6EcVPb%{ny^Irn-;WG%(flO$?Cd&esIBQDU)#nK!FYn?x8@nf1p5apNZ&O zViM#}L?Z|?g`!MZiirUzkjXaQ{9q%m{2L7?0Zm?7_`_di2dHe;?FuIAvG>NF9(A^e zc>W!u@p;=}6hC(JZ4@Q|3glX8iJ6)z@8}J5thuXZQ{(Dt8Hao~OHG7I0j>X~yoJj# zbm@>;bYyU=^2f=IGB-VR0nSlEg5966In)t>nBI6Ux`EmzKW{49z$`6k>j5CmP8z2K zC`s3wGuBz?ose`>r7oy{HXP^pRg~nM#Ltc&qe7z^Tu$^lgUQY~&O)Nu{rXo0jtnPu zSHB6Q_l}4{>T>J`GO;5tngzcPZh71E&#I39bVh7$5+E9;YT3^*CcC>kP@l#KqzrqIY7;NSOws9OQJd4Rk~L`fqzgY>$lnYIb_>h* zA)tc`n@$nExp+ZVSzzS;jv9THQ~xM9B(wq=f{wZBtth%ZQ|fKscpiCUOERDGv{rJy zr)VMp(|w2N-~iB=WEO_s#R|G}hTo<=J2h zTSQ%oLp0BSSP2^5xd#DpDGmGq0ddh;SW&@-f?+1{BGz+E3pb~aT-CAo=Rw|M6|?kH z6;W7yVD`+L1jr)NPH@=>)u$J;rxTpu5hhi0hD`lkS6$L$3_n>dad}+|zP=nhwK^H! z^V*yE+njUrMV0Q0^S%UkH87LI+HR~s&Z+K#`~qyiURS3faGX_Ws1^IkG<@Q5t-ALE zMi07z9R11bQODW`#r&2(oag`&dKnR7Pu;U&&*(NxYiqWGG)>4dQ|3oFwl1`N3ru(H zS{M$!iKKBYaHHli*V^7;9ss zTze%zW}__C`X!{hdnO37`tl>L0{K|gf09H5a#%lcIjY5Hw`zG}JFh#3GQH+y`fNim zHO!k6PJ~)D*ImlI9v@mgBxi+QGp%KVsKc@_e|gljmQqT|S&rl2l(EYCdER0vB(QB; zx4kunWv=6p^DqJlGhww=O-IiVSPkOcU)^fhQzm1_7V7vQ?>&EL4T%1jh!Erw3JnR> zvay(!?)*xf{?7ajo>GslKvewd>7u42C!Y~xt|-54ser|jY<$m1{){UCNXiMD+;zy- z(o*uLi;Rp2XWaDQ_(+FqBFx#fsb8K3F-EK3Rnn*|`JO&tRcf9R_uF&$O@W1tuEkQ% zP)7L^cj$VA-sfkI1;~BF?-q34_`s~{tO1EAdbiaF8Ej^Dx6NIS3g~DO7fF2=^GGAPo}xn z%Tq8bC8P??swCLLH|CK5DOS}#3g#hy70mz3*lsqEtBXLiM$04{g;Vk-p;{XfVy z-ll*3@MR*Ci~ED)OJPZo zsrQlOpqs~$s0q4r9Si)l284vnK4y6Akz-f@#BQI0(}5CsSF^-%LqODRxGB%uO_E|A zQD7o9BKYVL&c;#kQJ_E|xuCDR{K0eCI{zuHL*w4;N!aqD{bhS8&FmE8OjrZ=aw6==z z7v;CPNby&Baz9baX1?{AYY&&=AdAK7UxV%tWljhKMO7H03!1by=Uz@^YsA%lkJlak z))CX%p3XGIbiFKjt*WV6Tl+CKyRxYvbH{X-NGBec%_-*{()J+j&p-G2-p|Zfc}>M><{O6^tmbulL9ShJH=K9A zmIMN4U;B;gkM6JWg5J7bjpZaD9KUpufOXIZ&HP!1rE=Ce7tw=*I&54N^ z4AtIw`FM@q+JWF>yO`d+JQYe=Wy<2DMH-kvXWQ^5YN(!QN_o|LhA^OnV+o@U*GMdN z*qOnCOmJ)*g5(>H4HitY?h}MgRlfiULTkyLdemC=Ub}^aj*V`#K%ZH+dYI5)2_>zv z*2EIx_n++bmcJ5&WQl+TAx0oUXmG0QjD)O&d(D~F%uIdyHm#f~nk_RSG{?f^P;W)x z5phAs##5`8@K{RmWhmCrDcx zh4J+Rv_OEzUS3Kry`DBfp}fr&I9&1DIWmvygvrwD?`Ym+)6jLXW*6{X5TcrL_kgpm zLp`7E5B)e?CU8Y$`^q~IsAN|zJ=rkL;uH_(}{D2HbMS=}=^r_I&ZV z@vun01wzbe6I7L)fO`ek9CL1X`Eg6oi2Z2g{lvrcsK7kK4%Dk-__rMz3bnX-F zy?1;os0@qK5Y3(yHr7PO^x-Me8b2xnPc=E+@iLE4anvvmJgbk*4&1Q5`avn>`UWQy z^ZSyZbV&{v-|R3bGr>5q3E@P)o4RqVzEVwAO)g+5;MH)oiIsy%)*M+IBN9vh z={d8?Sq*B!uvZQ+P`MQ)PM0silX>gn`A^h%7=?5RYG0!&`TLGUn59v2Z-2#(|Fyd2 z>PN{O9#f#x_Hov7qlUA%>!wo~g8kSxSC*LSiA74{#?2`GjQk6qS$RBW=9zDG4Z=nhlEC2nl+UOM4!H4osd;llaX7)! zr>rn{C~Vvgww40>Pd+mJ2+Cxfif$f3N2k)XY-$gBBlk8K+KOQ z#V^k9E?Ym6#w+<1W|y$F=eg++Rtyg6{Na57PU7nM zyPT{$MWVx`e4iOYUNZfn&EL=%B@4<77j6-rpNDTZI!>IZ9gL1zVAOecDcL)nX}laz z0xiYT+~kH?(%!By6x-WbN77rHHw~%}hz4Sm$y;0f<`UWhJ5dYuN`W49YC8#Hbocnm znB7;6zM4BR*mnx1)1-CW7?J}blIQc~f5Uy6*b}e{*>1T`=dB~`1s|(y+h?>sBWDD7 zoo4No4w;Ptb^>0~(gf_$<}fEP{?qRg2JRRxEmd#1hVqK>_}6GSwL*@8?@42Bkd; zQ8E?OkapNAnXZR?$g7L$+3YP_n${JMBV6<4)WSj9^($ANfc)sHhH~o{I$oWwpvG8<&+cHxb`KTt5vw=`E=3``LX;W2W+_=TkNnsR`&j;YhkD z{E7iCji8{_xeNQ+cgGs7VQt^?9%|c4FJ*I$60GprNW@-v2ky@4x@Kyb_&}SfN2_mg z+xwKQXt|Jn`_z@iS5ItZU7<9F`^Orr(pAN)fmjn0tm(@6X>o+_4OO2Wjf{RmAKBZ5Ej|+}fIjm$q z1P^}n`DowtM~(VIzd++rg@yTI+e%K^{g`bvlXp?%OXf%O&@w=c>T3I#@KpV!6B1GW z|r{`o^0ZL(DZ}(xJ09BuxbE3p?gc;V8Gy;1wi)T0JS0^xr)Gm z8?6<9>~SBJEhf_TI$8HQe4WEjw`N)UZa?XSnBu9g&g1X(O0?m7xztt=;SJGkc;a5g zFdqmn7LHxWmhMIyDlOZg&X+sgP0;yHGxA{7wGHeWc+u*nLv_3EM=+2BZOi;0HVgBqBymM(&hk?r-ky*uu&eNlz(%hckj*DjUtb)KXlz0E-<=dx(N zM)fWWc{ARtwrBDXa1#gFk@E~uUE}2HYA};4Pc%oadSrupl9IwGV+McoCPAfca=`g0 zq`xLK=kRI5Oo1B>VNO$fL(} zv@hqK7;Zvt)Yg>_2Y6<8Lbb7sgQeyyWMEHR2o>%S^f=GnhC%ZZ*15I>Po`C_WY$j` z(_Ux_gmP~}?3pSmVgz_OgG3}DotGDIdZLzhlkNMbI zT@mzlWkDTMCozj-cWF;+)R7XBmQm1p8i_GAyQ@3>bNuBO#TB!%1+9`@1|ocJ1}J)^ z3pGP`@?ok8@dhi4;mSf)%&&SXc;GAVG&#|;6#u@#Qz0{Sh(_6u7%m!1zD43s{ z3~6cf7nhkem1p9e1{-?^@S3kv!W!wJ9k~V8t`nl$;P3lP)mhL`$9_vj{Q+)GV%sg`^fgO!^*&k+IsmWOCeKc@saD*Ugg-t#7aH$H6mW1n)d~KbPq2N z&|RX*H|*Sq+y-y9Lg1GgdwH2;_DPUtgn$enCya|623wqa zy`X{RYNy6m9$(c}?zp50D(79dP?oo8orZI@n{90&~P0qigU z0Bw)ucc1q%AvT&R{t}-80rBY|zKU{7E~3D5UUJd+@bBW2TTs|LAO1M6J(xH$& zi2M%#r26jw5dF)0^~u_NV9gKb50>=jZT@~U((+#<+5G=n_lSI^x$%cZqqng8q2?y; zzsC)!>z8@{yP74A`e$13FLIAKDr(gbI-ua8Zsf?!)Bm+u{*rQ5c`P9{FroXPpE!y8 ziFtHU;o0q<6@->uN^Z&Vz)$#(Wh3@K!$8J5p8y5wUzh(G>%u|!wvzb$Sv)Sf=-+D8 zldQng4KTh$*=Ob=dnSoXT~Ea|u5s0FHT%xcB?$yxih>>0`7<1L~<>X{?b?8*d0J>9(d+ScOaX56Ef)8O0WFdwd1w(%vV_()PkirS`Kil^YHEV z^{)IX=Fk(^D`9mG)e%{spOErf38I<6ZCBWlZFt&aX_4hfS{^hKA;gR)11k7&tL#=;R#l-Bq;5re@W<`# zv9OV0n8IpuSy~vIaktmxBD1JCtsb6!c&PyqSN!DB3aq%Z69w$ZoJ~_%3|GJ;Gi2Dv zZev7i7du88&w&AkT~^e2Q>s2-LF|x7b1dKwjbpt5Kx(SmiAzs!&&XZKas~pMhad>9rF0Xg*jIs0kzOCMP?v-1$(HMS& zUacNjnxf;vbn?{=0lyu##4!8q(fOXD`f!pQPUm6iJrV@lq7JoG5Mv;FwcG6H`aL_< zf^tlEXm3B3Hderc)J!&a(3vmQXkdAnljs#>v1K1(N98_8n@Bi9vi7a(Lf1XK8PSYV zcOocV+H=*&TY0?bPv;SCGlDNfu#a@BuuxLQmemK97P5Fc$-UzqvHofW{5#HKjv$C^ zX=&zAv+Zuhpr=LjaEyY(ME(SOso&kl%gd;1VUl_6P7k;=!_i!QGm8uYiG;H<;rQ@( zdLbe~XR|(5cU4#?CX*le#Ohb;!#-GKdM+?I9Oj~fJUjWjHRjok=!3 zUBjR2>`}n|N2+|O;71f<^B(zUQ#9=4WC;WBHe_&lohgdX`^gJ;@a4tY-)jQB2gd2D zqcC>hUYdL1)3#RBu_d4jwGc1d_P)JTMz5J?_dhu$`&*OX`hCreTPKQHhnJ;pu=g1Y z(th)EGe-i*NNZbYF;!c1h8ZC>9^@% z(wk2lh1k_WwE9QJs0jFUh;g)>eV==5VHw_$Dk*fe@2>*7&{1i3pQ_9Z*+h73J^JLj zT^eOljdSNndh8lHj}>)q_e@D}0^O`KOXl(y4r%7|$g2^(Yir|&zsfTk3{aVuKC|7X zZpdq)Cz5dK<9*dd!B`1k-Z)QW?&lM?;lJLckemSV~^opfxCEv|H?6-|W2OvVA!YFQ7Y!h~94k;*qa6(7$GyE=^`@;^|2zcj0Iwf=2 zVGA3{t4MhJs!x}p=)hsjbIcjW7HMH`aG0wlrFVljo{(VuO`SwmU7QI#x|r6bA-dmV zYaM#~Q_wAL|Jlnz4P8~6I{UmtF;dE1*qT1XiWbm?Ry@~eV`{y2p`>-`^!)D%K!7is zv+K~GOS9&8_dS(__S#P^=$$+vV;+T@Y=(Uy=u4utRAXpiV_p;5vy58UC!req4pp_*1vfUtjpln^Hf_{)f@`|k z$qc4D$fqnWH}djriV50;PMRJ5mffKwuh~nz!u;2Fu;T+WHg9$3JS7cb?SYRzlnZ^Z z+`IPQt8dBUV|IvL#WGm>h;ZpD1^@g9i^X#_KTixrc}<18#A-5G zyVnB+B=Itj^ z{A*GS1HJ&a<|aFE-UhE0>N==!@e9Ogxe2-JJUL5hA5P5=O%O;w>KUQD-rAcV+5;im z^IXV&LY+6~RO5;ay*=pdgBZWDaM$&KaDZ_K^lnooy9CfG!UojG;a`zC(ciPz_g$aZ z-c~u|yfSO8&bD~rB4a#B+HmWjk>3!i1xptx3t{kiP_HieGY}z2~12 zQt3jHf-?DTKzMw|hAT(P^M5=GV7TbbcB@(7 zF0o0^arI8?iZZ_Iu{V5Ww#O-CDlI!Ut4F{EW?m+{OKh)RsQG1@+A##4)QwtK;<%wH zFUDbE3xt|urYG&|_NUSR)YlK&FzWzJ*%aMhxH-gs;J)XF>OCSvQYD-%vDIegd%djg z$M*G8eu~00?9vLKh&czVQSYJ`mWB9&yG_G+K#@4_ea0%BVv75TV0GY}2D04I@Dvag zeWQm$KP$Bm-I-#H3bpW~v^_v($8}1s@JZQ4uRrfg|g3WM-&_}joEApXYJWl*(D*76kZ8iUzrPg_n5}fG^Rk5*h zuk$jf&a^+ASfCJx#))6_L_77Gz2}bz{6G^tDTDKE@B8T{R-az1rkM0~#*ZVM-q`N9 z0h4K(xdD^A#k?nuv}?K1VK-lp4k|+_?cCnPqRMZoRCqnv=_Mp4W4-T5Qg@8jFu&T* zeR6LnG1T#wG20Ofdx=}8Z=OswsiYc_QiXsqH>_m+CtU7S8r@y$TI_r4IO6)dv2`1! zkB+5N74V6L&lhA*vF4JImRng6hYFbK`^+Ws z7R^{A-Yn}%dmP6nC3b1^IQAHkDQquqtXNw`WsH91qMbh_pjF-W*wNiuXiC^Qvau@M z5xEu~EIMxLPwOX;)<-f}BmauJo(zGCTjoBY_&t0Y6CA~nXTPgq@rJE7NWRi{qz;V~ z%|W|y`{Y$5^1#(bcsPIc@Vzrzzmx`V^eiT=&I)QQ;LL{U7$FyKK&w$ha4=nwC9oz1 zf7~%>qeI3Qy@5S=^OrN*$-G$TXE(&N#JP6b9$Pv2uRC+FflW;9XbLeeLYncJylGj7Z+`^gOglZIDq< znki$2q8h)f#WDYl7={we0>_s8C@1(wIZ4d;EeoCkP?pqFm8`?nreNmsZf-z}GXKVT zeYuQSwKhX0R02b?aU4}&m?Vo23b83s~f`r zG3odG{C??TZqpEs(zz`ep3dtfK?b&R?HWzP+gX=wkW`3Je>5c`Xh*he=O_1d=gyIm zsS#N26^DjE7@}es|EjXcZc_98nXL9w4GvlH8ENpK#f#m|Af4&fU-LpthjQf*va66N z7^+{If**R|lN!6?W*v^TMb-v9pY+*$URNzG#3s7tplZKZZSo-9XjOyO+h!U8lgm2$ zaMZ1DY+rtn2HY}+civB-a!jR}z*;drdvKe-7$PF+!-huz@3#fUk*|$%lPi!1fkDXq zjcR>d&3`LVW*8%XFWg4&vhHVXspIm|%L&Nv*r}d;V*Ef5L(cU05EJE@vS^0PZ}?&d z{4fc1=gRhfL8c+3T2(KzUDN=toH`!ycs>L0qa#&VlENY#l`-DOuyk`H?x5F?kggR; z(Lm^XL>XD77=I3UCvgd zX+zXXRv*c~7d$#M9@$IuiQ_~mJgF8MCsWBxOOJ1{jLUy{*{9nuSP8rAd&_x?k{Hhv zXmqfHVi50G`iFHmRVD~fs9`3P)I^KiNez60YDXF#%ULh&YDrS8i_xS(fhF)}_UY@hPj62oH6 zY2Z@jeq<(uI__CAL8=&b+ZKtLT2;zRW4=(N;x}A^iH~F%?c|x6?2%@ChB-iwc$4JZ zH)+jA|4gq0|1d$uKOqP93$7bD&S*DWvY}@s8_}p2(mBCKN%(@>7U+m|3QbuT&-3%R zggSr;+Bi1+1s<6^fSZ^Qc8pQs^TEE-DjvCC7OI@fy}JFbFk5aRBK#`G5w$X`@owX- zZQg4Eho@iAr2*G9-HSRW+c<6?1mafFHEf^|m-bELbHuvY`ryt$De#r}GXk#56=E7U zN#0+7MMjJ7u1ikt%-!NJ0o|Ag=xo9CE?_C+w(-XgZuaZ&8nRzQPHph8{5g`0!}DyT>@urp7+sjS7CLCI752NsQz~pcY?Et&fb9xFaRKQSav3 zDQ_z7k#GIa)$7<;cqyUlcMS)6bL;uLv5BRHTTo!q4axLX!%d@}|`G%V_HH-^zH<-x%x<5&s)bC>J40VbZAaX+CH4VWH8@INd zPB#Vi6Qh{5bzMMFGc%fYWx3)Y?0Cw?bRLpS^mc^T*L57P2IanzzZeUYRlcS*Nv0-o zZ2*RBL?C$PMhjvG@p7W(R9q3z1y--d#)wa+UCYnOHj05(4goYJyX}e9t2O*<_!8Dq z8imZQ%kJZXEjPE`3Mt^vU36Z|Tzw^#aq>vp(RMiFK_|8NOESn0#lmhM5{oQyy9OQn zqT%|RJsLe}v>PBXU2lpBr1Y{)4lgVm2~~;#Z&X|Qc=VuHh1YHI^mAFlMQ-Je;~!xm zob164dSrK>N~Z#V2BlkcUN5~Z(qD4Of85gB|98n65AkXq-ivYSWK4>B$)ehKAhgf z??|trVnaYY;z*DR#xB$^gk=+%_P>YR65_Rc+Dt6ZZ0Q(j;uZ$g6I>Zbrw_ZJ(Hz?s z8w$NNQZKG}m#7K>vTA*3XgtTBrVjKvg7Vd8pNSUCbp&)$4AilQ@C-Zf%y490JB?{C zqH8aSCk({p8~luV8BuM?Y=wT(2~5b_%hB~#1K^Ib0Pk@B;aNNr$blJ-cbhhw1;6>Z%)cbCMklG}*)uo&_QW3WaoeXd z)_l`|oO96uZALu2_i+=sX3)%_HGM40XPpWXVR*Q(Ya}l2=(Foar|<{Y;awIXEiu#9 zm47LL!=kfkBjs0_^lp?r=&CdLw^A)=n`@i$!27W?56MYLwWpRDBLVN3#Qt`H#Bkuk zvThWgEk)~{a?mvPT4NyS67X@uQ~R~qhpk}9)gRlkR47jH8Lo!!&r*+O;VC#*?!OERQCm)N3 z)I3q!D^@c7$j=Ft8-UU}o`~um#EWQ7GNpwC09l7Xqt&PKj7~G)KF+r8*I47&-t~(F zM&HUiW`F*#p5~R(fCU7{ zBCLNS$zb@Z-4x=+%6IS4_E3RczDy5Bz@wo|GC5ipFF2l2E#>O`s8Qtt#l79?_f>^*@#t>yO!!vgE_{s1nSD^P=GTJbTW4A4xmI~aHG{-hDt!tq5jA#VsL+ zYgkv{M*QNBtpeQI-dmu{g3P4n%b}6w37IP-8+ASXJE|>`s7jF=K>kfn@}Uioe^cy? zmf$bTTsCj`)jYq=w8PHaxwor#+UIJKWK(<8>rUh#4amRg->i+-Xb`yq@^921$(&>3 z=36b(yG2L)EcO*S(YuxhfAP6A=Nh;_NX4@_8mQcq*BX#ypNoz0JvJe0_hgBVtac(Q z$e-;}j}emcLpQSK=GrmBaiA?s{{Pfm4)sTvQLW6JqWey|+oohKj#ThTHeZcDIq0zc zYEGZ=_MGjj`60Hm0<8>KEJ2CL@j2?d!iS3+PhLh5d1>rqEz;%rTcx>8;f;)JZnV-gnX~puFI>o-rBc=qhltB3rTV} z!o^Uo9J!{oh$RJ?vk}@{sO$7^Tr!_pWCvmf8=H8KQ zA8=$VfEG_o6gg<2yEjHZmqnivvOTmK&d>hA% z*bq%y+O7g7=#*xfbW>-(m@`c= z(zxTtR&PWpPBlG;fx|o%ui#iFi$ud7hWuB>l}+LkD5Nn?_>R1&o@6$(A#GKUfutZb z!$ysfTMXxwytIV>Q!?owoQHs}{prnqll9lcR&%48&b0Aw{}5L)ES|gcJtIZC3;IIC z0W*8Q`~r{+R{%6XbRHmlFMpO;8}L&Vh~k9Oc$U*+X6R74b}=} zK#xZG4Y*Kjz5cRGPtyrlujR^iG2}JfmU23morc3RSv6H^yaej&iE8HCn#`&00PD3` zbR!z0qRFjg!rfAy#@(o8vQ=!LCLo^7_$)Cs0ME^8SOfV z6Yutj<;8TBd|m(E3$XZUvf`hZo6~?^<8!ThA2zmR+&Xwp_;B==y=`OGY!FwiZY^)s2flg}A zSlZduiv_@$Z6bwGwjoeM6Q}f&WlpnbI^H>E!qN#CDY~XSPCNaN>N5O!jJVYLo*|1& z59FoflZg4{Z;i(R^~8ZVDMV+ZkySyZIpu5?)5%PEO!|e9%H`aD2l=9gYLN^_FZSA> zZ|Wdt7@PR~v;LvOQaL9NVeo{D)77(-0(NS^!IMdv*U2Jm$9#30)pSaD4?ol5-2AKj z*OhkCq`uz;$PG9V~ zV*p$JO#xs2Ged~W41N;NhmZPo#mH3uDg;vnYRwl_!Cqhr8JUUcjZ+v98|Pp{ zFddOS1Zw1Pq>OGX0eqk4H9>n1q}M0=PfXtf4IV3En})^MrWX3x8Mr3AYl;^SrQSck zi*8g>*&>*aCI#9vD!)HJ)}S1--;-UL7od8xyz#c(ceJ3whpgd`zj)x&{v9%DdH5LysZcGf4edOrD21I)c4ZGGRUY#(N#}rN8x|hLha3~Wzk|K@ z;IVz}>WEC{zMVX#A^LTjj#H?{vg11EMl?~gBBacvWk5&1%qK-4#Bi!>E6@7Z^6hI+ z5lIb84cd947#lu!`hB~4KgcG*SJJB3GOkpA)8BdOZ6?jUCxtk(3X}iaKo>RL1!|)> zDn|E2)T_;E-e|}Dzge*z@H=~Lw~8IE(i_TU1V(sBMb+;=GGa{z<`>l%U*z4YZfDqX zx#Ypk6Bt2Z${qAwEnilW%k+ti{ZGYy#QZ1D<(+{0#U1{RtjCh&slT4k_71v9&O+0N zydPqRB!O$-8$~i+A>lkeNHY!iwyn!=^Z++9C5V9b{c}B&b^2R&8d5E}hrgbl!EV@t zYC(JwF*M$3KjhjLqlGl5^xgS*Juvb9?- zw1m04A1};KBdj!lvP{ebcHxsLKPefVA_mWNB0`zErSE|R!YSf}JneR*yp2nXC_gN5`o$YA~?e?eW<~ za{M2HTL^i&Cth=28=XAE=-@V>o6RJ4;fu*+nZRZ}6V&WJXS*DvwH74F&&U}O9wq}F zPt^&<)yabcRIXZBO_P#Z-^V{d5?lP{asKMFEv-{FI%4pvuo-$HCbP2bkW620CH5#E zG==1od&K{OenNF>#;K?4kA_KMZG+7~%1{^5KkzalbBxl86=AlZTcUJL>YT6l=~}kf zz-6GJzoM&KwdSCJ?Z~+f0Tp~)u^i->e>4cdevxIUK&nmUeX7lmhwkq&k#RWexgE;8 zmyTlAe#T(<^r^LSK9G73x0%9XjGe_f*Q-5dnb>e*v>H=F+#TMO5(0r%W!aGKXXGyg z#(hnLCtX&bHBbOz-OgX3Hg(HDs13uyQhF?X9eRGjIc|xG?xWlXMOa+RcDnrMpXd2K zs&+y?M-<0K1Q2FT^>u`HhT*=FJ_r--@ZY&>Nlfw?h8%M9Cs?k~kaKurok4t#MnXg* z*lk0*MK_bjVs#cwX|u75Q^-Km#b#W!~F1IyJ!X*U-cN(|R2IKR`k`hLK1j#z%2tPWm2$ z_~68a8&^$@zI|NAb){FP&nTXVZTBOFA0^^5_5Mv9V}#Ad>|RQj1G+Fa zpbLB-X8V!3E@k8g@1rzW$D=Q?2|R+jXA{^XXG(3H72>k!i7ZRd(1W`qYh%ICkeZ}J zqSzP}(FKp%Y6Bq{0D0u@5lYULe=e>KB9uPstu{GosP!qyr#Eq?v@-gM-iq69cHZGp z`~5jU6G%m!4bS01(7(2dI?nZThW?bdUSj6fBKa8XLX}8ZbS9-QuK$!47xnRXm9>?6 zazCP@2g#&(-fL7^!tOnA4Mn?8t7qaokEi^lq}rUCM8eMjruf|I4^ynQxWa>{W$HsK zDx4sdEMF&~tii-8-Q!WR*^zWoK0Y$i-^MC!R<6u8Bc-4e{KdYUQRCDoK8BKp9<=_h z;2EK~NzgpnJy9H4fG05H4q2-IUezpg=FU%g-P{*z10CL(vn&|o>a6T=rK$7Yr8%Oi z)8Zfo+*ww1E)_>HA6Vvv<=Ijd1n%2-E>-D|$Qv;gm6GhxG4YZbyy2LaI2tarbh<&= zyFuMW#{U7`BG{JQ`XP(>=8S&RrVxqv5(DdVX>?q*vqqICfH;`;Hxn2Iq;P|jqMQi| zH6ni8e0@sHFTQ2q`W41@A-AnpXHT9az>95N!QhYI5>V&*6%Ey>tH;9fV=&<5#!0@5 zjPM>}OTnoUNZmwG_GC2N*N)`=@^X`FUxw!094gFU)`f0}IdY!tK5~!OhP5*ZOCX>R z7x&koZRZF?QFk?wo+Dr%`Lp2bqHA9~ia&m2C!_9+hVr{L*lM(LW#!N3N$_4feHA-` z7)R@hlATSrkCn8BO6#Pba^uC1_9+d{OjUuf97RRU0Ef5mc&EVWFV9`C9@-dI!rJ99 zabC&w6tG3cM4r59fA!TFJ#Jw4hoDi(7QZX^JdJ=s_h_qfm6OGIApb+<%b*{hFnz7h z535&L(j-`{@}Vu@vyY&=BZ`idKn5q;q?H1vokbJnTA5X-Kqro~32FI(*4dGi0Y3zv z*DsB-u+)9PG|DW{7l-sY^MdYWLYWDfwh}auNaMP14l+4kTGoKBMo@Y+%>J{11mA?kWweYE3Sio zbg!rN{rFr$mT)jZ3R5u(Za#ucZb;wCHRU<0;6nqm%b=@oGui6>4Gg%lOZ-9t4CFrb zP9mh;Oikqv9p9fgvbW)Iym`feY#%T1s*mW`#oig=}{{ILZ7xm*03mD zy!r3WYHL+F#i#({A%8&o&=z8XJSbpIw3@+av^H=#%`X4vrG!7rH$KXJR^^(*NJK?t<0+ zA@JiI%?;(4>EZp8!`}ZH>Z};IeBe4&H2cSF(pFa=982sC_Sk>fEl%dUqdY+4mi~sV) zUmonul((`m#?TlvD|<lYd^ier@G6HGQ}|+mjR(71fIt z`Ljj;<7PF$cKj!`!n^nHS2l-JgEjt(&;Dg^wsPaap`k&wV{;_(o>we8nK?l9( zzdh6SYA>gQ9!MYhm!<2M%tL@Ml1KHd7^c8?hdj;WFVEazHbLh(H(egz{0>qqs6lz= z667UvPWO{b?bg zMuo-Mkr2VgMZx{VYa4g_NZvBxFNx%*P2X@jTTWOzPRqt4dPJwZ?9%A`C=ly4X@z1H zGH400qMDk7u$sh%j32OJef!!uV%`lZJ0Z>qRQ`Rx`qvs~_Foxd#P8TUyp{G+r4uqfk#>Rt-8bskPifz-vP?z{}yJVodA0F0+d_7bUt8PTM=vxm~=h zZreH2LQr`-$Nkxl#q1S|XX{S7KqP3O5T`ZD!f1LFO5eaSz&UHr6E{ie#+OsEcW=Wc zWpG*{gn_#nR-;YsmSY?`ZZGKxN7b1san8fJGk3rL@C^Y$>Kso|t(BqpVw>6T^&X2J z_tmj7SC~`j!MPV4xsRm;bkKt-bS~_VF{-Aai7izzVL3aOT$=)KyvOoqyQeO(da-@6 zXRcRzb3!B(M$$UL_?(w*rKW0c2AqmHgGvjZcFIsXxnVdxeLfA1YN>kX+X1$2$l%vR zYI=aoCr_6B@>Kt^DD!TxoxXl8S4<^zeEYum6vukM&xrRI?DYt@ieckTW2h-? zGi{fuPo8N5i9BugrAHTw4)U&Mm_DIo-3&-g*w@bnWt}}1==gT~`Mt2t@Bu?Mk+AmP zd!C~zyhiW#Zs@xPJDYS%n<{x!A{e7k7=jF1;Cd=m7cX83lw^@-#%dMl+TMLz-+x!A za2ZfY&u|l}yVF(_>yP&P#bh!3z&S8UP-30tql3Mrq~*Bm?k;iNxIR1|{3_LjtBSIW zLas4UR;TN82cK0fNCbk+yT$N=A!B{=SM?eh`8 z(;FMHCu+uIGB$lF(k<*CLL!?1VNFH0&#mD5I!mX-?>Qu>KrlPGIxjKY#$Otfa>yh_ zI)>6(G-|X@tng9Z(9`;)1f(lIvXVnXUmo z;DAu&(S2UoDlc8VU**6}rhl+rzbI{VJ42V}b}ACVKVY=ML-EbxiEG|TTv_-WAJO2x ztyRFd718`G;ZO6pcEm>K@$6cX{Bo72CW#u>*jjYiIsEu2J`(@t`t3sBG4{}nc}*EB zBkh`LB3FR%4hHmYys%=);$aAhuF?8p*VG9IGdrY@YX5F5Y5Um5avzBB$x;%RmV9%B zxg##U7-9qG#)b`O)$3IkhiswqxUDFN&(=VL@@G}}N-w`~lrx7$aGg^pYD7p&tX)`4 zU!!wsingCeM{05H^AV(N{M1G{+?dH_DCIqlv@!HHsD0r>H$SlLgsL2MiA6ALEiRcT zp7a5>_{jp;udVFH>!EU^0o{^^oOmn_&7VrtN!6=DR-~+C_Ogm2tam$K~TkZlJ1_JtibC=EA$t?ahs}!_M`sSe#D51L7W_L)jj!_*c zSPFHd0r#Ml&s+Q4ANSY`b(cpJ&i5zm-tx|5Z@>8rw?vkeePai=PnfNUta+X2_kpO{ zs#`w~;?184TZ|K9=Cb4I_MEXin+_MW+@+zF6c{ha1i|~Z?CWP-Wy{Bst=?v=FU?(I zWq0HF7ilT4NHVGpFJ3x7Tk_sFM1Y3Oc(MoP0drCu&dqx@eLw^I_ z7U|_0A#@p?DI6vh;E={>F`+vGpL4|Eo(|U%{ZN#8b}Q(1j~k}SeFr3L#e*r*33~U? zL7L|HEA~;tMqet&ca6}q$MfB5&?nTpr?r6_R00yw;K94BavSNb_e(BKRDKIs$Sxda z=~LJr59CHK#!p?Qipuf9oxVHfsVoD)Rm9UOYRp1S+1TfgqoEYxXr%B9W}lNMehl0Ry?cj<2_ zIAFuOP*;6|-Yy>nhQ7}Jwbg7lq#*sm=f`kg60Qz~#DZd1vxi$6rED6+j%I@XTK_Z0 zr9G?VUQg!7Vi^Gj{#!53xwm|73`|IwVK6@fZT-Nw(4d?*^~1aoMp{tut^z#2+AA5+ zw3s;oWAYnM`cyM^zEyK&5of`18CFraj4c`~HSbc#>e#1>PV=|m?)T4S$4;-Np9NQu zTnX4m!q*_$zC_0I?$~CPq3BoA9J{kg_hoQ}W4f(#T0ECXlEvv~qTzDmYNn686t4gx@o%Bv3KkyO7ibKP zJr~|ZrW0mdd4Oi;o`Y(p(wZ@ZKIhDR@(myI^s+Q1#SL-iM)TZ9k8t)>u#KpVf^`e> zER7^b%aB+y>r(Gr^LWpJUN30@8Q|k4uskCK?W3Bh+n2k*`umJ04SNQpSmt^WdPz5NXxlLP1AYB!j%``;Le)*Osb*p1yMPP~U?r^s4XT|SOljxHdT z-2VxVxGyna%ixy*Rqr9rKGSs;4{P2guvKoc5^+vQyj~krz`qTdXf75F8)cfIZoAf0 zFM}#UE$k;Cxprg5ebag^)5=ZmXkGJ+QHPj8s}#Bn=sr^kxiSJ($xd!~<_LF6XW`TJNC4_{0Oq-eBv1d=(7u#{Kj zl6WgxXIH{SVBX-)KexrsJ|xn;0iV%Y7iKk8P+u2I+m+mRYf}wNr7MfVuS?I`CbMtm zdm$ky?D(=$CHCh;=;Psg*zfInUoll?37r4?3Z7 zh<5cx2O|dB%qhP*BJNBNbQ+%UL2Kg+FHI7~=O0+G#HO6WXf~$Fm7;|Y$r!lVb?FHV znHHu&6=(^FZlwGL#IE=vuq^35x3>b0^V}nXIDUyTilKP*R;#04vGST2?oX;6v%Ut0 zJNsm1t>sQi0;we)a-+>OQoxr`6T^nhoarym07KSZP6Bp@TSN_r2o6a5& zl-@-W24iaX9wOLdHlfMLZHLhuOR}UJ)-H{n%FV=IDfgSpMk$@ARLg{L@ye!2lLTx&`kh_rAVtL%;PXC|me zy)YPK7b=N+EyuIg_-CAME6a*B zN>%$N1~0G~C$~;_ch>{eoL+IU*@A?giFr0X0Ec*h^K`<2uz_EzBCCg3YD(6mI=Cci zQLD@3%O|||61-$(^7J1p?H?b(URk1 z#D%wT#z%%fU8iV|DV?2*sF`2)u_FBXp8NH_l! zyH!H^vPY=}T^WJ+4umy^p}2kHFc=AaPeTD|0=Ru`7}vO|$c!CfXP0Fr1EISJtbChh z^Pe3|ay&zXW41g@Wo>KWYyTsPI5*0p`a_$u!Dh^bZNH01X zdc*9PcpC;2;ub^!*;x&pYM;*I+ZyVD8Of*+q-N|9$p`4_ul^L2g+|D2C~L=f_c124 zRpY@n^(AWL=-gwcFZwswTPY*-tXeKyg!bA?4j$MU2IVd0Q_S~^4^wfyX}fX36F22Q zucvhR3p98k&ttyOzU2z87^!S=q=O0Yp!DJK7BYVMIbZB~B4c>rz1y*8 zl%#T%jRc6?qGR= zRcm$Wr^24NM>2*bZ3WpH*ILKz`8Vdjp{z@$hc5Kl;!_lfaorKH@e;TW$=O-=FIQa( z+NX+VjpwvT#XPFYdYHW#b5{e>xZQ$|rSVwx=&z06^EwsWQzdW+tDpZK6Vp1%U|2f$ z=6E5L>LES$#!Tyu_4$QiNO3+*wOZdgni^=WmXz4;mFKtSkW37!m79ry^j!amrrY;k zZjJGIbJyG#(WK=Au^Los+(Xe4X9toFGtw%j2K=`-6Xb!GLW7wJ7krqO6@IZ{foHwL zoC2_mU-mWCd8=hsTSWiVS;@cVSR?YRblBKVD6L*~-HEN?_w?vGi}R$#vC$b`!r-_t zZ)_LnYATw@(#I5+T0uVtCP*6_Y6JgQ4L{`Rj3bNw#LLONMn|zm0S5)wl~7=FGPoe) zS=>zghaOI8RO76xE3TDf9I zXB-<-FGR#lvof4J7v-b-k{D4K2HW#p50sC-Uf5?#*!VEwZo?ad7yi6iOKn1-zu*$@ zsg825IOt*}9@ZFT`yKs4OJ_{aFnEeIw$Z&qdW=`JSm&V-aWM;0?OMf5gMP^{6a(Xx z%&jlaxxRVw#bVweB?sDK>hHd>>{iq&mwpyQ)#IuL$b+t1ZnM_CKo;-hc}ql$YQilR zh6~Z3w|M~OZFz}j!#*uUP{6$H0jAoxwf8|@H`#e~`gFtWHFSWvGo~o#H-scdM|!#i zWrw$Jb2*B0t}Z-xj9Iy*JGbTL)|M%XH&zj*SjV!%FZ&baO zUoPEzIQjF3Z#ao_6xk9^?d!@-7wyZp3BNZ#ue^Nw3UT4OfyL1zc{f$hYS+9)>k+V%Y{2-;}vx6 zYmAs*?DE;zQo)OG?buC{F3&D4@hRFR=Kdi8Xn#>GcM)GpVl9=Q>}|j$?Cv>MShju# z2+y{K&*BYgJLfW_wI%V2l8TCl-`x})WFVV0F}0f!>&Z!tV|qd?Dc}6!MV?RQ9q!&m zvdW6HUt`SMVip8E;AVBBXpP-Y?gj2(aVu9m254*UaJEA608Mzrt{5!F%#Rbq-Ygpc z5;Yp#cWdF>HU9BRcCOX}*EvBXx1me6Tx>E;m~C60XE98vz+C%GPCsedyv%c z`@^uEXJ0KwEmyp~Z46&iPi)KU*%Tb73&#)m-+igPT9u<5DrhJcMrfs*F2nQO8iNIA zr%N5&ruYh}rj1b{W4*E(*el85%rJ+nt^|h#M%AZ^(3j4m%&Kp#g4OR;v}0_xbO>9o z1a_qS?s$`OzMCpSeXQ6M@hsv6u#ZoBGuX2(WA(ta>#mCp1t9wOK?#nSBDYnRz6Z@_ zdh2Itn~BAK+_CG4ZC+@q8h=`xRC6sbmfXe`tG~=Z6={v_G1t;F8aPx_8{gU0|IzP_ z7ZNfKZ^7H@INYQ2^+V$iTflGKGF$%RE*;I#sk?5rhBD_ue#VjL-k6rHBdrmdn6OQ( zm!U)favF`Nf(kRp%UR33?-IzIsqOGP7fDvsk+F9~%_cqdP0sU&c67`Q*a8M|AH|UQ zy@wq1riW9*ZPs2TyJz>8r) zC`znvy}6n$<8Ip$s3QJ+Og-VM|5#jvfBoD%S+@RLKerD2|5t#}#Q)eeAJS$x;KlK< z|1?$iZ}>?xaIYi#iA;U@6aRE1_XD?y+<89HtRlw+-Rap6|HYjx-Yfq}CH3!%H1+Ar z{(b%LaV?h++3u$l+uZ}EmanzajQJl@;AGZSJPxn$OV1S#`+^uX5^A}(bUN2)jQZ0X z$q4?qa^j&Ab=a^qBx|mD?ra+W$2_U!Aod>{^Z~NdfyL^OV81ONP`jIil6GhQ!S((f z!DAkW zBeQbeFVI+pmFrtUH6-6Je-KWG)T0imMiwYGTxAVmQyY1YA8oUUc{tj9HF{| zqCpA6nqLn0my?%^a%SiFJ&Ff5b!QkQFP$9ZK1u!X6VB<1`yS=J>ZSn5mpmp6UkcnH zO~i7^-}q-310h7b9t(8=u*#L57^RQ^R!*MXO!`lL)ItK*ZzwXR62>kms|+IZohyPy zaV}3XFzD`cJY!JityIn;++XG7#V(D;iEf(mNZe5UOs<1G_xmDa`tyw}rL4=o2gm(S z67CA!+Jnk@`^FQJV9fNv**3Z(#R$5Kv|jv>SmTp#ihxu|#ME#FUWM54@t`NhAe{X; zsmQO3a-f(l?cp5g`u$`>i?dhd0g4A5auQpPl;3}UU zK_>`^iEvJFtEF7Qz7@fVTW#v9!suld==reuF-=v<@+=czn>u^!H_NEhZbD;wtQLy0 z#eOsZ3HLEW*0z&OU=sqmFTjI@u82X{9uIh+`UUcj>L?7B=$nuVu+?|?UADk!^IxL8l$CN(bJ&~Kg^}N0s zk%{v2!v$G*qHg=~>g$jaCp+#xQSudT6rl@xJI)x`EEzOf@bc6y8FY;OM=-(2 zMlKM<;jm?FeM~AIyjToY{FSaV~^&>=cq35+|Jhz0y=o0{rO7H z!{3wia6bF2Fh+*Prs>Jm?Xq;0w>`V-hIMH4k*p=={P+j?Qz5e&T&tbD7J|`I=j6sc zZ0umuCazL23}@7)o(b`02PwL(r{G}k|7!JDHhkLEeEPeYm=dfDtBqV*JC!}&Y#XVW zJ6b`d90U~zi@7^_T8g20-yg1C4=7D+1g;5|WZB4S!Y?6+h@Hi-%{E&x^(MdcicKFa zFI(ykE$!736dA65yp?;VbFsX$qe|>jEF*d1qj7|_{2f)5*o?K|m!4r3TW8-lyBx}8 zpNw^^x(epL$c=nT7m#_!W6e(+4>)q@K%H;iR?<`hbt+t>Okwl$>aHPPLjuzq#NVTi z!U1+VrloJUj^mve8HMnhhdK!*96GS}4HNq_+wqnCIuZEfPG8zvn}1%ii+Vi8bQ1MS z(UgLI&@HBO%3aX{HIOVt!O$Zb6AJ)%(&kthJjF4m>gyG>9ER(8O`Vvh1n_AZ);Kbb zgy23_k-%Jye3Th6C2lcdWJykco#1F%PgQD^7Z>r#Lh%n%rCQC$iDHY}knxUMrksSw zKUK6=q&|13jtw$iHwGriiqw5LLmmBu+(*)C;$A;JX~)_G;^XvLvE47TqNLdy{U;=D zQ1m4swqE^<6%l8FMXyKxPoqT2E{Yq+FIEERO^uy7Y(!dJ@;*`|(*0zJo`E|`xo(J2 zS7q2$!764h2Uw8h|Di0%5bgn4Yd-Y-sG(Mtx+28S)Z@Xw*}7TUijAP}WMF;+T*3t- zXUttV?%VWE+p%MQ9=jjR%Nf#6$-yo>SE7AM0^j<|QP1UTYykJ~fcqaaHkN*C`N!W9 zH&~%^XU8OF`c!?nSKFsR!p_0`G9RIRu>cV-$e9DnCuML#;qY;`u5tosWI#8}a{7AS z-lC_V*j~(2=BwvG=S32p>rx(-kAO|n8#q*Et=)cr8V*+9-tF@}v#Twiu94!f1xvga zy%vrH`gu%qp|QDZZ6DswscYP7Hq9&ctaF{{*<(`9_U)qbs#I7LXlD!DIrkH6(2u(r z9>~3D7LDPVQHkRUobAM$o!>VG$EZMFEbYpZ!tx* zK^tV3J7u_9rLEI=P+q5X&JpuaxXVLN>O&YZt- zhs@y``rbkAC)lUl3L1W#A@9VI*`R>uKJ9JRWZfW$L3Z&{5FE1RULD-^4C~?--%TAL z*OKJi|875vZiqfOpz}a@D=_woc}&4z{%xSB_VC2;DVc}O)^}R@nD3vieW0RSH`BU5 z8tK_`+-J;Y8pNY_Q@IorTv{|D(|1=$fbJv%a9dl%uEPpw7<3v_&&rEE+dXMf_nf7( zh?K&*dWUrlwd@|h{herm{T)->-LwYMb>Ij~BorXP2uiayi9Z>kDA^3JsHNSox~t4q z-WkM+k*$yqHS<$#0s>SHI zwL|J_=Y&UyjD;65(n7z|Wr)$bj-G?G1%$xC<&tN+bXg8dr|TLUZ-(DdgPXE*c8(f_MXL5 zr6zMhG{)$6MPZ2p;p%*4MDAZrsdLfes!=TYlvo>qJ> z7RcYuDw;(7w0>Qw{ayoX|JImYq6U@vM+Syweqwdx!Xw$9>x09O{e4H3f|K)dWf;0+!qyuI3sdehQ4_mx~QVk-=r zGWD38dep<@=sEpvyhqi0x|4}GMN187h&T9j^C;)@rJ>RM?p-xjx6gjCD0$FgZ9s{R z#%VDyl=TO(&!)Dpk}_Lvjhnx;FlxrfHn_KUa;Z7uHCYi>%W`U>6+&0I?y?b&QjLVwkrY6gQJNab$8U28r^_O@Igzxg*( zaZlUrsJQ3r)Y0qV>t*Gs^^r^E2X%OZwYusJ0oK?E&TBRNBQsrlhE`B>(3^JUsGquX zbw_NyQhiQ+P1RJDZj=G_T`xPvNxPlC>_L!?m-EiHabkQ7gJrU}hf}fx^Yl>xJwvF{?O^?DM75MZIFnm{=$2?{^VxwB& z>qt^;>w`|N-;Coxo3XH}n$twe$dV*(&VNWzEPXSo*puOoGtQE>AglwSN$ETDdgGJ_ zo1cW0JXY1+_T|3eOtUgft}JgdmEDg=mm`+jYmeE~1Z`yMtk}}bYGq32ge0AAF z-s^S`ShqQuuJR?VJa*s4a%?ZhI(oI(U9HI>nfKXoZ+V~4LKEAxe3gVtPb!!F70!pg zwHmhd0Zw~DVMUD~Vg3?gsu46e+u&&5urI6&U%8JYa&(n|n$3ySyP@2Y7M&AC%ie-F zLwfBnq>M}FLgPOFNX;?d3UeLu$X(2#so1|d8xGIc^Ks)y+?1KY^LFm8PXKx9z~ zh%77`hD-j17Fu>Duo{q`eIH%pc2?V1LMGs3)%eTQ);P zHD>Dd28dcMo(`XUF}Ho(!SpuslpH4jnG0)cJfOWpC2r)q9L0w1Eb1PCOb0BeA;!8( znkC1*JyGvy1pUiS=oCs6&v;Y@e-eBr5i0{{yGufLTI^+)E-*eS>c@EL)B43~$Z2%W zuPw8rN3ikr?8kFFFTpj{7X#^+FYm$A)8c)KF)V6IvMAnyK&7BY7rfO4t=XMRgDP>C{kc~-rqZ6L=gejT_O6*6%NfZHv6wwQ|JDKHN3n4h`n*z;2Kr%45w|WAljkCvv@>hErfme! z_ZLZ5c+he(OyGDAWAiQAL4o7bkk)9zoB43rDCVHYVO3a7JUbV2o=i*`TNRga2yA9F z8(JQMFCLKG5TZIp&(O~c&N{@}&Lh_g3msFLEY)t6XB$|8-FUFdSO<40-{k>vj8s7X zGnMaoa9+XlL=07^)@t#t?^fdDs{NfM(2YMBh%kkQaGGuK^8Em_dA=vdaYML!zfC%h zx{&kHkWabk|uO{A%5LC z*E5kHx`fqC&RBU<3b`Ja2^`)$rF5~hj72EDf(h3nExr4UNNx9@`KuwY)n8%Z5pT?A z(q)EaZsJzam|Bq)EH65~an89=iDVrn>M@etzb@+6qvZp89lN2rz?}`usM^egx14TyyjL3ed2v4^l3h5wg%yMn zU$gEyJ(f07We0QU5*GvO%I73%93bo$IXkWu`4ztHt953MH1NqfqY-_`7##b}P7i9U zNgY6~mZo@qb-g;0I(jR`Vv2jxWWO+3kM~_{fDa|dcdokaLZ5@h=|=u^K)pR(3gFzr zAq-5`vnIN^8vnnrmsNKjrK4>8{MWU#)W#H}dpbt#UQ}}QrL+@<()>|UB| za%2VxHwc?Ia2Qz5Pmpg5B|Yd1;w&7{AP);=XinF715GVz!eR{n2N@a91K|LG6OKQX zM?PXUX9^H{MUIy-JP88f@*MWq@uk3&4%S$61lUHUeAlu?!}-E~BGCf-;l?)ETtSFn zy)t4ayx2j#dMN7VDdh=#X;`XLdjr25M&)T9miHrl#FbW2-V-}UgCO2fP59~i(7TuBg@xP_CX8yn>VljHzL;R}}PsIP<7Z=-;bl~6D|J=2lzbY24Y%-l%yZu-H`BU}B zE(5^qFNSf=1zObcIh6qGf-kSU}ncEHLzu=VI?4AEqM*)ARrxlN!|J2$3xAnit zwZafF$Tkjodpm+g&ite+No{DBcipkI%R4+q%|*muVacu^mvc9lK5q6}UE2omT5vuk zHU19IPBlTsmKXLst=u8M>cD9ni07o@Ctx3--%9=6aO$$I#y4s`7GN;W4`b6IvE$Px zYeS6BfN=*?V=vM*uX$@3a~j(&feFGM93N->i5VJQdgM`g8dlb5M|~y3$qv{jj%Iy( z%jkRNwkylt8Zk0Z7T?zfYX}xRRI8bJvT6R^d zFsz|*XYxK~HR4-Wo4LW=r`vmYD8eco)7O~3BN)6q1K`X&u#QKaYercZ}87SF)R0L(A;d0%(O91Jng& z9hhUf9d!sQL&1q(lcfS6;+=K)V9EKhR=u@as`}(7%}~env5FqjQ3A0#1fE1JKFY(B zCan>O0&uHhvEut~H`@9S@frF{e5_{Gd0`#>>^)L$ozn?kQIXnZ#WyFjXAC{(b+CaY zcU3oaLrF!-?vVf^N0A=RNy-9$LDpMmER$g2uCsA>Y+LqipGVtdR&qtnN6zYE5uKjR zNByCu^ICtao!depcIy82+$5n@5t7lOyjj}h9^(X%~CcX2Q7;j!S*RBun418QAw!53F5 z2+GEBKP?`i3j&Lb`08^pc+ClF^bbmbHY!E&O?aekj(*11K^#OnYQo3A(qYll*>h;a zvqzClG(#`6?CSL%(BFrRnytbm@)q=B^Sbfpt>C4K)S!eK(_63V9nyr1UqWRUu4;+J zH#yknW5##j=@h<%)Z=|2V~eBL)MWnzK3$mU>`up7REmi#Yp2fqSd7LnpE%U%6>0|z z(W=zU!bp*0X1BE2%ILFKonmqxlKr1wDrCO9jP)Bb4MOJdLs+=npWH$ehbJw+Ppw^t z=}`PsMW>O3#*}XMF!6MSj+o_Ef9_RUkX;vd?%&}G?TL>kEe9SvHOkC8jv2s5+JnQF zura2q4`0NLhJ~Qlr}$pl7y?WDA?pgw{Y&*Jr8b-FHCuB}{&BLFd752bZd>-ooi&~% ziw4pUqs(tfe6hqWOX&scal!#{flI;L08CPwI{th zE9l;WM~=tVvO99Da=-PxTRVGtnMTK$Y`c{%@ZC1n6^ZDzD0n9ds>OPzujTzLUbP$C z!%xLPwr5&}J6rAZi`!!_4YI&jmiaL2nAi>x_le>xz(39e(&9Kg?{Pd_{;=UUkxU0u zQNd*$F&vF4FPV+;?Eu$TF{lQI>d_V)eKQ8k;B>suWZIXcb-;>x5S|2l|4qP7u8%IC zcVFUlZd`V2;_!Y}NrvWQQHEriRKLy{tx{=!Ud!fiy)8VlkG{-#JyC)oLAGz2*?N*G zD*AE}6HA5ANszNsheVeebCgMpXXZf zpA)qEVBvr_e@IsVny!O%kPrk-wq}~|=`}?f4zcrL@9errcmzXg>;xSTW@Y9l5MKq_@~tHfEV#^L;@Z7R%dSgY|DMHn zzTXejp%irz*8?vd4pYEs{v4pJDl?_o?&j@sIwxwccD>mk#6OuAy9HMl6NY!~Lh#L7 zM|VdI0&k^OYwEf^O9SXxQzjUXebeO(3dQ=*G2ix~d3w*06SnVk`Ob=ejCp~DnQ1p> zdt2(8ZKP+$4=Pveo19G)nPPA3(K+-OSsy|uyxUFToOSti67>NWTstm-L{KH-iEw?0 zTctstcz!SjggL=N-vI?KEm_DwFn(-J3H!tkjU-Gl_+{GH16gC(-G&*y$1!2^th$!W zx5;sje_D&{34hf>e~z|Ac;5O)Dp_}=JPK43wuH>^XBv)qWs0nyf+z4VZkIYaCv(2d zs9PFXAIvE?Szi$zzsUCw?tCbn`wB!FirVwEH+WHHR1a0S`NtJJY*{>moSXmICqZxR zq~-KP-4j7y4`ARVwq_M5(95CV>>T^%tY+xYs(2y@nTy}*cCs_Pso`BN&sQaB4#FtTV?hzU zoSv?B{Et)p2n`(>eaC<9_1{1mg8$szTs+5a->l1=-1{hbMwSMpsPJFSSp7w^F>xuvCo-PU^-|z*M)6YE zlQAfBD`%Z6(3|sHbAY!J^~PhNxy%H|6%zL9afR)l#U9Kwv22UcY=TYjg@V;mAl^_a-K<_%#N|glP2R?B>Kjm(u&qow z{CVoGM`=>WYIR+6h5Bn^FM+cDtM=DZs1I>>zd>&akRz(<{xiHUY0xVL|@3=J`c%8%#+@BYv=RUM}f%XR>hl9Avg1_wd&A`0x7oX3*$3K;;#pT z`Lx?O`D5vQk?mmVbzDGToZi1XQ%)p`5_CtqYE{|mkv9Go-TQ00n8_vNkr z7fL7AtXT7EmV+ZTuDU_Bb?+h(9qCa@EQ{#yOCwZuNw_)K{F-FzM-PhL=` zn_hOa-M5aPWIC<(@#{H@y#OvQ;-PwXc#9p^USts)6B@(K8DHH6yz=g=z0YDEE}$nC z)E1AJY)i9MjR&wGmxk8~%VWINua{nMtFK$yf){taG7qAVPg|=Lu)4~*2ZmgpOy~GP z@f-Hbhk)r^&;hX}U^&N;Fvh{?b=u z^fINIV^0DBv`n7dbrGe8i_$%v*Dm<;px%J0TVE`srBS55lfqi9FW21q`&w;vO;ug* z&4J@E-ua$xqV;6QRocda5pMKMu3{u<`5cX>iyCd7O-dpx)(Z;4lUdCdnjobAU24W3 z$(!^CBKqs8|K51^m**=uhX8(b*e^7GSlSt1wzZZhG~h(GM;Ajfd9q-M`MIYbVdex= zscKXxG3u+u$nuG6Q%1zdn#i-LXFNY7o(2bZhyrNq>{ZG&0OpSi*lVVl#F1ro8R!k>Z^Z zq%K$_c@P>4AbvS%r4|Y)WvS6LKnOI*$;6v*_O2gEK0+h@F>D;2f<2MD40oYfMZ2yG ziC1DGFPncR5fB3X&f1pBF+U#y5yqi zmer~>A%mV$*g1jIHrb;Lm%Bu~L|A(auHuPY zd1eU;1RN7oef5D);W14!@f(oopFhpJGPoCaoFVuN zlxIPkMOF#=@ECn=3GYyH^2brd^%|Kg8C0yHrAyK0Zt~VpRbfN3uQ0Y2!s=DJr;v1vYU9(G}FJEMWrM?pL)PFdg3*s^Z+6h|j74 znwrE~xcyoV_50LFglz+)Pu=qqRv>byltH zesOt=+SKPAovto-%xT3`pBL&m1S_2OLh5ClC%;6}lw9ZR-PSdKKY}4&F0gv}5`djU zj;fp4t9m;j*YMh{Pd-z~0Wp}w$w7LH{KN=U1X7sJR}d8+KjRv#D&xv`TQNL*oB3x$ z&iSBGlf2WI)vU3?}j~L~%%yqW>#kIml*(}X$*+Z%VKczgm zE6IbX&tpljJv_1IpH?%(-x1b~?RDIlN3gZ4nEI~yqC=~soRH56%3mxWk}+j2#Q$K%JjPwl^8={_JLphLPN-SZjB2a2FAH99Pu(cE0gJY{<^)Vz8nl&Emq zf^YuTvT{6VrtdsIk{LruT(JG0a`{hGHP`T&@v{Kx`thS=77yq8lNI)$3tpFwUCXt3 zzu4p1AYW7wQ%8C3oPBHACa|x5=17c^!n5)hd9CdM=S_m1`{CX@Q((Sgy~T#AHizzA zv5scg_=PT&dEd+phTe@a40*?5`D)yV z@&^%Fj;vI&F_{ZK3}$oS*+?7yIBJPDz$7!mUbyG^@$+V44($rEGu?`ymU7bBS>&v_ z_N7K*YGb`o1U!9&gUlB_Par_EnZWgme2*XE?aeSKOJ6V^K3P4t?Y5+6pnb)KBZgdq zy0tmnQza|s0n|u&1nQ>wv8wxn`X%2X#$Rf5+9~a@rIHUk#QP`mdEF%q!vJbT%8k7R zGl__K2*#X2R^DLgP(_-yuSCbRiR?9j7iDA8vmFcjwGxZH?-=h6F2kMrdg%BOux@Vd z+F!P$f6GI-^^xcrs)4bod%(2v&J%8>{t|3V%=ZdVxo7*qeg64+-K@bO$2o@#INZr7 zWsJpTyOqrJoqVbZxkD#1mlEkdc=594ajcTe2=RP9`8OP+2;-*Ko&#m}$uk<{5TmPr zR&QEkJ4Oa4s!qhHoRHtWo4xl|YgiFuZzF(Cy#dRhM``6<US3iqw8Jq@VY&?{dKPH|G1-Srv6xb&KX7LabGlH2Gi>YbLelrJUkH+r zQme4M=OSdeWip@AxWUL838e0%uioKw;_G3xz=)1qS13k9R~wfMl2x;WRvx_VlT zky+;wQy;?I)1;YnK6dPqWFwD>mB*gv7q-B*5Z~`@enh$3fHSoLUYv^WZ*^h1Jj{9> zMHHwG!*U-tb5dK`TA?VjwSbam=HX{P+T|fe%f$j{fbZ_Uz3}cig)40LbSN#GFQl41)FXWfjjNNol`O{biO-&p@#e&eDN44xWUxs0w>6o6L=mx)O z!dn6qEq*)ZYx4~cvf)bxjR#LoaYEYaYZlhL4{7TMsubF9oZZOM6W8L97&>W z*j*WGcv?r({~}A^&0k|%R5|`mI8V-V-3|)3qtR`K5uKv#0+|0e&%Gr+UP%Wl=z`OB^K8ww8SDH_tG~zvzrrZ@%c5w zTfOGn)qm-EMh3lx)FwE1IT-qTtJdv22y&?Zfk-KTfzm|RlkMDJ%Jk1=r^2TT@h_aI zr858iRR7LbEdCqj@;`QM)4lM?crRs?`#{ie_rJ2G%jJhpOL~8q`5|w>I&;YS z5kztIprEZj@03F5w7Kc7u6-my=JNO4<86xSI^s`Np>nfa&K*KsnnK%Pg*oY)36IPf zZh)wm?RCa^?%?ACpUlV0v0C=vOKY{q>Mgj>Q$X$Wn%;byq9qz zslrNE^1AUsM&ix{Z{zndk&vaiM3&0zuo?jNa6BNIvS(OG{5vfiZZ9vY0T2HzVvP-Q zqs1PF(MD6d75RA=FN!yvN!hZi6~~biIWjM{mV$7K-K(tCVx5no6BTP0V|Wx~{Bfor zZU}&dmt~c`6l(7W&s^p#SvBD&$%mEO19g;^E6!d|dqz$vinv8VLfX;yhBvXUGO*xj zbrPC436+tP1H(|6&AR9oA;zk*95x5=2ry$!iA0rD0r*47jK|E6ag$3#E6_0)1g{1P zO^=W%rrs~as#7E;=YEKtVRMjpU12SSTGs2xw_Egu+lg++@}7{vEu~_&GL91h7d_** zpmXDa0f(^FE$2wkB4?sz2e`w1;8@*=Hrw=i!!kS2l)DmK$A2mN_eskf^tVN1thv~O zK+}g!HHV}tx696A(O9-SWI9=ZJp%#LKUOcG{c9G$e@S6Z-Is*l6P*_&u=g5!Ox`>5 z;h<%IO!y1V#q&w2C9#U$Z>RUw+HDSe`eT?aK=#dVE9a|T=Hts5qGK@Swx|!?nDA^V z&$jsbD>ke=4nJk|NH=wE2H$f_qT~nf%5$Th7FuN0z}rvfBp~fOjKS(n+{ww#^U?YC z1|-j`-GNOxcgD3_{sxg6!SQyz_cHwsbi#hcfMFbB3&|KR-sxMCunhVq!#K`Qt3yK8 zkm2b9-rtiu-z<9;@2sl_zEvtY8xjGA zaThTJRVU-Rp8HrKsUz_Zx1Vwkcrvlr>`b`w2D-z;Gvqbip|6FWmDN8P#-)e`{r=9G z#@DevV#VGQt*>Zxg_w_Z5MVqTp3awYD{Xw%T93=?#|(~Zl#i-2b@BQvgJ-gGfVhf` zzCJLej-Cz^y5zUxqw>h!sid<3W6_>`A6Q3>rFJyS1sa2&nBATL--Rb()LwW&L*`n+ zQ)tj@u;;G3^jIh7oI&PL+2zpKfhC}ZfLRZ$)e1-!5KBh2Q0R_?YuW~5WDH1K!bj#b zB=v`F#!brn6?Dl>Cs5Xhf034(IK;$#YzZ_BCPoAguVslUeNUPCcABLs4Xug1^Yxx< zvY-_A&ThS+`1ajJ!|Qbk@b)Gol+Wm!KLFm<3tb80muxQ6URBO71`IeeQk(P{OL&F) z&qSY^FhFt8?mqKwHYzO8t6vn(aCf>qnFd)@n{~VKv|N!7K>;L0_H)M?>8|7~qb}sm zS9f%F=NX{*i5JPjV5}RpJmJKo@7r>@%LNwYjMjAaLA+>8 zG&n#Fs%EG7%(kFVVJ{Y| z9R}KaKN>3%K}S1?w8~vtvHM$!44^lHmlXHj>q3#E+P%u&;J(!~FJX+L$33-g3{&6wA0pNq>HI!^krMh6Iz_1_a$I??YDvYM;UH=7;y z(*Pawk7D5VBq%AV(nQm|x8Uhn(hyO{p3dlxJevVWZ^tdmgv&y)FQ*bB$fL6bBokg_ z#qM8Cc>trhR$%9T*mGZAS5pfZ#Z|e_M+`61)hsF~E|u8T2XHbX7kKjzO*c9awGaL1 zL*{S^cwwj@p7y;?SMjL=-qfiXXluJ4S%7F^E0ncZ0rSC}wr*%v5I10$II$)VRRVF& z0ZgCRXvOl8esyVHfQ1_6 zu^Lbf|7BSaAxUDlLkYz>N;cB~G3CD5bJYM{;4X;b7SvLV+F%HIHh$FzIGWTuk;Qml z=5EBV(GMBgthiDjI_##5aMJT|ShlrMcKyW3MD!R_?=0*Qd08bq6ImWf67Z{lg zuU&wp52T=|Tu-rDR_rLhp2B#|Nxr~)<2ZjPvog{J4w?mS_INAcX2-i0dO)N;Du{F3 zmUx~ZGgpRBIA*9Q8Gr(+jMNWFLKL8gzWm22i-iQ~6JQc|A9HBheNumwi|lh<&S}%B z>yg!ceO}VHBQ64Zt6<#zNFl^|kLsoGx|uhfjgeOz3W9Gpx{)H@UrJXNKC0X?0!d?$`SPa;)nC#`5Zgm9lC(jInvDk zL*82k#kI9-y96g#kf6bW1$Vaq2`&MGyF+kyNN@;Fa7%&)cZbH^AvlfGxV!b4thL^~ z_Nwz%ol~`cy!)$f{79NvA<%P<9&?W8dhYw)qXJcBka!3)&Rpgydcpb52c>nut)4CI z^L}uU(2TT7hu4y$7tBrmE!=utj5f&?h z3u(rCSu9m^kKf{^6Yg!y7-Lo%sFOwhujGri!Yf#jbCYa_d~hU% zhZS&pLjt(v06FA_mM?MzjJ9pN0#=c_@%yA1SB>WTERa`_k=TW&Lfk9{5^5}EvJmV> z^@+S?oF*<`FI}cD(>x61M4qi8cbLJQAli=4+HdRGznm^3p>#miI-ZBL6d)%fd}^Rv z7k3aFwNWgUyP=^1%8DEd)vka#v3P_$!U-%&eLW4ZXnL@oEjtL!wLxRaBl~`xjm@q; zY;t@7^j7RJUPdJo_i%7t+V+$^1dHPIuzt@#hpgli^0b)@UpF=QY`;1!^YM4Ma?SA` zHps7BwVaZiBjbwrv^4uK+7-TiKKB5Q#>_aQanF!<1qFTAkE?7%eE6g*gR% z0i)VdJ!w0*BnU+uaUmfh{#Bn?nq->Y-Eu30t8!d7Yy7mu#1B5=CVlYDd@BNob=kBg z+|iFo=(~KXPh!$aP+$Hn~8;@CQAQ)4w;4#KU7QftnZf~CtvGzNL&g22c zEg$+oEwiDaBfFnQQvy5m!|gI|TRX>qOAO1SYFg3J!;EJ2zRQH$Rt19qSr*qfVIlvD zTFmU|&hOUh+|;7ctUl9%diTSE*uY}~tsfj*0e2DGvq?_2XM3vZ6Sw(@2Fkj|V(7_S zH1}OS2_?iSgkLv;eil1~rEJ9PRPZd=aecIkew9)1}vKPq#I{u)%Z*<}0 zhQr~A!KdEK!9=mO(J+?ZG(b4YAn<9ln`t(2$VWf7!m$YQWDQr1o8W#~&%46K#vsyj zfRNTw5n_3fU*{T0)uGn5K}bTRBXMhFq7C76c-Zj43$*Qir4@hEim^pmsV&mi@ccwY50 z=BHw`u@xCiE^{RYL7?PSkL@h-4zaXBKewZ>09$J9*lMDbzsT}XqBqObN1$Pj(5-0C zp_YSa=n4?NAcLl@qB?pS^5>4cGBbFQ0=n)3$9gD z37wWmQ!B^4rCbRYIjuZAem$$uRr1%|*c{olN5x4%V`n}v&G%yk`RU^(Y6knSC54Xa z!l)cnNG-qOY^znKcc;nh9+3TE`l93c{Ha2e*U;fVOd5*7FkV$#?zOBpO|ux_1F?Tqs4f7cL;by5Mi`4$PM(sPt|IeItBT+#Nbd;CDlDESdA9O^0``OpAIX zJuY&gM=-DpwbbVoA1{43e$48Md9EGs`tv)E0NNn6OwESa93(Mj1Lf-QdyxpU?a-8B zhlCYy!a(Iq=6Z@a<_ka@B#3_%eE3e}ma!-rgUS+Y(XlJdkW@j$3VPi7_*omBz!$x= z*P8W78+7C>5JpIi+a!bw$zf-l`tp$KYQEHoI1+*%lPXx#p5P#H{5B87oz)b@Y05qTJrbiX^i93ud#2Jy)FoX>| zI-*TvjV7ujP+A>e6VG~No9M+SRC3`bY=?+3&ZKYGhS(70@thN71 zCPPkZLO&`pooyRb9p*J`kGM!W8sVbvXuhT_ab8;V>X8C1Gh+zV=efrH;;TxG8K@Uv z45O8U$E#-*vD%!`?o-%5r#E)&pbL~ZQJ74dx6COSg};clepItA6*>0LW;R2|ptarC zx#Y{-+9Y(i_)9I+aDHAO_3qusDCTu-D!tvy-|(k~GT225>65o5(sOtEhC5nGm-_N@ zZnaf360Nrbva0CGrPPex5QBq_;kpkOnpnc?Jt~|0l+nBKHy@J@%p`pg9_*SUEubGF zDChQhkNHd`X&e`NRFX8jqI%SAUTklFHstI3oe{uxtmLr48UkYKhTu!wkrU&gIv-jD z7~9H#tle4AtQJ^p_bpZyF7Uzb>rBlDa<&nzv5zfcme5IPcE`*%w7M_%%!c~qCh4i4 zND3|f#`t!LmTKQ^l9|EXIjq{;H6gFt zNp=^~E6jLS-RSQGFcf`&m@7>3e}<7g7;z?eR()EOcVG1OXV1t+D}yrO%UH449fMr$Y!DQmf?wIp>_-#u(T73)m%!lI4(=2zv7(RNdiFzcOn)ZPO(oP=VaIrI` zatr8a-xNgkn{NM%oFOtD`=L%XlFjU^zgWF41H@z;9T-K-Fp`83j1fcRV{={(30>R{ z2&Oxcni+bizsfSNw9HES8L`R8?Rk7#4|cA|UA46M%jPX9;*SD*aEh=TIqL@Z?a~B- z-JzPWpxM}CFowp1wa&s37O7V`OS?H^)UU00U3ux5`m(Q8b0@!~S6qHnU-6XdxzR=o zIk2?M6PPCy5vxz*+iK{xXBIDgtN{i5Q2OG2RyhalZs_;UI_$ZqkTy7n*GuLKfspPZ zQftq}K>18Pj!u4z%0Ow&kJkV~glCp(-K#&RVrGQCBYU1vH$G?xhvwh3EaCi7A2u5Q zL>fWEt~umKz!3%aAx5Jv!b_wj?;pzz6LG3Z@sL{rZIC3v3g;*F?TBRAiBS7ui1pDy z?i+@TbO~C8QM2p?A14pf)ya48gkv=6AtCp#WR;kaMtOKX0Xm|;h)}`n{-86T?bpk0 zBmu@=#usov?_~Veuu1vZA`o6gflsIzbUo7g8%-bp+stuBM9+A-uz6BVM=dsKPBuvl+E>6E^Ye5QX0iOi{U zKXQtn0yJORI!XZMriPsT@(I+S7ef&;%f&b()ju0`YPq*x{p9>o$kddY_p+sGuBZXctE@- z4)+wp^Nr77cT9?144!sW*;IF;o8yk~IYQpi39CR!V7eUsCR;^)-ER`RH9vaA9?I5dU{OCGtw=KIAz{|h0SH|7)y&_)oH8TQB zfXAC?9`&J>-$EtzUJg_vV7B4`dEN`Q5M=@`XBtiO5>GXf_`!K#Mgl=ZqYHh50S7KD zjMPrY%FSswp;>XkNN`O|pLeqpyys>soPnGBdlv`bMq++MVLbP$!>{SHDxb=pzo2qT z#<%dM%)f$&bUdNITYc`1U|R1E1wMWHbgW0>c{-e>JNNu7dnu{-Nogc8LTvl#Sm;Xf zj(OL-rZ&@OcGGmLuvM_in-FviWk6AEZHuBzYJB@z{;a0#RQZnxoDHuk%+%3Bcn!|m zIaEMu)M85S=S>mF(@R?pE|L8C#2%p1k;6(^qawo4jn|1li%H08>SYG84+uBF&)4PE z7zPmY;;6}vd^nf2u%1bNh*s-A_W&F1Q4$H8bcWa&cZ?0{z zN9_;et+G#HM3%3gN=f&j59?1Jx z;)RjJ#TM&_Nwl8=#I%Di7VTjBwV(Gd*JX4qDEio0jUuHyhA^2x@ZvxZHh2+-ir+W? zZ)ZZM_YZ-9WW;Eo|9&EF&EYodt^(*oi>uy#l7AGys~x;)ui?cAaoBY+UzCgE*;G1K zLh+hZNya@GbC(~Pn&`EHA>bv`3olyH#m@dDKn^e`EEYQ|ju~->c7l0IOwr%OjK7fO$lJ+ten>gp1d?OENDhT3uLqorQ6 zN6&L}@G+;?_gk`VAkauGDCV93)jNmI9dKi;ck4ymnKGJC28O4^;&2&v8XD0Z=atKN z&bH~11_c9IOG5AChtRoKJD_NZdE3q#m`NH5l~lp>7rKLt= z5s-#aO+_0Z2&lJ+I(lXo=u;wKJ9V|@qSXU}fUIdKGY}Yk*VJBzEJa>!iTUL;F{#OI7Eo)^r)a7x zHoR@oEuT^;T{z4f7HIJD*yE|vtTYrL*7J@sJ$hMr7dh@+1M91HE59K2*ve$S@BGZb zWl<_?1o5RdgySY9?B4SN>7#&cw*J*6{8eKLP@DvNxe%>qrSuf>3B%qkN|(7J2Iq#` zob|9-t+#WWR>22P>}dt;`cqR0^7)TfE&q&od;89Hl9|YH{6Eiks(!qThphfDlJjsT z72r6a|9=X4LZ0-3Aq;oA6LI<2h9XT}5X0(RFL%ji?KsVrl9->xGny@Pvi4A^^l4Y# zGOpu|O=T^FRz;_N1}@|Vuu`42hR{P`R>oLCma!T2S+7k{{gsV@>25DC;=9QonYx?` z$|Ui!#hXzxa;<_f17*H^0rDaAd_o&zg8zmLEnL%DG+$Cq#`Rj!o_gG)h=CO;*2lsC zU$;tKI>t;;dIT&c*a*MrIRlC_Khe=&)j;+?s)4s8OTEFHb|=#4*QfgC)BbhG{2^z1 zn$eRX-e8T?4qvE%yV)(ydE@SoY>5!Dj5_)$KaKsnJoQ<^<<}B;wwOo85CDP4LqKLOmYLrq{ z6?#r2CG4~uoUY{A)4rXbuX#I&Qucp|w0t@HLZ#>4X0jO%7v{6Ac`Du$bG_Xl8N)en zzQ%kxjT2~SP>u23W|knXF7bno^XK!8UucDP-Sczmh*xHB-Qcb=#Ri}%gK;^g$Ef68 ztA=w0O<^SL0fLgR#9uE=7&L3c?L=}{eE&xKuU>#y!HczzVwZxl_Ze`68Agsr)xXf5 zw|XUG^~HGw>qJd!+? zY#+A{qnu2@bfRzl#0)nN)6(J_S(Ur(t-`sjPg`bI19`bSd1QCr%%mrb3>DT24Z?H! z>UTfWh|_U=N=Lc4Lwdw}Ase@UQO{gD$OjR=YCszG$x3qqU-`tRv?Sp;h)$0L5Zsy2 zI`%vyUX!}9-B#rZa-K@Br4p)cEZ_l;-$=p2(&x1&RLdUtbZ6lxR#^AhpscGgH+b$^ zW_i-o#n~V{QG@DjG3P4f)6>Q4f;C}7l%Da|F%vC!$4PL>?pJum^PQGTOV)A) zQPL#JNw-#??bMk|9}+%3S7hEf%8RWdD~{42-G-8593f8QrsrekJomH#SpfF=aCFHvLWprHbi`HrKU8tFDen4*7emdqia^wZH&IJ4dypGEPi$cg0=d@~=S9Gw{^L`DsjfVmh+9PC}DsF7!3YmNrM5(GU`` z!dcFZ+NrnHHzJb^L=E-1u|pf0qzHJw6kEj+ro`FLH;(0VI#Qj@?}ocL@RZ4IW(LS6 zpM~Agrg5w7hAS`brCXmQC+99Wc(j?79hz!w?vxxDhgH~Y1zgRnv&riaI%%3pObE@p z+Tw}2cBIY~+EpD?pe3{Vhsyhzo8{#jQ7yE!J1?@w&+W++L1iuux*}wZs|MvCYnd2{ zz;Vq%sQsr?D>D>H30dzKbVXulE)6=gxL(`!kjLGko2v%g5%9kQeB0Cgpstmt;cSB~ z8>YVDUlRj);AWa1yTII`-5z>FaXP22iH26BXh90>bp-Ev-P!@11foui@lP;05N5*z zi+wGv2t3pr34ew8&8uHpKZ>E8Bq~oL;fCA#mGrp_%^ab^VqTcQo)EjPs!m2Qx>o_o z`aXkA|p0h1A5GB|6E9!EF-gIS~Cuwv7pS_mHoBiP^;i`Q~KYo4pqu(&; zpm=n*^7RPg1gO-+Z(XB@dC;uC8kZk;$Y{ROU*-t2@hL_4FN~D>%dXuFv)w=8+JQl% z$pP*tXEBtzuZJsAk(k)&z5K+@8@`q9=r9l(uTJ8|eZ8>l5mhbbsWI~uTEd4smEJ!|d@tc!wme^99)VamS z*z-uAU~#^=T0>wAMhFOZ0`xOpXJ7>*QKdnel^D!1D(?%%my;80#SPVzQ|m!wR0`{=eomYB^~qqaBR&b5 zpizbq+?ku77|)++XAkI!5RK3oJKcV==pW{v8xST`xL#A1F?j|&i#4CYi%$<)PmaYI zDWKNAb{7+DO~1fBS`L9|@Lr+$h1;7ym#%w{`lkAtN4tLdj7g_S6;_w`QcKAvl5*o9w0v7~_>_N@lu4tFj}?1{3KMk=C||Bo z4EGkph6aU%Q`WHJf660OREuO@TA)v692Bnv1t1Bz54JnE4oqRXeH3iwg9R^nTt^Tq zChSEB-qW6);Fp-XD4^7V-HA^`u?X(eDm({C!!OY{Yjg84DqM*IN9|rqwTig^>%*8>vP)U0+qpOuHwqcYTFP ziuEfy>YK9iJ`Lo;P>`RRSl`96rGe4clCw4gpLKwyjl9?iU+l7!Hh=qC>7w1Q_0>R2 z{R}V-MFrTP^#RG;zc#F=W=R`Z^G3ygFhnMi${Nm!;v*enT1~|>#M;XBGIVt#+?5SnPQjo|Zi5l&MiLM8@pl6D((c+NlL<%e$%eA+&0wm#0BEKzA~ol0uqYJOdfuW_ud znYxxehafV%P)pL>3|97bnt3jyu{{@uv*|D=z>9sS=gqrNWc-Gy`)(6~ zI^(@d@+A_9Zsd$-EUXT1Ij5cSt}bJeQY6%BFrTndrx45W{8WCZrPToW$mf zecol|5n!)xTM_tbptnX5Gh(CTrk20|q$bzR2*3E5LAA`vjO2eSErx|l= z6$@9V!ss`)^IK1v^{ou#5t!pRt`paJz7Bfd*S^E$)CeZ>cT7UTY9voWBDIRHnt-1r z2<{i<`ph3o>X!7Ls|lRA%B}H?bE9c z%TuHkn05^l#I>WK1wxWrm+HOS-z|FL0Pb^U8QVlya{aaP-u$iMcU|#k*W2Q>W>Y?0 z)$HFL@I5aq2Gq%NR2>hizW(H=NlN-Up%fVL=9l?QsFMx04a}BU(388-aGAkl zuk^rmqr@!XcVN$4woK-`tA`DU+_)i?25&hrL1*ZOCyMN!+}YV&;ru&xUg4k|9{fhS z`|{^<8EW>_9zuU}RlNBS-TKoftI5MGCWOQM>f~jJq5IcoQD{U-;OESe1;=qcab5^Bz7b_VWTeA7L5DF zA^fw?E?0Z`L~aMilRYgBDGIW6T9*7U3$vw=CinG6K$v1Z@ln#z$wa*-OcC-{h`{}hEZHOVJbNu73PNiEt zo%g787MooaRRw|mkNB5lrTNCo5W<>~;cMW7V9HCUAk|fKdVNYGntyt3lRj8aC{HNh zN;guJW>5wS;-g%q$S)vJOt^Ur*GjSwuJXf zPa1yrTq=t#;#Ugko2%2qh|Mm!)R3wyq`G`u^`2z{p8VVKiOdKTG>%N84Xe5PT+wG_ zAhrkyo~p1<3!rbC24>zca`shDZu8=TuXm?2yQgArm0-|@l9Pv}D{Wr$HC>uMqLVuJ z(Fp0VNP2=C%lF4rosanWAT<}jo6cO0{MJq-If-k{J*HQzM^PuVC{EMN`A^yUE|(qn zo}jJ5xc3bUORtlBSAT{&4_en^?Pc6k9#B%BgAauMs-f1Vxd8IXK*+0Na8M3-{4<+I z^CdgSqKg(D-Sv3_MqEyP>Yjfoh_dIr6wLIVH?wa8!05#IfZv}d6BDvTtE{m0BZD^_LEmnmYhp@2IlM;?(Ij@ z>;A=yc1g2={Tr!tN3A7n)=T<-dDAhIss9%*dSUD93_H5$N4YU%x+`GCGEyD;2LMqn z0`hmusf_6qd7RiOU8w&~1TuXm3%Jltv1tL=sQT|jARYRr0~;MWeb}_AqJP1q|AH$y ziGR34|JT<~FaC4K5dL`keYs9HXVd=>1)WmX1DEd!A;l6Y1Ig5VEVDCSk6up~D}i@| zq2<5BC9K%5a!+UCpJR&sY1jSF@qflKDK0cES~Q~-JAMQ70^uceal9(mb@S%r7s??6 zsvHzbnCz}wNX55`%vMRGTVb{o8ZfZM)zAZbJ>7Fcs~NkMcpgcd<@o~MzWLz5)vBB0 zGabj)>j<|;{?N2E_5CVNP{J1$#)8?a8zi|MzxDTY`F$u7MJ9T(it9V2wV#ydLWx?x zRqyok&QDVTGNP|JqdA=1b_(k|BeWxyFD~EG6YJQXD&A?b3li;VhJ;XOB-lRSGo}jr zXO=`{<=s0<1D_-sCL4C4G}VkAi-az#9a)dYq7sQG9nsGn9!t8BVOwr5!)TVFkH*}Q1-(zqutv8l%I_J~7Gp)PA7Tjs!Te;s79QQtp0 zXg1zu@;XYLFk6!`T2YqL_&2^uvChC?ba!U6q6TFgp#T(>+s>(q!iQOrHrSY8^*pU= zGon=q*r7cR;D=Uad1(D+`z5>U?1vsuUXQy~!raEk6jnZ2D3yD0yx;j-|IV~%?*YgK zdW9+1+i;`O)1$ihIl`<0&E12JaBpeD?$Q4}&K=J9^&+Lq(=zsp_z!@jYBiPOg@i%F zSBqZT=-gYMy!pu)6*HPhK^dPI`Y@^$PPEj~W*BgzimdVK>TIY_#NJ8;Q@?ci3)TgB zHsX@#wyK)#;{(;>=@=71llsXGcl;%a;;GR9Nub`#juN=u<1V{-@pgJ{5@c%MV}e^Z z{=1jeeVFElT75V|IPRvAbrL%dO)QKL(MDfQ+H%6q^I&Ftp@dD0qKMroEHPtr5w20; zSvhob*Yy2cpRFhqol&!O-$|}Pujvt1A3Sa`LDS;n`Y?*ORa8`hvbW%QX=nhUt0#s} zN{IC!VIr>lL^<^s#3Q%|DD5&m)$GpmjRQWN@~u(q<-+sxb!_80;R6b(9LTz&gCemI zZp%K&i0OzpXlvDP%V=5xNC`>>Y~vOO`+l6@=3UZKSQ4bPqoD0f=1%-&8~6G)%Fx0s z-c6Dw=KPxRd3Kh`$7LzBPKCuW{Q8gvwA87fRNs@356iV(k3kM%+)+5wV&YkHkje#IaulcWXJwLK+uul7YWJ9rT z#b+guWk7D6Bhb$0F02Fka`nicslE)2xYGBIBFutRULJ^6*TFthYmC$p9qD-)ap16p zMmAke+|W=X+D{u#cscapksZo#Z&){8V(cVlcm47pz}x#VI#%#xNa`_mlX7EApy5Z# zOTnzFHy0kk50=QCk{}rgl)&nPHm(`odOm*K$2-#XOrKQ4cyM;#MDT2>%}5IM+l>%@ zA4GJodsIg0w85^Kq)Ms>)O3avqIf^xb|NVy9|cgv~KfB(BXnX7(8Uh$bGq5ca0d>{?C&@pH~gZni{HKL7MVEBUJ9J%~RjX18)W&m`98#Q7rc-e&QU?p+Do0UhucTR6SsvV*TB#Cx0pGZU zEwAR>C@HhGIP3HxVN>Z{WTSoaq-((l8s`YxF!T5HXX;yVNjG?BNnXe;TzDZhbWwX~ zi*tK)=l*sO&q`GEj)fpYuB+IGk!5$&b{QS7rHvAYHF`|SpWP`opo=X@0&MLeq_atT z^{YHZK>S_NxS-F$S8hrHJ?kB3Tu>b#qzK_E4uNbG0JU#f-NUksJ_u&?i&jhKou((F zINXwy!Rr1hspb8i_#+c}TRMuU^4))x2>)bD0|(1}cU9nZ(B{gZ$5#gxaOe8=sD97G zm$d%XgB_`PX4nGR{4b!H>=1mUJR75A?o{9hQF45Q-?t~dQ>*ocdGhc zLjc^H^|)MyZ0U&_9Z4p?yeH4DtZzC#{o;^BIJNb%*=eBWt{mkVV#&JgGl+JM?=@jj zm5)YgQ?AimY|;D}A3ywMvFSx(9LK;Xojvj{Bjq*buJgNVGdC==x5zdfr9U)_@H7Tn z_UHDxkFGv!T>3NNN|r~vd;!B z&YmOP7^GheEn<|K}!+?gH9F-RkhU#FZSwU(aV55j#_x&!#fVVzm+ zFWZuP%_y>VMo!U*e61B|3w~%!)JUZ|Hk!vft$P{6W{1|Y0BogGt7f&0TA}bhI7e$? zZmrpn_<$YoRB-l{3fhisxH$R(Bddtrq}A z?_gLxxfMh-!lx34Z{1hcJGzPBK~sg4=!VK+ID-Dv<|`n-hZE1D@?3BS-p$u<9^HX8 z$dBX8qh8wru*3@$zZc}eG??teMbODHlTJM+DQ*H;G!O#JC zs2ZMpR>u^fUBw~mY--KQJcirYu;GXE{vtVf`OT~&Z!x70@O7>jX1#=)^X)v642~Bz zo?q3?n%~dk1JOMR0i#6i!?8a!(j*<6l10Z;RdG4JVwBctPS*5k*<8m+s-Ls7!hm*q z;pHZ~;EN-PCRXU-SWlADK46$Nps3GU%&v67y$+a0>Gbe8UkZGsF}Qe>Ren|=`W=Cl zA!;>LdJc=Vn42+`S@~0#pC7a4-Xg$;GD#5JBMq#-JlV$uZqG`2r#RiXhlK5Dj;`e{ zpzPquYK%SE$E5`sikN#{4B0K>ZV0467j>!7(%(*V^+(7_^BU`J;}G%1dDyrByX-Xe zJjc;?>Yr4bFq z?36pCHUd0f$ps9Dt$#;PSNF0TbRfUxe#l5k$qbLRhC`Zk7?d+BV<>rK_+5v?K_5BH z%yY9)TW;Yldy2rhSV)Z+;nJ_?VmnU5n{m^(DD0nJ6Jw7b- z4>0sFUIB|O9kUiT7)|Hw7RPIW91r5bChOS0pc!1^!HUb@zi9M=WEr8!kn%oJL+rU) zT+LmVhC3=#V-I$tIUj?g;Dg)hX)jQZ6=^abG3)7XqHzIJo#*%XqZU_TVF>Ln5KZszUyrFm zi6Nw4bB{ALbNqDsAA2eybQ6c;Ea{xzBzISvA|!yx{@QZ?@vO?FvJB{j{1F&V*44^g z*Ac|MyUr5y1iNC9yTTBN~cT z+qgAKiw}q0s2XUfh%X`BFw4(GLNHQ4f1z&s48zKZ`0B#(_(S*QG>ByM90e1*@lL4c z!SYSj%uO5T$?dc3HGC4r zK_{&Fvr;b96t)GFncj)KIP+54neLWW$`eeX!TX0g4u=WypY%_{AxBnBAGc4QyN;}| zg(*=5;pf4>qgqrkd-CvOjvV8&YGQ=E6UhG63*d`oI9q1pIn8Cg;%@L9RYme6ANvS$ zONJ5$?W7fyF}S4~z7FNmR*{Xh@bg@i`WoNz1dVyIWo8?WO8P94{MJnus=niU-z{GZ zo2yuM8O)^Bk@WruYK7Bj7Q>MR1$$=-pci>+>W7gGCKy|gND1>`J zJYXZMNMdxriJY}ePF0K$Y>G5fjnUmE+27i-FQgpGLLR;~Y^}jK)|iWq(ey@eg_g>U zl_jc~xM`0hSV$=7w-^1R;H4krhlz+rS}g@tUmHG=j1%hhWx|S(XFJFvHftnSStfjG zqggmYGT&sqh`EF(shcITr}hk1Vh)}l(B)994}7tKOCnMzd<}?)8t|`DgNAr5Xt&7R zUQE&?q(>6{DIR*`oO?21Ob0yBEnOHZC#MY5>*iYjvQEMFVub6#$nEY`JigY+#;+_zs)^O5v@<8zkTZHG;H)#ycb!(A+jR0KS&gPKQ`M6*aqF zz{rKqn3l5{z_(yu88(>-3|1~C2a=2)dwDM>iiH?N-1hPiw!^RGpU0H!dJhg9Cm{DJ^88%lJ`%_gw)jXR3!@Rt1)uav5+ z5A{8RW`!jyPBx{b$%a@HuttJ|(j7p?5_ymzPYWB#8Df8j;pGoibboos-CF%FcP-2# zRdeCKnAzxUBzWMGl{IQ~Yai!Jq)yZ4t?^OdjzYm?t*I9>^kUbj9T(TTWy~XtZRW~B zwXv49SLZtW`e!9Q;*Uyt;geE`>YlsN#U{Y~f!12cdhGrdPXo?cXXXHC)I7oy8zp$P z4gpzxiYxUDIaROl%$Js0Ku$#>)K~!bcShd_6B7{wN+CFO22`H&q;h5_Ch(Tq7!NbN z;w63S{nnyMYlfABEe}eVUmT=Yex>0~;B7l?b5>Qz&;8N4-73eu>}(H*MR?MvO6BTX z%4wg4;J_~rmSiStCj`!3CcN1a(PmRIcNTv83~FV|&q|Kg8$_ZZH{b4%FZ}u?8BPn+ z3pH{Sj`8CU7NbfQ0-W6lUQ#+E0%q{f#x+J$zzgDfW<;95g?WY2A~CL-+pJWwUgxDe zi_G@;J}R$qI$gEyEz3vj`m?bc+wl$AjIa=E94cgCm#TLr5gevlj`(?;{!2b>Fhr0DQgS2+gm(}&m1Z7qnwk#R~R1*S)D0p8HVA~iSPBk&QF8pFX)Wz>}m?a#m z)Q?C@pS|;ZKS$A~@X_dX)PsVf;1|`cy{nhyxxC5Sq`Jr)XdOm}l%;WZFfezheKKxq z{1AQ~d#e1nS`&63{<(@;*^M9WsC+vL;g<(Jo2WEv(`n_z2Aq5EW{${Fq4U==r@`UEyJWz*yn|lK@oI&6s6?RWP!sEabP6pS4SS$Az8O+ zlv{>))+46vpDjpzA;cYPSte7*CPw>+@IjM&G}wy^bb`*Al{H!Y5YWbTj7f{5%_WWkx&q*+I7d_G2K%UcaxGU_FE!VofJ zTB8WhIMMaPHZhc}{Ku4B{lpba-YOk0h-%`OFj}vddzVY{Vw10@O>J{>Dv{_~SyBW{ zpASo-(+EG}g-_?4icFeSl*02KQ>Xwgp$!yQSY#?4jop+f{OgqA5!xs%wQCeg7Ik;q zUa!mGpk{lO(Hf6ECVEzJn>B1_?n&3fkLp{Vf{X}e6 zQNU-%e)KRyn^6tckQ=U75eVl=S9xG`J`({fWvUr1Y3GDEp^po=Tc(k`q#G46Cph?Pg(iuFfA+_}fDW7)gyKjkc06bK z@3Li*c!1I^u-$9ywE*^Nl5yoWscq&#|IHEZWl|#y@G-;BpbktU%V^`cT1Mt`O1T}G z|H@&L^9PNXqPLpCV(ix#m>f<{@r)W zow0hk+Cj$Jm4Tn|^iO%6%fAUV)fuOTj7kyPhYh^>15o|_$7z*$YRFCFHV7i0PssTX z!~}ftDdov5>!0J_9@A{gWd_WCb@)TPbN>N_fS&|H{@;$Y{wH|tUpUN^K$H{1azIs- zyR`l%IG1^ZWdu0=*N5dv57c+8!<}6GdA<)YXq2I!8Oj1ZZ$*k?0Tc>-^a8G!WYo8r z?H9iz=g-7Tz5bpLu~`C7)8O8pP~yd~cI{?b-0RDOrS6JsNfwt_Ux!^?2ZZUBB1>>3 z6L85zjs|ycFW@Z3Sd%L&#l5dGqK$i_9^O$ucoeCZk*aH<5pkkzjkQmlAzBvLPL-mz z9KQeivN9w6fgkaU9i&uR>F7F|K<`+-v&0%pMz1aDckocZ2i}<{_&gZvo#q_Hy;+Js z2lErjJx(+ImO#?M>OYdE=aTKJyY0gA_r!AK`|u|N1kQ+e?y}yl5%LPdLt>G?HAEiY zelx-Qh{w+#TQ{@yD($6gXL`tPk79P-Ov~T!uW$41Q+Wntpsu5&SOqtQ)f+{+8P2eq zA!69#naZ~=$?LjLFeLyz(sWtRS6biG`}_TDu>TR~^Y^I)Ud3wt8<)WU`LPCaiKh0U zg?$=hEo{S{{u18XVQL$&!7EXbH|np1;&-uBlfj#Jt(!)-??NcYmU=% zqn_YtAS=-p*j^IYY4&hA0teI{%x6?BfperIqsmY_;*{rB9OM0_qn$jldB8#`)z$sy z)@o<>XkMHxWTD-*=^_Vn`$8=D-ta*j24MUlFUB}Bdzt@sqWGeBW|Jo~n~8}?gqN_Tw{-rC+J>@T|ZArHZz?k4N5$n@T` zRkC=ku>_Is_eLmwaQ?w;k4vyaIGPA`U8LpF#eQW}=1KU+F^{+8I)edp;7x>11vwBF z+76Cv0oMyZCB!o*{}%*eici-Iay!jwWytyBcZu+gi5!0N|ED@%6+k>iuE#o%1MU>SZKYZ9zYkGJs9#-oG)fu;w;x`%Nduu<{Lyt(Hf!) z%BYG3UPp<}p<>g!YRzA+u)lP2P>c9fMPP{140}&!^;{?%Y?L&FluSnZ9;sMv$fYwDY!Dj&w&tZsXe1-%XVH%XI(qxxdT{8R9Oi7z_x=4G7aDt|Nbl&t zV6#0wNB#10A6>Lb+t5ze{T-X~ zRz_jx-^6Ia7$iza;=sucZ7E_&jWOW2$8FRJ`e>dDoB?Wq=zECUj*qOMS3JaFp(M&6{Q+Lij)9S zqy_1{2}qSLHGrY_&_if>5pQ`v-u#@MIeX^p%xUw@PWq6k-+SXmpb)ro3DN6Ub zgt!XRg%LrK^6wQa7KQWGYAd%V546dswS{FxOZVO-E!*B{5l5&11r?W(K*K|~G<~h0 z?k#_fVfXU&rbX=VI+vL|g`Q%h zAJMvI%L~H|WjCDtHEK(QqT#KgUe84%RBgOSC+u(ANjDqt$qEDXISf{41%lBtSJYdd zy_F&nZeTD+QfeyI;#%Z7od0Nl!(L@vZEoHz%XP%f!lU)toZFao&>I?Rnx~3+t7di= zsG{QHv~>5WB3Qwl+{^sBRE~g{4aW6URy**xdzG!7;v^!qKm*UOi>1qm?!Bic36{!P)-WL<}aY%%C?Qe$KF zo-YRFc}eH1d9mF%RD(HAYmOvCI3$)Kr9cZZ9)*b=U(&c=f&kio6vIRkm1io)8=0Bk zQAnH`Nete(2*IIBC2J$Fwj|TV5)?R7h#GMkk+6}t1cM5|%_ZqB`$sK3ozByV1n#;VG;1EX|0#c?MYkeJp4k?Pz6sIQ&Sf` zTZ@TQGI7y7y(O_L2^SP!&KqVG#I-$6@^5BH`HCXC37mJ|vj}VK_ zk7ajvpS<{mcwB$A;U7tvHyNLmct3Rd6K5?@L?^-~Rxz~l$zjWDyFE#7|G68Ciz9W$ zL&0?HU+4L5@vUTm$|oRAqY&}>@sGe4WF>&_>T(_^-Ut7}v06q4>`F*>;p?`E^LPw^Jx+z+z&}te*I-IV}h%MK%vL@D1sz!s8>n zGb3|n*@#saibKt$Yz5TWxiXBeLUAOF7}azjk`Fa*l062{Ln80Q9#rliAAoLHds)an z9YF(0V`Wu(Dz-dI?K#lSmGstiz4YzH;FmOO0x~BT<$-_H-{POXM9$9F$eLhix#m8q zrx$`cdfaEkb+D#*)UxQg|C{g^@EtR8N>iGM9euz-eeYS6mGu>zlc%gCC*eR_;(s)U zr}$X_ApicIj2+{SFDW{(q2XQJKFc)A6>arsE@e)^C(X^-cZK-Av0e5icA3_?36r$3 z-z({cXZZ`8a%#zVak+$6fTB$cjZffWu(rd{h(A-TO&_cs?R^+%!m=M?eF;{wGnaO% zlz9*dFzn{}6Mk@zxQf@44@8Tifjbx524iLl*~9ia>5uM>Vw^Qi;>bY@lgG-J?eoQ$ ztzzo|yFR2aUAR>z`sE)7&o{&z1P>SpIA*^J^aJo`#4*~u9KyA=F*TYfCRKHmn}hk0_Xrlr zu#=1!Q0Ivc?F#k}YQSBRs>N5wl7_Xu{Bya|)?s(O zz1Z@Kh_&FLujSJ9qAN}`-#$Oy;uYq(5o)U4(Q%7behRve>#=B0<#xbHlW&pGI^1>C zK+C=PQkfku>2XJ?VWaDd{l=wx>IM8lLfnWQuJrVp&2@>p06*h>)<rmVhLgFN>t5 z-TRE3|6t*v0)C)m{->egK~T@rj{)M)@s`+oiFw+yq$$qSfKSnA8jj7mhDbKBrqvYV z_IA0u&f8>jXn6DDUJ(MH(ph{+++;NLJdy1V%NDJ%vp&CNu(C&ne}s>LV=<}* zwX5Z`d|28OyJZ(E6&qE-sgD;de_sZC_A!pnJHMkRBQ^E?#-O4ydAtPd1Z5)S|E4jo zSaY~EUeac{68?j+2dorDt;??0`PP)7O|GNS4bDTLpbFfh=5 z$h6H#VcultvDWes?gv+2ak%lS6|0ui3^-=Mz#b0Nrt4m@=H?T*+7`e~HWc;Ai5(zY z;&+;#<3?c!M?p4USuiVbG|E_e(#%WsA17QnW8+tx_x4yaE4aI=#Kgd@*{JB;$k8Wn zUXJ+Nq~n;d-QMTxX~KUr@MUStTx4@uc=8Ra=K&2sk4y{A_rx@{P_A=99dFfT`*)|O z^_%E*P0jj!ASEocOmTmRgkk5PT?+-baZhwKahbcU4EfI^3o$pFYU$+dJGvEbpz9*@ zOp&UYSjA7wK<5MAsgHW0O{9jpqtJqq{8_-Me{XfUYemn1s4aKSE9Hw4lF;3d3s<`p z48ssU^RE$89#Hk6mX=HO^z_2aiX7iwU3eg>tzC#&@)Rtd34?&!G8)C;C)mN*6She6J$}@q{F!5pCzOoe1I=c+jXHZ$&+vR zopzzeJ*;%{(Mm|9h#u)!W*O`w*~S$yH`YdHkE=(|N$7#&7E=zLX={m2rw7MA{)*z$ zu7wRz;2NQ%)}vUHFlOGoqrVRQE*vbU$0ELn(5; z?~>4b5uPyjVZ<>SGq>v+xrln)fGRvbNT}WHjoWS^L^-niix&295`Vf+N%trE6ZNdT zp@QKZh;7}I1wYJGfaC?huTuWW_Q{hB&8Xbm0SNim7hX1&2RGq_tgZb9@dC*g>r2BE z_0PxrN&ZJ=d8cN^M^Y6jfh^At=!j!W(L`J`vZqMrIG`$dx~^2oEkGbR9lC{BA*|8##q zDOc*b-&LF%J;RG~{c-@a)$z6S0BhLy`5%w{_<<5Nmwz>QvD(-@qzvDq$uQzKdMeb# z2Qr{vo~}bg^HGc*di_f_TopWPrzVPY5VtqY6v*pFCwxB`MTs%*eHWy&)*4i#`29AM zey1VW_Q|J2ODcu7{=aJh)t}ENN;ZZvv(91{@R!_&3@WQ0`8hlx87}@Q z$(n$sHV5VPhqPH{S(=IDPl0-k0oE#m;`z(i$6^gwgP=Xv@_24p^AE zNcGftJT3K|JmW!u`$`{R^qWH#m#A2+JVj7*H6{mrn37VzOS|hE?PmNav#Kip(|g@|N*8Mk0Gz+1 zp{!^)zlH@@;#B^c3IMODO=F#7AH{ZwhR#<}-QX3ZapYY0>XR(wi5hx%*EvR`lg|`lRhItukjj(B zt7KK8U}GU&V$~+pPNhGn?S91H8QbFh44#aZ5+|AhbXhle-c{ zF+5%#3vW9ksF1y~y4tn7yW81&s{bE=wd-NVPGxAM`L({Kl@+sV`IXq&P-@#VRvX8w zYibM_T04O|=a~Tj^N|ld^Kx>hQMAk|`uR-NCeWic>ZfNd6^<^DCxLXA*-i~+M`oPb zJ_-5K=-g#`jkBlixf@IXfEW1&w16@rVE(gBB6X(5-M^&Kc!=h;RMX|A8U$p84P OT0=!w8S}vU<$nQ|b|e)5 literal 0 HcmV?d00001 diff --git a/doc/assets/crowdb-iceberg.png b/doc/assets/crowdb-iceberg.png new file mode 100644 index 0000000000000000000000000000000000000000..af5c1a25a542c5f8f51ca7deaa066c51df9baf5a GIT binary patch literal 293183 zcmZ^}XIN7~xHSr*AWBi1^j<`2=%GcVOYgmg9(wPnAT@LZq>41@O+Z=@gwTUX??UJu z0)(E6=iGDedA{?V{F!9(?wM!aS$oZT*WNLj>WYLgU{70HS1$)Q z??aqU>H8p_e?hXIwpQMbZmx{FjxM%Xx(+^!{KAatZoZ5H`~o73{DR_w0^$H5qox|8 ztd6d4HntNM79*CjoQ$r2&R$1QyP4jn&FkF6Ml_IvSV>9gjemq9^JAuHNp4MSd_D_3 zYrV^d%jx=lJ|<0p`}6sCrWNIF6+?UE&o0x>fJlW74j3t zQ7`qB)U}+B>EHzr+;I1yD&zatuQ$=Cm0ttEpc<*^EUyG0+zkqBzFfe=nB_%(woQ~C zJn`B6vEQFz_4=-Xh`#!zZq4V3d!oHwgz25(E3oJp1-B*&*Ghf%)P5*nV9PGSMs-Yov23~xyP2JQ#SR&wye!T}}w;gl>ud+yI^gz$F_^zYJwp-GjcMU|l+?;H=nfiID00DK`_(p*QZ4t_+ zOZpL-sb1ZbIQX?xNJ&mMRoqP;puII+zs zW+UcQB{FS`z*`Al8}SE0S;RxB9czOlVgkjs&|9PY1{PFcHWKH^bio zdS4~Eruz6fn2_T0u2{IR{?&_ZVr}-~2?WcgBqC7zNd#klvS*L@=bE|$B$RD;rUCIB z{OsDE8zEu6*mosU#|ccS{|=1A(w}>#lXeLqL{QcW5X*KZqRioK<_rQ9RCqD0$estB zgMu}$=oIlG+%eGJRj$6DKe0x>G@4)Yvh6&3?1M!Duw!S3A5&;I*?gJz=!t-GrBo6! zt#WCo;)f5Hk+Dqm&y>kVklEz^-0~6;zavEiX{)Z~+@sx-g2p&NRW%~h9kJEQ2L5Bj zA;QE~iD+}Kk~}OL#qPN?<{IT^eRYTre632k`O}bFVU7W(NL2xD@dW!Y7aFozi-3fE z5Kh1A5u4gQ6ih0X^@7@@mWks`bmYw$Yk(dS{NDDC*4oIdQwU{G)zNRuJpMF8P8F6A z>h@-%y0M3{Zc5Wfe8Nek-DiyqU;NGs%9`p?owA(>Eq>;tY4Tx?Aw=6yR)WC`LCJ!) zsim}k%(N;&r;3PZ<-_H(w|;U_9$xdxI@&tFo#msP2(C8cLyB#wEM*IL^xH(^Gvy!onjMoDX;hxevTw?DR+r_xnBSAI9^%p{X8}H{9}p-Z^$aTs-_?D_5$%)YQ&L|f z?lj+k5HuSxFZxV#34A#ijuMjfGO-dEeDWK`= zj=wgYdo^=PKfu!YWAh1h$`!ujgI#g4WwzcdyZRE%3Jn(IAD_}#vnQm6&n(z*13cj= z_yP)mqMYNY4?abqjO!7{={d|({a{3We6 zGyI*Gc^{KNz8Balxh`7}F7Z2n56n#3P>IhuAm|QD@!__Il3Q>TzYJ}BuQ@HY+!e$Z zif?o!+eg?cZFJ_Sgg?EVq{H6(u2H}sTSQ_PE4-o-!7%juH<@8>=u&bl*E7`L56 zG2kG6nFqC8k_=5rLsU0&5BANrMCjK?M z>2u!DY7M7C&xF;@(#Jc$iqm@RW#RHok+i+}IL9i~Vat|}wQg*834k$$+CJh=(*#rE zJ2Ct5=yE_*WuejZqnH8cuuO|i%+vNtlRuqFJJB9Qn?;~_Qj1oqzWdPqd5_%KvhmXA zJJPr27QxT%xhh%qnFivU)Z@mhHT~%qQCMl+SydHVD8>`=_(Y4@Q_<3X5d{>8{Ejh`pRVE-K?IgUyh$ty z4s`msJ!E+9s?PE#uSjq6Gg;b)&AZ6FWoI_sEFio=SlK3%w*3lCkupQx~0bN+P zq3mJ}0G^~xkvn3dtqUTrjVE4_ic#G`V-*JK!+$(ET%V|s_l41S8xzu znUmqKa2O8HXiQHND2lEq_s!AA!66y9YsuG~-a+ZLHi?7=QSkIB)8^?W)~N?O8Tdrk zQPBOCc+<(=8*%8%I;BQJ%5hf1Umf4aijc2ikycZWdE003O3dC%h=GxOLV$ z-5rg{ux;<76sGXT>e0lBBok`#RTuV-dC}}LUy_y9>jZVl)lwUYgHcUxHYXK!8MC-N z7RJ)kdk+3{VFnr+-1Qs`EQ}qNI z(p#5bV2m0;W3D_vT+J$dZ^@|CA!~2*$k2GNJULw{^($cx_fLH`x#J4ny;kS-!yHG{q#sQK~m4X zdL)*eI;JGX5N{aEaOEh1%%Wg7rU7R`>!E5{6V5sWe_kvp4@Wpafbw)vtEs6pw3ZlTa(IB)DotiTJi$vXsac?Ylhk4RBvlZm%b>19qDt4Oqr z_k-oNC}__kPWc#_38E(I;FOC|Bd$wA?kSTuHBlAqa8vx<%AS`-eP$wSFHtphReFda%IjMd!nVEI47+%eTf;&P-cO|22t901!-D^(6OYww zbb1`3;-75Pqn(%FPiAa~ZXro|iEUJ<`%!uq(mq|9`K5bC+y|=T&B97?HGSA%j^fNit9oAsy31fcV2#@u*;pP zq6|k=3UR_DbmV&Jqo5(!X1mI>h+tF#;hl|>tO^-?W#OkyC>P7b1x1@WC~YT?xd zxxtO6riF}+kwM{(ea=p&j=$R#i^_4g(ASiy96r3B628>SOO(dhyrXi$L1q?N!ZlQw zc(r=*!`{*K(DNlon6sAaW%2>GJ2tDgx}FbCtcEE+-@^>2|Pucok3f-k-qp@k^YQxEJBcpBV7YjE38hr zZCFp>j3oFw_pU#)@;y}gHTjMJ1-Pq7u1TS%EVFJ&mPlLo%fMw$c0X&(FV?(w-XG8s z3wyB3U}Uf$PH(LOfEV}k796M=`J1?imJKMTHJ%t^{q{5{#IfypLH+o?b&j|3Ivoy9 zxHs-AalMbb-lvTu-o@fd-G5j)S^Ia$ilN75J*5*x9SZ*ShBlR`RkG3vqJtd9O}Z2) zJJ|cVT9tcc@5?O16h`dl4U^aa{SO3SLU<6l7Y;&c1&T^|_5i3z&J+x>=^f4VE}9?J z_>-^jaepDAhZ7=z5P>zEf>Zs+@t3$It$Sz@e(vf|T%bM?hRt0tz<1m^M)*>6R);L$ z-FsR9V(5(f#I*h)n0<^|bGL}`Ym0yCMLg&C5cY69XvV`9R!cgVPO=kzDAdp2C$#;L z1Z)9!onp;uT)nY-9_Z(7Y*Ro$sn#T6VO7~@U~D*k`a+8!+@+z@I+`XiTp>l&vVry0 zp>fp539^U`p zD73j^ix2_0})$f*7WcPA#OQZj5j*v|0?EzPf%OBSF$-TNMPpE6{3lrMl z$5q>(9yOM~QiGRck+2|C?-tw2$u;M?$#0vX9>!X+&PTaKI`+fGP2>aOw3-^upX3Jv zs0^rI4!lWGzT&R*YW)j*j!t1m#j`O#t%0@N|hMz&Q;j#n-M4Ec! zII^+?L_}}fl9^MFT})=1TxdGX6Jw6|i^%YqVp2bd@sn6a=RznesdEy?Rpm-rsBF_n`{rVO;~vbBFaUNYvMY z_HEs96G@lZq>;7L6ADk7$D)tkO^KUMzAsmyU0xC|HFV5?>_N{AxjCn7kV*wLAzae& zKIU=bHV8v5U#I$8w<^=c zCf8xZK;(z3fwO8yNA96fuZocl#B|c}mdUM`S1}}%v!B*?DrHh1C})~jU?j_B{lH!r z_?$)0{xux(tuB|>&hi~UZw&T08N}&Y{$c8eC#k*py~fUyE}MERSwcv}^;@t>q1S}c z^XF8g@d%ba3`2+aA>aP<9q5gJM!=%rSX+^(v=ZP&OC(jQyXd& z%LgccnOkDl)@AB=oDBV!I-lVc2XkWdLaMZuc34fhSLc5Z%$MYaKQ{q2;Sbe7Zm?U50$#_UxLU<*feyRYFjk})S zhy^)r%n66|5JI^1^wgv=+#c$KRHTJ)0P{m)H9Bn!pVqV%_{mvwRhOO6!EefM+kO?@ z;#AYySoq--ycI4aLAk$>B+-Pg>V|74u$e)MYN`P3V z#`7ey*``pQc@gI?_ey=^_k1fkgdmqw_7#y`>mt#TM02#k4M4O55WZ&o>x{R!oCpm2F<%Nglzr#-&w-O5ZM3M+Zo z78o)no|VZRLA^_$^^~v&SCsnmmawv@V9|`j%M@pqCz{aeeB9agF*10mTw9YBTgjjT z3Aq|mEB0^JVEz0Oy~p+JaBOe7EVA)WvaEZv`Q=KL!eZi!7aQ^w zAF5c{pAjFc0$Hsr&;-V*=zXAt^pPFbR<-Tu@g6}rJb4nl?kyYPY)u-bQo~Sqryfhz zK7B4XhOgFeIchUGC#cu_04cHz&RADSy50(0`E@L%X zA@#M-Hv6b0&nH)?Q=anTVGQBdn0T{n2NC0*H0z&sg<0;YHv&}&VBIiPz?PXnSz?7N zH@^PyLY0#j0+N&FmEAunFHdUoRWX*lzRH|>2PTQrD3x3C8;fTZ>y}n+M_AdDqqVh4 zTq&4#UTfl1zOWO_-e-)uX8+7xGc7%}x#;}W(Px_b(JTGF*Mh{J70A6LM&H$G(d=`9 z)ZW)>D-&xuUnUo_R(&!TUdxJnm2raparGXLC|Hv{Wrjc7ok>t5Y_LIDuNT*NO{oc` z=#G3VDIW!GZSopvnu@xk6mgJrVL`pVa>0)&cRkQc&#Q$IX8{^|!;Ro|?Gy z%kg8QvxltGxYj8-$;yFx5O&dAk(LKV?i>$imALlbwpO=IslTUK5zT%mqy?SxDLgkO zsr1~R$%yAsv&zm}+V>gXSa2?{t|t!FZb{bI8;kV!uBc`7WLv(dvXH8J@?hU8~U z4FSHCImS|Co6%vz4%+Ed8927>ZQOQX3QlEj+?` z6+NzschcF%Z~AD$P+Bqs0lKAp^rn1SH3!Oj40D9Q|GEJC0D$CLr<^N zN8(L&*g`D{mExtQ6`a3ocW?|RUsvT0f3}6#pa}kUK)kf4-WJG#8Em|AKRaWJ1{}|n z=Z`e&HaJoi@UjfjJPN&DPsb~~$d&wiZvYOv6g)5I zdrMd_!wyz$J0{68jjFzNf>a^|!>=5&<{dpb1xj?04&P_#JOkr{_S(m-TL$7y;ybOX zCcjF~-DFN)r)-vr)bVU39nX<<_1l}!=~SP2K*y}NxFJsC(xsP#REOl2(RC%9-}59b zu}9*6_?nAk+0NfJE%%1XH|iO`Dwp38pr>xj%(^V{RnBYZ%Jn_E(Wo+NHh!h;xt3w~ zD&A}Uaip~-PFeN(MhY3x+RF#2x@^DL0qGxfb%xXhjFP+> z9L}zH=0)j_Btv11MKSbi(OdoaU^%F$QO{`kco+3Ij;3ANPha4Js%&s-`yvOHNNJzN z!sYBQAZeMm*oB|!6)Jpp^K?G|!QWnuPH=|nPH6)bOXqv%B)8v93Uf{osLtoRma2qV z2hE>T7G22EHVr%ph{9epQx-5D(@&M1uxCtF&8go2ay%}mQgVhi`oXHQ;ncFkuM3iLw$G>f}a>0amPteOWH0Hx_YJOma7?3hqJ~ueqG31 zfbsP)#wm63$&@`WoiPqrQuf=8K_}TLesjrDjblpMkp{litCze|X33xIVq))bQ`@mE z%Hlig{mwpdsUM|8hqWC0s2(@dTG0M;-Y2q1?3Cq~l?-4Mnkh?UPNc6of57(7*@_1%biR)S?TEP4^J?J;1(v(q4 zqW<4%$gEXT3i}5Zg*fke{qR!CIkQB8coKEe>7=4emWlm0`PPh)#(hd@OV znAnAQSg-zD>w-(#uT3U#9;et&K{g4SWRAYtN_t>@iL|*pMGY+M&&2E+`f{ls#KZL~ zw!OKHD!LP*48!w;{%q^|Z&DtS%ZB6Vnm;WP;-t7DRfW3Q2M0@Z#Nc0%A}4NPkC}1r zACg$rDmW`3BeJ;VDVU~;TKUf9awRqi5Qc9EPH4Tdyy?LX;ahN`!rN!Nd3Wa1l=imm zLPT!|tvPi5M7Y5vlE&OTb)+J(ZNZ{;?K-mEXzW>)zi%wU41`qg5PDL)E}jW8=SMvOK;Zg}Pr_8|^Z* zrV!f~<~mB`vYjEWTGt~iV2(-cnNM#trl|;V@Ir)u)?4gaXRrhd+~D*XQ*}?{LMT z=zEs?#iiH$WS(gCuVu0+{V@ZxQ9XB5nPiPS-^sfXtH+RdqS3+h4l}@`dOOVF$8fRX@cIRWo$smY$!wqdP_T z+*j7~*Vd|1dT(z>4bcu7>rYN#zIPAy5$|%mOG{7NxOr{|c1{^rzX7(&Y{jR%b4NT* z9tw{?W^Gjvl~C8j==QvSy|~*b-GtU|Pto~O+Nr_RkMBtxvGGhu>r>+q)3#)C5C<+# zj_f{_6Y|PfNoX4>%pt(GSeGTM_BP}=vEgEJ3HX`C38l&OhK|H&^tT!D2%^B)rA(Rw zx<^KWk^Z*)^)6693|sg0AG*b>b}fwQDgmeNz|OcB`0*`$okg<4TaMH9`MJZFc82y!yyq_o83eo$p zv~!_VSFZ=z10x85v()(HGn2T8{;g^_JK}L_s1w4Fq;8o5>Q*Y+6jKG6)^z(1zk6eB zzV*}oFPgGb);GwGiz2{I;LA2};T07C_X=rEykoAmH8k*!lWq3%Qxe7UPWs)f-}%2A zHKu0OCz&4Abx{E?%Xrp^TFYeQFe^W|*e1$D`}AG=-fMQ)5@^mZwvRs>`1Nn_8pk_$j_` zy%q?gy*3J_qR<9u$`G_RtpcZ8R-5oQN|4xBr+<%4{ z7AG^67ij&)Th|PSz*sKn&F-&KV-2x^n2E!fu%vgC__r^w>$rN+i`I#ofW)J?H=;jTg?7P>vI$D_9GaDCZN7yVQ$ z9+d~_BR)yO`qO{El@7&#O0V8}(b+Th`u4Q|TBP1{eas>YT+RO&sLg=*);WFgwS2eu z<#E<`%5VzlzHc-fHpB9Z!cJHfAK#P#TNv%%xV?EQ${zAgy-&l{Za`TbwPjw|v-Dr@ z0!+KidpfLA92-;uBFt-Y?OBie+EYI~`kE&~&06hv^U!|1kpxmkLFyEs;RN?j5o;Tp zIHmg1s+&+AZhw|9Ohtcxs`bNVfL45eeV{5`Xiu0ih7r4^+3*L(A1cw-aLar!vob7D;pFH!zJMTg>Il4r( z1)KOh*KFn~2zE1tmN2T@VIiENq_C;#Oj&2*D|#WpD4a|{h=Xh}5^u6-qNb#doU_q) zDRW_s%+_@JE6W-;L`_|7tbUj;+je09-@7cgPt8L&RTFYIwKWTzIxI zNG*CSH7OK8qul8f^xY7|F#;CKf?ZL%(3X3Mxmdjs6j777t8$DT(&KYg|I7r|{xi;{ z4RO;}7dRBCa!Z}CPu|R&^O($;v#N(XDQ{R5VuXEdw2|6${J3VR!$lqG!z3e-?K3sK zckip~sFt0jCt4W+Mthptg#Dtx47+xRyPj`uW0g4jj%Pq+)r1R@ zc)r>1Wn+E2OA$?qD6eQ9hYb&;N+!Es@#w<#-0$J%-w%zYz5+QpqQkbl&4fnU71eo2 zuG54WvcF$&Rx}m$MwB0l!~~5k$W61Wxei07KakChceB@2O)I4ahApPjQUl9ECCXTY zAxRfL0({obZkI%09ZQhd0y9!;f+}+sj#ZmU-D{!j12y&i1(y@zulVv~PSin3S!dkp zJ7MKz!lB_zNOr>Z4d~=~s#3^6RE+C%ySivGaB+bY!V~yg&C+;*{Sj;Hl6{_LEx%F! z?;IP!ma>jn*7b_F!t`?^yh8G1MzQp!UY>Dxd3M1&w{P;euVU23G@QIbNANEYD%fTX zfcCN2tgmul?{KmdhID4neYeF3-bco%@*>Q6+a>k^Spu$S-(OdhelEDgo^b28Y0p!d zB>VBbH(e>aqee2~b)OM#MJyq)l-$b4(f-23gI#vACm%PH&W7k`rqy2Fe}7Y;`LF2~ zWIct5aECcth!*}w$A9+o@!!mB@I*YLG;dBA0bq@zeX{?*Oa9+0|9cZ!c(fEyLZJ2t z=hocKqWS(&=Knv#e~-H#V?~q*boeE}nTiJNZi%I!XzA5aGq3x<#}^#YeDtF%AtGGq z_%+vFGq2;HS8G#p-J?q!PA?Asy9aaC`_Gm9AM+Gw&Ln4k$rnUoH{s5ZaN{@BWb3_@ zWT|X41lLYko6G$YvsdCw_GB+Uh1MXOUFQdHF7dPihWeX$ex`R!4|YFIz|PK`92nlC z7IC8j?{2t2RiWdD(-F_keNg{PD!PgM54XKv;o{-Wy6un=ExL)bbo9VsR?MI1W^-V~y?zmRn`Pd~zPWhaL;?~M>o3>i%FNsXNDAE0iZ<5rQP#%DMhYNJM2>%g zcU7$Zx0#vQB7R;G(oj(?_ACJL>fNvIDi_Sv$6vPB9w7i?zkBCijG;3naqcG{41#k6 zMo{LTpGj}@Y$U5e6B7lSe7N}6Tik3{0>H5<+>{?0A=3hq0`)c7I{ym@&L1#6)UOK7 z!GUv#c^*inyJyFpk(sD}Li~Y-UA_$>QBrawC)_BcsX>b(-J-yvv8q?3e98Dk)54R*AfgvZtup^K> zTf}W?dW7?TL2%ILmE`BvQ0~kyequM3b=LThpG;nM$3dkGqpI`vBgn*tyA|>tN`4XR zBT8f~ydLv90}b18P|%KYTlPk~*%ZLtOg!KTLg-svd1{y_0P%fWhl-XuVobm&ELF_r zWrOejoOX`Z|2-EsxDAddyBsfC0efCz*Tw0o%l*|Goy1M3yjQ^AyXzS;X4r}Up$7un zl0eM9fA+a`z9N0-V+bvSao__{kod5k=yFTqq2(cy1*9IeN+8Jp5qNl$NI>U93zSvA zjKSb3xcTRdg_%VWvRrse^;k%Ciq^iMq}@u=o?D;JooPXBkbs#}sq!=Z^Rf;6tM8Et zb(__NO%?s$Z}Dd75AGPR~BWQ-;vg^tQP1xH~69E52uS8q}?jaA% zQ*4?xG(aAL(oejW7m`IKHFU9+fsWxPyhjux8EhY>nTQXzJzulvah*e<#^xO zR00txJWpjVih%ndo&6mISwt%fzydlYBX&ELPXWGv1KPto`u@It0UlPzZ16<7%r|NI zhSPP6c$%aL;tUHD*CS+3kEuh$WjsYs?yI-7b!R3E#aQiFUb1lvrEK&G(-`NH9XV(v~O{gG7>A&cq%gMBV|Hv4}OZ}ziL9EI07qzrZXX+6llu)wrCk1k!C{>kmzb_?KAi8|PftuY^w<%>kF-4K|c!-?HjLP5xSK1{@9 zN66xr|G6dm>;Jao7lS`BOJSkpP5;RvV)^5$k|-?qM-F=Jd`?Qosxv5gcE0}AwFl4gnzORbfKKB{}e9`*rMZ< zhYl3k6f<`9JCbs~Z}6IxhUGTHPF@}a__HNe%^tOX4|4-o70;LXUZTAhAO-0?;WDuT zP;AfcOaj(DxCbsS?P5W!P_ZoK>CD|2-{1qnqz|7$>53@r@q$Y|35W;*E<3tSXXglR zmsmmG&b9HN^}x|4I?#2O@XgS=#m1k++YdM!!?r6aY~0565uHD7p8SiV<@Y;4EtTBE zTr2%{^#zvXMVy8ESRbfs_pcF+fQp^nwEOZ_iB#yJNhk)5^<(d8*l7RO);R3$>_txB z00#^@8hRI3X@c2>uO=>`UEzIwmxI%HR*7>SS6u40I}@CyUW>%&t&oqUU59;4hgY-= zH@|6JubN+Yp7CVoU!V&<(xS&YSG!Rfb_bQb$clrN_Y4@i<;z_z!?s`Ul#(lSIVc#~ z^Ms4S>!%Q9AyOaniZv5%Dt*Cg(%9RUvKb+;Fad0UL_cQk2dN!JT4{bMa>_1*y84mCoATW43GjW;P@u3%Ijhd_*3 zjzcZYKX&#@mhliWNMF>#`)37Yi^{OcP ztV)XI)uHdDXwZc6#ciu$!z0=5tOzFS;k8jmR>9}Ai`#iZYh|Un5d}n!od&uZnJRSUv6)(bieM?iY`rcY@LIKl?ofRJ0rea&Irj1#Y`tU1h8`(`>&a|s9WLLac@L~p$tq<5vh~} zL%t^;c)c)3qyC!=+}0!0pn(in02lXN$kI_?{%W{v_K(xiYHJnZ%%;q7%qhsTyNT-p zQ7SDfweN54dGImk95xD=u0sA%akaTAi(DFSlKtG4hQjPBO_+zW4_+G`wYxY3^%jTu(#+fE4|9gy7z7{HY#uc;%+w_zV|2&Q!-YjX_tAW| z`#boN>VXVycWxG7ee{0t*&trNQBZzY$U(q$o2tbXL%`;>hFwxAR|#quv>nVV86Z;O zc1ge78uEy?JwnjjOZsp})qNJU+!uYGn(z0@t@A>X!r2&JHN7-=eoMUBW6=4DnC?Of zc9bh&xE#b=-HhY=3XdV=;doS7?^y*$Cg_CH&-sb2AhGqaa;baQ;Z!9i+s$h z`c9jSQ9J)dW8T}DF#joeY8@%(#rFMeiyt_G_QQV{8b%*j+IXjVjY`MOAO3(e-)B)k z1fIqz+rCX$1F5)p!pB{Xyvs!2bmp(lxU-0cA;lBgEIAY*hKcLfcZiI{7&$HMxw2kKUevMp-&Zhw1$A7-SH{Uf%mx;1@eG%QXu=jxRm;Zpyg zS1UmHSqE0_Yd6z^yLEgg#WiNVa{$XTy@ew* zht~<_R$SdMn31@0ao~Cj5WLQDInU*HF{zdaxQ%f0*f1*Pzb(gGUqTmkjIK%N%UoB0 zm%2W=$H!Ofkv=ZME z&(Qx5C->gsD7giO0u9CeySdPCR~3A5GY&jMTwff>k_KS{B?QhiDfn{WSM!G}2lLbX zm-|lW;8%K9VDfS?U>s->uKKWe*OrD6fw6a!mzRxg*()>dE7u5W#hOpOk-uQ0&iV%yauhMG5aqJ z-bXf4Z0_-5LVtcBa&?c-$&zu76P#lvB%xt2R}YILGao*_Mp?YLdoCzK_?ux-J-75Z zuDJ%aJ3k@J4%SVqSRFFw%ah}WRodx|-GI*Se!V_h8^E=EH5*bne>$AVhB>+{J=x|8 ztDom^6SJ?Il3szR-&9fw*1~?Z#?u&yir|jcO1%iXAsx;UL_gBXIc|;P-RNoyJ5J;9 zm(1T^ShOD}b2L3Y)^I;wXbD~q+q*RAAqb@WOu1Z$&hcKw#|l0`Sv>fAufIlJ^S_QA zP6zMPV0!SDQM~k_r&_L^$m-JITP;Oe1XYcAuI$Dr$9hlsWTMw?VpunrOJ91$r;pxC zDC4FXhG^Ha~oJiitSF;to@vADBI1_O9UaBVP51kQBl}pd%tB31E;8H zefyc5=V7@~^q~>|cCHDkrvJR%W<{%g=XBZXP6p*Zbv!fp>1IfHy54SIdpXlp`1sJMIrNun<4Q}B z`K^xY1!~&!Qmy&dHYPVCr=j6N{I5)jpbx?Z%On0l-7<+h=q0$qQlARN$oj@LL#joh zOX%vf6ST&19}HP3Hk_W@ey~%a zmiJ+XCBj1E?$9+TfA;#k_Et6yc&Q;hyby#*>AZup^ZJjZJ`pOR?VfcDqOn&GO5^Ih z;yJLw#Z!L@UFeA9sX~%d8g+blZr=5wW&DTL$V+c~ubB^2&PEDot+1NCl-TgI0s=dBx+4YfQTVQNwoBCPue20OZ!(O{y_eL|vL{pE^3knr6f2e5b-n%j0{z^?#%_ZzE*&l^qk zu9yEtB3w943Z;w7?;2XV!RB2!v^PCp&#!G=T0w0f>^5|Xk%e>BG(T)Fr#g;=dGkK= zNyR@^H|f?w*$a-S)w=eEEV!RdF0?wm$=iNj>e=;~!*lq6=;y9N*U^T9x;J4rjD?u#kHN{h3{ehdD;Y#7M@PYbEr4f|g`#F6X&;2+7~myP@a z;)7dTcCOc#Yg8k|p!yuD>%&pZ4dqSCZ;yV; zOTlAwG7Q?1?^o{Q{waaClsvGO6GQr5!YwS*mbl8=p;JcsdR*f2t^@5c)h&B@w=)^M zeqaHxwWQ7+^thO~z34O#B^hEcNt6o3hV2f#ew)muII=w5+}vl_uVb$W3( zOZ|w292mm0#sDRRzEt1xdW-xc4Z$t{3ml`qn_fuA`39i8**#~_W30SmwSQg3KMC!S*6o_`=bVQ+wwnoGZWH5+qdwsh0x zX4yNaSGin^I>ZupSTbEgSNnM$hIFo4gu3$}fC=W`lhB>J%)Ga|LwTC-j)$(pdNX)2 zb2_1}rQ<7VZvJ#b29_84yz6iwWIZG8}CNtF?DtTHVA7i;OF z74=yvh$16ICRU3`;`G^4(6NO7X&w2^R^+`wM)$8c+86fC$l#WY<&>2TzNJ~V*>63g z^*jm|ai(3W5|P}5F8#s*r7`#g?7YeRV)ss#NP_I}aBLj#WvM=or*7ykbX)hQy5dwN z66kw;N3a}9Lz7>=VdFQ~6*?>)!s&J0?6E%C+_--s9LQyO8ApC}P~5(^#r5MLVdO6w zd@y9u`@4L%Wfx&a?kgGdpM{w9b{Fvpa@2f2FR$);e-Xb81(R|L%h=uwOM6No)sM!k z5`(v*i^HO@PUs3}oET~WWq1`- z>$(W?u-ED52f$AxYE}{|MEFJ4pQ#6|Q<)y6FNY6xA_%#BJ?Huo^8MD){_cxnQ3*Lzw%=*How-G;VVN{*Ke>N z3VFEP6OrHb0c?Tz1M;WQJ5&kwo1I#0pJK$L_mS));6)DBgylv>TFpUa;8SY+``=Ij z)AQ<9M`I=}2Ml+~QITAB^sNz3&4ZZZHW#~i68GY*Km9l9@}FL&zlZz5tbJql7lylT zL}$0ny^vur#ju3B-zH`P!MKbxwW9$epWC>9Z=oUL8p&c|ukOjD``!f^9^WwW3toK6 zDSB<{)g#kzbh^H)rrl#abRn0&dk~WJQ4qZe@dFLVCrZNomap~AeP#-7PP_0f{-Rwi z&-%qxtBn>bE_=&ARWAKV2kOv0&7hSH`>HXR@M}YT5wb67i7<_vt_o4;oF}bZJlm$i zqZ90h9#B_OI0#|8qD}7 z;Sa}xqVDTFd(aQNQP8QQq5sXlzw$!l`~M~DY01yESz4U6%HW!bgOha8&TUT2c?;jo zqN~{^!eU_rzmK7B9&?1g43kDTYlxK;uiSVc?0N?$HN@gqE!EMyOY}qXbpj$TLy>zl4NVShxo7?9&XM^XX`CAIjm8b}Y3UQNK8tzQlj zYYg!6qu_vD=@OZu!LXx5kfVj)?jjde*x|ewN{??nFRVS>Zewy@!4S5mrH{5eJDQxh z9z~k*nf( zyR~)3Biys5=jouPxgt&St;* z5xh&eeK9qX$)77~QXj-{Fcb;-KQ85#%DpUXKKbZoHYz*+j-2K~f%>m(4Jf@An=)c|t_&-QxrFUT8Z)xDZqDMJul(|t-(Yc6Pik2b| zq0b&@99NNQ49GKF|A(QfVd8x}7q`@CQgf|_e7CJL3k!KlDf^aRm+D|)Yum3>AS^Jd zZOUa$iqB%$L$KtyktVbsXIFn5e|}rTxkMl){!Gl``f@Z=bjDU4A$EAon^LQqFaCXc z?(&62$A^_hNd6HIaocs9)JmBX0TIX{C%l|mCG9Ig>bgBkFI7TM2(c;!-!NLoaAE)V z(0r}Ps@E5VCqe@0IaWbS0}8WrnIbc`8{e4@f^E)pUAT~ebMJZg|Mv)tm643O_L}RPKG%-D zNAJKE=6t2@r0)7Nbtfd~xcRR0X{8I_RLgOZ3>wBL3anc!iDJwg-|_;w^P-|+oOicS z+?R_wy3gvjUaRI#`NXv`Jy0%yQ-ybWy4q4vZWnfJxY>E&Z3Y{k9}n?oa9pm6Ut&D6 z8$ynSLH<&gzv>FvfBhbVUF-~L5Pl#Re&9nTac3LMf>y7gq>#e1ue2zvaDrNMPsz3Zm(B7d>;{hEuQLuV|JP!!gqML=N$F}{r#yRFA&S~ z&Yd!bb4YC0gzs5hEe$gA2YOmM__uFyf1;X)%jk=X*i|%I>mEP)>OR@ooIW;Apf^O! zMdaI8z~L*vPgQeM8o#UcaCQcr$9|$T6HRz`0R{+M{ayvORz0~G0);$0R~LYQ%1+^J zQh$BtMJ#6hGT%#nsmy0zg}e4a6y^pMVtE(`oCO^zk<;GhkqYQ*~qpG77;;$xv%#anudA~VeY_WZQAFtJyzXMB(L~B&1S9;|9W8Kq zQ@49&+qpeZL-&#POPewVtNn<$-9%kuvAm=fDqJs1H6G57OoY+KCog}|w@vKXf^+?p z%|vFW>%F=XoEf0g{GSt|er~+*$4ENAOakVt-(nNX|FRggUq;df!;5xnlqtLR)zerJQ ztrTHJpFl@RH9URg73Ji3fp8%q!M5YR=1p$bkSogp`Si~c`{5*NUw;Jj%_g}Ue#!#p z*-l;-O{T9njvcz}tLVgSd|WdB`XQ}osgjOB)6hvf!;ad}Q>caCy}_m=aio-P-R}<0 z!tj=OzH;ODK|?qaZjqIWzUquwSLu;8IR!R*trOO(pri!*Y z)EThRz8ba^Yu3XH?T#wJWi>q#mj^{hL{YMlzbczsww22wVGZX zLppJ?sd;ePhY*=}aPHiNeAR0(jJL6Uz*kuMfG6=Z%M35m6P;!ok-X4aY1RjdH6 z4S7!c(3qBe?16Sg2)FZm2aDT32ON6rpnr-w4lj5$^KBGX$b-1kXyGP|B0W!Z#^?^q zd8D(&FdftCM$tX7&PY|%S;E%`5i>MriUUQa-_)#fq<90 zRvW_c)FuyI$p)?tex6czo8DULS!O3^7T#X0H?=gA653vJqEA*%PCU`eLBsfdwq*P< zacKTSGDwLZsyDit<~&Q+NHV$JMHb#JqH7-CZq?5*e`MLZfz9+&p^HZ+RJUg~pKGu? zS!<;*W%0G=9_~%m0OYwAu`H@+1i!&4D&}&EAGx#B`(&g<$?C->#%^bB#2)#Akp& zzGSq~lw?l*@%=HEt=Vk&)>xXia=uyQvJSUjSJzTO@WD@7Yin!5<*?6W45xYy#g!Ai zKVpT9_l_DeUFOr}T8&=eM23iy`hI4?+l}^E-zzIuA3w9H(L#cMB_>w?9DVocLqM;w zgC>JyX-S10dnieXE%(Eyhv*Fp_46d2?Fyv7=%hCRhp2<+p^+xtg_Qv^XS)udX|r;R z&flxZZ>)bsZeqouuiuyf*YD`80=}X-yC=&8lm>0TjvRJqHWN$&ydV(z^90RIlJvBZ z;x@7~VNk6ssM>Y}&ZRuu5tk9xyPdw^6q-AXMi+77rHMXlVzjSp z*0HSYzvk$Jx}EGbx*7vw8|P`rlPu12bKTYI=zqHgFF6PZcUoZ`GMafeyQ~8pCjG^? z;(?o&N`xFv;`J;-Bp5=9#JnguChN7^`-78F4)*j$+YyliIJ9N{Re^)QQwM{X7gwJs z$p3v^ou)KcqrpAh(U6hFtl~H`BcS1ZI*ZXiEzLA;TDfrFjGPM7t@%Q>1TAF#LSb$! z(}y1W^bokeP$oXsf+8#2NwtV0qri!_C?$bAW_|+PABDFViRBOwyMU{tmKIyp7)V%| z7Ea$nYEw7wGSjT;MP`E%dADy0-+{-)54=5>v}5_TKTRhP+FdV8!{&Xj)Nar0UWOJ( z=8m1*MEr@LliPA*-HsR&wJ{UFI%y{N79Q=LXLDGO@+UUot%S)$2RKSy3&<{JMm`lS@aGOj!-F2h6>UOT5A zhgk`qz0cR98_$>8a>k5{xvg?|Ph8#4!PGM)jq__;M_7gC=Iu}2VYX}8@*6^j!Jo<8 z&z5_K@q=_%Y2Lr75k80MRp4*v=b|bsEGWJmux@at%@zeWVT;CM$9ISOWE;(gJI}X2 ztK`GOEbw|b->QGKeV&QV%E*M>T45fj5Bikz@S*7v9?M<^li;B0DFV;p#n8XODw4<{^G1nw!Z z%~Z4aX3x3Rk+gT`0;I5<>N-saD|xKM`ZeJhu)pr&&7ZSzx(Bp zS9WM4F}i~(=O-=hiaT}<6LXHD`{~ZC8coNxiBl^%-U@RFEb>%yhtLm-%;y))CYN)& z1=G6ki?-CE#@+Q9<;$(bGdri&-5OY5*A0lITN|QW^OHF8lU!}^t>&?&$Jt&b+Z#Ly z)a0EuI4NxZc!d{lfT~H%CCP}{sPkjY`bV_PkMd-A(Zcdlo3>gAZiWZ@ZB*i6;=R0E zj8t(}QnLniI!y6e->X@#49G>&!t50H$I84e5RWrp>>!OclZoHnkP&;Y@Y53=E(cI4 zp_dp#=>5D50oCh=G5?Fnv6d%;ec^i`j>8*KHujdC?l7ORX@;!4WsDl-_@t!9Of(Mf z*>H;{3<8$iIp1Yn@%-A-h!9yJi}^x!0y_v`$=Sa7tWD3QpMlGMJ+R-z*leH0ySV6L z;|`ajex=TV^pBA_IZ&RRpPzij!NFG6Qd|Am1$Eq@kWX$n=H9;alse9mfr>@>Ml*5% zRSsYT!#;D=JXMYYSgj`7%R;cs06DU$Y%#&n^anaO+ODj}b*t%UNUVPaq*M)=tfsN9 zI<9#Yb3C5OR^tM0n@&>Cp|Q6bT1R1$^$OqAY1IVnYXl>8%>BV>&GVzXqIDlkg#}$ zEnx?d zPcBsKH%Vrz{3$iZ;bb0YnM3jmncrMid<3p19vu=e1;ooA=rY)=1Tt(a`5#>8++{if zFcqzLjkTH1$hpdwyIWFoCQn9f0gVqZ_CF4VuWYFc)Glv}5#JHq%zZZ+~#j)fCd=9KyJsaw&?dk z%o@%s$%&Vq&tvUIs2$|Q?^9E%U1eWCd7Src(bLfQW85EwyYfoK)F~g8?bY=VT$QbA zG2$2i4ux%3YSY;z%-O0T(psO%h$^Loo}J}W`&N?!yP;2FtoAEOp=iI+e#3uFgeqG_ z$<#C2&@1=Y^7&-E?z%t1Y+sB39u0$L;Wd6LPFVjQEIW(_{eAgcBqKB}mRAVyXc}|L z_4GxzZRl@NU#ENRmrzPb!YHX}_l}(SzE5hZ-{rSmF&w*i>1cWQQR4}Q-KNmDUAmM1 zJVZn2?zqQWtg5FitKE>%q){)lL5KvOq4t|aRV9H{9+@ z)yqxM3TjE$hDVGw{XcKastgCA>=_cmlZAQ)P%DVfy+=;c@ibxo?eZc+6B@};9}`g( z6>&k&NYH^1z$T_F@^wq*3Oc3Pu(#S?jOlEbuQVzRR?Lu-QvQhWKs52%mz9BW-89Hm znVx``iD-h9p9tCHg#Wip;tfjXa=ng}zF_zX{wD?%Rc~cAy`CphChLS;Ou~1)i|i6k z{pRd7S`4#uUW@H+grrO_NC79S#O)V33u*3CW)p%oL@?ON$+nC2Eo8hVP!CnBLW2p< zpTu+m@7uz0e)Z4r$xbZ=sM3g#mGL~BTS#|1M6K)s5#$Pancii0oDxKW`hK(XNOhcP z+qYREb*>#R)qpTbo;Glo8u!ANTznW(T?<*hnCR-i~b6__|{em>lUPxN%$ z?nSm=+}D{^fT_PQel5E5fPAg;JbX=>NS)PtvO>MZ_MBvI zI8a7Pl0q~!3~rfLH2ng=)=fFF(n2WwYTWC}SHmftd!gF8#kl|QCXMe>kLh9JXQ<{X zmCsUj)f>a=PW`R#P3gD-IkHfK>b*z&SE;s^!}O%JW}+zSB7DCL8`C`ub;D1mzv&5i zyibKYJ7SYT@u)U89QB`VOUVIiJFrg*e=T`8xqqcPiSVq3O>8s@3^}2rEpqMk8DsYy z`_A#LY=Wmbqd02KMu^<<33gS4yU@p7g~f;`)zEkG{%qw}8#9CP()>}VX~UAHjg#L4 zpQ1)i?}O>YR;Uj%#qy16D-fnZ=;Aeh19j}b)j{}>)6Ru8CR+3rD6BgWK4z@v+iz|=@zH3o^Wyy{9t~)CUDogrfn{hh zcxmqLJNHf4ALd-&#MxdgfVA)lsRiIE3*N>4EJuo%g2asD5D+&GpCj_y;nVL3Dx zQ8I?v`MH{|+6Y47jkU2#Vs1+IL@Sp{3?H|Mz9cm^Hg>`kHdxw^v>$xdmhE-t%^ovb z(3W(R{1f@wd!$Tc=v>ZPnx^DWekZU>=Et%JG__Yte-0w@_j8@O^E6rS>LCu505;H1=Gp6ry_UF*HWSsvB{4tSpMBorO0KuJUnyIlYAL2W~rhrvn#2$c!TCd8}e`#K`r?avC zSy|F=v&?sDw6aiP`qX;COst|3r#SrMIbXs30QG#*zjMwkE&-afkT?4X(!ong60_3w zs#E6>6YHU-i#86*{|N1d0yCy+$GuCDc7>?oXP*=sZIHNQ&uz&N_wXiC#~HL_Q}HKd zlKngGBi~Rk-`gwKs|&qv=D`_%Dwwj-E}M^o#)42)C=JllEBljNNytnRvp=m-pX5=2 zW)`kVpV?>j^%PBHWESZyU<6no{SsyG>x=4A1%}sr0uZykf}ZS~-*-)h^_!42Xrdkt z*LVsFStw-4Ce~8xsPJaxVJ#ni{cwF-%hkBLwVak*>Whz4R<`ZyT;E-Phz@X9=dss%R1nXsFEeeY)-;jtG|uIxtSX z+)3}y25;8^nvWM#W)pVer609Z*==k|z5DxjoM#Ht%<3p{nWzq?Vd+3fO>Gkq~SxA()uAo(43 zD;AH5om>AQCt-TpG3>9s{=Wl#T2`{J!AWYhWt(=v zXqw3F>}W9sQOOZ=qxxqx36W8}r+ka_3Kr=#Qs^%eR#swkoQl<+4FhdQJ!YF}thVvc zQ7a4&Jma0p>+RbI9l+Yn>+VxDBeVw=JM>R`{fh71M1u4iL>`Opx~&^Z6GH!U`XOX$ zD92`J*H22-oh6!gp^7~ga{lp;^0;x4blJDUM^#XyCgXb(kT615dxw2-yRRp7Z*q&A zp5`=^w;SwWnjoP&j1Ob+%Vc}-Gm9B)*^+#ov_}at-tIj7VMap4=pJQP?%S>n(zEKn zssMimyLeEmV=cq1$Lds?IVJjMVnahe?p5R9tOXzi9_qF;9}B8%bMK*Zw-J&@R&l?% zJi4Vf9NB92hwfIs5$m$A@B9Ng?T!Kwmgau1;13Stv%bg1Rn^g`6hr$9%+NvnPL$Pm z%D%yPsFas1d}v!HIExee4&5uYg;u9UJD!t9$FLc50RvK!s#qo3*-DT_!=R#MLBw|nLca5m#U>tAHpZk2cabdR0RI|D*Sr9 z*;%ij6(v-3WU$sNsY_&o#jUyBoqR)OD43=a?~?K41CiY$L*8xHefUqzJAX!o?jxhsBC!5}qza>#g@pkJwfC1Z5#t%=o4FiGmiI@FDg#?`W#bY^ za17MJ3;|diooIntw0F86HK$Pjb2*rphX~z9aUVW?czdSQ(*skjgGNqHQH?jK91-!K z5Q2_}oO;5Yd67K@`^3aHubX>`|8_tA_01vV!J8*A9!_}lH_9)s8nPV+DgSd3(6RGp z)&$iAX?fZHv+5m;`~OM8v2CrOu+X?}<J};;;1QczOX`E2LL zxryo_r%%U9m3-&|WP=vb&{kxwuK%R;pcYmrD&Dv4(AO%blBoFU<$=Q{m2nRY^f~j) zIanC}=IjTQ%ZM4b*qQNIKfCEvOkzD^+9Id|tt)z8`ARN)|9fr@pr@yw!0qxe?Q&O6 za*p40;b09fk)GSJT_zVuZaX1Cs|kP)dA_Y`y-BM$9KJ>mOEd658y@cXdFvS-;rf-v zRgIX~54MroXN^zh%~k9XUY7BMAhc_^hVI|J0MG*1WJMwkm#wUv^j-`iDLE@=a>8>_ zP4fHq!GdS+ver{Uha*nQ8Hw^v>t7XOTiCPywFGG9rlh3&B;o(}_KRr*g-Lk*1k6K1 zPL^96ra&_TQRWqQkqUwuE=JR->yq&~GnM@-f8 zhwGgb%DCS(P}5X%G`h3zeEGs^$gDkW@$`K}#+lph!*PDBHe~NAA1^w^ZI4F~>g;-0 zm+R;83w3d|Wo-xv=XA8R2>5LFFi^?M8bA;pVLtMHTo7w z&VCx)V-co#(-vBAwGIREnbB&}M`D>~yoB^&Xnt)hLOikugf;`MX>vUZQn=Ru9{G?t{xTJpJ$vJ@4-}Bh` z!;>W5@LiLG+nR>v$EE6hR5YwNQS8k}jVsa$vNmd1Kqa4?h0!07_RpZl1}z2z{NAv4 z#2cLWCL5sRrm{&$rq#?xMh0?IPwGQB#9IHs7*mwypw82C2cIudd=fiuHyrMNOY?MQ zrMKVeRLh8CWW=}A+R`H$I(8whP$ib|wX@2@_3Qlbz-G^dvr%mvNXBKTFC@j*ryaZ*OL1 z=jMrSg-)Z*cfH;nBxi??8(RWX7y&l}$7oB6Yk76+slDpS$;#>)1JJz1Vuv07!kQay z3n90Ph8of@^9=}Wb7!dOuKnszjC@A(DTcoZ_-YR=nVTtFkWkYM-oN!! z5+3er=+67IjfJThLF+6|1rOmz&3UG$K&q;aH}w#LG}k@#OsALE1-Ivl>R$LhF3#Fl z%k)r+p0+ge3R%8ca%o;4nT4L;=2OdUw&SzkPZ~m&@r_9uCiuF(ZL}mz;p%;|?()-; zPt<$?X{IpV!$cZ8PH}FS&)y^V-N{%Sm(I0wLwMhSPF`=GL>99>#qif?87V5D8@Asu zjp6`)|E^j)G#j{Ga5*|)rMx{G?_4G3q0qB6Yy&MSc4oEikgQMNH@FFBx!eiz=wk|i ze}T~kTdw0Jmm0m@^3zBG$9CB-r(j#%x_vx}AHIAx?M0JWqiHBK3BTz>dVGDnp3=Q+ zpO5Hs55Wb6m;AMB4%tuKr0pzS$_N)#*75U<-9-u|jDhH%$osj2XPSl2_ftlQ`2e3xXHg>^67KYuWP;7{81fB=(N*%~qG z@5oNll#!*&#hnhcv5EOvX!#UYr)4xh-o$deuM4Afv5nIHv|3j&!Q60*+Aip_beiSy z#R~d#?7Pa$NESOT#N-Bp)j79a?Dwz(tZqEe(y+9YmA5gpnpG{$_4}q}A&g2SzbuA# z=aCSP)9;nTxKD(%va(jQ=hi13t(PrCxt2$|iEMAD-o+>);q2HAK;2fT zNAoznCR9=t2IRA+9cKuk_c@Bg@Fgj8YOd8LbJ+z=?s+2lk7o7A@kyylVHg13UO`@7 zFdoqfR?U$K^u1mv24jmSPN^+-5a>onTaLFRJ+DiOFXrdmmRMtF)q&hs?bnO-S2i!_ zAYIn-`B2VZmCbu$H~5*|^!`3kKfqp5&~wOV-UrGg%@5S2BO{CLY>_$od_hJZhT^t* zTP}fJxdk6`D#|qBSdI~!SX^)L+pq4rYJ85l5HJV_%60t~seq1BJZD>s(T2KrGpo@V zY2K$Np{95m^k6nNM@|1k)FoyLxETuy%ihcD^uyHn?xeEHtwTg~>SQejF$u}0#m>UZ z5H-MUa&`}UeCA!rhESi37vcKdHo#m5}hK4hX#sK08zzYvwjqJf!b z1P5~GywFq*4OsvmmcLV2pVxu8a~>`8gTo)Sbh8bukMTWy_G|gs&VR4o*SnuxN=xhW z;TjvIrJ>Ulswb<>kvviE=7~j7_{;RxXG(k2;2~J*yR!a4%Uq6c=-}yqzN)CX7d}oe zR5+RT=lq|8CO3S;ezXg$dp~|mVZIMQVak~MnX?V|aNlk)67dFF6L6mi#5F9a&ke}Z zZLB_;Q9SU!`klXCOi`njoC>F8pf{GVU_iBboyZFCOpP*ZCR(gZo%I95goj5^({ZH` zXXs{Au4U9^?cNA|0zWE#gFgGD#!I!Sj{vwSaCm<|&JERl;1HFycYKMOUM8}v6e*znq&Xu*^hMYF!6xdOEpFJ&f+%d1lz zr!bo?%rhGE59ZG;3qHqD3|0x=&D_$byvtdQ3{H9vgOKe_uqQ5_>3S6x>K7J;H{3X9 z^Bp{>p=e&WKvQ|L+WtCcA#!pjuf6%!Z7+tX>x|D{ac&vxHZ#Itf`uh@ywQ~h-kb@m zbGt2d45s_rUajRqb&1_azPPZ#Q+p7vtIvM;=0VPB6Ia*lGW$i$m^rDPtzDLn=A?KU zN$)R*hT5*5X|<(v;o&?*=*Oun+6%l>#FEp9H#ATdGtz5vc~y9H`H2k<)>#|^qZ=JY zMod7&!ZtON(>>(2?)&=GyqImyDJ$4Goi11Fw{H)MoSc#9|1^P+){|a07C? z_N?*3G)JI)NQh0Z+gO^j_bzO5|4<$1tG-H%?IltNN*!fMuNV|ovFaTSHub{l>!^1%F2L)S%&IkOm= zW|YV1FMa|xrzGnMpjE}P(a~Y(4l3`^RPQ!u^6o_`oB2Mo=1=5&+UO2i%1G?BG99RW zyq@H{c5T%=E0PB^n6Y{YUeO+R>M6`i$_}kG90v>9d+D2PGip}J&SaYVf4yZn6#`Rk)bipjz zuvXCp<`tBN)|Og_OC2g9QawCGWjJF(ds(bCpQ6U(pnm!=o&jXzog~=Imv;6ByUc37 zshX$;AV(YAUtPbL57`X+%~FoG@(NZsrPuHLKuD4hC1B&Yzq%1{vZH+N=R3@oX0&+Ho7>z9Zi=0go%=gYS!D=&aR5 zI#=}20Un7TuuI-xRN#qdp3x8|^k?gQ9%a5yrwcDm((i_x+lsd+&akF}4k+G7Q`uc_zNEB+=z1pvBK^Ua9*bG%6Fd+=#1*>Z z^suPsK1LV+KKp?^v*06=bjM8dX6voeK)w%h~-C;%HwtJ5$hPxl}%JcNwpx1GUKik&yXiQ7jQG`r__uUz22P9@A1j!7Z zeUFgsPGBjY-lHkIoQg#Mx562$ikVPH0K9}!@$&ts_=EcSXRP{X)b=`HZ3bZuDBDMK zTs&>X+2vbQ39i!z9pC0eA@21w48^B}p(B}p$K9EI^o*|N zf~g5|*ulN0caAI84z;53Y%p^B#ZYe&4h~2A%Z6pn-Y)daPh!_|I#pHYK#DrOw4C(T zEURfc;C2ph(Q|z=K5X>`GSQX2l@w3$Fbn~Q#BO+I+1V@@d9?moOdVsJj_%#Xe*K-Fj8^3m3I&A1{%IsA2l~XrncykbvYt``xd;=#>Pr#XowM&#gT4jGrt)T)3fDTW3Ik3 z+S0@+>B_MD-7qjE=c@-u&s}zBX1T@VRxemvNh`dt9<^*PX2yd5CL%*9jK@-Kx=%WP$q`(mG}j$^1s4 z*sw4^&^rt8CQfB5c#tPk0Vd+=%JVQ@Qmf{RaXx5nZ|`|;Po+n1)Fpp_=TuLY%MuLB@Oz+#-V^oa^*dB|qpWa}%`xdj?Y5Ih3&WL8cSh`f zs;CTskp8O9>G62Cj=LOxlD6LN6MnzYar@%R#uTVZtmCDR3>5V70vcX7gC^Pt*f_yL zq$rSUmEM;>^O%c^>$%S`-i+c!jc?^TLWDF(^!N@urR=JU=_ zJDU{0fdcv}^5dP@)|1Ru8Gthudu7*G(&r;?7-&W#`2mB(XL@fN>PV6EeJAvHB9$(& zO2WlW?DAlK?Q`dP=;TxS$4|Sq7~2fO-4Ek`|MyNGA`oYl0q!;?o$vDf_xy7naG@Lk zfF@Q>Wqt?D1EJd0XFoM}&vG-1H+60LkJG3xh-=((`YfH-4)@T!CX`uI(|+pWrUTTs z+O#UJF`X$wKlgx^5)(V@8UIvy)}EjUVDmlG+w3cSoJu@^rlWR@GyOQv4cj@-KXR38 zO&$7@VgUUa6mG7MQK31CDNka-=)cYF(6_xWz|`0R;Gi=DLsO=g#!&S#4831jR&gXW zE2!E=bc#p|O@Hy;t&T!lLD5e&6LKc=JvPLpMf}2IKibVaO$!OyWtwbnmoQWrf_kBe zg`~2ot@098uT7r~Xe#D9wfVpbxUuyAIelRr7RSxAR!qpA2RkzcF9mtGiMgUOtN)rN z@mbn(VV%$?a_1Y1^?Si0J+MA%nTWVFt52hm3{T{9~8nf9Z^r*N9gD;9^2soFOGwM_hA0Zaya*}3cw0?sHjLOC=~suCI6ok zat38TS5ew3@Xv9o|Iejy{LIQMu@KrcMG=I{yCw^m$hu}W*t6qn-0wL7c)v*3= zL-ntbpco5YA1Tr{2K>ci5Hvkrt@Yi`MxuXU6XPrM!QjqqpftuZuk zYdg^@9j%^w?|U7X!2VAvx+8~C9~di`Lychpiwro{(*a#S5 zcu4(YGX#Ui&pxo$a3?1hCMDxc`(jfvh>1xgrY&Y9B*4iha6oH6TTy+$Yv^QcAB((; z#$$T%zZF+XND~knfey1IqF1Rd6OycZ7w_r7agDg$|@%eFE0is-!A_!7S($1PM; zGy<*I{~ui>6}#ow3@voMS-G^aDRt!0lW!!{U}qS{54-m@KrO>>+#!vXD}Rem{7qS3 z1Yq)NHkhB1cS{@Xa}5*p74D7|QT_E8o^fp(WKwb~qE4Lhh8y~VSKI!tYR>Mx5pQ#@ zvr2)dPcCX2+4M^v>a1J1|y}iA`1N2usGJ6~)XK&G|v$O!PKW~j{w2}M8-;N!GP|b(bRH3eJ(ewVJbhb9Q zsHq|&qK*Fo^A>e-8YPUzl8DV^-&{cOT-&~7Z@#*ZGK=Q@>4&soV#>jdo+UU+&amIV(i*WL69X?cpKQfQ{Ws~> zG;N@}DiAcq&zmfjfxM3?$EKeS@xA@#50?qEt=g(q>fGi$xlGn97fCd# z>m*hDI2<(f{*y87QQXM&&18?1+O9?$^fj{3tkL4^elUJ?8Iz}t^$_DvleMoZo$G!9 zRM=tW%gtELkR$eHfy(q30RwlO5$Y}l|BWl<}(WqzEofK}2^5w<8S6sM%5 ze3z3ZrY+41Wr*iqNO2GTJ~F)!**8+~P3I#-m}kAJ{UV9lQaAUQvt6&ILyA-=KI*zN zvh7E;@l)q5X?)$y9*czg^VEN{EIykSN@Hqg0`3DDS<6-=Ff3c2*$-viNc86Ws!@Ry=fNd}!9mMH~3A4@_WoN9}OoZ-uL zw>8P?Ug2;X{zYxqSP(8;+l;t(QWmVQ3jS}JZ+S6QEiQKQ#d`}(E-e}eq4lPV&waxBYrLxnS(pHW6mo8oM zsim8Z)NAYcu8rHty_aCQAK7pk4c>L&@`_mOuwXrZ@qle>J2b@>ywb4G(do7E7rZw} zw~FdGE0%L*1omTCr`>c6zwdMuJ~t;#>9C+?2My$~=RI#cgB#qM+sG#FgDH#TGWrjk zfXdKSkotk^LWM_!pAVU}T1-Ah3VGJ~ev?oR;sw^);OFqB!KcQS2CnUzbxelIYN*Jt z0Ra$TX7&@_B*RiPL<^34h6ji!ytg|Zd--hXHMnkw=z4amGbzU(iLYre{1bM#nV})+ zY;7!ba$~=@%ENTDIa{}~zUJcmdT|C|EEN4CmE1IkQ} zD4|}af=RpU#ikbuP!;5-@|DCp3DRi`@ORik|DDm;1&?+lof|W@s&u(q#Jj|{u6->9 zdRx9*9rw(>zMSWnNJYMTpI?&dPF!WUFBuqJ_D}5AMbQ>oW?VM4{7>gzo-ZJ`<&HTw zEK^Ip;iY2BxNN+yZM0#TVsTUhRzf$3ad@lzE{?}x3LOFinmFJ0R2#L>?_-DMX4 z=INUVDqfy{}VC9NHDkhFxdm|71u?BGi z0}@4{2OgL^2*IySx|>Z$%EIiv9468p zElG|AIl*XfRr$_FVRW%ggiE92OwVT!>1t-ubgJ;UKuAqIHHCsSxvwZ%a@NHsfh0U> zS~4tIZna4yG{Q*#9~jq%oLcDP%<1v)RfLL;lc`$5OoP)~1{J|fM2a7HagS}D$S8;- z?o2(uhaQWn z$zub)GifYzL9w-6(tRxN*m*k?8Fhsw&Qf-xMDRr)wiNwE>_Ir$fD;V4Y z_}R)z@YqVQB4i>x1RTh3r4SP)*0m2@i?5%jq@}bzztVo&b-oap>b_7?s#;v8sQ&nT z4_@sd}SjF_A4@>?^B%BTP74zz-)WZ@G$gXL>Erf5kQY z$(-6`(Qd4rBX}Ie(CFHanK&^kihwBAR38DKljdgwVF#jQVpgVaj z`41a7)*}Kg6OMN@hO*D$th>NAYdD?>YU;A3&yG?^oifZKA|e&FGktx1J8WM7pu_29 z$mGZfIW;vxKxwmQX-xgj)fj!U?AdMw_v&Uv&YM2mHe*8B0kTP-FinE26o%N6Hw6KvTY^8borXW+giZg zXyb=Z`t7#&s+iNTq0naXF;Vk^&~TJwO3sazGlu2G$$~LMnk$L&JZ&Xd9w`TKZqGs? zVQkS(f5ULGPFAZuDpx1^$r<0%MQ%+L~Gw77s zRLija_WPt@=LIG%zs`tl6ie1h2X0$d z4AHg)t=5LX=23gok)QU9QJd>^QU1Q|DMTycW0}6#X5Qc0asxT5#?3`FMsp+&OFlLz zT8n;|^BezVG=BWO{z+a%TVkRzxxD6X=WMFu;LCC-y9wS-Rc8dO{<7&0RA34f;7Xhh zGm*r?B6XZ0tt%lj*FLJ_+7F?iV<3>3ts1Q*JNVWR|fc*r`CIu^{|5XubQQnz7I4@=O2(XR`AgWriXW4x;oQb zn_C=54JoDp(3I91y_nWLA6mR|H@+9|@ry~Y8eef1bOEkl(0n$^nt4pPM-6>b$LE4_ zKW?=5zuf4f9w@z;OO?9zN_9T=H;~Jr9a?G$^k>wOP`KN;4EtyxR5Ed3bZmJ2C_2LI z6_qZS$rIGN6OKBl)|%l)+R+>8eQ9Gs9=bM6{BFEl|AHW+&R-g6!?MPlCFim52B*EN zmJhC;>K5opBp!f|^ajX11B_$gK-ZNf$Pmh!yc%aBqB%um!}DALqkf#sEjQ)n z3kg%ou8B~@VazvZ4rGjR@)$3gTyN5f)4DFg_cb(|Xwox}ke&Zh@OC2%sPG*eK>PxaQ80*o z_x34+NvS$(%+2A#MTXkX#-UZu3ByVMb8cx;AfrPazZJg$j45p%9Q4;Zf{laIYn`Y% z#$(Pq2X`=N0QaPa+y(R&aGoG$^QWW9kGCFM&&T_Y1`h^fYd)%1w>u|}puEYsu0mBL z&#Ok&F9hL?w^h&0UJf&L-kG5(M?Rz1Y*N|~rYDTa&6wb_gSu-@_aryd`(1dZGs2*8 z__G^z0O(TYs%?DtyOXaIO9p@7`d;r`$W6=ky)!T^B} zub8^dEy($+PyQs7oRykIRzz3X6w#cNI=eIcr3WgPU-G-p?$^>&4hvbs_C@DfGYGKn zqT}8Ctfe04LfG%^6&?Zu-b=_4Dx{|7tDM{?3t0-S_+z_D+8&Q6eSI1T^kKb9ql;?g z)WkEBy)}3Z%>%}hI7+t{KdN+D9}r9QM673Vsw26+1Iy7>qq}W7w0$kJ17}*HP$U6c z=n;CmVoT4vr}%NwvPfD*^}YK(ZC@;-i6vtLYt9IQV}7%lE(R_Q&(clIvl zr!jp?#cXMILR-!LfOk!KpQ7QlD_<;vjvLyAGM$m=R#P^gOD?sHLiRQtifpfx;Kor7 z+x91@nGN3n&ZDWb@Q}157C@$Yiebx7HXXEygZW2`hcC};F2|oeq>N+L+B+5SHwOCi z3~qWJ@1Wq4xa5mM7QTkvKaWV^;gjRy^*S%;gf9g%&>;%VS%SAhQ4b>e6{|ke2h+c? zL{ztP*W`DP77?~mW=1Y%?ew172d`b@55yez$Gi*i4O#O03e3_p91Qj|H_F(_c{AZp-E$(3ab2FhR9mLLRW^lccWM zBCm`yAnUtLzpr>x$dI2#X*%YD!(lMm9$8OxMSyWlC&B(o;LF-ys%QGxkau=Ey6(bp zBm2ZPf;hnH#?mmRn!nrN)tZ98S9SC{cnIctC~bYBUm{8U^`oHq{eeZo%eQYv^`i>I zOJRqsps6G~X%8pLH`_**hZ(KU6S{t`59DFB6%<$Hle8E1xMG#<0J7!x{>ewb0!q>^ z+%>?!#;XDC-mOvF--8_-Y^R+@mmipgeQ!N$1lq&e&xM{)S%nKvSoX%C77Z@GBbx#zXx3}6Z@J< zx$6~R0K%b zvBKIsxQwhfYqpw`cBK?|7Wi$AwD+9`wV!{`Jd{5^EwI(NQY#Bl`Uef;VJQQQ%}isJ zPbtX#C^A<&&~fS|u(`e1#3*#EF)AgS|7dQMd(Bb3-Fn1q^_Ci< z@U#c&+DWG=CeZ*H-7X10HsZM8`MOX1UJUY(o!R$*BpM74bHKm`7LWJDCGG8gQ^U*l zX#pQ1voWtaa8vUnFo4*NPf=YfK9Y@&7G8FE5ZeE3m1QFD>=IR6?Yj$_?YSu%Oe1qHN3sd|D133daE2A(TZalP(qh@?6zdCRj4wHP{+`r8coXoY1WxDcUZM}d z#tM{yU@-}K)HcwLr3h}j(-fy;`oQ3a(hWH5OYnuO;lubZf{wT792{Y7G*z8ER<91i z*$}DX77vs+0~fhgZI*e_r1h}hpAlMI3rh|$M0tyZ-~tyzgzH|@$WI7USsv3@iXyDu zJbTCeMWSB379D5<6o#+@%4=|39|_TZs;P;XlU$`}i@!BW^CWDTAo`VGOtXo0|2$cZ zx+b-&9^8XfT5jWwfb50Cjyr|%{Rj>qXI;M)*<#6rMAOQXnOxO@(pk*k0bM%8t0X?q z^UlMkBWFvOwug_6Nrdv;SnZ-J;HoW{ktX@|6~v#PlzK$4nG)JIemaDJa?YE;9V_UF zL)tsG>-N#Gt03|#v!U{_?E=#Zs$Y8qq0^v&T$<^CN(1p`wJ3gn>vGGWY|m7;xV4_F z!-3mVQ1jA?Od5hE&hFmzK=_lQ>mrX!R&@dr(r+D>2N|R*&W9vMs^`JU<*LD7lg0z8 z5P(zD03wvBOC=vSd|F=JVunu#_Lw|9)v(PF?E6)sk*&5k=assqHXbLFu#N)W(RM&!4HG=FbXGEvcR5}NUDIg{F4VEJM2>9R$9#0>Mg zp7?QH>T28IVl-FJni?>&bSzlNTv9gi!p15T``Eu3@z#k*4J=*XZJenS<($o!gE5M1 zdTd@nA#AuOmDcGDwR^v6EmX9csh}YxjRl>lDJRsL4nZ|7lz+0U;Sz!0o%kC z@1PHb-H3W*MXP)fFZO>yb#nI*yL%RW zmKN3)1)+6|tfOdbz>TPGQDzEVVIL;6h9{2AGw7fMq3!ID#Wr|R@{(;$Nn*`1n`n2Q zr?O>$?1@S>pVYeAtuT={7>$k-5#%0X!cnj+M?Wep z7a3SqRt5~9%hc4Q(Y)DJpyA2mJ=MIk<5Gt=)32E|+C?CNI4}2l?cER&`87G7Ww%%h zN*20BJ0DN|t>b3*9cfi*UG5l1BD=}FCtQkU2oiTyt(N12jtsDnBHB*sbw(yPX}_h< zHVnV+w>L&dV%&}Ce99$vd$f@Tj*Bk6tT$c!AxXIg#vjBX*=il9B7dwfzdueen|=xO zD$?*$>>~p;bR()xPe0he!cgt`lcRXfuxfjKe5)`GQ1ThLJ)N8{2xKAo9O(bC=#7X6 zWVDtZ(v1&OL1~{ARZ=F0l4YQBhvHA#Wf!nr)T{Dnu8oHwdm_jBlLDJsPYf*g)|0MO zlWJL43gJW!3Eusr!7WIH|E1rU=08iWeMS0XxX0ZjGQ%FXQOJR~1I+JUJtDc_%+VSn zd@S?ki8k=vKDu(=aBW7YKqwf+is;nTb8I-cDEnY2_Tg(_yh?}LiH;-OB(;Gg6t7)} zmLLF8gm8j9fwq%3tEt$y1|EGNYo0F+LE)3dtjCP9vH~--oC&SeNLy+Tcmaz(4!1B1 zCoaN$aPLNkNh98H!F>=g`R07eE~{VLxjP8HRi;H~JNzk2_tf-pl2O_I9Ul}8`IKkgu>j)0D^=B}cIJ5E- z$&p+;B)laO%)9C)O8zESh%^#Ia`TFe)Ngpv6+H65i%X$YYTTE`>|GOvvSJS_9hhQs38?BFFR2qvbXPcHlP&2duh_U-%U#Q zp=OLVh1E)bP8$VpHMUjR|B_%b*9|@IJ|2Y(C~D56`b9VB*SI1SDL%iYGElpmz9CXD znO^~Pc^I-%3E&~Xy3hxQ(Z)|y@kvP%a&lwRjel+PvnHc%H{**6KMGmHo!EzAPsO)G z5qrPbVngAq@c^2g*(@~>Yt_-E6Pq1UNOg8-9k%Zw@#ybES;}yS^Q|{!`>6s$kJdvr zP;!f4?`=2x_Px;@i*zI|etRDRR^j1eTiMgRQO8x4dco_Mh~SQniLs~&t3=8Ua_H59 zxX~*ua28BLhrm$7>n$GByGX-@x$df6DP4iJ5#bQxaTOgt7~g)O$osd42xY{l-&GdM z8OB9M>7BS@{8BryNOb=wu9TP}N1pG@~IO6lE-730sX(dy~5Mj`~ z1U!7n9wfp9BQY>=NRWZqT_Q#n9|ziXyj4t$3PGJw^fHsjfO>|;0hZmB=jMGl=4HEr1Tc;!3wGP-_`b*%&NiNnpDA9L0WtT$Zwwg z)^%i5PVw+3d!U1Q=2W$#{G8}=Ho=lw7na_5A(#wSSmhm>${D#ArUZ(^{7E zR)#D<<)UXtT=p#8X~fchvWhANtQhqwP8BMxNX3K1jfgQqKnI`L!gca(G@uUdsI~)a zm#}K1JKz3g6n_cOFqCAe&|}^7PQW6d5sH9?g>~?0v^`yIZ7eT!RMIPdXmSZZ8ED$x zZ~dw$&-)C;fo4ob&Cw?pnMZ#Fk6u5bU{%O0EhA?vtF(hSYRn6uL7j2~-Yrh>u;c|l zi?Dgt{Fb7j4`d=&oZP!v0#+z@?dZ=WFkd~BDRjIo3a7n4avhsIR^}^+goGHn-YJ!m zdR8T+BYrR2o?epoXFK096mVN=^rlzX{NQm*)=RbRUA<+zy6`h|!dPidoz>$IhH3Z= z3jB#6tU7Fw*Ur-PeD?zagzz(36SO}8jq(xYUYEg4M8M%}0dp_o8R6{sz)P>~gj-oj zF4kDoSnf7_59QThc1-67?AZWlg7oTy5<2F=SZ}`^U)uruF|lIRz!2r-L9waQZ2iwj zxt4tr4{%xMS#1PLjx5?IIhKCtNazOEeYeyiIpuUed~tHc#BpK-U%8R49l?XVUtRPz z;N(PBroNx`OkPSl*8-N%-Cy_26v^_xXN@e#KNAXhb2X}YbzR>{8++yw^1b>L-a1ta zEF*VIMYK90v);L27(lvybKRLhhsO`P&*&eEK+^26C!BD^pOC}fGbnf{>9pDZz^;N9 zSIr-IWyD;dxD#?Z#113?hzJlrI1YxDF3!aJXeiDU-u(ReF4Z5Z0eUKB9KteLRp`&_e~jLkLy!!1yjKzOu&T8GKmGa=U(YN$y` z#3V7lw?w?2n@g^{)38ccRvVWhgMO>j$nJClVYL9>9&AgO??PP;lkzVvZ4t0~D)3uw zY!4105g+m8Kg?vTSc0VjurUA%GofhYyyp+ar}o{8x8y|n^MiE$^S+lWB_BB_gYXQw zp78|4)h9M9;=>d#38wIJ?;{*j>T-e9^LC>uSWXI&K&5b}+NJm25bINCVdaXY@eJ8U zfIy%WyJf>oeNE`=rrjss`@1y{dWP2*EhVQm9?A9LG1~R7&-RR{q!YtW{>oP#+Q#HE zhR@eNYfkHOlKGuu!5+huBVV77Nf-JA#u85e#+|inEY2_{r|+z3*BLYzF)4x3|B2W)WJN_+;}UL?=1$Ih&lZo7f?Cd6k$x{hewuBB zyI7~30A#r6p8c0hM_Xl91nE0 zf0_0KZ^)roJ-R=$PIF12DXCz6Sww!)cd;Fuv4T2~_+dT{Bgr@qN~Zu4VEEz!2k)qS z`EAHubC&G)0rC|PieusmM;mx>ty4Re4G9O=MtKav3yaA^P zFE8LVoRZ7qE+H^<>%MF{ohme#_ z`3aJ!rQ2gPwO`(~EpY{fl4`+yGmzNt5XmlO#Djc07}AU{uE z{o{vN`w}?G04n_GG@CvM6~aJ2<)4xvnFNA@#cfSky^XEJ!tK(bxMI@ulY@};Y3!CW zNy7iI9hGW=ecp74>l`LuV*rL-0})Q^Rt%R^EO#Z#0`?MHI(qvchHMa8Ms52N9|>bd z=YiG?_1J#aAkg7_Rer>u!AtG)qXjJSq~caM|JxUP4UYR92Ck0AxR)A?%nFQwjYr`n zVb5sTQsJ$Hs@2r%kb`C{WUhCgg^m;n-fhN3^Qdo54(1~gxzcN`%bDO%_O1eA**$FG z>{!cf{L$63=f*!I;YmW0Yd`e=TCIwl4VG+m@Hs+ZYEW7ihx`6O5Pvh7|7 z4aU?44~!AQM=kq7;FjM zNQ4`A=twSNHrn=LR@dcETO}`@P*aH?2&Qt=B`n9AnN z@F$zBwu!Imv+2FfSY~2>MmB7K-UvIkG<`;whFaC9+S4E{TSPG6oS00Qn};Xp=9z5~ z@KjuBa=a`zb$AC((VrX4QGyH#{d)CYlX~5G9~RsFonHj|Hd#qg@FHP1YmA&Ux1dzx zq(%EBP})yS%L-5RYPf>of|?nWIGApbI|eR-hSRQ3P)y7Jw4x<#J4-(#$jI= zoRa74e8YmRz#@9ZBwi8p)kL_Zzb@T_@|d&A(R^juYK!%dcL`i@A_xKBm?rEp>Myp z#e+PCA1sAvA0-}=Cuw$CR-1s+77vklaus%(*Up$K8pnFi9b@KKDBs5y8lSYSbzkrr zN{d+cx`%lv0FPL;>qgO}MeJc$I%0k?(k#asoB%+en?5-vYOa9ZJEDq0z zyw8r@BlSp(VOO$!LaGr^92+uLxqtnzn(>3Pbs%-x2$2mQV4v1+{ISn=P)B>L$Y)!$Qk4_24 zQ?+F<883V@gA)C;R|;V|s^T&3;CD{>Iu?g|W^)5OGM@AgEeC^>x zwz=?L2G5MwUBKtuh~+Pej}L^_n@?qB_girzG+rL5Go##x763gvQg%J=qrES0Ett0? zyjf&!L+g!UoEPr^Uxrl|4x|!_6hYxaWe0ba4{pH0M3F5x+j|??0ck}NKPPpzXBmY$ zcI@w%_aXx*FT4Br@tMcFd1t)VOoGydAs>*Vi;WI~K|s0HQF>x_jcGZ>PU7&9E7x;B z|77$jmeZxk$0b@uuL6QiP4}OY1dEUX`{X=atZ>E`l;*at2rNL?RVzHhlI6jjJp%m2 z#D~Ir@8Tf4lMXnf7=0f5V(B!I66-S-(_QWsFDGe5d4ru@tLB5Z!Q*yyO5sTfpN>)& zcd)9CXpJu{pBxH05bV36Mckf}O`ci=j#-nA7WW;j&yH`h0IXD{;qD&PSoYI=dacSL zwr!4B()Aa;324*)%k`j^jjyKs$jgR*PlETZkmpG0b#kEt=m1IFi7;SK!Tiqb0bjW@ z$gH+~#1cBRWqqwp)8=5|!k3~I{iOCUjAGwXFgUtVtT}sWsQ3ybh?2ZNT^2Ji5Y9z!WqLCXi@??y5CiIGlPxwB?}lc!+YjbeM}O2I5YP)p zu!0;~Z{|$2L_$J~S42~=1--RhABQQX^E=Qmzk|KmXFpP42&vAR5{+59bK0OW^7N1K zN*p$=IxBaA?R} zF*j-)WdwA~_CFl^F0vOj`9wjzi< zyQnxv^u@+4F3ejPBoS~`^tV-8SgC^FZPBUO|1C^d=#cbZdKTo4&qI?`7PC*-6Bzbf z?8{@4GBUE4N$ME0v`FFFj19Heu%>f&M@6!V1UXl&o3G>zpQsvC>hoa<(ebt#VZP(q z3~pJj9+ub2|Anb{`T}k)lY9T=&TckgUUsdYkCibVSq^BHr%(79@BszZ?4>739j3;N z0BV`yQ%;H2r-v8+2HHyVljDC*=l?ywBC`{r=v$dlF^nk~7_15n>#>hZqr#eH5~J9n z52)bprfkQYeO0Y%FE5=lEy+E5@XUsPG$BydAy~_5sEGf2VRcYOU(H!C;|zit(fbXN zEKiV>G|Rx&sF|yCRrTZ&R!JWeu8V6VoSTc zl6KV7vrG7pfbp3HRV%_hT!N9dEunyE&CuV)By_JfI`scs#sBxTFe!97V;~4*&?x;d zLX1XYJBvii0#whLiTT)xBj#hFwfI-fb@CpkvketJye9p`j%!!o84s%6@P^Y8)6P%6Z-d< zzwjj{;$RUj6Z+LCahIoX_lVes0;;MIuCLKZ8|O~$^lk8={3f-gWt@>%(il_6)dZwC ztiC9xBFTwis|{%E5!2oH_(iN25jn!V|1whKKDw<<%#B`P zkP-C!>K2`7$fd;!zgk)(YT>R5Q{xJLvjck8vZy?OOPDs_w|7CiM6rUw(_tUSF8!iQc>r-oB-AbYWYu`Hi!W8`s85xQPjC_U{h6b5ryyghgq!j=2lf2HhNGNqKxXu(Z9TFQI^8adC zoKrL{Uk!8?WdQ+wj6Unr>i?=n+1#A}i&)tM9X%^ty|eRw%<4aq6M}^@UDnn5P^YS* zUE5s$y+A^Tn)_9Du9P*^SQ*(2n0vOINVc{G9Kjb4m!COfA_947Y zmEc7et(6VZs2%E{|JFPs{a*^i|91p4Ddi-xWWoiL64eu{L6-&yE_3v;Z_19us`BEy zi!}G#fr>41uhst^?>aaw`_Bevc?hV;@HzQSu{*iiBcn1zFQm`*3mpq3xuIW6*)qz5 zR(h*7Y#>zJMra-IckVpW6K{JGbyUPf|6~N@QXsAM55k)NyG(Fus8TFcFmtGQB9(Dr z1@xu2r+USDw;8S8U|d{$C3xAIilT%aF7WrgpjE3rgBmed)o_9z<&y_#^%bFEN}tXv zYro=OyU;%he)$4!!wIa}T(P@{Hn=O)b7Z>AJbA(0`sR+E<{X-1Fz_LUwZH6ji&FPX z^8VJAmjHrD$+86(8l8J4h?9OLJqv8SB3a;VmK<9SNruR+6#jNB-TN&Mos_ zG?(9;*7K#8MT@~NJD?}v|8U|zdpyKDsBu*J=@U<6&Ie>n(%k<|)&~B>%B?al1XWsJ z{`UvsKh3;+cn$@E*o!bC?;9!-{<98ndI$`|#x?f&x$)PO;FxnMb^nuY^LI(zyj4@> z&k*WE{(mT2ms~aYLm3`@p^MsO!zmgnoYCmHhZQ1H8}5xY{YjkN08}R@|oFLSvylPq?*nY95}t^3uP@WHh5ymiQCnD4?Q>W5AZ4?hJ$rtmZQd zW-|i=OGP+c$#XlMua-KPFZz$)1Y4m3U?Oi$NAgcA&SV^=K8o!L-`~l*GP2-ARRA=3 zK#1>riiO?`52`w=^}TKcsRJ3TB33vP6o6?rlPRfWK-1S+?yITzc9lA3fO!5q0W#9{ z)gxV1W1|rZ!GB}!wUzpzvEGXAJ?U6~R8nz8AOZI?)&2yTyvmIN(v%|uTH*UeqqWW} z)@;sF{r?9W3?3y-b#kXoo+S=mJm-e5Pr2ZDilftII_Z~#H zH33WOe~NW4w01dnBH-bU(_YyY{eaG0%XV@~K3m5lY5zE7G~ zn1;$J^^kRs9gEWTUHyO4X|!X}<@(}q6J!r*pGyq}=et#DY|rGd{l*=+MNmNFkasW=iBHh>1TEr94hW=2=H_qePIN@8qO8g8)ckiFj(KO zfAOBsDtp_tkPLy|2G7DWo|+(M*5^UD{j#Ouh24us&o9p2axNZ+3n7q$~>jis1Ln8~#tQn{ZsQ!BkihH8# zhTekr+PZQpzJ%QNZr98cTDI} z!$J`WbR6FWO4*NI6k#~K(T6y}0QwA>AkYk~{G!_=I?UwMiM%Zo#0W>K&UFhukfUjRyvk%_GZyUuqN{ ztyg3))Faf`F=YWY23Bo-yh`$Xm<_OLVnm}Byn~#ScVv0oKY7@F?yRkxxCeEBAY9u! z5M2)u-hkz>Ygmg34(icc<%}7J-mf;1Xd&T!&1=r@{HIqEtmnh%sr)7|i*m=KGu~-) zat($TDr27LYkluNDaayo`95+)Cm%{a{lc}IFgX%M|9PzWzSZODG#+`y<0pt5{YR?P zs+syt3=6vz1MG^oTeDvP?`GbN&G5ZXu|3%zkB0U<+1eY^LSbNLMkzBbjR&AG5|j83 z+}+hSjWRtwQ@lL%o()5_nnU_Y>dZaK^b*Y2ihg~w5XIp0N@z=j7j>7(;97VFq0!2A z-{HC(>l2$ji(UhKZ4E=Uol{V#IWgVPHTp>))063{*&KZAblG$V=Ec@2XYsAxYE z)X#e7uddN_E#;C=ZanBpp9VUNx;>i;eNF8-%2u!hM(I^ z;dH?Qbu>kCO2a=s7U`5}3&g+|fhrUBSt388m+u}NyYQWFo|~-kcm2?6xVW2Ssce}L zrYSd`=sUz#n$#uZtXT1 ztt`tdR=eKp?4(aJMlqhK)m$z=6L|hOlLU6+N6{_@0nN3a0aZOTk7dxSQGgeih=da1 zQSyD(`qk=HE4j2_O(Q#FT_wQ9d|n>Wi;za%Gy9!CzMJ!v%z`zlmo0I}kAb$7tgXUo z9pcl~jF!q;TT*2X?Ckfe-AzVi4rRSE-3}wdMo>ML;5nh%8_v>xI)2gYU8-ss^tq{l zvtHxgYwHoQL%4TcF!k-9a^-A38>Lx=L6y9dTX*X}v*&{q%I|tVw)mfRyd%$C>U;&y zI#ySb{xI8=t-qz4F_#Lt61jV$L<8?ZvqTSQi|Z4T`;n+tw+-dhv+j4T#~?MK-ld5< zI)zjDo01Es8I7lJcf6a29hDyRMawiAc_DBMp1xF8@lb=Elt)r(Oj(1OtIVaoonGM= zS@o-*_RGp_d)GHv%p7~Yzt2v_5WBlvhg}%eO?t_0eYzF#kHKs=vp zmo1%qEC5Y&r#=k=d}0d*bpz!LBE4OVO36U9Cu;}`OQTJoWmnB7ogX=N8jqMxwnQ># zXVa{M-lv_UTF~A-feFB9pQ6C$n=LbV#UClNLYqWSDI76DwhIGE??=RBZN(nK-8yLx zT>^N%I^GvqSp&h9DObn9T7u=}6?@VKTjQf0SNHqhnvrFz_|92)8_NHvDro!^5|h<~iw3j4l78BN+XkrHnzSXgT`#{ZfW?wM=BKT!5gAqE;B0B9pS z7KI&d?Q*lB76JGd?Wax84CWh7JK_h%w|#sd4`x!QO0%<(z5}asOs?xw>9>0ut*Dpr zexmW%j4Xp~bM*C>`c{gbG6|jXrI?|=3yOZ}0q9ymerYi090%*Z$K+@1} zdWGCAJCE``+CI|VC&4a6oO1_sZ;SMkExcFJurSK}N;o@-ABJUUm&D$j_xb6(sf&f} z3cQD;IK2Qx*GtVp=^xfytPgI=vp}DT_8mGAZO)>%^IKV8g4EOg(Dce*Y0vDe#&j=wi z&xWFHK0BTXa)*KT6L5#Nu%cEjgY`bQ1rp3xrkUsGI#8+rF?p@Ay@Ml#^ZM_ns%G=h2I+ z)AE$rMNGU0)_gJbV7ZPAGhyw4%rIv-;+|l~XW3D=tB#|{whx!RVf9YFOzk(Yy6!s< zkX=t@eG0zGkIWn61<);qjRAwu_lG>ea_uqr2a{ljkS@5gYVPYD;e)HK+uuLWAX0lm zNx8y2^|Xf~xNp(t24M^g&WN2u)du(0uBI56$}4JHn-CQDEb7%pNhdW=gU+OYtMA6S zG;cJ>o9@9u5ha~we(g>qly+Xhjk;~+_wWG8B=%pd*B!scd|xCbbumakn?+II=N_#+ zkKni6iHygvsy%HyA(6Mg!YGG0mv3a@w&n?gvj;s^zSCgyb6;UwoPu&$_lUenqU<^g zNo7ft4_+dC+a0;Z6z&6Ej7Bo!lb9%J91H+=MuWPN!p~u{QZXJukJ;tB{m-_>J5_H# zQIV}F`CLAyD70tr+WGd*m=$GzN-ei|=6hbWAX#*^yS`Uvf@o1_$v6pFAeBBE>Ek~) zH73|v9*$qcuWo%91G$?LZ5B1)K1;Yd4ek{Y_}RsBUXlXd5N_UiQ~sFUid806B#>?U z|Ivxjn+rNIO5k){&ALr`IlD`0f?#TjgSRVr4t)4zx+zF&pPlfA>HDk?ZA-KX>OKM? zP8*RMyp5Rn-!UC4E~3pZ9-&S5p=I>R$1v~e-DK;R0q3%#>TgU09y%-SXJOSY3#xak z`5!NE;dx)P&4_t~%cfBXxx@3aUDPF`i6sN8-8%Q3DqtP zv3$}xc|-CIl%U9mLLCpfDCiIKy zhsRc{cj_LTrgJ%Ob0MdTQYV0=|6hoeZ*`XOVsj)4o>A{0&jGSm*pE-~t?u?7IqLS_ z_rh(}3-j8H({evI=)#AcNNvbHn*^tlO(dE|qw#Kj~4Dq_f70?;D!a(Y`rc}C}Mv$Q|0+9_#X)svFOJxE=nb`s39UM;To=S{Ztwe@e- zJY}OhNRKf;Z%FSh_7UJ&7kU??8mUrAmi$Ama-67{F*)4RURn}usnV5e21**r`<+0#}^ge8+FD0UVFTs=pwh6&P&DE#4V+b{D@eadI;eK|9ISK zw-=m2z8^?R_90!?Y+d8^XJFF%>szG&ND9g~%s4BPaGNmwtP_ z@dvuHU}Nus%6_M3nmF{yNmHLx)iKfaau|wngpVQy4qy^f602JGTT62pzBprP$=LXb zeAUJR!6UJ_k+72`FvCqI)tGER_sQvqPq03^{JcAfvp(P8Md31s$v+Oyr zGx7fNx#Y=`j-UMtEp49pO8#h`iw{EY0DEBqF|;VY*hs|iwWnbbn-wPFz&FWXo8yVQ zoFpaC7*qY_MvswG^r-Pn0w?_6rD(*$?fM#5q*1ZW>7Qc}kw#ROoVp&LWt=PQm-JGHGBWX78#L_+ z_;6_W)m=z4y0Aexxjr-~%+=`OS6rH6rGaa6nQ*UZ-1XABc}wfsB`TG}ohg$%oR*QFdZ7J;LRQxP188l?n`BMe?5BvDNgm=i4 z?CSdln2_j<-)XHL0oyCBf3=Gj_>9$C8fQ(?vh9Nd?3TjkkX!V{&k)1k+ zcE|Gv$(;Pj2@DSLiB$TtR^pQ3G&AEi*03@oiX79iIfeUmPC|tixyjE)P_hj;MmBI$ z7~u1JXC_;!qgt&8t`CT^h9E-BV_8j;(_OFlMA1>Tu^+MBS#0iciOJV2U(>JRc{ojEiv?nF)~mKeZnyc}d(5 z9yWRBXq!Z$n2bMLYKD~9p-CPfe=z*R;q<#PoBY82=?_^TYxJBzZ5sj{b=1Wxw<;pc zyMtU5vIK;jQ2^B$&_}PQIyHEmZeNnC*+FS%`Bmx3(hel}wF^3NA(@E1 zin#8ex|hf>v-wq;!?DS8E1DG4tL$73y$zw_%q7O46O)i!>Nr_FUe)uWo}z-B5Wh>| znwM}F@sPavVW8nKb(SVa%&!UI(#;*CgS=A{jQ6vZuNg!{4ll%A-EpTTXRE%JWPkXD zyEb6(W30^#PS@ACP2eWWRu#JAW_ch~hkD02Hp0!&CmniGxhqg_i8(GkY`=Nl0LfqB zpHto+8lHi{#I{Y_1HQ-#bxBgSrkG@au_pjLZPF2mE5i}S5sf*5qQ=MX-r02x)3pjN z;dSW+5UDt_mM6b6umO^y!m zFB9&Pvm4Qke0m*65*HzGs1&bfG)IhySMm&1UA7*g>vAki?};p6kk;wLMjUQ)$<2@z z>JIV0*3{r2FE~}^59N|Gk!t>c+cmn1ED(ah=J6RK8l7z^)+u(Rw>jVJJc+`=8A#w1 z0{7&(O*sn(K7Cs4>S35%yi|Iff31R2uv>ric$xV3vJ-T`(`K4 zS3}NqK&tdbr8Me?!&X>pUGty3<54;6Y{KP?1v1mo1>(H3z8+sKWL?@lwLKI>ndr@k zrf${wd>bzl$PPM*AIvYU_ri-nr#deh9D7%xby5LCP1fah`3Lzkk{((ama}AuU zqY?(k6E3YLq2H>HZ2UAxw8?T#TrM#`IdS#6yn8yNKOZlVf1s|ei_q$Ww zid&U(N8`MXOm**~*6OZ~ORRkc^u>Tz@fYwLA2Z{3Q^Cr50g_Xd{c`;gXld)x(-&_w zr9v76|6nyK0xKV`5NikziVM>DH>Ei-`LEieCKaI;vREUB%FSjXLlCk2o(yXma9eHb@NfIf zNbU0I?Ks<9QuvCBxNwhE(B+9uT(6LFQ5^D(@xEEIIn`+ty4=rW@$yIM^bh{wwJ0<{ zP|E%BG*j&(j&j4@!B{ZFq8i)ipho=K%5JSDEd}EoS*46H9sK8W2I>i0F;2l13=C>- zrIDDAtL#x#%SlWQmF0vX^;1NuD*5A={i+Y6P)X!t#3=`hv<29>GsIKoDV4$vDl9P95x@vU!$h$rg;^*>{V>=Mi znq(y|1XsVxAMv-&nTb!A^IW%_LOk&kN~q(z4@7N3YJqSK_V)^1~&`o7cDGRbF-&P~};7pyiFOHsj%}=jRHhl#_G>? z#5_bf7t#$I+M8nMsH8s$TseZw7{p09;gbQSaf7#4!L-c3OXGo)} zhSzY^e2xE!c=3lU=tl=tO>&B11-!NCNHienZD|v^5>f|sGkRV{=563pA@f+$ZKW$q zSS>mAkXD~1-xnKpKXOsgzRYxP+?S_8$F3B4G#EhuLWk$J15jF)hUXnwgRD-5OSEN@) z$NzOd{+`xaTQHE0&ul)WhQ_HfU+RV2SizcrPE`KeDNqgxrYb%=O@6OlQp8DKY}as{ zRmcKLzz%q@T>}g#5_Tn!WW(5cVO7X-a=f{0N}A3K{6xI>e)zBqP?EtOT;^kX%LCs) z@Re2;=69H)%)89A-rK;So>i0kSBR`7%#1@KMr)#1LCh8&J#K(-_t6amol5Sn&iB1a zoFtOoijJfVJ0;n=Xy}VR9y3H{0sbhhju*cGz=@lw3&kJb76gRixnrv=!bMc>wb-dj z@|2@N69L~BN|g=4>+#m@zxJRj@9w=y%)Rm7bY4uAWnQU%!L=Gf_@d=h0uH1n2O~d& z$Sl9a&$yX9a)ceXb_eH)7Mke*q{F`p3Gmnd>2Ttsy@|wH?e=AI>L^U-O6JAP=$G63 zbHm{rx6;k7Uo3}dQ`E3P{V2>GvBY3j(3^DeEqIf`%WlLOw*T&IP})}@m}Ume(B{;1 zJ*mP(ud%338~@ff_f^*7Gytz%4V4>(N@XI+b32bR)r8rlx*~C%-;tJq8+dr0J78OQ zN_k{Vza}7)OM=G%6GcWvc-E(^qLu}#C+NYu1C%|{JiRs;wH1^tN7#~@suD$bg;m*b z4C+ON+czPz9zWBHt=E{?WTMDoS9Tvbg3z<;H!R6*==g4c6OQ4t2>a@wxPzumT!RM)Zo%E%gF6IwcXto&5MXh4hu{vu-QC^Yb>TMe-B(w2 zb@$yryS24Dv(q#4>*?v|>267xL<>FkY&V++QkqYLe6x~TzYmQA?518+qm(M+M-Tob zZ%9I`hEUB){lxinMlHMtI?IX@v|UDEj|ywxGJx#^^&-=+CgY{P@7KQ*DU)}&{0EtM zL409M(Q-OC6kD62B!64*#hpyJ^q4DPFL|rmO3J)IyaEr(waBDOOWNubp-#Fod=ob? z-|Xg@Ea48S$w^vv*7c7SDAC%vhD?WW++l?ka?($S9&MT7eauRkc$EYyUq(Q(3Kz0M z(OuwdBTV`QpbCftkH}KWw-o})^j<6hcT5^nN0;u&`APplinrntKXc(M<$V?bm%`v& zi*dHwU2jyM|6KK zqsT?$wEC4Smq0MZGqEvTHLY?~K*U(0Gw=9qS*B&Vso+(P!$bx-r+m1mqs}jRAK!%0 zRQpwrj(+A^`rmmIXa!|_8Gp8LVXmz=kaR!vP-K(p#W_eVRXJMauqw(c{7w4&RyJ9} z5)3a)Oc;>?M<$=0XJs!(@GlBNMDH2X2f_1ZWZTt1*OkE z>PeirbQ#l=ODnYz{iuO?S?28iWNq*k-8Pr+*E!GMnZx(b@WdAPUr+$c!K2cnJ! zcc3hxP~WGaN31=&(jyht1l3z+hUXaRD6CqDtiZ1q(b2orR$7O&6`os|B8u`?WKD}2 z=SnZ_cJe5E>~KL!xe%aE2(g61%HsXNL*eeg;tD9a`7y2$jj^{ntLgcc*QvVczFIZE z7zGDj5dPa#gfCNW(wl#{|GGAKz4Qk~TY?nYSQp1&`S1Oz4wfn6TH2qsl>E)TmU=oK zPhXeO|EQ`MVu1(z+VS#43PVMo9Xu|;Mn1*q_z$MSrvXKAVXco7Wm(qO3l;K~I(xO4%mG^7S;^@60;M2} zltoWjBV7BAHVvZ?$CoeV@0q6j0AaQG{$;GMDgA}sgeFbnin~;n&J=E<-1wBAf9}Ih#e7Ij$W~ z{`NQVkUw`uI<9xE^gOtuE*R-|BKA+BuLP!8^7|+*UR2$0hUcc@S@w>;ygP=8V8_*3 zFSYAH*=2NvJ}_3B>53y~DL7xFm}m~0wMEu zL?cnsbUtBPLC4J&!YemE^0^yt(;YS{p11X~3+R5d?y$!jW>x~3LhR&W3IQ)QCmH^N zq#Y|2){>iQ8GVB-WNhG6iEr!c0nHq$C=Miz#Yg;=ELA(B{4-J|Bd|MrFtF(RyU0|y zl#FIGQ-rG1hv(^I0*@^ne3_@N_^6le5t;sZL^u+C$`YwtdzLsD3K5$??^Cgkx}u)` z4;w{?pDJz1z8l&3PNa{lX52iQeZa&bI6@K!{0NnyMsc@NA78~6WRkxUrzHi3sEOH= z6gA_rF(+JNyi4f3bzu%nGtxV&(8>s*J>rX3+>%O`(AwrOWNfJ;`i@0IHlZD3!KJ+Svl z#IkiWx?t@^rDGAjAEs6U5|F^M(d$jM6K|+2NMD5ie9VO@#u6lz8$d-#sMF&P$Md2- z;{%9v&f;rb89X7AFg-+NZ%clkXAr311isZQfuo9~F-pufeq>e}_{ts~473^5u|zt8 zi7Xp&2x3!?1d*rQ^J6Z>l1P=wyM}7TmAjD4xFbp*6&#pl|Nf{W!Kia(F!Z$pQA^IPS?c{_wupmDfKR zk+@}wHYPz(y4e{Nff^~5#gm-6V)=8i`JS?^Q84h6i!cW}Os$f_yzi{abX9cZskRwA zn4xLIri8^iCUX&yTcRPHnm1-czEWG-HAicXH!D}!y0x)rvt(gj?{r}L%^EHZl-?@M zY9sj*f9l9(_ju>My*c}rb1U78D;1f@?lAX8%;UL;Krk3&e4TsIMG)zz)hIPL9e1vU zcwD=P+D@3R1i!|V&`RxJtMXI6ERmqJNNLNbs3%mHAl`Gsun8lH*2d0k`YCd~B`Ixb zre{0C9HKC_;#&pagM)YolKIbZW$t`+ne+8l7Q|5N;|U+Ph??@{R^8d6@|$|ejuLY{ zFA@h-O0@x0wrKpOSexV}cCN;;QX3DkP9e z(Hf{Oo%J~06jaEl-iZCLBYCHqE+JxmD65qV z6?4Vd&1u9ypQxP2tXFbO%2xNNQLHaRJ|W*%|JLe*Wwgqyq65cd_y^Q`{t;6JmKDW` zE+iIX%s`h=dyaZaeo<+hSqSw_e)1+L21U>FuVAp2!=8>l~)yNeURehBFHRXf#d61NEYGg%u&O!JQ_U$ zjSrsEPZNR3Z{`z8vr`cZCghbMjF4p7I<^P-| zExlYjfi+t~r%&b(Uk!M6+^6uQgAK*Y@!c%X+4#ME`_heV++P)5vT3?BN-4FmmRE)B zfd9$bzL})K@ZRS?!Y|5W+~#oYctJtETw2rh<9XC24b=kml0!5iDsa_VzH7cjF~x-2rrL@rdf8wab|Dn6#&fh82n8 zr{@L3z&TX^BJciv8ylONPI8d*b;>wN!8TCbnf$`Qm$|0+Kv``GSSvOSg<5JP60ema4W+}ofjo?CZq8@>hWuGz3O4z(H|d2^j1KJG_C4D zTL0!MsRV=`Q26>jeuy3w@F)2oVsC$#ihVVlORX!Lx>nZ18a~n?eMu6M#HloN*#-XW zgl0ssXffY5Lg)A@=#YYb$r}RLnR{@(8G9jCsPwU+at+lJrj`-Y8CdBlK8T00JiY6Y z7iBzHn5QeTc6T&hl6pxuz!>^immDg1#PMH0>i3f<_z*!4O~7ImZECEWSnzXGMR?V zzNpXY3t`6hw=K*S!nu1xYtk+$I$e$qxv9$Y>5otHQ71uuh^t|xmX+}gk6PGfkd$

    Tvu0I>_~T{Zpg5dGFXt zFStwNOdvFum1##_%7d;*Oyg9{iZ+T{3x)Uq!7e8K{#tlr7eu_7Cx`rL+($l=8ue?N zHTU6tXBJev$khuLZR{gug_@T}(vO=1fwfs>afJ8LRwQ2DRlGi}`Hpc}V=bufd z?cHK>b}Xy%!9l%89`vWmg1*Adb==&xfXNq$J8U8~Qr}dxOIG!w$~HzMF0(OJgFwTr z%23-9JAPaJ&G+0*w$oxwsc$E)Qr`cMvZm(TWUX-R<=uIa{59%z0`J0o(sFe+2Ss8* zGIGjIa!}zno=f|x%Duowwy)K;F0b8$JcuZ2#wk^^NQ zRb@s;9gP?@p>MqjEeyt))EO_mS#vh>;#UJ@-P9sL<{RrSoYI40do{p!U?ChyZ8TB~ zzu>R(JyzIcczCk1v_OL9H#X7f=ldHU7p_k)uZjbHP@^*IGy;O?K|E>}0W)gtW*Ik$%4shBF4n2za3eqw)mXZ$S@GNNNR0 zxWZt>i>ijzWeF?1Y+q1tL%sfF=Yo}0?O`sc>j^Dk#^0q_;ZG1}Q=2`3djhp>ND+v? z;0o{ehry|y4qrk&Oc<^+R5*?W=pbjI<*me)5vREG6r)({C+2PyH}Z%_vA0%gHLD{8 z-#l6;M#zksi1_NFOj043%qn>YkFv^R*i;nnn!I&($y{ZVY6oVm#+8%%VlaV15QVDn zLME~CiBgmwYYEtTf=?IV($Szc;ARQ`O5ZK4>Ocv)xCf(sT~5nBHuKG#sd|s-#9tel zzju=Na__8gs3dSH(y})&k!foYg(asO9gTf8}w&)l)*C)^j2{_LP2`JFY z`Z7??_efAHTpv#U$(?|a;9=y(a$m3A-!yRe4Chv0fuGkbC)*A@_qDi+_&I6}ACnn(Kj5C){)?#hKL}G~cPUAH=_EGN0 zQdJmq+S=#|e)oYu5$uZQ99?}q<9flWAhfln-M?h!yMJccJT{X32h%i0_#u5-9ZF>D zc~mSwK_S01y+I1ir6hgre4%LnxY89kNzPdZ-DI98Mc{-}-USRpiC?*aFGGEL#4HDx zcITl=VDqI{KtWP}ytCB-7tU{Q<0Sb@*`!_<)Pn-frS`Z$WZ{OafkoU-N~3NEDGtBq z+($Gp@Qr5^wM)KZ0Ny<@HQGjH;>FiPhJ5c;VpJwSw3_qfo<+JBa|wz>`%*sS<~o zcVpb-qs7Qy8dAN`!7H)WiX7O3;{mNlDQ|?am5LEBFABWdirU6+UZ;^ObxM;15#gP} zKRVuB*A4IZUg2bomq z&0&?9GYy}A&l)#U2fAoElQ{6zDxs6NnSQ#BSS&$Y2@s+vUxG5A+RJeNrdP9~8(LxK zZK)FXHw6~&@L^}t<;FZLE`I%XiU+fO(tm2$ZO}JEI+eocxZ@mRIDl~Uxwymspe}^h z30RuNmiby}#IA`*WGt#QG*(GC%DzUu1{lZCj;w7s7M_v)7-X+f3XIX$wQ`LFcL30t z`b;}S!$&PEn}~hGmwq2*^E03RHI=)_tsJ8-6CmXmHtL4Th{wUsiPF>FUH8Glm-(n1N zlV_${;7qHAtYBsQGHHY_K6C%RKD_c0-|m!lIc+9?PMFR=>t5upQEaI~v83rOkUeXN zMnW_)CWfmk;lt*0d{oTS;Za6k`t$l)#94=`!AIT*_QwKoCZ2}7x++Yax?xH7XVJHcPK<^X8K5)ThUS2sjhjTkg5r`Fg^^=wIw91c1S_G9m)1LVHSg| zk(_0dQMrKkK$qxy?+C>#osr;D)5mG-V_Eta^xpleIC&5a0AGgV(9jCxj>;LAsVKqN zFON%HmBO0h9@$ktYF&=MC~(zc3!WM?IdkG}oKEA*_B5h?!SZyeYJx)TXhYWgT5bK- zr59gbiReO~2qBz^&AQ;J3p>PQ{5UXrnBmZMdzEfgzm`#4I%^yB4TMmRzU2fbaGk;$Peh|aCmB%<2CS3v%;y*S-Nnu z=(ppurm5OsNJF3d=g+SMnxe8YEy}?MXSMp6x-6UqR8$pxSv5DuC2w1hmk4BYcY*NF z*$+%mVYE;@pPjAGkES=W9Z9^~(%aDI)27f=8zCVPx;QZ^`lK#As_6HWM{GZjpx#%(3 zz*sr&3;rCqUBD(%$qIKoIDlFPo<0Hiif`*Kg#GHpmtI>~sX*rU4gG8&nQv4+TLQB& z6<}dIH2FX{LGZCQbjfZjmL`vGeNtnOP z?i8GNa^+qF(P$i~2*Y|kTa&-YobLSCTb<>+g`CpEnMav#j-bM)oDwkO0+iBux{q@=qw?9kit;=9q z_J91X?3{<&#dCj_yk2Ziz*viYM#}b?4d*Vz3S5Hn6;i6 zE?RCWl;5jTxHYt0aTCL33s5c96Vz3hgb-FSKEkVXq;43viwDh1oPeQ2a;ZT-8HO@% zpoi5C*H5w%=FcAB__&i1no{l?15bxn5yppQ*c`{VJ;dG#FBgFZWi@ROc_@hzsLje8 z(-Fh$;^j-LR)0_yVndBhZAfKLM-)cG6bc|e(7-Ztm#^J4lv`_Fs`r5lfFYF_!&~fO zw(;XUjASWSB-;V^t{uj>5v?&6bZ=5!IuVZ*Kht$2e+wuf66W!=SM|q|ky!N}(xj0= zSS^9MW=Ca)<-v}%hyABTXXuRRM<|Z2g9?9GgiHH;lTlq75IgeKv)*rCmWSO-FmAWD z@c+Q9U0O5EvN>{agYlLg4^NfaG9#v;hmjjA+LMrvnK$`;y@m-`keNgp+Z_JRfy(9{ zik3TBRF~nbpEH~>ch?@$oGnF78?gYcF{1y#=WWIFC`Y_~uz4f8T&^DE6Z zt!EvZ1F6dWo3S_wc}Y{r5fTZN#-)1@QiV_J*>eN_T&v|VjP-O#?BD7+FgN+1>9tXA zbVOOb%$iZBhai&=R;FhB#fgT;RSj=?WD6kFucR`1Ij5>K9c^UYfu(BkCBE5i$Yrq6 zt?fP39Y$E@r!>VGMj-$(w(nn-mN5Vqb1KJT^9nkI9bxcl5G|u8G*JV%A1L9Pb1hk8 zcoYjY)Bv-6a(}teT6dtW+?-^@OnydPYFUWH$~594Ur`g(jSjmhRy?zz-*Hf21l$aS z8cp0!WAxEliqo0UQ1pHBkw6XSuPU9A#*DTM`5V%jez>k*$b5{y;C@b+*m=dJ@CZmS?TF=YDK#%L!Nx!xK!bln0h-Ltfqa!>f|$z zXu`ERi|@K?QqkycDb`up_@^@0kIC-w_>L*MKHB0B2h9ZdFJ>oi@I=k2<$u2F-_9TU zcDy}V(R})aH|yzz-;~hjCw7_as=x{Fy4u)r07 zj8dLJFe3bl!=UcV!;pP)lWu)E(+}r$*6VIIO_1j94E==)5LpfkiUtCONzECCt|$p) zR|RULb?%W3)``jVvzt(_xl)|@Y zk`WOz7`E?6d}LdQbbwc$Yj*xt)pj27pNcM6TAHwphf7|!jGk@$kz;{Vj5mqlUCCuN zp+Nv<1zK*>6KfRk&x|lh*wR-Od_mgT`%WtJ?o&@i7+0d2qS;3skLsH2;gR~(EWW5x zEE+=;q~@k?^EL~qR$~2(1DdUSa#F9 z{z2<*z!!!B`u_6Cw-qZP=SgiX^~oJ%CHm~sE`qq%*qe8w)ca3aZkgx4wl8kj0Ohrw z;WguK@cE!=-VhnD7psF~YHFVI93*TL9N5Mj7}YVY$EUt4anqTMz}}b7qnqlt3Y<^6 zjGV+w#Qt)dY>)4)7)M`14xo^Pl^nlQP+|ynk*})!y+VfU>n!pV@SBbFdRKPqbkC%j zb7aBD%)eor^Wr{p%|cHoR@7AafDnAh?ySXzR}`^l{)D4 z3uv5m1zRn5%C>Y;Zj_Mtyi+wmTn|vo(qCmWq(@{aHNGL6_X2O*X|nTYzl))^zgJp! zOg2Kt7h@fqUWBaHhwyH|+Anr}a_z~tVSHu$ z6VK+F*%rrq&9AD$#Kh~8KNKJi@x%-BY$yGWUHe&!@L5JC3DD-kiJVPGzh3aqMRdx% zR_qlR(O7k_$fPn_PiUA9NlzFM@oCR_z*{r~6@IYi3^;uZ{h;^Hn3ZN#HC^dH-Ou(e zn;CB6sLeY_sN%dULw&Tm1u3!FFUPy7d~Nnmh>=GHi>}3tZzT)2o0@+m4&FI8<;IJm{z&<-BmT@N?ev zLTIKy*6S<{@c!L+ACP&X-ISQvC1nJG8YuO?!M6VeouR|~Q_W7sf-9R`=ytFKOxgpG8bn>@;dQ6Fy>0c}va#=!>PSyPknlBvn5>3<{P1e5P3XQoQzN=ND z3b#vRwGVCC%J)W<1Tz}`!e*b~YdcGt3p<$^>1~0Tebp@LE`fxP6Zv}(YAf7w7bUj5 z6<*d6>79UssVW0MpHdl1qVBh&N+RSox%l1x@NAIsa+ZF9gGzIDx84t6v%S~5;|z9L zdTidy_;p*d$E`clW&(e8`&tYMI`J@8YWF{9K*4O*|9-Geczz+q6I!aw^^J8! zyc+l%qOXEm=eSY2MCs@^&(ZC7p3ai>^nyn@h30GQ-hTpBZ6(*4pnOTQq2-$R>s?%L zr^-?Q8_DKNPk;v#DkwW%h5VuS%Ix~t8bXbj!b09pq%dSDz{cq~R713jY`=Zl`~~Fp zfSJA;EQ|Iq_@a-=*W1G$=dddR)#r~n_A55SkOZuTv5u?GlJ4pN|L>i-f}^#;k)E>P z%*H@jZ^L5F>TUBXx4Dnu%P)Tof#*Y5_o^O$zFL@(KUKImL-SPJJO?`LLxQe<4~o?$ zD0Ic)pyXtY@_hoLwik+2WN8^Vn`T73DsFBrKkiBV^fuGUxGbE|RSE*(BfK)XS?C+9 z>Qard2d?B=E3#lsQQyqyb?v4?j=WGzXFH0`=f{&;>WaWLB##MYNH0 zUfnTQaulT|Qfs_k%6R6*3eNM|kkkkK9KVLkm0UC+x_D}$D$IUk3h8)3Ij6xL_Zzpy zt9MmVD58FL{_Qn2h!D#k!|ezY*lU!Uc%KV!f0CsRljE?!nij)_k7Gh@cYwY#7MX#@ zNvk)hHOYmb!&I%t;mrh*B7#p%jZZ{hX)T_JiM%YwL%NRg<&qUhGQ7Mo6~6C z29@I%F@VoYA+p)7N!nw>O5{oZiW|00?*$5)cbti=+J>EpQ(UqijymsW7mV3>lqe_1$J6Z4!Wf_u)a{y$Fr-LO} zZ^glXuwGrRFqRxnqqp-U0xU|#P_`7&)w>7J<7f44t`6#^Os+QSt50~AzGbTHTZp3x z=)}0dv&TGpdr_)Kx}LKC-d3-qHCV$dO3))Z-yJZo*~z7QxO}Y32l4Fr0ydn+!`L|k zlmt67lCB~~@9Q`Id8*(2;Dbxf4TeTFf5gI3uoyyXZ%G>G^F1|@K?4>d8BY8R`}@Xo z41bgIM(I&>4o8C0NgXBIP~FIB`8=^nW`qxrnJgmO+q!k;542s&B@P+fq z%`JFDnXO(!>Y_^V*XBvC7^!9%FJ;VkrWA47WY?aU6faK8O76jX*xap`FBx!a~$Rf8^K&FX(B?Lm!QLe*b~#vKb4zRgk0iJIa{`7 z{Mu;$Jo1su3Xn35K-$=3B^k4y=Ri*hdf=w_5c8H(rO@Un%-%6#0%YiA@bpE1vw}G3m*UxYN+^5ZLyzoz#$-g{Y8Wfz=-@7 zG2%A-@gUM?o!Mthl*#j74;w*mm^gh$XjaO7-+bCyS`xtfukBRuo zz5YE@4>;#mC(zo4%d25OD@G_NDBbVnufCb3Wg%k@H^TzByJR=2MB6!&2N|~W+T}0Uf6ZaJJ`bRL zU|)Y77{%!xA7_GJ_n^d!6@ZH*cT9Y6llleFpW~1gcYr{ZX3+sOs|?W@Pm72Rk6+hQ zJqped3H(JrWI1vXG6!We{W|W~o2qkWY=$1oFwu|#ueg#A@J*aFValdV^rboxh|{UZ zJrkF&HK8mAnf=FFnm)~GDfn$`z9RKK73p9r@}LC8RIVpU*+}fk^(!NkCqvAdj?60Bahgrvn9+p0Tunr7yi9~<9Gu3lm8Eu!FWRIPh zWM^ghq+fW*X-I5N<%9&RE>al|#p^s0GzU@LUUnH(#xq+$^WSK1nk#1|iQ5ykd=G4% z?&_2$&Q=om%o29Z2pCQtwrmUl_^?tFj4nTZXJ+>g&Ua##iS*=jXT<7bb7|w^z~-I# zj52eYtt+d9ttCmhU$@|(gduyDxq9+B8k7aa@>Gf2{PI1iVWBo*Cb9C2oz%4K5YC=@ zHA1%eLq?`4kK}=Il{=N{mbMF_pbS8@E$Aar< zOzWLHx|Py{Bu|*1n?3y{oeJ+(c#F<49?%UV%^Hbvr_w7zD;ef1VLI)ur(NK$TbIPa zEP_+(9~-|(2=#aWTp>rnIMGaP>M=oGqq=Iw<_FkS6EY>)UE|c?@YegbRW65dYQyV&} zAIqm#eXZ8{ODl2ik}@U2=l(k-TZW8^H4W9C|7I+E;NFs@-P%i!j2zXD1B8yR%kaKa+ssPsjKxEe+TS|I^)pG_RQ&|JT5U5F#BF-1tmf&?x)f@b9c&+9=sI}YtL4K4jD%a=TM`BeG$clO(~4Y~gU zsUEfKWwWI1f)WF&Y!qJWo(X%2_B(agAEVc0SHH5$_}WH7$W7T$Bz+xxY1g;N8J zm7F0%ybI?z7hVO@q}G)VXYrppLk$Pugew}8UEQf^skyM9-xjfPrMR8J8TA>v0PntN zuP9(pE<48BThN9-SlUMuc^fQ_rW0T#C)rh!yTVlicr^%%gr^5;Sp;KEIcOV^^q?AI zF)CkVq$aA5U!1xS)7V~c(^%Ez$4plzL)Vj=e#m;yF29U1nZ+rxQh=PD(upg!O4U(( zNvj3fn#0+N65M)3L*KTr!ZK-ANRH@=M^#WMxWf%9>||UEh7U=515!LM)D1zrs)ESW zfmgvMeTB2^lo4v|8FDI}93aCmk-^o(a&^Aml!GwVlj}yLhWdEZxKVFPoG8Y(zEKFW zyVLVhV38;4n9PbYV@XfDl}o)nRu%s02ajod6udfYVydQahwcx>u?nTciO_Q=d|+dL z@q>drdlB=2dr>EkeOsU3e_U6fBw$AuCnKn~V5=E0J>8!VwofG(9=C}kAEjVJWyU@_ zC~TP&_MUtwLsN!p62O*|;(%X&ZKb$7yerdD9%}}<;9C5MId!)lZ@QkP=pQMBD);KO zP9lG>Pp9A=JgV-ai>36aO;b)Citv%B)J0L)MNtyZ>RK|E)OgG!QtLDE{4E1d`R{A#O_yB)&>0q z1YaVO!9V8$!0zg@4q>>1Z|WqS$?l1Ai}g|Bu8uR&P#;mj4N-g{Av+&Qw49{mCJuje z`kc(#6xdj^jGPp4rT5(QP-J^Dq>Qqk22dOuwP&DOZ^4+e9omVHIM7Yr=y|HTiPL++7df!ZHU-eWB^#AB&T%3J) z{7?aw*B$A(DJFkFDO8RX+(r|iQE#2*tz(Y+n2@w*^>*~LdcNhYrUXaeB^v68ubtK?Hg&jwq&0ai~w45FcOpOt?MhXV&fJTuB` zS-mmv)mTdjbjJwVVaDcNi<9F2;E~jE)eaGdcL+Kq#UX!s4ZG*lKXx;4<3L+!aI1#+ zGZm>;^Y2*S;EpY#l-?sgUR_iz2UwSYhwIA+LKSd8ZF~@VD$&0q1XrtH3oxjnU@t)$ zf?E4w2|qG+%QUj%ULHW2w3fY{#y@oMLLl@MOrXyF32dWOb8E#XUEF1-(?}KHVWy2W z`)fDw6CigQz4fPDF&S@0eB_i`HZ4WmFv+>wHubUOra!Q;VR@3^z>@FrnAD9Z@go

    7vWL0CV@~nT^LMAYrY4qT zAl^%Q6pn+Osb{j30EJZaP1mPNw6+8t?fMv_dxzcd7>=ul)TMc_lMzP$Ks!&97fPz* z;Z;OhD)SX5n6vkWK<-{sfMTJ%a?d*Vgg@i~b?gUj7^ID)cet13$#Ft<^5BePLc?7^ z+EjrWV9-yp}lvs^VEHQ(=j zi#{_-+d2x?<7Kw9s@xX_(f-X54K91w%d;+ToT=tdE;@SJ_n9EQAxtT#`oF=_xbt_J zgI~6DOm+GSu=4GqBx#!5j|7wsyYoUVuB6_U8gz6AD8Z%`Dgu9ivV2*{Q7)kP%<3~pdP;^-R!1_^MVdYud*X-72-WQVvFM3 z(|y`4j^w}p#9Q5`e=S&EmP`Qa@2cPU3WYu;#mxQ8Y?)cqTbv8uT05-9tfnDct-+Z+ zn+nJ=nF`35UEA!<8?H?v^&)moXJ;Py@OV5wYWXT=D{pR#%B1Hq`?vDNfys95Hc+Vz zshO=L0Ue!rMM{un*D1&eFKU~}k6G=#BMm@1{v_w1M-XTv$5l|q&B4D{Tfp!GvsWaP zbA!jH2IqUmxx&v+&Z<;sf4THu%At`KPHJ+#7tYr>eW%~^c1w9Nk2!${&?>CrLx)Fy z^n{Nac1QO?*75@ots!jjb<17}?n|EMiJq@AqDN%HYt+X*&ITAr`31k*1G&_RwO@73 zGvU8A*H;alY+oRrF6p#ioGsZr#ElK_o(>S|2Xa6;E7x-U{j#8T2ChxLb{Jn zivbLP4%$=AL+^2gl}k=T zP+z%9EA*04h80@q^s@u*&VBC=EcZVB<4ZWii*|ha>>e%p7`-8OPEO|G;jO{HDd|m2 zHj*FeNOdE|SlONaU6p2d^%HJhR~O8#F(f~F;L5;Q(nW~F-x2a6%1Yoz=HL51&V-C# zr75g%`jbWV7Em!N&1HVuh+5|8TBXMDlsc`w=$oIQLdaM}vxJ8w7hshb0mU&oik&(} z&3+_~NepbVVxIc9jPZZ}rPsN=Mf8R3`0P^1V*3_XHsvETWKE8|?{&nHT9EpAJd><7E6x7;+iTxC!^vjMDNia}^ zDMe=4|Fd}f>qMnvm60yj(=g!>lF5TX#wASdgh!i7^Mn~M$-A5e)fb}<*gI7(UU-#zbEEzYAyk+I*lWMU8d`&ri9X; z75;th2O0);LB;m%?_;jp7TXE@gMR8ZDoJ}!m5_1M?jofB3GwA+fxR|`OYLCk-l_WvVpgRgG~h*U_7ks3uWZ|F(k}g((<^NM>p<&59_&W3` z;t#sd9||`lv1EYJz`7Uc$Eyf=(ERVu{bWAefv;ZZNYY3F+w@#U3wAz^kBtS6Tefey z@fu41v(jw_l6eC=cLV79%?f1yjF#4K|7m>v%8-Njr0Uw}+t-nQk{*^Rl-B`Aj6pou z_Fn@2?=0W%<~GBl#bbf;-TyXb3pWBWhGescW0S4|MeN^`Fs$D`W_tHB& z8NBGZYMMd_2-+(pDXbA;_qW9U@$q^18*}X3Mh!kx; zf)I8vlSLzOLoS$#COiHtZkZJkHi8%q+&p-&lAZW#bKwmIw^vIm($n9hy1-Z!wLjLZ zODl_h?TdKl6cTpd>d7Rc-W~VF2e@y&XZTmYENW1P!x9NT94I*}yW=y=Y9Ld+OoulaS?x0_*F7}ZVlgW`+z7s9=W;;KF13$vq z%-(68?A|3N2@FE)hw}FeSE-))136UO45@~aePpb@mknNWS1qrHhaML%7}Qa=)VmxY zEqOm6Q_s7vWuxY-{On)>XhO-0vv0i)Zt?c}uu=7#di#ep)Do~^Rrw60@Q%ik@niUR zM_ZF}7t2Z-z+)mBwr{#*LwbS!@f^WW12#VjbVL#@i>Ulj+YjbkX=$SCoV9c%+Xn2T zYtdjBh=mXx0$ZK_Fw_E@P;CbOhW)NMBVzD5_9DVtFGtLeJRRtDaPh7okyRf1Nfzar zB?~@%QG$IAJBEa1Z5Q~DUXFwY!x>#8b;|}hp^Vg+EIhPPfX2aTzObRX)q_did&#H7 zzz(WIqdx!sN_*$@Wrf%uB#DcY8kC7XF!hfW`BW}yZLPGOiyC$`L#=g}_AlcbY0|>3 zDByRz$6q^PnN@jC&JCIovRc37<8!t9<=e*&9%1;bu(Kt3r*8EhhoTh7ZRGKhHZ` zO{x0fp0gD5-a*w2mUmg!4b|yw=Ovx;pSO)x5M!Wx@w(_7JLrp}LAG$jr=IxraxMe# zrb6@Y6!tcAl2omE_1zGKgA}8_Ssz34Z34>QGqO#IK7r~z)Ai$qa~U4-FI@jL?8TT# zEtK}iGO%vL@Rox0&DuFOGX&}-vQ6-|kZtj09k}1kKX$9xO@AIQuE5bLzL)*QZ>aE9 znq$KvakJs;f2$W^8{c*p*n#CS}91IUN1tYHXrF%sPuRkU+UM zr~VDi4_0fg46DUgxH?yUaKe;C^m%A?eVDIbzeXP3 zfRU4vE(OPB8X}!|2?G+ju$}uq8GGAXA=O%3Qp(X!FJ(x_iLMyR%{a12Uz)IJT?i0i z*9V_W+rrPZy5hl)WH_Jt%fUUt_%Og)H)d!(;S~`X1NW)Ho;|O$8O7gxn4_R4)&6@m1yz$#D%m5+@4i);P@S)>K=8TchL z)9Xq-)_0_i_`wZ(t|Kq7L70Z9K>C9d_Jj_Lzl45|>Jty)`@2K1)1@mPr(wS|0s_Ld zClrg2{l3^Ki+yWkHCc%2xm!C&IxLxe`+J@hpocPxvclqAgiY# zN;#h6`&S+(2wDQ69`~WFN!)J%6y;QP(Ow!QD2{E>q+g)EYzdJ7=Z4pMvXq;|iph1) zm&o-^8);$TW1{Y|(I1ORX4M=l$G1Ouygx21?Aqm!azq4sSeQ-#(Qvqa02prHdp)=F zY{gS1N5?v4VdLv?sUrI9fF#2HGUsoRkea$p@$(C#8hXR`???NG#%vGDU;dV>?dkx5 z4*egD6mVTL_{^==YdHwOLXvsoVI79;hL#ijf6W@;!mldIE(JXn5~R_?4P4!6V{TS_ zMZFei2`S5Qi!VQ5)(3k~E5A{o*wOmtB`=p__^xQLs!Qtr;|L1n~JFVZ-`T>JjK;94N*%a zaO4NK%dz=HynY9NE?#Hz#>Lki`VC>jI&IZ*>-1RyAm5QC`1#j|KA-E@0 zu0*GfZAXn(64>KAkv3HbPlZn*rZ*J$l+OQ%#`2r2o)YcgJS&lMpZ_2U? zM#r3{09%rMr6&@CPN_7R_!B_NwyP9`X3}*pt+|WZZW3pqF@VrxKnA2RfG&eT@vjVsG09Gx1nZ=QB3YXm$#71GMV-KuIKcQmw(B8TL-8Bau zZX58q^f>KNisuK`fRwqXafGTN;VfM3Wqr<39tW6nwZb)YJ!o%`qG8xH1-&M`Q%d|P zqq+n2M3cl&V^ZK$3qW-R!9=Q06#&xKrCmxvkP&a5ZIW-y(@4(EFsxJbG&HcRXBzG2utt;bj-?)8=+*4Fb|C2%yKxq^wQJSu|R0e*@m+A(J$Sq?a*uk^q_iMn5=q!axV$MxbdPv`)%wc>=OJ6&uG zSH?ACt|hv?2uD!ebk485ZbyI2p?IGG#?^|b1Me@|(jLQR7!gQ}#^>&600@6BL->a< z7`IGkA~Avy)B)%+!iSxhr4BUq3uI#(GAZ2V8&ku&+Ph@$7p^?3}?YjEmi_PJU#H$aa4m~*ypVKl{Tlvw3LGG<`llg=!}Kl3n~H|%4A z*(Oat;M7qviOd8sPwU6x+-*}eT$F*h)<(6e zu|%w@Tg`+YC~v4L_`uzt?!X%p41Tm%XBC>(GS_?Dn>VC|q2idPKNoY}(_7uEBVVO< zx71->(n;}8%%r;!OI^P!jii!&zM}bvffuUZ-P85hUbqWKQ7`C6eu(J*g~I1HmPpeI zm73vfr3JjwNCLfksX3E!Hh3^5!wx@(xq1`|dyUF%Xwabwx!PKrS<)3gMO$ukc@Bf! zk^I6cwq@Ts!=Zpf)iG!YiM%uD{yDQ0)woTeUTW~hOUt?wK~Xdez2Wc%QeTlmz-YWV z1l}4!^2U_fe{TJtGCdF9xy#3s?7(e^RS2Pby9@X3Q}-Dr>aV5$GV=U5J(uwwpOfQQ zP1r-M%|dk=7<|nMU33Rg2)PNaH`XnjNo_rDqrbA78zwUF>1huLB70$s6W7QIC@P5z z7CZ`(38YB}bdIVo8{pKL-vo^7KAx=0lZ+>=5gc=EgSCSD7Tb#pkGJDjjBy`DGGbN9 zgQFWFGf>^>j^>)uej#%aw)T@1RlYFh0Y5-=7WnAX%hzm+wVE)cGt*2d*c_ROPis{lr05ba=6wRocEVhO57EZK~c!DXD36;33|%7 zc@FGF8edf;iG5Kn&(&a0!t#&lTFO5S6%deB6#0bodve#dgr{0dI@#=w++TN&0+Jk>}(~~nM2ef^ok!*}As0+oYD&2#dx!D5ZbC-dySgB@A@Ph^){S*yu*$j?P z)M^AJ`$a9mb(p4E1k;H)7V z?AIp<7km@de96}9_M?F)V*MS$GTm)FpCAUMY?=CNLu`~j*OQ*a`b11E*|GmFP2aV; zqL^c-zy!xFkM_6FwlG3g?Q$@fUav{g@s48B^DYZ&x!9;svX4b38W;+@u$TklFW$QD z?<{;pfAqOg6MOrXj8+{du4+^6XFD8V^tPHXR4m6O1bjOSuoy5HM;GL-9Q)<&<8PcT z&c`52tCx@^qj1cygC1nt;&=6N4Uaydhh}ym<84cx9LBM56?5s*{zA z=a3Y+;KySx7jxu7Cn^z$%Y+APPu9s*O-R)2q|#%C<<+}=KeGcqRmD-&Zv1Eom)0Zh zv=TXGF7@DycHA^DIf^Q5pz4#23B)@>LSfO5Li$R%GsuQ?AAVtEKWs?hsthIwsQ7JE zbnw!N@Jkz9Bb2cSC|7j0ZdmDSd3|{*;ZQV*8vI1UwmihbFa5mwj?t+_qAEbnly;oTCusDp}9-7tmLEwvWqQzbxpP7>hu*8 zt`9ywh0ZYaEdP@Og17!)SSg&_G&rT*Xw%}qGBZy)(AOu0AP@kIY~(dOns=Ybg|*pn z(S%^Hc433s*H5|34`#F>IXt`CaM^xdgxI3|LF}!XfU*Fmq35b#LJ=6&ZlTpZy{tz{c?^(ia5`RlQT9#edO*S4S`yOtdHLJ>dX)}g?*9t3_y#hWMyh@@+O zGH+Mx)wzLpXpI0Ip_q^)?i2jurT#1O<**E2@XOlD)8LO1xK^WPI%{l$D43>e%WWoW z%aya)<(YIt$yHacFF9C~jVo;lscBH0mQCC%?mmO#QoW7+9B)b0A6-L>jEM2c4(TX=pUlNzurb&-@Rp+b}USo z@9Z4^G4DvU`jSa0(n-jH0yP2rOmfAA$%=u&qB=GUHlV_fW_KY%vrXi{_e z3ZMirF|qzZ!S*5cw)eJ6MP>I18U{DpR*<%pKy)~1%_bE8jS`*k*c`FC3JHN-b; z(po%*bTWxKh=E`rT~438pJy1O^2=8B&s2ucEwh*(W*Z}{A6ZN%)J;rQM91c2_0?7~ zmZouL8Fi?sL+zEuL=T7Nm_6FW2L!h9OcvB#<5{-is4cma1LXUXFu6X#?aFHKfc_QRDpU37d4?fs|W3m9TJ z*+Ohf$sLa5t4MX)gd!m2z-4``DpEG00s~t-5cd;+o71W2}lkq@&h!$4k zRtem2vgcLTrwwt5grJ7gG>> zKN{4(#C0H0w!xwR#bLn$KQ`8!`ut7Pg44 z^-`+eE}vt7=UhdUSgTZ{MZ%A#saAgM3qIquTzDK?YU$h7d~_rSFG7#J>!KH(XF^88 zw1(3@alepw@-K>?tBwRN}1(#XS!Ljy4Zii$w|{!9-}WyL=qw! zj%ABjq#&c9OnmO*CaNX_`H{0hB$oRTeTrN2zkYQ*Gwsz7IJ}Cosy*MF#Al#(^5)1+ zTcQ;ap~xFgtr{mc=(6ygo7fsit_BxizXhdt0D)EwJIXAr0)=0=ZsYo;fpU`O@%ml2 zZgSDu#dfzIc9gq9yQiwONUT4*+!5`n%%Vzf_Gk0s!so0R;n$g0L)l5iav2;LL+VYi ze9p9F76pQUtP5Wis!({qzM_t=!?}!*(#AeoDLH*2ZA%g~i?&kTI;`u^paK5A>~#(pxMvk$+0|s*MOPMD_p3#p*S&CXX9MN0dC7 z)6t=zzwuSQ!#{e;#QUuC8k66Dg~4F`hQ31+&fA5*$lbz*bfOxawW}EJI?-CGuYitQ z{Qzq_$C6A`&71S6uMN$BHwB)Tdrn}jQ>*#}`wRO+)7``l_I*dFy^^5`uHj3WK8V!B ztBnaJhgWR+iY$gq%jpcvS^=6cT|&jcGeB?u_u&-0+H9Y6cZd43Vv?Kao;aFP_+~@Z z2jsgoo74FbQN~$-`;fiQ0tdhpM@%_Pefj~*&BP=}e{lD|eF4g@vd%Kso}a!WfAyTP znyrM&`IB~`NRJ4dD$dJa9-r>?)Ehw>*i?6<^a{jXKjPIpC3&{eL=SJQ+cj*oc!L_| z_s?`^Ut#z`1{>cJ%0ld;sNB?H{23!aJi@Vp6ZvuCWCd9AM{jZ@b>;w&p-`fOjFu1F zT_Zcmr7{{lhP6BZ08Kg{pZhe9V!hr2TWhw?DlXn$enUA(s|X(HvtQMrcS?~1U5n^! z->J4vu~2daj;}p88W`f&H56_8Yyx7?#r*cO411}eqty=o4tu85Te!?ef}ga7erTJm#VFWpHYsl@QGR19>pqBVO^fKxVqqbZ?<1k?6gkzY;wVW(jc)2ZJAW6Bz4< zVUgE|>a%G0vV|FW(ycwRc;-Yfa($Znp(ssQxR#>hV+NOxqN={Ac^5;@<;5pu`A46Z zWU=sliiS*tQEgz3mbS;xt?^Kq9WurSR;DmISKUbMNTMMhS(^TQ?bN`D;9ENNX> ze+}2yuZ8PwO=I*{3#A7c7Uzp+p(G0@Z?Z|$!$r^;IMygYf^EN&J}9|tHk@_voH>^} z4{8k0n)Sh!Q{=5-&)GHhC6*bcPX-fteR!|w;thP_+-j)ulIjq>4@kFsuF!HuqF?ax zhFH-^u5n*WXkU6}NW$@l={&IJC}_!MR;kb{3(N?(W8}%D7#`pec-rl}c5CK{H#^0W zjy735(>+K`x`azJvrEvBvPM-f2uE^(Uy2z}i&`RSA&k<_LDpBJLxz`ZLBrCUEMUV~s2w(1b-fI)9C%&Og-0EkYKY!?!keuAxK_lgH9y?;8u{D-}-dAe< z*jwLWnD1ffiA6D!aTyoUFOsMDD4-dc*s|+K*esMNyEt#0GnJ+C0BdnrZ2y^Y^yr1p z;zQ+4UAN|D(4wL&{e2j>^SKwKpcw0qOjH}B4eXvIDveGBtk}KB4XeuO!e@+mlD+`$ zV7fg@QLoEZtiC@xNCL!MDS)1faA%R^eyG`#Pru;3N?_2cbHlvc=Uds35mKP?P2Jeo zAd3NPg0fGM3ES3gA2j*BxRBuzLPJqYt9cge_;w*DEAPZI7f> zY416Tdzj)L)x5u>j-eEh%@OrQ!(xjlD@*ddyQ;W`O59&Uo^FlhxX;=`%5;YA@|N?Q zQ%Zl8=%l2kT1vLw)y8v+Z`2x{agcJya|;WfoSgLUdWefxj)rU5kn%AX=KWXi!f9wS zb#Cpwb+z8Tve_n^bK4rGN)j%r?m#$1ss|diKBbv<>3Foh{MAATlS{afPQb7 zCyAEq0-6N{Lx3FE$n?jtP@7oxg4(Go%C0Ws;=*NOV!}!APh{se$P?ETKYoY;fIG9M zon2k2lc#^Y4-{hiW|IpbJDzhfD+X|5UOepGb=q_h{u4f>b`nx)4GHU4;A~j51Iln{ z&?UzFk^2{nORZ6$3nT*Ex8MPc%^ZdoKx_PAqy>1@M(s-L8juyQ!H=H9kcYV4l6EmG z`+ta$ZvdQD_<9qwvlYF(1g#1mng1dL%JtPSEX)*@Mv`WixZMyb=tbe*HqW^E@H1YZ zI>9*YmOKF+VMAY^{^bQj9WSs6gkoLb&znpNnYRUYLQ&G6E4XgC6(%SLy-fnJ@KH{~ z=j+^mTGqtHj}#;ehZKHHegs|;%FFFlkdYM(*RacBd7>b`akP>&ADW+^AVBd4HrdbX zU~u~NUj=no_U_7e#qCi~?KbXW#nMr>L(-|r7S!L%ycIvPMQL`@Ijz~+)?5q7#QGzn zHqc^DpU{Hfg9Z@mm@%b(oEHem%g9h4wMn1d5cJkLaMDeM_lpd@mQ?#(|41z(BQv=B zrxSu*MD|^H*aEq#(+|9a@h6mq!0?P$R8@)RF9(sr1+8usDga%9*RZ%N0BCl99t-sH zOiWCgEQFa>Y${n+TB81x@BZWZ{`-duXbT!Xo+ZDh-OW3oZN%uY6T*QI{T!qYWKM#A zr}}DG)Kx0&?YI`1B@7MgKiu+)pO6O(I(Y+#LF4{~vzgIl^q{j3OdH8 zJ(sP2j6+~7xC4@w=#VoI^e>EgE&fn0x%gZn6MDqpWP944$Y$m z#Urt=x;4otRyy;n0=_3IilGUcF8B7)O%KBVm*6vQc(+mKhydbz++}=9pJVW*xw(Wm z+I^@E9r{&{O%Lg{S7XKoEm>bIdFQHhYh0;>_y&C%z-sK=Z_Pl(Kp@A;pq@2_cNOmr zYV*iNtFhf1VNx!fKuuFpu;28Eh0hatYKZ7UPTO3Mw$A^5o;R+-eY=Jf-TfBq$exV$ zFQ3U}qSk)~_|{KO+B@b8j2ZH*nT$xSuDrwQmNbvgcf1*U>I?LSDQ2EV>B7d~X7nsK z(UQKtz;CaD1)ASAwG)z^V9Kz=)eR9^RG45QbDLKNZUdvEGBM_{#F;M#aFh6b#0%)T z54CJ)ANy{cGxx`J^V{^{9uI6_z`1(?*>$D((}|^)5RjdL4vq7UAKzIG7%=VJSPhum z9sJGaHYis&58pafiWGL$)6g=^pPpR}>~j382zS2hzlzxT02Kuqaz=@~p`uF zho_FSAjZGGd*~KKrmIM_JD5*^6Q|Q{$gx944(ZQ2YTe(8m^y8{JnH?cp@{SjmqN!a z>n)fqbceIDW5&PT0UmP#9UIaLJ#p!>UfQI)b`KqY$6R%vpn;c0nh=UVn`Gjr@5YFwOJg7oq`BM8nwSkyVNzXMSpcpP9Ji2hm_=+RPTiLm($b(KsvAT;FOv!uyY=lpE6ODB(EqSSfJ`hz_O@i+YL<}4e9?#VLh=nvb*i=Di*1?FSdu8nOYD_56KDrqa7ml!Z8_%cN5*1cUr+|l+x ziy4yGiS1*kHmJs7voKWVEp{V_vW23Q`-5=Mw@KIM?83?^6UeOj^u6WI>jo^>yE=Ya zDbjA_SIm9RsM&b`Hy)||l{Q5$Y`r41a^&guXbqp{dlkZX&TccTmPhH>1PW<3Y~)Lr zx6O__8}i>5^?p8HG8td8ZYzvyGjk9Wx2qzs(WwCal9f99Y2`{=9jA{j{%RJP4hnD&N;bZEjc1u%vV~ zIcai-cU>n9T8(+5L6W@BZ#0EH5yTkZ*K+dS4zJ{dD#plBk{X#YE9YyXC9|ZCbdAjER{9MD&ZaEZ?ghDO-fSPgFH$C|n#C(T zA9ObeicHRxtw}F_xKq076+c@#f!DBUvl(Bg`hRSF1ymdVx-Blnp;)ov?(P+!I0cHk zLvalfq|icfcL`2$cXyXUad!(ET*6E5J^y>pdGF0y%wz>JtjYZ5)4liCIbZU^<iXO4+|B?5FI_W%&p#MVi^_GX)6CVUiUwgvU|K@JMnQ z^w2-Hq4(`U=?yM8bLN}nM3KB!a2@CSv55C|K4lEi?OeeHf3|lw+kfq$&b?BZYPOMn zQe5QiFd8>(JLh71vz8<<>Dx0gZ9sbNi7r-a)s$|<{C>N`@psLF{}WWl-|+C6+>CLY zw)%kg3cQ$Rte}kQnW^aa$IDslOni58ukvAPFBYZF%LEgH{MvJKwb91la_(%30-*e_ z2i1I_H@Rx?@%)!RlB?wk!!RkF7XmZs0o_wPJh0gsUVWX^8K&^EQ!by!@Lq~tKc;<{ zG(f9*=}#qISTwcIU~`E&UYT}NCQT$&5paz>YL|n@cVKd76cl~BD&p!d%eC6-&R_%u zDuON`gpH8mrSMGyd3^DDs`57p1tyJ+_nE<%l(V2W&_<@a)_2;Y)WmdFNC^(_-YtBD zBahOaD5NrEDBF!G1?+gZUVeV9I@)OrYh|gIEOynFhRq+#JZllFp3VM9@Y%A*M4_cW z0u;TIbsuT)3;(d24(H?_Ph1R{)<5cO_%44kXmT~wMBrH%IB<8$nVvpLq_Q^7wph(Wb)FHLUxn~TP6!t92vh1@V=C(J@-J<8!Y=C2}m#J zgebk6kel1utmk*V5Q4%Xc9rl}tky8!(SH!#Ngo}34Q^1WI~&+maTC#dpXZln`TcuR z;kCNKuLolandldk((4se?b3!!uUy|oG9OwghmQ=k{U2PsMJle`FfCn`|3d#3hrpRQ zk`QVSM{O0bDx*`FeP^U8_$x0Xxcko;6m#~&rFzL(siz7wrPHQM-9T4;WFGTyaIxN; zr}kw9xd60ZP=w)fZjLwNRk=}%2tA; zPz~0(Dn>k^y127ofSHCND!SJ)SKYc<0=DvQyJ|DZ^+Yn=oM$5S z5L%De${K7_d(3^BP*?by?-tK*EveqWtH#lfiCC0@SQ=(CNaa`%jS-V?bt7-R9 zypv*#{1*9GnJRqepfg3>1H*sUBklmMvL=;$)Fty^p^ykUVnd`kywgg_@-Rw0!P5mZ z^VOK;(@tkqLF!~Xl&a3Bo}zH!qvs=W-y|_qPx!~#{&P4xh6?;-OvvHNR8P&V(N-_L zGwCc3cKOa=BcQ6`*N?sz@(qJPD~YF!fg1`>%?hcN{Z&ufL`jv-2bS_tU3k=s&GpMo zt6I6T8v!GC!1*Qytxh7E9}dXt89mCX!_qTR$W;IRYvzOK$!dN@%0~`KhdK}jQ}|>k znKSDI?SHq5n@368(Y=4+l#kF=9h6cnIPdxW4B@1wI+BMh^`G+u`tIxdrPYQalm}0* zh=p?&Rt{#N6m}Jg&FA5Z7DZU}LwN-e#$MY>Bw!ar6YO6V8LzdaC`(t_?KPH(>%?OF zQU*OM%dde(YC_F);x!ZZPIU2-)-j_z=O(iHYQ(*YpQTVoP#?CzQ&clGSlVQ>8|Q%S zMO7%h$ZMU^1UU?0uxiAE_=CE{R~mI9X=jecKE-$UC@WSNRwEq|;cn{PX~|Kw5G2Eb z)~V7+N#_t3SEGw}z>L_Z=XpZ}%3m@uMh>u|$Hfeoa6!Q%$1(jKHpLB{KfF=d9)Vq| zWT^CG*zXo4->`e(!|8V17oE{~Q6Sq)&$cuCed%qH)^qRE`61@bs_bRB7I%rDtwK4^ zH_y~A%J%PGwZt`NJmc}V&*KB}&&8S&F%+!4rZHrGQ;wE%n+6EyW6 z{!CF@whTZh(s4zjGKb#3PJk~kvGc5IRQJOr!X7^Ba)A*sHwmF358TZo6n;LHOS1c1!V&`%BtW zR=ij5AhDQLutO0(?>@U@A zDKpBko(Dn``6SS+*6%5Z$M@?+W;)J%5k#w5J%DlTHLr41Gyk92){r&64o<=Rpv6wK zJ$nKY?n@t2h~$9!XRd-EX&k`YJu~ zZj`~Yj{wcBnJagcMp=ucaCX>)S|x$xgOQjbmMvH4MwVt6af8#AF#F} ztz{yN7G~Bq@7H`&`0mLN5D_Oia4<1oy>P(XF($q;>G4 zw&^Iv6E7b72d6vvDL`z!rj#w68#+6C}x!iP;1U;a6*S?6~4IO5(-VqzML z$!H4G#(3md2$CD=rLcY2m+N#^?PuW|Sz_ zogTLtB7{7W(4D4~Xn96It|u`eXrwaXwhG%9-dmh?3JJA23#W^vao>&4TQ7sDzkb)Q zUN8+^OAbK9pe|oGes_Y@ae9wiZkVc6kyLpB%%k~U=NY?(MgBBZ$sD=VK05H>rKB zEiOeG+zuh#G0oJ)@M;;Xi4hsyvERS6Rj$*tonQSJ^mHX^_Mg3z( zqhga>;sSTy#CLjXOwSm=u%M7eIvCV)@J0!EaWPaoOzKooGH@@M3^6W$u+x3Pf#X(f zJ?A9(89hrJ@G7yS`eNN*h=fW3xZ>Y-uj>e1CiY$IXmHR~(Y_ynThXFPxVqwlE|a-0 z-lV@6C>5YmB4s-nb1~GWrVLTccR*+wh1KQ1vJizM$1g?1*fVU;(%(a>Y+wxGg|aw==pYw0^q!d){|RGb)2C(K8Xz44sg z?PvlLlD_=m^{KqaTDZ378(GR5fttY^2(+-l=}4MuT1y9cp!-rI>k1Q=sy4U*jtO$y<52sVj6@2xPH*^7;IINpkQw z!OO@mYdl!T&(8<{k_}-(=@1B8z~t@}8A1>9U^yutCt116a7a z3^94jR?d9!JXpgQ%Zol^=oLsFc0Y4})4tykqzJj%20!0AZYc*IcE~4Og;||Q#?loD z%dI^yiXH)S+*FMkPCFG38pg}C)gt$GZ*qe0XkKnl3e;0PQSnnIhjS1Io#6`ulc^X- z;FXspa&JLhyH5Ck10jKBhZ~T~mi+o)&JN0*rxkFv*JAn8A1DDF`BFpF^YvE(V}94u zi7)uhdIBbunxRtx1?0-F=-H;^{*1H}ulOiLWpt->@WL3@!oqJPk{Wv_$o$gE(7K-cH=7p?lNknv<(9rlFHqKIU!ZJM$IW9$x?n|4KLN%0Ro&+t7k2?i zZP)&rbb+!QrPd0h*KCT-^0NzV7b7Rqr+f_Y%-hjnfByVguJ?w)xy|VZ{pOyz0^KFy z_SzD>>m(^a2dIN6moxXnuh;ImiZbc$uX=r_)#a4oYXJ^p0W)54J2OTR=M(8vomI+h z!p$?yCo`rCC$_=irGhK@f5aj8RWCpZBryz$p_xkTyum3gxjRQL+sTH@K1L8>oLvzs z8CMdwLzDZOe2$uwVB0*8tkvj+3s%f>c zTzzbsw=c2e#l^+d0Tg^Vf=Y@rkJC2@^8E$XuKiR2F&LIwl8ZZ0R-9`~5r@qW!qviV(URQSU{w$=GC|XSo=0GiOcugKq)R>4sgz6JdZZ*HCEYdYOl6cER z7$SZFzTcLui9Isu-+zj%InQXqYux##O53_TEK8)6HL@x1YrWjT9@3dHf=y;T6gex% z`CcRB++!QE#7JlCeqy1QAKySYid%*A;T2t#s~^LRbyEf%BjcvX^GbJOTG(g?nd?9# zeVjT^Qo8GYf042DQq)~qgO#8MzQ$X@!clr_xSLFcg(Y(#81^ES&}(ZN#nQ2DOgV}{ zxKbqQlh}r0zq>bM#-XzQ1f9OsX?OuVCFzU1h;ppUgJlTo^J|VVPaTf3nM0AfD2hR{ z&lQSqMe$kiVp=lK@6uB+W_5!f%9qqSP33^f>%l^8S0ey{47nnw)S#h))DuTt{T06t z^1+sA`FLtP2~AC`8NUK(y|MFrCQSH@nPCH{Kd$foXk6cLs*JAQL)7aF#VGC1a+p7< zH*vF5!+x0=6uVLE31&rpEcRg}?WOEYhm|JVzqYpZv(#Ai3(l`NR_kWUF0|vXIH;*m znC(At_dVUu5;mgSPvdaaT%Ekx@KE18#HmLVG zrQkesl{5YJURqAX-8pGRxJY^F-cjYX$u0|?W(AKV(Dgo_2+pb`h+57BC&4j`TtXDR)+OK~H&w>mxU+!$>!RO3?RTQjWjrIhqvvWxn zLJ0WcqBgJ421d89Z1Wfmo8R8w6)R9SF;YnObw_q`U(_P&WJWuG+j3v}9Lvw9oygrU zeU!uhbT{1iC1W9R>%G^6!jx>k-xx|knYQ9PmJJ(v;^K@dqu1G^1F&*=gQ8bZ2Z4a= zB7$oL5qhXDj=Dg8L&)crl4ZAA2Uy zqVB}r=Y4%&_QZY~hcQ2v&#)-8Rkg8g4-Yo3F&;OgzV<|5%rxmr-FbyU)1W~0fiVLBPSH})a4iM1$jIB<)P<7vP|>9~ zl}VXmkn#nmQz>I;ZkzZzwsFBCZ_)_LU9uClA5%?KKk_tQ7Kxs0?^EyG_Mx)r5P2+V zWrrtgjnT|OQK(v!Bg;@AD-#d>3#U=|CHn2=(seKLOuBi*Vphj{|FdL8D{ZXxB+v$yg&yY*J-=jOnq{s2Z@kdmHn$tyx)%%kgLPhrP ze;kzvd+yz(*|58u-)0<|b>@!^Al?B;WiX()XhK)XV1aQ(itM-6Iw865>RU?Q1O6 z*ii5v@%34A;XbYx)Thl%>t`^tz>&^mh|QHy$i&BrtF~774oaWG$qX@RnDLP~x9}9d zhqY1wSn(>zZvaOs>8J`B%57X`?Va$6uS7#v0k5@mS(L>tuDIgc{4rYN=#tp~5C?Jhr^Pt=&j%L~>uc&2?2`WyDwKWZ&_$L$w8b=npwoT_F=YEB!RX zLS!n?7vpIy>2dZpjQ9#(MUv$Udhd~Q#hYM$zU`-rD~|V>!6xVH*jtmGej~xITQcUi zUjJgU-@*+_^!R5@baXX{%{E~g+AuBvFYqHbK55^~Hs|L}TJ0y^YZ8>X1<-+K1Gu zbowCY9gY8eicP1C%Q|hey2P2ZeTE#J=%hwtQ;su_Nlma?2asEl((+YOsSMt};Z_vm z`!^|Lq_a}p!#O9m)x^LQ8H=|pTRn$L8)L5-C%MjZL4uM--kXm$BMN?!j&`Uwrmo+L zo%mVSLByQA`phn{=tyzGbyi|$Ad*ZpZL9yI=WY8fI&*65I04Kf0@i&%4ABDH>;ChG zmI@eiuTH#d?#SHA4nY_EFh#+{bbk7=*K$&SCk05|6tZ^sCQ5}0(pX!H3FGj zK%5OJ<$X(CjAf+btf(7@hv=!2j~&M+ckZtfMS!yJQiIfp>rL(`bNv$_r$ zSG2yLD#bk0L@K>_9s2v_a`N`9+HKT1L!z2IpNwLi^ka8)aJADBXWb#+cR|Y0pvua4 z1IcLf6FP$W42k^E#nLegNC2nmYui(Obv!oL%W9tmo8@{^xT= zdGr02^%OC%9uc_O?K ze%moOc@(A*T07VgQhwrE=V?J>AFzYix;_j2_X#ixqeeSseL1wC#J-_Y^ zM^kaLJsk;}#${Cba<@|YBtmIz@yP0boNzxndu6Uo;3?RBwQuRT!1R+){#(O8Z(@^x zk26i-tr3hmefBiz+OKS5^p$*+A%f0ns#NjrUb+lk zVgX7rS~_Ybl-3Z3982MO9}cpis1MT66TnNBml;-5llvLLP<6aN#FU(F8`^p+cV zfS3;oJJZ%&{}V6!1hQ^Y?TxoY?~4F(?sKZg9snYp+O5-VXOCiGjE09kIu`~&ncVl6 zh7Asq$|v4(;FZK-n04S_YIqd;;$(1`)~*tYOXgd8z`+u z^9+|4oq1={oA4K^MFPL7;8<)RqAx^C)wsLU-Oe6DRC7{R1yF6g zF*~{gzCX6TdAh!)9|f>K#2AE2h&_5A{(M7R77dFV0@gz`;b47Ya{q|0Z~^RosAPrh zPS+G;D54FHEp*)5x`jf{`T1OT{S~PdkLwYxXOmm^wK`%iJy381K?I45ga102#0K~b z7GQgo8|`B(aq78MJdE{t)oKel&MnT{*l^f2Y};wq9+hy1uh?K3_(Z#jW-3ol>o3@U zO6h#o-_wMOH9LWIk%i~z)=!N}{u3nX62oD~> zOhT=S3D#qurrkWsXikxh4ZWk?Oz z!%`fN!`b%P#~*%1=bFD?emgFJ72dkCm~q`qDS3sOOc6l-Bwu@f0RRSdzegEB!|7&(BLj7Ou^0 zg;JE23HcSHZz-$$Dui1TSM!R7hS}zaBdHvd6_eQke}TGdI052*xBN=BGyo!;JXCi| z@9**&wwD)=jyrBbH1qket7ffOkN%Ozs2(uWAG6XV2WXX37>(_Wt5FjO_@GZ=a>g^N#J*sk_xoisOoxEF-NCr?T3f z_H7h<-&3m1&Z0-FQ+>TAKQYAY53Od4KO&7Yz&_Ki$d71U$=)=(;7U(<*uBfZmZsB7 zffI|6!L;P1q=eyTCY40fJV^oub?b`){Ndm};bZgqPhdtdP)f9dlOR|HW4`d(bGu{b zTILr8=EcTk0O;ob4wH(tGh@hxmSS3yjr^`xPi?su;N-GyybT7i{Balc1-iP07osoI#r~ zQ2a=s6dMMrSv);30`h_@GoolP78iMPXzg8KsH^|lVwa=hds$_;&HHDU``O=gr|98L zYitR%IP-uU#?sc}0Lm6m2c>-SqTK_`zQeiHmFvT3M7|<4MKZ9P7+cC!SWpz3l@-j& z9$9w%_#!i37DcpYb=uUDkKKAELkcUBB}Az0>Dw=bqPd@-?{{C(>P z#?<+<+7c9VuNC?lBm5w+Qw5>~b()zEx@cG7uDH<=$kg>Z_M*BusD&4WOblq6y#+;q zKl~IFa@y#UPt|Pyl`53j!T$Ak$@&+3mMc@w&O)Rj`|Z4_Xvnn#>Wu}W&6h%=ULr4yQ@7@UMZ>Tp=tbP^I7V{thy^xmvU&?5B zv5F~*GAjDPeAD`e=IVYZaMAS>vQsINRs+bRf4|Lr%IACQKT~WJX9d)>&7E#uZ}S2N z$SU0=TRA$7V4~4&ZrwGb7tAxk4g73UijM)q;f*GM?yG7l^@1g&veFR3!^aXbKoK1;7_tfN?6un?~xSp@*Z-#u#^%69B@Z)e2F?UwEHv~ z)_MNhpR`x_hSPY9k$2G(b3ro5>v-wHve*U5Ym)kAYulklt=V1c>)zVHysxD38UqE_$iMJOlP`^oLBD^lp}@d@CN409HI~e@+@SGL z)_WR&gBg^QmqTt^C_aPRetJ;qbj73hnfM$-$lynwSgdFx!ue``G`e_ZwzK78>iYf* zPY_^0iJ{J|xil)B^(~1%4q0jA*@SE^969>(Le;WNJdKsE&X8U%9)(Y1-uwQzJ0H;U zTGmGR{;tR^=cnhaNr-u54U$#D*ig(ycxMrEc)b|Eg0_%^6=JmeLS8uhQLN` z@Qc)x`sUU`id?j|I%vV3>}LT+;zz`)C)9k=suO6k_2VIf4ReO9JbXCkoPJkv_|_$x z{O;H8s#{j*ioIwfr_X$~cn%{LO9GwoT*J&+za|Yf3rz8-!pJ3#f|HXfPI<9n?T&JP zclV=9-18CE*;^54X(fm}yat}aU zykT^Pklt_IXp}N$4cKlB%-f1_uwq!RRXpzmC~!xnjn{$a6Fz6Lsl(&l-_zO7ta5gS|-*PSRwwWi6ohP&eMX7hx@fFgF? zTR@nOp8iJyM9Xq+os`H3$WFv|d%Hat`OKkXzkU{8GX>&^M)W%pnjsL3~0m_j*e`u~s7jW!DnL1Rn1IP3A-IuO^9>-}$du3zU{ z?7kz3*q^e>T+PlfKr06^$9!Ve(t2BXJYNlsaGp%23|!lVJV503z~AGZSNs8#xn+J` zVxgFRb%Ay5C~865PfI)pd8NN$(mgmI*q?C0lkM`6awxfLD;C7%t7A9N(}T>sW&6!6 z*$q9QK-T_ELN)F)1QmGa>M7%n9nLBJP%q(aqUNoYKTsOfwwZ3t!DDVf@s_ z#4!}_RA3w))uY^fyna|mAViK${=1XQyT=880))_y46D&EfW9_^_h@M5VaR402P3-s zXAHd0RV2Y+mTfmw*Va`!#zh!>IkY#{?HcR!@1mJl1^&mPiL-6L-E^YEdx3rS>6Ka+ zCtBWx@(Uo=Yil;qvPnq=5T9Yb{mzaK`;&`6KRpeNmMITReI|fUYl(e0Hxp*<7GDtw zre`?Q!y=*4I6pOfGdbB>e6>wKAk4%9zV&yKYn(QIz;|NEq+v%+KT(W)g+X1fr%sEu zfZ=IA$GhROA>>>k61)qQ5mOs~6%`>tfvze=<@UMxgT1_WNbPSF?LZrDJ4}0aSCR|o z*LbKD-($K8EigjG<8w@~81iw+*Q~a^A-rPnaoE+q^CrJg(2t30?z&P}Yw-2i0ur99 zg{I{VjRt9GQk{hUg6J8FdSl3QxBsPY80QOna9o%}XhVaXeOyE$M8l5GV67l_SWMD2 zGZO)eOz0Qg6i~(|`XYE?jxq}Y3J*svCj@Np9=xtO(;o>t$htZe3s(IU%*{>y_3Bhb ziM5D;kDPp+OxjLY{ixpdW8`-yHvX2WBKP;X0MTC=oFx7jU_ibT0tU_>2kB93BLS&X z2Ab}k`0=F~m&nMUK&8VgeclDi@I1TsPY^_By;OeIjHF&~9*?%fLyWm?ro}V~6-|p5 zEdn8+r+BAdX9iJ_k=ahbYl*aL&ZrnmH(&)ZJA~cLI(j}8z%H$-Uz{O}0Gh_KnZCj; zoqryethKDFR`o||zo$~n_*gfH$mdl^yh_$nmMp^SeT~(g`7coKKZjd7f&kuGIT)wo zlyz3TB`C9a!TDp9{r+P$6@fsRq!Nls9y+!uv48(gfBAtrb>8$3-x=ZvlI_WjEX33s zSI%55&W%SrUmb%c;_cvD)1?>S;qPaKJ@)8=_oaprYz9%kGktiR`%v$2{F1uY8wMI^ zGrZQZ&E2jLF`*k+C@);7>scz0-*tXs@#kWb;;UiAG9KSPlHIv}z!^{(Pa&e;8q_+> zn96cE@xC~0bJ31B4b>J*&!LqtS(89TGU%0HkghRx5XuY|wCHLo8S~k74xUKu!k8SG zZdVsY0nUJM!y^xVUIQh;hzu37aN%c>okjm#``@F>q>TUrPtnJwZ8j2;v>EE3vP3*? zzJ3X0LFxJVbYMn&ohH}8#l(BaPEWP15_T}aJ*Qw4CRgS2*KAPQu2Kfr#7j)n*~6j% z_>`AX%r#QM)MyzjdYAnMJ)%+=c@K?QgL!86cdNka*1wnqfBAtsYa1O)m8mQ`Fv`^^ z&VZFfVgx>41qh!{%EA8!Lb#FhvmYC|cq&2wGrKqzOe4=bar07dweYWlgqbIpHNRZ6 zfEA5fwgh1NE!iy-`hU|DfWdOjt4GDlR?b!9^~i@a&%WaLN$*UU%DWbra6$hyL-|n> z8tBTgLH-UmD&Um^ll5tFWV2=hyJenYbHY-cXYv}aO=nH39TlVfDVReqvnFk9piC~4 zJDG^J_Ros+YrJmcFYn&XA%^q{KSAN}2yqGfubzQoz`o{+vs)kEie3G<`TYlAcW*CZ z{BUjcwlE`P4K!ZUk8v+O zEf-NTQOf`84`3h^MoLOgPwy=^5$7b9|5xTQ$IIJ0C3^*NMEv}UkKf}z*;laTlT4uk zrwk(dyJ{5c&TKSXvB%4|TZ9LyLxyJo*0X^TaWY zjfvToCfa3CPtSuhv1VXmaBbas)P_YJUOZv_YY4?miF=*qib+W z?;auc=TQHqn+Ov#NA54Kvs}AL@Fsk4a%xRY4eK7&HjL;qLhy$zBJhbEEPWqI5jKRZ zb;?oMG8dHg! zuZxn}3Cc!-kJNPcBSwwH=ESj_eL?RvnN3vs`0k@Jaqe;QQb;4@RjSF};Tk zJaxH9P(Na#zFkn22TLQN@V9Qj&hFaG1ZcguI`5mAaqAXjf(fURh>3~um51s(kF4%` z{$WL&-wvzD*Q`Dp`SK5+uz3OajNp_g%*@gvq>l0P=byq~OgLJp1V)_!7%dbkjiZqv zA+S}R<>~EhI2HNNMgKQ;AMQ>YSiP)quPw!a<>3P_L9~w1f~CTQio?VSP}lS@*;oJ1 zC+1sNF+hPLY5=Csi4>p~vK^=rV(%d&e{p`*b>gw3sd@@2c350-oVc9x;UQ*$>Jd0;uXZHK)lSX3qJSpZ_a!2vb}KyNO)Dox?Sb z&MsW=r5Wp?`J(o5x%H*!E}L6f#MSwrjuLTxu7PQw3Zx<&@_3`%UNdhcPZnN4qxjGc z`?G%&q@G(dopp+Y+>t8*kOPK62YP4Q0($u2d(6Tgkyv0gqUK%A4#Va^-y%BGx^n$S zm)$Vkcs@T#F{1e4vBveX9cxX7%}qQ^N+6q-n~bRQdiw71(cAo& z{Oyy7X+; z^Dsyg@pLO{y}oY?9oeU2ca(4$y~X`iBk5?$W2Mk5ePe90E~-I*)$LO!1hQF!@=8Ac z0*hSeAb%%9z<%i)DtJA)r^9y`*zSUih>Bgp{XWk(gb5)Xh5Kw9r*w}{X{|N6txZfO zvLpTHPfeJMaCyGw20lgOz**0>jLU(1ZkvOZGey)54V52adx0;{0GKc*W!l|(U%!2k zRdg0C<=NbXUkA41!DJmjE5X7f-?Z>QWc&k@V{4W3~>hs+M5|GN0!GM zG6&JDDdYlF2hlb^k_uM{E*dHBHZvC19LNGpKN*CS*QmVnd|M1sG8uSOScoH(-7j4> zfx83byZXqkrF}NT|6%fddV3=DO6UeQn;bxHF>br2{O*RKA+x_}e?U?8R+QAeynBF? zxQZDf6(~J(+DkgSJe-DOc{!2}suI0o@bg#V3h|(Se|%bZS$eQuH?F}qzA9~67{|MF z`Mc1b?#B=0B8`b(4(lD}uP`vIX9OcZEbr5>NSZW+i2Z2A?~5Cs zL9569D5vCpEiweXfj`eDx|CHjDS6wr`f?NXEWXN?(SBKa>(B;)1^*+@G;hMcYq&>F z#JdP)Q96u_8l;RB4n!5Hia-3p{QH<(5HQRD-YL$4a(V~3-0n#JzIRwTQC?D_$~kwl za^I-pY2(S%x+ghtNBv9!U9g**Ne}h+iIoGoM)}5-S zUf-4Tm9mz8l*-43IVT~PDv&_99P`Zz&G~h&zO7@>MpQexXiL(psB3qk!j_8du*2`@ zgMct1Y0~tCCSJrM-k`p58@|Ko5Hwcpp2w+)gsvNPf{$?LwBVMI=K4ilBEL&^oe3CK z)JI@Di>$VsxN&tU+|rVXwcI0H5n309T_j|JC0aY0Z7sRnNZupaf|x|REay>|^U0hm z1<`tyJH;%tRjBX@vpHhHuG`!^AJnmx@U#{Y;bzycF_U(G|DbN(I4{)D{tJz>w{+s9 zF&zBPbjtxOEuD~iP%l;aU&VMYNOt98M zo0Hs&A7DGjoFdvAJ42CSu}xW(HIZg$X}GK*A?yixv=Pvzx2yd(TU<;a2rC253N5!_6g@Aw=BSTj_flS8LO z!PCmDsm~a9Oa{Hc4m7xKUqLs8S)kf3#7&x(dk5xtSN2fgZ3h=bdyuFTAe0Wj}S zX@@61F;NL7y)?PN!N(U-uGg5=@bp=iWnO=VSVcu;#Q`wu;{U>cv}^|>w&JcCu5CEy zMkmjQmb~ew?|*tY))jbafktrCoyu+C_&0^$P~;AYc)#{h{Z0D#59fKvvA4DIrT%Dt z-?FX$31^+o!qJpghK{Dl`yqAzV5wd-`WZG~O-{1scp70+(eNp6hS-IGOd6Y?s$|F+Z1o5nUPKoiN$JX8idCC zbx9jLG<^oG$pdNrj#o*a{6 zl$7Azk`5^Wend;%tM+)&hhQ_5FdTcU=sb4B$HoFWxta8#U8|Q4^QWgL zn`METuR50AD5-~}8ldHd$iB%*5G(>!IqnoSA-1rv(D(0+^z{5RVG^N%|v+MjDv+=Q^x1HGm9lnpT zxra*sJWLg6nnsd_e$TXNeCTDcTU>GRWHR==;sl{sk-TtbcsfLGJ66VH5_W{983z4gidAuZ}!U#aj{tws8nP0UXE%S>5d)G zS#t5vzUfNMDp-=WmI~3_K zZ+c9*o3>_vP7#q>A_3$4?frJfRn@a>E!1U8xNbladg#odvDCaP+pe zOgboR2D>tvsJ)qT4gOr5+EVy=Up?JCPr=MZ!@a1)rX)J=&vpm{{(6P4sFmR9J@e?= zm_L+Y0x@sE$#&0&ZK>ls=kwWC6uOEtnc-d2P4!;*NU@?k&q8L(WPojW=z1yaIlruf zWdtiOqve02!^azX5MRCAazpG2aJ0Nkzd3iH119TI@S@W*-ncRXf<7_9dHCS965wmM zdSJmJBj1CU8>3b8PWKk7$#2dSax{(w;OI^kKfZhSwi*zXpPx?xGZva`Mw+%|%)`T1 zN2}?5tLRvDxBrBNF3h#Wgf)W9-wZp+hH%z7kh2vU4WC@s$YDuUzLNQCPH_lYR2=>( zbeqnhd7DEb^^xIL;))4IA2{!AR5<)oz<4Xc_oGwd84Z^2Mz(#T=~P;6z0J<=>rP@% zKen5%e1U`BTw=~u8huLUAmrRu$`bo=evSsc8?3IgUdz}wN>P@Vf7J4t^9)?V(kAzxb6KInvPVcW3n+b*h+Nue|eKxqSsVr*`zO zt61WrR#r=v>@^!Ly}qA+Dxs=v7~jvdhX`vNO&34y{`nC_ep#b7w49;njFt2HETea9 zKgxduM!|?*@OUy~UoM_(+AlF4WNfS1P9TYgj_dh917`PwN+oy7XZf!^)H2e?tDjqC zC)B)y^|N62({gu&5R+W!C#ASgTx=}%7e(8x{e2idp7m>4mb1R0AqR-F?9SH#+$33J zG0Na^A|}l3(O*-*=)2b9t}r{9F5qB1Gk{eT`FJaQmeEjFl^yWW#%3X$?2_YugnN=} zPSWRjH(IEl&QH|R8h>`#k>u0SPr-uX;>X)9reaSl@8f~`6-pn$_ub=YShn3Nr7Y0* zV$^=D^d6%2G@adZ4pty7e?W5}LU?!cTQLrQtF**60>st4 zw*7G1l*~wV(A6MV1nJKr2qlc7y|9yWbuyTNYa{<8x8`xRUC~d8AnN35nCV~a$NLgkNZJx6KSC zw9EL=(fYXBJntj@Gx(y$iAs>LVqFuG)5Ad^|Ji-`=gQ)=u>6+1PtjQoY+O~$$oGtK zqL&aC^k{31%9AAF%-2Mkz{ycq)rp7*_YHn7Qe^N{Oeks5$ixU;EBqj*cpCce$;%Zm zZjYp4*SzhPGvC?xM$YYJ*IC=*r!xX7jixv0R&RcJAiu0&U_U#KaP3qu7|5mO2Fnnkh88mhb6o@bxnqS zef^5HP@Ic>(IqFl(d4o9)YS#}Q2p0~1pE`T57)LAw$tBHDO0*0u8$YIgIxgEv+X`A zC|Dg?tF3^zx=$|z?EkW0F7(bxBCt5IPZ-ux23V3=F z3sVU5I=y)L8sQ|(q2P1N@H=0QAU8L6QGKxUcZ>e%OcKm?O|#MEAGylPG7Y-8z%jWe znUcuNiYU~d@dYe$okJjbRaLmPi(TuTa*=XxBTocVvz@7t7C%g^{L)>{@K$!+5~>SK z26+ZghYWD-s5o+86fhV|%3oF}Jsls$jv2N52uu^%bKJ~i%qko%<-C}i4uL}*Rw#rl z*h(IZ+u_*=Rp@UKx63}|qEI#(^J0Mh4{vW76xX(`fg(YJLvYvN?$Qz5J-7x7?%pK0 zdvHn6;O_1Y0fM_W?yhff&fR;TbMLEq|K6->SiQP>_0%EXuzJ4$RG|YnqU+{(`-{Xn zcX`&mu(UV%lkW3Qw5W#nS0z>}tu-BeZ~h1BidhMKImJ#l=0YPGbZQ;uOL-)i6&819{kf z=PQsoM3D#o9jFO`3luwZXHh6$C+hPKYi%mij+hT zY6Axn)hzCZXlDOkk-EErY=Hpl9}~GUXz62}y&_f=KaG+*g>sga=J1zG7uPjtL z?&}*ooDK)Cc%B-!bGAa7TJ4FPd1H#kKdmcL3HzFHDtU8!sBEd>KZ=!o{Zr1X53ih_KL!M&@95D+^(1T<(M zp&L$hbK@72=uGs;ZhgyrPfeuS06?aZRf7?itE}SCY%_hc@1v*oy_te?;x}cs=bw5Z ze)|Qn$j!3}mO{Z~0))4&iM%dvDe)ozc$?y9iPT(G%czT--^YQGfz6v^|4)SI=;$zh zANUf|$7g^3CDwDro^d{EWM7eZPS*=Pp0s*ca~F#(c+?3KJ)b`?;XZ8WqLKbe%DN7u zsKX}29rbm0B=pjzoVWpmwQi8elYeX5zW1_m`j9t0@HK&%{53gwL!=)-oy32gi+T{YblBalU{-&Ucb1uCqE%u@+$97O3buEF$#vtXGfdq^#x{{pzv(1_tt6v{-^ z)xF7Au@(EPmk-m)LUJhePo~n+p+&%=h_cW~|K=0B1nTvD4i454afr;DtaL_z0-$an zNg9hKX$C3qLz^XcOjzWBqsQhs*oFCE#Am{;w)R0R6;E0*rwFtFf>-8Y2LD$$zv@dF8m) zM1rq?v5Eh$hfKVkKtHxTzz`KJ)Y-{lm$Uzmul?)t|Gm#IX@V)I+~&R5YA(7se-oi^ zJUGk_rrRtwpy>o5+nasoQuP0?C!J~M07HtA&fp;%K7@H&CumQ*{jKWY0iJ6($x1It z`&I@1e-*lE@U;lV2ELJ(0FGZsuaBG^0|7OaGz0g8)6bOgfV&V-#(z!QgK2?pFB}H| zyah($e{@l&-JdBonu~>tAc+nB)6?f~_Jeoy=75vuvXKE%B?%7dlLvcllf$0Xv4BzE z4%Q;M;eU~)e%m-KE<%lRW`6*}qY-3`GYMmi<#>7W$~X35&u8aYmp)zxe@f+SEId;2 zC9wb|-s)}+N-PCiPLm*HXtuhGuEV$4ZuT5KD`bT=$H~2|NzE-KTr~kk6+9i2li3wr ze;GLdyJVHR-uXld9_*?MU|$=f;}wJr%D#bl@vv}K@)d_A;x*)Gmr~;d&MyJTkOP3T z)-ID3GJ)Vdd~32F`zQAnb9rw>mRAR5C|y_J(Y&q+2RzZIm#6-OuL9XSih>p;4ZiY@ zW?RnQ_ry_m)2s;uNs5-Lg0Dq4IO}NwUSrYUu5F5uP#dIIoi zZ)#yomN_z&(_xV-^GRQc@JU{OZpz&3a)&fEQGR+ggvxwMuo%0T$fNA?W3~E{21K02 zM3dQ`f4?fAtYJnI(L4PISK4Ko;w;L98z8-ZHwVn`O-q{)Qx&vV@IkTmZa&6A)RYcA z4NhN<^xpx>t0ADg)|4I#k_Q5?BG>HIblGi&8JQ=>R|D<0Wg}_5cibzP3kG1=5K=Im z$i3diLfg|f=Aq@a)n0|U>yg)19>yrC;AP2*{M_wUamJMEu#Wf}v{O*o&&^r1NH5Mf zzknt4nIfV*$3;*KV}yw)H8Am(vZH*Z2k)SP%Zl*e=HQrg;jkJcICuVt|4c*472Ek> ztWhlYV$hCZoOl!S1Ny~)L>R|)hZbz;wN*_GVVnhoXw`g{f18vfwlZ%f=l>**pNXgN zj%)zcl4Cu;b}1dp@3YJGS2r^VlhhNx(%!IlkgM%yW!|JbNo z+?CQN=#&0Gc7iSH$4Tv4JW^3~bgKT;=ksl-t;X{{?bFk;j(~Gx zq26Po%VBR;kIb!em+8yRhX*fKyBx?iFZjMBGiT`(Kyz2CsUtQ92msdrZ)L2W27Cc8 z6hIJ>0WgTE`uDXMzE%{n?t8g~)Y0?`BfqLr`U+&hKk<)HB26;7oTvkN1J{03vqf#( z64Zy<<0crKy>53BOY@F7H%6{fboq6XV8R0I2+ot#TPl#^SIbsm^BRP*CgaQxE8wjO>H^`jz|q!?4NjM zqRCP#^0@T2=pFJpzeEmv2IJ@E%9DCK?ZHJ}EhZ5Oeho7LXw*9~8q}``#lwZ0;g_H? zfwGExp-U_!lBP$dNCH6{9gdO^VQNx zJc6ZdyP+HnuSizDJNlL9Ln&#Wx}?;jJzaL^5Jzn7Z7$6J4 z2n8147CtBxPK>+0L)o7$7BJpIGh2q17P|jkEryt*t~zd)*E}-Mo46jpE3!nI?aB4k z&$lM-)3H9)bHvr`)2>5 zEzNo57n%|=zeAGEQX}Wn@_iEbzU`}5udr}&d)K;xSl?AcjzXn>e+_-vjyRCiG^Hg# z7-q;*ERcPAu`SGL5DUB-wX6aTo#Ty z5JJE?BKL%#wG%xjpl3){zm?A-uhBBYCLVJ7U8&3KC7b%z0XpHWmEkf@NmN&X|N9tD zhNbU*3rc}wm&^ds#!!sOQBrYud1o&;_;%^LV?`IvKf5*N3-ar;a_-lFfYEO&x<6e4 z-9J#^Hp&AzK0a?+(H#N+9{1HJ&;0`;jF;k>yQ)AaVsQT>a|(aOa*N}6 zLv52(n^aUr6zXG#0MjIwHRK)EV59L#I(Axd^ViR^eR7HEr|_aK>P8g&V1>!j>gLPm z@j*|jd!FOpOmvi&s?B@^ZS3Po+V?Z_y^Hf?z%>r5d~0ybe|RPI7XTg)C+Fxtez3Jw ze8i0L5b&tZ>i;mACkO3z^AqU>z@V#XIBrFlZDh_USy;bqn_2#)rKh3l-(zIB6}okC z?1~CRLO^XoAi6m_`^TY0KgNH@x*S6p0Nj9%M(+ims0_fcdY${Rk3~&Q@P9b4;tDb; znzW0^SX3EF&X8^t7XKX;rG($P23DW)bh+i|=c{AcLYleG_XD+NnnXgWU{~-!IeU8r zclVfQtha>a;0{Ji3XYEtBEW26GLJs4`F0md6ZYNetY877j4u`uJ-rV4La)H7^zXw;G&5|buUR_Hs51ID` zH1O)M7(&m@oyanAn}VQ`2qb|Qf2I81TTMTv0x34aeER>|-P0kk}K>+R~Q*-*dOmg91ZZ9yx6eZ;8PrahYk zUaJm_7)JTE{E+%c9HWScZd+Nd|B!}us=9{+fTTSSesR0M3m5^+PgKm+aCvd=KSjS7 z8S%KUc}{J2gf^{@@(C<=bS!my17Qyd&Ckm#KwU{8#snWR7Z;upZU4*$>~fFGsLd@> zJEWz62=jd_pW}q&Tvd!5qX;DNQ4AH| z-FxGd=B~D{0aB|s8?K`aP8kB>K8Pk%c~JSeRFtvtIANBxU1Q%~-nJ}*Qb^Ou?Hwsa zg@@xS;ANh#<<^OsveawpLs=ouAog1<;dS~z&4knC&R15STsukYBAq;8ZTk?-PN*h# zn!VwVeG&uDhlL3vFu}~i57@Iw(Vjf%dv!jS4W$oF>{jc1#h_86wZQPkI!ol7KEE<$FtSn+jQBVtms0y{xie2U0;{q6s>I=IZDz0% zDk5*fReFUJ?D=9tQ4o+T;B-r{~pkdPU~Kq3L=Tw%b9)v*8~0bJs-f zv*Fc#S$c)O@ry({FgmyiP^_7Ho5!)DJ^pmSRVW1u???xMHWjawikr@y_){Yzg3FYT zdECfr*CSh3t)B+kl2csreQ`t6&&{C#_Zqb?#MaM}Xm0mGbvPR`V?6=fbEF?BcRhw4 zyX9@*SjScqRmc$a3=)3DbYBHIAIlpV>j@eik;qv+gmAgYi#tzkG^Sy5F)2|IxTp~_nw3$l*kbtLPh{S-d^-E6=@+c`k-%!RoSCAo z16yj(S@%u%=~C?B!}Su?ZpF1+Z8}BUp)Y5E!-y{WM39)*-I)yW@M-V#7LRyDrmR(m z>aw~)ip-C^kYEM0-OPI?gp8S4=CoT=$D@_o1&OMV`+l>*U}^-bK-9^E`*%)=3^3W*QijcH{33Aum%lPP>G zYup2-Ta`&XEyLKqL~>NWwSO#EnWyu2FsrHG9@7QV>%J9V=1yWGdpo2tdpnB?|dNOL%JIjJ}hx!y$b$ig%9k!Z;?daqmj2`1O|!CVXo1fCu}-Xd9DLTGV#5)olT~kk(3}jh$~Vg z=Dq0iu6?niS(QK;32$V2|Bu?%uW=XFnH4)jnf+nlq=K3VTjU&>h|>(G0OKZ8rt9u2 zs#)woWNgj*9-UU1emF}(_$W_*ZNWREF&PSW=3w&;BHV+Wk$Q^0n~N=e>Un5Ot}ara zv~7@(3iJE+U(dI{wmd2&K<&S@8!s6}jHi6j=M*RQbN0QT7JEL|kaUm*>Pjq_ zf)yR2tz{tPPTqt)_u$XQ9b9*pC}xnt^I|?D|B0=Hl(sPO^Y`7$f+?dTC~u{Vmq2D! zMi6=_BvXA~7?Pal%8>Z#(46moP|w{3P58y%i%(s%T(#QkOb#YC3xm!sebH~X-qFhm z%vAdj_&xkkRtumx8lPcP;=mhKGx{!bB1{4I?ln*Lqy9x ziPP3Hjhk^u4h}xoE_=fLQEP^QE81KTCmV_+=(66eF8i6;ySyx=N8&lM=pix>Dbd8_Jp!W_KNwK!iwk`Rp7ZU&ScH&Z1&dY_*L8IwY7 zEa>$-ml_~ad&8Asrk#jq$uE-Yuge)KXtN_%j?VXbVz$)}(V`#uRh>j*oV6^JnVG6>{MpCLdL|b%lTa4+BBQm&2p#d*PchBUXQVp9x5tJ7= zO^0Doljcnbjykru6O^8CYWfUWwhUX65WUPmSGx1=2w?E(D>5T4gZa~F?Wt8~jjpOX z{-P6h)J9hka{jP(?UY`hP4F7@Q~rdhNzlNj3nhPoMB_{5cHdy8kvcpXK(I-@D; zV+2$6)DWg#r?A<})9P$96!QM-NLV_V+!g?mZ|xtL2S(tBkNleMH<#8Z$GbA-R||m` z>x+DE!07A!g>+`oh>hLb;8I&9>9&(gWNA+VqIX!Xy5iC+kWd;<<$e8O+4I`*O0XSq zPg1l)c^T}4!kNgwMTi5WelaKAUy>i+WwJSkmh;^W8H`=kiZpOc6V;gv_re@03BQox|?-H&>~WR%0l*EkfTQ8Qw89D5jqeYPbsa z>#*^X{65fYonvukW^^wU5nqIp$U(lU%)>cl$rMp4uTRO2U4=nL&VD3y$(aio$TR?r45e60Y4I;qh3}#VU<6=+40i4#3p^BI4?GGej)|XATu~jrZ_+nEfmP9q)k=zHpf9Hll z|C~0cpOG>+kv)z=x_Fv%|55Dh9djQUT~f-#V#4U4W}Mqmz+8_1a(6`fM~WZ3T}OKo zPZI^QK6e%gq=;GWKkT)7YsQO-l-~1G+sN^mFvJadJ1WO?J3%(OHNczc7T8HoS5X>9 zv<{XyIbQetL}+;xYjJpmv7D@0uW(Qe)~eZMiSd8d;5))^cl2%=CQFX6s+pqb{i^JJI)kO`t2L)uwL})A4c6 zR1+Sa7&jQH1l5z3aL}UfHHAv3JMsrIoqF zIyOCzZ|(y_()-0XM_(|q!4 z7Rd(-<~i(pOp+5S1bbyCH^F;{G(IWXOAWMbVfsz}^yn1nR{HgL1Y|GnMCd znDP(-^^!$V$WOXRYo%UVo}=M*3*=nuI+xiE)0iaK=-z2m`tC*CkP*S$(JqaI)Xogp zH*_N23tDKsItu6S{H)#SYK<$zuw>!2qu!*8eATB;U--D*Ir3)a{j@31A>qg`9?YB1 z(;Q$$#9o38Zp5S6h}B~K*~T)Xrh-)PYJdAaQdIy328lHIAt(oz&z{z+XK=)iU>BA+ z%y_@mesMmVDk0r|wsbG@@`RV#H2v+n2ThU-XyiUIpZl#h!QIft)ogdLuNJLUKu;TE${1~_fzjMSSw33xI zDgeLR*UG#P?H%tw%7a*z!5Z|`ZRKZG+$pp9`fI%>gC&4l$wdRfn-xDXkhI+Bg&Du}+zY{{3F>Mw4YQA+8vhhp(ssaUIU1!#78%!bo`GVM5$ zQ3z&s8>a|KqspOCkU@=9qk3cop+%=p#hk^h_|7yyTE^M1(6>d!%d1TGirZU3_R7ld z>94TpOvAoG^Fdcd>GxcHac^im_>Hxks@q-i(KwmSkzeh=yrE>>YQ0xW!fprPPda=rT3mKE>=uMY9o8XpxAxXl%mMBVfbRx z@S}J(1e@(G9Clhtm(k7rx9m!EMO6(+%_(<34$rOI6E4IP1jsUW->?O1&hQZWK0zS^ zpNxDOQGQW02MW`&o}zapN@kvhrC^XaMyoizM4>~4wEescJHp+X+QN-U>t*Q7`Z2;( z8NV4t4?frFDIDW$7z1bA+s=$nqRHs1gEWAW-(_@T`Dl@Cu*kH-3?^d#RicoA^bOCi4d0;DPK6NIi zweO;ImJ-NV9bhzal@;e1tPD2})|iL|@U_UxofHHNyw3J;@XNl=?o6wfcXYe;|sbc`!0 zVo$#@Q9O-a8K3i;$0L3|`o7M@5ReDf*ykI2l6zqLwDM0ewSF4F_X!>_ z0z=j7w|41Ho03OSey3+X|X$;!ff2PLRWt(sSUG|L5V|IiCdl>Cg%_n4fExEB2oBfwL=8ZfmmszdV%_ z6qRn<)l}{+ymYN4J#FB3wcFOy>k=IuI^g|+i$)#vq4JlIMjlTJg|=J4Q=<~jHgxWA z1~(Z=$Q|W(hAdKjM?za2He0u)UmzQwoSE@yLBo6luJOvYqr{-!itB%1XnkthMgcM5#Omva+9PQ&>!~eN!{4Zz3~tjsb&4G2k8$^z zuO|-9`>W*h7UVaU(y5RO7mD0WY(?JhLK?y~0j|yDN-j+56`&2B7QnYO6&aO}hf8LQ zLS8?^XXHt0JpMX6WDHt+n2U7R(F0%nMy|V8Pi(Ib+ntUAFMC9iR$zc*H%%%tSKBYCx_HiHo)&f+qGo@)&w6Azqmn|`)I8j$ZwS=}x zZF%m6TXQt(U%TdJEg0Bx*V0EQ-Lwrak-jwQ_`JvW7={&foS) zO4OdJ>gxCnpET~4Z-90EQ`(Dt$LjS9KAK;0&bM{m+)#z|1y>rtPA&--k}D!>kd`le zPT<6KXHZkypr;kH%EY7&Co@UZbm%rvm#qYIR+5eLzyf;bBx8KF zzIvSQkbcDX(+{eJvm2=2}_nBCoE# z*z*r8x1<-B?>E$XlS3uQcM))Z!7#imu(Bhh(?p1+58L?&QG})XLF5hZO!XrWp}*c? zwvID^cBzz30P5uF`C!T2O?7Lo@6?tfS(a9iB08xY0Wp;c9!DwUt5$$-EJLt~eOHP4 z#hc>L&`{MPjog}F21^*2)nCQieS-}&KVyDIk2dIUJ&uz#^?XJYg`q%_;XrH~bGBzc zPlND7L^$IayEek(vtA6n2s9wdI1vsOcn3nApl-`AEKjp#m7=IUCVy(_zP4@O0mBB z$TkjJIJz*Tcp2z+duv=Qq*XxuHf0&s2$4J8jl<-#)yfB@Zd8;Iq@p{1j+*tepEHh9 z9qL$>)~~KgAI&N#9bY3QprP!zUO?Vb2d-`OO;BC3;v=37A1 z1&*}x$MfF_C`~3PLkTQ=1(dzfM7F(MU4H1x?kyfnth}jm&)Ra&ML}G}Qz6V@%w)n5 z);q%85#AxQP3;Mp(i7j(t!{VRr;jCjaQ`%J6AsC%dsX|_S16nwu=)7$+c!EshDw+^MCV)mD(gN@>(lba-t54);2xS5k=G!6TCf^SSV}@@sYi4Rk*R+ zHD4#aPqG&&u!_taXeO(azePQX6_(Lhup#au10${cV1q#|C{As<@<+x|OvsS5U#O7H zSPr}mlTo;h-F&(seeFj^eH1bm-tJ$<8Dk5{n|_#XV*WldG>sv008OjA68-}N!~`1K z=z>*4Vs5zSiB=`}SmcjNjSHRT0oF{_-?&?y(qy_D&Qvni-w=a!O9ckQA16fQKl3O* zT`06>ss11ovZeqYmYZh6ayg(ItE>GqA%1jMr-{Q6QeWUmKtvqK@A!tyt1`|ZD8m0z zyIlB;OTkrUt$(>BCkI!xPbDK$QmL;g*~=UMeMW$l{sXrZ4PAGPr@xTLcw;h|;Mq|5 zo80Jur3d>vOaB)3?>(7qA9qMx&-)p#*Jp;KDw&^6udbMO+zKBT%F`M2It!yL74$kE zSU=osEZI8b`XK(Kt}q&%q1Lp*pJD#&*w=JBS=3d&b^Q7M8xB(+t^hKBr8Aw_BSLgz zOTrax{#tw{odiuo+a|Wa5lvpe=++963&&~{r(3NLZV&(c+-aAH01EhPy+2;Ce#^Jn zpnUenvkzxzw&ZJy&?Bfk9Cam|Cu**o?!CwK4AO?z)8BJY$a;6QuTGy7`n*e@2*v^* z3c@|XJXE1)QgctjsbS)dle00@EiJFKi`4PYd7~|$9s8ieZ$1*Klg+Agx+A{@ogk5U zsrE)&W*Xfsk+3CF$DEuqh-ftXQ@<`ZL$Vb7b5^CH+RJQI znbqe~_hjCZjo`4<;mx04Yog=*(+~8x<8Gd3o*`shZ5!`NQ$&!PI{ZIPOKtyDB@Ibw-&K(Sh#VK!{Lp{5I_Ekk%FM=!1qLU zK>rmwQkjF}fq>6~Cp+a<1-xU4Ya`!dx)y(2#q3IOo*3u|=_2D4>JC0Uao8@Mmk1fT z*{?B}W~zjbGB)T1&BeJH?GycT@Eymf-oEX3F762d4D8ivvX{LWK4is6phiX>z#C6l z)#+HP4;$MhEzWzUE>m8Yb*qBT?BrnMn&+1Zk}8d}tk0wH_DFh9?{kR=An>A6mDoNp zx{7t5Z>Kra(U|Di!5HB`-tcKW2)%ZlUJ}TohwGKy9yd((I;&HVtegh5iEJOCpX``x zJj@GH$Nym;#}IG3&toTd#;w}GCiL2<>9_}3f}QY%{VMX@o3BoJ2BD4rXwWV#rPKgP z@)H(&g=$j?v@!4V^LJbK%Z7E+3scvXd5sy3$lOzoK+oSBR`$;SM}l!Eb=p~s%YhY) zBC?;lE>|aMx+riK;%uQJhO5(9F zM!p(W+|V|69l^Lt%-0VemgVZ7h_o)1rXKzyh&k9Ry|`rlv9nuY5dqucK9&(Ku5vdt z%y~Ty{t*wCmZv8V93PXXns%jQTz+aSJoF=)``9TYK4l3C&Ah}-+CLKgf5XRTo9c6Bm6 zUcY}z55~h`iMd5GBy%9r%M}4A&i7ZFsQ$AKL>mZg9KCRqznX)gN*LfQIbYE8kFNh> zu+xd&YpS59bfEaxi2iND``6$40AS|(Z*wZ*dWDFbkN^Fg?3i-8zkC7ydNL5;1{7bO zxS02MpJbemDm@&r;RlQ5hQ-S=g2>9F-o-C#l+_@IQB*VpC1RV z&K6!wiWbVc^T`7P1qKSr$qPnrtN3Q)RK*t%>4XLE)#=RLcsSRl=<&@e-e@Z$%^sM! zl4s)uD%zqP!r0BF^Sv&Hkl$5pI5nP}Ty$?aH46TRlZ-tqx`M)mem5oM(c112l<4L< z*7RaOYcfPaJ7%#{dfq>^FauwRP4sSqsdl};Jk8qy{_992RhMtkVkE`H^TD+%TD*2B z5+bs_EB-&-UZ;KfHuAO(r|HFIFDw@D(#??yc;%sFYF{si@}n>|Fscp#3UEBZ@AL+I zr}x&Dm|vZ*N)G!`ogS(CRQdsrr&zo$-Vnw^cBnEZC6qF#QWO%pRQUS$3ikJOc1^~$ zuWoapN93@L2{ZSLiL$tSv3DTd^|{38RY-FhJaliEv$`|n280ds&R;+^!6&5>Xg60S z>1?tgld`sqRChb-?HDDul;-mM?=Y)xcB3Z0DT-Wq$ zTuK~Mf@*V~xj*ui``{s5AKT1EYNHtSKGfA8G~MH04thw-&l3N>M%o^E=3J7$q1Kpm zsk$n1bvGVW#@a7*cr)v_@N`BvwzM2r6SdOcB}4Qq-l&sg)QC`NtzaJk>BHix1;T$V z7EZ?$){VD+TIlZ9fLWuwbiZ8^cSD~0%)N}L=yjvt^`xYP;EVWCo|0iltYP^`L+H}{ zvbkA`raqX?8@dMl9Z>bMj1EdlzP;{7v;FMEvu#Dod-&;%R|del=G(JBj;?kx$eGVu zrg^2q(Cs+x1iaz(VL_xR1N(Wzy^q$yG*@wn$@9zI#$_Th#h=jT5*+Pc>#(S^FTp0? ze?Xt8Dp};U=H2C2KO&n-x#3nDkiL%kGU7dXg)!dQeU9#$V|z_bE5D%Y8r#eNnUpw0 z!HPP7v(`XCgna~wVLHc9fq~ZT3y7E?w=G2azssUeR0}PUd{tE&+gaRFPGBx*a~=S4 zL%h5ZaNZp^Kh>B7Q-Y7enkzU`e-$B&3;(!sU#6A3-4W>p)g)C{iocd%fgzYvLlQY% zG2?^;kfOBJ(s(x^oE~N2EbGR9SUGzCa>r3`$9^Iu&w!LbY}eOcD~>a=f-X_9ksSKG z{}*kT%zqIBuweWTds04i;cN8}KG49h--+)6&I}^ZD__8XXT? zP4pr?E-``9V-=Oo3Yfcd>g#P|+W|p>9ET+zb08X0V-+t~YHzq($7sQF@|P)jtH7~Q zh?|kJmWw=%2_}k z2O5{*u=lR);Po8%GC_5*Yziro{4vq}S)@frYw{4X5=e9|CK`otFUL0)SKg?`Sj+K{K%;ig%se3hYOcf_a=?CBQi|qVn4|03w<2Xz@<~St;+vy zgvi^qO#52tpXuDpQnsj>xSSBtsXWr=m{8f%C#P&f>IzyJe!K>CU+hlKR@+m}=wQq2 z^H|T;l|2#Jaz5Q54<&k!B^A*52zZfw-n#8(Ak2MPE!}pLfu0!u|@50|v2aw9y&RVPLRzMziR( z;I^F1ojvW&ixQT-kedi3grel;4+%Jjtd}pi+iw$tHClX2g*~uw7Q>%e?$L!kE~%=; z@WtTVhwraJ6&izxXlQF!Na*XvrG;57ZXF23Qu4)@8k7z;K=|M{cQT)TpEgbCcA%u% z+z)4V@B1|%7=G37dS&C-cG##l9M4`LJp%VQSrItp>l+PK@|Ivg^Ga*-(+-{E z@)5{9V21InKZvBeF+SgF=Cejib$}|S2A}d}^TMmsnDd273IEtc;jiPUED^2d!kiS2 z29{fapEWBt1h`0HVv#WA*Nb^-6B(n=H}6#kX63E>_2w)3GD)CX<(3?7=HG-oxL>TV z%a3fPY;A9LzoA+Ii3$3STU`JxWcK!ZaHY?5W41UfREv1J0r{t91i z2!^L)3OP=Lc`ig_t}#FOUZRGUs_E+04;ihLVEs5b5BW(NM75)lK8u?3X;Cgo-Zjc%rj~;UOMp4soc<)h7jj5=hVPnl_ z-MM!zXBPE^g(43OjJ_ggqzd@@gZx8E9XXSbuX9l%Y`N{V3j-7GyqNR$$xr`Y5oSR4 zUZ}N(?jI{_9J?BAa%@-+WxUc9&R4WYVtuac8emMS2t7)*_q~mFblr!F#yfvqk?)FOVw$vLYZVdreNMmGWgN zBg5F*%@doI>~6`Ofav(a`K|R7>5)fW@Zw~65x{PrjI%1}-WV&e6g=8l!=8{~%gveH z^~nBn?sF>KR=))WAwWq5VM)r}2F&gU#wNz83HNPoboDE3>`s*Mf`Lqw#ZSj&C0GoM zMMqu@Y4TRJgM0RgL$K)R3NU`Y?1U}uTV!O@d@knZFCjyOQu6XsEvKd2z z{Y$PZU231ui!&T&bgxhPbNZk(s+~=B9{_-ZdwP0^G5S|XW`*_osh>JeiyhT-U0p4lI#!~r znacrXM%n|S<+V`P;3l>T7tAiFJH>0VgVX)r`$T*v0`bLZoy<{S@z6?vO_bcsEMNF{ zqOVmR-*7PuCJURuR*fkM0rH7qeRt++!Q)%qXhmJA4gAdaF*i{mc}CWRz-Q@3tpCVi zln$jmtjS~93eEkYWg=yyP2e3#p_9#k{3rjREu)M2dj-qPo*uFIPx0rAoA;y6fOkA{ zPfyS0=I6<(Pqr>c4-M1&WL_t>A~&lmbP@Tvc^H^;g?H}x^69^dt79`V0H&Hj5176* z^=x=6@)s|J$kz4L1u3~VfR^w;M;56-UpRZM$`M@ z)E%KJ3d}GrSzxaK!a(CF|1pd2#c^tDqwMa(D}ck_^opseO-eEXRd*)H@~7nI%#i-W zc#er_VFc&i0FT(7L#*QZaYy=!*mW)L1Ag(X+z7)U?3tqpW8 zqWRA4hUVdHh;=yS^4epDKV6>G%YZTp^n{lubc(mW(PlKAYRmP$JO5wW_?AP%*TQxg z?H*NIl+){ul#}cxBkY*hzgwuZE8Ra)nBW4UF0_G$mCREGIV=h(ep4V63o${0qX$YC zP3u>L+9h|?6tXu;VQSrmsbERl_k=4pVuyXhx zh9Sj!0c%+Lz#G2ETG4-g2)K#~i`zR2lZ=~xmf=ryz0UST09{dkZ=it(>BAm&dLtrp zRDdOLmv$&G7MYswO_5SdAcM@CiX;D)nnavd`N?1@C0f@{nH%S{}n!8;3vUQ zu9bcfdbNQ*6VyNx7%Gh?!BJAYqVz#oODw`VGR=XQf6FH1zhylZ9l*96L+eUi6blq4 z?=EAsbN!F-tCt^<>zrKrzk!lF|2I%_qW}Mod(Dd^fI`JZL-n&o0^4u37<;>q45Y$d1yh7Ku zO@?ossCpu?afy6)z57&+h9K-_1EYo9HHZB5*C&dv5qhC7Xgp}Hm zAams6S}PBqjzWVWCAJ^xy>%qO8G>P#1U5qGf3J89CkE;G@)8BmUjiC)K(QAMF9~Gd zyVCm%773t3e;-6dhskS$PjAbV`)lLvdZ}-OEvP#o3k@aNPSE}n&1uJVLH$N;Tf-xH zX%lVS@S$4B>+R8O{AK3vwx(0DY0#^0Klgpg#C)9Pvzb2^3cLQEE=0E}4d!h|Mapbd z_#lQ~;kSYK3{KU?=8_j0n=}DchWCI(M!f5E6*?D&Qn`ML|IM&AtxWRBTwD=>08v6g zi+@F};$h2RY2zMK8A>qpph?aqOS+Fm`rqAQ{;9*OEjd1a9912ZGP$h##D02~@v1~Y z_$UvE1VJUK>bJ)HxU-zx%ma$yY4eP(w-hwsd3Smw=5@Bqz>#ivQEyND@vgdaW-^dw z-)x#iuOF?iW|8+6UU}I)gIo3;%Pf$HMTR)m(@O~3D4WV7fC zV^xxGVSGKXh=812)~SY0^Zha!1D{Td`kCg(64KdR3{X}`HC#VX2F09+@M5Ud2hIik z?D~yRUtoqXT#|{9scD6n>a^(IG5|?nU!3-0w{5M9GzyAAt849@-41g<2b~1lW6js9KJiKm#8lYBTMjARUUk3hcTdX zTp&~~C6sUDDbk4fg@XF`-T`REc}#v2Ea3Q&XFRkrkT8NIfsGgLUuzq{{*2M^eFV|= zC?c0U^DNtO2pO5Y9(BZ9epx}W$Us@dGd>Zt9J2BT{jBdBPpkJMMTX~7Cbej|(M2>` z5|t6b1p0LA^6M&ZvUL*1Fzt~Vp}x7Ei2`~>~~V@LCN{_P*kd) z-r`t9)d32oLW4t`rsW;e0bmlpZyX#h$2}9<4g#;z%TB}*>1v2d4hBBC}0uAP`_uk!J`H)&;S4KtbtL#p;v5s7;e|X0QTS_9iTo{pK zoX^)vsuHT7e*xjgQ|>&RN>g5vI?Lvr{Fh0O&yqyxPN1sAe(?Tl1O(iU@)y}rnY zjrp{j+AK9mn&bkwe5*NSZ@tT0{#j(n`Ia_@@!VBJ_cz{17nj-Y_pSchKq2y4Y6oi! zkg?Dy6gAsm-~Zt=z*U3{5Q#pK(+o>_irR*w z?Jhzyu)m1q7u11Z{0@H`cj^BFB;42k{W;uUUoiPT@UCJjJK5`Ah5(!ZffxzA@^*7QBhQM~T&U$sIyD+WgV5R0_HLa9QTIJ>f#?`egq__W-D`2xZ`(G$% z?+^9a!SezpSr=^-774iT0R#AtUF_WZis?Vr=7#-0x z3~9MIZ%ksU-Ds;?Wd#~I#0D5}pDo{n$(8K5_DwE|{EQuLNam*EAhax*P^#nL;PvT5 zNCkD^6Ly^h(XzXkuFHziDj29a%2| z#{_Vx| zE;02`EXf=P}P_!DEnod9{Le5r#zN{HS#%KZU-z)7S-~TU&s@HwFCar<=_w(u?dOZ!MxpN-l3k*!w&f-r^;pZ>&|28;#?@sojdu2Bq9YF}Z)es)K zI!_pcKA47}!U+5>)5r7mYG(H!^~vBlyLy(jxgjq-z(@Jou+JP$DjaP$=EV^ik2zmN zR^t7#QeR2`AIjb`EUqmH8-)-c1WSTD1b25!f`#C2jk~*ha1ZY8?(Xg$+}+)^@6MT- zZ_b(T{ePF}X(Isscs*-{dl&;l4*t$6E^Jh&sE#uG^%D zMugao?-X=&OWRkk0%N!Z8t7w}Y!B7uK!LM z^O?|EYQ*3znOka*8#jCGF);u`YyT65j$6+9QD-)`Imx}Y-HY={biOq`C+k=At=fN~ zY2;lqvIvbDi(NqfG0C=RcBM6}+0Ml*!M#W7PHFdlNGH{T8p)JlA&)(2)Ms@x8vM+< zmV6pH%5X~EZ9q#xI-Gul@0{F0pLd}UHppgkS3;sauSYhD`DG|G^;q;ruSMBYZ3QX4 z?(3A*soTA-e*k7Bp2pa+Li6`q^RbQMQ-jOUH97g1uG3O#Q7pdD$@=51m-KM%aV5jb zeP&PLF!wMF#C7wk$nzcXOEb31S$eWeupU`EXXx@s>M`dp-(I6GoQ@CQk`+Ca!%1Xw zZvWw>?~z7iWDv#=@4V?au#;V+%5%{TY)78;C0IdQ63Pe92w?JCPf^zI?HE)a1kBK= zHD$nMf8wnF*0d%pdf6mrfo#P6>z9V3*0s5)=q2*D*(-*m2r2F3s+9IOp1P=OTWqN> zOWxaO54aHNN2+s)8QMS++F1`Zjg0=-I2Xc6=y9uNky112`-7u#KIjP8Z?v^M7K*%r z{ngrVar$4# zZP(KQRTsQWDI*bShC(_jFIWG~8k$)B4M}{l@(ITVz{5^kA;F}sL{NMe@#0(~FB-~2 zu7akP5_H&=U_Ko$u=YGo<~ew^zI1L<@4aPbJ39Do-=bEsTbgLB09&^@Xf2{rsJ}TNYVTBrzlwJH?#z z7k9bMl$@|DMBAHN7z^=i0)~;*JL>P>Bcn{6D1HtYA=KYNg;-qUiLW`!lZk7q9eZ&S zYL)zr=p)tCv4G4gv3ZDQ`1?&4IAZd?M_9Ir&^wrw6@}3aMrf0t47;23HD*kT<0Vz> z<$oYeNk;kt@|VYsQ@`iP#odqUIFCMZF@PSt=GD?IA3^_mA>}AcKr8SPn@2+ezm=J?I&<) zDYsGsSiSJ3N0ul=Hdz^x0+jPt&V4(nGSY8Fy#{^^h{J zq$HX-8WJy zbN8J;&iL%}x$-8zmrU43`%Xyd2A*tgxqQX8DXDNAG|4^9L32cg^TcvVNh;mou4VY=74yoEF+ zosC|63vr9fHk#VCWvNJN2$7Ifn^|l5k&`n2ke5)lep9oJJZI_q8}8LW$_BhhpM1H< z*7%{H-ET!A2svEdE~u@VPbIW8`>h*aaut>iGPYUE02r`0(Dn}K72tgq{lLHZhZw2t zlCGhY$Pv1azxi#o*68|HUl@Qar-aQhPQcpX>`lJzsakm&FqqqhDCuHON%>(7ZQ16^ zii+Bt=XM^|F+iT09vH0!tP22Hot&H+xYI($Di5(>Z_jj|dqfd6<{F1{!<2=({|Vec zin+O39i@|hus4E4_KeRu#Ig|%MrrTeUy2edOZyqm=^Lf#L3q+^2>I@mDMhm_C9ssF z_3IOpZ)nbnI%*MzUnpK#IEe zgN1)gPFbeFi4FbqX)%hWcb7|+{Gmw z$b8BOB+zH!rpvAm+OT%~fQ}zDyhBL( zQ1WfX{6Dn-4AgC^J$R0M^17}0`QDRTWIESu8wUr2O*7Q}>5=fs%$~R%ApdLVotxIU3v6f+u;>fg}hdDNtRY7VkMrqkKj9%g`4_S_7c%yx2BQo#17wE2K6k={A4 zxa9P@!D&|hv<4Z;wR7?q)1scf37p`Af)af2&TvqEErH1EC)swl$f{slob*gTEa1oj z*bT&h=XHRsE5zSiP)>765d~4F?8)TFZ1VGs6n0TQw_m_R&>aXPzCYgLmG@k2uW&}(pyra3Yt{pp_OU8@aWctYxX9_7^u;Z*- za|(U?eFA=~j?>W%284308>8aUpr-oiLNmc=66@XxyXKqqyp3!l3^ezt$3ZzwgOmE< zh087E8k3dqfP9|ujh?qDlCY!SR4bgfRQX=4*u1?vm+iq@BhE*bxDPuYh+i4vfV*Zm zYxQK?zcF?eC1FPh)7=&G%xq}*b#ce`9J2L!uX!w$D>^b#avY#s=HH+*7<wJgM{k-C<|z zMO?NW2xZadO>1FGgpwZqa)hdrQt1v(?K+DqV8Ol7{+OCWZrvj4*;9aCvdt z2ilxq1?3~+b1ide%yInSG&e9NG1dYxZR*(6lTrSEpliJ&bK~j7I+h*#ClXVcZ_Vm{ zq_)c9JzPIMe|YdlJk(V|bOt-x9Hs1`)U8zFBXM)nv4POvXKmlTIH~ zcd_C)fB+^JLYTc+<3*PC%`{7H*Ar4uyh|Px7StnA(%^&$Xr|65#HRF%Y{G&~w z>mB6XNhD=Mtq&Zf4uIDpTSi`b zUuj7rz-bUtF@}Ti@nruwT#l=(KFD`q^bO{V-;C_bJUyWo-wTY@%S6LrRYEYieQr{`T}0F2YYiv#IT2*4k{fA`U<{O-JAm-e)~G=t2Nd7%&2ckenK-_-2l2 z;}y~y&_8{qmZQat{1gRPettJ=@xTPoe%05b%J5dj&lo@d7txv-_pr8(OeW?|oU1IM z0t|=3gjMyQYTVVykXN9DqKJT<^~ihterd_);eOE2*>2iG1C-V!YR}m6=74_#&ObTs zs*5@uAY>W)VRilP`SWAd-U(lR+O$3gAi9yaD=b_Q>i(L7nnP3dY@%*EVxRuhpzs!A zy4swyS2)!oc6b7VQqoC(H>v;E?hB6nWdwh=EJyb>JDf>~w--BI#TphCP~@+n%}v}5 z&8yzYD5?%s<-$L_E|ueu_VIFi>Wq)2pCiB99(%&?-m$6jSQ$Dz=@k-6aCb8;xwwJP zXlOXkagU9QtFRm&Zvo$qFbC(Bl*9}unir=(J;0_+dI*k*7#C6s3f2{QtQN|VQ&E@Z z#yn^QOim z?Mc<)PQIMlkS2XD1G^XK_ag-#nA_d8~~mc zDkls7iA3iA%en*P2r5C&< zTP8}Xk^D0>rRtl0FhD+ns;wPwMWIbVL^Nu8YPQ$N+;d0{%uE@2YwWjU)1r=V}-Tu{PDm(%KuW7&xo4JB- zwr%HJZ6<({Wod0$Hv3ad1vd4UC)Z%n8wHduYe|>z7qWM~2rS>tYrQp&W<0qlmF{zG7nP5whyaH=j zO_7HkP_F=->LEU{`%x0Z1o1W&c*O2woWT72vZw!gX6?K{&8sN^3=nzBSWxCM(8Cwk zwnT2)w5PLyBdE*qN7BjpBSsDMUz#i%-H%@o8ZQiv+0?=sT^+%KM#BuR*Pi2m%!cN7 zcJ!Z(7TJQ)&~|Vo#n&H{);3x_U&Fq#Bal-w$69W-7cidn->fli&hp#Ly+nNc0QBG+ z=ni&8R4h=LmUx0w+ijojpBfF$C4Buv&Uf~%DC%Rn;NY-Hdxe1EjI_cL?didnHlf1A z!|kB?^*K^f&LB;slRf(JR$=Y0_{p=LB1d&6I4v66Zo76|&gb4iJg@hf370*Zff3+f z70b4b(+ZQFkC>SIZ>$@y1@%&t)g4yNJ4Wi=^F_H2w*-Xxg|^B+Ds!NI5MM@~9j z;V`A;A%ojug2|HGdU%~XHh*RH`q-aQSa>4{Ab_)m(b)rPE{2N=Wt*847Tl@vY zzwAzBf#2mgqK{;q1Xl(n|Ej7j+TGXzNKyc{x9)BMpu0L}l9O8OoZ%kG;}DSgfx?H1 z+STcB6VVYeS2>stB_|4&laXudV2e_Te&dMb?U;4(7$*7o<4KHJW&MrjI5H#%mM%Uz zrG)q`J2El#3x*>sFg|gzuyrE}nEO~6i=l<^F1w0hVI^}NZy7zAwEi%t+mE;uRd?<~ zMUz#`{9$R%Npl_!OyZ70QmJix(Y3K6J-(WY`xAxYzkN!aP97Oyf4mU<+%(g(2B>f z@PCR+=RZUIt+;d^8GVzcBF%h$&o@6G_U|xyV}kSg!NDo!HfW(`! zkY@b?q(*?G{!T%U!C)?rA$?s`DE1Nnh5~h~QkDN5V(~|d zZ;v=RJMAa`{ybH4tj{?=Y(pXTGP#oZAJh1+?<{aq_q8tmMMnR8J1{WCm6ghOEwXWYY zPCo!FVL)hXoc^6pd}P&BmN0MlpCaz>h`XnEm%r7noJ{QKM<6ia4s7Y=3`Vzh26gu_ z_je)wO6p)o7t#ulwU3*Y^Y^syhN1wC4Gre+hz8n=y%X(AYIE|S%lGGog*0eDr zxbfFO*5_HY-M#g#BN3}=bf;uDh?R2G$h=MO`oREzYw!HxCYymjA0R!lFZrV=f%`Cj z)3i2pUs>s^=RVwj=H8j9?b2^Qw}GD z#nwxce_aqHkBfxk%z1GC01K#{pOa3(p#VX>uvWE7Aq9PQMaD%mGenC#D=-0AvUmBw zz@pifBC;3f!R|Ewn7T&6oZ;=H!$fLmQyD$&D0-D6o&IE3EoOOg-^<)XDaBX>5RlR(9wCP8Q-j}P$du=} zzU$cJHbSWHb#pXa{dI3jOG^1xz4R(DRBGjSP>VYEfWzC#GE)+Rj^!fm(h)X~g$w3~ zf1RhuwRf&#NDjG+4E1LcdJyY#L3AfR75S%Hp8D+m5sisr$vYK%uVP4vk2YKLOZzUW zr4Uj#+&7hMKL55bHbj@#=jZG9{J`%Jy9I4EH?Yr3lWd>w=kOQ(wYY8%g}vz~aEv+53GI_82~El<$(My^nS{PWPWa?8E0-EMFc!E!&uR7&AcD>D``4 zk~>H{a@BDKR=P9n3>I@F7#1IW|7(?dSU3-^?9mR>cy`CDVGg$zzVp!w736r$ls|_> z#0lB*aA+e)rvadyQUf^14IsL|)Y5#*v#OTKETo(?PCf40 zUplE*^_OocNjELyGx|1x^0-v;t<3~;RDj6c=3REcEGi^JqW;KPLKVyNGTktP=Hp6CiSOrTHmUUj-pTTz+= zp>vOI{zy?l2ehnVEq%T9{J`q{N3lhzt?*&X-MTfr?Uh_u&g(0Up=SGRg&cw)QQ<`m z546Y!>W615CQ=o(`B=snobn*Q%T^lRf`4VwFLCt7wb#acmhS`FK*3?#x!eQw-!jM* zPQWe1`%XgqI?d4Dkg9c-0pN>=wK&|WkWAC1V$6y9(4XORQ)6~^$=MZSYlZ*LQ*U?W z{OOR$3H@4H5xXa+vfl=_v*V;Hpb_ueCkQW82{?RfSbppKFh1f1p9L_^(aqvV9=3ecNzr3z0m`R=r9V?(Tl%5E`Hs6b3 zx--YWT2R1g=DakuJQJF>KSvG72n6AAN7hITFT~n1yCVpP!r&|byY?S;1`LuvSi@Xp z4?_O5sBv+^VHnXXpE8Pvos!vS8_4&kcEoYV?hUwTr%2GqKFA*N*Lhmtc&+n}dDgd9 z8$i3n^cLW>L-SOejG0b6{|6!9I8x5aa*a;(xG%YrOZRGvb~+x@(4%$p_XPhc`=gQm z^E!1XDdGbPU&8;i6|E_+b*tJRPvy^E^&s=-lOf*h+#*^!B z!`4{j;JI#t^*hcUKS?*llDdM^TU{eUipB`|UuA#!Q1~SHU|^c#^n9cE`a&m+L*=Kf ziMT`q5*wBs4x1Vs(NrxXb04*-wPLQMkx&v>l3^Epk2)vA*4O|k_gU^R5Tj4t0F`rK z=dR!DrwvwlL2G)1NW}K?t}Wjcr{UOM%AK<(35UQtBHs_+{`RE+X`)~-gGeBSrTiCc z7Z&U749p(br6us%GymLvvj|6-ipYZI&H{PrwT*vrXcT_Q+f(=9^-NwdCi8?jP0p3w zdd%Uz$<0E+SwDKX27h1TNZUn~YAgmL4W?rO=+89aG@MK3NLHQiLAUT6OYO#FyJT#s zIDd%;ulKKg5fA>y%>efnZdLXYpf_Q}w{nK0^?P3F!R3X@gI7cTjLFgN{PQ;Q#x^*n zyDi-YZ>hooqLLcpQwmo&`5tD4I?Gb)IBFXR%ctFT<-kOn&KL)SnV;`tbgj2}^Fg4~ zD=irh>j&_d)*yR@@x{~gi)Q2ecC52C^rF*U=FvKG<*~-dpj`XJDj2S%TbGf%9jW6C zU+tUrA;K8$n2{a#*rW44F^r8&bzaN5ggB)r$A=K>pgLmLb?n{I0|i&g6;<{R-S%GW z<|5Y}g|=#v<;$)QIPGCEXPU!LZe`rzSMC1`4*sFt z2&3)6h{n<&<|8VvUwaD*Y+QeWt`S9%K%l>~9l^+8OD(Ft+nXQ@1lh@@kza0IogvwYB{=g`KaHgTC{W(r6K1o}!VP+DNvW<$`uq5UB{|5o--+H-Phax5=N8tzcRjwzP_9#1YcBCRPfN=P-#+S_(!M< zD$BWAG$hawMSgP8rT?TeV;R4j(^56a$jGS_a>r`agJS;1sx_)VI)WKzFmO(`IixJH z1{}$dG31tT+oRk0TX1sM?|KL$$GZ==mYBnwOWFZ-JHI_J9r@Tx#%iWQFP>YTT&iQBqYz~?R@Pg}SUs5%%)4026BI0JvMKKzXx8h=6bcQx^6-vgJ@Y;7^t z>rcax@E|#?TLw1lP2yU!1w?MG5#O*Z+aYMi^qzB}D2z}NFqtPpsx|%ui)mK6*i^Xh ze_{=Jkm4Q8!bOv3UIMcv8aID~b2VZRSW`#Rmtr)nR;e-yq`ovAn$*jsc ze!hOe>C>2Xppq=voA6gjp=_<&v7M^xJF(`qh*%0>;$*y7o4bYj$EF9aQCOy1w6Iun ziA3NQF}qtY$`V>tMLvvr~vqaZAjujuQ?#^U>MH zC%b)(wtF0At5TP{_9?t-Hf;!M(cs>?vsY-ZvL$tdP|P79w?{`=ZW)n=;B*hj9Eg&d zdZ%4w4X8*of0kVH5y{SiJU@8kem~kStpKgoz9P7atYXA+l7Rp8k(wXO4IX_&mK+;v zxjtlpsL{ZyR4I*#OCtsM6MSE7X&ZF;_-)x~45GH$C~&vZZ7?9d)@=Z4WA3zDtR$!0 zEdZ@gr^9ERUqX8uFBzz$Nwb6Z3N}75<*60ZMV=_Au%T!td;ZqjLJoJ8MlxFh>dzT( z-wqmBb&JyJqxp!v$y8%=_N^ITzz9$m78j;3(L`oeH_ zb2j%?Qd*ZZ?w4%hHs=wR@#l2a+>E_qsvXsSgr~J?@1Xocm@LSF?a|q2NsMpR5IA~% z_4a;ychbKGzD3H-HN0ph_sUQA^t&^Z*eymAgShYZ`s(CVx2JieIMa+sZfK~HbWfJd z?k2Q*#|?q`n!o3~>xa3ywF;F1?BGO2zgeX+4p#D6wa2}a=$ZY9Up^*2UDM-wKFF=z7KeZL z#JWJk_rqv5sg;Kj3f5+hGcPvlrrUZia9pQePQ7rh2YVAdzY{szei}?hN#H*caA*`^ zzRC4ALVNmDlnb$yMpJdC7nv&OC4@5cdxl$umlgFciHBxKKPC<;n3|5gV0-g+gV&18 zyyeL%kHIHvC1f1zn8$!}=FMz8r%Rz<*IjdQ>a#YjxmIBlizOvp7`MgM@$_8tet5yf zhZzLxBv%+GXTW3KJw*P`qu}r7i=@@Ph!zvKs981wW{i(mbxodTOYUB3pufMvxR_Vx|W-vX0GhIi?_8VKD*-XCzwLfiw^`u4>R(VxEYjfuybGg_t$tXp)H zZhMLwT0P$Hs~+c`I`2K5Kclwtc?DXRUkPUX&`^a5AaV6?K=`9kCF1#x*BH>n`g8yZ zQ{HRu`i)i7?D`H+{gmRVU#aitY7VCM{A16KT~PiQ+Qwv8pJ5J1+x(M2{1kP|CheV1 z|By4mt;My>o%o-v_IY^qxUD4^bKG|4o9nMlci{0<>aBrHD7!;F&(3OKD7~4K=;!s- zRj0bMiTzvID7N$2W9Vn&YcXGBm(bbLl_VM;rT(!R2YX_0Kj?M`r?Gwx%mMNs>KW~B zJP_)+sX5EX?MUO@n%fexD`zcteJQpQfi34cVjEQP@i2RnsB0=EFPi@F&XaMr{%5f9 z3%48YEt(v8`#059yC`H3s&!`40xCC>WLwu2J-x1;A@q_rOtzvjIivE!Y>x>S356$nrYyt4ti)L z#Az8cSV0luaXt+gS#&r&C~VlDtZ2gLhmMdNWNqzz5uRf z`CR!Q$8Hab9#_TiAqx57#z|%xUagMFJR{r~k&Yq0Afqu}l8#bqr$I}RtaAPz0ps%tDict zAo5yWlQh2n-6`uaKcZ`QO>TaNXv0FdrGp5KkBY}Ta^~hL{mXNsbO-4k@ThmN)DLSnxAOZ9=LL!x@c%+6> zFj9j2{eXrsdpgxbjLTHAdY9&CN9Z=83n4ErzrNVSnrk@Drs@DU@L_ZJGbA>k`1()oWF-Ay@R$iKBPvuh*Oy?AIh1wW}8Z-96=<){rglr+wm4CPL+m_W}}YxQAup_< zpz|fHg0CPF*UmcBw>nXX`+lWX!Y9`@j8!QpD44C@qeZ+}lLWwz#+wz_s^Rp#J|MO| z9}^=b-c2{v;;gEB_P0H|Os_&&?WudnD=5ss)nVQ|Q(vFr1ZzJzBF*}K*fb!NLiGE< zF0Y>$hAvqquMorLC<9SYM3IWqkoTWjfNF4iQE+fJDTlpP=H~YoNNm=jQ1Xn8qa3ln z-Z2`SL6!Ua*7@E`?EhW}h}TS(F;v}D8s^YjMV2;PJ-t48?ytCsI;h=yF?yyy#XoU! zSQcDfUR=*=sCuqO=Cm&gil;X%EGgN(l6vi%AWz%%JUxp@PR4oQ-?=QmxoP(@9~s=t z6u~cc)EG>hHdAi?NfZHwo13+;JKmO1?345ucx~$(M}AG0o+`ep7JeL)?}8O##mbnJ zZhc3TBm$3&98pws2w4cUS8|?AKRu7!nybs)z2-L#k@+?t)?PgDN((I%LXE1Uf)?$w zg@zSV_>2guOLOzT6}v9V8B^wVIh1vZW5;(FR^!mJzTflU`MM+8|JnM6*JXzLMHN9J z>ugfYpX1}-wmEREq51iAXYZN7M555-F#YDTmAhc|Km-|V)2om2El^D&pRV21?v)5t zGNrD)g$|3w_XGRy#!Z0+hx6`S^?C#NDWi?;AJli@9E}aOvo=I%!aJg|m+tvS3zK1r zs*b7+>MN90ML8)yOtE&JBspG@ZQvF4^8c{uPgRX3tD@q+>5Or@f^Qbb1Z=i- z**1qEt?<}F`0qp|ZJ=~_$E-R=>>qmxXeaBZCbVx4AUOg0YM%Fxk;4NliG%EkqK!Xa8jV5Lc*`v@ z8jK~YX&spDrt^w_tg$uD#5zA!c+d448;2g|oTZy)MQNAbUq&^W)Vr2RvS8L)LkecL z6Fu#i)27(1ymPgMahzV2;a|=ww{`kMx*H$NZR7W6NlvOf>Yc#S-1qKLQLE%xwzlzt zfu?(-W_rdD+~~xV9#gk+Y2YC=^m4L0LIU@)yo?II zpi#jTx~<7iXdhl&?lnfYKO{W*QJ9+CLs;({(^!CrC=TQaZL7(!HhTlmrMDwKwjK?#Y9pLomRH^WA z+EUjQ-&5F2;mW*T?2GiGfRzQ{@$}{%6)5sHi|=#PI%-#?3=y?0EwyBOX;uNo`-TnZ z9v)^4UD4^I2mTv$IT2(s!Sc>3PXGPEpai@|y_TtQJ1>XtxNs=JiL54E#^|t$6!jGc zwOVYWSRJ9cl^m~5C+vN+Uh#Yp`D=03q3TJ)q?%Xu9ItjV7d`xP9V1|BgRbSL1ttev zY6&D321CTJ*?j9Oa8Q%3R0Vl&S8=&hC1B;he8st?s+<;{)-3mq0>!oQ1al&>p!Ztx zh@r2J$5CGFzdrIUj^l6DN`$j8K`L39X_QjG(Ar$-4;)!C5;38+qEG2&kJUSy_dJo% z1m{^`M#Y*>;}el}hKgyicsg&P`QOfL=um4Zb%9oc#JWnT_5)gaTd=Y(?%vWsla!|p zKSx9}#Cl}`qqcujh9VpVNmfR+Iy2hx-TyQTX>Fx`{7yFN783blN1-eVUdMWEA%RbJ zWuR}Z{{^?fg(^RSb)@q63Q{zXSF#f5|0&d;aBrEgc$Ssn^V}QyIBhmH<9`H!dM#6a z)pr#!JuW4-;4ME(;}r0a44Q$%ony$myIoc>uTV1RL5wV$9RCEc?RH0@MwY?2Cb6zX+6@SIjZBiF)e^;@pB0Jzk)qR@nf}Y_O~5 zi>KK)EWllM9As!Q@5ow3pMhhIpLvc~kA(90iivq|88!H(;zgPB z&ig7rI*RBSCa?_JE-d(ijDqGg`V+sE^Il2g)7Qj%Lc;?^EibGt_F8!KsFMm7S{U_l z7q)an!H7LZ9GtaT>aMhCJ9`x>Qqp-vUleRH#NJ&bPrShp>Kq+S<*rfZuY}FR|1zJrc21$;0m;$?c<7?*`NoB#aD6}llxzn(%|BRKJ{w* zj=gURTp+QK(h)8lw+kfmVhnMJ`oy0`SQ=M>C7RPClwG_K=ty!ERnKJkedSK@upG+j z45vGO34VaxpwdMV_3=>s#` znvhWOy0;vnF30t0q?UjD5JnF=6Wa28y$KL*LjiQsjB(@D-EL$Jh5xUy@R^imPnJq> zO@8)>xO!jCd6WIBG+xGjU;?hxX80Rds!_aQ%<1C1iuX+wnCNNrrntneGcFO?wj`CJ z+(8jxy7o7ZiPU|zYpvPzKk^GdpjLPC<%H)&x;VnG=k~m@i87pSS7V?r9V1@T>pRyV zQ8nzK^%6}kk_*}cB~4X#av5E6|8XsPPQDmBH(H)9l*=F#!qI^D6lLoSBWv8=Yp)JH zJUn(ztF*OxxyeCN78^46WD?D?_>L|*6G=Rf3ed1v@fN_X=l+R%8SkGbpiWzLlv~}Q zq90itHnK8sK8gG|sR0Mm6K^?2anSzu2rGiqx-U6ADo;tJNyiQvJL!6Sr6>Irg(-48 zRjPN&8g02KYUVqu0V`9jlBU;AH_U#jw~Ep>GnXiFoNvgWi%t@0%o3em8SAGjFi8dS zF=d8+K{=~-v&ZdzG4&*%%8PkBo3mpxnojQw!@zU`n+xiw?~9R2`4PwO?liVk8o0c%1Wayvm+g2sYS`lxY!lNH{0_d*87D^- ztx3d{?bj=s9o1-{7Jl>!0|1oJtdl^kr-NJ0itC zaUO%^jU>}u(%|6y;HAps{z?)b8_sIZ%d-CW)nFp-ke+)dMayyGtMuxig4Y$RGhl)^ z8h3Gk7%ic-@V4I*rUbf*NKegM#s1#;aeCGtBRS;2ye;?8s^GpkWGvj^Vb8f}kMBa0 zkDuHODG!SI$vN}Au4_XUj1IpK5WKM#>20C*jtx>{#IKOnlk4n-I>T>ss+E~=GNxE* zkFEhHRc;$rQQu)E;X@j!e_6V-hFQ_pdVSLTWx%34^j$|z)tfijk2QYV_idCTnii)_ zr~U?HbxF&sF3JESAWB+zuezTGN|<{@ zx7yiHGp9wD+q~!RJ{|8l$$gFPCT|4v$!2&eiTOA&Gpo1Xh1%fA>R^`)7SGH)CFCmW{rjw0&40}>gfgVd<w zK$W4V?fb;!$DRdsT)V#MZ3618;a`=KA0baT#qGq|f?4!|#NG++Tj+{=82^BFfoUtd z9;uYa(2|#ZcpO}oJ(Nco!Ej&LnDDdj$}QogPpf+^=qKMH6ko!+*zM0rK|aPrJ^i=L zWZZy*!yJRLn)uDCGaxi9berB({?&0&4WX1$ir9V~pC<`UfII5d0=4zU9S#q}^A#q@ zb-lW$k1v4C96S3n!B+W!I@Z-3rSDAJu5os@CWBcbnvjvSRGJ&TU-~G_)O((Ha5uG% z6)R0Bm#(z^j+n=m)ok)-`i(Tfm!u3jTuVrLmHUNh_XqaAZIQmYxDuOINdJyXPP z2Zu|10B!y!)L6!6X@lHn&Bf)no{Y^mAu{des!I&lW+JIDh)=9qJhDFwG`^q!#uk%^ zg=8l@Vm#f%f4Zk(&O_>Z+)>W@EK+?fO@Xq8zxRdEJGQ3&tiN1}g*q%g4`hDsGSL)K zbA~jvfsDLS3@JTv;~I!0f?zgmdF&neLq_qvYs%xq_?ky%-u*SvNPWV!Ip=&zji7kA zuF0BDi5q8BL43S6qH9RK|BMIb0(QCUx5-5GtG&RqYY#14>Z=mKYtZdO&at~17QDS6 zWq8(~JuY3y7rvcV5$nlu<|!ZEF89R#@_U4YWqiayXH;ObuWWQk0V$Wf!a5#nSgNpO zr2=PI2zi?&_2u2=U2p#hcJ8#rG4qNW79VvQM)S`3Ur^Y4ixaxy@Zbp_(fBx`85#ka+h$WzT?(~gU#1HI=D$#4lXE#Y2LnPBlbcAXM7C<{Zj1L3 z+g17!pM;Nu_LA6Cuf4tZYipS}T^z{gVb48ddyYe(+*;1><~Fl@nTQARQ+`#su?alm zx?&clEsw_Ch+@*hF4S71EtyO`ugSi2_dRu<4XMY?Zv%>`*QR>8V0q# z8nq<7kNG1M=2Il(=sKl1_A$H!Ua~~RF3s*}18z+xo@hF6BATHqb3r*qaMo7j#TFcA zV*)?_d~`kvph0%@o+neLy-S?M9)O?u;@BMj26EXZ9r~;-Ixpzv| zS4Bfi?d2wfTcm;q#U*Wch+l%|(^5nyv8bH~j|NQfzEni~+&mEb)eaj+U{e`Gd4CU^ z)$5ZUL2?`{OvQ#4~<8`!ZefGpp&b z$M?POWMfVrS56OjZhD5zXs!g#CK@$+8%UDAfS(fU7DBC!Sz2$n-ehkw#Bn&_<=DL= z>tPOo9$%I^%YbZYWCUk9s7L&OAFL*?mOGYX z38Vm#ctN@$0SxbKYFGJM`-F_`><1o-PwCI}PPU2exH3pI#5fsE#nL8=Bf+(5jA_3s z$Y_)1E$}XEerDzIl`ntWZI3$E)i@p9k#@GUQ-wyFH#BwO8E*dQU69_|p^1XmJP7F} zCeo!5kzYUC0d~FUzM_Pr?L(~+Tg~(Qg)!)t`_}K)`?yh$z%mO7LYgt3R#(V`F7DE& z#ECF?{@3b#$w+UFA=cHMJbhey==d9gO`r8LBTxnGTNXjq=oX{X2YzJtM^2*5@~pEG zNR%&>2Hh@oFC|$`+fk14ne7pK_`E!TZ8Lj-55 z-CeuCN*nW`G?CR)f{<4edYGvW*&{u`$KH%5gH07yh^~x+4ttr~X+5_fAr$6#rfhx5 zli_je$jRLL^8$J2eed42rxwr43u{iU!rCzq`kT%Kl!(`Pj>q9rXZ&i@CD6rpf2S#d zE}Sm?XkS@zvUaQgo4WK)m_;2H(TyA=eE#cC+S@|X02n?2H}#_7Id6EFKA>{pA6rI=Db4~Fd#UiOK) zntF9d8oQ_ih07;9_2z$U1a@AZ6%uimQ)_3chM(Ps=V|OKL>N}l!lro=LYOZ~f_z}I z>amqgYQ6=0hvChrINE9~5ogY>iWon|8-dl%yf2~6*i9_E*&B7kO_z2@ObhwgfdfKQ z;#qVQ+=4ODMn4e2otGMic)U>O+nO_=${S@}lYgwMm=(wPT5p)66gx4qFTzge_b(ZCrtN$ z3NU;GIWKj0OhL2WfI5Qp-2)RQ(MpBAqm`VVS!d~@t}HcAGf;?HnJ$4b9gS~qJX5}Yt)MO+&|rsF$sxf1Y9195y^IBy`9%lpXj)SPmugM~o$9M=aIwpD}{s9SU#YN#|L8(9%oExixdO2DxA!0yH*eO;}!C1EQ`KC2*~&KS%gid#E#;EC z@51uG9SnXhLG&$;kDHg1d{dCCpTU8-r7jcNs|tEi$^9~lqX&U+^mN?`3w@U6;WQ&z zTyW21<81GuCkI)q`}0UJmO8EN`py=*lz_MScakkry!k@zJvftH0}pE7ksj0Ww}d*# z5)S}Fb*fb7@@Yyb#UNTarsE3y13yvo_|MY9))cuHw}<8@uK3 z-{;LBn=#?6<*P;=`YHp$21}wMYNXcgXKyhc*XEawfVi+d322kpZiSY+4Wf05B#j%1 z!{1px+0AjY7;*Ti+CqJ$?Ni+i4p5xd!<(PNU}j$Rx;=1;sv6ZJ8>`D#bWofysx^Y@ zujc7~P2^nK!rYugD>W<)l&{4czSs=mN&_sr9HTP~icLJa`v61>vZu)6_$|^C+MFwK zQi+fMza7%YA2up1MFcYCv;u&<`XC{q$hf}1&-J8G=8eq2BeNqW-D*fHnx>5g2g4wb zDu8NPti#GqrL>1VXtguF_fN6Ew9~;Z)X|n7BtM&cbgkRTdh3@p=Q2rM!d1SG&a*ddlZV$^R9hnd zm%=j71eI3t0dq{Taw!#En#S!0oJFQ7Glql+3vDSX+T2x6MWM%6;_d$ZUaS9rR6O;rZMp{sw)0Paamd4{HPn6yi_~-psVlaf~51xMn4;#jXehVwBobe z9)Fm<9@U%S8KoQ4AAPbFy>s#FNqK|748^FQ$f+O;V(WFmo}{Fj)uFEIlAm=;6m)3wuBD!pS>F^pPqmLgOCId# zW5v1;O&f!6WeFpIuT`A6#T)T4x0$cfO&36JKm!}CCa@up>T_jh;cx8-JvES%#lh7E zz}U`C*iJyWy=SkcEq?@QSi6U}_jyE=D>d8QTzBQxybBrC(A5_*2u>2$os8-8v>c=6 z?6qFf_x~(E?|F9uj_qu$Uf&L$k~O}^wL?qdL5}wd$*MXDdpdaOM`q0WV$5igh-PYy z)zF}&&Yv{#%V-uC#!uFalq%OSxN;I({DKF~hMzIHl;XW|slJ5a$Hv}TlU(V%)iELp z0MZgoESf5sxfxX_fy4%kJl)Rju#Ze;fiav}ajJf17Uq#C+FZE$q{eCMSoQ69%lIUm zkYHFkq&0bpz7_|jY>nr@F1~VxLWf6HK*?txcPri>xI0tP6-x%F)nX=GZAL%0(_*JN zZNrfX(hX=yRIVFsa zU}K4AcuJ)O4`=OzN-gAtC{2WJZcR?wUDs{JvGKep`TPs`+TQYEcIQDM2^z88rpBmI z)+G#vyN0g1sN3TmoWqA2=-(t0)YTmL&WU>x`fq=GD$4CuETL^Zzbq*`U{mF4_*&V! z=!7Q^QIvR{JB`K8kuYFwJ7~rq3P=37u7Na*f<9tA9fVM#fbahQF!q&EadpeK5G(;O-8=-QC^Y9fG?AcbDMq?(XjH(m><-n(y5E?mg%Id4CwY2cuyxt5r2?&MFyg zX(k=TI|&yhZo?l|J|a)Ih{R{U6{v4T?yleZecd}hNiF#P<0sjaK+RsjOYUo3&9n0; zvx8aVq2qmTyy_~L6SoURI}O#hJq>B#P+jOBear4>a^^Tcfe8%DTO5(n%1cTXlUWXa zhWNwvFeTuM&k|#7zX#pKUEU)1#b^ouGqc%vxwQKm<@OJr&N%POc4l|HI_;%4t-aaw7dwmkt(dk_ajOZYltP!cw)3^RIt%WK>Qt?X z|3PEYR`=+Xx%xHBSB+?A&;IHcu4T8XFoo!rfi6&1=DLf_4*`M=YT;-5i_d5kYU@Vj zDH9pd11aN6@8}Xo^uuHlLH&o+ip9;`$W?a#Wf9)w`rUhAt?|gD!Cp;cf$WlwEP zCJ#@WI#2EghGarQBj_5h2rYlB>~*nbBw_H`F@X2nqAx6`Dcn3lck6q8LUi8RaG>+| zjQ*3DZpI5(j3*`aHIrGl!G|W($5QLMdWB(k8qUej23@m<=)Yj@qmEiVY{{JHvq+5e zn0{Ukhrw$Cz80y#K&=id%$en&8GVJ3XKT%Meah*ag(#>t*!HA%yorVnZ2Q@OZ+qst z_yCzrJE=Er@+JS!@K4)~Ku7)2=VO&q?RXQLAg(FoSr-jj!~BvaoOiUUa}NZY#CI-$cSM7hfZ6^=b93oV;lB6g zF8nR`&_E=n6UR=x#_&pP!2X1(aL_7eVVE~qFHtJqYR%amAC)8N(k7hH^bS6XEbBL6 zh11dk)P3*n^an4uhQYAO((BtOk>hCV)|1J;d`fn5>0LYiU_SUWV*nB4+3^N!e2XQ5;vGFq;B zr(xI32&Q$OHdS6#G(9%a<%r_>6f`Sw+9j1V&=H5iXg1tz3f(z0r9Gd)X$7XY-OW^Q zmNxwBsX8j)>q(@t+*B}+bX9C`qZ%{rEA+`PC?R}tReW6W=8>E%`^QI5breddRTc$T z6D9-m0Rg2t$Uwxd_Z@ouQw4nzuR*>xm}rx~m4RCC9dvV23bz%BsCwj!x%apBY^B1p zPCr0^6)1VG{#pvy8R~6SXL_=QzMao$dYs{{VKrhWe{tpCm%US#wBir_Je@Oi!RLlA~ua+lw<7f8a{88d1UZa8`RG z|J8D|kvC928f&SkH6BJpOH8QUiw7nkx|Xz$(gnOK`2 zr;6IXr(B21P)X-4Me61L~h=QD>Fmhh2U3NvE@c+OMz; zx&vm@ab^jalqEYK%AdTm2>@nuO+UH1n&N9pr6fzwZCEqK6%HWfxLAtV6u;@^z3Ngz zz0>QSawPNJo5^x{pdc>~RvRoXqa|JIkEub|AgbC;2csP5H2N%0RlD_1EC&0z*Zgj4 zH@UF0nuMOG?5v>chk|nPqI=6gi=XI*cQfK3YTc`$ zXGcuSCE6(@A%N1q%s?$zSJFaxufuv0vm-(Dx1G%1#5#2~_7PmRw{+$Q z#8ADHKC&0uycDJzk2BHMviG%3Hi-^jD(2t8y`Y9>Lqz^pHa^UcLD|KdX{QC-4lxEa z?_8Dyl;oiTTF^)bm3-5n5GsU!Bj{&@G8)ilAx<0T*AfH=Dl6_Q(I>L!N%sbPSvkb~ z8ggvEUV&+H*rzJ+qv)a<^7RN7Ppr`e?`{d?Ka%R{LpzIvU$9$~186t5x3Q7xh(Y|O zm48uJ_IEC%R%zOODJ)?cSVw(KqRw(ku5rri-;yBB*vvF$DYg*epE-A*VlGPF6fOXg zrN23CXSDqtlUQSG6TRQD#e&-eJH|WfX7w38MaLq3)_r4HQn;{=A};fqO;dtn1umnf z92I%@9?aXpFRIh>a_cvT*`0HGrY~k3>flRF?XLmZY7(M(Rbl=pBx7i=7jvYHHdtQX zyYx3v1A8t>q~^fU`jg?9Bh<8hNt;u3v>sk-^V5`=AEfz|3E*N)vJPvAMPLVaG=pb|@4qn$#vqq@` z1FebcBL|Omg74R=^GCnMLgJmL8x_Oku|4cs)dx*7_28$+yI{D$6);b#%|F`QZC|41 zTKl^SrC^THEohUr+ePapwl-*k9{S3EdfyQ8ZRBvh53~M!Wz3{+XOdd{mtW(PFz-*P zY0TkvDo6QLtT8v?qmrzKA+0+x_4FmeCx=_W>jGt|hQ$lMo9g`r~*D~rMqqU;CO=bFd zXm5T{CYII0L^3sG?SGFw>>Mx|gaH@O#&7sb2KSQBv zd5at$R1C3P=<_td2jbX3IFUUET+OG4&b?3ei6S`%X{EtkO<~^MXpP)xvz_08R%utB zkK@WH@5qDSaHHfDl}oLa7*`i;gmJNC!los?XB_#t%od!*u&JNAe%-97KdV?1xOH2+ ztk0!kZhnsePO3~@B#)*CP$?=(rfujc_mC6r)C?`Rr*4-SOuXD`il*mn%`?Upi9}{+ zMf)S(S{JU{>+JTYr? zf4|)9N8HS!Fi?l$%t{*vC~X=}2caOJ_#U^8qt^&1+DuaJDRr#L{gNIj;#wc>Dm&Cy zHBRzXX>&?UCd=tK4$Ok zu_dP_p7vD9U7CYHT!VfO{6ETn`Js+rCL0h#jn+n5y~fL2fy{{))hu>YOQWTT;rS*h zzSQr{N6D7?2y-ux8!64OL9`}^+KfSHmAT7T1aK@WFyQ1I{Dkhisd0{Sy1o#DS#f?m zFYE(c$Wa@w4C*2Ror4HiE@VRE0RB=v|zL>A~XDP zKCttu^QV~MYPB0r6Ku{!$v&g;QArK0b?>O3^oRA$IIucB6X&Iv_V)CZtmTe!%9D+ECY9v?q{bW;o=%t&GES%L=5YSfh78l=nZ#L` zuV8gIil$C79u(yiXoI6$Yf2p#Gb5Nq2sCx3=aO!G^a%(&mW%gCN&v^%4Q>^c%o8it zdfa{+Cc7*_1vUp%##yRJ8jsdkj*loP013=Bh~XB<<|nVc6)2>Jz}2VlIqWeE(YrGx z*)E)(Be{5|1%%{Z518w>64P1lFa$H2h7^*&<*C)&D3!rw?$rnQSpa3+ zk)U9gA5c=e&*s+H{&c?ngT|U-xs*r0|6yda%W42a$4o1rKnx zD2!2V`D*N%48u!OWH0K+SU~_q+sOlhVXO2<&#}^TvQJrm|FD5duMgnjz`(bl86ZM9 zD`)v#C9AWRQC;K%DAgUI*NMyF?NhRyMP7q&rk41j;l~)zl!Kbpv6i=2WiM3nkp~Yz z>eNe0$p`I_3TX*+mlBE!9^#F9StU7fIMQd=Hfr^{uHrasqh%@H7?&%XXa&;eDq^$` zMl{#-(KVWgxb2r2dz$;jQrerZx~p+rxur4-b`o*_wc5_g5fUQbTI!ld?FyJ6r8LJ+ zlEJ9RYu(uQ`NLdX-pRBvHj3W#*roSsmwv(X5j((RA zoz;cDB0ZPb(s1|iErEQ|G^7Y?aeoOFv~g3D;O*?ceR2Dd4r+F3W&2h+7%uwm0;@w> z(c)Wxva@2o>A=nCEWm8NL>Ene3!nOK)ZrBt73@g5ZJU9CL_hMyi4M-~9|DB06{jfq1vi|v0!HQ+ z1k#{r;$yx3^ zT@SAC=k2TUpeL1i<@b0ey+nxaPk-S6>-4bv5?IX0=D?OWea?1zm=lfRI{c;LU;58@ z6>ab3m8dgci)7X!mGc5ei}hddi{Kk|xYH0t6ISf)_JDmG&Howy#}k5Do$lSEPv`3e zzcN+i?%}6qR@y$B^~uhcF&7)}QO?>;tBYJF@lQq9&K4D#Q2LfmTFoEq7MTY;ajSeePhEv7Iqy45@Gn4X5o6h7U<@N^d^*UbYF0Hl{6psNlxiIr&3IW176R9MHQ7 zsMA`(dNR3t0T;hIv^Q_2h;=G;0yo!v9XmO1>ipin zRbD$T;>0>o#cMUnhi>})zw%Ml8i$77CexO+OZ$4f!o!=A_FHI^IWk_|UC<+FKX>{P z|GPvD^0)A8)FG%aDX)sx#0&c@yK`NPo=hZkc1?{RNYz?(QEK$lyFQ|%MEqjT#bo~w zpAoSBE;1;sHFe(&T<1)@jVzGrW&^saD8EcKeVH?yv^89c_2S)+eOEvLEvcOs z-2PtJ&JL_OrnyjWIJx(DwQ*0NZ4zcb1iBM1kGs&*H6|S9lKy1wz6j z(r+Jvybo!ZP%HZ86hrevBj_-C4p)&dgJgHR`Bq<-7|rSw)9pYJ5K%bUf1kg9mhI(W z{ToIf0dNv$O$Y3k`L*y1gdeEu5EK|lsc$SgnGs-ucXxP z3jtMUR%({$1;Sq53`63KmE*XHK!*=hVR(M~*1X1)4Q0;m5!F#I&#j{qdX@nKiufz_ z!RS=-)eV*Ez|v1fK19vn+>lMw(qm4Lotb|0U9_`a$6I~#QAmFJ4t9Ql*Yoz(fZc@i zY^ouB1#x%cFlatSIh8&$%f&}#-Y9;!SixchPG&LX;7J!s*Z-4n!tbXsSqP85^cB!P4omhOwOHpdN zT+@F`U52#7D)EUabr~0zB7bCHs4JK^$BH8ahoBNRsQJwp#ozvGvvaa_f83-;$TB-Ac&j5QY%;!1 zn1+yNpA@xRYAD11%L;ME!Pv40<5Q5s$0L?s9?jdz@C;w#euUND#QKI9;m zyIi7B9%G}|bgM$TB3+K*46HZDLD663Yk(*Wi}7y5lPf8>*a(&MEs%DJ1Axrx+$(`Qd0cdWpw_ z{~HbpGo6xW`m$yh)3m*4$8D4Bn&qWIYKMX~b3azy_ms|N#2F2&K^p6;(}XSV!8mcf zpZ)GKOQ_VpPd!Wj0wIrDYitpzWI4>?S6$U6*#G$sf#Xan}`1lO>MP%h?VdB(1{@ez^zm3TB z&*)d5)yqW$D-TBX%cltUtj_lf-m(XS8>Q28v1PM+;K)AE;!R6Yap<(NZ7NI-mSCkP zVzYB;gMdwt!?dmz&@m2eE;8BFI8oz)y%E6K>M_6CAU1(@6Zegcsre#_;uwg(k7gi% z{WrORdJjb24^4eV@vuLP^}HFO!Q!u@@yMX2`Ho+ia<`{{Wk-WEnQf3^CR?GXmnzgz ze5+3>Mcq|wg5Dy`+w~|VVlF)M_kd$8jmF?vp`qya%$%R_o&7EyF8kVl4ol?z=2^%i z7=)HK5+7$zPk3`Uq)s+k>0ed~&pEc)8m8S^Y&cm4d>y694L)sZsno3}4tS<*emafy zWRd=&nEOcl_>qqbMt@oBW4<-IRVTl;K5uZX+uv9-4&U0mQy z79kNxfwq~ECxFPxVZPqnTW*vzG)j!7S(B)6wa!k1Rzm$%f@rx` z6+3=`VQ}ARaGp&(f(ifqW&%x&Rqj7i@t@xYy+G;53;gG6fxfYr;`l*_m=;I%Bqfch zVgIDlxUg0pHC=jVph^dSZ?QJ#HH3q5PeLbMS*QQ&yo)g;*^tqVgwbu;%8`d#Ae8xx z&^iCE1y7jMo*cSYp5I-2vG<>U!xRU-`Ni&M;bB}mZDv&mt6O^wJC}cm*vjOHt(J#~ zy~iKNPV0C+WXp}Tu6*xCJ-3d2vJQpibSn!=>|DQ!7L#mOL?ZKEZv;%BSURrl_&uti zg~C*{hRDd~!B(XiKvl5*yUw@um*fUF-c!#@1J-$r~fT}xt_-FQd>@p!RnK?ek5`KHYMRk#&Fuv`t$r!cPy8HGQ2$g5|!P^ZwlC6sEu;l90i4&{V0Fz5=WGvP>Z|V0qxSp$XVoG7o zV213h&KMejYV$UvvdaU8Vst_ph)GCaKbe_5D!?(SU5zP}jda*!2X1s?NvKxhyxmAL zX&`EtSJ&!EfT#eE zdAnChva`=XqB!F|0?sHs7WAMcX~T0X7b_>S!}B|p)j@?euLS@p04aA|6y{?ABOMk% z@<$v1K8o54V71mEDE}YypwDVd$zPeA@LpHg>M&weDl}!U-tHV1Dm(jmQ+2~#x;Ilu zhMuT2eWP|P_BN=L@^VmnhAV@K*;{pKGeS z?{`~=?qoKAWen7;zi4CHYJzT*plrE|S)-XZXPCNS3oNE4RMbXqHJBgJTzv+2vIjBXUYje$4VN%iR6(oME9Q%s1yixF>KBmDCzMDw$kC4yXG$ zh>-wvVa73|D$;CIOfEO_)6)B97tZQ$Sfc(xuF!PHxDde+#@7ta)%NyCDI1S^KKmd*j&Cp;ZzLkKQPq;a*&lQ@>5_mBEc)32kVdLiNv4wURlWis)`RU+6@2>_#ptD;61`x0an}14zw7ug`}K$t%kbI6gBW@a#L@W2$e~W)Ja7q;Z2&h&%8X5WBH+gd)z8wM=tPL zDc~0tkRWw-|JKv7Zg=)2T7!`?+CPBQ?>U)!TzBN+I`O7TOROTK|28{~bN3T-sOIlo zQhk66*{A}em}b^5{cCUeJ1kY6iOL3POVahy)ArEg$35KP{wfN;Zf>+yf8JKBKZyk% z=kak^FDBNcM(2Vzuu1}TgK-a?U z;F7E95}Cu2E2}xC8CIi2appx)WxVN~e%^bU3xILb$#|KO1EByVT9>-^6$~K}QO8ed zyM&kbWpNdVT-`)2h1L4UgJ<-q^hOw>D-K?q22C*I{V^Tkx2In*$55} zvLOk7dWdwU<&>ET9ucmlq9lDSCJ|<$E_<$sHBRGmx713)qD1=7|2*+L$sqf0E`W!R z17?X+`Fg|jOGP%EMi>Emrd>zFh#-G1|&ij3GTUBhROO&hB7ZQR?V z2VLJ9|GK`haC3)ZWu-np!M8o{rWk_qc)rD zTa%{Yb}$YzMICR2xU3&!Z4D4XJo@UYu zv5P0v`3aMNZ5K#3Jd6>SkI+oJ==2T&2vAU)cHzm7zf%y40v!QGheA$s70r z4>$8Tr{^N;&TSt)QD3Cs?ryNQ!az!`${d#xT*@l1LE|xMVI5=x4xI}3r#hW?d;ABS z>grB`TGG@nInh_f?)W6_?7>?nCUlYWq1^t-9X}H{6)krVWe^OmN4Q>!X!YfFGe|W- zA|;qjndwBe0O%(qrfqHpYIC@gBIUcbs~@kQ2CJrB`d@}eShKe39 zdu$6A&Ct^s&kYCf{87_~wnwR`H>``ErflrC?K}TLIZ$9{*xB9f62Buy;Q*6_GBmCyE~*i=!O) zOh;!((;97Db8-xRGkf~kef)R9C$eEZq2wsv_r`#-o~NQ>c+aThzkgJDQzTuF1Q+n& z^bfEVF3aDF&+ApN+wtbkaLc`ayg@u=+f}?)Gy;Crxze4qLo@{M1vv=@%{@|Qv|%uK zaAFR4%CpS5>8{7aCKw!lIb+iuxVECuXvFW6S2JlA4!#~I%!KudU(Pafy}nHaI|UhS#|MNtem&&=gj;mx#xd5^dsN&!c^v8E)|d zl`vAT`HixV3R0)ZbW_1otn&0a(oXv5XE=Oki`(7|*?_Yg+p~**UMlpZwR^&ta3?1`5#ITtzp?j6k2M9J4b>)e8*3qQIUcol z*)RCZP22TxRTM0JTEBfgI_a`mIHH%q2=P`Oe4QN7<4P5G3_*Oo0Ur~u0n-&r?RkBL zNKQzWaWfL|xCmIB^q)#m20q7tuf0#9vK*-A9NLmd;Yu%0V1hOQa=f9msQV z>jb?Tf+Di`2TFGSh7z>Fdl$4Y;Fv9!8H1KOxG(MfW~Fc|dwOy@u`M06@rJim`(Kah zU~%fH7jAr_8u_7fy&pc=kZAX62+?qV_GZp}t~Yzx?spMutk0_gaiu8S{Bz#4f1<~4 zk4yi9m?&5yELHbPg8s>Ek01~NK&!Wl$4IexwM{ErL&X-dp#h|4PKCyD ze`ol7J}r1)Bh_mmUiWGtG(>E1d$S|kUo5_V${;ky_7sdXu-%(dNM$9z+LKQ0;Hj$kWKc7%5J_@@Hm_2i&FrvN_CEP?20g_?lGm( zPv2OeA*=Ykb0(;4wCY@MH$UG>E3F~;7m(||Nj`zsc0yO>xul?TO9&@ zwAqOhw(H7F>7LZlBu+*$Wu=H>Xvy6{y8NslZnMZnvHF#R1Ak9x)<5^D4&0_;EQaKv zr32a2k_K0Bob)<>=u+D1phr4b(V1&=6rZV(;Kh~zoKU`;nWJS){2v<8} zKfE`nk%6I13mppF$^q(X3+P$y;nOm;IhsNW|akl*)1ORzZyT zNH{3r2slH4@3EUYeZLdz%3i>zf=mxc7dw#V%btA0Vel%lcYFa`+wjg31hw#*^>|He zbov$wgO*=T?>XIgd2&v)4Wr7>T>VRT!bBSJK)50G(9c)FQ#_TA8&`(oqh^k2jI}5m z!9}nu?_=wVa6=pnmP}wx&#WpoCUIR^e#pthfw>wx8A8Xtf?7box*@i7EJL-j| zov*L1Zw>hCrTuVz)01({PHz+QYUr04FbXvQSARg$SZXqw41UmEt?b!aG{H62s59hCjom!+A23S~7@g0%* zzKo_VW@OLDgC$(K`s`*oW!DmB4xbB9Jwh?h~!099;9)J!kM3sXvzv#geN z7D^tk$Xd~BL{L}qDPN@d+Z5!?M>Ka$7*?(v!O5#{ppH-iBxi~EOlqvl+j3#*nUWMM zW*3epn;>UNU22XPuh+g6HZz`gr)MZ|Tw!{7QY^$vm{~{f)IlpIFpJ4H@2HVa5S8y? z5H_pA$csn9jU7!OEMRxX@pgILvvMa#q{-cGTP#aXQK??J6y{z`kx8-wMG?8@Pcq6s z4vzMq-Rq6>(q-9}= z-Ttw8g+WeP&@K>09kJg!ZEn8(@v&=etZfk`aZkm~(Uov=fXZmj;J%jFMCF`2ppZ%Z z0Idyc)FXp?iY>|RjNi$awntrmlvGL^@hs+gk!K&!6-YP&2V8EY_vKc?wSJ?gKzP@= z;S(D*GZu-baPs~}gDQ7X(%UNqKIwsp`$!QN1d~&l`Bq4WrlOvRsE%rBaJ;b$5$S+D zP^*;3g_gpupsGC2*X#SDojmUAUzE%=m9Xc9DfM=45vHykpuQcJi>Gh51g0n?v7{mE zob&h3ALj*`{oiEKH+%xN`H@l)6E8eWk!we-Sih8wmIQHa1xR}>4ioA;F=2dMHGo~ ztb28COi>g4+|Vn^K!j(Be{MwD@97Y$|4)Y(EoW9~DB$~R`h zr?B8C6`l}NzSlXtR;Q`cTe|$Quzmj5A~y3t^U}%{jRpCM3Noz4lDlBa)Cr@%CgCTD zIX?UBDyrl)$-`kD5znU!Nib2>PB&&H)H5N4Gg?YYwik*(j>MT}X@4Q+xDIC;1z~6; z4x=x@E!-0~)xJ%vc4Hv(7~g6>)v_wz)n5yfda}M^fxhH1{GvY6lYls1kH}zlhrtAs zbjd#i291C-@WQ=F_iE&?SY{I5RSvUK0uw@`%{}EZ_!A()Q zzDQ$RiL<|e@?!q0xeDT1MtBu!vGZy^^$5)Pm+&0B>LlVL~d#kuXUGD7Y^=smxpDx zG;Hctm5aMUmrawe*gr6#irsq_DaJmKh|}3{u5(*Mj~FNck=Qy&XB|);7m$lTa1>K` z*g!@5f(&<$vUM7mYkGxy<|4%^HZsp3edsy1^8mQa;)u3`z-<%r{xYY}I82Eib9Mvy zXI8-C;OhrzzoaS4?ptsk=!rvN^})s(tBIpHeR%XIFknSRYWjpl%DN;8t%o@5B{K|R8a`Z z$9KuRrb$t}NBpm0hhY5t5Yr>B+8=&r6JN1Xv;__(A@*++k6T*G$5t14+j|U>$g&b% zO7#~J_}<0YH*4xmj3ot6-bo%-oDA%(vTQhbqpFT$4!hdZqPwM1e-@XOc#i*zdpXwM zZtQR|bdvX&^nHW$pd`DG(49jy}bgnan~>PO$e4 zSl1`IpV^37rJNQ={+|)%auomKJhKD?a6OGcMR@NGrkW^qyJ;BYvr{F2tNE}Q6`J}Dk2XRSYMwon1$yBGUjY09NPF;! zKWO`{S0%r3fDAD#D&`21Wl(5K$HAMTsB<}bm^h?Lhy=K8ExTXe{1Y@x107ww~{}LB3tm6m?=DO_T zxDCUMk9*8xwc)V|);ndMbk$oes-`r?-H%M}Rgt1YN8&_nNb%iwi9+93aFjqS`VWXc zT4Y@p;f)f!=A(o9VD{X3@Z3NWxMUEXrg1F387J~adP7EYWeeB=^-Pam<4uY#vud-+ zVAID4C@JwJ3ZUO}_->pn+#Xx;<`~)Y4=EtIyGb_(q7x*b47ra`#t~u13#a-eD#>K^6WNV4xx#d=?w|< zye!=Jw9RP`7gli;m7~wC1=;>`S@EsZS&-?p!F#MOMwU>8w8mYTtUFF6?sAyld)KSo z+v+0)70!i%%wNZTIH8U@=^@MYR5yA>tCHwYy$5f;eXmKCDE3ydQ)V>u9yG)4!u)Q! zWfiqk=Moy?>sH$s_ZkIFG4^v%Qdi_#s}h{^_4XiNpP*Ow#9__j^SShU@R zz&2h(?pd7-(S-=M!OhQX)#3L^UIVF(hjc()1UTnZIXH)<45;0K=2%ppzs{=t2Zg@o z2jQLG!b_`Hp7hsDiMzb^Wk*a#G)~fr%(Wh?(gW1dq(<4OV*!MuC}97(7wu5dWjFLHjqP7 z!6*Y;?2=yB)n`<%#F=LEHh<7l;<=c}!pXb6Cos*N2hI9D6`#`OZe$8vYT>yaJ2eSf z4baW=AGD}ZiB_q=SOo?{f`rE~2!eUHU?m$z6;&}W6IAXZgDAjg&$;~>YO?UfcL-sT%`uurUFR<*QVW`Of$TraT@yNStV@t{^;bC37Bdcz zZ?#G2H(+)9o9I(RyiyvJuq?uFEUjDojW*2}F0?1cZed*SR^cTJgwjh#JZLVwBQ$EF zn7ynzW$Xsb->S&{cM2!o1AJY+KO(hWDj8e|QHk2syt~X%r#hS!JL3eopMXgKV+ZK5A#adE0^$ zM)w{}46If2creICw+M|~5xV2l9TwHSYID`_s0}qEOD)WvN9=pIW=!Q+M45k}2O8A( z30fb>*7@}(j|4St8p!yWJQ^ljj1dR zdzkZZCQ=LHV@v4Y5suV;W7nnVxwD-2J>@g7PPtAByGt?Zi@42eEkej(i80a;Ab;!b z+BqW5H`&zPk7jyfupwCAOc&64X%13@ED?~D!)_YMjyi1qt#^rh0cxLVnO-ql?P!0 z+H{s4MX@$te|-9{r(L@;PCBSL$qh_K-%gJ^z2BSds&(31*j>NHY1^p0;jdqj%PmHw zMLrVMT`W1pVIUb`NdTi>{R%xKC%n`j?RxW{P7(dWmY}PKYhWck}LTB@jw-970l@uPV=|(&jo|=tU*jB+hC2+rz$O);;q>FbcrJIzjg5(Z5bW! z+0dP=Ag)KQYsvlkoOitUx6o%@cqkw#-8)Oh`(>gOgb6i4bry&{}fxd zYrTdiJjr8wQ2^1yl~NWrsufRkSP`Va{+#k|`$Uf!k6}X(9<<#f7(I$OkZIhZP7gQA zwq`@c`y!Y5u!hi@NE()3JxTHYNG*JjHXL6`s_p=y)}mbFrW*Hfo6=-ao%IfE3RM+A z<$YTtd;E2kG4eF-aODCqn@Z1B&sHuyhzPYFXXnYKwV$y$PBQ!lXjRStz!aPlkvM+6 znjI6jJVyOdZyoQ5K2Gif`-YrVbC$#znp}frAQp$s)ImB7vsl6I^n10B`_;Sq-feU4 z@xWf;-7;p<>+P$z&-H^3ZrCm;rR(6T1p)@(#+zx@=TA+Xlkf$$&)im<#6B{exl&Fy zN9z?Cj32BJIqwTHG-4B3ETYt>Wg45<@^J_!iEc=#-Z3xE2#8QBpH(PgvrEz3x_8$H*;+yY1L!-`$}L78is9zrDr5k=;!<4B2%-e))Tz+~mz zqb9Tf4PLe_5Tqc=PECnorYRrmT?>t}tXuA z`hnh=I?W->N4V*|F!1Wbg4Ue9;=)YSQlRS2bLG#FKjPc5t@AdZl{a7)I_Y(PQi6=U zx$8j;xee``o78jp9T41~&FvZ^kT*lGFE4s~e?Cs?l-G2j14N-YOi(dqw0!vNe!qa+ zY6`B==)F?*stF*MHiV4C9@+{PrqjC~jih-$9y6Qil->sRNm*eeAX$yJ19}~wvrLYT z4ur0zE302lCpAl`M6lTs97JN9R6So>aPIB<3rH4^nby*zqfd{s*%bbVg+;b*oXb?d z)zbIx&1OQLHo`*lFN+veyedyUWFRa!FS@rjt0>)IKyOa4AAU}~l7y6d?)kn6cs#lF zsj{8cIL8gs`e1I^0ld5=lV2D>+`T>!KasU*B4fQB}1SP*l4HRv0o-43SB4oe8w>_3+T!O&I zGsJCDK>-lma>it@RTZJ4@N&)P_~7~H29Z9f#Sv`RB%X!2yFKf9j1;NxI{cr-{CH}j z0doX;OCe zJOcz=Z;yua-Z`-vX)ZK76vY;DAYLrqGw-BP2&~0IS9w5O6dXbj_Y>@Pl!mn#ocXdG2=3DUK-&IXyJZ(VLVW3bFDLrOrz5x=u(;Knc0*g3E}(Y_xp{*!v_$cA_@YM%_1@954{t4nIgaeMSrwi`^6`+rG@htnD<9!u)#52+uSb6tkzpq>5S~19ig{44~*Zt zgHbyd(`aDzW$>NvU$%k)z=z7?&-jCtrZ*TM?x8=(hX7ZM&fFFa4hQj52>`2S3;062 z;?kh#$g>8_%>IQgp5 zucYc%yQ_0ykX`jZh5NUyle_<4HpqGL`0OI;9Rl$>?T^fyr_0@#855osIW?u-gDMtc zbm;wTj`YL?d0<6A?%i8`Q_P#1A&aK-LC&7a|f#3-c+#wJo zxVyW%yCx9a-3jg#?k>Sy3U~Ly3hCnf<96TEPu*|*&@U8K3sS=$PduPLeMcsLF;re_=KsmQFMUXYf?Lla|A-B7;rw(T?tJ|QC&m$>+?+{=6cdbH)4=ZJru`{afGLL)P zX@OoDl}@mSlkHek*{POED%cr=ke}EfWRoc1VqQFDit&)c;(O?79hOox*!2X}pU?7$!guV}J@4B~8;q||HAJm2Q5*!9c zdB*t_7AR_J_ArqVj`uqzj`|fSxX<_GbdeK}b9IG^i%s2;l1ZPx#Qyz@l#$6h51b$v zkq8^lq`F8(kG-7mxj8H=5|7T>A$veyZCPD9xTI;nN3!th+zP!UiXsEY*4ASFl_Vi+ z{{GJ4I2MhNGZeyhZZ8BO3bN&a&~y!{e71!-@WY|dVX{(tn|R%7KT`k4bn5s^mOxI8 z%Vcq#;} zD&zgZ?d6ucV{DP$|8EbZl_=fp&aKjO#f}x(GtN}@>`3lZ=ewisb3KnB(RbT3P)cmx!yBaTG;SI_yagwB1#giG@jcHjkBot10mbP0pBQa+hT$ylpI-Ch zpDZ;A2h?G4ag?P523i0~=HB@HU=wedGL4LwAf%1?khOO9glGG(tG^uYv{n?`LM9M5 zgoP0H!{pf7PTUbR&r96bIK$ddJ8j~d{zjTL|54q5u-qZ1kbgzI*!N6U?GO(qiHe*R z_XZh5D@LDJp;MB78wBB6xU-y(pp9MnFkl6&yVEnNTpdMRx-NKprmo#v_fkSVkzv=a zABV&)HI5HYWgVQ9Hr+9G1>K<6ry_VYA3ggMT4deqsg3>ZLErL242s~U4%4TX{#&$e z^0)XW4kF0NE-hqw@CFv9`8Be$GX*rNp_=*3@|hOySL+w0mJN)^htn!PxjVXksSZX( zB-fwT&T0paMm4dGqM}qoJb>Pib6+=Lr}E0qI}_Z)4=i z-PTQE>+{W4w9J;;QSEvlQQDCWkkSnGarixPn)aO(gjE%N6;_jrsw@q}z4E;%5*hRT z&EM=KA>b(*jXWoORXf~fo7sEKy1*=poYcm~F{(KP9!`REWDVB2<dg_~aE$~qZ` z&9)PuIF^Bp?Ad=zlzwQ=JL3y4&dmP6-|ZiydvX;&eS{}~{DZ#jy`}q41D_y~OEd(E zfaZko`_T5cXFU)}3q%L<$LZe(_FJJ#;r~jV5Qebu|J4cq@2e9ue+QoLz$tQwZye93 zwa!6Iq>5fB756jq3OT=B{yg^ z;2(uZ75+Szqb$_BQ8p&y7PbEBob6itV#4`~7PWm0dx0b9z z>8TQA7i)B9R>{4JLz2IJ8;|b;xdz_)wNp-I?NNB@3n6tT8Z6&Ny_teR{pU_rM&7%q z*3^VcV@eLN#j`J8#1FjUG z@h%=?Yx*L`X$+q*BRCy>E_nWUIng~A)|&1@6c03{tLaixzW4ZmIN~4f`+Ez$={}={ zY(@O4rwH`O?BqBQaz|>BdZN!R2U5Eqds4M7_tRmjc>`&~Y=)Io^b&EHFNK3u)`pK90Sox7la-zs`f!AdUAUTEQ!p*<(eHxvLO;f3Dpe)4 zaBQcLf4C;izz?wp3No`Z9or$LobdkBFlzOLPeL*>S`Af)loG$sVlvMooyZeN)82QT zFSfBRC)G?BNpcx}uVjeUj{|Aglbrfu&GjAe(4KD8Y+|0O+!!qj1dV5liU) zR{krQ^hQu)ay1nYiS|IeW7LY3hDEt5_uibEcP+httIu1D`;F!g^=x|u5a2QM%WoyzfrZvfJj zMW4!N$=$b;I^M7mDv!g?OtEu$EbMM{x5VQ3YsrRyJxW#EEoMIH5OG!$jtbw#MecGhQm*~{cDjNM^nr7RU)(4E+Bfi2Agxm#TPiyzF`o&c@3OmN^ z@9Ki6#kdi}WHp8sZ46n3Y^u2QUns|rK3ZIUHJl&FbNKq1=TGzQgzXw^fgNqElpz6L z@$MBmU3nUqz6`CdXT3AMOQ+p;{!Jk#k`9RmA&D&EetEZ_GS`|QW$pcXquD-+M3C$B zx0Ym_=O2$W$PZV?zxyJr+X5j#}e z7gPLfIE^6;35Xq-!(5I^*bsk;0{GjBE2Nc~hmE$EI5d|oSsT^Lx5G8U1*D7&xwD(b|gW)d^PlpfhB7BOj=5N%qo=y46(6ki12Qi4S z(0EO;I!<}9$pq_gaU`!az~$&xWfLP~(Gp~RtVctPj^WTUQ1N>cpYBI_OoLB%Qqsvf zGArsc{gqi;^Jy?*?fXFx2%M@KCl7{ux!5sNf7YYkD@ew7`xx18p$c9wIQ?2UK@ueH z2)D`m*PW6+saVbN{skm5nz);yxNbBrPGI~D)vyv;YJrv8T7xa1yqq`;LcC_P)0Jr{gBT{+==Vdzb_4e1-T3kxLE_U3qJZ7> z?N{U1UrZGkvOevl4#n~hm!V+OYKr#%e2Au6Epp8PZqh4_)>uTy6b zit!^2cvv7#ZW$tuP?j0ogh(uYmmK5kfHZgwKm6L^>Ven+5lC7)QvbGEs9CDcuDj7G z&mgu3+2=ysfMP@`&)6}P_EXTN?wKv-d=*v*&dd5~96D-h|zfRLV-Z zkIWu|bY7@m$oAE6qW>cqIp2q96_yCV8lCzmk4L#mr=&_kc*Na*)9S*FR&vVL4a{j* z!y^$IC<_gF%7jgf0<11b2*xKEvTL?;;3vOsEc70{*uW6Eni@(1`+n_E+(bv@xLL{e ze|kHHwBn|734&%g8yROsGSwj}w)%$BmwG#1X1T}N`H-?AcHr^sz|R% zZIr`S_0Bbaau-3mU+#sdC4?~Tzj<1wITMOeQyUuO6k*bpSgnfK?B_T1*r|lpVzp7q zDDYQiV%e(W&s1iR|2)}{9Amk~#>Gl(0gWyyDrF&f*P!pEvIY)5gyy$wv|%HK*I86V z8NQ6TuJ@T2BnW{BN2a1=y>d_~SbTXe0U@~+7MZ;PaH*rdXjpPZ=Oypa;#d(7vmUy#K%FENz#)py?&Kf(s z%S)hy^w^HPB}{C$X2|ry8newp(2*DUn^4Au?^Iz@usH#gzKwtC(4(5=a(}5Ntlfv-Lp-ZrXA=`fR*~a_SAd#`iYNJ94QpZAw`6w@U6mDc zguS2LuN>Exr9@grOUdCP)3}UK#2gm`Q81q5tNX@x2|&Uw>}4( zqbpt${7#To2WLL(cS0rAaq)HjR8}ZEDM-W1ZGkS}u*)0J;CJZ#s7wd?F{6bJ?@^ko zq2*-p)OqytD5Nwm&)j_|qItq0x*^l!BT6)$Ul@cr@X4fb=m*`0He@vpdgR<%YJg2G zhUZ6*?J$J(i`)aWs*Q?Wga~ouVF@YnId>c7$TB%Ugf$~_BXLG5oE#r>TG;cy#=|`- z#`t3?X!wgsQ_f%0^FY)GSuC}2f)wnHG3Ak;7cGyZFYI9LZSqLw5yUW0VuqKa-3-|Z z0j@UGaAMPdaF}^M#rU`%t7@UU(#)zj8TBvs7F92`IBmid?%&2JT{_|Ana`yRbtHKP z@^Ql?mF*pSuLbP*uybamRV6V?xQl2zkKDA`3jpMFKfmM?gcqbZVxP55kX3QWcsa*|PPC=58${P7d);~{5h%$;)AxITA&7WpcqIHys9$D40xxKnpQ{?{h>qWPr zYn2p5Z;JR)3#B8QL9igL5&*&;W{UjBann3FP~b~imL(z4ZHg7*&r0%m4L07uxQZEk z<{T!bFY6KxnNkGfy8IgeE-^mVkS}8xhFM3zwvVvl3_L}!qpwDOdFw-cRy)QB>sqfA z0pdf(xML{YH{%d;J}2iZ-N$576hGDgg-W`<{)&%-hTToaAF1w8t5Nx&Ppc3&q)2&d z$12KTX*iX)J-1#alZ$_!8?=~FRz(r{&reCtpLE>AiB#|Sr=W_i zY?fK@Hw%sS!<7*u&i|CP!KAV3e#SIYBDiPMw3aRk4Da4ty|*lb&c~5onRIc>f3jxQ z?cJ;8a)tVvN|8~&=I+vzh4WVQ-BSH?c`IH;w|$dZ=1E!l^MoON(S+wG9qVsY@aSBl zkk%5OWx3o5X-_cxC7)(Z*1|84*-0%749)0n^O}n-23J!q`kCnEuM&})TWaGQUSq9U z)f>4As(K|QZPI^{9TEm|{knhMCiT}`PwoHxM zYbss&B71T(g+mfl)IOfSDFP zEulf}94c!3fTzql{l$wy{7V-$SJwV_msw)38d%0!2Sd30^od7^DUwMmro6VRWa?SeIX#z8~^ z%VE^EEKlV8C4E<64K8@Xcwx#qDf*&Uyz0}A3TzTm4qG%;kTcwq0W`PSaq>*94F$QU zpjZ`MUz-nwK?jewcXn~+cz>-8ar+svp{3-e3a2N|P0*HyQIoKxHr$7NyqInNR1YlS zN;ox_+0Dx_m;nKS7(pAGiEXSjJmbj&jxu1-=BXQtkvcYj6}Y|HK8}=b83ZMxMh4^4 ze(BiU5Zm;!PLVEFHL+OY)*|-RWDn3(AY&vlJ5$a-MNk#x5-3BQIptwBq;bRcNp}?` zH@*~@vv*Pys7m%NM~{;&3Z%Dyp4F$n;H>KFS-vK~8y;h+ZaVu; z=hEc)xLrXv;C(Y&zwm)=^`d<+H7*}ML#aEh{rBo157y3p0EFj>dKF-VNL%jYJCAJV z==isjy@l1@a^Xqwh(gf% z`KL>s2Rm2n#X{xFE7Wh{L_zCIX2^jo<L?9Vi?}``e#c$G# zd*{tW=+^vSIRPKsamoz`j*cEaMORWI=6Dv%n=X$pM;_u&ST4qVknz47cjx-1pad+d zw!g?m0In2;TyyD8A!n)vuoU6EyQ9jl(6PjQBF&x4_h&H=%iy# z8cKgEx_JwOtA~vx{5*BSEv%nytM1y- z^DBZ+l31>W<5QSgI=D}>`DaiGkXb`zHskb!hd{J5q$QVls5eECrr>x0$NB)T=faDP ztNGx>TB*BLkMeOQJ*9=>X9hrD8}#9slvGYo50!vme&D!W1KT*(WsH zrWT$ccs^f#f8RUxrFw~*9VM(|Sc zGd0CsOMG9T7*4a7jjGVoTCkwXEO>#psR|YS_>uh))4&0EDjfN^^qwniVCg(Vrz?mm z)#>*|Ca|Z6U8KLReb0;D_#wA!tzi#f1csJW#@UvIEiB4E+|DQe1h^DvQCta^7fdXF zAd5}23fL}N;;-ZMgc23EcendHv*qq)M4g0mukZ3Mx5nV+#JI~lkKJdIWi&N?2^1N2 zlpy|chxFXGBCm}N<98ec(o=f)_$eAQ1g9J&+!)DUNoltx=J8DXDT%83gufbpDDX5J)4fx9OJU96Ut4X!G( zApv|F)m~UUZ{5%VCIzA`je8g=U8^rL$dQZ6j?>B#YV$U4=0dEYQfEn4S3{kIh&o~t zcBjlIOj%xko~BflTDAh~SP`r}+m?S2-54qHUHB}L-HsX9Sdu1%eJWqY?*26OeCaes zNhUwB1K6H{RhLxxiYqTrO|*>m4Y>$ifd@n}i|)#x(cPfUn&BHgd2!D(!^t*U-GPstZ|n?c3fH6dG>*iQ&L=3n3#M1YgXxXmJ=ep+QdhgPTj6-@iv(5cz2^G z?JQ1#?x#^6Q0}GKeQ>PvUybevA2kw>D8fK-85(T~qnBUal2EdEwZ~nedsHVT*#1=U zR7_RWgGhfH5t*4Ygkso{!c!xXHYMoLv$c-$X$0Fk8gIr#?F?Ex zQUx3llc+Z5Ywo!lhxmEDxYGwK zGg!R+fXLC~FE<_-5nflt*VQ87N8c`gQ*cau(~M$xwz=y2$cJ52_-WNjaiGvN-ijr@ zl{Y}hlhfYZkQtA&4k=-FR_ABfT z2wHj72u;TCT5mT2YUDix_RW=5A6ykI>er9|?*|eSR!*T|5m^z0*66M5eDa(zeGc-rc7yDQT=Vl%!vo zY@xdaACN%SU6O)ht|?7;O_FNSt{R)W^ITrEEai3eX{$h!BT9#^CR@eWmMS!r>Sq%i zC=+?;uq|WWTQ+@$+C^K_l74I-J!^1=6YF0mj$|LOT<#QhJYfG@yMsdbN{(;SbT=%- z=ICC_%SI^yl0Ic&23J~{vCW03mqoQtJ7q9h_&%i3%Sp^5nbMf%`Ots(9+etI=TugN z-{JsS)E8jrsV@(;L_ygHb8Qhx?1V(H9A^JOSH6?{q^ombTiqU?LXBp{WG-u3S_sbH z35S+FF{c6~v~h+kF}Pw|^st(YlJROq>%E&L7Vrq0xRhUbY5mS)1fUgB7Kgr{ec$3Y zPJQV(B7M$|bS6h}TMMcsoj61HLB@LCAD?xv_gHg*gjLB)`2IYr_1K#J|o7eaC6S@S(J zm%S41nfIqir53DGxVCxL_k+Z2d)I!?vJR8RqV4ix9v;k=KUu>(V;+K@Lfu)hGRj6u zk`xrJ`)dV%*J}ZoNm!~~d!aARB0FR;iBG@B@=cQ`lXykVBB*PIS0rRC4*!Zq-N81l zbvUI)%4~bzB3=UzLs>x$iXoT@pxoJzJ%d1pfC24}UP zE=>dAb7#Oop&~LjP_3qSIXgvSZ1|%}g>b+3$Dfg283Y10EZv;euITVG;^?%Uj#=`C z5cy-e*sh2#`3hDSY{$!i{Dto%S$j6(d>~(#t4=4sp3F(8 zO^d577IG{2^u9a863a1yWXi`o59Sgiv^+gD()$Vx*L%euVZ5L5V`F5d3JOCMDxoJJ z`X@4_Es;D=FRY6pkp_~AY=Op%@P=@(3u9nOLBG1PSzYFewM$;p=9EMLKvZ4jv#9E% zySa@^MN{tKpV{=SKa@t)2A-J}FQi#Nj(+nHIXjC@aj7lTv!b=2tTf;3Z1jphm{5sg zrz)mAhPq|$ItTX`EcjtQmL@z@971BQMs!aX0$&8|B3_3I0)4L5%_H@Q0`>ZJ@lN@O z`W{r9!J}4Nk~(L{iY$bj9?6=1Y1tmEo6>8a2RIQx!vW7GX=PNM zq80M44WW)^sLMxUQ^W17X{vpCY{T8COg@}93h)0AjP0Ch|7a?T5|X-$Wubakj$01$ z-Dk|3=N~7A^d_r|Q6u9UI!QLGtysgBMryfGOW~B2vRpwPl>;d2F)3cOtrP9rDc;*> zLL1rg9An0RZ4Q~75Ql|yXdE-qZaUReS})hTkhrNG?fGne~vFnqs1WyFxt?@GK~&GMktaoc=sQ2)O7npk=eM~^5M3D=;bLt*YgjA#qt{BM zcC^!Ze@+Om(W6_7P1q6w9WCUrc(|WA^fb!BhF6obyFnc@b84qArO9(D_R=j?pTG{1 zGSfmJ=^D;wO>z+I<5*~Sp?&2HIf$lbvO1%LYg5adF2t{m?Fo? z-#inLo~`a+x$l}q8{{Gib<8<^K>+J5{LFK#kH~9N8I|J`-nd^Nz?mFH#)5RRO_9*K zr7l7H{yP;zfmaPeLK2e|WfF@d@P{O1`!|W3vJiWrC81qsA&w<@T;*h_$LINUUb`wl zexgZ+0IX&^T69aIE3dreK#CL8jK+m0ZA6wDs<29U>NZa1CFr(Eaoy~Cv9S~1#~rTY zY@?K{E8mx1!|SSAb8N?3*PogzP9O_O@(u$5K{wHJk(7qPr{~j@6?R=f?9DO6@%oin zS==mxK8)GSEY4{*EM6d3e*K6gaT!>c*rNmFc93J65LG2(4NK~(ez|YqR5+JWaK0hp zfHKj;ELoo=Xi!xj;=t$fb@LPUVW(Y&#HXr5>xR`%(2#~}!5BSa6&=K9vm&Yh&59Fb z_JON{fQUFywdZl^mamM>4OYc+ZjdbQfPAb$O-we*(L zgD1M&`)5voLBnm19CoyGXO=z^{B!lsatT@ert^As=T!)D=+g~Rf`y@Ad!Ou8>h7N_ z1s~^4UrKTh-bG7Ma2rqruy=ZaNL!{peBHoNluKFSC_q<`0?=`1iO=7W^0EABu_Ysw zbWc{=@%BKOQ~18I2xs~;RfESz*2OhqIWiYDQ~IecWXR(fXRRVRvp0z(SDHwqIvRnP z{lM!^^xVAQAOb0h&>8yWKpVTR1CmbJ@Szj0_suggG|<%ueC}dz5){&eSL^)oXfbB} zeDjgYL{%DmO?yr9cSFxgf$fElr-^gCFeqhQIk^-JMT#2yotK49u_~eHV5B`ILl~c} z8nF_x3Wt8&6nGf>mE}R$E2K-=`eY1Y1L+scQ}>B`v6uA~%r0|?n?MzzL&Cls3049Z zH&QL7Yq}aBT3N~~Ahq1kPnc{zx9Q*j?@aE&j{^WM4zrqI?vwsay{J?p0|d?DKXRoH zGqqvT<{*o$moB-0AtsYP?3BB(M4gnCcR}gCx?UlxN}jmQeVD=%j6qudT>)bL zbvFnC`o+5_i}I%qa#})lqRS7jB}NTX1^Z^o#oAdqV(x!-aR6Pe_0<%St`0X4ze7S6 zDxy934$c>BkGIh^D7`@Xt;R(8?K|Yf zCra~?k5s7W3HxQPYet7E)sEF63Hmgcp<%WVxc&Zg@y2ov3kadE3*FRY*YI-(Y-)ZM zT>x+B{tR>2CSDsm@4l{zAiwaebjy(5UlmRNXvm|~Q;O&a0iHzIj*fABHONF z7H`o_QI)xTZcA;lzh2lHQv2GSQ7t6MZPD5#AR(7X6<%!~{ty^qbFv>}O|VNDo>m4v zRSmeXYNVY`=qVu&>tOxV)UT8kT}Ii(9hFsR%U1fa2Ylx4pol_8)7UhoKIR}McV9!B zMnMn=eGeU8&%s{QZj(f;*+1Kaoss>Bh$wA}6tVQhivzC0X$Dk@nl6=Ag3((JZSFI0 z;hU5iWHpeZ2FdD-W3qqF`kCsoCVLKdG^XNaN3Lh+oUDF@FG6L)O4g>8BYQ)E;ahbUjjMgG+7>w)szK@QP2q-DhL&;o*MYE^s~0*QFjnh;Ke-2akdI67 z0JYIqYjS~>5&?6fgB(#Z41e8Y)pHZCXCa{D>H+H$K^7stJnx(&Kz*#ZQ)fuXNWkis z))Lpb4a*mqo2fbB2FA4BPHmd_IISOlxy(9P=8UoBnQbx#BIf#0d})9jnKG&@%(<)P zSE-&}AX_&fV9niHGIZPJuz+g^CqhNP;1QFI-S0&&W)EU-E-fJ)v*(15w6HWRk;r{pZT@CM@+boL zDy^9j;qAdz{@adt>c87NM-#>HhjTN5s&7Ig_X?exuX7;6Jg4IEB=u~Ev=fD4Fsoi2jyXkD50(>$LmVlMo_Z3;X z@Ux$6iq=>~X3nxnKMnk^lKR+cG9cCa;*q%4c}8NG!5AE&i;uN`wFzBzO8qhfec+R? zk)g>CpE4Xl&H<;^ALKh_^ll#UQrmD0Tg22co@VPf&rsGUzDdRO@tGk#u=n9lAL!$2 zFTC}WBs?OgrvTtr-HI^Ff^^UM0WiXgJ0Jr9ML)m}(LmO*GN%I`tbcK~ns$!_hlll$R5(X4h`5ah?CvQ2L~XxH;lVvI(3SHpFK zZME(h5`jhE1pn228C=e69N+4PtaQfR_1fQV;e`$#y)o?FRur>Rhwfe~MQta*8|xwl zTt02+r9mCdZt>qXI0v}7u>_4*>K~n8kJ8J(e;jHqU()#zz?|uUo~prXME@CGg=`@# zYU4teYoBna6c1ks!B&VbTYn>GYP36gAb*KF#NxVa;dpDz$qJcnNVAugjG@~B!BNI5 z6{i%sE?8DGhR)Xb32@=f?riTL?!Hxlx@I`v3AQ7W>b~6EfNWdH z=pa{pH2WN!oQ)&NOnL}8rKMrly6u+hC+;DvN1LxJ<+bn0`=Tp#Xv~QD&U02Xks4Rm zTJRf2l1qzuiWEJz_*9D0v*VQf^d=DWI7tL%ORr|jTT4K+0!BaFnbNQT;;gL!4}x~^ zor^s7vzVl>=!W!b#y90RRxkudml67x;xlwwHBens|w#Q%7**g$oqT~zUeO`w^j3jAMMU`6RZdL zJGFbJ#|{QSN_BGeoCTl=Elz z4z&f%kHHv6{|Hm2J~!#F16*7IjP;M$if1B=MExqn`dbzDmUWID)KHmv;u}<)bSFN=>I4TYe z{V*F|$91?p(Qt0D9rd#!@96N&;B`xy5L_SI5=EwC6hQoTB`DB6xZkntaw;iDLv2a{ zg)VS!MJZwI#x0URTq~)sgog|06Ddiwymo#<#MNp?$f@ZS*~H9aA?M1Hx4`yRV?B0IMbV&0GCh^ zh4Tz=81Yu{{pU+)c2x`XC(UR^6w+I^%~-##YKTjx?H$lAij5!x`r@S)p0kd4YksSA z>SFA*W3v~1z36)gT3-jQxV1HLRR>Qb1PB2ZGh(K%yQq?qlK8J%aJpU$QHzUemdV)! zcZAu#EvIQ9-4^HHavV04K59Il#G@XW-44Y+K@-}X9M;4`ADN8ZZ#sX-AC%pzA^)WD zxX^@Kt|A27Cn%IA_%5hNqKPNZ@Dsn2uPbWqyqa;wT2Q5QAbYclaVPS?)>!(PpQ$KC zOCEvT*2XQO1$|#$ZoEoRTEp}C8_NDm)(Z2L0CCG&_k&A_o#OkqQq=!^@OPI({_lKe z8p$9)zDG1;z(G^B1rf$Nw`++TE3=)QkplA5~nPE7Bu@`C5osN}?>y7EX1 zKz#DraxLQJ6?lew)e{tqn6k7%$>Sg7|WSDcZ=ShJ0)FiJnteQMBOd@ z`^)8A$@$3G1|jl1ij3;<(1Dou5jC@QB{Dv~$1K}d_yrHqVxf|xS3Ghz%d4HP-JWzN zwyJ5S9m`J-bQbyaw{K$d<@F`x`L&$|Yj95{`{L%n42sKlW%M$#0X!A9HEPq??Q>>z;ZnFTOjyakvt@eGaaojGFkU5o{?T%|o2vZjn^lrCHt1?~{*^dpg4 zgq$Qa+2c2LqHB(OoBP@yk**gC99BNXE#6`Nem}?2;e7Yba9wVv89YKqpGaW$)BN`> zET01!Xh-|$#VU{$>hH9iJ)^JScy3l5*@g#}BL_1o4ygtUfa)F=dW;*}W!~{-r;gS`QI(qM3-w z{!4Q#XgK_on(_Do_gyT)q86ob)s<*XpyLMZg<(CL4Kk@CDk{p#$`|838Yf_H8in30 zQy`6%2JtK5I3vl0S9S90)q=}|BM#(uDE-<3sr(|hM-M3i$w=j2Syfb=uMQ+_$9*wz z$M&vuN`il)BW!T`fH$X_k;C`;YB$fxmSH4IOEvFpLbZmU(eJNL8$m_zOYskSwJ%Qo z-9h>H`+|+LUUpC6Of@vuonQQzAS<(l&5IY|6@sb%kBQBw+JgIgf@+Sv>(mCGL|wzF>yHoas!ou}?>Q^txX< zUDeojrEI3g)dIoFT#3^N3|q_XZhs27`wrFD$9{NYjmoxq+f&yo8dOmSXR!bTfH@bP zzAk^%)X#SHHd99_H;GU!O)$CV+kWhyiW3u5a;OWwK1SbWQMPj{ee1CI-*)2)eka-! zD)`RCzGjv~=3&3p^&-9Fd1&+=kF2A$`ql|PoiHqA)eKJRkchqz*gSznO$}Z=(fSd_<{-5;r9Ts{stpu-o5A#f`X69%jK2{_~xGlsxftZ~+(h~uI zQ${^SeSIBh+DF~7^$+AC3^eOebxOoUqZ!Z}EXvW34{pkujEYIVfxKdCA@O$N5VfN3 zP8`G!8{IvB1fq-l;~& z&gP}sP`kSggVjuGDh`K=9;e62OzQ(Sjr9Pnm$Jm<dzuYTofA|} z`v^%wV&Z&<8G}FwAOy}lbin0!KC=<`%hqXu)K|jmDv#H2CNu>Njp3F12NC{De5m1J zwX+4etQ(fYR)N8ZiR`DR!gxAmWmQFoLrk1R94KW~)ycIb*N6@QCj>bGg2+vPp8*}c z;p0t-_rorpO-;JZQPR6ft}7SPY+GGFnHVt{`+>B(bz7I)h`~IuE69Yso!uKDy-)}0 zB-PBj_4mss1WLN;;`93!k*3#8%b91S=Jc|$dg?3szfd5}Uw0=9_#wylg4ftXY%3k` znZ#}UM5a~tly23y)Te5z&x5$74sknLNAX{iu{H0y`Q#xWBVoGG`Koaax@C03;NKH{ z1G&3(lEJaDOaiB*`Ht*Y6OIQxQ>T!!UuFaS_7I#ho5=_MUP{?AG*7^2sjx z0NOEsIk$5-Z_ z4or2P?KY>&F(rD%a!T_H2o`*}?6tb~V)I#bsuf;;tuFaEj42_5J8`U}r$4M`Ho5%* zybK`Y{(n;*tcoY&^@KyMxXpnOAd(v_=q%7Fy3I(vdKhDj`)}(Sf=&M45o-@XuaDCm?Z{A_9_~;= zuS~(RnS90N#T&B{S`8;1xtn!X3uqy(Xg7Y&7$1Vfq3k>QB&SNwX+O7yUXBDEv^_a=q1H<13hK{pCDewd!1~nTo}Hws=@mAzmV%A&@nY46S$n z#hN@K4YlHS5t`{iA@1TV8rrRf%o3^0ob0*#E zmVb}!@_59O)X}q-$`(hOsRyZ%)dJQ=RjOUK(>w&}E34{BstSM}VaxzI4 z2a}tl>36tLgVALBpwy~*kZt2{;L6Ghqh5zMfq!W!{kRvX)CaIlGF}|=>IC@f+B+*P zG`+kLk>(Of%GXRV3Yf@F<2zG ztUP~iJtq_29DQKfz#Yj^$Y+3It-Q7P>sfpIhZz!1)(7CR&!o(btz16N;z2S7g?4?k z)^SpO9bwE|se$X4y^zI=O)u*e54baABYhFdpq?k%<=YSI+UD0fn8M1fO-p?wpU(5Q zh(6HzkGX}nn}jlHE+t;dYVrnH92l@0m9nJHCp+CbnLIPwzOT!4^1fU; z%0x|5ZQnlNT;Fm>eCA-?O_V^1mN9n!QcQ!)yk~l~nNVWAsPRKW^!teAyNo31E2$SX zUCp-Sz<&9-M}@}&~|O68Z?9b0v6>p1)q!-burUm_9`hg%4W$no~ahG9V@=8$boUmqBrF5?=+ z>MN;~@#C!^^S`!uQ~m1kb&m8~&W##FC+mvDepHij7@qh4tskYrOah|Z#^X(oxhcewErn$9w;{NuAIB=$I|0=tR>TmGHJ zLH=svm-mzAp9C`)4G;CU3B8%D3)afjgU?(S{@f?IHDoZ#;67Tn$4-ThXc^V&Jz{lloP zG3eg4Ys=hgtvRPNaASreS>`XV_wK5PzdGPb;*`&K^hl^T6|DAEtiqW$RT; zqqP7F{5x_2b{Ox3f08wa1t2Z5&c8O%|MuO%tmF!*|8ei1kIT9!fZyJ~H{AG@ejl6c zx2)6g`p;KCTLMz6X?)+6i5fjnu%}-zj)NM5=q;AC6NC?}oyCO^MQQvo@|XI=Xi{2K z1#MG(YkrzK)~^$J3D!Jvni-^g(MPOfFtiplaKyBM?zJIes{iG0(Vt^8#Ner6VTb!W zldv|WMkWunppocnvvPH$e;x0e=945T153v8a%)eJ8TjJi;PO8-K{cgryCMg))3h+7>lcu2?vY8_Alj~ zVuNne*7PeJb>z|b?>o4V*dypz-yo{6B9?=NmSoZ5m-@s&NYi(;6>{uK;hS{H=4S5< z`pQ;YK{KuRrpVpbHgY5<*=toNuBAvEB7@2JRNgD8I-ZDc&|f(^?f5oi?K9pB-l#?i z#@~*zFe6%8O$JCX>7wGj5maL%Ir@vWA&BwDgWyUZHbtt@0F1kCo9TzS7@$zC?oApk zW@cFGP|??k^E=aQW0S?v zhZKumpc=~W3sUST5fM#8kxZlB*eq5L|8<909g73fj-B`c@xp&v};zIjvUZGvepnJ(d zt?6aa`8ly*Xr;YrK^Ryj2>**CmYNgT(JANZ4eolTvmRMTDD(&a_I0PD;t`qccbLC; zz%mw|B}{3$^^Bf3aWw1&Py;z3vGJ^SsE`M#Of=*xpICZy}i0qk+OhXU1Q@&v$Uu>-!uRluqQlNNlV{&*bbzFp9&*-jdF? z=9RpKPm=tJGsKOFCOkdnXWwzB%#Fke(>>mD&@vl8+HoG}ZzcT^l<#OG>?)jBwS`CqFW z_Sy$PWqvsc<4FN6t~vx~k!gdc)oE^g$U7{AkzcO*I`}f9^NogSP!?^zxu}@G z(k%F4@x+g70-g`HKkk}fR9#ZqD9^@q_Gav@N*JDRI9^I*wrnv}Dfd@y4EZNbTmO_v z8p5r_=#!G@`WdGHc&(DV7p|lPwX_^(+9{A8h7=33~+0T8DDC46)PBPrwJ(FvSJFFzxU@t z-^bziG-suxQi-*kmoQca4fR0-G?-~QeoUUkxD{yQnFZr&(`Ps92kl9ee$;{)6wpyo zTz2|rYH2Bxi_FIZ&hmOBO|P6_sd)kQvgG#$Jc02{uxDiuIDRH4xnJh^ByI5jNm3fa zAuz!@!=TZ08emrC8K*VZf8uTO1(^nWd>W%+u^l(fTjv9QJTS9#fezCqupjpiz_VIu z4PePIWTT5OGWihZVjNpV77{nz`ul{n!Zpsv$qns#V@fTxCDA_!5ZIU>N@2?>|J_^L zECr>tFO)DdtMxv|T^npum*q~!fHVKA3Uvw_{o9eZI>4xUoH{rnowXk8i)&1;E$rQg z3EbF`e{*PymgCZ0=$U<~%cllPzdmppR^4u;VMQF2Xn|pYQm|v<#`M3F^4<4ljB8l+ zMNIQ-nUZ~yU@f(kx5erJweFVk>mZ9fC{3P3qopb!|9YA=Y zf|@1m{_w1eMVBjt6S0TQg)8rWN}NueZns2ky75^)D9cDF+OiXNUxs{eDP_O_e-{^NQs6Q`(!| zXI22V0VEj6`XzTKDI9qWTzQ9XlY2}xUML?KN(M^P$sCtv6jCYnNldDYU5KFa1SVtl z82{uq&4N9wvfnvJK{oH{OyO9dTGVcTmLx|e@M6hrl4RL?&pdNWwF0~ouvPPJHBHEN z+CmBT(b_gkRvzVk5RIXf{rhgOa`iqS4P0>X!6&hY)gUdC6B9oQ;WWVL<$c0$LjzEX zXdf#i2L}4Ll5J8N-2og(jy!=X)^gpIHhnIymkZgBm`1+`iP%4XSPTYLdxLQ8`Pbm- z5GiO08bIi=?c}IiuuOg2$w>#k<%!V3M^fY5MuuN~gv_mp)NoZQW~P(Z*Pz$U?-qal z|B$ff`95XUVL}RW8Z0J94#0()eDwF<(1G8ppsoH?%@8|nm7D~a^{j9Ep82Oye%BZY z9OeqgE>>&I`B?tz(>3kwl3p*>PhVt=Y7Z?G>^?5}xiWH5)V>awT3J0JlO=Q3-^K}}nl{34Mi-W_d4BtD|Kl9NN;!p*7j;y4)zl@j-D$x`oyVC9+wsPNnA2G}& zjZSd@b0#rxQ43s+6j-oPd`$lzu%%;>c`9j@VUs`bp>{J%TYWR0R zbv`ch(F+07Ctahue)VCzAu#IBBC54D>7Wx*n|GE=>Mx$B{s)+6e+J}kOx=sjEawf% zMg-Pz7?XvoNhlUT5uYI+FS{<&(B`n+**$K4jBR3F4C7TX3kJIt*zcPULjpOlJ-LHo4sCN&`IkZEv(N1*4{Z^n&GWrU0DCY) z@KGjoj+K7{QH|eiGsB|ng00?Efd=Xv#*q9_eB4b)PmpKklINfAXOiC}rqD5g0#DKf z;aqehl>k=T?}qA5d{Ye3{L%jYudi5qvRERrh0kp|bUIzbZug@sdmzm{(3*o5)-7Ik z%Uos?65mNV^oGnkP6w${s=w}*-6AM&PoJ~5R=Zu_#Eipnf6guljJAlL!-pF_@S}2w zM}IhUx))wd=YddR9yNBReS@6%EK@~+>isn682H5Epte`?mDn4bZXfAl@eBrqn+|`3 z5QC9JGsw^Htsp?sd%h=#PLxq!kM(+d^U!oJA`BzViH#`;k#u)v^3bkf8^!mO%e;XBk_ww(Z`rpgy&ZPLQPO>ZZ=o((4P{A_0gQJzvfvisT^T;Db^| z$7Xi73Ki@>4&e|HHW2PoKuxsP9I@QUw)IOKci$9NwSw4+O_5tnZM;=h-eOOBmZWNG z&(>^23&`?|feHo+vmTMbRYe?$XzJGl% zBfaDZK-b#3JbRtvJ?O(;-z6HcK}Rq9`+U`-)?8MzWY(!}PwbDkaK_)&CnDG?OpBJS zE=UoYdUhMc^w$PNpE467k$Q4)cAK(|?*iIOWD7mD^x4-G+2N+i7^D zCw=m7&ueUw&GC{iA&7TRmLY52rmVXY6LzpMKHdSYVrc!?B(^<XdZ^*fb2;Y+i);{ej(IX?S1?`)Yfj4?d=)1{T`V7zjGvA9yg88G|rq8WQID zbcpwkyQa^72*G+rHp0rO@?c9iZkx`c1u?pfvOinM6X9pAB~25pM&u8 z(yUCSv{1b(`PR!pX}M(6^*ln$YQz1b(OZ*%dm=uM-bjdg2-#(CK}Fwuuw%4QXM(}; z(2}71yWU`hsj+qS_(JnW#1`4p&y+BjamOo?KWmtT^?^Gl1oSZs8{d{@U(=Wpk6Jet zVs!~Sd{E5X$l$K-Dk;qz{S7NDaL*#&UdrMK>9(H{9bR3t&Q|c-sIu>g3-YTL(yc@= z_k)z>&fN>>Ko;);4Oj-ylnYAQ5lx`(%G5PiPOha~-8at4dxc?*tea@`s2o-VmJ|u+ zCV23KgGG5bt0S}D)AGEGyA*VpT$3STOK$J|t}c1?=NN_Kk&^)QC5bgq_=UQK2$Xiz z855WqUNwV6Dt!o?FD`rgbHkZNvwG+>x+vBdhcUUw@Al;r|DZCntR!sLkVR&OTD~SY zXuU;L2d=h9bnQ*AEvTv5EXNWslyI45)6=E>Hl+(?oE#|iDYFU%(;3WS%9;K#QrP^W ziD-8GJnJi0g~xu2vx7Yt2A_wjdCh7-Lh_(cG>95fjlGzIAeekId=#xKM`!ZxtqF_6{@Wx&T#@bXANm$Lj_Od6 zORKZj?vZmO@8x(sqvs~A^-qNpX@5#d`R*t1 z#J{S4Ia-&8!1rQLGh`7YOQSgy0T5Pt`v6UP4l`=A|ODXb(Z(e zKaGQh8*=}xqHCj;F!sXevw5fJZ=_&j0ab{UR#Di)HTFP^ag~rc!onj-dL(M@MsT8<)1WeVd!9k~)u6{IP7?1GqDa#ycS+!0+dGf6W9$ zY%}axuFflYJX4r@pT3(>TAzw~rw7+pVyS4N0`d^^`zuRnr{Vy=z7sVWe3q{(iQT{Z zd<5!KLRC955A+_>iKw)0RjvtuGpmw1F2ffSN)%unmL}`Y6xcAAhbc8M_0gHd8!k0l zDoCnQzM9QjNm9zJfneg3SOTn;Ec}g+-*@OQciz5`DN0~N3u5P7*B$xUQ^oD&SAJWo zD?}`|UiNgm6(D}W1mQ()Zu$mM6}YYwG6sLDm$awqR66nvgJdf^>6=ov+QcxgSzdAa z@UnyaBpw|#DtUswOX{U2w*vkhBPeyOBxw#ylQ)(ta-TX6d-?g!z zKc>b4l8t!=V1^Xzt!PAcdAe&ktbDI)MWF|~aea_FO2EhMl9ZN#-=mr3m-VZtb7jIA z0*}(UIQa#4%~&w-#J$V&#cjiVq{raBCb|k2$O#ZD!UYq4Tq&5Gb=2#R zk5%}R3oW>UlvunEaam{2ffWP;Sfi7UYeIVP`IxrVDsPtR&eGGLVk+XD47}vpHAm)8 zpyqC33$R?KXQQ%l^W7;qhwE-Ax3`P2>J6E=+8&xR9Zw}xZQ@gf38SNJ0yk?tFC2NV zZxYOkWD1rSCAkW=OI>3+j*y+$bOIxzI$0br3Imi7$_b;YVNTN=Kiq(NE<>WX9fOJW zUL8@|hW2{V8{_6^_e+QjryYsoBr$Rc+HTzuPja^0h-$)5n0QLDJEwl?+3*Z5qg8D| z6o-UAY|U#HWH+HlYa9<25N+(=S$9tI24*3f@87q=;FxfFp`PNPA=5d&g4)E~1Krjf z_ZAk!NhP^M0o6zglDcmo0b&8WhdWMBIm;&zEl!FC4;a+z5&c(YeVm^yU8xuFZk^|( zVO2C(On`0U)+Y`!WCFtWU-xr~vd-ps7HvHK8^u#Cp9rMhB47sW33|@yb(YiM#~Z%s z?r|L-Zu@d(HXUGn!U^SY!R(#MJlK|3D7Tub`cQ`Wg_JS$9YMghw~H43MF}=6sgsMp z+2-^Xr6j_FUzrT43<6R|jFwnD$w@Ta(luq_qpUY4DN2SujL7H8lgea+a=TM*0oaKY zF)x~cQ~~s1iw9zJ*eDo}iZR)bQIDG|EJ@_6&p$e!wt|;XgOSN*2^H|?O;Gzigu4yS z4tG!usjzUgmZOFB>6=3ZPAr05IAQZ>n9J+p z?|P>a-+7m1-sIeQ-KfN zP+PeT%)BjnCuro@by=ua?W&!hT49){vPv4W)jkVT6T-T+vOjo6l zo0WVk%H5!^JJo|9D8UY_wLOC)ZaK1ARu^x~tvcnb)N~X+*!GneL8GI3Gso?!qxWHR z=ys$SJ?UH+K5eDLc)b_KbZNOldQA8x8ox);) z)Utd;LUT1V9ir-Jk`-I$G~=kJib!tGMFH96EkX9Ag1_6_8i(i0$_UsQytokxU2d&@ z%`jj{Sm~7@Xd$w7mLsgdS*1PWHafdHUaZ_XFmq8DKFv66GKA-&n%U_3-TC4)JBs4) zA=IiV0&j^KwLaFJE1;nthS4rQ1STW2h1L8giMl-Ip=S1^Js%4q?SIz-;0*O@Jld)A z@lOrF7nRMR4$>5uo{vxC!8kvWsds34wHzJeqT~!NSe8Bp4A}j+v$}UYTjJmGej1$!ys|ANRQ|s2rzg3fj2l+#5#b%^ zQ4)2drwxI9VV0HSJB;bwc5M|<_x`+a^`2ojg-KJjuFN%jJ#?)4dzzj zL5U3a$F`U8InEm@o-72|Ksu%>hMAx0Zk6Fhlf;ayzkkK+@XhYlKA*(F@FP2k^7kfH zCQJ{%mkd}aQDYB7%zwthISp%`5m;Re@HG9XYwiB|VJUyhn-ue^mr=3!<;r%amvPVx zxzAAKCs$VwJ_Z^TwUH8{h`HPdl%dsYTy}@}aIiJIaO~{{1;Np;vC-Cu>l0NaNtmhm zmbv&11@h1KsQGR72CPf+$eT>=$}1E+_5Md=ET-&H>KZcBjMPeHuBI%ip>Xt++!tr) zVm2I+-@!}HFhj$|jMZ_K3O1dh1wELL-y>Y7-$+VQB% zLx)BN!%n%S;tvy|OJ@4mh?M-<8QvkQQaiLm_DsK@c!w3c6@0oUv2I{z$-8RHKJkrq z0pxV_)v5s9C`n1BCXL(DDBE>?M$LDUNmUWvggWzLa&b`of^SOAjJ{XgZI-(02Bh)8 zjUy@|-ii-u&NkJw7+ge`3i>|nV5De9s5d+f*oR_V=?mGshlF9ID>kN(C{FWI@3bEc zr?kL`_GNjRUJA@kOERv~IlWa}0ySv1TC|~>K%Km-WIcb_O+n(bl11=8;~cERN6bi| z&Ye4h#Y~FgG>DbxdA76AP?a)?x zQ6mh#2WXk>4rM64-QCJOqi~@jl@Y~9d|cn_eIdq zQh#^twwuEc9bE)m>zS#3HV1{)1y{vzY$`F&+s_2pjFBwD-dvowX3zNe7Zm~1lf;)u z+X}geJ^fc?*6H=XX{YA~ucULVo4kuvN8^1pFqcd}{|HmTWImye)8Z}tQ>tJ7TI;^_ z%~+7Nu@{b~wWeXsPioPr5+Q5`(keeC+z&||)ZDx}tK~fVw28`|a(A~F0}#b6Ci?R# zOjb;zmi-CV;H62;w6Uir2K?pP_#<@L%23p_T*7? z0n2AILVnYYW`F50pnApX!*ulVz~(6I+n5ZIT@Oher5iEI0MsY}XXk^2^CQ{d5Cv6f zhHkdJQlb39mjfYwLJX<1K}%KMX$uMxrEaXyf~plzESsGi;1tuN3szCY?8%}Ox9ets~a}P_-`|7^*0#`K++UB zloUu<17@#J<>|%+XRvivobwgkbLK=YCZOFX11xmwW_HyiX>Amk{Y|`g$2*&sM9F{Vj%WWzwG~ zQKFPz6pw(3Dz^YOo+GNvOoQnG;pYJDoJ_<}eLglO4jAc7Q1hlfO?MnvYSj*&_EEO$ z*+RdlqFx4G*uz=Y-Uq?Je2+vKAZQ}@tW_u5*Y}%~v2mwuK0%}Gmh(^C8rcSSZIm8h zQO|>*0}%xlG6eeS5hq>fUmV~$jNbj^E&^207`_nyDDttcT=NC88Un%1f$9#pUD0NY zKa}eOQuJTHrIlQ^aXl@6akTzX?N_ElO7yt9)I($amj?|{|MR=7WSGmBgNY zsz}vWR|3m|&?qHJwrBXg5Tzkj8Td`Jd5*K|yrhvo2PW9q%=vI@LTno0P~Dzn)(KF= zE9(6&`Pc;oof<$WQQ2RQcQ}L;f73f$3a$Q`JU^~U{AkClEG#Od4+mE6`K(wUX>ekj zt3i>IH>xk0`8Lp@u9(Xaf2Z20^&s-7Bz*tD{DlK({XlQ0b>T9ds@H6*RD+KJg(pA)a1%WlKg zhm#gLWBixm*X8CdAG!fq+nfcvzZFD<@^5IONQ}&c@0Gh z1Oee9vdR`w_@23Q$$`|oE-&>0=bvIsAfUoG>Y~kctJqMwOz4#oT$VrZboikNOh+Ua z{qyFccSXwI=URemVvCsrMl#NTbMM20nJhC%AW({&XD~jMrkSFw!iPXP>P+8*;6qSQ zwgTFqrIp>U+x~9g{uy;H;AsKjr!S(uJ=-q{hYZn%~h63%bU~pt`Wpywi zx~IdQP%+dM13c{=%>1G%%V5O9cIv6X{Cb-yI+WM`)Ca1EXPUX6>Ux{!4Eux=s1Bpn zw~70~`)B+Y?O}M}vfPQSmDkES4zB20z+r4BSZq@zzta6Xsh)vK?B&3Fg1dxk05IHF zHljB+0cU@OR8CVlE<(PG{Z+E3c6F`ZozUun+%@~gXT>Y3WvkTMYaim6(IAzpdXaR- zZ)4uzhDLgiL*>i0$@$&QRNSFDW7gt?v=(449qf&2$+q<({ zo&KL@IRoGZKQ7UHQ=VF|OQQGSlZ~XwrSJRQGy1Fqj0$pOkqCPu55Q@93S3izpop6B z_Mi~h5pXF{LB#c#KO{Y~r8OLnFworV*umO6PDE7O*xvRy}Pw4K+;YZXp=sRGPk z%(S~Xl}giUz4rB>1-u+(bafZq-qn@PA%jw%6&${!aGnhAfmxWcE2kjd>Q-xs*YnUH zsL;c(S!wrs8=&ZFWd@%>$8HH0Dkn2E3SD`xQrF}{S?cgaAh*^0U;!f*imF{aN;7WB zINZjCFjnVUpE`IvQNLXpuC}Kn2d<1HG_!gVQ4Zu3o-BIvnlSGy zRJ$48)8n3aBJr+7*NXgwBbUhu1 zsj#=a;O9}x6yttZ1a^r}E{#+ykjtFQ9+RPUzqofT6W%DJVM83mh9vq{u%~0RTAuPP z?T-VeLT?L?_)u1pq*YarTdqNxD6G*utCFXSl*2NWbI`3&Y_K5H2^i??33rIObmF=> z0_Ku1l4uT6%K>4Hl#xd_R5gc+)n30?Kb~i|zU!e~TXPV!XhuT&l$E{*8ouCeiJ?FR zMnn`6&CvZ&cRs^ji{NIqma}qA?^U`rw!>=^KsTL21CKZjKuGmhBd_F4g%H}N+?2P7 zu2RC6q{_T04FwZsjD$&4>a!ZN(L!y z?@u0QN(WKs@dbq->7}S#FbO4Z!f19+@AkHj{MA8hxk)G^~hChV(IVx3j;2pnJl}gvyCX-3SfLmF19WjeD*J=qUpA zeA_5|DFH8?E8_!Nls6B&!T?I@q~0yDKk@mYPKrV2aIqQ!kN}P8?5CmX0%dj%pcu#kkzb0dSr(wLAoO)l)DQIudJ8htUE%HLU|iZSqww{nwujv?VOH-Bx%=#K0$a%W9@b-8h!xnu=J?GlCRFcA zpB08yPhr&&ZA)EWbeK4w8H{X9mv`l7bj)a42udG$6rP$_E_ja-9wZ}JFqjya<0o#C`O)pzZ$i*Gzx~Jsk{hO}4ipPc#QfAZ}Y)!q( z7hGV1)_0C^adDMtx4NVMpi(Kb?aa4X0jHfh33FeL)rCrZyg=RFJdMzps=HcuKRZ`8$u*!y)rn(X1zMbVDji&w+Es5N z4HrbG3N;@QLa<9I{K4~kR+KI#h(t`R(bFUWcCmAy)dhe}4U`|OV}I`wrTRGN8I7s3 z1LEmYDFvJPge*y*N~%B_=*z;eE`myu+E2Ob0Rk zA-2Md%HY}DMy$L;_Do&uf^h6mSJf)=$u;5FrC-vL^hV8G*Zg`masJBoL-!pO>HCmD4qdIPkAG2 zH>pGSAf3wVuM}-h7d|%(?$`_!4l>mrt^pUm#q29*(#_!Hx!Q=vhQe(t&!8Ka(^cuj z3`J~H$dB+hy(`#Dwi(D&SXWZ@hWiH^tz%2_r?dnV`OfFsmq_BHK<^Rtj8Ki3EY*2> zlG*IYN;7tQu#cD7gxw~ZCTw2OBY%6aRBa!s+zy|O*~)ZEW#Rf%(w*9vFrp)2-zQbQ zT}3g;N+Yx$95{dtaabu?_okpXouYX;7bSW&dvDwm^S9LTr$0!lh*@q87QSk#KwiA75Ezgfx-05A9X)&q5nA{AK`RfX?EO%FP_j_}A z&I%tIVpd}BVEn>+iB6igd*DgCOjWT+W+l71A+c03qiS<{WTfuoD8KmL_uCs|fKn-8 z$(HCzz9Mv}iH^2va7uuIU?GCq z)OW$MnEmoKB_BtbiPruGe@Q1S5^_ver1hM+En);SGo6f$CHW*m66!wIkR_IdJf@^B zax1?mJ9wHh>K7=t6!IZjw&rI9&e)G9++@+rCX}hwTFXC{&7VmJ@92z$+IwAeW#I5V zde0v7`XAqmXQNn2^~Pa{8mepb1CKAjF7mI(Pjg4kf4;}G!WY56BZ#s) zwf)BHU>(HR0WNCU%~*E1IQywEcMGBD;}Y_%yK#38c2fSqMRa%A>WNbN@7(zNRsZ~X zu6aqyXSXxo>&bTEDSK*dYD$Ui?H`!Gpj^XyWn1OpcGc@*-W^MIRak08_j}U;kYBoq z!=UK)D^>G^QM#8NvB{Wc|K1YY+H{>xGApuH6pXd2m)<7T^-H;IOOtzSl{HIw`@=hl z7!-<^SIgKXIQXdSC%S9T%gzYodN)9IystYpTcBI2mt05#9>Fu%?JD;Cwk?XTeb`_y zgUoKDs~5aFdIHduMaJ*n!MZW0&R52!c)mSVoBVkYCCkXQYVntYwKHkFa}Aenf8@Sv zaFg?XCQN#}Y1b?)6Iin0+KaN>vws*^s#+_ar5d4~^8Uqkk})|YZP%x6{|>_B%%d?| zy|JeEPp;P2`me12(=q?_4)gWnvg`MDUWp)&m4}-jom`y8#BWS@GFBL4Kh~a+Y%&N^ zK?n2QDKycKd&m!SZQRa@$q)IM8nsbBcwTIs?GrvAUgX4HKN358-uhUsr1TB6Y5eFXe*_`4pz~@`v+3R&jgV!4V?@;VTrxW(goMxk*56ebmrnBXCeQJFCI}()d}{U#>2GHz zN5S(-17>ZJsht@Fx-?Sfwl2cftL>VabCOlBGBw|zU%DsU8+-1yF1nU?<^D5yB(6>H z7H+r7@xzlz3zu6$cO~-u2hOWLX-3*VjfOE{TLR z4wlaJK#mQGcdH(lKcZ$mJHSJ0WRzw+DrGh6ed|`FusFP*+C9uk1J<_LI6)#~Fp&2v z3M6hvJ}5RTqg0Z=M_prj5SFHqLS5)T3uc&f{mi9S7=GS}upBI9E6ckMCiT9s>`EO; zEo_k33#+<6C4K=F#%n!Xkg7KC)wIpcVK8noo)1(q6zDB!6N*etzj zR9$#%SVX?;1K!V-I7|gWJI@I0HFvAKED%kAHKPus2bI)8i-iOLx6>QQNMuif}SzgDZe9bT+bzJ9bf z1}-1#(dQWZGoSd;&GFOHYg5+D&c;kGZj>l@FIc=hyC^<9Ok({o-hzR5z8M+JBfh&e zW5c+)VCVGo(zGryIO-$Q7fDkZ$sf$wTb@stFP*lX733{^yuYW`bdAWx1)8k2P?1h= z;m*0LS8T3B;+q#4JAX3zl3``$eHb%VXW4U>=zINa)L|7SN;6>e^mL_`ZU_C619@<= zdT&n)moytmayuhK+4yka0qA0Ik^`O&Iz>+_>S}}WREb$>G>OOd58|54cPkeSi@dpe zU40}GWG2&91ayBdYka=@z>uK+p9K}zF|4iePj}d&W$dED;MXt~L>k-VwMKv?Qo-#2 zg*+^3>vZDt57hn$YBA4%FwDV5&0KlQN8~h+MPXT4o))JG0X{p{-0jJO%PQ}Kz6vPK zyDV2??&+S`0R}<`b6ZYLuJgscT?3>2pvR{|brjO}(2(IYB~pr(^ZAah!f=P&=}p_` z%AYLp=E^77ppOI~tjepog>d^km2^J!zbW*zJG>3VQ>mn^lz-xN1Wdd&+dPw>T3Qd* z-aZ$V*8QX@oh^YB( zPtf*o0k`9w+Zuu!Lh?dZooss-*t^K%fG{8OB80tD)6{?K&z=>--Ig5)MHLl?s(bdM zxst!D49NLd$jHcOr3PB1T(h$<4HY7^m`a8lb%^feX?#O_33V`;vtHir;{Dsxu4k7w zYHv7)1OvfK3cdzg*2VrGV4tYiPm97mZg$wCea_#Xm2}fqaJNpB9;gvHC2{ZpYs&BN+O*%y0;k249VgNz) zfy)afBmb;ps2CmsLqZK#)R{zl+|&SY6Giiml{ualM=Dq03=Jd^$7hq_$7D^xgf^uw z8_6rkeqj08fKJ`l?hpmWTp>scba;}EBYcAS+H4Ftevv51Ugt!eIE0TI_@{76VfpP@ z0a3e{i#^>2`8ljus$^>4xv8#S-9D?+qAY4fbO)3CfW4UT1XQ2DIc4E6q9#GRK3&U= zWvx&CZ$107fwFuoBLDiSnyq$pOHSs3X_4VD(UjO1buz+l8#*MRqZn6Mf#R3kj_br^nbqbA?*Bduh$KaT>q=Ix626X$G>n%Js%Q zu!dMvQ`UdGZ-2&Ql|mGGC+ncbvx%*|Q(U@M|7q@l=WWb0h&_4Ru0s>f`9S@EepJedRb0{drPVaMZZhtV|d?mLsVtv7CzzV=(Ajm0+ZZjrW* zyqhzJ#>3lH^6I#F*7jfcd0QghsMrzBeg1%m!?VghU(jr`Wovls6nE)j{U;Z7-4VR7aR>gk=lSw0RXJm&vKvMO;pZ(0=eFG_rA0>&C#WE z{ppPnOy#YLK)So>K51)9LN04Fuv7AJfdvpx`HOX%klG~wa{8=7+hgAcO5k4M{plor zn9Tno!@pON5B;ZO*c^{Wj~{|$?J85l@il^R9ud|EUZA%o%0s;DP)>1>mqE(_Z=(&e zV{yA4i{tUDfB=k~me$BdITA}sDudOneoK=hqkDHbI(buLoYfm!E@ynd1!`7-<;wLTw^A~h(cf&! zq{8cob=BL#O(STALL5Ma4_en^ZM+_VsjbO9k!d}zK8CA#l#*pCfd|EsjEut&hUk=B z6~?TvbS4I3-}_7a%xSxH+aR-on-lIb-5@lnIsQCcDnzuqTz*kMyKsHArwBOmaS`1t zVoco%HScxGWQ%W>fY68$F&MxugKP!hVj*K{Gf8D_T$@Tp9nwGIr2jp-go>GYlJAs) zcJJGgLppq1i^>(5tTqW#JllWYvtOA=TVCIF*}x^;9TMVFzm2svL(rpbvoy6{?Tu9e zLk=8aG2nMFLJL{(+8yu2YhrY2dcJzoK6f`7(L-@%Jw}VekqC3z9%OzyJBEu3JK%IsP0GIoLuz6}wl)Wtgf1jRIPtET znd`yXl;HF<{alb7rPO^zLS@trY)Av$;(QN!dbD#@lARiQw*LqK=N!X1&FA!&a9|K^ z0VvUhv`s=Ie3`QfbJm7NH7b#J?$wyFjSaVD~0;cgooYreT1*v2t?k8r2IJ%_%CFH4Xo}WWA0}+mY(Rq zLQ&k=G-5xSNeJ%Bf*!E`#&P96lz|XYnyJ5B^~0t6RiJBH=nxxs281a{J|RDCKX@ zU2F#@+y~?kfl~mxQ*&Zu2(p9TR&-M|vw0)Z_3mnPYr*ihjmfeYvbuj$djbN^CYluW zzmnVHljbh~&VD2lV2cv+9rP{8zE5}BfCg!oZ?*QzO3swv4qJn)am*#GwbedqjLRD1Oqp^wcG{m)M~ ze~%m;CT6$~)K&%>L4cKKqo#Mvm1y09*P5rT)5_V6Pn(kW?(*nuH}|OR!s0$-qVs~t zAKU*vPkR16PjWorwu?&F{_i{f*VXdLc3IY^&B!KsrP04}Aa$*U*(?a#+NQzVx0zx^ z9~EB(6U-tRmh?=y;z3&^oHT+0JLxp7g4eQa7QO$M!MeRz>jVL0yM}foa+kMOb4$A+ zI&uw=X;o5dU*x^w4`bkKrYP8*%zcJLT5S(^MK~h_oJQg({W)YaytoNvTvxz%A2EYrUM~v}a;n$|0qxw0S?Jig zR6txU1|ZI&XQ9+4%73=**-;!QS=@}Q)@C+nHQm*};9Ua!KSbc44J7~c#ee=hut4)V z#r!4gZeqxt6M_YH^VpwPtapq7Vs35J^#i0``+(BQlMMMu7@EjIyatvm?m2>ybzq1< zPDs~tM#tXgzsZPF8@YM;#pR$Cq5P=6mvi5p$0QmU!eR`KbP6MsXNf%(|o~RGmdU(OJX`R*= z%7_Io^*Kkf8`h*zF=>pS{S{-XnMU4-`OC~jPTJ&6zf%#3O=S1md7`nxmj*ZF8m(h= z3=w{v^4VkWE2HFdGEAGa+!fw;tsZmaaV2kXdV2fZ*tT5z(jQCj#0>ekZ_AtBHUD|) z591v55%(|H>O7|735XhLpo|K?^U^=UQw0b*?mh=#rx&|fo5PWz*JMqJtJ6omhW6D=U!FkA?Ly{X|e}o;o+wYMlx#epJ@Osw{za}X-L~+eAT0#D!nd8 zfBA~CaE7d?#0u{_)uBMx(nfvpYwfmr>1Zjyz-8w##}Ei7 z$~#EIb63wwfaWL%|KRF|@XQgo@3v3}Ix8WfaHcGm!TPzXz|WFL?{G|AM+o!sOI*^G zHRYX`V?!K9bgdi1(3(`g>bp~4hG)AjF#=>*kv+@4Uk{&$5=rM${n4%`rjjnYrmpX% z{2~ z8UN&HdnORZEi$uzu%H^K#pR+fJE4NLkxcKSsqEF5H?e226vsc%b1{Sr2>MNY(;ZCe zZJ@MZP&Nw@TO1tJyFoFN&SkI@!o&WWY3h&7;zUa)1eN1d! z9HNujQ{b)6-WKM9nqOt!j*Uj6cR}U)@*rC?Dr-G#tZ^x@DoK^O+?&WSkP$on*0i0` zHPiFzO%&7|v0H02g}s^oZYG>4N4v54fzfc$l1wNtdrm-F{<_w|&xD??y)p)N1B9k1 z2~%HTJfP2t=eYhnyz z=AOJKW{H3qgHQ4gjU;8agvivCd9{C1_4=%ti9>KOGlE(EG~S5(&x7GR&NqgXM!7ky z6m7~J3I`S@7wAtJpBA@`;|VqpywKXj25X$sO<@@Ilo1H9?SyQK4Q&NbV12%=FD0A( z7X{bF0kCc!WcOTOuiXxBmI??7XQ5gI!F70hj9Y75JR&HIdc^VGzCf_Pil#qUa9; zY&DA&%U(E?$GM|>j>gRXsNW-?UKO3Yi3R1pw^bf@{INNPalSd6LxIdI+BKa!+-j=UXsc8H2HF^M+lC!;Q?oP4;?(Gm|g z^2YGK=0NCIS8+E_FA8%=V|hzg!OH^mVBPxa-dB|Lsp!~iQdAjk$>-&RVb`J`4T!?) zgo2aHrt_@v`!u3*o{-}yN*w~LlVZ;av);6bL<($`C1Yy?l*&MUPxTo^`%tN)*&zR1 z*bI|kT2Sd^QyG10*RF>w&!8*=3m^)ZI->X-+id!fi^Obk{Tgu4A?9&st1O0-JxL3i z`23j{+&{#lzxiv$H2piVUZ3k_A1!G<8gv!R9bjeww&JGQDMWD1VCRW6SI`~}$&gA-C0=uM1*la9NB;R&@@~b_4Kza)K zQ@Y0qhu^}RSM>%(VFoy%Uni=Qt93!UnEBCi_PCMtb(1HL1~xe2OvZAamRz_9yw|Z# zM_xce!wTQ;lfZQ2insw|ZwOGbLPn`2fK9MFn5f}AZTJL9vEP*+o9dn9nG+(oG>1Z% zo5RYsI#W%E%CjK)8ckfqMnjry{Ac(mXnZ=5j+EN~N)T{w#t@emp*6FZt)lcIT;OO{ z<6yn-BjQg(cl-=5{zRPCcyoL3NeaS1`GkbYjf-EMM$e%fSALo^#t6GvjrI4&c^aDr z^y%vaY03O|#16(RFJY2@pJi;1&|xT>}IUZoxgcyB=JFTX1)GcXxN^;O_2B=lkxRng7mI z0d?p)aH@Ony?XW9>v`XhQj%UadDyTEB*PK)1AL|gif;|cv3i1&z_0=>=j6FD77S5u zy-IU0tn-zbKU=@5)=|gTI8f#o9DltEv5yNwCcs1W6465T6S4GS$(ueFY`5iH`FKDE#iM;}<^LvcYXI^V(=<;CAaBJ~6sBL7 znqwQg-O6hN&Hh?BN5(GO0p#rt3G-j_);&0&;^E1BnTfE?CgF)}IipVZwAv{*vV}MG zx-}3_1B;`pb=+CWQe5{mGQh$Ebixlht5w0P55mhPh<3B!xXpQ-reIMxwQNI5Y_BCt zlS%lnlDnpq94>l6G&&TzN;2z)b?Ut(wiYWLojtDy^jY?NVk`aqb)b4}uR-&~+6bAS zbs7zWkW=!ph(n-hwwY|TIeB}#H0NY!XQJ%WW_Jz6=p2ms4YNDZpcwT+P5o~mg%LdJ z-#Rlqa9xbH zR6Vl*wPzl{}D z?ZUbos`>uG;q$5S{!6mA{VSRO4Yw_$p_RV72+m{Y#(jgyd4x0WYSe5?B^5#U#g|K_ z=>Z0o=$FEi;q?kSY^RexW5{0aKbgem^`wXkk(z#kky3rP=8~gfee0Y4Z-SZ}F^zFE z$52SBW~$X!NW7Vu`>V(4ff(tsuzWxJu<+yum_9{T0z2tR*06jzFvW>2CuU-!1HI0N zNIO{kllFU%t@FtJ`OV<(uwW}PGb7#|CE2iMF)6XH{W@LdblC|0;jK{TlMonWwo`yQ zE`Z3~c=pr6=;!1_R<^&tYGk6JYA}V~FKunT!fJf#kG!7OSO)iLngyZGXHv9`M~=ah98 zId_kcD=7Es3niE>_M`H{s{%2#z@Y^%CJd+eqPd zq8}qE=58X=zxJV#=}iE+vr2mpnOD;fub(QI=VzLjtS`Q^nv-0#6A^Qr@`T<+==U}78>iB4kAU8_cl?MWVu_0R zB$lBSsGW0i0BtS2Wp224$@hWaf+U1#qrP0ys6m@K-T}yLH{&mSPCt~tczm9`nwosp zC64KKprE*qP#f3|lybYbCHabJD2Pg17Vfy#U_7bn#B8_l{g>8i*!Vq(R>?_$L$lCi zoY3nB#>mPu{p_G5X}3RC)lc@QZs9pOdz1R4Xm}4KkPDa1EyDpa3Ra8?*>@_RGTx?@&$E9}|B5p$E$4-D`u;?mx7L zk_dI1R8C1B8RxK+@HFkC$HyIt`Y;LCqEuNmWNn1Xt<(9|>4}Q0Y;A3A7&h zT=fx{FS~k0E5IB}O7-hBmbr2N(^fR^T62kKAG+$iR3f#)adm5ReS$IarzQ)}N@wzC zTdh--oTb z+i#4-u%1$O4w)iw1TVh2d(XxKzOGjzc!>!`C+_5{74lFmC~ZI(Hvv7am}d8_Vq z1#~l}5%a#8`=P}TdO}u@yj56P< zE4|x<#uohxHz0%1?(yx=vnt$};`Rq7ODItcE>mpo=hLJK_LGf6TekoX!*?zfJ+=(; z-BZIaPKCLOQ&q)aJNbn=u-`+nrumvHgxyQSH`gp`DczypDse=oW7Xw7Gk?5O?b@^S z_GcEqX~lN^(VZMwG0d!X>c;Drx`k)OLd`hX6^(4C=e0y0Ugpsu(d$yID}CWMqMHYG zSCe=0dcBcMT}po1J=0T|+J_y~nP*!gcze>Mz}tPk5CAuGyD1A@2+PVCRP!u{ z(oLY2X06}IYTM62A@!R^Qk4_xvfVj?KSa1W8WIS@Xt+IQp2_-;?@Iys!@#(hk?ISa zT?&R|ka^D#_zHRkzEWwQe|J5WA2i2kRgfMu+j z$T$61Y|uzzptx1mcJ9MB4z%>Hh{VP@HY@V}H$IRP5td>spY?P0XDqn@1~?*}9^4)Q z3jSrHU_2^+U6UEpsY7v9)5fHxBvq9%)SA3T)22j~q&#YxxPoL7S}HQ}VjnyPQl{K+ zaV|0@3frhaFp=;3U$DOQVOj+a}7~mxjzcFAp}-)$SpwwmdAQ?^{krdTPL!7GZ9+b|ly0ag&QZ>}>TJjQCxJ7e2Kg^pyU#5mF8JO}m=*je#P)cgFfdRPg{^EmWBN}&imYf_FOO3Oz zybMH-xWE$y7o+~zEm3~Edc;4mUZOIOu+-QJ@^U8xCP!Z&|5AEAR!K4R^yShzwlXk! z$JlI!T6Ve`tWi~UOgErigiysJ)?YM#x{>>P#+E$%>V|b+k{L;Geh#SeyS>#So$mA& zK_A5G;bABYAz=2O^prq7s3t(-a`Bc*XzESm5J0GeE-y5V`KMz2M`t^G z>QU_io^?_mI;REq*Vv*oyavDz?2QM%jDw`(a!efz5NcmVa- zlq43MBT#wiXdox?-e+{uh$wuMG60pW*EAt&oR_(gj~tD2 zsLU}a!e-B9vbTM);5S+JUALq%xl)SxV-kRFw(w#vb3Gf^ILVsXHRU?wE8aIne$jj@ z>vZ|+eBnKQjM0a5g|3Oiol2+9#xRt(GB{Nh-q0E)=tN!+?zHE~c5m*%CoR{JwU> z`DcKu;U&u7!-C5<%F;C^7Mjzxv^3pXT?Op8J|ZExH;rtlh{yVPnNi7L%_J&P~EUo^Q3{-MCB%5%VHD1Z#vB- zkV*4k4&Qf3?KcHp^gw9S6mtGqU?QZfl1@h}N8POC*L6;*AP)KM5OISKs zxWO$Yhf{BHVd7O3SdQay`hOim<)}5wtp!q1(bDa=55?M25pk9MH=mhR#7(u!8wQe0 zKYXX9gY50YTQz`5poNKjqJsTii15{(f4Q2RkfOoqs&6!N$u$_OdtNEek98M~oj9W} z1}eB1%e|F=|46Dne0FVc{Fkx_BX9eew>r1BD-T2fkLp~=VpsumFL%lTV-Sgv%CvW% z)cX!8n*zSpjaW+Q=#oNoW+!@s$TgTl7c_AjTmItj8?jB)ij^uq@ikT_C~#MnII=K* z@`@=GWvQ9wRXU)gN!S|T&ZY23z&n)JT}=H#pzT!z2km5i701KEr^Rp0Y>sCdXhv6?FP4FtRXYpIJ?KK;XiHgb^LAIn z;!&i5AzfehGEgAl_x{+E4(0y~A!K`7`EQw9ihAhR!+@5bjH>SvwND_A1^zUfBaOW; zn|?F zgD0244ozV!NB_XEG;#0|w|dXE=y!Y=j;v;Ww7654MH*QPw*LaKxBvj#hb&%JhmHUM zu-2153E>>Hl8wEZmb9}~et_;6eF8>54|4itdb4AdOlQ>+{^Sjo$gQZ^Z3DW@5NI)xmX)e`SrNO#XGWl5XGEpN8 zcRwsVj=4pL&zg6Rt@fi4%n}VJH9mix#RULZp>wqv1&TptY@An;t$3o1SNO@$qeAa2 znoX<`^3f9vG$uQ-)SW=a)t-keC~?5Tmj((>*kInQ_243q6k9f<0pcdwzY7X+_ocFA zIXe78_RG;Gs+Tel)S5hczy9Wjw5FiMP%w}Sp~sYsPM4T^RFP+@{;}8)K!2ri?wNs( zK-HJS+Cy%nJ0xZ>yMj|Per0I=8*3~VlWA^b=c_M0GrHa23K#NT_dxM^#w2Uh~pPw+(e8?P63rTu9@?MxfvU%y};a(Y4(Q~{|V zgZV*Ia!dmll}knZWe0&H>#bQchr!3~(C%0-=-j2=ZC4DPC63rA`quuzv~q@rLQ`^g zQ@SHj$8PoVZuPU?irfQM>F1wzR99NA0rT}{*q2Mx%G0p%YWaX`u)zCc44-y)%;o9g zod@F|fNQWoztomo9V5sY!h?4eyN(4=i?RAJDj!x2zLE^T?!{*IK2OuS=lSMc+HabN zPP-9(lVw?(fm~YvQGNKkt~8`p8o>0Is3JXKLrgb9VOT8H%rvR#jF+fb^EfCa6ZwB@ z8I`v4qW|DhwFtZor@-wWYH}2sF2cUifFxa73r)iHHhgo_LOKaZ*XA?K{h3!qQ>vGq ztHWv3_lL=sp?%2xu(Z&)pFH8P>`r9rQIsKL>tmgH@k5QI?wU!kCRb*03-;c^WO7}u z5yK0~f8{r6rhN@>5Peh9&Ijf#3~LLO6Ud*i(l$aFy(v*ais~|=KW3y6iox5>Yz9tW z`|=$M?T}0*md>@K^RUsurn#77Dbyl1Hl2_iTKtm~b1hA5e?$)^B5crMJ}v(aTqzn`M;t zlLj{cy2>j*nO)Z@js*OilA!>c&o|u=*;MmGU{u64C+Z4ADU5S-ORKjmRt?l!u?Rh) zN>=ocUlZ3PLOk?on&(8a=<9XbRIWe@5{hL3HWd2;+B+wVYSCF*`dU?ueKv*r-HE*9 zKGDcq4_8AoMeX^xsM^=Li9Md5B-9MK#ERq2b5%uk=#Od(x<*v80&eQ02hnkbxvX#Z zZN{{#V8I+s%3AbQ$`g*K@BI52u`p#LwRt*(a&uI28$>38e0VuhEPD&h`K)iw_=9>w zbNj%L5(VsBx8*I$E^dXpKb555zYd57-a$uH9wtEdx`=YvRGxY?^q6<~V77|xu!vZ+ zSUb`6J?mNUn}$5(Pw ztOnQYv*W1(F9nF9V7XzcVUaT-E-!n@iK@JwRvqqWa$yx@_xteH!VOYb=fy&Q$B1 zwl=bj=cC=;`+z_d8I(`%d;tM-o>{Wdp7_j~@6T-j1JC<+k8>k_a^Oc(Mx!cTL#>gI z&;4zK)_i#M2ywn@#$QYp&7+Ug3@;lhid?ux1N521K@8uj_=1TOL#vf#aPidO!X2rM z?54!V8MC<_ChzXFR%Wx7;Ap9Lk2xm}9cI(^A=GrF3=(W0^mv`WeG<4cla^Ji4nog8 zIlr)NRs%n%9xrQq@vPqPe0=d3(DNp23BjsVp6|iI~ef( z5)SeF!RMxNRCw1XYky5JMP#5&BL<#-5AM9aA)7*QgFnp{w3(V@pNWqyXP`o{}r;7bcpAxV~#Xj45_^ zl{6tC2D_hEOPh&x%R7xG-sQ<4v^SPON->bN?F38eA9IJ>>0m>iA`G<>BcV)f7)!VS z{3w09u(2GCl_c&sHzlq}%Oje1U7X_NM_devl_~gpzklo7@BotJpF8Iyw& zmrzfn){g1vY7M&@3e}H*x7Qg9wPInV+9-w+x@Wb z`J-8L&XZ+T{sJPtBfBc^iuR(9MlDjIne?pE|HCj(^s*_*JwC|+p-RkdTOAs6YU zu}Ug7v{!fTJ3zHswR?;#8}&7?`LunV*LkK4ytYC(M{r46r*>E#_K(5gOp(Q?=|m4o zd;EZN)-v=&$+~wXcfBF0&GAul3Q0>lRcq_fcgGmB?^62OwE%JKHsb`-&lBz_%mu>8 z9fFtO(;pN_Bv%92Zf?U$Qp)5mdFF$lpp^ZE2puBi;5s;HD2yt0@?RWAoeg{;ztZz@ zo-;6ow}!KQL7T?keqU(pgtM-`CtMFBb-=@!4K7X__;Mo3PUu0^qju(=MppA;!qmnY zez+~X-ofD04B9x(=_G$9O*{W4wd8^BukwWNX@wjtBDx-FB1yRaV=0dyg`%jLNGak( z#|-wJ-@8nJbnSeZOkeGcV?j6e#b$MY@@+DcJI1)qj6!n%5DLaG#D%6vdvTc|La~t9 zkyM?Gt}S^RHA`Lm`q@d@21sB}8l+ zpe*JJ8xsh(qLHp*+UYjqBWR?3gk3D3$rxKRee#}g0=hIS1JyMNf;5JHTB<;ptSUmF ze1a}v+Og^^CwxMSF6Cak%woEB#zB~FMR+&WVFeNYLWu+h=8q>0qjQBEb;9T?d9R~Iq^>+h%ACtA5f98} zrjGvpkKJXy%K7Xj#smdqZTEXd1-^RsUrfm1CHgbyIRtHP$o+q!H?vs?65tC~bqYyu zB0*_po_nTO(h2ro-*AhqJklGY$Z{lmj<(5=U3y4Tts0(@PA5o>SkwQJXBG>|JD*(i zc1tkbTwmIw2uknVBtfU53b)C~Xcg0FPiJk^K8EbKZq+!$&Gmfr^RxBgVVlM699o0OW}i)-NdDCk zk{9lqN0+%Pv_y6TUUC7YDkmp=-%Di%10?}PD!IR$$a7z;GFe0YgMjDPJ^=^D{HlF= zxf~biuf*L#V;dTEFFaXG$Y*#M`5RjbT-SP!OG0Q{m<$V|qUYBfXz1+d1ae^PYfR%@ z;j_I!Ep@x0h^9VYhgLYRR3F}L7ZGH5{xdgpcBa$oga*3caQ?|;v6ZI8gG{Y`pq$-a zM^@!%h~31+7hU^}$Ze}%J^fU#A1=yxbhk*y3ol-jAB!t&&|Rkb)QTuFN*U+AOo+k7 z{KFVqEh0?m8tR*6Y8kI8GFwZn$s^B=Kp^^tS6l8FVTT+p?qQ(deshD`-#zfjF6Gx} zMf9{&#mAk&G@pp%EwG*K8hN)_-J1An!p7NW{_9JNtn4{XdDr~pYALyhI1nwVOrgE zs5xpd+-!G0D|V{t(=K1odNo4+SEh~a8jgNe@_ z<|OT5Bt%M7ab+LxqgV#7mmDK#uq0)PmDv)CR~q5*^&`^7)fa2|5(iVl?4T5AGwj2{ zjNKk4qd8dYrnuHXshqOp--hF8m{!BW!kx6%U($~UNtqu6MQUUWZm-Z)tSSgE{GZ|& zSh-Ry?HnuFJ)2g&ekTr#!qH?HpA(^t4#_y#GB790h!`)tNdKI*r$sq^ zNl1&7GVGJiN|#@FePkY4Q+udyD)Ufb{rEPAtyK8YEhTQOtK(}6vjK|orpBi;BbTRX zdnVMq%kp;TnAf#;S!}yXbs!jbYwT5V`Nh|p+AS#B&hipc#%>|$A$eFVi}kSiwXdT{ z94?Hensl5dW;my%qzqJ1l3QAaux1%Z*U)=v`MylY+b4mNB_KV7ViZNQhElFcvwgcz z>~&?<)g+6bK(wi@yjR_1lH4X4t-a-cD0I8ub*qBLtm=*&Zr7TOX0d`*xBW>`t24rj zoZHW-&(T$X(D+G@Y}vVQx2?Qx>t#vLYMi`0WATbmqAt<+bScQPO>@F*)%p%TBY6#( z9aX#O$$6H~1E`T)0GVBHZ*Epr7L?cJr7fNFb%ATOl$q#Ew>G=tqTh#h`P(){Xl#rmuGzbR%ZbOf$mZT2N^@9x722>2jGBcd!$Q1mExV?UJaGrd@bn zaIm%o!Gall*eh0ygb}mt?%0tpMv)z=-o~NHr}if#IkB*Ue|XeFClIoCdGE&)oZ$p9 zGxPR^|Kd?`d#-0CE@V8Su;TN9@fJvt-a{?AD~nohx6* z9VhrFC^xH($ZW?U-Ltg(&G`^6d1$h2i-(OKu>Exs5PZzNB{(a*dO@d1Y1ToWZpElx z1*Dor@l#f9OwamvB=#u2)isL^O^l31VUC>`7v4Y?e9Y0!w-CE>lxCUdm>4Ov(bN}~ zquhx@8JTot(l zZ%0YpMi4l)A9JZ#tLcfO&?WYw6FB4UhJVxGyh&0E&VSBvs3CV|K5=tRTxSD5uVr+GnU?ai5VHz z5*^8fVD7qhSOT~*NF>h)53qzs!k3rp;^pec{xXgYKt|@(PMm**-Y*SumxJ6g>0UBfTc-syBNp;|8Un$MP~GHbc+ILJr0e_t-+Nj;ss z8VsE0tGT@e^Xcdljt8`f6iu==I+9yS{_ zDE?%0g1PwG;SyN6ah>kM@wxtV{%M!o?(z>|Z%tk1=@q+5ri=O@z#?D=x0fz#4XeUz z!cuJqQ38$Kd@IccW5jV^lWE}Tq;~7n- z*+aG3vE3*HfZE`xOV@alMR01I%z;VSf`aQM6vXXmDW$SRW6j_|E&;if1Q2uo`^5gf z(lxLGLm7o39=xsk_SzD9@o?A2fEGIZy97i+ab+a(=R(t&Vs47x_=R0KKSt4SEmV<0 zq+6(rUg(K%Ioy8vwXGtJ|CH8TYe^COb-AubZ=i*DBGct+_Z#ysmb5!(T}q12Z!3vK zcc+%i;kur!zfd#&S7A-mSQ_Jlkh^MRS&+S{XrdJS%key`B5 zlX+or)w|5j_J@+gaizwkld@SwJ>}YiiPJ9E?%Ot}M~0&f?!0d_%V?WLyjBjVcc4%i z-lD{dUo*oB7l*b)OIkNbxbBZApE*cE5wQ>k7uw^&cX;2glVc7wz+-D^d_v!3 z$j3n@b0wVGC(k}LGpYMrtHaD+wYK|PVrm@B#-p9qN|wTbN$&0gC#$3+ezoi9c5xQ= zZG|uN?PxVVI2fUgozPHWrT?!-Y}c>?r~4xbkLMkTH%xt0(7Q8*Cr0!soUj4pYfhn9HGy4ipDP^wnD>x;Sur*l(; z`tX!RDkM6d*V6I|@^XsSpSVxlw@s)2Dk`&mC55NbY%5m!222yO^E+bu#_{=EWV5G_ zsehz4uRo`lxhyyCv7n(O4=~O>$2W3E87Op5&a<5zU<8MEOiBbTr%Hq~#`}s@j^J;G zRQtXBhBe(;i;;$ow?3F59>Z7RmncO&C7x6~17B^6t2rF2o;e=hARav0!mgG*%->8h z9OEq<)e^47Dt*+Xd>k2AWaQ@HJ8Dl!ztJ8GU*w-O5>XSj&%HTcMu(*O&skSIH8rg7 z88db+w{>xW_DLHQ6vQcAQ+Zm02Sjgua!^L_GI@JC<&d{`PVcX5)@qW0SDR;z(<(fVX_SH?B+v{ z;JcyW1RGqsbiq#DlFUDz0kP=W{+0zeeq6fh5*2hG4}Cs&^!E#NFSfQ28Nz$1i7EZQ ziNC|&XM6zWSucg2oK5nxn=HwdygHNfuKX)nPHXKBdOHrCCTw(iYZNmaZ%1@eo4{`JX7I`&@~}2<5b#04&nxxB__drk!rvsr5;#UL(1=w#Ez`wK}*A zanDBWyP5Tc>!=hGleB0^P7ehFuI&{8h=`@B<(}J%#b)eng9|dME;#YQbmtbk#1KoS z%XaZ*>bIUR7!c%=qJ!KfSqerxefZEoxPx3@J0P5rce0i-@Dj-%)7o~6hxaJXlwJu=r;qm(Nlt;`PkxV92fCco?@VJFq`?4?3!o!Xd6&2OrC1S;Nd!>VFHt_>k0QaVQj={wV>fJ>#MsFRcH&7qX42@Zr>sXBW6v1|0vx>3fk zdz&+OuRp83Rqm`R-D;_0dmfwz9K?U1JF5F5%9A)Y7fbfDoRH>tI>PcSKRV8HRNxku zC1Ih~5avn=XhiTK-i2<0YY!QRK9Q9lDX7o2-<&ULOofi0Ek4K>54obGZ$}y})tuew zQs6M=TD!2VbrT==M-HUqQF87!5RbqBL=SXR(&mPL{^!{DSp1Ze>)e%@hn!pY3VaG= zyg%3gB5~k@q}Hr0ZhW)xWS6(gnY)JGOP$5?h29!`Fr8)z0u=={?T&4b_^{I0Hy-m( zk92c7!4YxvIk^gE!StFXYHwY=0x4>>B3qkXxtm$I%XCdDKZ@tZ+~Qx5`8y*vXNp)2 za^nX_AL5Jx?($0O2=>O)x28ysTuDiLXL*%5M^BOz>;e4w{>j{j2xq{|sx{1q4YnR1=7X0cLf@h!4WV;OxYfJp7ZH%uBUJ~_ft>GAiwX0gh<1FRSNIb=wOZ42hV{d0ZGAV50 zp5*_XcKJ2TYMfby7#Y!{5Y(*Y78MkHCUwEn(rliTn{O2H3h0&Z$la7^VRaaBSe|rQ zlaLV2F9M$&ADe7sSZA((yLfy40D8ilv-q@aSGW%qtT$X_pk8lV=2o83#|Dw2#;I4C zvX@_-Z)0t}{Wo&!at$?B8yN$ewXOB2eA^z%XL}>u245Uc5RQ+JU2F&52fUcj3}cPA z1|=0~PWFb>oMjY0YG&KinzRBI-xY>7Eac&DvRe*Ls4~fS?iXD!yfVM_<~?66uDApi zHn-u?308TEJkF+a5lKm@@}+lXE0v}rA;QQUNkJlGPKlxTJd?XOEoWEFVX7>j>^q}c zCX{_`q^KH&Z_2%6=mp7yvc^|C?g_{S(};0?t0r*JYmXE2penkU==D zvj5Ib7PV_U|0A_rWp?!`HQlqQ&#V`&@ABpKP2S0;)U0m0jWn0sOeCF)cO`0bHNU}3 zL?)@;LTu8gujo!(MysRDU4mhy=RYNB-s{F}O7tSR*<#1j$490H0pB3(4);hoc=UlnW+JcNS0^S7vulkK&rG zJAFj+$JiA;^Ylar;Su{)x4L$s(O&5b(xr|J2$#mso=`sD#$fzS-L@zADm}HuTaPs9 zT0>J)Sr>LDyji+nQ8$ie2sE00_PpI3jPIbuUccq7%@{Q(70=<)eOlu{Y`VXZ5L*=P zd~hT5bo-F**%s>!ovyQ7XTfIV)EGnZJv(T1buLLO*Yz>&QA73iwC;qXi}wA5^%SGM z3zwi-OTjO3ErFS&p9m5dZd$KcHVV+P5Ht3e$e@6o#5o3}-d3WmT_NVKtMLhUChD8lHH7*3HxMbw z*R!Ro-v5#F6I<@fPr~qqJSAiV673xP-m@{zwBEeGD^Y(zy_8ObFw^PL-t+7TlmP2W zjr>Xra|`Oocm-4|qFzQjiwc<$F2^U^vf;{GxDess;MlC5bLQMVTG{=bmRc9=Gm z6_Hj;!IzwqStT_Pro+;NxdmEI|@oU zvE^#?QuQa-`6o9!e9zy`Y3(h9Qj@)iDhOjSPE$gnchU}J@mCgPo}q4uY)C4qkkyNFAi(w7n&Rp zB{1nt4}Z-TC5D7Z^v1e?hK7e5*C(?NN7+gVssG&_fy@k=R*UVBAZazSlP(iMz2=F`|Ibcz?ge5&&)uyU5;+ zj8?|+%e)adE31Sx{-eqf6%(Qg)64BsAJ;OdC!Fl;zJwng8qF;@Qz})T`H67Zef5Yo zh1*tnp1;QSEXdcczX;G7;Ogj&f#dXaMl9vJ?J3kl9f4DtX+#CR2yored>#*gKsFi= zRoRnZ5jOnrm`AQ>*;3<;c&Qg_MP+0cshIUV$J|0xedLf&CJIq z>I{0PXmK>oA zoq)&4w$a=ccaM2$Y`xQZv=m|wMw0JaMkK%~@hM7cUq0j^EB=)Vkfu4KIU8R(uwtU5 z4Ov{4)QHFG))Lg&W`U(#PGF@6>S;pRE3*y^!1$AZTu!3u{<$w)1=d^J_Di;ix=9y@ z^E5gib`Vg4bEQPo_?Ba*&S?J-7Q3S`gJg{U{vJ}Sp3(@HqQM$ ztj>UVxzhupKV~$rX#y@$D-g!h?+G>nq>mHE!wL&c%ha!FEK;zkL;OcIw8aU`9*m= zjT+8?R|LvL))EWFJQb@n@x1fVkfTYz*fCelZdpNKL&b)*B3T$UFm%(8i)EkkQ?{qV zW7rph|1(Z{&`I{Uz4gjiLCFI7B;PCVUf$f|)2!zL@NKCObei&PXQawt1W-(ZzmtJJ z1F7TuYNLC%X;>&vB6$pF-C-^>Wx7h*?f@5ZxfchaRcr4Dzo+r`*PSelxd?cjmoTMa z83V0^a2K{%mETRX@$lI>=bwNgTw~MoAhZVEH;3jB#cuzbK+54}ujy%5Gq$i8EQksx zE2&1cq27gVV2=Es0s8k<12O!>J?OCvrz!@^xJ<6XZ2B_Jpta&? zUC1s>Ky~N;BPReR8~j&J04t@VvXmXg$wl7wRVC(6W@74B+LUG8z4Z~&tJ$B}I}cR3 zqGQu7spzydV=Q*28d$1wKgAn!{ieR|QI0F)shj`LtJy=J@D7kqKP`yviUA(bGaA-H zs`t93e9%lVV zlKXeU^$<+96e0p59h>t4oksRFgEr{rHchd~vyAo+T4+Xh-4A5S4yUYPeyf#8^VIbf z-=#WOeZ?(huTspVT`-7j)4S6tKx?9^$UEGl;Iowwr{F(xi`b~PoMZ`{5t z_qaSd>6nIGhxP5@;bOi0T6d&uWrNJ6Drpa@Y5|Am)+ZKTH~I{lr#G0!y0zNYJ|=fl z68GjKYhKLZeckftU$C+hs7V72A2+T~%DRd9;{SwQsa|`0>B}->Gc6~*F7>=fIzmWG z#tjjJv6x`V+6&DrCr`L_A32`KjI^D~vl97x4ci^dJdg-+->H*-Xk+=f6`Pqs<~3N1 zwRti4u8yzf&md{Fi{YF!LUK(7@=x&nPoA+hDLmOwn0oX0rgQ9HuYZP1+A$a`J*?^? z2)kYNUbf2d!r?FoIp(`2g!J9a=q2T9Eu5Hk+%;up7L__pS2B%g^gKVOX>S;gbr@$UMSF;;I;{^zLBi~ zWTo$t`qNxl67NavR9j9SJ)Y+a-1|XEP45A9oBn^BP4KGc*ZDZ`@3P@X0EZ%G4#C5W zpc(nIUYLHdCw-LV_F(&+|9mivFauJs;goDy9od-iu&>CY+Ch>=dA zo3mp||9W|2vwww(&w=-|qBQKdHqLt^4LJkGeClEQz6hwjRnHT&bTXPrn;aE({6`3ef?SLBm;Aoza$r=i zI`(eu_nnnOJErAj8K5so_36`*Om{eaMt{~Xyig=Q-e{3q&op#wAdh%>ej{H((qFk& zb_N9`M#cPMtUDNQP)_TmB@Nc4ClET|4y(^^4?5V#o+CQxno!RFC{9z3`hXPjCS=%l ziO_wQ%S1oxV|N#y@R|}{vrmqbe)l29NP4R_zbc`E7t8I-?=&0~^*pq;gZIJl^lH@z zDg(~!NM}yvZgj}r3R@}*FMK(vvYlEI=i4wXGpr$ZqB6Qp9^9w%f`hTW5{jXC{R~W0 zAMI{BK7){7qb8pl%RnjC6ORg>A{-Jxln|qMX zFPg1iOdVbLnH`R?DD^!7dwhLfGGHix2}lXNkyaUNf9X`nm;34@p$gweaK>nG1IZNb zBXghJ!fJ%Oi=|C^ux-JymLwhsG(IZ_d04=bhld-qZtSh8vadztdl(;nc~2}9yG+1B zrjPa~*Z5Sk)QdPodoOjvBJk8sIQXKYB>)+TGow|EyNb~iNS zw|3Fg5}B@+J|!(hHj%6(^ZC{z7g8>s~0%z^z(XG067L-XGb?--jJzglQ zWU(QSh*No`TEiBFq(^6xtW@El4SCAj)-ZdYg6jqwVAuY@Tz6d0Ymj+E^dm!)>WFvI zyR!libCB=!{Jgp>QD*cvS@D``fzR8+x0m5(Ig6WHaKQQ^sC1dhP&6pt7e#g2YC~G3*LTox_P}xT-zdAd?mIb9?( z%@tsnCXPpAcOLv~AWw=cZu$U2?=PxxH8J}`XI+a{G>qJ6L3KB7lK;Dhvsvj0zupYr zvb?k1RxK-~dy#}L{ku+}>0S=V5e>aM(U0H&f92!wwhpa$s|BRlljMqoX^H=FqN!;ttnVWPD5r!g{REFY6HgvfZD(u4QiMP zBE@|RzJzU;e3tUb_PQ=QXCIKatStk{&7^1*Ter~RrnUWAREcCY9Xh&hDW5RW;L^yD zi9o#;cW;o7$2!v7=@@B$f`sjg2CR$nTvO>hnW@~KjjYjAF*xk8hYH!{-E?P|sJ27_ z&2(%X#Ldl1uo3q3e6D)=gLxRKr6`w~=+hiXs=x!`kr)wBYxtANJ71z#PHcUz0wvR1 zH7iTIKPJ}S?F-UbV@MZ4IY2zj{1!y zgQIcdSoZsE+D{~MDeT*vUs;H}gt(tdu^>}J#6r}i#Dz0DKp#aTOrXB+>=yHt*`MA=DPK?B3BnHFiH=J79-SHYS-=7n_Fa8I78H@#Gzds-I;9&F zq#LBWyEfe&(%s##>2B`B`<`>Z@7(*3W6OX&o>+6PIp_LCTxfecS{mZK4UG%hX~8!F z#p!ahdqP2P?@@uNB@6~KF;wUDpG#1%6q`Y0l@%Oe;#&m4bYDddLd}`CX@A1lj!}JI zWChl>n-?5<^D3Q05bU|5Awtm|it?cXf#I_6hF)X^(qH>FHr%aeyb6-ZrtXURW0M#K zG3Pe;`<{+FHrt~%9w_DDfh8D%P_NK>u{A?>wlzP6I!g zjMe=&PQe^tbz|_T-k*9tR@CBSf3I8FkQ3jvSV#qPA zbJkA}BoJ0K&;Rr=nY=$9e$#QY{CeAxV=i@hqyRdp^r6*?STJ_-prE< zK1?;nRj)r|O;98=QlTlKB^^a=$$_6pl{Oz&pT z!P);?nlOwc=ao?n%N_Pw28f-7kj%}HNW2-!S5i5H#qUA^xND!yh+I*+tG-VZ88TE1 zK!=ADxXZQn?SpX5t$msQ<~Va$uY~0yTBHlEZX(*Q2W3|S%n9> zD)Ouq=}#O#h@uf)ry4=Vtvdl>o@BCLaVR@eH_J>@2rg$#U(dwT)LyyPCgfrZr;ySK z#&+KBGnl)v@>XH5SV?ucqDhEB3&fS|iK-U}GFKA(yyk2e(fOOILYr6_Uup~;_|P~O zS6Mc|Q=0?u)Xuhk3i%FrYAb;QSBwkYqpBmp8N{K1-+c&pL=q*Xc~0&kJVUf&eV*&; zL-xNNdN-$_+_jP6bb%;}^YFzeS16Z?x%J)6D7H$Lov%Z5o7WO?t{uX}7@(ydw1dwi z;x}!2?)DWT8-*$O$vjrzf$hiS!2x|Y_xA_`T$c-9gO^7znx}KbR+1_kS+o;jWqRcn zIAeB9 zjqiWSgmfVPrx2orv~|l3BYua8xN~xfT>#w?zJXxXxR|-!V%vjjeb@cn(ODT%~&1tzqd> zNb$DQto-u+wJpS8`aC;@P;-L4olmox0<2Sl?-U+V!P^(V)P)IUozg!weglXI4Wl;`iUR@VZyWW%-?M}3-0AvpaiVMsbTcFn+#0P={Il-5sRr%L9jQ8$5VpZA=il_5>3k%>;U6MK=f z2qv3N*H#`^h9E1$MKiW|&_)!w6$b^|3}q*T>cE_7Sz~s_}UGKwrZI5#L#* ztoCZjX}#uRi76>CviDpDjF4&(UDFwcI(}5K$7kXM6Nj!!T_oBgUSFvCC9%LOpfs79%y59uW3Um=F4?rO%Y zViNX>`2-GDN#Ln^vG!`WgI62fPi0HF0^U^_y$dh2D`_KR&F6 zc)X3qXV$G#d$k?RyeqVz0)KV`lC47ED@pw6r9iKf``zxbM_atM7hyf}n!IL5dBAO* zupI&i%m%Om(VZxH=prxnJq|6rSl-i4r_pI==y=N?sxVwUg>o1;lEB(Vy z_tUSNv1HXOiM+f{Nyi6b^Eip4{xZ6!EPIwxk+O8KloJ~;Q=Q9*_rpxWnSXP2K%QEx zcHg|`4Srn6s>nN2j-FAHr7F=$7>H-Gac*}J6#0hOz`U~gSK{0N!PT!dy%l+5gk zNZ-w$jiL{6=t%tL(j?ram!T`yNJ-Zz@v=Z~(7xEy2~cjHOZ)S4tT5Avru^K1pt-SD zaxN~EqO_x`BuNc(K_G&HlxtRC-|UN0dzO>eg|RSrzL1xFasSGqEg9J zoZI&~(>DZcK5wZL;4AoWr!!PX;n8;38abEeZ7v@aMi-A!bGlcTLzl&ndnP%R1rRm( zq+e(aFXr7BYdKstGKaakA%J;zF%$tX?=FD$=X#E4_*S?#B3$^|7Sw2)^uDd)ok4ATE#Cgu?JxX`LH29{WUUVT|L2YVahLt=cwhKrErG!IW&$Z6hoW_HBV)mUVBN?zk08L zS|oIewlt4^U`fh-lk#0QarfWIrdmEUJ0$ly?uhgN-G-{K2DIs^Lwfi4 z_pJDZx_voyI&M`%zZ!7d0u!2vSMJ%IEcxbHJ>-$?4P2JgH&gi zliU4b2y0r4G|bkW-%q&iS-H37GCI65zHP-HTj-Wi+$=tWZt&-POKX7HnmL*#$987n z&t)7s_^g5cxqYdO)>RsIf;&%_7qJH<4CIJ#>`4luPQpU+q`s1!GuSY$DUqbR>Odmc z_cai!V#NJlDj}gGEmp}A-SI!FJvW!?CWEQ{Eur8qBZ_QRQsBPHd8qezA%oa5{T|A6 z#2-UHu=yNUKJgv$N`aM3etLH7LwZ#GgQbS||4|#uu*33AkVpP2qIj@8e3d#~V8fOfKQCMQh z>%TWW;^p2c%nx%hAhFZ!`l)--SgU)8Io#&Yxp-a4K_pBqj~e-+}+6cn)M&)U>JiKG4>Vu$=jg& z3jV?O`qCA#c4cOAq_9Y%i2)vzeivGRbNAeo7bfUNpt3gtrCIQY`NoKnwbz%{=IHpU z34AA+Vv*u$iXpS%>yG}N6$B&S-}xqG%n|A*fNZs#S~ssY zD|f2f)c~Q>3w5zb+|=0voV&YymQ&HwFS{DQ`+|lID>*9Pl!-WWS65>Qks)**SuE%} z!u5-uqEf!+kKB}5uuo5-x!ow7oe^$co1&JBvQVNmI@B13Y37(bqx!SH8>0HlyU0*2 zQ@^@!wy`c#fwQj0UbOm;kd#tprUnBD48`G0+k}{^+c~k&c9nDtO9y=@$BTKUnGQz( zz?4$KqPcF^-D;^cyKR)iD)7Z$e+Koy?8}6lSk_eLCFw|mM-6ACBa`t@W49{inbg;X zkVWW=ZEN>I``hQ~k5OUDvlHchtt4r`a zoHjXQ@AoxgqH7{aT_cJ}4qI3xVC`7H$5EisS*<8(Cp1C9NJs2+DkFkaLVe?ncc z;Q029{vfMO%jicNDNJ}@-Lm|HgP$TKU%qxK$4+Gi;{}auMOIPxO%_`y@|0~AMGxNyEt0_lq;C!V=t{P& z+zQtxM(*Sb1Kn~&Icin*nc*EuAB;-u^S+?8UlZ-9gZNDO)LgZlpYWM*Mo~k?P*HaO zcD-f@f>o8a$w|;qw&N~H5}_P%LgwhCXBo>rTu$pvG_LS)!UYRZ)8~5FSS%m5MeHfP zsSC_4Ym4uxnf0g(PgM~P9ywh7(Tae!qfj=DJ>^|J`^vZk!Caf0CI?$P#pBcwmX?K_ z?CeYwyc|BJ_52K;Npp){7U>el-dPwHdO5OS!1_mT`g0hWARb;oB2nk?1}-z>S0?q5 z0$zaMFx_%DRAm46YBS_<3J@dc(6udLjOPX!PX-^`HZ2(B3M;K;f`-7IAFKs0`2{b@ zP&HE(_BudZc@?YO84n(fe(fBzI!WAhT{VM&`?xZq;kpk8khxR^z0~l#Kff|54LVR$ z-~Fnmd^o*4Gnzd^J@YD{Nz8$zUD=7}pGO2=P<1r8tlrO3t%ob8%2mZn*N814ixHb; zz*p}7tcj^Ck-;(6(xm&rOkQM0y@E`nR#H-oYs^{l=c{xwPWJG7^|HzsoiZsZ(z1y3 z6u6+CjN)R_QBIbRLgpFj3(;&7on!8e5%?dQPA?BTlP4hsjM8RbDqj~H55+4QWA1Cl z<}P#N9(M664Q**Nz<=*!XyTKTR@_8k{KBoqk5T2hc46)mV2Kn1cysf*B);Fd+U-Z! zAena=FQg)$82RVPeq6j%=IF7$)?+jc_b&~D+dw-+QOU}Gm*x&8nIJpQFHZ?u?ZZK) zU*2r=>_0-d0GTU-$*Y(r{8rE%iD0!gx(5BPIgHmC*UNO!Gnelze+!?MsOnQcTVMuN za$LbUzBqFa6>0iEe6YM{)l>8e!q{BpbmpeCZbz$F7{_3#&}U2P(|Y3k&M=7$TKU1f zSeO@Xy`Ae?a;v~b!$fOq0CGhq=-n3teGVi2t@hnmu$cBzi!5QqNTC>TmD>WGxo7g2 zl_%RA^S<&dE$6F`XtY&STh_d3Tdw9xq)}y@VPUW>npA!!YO=?Tqm6&$bK&4(S9Id5x{#r~e)1Dq<-1@U%;L3ljI9eRZLd>s4& z92I1Kg;7M`weNp=t~_R*$Z<4>)EOSjT7PRS)q7o5Le8auRqU$q;g>eQAh~DMl;i;& zze@lZ-`vVSc!;LhWUMic2|PIt$CLMdFs~3<(N5IH{`2wD-n^n}P}p+DuZZZrQcA8# zrcdpEhu|KEm^sXMey*)`2l0hk@||ejM*-9c|Kg?^&s`uTXr~?IhVIki1yba6SdA{RC|# zu$nDsGrbqN2jfYQcUV7|hn$pbq^}yvqsUN|#7iqqh{>e9)o78+n~RV!l2ZE|aulrp zh30&)I=JU9_LsO)3`aeo!DgZYsmIatKg^UU}3LL=@=aQ8QY&$^O)BF~#^CD2z6XAMSy3<7 zI=T;}uzNdBfJyhP`5xvtdGlv6=uFzO6mw2wOSd0gUymQ$Zxt=WW`)qAv3y6FJEAWmPdeiB5xYQY&g8DO zuMP4J>l0L5zFfqi;aV-NVKm~Q?&dAdCO9c{`-H+aO!GsbOL&YaTmn&Z%! zQzq7e3;~aWd9$!X1~ZZT(Wlr69L{UeW7V0jCAN$l@ZfXNkFfS1g7BUrgn3DyF!pMp zMCCnqYCK#ufjOc`GehdOZu!O#si9xYkN25Q%A*d)pr7|84VJ~}|E2=Q1aD?j zJTIMS1=ELiS6PKgK{&4|84qeTt%mb|<^ z4JBJ7`0(!SW{%jHV~)#YCS0190cn&HQ$TgHNgZKN2sj zWpQ-P?H&ZVJ`j<<@-b$uZiU;^bo^0VzU`0`M^Gm5}& zbVr3?KbAR5w=4dcV-u6r(<#3RBi(*?0NeA2t{c(qmWs3(Ik!(i)?NAx%)BT<_4?jgHn@SrgS7H;rc3mXijhJ>N-P=3cbuL! ztgO(vKmXk!Db0|agn(X=C0*F>58fKnEt;3ku;d<19(WD;*M4E~3k731QONz4ig->w@_!V+k0 zupT-toZ9C$X6ih$Bp%D=zl%xsf!Zi4R&FDUd+w-EzQ~5q$sQA zz>IJi-HMa%V`IqD+{E{6L;V(QDUP~qy za-|pOVvYTXslxv44$UIF&19Fe| zc~ytvT@4agJWTrOabf8JvQ#uy)bvd?!n;hPTEWXR_{}wEQRGush)b}9f=w*0Wj|u6 zVwm8-oRI{W&WC|#xyW~`I|LlO1kDq&i+yzy9_xu8_z@&5so6XkFe%7()4p}&ody;s zkp`9ah(oq^9*^B!9#`DvPM=JwjO<)kMK5m^Mtui$&5kFEpL_2OQW;U))jAURh2Q4! zPraxZ;ENnz&7>Cl@dN%=n)e(A7nP_NMbO1-`ktW)U@qtT>)2?v6lt7?jVs?{$;wYn z4c|<#DbX_;7@*{JdmZqPqt+z6ihWIH_=uP`7&@g|U*X#%vI{`J78PqZpS0zs_LREF7PbHWxzgDV*=CI6&8TK_pzztxPUJo-5BdeDDRcGh=yZBW&S4|zue zr&3^=EGo9blic1y_PU7#teMzr^4@7h!_Qt`-VPkkV@!$qtvA(&W_%hXR(VevOTC$q zxk|+93v%3vjCr7-T*zj-u9vc4ogKdMZa!ueqK7+n5tzfiXTPez4qW`>9~8vqNoEHg z#)b&`6kGGK{T@*-w?|QU5xslcmW`M^)+KE!e7jsL&teL!*}D)8*~DCUn8kqv?&;*{LZjJb%2iv%aQ{ z9y{=ibHbA0v@s2@1<_t;r~$O~)9J?b{UdRFmhky|jF)=M;mF3(vBh?)9~TB7KMlfP zmu`{QSgieYJxS{6PXeg!F&8O~%C{}2tG_fM$MaWN&S6Hk_iH3+96@|DBN9L2ZkA$y zrNl<$gAz2@;}j~DtWZ_+``N`J`qAxdY}H-?ahty3)_Y41grU}I#`A?;+ceeZW`>X($*0PEkv6?I?5 z@N1Ie-=INz!@!2B;zxi%Dd&@f#e6>jNdB71VAgJcO}P$Fv8m@=%$~QWFToY9I%KBCv?awbNLP%s9yQzI^3U)=?D*g7fAsWG-D#HE(8ySWXJ^njXtjkBfD^q zC%0=YIvI|ce=3gZC4VS%f1V6X+y4{JeMpvn?&J#`pyq=bCv%tJxI8^keJk-T^crgQ zs|A4J3f-aR!ugk-ZNZDS!!*wd`cK_ztSO4OI#v`SWUU3pct^-rVe|>aE7W@njZULm zPAUi*xe#tv)|hZEE}kmKr8#4*3$J00$Is2tB;3B|H5AblAM@>Ef*jJ zZ|h?ov#RNjXnsntx57(W;Fj5vjf@nC<9*h#Jc3`I#C(CDN(jn#hu-m`8JE@dk$@euJD>R5x&$D@faW~t z>&6@xb-vNvGmK0+&n3QL0MhFJ29sF#VewvGnpsoj>j+LK3z~b?p0+?@4P+-*+$2@8XHQZr4%GN$NE-W z7Mb|=e#7&6`FCP&nKrj~!h^~63U&3>8*RU2avv>`@tGGN9{U!WyQHF%hF=5*M4|V` zTPFoNy^#0D!}t>|q55uP7i5VI)#dQU$&41xKM>kjsQ;ugZuaM30Y5dldHUZw`1fAg zpM?Zofsd}6&Zo^8lX-j=$svS7R>*Qh3-$&ebb`e`K0skW_=*>Azy)=V&`I#nP9%-O z(im$5mDr}>#Hh@ zf~^of^6M+o7cM^&CsB|}%q$vtgx4KaGx;-z*GUZ^7Vxx4h_%igLvH;k{L&=~B-~(= zuycQ8{%y+!PH_ENk_^QY>yxQpw{2`4Xhv^X#Mc~JBX~RcmPC`+X3Lf?;r{w;MX1Ji zhYU*GPwsZL4fqj1mf`ka zobJ#*hKt*tfW)Vvt+TCx(B#H`fZm}-m9EB`)gxDMb)Gh|323db>vRDI>h)Nery=Bp zaJo}@xeKY&F;VlxJ4PN$CRZV;G(n`)y(}=gJJa3+W=L^01oc%LH;2Qy{~Z^=nH>Y& zt#9%%I{}-~`+~nnIO@Y-{yWM%x!EIXi=`&?G|2{t{$CQ9+`;5tC&Fjyts-Z#vs3WJ zOXk70Om%X|qq9*8?dwuoUZvd-JF`WzpLC*mzK|lj*c}W6OFEBcaUiyi zWQt2-0L#e#USm1jKM&u!^_E^0>f#sf8u?S(L+W(&ni!6U zUyA>*@LZ#FR#a}p@13A+ak@TC(uXe$A8Dr&awmpo#o27;iKCM8g{{&eVpTFHruROF z?CpZG-Elz^G_tn?DABjf8?C!SB z*kpB|40Bp8c%u-bFMCh^-|VB1QccbXUQNe*eNFsR^B%E0`7vQL+xe-{*f{67x=BYG z%6&K+lMK^#J(qxki#M)tEcKVARlc~8gnJvd{5YwMP|zKfDLMH=xoQjI+SpvuF?(yy zg)LniGRuj_^ed92+Vq$jwCnn+^#$t|gw^;xR@(3+lNE(TixyQ|hX{ehM|V ztp9?*gkF&f1$?Uwl61T3vA@VWOJI{r<@HM{h0Dpw;WX8k1QZ@_j6lfG0}r|VwkFR% z5oyCo>(?F=!|s8#+JwoL@z5x;_4xBSs~JW*(-MD4CF}qK1CbkWP_9 z3=En=wTK8uGLW3jQb@BV#DyPj6V@2-B{xf?n?@xL2J-@CAWVnpb+_2;Cww$A>VsI_ zgF4G!8`)Mnrn0P%4iw!7ouZ_*uWU0Ro=-4-DO%gEe>=0cU$W-IC!E}; zM#v+Ogl!aaD(=vgH|rhKgE7Ts<%oV*g={Tzk)Bk`0X8KhOk50rwy8dUXFn0w%_|>G1Sd>tB=}5 zdpo7~ic0Wj6E`Q9BkCy;{>TI(8sgzC3eGOol2QXL)RlMvjh%np7v`{9Ins9dgp2Pc zC3vijQ#C5ce~zd*8j4!00Wa95Lrd8-Ng!j6!j^4;aBx(8*l8Ok?Tar@k{IJv`^hd3GQt>h^=fr?5eeAWMbh{}o15V< zT%6W`ST!EmoPgvoxPn+b`|mTRHw0ZeBnBH)I4#Y3CwwH?4_G$7)mwO`^ylYpc=k1- zd~ZaAShSnt;C|1%Cx4cxB@=^rv9m8zYT1PTVKLoVPOb1t3FLdm!DrE04X%?tco^An z_^J~iaCpOpeYm(dpyRYZ8Mv+=gH0tULC%0MDbv$T1 zEP-KE*0=Hd+t%-IhF3v?3n4@T=Yb%$IBPrFS-0x!Kz*l92#p#a`uK#hIrI;14oP01 zzJIgYfIt*rtn%N{WTj+Z5by-T=)Bi^KD)LjC6kzWdKH3*_MrHH@&iLQk#p@J!QFFM zq&rK<^3@ykp+q)7G3uyV4W_(tCalzOz|9`z2HSWrkB+hF%NoE~X4)y&umG_pg%P;S zmI(lVpOFAYd0IVx0|We0d9sLft;qbv`{xX*H5&b|q51J)pIz5ONa@MQuaq7icF4(-y z#HsZm|A|74!m4(1GDUqjZDYiX3`X?%2^1L-@v{(b3bK_#%VEL^DXKKye2VDKuTa`= zN&#ZZdV`S&xgmW_a7EL{#*)azm&;?y2b0{w&@mmMc&=+Wp=;j*(iB+da&K2`e?HB6 zpXLHYi}GIOP0DT8ag}Tg*G?JPmUP-8lI#L!)~55;UONeyx#UOL`eRCVJVP{XTr^0QLAZ zRh)PTYdztaXT`C(Uj)B?eU(#H^(XoG>Rh_wkV1Z-zOu@6XLwU>Ti4opWO^t~IboZ} zP0^*w*jhWK_D?W8yn9?dMqKPQT>0H&EfP6bs>mf;jzf~8Md;&iX~Tgqf^*Nd=P}Xg zVprmyS(2WcfupQv=+o}lYvpYC8yUxhmmN6CoWs1_*6p<9&2F97GLoKDyau{4_xVS4 z98Zl&WX*T%rgy{-XTe|ROP}8g+X}XWgoonNthY~T(=ybKJ*%2@f+-lHH+sW(9&i`>BZ&6B(b@LA+;U+6QjsIPj@-p9$W(hN#uS#fQ2h*IME;t*{ zjrR9^B6s+^hR@dJx-D_5FzpZ^I+NsbR5krVtaFH8CcT4(yCRafoaAq3M zvq(G&hfOysx1W_)H7ZC?oV~oVe}9dDb>ZxV&{B^cKX4x!we$D#rR zTuRRBm}2dk_}t8Lt|9ELOwWz_tDinKI#NN*CD+l87v`3Nmp3ScS0EMB56CtGqymm1 z1xmDBj@)L{z;=QK70z#{D3TK`Th)wn&WO1{I|l?wVTNy>S5QAMrnN23jb$gjQVvPV zW3wph!An;#+fqN8Iza)<3nXL^u>zM)2$g2sJ61fi|H-iARp(%3NR2IlZFt z|60Lg3hmzh02{xs(~lpbdcD)AjK14vWj~vQzYB3$u{+w~2#soa+scKNe#&sRQ@#dN zw`mtke}S~=+pI&ol1nX9YTSPDon1=G2uAT7hfM2{2$Lm8FW@Vn?PE6Fsaske$Q==Y z8+CQJrr-IE zxrq9ubp5c!tBW$8PtV(#5l$R8g%yNme60Te>7Z?{=bJeKk(7+rwl=~k1UABQ562<6 zznaC>>n)y0Wz_;!*a2cRuGC7(;E3TsoO^{K_;$r9F_E}ali?o1yw0U9H&W_aj|Ek* zMeht~=eSu|^DTa!_V#RJK)q7=8#v#~mLMcP*-6ZBw!42QBcPVkKLU!MYGAR}-x0kw zew@2MsICq3is`f~VqCO9-N|A4tbuwg^EX`eYZ#7liWVX*@ZZ7icpsj6r~k8)d|Wim zr4(5%pX{|Bi(yc^*;zOzqCNyv$I4BXLq1Ds2CUE{jhYw$T?B0DQ0iDghjB{b^6z`i zde!+FRY#%=FnCs^W<801&n|V-+OLpXp3N$u*G()9i_L?*fUoikzwP%fU$ghZW+cf; zgXh>Y?3pDW+apmOnbUe8OfEbK_HS&k<&-&m5JUw^6z@@qKE*j7#|Sk!1)tfO_Sz)Y zShjmaF4dQx>fXu_sqy{WNZ{mOX%~}d4hY(EF==>fZ`Cf zCk>Mv$eWaE3AqR9B(A0~za8S^VQtr*^wgr9&$14up>Z89B+5Scu;RP^FBZVtri&$C zi1_2=CpBwhV0BF0q^P8I1g1E7Iwk-ej3OXw2oy#&bDUv1Fm1yn@W7&A?jiZ}uM+8h zUz0@jOOFlI8KLf&wG|bW6cSm%D+wTX%GWwDi{P2AtKMeE6KScKmHMhqZgIb6HR;3} zQv}hq0%;uL;pwJxx156i9}C`!)(Oq`t<&T;bEF~CB-?*RE3l#pr?_o3pEdUiJ_19l zybFV=S43*K#KRwB2H;dmOc~t1f^fO1lKxLh0}6_Ol{%WxlvlvGZtG&Cb+jT>9Bd2+ zbBve-=Fgee{-d37aRZ-df|FQd;W%K_Y%)I`Nkim24xv`~?_%ViFLk0#5PQ)%ef)8Q z74FK@d*8mj`QTika5lPFK}Wj4yLwO@;h3KO@Hnue{1rVV512R}P4U}=gF6%Z7<{R< zYVee7cLn1{3ebjX{;SaW=eyT=WMBcvq0?kW)~Eb^L$bZaDoP2+ zw}tlx(|f3zwMZ1)DrRj;|MyFHxz^KrARm4V5IEZ1(+^8!rtzGq?->rN%fV^tDK)&^ zFqLt&;`8g!tkxtIkG0ef;fu;B%afPC$J?mOl zs!*MwN;F!@1EckHIDZxo&^cdASiLVC(nw7q^Vwq-$gvJac&4e1?@0@g)g;_N+Hdhi zZG0kH8uNJ-DIGwt8`}Ad$7a7sA5yzd^rTwkM4X*YjwySQPLeB98zBR=99ZLW;K#7? zc(mJR3#7Kw&X&|fS@%=qT1jiCRamwl5D1&y@4>TbShSvqe<8@1gx=U&=_JEU z_$mBe+N{e09+!g7KUII`=b#y9VRg*UMw_g$N6pxjM6JTLRj4E$&Ze@QLN(#wgMEvA zPCPs2P(BL&}I<7tLM7h=hk#znh-uIKHdKVECEB8NcPSCTSME#u+ zb;tb@ho$hu(i^~L(>_Etdq`+7LOM#5(RaJvl3_|680Bjn6MLM$%iCADvYHFfuUdpq zIp~c^JX44BJ(siU>49p(N!9XTHqGvRf@0GP23*eTMXMa;b!EVs7w*ep1Emw8XLs9q zrvNM7^Rq=k;JS|3J5dkl$L9OB+M=ha9gUL3l4n_*kyH2bOxguwi7YoT=6rank%s#a z&oYDY->%_41(~3~jce*S9;}o;oCoeqA@G57IQg|7hVD#6iL*(|ljD)_w!;!t94*yN z#n0@p<`yqFjgQnFN>^M))4BWPq{GYCoYP#>Dn>p{iBNx7Dr^t9#-^avb}|o%7T2A2({edUoR$4$E>rEBW0`d>JgR`0!Hz?gMO$>Gk>o zE|*pB1-HLjE)pkc=4%6h<9+ynK-(*#CfWK&Wj(-|6w2bf;KEO2+76~9r)d&hd zc;`4L7)vv|9~EuQ>y^Zp@N3Co7JsvX$B5!QmUiCTvUJ;6@q=r(%yw*j6A`W&H+M!l zTH;6LO%qormwf!teqfe*MBzG*2X>P7S%G_>FCilV_n#>QTux!sz>xK=S5N;CzcIz%xAIkNx+00T!h52M&8h2gH39hV=TPiKsFgtT3UPRZNXXiSq@4fAx z^i>Gi&Gn@eaAvXPXShx@e@ZhyZjaw(3Ea(47LHw13At;%-K-)VB9=B{ZBv3PBotTl zwRO7CZDS~7qu_koj5nUuQCf|}{M2+YjZE_zDlgA=xX`ix31zOl4E<24dM7o=rqiUs zYMO1eL{=srTA+@9DZIjv<_4V7=&vM5R37T?2y3^o}Gd5Edi~mBt70$D{B~P`rU6^ zNyNcBSrneuk0&d$*7{{*nK2)6N3_pg3it~AQ&57rsn@KwXe zu=k1$*b5%$MIzxG{_%s-ZY8O&BK9%iEn5PQaRT9`Eox7!M*^!0Z)rg!s+*0-*JHva z!c(H$&W+h0b=czC31V;99z0Hr)gNzKkrwSS-&Kss0|#b6onj4aTj z3(-gj)ld=$35zHwJN<2lhK-n=?m`n3kg)NWL?G?Lj_Yw`2z>bkLse(UCkq3B%o?q> z|aaJL>Y4a3%?N^!(w&Bgq9xJ&8TL6 zR#8Auu{kv!gh*Tw%WNJ=>vhkHR;h&#jQLO~1ZES;7#z;`H+UH<3m@&*`9cgN5p1CP z0n+p0XQdLcAt!VL;oiaq2-4h#de36m-|nxaat<~6q=>9w z*yRAUwg~C9f^a$m(E_xsXE#_!V2iM*a zPG(juRxPE^qkw)qQTC^TYG&P{v_Q9jFpc5dC6`S=luAd0jHMR<+9aNX-4tDgp;&en zmxJa0)VNRw%c@^8q03({E`&|2uP_A#5OI|Ap^O-41t=e!n{mW#7i}|D^U9yo1GP5D zGCz_P0|n2nWF_;2#5{!6J~iw1-O^Rc?hnmdy1TLB0mvgw{VB&uk#aN6#iMbBl^R)n zk6;wf5Wvv*RxsmWk%`QeipVwHI}Qw(vN}aX?n!x zf#G@+VW`ugmjw6QzC0{pO3skidrWt${gG228f6sal~|2G7Gnu7%5*SEAnzAyBN8Ay5+8&YEQ$ zb?W{O7nH>ycZFx`n5V~U)ln{hhMjU^JCklAbaJdNahJixAsIFI_K*0SNS)kqIP{Q5iu1fnMznzPhK5 z5^EpRzc&6qF~b04aB+b3W-V4Z1n!?@2n8Z8ega{#qG~T)}3OO)L{X zgcqbxS5}E}M3g=%;i`oym5@ir+-xN|Z4srf|7AYO|F#&u2pL_2ys6sZe&w5v9FVdf z1v;-A;NFTT@rfgTekE&~GsQiPntq>;vmi1A*&~r> ze1T&-l<>7F1C5qL{mf@#NPt0BU~#s|XXouo22+t+T%Ku1L)|ZKEi17BmZ%b93u6Ac zC!B@XyqU|1zjXE#im&(utI$XY@PK-!>)5-e6R|lrfWx>98kujzUgOy#w_zG$%GPzn z6(jU_zO6;x!9zbzl!h^6JuoWt!Q%&cPibGbb%%S8eG#iJz#8)qRb#A}cyqWdwwmws zm$e8Da5IwfXr@hF>X_55vkJN=x=%MWoXHj1_6nseer6SJn2kfiEkzk7#-6RY)C@}- zG;*_ej{tSpTKE@)IDKp6$@Gv*J8xB2n)+Kuk4PtHriA4i*psLPq{U_Tz}w(>WvrpK z9LiIkU%F^{8@THhI28Sn@X`uz-oI*5&Ho|ak07=sF7){~A?gL1!6rkNjhuELBpU)L zsbSX2NuRo3-&rD)_0^3+xa>5~_qr?ZW=vfb+lGw&Mo*QCVo#o%5Fdb>jEUvf^3v*? z&AU6Du^e(X`^c-EIKTffVVmFert_*^@AaJv$3=;OOb*v0P4#A;rA|-Cv9v;m#K%UL zRbs)&S)ZBxMAswD8fz(lIN!f)Nj&HB7=?dDwAlP`dbAoMJ~1B*XfPv$`wNZ!QSW0Z zZW3~`~TtUt)rrf-hW{c1nKS)Y3Xi| z?vn2A?gk~LOX)_syGvl`?(Qy!8k+a;z4!j&{R0*Zvsh>Le$Ia4d?MoB`8@bs7ilIm z+Sp3LM?wLGi7tp>*(0w-^IWr~ zJ<~UojY`vOW%)JkRW<}wmv`#|#Z#UPIPvZ0NjSx$E3uH=?GDbs$A@2g;&sP2s`q2* zyf1DYzr>p03Jau4_@?1!R#9t<_uOn%9?I0R5HTVa()`Gac&t`&w9fDE8N%0tvxu0` zawO%?rbPEs9^c=|x#?c($7W(;xRv=-hiM0;Gjy`QlJw9V)94fMKBI0i%ljjCwZ}zh zrx(k4K3C0#;b%I5cuH*2(Ha%Ui>?%=u93PA=)tNeyUJ>nA54~h^gi>l%Z!~>ca*`M z-{M@Jy-#g5MJMN{PhsNCznE<0k1LvgZE+`P&pwTP9D1!5X!_euP5f7@W?fwt^dAB1d{M~Adk!^U z6vaD1-6%LnE#~arisxmTlZ?FW4LDIPlFBDuog1F#_xW@~`;Wa{z5ij%Gg8aG2`3e& z$#OjitcC9w(0{%<4oS#_ef7_6KJXtcv#~pIcQ$26eP@_}jA8 zdgksi@*+;Zk-y-L2C{W>ed|JbWY5d#JaZQFf2Bbm&YabcGVcvdSsD?yHu)MKwII9f){?x z?s$xiER`b^dQpxK`V^EBBxDFk0Ek$Aup|A&smiIn%6?oHURGvSvS)p4y&t+)IGwg< zJ!Niel_t2oO?8Y(!SIDjCS8P(nUb{tMTUin^`o<4+yyE0#|Y-nGJ7}v9o~DR8KeuV z87}*R2f7EX&yz&_3f*z*DG)(LUu#ZE6rpM!jzHXus*YyZFQY{A3VoK!%50cR253Fi zkKXx2L!daND%+FWhCWn8b6W%CnDJJs`I1cmTZ)G|QWgih_o?EBcZSft=h@@T2FN8P zNj6Pw#G^wKn8#g#-;H9yO7DoqY0H~30ZUfU)pt>Xsc?xOEq;OOvn3Tp&TA`o{`(Ubn6sP zONkXUA-aw2ShBSMbH1T3cCqh}Y`t74L3!ps%z3k$M!yub92$T*FZ);|Z&iw#){grw zk|lmEOujhD7%9}4>d@}s1Eo66kc^>xlAW^7yl%%`LcLk=BV*fmjxFW(GuHz;mSBRB zeT%8D!-keEFdcIKH?O2xl*kf8`pC~LJ2 z#v{~?E{cz!U|@^f2T?|B)El6YU7R^pX9FZdhLTKI?t|}#cmDoP3m<^trUl%-&tgFM zvorXAaA(suGJ29Y@EZfTIA{BgDE8N?+8&+@MUFA0N4xYbQrc2?i^bgz2HY#^C?DU? zW+rNN%I{__HmA9nH%&P*Bj`hkPT^9sIy?#@i5ousBbVs9(Ghv*Gv*#?sFL4pbo7Ke zD7+O)Pa?+TfMHl2DlAzqHmEc^K0l?aOf1I3w1LF7mxX!LjvFoPb5GZ)M)6JK@ipE1 z2_JF(jDz}eG{O~^VHWxnP2_3I{NO#Kdy=LU-jM34G!*3p^Ge6%X5k>A7~4l{h7NAQ zj;bJy4I-@6bA!1S8ZJ%U`qLI=x|Q&O{z)*h$b^-qBy&<&bTaruFfA>8>_p3xKl0i0 zjMtaulr<(_Rh6EZ;4MR(;cf0|b0~GpAz2`PG{oJ;$;mgu;qyhNNLPQDNm1lqOik`d zaH@EWsz~}ep2sH9zIDM7+fXl^RNr1(L>e|VRA|^CWw{dO0EjvY@5kSnyFgb{5dA)7OTF%fNdavPM``STTXz^Dkgp7*77F-AQ- zDQve11|QNYV>ligh`U;VvgGSFgaeOCK>&YXG>7;QHuUPBsK4g4b4hiP&^L2JK}vnn z(F=)?o!<_U*yt++=}Wf+2g}qcpDM!dx0qE~)fDPCPcv$@{hI~q5uV0%kAR1o5a z2`E8N?)h{@u=(G@w1jF9f8!W}?Y}?Yx6j9zNvS6vm~%8b6eDVvvbAC^9ALMm)?&#Z zfm?=+XU4b4^*^>Td;Ts`d}FoS``R>y&?o9-+u3e(hDJTqI32D9XN*_i5oDef{4HZ` z@{7{96?b*)Nb&xuHR$NPp8PUQYp1b9c6IEz>FTHon?crZ*2C5NBvE9eb?bjC_I_y& zpS(~deWU(WY0o{61irlOvZb_jsk^P=77N>+geOYVED8#gC=)*s%9NM1JsIUi>!rzF zw_IZqf}gc)TPYirGv9_gtwFLlcS+9vB{~xuM*hUzWxL%rAdqaOld-QlcGSquBlV>$@j~(xejl$^x2tWO9^SavJMql z9N1POUh{6=(p<5q{8H3sYk#N8#5E~oCz9wlnf786boKtkEdzJ;(_Du9@ontjRrVMn z`T~n?Uqiiec5?x6Qo$jU5L9EO*>UyE8Y6Tu6|Sy(0^E7H*l*{RJ-&AvySiQ;SFLM* z-njGmML2$@^i~758+(TjLwDX*Q_Zt-Do$B-4P4iys)w7VuSycuEcz@&AbufOW#?ECPF$eU&MA4m2-TDd*ztgg)e)<;p)m=nWYQ`Ue+-NyBnWes zk+DH#?bjS0CX}1rwh@W`HW@N*Zal%hzL6vm-muj=8(}O%kAoCWxR{U>ZFB+4S%kB* zccJ3Ytn$H6j6HIo`+Pw?&bM}pRz2CK%73Kc}{4NkX7rttX)OT1xrle|HP9 z_)gdS@Zc0%#YJ$X9JQbH8l%^8Hs%fChK%Z(uiOF|`YBH;5YV{!S;pgY^;L}7+M4K5 zae=R<_zURUpY7QWF6{xEiS^>hMrsQp8%&g6-nfbBYEf2?zElENI7M(m1FMoJnb##= zwyEszNIOtSJGgd?SqU=kO7_{4gs!*)mhz285hwH*32hsYUGg46G1w`7y$iFwHJvTH zno8hQat$8S><*%+H5K{v(g}@+n<#9w$5E?w>~s^5(31Gf3>yDzT+Q`@I%*6V3f&W< zYqo4F)?u;|WuOC<^X8uIo7-UaBgfh!U|}08Ghk_d-N|ieXxI)dOrJLVnV2vu9(?T9 zc-t0$GgNd^G1&tzzN0!buVAS=tUQ6Ol1n`N>ptGHQ?qBr2>Ks{beyh)qkz6bzI+BD z$;Di*B=RGqN~>sHJ;L2Qu;j?SixD9rl8~2FsCCYkX8F4m$+268UN|y9oTt6};oP)- zXw3-Xxw*_uZ$O)g{lDeUEkK-SiZQIe1?gBEgn$Xp5cNd8q{%`+F#7_VdJclbxyp|; zB>{;S!wG9|CxJ|&pOVjT=!UBT&A9y`+;tgOxddW^cy@Dx76VomourdXFM7AI@DxpB zYpZ)-{x#8flool(9V^@dOJwRI0#~2CjL@oe?xO8ywtaNixfMmv*uAF<=?+luOxIS--YznU$p=TLJy4=gq?QGD4*7H!BvLGQ#Cb z4*U;{+N~|TYqM(ZGd??eLaAuCUc#OaTPSzP;zydcNPM*`zE z9JK^&$=T-IMl!Fzd4uo9v$*^3BO7-KsVCA-#8D8IJ3DfBvGx;;BO;HfQ;Jo8mHpz9 z0vPjQV$3=G(E^f;uOzCPqnZRy^9Eua^Nj|NT4T2bX)~HaOc{m-#Db8Qm>X(hS|)A< zf2)~+ErdFFMD1&Rj$S;s8_-Mx(G?_`vV`L@z3dPVhXjd7nuvR;ud{%r+1oq6Yu(74 zI74aGZxZ6x1{NDFb)GFv!#g=k-s{JqKNI*@ETGV8d9;tzZ;pt)VF7L^;Q9f9+qvt9 zMbU&~*%c%%W!cSkr~iLg07$9P@5+I>m319o>v^z{s5aS_Dlqv_1WYx79;!Lwq>99( z9zyGW#ykBluxd^OltnS>N1lpi&Bvp3-HT5+{}e7FK1DlvO+L8*Ek3nPxVun|A>ndW zwufaN&<*&;z{mxJ!@s!eW3|IrhllSrCC9C9vFwk3tRgI$v6Uj5;r$7pD4B!zZIIvR z{!dMT-R7+Do~=DBev7%PML_5?D4mh`#BZw9c{~fcLSN$|93_Xm@mpxBBs0#B1;1RR z_H7t7j`YDM_ko3#9cLe0Gncza<7=a>yVjsGe;8C0C266Q=zinRS9QAV!sPhdB{|pd z`?=pwJWD~!BXQO7ha3pY-{X-^tf-zNRfKlCO$X)+rl4)4r3|NO7lpJx zhi9A<4*gmN|HNj1%IyWaITWs?=N*3Zafed*`{bzBJ@=|iCAb=U)v}bKW|xgh$o78E zk!WaxQ);8<`W|IE^7&(nLD+V=iDn8ZcJ_tk!~4KD7=S&Y^_bnq0rS$dxt5;J$%d3J zna>TTXqee}KgRJLl_rr_qsLvJG|}bG%lwvJI4&ZxA<7gr7kY1sYuwP73&93=q;Er_nC}k zerB!+r-sk&+Om^p8@bA3gy-FmQ?R48oj*628Z^gyu0DT+2iK|P^EhQ>%=G`_>gWrt z?-`9=lRkagvaGr?4``WFhu>+nuV^wgs2)JB_wfRw{%ZEjhtzu%sSi!8hD}kXRDU9l z$fIPo<*uZDk$)KDg|l^jty4@K==;(=jEya8k8)$nS;E8Yxcl*X9HI450%LbhyD1v` z+C<-CrV$VHFwdw$>EhPRfG$$iI}pG(==n)l8MGxF z4Cds_8=+o}^rwU3hXRXF`B^%B!TL+5GUK^@)>P!Q;ZOR#@Zh-|OE|DnoXGe%1=0aM zUe{pZDuhMiRVFeIywfuk9YL*p~huTm|Q)u+)k=s{WauJQ| z@-^LJ!;vA{{zBftXH6u3vH2M~3mMVkd>Lkb9Ff}eSbJMgcQ`03p%W;2g}DBc;_}Gg z7(_`>igXm-XLDZuQkGs$AJxtXb_Xe}C=rEE1-sh$*Bl>-t-o&4JcMM&H8X_PXd{L! zq^sGhvDg|6^YJ`;9^Vha=uU*p1^TsLfh5s7zw_jB;BvgR5AdCx3V64q9sKRmvMXOX zz(cS8!MwaliV2W-G&W>EC$hbpY^-ncryKDmhr!2a~Aa9-67@lpOQ3p-&a%`M0Pn!L~yOPr8e_t_SI$w~^ z-D<_^J>cw!xLcA!5knwHw8l(4wq7u5ko8^qL%#(@K5UBgOILl_fEJUK@qIh?Zdm2c z3FWMvR8W_hi;cxKXZBCDp!tRbW+y{pordcc(g0dGPaqsL86sBo|0!fnF8{{Wz z|2oyV87{J?UAPgsDC6tT+$0{4KWa5c|M`z9|q2IR1(X7>OztZZ=m zLlshO%pQ0`=rImFPoLL#Fh^LbBux51p#lod^XP~n#BXXveIUXGfUA) z7eAUw5>=-67!s&{q1gR>9l0!ZaSGMrlw@#mNRgShdwm(;&!1g+IhnBJZwyx>FC=tE zw~%$5wc-0$_~i~&EzEBGN%bVEhf!D)OY2d`H#GZ4%USjK(L{^wqzE za)p;0{aeuJ&jnf$Vr^%n($oeDx*vB$q&P2hVY)y}#IMuP+$lsD=}c4ijUz~KIkGy~ zZb*upz7MR=G57mAdx_;baX~)fn4;kqTk9=NpPF=H-5#2@gp#)Wdg>f)Vq6jNC5@P5 z&R)KuEp4EZE9tW2#1^CWQYvM?ZvnKupKL`fuSojdw*&0lc!g)jk=X7{^2&^23@-9A zeLmr-htsI$XM|P*8*1*F244S=3l|VmADgVyS^?&jfB*^l5MF>o;C|my=spXuwYBjz zeps`s+)16a*dY7uNfu12bc=kRJE}6wD<$QBAJfLd;I0N`ykf%zrZq2j=^OEEb$o}m zxKc?od%hPu@?Z(U4cw)Q*^P*b_c!~{%5INi;v@>sZ}V<8@y{N> zeS7t*-)e}=V_)~NqCIK_LTfMnZu=8b&4}mg&ktD0P$qd=kSaf-vEi-WZ~~Ne95pW8 z$ovCG9ID;8-z~9+lGwLTdrnEGmG>3)$K2&|xCl>~| zU|*AMAr(uROdM$3BGw1VZ^d`KH6PyRcb?&5>5-IGQpL~$ ziPED?eZ@YMWUNDmGg}-egQPA#ZS&6uXgLhH9gW4+C^Cb|UR$^3PxFlzI|I-5c9xQ? zK681cp5dwf0{Jjie4=)w^g;G|y5eCN8o{lh)A&&K!DJlOP@q3KbjLR{UXsV$IV!}R z^pRj?y=wIq#s7{dEyw|Eo)><5-hscl0B9sbD{g#+Jrwx&@6sfv%H!%8?(KNw0hk=C zrDbJhP|dDK7`W}QbAF2rb(qIB6DwahfvSz+=n>WS+HVJt_%mA?V~_-=%8j7HOvF|2#I)r$CKZI!Q9f4}4+q5g6YyS&Le@1?T5229Wp zSAL#j4%G7yo^>tKz9H41Hc3YRBD28!;UAl^VLh$-r*`7~V6eSE4a4d&6UOgrD;Up2 zZM1*}wwt8JX~1$tXX#^@t@b)*pEq3_T)SF{qlybzacAv^n%JfvWhSL ziJvsY1=rT9^&5ccBol*<8UK>1vtfLN+jJqGY@kYL3DhySpgAa=mWfBKD6K+uT7k9Y zlXlv(^TW_@af59&;R*k-hWdG;@vFa5Ps?*`X=v(N6Led%Q?lL&(80}V^&w8N3H?c9 zK4-Gy&C0dJt9Wq89v_aGi)>9F7ZNg@*68u+QJatvw}dKO{hgdCpLHJPfc>Vfv4)zhztxofB7iVn)&%atY#Gn*G{gd z*MfOD5zxRRAHoN!O8ep7z1!{}Ai z8cKF5Zvh(<3)yu2;Z$0M1JACEXlk$#MbSE;*I_uE4_V7D071|(Dt7#kqOM>3pNf?f z2kC?wvwi)AXF2bcGOdVUo|VD(j6ugBvJo2cm5TocZ#^1$ulVJwgfD#?AwI%C+#zeK zn?1*GE4(oVrM3Lq2gsEE`Z{ozY`K>3smz&1euVgvIi*B^vcIjgRT}~ZWtaxD{l5sE zYN`;%XL2t$H<(6tG`7c$l>U8N={yWO`Tk$bk^h1lW8R4G)v-uUpM#qVi1Y-U&bJiz z`dw5rgqEmS>H$Y9$%S)QN7uc)rO8jy`9Y1CXKj2l`a4p52ZSR=WsG&sL*IG|NQaAA z4eEV58%Y5EJaB6PtM{L8eM~D8em^lCNDV~%AAR^c9ws~ed=xU-o$u;8d^0oQyW<`u z9q%^RPxLE0B!UIxASC*>YuD%UI$<5l4K@R86VD;DzZ98GEfxhfYPsfXE*%1L4+rYp zk`L>wtoLZj$Ey9W&-I@Vmv$3IZEiUFoW^C~6OJv58}}&BnOHOiQZu2Gylqo9 zS?2z$V{r)GRnUC*%axh|rF6ZcHGNBcb=W+_#eqXcfcbpvoB*Lf2M^K! zw#cb`ko!iI5@l9EL|{AVcyTDR_HWe+FVeem+=-p(f~qs|Fn8B~r=ygSGHSwFiC0S% z(=uv5exnFv+Kn)_%F9gsZ$1%Mc#x>1E{DD8)4?ZznEwvtd_{Df1sYdICWYIKDl>zR z&1K!V(e@5*E}0RX=Uyv=*X{jgegz@7+qBe0=en3en!?MFLyPFik38=9W;znl`MGeE zvz4X;aJb5mOA%MnwsFKb_Pn3FWYc-nW^%^obaJT2$m@;`=CbjZ&MbyGKYGdBse7myY0(qy? ze|9BqAuYG44)HtjF}%CD`}aK3%OCdq!s7c3?c}ODD0I)T(pa89Y%RWVVNVj7A#&gU zO^&*2=bOAAR@J%(#{Ga*2Jbzw-@|c9W##@%A2E`bA&iHm&p$VHDpI@09N`bqMqPe) zX-)zDoBYe8#idqwX%+u!WVZjLavl-<17j*WCYy;DtenG&AUR>9OUoqFk+ouxTZ zC;}M{-1`6H4grcocZ&w~gex^z%)-&W1tBy&2c3e`i;0xt%1ZNtTY13NwSR@qAop-p zi%Kzj1Q1#&*YEr(@Cso9eidzVKP#^644%)23!AycxB}7nHVUP(>=aE_>a~^4^X$H$ zd%9lq6UyRV3ZGg(tnaBnmJ(+U{`KdouKS)*ZgW+coc0wRd50yn>q7h8JVP~)vU zDmo`Oow;Rqt`ogdg&iNudOqIOp7l1AKj>f`jN@RqzJ3HED(a(8iW|#_7i7$`bNnIU zM^Kn9c5;yh=`tU8oC7b^*9$@K7mGvY`Hp9TvLvRwWDSS+aHs`(ZSGM4+gcbNcI5lf z8r%lgo)z! zN@fklqmOAma4!#_mq`&_xNLtxEUb*Jb)SHKLOKnX!c8Q5d%eS{d`X=!ckGfJFeZj; z08L`yc<26sP!yL6 zZ6)+PE-Ldnc6Ne3l`|g|c?C*~I<~*xTz_~+#Lwp<-QesL79Z)OxbbW8zt6k3pPZIJ zZe{q-i!dUj1R)j|e6)fPa@wJ``-$9V}>fo!(I`un((8V*w~<19gy0BHn+r9^R0H#RaYF^Aq{i!iFdBLuR?6&a$#7 zWk)JfMLp8e(o)Nf^&gL6LVbXQHF#rmg3vB(-NRcYF2GCbUWGcEKH9tT+^zG7uho5K zx}%6XMuyt?#Ruzq{jB1>pP1>ItF;$Cb!S6EZ|V#+EhA%Itw=$wNWb$7zr9|izh*;C z^U1u=B}}WJhaN{P8dqd`I^pl%zppQ^O$Xz0Rkh0AOe$jsPu_(edNh#& z`?SA9taA^xHOik^of-?u*$7T}l*bw98ftXtS>6(J@JjFQnDO-WBB0dpJ>~U;BDGHq zxxm&5^yW>U)w9vUPX9h7Ql}e6zuB><<+{QV{DF z>USia|QfUBfeTcTZr z(g@4TTr7~y%y5tO=@bYXagk%k>S~FsWyS7X^ z5Tw|5K}xK6&ZRY`K5*y}OWGCS%3t=1Gv`gm#D@O*mt2f;1?4!x;*(K+wkarzOS2^9 ze#gd2o?6%Ped=A&5Da%v$xR+c6Iqi#IqmS0!dAQo6wSyh>T~vLWPUozReVFUCkm)# zA;%n({axw!*{6U53>h6G_TLKR{7wkqUEgb5O-23GeVDyR1(WA_k>ujX;xEmge!l?) z|GRSencnvtV}E^gGL|ek1Eo_dVJ-7{{TXgjcA-f)5b$&#?Z1p{=MPv`8=1OkiBwQX z4x_%9?D-KBgMb?`f%)2Xv9&ol)flz?nK4Q4tJAS6mz@RfLfS+GAw6{WQa^U&9=}l5 zGnAwx^*KIsobL?$cqYRi8x7QJ!A{>~hdy1+W8f@pWz{O4f%uiHVb0sFm26FEh=u_WB|B)|d2Y<)0KICK}HUi-{ zn*~BzN|{}K)fhrqWo6|E8fKGvSKITymX+*S4|XBfp#i?eZI>pY=kwb(D{yfU+hT`3 zd|3kG)P_(7lsr{4?OjajSl_x9g+*#fvz2oAitr;!jH;Uc_-1_)uG7&s%KE0g1hu4$ z9_5*nq=C63WdYzmv0>-QPr5OMf4gO7@*i~YQvc@ye{}Mp?a~|{lmxMCu6MUWT>Z{P z79JI%29Q+MRo|akZgdI&ndZ`D3-aV``SG1XCNGQA@M!BUOjryG(!|88p^GCkVL%b~*m>JIL^?#YedZ)$304v7X_$>Ri!zjPn7 zBG4)rINXH#4v7O>K?%r`N?fNIqJ*cr) z&i26fPr=V3%u>ZnL?}|YWF1QG`zV{Pc44$|FxUt0_4)IMG3@6pefPd4Z2cf8?v?u^ zBJbjGI)Gj8A(qPH>cYWF4yZS*sJnpa*8W=x+ys7U!a} zpOA!m$csx$KOdCkyY>12Ra3qs6H}JY3pNok@lwlkuf)RreXgp%fGZvoqn@~xG`xWs zNl(WK@w@T>$OTYXjyp&M&vZyKxq|`*`_qgh0WL4v-H9P*`7__UA_f1Epe^)WVm9`B zVnsX`7LwOAX~hYU+D>Sltc9TOYeiWhTjo>JWc8F=mCy%Cy`oxD~fT67E0`%eh6{r0-j z-@O08RP6&{T-WU=t?0G>MPXTAvwV!-vSG?%b!^*qP*#y=r1)*%MrATYZokBcAc}Bi zkgjdsbUiAIBNNJtFg4Be^X5xBT10b&X4Gq8ejS{^MF2^df*vgo7{3V83G+3%P14Eu zK=q64_+mpjiXLgfMPe+cXVuf7Yv=Q}kgj4y4vGM;k$_)rkB34qek(nJ+@pOa z&EudZBo(MWc&`s(_%hj;K%dBDA1nT*R&65l! zo@%a=sXa-iR^M1wrvo9vQG-{Xu2xU`IzX<*RqVnK_0?iv5K5%J@af5Se>`(;-iTcm zgME3Cc&!!M@!U3d?P^xR2WTXSim)-*Gu)!|5_=Y59ZFNaty5T=5KyXr*sypVYP|nJ z^UfN!4D=lTIy14K`i?E#!oJ@S{Am+Jjvh!__mcefbxTatqUz=~nSseyq`%<&UzwoA zSsBibI)ij(=P9K^f3$@syWt$c9txQ8tnkA8kF^X&vpWQaA;61MGJ-3w@TkErG0pNd zcEkwpx=1hp1tyyafBu#E+>hWy3U~q`^z``-FTEMKHPU4uk=4a@n$e~s_kSn+5un)} z%!i>>%bn!P_C3Psksm}nzlgzJN{^-g(HwMbIBuT}G~|e`|13K^`|L{cZJ1iGNkbwJ z42kr7<~j_#_dgaC#X7$%GU|FG18MEksO0>5ZF6a7*Uun}gORE+T-q4Hz$+7PnR)%p zPkWn7skIkXBybAc}OqLoYIP4CSvSp7pLSE(9xRECw1IQH((KJ9A53nh+V zdMFt$d>uk54{Ff;CDHOE^~cMpysPjX%ODw+bnb_AZu1zQtldBzwAQ=+e?(veK)x;3 zvGY$3cw(GL_=JZT(k7YMb`NQys6^=HVJuP42=)2adBJWdJ8enN(`(gCovccS9qQ6V z0Sx2*XY#0zI@)b=6Lo*eP}9)3itS>uJ7IoD!|Rc>kj!X#!W?`j{dTmH9l@#G)tKKU z#Cq6PJVGi8_d($0G;~|-I}DfUI;{jtkq%=5m*LMMH<658xt?rf``L2r6FcCydU1U^ zu75%|`6b`!h$AI~V}XRGXk;y)JHQsV2*L=m*dA!UCU~nBnkB+o7ctq9I6zc8xn=*GcOres?EUwEST3sNE ztr&X}Gx~`pxP>{J2%em$-AK9zv*3*I1n3mY(5SF~GAHt95Typbzn2{_7 z8mhWp>&?GOCaT!F-S;dV6=|R&#CP1D?)wSW1?=Z36c+#g1tY7J4&}awzvDKdBc-+l z_Bq&+?KhwUeNRcA+5esJqHW+*o7{l7`f=vz>N2D!O4i5VR+1O&PMlF(`NiQx>0NPUY4vkg zYg*~W$l>Hzd}$yEligYkZ(KOP7n-9|luNR-lc_D9`ArY?G_i~C|6u|CD_DOTWDLt- zON!@;JX@`@0Z7|*BJ7Z1nVZ7O{AYQ+Rx}aa+)HD0V!gH|H@tV;?nk0#OIjwsGBWjm*$`RqyY8i>-vu@TyD6;lBCG8$a#8OJX6e0^Eoe(G^5oTrL^e$Q529vOc@f{{Cnb6k;9& z)6SBUcmTBcQ>Qd=b&h{?;K72}A!fIff4(i;)@+|_z*V^H9j#sp35}Pa#eYP5MtmKMab{l+;fzrn! zTvc!U)vjA0rGJR(jSpAn54G`sY~?LPgD(EyVN72Q^5TsY3pN=EjRNm#Lj}7BS5}at zHxSiFX}3(c$IRv(Ah5%TQ^sRphM#k;{&9#Pd`YiomH&j8XnPQ+6`h&S=fn2Jo7zgt zi)f9X*v}@bx2X|_6tD%y=B6RC%?DZ#fXNu7S0SmDJog6sF-Jz0WpMj9OX8AodtL}}eQ?K%~Ju0~Mu zzYGh?EVD1-SXTy=W^MnAsh|PP`=lx3qRXC=UU}|0^p9$?{1Q+p(pz(|2^M%3?x#-p zovu&4`-mO&D2fzXn~tlvqRevs`t$A!CiXlh${y@(2o9vzVg6+y!KHz>;+0F;x3KQ~ zEca9kE%LmoD&OxFegCX8CbGXY{8>3M1v}Ye&O`Y+R}N{a`nBL=61AC1UgI>c&Ngg} z@Uk*X-x!Vg&>wM)Y{I;6DT#fzC_;|6NoV5>D=7Adc)K}fQhlDVzORfb*-?}o?xfC> zXqT4?`|1o^{U$r%_~`@XLY37gDVC34Dtu47raYAUjmuB`gfoyJ3ANeijy&TvVBXcU0OUcjqax7L+mXJGn{i zdJ6ZI?-#sc$hK;(^ZkLQFvO9)N-&qqZE0t8w7mO7pE!vX0446czb9qdJHn04UxmOR z9(46=f5TK+Fx~m%6l9{KbvZ6~*%o68`g`E%7N08Zvd!#H+!J*G+KDIVZ?=fZL(=Ik zLp1p7YD}P`aBXsWh4_&?rYq|HqinY=ju3Hc+Dal85UPLAC8;*6eUcG?D(to!*z6D4WZ=D>*dhenpe#-^QPVvdtVy)AJMl z8KHEK!ttH2${y!aqT*+}>exHCR&o(Ce|3de28BhtcP(Go`}%~;Lle)LGk7AijqLjb zsVasf?2fskVyau1z=mL0^O)tZ{V7YFPUg(fk`XDU_p%BYWfm2m$VZ|un6pBz3=R}O z>}X~s4LpRemsdsiEXC4ZfLI?E1)XJb@=j={xg^kE=ai$Y9sZir@9c$`7mmgD{*1Wq zk}l`X5F32@z+SxeP3?!%h0f@TWAG>@04%~_V$%D5e{>)%=!e`tx_iT#*a<@A};UBMK5Qj*Unnj}^$N$Bb-tXIPrX#iS zN&-rJQfF*a5v;x*#;h)r73;N3Mxim-vewvn7-$VV>!8dF*@3kJ`RJSIa5(QRdOHev2hdiP=wo4qSx- zY-45@mQ>qDq4~nIW~B((K>~5KWr^I@%<(>^qz;P65;~6^86Eh9nE>EJZ2m{4?Sn?B zUxBJXuT~3JP_Voc1qJEny{{N_)OT}z`8lXM-NSZa+}@*N^YNm#+8ACf^!AA-s(_Qj z_E~FVq|x8*vXe2+`E$xpifznWWB!G^4<6#A<54vPD1B!I)$KKy8pxgGhCs5;M;Sgl}XQ|ny5KC0F;wQW;#T+QX9pik=)W{PCBc;FKFjLRFj@4 z#hWs-+1^c330-t`+sKU*YrS~jP$^W5oH{OcL2kS5lA%6SWVIvcWKy4R%#>^rTPYMJ z!gh9kI#x`{XS01DhyP>kgy9B&+;k>-@p+L>aReZ0ydL$P%YR;8zXG`R{}({Gsej?K zTJGN*6#6~QDBL-AFRtDkM_Aw_Gm&If()LsV;*V&b#6b0# zAxVn_rX-i*z(Pw&m-GyHG3%E;sPGTbioZE`-|)kuo6G+|7DCYZ;i$1xtDA;j^W2m4r zayjSp$2^s}(7*c;td&C=oD7R)fvQU!ImD9^x>XO~+rut*-Mb(Q=*m!4JMbK>aK>f) zmVCHvIAwLGKID37&NJ$E2%9-e=jH^S{K~Bi@9!F4qlRqnmG^L+vbYY+UoNLnAW>fp z%1M7(njEb9?T&^T;yTdmK?+#YXZ1v(1S%RB3@6p|%Z0KhcoZ2~R}uu}!*xR3iH$D#;CFRkfotrGw;rVOx>9H5$mMsf!YMar^>CmN|U1z2RozQA8B%Z6&wj10^u(jp=V7_7ZUrS$4XcZRDC4I zD;eFPqIukpEM_ygu#deB9Gx8o9cRRU#e~kmt!g{$K(o-SM)8IN>Eoa|6eY(mE<_8M zRMn7~;PB*hXva_(s=SXPWcgLJa9Ys3UJ{n5A#_I(;iO1Usdq)=LXFbEG}J8|IS&3f z^2(esm^|~3Kq!66gM+^S{4r~cFfx#(lOu93oa%Apkq2UOnt(^uOofKskR(@!-}{pP z>zY*A14`+!4E-))EDLESy5UDYVW|3#O*G4B}?!fH9Ly=ZTSN^x>0O3{|A4_>U+=wyNmWg4yEn6Ox85k~HlokZMow6nNVn(Vzlke4fD^hVK ze(5d-9;n`u^P@u^8~iXze6)7TtM{D!iT6R8y+m0KAt1P`5Mn-TfQvS5n6e65>5z_y ziKCqX53fqK=eC2b>Vxoz=hLe^D2>`+^K#DeMOAL0kH1@qbv1tfUaF!=^ZgJH$h^I)1#Agv=&~ZL@JFFoe=&8Qa#PYR|X} zc$>UzScydu4LlZ*rzl^vZ??0<>+yAMx(ajpY$q~nR$3b1GI$3BTx)YZzG^NTW_eqq z-Rp2AXqmp+j)7*({kB1=9*QPPuV42mclG?BBW)Vm7_vgh8EgI2xf9oiMSNnD7y;@8 z!M5g&0_P7l9Oh-Al<)~f5QEu=vP7GQqlHymYg)^j7Z{|&=t6zgA>^Ch^u^a}bccT+ zBnMUvMJnG>--O>_`YclY9lLP{j&sMpoXatSt`pVw8 zn_&-}p8yu)+xM7F9>AQQi{;E-lPH`ub55lcHUMGyy@Q9uT0eRA^pe@fif9?Z=yDobmoA% zd`-!E)20Lv$HVq=}_UDzSR zWZZSajZTGM8KZkoY9=q9Ii!<>ofG6;ShIob?nS9n_m4ShSp4V2!Q*MsTVnF8A=m`` znA=-%!To2f&Jm5YnALSd`h;dth2UI1?TWLF3O{Y~hC=Q7;n`2dfV}R+iFf)2{|OOsg5JVH84P zZhdVec?*eG^z&DPyu6RzzrR{Hwf5u=h|%!kKBkm_(1PM@B=RA-vd^LJ3oKuCqF5kU z)`(4zvZju&uJg8?f=+EoPM9J;4H)s?&{JYNt8iD7sC;FDw~M_$X1zBEKttEiq% z`F5)c3icG3Ir~@@t0{x~XOGNSRzDS^DOwdh_<6q?xt!Fc@jL`b<3*@)C&_oZq48F{ zr?w-tc@XL;GkPh$5E?VUtTZ_lA2<)@Cms0&n8thU%TL>cqOlQv8>IJ1HG(3c$Y!Me z}uSIwnzvL2^O3HK^k|L5Q4kAHcs&18Z>xtcXxMp zYuw%4-J9Ea-g@ht=iXm%tNO#PuIg25FPS}Mj5!P})GIJOv+OlbF)e?R|2|V1d-|j_ z$Wbic9ugL1iN{OlGy5EJ^u$54_oC7ol^!GlCmU}FePH%hOK){6RO0|$nN(lp7>PG) z(q~r7GP%*pP|4e=8$C3Pv7bLz(du%2=PT0tj^4ga;7s$|_EvDM2@CJDWo%Kxo!wHY zoZ4+LZ(-kS`6TAk$k!hkV|%MFiYL}3>O}0xNsX!`N^=;x=u!q?-XUAA@I~4y214Gl zx&Su|xZ@1m!6!;ar)rWl&7_d@iv!Y`oiEjfKxiEJ))agpqiUesLr0THUNEo=8w*5;cHtqY?+o}#8xV78*se<{6N_MLKPih@-^`cy&M z(hDP|xcE5n0VO=%+)8@z%&b755Q|FitN69>Dy>h|^wtF^Nlergb(>&*V3k0Ikw44% zWd9bA8S4&Lrn|f8i-%UX(vZl)@u#2j7y@<-lagqmtiuegx?;}bZ0TW@)whq4$1E<% z%x5%0Ng1L-d>c$LZMHN7BPz-jNFfm}EOydZfjTLbsG-1$M^vH)FtDj*`af zIuKXzErKe?5U<{Vf-^$>e*MdT%2V#m`Z zZh9$k3#Lux9K16f*~x~{rQdQolz+lnDW`jjf~FHcwj;h~;+~N}8Wd!v;;TK8tBf5g zv7mf>ag#;?i)jy%ZP8Vw0V|A?!5{sPK}Iit@NRo-x}p=pw%F(H)@2`O9L(ke4ey0k0bvl3KSLjYj!PyVuMm!I7#p z5yV@BXVl(lam~3^C=_r-(UKWCR#D?tg3#_~*)t(@tCqQtK9e%qktAp#a|H4mQBETS zF!ci}O7DEmq?;rZ=KNfqKfB5wVv@NXh-5vL&*rc6Ok|^tvhX!-jp2wb3tum8+;xF2 zlosT$7RfZf#k2)@Nqss*fW{6M@AR+yfuPM2m)eZAO0STZAErmW(P=|r%r3&ba^C>D zYX?}B^XI%lA*1eyv~CBfR?Wp(i9u5^aJKeI12=HR!K zHE;4(rQLl$+zl67#bO*0w9l`kXB+~0_B)!Uhrb<(%Pm#%N>kaw78Z*Nd$t-IIzl~{#_j?%4>9j&ZhIPR-XUty9_4=< z1bW4$uQj-AE3BKAm-Sm$O?c!tpxw(YPneL@mcK3SGT5?P% z)<4J|W((Mf)itiK);(BsIf1rX5%V><@562*mOJD28DxElPyU_^n#Dx>j);LY!3T=%%7@78?(u+vbyk!t=0{AJv9Vn;i=8s zAHR(TIQgyyhTzy(t=diOEqsTsodxssiEdhUP#SjgRkJIEp?B2kr=isa5JP$!lvHJu z3l6DQsj!dgFK#kt;Q$VD(sw7@;aU@PeO4!H>FWAJR!w>utx%7VC7Jii4RqSE-~N2{ zU<5%2l5zw;ZtEYHNDT_C)WTW5mmvq(eW{-jd8zBftIB=Xcj>X^N6memi1iG`s!F^w z_V6Viee#;Qq^c+w$G^M^cSU23jl@p_JFcK-^g{9Oo7=HD$!^gzvM!D zNkA1*P>8D@C4sXmi`Q6N|Y@aPM=)eg1OqLtWu) zZGOD7@WSV%A<$8u(X2{SzC4*mPZK*XX*dYG9>R-!f+*Kd>f5)JCUTA$Or5V~+YhpW z+$twUGBmrMBw!KQ4iMSiF_AVE4h3Xb0Q4`%!c@Nyz4YL}TW%Gnv%gK_VZI zbcgS=%uMQNjGtO$W@?nYymoFS&-RLdfS~WPn%W0PyFGDFH1u-9wWYbTw#bG7_pS z6m~o96uB;Y$tw()FKx0;Fk!~euMq6IK(M}JnO@z3`8r)+y9n<{VfA3Vmqp-1$`n>G zAII${pN=NVX~vM$G!%AuY@{!)b`#9V859nrDpqK1yM(e;ldlb7ttdynRd|ZfcY86j z5TF%iwHAof4J0sB%OyFhG^k4i$a?v7g)d3yixVC*&QA{{*ekWIJ{2Zb$4GeGSf3HI z#?4~?;^WyR_0=AASr9rJLJGN$m(M}&THnW55))hc22A5SuRVw!=cK8}RJ}Ob2H`-v zJlqMn=b&0%n~N|c)Gz53G?W=K@;> zx`7nx?axqa%?RF043l(_fg(QdoNT}CSm5r@>2n#!LT!X?`3W(>0a_(TBPxNuEkdc7 zZAIqZ=nH#7b^?=&XJ;gQFS$B!7cL=%QEP&zfyBqflJy;>(DVy~u_|hJ3H)HpGbh<0 z%rxK-RU^X6sU06OjMuatc`;GeeTTV%<4e7y@k6^qrzdehK) zLE+h_8tT@eK*QMUt*^{s`W`=@KOwPGb@X>bb^CfKU=ayzcFC|B)Hbwhj#f?GYoyx$ znJ{N$Veah{>T<|j8Z~0n6R%$u1|=7LkoYeaAenfYNv@zTQMd}ogh>_ooq12*795SquYeG-%CD`#pEO6a`-4gRe> z6?M=CHbMo3GwWi$WOfqI&n!`{CYWvk))7UZ{iQB`sl`|Qxus^es!4e zyIgWRAc@t5U2i03{58T>?YQ_LSXP>xpF4db&{6w1h}M%lk;{}=?t8Nnp}aArU<=mi z!8&t2!|(5taF}@5fl~Kio+%t2C6Ww%GIGDPMKd&_+|nsY$7QV_lApRtV@GpAc)!jt zZ0^)}Qs@6-&)(y@-`gD!Yik8{ua-l0lB4qd6QinuT9(*)IdyVO) zbwph-d3KOV+3(ft6sifMPz!WxjeTLkWpn%To)PYKV$%Ov57JQMbzb3m3sVLm1w zqURZb{V214t$vDT_PMs=x}VRd|AVN! z9$cOcE-*c5pjS(3FX9ps6Ys- zE5CCS>H5Hp(Q~O?kCiO+sSDF#3y)PiXscZRH)NUgd=9VuP(7jY-o($41IFt=*_LLR z18@!ZSuCdzP&B`O>S~n8xvYyd{6Tf>Z!%$IG8#s{B4;NHdwpOO=$Z^DF~efo%A;}{ z<};B&X{mL{?g`xegBrQ)o90={)~uzpKwKcoLM#Fuo7v^y3W%<)t4OtPwuS1z1kk)w;Acc%IxwNPcnLe&!=0>M2!8DorL5-icMo^}JxYcjUlT zfh5jVsQzgWpKL$Rd;-RGOEA7NN0!4|L|*&i<%QFHn3ASs_d|-TTJ1SSg|^{x?fIGc z%Zs1Y;ZWH8B9~KSUWNOTs}B9+cqjQJg$ ztKmx3TlWnm(JFj#lK0CgaHb0ILe3ir+9|><9VoPZ$^BztGLUao2vizmP6mu8i#4Jz zyj?RNhunTC9VfRu4g!(n*}fRnRMXa|M=5r82DAW}I0h~T-<5!;xkAV)O$UNzG%v#WKC!Ixc%{fZ+ZQ^xlm*$ zk!CPi5JP9fXbr$t<9%v`a;hCf^D76Bb>iFIIx<7L>8lw|s43&7_zdsKTGCizOi5zf z=Ni(ue|VVX4*N$KD=mBz>%r~vT6A!&;GpkF&*Ky|3(}9jkm_G4$|q6J=ZdEy&t83 zNZ2oVZq-w}^?$LVv^+M!$cAO&YYangFsX9#LV4m4jII%|@h0w;A~_|!KOH)tix1TI znLEzOj7%RX-CVCa8@EyNsAag&2?oy9BCs*kA;YDS`cM3NW#H35Ww`C(fAVMcx&nrw zI>WbH~+?A2-|D+E%1cjXJFO6QzEqzNNsVFaPpkXXC~s7VjPdCO#ae|O(MJ^Q&jKdz&e4vokHnc-lNyU<$2U4jzN_}3SY9Z-4< z#wQ=&ZxgvjQ#D{ANso6=LNTsZK;k5T1LFKI%`1W6P=^VKd_=Gvdd0ib{qO()ma~al zu2Y0x{ZfXMCO6ewNm1VFp@PoboR=Ey!qw7C>n{S!0k#erta0MS$vuE&qOBF zOPe;I7%!bs2XjhC@_W{|X(UMjxA&{%6m+t*-&8#Ch&hvNg>mI_Px5oEE#{+dTzp%x zP{_A&KVf`*0I}`Q@KLhvJ}cs_>j}}zZSTSn=)Y(>kuWJ$o@vy!tnGwbVHZ+A1PYG8 zx6n`zxh9C&0Q5JvFH1RjB3+)-Ig?_Pdpi@HD|vyG#1NYzUGY_=2tew zPi>5IkJtflG>d?X^2nGPCPBXC1~6qXgnI7EsP9V zbU*N#`jG@YM8)p5Tz>%6x(s@h?})hC_gJ@uz*@>e>^Q^H1KI-jbw67<1@D|ZFHn|r zY%cdWSUI0=fch6uC_H*EzTLQH7k%@Bi9$lqr}rcfaGAD)b>r=v);MgcAOtqw^J{l6 zGmz7Lbtl+uV_ip;8&YE=#yi8E**eaL&CrRrYG@#$uw@yt8Bl1_$3RZzH?VDc7&?P> zCYqEWVHv6VBXNDrfTn5&Vhl94+*5pA>iJ^gY)GR0ElJI&qs0=@TOchLY!1oU9zE$p z+Jz&KzVnfLhiDm%IO-Zwpo-I!x>?edchk`umxl_RTS9Mq>m=OL9q!HpY_(*{(H#K} zfH4wM6V=wk%Bt}{_s$+2#R){eJ(;8*9I>NAK+}rGiHrBKG)D)wDa8QTGV`h$Z{7fifNUPX;Ub;zALy zc7bzzr`zGjAXwy?iA>2t1{Kab_A^5mTbg2TB--uCKEYMwt|f?bc+#phty;#H^)I;3 z?oc+q(pU-+%*t8Q!4z=$s^e1JK>MVZtrPMlf2J&{-oY>rkrEk;-p+Jbsaqk=ABc__ z?thcC?Wd^0uHx$FrvEiVG#Y2MN zY+q&;1*L$-`!S9$g|D%9zqVKtsS+5s(wYzTLCbp|J5GmnOPy*#wAJe+pq&cYb`Zfo zGV`}Rfp)h>WU4LgqG~rC-=CGYY4ZC^n=N)754bK6fm<=?A19`iBaWQ0jB_jN0*au# zqX}`Y^g>8GN{%m;R8)lePg;f&S7)&wqaYERY-<0-vSy-KWD> zxYau}vAGl15nQ^i?W);WlyZw_S%$Zc46=O#(JuR8hqMn~Ehd7Yg62P!o z-zeCUOvE^F#MS66ZT_I5fm%oRKU@bV*9x=%&#k}Vq}KYcRwpTqO|2vbeqMx*)QSasn8!z)XbkjU$Lz}mjMqPb z?*}3k9x4b0hE{B`Eh&0u^A5ck+q^Cbnq40a1z#BH-k#OQX6G^qSi+Uy>+R>jG6}O~ z&*Fm3f@T|{eMDk;HA?$u&Y1hk@TMF|TrpaU#*Uc(+B`2L$h!FNj^IE|j0$fS9gbh# zW+4rCP{U=q*g!9HcrP41IS0Jz$);C+;9&FB;jM3I)amizb-g-VqZt^6ab7cmI(+E+vQs2<)g>Y*bmE6WKNdZG$o5k&Z7FaM=H z0->p=UH@I#-OWC@g_O23JeCCXxdF_X6LEQ3siF1ika;B*W=M6X^4r8TNwfeJO5V^h zc4_(zR4B5hw#09a+Y5qk9g{YOi!C8B1?|~t6I?IZ(M+a-@SWqtE)iD*2ymL=}}%Um77qrt}?L9d}0rF)8x(@bN7^9>fReu zoDBchok%THcpScg1HW2ZB21Wtszs(;T%V7NnU~<%ttV1KnhfCDnlimzZ;-cM z?)Q$Uxx^)f?UQ_FUgoaz+xjGWvoMe(@X8(!!0~#vfOZnQj>C#2CSH(?PGUAk?8As8 zWq>`0&f(Umj{-igz0W)ve6t_eZgO!st~${YLGK`SS>SR?IN_a{HthQszQoRqpo^yrJu~M;ISwoRITAgC&T&7T@oc-`nPIZ}cAzLKE~wgF zl*v{EI$+dWa7C!(_FW40vPc)fx1XSJYvNQD##{^s<&2$`rW)IvYh}kdFe>qqR)t&J z->VXY3a^PWR=CggV;}N(;WqgBpJ45O_VYXZ?VARaRDw6u*Rq~2L2!Fih&{09nf#2P z=l63z+e^m%;ang*X8EmzRVc>-8x^?xRI~NHZc$k8_(M+QYXR zsGsc(A9%j_X7!Lsc%90jYOrhC1B^LU-!zW-Iky{5%gOy8E~BBsEx&r+p{T(ypm`}s zLUukuFjUmRI-Qz0J##K|uD;gCw?4>spPt+WY%`6ElkGQ5#?c-(sK+9uHP}e{{rN|b z(*~!9fGM5m%(mVH?}RYI4WXQ~Ned{9Ws88I>^`;KZ%W4#r+A@-rURd-`t zxLh9x70z?VD4nppFIMxbhzfFOqw7}DrRX$ryR^>W918zt4za;Xw^1cO$hY}W(Z3Ok zV_NWVZ&zyFJ-w&Ro$(|HEMj~X3U%}Hh>K>G-mUS>$zU~7Ht)A`_uwa^Te8L(&Ug<# zr*Pz7wLXl2&@}_CFp9% zR-`z7qPN7oaPVU0N_CF!aGP|0wP9u2%UZuU3!zxB;&{5b7295oQKITNmtUl6PjEN+ z_8S?ZmK?ge(t-{}6+zAd?V)|g#zXIl)B5N2hPIVV z&v6ikTvP7pO!`7^)F?8yj;HJKe*SU9yzqO->4CB}-0NVlf&SO~`$KI^Bgx4ocD*Ik)|KGv3VMV2-daql-b~fMP-L9$#7sk`vgO~L`$$Chd9lsX#!qZZ_=iMw5aui z7*Js_)!oN-L9eiG9d7C5D>`>@oI^l9llfT4<27ORaX!URg?yvuZRi(19Uo=@awt-9 z9t?0BMS2OUf7~_x^E3@kL0mSB%WcoPp44sDd0|l-(kzZ3z>7SMsC+u^nafyk>B@cm zM0lSQH6tT3&tvsXJI-wuSI$<}af7=%67px!iI+R6XdAeXAUIZp2JzU+C0GCT-hbUm zL1M@Tp#oOodhQ*k6FjQTg?rZ3QZ&>1OK8QaIW|R1aN?_NQ<6fjj0;suC7HJOzCcT< zH+B8XvGd>cpXO*vEW0pt@;vL@4N3 zbXGZXHi-#NA#Q>lNZ=q0I;CgWUeoK5iCpPAtc6v}ll}Mkh13U0J|RvV7{Yf@Lo*nS z<4zp3LDN@skG6THlrA$k(+2{~Pi&%_B=T&#Djxl+^01YqQ*=#4c7ncuTmxUKU5!Yr zaqO}*2feJ27#H@gc0kIMN1ckgoz07kHDt?EQ=$Q8N?Khwrl;#{((Vu+4iz#-u?ErU zOtyWd(fsb8k4H|F5P7wB&Pi4~YQY~|6MrOf(jjYU?Y4tfd`-&@aqEUv|DW}oXZ<_p zbrQjTsuYe7;E}?>PGq$?4afy3H(sdRiQ-a}Gb}--Bjxxx)W@i%Qt4%Zd%V0i++qQ?|uUhx$#^z=E`6wpO$r* zR3)NRS>DfGe|xL_Yc9W4C4BnXH)24j9??x%B_*Wa6Sytk0wL~d?PELSngqTPym>A< zIt0~3;H>k+9E1LB{@1;~{f~RUp8q$l`rm_YHB3m#D z4(zb(DC;h3fUgtNLYpX51M)%B%c9S)G^`QNcxlZ}()^ny3K`c6#`@$42@S(9aAcb>v-zG=uG#!)V=rqnZ1)>Ym;;pxA-tR&kc6@MkZomSwn=H{ zkp_PM2Rj#9`qqj>UCk``qdY=#wB%I<(r<4*yzN~{{Okj~H&Grrbqv>sJie2J%3_lR zZ2=edu&@EDnzb8pFq#KhVn%xZcUXCqPTvKR$Nm2Z5>HNGumOVAAR?2tJ6GMkF7?Ea zR!#;2A3=I8SgY=siBaZfM@tA$&O|`n*FT2^pRO*t`A)2;e*O3rf6@Ch+FD|_irYM~ zqDJ^BF~$b zzz_+59>=_YNrAOou8zuRd&t%Jaw^A4mu`&gHBi~ES`YH@=_Qon9U0&C%l%cF!4elX z*N?;C+M$=lOiEggyU)GV6F@!*wRaoyyRFww%+C0xafo32jQ?Ox=J2Q(vfSjR3E6tJ`kCf>J>ckd@e>j83R{vK*5k8ilV0x+-${SjX7+{;T5X#Ay7&D(FzV3{KNPux@A$oxL68$ zuF6aR(T?l&6Rel?jL8tuLM*U69Ev`K2jkze=GAY~{m zMV!k7km3n3G+)#eIz}LSOOB!*D zOuz2j1?2sA8BFP3j@;8ZH`IlR_g03tQxv7^``wR4vp~7NS}ztq&Lg~Q3!Z5mek`5I z&F!|sssiF+((O@^)~eI3>j$~06fI7-yD5cZuFzE{Uwjp~F=xj*2%R#3;6fg&wUuq9 zv?Wf7if(2uy)BzU5(1*xDt1V%-{7*^yovI5c2xQ9zEbBO5#Dz$PZ|2=;Q8KkP~d=! z{^Z``cpl5^f@S%QJ?Sxrv7L=yr8;C?u>E&;Oqq@9AxYnYyS}Mngu}&mPT#}pt&6b; zDkmxS9E@o32VsC-`yYTD#$N*t!&i5JMQiMSuY)kwxTc&mq?@)fTL-mth|Z%dO;ErB zU`CcwgQwRvMy1!%b!KBU@>9`$k$f;I4I1zSM3 z=2B{e@YZVw9-W4C*(de_X11tl&(IUAqrFr)7o`eK7Y?PF9fHAX#M99o2L>is^BZr*;Lh3BZW?L&#U!5_Wx zH+3;i50GG;X1c{6x-lj0;uMTR>{yOu@nER<()#;xLV1n^zw=>#4?_muoA92we!?{A z4a6$FqSUzsLy#=gLB7e%A0UCww0$6NGMX`RjUJED(GKIgC7w)zthM){GvlrgX%!^=1y z$h{+Xdm_`7tYmVO#ivxJdmGxi<=6N=mL!$)2eYuOEA|MfYc3)UkSxuXn9Lz&*Mqp6z;bPwLNi3zW=31FdYGCyX|aPnP9x`bW}x5w z3B}+Ez6HQ7FC_FtmW?{*%+|9Pos=^C(ER485IQ(4YZ8_L;|UZWW*~ELq&d`R0Z@a0 zkFiW_qdf?#*f*1*gDXjWFQUfH-w8mNw9YXxEaCEL5rdB!J6{H}D%OANp^Q{tt8?TJ zQEX70!^+6hAr7Ss0hl2LP;$##zU*gOpZJT%769Xr;oJxQ&0vk#L{O~hDESIJM6yLr zV|eMw>TE@NDi`NYNXnzzWhzqp>F|pry!=WUsTi zhrUX>hTot%b0d>;N>fvu5z6=TvtnHz_!Eo|TNb@LrH9&AOLcIAx<=wyuy3L+>1-C9 zholK}+X^4z?^;$9MMO;)@-b;VuT5+!EyYI*he7e~n{}t%HuIfWDi0hNzZR zn}FnmQM~DXl=8if*|+?t=oFFYl6hcA`av(-RLnNwt@#aixw7Djj?_V}5eqUcwV=G{ z+^|dcGe!ci_tZJms&t@DNio*+hSg&<7thrZb5-VmXf*{X zj^^xFIhn%k<|ElpePX7pw6193;h6O0i1g!F)_d0ErQ$dkBu^x6U%B@%(Mda6hN*Qv zQ+pctQ&~R9y;s8?ezQV0CyQw9EM4gNKUfr#<3>xw46g7Co2dnhA6WE@FLy^!3`pl(VJgsp0@{y1@%M4$MX}yHH9%%Fd>U~?BQ-h4mz+vx< zdltz~CR}fNYTb4-zS7uJnof1|{>Bn0R#=3q`RoH2gqE`8?i+Ta1ZDX~J8Ug!PlPvP z1pHc*mstci@2LJ}E0IJbj!K5*5Cdf_u&$n@=JK_#jjJ2( zlb1z^gk6QpbUQYTU1w0Uv0|c{lmViwIH7U)SZ-?{!#Z#ZS(Fl!M4c|3XV~7t8{(qV z;zjS$k?5#O7nCn%;8^ANmuyVM%poTGsc`UdIWYy2*WeGq=u zm01!#))ZJ{6u5H&nm{=}=`D`C(jVBxaMnfCU_d%-AJ`W_z`2g8c>^+QYPE!a9btq? zBsgJZLcbY87N;Zv#Y1zl^xZl*=7k23LHA&btA#PcoM-}ZIv93?Bv{(z#2Bd#o#lA4 zMF*M1c0TJKB9gWq_W3Li=7q#HzN&$A&*UT}X6`D+RolAbY& zPHYS@429c{cwIQhU1WkPeC`wXr8ag~*ltqctwiT*h49|QU%Z_$3EnvJmx(tYi^wON zmZ`OEIe(Edx3-T<8z`cXzUTkNe2YiIH`%PU0DiIw4)^luVSMe)ZJ@3$OgqR}8uUK2h6(De zT3h&0**Yp>)OGdx2A=jd&dWT(>UY%Y~sVHue#LTM)tTTH_KwBDIr_j*<;*FDJk zBP`NjN{iy)Y-Vyg;Hj{mO;=Ara^a>a3^jOTa-VWLt;V0T`uORvtUkaEiANYA3-&q7 z3QKyb$jgP`bo|3JeuWHr=0o^UZ&-?O{T0IXelOd*LfW>Q2d^f7zRYzSAG?ijT1s+f z#?Se5Veybi$QYJ4S=wGPSv7+nJD%D~*?mrr{mt-g)q8 zVx9T2Lm?qR8aP7C8qa9I*yzd_-+y~W>;pE!uc4COx~=x&dy>{(6)@XgsW`yn-4{74 zkpSK`s_QNo<}~O{6AY9^Al9joeElyV_r4IZ3dxbs?~}qlmYnbM2!(n z3V?A@#%*ecvP=IH$693OGG7%5!~Tqj1NgxZcLBYiaL^ptF0nPl`JFoJTW#sBmXUI7~YjTF~yW^!5`o)mY4 zh|gv&M`cL>-TmSH#<{_6qWim=czR<)AM6ox82j#6xcL$aq7uIasndE|4~Y9kF8xLdE3>gA15r zPW77qtbqa`I_Lr)`EIYYyJ^;pB&AWj2j!atFGc$A8VuXLnGUmAZr13KPbrPAFXjmFD~|qje$cV_3|w)K5=n6w-Xup1CWIzoN7U)2sy;K zUw5gb0OcL>km!|z=Kb5%y({S~w+&U065xGd%l8rvE8yy1{#l*T&Op>m2YS1;ZE4#T zAHR%Gm%Jv?*a+k`a00wFRLk7 zb24(~5D>BUEk=*9=TLbmC7U4v7Gu_fi|HM7VMgASnmqFe+l%|oD!acJUuaeZPM8&_=nxZBnT2FrbRP4n0cUZD&k{Jm!q0+xTJGnMb{s@; z&&_frRn_7>(EOsFesAIFATy)BkkN)3BO^k;)*q2mg+j7~Zmb9B2Z@?%ah8(-UEXcm zOmA)G+2lwE_-y7d6TT%bmaqT>5c}`vnO@S(QP8KNGi)q9I~pW*XMS0gx{$_o8`1_= zjbFa74uKj3K_`Ka@erRf(ze1Kx?)e)MaI2`b7s!x?L(j#P2v@@{3`$9Dg8f1w_e9l z+Qfo+BNofwz5pv40;y!wuo9Ci2;s_$Q6r$ppiTBBVA#A)u5>c|bF&=xv7R*Pa^bf} z8Wz!V{F}O~XrT#g76Xg`_oXO$u ze%H;NXHXCb9IaGpie|^v*61=a&PUAJ!P_UUABpy=RZQ-J0VXzQDfPcKD(L#$e$ter zcstXZ)d=X!Qk`Yu+_#9z9WcUHdGe$|7yW*7miGF8jhP=ayciR8M2) zr6sS3lBOonGZMTtzg(>N#uY87Ic!UUycm3qUmmGLQ$ubAjbFZMrDqKkk<`>l%F|&Z z3iI_oSj8<}PHbaF-f<>Gs36qI5*rfiPja`0^uUp3HY6!X5*!Ivu;jHq^M+V@=WASdpXDG!RcDm4 zlK>lpU}UUGfLYDERD@2`H-!FS>P7w@N&lpli_c*%N%A&ex*HIr5%Yxb?~L^Cji>`S|bTODwl#%0D=;(Mj}bAq+v?^uoDjDsw`b;xco5EtwrM`=hDK_;X3h zrHaXZqS*zjS=9C5(^@J?MjO7Pd>0LH6l;K$s|4{i`7QO?YFAr0%PMv)4^i_U^U~FQ zx*8)>79h-X)^*dx7x~cVHln*|={ZvzeI9MeyJ7Cu=Cb68Nri`b_#88MTkjuC0}VF% zay^*^A)}hG5J(1x3?&&~M3~LjEPVGkZLGh+@_b#fqN*&u=*fxljDg+C@_L_umFe@h zvYTM{qo>VfPGA2rY)2*19cDpExpS@7%ffTkc~xM8da}T*6xCCKm01nC=DfF+z0O9X z|FJKc0<&>Jbd*v^Q>)Kx=w21r>@zS2COMzv{5Qi9rn7SB^Qum01C zW|32!KylKR&U4Y-q~YSV+%V6p&oTB<5Ur-i^AS^Tp1f`6qsCoDy7Zq8G^n)iI$sn$6Y+@rm5xx}*M;P#Iob@RP3 zcp!sOxUC3;Oy$R_yxiX7^i=JYiY7mCcNLI~eEO~h6`Ou-sd+ndv?>_QtmM@(B)ogx zOQzS~)yOg%m&D7wiwEM`@wZMks{sXdyrZiOONyKS}mrxeufdiGLA>b}I5K*vYb7h#yoHYRx7z=bb+k$zSGgY~i`RQ45Nn zF6UuPd!sS12CjqAvv)v1#S>VNy6c~}@wL|h5slsOH*YJ9PoX1~v3^Nm^}OQCP!xq3 zaOT3sJpnfOu9j0rl1)Xsl0z0I22Dk^dy%uV_os#NZuA&|qQBnr9?hd2ouK`2cUY{~=&s|)wdaX%2z^*HrM9Q|BJ=S;0q8i?$Q?t3WA0#X0Wq~56Htk*0OsZ&v` zKIFIP&e%E3ozr475Bg#+H;z<)Y5}w!C^B4C>&({?2upeWqL6m*bVNs4zjIDbZLE7u zBcSFtK8nxg5#W}UR_H_+F7RN<&SgPUMFY#ULM&TI@_^Uf*z+N*zn&zt?06lRFxNIS zk7hRzCQ{z)4C$y@#tK2C$dr3|fBDCCVAv-B@k=llQZ$|L&24pWl@bk@V`r?rPPjFA zyQH)`@pEr`l5Tu8XHHrn<(-qMNxE5}B_w!xXO3Rmc%Wptzy*l?&=&E*lWCiEL{sOX zhyyc}#Ql$Hy*cjl4d$l=RHx>8ZUt1k?;-;qH&uq}t8tD!e&(TI5nWF2+cE~wC4P`- zB#pgHGM@38mSvMd_S%g66OR!{K^vFb)#bk)Gd$N*mVjxsV>zE|9h!++w-zCDE3`ba zu0k0i_^SA<^*8;*i-L!{+3Je0NXK^*4!W#dujxbDl1{;lMg0V#G}u^CKOp$ua86 zX2$z5`zedqoN#fIowA?>#f2yHq~>iZvy1bfa&wlk{kacr+?G{fEttR7{r7P?gcs~g zf@3P^>qsmc=Il&26A{>~X+aMMYNl%Jv*X#Ca+aUJqFm3Da&iWG97QZM7gCJ`>RFj~ zi1LrbX>i_Yh6Pu!75FVFp;CjvF0d{DYWzMG0pcC8L)O&8qF^@S!fu-r4@%u%5OSt5 z;n`391`WV?Ylaa0l7zSm3qi%q^gvou(VBsmGMz|Lt{ma;go2WW7G`%Z0LbLhyk+pI zsKAnXT}vp3BQ?`Uc7^X;Qp>C;utpeicqAg*PAi;w>gy8ND-NA#qyE}y7p5O!z?iSqL8n@u? z?(R-EFrDw4duQ&QdHG-dr+F#5y697N>e$}2 zP+yYt6A;#n9B1Y21@Gs+Is@@3K|~TgFZ=j9sw>SfM5_&ECtGqB6K~ziJ)-|6MFyOy zr&93dH|rc^iMp7buh<~UYL~kf*GF1feiB~Auyt`gDb#^aF5=`a;;ci0Y9QgiY7=GXfM8ydB^(^(QaS{W2Cppf*n-Xa_2--Ko za?ALu*s=y4@U{jGIKTG-wvM9#&YgYbW!NM=n8w*dezKs5Djis##_TzccLOj# z_`h(!+QX&4{o3&6)>eg7r*0U&_1ysK4_N&J8ZYQuLUv6H=x1SgSecCe(`s_D)*2O^ z9uP{JD_XB=&`_>yp{swdzbTp)Wgvmz_UZ&(+8*R%|KR~AUAEewq4h&T#%UK;6(7eHVmH-r7zoO~6B$3obu4G$1fl~+$v zQnwSmMYUL{^nW~zWC9#4!pmpAE!Usl3PanF(3K%bJn#8N=pWq6cJT{Yb=G{hsFo*q zqceQhr{$NB*XhAF7Ax%cEMM6B;F9>WlHScKy~U$m(8r+WEUinkHaSX@SI zLGT&Phrjp;$5q8Ndr^0{suHbQ`>3EL6Y$6vsnw6v0KvBzEF1aR zMCqU7+qc;qlLiXUnur2i!Np(au|`X;zNG-A_ZLwS*9~(qu|i4kBeE*tMW;+w=3im+ zy~@iIn|LCm&CW)CkChXCMeXm^&}~b&5|X?Km;P8&>3?<2D=~0zaY+gMnYb{JMbi#Q zvalH2$j3y*%JPh$Rk$uW2vr!TV&O;#-7^s+g7PtqlUcn;<4lCg5N_3_f6`A<0{RaM zK4OX!N!aRAlCFY}rF9PEecyJ6Mxr{EOI-JnmqaaEef8^~t7H3Z24z~^eXYv?Ev|1k zmj_N3>m2J@A!i;)H1jMdD9&@>`xqz?)elWxk!X7Me;~^jRjL!~M;|y3?@;oruXwZC z032^R#TtJr&#YCwbEK%CuB;6A4ofooUTp$Zijw|7dX-y!B3e&cUY~j!1#hfaxt+ZW z=Y7sR1|}fA&&yWx+b;rcI-=o@QaUJW?duA@F-yE%9dKZ%l&^%|(j<%0u82!T?@^=GZNT=N}jd=ra+as2JINqz4SphcXy|B%Sp| zLWnIIu~EGyzvw4_VI+HAY+fT8&0MzdB$}~lyvVFM#hwZpqZfk}YlP8_o;J1|{(Tf_ zM6?+&QAG~XMkI`cz`sZJi%0lLi6jHvGkc5|X{b{Ic-17dV;6Qei$wTz1FSF^jDe9W zJb4gGC=gO7Mkc+!J~ruj|6JwiE}%0^&dv-ERYZ;^`SG&j+7PcTOw_ZpSR4UmYF;W& z3>31SF;{uJ7I~m#-;i~+VSF`A3tQSN$j??t6>F%1RZxoR;(GiriY8gail7$B}g7qN4qe*@yFNr?ww%mX#G&Ja66GgS-El$l3~ zx@n%)hArBWk9P%83ZXIgjMvC>XiZS04tU7ASCq84gf<$K_?Fu^KQ(bB zMLs(;c8{ziX`y7>plp{esR}dW^F&HC)o@ zwe7GM`-q^G>J-h(0cUCOcuw|;^lcX*((A94tzb8=#lxqEctB4|nn`bRWBepMNww28 zDGB?;B;$>u$74AKmV& zp6jvexOJRZKZ75YCi=$7?G_NZA&*=`(L8YN41>Sr!n?-vTywH#L$ENG&bQ)bHzs*k zTHPqOl}XK=0}E}|OEtN#$6zY{0}C*^w`ePnXF%s~)d#KP5jT6B5axO~bmx3@ zYyP^5-0zT*4}0x8;`RAI%XSR0u~FjK2RVE{t6QVA{Gel8t_$!l;}f$IF4t6e+A?-l zIL+<`h2WurwVFSMf0Haks;oyl`q6r~AM5(mi%d8^wRu-W>ZEaIjjEH#v?JOTSgJ>ER9Fov*bS(!st5z3Rf z1;fnehszktgvH5OkEGU?)I`NEiDW^2z5dW#Tb8b~4=qA090GX}6F~xt89=Oa2*L0D z+3Z~7#g-VjW&?~YEs4H0!Tw9qa1Bs&esT3f&`WCUvx4GP<8%2wvwWKSc5_ht(Vn8B zlIUh_0ILPl)~FAf;)>s^$34l|ia$O5kXy{<_wEJ@Qq!)vf^%!P=PGNe*T>>AwKy#U z?f$DGQ4psMIo_6Td`yV@N^{;2sU>PhgWDU9!1*o?^EwLIi?>KDc-qb6 z*C9omFas*-&%RK$OW+C>ummy+qd4>_3AaS%fPOF2oGr)fw-9qj-e(g2DO zfX(J9(6oieu`hQ&?Aq=5+&6xvx8_~blGB;CCyrF(QD1gbDd6j}_*R*11?c;@Mra<(&XZhquhl1=+`|K{}4f}B!eHopO$2m9o1SJ6k~+MsXk zd9Mw04|o`M?Jv6>FNaD1`{r8<(hiOHhe?0W?EU<(gH&D}gARwF0vXWsVwZPn zbbeqQ?vplO8a-b{vusofZfblWi|()wno^ajX62u$X$}6-QsuL?7uo@252jMZf61AX z9)%1GHTnM8DQSxaseMf^q=K_vO#KGAqS6x|&J^hvnv7a4x(HUOZnM2WoDf0fkaFHM zxh$El5`}z2n&}%$xSzbek2g?YZo`cmd~;ci>bz5+1SQd2sg$dMxSE?s>T@vrrmhh`SMdRW3 zKU2RiFKeY7xq11%q;B|S(m}Gde$PxG*SfZ+UJ1gRSlNeXeE;R8AtfPUEni%nZ9KjhKr$pZc@@V?v<6O$Pd<-I1dxI&MczCfK-Y6LBK3aZbrdVa3 zH$)>C*79#>2O3(taL0YB5%Ak!qp01ABy z9%(2*-oEU5vLLTEYa#1p%gw-0JGs4xcIlCvgHEPlgD>YaP$^sU+xs8w)`c zB6QX^SDKv;5Fq}OCVmLeBC=Ze*T%F){f@LNN$a2rXau*N4u|<|L2(T%Zw{b;b-;Hf zqcmiVY4$Qq8WR)f4{#>GXr?c23CcoV2l>YQbi|<&W{uza3^W@7A?YVBV&bTM%V3GQ z%GRc`-Q-uo;U08_y}dNAYOlty$MHfHgY%{7^ZAab{Y^<~C-7-Uc8;*wrvD>0j?dR} zfOM;U(;UHn$B&DIo*}hy{4zi#8!YKyH0|5+xIsM{pYZZU{cNnvZlGDBm3WEb1z_HL zc}<1s%y<{_;lo#sQVr&<`lQ#9=Gy%xIX?lyQ3VgvCB`)$FA@ga@$KHE0e3_1Av3EI zR8&>9L=2bvi@o_^u>oZ@t>C4J3Huhi$H{2lUQu+Q{AQvcds17cM&WDZedL-(Q#^E( zJwco}mK%IJBM20(4&sZ;hP>eluZdtJB0^2}MzqSbZ;{ZcqD3$7%AFYQk#4e1TkCVd#l8E>W#^v|2~`??cO2-ARP*YyFnE86!a zZ+L38ao^-qNH6CrF&|~Bj~Bo-&0fdz{xH1p%A^FaivGQLN7VD?U~<(MW6Nu|L&NN< z)XTVf(H0x`rT+1DGB57o*q~9d?(|(0G2e(yW!q5ICPk{>`Lzj|efy)iw~t@r9Z zF>MF#;(3hiUloP_?5_WOdRRL?+?VW)$yrI`EqJU~p|N?>MB)6ALNanU5g-h`ae&kQ z*c-zN#%)QD4=!%R_tDQJe(QSOQRt0VSU}XLH}HuSo~3GpFE4MrZ)c0uE1$C-Rbc!b z(03utwKoTwvk+~s!-l@bLb6B$zg5UJ&pPLoT;5V7CdL-Hu;C6i1Wm2>{yeMBQ)r_g zEtG^iJ%rS{4Jk5dK9V)pJ1XXrpJIss9e)e5&ho$8M;k5;L@)k_ccz25MS+OIF&3eRPS+4bFn*{RdhFQcOJs+%tKNOrTk_RT<$~qYlK1nl-}7{y_sX~9aY5fosC`J(GR3kAJzIHsp+zcunTGlE z*UWKTwPJyRCbRxu-Jld()XXYy*-7RSNg|{ zP@cbw6gMO0e_Z%qB?UGO{Y_Kwzm@$zuWixC{S!d^?>`AGmHj7R{eS$Vg8d)g|NkiI z|A!lZDqP^9j}1|z#V;_8d$hqSjy5%}=uj@4WwSRUjk7xxe1M=G@jDh7+z_vM(*1%+ zDlb?nRVoHo{D4}h`@8JI=rd{Vj+U#QM2KHv`R8W9Cp**Cr)=~2_ z%C%Esu8)%-2U6KQKhN7vnchgrq>rIx@GfP3GMVP z&pgfD|HW?3pFe2tijX!KTbayx@NS7mD6HuEmS3@1S;u@YjP!)4Wg1y*d`BTe>!w>z ztiPiq)O2+^&cdIt%OWV@OeUaXzNX+=T^?{hzF8ij$St2D?d#=)oWXow%1J}ntG0ba zv8KiP7TDS1h$)qk?uM-~$JF?8qHO^cH*M$oXUXbj>SK+Z*Pv3$1Z3iPwMP;}o@X*> z!?zm=wljwx*r}w{G+;*W8AJU#a{U1V__57MjZdvfFZX})cPk<@`&Sm}P>70p&c8CN z^aI7Eu0e7B+;>v!Q!4lc^PhdFY%FAr9S5nfRP?pCviw2{-_3C<1Y4Qx#Z-T#?|BMv z8TZ5$Y>WS8!Y;$~_g#Pdm_E5}Y-mF-l_J527wivpMdp}j4$4&N;iXm%PH-qw95sux3n;er|uInqP@S(K)gP%PcU$^ zzlLj7_B^l^r)MF$0`!R)=dpfs(%WtyKV+BM>jj96*g@zY;PIud34C3C&C%=0f z{hguLS&KmS$w?QrxI<^lW1~0W4BVPVhV2{{uG#U4;DsjTH=b+-_qzQ$`je6ZyN@wu z5B$*uKu>hAfU3-WHZ-3U(jE8Ma}(0ULywt}gz9w3ug*AXeFRlIlTx_6IUo2ylAC{d z;d-UPY$~2p%JdjSb3IU)Wq9*}WX*-^WKgfWTQ-kjVNb+Y8Eb-pduZLDvz7o(?v3K< zs7{gC$QZ@>Wz`&6b9+SJHkoO;7MvXZ_7q1lA3bLI+qqmQ-6kP2UrENh-&{R%R!{KE z{S8Ndp|UdywjCvw`S|{QSgfU26JUP}ua;Kmtc2$53Cg~CYbC>;%&LhT(s*tCWUGE& z4Nc_4%@l`67AAZQ=Yor+Qv+q+_)8yw0#ed%-#FzZ%w^?!qQ?2QhL!;6AXbmWP}=6_ zdLw3Qr8DU8@jT)ypy|z?7qOC<>I0B}{54>p-FL$a6*rsnIj?m+#NZcOjK#1wNi5Oc!-FxT*R}s}q)^+A%_SB5Bu|NWlv3nP`;E~<0CYk% zYZzuRNO*CKXeR>D)4IxKT841==(E3w7gI*2b{mXSvEN4{9>fo${hLbXk7R&)wMxC) zdf!IGv0Ph1ZheYvdD9)-#EZM4u$uuq3!p5=bTWW4;)|igo@>;LV%CN$KdZy2>FNDr z@F=<+@rw{R&E2MsHd2yT(Xoh+cUz#8$VYMYhpzhm$-|v`STpWzb_j3@rYdE)sO1+r zp?AoWFHD_i$_1F$n^27~&A@s3-iyxCvwQMY?@wM>PfE>{nBo=tULlmL4j}a7ec(o> zMx`?9CuD2{_J?1Sey)V^kzjoe3wgiN`2CV1_{y>&V4-=OZ7%z}1O9eqs;D5GAcrfj zihi+J5(E|t-UjA^=mFn}Y)&cF5s0WVXGp{zQb_h*qH7ft6mVsR7gru)+Je_#8a5@X(H{*58`{hwwwP(!7tHzjCfa&Bled*Tf&v7yJA@)}nS+2sx zUIgqf<#{Eqd6VB%sGr|J1M9gIfK$G+`bv}jBG^o(7dou0Oo@10UgRix3I>&Pn$5Eu zh`kxX)1k~`4-1!~LC;cH7amZ?{cX6M`bev*q8dWv5r%Z_bPA_6EnA-QAW$H9f1f)+ zfWAQ?vdrC=^f3voWsKW+@dc@t8jWV}Y`x0R;$s}%78OB(!u{r> zOz65yhW&3moH^eOW!uWU6Wa_e2?^q$<(1WP26S%9a2P`^SHQFzf9P z$_V3ig!)HEj5x#=NOBs<>bIqBm05{d*7uI_D5s}*JHD-Qcsm^KTIC&IF>92;^bRPB z>BPaKqHC1Dq#NXTD!2vZzXTUOwiZ{g{^Hh@{2M|GPq5w=%zXa7AVcH{k52JgV}P29@r3?^E_6r^vl_GO6hByU@SP{tzpR`drgdJOnqlwk zC@a>bZAC{gW__58Cxi2W5GU@Rn!gG&;DBA@YCjURRnl0|9~i4>&FZ?Q$l@0@Gd?y9 zwT;F3Rcz+ZN{ocq(6TUR|M5N)`vos*jOn(pW-_cluY$W2 zDwgY+yl^q+I87ou6<#VZ$m324qClmpgDx};_KQh>G zz%36ibOtTELEx)y0 zz?aFw7!~`zI(9_8zgrhAYjFuH&J?;?BJx4U4$5sVXc*EnncOK+WL?irq;4U{v&wO`AX1Ufa_ zkwrZIDqm!R^i(Y~I5=#&lFyUZ#wjM3EHTsZZE@*}e_e0=+*vWRH4ukSVu*<=zP%$j z((B}gD-fgT`)FNoAA=KJag{Nh)_B?@PBxiX3_SMwX9(z2^_1NRVBQBB4orB1?++E` zH2maXlg;TsB96lqZ?D~hE8~#sixWjgDB>rSuK$`e9Nf*$SkSnrXetS0kNRk!u+Ic@ z`cb0)uD`|y0T+SR#{{1Qj*MCFkAw=UdTm&4IdvHc9JwcgaZ@c%ec$3v8a-Q*X|1ac zlQ0|GfzRrFkF+8R#d+QJWWTQTNCLKQT3ZT188ijy>^<49WXfr$qlqJ$i_FG02@cla zVHPd3FW*vj9i1-d=z2cDqJBr8B6z(Jk>)~=;dU&t5Qf*+c3uN(LY6lvfexPza%c#i zuji6Df3PXk*y^JsYA9>zTHZp!u-D&2=)?@dcD$XMo>#Q5HCk)VYFU=#RDiKQC;ah4 zx*?%{MW-IwY~wWA+Bfayl~7vw$VSuNE3~jM>htk#_DxVuU?5GEaaO`q1Z?*$zX&F- zk|$v&LS#~;9fPl=GUS<&Ie1-=EyR(WM0d`^DojCV?A!(XIPDM;cjJ#SdXCG+As^!J zZ`A$C9Ig6lZyPbfz|{Xm+gmULbUBynbhfcr-6Q2|jvzeE?tarPR(?J|1IijZ-wNT_ z#lcGL<@-$q@0MI(@X%vwD(GK8nIVOEe_c4`EBi7{Y+TQTW#788oDXZvYI&({ob2E% z=Z(A1_>BpgOT+a;-ZuF15tpKZ?*p8jA4%#HR#A@1pt}_shDG$o#T-Ui!oR-QcWPdg zwt^pVh0Fx8MchN&9Ru>ih>QigSNED}nWm?Enni7Fj-OGclT5!>Ufa!MwU(BICAWtT z2Ixe%c|PPt69x+T;UWeEgr(g4JRV$Oj#eOv7X-$~bPd{a8r1Z*EwfpwEUr$WFTcuC z(K!k_WD<1U;LgJtXdx}uc(4paY1A;Shb}rwJ>JdaqN7Vz-rk>}NY0yxGKQ>{emxNQPj>+ePme*sF{Gf{T(kG*-HNz?I(8#2l{qHs)x?H&7FZ;-z)jFWh6nml^ zdV#kY5~7KP%5TA5$ReJ`o}&E}#zQofY_kKU-@=v2Sz2a<@U)VS<&=t~Go`IgjF_w# zEqc3Z#BEt?hZMr&A&39qw8VrWZ$KavEH*Iu@pYrGp&)Z@xZyrB*-mnycGu2TzhyV5 zG5Dh>WzO^S<>91bHP`jj2X6~j!orNDCupj|H1njTZa5(IMHIesA{T@l5HIYeU}iE~ z4Rz#7sM4e`_>IkR4a>*OrQ>a8e-z6tHy0(^QU@W#T|ivR${8r7A9_Zz(n7b(=_sjw zSKV2naAalu+!S#;>RJ)gS($z3c8cCI59PZAVi4>Eg0Bo|7NMzNrv29fEO4@+^P@ZIRFmX}z^N#5YIFPc5yhd^u59 z?<}7JKN!8I{?q8<0S*%C-Qi7OTJ+1iOT+CrkX^4l4B6x+J3`aAo;CA&fz3rfMKGBA zdV^9lAZN?9I1v;A)gXI&P8>L<74{fjV2DJ+Stl5hcIk(OO-39s6yd`5|8P8T&D9hQ~)vJw4Af1eK=ICya#ndRnu z`rx}%ll{&)(E3!fBs$x*y4^tk{?jBM!y_d8{Esv@)Xu0#uIQbIVy~J=p~l?K^E=+V z4jHsL*7j~ z_ucDk=wlrc0t4R>aiDx@r_p>R7nITwez{rX02s*a`UleL<@ zi|2M?+r1sRU9nH+o@E%UDB3=up+{#V=K_ zT<0lu9WA*Iwf08h4#1Y3C*}&H-&u1*XjRQB3lBb{TJQ2-g;22xaife@h>D5b_cT1y zUCGcFU-Neb|Dw#ivwL+e=UiGaPz~+N$xXzoxpv3w3BD0s)Cg<-h;KH#Qib2Y zTF|wuu}>`*v|$u>pwK>0#swFj1%LhIg3{_m*n>hSElqry)G8q4tq6HHN#N`qfBBIG zVgRHDBz3JUhOmy0*sno#9Kav$-BaAHz(G%W+0Bb&W85MWhQWRGn$ogG`uKW4p8voC z0B)}SLjYrssDJwy=?hVr;b6v4flQLSMtS^>Y0Ha2HN4EV<0?L~+VolSf`UN$!`iH^ zH*@%>TBKQj{r>LXvW>nuJw z;f)xbCE*CBYDwrJ4784p*Ars7DW&b66;f*lVy971XLv{qBuQHM(@=w*UBAdcDoWZk_JWY&0-8i9Po0o|~LQC)pW5dY}b7 ztj`cdvJ?#r(F(0?x$6)RK^1ymI=HW|uE{vqkL_1?D_e(^UrpK9^@gfoh{O-CQ!Ar| zq-z=M-lSHplDyLH%$BVba5Av8zv|?Fz-Gg4Ym3oJb2S8@CN{2_+%R-dH~8W1#nl0M zlvew2JiJ%yfk_p&VT8GR_S6>AtVch%ph96DgMwi{^DjieMMk$jWY8Kls?9{1^Ks}C z@8u$7j+>Zh+u$*XN|amNp>`TKZ|bO|34ETzW#-*p;88ls4no4!2pF-Wt^^Fv;i=6I z=)3+?X>Y1K3~^t&_p%C2Wesu9PwsnEOY)(yJkRVuH#xEOd1@|XWY zFuGr1y9)#-_bh?3met-ZDXHhDl$D9(3zdI;=%9i)LR3E7!j3EG^q!C2*DmaIDuaUI z#KCm+Pfq7`;=kLPu)&wbWzQhoYYUnOA6?8zhx5E%8`y2*fXnL=vF*I?X}$}EAwb{n zN2Mm3vM3ETr^G_YscE3JrFt8k#ETQwTue-YJp;MLq@=po?*$2;uiWaEsO5#yha-ya zCsAcL+GCA)WoJ1(*-5RI08|EKJq{97ADm|b)Wme@Tovz^_U3`XwB-p9;I6WExj$1{ zG-lE_hu^@~(b=zVtUeB5JMA5yu0KM>zt2!;-ZK!jD++z7&3MvZZ#m0&-Fq*NTv%OP z`W*ci0LG~GS3gxO&3RI)7ko<7aWK&r4k3N9e2YyeKfh&Z3vLSboL3Hju@BR-WJ0tlCTHQQnT>l3&9KoU;{BOO z7u}Ep6x|ijSf$gKhD+S^WERvlP7Hm^(E5%CHg>7y zSl}Yn!@0c)`M|nX>fZ5#MKQjX@0bCIGv>{$P7Oc!Ti-3O>2llB^?d966AbbBAc+4Q zG;Iz4)j%1arm<}_b5BYEC!`Gn)aD4v5|LG9e|&u#1b=k2y9Gr-1U6O7YSeftb4u0L zDCNa97j}f0ES*7NrvzB9s>{)x0Jf1;?ddw4ZnV9D!vmLa;2K{4hy_tm?V>htR)(j) z(7uTzfIHGaM-Puzz)^1k&N?rHKjfX$CO>y7J)48UKrp6&@FqE|E6<`0C-qqg_wcHW zXf`}?J@>pE>#Y1~k}IL$XtZ^{_q;FRTh4_$NMuRWCw+eBg#OK+!JPo@n?bE)v=+ed zOFbRFe;pJt-&T0ZAg|w)4-}R9Mxm);nbyUQ z!;z~v#7U~njh91Rr+NEFV0fv7doF`r&f+{LulxqDlQ>WwkJi#2DX>5MG>{*Go)E$% zciVZoB8Jyw??!v5DC}WpX2qdh8KN{0LFg@WtBGPWlA9fP4 zQ=aG(8ib15uZ?gh?b~J~TKIJM1Ly)CO^l9ZWN7(UTHwH_=@Jr8?}-<~9~|NsjHTEp zubrRX(e_;@$2$W#)XBD@W>5;y=b)kPN2ux-;R7rj4p*}N`vGP1-BM41$T4Vhb?GQA zKHez8KhF;*D`j^@J=N!Jdmw*Cgl1EqB&KCIvKzAQArOd!!s(ZiWu(NZsM2a4FPeZD zZg-->eai82zK&FL)>t-pqbA#zA0!6gw)de9Zd)kNsL=?|oMN>LnlHj|RYr9lKh%wc z5=&+-O{z@U0M@h|NF=mKyeE<6>efnjP?m{jbm|3-2Y>M`uG|uN*vfRRzg_E6^0H%z z<~>f8RTBEPIZ#TKrlp-fJdEiP{9Za`9`tj3cEhO_9@QX6-;$<;L^2QkMH@9LRk~~l z2Bk3s>j=R&nDvYmHm~}0U@G!-mbu)kxRPWmiXdFqMFG0Yg#Pm%H?b+*f%TrUFFSDu znsX%_IlR`DxKPd8^Ir6L^<~rUY3X(+;ywtf@YQGfE#yR#XViSkr1Gon1=eq&iLG3P za-;3viXu1sA~#(Tgn}tiVjJ_wrnTrKRC({bLy4(wgOM7Bmy%;zol7Sjj%R|~_ z*OLkCiq5?XSv_soHHIRTo570x7Kut=Ie|$=riUP)h_*C-VwvhMcmyn81$rIyMt%9Z z*h;;)7AB~`m!|;tEP@@2pGFGp3+$cphBFh|tZ?}|&L7$uS`06io^<66yu}PDlIi;U z^c}4#bo_IlgjvY2BNbG%z}bh3-#v%q^IU1;ENJo;TLUGxMG?`-#*nGxWVHBKe;>jQ z&=Jl3@}Lal$uD+e5*NtpJZfKA<+B;^ZP^uzlr{;F77ZUHvJ+#ca1iQE?4DB?o*@AW z2lxS<<>DLxjlmPcTH&|r|AGy4?^THs+rU-4V2rZsug0KOlN?vhnOS8UI>R}kE~cny zjHLE}mPDRMX1$tLqQB-!l(3K%H$>XztD(TOC!~O1jd01st57$b?J{A{S<$QHP2W11 z0}KnCcwW=oEYug1&f1h!ko$f48OPUTpW_l~Um@Uxf5TAJ>ow>o;NHoVWHoG?qkU zR!`iPW)5E@`W%PQ+S~L>#bMTdv`wyTbow4PeA4*YdN+USRJt%iJrqR`yIRqAj+8!C z&f!Wd?^nmeC60YFGc`QEUqZboLnf=}j#PR64eoU$axOP3et0Y$`(~g22#n)=sc^Za z$|QamUDLEMSE5hVXefhf)=3SJ(@zoF^i?ba}8^>^@o;vU8GptE#1MsSg=nspI$c!Tt`Kq*2bYrE`r z3C^B0>X|%heX*IjJ9-IP=bB}iH8(yjgEdCs7deBV+Ms()e}#OI9B(=gA+l2^wwy%Z zvNLg3VfA??tnyh@k4Z)USqXfac!Sqi1KmB53H?D&K}O;_-GB|((cX0+&^lU{gf9L0 z{nIj}6)M`*xd=Z~E9YT5$xL*DM$7|?>7J8yQk+7b}|XA4)i|Ga`N0u9ie$OP6_c;F|A zx8pEwWlBMGtO$Q64or8vBr%}n%mnyHz7ZpQtAx!ITvRp3VC3u!TKa6a>xpYY^2vC~ zN%sBySLF-hA{X_a>P*>11Nc(~G@R$tRvdULXT-gM@ffsw#zPAFAj=bLf`=PLJ}o^7 zt$Ng1o^+!Qq`g9(hKFnmB3=)qI#0dh!bkhw`go&90AnG!YFha>YpUSoDZgRUZ^kxKCr+0I)vAV(zW$ONXnWXndIHoyQu|}u3 zvnTmk8D(^7#OZt04n~Ciy9chzGgGkqVU48Bdaf+ZBpN%{GVZPImxL z^CAD-U3O*nvRL4(KYP`ee&tr|x&)-V7d_?SDw<66{n4O3lLy zc7f7X@|05qIEtN0;dKMpE^#y*^1a6~Ai0>KAVmNhzqjCEbS~B1<$hS@`0gT9s;gM6 zr?-%lVrUvf&%ZV8G&n3vmHUW4-oGiGg-<<_Ik2ONek=@y5#~yBhDEcr&_R{Df-#8j z!X)osTBdMY;ulASx*v=s2D?dUqxM$A)boNJxJ z6`YLx=1`TDU17oc)Sk(0oshRHP<0bQ4v-GXwG4|Oi(1EnZu3~{y=$@a%XD0($%ISS ztuY>)&B-kL{Tm6MI_mRzX*SWXFRBv>Ep}zuWUcY(P<8+TbGk~tx0ye~2a!q&N^joq zduT>gN)B6P?nJ-VV=zu#wZbnpulKtwLVcxF<@eW0H&P>dFO)609)h(MWwVfg?Qm(v z{U;0|Qy5?UY}82d+~{MdE>{)+Qhhb4!RXk%_B4zT=q)rHy%|x~( z0vo$@j&&`@buro=6AWigZxl+G5&dtFxa(|QLcA}*A0hPK=Aq{^w%+w`{#WBs*-!9T z3CW*NIt?M-T=>68x7%N6Xjg6W(KNEK{vQwI|M5Wnzwtn-ZT0^6uT+MPGv*mPkZ!w2 z_c5#uy@*D5( z8FAWEQ#9wX9pFr;gaZVL+Zt{Y$xfpXycYg%)P!YYmo`$QrzXi{L2k4*0~&i%?`0EX zH*AtTc5aiEmXzkpt^+d7F#(;(oe_o4`*#%0SJz7xn}V-rqzxU3{&VIwXz-RmAX3W% z-)~fbu3%t?#N%Z_ryEsDofX@I#|$%e-P6}kB49o58bU$B5U=|vQ{ADSh^PzaQ!kw1 zfFY1yr~8Ye7k5kk%hCI14pYLupgJ-6!7fwRa+E5;z4t`=e&{kj z2v+(V<>Y@pNLl*%c?+U`vdMTHE8m*PZ`k-z0p)Lo%&B3=7~piTxRn+)sKzD`M?m)k zl&L-X(f>$C3iGfAnWecZ#&CtuG8N$*OBv`!oE4L(Cf{G=8UY#bOqI?KR5$DAgoVD& zzX(prXSoY+Yw@p;&YGMJ>6IG|*93VV+*3k*25i~eO{eg-Z70t!(=>taMOlU0N$)1* zd3-uB|5WW@=ibE1jxEWFPtR50cogG%ao<;RKeo}br#Wh*l3{Q^f|Cz;$k6_N}1&?s%;BnHm8()S>kMxr8ZBmmW@*WDhO56B8EU5 zYvHL}8TvlrK{HThgYD;+WVDaZ?C<|X#0u=KJDki~@~-fRo2Z6@5$bq9>EC~yn{(#L5W+WYNj0o#Fi@a-*=f&C~qu5;R6ZSjN0*54vWsvCsi z4!v*gyuUr;RqG{<(^M8qrSL>DdEg!gh%{Sd-|#{$Oy2ku1vZIqw-$Ex;1c@t`J}eh zV|~$jFS>IYf+{DwfJOy`I%^(nugC3jm!8qpkTHGI`hT>(AZMJGvn8`Q-x)M26t%T~ zRu^kLrpXZs81q5w+Nk8;lb=a0MP-zF>-jxw*e$x*ft3U$KRg@p(paEb?71l5uduvS zc^FnwQyyVa$B!95M3C^nGpNq+$BUQJsDQq*#?m#$An-Vj2>d#~qkmh_i#5v~HMu>% z1X%b7Q2%Cw@XyTb`9b~cR|37tNLX7iEDhzvXj8?(VsUgVOO z>sBF%j2`qYzu{GVa9hDwt=;#j1GpR?g zst&#cwqY%(Fq?upGZqhu|z2_c?v zF)eun@s*F#;+&kmcaFf~sDF_o<|7Li5>o0w_j_yC^p!knD6Qlu#XrqLV`gAWv*TYa zGRVv74$@y*$;$x}dVe)!j^?hi zJ};FVEQ3`3sTFYKKt1x~*Mw7@nZK18%hl;Rn80y6b$6jhsCYr~dE^r;(H@%o7CaJ& z-=JQNy+Nj299wso^Ly5f|E1{Xl~nRVTwvI3MvPZrF16(WzGr=@G06{^K77|oGg0g3 z(T_?^ZUE7iu?}QOW%nWzUZtasjZnh*TRa zKJI_&={oSozupv5c*i!yZiV9=gT_gtZ;j5+PGLr#_azxaQ>D9D_U$!(IWU|{wz8To zST^{!{O}=YwYM_x5y|W?7q{6m)?l?>F>N1*M4jKAfQnDk8oyeYoP;#SqdEGyZ-vo# z7(IkLpDe^?`Msq0G3gAQt**ml-yU=lImmW5PUj`dQEzSY)G)iE!uVt%nXQ4~67Fuj zA3pj*E7bctFn)e$T%u3wJ{O!N61&FBg7=={N?f$~ZORMusnB4qzSPX9eT zn6%C(@g6>#`jMVrW2bTZMgYz~EJf)D-grfAVZLzxS7CdZ)X)tjvY z#Di}&7w2;U=<9y>iFayjpYtk~WxfYjsSV99KVA*9Xx4h-thOSsZf9r-KKuF5(b5`j z=2CH5@AL|s%NA-)n}a%T3`WxVG8|me-4?8 zY{nBVbzIn>!`SnUB|l%z3+5n^0zq@A%fzOyq;h zTxal4KOVcebaIx6tv3j80s;cA`P5Dut*dqS2Y)t<@{;ezD8EblwQ@NOswkj$7guv= zFdKY2?$mEzcG?CRqw{kWcnr^#g=xtuW+|qtF=MC7!GNyr2a|ol2?e7wf;&stJ=Ix{ zrah8%w)y1V5BT!L;IIN~9_+r@-%N-HH5?45ty?IBOK5(^ot=0aLpWo2xHW99eEZ0w z3yX_v?hCX_fRi7KrnwvQwQd(gcz%?wuC7J1_TIG&sBwop%F6A=b0>Vx2NSX9?ZcnT z)oW>lGB^qqoc3<4d|#v#Xx8PuYfJYYZ(Cf!9E(721?4F`m@=)lpO476>yaugR<5@h zF#oktplwtYoLAY0dJbV zJ+lAI>%O9LdN3R$&V28ah(L_Z?eKH=YFCC)hcnLKppwaEYYs%EaOah1hHO!@_1x@y zhS`}gTtCY6mL-y4yj{o&^t!f8S!OVpZB#k&ySSy_&Oh@3h%(@}*K?ouaA1IZadkN` zdD%q-m}lBxfyC6Ke?RBoI9RK7d-Q_#j2tTtF0%Gl=I}jsNIi$^Q!^{yvae zZZ@4IjVYJEJ)-v&ECXDHeR?Cq^9#d`47%~7w? zvyp7U+n?bqHP^>1HgYB$F36ukG13-czu-8Zt6WY%l% zO-=5@#xnqII_bR-|M}tI_NMB5+2LU7xmx<1jIZq7h94E~w^|OX*%l@kDqLjATd3gK z>m3oacXPVBR|`gbh2%-+%d{%>J?|GfMtkV~s^@WWkJR$jbQV<#Z+g5h<&p}k&< z+zP^b8EUz$j>x)z_j!VSSLL_v_TYLHn3Ti=?fNK$*kk!JNnh=z>ej|;`D=Q^{`;)5 zm8s~wDHutpL8AIq&BVfjS-(l~-h?mP+_r=ZefAIE}L2-56x+sAF2^xZX2m}c3 zt|7tQ-Q7F5I|L`e-QC^Y-D%vtai@_!o%g%D&e`AoZq@y9>-^{{=&tUy=9*)UIp!G8 zGai7#BcxKhBO+brcCwDsaxN(xx6*6fnxTk2k_H!-L6s|*eJzO1@+4wO`Q=W}^VJSE z;M*Z&RnKCsh?%$J)j#5fI9^~jCq7v9%*O5d^f6x7ZgwJpN8EiGyUH3Z-yCzO}ow1kJ#{`DRY+x5@1YwUl9{S7Gm03=TWl z-9b|H%bn;Dn1}mdr#dTOR8`f%Vh4YGS(&tlo3kfOq38tERRf?4h(K9=SE%@+ zc{GvndOWJ~!E{iq?x$UEQKwwD>m3^y>0qhd>_f4h&3-8(M|>yF>OFqTt!o>fQ0{iK z;NfoHBl3c8WSE8J;#3CIvrMFVmtM6^W-{)|pWT;fc3cGCGvK5%3aa@I%)fPe2WY=L zUD-K&_;fy=irf`^TYYmb=sQSy-aiRl#i}cQLqlJQc=C3p$cWcoSJMlI-QfQV3$WfE zu++wV8SJIg;fdcGj5-R@Z3i2vR=(m@F{IBxce=6qu?8hl<50|h_m_SATRo58{ze73 zV(kOD-fI9y&ls^bURKLhtP~C_S&7h~lYpm-bDSYKG$d34wk(fH-QAZYogHt8+Ld5d zH)^c~UeuAr#+DpcskS}zyJvpwqZ?1Rxa8wnO|x0Ta0G1TUA@auFlnqA12-#`j`UjY zc`{n`TlM!Y^03VMshq7;jGSf5paP67#->c6eoe z_PXcXd%C+n_fJnJ)@gl7=A@K0Tx(fu_3`JZy0>_n=trxz`RzR(i1)|8#}V%oBIOn; zDRvpU}$2o?s@g)9mF9ze8`ABa&u5}4cJ-^V;9Z2I${iWAgGvI}TtG8?R zVz8bJAtU|KsrCw<&WyTv@k1GqOO}DoSi~Byd}}?-r-LUrXgrk=YS%uxGzf)0^*xL|(Hm!lrnsxyQx5vUa+e!35lrhGO+E0oO;y6=6{=J^csa2zhfcR5$f zw@}I`lTtEmVg|VCtG=ppeFOt84KWH|t%~CA^d{1?r>uLPrWs(Ki=HwRe0(tDM@7=O zyyKuVmTzK(THvo|F+GwFTu%!P2!KoFRMG9o%lTFt>QtJ*Z2W*1#;CjNhe-0t!hsKF z<4j^_lHcQU;iO-inr=6w?4^mrLh}ii>deEmHdF$0@Vc?uhstuIm94JvV3e^jChg`6 z%7`xC860W4=u>BQ{PpyQJ7+pwbxwjQ`#Wbcj7#+~) z0Z9d1weGxl@5uj8@3Xx=zg~+exc4Hz6;E6L(kMydb5(kEI}3SPDsZebzpWb2d}%+N zdz^eL$nM*JT`=#WSaNf<$T=*~6b+5U%Whf-F@Vi&>F^|9Kd|RbGo?)mrzqOA% z&lK7MUiZT+0=Z3~W^*Pn5AXZmqR=KaWnvyce#5UYf(C3AtCKRr9>JUk^CeH&OddbA z!incRLG6v9L#+a8>VUhGuN-z@ z7cVNQcYx?W|A8D=G>~EU+6#RtZ46fp50cUqT{%HRMPoNiGH=9_W8Qxg>igF==i~Sv zc*K9RcOeo`nV0{uF39?bp;D&*^X;YArSR>JxTn)2K41DNxM^o+7$z4n>dN4(n!ov|y@dy{hy;}JJ+M_(UN znqLXlh67%IhbQrL5e}Z?GP&L$-86y-UY$!F2=lUt2}H&mnx)+>1T^+}6MC$yX%HJb zU%Bsp=}(WT9Zk>lC0tH^NVY87yN=bs-5U>9$dxFvwcaYYenE?T{~j=?zVb`X^tKzH z$r?{~Q6`EvY^C%^X$Xk-J#^-P;(urE7r|FuBlTvwh_WFvx^)juQ8sTMTPqT-$BV{! z=9@*16ot~CRTgvwpVE`b2ws=mSV-w_JL`8gL@Fj-oT8bf)7kuW)7iO5!;R)*I3-wN zWl16;9>a?H<3@Ogw7M@kC&+TMdaUj&zZLlOU;(lB0t}A3hQE)K$W%7yFSZ72DB;97 zD~cBX%>=(`0<-ALf3lAJM8|{wGGCE3lKH+AcdLWJ-)1c*S)L`loArEJ_+=Sfn6Pv5 z;z=I1?zn6j<|j&u3r^PS z13sl~`Qw1c9tHj)K6l!$rf0&5AJYh|bmILJjlD^UX(HKs9K^K@w%MqP_tBKL&=Xm2 zp++Stt=-Vlsxb#UUvG1(YC1?2&9)e|Ms1e-#H~%vXQ(LAaHvT$>KQZ_G#lVwutBsyhZf=!g2}av5RGHtb&o?%Xm^F2s!Nz9+V%^dlRNEPsNx zSxl97d*pU^E&0mbkRJG@RwK>0Txz0YtBrY&9l>#SQ$A^Oz3Fi*O11%EMO#MhJGExN z_`*T0D=kem9gL(f@ePl{Z}P)@6=4Om(g9R6-4ysviOW97c;VGoToV?%A$`>&x~wY$ zd1I`dr>9)it{9fcX-r@aqXOo~`eYM0c3=Ze?u$^{e^Ig~tgmK)zT(GRlGJN!-nT;Cz9;PB;j0 zN1dc~=WP6=66s!6HnOD}(=p3%dpbJ7bJVgo%p5;H_$O1;7aYkEJ>@Ar;{H`!mAr9$ zY4c3^fgg4U-!b$nvn=0Ib)=2czPPhmO7NU*Aty8UAi~=#H+OACjuEWKP624ATs-5O zXET13!q7i4-PIoeRs_SzyxgxwG+Bj}HcYb;VPyqS+#OiRD6XRY+VShz5M>^$P)ngt1ds5eDZLp~25 zqJCZ9K_QJs{z7NSZbs@A|K5ODK5O4uefk5-mQ4m*xv4fpeqGY*mx0p2hb#mJKW@(C zFBp>_!0K|k>VHC=PhelybyPqP(xH1?_*Ufj>z!17FXZA)H$=0A346Ra5gSysu3)y0 z73YpRG{T);Uok^&DKDxElj>(9f7eoB&VVPdo819AON7`~l;$IBR!dMJ_REmZ|LmP5 z`E2OM@-gIyf^sEHYsLPQt}M*M&Bh>9qQ#bk_b+VP?aKC!rz49_9zz0Ya_-MW-^$c9 zxbBmrx9{pvXr8o)`9gN%%)yPSStis9Uy~^+G^ISRNM=}_tE}S$CxDaJ#+6y`lSQ^G zj7^of3S^W8BD6a~=8~8nis#>si&L`mJTJrOgXD4YPxKXhWb(&tIndMEdy)v+ljp4A z+*-8%sL7D$mBSXjOR5fOJ29qood574S-N3eNcV~jR{PBlnXV%EKHas0it+ELm7mc| zwvQ)=z6=vh;Sa5J2AJf+D(LYuQlTUK_!3*e8q;pYTAgXCDsX;Whh{Q;C;icAM_hVI zbF-!RNz5QmD-^q)|uXOCFh2aDv8gu{8>Z%zo(Rhd%XP`Ny$mCrhavrxAHJqaC)A8rZroJ+6HZ_*!=wnKklHx8zv{JRqQrSm8=H z`1WWM7(GG1`wxQgL)PMaHneSIhdjNP;Sa)*ITe$a%C*#F=BXV`AN6S_^GA|DiIWCP z3({9N2+*nvUY^SFy5Nr=Q)T=MUPlfZII7K=+Fqo}-B!v(M=LwOG%}^H&Q>LX@D287 ze8DD{fU!fp=1Ds(ANg*6rZ_RWqr8*Wekus4Sxy_<34(`qh`M`(nkRZ&Vc#w<)xqmF z<+HF~b5mwi=5sCSrmEwPRf$0-wIr;(FA-ZT&)WfB_0Y)8t;=njs%+?h&Q-=?}lwa zU|N&Z5$Xon0rDtCG3zZ65zz>TW}%t?Jg@YXPd|A57wO=Rq~JkC*#?Iwg~@ zbVpJZQY7RbS?c&MFMH?PP%8n=GzUE!1~5ALV;2Z9Jz%L?<_zp1{2>Wsab*0qQE zW*fbGup`oP!nR%IwQ%(m!i^w2sO4hDe5odyX`qZXsS_$rE^B(Ei;|&~%9I;8pk1s2 zez#&IUBzN5sMKHo!!=WbdN{jq0r~&X0H4MSkXT#|I7ZR=xfS@^c7~mN)Hrv=k*4 zZK!>~z1$a0I1?!O_M;S_*tesNMkV$~TZij!I>L;H!fS`+`9}i{_-&9`s?>4;i<51g zca)y)JZF=g6TWxxj25e~dS+X7LN`+6Nr}c)FQHh4XecSz_WP1*Ihei-GdY_DcQS7s>iM_UpbxUiP@1E8>A>_tHz6W>$_#NEdllyvWYO;_iCVXrM;_$( z`+1VwjVzBdH=!W+m^1nP3!f!_LG5|?_aE(nH1vPAvL`QOMF~YaSd%|F^~0&9pL0~J zZM{I1=;U5PY}|mTjzz6ZpYl*c8Yqm2;S}T>=d);d0{T7J6wYM^&SwB)Xasz7|IgV< z-t8ev1>4~a2y&;uAx};fG%9>W0pDXD5#V*#kky6R+Rne2-r$4ql+vJIYK;f#VKd<#tln}pt^ZdG+ zE^R7&N?HjG&=;sGQUDkD-gxmFRs)GiVflZRmCm)}}rKRka$xc!$>$FvZ2=Jiz5a|u?gML{sYLvSa<54qV z6Z4}pVG9aeAEYuI(i$y-0hPh!jwU!%9|BocfdNxTE6mCaG1MK5EJl`sV<1rs93Foa zDp-6$FE(AZR3*Xz$`VB-?xNz??(+IjNo3OJHDcyWc9jZ`d>mQhzG{=oH#^ZQOLcCf z17Sx$k9-S0MTLq4*K#$Y_Ua-tjs_4cd;jS5m9V=umLS4J48B~zVI9|0>i}qI=kUb8pd9%NJmo6lgu(Qj0Ci_Zzrv|sYbl|wt|d{NkST>0T!W&~fqEjiDXK+F z5LUZpDZ%+I^5*6`woUHGRk1z?A+t}sd}pRa^O*JnGD)C{TpnquY^cCC59J16{=u7VCP>u4LI#^ z&PPmA+w^KQYzXP9vM3nX?iMaSc%1Hb58XcQ9lt7whKsX(jcyKmId zDoifBYXq+ruBy*5>2B--M*LC=)f(``H3T}JCoxIODL-8})zPbDm2M3Ss*$=oDh=th z2FC?Fs{d4touZAX78WRSChe{iBaB$tpdh0U`S!$GCWmYkrgYm{n%6hv7e{epikwhJ z;hce#tJcUt{p2w9MKre++$=*YeuzDkP&h)Sas)Sa%>`v>o=n21%&Vl)asV zP=t{Y+rnc1_a5v@G}e_f|Xm@&*PhfM1Fb$GUF9U|{3*kmqhtfj{6 zyIH&P24kheU#Z4SQ6;?L=Tom2tSz%M>W*pf|8mL8r0MHV6anLOnz@iyAYeY?3^N9% zXgFinXHK0ul17==tVBkV%4XvIy|=WWW25f9bA~*7@Zp%fP*%d>?pUnS*-ZQicw1Yn zQaNxlr7hHy z%A~Lb>HxoFqLQ~S$shG=orI{lNdFKK5?7N{B!ARb;u%nUFt_<6tQ2sxN|&7Oy9d;6 z@M^wdv{hyOpHj_FtO5mbp!4$2YG6K`Pacp^q7?pV$#r+f8kdj3iC$T0EM^2BEk&>?j-R<2DW`fP?3&q zN1l*jdvN0G+6H7BvmFo4X~ZVG^Q5ZfuzuCMoYVc$H8mVaT|6c=wAT_ZuOP?~oCqgl zoqnaUgpz^a*jlS#+V`8}O&o!EGJA@Q8xEc9qj844u{lJ)u2pbwl|1cDbdl>JF0S$& z0~ar^KgKG_sYs=>sk67KpD067nhG_8t$7>UNf^P{80iYr$tiV#ERVnK^Rg9d0}TP} zH(0=f_t#cg^-hIBQKX6Tm&+%BfE|h(CakUzIElNh%p=(C$n>U|HYxt6FG>KiTvT#z zU^6mPI>C5}h#9n$5}X*&dsqz+w7B!a!`k($^+KZ&gSRvfAtr3uL{U#;yFo(fOKMKX zOca{$%P!N^N;J9D0f`8i3%WP&{&oYm@F(?Z66%&2q;TK7sXRCv*D}Hb6Cd zN+0X|_~r$%?lC#WJ{$5I(wkITPT0V`mnRf=rHYE+L&gKvly1Og+_)G-GoX`PXE`DJ zWV{}T-K-G`_sQz5GFQD8^#>cNDy@gpB(~E+cx6XUn5|2!o}kJ4ZI{tANN_a8m0qcv zi^yjT2V9_wJJdTq%bL3fVr+Z*2F8&C8LkLhvVKAzAy;^F-tlQzc$pF^ta_M~Q1zwg z{TEEp+Ts)Sei-UKgFuF&R_A`=poR5}CT`_L*-L$M>!r9Vu;(`UGcEtutS0xV(OHbS z4A;hz3t3?_-rBHBuoUiU%F|RX=ArX}I%2II89LS^Qvp=uKE)&*jZL73l(Rqk?g+J# z0r1B#orl>&jqRnLL4Gt=J}eg55+12?U#A$pcF8}Wl#u2}kQSyGT|o_%g(SWDSbQOr zBb6dzxx`lH)|yLOxuKyco!#8@%*(82|J`ESUu5Yc-KmV1Ny06b6N zuv$x)&R)=)fg5XW7A{4k6-D+|#iE2HB47hhGm$zwETap1iKt2D*T+Z)HM>Kd8X*y} zl|+`BB&H13CGSW@Eq7Vlg8;YE;q!|Wr{+i$5DTWuAE>2b$laN){)CI>BXqY}c&9a& z&k)xtUj2x&kKB0ccq}A5--jw$F^`Eft}efoLqm(m9wjp;EOUFFkTk6RfaNDr@OztR zEXy|)d7)TI$9&g+q$8;ZS&a}Mb3bQWlC=0bNj@Hnt-F%7%!G$@uuw@|^HDwFBW92( zYfIj|wfLlPqZd19R=VlQecRavm8s4^VGQq`)<&w_Nc9N&V)Hd#&gUn6UEUC%d@b?> ztV;WW?*id}cp|q9CYfv>F-Ssx8Ct7hW9`LM=?mc7R{l;(mjg@d-6SJ;)3PWiYDybm za8ctp%N|K2Lhc24u&6lm44{4|*eZJ!=|e2OjMy2aqZ0^4S}E`*EeQu09uR0M`c76lRoIdc|Uu|Ai3DH>qPWUvodba<(ckqKB+_#bE zRqk|)XVA<{12hrXXDQ=JSH!>XEs#L*ar#Qt+A$DlMyq|Jaka#>N+?&3AXknklEX@? z^PmxNH0I&N1X(OF+>8TO$Pj8<>g@MzV@nG~>p!13EGB3eWJI^@`sI)rpBQIQgldSq zT;5)piaS!8LDXc}q~Iqy7?}+M(ZNaYdRp8!pIqN+MXO}U)&)8^yHfub79e~Jvg0{U?fl#-`7!Ir1c-BuPt!qty@VsNNj_}AZ z5@MapcLnZ*y?XKz(SCJrc7>d?;US5wqPy{f_z!lTULLHh<<>N#GXzl@Nc(z`Q2rCf zv6i!Rxf9JYAT&7TdJQKmSU1}*CUvX=grGC$y>(;b6i6=-pPBHL&neh2k;8lhVTVY| zZKmvoE?Dnh-8h@hH6jl0PsQ|>5zr$e;dcId4N<_B#>6pZ@4XDH^do<{P-Lad^(#9xJ5=+PhAYBbFC(48_5T z3zM^trxBkYlBHVk+usv+kE|_hW)}0$nTHHkvpTXjE&9y)Es+ta9F-OG$0PA z8H*zSDJ}>4!$8Uv zKeZ2?_Qhw{yst5dNO7H7ZpPa=D)FBqQb^ODa(8KJh*Lq`VU4GT1|R%W>nAL=cTv<8 z8Rjd^tBPG;VG_ftwP~b`g4{K%S&SMSdQUBRm&9dQshIG`hU5yAz|gH7UX2O>@EzUmROJDzU=KnH(We=hxKo8PiSXa0N)V_asVii4IY1)*Kt zGL3u(0S(qmdTw2fbJF__#u2AI_ypGNT=I48_*jZhLEj|00TGT}8>6T{6n3yTmE0&J zmKz$e95~ESuMML%IrnXX!OKv<2|isAf?~q>=lm@PX*@#B4%tG5L`hW(+~BCgfE=kW zpw;^DftGzgBN60-E0*CgRztv5K2VR(_0SKA%Dp(rq0?WpwD{sX;sp#;C%g78ml5XP zw%f7e40hg-%|54&TbINZc-8tTr2R?ST;$v3Pk1vM(u|vim}*|c+9ja{PPZah@N{my z{|rY=hd1Y?a%4aX3E9h33D%tV%i;Wt_&~>w`Ux?Bn%!VXnhKPJj|dIabxka2N{*6D zuUserH9B16l@(%TMVH8Dk%?CHkG636f3FY_urkA~Wij_dq|(nz{%+v+l~Vo%G0KLw zZ18YM)*u_TEP;E_gZ$O$0XzDqe=18I247oIx8z65O!sW78I_55(1-J|oedVly{5R{ z+##h@WWP1HI>!Vy`)J0dfU_E(m78uu_y0*>S2h1>g|g*g%eKEqjjDcf8ACDN#k)kS+z4b zb>lR~oc9;7VsTsvtt>A{+H8GaK)Z>AegA$ci`w#8#C^x}M$mY**s6;FlxbsWjmh{X zUz^itINg9dQACr$@eGJq9_wEKO*O3A0d0)nPrKmz2ER(;!mE|?TvygfV6r1?o zgg9*^sYqi=Du)~uJO4Ii>SKl*RQaK`w1_h`RvQo14xA zV)Z>4c|EzYBMD;_@rpgDszKqJ>+_PnRfpgm5gGkN%xy1#PL|~D4f2#h45{x!#mxgIJh)MW=eFg>1cpd@-m{$#M&r4T>P+^`ZHu5n|(9b=TE*)gxHP zv%_4oX;~dQ^0Y|HT?ZFxHIF>AzWGB!X9|ZHGjEIr(C#wkKFst6VQw6(G-yl3@3rsO z0O`2?)Tb_7G{(qNmWdnp_sRQ&ouIq8`uV-_x~qOq<~Rir2Tt$hiIon{ z@PWnzF?2e2I%Psp`%;1|+%G+o$^id^NXf*%ifY*xPm}7I`3A?q#1wWff%sn76WH1= zC$YnxA7Z2K%lM5!4x#wReu|b9CYRp@1c1@eGpIzryng=)w4jq<*6;SW?PW_a24C1p z3M^F{s9&Kp$mMGq_HBL`7%B6d**G5$#s2N-IE7L;vlL?==-a#^E+f02``%BVEwR!| zca|9$y~4oI_;XaRfI;WFPzFP+2_t-ta~c1}e7A%6{6Q>VF1qC;M%fsX{U_(QnjmVH zidyYR>KfsZovP?fL?Gp?gZ9f6pV^ z0xr$g{vN4_V7qKaWlSTU@Qq&-xu?b-p%f+J396&aEo@s?wZycRdpV1x)Z*%4ia|zw z-^Ado;!6rDJ4%|i27yDHDe=e!AZ~ff)gMy!J49^MH)J~YW3m-sQ7aL6n+vH3BD{!& zM@n1c)l&h03>NB3E=O#e6RVO4-;_Y>FS!OvOkQ#_)sjWyJfwr2_ZttOfA)!P@?E?7Y5`;A(cWV1Q`b z5R9PKK}RFB_#E1}kb+mmHrt{tCCZ2Czr5(o+Sbw{zjHKylRo!{#Do1c*H8keVTPcN z<(hU&D=1c0omQ}`;WIuIjQaD{=%2!6UE8*ypu|*cj z$6A!6{w9o<@=`c1Oa4gBweXdUTaLI$325O=>{}f`Q@RsWfI9M!n}X$g3vv}Hx#u=M zdzYH(NCx&lHO8MIGcE=Lz(->rFv0QKbuw;Z=np`y+QU28y4Q$^B@ySPnTn-L3|DSw zkuQ*S3K-yf30TN?gO*M2Doy0I;H7klYRzl|g{x1vtv`9afQpjk$mIN9C~!Wn6s{h|OWS%xI70f4>6|#}hO56u_Z7Y@9GvU9}7RZuY00 zF+k@f3gN)gkL@@rN)EX*+)^$~e@Jt|i<{|F3ZOI>49s@|ZW8 zbi*15S2H>I@^!x(3+TS|zTaT47zV2mTJuzVBRNs~p z4z!UnR1$zqW1GuygLm_}7Yt5dY&0c=dY9`ZkjLrVldwW_F0?h_~cT1^V%Zp^g2UPfJVM^z`nWD?L!>&+^+cjk_l5 zQP%~Cr%Kk$Pxqo57f?A}NO?Tm=GJ|nn5@gz*-uBcGHQyLf?Jyzc1`k%lEJb6o3+PJ z32moxCis7kwP)uZT$y_O|BUqpolko%qW4M zRW?A~#!`k~E0twl$y+VKTP{y2K=zp8QryiB(|4Gdc1pe(ADr@BBlY)6fXs~eV|`7Y zU!yrJ&zC|oue^&cM&g%WwZ23PKB_!|k07tsN<5l%?u+}ANMwC$@`=vHP^_JmC*&0D!LZc9`eP!Y7YUUc2?;s)oCaP77@y zhAr|}9zF%{S6J-@L?j`1c|zZj_Z3Pqe~<8_0Hpt4Z6 z??8WH~wP|eV!$V|QxDK4qM&Y+)PCMazFMd*iU!*>Kfugg)Zs+t} zqob)C-n5&kcFxu2Td>jv8)tV%^CiMFW~mivf_4KcYiWAYrac>vRnIaSe|y89b%3Gl zx4+Nzzi_yHq$}+meZx1+ql@(~v^976*PSzvEy#gThWDLYPC>h9b~AXhdI3r=aXU#J zbGKTx;awN07`H~6a)rKeBEHlk_ZVV#y!h>TI?uB`ow4)TpS@IVy4?eYbv?O%76MF! zl5B7oG{5DF%C=okIHs`M3_M9e`p1%IA5GN=pwz>?cr{U0^UYe$li4*Lk$Yd58lMNh zo}O<5Z9F()KMaoNIPG}6T--x3J$&sT*Vu^np1nbT6z#8Cj=ukm4E}2%|NPaLZfXaC znV`XC>0F6ivi9QfzK^omC0zh?RDJLjduh3Ly-=ba2}qLe{PQ3|hdY7G0tKw6r4caO zZlf$zX!~njC!pXxz(T1`k3HWw1{e-JvmXnh?qjTKB{R{zkMW;d-rDbtU`LqA#wL9T z*K5o$=Y0+hA^G{`%Moi0`dKG3_m{r&XeejEvYv3`M6%Zy92pl^nZ)jaM$F^5+j`C- zUpjvxA$Kr#G@Hilm6GW*GdtXfxp7kB=G~4-g_5>t3?rmKWHAN9^Lq5V6(GquUeSRG zRWAQ;gADx8Ku!Nb+AL_y;<(Gz25kpdXMx^z1R-epxrc<#x7k%IB3gA@8%dY)gg!=z z*)4~f1*J~BClu(h21lBNo6h(mw6U3f&a~AY-*%hjasX3IqTvH(2TKnXAZLzu=dw3B z$5)cVK?s0QP!MobLiz&qe1TBRWFlwPLId=K#I5|(@l`{Cuz;m zTB?vYkt{Ua-s3e1L->*Tid?eu99RmZ4M_&^0Fxai(m>i94zhy>aImls*O?Ii=?ApV zhyK;;sLsh7&Ju{bFFxzX#~o4^PhMOnKfd7qfNkHZSWV<$**W9xla|xJNg|!S>4zw) zYW8fD++ExPZTR9lns(YD=Fc&|AW5F<%(h6#2T7;rL@!MkzE?Bp#%1e6CW$)Ex|>i; zB#6%XnSPFBHG27Yb!5}-i8fTK$xOad=hu1b$OsZkre?ngI^#a5a%E`D(?6C<4)&ts z(d#VPi}9SK2>uh~R=cFM6ov~*I4pay!BJ6BS;gT^&oA!WubmXgZ)+>tS&T)6bjx|O zWHdcMMmJu;Ks(6x>Si^dKlj zukgA3l4-VJa#JwJc27(#8g}zMGGvl>0Ns=Aa1g%Mdlsaldw+YoqXz-^oL+mq+yc|1 zj(XJ!Wu2a%<`TzUqo2Dswd+3Mz7Z4@%y>MlJy}#3Yk20%DU{v`rDA!!fJ25n3<|m) z7thzF@$@sEwTDdSip4f;bFy)G_eBx+dp>L-{(ZqbPc{IXU%?;=>9$rWUxY4?bK>or zquH`yO{wj4hzl1(ty+!$|6mtvdvu?Rsj=RHh0uu05YoCa<4s!MwJ`^R9D%viYAjng zaD+esVJnc@>eXx}GcygiZO8M`gq3uzb8umN67|qi?Dche$%uo~A-~aOuv6@{D}=Gr z5oN#x8KBU#6-;<>)$+#-C;-<59n;5;ruLw6CFpQo{d5JRA|kLoFZ_bg1*+^veV*fr zm5dgvb06A_UY<4}`&Ss`E&S2Icr_O16ZML=E6EPbe6l0N<=IpFa7(tO5qp$5Pk{e_U>l&{pntBjQdiE?%S~Pr&@sN%Oyt?UMgeWi+u( z<$)kr%MOZnd%A4r6e!~5;&+vKwj@RO$ zYZZ`YDu!8Ux7K&!7HOk2^kjpTev-L`psG_XE(xqff?8U`r>sb4KBX>(s8dLEhQ zFhRb+2rvQvwCR5x5X7mjG`ycOi~F!4j3$OTnn=I5)!+HqYTOPV`ZP0N>o9jJ1m9fj zV&dn$x^u^S++PQfM&wOcQ<{yM!hHMI)besqu2jN> z>i{NKO3K4ZtItm3WEp+~=w#fadZ*%Pe}gM=hkJfOw%)~lkhBu)hkLx$iKcf9J;{E| z>jZc62uvy4*>q#bMo|A`{2mxW9QmAOS^G%+qs7HEV#AS@wy}k*;3`HXg(XX z{g5n9^?%51-DhKN++ZtAkD5Wzb}0ORaSgwuBK}SC zEU|&m--+Cp4s-|V>zDlxWMSwB!T&G(9Tet&fhhjl2%$!gXZRm@ zJ$RIfs4lR9A9_p8Z z!GK&0%+j-i^y)Mtkauv2Ho{IcHTr26WZ$2*iWw;KO_n4Jf}e+X-r?b&3mx;T?`Q6d|<6FUxy z)Z9NqZ+o9njQ7jMo<^)mzBZt`SibcQNeDE9bZsk@HkqlwkDubY!=V1g4>Hz?9rYST zuF^65;VzEyz>9C3@@@v%g-P(;92_Xg>x{p}<Yvox}QK=AJQZyC4qWkTmn$MzpzHU@qTxJ+j4`?*k~cczHU zodwY?YKCYTF;mI#5tr_C4isKqA!EStLhgr}A1fj?zk{!g3nGA94tc)twWPT-^LmPP z2>W@3%v3yY)3>vnfjA2S_XPEK4nG$~q)uXe^n*4MqPm&N$s)_wG~7bGk@KUHGyeb3H1W@oIw^4wNyx_Ybm#|8&1VdV`-S4X!*l8tD^ai{hbaEZF@HjP3wzVZaMMaEpnO{?cyCFJ0O3{e$GMy-kT{VY6~3Qy zud7@guK9zE$E0d=GMpqPDZJwb%sDhlA}lwU#0)KCUzjdKo8f*kg0YF=AWQY{mnlnW ztjVQ6yb+e2BU}>L=F0T3ns4^m(-*Rd2}vmG^LYPwa!=PJ7;?<&G$bN&6U!^k-ySxI zz9Ds2CaUP&Jn_;M+HL!Um_mZ(cxmN&`bhDRN%1k0gB#Qq5`5t{c{#RiarlCzZ1pCZ zQC(sfV~UE-;Bxi*-+6GW`8{$9a((?lzJPP_UTRw{{Cuxr=$#{pGo4B0)ian#4N7{G z^NheoUzJggNoXM7!?MvHziv%)8m6KP)vI)k!*4viu(mb!>E=;2BTVn)G@FV27@}LP z^mIob!v3GgtzG^$G=kS7c)>{+!Z+BS2 zyc>ffTPN%W8BGg2theV9YsN!AY;QaNEBfejNrFxeerUvNQ_|YGPD(faAH?3gR8Kn_ z9?4r!>7z`;dk>ETEhYGuF*fhpS)Xcp6PjR%w&?BQ558EI_ov+1MI(!NrqaRf{U#IL zn5W7(mC)8QY(ag2@XwbUmw6B#wDh+}fBhv?a4-C4w>q7o(S2F6ei0C?YF-3yZPcFkDdqmK z;2j~m79;M)-_U?F4@g1BuHv3-7b+KKCfd^vWYiR_(y}Zf9ScR+$ZjBoIi=X=`w*a( zDycOXj?}_i8GvJI`Ncs_Q|zZz|FUw#Cl-Azd`4a;bd*&1gBZnfj(xu*U>X2_ek_zR zxtpP%TU6;1K6L=RJ#wmmd`oI9U6+!U{udSics&(~+Sa5`!~9-? zueJ2jTA;2}y^*ocJ{NQyo8FD@YW_}cVp~mN&~5KG*GDDYy*@dT&+LFSD9ySNN?^iB z1;1@%ZyzCl58%_i6WOm&|DK1#g&O5fPN;N7VS7d4-CT|EvVHq}Tu5MUVGLUBss(<; z8PtqL9qm9mPR*9#mvpj4xc*3KATFMLc(cZrd^X~ud~%NO_BtGomA-Psvld+c44*Kb z9%gWLTDH71`8;pY7e&(YnUikfs|WHG(f3OABf7r`4F;Nj!T!ZYsNrgJ+PQA-Dr85~ zp=w$)L(B_C4a;2Wf21s*m;B@yJ45Nb+mWufM5TkQemSk1Y+=TW0^AsCwIK@wW!>t~ z`nWlKMI?D?UGR2!GZsN9lZslEh45Q}gNp0j?5uDdkxO0jA7(<<^sp&ent%+H9|~c)c;kHq3Xk`bB&FkT~d>viKcz z*8RptSPW|xna?cNXP!MF6^Q`tw!dPT6Y=WQ%B_g|J074lGr`MHT9` zAI+8Om+X7zgEXX;Lo5!bT^$aG0jmjKFPF!4Yi31d?WkL1Y*`og+7SN%JrU5LigJfp z@49`=i__rrDqh$^?X?2?+d-6=@0ECBW`q7{HY&X?_xd90%+bcQg~w$`?%~fa@H>CW1i?f3U#Hji zi=;$b{qusbE7_g$M)gMJ>#l#24S)Z>PZzGm?41P{uDn^jewjyvYUpTYhL8^; zwhgSyWJ=+>N%x-+v=IR;&VCHyOyMa+Ogm5dxqu2qbL(8U2V5r&35>fM7LUmfe=du( zp7i1FTFfhx*FJXCB0pW>zix%yj%yL8=eX$*%iiy~(aY5aQjCx{ou&X5emCs-KPaW| z<;95-F_6znb;mfBC^^GIrZKVV+rJu~Ujc4+6;f$N@&=WI@>mnfTEHLlSTdofZ6se` zqsnwNg|w}p6xPlvB)=fmCFAF>mj*b2>Zo8Sv3b3xt2bi>LQ~-zh;1*fJ2L=Nx|xaw zrKS}bdr;v#N+?QbQT<}&N{Xq-Mi5sZW+t45r7|9#yoP&f{mdTmpn`;%FkIdPQI+?U zzX$@e6%Dxt6lQQ30dR+ua#^HyMC>%>_V3Hb3hizyD>Smd&KS43dH>BzMOedON&uN)S~3+B z&6EYHc5>g^Bi5X0%GT{NDelRAXCCYn2^+tBW#J8s%LT~3S#9wr;BsioyWnEjXo^ED zWzMa1N*H@0pk3QatfNce5$H2Avdo&d9;n{?IL0M+g#T*x*@&=fwyGuna;^Aj5FV_n zJVIpjxc3QD$t&vm-d(-QmQVv7X$}mgpkavILtiW$52ci;pro-zn*+eLPP!~EJv}l@ zhPTDkeY1VYF&jlW+^QcJk+wWUB%LJLF4C2ux-g?m)-BgJ3xAIg@SDd%%`GfKl{C`u zh7c=!Zgc1KcSHfMwmU-9M5@%Z4c%L{*giiKMDwR$8h6@11K;KRnc z=+lFQ+0nyHTY<^qeJ`XYfx)Od0)v6xn(qY;mw%^)jsIY#-N0Rn_)>=@r%iGM9r7j0 zoc8w}-chhxYJWHjygiw8e}g3nk+k;H73ono0g8qv{sy_>qyq{yfxKXA!hHeQBXSQH zNwEc()R;6vt8z+Zf43CJ6I3)Ydh^r+%uM z>%<>IrDA zZ;eO0eQPp}L2v2Ul}}eNuqleV=}%@tz3vnYAdj^=|FyF^DTJ?mrfV*zMS%O%PNuNp z>Q%CZ-&^$epE_GCydM15Mjr9w(we+CdKc23e{)3!zxwEc+*ij*II4neOy!J>2y#CM zzR^7syNpJBc3evreBKx7?=*=Z1|b|4P)o@;Ip_#eCv z#DIO;gVXm<2^y~!7&S^9@$OdWD^q9Xm6Js07Zq-!e6T#zDj?%sf`nD6${V&M1IvKS z89LkEJCm^p=!J|W{vkd3ALR=K2E0R_(EWk$iJ2T1DM4Nmbr$YYEqUR~@qi@EQ}6+x z%%6z*^9irHAlVmEA2*tWJ@lsMsh=bC`ht5+!9Y{KuC)g!36jg_~0r#s5H2kx{8c;ehB}{cLg+u&!nC(u_jMn!xXS=6h*7&Um)7mG*#aT0~>*imn;Wg%KW{*9E;f zFOQ7zRfK+NC-|}+yJkIEgA~=*^E_JHIBaGn2g6hwY%2d++XbIza zGB>y-y3{|)+$l^x4&uo*2zQF+i_)vn%NCOAXr0jilYi-@8Uw`!=`vC~X8YFZk*`c> zZ)(>mBqf|YdSBv}l4KAG=@$MVedrZfF6xam*IU8jiPcaxUHm*%%V#W#P<<^FQDT#0 zP_o+H-;HuiXIuRqB;@^2&~{LUuA%YXr>8mW*Au6*1QxC|mi^WEgYT}BDJNdd(lOs` z{n#E^`%-OS<8KbUpC55Ft$40$=H*@Vs4@*0)yBIOp}x)TvJ9?&U_=bDei zXAE0A!@S87miiVleA0Hz{9m_t)1wB-FY_$96+5f(=*Q#J(hs`{%qdrPgDT5L!r+gC zpbm0k5=O_O%fqZLb}h*6pw?82kG%YW7pg={@P)dwH4k8<%7#KcvrMad{8~yl=jt}d z_%oN+K)vGLlYd)98QJ}pD716M*7G)VY(g7(!J@4D8M%yxg-R>?h|8A-;|%Xda5zIs zDd^>T?H4;|$T8OfTi=A7S^se4gJHw?QbzMD63594sCH|j(r;c9$P(1M1p9S2Aj@XC zulfW>P0siFx*_qRdm@B1#c#*njWoq{fA5BT4K~yjJlBN~Qp`C2Z20)$dJ-2mb~bHh ztw_x|t%xkbQovz+Amwl=Nmr;&z3;w0M7_v!0b%D5Sv06nxxzC}v4$jary6=1vJYx2 zC;(>d-Uv+lNm<>b`KM{lw!tS=(I8h;$6<5S0!!@O_&hF}q$F-5kX)amxr-(GW7_E= zt|mtL617DxlepYWgHvO~OBMT}}c+8gXdseoJ5Xo8Qry6mLwx3?}oNFKdeBWM>ui(-xY!u!8NXZ~=KT*C|ydNCS@%wJ=-7){}qj;M*-EDZ}$HyMW zN4Fcp4|gpT)c!f`u%k@9^crbJ_y!*Vjog zdm5PY%U|@RL=%!;hw6S*@@*9&5*>6*d;VS>$zNCY=Ed^$bI#|GW&(zX!&JW|_(6Co zn&>NH%5Qb!J3{Z2D9%;+9(2~uM{xmp<9S$P?lqE=51+}{vaoG*Yf;!UhFY9(f<*de zCdeYPyI$y+XQ>xtlAUnorW^@0Gxj-Ro8tAY784+jFXJ~~lS#@*)p@rG5}U7G7% zmag*ckyjAy5t62Ov7_j_auV#_XiRH0YGE-V=r>=1s|_czu;Ju=S{m}KRXem_M0$IZ zBR7!j+vPM!GPU8CU{qrYK-FQnM}sXAgeh|80VO&Q{BQKfeZ9H!}E@LAveh!3wnPPRzn9Z z&j{vpll##EO|$xymhr=HubyUfu9lG;uXS5i@oIHDJ%znZDO}Er0F7;BKH*JvqdGT) zH~RorgGkUwI8;CJzV~U)s@>i6e8FeSA9D9W#%i+8*q}{Lf-mLQM<=yq3 z0Hjq1Vy|pYgw@{dI?h(J$%jP8;`juP>OZUiAS@zVC z9NOW|m7nh}owq4S6Kv{~ncbzjMTgWDDSIVib8RC;tSsmZ3S6S6y$;d;x*dRUbXCF> zekHawC3RWE79oQb89&cta{x`0Z@_i{N+-IPg{Y#)vNt&MCoIp4*7W@NwoqcpHm6VP z9rrj){kq%tT(AWR|Gt-j)#UR~%@Ga%bRFVDxUT5eesSwXTX%)W3? z|C(;hQQ3N8Ck^b0+~JCfLN`CH6*_!a$&t9&Y)rKxfMy^r^HJdBy}A9uc_>w^j3MRj zFLsaj3wnZc{>y9bpjEfx5aaSU7Vm2)0T+WzNTIJI^Xr-D>k4mR`ThEKqM{E8)W1CuGZ*S}PqN;lcCTfK6HdphJN zX0hs!B~tL(VcfHRUdv%_UmH})UL+4+ktSO14-rovzq-G84~#Rq({{WBr;w^|{ogZj zn;vUFR+CMI${6$}0BP0?a$FnSMVlPvsLSslqj&UsqXZLda7a=GUE2a2`YY|JuIO!@ z04y@9>v9D;6wy^o&q5>E_b;)uI@S+x00LvjZcwd z@D^HV=qy{{C~zNx6dJkMXSUkU^mQ!cBLlbp(h^7NZSz?`94rh3aPVMT$f1N_!6UbD z1pK1ZPQbF-qTkE3F33cy%R_EC47f4ZHA$ywztt`49KzyOMgkCLUM%olYFvof*G+#5 zi`gBH>)m%8e@sdJd;mz3+usPtzWBaVhnE~(R;bH9m7mu8xFc!0rajajVnE8GU{?i?G9Imb6yXKbN=uM?*bw2 zxh>JwT=WM3pPHilXk&TGlw9=Gz*sTqvnt0)`CAyys3m>a7q8h?INerGt$mw zfk2_O6!YkffWljH2O-MhQEGJlzLp-v%6_P`u4-{LY!>TNbo-vS=^6*`QG#=S7-NXb83 zr=*^W30}!^^Ds7Mb^bsaj3z1A3K5Qhh{I-enSEujZ}+c7PPLjS=It!Ej01(Lc;gtL zGiD4XZgm|nxar*~^&$iZ{Xvp{>B~ur@p7&Hn_F!mgag=!VCi!r*TB@+reNnW^=2(6 z6~;tMf6kfII?X&-wGu;=hK&cE17E8&WoC($-Zm=ujr)cX2nn=0p}0sf(=d_85k{5xm3h%nv*{shCUV)DgD zi;-jP;k<0{Cr?LAD;vJ<3LRF*$8_`8pQjp9Xx;LB% z0G@{Jkg-g}+sCb($0K@^^N0o01()a`Ul^cZ(gWMw9!nMgLP{*Gtt+l|LWVh|BmkJx z<_6FoH62J+??I7F9oM>9H_A5?k7FwB$}J&fBbzL5j@~xkIn|S1~#WD5XeLF)~t2(+hQ~urgPrAwZUzQ4MILy#VNvz5qZZ<}G@}9)Any&y@0Ir0guz+Y{VvYTV3%ymm&|^(r zMAnaF4iru+_-;E$)uDNErp)6R!SuRWj*Zuc-j?Z`n)|CG$+wMEffswLnZ-^-1IrQ^ z#QWZy9pL$@7YOxLmED_%A5mU}4FQG9XC*xwl?o3pgp;t&nH(Cz1QvR3w+N%f7qgf+ zL{U09nP^ca!*x!@>-?dI8v!QnSoMThbrQrbhiIZmEz`5pD?B9Zr%Qc+1px46)T=i% z{yeeh&BJx{S(_Os@;YbB^fRicDbD#kP=F4b`MS~mz(8yuHfcuw22cJT%CrGme7@(L zlg$zZ&u6(DC1Mgut>~_wdcG3=tKLaKox8^Gu8C3^<3)vKj<6mc#OYE1EYNnd{Z=@1 z1V}G8#kD6i_B@Z}V#WUiO1$M3q%=x$V19Sj6x&FZ%^L zum?vp+f*bZGNiN2#QUk}9lT_eF3r?b*7%&Mz;p0DIN9{%DOlSKEqC|$qsg6eE#7Tq zM;)XIvmb$9;|#2AJMijHTipTo7$y8vlv9IZ32{kTC-1b7W$t^iDNGCyPDa$r=WBTg z{Ef!T>WoE-3!&*=Dvoqu;JqtJ+%=1U7onRC0)+8G^gTDqDJrbMc)g8d%tn7Z3RE%Y z>gRzczvA`n&aZarAhFQFRNCD!cv?p^U$dcL*|pA|*tUvg&Y0!2l~I`#XC}9}W1JOu zz-RvI1c4c+0KeMxk(;OE9(U)GnYPd)qmC+{-BA@9!g~hbYsnG2D;xz9kK?&x^TP*%_ck&BobtUAiaBq!O;p_dejLH~B+$0n!G`Np`)`_c=pHW$)dIGUq<-OV2-$ULE_?WoT5k8xke~ znZ(ttkOwRa4iM-iF^p1rDvvNuwUTrBi71vs`Q4bX=t4)iLwmd`er-kn`U?+L-3JUq zp#qxyoPw;E*3`5)&Pl?=4!!{Y3k_^^1W!xW8n4&$P{G4}cpAe3J56rlWvfk5i8;kd zKz=l@4V@cWCI^eX4&U<)FN0?(?*^Md(rlijXyn(_mCjLF>fsS_Ps0Kq3>%WFcVB<; z0yevsd!3MOqDWkH$PSMUo7gpFmC4A&uC9<_*{HS+YYJ?(v)M&?P0LwtRE4UoN>$a$ z7c7!0g?3{1imA2}jb_HPwuH74L)JmaGa>pwe{^$RBIC$ruudhc)vl`jiYf9sY|(s1 z^pEmoDh{S40BtrlwyeFYe60Z$&=5E+rj1#+3@g)2DWQ14I>h!_$K4VEG zag3qN4-1b_Wv$#`sXZC*XyBsLaL8YGu=B70r}cJK^Fu=aH2y+n{}AuZ>}T6bym)@3 z32U3_bv}8k$2;$}&uP&;8PfDL<2u3Z2XD&KOi+qAZ(7^lIf}pkhD(IjA18V*eCciz zty@Crsfp14OYkR;_Qqw*Ug`rDCG&3LTXmIyB@OU+s!*&GC~ty!D{f-Tb;)Q;Z_Tq7 zr_9K(2n`2kq(08F6nUp1FCy_0Q-R3xlsRu1<~vUsl&*_N^x}#$)MRzJ`w1_=QB}52 zNs+N63uE++6d_iEcy9#GPEyB$bPUzSfjagtop2=)h|*{&f29J4p!GVS1x*d7!DwnA zCrwg9u@&*Cw76im+|IkGsw#n4fHSOaRj%jPB{yZ?G@Jg0iYDFv11Qg%J*{KnzCe7Q zNpqkoF`;l-^Go*-BZGT$qam%r-+G_c=JSch@3oU3&u~`(ZiZYJj}>f%q$%QdWf6fr z5y$rxxpL^O?ox6u4+UDFxGO%&{_bm7hX;%_o#3fPztY_H`%FK*fAiK>3ygwp$LPfz z+k#c0QYV<97FLPSUEtOg6X$Mh#>q>B{N%vu#A6b0-;?RLaDM{?a2k?!s|fS?SrhJiY1?}qJAWtJ<)Blm3M4L&I- zhjG(=9R+oo#RH_i) ztKh9pW^fK(PZ`?-#jb^OJJi8`b?K?OLfF{8d*75yAK7?l5goQ-T&>*^`gqJl&4oES z#QT?79imk9>W}x5e*0N!#fg+{`Lb4&2n-9`Tu(6a*M^wXrsYqZC_v^l+e;#jJ*+v- zZW*!`SHE0*W)`UAy!MpgGAl=;;N$-J%7$xYA9m*vC0wk}BOA5P1RLQY-~Xo;;BHJ} z>Qr-J*_TkcFN0QcNc500F+(X}sjjZ1i6ubK2H~mplW`ydW>alv6Gzk>tSYT#9HAml zgrfFzMZ`iMRoufVb0Dkf`S9jk3Zpj=A3Y;><8n zOm1{dS{xd`sebjAARwZ!P)+)4P4REtUgHHluZExMt$oXT>-~qt3q!rl?Ia?}&F4Kw(@5v5 zAj$5^3@Zy&ZPz*|7FO)B<#zSL0NK{ZUfx4WPI+syZFE=xM!V%#8X8?KsB)ex=q87_ z+M^eVBcj%zV9l-jq`so%kC*k*3iLk5^ro%z%xS4{)drP3YirWtiGYMyICzch#Yes$ z3F!v%kIO2}e#vn`8;aJ+weBI}?hd*kLvVh)C?<2sB7Hup{EJ3o61yiFaDHgMKF<>7 zanc=7@clLP?1`Dd#~)QNk)&^1)M$Fy2`kjilYOz$?Q0zkz4XK3B_bIJRGl&PJSAn7 zLi&k>XFEx5nM>)P)V^2UB%|oBuRD2- z*E1sWX4-yYwgxAB|5a(cy0zA}pf`sAywy69&R@3}eW*WkeL~IcXioQ8Fs>||)Kzss8jr&+bXStdw=WV6B}TWjtMyJ;|kf`0TN+ zWeJ^y%PqQxSCX0esJ^4WjvlrR-8(@RTyNEc_< zvaBn$<87NWGzeoI{>149kBA4i3NfcCv{cPmHT8fj<0Jf3mj*VrLkw0(`#sGy8P`yc zEpQb3(T_TtUs34U`d@Dq#Q6sZ8+L^n7dgo2^e#K`j&5~Y68{KYq;$f=Jj}l=K=>== zakGog#wDpPa%pb#Ato_yWGLgJz(2N0EI=*1_!2DF6N{pIn_zgRnAl;7P zAG&YsNgGo-PwT%cM=2@Aib!&4!s)~=pC}6uNbJ}aY#G3M5lmG}Vlxt*`g?$3%g;Rq zgW+q)F{N!Y&PuyheH_|43g+0u0?>W4*=9oqMX(hZV!}zii3(p;v)6?E=LiNwL*>HJ zv_rA$>q&bLn~}QttNUQo9FGB1C@u^v_IHl@j~6EEe?CDaUHb2arT_TGCP?&uQ2xR; z{{Kac4BHD}@4x*_{%eQ+A7(SOK>7A>ReABEx>=;$(WA+p96=zu9>V zgxU|D=T3E9og0CPV^MA^z`u6}3Lq4DTz8*$VdV51U`d1v?gtAp!bgT7S!ZUlCW=*q`xQ#U! znGn$nkyYA?UCoL;k=geM5MQE5p@Z;h?|c>J!y}X_zT0R==3fHgIHLK-dahf0GRN1gLC} zq4%9sCB{&vN6x`Alr=^CJm;T0bW>B#mJ7u!U22A1FSXj!_vNZkOI zvbaG3gBC4A`sj>RC3MZGfo5oC+p%2JdRDm%-c(aL-(ofXK2}U#`lE^8e12hLZs93@ z7jc>(-bs8VfI4HT}rx z)pwgG;f%^ptd}L+E{lP8LI^w~TdQ8TOY2vtWH?9Z4K&5|Q}Y6&k_1B9`_6(Mg@?M{ zr*pih&(7Iz7%3{)6L_}Ja`9BF$g|Fn&a`8U48i@%)Z^p*)8EIdu7RTw^ObucqWOH4 z2o@tB(`p0}ClSrCs%hEE52kS)?mq$!BP`c1m@-QIQg=zOD~M4G(f&+g#VWtuuG|?xiEbZjdQpD=Uc0bdZ{|;9q~0gkQBHH6WtPs1`!xk_M$o3gKnb3kQs1S_ zS;*?`D%lgRZaX9YEFGPVC0)((4p5`iV1oWp8qW;UmO0CI0)LT~A+Y;1bMfjPJ!ZHf zUHWMw&HJ++Es-QLl$%=mZ~3NcY9)YM}4fYXZ1hL_i8t2Iu^DWpK;#b=#2oBi$i)1s|q_ z!Ej0H?T|<=btLGF`YQoZPxw$}LnCSmifKW$%QwvSJlapx48NmZA^}eer|7$XMZudC zdwIeo>@d?ITxo*2YGt4J!Mvr^Xio+l{|!=`NYb?$VHNgeN|KWL`Dc>mT!|#M3koz= z8)rS*A>*TR@%FmCalaFTqag2zb}VoNZr%w^&?wo|qc-^b)pF*p2!iE!iS62hCsAMy zarn8|5t5TM+I0r%rh12nmEfyo$x+vjh!vc5Gky*T5E6Ah`;5S04(JCi#z=|k5uW+i zE;Od%-Bp5RyIr?M42z}|9}jp~W75!{+SuD|Z_&PvrHHOlYF4h@ewZk2eb@F{3lz~8 zwHEgBLuE9bGX%+=M6^J#Yg8|9Vx1`C0zKL*Bl55m6rx#)@W5zR+BkCOqm3*_LC=Qa z#0aokRyJmae2&mYAf6S#llZB#chBtansQXx0mUnhD*By8#p}qA4q5CBdzX(H^Ru_6 zg>Yh5q$Skx;WH{%vw$E^z%+Q@y)^fKo5Gjm2Mu0?{qUMEAJP>4SKg6y*jyC$Zwhih znsvCd)Rg&s8!Sg4jz;s}^}Vg-Tf&VA>!P?FIstdKl=UkX4He-2{Y5_&fzOwHq#&t1gVJYF3z|A&sR#4guW3v3NO?fth3C4y7<5bcf z11&FaBw8Mh<(K_tkG+81A7ERGgC!6R+lp!BEDY=YYq1I?lbp3C6tO0HHiVagEqr{? zdx7CKHe>X=`55QjM0e+(92F^i`;oeDF1ZMKOsl4BGhu&>YC55n;QX6u*nU*kHH7zMl)BBkf%6}uAHnN&B z{BCfF%V1S?eW$&XL|_ZAs;JUu)W6*GIsM)*l=Elsk!BH-N1UQP<=jd`PTpXKOz`l`d~cO&l>q;?Erv@)t-T%NZ+~A*h$1bptFh zcy`3ASKG*x9D0qh?NX$|-y9?+n=0M1E(b;;&NfG^5v`5%^_%|4&c zdk|uAOk(QleEou8>xYuT(6+2~tDo6m>WcY5Gee5zOe39Qr*;5f==!5phD`brhkeKP zDx=)LvfU#Mj!auLWZ`lxXJ^Z7qPBQHog>|KH*+k|m5C+^*xOoD;aF-Wsri*>-4o0a zGJR>0YA8U3fE~X8lVsFR0JGX@_f5(H6)Y+0djjD%lOK3YFrr@=GJB^Fy4o5dHBDqX zpZj^!najJ}awCT^DeIhYmjAztb`Q3o;v$0+1>diO5Vp^XyUgAE9;d0VW(?n+&wn6x z=W6cJbaAS|7>O1)9|@(cbHI9)lh%2)ub?(UU&$*(V5KMiskO$7V?`!nk%3xDT4qsW z{AIss-Ok343g&!~^7(PV#Z~332^Sa7D|NY^`4y|@Dy44Hx}g43LiRLaX-0bH9X3re za5WPuSmAO1M)uqNwG*9likUUioQAN1FmMi+$jKecistdWU;Z^aEKhy7`C7^%YR2h4 z6`tpihLI-I!JG1iIoY=GcODv*D5&;FqCl6VQQLu**tVW4QxT9Co9ZVIj|w(Z#QyTY z`TwNF{Y`wP!rEgQ9NdwMiA-7oql zOeqg+imd}c3l&a#kI=i9V~Drf?6>yYmp$m<27j2YarSjYQusYT+f)G@%mXj5Uz=5} zZyTUcLQkS?#aPFu(41VQuO8?rb9v<_M;#Y%0MHthpV0rRA^I5noe75zu7-&9K~^H_ zz$KfY9m8h4@!`o3k>EohoP*WTbz^xs{SW9Wbk%+x_WI=2b_Qa$VRK}{yMZ4tYja%Q zK@hqI%v4TxwrlK74u4$rF@6bd%>Yx=?^zLVk!~M|&B*ug&9<^6E&4m?@bBgtTk#@O zQ&t_^&DOr!pA2eRgdQ>vNwq{d;@D1L(ZlLT6SYh-R8~vkq@1LCciyQ|^Y1vXIz~;* ziZKunMp8ZE*EY7>2BwqJbkPh|9I}|!{7dlL;`rF?>6SW*J{ezVkRE6iM%>2lg8$rp zTUN$xSBMZWq#9uzeaa_DI{#p?@QmWALV7uUwsn^bWN`8N7R6}h&J2AvCDeWwuAli( zh2g1pQG}Hk%^4_oT(zMIV;u|-PBo-SglYCJC#e2%Ibct>r8IU7e%JF)c%U0p5M@=5d&H*fi8DH)`C{Hh>Ei9>{#IZZShGe`yOY4PR=7 zjGjNCw7)mnQuOr&ZE}0M{#25 z(W`lX-@@&PQ%-qnOmw>!T!P)bdfTL{tM(r(-g)&z$VmBmT2xn3_g{o1(A(;u+$3Ga zWGKw2Rk{$Ke2a<^n3DN?xE1TMHaA}zpL6`s8whbsrgRtB^vRI`AunJVN>W&t!a*vZ z!%nJM_;**v_S>wPY8}1DGR@u>GdkR3Of{$jc{wk^W_B`-_82Qf6YrbRr-{WzrP&R% zH9~8Of&+%ywUKaf9-PW2|Gv!<^StVn%0JF1k4b6(Ga;p;Ru23EEX1<;BI?fJI2uUwj^7SS3{U;xSmPfhkta~fY?errT9lm?EN zOl-~d14lh6R&)`C@4fm4!*M3QJ4?VmFuoWiMW+;Z?yy_kOP2S(gS&z?;i^px3E9_K zi@rdw{-|OIo;qtfUx(`mZVt%B@BH&A&rvS<5WU9m^oAmg{8Eq0tEDP3KAVwrVv&kIR}r6eZZS&pU! z{+UB?C{9W&G-bezv+Wx;Phr+8F|ZIws#0BZQ9T>{xFkQ(UL0d8hRlg z0$g)&av};OV`xj^Z7C|$x6{W;J7vjAH6>-UqbOKno3Ef-#QDZ@(R4<2K#avRjd z{>WD1!7WgHUQ#F{&!>8oOEcN1>l7A#JE+kI`%~z##UYBQLcj6lF3eEt$T@E;Hlqb= zJ{7y=0g3`wEmTw55c0ayK0IT0zh@?&ZPO7`cirxb`^3>P-hJiZrJZeOvsN>Tf7HLW z{TasL4(#-eWW_9Kwqs8dCo=Nvs6M6Azv4U(Vs57dEx|(6Msq|gU3yq-5cJU;rt579 zsClXX(L5fan3$e>aL~){*~uoxxjv3q5LyU-4xU;j`DVGI9?~WTmQ>-i7>NAjhOf2H z%wG|<^}-B?bA z(w1Q4+_#F%@)Mr?PvS~m^=YCb#WB{y?{ zjW>c~k}B|J`g==X94m7Db~ zGrZf!LDS^~oyte83gE@ZYmL)raZGQ*u)Ud`R?Rt~qslQ$$uT|JtepXU&507pfKso5 zS8_Ntk7dK<%`>W+`?(x7p2EZ{El;+=Tj%Z5%Hf|1WOH*Uq8gmOZvE+Q5&Nq6U5P-j zyB|*Ip*Y&}#R*k41@VTOH$O_47uCEEG28?(E#W&L`l<$>tsd=Bmo1araK>J*==KHbWJFhw zFQSzhaAKi%4`x%lugOCC#wh=E75?tg@V=#ex~~e; zvt6Z~kNyO~ty5n%%LZJpFC5GRO@6i*Q#I+%Uw#p0EKBpo&j+}UOHIe3%*U}iJ*|S< z?1AW-kYi7`1aBlT;6~({rX;Lr9IjPvc!9ydBqk6SWBY-6RLCgzz1D}P7&94yy^T&L zv_PUsi9-d{DSSrp_vKFr%#U`c_8HpJr=c;63Am0Yi^ceoay6LGWHz?v;b&NLXsW@) zX`-8C3ztVrVBtSld=1-AH_# z85jld_O7hpXSoYNZ+M*t`H%d@9Jf3-}>ciaaI|K7% zc3T`f7DISv00Fn9jW1{{yqz}0{pC(OpKOSHPv|v}d+8V<|aGW)#ut}IC-WZ3_^K;+$Y3E&ufAQbTLKaf)}n zy%Q@)~V0AEnmjyKys?=khkw=P?;6$Kf$rryRD zmzzdMy?0u~*FH341ubwsA;bvnstp)v&x6>LPH?GSpP>5sKKGmM@0$Z^-6kgu_gB4u z`^&u_-&lByY-a7oxmS+sh`Ack)%-%jEoCdLXIb?TC{3HU;RO+JrPtqBd$aCyWa-f2 zD`gm+7MicED9DyHxNB$xrF@M-a$kC|tg!BEfAOqWX$p}HrfNPyWsy<0Q0K?GX>U#c zqXf~k-M;NB9ZLx;pvmwqx)Ecbjkio6Zz0Pwz`3kG-MWwkG%=bmN>d;Dd9S;ym|6MR zifwhy#3Y?xE>z_RwGf?XqL*2tm*@0Zt|*0V`A1k$4RrEhC=LH7dbX9{aoz)a+SdUXEDobehexA(=jY=p`ECCV}XTR$;Q9r8Do; zN;qzqcg&Y-I3Eg-Bo$g%I4m><4*?B!E1FxwChA}T2`RsBj$=L6SVVzHknr4=nRRE; zs_)MHn@tMxY0L>LTsF4_JL2*#@2KlicOILJ?fQXBPe)k8%wFif3A08;|D=}cQd5;a zVs0Xqs>bd9nd~eg9z>4Tvk~{}va_M2^Kl<|Dm`G+fLagze`*1Aty4j&EN4?(+@K5O zcnBHG!A$a!0CS{TOte}|(h@=)gxV{)B-t_Q*?0YThVAkI&|Hspb05hw#ZdGKEJXz@ z{oz>$_uzq=kBUucCjIq0*SFH77*SuUNZ7Vow`b1@0hCx?ebsjt+$i>g$)?mB<-|AX zhwW*mNbC2wj_|&mwVYf`vO&Dvzv;31gs$EqXfXYr;?p|!5zx4ft1}Q1{CwclD`_!D zrzk6~dk0Apxm)si)8jznmex_~xF6Jz+EBe z(`|}-64H(WUd-)Y+isLNVf+2C^kGW6|yw~o6h+gQAxT+Xa9YEM$T~3(BZK^M z>)4f?Yt!Fz{75W>Tje8@(R|C^>s6o%e`>?dr`tqskZaTM_9LI$r~8h}->0sngL9ar zPiS{_99lQ#<`Xo%tHqeEI6AC_HmfeJh_ggb7zjNo;-ZJXx@@A9GR6j&yO-nSy6522 zPdGEJ2JbD?q7EiVwbmV8R@p1WRq=y-fhV*&8o7$;(6>i)T9Cs$gJBA2pCzw=TdKEP z!Tld-kpMMmOhfuFXlPMUMV`mVk;ftHFTSA$+_^pISVQ}7B8cgWn?j;5uQh6IdjH&E zKLitBFMDdLd}@txU$iUsmzb72$3vu}Cz*qbmP@nef_K=wD&*`jt-}jJlOA~<-MKN< zMIP998wG5h8r>gs($c-b&CM4HVbKe1$vdKdZ({zQ1fP|itgR8Z=idY((`jG3quM8g z&)!f&^hNF2we3o$g|yV6YrmBeqF;E1_^zOTi3A|Ae@d!absx(8)Fb4y{;;Y~F;SZNAN6+A;lI@jt8+`}^`W z%u}>XF_2~Mijs()1%5~9nD1?>W+sQjfgN6XN7>N#G#hejeB6yx?uRC3nlk}RS}Cg% zb6SR5@_tZCKoX~?t3-u!%?OrHy=}o`B5%8J|EKMYFdc1>YE65_I&3^<#x}9yG9{{tBTaryP z!I=6+QW|_DFVB+a2ygz%03P*BpI1I?JRO@mEq#DJn|7YYImM2z?a7h@fv?{xvB=2) zmS$fPT_(3?^n4?PgJ2|rvyfdbXY3l@8T>zJAzzd*;}uYb3qP@DsP(0$5G^XLEn0=S zOB=s$lCD9lx9D`l$I^AQk~@q{rDo+x%bh|Q7R+VsF&D*?^1_ab9Z07vD@!zq7Z@K= zaoZKRrWwlGRRR8rR~?P?CB|gd#aiapBKY=gJXD3@3vv}{El&}hjZnq~4cnver%*z$ zlsP6jeJwN1M`;oqx9Vuy0WugyZx#6uV~Z0frdJlnyJT4QOt*9O115f9GM0CG*uuImkKiK z+twbKhIe#cmtIy4*GLn1Xxz_yfs)RvpN;Sf?Y?44+*yx`{5&J72-8T3VB-Q)^<_&a z!1_C=XW2ENJ*birOAwhh@O6e~6w=(goTdXY(zduGD;|-Lw!S2hh7~l2H~So$n-CV> zu_m{ig0JHq($If8&A1*4fFf}9sd*^VVlRv-D6iD2Nq5$&_KJ88bK~^>PU89}vtBpz z!ivQ>IoWzrNr4BOUN^Jc!gbYpV(*i-RAN~Wl$Je=pWYB>q_y%P%J<7{or-c-RV2Sl zNQw8QwV)c=mF>$#Ql@9y`}X0)ymM`$94mTjug21hb0`OA9mrz8$0oU#TgCHH?^E}e z2I5qNU#KTtuKq?{Epv7BKK$w&U%FZ6NJB()b6nzH#+~`Bx+?oP%UDZowy#~} z+GrnSb}p~WF;k(i1KAZ=1#I{`%~-R1mb#FE6;>{|yQSdjWDc8UA& z-gE%SXLg!NP{-YTa3U-iG`V*KmY(L8rZX;)+Zh%ZwXLbx^IWrkrD91&5Kmp3aQ)y! z#~@GXExp5FqhOnEJLMwbS3i@DNh7qIzK^mhTIi~dXUkqeJuP8+4f5&|jPtp`2FKRg zGt9WIjlLi*D_FQ0WhSwC%v+$-QHY=lZLC6a&PIxH8GGn-bs+er{ie&{{=;*A*>!$A zXJ0JLfkG}y{9VlmEv?2bZe`C6|2wBrg8s3+gp7oEymCrS#Ih%Zi&<3z@8Vcb7^Yq| z$zB~-;Is$QH}{om6^!q79Y3!oXk7m^igxtp1~Fcj7PL=AtIBRJm;SIw- zj}k4jrvqJB*#@bJBaqh|gEHYDiA&yePErP=Sq;RpqGWvi+c!X)owGgp;s!wI)A(Vl z+H8dQ*=gD=MMaH~4U+>4*}DCpa<>c_jhzGs5m8hdOEvh`YVw_Bkx+Ah=8@4#N!I!q zPvSsh78~fuqu9r{6}BiQ!*wbM`^43k5I@VKaszpN6nNL_20;kR>`k%q*xQnhn&Y|H z&;cOpkG6R0$W+$F{(7w3&D(~jGA63;et{Ds2WdP!?!ukngSQIfVzR$(+IJOV=SwPc zO#FHK*!rfy3sFUy&w+I22nNmEdX*4QQ*Tt_H2F=nGon7)efNN;(BUN}5YO5QTm)or zGE$Abw4UZiui+EJNa~D_Ips$m(i@aJTN6sn~Q75hH!{slVXp#hc`W+2mH;3 zq(2|R#fqO$I`y9}@LTm*%E`5vpD+5JwS#Z&I`kVLZ$I{8#2n0?w>>SN--VW3EIzhH zVew~uZ>aeU>lV=4KmXB%@f%u?+Wl>E_9@YHl?$S+>tR|c@@wM2+S`}vxZ>e^>&3_T zt{0Q7)H`Hgn!5wiQc`V5PQF++%fBJ8qMC!C4!_>K;uDskJnt{NtP_*;@?QWxB1u(2 z_A;0hHa(8<8Ch%-?+J2zL)tQgxIlPkaUECCjyCq^=w3bFrybv?j(|}(1p83dbg=tO zvwu4F>ulzE4`nKWloRevIb)M$D?ulSt@-bJbe-??68wo2!e(Sm2-=SYPZjN_`74HI zr_}jJtdvhSkm8+8TXi+4yWm2raIx0LM9-U}A{yXK-#K%2)bPN2f zZCSKdPoN@t%0lskY=(&0h@mw%rP#1OaE;&7Sid!@eO&ProMf;mC4(=gc#6)~3>%Gc1b$|Pnvyf*JUM5N#GET@jhRG4m{AUHUzCy>;0Nxn~L zT;)v9|D2bkck2BY&C8CVkpCN3?0S+re1dn_-)sN)vk8-n;>8pRDUta5?>{d8$E#cP znE#zSwfYy-XZgQSrAyW9=wuWRp;VoU1VR`cIUSjneJn1a`{ zgXG*_xcTNEU&hCPBrB_NRr9~J2Yfds(#Nky$8-EB@NNvClt}-dZ2||`jQ<$LrU(d#o-nC2%C{_pOD=xw(}C=$ryoA zQcYom&FDF$rOa-V+xC;+o&oMc{}>*=99QsJS5a$9_Ay(8z5nS&zoC+v+9?2%rrCFx z5l5c*F{veUTYwlBae=_sZ{{ZTHmVNscXcx9b-PCn)12pTZ%snD`(zXR+13v1EH>%4 zH{~lgbW7q&!Uz;Tdck0oEWO(&z2+%AUpKh)Ycc{}IU)^SXlBhUnR-CF*Tq@9hxiTa zvay$}v*vrK&+>e1kL&)%&NF*IJ4F)Nno^{9O7a#y%>DjC&p-BqjWi*%kKBoJllrHn zIy~xisC@XAc&G;1;QUnl$q3n7kC!e?U&p%F!fbcS38Xx%5wvaUruoxJSXhA9klR)K z-oJ+an=6xe@PU+$Vo2GjR9ssksZ|ZD?#PEve;+4rUok3H<3!w>-Jc0nA#DYmYh%|^ zjIhSoNX_%Sva;TK&-Jd}HNwY8g}Nz{yKRb_&&%peQpRc@>!a0U1svG4q4Up>X3{?bLDV_x6_c8;mJTRU5vhs$q?`_u)#I-OhGB>)8RTM!%@-0hSH^Ob})v7zlH# zu{I)i`g2)#bT=Tb8$P>*?-{GEuB`NPM&egY&{ik2v}L-pdxawy#u>sHyojt@a+)JJ z4Eh;#FW7TMxwly8HtlM6dG8KvYfZCkVfj#GH07wg zeQGPgP>#rBV#>QRppeVrQK@3e(tMk7t`B~2M=zSJSy{F~PW%2W%LaE)>4`z^VB3~owoNG6;LupA zeQt&(#_G^#<(%pAz`8l+?Lz75u5VU9J)KU>>Wtp~L9ptf^gAfe)V4X;Xo+4~9L5qM zy;;>g+RCX?WsKlNeJ_Y@YRzp&=A?J7N3SO@fu_Q0x~sBy2g}?cGo*HM!K^3JPxS{m zfO1&GRh+q4V1w)(tI9(p&)d)W^44&@Y5(@+au}8Be8~ z2xMuL!c=gZ0;2o}p?!}Brg0F*%k7{wUGsG>ce5?WpbI$6HEY698E3FXZ@qq_X#qWd z@R1x|>J)k$W5B$ai6+x%_nb$^(^2yh;HsavtND#I3wB+J*O=dIo z+)Nvt2b%d_@j`o12;bZbmMhQ7Qzio$%}f1LfL=6b#f00$ui;0cP!gdmDng2bH;nVF z3Gt@H?pL$at6IDX(-3PtW-BnlcVCttbo;?QDZXAY671mWvpP$oYJJ0NZ~ZS{wtMdw z(}<(D%JgITn8N4et z#d%e@<1|(4fo4)o8ogU7aZ*`r-?3T>7~V)aIexICfBSSu!${km{SxyW;%P}`5r6R) z1-=zW=sTiG-oglaw}?@qqFGH({M7?pGX>>tPRQ&h%*xlkKOdb0wq|g*T%>W@8q9^9 zqj|O|Jt-dGTAbWZL&VaN^rNqe*dtM8sOEmEZlQV z&aDTs#iat!I|aL@rsfFjn#%X3rhFaDM|Nk1`n21n4Iq_x(TYzd0)|&kpdJDo8LeT& z$u*9cgIKw6OS8uR&){8QNRpvO**X0-4^?7mC(B4yS1P+~0{WwS^;4oL0Sim@K@N5G zc25u7iI=^O@}7L)C>oGrpj*-JWqj_KnfA8cK{Sa{Gx)}5HEcz=D4<3yy2*sT`a8UFA{1@DSeQnjXV>I1tLhd2=7d}7yS{jm)m9x>L{LvsnwnJf_8Y1((o#j z+o_WaYq-qVa~c7V7{O@kbZj)Hg#U}?#7u?oyx*ukVP>=NDa2;SR(47y^sZcp3!a1( z2|1L%YY|??B{`hPIU5XM6cwQ8Y>)lExpVnL)_9i{T-_%NjGQI&Vq8 z&^!}l^K3|O9jL2Na>F66-Up6rDVybiE@(rd9_w@^h<+SMU6D(TUwKf}%YCMden(Xx zJhbB5`sH`Om^D*b-lhgqK8u<00g1ds@}N%5l0H<>L~3%N+zJMX{?6uvP=6t^tKt33 zJ;KQRdaV^;_Gr{=)m z?UPYre&NRiyGl;bqwt>A06n;{d3=}*aSDQFy#miw4JUjioOZ(mRgAr6aCmGAFGs1g36MtIPx3I?NKlfGl+*5DGGIw-Bc7!Hv!KZh`Tt;Bx-M%e} zl)E+paunyLz|xnI7^KA+UI-8Ue!pC2FPn7zLZMZ$se%$)e@|&1W#M}Vl))+6jm=f) zf=6CTGe9ifJDc4!EJgB4W0I`%_714GYSQCVoYEYoJanusJ|o&EZ+g#*uL}EPasA!1 zA3^rnfvdvy){njgyPu8NtK=TzmGU%83>yhcBWe8cnWUO!M7Zah!%G6}x zi%QFSP$PLuTdd7e@kh3Ra~`^Ni(jpc$rltOjqQ$G6JHAYunTS;DshcU50=5IDn8SQ z=Dg83HEp$6FgkpEEv`2_toLdh6Ys`%^T4&q7tNQ>I)9#|Q*&cuH*#P?x8z}5>xBqknU@-? z*P;p8LM^#lr*aVUF5ay!Sq-^o4AJ8wV;cYGUYlUmh>d;?Xtc_pPkCxGa+XP-kh4Bz zNWI`7LrE#`>VTnc3ddD}>L**n+0<9NqKUCd4Jmv{C@}x6+Eu<0p7Yn!S+YGM2r$cx zuM7e92&s=07e0b1rRfb05newF@a4XuT`FF7n;2nFdR!;RrPdv^(~wVI7FoO;&&F`7 zI-SvvO>iYYq+idZQM>7&B?8CRTT@^VM_)b&$x|I765s3Z)^N^*eE)7LFDP?rf3a$O z02!?as-kaJ9NR@xXY*w60PcRP_EnLBY}N^E>dSun)SiDAC*YI9oe*gEEq6>~uo~5= z!$O=|pm(vZLqtEJzx4{{-VQ`X$<&vEezrT^A)|G4767&p?-gCH_*Tdph%XX6RrUw1 zAkZR37^6AYHio98jP)&B(CX`cDM$?^Bs2Q_H}2U)ma&WK&$EM zmAJjY6&WCSt$4j`VnhS1Sy|nZ4E|-T=)17Oi8S}D2bbNlUWEaPWM1te>hllH*T87a z#<(gg;OOU6u6J-rG8PtVQIMcenqYrG+xh=sF6~ecB5AXBF2k?9G^%9AAZD3YzSP2f z`(ZJJ44FG8QAUYlkj)eTnRo>HJeklTuAZh5_@wSGZ%jwH>Ec5PLPN%>%1=k+H&ccr zMOAJ)027E_G=&2jMghS=saMlfyQ!Hx-{oqdM4Q}B(CCXp!OPqYM*Xpp8S!0aP)P4N zw8mOVpbn9xx?oVfT5DM824vK8!Z#8FAb04T$nEX9?$wA5A-4L$Q(E&Th#UQsLr#p! zM(kyiTRIT7bzlO}`$4m#nTa-^k_&B~(+5F29zrN%b1glNlj2f-@+O$wa86(zaU9SH z#YkOhvKv!!v6!-T8TB<6{^V0vztfGZA1`@Bk@y-{Yq|EO@huVOh~S>zM~J%DVSxNx zHQlhR$Ra&@da>!$5wDS)c&WmYy@!82lHLes?AhhDI?3L_Vp_lqPJe6J2=B0?WF0|h z-Eu$JbL1H&Cfmprx}&62`X4Mn{s_SKiawcr?qZoS@S@dua<8ml^nsFvsmc{meZ`#e zUg1#F{nYu%Vl2ARP0Cyt2?Rbk#Q;8<3ynb_n}L7G4atblzCT_I`}urh48`MM!&G<0 zLU?ArIB)_{OEz)PXXKao1ZEo{*3|e6*4I#kHnDw}r<4tzC@nZi&}WAoD16v?>r0L^ z1pB5a_m;9RQkZ=9aSu$Wz)Dda%k=EpyyW#JIwi!y9hvRY&3^eLuf4CYuQOQr)$2#S z@#rpgd5Oa!>h{@Am6q^;0*P20{K%X_jaZ|NE4EaGMYD!8usuSt_p*WR614Fi@PX6CG)8YKzH$SiG$zt5zabOq#Am`^K9- zorKR$bMp#$INIss53$za{(|WRjWk?2ClBb{v5JsQ_Sn_`N;AvQSO}PP;_z2%pQ*V= zq$JA}SO^}!{W!9E=vCQ_Pi8|+y*v`g$}r4rYY?>%c}R=^uA)FD$%F4`wAsQmt<4FK z{(a1IsM#OewRvH7#Ld)4hMuwD0_>@)mt!~!-u%Sr^f~CALaOF&@l`fvR5y44D%FKj zQvTR|L;LKGt5)Wy=MprpaHK3A`#%1cxy7AyYZccGg1s^pZ_Q5u-Iw_Y^U|E&lynNX z?80TcJkjm)k-mrWN$?_d{a$h`f!y7KD^?kHwk^N~KUf**R|f6o4EB>RC+@jk_n14C z1?s)MH;N)?;(y|^WF{RG|5x~|VAbsZQ~2x$kU&kMz(7dzh=3Dir8Q0j9fqU21{cdI zbS?%?4^p_Sf8WoNH6G%{Kz6nG%Seg%pFRLP{$^SlnLllM96c&0%6+rPVKFirqd?=N zQPoEAe%9|G?B6*IV#Tw}8Dn$E<4dTGk+b%#|LNF*B$q~)QlCH=2eJOFc1S*9&RfEL zxfms_Lo`0=hhk9q&$9L~6k**Vz6u(GJ+jqj>&DT)orYiQnO)Z&FL&LZwAg+ylw?;n zO0yEX=5`N?LWbpR&>lXdr!F|>^j82Pj~MvHOTdO}c0K3A*3`fM6cwL%MU*$v{KP6y zsyP7-9qQVHLjo{ldtd*8bCzk65@KX_$EAHcfo|cYdD~!ipC4#@3)vCEOAczj-Tp!r z<{N07K1HlleSTQkKX0cr1hSa-R*I3df^C2@mPj%$yJ5r8;TH`Tp#Wzl9m1aB*;n8}Y2=RE&Ah5n;a>!oge~U4jh@8A3b0g?Zl1o~w#4C1^;;S; zYV^T<l!6D>7J(vW}pVgD}7uu7tEz0D~CY0O0!)~eexgJ zWgXC>f8xhpiEH!Dcn~&a^`8FJEs06*2?!w;5&!~9$jSC#B%0;ccF@`eWhc) z+gWAjQMAtL^g?H*_1+!Rg-o%XH!2{(L_U1W%`7!^*xT=Iu}F=cgnQ%50CY}Njo5-1 z5+z|9{{3$V8)4|b-$zauZLcrtpLLV>vqBf@w##`#K^97L<)up4$(ig}5>olk14M|X zjbvQ25lbAg(&aAuW^mY1mJYQA4Oy0#@~s+2spF6r4xPTU2h3(JcRIhlkYPo1i3RVdMS7Kutg3lDZ z9^dTBD3tN#Y5=}`0R84G+G_zuwnBDAqnZJxS|z2ur6*Sjd8`R(_ve_C4DG(eIptG( zWjH7+8}~Bk+DY?YFuAt+I34O=FU{YxHbu$~7Ib%rSxsfXniNNFZEVO;Q-JM^^Tg)3TI71xh5 zSunxFE$v<7P^AeB7knO;T2Fymua}EmC}CJ9C0?FqI`((AKnq)D?}MAi4CWsXwbM@9 z`yN-GJTAdeE@}8^LtI3`3_ZC@suUJ1Bo94j&Ny>;^@f;{z+hC@xzM^TK$n)ub zSix!ps7G5Wo}mp-M-QG)dOu@+or@}YpgT!P94X0)H0%Rmi!4o4tLQs*D(;qmu7rJE zdT^$RU7Ty-5Gh)8KSGa0&Z)OX_Ldd1>kVlVCjprFitUqMhmif&{ZL0w*wWzd#GCcG zA4ZjHVm<+ySrcMn*e5ggZuL`Y7>ij5Pn{@b`iNJ;q)&q8mEGTB$+cvar-1d}bO|S? zo+8tt5`jYv_$sPvUuH%>2?ESMoZt9um3{YNp-rxyt@flo0(k&-3OK)XCAY`!?M7#x zB#4=qju`8XNba^b#>OlC`YgCHgVti6b^WpbIPzgQ{*uJaeZ1YGAh#!ScP87sUo(kN z;y%@K{O+;{j#J;P^Jziq0d>6c^Ydzss1R&6MCLmfVtBz>-VXvKV>Fjl)MhFdS--~Rt+b)y%m0PfEh#>&nZo7)(X*Ki z^N!EUIIE;<2&fH>e&qv}xyNo>xi$y?H;j7RD&599$l>iW zFtk79``wMZtiyzRDp4crewD;o^BlZ+)qGw@i2!L7s0$zgIez_9Y(Ei7;Di4G?dLx# zcIsiQPU?xcWyQ$|{t9loj$0NJ`!hs+9Rn|TNQQaif)TvU948d&$ z&30K!T@CHOAN|Bw%SIw)rm~}0Z-+$L-UCGZ;VTh1G4KU`MXP~K%LvA!6`M&NCbjCq zzQs+b`?&2Y`fz{fVkHgJ+hegJz27)C*Kaydq^U6wWy*aidgA%=5dD92`|vA&E}n3rOO>&GdwqcNErq((%dkf3K2hGnD*0)BB*%dn;4{ zN9rdcZ+!I(-~DN18}}rd<9YO9{OgI`e*CXg@5}wakbidD^(%ame8!Gp68nsp>DHkW z1rg{xyD#lu7b{BGYj?x)kC_0khV(6(sH;4yGt#G?<;G>LGq!skTLC?UKao!@wtp$n z`{hu;_j_s6tjxw-w5y1AafibW+sciK%;Fhw%&G7UtYCOjjx+rN4ctwnPg`sL&WWI$ z#4p>sjfB>0cZG`TMV2B>cot`-j_HV|dJ3$MX9@25Nx!*NT^|zMFD5Yf)kB3gr+)lH zqG);j@Ry=0?h4P|xlW*8xgKb%Z)XPc^YU5@Ez%WcGu%X2vaX4J9ba|yN6#ReS)&V* zh!hq2aU1S~#_`^mD1ffS4^DX9KN3q4+fZKCQN%OLy^2OrXgF=Mn|O7oIev zS}U0^aEdvr+W2rN_Ad81m~bUa+QY5|%?m`egc9x`stKRckU2@bE;ZwKwXITHy@f~( zZAl82&DYZRWKnvP`)}{HOexve`1sld#^#=n`&O7(C316hb61GZIQ|3REoP`FW72o4 zI7s5VUuD{e^#zS|)K6*Y%dQ>`wi~BSUKAoS`#7XlbfB$bG-UECv>2j_`#Ptc>}rU+ zv#dW<=qaO5WsCkcA~M_8iVjcxvYSES@X$W#)@xmX6Tg8JQFR_cV2&D}^ZxT8LDLCf@+_fAkqOgpKw-`Rf5{_|+SQm#YSgr7>5=un>dQOl71Z+YV|&Hj|W zDP1dh1iaQ0mw$(n@!S6mN^UlVqQVDa=0D5w>LCw<2oTmR;8>>d3chz6nI&|=K5`f+ zU(6yY!X{^`mfEyNbH=A7zqN@M%lO2B7q^96t2muq=ajvHFpHZ>>2O{Hk$U4`xH}0r-8B9QEBQi& zgXaO%N!&I)`n%!_7hxzmV{HfWV*31H)P}i#GQTq6nlV;!Q+oP&RIgFg3H@mzQWS}c zv9wG-7Gez8Vwto+$*J&VbidTfvxP`jde>h{R)b**+shh2BHK8WSBom^iO9tkK$ojC zO#+C-15+#?_GxO%teDlUCul^&mZ*fEfz2PBy9l{!0 zZ;v~?OPP&sGWP;RGBrhwRkU9vE7QL%I2uE(FwWNOTIF9XPLRchrt;(ac@%1_F<(QV zo3iSvpWv1f@HUHEZQ5N~Ij)fin&LULn|JbTcKWnc%y7`3rCP7sA_$fAPnQUMw?exP|aDUiK5cXrkoyfL5&i5tS&)PMAOjKc1K% zcs;GGB?m3Vg^-#WL`WO`2^dcHa6wpNAhtNJBIQ1Rc+$1h-jr%v2*Ly!D~k&pclah@ z2y(q8=Lr_W*!wB4cxJc6WrdFY6~M`zg@#i}wI=M?7? zHlmd_+f8q7mQlEFa6I60O849JS7pz4+&!I{sX>p!@Keu9@SaNRHQLG8fXPIVBRAtH zrOPUk^qS5fQ*N-UDK#c0q4&G-_n0x|BBipeTyYGBOM{aJMxsw31r1N7tvShMVnEt7OJi}(&kmV4{p0V9$dF?&IFmUk7 z=}Bk0nrOoC*EnOsL5l2ajMK`ZRq@qbOA_1;!#kV+7KL=D<6Lw-`r?nazXj7z#KVB% z#?J0V-nMKNpp@SwTx95p%W|+p)6$Zw)8{{ntr0FR=MOTb9HX(;5NZVjy2%ffgo=^z;?ix)qLxYactJ2R6J%4z5e^&u?2HBk`d7fgmo6r?S=h?MZvY!>g#_#5a=) z(WSR+d8C~QgTrW8y2u7N)S3+E{}`^4Wb(ye)&1`~_+nX+5m z=*mXCyV!_5$iFzynQc;`o!?+VMjlpY+gS`*jSfUff@WS_rCh~kVs4y6yuo~xOpwC=VV=m(J9)@84u4%P4tp z?XWCMZ0_Tnc-#RCqv4XJ@|^shuMR5kudBBjEDj}>mH@5$kqt+qlStd z9$a+kL^C4}*=5|j?G7teD<=*)t_t>@c_cEgBJd%lU;Cef)Z3*1z$`M|LR}WmNp$vg zN&Co~%80O!Y}~1Zm9~FGvO2;&XN}u^PYhOy=Pc6BiP4*HYP_vE%CQ5SbXhAGvEi(E z0M$IyF(*i!TwkiphHhsR=qkWXranQh$G?7f=hYQrqC83xdn^TE;%h^UK{nH5t}>*~ z9Kzl1*f5;zn4NHLjJbF?H1|FxtbdyLG5m?Z&F!t~D9}=xjPhh^FQudM=RNid>hf5f z3tunX@YS-dw*H)j$88aAva=0+4`V6@>1SDSL()-<;p8rOzn$nlpm-6?t8j!`RT%D_ zw)p6$mh-q*uakU*FGkPCBs^ zJ)MU!wJ9ytqL|4L7?H5;jVuopOf1lP!BB*exLybyuPyj#L9@15C87VTUd4JOq$D<( zZO;}=b>pyxU6XRmV$+$S)!WrWW;qKqMteDRhjdp`QVCFwoT|$zFv6&|6^rpj%!|vK zxRpXl=aqcSN}mQ9D)v2CDn%$ZS$VTKMU*;n$+5E z)BOmS8y4MGHtF!uX{czyASa4%+ z172-EdyBoA+(5c(_$7>XSX0!Z%=Rj9I`h)OLR#d^Eq9pE84&bVoY8!I8{$*6iu1)(e60 zyPELzb&0zLxieQw(cYH)FP9&ken{`p=s}Pi`Gk)SoUumpvuVR?!_o z-@DYG=g37Wa5+Uq^TII=sjuPdOTic{D=$;R=`l*r^T49mWZ!*LwusS|<;Q+80s!-K zM{e!WA)UA0$SnBBiuOV9mO(AYman&S!@54>MxVuFGq!$T^AMc-OywtR0(o7Ul~`@D z*HELS@b zP>=mwd@B}Cy@d`8$NIW`NN7KO`ZP&QULNTrxj^uM`B#Ma1-60*bIQ)z>@}*7JqTT) zY`Cvuyg6xQY`3~7doNggqiSK9-&h2yUFkj3c_Txo(WTz+|;3H5$3M-s+{0k~TV%5+_+VIxkY) zo%}hg-V2qzL3QJUPjX29^}LvvcTt2VO!Dy4e>=7}!GBMYqW`}6-;Xh z{vJZ{xxQuAWvTa=O_Ubl*#G&)KVHRr1iwK0|Nk{YN~Eew=_2}JDsF351uNsedl0W$ z9_DKQ*B%^LhyO812{hMX;i+-Ugv@rv$=+A(> znij9CN^>xtb#C7Rq)QH^^52z%O$@R$WoMRTq-s&6hDPzfeI|Ig#}{Nv${7+*9|hs$ zX$RcG^5akGf#M3rD4fx@O z(6!l+KDV)7yTTd*(4^yB6I%h{=-=suBRm<6|E>$LF^EdM^;`s>YY`N>p<+qiJl1)u z9YI(Ur1A~H0SWnxS!BMh;|#qbo6Da_uE|n$T;g#}+=K=QjHryoT#~w2t%j7tT)C#U zZNZjP?8*{W@mlc5;*~fxtBv@(Z`-jMTv=fXa?gi}lz45rBa27w?C**l7@x^7{y=Ai zxP*+KTkNW)R@lvxYEAJs+`Z@qvGEkEOL!Y!Y_Hx27*0Uk7rTU$g24zgr5U8;R{!5#i zgtrade2B157<*kV(w&y3QW7i;p<4^G&1MKEaRJUFPXu3xO)|736O3_-R;Z`8-E#xV zmISwDoPJJu3qLX-BUlE&LkUQGcq5A*a8+SypyC2auiyps;m4U&q*4G(B4Ao(k(b>BxKeqN}%J~(yrH;SB_iLwz zJ+!LW^Q-pq{$nOeW9Rj-S;}3(wn(A^A;3;x6J<1?cDtu#aAZ}=cEVkUQ$|ZzlW)%T zm6X&7L;h~%v!`x*bWcu7OVQl}bW=n8Lh?;?Mvs;R+yxjNgu=%&GfSV+p@KpZx1k9Z z6#uyA;O2Hpt3D_N`@3}(v&N0=a@roN;0-mNy1O~Xi8Vqi(5^YL4S|1ceL~(;K~$8P z>>uG83QE;aUZ%8(toWKsjP?!rkuyay)M%&6M|OnpiSCo66Ds~KZl5+3_KUf*@D3>Q z$dBlj=++jSX}o3e-BvVrJDs82hAdxfs|Y!>#gQ{%4_q?rO7MASHOVJ>r@>6VXkVBW zo<|2{T2=&Zsm@XA47jw+AI@H!$kZ{oPk!C$6QNJod_5bp?lLps?)<*uUe^MqP_x<#@tS;V!^01dQr&lp0D|1W+FdoVz|K^ zmHt`;l`*L&X03dCwUl$WtmGs8o@V0MxzH6xooMGqg9aOnnjPP^6Z8lYs5^2!LSnUK z#}IJGk3m+m-`h@WY?zm@T-}n5=K8>xNf^yY72xQ@1&H)u;WQZb_?d$3NuI`PG!6Yx2bpM3R8#@uV~X`wW!h!wosP~)3>q*mWs(mDxvdLy>k91W_w)+=*4DVdep zGsT|@7Z(nO2U*_AfY@MZ+HP zzMTH1{2doO`H}6aVsh5Q2}DY5f0cDB=SpTMH@jVXsIQ)S@S}ex^n4J=JsKtm)5Bud z)ZOZ;Ye?-=Gv^V?t>?{JP_g@DYRRlEztV;6;qTqw3qwX815`yY?YTC~*KMs{zX~_Y zk8!+I(5&K*ZDL5ud@I^yp00AUn{X=$a!%2ISTiw(m^EAcE(%yM!$=~ZQmk-)GY(Z}c)Ta$b9{x-+Jv#U0kj-ai@{93b2!0Dg>Q4tOOKMmx9F#o zg7a-wsl6(hy=ilAB(me4+Tc3sHNp@vaqHXTBhRBQc2OA@Ac{-HS> zSC;^{V~Z|#qwTxG?$+EMflGysp5~5RoAZqk5miVAA3g2_Q`6D$q+Yjrrds z%&J}0Nq4C7TXNeJ`}Z>TJu32az&*NGeQQ_$R3B!6 zgLRowt9@UdtxmZU6;5#?JUz8LuS?vkaz=$X>gq!XU2~rwRT%Tic>xR!=MSbQEc3|M zEKd40CGk0O=pu;`f zO|`#EH=j|u&ge=D|7=d7Uoh<5JG!OAEK(o)saHllp7*kz^3lHFD|+cs2=!Y3;sGd` z&PL8qX@WZ}G0*V$u%7;7HbL_HBYF&kA{2t&RefnykB^ANC7<2rzs7lyBNEQtw4kvc zInfqD6>FG@wIL*WMc2=AO7iQ%jUaTY(N4%?m{S8UTcsIO?shc{Kzp1A|N&T=&}1M_Bl8Hoq7Q>)R5bU-9})wA#AmUjq=KWGIo_bf2d!;m5}L*)HhaDBbSi-_Hn>eJuoY^>LxH@bV<-q7?A+Yn9$Yo z81GF`39p_iBrz39$17~1@sd}Gj&8ok(>6G=5C^$EyfO@1{xbg$pykqC8|puA2ZQ(dsfcrw{BG6}$+QS0h=P!5!__*}<~p=XD3Oz^ON z5C3JIQW$kR`QK*Ab04Qa(_C;VYHCUEJONwv>()6wP7^7lyO<-e^%y(Ry7N~$&XpVC zDm;>>SbB{Li%0Us8?z@Rchsxy4rLYShpM|C1;KN&`X)z*u`cIB;{}DUr`1u0> z!45!N?HkWg$kQ7Ij}WsgNS+G}>sj{ub&L6d^C2dY3u$Z8ScY3g-mD{kIB*3==cjN8 z?otRs=@Yrdwt&*tXV{2d9 z=EAL@!dews-oO{*_gjO#lbxe{W1SZUqOZY+Oc&1w)Np&>2O7~8;PYjleNSGeuqrAm zLOn)$rl|c>>ideg>@K~f!iyh9Z_leQn|!I^Q@EOXmjb!YP(-qx?IAoJz^FA0s1JC! z_sxrW%tII!s@U9cCPqT4;!r<7izL%)5sKqipXt0YgM9-2wZ5luv(3SO$?c=dAJo*L=Q$c% zyRzqHwvoFPT3@?~Um0CE=PijIsRvW?g;Lw8b zOc3JlQi?V_Xq@bDN$2iK7g`|L!_zHZtNM37`SdDr-~A|&yZzx$I$7vJzU>nKkt=zR zuF6p2V?k0MnSuVk2xv2gPT)-TWrK6;PF5An2s;#`)ac}gJi<+qn1KSB!f@hm6ZsHo z(;Jkhl%~g%to!-PUb$Y`0mGWZbgC1tqX49flVIbsr;0lhvz0+BeT7|rc_^Z<>!|^( z9^>=92LgWsul?QV;M#E6JT_LPHU;zTg7X1}_SW2^EsNT#@4$BWDOnKwa0_OvvTvgroXgL5o2()>0NvgF62As;IGn3821GB(KXKJtzbWfRXR(*V248EpE z9@Df+=Yv|4`<`lidXBli8C~b9v6J=djsnR_K-o~_-m2(;42k1B{;Fb*(H*cY$KWo$ z+LikPq}=g=c?l+m{U3&a~ew=?~?`yVmXzyRw-kWckpwGIL5ZBf0lvKK^_^SXPygd4+kWg74tT z=CcZ80@p1jHJ=fZLen`&^vy>KY{+GzLACcL*z_^ottVIwr2Gl0{C&V4?+0LM*eBt#pK`gxYGrBYGrX zq8i|>u6^5hK?VUk<9tuHNCyhcm?nFe2|EyI%-e6)yy!QvWM5o-y^z?ArRO>#rfA*Z z*t&YpVuWD0orC8=Acj=Bb);8366?PQrr!J}B&BGm`p4f}8O=9G3fl<5S@BU*Uh*Gscg zhaoPA6qz?Y^}Wp6c7(3TJckncfMJ!^MbPT>8~me>w1gELjvR$x&QSq%a;NHhnEv`u zcnmZ>kZ|4N2BwWU?<}tF^}o66Wph#Nm3A6GR?-q|?XAy=ze<5S(SaN$k5uvrTK@YaCGMbiQOL`m`U!_2F9ebUH=XVC5 zLuO!$k4$K~z^&5_KpAQtwtdffZD7AnnPb*J_ZeXJvtR&?s|{8qTl2-Dl$uy02GU9C z>*H;eV%@LI#^`v@snk$j_Nu(COh;^-`LN8;%13}tCUa}Z;x515Z4t6Rq-+VcMRH7`orm!@N zC~F1>Tl@i!7rRPq&>4lRv|Tu|4Nrau(lgW^>{^%o+KY)Y#FY|n^sNTgWMOFmuAGl; zGdmHViQJdI+UXDCf!=N~5 z?W>eNF4*DFSJck7Yb+VRU-bCp+vE$j<&~FPmCIcb!-2D2)J>+{O!y-8A;}X5z{^;W zqpwSp_cSo=VgAiskb;@C755eunx+KW;cqhZc9sB#E>s`~BQ3oB8cYOuJA(J&?d>wm zOiDNk52a}-L|EjVM>-!54tB{=>ilW7!7=yK9`4z7uZbSnooBp!uu8YUC=$b76g7<$ z3C>X(qdAtrEDL+6T9jWp(SKOPjjteYcy38tro(CL_xV*Hq7zTOcwiNfkx%r}OJn!4 zppjN9ma@8rB(-HULvMz;(DunwY^na#_Zlji@3i-};jZxo3o8!HQt?-2T&qYV`4dY2rBU`rpfV-0?7YDjo3M0hjW;Km-{GwJ&WzN(7Wp@R3HC@!NQEKPa^ zO60&PM-Hi&LzhgfvC;Uh`08`BVW)(aQCxOuX)Z=O+;WaSy|p7gdg4XB4e3*{DEg~E zF#GmTWjoqy_vxEmf2;&W{kXtCUQnl8(`H#I;tyU1>lZip^hvLUI9U|9oO9%F+EE!u zjIcD(EJU*0$>>CLhQi|FG*MCFG=hv9-kD9iq^5RMjx2^0SLQKQqgv0b_f$gM-Rh=} zt%a`B9y-wbo2Ub@{6FgJ`>b>Z*(Lzf_ey!Aesu=_BCjMeHmuREp<~J2{RvGU$3RXa z=0Z3b1akSejNNnDFT%tQo>i_tA+8h2Ply~!^;2Hh(u8!a6z$=c&>LY^R152F8;s0u zC$dH}(DmMU<08Opa;9%#8vTRxrnfsf1oC%htytz^K zo;0E0o&=mZ8B9)vLIU*J{QLluHul}8@B7&?YQ3yT)0nL&ahE&nj@**MMTv?;9RaQx zU@Cm-kDEg0w_@+ObogICqu6#Afo{Hb-c#Cs_~>yDjTvyxvAbKsyzNCpiOVlvbdD$y zI0djF@{LpR4oB?W411jc_D5zgZj{m-v@9-kh|)fKblM>K)qb)+5@?A``QE8??TYc# zxKeU!&&82Asw_X$P8%J3BJzBAHwrA1ewQ!ix)s$qxi1s~J`wj7dNDL?rp8{Ln8(rf zP1Ovmem5*0Yl8DKsrEtwi#R4;6BC6}REV<2qjo#sPj1Rsb;+%l=(P6J}5P~ zJ|_6ZBDs!qCBI?VOxb=R!^*Amn-`8K41+mQRUUXy-~dI+q;d%&vS;&hp;~_G39-m?zLw>2Q9sQmPF{6R>=ON4l z^1MiYLsm=J{A!Ap{YiDD!|wNs6j##UaMZ?dV%~TiDjd*9GGJ)9UCCl5uMU@8M9f>M zJ+MDvlA27e=n)$Gw5{H@A7S2{Ph{S&+JmvM_0j?l%s+5zIy=GFe$;Uwa5TSCWLe$n zW$#*;UNiR1nHcRKu$>24W;-l^!n>S2%Q~B3wSSA}wdd`nx!*fQ{`za+p{9^5kEOw-J zE7W-`0?{@dz~w}y09WAHA$RI(8NRSi3Ol5OfM0M+9Di;7sA$5*>>8EW+jyr5SRJzi1;3qQ$qJ%>&iw4W?MSc}FG7_bl(VaPX}Pc`welDV_}Zv88RA(B zUquvgJNzDyO^79>Z`l>T^PoTh@-+;vd%9i@>vAfbGL1}_7a5UhTzP(U|0{?J>9Nz` z_F@wco_u9VX*BYilW4uA+h-l>zS`ni&A|F1AwjHsj>KZRG}X%^)ODXkFp+KggYI3W zj@7)IroT!--nGUNVxjX3ZU^oR zMemc&iDOLbh}Lt$p%Q3GW}>R7{#vrbIS>nBQtV7N)_2k$LF#5#;-$MP)^Ycp)aO{E zcRwBRR8)S~0UV>mFt|l}r|8LT$V6iRb0Jt*TN62$aJ|M43v+1HA_FDd~S_UK4Q8h0qWlyO6{s^ zz7BjDXy^r$Pi+L;1VVK-^EuiYd+Y>zhuz-;YT_SS9f|Ryt za>Z{wr20!I7_UPa7X6~FiEFgZ)zG=n`;*F?j3px{-l*QLg^-}#NV29ehdR40|*Kc+{+4t?d@$^riJngK)-3`wKfP8k8 z&+$(-vBxMeXklng_&mP0k_$RJhowE2GQ;jbl3 zp*P^Sdn2%+?Fg%VR>Ny-_rORZMkc;;K;gW5lD)F!R)t9M{Zz%@HmWkr%*BIbovj5$ z6{Y`uWEZ*n+EPV1DoFyTTJBUgV$R*yM%2KIsi{Rs%b1M}!@wZn>4Fq0!zS)eHQUtU zx0a{h*jBWC9$x9<2WQT9_r6!62I5!6-mzsqPw|BmWT+8dZk_l{a%6||EwisF9@v)6 z5*(zo6L@S`?||v0F`j)R!Uo+V-oglmUEwXUjzG()oEl$--PkB7xGLYpQXyqkBdejuLcsK}r^v&~s zp=dCi7^HEd30k3dmd}(uTSxE_#khqU;3V&|(8chw4-~P})?fgwq&3r07NkB3z{V5pM%-S6&6bS}J=T2cmm?ZvsePN1DP`b*+R}z-Ogmht@w1ycL7!i{pOq zypJ6*j-Ttj+aMQSDMqcq6yix~3k5WDZAA|)G3Z@VS>W>uUYp{qF zyil1-@5RIihE&PDJ*bqyHS8yG5E*pdije52XVqb3Sa9#G;PB#L@(r0Qg~i0~&&MW2 zHF}ZLQk$-LSBjap;xr*V*|Rq^0bX|tIW6qJZla>-d3e@4iO1+EW<2VG8?q-Blm1O3 zht$VnKvo2wnT>9arn~^h66^@Sz_uSj7VW~skV8Y}`k}ZdtJRd^UvW0Q?Bz731z4{` z;1aSw-d*G;1`aK@ILbosE)L@nw35Bgwf-W}?5x(DckFO0(ONWL(B@4{;2<|ViHUfe zAMS1h6pt57pWvuq^9=U1X8T5DD@~{@Fwzm4Lw77{MJ4!xuBv=+*7rVUp3kw5yShkJ zsym*f6!3(lzdYkAM$wzCxV{`(t{t2lvN%h!q4GR6q!y4SD@``x< z&(D)fjECV+NH4;&G1zc9wt_I(!!C{2|LeW@ssTK! zP51{_In<7HvRQYzO3woSWj5-^p|hh`pUy{gPCt*Rr&!Tm)JQr3=7d?tsi07;u(6I< z9T0i`og_xZ!%Ai!AxcQU^wmuv%P?}`-I^XLsOjnC-zZR9EuN>;rI77Z?)HWX31M4W*K*Ch!O4*g=9E0+YAo6g^zw;;-scJlC3XW-BHxX zac~5D09c^gwIThw zp%<)~=3z~z!JB;T#|Mnkw$7$^Z-zU8TY1)TJY}G~jaXN)k>TCDx`%n4ne{7+K&5mF8VM{l>y`Qu-aA)*W>$Khx!eznh| zs(F8HRyf4k&>xbE_=Gq~a2(zka^RPye70DaxZrIexH=jG7(2S@g8VO{!rJ%kpx4~+ zGMbd<4gLN+_Y)6rjfI<7^RH+`L0d0Cn!m`)0jxG{j7S>>1}ChNsiQI~i5g17R>`8Y zzBKpU%EDC!G9v8=SIzB1^7E1BWB$8r4{xzn{p`TG6A>I#hpijJMb6|}i)jwtmywZK z0Qq|#J}1pO%HWIK<6!@}>`%t4sX(qWsuI74|G+d6TJPHb8-lF;F9eycDRhhA4~j@f z`@i&U9AjEbeIy=M=4a8mEqtOCiIs-DB_38 zB)I%_b6Vrc1B6%Wb_GLgGAc^kutgFe3Zrw|7Zp3?21jQPH*21~&+@Vj0Y^T8&vWXF z?c=h?1~@ZBNGte1ba z#lD?YQE5X^8prqeYcl>dh-S8xznbBM_TC2iu=* zqfdNCDNfQYd*$;X9+S2P?t3q*wT<^w4F!nVgyUc}<1STEES#O3 z_l6?ut+r=@!qOzSMg&NGop_bAxq3HM9jEri!;+!`HmXZZ;DIR`ITYoSONR>c9UX}c z<^XBGy??H9F&0SUeK2Z%JzEgL+AF3(9ULzLvtTxX`oFQg3q%d#NAghlkYw8M&$RYV zCtI{I*OId2<96t8WtL*1#aa}`F(52E+-F`0@#^Lo`-d!bl?{orNtf^is{3mjrbe+h z)d#mXRk(Yb<4`S!d;LIlf*%*~%PD&wHVp3?eJ6xCgWDDsFmDI4SXe0v{t=1u<1{|F z7#;|{sDKbf%4evP2fEmfqCJ-$lrms+>ljAWYn90B z>w5%)7Xv_Q<@dh{C#=#zX|tWTZ!<*OToCnG{|HwPB&1_2dc1$#6Se1d5E2Qjy}2yqsL@1uPHafVJ&eKL`uB& z%XqpD1D{GsQ5F6hXnT`Z6*HIiYDJEO{b_U$iO z>~nwX7z$&hniuQInM_~02u>dKTT`#P2nDje^>zE1{m%@Ag_7f1O)TucL%heRhVB!{ z?`&8|cM~FD3oC`aq1HR3s=Nq*K8Ksm0{0?oxN9yxAM1q^4FSS57yxxn+0t4;I%YC! z%b1s!@sW^C4tAx&aYNZ#N}&GZA6z>48-2&yXTS)5vOZzkj9q3_+HXV9*Cm=L; z?Ek!~AD=poUgypY(2q4?iBDdCW|)ye$N1OMX{ zDhIs9|M&L|QgVc4X~?Pn%_Ir*)F$DY2lv)cM_-7N6N@@B?olLi&e)+E2ZVGF;wEdr zuqtm0C;fNeFR(|o>)k7HdMYt&o#A-?(fD{#zM6uUGk6aV;=!@PyUG|h1EV|jbj~`z zARkR2vCtd{l`xHVyCLBpQE_Ze{P?96Kj{1brpj7d%Mbm`Rc}bbGj4`%r~T8H9lV98 z)44(k2N^Z~v#u|fXDXxW){#5hy}8hN5#Efe&b#e$o=>?(+_%A2=%c~G=-h!Z zmHKP&*JUFX;@WFN?H`Jr7}#KZDf|miAI1I6M9_mFV9yQG+#TKsJQzLO&7C$Bw9Y$7 zUM%}kXv$`bgiz(+*&IR-><+e;-BzOyx34^j{KpP0BBbRkrjIudT!H!nhuruT?q7Fq zCU>2xEqdTZ%P^UVnE$k*hG?lJtJamJ(YbS}XC%i7YUj_wejJ;f)8CRyHJ3W&wEXd# zkJXbwBTdcW87#vDl%Vo8+T`)x<7;Q*Yw05qWWzTy>pWS!wcumxdk_p*&aQ@CZ+|yD z<^428osw?Xa^w9YMct{kq!oCA0Wmq-$aH)C1zpEVq|P1HSry?c!P6QB#h<5z9PfHA z1Sl7WG5%Lot_-KPa>A2|Ll~4R(-UY-8d|*YLMhq^Kom$TgfC5Z3Juk2Fs)#Sfjs zBpz!L#}QV;DaBYBYEm%F8iLQyv1kkZG!y9m(Hc!XROH)7WdWT7Nfw&%k3EjLOBBDp zHPc}#SQ_TEB-Wck_&pW_feq;dDL=B2q>+^esq9~P{~U{O3!E?*q^y*paaNnN<(K54 z9~faEGDi-LM2M=1_To5cb}%}u!_DD-91R(NLTY!q|SZCNY21nn}$F*~0JKOZO``yXG!vu*j?aq;}UhImF%)nt6UWM$aua}Nt zc|puok8b>ztd9mF^}Pd&3RlC=8_nZ-{i|YAUQY!D`$?+Ki-3044G@f@UYqrK^>$?W z;7Jn%8aJx=aPg{luUj9Bl%aNpW9iHgX`k{XV@h!*evV`*-EFsMYu994o&csG=#x4Y7 zxa=bJCr>nD@*bv2n*80Y=ktmIa1dfzWQGl!P{IsK;LD~Yc0v&y4F2eKCy+Do;j3GzQf2xvl96>{1k5mZQp1#;veod#nSu|u%P~BXG*;7*uC&fY|IOX3oJ`tw|06% zxz^=o{YH7s)@b5SPMm;1LU3>^zZ3R*j%r=>FMh4@-oWY9#>$YAOmi(!5Oh*(7W+-; zhH?LC#-MJ~oBP}Q%h}qM8j`M(A>oTnJt=a{?yLm+?R8^{QqNc}k@=t1ur>7BMAYHO zxf|~N_Q{_g55&z7=)(Wc2xkaIHA#&CF3Us+nfzdO=0FyPeeSsRThZg3F`5B8l;=JU zZ(q1|vFDiO4|GAAvU+tPkV^aQLQl&A<*}_JvN7}P-@Se9uA+AoYFR zi^y=0hpiFaQgfB_Gw8mC0U`*6gZ}@J-VNVJCll~TTwe{ncHrVMEgO74`poZ+q<8|I zSSJI0*4mHUfC|(5t+I%&N}U+e<%4HgBmYr@NrQ`* zg*VWz$g}ohSFTX}0SBKs(K*#s`;!mpUfVYZcSAQ?vmBe7R3-Y1t)V6XkGn?`1X9cp zEEn4oh+n%`dz8^RB;C65WZ&E1-qp9N?(JwYwOOp1p$$TCGTk4y#IHu8QS~+174ivH z3P{9xxyo$0$GK@foz4iK6%B_PdS~(3=w*vP@N^o;O*saIQ{ZHk5owE=;?H%OyX^}nmCLGpI>ibHtsH92I}dmfC*Mg*Hyb>j_F8a z=>BXhjlWqSjB=0q@}>@`_zhf|ca*?dC$7O%O2`Flnz?-giSg3FJBsI&Z61m0Iz>k{^ZyUbz zd1@5#;&V;pqKb)rrgin5__I3CJhO^H9OyN(s&g{T$jW6RI=g((K}m>}kPNKP%Y0t) zv~y4p##}qy9ywY>d1+K3%v*t3iP|0AqTcosNEDCB&n$+N~$Jq95m@WdW6Zv+w0^tBvhm} z>gj(Z=WzX~9(v5kx-PG-hXXnp)=K&#NmR?Wd6u)^i%Wp7;yBnU`0TOd6t(5wkY=} zWrET&X(*!tw7a@WJi>^ha7?LOr*;zbalb+u48}EwkOz_At65!FtX7?O_J;>hIg=G4 z`f9;3A*YfZl>G&S5iqOC-kky@NiNb4?fF-ZgQT4|fa)Ebw>R*F$IDd;gOrWsLqhv% zdP<-5a0=ydLP&9l(*umo>ExIrQWu~`E@5)R0ec0{Uj*h}nK^rISB1KsM2_a$5^7aO`m z6!q#XV9KM&U*5_HY~mP|`#`7LpZbdVl(6Dynn469bxXG{t46Ksk>X3!Dm8JKo!hzi z$vNJaiThwNBY#@kuv4c~^G(N5zwN5xrPE|b&xGGVN20JE-dhq$=q`tT;71Q8`Ga72 zUuvprN=9Jouw`|lA>)c7FnyrB)n)Q5uel{iP97%)cyRQ9jO!{a#x7+rCHi68jQzxo z01aa6;u5%DQe4{>A>E3TGNMR;%(oHuDGFKFWm;1`rtpze?*$s;tzDg%Vj?gdbArN0 zj@ILX)OS*8->~uCgf|!r;dGx%s%BD*&FibpePG8^roSIs$Ib4FpYHX2>fJ=J18X$2 z_wuI=W^rt+l@Gw06xh<@{O#+5D$)Hbd5oqad029MeU1XFsMzDSA64+K;V*68TB)3>|<=3RQ?lX8NL#VUA%Z0y6+(e@AC|5twA;R^@ZB#qxgnSy2XrITg!OR zPBba+EPtRu9JXkj=Sid{rpjmib+>dq)>Fio8As6Q`kPN{80N+W!7qyAadYApH0i#4 zo>2bi^309RrWd1ZJzgBWu+Ra5Vyu+ZZfj5`H@?{~i%Faml}|hHKiOw72_Oa#BCR8g z>fiW&SG(H#{b-Il1bH8P=gi3GC=M2&+=1l;V&xQ!&n+kVTk$`}^3)Yar0Qq926Wq*5|<>9UGYxjGS^u& zupBr)(&)^0A;iYvIHX2=u?xY&K&LkJw9`}&X4kmoq}zxaJ^pjYOkv2O+3r1(Aareg zxBH{|5RIfo$aR%e%9Gn$psIaI;wqwkRF=EuBCTI)-tI?aKS43ZGJzMvA!;~Hf9#jr z3$Ry-&DPZn7zyCUm!D3N*@vArKaO625B(irM`(mi)bUr} zZu7+K(xGaAFMd&hhu@Rj$8mX><)a$ra-)+^o+tpC(GsZ{%PQ!|?hTFjDz-W#zEGss zw2JNxcC3*)+*lv|X5tw5u}v?9?KU$xH_t3ZdsnGBYsYuMb|+%i<&0j&cJn|EYbP)% z)Br0a?LoHzjMA$x5Aj8tCZ%POn>O)kK{nSxl2>cfLQGEb=F+f;!N#W@h`*E|8~20Y zGla+f%E<^nXfOO>%3=nx#8K|7Mm3q(`wRQ}zq3O0f1-#wO)ptZ|Ky}Vn024r`0%MTS)X6`Qk$j2gNKU=zxFw6bV3(ifzDs(}zJ~7IOUz2eoBXq*_Kio# z5=d5*=irz#WC*L@LcVimqel{IBq&cIUf*ahT=}^LlT*L)9Y`vaFSmum*VySAd^S#HFLI}jb=OID zvvS}4`Q>0sU&YO&H$z_PA0{%4~*4=mJvbV z(Uck-TD&>EvfdWAv6Z&*<^(+nvRdX24YR8AG$N^c>8*b&S-XF&Z z3=F+gxs$oFXwr&4T}!8nQOJ`0bju4`PfPtW&%P>13hg?->v#27=$EIo-BblcBJoO*NKlUjG{s|U3o+~3(0JGF#+HkJj2oWNzO zQMq~1g-nPU-rg~#v;BNo%Tnkx7nq4=LPN{$K3U62?`p%t;A-&{njk~^%IC{)s7A5B zj>%j#ds{~Ve+!9oC`BlnyNd#@wZGfH__0}s% zq2Ox#Uq&p1&(FOPh=>H$!hB40e3rYv=5ssg@mR#J<~Ubci8QB z>vxT>Hkbm`_R9T$uTCpOT%S-h%*?PWy{5||W)y_;_s?)`j>KUL6@A9UWtut(# z(q>BqV%zdtlT23<0PibI8uHH(6`87n4Ka9op{SIIll&l;T9x0LC{> zaBO=sk@3k(pq^;p^pg1a)|BbO8VmXcU|mb$27h=+eo--?4tOTj+1RSHr14!TU$OV{ zcampW$T`aJ;ZVVVHMbqi|Jo0jp(XXPKt|}?AhJ;?$@dPs%AL}^`6v7(!^?xkIoTZN zduz;a1MB`{3O!AHn9dE&nq%g|ce6=;ri9g7>!Fv?0h7;)E;rA{e_hA=0_J*~5}WpC zXNonE5zpD(qNgPcpER|sMU17dS0gq1|EwvooWRD4q9lra@sSH5idnG{{|!}k>C$Mk z+TK1iN(wEUhMF2x7xSezO`&tG*B^tM5HFngcb=4VKHsr`=7XC4^R)-QsDaFRdTfoD zo|Aw{Ssp#@n$#i1`sSu3n2(rOj>=ep3$aWwG$EbYObM={l;R; zt0+UEDgBt=7im^U!uMsl^`pG8>{~{nG=*53{3D>BFU@cYQbBFU1h37(E)&~)=2VZqMqjzY?mlHRb4+K!Fc#`{F7&B81bmKesfcmey{ZvQo0%^4p6*e_ z)fCbpBX5#boRKD-gl1^YhS_`^sS`R4c3?uSavkNkfQ)zNV&mD%ZKr7e)wI(c>iyFP zSnXmGdCtF$)d+w2S>YdHfx!Bb^Z#Z6O6z=l9_gS(GP4Z9<0uE0E@;mk_BS8_WdS{FAHUmtPXX$rHL}Tf6*9`h`+O0tCNq*A9Nk`nN1)>@W}hoE;S&V+^$L!%V-@yfR831l6&!M4HcO6`_`Y9sL1 zKL@BPUtI-V!bccI_C<=%46%Sa1No6HZP*rFUf`E5uW1e2?WvmY4Q<-zjc22nv*D{b zNnHmJcjp54Nwwac^MJ{(!W66niSdeifS(fdWCVNr8hV zo?)SDE`;eEB}Dj$JQTRMMoWWvaTXHX{$yrC%&MI-5`-v5n=*N*344!F+B=G7 zoY5yV7H)P>0|UI|xn10LY~zlMMFJ)_)Q>|=j-4EV79`*JSFFufC9UW)iywbHJ3 zy!xr9L}vDvl3>Si^I+4Z?&;|@b1yj#@vi{UGe-QF`YWa2q?q%LgXxx47Z2e@Yq#2w z<$!iOdQ&tzW9*%9XVBzV;ty)f5l{J$b?6 ziOa4m!(Km=Rz&y5GbO$&sB9UdBxUeU6TZ->H@KCv(QR8eUG075(}Rr<-Jq|Zce;Qx zsn4-hCSUxO_?8#Q1y{*+zl<=hEClTYrPU6bU<$zpZWJ1Xb8rX|DQTAT`+h!q9#&ik zc4PS)3D!?m7YD_TOUJt`c;IrACSx;M>o_!MEvZz;=$Y@NCg96UHHPlTzg@4P|7C9n z`re)#mY@i0)06Lt*n#fuw^tMh_Rmj!WSr5|+7Vaj+BrW?DE;gV`_==vWt^PtrSc9P z?xXYZ-E|e-I|^b$n_`__&+-V7(4N%BeMZ&F?dgTB>e{L+LXOCVTeHh%@}7>WyenF5 zWwEJMkCvXW*;0>X9S0^Hj`cVhpLbG(I88Hzk;j5Dy*$sICNC@cu`?Bn{(ASu_e@U!XPIm3({vOq!c1LNN~(a%#fL=7<_`v%XtIhJ^yO@?lYaN;?SZXCKm3DkO)qIu0!kHm4n zJ@~~#o80H^Tp~Khmt{d<;Em||LT771u+~(^?vnqZ-8D}NTSVI*X0a^D$GtyA&wHL| zhf9%F+$?Pu9@%Rf)CRt-0~Q z#xcJz4rEI*b^qz?7I=U{FdpLigXU@6@}ekDPiqfV7rd!=d(}sm-G0K)R;l~N6^h&W z<}kiC&2&5gRtFqCdp*}BaZn2c2Xf%eX#0tw7Tg2kR{LjwoKT{vH zPwpIEBb5>dWhCB6;Nq%Z?$4p+2@fw4l@AUToJcWaIbUmeppl2^rSP|{1g`8AG#|d9 zgWM)G_1U=TmCb=PuD|*YURm(7eBS+-IZ(GWvN%N!>M6u9Lgn4rVz(q%wiLC>mrs#V zm{k`Rrcg4t;OjXV|5^FumB01agSKVTwnY7N@{pqhy4k%3(229A{R$PMAXKT+?ki|2$=J)(uu^v+hi<7}{k@dZt-(_Ym=kFMUG+K)LiM*lKC`D4&#Y@P-2F}>b-hT?m2E9T{pg3*L5%ZO;~r_6*iZj zYG(NlmC&D8=2Y)|W5_ZW+rQE9uj`_U0)$50irxzjw}#iv&9a8!vS0ZV*e6AAVDnewo&Q=;g_@cqQBSt(|kLy&Zp6+fc<4U$y+}(!;95eOS0nk@lv{s;oW{;@!#x5;^Ud=_ zY0U>{=3kdu4dcL6A@ADh(@C7}gKe%~YEZYybdh?=8xA7rH7$ICwhjyqO zwp=i>(#6|ENvpUJs$oBw^$;GvXwx6)teZ++f@nShvf;!3^0#~zWJ)YL3ntA+RNpHG z57&xkOsfi;y*W|vC!}f?rk@}8G6uHh#TONca~gT3X7?pflz%+M8aF=z9(*U|vwU9R z4^h23@yNQMT^YmGkAyKOn9lCR2~TUeR;={Jtsujw=nmZtm^a`w8%-qwXM!t5Xy#@Q0MtASX5+g{1wkdHcf%zB=7RD*n-L^$gS0p zSIbg#k80MCdhU>UrvKBy&7@y$2v?&yu)ti*`12RlWz;BGy1RFn-1Ic{)l8<17GxT) zO#iLwjY5tP&Yp}wa_3E2N8t5_Siw}*r(+SXLjx772Mz zMqQ>=g*6}%==d_FAPM?`FUpDI%a56tn-dSyT@O<|7B4L+{f#HFs*vw1E!MSFlwNIK z_L|&jE_1;z14{zCqFJu4T5xus6nw_?T&8_?)u>fDWyp(jpDCjS=tPXV6TeKwJ@ug! z7_b~P7i(O}1rGO^lhe~K)-ribUk{fu3{#cfi*)yI4Qt|{%$J+U{t<=Iv;zg+$Jl>EjXFdKVq!2!NPF28#9y87 z5tU~qTVqWhHB;K)4g8iLk(V)AjvGARZgy(m;XrEpd*%Vt*{?)W1*ge=9JhxQob@Nag(!YT;SdjA@>l^iXQ8HQ z3P})0etwtTJ06`@s{2<*s;AgH5QW%z^qWK^u4XnQrPD^Ijz()nLRgOlt0`1erd%!# zcNMy+O8isFA6a3kb*bvg<_Vlimv(SK2M)E~%(@1MA|cX3LWc`k89t4{8c)qi;0X@K z??!qL6P6tG(4+xp6PYX?qO3eab`hysdT zO=sE&Hemmi^}{-0xx2OzoFk(0W&QrUIidpAHKj*yHpdPAv<}#x0$~3?j82Dy3#@x@ z;8caW4|f++KKjYY?rz3Z#p20YM1Ik)S)CBka<`%o_jQpNs(bg@Z@3)F2V2i`<4oZB zYOrTGpAP-iAy=eqOS=CY@V4#o81Yw4a`B8$Ao1Tg0aed=3Eo zTW zg^o;z=l`Hr?j0+X|$P+`X5sX zd>xW3Qh-=!K@F0y%9Nar2^J7$^c>ShW894%HBOQsa01o z@3%$RcreS~=e`PnV6P&IOo|0)GGL}8E%>E<*$Jxus}~9_DUFGlLqsmf%lUg_m5GD` zS5>+8BNRtDj%CLxx@^|vjqbj39P+Tsg;H@~1&*_UA()rBtQ9}CzXR024+0k)~8KBqu#s` zi*FR}d^DkEYXbzeY_LclINd5C5Vi>;Ki$-qr~Ad!LT^dZC!Xj!n3u}6?Wn0s{C?+h z(Zu75yyKrpXIxypCyo)Z_NeR|%j8RMTdC*%Ud`m{(bV1-^SMFfKV)M3Q<~b2;La8n^xk6>al&z)!TWi1 z9lWFp$K8$%xs3dz?Nd!$+^EQi5_V|JHYv`NY^0~FvWR-Wwd|C?nRiMXTK!>8DacH$ zXJlZuZ5Lk7sk<|d;@C>LG*pYW2v+AhUb;$YS~|R z9WZ=HDN9XuWH<+~amaUhT2_KhN334^InthEmuuRgG%b=OOJCQ6iln$iyg3*{ek9{k z4D3TPjLHVP2`dd#)5cc&G*z=%@9oBR9H!mMgFJC?>l&Q&y&U;QnfESa>E~;UzhbnH zC#Sn{oBZFxwUG8nNXE0 z3|w0a-GG4L1yyrc`d3d2Dt?Cv5V&8j0_`Uc-}abjsm?3um@uMjyA64QGh-Y;(BU5le-d_e@p5D7)FACo=Z!!lyvI^#}R$cNJ`r8?Y%?S zy|Hg6N!s*xb}8}(aB|qO0$I}?VXG2oQ&J!r_8~$cmuJeq^2!0U>8xSp75QSJ^EK)G z+)j6c@2RqzK;26pHq{wTouiF>-@}1B} z!^irBQtFG>mc7Ub-Pof=zTjYWh#t8F$%-_oISCf`_yawQc6o=e;o3jk+q zJS8cfqb}uHDYrCjY6FbkJ$iKim9Ajdub)Lwnx|5>smT13KgNtcl}4*4*OCVMeM^k)GBdHIIhp#JRn*Y4X6E zz^2v*`4x&6zpI7cuL8M?P$A2%9Kam~6kI(*_jg*cYd?b`KRj9Dc054aW?X0k|GKuK z61_a>?w#UGQ{eO5k;XshE=UFG#^LX~GtRH=;6uke6;+iDPMq~qyfC7r1ax}sBdmWo z!9Mt&s&XBk1akMkvG&_Z!{hqkyEcXxLUQlPlIhvM$8!5vcE z-6c2#&zt`5d7gQn=bjI<*1R)oX06EwNKSTk&Nsko$@MIbw3Qy(w9HP?B zMfUt&SkdrgSFW$Wq6S_JF^fp33y4KgxcgQ^qGp2vc>WU4R*_a1icYW!ht}>vGc#WP z_Ewa>2GP;aPy8yk<_S9U!z6#kaBk&RG}(1_d}+F5@OCO-jGK)g%YjlclJ~=hE2wcS z@4b?yki$riy+Z(v$bjrOw~~ySCsrO|xZ&nnQ#tuT{r4xT*b?Dc5(AcPoh~*OVOKR z{mA??7vEad;x}bhR1)uydA|n7>G26=4Vmi~TFgFlF0JP7p96x3bfWA^!y(EQK(U=e zBAplzc39Lj(nzY$!;6PE+K0U!MZa|F;Ui3BAU1L|tLEI!o-#1~bZ~cvLBqY!uZjE2 z62pXnHB@TZoN&D4BWsDRGtXbWuJu7&DtOfy-6^+`8(vv~55E8H&pCurD+7MP_2hs5 z{paETeC8D}_zL|$U;XbbLXKqq6)*qKo&9a6ZN~Zyeh2@pr%m2?6vF@B_;0(p)PGeT z{`)7I#{cnP{;jKC7s`K1=l}LbMQsd`&I@>tzFI+ilq>sW|D^?fEC}Oq%Fp-XG$0Uu zmcIX-Wrxv6D*5IA*0RaZ=bhVJBqLm_#EWX^?|u9CtNCAd(rl@}|Gj&7?JINW|C#&l z5J`=noTQ``8^H3YNummFvk7LY%=*@=(!pVPs9A!MRNMJmy9Z(Ag#zUT102dCw`jfQ z4~FNb;ls&IxUZ=wq@>@~Iyfvm82dX`LkRF3rDJUEtF)E^N#?$C{c?FHLS09YJ<;RD-TTHa`XqnJbm>a z-ysiHl8TkmLlqyC`I>H?m<%m3`jS+w8+twz9QfZp)ho{=GBf>|?^c<6aDkKVZnm83 zl1K5i%?&RaMDV?Z-`Spj0WDtT;U_C;nwic_gJwedxxgId0cWp(Mcgt6aj*PI!khJX z`kV#gVbvkKsN_uDYvF>|75kFjhS>SLl-3wj0}MOcVA(TtPa1sFN)x9(gQzTLL#r~g zqqGRW*95}Od6ku?Z1@Fi<@rq&nVjN$^(IFW#8kP{e}~RlsFd+IKeqB-Ok#7yp;8)j z&yuxtwbIN<72zQ8I^_};_4+zEOuk;8=VHV5hHrW9IN`p`aP+j2!erP{>PX5z{pUw& z5;S}|-NK9IAB0heq(2t+^GY9(@l0NM8_YZ0ULX4QG^-uCKtR|OwmkVs?{Q^VW3^mT z#%hCo&Q=>GKWI-DvEP+N{K>+m$-hv_&7LqKxMvzq$$t2m1p#;AH6tefVLn{r* zi>6dI4U{Df+qXIaAJ8<8s4Hx{T_=Yeq*0nvtJ@)LWGv~AtZNR@GfHigTNlRZzzcSG ziJeP9A!Ub)GyAu+j&ybYOyi3I8QT(!ACj6zH_xLmp(WiUeDBw)*eTFq-7?oTV}G0Q z_5t*X*yrJmj#EBmk$dhn0+%6r4-+QU^hSOaQJ%br_T>%)o!~?ixNGDNt5`ggayKlg z*!!Kgn*`6v3+QEmnm4NHX{Z)11VQb3?ZE@Ae4tlwd&=)!fub

    =wkCjPBA9KG+otb~R~cLW6aI2E&&vn8B?QG}}4VqBmkkeLy^>`6EsS zi|dIT{2CG$7sdrkHj#rzLNMI>s3O{l0yVFlME5;fPn4}KzG~BQjr{~0DCa~psDQOA z7!do6tjq7>hErGHYc!NB$!jWUn0rEH<1~-W(yPTGoUKV+Y?IO&%_3b9#CTfAu_d4V zu37a;2ZYoa2|f~F22ab87v7ue-Z_j`QA27pU!{hUq6{l4(2a4~$vdxL*EX$iMKJig zKK5(;>iAX*#$rhm<)l8ACxne*hECts4n6GBGt<~aALAh%S{qTV1Yg`U8P9e^0e{+9Z@(+EkQClyp(Hw}` zy#yc7Fo$PsY;5eUO=xt`=f-AXBn^Exx5IrB(ngM7-_5{)hzO!e{f3@=;Pz07Rx(uP zMm*TBFuSe@-hwd%43KcmKogqLDhiR+#Rl{09~dCd)?mQM5N7uWuAjnQKm+;`j3(&c znIddsV-|)Po7KQ6iAk8Sg<}4aoY>V}rd!>Y+3r#~K`vx*dA5kA+IgE6lqxIh@hONW zXNiw6_|abvG}-a#6~T%N%}t8qk-SMnT(^9HmMG((FRmb{lbuY$!xgPhz?QlUjLFJ7 zhYApUbXUlx{?%f5L?QoRbd$l%X|78!r;am0d5`;+#sUecCmI>mNBNeU{7vHf66qr) z@D9JtTAIH|28m-)hKiy(Y`H&m7ZEQne5UC*%`99!3R1*&Diwwn;w8 zWkbzh(pM!Q&di%UrlMU-gp&Fql3|WrrWE86JKW<2g}VI$dQjA82_6%wQ>j^4edR}T;K&L?#H(gcs6xb>k{5YElWI?+=V()RRw+srpbT6!_Ak9SlwAMv;h;4I}K- znVP3NOY(}Gqrj+EhzwF~gHh+77fzrx7|6kQcI7q5Z|!_0oFU{~gVC{C z^*WZ+Yi);xux^JUm5-pytLpEjlZ|Z12XK5t+;t;IgOMi28t+($KHpK-vQ167*Z^Nv zx{47qcLUf6(fuZT*l)DX4PtwytA$QFkWu_eC z{Gf9-=6A#?>hc#&pzsY+t9XZPrTDr-V$Q1s8g&856{SD=-C2GqrLy)dWcO$=ad6y=(*hE?9hn38#4}Ou6Cc@Xa2SG-(ki4W2JsKwUBFU0oXJNMJ8(megtt zi4`;Y@hQKtrY-#x7C=LJ5eNGtu;`G=qJjczk+27?2Z(D_)eDGk; zHzl)8x-pltOmlTb^sUP65WW1!3?WSW^znzPDi2tT|7MDDmMZQ`qOeEx5uXO-D+@*o zX<1_Jv=Bluf)*Ov(0p~Q%7Rp2TUEUgz5bFMX4tXDMbZ-=4{`DOKO~-a?bx?aP_&@d z5BL{|gJckR>97N<&bV8^IEsWm(l<97mhOsG71WEp{)9i0zRT1Q5 zYwhHJTr3}Gq)J`O>JQPyC3!Jq3*Dcm6bzDT^u%{Zcnl@0lP@##rNY(R{P>RJr160U ziZu~Yp(-nap#Gh4ILgC%A&s`LuiBQWU=(iQ?iXP(T=%XtR3)Y&O9R-CMaBEG9^_l| zw4p6+_t2A9W5liEh001Mk8D70q~Z}2W6^qWb5#oYPKT`=J)0j{0N;^D)IYUMk!+M* z@Y0mJsO9G%rKw_b^hw%xCX1R!G4i-!Zb)de#`ok_pnA1b#Z%i7_Sa`6LJldYce%p& z_)*26gBt0AV(SDvezn63nP@JpAz#z}9JZ=(t-_`AZ^NuqFu zxnsAt-F`)D)r%zU9Ji<#k6Uk3CMJ@SlZ9M1S4`)D`DeBao&`PrtU9Ba#)r(v%;yBWSf!&yW*?q=YhveX4`Th>AYB_>#+m z-N;!AL1K4pH_Kv~Wz=`ZAFi)62mZ{RMH#%s2nP4~9~!m8yEUMT$#w(Nsb&jfg2!~q z-Rw}Y^)W#hl@woqlg@XUbDO9t7n7caJ9)y^yHo&{lMUQ4k?nA?+yE4(ixx0zkFnoR z|5;oXUzM-o{-u$(x>neTTs}Z9NqlPQa6pcs^&!0s?;uT84r`q8W4S4-YD!uu3Tn=x z)6RgexsxcSvRZ;f%Zx_pLlRaGufwVoYeKy}HGR4VTczYi=tF=#TPd&ZfDA-#$493H zK>4OS*Z)?J-w6F2B1}nNDs#Hzo_zUwtuKANWVF)PDLp-XbYvtmDLKFOh(P8XyAML9 zTDN81BJpNk16+frz>)B&QBoeaUU$Dm>Ms@>31b&F zYjDO_Z7uB5Met}$HCEwL(OqN*<900n9(EQ=)?JLgW^?=;d<2Ca#%Zb_OC&^vuwfQh zA7pd0YqTROzYG+?s}xPxfOT-K^FSI!) z6tr$n?#XYFO%Tt$Wf@g>Y!dlYwht=sTZwx4wuN~v-5=DO_HgoLk_Cb%Ugepz#G`|p zP?gn29rGz6{pGg^mE$)S?NmW1+(=;E4)L3FutsXOjI2hB75Gb$Ggx~E0Pv=`a2*hL zP1wfQy{%eB8*DxQU#EY#SD#Ak6q(;&K--GK=fxP)4nms8Kz2I661V}*uh68Tw~#q`nm?VHm#8DZ(~xS!Z9&1Vg^$@ zv0lAy{VZ{YD04^5S7_>q`NpQ39DcE=NaL_eg*Mp>**{joV_u=ZcG#aR%1gqp*Km%3 z{S!*FkqZt=D!s_ifOdIJ{cb*&G+M6u14cTL3lJ&+WTPxnH5jw66fBl{C0kzI?@qf9 zihpE#T8b-U)TM&gd+nn|M~fbs-l<39(pV2&c6TGzxOp`?E?Y8h# zd?jU1eZD5qEh0ylSa`Z-!;g}IoS1l?;F724@Dn#pSf9QN(c&&X-yAaP1#;!d!tSuf3e? zW>{_ZN?0H^MA zoGI#7q&tOjhs-~>=xw>0WZzkiI7YT^b+R7{ipMS;PYTbWY zE>{cUq^T&5fROWP1K1u;DA|Dz1k zl^#HY{VV-HAOMwWE*?fUHa2?JRrdY;{RtBj9w2Y7`Bk$Sq6|*52vA5)xE4y0y$uxl z@L{X)589u@rt2z}&QuzuXQZ1A=L4vhJD~N#<&_jX$i6MR05C!Mj*YlkDdjm=w5c{n zUm2;vI@r`S=j0CjpeQWiR{*(+0g$WFI=47KF~2#d=6k?6m2Z&a)*L{Hf8w*RDBU69 z2`M710r*hulc|yKpqV&~I)zhpNlH#F1VFOyIab$gr-yMr>b$A%M_fD2^qdT!OPQ9^ z*uwj3+2zR^3HAb&L(}Q21qG!+O`4k%3SQ);mKA_Rw(leYV3c~p!J9qes5Feq{cd;H z9$z;a<;(rUyS+4_NkS(PDgor2E0CujWYyKyGOsE9d5*0A~%*6!- zAeBl@M*o4YK2rM*$^r~jy@5OY4kZI(1i+G*?+9;IV^Csm0P+3he=xZKD8PlS+gs+g zA(?FdZ$Z2N9>n+GDE(;D3jyqQrQE5-DUVDKM3`M(3H4sZkD zXQoe{xBgA7VW<(?Gmr59Zie8Sz5!ssf2E|f{~~(}FqxP6nCip- zDOV|0bRjIu_}`Gg#Pn}DHcV1{6VPY#Xa5oFuvV!Ef#g$_<{mQdhXEZmF`-n#@h^J9 z8X_kq>Swzp>i-ye=73G`E4dCe{rlM`+l+|IOfw&CootCu+9a9fA5P?#M*c$<)2DjT z&0z_Y5jZR z|L||CebRneUwXBXAh#{sGr%;SoXp(SpcorKbCT-*RmjWdvMZCjy#X{eDX_w@s!Bx+ zw*R$hob9rh1ZJRn)vr9%dIWY;%RVV1pre*OIT8q z^XuvqB;Ib0)ZypXM?B(6ZO7kDjPL6&7bhjFT>V@7ewxvW?QlgA; z|H1K}Jb`gg2qjr&gB?leNvS{khZrnGSn~ z)5^;5^zaCoJDt(u0hy*N|9JkqWn;b#c0y_B@Hm)Z7dpihU^D~cYuNG4_(@(hkl3nU zxfFBKCUfgwMnbEUeRE!Ese4yA6i`}W_SpLDfi}uc#pZicE35jcq3#qaofD`3FuwNV1HNQ+lmSS%u7CTM#h)2|7$!hQy$tZ7I9uCC q5S9*K?e)Ch|7p^|Vl?329Ko;0Iuq}qmzKZh>)$cHU9SD;<^KQ)2Dd=~ literal 0 HcmV?d00001 diff --git a/doc/assets/crowdb-s3.png b/doc/assets/crowdb-s3.png new file mode 100644 index 0000000000000000000000000000000000000000..43d36df1b3cc8ca18cc208ccd0f0367cc7a27c2b GIT binary patch literal 148149 zcmZ^~byQrzvn~oDfsha&SO#|pK?e(iCJ@{bEVvC8+$C6m1b3Igg1fsD7<_P-2|7q% z@PWY|=e>8%d+Yx0-T(C3y{dQDu3cUA)z`hlKC8&SdO`650|Vogyxb=>42n2%qW7ex12 znNCVwqbLvc4+Db^L;ll84Nu6w4)69XO)tcqLCB5QreGHdjN=5h$?R*ZiCAxZFAWNkc3YeIrr0<-?pA)Ij({DAS zRB99c-HD=v+x_+DeUYzNVn5Ky&7;oN|G-?Dv)$k*-TBr^xLbu=#<<_H#8z0U&h;Q; z`BHU2MvM;#drRV=`$t1^A|8t@137}jOe*{}f|@+kEk|9Gi6|Fe8^HsO2P;F3BN?}U z2#`W`OkZe&d7wm+v`8^fAkerY?`VE2(GeEs0@dz%@RMM!CflT9o7sxxNgAM#Qouz} z&zTJ-2lRY2jT8dctYMO0e|OR3!&ZE<-zuvT)(U6EWRsB#!@DXCs!3~w4|o)QROwq7 zpuqVw7>oH5U}Q45^*k&T7z}sTb>d{7DdjN*a*EoZi-b%qfh;6@^iBflX(M#e$-l9h(B!QDCp9~g$NDWuA6A`K}` zp<1LM%NEujE56mR55xHor_NqssTZe|&S^pCRc6(&VXwH78v*SQzTns=bMDnaUPAYtD|>Pe(3{5xKzQdUT0DTJ9SbQW zW{^SOp;)5K0DQKO!#YdkLG{n16yj{VEh2z_%fhL!+-%X$x0n!CB3Q1$7J|TJqLRX3w+m$(zG}d>#M34Z$%PMiao-ul5RgZl2Yrh7GYufw zPXC&8nfFs5`4iOiBnfP6{G)JMw+h9@yifSt4TA`1!NLMPAyaEK%UO2n`+3eDKl+M( zgI!e#Hz=}%gmF#|ULy6A#4M|O*G+Kgff(kkrnCh)Sxayxqtx|MQu>+d2-|IE_MOsU z6{Ke_k8ZQA&`0LaWA@$avd{A{y+h)HJ}3bY-P#;7OSD+|N-}7p7nsP*b6-R??BMRH z4aZ%WQ~CTnU+i}94q%8y-3`0gTdbiW%kMcLXL^q?FA@ljx>aT=Dt8BNve&RrJy6}P z{fsoQ2?Lr|1T2ppEgK*FQ^W77u|sXWne?b zKp3H9pNDWN1lP?7&k)u}a;%ACq}fN*j%`DP6uSnlStO+pMp&N>eh&I_t#F%Istt+fKg}18SEtM1@D-* zS!Xd3?-;_Uu%KF=?Ai1s*|99ul|g3)YUSUsZC*5>`KP!IJX#kDcmkfCXqv{dTXLk) z`I1NtlRI!~%kJZ-sQ?^?*i55vBCdpnjFlLEQn1tDBpt*?t$8*`Mcr0U;KK{w@+uydkJtzA)Kv1{c1bDp z0UTjA5}%+h_O@J89xhN%9O0t^RWs9M5Iu7XHtZ{J%ywkW#z8(wf0ZENJ_SO=6WPyA zKXNzW7X(w)@}*Ggbrar7lm{ADM9vnD!Gyd2zAM`t;N|C!N@TceRgFwrK!T$%kwaf9 zy*E8Zu2B0$LwAnC(fuqs&M$(~EUAg^fD;I|e z`e0D+*;J{W$?~(nJ~J6;4y2^-nc)s+OIPfEs1rV7%a4M!({(zg!fi&xV?&s9+7d=L zQ#Q0->T|w%f3wG50T70-Z0+jeKx9H zO;{zK;z9$)o1;8ot+RR!4dZqww;}_vmN&&Rh2Rd!FYv)J~)_4+T{?pg~Rl8Y#|pah+U>m$QJYiiq*+eO$;i0QerSm57l-3 zfP8+JPD0$d9H;$WUI{YO)!QsclOXg)Ec%Kc#Qg5iD3^?yv%mOv(>H|c0afn&8>5$u z!bZvWG~gNS2~0w%R{*t%g(cAJ1he5v4LHPFjjRmV=q4i|z>T)h$c9TsW7|a4)6ZaN zXi7)QZ+hHuvS*L59fVP+k&MlOWQ%RWhBD8A2UO5-2Tqk&6v*L*F}9#OB1`_eKJ9%! zqg%__`%i5RTYwJ@x|1hVD)1EP3!#=OP(Ya|%R#E(?+hrweB~#T`+)bX z`|rL=P`p%ky3*)}$kgbsc2s*0+NRXfH-C!B^6uTWIN2H!yKo?UzKt9%gR>)Srt&Q^Ok=&*Gy9Z68t#q)+e zp1X+hTYG~9y*DY~I-;$aL)4Jc0(q9{@KnT6NZq23O#98-<=pFv-g!n#eV5rL9 zjN90_X1ap6z|f^0tsqsR2apYOS3v)F#oY_Xr1?~KlguOlvrX4XFqQeOKx0Fxa4D9m zbypbFzAW3J<9xkUMmQCk4*QsQ{Y^Ao9b;1?q0%X7VW9O0e6eOe>q})3ZeMWHNi3!? zn#hbj6krx~>rO-&u%p)Ql3pbz9Kt-Aq!D6e+NYH=am`d%&f89EC)IFW4u~OtprrP#Ma-b;_+su%fv{ZQSK`imoMz#|Ar3dfF*wp&It-x7G$O) z6+m(7h0}+PH5TBiFAwwuax*OznIh%PGr6-6a~fW|a*n%slUVU@N_k*Lkm#&shk6GE z$iDbS8(fpb`fQQ|uU^;=N+>Dj1y}3!Xf-1Oe)6K0t4a103FEeaEGcxbut^Vr1t?yf zw#WowPU1?ge)V06m+aG|G18p56W1OZn%!U4j+&&DsK6IcK?JgJ&$}X_RfC|WPT2_v z8x+B?igp7g--x&Wbu~P(PdPEhlax1y48$ zq{R&D_)sF)CsV=JH*r82m$CpzUx<~RVFLIIZNU)tch>94NUKA6moeCq;QX(BgtE~H zI!#8=#>aujE?YTD{8c(CRAUz7&W{#+nh%qRLkQe;)yQiN zbX+YuXLq2MDp#Ov@kJWmgiKxb9n+xbV^)5)K~gm1egvUIu*u6L0ENQ} zwS?wONt8L{V}FD$?*9O=e+;g{-s}i(!L-bGgnG$74}j!_6Lt#JZdfV_PaPszTe4+- z9j^TBW3%Cf33k@446vLOVzsAv_Jx8V$iEPhCBVV+8v7t$u+^*jkC7d(1-uo2KLV(L zu#poEXM@_OV3tXW&j zmdo?o^k0uLoU6F$n*GKDC^w22nUoKF)zzL};rCwxzX6UF4vZ6U8wiWNnM zBkza>R8;G~QHUsH3bUyA#;OHORJjSDka0spUb8->P?m&H$V244)nWOm%ed24z0&^z z9HnQPk1WBqAvf=syi!6_=sYlsqiby zOlc`UdD#=WNFl!KP7h7?ocdqvXotPc`rjmre?_PZ$8|s0vB$d!6Qrn8FG10`U6}E^!H+ji6yAM+2(Qy+t zC9a^K!-Rh$`noZiDhjGpgQYSNyQ<*&q(B0(*X0vx<+^Hk{erPW0063-(s$Uw!2l|% zM0ZAMmQ9hyI`9Dg_YNB3O%ZuEB1t0FXHLdU-Gn0ZN{w?W!eIH#y_L|oFGPJ~Ld4#s z9}Ab_chDfCum#i1h3(ckmssQ82_{=z>R%)Y;a?cy0SCY}(^`^L9mMhv^`}bTQqFi0 zLhSe!TCBS9`mvOVDS*CafJ?eQ5sr$)49iIcQqIKqAX2_|ye0ZP4pRSv5uE;sLe`UA za6A80%myP(YPsA=E>$_9aSb=A+c4luw6XQ{fzt%Ot*IFT}bEwkl)2hTIp{&3#)1VKAxQa zD}l!8emp!vC^OJF-Bj7GE7Vx+vq|k$opwY;>tVC?r-VYdm(e#{8`nq*HU_^@FJI~Cuz{zP&#nr^AXth zw2X!a{(iA$C?aGyY><%wm9LsO12220+O$k6QiN%$^7^1+$ws-D{Pplq!ZRC?{044^ zu+C--35)-T@shD#pFQrH9wsiQ%4yOI2P>e7Y5SrF52oG?m!?$E38#mo&72EoR)+RRRk8I{(iqi8v#`9( zg$hf}NMhCWM9HvxSDxy}V-utZXVS}fLXvk@^pVVg|JMoIB+O1btfV&XleBuMJqAB+ z_H5BN(!laCCydMxmS1<}LqQg~v1l9^mKL!}`X~7>2yYhfru(AuRIkK0+0AJa1HIx` z6h`sVy+8(;ubKsSJ=V1Xp|rU#gO;|86@CYKu{p}6e}1vgsKY*FwORkX6F)xaYqs;$ zd&h-Bfc#5^jI98s8lQuVp_wxD6E$rkO~Z(!zW1aLpknca{x<@<+C$C5Bc9)gXyl%B zTX)&SN7})=t!qBac&-xGk*Mo%0}Win(gU9ufeZ-gBNqDLQ=0?DRq}CxsHLp8!nxnb z9@5Hny$koXj3H3-_*naR^8IiHlW>-`35E%(0`0t$jxrV5eePNgXhL78H7&I4t378uQ4TTpM8(H`2Ie^N_SO2F(7KG*my7zc^vgb-C($RArVn6Bz1&%1lFVUP-oA1GaeE!bq{&xWvVWo4 z+RXNAf~%dMB0&+J=xuqI_#_G<-#~Jtht$pNoP6!v*3B9y;8~Sk4*kPsB`T>Uh8Y$U zkji9f7Ed#qJ|tbFT#3Q9g8PwdN}(+tc2L=%o+EuV#(=3zlIOl!+>!jWj7sFv+(c1* zVQ0oZ9KW#t+s6h$qlVVvt#geG!6-jl)`5a03KoZ!(FS36ii)-<9Ze&tGZThsN}cxd z%hqo$_GFscIw~A3B=gbqI#mztY~2IK-1vy`e?$=}G^#ZKR^ zj-qrfyamwtkX={d3ImVI8u^4JH*2C53*M|pCTL=plIa7JFM{#ob<1WetF;zh$xvinJa%EIlxi7+}EYt3a@e(Sinkc z$3*G+A+S4ZoAPMBAhcEb@;7I;sUnR{tNKMYPzO}ZeeGM#%2sa$6ka`N{e{NPZ*<>l z=t4bXfy`CT-!9yEtEu9WZ5}NIwF0^(gBreNPu~Wgqh9dzX-|{rs@hlo*pbl};i4R? zwe$Ce_xnjVXzIQzmENDCrD#h{KgoBOudM6o*pHRY)^9SY(8%8!93OL~Ks3I_>(9Yf zGZ-0SW}_PX@|2}Sp|ZicL3J2&TFEh;WQac0_^7~?<5@{za*`P@J;GvYA2r+LVBdXq zbu$;w@;#REHzRB0Yv7vMYl$)y9GgenRq{qNkb<2t319<>`yB;9&$z|-IRxHcJ!N|q zsWnVdu~gx$hPxA)TPiwT^h!-fahodB{*BF9K9<&ezBV7v=7-iyuDh^IyJc|f2C>ru z3;Px|wyPWSkd?HaGX9sU06bo&;q(It-Oomb8sOa8b)&VQ%(Z^FZ;7d3ItvUpsB-uK zRd7r{CtR%QIe*MM*C2GtW@TQZbYyj$=qAuSrlVCVXz~p&{_R}2GuZj2gDOag;jZ&( zfeggdX8SvJqZ2P@@Q2cyt+$#(&U>s;K3X^-spstiB`I;%oilHy(?28h zs2SWaeAvkb!L zwVB$?)1;wV-X9#A^<&nDR9GeWYOhQBKgQNjH21Il41cQw`xr!*X1Oh@8_-s1{_CSc z`txW2=hTqyt1MHiuVTYI3PQ^IQt_tIpPPt%!vg@;hm2pAeiM6wUou_is%S)4s8gYR zO;>R$J_u6u{Z25M>wJem*Fc41%EzM}=Re?te6c1t{EZ15dvUb>;kHVB;N47V- zi~MA#8fz?oT{tEbOPI6P`0M+CMQ8)WC^8*j--n?SXIB@lYi8;Y2tjZBJ-8y(V+;GW z;}N=K&k3D;S$r4=%u>(Sk*C*;(mDKWjg)A`dOqmM<5kO$PBKGYud7_-JETkMF4{-o zS`?>J)GIRZqY6Mlm2;{~S0G3HNF6g9J?ZC|GCA z*lM5Gu*~|_uP zz#k^8Jfi?qnNyhKaQPWdiT!%5w(>)My2^e`+2%Y6n4B{(0pmxxKqFgO&U3muX)y0% z4?_eYsnnA^cjn<2l`9&%ve&EMO^eFYrD6)6sQ_n8QD71gD)UjwzMcjxt#yH48o=}A z1G0-wuY6>_?u$0H-96E~o<`vu`L8tqy&T?qaEkueeETI7pENk zi!`dbr}C}VqkTE|GOJ|sfDZpYxoBT!{C)q6u)BPx1hUy+xg%x58&d~Kv5zYI<7R0C znMn*(Y7Ca_>Yl|;`_n%Gj)JPMqvO{QKczq2Uv?H>#knxd`;xV9{*Jyh! zNX~spC{{r0yp69dvcQwM=!2?}D*Yh7GsVvxLjEsF4af9*LUAsv*qjilLvrg)w5}Zg z-ll*EFS9{Sx|Fhl?@3JE)$AIGoYDI1Grc4=QQG)~Cc)TnuC85)A!}qg3DJXiG5q)a zcP)QRjSo|#tKU1mJm_!Wv1ttLPYZ#c-_g_HXntvrTGI8zZV3f%3a6&UfUzwL z^EC7jbnv!9TGGr_6HILRQ(i*Ak|S&ZNeuOvqFnC1yf`TKh*p1u(LRqb+7fYW@oqJ2 z!joJ4Tcj}U@cu%vpG;)Hee*(4>sMlQEuXQ1yw*-&U)+AxKr72RtLssMSEIU{+(%nS z8FxSHmn}iCyFwlLcYczq2XO-Kp>Ey$rsXUeQA?Bgrk?*$ZV0K@5<|ElOa{*38?2Yi zon`5ozd`CCMSOWXn?!h2?Mt;ez_jiRYjoe|h|V|a8LqtKwsT&27cZ?(Qe4PxDy5&7 zz()yWqSec|6Zn!?4AcyQD%+LsbWMp^f%Sap&`8{RM}|w*oXJ6cn#NG1X>3bTRw^O2 zCITaf>~;PG`nY6{TE&^+!h3_3iB3jRBHV8hnXoT|no2^?)w`HR$LZr_isV_@hyKW( zl?dygd5i*Zj-kqAvPq4DwV%NizZY{}M)kcRtHYYHjADG z3GE*_Z;Y38PEiez%heb-*(=$5{8h+h`IU8LnVU5e!ncnU=q%p<&Zf}L!U@g!H4fbG z#ocrH@->R0+dB8NwL8}%5t1>{8zoPhGkuZ}q2uhn@oM|O&jQ4+ilW_?8>!NprEf`X z@N(yE2FT@afyjL0fa10&>@XV&i5{Kn@;=?t{-3&?3F+clOfY+Xfe!3quB5#EzdCo8 z%E2M6s`#~zpWPQ;dLC}7@i3#+=Q7_N-2bHd!*Bj^qiCn~;pqeSvqd{tI9Pd8M4;6u zJPOW@mffUe?qCg8O5F{k!Le%#bLP@|S>6Be^`V2!a1Qtutwoe=^d?q^0YbK%z4GN7I$0!Iql^OKF#dKRp9$lUG8w1qCwW;72`tgZ@5m(M@3 z=lpFA{WaN)!>J@|r+1K0SS*cfIjqBGGqZAkoA`{NRKCPW8O*L`eb^3c8DdlDs+5); z)Y(agr2jT2Hxg9rN}%3V6N%2)C}zo>YoFg;1kIf`?q3>uELfdWu|+F<=VZNecLmDj z84W)XS~}xbR?1e(_!E2mp^8=h?0sni(Ga~g{vSdqf)FmjCt)Ae=n8a~Ih=k^iz>UU z$y`v!FNBd;JO3WzG_QYEqm@Z-4S(Ny{2`&AT<294@N4N!I(J%RQMzA%yxPw!>-6@h ze4_?2-SX^&9Uz6Wk#D3R{;UwBktcUAqww%yj=z7q^v(OW(O4kc-Vtd@!+SPXa*wpR zCi(rkK{qA4E^GHg9vyH4(`MD=1BD20iBkzaAP6RJF=eP|5oYnx@ZY@Y!w1Adqyri4 zzP2?Kqtst!bun3sb?`7XE|7Qz$0#RaeV?Jc87ZS5G6kR>B zP6^nCE>PNST z26Vr+LJC6lkxg8ZDNibf7${$lyH3{WEAKV6>gCL?j!xWS?7uhTHT1~0exbi)s+d2v z_LW95XOkmHv%e6BHvhQSaw|^u(u4TLuU?lyXc7vEWFdL+OEUNK+y83T$%)Vc8Cqhk zT^WzO#{cxc0r=S-2ueQ3b6Dt}1N=z4X;c26A^-2l|9`{%uH7T+o78mAhpzc$_#xZ> z_n`lt=YOODYu$Iixf0ZT&jlf2S)tb&eTvSg1Zs{YjZIj2+Y%P)z4%bk_H56{|Mzz9 zeF-6aLjTnLh|1E_>?u;L&XRgC&IQ3cHC}$ucDLFmX_5i}@4CNh&9wG^Z=SsY>2JZu zGT6Pa`F@uULH+y2?C0nhJ3ym1*7qr9J#HP*T=MI#=Z7)}&2Xf{a_?P?J;$&A)67+G z>7NSPFO3fFl97!{gck4-n2o_|Gi26({ql(J3E-N$e<3!oMLuz;p|Qp%KFQd3RiBLl zvNlHBn)vuYY7UnJzK#2`zZ|EW);Ap@EH~=PT91pnTYcOD|KHlRn&3J8@xszT8lQMi z+)J`rYfDpE4rz*;g#I(csSUSCe1BaLpBW?9JYX|yu8y%gJ?`*!WSrJ?Ecnq~O2*kT zOHCdB8jVkjlnW?PQW*RER_&_HKCScf^i2IW9gGo_y>NY7Uwz!pV_YBvK3p1DsC7HI zw^#Y!EAx3_052wz5NtN|TguZ)_MOPLe>i00*&})?V$q9@d39&1wrHYG zm!B^i?7uzge_2O{eYT6OqzI}A1oz*7!<#mNhgKnE;--+@;*+B;q+#RF&??b2Z-@&9 z01HO$?EZ5Bu^vLxM=^kexQV_O=N%<|zt)*OZVac75m|#f%N}H&HrAH4=(^TV@Dl;{;@{_A64+Ux&Qm@A&q)E+H87LWbyn; zMX-w2a=JAW&jDb07VIP_)|?b~Fh2=Q`^L8e^gVh85IGjj>gfbK=0pH3QjlH|CIivb z8kzX3FpPhmS$e3dM00j6$4OZU{O$tUJE`FJ<#5XZn(DB{%)UqOU$GqE_(v;#evdj` z@oYbbA9FP0om&Z&W54*sg<@=5cIQU%PmZwS# zE;N!;@q3VI76tJo8}qK2>kH;HiAd#BZ6luoWz-amEo#660NL@(9!ewMC-A!=yK%3K zY{k~h7d9RSiQQA`G zRFgVz^59|0`##ItPlXzb4`1j9$OX}}yC8l&S%|SNVRZgjTD4pLq2y{!EnttfTns10 zt1=qL@>m@@MqgA+UL}llqT3r?9{YW+;#JSKA5PdX#C?Xck$jBUJWUCUkD-kIX9s#M zE`%1pTPjAhR(0vWa*cOi{U5JsZ~urqntO`uw$sSVS`zdy)aq~$-&|L>VBjh_%Ly4xE-PQ9HQ^7YW^p6h3K;aEnR z_ur4AjWP2AhPO2_E_<>Py~pTRkD5xrEnCKXhalHy*(q=x=i`sPo83MTJm2;Uyjc9Lt5Wo*LH1B#jHK*`ck{q^x243 z`CnD8n|F^=Y&#s<$L&pfQ!#!&lRVaNyRkghhNNk4gdBcY+6{7~EgJmk$kV$-w>jKI zHDMgLaNoXS%*#`~xqhGJ&&8T~>@0qE&0TA<$xRvIf79u~CoT?4;eKfAupTI>68*LJ zn8z*};P5geX%@CWMg;sqNBm0UtYsZ;)UEm6&UfQRN)dF)=ns8{$m|PjF@VyxW!(5* z&HLPNcMYT2R!xqtJRC1?)<)#3^kEQkp8JK>6_*Rkt4l;@&vDj_hnRr~MJrCt-*S)R z(mJC*uSb%BO(=uT^ITO&|F%V?l%B60G4H!>Bn|yGoUxlKw(cprdfc*a@K9AphdTpp zqoOyA?RRj)ZpCC8C>^Da|3J+3;=II|+gFJX9cAf-Vs=PFe8?h3M8(nDBvyinu9_ zt-&8|`<5(r|JL}pH5bozwo&bKwdkyt%z?X;+sm#S{SPF|MooFkL5~dMCh=c540WXHJY_Uuf$XPjhbNXM5^*OMHcWfYmr+l z>V2jkl`Lm!Ki%?THTv~Y=QWPLKWgWV3Q+Ms^O0Ye3rsYq>jv7i)FpBAOU%evjJNey z3M_kklQ^x<-kpB|%g_tfqW|diO%H}%h)g>_7tcS~;x1!urzrHrjVlmc^@R57VmfW1 zB?(kVz*M9pm`2E_cjN_u&3)QcB@DsC8lYAZB!y~Yx6)YB{gmsX4P7*jGIy*VSx7cn zyF<^P${O85@5LJ|KW(qt!|P--LnU$|1eST2%H;tJP^J4!8D8R^$;>@@sjTn1(t!O~ zd{0i>b{gn=ej$bwiuI@h9aLbo?)z-vZq?X9xD~ z@P>jcvwHsN)L{sn{Mfk_m+T_=!#1aG7VbzgO^a%@dTMX^Hq`_2 zJ~Cp>iy_Eln@fDcyr=8>@nlrSh&HN0^|-*{7BBe$+X20*dfJoAsgIWZT3vxNstqgg zKc)rw4Sc=6WR*}B+Y~Z%hCOTF_supBSne)Jx&3)kwEtao8Q;7|2ZFLpIo;VDXn81g zaNc~l$0L9{^Bok4;#wX5li2v<@5A!P|4g3iE-zydgk(~=UzOMl@&{#ip}S`!qgmC) zgfQmre%e&FCtldF0)$U|^{?PumiDNo>*4pbo)(9`*oYR?A(Hp3{9#Z8?oSjXY5>f% zoD4%a^~j%Tdrx~@&vBNm$C5%WThBbdS_Y2aO?O1@`kua2U2ozRPhr;6I<2$ugTk-~ z5I^TTGvT1!HlOw7rkWa*K9$1@ci|8!-=};b{ylCstlIpiZ=39JHR=c~uESVe?yKf) zZ{ZJvtILL;dBOyTa_%j*iu09e6 z=XjZBJ-DsP@=tR2M7+8XvNs}ilY0|szk18AeWmsv5Qy6^T{gq#@}>XS<5v~BaVv7! zS`R_aa#83t*?|kQdy=bf`mU#)P6otb$SmI-E_mB{9__b>EI$99m+rDrh`>K@#t<`L z0`S2u4^-Dw3GbEr)sQl(4qX^>y=f#75!O2JFMtc`84ncY5gxWg-agCRi)Kt21+ND# z;&~}XzAmR1`om5(4j$?`)48|`zVi+DR^Gm|hiCY0(Yv-D)YCDlkbT?rPaa&@RhG3r_{5-fZyz033xA3Kh*Z2cU-jK|%H_E6h89WZ} zpHrz*+1N7je|7tp*k_Ns^IboHz$d-yfCB;UdJvE9_@`^%UWEhpa5?OfNIDp3+P5eE z>Nr3;&vSe2a8jsl_-CajyV~M?o7KNBA{fER&Qq*h+hg1YV@(oQ_LZ&QHo5%^y#d@H zWYnndd4$j2DZ91=c6mqO!NU%d3fdpueKJC@aevWvMI#p6b&Y$!pP_Epy8gw2=7Cd1 zV$;mR@Z<;NX00BT20`#y`MZsZc;3ju2j?7of-fvPf_jRq`|A54j=isc-gJ)4S9L0f zvkK~u3wFg4WV*c=anK*SIC!#vnVjXxtLXh-NW(XxO%}_M-NV60gjN`-q0(|^*m?H1 zuTx@8qmXv^w198Hb7XbJZA6$QyV9MTz+NJj*+GBscMH}^haD~Q&n0_wwhI$0)w<5b z9rAe0KTX#PILn4>+kWw))AJxPg@(!;x`A>W28QsS!N%-Nl>alr8v^*f+E|c+yu!l* z@Q2*aM)R0c4~*p|q%j&oZtF$k`gi@+LH0Z=**^!^UBf1>TXs(_y)$epv80uhJIxED zDC=0wg5Xu`DBXOye!s+;8+V8jrCI++m*#_`2GM%2)OS%QHUnJic3r;s)0TKdQoq$T z&(R`6H(T32_q6^EFZjzFge)JSH-wpb<*)FfK+po2WxX^7lb}R>_`Tqe(x_Udkx>h`MJ;w11+9CPdaZUF7IChgJ@(}ps0Nj)+|R+jO@GH z6pX70oVB`vHmma+ffWBc+N;0bYyFQXw3oAK^&~M{Vn6f`=un`j)^n0UO+~cv(y8)h zJ;u}ZBAwOe$b5imjQQa(X4(H?k2!@`l=oC|K(FRj8|Haf2C`Zc-EZBg8K6bz!8-%T z{`|q9a^E$D32*Y_>$Rs+i+G&=l_GE%L-`Lj)(pK1OyLj}++pLEJ@Z`V|)`jkVjtmh%Ek!sR zt^9E#UjlWEC90J@6fa$OOi&q=>+9VvZ#2PobDZL&;X(breC0fUl}eP)Vl1b006;nI z@edtXQBj(?hvi+xh6I!_wD1=y18*V8&$gOOJR z4%yD->Y%&r5h2@LafbY+%m#LcZOby_N#KZ(#fhXvGp-Dccj~gAGw1x8h|cpzo@yWi zCKk1cpD}>fl=-UcF-6iEjK3n--_*Y4F!Wo;Y5w~o`?RI&?zPPUTUV1YOUfOgj1 zU|hMZ5xHtlHm9!hmn{p?sHm2sQt&SCKj;m^w^UwN>k!Z0lhG`J2*w!g-of3sd3ky* z*K9pTHN$c=MV;^q7{u5+YqUXo&XbcgZ zAeC()A2jBzM3qpSW{iCwP195Og&AGJV0a$cA#!@zrhM^^Sh$e(J6#?A%59$kUzSFG z9E-JkWtRSr@9XEA+SOu{;KW4i67`f1AwO9WqZy@VSK_J$IY196p~H1UiqjcZjMIf3 zyUu%IXk-r;)@&VsWSr0jPo36z0|y7&DGAY?HVwL+kTUT^dL&#OGiPIGH7AYTo52E~ z-CUf!Q{72IK-!iM{j)|BXgzA1TqAS4-ssgRt%`x)dDlfi_XT23qWsbLZ|>0<+8sN< zM_%fJ?+3nN^AA+VLFS4JlKTpQGzfBu&P(CkzC*T#Dn6XLsJ`vKy7uk*tC(W&=)22% zuAT=OYu%-@W!F#%-w?y9fyt5N8n+D5m+P_8IA;MKMx(x{sx23zTOt?4==zw0P>k2v z&S?v`9B|kqglnhcO;?aCjlmc}y{vVNQ~j5; z%4E0Zpn3Ch^>G6{EVqE_CQf3rT#DvzNOnu%qv6Jv+g@I7d3(>^bMcMeJV$VQ|6un= z(gKb$8%CtG{C@OQ<~@-M_+bfM(ZO4CixklE_Yo1PvvjojVErw42k0+j{0{++R81rA zg+)b1dhgIO!D-yXoNalYB_g3J9T73~eb5DdG9-;b7!_jnkHiy`IZE2j2@ykpHsg z?eUR7q?o~~^#5H=Mzw#xWU;@wA7<+p{7>d~!3g5ALw?Ax8`4>QAe$N)8GYJu;Ma|+ zyuqxq9D8DhdAV36;@k9$;*n`aun{3t`)f(r|7%tFE%_YGjCaI3*NqFvQi;A%HKK8d zXPDE4vZD39$FVpved$u*7laaCS6@2M)|$@J|KM*A5Uh;jba{)}$O;-ek1ZLFWyVyP z9PV6;TO1FJV5YYv#>F<0gm~O6((jR1s5%k}&akIS@j!@5h{wlB9M%J0uY9@gPbur$l#hOP%FO%tW@+6w4# zSw|s_9>8;=jbwju`{^D0?yeuH;QB7qRqKV4lG1SxsxtyqXUQkZ+uLH<5&akuI#k1% z-J@g9{{G>0TJv}q7PJ5JGVqm$)3^aK#1(oYbg`AuatV!6<8j;w;{XI!w0VmzPN=D% z7qU-FVX9Zd=JUMww#1#QLE7!E;z>a%z8l#f|7&N9+BgA^+tiG%`)`$9`ogvfqcR2QNA1c&3UQK7iA&fD?!YrqfLmg@xMb}J>S zx)_{Y{pQffl??F!D_ius@NT{ip!sJl6-{l^yzk;asyWtmx4DASHol%Lay)%q)^gu- z_*A6(mN|sCEwIKS>pTdLqIe*?{y-(wxE*KqFGkX@nu?oCPvx|nY&W41f}jDvV?svt za4^)P<3r{Jojtxjq~~T#I}cf!SuMVNPx-gu&nlSP{W_EUuc_SJvv;e zCEC_mj+U_A$M~?j!cQ}!#}lHvH;(o%NghfTrFq2V!h-(*e?9zM$VUjWV7|+Z#Dc41@44ZlqXu@7ujy@#SYw!R51QAJ zUDysjFCEZh1ddAw^G+z`_l}3-N9KnK?A+LIT-I(Rx^-S+%Cy{N-xJJ>H>ZfW|0g}Z z4q8uMc0l}_&%Cd1n|c2j2O3{iXRMxhNIwrK^4#xJcYY+MVUN7@;53;*`~7c^!%2jp z?PwY(GgG7~0GR(&Drx<4EX&VBIKslc8T`j$?JRtoO2h^`#puEJ2wt76vLNqo%AO=| zTRG5jJC|MiNZH&%oYpoCm`u_@zE;@{XR4j2n!)o`{=Monm*))~>m}BB_!6FwpfU=@ z#NT)c+YXHVZlvf2H<) zE0L-edeSt%+By&;6kyaT?q_RD31mXMjX__Hb9@A~-=}?@Y5(lm*|e6%=Z{$>ilF-2 z!mi;Z94&h57%gf#bOe_$#bxc-*lw$6_vVFP4Iqzkx={3n{zr>Gs9_`VY;#%ap{AmL>Hgl9hQ{PBII8`yx5;@MRk@wzjkJODL94f_`)|)?Rae$5wu1;J z4=zAHL%ENi$9ZdA(S5Pdw1Z;O3-Z%gjC z{7m0%QwyW?8LRW{L9#mhf?&l_Hl}4@JU@NT`~;|2V&7r@fARGdKyfu$+dv3}Kp;2- z4H|ra1Pu^^2Z!M9?(PuW-GWRKG`KqqFt`TSf#B|L!++o1+E@15`ft?~J=C4PJ*SU6 z=jqcuSIu?^QLs*xBc2qJSe_VaU9Z<$k8tjrhi-~hw#ThnW%85L#VRv^1KQfZaAf(5J+~Kp42$lTSNSv?lURP z8`Rr)y26s!jRb4VJZ@4>f1GvkUTLncR6Gg=d%Tet4y_exO0AVY!J`*CQ-~2HnucEo z(EHl<@*OtWB_+yh_ZV;<8XBuLx8}C@^{{uMRi_DTThyf$*4rA>WVc5VKXI?A_|BiY zQSSY&fE=>_Xm->FEiM{wi}f276lkTV@y84N@e#p-OF>vc;q(5RLe3bz-65VjH}S1> zGtvS9&S{3-H>&%s6E@F~zPm|&q|d!tUq8Q!{Xi4SF} zyJO)#*&j;PDzROL*y;Jgt1~<{Z&Lf_#@xH3jZ`wh%Z@p$hUY=yEF;x*`LbGR7*C)e z5HTGUDN1Dh&9%1=LMn9%Ohv4V_rUmYemGPdpXBN=uVPwOSA`ha5-qLW*niIYB{p&R zvL)2C66fvRGvp`3syXJ7fzp#*1-X-IedY&smq}yA7zJ~&-FRI*3QwTk-x5M@+Up(` zsCjrg-p&)trYQzuW9VbvRwB2$f3dwMmHY)t@B0Z`*yn|kacPsY!9Scy;pu`Y{3!d% zzHqn_CXF$@4jE@8oU+A2X#3zF-F!wo%k)#HXO&V~R*W`)a-Z!DrMbLW&(7TsVRwlH4POpcXuMX#BhC* zs^~>q?nn61SrqS{jBUduBk)0uO3N4MZMPwy0$eDl;rd6ow)a>U^=0%kEc*A%7}ax& z<}wGb-!qqV6E)!+U0&Vj&)<2!^g?2IGU1S7su+KwcIZc$4x2N1K@ytBxn4hN4|VVJ zzIHYBn`@h*XH;!z|1z^ARI%Q7XFM_zqy{!#e@cCAdV{3B=^S!@Y9Ou(=}UBjCkx~u zxSGoQBsBZo(WLHY51_Jie^&m`9rCnft|F}ED|h4Lh!>BQFCF@HQoWVlD-%1Dd$cXk z<+$#?$HQZ#i{1i`X^>L$%_bYN~%~;Wrso(}r`OQmo>N&OTPE>VoR&C!fNAi@< z`VPNq0%uerxqq?uLbLnY1AbOmAGBI17o(f<|V-X^V<0wN@vmXup}`|r(46t3pK3#ZRbH7CeD(@ zow{=+@WH{?xB3hsy)|qyeom79w)OBzVFgQV*SL$)c!#nWN!sbPCw&oXMA>YKO z%~K3gt)br_1ju%7v_r^cVM{G5$rQ; zRv%$F^r#7(G93ubPNhjHVpH{CJlsHNA#Ysf3iMTt0hZ>;(_!Swmk5_Pmv2erlLw=r zmL0~Hf^{NO?kzkeF4E7V1WMTovf@5Az&_;-idd+~qI+>49TPF@rbc$<8t-G0;HOd6 z+XU)mf_uk3&Jf&5P<`3m{d#-Zq5o2uX=dSPsSBswiduqJ;NY}8dps$MJN<!5kch+AFF^1v!-3gM|LeeHiNZSka2x>T3xw{vp6^+{xlJLB|E^<9e zqn0_GIYdyiKipFgoSjo(^7q&goKE7)pUqhK>B}2UQDPpeq*gIC<7EmE@i%g>vl%)0 z!g5>6^W*M6p;Ds7m*0)m3VmxuG za{U}jniSCUwfdL%8|_H4^cJM%TTwhMSx?Q3X|APz^7a&AkK{VbMLAUN!seJ4mFm}} zq^4a@twP>4A<1f~d&aI1)<@BohRa~7U2fg)O_|`m9Y@tW;Ue&BO);3s8fc%COd;e- zQQyj;@e{)D(O3Gh@KuiLuZxVnsnq1NBhR6iF@GzyrY|E)Ye!m2b{9c3&K7PM z3uc?yP*#r%t9+pCsuuCo{&~(Y^(tnMnXb@q?BCeo7yJsq9HZpE#W(5qQviwNGvI%R-`Nl*sv2^W}Zq&S&K{ODC7`gmERo z2SZG)I+L+tvsWGZaRB%0q;wlDdx(pHGMCy67B0vhH_lj1z!_Ka(&x%pz#HvYtH%>b z#ZMWDy8|2S#&M`ZZo~tT5wB?le=iMRzH6?$BoU4c5=o%_AxHTd zW#H5O|jiVJuliLkXKM0z&QR#{JBN&7XR&(*f#nH?$bC<|%W zy)Ij3IDdpPyNFTqxG}x3Cngy)|Pel?`w;fm965`*@X&_q!a&F^} zIji}opw7RLp}+a1W#Ep5f+Z8wLblI8?IM7wI>R$Cw6=NueCDi!#O`DuBl^DOlE996 zR$R(yg#2N^T(yr`GwI-t{VHEbZ+mH04V!I& z3R28;hapn8hsRLEW5DeJY9c4!k0Oc9O)_Lz0wrAPor5=Yc5Vc9^Y5vsGOb}6ee;X& z+vXcnm6b?ox<%LQL)fd_IDFQq96VY?LwK*xj*eCX`0ctZ>-1|iK^V9M8Nta~uJv9m zWGugAsD4SDL8UlT6HJrhER*6)S!t)VWxnR&`2|P1L88m;ynLbp)1C>5H=sf_BByjk z(rQDVN#nh;u2#Y8oi!oN|%lhI{XE#@Zv3e zg`v1CPwk`3!-?kF@kynqxY%#|d3z75H*8FZnre;T*ijQn0Ryg#QyqUZGHNuP<7am3 zg9X|fgK&E&1M#2((C;Oa6jKW<6~VBWX9W$pJJ#P;pdoLYTK$&Wh7Tv^8DHTiClBcy z(<-yIHgsIt5wfl1la#H^GZj=BOqj|52gh)tj8W63%L{^amTj+=S}`cNg5sf}ipt8| zXIcvClEMaxfos9Ryu{AC3;;SqtY46|jK%Z3kTHUVL?bX??^^k~o?);y8{^D{=DvCr zqPT3@U$vp_weh*Ean+H_4yFq3_^hP)=@9-tg?ZTcJPdo$8S28US6L+JRaA>FsEI3* zY7k53`}>*a3*L%u?qRLNV7ZS84Ac>FZImD^Y%@Caj;d zyGoi5O@}Y1=>!}4GtQ7m@&)_fVIA~voC#d2qjIhmKn&iXyYD^7v8z%{9GvOplh?Kol@K~v~A`K0UOGsU0982m1^L?tK#Jf zV`-99Zr*v{IxhJ1^P}#Dy|T5%J~Hp)`Op&9^MGz)B}=<;&x^IBC7d+io%nB?o>dI6UaznU#zP(2V5?#BRwL|LzZOsADtTG7vf3 z=Z?`TBMUa#Giy0MX`)ELUJNeLP3bNy9$CM&z--fPjK;yz?@GY&l}w2&j=Ny0ce>oz z;yB=7TDG2#jg-8_5J;@-PiB6+{qP0Kju(4OaHy{~yo15-_x@=Q2N&T-?NMFPItfGq zHHPN8nwH$p>;CAC@O$J>btdt^mdW}q8Feg_*pc?8E&h~)A|z?%z!4(R&nja9jI;!+ zp5V^Dm4{>7n<$mL+LrLY@*g!ELXw4V^FEJlKH-s(KlPCEYD%JJXSY?!m;mcLb-m8k z^>+Le>D2ZeehTlPi3Sn{kVj86g0)t29dy$la&7z6P1gm{wmz0&E1HsFQBF~cJ(4T6 z+HMAOe~F{&%r&q+#l9e`i-?JK=~Wv;Azk@+@SO4L1^1HS-llA(AakIvk6b$*hM`*s z=hod68IF_0b7Z9Jhk&L$P0X_%?*9G0TFy76U(!HWGgwu>7Kn}I%c74`=r{xxDfJt8 ze{!{NPZG(z#xG^!*uf3+RHnOpFa)Kv$v zrxOQ;FSOir6@$dpWG{N=3| ztci5G?Q>>PDBoIse7PXuo}!lk9*a^vH{@Y7V$iiCa^l{ChevQa>?Yj@14}46u-mnq zkpnI$jiekxP$<1LTk9oX-1FldyYmZIC0qA!emi1+H({0)zH{FwaWh|-4w;2FX4W+{#RF&-$X4VKB>UCV_6#|-?ie(|=a9P!w+b#&qC zA!Mi|5bfp0daDQb<2#`te27k8aBM)mImv=r%ZQacM0E2j#j5Q~33V!lg2yDQ5&5D! zn9hf#O&9Bj(cvLqCq#US%Am=yi|UJ$`jn~Fb_;F_^;bO~=ZOJOlqPB!;nWX1dr|J) zkZ=(#(|32twiZ3p5=G%hFmXr<)O0FV*{~lC@zU`A$O4R*;==v*_SxUGXy^p z8gLPY{G5VzPsli*8ywk_HB769QS8^P(h_>%#dLzQJC9!i4@(blYBavr&=x2>{Lr>) z!|d*bi9XG(h@bnyJ1^t<9M-7LM%KH*8REeEyNEF_SD!!Qit9n5Rp7=Fy|4UW@V65^ zXO(Y>`!5Y`IgRVE~RhzgpTBTu)$gZFHZGWp5v<$Am7!Ax~#f*(0QW|e)<^@dQd@2AvWF*`Y$ zk3)e#qzm~f?4YaL%tk$~ikkTLn=NY{HRKz(+Dzyy#m0F5w=YeX7^lhUjXuX4Pm|Z( zjr=jWP@%k$@hu;^mJ z-=>gs1N}$+{fQHok%a>rZ-Pr>ETCoHT>Ard=e4~;@Uo7hYBk&9$E%EGt}OnzWUa1c zq3gOSc6XBWa&8tlOrV33&eGCqPEE0Yc=^!Mc&3g72)(&G+#xM|V&0C=cZi8Eq|*_A z1TwMKRknb|io@eJ>eia2uvv;s(||^hH+bht=+|#LSNs07oUw_OUS(B#(T{4>zavzAWF*3b5CvHtn28MWz)k66w|4f{*_p)ro)@p%Ic z#Vjq){)2kTc%X7nGDNCNqq&^^y1>$Qy`FT-woSOTan@&5?$rBqE#{`-K~5m~fTVdm z^{ZBaUpcw=$)rewWf2;(Y?=(+z(CO(CMJ5m&W@sFE03EE4I;WHX8{BE0Ur=zm*wLS zEaM87l}u3>TJw7ejLKA0{IjU6 z!=%!2;i~cVbNTs5=(Ar!L!;9r`{KpseJ3Jluf`8VU;N$$>q^b1aC^8OVrqw~{d)PA zd~UEKBBS&CAc9>zT_7NPjjkhhxcz0S7uvK-OrHmql#5QEZ;zLjBaIBy`xPrMCz#Kc z0c>f)c}t#t;Tfp%{Obg^boL_X<~@1Sj*_=78(kt=^;{6IdFR@tVYTMXaiF~W5$x*Z z?g?@rA_i3Emoe3QzAs^g&kPNJ5Oci^>VHhxFUrCy`S(E!Fi^xSr#Ji{xyzwxie!vJ{W^ z`TDgBBFTedx+eILQNgNsQa{hx`!uV;*=Bh(ZmMw?QOu?&J(-=3q8UBIkr5*;CF#FR zSoGCydP>AHHq=-CPGvMM|G{z|?=X!IGqyJn4fOOL`3ZvX7ew_o@27LTbF<8*PvYGxU$f+J%o^&*IT`n~oNm|F z65;IdpReD&q}i_lDUF~Ndv(~oJb#7kV|q*ssN?bh8pueOcq5xSk*uj`_p7G2uU}4i zx^AYcO2zy^$9_d(qFTGrT?{xz>Woy}AHGsyx2!ubFepM6^N%q*`=@Q2kB@sMD9bjp z*-)|UEf=G@$u|c@ieq)E?fsPQdxmm+_r#y#i!YKeD4kh--*vNeb_lYREjzOG@Ex-i zHdmsWiK$fxeMYQSiW)n!u)arNhoDU^wE2xNatgC>-P=u(1T?FG`%PX`e#o?%(c+(v z=L7k8ixc zWBy&sL`7o)x(#2`@}il2j$$-(#^&|9JR9W}df`I!dh5g!4H=1t%oPKQw`^G?1FC+*m}QdU{(w5>L**AO9-3w7{nX%Nu;(xG2GuT``t zIKWu^(`9BArF|*>MC#eWRT{%Zx@4HtYSOy#0259&OB=k zOb0zS_KK*4nBppCg0hj)6voc@mk!EWTP*T&tm5vdDZ>tHL3p_aM96<;3%YlP`wV%$ zrkqxvqp-|HSy@L!RGW0NhP7HGM!ps?u(dW$i-cwXM@rP2#(|(BUB)?Z^vmC;P_lj% z%b%X8Ya`H+VxNH<`vb0*jLg#_&IXf9F{mF!7S5?MPb^?YYG zEl#FHNn?Kb7V*=sYxZaQKZeZmWyCoDE;>=sTwG)%L?i@Xzd{i-v?uM>T+h(4pvI&U z42*xVO8;*4qEUD5v>!R(;uD0-96Wac66k+ku$bL-+#5@ zZM!99{0!80m z|8odb<4aLtea{942WDH%u!6hAzQ;)X^ZSbCF0kX^;8cVL8&&e!d#MQjV}5~qb6Yh4 z7_JyUCMwn&_Y&iaWAN3z7f&lk{^tT$v}Wz8?C zN(pM<@iwKNJo&F1!kqa5DZb|8ix6z}S}FU9>GHmG`e;&FgzO(v9~MT6`WC{h6VPo? zi9j!55a#sP7ym?#B7Zc;&Dj zt~{ohb-T98U12AOhci-cJIfl0bVjwBH@9oC^AEEylPEsv!k*&VtHd(z<*{12zr-2M zOP)=k=lt*j?{J|vq8zgO>rDN?wXAJTPL7N>A}t7cnpI=S@A2x2%yVSZdBY<#keOWV zU3j?EWCKhzh`)pEebEy#Ped+ys{_r5v$L38(gTCr2P}mIa4#8QRPizaII|PLgyPA?|o3fqn^omXPd67g+h#EH*GS_cQaAhJlK)lpIl6yiZ$N#!x{t2g>K^n=H?`7 z>gNx%9oJ+r1kYp)aT5y}*~}ixbeQ#cz#HfUHM4C^KfYu}@AYe6)9$tDIpXDv)x?k% zs6C!sxklGgJNInOSXQ6(s+m>N(}xyNcmm<`=>ANvf4MpID%<0Bn|~Y`2G-1Hx6Dw8 zRq&wxxfaVs`taY3=U4jH*BoWrASf9 z1dm?so>)ZX*_D5$vQ`B=vT5SaVN@akgO3?FtSH9D_E$Nv)y_*TaXwJz(<&q$s{hmi zoD;wBl^-QSYX2i6Rd@OwDvBrD;G7xbskfStaC4amfTZ+-@TllKiKC@V-$ReZvo7Tp z__a~V_3q$p-N>=M@53zab&KvMy5J>1zS(-g z1?0i{9K^X;=2=7!P!55pysW~!h9|2 zrL^gJ5Sor5S$0XSm1Y#60q1gY=raiDHwp&vM;9?* z(?#hzbPHTimw#J$WnsfCGDt}$tb!Y|6esMm3%oA7K}=y8tRfV}o{ce#P7%<$S{ zD}KD0+cYw`{_#SdU=)s&XaG8y-y`BOWv-M8x7KKk7&UE`fgcJWWO}TK0p>$4LpPh1 zHS7lV8Caduo1~c8w6{Nr=5`!7#-=&n?pVq^;OWe?)~eo|g>5@CuR0 z?_x?DUajTOti3Xfma-Z4)w9m;O(I(Uk|&9>UF!?~xpGKP0UKm;|6WOYv|Woc z6*lh?n%`59*XFja9n@`EtE{-zp`s#f>-$=tbOO+ zK(FtEv>^@-k?+#Y{rUH0V5M~NgRF{4Xq29r9=i0VcL%UfX>%x&wS*$$yHU^a1}eqm z@-b$uMfdYdo8^vebh=4kmq|47;Obm835}U^&B&{QY3(O#WepKPnZWVRAX}w%hj$A( z*5eVd=|Sxgm!DZUhx$&)d5FkThqa2kq~J$Mloy5H7p!bAOnDod(5y3ivMI>-txRYS z{@@Ly`LG?7u{VRzd|J1#<6+2W_qfOK{Az>4b2WJJZZ`3~;~vZBl$0t9F*-=wxgio9 z;Ry=X=H{}Xq=PRxV3A1ryQ8*7zmF$8?v@`7XHrTtdCXrb+cmd}+}bO=Ib4yA)|0-P z$b3Q4xP5DJu>~43^(-h@E4qD6T;=A;M^K!+f6+T?mJhUX%i2=5D0t&O8**Ti6VDmn z<@c25jCz&u?P^->z?)nyELj_GHc{?!u|_DpV7Hjze*-Y{`OrNN7YSyXVjU)SVQM(4L-x#COa+2KGH&=r{J8P9?XSB#W$sWxJM)1E1K5QOx98 z;@se86m1XZM$es`ZjNuZ)H&|ZN1ClI1K02lMxlEr3YzNeYN%BirVuzE?ad9K-08`B zeq!K66!Hr}oAA7<5)A*UA5;V_cbLBAvF9Cu{KoemO&$*ga?cN6d{7sq!5$m4UY z$z*B4PDw?v!-84@(dDRh=Ah^3tjc5RQ(c(`^e43hl75{!F}p1(#jtm_YouyJ%Ju5)rrE27V^t6F7ECJ=u-r*Y8uPE7OE+ zKU%w}sA}n~4#dc2oOr%>huIE9eAQDsJU-bgIdfGRhUsya_0cf0#!L$O>@I-a=k@DK zG>Ib=#QqFdxwthhWmUAncTRxUh!zaB9VW#leW-re+)VxIg|d{=xStbU3iyU|z&F?s zT@RUF;u9!pYM>by^g6dU(aKEM$vgMP*R=VBg~gbjMc_-cOy{`SQ5hL&v~Dk;EO48e zavU7$aeE9UeEC=c;NQy13p-CS3kJnXSu=#%n$fWyBn2h)+58RB7!qMz1l@8}fB*6w zOQKNK0e}ETt$WG`YADUBBUyJlH2j0Mse@XmWA%^9_lvGH=X*2 zxjDC@XFd{X^h$k_h924<|E?aTIMEb!#y~KWZ%cEc)00su*ZrK7q>trP<=c;g;FQTm{5Y0549VUd@NkI>*G zon>;eiJ`0{zozPWX955pR$Jcx!OsIF8=6ys0@Q)P`*GBnhx)t1j1piu#kq&d* z<6S8AWxaZ-JD)>3IA-izpoO8d6;Sx9pa5uSH6VMUx4Rg+_j}+9-hBWiG~#U@ZUrTx z9Czx$oreUrt?l05AfpK=@g7wTHazM*u1uurmgvnK7CsG38%`M!vLm#=UY;E)C*ZK% zP5xs|UXH?xoFvsuNlO_vg#D-Xi>|v*Pd5(K@tyNJOFN17{>ZzbHNPcpm>XaF;B5m2 zreXTLm3oUR>QgLXJ(^)c;xi)gK`+wHGmUbkOzXY^2)w=S)YuJ3TBU2HEC3m8yo z>@~^fML#sVDl4l<^`x7TpJ>9DS{sUri5xHIUetm4Eb$uoU0s!ebl6f9_^rl`0lW@? zxW!u=rZ#v)_>8W!v-Umq!DyQMYacnm^t>}aNpa0)WA-gwSKbvx#Vi~rwFHP88@U?` zJtiN^)-F}$Kzg;ky}apTmP5j)x0lgcLGty_yx$IpuXkSP<%a~VFsuK z`+dvVACOW7HKmzn3Gm7rFthAU=oWl%R+^1}b4?{#E$&y2L;<*-N%P^LzAKMr0!TV} zjXIH2Sj=^(O8&}HNg?xmYCBiAyQ%BukpMo45tBZX{&miIe3?CHYwfMMa24}4vch_} z|Mh#0kiJ6meTl}FegOB20DSie1B=>nn6#V$P^N|WM`4yw(V2^wFF5t)s0BLZ1`Fn- z`T~>+#4i@pn}~Je?Zcn|(it+TU;H={@&eEx>b0Tje*uZJfIGf-azHpw4X4EeoYjs) zCt6gSJ`%JiW}}v#ydakn0>6;g^$br8$sA#OJaHpv0rEVxGD<<-bO4#Hf3YfCiJZ|j zk#}|Vxr^2F4oD2e+(>=A!fNw=6abuj^zaG|0}DQQW-vvi_~Y2H+{({Vlm7iV830Qz zJswq;=(MCwj0(i&W?Ov6EFBKX^x*;u4NHsFC{E@9@3K(_fk*evsxwMUi@en71)w%F z&@bylzx*?lKQ z`6aheMR0NP*Sk%UAh96Mi|K`i(@Xxoh3d%eYBS{uH!MBB^VcyIAm6Kd86fiDo8+~e z&j?i;oo&j+Rx@>v6MuSsO3Z2Ud`rF+2?3a?O?)Qzb2W&*-N+@1O{*<;?89xU%)^|G zw$BV(1wSbQ-sa|M53YQL^U@;90T3syd)a(DRDg3yUR=S{^u7w&UDYXfA2(Cgwd zy0gJxvgv{toQ$;gZtkV7QQt>Y1OR??mPo`3>^Q=kk6scG_YY$UihlgWXd)&Cd;t-7 zHjK=3scd&aUT8Bjl5I0`#x$4@?=K#02N?)u_vgu}4M+Wcr$THkk^oDw?0v6TjPv%} z4et-5v9JH~t?phUCfx4?hc8-lwy0#>viSwmnuV}Z2P3KZzQ(qUxbnM8?7Hp#v|etg z9KM4+Y_A!Y>Oi-ub!{#LPx={Tve}<(>#7Jf;Q)k=)BC82BeS-?P)@MX!+73-@BZ5N z$KD^!SLFicQS;(!F6VDu|BAJ+!LboG3c1nP4fQlz5*P7bAT2~r!RqKObSn`ugd;w{7xq{cCMk_9ux5Q-k$o&~gxF3ViX0$^MxB$|`&`p)gY1vZ*Ec=(3v9WiQ{zcsF z2u+k>O*aW3S5!b+zVyZ>BER_iT|hg2Ef{;l{^j|NgPK}AxL)Xga)mHcYF56G3J`sp z|Np#!09t{AZO{#VkD7~dHf02K#ene$w;Ttgj!Vc_Hm!!RygY3s4T%UnT26GIUf~ zTrw(o>c!}O?!x`V{}>1r)hYFceNg<@d2}cUQMZH+cWMQbQHLCg$9Bm zg=IZ70<$on?bzpr#jLx>-~I{$|2Bdc`cTzTp3gsMC3CymK~c9hDV!29DAbZcP$>&3 z>|^MNT=P@LtD0hX`uT! z#Ng%FK^TY_3X(~ai7rq5ZxK*Mb7l+^h0>WPsir8em36OHD65uk$oyBYqVpLhy)jv7 zNV-bO%7v%5|5xj4G|Wn4u=jJ%oP|>EzkDRcWKy)HNVb*#Kia*cRKmrNjmVaP}Ua(q@-`a2WVSiJmh3Yr?y8nq&baxr0pBsdDr+XG(T@ zsHhAJ3*UUtZ~BoqN?IahCMt@NM|<-^w^sAOz17xxX+)1mEU%?Tc>1<3^nn)t<4NBa;!bKP zd(l96bZp?RLMRO^dybOmfQINGND{RJ#KhBVu|trGsvJ$oXvLn~3`%ZpFUJx3#}xOX zP6Rxa<+IXA>_NuLgiTUnDu?~x{j3(Y;%>;!Pev0Itayl}psa*9u1^kM0}}~)BSRh^ zj?^2eICCo{d!NyXVatTm5N(u5s}?#MC8-9^Q0)Gi2Tx&q_+=TOP9IQ1B!kZ?9 zoUi#l{-3TeC?T(BtK(T@=WdO%qV7{u2}vZ6>sUX-fuuo%G~P2>!JBnz_}%`*`sQW` z4Y`Gx97#ZSC2}GayGjG73jHIQVlRK3v*GFO_nzo}OFAN4aY_L+~*-*ygX=Jj#2=7wG-s{9E2ZY z>ysaMddrTlYPipe5*qx$ld40~t1t)WKlA;szwvVVNovh+*owQ5)n7Ojpn{w>r6p+0 z(yBPwOVg#luB>jQ%IhEA9XZ|}V`q_P)I2GO{*88`&kj0Z#B_P96F3w5vjpYKWOVcd z!EV?;hnyRz0kCqGI=Y@T`7dp2KhP750bQCNItvqTl9x*h6 zE;ldsO^rEdAvx2b;fd6%j!7bjZnTHo|O)0=kgTKKFCd*ev*#(NIG^x&&3+h za4l-MhJ&!%3!=AjdIP;3NM+Uv#guon=nb-UWW#_+lTBr7Xq8-rZVPTi$LSw~jR zm)7bkwZePBmmrBE!D20HZN*A%>oiR~X6v#dry|IX`*1zEG#Rhf=!+bP_p83;+n$t` zOMY{=)eEBm7V2O80s0;+`S+&^5~9}HNLcWb@Z0_pw8Z?=A!Wa5Q8y+X(2gUW%-821 zEF55I$w_!PgsGC&-bi`fV(Z2>n+G%5=q70}Rp6sSk5eKpkAk2jJIG}woms}Wy>~T$ z=!gVwNrBI63c(T52ir{GgcKp5s+(8w*nYm)K+Y6k* zZ34JN=6WLnRz(Xe67*@L0r4|Z#6rDH<)i?s(2c=(*736xyUez5%ExkkVvZIk#Y{(xrne7-tsktO_(3KdpYyZsFoGAC@0pH3$`NS$3*lu?gs{fgM2hz z!wG$T8GeJaRchB)%~lRBA2R`akV$)+ypl8ey`~`ya(`65_xNTRUW}?y>5P5WcKyvI zTpXL;aoS{tlWt9PHB*=I%q>3ahS!OpV)dfSQU9zyWfPOPFo{+YORB@Pk=(*e95GPzxTV zsT9ij=6&Rp#=*nY`?r2~nPXD9U?(z*z}^)`w%>b?i(feT!fQI#h+Lq0v-P&GH6oX+ z8-Ij67<~loI$(uR7!E2=R+&DV|mLq3IEG}9ojneo z9Wm>}Fu6|xdqME=j0RKdS7=6dYbofEjW)OXQ1hTyidC_XK5itDm>3ct2hi>hP7$MO zBpeUK3aX9{YFm`Xk9S-ZFHLRJwc6XBWZ))vozyeXb8;+p#wFrPYU#-aKhBDv_k`j` zq4!RrO>uq+6kjFT>OQ@1((FWA1ckI89eay zeCSUi9yg4(Y~Hlx+324FbXY~Ov>%3OJ)uU7WwwnzgoNo1)b%h>(7m1R=JK-HN(4z~ z-$|d9!f%<5{b4y&+HS0VccosAIq&J&WweXNv)*y!@RqmHQ7 zO%Dc*ledrA7Xrya@y@5}J-Qs}#cTr`%SWZ%Q<&fhE_Mmd<8l55kOS_K;3Io86YgAv z8;((EUNvYhR1`zn8>Qf)#NaGdBP$_uOvs0_{&82>v5M<#U)gWvi&QFbGK2i2+G};d znxGP3=&0X?!r;`~eR|Mk!4>C7dcWDI!06^jCV^1h_O5U@5cK5D<3mXWuID80{GDel zO^nGr*@WcM3AP>F2hD2-`n#IdXz?ASbH_Jr^rKS1z^BFIj5JqUoX0bPg9V*^KKFom`QPiW!`I9EX&|@>oaFawxGqyVn9^_FurL^QSBaIV*M$M{XapQi z6wy0zl_^Xhj%l;x zdb>`QXYkAJ3&Chb?)%^bojr}q`M^KSVrz*vv2~9ip>>tQ8}Tps5W4f9T7YrX zxisZz1|R(YV2=2;!2^hATth?Gt!R|U3ksF1r~TMDJqk)nv){hBs2Nmh*#rfLuAOnD0fq72xBU_qk0ZmwwC~>G`~ME7em2e5Ahe11#es`_*Nwmpelq47*_{H4 zaLAcJC>**c1OlZ=9cf`g<SJ0Z{e5Aycq`Qm5s5U;mGfIg2fP`30`OpII zNUzq}5;Pm>YAnBNqG$fIFopM81QqTkmW6?B_S6(+c?kk7j|@2mEY6S}2H#%>_~k8Uw1)$H<*CNMT<&Fh zFeB)Pe)q1;k4~#tF*G}J;oiE@GsCgX>-{&Hz0Yr!U@~N|ED&`AR>|)N6Ow+c<_L{5 z+aOXQAJ(%<-wyA(^}2hXr1Wf6ekGERd6!mvC|0^NNiZS$Wc>9bJ*ej$f-he_Th>ss zLl1`8>M%D)p_SLr=iiH!Nmzn%^rolOL%U5`V&!G8mnX9voXCw8dXNZwGG%8gjyRnx zso2GzC7D(Dbn{!wu+f(6@sxqzXOoW+JWH~FomR;~dT^sC&(7L5sjT z?rH)uI5qbaJt!-S(0`rpN0kL*@J^WE6F8lbZG2j2yYP{)yye)sm2LU!*eMxbk0UHe ztIzxU3GxfCN0LT9y5wZG){e_vxnDJ-dTR=sz9=qlLU+q0rBhE9FND+*U!vdKsh+`C z@z<}e(S*ReTY z;V_0&x8;+1IAiY_1^$RNV@c53n%{)fZCl4IOaG7|N}A{Y6eQ+!b6$|P(X$tNVbHngK||^tEhhEOPP5`F zqq8KVDY4C3aF0{u%4lqf#<3^LOySkc-**5O3508D_`xD(==1)C`V~ zqvnahBh*_beLm0~V~6;<6`hoFsrCCSdVr~9SbBVi1jjZJ><4NUFL z$~JOLtlsPjMrzTW_jbHLeAn91y!T160e*f_6|1%V*7_B96xpXJe578dxw{*F>-P%J zU+5NS^U1#7Xl85NNbJ6iX%v`5$-8m?bT)T6vc`iV|LrJkVTaEh$xG0aoJYgNXn{YL zyQ;u`UQ7b#ebe*QGfFt+byl&YR!c@8P}&e+|H-v=7=9m})adhjPS#)!kIyBnLlTYg z$6r*7%hYET-^+eIuImL4Q_oj`VB>ilt<6*XJDt*LzqzD6>0pfb6cN6M7rE6HuF7kE z8k6f;zNb2hASoxex|vZh)ke-^|MCecCek;o3d>0sQoyk?Fods@N$pFPK_E`pQ*JbY z{7>AaZme-p<+D0-S1GMFg}T1-+0Dsiwv!;2IB1_GO4w^7J(Utm+}InKV!7i#hM)gfOE*y2vCrGH9~#zXqBjc;WX+baiQ!l_WBfU>(22|&53 zQBg(ix{-7-_hr_spU_uh^3Q3H2W$Q4aFBXSCTk}5D=Z@1K!%}N z`Ty9u%CI=Hr44}~fuO-HxVw7@L4vykcM{y)gIjQScbCC^a1HJb!F8~KZ}#3Vd+*)d zU#FjWx_kOmol~dut&d$|OX>vB2o@h-k^v*L5kzoWdT##hl-Pzr6NWuMc+{-5yT?VQ zzE|9{kAUt&@mQl%)bH$*@Z|fqsaCw+KkO`eWrZ9IlbM% zjMeVIdM>4iJy!>V#52aP^N8cL<(`dg&D;MbWfJ@#ku2!IchAfS6!3nd6Wul}U1f?E zay-$TCO*SVl@{@9+o|Kyx{?@XdEK)?cgy!N?Zj9nv%h-nHk?wXty1g^W(4t%Hu^r->oC zxoT-@HZLU)XC}8C)mZjyytDag*N5wn_ssV{-@Jny18b5(XmdCWQ(lLu4}9U7ckIvf zBs*(0>T&fR{dmsD@J1XNd{i|DBuIme$Oi7R9_l_2TuL6eRafqSF!l4jR22cDk3$)z zK&R1kS5A{7UH#qDxD!H}1+Usm={;+K@7uV1@zs#dpIG&ttvEPX=a)zmEMJF_|Csh9 z=RzZZOLP0@p}el!=3-%zE_(y@=0Gc6H{9BjC|l-;DcjLr3Xj9uW?#bNVIi3k$N^GxJJzV9nTzEf~fJQ&w0eCnTz# z65BJMWiq&f5!Fxk#+Vc9yj^^#;yZ?OqiaL)!TP;NR?((?-*e(C+!DD#>iVaAF(rAJ zhw&MofN2B(xtos1FVy#p3@z-3X*e@AEwSRVyAip^?GbI;i%Y-o|M=1A{5$(`iU0Ot z>xDcZ;Ms_LXh^8%q7BA1YG36gV`EiZ|;YvFk2%;K@)CuoHa2G*K!h0MH*FUCN0je*`K zf!>E-zwm3ey}Kt$p1&vn1M-C2#wbFg{P z`3C%TD!Zi*uR^%v+lm8rM=_9Wi-hS|)84BVc^@h0VLwisHPPE$`~DL_&C=*6yd&nQ zlejJmB9G19kcN!668U%t)%=u^_j=e0lQNw_>SB$(3M#JN%g!gVwlcT_s4n^pmO&9Q z)ln*5K2WN7%QXyiIo}2oROEp+|+%}*crmYeq3%7oK6E8x@-kl#c)sVp;Y3qo|{S}~Pxr!%9PFz>wuk);&DV8vsphra+MAr%X$aa2d zTPrMEP@GWsmX3)|e`p!WB98UccHV?Nys0)kingVxq?Hu)5*8ek^W-+`dy{TJ}3B)cs=k`kif0O zy9VoA@%je)c2DhtiqC^%<$EJ#y3-G@Pftb`g|S*SU$vu9BmPiF*86u&STTG$tf`%`q^l$aC}=zw#bwbijrX_RL1=6s7M8W)yxa zwil1Q9t$5@Xu}2K#j!(qy0X8>UIzDH6L^rr1UXYOtVp;$&~@A&dqt_Lqp%NvePe_> z0?Zv)D<_^mIweZY$?^|`RRCvgjM!-#@5|KSm6Z>9RE_+%4 z0Dzr6zR+lwA8BbGUz7sbxtn4b30#DR@WwW~mR_pc`FLUKGBWeTx1em#w(6=-f-vY_ zS)*7kjCS*59^kA+*Qh)0R`h>8mUCeTI4~f3ZH3Sg457i_Iae9Eokj{kdbNu_@ZOCI z#_LXX#k^CBYKA^)LRv8HNB8(8G)i&Z{5>={Op<@cLMe7wQ&V9-7GBW2*3vJU6 zh&FfvbMg~j>#PIuT{HvFqa$^Py+9$tu>tnRQyj-DiTU%ro;Ot5F61tPcHDlW4W4b^ zI6cCT%8f(bmwItTZFX*hFpt{K?9oje{MVz4oWfqsmf~fsD)7%b4rBttM0Ng9XSRnn zuRb!j7Z{)E_CH!UZ8C- z%q3U~VOZ$6-OK~Qj%4Z)Y>u&n+GojG$9-cF0x0iCb>BVzw6#qHC2+rUD13P; zDM!Z5opg71Ck~29JKE`mK-%sL<8ZXdDaT{&C_pWQ_M?*oCipu@D%fs2V*e~DHXt7+ zE(MAN61emaQLb$>tw48u>m|kvgq$`@nE`i|cpp7GVINq(>#FfIn-MJ65x$dYjqz5e zn5=pa4X;qm99YgirmECy*^iF*>xPaVq@ad4+uj8A&a*v@Uw8-eu#!C4cWxTg+(TQZpi#bfDLoGQsiym+HNV$;vH%FJE@O@KK$ zz10PB+Qs{(l|AaShHJj%L}j&KWFEV(bnkguy^CzGZN)#7IXjCMPY<3e8a1Dc0~*M; zNIB?;KPC`|g=sfnNsW)nD_*|Rbw83`DOw{o@M?D7<1kdx-SE*Jq(WQFIuy>5l}-IH znoR)6)nRfQCRbO-+BA3Z>3+dtJiq%_f&P=C3iKk?6aL zt-H*fk=4e4lUNdZ9lH9(CN<^{_fvkOKh>A4o_LgHZR%w@6+2dmsU&d+TQmHC?$*`qc&I-hPVlzB2*`Z0NQ zMKnKiY3ceT?!wEJsAG6x;T~BVb+4d2Ws*tZdvv63A83ObI@5hgMmbU6OY_{a(3^eM z`ldGacr2I@nEae~Xg#WL5|#D+$c(k8IWfumkwZ-N1n)Yph^Ia1(;ePCQha``$yYk9 z#K7%lcN7<}K3ArX*V>&^sg_`AB*b5F5xcJQ<0RqNd47!LS= zr-u8sINBb2CT4d@877r*(S~_0nwfZR&Vi+FPYmHK|%|zcDo>VT_Y6}-MS+}lH z8G0IxMcwLRvgKI#W3MBIRb3m%`C12>S<_?M_MYb`#7Kf09@7ZQ-(v|$F^wc@OJkP z7u=c7CLu+qZQDGQA5TTknkN3bLY9tHc?Qfqq7x% zU&!HA^9Trqo|`&uKXN(Fj<)NV7Bj#>fywJw=KK~u{48y6-plA82T=N9{mZA8jvGOasfo8HZa`kgAS zxHcJzG24`}Q2%f@Ri z%|g4gn;G2r8~m?_CGR4My0cFSG_w$+4Dv#0*C$9_2=n$wNrl{?rCk8FB(l_Ju4JxY zG`0QYHOt1Qb$*`HM|9!j1H&ocEF#aErSqv1>v8GP!GXL@gA^<$L^$O{4ArT+Y+B;Q zdWR`-|8NEetk|CsGK?hFvG)c7Cf=E@u?80FFjrq7PZWG831`#t;girDJpVqErzc=g zNJU|;Ag3w96x`vSRN+Np{OhLV$>IKVfZmlO&$LyKUYhrMS1oMME35U?f47Y5N9Gxj z)g7g-(3yIBH8TCs5#IwPJ#zhs?980aLk(r4L(7bn?do;oPx@sAk%6MBI-zIuLhtp@ zYeyCzl_eCqb24aMPn)AYJrTO{gN@lV=^?&QbuHdJs*-GL0&1&e6)kYV-V#UXK3dA6 zZErPcpJxVgfGMkoI|l(y+xUD6i<1gjMlD1%;!RYSD2FU{#z!)-Xl+U2a($Q8^9kLb z{sbqcUd{c8u(&3kS*4eGDfT5s+F6q$RP@nsLW}tv;|F&b%=5`c;(};m8Wx0&A4z(@Xp}g$^7`szVpE;-S+KKZ5i+VKP4HPDVm6Z6| zGv%#YzX}{X7&s#)t#E#QAnPv_z5yb${X*u8AnM6O3?%C5t8;qZKkKnOdXJMPHq@in zP+r_ZZg5=f;Xas{P}JB&f=)q!};tzd^nxbt|ZJHoYFQsaLOo=wU6c!P~zpDU$h*;gW9p#4#4^< zlbq(ud4bb*2jbDys*kv7ouMgqv>hXgzA7t0?3)fJ&3N(JBfWO0Mec$)Hq2cmPbun>3_+lAptHb$fjT!gNVxndECPJT3n<_EmKlTupn8XQR$AqN!J z5qY*kx`I0G4HGA_P@$amRzYyju{T_WMo<#7+VMlCMkjSMKGvY;kK*L>ng+5E3X3H4 zN924?AbrABs%3FJjo1ut$9|EsMstowvVZ}G0dV`d!BOyHaAkm_ps`%cXCy6=KVN1@$W0=Bp^${LJijHPP z&d}o<$$)HHZ~ZX}PgJF`Vk%OT!RwmZuV_0jGzEZiG2J7mJyC^EOf5SyQ% zGJTv@EBmBiG^B;(b`0`?`Ir({)aVA0$yBl_-4GRNTxj>RCL2lH<*U=fy5xte{bUq~ z@XT*tw7fTG1n-URJir8NbadVn za+pUw&c*@!x%F-wdLw98t~8t)a<}m;HTkjjf$IVTKhp+c zAeeyRWtibx`>SK+Lx88NJcY|}YUx|*pn)(VGp*C9imH}M;E&}a*FLslt zTj%A9JeOg2$I>h51mL|gK9E_3eiKbFXfi=<0r1=-B)*4fu8SMw`4ZI*AgDr2-CeXW zm(NO>xB#ZiR>3gcR(kpsq}3v-e35*lOXyoB-j1QyX-1k^d{pHVtV|jE8L_|Un*TjH zrE@zbxyg;6Hlgo~yM9f0p_wuZafaAb-%0xS==#p6gtHq6JjGjjrdHn~3V$j065XXTkvL>XQ>7{<`A(;FV~-J}zrMm{ZSy;sXSKpaYGJ z*7*3uoRTgIq;wp^>FTWc_FFbtuo10I9rwL&;h%nnm@}qFH1G3sav2iSg7>#3Yptz0 z32i%`d|AMkh6>d(iwGiil(_z%8*F8ZQr}mjTFWEKMKvpMR4`{{-ss96PN^Nax@D(S$e`Zd>&F=eCA9x3=glt z<<@h;7hc4=hXR0HLBHip?-Yru$k!{zUJa4yA+X{o_WJ!Jeg&Poo5EGShcM+Rg)fIh znJru#TQlt!pG}2pBN(zW`t&+_dluV2K90I(0EAQIl@Rso?MHUXTwAKJ5EK-4x`{Yu zSg9h4Q>FJAciTB>M}>EgD9yk3pI=+4+e>pO9QJA+ziqdH+|)Ae42-_``RH+@tigeGs^nrNZtb zFcbESPzHVX>nkA)5+{_z*_*NJo0k>H- zPg**AR)`fpc`azeXbJe9a`t|;%BpD3JA=anu0QZK1>_)K?%W%M@oiOStNaKB>d&Kifv!RGI<|+xcI0e>SdIv zvdiiDxh+2y*69E@AAFv;PXA`_;!PMGhnT~FEmhMhYtm-k$50&UwKq3rTt7j|>;WIf zJ`BxGQg-a)CGG3y6}wk=0K7g?VArK=T^dS-55HfU%I((C)Dg@h?Zw1A0(+FMXq#P8 zn?c1Nn@&}ol7ddN?C0xQufr>dTaK9C(~}1>t1+7nm0Iznu?@~;48%(Ke7_o(xLd3s zKji#ICU~LrqapIf<|V_Ye8GziX;{6>FC*oyGV(2@&9Mv;rS%4th07Z`_f4quN994j zwaM<$zJUc%EOrDX1(lCdEpedRox1k67`tF9`Y4SfS{pb!Q^KA8ip6s4!l|<$CDd+d zKK<-nOb%p^CL`X47Uyij?IR1hafEHYh{M_5BJ2U*HU^oiBQsyT&IRQ0+P#PYu8 zeACDYi}hfs1qBeNg8i%B&AfdT#C*DPqJiT%1_YAWl+1yATK49%Zs4zW^}OOqF~yiN zXE%tA{*sWPB^&dkT5M!H9go+r=piPSGd`4@%){ph0rqBBOO>!w6tSZ8Up`IO(u&M( zuJ(mQJZ}9*qbx02^p@pT9~-D@XzX?t&aBhYmf~2zq?OSNU9HkwO7G^g*3c=9_qa^L zvTWo*_9GBOEw09mbzJZWPvAzxum!c$%V&8xTG7!}k3iy2L(Cr9p{V8wK)vYoRSSNa zLp(1A67hxP*{savq$@(=iL9)4GO28N-RoW!rIzn-u(hi@5ec5xQBO-QA3fhC#M^I3 zavd==T%p!WzK+y^Kp-ZkK|D0()~9%J+}VMpW|HBED9-xlsvkceB}Cg;sAe!zx_Ut5 z_UoC-%Q_nbZtBTxURm^jE{x%8{g-Dx8LaHYD!0v3ns4;IaikC@Ok+fB6m;yvB=;K`8C_ z6H84;5T;q5&NN*L>rZr^{M~ek*OAzh(^wqi>K1B~zLW7FCwkR+w{oH9R`Q~wH6Rb5YXqYxLFUzx=#Ik0=Vo-v@E-)M9Y_+$QZ z_7B9wola>73Uf~!n*TryeTl;07|(AoEXK=MXQ!z${qe`@Y50r}@eo?aLAUL5^oP#t zgmw1x+EtOH^?zDLT3TTO^mRk*;Ek*oZQG^Vl@;wj4CF6j+e6e@<~7avCb*YUto1e$ zs-P&`TBK#ZbZoUygClVUtqH5L(o-VV=Hd9iRBB9zY~}j958?ESy{52PUfDo~#$j_v z?jO9k+=dkp+TsU(EGg?pnNu(n*CFw4ZtI4P%zD}Z9!tu7E2++ZjTik6vp%&rtj@@` z!RmG{u#m1pFy4ce7*M@$|Cj7?n+fysOT+xwY8E7e69y}M8GD$|xKiVHQCD{#XW5Z+ zzM*3%&9;-n`sXxW+%54D{^{xe*Tq)@@c`(arNxl2T4kl_M)G@$Lf(>`orv&&?vwuH zQ~%sI{_XsVSY|dyKbB-v(=+rtnG{s~SXTqPPppo9{3RP53zwX%H}nu!%<)mO+*DZ$ zD5|7HLn=Cihl3Z5YUEBlaP7(Rp<150bZc1|%Nj;{dEJGv$M98nXsN9h4J-L328%E2 z4er3Tfa+f=b?%uB8<)wu^tV1`hZPq~Zn&nw#kY}vhP;?n)Eh6TXIrHKHLuhQcVQVF zIr=+`g;ww!n}859mbYzI*8xMf!Y{vJeJu&f-{Sq>W5wu7Wd>H>P*}wqL-15r4cA-A zP`sDdbOE?w!>sQTnw%*9xDTgcDB5dbjabT(@{#n9we;Y-@oMQx(hAO6U znQ?a_M~1vLG%PHq`uyOlG}xQD(HR{I_NhZHjJD6R#;|7LJae)falc1f>9ji<(E-cY z*aQaM>aX={gRi^Ku9P%f;|B-Lb8`*9unLl^wC7cy(|Bir>USZn3C=?88L$ZwZ9Q%{ zXc())6mC!;zd+#L=!tH&^ttdE2}eeNYdJ76x3>!A82_T+e^H(DNr<^$%*~2u;49>j zfSIhpz#((g@cjJh`1liASUH;in5F+u=X?O(a?0`)Xj<1XEHtlAbz$yr2w&Qc5kTp% zLp1cEV$$A_Bea=+yhTL*b$_J*vWT!?E8o&zOKH$!!lQUAjo{m;hgsD!yS^be{v9(7 zhseR$n4gT?5|)mhHT7>B3G#|)07kV&MKZKU0wT}c0O@}Kn*UuFp{+d%Y!{)UA1-D~ zp=A=D`lrtv1&Eo%Ct@4bHj<+eul?g4)c#abU{zdy?*Z z>?zq$nyg6)bi6?!=7-yia;G=;&X^X@2@QDy9eUkQiYFbqG5m)&Rwpu@MkFI+-})hC zF-*w&O4mKf>R+S`00d@!;f5}nNI`$uK4h2cL%1CH)ncC@#j#P$8&*fIzyJLA$iydJ zH~Z1mmBPRP-SLoc{U7I(AS5hC*^i4%Rm|4(PgKuqt%t4a{$!GkQCj+sjALyy;y_2| zD)sjs9cV=PFBq49ea6m~5tl;fgqFzTfn=^#&Bk@Tlgxi+{qHLrEF&>7yiCm9?@e_W zcx#mve=Dd293_q zvB%EwEWEWmh0jX~LU=7F2xh3{b72{vC%@-MXlLfDpj1@5J zTQ$9W3?CADe|T!|$3{hYlAWKAEa5qi z&HF*4nJ5~#O7@724p49vu!hXTiYVyMYNbCew6jM$E}@9qRPLo%3C6~}p#r+Ihw&g> zD|@46z=FHP_n%yvKc))!kizeB`Vr9TZ|}mphDqIpyEe!AKO26~2hXC3!0mQ@f|d04 zC#CM2C_;UZ#LJ$IO;xYv6!sKn+&HG&uk{Ib!L_kHGw;4$Y1AzX2*>khal^yI5nw$R zmX|pnw0?8`U*nQ)Xk2+jfH4tmS7hXZwP-TS8KjdlfvDw`{F_O(Ka5T%nASPLiEHb| zNe^9aUij(32fhT+z_rFm`A=Um7P04|qiz=Q1LdhX6Z16X#@Fgk=^shz((`?*S z;VIeOArmC#e_fMu_C*){X0*-FxkT5emXyvb+vPHgz)B~i=BqiqYTFx>#&*sm?@eFc z#O|h-Om3lvSS#Acg@m?%hLCF8##hM(iK+x8>x=S^^2}!sQe*phWb-e5{_a&%E z*Ju^K(@cf~O)#cNXsLUth#KQ4c~H3wR71Zj{VY_+M_iZTtt`rFDGm9~6TRBwg5VSd zWha=fwQ$6%Onx`HG+~1a`p*3|3&Mtd3Njp_yw(-b;3&K6U#n@`yTIkYoGvjBblpg3 zu&*|5+%I1H9z3tuzgS}Atu{+}>DR{bfL`l@PmtkUV9chexO`5i8YeVP!AB!c*7h_x z-2}yJLY@>Z&z5vGK*MUqLgUvWD1%D_d+o0wVWpU? zmwR4o0{vj9J33D&p2~Y#2kV2^{rB-iYkilI;|j;n)Su~buGZAyY5A?%+TH@+%Co$n zHpOWWon;Ep3swQr%wDdQG4i0x0uWIdkA7LXF%4=rk(Yn5x-Lr z8Dx)vPG*wMLnP_whFP_&HXthE|179aAImd4-Z@I^DB``?TH5altA z*g`)Zlg!R#_t0;dd>|mj`;febC_9iqcOI%tk;~6}SpB@0bgbhFUhgEor^`I@n{7m! zjrPRJ1-p(LUT>{$c~_=K1I_Sj`qHLOqc$(`YmBp8>n1qAZZQ8}w8p$(aKWwL3!H)1 zgBcz-Fz383*<4U+q2=bDuAI2`XLaU>aOme_RneE2th*$G#9=yMET4aUF#qx{D^4&? zi3Y0N3;1JU%-jB|OC&Z{$qwFkJH>LHsMH;OxrzIN$b;Q583t-TM~IA`#Q7`SNxA==@(LF1|N z-4u+lN!z2K^Ygkr+sL>b{2ooq?NQKMo=YdV%CN+c%jNsI{X46Y$Z__gaNwf^ zu)&E2rw!@i;kVEo^hq&?7tA|qV_F=G?F9Szl=Fn37H9nIRTs}WGWPcuTq6FWrWn>b zZd0V&M6Yt#^DWQoQwA7rHNHhuIUenNOHj9+*@HoG%K3N}rj3wY&va%~*(jn+2Sveo z6i=#}2Y&9H=2+ZUgX-x5(-~azZ0}K|Vd-a8e)kwoA#e+Y>CPwF=EM}*6qMNN(mrk! z-8qN@keWujXJU~E)T-7*#x0{O>G_xXI_Q!bS*fl)lAzdipQwU#HsCw51;spG70tmy zLZVO;-MRWRvNOXcTXtr{E_IWzld)k6fJ8DV=;M{}7`$ZbQx`{Wr#{&!ss-!b_S&6X?#C zUwZZSBB;ja2JKqXcL~#KCzQ7con~xY4~4ZB(^9)1UKnTa&#fUM>@kkdHD;ax-*AFxK9J{(zE>U%W!{@;AMOBxuRBH%D-p>8>P3iPhtjjv2uF}g>#>yxFPu%{epC2IGjrNWzkby-11emPYUj0gg2$@(g zUc;dJds<-`R`q1wiBBk8qoxY8d!K~gJW`ElvcQ>L zCOB*HnqhGuwQ2uq0Sr379)8soBuMv0tG5*=1}S1ul2V+m<_q#dFL|-t6b8gf%HR!C z>~xLk9L3Xk4A#0fi37X~@a4np;rIPt8GF!vY^o-HC-nF&keFBx%Y3M;}_8pu$k&QXNnz7uyP4%U%{D@mbRD zx8bK56@upB79Loh{gS#kBjMMEy22yZgxzDfeS}kZy1--DT<>&;Rn_0_6iVpzsa?Q$ zgXIPeLs9+M(~V3pL2gp%6A@?9)Lo`^Q&tDYr!QV+?j#2T`F%8PrUYofYLaKaBKupr z`H{z;;nYOXoG2Zc>|9}0@Nh#!mf&Z7+qj42A@+{y{;R$EzrW!^s( zO!sG8HEk4zg4{E_#NPtaB&csl$7N`o9p2^bF_yn99y>9Dgt_sP&Wy3cK!nni)A1G$ z-6@sp0Al&y$87!$pi{R$?FA5JW+VUpBI#wTy{s{2#(>6%DLT+Mjepb% z&3vLefCVYdSxog~9cn!L+17X^?&i zT`#Nnki*5NMZnodAT`cI;>SmO&c&=J^*+V?VUk(TEdFKHGrU z_W>ryL$86wp%I5xy#OB~W zT^^mrr&W7(J0ooFQLQnEUK<tt-o08Ls|wXAJ_64a(#}}=BrGw9>^5c(=J2OG zCQmoqod%ms1!f_h>d9bL1x=Ie?Be9}+mr5Zf{$ttAfF7JStA}qyPTKtryW5qV-L~| zN~bXoPft+8sN+w~`vZJplI2L?(FtiZ`MvIeRWCMehX>V>*JyX6rsN-?9nwAU>$JeCxaPpKQKtw^fhn zKU?9ZJ0Hy}TNt9`<5Il`i?0@o9FR!78&Pdudp3B(S01j0z_86T8;LgJRDDGv0L8O< zYrku3mJYQ*Esh*t6OUXC9_t2V7HE~y2gjRKZ;m*glJKEzJ=lw>6OdSa-D=n6Ud*{; z*LAxP?64176S%3nzvyQ7arMH|ltRP2x2}Gh2U_u&G}xD408l2RLK>ST369LNFJ0Y? zbZ%*UHR^513rW4h@8x6MEu)ma;1XQ~cg%zRJK~6;x|DZzu7smrZu#3M#hl3Sp_nW4 z3hurYO4Cf_Wg`^NBUVpz-QjMobF!i15n~3hbfp$Sd70n1UrHW{tpyG*D`n*|x zf^H|qFs~O^TIJBmOM;ZiZ_Yfu9Iv`*tK2*TFK~L1VPKF$B50%w^-rk^3zfcoEhzZx z{8{U!MRo00UhEsDY_jmJ_an&5EHF>cP_nW= zv_GDJJq26`)33nZ6Z`3QcRlG7o*j^VUF2UgspyXmLrN-}UHWT55I-ZudA_qt8l)}x zeuKf$5XEA`w}yFgX-mIHS~wOIJa3E>(;4QAZS@t)g?(4-5yg7>`NYEV<(4z7mTTH; zROabp)Ya)e9#D)IZNWFuZCH(!>u1CbOO8Z zO^GBSO_!jWBm7y*iZgt0%ILFqu%J-peAtMpbMR=poC+fpCJImR9!OW(VI_3J?3!fBqG_=X^a*=`lnh|HTtayA$vdCZ_Ly> z!LaNdBLiG_uZ5D`vCwsvFn{I2zKy91#&ZbvI1spd^U5om&W5+-D$8vBSu_=va?%^Jc}%PTyT*)`I%1iO-!jWrV7N@kVw!_8PDE1nSBroM`!4G?}KuzEBkt< zD|^YK`;49BEAIhv{8aLbF`um2DxIDep=Y?POwe?T$$=s;;@FO{?1g<(e>UP|y~A0< zexQSKWU*uCM%l-{2ztL#PqJp-lfW(6`oPtKPp7HEr`L>%8U1Ktz?#`~l-- zyr5!eRF>^L-cip(XeJv?;M<$L%x%P|Mg11>w}yv(kyRA2%{w00ZERr&!nLrYH4oy* z?@5{9*wGf8D7bZW(?+Y!Fwyrkcq6c3o{9V1I?nOVIUc zO%Rc&w>rP6+P?p7mqX0+7)&FE>NE1L2^sUa!cUD6#-ef}M!@f!xq=mY#Aa!dCgX_KTUey@o`#d|koQ+#@=}|5~;g+P;Id>HJOVcQ_GuVxUr7`o(Sc z4sx;PG#xC-BXZTCJ%J^4z(RiWF^^ zZ`*G_vTLD#&Z3-ow6TM2B<$~8vl-|lc8ibKNolS&u(qd__k0rc(YoiF!Qsk&@yV4O z<;7vwUr!KsJ}lvPQonU+T!wDWnryZ|PO%x1|JvaUD7kWn3zgzv zP5NAryABe(b_rJs@w!IBmUo;g-}U>2$E1WkYT#B0ga7cw?N@utw(wTNz2%nv>G(o$ z6=TSy(#B*yI?y_iS!*PDe3fMx?&{cV-9gKQJ$E#vT-pVlz^c-V=*4fpeth2so3`lu zCH=R+4+$Au9M`|N*i0{$%~B7}aw1GCvSM z$gYkj%rvqeWO|q{alGk-%IXD{IBw8ZFlH$q_9&yQd8;7&fV|ogN0p%1xwf5!+^>a%bk$oOgQno?(9?xG2gv2e40M8EM%6r} zxh^lZT`teBaomjT@a5i5nhr#S$1}h5CLMBeFOD_R7=HAm)tvWz9(#ot{^qqd1fIue zy@VbGx>@qx_*K&in2WwnhjX9Upf@5QK9HI1`81`x%;DxoUKaTPCSIN2Q%^|L*Pu$p zZM%XAgW-71eC+sn%ku@z`|*sGds8;K2@1b(~jAKae zHpYKI>1k{E2<5*;U2jBjblPrNRiGAFugQJ9*I; zj=W9Bab;V33Iyy&0x0&$?AelyLNm|dWapxctnA;YHl)e!3rb{didcFb8HDkT zW$(LvE`K-|97ZXNGV13$*D*{RGuoY>^zPAEL9#ZyN^D!Qdl8!~wJQhnf?a~!?U$Uf z<-a+c+toRPQPswIex%#AVR57eH<5WAJ#7E-_~5u)>2%6JRU@{;qseDCvGU3K45~7c znQbd*o}ei9djV#XPqVK_ODX|3%K!WP(C?+o&b*Gd0n{{P_JtP2Yo%*o&Ez5G%KMF? z?rl2YvVQjGYOH{y`7HGHntNEotD$Y5s`cJtbajWt1a3B?45W|C4&K4K5g8Y6g4y!D><4 z#d?+Fi8_qN&<zH~yGdQOu%pZd~+0sWmg6t9pG*Z4SKmI`k5xx^#hspNa z#qjSDe^^6NSKQ7HJ*o$+@_6qni-=EV4@a8{d`Q~kbbj|={|ULgVOvV&B&V6oIzDE& zO~U4O{#yq#Ga@6&8MG%aa)fWw-xCPtIREjMz07vl2TcSEba7XecRU3A6u&pcZ-;nY zxLuVXNs%6}w;H`{>Qa~HEG_wD6*qH(GV3k$;019@GSCC@RX0uUWPsZ`d?I1^tyG&r%Va z*R79uLFpC`U?F+@a9z)+d_qtbNm9t*XJh0y$X%+sQlD#J0-~YYUg7f>ZsVYdYClC@ z3knCJ33HFHAWy6L^v_Gr|2X@4ya9Fg{>wmPj^!2pEuc5F^~8J0I(_^&pHMFN9K{Hk z+WJZOYue>6JD?fznVi|nN%^v_)Cg%Dfa#k48NB%>Jwd|eY~K1^(?D-7UY#VpP|m@B ziqeZY-{DqsD4XZFfdD9|lUih!RDJtl4d7sIlD2UE({mc`>3ura$p+PyCR zoIx2b4BVu#bftEXeu&KIgX)}RNWDiXamI|PNwVw5!UXiY@Yyg+<{bYFTw(De3OQPj z`}&?FqZg9nOTSzygzqath37Edqv5xG1wsBw)uga5xvlXn}p7oOeOQ-1pSig3ia z6&mS8LdG=Yt5A3t9Qu0W56h;=>J9w}N?L}rE9kOXDb}DH)4@U4rQqbQgx=EovuaCe zMWd{!kf%Vr<))}^473lLf#Kl5dj$!n0O6TH9382qOq?3y3(rJmaH)vbD}HUUtRL^N z$CCT$JY?PSex9wHs6UwUannNEums&~r=qK47;%cIqJJF`Hl33HI%)c@38g6}<4FFK zE1H@pdM8y!Z?39y51e4n+ltzuTcMkCDxmmrc*@weqHziBt4cA{-ZNTC<)-ZYz@A*Oe z9k)w4cA%75i0Z2cE&2`7i^9PBB~*O*r~hN?E5q7smvx~Ox8e>3in}`$DHMlN+zIaP z4#l;&yR^8wy9Fx}9D=*Mob)?q?X~tkJ3mOS5Xk$EJtOzbQ0)O+cH$oUXuMx-kmkoo zH$&81Z$q;cXKn1*!F9S4I_&A>4onvu{!U|f&u+e$PdWzm5L%VKZ@q7GIG!XI+4zh| zhee(hg5l>v);qpYQ?+~4l2bFSkah7wnApt)!~da2V$joX96f1)wa`KyHo7+M6s7%< zo?i;iurTphfA`{H0Wws7W)lw-%K&$`w_6pnU&<%b_7O=bKu&zvvnoNl2FwK;!J=Ig*o8lFvee}AV)OTX{L0g?lQVzU z6PuQHm^B_u+d@LR-~Ez`Nt=4Lr8u04FbTeMAM3k?&XIb$J9fY;H$^4Bd>1Y9_rPD$ zyv3*x${oNz++89q-Bn0SLSY49^caul^wygdpS$Rx_-rk0?2$-3mC?*Bu%z8(x20H| zQGu7;IAOH3StZdv`3|=nYq9?L#nLIRHCu+iG}O-$E^WMRYq0rb_p^Pk$@x8CJL!{- z52-~Cq)UP>_F_Gn`;n@DRya2sOy=8wa3Gvcej=-R``97(H0c=CVQ28U?IYAX%D~}= zXEwd;jKFo+0_pt|>VpaRH2xFanRIk;dZe}y=hD8b%R3u(k4$z`LkT zm~h2?nb#DR)W<*AxI3=KU%ko1qFjddK(+oRM2^F+oAPVg=L|)24SAmM)4u1Z*Vmc@ zG%Uiy40URNllae*!{YN;9Pcq^-()B_L`(t=+sNxyGAuk63yskR(&9i?C0 zw$oAg-9~i+M6}#az=-c+2Pmjfl6k0;nq@nt)4{t#*MXVVNPuw5PZ=_H4p#?o&Hcy)~YK z=Al6Os>zTz#hIeQd&nqS(KH_+tER;ZclhaAaoJJs&N19JqDA0uRX*p_wWK)xS_^_x z)2DXq%%sm+>_m6hxm1#3OU_h`eq51_*rHU{`Zej#U3VX;_i1X6!p}iSQ;lK|@3UN)7}xEVm23aM*q zR$HpC7;0s@4;@t0)FMi5`+Bn@kOZmyyep{R^#N%NN1r|svnyz+_(frpsDX4PwkM6o zhex}uFK8BVWn4Q+6sn%1f}UYuF3=GKAAzHoqq3{`6;hv=iAP!RRZI|fRN)_+=@{S> za$xPcPlTPx3`$=*2?Z;^mA;;^!F8MW5E3iwEt^hV#r5$nXd?c;9@235ngcjTJr5P9 zK~qFDNk!nyI23-(Yk#hROSLy6t~nkFxRiXlHwbl zoY~qy-+l&b8pF6Nw=Yj@Rf3AWW#fOWpTKw+yTG~#JDu1dVP)sfqhWHY$+S0}3wB-i;{Fr5C)^{qlvW9U_`jzyYdvg!jB<9;8 zZL6|~B3(=`%1;sZ7+7sI`6$-u^7Vla9>Wq-^;8(UVva}nzuTL)&u=CDo@skDcPk5q zFn+J{My#ejGr z@h@&Gt#l&>YrGHC1>d%PhQ0!!>=SZoGr!+G=y13SI-K-gaHJzx^F$rde41h&mNzs- zvM^x^7wV#t{K#4?a`Nt^(2T@vR0qV|SS+5QCuD947q!_x7+auEg#2)DG=hF2v-cJG zeqieCQCeq~gqgy4wHStqJJDfTB>>Z)y3N zjbv#UUt#5bI44!PbG!HicUDKb-pbE^3~KW%ar-n>0`~8jf6P!m(uAZq3ZW8AaT&C|95anF=Bo@xwjPd0Mj$TV5TWqfT^TRf&S02?otgywi$ zupj#YyI*;LZ6k~Kj|M(lb0=a^HQe~teG;)sl8MF8kWIFGgL)RTHMcMB#Ik8Ai%~>_ z69!7vh>C;XOj*`rW&so_L|@&_*(cUKasrp~1T`bm?o3z_c?883-wect4M3smKb7Gy zP>}}ZU(?Ywu=%s^N6xT}LVOt3w>y}=+>@|wK_ZpMl!X<#@wYD<0y*`!$k5ooq$Hl}j zNlm{EOL_a|dWUiRP(5PV3#pl$>a50=8njd^>YqA0oOaeeg!`8D(z`iBHnm=7$#Rj zEesmxhe)`YGX`HI2!Tf<7)@$;iOEJRj0RqV3ToP7=%*-%=K~EldszH`b0NnQAdh^9 zh_KAt|Cb}dBBX4!S>XR0#Y(X^MeP-}R5`Uw4746xtX$4h>%hA%F&kd4fL0bU+09}< z`Ho5&oo0`%2nft^*?KIyVy*E+b{$7^?zm6MNz#R~KLgwb}?MBNBI$!fMs zn~z2^8ZSM%D{rlrBR7+hQ)iqJ#F3L$+SLIJ#Dl{!*YEaOSexv%UZ`pNw{wswdFVy< zl(>e)35UU9Pl1@#cr`+l6R!+sKw&gRxiaOA`1Wc)aE1xNufC#8^l8tvi6C=44D3MR z-!5GZGc77=s!|Q$v?@d3PD5M0+3}K(jg+0-wx1Z@!aNE`V6S5%`R=D&H%LpNy{u?yL&+6Xp$+b9O z%H9&U{i^7LdX$9Wwf8K4-xvmtuvKOp5gx@<%jVn_5{%kQAn{Mz&x^g9aVpq1I2k-q z6cJsziB@vxH7^2m^+;q)6h5~|y+#+IzQDN-S+6W;VEotBgmG4Gp)#gAUMVNjJ0!f^ zU>9?VeTC4WZv()&sfwznIv3nzzVHPIr25SRiQjEfKBhenChJvC)HPhF6=we zKbIZFTUi64?9zHhrwtxcS%EZ`?M@lo96hed0T~6&&#!!p+Uf$Qbp;2OFE0eCHmhG# z1tVeK%EAr7Z4o7u0HzL|Qp^vD<~1diu2H5vk3!^{Kf=5z%O}Y+-*bI*+Y1iM1TKpE zQ9gYlYMF&+xSb^> zHB+niV)@4l7<+hMRC`*FTNq;np z&v0Fnj*Ro1X>!%qg8VNu%w%u;EUN@kpL?`A9~fJxrD5?7#&*wNpN(r7h%o;xv=Dj8 zR6tSF7qJ!<<*b9-JgJ0u9+1{9MKkK#*Zy>R6!=ax;O=0u3ukO|$OxIAqS4l-zLP$o zbp1|~X(Kbt{v4*g^>zl=vb6pKNf$O0(^fMD_iw{I9Fw0WRhsjkct4@&pFK`e+O3hS z2tM*t3+6;2=YAp5$(grqyV^;&X|#2yM~A-h=p!H^gE3}5J2i~L=lFRv*fHGtEv(h( z(~J0RJf{`zSe<1|PTI9ywx2I9!&P6Ht+MERV*_I*cYyXF3gDU&xf*5{2ooO7DV}hu zBh*e!(ARnHE4j`ZfykpX)TKt&DG>?66;CweIUwo+Pr%!m_U;?5JUL8UvP|RYie>yX z7S2xk+6rSpW+mbkzqkI7EjP(G6eIedlB2*d2xVtS|EqZBWYCC}^KatJaft(9MG$l{ zJdy3>s~y;HULf9;z9X-b(Pg#@*(*D0{L|NEsw*d24~E}RJ2`ySW&5KG!(HOSdUwJ| zYl8(ZBqMugT8kcbX>GQ1m~L}xEr8aIyBPH`GHICEXWtxF#x0?C4BDT=3yaP*%m$QJ zIjy#P!Z+$5!*g)~S)XhQ!9sRF|#uZgBY8)Kwqgn&*;MaE12yZ3AYM-rB#y>4%9bax0)jtGA>HCcbAx+ea1e;7`^T)Z?|Caf z2^#%IoCGdIjh~nQvXrj$*_E{m@!7r6fAwi$@MSzWXLk#*pp)MmunFcQy7lXERAB0v z?yfW3FI1raO#OvkA3l80V_tb}V6}uTA`3;OElVgsih;q0Osuo^z8mw)wTBd^ImQsq zYJo;AYhAYTxP%LtLKOm(z>&IL?*ZR@JkhIc12DGEwI_T>6adIAEBK^9<#0d ziL-)3)}JYQej$4DI)&cP_-7R>@Y9qY`;b)=wVcVS%2F`wv3~lryF9)SVQ0br8b)BK5%>+Uyj~ z)%G=QfUI+3#YKu5KpY);yxrvSsV*?9g(f+36U&!eeej|Bv5ahlStu`0LV~sa9KHK+ zCpxw*khQGgpM!~Gm3%BC?vlI7Id-dkf2nG*r@eg*c5#2M3#+kJMp7expUae=LqDvF z#7y`~ZCTMMj`_C8hI$q@HORn@T^5(7*7ytcZWum=O=mMb+L__)9Za{Uty_KuRON&> zpBWl59!`( zx@nOO;=~=mx-A)xj0;)JyxOpKALeek>#3nsOX=Ocq_z=y*o-0~0F7b^;FZt$#GGmj zdU{>~g}@I_bM~#zOuusHP@v6_h(8NmOGmu5h@+zb{gYTaG~R9fDsMRXdHa#a{;!X< z>HQeX)?}{bixhCI7ct7b!XT+)EKf}v9p61?y~7KVtFC3&WBdp`fL;W6TMyFD^{*tq z$}o-hN3dN7(oyR|Dys!`89MIRePT2YjB2x z?Mnntwns<{5u|N>>%Eq)E5n^!dp5E*tk@=+?`zkB`({TGBnQHXfr9@;`__CbrUm=G z`YKe>1H6O2ICDZLoW&mZ*J9%Q?!!@<7|Us&%~%*j+U>FW?0n+}vOAx_0Y?Xz?(XYTdTy61SbY~Srv!ugPF!y7 z7aXDs!V}xUxAz9ctLG)D-WAcREU||7_ZH=~x|7=_k~7zBLoMKr@0k{4ta)#c0eK(d zv3480SA^yy3osN_angzuZ1w7H{{Mn2e{?p=i{(CDFrL1YqKdvg+!YaW1zc|G_2OZB?f6d5 zip6#eE8M(3n7M4b*M>H^@O>)v9tDVTcBPY2wBoRA^?dj~Do^Kn<4JJA76a?_DZV*n zZ7;pttl8yK-B|sz!E%5OXk_3w)e$1`i!jII%eW7?xUTj5y@H|yQ0u>i53;CACk~rG z@M31R5=iA-T)6b$Sbmp%&k~0u#Qu4C^~&*5x3&T}P|fz+ zrkmx~(G}s7b~Y$^!6C~C$GicWA_~0iehQozVm#d`pV49jGGett*C;KJ>S|8H_qn9Ai4cy*@x|$F!3Wda~hVX>3qXpqehW zG6ZFLIfB%`%VigA7m|Hk?@ak@g@EhFpa)l#Alw=D6>!P!Zg|mzX^u+1-q}>s_b<}` zdFBJ`*gT}oJ;Ah?->Y6D2zloy-Va1IzlxO$Bz#@d)t2s~NepwcCN51ev@xRmKyw=y zvCgly6Q2U^CZ0|Y>t5%E4{0h^EhUh-8m14KQj6EHg1@xjimSn@Dv?TkNX#k=d*&io z&6ddo8tLGW-fl1VV0(W<-*8O&nN^eZLE9<)M??RZtTdz=dRtmqSzP`{+I7%Xr!qnR z2c(PkUyp4T$~!63uC_!{l~beL<4W)Q97WCcK~afytCpR!zJslVQLj6oStQmwXtDwb z;BReY!lLR**VjriiZqPx%0}(XRWx*ojUmfg`Z0i{uhMKK`1|o0lTk?#%bl=5_`Ec_ z(exMAs&&Xlnkr8 zZJW%=t2fW+mZM9{ru%Q3i=tai9w^?Xq;BuU)^@Ah!2^m9bBGma1YCAIM}q7vfnK&9 zk#4vmNG-pfpHDcmpEt}hNf+7Q5Q{lI68%;lgs))e7h5jhQu!Zj)bLsnbmIGyGtRbJ zT68s8El^ZCg&YrXFyce59=DUV%Q)(47aM&9eqx!ed{{LWrgl`q35`MRP{ES8UD>h9Hk%i;x4CmQu1p);rJ^T@t$^YN<4zh&! z+);hQj(tXOds}_--=5rGiagBX`yfQu8@hrV)c&@sE0Bfc=X=wy;$+TJMA>Il{LhLk zpRh0nN_J~pT7g#GRa=oYt@W|LgoH1Ek(RdGvSy005HaoH;eQrshcA8&Q#*X~!z23K7qGxjDeQ&D>_1hDk)`1}99f)t-$ zUthQJ_|I>cx_4Y*g#Xvtv;8Tb^8M43@;+l>`RMHb_Fqt##KbMu#zU4>M3oI85V!xi zR@QNz?NLcSjLUyCLJ5gM=YQT7B+7@>o;CGie$_^#CM#pBd4B%#-@ASx_oF}`)YTLK zRlw6@$|3*!{Es92m+=4fh_8!bWBe5S+tOZLb@~%4RMOOc=Mcm!w4j{M7?oac>>bJ4 z&rbdC6sxQpT;7*gb-_u>j`&|c!B|yUSUV%1*B3rp^#3mtPMAgYgkiB&jN7~q#rsQ7NM?vlhy%;&tIE5I9$wiosIGso0Iw)(ul1)QA$c#YQ-8<= zZ}l~A51u}mCHm8)Du}<0xCAkOEESBL7+V7X^Y!iV4;K|!ZK|5HHea;Cd#Ew;Z#xBw zohDU&`qYA~m;h5|Ty6Y3(dwt6yL)qK=o+3?x!C;=OKT1y$!_{obYr?iso|xDlNWcX zI+xaXAJW;gEOKzBm|jgEUBmk)-^bYKv%Mw29<$!;oZX>@X2Eg_(V$p;JeGfrixt)n zfbrAPcM##x#f_d$8Us!|aN4Pi_onEXsW^~aw90Xw@dcK#oG7U1+Y8ibyxp`hu9hRI zKtg5;x!_ykvHc^Kf(AcC0pB!zj#xhV;M?|aqR_j%U(kXB+7A|MZH)3vzp{|s5)Me@ zzndcrZs}`sTj8eZ+5zlS=I_4Yw;=@6sOKW`_W9Q-Io&fWBrB(LXU zGQ5p2mqa-f!@lE~e$wrvgDJ2RsBAvt_iRMYv%W*Ze7nMxCZO^=Re`9USsDib$o1pK zWE(YQx>XYv>lDFEU5OETtmGVc$++^_JSP-lp*y@ElhT5%C_!xtaw^)aqQzqK6Mjh# z40ZNi3pZ{y1;Wt(oZ*frAumNH+uL~6PKszn%r9^(6qo9T5$zG1eM@z2uK>Z9DZnRO z=gkNPmhM9viu#2Ul(TQWJSX{sv4)@b2~CCB;w_vw!Vi~lBYVF1 zjuK<~jZip&oYykjk}+AZMA=RZ=bTBq02FU2%gMekO9$Y05Lp%jCh{$0wFUmre;Zw6 za!TU^;J!UI1n=TGm;6G67^Jxe<;0>_F4dGBZU1MMCPzT0g&$Cm89%{Y~t&u)2KK0m&)*Q-JHnfEJaf--1l5VMwcb3CA2br zy2HP(buPc*o$vZ!rL}={?{2TORT^y(GaOepcLLgEj zJSlNiO+6G<^~nvjjAw6aP=4)_PVl<>yV{upm|4N>h!js${>4hM(4QouaicW2HgNTr z1-;wyf`+r0WIl7w8c5fZ)fH%>UqwE70%!Nu3g&9D#}Zp^F5CapOXfCcvaVDT4q@1O2|uYniHK3+E(V) z+Q6xji2e$cp@I-Veoxk_;N#IEsM+G`?aWiMt<#)%G$B17*Mv6 z4hJN$UQ**w#5)S=3^6nO zKu81MHaZ{2o(?U!(+MH*=BfpeX<+WHb40npQEPfvfzS_u#bjlKW>DKJjVqR9WZZ`5 zYo*XgiYnyqCQXTkD_4UtM%hqE zrm{+N)Ac!54}NXnd{)Xuyo;ffi6S}#kz$K|uQGE#7!_~a+r>0qLsrG%MRWf3yIU8H zIud>G(Wva5q6K0$43BeQE35tX1WJ1@)BKVpt{2{duZpIq#X~?$L(bPrI28k^<_PpP zY83(U&az!L?h1-PHJGPyX-W07=aBs032QSey4o$`k<8+eFFZ%UOrwTjSc} z82{`v!D7H`A0$HN-Yn1h@J=9ho6If7$ox!Uk&*d$`KRXPbXofYNbC3!Qq0Me6k};l z+pK{d_jCM|aY^pnO}R1$Q|hX+?F_iAs!$3*v@*5|)ERKGo-8S)O(D4|xjMuXKvOhE zIFKa^LQjU#MG;Q*?$OnnnY#N0<;Slp6e1i_mHHYhy{!s6>$1WLM^=#1_6L>e-8man z(gxWF|LHcU-;qZ#c>em*kLT(?g)eA+zv)Jixe%GK2unFu`eNW=#Di51X+zXr@vZhg zshC^kduM&;jm+C2UzUfySB@&E-_j7DFa2$%fP5gIgB1o+`Q}e~1i#zGnRW9JdUj={ zug=0_vSV-|a2{?*#whxFLt_GQlQJdDwlb0Z84~2Xc4q_{$#k6x97@_JvuDzEsn?;N zEwLo!)}Zknz_cv9gX;|txiX8beX%ICl>>Bd7?gKz2)Yrwp*UPUL}?b{1nHNElcqo* zMN{ruEPP~T455F>gg3A3RM3bf@rC2G2XATK#?&>de+8z>N^*Rvr1FKcw|xkF1L{Yw zQx)J(+F>gUD?QGa~4qta7VlC!-)rE)xfzdq8!hZ5%B`noL1S7 z6qoR6=g7!?8qFCU3Ow%{i=|>+-#(S(dn}^x+-KQZUVax!-~~(|lM2HvP`l;bwcLj$?;b z<`jib<{|YgCZMA`_7jmYw%g7C=#xNSD~!c}07~Y*t3Cn7C8e*!{|g|_#+b&ZXQ`Zc z9+*)Kso(M)L@6up8Qiidf76lkP^H@N8&@dFAnkYp(=wLU(GLcU5<~sOTjT{LDdj<2PT%f;Me5rX6q_!j5Oj$(A6c2##T0#KN7QpQ zf(9l0{r5GE zbKEPNx9wJoH>pyB+gIK!9-ie9u=z6bNSS9!OHUKWa{x~ZAxH9W=j6_WF9e3ox@pF0 z{cMjH8HlLPb&zo2E6^2uJ7LTMXudeG2<~{My;6}qMBqx95P6l!(azqv&}-p6*4B6f z&{wc@L~nxh=RHD9wDIDbH*-*Y%(;VfTdZ}ve!L{7^Q>R`R#L9DGba0zO>AH!91!A* z&2}TXGJ8i)Jh+fTrbKS1t`O+IhaM&00-aE+uo3pLs)#-S)?MP=&Z;jBTxYL8Hnh4MaG-f3(@t z;a(Ie<>*tbrz~7C$gy=*+S2}!?YC`;`A|)J9o{Eo< zYG8$)ssUcA}xf-!fonYXo-M46McHd{8AK5ugo)3QW9 z3^Bn(zzsjTb)<;X79Vh%j_wI9C(3-R2uB|i|IMgQsdOEGWNJP|L%@n@Z`-csgBZdE zv4&F7n6t3jJge6Y+e7cyztq4lm}$OA3kbC(M{dd_N2k@af)a1*P6o;HCOqWR&<1}H zTe+-$T1;Wi^s5DOO&kc7BI2njnC|6xB>v$AFXiHpR(c%k?v7bxWZ$tk)Ec8VlEoHE z{vl32>$2Alx52Vr<|Vqkre|Ou(D5)3!CtQyDh2H*2nPQz@t3+QE2(S|nMOorsb- zUOaqEqaFKUD@m!-XeYFhCeMCH3k_;eT3Q;PhzMY4D5@t3{3 ziD5|ih_7QGdGH;>gk7Ra6AXFh?DJGZZJ`qD=ut*#fw>M}QpAQgKaN877Cr;(un_sM zfAo~l%v+aHo%a^ zs3WX8pwZRR{iUI!F5U#0ms3q8+XwZbnaoZG^XbL@cVJBQO-a&6E00u-RJ?}N+}|3A zF1J5=eSvSIZ-H{|XF&??nVMhYxRHHAVuDXP89`~?Bu&w>{3>7%Mg@cm8Fqz^%)lZO zi$6uG+v~LM)Il0ab;T}Ey}99SL7ar#w}oEoR1ey!$nQf~|K1mpq>H zSe-`1`dU#&2JmFNug5U{%)+TK8OgQAtG@Mau#(gs9^I$Gp!aPpo!Le@ZdM z!uIISw$6@GT#Esdpyssho8X}PHx`taI=(-M5Td;MD$wv_{r64rs<}S@&!HkLqQy(8 z(a7xNzp{79P9MGO|Hb-XYhxrM24;!#}zvk z)8oLh%w?`nzBaw_b1?++eg?WH;-0c0tmpi_e*{V!TZihpjWl?z6g@^;J z3C>s43zLTf6qzL)muGcu#3vpMy$*C6p?r-_Gm@@SuW8dM#ty>?UWuS4IV1n*sUSgh;`p(QnCEvaymoBY-VgPu6&=DJ3cX3TQts^$@lmN&T2J} zsAAa-Ff@OoGhMWX0=zFNl8^9Vb0MFf$2r$!y@?MX-6=085P8x5i3wzL8(j1gF0CQl z<1R>JYoLjlUY{yDGs?fY169chg;YF;uZ_lDznBw}AG~>gCGYoJ@3Xuc$l<#nHlu*Z z3dCCdzaJ|{hIchAy1Z}y{DLL)5{E&gUI_6A!-69eYp3@E`+{R_&tll|3NfpvD;5JV zQ#n%_Z1LUsu$w%hCOyD+%$77%&~CvBiWi7TKClzc6kE6Sc1P9Mf3MV0fcCnQTT*WUrOg#ku^MW9JEgNo*NWiM0lh$d= zj*g$^kjrt%*!mbOd>xMP2kbjStH2Yw9>u!(gz4c9 zUpNaLuZX#)OkZa=z)jpjiRFJ&;yRt|=7$s8wccz_K`u`AxQ z*-p975B#>+5XwG4-aT*8W(F+3;rBYKSklbYZo(ip1zSYHc^Fdgr{JoL9M861GTjS6Pf_>2BK*H> zDT41=*xd4q+|&ub-@B3WKtw5QPkv`5yTuz>lvb8?|0d@9ETIafP@wq|Bd_fjhMPvLxD>;t* zk?v`vb>`1mgc}15w*NT7yaGYL@f$CD&Y&QURpt?291+f=#MO7})r0L-?580V$rgaT zU&@Nz|7F&j!(cpH$q38v(8@}nRmcjSP>$cWwcLB?uJRI5uxY*R`bplCo`S5&>$pQ^ zbhx`h9hO+w8|e<0qVQ`yos`0h*zf?J5D~;^Z0!^;da}9p5T_Tu4v#DPgfYw}DoUXl zu1N@G!B&lK3P{yCXk>?yd2i7_@UMO~}I zy=tLOJsXSy8x$jF2@_2$dLXr)f-5~Ed z?i3deFZO+G_xthPD^2~pBVo+9_b`NRWV;F{AzCRBqU=aWkOOU!H8&DX@~&_d-^$*0 zAtyEo#q}&kJf(`}b@;V}tw+p~=1^57OrJFLm5eMDr+n3uJy(Td$^XTi3}@eAGW~Ve z+_6jaA4z{6B6v&B2I~90z?J}79Kdjk^Ip?Vk5VzjXj{XZxfvjiQA_JnkW?T$grK%a z>2L)g-WnJe8yxn6CU(1CUT!>R0BH1e-^|stf0q{p>d+HKyCtO2qOE0VUtDKAuV^a$A^JKsW>Dt=%LleuT@PfK$!Njv`r$q;|=g+O|SOM>MEAqu~Cs zuYXPS5uyo#UbskU7(^P?-rO1asNI_03|u9KC6ByXl@!8K@(TA9k4#_lcABj2&tve) z9DJLEVp?yaG^zvaN#aX)Nn}DDK%|R#UeHBl*U92-5LpsTMgzX zUg7lpnBZaxbWg>`eq{ftRXJB|=9tSltWd;X9844fB!9^MHP|nH@d*i?U zof9ZB-BuxRTY_n^zx(-H_s7mR82N%W`}|&WZn?x- zONe32=7Nou5*b80mc&x``VPlL{u-Ie=La)=fD|M2tg#CJj80o`egs*HlYQ?aoBEtY zM($~~1Y8={DD$~DYn>hGLsSh&j#E`3XyW~Fa?*!yWe$PML7?x`fbMTwH6=UU7AQxD zF$JTapF35@b6Vn75#f~DrYsg|+qIj)I?&G3Pa16o^bBab8*DG?oi9_Q=ah?-aTqVJ zQXQJ>%5Lgztt`9>wyx zc;j6>(=$R|M^+ocW`nBzaCFVsv+*zYLx;*bPj!z^mHjqolC%9;S~0gDIs(#s%`DEP z)d_j|43dev$5{r!WDIoi5QiReV9u&jN$L!TM97CC^651L9dkDxpO3DzFss-EW9yk zPqe!VXAz)Nhih|CV&_#*%`t!I!5TB<-FhOyU^q=8{OL~8Q(eiLv5k=_wpDxI^ajtr z(M3ukxhn&QR1^ym5GrbJ5%h~(HdFm&evu!4uG>1D zS>5tiOLS|H^aon2^IxM*(-m#uq+5w)1X>MDJnh=ZIpPZ$VTTS?m5GR^NLT#*Qj;@j z+~58lSwm`h1dw*IZa!NKpC#`6v2CbCwb|6^^^WsqO(@gG3G+PAWHWR0`C(Jo{UJ>1 zVwLdkx3PeC9Gs{E8^mv@tJ<7JL*oXVx-_iyMK9uTs`jV+8C7jsSnPEidTT0(HK!vB z>(>R9T0I~yL4Y>xarz>uVpS*O+w6~k1e&SJpV_MkUaVJPq$j(+%{-@}UEF7lwkH;R zrrdp&R+_@~blG}7nqD>Z9VK$PSq1N47m#HOJz5eTebC#}xeCvy&3PO-nHcE&16?{E zHHxG#d`A3#zkV>4I|#uPOk$oOoCfR)C#qpZtYyGg}rhJY8A}7y87AsF2Q!0jgu}9&)-`VK~ zP^_%DZcz{WfEdme2LmauEt~{5On{8l(XV-njb^rC$&2ToI~`+;*HYH)&xI{|l4SI* zPxoSgeg{H|KP*=~FKV2Sytv=zazQQX^cz#ee93vzdNHFnIX=IX?l6aG^iY_+Ivq(^VAk z?9Vn)wl%$!R2G_2+xy0QVMvt9WBBRHH(Vk%VBe)+e`Pro2cB|z@YHPfvZ6mjbpWLB zGF6ni)H2%`K(%RwSHnz;W^?ST#;B)Q|1AgC_7>ZZst{RkU{{I6;IkL~&+h=QOimfY zIJ1-A+mb(iCRP;Bx9Ra7ygR>H{p=o?d|rWkv6>Q5Af;jn60}BjQx=#mY++D$W9oKu zV!fyt0&PE`W?&JUgJv37KkOPt;rt;)wR9Z*D)+^W?dNKw*VcoA2=~Cc2;hgNC#;&1 zsi8jBn5%!1;z29EmF3={xvcd?6MKdiJWSMt@W(HWJgG1^F`vc~Ri`60tTUH>-h}*S zm?hYFa#J_8);15f!RAUR zVskbakP9UyCWgr8_&nWU_1C(-K}HGyb*WhasWgub!i)%vK+6l(VuSZ-h9X;Swjvjx z4;}983bt0}U`?`#&d<&d249g)AMnZY2|Z^Di}0|za>Q-z$hcQ=gYzsFbHWqgRjww# z+%vBEnaR5YP+hdS#7WvBY&Hk<2Ou$kZy)v(A_`G!Lf8RPNU)DJa%|toG!z^Oc_%o{ zPMGkFOF4=Qel;F7xPRXwj)ji{qBiYNgGjEn9r*KgI4)%tws36tF)OVR6C+=#q}83h z$9ZxU=YoEqrvF5W6R%{hWbMQ8Cl9S3gB$0ZFO?Pi_Wy#xhZWrpgfZK zx#`s0_+f^Q6-juEj3 zJdSJzA^A`yS>{S4`6+{W9JPhg9`V zVD1PVQ5_d_5Xkikf2x|lDzoSfIo{rCZi$VXXDY z__+6Uo!6t@{G@2u8`Mbd-;kjbh0gGp|7V2CmD-=*bBQ51-w;4}qNcAtchpzIoJ(y> zDiartId0yIKGUDCca`hi?7rTO9EEe(jrzlO)f(EW6BqS|L*z;QM}A(=_tLAE(^9zoX%|%{DsB z<+BuHA8SXOfwp>hLlg(hwr9LJ+{_Qb$_r7Yx}yE-q@OLO9PJ%MmU!&*(+h+unWP+* zsy>?{wKB5q2WqnI`Upk>p;vqw^Ho0Q6oW0H|JpdGA_uaGC6l`qo-hH8V zlTOr-SVtVI0E^wM_<2}FaoqN!r>6Rveft$@H_;2Xy&GoKc!Dg!pBI^Fbdl)<4S zfXUq*x#w#eYc4w+U|DO#C>hIMQ%Bu-t!kd}>)^tR2zXg(wScUfwPE1sNcUKiJ7)8$ z@nV@QKKQK@StMt75m$b`-q8teV!8iT9P*|^Fw=TW|JX14tm1~p8p{Rl-T%YZJIBY_ zt&PHI(9XTBy0w?qR<72}f7H zfHLu8zH<4r;T7lIGoJWEus%SCDeuCgb$nk=?Q9qRa^|OZ0GK%pp_-piF0QmtC^nTK zn%(4TGw`>C6MYacu}GOJ@~{IqfEFE|cx4gLa-)+|eTw{09KnT0#eg*H8FJl{OdESa z{am7_4d^!f`55gO$j;eNSF~K2jCVBl@zoDJlsZ@c_LgWQeP7C zH<{~pr1sCB^`u#$-Gx}0HEs6qz|w`8{_fFlr7P4vEv)g zCMS@-obOE9SVTZ^6!!SWqvT~1T~N;fW29KVxal^nxd_*qTE{+;S$RUWn2-u;)d5mB zpeTQ-)LcUZsKw!rqec0Ijf}ynA*MiTf`d!0HB$=qUl0wxKV)dz7??GkL{r^T9lY9OX_najlQzU}7XA8Fwvuw8c}6HN^v6?l)}14hsr;fDziw>%T9;9>8yRdsmh@zgI(q!c>7OA5Hatn zoO637!oiP^YHa<1crQjMmS^Gp$H;V`EB5Ua&R^Y=n!{(ik{8IKMAiI2{nNk5wDRvp z$8NMsv{j&oh@1pgV1C{9X&c|S6v)EjjmL1ojvkqsL&g9druD7s&DFxvk4meuiR@KS zC+Yn4w2+U4WuzWtvRQtLlSJ@&#~W%yNrt?q2kd$zrVb#c7CXc1YpTC%bv zxBdL%sn$xY{ZXv)Nf$N4ZOPXZP5m*fpn8w|8m2|Ee;nOL;rfG4*rPYr9=|7T;_{Vi zEweK6ozc$IG&vB4=}zBxtn@J~JPo=+N+aP8Sc-?y;R16_X`fgTIc`o;>1t2L>4@Jg zv#R0FYlxb@ckhX`Pl8?dMHcwU1kBf{hEV0gP>&q3MXY4eagJ*WocQQQzTJI`mUC7} z!o=<9lvFYW{ic~hIwdJPpvb`S^U)LmIzYLd4D;Wt5o`_H>uzMG&(tdA{GCg$l=RdK zaPk;hWYs^o=c@$7qTlU#H?lY!Dfd!qNdA3gkkx)iMK6ZlO33b~Su&SLgEp>0kQ?3D zJW8!Tyu2YNJwVs(h_h+JaYLBbQEIn-^>?al*p6mA&#AlU_UWc?tDRzt$J4Z2@ODR( zFoGF6C*pa8#Q~d?w!`xW0Tj5qaG63oeXBfVI&k~N^C-JSt|}_Tqk7|o5)S*Ro=S7E z-^w&fw|58`3<=`+jf(Yo$ad{#5vzF+&q8l||By*RN2Do%*zTWcb2HRh6dJx$aV;Xq z{CF(pUY$KEnVJ^>J&jhHwIxTQkHx7tA}r}&a;8STh|53gyg7quGyikmudEO^kFr)Njvc+WU*haJ@7p;O@U?A)?o5;n3D?bYC|^9ocZ&08Odh@`!@H?oPZN%1 z@QH6ElYBBoykLh?qH6Ourg_jG{lSVCzUkhh{p{|wQ*%VaeZ0t9dkaT+M)GdvT@FzO z3*ZNO@l&3U!4hBW_tsEy5S4YQ);R{pl&ZLk3?R6 zi-wM$NAcn((G^xlcbsB3_El=1?7F(1kw~A{vYYiJvUvJMIt1W7K-o%DsvzWR2?j$D z8rhAv&%x+)BKjv9GE3dLN*^14{d)%& z-D9it^_iplvhUhK?eFSWFofJHRO9>fqhg7QJQoi!2Id|wUFRoYUKHte)K73v4B(TC8@Erla4*adtf+WM?L5%!R`o!Cg@AE4-|Co1`Q5f>V7dg199XRX-8ws`r3 zoi*b6_bY`P?~5a!K>J7*P~?jubVSeF8Rrit#o63&q8|K+X3W$gF7`M9owEK7jx-?X zt{PzC5v_dN+s~Fb@*G$~xXQnh)hb-H#5C_kB!r#vwN39!w+B%r$R5*YQyzeED0nmV zb!52!((i7JWpUM^hpng5Vvn7*pqXivFJys99`G{c|FAo)>L%zl{{oFpeMWJe?RD@F zumBMcA_M1Yl;ebk>q(RG9Fs~LYPF!~t2|B43MiO@f!eN6mc9eul!BRyaL8#ZpEViV zJ0}VxBuU!YLO?bpgVpq+s`e-33bvzQ(+SRkKvje)Nnv6jh`vDa=W`=v+b_Skea0%v z2W5|z=&<_>q^WQ3*|%$AN9dYKg8aoS?{M2(h%MtqwhPjTN}bD~Ars0mPe(GJ8VDO~X-3M(lKGG)2M)WMPL7!IeyO_w@LnaQE= z#Z$`8GHbn)*)Dji+(>LxQ~;{KE+Hc8f4(?9wom0rzTM92czv+m?n7j>I9>%%YYZWD zC*{zYS-umYO^NotkHw|240nGLs{Dy5mpPICtmGks$!r?SAy~ZpD}d;Q2qk94r+m*w zU)_xS3hM;L@qf5{8(-1D58`e8J0>dZk+S4u(1+~DaCb-Ng=H6I9 zALIYG^QZEtdmtYvaL3-9FQC{ncmJY!*qVHyjV}}*CizM$<2PUy3l5UZsBGIVN8|>p z83yOCTp9T{!uu-@869$WVR^$hPH=@s$M7*EHx*f>(Sz|wXqDkaGzl|2zQj0(m=*8H zXUdhn5`D|oR)nMKAjKqq{o<2ixkh{W{rMPDueEoisLR*YTG7*1s5)&&T`AWog%MF@?Nc*y26rfp$zS@Jz0#<-PV4FQVmi zn$0d~O>~-K5yjI7ERUaR$;xya4kK=`YtyeAb#H&fCHEXL_ER!O2AeCwz>Zm34D(}b9?10 zFL0XBlK?gOwM6|wr0kohhzkxn2fu6B!TF*7V!1q;`?x#s_Sr)<11gC)gG2o4v^%!0 zS2>SN%`G~!mJIgE0ol(}Rj|4*oH}&KMOWpXF|r7_IddxB5LOBSm6u9XM+pbXjEd%H zh=#Clk3q?otI2+@W z@%Cy!F4g6*7(Jb;S=9e-7W~b*4-R4HJGttT$;7Xku9?sD%7vFc$g4bN3X8Qw9!@oJzQtTsM7eG~j_Q+}gst_@FzdN?Q@d7qreIc+vz8PEZ4mojj`|bA~$Lu4an>cbV)}G zEc7Dbrm`I`R@si>hlBr=8(dMNb%P>|J!BQ9#&GnyXBGirjF~o1TtwLHt)86MGen0B z=4|>@e<2|EA@?tK@9cK2o^aj%q26Q)=Xlz+sR*UOCxKUvc(1+D`m7`;WC)Cbum>Op z=(Xgj_j^?gtQKF&xo`BKA#$-(ol#$f!!s)aWB1F0?H=G2ha+xteRRLCX8Zmo-qD$Y zXsnts!v2eZK*2AFi!01amvCSNt4$lHfPC+o(Z~Yywc|A%LS(N zz3KA9413JAe0!zM9N^6=6X?F*Q{y+CyUvDpS{N}Z}J z0TfIK#^G&+jj&*aeE)YXXlp#xIh^VL%m_HU{dZ2;U;U(}JN?CH)*h+T(GfpW+1m5{ zp#`{d@na0j+H}uuFKX)ktC}R`tl<+h`|&jCHyf^>ug&7Zf4;n_K8D2MFQxzKiZl1m zi_WzmiCg9GGLyt z&M%SrU;n?O!9PD+go*k82V?$c9`Fm8F%XFV^VI*o8DuQ=@&Az_0*(kWY@_q|BnW_W z+nBAb9Q*6MOOG2<}joLtbVanm#PQlj6=dKVXNrZND(=5_ zOwbv9!A7weHwP!7klfi58=p@*(n(Sux$_^+WQey#UNYxD+b6Q?F5f`T4C++dHPR(P zLqMSv_%wK2LG6~08n3sMH<~$5qIjrlPmTX8!2G`(c(>FxD2~b3D2zRU}JeIMT z?+g|e4kHqN3l)V3UHdF7wo5Itdw+{hOcol;`&@+x5nsq1Kap#UEmigA_W^g^r z{NH1O!5()pWIW-Vh}&M`rJHj8*8-d;I1~l*$p*EUgAoGVt%D*F&PwKoNwe>=H}WL@ zD`Y4%zMdA6?Kekj+gXfnbOsRdsx6mLx65AMwS8f!*IYrbE*~##p0e@KhM2X2&2#n& zC5)_ZAKV4mX!nn5LAg!zHXTj72AWvR`I>4>wzSUzwgdOV_$(XD&&FoTpD16X#Oi5q z(s_W#i*6jeR&30xL(8-HmwOl-a#{V~kBaYYc|jJbW-7fqqCe1`WQhU$NVaetg_`Dn zYw=}vlgm`eF%9}kws-O$HD|zeWi}W<4_@MZ;g5z`4N=YY*f36{a!tM9_JkJUL(G1~ zWOFu?J3=^DZAR;Vg<_A}{9Q@gPSomrQ$J=(iLGIAK;_`#KD^vl%T@8l@g)jM*FULZ zU=MJ%Ck+5&eoB?Pj_+3(2h*v!M5yNc-;f3CDzQJIfyUH*P8h)nisIUbV{^T2{)?0& zf4Ai7iT7WUH~CBt36mK zPx#08CX|a(y7{yK`SEVF98j2rf+4}}nhk-2Ioor8gPpF}r@uxqHf8q718Su~QUe=D zXUGdzy%4n8;#+11oU{e|ylY)>WkNofMl5`pH~yF>nCD~Zz(Ddvdq&z?oQ^`uI$Yz; zUwdCRua8jZ&&?HRT3wEbtbUPpAVMax%Z>5+A3&8pc<6j0F>?=2QL!k4@)Yh+Sq*A? zCY7lzWoWv@xMnB{>F(#da?KOgbLW9g+beyK^?TK-R+BWsVvvn522a>wUtVSGBa%=j{1Cvgn3e z^XOVk-I}zXKe9HJjvYPZVZOdxfee&}e!*W+HN^?`lfzR{V%VMa6n^I@92QaMZA>}p zA)gT(V|{7z{Ud5&{r5Cvun3p9{Jg_3=2z7elea@E#|)MR9h*cFl>sEiZYHvp{0*O9D~}^4z4MlF!<_xK!IDhC^W!X~IPbV;bINJx_xyT?$g zZ#|_v3KGe~&$&3kNisY4>i|1IXnk1& zAveCd=jZ>6Be(E>%e?gE7Pm<91;sr#I3^zSma?m$evT^OlU%Ds(DOJ((aGnXt6ko> zxTDrSkN&x{Rn}wjHs(!1Y;`g)h6^XkFCg=AaXyRKJIeO0wov2RHG{6udN5&vBc(Am zHfT|U%5?7O&U3R*OO3Aqbl*@p z%dY!7T|n6t106ZQhQWaAY4f1YE~Yiio7_NElQDV%ye^Z%Vya7`+FU!$ZHt*LHizLO@_;t*<9QeUww- zwSxh6;vGWizR__hQ_rW8ns5pKi0CbUSNZwo!}d+KyZSD#gxZ2+Em|1kyWf(4?||i} z=e>UmNoQJO{%y5po3xLp)I+iTBUhCEgLJ4vERP@i;dNr1OIc@I)%9IX@rn`@2YKso zJ#y9>T0k)Ueh#swS5W2i%`)~oPF{ApP)gg6%%<67Q&!~rwU~*qUMaSDh?=s^!XM&g z4P8Wcxr-L3QjS5d9ff1oe^o{{w#q(VA;t~#Vk!Cm#pq4zZCU$FnGncLggqkh@!sIW zAV0oK(2D39yW1bzs|6*A_7}`uNh*<3*D@`M-om4lLdjAhNh`nFK(<|pP7?3pz@OYq zsZXPA9hXzWWBuB_TWS8xt{UTmy+S|7+OO>fh9|@)0=|fT|7FS&9g)Hz2=3KN0WA^z zYr#kx>i%xkrJxR%M4BJJ*^lOJ8FCbQL#Y{#35r;JZtMK*A*dCzZA1#_0@f=+H zw%kq9EhP3G40CM;Qq|c*!F?ILST?`s*!9IaGa7m3++#7CLcKJ}_Hik-W?Pt+#gsa{ zS_?-#0`rqtFU$2d2$DXOTkBJQx4Lex7^hX zN2EP>=&vrr-6Odgkl`5Yq*@n=ELX(tp}0vkH32sHHPVYHn1cQq z9;Dpnho>PvwzhW=Uy#RZpQ9Sh1?{)(em9%5=*h>`4oE zFPR5dBzz#ayMe3{SUFS27>vUFW%-Nj@X%eYTn`Nk=bTh2@o>5Hw#reh_9E z`#d?FY4@en_l@;$8m;uvt5f4ZLm3CBPKP93W8Pzed1eETCM{ps_i5aY$bsHgl;E(R zx!$Ke0iOKbWgqNM;9QLNu0e04+2>W>u0qNR6_h*aYgaPkWPp0a+m=_c;_yfZGpIqE z*-9Gi@vQRTinN~NZoeXpk5yYpHsbTX-$=Y$s)@@^{U>1*3m3@}=YdQE+~_??N+>_u z01!D`e1SacV`PeWuju6s(cx)_S9P%MbPutu34n_S0QBsr0t8CQ+KJ3EXX7)s7Xew7 z`Ls>Xmo+!4Fhe)bt%xQ=ibjWT*s?o%vnIrA=R2ExP6SztET&uwfwbwD574PXuc7Z1 z2>#%bjp;@=5884}f2&Uo?##hsTG2nBULjt&QuN;T#Gl*Nhjv5KFt0qNYpD9Rv7sDX zWpgZw*@7;(d>@xvFKghs&x?VYHOF218Eie=eHkn0t>9$hFW_U{I9`e~#@;-HhCxTz z?Lk8+!EzsJwpysz4L@Ej;c9OFKx$|)uzo|L^CtH_f^gatBm+0xhS`-gRf@^pK;l4! zqkFD{d+|Dvw16M*4Fg_jNhW)}U#pSXj>FXIzHgcop}l-ATWM``ds2vAw!tE7|7Om(6_gXHEkCvIU(H;kQ@c1*7nohpcVqZ5`BFRcfbh`4M$eS2EMED`rxqU zPtS=37i00Yr>}$eqX8JABlAkZ6t65ExMKY!+Xp!+_GH~^Tk#0nJ3Q+>q_#FWWE!k% z;DE?0Qb@P%D^B&nSf4Mr3d_A`3|FKbTd6n83frSQnJWHpb{+e&pE- zH{Xqpp63PJ{AvBo0ehxkCEZRl`t!Sy^^rbH6ZjN6|1uU|SS$x&>6EHRsdE zpp+c2SX=^!-NG{w)w(z#u9uL{8MbkZzm-QD4+yz2q=ki<+a4I#LKWA5Y1n0Ix?ULGR5&$}2eCVbBoW&+RYiS=n9oo@Yq6}T zP{AzpkCGGgV=r%5Pw85}?P+sc0~>zqtGTJ|emE&>L(P%2#`x0$dhKp+f5%&r+!Rh# zb;z}$K~!3lNZWg8?avqFJMZJA=iJfu7$c?bQw(-VXsE$tI#sL zYdG7%@zEdU4_Hsox}Wy`Dn!PKcwRa0s=}`Pa94@$G+oZ zf@7=cFFhDLUVM=y;MB|^6LxtsSAz{OEDUwbBB{GeColKi2XINSsG`__+J?J+{_$t? zwzsKo+)L?t$=ZQTf72l#X(n!fN_Y2>#BUYisv#Y6k3>53j=6VbN|1K%9U)*`9~@3^ z`R*%rn|T28C)S_?$)>xJ-KqM54Ck4X5p%i0kK0?vfEb0?9jvDk>aunq7ctyC*Eblr7e?yQVtq^ZE4RUjos&v@wUd+l3`Gd)%<5{s4Iu8la+=f zjF>(5rF-cupAkVR2o7#C6RWotCjo~bnJI8HO z(M6DxwyZ4TO;4vPeYU;|%2-@;OR(dLW@o@hF{pq#qUMkD6%}iCJP?-LamGRpMUn`G zo>qv%9G)?HYj6M(rCGE^Er*&!x)TEuB43nL&N}XzZ2Y%&JxOhe6a@SU56nbwH7AMA z$<{kW5`N+($Yv0ix+`LF#zAKqjNaKFULoyDUxC$oX(@)zKhdnL7&_8B3PyN^e>e7& zAtjjlWVEapsC{jO56@z826~Xqg~4{;!Jh0-XcFgezKS8_)RMx&_Sg&oTSg7NeNHN zrVzwwA0o<=&y*zq=elYtgAc|+-?aZHZLM{ z?|XI*-XS^NC_v16@VYzWq20gs+ITo-TlM(1`g_$sq^J#Pknjd~X@|Z}N>raAAgMbM zhjfm<4ap)rWe{MS7)IAeZ}AioNmE5`Pf(0=CU_Qb_R|Rl^dfIRVcNaWk`WQuPo#Cf zf9;|(@8P5~fBUEz$OcWPC&NOoY%BiY$-vBj|?j z9^WSM6sOpeCQT}tr>n_MpWAgEzZ^fXTW>r5Pj0A3QEWhi>s)`K`uEAkUULI9hu2xG zV)LkHYf{Lo&3U3IO}AyJK&-&W+Z_^4o^`C_3E{Fi^HIC;=b>~v%m^BA^j1Fptyrme z?!5i&l?%e@Y=v;TX|%P6lSP`I7{^1i66~B*>~9l4ww;dtCiCq7kK21|PzjQYjt`n@ zy~51`_ zvA-eVIDwn=?c*B=!1}v>1%t7SNrlZ_Mryk9ru31>ye;lj`p=gL?bRnBrIf6|=F%vD zU-=)i+H)m~{k1I$Fj#5CcM7tcb;m*ST4+UXhHfZ?JuzaA#Hjxa z1D;6pspu4k(@N;Ph$``VvUx#!Wu#1J3eO45Vg5B)4^ga$-TC1k@4T;(PJTTx33Bzv;wj1d!L8)WDPa8IuJQ$EBKwML%xfzyJDtslup9;Jz$+6oK29a+aWqSrd`QWj0apM$KCSO-Yi_vw`McZ2CUo z&$0cwE>4E%>!*I)7O*KN3;mBA`p+8wfeIeqD(~FblFF4+JMQn}6{7oBH!eqPbN%`_ zkCsj7r*+y)J_qeOC^SLtD1%Ux|3LCTL-eetL~$U|-Q+HwlLg7dzaNe)DN&q6FBoZ% z_&q&e^^wH>4UvDJ0r<^V@8r@0rbU?fqsu>K%fnD=yh5KZ|1b^>4mPs!Z;z#MMR4H! z-xK|LfV50E*gvUXcx>1PAGO55cQWu$Lq@q-{l48e>s!E_!)+wd_qjgGF*isf5HpJo zU)Bm$s^CqT59^(dMZd_G;Ggy2^W4}l+NB9t3)9Ce72!LPBS-mG-{0dPtNlp{P&hyR z+=PHg{4587hxjIQ9{XjhGSai`-zdxw^H12HnbAS_AwYPH)O7d}y*FC>djvNEI2q8$ zfY0buXBD7vtd1?m*fUW=29WTe6c>D zmvlA5^L$QbMM$nrn$TAtBklRxWcbyO)8tI6J^R_m*CV2cdL2_lDmp$fNH-pI)2Xc$ zTV``jN9a$O#geBlm$!&&Xz&YeC+ zJ>l}A?R2hzF}cx@9drE3{I%#bytQVV+J%K!Qe<;7 z+Z4XVo=n4F7-#e2IFNm+DYab(TnGH~sS%m%)ys_Up0Ho}H&5m z=<^?Ev)F5L5wP^-QxA`C4Hb*lGyNzE$7+d^I3qpY*Q@h259aD7H+85dznds5%MrWJ zb4%>JMOx|%{bzI&SqyJdmtICFLLw#t%Lqdo-q;DD(4uplZvh?2!c?~#vs?Hm$&Rg3 z#A6I9jO(}>(CQjvZp;(NlYo73wUG2cOE2&ik1X)Chqg!8yowr6jq$k|ozZg1#)L0c zd@b~m{rm?}ylB+49){%imQYT6S|QN@ziSisepy2~ck1hIdlG$ysSQ?B*tT&UgWew6 z=giu?^%0T>+5vtp1H7-xqLYpqMf0E%qkKhn7`lV=;pF_|U-2V{!IsN{qB7wkHn_m^ zgU#<0)qu`(#NI{Jnq9P*TCO%p^y$YV&b{_+X}we0mTe{r<&eN6(2 zS}GaYAxej?B@2x0H^dlD{^v94lFvVhdDd~T`@?Urw#NSk>>(s++H1<-sX88ryA{8D z%KPp?$~?w%_zRsLf`cONjYoK0M+PkJl%TsU28Q}{2T1!Pr5%-7`{LaBNGZy~v>Fm$ zT~Zz@tO&&Sv@m%sm}8)OF4DbzNFeIpFuaCgS4CE}WiywpQ73VIsXe0%m_jZGGT&TW z8s}@sw@!LYNd-jAeq-T?%uNlMF6ZRq$NgTUYh*@Xy8UG>(aw8Bp^^d-v;jI4TL_dk z?g`w?P2^%&aDtWGGh#9T?=S?-ECxYgEL}uJO-W~~7LGM8R?Yt}kLt?(tg+!c%-tzp zPJAA^)(>+M&n<_ax8}{by-9|}^x2(Li;)*rTp}B!DFMY4>_}=o==!DX7#pJ}^gN>= z*e}DVRls~Z+Vjj01aogUs)?@+h`X}`jxvZz4hYlrLB-8$(271a z{v#`7(bNRM4*zsf7w8%?q%YsTNe!Io{s3PDGblrJHsC4f2!rzNn~b~bZ!#TI{&T3# zKmF(VeZi~NbOCNT*AsQIv3c%6OEh@BeZ$!Rpk{;1%N;-g7&vqK?C@u!Qu$4me z4#mBu(FCtI1l48*n~dY>-lC^W&K(B_4!w>{no4;`y#rR1oN3cfpI7pd1Ml>#fZxlu1VTJ^%3t{u(`IgS?!RWJz7;Bb)D zKzCq_KLa_b9%1L#)k}ftwhZ#Iu(DTGSfn$JxY);}v&ksJS9t?0A9ah5PY5@LWjL9- zdb#??zvx=nda}pq;>=DZ&d2s}9&OKBUG+~BX&WLa8ZLEgnyz#m5~HxLlXWMI`7tZ8 z=bgH=J>r5qK*2JzS)qI*=|t*MI7^ecer(51W0@YOzL>rJ^T@qz0a*RK*Q)J&Wx7VE zGF}1AX7uLrN44t+x#FiGLkJT!zaOTuO|Ee91cWN-xe@vI3-b;fJ%z-z;h4DI+2Dw71Fe>w7vxgAk zTaw)>a~Qha58XChl{vB5r+lxZS(DpaFRy|7<18r7r}ET@c4LQ6UB|aDu7o->jxkU; zmC1X!|LzkIyTweD9 z-XAR7sa2$3{YjLnhyIfb& z?8eA5vhISk)z`2Uy6}3c!8J|d3T7$NU+PttWO5!pM!K}3^4Ek;V^CW%NjR`{U@mF; ztvGvEB>}-yaSxGW8yZ;2&?@2n-0Oynz&PIjrGTwJOMEcTn!zFSo(r-f2+3f((sz+D zz}$GTnIn>?(iDRp?2j5WHb>56L2Xl^O>bFQ;9rSaaAR_a>jQ^1PjrOKwZ&&sM63IsPZ z1xrkiSEH?RbkJ>SsBr2ZBOc943hK|jU2zF%h{A!r@o$k}>lF_RlH&Jp;_INy)-=i@ zixr&tk1}O23F&Sxe5~a~OqtYxJL*g%uN!GPg zYJ!2rkmoynolVyoJ8&c(zw-L*YS+AAD#u#wEMyuHSvgRtI9y$x-Y%)*NDui1rk2V? z!+TEWNR7~iE9Dg(d8a)T=i0vluRtAflmyFP&^tQD~GHmw3D~4Q;SU%(iadmWLhvx z302!CPEN;|4%pD`Y!D6)5_P=^d?`xTUGxFX1{d94PUs`qEDqUkH4z6ESVO7v%@f8? zmLts$G)wU%+A5@DuIT+ehG7>6LGvf8ivvRNo-;9yW@dhD_)t- z8Kj=*mt!mvqtXqgJFxR0O)uz)uJ$@LsTG2!G0Xc3S-yPIzKxWQex3t?J zhr+ZC-88$;WM>WN2z*sQ#++dOTi0o=qruxVhmHj-*`Xjcq51P-vYxnGz2lemjL)r*X@5uir(Q245=d3BnwZ>UshTq5KBeVNJ;&+h_PJEBy&U1_cL#JIgh^G@{L)`; z!+o}5)aF@9`W`E^PceLX>iQ-vgrbOOWROG0d&#UJoBv_v=m-NHRNH5W;fFiD(Cp&0 z34yTTimH)7Yp@-)RUMC-N(PM*6Z`XFDXYSc)2<%X(kh#MV%d#(6a`Z4Yf0-Hw8DS{ zFNm>WgmJnd+z7zt|5myztTvuODWx)_YKD5VKOU{H8_bI-?0-U5P{d_<2bZb4m_=5K zxOoyhTzF#wR=P0|67+-FvwC$WV(d#!7DGtd-l|Z+Wt>=`GUoQWq(P6Sja~GRDJup^ z(MCNZfNyXThmiw;%P~&=sOCV~HP+MU<^BufQSg-15i@_OP4=~2d(~Vs{8@K`%~PYi zJEIdu_|ed`Grk#RCWq{d zd$|)7wn`%}fn|ZB`8cP3N1^u742!%YgW>O5Hmt$=%gNGE784f33lG(0TX9rZwI^EF zXaWquJ%R3{$rHs3HBNbt_zf|CRh;1^*XV{e zIul*YfRz#BX=rP%vLW^uTnoSMJgUK1J4h4JEB>>dOYQJ)#UsA%0zGS%j2 zc*$y(;FTy*rn{ptzr4qi1{&R?V)b4?094f8ohWT4Y{*NUHqlVJ(3h*g)gD8y@o394 zW*rono?F}S(GbJqgcO%WFJ^rn!AJURRXs#R)>?u4X%Wkn*2}240OjI>cn8Rtr_(>U zjS|KxDkqR$B-PSv>U=2OOBr6J7e3zk-6lq(c(T>k!xwja-n6nn@I_lxeKQ{-C?qd# z=}8Ti$m`Nryq7Tk3hY%nlq@)V=TZmVX8`?%*=~;JkVaoOzEw*VkNotr8!syyhpsA~ zYBfS6h91?gDnrCwah(ELcs*kd4fOWY+hlY!k8!ewEep4-ecPwh(g5bni@9?bVLu_R zQPc2L!?~f}^ZMk=ggu_ikgjSZ&-4G$!51xixm_cCdeH4Edz@Z8)d6s4NP{e8 z3`C{Kc&jy(exM5mi4~8W^RfJb?B0DTQL9V}`UOc$x5L!$=)P16KRQ36e_Xf+pm9vj zT$MK{L(T*Z$IYXV&O-RbYf&jz-3B2|ROzZ|u#O*yg945hG{jxqr8e1aGD^DhovI9< zT1_0QsHBGf-fkbRt;N4lls!Bm3NP|SL>&uG&f4U?Ex2kC`dXxUYj zn}|0yA4SuKWZud{4FW@3=Q2whMz*Re<%IHCfhS!jyPA|HkVPM8`S@3IAgRKqZOBp9 z&G=V_F#l-MZ>Wxe@)w-84f_!Ujr`ZJz4e|;(9FuBW25C&t*|DvCTzRq8e|ire1XIC zq==`%cH7xBN(f?7XIh}~K54XNvk(XLzOh^VicyZ?vE{KE;O0Tt zYUn;H6A&w=gjwMXDS+M}GBH*(+=Ig(GPC{XR9x5m{-HQWIZ=Th8kQndnaXZ(g70W3 zkExY?*drlY{mY*(US3n<9_pY*WR5!9r^pz z#w`q&%nFG)N#;M(%BN&O;9ChLt+5noXM~avqGIBblp6YcxcMh8X{abe5#E#4d`Qv? zX}d8^3>G*@UFUGqtzg?BK6wEJc?c^e?uC&+o{mohpL~ZCkZ{Q;h)f*w8l+;&SmzOJ zW0!y7E6>?f;-x$e4mY4^3u*6|JVyxU^PHLVrPItDm^tWX0*Ltl^$YJk|LFf=ata=^#Xe zKsFyru53sQcR!QTA=A0o6@$qMfm5OENPd|M4d^}IIDmXD%4j)4-siC)+q28KM&*Iv zhGbUhRYW!h)H+-f7F&4Tt~x0OclD$?2hkzl%rgxP4t7`<9Qahrq(FzD9aW{{8)^|# z^zzdbLl;sdRYj*I3DglncDF8;LtEyOWD5cF)QgyLolP7tyf>djYFsNyj0JKWwM=E$ zu@g#(s7a$9D)^O47Dkn@RXCYx%+>DokeOZ!w+!;P?dK8kj_>#96J5ih$vZ@UQf}HHTkdJ|`(jK_=SRNcW{- zA4#6v_d^3LDLS{&NV@coHks7I)Z&(&Y>W&l_`!!**(e;ltgYKdb|8yxf@YQLU~mnr zCa~Py&`Pm>1190h7qJ)J*y9gS`X4t_IGoYSqHU}sBUM(M-;RMT(_P7y_@`S1N>Izd zM3&1Qr&&BC+)$FI1V4Z}^IvN;D5&Fkl?pZ!if#A!Zw;peV#xRg(ubfQ=`8*uFEU26J7mqjxo2!tVb|N@cAu&5T$)lM zuJ|PVGrIp-3{+cao$bK|o>8Nk?p#ld+B#~hXZ$@5>*?|kh}I~PBf@yPO{-=7_3Vh; zs0ODwPG~uQTOl#yYN)SakY3s2)Gze(I&9hbISWuql&<-YUfYuqlOQC+OFl~u_(PdE zQ%9W`r$GU^R2{4dHWWtfoUH1|7dV@AlWwVO63|iusr3eLco){BSrzoJH6g>Q%d*P> z27vVNpjr4wd@UEJLd8ixe+|=IO-P^zq}eJ@`4U{9tC?l4wr~byQ{s!TAxmGWhOD+@ zKtqMK!jS1fQY~J z=YbZppIXu8b}l-pt0+qddYT2@$&NQV$LCUm&HeAE-R5_i8p*}w-@I5)*IY)iny31H zNebI@6f2@?7X<3iXC1dnQn#87K!#;1qo^#g&25~tCT+ul{pR?(%4~>tkaKqKZ$DgT zn$Q0%+~0Ov@jLjEw)HhA$>&+BE5&PvZBojE^asHAF>R#FKq1a>^DODZe1?T3J^6{y zlv-uw+8206BtGr`3dvJBf4Z%k+O~s02kuQXtaS5IK*qxxg?-ewpn>LziCr|gpx#-^ zY)=XEBztc>2T_onw;+wJtDi#(a#aTd(Dq=wKiZ(8Yiz9Q!r5P48OdbA?zSmw>ISB$ zyUJoEsm9=I_cEFiqr@rn!e0R|rEw}$(_B^U4Bl76H6O3i+5Gh^$ue@NJ9Z%l(w#CIAAu5E$LkrSBoP!a zuW92vFNEnY*f6TG((5`LP`zQ~6$&ayxUqXndeL+)tW=tdm~C*~t~a2xybaOT8E*QJ zY=(K@MTM64&X%@#`+(ZJJQ+SS5!c4Q)?cNTi~952s@;}-l4`L>19LLO+OHv@oo|en z?lBv}(z@L&jdDHTzHZN@k}}!g=5V%;JXc61w{3QVw9xK6xD&NsNUL7;m_(&h1jj|8 zSi+J@nMS1)*n&UA$n@~)a4MT2Y9Kq`+)HU16LRZhL@~=(02pOVA(N8bJP3??P6VK@Q zmYx!WQY-!Otp&q{8Ry32)Df0xsE*@DvUKIh54_hdYi?cyG^3PH)M^lsF$s{$V>Lsd zs6nBC>t0ht5BL0u>L)^8mtr=^y|dnu{$ge&AsIAiajbc|x7N8-js6|XWpQb~!7Bj^ z%!aKUNUQ&5OmU)HhLGUy`4!AidD>i0i1s4-pE6NLNHH1X#9x+T8xt&zZX5+^Ri{8N zmp9fmy=xYzDC@QxW(TA{Xpbh1lcrreqOa)fQ?@wd$?oWy{2-;1+F120TVMnD^%oZ1)J@uw->7yXqOCG?{a`X#b0a^M z_B6B-=3sM~k-72mVxEhFPr}6zF)D&q^?#IvTDIIynxRQ4z5f2qHfsj$>Za$VA+2G_ zBZ;{VdF)QcIV`PL7gUS`$KkjR0WGA?-Yi(Y#co2`-OQnTd@vSt&t2L2Wu$DNd0GJt zaEdFppGuNLfSxIWC6?k6gTjg}dl3s~drwK&ZKluMx`^tbg zmTg@U5(vRU2yVfHyK91bf)m^!xVyVsa2SHSyAAH{?(Q%+49uHk-@VT{@7(wA)sLQ@ zmg??RRjZbKwN@F9eRX~A#fn#?W)eXWsp;jrMaT3n{f1E6Om;GBFM*rF*WYFtMIQON z;oUIiynf{bE$V!-6RmD6F$~w?g@Jl)+1$2b*t$r~{j=6Lkn*PsM!un#DXcTMF(iqO zV+nKbpdi@7N0blE1JID#QDRUp;$r{Ev5bw*`QzDS*TjoGPle=dT?xH?%lA_({a@4e z_qKx0U{sDzG!M>L{5r-=5fI)oEV{Mg2mV|!M;yvK;ZeZZ-t}%hV&unO!;rP0+-%8= zy@D_9k}p5lXauxy9Ff8`a_O!$p~mG!^m3UAsCUJQhw=)hh0s8gnbSHuD>WlO!C!ZS zY&pf~e?Eg|(dz}UOY<1?*AD#jp{sG_KwVB>@rFTC0^ow2;WA2FvQ3f%2DxnlC(2{> z29`|=r(?u3xSf3>veRkXtr9>}y3I-0Oz{nhGhM_NtgWa-DXiTftX(yWFhArRz+6HS zj+r>uXov9Y`pm6^mBkls^|~!n%{{GZ?19p-3q~lMy!oQvc*eKVGdWdQbj-^>Lz7KB z4>XL}GSgTV8y1-g*i#b@T4D~bNy~@S%gSBJ-|D&4<=W(bd!;%4p1G0KLd;^O0663QMTpJCz*(8_BmyXVNU}kFL_g^S$>PSKlf{bTA;i z&GiHY3Xu?gXoL)V(htJ+-t;{*x_Y40m1QS!jWkzi;r57aHGjF1y&+4q91|CSLiZ%6 zSkHuMc93{xy~MT~4M^Iw2NHs>+=?j;MN7MlyRmn@+= zM%{VhGS58PIX3boWWXWmM_y!VmDz>0IKOmOS}?{|IlB&O2HjpBprzjNLxUwo<2t2W z{g|Cs$JU1$44T-2w1H~92^n&B^hQK4Xl8ACk6oH-E6dK)Qj}M03JJ#&;(h#k$?Yi2 z(X~r>{5$T+wJPYbh4uP*yMQZg(E$CA*TbgthDNp|21Jxo$Oc-Wd*)5P5i%Wh39n&g zW2p(P z$f}O212{*q<=RoReUM~{5GV&0>vUXZekgOcasTwh!CJtyheQxueJe@gCb)htkN_V6 zheQSq;=etH@@oioMV81tJdfG5fk?f#fJq~g++g9;ZaHeXo?oBfXd!O=wajlc7K{@7LFih3$>_j zy{{?xjO4=I$$x1CTqIiahLwKFtpDfQA@eUhgA12*LsqxD)F_=!a1 zuTA%WTym0aY;LgtzKz#zbShP_-cS|!%UgB~PXX79g%E50RBi&pQ^kZ=F4_CF_{;lU z%if=ut3MiCi02*TgGRdr;YH!ff4ByQ zR6qEei4m>daUwhvr7FL_C;jZqa-jy(1e&s&hFbG?4pc9mIjbmT430rMMoLrJe06@N zXY^#B24G6`Z&>wvBX9KgsMLpT?a4$=jh<@^h4%#uHLJ*2NZa%l&DnrkPxD;zpRw99 zkf-*ln0jF%!V-(iHkn~bW{0;}j$B&mS(7&QLsqfH`vYE>6up@1BKl-mrt;2MiJvJ? zSvB>GZm-Q$4GXv<j` z|E%-&NXSS>{H>9NH9mXUub&QB*|-Mzr<9)-LezLre?h07{=H!oPH7_mu5kvPInI{i z)dM*+8vH9}p}cY9O`{q(#=`85;ZH%~sm#1kQ7q+;D7GwaawAw-2oV&R3mcOy(Ff(2 zQ^?|vP%h=^CFGL3-fHyHA=A3O3;3feV!y679{E~juklc7EiM`f*tX+FiZ}%d3rV38 z1-5{AWlBSVcw;}SH{)&Sc$$Eh^d8zK0oSc)_=Raqc{)7N%Iw=|YJIs&f+KIg42=rE<_^qU}z|d#tm=8Z)6Z{c2|FD=zw(w)ywZ*t<&E^wbpG%vwycYAU za=_ZL8aMMw^B$(>vAb%6Dvx3z#ir)&A| zCtiu)AZ#10%>rJqq$*xLh-m0I4-4OzU$7)zymAMggz=9kn8d5)(E zR}a^lF>dALbK=aPaN69O*?0BesU4nCeP25WmT;)K}DaE`KtOu(|xI7e}qf;_jB|Ewy+(_kMUT~)iJkp z>`XrfQRnZ?4V3-Pu&=dXjH_UZ6O(FP=^c53Sg1~wlvS7nE=gxvuJ1)#=smhC8U=Kj zu84PyJ0@)F?>})`vOeVco!=AJ9?#Wa_fSLTB(7 zE3%AGax-g}7nr{tV;cbwpa)h{$l|7WaX1`t6fyF>4lv_D6vY_&1G^(cCPj`Ucos40 zEU${H{~ustaGSti9_Oee=n5NVrljvWHcqvO`fQF7tgnj?aEdFS*yxW2`>1OnEOh;W%Qx?Y-9CF6|i8Pk3#FiTfcNpK;r}5Dt;Xj8?HN&Xh6F zlhmz527@+@4`))-k65l_c-VIjUtKrBGYiU4;G^vhK4nrOvwC!;e$zZ*WT|c6DSdtC z#SOqRm$LX4Z}pKFAw+{r_$~Ip2O@RoG`-84l&$}t;=lpJ6OP(fTy~uTm_JV z3}C>v1|9h!bKVQx5BTBxz1UGh#!SF8=o4rG!z8iYcHd@S*qh}jKI?tJZG2=L>Wj;B zDmTm-zp6+q;QSj401XyxYbKhF2?)o&c^}Fi-hY`K@18J7orn&$m)+>$5SywFAc)Yy z?th0Wc4~N3uPATF%4(*@7~NAM!k}eW3*@-_;#9R+#k7sx|800&RZZXGg7U2t*rNZx z*ZX(tC)%Bg0adHg*S@==SLkC6ttp~6-XcBleih$Kv`FRru{*H2(#O?g2Y2&r=x!_} zE*$llNz1@@m7=SNsu-d+{#Koc4^tsM`GoL%t%fGHAH&NGYFQWyZLEkNn={C?IEU zdAy~Nk$gOQG#cas&xKp_69Dc9zfR>;SbZiml>b(4SRKT?z8$q4l&0v(W&KhsX*3Tq zy3u3o=_-Ct^=sM-faTgE&Vs1_whn1Q@+w3%@-!(jEa3IZ?g{aOFl`Y3b@$>tcGzzE z{=qCR;#9MX#7=&Nv{p)e{K?wzMnH>)d!;pczNO-gnr)_N?c|PaO1e~HB~LkeHE^Nl zCZPOFH}R+;24+vLM+pdmB6J&IzE~L~1BC3>2T$ef`ROS}YkgZV04zXb$(3F_*ke6+ z;nS#<3WL+1G8!B2|ABjNnk;Tj!j){&3as zJ*lXx_nSoG|G>lly>5T!?mwWI|2X#h$=#$B|F4JQ{Elk+ACLU~_ab=UzdJ7UM6j6j z|AMxLp5WE{WAps4sy5O7;EVtJ8Bs}p_|Siy`2YS1S+p}hbmxGu4l+`d1r?VXgTqYe zUsY)074y-p9_cz)w63vU{|5B)zQL#~r=u>IqAgMl;+t?tpPu*^SvCiZt{Px}(+MEH zOJ{SmA!)veR%U=9*H7;9D7?#C*6{nwzo8Un|7%fvcbv>`T6HTF!Z|)YcU1&#l&IFl z(TvSsO!}jmrdK2}0Oi(iTIUMCa-0nv5vH5w6e28OaiLvdd`6?fX7fr6vHEUvAun@G zV9q$k*VQpjcF4DehBuV5ALZ2*2stG`TdTga7FI7>yAz zWoJ4$EqETgfIoWXy*K`Xb*|tePXH<8DQ&$0?`tWXBcBKM@(dial{NV_g+pn1rN@AZ zfrEDUx^aEA{WbB`J@xSe#uKE@Fu!Wdc=bKU7_fo0B;rt(zKz_!f`%-F!f&m!2Z$om zd#}^gUvN0as8z;O-#b~BR<*?@lQ{x!azh&5o-TjXJ^N87Hq&7ji&f7dZTz@^ zPs=U0)#XX;ct1jM%slVn8LpCk;$BIPyvNbEwORwGHm5MYTJ*vRokdEqACKSyx@qA;T?!lbc_?};pE>SOVE)spn2&-7= zg%VvFFU5H;jRn(~EHS-3a;g_+B;Y&eb&7X>F|ug;$xlY#I&)(7dZ7Vu`C`<{uT}Cv zp+l2@b>j;sVs&Utm~Mpw%mFtJyDU-E<)y{1w`*GVtX$IL&#|bXj2698TCL7E$13j3 z*96MPW4X0r49COXJim(cMYI}-GVWw4Ia{e19Et`u#wZz`CLC?kUDq|_L>}a-lX+`6 z^n-w-!!u&zfajQsdA!`sf!F9X=3MPqEXMB~M09x2Isd7IV@r3oEQo!%|(y@W2$HkPM76Hp!4uK=*2d2Bdy!?h+r14N zwVD{n6y>@v`n-bGw|-z9c%p4ZOnu18`a{ew1%?`6u^4GQ6&Vw)+=8N{JK>20ptGtp z?hxPg#PR)>?__ON){(V^&6_^;yFwO$lo8!!=dNnlLu1Ju$lysVB#|4s%n}NfuIVdHPU)X?IUYh0r?q7BTlD7YAvw ze;7wUTTW)HdwO`-g+@!V)M>3j*>aIjQ{Y9bLgAr9Z?zk;toh1kTg=;`HrdG-#+u|` zMSEY+r#?jDbB~;WH74c!-=&&~U5et;;^P+rFeR3AH5hZ=5!j@JFN5-KFd2)~SMPra z%A$xOH;-3(pihr2e{^gX_?$0^`jIMC;wg`{AVJz*1Gn+Me*D?iJ1lqTck-fwFX_ca zbFmag1Hy#7o$*%&By(~GyPfG81KGj^YAbo2vyCR`Q;c4t z35RTWI_i`W`}(Fy96tmdSa<-%4(Z#7ZTe?A0sQRmEhx70i1w__ak&Eg@a^oTf_d&p zE2&IPyzXQAi>VK1%QZy<6$=O+HfL};XUm9hzSBS01xaM#jNr|}#AC>$9a`^fGkMZz ziKrBRCZ$xbl6`$4C69ZM?^LA+QmyOe6Ut6-G)U(b>|}d;c&iNS*WTDO@JA;=o zP`2N5RViTGsgG8^ywDjqWr!^K_1o>5-^JCjk3wrFPK{b0mn{cZpP<%;U*a+tA7pD8 zld7HM2w&iM%gPR3s%Dsp@*3^PQ8$E-OYJ&2_9cq=F)gZDPUn~4-uEjDD_Z9pIywu6 z!5(HBQU6GjgqA1H9!*}>rQxx9Z)gj!!Rb~v0ZW4yOt<8Bqilg-pG#2$&N|xLUEOvh z_Y?U&Eu~Wz+d=E>Wra}G4;4*?&cr0miYkE}oZe0<=Jgtolf^8RQ6V3$bz;(G=6BSk z5?mk7bd&Ieka9eu@+kDTI@HsT5_8Z5wcv{E4K^uV*;%p|e9%`E3~^wTwm7(JqSN5{ zGQaXUkt2>X`3<4$O+op>c#ABqjg2XJGWn}q|gC0m8VOahqUVQ56m9E(~nRe z(b!*0jy_ad-*oo~55*gEPZcU*<9SKY#FSMXMCRv{F19n3aC*Sb*jX{^npX=rV_-P& zw2I{8!Y@?RPurg6E_;~AIvsI8{1h_Ru1Z3mlLQRSl440KzBUuw@f)n65S=N*B1@d# z?Z%ETWD>;yl()xgv!7ox;C)rA>Cr!a1$Qg0?;?y*Wz-;HS!>KhjUSvPKa2j-)#}?|ToGfTLODBnH);}V% zmFS>t1Dt8Tx;YjE%>|9`t*z@_nJTvOOp;8!MbsoiILM7`iIWmFqA$XAV#xYpp&!(^ zL4lL};n73)3E&Y>(hU1qs_*MJUB7>|0EeMsE2cxZJsE&o;xIzvW7w3NfbqQz#)Da- zq7}cd4PoQ4DiB?XNFcDDgMoYg-Pq_oG$XK4s8FKrm92&NweiQx;am4+zVPC}z63hx zDqOL;-vQ9_mC{jKz@f+psRFt)byM@Vu70x)lECi%mKT`!rX~z4-al$hCM!wn@~Fd$ zl%8G$4d(H*uKaj+e-dRfw78gq)R;&r(O;FFErnhLJ8c@)7u~_BCioCQ20C=gaqz(S z#Lii773i`7i`9C{Evk(CwoZqsI=@oW!Wj-!_R|UYecokM-&aMwNX2jYNP;YXg^XiK zP|Fjg5o-VbtW`Y~*8q}6PfPutS)>fR>ulwcuBirr7s6U{B+vVYvBl zYfo(rndbt~j|K(Sd8Owq%u}O28I{Gs?hhf?;lkLZRG4k;EsYXPg?xHeCY8*pgHi7b ze*3CXs1uR-FhSEq59O01XMH^>!@-9{8g;5{{C{I`3ab&$dJ4lf9{QgwPrh&H=tq9@ z#GJFo$yJsgUB3#nk;b{jRGdv42^5fot5h7Iz`%f;W4;%b1cfHKOIa_eU3r$m4je`OVldxd<#+P#n#;SK z^-><^*}`4i_XEdn1U#+-a{`rjzU0k297-JpD%Sv75#(TOC05x3#{6Q3GM-Z6+aa@- zy>|Pr_>bWVeq=>`ABhRqurv;Q=XfCCJhC_}HCIcPCcRnD-6%+ZU_(sSa@h5%Ez$&C z4lOb5i`I4MsFPo~A5ZY*anF(hoQ~E|=;0hng^k_I`m^cnw_P}_PH4&*GRBVw%ymW* z0X|kjc^{>*ik~pytj64zXvpOk?LNoL+E-t(YgCl)L788_prh%y2ltA z7dj~b!_kKhAD(l3Ffmcf^>&;WKi}V?B31*citR$WvkKf1LY8-myN|e^1J&va-F|!RDWzppvl@o$45U zB2cIAo>6Zt0Z!{b$`5HSSYLb~(Y-J=oc3?b5py4K9C@{>dPRWwR^#53iDuJ2=aPJF zHradcN(rgVP~@7%?cTME%5v&-EWY_)gB^0b;-;#&v)Ff0E=jA;;fI?|E^YXWT<`4x zo+T}EXlGSLsgIgof-1G;Skn|ql*_Pubq;Ig^5JZFx+T>REJaA6j_*&n6ciKY1ZIWq zcY}}czr+ctrKr6!j#3bljQNfl=f1<51v->J-rLy{kT^`(hn{-*qo8>-)W=X-M&k27 z>2RG{jpkj^YD&LYd2QObeR$4c{Y@idgM<^ZK^7^%VihJCkrvwP-4E7h0pML~NuhGF^G zA70bsxAqbm%ZQOev)KPtKy!{jh`yJH3B-ygDV5Sw*JIyNMc;OnaOiN-5a)heE$7=4d{w^0xm5w{*-$(Z61Kvd04!*-^nWY&9-21s0t8Lh8_(+HdxP9CDmtJ? zo)>dqajjJw*)*+KIG>d_)yKO}sQff?yD*&J`hAXJH5RH!N6LXO&)Xkwut}_Lw+36+a6J3qMbL&OhNY@TyFG3;w6{_dF0gy@ts8x(Pinfn$1;eqdajsAOCmfD$DZ zH2X2=s4?x$q;>ino!4MID4mL&LGP#-q71Jids@s+d!AtPtQk%8XDO3?SVM=J-egAUO1JRo?2rwnH~xf+Xb$*WsgQTN3BkA;eQp+Lw88u z^TYVF6gD)$JyF-cQ5=r7C*+^^{cETHOcvok8v)u9Z7k#_|JQZF;L)GIi2t+_$XH3h z|89GQ4Eb%R;#B277xueQL5Z`?Y{VJ{WyX{Sr7bdgc?CwTCjIuV}r^=yv z`rM?1pJep{`{%>Wg!cJ%+De8hDmmOZ2Er)Dvuk&Z`*!j8eYqqK40ad$rKOBUE5zN4 zj~&s_3UTsgC%PFH+avkdZ;aeKtEfB>ZrJIUFGz5nJ=79RX#^=EDTCkj7Er+=l#K-}nK9Re;Xab+U%vU( z_jBQ@ZRcZ{nCQEUZfE)Z?gA{acc&H_jO+^i)<)7H9mcmqNc;F5-u|@iuF}TxRx3($ zB=T`cYvNi-Y(6}paW4FavRMFi-_{drqi&~;TPI-N*&sknQHo~7orcKMzlVb{m|jX6>M<4Q{*UvM8PDBZMnp zqrYz!%M$&K`(yQ(>F!6u6C$BseK;}c5O+6Xtl)?T;;U5dbW4?z4Q2hK6>3RJ{y(w=suR;XX?{YqZ!equIY)U#Gy3gGf+b zxlTkV6Bc_toL>Di9G0DVHND4|)x^mQ8>ppx>ip?vllm@`lMQtve3fcEWBe@voy$5f zL1X2Uza;@)Y^yUa;_QyjQnL?-mRjh^qO858Sx1AQn@q3cH+{Dx25R=WjYyU!8HywBFiGUJR^2j{#O?gJx?heDCr` z@0zIDxEQNGz!L(IdF8jca%AB3i!N!KURpi3s2x8rOB~}eas_$s6p+YBT_yg_6QLkb5_BuyuA+b@?Eme_fF;aJ2TuvZ=@K$%Zb8>KSzg1^)dUr7c zd8`ow{G$ zz+c4pi(KhGtGu|vd!$8GOjs+>Utc2jhdP0U1I1WeiUZ2J0;g)#u0J5NJb#m$h;bO! zP7gq|-pz@%nU8B@&Yn}FGT_u|_nUNPbyAZ`DwC#RhcHuQCv|!l z0jH)8yLQOHmir0Cv$B4r=JGtitGQu5oEt-|k0%z#6w0N^F4Md)8TeV^e|BMU8UbeL zI~$V`+*u>bkn1jQmNIl0M1+NZBo%hOn%!CK*GO$A2-4QS+9orn9KKt_2)u5WH=-bm ziAfb&tV&F7&hy5+T@5K;5V_gh;9cg4d`+b`e`Be04<@rcf~M8mvlsPd@t{IX$c;@+ z9Spq>F=(#M6#80AjklV~``{hF93PTHQ06=d^l6+@`7)gaGVnWr7*@zpmyc-tUMaK& z1y0+QLAdramm@9#R^?TJ)?;)PhR=$a`35n~EXj>5dzDN0 zoqQISypa1kAB5^2-yV*im}~TgCfn)4G8#b>4%{qyLX3@14;~bM@wgpnj+MUX#5Mh? zd^H32s+evpJyWYvklMsZH{D8p#lsY>vVfag?U){%)`@=d^(kiE>9F2Z^JR$Fren*> zkopSVO0T{?R`tUcEi?a0=ir@Y&jz`aI-zv>oWaNbd&1y-pxf*(fi;2F7`RbNVkuRqH?(lreJ+sgSoV98 z=L&n+A1O+l>$ca4$pFMj!pnZ42eIt#hTypgYEwkG2U8Gw_eBWvH+xG!&N~k74j_Ch z>h{Kymr?o>(0Jmbw~RW^{FdSP;?ocpA2GYEUfE2!Fy4e7r>)HAsrlg*q-56Uie9*e zHK-+C%Sb#p>^A=j8X6O!1@c6Igu|1@NV5%bdgoMw^05Lmkwzro`u#L;37lks5;WsHF&5 z$J+B{KGBKq^YBVd^0LbeD>>IMF%dhYL@S#f-| z$kFMZNh|vox8@Q+h21-iZvoAk$KEt}d$ibS!ap-qv2mm%RIC|x)|wI8*q!*xdPUG1 z+q2u)?R&7^?-Br99LSp^U)F_mmz0^5?M2wlwYe}EB1kl+x9`z{Ol0@xXIXC@jWuI@ zE=H6SMgSsTZ28IEMW2^O5YpY@M_N}T>eBBsloe9N_U+1fj<(he=A0-N;a37(Ad;)j zM_$LaFA?*%9Hw3pHs7+hRD*h{e(nE_1!$lf4+5i#oj*Y8HfaOE$<-$OR-9-jt!njT zL|rGJ&iLn2HRq5FHu_m;%!2#1#ZCr`aEhRsh%J|mE_59Eh{1g3P}JgNs(7f+)tyK@ zUd@n`t&3>lsOhK&ovLLM(^*!^rzMJ<`Y;SU4+iWcB-|-FT8;ml^@O`QmIm3~#;Q-h zgQ&|UDs!w@)6fmSHaO$+y|Z@d)>4L0@v?(#)erU(YNaz$Wh-hmOc2+0SQ;@Osp$(1}`sAl~?0~ zrw4aq=Xq27dg>wQzK*tNRmY$KN1zaFU*y{p)#h0&MZc@zn>iDOBIXv5>{RmAP3ij4E_vHFPvZoEJoZEYHgdc+p7L7u^hWBka})QzPA zk{8`0;XR9QfC9cx;UJ>FCLfSg{5CiWt53Q_d=zF`+L?pB(2O!1BK>k*FXDS>Y5}wnDfH8pMUC4EE)(U$WlYY#*@4uB%ZN4e4 zB|Tc6)VbJR9D|xC&vTgHeL4!>}uT-C+*v*RfXp=DLldisqkJXTztEwW3Wg}Ez3{AQg8&NAt%wIkSl@Ri`{ z_53KPw)uWnocgSkroFB2jcd@o?se4qz%!~YU-Yos(s2X*xK>=H=JdQsSyEhe_in zdcJpLdbp-BrIV?gB{IS2A}p3OLv-RG`~f}K4Jlu{frtVXv7^;E?)>l#x0aR0M|P|_ z3#og?u_&=1UXg}2A|FjL#s)hf+u2g$zWZV)sQCQ5qy?y0GXRctDo)qV&U*s*?Qq~Q z8PHcDbGZDj2}$%T&d*Qp7yD!k`w8-Cg;NMKv^_A-k=s_Sb9%0nZbqqX~J@cUoW1JB{3r=5(0 zXzV<~U^+bH%ES~-W1Awvcz?b-v~d}^L>FSP+0E%o=RMThyU>O6S}jVGB#)sw=z#A+wJP8M2Fac0DK5r_sm9DGuuIKc0?_v}Sz;R#=T20PQrDbJ znPdIjl_JU=$1OjEcP$qUk5abxV*k_2DV12P>9BJnxU#N;k|h400}@>Mu}riZQ>xsB zI5dP_Nh&Ow#@)Mkgi>w5l>_XrXiIf$v`Jj?JIRqC&vH@yQlP~B8KeG_43Y=!I;HxN z22Wc5XQ@h@Hjw+H)R<`?5uXtxNbE$&h0_yuG~-gM9jZ|3UR<4i@UM;ZTOPiq&0vds zU3hcZryNtQ#~NlZeTQamq2FUYKi}ZYyR}E5U(7EO@uzF`YmeVQM?GCP#`;N#;kt|G5+# ziYGq3QhI*loxP-7HL_*7NSGp6J^`VTyMk#U$*VBP!3qN$_o;I=p~-EJt*7+Lj4lC! zI_lTr6SYbW*;9vL1_uW@T|i`O>PueT)I|!(!tHQEc(h%;;U;?23(~J?d&F#_X z)f}9>ldct9%M%wlw(}Z6-_-=9T|Bv$WlH6_##Su5IoQhrNd?l((UowvMH^T8vsipP zxb1>K?~x)+cDq3Fj6JaB;{8cIGKo>|EXYC-`kG#%ZDi1O!KL48VG5wifBf2Ca_^r8 zgl;sx!^v<*5kF@(kwJ>#yWWq*rrFSwI|AqR`x?r)B1YniFFW&Ne2yB@r1_lu4nz=m5JINr)Q9LPeD)Ak!eOFd(mPFa*~C}yHR zwEzfvzZpbsUgw6kSd3MV^C*7D8uebmF6-eo3u<=ROmKIM%t3%X!?WyGj2+6{nkiDB zMkM>8>|zrUw)7Z53_A1}zW<14Oy&HzA_noq{63S~KMBp8XRFH45=FZm5U$;j zP6EnT{%>zk#jyk?g-R%A~AYc@GcTA8}#1Zd43Z`_eo@P!9VBq(6 zy1o&j!MVA!E8)>wCs<6p>sT>0WBX>i&$61%;iK|=%QY%*ZDorU)x$5FcvE@~oD_Km zM7elv(7xwihNshWc^dIq-qiBgp|6uU==COl{L~@5)Pb0URu!KqCu_WPK^@kZ+(2&h zmSyv_9npHRE|&F@YqzAlphowH>@Nj1xaBC!e6)I+T;c$FV-q0AB`Pu4={Uv zvb8?mN7E?FN*W!Ct& zJH#U@mPOXSG2P+J;Dz?9o|D{Xk~TsDX*?IpS%YR2C}V8*`=rovgZs<>gYb!H)kGV7 z49gBT%~gX(64feuB8&6%!L1s29o>cq_Lq(o7me^C>S+p>OFvW( z*dIM80+TB#d(gO;5!_5=;K~2_eA@%?-O7vE5yGKI?R4@5l2#5Xm|b@JM`I3WC)%Ov zPaV6Fq44fRq*x;Xhh$#}zLWN8Bf_w4ar zjs2K=HOwt~!t8_%_dz=V>Hv9W5x|`D4RSGe<(3VlT27Rxn~8gwn!cHW^7Ws-vOV@I z5`GTygm#L2k>3&j^}&Lj>I|GJcNw}qc!Y2;8&qLL2>$Bq&440xsl*x@U zc5F*F{44E{)MQKol5@7A=|j5(H6E^%{@^!y+LAr@Dy0Bj6uon5JU6>y-kW0zmkw<1 z$5e*-mG|s{hD#R5ngS8$W(>@jr~Jvy`aU+ih0sxa1u=cvQe z!08}lh}wdzDm4G0c_Lc9*;$^QutH>sr;q8D_IkGdv;WG>TZMEQ6KwDsCymy)?g7%| zxSvsS-b5<1RG;HAgT-YEG^bt!2a6-yTy}%&K~|b;iC^WDk#yGDY5WMuiZ0WI`QwB? zS92y<$QfDV!B$M3%-G*hL=iXeuNx6p?6D@4uE`jcMYOuvip$6vSU0Q{hRr8bAe0Q% zR&vnAtWJ=s)=&ivJ8hod!a!~He>n`G-Zs(QU4z~5(40dy*R92O0NMH9?lEQ#NFtXX z$pJ5Sqx*gRMPxds{F{{ z|7|OS+RGw64kshQ;zE*0NVwa2mJq1;0-`ls)39gLRM(8+)DCpK(>7@Sr_*|-)08y< zDs&N(*!N1OqAH(EJOOUal9$gzX~v?lKMnJ1t~l=>QGJYs%Iu5#kUcTeGOS1;%|!Pb zL#S%E$>~H&TtdP>g7Ki!3e+*BfB~I>hJuG|-lY=wmUFxzg!UmvGu39Mc)BHpf%#Nf zQIXC-8`gAG<($na(pQM|wsWmggLR4^xg-R_bGY)p*ShN;orYY^RcdqehrR+lYpJyS zvast1MQvqPdDRywLJnZ+NBU#SN3)FDfd@t!3f&=9t$PXYR|G7e(bxYwjz4 zG#MQaYa_}_&mYt{Yu>f4Uw2_#rL~3KlU{4EGtYV;coM$~>}J&W!LS=s#XNJ^u#I1T zi3B<2s=*g=W5bzM&T`0@eiU{fG`Ekdu%~ zdoFjc)8%9Ot$=v0SRdvcJEJ%c)V&snE>M8J>fqyd*elVz-WNJ(Z!5 zll*w1&eFjrI$o~+W;L(r=XUwl307WJ;e`Os4Cx!MybHB;Bt2g2tbzBsw}!qYjPM#q z%Gq>QzA8%7x9ij2%nO8fchL-6lFFBG+xGC9jeePywe1M)v@siFb(aR_)6=0;fx~MR zSHTI?_#^79el_6(@>X>j;q98m-=de|6I6gmeDO%x!tOzc(+4wr=NP1Qvfsv5RZB9#z8RcJHVe%^t6T(xLR3xmVMZQt=4P_)d#3ufU*EBj(xT58N3E?Nwz}Ids zq&`LQkHf|FIoxTfe_36%SC$FOH9mtsxR!qJ*lyy2<%9fITteDFU(4mCH{A{!g`6ze z$x&>Ezv0PvqNgtArxD%H8So*Y5yaF*s9|G}(HR!Wn@LW)^ z@)BBui7*0BazAme5cK?nB}T%XeSI6DfYVHpzU-h7g2$Rl8-!S98ftoELXu5Uv9xFY zlyza@PP5_~pp&r{*FTEJd#{JdpP4>bT-)HXi8mb`9)fCXw$C_O8e@-W>UZiY?lZq#j ze;%Y}&FQ}?=ikwHnUoPLbQ8L9m60@@IXD(_3bmoitiO04(t3Ml?+P|NJ`rOqGJ`pV z$;-R*Z2PGj*L#vguBbMx?@j!crM>NmU6 z(6(d4pR|~`0qc6%Sv6)2esfML@WD@Pg@!tzR+cu_BI3TrqUMy}@HUFuC081Gun?5l0b64v3&Ak^5w)0gd zLm+Bw?X!<`Q?h$zG23lf7FPH^bab!knlZ{5@X(IUU{Ltis{CGJ>Bt${`o|9_Dc3vi z`_YzxE-$R2)<3;QJauk@3=i#CVy4u$)w(a;lUdFGRhvV+Popsd)IeAQjB`@Xpm^Env z#E$IP0eq!7TV+R>?evn+^VKHFt>3Fv*@2D<#yWlZ#j8#>gwv zwElv2m|=6Ly=sS@$#tw{z$IyXI8qhvf#sFP-*s-vKZems3DUpxxzHWB5 zs!=vM4Cr+<7nY>s_bqC>1pgxxJmZ(GOEMr@Mz-SLpNcgMMz#OSOIm5$46WM2A7IGx z@ltQ$AjoQ;^r@pewCQ|emb|p5}ljhus1V(hkwm^ zqMR3Jn_!vuunXe34~yKQ<$iLn%=XrU$)*Q*6;F03^-uUxQ^8moy=;zLKS+{=+V8lI z2%b*~G4vZE?P_ep(R)6)tGl~qF8;OzCwQ$fM;hYT%*;-{V~Bf!B`HvF6jDf{71Z_F zrso60Pwr3FoG>g7G60Q}gOEnvPj3l&_Ed{~ay41~9&R^RKMF`IahxU&Tc{c?X@~@* z4Kh)^?yC>*#^B{;n%lAW75zpWIk=+mJ-pbKXSw~;pkB|FX$>|{nzc4f$Arx*vHwHe zTeh{;Hs8Zg+zS-9777%1f>WVTAdnVX+_ktn1b2txR*Dt3;$B>X6l?I{?v?=m^tyl7 ztsmeyp7RCSFF2Adv*(;yYt2jwK)eBBJP>D*zpnpHFq+hLlI1ENvZvX|=|<9FfO+o) zsB+N1X&2UC#J0*Her_5ivREV-`Xl5}T50@x>6Kyi9^!7dP~a@*4V(Knia-j9x2=*% z4d$SY1HqxIL|Eu)vHOC*3h%-KgsORU9$VGoSuishyyQpF+EM4!nI+KoEa(W&;gj?j zJvivfE4fSe&2inAj;A_l#ap!92MQogSw47sC2e}#%^M~w~9g-8~@M8J=MdrV!K}cogIKbo=Q_EECt`{0xv&T@R2mi@ZO~El0n$@QwHy!vb^k*o|ZQWjB+=WvXVf0 zGSE!1VmGD>sPf!|FZeClo@89ldYaWhsi(uj##6VPKs8 zRbG|Pn~74$m@(0a==8XUV58FfZ%ij%njbDVLUTJ2oZ@6{esg_L1m=|d^0T8~t!3K2 zJ3)21BD*$g8pYk-XI(&0DB{H(Ghiw_3?XYd>x{z4s3I`J^^Ho%6Fm@Se!NfWW>gOJ zvloEv9feYTJ$~xZU!qr0si+G;p}^3$X6?tIX~X`n0Ge0RfMiYYagO6(pkyGch=Vx39T zmwn46k{?W=j&7#OAO;qeF(337l4!Xc$Qq6s@`oxDsnV?#y2J{J!adb0aRPU7r zdnq~4;B#f@jd}tdFRz!U<6$##2P;LBmA-NBYO(}1X+*ERp(6BUJBYQpj-#@|Xy9)?2Y&6P^WP5QEB_%6USCrPJ8N6hbN4>T5k)IZIfo|zhVupX+-5cl6 z_HjQJo7Qa=nU{-5uh6x4IiE?0UJ}Sk&IMrtgnuq(zpY=yNETjj0x42^>VzYMK-Ny) z1qd)hGW)HTFuDtF_bgfzO}G2k@>hrTJPw&06W?OOR}2+3$r=26QPlOwcl62DuG@oZ z7u25q?K>PbKaUqO1cvmEpYC$3l+b(z-;vRZer2ad+QJ`jX`Md_{_2sP8_w#EFu53^ zmQ4RE?tLulCoS65ddRI^n5BJ?T)dGJV=nWtb^T*{Ko*tF?AfK2{D}TcIF%&YU+E-N zwR+m_jhrvum){ymRdIsijR0}zHut_Lmcv?Tv8J1B^x5-DFt$GdUX^j*8l7whDjzUA zr{}bMk&4U+2{HL<%=wtiOW;azV5Rqt36c9)vnB4C-=A98kodA(1zIbzLJ=GRA zx4~{QtOe zwVm?XmA)H~zBnHN&&t^^TByTmrJD#Tf?^yy|AeF4-jKjHA3thCg?ny7^kYXc?xkU= zDJ$~hl6=m#m~?iQgYXvIuV6*75bduovg#NTr}aCTQDYW2{VF+$g|sLI+M}{#)C8FOl?QDu3`)( zcJ=0B>7F0txefX-hJ5hpFi;yKrY6+xfzL)6j+aF6DOi_fNnz$}^~D|EWA6&_J9!_t zugz?}SCL-XcgV6&sP?9(!MZf+`&Lw?QP#+yHTcB>YIWH7Pvr7fm7Uu|EXbQ-bqQ6q zF^54kA*p8<-%BZ!nK7kUD?2+InXZ|lWJ-7>9C_PHPS35BG&f#amlw~;$Z(6d1L~yA zwGXqiI_~@~(6(u(Qf#wc<6Ls_D6Kfo4MH2T>hOwAWgW+gEu;c+lvk2oJxfJVzm&i{ z&wd%v{o3Fcwyfklj<217Y%v;itGFyxG#i6WSjUge1ph&EoMM?&bHw|3g1Vd97tZ5j zrq<SmQ8lf8#R8gMKu@%82Zy0F$*v(a&dkKo%oL{ul8z`prTh6|b zptJ32vK1lf=VG44=&O4rF%EMO3k@alZu7tEwB?`a7+h$QH_0iF>#rjQ>LkAe*kRG+ zc*QmLiZ-Hj0|L63WE!F`*frZYpyZW;QM*|U-}DDklSg;R5k}^KQ=4pkJnjnGuf0%WoXz@T z);{Nu0r3TlK@m)jCL~DW7pc$fGIlR^+*VPn0hdruX-A_ zAmb?;{p^5}aw#6@X)c__MK}}Fnv%<5-3XTxcn(IF{QdFcGq#$AC8@^0vBfUJls7kJxIcTVe1Y&SWP;R{oi&M z@ALu5Pzb9di!%l>#=fwA=5;o zNge)aZ4YB3>4bk>>ED0W8ULF$8PE@Jp*Vn#nqIJKTBlECu^e_xAZy*Pxmn78zx$at zdQ+flmB8js$sw&Z=J!%nDfENW&dqK&xvBd?B=vc9j^R6{bj7#6aW!{tK~9xp{rn_Yhf~y5T2t!B_K5>aGqB57yDP=D%7rt#yUtw1IxbWN*|y#x@~NR`ITo zn|tKB6-*d|$av#&*WutPU;QVK*O4s%8{@*bMl&obakr|S#hWP4D$&B6kzhmnvplEy zA5M5vM_2cnd^sN04{!8#AloLE%Imem^IA5vR~er1*MswyCxj%;%fMkL#Qjj(KMGV3 zmaL%U52da)L(17@JMY&4D#&ByY|hdh-hWyE|GFdDn|M5R`J23JEQ-VEBcp+Jhab|- zK@{*<6a6;mj%XE6qO4drRdvwaVFR(atwemm5pR7z$z|Hn*R3i&eD@a_xj1n9EBpRx zasOPJ=7)070Zuhdrzr&h`Dic zRmVEwK+Z1Qcyi9S^S35GT?6TF*CO9#2MJ%sSX3wJ?&)vhiP^riszZXe_zSq@o_s%! zUVgju9*f+;WFew}r^G%r8|oJ+)zxroSn7|H@3yG%c!btxvk_M;IuY0}utP;r&6FHx zr@>1@c-k?tvF(u{fFfe7=|-(@kdu`H?d@&r*xrg-eGBi$oGiepDq0iWGH;y|@}nq? z*|>U~Sa|X6xeMzh-a#NR?)0Y_D?pCW zWMF{pSjmpXPD;zv>ku~Bf7sc>Bl2Qe4~s&f+M!{o*UzEb;>h)h*F;)%^1kM~k!6Z( z;+Ct$yfn%2NSB?^>xu)AukS|DKZ=ApeiOYV1ZET1(~5hp25*46*5~~gB4~x^{^=cb zV!-xm><%lAqSqBq+BqCQtQoE>v?Zu7r7KO+Q&hzQD)u@b-qvr*PLX5+mfvpI!QatJ z#JYH8c9QhTa>WZ=}!qZFZ;U##Py;Nv&swZe7mTy+KFf|=UJKD%^W3NyYQvGy?* zvn$xo3-W0dS~p%Qv2Mt&TNm!RXbmjxYA)=^7ND*(4BV-GVomaq`p4N8w4C3 zkCbxEyF95@JEXMa?&&wY>R0uqKYDP=(;n&1 z%RgxnIdW`{Ah2F_wbR~{$Ve#f$osV(hk9=e04dAqe5=Vbp7smm%(a!sV7tkZJNIt1ra(V_DAziCgXJuS*)H|)Wgfy!B z!tQ^jW5Y3#h|x1j_}atu+P2g^;7*j>MqELD?7`h(q$dI?0~rl+GbW{l^y`_7U$giE zmBpoEc~-iI*xt*kDKi-vG*5PZb`}rRuS_*F8C_$al4~4TZcSK^@*Zl;_s@?|Tfctb7vX*uoyfH|HLQn49il2$$A+pFRcvZVnld;2pcK+l= zVsis~Y`L;-1wTIUl}}o&B_7gC9O+o)wSLta>Piun5@v)`DWuxle$ zuLkDvt@s^l@pRDXam>QWFOnEc31vi8i{U}7bL?(Ejp*o(fGh1)6A478{=qUL>g+&( z)TSlY@TEAbJj4}t&stPnIaE7AEZ?3yRCl3e?hl7H>;5 z4yh`_Q2SIKvA>Rp^6NJ0kVbin@jRtQ|7Z)g3R@k?ays0~6n9JYIy4`{`HAT6_a^;> z0FoqQd7XW{6KuaTQe4_6R|Y%NNc$_th^HW83mG|{euYKmY_`d{-N#}WLst*b*Yh7YTeBxp9JESI zg-=YA67>rvfXESXV?lDzT#4zY{z(i6PYd3!MNPB_bR%nBPpca|3cZL}DwUYN`Ycrs zg*RXN#5MGU7Y9gl9Y&k1R>^HkT42mzzDX1O>Yyi{*HB}rh;Gr#yrpAS*k_Dp(M2eA zBOFj>G^XE_ZT3ga5uagJYifVmgY4(nsM@vGC3%I|t0a`Soh(r7NKA|G8zhxBu1zwd zfAvpug>rDP1&kw>347^&tFK#eDq$Tve+`d&C}ZKDdS?v4BR``4B_fhD2s-RmvqMSD z&gd$OdXjzf6g6?4%1QKkw19Eov!Yg(4XsM+TlR|fd+j>uE~@qGzRSFlU_jPE6wj%O6i^n zt$WN;IN?v3IRZ?FEhk%F9U+ukk{U^76uFu3`E1t;Q#)I@ zY)zAnI^^>3wC5;W2Ciqk6M*5iEtIQq?4~hl-V9p4hLy!N27Mx`H&Ur1S(-~BnZMkT zvlcT8{tmgxe`{`rA($&WpA}S?{J!F`@rIhj#6)0YrQSwA!l_!U2*RL=`J-JZ>Z@sPT$=-3|9f^n@dk zHl9VlfgQA?n8Jq&>siciET#qm%llY>K0AC-&@BPSUP6^2p?J@TTO{{MOq~?@a<33}U`{iqg zdF@9ez{-{~R`jxEyaf~Et~o8AI4wjim4DV|ijLD14;zl1_&7!&FxZo=Fs$#|Evw+o zXrXz7LwzaB4%i+ITRJFKpLP`OV&?ee?6C}>0Wh5B4q^pALb&1)*F#dt=3=OauXL3o zc-3qN1rm$Kr(nFbtei|ExO3W{B@AA0InD~)Mf&7ZLdi-7f|>%TRAu>DeTVHnt5H`w z49>Mi?$G-EKsQP})z2|zK9Rw#M*(muv%KT7YUpUw{5{NS{v60X5bkg;Za{(PRdso9WwjoVhuQjal6y8w4#`71Mc-*`HKY+sCA zalL}9^`VPgMIwDXKfP(-^2sz^fl4C;-=ogtRs8VJCk4M8)VWl>Gm*C9>4^fa=Fm8| z*-w&ci9w29xK5N8Uk$`h@>CmIh()C2>;!n ze%EQ{+HEujLMf*1$;YUim@>thLaMt}otLslqq9xikt5L+rOec;N7)YI8Mmojdl0O;30ZWQ~qBQ7tIJS|E?SgNIHrRu7>`+RhD#4q3 zF9gcxIqtg7fnQt`I|#x$8>{Hys3kCACZ$S>Dvn@C%EC*x?3!8(;u50DkQ0|}6$=Dk zR6f_H)RjEp4HW7f>evU)GNbFXe*)XI)J(nuDxkR$No1bp2*4*!(mptsIw|po-eCHe zDQz|get+LZwS}mo1}2@ZJ>vCTe7%r1V+8YV)%4N&b=5g(B*Ebt;~p2xC&Y!_8R&`HKF3Hz>Xth$HO6c_>d>ds<$R-|*(zZ#y!o&^i>0_rsK&kg^0PCp zrgHoBr&=9uyXL{hmYZe8#U+7Mp>_d!3_44@zPt$5!YGrnG~p^i!=7^-^d7FSzQ@4V zYq*xeu*I*;LYz|Po_Ah^bc9>ueSn?qjaN|hhyIubF;$l7h|$4-g0C&nW7qC2hU^w} z1m+;`g|onFfKheoGp4jJQIpKtlw9|M6=sa5@6j#NrC>y=|NRV;qL#9#q&jmFhxK7? zjD?TiN&E=0373-K^L}bm3V(k!@d4j=F1qnRJ4`$yWO`lhDqfZHN`thuEo0nf-QClZ znWV;oAso=yl$q2RZ08C+_O*P{rggot_@`4_ebDhmP#@CO(MdcC<(C-m`+b9l9p16I z=@yeT@KDY=S%=4j=~h_Lz(AJ+&dy#M-2d&Gnz&VFukRJo%ImMkg*dL5N+X70z2`B; z(tmi|&lBoOVqy9Es-I?S@!G9iu7Uob`uTyee(ObgvPSqqdy~jGJ;CLaR~!cX;qCY$YZrN;-vA(& z68Tqpfd>Hu8S_}n>}FAB48Je;lq6d?fn(&HvNYiUss>SUI;#)?mp~R?htKA_p78Ot za2))jZQy!xIc<{x%h#(HV(InAeKb~KExY5IINHDR-e*{;c1%$U_p*}kZ!G_yZVSmX z6cX;s>zAKFXKKkdM}zZP!n5t zmSP4oCE^P=lx$ziHwpZWf&b&vmzb2E;P=GbaDd0Y`97ugXH_+H27LM+ak;CLu^;Mu z?IIDHrP*Ddl}xrMc=)bhOd0}DO}Z%clxccZA>7|mFjT%>o%DXW;ap69q%3vrwWrvJ zp_4{qxS)botJ^Q)Xs&rC3hU^}zz z>lj`f;aIb0r*jzA5w)lssgm6TasH|r)%|ErYPb$q4ac`p%t3{I6BoPLKj?46cAQ3| zXvBia?xOQ$4oo&U&NN&`FhAj4St$(qUuO$PRPc0!Y?c+?-0Fa-4yzFTxt zbV$^CompVu?3`Cassn7=R(eNH%{lmtxHRpRLCcQSMH_7kV0T_w8 z#2EWsu8vq4;1gIdAQDpT0>ZK;n(JNNsRCG=H;a-!g*OvJ=Z;vY-)b=NM@fthXg#N#Ba|Mri69lB+b{EvG2koSKZ znk*Rp;pYEyHFf{pKOXpP#_YZB>Zp3hAcs&D@5|T|GDDDf&bfG#Vz&!7blqi z`7n#J?Yx6i%k`jx4BvWN%Ue`j-8)_C9dnWPTtk!CeRL`8vm2hvyJx1~m+XiIvTcIT z4z^VTS^Xg$a`xgNeBV}Hz1T71U6*RgV;s-gu&qiSyJBd>??Yd2h%eB=A!6YGUxWIR z*?o+)wQj$I)ALG_C7Me1s~r^z?Oh0;#pXN9mWAv#(HZK#e5&s6w(XJ^f7;i48PLh< zEZX9OEk4wIp10f_N$c;FW9TtL!9PTg@ra$s^0r+QE5ODCJ<8E`4&2LbIS`XwhO8e! znogeIJB^C`irNaDK(zDwp;lI$rHYkECTq8;b-XdAadYJCT0tgS;EIJA@SogHI_plK z&hJkgS5sbXoi6V0!Z%;Qv#yfTNjQoVyEMELl>x2HyL$7($|HR$^xMElTa(y2;gGTc z-0IC7*o%YNb(GROVsG)TjK}T%TjB#vG zl#L#3&uTu#Nq9ev`iIb2;7lYs2M3EHb3RC5aG|v_~ z-GS>r_DT$=R}sU6T%PYQXYF!hHhxj~m3*an=JODIqD#N5B2JEOmLrJf@2tqj>YC3qW@p@6l7b^yMOtn7 z_P{S}$A9^O#wBi>D_2K}jEf)kO^^dX2JNqMRhnu{`pV6xrCmOw3nb8cWkDftpys%Lrz|! zE&s58bILWdFaLAz#N)iYDg>~+{7S0=H7B6IbhWzr)nm69+)oP*!reQI!#C%9ZR$iZ zVxNj&-%ox;fHQ*iL)p6iRPNYvYDvGeRj+KSW&i8m#QAeYVGUe*5L4@tGp3s4tJW+Yz2wt5eQYL^ z`4($wP4^>AP9-_gh3S_A;%D}&N*54B*0_5 zIq_dB%aDtFhv-w2?788>*rkos{ui>6-Z79(P8N1F`x52Z6Ijz#%MN08CSC5g;I{xY z6Kzrv!~xt|QZKo)8pD`IzEO~V#Bb@~ge_5d-(RTg#+$2}%JkSmGOWVirT9IjV?sCf zAj8=UGS(bMh>PVDNtSvu(4%?fx3Rt6HM1#tM6Y zRsT4}EsXd^$`0I^Rh!Z;p!0pojs-L=Z*q`%v#-o;Gn9-P8_IfSQZUM?hqV|eZf|zO z*{)=t=de8CTzNFalhM9v^b$pW*!<*x2k_zOv`s^Zkdd(-mNktAN|u=7{r zHz0PQ!?yyVl4Vc?-Lh|O@SG}K^KQ($KIdX}_1WFASbOR`1gqBIjTkCOD!K#D9tVFz zl8-(Wf}jZSaH@^gI?ytfUmR|>>lUGFiYPc@7lp+*l_<`>l605{s!^EW#A9`y`=vKkd-mFFPZ&0lhc~SDb4Z0fQgyDc_zR=^YfWugSIO z>3+BO=Bv$N82M&1f9-Y<04gq=AH{5OT(A-yZ8!@|yFJzJX>KUeEwAaZy$N6dJwJw> zrf8B`=3T;I7JJu{kt{^3Nx{poxl2tksrM|EgjjB^*D=AWtks4fba%iv1x*w*GE2d4 zuP4FZ%0AgWQ;u(m>>7>58p>k!_66!W%KGm=|LJD?jGfueuj_p~rrwMTCoME%p5|1_ ziz7{;MD)nJ2YT={G8^Psw#o_Kzx?N5naZK^&CceDewXdME+L;WJn zJ&dGtI-}%CHByipMklohxo0r!E^B9gI%(V#50}|oM{$;q(JdWyg&vLj)7?t{O{*Yd z>a@Jami6t);&YK-y&aZIr!gU3Xx|UNbXH$S2zoUVZvol|_EY+gXVj%XxY-oMP2GZlNPgwXu)$?6?ANq2C&FZi zkbE>1bbW(ny*M7VD%mG+OdocO(EV0s@IwE)_kHj25_0p|G)wew7g(hW`7jJkdO>3l z&5Up-neV(&#WDGv(%tJ{YR5T5x3R66va4KeElH^;97k7|b?h{H+zbho*j zf!>G`SPmb0O#dNfw{F)9LUgCCDML1|tIC}^0ovd;Tc>1TUp zwHMdX;IxW8%6RD5igLMddD9gFJd-twSu+@QmmRnGA>WjyBD2;amo{QgPVRwPgdVS3 zrIucq6+n8SEw;f1I@ruA&T92P)(6z=l6J1hU$oM_&(&h$9BjE+d&$W*UuCC1$NN&HahnAAXN+SrPj};cA=I^La$<#~`c$zD$p1XxO} z^mVN#uA^+U?Wvypfw&$uw|=;h5LuEtqkr=m?njzd z%qkCb8=`S`^@%GAK)T9}A`NXx`K%#Z6KPRucwC@JgH1Z1wvXq1`(>W9S3cDt%YZ!L z2K1A@gP>^3wdY%zdz3#>_q*zU22XF7Z4~jAJ69{t+^;9urJ~=p*|zwkIf^VpL`Vp9t>A0kGl^Ooi6X9?;rrUrs_V8!Eid~dP0vVK zBOnO_?e57V`-p>ryZzBo#=I48_V3~ynYojJ&IjmehPnA{f4Ql{panb=GZW(@&%lh> zrnq>Az`Zal-GI!<9gpS6QwIHeu!%Z(4&mMLV6=J5#cQ19s)~Jmq#J4&dPxOQjsh2{ z-oub1OY5s_=VyD&{AQU42Hvr5{dN)tBz;Hw;7ni{Qn^8!gc!2-w4-F3dEV}0E@hF` zGIeotcrNMdS8kPDE4emAPnR`pq<`(S;`rk5*P&oog$>Q>MnZ~Z5*_KQ;yDW63h@Qi zk<_^ke34`$&cd(2_`9d44{vS7I_QTBkrkF-mfaErI$9j`l>lS1wB39wV3d7H@d;Qw zR@l8W?lV67I4mrRW=XK_Cx|DorM5w0l76*wE(WFz)V_LjQ^kkoa#nO&-sVld)q5<<)H<^3rlzKM7|L=2D>$J>8j!BS@sGLYza4wKxwg9{1W2-`X}x z+SXK~J#i*29wuqDNYj_5!^k(Tx1#*GDQ=GaTY6}N^Da2JCGv{oTOu{SZJCU3wt%C;MrGeB!{wYG(?(V#BhVZE^OCL%H{* zyi&OGjVGgc$k01p`g!Ekggj+n`iHi)tNbTgW<4kO%0EiedX|n98pR!VcALftFwtU| zL8bAmfQ+eS9s?yy^V{?7!D>LDbp%_9h-E_eq$VS2i{4(~H z?yhBcbpff~++f9)#!IeGq#}DiWVZeUX(hMQob?B31(cD*H$_!_cLQm+qs3|t;^sC& zC#9*t^`(!NSSXY)^vhW`eBtqV9h;*#>F(|Td8G{1>Q;=}2AY`x9{E-jQHq#jV-xicXfMe(&R2S0{@2ia z>(wXpc3sakefoh>Mj9qrakTlO*Wp9QxtcF8T7`Ir0r`8XKE*a{I{{)x>av%ZFaqwD z4c7r6HWPbys2`>jCCb(=-S>k8nU6ZOMaucc#ZY`6R0w0T@WN01Kus#TTqgR!HRqpt!sR%5lO0%7BdS}m ze{wG1b6i*)c`q?z?*f%Bn$ ze$X;|U9OVew=x zv^1)9MD~>kD{B@IKb4Nw!8u_6JfK&nc5rNPeET) z(o0r*RzA(asPfWx_^nL724*c0T~S3LObC**Bp7NJC0U$M6%s6$_^xFOPgp3kk11B_ z-Tv%JrrBd3y|hp`4NuKscM|#q!)SiJUoNKw*0TB0jkP@oxMPRa_+3jS4@|%Kr(C73 zG(2PhA1%2MmNw;#NGbx~1J<^r7{b5I+Ugla{(M{d26f)2XO^oa-a@T|q3nTJcCOG;-S zz3p#D-j>RSaeQs|6M0h$z^-m)hiClTQi`g(L312%0jsDh%YLHp*?dFsThN}zvjAgO zpx~8s*JaM>jK^pU{BW#??KEY81-$1?zIo2{9q!KJGU|IdOIPNxr|X(C)K@)M-D#~7 zW5ZH%<}M4dmi2a~>^CAEV7sV%StBprocSO-!nl2MDZ{leV4;-vr@KJI)J>cBNuTwk zCjsO3&D-;UR%QZW?v*yXp94K@*ELkcLHDUWsi=1Rd2jGPtj70b^XT9DK#k__r!g&~ zcl()E`O0uxz~PXD;RnrGzD0S&eQYqx>4SnTJL_erL%W3l31M)20$x6?!Jb>Y^S%&) zuagIRYHgTygP-x%S50trras_!{?r&2+3Cb!CeAf7FCLyO2sx?$3*ikoM~u9BWngoVn7#O*pi!tz}G@o@>C zTApa{3%D2^8~A>c&XRkFzi@Soo?Ye>1)%_=*%VE5^cf~|AtGdouy#a^us@M5!J_NN zB^%MqlObzRg@#;amlr3ok&}ErpcRXWLMoUe~C1jB0pmiG?j|nj3Z87{U|?(k(|#&mZu4oy4l~# zPPzXro<@Y@xBDPltFa~_Phvux!Cqv;u6grYpV7MATFlrV6)oUg&TRY-Ejgeo^Y6H> zBj67lJTQI#E}&1+SN7$jNPxCcqk**7>xHoj=vLs)gC{V&%4K zj6B6hX31h!0=oj;x4vJb8`#La^GY~R#4#PxT&ucw)5rQ=(o z{y_5gx!$Glhv{d3->N^D{-5`^k}&sIR1%cWb_`!Dzij1Vl{DPNW>7T`In?1>(mkZ| z=>$iZMWUDSl2q({7yRqVdp-IyT%P9yMyISZsSA1k zi3M23e@X?fM==iR4cE~l%Tit!WljSRI|4MEyh1>9gJtnJ_^C}~naDx1hsI-725N8h zb#o>Xde0ri?d#DpEhnu^eAyX1GlDZn2Zk`9P1qznnxFX)WPKRVtDGlg+Lg|~!h6&y z0Zh?7xo0?cX87gpY#)o;g2|bd*MigGbHu(%j+iW#3w`aa6ApFBXZDF35(S!dWiIxo=q+uI@UF={4IBevXd^|V2g#uUxSqeJ0o zbjuCQSTDB8=CM$XE3eNE_0I&qd}i;-Rx%j9Z7^i>B30gGHX_!;$73mqI-bE-rj5LE zMSY3@_R3t#`8?F7^7NJh{%$a!@ogZE=Ev6(vI6y0%uzaDfEhQDJ9?iJ2hqcGVpr2d ze9gG%J-Z7vE!}dX)%mUTHC_+!aa09dsn?n8B%7ol>3GTyxNUWrAaaMZ@+1 zjvdbaE5#q?(VQF|?X*>W(i}t>-vX8NbwQSl{rl3KpmFyZ8bM@S`?XOayXC)AdM?^e zl~ZCnXGBEmRq^#DZhSvUS2q!n6K?KsA-~6mdq-BkzlCvy{-s1<`eT}MxRZrVwg znNoKIPUL*p;5>#otGP&hz^wVmdf)-u^#xKl)}_RoVx)wgn1~@D%$><|o_6i3`vxK`xL8{rv9M73Ygi;`1 z@IQhk*{jwb|A7ldje0gai?{+iFBP{{l;46?Ro#;rSIcyM4U&FIS(WF8h~2r2^xQ65 z-fUC#=?nZ}1^2dSPDbhNH;F{Jk~atBjMh2G_kvC+`G`sIg5?ogL|d@ufwp0cURNLQ z9)Uhsz^93?A~lUGLagF_b3HTzXB!awObEQz=HM}06NaGsZ@%=9P!1gvlv7FZB}7^+jtYyvjNdC zRAk{|!kespk=o73A>`i!B$Xn|Y^vHZ&ZcR^a9ScSI{|cfhg{5ExTMi5p(P5K8S6Qs z`3>$zr6F8(Ir_JN(34@@#U6;lGhu#WRWe_1owFPPt?wU((X@@wtJOLugNUbCA3tvJ zyYCXHeEtMQv$$F&GiYeq+g@Ok!p)_-dqUPxh1w+lvD-Pep*@2aUvZttd@WNG!^4!@Gp&s(W`aDG!-GN=pQ3K4-JXgu)2e%&5kPIv4@c%sbf22g z;{g^I%2*iP@cxTBAJ8A$UQAm>Lx?sZgqEO5j-zTc@z7bPxw(kdz`*Z&8?QQz^W51x zWw&p+o|~oNY)eMO6mqqm^ii)>WSMkP57%G7dFv>LrecM=T09kRqiGe|%6x)oflV;L zUAf#$E!R@Ca(C!teKXb)*DdM7aOjtVQXHi!qfz@p+i`Rj!ULeL9fOtxpZ_XdR`^0q z5>tW37}MT}l&~$RP!{sx9Phj<&r|$XdI}@d`aD*EZip9TDg1Cd%{=RjC`;wML_)7`d^!;@%8`R0XNL&Eoc{F@ATN@V2l zKW5sdaOv~Ue<GETnS%A~V?D%BwmAg*M>bls{(ubsY(6z35 z-#^qBswRy7NZB(?FY9%smRYCl!tD((yCvWqhlhZ?gjfWzPs`&K8y2J(IS#UQb{pZ7 z|DoC2jKd_k2F`UQB z`)=4Ol}vNdFpyKAVtPciW|My5azGXg35y?~Ec-=DLP;0IIn_K+r z@Lw_vMBdyddYQcNRdgat7_D7i1Ac3cKMU6OWJtQo`Mf*!M;s5`L_uBFj?8*F(5Tp5w1jlGSbNEg?y8(i4-n7Mxf8W;)X z6`wU5FY>xJd1nJ0BcN=lc(30hQ|mRQP<{lVg>3&X_ zy9YJj=JN$aYyIjXL?I?L#W-K@@f*XT14sX-U z?(y|0T8Q8CqYrRT@Mxm2IRQSz>7~1DqZFWebQQK9a!s~BGwwu#? z@?&Q_Y*M&RgZbQFx8RkaCtsz$S1X-zhQD#Kebchimtyg7Gu5+tH!oAB3u?6%T2Q!; zeWEQheV`j}bl(xgE4J+vlMI*D`-@HZkt&UcJ6J+;3AadT*+7R*=?31K1a|8ZI%fI} zi7x726F^t;3G}MGt1v1FJlf9LIq9=3`Pg_F8-PdF9P;>wNT#gI*#kLM$tv-=2RQ5G z2X>k2?p0@&2y#q=N~g?S{IdPZ_A6nZTXs4LAhVA86P2~AYnp?}$WHT=n#<77F$|Ie zB~5q5`c;vR1eE8OZnR+$Ti3Zq?&j}vZ*lAIob z--5A(**CMVf-6o4_vd;hDOy(d&^Kk7G@Cuyd@OgH_3w=U9*1f40}sHk2o{rRD8A=- z^xE<$i#LOB#-dz@vDVkK=qv4aOB(WmrP<{^E_b#>d`{XD{j-uFA9M5c_Nwr>hMdk) z;NVs^(%T`<{tC__)G7q(TI$xx;5&1ra_Q*<0gSkmFxHN%=ywqjSnt zU?Vv(sT=BkO9In1*r!O^#&^-0gJI-)67OrtZ9=)8He34k5q?%XH7Fd6CN#c(qrVj` zFwPq!AEvuYH6XV{&- z`g&@H0B`yTSUxlAP&<#zlK6k#RM`n+d-BlIytL()9@%+Er_Zx1=#x4~W8>`ZDj0j+ z3DWQAHQ#yl|JZvAsJND!Y1cF1~-p`E;>IT@zW-o7BN{x_P`6CIIiCciW4m-fpzswgTV@tkKF|-a z4^8f^Wp|KZWqs>rF)#qCn_A9p$SYW3Nm#xpRp3=c6*#Q;{McA z&mqIOr6(yq)UdtwR-tjUuiJu#BJ0{@$nC1Be z0L9ZH6)Qs)=t!lwdx4|kOQdrDfaqRVM+?aRCQPva)HW*>HiJ;MR-FvVdo>W+fSuQe zfTUd3j=tf&V}9p61!J*!LA(6!eA~;_inHTq6byEm;yuy$vtA^vvUgi(RV6!Jh%O|f zc1yOp-$hmIb=ZqZ3zVJ8L7;%W(&UCUtQ6iR^DxY!trpCW?SNK>Y7->~! z(j77RaIg1En~nW2$c z6(8LHSVoiKSY4U$cie7X%U=|dhuSbOJX}6ap&&0eLMS z*}V-~BY!%3lF!iBSGd)7}&j2PvjJlwx z&ZYr3rzJc-$|#2GD_ck~uxD*As3;?9(u6rhcwc7v-XpUZkEXFV#iZw}gbvA*K@ZhZ zEbJT7FhL*sqYAaJ#5|=3(Hrl59?eM&=7y^~^GV3^ESG^}oc5lPEa*5chc0jiqZX)N z!jJp90r4Hg;yjL}j8!x`LAKST!*GUmW!z6O8w8q7n5QRn!oc6-7QX47iJjTx$a%{e zZ>X|u!6iLT;#qxdg%{OLFi27kBoMjL)zV&X_fZZiU9IJ=*@J|c(0n_;N{4{fcPcnC zOXEb_Yd<{bJr&|-s`3dSQ1hjKe?t6yXyIi5!8M2}z9cwzO-9+q7s$=vyH)Fr9wu_2 zEYtFRmYC_ya8Jg$6y>SHr0O`0DD7@5ak^D)XPaS`;@%Vz>w*f1{;j>Y+XbceT4$-G z#ofms1SCk^k#luLHH1r+ zL)}_hP%><(2Hxt_2d~zr+b%e9VQ=T1BjuA$ol@3|o)G*hj!b2mvJ~9hF5~$TF{v1S z$OKC{8%l#tBx)uR`)mytI2k- z)m~Rs3b{&5kl5LAub^99z2DczmwZ;8K}Qz&4u)ccBjlU49i-*@&nHKozF40bPf-I) z_4YRBK`~tlPVEWV?yra~*q*%F84zu=Lls@0WS5s=aCvem`n-!1wh{>LQQvowQ6(nP z(7hCCMzUV%{!1)nBCE;b*n6M(sz7WkV#~!W zrY%RzYk=D9=^+9P(`5Q&Pgb3 zu^2=-( zu_Tl{dEpTm$ER;b1G9H~U8F~~q3ljt`Iq?X_>fm`SK7-=x0gc)l%NR_m+EbEbE{>TO14pB$ObB?!@yV6 z%&4uh^M%~A3*XkX@;0*QV`QsJ&8X>l(z6b$_01KwM!OTNhbhrG6N;2dJZ+@qy4LN> z2%ic{NoxBb*U+ww2(9cLC~e*r85YmRYNeg#pWd53)}yRr`QK6lBi) zQr}=X{@v{1jGk=an_;i8QmF+8=8bZDJ6a`1?(JZ$Q{Bt?Ou|{NTtV>6VP{1j?T1^9 z28Tx1XrkhVM+OVQF5GhX-lBdlM1t2pAlP>fA7X?Rd4f*5lmjW?VoKxcTE-Jb0u_Sn zGQlbK&$qipkTv)<=wkgc6rPgp+vNB{iL8S8ja7c2y=n1|(;|?rnNR-T^ho zqDf`rdDNeV)-f!U>{lA~QsV6(One^u1lnOCn zJ6HoKM?`t=T~1i+=0LRASt=qkKffI7wiYH|RE_P(=VlT}iV}Kkk1(8NV=eroFYqKR zmH74WXFI@QS2r!*D&@dM*}fq_nF_4YG&GXm<$B|TBe=?N_na;Xu)OHe?mq5PznS5! zWxX)c-Sm!Lk$lOgR637~t{#?v?Qw&_m zy1mg$h3_r#{YK@5YrIeg^c&NtP#-!Y2C&nwWum$fix)qbOvbBIv$oNykVkNH2Q&t= z5y)}MaIJnFna$96X**0IS|=9Wbks7E7ao03bHDLcQ=VvHNX963$vDuEXoWs`G{^Eh z3=>78=eE8Ue2}w_t(F8<$t9S!fuT!8iBniMMznw)F79$TTPBC>Z6&5bU2!*(_}!c!7)|j$9~VDWYWP$p zp71E&6-TEYT|THpEgrY73;d7(!q&TOqz(S(i1cc9nS6mhG#;H|U*V6LW!cALyP!+P(e zo>2LvDF+p2fYE*2g2$rngMm5O*&!4SU)kG-+7a)}7AQTo{4iOuuX4FL?c&N-%>x45 zyRw+D(+N&W?iUQatTHRFs48N{U;Gz+>i*>s4Yo-@5K1z(yhgbM1J99mG1foOoRvHF< zBLwC+?mbSz9&dZp1Z^2y_#R<_GJRAvHlH3SC zV~cgaP8=P0mp)*?wlS}xR!;f+%bXVaY=PdHfG4`zOYh20RXx5zM_+Atc&5h}7Qar~ zt@j4!PQH8ky7r8?hE*rzvV3m`|9#sdGC#X3x=5eKrcx*UKEu0S0R%~@V?M1D%&Jjs z16OO=Q10zc2RBC4%WJOVU;`YAN9wFpx0DKoyUQ;<&BWB?;#KuObR0OZKxR?klzMK+ z{bU(zX^1*{J~H-(H#Ea(^su~&o)0$xnR)NsxCz(1Y%lXlTs;Zt)AxD`Wb+CQ$lD_p zvLI-tA32D|{RkAR+pn1zcwdrRKPP*TH^V4Up7vd&EUvakR@@6O*p~^hEZN_rgE{_P z#?2go{8g?P-CZy9#>TsIF9^o5FxK}2*wVu_o6B`W`?S~1#NMKJ%w?xo?3_w-ceV-O zY^_5j7ABeG<9kL>Hp{it8n)rB$bp7rrA+XLQ$<+Z*LlYQ7I6=4#-f6FeV=kf84{kl zpETXQpM+m>TW4l^am#UvX8xmu{jA5+Sl;&rere&yD!pwN-}IJN$_w6aA`JY(hLg0-+$%WeZhrT73PK;lQhz-h~~b;qafc;=ZWy zyN5`frFn<_Fnl?8c!-}ot+e*bGJ8pJmWo(qrf_CPNYNL;wt1h9(uw@0_bsml&zSJS zH9%e#$etSaVT7z5R|6l_*8-AMXD9MkUQBLz_sQN)==xwvzW^NX^YeYQaBTsCsMFpa zH6ry3LJ4EIsg}xO&cjC0l1hi1|8)oB7$Y zf7ru!inUF;tHLGU+$WE5zbVjmB}@a=u5gS{Uu?Gs=wGZ|&R(>8?jvvn$JuY|z+!Mz zQCh&}&ZQLDB_X(XlIHAYS#-4#`7cG@yVQh$JOdgEG30jM)HK<%)77Xg-xdq(W)R(@ z81l7Vlmzg=QKD~^3rHfcs!s`7&Rz@b0YsOYS?|odmqn6;!y%*(ne(l!*lMqz!1gO! zsG?!C+0Qvt3pmWoeKC+Lj{tw16C<+wo-DhDu{%LWllsXA79ft22C&Vyuus2Tr0~Oc zNKq0|BYi28p-L!PJ2eA`f+8wR&79U@)`91RAZPH%!EWyWo}B&pvzz6r)T*>-J4&cC z%2)P9|0Z8Ds=kO65r>nLD0Ekx#{C!L8n$~}?>o)};DnzRqou6LEeG=^<(3}6 z1$i4@-yjg0=-xQASlcwRlpUuyi(dFXlJ;xMQ*_sK0%lG`YKM0OZ*(@lWWjsgvy7T4 zeUwk;2ve29C%%}f7WUo0h`C}~wcUC+Jc)6_I1;SVr&)H zQ@b@!MTfpLZgP#>!toG%hApV=w-q+>UB3DZjc?0KJp|zIxL9}{ZaU@Ue0ss+ru%h_ ztS7LrkTx>CvvRRTl6}F zPQ|KXyW|!X#p#+%DMY%~e3|^YyP$i@tI?7Yb=tyEr8%)k4%GyWVkojo{!?ufjcX{G zJFAm<41KRPozc9AurpMbURt|VEH)NDSYp*@1fE{65G-{gOPXDph9cCwA+9$9sK!X{ zjBm5wNJE6h@9_OF@)+_#CuTbR>GiN6K&?s;b@}VWBeW{R@cPtdBV@9r?@#&PA$&F; zV{i~?CP43Qb;qEv?8*cohYW<#SfI&~gy3`2V#VRg0XaJ?7i$AgcWsx>ln4|{_;ycc zR5eUGp17lVqe_zlz5DPf(Mx!>;whJ6ig=Sp;}srp0faHP5e3i-2A4l(qj}+5j5C~H zIpyaat0N$=CKnf;x2!pQZU;CS-X20o_k23g!mErcJ8`)ljAT`gbDcYqc-%13%MH)d zWR^D+9+hizXt7hBR7P+1mboazXm)vuUZCvR;DooHg9xiU2``*5? zF(U4UZTYOwZVjdJ>a`z?ev_JZGJ7a+SsSy%dA37oV`F#e&p%NMe44)eI@6BEZSW#c zhc%hn0*}A*RmTIFF0aBI#GX>L5p=&?nY5H$x~0G;#ucV_>uP_zDbZ8oeRVv-SirQh zk~QYFNhN^z!<@`{8MdSQbaCefAY)W}n|Ush31QgX9#IrF~Y>sb3t2lrLD{l}EV3qH5QQFUW4-}B!`Ce_R+#FvY`7C?e%Z9Yu7*yns zxD3{>z*%t*SRU?#sQh?FuSkRxQYkn->$8x^Ay!cI!^2!6R{!E)xG5l|sLvE{W{^ns z6O3MFa$;8&hL&htS?(D)Jr5ITuT0N&ed#E~F6kHgJsUvob`S4uz4JdMm-ZjLQa~1( zw(a6Npbu8}mbsx467k8|srtj7JIwHhf8`4eQ2b!+9TDF)cj-N@x~-c(CMBDQdxpog z{x{A0CoSa*1MJYcfF4Aw^3w~0<73nel_TQM97$rOK=6ZHL`LHf)bymI#o{d*k`hTr zx?qRB&y5Nfv+}GEOTs`tmdShQOfSsa!ZB%iXu*KZq>rY>@0QYJb6R>|XL&0K)quJQ zp+IX8d{}3seAEc^gx8AJKY&x==q*eWcQ5Nb$b!6>f~48j{@HBJ91o`r+A}@JUM{ zh11hbP7pRNJkWhh)RVjEzt}fO8pr`p+Xpp8=Jg}!R`V^$i7`t~5J5+c*XEB5@E?wS zf~Q!_?m?GtK3_A*6tX+ct@E#!Cyl?>3ePqvqyvoMIYQXY&jW}tN?7v?D>3Bm$G8VR z+Kofax=SWFbvM-y+yZRaS+g4Ryc-yeUzi;ECfi5E1*g;Pd){kfQSAFfld1uCFd7cb zC0YDHe%~@8W*hgLAgx$4FC3TM9$j9jaIAOL9`pJ)tSPuQHD`WfvM0atwRS_5&G+t4 z7lMiAUsf{Go?pzOU>oyad_Ag$4wSq56{XLMz|_nD!h#P7iti-zp-~YG5O}cd<7qLK zn{T;10u5G(=%ax}-^id<7+ca(9hv;Ikl}F1gK%AF2@u;)1X_}!;S4?aq!l{J=kmJ#@q>Q=7n+{TUBMU-6 zR8ehgrq(dhzz5gF7gTvg!5|!=Gr@O9-lg{{o>VDCOIxvo(76ZCzVVQ+XtC;B8*fTu zI0Q+XFY0UkWzeDBCfMGskRBHG{rhkLj1JP6T7@@X5wL`AE|wU?S{xZ00yJz1^Q{+` zc&(YfhhJR~KjD>}DQa_dI)s_tdfmGQK2+CKS69Q2&dPYalwbp z^|Z86`;I#L!^6HNN*iL~bg^j$cic%9i$e!x=DvV7#I}xPXs>Sg!|{`%=pW&d8-v?d zX9#YqrKm621;N#SxHVm?7#mG4+MS6>B{}qXANl6<^Uzp4q~4-<^@Dtu+K3ekE-Cw9 z3}zeZhVQPj)Q(Q-$rS{Pm5^R0tk=WFDtcvC49h!NPxXCSJO&(5{(7+@y($qFKNvUG zt_#mF*LdB+^c|A`B2+qWSh=hD#%}+${U5u+Tw&!{^ zDzyFFp-D~MBm)micS^mD7?+?b<`D((;ZH(>9?>gU$fFD-nZ0&c>aoxy@>pzIAS;h2 zT$)k?{Hs<+j3}X4ua(_hV#+=1vTCOC>AR+pZ@S2|NVyOE0HxZq!-%7}8MhiMnxD^~ z0u>a}Vc#NE)%lmJ2<&p;=Vfo--wOY5_URRr{-alayzAFblj1*R|65=%xNWTgOVc_` z5!Z(r(M3tw0mohj(j&hvt*Rbdb6j=iTXU^Fe@_?{d#Blhf074xLAjs>Wa2?Lo|2y4&kl6>pV(^FeAh_yZr@ z`;AX}R^NY_+Zn~YhH}8_^9Yd(kYQ@FJS^tTXbSMBTekd$MZK312ry5lVKvcU|Drkf znj)&E-l=_SgUTFI67=oFOi0>a{$rt`UMqMDC3UxzOld9r#yeP6cEQI41w-#57IpssMF{dB#b19>rW@$!XqMb9^ca#PKEg- zh%Pg!*xA{kU{WmX)D)_*?>T~7sCjrIeeQ4Ip_wNPMOe7LzkC6HQ5Wq0HUki(2M^QU z9<(Zq1zlW)9(;T-l)ryEAymrQjf=6g{&16Yw$$V~19;s5Vi0sEsux^_E8pn29C5Vq zFi=y&r;&>`zrQ&@qHPK7eY9qbS}w+aDb!cSaB~^c1~$#AtGn_Pd-!d@Hx_c?U`o{W zG)Y?OY_%1`30jymBrLC@B9qgkN6i9+PxPdBn^`Q^ZrLZ2NboUSen(s05UHh=i_;MH zL&WAl(%9teqt?0;eEWt)9-%a}$5_Tv-5$m7m`!?=T1~lDH#fT;?gcl<1;m$OJ5l0Z zM5`qFBH52GQ+@r4LCJ>z>QdCPVnz-^CYwFFHisUs(u9UZWS5jg1%LN&-Jg9WC58B| zD8XT!!LT~`wcgt<2syZ7Y8s{{OT6lTCd(Id{<6-5HzCl|DVqr@d%fS;@u7&I|*b0IUPan1~T&?1za2HMU?$8zsz& z3I|$xtoDudfPpE+VgB3atgHu%$|6!h$jGoX#6h0diSmFUw z3yak~tJQF33y}K+ifoL0m+^_&X?&qqMsv-8W5Wj(zxeoAT3Y%w$9jQypb8{&5iXL5 zGyI3o{V``sL1Cf!z?Rba&O}J3M8JEOh7$-vZNtUv755-4{64u{`BZpP62Dn4-TvT$ zgqxc{p$WrlJ+&a-=R|gtl#s)|yp`jxcD#QZ&bu#v1okkPo}Yg#u0x7+b*rYQ)!Gb4jDyJyW_ps@Hk1DkDYh49VAo{EuCoBXwMI!{Uf>e;o| zX1+DJCWX_a8;{KTLNh4`J><2TUe_+WMr>wtu#fjG?8A4s?vWAKcPL2*cG=&ZTw5?7 zp9aU{9>60LDm5mr(R3ASK5*8&5XzQ|LU~4KhLn(y;04)nJUzqWFz$Q_#|5y?FR4yo zf9I{Nr?*(ag@AxaEaZcb#&2sn)vv=K+L~_aMRWK3HCivq+CSl&Q#u6H=_*SunBcJKDk)67*Bmm z&@B0?Yu_dw9udLF$hbAS+OfZo;{HaEMlLF^1oNHG_m8|q8e=U67$<0GRn-!&U*mMi z$P{Z=e{Mvq*A*~*WWTlk(}(5PF--aRXOtqlp^=f8)egCp^>ry3_uD}%eSQ6Xa6`!S z^t+pjn6EW?!Kam6aFD}AsbwE;7~5WZd4hX$aeykWuFhjqV$(AYuPG&Ya(T&c)anDr z#LQGNXB{Xf?N+Q7F(9-2a32;O-F@JCb}?h+Ads$?ceE@Pob?eQH7%p6y4vB#3YDak z6dG0y3?+IqMH(y}c`Cg32&Rj*Fk4f3i(*)VRJGrQxA^+?yts%7`(L9Ax_^ ze+vZyDdB47Nnpo|2fL1u8A><}1@y206nrx4E?ZEALx)GqW;$yKHf2t3Zc3XxVdnYw z{D(3SSnw0T?*yMM%b>+)f4)AuXVd+|NOCgdp}6#0*0*n;a}~sL8jI6UD;-5|-ZMri zDb=N>6_%7xJ$p84BIuoc`#EVw^fsttHK%Ulk&)3VT;1ZK3A{`-FYlGMmp?b9?{6&B z@!^jMYL4CJL#deJR#v3dH6(m?3*shf_z&>@b-JY{J6^2K(vY2wg*01HOQ@fp_+xBJ zY8ZCH@MIY$8t>Og75XtOna@W@{ ze@cs+F!L~o2nxsVcX&Zg6zmTKE?aI=J|!jX88zytK+!pYK(eZfmI1O%FD@_HPNydh&}+GSMvMyCf>v&kOvwtCEw>RweK_#L~(uv{ToKJ@+F7=hV`mzbPwin3G zj(_uN&6#YZXl^cYiUdJiVc_HxJTEY9Ve#Vc9OLWH2|&<<`5jCCuCE-oSfwI zYO5EE`g(dYEf1XM2Qx9OX?&SgB^8Atuh*o_K|%B(>s5Y_D1qxc+B!OO4FGJ<%R?$* z>dy*nDUumYr30-fAroP99F(_U@>C)@)-CZUa z&8P70k;ELV*z@5M5P0m{by#=z^hleSH1*v7oRliONrh_cXhf;}BiK_0$9-Aa4s1{A zmgio-eJd~DhZ~Hk^0maIFYWNgceZ>LmNFd&M$^rG*TH$+rHFRQ8-*en`R5jHZb-`c zaT4($l7ya)*^OKq8%shGe6qkfV*3Ip< zk~TIrHj*OF!}>NhiyFp=j@u)`!^7hs)l<`>8}^?*B|j1e7KkDtBdh6aztYnq_wn($ z0&Kaaa-#ye$8OSG`C0J!6`i)@3Ihxmj% zkdMno^MlPMia>W{m`o%W9y7}SogHq@vy%7)ZHhzy$~qyAxV!^rLc-IGEo}QAhN7@~ zia;MhbCCbUj_F>a5}fs*W)DnCArf*GRt^b>jC4`g&@d`0*$gi#&70fV82#$C@8Y7% z<^I@e#!JS`40%Q$AhI`8Ch|_!+nZdc*4nr|Fw8AYQFG9oi)-55S}19tD|2x!Coj6k zn)CSL%tUU>Ra<9d5iTmPTNZ17#BBC>Rea|D2qyl(-a6alNvLeW-2kxQw3@ayHd!xt z&TcVTFgBR!%{q|8zE?#C1Nhabgk&+5uE;H_CJVvo>FIMu#x^jxbJk%!EG#T{Zf>x- z$&-jrIH>>BvA_QnF~2Rs13dh!O^-!z-{3-9odbT|+P5M!j7J+U9dLt!kYN*!>m15| zi_oiP)!N|s`T6{CI~$u^QS-ryC?o%70Vh6S$@j3(bO7FGS=V}2c8hI*_B;ERNRavd zBOVpO%kvx{Oge~;GF_BvkH#K(G?~`ue@u8_k9Fk9-He zv1ID;fhVxd#(zKl*OWOnf3WHp7}&F(k@Sd7N+PVPb^*RNSxp~b_xY>yFopf-De`-m z_M5M*%Z35kwAPUFh&8GiU2}9vQ#M&$7@zq2!-W5OIDJ^w%NvnH$BWmq-Is`+oqdqN z09IiCg0w-Ytis+_;J>Z*FIiqtXyK56WY-3!JYAun`q>Ubx8GZ3cWq{UT?z&-j@vlG zw75><|4K&^-g2!SoqaC`p8DkE*ZuJ~t>@*@_q+puIbX4PW$Dt=Wk<_oQCv_kvHhTvQuhOzH4S>eQf%o7^pv0+_`dfW+gmG$@ zHW0p0a_$JmKGi1tAIHEiH&*(dKj0I$>xW&HhFN8w^d;rSF3fnBhy;HNMF z6jV?V+BpCGbBJ_BZt){RoLGVAAK!&vHN`B2@`Vz))9??bNLH~|#oMf`*Djl36LMaL zaEBgE43Yy<+%HR>a@dN3@^f$~;8>q(6+eK}Ov!BFpl*2!j@TOdMB*k&xIvikqqCU+ z`%{;@nf*eHNDL%7c%dYAbN9TIurpgDYe^z`u?9-$3}LKg)6Kba+WejuHH|q6|J|hv ztrq6{61we(8LjVpsBDdKw?0HN5)RKCDyTG@ton2zcnbXS3ut(M7hflFZpEc~qXa8W*bMZ{O zHn1nJh59gcP^U63UaN?@-5x!`s7pgF1)Q`wMf6!>Q0<&Hyi6mB0ix~C`ED;l??1j_ zz=`iX*E!nk=j42yDbqdPRqUcW6s3`aGjPws!MasK2d-!kie}hUrnoY~$jv2ar;#cr zsWxTv4$Dmox4(v^K$_P{PR9!X!b29GvfMA5&4!2#ifA~GM;o+A4xeOF8!rgkDyO}y z z)0Vc^tFQATJBsS=Ew~GBj)_HQecETy(gw+~hr9C2F`&KAsf%BiA7U=)WxAX`s1iDK zU06nv;C#f!d}4FMe8j#Sy@>3CX#U9+%o-Vi-kR8uI3hI6oqc3F{@}^h7mFegK6}-| zelnMd0;4mrg+z5(Do%{Zh>uy$%}e+%PiJXwg6n#(ujrPbTp{#ogzI^{!Hi<}PfgNJ zEhEDjUhsU%3jiLs=`gQU3e|JbQMAnVv;n9ntzUQ)%X-#_nW@W0QyCvgW_q&6)UkD& z&GcB|2s={L-XFhd*C{GwV(Z}e5)W||Xrgp{?+5@S{)&qMBf@m@pNmi%UR$NxhVO`1 zY1AI^!>K(%l#^CXxW|ktIgpP`)>^PvV0ud_+M10m+B*IjH)m1csnvySV%cEUc^bCc z!BmSH5nn_VCB|3hoig{`uLfsYN{^Eac!Le&B+e(V6(i=mtzwpjxg(~<)t@{p`9X2= zmfd()>m%H^+9)f2FcZ>P7$`Lg0x>DQC4Bn$9!`3Kax@rRGY=;>h40joMW86fX@sYf z0kxoG-%C8eg`Q%|GdS(GM-gp}-c;z6+N}{q>9i`qS5=!=hf}xQ&rg+%OpV2FD=x_u zKKtT&Iu>~@<>@7w;BcO>jmsufoX>&X&-qo#{TUZNxsEX^XuG}K#!N;CQ>rT8F7b3f zpKEzT;zSN&=mBz^1xa*XHPpFRMz8}7TC-zX6qQ>@>m0O&gBb&jW){1o-<`~N)Sj*N zYX|H3qNC+PS3n0cPfGQBv#_I4EvjO0kI^Esj20(^&w3H`wGK?x=XF1qpy00C8bNx^ zo4qXIRYdBO+aD}-IYm))YzYtQmR(tNfbcU8XBSd^-e;l|-p8U#Ly^WejrADlyDtnn zLX8(LYCk#uPb;9DFBZ%Yl#xrI1d%T#iqY!MLi?=nS*jBfKF^9ds z-II#q)dV?T?>F`+B7Mt3fjPu8xSm%o1fo8aBX_Gn^)9{sa(wfzoH}ZCYb|AYn*(Cy z&YleZB8ktbnV@~lu!Krs8vKSCH#B*Gs;RLX1_L3Wd^2(5#T7#4^#saChQ;9x`!YN zUreSwkr^^{93)us`~Z(G#$vj%p%8|C%3*8o&lul~n_H|~=O;($8+KPftf+!T3yo+2 z<$eie_4U3BT!~yS=A?Sk>`OV)7$@Tw;xM;)qOF!9VnGer#6F&+i7CIcC-RoU@bVbN z{*Co-ll&pYZ11CTJ;x+gcoXlBo7!Gpzy;ZO2N+Z|!tpU;8hc9ma#;=iP}{M2N&Ia~ z7_FAdaF5H?5lf>5Jmo%1$L6=ZV{u%~>=XX>moFpRGn*3HU)aC3wUBZp9x4$xAyA-f zN(fQ_y?1OV=pX2?mBuv^%>68)xJ?)*`Po-g2e7e@urIh{I)j8mN;;eQg$=mBz@=MP zp73OB13!G~G7#fKWv}r2PdulRcZw)uQgSN>jsEHV6I(m2BP1UUG%i01F*7`i^k+Qb zF)yLYC$T4pt*4{Bp-=ZB%d;J?*NeB+8V>fBQQyjoYiTf=CGF*R!S+F zx7aEG_jE^67c&+H>fpar=7|FxqQhkA=eiA_7%07hOWtNC9lS3bVz{(tJ>XWg{^Ixi zCRluPp26)$o5ig5TKij|oN^exm76&B{MfU*_9yMk+`YoHa6GT;Y%u9$5yp%tPA_7% zKWY*RfJ94!<=i6gK^w46%*98|yGD2braS2bqp*u$a^}YJ zW7Z)Y&JXnXA<{;ZW+$6E5py2S1W#yEW1Y6*oL4ZH4B~v}G1%eQG>F}APv0Y?V%Zs5 zt>;t-ZHJDemvN zzERHy81nfv#n9JA^k)?Hx^~40b0FD|y>5&Mm}1kNa(^Gql_ehROa2H#`ieA>)b|Pfo^DQPyKxH6z}bMB&OZ0r|cp!j;-O`LTBv zeY8HzFyql*qD2#Kq%mXxsZuuVoy5se7(W~rp;5y46jfP;n-%vz;F|cw+&W(bD16cRbjN!lQ<%@ zez4SM>Y3k7KSFUz@46PHy~eJ`88Q;r%?vRSX8eQKcS!auIZ@YqjnzB=NyGB%VK#H4 z7+(nnWvuQ^UmOgb!sH%s%85yc{ii=~o9 zFA4HuYq4`KaUY$sB%r>4+}u7wk9xNgUKV9o{uU) zWQ0V-kh4eR1haS~7ouw1)(I%ZoR-nu1~9k*<}nr1F3D!D5-Zk0EtSIFl!iE&jx-=k zdT91dY0~_Ao9-q1J183}(L=YqLH`}f;7V$m?R#v|AI|~IEBH9!Ew|^5-UyOtheE+= zKqx9drRS84vh*|2lF_vSJM8dNl!ejpjdo-ro&YoqmG$mgZ_1<&7Z!sBzrIMhlF{+m zv7V*Ov)YTjJGTlX!F#s?7XS61zQ8V-nP2hTh3Oj0ZKIk1=u!AVnFOf{3Bn4QU9W z&Ch}pPn8o_1^ib@-HMek)t-sgl?!18>v03amY6lKCQu8kF^x82fb0Fy-Y@KW3QXMj zrlval;JjWSpkUD8i0xMJd-Tq{jVYRngpO&cG!9)|Ybyi0+TJP!>;*cLo0R63rUp+# zzI;^fky1GWB*xzF44C>pZj-WIsi9mR+D|n&;T?5eL0r2f@8Qm;dT(OkJy(c=t zzi9IEf_pq_5p+ww3A%a0V#?ddxt=2m{juQ-~kFsUEOp-CO% zd^#3|pAA1PuWt=a>8(TWIOYaO`+8W*5x!kERej1ErizF-zbA0`rhTu4YFVv0DriFk z=`+uk&&U7YA&wdY!wj zXL(rD57JkAB<~&S;0U8?-n@d`#pUX-=<-+)oIKdyd&Bvh;1d(ix3)pRiZpNV>Nal}2N@A^NUmOSRw2)@N5ROaC67^c&P}dq@9XC`b;Gj0AMG zf8hI>@5so9n;EBQN@nO~S5urKQuA!32 znp^k+TW8Jd5QrNz)sQ%t3;blDCOh?=g?{OVrzPceaMK#8YbjXn&J6H5@eJw6hEb@A zT;&nEjo_Wik`7+?(3!zTRx1Z}?>1;glxlg&Z&9jwK*P?ke}rPa(4ktLd_jEJpQK$# z>zny@NJQAxs@I`Aqz5SwWwiVPlfW!dLcB4Ajfoo6aJML9tyK~HLkr+|6#2T?2%hKg zhS|5zrj@;dM#&2C8VF4&kvgKc>1NZve!iU}>MDrNXr0*!y&htfMyh|Vd5JhxL z%4|Af+)*>P0GS{pez)j%u<|ospYhEFfV0qz3fOdyW?3sTq)>|@u17EcE&upN2a$Q% z3s?nhP729~mwg5yF9VHj?b5eqp{@tFmI557-)Iu_o_p%kx%JVoXlE{OTJ!2{*)hX* zH9(J&#q5dzULg6jenNoAy zt!-fLGWGJ2i4f#F%rex0;<@z87XT757vFr%EIKe@{Z}7O@7H%;xLTgH!CaqUUKW^R z_b46<`hh{a9;kn;_DU-g#CLXTsU44K$+!;9l`5(bBWZRVTe@VH3lC{Qnylr1kr?cM z>{9m6^`2NP*>+(HpH;!}_L{Skd@Y^~<~FRw&}28U%Vj*NokYxd9#>#K${xf#FOMoM+%;LPGi=_G@^f?j| zA!^bJwPfJ4PGz)5^o>`}pPFY2r9|piw*>NTJ=}svDD55f;Qlzo6~Dm$>kJMD|NWn5 zaGgjbzfL**QT{yr=h1&4@c$A5I*{A@!NL8@+v^ejUx>9#&i_A;Z{UN=+x3pxmS4Df zWm_kK_-9McYCKo`QKRjK>%4Cvx;g;tKg(OU_H7rx^Kvf>+c`Qr^PDv$5dNxe9m$~l zs~^S)n17!7C5!*nCjIEI`=Mw&zfS*o^dAWP1A%`a@V^KG(5LT(&jz`z4u0{Q+2YN~ zLbaX{r|sbq0pveR4bu`WfZnbLzql4$S`8TmMQPJ7-ex$F7!VLp{TH9LosJ#)+H88?d5)TI59fvcYQU-<=6bwP=U{U18=}EN zn3DE{z%ck}{;vnECr;Z?Daj~dLPWUQufiHjEze))t<(R^(f{!fm_3BWm`P*EbTd9| zx?H;6h*Q~`VTalrDE%7cv`WwYik_{D83Q0-$$dM`;P7VkTfncoHRk->PG?&~^)O~h zMovDoPz|B|^PZoL)-=Vh=gM9FOL?zma8S3--iX5MxJ}Sut#ke_+~Tu2!U%$y@ttIIYJWzBoMGU0+|V$IL7s{8?Ns-{arC9wR2I zQKqlwg#Kq~D?KYWx5Y?i2DsL4WE<$)4EZ;aYr%w0n9QhAs;lMx_}BA%tQN!RDGWd_ z$0g6Brt05bdb$?DK76q^v)vs*0C#T`eoZO1i7k+-Jj2s{^~KG z&3_bI{2xsn|F>2wKi^saYLgO~5n0IcoIoBe&vVIh9QBJ)^xaD+OOt&;pt`y`H4+`X z4#XPUGAo%jTK>8=+&wd{sljB(!Sw#_Up8Ssj(^KBzrE@=JMzCf_~_rtPvE}=)b@X4 zyV1Xev45I{gZl@zFq-@a2>$@#U#$7R#)K6t4@&-KhQAE-bpbK?M}IG{iTxThZHWn@Q?0Of+VIS>I4G{9CWt0jFT>~MH{+tx-kA}?i zFM5tz7H`tg{#p$Chh4$I^Zzp-SpQ$kU2ySt#N!|Fxv-8}f_2a1$nwIx+yxsWzO8C! z|L6;APUK8&XHUO-)czW~RBdE#JUm@}P`&lhq+@%RkW(~O)>wfIC+jdnYxpu+DsI$+yrQ~c^+Sz!#gAMOK zxcB<-mbSgkv(Ac#u8ob^U)tohb)nrEG1H$PE3ZD>{6r4sws-3fFTPdzZO^^)VxjYW zFU+{JYsXfp;~W3Y(tW-A{roFm9xjfWw`cvwFR@quEV;Gx7OXY&Alk4m`D)4jTlXH{ zx@o=Q=ey=#ChtQJZ!of5ylR!KtZeV|p9-DV7#JAMJY5_^y2ZL(|NeY3eYV}!tT$Jq zR=*6`YP{vvT}gP2e!zU?_qcAmYn#5$`o5V_;_A9%lA*JGE?8WT3gtBoySgSkRDSid z?)BR;VF6H`Zt+@k)vGmYcO+go>bhX#MU|eZ=}}kD?Z2S_RdT>QX@^YOkKIBrE$iQ( zil1X+`|s#;O>+jdU_j{oV!-2mg*Ng8wE!A#3mSPmi zshw9rDF0i$2oeD|<`gh9 zBPtxmhMfJS>H8mUW4*3xXCGh{0&DTE={Jtb Z{%YQ|B_~=ÀI!4tT!ah+2_696ihY*+vQ literal 0 HcmV?d00001 diff --git a/doc/assets/dataset-read.svg b/doc/assets/dataset-read.svg new file mode 100644 index 000000000..54e93f689 --- /dev/null +++ b/doc/assets/dataset-read.svg @@ -0,0 +1,78 @@ + + +Dataset read: selection, storage, and deliveryResolve immutable input, select metadata first, and fetch only the projected field payloads. + + + + + + + + + + +Dataset read: selection, storage, and delivery +Resolve immutable input, select metadata first, and fetch only the projected field payloads. + +Dataset + Snapshot or latest/stable + optional Split + fields + read policy +HTTP Access Server or trusted native API + +Bind immutable input +Snapshot ID + optional Split revision + +Read plan + continuation +Selection, projection, order / shuffle +Worker partition and position + +Chunk-KV metadata +Snapshot, Manifest, Split membership +Sample fields and payload locators + + + + +Select a bounded batch + +Inspect metadata in bounded windows +Schema, membership, and selection predicates +Split filter → shuffle / order → worker partition + + +Project the requested fields + +Resolve field locators +Inline values or completed uploaded fields +Immutable upload description + part receipts + + + +Inline field +Value is already in metadata + +Uploaded field + +Ordered part receipts +Lengths, digests, and locations + +Chunk client range / stream reads +DiskIO · TCP / RDMA · mirror or EC + + + + +Verify and deliver the batch +Frame / field integrity and exact lengths +HTTP: header → payload → terminators → EOF +Native: direct RDMA path → retained host buffers + + + +Application processes the batch +Save the next continuation only after success + + +Next request uses the saved continuation +Each request checks access and deletion state. A continuation replays a batch without a persistent read session. +Fetching, EOF, and releasing buffers do not acknowledge application progress. +CROWDB + diff --git a/doc/assets/disk-allocation.svg b/doc/assets/disk-allocation.svg new file mode 100644 index 000000000..810c60809 --- /dev/null +++ b/doc/assets/disk-allocation.svg @@ -0,0 +1,109 @@ + + +Disk allocation across multiple nodesDisk groups route allocation; durable records track ownership of disk zones and blocks. + + + + + + + + + + +Disk allocation across multiple nodes +Disk groups route allocation; durable records track ownership of disk zones and blocks. + +ChunkDB +Requests space for a chunk + +DiskDB client +Route by disk group and current owner + +Group 0 · System metadata +Nodes, disks, and disk-group bindings + + + + + +Node 1 + +DiskDB owner 1 +Allocation, free, scanner, and recovery + + +Disk group 1 + + +Disk 1 +Zones +Busy / free +blocks + +Disk 2 +Zones +Busy / free +blocks + +Disk 3 +Zones +Busy / free +blocks + + +Node 2 + +DiskDB owner 2 +Allocation, free, scanner, and recovery + + +Disk group 2 + + +Disk 1 +Zones +Busy / free +blocks + +Disk 2 +Zones +Busy / free +blocks + +Disk 3 +Zones +Busy / free +blocks + + +Node 3 + +DiskDB owner 3 +Allocation, free, scanner, and recovery + + +Disk group 3 + + +Disk 1 +Zones +Busy / free +blocks + +Disk 2 +Zones +Busy / free +blocks + +Disk 3 +Zones +Busy / free +blocks + + +Paxos-KV data groups +Durable allocation records and ownership · each disk group has an explicit data-group binding +Working bitmaps are rebuilt from records. Payload transfers use the separate DiskIO path. +CROWDB + diff --git a/doc/assets/disk-io.svg b/doc/assets/disk-io.svg new file mode 100644 index 000000000..46076f76e --- /dev/null +++ b/doc/assets/disk-io.svg @@ -0,0 +1,74 @@ + + +Disk I/O over RDMA or TCPTransport buffers stay owned through network, device I/O, and response completion. + + + + + + + + + + +Disk I/O over RDMA or TCP +Transport buffers stay owned through network, device I/O, and response completion. + +Client + +Chunk client +Payload operations + +DiskIO client +Route Segment to its storage node + +RPC request +Read / write / fsync +Owned payload buffers + + + +RDMA +Registered memory +SEND / RECV + completion queues + +TCP +Framed control + payload + + + +Storage node + +crowdb-rpc → DiskIO +Receive request and retain its buffer +Validate Segment, range, and payload length + + + +Retained I/O buffers +Eligible RDMA paths reuse registered receive storage +Alignment edges use explicit staging buffers + + +I/O engine +io_uring or platform fallback + + +Local disk 1 + + +Local disk 2 + + +Local disk 3 + +Same DiskIO operations +on either transport + +Read payload / write or fsync result → selected transport → caller +Synchronized writes respond after the write and disk flush complete. +Read buffers remain retained through response SEND and caller release. + + +CROWDB + diff --git a/doc/assets/iceberg-commit.svg b/doc/assets/iceberg-commit.svg new file mode 100644 index 000000000..93c57cf58 --- /dev/null +++ b/doc/assets/iceberg-commit.svg @@ -0,0 +1,54 @@ + + +Iceberg write: native files to an atomic table commitThe native upload path returns a storage-bearing URI before the engine builds its manifests. + + + + + + + + + + +Iceberg write: native files to an atomic table commit +The native upload path returns a storage-bearing URI before the engine builds its manifests. + +1. Native Parquet upload +Engine / loader → Iceberg access +Bounded streaming + checksum verification + +2. Durable Chunk storage +Chunk client → DiskIO → disks +Complete file bytes and an owner-bound root + +3. Publish immutable file +Sign the root descriptor into its URI +Create the file record in Chunk-KV + + + + +Returned URI carries the Chunk location +s3://iceberg/<table>/f/v1/<name>.<locator>.parquet +Signed locator: file identity + length + integrity + Chunk directory root + + +4. Build standard manifests +Use the returned URI as file_path +Store manifests and the manifest list + +5. REST catalog commit +Validate requirements against table state +Prepare immutable metadata JSON + +6. Publish the table head +One compare-and-swap in Chunk-KV +The new snapshot becomes visible + + + +Iceberg files live in CROWDB Chunk storage +S3-compatible FileIO is a protocol surface; this path needs no separate S3 storage backend. +CROWDB + diff --git a/doc/assets/iceberg-read.svg b/doc/assets/iceberg-read.svg new file mode 100644 index 000000000..7de358660 --- /dev/null +++ b/doc/assets/iceberg-read.svg @@ -0,0 +1,55 @@ + + +Iceberg read: the URI already carries the storage locationNative direct URIs remove the Iceberg file-record lookup from each data-file read. + + + + + + + + + + +Iceberg read: the URI already carries the storage location +Native direct URIs remove the Iceberg file-record lookup from each data-file read. + +Load table + select snapshot +REST catalog and canonical table metadata + +Read manifests +Manifest list → manifests → selected files + +Select a Parquet file URI +The same URI published by native upload + + + + +URI → authenticated native descriptor +Chunk ID + root offset + lengths + integrity + file identity +A stable storage reference, independent of current replica addresses + + +Direct file read · no Iceberg file metadata lookup + +Validate grant + locator +Check scope and signature +Reconstruct the descriptor from the URI + +Read the verified Chunk root +Follow only the requested file range +Bounded directory traversal + +Chunk client → DiskIO +Resolve mirror / EC layout +Read the selected bytes from disks + + + + +Stream verified bytes to the analytical engine +GET / Range GET · bounded buffers · consumer backpressure +Catalog planning and Chunk layout routing remain; the per-file Iceberg KV lookup is removed. +CROWDB + diff --git a/doc/assets/paxos-kv.svg b/doc/assets/paxos-kv.svg new file mode 100644 index 000000000..84be71224 --- /dev/null +++ b/doc/assets/paxos-kv.svg @@ -0,0 +1,136 @@ + + +Paxos-KV: store, groups, and replicasA store contains multiple groups; each group replicates its own state across nodes. + + + + + + + + + + +Paxos-KV: store, groups, and replicas +A store contains multiple groups; each group replicates its own state across nodes. + +Store S + +Group 0 +System metadata +Paxos replication within this group + +Replica · Node 1 · leader + +WAL + +crowdb-tree + +Local disk + + + +Replica · Node 2 + +WAL + +crowdb-tree + +Local disk + + + +Replica · Node 3 + +WAL + +crowdb-tree + +Local disk + + + + + + + +Group 1 +KV data +Paxos replication within this group + +Replica · Node 1 + +WAL + +crowdb-tree + +Local disk + + + +Replica · Node 2 · leader + +WAL + +crowdb-tree + +Local disk + + + +Replica · Node 3 + +WAL + +crowdb-tree + +Local disk + + + + + + + +Group X +KV data +Paxos replication within this group + +Replica · Node 1 + +WAL + +crowdb-tree + +Local disk + + + +Replica · Node 2 + +WAL + +crowdb-tree + +Local disk + + + +Replica · Node 3 · leader + +WAL + +crowdb-tree + +Local disk + + + + + + + +Client routing: store → group → replica +Groups have independent consensus, WAL, and Tree state. Their leaders can be on different nodes. +CROWDB + diff --git a/doc/assets/s3-publication.svg b/doc/assets/s3-publication.svg new file mode 100644 index 000000000..c9e9517d8 --- /dev/null +++ b/doc/assets/s3-publication.svg @@ -0,0 +1,62 @@ + + +S3 I/O: bucket and key to Chunk storagePUT stores bytes before publishing the object; GET uses object metadata to locate those bytes. + + + + + + + + + + +S3 I/O: bucket and key to Chunk storage +PUT stores bytes before publishing the object; GET uses object metadata to locate those bytes. + +PUT / upload + +S3 client → S3 access +PUT bucket / key +Validate request and body + + +Chunk client → DiskIO +Stream to shared Chunk storage +Large or small writer + + +Publish object metadata +Chunk-KV: bucket + key +Locations, length, ETag, attributes + + +Return success +Object is visible by key +Payload completion precedes publication +Multipart uses the same storage path; Complete publishes the selected parts as one object. + +GET / Range GET + +S3 client → S3 access +GET bucket / key +Optional byte range + + +Read object metadata +Chunk-KV: resolve bucket + key +Select locations and requested range + + +Chunk client → DiskIO +Read mirror / EC layout +Verify the requested bytes + + +Stream the response +Return bytes as the client consumes +Bounded buffers and backpressure +HEAD and listing use metadata; reading an object payload follows the Chunk storage path. +The S3 access library connects directly to CROWDB metadata and payload services. +CROWDB + diff --git a/doc/assets/tree-flush.svg b/doc/assets/tree-flush.svg new file mode 100644 index 000000000..e28547970 --- /dev/null +++ b/doc/assets/tree-flush.svg @@ -0,0 +1,83 @@ + + +Tree flush: fork down the tree, join toward the rootIndependent subtrees advance concurrently; snapshot persistence follows the same dependencies. + + + + + + + + + + +Tree flush: fork down the tree, join toward the root +Independent subtrees advance concurrently; snapshot persistence follows the same dependencies. + +One round · fixed MemTables and version frontier +Select memory-only flush or flush with snapshot at the start. + +Root +Fork by index separators + + + +Index [a, m) +Fork child range jobs + + +Index [m, z) +Fork child range jobs +FORK + + +Leaf [a, f) +Receive this range’s inputs +Consolidate / split → final pages +Persistent: write final pages + + + +Leaf [f, m) +Receive this range’s inputs +Consolidate / split → final pages +Persistent: write final pages + + + +Leaf [m, t) +Receive this range’s inputs +Consolidate / split → final pages +Persistent: write final pages + + + +Leaf [t, z) +Receive this range’s inputs +Consolidate / split → final pages +Persistent: write final pages + +JOIN + +Join + consolidate index +Wait only for this parent’s children +Merge siblings and update separators + + +Join + consolidate index +Wait only for this parent’s children +Merge siblings and update separators + + +Join at root +Complete page, mapping, and coverage results + + + +flush +Publish the complete memory tree + +flush with snapshot +Changed mapping segments → barriers → durable root +CROWDB + diff --git a/doc/assets/tree.svg b/doc/assets/tree.svg new file mode 100644 index 000000000..e63849233 --- /dev/null +++ b/doc/assets/tree.svg @@ -0,0 +1,93 @@ + + +crowdb-tree: concurrent ingestion, ordered pages, two backendsCAS-based skiplist tables feed a Bw-tree with versioned page mapping and retained read views. + + + + + + + + + + +crowdb-tree: concurrent ingestion, ordered pages, two backends +CAS-based skiplist tables feed a Bw-tree with versioned page mapping and retained read views. + +Versioned put / delete · point reads · ordered scans +Consensus slots preserve logical order while independent updates complete concurrently. + +Concurrent skiplist tables · L0 + +a + + +g + +a + + +e + + +g + +a + + +c + + +e + + +g +CAS publishes nodes and immutable version sets + +Bw-tree · immutable pages and overlays + +Mapping table · PID → page version +A retained mapping keeps every page lookup consistent + +Root / index +Ordered separators + + + +Leaf [a, m) +Retained base + ordered overlays + + +Leaf [m, z) +Retained base + ordered overlays + +flush + +Readers retain their view +Consistent sources + mapping + page owners +New page publication preserves existing readers + + +Two outputs from one flush design + +flush +Publish memory pages and mapping + +flush with snapshot +Persist final pages, changed mapping, and durable root + +PageStore +Persistent pages, mapping segments, and recoverable roots + + + +Local disk backend +Pages on local disks +Paxos-KV replicas + + +Chunk backend +Pages in shared Chunks +Chunk-KV partitions +CROWDB + diff --git a/doc/design/README.md b/doc/design/README.md new file mode 100644 index 000000000..3b181089d --- /dev/null +++ b/doc/design/README.md @@ -0,0 +1,59 @@ + + + +# CROWDB Design + +Start with the product design, then use Scope to choose an architecture or component article. These English overviews explain the design; component READMEs provide usage and operational instructions. + +## Product and architecture + +| File name | Scope | +| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| [design-crowdb.md](design-crowdb.md) | Workloads, shared storage, independent scaling, coordinated recovery, and design advantages. | +| [design-crowdb-architecture.md](design-crowdb-architecture.md) | Component roles, logical/physical placement, serving capacity, recovery, and resource control. | + +## Components + +| File name | Scope | +| ------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | +| [design-crowdb-access-iceberg.md](design-crowdb-access-iceberg.md) | Native writes, direct-URI reads, table commits, and integrated file lifecycle. | +| [design-crowdb-access-dataset.md](design-crowdb-access-dataset.md) | Sample-oriented storage, small payloads, Snapshot/Split selection, and direct training I/O. | +| [design-crowdb-access-server.md](design-crowdb-access-server.md) | Host, S3 buckets and objects, streaming, multipart, and authentication. | +| [design-crowdb-sdk-native.md](design-crowdb-sdk-native.md) | Direct RDMA, gateway bypass, caller-held capacity, retained buffers, and GPU integration direction. | +| [design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md) | Concurrent slot writes, per-key apply ordering, stores/groups/replicas, and durable metadata. | +| [design-crowdb-chunk-kv.md](design-crowdb-chunk-kv.md) | Ordered metadata, independent payload placement, smooth split/transfer, and serving capacity. | +| [design-crowdb-tree.md](design-crowdb-tree.md) | Concurrent skiplists, Bw-tree mapping, fork/join flush, external-WAL recovery, and dual backends. | +| [design-crowdb-chunk-stream.md](design-crowdb-chunk-stream.md) | Reusable durable byte streams, journal composition, rollover/recovery/trim, and future interfaces. | +| [design-crowdb-chunkdb.md](design-crowdb-chunkdb.md) | Chunk placement, large/small write pipelines, EC, read flow, and repair. | +| [design-crowdb-diskdb.md](design-crowdb-diskdb.md) | Disk/zone allocation, durable records, capacity accounting, and recovery. | +| [design-crowdb-diskio.md](design-crowdb-diskio.md) | Addressed byte I/O, asynchronous backends, completion, and buffer lifetime. | +| [design-crowdb-rpc.md](design-crowdb-rpc.md) | RDMA data paths, buffer ownership, resource backpressure, asynchronous completion, and TCP fallback. | +| [design-crowdb-monitor.md](design-crowdb-monitor.md) | System metadata in Group 0, node discovery, service supervision, and console operations. | +| [design-crowdb-deployment.md](design-crowdb-deployment.md) | OCI packaging, runtime independence, host networking/devices, and CPU/memory limits. | + +## Usage and validation + +| File name | Scope | +| -------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| [sdk/README.md](../../sdk/README.md) | Portable Dataset HTTP SDK for Rust/Python, examples, packaging, and limits. | +| [lib/crowdb-sdk-native/README.md](../../lib/crowdb-sdk-native/README.md) | Native Rust API usage, memory budgets, transport options, and shutdown. | +| [lib/crowdb-sdk-native-python/README.md](../../lib/crowdb-sdk-native-python/README.md) | Trusted native Python packaging, buffers, and application lifetime. | +| [container/single-node-container/README.md](../../container/single-node-container/README.md) | Local node storage, listener setup, and client examples. | +| [container/oci-image/README.md](../../container/oci-image/README.md) | OCI build, launch, host deployment, persistent data, and lifecycle. | +| [app/crowdb-access-server/tests/README.md](../../app/crowdb-access-server/tests/README.md) | Access acceptance evidence and client/engine validation boundaries. | +| [e2e-test/README.md](../../e2e-test/README.md) | Container suites, deployment/restart checks, service ownership, and artifacts. | +| [e2e-test/clients/ecosystem/README.md](../../e2e-test/clients/ecosystem/README.md) | Pinned external client/engine matrix and manual acceptance. | +| [lib/crowdb-tree/bench/README.md](../../lib/crowdb-tree/bench/README.md) | Targeted Tree benchmark conditions, reproduction, and interpretation. | +| [tools/README.md](../../tools/README.md) | Tool directories, Pixi task entry points, and validation commands. | + +## Project files + +| File name | Scope | +| ---------------------------------------------- | ---------------------------------------------------------- | +| [README.md](../../README.md) | Product introduction and deployment options. | +| [CONTRIBUTING.md](../../CONTRIBUTING.md) | Environment, development workflow, and contribution rules. | +| [AGENTS.md](../../AGENTS.md) | Repository rules for coding and documentation work. | +| [VERSION](../../VERSION) | Canonical development version. | +| [CHANGELOG.md](../../CHANGELOG.md) | Release history. | +| [SECURITY.md](../../SECURITY.md) | Vulnerability reporting. | +| [CODE_OF_CONDUCT.md](../../CODE_OF_CONDUCT.md) | Community conduct. | diff --git a/doc/design/access-server/dataset/design-crowdb-access-dataset.md b/doc/design/access-server/dataset/design-crowdb-access-dataset.md deleted file mode 100644 index 0a7da892d..000000000 --- a/doc/design/access-server/dataset/design-crowdb-access-dataset.md +++ /dev/null @@ -1,589 +0,0 @@ - - - -# CROWDB - Design: Dataset - -Dataset is CROWDB's sample-oriented data domain for AI, analytics, and data -preparation. It owns dataset metadata, immutable snapshots, sample selection, -field projection, bounded reads, and payload lifetime while using CROWDB KV and -chunks for durable state and bytes. - -The normative object and behavior contract is [CROWDB Dataset Specification](spec-crowdb-dataset.md). This design document describes the implementation architecture that realizes that contract; it does not redefine Dataset semantics. - -This design is independent of S3 and Iceberg. A Dataset may record an external -source reference, but that reference does not replace Dataset publication, -selection, or reclamation authority. - -## Table of contents - -1. [Scope and boundary](#1-scope-and-boundary) -2. [Authority and naming](#2-authority-and-naming) -3. [Dataset and snapshot model](#3-dataset-and-snapshot-model) -4. [Manifest, schema, samples, and fields](#4-manifest-schema-samples-and-fields) -5. [Selection and read plans](#5-selection-and-read-plans) -6. [Access surfaces and bounded delivery](#6-access-surfaces-and-bounded-delivery) -7. [Progress and lifecycle](#7-progress-and-lifecycle) -8. [Interactions](#8-interactions) -9. [Correctness invariants](#9-correctness-invariants) -10. [Implementation and I/O flows](#10-implementation-and-io-flows) -11. [Risks and open questions](#11-risks-and-open-questions) -12. [Empty Dataset retirement](#12-empty-dataset-retirement) - -## 1. Scope and boundary - -Dataset provides a stable logical model for samples and heterogeneous fields. It -supports HTTP access through an independently packaged Rust SDK and Python -wrapper, plus trusted direct storage access through the shared native runtime -and its optional Python extension. Both surfaces preserve the same selection -and integrity semantics. It does not own disk placement, replication, -erasure coding, repair, or physical chunk layout. It does not become a query -engine or a training-framework-specific storage protocol. - -The core objects are Dataset, Snapshot, Manifest, Schema, Sample, Field, -Split revision, Selection, Read Plan, and Cursor. Worker scheduling, tensor decoding, collation, -and model checkpoints remain in adapters or execution systems above Dataset. - -## 2. Authority and naming - -Dataset authority always supplies the fixed `dataset-ns` default/root namespace. Every Dataset -identity carries that namespace, even though the first implementation exposes no -namespace create, delete, or mutation operation. Dataset names are unique within -the supplied namespace. The representation keeps namespace in the key so a -future multi-namespace authority can be added without changing Dataset identity, -snapshot inheritance, chunk placement, or read ordering. - -## 3. Dataset and snapshot model - -A Dataset owns metadata, snapshots, manifests, samples, fields, and the payload -references needed to read them. A Snapshot is an immutable published view -identified by an opaque `snapshot_id`. `latest` and `stable` are mutable names -that resolve to real snapshot IDs; cursors and read plans store the real ID. - -A Snapshot has at most one `parent_snapshot_id`. A child applies additions, -deletions, and membership rules to its parent; the numeric order of IDs never -implies ancestry. Any committed same-Dataset Snapshot can be the parent; multiple -parentless roots form a forest. Multi-parent merging is not supported. Old -Snapshots remain readable until reclamation has passed their protection checks. - -Publication builds and validates the manifest, schema binding, indexes, payload -references, checksums, and durability state before atomically advancing a -mutable name. Readers observe either the old complete snapshot or the new -complete snapshot. - -## 4. Manifest, schema, samples, and fields - -The Manifest is the authority for a Snapshot's schema binding, sample membership, -field locators, checksums and parent delta. Split sets are separate immutable -revisions owned by that Snapshot. The Manifest -may be partitioned and indexed; a single hot key cannot be required for all -reads. The current Schema records a version and field names/requiredness; it -does not declare types, MIME, nested structures or semantic media metadata. - -`sample_id` is a stable opaque byte sequence supplied by the importer, represented -as canonical padded base64 in JSON. Within one -snapshot, `(sample_id, field)` has one logical value. A field is either an inline -KV value for bounded metadata or an opaque chunk location string for larger -payloads. Dataset passes the location to the chunk system and never parses or -constructs its physical meaning. Copy-on-write shares unchanged values between -snapshots; field and sample deletion uses tombstones so historical snapshots -remain valid. - -A sample may contain image, video, audio, text, label, embedding, or metadata -fields. Fields are aligned by snapshot and sample identity, while decoding and -tensor composition belong to the consumer adapter. - -## 5. Selection and read plans - -A Split is a named logical subset of a Snapshot. One immutable Split revision -contains the complete set of names, definitions, membership and overlap/coverage -policy. A Snapshot's currentSplit points to one revision or is absent. Publishing -a revision changes only this pointer, never Dataset latest or the data Snapshot. -A Selection in a Read Plan chooses -samples using prefix, range, explicit IDs, keyword, field match, and boolean -composition. Prefix, range, and explicit IDs can be checked for overlap directly; -metadata predicates require bounded work against one fixed Snapshot. Defaults -are disjoint membership and allowed unassigned Samples. A committed empty set -differs from no Split revision. All committed revisions live until owner reclaim. - -The [control lifecycle](design-dataset-control.md) specifies durable composition, -frozen Split inheritance, publication attempts, bounded inspection and cleanup. - -A Read Plan binds one snapshot ID to selection, ordering, optional shuffle -parameters, and batch policy. The implementation supports bounded sample and -group shuffle. -Sample shuffle loads only sample IDs and lightweight metadata into bounded groups; -a selection that fits one group completes in one pass, while a larger selection -uses multiple groups with the same deterministic contract. Group shuffle remains -an explicit plan type for prefix, metadata, session, device, time-window, or -association-key grouping. Every shuffle has sample-count, retained-byte, batch, -prefetch, and in-flight limits and is rejected before payload reads when a limit -would be exceeded. Shuffle never moves or rewrites published payload. - -## 6. Access surfaces and bounded delivery - -HTTP is the portable Dataset surface used by the public Rust SDK and its -Python wrapper. The trusted native facade reuses in-process Dataset authority -and reaches Chunk-KV and ChunkIO/DiskIO through the shared runtime. Its bounded -versioned routing observations do not replace storage authority. See the -[native runtime contract](../design-crowdb-native-sdk.md) and -[native SDK guide](../../../../lib/crowdb-sdk-native/README.md). - -`GetSample`, `GetBatch`, and bounded Scan resolve metadata first, apply field -projection, and then read inline values or chunk ranges/streams. Batch size, -read window, prefetch, cache, and in-flight requests are bounded and propagate -backpressure. Direct GPU delivery may use short-lived capabilities and a CPU -fallback, but it must preserve the same selection and integrity result. - -## 7. Progress and lifecycle - -A cursor is valid only for its bound Read Plan and snapshot. Ordered reads store a -sample position; grouped shuffle stores group identity or index and an offset -within the group. The client advances a cursor after batch confirmation. A crash -may replay the last unconfirmed batch, but may not skip or change the snapshot. -Epoch and sampler state remain client state. - -Read admission checks the selected snapshot's authoritative deletion state without -registering a reader. Deletion durably fences new reads and records a configured grace-period -`reclaim_after` deadline (twenty minutes by default); retries and restarts preserve that deadline. Existing -streams can finish using resolved metadata and payloads during the grace period. -Unusually long reads may fail after reclamation; partial delivery is never complete -success. Access Servers and native clients do not acknowledge deletion. Client -checkpoints and local request/buffer admission have independent lifetimes. EOF, -cancel and drop release local request admission without remote cleanup writes; -caller-held buffers remain valid and charged until their final release. - -Dataset owns the lifetime of its metadata and payload references. Reclamation -preserves explicit retention, live snapshot ancestry and shared payload owners. -External S3 or Iceberg references are provenance unless an explicit cross-model -retention contract says otherwise. Physical I/O identity checks and buffer ownership -remain necessary when storage is reclaimed or reused. - -## 8. Interactions - -Dataset metadata uses KV authority and can use chunk-backed range state for -large manifests or indexes. Chunk I/O owns location validation, range/stream -reads, checksums, and physical recovery. Access Server exposes HTTP and health -boundaries; a direct client observes topology but never publishes placement. -Framework adapters map Read Plans to PyTorch `IterableDataset`, TensorFlow/JAX -iterators, Hugging Face streaming examples, or Ray blocks without adding storage -objects. - -S3 and Iceberg remain peer metadata domains. Dataset can consume immutable data -from either through an explicit source reference, but cannot commit their tables, -overwrite their objects, or reclaim their owned bytes. - -The Dataset library owns these semantics and storage clients. The existing Access -Server may expose Dataset HTTP on its own configured listener, alongside S3 and -Iceberg. The Dataset SDK has an independent Rust workspace and release version -in the CrowDB repository. Its HTTP client uses reqwest, Tokio and rustls; the -Python package calls that implementation through PyO3. Clients share pooled -connections, atomic request/byte admission and process-owned runtime resources. -Blocking native waits release the GIL; live handles inherited across fork are -rejected. File transfers and batched iteration bound memory independently of -file size. The optional native Python extension calls the shared Rust native -facade, retaining the same bounded ownership and runtime shutdown contracts. -Direct metadata and chunk access use the runtime's configured TCP/RDMA policy -without implicit HTTP fallback. Tensor decoding and collation stay above both -SDK surfaces; optional local staging remains a separate extension. - -## 9. Correctness invariants - -- **DATA-I1 — Namespace identity:** every Dataset carries the configured - default/root namespace, and a Dataset name is unique within it. -- **DATA-I2 — Snapshot consistency:** one operation resolves exactly one immutable - snapshot and never mixes fields from another snapshot. -- **DATA-I3 — Atomic publication:** readers observe no manifest or one complete - published snapshot. -- **DATA-I4 — Explicit ancestry:** only the selected snapshot's parent chain - explains inherited state; ID ordering is irrelevant. -- **DATA-I5 — Opaque locations:** Dataset does not interpret physical chunk - locations. -- **DATA-I6 — Logical shuffle:** ordering and shuffle do not move published - payload. -- **DATA-I7 — Bounded delivery:** batch, window, prefetch, cache, and in-flight - limits bound memory and registration. -- **DATA-I8 — Cursor binding:** a cursor and Read Plan cannot resume against a - different snapshot or selection identity. -- **DATA-I9 — Safe reclamation:** retention and live ownership protect reachable - metadata and payload. Deletion rejects new reads and delays reclamation for the - grace period; overdue reads may fail without returning truncated success. -- **DATA-I10 — Surface equivalence:** HTTP and native access return the same - logical selection and integrity result. - -## 10. Implementation and I/O flows - -The implementation is split into a `crowdb-access-dataset` library and a thin -`crowdb-access-server` Dataset listener, following the S3 and Iceberg boundary. -`DatasetAuthority` owns keys, records, publication state, ancestry, retention, -through the `DatasetStore` interface; read checkpoints stay with the caller. `DatasetReadService` -owns admission, bounded delivery, plan/cursor execution, and request lifetime. -`DatasetHttpService` owns HTTP framing, authentication, route/status mapping, -shutdown, and health wiring. `ChunkKvDatasetStore` adapts Dataset metadata and -payload access to the chunk-backed KV client without exposing physical layout -to the authority. - -### 10.1 Upload and publication flow - -`DatasetUploads` persists bounded upload resources, immutable numbered part -receipts and Dataset-scoped ownership in `DatasetStore`. -`ChunkPayloadStorage` writes parts through the shared small-object pipeline with -Dataset chunk type 7. Each part is at most 4 MiB; 10000 parts bound an upload to -39.0625 GiB. A write intent records the exact physical location before payload I/O. -Successful durable completion accepts the receipt. Recovery accepts an intent -only after the chunk's acknowledged cursor covers the location and the bytes -pass integrity checks. Readable but unconfirmed bytes remain pending. A fenced -abandoned allocation is reclaimed before replacing it under the same part ID. -Upload resources hold at most 256 active and 4096 retired owner identities. A full -retirement list closes admission to new owners while existing owner retries remain -valid. Retiring an owner not recorded in a full list upgrades the same resource to -version 5 before success, even if that owner has not yet claimed it; an active owner -is removed in that CAS. This prevents an older writer paused before its claim from -reintroducing the owner after cleanup. Readers supporting only versions 2 through 4 -reject this final fence. Older Access servers and native SDKs must upgrade before -reading or writing that saturated resource. New mutators retain version 5 and -require its retirement list to remain full. -Ordinary resources use version 4. The first upgrade costs one GET and one CAS; -repeating retirement of an absent owner in version 5 needs one GET and no write. -Upload storage uses an independent binary envelope: a 52-byte fixed header -followed by sixteen bytes per active or retired owner. Counts and exact encoded -length are checked before allocating identity vectors; the maximum authority -contains 4352 identities in 69,684 bytes. Public `UploadRecord` JSON and its -logical versions remain unchanged. Part receipts retain their JSON encoding. -Readers accept legacy JSON versions 2 through 5 with the same validation and -identity representation; writers emit the binary envelope. Conditions compare -the actual stored bytes, so a paused JSON mutation cannot overwrite a converted -resource. JSON-only Access servers and direct native clients reject this -envelope; their storage readers and writers require a coordinated upgrade. -Completion verifies every part and the whole declared MD5 with bounded buffers. -For one part whose declared length and MD5 match the whole upload, verification -hashes the stored bytes once and reuses that exact-range result. Multiple part -digests cannot be concatenated into the whole-upload MD5. - -Expiry scans bounded pages of upload authorities directly. Each caller retains -its scan cursor locally; restart repeats the scan and loses no cleanup obligation. -Discovery reads at most twenty keys / 1 MiB and seeks past each upload's child -subtree. A bounded local queue schedules uploads in turn; each cleanup window -scans at most 32 actual part records / 1 MiB and attempts at most 64 range -notifications. Discovery and cleanup each have a two-second deadline. Sparse -uploads do not perform a point GET for every declared but absent part. -Each cleanup authority point read has a fixed 352 KiB budget. Canonical JSON needs -at most 66 bytes per owner including a separator, plus 1 KiB for other fields; -4352 owner identities and a metadata key shorter than 64 KiB fit this bound. -Noncanonical or otherwise oversized records still return a visible limit error. -Accepted range retirement permits deletion of that aborted upload's part record; -the upload cannot accept a new part or owner. Deferred or uncertain notification -retains the exact range. A full queue can discard a hint only after serving it, -leaving a discovery slot; removed part records ensure useful progress survives -hint eviction and restart. There is no persistent scan or queue record. -Independent upload creation writes only its own authority. No shared registry tail, -registration generation, registry fence or durable scan cursor participates in -creation, part upload or owner release. Corrupt/failed entries are reported and -retried next cycle without starving subsequent records. Legacy registry pages are -ignored. Claimed payloads and terminal owner revocation remain authoritative; -removing an owner cannot release another owner's bytes. Immutable Accepted part -records are the sole part-listing facts. Listing and completion scan at most -128 records / 1 MiB per page, with sixteen concurrent disjoint prefixes of 256 -part IDs. A part write updates only its own record; it does not rewrite a shared -index page. Legacy index records are ignored. - -Prepare validates the complete current sample membership, schema, parent and -operation token, then persists an invisible candidate and claims completed -uploads. New caller tokens directly select the immutable Snapshot ID; they do -not create a separate operation-to-Snapshot mapping. Existing legacy mappings -remain readable and select their original Snapshot. Retirement retains a small -negative fence for legacy delayed writers and the selected Snapshot resource. -Omitted samples disappear; surviving samples inherit omitted optional -fields through explicit ancestry. Tombstones remove inherited values. Required -field names cannot be removed from a child schema. - -Publish reuses the prepared Snapshot ID, marks its complete record Published, -and CAS-advances the head. A Published Snapshot record can precede head -advancement; a successful publish response establishes the completed operation. -Lost responses are resolved by inspecting status and retrying identical content -with the same operation token. Historical retries cannot rewind a newer head. -Divergent retries conflict. The first Snapshot resource write includes a fixed -internal `gc_after` deadline twenty-four hours after creation; content bindings -inherit it. Retries and publication-state changes preserve that deadline. The -background worker scans Snapshot resources directly, including writes interrupted -before the content binding exists. Each page has at most twenty rows, nine MiB -and a two-second deadline; its position is discardable and restarts from the -beginning. Head, retention and ancestry checks protect referenced resources -without writes. Eligible resources use the ordinary authoritative reclamation -fence and configured deletion grace. Eligibility alone does not revoke reads. -An old resource inherits its existing content-binding deadline. If neither -record has one, the worker assigns a fixed deadline twenty-four hours after its -first observation. Legacy opaque resources without a supported Manifest remain -excluded from automatic retirement because their payload references are unknown. -No per-resource discovery task is created. -Upload expiry does not reclaim claimed candidate payloads. - -### 10.2 Read flow - -`DatasetReads` resolves latest/stable and optional currentSplit into immutable -versions and returns a self-contained continuation. The server stores no read -record, selected-ID pages, progress, TTL or cancellation state. Every next request -supplies its continuation and passes ordinary authentication and deletion admission. -The token binds Dataset, Snapshot, Split, selection, projection, worker partition, -shuffle seed/group size and position. It is a cursor, not an authorization grant. - -Selection uses bounded inspection windows without fetching a full Manifest. -Within one next request, selection windows share admission, immutable summaries, -schema and compiled predicates. Explicit IDs are validated once. This prepared -query is local to the request and binds its authority, Dataset and Snapshot; -independent page requests still perform their own admission. Per-window storage -budgets and the overall group deadline remain enforced. -Split filtering precedes deterministic shuffle and worker partitioning. The cursor -carries a group start and offset, not the Dataset's selected IDs. A partly consumed -shuffle group is reconstructed from immutable metadata; this bounds memory but -adds metadata reads when groups are much larger than delivered batches. Groups -are capped at 100000 selected IDs and 8 MiB of IDs; selection work has a two-second -deadline and at most 5000 scan windows of up to 20 candidates. Serialized cursor -content is capped at 256 KiB before base64 encoding. Invalid or excessive work -returns an explicit error. Stream metadata and headers remain capped at 8 MiB. - -Repeating a continuation replays its batch, including from another Access Server -or native client. After successful application processing, the caller saves the -returned next continuation in its own checkpoint. EOF and buffer release never -save progress. There is no remote ack, resume or close; cancelling a request stops -its local producer. The SDK iterable's acknowledgement is a local operation. - -The public binary response streams a bounded JSON header, integrity-checked -payload frames, field terminators and a batch terminator. Success also requires -HTTP EOF with no trailing bytes. Each active transfer owns local admission -credits through delivery/drop. Snapshot reads perform no persistent activity -updates or heartbeat writes. Cancellation stops the producer and returns credits; -retained SDK-owned buffers keep their byte credits until released. -For an uploaded field, HTTP and native readers validate one immutable upload -description per request, then fetch each part record once. Part indices, lengths -and payload digests remain checked; EOF does not reload metadata. The description -is request-owned and registers no remote reader or retention. New requests still -perform admission, and reclaimed parts fail instead of producing successful EOF. -Native delivery reuses a single part's successful-EOF length and MD5 proof only -when the part is the complete field range and matches the field's expected digest. -Partial data and failed streams provide no proof. Multiple parts still require -the independent whole-field digest, and inline fields remain checked directly. -The internal legacy scan adapter also returns a caller-owned cursor without -loading or persisting progress. Requests have no shared inactivity TTL. -Legacy scan planning retains one flat ordered ID vector, extracts only the -current window by moving its IDs, and releases the remaining vector before -payload delivery. Batch adapters move IDs into windows. Selection and budget -accounting share one pass; sample shuffle computes its plan identity once and -orders by the existing MD5 seed/epoch/plan/ID key with binary-ID tie breaking. -The legacy path still scans its full selection; flat planning does not provide -the incremental metadata behavior of inspection-backed reads. - -Binding reuses its committed-Snapshot admission. Independent selection, preview -and batch calls retain their own admission. Combined continuation selection and -delivery share one request-local Snapshot/schema/membership proof under separate -work budgets; wrappers do not repeat the same deletion lookup. Empty and legacy -batch entry points retain independent admission. No proof is cached across requests. - -Legacy Snapshots without a stored visibility proof are checked against head and -anchor ancestry using bounded, cancellable reads. The traversal validates the -Snapshot identity and publication state, rejects cycles and never fills a -persistent proof cache. Existing proofs remain readable; modern unpublished -candidates cannot use legacy ancestry as evidence of publication. - -### 10.3 Retention and reclaim flow - -Reclaim atomically fences new reads, invalidates stale head publishers with a -head nonce, and rechecks latest, stable, explicit retention and all live committed -child branches. Temporary child admission fences do not prove retirement. A -reclaimer owns a CAS token; only that owner can reopen admission or enter the -irreversible retiring phase. Concurrent stale reclaimers cannot undo retirement. -After the terminal deletion mark, cleanup waits until the persisted grace -deadline. The Access Server scans bounded pages of deletion records and resumes -cleanup independently of reader processes. An Aborted Snapshot tombstone preserves -the original Manifest while payload cleanup is pending and prevents publication retry resurrection. -REST and bincode deletion requests only establish this mark and observe durable -completion; repeated requests do not drive metadata deletion or accept a client -cleanup cursor. Background workers release upload ownership for that operation -only; other Snapshot owners preserve shared bytes. Owner, legacy Split-draft, -index-fence and metadata scan positions are local, bound to one Dataset/Snapshot. -Lost positions repeat idempotent retirement without a stored progress record. - -Background discovery reads at most twenty deletion marks / 512 KiB and keeps -at most 32 local cleanup hints. Each window has a two-second deadline. Pending -work runs at a 50 ms interval; failures back off to 500 ms and rotate the queue, -so one slow Snapshot cannot starve another. An empty backlog waits fifteen -seconds. Restart rescans durable marks. Metadata deletion starts only after the -owner cursor completes; clients cannot skip that boundary. Accepted range -notifications end Access responsibility without waiting for physical capacity -recovery. Shared range retirement does not count as a reclaimed whole chunk. - -Local queues reserve a full discovery page against failed work. Each hint tracks -whether its previous window failed. Successful incomplete windows keep their -positions. Failed hints may occupy at most `32 - discovery_page_items` slots; -overflow discards the oldest failed hint, leaving every durable resource fact -untouched. This opens discovery capacity even when 32 operations never respond. -An outer deadline marks a hint failed only while its window is active, not while -discovery or legacy-checkpoint sweeping runs. Dropped positions repeat idempotent -retirement. After discarding a hint, a completed discovery pass restarts when its -remaining hints have all failed, so no discarded resource is permanently skipped. Empty -passes retain the normal fifteen-second idle interval. No scheduler position or -failure bit is persisted. - -Metadata cleanup deletes at most twenty keys per local window. It removes -obsolete Manifest storage and inspection projections while retaining small -permanent CAS fences for all possible immutable index and Split revision/member -writes. A failed fence write does not advance the local row position. Commit -receipts, head fences and terminal retirement evidence remain. A retry helps the -final status marker if cleanup completed before the response or status write -failed. Old upload, metadata, Split-discovery and index checkpoint keys are swept -in pages of twenty only after authoritative Snapshot completion. The sweep reads -no checkpoint payload and never deletes the completion or immutable-write fences; -its scan position is also local. Empty legacy scans wait fifteen seconds. - -Legacy Composition cleanup scans existing draft facts into a separate local queue -of at most 32 operations, discovering eight drafts / 9 MiB at a time. Each -two-second window advances candidate fences, upload owner retirement, scratch -fences or registered edit fences. The queue rotates failures and slow operations; -restart rediscovers every obligation from the retained draft, work-page and edit -registration facts. No candidate, scratch, edit or upload-retirement checkpoint is -written. Complete result owners remain independent of draft abandonment. Old -checkpoint keys are deleted after the local pass; immutable discovery facts and -negative fences remain. The retired HTTP Composition cleanup action returns 501 -before storage, and no client supplies or confirms background cleanup progress. - -Composition evaluation work pages record their possible row writes before applying -those writes. Obsolete-window cleanup seals rows before sealing their work page, -then seals the first absent work page to exclude delayed next-window creation. -Split evaluation similarly registers immutable page keys/digests before writing -members. Split cleanup processes at most twenty page references per local window. A -separate background queue discovers eight historical drafts / 9 MiB at a time and -keeps at most 32 hints. Snapshot and Composition cleanup embed the same local -Split cursor; no Split checkpoint or work-format migration is written. Restart -repeats idempotent fences. Complete final roots preserve their member pages; an -absent root competes with cleanup on that same key before member retirement. -Readable Snapshot owners wait for the persisted deletion grace. A permanently -sealed incomplete root can retire without creating or waiting for a reader -deadline. Grace waiting and initial work still protected by a live owner leave -the current queue pass and are rescanned after fifteen seconds. The retired -HTTP Split cleanup action returns 501 before storage. Work-intent facts and -negative fences remain; old drafts without that format are retired with their -Snapshot metadata instead of receiving a process-format upgrade. Terminal Snapshot cleanup walks Dataset draft -roots in resumable single-root windows, including drafts without a published -revision summary. Under that terminal fence it can upgrade an older root's work -marker before retiring its stored pages. Split revision summaries remain valid -small records and are filtered by owner/commit visibility. - -The composed reclaim operation advances upload owners, Split drafts, index fences -and ordinary metadata cleanup with a ten-second deadline. Each component retains -its own restart cursor; completion does not claim physical upload deletion. - -Obsolete composition candidates are discovered by deterministic operation IDs, -even when preparation failed before storing a pointer in the draft. Missing -operation/Snapshot reservations become permanent fences. Existing uncommitted -candidates enter terminal retirement before cleanup enumerates every possible -Manifest/index key from the retained input. This seals writes not yet sent by a -failed preparer. Small valid retired inventory/child summaries and write fences -survive general cleanup. Candidate recovery caps its original record at 9 MiB -and metadata window at 9 MiB with a two-second operation deadline. - -Managed composition uploads register staging and candidate owners durably before -claiming payloads. Publication requires complete candidate ownership. Terminal -cleanup retires the staging owner and obsolete candidate owners, preserving -published candidates and independent owners of shared uploads. Snapshot upload -reclaim walks at most twenty retained Manifest field positions per step after -terminal admission fencing. It revokes only that Snapshot publication's owner; -physical cleanup stays discoverable from durable upload authorities. The retained Manifest -allows owner cleanup even after inspection indexes have been retired. - -Composition version 3 registers each edit ID in the root before writing its -payload page. Up to 4096 registered edits, including failed attempts, are allowed. -Terminal cleanup seals these pages in bounded windows and keeps immutable edit -receipts. Older composition writers reject version 3 rather than ignoring the -registration protocol. Existing records remain readable; editing upgrades the -root while preserving its journal and payload ownership. - -`retire_owner` permanently revokes one publication's upload claim with the same -session CAS that removes its ownership. A delayed claim for that operation then -fails; other owners remain valid. Resources retain at most 4096 retired owner IDs. -Below that limit, retirement records a negative fence even for an absent owner. -At the limit, new claims report capacity; retirement upgrades the resource to -version 5 before removing an unrecorded owner or acknowledging its absence. -The full list remains unchanged, so retired owners cannot rejoin and the final -active owner can still be removed before abort or expiry. Ordinary `release` -remains reversible below saturation; a released owner cannot rejoin a full resource. - -Upload expiry uses local continuation/queue hints and existing upload/part resource -facts. Shared registry registration and removal are absent. Snapshot deletion -sweeps also yield after a two-second page deadline, keeping durable deletion marks -and advancing past a stalled item. The listener schedules snapshot pages every -fifteen seconds and upload catch-up windows fifty milliseconds apart; failed -upload windows back off five hundred milliseconds while other work remains. -Idle discovery runs every fifteen seconds. Limits bound work per window rather -than promise a maximum reclamation latency or measured foreground p99. - -### 10.4 Transport and compatibility - -The public versioned HTTP surface uses JSON control records, binary payload -streams and bearer authentication. Legacy bincode routes remain an internal -compatibility interface. HTTP/1.1 keep-alive has separate bounded idle connection -and executing request admission, header deadlines and graceful shutdown. -Default executing admission is eight requests. Early authentication/capacity -rejection drains a bounded body without materializing it so valid part writers -receive the structured error. Oversized or stalled rejected bodies close their -connection after a bounded drain; they cannot consume unbounded resources. -Uploads and downloads stream with bounded concurrency and backpressure. The SDK -retries only documented idempotent requests, preserving replayable bytes and -operation IDs under one overall deadline. Authentication, integrity and semantic -conflicts are not blanket retried. File downloads use a private temporary file -and rename only after successful integrity verification. - -Head envelopes decode legacy versions and use version 3 for commit-attempt -markers. Access envelopes decode versions 1 through 4 and write version 4 with -reclaim ownership, terminal deletion, grace deadline, cleanup target and completion. -Legacy reader activity is ignored; all authority writers must use this contract. -Upload sessions decode versions 2 through 4 and write version 4 without registry -fields. Permanent retired-owner identities still reject delayed claims. This -metadata upgrade is one-way: do not downgrade an upgraded authority or mix old -and new authority writers against the same Dataset store. Immutable Dataset/Snapshot/Manifest encodings -remain compatible. Persisted read-session records and legacy cursor keys are -unused. Session read routes are absent; clients use `POST reads/bind` and -`POST reads/next`. Upgraded clients and servers must deploy together. Control v2 -advertises `stateless-reads` so an SDK cannot treat the retired session capability -as support for this contract. Physical purging of unused keys is an explicit -maintenance operation, not a read-path side effect. - -Hot-path synchronization uses atomics, immutable owned views, CAS and bounded -notification channels. No application-wide request lock serializes independent -operations. Producer admission includes queued native buffers and metadata -parsing, while caller-retained completed results belong to the application. -Functional resource checks verify these ownership rules. Benchmark drivers, -measurements and performance thresholds are outside this implementation. - -## 11. Risks and open questions - -Future namespaces add metadata fan-out and authorization scope. The first -implementation intentionally has no namespace operations; it only carries the -configured default/root namespace in every identity. - -Manifest partitioning, predicate indexes, cache invalidation, and direct GPU -transport require workload measurements. They are independent extensions and do -not alter the Dataset identity or snapshot contract. - -## 12. Empty Dataset retirement - -- Dataset root records carry a deletion receipt and bounded creation reservations. - Snapshot/upload creation reserves its child key through the root CAS before the - first child CAS. Deletion settles materialized reservations, checks all live - content, then compares the same root version. Concurrent new creation therefore - invalidates the emptiness decision. No per-Dataset lock is required. -- At most 32 outstanding creation reservations are admitted. An abandoned reservation - expires after twenty minutes, independently of deletion grace. Before removing an - expired reservation, the worker CAS-installs a permanent child tombstone if the - child is still absent. A late creator cannot revive an operation after name reuse. -- Marked roots are hidden from discovery and normal opens. The cleanup worker uses - bounded pages and an expendable cursor; roots own durable deadlines and progress. - Because acceptance requires no live content, due cleanup only retires the namespace - reservation. It preserves the minimal root receipt and existing child fences. -- `dataset.reclaim_grace_seconds` is shared by Dataset, Snapshot and Split revision - deletion, defaults to 1200, and is positive in Access Server configuration. The - worker polls Dataset/Snapshot marks at min(grace, 15) seconds. Configuration changes - affect future marks; persisted deadlines never move. Restarted workers can resume - from the start, and duplicate workers use CAS rather than unconditional deletion. -- HTTP v2 advertises `dataset-delete`. DELETE of the Dataset requires `{id}`; - `GET .../deletion/{id}` reconciles its receipt. Nonempty returns 409 - `dataset_not_empty`; an identity mismatch returns 409. Native `DatasetClient::delete` - uses the same authority and expected-ID contract. diff --git a/doc/design/access-server/dataset/design-dataset-control.md b/doc/design/access-server/dataset/design-dataset-control.md deleted file mode 100644 index 7bd7bd809..000000000 --- a/doc/design/access-server/dataset/design-dataset-control.md +++ /dev/null @@ -1,438 +0,0 @@ - - - -# CROWDB - Design: Dataset Control Lifecycle - -Depends on: [Dataset architecture](design-crowdb-access-dataset.md). -Satisfies: [Dataset specification](spec-crowdb-dataset.md). -Wire contract: [control API v2](openapi-crowdb-dataset-v2.yaml). - -This document describes Snapshot composition, whole-set Split revisions and their -bounded control operations. Payload delivery retains the existing binary framing. - -## Table of contents - -1. [Identity and authority](#1-identity-and-authority) -2. [Composition and inheritance](#2-composition-and-inheritance) -3. [Split evaluation and publication](#3-split-evaluation-and-publication) -4. [Bounded observation and reads](#4-bounded-observation-and-reads) -5. [Ownership and cleanup](#5-ownership-and-cleanup) -6. [Compatibility and invariants](#6-compatibility-and-invariants) - -## 1. Identity and authority - -Control v2 advertises `dataset-identity-fence`. A caller may bind a named route to -one active Dataset ID with `X-Crowdb-Dataset-Id`, a single canonical nonzero -32-character lowercase hex value. The request-owned authority carries that -expected ID through Dataset reads and root resource reservation CAS; checking -only the initial HTTP observation would leave a deletion/name-reuse race. A -replacement returns 409 `dataset_replaced`, a marked/missing Dataset returns -404, and malformed/duplicate headers return 400. Omission preserves existing -clients. This active-instance guard is omitted when retrying deletion receipts -after a successful mark; those operations already bind their immutable ID. - -A Dataset has opaque latest/stable Snapshot references. A Snapshot may declare any -complete available same-Dataset parent or no parent. Its parent is independent of the head -generation used for publication CAS. Multiple roots and off-latest branches remain -discoverable. A preparation binding or a Published flag alone is not a complete result. -The final Snapshot resource root is installed after Manifest/index rows, the initial -Split revision and necessary payload ownership are ready. The current root uses -version 2 of the existing candidate binding; new staging uses version 3 and is not -readable. Old version 1 bindings use existing publication evidence, or read-only -validation of their Published content identity/digest and complete initial Split. -They never acquire new publication proofs during reads or head replacement. Complete results are -readable, enumerable, retainable and usable as parents/stable targets without winning -latest. ID reads consult this root rather than publication receipts or mutable heads. - -Publication selects a complete candidate with an exact expected head generation. -It writes only the fenced head, without attempt/response records, success proofs -or Composition committing/published progress. Compatibility attempt IDs are echoed -in the immediate response and do not create request history. Retrying the same -current latest confirms its current effect; after another Snapshot replaces it, -the stale condition returns the observed latest/stable/generation without rewinding -the head or querying again. Unknown storage outcomes propagate unchanged. The -caller decides how to use an observed base; the server never rebases content. -Existing receipts and attempt-bearing heads are decoded read-only. Before replacing -an old head without complete resource roots, the existing bounded legacy ancestry -anchor preserves those resources; new version 4 heads need no receipt helping or -anchor lookup. Stable-only legacy roots are included in that compatibility index. - -The v1 JSON adapter prepares content once before acquiring payload owners. It then -completes that resource and selects it using the existing latest-parent condition; -it does not rerun Manifest/index preparation. Authority validation replaces the -old operation/head preflight. Responses use the known immutable metadata plus one -latest observation, without rereading operation/Snapshot records or copying the -Manifest for a second preparation. - -`PUT .../datasets/{dataset}/head` selects an existing complete Snapshot from -`snapshot` and explicit nullable `expected_head`, under `snapshot-head-update`. -It reads no draft or attempt and returns the same latest/stable/generation shape -as GET head. Stable is preserved; failed conditions include `observed_head` in that -shape. GET head reads Dataset identity and head once each. Rust exposes -`ControlClient::update_snapshot_head` and `Head::from_conflict`; Python exposes -`update_snapshot_head` and `ConflictError.observed_head` for synchronous and async -clients. Missing capability never invokes a composition or v1 publication fallback. - -Each Snapshot owns immutable Split-set revisions and an independently fenced -currentSplit pointer. Names are scoped to Dataset/Snapshot/revision. Publishing a -new revision replaces the complete set, including rename/removal; old revisions -remain readable until explicit deletion or owner retirement closes admission. -An empty set is a revision, distinct from an absent pointer. - -## 2. Composition and inheritance - -The native `prepare_composition_resource` accepts a caller-owned Snapshot ID and -complete parent/source/schema/changes/Split inputs. It computes a bounded final -Manifest directly, then writes its immutable partitions and inspection index, -initial Split membership and final payload owners. It completes the existing -resource root last; latest/stable are updated separately. There is no operation -mapping, draft, edit receipt, working row, window intent, staging owner or durable -computation cursor. The native input and final Manifest are bounded to 8 MiB and -the request to 30 seconds, with twenty-Sample source windows. Partial field patches -keep other effective parent fields; source filtering does not prevent an explicit -upsert from reintroducing a parent Sample. Each patch ID appears once. Frozen -Split sources and inherited definitions name exact revisions; retries never resolve -currentSplit again. Patch application moves existing rows and tracks encoded size -incrementally rather than serializing the growing Manifest after every change. - -A retry recomputes incomplete work and reuses matching complete content. The -existing content binding follows all final Manifest/index rows, so retrying a -failed owner claim does not rewrite those rows; it still checks unfinished initial -membership and owners before completion. The Snapshot ID fixes its Manifest and initial Split binding; different final content -conflicts, including removing an existing initial Split. Identical complete retries -write no state. Unknown completion replies return directly without a confirmation -read. Resource preparation checks Dataset identity without loading the head. -Managed upload metadata is validated once per unique upload before output writes, -and the final Snapshot owner is acquired before completion. The final Manifest -finds every owner for cleanup; there is no parallel ownership-progress list. Opaque -legacy locators inherited from a parent retain their existing storage contract; -new chunk patches must be valid managed upload locators. - -Resource prefixes and the immutable Snapshot record expose interrupted output to -owner retirement. Existing member pages are sealed after the grace period, and -recomputation cannot complete a retired Snapshot. The first Snapshot write stores -its fixed internal `gc_after`; a bounded full scan discovers abandoned resources -even before their content binding exists. Terminal deletion seals that binding, -including a missing key, so delayed creation cannot complete the resource. -`PUT snapshots/{snapshot}` exposes this operation with `composition-preparation`, -when an upload service is configured. Rust `PrepareSnapshot` and sync/async Python -`prepare_snapshot` preserve the same ID/input and never query workflow status or -downgrade. Explicit nullable parent/schema and strict DTOs reject process IDs. -Upload values carry id/length/md5, converted to locators without a metadata query; -the authority checks each unique upload before claiming it. Descriptor mismatches -return conflict. HTTP body/input/Manifest are bounded to 8 MiB. Declared oversized -bodies use the existing bounded discard before response, avoiding an unread-body -reset that can truncate the 413 JSON. Composition and standalone Split workflow -writes return 501 before storage access, and the workflow SDK methods are removed. -Only legacy HTTP observation, abandonment and cleanup remain for existing records. - -Legacy `CompositionRecord` stores old input revisions, source IDs, edits, work -progress, candidate and ownership lists. These formats remain cleanup inputs; -current routes neither append edits nor resume them. Status only reads the stored -draft and never helps receipts, queries commit ledgers or advances publication. -Legacy Committing is presented as Prepared without changing stored bytes. A caller -may select an already complete resource with a direct head update; the status does -not prove that a historical request selected the head. - -The default source is all parent Samples. Selection filters and a frozen named -Split can restrict the source, with their intersection when both are present. -A source S1/r2/train produces a child with parent S1. It copies exact frozen -membership/provenance, not the future currentSplit value. Inherited Split members -intersect surviving child IDs; newly added IDs remain unassigned unless explicit -assignments or reevaluated definitions include them. - -Sample upsert replaces the supplied field bindings while preserving other effective -fields; remove excludes its ID. Effective inherited values are resolved server-side -and materialized as bindings in the complete child Manifest. Inspection reports the -child as the binding source after this copy; chunk payloads remain shared. Sparse -Manifests created through direct publication can still resolve omitted optional -fields from an ancestor and report that ancestor as the source. An unchanged -required field must remain represented in the final complete membership. Tombstone, missing and empty inline -value remain distinct. The current Schema contains only version/name/requiredness. - -Computation reads windows of at most twenty Samples into a bounded request-local -Manifest builder. It writes final content directly; no redo window or working row -precedes it. Final resource keys reject conflicting immutable content. The complete -root makes data and initial Splits available together, and the independent data -head transition selects that resource as latest. - -## 3. Split evaluation and publication - -Standalone Split preparation accepts an owner Snapshot, caller-owned revision ID, -policy and complete definitions in one resource PUT. Preparation checks source -availability; deletion may interrupt unfinished work. It requires valid overlap/ -coverage policy. Defaults are disjoint and allow-unassigned. Frozen inherited -membership is explicit; reevaluation is a deliberate definition change. The caller -captures the intended currentSplit revision/generation before computation and keeps -that base separate from the immutable result. - -Predicates support all, explicit IDs, prefix, range, keyword, inline field match and -boolean composition over the selected universe. Effective optional inheritance and -tombstones are resolved through metadata. Payload contents are not searched. - -Weighted partition uses evaluator version 1: SHA-256 of `crowdb-split-v1\0`, seed -as u64 big endian, Sample-ID length as u32 big endian, then exact ID bytes. Interpret -the first eight digest bytes as u64 big endian. Its bucket is -`(u128(hash) * sum(weights)) >> 64`; ordered cumulative weights select the name. -Weight sums are checked u32 values. This is deterministic approximate partitioning, -not a promise of exact cardinality. Overlap/coverage validation remains independent. - -Membership pages are immutable and checksum-bound. Preparation persists the whole -revision before currentSplit CAS. A conflict preserves the candidate and caller -inputs; retry never silently rebases definitions or overwrites another revision. -The native `prepare_split_revision` operation accepts caller-owned revision ID, -policy and complete definitions. It recomputes incomplete work from those inputs, -writing only final member pages, the resource enumeration index and complete -revision root. The index precedes pages; the root is installed last. No draft, -edit receipt, window intent or phase cursor is written. Input and final bindings -are capped at 1 MiB; computation uses twenty-Sample windows within a 30-second -request deadline. It retains one window and the bounded bindings, encodes each -member page once, and tracks binding size incrementally. An identical complete -resource is reusable without recomputation; different input at that ID conflicts. -Unknown writes propagate without confirmation reads. Frozen inherited sources use -the supplied revision, regardless of later head changes. Snapshot retirement -finds both complete and partial output through resource prefixes without drafts; -complete revisions have no independent automatic expiry. Explicit revision deletion -and its bounded resweeps are described under ownership and cleanup. -The complete `SplitRevisionRecord` is the final result: an explicit revision ID -can be read, listed or used as an inherited source without winning currentSplit. -Read admission still checks the owning Snapshot. Reads load that immutable record -directly without Split publication proofs, currentSplit lookups or helper writes. -Standalone Split publication writes only currentSplit. It creates no attempt, -response receipt, per-revision success proof or draft committing/published state. -A retry returns the current generation when it still names the same revision; -after a later revision replaces it, the stale request returns a precondition -conflict without replaying historical success or rewinding the pointer. A definite -CAS conflict uses its returned head; a transport error remains unknown. Legacy -receipts and attempt-bearing heads remain readable without backfilling receipts. -`PUT snapshots/{snapshot}/split-revisions/{revision}` exposes preparation with the -`split-preparation` capability. Rust `PrepareSplitRevision` and synchronous/async -Python `prepare_split_revision` retain the exact resource ID and complete input on -retry; an error carries the revision identity. Missing capability never invokes a -draft or v1 mutation. Standalone begin/edit/evaluate/prepare/publish workflow routes -return 501 before storage access; SDK workflow methods are removed. Only legacy -draft observation, abandonment and cleanup remain for existing records. Initial Split computation in `prepare_snapshot` shares the same pure result -preparation against its fixed incomplete owner; it writes no draft or progress. -Legacy internal draft drivers remain for migration/cleanup and are not called by -the resource preparation routes. -`PUT snapshots/{snapshot}/split-head` accepts the target revision and both explicit -nullable expected revision/generation. It requires the `split-head-update` -capability and has no draft or attempt identity. The failed head condition is -returned as `observed_head`, including an observed absence. It is an observation, -not a promise that another writer has not changed the head since. The caller may -review the new base and use it in a new conditional request; no rebase is implicit. -Rust exposes the observation through `SplitHead::from_conflict`; Python exposes -`ConflictError.observed_head`. Missing legacy observations do not trigger an -automatic head query or a mutation downgrade. - -## 4. Bounded observation and reads - -`inspection_split_page` and `POST snapshots/{snapshot}/split-preview` evaluate one -Split definition over a bounded committed membership window. They reuse the same -rule compiler and evaluator as Split preparation, including deterministic weighted -partitions and frozen inheritance. The cursor binds the Snapshot and full rule; -zero matching rows may still carry a scan continuation. Preview writes no resource, -progress or read session and reads no uploaded payload bytes. The `split-preview` -capability advertises this operation; the console proxy permits it in readonly mode. - -Inspection projections contain complete summaries, Schema rows, Sample IDs, field -metadata and separate value bindings. Ordinary inspection and indexed read planning -do not load a complete Manifest or chunk payload. Inventory and parent/child indexes -filter visibility through the complete resource root, with read-only compatibility -for old publication evidence, not merely candidate record state. - -Preparing an index overlaps at most sixteen immutable row writes, with at most -2 MiB of encoded keys and values queued. Encoding holds at most one additional -row under the existing per-row bound. No row is copied into another batch buffer. -All row writes must succeed before inventory, parent/child and ready summaries -are installed. Failure leaves the index unavailable; repeating the request checks -the same immutable rows. This bounds parallel work while preserving the existing -number of conditional mutations and each row's identity. - -List pages default to twenty items and cap at one hundred with a 1 MiB metadata -window. Exact matrices admit twenty IDs by twelve fields. IDs preserve up to 16 KiB -of binary data; Schema admits up to 256 fields with names up to 1024 bytes. Projection -and selection use aggregate byte/deadline/cancellation budgets. Native bounded reads -reject oversized external values before materializing their contents. - -Continuation delivery resolves selection and field locators within one request. -It reuses that request's Snapshot admission, schema and already scanned membership; -Split visibility checks reuse admission for the same Snapshot. No admission handle -is returned to callers. A separate request checks deletion again, including empty -results. Explicit batch reads remain independent admitted requests. HTTP delivery -moves inline payload owners from resolved locators into output buffers without -cloning the payload. - -Delivery resolves membership and field bindings in windows of at most sixteen -cells. Independent point reads overlap within one ancestry level; unresolved -fields proceed to the parent only after its read-only admission. Ancestor summaries -remain local to the request. Each read reserves its response cap before dispatch, -with at most sixteen requests and 2 MiB of response capacity in flight. Completed -rows consume the aggregate metadata budget, including responses waiting behind -an earlier read. Unused reservations are released before admitting a smaller final -read. Request order, missing fields, tombstones and locator integrity checks remain -independent of completion order. All windows share the request deadline and -cancellation; no reader state is persisted. - -The bounded compatibility part reader checks the declared logical length before -allocation. A single buffer covering that complete object is transferred with its -allocation owner after successful EOF. Fragmented objects allocate their final -buffer once; each source buffer is copied and released before the next storage -pull, so a one-slot reader can make progress. Short, oversized and failed terminal -reads remain errors. Whole-part digest verification still runs on the returned -bytes before acceptance. - -Old snapshots without projections require an explicit complete inspection rebuild. -One request bounds the source record at 9 MiB, its embedded Manifest at 8 MiB and -the entire rebuild at thirty seconds. Final rows use at most sixteen pending -writes and 2 MiB of encoded keys and values in flight. The builder writes complete -rows, inventory and parent-child discovery before installing the immutable summary -root. Partial rows are invisible without that root. A failed request returns its -error and a retry recomputes from the source; it saves no preparation cursor or -request receipt. An existing summary returns without writes. Cleanup and the -finalizer conditionally replace the same root, so a retired root rejects a delayed -finalizer. - -Status can decode historical preparation progress when no summary exists, but a -complete rebuild never resumes or rewrites that progress. The summary takes -precedence over historical progress, including corrupt progress. New successful -responses retain zero revision and row-count fields for wire compatibility. -Unsupported or oversized sources return errors instead of empty Snapshots. Field -preview is explicit and capped at 64 KiB after read-only admission; it does not -create a read plan or acknowledge application consumption. - -V2 read planning pins Snapshot and optional Split revision once, intersects Selection -with Split membership, then orders/shuffles and partitions workers before fetching -projected payloads. Planning advances twenty-Sample windows with fixed total ID/byte -limits. Fetch/preview/status never save consumption progress. The client saves the next -continuation after processing; acknowledgement is local. Cancel stops the current -request without a reader registry or remote close. - -## 5. Ownership and cleanup - -Incomplete-candidate cleanup and finalization conditionally replace the same -resource root. A cleanup seal prevents a delayed finalizer from exposing retired -rows. If the complete root wins, abandoning or superseding the draft preserves its -Manifest, initial Split members and payload owners; ordinary Snapshot reclamation -later retires them under deletion admission and grace rules. Completion transport -errors propagate without confirmation reads. Definite cleanup conflicts use the -returned value without reloading the root. - -The parent/child index is installed before the child's complete root. Finalization -rechecks parent availability after that index write. Parent deletion conservatively -protects any indexed child without terminal retirement, including an incomplete -child awaiting cleanup. Thus deletion either observes the child or the child's -last parent check rejects completion. Readers never register in this protocol. -Reclaim admission may remove an exact child-index entry after validating its -binding and the child's permanent retirement authority. Completed removals shrink -later bounded scans; the retirement authority remains even if a delayed writer -recreates the derived index. A failed or timed-out index deletion returns its -unknown outcome without compensation writes. A fresh request rechecks authority -and can resume this resource cleanup; ordinary inspection remains read-only. -Discovery and per-resource positions stay in memory. Restart may repeat already -sealed prefixes, so progress requires enough consecutive bounded windows to pass -those prefixes. Grace deadlines specify earliest reclamation, not a completion -time or a guarantee under arbitrary restart frequency. - -Uploads are Dataset-scoped and content-verified. Resource preparation claims only -the final Snapshot owner for each unique managed upload in its final Manifest, -including unchanged parent uploads. Completion requires all those owners. The -Manifest supplies cleanup's owner references without a second process list. -Retiring one owner preserves other owners; permanent revocation rejects delayed -claims. Legacy staging/candidate lists remain cleanup inputs for old records. - -Legacy draft abandonment and cleanup use their existing bounded cursors and seals -for old work, including keys that could still receive delayed writes. Stored edit -receipts and progress can be decoded and reclaimed, but resource preparation never -creates or resumes them. Their independent discovery and replacement remain part -of migration work; removing the last discovery path would lose owned garbage. - -Legacy draft cleanup competes with preparation on the final revision key. If the -complete root already exists, including a root returned by a failed cleanup CAS, cleanup -preserves its member pages and summary while retiring evaluation intents. If -cleanup seals an absent root first, a delayed preparation cannot publish it. -Abandoning the draft does not delete an already complete revision. Retiring the -owning Snapshot still permits revision cleanup and closes new read admission. -Sources use read-only availability checks. Legacy `source_lease_deadline` is only -a compatibility field; resource preparation has no reader lease or draft. - -Complete Split revisions normally remain owned by their Snapshot and are reclaimed -with it; there is no independent revision TTL. Explicit -`DELETE snapshots/{snapshot}/split-revisions/{revision}` takes only those two IDs -and an empty body. It rejects the current head and the Snapshot's immutable -initial revision with 409, including an initial revision that is no longer current. -A missing revision returns 404. A successful mark immediately closes new revision -reads and frozen-source admission; ordinary requests return 404. Existing bounded -read limits still apply and may return a limit error before the closed record is -decoded. Previously admitted -work may continue until cleanup, and may fail if it outlives the grace period. - -Deletion state and the fixed configured (default twenty-minute) deadline replace the complete root on -the same revision key, retaining its page bindings for cleanup. The head generation -is fenced against a publisher that observed the revision before deletion. There -is no separate deletion task, reader session or request ledger. Repeating DELETE -preserves the stored deadline; an unknown mutation returns without a confirmation -read, and the client retries the same two IDs. The bounded background worker scans -revision resources using discardable local cursors, seals membership metadata and -rescans terminal resources for delayed member writes. It does not delete Samples, -upload owners or chunk payloads. If Snapshot deletion has already closed every -owner read and its own configured (default twenty-minute) grace has elapsed, revision cleanup may use -that earlier owner fence without waiting for the revision's later deadline. - -DELETE returns 200 with `snapshot`, `revision`, `complete` and `reclaim_after`. -Pending deletion reports `complete=false` and the fixed absolute Unix-seconds -deadline as a canonical decimal string. A terminal resource reports -`complete=true` and an explicit null deadline; it retains no historical response -from which to reconstruct the old deadline. The `split-revision-delete` capability -is independent of revision reads and preparation. Rust and synchronous/async Python -expose `delete_split_revision(snapshot, revision)`; missing capability never invokes -legacy draft abandonment. The native adapter calls the same authority under local -admission with no additional metadata GET and no automatic unknown-outcome retry. - -Snapshot reclaim checks latest/stable, retained Snapshots, live children, candidate -ownership under a terminal owner fence. New deletion marks have a durable configured (default twenty-minute) grace deadline; -existing marks keep their persisted absolute deadline. The grace period -separates deletion admission from cleanup. After that deadline it retires upload owners, -Snapshot-scoped Split drafts, possible index writes and ordinary metadata in phases. -Completion reports logical retirement; physical upload deletion remains deferred -to the bounded upload-authority scan. Complete Split metadata also supports the -explicit revision deletion described above. - -`GET snapshots/{snapshot}/reclaim` is bounded, read-only observation. It returns -publication operation identity where available, latest/stable/retained flags, -`reclaim_after` and the compatibility `active_read_lease=false` field, and not_started/pending/completed retirement state. Parent reachability is -explicitly not evaluated in this small read; observed flags are never permission -to reclaim. Rust/Python `control().lifecycle(snapshot)` share this contract. -Metadata HTTP reads remain capped at five seconds, while bounded mutation steps -use their configured mutation deadline (normally 60 seconds). - -## 6. Compatibility and invariants - -Control v2 exposes strict DTOs and capability discovery. Required null preconditions -cannot be omitted; u64s use decimal strings; binary IDs use canonical base64. Clients -check capabilities before mutation and never downgrade a Split-aware operation to v1. -Existing v1 routes, persisted read interpretation and binary framing remain available. -Rust and Python control clients share transport/admission/retry/cancellation semantics. -SDK clones share a fixed atomic bitset of capabilities previously advertised by -the versioned endpoint. Positive hints remove discovery GETs from warm control -calls; absent capabilities are rediscovered, never downgraded. Explicit -`capabilities()` queries replace hints after a valid response and clear them for -an unsupported API version. Unknown transport outcomes do not imply capability -revocation. Concurrent cold calls may duplicate discovery under the existing -HTTP budget; there is no discovery lock or server session. Every actual operation -still passes server authentication, capacity, deletion admission and mutation -preconditions. Server redeployment can invalidate a hint and make the operation -fail; explicit discovery refreshes it. A preflight GET could not eliminate that -deployment window either. - -- **I1 — Immutable data:** committed Snapshot content never changes. -- **I2 — Independent revision:** Split publication cannot move Dataset latest. -- **I3 — Frozen source:** source revision identity survives later pointer changes. -- **I4 — Complete publication:** data, initial membership and upload ownership are - ready before the visible head transition. -- **I5 — Exact retry:** a successful attempt cannot create another Snapshot or - rewind a later head. -- **I6 — Bounded work:** page, byte and time budgets constrain underlying storage - work as well as response length. -- **I7 — Terminal retirement:** delayed writers cannot restore retired work or - acquire a revoked owner after cleanup. diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml deleted file mode 100644 index 31359f8b6..000000000 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v1.yaml +++ /dev/null @@ -1,2535 +0,0 @@ -# Copyright 2026-present Gian -# Licensed under the Apache License, Version 2.0. -{ - "openapi": "3.1.0", - "info": { - "title": "CrowDB Dataset HTTP API", - "version": "1.0.0", - "description": "Public Dataset REST v1 wire contract. Unknown optional JSON properties are ignored. Missing required properties and unknown variants are rejected. This document does not expose persisted bincode records." - }, - "security": [ - { - "Bearer": [] - } - ], - "paths": { - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}": { - "put": { - "operationId": "CreateDataset", - "description": "Duplicate identity returns conflict. A lost create reply is recovered by OpenDataset; never retry creation with a new name.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Dataset" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - }, - "get": { - "operationId": "OpenDataset", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Dataset" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots/prepare": { - "post": { - "operationId": "PrepareSnapshot", - "description": "Same canonical content/token recovers the same candidate. Complete preparation persists the manifest but never advances latest. Payload uploads are fenced against cleanup before binding.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Snapshot" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - }, - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Publication" - } - } - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots/publish": { - "post": { - "operationId": "PublishSnapshot", - "description": "Returns only after complete publication and head advancement. Lost replies are recovered with the same token and content. A published operation with is_latest=false remains addressable; status does not claim head success.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Snapshot" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - }, - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Publication" - } - } - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/operations/{operation}": { - "get": { - "operationId": "GetOperation", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "operation", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Snapshot" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots": { - "get": { - "operationId": "ListSnapshots", - "description": "Latest ancestry, newest first. next is the next immutable ancestor. An omitted starting cursor resolves latest once.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "query", - "name": "after", - "schema": { - "$ref": "#/components/schemas/Id" - } - }, - { - "in": "query", - "name": "limit", - "schema": { - "type": "integer", - "minimum": 1, - "maximum": 1000, - "default": 100 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SnapshotPage" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots/{snapshot}": { - "get": { - "operationId": "GetSnapshot", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "snapshot", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Snapshot" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots/{snapshot}/manifest": { - "get": { - "operationId": "GetManifest", - "description": "Bounded JSON manifest; response over metadata limit is rejected, never truncated.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "snapshot", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Manifest" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/names/stable": { - "put": { - "operationId": "SetStable", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Dataset" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - }, - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SnapshotRequest" - } - } - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots/{snapshot}/retain": { - "post": { - "operationId": "RetainSnapshot", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "snapshot", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Snapshot" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots/{snapshot}/release": { - "post": { - "operationId": "ReleaseSnapshot", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "snapshot", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Snapshot" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/snapshots/{snapshot}/reclaim": { - "post": { - "operationId": "ReclaimSnapshot", - "description": "Deletion marks the snapshot unavailable to new requests and persists a twenty-minute grace deadline before background cleanup. This empty-body action only marks deletion and observes status; repeated requests do not drive metadata cleanup or accept a client progress cursor. Latest/stable/retained/parent snapshots conflict. Active readers do not block deletion; overdue reads may fail. Physical capacity recovery remains a separate ChunkDB responsibility.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "snapshot", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Reclaim" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - }, - "get": { - "operationId": "GetReclaimStatus", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "snapshot", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Reclaim" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/uploads": { - "post": { - "operationId": "CreateUpload", - "description": "Dataset-scoped token. At most 10000 parts, each at most 4 MiB. Retrying identical declarations returns status without extending expiry.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Upload" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - }, - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UploadRequest" - } - } - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/uploads/{upload}": { - "get": { - "operationId": "GetUpload", - "description": "", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "upload", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Upload" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "operationId": "AbortUpload", - "description": "Idempotent abort for unclaimed data. Claimed data conflicts. Cleanup may remain pending; metadata may not claim physical completion before storage reclamation acknowledges it.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "upload", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Upload" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/uploads/{upload}/parts/{index}": { - "put": { - "operationId": "PutUploadPart", - "description": "Exactly the declared part length. Immutable index/digest: identical retry succeeds, divergent content conflicts. Allocation intents and storage acknowledgements survive crash. Returns one bounded part receipt, never a growing upload list.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "upload", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - }, - { - "in": "path", - "name": "index", - "required": true, - "schema": { - "type": "integer", - "minimum": 0, - "maximum": 9999 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Part" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - }, - "requestBody": { - "required": true, - "content": { - "application/octet-stream": { - "schema": { - "type": "string", - "format": "binary", - "maxLength": 4194304 - } - } - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/uploads/{upload}/complete": { - "post": { - "operationId": "CompleteUpload", - "description": "Checks contiguous parts, full length and whole MD5 before publication eligibility. Completion is not snapshot publication; open/complete uploads expire after 24 hours of server time.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "upload", - "required": true, - "schema": { - "$ref": "#/components/schemas/Id" - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Upload" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/reads/next": { - "post": { - "operationId": "NextBatch", - "description": "Replay the caller continuation. Save next only after successful processing; no remote acknowledgement or close.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/vnd.crowdb.dataset-batch.v1": { - "schema": { - "type": "string", - "format": "binary" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - }, - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ReadContinuation" - } - } - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/reads/bind": { - "post": { - "operationId": "BindRead", - "description": "Resolve immutable versions into a client-owned continuation. No read record or progress is stored.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ReadBinding" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - }, - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ReadPlan" - } - } - } - } - } - }, - "/api/dataset/v1/namespaces/{namespace}/datasets/{dataset}/reads": { - "post": { - "operationId": "GetBatch", - "description": "GetSample uses one sample ID. One streaming response for all requested fields in sample/projection order, with field and frame integrity. Unknown samples/projections reject before payload reads. Missing and tombstone fields are explicit.", - "parameters": [ - { - "in": "path", - "name": "namespace", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - }, - { - "in": "path", - "name": "dataset", - "required": true, - "schema": { - "type": "string", - "minLength": 1, - "maxLength": 255 - } - } - ], - "responses": { - "200": { - "description": "Successful operation", - "content": { - "application/vnd.crowdb.dataset-batch.v1": { - "schema": { - "type": "string", - "format": "binary" - } - } - } - }, - "400": { - "$ref": "#/components/responses/Error" - }, - "401": { - "$ref": "#/components/responses/Error" - }, - "404": { - "$ref": "#/components/responses/Error" - }, - "409": { - "$ref": "#/components/responses/Error" - }, - "413": { - "$ref": "#/components/responses/Error" - }, - "429": { - "$ref": "#/components/responses/Error" - }, - "500": { - "$ref": "#/components/responses/Error" - }, - "501": { - "$ref": "#/components/responses/Error" - }, - "503": { - "$ref": "#/components/responses/Error" - }, - "504": { - "$ref": "#/components/responses/Error" - }, - "499": { - "$ref": "#/components/responses/Error" - } - }, - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BatchRequest" - } - } - } - } - } - } - }, - "components": { - "securitySchemes": { - "Bearer": { - "type": "http", - "scheme": "bearer" - } - }, - "schemas": { - "Id": { - "type": "string", - "pattern": "^(?!0{32}$)[0-9a-f]{32}$" - }, - "Md5": { - "type": "string", - "pattern": "^[0-9a-f]{32}$" - }, - "SampleId": { - "type": "string", - "format": "byte", - "minLength": 1, - "maxLength": 1368, - "description": "Canonical padded RFC 4648 base64 of 1..1024 opaque sample-ID bytes." - }, - "Error": { - "type": "object", - "properties": { - "code": { - "type": "string", - "maxLength": 255 - }, - "message": { - "type": "string", - "maxLength": 1024 - }, - "request_id": { - "$ref": "#/components/schemas/Id" - }, - "retryable": { - "type": "boolean" - } - }, - "required": [ - "code", - "message", - "request_id", - "retryable" - ] - }, - "Dataset": { - "type": "object", - "properties": { - "id": { - "$ref": "#/components/schemas/Id" - }, - "namespace": { - "const": "dataset-ns" - }, - "name": { - "type": "string", - "maxLength": 255 - }, - "latest": { - "anyOf": [ - { - "$ref": "#/components/schemas/Id" - }, - { - "type": "null" - } - ] - }, - "stable": { - "anyOf": [ - { - "$ref": "#/components/schemas/Id" - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "id", - "namespace", - "name", - "latest", - "stable" - ] - }, - "FieldDefinition": { - "type": "object", - "properties": { - "name": { - "type": "string", - "maxLength": 255 - }, - "required": { - "type": "boolean" - } - }, - "required": [ - "name", - "required" - ] - }, - "FieldValue": { - "oneOf": [ - { - "type": "object", - "properties": { - "kind": { - "const": "inline" - }, - "data": { - "type": "string", - "format": "byte", - "maxLength": 87384 - } - }, - "required": [ - "kind", - "data" - ] - }, - { - "type": "object", - "properties": { - "kind": { - "const": "payload" - }, - "upload": { - "$ref": "#/components/schemas/Id" - } - }, - "required": [ - "kind", - "upload" - ] - }, - { - "type": "object", - "properties": { - "kind": { - "const": "tombstone" - } - }, - "required": [ - "kind" - ] - } - ] - }, - "Sample": { - "type": "object", - "properties": { - "id": { - "$ref": "#/components/schemas/SampleId" - }, - "fields": { - "type": "object", - "maxProperties": 256, - "additionalProperties": { - "$ref": "#/components/schemas/FieldValue" - } - } - }, - "required": [ - "id", - "fields" - ] - }, - "Manifest": { - "type": "object", - "properties": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/FieldDefinition" - }, - "maxItems": 256 - }, - "samples": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Sample" - }, - "maxItems": 1000000 - } - }, - "required": [ - "schema", - "samples" - ] - }, - "Publication": { - "type": "object", - "properties": { - "operation": { - "$ref": "#/components/schemas/Id" - }, - "parent": { - "anyOf": [ - { - "$ref": "#/components/schemas/Id" - }, - { - "type": "null" - } - ] - }, - "manifest": { - "$ref": "#/components/schemas/Manifest" - } - }, - "required": [ - "operation", - "parent", - "manifest" - ] - }, - "Snapshot": { - "type": "object", - "properties": { - "id": { - "$ref": "#/components/schemas/Id" - }, - "parent": { - "anyOf": [ - { - "$ref": "#/components/schemas/Id" - }, - { - "type": "null" - } - ] - }, - "operation": { - "$ref": "#/components/schemas/Id" - }, - "state": { - "enum": [ - "prepared", - "published", - "aborted" - ] - }, - "is_latest": { - "type": "boolean" - } - }, - "required": [ - "id", - "parent", - "operation", - "state", - "is_latest" - ] - }, - "SnapshotPage": { - "type": "object", - "properties": { - "snapshots": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Snapshot" - }, - "maxItems": 1000 - }, - "next": { - "anyOf": [ - { - "$ref": "#/components/schemas/Id" - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "snapshots", - "next" - ] - }, - "UploadRequest": { - "type": "object", - "properties": { - "id": { - "$ref": "#/components/schemas/Id" - }, - "length": { - "type": "integer", - "minimum": 1, - "maximum": 41943040000 - }, - "part_size": { - "type": "integer", - "minimum": 1, - "maximum": 4194304 - }, - "md5": { - "$ref": "#/components/schemas/Md5" - } - }, - "required": [ - "id", - "length", - "part_size", - "md5" - ] - }, - "Part": { - "type": "object", - "properties": { - "index": { - "type": "integer", - "minimum": 0, - "maximum": 9999 - }, - "length": { - "type": "integer", - "minimum": 1, - "maximum": 4194304 - }, - "md5": { - "$ref": "#/components/schemas/Md5" - } - }, - "required": [ - "index", - "length", - "md5" - ] - }, - "Upload": { - "type": "object", - "properties": { - "id": { - "$ref": "#/components/schemas/Id" - }, - "length": { - "type": "integer" - }, - "part_size": { - "type": "integer" - }, - "md5": { - "$ref": "#/components/schemas/Md5" - }, - "state": { - "enum": [ - "open", - "complete", - "claimed", - "aborted" - ] - }, - "expires_at": { - "type": "integer" - }, - "parts": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Part" - }, - "maxItems": 10000 - }, - "cleanup_complete": { - "type": "boolean", - "description": "True only after all unowned physical ranges have been reclaimed. Deferred cleanup remains false." - } - }, - "required": [ - "id", - "length", - "part_size", - "md5", - "state", - "expires_at", - "parts" - ] - }, - "BatchRequest": { - "type": "object", - "properties": { - "snapshot": { - "$ref": "#/components/schemas/Id" - }, - "sample_ids": { - "type": "array", - "items": { - "$ref": "#/components/schemas/SampleId" - }, - "maxItems": 10000 - }, - "fields": { - "type": "array", - "items": { - "type": "string", - "maxLength": 255 - }, - "maxItems": 256 - } - }, - "required": [ - "snapshot", - "sample_ids", - "fields" - ] - }, - "BatchField": { - "type": "object", - "properties": { - "sample": { - "$ref": "#/components/schemas/SampleId" - }, - "field": { - "type": "string", - "maxLength": 255 - }, - "state": { - "enum": [ - "value", - "missing", - "tombstone" - ] - }, - "length": { - "type": "integer", - "minimum": 0 - }, - "md5": { - "anyOf": [ - { - "$ref": "#/components/schemas/Md5" - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "sample", - "field", - "state", - "length", - "md5" - ] - }, - "BatchHeader": { - "type": "object", - "properties": { - "snapshot": { - "$ref": "#/components/schemas/Id" - }, - "sample_ids": { - "type": "array", - "items": { - "$ref": "#/components/schemas/SampleId" - }, - "maxItems": 10000 - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/components/schemas/BatchField" - }, - "maxItems": 10000 - }, - "next": { - "anyOf": [ - { - "type": "string", - "maxLength": 4096 - }, - { - "type": "null" - } - ] - }, - "end": { - "type": "boolean" - } - }, - "required": [ - "snapshot", - "sample_ids", - "fields", - "next", - "end" - ] - }, - "WorkerPartition": { - "type": "object", - "properties": { - "worker": { - "type": "integer", - "minimum": 0, - "maximum": 65535 - }, - "workers": { - "type": "integer", - "minimum": 1, - "maximum": 65535 - }, - "rank": { - "type": "integer", - "minimum": 0, - "maximum": 65535 - }, - "world_size": { - "type": "integer", - "minimum": 1, - "maximum": 65535 - } - }, - "required": [ - "worker", - "workers", - "rank", - "world_size" - ] - }, - "Selection": { - "oneOf": [ - { - "type": "object", - "properties": { - "kind": { - "const": "explicit" - }, - "ids": { - "type": "array", - "items": { - "$ref": "#/components/schemas/SampleId" - }, - "maxItems": 10000 - } - }, - "required": [ - "kind", - "ids" - ] - }, - { - "type": "object", - "properties": { - "kind": { - "enum": [ - "prefix", - "keyword" - ] - }, - "value": { - "$ref": "#/components/schemas/SampleId" - } - }, - "required": [ - "kind", - "value" - ] - }, - { - "type": "object", - "properties": { - "kind": { - "const": "range" - }, - "start": { - "$ref": "#/components/schemas/SampleId" - }, - "end": { - "$ref": "#/components/schemas/SampleId" - } - }, - "required": [ - "kind", - "start", - "end" - ] - }, - { - "type": "object", - "properties": { - "kind": { - "const": "field_match" - }, - "field": { - "type": "string", - "maxLength": 255 - }, - "value": { - "$ref": "#/components/schemas/SampleId" - } - }, - "required": [ - "kind", - "field", - "value" - ] - }, - { - "type": "object", - "properties": { - "kind": { - "enum": [ - "and", - "or" - ] - }, - "items": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Selection" - }, - "maxItems": 64 - } - }, - "required": [ - "kind", - "items" - ] - }, - { - "type": "object", - "properties": { - "kind": { - "const": "not" - }, - "item": { - "$ref": "#/components/schemas/Selection" - } - }, - "required": [ - "kind", - "item" - ] - } - ] - }, - "Shuffle": { - "type": "object", - "properties": { - "seed": { - "type": "integer", - "minimum": 0, - "maximum": 18446744073709551615 - }, - "group_size": { - "type": "integer", - "minimum": 1, - "maximum": 1000000 - } - }, - "required": [ - "seed", - "group_size" - ] - }, - "ReadPlan": { - "type": "object", - "properties": { - "snapshot": { - "type": "string", - "maxLength": 32 - }, - "selection": { - "$ref": "#/components/schemas/Selection" - }, - "fields": { - "type": "array", - "items": { - "type": "string", - "maxLength": 255 - }, - "maxItems": 256 - }, - "batch_size": { - "type": "integer", - "minimum": 1, - "maximum": 10000 - }, - "partition": { - "$ref": "#/components/schemas/WorkerPartition" - }, - "shuffle": { - "anyOf": [ - { - "$ref": "#/components/schemas/Shuffle" - }, - { - "type": "null" - } - ] - } - }, - "required": [ - "snapshot", - "selection", - "fields", - "batch_size", - "partition", - "shuffle" - ] - }, - "SnapshotRequest": { - "type": "object", - "properties": { - "snapshot": { - "$ref": "#/components/schemas/Id" - } - }, - "required": [ - "snapshot" - ] - }, - "Reclaim": { - "type": "object", - "properties": { - "snapshot": { - "$ref": "#/components/schemas/Id" - }, - "state": { - "enum": [ - "pending", - "completed" - ] - }, - "chunks_reclaimed": { - "type": "integer", - "minimum": 0 - } - }, - "required": [ - "snapshot", - "state", - "chunks_reclaimed" - ] - }, - "ReadBinding": { - "type": "object", - "required": [ - "snapshot", - "cursor" - ], - "properties": { - "snapshot": { - "type": "string" - }, - "cursor": { - "type": "string", - "maxLength": 349530 - } - } - }, - "ReadContinuation": { - "type": "object", - "additionalProperties": false, - "required": [ - "cursor" - ], - "properties": { - "cursor": { - "type": "string", - "maxLength": 349530 - } - } - } - }, - "responses": { - "Error": { - "description": "Stable structured failure; credentials and payload contents are never included. Codes: invalid_request, unauthorized, not_found, conflict, request_too_large, capacity, integrity, cancelled, unavailable, timeout, unsupported. Only capacity/unavailable are retryable when the operation permits replay.", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - } - }, - "x-crowdb-wire": { - "identity": "IDs are lowercase 32-hex nonzero 16-byte identities. MD5 is lowercase 32 hex; MD5 is an integrity check, not authentication. Dataset names are UTF-8, percent-encoded as one path segment, compared without Unicode normalization. Only dataset-ns exists in v1. Sample IDs are canonical padded base64, never interpreted as paths.", - "bounds": "JSON request/response at most 8 MiB before allocation; inline decoded values at most 64 KiB, schema/projection 256 fields, sample IDs at most 1024 bytes. Resource credits cover active request bodies and actual response delivery. Request/stream deadlines and deployment admission budgets are explicit configuration. Stateless read groups admit 100000 selected IDs and 8 MiB of ID bytes; continuation JSON is capped at 256 KiB before encoding. There are no persisted read pages or progress records. Manifest membership is the complete current sample set; omitted samples are removed, while omitted optional fields on surviving samples inherit by ancestry. Required fields must be present in each sample; explicit tombstones hide parent fields.", - "batch": "Content-Type application/vnd.crowdb.dataset-batch.v1. Magic is 8 ASCII bytes CDBDS001. Each frame: type u8, body length u32 big endian, then body. Types: 1 JSON BatchHeader (at most 8 MiB); 2 data (16-byte MD5 followed by 1..4 MiB bytes); 3 field end (zero length); 0 batch end (zero length); 4 terminal JSON Error (at most 64 KiB, only after the header). Header declares every sample/field in requested order. Value fields are delivered sequentially with one or more type-2 frames and one type-3; empty values have just type-3. Missing/tombstone have no data frames. Validate each frame MD5, exact whole-field length/MD5, field ends, batch end and HTTP EOF with no trailing bytes. A batch is successful only after completion. A streaming destination may contain verified partial data on error and must be discarded; download-to-path publishes only a verified temporary file. A type-4 frame reports a failed operation, may follow verified partial data, and never permits successful completion. It carries the same structured error model as control responses. The SDK must report the error and discard partial output.", - "retry": "SDK uses one overall deadline and stable logical tokens; bounded attempts/backoff only on documented idempotent operations with replayable bodies. Do not retry authorization, integrity or semantic conflicts. Ambiguous mutation errors retain operation/upload identity for status recovery. Never invent new tokens on retry.", - "integrity": "Seekable file convenience upload makes a bounded digest pass then a bounded transfer pass. Upload complete independently verifies durable parts and whole MD5. These are deliberate integrity passes, not whole-file buffering. Nonseekable uploads require known length/digest or later explicit streaming-session design.", - "lifecycle": "Remote durability is guaranteed after successful snapshot publication; local staging/checkpoint flush semantics are future SDK work. Reads perform no persistent activity registration or renewal. Snapshot deletion rejects new requests and defers cleanup for twenty minutes; admitted reads may complete or fail if overdue. Local byte credits survive until streaming completes/drops. Keep-alive uses bounded idle/header timeouts and shutdown drain. Upload ownership and reclaim are durable authority state, never client authority. Whole-file integrity completion has a 30-minute server deadline; clients use their configurable transfer deadline. Candidate reclamation retains an aborted operation tombstone; publication retry cannot resurrect it. chunks_reclaimed counts whole chunks, not retired shared ranges." - } -} diff --git a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml b/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml deleted file mode 100644 index 43e050fa1..000000000 --- a/doc/design/access-server/dataset/openapi-crowdb-dataset-v2.yaml +++ /dev/null @@ -1,2002 +0,0 @@ -# Copyright 2026-present Gian -# Licensed under the Apache License, Version 2.0. -openapi: 3.1.0 -info: - title: CrowDB Dataset control API v2 - version: 2.0.0 - description: >- - Capability-gated Dataset control operations. Clients must discover required - capabilities before mutation. An absent capability is unavailable; clients - must not downgrade a v2 mutation to v1. The capability list advertises only - implemented operation families. Existing v1 control and binary framing remain - available under their original routes. Metadata GET routes require - bounded-inspection; explicit POST inspection preparation requires - inspection-preparation. Dataset creation, stable/retention changes and receipts - require dataset-management; direct Snapshot head updates require snapshot-head-update; - Split reads require split-revisions; read-only rule previews require split-preview; complete Split preparation requires split-preparation; - direct Split head updates require split-head-update; explicit complete-revision - deletion requires split-revision-delete and never falls back to draft abandonment; versioned - stateless reads require stateless-reads. Upload, Snapshot composition preparation and retirement - require uploads, composition-preparation and snapshot-reclaim respectively, advertised only - when the upload service is configured. Query parameters are strict and duplicate - keys fail. Snapshot preparation JSON/input/Manifest are limited to 8 MiB; - standalone Split preparation JSON and final bindings are limited to 1 MiB. - Composition and standalone Split begin/edit/evaluate/prepare/publish workflow routes return 501; - legacy draft reads, abandonment and cleanup remain available for old records. - The dataset-identity-fence capability supports X-Crowdb-Dataset-Id on named - Dataset routes. The active-instance guard is checked during request admission - and root creation reservation CAS; a replacement returns 409 dataset_replaced, - missing/marked identities return 404, malformed/duplicate headers return 400. - Existing clients may omit it. Deletion receipt/status retries after marking - omit this active guard and retain their own expected immutable ID. -servers: - - url: /api/dataset/v2 -security: - - bearerAuth: [] -paths: - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - put: - operationId: prepareSnapshot - summary: Prepare complete Snapshot content from retained caller inputs - description: >- - Requires composition-preparation and a configured upload service. Caller - chooses the Snapshot ID and retains the complete parent/source/schema/ - Sample patches/Split definitions. Parent and schema are required nullable - fields; null schema inherits the fixed parent's schema. Frozen sources and - inherited definitions name exact revisions; currentSplit is never resolved - during retries. Body, native input and final Manifest are bounded to 8 MiB, - with 30 seconds for computation. Each patch ID occurs once; upsert patches - specified fields and preserves other effective parent fields. Uploaded - fields carry id/length/md5 from a completed result; conversion performs no - metadata query and the authority validates the descriptor before claiming - its final Snapshot owner. The complete root is installed last, after final - index, initial Split membership and payload ownership. Complete results - are readable by ID before head publication. Retry the same ID and full - input after unknown outcomes; different content or initial Split binding - conflicts. No editing/evaluation progress, operation mapping or staging - owner is stored. Capture the intended head base before computation, then - update the Dataset head separately. There is no workflow or v1 fallback. - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/PrepareSnapshot'} - responses: - '200': - description: Complete immutable result; latest/stable are unchanged - content: - application/json: - schema: {$ref: '#/components/schemas/PreparedSnapshot'} - '409': - description: Resource content, descriptor or Split policy conflicts - $ref: '#/components/responses/InspectionError' - '413': - description: Body, native input, final Manifest or computation exceeds bounds - $ref: '#/components/responses/InspectionError' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/head: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: datasetHead - summary: Read mutable names and their current opaque head generation - description: >- - Requires head-inspection capability. Generation covers the exact head - including its internal fence. An absent head has null generation. Stable - changes and reclaim fences may change generation without changing latest. - Clients must treat the token as opaque and preserve it without decoding. - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - responses: - '200': - description: One consistent observation of the current head - content: - application/json: - schema: {$ref: '#/components/schemas/Head'} - '404': - description: Dataset or namespace is unavailable - put: - operationId: updateSnapshotHead - summary: Select a complete Snapshot using an explicit head condition - description: >- - Requires snapshot-head-update. The input contains only the complete resource ID - and explicit nullable expected_head; no draft or attempt identity is read. - Stable is preserved. Same-current-target retries confirm the current effect; - stale conditions after another publication return observed_head without - a confirmation query or implicit content rebase. Unknown outcomes do not - create response history. The JSON body is limited to 4096 bytes. - parameters: - - {$ref: '#/components/parameters/dataset'} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/UpdateSnapshotHead'} - responses: - '200': - description: Current effect confirmed without a historical response receipt - content: - application/json: - schema: {$ref: '#/components/schemas/Head'} - default: - $ref: '#/components/responses/InspectionError' - /capabilities: - get: - operationId: datasetCapabilities - summary: Discover supported Dataset control operation families - responses: - '200': - description: Supported API version and capability names - headers: - x-request-id: - schema: {type: string} - content: - application/json: - schema: {$ref: '#/components/schemas/Capabilities'} - '401': - description: Missing or invalid configured bearer token - '429': - description: API admission capacity exhausted - '504': - description: Request deadline expired - /namespaces/dataset-ns/datasets: - get: - operationId: listDatasets - summary: Discover existing Dataset identities - parameters: - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/DatasetPage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshot-inventory: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotInventory - summary: List complete Snapshot inventory - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/SnapshotPage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/roots: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotRoots - summary: List complete parentless Snapshot roots - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/SnapshotPage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/children: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotChildren - summary: List complete children of a concrete Snapshot - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/SnapshotPage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/summary: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotSummary - summary: Read bounded complete Snapshot metadata - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/SnapshotSummary'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/schema: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotSchema - summary: Read an ordered Schema window - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/SchemaPage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/samples: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotSamples - summary: Read exact binary Sample IDs in byte order - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/SamplePage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-preview: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - operationId: previewSnapshotSplit - summary: Preview one Split definition without creating a revision - description: >- - Requires split-preview. Read-only bounded evaluation on a complete Snapshot, - including frozen inherited membership. Limit bounds scanned Samples, not - returned matches; an empty page can have a continuation. The cursor binds - the immutable scope and complete definition. No draft, evaluation progress, - resource or read session is written. Uploaded payload bytes are not read. - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/DraftSplit'} - responses: - '200': - description: Matching base64 Sample IDs and scan continuation. - content: - application/json: - schema: {$ref: '#/components/schemas/SamplePage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/matrix: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotMatrix - summary: Read effective field metadata without payload access - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - - in: query - name: fields - required: true - description: JSON-encoded array of 1 to 12 exact field names; continuation binds this projection. - schema: {type: string} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/MatrixPage'} - default: - $ref: '#/components/responses/InspectionError' - post: - operationId: exactSnapshotMatrix - summary: Project a bounded explicit ID window without creating a read session - description: Read-only query, including Sample IDs obtained from a Split member page. Retains request order. - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - requestBody: - required: true - content: - application/json: - schema: - type: object - additionalProperties: false - required: [sample_ids, fields] - properties: - sample_ids: {type: array, minItems: 1, maxItems: 20, uniqueItems: true, items: {type: string}} - fields: {type: array, minItems: 1, maxItems: 12, uniqueItems: true, items: {type: string}} - responses: - '200': - description: Effective field metadata only; no payload or consumption progress mutation. - content: - application/json: - schema: - type: object - required: [entries] - properties: - entries: {type: array, maxItems: 20, items: {$ref: '#/components/schemas/MatrixRow'}} - default: - $ref: '#/components/responses/InspectionError' - - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/contains: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotContains - summary: Look up exact binary Sample membership - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - in: query - name: sample - required: true - description: Canonical padded base64 of a nonempty Sample ID, at most 16384 decoded bytes. - schema: {type: string} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/Contains'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/inspection: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotInspection - summary: Read explicit metadata preparation status - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - responses: - '200': - description: Bounded metadata result; counts and byte lengths are decimal strings. - content: - application/json: - schema: {$ref: '#/components/schemas/InspectionPreparation'} - default: - $ref: '#/components/responses/InspectionError' - post: - operationId: prepareSnapshotInspection - summary: Rebuild complete legacy inspection metadata - description: Explicit mutation with a thirty-second deadline, a 9 MiB source-record limit and an 8 MiB embedded-Manifest limit. Writes complete index rows and installs the summary last without durable preparation progress. Retry rebuilds from the source or reuses an existing summary without writes. Errors propagate without confirmation reads. Does not change publication heads or acknowledge reads. - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - responses: - '200': - description: Complete metadata readiness; revision and rows_written are zero compatibility fields. - content: - application/json: - schema: {$ref: '#/components/schemas/InspectionPreparation'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-head: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - put: - operationId: updateSplitHead - summary: Conditionally update currentSplit from a complete resource - description: Requires split-head-update. Caller supplies the target and both explicit nullable preconditions, without a draft or attempt ID. A conflict carries observed_head for caller review; the server does not rebase or query again to confirm a CAS outcome. - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/UpdateSplitHead'} - responses: - '200': - description: Current effect confirmed, without a historical response receipt. - content: - application/json: - schema: {$ref: '#/components/schemas/SplitHead'} - default: - $ref: '#/components/responses/InspectionError' - get: - operationId: splitHead - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: snapshot - required: true - schema: {type: string} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitHead'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-revisions: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: splitHistory - description: Lists complete revisions, including results that have not updated currentSplit. Snapshot read admission applies. - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: snapshot - required: true - schema: {type: string} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitRevisionPage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-revisions/{revision}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - delete: - operationId: deleteSplitRevision - summary: Mark one historical complete Split revision for delayed metadata reclamation - description: >- - Requires split-revision-delete. Accepts only the two resource IDs, with no - query parameters or request body. The current Split and the Snapshot's - initial Split are protected. A successful mark rejects new reads and - persists one fixed reclaim_after deadline using dataset.reclaim_grace_seconds (default 1200). Repeating the - same DELETE preserves that deadline. Background cleanup seals membership - metadata; it does not delete sample payloads. The owner Snapshot may - reclaim this metadata after its own read admission closes and its own - configured (default twenty-minute) grace expires. Complete revisions otherwise follow their - owner Snapshot's lifetime, without an independent automatic TTL. No - request, response or progress session is persisted. Unknown outcomes - return directly; an explicit client retry repeats the same two-ID DELETE. - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - in: path - name: revision - required: true - schema: {$ref: '#/components/schemas/OpaqueId'} - responses: - '200': - description: Fixed deletion mark or terminal metadata reclamation status - content: - application/json: - schema: {$ref: '#/components/schemas/SplitRevisionDeletion'} - '404': - description: The scoped revision was not found - $ref: '#/components/responses/InspectionError' - '409': - description: Current or initial revision is protected, or the resource changed concurrently - $ref: '#/components/responses/InspectionError' - default: - $ref: '#/components/responses/InspectionError' - put: - operationId: prepareSplitRevision - summary: Prepare complete immutable membership from caller-owned inputs - description: >- - Requires split-preparation. The caller retains revision ID, policy and - complete definitions. No operation, edit or evaluation cursor is accepted. - Computation is bounded to 30 seconds and 1 MiB input/final bindings. - Retry the same ID and input after an unknown outcome; incomplete work is - recomputed, while an identical complete result is reused. Different input - at a complete ID conflicts. Frozen sources retain their supplied revision. - This operation does not update currentSplit; capture its intended base - before computation and update split-head separately. No draft fallback. - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - in: path - name: revision - required: true - schema: {$ref: '#/components/schemas/OpaqueId'} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/PrepareSplitRevision'} - responses: - '200': - description: Complete immutable resource, readable before head publication - content: - application/json: - schema: {$ref: '#/components/schemas/SplitRevision'} - '409': - description: Resource ID already has different input, or validation conflicts - $ref: '#/components/responses/InspectionError' - '413': - description: Request body exceeds 1 MiB - $ref: '#/components/responses/InspectionError' - default: - $ref: '#/components/responses/InspectionError' - get: - operationId: splitRevision - description: Reads a complete immutable revision by ID without requiring currentSplit publication. Snapshot read admission applies. Deletion-marked revisions are unavailable, normally with 404; existing bounded-work errors may take precedence. - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: snapshot - required: true - schema: {type: string} - - in: path - name: revision - required: true - schema: {type: string} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitRevision'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/split-revisions/{revision}/splits/{name}/members: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: splitMembers - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: snapshot - required: true - schema: {type: string} - - in: path - name: revision - required: true - schema: {type: string} - - in: path - name: name - required: true - schema: {type: string} - - {$ref: '#/components/parameters/after'} - - {$ref: '#/components/parameters/limit'} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SamplePage'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/split-drafts/{operation}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - deprecated: true - description: Read-only observation of an existing legacy draft; never resumes work. - operationId: splitDraft - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: operation - required: true - schema: {type: string} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitDraft'} - default: - $ref: '#/components/responses/InspectionError' - delete: - deprecated: true - description: Abandon an existing legacy draft for cleanup. - operationId: abandonSplitDraft - parameters: - - in: path - name: dataset - required: true - schema: {type: string} - - in: path - name: operation - required: true - schema: {type: string} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/DraftRevision'} - responses: - '200': - description: Revision-bound operation result - content: - application/json: - schema: {$ref: '#/components/schemas/SplitDraft'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/reads/bind: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - operationId: bindRead - description: Resolve immutable Snapshot and Split versions without creating read state. - parameters: - - {$ref: '#/components/parameters/dataset'} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/ReadPlanV2'} - responses: - '200': - description: Client-owned continuation; no TTL, remote acknowledgement or close. - content: - application/json: - schema: {$ref: '#/components/schemas/ReadBinding'} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/reads/next: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - operationId: nextReadBatch - description: Replay this continuation; save the returned next cursor after processing. - parameters: - - {$ref: '#/components/parameters/dataset'} - requestBody: - required: true - content: - application/json: - schema: {$ref: '#/components/schemas/ReadContinuation'} - responses: - '200': - description: Bounded batch; null next means end of selection. - content: - application/vnd.crowdb.dataset-batch.v1: - schema: {type: string, format: binary} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/preview: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: snapshotFieldPreview - summary: Read a bounded binary field prefix for text or hex presentation - description: >- - Never acknowledges an application read. Response bytes are capped at 64 KiB; - uploaded parts are independently integrity-checked with a 4 MiB per-part - allocation cap and five-second payload deadline. Full-field MD5 identifies - the entire value, not the truncated prefix. External legacy locators without - supported payload storage return an explicit error, never an arbitrary URL. - parameters: - - {$ref: '#/components/parameters/dataset'} - - {$ref: '#/components/parameters/snapshot'} - - in: query - name: sample - required: true - schema: {type: string, description: Canonical padded base64 Sample ID} - - in: query - name: field - required: true - schema: {type: string} - - in: query - name: max_bytes - schema: {type: integer, minimum: 1, maximum: 65536, default: 65536} - responses: - '200': - description: Binary prefix or explicit missing/tombstone state - content: - application/json: - schema: - type: object - required: [state, length, md5, data, truncated] - properties: - state: {enum: [value, missing, tombstone]} - length: {type: [string, 'null']} - md5: {type: [string, 'null']} - data: {type: [string, 'null'], description: Canonical padded base64; empty string is an empty value} - truncated: {type: boolean} - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/deletion/{id}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: datasetDeletionStatus - summary: Reconcile a deletion for the exact Dataset identity - parameters: - - $ref: '#/components/parameters/dataset' - - name: id - in: path - required: true - schema: {type: string, pattern: '^[0-9a-f]{32}$'} - responses: - '200': - description: Null before deletion, otherwise the durable receipt; identity mismatch is 409. - content: - application/json: - schema: - type: object - required: [deletion] - properties: - deletion: - anyOf: - - $ref: '#/components/schemas/DatasetDeletion' - - type: 'null' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - delete: - operationId: deleteDatasetV2 - summary: Mark an empty Dataset deleted - description: >- - Requires dataset-delete capability and exact Dataset ID. Nonempty (including - pending creation/cleanup) returns 409 dataset_not_empty. Successful marking - immediately hides the Dataset. Cleanup follows dataset.reclaim_grace_seconds - (default 1200). The name remains reserved until cleanup completes. Retries - preserve the deadline; stale IDs cannot delete a replacement Dataset. - parameters: - - $ref: '#/components/parameters/dataset' - requestBody: - required: true - content: - application/json: - schema: - type: object - additionalProperties: false - required: [id] - properties: - id: {type: string, pattern: '^[0-9a-f]{32}$'} - responses: - '200': - description: Durable deletion receipt, including idempotent retries. - content: - application/json: - schema: - $ref: '#/components/schemas/DatasetDeletion' - default: - $ref: '#/components/responses/InspectionError' - put: - operationId: createDatasetV2 - summary: Create a Dataset - parameters: - - $ref: '#/components/parameters/dataset' - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/DatasetInfo' - default: - $ref: '#/components/responses/InspectionError' - get: - operationId: getDatasetV2 - summary: Read Dataset metadata - parameters: - - $ref: '#/components/parameters/dataset' - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/DatasetInfo' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/names/stable: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - put: - operationId: setStableSnapshot - summary: Set stable without changing latest - parameters: - - $ref: '#/components/parameters/dataset' - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/DatasetInfo' - default: - $ref: '#/components/responses/InspectionError' - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/StableSnapshot' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/retain: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - operationId: retainSnapshotV2 - summary: Retain Snapshot retention - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: snapshot - required: true - schema: &id001 - $ref: '#/components/schemas/OpaqueId' - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/RetainedSnapshot' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/release: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - operationId: releaseSnapshotV2 - summary: Release Snapshot retention - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: snapshot - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/RetainedSnapshot' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/snapshots/{snapshot}/reclaim: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: observeSnapshotLifecycleV2 - summary: Observe bounded protection and retirement facts without mutation - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: snapshot - required: true - schema: *id001 - responses: - '200': - description: Observed facts, not permission to reclaim; parent reachability is not evaluated. - content: - application/json: - schema: - $ref: '#/components/schemas/SnapshotLifecycle' - default: - $ref: '#/components/responses/InspectionError' - post: - operationId: reclaimSnapshotV2 - summary: Mark Snapshot deletion and observe background retirement - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: snapshot - required: true - schema: *id001 - responses: - '200': - description: Durable deletion status. Background workers advance cleanup after the grace deadline. - content: - application/json: - schema: - $ref: '#/components/schemas/SnapshotReclaim' - default: - $ref: '#/components/responses/InspectionError' - description: Latest, stable, retained or live-parent Snapshots are protected. - This empty-body action only marks deletion and observes durable status; repeated calls - do not drive cleanup. Nonempty bodies, including cleanup cursor or completion inputs, - are rejected. Background workers retire owners before metadata. Physical capacity - recovery remains a separate ChunkDB responsibility. - /namespaces/dataset-ns/datasets/{dataset}/commits/{attempt}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: commitStatusV2 - summary: Read a legacy publication receipt - description: Returns existing legacy receipts without backfill. New Snapshot and Split publication creates no historical receipts; retry the same conditional request to confirm its current effect. - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: attempt - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/CommitStatus' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/split-drafts/{operation}/cleanup: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - deprecated: true - operationId: cleanupSplitDraft - summary: Retired Split cleanup action - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: operation - required: true - schema: *id001 - description: Returns unsupported after authentication and route validation, before storage. Background workers discover historical Split draft and work facts using local cursors. Readable Snapshot owners retain their deletion grace; permanently sealed incomplete roots do not require a reader deadline. - responses: - '501': - $ref: '#/components/responses/InspectionError' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/uploads: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - operationId: createUploadV2 - summary: Create a Dataset-scoped multipart upload - parameters: - - $ref: '#/components/parameters/dataset' - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/UploadStatus' - default: - $ref: '#/components/responses/InspectionError' - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CreateUpload' - /namespaces/dataset-ns/datasets/{dataset}/uploads/{upload}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - operationId: getUploadV2 - summary: Inspect upload progress - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: upload - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/UploadStatus' - default: - $ref: '#/components/responses/InspectionError' - delete: - operationId: abortUploadV2 - summary: Abort an unowned upload - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: upload - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/UploadStatus' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/uploads/{upload}/complete: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - operationId: completeUploadV2 - summary: Verify and complete upload parts - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: upload - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/UploadStatus' - default: - $ref: '#/components/responses/InspectionError' - /namespaces/dataset-ns/datasets/{dataset}/uploads/{upload}/parts/{index}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - put: - operationId: uploadPartV2 - summary: Write one bounded upload part - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: upload - required: true - schema: *id001 - - in: path - name: index - required: true - schema: - $ref: '#/components/schemas/Decimal' - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/UploadPart' - default: - $ref: '#/components/responses/InspectionError' - requestBody: - required: true - content: - application/octet-stream: - schema: - type: string - format: binary - maxLength: 4194304 - /namespaces/dataset-ns/datasets/{dataset}/compositions/{operation}: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - get: - deprecated: true - operationId: getComposition - summary: Observe an existing legacy composition without resuming it - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: operation - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/CompositionDraft' - default: - $ref: '#/components/responses/InspectionError' - delete: - deprecated: true - operationId: abandonComposition - summary: Abandon an unpublished composition - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: operation - required: true - schema: *id001 - responses: - '200': - description: Successful bounded operation. Integer counts are exact decimal strings. - content: - application/json: - schema: - $ref: '#/components/schemas/CompositionDraft' - default: - $ref: '#/components/responses/InspectionError' - description: Background workers clean existing draft resources after abandonment; no client cleanup action is required. Complete resource owners remain independent of the draft. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/DraftRevision' - /namespaces/dataset-ns/datasets/{dataset}/compositions/{operation}/cleanup: - parameters: - - {$ref: '#/components/parameters/datasetInstance'} - post: - deprecated: true - operationId: cleanupComposition - summary: Retired Composition cleanup action - parameters: - - $ref: '#/components/parameters/dataset' - - in: path - name: operation - required: true - schema: *id001 - responses: - '501': - $ref: '#/components/responses/InspectionError' - default: - $ref: '#/components/responses/InspectionError' - description: Returns unsupported after authentication and route validation, before storage. Background workers discover historical draft facts and retire them using local cursors; no client cleanup state or completion confirmation is accepted. -components: - parameters: - datasetInstance: - in: header - name: X-Crowdb-Dataset-Id - description: Optional active Dataset instance; omit after marking when retrying deletion receipts/status. - schema: {type: string, pattern: '^(?!0{32}$)[0-9a-f]{32}$'} - dataset: - in: path - name: dataset - required: true - schema: {type: string} - snapshot: - in: path - name: snapshot - required: true - schema: {type: string, pattern: '^[0-9a-f]{32}$'} - after: - in: query - name: after - description: Opaque canonical base64 continuation; preserve query and immutable scope. Empty pages may continue. - schema: {type: string, maxLength: 43692} - limit: - in: query - name: limit - schema: {type: integer, minimum: 1, maximum: 100, default: 20} - responses: - InspectionError: - description: >- - 400 invalid or duplicate query; 401 unauthorized; 404 unavailable; - 409 inspection_not_prepared; 413 read_limit; 422 inspection_unsupported; - 429 capacity; 499 cancelled; 500 integrity; 503 unavailable; 504 timeout. - Unknown query keys are rejected. No error triggers unbounded fallback. - content: - application/json: - schema: - type: object - required: [code, message, request_id, retryable] - properties: - code: {type: string} - message: {type: string} - request_id: {type: string} - retryable: {type: boolean} - observed_head: - oneOf: - - {$ref: '#/components/schemas/SplitHead'} - - {$ref: '#/components/schemas/Head'} - securitySchemes: - bearerAuth: - type: http - scheme: bearer - schemas: - Head: - type: object - required: [latest, stable, generation] - properties: - latest: - type: [string, 'null'] - pattern: '^[0-9a-f]{32}$' - stable: - type: [string, 'null'] - pattern: '^[0-9a-f]{32}$' - generation: - type: [string, 'null'] - pattern: '^[0-9a-f]{64}$' - UpdateSnapshotHead: - type: object - additionalProperties: false - required: [snapshot, expected_head] - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - expected_head: - type: [string, 'null'] - pattern: '^[0-9a-f]{64}$' - Capabilities: - type: object - required: [api_version, capabilities] - properties: - api_version: - type: integer - const: 2 - capabilities: - type: array - uniqueItems: true - items: {type: string} - description: Unknown names may be ignored; required names must be present. - DatasetDeletion: - type: object - required: [id, reclaim_after, complete] - properties: - id: {type: string, pattern: '^[0-9a-f]{32}$'} - reclaim_after: {type: string, pattern: '^[0-9]+$'} - complete: {type: boolean} - DatasetInfo: - type: object - required: [id, namespace, name, latest, stable] - properties: - id: {type: string} - namespace: {type: string} - name: {type: string} - latest: {type: [string, 'null']} - stable: {type: [string, 'null']} - SnapshotSummary: - type: object - required: [id, parent, schema_version, field_count, sample_count, manifest_sha256, inspection, reason] - properties: - id: {type: string} - parent: {type: [string, 'null']} - schema_version: {type: integer} - field_count: {type: integer} - sample_count: {type: string, pattern: '^[0-9]+$'} - manifest_sha256: {type: string, pattern: '^[0-9a-f]{64}$'} - inspection: {enum: [ready, unsupported]} - reason: {type: [string, 'null']} - FieldDefinition: - type: object - required: [name, required] - properties: - name: {type: string} - required: {type: boolean} - FieldMetadata: - type: object - required: [name, storage, length, md5, source_snapshot] - properties: - name: {type: string} - storage: {enum: [inline, payload, tombstone]} - length: {type: string, pattern: '^[0-9]+$'} - md5: {type: [string, 'null']} - source_snapshot: {type: string} - MatrixRow: - type: object - required: [id, fields] - properties: - id: {type: string, description: Canonical padded base64 Sample ID} - fields: - type: array - description: Requested field order; null means missing, distinct from tombstone. - items: - anyOf: - - {$ref: '#/components/schemas/FieldMetadata'} - - {type: 'null'} - Contains: - type: object - required: [contains] - properties: - contains: {type: boolean} - InspectionPreparation: - type: object - required: [snapshot, revision, rows_written, state, reason] - properties: - snapshot: {type: string} - revision: {type: string, pattern: '^[0-9]+$', description: Zero for complete rebuilds; historical GET progress can retain a nonzero value.} - rows_written: {type: string, pattern: '^[0-9]+$', description: Zero for complete rebuilds; historical GET progress can retain a nonzero value.} - state: {enum: [preparing, ready, unsupported], description: Preparing is observable only for historical progress through GET. New POST results describe a complete index.} - reason: {type: [string, 'null']} - DatasetPage: - type: object - required: [entries, after] - properties: - after: {type: [string, 'null']} - entries: - type: array - maxItems: 100 - items: {$ref: '#/components/schemas/DatasetInfo'} - SnapshotPage: - type: object - required: [entries, after] - properties: - after: {type: [string, 'null']} - entries: - type: array - maxItems: 100 - items: {$ref: '#/components/schemas/SnapshotSummary'} - SchemaPage: - type: object - required: [entries, after] - properties: - after: {type: [string, 'null']} - entries: - type: array - maxItems: 100 - items: {$ref: '#/components/schemas/FieldDefinition'} - SamplePage: - type: object - required: [entries, after] - properties: - after: {type: [string, 'null']} - entries: - type: array - maxItems: 100 - items: {type: string, description: Canonical padded base64 Sample ID} - MatrixPage: - type: object - required: [entries, after] - properties: - after: {type: [string, 'null']} - entries: - type: array - maxItems: 100 - items: {$ref: '#/components/schemas/MatrixRow'} - Decimal: - type: string - pattern: '^(0|[1-9][0-9]*)$' - description: Exact unsigned 64-bit integer encoded as a canonical decimal string. - OpaqueId: - type: string - pattern: '^[0-9a-f]{32}$' - description: Nonzero opaque identity. - NullableRevision: - type: [string, 'null'] - pattern: '^[0-9a-f]{32}$' - NullableGeneration: - type: [string, 'null'] - pattern: '^[0-9a-f]{64}$' - Predicate: - oneOf: - - type: object - additionalProperties: false - required: [kind] - properties: - kind: {const: all} - - type: object - additionalProperties: false - required: [kind, ids] - properties: - kind: {const: explicit} - ids: {type: array, maxItems: 10000, items: {type: string}} - - type: object - additionalProperties: false - required: [kind, value] - properties: - kind: {enum: [prefix, keyword]} - value: {type: string, description: Nonempty canonical padded base64} - - type: object - additionalProperties: false - required: [kind, start, end] - properties: - kind: {const: range} - start: {type: string} - end: {type: string} - - type: object - additionalProperties: false - required: [kind, field, value] - properties: - kind: {const: field_match} - field: {type: string} - value: {type: string, description: Canonical padded base64; empty is an exact empty inline value} - - type: object - additionalProperties: false - required: [kind, items] - properties: - kind: {enum: [and, or]} - items: {type: array, minItems: 1, items: {$ref: '#/components/schemas/Predicate'}} - - type: object - additionalProperties: false - required: [kind, item] - properties: - kind: {const: not} - item: {$ref: '#/components/schemas/Predicate'} - description: Maximum depth 32 and 1024 nodes. IDs are binary base64, never text normalization. - SplitPolicy: - type: object - additionalProperties: false - required: [overlap, coverage] - properties: - overlap: {enum: [disjoint, allow_overlap]} - coverage: {enum: [require_complete, allow_unassigned]} - SplitSource: - type: object - additionalProperties: false - required: [snapshot, revision, name] - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - revision: {$ref: '#/components/schemas/OpaqueId'} - name: {type: string, minLength: 1, description: Exact UTF-8 name, at most 255 bytes} - SplitOrigin: - oneOf: - - type: object - additionalProperties: false - required: [kind] - properties: - kind: {const: evaluated} - - type: object - additionalProperties: false - required: [kind, source] - properties: - kind: {const: inherited} - source: {$ref: '#/components/schemas/SplitSource'} - SplitDefinition: - oneOf: - - type: object - additionalProperties: false - required: [kind, predicate] - properties: - kind: {const: selection} - predicate: {$ref: '#/components/schemas/Predicate'} - - type: object - additionalProperties: false - required: [kind, seed, weights, index] - properties: - kind: {const: weighted_partition} - seed: {$ref: '#/components/schemas/Decimal'} - index: {type: integer, minimum: 0} - weights: - type: array - minItems: 1 - items: - type: object - additionalProperties: false - required: [name, weight] - properties: - name: {type: string} - weight: {type: integer, minimum: 1, maximum: 4294967295} - description: Uses sample-id-sha256-v1 with ordered weights and checked u32 sum. - DraftSplit: - type: object - additionalProperties: false - required: [name, definition, origin] - properties: - name: {type: string} - definition: {$ref: '#/components/schemas/SplitDefinition'} - origin: {$ref: '#/components/schemas/SplitOrigin'} - PrepareSplitRevision: - type: object - additionalProperties: false - required: [policy, splits] - properties: - policy: {$ref: '#/components/schemas/SplitPolicy'} - splits: {type: array, items: {$ref: '#/components/schemas/DraftSplit'}} - DraftRevision: - type: object - additionalProperties: false - required: [expected_revision] - properties: - expected_revision: {$ref: '#/components/schemas/Decimal'} - UpdateSplitHead: - type: object - additionalProperties: false - required: [revision, expected_revision, expected_head] - properties: - revision: {$ref: '#/components/schemas/OpaqueId'} - expected_revision: {$ref: '#/components/schemas/NullableRevision'} - expected_head: {$ref: '#/components/schemas/NullableGeneration'} - CommitTarget: - oneOf: - - type: object - required: [kind, snapshot] - properties: - kind: {const: snapshot} - snapshot: {$ref: '#/components/schemas/OpaqueId'} - - type: object - required: [kind, snapshot, revision] - properties: - kind: {const: split_revision} - snapshot: {$ref: '#/components/schemas/OpaqueId'} - revision: {$ref: '#/components/schemas/OpaqueId'} - DraftProgress: - type: object - required: [state] - properties: - state: {enum: [editing, evaluating, evaluated, prepared, committing, published, abandoned]} - input_revision: {$ref: '#/components/schemas/Decimal'} - next_page: {$ref: '#/components/schemas/Decimal'} - binding: {type: string} - target: {$ref: '#/components/schemas/CommitTarget'} - attempt: {$ref: '#/components/schemas/OpaqueId'} - SplitDraft: - type: object - required: [operation, snapshot, expected_revision, revision, policy, splits, progress] - properties: - operation: {$ref: '#/components/schemas/OpaqueId'} - snapshot: {$ref: '#/components/schemas/OpaqueId'} - expected_revision: {$ref: '#/components/schemas/NullableRevision'} - revision: {$ref: '#/components/schemas/Decimal'} - policy: {$ref: '#/components/schemas/SplitPolicy'} - splits: {type: array, items: {$ref: '#/components/schemas/DraftSplit'}} - progress: {$ref: '#/components/schemas/DraftProgress'} - SplitRevision: - type: object - required: [snapshot, revision, operation, evaluator_version, policy, splits] - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - revision: {$ref: '#/components/schemas/OpaqueId'} - operation: {$ref: '#/components/schemas/OpaqueId'} - evaluator_version: {const: 1} - policy: {$ref: '#/components/schemas/SplitPolicy'} - splits: - type: array - items: - type: object - required: [name, definition, origin, count] - properties: - name: {type: string} - definition: {$ref: '#/components/schemas/SplitDefinition'} - origin: {$ref: '#/components/schemas/SplitOrigin'} - count: {$ref: '#/components/schemas/Decimal'} - SplitHead: - type: object - required: [snapshot, revision, generation] - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - revision: {$ref: '#/components/schemas/NullableRevision'} - generation: {$ref: '#/components/schemas/NullableGeneration'} - SplitRevisionDeletion: - type: object - additionalProperties: false - required: [snapshot, revision, reclaim_after, complete] - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - revision: {$ref: '#/components/schemas/OpaqueId'} - reclaim_after: - description: Fixed Unix-seconds cleanup deadline while pending; null after terminal reclamation. - anyOf: - - {$ref: '#/components/schemas/Decimal'} - - {type: 'null'} - complete: {type: boolean} - SplitRevisionPage: - type: object - required: [entries, after] - properties: - after: {type: [string, 'null']} - entries: - type: array - maxItems: 100 - items: - type: object - required: [snapshot, revision, operation, split_count] - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - revision: {$ref: '#/components/schemas/OpaqueId'} - operation: {$ref: '#/components/schemas/OpaqueId'} - split_count: {type: integer} - SplitPreview: - type: object - required: [input_revision, next_page, complete, member_counts, validation] - properties: - input_revision: {$ref: '#/components/schemas/Decimal'} - next_page: {$ref: '#/components/schemas/Decimal'} - complete: {type: boolean} - member_counts: {type: array, items: {$ref: '#/components/schemas/Decimal'}} - validation: - type: object - required: [scanned, assigned, unassigned, overlapping, unassigned_examples, overlap_examples] - properties: - scanned: {$ref: '#/components/schemas/Decimal'} - assigned: {$ref: '#/components/schemas/Decimal'} - unassigned: {$ref: '#/components/schemas/Decimal'} - overlapping: {$ref: '#/components/schemas/Decimal'} - unassigned_examples: {type: array, items: {type: string}} - overlap_examples: {type: array, items: {type: string}} - CommitReceipt: - type: object - required: [attempt, target, expected_head, committed_head] - properties: - attempt: {$ref: '#/components/schemas/OpaqueId'} - target: {$ref: '#/components/schemas/CommitTarget'} - expected_head: {$ref: '#/components/schemas/NullableGeneration'} - committed_head: {type: string, pattern: '^[0-9a-f]{64}$'} - ReadPlanV2: - type: object - additionalProperties: false - required: [snapshot, selection, fields, batch_size, partition, shuffle, split] - description: >- - Snapshot latest/stable and null Split revision resolve into the returned - continuation. Selection and Split filtering precede shuffle and worker - partitioning. There is no persisted read plan, preparation or progress. - properties: - snapshot: {type: string, description: Concrete ID or latest/stable} - selection: {$ref: '#/components/schemas/Predicate'} - fields: {type: array, maxItems: 256, uniqueItems: true, items: {type: string}} - batch_size: {type: integer, minimum: 1, maximum: 10000} - partition: - type: object - additionalProperties: false - required: [worker, workers, rank, world_size] - properties: - worker: {type: integer, minimum: 0, maximum: 65535} - workers: {type: integer, minimum: 1, maximum: 65535} - rank: {type: integer, minimum: 0, maximum: 65535} - world_size: {type: integer, minimum: 1, maximum: 65535} - shuffle: - oneOf: - - {type: 'null'} - - type: object - additionalProperties: false - required: [seed, group_size] - properties: - seed: {$ref: '#/components/schemas/Decimal'} - group_size: {type: integer, minimum: 1, maximum: 100000} - split: - oneOf: - - {type: 'null'} - - type: object - additionalProperties: false - required: [revision, name] - properties: - revision: {$ref: '#/components/schemas/NullableRevision'} - name: {type: string} - ReadBinding: - type: object - required: [snapshot, split, cursor] - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - split: - oneOf: - - {type: 'null'} - - {$ref: '#/components/schemas/SplitSource'} - cursor: {type: string, maxLength: 349530} - ReadContinuation: - type: object - additionalProperties: false - required: [cursor] - properties: - cursor: {type: string, maxLength: 349530} - PrepareSnapshot: - type: object - additionalProperties: false - required: [parent, source, schema, samples, splits, policy] - properties: - parent: {$ref: '#/components/schemas/NullableRevision'} - source: {$ref: '#/components/schemas/CompositionSource'} - schema: - anyOf: - - {$ref: '#/components/schemas/CompositionSchema'} - - {type: 'null'} - samples: {type: array, items: {$ref: '#/components/schemas/SnapshotChange'}} - splits: {type: array, maxItems: 64, items: {$ref: '#/components/schemas/DraftSplit'}} - policy: {$ref: '#/components/schemas/SplitPolicy'} - SnapshotValue: - oneOf: - - type: object - additionalProperties: false - required: [kind, data] - properties: - kind: {const: inline} - data: {type: string, description: Canonical padded base64 of at most 65536 decoded bytes} - - type: object - additionalProperties: false - required: [kind, id, length, md5] - properties: - kind: {const: upload} - id: {$ref: '#/components/schemas/OpaqueId'} - length: {$ref: '#/components/schemas/Decimal'} - md5: {type: string, pattern: '^[0-9a-f]{32}$'} - - type: object - additionalProperties: false - required: [kind] - properties: - kind: {const: tombstone} - SnapshotField: - type: object - additionalProperties: false - required: [name, value] - properties: - name: {type: string, minLength: 1, description: At most 1024 UTF-8 bytes} - value: {$ref: '#/components/schemas/SnapshotValue'} - SnapshotChange: - oneOf: - - type: object - additionalProperties: false - required: [kind, id] - properties: - kind: {const: remove} - id: {type: string} - - type: object - additionalProperties: false - required: [kind, id, fields] - properties: - kind: {const: upsert} - id: {type: string} - fields: {type: array, maxItems: 256, items: {$ref: '#/components/schemas/SnapshotField'}} - description: Canonical base64 Sample IDs decode to 1 through 16384 bytes. Each ID occurs once; an upsert preserves unspecified effective parent fields. - PreparedSnapshot: - type: object - required: [snapshot, initial_split, sample_count] - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - initial_split: {$ref: '#/components/schemas/NullableRevision'} - sample_count: {$ref: '#/components/schemas/Decimal'} - CompositionSchema: - type: object - additionalProperties: false - required: - - version - - fields - properties: - version: - type: integer - minimum: 0 - maximum: 4294967295 - fields: - type: array - items: - $ref: '#/components/schemas/FieldDefinition' - maxItems: 256 - CompositionSource: - oneOf: - - type: object - additionalProperties: false - required: - - kind - properties: - kind: - const: all - - type: object - additionalProperties: false - required: - - kind - - predicate - properties: - kind: - const: selection - predicate: - $ref: '#/components/schemas/Predicate' - - type: object - additionalProperties: false - required: - - kind - - source - properties: - kind: - const: split - source: - $ref: '#/components/schemas/SplitSource' - selection: - anyOf: - - $ref: '#/components/schemas/Predicate' - - type: 'null' - CompositionUploadStatus: - type: object - additionalProperties: false - required: - - publication_ready - - staged_uploads - - candidate_uploads - properties: - publication_ready: {type: boolean} - staged_uploads: {$ref: '#/components/schemas/Decimal'} - candidate_uploads: {$ref: '#/components/schemas/Decimal'} - CompositionDraft: - type: object - additionalProperties: false - required: - - operation - - revision - - parent - - source - - parent_split_revision - - schema - - splits - - policy - - progress - - source_lease_deadline - - preparation - - uploads - properties: - operation: {$ref: '#/components/schemas/OpaqueId'} - revision: {$ref: '#/components/schemas/Decimal'} - parent: - $ref: '#/components/schemas/NullableRevision' - source: - $ref: '#/components/schemas/CompositionSource' - parent_split_revision: - $ref: '#/components/schemas/NullableRevision' - schema: - $ref: '#/components/schemas/CompositionSchema' - splits: {type: array, maxItems: 64, items: {$ref: '#/components/schemas/DraftSplit'}} - policy: - $ref: '#/components/schemas/SplitPolicy' - progress: - $ref: '#/components/schemas/DraftProgress' - source_lease_deadline: - <<: {$ref: '#/components/schemas/Decimal'} - description: Compatibility field; new drafts and edits use zero and hold no source lease. - preparation: - anyOf: - - type: object - additionalProperties: false - required: - - snapshot - - split_preview - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - split_preview: - anyOf: - - $ref: '#/components/schemas/SplitPreview' - - type: 'null' - - type: 'null' - uploads: - $ref: '#/components/schemas/CompositionUploadStatus' - CreateUpload: - type: object - additionalProperties: false - required: - - id - - length - - part_size - - md5 - properties: - id: {$ref: '#/components/schemas/OpaqueId'} - length: {$ref: '#/components/schemas/Decimal'} - part_size: - type: integer - minimum: 1 - maximum: 4194304 - md5: &composition006 - type: string - pattern: ^[0-9a-f]{32}$ - UploadPart: - type: object - additionalProperties: false - required: - - index - - length - - md5 - properties: - index: - type: integer - minimum: 0 - length: {$ref: '#/components/schemas/Decimal'} - md5: *composition006 - UploadStatus: - type: object - additionalProperties: false - required: - - id - - length - - part_size - - md5 - - state - - expires_at - - cleanup_complete - - parts - properties: - id: {$ref: '#/components/schemas/OpaqueId'} - length: {$ref: '#/components/schemas/Decimal'} - part_size: - type: integer - md5: *composition006 - state: - enum: - - open - - complete - - claimed - - aborted - expires_at: {$ref: '#/components/schemas/Decimal'} - cleanup_complete: {type: boolean} - parts: - type: array - items: - $ref: '#/components/schemas/UploadPart' - maxItems: 10000 - StableSnapshot: - type: object - additionalProperties: false - required: - - snapshot - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - RetainedSnapshot: - type: object - additionalProperties: false - required: - - snapshot - - retained - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - retained: {type: boolean} - SnapshotLifecycle: - type: object - required: [snapshot, operation, latest, stable, retained, active_read_lease, state, physical_cleanup_deferred, parent_protection] - properties: - snapshot: *id001 - operation: - type: [string, 'null'] - latest: - type: boolean - stable: - type: boolean - retained: - type: boolean - active_read_lease: - type: boolean - const: false - description: Compatibility field; readers do not register persistent activity. - reclaim_after: - type: [integer, 'null'] - minimum: 0 - description: Durable Unix-seconds deadline for destructive cleanup. - state: - type: string - enum: [not_started, pending, completed] - physical_cleanup_deferred: - type: boolean - parent_protection: - type: string - const: not_evaluated - SnapshotReclaim: - type: object - additionalProperties: false - required: - - snapshot - - phase - - complete - - physical_cleanup_deferred - properties: - snapshot: {$ref: '#/components/schemas/OpaqueId'} - phase: - enum: - - grace_period - - upload_owners - - split_drafts - - index_fences - - metadata - - complete - complete: {type: boolean} - reclaim_after: - type: integer - minimum: 0 - physical_cleanup_deferred: - const: true - CommitStatus: - type: object - additionalProperties: false - required: - - receipt - properties: - receipt: - anyOf: - - $ref: '#/components/schemas/CommitReceipt' - - type: 'null' diff --git a/doc/design/access-server/dataset/spec-crowdb-dataset.md b/doc/design/access-server/dataset/spec-crowdb-dataset.md deleted file mode 100644 index e0a5a7fcf..000000000 --- a/doc/design/access-server/dataset/spec-crowdb-dataset.md +++ /dev/null @@ -1,537 +0,0 @@ - - - -# CROWDB - Design: Dataset Specification - -This document is the normative Dataset contract. It defines the objects, -identities, state transitions, observable behavior, and compatibility rules -shared by Dataset authorities, HTTP clients, native clients, and SDKs. -The implementation design is described in -[Design: Dataset](design-crowdb-access-dataset.md); when the two documents -appear to disagree, this specification is authoritative. - -## Table of contents - -1. [Scope and terms](#1-scope-and-terms) -2. [Identity and names](#2-identity-and-names) -3. [Dataset records](#3-dataset-records) -4. [Operations](#4-operations) -5. [Manifest and field values](#5-manifest-and-field-values) -6. [Snapshot publication](#6-snapshot-publication) -7. [Selection, plans, and cursors](#7-selection-plans-and-cursors) -8. [Read behavior](#8-read-behavior) -9. [Retention and reclamation](#9-retention-and-reclamation) -10. [HTTP and native surfaces](#10-http-and-native-surfaces) -11. [Errors and compatibility](#11-errors-and-compatibility) -12. [Normative invariants](#12-normative-invariants) - -## 1. Scope and terms - -Dataset is a logical, sample-oriented collection. It stores metadata and -references to payload bytes; it does not define physical chunk layout, -replication, erasure coding, or framework tensor types. - -The following terms have one meaning throughout this specification: - -- **Dataset** is the named authority for immutable snapshots and their - metadata. -- **Snapshot** is an immutable view of a Dataset. It is identified by an - opaque `SnapshotId` and may name one parent Snapshot. -- **Manifest** is the complete, validated description of one Snapshot's - schema, sample membership, field values, partitions, and integrity metadata. -- **Schema** defines the fields, requiredness, and compatibility rules for a - Manifest. -- **Sample** is an opaque `sample_id` and its field map in one Snapshot. -- **Field** is a named value of one Sample. A missing field and a tombstone are - distinct: missing means no value is supplied by the visible ancestry; - tombstone explicitly hides an inherited value. -- **Locator** identifies payload bytes. Inline values are stored in metadata; - chunk locators are opaque strings interpreted only by the chunk subsystem. -- **Split revision** is an immutable complete set of named subsets, definitions, - membership and policies owned by one Snapshot. currentSplit is its mutable pointer. -- **Selection** identifies samples from one Snapshot. -- **Read Plan** binds a Selection, ordering, projection, batching, and optional - shuffle to one immutable Snapshot. -- **Cursor** records progress for one Read Plan and is acknowledged only after - the corresponding batch is accepted by the caller. -- **Deletion grace period** separates a durable Dataset, Snapshot or Split revision deletion - mark from destructive cleanup; it does not guarantee completion of every admitted read. - -Dataset does not infer semantics from a locator, a Snapshot ID's numeric -ordering, a sample ID's encoding, or an external source URI. - -## 2. Identity and names - -A Dataset identity is `(namespace, dataset_name)`. The initial authority uses -the configured `dataset-ns` root namespace for every Dataset. Names are unique -within that namespace and are compared as opaque UTF-8 names after validation. - -`SnapshotId` is immutable and opaque. `latest` and `stable` are mutable names -whose values are Snapshot IDs. A Read Plan, Cursor, or Manifest stores -the resolved Snapshot ID and never stores a mutable name in its place. - -A Snapshot has zero or one declared parent. Parentage is explicit; ID ordering -never implies ancestry. A child may add, replace, or tombstone samples and -fields while inheriting all values not overridden by its visible parent chain. -Any complete available same-Dataset Snapshot may be the parent, independently of latest. -Multiple parentless roots are permitted. Sample IDs are opaque bytes, represented -as canonical padded base64 in JSON; they MUST NOT be normalized as text or paths. - -## 3. Dataset records - -A Dataset record establishes that the identity exists and owns its metadata. -Creating an existing identity is a conflict. Opening a missing identity is a -not-found error. Whole-Dataset deletion requires the expected immutable Dataset ID. -A nonempty Dataset MUST be rejected: latest/stable, unreclaimed Snapshots (including -unpublished candidates), uploads, legacy drafts, or pending creation all count as -content. Empty deletion atomically marks the Dataset unavailable and persists a -`reclaim_after` deadline. Read/list hide it immediately; creation at that name is -rejected until the background worker completes retirement. Repetition preserves -the receipt and deadline. Name reuse creates a different Dataset ID; a stale -DELETE MUST conflict. Minimal identity receipts and retired child fences remain -for retry safety, rather than resurrecting old operations. - -Control v2's optional `X-Crowdb-Dataset-Id` active-instance guard MUST reject a -same-name replacement with 409 `dataset_replaced`. Malformed, zero, noncanonical -or duplicated values MUST fail with 400. Missing/marked identities return 404. -The advertised `dataset-identity-fence` capability covers both request admission -and creation reservation CAS, so deletion/name reuse cannot redirect a delayed -upload or Snapshot creator after its initial identity observation. Requests that -omit the header retain their existing contracts; deletion receipt/status retries -after marking use their own immutable ID without this active-instance guard. - -The Access Server setting `dataset.reclaim_grace_seconds` defaults to 1200 and -must be positive. Dataset, Snapshot and Split revision deletion use this value -when first marked. Existing deadlines are immutable across configuration changes -and restarts. Tests may set it to 1. Direct authorities expose -`with_reclaim_grace_seconds`; native clients default to 1200. - -A Snapshot record contains the Snapshot ID, optional parent, publication -operation identity, Manifest binding, and publication state. Publication states -are: - -- `Prepared`: the operation and candidate Snapshot are durable but the - Snapshot is not visible through `latest`. -- `Published`: the complete Manifest and binding are durable and the Snapshot - may become visible through `latest`. -- `Aborted`: the candidate is permanently unavailable for publication. - -A Manifest binding records the Snapshot it belongs to, its partition count, and -its integrity metadata. A binding for another Snapshot is invalid. Partitions -are an internal representation; readers observe one logical Manifest. - -## 4. Operations - -The Dataset authority provides these operations. Each operation is scoped by a -Dataset identity; operations that take a Snapshot ID MUST use the immutable ID, -not `latest` or `stable` as an implicit mutable reference. -The table summarizes the legacy publication/read surface. Control v2 adds bounded -catalog/tree inspection, immutable composition resources and independent Split revisions as -specified below and in its versioned OpenAPI contract. - -| Operation | Input | Successful result | -| --- | --- | --- | -| `CreateDataset` | Dataset identity | Creates the identity; duplicate creation conflicts. | -| `OpenDataset` | Dataset identity | Returns the Dataset record. | -| `PrepareSnapshot` | Parent, Manifest, operation token | Returns one durable candidate Snapshot ID that is not visible through `latest`. | -| `PublishSnapshot` | Parent, complete Manifest, operation token | Returns the published Snapshot ID and advances `latest` atomically. | -| `GetLatest` | Dataset identity | Returns the current Snapshot ID or no value for an empty Dataset. | -| `ListSnapshots` | Dataset identity | Returns published Snapshot records in declared ancestry order. | -| `GetSnapshot` | Snapshot ID | Returns the immutable Snapshot record. | -| `GetManifest` | Snapshot ID | Returns the complete validated logical Manifest. | -| `SetStable` | Published Snapshot ID | Sets the mutable `stable` name. | -| `RetainSnapshot` / `ReleaseSnapshot` | Snapshot ID | Adds or removes an explicit retention reference idempotently. | -| `ReclaimSnapshot` / `GetReclaimStatus` | Snapshot ID | Reclaims eligible state or returns durable reclaim progress. | -| `GetSample` / `GetBatch` | Snapshot ID, sample IDs, projection | Returns ordered samples and verified field values. | -| `Scan` | Read Plan, optional Cursor | Returns one bounded batch and the next Cursor. | -| `BindRead` / `NextBatch` | Query or continuation | Pins immutable versions or replays a caller-owned position without read-side writes. | -| `Cancel` | Bound read or service | Stops local admission and production without a remote reader release. | - -`PrepareSnapshot` and `PublishSnapshot` share the same operation token and -validation rules. A complete publication may be implemented as one call that -performs preparation and completion internally; it MUST have the same durable -state transitions. Data reads MUST NOT persist activity, selected-ID copies, progress, TTL or -reader registration. Checkpoints are owned by the client. - -## 5. Manifest and field values - -A Manifest MUST validate before publication. Validation rejects an empty -sample set, duplicate sample IDs, duplicate field names, missing required -fields, invalid checksums, invalid parent references, and incomplete or -inconsistent partition metadata. - -Within one Snapshot, `(sample_id, field_name)` has at most one value. A Schema -field can be required or optional. Required field names MUST remain in a child schema. Removing an inherited -value requires an explicit tombstone; readers never silently combine -incompatible schemas. A Manifest declares the complete current sample -membership: omitted samples disappear, while surviving samples may inherit -omitted optional fields through their declared parent. - -A Field value is exactly one of: - -- **Inline**: bounded bytes plus the declared digest. -- **Chunk**: an opaque location, range or stream metadata as defined by the - chunk subsystem, and the declared digest. -- **Tombstone**: an explicit deletion marker with no payload. - -Materialized reads verify the declared field digest before success. Streaming -reads return verified frames; whole-field integrity is established at field end -and partial output must be discarded if completion fails. External S3 or -Iceberg references are provenance and input metadata; they do not transfer -ownership of those systems' objects to Dataset. - -## 6. Snapshot publication - -Publication is an idempotent operation identified by an operation token. A -retry with the same token and equivalent parent and Manifest returns the same -Snapshot ID. A retry with divergent content is a conflict. - -The authority MUST perform publication in this order: - -1. Validate the Dataset, declared parent, operation token, and decoded Manifest. -2. Allocate or recover one candidate Snapshot ID for the operation. -3. Persist the prepared Snapshot record. -4. Persist every Manifest partition and the Snapshot binding. -5. Finish inspection metadata, initial Split membership and required payload owners. -6. Persist the published Snapshot record and install the final complete resource root. -7. Advance `latest` atomically to the complete Snapshot. - -Prepare completes the durable candidate without advancing latest. A failure -before step 6 leaves the candidate invisible and recoverable by the same -operation token. The Published flag alone does not authorize reads. A complete root -allows ID reads and references before step 7 succeeds; an -operation status alone is not proof that the head advanced. A successful Publish -completes the head transition. Retry identical content to finish an ambiguous -outcome; retries of historical publication MUST NOT rewind a newer head. A reader of `latest` MUST see either the previous -complete Snapshot or the new complete Snapshot, never a prepared or partial -Manifest. - -`stable` may be set only to an existing complete available Snapshot. Publishing a new -Snapshot never changes `stable` implicitly. A published Snapshot remains -addressable by ID until reclamation removes its metadata under Section 9. -Complete results that never won latest, including head-CAS losers, MUST also be -readable by ID, enumerable and usable as parents or stable targets. Preparation -records alone MUST NOT authorize these operations. Draft abandonment or supersession -MUST preserve the complete result and its payload owners. Incomplete cleanup MUST -seal the same final resource key that completion conditionally installs, preventing -late finalization from exposing reclaimed data. - -Control v2 separates candidate creation from publication attempts. An attempt MUST -bind the exact candidate and expected opaque head generation. Initial Split -membership and candidate upload ownership MUST be ready before the head transition. -New Snapshot publication MUST NOT persist attempt/response ledgers, success proofs -or draft publication progress. A response may echo the compatibility attempt ID; -it does not establish historical response replay. Retrying the same current target -can confirm its effect; a stale request after a later publication MUST NOT rewind -latest. A head conflict MUST return the observed latest/stable/generation without -a confirmation query. A reviewed retry uses that observation with caller-chosen -content; the server MUST NOT rebase. Unknown storage outcomes MUST propagate. -Existing legacy receipts remain readable without backfill. -Native Composition resource preparation MUST accept complete caller-owned inputs -and a fixed Snapshot ID without persisting editing or evaluation progress. Its -fixed source/Split revisions MUST survive retries unchanged. It MUST install the -complete resource root only after final Manifest/index, initial Split membership -and managed payload owners are ready; it MUST NOT move latest/stable. Input and -final Manifest are bounded to 8 MiB and computation to 30 seconds. Incomplete -computation is redone; identical complete content is reusable with zero writes. -Different resource content, including its initial Split binding, MUST conflict. -Unknown completion outcomes MUST NOT cause a confirmation query. REST exposes -this operation through `PUT snapshots/{snapshot}` with the -`composition-preparation` capability when uploads are configured. The JSON body -is also bounded to 8 MiB. Parent/schema are explicit nullable fields. Upload input -MUST include completed id/length/md5; DTO conversion MUST NOT add metadata queries. -The authority validates unique upload descriptors and returns conflict on mismatch. -Rust/Python SDKs MUST retain the same ID/full input without status queries or a -workflow fallback. Legacy Composition begin/edit/evaluate/prepare/publish and -cleanup routes return 501 before storage; only observation and abandonment remain -for old records. Background cleanup MUST rediscover old draft, work-page and edit -registration facts without storing candidate, scratch, edit or upload-retirement -progress. A lost local cursor repeats idempotent retirement and preserves complete -resource ownership and permanent fences against delayed writes. - -Composition and Split status reads MUST NOT reconstruct a publication request, -help edit receipts or persist recovery state. Legacy Committing drafts are -presented as Prepared without rewriting the stored draft or querying attempt, -receipt or head records. A caller may select a complete resource with the direct head API; this does not -prove that the historical request changed a head. -Direct Snapshot head updates MUST accept the complete Snapshot ID and explicit -nullable expected generation without an operation or draft identity. They MUST -preserve stable and require the `snapshot-head-update` capability in portable SDKs. -Their response and conflict observation use the latest/stable/generation head -shape. Unsupported direct updates MUST NOT fall back to composition or v1 writes. - -Publishing an independent Split revision MUST update only its Snapshot's currentSplit -pointer with an expected revision/head CAS. It MUST NOT create a data Snapshot or -change latest/stable. Old revisions remain addressable until explicit deletion -or owner reclamation. -Every complete immutable Split revision MUST also be readable by ID, enumerable -and usable as a frozen source before publication or after a publication conflict. -Abandoning a draft MUST NOT remove its complete result. A missing or cleanup-sealed -final revision record is unavailable; incomplete membership pages alone do not -constitute a result. Snapshot deletion admission still applies to each new request. -Independent Split publication MUST NOT persist an attempt, response receipt or -draft publication progress. A retry that observes the same current revision can -confirm the current effect; if a later revision has replaced it, the stale expected -base MUST fail without rewinding currentSplit. It does not promise historical -response replay. Unknown storage outcomes MUST propagate without a confirmation -query. Existing legacy receipts remain readable; new Split attempts are absent -from commit-status history. -Independent Split preparation MUST accept caller-owned revision ID, policy and -complete definitions through `PUT snapshots/{snapshot}/split-revisions/{revision}` -with the `split-preparation` capability. It MUST NOT persist editing/evaluation -progress or invoke draft APIs. Input and final bindings are bounded to 1 MiB and -computation to 30 seconds. An identical complete result MUST be reusable without -recomputation; different input at that ID MUST conflict. Incomplete work is retried -from client inputs. SDKs MUST retain the same ID/input and never downgrade to a -workflow mutation. Standalone begin/edit/evaluate/prepare/publish and cleanup -routes return 501; legacy draft observation and abandonment remain available. -Background cleanup MUST use local revision/page/item cursors and retain permanent -final-root and member fences. Reader-visible Snapshot owners wait for their -persisted deletion grace; incomplete content sealed at its final root has no -admitted readers and MUST NOT wait for a missing deadline. -Direct Split head updates MUST accept the complete resource ID and explicit -nullable expected revision and head generation without an operation/attempt ID or -draft lookup. A failed head condition MUST return the observed revision/generation, -including absence, without a confirmation query. The caller decides whether the -observed base is suitable for another update; the SDK MUST NOT rebase automatically -or fall back to a draft write when the direct-update capability is unavailable. - -## 7. Selection, plans, and cursors - -A Selection is evaluated against exactly one Snapshot. Supported selection -forms are explicit sample IDs, prefix or range, and bounded metadata predicates; -compound forms preserve their declared order and duplicate policy. Invalid or -ambiguous selections are rejected before payload reads. - -A Read Plan contains the resolved Snapshot ID, Selection, ordering, projection, -batch size, and optional deterministic shuffle parameters. The plan identity -includes all of those values. A plan cannot be resumed with a different -Snapshot, Selection, projection, ordering, or shuffle configuration. -Control v2 additionally pins a selected Split revision/name. Membership filtering -precedes shuffle and worker partitioning, which precede payload reads. A currentSplit -change MUST NOT alter an admitted plan or composition source. - -A Cursor is either at the start or contains a plan-bound group and offset. The -caller acknowledges a batch by saving the next Cursor in its own checkpoint. A failed or -unacknowledged batch may be replayed; a successful acknowledgement may not be -skipped or applied to another plan. - -Batch size, metadata bytes, prefetch, in-flight requests, and shuffle memory -are bounded. The authority rejects a plan that exceeds configured limits before -reading payload bytes. - -## 8. Read behavior - -`GetSample`, `GetBatch`, and bounded Scan first resolve Manifest metadata, -selection, projection, and ancestry, then fetch only the requested values. -Requested sample and projection order is preserved unless the plan explicitly -requests another ordering. Unknown samples, invalid projections, checksum -mismatches, and malformed cursors are errors. - -Each request checks authoritative snapshot availability at admission and MUST NOT -register, renew or release persistent reader activity. A durable deletion mark -rejects new requests. Previously admitted transfers may continue; neither EOF nor -cancellation waits for a remote cleanup operation. Local byte/request credits and -caller-owned buffers retain their existing lifetime guarantees. - -HTTP and native reads use the same authority semantics. Transport framing, -serialization, and topology discovery may differ, but the resolved Snapshot, -logical samples, field values, checksums, cursor result, and error class are -equivalent. - -## 9. Retention and reclamation - -`retain` and `release` apply to an immutable Snapshot ID. The latest Snapshot, -stable Snapshot, retained Snapshots, parents of live complete Snapshots on any -branch are protected from reclamation. Active reads do not prevent deletion. -The parent/child index MUST precede final child completion, which MUST recheck parent -availability. An indexed child protects its parent until terminal retirement even -while its own completion is pending; cleanup removes this conservative protection. - -A new Snapshot resource MUST persist its fixed twenty-four-hour `gc_after` -retirement eligibility deadline in its first write; content bindings inherit it. -Identical retries and publication-state changes MUST NOT renew it. Bounded -background scanning includes resources without a content binding and MAY mark -an expired complete or interrupted result only through the same authoritative -reference checks and publication fencing as explicit reclamation. Protected -results remain available; expiry eligibility alone is not a read-admission TTL. -Legacy resources inherit an existing binding deadline; otherwise their first -background observation assigns a fixed twenty-four-hour deadline. Unsupported -legacy opaque Manifest formats remain excluded from automatic retirement. - -Snapshot deletion first persists a terminal mark and a configured (default twenty-minute) -`reclaim_after` deadline. -Cleanup MUST NOT remove snapshot-owned metadata or payload references before -that deadline. The default applies to new marks; existing persisted absolute -deadlines remain unchanged. Restart and retry MUST preserve it. No Access Server or client -acknowledgements are required. Reads that exceed the grace period may return -an error after partial delivery; they MUST NOT report successful completion or -switch snapshots. Deletion completion does not revoke bytes already delivered. - -Reclamation is idempotent. It removes only unreachable Dataset-owned metadata -and payload references. A payload shared by two retained or readable -Snapshots remains available until its final Dataset owner is unreachable. -External S3, Iceberg, or other source references are never deleted by Dataset -reclamation. - -A reclaim status record reports the Snapshot, pending or completed state, and -reclaimed-count progress. Repeated requests and authority restarts preserve -that status and do not double-delete a payload. - -Complete Split revisions MUST default to reclamation with their owning Snapshot; -they MUST NOT acquire an independent automatic TTL. Explicit revision deletion -MUST name both Snapshot and revision IDs. The current Split head and the Snapshot's -initial Split revision MUST be protected, even when the initial revision is no -longer current. Deletion MUST close new revision reads and source admission, -preserve its fixed configured (default twenty-minute) grace across retries and restart, and fence -publishers admitted against an older head generation. Deletion state and exact -page bindings MUST reside on the existing revision resource. Deletion MUST NOT -create a reader session, request ledger or separate deletion task. - -Background cleanup MUST discover marked and terminal revision resources in bounded -pages with local cursors and preserve permanent fences against delayed member -writes. If the owning Snapshot has closed all read admission and completed its -own configured (default twenty-minute) grace, cleanup MAY use that owner fence before the revision's -later deadline. Revision deletion removes only Split membership and revision -metadata; it MUST NOT delete Samples, payload ownership or chunk data. - -## 10. HTTP and native surfaces - -The versioned public wire schemas, resource paths, binary framing and retry -contract are defined by [Dataset HTTP API v1](openapi-crowdb-dataset-v1.yaml) and -[Dataset control API v2](openapi-crowdb-dataset-v2.yaml). -The public surface uses JSON control records and binary payload streams; -persisted authority bincode records are not public DTOs. - -Control v2 MUST negotiate capabilities before mutation and MUST NOT downgrade -Split-aware inputs to v1. Required nullable preconditions MUST be supplied explicitly. -Unsigned 64-bit control values use canonical decimal strings. Legacy routes and -read-plan interpretations remain compatible; binary framing is unchanged. - -`DELETE snapshots/{snapshot}/split-revisions/{revision}` MUST accept an empty body -and require `split-revision-delete` in portable SDKs. A protected current or initial -revision returns 409; an absent resource or a new read of a closed revision returns -404. Existing bounded-work errors may take precedence over the closed-revision -404 response. Successful deletion and repeated deletion of a terminal resource return 200 -with `snapshot`, `revision`, `complete` and required nullable `reclaim_after`. -Pending status MUST carry the original absolute Unix-seconds deadline as a -canonical decimal string and `complete=false`; terminal status MUST carry null -and `complete=true`. Clients MUST NOT substitute legacy draft abandonment when -the capability or route is unavailable. Retrying after an unknown outcome uses -the same two IDs without a status or head query. The native adapter MUST add no -metadata lookup or implicit mutation retry to the shared authority operation. - -Composition takes retained complete caller inputs and creates an immutable -resource without edit/evaluation progress. Parent data defaults to all Samples; -Selection, frozen named Split and their intersection can restrict it. Source -S1/r2/train creates a child with parent S1 and frozen source membership. Inherited -Splits intersect surviving IDs; new IDs are unassigned until explicit assignment or -reevaluation. Defaults are disjoint/allow-unassigned; inheritance is explicit and -frozen to the supplied revision. Weighted -partition follows the versioned SHA-256 algorithm in the -[control lifecycle](design-dataset-control.md#3-split-evaluation-and-publication). - -Inspection MUST bound underlying metadata work, not merely paginate a fully loaded -Manifest. Committed inventory/children, Schema, Samples and projected field metadata -are separate bounded reads. Legacy inspection preparation MUST rebuild complete -metadata in one request without durable preparation progress. The source record -limit is 9 MiB, the embedded Manifest limit is 8 MiB and the request deadline is -thirty seconds. Final index writes MUST remain bounded at sixteen pending rows and -2 MiB of encoded keys and values. The immutable summary root MUST become visible -only after all rows and discovery entries are complete; cleanup MUST prevent a -delayed finalizer from replacing a retired root. Failures propagate without -confirmation reads, and retries rebuild from the source or reuse an existing root -without writes. Historical progress MAY be observed when the root is absent, but -MUST NOT drive or override a complete rebuild. Successful rebuilds return zero -revision and row-count compatibility fields. Unsupported or oversized sources MUST -return errors rather than empty results. Preview is opt-in, capped at 64 KiB and -MUST NOT acknowledge read consumption. - -The HTTP surface is the portable wire contract. It exposes Dataset creation and -opening, latest and Snapshot inspection, publication and preparation, Manifest -retrieval, retention, stable selection, reclamation status, bounded reads, read -queries and client-owned continuations. Cancellation is request-local. - -The native surface exposes the same operations through typed calls. Every -operation identifies the Dataset explicitly or through a bound service and -carries a Snapshot ID where the operation is Snapshot-scoped. - -HTTP uploads MUST use a Dataset-scoped opaque session ID, declared length, -part size and MD5. Numbered parts are immutable: identical retries reuse the -receipt; divergent bytes conflict. Declared part sizes are 1 byte through 4 MiB; -the final part may be shorter. At most 10000 parts are accepted. Completion -independently verifies part lengths/digests and the whole digest before producing a completed reference. -Preparation rejects incomplete, aborted or foreign references. Claimed payloads -remain protected until every candidate/Snapshot owner is released. Unowned -sessions expire; physical cleanup may remain pending and MUST report that state. - -Public reads bind the resolved immutable Snapshot, optional Split revision, -selection, projection, shuffle and worker partition in a caller-owned continuation. -Bind and next requests MUST NOT create persistent or required in-memory sessions. -The same continuation MUST replay its batch across independent server instances. -A successful consumer saves the next continuation; fetching, EOF and releasing -buffers MUST NOT advance shared progress. There is no remote ack, resume or close. -Missing next continuation means end of selection. Cancellation affects only the -current request. Each request authenticates and checks deletion admission. - -Selection windows, shuffle memory and request deadlines MUST remain bounded. -Groups admit at most 100000 selected IDs and 8 MiB of ID bytes; token JSON is capped -at 256 KiB before base64 encoding. Selected IDs are not copied into persistent -read records or the token. JSON metadata and binary headers are capped at 8 MiB. -Large fields stream rather than being materialized as a whole batch. - -Binary frame integrity is established at each frame, field integrity at field -end, and successful batch completion requires the batch terminator followed by -HTTP EOF. A caller-owned writer may contain a verified prefix on failure and -MUST discard partial output. File convenience downloads expose their destination -only after successful verification. SDK cancellation propagates to the same -Rust operation and returns producer resources; caller-retained completed data -has its own application lifetime. - -A configured HTTP bearer credential protects all Dataset routes. Deployments -that need separate management and data policies may enforce that distinction at -the service boundary without changing Dataset semantics. - -## 11. Errors and compatibility - -Errors preserve their logical class across surfaces: - -- invalid input, Manifest, selection, or cursor → client-invalid; -- missing Dataset, Snapshot, or Manifest → not-found; -- duplicate operation, parent race, protected Snapshot, or stale cursor → - conflict; -- checksum or corrupt durable state → integrity/unavailable; -- cancelled read or expired admission → cancelled; -- delivery-window or configured bound exceeded → backpressure/resource - exhausted; -- transient storage or service failure → unavailable and retryable only when - the operation is documented idempotent. - -Wire records are versioned. Unknown optional fields may be ignored, but a -reader MUST reject an unsupported version, missing required field, invalid -checksum, or incompatible state transition. New surfaces must preserve the -logical behavior in this specification before adding implementation-specific -extensions. - -## 12. Normative invariants - -- **DATA-S1 — Snapshot identity:** every operation resolves one immutable - Snapshot ID. -- **DATA-S2 — Complete visibility:** `latest` exposes only a published, - complete Manifest. -- **DATA-S3 — Explicit ancestry:** only declared parent links provide - inheritance. -- **DATA-S4 — Manifest uniqueness:** one Snapshot has one valid value per - `(sample_id, field_name)`. -- **DATA-S5 — Opaque location:** Dataset never parses or manufactures chunk - locations. -- **DATA-S6 — Plan binding:** a Cursor is valid only for its exact Read Plan. -- **DATA-S7 — Bounded delivery:** configured memory, batch, and in-flight - limits are enforced before payload admission. -- **DATA-S8 — Delayed reclamation:** deletion rejects new reads and defers - cleanup to its durable grace deadline; overdue reads may fail without - successful truncation or snapshot substitution. -- **DATA-S9 — Ownership-safe GC:** reclamation deletes only unowned Dataset - data and never external source objects. -- **DATA-S10 — Surface parity:** HTTP and native surfaces preserve the same - logical results and error classes. diff --git a/doc/design/access-server/design-crowdb-access-server.md b/doc/design/access-server/design-crowdb-access-server.md deleted file mode 100644 index bfe282e31..000000000 --- a/doc/design/access-server/design-crowdb-access-server.md +++ /dev/null @@ -1,320 +0,0 @@ - - - -# CROWDB - Design: Access Server - -CROWDB presents three first-class upper access models: S3, Iceberg, and -Dataset. Each model owns its semantics while sharing the same durable CROWDB -storage substrate. HTTP is the portable server interface. A trusted internal -native runtime exposes topology-aware storage and host-buffer delivery, with -S3 as its first typed frontend. Dataset retains its own semantic authority -when attaching to this shared runtime. - -## Table of contents - -1. [Intent](#1-intent) -2. [Access portfolio](#2-access-portfolio) -3. [Architecture](#3-architecture) -4. [Semantic ownership](#4-semantic-ownership) -5. [Data paths](#5-data-paths) -6. [Shared storage boundary](#6-shared-storage-boundary) -7. [Scale-out and security](#7-scale-out-and-security) -8. [Correctness invariants](#8-correctness-invariants) -9. [Direction and risks](#9-direction-and-risks) - -## 1. Intent - -The access architecture gives different users the model that fits their work -without forcing all data through an object-store abstraction: - -- existing applications use an S3-compatible object interface; -- table engines use an Iceberg catalog and table interface; and -- AI and data-intensive applications use Dataset concepts and can choose a - portable HTTP path or a direct native path. - -The Access Server terminates the HTTP surfaces. It is a stateless translator, -not a storage authority or a required relay for every client. The HTTP engine -is an implementation choice and is not part of the architecture contract. - -## 2. Access portfolio - -| Model | Semantic authority | Portable surface | Performance surface | -| ------- | ------------------------------------------ | ---------------- | ---------------------------- | -| S3 | Buckets, objects, multipart, S3 lifecycle | HTTP | Trusted native storage SDK | -| Iceberg | Catalogs, namespaces, tables, snapshots | HTTP | Delegated immutable FileIO | -| Dataset | Namespaces, snapshots, samples, fields | HTTP | Topology-aware native client | - -Related designs: - -- [S3](s3/design-crowdb-access-s3.md) -- [Trusted native SDK](design-crowdb-native-sdk.md) -- [Iceberg](iceberge/design-crowdb-iceberg.md) -- [Dataset](dataset/design-crowdb-access-dataset.md) -- [Accelerated transfer](s3/design-crowdb-access-s3-rdma.md) -- [Chunk I/O](../chunkio/design-crowdb-chunkio.md) -- [Chunk-KV](../chunkds/design-crowdb-chunk-kv.md) -- [RPC](../rpc/design-crowdb-rpc.md) - -## 3. Architecture - -```text - portable, stateless server access - - S3 client Iceberg client Dataset HTTP client - | HTTP | HTTP | HTTP - v v v -+------------+ +---------------+ +---------------+ -| S3 module | | Iceberg module| | Dataset module| -+------+-----+ +-------+-------+ +-------+-------+ - | | | - +---------------------+------------------------+ - | - CROWDB storage clients - | - v - Chunk-KV + chunk services - ^ - | - routed RPC / optional RDMA data plane - | - Shared native runtime - ^ - | - AI runtime / CPU or GPU memory - - topology-aware direct access -``` - -S3 and Iceberg expose HTTP protocols. The trusted native S3 facade reuses S3 -execution over the shared runtime and contacts metadata/data services directly. -Dataset's native adapter uses this same runtime while retaining Dataset -selection and acknowledgement semantics. The native runtime removes the -Access Server routing and payload hop. - -## 4. Semantic ownership - -S3, Iceberg, and Dataset are peers. None is implemented as a metadata convention -on another model: - -- S3 owns object names, overwrite behavior, listing, and S3 compatibility. -- Iceberg owns catalog, table, snapshot, commit, immutable file, and - reachability semantics. Its S3-shaped file locations do not make S3 the - authority. -- Dataset owns namespace-scoped identity, snapshot publication, sample and field - selection, batching, streaming, prefetch, and bounded delivery. - -The models may share physical bytes only through an explicit reference contract. -Sharing never transfers namespace, publication, authorization, or reclamation -authority. - -Each HTTP model has an independent listener, request types, configuration, -authentication policy, admission budget, metrics, and lifecycle. Shared -utilities may manage buffers, credentials, errors, and shutdown, but cannot -reinterpret model semantics. - -The `crowdb-access-server` executable starts the S3 and Iceberg listeners -together by default. The container supervises one access process for both -ports. Explicit `s3` and `iceberg` commands are reserved for focused tests -and management operations. -The combined entry point signals the other listener when either service -returns, then waits for both services to drain their owned small-write pools -before exiting. An unexpected Iceberg recovery or GC worker exit fails the -Iceberg listener, so the combined process also stops the S3 listener. A -terminated small-write manager fails its owning listener; an empty pipeline -route set remains recoverable while the manager continues restarting pipelines. - -The listeners own separate `ChunkIoClient` instances and small-write pools. -New S3 chunks use type `S3`; new Iceberg file chunks use type `IcebergTable`. -Each protocol may override the legacy common small-write policy and select -its own large-write EC, memory, prefetch, and mirror settings. Each protocol -library constructs its own storage clients and chooses foreground writers; -the executable handles HTTP framing, listener startup, and process shutdown. -Iceberg file-record construction also lives in the Iceberg library, including -the format rule shared by ordinary uploads and multipart completion. -Iceberg GC uses a separate chunk client with Iceberg's small-write policy. -The Iceberg library owns its GC storage budget and file-block adapters. -After proving a payload range unreachable, GC hands its exact byte range to -ChunkDB. A successful deletion notification completes Access's responsibility; -Access does not poll physical reclamation or retain progress for DiskDB capacity -release. `ReclaimOutcome::Accepted` means the notification was accepted (or the -chunk is already absent), not that disk space is reusable. Physical range -retirement, layout grace and compaction belong to ChunkDB and DiskDB. -Each small-write policy also chooses its chunk capacity; each protocol's -large-write policy chooses its own maximum chunk size. The deployment profile -sets RPC workers and DiskIO connections independently of these data limits. - -## 5. Data paths - -The ordinary path streams bounded data through the Access Server over HTTP. It -is the universal path and remains available without specialized hardware. - -S3 and Iceberg PUT and multipart part uploads share an object-scoped transfer -driver. One task polls the protocol's body decoder and a bounded write -consumer; it can continue receiving while earlier strips are in flight. -Completed receive owners are offered to the writer before their logical -payload views enter a separate MD5 and optional SHA-256 worker. The writer -uses separately bounded held-buffer and in-flight strip windows, and -processes strip completion in submission order. Small objects use the same -handoff and digest completion but retain their protocol-owned shared -small-write pipeline after the handoff. Authentication, checksum declarations, -metadata publication, multipart authority, and cleanup remain with each -protocol. The [Iceberg upload-flow design](iceberge/design-crowdb-iceberg-upload-flow.md) -describes scheduling and stage measurements in detail. - -The Dataset native path embeds routing, retry, bounded planning, streaming, and -buffer ownership in the application. It resolves one immutable dataset -snapshot and distributes work directly to responsible CROWDB services. A -topology view routes work but never grants placement authority. - -Native S3 and Dataset access use the shared CROWDB runtime and RC SEND/RECV -transport with registered host buffers. This path avoids an Access Server payload -bounce while preserving metadata authority, publication and integrity rules. -Host-memory ownership and admission remain explicit. GPU destinations, cuObject -negotiation and one-sided client-memory transfer are outside the current contract. -See [Native RDMA access](s3/design-crowdb-access-s3-rdma.md) and -[RPC RDMA](../rpc/design-crowdb-rpc-rdma.md). - -## 6. Shared storage boundary - -Chunk-KV is the common metadata and ordered-index substrate. Chunk services are -the common byte, durability, placement, repair, and buffer-ownership substrate. -Each access model stores only its own logical authority and opaque references to -data below this boundary. - -Physical segments, disks, replicas, and erasure-coding shards remain storage -concerns. Native clients observe versioned topology for scheduling and routing, -but cannot publish or alter physical placement. - -## 7. Scale-out and security - -Access Server instances keep no authoritative namespace or operation outcome. -Durable state lives in CROWDB, so any healthy instance can serve a later request -or retry. Protocol-local caches are discardable hints. - -Trusted internal native clients share Access Server's trust domain and receive -explicit tenant and operation context from their embedding system. They do not -obtain the S3 secret database or verify SigV4. External HTTP authentication -remains in the HTTP frontend. Dataset HTTP capability contracts remain in the -Dataset design; native embedding does not depend on external-user capability -issuance. Topology and memory descriptors alone grant no access. - -All paths bound connections, metadata work, stream windows, retained buffers, -registered memory, retries, and background work independently of total object, -table, or dataset size. - -The process has a separate HTTP health listener configured by `HealthConfig` -(or `CROWDB_ACCESS_HEALTH_LISTEN`), defaulting to loopback port 9094. Its -`/_crowdb/health/live` and `/_crowdb/health/ready` routes are independent of the -S3 and Iceberg listeners. All three ports are distinct. Health serves no -user-data routes; S3 requests cannot bypass authentication through a health -path. Readiness observes the same storage dependencies and cleanup backlog -as the data dispatcher. Retained launch records identify the dedicated health -address; a missing address requires deployment reconciliation before restart. - -One Access background sampler runs every second and publishes an immutable -health snapshot through ArcSwap. It independently checks the routed Chunk-KV -bucket namespace, ChunkDB and online disks through the process's storage clients. -Metadata and chunk checks run concurrently with a two-second dependency budget; -hardware discovery, ChunkDB and disk headers share that chunk-check budget. A -2100 ms outer watchdog bounds the full round and cancels unfinished work. Only -one sampling round can run at a time; missed one-second ticks are skipped rather -than overlapping or replayed. A failed round publishes unavailable evidence and -logs its generation, configured budgets, elapsed time and dependency errors. -Sampling continues after every failure, and a later successful generation -recovers readiness without a restart. Failure never extends the probe budget, -adds a retry or changes the response to Ready. Physical checks read existing disk -bytes without creating objects, allocating chunks or repairing storage. -ChunkDB point checks use the validated service route and actual owner/storage -reads. They do not initiate an ordinary periodic full registry refresh before -the query; cold discovery and failed-route refresh retain the existing rules -and shared deadline. A cached route never substitutes for a current response. -Each physical check reads a 4096-byte disk header through the storage client's -bounded priority connection group. It retains the same authoritative disk route -and shares the chunk check's total deadline. Bulk response bytes on a normal -connection cannot delay that header through TCP response ordering. A rejected, -short or late header still fails the observation; the priority lane supplies no -additional retry or grace period. -Credential-cache freshness, listener state, native-pool pressure and cleanup -backlog are included in the same snapshot. Application requests only record -their own errors and metrics; their outcomes cannot overwrite health evidence. - -Health requests read the snapshot without dependency I/O or a new sampling task. -Responses retain the check timestamp, generation, age, exact flags and diagnostic. -New observations replace old failures or successes. Evidence older than three -seconds from the sampling start, or an absent first sample, reports Unknown and -HTTP 503; readers never extend its validity or clear it to Ready. Completion -does not renew the start timestamp. Consecutive slow successful rounds can leave -an expired-evidence interval before the next round completes; retain that 503 -and recover only when a new current generation is published. Process stop is an immediate lifecycle -fence even when the cached sample was healthy. The sampler is cancelled with the -listener. Recovery needs neither application traffic nor an Access restart. - -Web independently samples local monitor evidence, signed protocol reads and -service diagnostics in its backend. Its node-observation endpoint only returns -that bounded-age snapshot, including original operation identities and producer -timestamps. Status readers aggregate cached member evidence with current -committed intent, so stop/delete and ownership fences override older samples. -Frontend query failures do not mutate these backend health observations. - -## 8. Correctness invariants - -- **AS-I1 — First-class models:** S3, Iceberg, and Dataset each own their - namespace, publication, authorization, compatibility, and reclamation rules. -- **AS-I2 — HTTP boundary:** S3 and Iceberg are HTTP server protocols; Dataset - HTTP is the portable Dataset server surface. -- **AS-I3 — Native directness:** trusted native access reaches responsible - CROWDB services without an Access Server request or payload hop. -- **AS-I4 — Semantic equivalence:** Dataset HTTP, native CPU, and accelerated - GPU paths select the same immutable snapshot and logical data. -- **AS-I5 — Storage authority:** no Access Server module or native client owns - physical chunk placement, replicas, erasure-coding shards, or disks. -- **AS-I6 — Stateless server:** an authoritative operation survives loss of the - Access Server instance that began it. -- **AS-I7 — Bounded ownership:** retained state is bounded independently of - object, table, or dataset size. -- **AS-I8 — Authorized direct access:** topology, endpoints, and memory - descriptors are not capabilities by themselves. -- **AS-I9 — Completion lifetime:** every buffer and registration outlives all - socket, RPC, storage, NIC, and GPU operations that reference it. -- **AS-I10 — Protocol storage ownership:** S3 and Iceberg use distinct chunk - types and independently admitted foreground write pools; each library owns - its file or object authority and storage policy. -- **AS-I11 — Shared upload progress:** one body reader and one write consumer - own each S3 or Iceberg PUT or multipart part. Socket readiness and write - completion can each resume an idle transfer without changing protocol - publication authority. - -Coordination follows the semantic owner, rather than the Access Server process: - -- Dataset read admission is read-only. Clients retain continuation progress; - fetching, acknowledgements and snapshot activity create no reader records. - Existing write-side draft journals recover partial work and fence late writes - after cleanup. -- Dataset upload expiry scans upload authorities; creation does not enter a - global registry. Shared payload owner records prevent one snapshot's retirement - from reclaiming another snapshot's bytes. -- Iceberg streamed uploads use independent session bounds and per-part writes. - Catalog-wide credit records are retained only for legacy recovery. Ordinary - table commits use one head CAS; GC fences protect committed file reachability. -- Namespace deletion rejects live children. The associated parent admission and - recovery state preserves that strict contract and is not a reader lease. -- S3 multipart coordinates accounting and exact completion selection within one - upload. Ordinary object reads do not register persistent activity. -- A failed CAS observation supplies the compared key, revision and value for an - allowed retry. Unknown transport outcomes follow the operation's retry contract; - application preconditions must not silently move to a newer base. Credential - bootstrap publishes one encrypted final resource through create-if-absent; - comparison losers reuse that value, while older peers and busy/unknown replies - require a direct resource read. It has no persisted preparation record. - -## 9. Direction and risks - -The main architectural direction is to keep HTTP broadly compatible while -making Dataset native access the performance path for AI workloads. RDMA and -GPU-direct delivery should reduce movement, not introduce a second semantic or -storage authority. - -The main risks are compatibility drift, stale client topology, capability -containment, cross-service backpressure, native resource lifetime, and uneven -GPU-direct hardware support. Every accelerated path must have a bounded CPU -fallback and must prove the same selection, integrity, and completion semantics. diff --git a/doc/design/access-server/design-crowdb-native-sdk.md b/doc/design/access-server/design-crowdb-native-sdk.md deleted file mode 100644 index c2467194d..000000000 --- a/doc/design/access-server/design-crowdb-native-sdk.md +++ /dev/null @@ -1,185 +0,0 @@ - - - -# CROWDB - Design: Trusted native storage SDK - -The native SDK embeds internal storage access in the application. Its shared -runtime, topology and host-buffer delivery support typed S3 and Dataset frontends. - -Depends on: [Access Server](design-crowdb-access-server.md), -[RDMA RPC](../rpc/design-crowdb-rpc-rdma.md), -[Chunk I/O](../chunkio/design-crowdb-chunkio.md). - -Satisfies: Access Server invariants AS-I3, AS-I5, AS-I7 and AS-I9. - -## Table of contents - -1. [Trust and semantic ownership](#1-trust-and-semantic-ownership) -2. [Shared runtime](#2-shared-runtime) -3. [Topology and selection](#3-topology-and-selection) -4. [Data and completion](#4-data-and-completion) -5. [Budgets and shutdown](#5-budgets-and-shutdown) -6. [Transport and integration](#6-transport-and-integration) -7. [Correctness invariants](#7-correctness-invariants) - -## 1. Trust and semantic ownership - -The embedding process is a trusted internal cluster component in the Access -Server trust domain. It supplies tenant and operation context and reaches -management, Chunk-KV, Chunkdb and DiskIO directly. Native operations do not -verify SigV4, receive a user-secret database or call Access Server to authorize -an operation. Network access to internal services belongs to the embedding -system's deployment boundary. - -`S3Executor` owns shared S3 namespace, metadata selection, conditions, -publication and multipart orchestration. HTTP supplies parsing, authentication -and HTTP/XML encoding; `S3Client` supplies typed context and results. Both -preserve S3 generation, checksum, visibility and uncertain-publication rules. -Copy captures a source generation. Multipart completion binds exact immutable -part generations. An admitted independent-part writer may acknowledge a later -replacement without changing that selection. Each accepted Complete execution -publishes the frozen selection through an unconditional object PUT, including -retries after success; it can overwrite a later ordinary PUT. No completion -response is recorded. [S3 publication](s3/design-crowdb-access-s3.md#4-publication-and-lifecycle) -defines the shared multipart resource modes and completion rules. - -`DatasetClient` attaches `DatasetAuthority`, `DatasetUploads` and -`DatasetReads` to these same clients. Complete Snapshot/Split preparation takes -caller-owned immutable inputs and resource identities. Snapshot completion follows -final upload-owner claims; it stores no editing or evaluation progress. Mutable -head updates use explicit expected bases and preserve structured conflict -observations. Complete results remain readable by ID independently of head -selection. Unknown mutations retain their resource identity, full input and base; -the SDK never retries or rebases automatically. Read selection, worker partition, -resource CAS and cleanup remain in the Dataset authority. - -## 2. Shared runtime - -`ClientRuntime` owns or borrows control, metadata and data clients. Business -handles share transport groups, discovery, topology observations and resource -budgets. Compatible writer policies share one bounded small-writer pool; -incompatible policies use distinct pools while sharing the transport owners. -S3 and Dataset chunks retain their respective types; native embedding introduces no generic user-data -chunk type. - -`connect` creates owned groups. `adopt_clients` transfers shutdown -responsibility for existing groups. `from_clients` borrows them, so shutting -down that facade cannot stop its original owner's workers. These ownership -choices are explicit rather than inferred from an Arc reference count. - -## 3. Topology and selection - -`TopologyView` retains catalog generation, independently versioned disk routes, -service observations and actual transport configuration/counters. Sources have -separate observation ages and are not a globally atomic topology epoch. An -independent refresh with unknown observation time reports unknown freshness. -Catalog regression or invalid pages cannot replace a valid cached generation. - -`ClientRuntime::read_flow_metrics` snapshots existing cumulative layout, stream -credit, storage-wait and frame-decode counters on the shared chunk reader. Writer -policy groups share these counters. Reading them performs no RPC or reset; a -delta covers the reader's concurrent work, not one request. Resource status -reports current charges and lifetime high water separately from compatibility -copy bytes. These observations do not measure all copies or per-trial peaks. - -`ResolvedObject` binds one published object record to a tenant and metadata -authority. `object_locality` returns a bounded page of logical extents with -layout revisions, fragment roles and rack/node/disk observations. Inspection -does not dial storage, publish placement or replace routing checks. Execution -uses current routes and layout fencing while preserving the selected object's -identity. A retained old view remains inspectable but cannot pin execution to -an old owner or force a deleted/expired layout into a read. - -## 4. Data and completion - -`ObjectBody` is a lazy pull stream of `BufferLease` values. Each lease exposes -request/stream identity, sequence, logical offset, length, allocation capacity -and registration identity. Storage frame/control bytes are excluded from the -logical object stream. `OwnedPayload` retains the allocation through RPC, -DiskIO, ChunkIO, metrics wrappers and S3 adaptation. - -Successful EOF requires all applicable length and integrity checks. Delivered -metadata or a valid prefix is not full-read success. Terminal errors remain -sticky across polling; early body drop is cancellation. Cancelling one pending -pull preserves its position when the body itself remains live. - -Ordinary input retains its allocation or uses a bounded adapter. A unique -producer allocation can be filled directly in final storage frame slots, -frozen and consumed by `write_framed`. Submitted input is immutable and retained -through physical completion. Acceptance of an input buffer is distinct from -durable object publication; only successful finalization establishes success. -A lost publication reply retains uncertainty and cannot authorize a blind -replay with a new identity. - -## 5. Budgets and shutdown - -Admission is lock-free and bounded across requests, producers and caller-held -allocation capacity. Shared views charge physical capacity once until the final -owner releases it. An allocation's registration budget is separate from SDK -held bytes, posted receives, SQ work requests and writer windows. - -Held-capacity exhaustion pauses production without blocking RPC completion -workers. Releasing the final owner returns capacity and wakes waiters; deadlines, -cancellation and shutdown remain observable. Unknown external owners reserve -bounded capacity before a compatibility copy. Open small strips can retain -input for replica repair after operation completion until seal or pool drain. - -Shutdown closes admission, drains owned writer groups and joins owned workers. -All waiters observe one shutdown result; cancelling a waiter does not cancel -the shared cleanup. Caller-held leases retain storage after shutdown, and final -release never reposts to a destroyed transport. A host-to-GPU copy holds its -host lease until the copy completes, independently of stream completion. - -## 6. Transport and integration - -Metadata and data groups retain their own TCP/RDMA policies. Disabled RDMA uses -TCP; eligible setup failures follow the configured fallback policy. Accepted -or partial RDMA reads fail terminally through routing, mirror/EC recovery and -object delivery; they cannot append a TCP suffix or switch replicas to conceal -an interrupted transfer. Ambiguous writes preserve uncertainty. - -Eligible registered producers avoid staging; eligible receives transfer their -allocation without a second SDK payload copy. Framing/control decoding, -alignment and EC construction/reconstruction retain their required copies. -The base lease API exposes host memory and opaque ownership, not verbs objects, -GPU pointers or RDMA READ/WRITE authority. - -The independent Dataset HTTP distribution has no dependency on this storage/FFI -runtime. A Dataset facade attaches to the common runtime and delivery contracts -while retaining Dataset authority, Snapshot/Split binding and caller-owned -checkpoints. Buffer release, download completion and saving a processing -checkpoint remain separate events. `BatchBody` delivers field metadata and -lazy payload leases with per-part and whole-field integrity checks. Read -admission checks authoritative Snapshot and Split deletion state without -persistent reader registration, renewal or release. A durable deletion mark -rejects new reads and records a fixed twenty-minute cleanup grace; admitted -streams may finish, and overdue reads may fail after reclamation. There is no -persistent read session or remote ack/close. A continuation replays the same -batch on an independent client; only the caller saves the next position. - -The optional native Python extension owns a shared runtime and exposes retained -immutable leases. Slicing shares allocation ownership; conversion to Python -bytes explicitly copies. Pulls are serialized per batch with a bounded request -channel and no automatic payload prefetch. Runtime shutdown preserves already -delivered leases. Forked processes create new runtimes. Python control commands -reuse the Rust facade and shared authority schema without another storage path. - -## 7. Correctness invariants - -- **NS-I1 — Trusted embedding:** native tenant/context execute inside the - internal-service trust domain without a per-operation HTTP authentication hop. -- **NS-I2 — Shared semantics:** HTTP and native frontends use the same selection, - durability, publication, multipart and Dataset cleanup authorities. -- **NS-I3 — Non-authoritative observation:** topology inspection never changes - placement or bypasses owner, generation or layout validation. -- **NS-I4 — Verified completion:** only successful EOF establishes a full read; - a terminal failure cannot later become successful EOF. -- **NS-I5 — Final-owner accounting:** completion does not release caller-held - capacity; the allocation and its charge survive every dependent owner. -- **NS-I6 — Safe shutdown:** closed admission and drained workers do not - invalidate bytes still held by a caller or asynchronous copy. -- **NS-I7 — Bounded transport failure:** accepted RDMA failures never append - TCP data or replay an ambiguous mutation. -- **NS-I8 — Explicit Dataset progress:** fetching, verified EOF and final buffer - release cannot save progress; the client checkpoints the next continuation - only after successful processing. diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md b/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md deleted file mode 100644 index 4dfbf222f..000000000 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md +++ /dev/null @@ -1,395 +0,0 @@ - - - -# CROWDB - Design: Iceberg File Upload Flow - -This document defines native injection and the FileIO upload path used by Iceberg clients and records -measurements from the single-node profile. It covers file publication, not table -snapshot publication. - -Depends on: [Native Iceberg Storage](design-crowdb-iceberg.md) and -[Chunk I/O](../../chunkio/design-crowdb-chunkio.md). - -## Table of contents - -1. [Client and server flow](#1-client-and-server-flow) -2. [Large write ownership and scheduling](#2-large-write-ownership-and-scheduling) -3. [Integrity and durable publication](#3-integrity-and-durable-publication) -4. [Measured costs](#4-measured-costs) - -## 1. Client and server flow - -### Native injection - -Native injection owns both byte ingestion and the subsequent construction of -Iceberg file references: - -1. Finish durable IcebergTable chunk writes and seal an owner-bound directory - root. Even a one-leaf native tree includes its file identity in that root. -2. Form `f/v1/..parquet` under the immutable - `///` prefix. Bind the locator's - unique file identity, length, descriptor and exact filename by signature. -3. Durably select the descriptor in the UUIDv7 injection operation before - publishing one `file/` record with create-only CAS. Return the one - `s3://iceberg//` URL after publication. A retry uses - the same operation/file identity and returns the selected URL; conflicts - never overwrite published content. -4. Construct standard manifests using that URL, write manifest lists and - metadata JSON, then publish the table head by its normal atomic commit. - Failed table commits leave unreferenced durable files for the existing - safe cleanup flow; they do not change a previously returned URL. - -Direct HEAD/GET/range requests validate cached permissions and the locator, -then read native storage without Iceberg file-location or file-state KV lookup. -The chunk layer still performs its normal replica/EC layout resolution. -HEAD and conditional reads probe the physical directory root; GET reuses the -verified directory in its pull reader. Reclaimed or reused roots fail closed. -List and cleanup still use the single file record. Logical deletion removes -the file from list; its known direct URL may read retained bytes until physical -GC. Reclaimed identities cannot alias newly allocated bytes. - -The native injection entry is a signed PUT with -`crowdb-native-upload=` on a table-scoped parquet upload URI. It requires -an explicit SHA-256 payload hash and returns one JSON `location` after durable -selection and publication. The operation binds its principal, full table path, -readable stem, payload digest and retry deadline. A missing publication is -completed from the selected receipt after response loss; a changed or deleted -publication cannot be replayed. The listener advertises the single-request byte -bound. Native loader parts stay within it; normal SDK multipart retains its -chosen logical URL. - -The loader journals the stable upload URI before transfer and the returned -location before adding standard manifest references. Its report preserves the -upload URI for recovery; it does not add a file-location field to Iceberg metadata. - -### Standard SDK uploads - -SDK location providers choose logical filenames before PUT. These uploads -retain that filename and resolve the single file record for reads. They do -not require a pre-upload reservation API, a second SDK location field, or -post-upload rewriting of manifests and path-based delete references. - -For a catalog without native injection, the TPC loader hashes each local Parquet file through bounded reads, checks -that its unique target is absent, then sends a known-size S3 request using the -vended native credentials. Below 256 MiB it uses a single PUT; larger files -use bounded multipart parts aligned to complete frame payloads. The upload -finishes before the loader imports its files and commits the table. Other -Iceberg FileIO clients can select their own multipart thresholds; PyArrow -output streams may use multipart even for small files. - -1. An optional client existence probe publishes nothing. Direct PUT validates - the request body, completes the Chunk writer and publishes one immutable - location descriptor with create-only CAS. -2. Multipart starts an independent session. Each UploadPart authenticates and - validates its body, finishes its writer and persists that part's locations. - Streamed parts update their own key without a session write for each part. -3. CompleteMultipart freezes the selected parts and composes their locations - into one descriptor without copying payload. Location publication is - create-only. Recovery can resume interrupted completion. -4. A later table-head CAS publishes Iceberg metadata referencing the uploaded - files. File uploads themselves do not publish a table snapshot. - -Writer selection uses the decoded length of each HTTP request, when available. -Payloads below the small-object threshold can use the small-object writer for -one frame or the shared-object writer for a longer request. Other requests use -the large writer. The total logical file size alone does not select the writer. - -## 2. Large write ownership and scheduling - -The single-use Iceberg `WriteObject` owns the request body, bounds, file -identity, writer, digest pipe, measurements, and final publication action. S3 -PUT and `UploadPart` use the same transfer driver and checksum worker while -retaining S3 authorization and publication. Its transfer coroutine polls the -body receiver and write consumer with one task waker. Only -the receiver fetches the socket body. When it offers a prepared buffer, the -same task immediately polls the consumer. If both sides are pending, the task -yields; a Hyper body-read event or a write completion wakes it again. An idle -writer therefore resumes when a slow socket supplies the next buffer. - -The receiver assembles up to 1 MiB of payload from body frames. The native -receive path prepares frame headers and CRC32C on the received owner before -handoff. CRC32C excludes the chunk ID; after placement the writer fills that -field and writes the frame without an object-sized copy. The receiver offers -the owner to a bounded channel with four held-buffer slots. A full channel -pauses further body reads until the consumer removes an owner. The digest -worker receives borrowed payload views after the write offer, so checksum work -can overlap receive and DiskIO without controlling write backpressure. Small objects have a per-object writer owning inline MD5/SHA-256 state. The -native receiver allocates exact payload plus frame overhead, updates the digest -as socket payload views arrive, and hands one complete owner directly to that -writer without a per-object handoff channel. The shared small-write pipeline -sets destination Chunk IDs and slices the owner across strip boundaries without -payload copies. Each pipeline confirms readable cursor publication before its -next batch; queued arrivals naturally aggregate during that wait. Hash routing -is a hint and can select another pipe with capacity. Shared chunks default to -256 MiB, with 32-strip mirror groups refilled asynchronously at half consumption. -EC conversion is disabled in single-node mode; enabled multi-node conversion -can materialize a strip image to compute parity. HTTP header parsing retains -its prefetched-body copy and reports its count and bytes through one bandwidth -metric, `access.http.receive.prefix_copy.bw`. - -The large chunk writer prepares strips ahead of demand. For a known object -size, it batches up to the configured strip-prefetch limit and requests the -next batch when half of the current one has been consumed. Mirror strips are -submitted to independent tasks, with at most four strip writes in flight by -default. Later writes may finish first, but completion is consumed in strip -order. Each completed task keeps its buffer until every preceding strip has -committed. A failed mirror segment is replaced and replayed from those retained -bytes before its strip completes; later completions remain held meanwhile. -If replacement is exhausted, the writer drains submitted IO, reads the -committed prefix from the old chunk one strip at a time, and replays it with -the retained suffix into a new chunk. Frame footers receive the new chunk ID; -the CRC does not cover that field. The old chunk is deleted only after the new -copy is durable. Rotation attempts are bounded. -At a full write window the coroutine awaits the oldest completion; -the completed owner queue can still retain four prepared buffers. The -`large_parallel_strip_writes` and `large_held_buffers` settings are separate. - -The following invariants apply: - -- **I1 — One body reader.** No second task reads an object's HTTP body. -- **I2 — Bounded ownership.** Receive buffers remain owned until the writer and - digest have consumed their views; the write queue controls backpressure. -- **I3 — Ordered durability.** A later strip result cannot make an earlier - failed strip successful or release its replay buffer. Chunk sealing waits - for every submitted strip, replacement, and required fsync. -- **I4 — Event-driven progress.** Socket readiness and write completion wake - the suspended coroutine. The write path does not spin or poll a timer for - capacity. - -The upload metrics distinguish elapsed body-frame polling from actual body -waits. A body wait is counted only when the body future returns `Pending`; its -duration ends when that frame or EOF becomes ready. Immediate frames add no -wait count, while writer-capacity and write-flow pause time remain separate. -`queued_buffers_peak` is the highest number of owners waiting in the handoff -queue during one upload. `mirror_uncommitted_peak` counts submitted strip -writes that have not yet committed in order, including writes whose DiskIO task -already finished. The counters describe different stages and are not added. -`mirror_active_write_peak` counts mirror data writes actually inside DiskIO; -it excludes completed tasks waiting for an earlier strip. -The native allocator exports current and peak retained bytes. The chunk client -also counts repair-driven rotations, replayed bytes, and rotation time. - -## 3. Integrity and durable publication - -Large metadata JSON, manifest lists and manifests follow the same native -object durability rules. Each keeps one bounded file descriptor; complete -snapshot arrays and file bytes are not copied into TableHead or an unbounded -KV value. Public `metadata/` and `statistics/` prefixes remain available for -bounded listing without opening object payloads. Small inline metadata and -large chunk-backed metadata return identical canonical FileIO bytes. - -The per-object OpenSSL digest state computes MD5 over ordered logical payload -views. Small objects update it synchronously in the receive task; large objects -use a checksum worker. It also computes SHA-256 when a signed payload requires it. It never -hashes frame headers or footers. The writer fills placement-dependent chunk -IDs after the receiver has calculated placement-independent CRC32C. Digest -failure, declared-length mismatch, failed DiskIO, or seal failure prevents -file and part publication. An upload is published only after the body has -ended, the digest has been verified, all writes and fsyncs have completed, -and chunk locations have been sealed. Multipart completion and table commit -are later, distinct authority operations. - -## 4. Measured costs - -On 2026-10-01, a local single-node container received a 100-MiB generated -object through the same `CrowdbFileIO` output API as the loader. The object was -not registered in an Iceberg table. Two runs took 6.23 s and 5.90 s. The second -run's stage and server-counter deltas were: - -| Measurement | Result | -| ------------------------------------- | ----------: | -| Create output stream, including probe | 0.322 s | -| Thirteen 8-MiB client writes | 0.075 s | -| Close and finish multipart transfer | 5.507 s | -| End-to-end elapsed time | 5.904 s | -| Effective logical throughput | 16.9 MiB/s | -| Successful FileIO HTTP requests | 12 | -| Catalog GET operations | 144 | -| Catalog compare-exchange operations | 16 | -| Small-write completions | 0 | - -The 12 successful requests are consistent with creating a session, uploading -parts, and completing it. The existence probe returns not found and is not in -the success count. The summed server dispatch time was 31.61 s across requests; -multipart part requests overlap, so this sum is not wall-clock latency. - -The client writes returned in 75 ms because PyArrow buffers or schedules the -transfers. The 5.5-s `close()` wait is the dominant observed client stage. -The zero small-write delta rules out the small-object queue as the path for -this 100-MiB sample. Catalog GET and CAS counts show metadata work remains -per multipart session and part, rather than a single final table update. -These counters do not yet isolate network transfer, chunk allocation, DiskIO, -or sealing within the 5.5-s wait. - -A second container run with the checksum worker used one prepared 1-MiB block -100 times through the same FileIO output stream. The 100-MiB upload took -5.014 s: 0.034 s to open, 0.104 s in client writes, and 4.876 s in `close()`. -The FileIO path still uses multipart. This result shows that checksum work is -not the only cost in its close stage; it does not isolate the remaining -multipart, transport, or storage costs. - -### Small files - -The same container and API were used for three generated objects. Each used -three successful FileIO requests, 45 catalog GETs, and seven catalog CAS -operations, even though the payload sizes differed. - -| Object size | Open | Client write | Close | Total | Small-write completions | -| ----------- | ------: | -----------: | ------: | ------: | ----------------------: | -| 64 KiB | 0.345 s | <0.001 s | 1.242 s | 1.587 s | 1 | -| 1 MiB | 0.328 s | <0.001 s | 1.213 s | 1.541 s | 0 | -| 5 MiB | 0.344 s | 0.001 s | 1.498 s | 1.844 s | 0 | - -The 64-KiB transfer completed through the small-write queue. The 1-MiB and -5-MiB requests did not increment that counter; the available metrics do not -separate shared-object from large-writer completions. Fixed FileIO session, -part, and completion work is material for a small object. The table-level -commit is not included in these timings. - -### Focused direct PUT - -The single-node small-cluster fixture uses one mirror copy and null DiskIO. -Its producer yields 100 references to one prepared 1-MiB buffer, with -Content-MD5 computed before timing and `UNSIGNED-PAYLOAD` in the request. The -direct 100-MiB PUT completed in 335.9 ms. The upload observation was 315.4 ms, -including 224.0 ms in body-frame polling and waiting, 47.4 ms across 30 -writer-capacity waits, 23.1 ms across two strip-preparation waits, 23.0 ms in -writer finish, and 47.9 ms in metadata publication. The sum of 101 strip-write -durations was 558.0 ms and digest CPU time was 234.3 ms. These stages overlap -and must not be added to estimate wall-clock time. Body-frame time includes -Hyper delivery and coroutine scheduling, not just socket reads. - -The container's multipart FileIO timings above have different transport, -storage, and publication work from this focused direct PUT. A comparable -FileIO profile must record part size, concurrency, topology, durability, -software revision, and raw stage samples before attributing its close time -to a specific server stage. - -### Frame-aligned FileIO follow-up - -The loader's PyArrow S3 output stream split a 100-MiB file into ten multipart -parts. On the local single-node container, the first current-tree run took -4.668 s. Its non-native receive path handed the writer 1 MiB of logical bytes -at a time, and the writer pushed each 64-KiB physical frame separately. -Consequently the mirror-strip batch path never ran: the measured peak was one -active mirror write, zero queued full mirror strips, and 1,610 individual -strip-push completions. Aligning receive batches to 16 frame payloads alone -left the run at 4.638 s because the writer still split them into single-frame -pushes. - -After the large async writer grouped those 16 complete frames into one 1-MiB -physical strip push, the same FileIO path completed in 0.993 s: 0.109 s to -open, 0.034 s in client write calls, and 0.850 s in close. The server recorded -ten completed part uploads, 100 MiB of logical data, 110 strip-push -completions, and peaks of four queued full mirror strips and four active -DiskIO writes. This is one local sample, not a stable throughput distribution. -The loader now chooses a single PUT below 256 MiB; one 100-MiB direct loader -upload took 1.091 s including local SHA-256, and a 256-MiB four-part loader -upload took 3.042 s. These timings include different client preparation and -must not be compared as pure server write latency. - -The updated loader and image then completed a fresh TPC-H SF1 load in 24.94 s -and SF10 in 81.30 s. Those wall times include generation, upload, eight table -snapshot commits, and remote manifest/footer/sample-scan verification. SF10 -uploaded 3,635,609,132 data bytes; all eight tables and 86,586,082 rows were -verified. The access upload metrics recorded zero failed requests and a peak of -four active mirror writes. Previous runs on this host took about 50 s and -232 s, respectively. These single-run totals show a material end-to-end -improvement, but they do not isolate upload-only wall time or constitute a -repeatable benchmark distribution. - -### Native small-object profiles - -A 2026-10-02 comparison used the same local single-node, one-mirror, -1-MiB-strip null-DiskIO fixture. EC conversion and Iceberg GC were disabled. -Every measured group uploaded 128 distinct paths with Content-MD5, then read -back every file. Concurrency was one or 32; the 32-client groups followed -128 warmup uploads at that concurrency. The reference revision was cd5bede8; -the measured implementation (c27f6fce) uses exact native owners, inline checksums, -owner-view packing, 32-strip prefetch and bounded concurrent transport frames. -These are individual process runs, not a throughput distribution or physical -SSD benchmark. Sequential readback and background maintenance change pipeline -state between groups; keep that limitation when comparing workloads. - -**Reference** - -| Payload | Clients | Elapsed s | p50 ms | p95 ms | p99 ms | GET / CAS / scan | -| ------------ | ------- | --------- | -------- | -------- | -------- | ---------------- | -| 1KiB | 1 | 5.264 | 40.451 | 45.914 | 58.831 | 501 / 128 / 18 | -| 1KiB | 32 | 0.393 | 79.955 | 100.023 | 106.175 | 384 / 128 / 0 | -| 512KiB | 1 | 14.434 | 109.813 | 151.723 | 157.257 | 658 / 128 / 42 | -| 512KiB | 32 | 9.575 | 2242.789 | 3572.333 | 3905.036 | 581 / 128 / 30 | -| aligned64KiB | 1 | 11.569 | 88.975 | 104.206 | 154.668 | 618 / 128 / 36 | -| aligned64KiB | 32 | 1.083 | 209.547 | 306.681 | 337.257 | 403 / 128 / 3 | - -**Native owner implementation** - -| Payload | Clients | Elapsed s | p50 ms | p95 ms | p99 ms | GET / CAS / scan | -| ------------ | ------- | --------- | ------- | -------- | -------- | ---------------- | -| 1KiB | 1 | 5.268 | 40.650 | 45.195 | 51.810 | 483 / 128 / 15 | -| 1KiB | 32 | 0.421 | 85.970 | 103.340 | 112.044 | 384 / 128 / 0 | -| 512KiB | 1 | 11.665 | 88.193 | 117.547 | 125.175 | 609 / 128 / 33 | -| 512KiB | 32 | 4.337 | 956.139 | 1515.353 | 1716.117 | 464 / 128 / 12 | -| aligned64KiB | 1 | 9.606 | 71.214 | 118.007 | 125.484 | 560 / 128 / 27 | -| aligned64KiB | 32 | 0.986 | 191.480 | 298.624 | 317.061 | 403 / 128 / 3 | - -The 512-KiB 32-client sample is 2.21 times faster; the 1-KiB 32-client -sample is 7.1% slower, while its sequential sample is essentially unchanged. -The aligned 32-client sample improves by 9.0%. These samples establish neither -universal improvement nor a stable tiny-object regression distribution. -All 768 measured uploads and readbacks succeeded. Each group recorded 128 -small completions, zero large strips and 128 writer handoffs. Actual native -frame preparation was 128 frames for 1 KiB and aligned payloads, and 1,152 -frames for 512 KiB; all cases allocated exactly 128 receive owners. - -For 128 512-KiB uploads, socket-direct payload bytes plus accepted HTTP prefix -copies sum to 67,108,864 bytes. Prefix copies were about 927 KiB per group; -all 1-KiB payloads arrived in the parser's prefetched prefix, totaling 131,072 -copied bytes. These are the accepted HTTP parser copies, not strip coalescing. -Pointer-identity tests independently check owner views and frame layout. - -The 1-KiB concurrent group used one pipeline and eleven batches, with no -reservation or scale-out during timing. Summed writer-finish time was 1.685 s; -summed file-publication time was 5.851 s. These concurrent stage sums are not -wall time. File publication and transport latency remain material even after -payload copies and per-object digest tasks are removed. The common-path -location publication still performs exactly one CAS per file. Cluster-wide -GET and scan deltas include maintenance; the quiet 1-KiB window observed -384 GETs and 128 CAS operations for 128 files. - -A complete zero-copy aggregate can exceed the transport's 16-view descriptor -bound. DiskIO submits its disjoint ranges at bounded concurrent depth within -the same batch and drains outcomes before one fsync and cursor publication. -Waiting for each subdivision separately doubled tiny-object concurrent time -in a diagnostic run; bounded overlap removed that doubled regression. - -### Consolidated acceptance - -After the native owner implementation, consolidated acceptance passed the full -Iceberg library and default Access Server suites, nine native HTTP scenarios, -three native catalog/namespace recovery cases with the pinned PyIceberg client, -file storage restart, all nonignored GC control/capacity cases, and eighteen -real-process shared-writer cases. These cover cross-strip readback, mirror repair, -conversion, restart takeover and elasticity. Separate acceptance fixes preserve -visible commit retries in the existing TableHead CAS, reduce retained response -reservations, and terminate small pipelines after a pre-batch allocation failure. - -The final 5-MiB release profile recorded PUT 82 ms (3 GET, 1 CAS), UploadPart -69 ms (5 GET, 1 CAS) and GET 24 ms (4 GET, no CAS). PUT and UploadPart each -recorded zero small completions, six large strips and six writer feeds; PUT used -one Chunk location. This is another null-DiskIO sample, not an SSD result or a -stable latency distribution. The native path assembles socket reads into 1-MiB -receive buffers before issuing large writes. - -The final release loader run used TPC-H SF=0.01 with eight upload workers and -TPC-DS SF=0.01 with twenty-four workers against the same single-node fixture. -All eight TPC-H and twenty-four TPC-DS tables uploaded, committed and passed row -count verification, with no observed 503. The combined test completed in 61.94 s; -that time includes cluster startup, generation, uploads, commits and verification. -Run IDs were `ff518ccf7029449cb7fae2d7fc0491ae` and -`0c321c0991ae44b0b1c8b7182d46ac09`. It is a functional concurrency acceptance -sample rather than an upload-only throughput comparison. - -The ignored exhaustive crash matrix and additional Java/engine integration -profiles were not rerun in this acceptance batch. The suites above and the -recorded profiles establish the tested scope. diff --git a/doc/design/access-server/iceberge/design-crowdb-iceberg.md b/doc/design/access-server/iceberge/design-crowdb-iceberg.md deleted file mode 100644 index 3703d9880..000000000 --- a/doc/design/access-server/iceberge/design-crowdb-iceberg.md +++ /dev/null @@ -1,1291 +0,0 @@ - - - -# CROWDB - Design: Native Iceberg Storage - -Iceberg is CROWDB's HTTP catalog and table-storage access model. Catalog, -namespace, table, snapshot, commit, and file concepts are first-class CROWDB -storage authorities, not management metadata layered over S3. - -Depends on: [Access Server](../design-crowdb-access-server.md), -[Chunk I/O](../../chunkio/design-crowdb-chunkio.md), and -[Chunk-KV](../../chunkds/design-crowdb-chunk-kv.md). - -The backed-up [REST Catalog OpenAPI](iceberg-rest-catalog-open-api-1.11.0.yaml) -and [Table Specification](iceberg-table-spec-1.11.0.md) are normative. - -## Table of contents - -1. [Intent and boundary](#1-intent-and-boundary) -2. [Authority model](#2-authority-model) -3. [HTTP and FileIO surfaces](#3-http-and-fileio-surfaces) -4. [Commit and lifecycle](#4-commit-and-lifecycle) -5. [Compatibility](#5-compatibility) -6. [Relationship to other access models](#6-relationship-to-other-access-models) -7. [Correctness invariants](#7-correctness-invariants) - -## 1. Intent and boundary - -The Iceberg module lets standard Iceberg engines use CROWDB without first -modeling tables as ordinary S3 objects. It terminates an independent HTTP -listener and owns Iceberg request, authentication, error, compatibility, and -lifecycle behavior. - -The module does not become a query engine and does not own physical placement, -replication, erasure coding, repair, or disks. Server-side scan planning and -table processing are separate capabilities rather than requirements of the -core storage authority. - -## 2. Authority model - -Catalogs contain multipart namespaces; namespaces contain named tables; tables -select immutable Iceberg metadata generations; metadata and snapshots reach -immutable table files. Stable identities separate durable authority from -renameable names. - -Standard Iceberg metadata is the recoverable table state. CROWDB may maintain -derived indexes or projections for scale, but they are disposable and cannot -become a second table authority. - -Table ALL/REFS loads read and verify canonical storage, run the complete bounded -metadata parser, and recheck the namespace and selected head before responding. -ALL preserves exact canonical bytes; REFS filters snapshots from the same validated -content. Table loads do not read or write projection pages, validation receipts, -or other persistent cache state, including on repeated and conditional requests. -Optional generation-local JSON projections used by file-level selection remain -disposable derived data with canonical fallback; they cannot replace table -validation, participate in commit proofs, or become file/publication authority. - -Iceberg metadata stores bounded logical records and opaque data references. -Physical chunk placement and storage topology remain below the access boundary. - -### Identity directory and file authority - -The Iceberg key codec carries its storage family and layout version separately -from the directory payload. A table-owned payload begins with -`///`; it has no redundant -`iceberg`, `namespace` or `table` labels before those IDs. The storage namespace -is frozen at table creation, including staged creation. The default namespace -has a durable ID under the same rules as named namespaces. - -```text -/ -├── head -├── namespace/name/ -> namespace-id -└── / - ├── head - ├── name/ -> (storage-namespace-id, table-id) - └── / - ├── head - ├── file/ - └── projection// -``` - -Catalog and namespace control records have their own typed scopes. Namespace -names use the existing unambiguous multipart-name encoding. Table-name lookup -returns the complete storage identity; internal tasks carry it as well, so no -table-ID reverse index is needed. A move updates live name mappings and the -head's live identity/epoch while retaining the storage path. Deleting the old -namespace cannot cascade into moved tables, and IDs are never reassigned. - -Each file has one `file/` record. The suffix is also -its public object-key suffix. Its value contains FileId, canonical length, -format, integrity metadata, native descriptor/root and lifecycle state. FileId -is a value/locator binding, not another file index. One create-only CAS publishes -the complete record after durable bytes; exact-key conflicts cannot overwrite -selected content. Retries preserve the selected file identity and URL. Workflow -records exist only where recovery or multipart/GC progress requires them, not -as three mandatory records for every file. - -Service recovery journals are separate from that directory. Their existing -catalog control scopes remain explicitly versioned binary keys, even in a -native catalog. They are not file lookup indexes and cannot produce an object -listing. The service must discover abandoned work before a table head exists -and after a table has disappeared, without a table-ID reverse index: - -- Legacy creation, commit and drop journals and payload pages are keyed by - operation identity. Their selected heads carry the frozen storage identity. - New creation and staging keep complete input only within the request. - Name moves use bounded namespace occupancy instead - of request journals; existing rename journals remain recoverable. -- Multipart sessions and parts are keyed by upload identity/part number. Catalog - admission records apply only to legacy credited sessions; new streamed sessions - use independent per-session bounds. - Every session and part carries and validates the full storage identity. -- Write intents/fences track allocated but potentially unpublished native - extents, including failed creation. Catalog maintenance scans their bounded - control scopes; the value binds the full table/file owner and exact chunk root. -- GC tasks, candidates, claims, proof nodes and pages belong to the recovery - worker. A retirement task can cover several tables and adopts interrupted - purge work. Claims coordinate physical deletion across tasks/generations; - candidate values retain the canonical filename and full storage identity. - -These records are retained only for retry, fencing and reclamation. They do not -add another published file authority or participate in direct GET/HEAD. GC data -discovery chooses v1 or v2 from the persisted catalog layout; independent -service-control scans never merge the two file namespaces. Head, file and -projection records use the native table directory. Historical native injection -receipts remain decodable for reclamation; uploads create no request receipt. - -### Standard metadata and bounded scale - -TableHead selects an immutable standard metadata JSON file. Its `snapshots` -array contains snapshot records and its `refs` map selects branches/tags; -snapshot-log and metadata-log retain their standard meanings. Snapshot records -reference manifest-list files, which reference manifest files, which describe -data/delete files. Reading v1 also supports its direct `manifests` alternative. -The backed-up v1/v2/v3 specification determines fields, sequence inheritance, -schema/partition evolution, delete matching and row lineage. No authoritative -snapshot or manifest KV index substitutes for those standard file contents. - -All of these file kinds use the same single file-record model. Large metadata -JSON, manifests and manifest lists are native chunk-backed objects. KV values -and TableHead never embed an unbounded metadata document or snapshot array. -Small eligible metadata may use the bounded inline representation; larger -objects and large extent sets use chunk payloads and bounded directory roots. -Changing storage representation preserves the canonical bytes returned by -FileIO. Table publication selects a completely durable metadata generation by -CAS; a losing commit cannot expose a partially written generation. - -Metadata files use useful public relative prefixes such as -`metadata/.metadata.json`, `metadata/.avro` and -`statistics/.puffin`. These are defaults, not restrictions on safe -client-chosen paths. The optional generation-bound projections described above -support file-level selection; they are not used by table ALL/REFS loads or duplicate -file authorities. Parser, collection and response budgets remain explicit even for -chunk-backed JSON. -An unsupported oversize request fails clearly rather than silently truncating -snapshots. ALL and REFS keep their standard response semantics; internal -projection paging does not introduce private REST pagination fields. - -New catalogs use the identity-directory layout version. Existing legacy -catalogs retain their own decoder, keys, URLs and read/write behavior. Layout -selection is catalog-bound; relabeling or mixing key versions in one scan is -rejected. In-place migration is not implicit in decoding. - -### Catalog foundation - -One active root selects a random stable CatalogId and activation epoch. Display -rename updates its authority without moving descendant keys. System-scoped -management receipts, audit and retry bindings survive catalog replacement; -resource records and retained REST response bodies are catalog-scoped. - -Initialize, rename, capability activation and clear use bounded single-key CAS state machines, not a -global lock or a multi-key transaction. A root retains the operation identity -until its durable outcome and audit can be recovered by any instance. Clear -fences admission, records a maintenance observation after the durable fence, -publishes an empty replacement under maintenance, and persists the grace proof -before reopening admission. Completion uses persisted lease, request, delegated -access and clock-skew limits, never shorter restart configuration. Retired -authorities remain unreachable; bounded retired-catalog cleanup follows the -persisted retention and owner proofs. - -Format capability bits are durable catalog authority. Zero means no advertised -table format service; startup never rewrites or widens a legacy zero profile. -An authenticated management operation explicitly activates a validated profile -under the root fence. It preserves the catalog ID, activation epoch, table keys, -name generation and admission bounds, advances config generation, and may only -add support. A resumed operation replays its original profile and audit result. -Clear creates a new zero-profile catalog that requires separate activation. - -The baseline has no root lease. Each HTTP connection closes after five minutes -without network progress; active streamed file bodies and multipart completion -heartbeats extend the idle deadline. Request dispatch has a separate deadline -starting at connection acceptance; incomplete request headers close at that -deadline, while an active response remains governed by network idleness. REST -and FileIO admission reject configured request timeouts exceeding the persisted -catalog request bound. Newly initialized runtime catalogs use a five-minute -request bound; -new catalogs also persist a fifteen-minute delegated-access bound. Restart never -increases persisted bounds. Explicit catalog clear may expand them componentwise -under the maintenance fence and waits the resulting full grace before admission; -neither clear nor smaller restart settings can shorten existing bounds. -Listeners stop admission before bounded draining; -startup and periodic reconciliation resume interrupted management operations. - -Management and audit ledgers use 4096 deterministic fast-hash slots with -exact-identity overflow keys. An occupied slot routes a different identity to -its own durable key. Their retention and authority checks belong to the -management operation contract. Client write identities use UUIDv7 issuance -time with a 30-second future-clock allowance; requests without a client key -receive independent internal identities. - -REST namespace and table writes do not allocate a shared HTTP retry binding -or cache terminal HTTP results. Each request validates its current authority -and input. Existing business drivers bind operation identities to their stable -resources and validate their own request preconditions; direct expected-base -table updates retain their head CAS contract. A failed request without a -business operation can be submitted again after the resource state changes. -Legacy HTTP bindings, overflow keys and response payloads remain decodable -for retention cleanup, with no producer on new request paths. - -Immutable operation payloads use 32-KiB pages with a 2-MiB aggregate limit. Each -reference binds catalog, operation, content digest and total size; readers validate -every page and the complete digest. Business operation drivers use these -references for their existing inputs and results. Legacy large HTTP response -manifests remain readable by cleanup; independent requests do not create them. - -New namespace creation writes only its final name reservation and namespace -authority. Before publishing a nested child, it places that child's ID in the -parent's `creating_child` occupancy; the child authority is already complete. -Parent deletion must settle or cancel this occupancy before proving emptiness. -Publication clears it through an exact parent update, preserving concurrent -properties. Top-level creation needs no parent occupancy. Helpers resolve an -interrupted create from these resource facts, without an input or response -journal. Cancellation and retired-catalog cleanup permanently retire the same -child authority slot before releasing its exact name reservation, including an -absent slot. A delayed first authority write therefore cannot resurrect it. -Occupied parents use a format that older writers reject. - -Legacy namespace creation, property and drop journals occupy a separate key -scope from HTTP responses. Retained records bind the request and stable target, -immutable mutation snapshots, phase revisions and bounded child-probe cursors. -Recovery validates their phase transitions, snapshot identities and cursor ranges. -Fresh namespace writes do not create these journals or payloads. - -Authoritative namespace reads resolve each parent/name mapping against the -selected stable authority and full canonical identifier. Reservations, stale -epochs, missing targets and tombstones are not visible; corruption is an error. -Active-context checks bracket resolution so retirement cannot turn an old-domain -lookup into a response from the replacement catalog. - -Property updates apply the request's assignments and removals directly to the -current namespace authority through one conditional write. This advances the -property and mutation revisions once, preserving the name epoch and admission -fence. A definite conflict supplies the next authority; the request reapplies its -changes while preserving other properties. Deletion, changed identity or a -pending lifecycle operation blocks that update. Active-context validation -brackets resolution and checks completion. Retry work is bounded. - -Property requests persist no input, operation journal, marker, response payload -or HTTP retry binding. A request that leaves the properties unchanged performs -no write. A lost reply is returned as an error; the client can submit its changes -again. Each response describes that request's actual removed and missing keys, -so a repeated removal can move from removed to missing. Repeating assignments -can overwrite a concurrent assignment to the same property. Other properties -are retained. Old property journals remain readable by recovery, including the -holder-bound marker dispatcher. Namespace creation and drop likewise use -resource facts without an input or response journal. -The catalog configuration does not advertise a universal idempotency-key -lifetime, since independent property updates do not retain operation responses. - -Legacy namespace creation recovery installs a parent/name reservation before a -parent authority CAS. Its nested admission leaves a pending-operation marker -and advances only the mutation revision; a top-level admission conditionally -validates the active root without replacing its management operation. Helpers resolve uncertain parent -writes before allowing subsequent parent mutation. Definitive admission conflicts -may retry with fresh parent snapshots while retaining the name reservation. -Publication selects an initial authority and replaces the reservation with its -published mapping; a creation marker remains until the result is durable. Abort -records retain their exact failure outcome before conditional reservation cleanup. -Recursive creation helping shares one bounded phase budget. Root-admission -backend identities include the individual creation operation, so a different -creator cannot reuse a cached no-op CAS result from before its reservation. - -Namespace drop must reject a nonempty namespace. Before fencing the selected -namespace, deletion settles its child-creation occupancy, rename occupancy and -legacy pending markers. These resource facts protect child publication and name -moves while the parent is checked for emptiness. - -Each drop selects the current namespace binding once. An exact CAS changes a -Ready authority to ownerless Dropping, advancing the admission fence and -mutation revision; an existing Dropping authority can be helped without another -fence. Lifecycle tag 4 identifies this state and requires no pending operation, -child occupancy or rename; older decoders reject it. Dropping remains readable -and counts as a live child when an ancestor is being deleted. - -Deletion scans both direct-child index ranges through bounded pages, validating -range, ordering, continuations and child bindings. Reservations are helped and -stale mappings are conditionally removed; corruption blocks the empty proof. A -live namespace or table child restores the exact Dropping authority to Ready, -advancing its admission fence and mutation revision while preserving properties, -name epoch and property revision, and returns a nonempty error. Only completion -of both ranges for the same exact Dropping authority permits its CAS to -permanent RetiredNamespace at the same authority key. The retired slot prevents -delayed authority creation from resurrecting that namespace identity. - -Scan cursors and empty proofs exist only within the current invocation. A CAS -conflict supplies the next validated authority without a confirmation GET. A -changed Dropping generation discards both range cursors and the old empty proof -before rescanning; a Ready authority restored by another helper ends that drop -attempt. Unknown write results return an error immediately without a follow-up -read or write. An installed Dropping fence survives cancellation, timeout and -exhausted work budgets, so a later request or mapping-repair pass can restart a -complete proof. No input, phase, scan cursor or response record is persisted. -Active-catalog validation brackets selection, precedes each authority transition -and checks completion. - -Terminal cleanup conditionally deletes only the raw name mapping selected by -that invocation. It cannot delete a replacement NamespaceId at the same name. A -fresh client retry selects the current name again and may delete a recreated -empty namespace; a nonempty replacement still fails. A missing current name -returns absence rather than a cached terminal response. Legacy owner-bound -lifecycle states and journals retain their recovery and exact-target cleanup. - -Current property and rename-admission CAS conflicts with the same-key retired -slot report a conflict without another authority read. Retained legacy work -treats that slot as final parent absence and aborts work not admitted before -retirement. Cleanup still fences an absent table-head slot before removing its -exact reservation. Captured authority payloads keep their identity and owner -validation; a retired current parent does not make an invalid capture acceptable. - -Table-child probes resolve published mappings against the selected table head. -Ordinary table creation leaves only a reserved name and its final head slot. -Probes retire an absent slot before clearing that exact reservation, or settle -the matching Ready head and reject deletion as nonempty. A creator reserves its -name before its last Ready-parent check, so a concurrent drop cannot both miss -the reservation and permit a later first head. Retained staged operations and -legacy lifecycle journals remain recoverable through their exact markers. -Stable namespace identity prevents old state from becoming visible under a -recreated namespace. Name moves occupy both participating parent -authorities before reserving the destination; deletion helps settle or fence the -move before taking a fresh emptiness proof. Corrupt -table authority blocks the emptiness proof. -Each listener alternates bounded recovery of retained namespace journals with -mapping repair, using local page cursors, work budgets and a wall-clock deadline. -Mapping repair helps child-creation reservations and ownerless Dropping resources, -including a full new child proof after an interrupted deletion, then conditionally -removes bindings disproved by authoritative state. Catalog changes invalidate -local cursors; cancellation preserves resource facts. Corruption and unresolved -reservations never authorize deletion; no sweep physically removes file bytes. -The listener exposes authenticated namespace listing, load and exists routes. - -Namespace list pages scan bounded direct-child ranges and validate each published -mapping against its authority and canonical parent spelling. Reserved and stale -entries are omitted; corruption fails the page. HMAC-authenticated continuations -bind the catalog activation, stable parent identity, spelling, page size and last -scanned key. A stale-only page can therefore be empty while retaining a token. -Unpaginated lists build a complete in-memory spool before success headers, capped -independently at 2 MiB, 1024 results and 4096 scanned mappings. A shared -128 MiB byte budget admits response spools, reserving each operation's maximum -while constructing the response (2 MiB for namespace lists and 4 MiB for table -responses). Serialization shrinks the reservation to the retained buffer capacity; -the response releases it on cancellation or completion. Atomic admission rejects -excess work without waiting. The request -deadline bounds construction before success headers. Dispatch stops before that -deadline, reserving the smaller of 100 ms or 10% of the request timeout for -emitting a bounded error response. Header receipt does not restart this budget. -A stalled transport closes after the independent idle timeout. Cancellation -drops the spool permit. Completed -responses stream in 16-KiB frames. Absent page tokens request complete results; -empty page tokens begin paginated mode. Tokens use a domain-separated signing key -derived from the configured credentials so equally configured listeners interoperate. - -## 3. HTTP and FileIO surfaces - -The REST Catalog is the portable control surface. It exposes only capabilities -CROWDB implements with compliant Iceberg semantics. - -The listener classifies complete method/path pairs before domain mutation and -uses the same fixed route set for discovery and protocol metric labels. Request -measurements use bounded atomic counters and follow response bodies through -completion or cancellation; streamed file bytes are measured when emitted. -Protocol counters never use principal, table name, token or raw path as a label. -An authenticated management-credential-only `GET /_crowdb/metrics` exposes a -bounded snapshot, including while catalog storage is unavailable. This local -diagnostic is not an Iceberg REST endpoint and is absent from `/v1/config`. - -The catalog listener exposes authenticated config and namespace REST. An absent or -empty warehouse selects the sole active catalog; other selectors fail with -`NoSuchWarehouseException`. Its endpoint list advertises installed namespace and -table read/create/commit/lifecycle/credential routes only when the persisted -format profile permits them. A zero-profile catalog returns unavailable config -rather than publishing a misleading set of false overrides; table routes reject -until management activation. Selected table versions gate load, HEAD, credential -refresh, create and commit. Version upgrades require each intermediate edge, -including direct v1-to-v3 requests. FileIO bytes alone do not identify a table -file's semantic kind or grant format-version authority. File grants intersect -the principal role with the selected version: published tables require write -support for upload permission, while staged drafts require create support. -Runtime table routes require a persisted delegation bound of at least fifteen -minutes. Legacy catalogs below that bound retain foundation-only service; activation -requires an explicit clear with expanded bounds and a listener restart after the -maintenance grace. Namespace mutations, table creation and lifecycle operations -use retained UUIDv7 retry identities bound to route, input, principal and catalog -activation. Ordinary existing-table commits publish through one head CAS without -a separate durable retry-result ledger; a retry whose result is hidden by a later -generation may report an uncertain outcome. Server errors -remain retryable, never terminal ledger outcomes. Exhausting the configured request -deadline leaves durable recovery evidence; subsecond completion is not guaranteed. -Static bearer credentials -separate reader, writer, management and clear roles; this is not an OAuth token -issuer. All four credentials are required and distinct. Writer has a separate -namespace-write capability and no catalog management or clear privilege; reader, -manager and clearer do not inherit namespace-write rights. All four can read the -configuration endpoint and namespaces. Only writer may invoke namespace or table mutations. -Management commands are separate from the Iceberg REST listener. Operational -configuration follows the selected deployment profile and its startup inputs. - -Iceberg FileIO uses reserved S3-shaped locations so existing Iceberg clients can -address immutable metadata and data files. The shape is a compatibility -contract, not delegation to the general S3 authority. File publication, -immutability, authorization, and deletion remain under Iceberg control. -Public URIs use the fixed virtual bucket `iceberg`: -`s3://iceberg////`. -Native typed locations use lower-case fixed-width hexadecimal for all three -128-bit IDs. Legacy compatibility buckets retain their base32 catalog component. -Relative UTF-8 object keys preserve case, literal percent signs, plus -signs and repeated internal slashes; the whole object key is bounded to 1,024 -bytes. Dot traversal, leading slash, backslash, controls, query and fragment -delimiters are rejected rather than normalized. HTTP percent decoding belongs -only at the transport boundary, not in stored S3-shaped locations. - -Native `ListObjectsV2` retains this address model. A list request must specify -the complete `///` prefix or a -descendant prefix and a table-bound delegated list capability; -catalog-wide discovery and general S3 credentials are rejected. Reader and -writer FileIO credentials include listing; cleanup-only credentials do not. -Exact-object HEAD/GET/PUT remain independently usable. PyArrow missing-file -probes may issue listing to distinguish a missing object from a directory. - -Empty, catalog-only, namespace-only and foreign-table prefixes fail before a -scan. HeadBucket/GetBucketLocation expose only the configured virtual bucket -and region to a valid grant; ListBuckets returns AccessDenied, and bucket -creation/deletion is unsupported. The URI host determines the bucket. Arrow -FileSelector adds a trailing slash to its key prefix and uses `/` as delimiter -for nonrecursive listing; an exact HEAD miss may probe `/` without -widening scope. These behaviors follow the -[Arrow S3 implementation](https://raw.githubusercontent.com/apache/arrow/apache-arrow-25.0.0/cpp/src/arrow/filesystem/s3fs.cc). - -Listing enumerates selected, fully published immutable file-location records, -including files awaiting a table commit or later reclamation. It excludes -incomplete uploads, unselected draft/losing candidates and logical deletion -markers. It does not enumerate the files reachable from a current snapshot; -clients use Iceberg metadata and manifests for that selection. Names, lengths -and ETags come from the selected record. Since file records do not store a -wall-clock publication time, listing returns the fixed Unix epoch timestamp; -clients must not use it as a file-age or reclamation signal. - -Each page scans at most 256 records and 4 MiB, possibly returning fewer than -the requested maximum of 1,000 keys. The supported delimiter is `/`; URL -encoding, zero-key pages and table-scoped `start-after` are supported. XML is -bounded to 2 MiB. Unsupported or ambiguous selectors fail before scanning. -Pagination is a forward live traversal rather than a snapshot: unchanged -eligible sets have no omissions or duplicates, publication behind the cursor -is seen by a new traversal, deletion ahead removes that entry, and publication -ahead may appear. Common-prefix rollups advance past the entire subtree. -Authenticated continuation tokens bind the principal, credential nonce, -catalog epoch, table, prefix, delimiter and encoding. Their fixed expiry is -no later than the originating grant, and invalid or foreign tokens fail before -storage access. A credential refresh requires starting a new traversal. - -List operates directly on `file/` records in public-key order. For example, -the public prefix `/metadata/` maps to the KV prefix -`/file/metadata/`; filename prefixes narrow that same range. -Pagination uses encoded prefix bounds and the last examined key, including -filtered tombstones and sparse pages. It never materializes or sorts the whole -table, opens metadata objects, scans unrelated file prefixes or enumerates -snapshots inside JSON. Returned keys omit the internal `file/` component. -The same limits apply whether a table has ten or millions of metadata files. - -Native file records bind FileId to exact location, kind, format and canonical length. -Legacy tree and inline records also bind a whole-file SHA-256 digest; streamed -records bind a bounded array of complete Chunk locations and an HTTP ETag, and -rely on verified 64-KiB storage frames rather than a whole-file digest. Eligible -metadata stores at most 16 KiB inline; bounded LZ4 -compression considers at most 64 KiB original input, and decoding verifies the -canonical length and digest. Location vectors are validated for exact logical -coverage and bounded by the record limit. Hints are non-authoritative and out-of-bounds -hints are ignored. One exact-location key stores the complete file descriptor. -Publication uses one create-only CAS without a preliminary existence GET; equal -content is resolved from the CAS conflict value. Losing candidates cannot -overwrite the selected bytes. Legacy file-ID/mapping pairs remain readable for -recovery. Prefix scans of current records require no file-ID join. -An SDK upload supplies a path and bytes, not the eventual Iceberg data/delete -use. FileIO treats client-supplied bytes as opaque. It verifies the declared -transport checksum and durable frame writes before publishing the descriptor; -format interpretation belongs to the client or to a CROWDB component that -constructs those bytes. Commit validates the selected table metadata and request -requirements in memory; it does not traverse client manifests or perform -per-file reference checks. Format validators remain available to components -that explicitly construct or examine those files. -The isolated native HTTP surface exposes signed immutable object reads/writes, -multipart operations and separately authorized single/batch cleanup, without -a general S3 bucket authority. - -### Native injection and direct reads - -Native injection finishes chunk writes and its bounded descriptor, publishes -one file record, then returns one canonical URL before constructing manifests. -For parquet, the public filename is -`f/v1/..parquet`. The URL-safe, dot-free locator carries -a unique FileId, length, integrity information and complete stable native -location or chunk-resident directory root. Its versioned signature binds the -full table identity, readable stem and unsigned locator fields; the signature -bytes are excluded from that signing input. Repeated readable stems are harmless; -a full upload retry can return a new filename for equivalent contents. The -client retains the returned URL for its eventual metadata commit. Unreferenced -copies are discovered through file and physical range ownership facts. No -operation record selects or replays an upload response. Locator and stem -together must fit the object-key limit. - -Ordinary SDK writers choose logical filenames before S3 PUT and keep those -URLs. A PUT-time allocation does not change a client's chosen path. These -files resolve their one file record; no response field or manifest rewrite -converts them into direct locators. Metadata/Avro files may retain useful -logical prefixes regardless of their native object size. Position-delete and -deletion-vector path references always use the original canonical URL. - -Direct HEAD/GET/Range GET validate request permissions and locator authenticity -and read the native descriptor without Iceberg file-location or file-tombstone -KV queries. Chunk-layer layout resolution remains available for replica/EC -routing, cold reads, repair and migration. A KV-resolved opaque file token is not a direct -locator. Large files use bounded chunk-resident directory traversal, not an -unbounded URL. Native chunk routing preserves repair/failover without embedding -transient replica addresses in URLs. Reclaimed identities and reused extents -must never make an old locator return a different file's bytes. - -Native trees retain an owner-bound directory root even when they contain one -leaf. Its digest and encoded full file identity distinguish an identical-content -replacement at a reused extent. Direct request preflight checks that root before -HEAD or conditional success, and GET reuses its verified bytes in the pull reader. - -Authorization uses bounded validated grant/context caches and signature, -scope and expiry checks. An authorization cache miss may load authority; it -must not become a per-file location/state lookup. Expired entries are not -accepted during refresh failure. Cache entries bind principal, catalog epoch, -full storage identity and operation rights, with validity bounded by the grant -and applicable authority lease. Fresh credential issuance still checks live -authority. Persistent locators have no credential expiry; retain signing-key -verification material for their files' lifetime. A locator is not a bearer -permission. Cold/warm authorization costs and storage resolution counts are -measured separately. - -Native catalog authority persists the default locator verification/signing key, -independent of access tokens. Key version 1 is reserved for that catalog key; -renaming and capability activation preserve it. Additional signing versions are -configured through `iceberg.locator_signing`, with an active version and retained -verification keys. Configured version 1 must match the persisted catalog key. -Every serving node uses the same configured versions and retains them until the -corresponding files are physically reclaimed. A lock-free cache selects the -current catalog's key ring; clearing to a new catalog selects its new default key. -Native authority uses record envelope 2 so a legacy server cannot silently ignore -the storage layout. Legacy authorities retain envelope 1 and omitted layout -fields decode as the legacy layout. - -Credential refresh URLs carry the table ID and frozen storage namespace ID. -The listener resolves that immutable head directly, so refreshing an old endpoint -after a move or namespace recreation does not require a table-ID reverse index. - -Legacy chunk-backed files use bounded leaf blocks and immutable chunk-resident directory -pages, with at most 256 children per page and eight directory levels. Each page -binds its catalog, table and file identity, child heights and covered byte count. -The writer retains only one partial leaf and bounded per-level frontiers. Native -block completion waits for the readable chunk cursor before publishing a root. -Pull readers retain one leaf and its current verified leaf-directory page, -bounded to 32 KiB independently of file length. Directory reuse is reader-local, -bound to the exact immutable root, and never shared across files or generations. -Readers verify directory/leaf digests and read no future block until requested; -full-file reads also verify the canonical digest. Range -parsing accepts one contiguous interval and rejects multiple ranges explicitly. -The HTTP pull-body adapter adds shared response admission and 16-KiB frames. -Only body polling starts a storage read; cancellation drops the in-flight read -before releasing admission. Exact remaining-byte hints track delivery, and storage -errors terminate the body rather than returning a successful truncated stream. -Legacy writer checkpoint decoding remains only for reclaiming existing physical -frontiers. GC verifies owner identity, storage checksum, frontier heights and -complete byte coverage before deleting any child. Foreground writers do not -produce or restore byte checkpoints. Versioned digest state in legacy records is -trusted-storage metadata, never a client authentication assertion. -Legacy native block writes persist an exact physical-range ownership intent in the -catalog before DiskIO. A shared-writer callback receives the assigned location; -an uncertain catalog reply stops the physical batch without a confirmation GET. -Explicit retries reconcile the existing range fact. This ledger also covers -process loss before file publication and legacy checkpoint ranges. Reclamation waits for the chunk readable -cursor or terminal state to settle any unconfirmed physical write. -Staged-tree readers validate physical roots, byte lengths and digests without -assigning a semantic file kind or declaring an incomplete multipart fragment to -be a valid complete-format file. Published-file reads retain record validation. -The legacy tree assembly byte engine consumes complete selected parts in ordinal -order within one request. It preflights the entire selection against the -intersection of request, grant, service and upload file limits before physical -copying. The copier retains bounded read frames and writer frontiers, verifies -each full part digest, and emits no intermediate checkpoints or progress CAS. -One next frame read can overlap the current writer push; either failure cancels -the other future and poisons that request-local copier. A complete-input retry -starts a new copier, while unowned physical ranges remain discoverable by GC. -Multipart session/part models retain independent resource limits and validate -phase coherence: publishing requires complete candidate bytes, published outcomes -require a selected FileId, and abort retains completion evidence without claiming -publication. Their FlatBuffers envelopes bind session and part identities to -separate catalog key scopes, retaining only bounded checkpoint references and -current-part digest state. Unknown phases and invalid revisions fail closed. -New multipart sessions publish their own authority without a catalog-wide -credit reserve or release. Persisted per-session limits bound staged bytes and -part count. Global credit reserve/release/journal drivers have been removed, -including compatibility write hooks. New sessions reject legacy credit input; -existing session and policy formats remain decodable. Recovery never reads or -settles a global policy. A legacy pending reservation without a session is ignored: -session admission preceded any permitted part writes. A pending credit release -does not change the published FileId or part references and is not replayed. -Existing sessions still expire or finish from their own resource facts; obsolete -policy/payload metadata remains available to legacy metadata cleanup without -keeping accounting alive or creating another upload. -The legacy tree multipart repository reserves one part mutation in the session before -changing its part authority. A bounded before/after snapshot and monotonically -increasing revisions make the write and fence release recoverable across servers. -Counts and current staged bytes are reserved once at the session CAS. Abort cannot -bypass an unresolved mutation; stale helpers cannot restore an older part. Abort -retains parts and completion evidence rather than deleting physical storage. -Legacy completion freezes an ordered part-number/revision/digest selection in immutable -payload pages, then changes the session phase by CAS to fence part replacement. -Selections are independently bounded to 10,000 entries and 420,007 encoded bytes. -A client Complete request copies the entire bounded selection in one attempt and -holds the existing upload concurrency permit through assembly and sealing. Its -300-second response-body deadline cancels all pending work. A selection exceeding -the request copy limit fails explicitly before any physical copy. Existing byte -checkpoint sessions reject continuation with an instruction to abort and create -a new upload; their original frontiers remain available to expiry and GC. -Assembled bytes remain unexposed until semantic sealing and immutable location -publication. -Streamed UploadPart writes one part key by CAS without changing the session for -each part. Create caps part count by the reserved staged-byte ceiling divided -by the per-part byte ceiling. Complete freezes an ordered selection with each -part's exact location bytes, length and MD5 ETag in bounded immutable payload -pages. A later replacement cannot change that selection. Completion composes -logical offsets across the selected locations and derives the multipart ETag -from the ordered raw part MD5 values; it does not read part data or assemble a -new chunk. Session phase CAS freezes publication against later selections. The -final file descriptor becomes visible through the immutable location publication -protocol. -A successful completion transition passes its exact installed session to the next -step; a definite CAS conflict passes the returned session observation. The HTTP -driver does not reload the session to confirm successful freeze, publication -preparation or completion. Stream publication reuses this request's selection -after checking its count and digest against the frozen binding, without reading -the immutable selection pages again. An uncertain part or completion mutation -returns its error without a confirmation GET. A client can retry its complete -part transfer or Complete request; the existing frozen selection and publication -facts continue to protect physical ownership and GC. -A recovery page scans at most four session authorities and performs one pending -legacy part settlement, logical expiry or frozen publication per session. It -validates the complete scan page before mutations, rejects foreign continuations, -and never starts or resumes byte assembly. Completing uploads wait for a client -request or expiry. Expiry retains physical parts and legacy frontiers and cannot -bypass an unresolved part mutation or a publication fence. -Publication freezes a caller-validated sealed file record in immutable payload -pages before the publication phase CAS. Recovery replays that exact record through -the immutable file repository and persists the selected FileId. Equal preexisting -bytes retain their original identity. Only a proven incompatible immutable location -permits the terminal Conflicted phase; uncertain writes and context failures do not -become false aborts. Canonical format validation remains the seal caller's contract. -Each native listener schedules this resource sweep independently of namespace -recovery, with one bounded scan and no copy-observation pass. Context changes -reset the cursor. Each session uses the persisted catalog request deadline; -timeout defers only that session so later entries can progress. A separate outer -budget bounds page and context work. Empty and terminal-only sweeps perform two -context checks and the scan, with no policy GET or mutation. There is no global -credit recovery or background byte-copy continuation. - - -Multipart part listing uses one upload-scoped scan with at most 256 records per -page. Numeric markers preserve gaps and resume strictly after the returned part -number. Current-session checks bracket each scan; concurrent mutations invalidate -the page rather than mixing pending counters with old part records. Expired or -terminal sessions and malformed storage pages are not reported as successful lists. - -Native HTTP upload staging holds a bounded concurrency credit. Known small -bodies receive an owner sized to payload plus 34 bytes per 64-KiB storage frame. -A per-object small writer computes declared MD5/SHA-256 synchronously as socket -views arrive, then hands the complete framed owner to the shared pipeline once. -The worker assigns chunk IDs and slices immutable views across strip boundaries; -ordinary mirror writes do not copy payload. The HTTP parser's prefetched-body -copy is measured by one bandwidth counter with count and bytes. -Large and unknown-length bodies receive through bounded 1-MiB owners. Routing -uses the configured fraction of a strip's data capacity, not socket frame size. -Unknown-length bodies choose a dedicated chunk. Per-pipeline disk completion and -readable-cursor publication finish before the next batch; requests accumulating -while publication waits form the next batch. Hash selects a preferred pipeline, -with atomic alternate-route admission before backpressure. Shared chunks default -to 256 MiB with 32-strip groups, asynchronously refilled at half remaining. -Allocation appends after reserved capacity, including hidden strips, and runs -outside the existing chunk publication guard. Exact length, declared checksum and -durable completion precede file publication. Failed uploads leave unpublished -allocation ownership to lower-layer recovery and applicable cleanup tasks. -See the [upload flow](design-crowdb-iceberg-upload-flow.md) for transfer details. - -Metadata JSON structural validation uses a bounded pull-reader bridge and an -ignored-value parser rather than retaining the metadata graph. A separate scanner -bounds nesting and verifies raw UTF-8 before parser scratch can grow. Admission -caps blocking workers; cancellation keeps its permit until the worker exits. -The validating full-file reader checks the canonical digest in the same storage -pass for chunked JSON; no independent preliminary full-file read is required. -This structural check does not replace Iceberg schema or commit validation. - -Avro writer-schema binary layouts compile to bounded named-reference graphs. -Decoded block validation checks datum widths, UTF-8, collection byte counts, -union/enum indexes and exact record consumption without retaining datum graphs. -Independent graph, recursion and visited-value limits also bound zero-byte values. -The record reader compiles its container's schema once, decodes one bounded block -per pull and permanently stops after failure or cancelled reads. Reader-schema -resolution and Iceberg logical/manifest semantics remain separate checks. - -Parquet and Puffin container probes derive footer ranges from canonical framing, -ignoring stored hints even when those hints happen to be in bounds. Their reads -retain one bounded leaf and only fixed-size framing bytes, independent of the -advertised footer size. Puffin probing also checks footer-start magic and reserved -flags. Container framing does not validate footer contents or data semantics. -Puffin metadata parsing separately caps encoded and decoded footer payloads at -1 MiB and bounds blob, field and property collections. It accepts plain JSON or -one sized, checksum-verified LZ4 frame and rejects overlapping blob ranges. Footer -deletion-vector descriptors validate their reserved snapshot/sequence markers, -uncompressed storage, referenced file and cardinality; manifest checks require -exact offset/length and referenced-file/cardinality agreement. The deletion-vector -reader then streams Roaring array, bitset and run containers, validates their -directories and cardinalities, and checks the blob's framing and CRC-32. It retains -one bounded container directory, not the deleted-position set; byte and bitmap -limits independently bound work. Snapshot-wide uniqueness and referenced data-file row-count checks belong to -explicit selected-file validation; normal metadata commit does not run them. -ORC probing retains at most 255 postscript bytes, checks protobuf wire framing -and resolves footer/metadata spans without decoding stripe directories. It accepts -legacy header-only magic and skips bounded unknown protobuf fields. - -Avro OCF framing uses a bounded header map and pull-based encoded blocks. Header -bytes, metadata count, block bytes and records per block have independent caps; -negative map blocks must match their declared byte lengths. Sync markers and -canonical block integrity are verified before a block returns. Errors or cancelled -reads poison the cursor rather than resuming at an ambiguous record boundary. -Null and raw-deflate block decoding enforce an independent decoded-byte cap; -truncated compressed data or unused suffixes fail closed. This layer does not -resolve Avro schemas or validate Iceberg manifest fields. - -Manifest inheritance is a separate constant-state semantic layer. It distinguishes -the manifest version from the containing table version: v1 sequences default to -zero, while new snapshots can assign row IDs to older manifests. Only added files -inherit missing sequence numbers; explicit file ages are preserved. Unassigned -data files advance the row-ID cursor in manifest order, including existing files -after an upgrade; delete files cannot carry row IDs. Invalid entries and arithmetic -overflow leave the cursor unchanged. Avro decoding and commit admission are not -yet connected to this resolver. - -Delegation tokens carry catalog activation epoch, table, principal fingerprint, -nonce, exact operation set, issue/expiry times and independent request/file byte -limits. Domain-separated HMAC authenticates bounded claims and derives per-grant -S3 credential material without a mutable credential registry. Verification requires -the operation's authorized context: writes and credential issuance check Ready -authority, while direct reads may reuse the bounded validated permission cache -described above. Ordinary FileIO grants exclude DELETE; -explicit cleanup grants include the separate delete operation. These token -primitives feed native request-signature verification through a request-local -credential provider. Only the shared SigV4 algorithm is reused; general S3 -credentials and metadata are never consulted. Header and presigned requests have -bounded authentication input and reject duplicate authentication fields. Grant -expiry remains exact even when signature timestamps allow clock skew. Table -credential issuance requires a matching Ready catalog authority and rejects -lifetimes above its persisted delegated-access bound, independently of the -signer's configured maximum. A zero persisted delegation bound disables issuance. -Credential-issuance callers freshly authorize the root and exact live table or -client-selected unpublished storage identity; serialization performs no storage reads. -An unpublished identity requires a Ready namespace, current writer authorization, -an exact table-ID and format-version selector, and the matching catalog layout. -Its final head slot must still be absent. No original-owner or draft lookup is -required. Published heads and retired slots cannot be relabeled as unpublished. -Grants last at most fifteen minutes. Published readers receive -read-only grants; only writers receive upload and multipart rights. Responses -configure the native S3 origin and SDK credential-refresh endpoint without embedding -long-lived secrets or changing canonical metadata bytes. Table-load ETags include -the SDK configuration as well as the selected-generation metadata representation; -changed endpoints cannot be hidden by a metadata-only conditional response. -Routed operation checks and streamed -request/response limits already enforce signed scopes and server budgets. -A session token alone never authenticates a request. -The native path-style request parser preserves decoded object-key bytes and limits -operations to immutable object reads/writes, multipart subresources and -explicitly authorized cleanup. Unknown -query operations, duplicate parameters and general buckets fail closed. HTTP -DELETE identifies either an upload abort or separately authorized object cleanup; -batch cleanup uses the S3 DeleteObjects request and per-key response contract. -These request primitives are attached to the native listener. - -Writes and reads stream through bounded CROWDB storage clients. Delegated FileIO -access may move immutable ranges without an Access Server payload bounce, but -cannot overwrite published files or bypass its operation-specific authorization. - -## 4. Commit and lifecycle - -A composite table read checks CatalogContext at entry and before returning. -Inside that request it reuses context-bound namespace, table and file resolution -without repeating root reads. Admission resolves complete namespace ancestry: -namespace identities cannot move, and strict nonempty deletion protects every -ancestor while its child remains live. The final checks revalidate the selected -head generation, leaf mapping, authority identity/name epoch and lifecycle. -Property updates do not invalidate this binding; Dropping remains visible until -Tombstone. Public standalone repository methods retain their own context guards. -There is no cross-request permission cache or reader registration. The same -context permits capability additions only; catalog replacement advances its epoch. -Capability revocation or authority-only retirement would require revisiting this -request-local admission contract. - -A table commit validates requirements against one selected table generation, -constructs a complete new metadata state, writes immutable candidate files, and -atomically publishes one new table head. Concurrent commits either publish from -the generation they validated or fail for the client to reconcile; they never -merge implicitly. - -Ordinary commits prepare immutable metadata and publish with one TableHead CAS. -The Catalog config advertises `crowdb.guarded-maintenance=true` when table writes -are installed. Console maintenance uses the optional CrowDB extension -`x-crowdb-base-metadata-location` on an existing-table commit to bind the complete -reviewed metadata generation. A changed base conflicts; guarded `remove-snapshots` -also rejects every current or named-reference target on that exact base. The head -CAS prevents a concurrent Branch or Tag from being silently removed. This guard -does not change ordinary Iceberg update semantics and cannot accompany `assert-create`. -An exact successful request replay returns its published metadata. Prepared metadata -may still publish while its base remains current; when a newer generation conceals -that operation's metadata file, the result stays uncertain rather than being rebased. - -Existing-table commits do not write a phase journal, per-file committed state or post-publication -head settlement. No server instance is a table leader or lock owner. A current -head carries the operation-named metadata pointer and an optional 32-byte digest -binding principal, route and request bytes. A matching visible retry returns that -metadata; changed input conflicts and old heads without this binding remain -uncertain. After a later generation hides that result, -an old request may return uncertain rather than silently reapplying its update. - -Table creation retains its complete input, evaluated metadata and response only -inside the request. It writes the final metadata `FileRecord`, reserves the exact -namespace/name, checks that the same parent is Ready, creates the final table head -with an expected-absent CAS, and conditionally publishes the reserved name. This -order is required: a reservation precedes the last parent check, so namespace -deletion either sees it or prevents that check from succeeding. The head contains -no pending request identity. Cancellation discards request-local work; complete -client retries may create new unowned resources and never replay a stored response. -Response headroom is checked before any resource publication. - -A `RetiredTable` record occupies the same storage head key as a published table. -It binds the immutable catalog, TableId and optional storage namespace: 32 raw ID -bytes in the legacy layout, 48 in the native layout, plus a bounded envelope. It -contains no request, owner, schema, input, response, phase or continuation. It is a -permanent terminal resource fact and never returns to an absent or Ready slot. -Name resolution treats it as missing; credential issuance, initial publication -and physical write admission reject it. GC never interprets it as a live head. - -When namespace deletion or reservation cleanup encounters a reserved table whose -head is absent, it creates this retirement record by CAS before removing the -exact reservation. A conflict supplies the current head or retirement value; -cleanup uses that value without a confirmation GET. If a Ready head won, cleanup -can publish its matching reserved name and strict nonempty namespace deletion -fails. If retirement won, the delayed initial-head CAS fails permanently. Removing -only the reservation is unsafe: a delayed expected-absent head creation could -otherwise expose an object after the namespace's empty proof. Exact name bytes and -an input-bound reservation identity prevent delayed cleanup from removing a later table. -An uncertain CAS reply stops that request; a later full retry reconciles final -resource facts. - -For inline metadata, a successful immediate create performs four conditional -resource writes: metadata authority, name reservation, initial head and published -name. It creates no operation, input-page or response-page keys. Chunked metadata -adds only the existing per-range physical write intents. Ordinary creation does -not modify the parent namespace authority. Retiring an absent head adds one CAS -only on cancellation or GC; its retained metadata has fixed size independent of -table schema or request size. - -New-table requests may carry unique provisional field IDs starting at zero, -as produced by Spark. Creation assigns positive durable IDs before validating -the canonical schema and remaps identifier, partition, sort and nested-default -references consistently. Negative or duplicate provisional IDs remain invalid. -Existing-table commits and stored metadata still require their durable IDs. - -Staging evaluates metadata and returns its complete initialization material, -including a fresh UUID and storage identity, without writing any server record. -A repeated stage request produces independent material. The client retains the -chosen response and sends complete standard assert-create updates when ready. -The final request initializes an empty metadata builder with the supplied UUID, -location, schema, partitioning, sorting and properties, preserving field IDs -already used by client-prepared files. It evaluates initial metadata and -requirements, then follows the same resource publication order as immediate -creation without scanning staged data files or client manifest chains. - -An assert-create name reservation uses a domain-separated, length-delimited -digest of catalog activation, namespace identity, TableId, name, principal and -complete request bytes. It excludes request IDs and evaluation clocks, allowing -identical complete retries to use the exact existing reservation. Ordinary create -uses a fresh reservation identity. Competing input cannot adopt another reserved -resource. If the initial head already won, a retry only settles its matching name -and returns a conflict; clients load the published table to learn the result. -No response replay or server-owned draft is required. Unknown storage replies -stop the current request without a confirmation read. - -Bounded background scans rotate creation, legacy update and lifecycle journals, -four records per page. Independent continuations reset on catalog activation -changes. Legacy unbound drafts retain their existing expiry. Creation cleanup -never writes missing metadata or publishes a first head: it occupies an absent -head slot with permanent retirement, settles exact name/head markers from final -resource facts, and clears only the captured parent marker revision. After those -facts are durable, it conditionally deletes the old creation record. Unknown -writes stop cleanup. Physical allocation discovery remains in write intents. -Retired-catalog GC permanently retires the exact legacy head slot before releasing -that record's protection, whether the first head won or remained absent. It waits -for the catalog's existing retry and request grace; exact-head CAS conflicts stop -that step unless the returned value is already retired. Final file authorities -and per-range write intents remain independent discovery sources for physical -reclamation. An old nonterminal phase cannot retain the catalog indefinitely. -Legacy update and lifecycle recovery retain their own bounded compatibility paths. -New creators leave only resource facts for reservation cleanup and GC. - -Logical drop resolves the currently named table and conditionally tombstones its -exact head, then conditionally removes that table's name mapping. It writes no -request, input, phase or response journal. A conflicting in-flight head update -returns conflict without rebasing. A new DELETE retry resolves the name again -and may delete a table recreated at that name. Background cleanup of an older -table remains bound to its captured table ID and cannot remove the replacement. - -Same/cross-namespace rename stores no new input, phase, response or lifecycle -journal. Request admission validates the route and principal without reading an -old request journal or encoding a replay payload. Each affected parent holds one -bounded `NamespaceRename` occupancy containing the source mapping, target name, expected head generation/fence and a -fresh per-attempt reservation identity. Parents are occupied in stable ID order; -failure to acquire the second cancels and releases the first, without waiting on -another rename. Definite parent CAS conflicts use the observed authority directly, -preserving concurrent property changes. Ordinary readers do not acquire occupancy. - -The target name is reserved before one exact-head CAS switches the canonical -identifier and name epoch. Table identity, UUID, metadata generation, digest and -file location stay fixed. The destination is published before the captured source -mapping and parent occupancies are conditionally cleared. A namespace cannot enter -Dropping while occupied, including a same-parent move where a scan could otherwise -miss both names across the switch. Mapping visibility remains qualified by the -head, and the old name never becomes an alias. - -Recovery reads the head and name facts: a switched head completes the destination; -an unchanged head is first conditionally fenced to invalidate delayed publication, -then the reservation and occupancy can be removed. Conflicts use their returned -head. Namespace mapping sweeps and name-reservation/deletion helpers recover -abandoned occupancy. Every new attempt has a distinct reservation identity, even -when an HTTP request key is reused, preventing delayed cleanup of a canceled -attempt from deleting a later reservation. Cleanup never removes a recreated name -with a different identity. Existing lifecycle journals still finish through their -legacy recovery driver. - -Rename replies are generated directly from current resource state. After a -completed move the source is missing; repeating that request can return 404. -A recreated source with an occupied target conflicts. There is no historical -response replay or implicit rebasing of an in-flight head CAS. - -Drop tombstones the selected head without traversing snapshots or deleting files. -A purge request persists a `TablePurgeTask` containing the candidate tombstoned -head and activation epoch before publishing the tombstone, indexed by table, -generation and metadata file. Its initial marker time is zero. GC admits it only -after observing the exact tombstone and fixes a twenty-minute delay from that -observation; an existing admitted task never renews its deadline. Older markers -keep their stored times. If the deletion did not win, GC first fences the exact -Ready predecessor before removing the marker, blocking a delayed tombstone CAS. -Unknown writes stop immediately. Deferred or canceled markers do not keep an -otherwise idle scanner spinning. A drop without an explicit -purge request leaves the underlying files in place. Purge reclaims table-scoped -files after the delay and owner checks, without snapshot reachability scans -inside the dropped table. The REST -drop/rename routes require independent writer credentials and return empty success -responses; stale table names fail normal load, exists, commit and credential refresh. - -Drop and replacement remove logical authority first. Purge and retired-catalog -cleanup reclaim only their captured inactive identities after retention and -owner checks. Snapshot expiration alone does not delete published files. -Explicit single/batch cleanup scans retained references under the caller's -no-future-reference contract described below. - -The reclamation proof binds current and retained historical metadata to their -captured heads. Its immutable traversal stack and compressed binary file-ID index -use content-addressed payload pages. A task CAS publishes the pending stack and -mark root together; a missing page is an error, including during a nonmembership -query. Physical deletion runs only for a tombstoned table or retired catalog; -the worker rechecks inactive authority before sweeping. A Ready table may retain -unreachable files until drop or clear rather than interrupt reads or commits. -Unfinished table operations and active multipart work defer table purge. -Completed operation retry records and terminal multipart sessions do not extend -the 20-minute purge delay. Background task advancement is enabled by default; -it uses a separate storage client pool, one-step concurrency admission, bounded -KV and chunk request/byte budgets, and durable retry state. The -chunk byte budget covers payload reads and writes. Range retirement sends a -bounded control descriptor and consumes one chunk-operation request without -charging the referenced range as transferred payload. Large ranges therefore -remain eligible for notification under the same limits. These are logical -operation and payload budgets, not exact routed-RPC or wire-byte accounting. -enabled scheduler admits persisted table purge markers and completed catalog clears; -management may also start inactive tasks. Operators can disable the scheduler -or adjust validated resource limits. Discovery starts at startup. Each catalog -with unfinished discovery or runnable work continues after the retry-base delay -(one second by default, at least 100 ms and at most the configured scan interval). -One window still advances at most one worker step under its original I/O and time -budgets. A completed discovery round returns to the scan interval, 30 minutes by -default. A purge becomes eligible 20 minutes after the drop; discovery may add up -to one idle scan interval before its first step. A known task retry deadline can -wake its catalog sooner, without shortening retention or other resource deadlines. - -Scheduler discovery positions are discardable, per-catalog memory. Purge, -multipart admission, retired-catalog admission and task lanes rotate their first -turn before awaiting storage. Due catalogs rotate independently; one catalog's -failure does not delay another catalog's due work. A failed lane reports its -original error and leaves other lanes eligible. A validated raw row advances its local position before key -or value decoding and before any admission or worker action. An error, deadline -or cancellation therefore retains that observed position. An empty page closes -that lane's pass; actual task progress reopens its source lane when more work is -runnable. Terminal, paused and future-retry tasks do not keep a round busy. -Failed or cancelled lanes are rescanned with capped exponential backoff until a -complete error-free pass, so a long-lived corrupt row cannot cause a hot loop. -Multipart admission runs once per discovery round. Local scheduling retains at -most the 64 configured catalogs and the current catalog; retired identities remain -discoverable through durable system markers. Restart loses these hints and rescans the unchanged -durable markers, ledgers and tasks. These hints grant no deletion authority and -do not replace task proof, permanent retirement fences or retry state. - -Active multipart and deleted-file cleanup skips candidates whose fixed retention -deadline has not arrived, so later eligible files can advance in the same pass. -Those candidates and their deadlines remain unchanged. Complete denotes the end -of that eligibility pass; periodic admission reopens it to discover newly eligible -resources. Eligibility may therefore wait for the next scan interval and bounded -drain time. Reopened passes retain a monotonically increasing sweep round, so a -completed candidate contributes to deletion totals once. Inactive table purge -and retired-catalog cleanup retain their task-wide retention and fence rules. - -The dedicated read-only local scan retains validated catalog/scope/prefix bounds, -strict ordering, positive revisions and item/byte limits while allowing an -undecodable raw key as its continuation. It uses the native exclusive continuation -and an actual routed catalog generation, including for keys longer than the -Iceberg record format permits. Generic scans and persisted GC task continuations -keep their existing format validation. This path still uses the legacy scan -transport: its byte limit is checked after materialization. A transport failure -or an oversized response without an observed key cannot advance this position; -it remains an explicit error, with no claim of a wire memory bound or universal -poison-row recovery. - -Provisioned disk capacity is the allocation boundary for both foreground files -and GC durable workspace. A failed GC workspace write retains the last durable -continuation and defers retry; it never substitutes an incomplete proof or -authorizes deletion. Committed files remain readable when new chunk allocation -fails. Progress resumes after capacity is restored through the normal storage -flow. Shared-chunk ranges remain pending while range deletion is unsupported. - -Publication and logical-URL resolution check catalog and table or draft -authority without request-level pins. Direct reads instead use the validated -permission cache and authenticated locator. Physical reclamation observes the -configured retention interval. Logical deletion hides the file from listing; -its tombstone rejects logical-URL resolution but is not consulted by direct -HEAD/GET/Range GET. With a valid grant, an already known direct URL can still -read retained bytes until physical reclamation. Logical deletion is not an -instantaneous direct-URL revocation guarantee. Legacy live tasks are retired without -further deletion, releasing an owned table fence. Retained and deferred -candidates remain durable work for later inactive passes. - -An ordinary inline file in a table purge has no physical deletion frontier. -After checking retention, head generation, purge fence and immutable claim, one -bounded worker step conditionally deletes its exact file authority and persists -the completed candidate and task count. It does not persist an intermediate -Deleting checkpoint. Part, assembly and non-inline candidates retain their -destructive traversal checkpoints. An unknown mutation response stops the step; -a subsequent worker resumes from the durable file, candidate and task facts. - -An already authorized FileIO GET or HEAD can continue within the configured -retention interval after logical deletion. Native storage arbitrates direct -read/reclamation races, returning the original bytes or a storage error, never -bytes from a reused allocation. Logical-URL admission rejects a reclaiming -record. Uploads and new credentials require a Ready table. After logical drop, -direct URLs remain readable under still-valid grants while bytes are retained; -physical purge and grant expiry bound that access. - -Retired catalog recovery scans system retry and management ledgers before file -deletion and after the final file rescan. Pending or retained bindings stop the -pass; exact-identity overflow entries remain independent of occupied primary -slots. After files are reclaimed, the worker conditionally removes expired -bindings, audits, projections and non-GC catalog records while preserving the -active root's management operation. Multipart parts have their own durable tree -candidates. Assembly checkpoints have separate claims and a durable frontier-root -index. Abandoned frontiers are authenticated before traversal; a conflicted final -tree is traversed once instead of revisiting its shared frontier. Published -sessions reclaim only the checkpoint block, preserving the assembled data tree. -Unpublished server-generated initial metadata is identified by Metadata/JSON -resource type and the exact generation-one metadata filename bound to its FileId. -Its storage table must match the absent or retired final head slot; metadata for -another table cannot select that resource as its canonical head metadata. -The active resource sweep may discover this candidate when its final head is -absent or retired. Its first observation and existing safety retention remain -persisted in the GC candidate; restart never restarts or shortens that interval. -Immediately before reclaiming it, GC creates the same final-head retirement fact -if still absent. A Ready head returned by the CAS conflict preserves the file; -retirement winning prevents all delayed initial publishers. Only then may the -existing exact-range deletion cursor notify ChunkDB. Acceptance of DeleteRange is -the access server's completion boundary; it does not poll physical release. - -This proof does not authorize deleting data, manifests or inherited references -merely because their owner head is absent. Their existing final-resource reference -proof remains required. Physical ranges written before a final metadata authority -still retain their BlockIntent discovery facts. Both catalog cleanup layouts keep -retirement records permanently and allow those terminal facts in their final empty -proof; removing them would reopen a delayed expected-absent creation slot. - -Retired-catalog cleanup leaves a permanent retirement value at every discovered -table head key. A scanned head uses an exact-value CAS with the scan bytes. -Before removing a name mapping, cleanup reads its complete storage head key once -and retires the absent slot or observed head. This is independent of creation -journals, including a reserved name whose first head never won. One concurrent -slot change can be consumed directly from the CAS observation; further changes -leave the durable cleanup cursor unchanged. Unknown replies stop the step. A -restarted worker recognizes the retirement value before removing the exact -mapping. File authorities, range intents and GC candidates remain the physical -discovery chain. Table purge keeps its tombstone head; this cleanup cannot retire -a table in the current active catalog. - -An active catalog has a separate resource cleanup task for expired terminal parts, -assembly checkpoints, explicit deleted-file records and unpublished initial -metadata. It never infers missing snapshot references for other published files -or fences a Ready table. Candidate progress is -durable across restarts. After every part and checkpoint is reclaimed, it removes -the terminal session's payload pages and session record; pending part settlement -or unreleased multipart credit keeps that session available for recovery. -Streamed file and part candidates instead reclaim their exact Chunk location -ranges after retention. A published selected part remains owned by the immutable -file descriptor; part cleanup does not reclaim it a second time. Legacy tree allocations -abandoned before publication retain FileWriteIntent ownership. Native streamed -writes rely on Chunk-KV WAL and chunk allocation lifecycle rather than a -per-frame Iceberg intent record. Active-catalog cleanup does not reclaim an intent merely because a -file descriptor is absent: a multipart session in `Publishing` may still own -those blocks. Retired-catalog and table-purge passes can reclaim intents after -their authority and owner checks. The disk leak scanner alone does not establish -Iceberg ownership. -Each physical step rechecks the terminal session and retention. The checkpoint -block is deleted after its children, and session cleanup requires its completed -claim. Block intents are swept after tree candidates, preserving reachable owners -and any unfinished file or assembly cursor. An owner fence prevents publication -or new block writes once orphan deletion begins. Superseded intents of a reachable -owner are conservatively retained until that owner becomes unreachable. - -Final catalog cleanup verifies that all candidates are complete and that no owner -is paused or quarantined. A durable retirement marker selects the cleanup owner -and rejects stale GC mutations before bounded deletion of claims, candidates, -proof pages, write fences and old tasks. The retired authority, permanent table -retirement values, winning task result and retirement marker remain. Late -unfinished records stop cleanup. -Uncertain progress responses are resolved by reading durable state; an unfenced -live proof whose authority changed terminates without deleting files. - -## 5. Compatibility - -CROWDB covers the core Iceberg format semantics for v1, v2, and v3, including -reading, creating, writing, and the defined version upgrades. Mandatory behavior -for the selected format version is not weakened. Optional features are exposed -only when their complete semantics are enabled. - -REST wire types, Iceberg domain state, and CROWDB storage records remain -separate. Unknown or disabled requirements and updates fail before mutation. -The backed-up specifications decide behavior when implementations differ. - -### Executable conformance profile - -The official Java oracle is Apache Iceberg 1.11.0; the official Rust REST -client is 0.10.0. The declared selected-file profile uses Parquet data/deletes, -Avro manifests and Puffin deletion vectors/statistics. ORC bytes can be stored, -but selected ORC validation and compute-engine certification are separate work. - -- **v1/v2/v3 metadata and upgrades:** `official_java_metadata_roundtrips_without_rewriting`, - `official_catalog_creates_commits_upgrades_stages_and_refreshes_native_credentials`, - and the official create/update/snapshot fixtures compare canonical metadata. - `TestIcebergVersionRows` reads actual rows before and after adjacent upgrades, - reads historical snapshots, expires them logically, and reloads after restart. -- **Selected data and deletes:** client-generated fixtures exercise original - rows, equality-delete visibility and historical snapshots. Commit treats client - manifests as opaque references. `TestIcebergSelectedFiles` publishes mismatched - data/delete declarations in isolated tables and checks the selected head and - snapshot inventory; its valid table still verifies actual rows, equality deletes - and historical reads. Selected-use rejection probes are explicit validator - tests rather than a normal publication gate. `TestIcebergFileOperations` checks - that upload credentials lack cleanup permission (403 `AccessDenied`) and that - rejection preserves the immutable file. Canonical file - validators separately cover position deletes, v3 lineage, deletion vectors, - defaults, nested/variant types, integer encodings and nullable values. -- **Statistics:** `TestIcebergCatalogWrites` and the official partition-statistics - fixtures cover publication, replay, evolution and staged creation. - `TestIcebergCommitErrors` verifies supported partition-statistics metadata - publication, exact inventory reload and removal; a foreign file location - rejects the whole update batch and preserves the selected head. Metadata - acceptance is separate from explicit Parquet content validation. Historical - omissions follow the explicit compatibility rules described above. -- **Discovery and authorization:** - `discovery_uses_installed_routes_and_unsupported_paths_leave_no_record` and - `explicit_partial_activation_limits_discovery_and_table_admission` exercise - installed/disabled route and version combinations. HTTP namespace, table, - lifecycle, credentials and body tests cover roles, malformed requests, limits, - cancellation, unchanged authority on rejection and bounded metrics. -- **Faults and retirement:** `official_rust_client_observes_lost_create_reply_on_another_listener`, - `official_rust_client_lost_reply_survives_native_storage_restart`, and the - Java response-loss/retired-catalog fixtures exercise listener changes and - restart. The SDKs do not automatically replay a lost mutation POST with the - same key; direct HTTP fault tests prove the server's same-key replay contract. -- **Apache REST Compatibility Kit:** the unmodified 1.11.0 runner verifies six - supported cases: namespace create, basic table create, rename, drop, - missing-table drop and table list. This is not a full-kit pass. Other cases - assume register/views or local filesystem locations that the native authority - deliberately rejects. Custom SDK fixtures are not described as kit results. -- **Large logical files:** `tib_address_space_range_reads_keep_fixed_windows_and_small_authority` - and `tib_reclamation_progress_serializes_a_bounded_resumable_cursor` use a - 1 TiB logical tree with repeated immutable blocks. They prove bounded windows - and resumable state, not physical TiB capacity. Oversized declared metadata - is rejected before I/O at the configured metadata budget. - -## 6. Relationship to other access models - -General S3 and Iceberg share chunk, Chunk-KV, transport, credential, and safe -utility mechanisms, but not semantic authority. S3 bucket records, overwrite, -delete, listing, and lifecycle behavior cannot publish or alter an Iceberg -table. - -Dataset may consume data described by an Iceberg table through an explicit, -generation-bound reference. That does not transfer snapshot, commit, file, or -reclamation authority to Dataset. - -The native file endpoint exposes separately authorized S3 `DeleteObject` and -`DeleteObjects` cleanup. A cleanup credential is requested through the table -credentials endpoint with `cleanup=true`; ordinary FileIO credentials cannot -delete objects. The server scans retained table metadata and refuses a path -that is referenced or whose reference status is uncertain. Successful deletion -records a logical tombstone and removes list visibility. The background worker -discovers that final tombstone and durably claims its exact old file before -physical reclamation after 20 minutes. DELETE writes no GC task or candidate; -restart discovers the same DeletedFile. Direct URLs remain readable until their -bytes are reclaimed, subject to grant validity. Their FileIds/locators are never -reused. A new logical-path PUT can immediately reuse its path: if the worker has -not queued the old file, PUT first retains its exact resource in the existing -GC claim and candidate records, then conditionally replaces the observed -DeletedFile. An uncertain write stops without a confirmation read. Either the -deletion marker or the independent GC candidate retains the old reclamation -obligation across failure. Batch deletion returns individual errors inside -HTTP 200, so callers must inspect every item. - -> **Warning:** Before deleting, the client must stop and resolve every writer -> and retry that might still commit the path, including unpublished metadata. -> The server cannot see a future commit during its reference scan. If a client -> later commits a deleted path, the snapshot can reference missing data. Do not -> use cleanup credentials in ordinary FileIO. A reader retaining old metadata -> must finish before the 20-minute physical reclamation deadline. - -## 7. Correctness invariants - -- **ICE-I1 — Native authority:** catalog, namespace, table, snapshot, commit, - file, and reclamation semantics belong to Iceberg rather than general S3. -- **ICE-I2 — Standard recoverability:** published standard Iceberg metadata is - sufficient to recover table state; derived projections are not authoritative. -- **ICE-I3 — Atomic table publication:** a table head selects exactly one - complete immutable metadata generation. -- **ICE-I4 — Validated concurrency:** a commit publishes only from the table - generation against which its requirements were validated. -- **ICE-I5 — Immutable files:** a published Iceberg file is never overwritten. -- **ICE-I6 — Reachability before reclamation:** physical deletion cannot precede - proof that no protected Iceberg state reaches the file. -- **ICE-I7 — Spec compliance:** supported v1, v2, and v3 behavior preserves all - mandatory semantics of the selected format version. -- **ICE-I8 — Bounded operation:** table size and history do not determine one - Access Server request's retained memory or unbounded work. -- **ICE-I9 — Live file retention:** a Ready table's published file is not - reclaimed solely because no retained snapshot refers to it; active cleanup - handles expired multipart remnants and explicitly deleted files with durable - owner checks. -- **ICE-I10 — Stable identity directory:** rename and cross-namespace move - preserve storage namespace, table path and every published file URL. -- **ICE-I11 — Single file authority:** one filename-keyed record selects each - published file; list and lifecycle use that record without secondary file indexes. -- **ICE-I12 — Direct native read:** a direct URL supplies its immutable native - descriptor without Iceberg file-location or file-state KV queries; native - chunk layout resolution and repair retain their normal behavior. - Logical deletion removes list visibility; physical GC ends byte availability. -- **ICE-I13 — Bounded metadata scale:** large metadata bytes live in native - objects, descriptors/head remain bounded, prefix listing reads bounded KV - pages, and optional projections cannot replace standard metadata authority. diff --git a/doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml b/doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml deleted file mode 100644 index 06d13ec13..000000000 --- a/doc/design/access-server/iceberge/iceberg-rest-catalog-open-api-1.11.0.yaml +++ /dev/null @@ -1,5285 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - ---- -openapi: 3.1.1 -info: - title: Apache Iceberg REST Catalog API - license: - name: Apache 2.0 - url: https://www.apache.org/licenses/LICENSE-2.0.html - version: 0.0.1 - description: - Defines the specification for the first version of the REST Catalog API. - Implementations should ideally support all Iceberg table spec versions. -servers: - - url: "{scheme}://{host}/{basePath}" - description: Server URL when the port can be inferred from the scheme - variables: - scheme: - description: The scheme of the URI, either http or https. - default: https - host: - description: The host address for the specified server - default: localhost - basePath: - description: Optional prefix to be appended to all routes - default: "" - - url: "{scheme}://{host}:{port}/{basePath}" - description: Generic base server URL, with all parts configurable - variables: - scheme: - description: The scheme of the URI, either http or https. - default: https - host: - description: The host address for the specified server - default: localhost - port: - description: The port used when addressing the host - default: "443" - basePath: - description: Optional prefix to be appended to all routes - default: "" -# All routes are currently configured using an Authorization header. -security: - - OAuth2: [catalog] - - BearerAuth: [] - -paths: - /v1/config: - - get: - tags: - - Configuration API - summary: List all catalog configuration settings - operationId: getConfig - parameters: - - name: warehouse - in: query - required: false - schema: - type: string - description: Warehouse location or identifier to request from the service - description: - " - All REST clients should first call this route to get catalog configuration - properties from the server to configure the catalog and its HTTP client. - Configuration from the server consists of two sets of key/value pairs. - - - defaults - properties that should be used as default configuration; applied before client configuration - - - overrides - properties that should be used to override client configuration; applied after defaults and client configuration - - - Catalog configuration is constructed by setting the defaults, then client- - provided configuration, and finally overrides. The final property set is then - used to configure the catalog. - - - For example, a default configuration property might set the size of the - client pool, which can be replaced with a client-specific setting. An - override might be used to set the warehouse location, which is stored - on the server rather than in client configuration. - - - Common catalog configuration settings are documented at - https://iceberg.apache.org/docs/latest/configuration/#catalog-properties - - - The catalog configuration also holds an optional `endpoints` field that contains information about the endpoints - supported by the server. If a server does not send the `endpoints` field, a default set of endpoints is assumed: - - - GET /v1/{prefix}/namespaces - - - POST /v1/{prefix}/namespaces - - - GET /v1/{prefix}/namespaces/{namespace} - - - DELETE /v1/{prefix}/namespaces/{namespace} - - - POST /v1/{prefix}/namespaces/{namespace}/properties - - - GET /v1/{prefix}/namespaces/{namespace}/tables - - - POST /v1/{prefix}/namespaces/{namespace}/tables - - - GET /v1/{prefix}/namespaces/{namespace}/tables/{table} - - - POST /v1/{prefix}/namespaces/{namespace}/tables/{table} - - - DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table} - - - POST /v1/{prefix}/namespaces/{namespace}/register - - - POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics - - - POST /v1/{prefix}/tables/rename - - - POST /v1/{prefix}/transactions/commit - " - responses: - 200: - description: Server specified configuration values. - content: - application/json: - schema: - $ref: '#/components/schemas/CatalogConfig' - example: { - "overrides": { - "warehouse": "s3://bucket/warehouse/" - }, - "defaults": { - "clients": "4" - }, - "idempotency-key-lifetime": "PT30M", - "endpoints": [ - "GET /v1/{prefix}/namespaces/{namespace}", - "GET /v1/{prefix}/namespaces", - "POST /v1/{prefix}/namespaces", - "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "GET /v1/{prefix}/namespaces/{namespace}/views/{view}" - ] - } - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - Warehouse provided in the `warehouse` query parameter is not found. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NoSuchWarehouseExample: - $ref: '#/components/examples/NoSuchWarehouseError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/oauth/tokens: - - post: - tags: - - OAuth2 API - summary: Get a token using an OAuth2 flow (DEPRECATED for REMOVAL) - deprecated: true - operationId: getToken - description: - The `oauth/tokens` endpoint is **DEPRECATED for REMOVAL**. It is _not_ recommended to - implement this endpoint, unless you are fully aware of the potential security implications. - - All clients are encouraged to explicitly set the configuration property `oauth2-server-uri` - to the correct OAuth endpoint. - - Deprecated since Iceberg (Java) 1.6.0. The endpoint and related types will be removed from - this spec in Iceberg (Java) 2.0. - - See [Security improvements in the Iceberg REST specification](https://github.com/apache/iceberg/issues/10537) - - - Exchange credentials for a token using the OAuth2 client credentials flow or token exchange. - - - This endpoint is used for three purposes - - - 1. To exchange client credentials (client ID and secret) for an access token - This uses the client credentials flow. - - 2. To exchange a client token and an identity token for a more specific access token - This uses the token exchange flow. - - 3. To exchange an access token for one with the same claims and a refreshed expiration period - This uses the token exchange flow. - - - For example, a catalog client may be configured with client credentials from the OAuth2 - Authorization flow. This client would exchange its client ID and secret for an access token - using the client credentials request with this endpoint (1). Subsequent requests would then - use that access token. - - - Some clients may also handle sessions that have additional user context. These clients would - use the token exchange flow to exchange a user token (the "subject" token) from the session - for a more specific access token for that user, using the catalog's access token as the - "actor" token (2). The user ID token is the "subject" token and can be any token type - allowed by the OAuth2 token exchange flow, including a unsecured JWT token with a sub claim. - This request should use the catalog's bearer token in the "Authorization" header. - - - Clients may also use the token exchange flow to refresh a token that is about to expire by - sending a token exchange request (3). The request's "subject" token should be the expiring - token. This request should use the subject token in the "Authorization" header. - requestBody: - required: true - content: - application/x-www-form-urlencoded: - schema: - $ref: '#/components/schemas/OAuthTokenRequest' - responses: - 200: - $ref: '#/components/responses/OAuthTokenResponse' - 400: - $ref: '#/components/responses/OAuthErrorResponse' - 401: - $ref: '#/components/responses/OAuthErrorResponse' - 5XX: - $ref: '#/components/responses/OAuthErrorResponse' - - /v1/{prefix}/namespaces: - parameters: - - $ref: '#/components/parameters/prefix' - - get: - tags: - - Catalog API - summary: List namespaces, optionally providing a parent namespace to list underneath - description: - List all namespaces at a certain level, optionally starting from a given parent namespace. - If table accounting.tax.paid.info exists, using 'SELECT NAMESPACE IN accounting' would - translate into `GET /namespaces?parent=accounting` and must return a namespace, ["accounting", "tax"] only. - Using 'SELECT NAMESPACE IN accounting.tax' would - translate into `GET /namespaces?parent=accounting%1Ftax` and must return a namespace, ["accounting", "tax", "paid"]. - If `parent` is not provided, all top-level namespaces should be listed. - operationId: listNamespaces - parameters: - - $ref: '#/components/parameters/page-token' - - $ref: '#/components/parameters/page-size' - - name: parent - in: query - description: - An optional namespace, underneath which to list namespaces. - If not provided, all top-level namespaces should be listed. - For backward compatibility, empty string is treated as absent for now. - If parent is a multipart namespace, the parts must be separated by the namespace separator as - indicated via the /config override `namespace-separator`, which defaults to the unit separator `0x1F` byte (url encoded `%1F`). - To be compatible with older clients, servers must use both the advertised separator and `0x1F` as valid separators when decoding namespaces. - The `namespace-separator` should be provided in a url encoded form. - required: false - schema: - type: string - example: "accounting%1Ftax" - responses: - 200: - $ref: '#/components/responses/ListNamespacesResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - Namespace provided in the `parent` query parameter is not found. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NoSuchNamespaceExample: - $ref: '#/components/examples/NoSuchNamespaceError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - post: - tags: - - Catalog API - summary: Create a namespace - parameters: - - $ref: '#/components/parameters/idempotency-key' - description: - Create a namespace, with an optional set of properties. - The server might also add properties, such as `last_modified_time` etc. - operationId: createNamespace - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CreateNamespaceRequest' - responses: - 200: - $ref: '#/components/responses/CreateNamespaceResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 406: - $ref: '#/components/responses/UnsupportedOperationResponse' - 409: - description: Conflict - The namespace already exists - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NamespaceAlreadyExists: - $ref: '#/components/examples/NamespaceAlreadyExistsError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - get: - tags: - - Catalog API - summary: Load the metadata properties for a namespace - operationId: loadNamespaceMetadata - description: Return all stored metadata properties for a given namespace - responses: - 200: - $ref: '#/components/responses/GetNamespaceResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - Namespace not found - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NoSuchNamespaceExample: - $ref: '#/components/examples/NoSuchNamespaceError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - head: - tags: - - Catalog API - summary: Check if a namespace exists - operationId: namespaceExists - description: - Check if a namespace exists. The response does not contain a body. - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - Namespace not found - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NoSuchNamespaceExample: - $ref: '#/components/examples/NoSuchNamespaceError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - delete: - tags: - - Catalog API - summary: Drop a namespace from the catalog. Namespace must be empty. - operationId: dropNamespace - parameters: - - $ref: '#/components/parameters/idempotency-key' - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - Namespace to delete does not exist. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NoSuchNamespaceExample: - $ref: '#/components/examples/NoSuchNamespaceError' - 409: - description: Not Empty - Namespace to delete is not empty. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NamespaceNotEmptyExample: - $ref: '#/components/examples/NamespaceNotEmptyError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/properties: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - post: - tags: - - Catalog API - summary: Set or remove properties on a namespace - operationId: updateProperties - parameters: - - $ref: '#/components/parameters/idempotency-key' - description: - Set and/or remove properties on a namespace. - The request body specifies a list of properties to remove and a map - of key value pairs to update. - - Properties that are not in the request are not modified or removed by this call. - - Server implementations are not required to support namespace properties. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/UpdateNamespacePropertiesRequest' - examples: - UpdateAndRemoveProperties: - $ref: '#/components/examples/UpdateAndRemoveNamespacePropertiesRequest' - responses: - 200: - $ref: '#/components/responses/UpdateNamespacePropertiesResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - Namespace not found - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NamespaceNotFound: - $ref: '#/components/examples/NoSuchNamespaceError' - 406: - $ref: '#/components/responses/UnsupportedOperationResponse' - 422: - description: Unprocessable Entity - A property key was included in both `removals` and `updates` - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - UnprocessableEntityDuplicateKey: - $ref: '#/components/examples/UnprocessableEntityDuplicateKey' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/tables: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - get: - tags: - - Catalog API - summary: List all table identifiers underneath a given namespace - description: Return all table identifiers under this namespace - operationId: listTables - parameters: - - $ref: '#/components/parameters/page-token' - - $ref: '#/components/parameters/page-size' - responses: - 200: - $ref: '#/components/responses/ListTablesResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - The namespace specified does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NamespaceNotFound: - $ref: '#/components/examples/NoSuchNamespaceError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - post: - tags: - - Catalog API - summary: Create a table in the given namespace - description: - Create a table or start a create transaction, like atomic CTAS. - - - If `stage-create` is false, the table is created immediately. - - - If `stage-create` is true, the table is not created, but table metadata is initialized and returned. - The service should prepare as needed for a commit to the table commit endpoint to complete the create - transaction. The client uses the returned metadata to begin a transaction. To commit the transaction, - the client sends all create and subsequent changes to the table commit route. Changes from the table - create operation include changes like AddSchemaUpdate and SetCurrentSchemaUpdate that set the initial - table state. - operationId: createTable - parameters: - - $ref: '#/components/parameters/data-access' - - $ref: '#/components/parameters/idempotency-key' - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CreateTableRequest' - responses: - 200: - $ref: '#/components/responses/CreateTableResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - The namespace specified does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NamespaceNotFound: - $ref: '#/components/examples/NoSuchNamespaceError' - 409: - description: Conflict - The identifier already exists as a table or view - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableAlreadyExists: - $ref: '#/components/examples/TableAlreadyExistsError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - $ref: '#/components/parameters/table' - - $ref: '#/components/parameters/data-access' - post: - tags: - - Catalog API - summary: Submit a scan for planning - description: > - Submits a scan for server-side planning. - - - Point-in-time scans are planned by passing snapshot-id to identify the - table snapshot to scan. Incremental scans are planned by passing both - start-snapshot-id and end-snapshot-id. Requests that include both point - in time config properties and incremental config properties are - invalid. If the request does not include either incremental or - point-in-time config properties, scan planning should produce a - point-in-time scan of the latest snapshot in the table's main branch. - - - Responses must include a valid status listed below. A "cancelled" status is considered invalid for this endpoint. - - - When "completed" the planning operation has produced plan tasks and - file scan tasks that must be returned in the response (not fetched - later by calling fetchPlanningResult) - - - When "submitted" the response must include a plan-id used to poll - fetchPlanningResult to fetch the planning result when it is ready - - - When "failed" the response must be a valid error response - - The response for a "completed" planning operation includes two types of - tasks (file scan tasks and plan tasks) and both may be included in the - response. Tasks must not be included for any other response status. - - - Responses that include a plan-id indicate that the service is holding - state or performing work for the client. - - - - Clients should use the plan-id to fetch results from - fetchPlanningResult when the response status is "submitted" - - - Clients should inform the service if planning results are no longer - needed by calling cancelPlanning. Cancellation is not necessary after - fetchScanTasks has been used to fetch scan tasks for each plan task. - operationId: planTableScan - parameters: - - $ref: '#/components/parameters/idempotency-key' - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/PlanTableScanRequest' - responses: - 200: - $ref: '#/components/responses/PlanTableScanResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - - NoSuchTableException, the table does not exist - - NoSuchNamespaceException, the namespace does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - NamespaceDoesNotExist: - $ref: '#/components/examples/NoSuchNamespaceError' - 406: - $ref: '#/components/responses/UnsupportedOperationResponse' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan/{plan-id}: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - $ref: '#/components/parameters/table' - - $ref: '#/components/parameters/plan-id' - - $ref: '#/components/parameters/data-access' - - get: - tags: - - Catalog API - summary: Fetches the result of scan planning for a plan-id - operationId: fetchPlanningResult - description: > - Fetches the result of scan planning for a plan-id. - - - Responses must include a valid status - - - When "completed" the planning operation has produced plan-tasks and - file-scan-tasks that must be returned in the response - - - When "submitted" the planning operation has not completed; the client - should wait to call this endpoint again to fetch a completed response - - - When "failed" the response must be a valid error response - - - When "cancelled" the plan-id is invalid and should be discarded - - - The response for a "completed" planning operation includes two types of - tasks (file scan tasks and plan tasks) and both may be included in the - response. Tasks must not be included for any other response status. - responses: - 200: - $ref: '#/components/responses/FetchPlanningResultResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - - NoSuchPlanIdException, the plan-id does not exist - - NoSuchTableException, the table does not exist - - NoSuchNamespaceException, the namespace does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - PlanIdDoesNotExist: - $ref: '#/components/examples/NoSuchPlanIdError' - TableDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - NamespaceDoesNotExist: - $ref: '#/components/examples/NoSuchNamespaceError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - delete: - tags: - - Catalog API - summary: Cancels scan planning for a plan-id - operationId: cancelPlanning - parameters: - - $ref: '#/components/parameters/idempotency-key' - description: > - Cancels scan planning for a plan-id. - - - This notifies the service that it can release resources held for the - scan. Clients should cancel scans that are no longer needed, either - while the plan-id returns a "submitted" status or while there are - remaining plan tasks that have not been fetched. - - - Cancellation is not necessary when - - - Scan tasks for each plan task have been fetched using fetchScanTasks - - - A plan-id has produced a "failed" or "cancelled" status from - planTableScan or fetchPlanningResult - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - - NoSuchPlanIdException, the plan-id does not exist - - NoSuchTableException, the table does not exist - - NoSuchNamespaceException, the namespace does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - PlanIdDoesNotExist: - $ref: '#/components/examples/NoSuchPlanIdError' - TableDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - NamespaceDoesNotExist: - $ref: '#/components/examples/NoSuchNamespaceError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - - /v1/{prefix}/namespaces/{namespace}/tables/{table}/tasks: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - $ref: '#/components/parameters/table' - - post: - tags: - - Catalog API - summary: Fetches result tasks for a plan task - operationId: fetchScanTasks - parameters: - - $ref: '#/components/parameters/idempotency-key' - description: Fetches result tasks for a plan task. - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/FetchScanTasksRequest' - responses: - 200: - $ref: '#/components/responses/FetchScanTasksResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - - NoSuchPlanTaskException, the plan-task does not exist - - NoSuchTableException, the table does not exist - - NoSuchNamespaceException, the namespace does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - PlanTaskDoesNotExist: - $ref: '#/components/examples/NoSuchPlanTaskError' - TableDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - NamespaceDoesNotExist: - $ref: '#/components/examples/NoSuchNamespaceError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/register: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - post: - tags: - - Catalog API - summary: Register a table in the given namespace using given metadata file location - parameters: - - $ref: '#/components/parameters/data-access' - - $ref: '#/components/parameters/idempotency-key' - description: - Register a table using given metadata file location. - - operationId: registerTable - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/RegisterTableRequest' - responses: - 200: - $ref: '#/components/responses/LoadTableResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - The namespace specified does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NamespaceNotFound: - $ref: '#/components/examples/NoSuchNamespaceError' - 409: - description: Conflict - The identifier already exists as a table or view - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableAlreadyExists: - $ref: '#/components/examples/TableAlreadyExistsError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/tables/{table}: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - $ref: '#/components/parameters/table' - - get: - tags: - - Catalog API - summary: Load a table from the catalog - operationId: loadTable - description: - Load a table from the catalog. - - - The response contains both configuration and table metadata. The configuration, if non-empty is used - as additional configuration for the table that overrides catalog configuration. For example, this - configuration may change the FileIO implementation to be used for the table. - - - The response also contains the table's full metadata, matching the table metadata JSON file. - - - The catalog configuration may contain credentials that should be used for subsequent requests for the - table. The configuration key "token" is used to pass an access token to be used as a bearer token - for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration - key. For example, "urn:ietf:params:oauth:token-type:jwt=". - parameters: - - $ref: '#/components/parameters/data-access' - - name: If-None-Match - in: header - description: - An optional header that allows the server to return 304 (Not Modified) if the metadata - is current. The content is the value of the ETag received in a CreateTableResponse, - LoadTableResponse or CommitTableResponse. - required: false - schema: - type: string - - in: query - name: snapshots - description: - The snapshots to return in the body of the metadata via the `snapshots` field. Setting - the value to `all` would return the full set of snapshots currently valid for the table. - Setting the value to `refs` would load all snapshots referenced by branches or tags. - - Default if no param is provided is `all`. - required: false - schema: - type: string - enum: [ all, refs ] - - $ref: '#/components/parameters/referenced-by' - responses: - 200: - $ref: '#/components/responses/LoadTableResponse' - 304: - description: - Not Modified - Based on the content of the 'If-None-Match' header the table metadata has - not changed since. - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchTableException, table to load does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableToLoadDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - post: - tags: - - Catalog API - summary: Commit updates to a table - operationId: updateTable - parameters: - - $ref: '#/components/parameters/idempotency-key' - description: - Commit updates to a table. - - - Commits have two parts, requirements and updates. Requirements are assertions that will be validated - before attempting to make and commit changes. For example, `assert-ref-snapshot-id` will check that a - named ref's snapshot ID has a certain value. - Server implementations are required to fail with a 400 status code - if any unknown updates or requirements are received. - - - Updates are changes to make to table metadata. For example, after asserting that the current main ref - is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new - snapshot id. - - - Create table transactions that are started by createTable with `stage-create` set to true are - committed using this route. Transactions should include all changes to the table, including table - initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. The `assert-create` requirement is - used to ensure that the table was not created concurrently. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CommitTableRequest' - responses: - 200: - $ref: '#/components/responses/CommitTableResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchTableException, table to load does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableToUpdateDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - 409: - description: - Conflict - CommitFailedException, one or more requirements failed. The client may retry. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 500: - description: - An unknown server-side problem occurred; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Internal Server Error", - "type": "CommitStateUnknownException", - "code": 500 - } - } - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 502: - description: - A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Invalid response from the upstream server", - "type": "CommitStateUnknownException", - "code": 502 - } - } - 504: - description: - A server-side gateway timeout occurred; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Gateway timed out during commit", - "type": "CommitStateUnknownException", - "code": 504 - } - } - 5XX: - description: - A server-side problem that might not be addressable on the client. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Bad Gateway", - "type": "InternalServerError", - "code": 502 - } - } - - delete: - tags: - - Catalog API - summary: Drop a table from the catalog - operationId: dropTable - description: Remove a table from the catalog - parameters: - - $ref: '#/components/parameters/idempotency-key' - - name: purgeRequested - in: query - required: false - description: Whether the user requested to purge the underlying table's data and metadata - schema: - type: boolean - default: false - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchTableException, Table to drop does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableToDeleteDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - head: - tags: - - Catalog API - summary: Check if a table exists - operationId: tableExists - description: - Check if a table exists within a given namespace. The response does not contain a body. - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchTableException, Table not found - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableToLoadDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - $ref: '#/components/parameters/table' - - get: - tags: - - Catalog API - summary: Load vended credentials for a table from the catalog - operationId: loadCredentials - parameters: - - name: planId - in: query - required: false - schema: - type: string - description: The plan ID that has been used for server-side scan planning - - $ref: '#/components/parameters/referenced-by' - description: Load vended credentials for a table from the catalog. - responses: - 200: - $ref: '#/components/responses/LoadCredentialsResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchTableException, table to load credentials for does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableToLoadDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/tables/{table}/sign: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - $ref: '#/components/parameters/table' - - post: - tags: - - Catalog API - summary: Remotely signs requests to object storage - operationId: signRequest - requestBody: - description: The request to be signed - content: - application/json: - schema: - $ref: '#/components/schemas/RemoteSignRequest' - required: true - responses: - 200: - $ref: '#/components/responses/RemoteSignResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/tables/rename: - parameters: - - $ref: '#/components/parameters/prefix' - - post: - tags: - - Catalog API - summary: Rename a table from its current name to a new name - parameters: - - $ref: '#/components/parameters/idempotency-key' - description: - Rename a table from one identifier to another. It's valid to move a table - across namespaces, but the server implementation is not required to support it. - operationId: renameTable - requestBody: - description: Current table identifier to rename and new table identifier to rename to - content: - application/json: - schema: - $ref: '#/components/schemas/RenameTableRequest' - examples: - RenameTableSameNamespace: - $ref: '#/components/examples/RenameTableSameNamespace' - required: true - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - - NoSuchTableException, Table to rename does not exist - - NoSuchNamespaceException, The target namespace of the new table identifier does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableToRenameDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - NamespaceToRenameToDoesNotExist: - $ref: '#/components/examples/NoSuchNamespaceError' - 406: - $ref: '#/components/responses/UnsupportedOperationResponse' - 409: - description: Conflict - The target identifier to rename to already exists as a table or view - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: - $ref: '#/components/examples/TableAlreadyExistsError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - $ref: '#/components/parameters/table' - - post: - tags: - - Catalog API - summary: Send a metrics report to this endpoint to be processed by the backend - operationId: reportMetrics - requestBody: - description: The request containing the metrics report to be sent - content: - application/json: - schema: - $ref: '#/components/schemas/ReportMetricsRequest' - required: true - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchTableException, table to load does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableToLoadDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/transactions/commit: - parameters: - - $ref: '#/components/parameters/prefix' - - post: - tags: - - Catalog API - summary: Commit updates to multiple tables in an atomic operation - operationId: commitTransaction - parameters: - - $ref: '#/components/parameters/idempotency-key' - requestBody: - description: - Commit updates to multiple tables in an atomic operation - - - A commit for a single table consists of a table identifier with requirements and updates. - Requirements are assertions that will be validated before attempting to make and commit changes. - For example, `assert-ref-snapshot-id` will check that a named ref's snapshot ID has a certain value. - Server implementations are required to fail with a 400 status code - if any unknown updates or requirements are received. - - Updates are changes to make to table metadata. For example, after asserting that the current main ref - is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new - snapshot id. - content: - application/json: - schema: - $ref: '#/components/schemas/CommitTransactionRequest' - required: true - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchTableException, table to load does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - TableToUpdateDoesNotExist: - $ref: '#/components/examples/NoSuchTableError' - 409: - description: - Conflict - CommitFailedException, one or more requirements failed. The client may retry. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 500: - description: - An unknown server-side problem occurred; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Internal Server Error", - "type": "CommitStateUnknownException", - "code": 500 - } - } - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 502: - description: - A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Invalid response from the upstream server", - "type": "CommitStateUnknownException", - "code": 502 - } - } - 504: - description: - A server-side gateway timeout occurred; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Gateway timed out during commit", - "type": "CommitStateUnknownException", - "code": 504 - } - } - 5XX: - description: - A server-side problem that might not be addressable on the client. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Bad Gateway", - "type": "InternalServerError", - "code": 502 - } - } - - /v1/{prefix}/namespaces/{namespace}/views: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - get: - tags: - - Catalog API - summary: List all view identifiers underneath a given namespace - description: Return all view identifiers under this namespace - operationId: listViews - parameters: - - $ref: '#/components/parameters/page-token' - - $ref: '#/components/parameters/page-size' - responses: - 200: - $ref: '#/components/responses/ListTablesResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - The namespace specified does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - examples: - NamespaceNotFound: - $ref: '#/components/examples/NoSuchNamespaceError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - post: - tags: - - Catalog API - summary: Create a view in the given namespace - description: - Create a view in the given namespace. - operationId: createView - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CreateViewRequest' - responses: - 200: - $ref: '#/components/responses/LoadViewResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - The namespace specified does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - examples: - NamespaceNotFound: - $ref: '#/components/examples/NoSuchNamespaceError' - 409: - description: Conflict - The identifier already exists as a table or view - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - examples: - ViewAlreadyExists: - $ref: '#/components/examples/ViewAlreadyExistsError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/views/{view}: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - $ref: '#/components/parameters/view' - - get: - tags: - - Catalog API - summary: Load a view from the catalog - operationId: loadView - description: - Load a view from the catalog. - - - The response contains both configuration and view metadata. The configuration, if non-empty is used - as additional configuration for the view that overrides catalog configuration. - - - The response also contains the view's full metadata, matching the view metadata JSON file. - - - The catalog configuration may contain credentials that should be used for subsequent requests for the - view. The configuration key "token" is used to pass an access token to be used as a bearer token - for view requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration - key. For example, "urn:ietf:params:oauth:token-type:jwt=". - parameters: - - $ref: '#/components/parameters/referenced-by' - responses: - 200: - $ref: '#/components/responses/LoadViewResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchViewException, view to load does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - examples: - ViewToLoadDoesNotExist: - $ref: '#/components/examples/NoSuchViewError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - post: - tags: - - Catalog API - summary: Replace a view - operationId: replaceView - parameters: - - $ref: '#/components/parameters/idempotency-key' - description: - Commit updates to a view. - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CommitViewRequest' - responses: - 200: - $ref: '#/components/responses/LoadViewResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchViewException, view to load does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - examples: - ViewToUpdateDoesNotExist: - $ref: '#/components/examples/NoSuchViewError' - 409: - description: - Conflict - CommitFailedException. The client may retry. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 500: - description: - An unknown server-side problem occurred; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - example: { - "error": { - "message": "Internal Server Error", - "type": "CommitStateUnknownException", - "code": 500 - } - } - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 502: - description: - A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - example: { - "error": { - "message": "Invalid response from the upstream server", - "type": "CommitStateUnknownException", - "code": 502 - } - } - 504: - description: - A server-side gateway timeout occurred; the commit state is unknown. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - example: { - "error": { - "message": "Gateway timed out during commit", - "type": "CommitStateUnknownException", - "code": 504 - } - } - 5XX: - description: - A server-side problem that might not be addressable on the client. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - example: { - "error": { - "message": "Bad Gateway", - "type": "InternalServerError", - "code": 502 - } - } - - delete: - tags: - - Catalog API - summary: Drop a view from the catalog - operationId: dropView - description: Remove a view from the catalog - parameters: - - $ref: '#/components/parameters/idempotency-key' - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - NoSuchViewException, view to drop does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - examples: - ViewToDeleteDoesNotExist: - $ref: '#/components/examples/NoSuchViewError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - head: - tags: - - Catalog API - summary: Check if a view exists - operationId: viewExists - description: - Check if a view exists within a given namespace. This request does not return a response body. - responses: - 204: - description: Success, no content - 400: - description: Bad Request - 401: - description: Unauthorized - 404: - description: Not Found - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/views/rename: - parameters: - - $ref: '#/components/parameters/prefix' - - post: - tags: - - Catalog API - summary: Rename a view from its current name to a new name - description: - Rename a view from one identifier to another. It's valid to move a view - across namespaces, but the server implementation is not required to support it. - operationId: renameView - parameters: - - $ref: '#/components/parameters/idempotency-key' - requestBody: - description: Current view identifier to rename and new view identifier to rename to - content: - application/json: - schema: - $ref: '#/components/schemas/RenameTableRequest' - examples: - RenameViewSameNamespace: - $ref: '#/components/examples/RenameViewSameNamespace' - required: true - responses: - 204: - description: Success, no content - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: - Not Found - - NoSuchViewException, view to rename does not exist - - NoSuchNamespaceException, The target namespace of the new identifier does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - examples: - ViewToRenameDoesNotExist: - $ref: '#/components/examples/NoSuchViewError' - NamespaceToRenameToDoesNotExist: - $ref: '#/components/examples/NoSuchNamespaceError' - 406: - $ref: '#/components/responses/UnsupportedOperationResponse' - 409: - description: Conflict - The target identifier to rename to already exists as a table or view - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - example: - $ref: '#/components/examples/ViewAlreadyExistsError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - - /v1/{prefix}/namespaces/{namespace}/register-view: - parameters: - - $ref: '#/components/parameters/prefix' - - $ref: '#/components/parameters/namespace' - - post: - tags: - - Catalog API - summary: Register a view in the catalog - parameters: - - $ref: '#/components/parameters/idempotency-key' - description: - Register a view in the given namespace using given metadata file location. - - operationId: registerView - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/RegisterViewRequest' - responses: - 200: - $ref: '#/components/responses/LoadViewResponse' - 400: - $ref: '#/components/responses/BadRequestErrorResponse' - 401: - $ref: '#/components/responses/UnauthorizedResponse' - 403: - $ref: '#/components/responses/ForbiddenResponse' - 404: - description: Not Found - The namespace specified does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - NamespaceNotFound: - $ref: '#/components/examples/NoSuchNamespaceError' - 409: - description: Conflict - The identifier already exists as a table or view - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - examples: - ViewAlreadyExists: - $ref: '#/components/examples/ViewAlreadyExistsError' - 419: - $ref: '#/components/responses/AuthenticationTimeoutResponse' - 503: - $ref: '#/components/responses/ServiceUnavailableResponse' - 5XX: - $ref: '#/components/responses/ServerErrorResponse' - -components: - ####################################################### - # Common Parameter Definitions Used In Several Routes # - ####################################################### - parameters: - namespace: - name: namespace - in: path - required: true - description: - A namespace identifier as a single string. - Multipart namespace parts must be separated by the namespace separator as - indicated via the /config override `namespace-separator`, which defaults to the unit separator `0x1F` byte (url encoded `%1F`). - To be compatible with older clients, servers must use both the advertised separator and `0x1F` as valid separators when decoding namespaces. - The `namespace-separator` should be provided in a url encoded form. - schema: - type: string - examples: - singlepart_namespace: - value: "accounting" - multipart_namespace: - value: "accounting%1Ftax" - - prefix: - name: prefix - in: path - schema: - type: string - required: true - description: An optional prefix in the path - - table: - name: table - in: path - description: A table name - required: true - schema: - type: string - example: "sales" - - plan-id: - name: plan-id - in: path - description: ID used to track a planning request - required: true - schema: - type: string - - view: - name: view - in: path - description: A view name - required: true - schema: - type: string - example: "sales" - - data-access: - name: X-Iceberg-Access-Delegation - in: header - description: > - Optional signal to the server that the client supports delegated access - via a comma-separated list of access mechanisms. The server may choose - to supply access via any or none of the requested mechanisms. - - - Specific properties and handling for `vended-credentials` and `remote-signing` - are documented in the `LoadTableResult` schema section of this spec document. - - required: false - schema: - type: string - enum: - - vended-credentials - - remote-signing - style: simple - explode: false - example: "vended-credentials,remote-signing" - - page-token: - name: pageToken - in: query - required: false - schema: - $ref: '#/components/schemas/PageToken' - - page-size: - name: pageSize - in: query - description: - For servers that support pagination, this signals an upper bound of the number of results that a client will receive. - For servers that do not support pagination, clients may receive results larger than the indicated `pageSize`. - required: false - schema: - type: integer - minimum: 1 - - etag: - name: ETag - in: header - description: - Identifies a unique version of the table metadata. - - Implementations that support ETags should produce unique tags for responses that return - different metadata content but represent the same version of table metadata. For example, - the `snapshots` query parameter may result in different metadata representations depending - on whether `refs` or `all` is provided, therefore should have distinct ETags. - required: false - schema: - type: string - - referenced-by: - name: referenced-by - in: query - description: - A comma-separated list of fully qualified view names (namespace and view name) representing the view - reference chain when an entity (table or view) is loaded via a view. The list should be ordered with the outermost view - first, followed by any intermediate views it references, down to the view that directly references the entity. - For a simple case where a view directly references the entity, the list contains a single view identifier. - For nested views (a view referencing another view which references the entity), the list contains multiple - view identifiers representing the complete dependency chain. - - The view identifier is a composite string of the format {namespace}{separator}{viewName}. - The namespace-separator (defined in /config) acts as the delimiter. - When parsing, the last occurrence of this separator identifies the boundary between the namespace and the view name. - Multipart namespaces must follow the encoding rules of the parent parameter in the list namespaces endpoint. - - Multiple view identifiers are separated by commas. Servers should split the parameter value on comma characters - to parse individual view identifiers. If view names contain commas, they must be url-encoded as %2C. - - Example with multiple views (where prod%1Fanalytics is a nested namespace which has a quarterly_view which references to monthly_view - which then references the entity being loaded) - prod%1Fanalytics%1Fquarterly_view,prod%1Fanalytics%1Fmonthly_view - required: false - schema: - type: string - - idempotency-key: - name: Idempotency-Key - in: header - required: false - schema: - type: string - format: uuid - minLength: 36 - maxLength: 36 - example: "017F22E2-79B0-7CC3-98C4-DC0C0C07398F" - description: | - Optional client-provided idempotency key for safe request retries. - - When present, the server ensures no additional effects for requests that carry the same - Idempotency-Key. If a prior request with this key has been finalized, the server returns - an equivalent final response without re-running the operation. The response body may - reflect a newer state of the catalog than existed at the time of the commit. - - Finalization rules: - - Finalize & replay: 200, 201, 204, and deterministic terminal 4xx (including 409 - such as AlreadyExists, NamespaceNotEmpty, etc.) - - Do not finalize (not stored/replayed): 5xx - - Key Requirements: - - Key format: UUIDv7 in string form (RFC 9562). - - The idempotency key must be globally unique (no reuse across different operations). - - Catalogs SHOULD NOT expire keys before the end of the advertised token lifetime. - - If Idempotency-Key is used, clients MUST reuse the same key when retrying the same - logical operation and MUST generate a new key for a different operation. - - ############################## - # Application Schema Objects # - ############################## - schemas: - - ErrorModel: - type: object - description: JSON error payload returned in a response with further details on the error - required: - - message - - type - - code - properties: - message: - type: string - description: Human-readable error message - type: - type: string - description: Internal type definition of the error - example: NoSuchNamespaceException - code: - type: integer - minimum: 400 - maximum: 600 - description: HTTP response code - example: 404 - stack: - type: array - items: - type: string - - CatalogConfig: - type: object - description: Server-provided configuration for the catalog. - required: - - defaults - - overrides - properties: - overrides: - type: object - additionalProperties: - type: string - description: - Properties that should be used to override client configuration; applied after defaults and client configuration. - defaults: - type: object - additionalProperties: - type: string - description: - Properties that should be used as default configuration; applied before client configuration. - endpoints: - type: array - items: - type: string - description: A list of endpoints that the server supports. The format of each endpoint must be " ". - The HTTP verb and the resource path must be separated by a space character. - example: [ - "GET /v1/{prefix}/namespaces/{namespace}", - "GET /v1/{prefix}/namespaces", - "POST /v1/{prefix}/namespaces", - "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "GET /v1/{prefix}/namespaces/{namespace}/views/{view}" - ] - idempotency-key-lifetime: - type: string - format: duration - description: - Client reuse window for an Idempotency-Key (ISO-8601 duration, e.g., PT30M, PT24H). - Interpreted as the maximum time from the first submission using a key to the last retry - during which a client may reuse that key. Servers SHOULD accept retries for at least this - duration and MAY include a grace period to account for delays/clock skew. Clients SHOULD NOT - reuse an Idempotency-Key after this window elapses; they SHOULD generate a new key for any - subsequent attempt. Presence of this field indicates the server supports Idempotency-Key - semantics for mutation endpoints. If absent, clients MUST assume idempotency is not supported. - example: PT30M - - CreateNamespaceRequest: - type: object - required: - - namespace - properties: - namespace: - $ref: '#/components/schemas/Namespace' - properties: - type: object - description: Configured string to string map of properties for the namespace - example: { "owner": "Hank Bendickson" } - default: { } - additionalProperties: - type: string - - UpdateNamespacePropertiesRequest: - type: object - properties: - removals: - type: array - uniqueItems: true - items: - type: string - example: [ "department", "access_group" ] - updates: - type: object - example: { "owner": "Hank Bendickson" } - additionalProperties: - type: string - - RenameTableRequest: - type: object - required: - - source - - destination - properties: - source: - $ref: '#/components/schemas/TableIdentifier' - destination: - $ref: '#/components/schemas/TableIdentifier' - - Namespace: - description: Reference to one or more levels of a namespace - type: array - items: - type: string - example: [ "accounting", "tax" ] - - PageToken: - description: - An opaque token that allows clients to make use of pagination for list APIs - (e.g. ListTables). Clients may initiate the first paginated request by sending an empty - query parameter `pageToken` to the server. - - Servers that support pagination should identify the `pageToken` parameter and return a - `next-page-token` in the response if there are more results available. After the initial - request, the value of `next-page-token` from each response must be used as the `pageToken` - parameter value for the next request. The server must return `null` value for the - `next-page-token` in the last response. - - Servers that support pagination must return all results in a single response with the value - of `next-page-token` set to `null` if the query parameter `pageToken` is not set in the - request. - - Servers that do not support pagination should ignore the `pageToken` parameter and return - all results in a single response. The `next-page-token` must be omitted from the response. - - Clients must interpret either `null` or missing response value of `next-page-token` as - the end of the listing results. - - type: string - nullable: true - - TableIdentifier: - type: object - required: - - namespace - - name - properties: - namespace: - $ref: '#/components/schemas/Namespace' - name: - type: string - nullable: false - - PrimitiveType: - type: string - example: - - "long" - - "string" - - "fixed[16]" - - "decimal(10,2)" - - StructField: - type: object - required: - - id - - name - - type - - required - properties: - id: - type: integer - name: - type: string - type: - $ref: '#/components/schemas/Type' - required: - type: boolean - doc: - type: string - initial-default: - $ref: "#/components/schemas/PrimitiveTypeValue" - write-default: - $ref: "#/components/schemas/PrimitiveTypeValue" - - StructType: - type: object - required: - - type - - fields - properties: - type: - type: string - const: "struct" - fields: - type: array - items: - $ref: '#/components/schemas/StructField' - - ListType: - type: object - required: - - type - - element-id - - element - - element-required - properties: - type: - type: string - const: "list" - element-id: - type: integer - element: - $ref: '#/components/schemas/Type' - element-required: - type: boolean - - MapType: - type: object - required: - - type - - key-id - - key - - value-id - - value - - value-required - properties: - type: - type: string - const: "map" - key-id: - type: integer - key: - $ref: '#/components/schemas/Type' - value-id: - type: integer - value: - $ref: '#/components/schemas/Type' - value-required: - type: boolean - - Type: - oneOf: - - $ref: '#/components/schemas/PrimitiveType' - - $ref: '#/components/schemas/StructType' - - $ref: '#/components/schemas/ListType' - - $ref: '#/components/schemas/MapType' - - Schema: - allOf: - - $ref: '#/components/schemas/StructType' - - type: object - properties: - schema-id: - type: integer - readOnly: true - identifier-field-ids: - type: array - items: - type: integer - - Expression: - oneOf: - - $ref: '#/components/schemas/TrueExpression' - - $ref: '#/components/schemas/FalseExpression' - - $ref: '#/components/schemas/AndOrExpression' - - $ref: '#/components/schemas/NotExpression' - - $ref: '#/components/schemas/SetExpression' - - $ref: '#/components/schemas/LiteralExpression' - - $ref: '#/components/schemas/UnaryExpression' - - ExpressionType: - type: string - example: - - "true" - - "false" - - "eq" - - "and" - - "or" - - "not" - - "in" - - "not-in" - - "lt" - - "lt-eq" - - "gt" - - "gt-eq" - - "not-eq" - - "starts-with" - - "not-starts-with" - - "is-null" - - "not-null" - - "is-nan" - - "not-nan" - - TrueExpression: - type: object - required: - - type - properties: - type: - $ref: '#/components/schemas/ExpressionType' - const: "true" - - FalseExpression: - type: object - required: - - type - properties: - type: - $ref: '#/components/schemas/ExpressionType' - const: "false" - - AndOrExpression: - type: object - required: - - type - - left - - right - properties: - type: - $ref: '#/components/schemas/ExpressionType' - enum: ["and", "or"] - left: - $ref: '#/components/schemas/Expression' - right: - $ref: '#/components/schemas/Expression' - - NotExpression: - type: object - required: - - type - - child - properties: - type: - $ref: '#/components/schemas/ExpressionType' - const: "not" - child: - $ref: '#/components/schemas/Expression' - - UnaryExpression: - type: object - required: - - type - - term - properties: - type: - $ref: '#/components/schemas/ExpressionType' - enum: ["is-null", "not-null", "is-nan", "not-nan"] - term: - $ref: '#/components/schemas/Term' - - LiteralExpression: - type: object - required: - - type - - term - - value - properties: - type: - $ref: '#/components/schemas/ExpressionType' - enum: ["lt", "lt-eq", "gt", "gt-eq", "eq", "not-eq", "starts-with", "not-starts-with"] - term: - $ref: '#/components/schemas/Term' - value: - $ref: '#/components/schemas/PrimitiveTypeValue' - - SetExpression: - type: object - required: - - type - - term - - values - properties: - type: - $ref: '#/components/schemas/ExpressionType' - enum: ["in", "not-in"] - term: - $ref: '#/components/schemas/Term' - values: - type: array - items: - $ref: '#/components/schemas/PrimitiveTypeValue' - - Term: - oneOf: - - $ref: '#/components/schemas/Reference' - - $ref: '#/components/schemas/TransformTerm' - - Reference: - type: string - example: - - "column-name" - - TransformTerm: - type: object - required: - - type - - transform - - term - properties: - type: - type: string - const: "transform" - transform: - $ref: '#/components/schemas/Transform' - term: - $ref: '#/components/schemas/Reference' - - Transform: - type: string - example: - - "identity" - - "year" - - "month" - - "day" - - "hour" - - "bucket[256]" - - "truncate[16]" - - PartitionField: - type: object - required: - - source-id - - transform - - name - properties: - field-id: - type: integer - source-id: - type: integer - name: - type: string - transform: - $ref: '#/components/schemas/Transform' - - PartitionSpec: - type: object - required: - - fields - properties: - spec-id: - type: integer - readOnly: true - fields: - type: array - items: - $ref: '#/components/schemas/PartitionField' - - SortDirection: - type: string - enum: ["asc", "desc"] - - NullOrder: - type: string - enum: ["nulls-first", "nulls-last"] - - SortField: - type: object - required: - - source-id - - transform - - direction - - null-order - properties: - source-id: - type: integer - transform: - $ref: '#/components/schemas/Transform' - direction: - $ref: '#/components/schemas/SortDirection' - null-order: - $ref: '#/components/schemas/NullOrder' - - SortOrder: - type: object - required: - - order-id - - fields - properties: - order-id: - type: integer - readOnly: true - fields: - type: array - items: - $ref: '#/components/schemas/SortField' - - EncryptedKey: - type: object - required: - - key-id - - encrypted-key-metadata - properties: - key-id: - type: string - encrypted-key-metadata: - type: string - format: byte # for compatibility - contentEncoding: base64 - encrypted-by-id: - type: string - properties: - type: object - additionalProperties: - type: string - - Snapshot: - type: object - required: - - snapshot-id - - timestamp-ms - - manifest-list - - summary - properties: - snapshot-id: - type: integer - format: int64 - parent-snapshot-id: - type: integer - format: int64 - sequence-number: - type: integer - format: int64 - timestamp-ms: - type: integer - format: int64 - manifest-list: - type: string - description: Location of the snapshot's manifest list file - first-row-id: - type: integer - format: int64 - description: The first _row_id assigned to the first row in the first data file in the first manifest - added-rows: - type: integer - format: int64 - description: The upper bound of the number of rows with assigned row IDs - summary: - type: object - required: - - operation - properties: - operation: - type: string - enum: ["append", "replace", "overwrite", "delete"] - additionalProperties: - type: string - schema-id: - type: integer - - SnapshotReference: - type: object - required: - - type - - snapshot-id - properties: - type: - type: string - enum: ["tag", "branch"] - snapshot-id: - type: integer - format: int64 - max-ref-age-ms: - type: integer - format: int64 - max-snapshot-age-ms: - type: integer - format: int64 - min-snapshots-to-keep: - type: integer - - SnapshotReferences: - type: object - additionalProperties: - $ref: '#/components/schemas/SnapshotReference' - - SnapshotLog: - type: array - items: - type: object - required: - - snapshot-id - - timestamp-ms - properties: - snapshot-id: - type: integer - format: int64 - timestamp-ms: - type: integer - format: int64 - - MetadataLog: - type: array - items: - type: object - required: - - metadata-file - - timestamp-ms - properties: - metadata-file: - type: string - timestamp-ms: - type: integer - format: int64 - - TableMetadata: - type: object - required: - - format-version - - table-uuid - properties: - format-version: - type: integer - minimum: 1 - maximum: 3 - table-uuid: - type: string - location: - type: string - last-updated-ms: - type: integer - format: int64 - next-row-id: - type: integer - format: int64 - description: A long higher than all assigned row IDs; the next snapshot's first-row-id. - properties: - type: object - additionalProperties: - type: string - # schema tracking - schemas: - type: array - items: - $ref: '#/components/schemas/Schema' - current-schema-id: - type: integer - last-column-id: - type: integer - # partition spec tracking - partition-specs: - type: array - items: - $ref: '#/components/schemas/PartitionSpec' - default-spec-id: - type: integer - last-partition-id: - type: integer - # sort order tracking - sort-orders: - type: array - items: - $ref: '#/components/schemas/SortOrder' - default-sort-order-id: - type: integer - # encryption - encryption-keys: - type: array - items: - $ref: '#/components/schemas/EncryptedKey' - # snapshot tracking - snapshots: - type: array - items: - $ref: '#/components/schemas/Snapshot' - refs: - $ref: '#/components/schemas/SnapshotReferences' - current-snapshot-id: - type: integer - format: int64 - last-sequence-number: - type: integer - format: int64 - # logs - snapshot-log: - $ref: '#/components/schemas/SnapshotLog' - metadata-log: - $ref: '#/components/schemas/MetadataLog' - # statistics - statistics: - type: array - items: - $ref: '#/components/schemas/StatisticsFile' - partition-statistics: - type: array - items: - $ref: '#/components/schemas/PartitionStatisticsFile' - - SQLViewRepresentation: - type: object - required: - - type - - sql - - dialect - properties: - type: - type: string - sql: - type: string - dialect: - type: string - - ViewRepresentation: - oneOf: - - $ref: '#/components/schemas/SQLViewRepresentation' - - ViewHistoryEntry: - type: object - required: - - version-id - - timestamp-ms - properties: - version-id: - type: integer - timestamp-ms: - type: integer - format: int64 - - ViewVersion: - type: object - required: - - version-id - - timestamp-ms - - schema-id - - summary - - representations - - default-namespace - properties: - version-id: - type: integer - timestamp-ms: - type: integer - format: int64 - schema-id: - type: integer - description: Schema ID to set as current, or -1 to set last added schema - summary: - type: object - additionalProperties: - type: string - representations: - type: array - items: - $ref: '#/components/schemas/ViewRepresentation' - default-catalog: - type: string - default-namespace: - $ref: '#/components/schemas/Namespace' - - ViewMetadata: - type: object - required: - - view-uuid - - format-version - - location - - current-version-id - - versions - - version-log - - schemas - properties: - view-uuid: - type: string - format-version: - type: integer - minimum: 1 - maximum: 1 - location: - type: string - current-version-id: - type: integer - versions: - type: array - items: - $ref: '#/components/schemas/ViewVersion' - version-log: - type: array - items: - $ref: '#/components/schemas/ViewHistoryEntry' - schemas: - type: array - items: - $ref: '#/components/schemas/Schema' - properties: - type: object - additionalProperties: - type: string - - BaseUpdate: - discriminator: - propertyName: action - mapping: - assign-uuid: '#/components/schemas/AssignUUIDUpdate' - upgrade-format-version: '#/components/schemas/UpgradeFormatVersionUpdate' - add-schema: '#/components/schemas/AddSchemaUpdate' - set-current-schema: '#/components/schemas/SetCurrentSchemaUpdate' - add-spec: '#/components/schemas/AddPartitionSpecUpdate' - set-default-spec: '#/components/schemas/SetDefaultSpecUpdate' - add-sort-order: '#/components/schemas/AddSortOrderUpdate' - set-default-sort-order: '#/components/schemas/SetDefaultSortOrderUpdate' - add-snapshot: '#/components/schemas/AddSnapshotUpdate' - set-snapshot-ref: '#/components/schemas/SetSnapshotRefUpdate' - remove-snapshots: '#/components/schemas/RemoveSnapshotsUpdate' - remove-snapshot-ref: '#/components/schemas/RemoveSnapshotRefUpdate' - set-location: '#/components/schemas/SetLocationUpdate' - set-properties: '#/components/schemas/SetPropertiesUpdate' - remove-properties: '#/components/schemas/RemovePropertiesUpdate' - add-view-version: '#/components/schemas/AddViewVersionUpdate' - set-current-view-version: '#/components/schemas/SetCurrentViewVersionUpdate' - set-statistics: '#/components/schemas/SetStatisticsUpdate' - remove-statistics: '#/components/schemas/RemoveStatisticsUpdate' - set-partition-statistics: '#/components/schemas/SetPartitionStatisticsUpdate' - remove-partition-statistics: '#/components/schemas/RemovePartitionStatisticsUpdate' - remove-partition-specs: '#/components/schemas/RemovePartitionSpecsUpdate' - remove-schemas: '#/components/schemas/RemoveSchemasUpdate' - add-encryption-key: '#/components/schemas/AddEncryptionKeyUpdate' - remove-encryption-key: '#/components/schemas/RemoveEncryptionKeyUpdate' - type: object - required: - - action - properties: - action: - type: string - - AssignUUIDUpdate: - description: Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - uuid - properties: - action: - type: string - const: "assign-uuid" - uuid: - type: string - - UpgradeFormatVersionUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - format-version - properties: - action: - type: string - const: "upgrade-format-version" - format-version: - type: integer - - AddSchemaUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - schema - properties: - action: - type: string - const: "add-schema" - schema: - $ref: '#/components/schemas/Schema' - last-column-id: - type: integer - deprecated: true - description: - This optional field is **DEPRECATED for REMOVAL** since it more safe to handle this internally, - and shouldn't be exposed to the clients. - - The highest assigned column ID for the table. This is used to ensure columns are always - assigned an unused ID when evolving schemas. When omitted, it will be computed on the server side. - - SetCurrentSchemaUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - schema-id - properties: - action: - type: string - const: "set-current-schema" - schema-id: - type: integer - description: Schema ID to set as current, or -1 to set last added schema - - AddPartitionSpecUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - spec - properties: - action: - type: string - const: "add-spec" - spec: - $ref: '#/components/schemas/PartitionSpec' - - SetDefaultSpecUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - spec-id - properties: - action: - type: string - const: "set-default-spec" - spec-id: - type: integer - description: Partition spec ID to set as the default, or -1 to set last added spec - - AddSortOrderUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - sort-order - properties: - action: - type: string - const: "add-sort-order" - sort-order: - $ref: '#/components/schemas/SortOrder' - - SetDefaultSortOrderUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - sort-order-id - properties: - action: - type: string - const: "set-default-sort-order" - sort-order-id: - type: integer - description: Sort order ID to set as the default, or -1 to set last added sort order - - AddSnapshotUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - snapshot - properties: - action: - type: string - const: "add-snapshot" - snapshot: - $ref: '#/components/schemas/Snapshot' - - SetSnapshotRefUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - - $ref: '#/components/schemas/SnapshotReference' - required: - - ref-name - properties: - action: - type: string - const: "set-snapshot-ref" - ref-name: - type: string - - RemoveSnapshotsUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - snapshot-ids - properties: - action: - type: string - const: "remove-snapshots" - snapshot-ids: - type: array - items: - type: integer - format: int64 - - RemoveSnapshotRefUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - ref-name - properties: - action: - type: string - const: "remove-snapshot-ref" - ref-name: - type: string - - SetLocationUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - location - properties: - action: - type: string - const: "set-location" - location: - type: string - - SetPropertiesUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - updates - properties: - action: - type: string - const: "set-properties" - updates: - type: object - additionalProperties: - type: string - - RemovePropertiesUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - removals - properties: - action: - type: string - const: "remove-properties" - removals: - type: array - items: - type: string - - AddViewVersionUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - view-version - properties: - action: - type: string - const: "add-view-version" - view-version: - $ref: '#/components/schemas/ViewVersion' - - SetCurrentViewVersionUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - view-version-id - properties: - action: - type: string - const: "set-current-view-version" - view-version-id: - type: integer - description: The view version id to set as current, or -1 to set last added view version id - - SetStatisticsUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - statistics - properties: - action: - type: string - const: "set-statistics" - snapshot-id: - type: integer - format: int64 - deprecated: true - description: - This optional field is **DEPRECATED for REMOVAL** since it contains redundant information. - Clients should use the `statistics.snapshot-id` field instead. - statistics: - $ref: '#/components/schemas/StatisticsFile' - - RemoveStatisticsUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - snapshot-id - properties: - action: - type: string - const: "remove-statistics" - snapshot-id: - type: integer - format: int64 - - SetPartitionStatisticsUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - partition-statistics - properties: - action: - type: string - const: "set-partition-statistics" - partition-statistics: - $ref: '#/components/schemas/PartitionStatisticsFile' - - RemovePartitionStatisticsUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - snapshot-id - properties: - action: - type: string - const: "remove-partition-statistics" - snapshot-id: - type: integer - format: int64 - - RemovePartitionSpecsUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - spec-ids - properties: - action: - type: string - const: "remove-partition-specs" - spec-ids: - type: array - items: - type: integer - - RemoveSchemasUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - schema-ids - properties: - action: - type: string - const: "remove-schemas" - schema-ids: - type: array - items: - type: integer - - AddEncryptionKeyUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - encryption-key - properties: - action: - type: string - const: "add-encryption-key" - encryption-key: - $ref: '#/components/schemas/EncryptedKey' - - RemoveEncryptionKeyUpdate: - allOf: - - $ref: '#/components/schemas/BaseUpdate' - required: - - key-id - properties: - action: - type: string - const: "remove-encryption-key" - key-id: - type: string - - TableUpdate: - anyOf: - - $ref: '#/components/schemas/AssignUUIDUpdate' - - $ref: '#/components/schemas/UpgradeFormatVersionUpdate' - - $ref: '#/components/schemas/AddSchemaUpdate' - - $ref: '#/components/schemas/SetCurrentSchemaUpdate' - - $ref: '#/components/schemas/AddPartitionSpecUpdate' - - $ref: '#/components/schemas/SetDefaultSpecUpdate' - - $ref: '#/components/schemas/AddSortOrderUpdate' - - $ref: '#/components/schemas/SetDefaultSortOrderUpdate' - - $ref: '#/components/schemas/AddSnapshotUpdate' - - $ref: '#/components/schemas/SetSnapshotRefUpdate' - - $ref: '#/components/schemas/RemoveSnapshotsUpdate' - - $ref: '#/components/schemas/RemoveSnapshotRefUpdate' - - $ref: '#/components/schemas/SetLocationUpdate' - - $ref: '#/components/schemas/SetPropertiesUpdate' - - $ref: '#/components/schemas/RemovePropertiesUpdate' - - $ref: '#/components/schemas/SetStatisticsUpdate' - - $ref: '#/components/schemas/RemoveStatisticsUpdate' - - $ref: '#/components/schemas/SetPartitionStatisticsUpdate' - - $ref: '#/components/schemas/RemovePartitionStatisticsUpdate' - - $ref: '#/components/schemas/RemovePartitionSpecsUpdate' - - $ref: '#/components/schemas/RemoveSchemasUpdate' - - $ref: '#/components/schemas/AddEncryptionKeyUpdate' - - $ref: '#/components/schemas/RemoveEncryptionKeyUpdate' - - ViewUpdate: - anyOf: - - $ref: '#/components/schemas/AssignUUIDUpdate' - - $ref: '#/components/schemas/UpgradeFormatVersionUpdate' - - $ref: '#/components/schemas/AddSchemaUpdate' - - $ref: '#/components/schemas/SetLocationUpdate' - - $ref: '#/components/schemas/SetPropertiesUpdate' - - $ref: '#/components/schemas/RemovePropertiesUpdate' - - $ref: '#/components/schemas/AddViewVersionUpdate' - - $ref: '#/components/schemas/SetCurrentViewVersionUpdate' - - TableRequirement: - type: object - discriminator: - propertyName: type - mapping: - assert-create: '#/components/schemas/AssertCreate' - assert-table-uuid: '#/components/schemas/AssertTableUUID' - assert-ref-snapshot-id: '#/components/schemas/AssertRefSnapshotId' - assert-last-assigned-field-id: '#/components/schemas/AssertLastAssignedFieldId' - assert-current-schema-id: '#/components/schemas/AssertCurrentSchemaId' - assert-last-assigned-partition-id: '#/components/schemas/AssertLastAssignedPartitionId' - assert-default-spec-id: '#/components/schemas/AssertDefaultSpecId' - assert-default-sort-order-id: '#/components/schemas/AssertDefaultSortOrderId' - properties: - type: - type: string - required: - - type - - AssertCreate: - type: object - description: The table must not already exist; used for create transactions - allOf: - - $ref: '#/components/schemas/TableRequirement' - required: - - type - properties: - type: - type: string - const: "assert-create" - - AssertTableUUID: - description: The table UUID must match the requirement's `uuid` - allOf: - - $ref: '#/components/schemas/TableRequirement' - required: - - type - - uuid - properties: - type: - type: string - const: "assert-table-uuid" - uuid: - type: string - - AssertRefSnapshotId: - description: | - The table branch or tag identified by the requirement's `ref` must reference the requirement's `snapshot-id`. - The `snapshot-id` field is required in this object, but in the case of a `null` - the ref must not already exist. - allOf: - - $ref: '#/components/schemas/TableRequirement' - required: - - ref - - snapshot-id - properties: - type: - type: string - const: "assert-ref-snapshot-id" - ref: - type: string - snapshot-id: - type: integer - format: int64 - nullable: true - - AssertLastAssignedFieldId: - description: - The table's last assigned column id must match the requirement's `last-assigned-field-id` - allOf: - - $ref: '#/components/schemas/TableRequirement' - required: - - last-assigned-field-id - properties: - type: - type: string - const: "assert-last-assigned-field-id" - last-assigned-field-id: - type: integer - - AssertCurrentSchemaId: - description: - The table's current schema id must match the requirement's `current-schema-id` - allOf: - - $ref: '#/components/schemas/TableRequirement' - required: - - current-schema-id - properties: - type: - type: string - const: "assert-current-schema-id" - current-schema-id: - type: integer - - AssertLastAssignedPartitionId: - description: - The table's last assigned partition id must match the requirement's `last-assigned-partition-id` - allOf: - - $ref: '#/components/schemas/TableRequirement' - required: - - last-assigned-partition-id - properties: - type: - type: string - const: "assert-last-assigned-partition-id" - last-assigned-partition-id: - type: integer - - AssertDefaultSpecId: - description: - The table's default spec id must match the requirement's `default-spec-id` - allOf: - - $ref: '#/components/schemas/TableRequirement' - required: - - default-spec-id - properties: - type: - type: string - const: "assert-default-spec-id" - default-spec-id: - type: integer - - AssertDefaultSortOrderId: - description: - The table's default sort order id must match the requirement's `default-sort-order-id` - allOf: - - $ref: '#/components/schemas/TableRequirement' - required: - - default-sort-order-id - properties: - type: - type: string - const: "assert-default-sort-order-id" - default-sort-order-id: - type: integer - - ViewRequirement: - type: object - discriminator: - propertyName: type - mapping: - assert-view-uuid: '#/components/schemas/AssertViewUUID' - oneOf: - - $ref: '#/components/schemas/AssertViewUUID' - - AssertViewUUID: - description: The view UUID must match the requirement's `uuid` - required: - - type - - uuid - properties: - type: - type: string - const: "assert-view-uuid" - uuid: - type: string - - StorageCredential: - type: object - required: - - prefix - - config - properties: - prefix: - type: string - description: Indicates a storage location prefix where the credential is relevant. Clients should choose the most - specific prefix (by selecting the longest prefix) if several credentials of the same type are available. - config: - type: object - additionalProperties: - type: string - - LoadCredentialsResponse: - type: object - required: - - storage-credentials - properties: - storage-credentials: - type: array - items: - $ref: '#/components/schemas/StorageCredential' - - LoadTableResult: - description: | - Result used when a table is successfully loaded. - - - The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. - Clients can check whether metadata has changed by comparing metadata locations after the table has been created. - - - The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. - - - The following configurations should be respected by clients: - - ## General Configurations - - - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled - - `scan-planning-mode`: Communicates to clients the supported planning mode. Clients should use this value to fail fast if the supported scanning mode is not available on the client. Valid values: - - `client`: Clients MUST use client-side scan planning - - `server`: Clients MUST use server-side scan planning via the `planTableScan` endpoint - - ## AWS Configurations - - The following configurations should be respected when working with tables stored in AWS S3 - - `client.region`: region to configure client for making requests to AWS - - `s3.access-key-id`: id for credentials that provide access to the data in S3 - - `s3.secret-access-key`: secret for credentials that provide access to data in S3 - - `s3.session-token`: if present, this value should be used for as the session token - - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `RemoteSignRequest` schema section of this spec document. - - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled - - ## Storage Credentials - - Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. - Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. - - ## Remote Signing - - If remote signing for a specific storage provider is enabled, clients must respect the following configurations when creating a remote signer client: - - `signer.endpoint`: the remote signer endpoint. Required. Can either be a relative path (to be resolved against `signer.uri`) or an absolute URI. - - `signer.uri`: the base URI to resolve `signer.endpoint` against. Optional. Only meaningful if `signer.endpoint` is a relative path. Defaults to the catalog's base URI if not set. - type: object - required: - - metadata - properties: - metadata-location: - type: string - description: May be null if the table is staged as part of a transaction - nullable: true - metadata: - $ref: '#/components/schemas/TableMetadata' - config: - type: object - additionalProperties: - type: string - storage-credentials: - type: array - items: - $ref: '#/components/schemas/StorageCredential' - - ScanTasks: - type: object - description: > - Scan and planning tasks for server-side scan planning - - - - `plan-tasks` contains opaque units of planning work - - - `file-scan-tasks` contains a partial or complete list of table scan tasks - - - `delete-files` contains delete files referenced by file scan tasks - - - Each plan task must be passed to the fetchScanTasks endpoint to fetch - the file scan tasks for the plan task. - - - The list of delete files must contain all delete files referenced by - the file scan tasks. - properties: - delete-files: - description: Delete files referenced by file scan tasks - type: array - items: - $ref: '#/components/schemas/DeleteFile' - file-scan-tasks: - type: array - items: - $ref: '#/components/schemas/FileScanTask' - plan-tasks: - type: array - items: - $ref: '#/components/schemas/PlanTask' - - CompletedPlanningResult: - type: object - description: Completed server-side planning result - allOf: - - $ref: '#/components/schemas/ScanTasks' - - type: object - required: - - status - properties: - status: - $ref: '#/components/schemas/PlanStatus' - const: "completed" - storage-credentials: - type: array - description: - Storage credentials for accessing the files returned in the scan result. - - If the server returns storage credentials as part of the completed scan - planning response, the expectation is for the client to use these credentials - to read the files returned in the FileScanTasks as part of the scan result. - items: - $ref: '#/components/schemas/StorageCredential' - - CompletedPlanningWithIDResult: - type: object - allOf: - - $ref: '#/components/schemas/CompletedPlanningResult' - - type: object - required: - - plan-id - properties: - plan-id: - description: ID used to track a planning request - type: string - - FailedPlanningResult: - type: object - description: Failed server-side planning result - allOf: - - $ref: '#/components/schemas/IcebergErrorResponse' - - type: object - required: - - status - properties: - status: - $ref: '#/components/schemas/PlanStatus' - const: "failed" - - AsyncPlanningResult: - type: object - required: - - status - - plan-id - properties: - status: - $ref: '#/components/schemas/PlanStatus' - const: "submitted" - plan-id: - description: ID used to track a planning request - type: string - - EmptyPlanningResult: - type: object - description: Empty server-side planning result - required: - - status - properties: - status: - $ref: '#/components/schemas/PlanStatus' - enum: ["submitted", "cancelled"] - - PlanStatus: - description: Status of a server-side planning operation - type: string - enum: ["completed", "submitted", "cancelled", "failed"] - - FetchPlanningResult: - type: object - description: Result of server-side scan planning for fetchPlanningResult - discriminator: - propertyName: status - mapping: - completed: '#/components/schemas/CompletedPlanningResult' - submitted: '#/components/schemas/EmptyPlanningResult' - cancelled: '#/components/schemas/EmptyPlanningResult' - failed: '#/components/schemas/FailedPlanningResult' - oneOf: - - $ref: '#/components/schemas/CompletedPlanningResult' - - $ref: '#/components/schemas/FailedPlanningResult' - - $ref: '#/components/schemas/EmptyPlanningResult' - - PlanTableScanResult: - type: object - description: Result of server-side scan planning for planTableScan - discriminator: - propertyName: status - mapping: - completed: '#/components/schemas/CompletedPlanningWithIDResult' - submitted: '#/components/schemas/AsyncPlanningResult' - cancelled: '#/components/schemas/EmptyPlanningResult' - failed: '#/components/schemas/FailedPlanningResult' - oneOf: - - $ref: '#/components/schemas/CompletedPlanningWithIDResult' - - $ref: '#/components/schemas/FailedPlanningResult' - - $ref: '#/components/schemas/AsyncPlanningResult' - - $ref: '#/components/schemas/EmptyPlanningResult' - - FetchScanTasksResult: - type: object - description: Response schema for fetchScanTasks - allOf: - - $ref: '#/components/schemas/ScanTasks' - - CommitTableRequest: - type: object - required: - - requirements - - updates - properties: - identifier: - description: Table identifier to update; must be present for CommitTransactionRequest - $ref: '#/components/schemas/TableIdentifier' - requirements: - type: array - items: - $ref: '#/components/schemas/TableRequirement' - updates: - type: array - items: - $ref: '#/components/schemas/TableUpdate' - - CommitViewRequest: - type: object - required: - - updates - properties: - identifier: - description: View identifier to update - $ref: '#/components/schemas/TableIdentifier' - requirements: - type: array - items: - $ref: '#/components/schemas/ViewRequirement' - updates: - type: array - items: - $ref: '#/components/schemas/ViewUpdate' - - CommitTransactionRequest: - type: object - required: - - table-changes - properties: - table-changes: - type: array - items: - description: Table commit request; must provide an `identifier` - $ref: '#/components/schemas/CommitTableRequest' - - CreateTableRequest: - type: object - required: - - name - - schema - properties: - name: - type: string - location: - type: string - schema: - $ref: '#/components/schemas/Schema' - partition-spec: - $ref: '#/components/schemas/PartitionSpec' - write-order: - $ref: '#/components/schemas/SortOrder' - stage-create: - type: boolean - properties: - type: object - additionalProperties: - type: string - - RegisterTableRequest: - type: object - required: - - name - - metadata-location - properties: - name: - type: string - metadata-location: - type: string - overwrite: - description: Whether to overwrite table metadata if the table already exists - type: boolean - default: false - - CreateViewRequest: - type: object - required: - - name - - schema - - view-version - - properties - properties: - name: - type: string - location: - type: string - schema: - $ref: '#/components/schemas/Schema' - view-version: - $ref: '#/components/schemas/ViewVersion' - description: The view version to create, will replace the schema-id sent within the view-version with the id assigned to the provided schema - properties: - type: object - additionalProperties: - type: string - - LoadViewResult: - description: | - Result used when a view is successfully loaded. - - - The view metadata JSON is returned in the `metadata` field. The corresponding file location of view metadata is returned in the `metadata-location` field. - Clients can check whether metadata has changed by comparing metadata locations after the view has been created. - - The `config` map returns view-specific configuration for the view's resources. - - The following configurations should be respected by clients: - - ## General Configurations - - - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled - type: object - required: - - metadata-location - - metadata - properties: - metadata-location: - type: string - metadata: - $ref: '#/components/schemas/ViewMetadata' - config: - type: object - additionalProperties: - type: string - - RegisterViewRequest: - type: object - required: - - name - - metadata-location - properties: - name: - type: string - metadata-location: - type: string - - TokenType: - type: string - enum: - - urn:ietf:params:oauth:token-type:access_token - - urn:ietf:params:oauth:token-type:refresh_token - - urn:ietf:params:oauth:token-type:id_token - - urn:ietf:params:oauth:token-type:saml1 - - urn:ietf:params:oauth:token-type:saml2 - - urn:ietf:params:oauth:token-type:jwt - description: - Token type identifier, from RFC 8693 Section 3 - - - See https://datatracker.ietf.org/doc/html/rfc8693#section-3 - - OAuthClientCredentialsRequest: - deprecated: true - description: - The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this - spec, see description of the endpoint. - - - OAuth2 client credentials request - - - See https://datatracker.ietf.org/doc/html/rfc6749#section-4.4 - type: object - required: - - grant_type - - client_id - - client_secret - properties: - grant_type: - type: string - enum: - - client_credentials - scope: - type: string - client_id: - type: string - description: - Client ID - - - This can be sent in the request body, but OAuth2 recommends sending it in - a Basic Authorization header. - client_secret: - type: string - description: - Client secret - - - This can be sent in the request body, but OAuth2 recommends sending it in - a Basic Authorization header. - - OAuthTokenExchangeRequest: - deprecated: true - description: - The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this - spec, see description of the endpoint. - - - OAuth2 token exchange request - - - See https://datatracker.ietf.org/doc/html/rfc8693 - type: object - required: - - grant_type - - subject_token - - subject_token_type - properties: - grant_type: - type: string - enum: - - urn:ietf:params:oauth:grant-type:token-exchange - scope: - type: string - requested_token_type: - $ref: '#/components/schemas/TokenType' - subject_token: - type: string - description: Subject token for token exchange request - subject_token_type: - $ref: '#/components/schemas/TokenType' - actor_token: - type: string - description: Actor token for token exchange request - actor_token_type: - $ref: '#/components/schemas/TokenType' - - OAuthTokenRequest: - deprecated: true - description: - The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this - spec, see description of the endpoint. - anyOf: - - $ref: '#/components/schemas/OAuthClientCredentialsRequest' - - $ref: '#/components/schemas/OAuthTokenExchangeRequest' - - CounterResult: - type: object - required: - - unit - - value - properties: - unit: - type: string - value: - type: integer - format: int64 - - TimerResult: - type: object - required: - - time-unit - - count - - total-duration - properties: - time-unit: - type: string - count: - type: integer - format: int64 - total-duration: - type: integer - format: int64 - - MetricResult: - anyOf: - - $ref: '#/components/schemas/CounterResult' - - $ref: '#/components/schemas/TimerResult' - - Metrics: - type: object - additionalProperties: - $ref: '#/components/schemas/MetricResult' - example: { - "metrics": { - "total-planning-duration": { - "count": 1, - "time-unit": "nanoseconds", - "total-duration": 2644235116 - }, - "result-data-files": { - "unit": "count", - "value": 1, - }, - "result-delete-files": { - "unit": "count", - "value": 0, - }, - "total-data-manifests": { - "unit": "count", - "value": 1, - }, - "total-delete-manifests": { - "unit": "count", - "value": 0, - }, - "scanned-data-manifests": { - "unit": "count", - "value": 1, - }, - "skipped-data-manifests": { - "unit": "count", - "value": 0, - }, - "total-file-size-bytes": { - "unit": "bytes", - "value": 10, - }, - "total-delete-file-size-bytes": { - "unit": "bytes", - "value": 0, - } - } - } - - ReportMetricsRequest: - anyOf: - - $ref: '#/components/schemas/ScanReport' - - $ref: '#/components/schemas/CommitReport' - required: - - report-type - properties: - report-type: - type: string - - ScanReport: - type: object - required: - - table-name - - snapshot-id - - filter - - schema-id - - projected-field-ids - - projected-field-names - - metrics - properties: - table-name: - type: string - snapshot-id: - type: integer - format: int64 - filter: - $ref: '#/components/schemas/Expression' - schema-id: - type: integer - projected-field-ids: - type: array - items: - type: integer - projected-field-names: - type: array - items: - type: string - metrics: - $ref: '#/components/schemas/Metrics' - metadata: - type: object - additionalProperties: - type: string - - CommitReport: - type: object - required: - - table-name - - snapshot-id - - sequence-number - - operation - - metrics - properties: - table-name: - type: string - snapshot-id: - type: integer - format: int64 - sequence-number: - type: integer - format: int64 - operation: - type: string - metrics: - $ref: '#/components/schemas/Metrics' - metadata: - type: object - additionalProperties: - type: string - - OAuthError: - deprecated: true - description: - The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this - spec, see description of the endpoint. - type: object - required: - - error - properties: - error: - type: string - enum: - - invalid_request - - invalid_client - - invalid_grant - - unauthorized_client - - unsupported_grant_type - - invalid_scope - error_description: - type: string - error_uri: - type: string - - OAuthTokenResponse: - deprecated: true - description: - The `oauth/tokens` endpoint and related schemas are **DEPRECATED for REMOVAL** from this - spec, see description of the endpoint. - type: object - required: - - access_token - - token_type - properties: - access_token: - type: string - description: - The access token, for client credentials or token exchange - token_type: - type: string - enum: - - bearer - - mac - - N_A - description: - Access token type for client credentials or token exchange - - - See https://datatracker.ietf.org/doc/html/rfc6749#section-7.1 - expires_in: - type: integer - description: - Lifetime of the access token in seconds for client credentials or token exchange - issued_token_type: - $ref: '#/components/schemas/TokenType' - refresh_token: - type: string - description: Refresh token for client credentials or token exchange - scope: - type: string - description: Authorization scope for client credentials or token exchange - - IcebergErrorResponse: - description: JSON wrapper for all error responses (non-2xx) - type: object - required: - - error - properties: - error: - $ref: '#/components/schemas/ErrorModel' - additionalProperties: false - example: - { - "error": { - "message": "The server does not support this operation", - "type": "UnsupportedOperationException", - "code": 406 - } - } - - CreateNamespaceResponse: - type: object - required: - - namespace - properties: - namespace: - $ref: '#/components/schemas/Namespace' - properties: - type: object - additionalProperties: - type: string - description: - Properties stored on the namespace, if supported by the server. - example: { "owner": "Ralph", "created_at": "1452120468" } - default: { } - - GetNamespaceResponse: - type: object - required: - - namespace - properties: - namespace: - $ref: '#/components/schemas/Namespace' - properties: - type: object - description: - Properties stored on the namespace, if supported by the server. - If the server does not support namespace properties, it should return null for this field. - If namespace properties are supported, but none are set, it should return an empty object. - additionalProperties: - type: string - example: { "owner": "Ralph", 'transient_lastDdlTime': '1452120468' } - default: { } - nullable: true - - ListTablesResponse: - type: object - properties: - next-page-token: - $ref: '#/components/schemas/PageToken' - identifiers: - type: array - uniqueItems: true - items: - $ref: '#/components/schemas/TableIdentifier' - - ListNamespacesResponse: - type: object - properties: - next-page-token: - $ref: '#/components/schemas/PageToken' - namespaces: - type: array - uniqueItems: true - items: - $ref: '#/components/schemas/Namespace' - - UpdateNamespacePropertiesResponse: - type: object - required: - - updated - - removed - properties: - updated: - description: List of property keys that were added or updated - type: array - uniqueItems: true - items: - type: string - removed: - description: List of properties that were removed - type: array - items: - type: string - missing: - type: array - items: - type: string - description: - List of properties requested for removal that were not found - in the namespace's properties. Represents a partial success response. - Server's do not need to implement this. - nullable: true - - CommitTableResponse: - type: object - required: - - metadata-location - - metadata - properties: - metadata-location: - type: string - metadata: - $ref: '#/components/schemas/TableMetadata' - - StatisticsFile: - type: object - required: - - snapshot-id - - statistics-path - - file-size-in-bytes - - file-footer-size-in-bytes - - blob-metadata - properties: - snapshot-id: - type: integer - format: int64 - statistics-path: - type: string - file-size-in-bytes: - type: integer - format: int64 - file-footer-size-in-bytes: - type: integer - format: int64 - blob-metadata: - type: array - items: - $ref: '#/components/schemas/BlobMetadata' - - BlobMetadata: - type: object - required: - - type - - snapshot-id - - sequence-number - - fields - properties: - type: - type: string - snapshot-id: - type: integer - format: int64 - sequence-number: - type: integer - format: int64 - fields: - type: array - items: - type: integer - properties: - type: object - additionalProperties: - type: string - - PartitionStatisticsFile: - type: object - required: - - snapshot-id - - statistics-path - - file-size-in-bytes - properties: - snapshot-id: - type: integer - format: int64 - statistics-path: - type: string - file-size-in-bytes: - type: integer - format: int64 - - BooleanTypeValue: - type: boolean - example: true - - IntegerTypeValue: - type: integer - example: 42 - - LongTypeValue: - type: integer - format: int64 - example: 9223372036854775807 - - FloatTypeValue: - type: number - format: float - example: 3.14 - - DoubleTypeValue: - type: number - format: double - example: 123.456 - - DecimalTypeValue: - type: string - description: - "Decimal type values are serialized as strings. Decimals with a positive scale serialize as numeric plain - text, while decimals with a negative scale use scientific notation and the exponent will be equal to the - negated scale. For instance, a decimal with a positive scale is '123.4500', with zero scale is '2', - and with a negative scale is '2E+20'" - example: "123.4500" - - StringTypeValue: - type: string - example: "hello" - - UUIDTypeValue: - type: string - format: uuid - pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' - maxLength: 36 - minLength: 36 - description: - "UUID type values are serialized as a 36-character lowercase string in standard UUID format as specified - by RFC-4122" - example: "eb26bdb1-a1d8-4aa6-990e-da940875492c" - - DateTypeValue: - type: string - format: date - description: - "Date type values follow the 'YYYY-MM-DD' ISO-8601 standard date format" - example: "2007-12-03" - - TimeTypeValue: - type: string - description: - "Time type values follow the 'HH:MM:SS.ssssss' ISO-8601 format with microsecond precision" - example: "22:31:08.123456" - - TimestampTypeValue: - type: string - description: - "Timestamp type values follow the 'YYYY-MM-DDTHH:MM:SS.ssssss' ISO-8601 format with microsecond precision" - example: "2007-12-03T10:15:30.123456" - - TimestampTzTypeValue: - type: string - description: - "TimestampTz type values follow the 'YYYY-MM-DDTHH:MM:SS.ssssss+00:00' ISO-8601 format with microsecond precision, - and a timezone offset (+00:00 for UTC)" - example: "2007-12-03T10:15:30.123456+00:00" - - TimestampNanoTypeValue: - type: string - description: - "Timestamp_ns type values follow the 'YYYY-MM-DDTHH:MM:SS.sssssssss' ISO-8601 format with nanosecond precision" - example: "2007-12-03T10:15:30.123456789" - - TimestampTzNanoTypeValue: - type: string - description: - "Timestamp_ns type values follow the 'YYYY-MM-DDTHH:MM:SS.sssssssss+00:00' ISO-8601 format with nanosecond - precision, and a timezone offset (+00:00 for UTC)" - example: "2007-12-03T10:15:30.123456789+00:00" - - FixedTypeValue: - type: string - description: - "Fixed length type values are stored and serialized as an uppercase hexadecimal string - preserving the fixed length" - example: "78797A" - - BinaryTypeValue: - type: string - description: - "Binary type values are stored and serialized as an uppercase hexadecimal string" - example: "78797A" - - CountMap: - type: object - properties: - keys: - type: array - items: - $ref: '#/components/schemas/IntegerTypeValue' - description: "List of integer column ids for each corresponding value" - values: - type: array - items: - $ref: '#/components/schemas/LongTypeValue' - description: "List of Long values, matched to 'keys' by index" - example: - { - "keys": [ 1, 2 ], - "values": [ 100, 200 ] - } - - ValueMap: - type: object - properties: - keys: - type: array - items: - $ref: '#/components/schemas/IntegerTypeValue' - description: "List of integer column ids for each corresponding value" - values: - type: array - items: - $ref: '#/components/schemas/PrimitiveTypeValue' - description: "List of primitive type values, matched to 'keys' by index" - example: - { - "keys": [ 1, 2 ], - "values": [ 100, "test" ] - } - - PrimitiveTypeValue: - oneOf: - - $ref: '#/components/schemas/BooleanTypeValue' - - $ref: '#/components/schemas/IntegerTypeValue' - - $ref: '#/components/schemas/LongTypeValue' - - $ref: '#/components/schemas/FloatTypeValue' - - $ref: '#/components/schemas/DoubleTypeValue' - - $ref: '#/components/schemas/DecimalTypeValue' - - $ref: '#/components/schemas/StringTypeValue' - - $ref: '#/components/schemas/UUIDTypeValue' - - $ref: '#/components/schemas/DateTypeValue' - - $ref: '#/components/schemas/TimeTypeValue' - - $ref: '#/components/schemas/TimestampTypeValue' - - $ref: '#/components/schemas/TimestampTzTypeValue' - - $ref: '#/components/schemas/TimestampNanoTypeValue' - - $ref: '#/components/schemas/TimestampTzNanoTypeValue' - - $ref: '#/components/schemas/FixedTypeValue' - - $ref: '#/components/schemas/BinaryTypeValue' - - FileFormat: - type: string - enum: - - avro - - orc - - parquet - - puffin - - ContentFile: - discriminator: - propertyName: content - mapping: - data: '#/components/schemas/DataFile' - position-deletes: '#/components/schemas/PositionDeleteFile' - equality-deletes: '#/components/schemas/EqualityDeleteFile' - type: object - required: - - spec-id - - partition - - content - - file-path - - file-format - - file-size-in-bytes - - record-count - properties: - content: - type: string - file-path: - type: string - file-format: - $ref: '#/components/schemas/FileFormat' - spec-id: - type: integer - partition: - type: array - items: - $ref: '#/components/schemas/PrimitiveTypeValue' - description: - A list of partition field values ordered based on the fields of - the partition spec specified by the `spec-id` - example: [1, "bar"] - file-size-in-bytes: - type: integer - format: int64 - description: "Total file size in bytes" - record-count: - type: integer - format: int64 - description: "Number of records in the file" - key-metadata: - allOf: - - $ref: '#/components/schemas/BinaryTypeValue' - description: "Encryption key metadata blob" - split-offsets: - type: array - items: - type: integer - format: int64 - description: "List of splittable offsets" - sort-order-id: - type: integer - - DataFile: - allOf: - - $ref: '#/components/schemas/ContentFile' - type: object - required: - - content - properties: - content: - type: string - const: "data" - first-row-id: - type: integer - format: int64 - description: "The first row ID assigned to the first row in the data file" - column-sizes: - allOf: - - $ref: '#/components/schemas/CountMap' - description: "Map of column id to total count, including null and NaN" - value-counts: - allOf: - - $ref: '#/components/schemas/CountMap' - description: "Map of column id to null value count" - null-value-counts: - allOf: - - $ref: '#/components/schemas/CountMap' - description: "Map of column id to null value count" - nan-value-counts: - allOf: - - $ref: '#/components/schemas/CountMap' - description: "Map of column id to number of NaN values in the column" - lower-bounds: - allOf: - - $ref: '#/components/schemas/ValueMap' - description: "Map of column id to lower bound primitive type values" - upper-bounds: - allOf: - - $ref: '#/components/schemas/ValueMap' - description: "Map of column id to upper bound primitive type values" - - DeleteFile: - discriminator: - propertyName: content - mapping: - position-deletes: '#/components/schemas/PositionDeleteFile' - equality-deletes: '#/components/schemas/EqualityDeleteFile' - oneOf: - - $ref: '#/components/schemas/PositionDeleteFile' - - $ref: '#/components/schemas/EqualityDeleteFile' - - PositionDeleteFile: - allOf: - - $ref: '#/components/schemas/ContentFile' - required: - - content - properties: - content: - type: string - const: "position-deletes" - content-offset: - type: integer - format: int64 - description: Offset within the delete file of delete content - content-size-in-bytes: - type: integer - format: int64 - description: Length, in bytes, of the delete content; required if content-offset is present - - EqualityDeleteFile: - allOf: - - $ref: '#/components/schemas/ContentFile' - required: - - content - properties: - content: - type: string - const: "equality-deletes" - equality-ids: - type: array - items: - type: integer - description: "List of equality field IDs" - - PlanTableScanRequest: - type: object - properties: - snapshot-id: - description: - Identifier for the snapshot to scan in a point-in-time scan - type: integer - format: int64 - select: - description: List of selected schema fields - type: array - items: - $ref: '#/components/schemas/FieldName' - filter: - description: - Expression used to filter the table data - $ref: '#/components/schemas/Expression' - min-rows-requested: - description: - The minimum number of rows requested for the scan. This is used as a hint - to the server to not have to return more rows than necessary. It is not required - for the server to return that many rows since the scan may not produce that - many rows. The server can also return more rows than requested. - type: integer - format: int64 - case-sensitive: - description: Enables case sensitive field matching for filter and select - type: boolean - default: true - use-snapshot-schema: - description: - Whether to use the schema at the time the snapshot was written. - - When time travelling, the snapshot schema should be used (true). - When scanning a branch, the table schema should be used (false). - type: boolean - default: false - start-snapshot-id: - description: Starting snapshot ID for an incremental scan (exclusive) - type: integer - format: int64 - end-snapshot-id: - description: - Ending snapshot ID for an incremental scan (inclusive). - - Required when start-snapshot-id is specified. - type: integer - format: int64 - stats-fields: - description: - List of fields for which the service should send column stats. - type: array - items: - $ref: '#/components/schemas/FieldName' - - FieldName: - description: - A full field name (including parent field names), such as those passed - in APIs like Java `Schema#findField(String name)`. - - The nested field name follows these rules - - Nested struct fields are named by concatenating field names at each - struct level using dot (`.`) delimiter, e.g. - employer.contact_info.address.zip_code - - Nested fields in a map key are named using the keyword `key`, e.g. - employee_address_map.key.first_name - - Nested fields in a map value are named using the keyword `value`, - e.g. employee_address_map.value.zip_code - - Nested fields in a list are named using the keyword `element`, e.g. - employees.element.first_name - type: string - - FetchScanTasksRequest: - type: object - required: - - plan-task - properties: - plan-task: - $ref: '#/components/schemas/PlanTask' - - PlanTask: - description: - An opaque string provided by the REST server that represents a - unit of work to produce file scan tasks for scan planning. This allows - clients to fetch tasks across multiple requests to accommodate large result sets. - type: string - - FileScanTask: - type: object - required: - - data-file - properties: - data-file: - $ref: '#/components/schemas/DataFile' - delete-file-references: - description: A list of indices in the delete files array (0-based) - type: array - items: - type: integer - residual-filter: - description: - An optional filter to be applied to rows in this file scan task. - - If the residual is not present, the client must produce the - residual or use the original filter. - allOf: - - $ref: '#/components/schemas/Expression' - - MultiValuedMap: - description: A map of string keys where each key can map to multiple string values. - type: object - additionalProperties: - type: array - items: - type: string - - RemoteSignRequest: - description: The request to be signed remotely. - type: object - required: - - region - - uri - - method - - headers - properties: - region: - type: string - uri: - type: string - method: - type: string - enum: ["PUT", "GET", "HEAD", "POST", "DELETE", "PATCH", "OPTIONS"] - headers: - $ref: '#/components/schemas/MultiValuedMap' - properties: - type: object - additionalProperties: - type: string - body: - type: string - description: Optional body of the request to send to the signing API. This should only be populated - for requests where the body of the message contains content which must be validated before a request is - signed, such as the S3 DeleteObjects call. - provider: - type: string - description: The storage provider for which the request is to be signed. The provider should correspond to - the scheme used for a storage native URI. For example `s3` for AWS S3 paths. For backwards compatibility, - if this is not specified, the provider is assumed to be `s3`. - - RemoteSignResult: - description: The result of a remote request signing operation. - type: object - required: - - uri - - headers - properties: - uri: - type: string - headers: - $ref: '#/components/schemas/MultiValuedMap' - - ############################# - # Reusable Response Objects # - ############################# - responses: - - OAuthTokenResponse: - description: OAuth2 token response for client credentials or token exchange - content: - application/json: - schema: - $ref: '#/components/schemas/OAuthTokenResponse' - - OAuthErrorResponse: - description: OAuth2 error response - content: - application/json: - schema: - $ref: '#/components/schemas/OAuthError' - - BadRequestErrorResponse: - description: - Indicates a bad request error. It could be caused by an unexpected request - body format or other forms of request validation failure, such as invalid json. - Usually serves application/json content, although in some cases simple text/plain content might - be returned by the server's middleware. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Malformed request", - "type": "BadRequestException", - "code": 400 - } - } - - # Note that this is a representative example response for use as a shorthand in the spec. - # The fields `message` and `type` as indicated here are not presently prescriptive. - UnauthorizedResponse: - description: - Unauthorized. The REST Catalog SHOULD respond with the 401 UnauthorizedResponse when - the access token provided is expired, revoked, malformed, or invalid for other reasons. - The client MAY request a new access token and retry the request. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Not authorized to make this request", - "type": "NotAuthorizedException", - "code": 401 - } - } - - # Note that this is a representative example response for use as a shorthand in the spec. - # The fields `message` and `type` as indicated here are not presently prescriptive. - ForbiddenResponse: - description: Forbidden. Authenticated user does not have the necessary permissions. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Not authorized to make this request", - "type": "NotAuthorizedException", - "code": 403 - } - } - - # Note that this is a representative example response for use as a shorthand in the spec. - # The fields `message` and `type` as indicated here are not presently prescriptive. - UnsupportedOperationResponse: - description: Not Acceptable / Unsupported Operation. The server does not support this operation. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorModel' - example: { - "error": { - "message": "The server does not support this operation", - "type": "UnsupportedOperationException", - "code": 406 - } - } - - IcebergErrorResponse: - description: JSON wrapper for all error responses (non-2xx) - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "The server does not support this operation", - "type": "UnsupportedOperationException", - "code": 406 - } } - - CreateNamespaceResponse: - description: - Represents a successful call to create a namespace. - Returns the namespace created, as well as any properties that were stored for the namespace, - including those the server might have added. Implementations are not required to support namespace - properties. - content: - application/json: - schema: - $ref: '#/components/schemas/CreateNamespaceResponse' - example: { - "namespace": ["accounting", "tax"], - "properties": { "owner": "Ralph", "created_at": "1452120468" } - } - - GetNamespaceResponse: - description: - Returns a namespace, as well as any properties stored on the namespace if namespace properties - are supported by the server. - content: - application/json: - schema: - $ref: '#/components/schemas/GetNamespaceResponse' - - ListTablesResponse: - description: A list of table identifiers - content: - application/json: - schema: - $ref: '#/components/schemas/ListTablesResponse' - examples: - ListTablesResponseNonEmpty: - $ref: '#/components/examples/ListTablesNonEmptyExample' - ListTablesResponseEmpty: - $ref: '#/components/examples/ListTablesEmptyExample' - - ListNamespacesResponse: - description: A list of namespaces - content: - application/json: - schema: - $ref: '#/components/schemas/ListNamespacesResponse' - examples: - NonEmptyResponse: - $ref: '#/components/examples/ListNamespacesNonEmptyExample' - EmptyResponse: - $ref: '#/components/examples/ListNamespacesEmptyExample' - - AuthenticationTimeoutResponse: - description: - This is an optional status response type that the REST Catalog can issue when the - token has expired. The client MAY request a new access token and retry the request. - 401 UnauthorizedResponse SHOULD be preferred over this response type on token expiry. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Credentials have timed out", - "type": "AuthenticationTimeoutException", - "code": 419 - } - } - - ServiceUnavailableResponse: - description: - The service is not ready to handle the request, request could have been partially processed. - - The service may additionally send a Retry-After header to indicate when to retry, a non idempotent - request should only be retried by the client when the Retry-After header is present. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Slow down", - "type": "SlowDownException", - "code": 503 - } - } - - ServerErrorResponse: - description: - A server-side problem that might not be addressable from the client - side. Used for server 5xx errors without more specific documentation in - individual routes. - content: - application/json: - schema: - $ref: '#/components/schemas/IcebergErrorResponse' - example: { - "error": { - "message": "Internal Server Error", - "type": "InternalServerError", - "code": 500 - } - } - - UpdateNamespacePropertiesResponse: - description: JSON data response for a synchronous update properties request. - content: - application/json: - schema: - $ref: '#/components/schemas/UpdateNamespacePropertiesResponse' - example: { - "updated": [ "owner" ], - "removed": [ "foo" ], - "missing": [ "bar" ] - } - - CreateTableResponse: - description: Table metadata result after creating a table - content: - application/json: - schema: - $ref: '#/components/schemas/LoadTableResult' - headers: - etag: - $ref: '#/components/parameters/etag' - - PlanTableScanResponse: - description: Result of submitting a table scan to plan - content: - application/json: - schema: - $ref: '#/components/schemas/PlanTableScanResult' - - FetchPlanningResultResponse: - description: Result of fetching a submitted scan planning operation - content: - application/json: - schema: - $ref: '#/components/schemas/FetchPlanningResult' - - FetchScanTasksResponse: - description: Result of retrieving additional plan tasks and file scan tasks. - content: - application/json: - schema: - $ref: '#/components/schemas/FetchScanTasksResult' - - LoadTableResponse: - description: Table metadata result when loading a table - content: - application/json: - schema: - $ref: '#/components/schemas/LoadTableResult' - headers: - etag: - $ref: '#/components/parameters/etag' - - LoadViewResponse: - description: View metadata result when loading a view - content: - application/json: - schema: - $ref: '#/components/schemas/LoadViewResult' - - CommitTableResponse: - description: - Response used when a table is successfully updated. - - The table metadata JSON is returned in the metadata field. The corresponding file location of table metadata must be returned in the metadata-location field. Clients can check whether metadata has changed by comparing metadata locations. - content: - application/json: - schema: - $ref: '#/components/schemas/CommitTableResponse' - headers: - etag: - $ref: '#/components/parameters/etag' - - LoadCredentialsResponse: - description: Table credentials result when loading credentials for a table - content: - application/json: - schema: - $ref: '#/components/schemas/LoadCredentialsResponse' - - RemoteSignResponse: - description: The response containing signed & unsigned headers. The server will also send - a Cache-Control header, indicating whether the response can be cached (Cache-Control = ["private"]) - or not (Cache-Control = ["no-cache"]). - content: - application/json: - schema: - $ref: '#/components/schemas/RemoteSignResult' - - ####################################### - # Common examples of different values # - ####################################### - examples: - - ListTablesEmptyExample: - summary: An empty list for a namespace with no tables - value: { - "identifiers": [ ] - } - - ListNamespacesEmptyExample: - summary: An empty list of namespaces - value: { - "namespaces": [ ] - } - - ListNamespacesNonEmptyExample: - summary: A non-empty list of namespaces - value: { - "namespaces": [ - ["accounting", "tax"], - ["accounting", "credits"] - ] - } - - ListTablesNonEmptyExample: - summary: A non-empty list of table identifiers - value: { - "identifiers": [ - { "namespace": ["accounting", "tax"], "name": "paid" }, - { "namespace": ["accounting", "tax"], "name": "owed" } - ] - } - - MultipartNamespaceAsPathVariable: - summary: A multi-part namespace, as represented in a path parameter - value: "accounting%1Ftax" - - NamespaceAsPathVariable: - summary: A single part namespace, as represented in a path parameter - value: "accounting" - - NamespaceAlreadyExistsError: - summary: The requested namespace already exists - value: { - "error": { - "message": "The given namespace already exists", - "type": "AlreadyExistsException", - "code": 409 - } - } - - NoSuchPlanIdError: - summary: The plan id does not exist - value: { - "error": { - "message": "The plan id does not exist", - "type": "NoSuchPlanIdException", - "code": 404 - } - } - - NoSuchPlanTaskError: - summary: The plan task does not exist - value: { - "error": { - "message": "The plan task does not exist", - "type": "NoSuchPlanTaskException", - "code": 404 - } - } - - NoSuchTableError: - summary: The requested table does not exist - value: { - "error": { - "message": "The given table does not exist", - "type": "NoSuchTableException", - "code": 404 - } - } - - NoSuchViewError: - summary: The requested view does not exist - value: { - "error": { - "message": "The given view does not exist", - "type": "NoSuchViewException", - "code": 404 - } - } - - NoSuchWarehouseError: - summary: The requested warehouse does not exist - value: { - "error": { - "message": "The given warehouse does not exist", - "type": "NoSuchWarehouseException", - "code": 404 - } - } - - NoSuchNamespaceError: - summary: The requested namespace does not exist - value: { - "error": { - "message": "The given namespace does not exist", - "type": "NoSuchNamespaceException", - "code": 404 - } - } - - NamespaceNotEmptyError: - summary: The requested namespace is not empty - value: { - "error": { - "message": "The given namespace is not empty", - "type": "NamespaceNotEmptyException", - "code": 409 - } - } - - RenameTableSameNamespace: - summary: Rename a table in the same namespace - value: { - "source": { "namespace": ["accounting", "tax"], "name": "paid" }, - "destination": { "namespace": ["accounting", "tax"], "name": "owed" } - } - - RenameViewSameNamespace: - summary: Rename a view in the same namespace - value: { - "source": { "namespace": [ "accounting", "tax" ], "name": "paid-view" }, - "destination": { "namespace": [ "accounting", "tax" ], "name": "owed-view" } - } - - TableAlreadyExistsError: - summary: The requested table identifier already exists - value: { - "error": { - "message": "The requested table identifier already exists", - "type": "AlreadyExistsException", - "code": 409 - } - } - - ViewAlreadyExistsError: - summary: The requested view identifier already exists - value: { - "error": { - "message": "The requested view identifier already exists", - "type": "AlreadyExistsException", - "code": 409 - } - } - - # This is an example response and is not meant to be prescriptive regarding the message or type. - UnprocessableEntityDuplicateKey: - summary: - The request body either has the same key multiple times in what should be a map with unique keys - or the request body has keys in two or more fields which should be disjoint sets. - value: { - "error": { - "message": "The request cannot be processed as there is a key present multiple times", - "type": "UnprocessableEntityException", - "code": 422 - } - } - - UpdateAndRemoveNamespacePropertiesRequest: - summary: An update namespace properties request with both properties to remove and properties to upsert. - value: { - "removals": [ "foo", "bar" ], - "updates": { "owner": "Raoul" } - } - - securitySchemes: - OAuth2: - type: oauth2 - description: - This scheme is used for OAuth2 authorization. - - - For unauthorized requests, services should return an appropriate 401 or - 403 response. Implementations must not return altered success (200) - responses when a request is unauthenticated or unauthorized. - - If a separate authorization server is used, substitute the tokenUrl with - the full token path of the external authorization server, and use the - resulting token to access the resources defined in the spec. - flows: - clientCredentials: - tokenUrl: /v1/oauth/tokens - scopes: - catalog: Allows interacting with the Config and Catalog APIs - BearerAuth: - type: http - scheme: bearer diff --git a/doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md b/doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md deleted file mode 100644 index 94651da0f..000000000 --- a/doc/design/access-server/iceberge/iceberg-table-spec-1.11.0.md +++ /dev/null @@ -1,1858 +0,0 @@ ---- -title: "Spec" ---- - - -# Iceberg Table Spec - -This is a specification for the Iceberg table format that is designed to manage a large, slow-changing collection of files in a distributed file system or key-value store as a table. - -## Format Versioning - -Versions 1, 2 and 3 of the Iceberg spec are complete and adopted by the community. - -**Version 4 is under active development and has not been formally adopted.** - -The format version number is incremented when new features are added that will break forward-compatibility---that is, when older readers would not read newer table features correctly. Tables may continue to be written with an older version of the spec to ensure compatibility by not using features that are not yet implemented by processing engines. - -### Version 1: Analytic Data Tables - -Version 1 of the Iceberg spec defines how to manage large analytic tables using immutable file formats: Parquet, Avro, and ORC. - -All version 1 data and metadata files are valid after upgrading a table to version 2. [Appendix E](#version-2) documents how to default version 2 fields when reading version 1 metadata. - -### Version 2: Row-level Deletes - -Version 2 of the Iceberg spec adds row-level updates and deletes for analytic tables with immutable files. - -The primary change in version 2 adds delete files to encode rows that are deleted in existing data files. This version can be used to delete or replace individual rows in immutable data files without rewriting the files. - -In addition to row-level deletes, version 2 makes some requirements stricter for writers. The full set of changes are listed in [Appendix E](#version-2). - -### Version 3: Extended Types and Capabilities - -Version 3 of the Iceberg spec extends data types and existing metadata structures to add new capabilities: - -* New data types: nanosecond timestamp(tz), unknown, variant, geometry, geography -* Default value support for columns -* Multi-argument transforms for partitioning and sorting -* Row Lineage tracking -* Binary deletion vectors -* Table encryption keys - -The full set of changes are listed in [Appendix E](#version-3). - -## Goals - -* **Serializable isolation** -- Reads will be isolated from concurrent writes and always use a committed snapshot of a table’s data. Writes will support removing and adding files in a single operation and are never partially visible. Readers will not acquire locks. -* **Speed** -- Operations will use O(1) remote calls to plan the files for a scan and not O(n) where n grows with the size of the table, like the number of partitions or files. -* **Scale** -- Job planning will be handled primarily by clients and not bottleneck on a central metadata store. Metadata will include information needed for cost-based optimization. -* **Evolution** -- Tables will support full schema and partition spec evolution. Schema evolution supports safe column add, drop, reorder and rename, including in nested structures. -* **Dependable types** -- Tables will provide well-defined and dependable support for a core set of types. -* **Storage separation** -- Partitioning will be table configuration. Reads will be planned using predicates on data values, not partition values. Tables will support evolving partition schemes. -* **Formats** -- Underlying data file formats will support identical schema evolution rules and types. Both read-optimized and write-optimized formats will be available. - -## Overview - -![Iceberg snapshot structure](https://iceberg.apache.org/assets/images/iceberg-metadata.png) - -This table format tracks individual data files in a table instead of directories. This allows writers to create data files in-place and only adds files to the table in an explicit commit. - -Table state is maintained in metadata files. All changes to table state create a new metadata file and replace the old metadata with an atomic swap. The table metadata file tracks the table schema, partitioning config, custom properties, and snapshots of the table contents. A snapshot represents the state of a table at some time and is used to access the complete set of data files in the table. - -Data files in snapshots are tracked by one or more manifest files that contain a row for each data file in the table, the file's partition data, and its metrics. The data in a snapshot is the union of all files in its manifests. Manifest files are reused across snapshots to avoid rewriting metadata that is slow-changing. Manifests can track data files with any subset of a table and are not associated with partitions. - -The manifests that make up a snapshot are stored in a manifest list file. Each manifest list stores metadata about manifests, including partition stats and data file counts. These stats are used to avoid reading manifests that are not required for an operation. - -### Optimistic Concurrency - -An atomic swap of one table metadata file for another provides the basis for serializable isolation. Readers use the snapshot that was current when they load the table metadata and are not affected by changes until they refresh and pick up a new metadata location. - -Writers create table metadata files optimistically, assuming that the current version will not be changed before the writer's commit. Once a writer has created an update, it commits by swapping the table’s metadata file pointer from the base version to the new version. - -If the snapshot on which an update is based is no longer current, the writer must retry the update based on the new current version. Some operations support retry by re-applying metadata changes and committing, under well-defined conditions. For example, a change that rewrites files can be applied to a new table snapshot if all of the rewritten files are still in the table. - -The conditions required by a write to successfully commit determines the isolation level. Writers can select what to validate and can make different isolation guarantees. - -### Sequence Numbers - -The relative age of data and delete files relies on a sequence number that is assigned to every successful commit. When a snapshot is created for a commit, it is optimistically assigned the next sequence number, and it is written into the snapshot's metadata. If the commit fails and must be retried, the sequence number is reassigned and written into new snapshot metadata. - -All manifests, data files, and delete files created for a snapshot inherit the snapshot's sequence number. Manifest file metadata in the manifest list stores a manifest's sequence number. New data and metadata file entries are written with `null` in place of a sequence number, which is replaced with the manifest's sequence number at read time. When a data or delete file is written to a new manifest (as "existing"), the inherited sequence number is written to ensure it does not change after it is first inherited. - -Inheriting the sequence number from manifest metadata allows writing a new manifest once and reusing it in commit retries. To change a sequence number for a retry, only the manifest list must be rewritten -- which would be rewritten anyway with the latest set of manifests. - -### Row-level Deletes - -Row-level deletes are stored in delete files. - -There are two types of row-level deletes: - -* **Position deletes** -- Mark a row deleted by data file path and the row position in the data file. Position deletes are encoded in a [_position delete file_](#position-delete-files) (V2) or [_deletion vector_](#deletion-vectors) (V3 or above). - -* **Equality deletes** -- Mark a row deleted by one or more column values, like id = 5. Equality deletes are encoded in [_equality delete file_](#equality-delete-files). - -Like data files, delete files are tracked by partition. In general, a delete file must be applied to older data files with the same partition; see [Scan Planning](#scan-planning) for details. Column metrics can be used to determine whether a delete file's rows overlap the contents of a data file or a scan range. - -### File System Operations - -Iceberg only requires that file systems support the following operations: - -* **In-place write** -- Files are not moved or altered once they are written. -* **Seekable reads** -- Data file formats require seek support. -* **Deletes** -- Tables delete files that are no longer used. - -These requirements are compatible with object stores, like S3. - -Tables do not require random-access writes. Once written, data and metadata files are immutable until they are deleted. - -Tables do not require rename, except for tables that use atomic rename to implement the commit operation for new metadata files. - -## Specification - -#### Terms - -* **Schema** -- Names and types of fields in a table. -* **Partition spec** -- A definition of how partition values are derived from data fields. -* **Snapshot** -- The state of a table at some point in time, including the set of all data files. -* **Manifest list** -- A file that lists manifest files; one per snapshot. -* **Manifest** -- A file that lists data or delete files; a subset of a snapshot. -* **Data file** -- A file that contains rows of a table. -* **Delete file** -- A file that encodes rows of a table that are deleted by position or data values. - -#### Writer requirements - -Some tables in this spec have columns that specify requirements for tables by version. These requirements are intended for writers when adding metadata files (including manifests files and manifest lists) to a table with the given version. - -| Requirement | Write behavior | -|-------------|----------------| -| (blank) | The field should be omitted | -| _optional_ | The field can be written or omitted | -| _required_ | The field must be written | - -Readers should be more permissive because v1 metadata files are allowed in v2 tables (or later) so that tables can be upgraded to without rewriting the metadata tree. For manifest list and manifest files, this table shows the expected read behavior for later versions: - -| v1 | v2 | v2+ read behavior | -|------------|------------|-------------------| -| | _optional_ | Read the field as _optional_ | -| | _required_ | Read the field as _optional_; it may be missing in v1 files | -| _optional_ | | Ignore the field | -| _optional_ | _optional_ | Read the field as _optional_ | -| _optional_ | _required_ | Read the field as _optional_; it may be missing in v1 files | -| _required_ | | Ignore the field | -| _required_ | _optional_ | Read the field as _optional_ | -| _required_ | _required_ | Fill in a default or throw an exception if the field is missing | - -If a later version is not shown, the requirement for a version is not changed from the most recent version shown. For example, v3 uses the same requirements as v2 if a table shows only v1 and v2 requirements. - -Readers may be more strict for metadata JSON files because the JSON files are not reused and will always match the table version. Required fields that were not present in or were optional in prior versions may be handled as required fields. For example, a v2 table that is missing `last-sequence-number` can throw an exception. - -#### Writing data files - -All columns must be written to data files even if they introduce redundancy with metadata stored in manifest files (e.g. columns with identity partition transforms). Writing all columns provides a backup in case of corruption or bugs in the metadata layer. - -Writers are not allowed to commit files with a partition spec that contains a field with an unknown transform. - -### Schemas and Data Types - -A table's **schema** is a list of named columns. All data types are either primitives or nested types, which are maps, lists, or structs. A table schema is also a struct type. - -For the representations of these types in Avro, ORC, and Parquet file formats, see Appendix A. - -#### Nested Types - -A **`struct`** is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema. Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type. Fields may have an optional comment or doc string. Fields can have [default values](#default-values). - -A **`list`** is a collection of values with some element type. The element field has an integer id that is unique in the table schema. Elements can be either optional or required. Element types may be any type. - -A **`map`** is a collection of key-value pairs with a key type and a value type. Both the key field and value field each have an integer id that is unique in the table schema. Map keys are required and map values can be either optional or required. Both map keys and map values may be any type, including nested types. - -#### Semi-structured Types - -A **`variant`** is a value that stores semi-structured data. The structure and data types in a variant are not necessarily consistent across rows in a table or data file. The variant type and binary encoding are defined in the [Parquet project](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md), with support currently available for V1. Support for Variant is added in Iceberg v3. - -Variants are similar to JSON with a wider set of primitive values including date, timestamp, timestamptz, binary, and decimals. - -Variant values may contain nested types: - -1. An array is an ordered collection of variant values. -2. An object is a collection of fields that are a string key and a variant value. - -As a semi-structured type, there are important differences between variant and Iceberg's other types: - -1. Variant arrays are similar to lists, but may contain any variant value rather than a fixed element type. -2. Variant objects are similar to structs, but may contain variable fields identified by name and field values may be any variant value rather than a fixed field type. - -#### Primitive Types - -Supported primitive types are defined in the table below. Primitive types added after v1 have an "added by" version that is the first spec version in which the type is allowed. For example, nanosecond-precision timestamps are part of the v3 spec; using v3 types in v1 or v2 tables can break forward compatibility. - -| Added by version | Primitive type | Description | Requirements | -|------------------|--------------------|--------------------------------------------------------------------------|--------------------------------------------------| -| [v3](#version-3) | **`unknown`** | Default / null column type used when a more specific type is not known | Must be optional with `null` defaults; not stored in data files | -| | **`boolean`** | True or false | | -| | **`int`** | 32-bit signed integers | Can promote to `long` | -| | **`long`** | 64-bit signed integers | | -| | **`float`** | [32-bit IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating point | Can promote to double | -| | **`double`** | [64-bit IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating point | | -| | **`decimal(P,S)`** | Fixed-point decimal; precision P, scale S | Scale is fixed, precision must be 38 or less | -| | **`date`** | Calendar date without timezone or time | | -| | **`time`** | Time of day, microsecond precision, without date, timezone | | -| | **`timestamp`** | Timestamp, microsecond precision, without timezone | [1] | -| | **`timestamptz`** | Timestamp, microsecond precision, with timezone | [2] | -| [v3](#version-3) | **`timestamp_ns`** | Timestamp, nanosecond precision, without timezone | [1] | -| [v3](#version-3) | **`timestamptz_ns`** | Timestamp, nanosecond precision, with timezone | [2] | -| | **`string`** | Arbitrary-length character sequences | Encoded with UTF-8 [3] | -| | **`uuid`** | Universally unique identifiers | Should use 16-byte fixed | -| | **`fixed(L)`** | Fixed-length byte array of length L | | -| | **`binary`** | Arbitrary-length byte array | | -| [v3](#version-3) | **`geometry(C)`** | Geospatial features from [OGC – Simple feature access][1001]. Edge-interpolation is always linear/planar. See [Appendix G](#appendix-g-geospatial-notes). Parameterized by CRS C. If not specified, C is `OGC:CRS84`. | | -| [v3](#version-3) | **`geography(C, A)`** | Geospatial features from [OGC – Simple feature access][1001]. See [Appendix G](#appendix-g-geospatial-notes). Parameterized by CRS C and edge-interpolation algorithm A. If not specified, C is `OGC:CRS84` and A is `spherical`. | - -Notes: - -1. Timestamp values _without time zone_ represent a date and time of day regardless of zone: the time value is independent of zone adjustments (`2017-11-16 17:10:34` is always retrieved as `2017-11-16 17:10:34`). -2. Timestamp values _with time zone_ represent a point in time: values are stored as UTC and do not retain a source time zone (`2017-11-16 17:10:34 PST` is stored/retrieved as `2017-11-17 01:10:34 UTC` and these values are considered identical). -3. Character strings must be stored as UTF-8 encoded byte arrays. - -For details on how to serialize a schema to JSON, see Appendix C. - -[1001]: "OGC Simple feature access" - -##### CRS - -For `geometry` and `geography` types, the parameter C refers to the CRS (coordinate reference system), a mapping of how coordinates refer to locations on Earth. - -For `geometry` type, the CRS does not affect geometric calculations, which are always Cartesian. - -The default CRS value `OGC:CRS84` means that the objects must be stored in longitude, latitude based on the WGS84 datum. - -Custom CRS values can be specified by a string of the format `type:identifier`, where `type` is one of the following values: - -* `srid`: [Spatial reference identifier](https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier), `identifier` is the SRID itself. -* `projjson`: [PROJJSON](https://proj.org/en/stable/specifications/projjson.html), `identifier` is the name of a table property where the projjson string is stored. - -For `geography` types, the custom CRS must be geographic, with longitudes bound by [-180, 180] and latitudes bound by [-90, 90]. - -##### Edge-Interpolation Algorithm - -For `geography` types, an additional parameter A specifies an algorithm for interpolating edges, and is one of the following values: - -* `spherical`: edges are interpolated as geodesics on a sphere. -* `vincenty`: [https://en.wikipedia.org/wiki/Vincenty%27s_formulae](https://en.wikipedia.org/wiki/Vincenty%27s_formulae) -* `thomas`: Thomas, Paul D. Spheroidal geodesics, reference systems, & local geometry. US Naval Oceanographic Office, 1970. -* `andoyer`: Thomas, Paul D. Mathematical models for navigation systems. US Naval Oceanographic Office, 1965. -* `karney`: [Karney, Charles FF. "Algorithms for geodesics." Journal of Geodesy 87 (2013): 43-55](https://link.springer.com/content/pdf/10.1007/s00190-012-0578-z.pdf), and [GeographicLib](https://geographiclib.sourceforge.io/) - -#### Default values - -Default values can be tracked for struct fields (both nested structs and the top-level schema's struct). There can be two defaults with a field: - -* `initial-default` is used to populate the field's value for all records that were written before the field was added to the schema -* `write-default` is used to populate the field's value for any records written after the field was added to the schema, if the writer does not supply the field's value - -The `initial-default` is set only when a field is added to an existing schema. The `write-default` is initially set to the same value as `initial-default` and can be changed through schema evolution. If either default is not set for an optional field, then the default value is null for compatibility with older spec versions. - -The `initial-default` and `write-default` produce SQL default value behavior, without rewriting data files. SQL default value behavior when a field is added handles all existing rows as though the rows were written with the new field's default value. Default value changes may only affect future records and all known fields are written into data files. Omitting a known field when writing a data file is never allowed. The write default for a field must be written if a field is not supplied to a write. If the write default for a required field is not set, the writer must fail. - -All columns of `unknown`, `variant`, `geometry`, and `geography` types must default to null. Non-null values for `initial-default` or `write-default` are invalid. - -Default values for the fields of a struct are tracked as `initial-default` and `write-default` at the field level. Default values for fields that are nested structs must not contain default values for the struct's fields (sub-fields). Sub-field defaults are tracked in sub-field's metadata. As a result, the default stored for a nested struct may be either null or a non-null struct with no field values. The effective default value is produced by setting each fields' default in a new struct. - -For example, a struct column `point` with fields `x` (default 0) and `y` (default 0) can be defaulted to `{"x": 0, "y": 0}` or `null`. A non-null default is stored by setting `initial-default` or `write-default` to an empty struct (`{}`) that will use field values set from each field's `initial-default` or `write-default`, respectively. - -| `point` default | `point.x` default | `point.y` default | Data value | Result value | -|-----------------|-------------------|-------------------|--------------|--------------| -| `null` | `0` | `0` | (missing) | `null` | -| `null` | `0` | `0` | `{"x": 3}` | `{"x": 3, "y": 0}` | -| `{}` | `0` | `0` | (missing) | `{"x": 0, "y": 0}` | -| `{}` | `0` | `0` | `{"y": -1}` | `{"x": 0, "y": -1}` | - -Default values are attributes of fields in schemas and serialized with fields in the JSON format. See [Appendix C](#appendix-c-json-serialization). - -#### Schema Evolution - -Schemas may be evolved by type promotion or adding, deleting, renaming, or reordering fields in structs (both nested structs and the top-level schema’s struct). - -Evolution applies changes to the table's current schema to produce a new schema that is identified by a unique schema ID, is added to the table's list of schemas, and is set as the table's current schema. - -Valid primitive type promotions are: - -| Primitive type | v1, v2 valid type promotions | v3+ valid type promotions | Requirements | -|------------------|------------------------------|------------------------------|--------------| -| `unknown` | | _any type_ | | -| `int` | `long` | `long` | | -| `date` | | `timestamp`, `timestamp_ns` | Promotion to `timestamptz` or `timestamptz_ns` is **not** allowed; values outside the promoted type's range must result in a runtime failure | -| `float` | `double` | `double` | | -| `decimal(P, S)` | `decimal(P', S)` if `P' > P` | `decimal(P', S)` if `P' > P` | Widen precision only | - -Iceberg's Avro manifest format does not store the type of lower and upper bounds, and type promotion does not rewrite existing bounds. For example, when a `float` is promoted to `double`, existing data file bounds are encoded as 4 little-endian bytes rather than 8 little-endian bytes for `double`. To correctly decode the value, the original type at the time the file was written must be inferred according to the following table: - -| Current type | Length of bounds | Inferred type at write time | -|------------------|------------------|-----------------------------| -| `long` | 4 bytes | `int` | -| `long` | 8 bytes | `long` | -| `double` | 4 bytes | `float` | -| `double` | 8 bytes | `double` | -| `timestamp` | 4 bytes | `date` | -| `timestamp` | 8 bytes | `timestamp` | -| `timestamp_ns` | 4 bytes | `date` | -| `timestamp_ns` | 8 bytes | `timestamp_ns` | -| `decimal(P, S)` | _any_ | `decimal(P', S)`; `P' <= P` | - -Type promotion is not allowed for a field that is referenced by `source-id` or `source-ids` of a partition field if the partition transform would produce a different value after promoting the type. For example, `bucket[N]` produces different hash values for `34` and `"34"` (2017239379 != -427558391) but the same value for `34` and `34L`; when an `int` field is the source for a bucket partition field, it may be promoted to `long` but not to `string`. This may happen for the following type promotion cases: - -* `date` to `timestamp` or `timestamp_ns` - -Any struct, including a top-level schema, can evolve through deleting fields, adding new fields, renaming existing fields, reordering existing fields, or promoting a primitive using the valid type promotions. Adding a new field assigns a new ID for that field and for any nested fields. Renaming an existing field must change the name, but not the field ID. Deleting a field removes it from the current schema. Field deletion cannot be rolled back unless the field was nullable or if the current snapshot has not changed. - -Grouping a subset of a struct’s fields into a nested struct is **not** allowed, nor is moving fields from a nested struct into its immediate parent struct (`struct ↔ struct>`). Evolving primitive types to structs is **not** allowed, nor is evolving a single-field struct to a primitive (`map ↔ map>`). - -Struct evolution requires the following rules for default values: - -* The `initial-default` must be set when a field is added and cannot change -* The `write-default` must be set when a field is added and may change -* When a required field is added, both defaults must be set to a non-null value -* When an optional field is added, the defaults may be null and should be explicitly set -* When a field that is a struct type is added, its default may only be null or a non-null struct with no field values. Default values for fields must be stored in field metadata. -* If a field value is missing from a struct's `initial-default`, the field's `initial-default` must be used for the field -* If a field value is missing from a struct's `write-default`, the field's `write-default` must be used for the field - -##### Column Projection - -Columns in Iceberg data files are selected by field id. The table schema's column names and order may change after a data file is written, and projection must be done using field ids. - -Values for field ids which are not present in a data file must be resolved according the following rules: - -* Return the value from partition metadata if an [Identity Transform](#partition-transforms) exists for the field and the partition value is present in the `partition` struct on `data_file` object in the manifest. This allows for metadata only migrations of Hive tables. -* Use `schema.name-mapping.default` metadata to map field id to columns without field id as described below and use the column if it is present. -* Return the default value if it has a defined `initial-default` (See [Default values](#default-values) section for more details). -* Return `null` in all other cases. - -For example, a file may be written with schema `1: a int, 2: b string, 3: c double` and read using projection schema `3: measurement, 2: name, 4: a`. This must select file columns `c` (renamed to `measurement`), `b` (now called `name`), and a column of `null` values called `a`; in that order. - -Tables may also define a property `schema.name-mapping.default` with a JSON name mapping containing a list of field mapping objects. These mappings provide fallback field ids to be used when a data file does not contain field id information. Each object should contain - -* `names`: A required list of 0 or more names for a field. -* `field-id`: An optional Iceberg field ID used when a field's name is present in `names` -* `fields`: An optional list of field mappings for child field of structs, maps, and lists. - -Field mapping fields are constrained by the following rules: - -* A name may contain `.` but this refers to a literal name, not a nested field. For example, `a.b` refers to a field named `a.b`, not child field `b` of field `a`. -* Each child field should be defined with their own field mapping under `fields`. -* Multiple values for `names` may be mapped to a single field ID to support cases where a field may have different names in different data files. For example, all Avro field aliases should be listed in `names`. -* Fields which exist only in the Iceberg schema and not in imported data files may use an empty `names` list. -* Fields that exist in imported files but not in the Iceberg schema may omit `field-id`. -* List types should contain a mapping in `fields` for `element`. -* Map types should contain mappings in `fields` for `key` and `value`. -* Struct types should contain mappings in `fields` for their child fields. - -For details on serialization, see [Appendix C](#name-mapping-serialization). - -#### Identifier Field IDs - -A schema can optionally track the set of primitive fields that identify rows in a table, using the property `identifier-field-ids` (see JSON encoding in Appendix C). - -Two rows are the "same"---that is, the rows represent the same entity---if the identifier fields are equal. However, uniqueness of rows by this identifier is not guaranteed or required by Iceberg and it is the responsibility of processing engines or data providers to enforce. - -Identifier fields may be nested in structs but cannot be nested within maps or lists. Float, double, and optional fields cannot be used as identifier fields and a nested field cannot be used as an identifier field if it is nested in an optional struct, to avoid null values in identifiers. - -#### Reserved Field IDs - -Iceberg tables must not use field ids greater than 2147483447 (`Integer.MAX_VALUE - 200`). This id range is reserved for metadata columns that can be used in user data schemas, like the `_file` column that holds the file path in which a row was stored. - -The set of metadata columns is: - -| Field id, name | Type | Description | -|----------------------------------|---------------|--------------------------------------------------------------------------------------------------------| -| **`2147483646 _file`** | `string` | Path of the file in which a row is stored | -| **`2147483645 _pos`** | `long` | Ordinal position of a row in the source data file, starting at `0` | -| **`2147483644 _deleted`** | `boolean` | Whether the row has been deleted | -| **`2147483643 _spec_id`** | `int` | Spec ID used to track the file containing a row | -| **`2147483642 _partition`** | `struct` | Partition to which a row belongs | -| **`2147483546 file_path`** | `string` | Path of a file, used in position-based delete files | -| **`2147483545 pos`** | `long` | Ordinal position of a row, used in position-based delete files | -| **`2147483544 row`** | `struct<...>` | Deleted row values, used in position-based delete files | -| **`2147483543 _change_type`** | `string` | The record type in the changelog (INSERT, DELETE, UPDATE_BEFORE, or UPDATE_AFTER) | -| **`2147483542 _change_ordinal`** | `int` | The order of the change | -| **`2147483541 _commit_snapshot_id`** | `long` | The snapshot ID in which the change occurred | -| **`2147483540 _row_id`** | `long` | A unique long assigned for row lineage, see [Row Lineage](#row-lineage) | -| **`2147483539 _last_updated_sequence_number`** | `long` | The sequence number which last updated this row, see [Row Lineage](#row-lineage) | - -#### Row Lineage - -In v3 and later, an Iceberg table must track row lineage fields for all newly created rows. Engines must maintain the `next-row-id` table field and the following row-level fields when writing data files: - -* `_row_id` a unique long identifier for every row within the table. The value is assigned via inheritance when a row is first added to the table. -* `_last_updated_sequence_number` the sequence number of the commit that last updated a row. The value is inherited when a row is first added or modified. - -These fields are assigned and updated by inheritance because the commit sequence number and starting row ID are not assigned until the snapshot is successfully committed. Inheritance is used to allow writing data and manifest files before values are known so that it is not necessary to rewrite data and manifest files when an optimistic commit is retried. - -Row lineage does not track lineage for rows updated via [Equality Deletes](#equality-delete-files), because engines using equality deletes avoid reading existing data before writing changes and can't provide the original row ID for the new rows. These updates are always treated as if the existing row was completely removed and a unique new row was added. - -##### Row lineage assignment - -When a row is added or modified, the `_last_updated_sequence_number` field is set to `null` so that it is inherited when reading. Similarly, the `_row_id` field for an added row is set to `null` and assigned when reading. - -A data file with only new rows for the table may omit the `_last_updated_sequence_number` and `_row_id`. If the columns are missing, readers should treat both columns as if they exist and are set to null for all rows. - -On read, if `_last_updated_sequence_number` is `null` it is assigned the `sequence_number` of the data file's manifest entry. The data sequence number of a data file is documented in [Sequence Number Inheritance](#sequence-number-inheritance). - -When `null`, a row's `_row_id` field is assigned to the `first_row_id` from its containing data file plus the row position in that data file (`_pos`). A data file's `first_row_id` field is assigned using inheritance and is documented in [First Row ID Inheritance](#first-row-id-inheritance). A manifest's `first_row_id` is assigned when writing the manifest list for a snapshot and is documented in [First Row ID Assignment](#first-row-id-assignment). A snapshot's `first-row-id` is set to the table's `next-row-id` and is documented in [Snapshot Row IDs](#snapshot-row-ids). - -When an existing row is moved to a different data file for any reason, writers should write `_row_id` and `_last_updated_sequence_number` according to the following rules: - -1. The row's existing non-null `_row_id` must be copied into the new data file -2. If the write has modified the row, the `_last_updated_sequence_number` field must be set to `null` (so that the modification's sequence number replaces the current value) -3. If the write has not modified the row, the existing non-null `_last_updated_sequence_number` value must be copied to the new data file - -Engines may model operations as deleting/inserting rows or as modifications to rows that preserve row ids. - -##### Row lineage example - -This example demonstrates how `_row_id` and `_last_updated_sequence_number` are assigned for a snapshot. This starts with a table with a `next-row-id` of 1000. - -Writing a new append snapshot creates snapshot metadata with `first-row-id` assigned to the table's `next-row-id`: - -```json -{ - "operation": "append", - "first-row-id": 1000, - ... -} -``` - -The snapshot's manifest list will contain existing manifests, plus new manifests that are each assigned a `first_row_id` based on the `added_rows_count` and `existing_rows_count` of preceding new manifests: - -| `manifest_path` | `added_rows_count` | `existing_rows_count` | `first_row_id` | -|-----------------|--------------------|-----------------------|--------------------| -| ... | ... | ... | ... | -| existing | 75 | 0 | 925 | -| added1 | 100 | 25 | 1000 | -| added2 | 0 | 100 | 1125 | -| added3 | 125 | 25 | 1225 | - -The existing manifests are written with the `first_row_id` assigned when the manifests were added to the table. - -The first added manifest, `added1`, is assigned the same `first_row_id` as the snapshot and each of the remaining added manifests are assigned a `first_row_id` based on the number of rows in preceding manifests that were assigned a `first_row_id`. - -Note that the second file, `added2`, changes the `first_row_id` of the next manifest even though it contains no added data files because any data file without a `first_row_id` could be assigned one, even if it has existing status. This is optional if the writer knows that existing data files in the manifest have assigned `first_row_id` values. - -Within `added1`, the first added manifest, each data file's `first_row_id` follows a similar pattern: - -| `status` | `file_path` | `record_count` | `first_row_id` | -|----------|-------------|----------------|----------------| -| EXISTING | data1 | 25 | 800 | -| ADDED | data2 | 50 | null (1000) | -| ADDED | data3 | 50 | null (1050) | - -The `first_row_id` of the EXISTING file `data1` was already assigned, so the file metadata was copied into manifest `added1`. - -Files `data2` and `data3` are written with `null` for `first_row_id` and are assigned `first_row_id` at read time based on the manifest's `first_row_id` and the `record_count` of previous files without `first_row_id` in this manifest: (1,000 + 0) and (1,000 + 50). - -The snapshot then populates the total number of `added-rows` based on the sum of all added rows in the manifests: 100 (50 + 50) - -When the new snapshot is committed, the table's `next-row-id` must also be updated (even if the new snapshot is not in the main branch). Because 375 rows were in data files in manifests that were assigned a `first_row_id` (`added1` 100+25, `added2` 0+100, `added3` 125+25) the new value is 1,000 + 375 = 1,375. - -##### Row Lineage for Upgraded Tables - -When a table is upgraded to v3, its `next-row-id` is initialized to 0 and existing snapshots are not modified (that is, `first-row-id` remains unset or null). For such snapshots without `first-row-id`, `first_row_id` values for data files and data manifests are null, and values for `_row_id` are read as null for all rows. When `first_row_id` is null, inherited row ID values are also null. - -Snapshots that are created after upgrading to v3 must set the snapshot's `first-row-id` and assign row IDs to existing and added files in the snapshot. When writing the manifest list, all data manifests must be assigned a `first_row_id`, which assigns a `first_row_id` to all data files via inheritance. - -Note that: - -* Snapshots created before upgrading to v3 do not have row IDs. -* After upgrading, new snapshots in different branches will assign disjoint ID ranges to existing data files, based on the table's `next-row-id` when the snapshot is committed. For a data file in multiple branches, a writer may write the `first_row_id` from another branch or may assign a new `first_row_id` to the data file (to avoid large metadata rewrites). -* Existing rows will inherit `_last_updated_sequence_number` from their containing data file. - -### Partitioning - -Data files are stored in manifests with a tuple of partition values that are used in scans to filter out files that cannot contain records that match the scan’s filter predicate. Partition values for a data file must be the same for all records stored in the data file. (Manifests store data files from any partition, as long as the partition spec is the same for the data files.) - -Tables are configured with a **partition spec** that defines how to produce a tuple of partition values from a record. A partition spec has a list of fields that consist of: - -* A **source column id** or a list of **source column ids** from the table’s schema -* A **partition field id** that is used to identify a partition field and is unique within a partition spec. In v2 table metadata, it is unique across all partition specs. -* A **transform** that is applied to the source column(s) to produce a partition value -* A **partition name** - -The source columns, selected by ids, must be a primitive type and cannot be contained in a map or list, but may be nested in a struct. For details on how to serialize a partition spec to JSON, see Appendix C. - -Partition specs capture the transform from table data to partition values. This is used to transform predicates to partition predicates, in addition to transforming data values. Deriving partition predicates from column predicates on the table data is used to separate the logical queries from physical storage: the partitioning can change and the correct partition filters are always derived from column predicates. This simplifies queries because users don’t have to supply both logical predicates and partition predicates. For more information, see Scan Planning below. - -Partition fields that use an unknown transform can be read by ignoring the partition field for the purpose of filtering data files during scan planning. In v1 and v2, readers should ignore fields with unknown transforms while reading; this behavior is required in v3. Writers are not allowed to commit data using a partition spec that contains a field with an unknown transform. - -Two partition specs are considered equivalent with each other if they have the same number of fields and for each corresponding field, the fields have the same source column IDs, transform definition and partition name. Writers must not create a new partition spec if there already exists a compatible partition spec defined in the table. - -Partition field IDs must be reused if an existing partition spec contains an equivalent field. - -#### Partition Transforms - -| Transform name | Description | Source types | Result type | -|-------------------|--------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------|-------------| -| **`identity`** | Source value, unmodified | Any except for `geometry`, `geography`, and `variant` | Source type | -| **`bucket[N]`** | Hash of value, mod `N` (see below) | `int`, `long`, `decimal`, `date`, `time`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns`, `string`, `uuid`, `fixed`, `binary` | `int` | -| **`truncate[W]`** | Value truncated to width `W` (see below) | `int`, `long`, `decimal`, `string`, `binary` | Source type | -| **`year`** | Extract a date or timestamp year, as years from 1970 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | -| **`month`** | Extract a date or timestamp month, as months from 1970-01-01 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | -| **`day`** | Extract a date or timestamp day, as days from 1970-01-01 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | -| **`hour`** | Extract a timestamp hour, as hours from 1970-01-01 00:00:00 | `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | -| **`void`** | Always produces `null` | Any | Source type or `int` | - -All transforms must return `null` for a `null` input value. - -The `void` transform may be used to replace the transform in an existing partition field so that the field is effectively dropped in v1 tables. See partition evolution below. - -#### Bucket Transform Details - -Bucket partition transforms use a 32-bit hash of the source value. The 32-bit hash implementation is the 32-bit Murmur3 hash, x86 variant, seeded with 0. - -Transforms are parameterized by a number of buckets [1], `N`. The hash mod `N` must produce a positive value by first discarding the sign bit of the hash value. In pseudo-code, the function is: - -``` - def bucket_N(x) = (murmur3_x86_32_hash(x) & Integer.MAX_VALUE) % N -``` - -Notes: - -1. Changing the number of buckets as a table grows is possible by evolving the partition spec. - -For hash function details by type, see Appendix B. - -#### Truncate Transform Details - -| **Type** | **Config** | **Truncate specification** | **Examples** | -|---------------|-----------------------|------------------------------------------------------------------|----------------------------------| -| **`int`** | `W`, width | `v - (v % W)` remainders must be positive [1] | `W=10`: `1` ￫ `0`, `-1` ￫ `-10` | -| **`long`** | `W`, width | `v - (v % W)` remainders must be positive [1] | `W=10`: `1` ￫ `0`, `-1` ￫ `-10` | -| **`decimal`** | `W`, width (no scale) | `scaled_W = decimal(W, scale(v))` `v - (v % scaled_W)` [1, 2] | `W=50`, `s=2`: `10.65` ￫ `10.50` | -| **`string`** | `L`, length | Substring of length `L`: `v.substring(0, L)` [3] | `L=3`: `iceberg` ￫ `ice` | -| **`binary`** | `L`, length | Sub array of length `L`: `v.subarray(0, L)` [4] | `L=3`: `\x01\x02\x03\x04\x05` ￫ `\x01\x02\x03` | - -Notes: - -1. The remainder, `v % W`, must be positive. For languages where `%` can produce negative values, the correct truncate function is: `v - (((v % W) + W) % W)` -2. The width, `W`, used to truncate decimal values is applied using the scale of the decimal column to avoid additional (and potentially conflicting) parameters. -3. Strings are truncated to a valid UTF-8 string with no more than `L` code points. -4. In contrast to strings, binary values do not have an assumed encoding and are truncated to `L` bytes. - -#### Partition Evolution - -Table partitioning can be evolved by adding, removing, renaming, or reordering partition spec fields. - -Changing a partition spec produces a new spec identified by a unique spec ID that is added to the table's list of partition specs and may be set as the table's default spec. - -When evolving a spec, changes should not cause partition field IDs to change because the partition field IDs are used as the partition tuple field IDs in manifest files. - -In v2, partition field IDs must be explicitly tracked for each partition field. New IDs are assigned based on the last assigned partition ID in table metadata. - -In v1, partition field IDs were not tracked, but were assigned sequentially starting at 1000 in the reference implementation. This assignment caused problems when reading metadata tables based on manifest files from multiple specs because partition fields with the same ID may contain different data types. For compatibility with old versions, the following rules are recommended for partition evolution in v1 tables: - -1. Do not reorder partition fields -2. Do not drop partition fields; instead replace the field's transform with the `void` transform -3. Only add partition fields at the end of the previous partition spec - -### Sorting - -Users can sort their data within partitions by columns to gain performance. The information on how the data is sorted can be declared per data or delete file, by a **sort order**. - -A sort order is defined by a sort order id and a list of sort fields. The order of the sort fields within the list defines the order in which the sort is applied to the data. Each sort field consists of: - -* A **source column id** or a list of **source column ids** from the table's schema -* A **transform** that is used to produce values to be sorted on from the source column(s). This is the same transform as described in [partition transforms](#partition-transforms). -* A **sort direction**, that can only be either `asc` or `desc` -* A **null order** that describes the order of null values when sorted. Can only be either `nulls-first` or `nulls-last` - -For details on how to serialize a sort order to JSON, see Appendix C. - -Order id `0` is reserved for the unsorted order. - -Sorting floating-point numbers should produce the following behavior: `-NaN` < `-Infinity` < `-value` < `-0` < `0` < `value` < `Infinity` < `NaN`. This aligns with the implementation of Java floating-point types comparisons. - -A data or delete file is associated with a sort order by the sort order's id within [a manifest](#manifests). Therefore, the table must declare all the sort orders for lookup. A table could also be configured with a default sort order id, indicating how the new data should be sorted by default. Writers should use this default sort order to sort the data on write, but are not required to if the default order is prohibitively expensive, as it would be for streaming writes. - -### Manifests - -A manifest is an immutable Avro file that lists data files or delete files, along with each file’s partition data tuple, metrics, and tracking information. One or more manifest files are used to store a [snapshot](#snapshots), which tracks all of the files in a table at some point in time. Manifests are tracked by a [manifest list](#manifest-lists) for each table snapshot. - -A manifest is a valid Iceberg data file: files must use valid Iceberg formats, schemas, and column projection. - -A manifest may store either data files or delete files, but not both because manifests that contain delete files are scanned first during job planning. Whether a manifest is a data manifest or a delete manifest is stored in manifest metadata. - -A manifest stores files for a single partition spec. When a table’s partition spec changes, old files remain in the older manifest and newer files are written to a new manifest. This is required because a manifest file’s schema is based on its partition spec (see below). The partition spec of each manifest is also used to transform predicates on the table's data rows into predicates on partition values that are used during job planning to select files from a manifest. - -A manifest file must store the partition spec and other metadata as properties in the Avro file's key-value metadata: - -=== "v1 - v3" - | v1 | v2 and v3 | Key | Value | - |------------|------------|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------| - | _required_ | _required_ | `schema` | JSON representation of the table schema at the time the manifest was written | - | _optional_ | _required_ | `schema-id` | ID of the schema used to write the manifest as a string | - | _required_ | _required_ | `partition-spec` | JSON representation of only the partition fields array of the partition spec used to write the manifest. See [Appendix C](#partition-specs) | - | _optional_ | _required_ | `partition-spec-id` | ID of the partition spec used to write the manifest as a string | - | _optional_ | _required_ | `format-version` | Table format version number of the manifest as a string | - | | _required_ | `content` | Type of content files tracked by the manifest: "data" or "deletes" | - -The schema of a manifest file is defined by the `manifest_entry` struct, described in the following section. - -#### Manifest Entry Fields - -The `manifest_entry` struct consists of the following fields: - -=== "v1 - v3" - | v1 | v2 and v3 | Field id, name | Type | Description | - | ---------- | ---------- |-------------------------------|-----------------------------------------------------------|-------------| - | _required_ | _required_ | **`0 status`** | `int` with meaning: `0: EXISTING` `1: ADDED` `2: DELETED` | Used to track additions and deletions. Deletes are informational only and not used in scans. | - | _required_ | _optional_ | **`1 snapshot_id`** | `long` | Snapshot id where the file was added, or deleted if status is 2. Inherited when null. | - | | _optional_ | **`3 sequence_number`** | `long` | Data sequence number of the file. Inherited when null and status is 1 (added). | - | | _optional_ | **`4 file_sequence_number`** | `long` | File sequence number indicating when the file was added. Inherited when null and status is 1 (added). | - | _required_ | _required_ | **`2 data_file`** | `data_file` `struct` (see below) | File path, partition tuple, metrics, ... | - -The manifest entry fields are used to keep track of the snapshot in which files were added or logically deleted. The `data_file` struct, defined below, is nested inside the manifest entry so that it can be easily passed to job planning without the manifest entry fields. - -When a file is added to the dataset, its manifest entry should store the snapshot ID in which the file was added and set status to 1 (added). - -When a file is replaced or deleted from the dataset, its manifest entry fields store the snapshot ID in which the file was deleted and status 2 (deleted). The file may be deleted from the file system when the snapshot in which it was deleted is garbage collected, assuming that older snapshots have also been garbage collected [1]. - -Iceberg v2 adds data and file sequence numbers to the entry and makes the snapshot ID optional. Values for these fields are inherited from manifest metadata when `null`. That is, if the field is `null` for an entry, then the entry must inherit its value from the manifest file's metadata, stored in the manifest list. -The `sequence_number` field represents the data sequence number and must never change after a file is added to the dataset. The data sequence number represents a relative age of the file content and should be used for planning which delete files apply to a data file. -The `file_sequence_number` field represents the sequence number of the snapshot that added the file and must also remain unchanged upon assigning at commit. The file sequence number can't be used for pruning delete files as the data within the file may have an older data sequence number. -The data and file sequence numbers are inherited only if the entry status is 1 (added). If the entry status is 0 (existing) or 2 (deleted), the entry must include both sequence numbers explicitly. - -Notes: - -1. Technically, data files can be deleted when the last snapshot that contains the file as “live” data is garbage collected. But this is harder to detect and requires finding the diff of multiple snapshots. It is easier to track what files are deleted in a snapshot and delete them when that snapshot expires. It is not recommended to add a deleted file back to a table. Adding a deleted file can lead to edge cases where incremental deletes can break table snapshots. -2. Manifest list files are required in v2, so that the `sequence_number` and `snapshot_id` to inherit are always available. - -##### Data File Fields - -The `data_file` struct consists of the following fields: - -=== "v1 - v3" - | v1 | v2 | v3 | Field id, name | Type | Description | - | ---------- |------------|------------|-----------------------------------|-----------------------------------------------------------------------------|-------------| - | | _required_ | _required_ | **`134 content`** | `int` with meaning: `0: DATA`, `1: POSITION DELETES`, `2: EQUALITY DELETES` | Type of content stored by the data file: data, equality deletes, or position deletes (all v1 files are data files) | - | _required_ | _required_ | _required_ | **`100 file_path`** | `string` | Full URI for the file with FS scheme | - | _required_ | _required_ | _required_ | **`101 file_format`** | `string` | String file format name, `avro`, `orc`, `parquet`, or `puffin` | - | _required_ | _required_ | _required_ | **`102 partition`** | `struct<...>` | Partition data tuple, schema based on the partition spec output using partition field ids for the struct field ids | - | _required_ | _required_ | _required_ | **`103 record_count`** | `long` | Number of records in this file, or the cardinality of a deletion vector | - | _required_ | _required_ | _required_ | **`104 file_size_in_bytes`** | `long` | Total file size in bytes | - | _required_ | | | ~~**`105 block_size_in_bytes`**~~ | `long` | **Deprecated. Always write a default in v1. Do not write in v2 or v3.** | - | _optional_ | | | ~~**`106 file_ordinal`**~~ | `int` | **Deprecated. Do not write.** | - | _optional_ | | | ~~**`107 sort_columns`**~~ | `list<112: int>` | **Deprecated. Do not write.** | - | _optional_ | _optional_ | _optional_ | **`108 column_sizes`** | `map<117: int, 118: long>` | Map from column id to the total size on disk of all regions that store the column. **Does not include bytes necessary to read other columns, like footers.** Leave null for row-oriented formats (Avro) | - | _optional_ | _optional_ | _optional_ | **`109 value_counts`** | `map<119: int, 120: long>` | Map from column id to number of values in the column (including null and NaN values) | - | _optional_ | _optional_ | _optional_ | **`110 null_value_counts`** | `map<121: int, 122: long>` | Map from column id to number of null values in the column | - | _optional_ | _optional_ | _optional_ | **`137 nan_value_counts`** | `map<138: int, 139: long>` | Map from column id to number of NaN values in the column | - | _optional_ | _optional_ | | ~~**`111 distinct_counts`**~~ | `map<123: int, 124: long>` | **Deprecated. Do not write.** | - | _optional_ | _optional_ | _optional_ | **`125 lower_bounds`** | `map<126: int, 127: binary>` | Map from column id to lower bound in the column serialized as binary [1]. Each value must be less than or equal to all non-null, non-NaN values in the column for the file [2] | - | _optional_ | _optional_ | _optional_ | **`128 upper_bounds`** | `map<129: int, 130: binary>` | Map from column id to upper bound in the column serialized as binary [1]. Each value must be greater than or equal to all non-null, non-Nan values in the column for the file [2] | - | _optional_ | _optional_ | _optional_ | **`131 key_metadata`** | `binary` | Implementation-specific key metadata for encryption | - | _optional_ | _optional_ | _optional_ | **`132 split_offsets`** | `list<133: long>` | Split offsets for the data file. For example, all row group offsets in a Parquet file. Must be sorted ascending | - | | _optional_ | _optional_ | **`135 equality_ids`** | `list<136: int>` | Field ids used to determine row equality in equality delete files. Required when `content=2` and should be null otherwise. Fields with ids listed in this column must be present in the delete file | - | _optional_ | _optional_ | _optional_ | **`140 sort_order_id`** | `int` | ID representing sort order for this file [3]. | - | | | _optional_ | **`142 first_row_id`** | `long` | The `_row_id` for the first row in the data file. See [First Row ID Inheritance](#first-row-id-inheritance) | - | | _optional_ | _optional_ | **`143 referenced_data_file`** | `string` | Fully qualified location (URI with FS scheme) of a data file that all deletes reference [4] | - | | | _optional_ | **`144 content_offset`** | `long` | The offset in the file where the content starts [5] | - | | | _optional_ | **`145 content_size_in_bytes`** | `long` | The length of a referenced content stored in the file; required if `content_offset` is present [5] | - -The `partition` struct stores the tuple of partition values for each file. Its type is derived from the partition fields of the partition spec used to write the manifest file. In v2, the partition struct's field ids must match the ids from the partition spec. - -The column metrics maps are used when filtering to select both data and delete files. For delete files, the metrics must store bounds and counts for all deleted rows, or must be omitted. Storing metrics for deleted rows ensures that the values can be used during job planning to find delete files that must be merged during a scan. - -Notes: - -1. Single-value serialization for lower and upper bounds is detailed in Appendix D. -2. For `float` and `double`, the value `-0.0` must precede `+0.0`, as in the IEEE 754 `totalOrder` predicate. NaNs are not permitted as lower or upper bounds. -3. If sort order ID is missing or unknown, then the order is assumed to be unsorted. Only data files and equality delete files should be written with a non-null order id. [Position deletes](#position-delete-files) are required to be sorted by file and position, not a table order, and should set sort order id to null. Readers must ignore sort order id for position delete files. -4. Position delete metadata can use `referenced_data_file` when all deletes tracked by the entry are in a single data file. Setting the referenced file is required for deletion vectors. -5. The `content_offset` and `content_size_in_bytes` fields are used to reference a specific blob for direct access to a deletion vector. For deletion vectors, these values are required and must exactly match the `offset` and `length` stored in the Puffin footer for the deletion vector blob. -6. The following field ids are reserved on `data_file`: 141. - -###### Bounds for Variant, Geometry, and Geography - -For Variant, values in the `lower_bounds` and `upper_bounds` maps store serialized Variant objects that contain lower or upper bounds respectively. The object keys for the bound-variants are normalized JSON path expressions that uniquely identify a field. The object values are primitive Variant representations of the lower or upper bound for that field. Including bounds for any field is optional and upper and lower bounds must have the same Variant type. - -Bounds for a field must be accurate for all non-null values of the field in a data file. Bounds for values within arrays must be accurate for all values in the array. Bounds must not be written to describe values with mixed Variant types (other than null). For example, a **measurement** field that contains int64 and null values may have bounds, but if the field also contained a string value such as **n/a** or **0** then the field may not have bounds. - -The Variant bounds objects are serialized by concatenating the [Variant encoding](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md) of the metadata (containing the normalized field paths) and the bounds object. -Field paths follow the JSON path format to use normalized path, such as `$['location']['latitude']` or `$['user.name']`. The special path `$` represents bounds for the variant root, indicating that the variant data consists of uniform primitive types, such as strings. - -Examples of valid field paths using normalized JSON path format are: - -* `$` -- the root of the Variant value -* `$['event_type']` -- the `event_type` field in a Variant object -* `$['user.name']` -- the `"user.name"` field in a Variant object -* `$['location']['latitude']` -- the `latitude` field nested within a `location` object -* `$['tags']` -- the `tags` array -* `$['addresses']['zip']` -- the `zip` field in an `addresses` array that contains objects - -For `geometry` and `geography` types, `lower_bounds` and `upper_bounds` are both points of the following coordinates X, Y, Z, and M (see Appendix G) which are the lower / upper bound of all objects in the file. - -For `geography` only, xmin (X value of `lower_bounds`) may be greater than xmax (X value of `upper_bounds`), in which case an object in this bounding box may match if it contains an X such that x >= xmin OR x <= xmax. In geographic terminology, the concepts of xmin, xmax, ymin, and ymax are also known as westernmost, easternmost, southernmost and northernmost, respectively. These points are further restricted to the canonical ranges of [-180..180] for X and [-90..90] for Y. - -When calculating upper and lower bounds for `geometry` and `geography`, null or NaN values in a coordinate dimension are skipped; for example, POINT (1 NaN) contributes a value to X but no values to Y, Z, or M dimension bounds. If a dimension has only null or NaN values, that dimension is omitted from the bounding box. If either the X or Y dimension is missing then the bounding box itself is not produced. - -#### Sequence Number Inheritance - -Manifests track the sequence number when a data or delete file was added to the table. - -When adding a new file, its data and file sequence numbers are set to `null` because the snapshot's sequence number is not assigned until the snapshot is successfully committed. When reading, sequence numbers are inherited by replacing `null` with the manifest's sequence number from the manifest list. -It is also possible to add a new file with data that logically belongs to an older sequence number. In that case, the data sequence number must be provided explicitly and not inherited. However, the file sequence number must be always assigned when the snapshot is successfully committed. - -When writing an existing file to a new manifest or marking an existing file as deleted, the data and file sequence numbers must be non-null and set to the original values that were either inherited or provided at the commit time. - -Inheriting sequence numbers through the metadata tree allows writing a new manifest without a known sequence number, so that a manifest can be written once and reused in commit retries. To change a sequence number for a retry, only the manifest list must be rewritten. - -When reading v1 manifests with no sequence number column, sequence numbers for all files must default to 0. - -#### First Row ID Inheritance - -When adding a new data file, its `first_row_id` field is set to `null` because it is not assigned until the snapshot is successfully committed. - -When reading, the `first_row_id` is assigned by replacing `null` with the manifest's `first_row_id` plus the sum of `record_count` for all data files that preceded the file in the manifest that also had a null `first_row_id`. - -The inherited value of `first_row_id` must be written into data file metadata when creating existing and deleted entries. The value of `first_row_id` for delete files is always `null`. - -Any null (unassigned) `first_row_id` must be assigned via inheritance, even if the data file is existing. This ensures that row IDs are assigned to existing data files in upgraded tables in the first commit after upgrading to v3. - -### Snapshots - -A snapshot consists of the following fields: -=== "v1 - v3" - | v1 | v2 | v3 | Field | Description | - | ---------- | ---------- |------------|------------------------------|-------------| - | _required_ | _required_ | _required_ | **`snapshot-id`** | A unique long ID | - | _optional_ | _optional_ | _optional_ | **`parent-snapshot-id`** | The snapshot ID of the snapshot's parent. Omitted for any snapshot with no parent | - | | _required_ | _required_ | **`sequence-number`** | A monotonically increasing long that tracks the order of changes to a table | - | _required_ | _required_ | _required_ | **`timestamp-ms`** | A timestamp when the snapshot was created, used for garbage collection and table inspection | - | _optional_ | _required_ | _required_ | **`manifest-list`** | The location of a manifest list for this snapshot that tracks manifest files with additional metadata | - | _optional_ | | | **`manifests`** | A list of manifest file locations. Must be omitted if `manifest-list` is present | - | _optional_ | _required_ | _required_ | **`summary`** | A string map that summarizes the snapshot changes, including `operation` as a _required_ field (see below) | - | _optional_ | _optional_ | _optional_ | **`schema-id`** | ID of the table's current schema when the snapshot was created | - | | | _required_ | **`first-row-id`** | The first `_row_id` assigned to the first row in the first data file in the first manifest, see [Row Lineage](#row-lineage) | - | | | _required_ | **`added-rows`** | The upper bound of the number of rows with assigned row IDs, see [Row Lineage](#row-lineage) | - | | | _optional_ | **`key-id`** | ID of the encryption key that encrypts the manifest list key metadata | - -The snapshot summary's `operation` field is used by some operations, like snapshot expiration, to skip processing certain snapshots. Possible `operation` values are: - -* `append` -- Only data files were added and no files were removed. -* `replace` -- Data and delete files were added and removed without changing table data; i.e., compaction, changing the data file format, or relocating data files. -* `overwrite` -- Data and delete files were added and removed in a logical overwrite operation. -* `delete` -- Data files were removed and their contents logically deleted and/or delete files were added to delete rows. - -For other optional snapshot summary fields, see [Appendix F](#optional-snapshot-summary-fields). - -Data and delete files for a snapshot can be stored in more than one manifest. This enables: - -* Appends can add a new manifest to minimize the amount of data written, instead of adding new records by rewriting and appending to an existing manifest. (This is called a “fast append”.) -* Tables can use multiple partition specs. A table’s partition configuration can evolve if, for example, its data volume changes. Each manifest uses a single partition spec, and queries do not need to change because partition filters are derived from data predicates. -* Large tables can be split across multiple manifests so that implementations can parallelize job planning or reduce the cost of rewriting a manifest. - -Manifests for a snapshot are tracked by a manifest list. - -Valid snapshots are stored as a list in table metadata. For serialization, see Appendix C. - -#### Snapshot Row IDs - -A snapshot's `first-row-id` is assigned to the table's current `next-row-id` on each commit attempt. If a commit is retried, the `first-row-id` must be reassigned based on the table's current `next-row-id`. The `first-row-id` field is required even if a commit does not assign any ID space. - -The snapshot's `first-row-id` is the starting `first_row_id` assigned to manifests in the snapshot's manifest list. - -The snapshot's `added-rows` captures the upper bound of the number of rows with assigned row IDs. -It can be used safely to increment the table's `next-row-id` during a commit. -It can be more than the number of rows added in this snapshot and include some existing rows, -see [Row Lineage Example](#row-lineage-example). - -### Manifest Lists - -Snapshots are embedded in table metadata, but the list of manifests for a snapshot are stored in a separate manifest list file. - -A new manifest list is written for each attempt to commit a snapshot because the list of manifests always changes to produce a new snapshot. When a manifest list is written, the (optimistic) sequence number of the snapshot is written for all new manifest files tracked by the list. - -A manifest list includes summary metadata that can be used to avoid scanning all of the manifests in a snapshot when planning a table scan. This includes the number of added, existing, and deleted files, and a summary of values for each field of the partition spec used to write the manifest. - -A manifest list is a valid Iceberg data file: files must use valid Iceberg formats, schemas, and column projection. - -Manifest list files store `manifest_file`, a struct with the following fields: - -=== "v1 - v3" - | v1 | v2 | v3 | Field id, name | Type | Description | - | ---------- | ---------- |------------|-------------------------------------|---------------------------------------------|-------------| - | _required_ | _required_ | _required_ | **`500 manifest_path`** | `string` | Location of the manifest file | - | _required_ | _required_ | _required_ | **`501 manifest_length`** | `long` | Length of the manifest file in bytes | - | _required_ | _required_ | _required_ | **`502 partition_spec_id`** | `int` | ID of a partition spec used to write the manifest; must be listed in table metadata `partition-specs` | - | | _required_ | _required_ | **`517 content`** | `int` with meaning: `0: data`, `1: deletes` | The type of files tracked by the manifest, either data or delete files; 0 for all v1 manifests | - | | _required_ | _required_ | **`515 sequence_number`** | `long` | The sequence number when the manifest was added to the table; use 0 when reading v1 manifest lists | - | | _required_ | _required_ | **`516 min_sequence_number`** | `long` | The minimum data sequence number of all live data or delete files in the manifest; use 0 when reading v1 manifest lists | - | _required_ | _required_ | _required_ | **`503 added_snapshot_id`** | `long` | ID of the snapshot where the manifest file was added | - | _optional_ | _required_ | _required_ | **`504 added_files_count`** | `int` | Number of entries in the manifest that have status `ADDED` (1), when `null` this is assumed to be non-zero | - | _optional_ | _required_ | _required_ | **`505 existing_files_count`** | `int` | Number of entries in the manifest that have status `EXISTING` (0), when `null` this is assumed to be non-zero | - | _optional_ | _required_ | _required_ | **`506 deleted_files_count`** | `int` | Number of entries in the manifest that have status `DELETED` (2), when `null` this is assumed to be non-zero | - | _optional_ | _required_ | _required_ | **`512 added_rows_count`** | `long` | Number of rows in all of files in the manifest that have status `ADDED`, when `null` this is assumed to be non-zero | - | _optional_ | _required_ | _required_ | **`513 existing_rows_count`** | `long` | Number of rows in all of files in the manifest that have status `EXISTING`, when `null` this is assumed to be non-zero | - | _optional_ | _required_ | _required_ | **`514 deleted_rows_count`** | `long` | Number of rows in all of files in the manifest that have status `DELETED`, when `null` this is assumed to be non-zero | - | _optional_ | _optional_ | _optional_ | **`507 partitions`** | `list<508: field_summary>` **(see below)** | A list of field summaries for each partition field in the spec. Each field in the list corresponds to a field in the manifest file’s partition spec. | - | _optional_ | _optional_ | _optional_ | **`519 key_metadata`** | `binary` | Implementation-specific key metadata for encryption | - | | | _optional_ | **`520 first_row_id`** | `long` | The starting `_row_id` to assign to rows added by `ADDED` data files [First Row ID Assignment](#first-row-id-assignment) | - -`field_summary` is a struct with the following fields: -=== "v1 - v3" - | v1 | v2 and v3 | Field id, name | Type | Description | - | ---------- | ---------- |-------------------------|---------------|-------------| - | _required_ | _required_ | **`509 contains_null`** | `boolean` | Whether the manifest contains at least one partition with a null value for the field | - | _optional_ | _optional_ | **`518 contains_nan`** | `boolean` | Whether the manifest contains at least one partition with a NaN value for the field | - | _optional_ | _optional_ | **`510 lower_bound`** | `bytes` [1] | Lower bound for the non-null, non-NaN values in the partition field, or null if all values are null or NaN [2] | - | _optional_ | _optional_ | **`511 upper_bound`** | `bytes` [1] | Upper bound for the non-null, non-NaN values in the partition field, or null if all values are null or NaN [2] | - -Notes: - -1. Lower and upper bounds are serialized to bytes using the single-object serialization in Appendix D. The type of used to encode the value is the type of the partition field data. -2. If -0.0 is a value of the partition field, the `lower_bound` must not be +0.0, and if +0.0 is a value of the partition field, the `upper_bound` must not be -0.0. - -#### First Row ID Assignment - -The `first_row_id` for existing manifests must be preserved when writing a new manifest list. The value of `first_row_id` for delete manifests is always `null`. The `first_row_id` is only assigned for data manifests that do not have a `first_row_id`. Assignment must account for data files that will be assigned `first_row_id` values when the manifest is read. - -The first manifest without a `first_row_id` is assigned a value that is greater than or equal to the `first_row_id` of the snapshot. Subsequent manifests without a `first_row_id` are assigned one based on the previous manifest to be assigned a `first_row_id`. Each assigned `first_row_id` must increase by the row count of all files that will be assigned a `first_row_id` via inheritance in the last assigned manifest. That is, each `first_row_id` must be greater than or equal to the last assigned `first_row_id` plus the total record count of data files with a null `first_row_id` in the last assigned manifest. - -A simple and valid approach is to estimate the number of rows in data files that will be assigned a `first_row_id` using the manifest's `added_rows_count` and `existing_rows_count`: `first_row_id = last_assigned.first_row_id + last_assigned.added_rows_count + last_assigned.existing_rows_count`. - -### Scan Planning - -Scans are planned by reading the manifest files for the current snapshot. Deleted entries in data and delete manifests (those marked with status "DELETED") are not used in a scan. - -Manifests that contain no matching files, determined using either file counts or partition summaries, may be skipped. - -For each manifest, scan predicates, which filter data rows, are converted to partition predicates, which filter partition tuples. These partition predicates are used to select relevant data files, delete files, and deletion vector metadata. Conversion uses the partition spec that was used to write the manifest file regardless of the current partition spec. - -Scan predicates are converted to partition predicates using an _inclusive projection_: if a scan predicate matches a row, then the partition predicate must match that row’s partition. This is called _inclusive_ [1] because rows that do not match the scan predicate may be included in the scan by the partition predicate. - -For example, an `events` table with a timestamp column named `ts` that is partitioned by `ts_day=day(ts)` is queried by users with ranges over the timestamp column: `ts > X`. The inclusive projection is `ts_day >= day(X)`, which is used to select files that may have matching rows. Note that, in most cases, timestamps just before `X` will be included in the scan because the file contains rows that match the predicate and rows that do not match the predicate. - -The inclusive projection for an unknown partition transform is _true_ because the partition field is ignored and not used in filtering. - -Scan predicates are also used to filter data and delete files using column bounds and counts that are stored by field id in manifests. The same filter logic can be used for both data and delete files because both store metrics of the rows either inserted or deleted. If metrics show that a delete file has no rows that match a scan predicate, it may be ignored just as a data file would be ignored [2]. - -Data files that match the query filter must be read by the scan. - -Note that for any snapshot, all file paths marked with "ADDED" or "EXISTING" may appear at most once across all manifest files in the snapshot. If a file path appears more than once, the results of the scan are undefined. Reader implementations may raise an error in this case, but are not required to do so. - -Delete files and deletion vector metadata that match the filters must be applied to data files at read time, limited by the following scope rules. - -* A deletion vector must be applied to a data file when all of the following are true: - - The data file's `file_path` is equal to the deletion vector's `referenced_data_file` - - The data file's data sequence number is _less than or equal to_ the deletion vector's data sequence number - - The data file's partition (both spec and partition values) is equal [4] to the deletion vector's partition -* A _position_ delete file must be applied to a data file when all of the following are true: - - The data file's `file_path` is equal to the delete file's `referenced_data_file` if it is non-null - - The data file's data sequence number is _less than or equal to_ the delete file's data sequence number - - The data file's partition (both spec and partition values) is equal [4] to the delete file's partition - - There is no deletion vector that must be applied to the data file (when added, such a vector must contain all deletes from existing position delete files) -* An _equality_ delete file must be applied to a data file when all of the following are true: - - The data file's data sequence number is _strictly less than_ the delete's data sequence number - - The data file's partition (both spec id and partition values) is equal [4] to the delete file's partition _or_ the delete file's partition spec is unpartitioned - -In general, deletes are applied only to data files that are older and in the same partition, except for two special cases: - -* Equality delete files stored with an unpartitioned spec are applied as global deletes. Otherwise, delete files do not apply to files in other partitions. -* Position deletes (vectors and files) must be applied to data files from the same commit, when the data and delete file data sequence numbers are equal. This allows deleting rows that were added in the same commit. - -Notes: - -1. An alternative, *strict projection*, creates a partition predicate that will match a file if all of the rows in the file must match the scan predicate. These projections are used to calculate the residual predicates for each file in a scan. -2. For example, if `file_a` has rows with `id` between 1 and 10 and a delete file contains rows with `id` between 1 and 4, a scan for `id = 9` may ignore the delete file because none of the deletes can match a row that will be selected. -3. Floating point partition values are considered equal if their IEEE 754 floating-point "single format" bit layout are equal with NaNs normalized to have only the most significant mantissa bit set (the equivalent of calling `Float.floatToIntBits` or `Double.doubleToLongBits` in Java). The Avro specification requires all floating point values to be encoded in this format. -4. Unknown partition transforms do not affect partition equality. Although partition fields with unknown transforms are ignored for filtering, the result of an unknown transform is still used when testing whether partition values are equal. - -### Snapshot References - -Iceberg tables keep track of branches and tags using snapshot references. -Tags are labels for individual snapshots. Branches are mutable named references that can be updated by committing a new snapshot as the branch's referenced snapshot using the [Commit Conflict Resolution and Retry](#commit-conflict-resolution-and-retry) procedures. - -The snapshot reference object records all the information of a reference including snapshot ID, reference type and [Snapshot Retention Policy](#snapshot-retention-policy). - -=== "v1 - v3" - | v1 | v2 and v3 | Field name | Type | Description | - | ---------- | ---------- | ---------------------------- | --------- | ----------- | - | _required_ | _required_ | **`snapshot-id`** | `long` | A reference's snapshot ID. The tagged snapshot or latest snapshot of a branch. | - | _required_ | _required_ | **`type`** | `string` | Type of the reference, `tag` or `branch` | - | _optional_ | _optional_ | **`min-snapshots-to-keep`** | `int` | For `branch` type only, a positive number for the minimum number of snapshots to keep in a branch while expiring snapshots. Defaults to table property `history.expire.min-snapshots-to-keep`. | - | _optional_ | _optional_ | **`max-snapshot-age-ms`** | `long` | For `branch` type only, a positive number for the max age of snapshots to keep when expiring, including the latest snapshot. Defaults to table property `history.expire.max-snapshot-age-ms`. | - | _optional_ | _optional_ | **`max-ref-age-ms`** | `long` | For snapshot references except the `main` branch, a positive number for the max age of the snapshot reference to keep while expiring snapshots. Defaults to table property `history.expire.max-ref-age-ms`. The `main` branch never expires. | - -Valid snapshot references are stored as the values of the `refs` map in table metadata. For serialization, see Appendix C. - -### Snapshot Retention Policy - -Table snapshots expire and are removed from metadata to allow removed or replaced data files to be physically deleted. -The snapshot expiration procedure removes snapshots from table metadata and applies the table's retention policy. -Retention policy can be configured both globally and on snapshot reference through properties `min-snapshots-to-keep`, `max-snapshot-age-ms` and `max-ref-age-ms`. - -When expiring snapshots, retention policies in table and snapshot references are evaluated in the following way: - -1. Start with an empty set of snapshots to retain -2. Remove any refs (other than main) where the referenced snapshot is older than `max-ref-age-ms` -3. For each branch and tag, add the referenced snapshot to the retained set -4. For each branch, add its ancestors to the retained set until: - 1. The snapshot is older than `max-snapshot-age-ms`, AND - 2. The snapshot is not one of the first `min-snapshots-to-keep` in the branch (including the branch's referenced snapshot) -5. Expire any snapshot not in the set of snapshots to retain. - -### Table Metadata - -Table metadata is stored as JSON. Each table metadata change creates a new table metadata file that is committed by an atomic operation. This operation is used to ensure that a new version of table metadata replaces the version on which it was based. This produces a linear history of table versions and ensures that concurrent writes are not lost. - -The atomic operation used to commit metadata depends on how tables are tracked and is not standardized by this spec. See the sections below for examples. - -#### Table Metadata Fields - -Table metadata consists of the following fields: - -=== "v1 - v3" - | v1 | v2 | v3 | Field | Description | - | ---------- | ---------- |------------|-----------------------------| ------------| - | _required_ | _required_ | _required_ | **`format-version`** | An integer version number for the format. Implementations must throw an exception if a table’s version is higher than the supported version. | - | _optional_ | _required_ | _required_ | **`table-uuid`** | A UUID that identifies the table, generated when the table is created. Implementations must throw an exception if a table’s UUID does not match the expected UUID after refreshing metadata. | - | _required_ | _required_ | _required_ | **`location`** | The table’s base location. This is used by writers to determine where to store data files, manifest files, and table metadata files. | - | | _required_ | _required_ | **`last-sequence-number`** | The table’s highest assigned sequence number, a monotonically increasing long that tracks the order of snapshots in a table. | - | _required_ | _required_ | _required_ | **`last-updated-ms`** | Timestamp in milliseconds from the unix epoch when the table was last updated. Each table metadata file should update this field just before writing. | - | _required_ | _required_ | _required_ | **`last-column-id`** | An integer; the highest assigned column ID for the table. This is used to ensure columns are always assigned an unused ID when evolving schemas. | - | _required_ | | | **`schema`** | The table’s current schema. (**Deprecated**: use `schemas` and `current-schema-id` instead) | - | _optional_ | _required_ | _required_ | **`schemas`** | A list of schemas, stored as objects with `schema-id`. | - | _optional_ | _required_ | _required_ | **`current-schema-id`** | ID of the table’s current schema. | - | _required_ | | | **`partition-spec`** | The table’s current partition spec, stored as only fields. (**Deprecated**: use `partition-specs` and `default-spec-id` instead) Note that this is used by writers to partition data, but is not used when reading because reads use the specs stored in manifest files. | - | _optional_ | _required_ | _required_ | **`partition-specs`** | A list of partition specs, stored as full partition spec objects. | - | _optional_ | _required_ | _required_ | **`default-spec-id`** | ID of the "current" spec that writers should use by default. | - | _optional_ | _required_ | _required_ | **`last-partition-id`** | An integer; the highest assigned partition field ID across all partition specs for the table. This is used to ensure partition fields are always assigned an unused ID when evolving specs. | - | _optional_ | _optional_ | _optional_ | **`properties`** | A string to string map of table properties. This is used to control settings that affect reading and writing and is not intended to be used for arbitrary metadata. For example, `commit.retry.num-retries` is used to control the number of commit retries. | - | _optional_ | _optional_ | _optional_ | **`current-snapshot-id`** | `long` ID of the current table snapshot; must be the same as the current ID of the `main` branch in `refs`. | - | _optional_ | _optional_ | _optional_ | **`snapshots`** | A list of valid snapshots. Valid snapshots are snapshots for which all data files exist in the file system. A data file must not be deleted from the file system until the last snapshot in which it was listed is garbage collected. | - | _optional_ | _optional_ | _optional_ | **`snapshot-log`** | A list (optional) of timestamp and snapshot ID pairs that encodes changes to the current snapshot for the table. Each time the current-snapshot-id is changed, a new entry should be added with the last-updated-ms and the new current-snapshot-id. When snapshots are expired from the list of valid snapshots, all entries before a snapshot that has expired should be removed. | - | _optional_ | _optional_ | _optional_ | **`metadata-log`** | A list (optional) of timestamp and metadata file location pairs that encodes changes to the previous metadata files for the table. Each time a new metadata file is created, a new entry of the previous metadata file location should be added to the list. Tables can be configured to remove oldest metadata log entries and keep a fixed-size log of the most recent entries after a commit. | - | _optional_ | _required_ | _required_ | **`sort-orders`** | A list of sort orders, stored as full sort order objects. | - | _optional_ | _required_ | _required_ | **`default-sort-order-id`** | Default sort order id of the table. Note that this could be used by writers, but is not used when reading because reads use the specs stored in manifest files. | - | | _optional_ | _optional_ | **`refs`** | A map of snapshot references. The map keys are the unique snapshot reference names in the table, and the map values are snapshot reference objects. There is always a `main` branch reference pointing to the `current-snapshot-id` even if the `refs` map is null. | - | _optional_ | _optional_ | _optional_ | **`statistics`** | A list (optional) of [table statistics](#table-statistics). | - | _optional_ | _optional_ | _optional_ | **`partition-statistics`** | A list (optional) of [partition statistics](#partition-statistics). | - | | | _required_ | **`next-row-id`** | A `long` higher than all assigned row IDs; the next snapshot’s `first-row-id`. See [Row Lineage](#row-lineage). | - | | | _optional_ | **`encryption-keys`** | A list (optional) of [encryption keys](#encryption-keys) used for table encryption. | - -For serialization details, see Appendix C. - -When a new snapshot is added, the table's `next-row-id` should be increased by the sum of `record_count` for all data files that will be assigned a `first_row_id` via inheritance in the snapshot. The `next-row-id` must always be higher than any assigned row ID in the table. - -A simple and valid approach is estimate of the number of rows in data files that will be assigned a `first_row_id` using the manifests' `added_rows_count` and `existing_rows_count`. Using the last assigned manifest, this is `next-row-id = last_assigned.first_row_id + last_assigned.added_rows_count + last_assigned.existing_rows_count`. - -#### Table Statistics - -Table statistics files are valid [Puffin files](puffin-spec.md). Statistics are informational. A reader can choose to -ignore statistics information. Statistics support is not required to read the table correctly. A table can contain -many statistics files associated with different table snapshots. - -Statistics files metadata within `statistics` table metadata field is a struct with the following fields: - -=== "v1 - v3" - | v1 | v2 and v3 | Field name | Type | Description | - | ---------- | ---------- |---------------------------------|-----------------------|-------------| - | _required_ | _required_ | **`snapshot-id`** | `long` | ID of the Iceberg table's snapshot the statistics file is associated with. | - | _required_ | _required_ | **`statistics-path`** | `string` | Path of the statistics file. See [Puffin file format](puffin-spec.md). | - | _required_ | _required_ | **`file-size-in-bytes`** | `long` | Size of the statistics file. | - | _required_ | _required_ | **`file-footer-size-in-bytes`** | `long` | Total size of the statistics file's footer (not the footer payload size). See [Puffin file format](puffin-spec.md) for footer definition. | - | _optional_ | _optional_ | **`key-metadata`** | | Base64-encoded implementation-specific key metadata for encryption. | - | _required_ | _required_ | **`blob-metadata`** | `list` (see below) | A list of the blob metadata for statistics contained in the file with structure described below. | - -Blob metadata is a struct with the following fields: - -=== "v1 - v3" - | v1 | v2 and v3 | Field name | Type | Description | - | ---------- | ---------- |-----------------------|-----------------------|-------------| - | _required_ | _required_ | **`type`** | `string` | Type of the blob. Matches Blob type in the Puffin file. | - | _required_ | _required_ | **`snapshot-id`** | `long` | ID of the Iceberg table's snapshot the blob was computed from. | - | _required_ | _required_ | **`sequence-number`** | `long` | Sequence number of the Iceberg table's snapshot the blob was computed from. | - | _required_ | _required_ | **`fields`** | `list` | Ordered list of fields, given by field ID, on which the statistic was calculated. | - | _optional_ | _optional_ | **`properties`** | `map` | Additional properties associated with the statistic. Subset of Blob properties in the Puffin file. | - -#### Partition Statistics - -Partition statistics files are based on [partition statistics file spec](#partition-statistics-file). -Partition statistics are not required for reading or planning and readers may ignore them. -Each table snapshot may be associated with at most one partition statistics file. -A writer can optionally write the partition statistics file during each write operation, or it can also be computed on demand. -Partition statistics file must be registered in the table metadata file to be considered as a valid statistics file for the reader. - -`partition-statistics` field of table metadata is an optional list of structs with the following fields: - -=== "v1 - v3" - | v1 | v2 | v3 | Field name | Type | Description | - | ---------- | ---------- |------------|--------------------------|----------|-------------| - | _required_ | _required_ | _required_ | **`snapshot-id`** | `long` | ID of the Iceberg table's snapshot the partition statistics file is associated with. | - | _required_ | _required_ | _required_ | **`statistics-path`** | `string` | Path of the partition statistics file. See [Partition statistics file](#partition-statistics-file). | - | _required_ | _required_ | _required_ | **`file-size-in-bytes`** | `long` | Size of the partition statistics file. | - -##### Partition Statistics File - -Statistics information for each unique partition tuple is stored as a row in any of the data file format of the table (for example, Parquet or ORC). -These rows must be sorted (in ascending manner with NULL FIRST) by `partition` field to optimize filtering rows while scanning. - -The schema of the partition statistics file is as follows: - -=== "v1 - v3" - | v1 | v2 | v3 | Field id, name | Type | Description | - | ---------- | ---------- |------------|------------------------------------------|--------------|-------------| - | _required_ | _required_ | _required_ | **`1 partition`** | `struct<..>` | Partition data tuple, schema based on the unified partition type considering all specs in a table | - | _required_ | _required_ | _required_ | **`2 spec_id`** | `int` | Partition spec id | - | _required_ | _required_ | _required_ | **`3 data_record_count`** | `long` | Count of records in data files | - | _required_ | _required_ | _required_ | **`4 data_file_count`** | `int` | Count of data files | - | _required_ | _required_ | _required_ | **`5 total_data_file_size_in_bytes`** | `long` | Total size of data files in bytes | - | _optional_ | _optional_ | _required_ | **`6 position_delete_record_count`** | `long` | Count of position deletes across position delete files and deletion vectors | - | _optional_ | _optional_ | _required_ | **`7 position_delete_file_count`** | `int` | Count of position delete files ignoring deletion vectors | - | | | _required_ | **`13 dv_count`** | `int` | Count of deletion vectors | - | _optional_ | _optional_ | _required_ | **`8 equality_delete_record_count`** | `long` | Count of records in equality delete files | - | _optional_ | _optional_ | _required_ | **`9 equality_delete_file_count`** | `int` | Count of equality delete files | - | _optional_ | _optional_ | _optional_ | **`10 total_record_count`** | `long` | Accurate count of records in a partition after applying deletes if any | - | _optional_ | _optional_ | _optional_ | **`11 last_updated_at`** | `long` | Timestamp in milliseconds from the unix epoch when the partition was last updated | - | _optional_ | _optional_ | _optional_ | **`12 last_updated_snapshot_id`** | `long` | ID of snapshot that last updated this partition | - -Note that partition data tuple's schema is based on the partition spec output using partition field ids for the struct field ids. -The unified partition type is a struct containing all fields that have ever been a part of any spec in the table -and sorted by the field ids in ascending order. -In other words, the struct fields represent a union of all known partition fields sorted in ascending order by the field ids. - -For example, - -1. `spec#0` has two fields `{field#1, field#2}` -and then the table has evolved into `spec#1` which has three fields `{field#1, field#2, field#3}`. -The unified partition type looks like `Struct`. - -2. `spec#0` has two fields `{field#1, field#2}` -and then the table has evolved into `spec#1` which has just one field `{field#2}`. -The unified partition type looks like `Struct`. - -When a v2 table is upgraded to v3 or later, the `position_delete_record_count` field must account for all position deletes, including those from remaining v2 position delete files and any deletion vectors added after the upgrade. - -Calculating `total_record_count` for a table with equality deletes or v2 position delete files requires reading data. In such cases, implementations may omit this field and must write `NULL`, indicating that the exact record count in a partition is unknown. -If a table has no deletes or only deletion vectors, implementations are encouraged to populate `total_record_count` using metadata in manifests. - -#### Encryption Keys - -Keys used for table encryption can be tracked in table metadata as a list named `encryption-keys`. The schema of each key is a struct with the following fields: -=== "v1 - v3" - | v1 | v2 | v3 | Field name | Type | Description | - |----|----|------------|-------------------------------|-----------------------|-------------| - | | | _required_ | **`key-id`** | `string` | ID of the encryption key | - | | | _required_ | **`encrypted-key-metadata`** | `string` | Encrypted key and metadata, base64 encoded [1] | - | | | _optional_ | **`encrypted-by-id`** | `string` | Optional ID of the key used to encrypt or wrap `key-metadata` | - | | | _optional_ | **`properties`** | `map` | A string to string map of additional metadata used by the table's encryption scheme | - -Notes: - -1. The format of encrypted key metadata is determined by the table's encryption scheme and can be a wrapped format specific to the table's KMS provider. - -### Commit Conflict Resolution and Retry - -When two commits happen at the same time and are based on the same version, only one commit will succeed. In most cases, the failed commit can be applied to the new current version of table metadata and retried. Updates verify the conditions under which they can be applied to a new version and retry if those conditions are met. - -* Append operations have no requirements and can always be applied. -* Replace operations must verify that the files that will be deleted are still in the table. Examples of replace operations include format changes (replace an Avro file with a Parquet file) and compactions (several files are replaced with a single file that contains the same rows). -* Delete operations must verify that specific files to delete are still in the table. Delete operations based on expressions can always be applied (e.g., where timestamp < X). -* Table schema updates and partition spec changes must validate that the schema has not changed between the base version and the current version. - -#### File System Tables - -_Note: This file system based scheme to commit a metadata file is **deprecated** and will be removed in version 4 of this spec. The scheme is **unsafe** in object stores and local file systems._ - -An atomic swap can be implemented using atomic rename in file systems that support it, like HDFS [1]. - -Each version of table metadata is stored in a metadata folder under the table’s base location using a file naming scheme that includes a version number, `V`: `v.metadata.json`. To commit a new metadata version, `V+1`, the writer performs the following steps: - -1. Read the current table metadata version `V`. -2. Create new table metadata based on version `V`. -3. Write the new table metadata to a unique file: `.metadata.json`. -4. Rename the unique file to the well-known file for version `V`: `v.metadata.json`. - 1. If the rename succeeds, the commit succeeded and `V+1` is the table’s current version - 2. If the rename fails, go back to step 1. - -Notes: - -1. The file system table scheme is implemented in [HadoopTableOperations](../javadoc/{{ icebergVersion }}/org/apache/iceberg/hadoop/HadoopTableOperations.html). - -#### Metastore Tables - -The atomic swap needed to commit new versions of table metadata can be implemented by storing a pointer in a metastore or database that is updated with a check-and-put operation [1]. The check-and-put validates that the version of the table that a write is based on is still current and then makes the new metadata from the write the current version. - -Each version of table metadata is stored in a metadata folder under the table’s base location using a naming scheme that includes a version and UUID: `-.metadata.json`. To commit a new metadata version, `V+1`, the writer performs the following steps: - -1. Create a new table metadata file based on the current metadata. -2. Write the new table metadata to a unique file: `-.metadata.json`. -3. Request that the metastore swap the table’s metadata pointer from the location of `V` to the location of `V+1`. - 1. If the swap succeeds, the commit succeeded. `V` was still the latest metadata version and the metadata file for `V+1` is now the current metadata. - 2. If the swap fails, another writer has already created `V+1`. The current writer goes back to step 1. - -Notes: - -1. The metastore table scheme is partly implemented in [BaseMetastoreTableOperations](../javadoc/{{ icebergVersion }}/org/apache/iceberg/BaseMetastoreTableOperations.html). - -### Delete Formats - -This section details how to encode row-level deletes in Iceberg delete files. Row-level deletes are added by v2 and are not supported in v1. Deletion vectors are added in v3 and are not supported in v2 or earlier. Position delete files must not be added to v3 tables, but existing position delete files are valid. - -There are different formats for encoding row-level deletes: - -* Deletion vectors (DVs) identify deleted rows within a single referenced data file by position in a bitmap -* Position delete files identify deleted rows by file location and row position (**deprecated** in v3) -* Equality delete files identify deleted rows by the value of one or more columns - -Deletion vectors are a binary representation of deletes for a single data file that is more efficient at execution time than position delete files. Unlike equality or position delete files, there can be at most one deletion vector for a given data file in a snapshot. Writers must ensure that there is at most one deletion vector per data file and must merge new deletes with existing vectors or position delete files. -When removing a data file, writers must also remove any deletion vector that applies to that data file from delete manifests. Writers are not required to rewrite Puffin files that contain the removed deletion vectors. - -Row-level delete files (both equality and position delete files) are valid Iceberg data files: files must use valid Iceberg formats, schemas, and column projection. It is recommended that these delete files are written using the table's default file format. - -Row-level delete files and deletion vectors are tracked by manifests. A separate set of manifests is used for delete files and DVs, but the same manifest schema is used for both data and delete manifests. Deletion vectors are tracked individually by file location, offset, and length within the containing file. Deletion vector metadata must include the referenced data file. - -Both position and equality delete files allow encoding deleted row values with a delete. This can be used to reconstruct a stream of changes to a table. - -#### Deletion Vectors - -Deletion vectors identify deleted rows of a file by encoding deleted positions in a bitmap. A set bit at position P indicates that the row at position P is deleted. - -These vectors are stored using the `deletion-vector-v1` blob definition from the [Puffin spec][puffin-spec]. - -Deletion vectors support positive 64-bit positions, but are optimized for cases where most positions fit in 32 bits by using a collection of 32-bit Roaring bitmaps. 64-bit positions are divided into a 32-bit "key" using the most significant 4 bytes and a 32-bit sub-position using the least significant 4 bytes. For each key in the set of positions, a 32-bit Roaring bitmap is maintained to store a set of 32-bit sub-positions for that key. - -To test whether a certain position is set, its most significant 4 bytes (the key) are used to find a 32-bit bitmap and the least significant 4 bytes (the sub-position) are tested for inclusion in the bitmap. If a bitmap is not found for the key, then it is not set. - -Delete manifests track deletion vectors individually by the containing file location (`file_path`), starting offset of the DV blob (`content_offset`), and total length of the blob (`content_size_in_bytes`). Multiple deletion vectors can be stored in the same file. There are no restrictions on the data files that can be referenced by deletion vectors in the same Puffin file. - -At most one deletion vector is allowed per data file in a snapshot. If a DV is written for a data file, it must replace all previously written position delete files so that when a DV is present, readers can safely ignore matching position delete files. - -[puffin-spec]: https://iceberg.apache.org/puffin-spec/ - -#### Position Delete Files - -Position-based delete files identify deleted rows by file and position in one or more data files, and may optionally contain the deleted row. - -_Note: Position delete files are **deprecated** in v3. Existing position deletes must be written to delete vectors when updating the position deletes for a data file._ - -A data row is deleted if there is an entry in a position delete file for the row's file and position in the data file, starting at 0. - -Position-based delete files store `file_position_delete`, a struct with the following fields: - -| Field id, name | Type | Description | -|-----------------------------|----------------------------|-------------| -| **`2147483546 file_path`** | `string` | Full URI of a data file with FS scheme. This must match the `file_path` of the target data file in a manifest entry | -| **`2147483545 pos`** | `long` | Ordinal position of a deleted row in the target data file identified by `file_path`, starting at `0` | -| **`2147483544 row`** | `required struct<...>` [1] | Deleted row values. Omit the column when not storing deleted rows. | - -1. When present in the delete file, `row` is required because all delete entries must include the row values. - -When the deleted row column is present, its schema may be any subset of the table schema and must use field ids matching the table. - -To ensure the accuracy of statistics, all delete entries must include row values, or the column must be omitted (this is why the column type is `required`). - -The rows in the delete file must be sorted by `file_path` then `pos` to optimize filtering rows while scanning. - -* Sorting by `file_path` allows filter pushdown by file in columnar storage formats. -* Sorting by `pos` allows filtering rows while scanning, to avoid keeping deletes in memory. - -#### Equality Delete Files - -Equality delete files identify deleted rows in a collection of data files by one or more column values, and may optionally contain additional columns of the deleted row. - -Equality delete files store any subset of a table's columns and use the table's field ids. The _delete columns_ are the columns of the delete file used to match data rows. Delete columns are identified by id in the delete file [metadata column `equality_ids`](#manifests). The column restrictions for columns used in equality delete files are the same as those for [identifier fields](#identifier-field-ids) with the exception that optional columns and columns nested under optional structs are allowed (if a parent struct column is null it implies the leaf column is null). - -A data row is deleted if its values are equal to all delete columns for any row in an equality delete file that applies to the row's data file (see [`Scan Planning`](#scan-planning)). - -Each row of the delete file produces one equality predicate that matches any row where the delete columns are equal. Multiple columns can be thought of as an `AND` of equality predicates. A `null` value in a delete column matches a row if the row's value is `null`, equivalent to `col IS NULL`. - -For example, a table with the following data: - -```text - 1: id | 2: category | 3: name --------|-------------|--------- - 1 | marsupial | Koala - 2 | toy | Teddy - 3 | NULL | Grizzly - 4 | NULL | Polar -``` - -The delete `id = 3` could be written as either of the following equality delete files: - -```text -equality_ids=[1] - - 1: id -------- - 3 -``` - -```text -equality_ids=[1] - - 1: id | 2: category | 3: name --------|-------------|--------- - 3 | NULL | Grizzly -``` - -The delete `id = 4 AND category IS NULL` could be written as the following equality delete file: - -```text -equality_ids=[1, 2] - - 1: id | 2: category | 3: name --------|-------------|--------- - 4 | NULL | Polar -``` - -If a delete column in an equality delete file is later dropped from the table, it must still be used when applying the equality deletes. If a column was added to a table and later used as a delete column in an equality delete file, the column value is read for older data files using normal projection rules (defaults to `null`). - -#### Delete File Stats - -Manifests hold the same statistics for delete files and data files. For delete files, the metrics describe the values that were deleted. - -## Appendix A: Format-specific Requirements - -### Avro - -**Data Type Mappings** - -Values should be stored in Avro using the Avro types and logical type annotations in the table below. - -Optional fields, array elements, and map values must be wrapped in an Avro `union` with `null`. This is the only union type allowed in Iceberg data files. - -Optional fields without an Iceberg default must set the Avro field default value to null. Fields with a non-null Iceberg default must convert the default to an equivalent Avro default. - -Maps with non-string keys must use an array representation with the `map` logical type. The array representation or Avro’s map type may be used for maps with string keys. - -|Type|Avro type|Notes| -|--- |--- |--- | -|**`unknown`**|`null` or omitted|| -|**`boolean`**|`boolean`|| -|**`int`**|`int`|| -|**`long`**|`long`|| -|**`float`**|`float`|| -|**`double`**|`double`|| -|**`decimal(P,S)`**|`{ "type": "fixed",`
      `"size": minBytesRequired(P),`
      `"logicalType": "decimal",`
      `"precision": P,`
      `"scale": S }`|Stored as fixed using the minimum number of bytes for the given precision.| -|**`date`**|`{ "type": "int",`
      `"logicalType": "date" }`|Stores days from 1970-01-01.| -|**`time`**|`{ "type": "long",`
      `"logicalType": "time-micros" }`|Stores microseconds from midnight.| -|**`timestamp`** | `{ "type": "long",`
      `"logicalType": "timestamp-micros",`
      `"adjust-to-utc": false }` | Stores microseconds from 1970-01-01 00:00:00.000000. [1] | -|**`timestamptz`** | `{ "type": "long",`
      `"logicalType": "timestamp-micros",`
      `"adjust-to-utc": true }` | Stores microseconds from 1970-01-01 00:00:00.000000 UTC. [1] | -|**`timestamp_ns`** | `{ "type": "long",`
      `"logicalType": "timestamp-nanos",`
      `"adjust-to-utc": false }` | Stores nanoseconds from 1970-01-01 00:00:00.000000000. [1], [2] | -|**`timestamptz_ns`** | `{ "type": "long",`
      `"logicalType": "timestamp-nanos",`
      `"adjust-to-utc": true }` | Stores nanoseconds from 1970-01-01 00:00:00.000000000 UTC. [1], [2] | -|**`string`**|`string`|| -|**`uuid`**|`{ "type": "fixed",`
      `"size": 16,`
      `"logicalType": "uuid" }`|| -|**`fixed(L)`**|`{ "type": "fixed",`
      `"size": L }`|| -|**`binary`**|`bytes`|| -|**`struct`**|`record`|| -|**`list`**|`array`|| -|**`map`**|`array` of key-value records, or `map` when keys are strings (optional).|Array storage must use logical type name `map` and must store elements that are 2-field records. The first field is a non-null key and the second field is the value.| -|**`variant`**|`record` with `metadata` and `value` fields. `metadata` and `value` must not be assigned field IDs and the fields are accessed through names. |Shredding is not supported in Avro.| -|**`geometry`**|`bytes`|WKB format, see [Appendix G](#appendix-g-geospatial-notes)| -|**`geography`**|`bytes`|WKB format, see [Appendix G](#appendix-g-geospatial-notes)| - -Notes: - -1. Avro type annotation `adjust-to-utc` is an Iceberg convention; default value is `false` if not present. -2. Avro logical type `timestamp-nanos` is an Iceberg convention; the Avro specification does not define this type. - -**Field IDs** - -Iceberg struct, list, and map types identify nested types by ID. When writing data to Avro files, these IDs must be stored in the Avro schema to support ID-based column pruning. - -IDs are stored as JSON integers in the following locations: - -|ID|Avro schema location|Property|Example| -|--- |--- |--- |--- | -|**Struct field**|Record field object|`field-id`|`{ "type": "record", ...`
      `"fields": [`
        `{ "name": "l",`
          `"type": ["null", "long"],`
          `"default": null,`
          `"field-id": 8 }`
      `] }`| -|**List element**|Array schema object|`element-id`|`{ "type": "array",`
      `"items": "int",`
      `"element-id": 9 }`| -|**String map key**|Map schema object|`key-id`|`{ "type": "map",`
      `"values": "int",`
      `"key-id": 10,`
      `"value-id": 11 }`| -|**String map value**|Map schema object|`value-id`|| -|**Map key, value**|Key, value fields in the element record.|`field-id`|`{ "type": "array",`
      `"logicalType": "map",`
      `"items": {`
        `"type": "record",`
        `"name": "k12_v13",`
        `"fields": [`
          `{ "name": "key",`
            `"type": "int",`
            `"field-id": 12 },`
          `{ "name": "value",`
            `"type": "string",`
            `"field-id": 13 }`
        `] } }`| - -Note that the string map case is for maps where the key type is a string. Using Avro’s map type in this case is optional. Maps with string keys may be stored as arrays. - -### Parquet - -**Data Type Mappings** - -Values should be stored in Parquet using the types and logical type annotations in the table below. Column IDs are required to be stored as [field IDs](http://github.com/apache/parquet-format/blob/40699d05bd24181de6b1457babbee2c16dce3803/src/main/thrift/parquet.thrift#L459) on the parquet schema. - -Lists must use the [3-level representation](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists). - -| Type | Parquet physical type | Logical type | Notes | -|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------|----------------------------------------------------------------| -| **`unknown`** | None | | Omit from data files | -| **`boolean`** | `boolean` | | | -| **`int`** | `int32` | | | -| **`long`** | `int64` | | | -| **`float`** | `float` | | | -| **`double`** | `double` | | | -| **`decimal(P,S)`** | `P <= 9`: `int32`,
    `P <= 18`: `int64`,
    `fixed` otherwise | `DECIMAL(P,S)` | Fixed must use the minimum number of bytes that can store `P`. | -| **`date`** | `int32` | `DATE` | Stores days from 1970-01-01. | -| **`time`** | `int64` | `TIME_MICROS` with `adjustToUtc=false` | Stores microseconds from midnight. | -| **`timestamp`** | `int64` | `TIMESTAMP_MICROS` with `adjustToUtc=false` | Stores microseconds from 1970-01-01 00:00:00.000000. | -| **`timestamptz`** | `int64` | `TIMESTAMP_MICROS` with `adjustToUtc=true` | Stores microseconds from 1970-01-01 00:00:00.000000 UTC. | -| **`timestamp_ns`** | `int64` | `TIMESTAMP_NANOS` with `adjustToUtc=false` | Stores nanoseconds from 1970-01-01 00:00:00.000000000. | -| **`timestamptz_ns`** | `int64` | `TIMESTAMP_NANOS` with `adjustToUtc=true` | Stores nanoseconds from 1970-01-01 00:00:00.000000000 UTC. | -| **`string`** | `binary` | `UTF8` | Encoding must be UTF-8. | -| **`uuid`** | `fixed_len_byte_array[16]` | `UUID` | | -| **`fixed(L)`** | `fixed_len_byte_array[L]` | | | -| **`binary`** | `binary` | | | -| **`struct`** | `group` | | | -| **`list`** | `3-level list` | `LIST` | See Parquet docs for 3-level representation. | -| **`map`** | `3-level map` | `MAP` | See Parquet docs for 3-level representation. | -| **`variant`** | `group` with `metadata` and `value` fields. `metadata` and `value` must not be assigned field IDs and the fields are accessed through names. | `VARIANT` | See Parquet docs for [Variant encoding](https://github.com/apache/parquet-format/blob/master/VariantEncoding.md) and [Variant shredding encoding](https://github.com/apache/parquet-format/blob/master/VariantShredding.md). | -| **`geometry`** | `binary` | `GEOMETRY` | WKB format, see [Appendix G](#appendix-g-geospatial-notes). | -| **`geography`** | `binary` | `GEOGRAPHY` | WKB format, see [Appendix G](#appendix-g-geospatial-notes). | - -When reading an `unknown` column, any corresponding column must be ignored and replaced with `null` values. - -### ORC - -**Data Type Mappings** - -| Type | ORC type | ORC type attributes | Notes | -|--------------------|---------------------|------------------------------------------------------|-----------------------------------------------------------------------------------------| -| **`unknown`** | None | | Omit from data files | -| **`boolean`** | `boolean` | | | -| **`int`** | `int` | | ORC `tinyint` and `smallint` would also map to **`int`**. | -| **`long`** | `long` | | | -| **`float`** | `float` | | | -| **`double`** | `double` | | | -| **`decimal(P,S)`** | `decimal` | | | -| **`date`** | `date` | | | -| **`time`** | `long` | `iceberg.long-type`=`TIME` | Stores microseconds from midnight. | -| **`timestamp`** | `timestamp` | `iceberg.timestamp-unit`=`MICROS` | Stores microseconds from 2015-01-01 00:00:00.000000. [1], [2] | -| **`timestamptz`** | `timestamp_instant` | `iceberg.timestamp-unit`=`MICROS` | Stores microseconds from 2015-01-01 00:00:00.000000 UTC. [1], [2] | -| **`timestamp_ns`** | `timestamp` | `iceberg.timestamp-unit`=`NANOS` | Stores nanoseconds from 2015-01-01 00:00:00.000000000. [1] | -| **`timestamptz_ns`** | `timestamp_instant` | `iceberg.timestamp-unit`=`NANOS` | Stores nanoseconds from 2015-01-01 00:00:00.000000000 UTC. [1] | -| **`string`** | `string` | | ORC `varchar` and `char` would also map to **`string`**. | -| **`uuid`** | `binary` | `iceberg.binary-type`=`UUID` | | -| **`fixed(L)`** | `binary` | `iceberg.binary-type`=`FIXED` & `iceberg.length`=`L` | The length would not be checked by the ORC reader and should be checked by the adapter. | -| **`binary`** | `binary` | | | -| **`struct`** | `struct` | | | -| **`list`** | `array` | | | -| **`map`** | `map` | | | -| **`variant`** | `struct` with `metadata` and `value` fields. `metadata` and `value` must not be assigned field IDs. | `iceberg.struct-type`=`VARIANT` | Shredding is not supported in ORC. | -| **`geometry`** | `binary` | `iceberg.binary-type`=`GEOMETRY` | WKB format, see [Appendix G](#appendix-g-geospatial-notes). | -| **`geography`** | `binary` | `iceberg.binary-type`=`GEOGRAPHY` | WKB format, see [Appendix G](#appendix-g-geospatial-notes). | - -Notes: - -1. ORC's [TimestampColumnVector](https://orc.apache.org/api/hive-storage-api/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.html) consists of a time field (milliseconds since epoch) and a nanos field (nanoseconds within the second). Hence the milliseconds within the second are reported twice; once in the time field and again in the nanos field. The read adapter should only use milliseconds within the second from one of these fields. The write adapter should also report milliseconds within the second twice; once in the time field and again in the nanos field. ORC writer is expected to correctly consider millis information from one of the fields. More details at https://issues.apache.org/jira/browse/ORC-546 -2. ORC `timestamp` and `timestamp_instant` values store nanosecond precision. Iceberg ORC writers for Iceberg types `timestamp` and `timestamptz` **must** truncate nanoseconds to microseconds. `iceberg.timestamp-unit` is assumed to be `MICROS` if not present. - -One of the interesting challenges with this is how to map Iceberg’s schema evolution (id based) on to ORC’s (name based). In theory, we could use Iceberg’s column ids as the column and field names, but that would be inconvenient. - -The column IDs must be stored in ORC type attributes using the key `iceberg.id`, and `iceberg.required` to store `"true"` if the Iceberg column is required, otherwise it will be optional. - -Iceberg would build the desired reader schema with their schema evolution rules and pass that down to the ORC reader, which would then use its schema evolution to map that to the writer’s schema. Basically, Iceberg would need to change the names of columns and fields to get the desired mapping. - -|Iceberg writer|ORC writer|Iceberg reader|ORC reader| -|--- |--- |--- |--- | -|`struct    `|`struct`|`struct`|`struct`| -|`struct>`|`struct>`|`struct>`|`struct>`| - -## Appendix B: 32-bit Hash Requirements - -The 32-bit hash implementation is 32-bit Murmur3 hash, x86 variant, seeded with 0. - -| Primitive type | Hash specification | Test value | -|--------------------|-------------------------------------------|--------------------------------------------| -| **`int`** | `hashLong(long(v))` [1] | `34` ￫ `2017239379` | -| **`long`** | `hashBytes(littleEndianBytes(v))` | `34L` ￫ `2017239379` | -| **`decimal(P,S)`** | `hashBytes(minBigEndian(unscaled(v)))`[2] | `14.20` ￫ `-500754589` | -| **`date`** | `hashInt(daysFromUnixEpoch(v))` | `2017-11-16` ￫ `-653330422` | -| **`time`** | `hashLong(microsecsFromMidnight(v))` | `22:31:08` ￫ `-662762989` | -| **`timestamp`** | `hashLong(microsecsFromUnixEpoch(v))` | `2017-11-16T22:31:08` ￫ `-2047944441`
    `2017-11-16T22:31:08.000001` ￫ `-1207196810` | -| **`timestamptz`** | `hashLong(microsecsFromUnixEpoch(v))` | `2017-11-16T14:31:08-08:00` ￫ `-2047944441`
    `2017-11-16T14:31:08.000001-08:00` ￫ `-1207196810` | -| **`timestamp_ns`** | `hashLong(microsecsFromUnixEpoch(v))` [3] | `2017-11-16T22:31:08` ￫ `-2047944441`
    `2017-11-16T22:31:08.000001001` ￫ `-1207196810` | -| **`timestamptz_ns`** | `hashLong(microsecsFromUnixEpoch(v))` [3]| `2017-11-16T14:31:08-08:00` ￫ `-2047944441`
    `2017-11-16T14:31:08.000001001-08:00` ￫ `-1207196810` | -| **`string`** | `hashBytes(utf8Bytes(v))` | `iceberg` ￫ `1210000089` | -| **`uuid`** | `hashBytes(uuidBytes(v))` [4] | `f79c3e09-677c-4bbd-a479-3f349cb785e7` ￫ `1488055340` | -| **`fixed(L)`** | `hashBytes(v)` | `00 01 02 03` ￫ `-188683207` | -| **`binary`** | `hashBytes(v)` | `00 01 02 03` ￫ `-188683207` | - -The types below are not currently valid for bucketing, and so are not hashed. However, if that changes and a hash value is needed, the following table shall apply: - -| Primitive type | Hash specification | Test value | -|--------------------|-------------------------------------------|--------------------------------------------| -| **`unknown`** | always `null` | | -| **`boolean`** | `false: hashInt(0)`, `true: hashInt(1)` | `true` ￫ `1392991556` | -| **`float`** | `hashLong(doubleToLongBits(double(v))` [5]| `1.0F` ￫ `-142385009`, `0.0F` ￫ `1669671676`, `-0.0F` ￫ `1669671676` | -| **`double`** | `hashLong(doubleToLongBits(v))` [5]| `1.0D` ￫ `-142385009`, `0.0D` ￫ `1669671676`, `-0.0D` ￫ `1669671676` | - -A 32-bit hash is not defined for `variant` because there are multiple representations for equivalent values. - -Notes: - -1. Integer and long hash results must be identical for all integer values. This ensures that schema evolution does not change bucket partition values if integer types are promoted. -2. Decimal values are hashed using the minimum number of bytes required to hold the unscaled value as a two’s complement big-endian; this representation does not include padding bytes required for storage in a fixed-length array. -Hash results are not dependent on decimal scale, which is part of the type, not the data value. -3. Nanosecond timestamps must be converted to microsecond precision before hashing to ensure timestamps have the same hash value. -4. UUIDs are encoded using big endian. The test UUID for the example above is: `f79c3e09-677c-4bbd-a479-3f349cb785e7`. This UUID encoded as a byte array is: -`F7 9C 3E 09 67 7C 4B BD A4 79 3F 34 9C B7 85 E7` -5. `doubleToLongBits` must give the IEEE 754 compliant bit representation of the double value. All `NaN` bit patterns must be canonicalized to `0x7ff8000000000000L`. Negative zero (`-0.0`) must be canonicalized to positive zero (`0.0`). Float hash values are the result of hashing the float cast to double to ensure that schema evolution does not change hash values if float types are promoted. - -## Appendix C: JSON serialization - -### Schemas - -Schemas are serialized as a JSON object with the same fields as a struct in the table below, and the following additional fields: - -| v1 | v2 |Field|JSON representation|Example| -| ---------- | ---------- |--- |--- |--- | -| _optional_ | _required_ |**`schema-id`**|`JSON int`|`0`| -| _optional_ | _optional_ |**`identifier-field-ids`**|`JSON list of ints`|`[1, 2]`| - -Types are serialized according to this table: - -|Type|JSON representation|Example| -|--- |--- |--- | -|**`unknown`**|`JSON string: "unknown"`|`"unknown"`| -|**`boolean`**|`JSON string: "boolean"`|`"boolean"`| -|**`int`**|`JSON string: "int"`|`"int"`| -|**`long`**|`JSON string: "long"`|`"long"`| -|**`float`**|`JSON string: "float"`|`"float"`| -|**`double`**|`JSON string: "double"`|`"double"`| -|**`date`**|`JSON string: "date"`|`"date"`| -|**`time`**|`JSON string: "time"`|`"time"`| -|**`timestamp, microseconds, without zone`**|`JSON string: "timestamp"`|`"timestamp"`| -|**`timestamp, microseconds, with zone`**|`JSON string: "timestamptz"`|`"timestamptz"`| -|**`timestamp, nanoseconds, without zone`**|`JSON string: "timestamp_ns"`|`"timestamp_ns"`| -|**`timestamp, nanoseconds, with zone`**|`JSON string: "timestamptz_ns"`|`"timestamptz_ns"`| -|**`string`**|`JSON string: "string"`|`"string"`| -|**`uuid`**|`JSON string: "uuid"`|`"uuid"`| -|**`fixed(L)`**|`JSON string: "fixed[]"`|`"fixed[16]"`| -|**`binary`**|`JSON string: "binary"`|`"binary"`| -|**`decimal(P, S)`**|`JSON string: "decimal(

    ,)"`|`"decimal(9,2)"`,
    `"decimal(9, 2)"`| -|**`struct`**|`JSON object: {`
      `"type": "struct",`
      `"fields": [ {`
        `"id": ,`
        `"name": ,`
        `"required": ,`
        `"type": ,`
        `"doc": ,`
        `"initial-default": ,`
        `"write-default": `
        `}, ...`
      `] }`|`{`
      `"type": "struct",`
      `"fields": [ {`
        `"id": 1,`
        `"name": "id",`
        `"required": true,`
        `"type": "uuid",`
        `"initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",`
        `"write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"`
      `}, {`
        `"id": 2,`
        `"name": "data",`
        `"required": false,`
        `"type": {`
          `"type": "list",`
          `...`
        `}`
      `} ]`
    `}`| -|**`list`**|`JSON object: {`
      `"type": "list",`
      `"element-id": ,`
      `"element-required": `
      `"element": `
    `}`|`{`
      `"type": "list",`
      `"element-id": 3,`
      `"element-required": true,`
      `"element": "string"`
    `}`| -|**`map`**|`JSON object: {`
      `"type": "map",`
      `"key-id": ,`
      `"key": ,`
      `"value-id": ,`
      `"value-required": `
      `"value": `
    `}`|`{`
      `"type": "map",`
      `"key-id": 4,`
      `"key": "string",`
      `"value-id": 5,`
      `"value-required": false,`
      `"value": "double"`
    `}`| -| **`variant`**| `JSON string: "variant"`|`"variant"`| -| **`geometry(C)`** |`JSON string: "geometry()"`|`"geometry(srid:4326)"`| -| **`geography(C, A)`** |`JSON string: "geography(,)"`|`"geography(srid:4326,spherical)"`| - -Note that default values are serialized using the JSON single-value serialization in [Appendix D](#appendix-d-single-value-serialization). - -### Partition Specs - -Partition specs are serialized as a JSON object with the following fields: - -|Field|JSON representation|Example| -|--- |--- |--- | -|**`spec-id`**|`JSON int`|`0`| -|**`fields`**|`JSON list: [`
      `,`
      `...`
    `]`|`[ {`
      `"source-id": 4,`
      `"field-id": 1000,`
      `"name": "ts_day",`
      `"transform": "day"`
    `}, {`
      `"source-id": 1,`
      `"field-id": 1001,`
      `"name": "id_bucket",`
      `"transform": "bucket[16]"`
    `} ]`| - -Each partition field in `fields` is stored as a JSON object with the following properties. - -| V1 | V2 | V3 | Field | JSON representation | Example | -|----------|----------|----------|------------------|---------------------|--------------| -| required | required | optional | **`source-id`** | `JSON int` | 1 | -| | | optional | **`source-ids`** | `JSON list of ints` | `[1,2]` | -| | required | required | **`field-id`** | `JSON int` | 1000 | -| required | required | required | **`name`** | `JSON string` | `id_bucket` | -| required | required | required | **`transform`** | `JSON string` | `bucket[16]` | - -Notes: - -1. For partition fields with a transform with a single argument, only `source-id` is written. In case of a multi-argument transform, only `source-ids` is written. - -Supported partition transforms are listed below. - -|Transform or Field|JSON representation|Example| -|--- |--- |--- | -|**`identity`**|`JSON string: "identity"`|`"identity"`| -|**`bucket[N]`**|`JSON string: "bucket[]"`|`"bucket[16]"`| -|**`truncate[W]`**|`JSON string: "truncate[]"`|`"truncate[20]"`| -|**`year`**|`JSON string: "year"`|`"year"`| -|**`month`**|`JSON string: "month"`|`"month"`| -|**`day`**|`JSON string: "day"`|`"day"`| -|**`hour`**|`JSON string: "hour"`|`"hour"`| - -In some cases partition specs are stored using only the field list instead of the object format that includes the spec ID, like the deprecated `partition-spec` field in table metadata. The object format should be used unless otherwise noted in this spec. - -The `field-id` property was added for each partition field in v2. In v1, the reference implementation assigned field ids sequentially in each spec starting at 1,000. See Partition Evolution for more details. - -Older versions of the reference implementation can read tables with transforms unknown to it, ignoring them. But other implementations may break if they encounter unknown transforms. All v3 readers are required to read tables with unknown transforms, ignoring them. Writers should not write using partition specs that use unknown transforms. - -### Sort Orders - -Sort orders are serialized as a list of JSON objects, each of which contains the following fields: - -|Field|JSON representation|Example| -|--- |--- |--- | -|**`order-id`**|`JSON int`|`1`| -|**`fields`**|`JSON list: [`
      `,`
      `...`
    `]`|`[ {`
      `"transform": "identity",`
      `"source-id": 2,`
      `"direction": "asc",`
      `"null-order": "nulls-first"`
      `}, {`
      `"transform": "bucket[4]",`
      `"source-id": 3,`
      `"direction": "desc",`
      `"null-order": "nulls-last"`
    `} ]`| - -Each sort field in the fields list is stored as an object with the following properties: - -| V1 | V2 | V3 | Field | JSON representation | Example | -|----------|----------|----------|------------------|---------------------|-------------| -| required | required | required | **`transform`** | `JSON string` | `bucket[4]` | -| required | required | optional | **`source-id`** | `JSON int` | 1 | -| | | optional | **`source-ids`** | `JSON list of ints` | `[1,2]` | -| required | required | required | **`direction`** | `JSON string` | `asc` | -| required | required | required | **`null-order`** | `JSON string` | `nulls-last`| - -Notes: - -1. For sort fields with a transform with a single argument, only `source-id` is written. In case of a multi-argument transform, only `source-ids` is written. - -Older versions of the reference implementation can read tables with transforms unknown to it, ignoring them. But other implementations may break if they encounter unknown transforms. All v3 readers are required to read tables with unknown transforms, ignoring them. - -The following table describes the possible values for the some of the field within sort field: - -|Field|JSON representation|Possible values| -|--- |--- |--- | -|**`direction`**|`JSON string`|`"asc", "desc"`| -|**`null-order`**|`JSON string`|`"nulls-first", "nulls-last"`| - -### Table Metadata and Snapshots - -Table metadata is serialized as a JSON object according to the following table. Snapshots are not serialized separately. Instead, they are stored in the table metadata JSON. - -A metadata JSON file may be compressed with [GZIP](https://datatracker.ietf.org/doc/html/rfc1952). - -|Metadata field|JSON representation|Example| -|--- |--- |--- | -|**`format-version`**|`JSON int`|`1`| -|**`table-uuid`**|`JSON string`|`"fb072c92-a02b-11e9-ae9c-1bb7bc9eca94"`| -|**`location`**|`JSON string`|`"s3://b/wh/data.db/table"`| -|**`last-updated-ms`**|`JSON long`|`1515100955770`| -|**`last-column-id`**|`JSON int`|`22`| -|**`schema`**|`JSON schema (object)`|`See above, read schemas instead`| -|**`schemas`**|`JSON schemas (list of objects)`|`See above`| -|**`current-schema-id`**|`JSON int`|`0`| -|**`partition-spec`**|`JSON partition fields (list)`|`See above, read partition-specs instead`| -|**`partition-specs`**|`JSON partition specs (list of objects)`|`See above`| -|**`default-spec-id`**|`JSON int`|`0`| -|**`last-partition-id`**|`JSON int`|`1000`| -|**`properties`**|`JSON object: {`
      `"": "",`
      `...`
    `}`|`{`
      `"write.format.default": "avro",`
      `"commit.retry.num-retries": "4"`
    `}`| -|**`current-snapshot-id`**|`JSON long`|`3051729675574597004`| -|**`snapshots`**|`JSON list of objects: [ {`
      `"snapshot-id": ,`
      `"timestamp-ms": ,`
      `"summary": {`
        `"operation": ,`
        `... },`
      `"manifest-list": "",`
      `"schema-id": ""`
      `},`
      `...`
    `]`|`[ {`
      `"snapshot-id": 3051729675574597004,`
      `"timestamp-ms": 1515100955770,`
      `"summary": {`
        `"operation": "append"`
      `},`
      `"manifest-list": "s3://b/wh/.../s1.avro"`
      `"schema-id": 0`
    `} ]`| -|**`snapshot-log`**|`JSON list of objects: [`
      `{`
      `"snapshot-id": ,`
      `"timestamp-ms":`
      `},`
      `...`
    `]`|`[ {`
      `"snapshot-id": 30517296...,`
      `"timestamp-ms": 1515100...`
    `} ]`| -|**`metadata-log`**|`JSON list of objects: [`
      `{`
      `"metadata-file": ,`
      `"timestamp-ms":`
      `},`
      `...`
    `]`|`[ {`
      `"metadata-file": "s3://bucket/.../v1.json",`
      `"timestamp-ms": 1515100...`
    `} ]` | -|**`sort-orders`**|`JSON sort orders (list of sort field object)`|`See above`| -|**`default-sort-order-id`**|`JSON int`|`0`| -|**`refs`**|`JSON map with string key and object value:`
    `{`
      `"": {`
      `"snapshot-id": ,`
      `"type": ,`
      `"max-ref-age-ms": ,`
      `...`
      `}`
      `...`
    `}`|`{`
      `"test": {`
      `"snapshot-id": 123456789000,`
      `"type": "tag",`
      `"max-ref-age-ms": 10000000`
      `}`
    `}`| -|**`encryption-keys`**|`JSON list of encryption key objects`|`[ {"key-id": "5f819b", "key-metadata": "aWNlYmVyZwo="} ]`| - -### Name Mapping Serialization - -Name mapping is serialized as a list of field mapping JSON Objects which are serialized as follows - -|Field mapping field|JSON representation|Example| -|--- |--- |--- | -|**`names`**|`JSON list of strings`|`["latitude", "lat"]`| -|**`field-id`**|`JSON int`|`1`| -|**`fields`**|`JSON field mappings (list of objects)`|`[{`
      `"field-id": 4,`
      `"names": ["latitude", "lat"]`
    `}, {`
      `"field-id": 5,`
      `"names": ["longitude", "long"]`
    `}]`| - -Example -```json -[ { "field-id": 1, "names": ["id", "record_id"] }, - { "field-id": 2, "names": ["data"] }, - { "field-id": 3, "names": ["location"], "fields": [ - { "field-id": 4, "names": ["latitude", "lat"] }, - { "field-id": 5, "names": ["longitude", "long"] } - ] } ] -``` - -## Appendix D: Single-value serialization - -### Binary single-value serialization - -This serialization scheme is for storing single values as individual binary values. - -| Type | Binary serialization | -|------------------------------|--------------------------------------------------------------------------------------------------------------| -| **`unknown`** | Not supported | -| **`boolean`** | `0x00` for false, non-zero byte for true | -| **`int`** | Stored as 4-byte little-endian | -| **`long`** | Stored as 8-byte little-endian | -| **`float`** | Stored as 4-byte little-endian | -| **`double`** | Stored as 8-byte little-endian | -| **`date`** | Stores days from the 1970-01-01 in an 4-byte little-endian int | -| **`time`** | Stores microseconds from midnight in an 8-byte little-endian long | -| **`timestamp`** | Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian long | -| **`timestamptz`** | Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte little-endian long | -| **`timestamp_ns`** | Stores nanoseconds from 1970-01-01 00:00:00.000000000 in an 8-byte little-endian long | -| **`timestamptz_ns`** | Stores nanoseconds from 1970-01-01 00:00:00.000000000 UTC in an 8-byte little-endian long | -| **`string`** | UTF-8 bytes (without length) | -| **`uuid`** | 16-byte big-endian value, see example in Appendix B | -| **`fixed(L)`** | Binary value | -| **`binary`** | Binary value (without length) | -| **`decimal(P, S)`** | Stores unscaled value as two’s-complement big-endian binary, using the minimum number of bytes for the value | -| **`struct`** | Not supported | -| **`list`** | Not supported | -| **`map`** | Not supported | -| **`variant`** | Not supported | -| **`geometry`** | WKB format, see [Appendix G](#appendix-g-geospatial-notes) | -| **`geography`** | WKB format, see [Appendix G](#appendix-g-geospatial-notes) | - -### Bound serialization - -The binary single-value serialization can be used to store the lower and upper bounds maps of manifest files, except as specified by the following table. - -| Type | Binary serialization | -|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **`geometry`** | A single point, encoded as a x:y:z:m concatenation of its 8-byte little-endian IEEE 754 coordinate values. x and y are mandatory. This becomes x:y if z and m are both unset, x:y:z if only m is unset, and x:y:NaN:m if only z is unset. | -| **`geography`** | A single point, encoded as a x:y:z:m concatenation of its 8-byte little-endian IEEE 754 coordinate values. x and y are mandatory. This becomes x:y if z and m are both unset, x:y:z if only m is unset, and x:y:NaN:m if only z is unset. | -| **`variant`** | A serialized Variant of encoded v1 metadata concatenated with an encoded variant object. Object keys are normalized JSON paths to identify fields; values are lower or upper bound values. | - -### JSON single-value serialization - - Single values are serialized as JSON by type according to the following table: - -| Type | JSON representation | Example | Description | -| ------------------ | ----------------------------------------- | ------------------------------------------ | -- | -| **`boolean`** | **`JSON boolean`** | `true` | | -| **`int`** | **`JSON int`** | `34` | | -| **`long`** | **`JSON long`** | `34` | | -| **`float`** | **`JSON number`** | `1.0` | | -| **`double`** | **`JSON number`** | `1.0` | | -| **`decimal(P,S)`** | **`JSON string`** | `"14.20"`, `"2E+20"` | Stores the string representation of the decimal value, specifically, for values with a positive scale, the number of digits to the right of the decimal point is used to indicate scale, for values with a negative scale, the scientific notation is used and the exponent must equal the negated scale | -| **`date`** | **`JSON string`** | `"2017-11-16"` | Stores ISO-8601 standard date | -| **`time`** | **`JSON string`** | `"22:31:08.123456"` | Stores ISO-8601 standard time with microsecond precision | -| **`timestamp`** | **`JSON string`** | `"2017-11-16T22:31:08.123456"` | Stores ISO-8601 standard timestamp with microsecond precision; must not include a zone offset | -| **`timestamptz`** | **`JSON string`** | `"2017-11-16T22:31:08.123456+00:00"` | Stores ISO-8601 standard timestamp with microsecond precision; must include a zone offset and it must be '+00:00' | -| **`timestamp_ns`** | **`JSON string`** | `"2017-11-16T22:31:08.123456789"` | Stores ISO-8601 standard timestamp with nanosecond precision; must not include a zone offset | -| **`timestamptz_ns`** | **`JSON string`** | `"2017-11-16T22:31:08.123456789+00:00"` | Stores ISO-8601 standard timestamp with nanosecond precision; must include a zone offset and it must be '+00:00' | -| **`string`** | **`JSON string`** | `"iceberg"` | | -| **`uuid`** | **`JSON string`** | `"f79c3e09-677c-4bbd-a479-3f349cb785e7"` | Stores the lowercase uuid string | -| **`fixed(L)`** | **`JSON string`** | `"000102ff"` | Stored as a hexadecimal string | -| **`binary`** | **`JSON string`** | `"000102ff"` | Stored as a hexadecimal string | -| **`struct`** | **`JSON object by field ID`** | `{"1": 1, "2": "bar"}` | Stores struct fields using the field ID as the JSON field name; field values are stored using this JSON single-value format | -| **`list`** | **`JSON array of values`** | `[1, 2, 3]` | Stores a JSON array of values that are serialized using this JSON single-value format | -| **`map`** | **`JSON object of key and value arrays`** | `{ "keys": ["a", "b"], "values": [1, 2] }` | Stores arrays of keys and values; individual keys and values are serialized using this JSON single-value format | -| **`geometry`** | **`JSON string`** | `POINT (30 10)` | Stored using WKT representation, see [Appendix G](#appendix-g-geospatial-notes) | -| **`geography`** | **`JSON string`** | `POINT (30 10)` | Stored using WKT representation, see [Appendix G](#appendix-g-geospatial-notes) | - -## Appendix E: Format version changes - -### Version 3 - -Default values are added to struct fields in v3. - -* The `write-default` is a forward-compatible change because it is only used at write time. Old writers will fail because the field is missing. -* Tables with `initial-default` will be read correctly by older readers if `initial-default` is always null for optional fields. Otherwise, old readers will default optional columns with null. Old readers will fail to read required fields which are populated by `initial-default` because that default is not supported. - -Types `variant`, `geometry`, `geography`, `unknown`, `timestamp_ns`, and `timestamptz_ns` are added in v3. - -All readers are required to read tables with unknown partition transforms, ignoring the unsupported partition fields when filtering. - -Writing v3 metadata: - -* Partition Field and Sort Field JSON: - * `source-ids` was added and must be written in the case of a multi-argument transform. - * `source-id` must be written in the case of single-argument transforms. - -Row-level delete changes: - -* Deletion vectors are added in v3, stored using the Puffin `deletion-vector-v1` blob type -* Manifests are updated to track deletion vectors: - * `referenced_data_file` was added and can be used for both deletion vectors (required) and v2 position delete files that contain deletes for only one data file (optional) - * `content_offset` was added and must match the deletion vector blob's offset in a Puffin file - * `content_size_in_bytes` was added and must match the deletion vector blob's length in a Puffin file -* Deletion vectors are maintained synchronously: Writers must merge DVs (and older position delete files) to ensure there is at most one DV per data file - * Readers can safely ignore position delete files if there is a DV for a data file -* Writers are not allowed to add new position delete files to v3 tables -* Existing position delete files are valid in tables that have been upgraded from v2 - * These position delete files must be merged into the DV for a data file when one is created - * Position delete files that contain deletes for more than one data file need to be kept in table metadata until all deletes are replaced by DVs - -Row lineage changes: - -* Writers must set the table's `next-row-id` and use the existing `next-row-id` as the `first-row-id` when creating new snapshots - * When a table is upgraded to v3, `next_row_id` should be initialized to 0 - * When committing a new snapshot `next-row-id` must be incremented by at least the number of newly assigned row ids in the snapshot - * It is recommended to increment `next-row-id` by the total `added_rows_count` and `existing_rows_count` of all manifests assigned a `first_row_id` -* Writers must assign a `first_row_id` to new data manifests when writing a manifest list - * When writing a new manifest list each `first_row_id` must be incremented by at least the number of newly assigned row ids in the manifest - * It is recommended to increment `first_row_id` by a manifest's `added_rows_count` and `existing_rows_count` -* When writing a manifest, new data files must be written with a null `first_row_id` so that the value is assigned at read time based on the manifest's `first_row_id` -* When a manifest has a non-null `first_row_id`, readers must assign a `first_row_id` to any data file that has a missing or null value in that manifest - * Readers must increment `first_row_id` by the data file's `record_count` -* When writing an existing data file into a new manifest, its `first_row_id` must be written into the manifest -* When a data file has a non-null `first_row_id`, readers must: - * Replace any null or missing `_row_id` with the data file's `first_row_id` plus the row's `_pos` - * Replace any null or missing `_last_updated_sequence_number` to the data file's `data_sequence_number` - * Read any non-null `_row_id` or `_last_updated_sequence_number` without modification -* When a data file has a null `first_row_id`, readers must produce null for `_row_id` and `_last_updated_sequence_number` -* When writing an existing row into a new data file, writers must write `_row_id` and `_last_updated_sequence_number` if they are non-null - -Encryption changes: - -* Encryption keys are tracked by table metadata `encryption-keys` -* The encryption key used for a snapshot is specified by `key-id` - -### Version 2 - -Writing v1 metadata: - -* Table metadata field `last-sequence-number` should not be written -* Snapshot field `sequence-number` should not be written -* Manifest list field `sequence-number` should not be written -* Manifest list field `min-sequence-number` should not be written -* Manifest list field `content` must be 0 (data) or omitted -* Manifest entry field `sequence_number` should not be written -* Manifest entry field `file_sequence_number` should not be written -* Data file field `content` must be 0 (data) or omitted - -Reading v1 metadata for v2: - -* Table metadata field `last-sequence-number` must default to 0 -* Snapshot field `sequence-number` must default to 0 -* Manifest list field `sequence-number` must default to 0 -* Manifest list field `min-sequence-number` must default to 0 -* Manifest list field `content` must default to 0 (data) -* Manifest entry field `sequence_number` must default to 0 -* Manifest entry field `file_sequence_number` must default to 0 -* Data file field `content` must default to 0 (data) - -Writing v2 metadata: - -* Table metadata JSON: - * `last-sequence-number` was added and is required; default to 0 when reading v1 metadata - * `table-uuid` is now required - * `current-schema-id` is now required - * `schemas` is now required - * `partition-specs` is now required - * `default-spec-id` is now required - * `last-partition-id` is now required - * `sort-orders` is now required - * `default-sort-order-id` is now required - * `schema` is no longer required and should be omitted; use `schemas` and `current-schema-id` instead - * `partition-spec` is no longer required and should be omitted; use `partition-specs` and `default-spec-id` instead -* Snapshot JSON: - * `sequence-number` was added and is required; default to 0 when reading v1 metadata - * `manifest-list` is now required - * `manifests` is no longer required and should be omitted; always use `manifest-list` instead -* Manifest list `manifest_file`: - * `content` was added and is required; 0=data, 1=deletes; default to 0 when reading v1 manifest lists - * `sequence_number` was added and is required - * `min_sequence_number` was added and is required - * `added_files_count` is now required - * `existing_files_count` is now required - * `deleted_files_count` is now required - * `added_rows_count` is now required - * `existing_rows_count` is now required - * `deleted_rows_count` is now required -* Manifest key-value metadata: - * `schema-id` is now required - * `partition-spec-id` is now required - * `format-version` is now required - * `content` was added and is required (must be "data" or "deletes") -* Manifest `manifest_entry`: - * `snapshot_id` is now optional to support inheritance - * `sequence_number` was added and is optional, to support inheritance - * `file_sequence_number` was added and is optional, to support inheritance -* Manifest `data_file`: - * `content` was added and is required; 0=data, 1=position deletes, 2=equality deletes; default to 0 when reading v1 manifests - * `equality_ids` was added, to be used for equality deletes only - * `block_size_in_bytes` was removed (breaks v1 reader compatibility) - * `file_ordinal` was removed - * `sort_columns` was removed - -Note that these requirements apply when writing data to a v2 table. Tables that are upgraded from v1 may contain metadata that does not follow these requirements. Implementations should remain backward-compatible with v1 metadata requirements. - -## Appendix F: Implementation Notes - -This section covers topics not required by the specification but recommendations for systems implementing the Iceberg specification to help maintain a uniform experience. - -### Point in Time Reads (Time Travel) - -Iceberg supports two types of histories for tables. A history of previous "current snapshots" stored in ["snapshot-log" table metadata](#table-metadata-fields) and [parent-child lineage stored in "snapshots"](#table-metadata-fields). These two histories -might indicate different snapshot IDs for a specific timestamp. The discrepancies can be caused by a variety of table operations (e.g. updating the `current-snapshot-id` can be used to set the snapshot of a table to any arbitrary snapshot, which might have a lineage derived from a table branch or no lineage at all). - -When processing point in time queries implementations should use "snapshot-log" metadata to lookup the table state at the given point in time. This ensures time-travel queries reflect the state of the table at the provided timestamp. For example a SQL query like `SELECT * FROM prod.db.table TIMESTAMP AS OF '1986-10-26 01:21:00Z';` would find the snapshot of the Iceberg table just prior to '1986-10-26 01:21:00 UTC' in the snapshot logs and use the metadata from that snapshot to perform the scan of the table. If no snapshot exists prior to the timestamp given or "snapshot-log" is not populated (it is an optional field), then systems should raise an informative error message about the missing metadata. - -### Optional Snapshot Summary Fields - -Snapshot summary can include metrics fields to track numeric stats of the snapshot (see [Metrics](#metrics)) and operational details (see [Other Fields](#other-fields)). The value of these fields should be of string type (e.g., `"120"`). - -#### Metrics - -| Field | Description | -|-------------------------------------|--------------------------------------------------------------------------------------------------| -| **`added-data-files`** | Number of data files added in the snapshot | -| **`deleted-data-files`** | Number of data files deleted in the snapshot | -| **`total-data-files`** | Total number of live data files in the snapshot | -| **`added-delete-files`** | Number of positional/equality delete files and deletion vectors added in the snapshot | -| **`added-equality-delete-files`** | Number of equality delete files added in the snapshot | -| **`removed-equality-delete-files`** | Number of equality delete files removed in the snapshot | -| **`added-position-delete-files`** | Number of position delete files added in the snapshot | -| **`removed-position-delete-files`** | Number of position delete files removed in the snapshot | -| **`added-dvs`** | Number of deletion vectors added in the snapshot | -| **`removed-dvs`** | Number of deletion vectors removed in the snapshot | -| **`removed-delete-files`** | Number of positional/equality delete files and deletion vectors removed in the snapshot | -| **`total-delete-files`** | Total number of live positional/equality delete files and deletion vectors in the snapshot | -| **`added-records`** | Number of records added in the snapshot | -| **`deleted-records`** | Number of records deleted in the snapshot | -| **`total-records`** | Total number of records in the snapshot | -| **`added-files-size`** | The size of files added in the snapshot | -| **`removed-files-size`** | The size of files removed in the snapshot | -| **`total-files-size`** | Total size of live files in the snapshot | -| **`added-position-deletes`** | Number of position delete records added in the snapshot | -| **`removed-position-deletes`** | Number of position delete records removed in the snapshot | -| **`total-position-deletes`** | Total number of position delete records in the snapshot | -| **`added-equality-deletes`** | Number of equality delete records added in the snapshot | -| **`removed-equality-deletes`** | Number of equality delete records removed in the snapshot | -| **`total-equality-deletes`** | Total number of equality delete records in the snapshot | -| **`deleted-duplicate-files`** | Number of duplicate files deleted (duplicates are files recorded more than once in the manifest) | -| **`changed-partition-count`** | Number of partitions with files added or removed in the snapshot | -| **`manifests-created`** | Number of manifest files created in the snapshot | -| **`manifests-kept`** | Number of manifest files kept in the snapshot | -| **`manifests-replaced`** | Number of manifest files replaced in the snapshot | -| **`entries-processed`** | Number of manifest entries processed in the snapshot | - -#### Other Fields - -| Field | Example | Description | -|--------------------------|------------|-----------------------------------------------------------------| -| **`wap.id`** | "12345678" | The Write-Audit-Publish id of a staged snapshot | -| **`published-wap-id`** | "12345678" | The Write-Audit-Publish id of a snapshot already been published | -| **`source-snapshot-id`** | "12345678" | The original id of a cherry-picked snapshot | -| **`engine-name`** | "spark" | Name of the engine that created the snapshot | -| **`engine-version`** | "3.5.4" | Version of the engine that created the snapshot | - -### Assignment of Snapshot IDs and `current-snapshot-id` - -Writers should produce positive values for snapshot ids in a manner that minimizes the probability of id collisions and should verify the id does not conflict with existing snapshots. Producing snapshot ids based on timestamps alone is not recommended as it increases the potential for collisions. - -The reference Java implementation uses a type 4 uuid and XORs the 8 most significant bytes with the 8 least significant bytes then ANDs with the maximum long value to arrive at a pseudo-random snapshot id with a low probability of collision. - -Java writes `-1` for "no current snapshot" with V1 and V2 tables and considers this equivalent to omitted or `null`. This has never been formalized in the spec, but for compatibility, other implementations can accept `-1` as `null`. Java will no longer write `-1` and will use `null` for "no current snapshot" for all tables with a version greater than or equal to V3. - -### Naming for GZIP compressed Metadata JSON files - -Some implementations require that GZIP compressed files have the suffix `.gz.metadata.json` to be read correctly. The Java reference implementation can additionally read GZIP compressed files with the suffix `metadata.json.gz`. - -### Position Delete Files with Row Data - -Although the spec allows for including the deleted row itself (in addition to the path and position of the row in the data file) in v2 position delete files, writing the row is optional and no implementation currently writes it. The ability to write and read the row is supported in the Java implementation but is deprecated in version 1.11.0. - -## Appendix G: Geospatial Notes - -The Geometry and Geography class hierarchy and its Well-known text (WKT) and Well-known binary (WKB) serializations (ISO supporting XY, XYZ, XYM, XYZM) are defined by [OpenGIS Implementation Specification for Geographic information – Simple feature access – Part 1: Common architecture](https://portal.ogc.org/files/?artifact_id=25355), from [OGC (Open Geospatial Consortium)](https://www.ogc.org/standard/sfa/). - -Points are always defined by the coordinates X, Y, Z (optional), and M (optional), in this order. X is the longitude/easting, Y is the latitude/northing, and Z is usually the height, or elevation. M is a fourth optional dimension, for example a linear reference value (e.g., highway milepost value), a timestamp, or some other value as defined by the CRS. - -The version of the OGC standard first used here is 1.2.1, but future versions may also be used if the WKB representation remains wire-compatible. diff --git a/doc/design/access-server/s3/design-crowdb-access-s3-rdma.md b/doc/design/access-server/s3/design-crowdb-access-s3-rdma.md deleted file mode 100644 index 6c047f69e..000000000 --- a/doc/design/access-server/s3/design-crowdb-access-s3-rdma.md +++ /dev/null @@ -1,67 +0,0 @@ - - - -# CROWDB - Design: Native S3 and Dataset RDMA Access - -Native S3 and Dataset clients use CROWDB's shared RPC transport for direct -storage access. The HTTP S3 endpoint retains its protocol and authorization -contract; it does not negotiate an alternate cuObject data plane. - -Depends on: [Access Server](../design-crowdb-access-server.md), -[S3](design-crowdb-access-s3.md), -[Dataset](../dataset/design-crowdb-access-dataset.md), and -[RPC RDMA](../../rpc/design-crowdb-rpc-rdma.md). - -Satisfies: bounded native storage access without an Access Server payload bounce. - -## Table of contents - -1. [Access paths](#1-access-paths) -2. [Transport and buffers](#2-transport-and-buffers) -3. [Correctness and scope](#3-correctness-and-scope) - -## 1. Access paths - -```text -HTTP client -> Access Server -> shared metadata/chunk services - -Trusted native S3 / Dataset client - -> shared ClientRuntime - -> metadata authority and immutable content resolution - -> ChunkIO / DiskIO RPC or stream - -> registered host buffers over RC SEND/RECV -``` - -Native clients resolve immutable object generations or Dataset Snapshots and -use the existing metadata, publication, durability, integrity and reclamation -contracts. Transport selection changes byte delivery, not logical ownership. -Native clients are trusted storage participants; HTTP authorization is not -implicitly inherited by direct native access. - -## 2. Transport and buffers - -RDMA uses RC SEND/RECV, RDMA CM connections, registered host-buffer arenas, -bounded completion polling and explicit buffer ownership. Endpoint capability -preflight and fallback follow the shared RPC transport contract. An established -connection does not switch transports in the middle of an operation. Callers -retain request identities and treat interrupted mutations as uncertain. - -The native SDK owns request and byte admission. Returned host-memory leases -remain valid until released, including after runtime shutdown. Dataset fields -and S3 streaming bodies preserve their existing integrity checks and bounded -lifetime. Device selection, class-specific frame capacities, diagnostics and -hardware verification are defined by the RPC design and native SDK operations -instructions. - -## 3. Correctness and scope - -- I1: Transport selection never changes immutable generation/Snapshot binding, - exact offsets, integrity validation, publication or durable completion. -- I2: Buffer ownership persists through asynchronous completion and consumer use. -- I3: Partial or uncertain transfers are not reported as complete logical results. -- I4: Native admission and the shared RPC fallback policy bound resource use; - failed transfers do not silently splice data from another transport. -- Host-memory RC transfer is the supported path. cuObject/DC negotiation, - one-sided client-memory READ/WRITE, GPU pointer registration and direct GPU - destinations are outside this contract. Their absence is not a missing step - in native host-buffer RDMA support. diff --git a/doc/design/access-server/s3/design-crowdb-access-s3.md b/doc/design/access-server/s3/design-crowdb-access-s3.md deleted file mode 100644 index b48c004e7..000000000 --- a/doc/design/access-server/s3/design-crowdb-access-s3.md +++ /dev/null @@ -1,345 +0,0 @@ - - - -# CROWDB - Design: Access Server S3 - -S3 is CROWDB's HTTP object-storage access model. It provides the core S3 -behavior needed by existing tools while retaining CROWDB's bounded streaming, -atomic publication, and distributed storage properties. - -Depends on: [Access Server](../design-crowdb-access-server.md), -[Chunk I/O](../../chunkio/design-crowdb-chunkio.md), and -[Chunk-KV](../../chunkds/design-crowdb-chunk-kv.md). - -Satisfies: S3-compatible access without making S3 the universal abstraction for -Iceberg, Dataset, or CROWDB internals. - -## Table of contents - -1. [Intent and boundary](#1-intent-and-boundary) -2. [Authority model](#2-authority-model) -3. [HTTP data path](#3-http-data-path) -4. [Publication and lifecycle](#4-publication-and-lifecycle) -5. [Relationship to other access models](#5-relationship-to-other-access-models) -6. [Correctness invariants](#6-correctness-invariants) - -## 1. Intent and boundary - -The S3 module terminates an independent HTTP listener and owns S3 request, -authentication, error, namespace, and compatibility behavior. Its supported -surface is deliberately smaller than the complete AWS product surface. An -unsupported operation returns a stable S3-shaped error and does not mutate -state. - -The module does not own physical data placement, replication, erasure coding, -repair, or disk lifecycle. The HTTP implementation library is not part of the -S3 contract. - -## 2. Authority model - -S3 bucket and object records are first-class CROWDB metadata. Bucket names map -to stable identities, while object keys form an ordered namespace within a -bucket identity. Object records select complete immutable data generations and -carry the logical information needed for S3 reads and integrity checks. - -Bucket creation first attempts create-if-absent. A returned active mapping is -the result; a tombstone is replaced with a fresh bucket identity through CAS. -Definite conflicts reuse the returned mapping directly, with bounded attempts -and no confirmation GET. Missing observations permit create-if-absent again. -Transport errors propagate without treating an unknown result as a conflict. - -S3 is authoritative for bucket and object visibility, overwrite behavior, -listing position, multipart publication, deletion, and S3 credentials. It -stores opaque data references rather than physical storage topology. - -The current listener uses one configured tenant for all accepted credentials. -Signature verification authenticates a key but does not pass a user/grant -identity to operations. Accepted keys therefore share that listener namespace; -per-user bucket ACLs and IAM policies are not part of the installed surface. -Namespace isolation currently means the configured tenant/bucket identity -boundaries, not isolation between multiple accepted users on one listener. -Presigned request expiry is enforced independently of the allowed future-clock -skew; clock tolerance cannot extend an issued URL's lifetime. - -User bootstrap reads the bounded final credential inventory to preserve existing -keys and reject already ambiguous or disabled users. If absent, it derives a -stable public access-key identifier from the user and conditionally creates one -final encrypted credential. The secret and encryption nonce remain random; -the public identifier is not an authentication secret. A collision cannot replace -another user: the losing caller reads the exact target and validates its user, -ciphertext and enabled state. Successful creation returns its known token without -another inventory scan. There is no persistent bootstrap preparation or response. -Group-0 CAS currently supplies only a conflicting revision, so that exceptional -path needs one authoritative GET. An absent value after an uncertain write remains -an error for the caller to retry. - -Explicit token issuance can still create multiple credentials for a user; -bootstrap does not provide an atomic uniqueness guarantee against that independent -administrative operation. Existing duplicates fail the inventory check. Unused -legacy preparation records are neither authentication data nor a source for -restoring a deleted or disabled final credential. Deleting the final credential -and invoking bootstrap creates a fresh random secret; ordinary lookup never -creates data. Credential snapshots continue to load only final encrypted records. - -## 3. HTTP data path - -PUT and multipart upload stream HTTP bodies into bounded CROWDB writers. GET -streams owner-backed CROWDB data into an HTTP response. Backpressure bounds -memory and storage work independently of object size; slow peers cannot create -unbounded buffering. - -Uploads use the shared bounded `UploadBody` decoder. It verifies supported -CRC32, CRC32C, CRC64NVME, SHA1 and SHA256 headers or declared AWS trailers, -alongside Content-MD5 and signed payload hashes. Streaming SigV4 first verifies -the request seed, then every signed chunk and signed trailer. Unsigned chunks -require a verified declared trailer. Encoded and decoded lengths, frame size, -terminal chunks and trailing bytes are checked before publication; malformed -or corrupt bodies cannot replace a selected object. Checksum calculation on -requests is supported; arbitrary response checksum negotiation is outside the -installed surface. - -Presigned PUT and UploadPart also accept those checksum values and their SDK -algorithm declaration in the signed query. Authentication verifies the original -URI first; normalized checksum fields then enter the same upload verifier. -Duplicate query fields, header/query overlap and declarations on other -operations are rejected before publication. Signed query values cannot bypass -body integrity checks. - -The ordinary HTTP path is always available. An optional direct data plane may -move an authenticated object range between DiskIO and registered client memory -without relaying payload through Access Server memory. HTTP remains the control -and compatibility surface, and acceleration cannot change S3 semantics. - -## 4. Publication and lifecycle - -An object becomes visible only after its bytes and integrity state are complete. -Publication selects the complete object generation atomically. A failed or -losing upload remains invisible and is reclaimed asynchronously. - -Reads retain one published generation for the operation. Overwrite selects a -new complete generation; it never mutates bytes observed by an existing read. -Delete removes logical visibility before physical reclamation. Bucket deletion -cannot expose objects from an earlier bucket identity if the name is later -reused. - -Listings are ordered and continuation-safe within their documented consistency -model. Continuation state is opaque and bound to the original request scope. -Bucket paths accept a single trailing slash. ListObjectsV2 supports URL encoding -of XML-incompatible keys and selected prefix/delimiter fields. ListBuckets -reports the Unix epoch as a stable CreationDate placeholder because bucket -records do not retain creation timestamps. -Bucket listings skip object and multipart rows sharing the metadata interval only -after validating their complete key and value identities; unknown or corrupt rows fail. - -User metadata is accepted through x-amz-meta-* headers on ordinary PUT, -multipart initiation and CopyObject. Nonempty HTTP token names are normalized -to lowercase; values are opaque printable ASCII strings. Combined key/value -bytes, excluding the header prefix, are limited to 2 KiB. Duplicate names, -invalid values and unsupported operation placement are rejected before writes. -The complete attribute map is published with the object's immutable generation -and returned on HEAD, GET and ranged GET. A new upload without metadata clears -the previous generation's attributes. Multipart sessions persist initiation -attributes across part replacement and recovery and publish them on completion. -The session schema has a distinct version; old-version data migration is outside -the supported contract. - -STANDARD is the sole storage class reported by listings and accepted explicitly -on PUT, multipart initiation and CopyObject. Other classes are rejected; this -selector does not alter the configured storage policy or introduce tiering. - -Server-side CopyObject and UploadPartCopy resolve source and destination through -the configured S3 tenant. Copy selects one immutable source record before -returning response headers; subsequent source overwrite or logical deletion -does not select new bytes. The source is streamed through bounded readers and -writers, respecting writer capacity, and publication uses the ordinary object -or multipart-part fences. No source reference is shared with the destination. -Copy never deletes the source. Unpublished candidates use ordinary reclamation. - -CopyObject supports at most 5 GiB and copies the entire payload. COPY preserves -Content-Type and user metadata; REPLACE selects the supplied Content-Type -or application/octet-stream and replaces the entire user metadata map, including -clearing it when omitted. A self-copy requires REPLACE. -Cache/disposition/encoding/language/expiry metadata, version selectors, tags, -encryption, storage-class changes and destination conditions are unsupported -and rejected. Source ETag/date conditions apply to the captured generation; -their failure returns PreconditionFailed. Copy selectors must be signed. -The optional SDK x-id query marker must match CopyObject or UploadPartCopy and -occur at most once; it does not change operation selection. - -UploadPartCopy supports complete objects and explicit inclusive byte ranges -from source objects larger than 5 MiB, with the session's normal part bounds. -It replaces a part using the durable generation-selection protocol below. -Neither interrupted copy path publishes partial bytes. After request validation -and source selection, the HTTP 200 response carries whitespace keepalives and -ends with CopyObjectResult, CopyPartResult, or an embedded Error. Clients must -consume and validate the whole response. The body owns the copy future, cancels -it on disconnect, and caps execution at 300 seconds. A retry after response loss -can select a newer source generation; copy is not an exactly-once operation. - -Destination mutation conditions (`If-Match`, `If-None-Match` and date conditions) -are rejected before dispatch. Object GET/HEAD and copy-source conditions remain -supported. Ordinary object publication and deletion do not acquire a preliminary -object read or conditional mutation. - -DeleteObjects validates its entire request before issuing a mutation: at most -1,000 nonempty UTF-8/XML keys of 1,024 bytes each, a 2 MiB body, and no versions -or conditional object selectors. Content-MD5 is supported; verified CRC32 may -replace it for the current SDK serializer. This is an explicit integrity -compatibility extension to the general-bucket MD5 rule. Every supplied supported -checksum and signed payload hash is verified; unknown integrity extensions are -rejected. Neither missing integrity nor malformed XML can delete any key. - -A valid batch uses the same logical deletion as DeleteObject, sequentially in -input order. Each occurrence of a duplicate key gets its own result, and absent -keys are successes. Failures do not undo preceding successes; Quiet omits only -success entries. After validation, the response streams the XML opening, each -ordered result, and the closing tag. Quiet successes send XML whitespace to -maintain read progress. The response body owns the deletion stream; dropping -it cancels remaining work. Batch deletion is -not a transaction or an exactly-once operation: retries converge for keys with -no intervening writes, but can delete a new unversioned PUT after response loss. -Physical reclamation remains independent of this request. - -Multipart uploads keep a durable `MultipartSessionRecord`, current part pointers, -and immutable `MultipartPartRecord` generations under one upload prefix. The -upload resource binds admission bounds, expiration and the completion selection. -Replacing a part number retains its previous generation as reference evidence; -expiry does not remove that evidence. - -Independent-part uploads use the `S3MS\x05` resource format. They require the -checked product of `max_parts` and `max_part_bytes` to fit within -`max_staged_bytes`, so every valid set of current parts fits the staged-byte -bound without a shared accounting update. The ordinary bounds are 10,000 parts, -5 GiB per part, a 5 TiB completed object and a staged bound of 10,000 times -5 GiB. Completion separately enforces the object-size bound. These bounds do not -count retained superseded generations as current parts. Format 5 rejects pending -reservations and tighter or overflowing bounds. Codecs that do not support this -format reject it; resource CAS cannot change between independent and strict modes. - -`MultipartRepository::put_stream_part` admits an open upload, loads its current -part pointer, persists an immutable generation, then conditionally replaces that -part's pointer. It does not reserve or clear a shared upload mutation, and distinct -part numbers share no accounting CAS. An initially matching length and raw MD5 -return the retained part directly after admission. Once a writer enters the -mutation path, successful acknowledgement also observes the upload authority: -an observed abort or invalid open admission rejects it. This check reuses the -request's supplied admission timestamp and does not fence a later abort. Failed -acknowledgement does not remove persisted part evidence. - -An admitted part writer may acknowledge B after Complete freezes A for the same -part number. The frozen selection still references A's exact -generation and digest; publication and retries use A even when the current pointer -selects B or is absent. A request first admitted after the freeze is rejected. -Completing B requires a new upload identity. - -Strict uploads retain the reservation protocol and exact staged-byte accounting. -They include decoded `S3MS\x03` and `S3MS\x04` resources and uploads whose bounds -require a tighter quota or cannot establish the checked product bound. Their writer stores the immutable generation, -reserves the pointer update with an upload CAS, publishes the pointer, then clears -the reservation. A later request can finish an interrupted reservation; Complete -settles it before freezing. Publication also rechecks the selected current pointers. -Ordinary bounds do not upgrade a decoded strict resource to independent mode. - -In both modes, an unused immutable candidate does not block a part number: -matching length and raw MD5 reuse its retained generation and locations; -different contents select the next unused generation. Generation numbers increase -monotonically and can have gaps. Definite pointer and upload CAS conflicts consume -the returned observation rather than issuing a confirmation GET. Part progress -carries that observation into the next iteration, retaining already-written bytes. -Its five-second local work deadline returns a retryable busy result under sustained -contention. An uncertain mutation returns without a confirmation GET after the -routed client's retry budget; a later request retries the full input against the -retained resource facts. - -A bound upload is loaded through its authoritative key and validated against -bucket, object and upload identity; direct lookup does not read the listing index. -ListUploads and expiry scanning validate that index when enumerating resources. -Exact observed strict-format bytes are retained only in memory for conditional -updates; they do not add a response or recovery record. - -Completion validates ordered part numbers, raw MD5 values, minimum nonfinal size, -current generations and aggregate object length. `freeze_completion` binds the -selected numbers, revisions and digests under the upload-resource CAS. -`publish_completion` validates those immutable generation identities and digests, -composes chunk locations with adjusted logical offsets, and publishes one object -record through an unconditional object-key PUT. Independent publication does not -reload current part pointers. It never reads or concatenates part payload bytes. -The multipart ETag is the hexadecimal MD5 of the selected raw part MD5 values in -order, followed by the part count suffix. - -Every accepted Complete execution performs the object PUT, including retries after -an earlier success; it may overwrite a later ordinary PUT to the same name. The frozen selection is -the reference authority, not a recorded response. No predecessor, request digest, -cached response or publication-success transition is persisted. Abort and expiry -mark open uploads terminal while retaining their listing entries, pointers and -immutable generations. Publishing uploads remain bound, retaining the selected -generation references. Expiry reuses the authority loaded through the validated -listing entry and consumes the abort CAS observation without another confirmation -read. An uncertain mutation remains an error; later scanning rechecks the durable -resource state. - -The background expiry worker owns discardable tenant and upload key positions -for one configured tenant. Each two-second step scans one raw tenant metadata -key or at most 20 upload-index entries, with a four-MiB aggregate result budget. The legacy -scan transport materializes values before applying that aggregation limit; it -does not enforce the budget on wire responses or native value materialization. -The first sixteen bytes after the tenant prefix supply a BucketId hint; the -name mapping and its payload are not discovery authority. An old bucket remains -discoverable after deletion or same-name recreation. The worker then scans that -identity's existing listing index, validating each canonical index and upload -authority before expiry. Upload creation persists the index before its authority. -Long name keys may supply harmless extra hints; short keys advance by their -observed raw position. There is no bucket-count cutoff. - -Selecting a hint also saves the inclusive successor of the entire bucket -prefix. Finishing, failing or cancelling its index scan can therefore move to -later identities without scanning retained object, part and generation rows -one by one. A later complete round retries discarded hints. Successful partial -index progress advances before its point operation, so a corrupt row or -uncertain abort cannot pin subsequent uploads. A process restart scans the -configured tenant again without loading a persistent progress record. - -Bucket discovery requests one raw row at a time. If a materialized first row -exceeds the aggregate budget, the client returns only its validated key. The -worker retains its identity hint for the next window and reports an oversized-row -error, preserving the original metadata for later rounds. A short key instead -advances its raw position. A transport failure that -provides no validated key cannot be skipped safely and remains retryable. -Obtaining discovery keys without materializing their values requires a separate -key-only or native bounded-scan capability. - -Workers continue after 50 milliseconds while a round has more work, wait -15 seconds between complete rounds, and defer failures for 500 milliseconds. -These local cursors authorize no physical deletion. Physical range discovery, -reference checks and retirement fences remain separate reclamation duties; -marking an upload terminal does not report its physical storage reclaimed. - -## 5. Relationship to other access models - -Iceberg is not implemented as special objects in the S3 namespace. Its catalog, -table, file, commit, and reclamation records belong to the Iceberg authority, -even when an Iceberg client uses S3-shaped file locations. - -Dataset is not an S3 extension. A Dataset may explicitly refer to immutable -bytes associated with S3, but Dataset selection, batching, streaming, and -native topology access remain Dataset semantics. - -## 6. Correctness invariants - -- **S3-I1 — Atomic visibility:** a reader observes absence or one complete - published object generation, never partial upload bytes. -- **S3-I2 — Stable read:** one HEAD or GET retains one object generation for - the entire operation. -- **S3-I3 — Bounded streaming:** object size does not determine Access Server - memory consumption. -- **S3-I4 — Namespace isolation:** credentials, bucket identity, object key, - and continuation state cannot escape their authorized scope. -- **S3-I5 — Reclamation after invisibility:** physical deletion follows logical - removal and cannot restore visibility. -- **S3-I6 — Independent authority:** S3 records cannot publish, mutate, or - reclaim Iceberg or Dataset authority. -- **S3-I7 — Transport equivalence:** ordinary HTTP and accelerated transfer - produce the same S3 range, integrity, publication, and error outcome. -- **S3-I8 — Multipart selection fence:** a part pointer cannot advance after - completion freezes its selection. An interrupted pointer reservation is - settled before completion or abort proceeds. diff --git a/doc/design/chunkdb/chunkdb-allocate-flow-analysis.md b/doc/design/chunkdb/chunkdb-allocate-flow-analysis.md deleted file mode 100644 index cd4ca4a9a..000000000 --- a/doc/design/chunkdb/chunkdb-allocate-flow-analysis.md +++ /dev/null @@ -1,189 +0,0 @@ - - - -# ChunkDB Allocation Flow Analysis - -## Scope - -This analysis measures one-strip EC 8+4 chunk allocation on three co-located -logical nodes. Each node runs KV, DiskDB, and ChunkDB; the fixture has three -racks and three DiskDB data groups backed by three-replica KV groups. One -successful ChunkDB request allocates and commits twelve physical blocks. - -The implementation keeps strips sequential. Within one strip it groups blocks -by data group, sends one DiskDB allocation request for four blocks to each of -the three nodes, and runs those requests concurrently. After they return, -ChunkDB persists one Active chunk record and returns success. Three grouped -block-commit requests run asynchronously after publication. - -## Current Allocation Flow - -For one EC 8+4 strip, the success path is: - -1. ChunkDB selects twelve placements and groups them into three data-group - requests. Each request asks one DiskDB server for four blocks on four - different disks. The three requests run concurrently. -2. Each DiskDB server claims four bitmap ranges, constructs four - `BusyBlockValue` records with `state=Ok` and `commit_state=Tentative`, and - sends one four-key batch write to that data group's KV leader. After the KV - write succeeds, DiskDB caches the tentative values by `allocation_ts` and - returns the four segments to ChunkDB. -3. After all DiskDB responses return, ChunkDB constructs one Active chunk - containing the twelve segments and writes that chunk record to its KV - group. This is the allocation publication and durability boundary. -4. Once the Active chunk write succeeds, ChunkDB starts three asynchronous - commit RPCs, refreshes its chunk cache, and returns the Active chunk to the - client. The client does not wait for block commit. -5. Each DiskDB commit validates the physical identity, owner, and allocation - timestamp. A tentative-cache hit supplies the value without a KV read; a - miss reads it from KV. DiskDB changes `commit_state` to Committed, writes - all changed busy-block values in one batch, and removes exact cache entries - after success. An already Committed value is idempotent success. - -At client success, all twelve busy blocks and the referencing Active chunk are -durable. The blocks may remain Tentative until background commit completes. A -future reconciliation scanner can resolve that crash window by asking -ChunkDB whether the Active chunk references the same allocation incarnation, -then committing or freeing the block. - -## Environment and Results - -- Host: AMD Ryzen 9 5950X, 16 cores / 32 threads, 62 GiB RAM. -- Duration: 20 seconds per row. -- Capacity: four 4-TiB logical disks per DiskDB; 256-GiB zones. -- KV inflight/coalescing: 32/32. -- Command: `pixi run -- bash tools/benchmark/bench-chunkdb-regression.sh`. - -| Workload | Groups | Threads | Strips | EC | Client conn | ChunkDB conn | DiskDB conn | KV conn | Workers | Chunk/s | Block/s | p50 us | p99 us | Errors | Space | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---| -| allocate | 3 | 1 | 1 | 8+4 | 2 | 2 | 2 | 2 | 2 | 799 | 9,588 | 1,275 | 1,561 | 0 | exact | -| allocate | 3 | 16 | 1 | 8+4 | 2 | 2 | 2 | 2 | 2 | 8,800 | 105,600 | 1,719 | 3,481 | 0 | exact | -| allocate | 3 | 128 | 1 | 8+4 | 4 | 4 | 4 | 4 | 4 | 11,914 | 142,968 | 9,945 | 22,922 | 0 | exact | -| allocate | 3 | 256 | 1 | 8+4 | 4 | 4 | 4 | 4 | 4 | 12,685 | 152,220 | 18,427 | 49,050 | 0 | exact | -| allocate | 3 | 512 | 1 | 8+4 | 4 | 4 | 4 | 4 | 4 | 12,734 | 152,808 | 37,131 | 91,641 | 0 | exact | - -All rows stopped at the deadline and retained three KV, DiskDB, and ChunkDB -metrics/RPC log pairs plus the CLI pair. Clean artifacts are: - -- `bench-log/chunkdb-r98-final2-20260905-223329` (all rows). -- `bench-log/chunkdb-r98-common-latency-20260905-224613` (selected row). -- `bench-log/chunkdb-r98-confirm-2-20260905-224959` (confirmation). -- `bench-log/chunkdb-r98-confirm-3-20260905-225127` (confirmation). - -The selected 256-thread confirmation samples are 12,707, 12,460, and -12,634 chunks/s. Their median is 12,634 chunks/s and the maximum deviation is -1.4%, within the 10% stability bound. Every sample has zero errors and exact -space accounting. - -An R149 validation run on an Intel Core i9-7960X (16 cores / 32 threads) -confirmed the cache-assisted batched commit path after removing per-block -BusyBlock CAS. These numbers are a same-host regression sentinel and are not -directly comparable with the AMD baseline above. - -| Threads | Chunk/s | Block/s | Average us | p50 us | p99 us | Errors | Space | -|--------:|--------:|--------:|-----------:|-------:|-------:|-------:|:------| -| 1 | 238 | 2,856 | 4,195 | 4,188 | 5,408 | 0 | exact | -| 16 | 7,015 | 84,180 | 2,279 | 2,194 | 3,773 | 0 | exact | -| 128 | 9,795 | 117,540 | 13,046 | 12,647 | 22,392 | 0 | exact | -| 256 | 10,441 | 125,292 | 24,474 | 23,307 | 47,543 | 0 | exact | -| 512 | 9,702 | 116,424 | 52,647 | 48,765 | 112,289 | 0 | exact | - -The retained artifact is -`bench-log/chunkdb-regression-20260913-132220`. A focused same-host comparison -measured 151 chunk/s at one thread and 150 chunk/s at 16 threads with -per-block CAS, including a 4.24-second p99 at 16 threads. Restoring one -ordinary batched commit write per data group reached 249 and 6,920 chunk/s in -the focused run and removed the seconds-scale tail. - -## Bottleneck - -The peak is 12,734 chunks/s, or 152,808 physical blocks/s, at 512 threads. -Increasing from 256 to 512 threads adds only 0.4% throughput while p99 grows -from 49 ms to 92 ms, so 256 threads remains the better latency/throughput -operating point. - -DiskDB retains each freshly persisted tentative `BusyBlockValue` in a bounded -per-data-group cache keyed by `allocation_ts`. Commit validates the cached -physical identity and owner and builds one batch write without a preceding -read. A miss after restart or eviction falls back to KV. The historical -512-thread run reported 3,153,276 cache hits, exactly 262,773 chunks times -twelve blocks, and zero misses. - -At 256 threads, weighted ChunkDB metrics show: - -- grouped DiskDB allocation: 9.48 ms; -- the single Active metadata KV persistence: 5.08 ms; -- complete request: 18.93 ms; -- asynchronous block commit: 9.30 ms, outside response latency; -- placement: below 0.02 ms. - -Each DiskDB server receives exactly one four-block allocation RPC and one -four-block commit RPC per chunk. The earlier flow sent four one-block -allocation RPCs to each node and synchronously waited for commit. Grouping and -asynchronous commit raised the 256-thread row from 8,737 to 12,860 chunks/s -(47.2%) and reduced p50 from 28 ms to 19 ms. - -The response-path bottleneck is now the grouped DiskDB allocation round. At -512 threads it grows to 26.16 ms, while the single metadata write is 6.62 ms; -the asynchronous commit averages 23.72 ms. These concurrent writes share the -same KV/Paxos groups, so commit still contributes background queue pressure -even though callers no longer wait for it. Future DiskDB reconciliation must -resolve tentative blocks after a ChunkDB crash by checking whether the Active -chunk still references the exact allocation incarnation, then committing or -freeing it. - -### Allocation latency attribution - -At 256 threads with four connections and four RPC workers, one three-node -allocation round averages 9.48 ms. The average DiskDB `AllocateBlocks` request -is 6.02 ms, including 4.99 ms in its KV batch write. Bitmap selection is -0.003 ms and response construction is 0.007 ms. Therefore DiskDB's local -allocator is not the bottleneck: about 83% of server request latency is the -KV/Paxos write. The remaining 3.46 ms between average server latency and the -client round includes transport queueing and waiting for the slowest of three -parallel responses. - -A 256-thread connection/worker sweep produced: - -| Connections/workers | Chunk/s | Allocation round | DiskDB request | DiskDB KV persist | Request latency | Result | -|---:|---:|---:|---:|---:|---:|---| -| 2 | 14,129 | 10.46 ms | 7.26 ms | 6.45 ms | 16.98 ms | exact, zero errors | -| 4 | 12,860 | 9.48 ms | 6.02 ms | 4.99 ms | 18.93 ms | exact, zero errors | -| 8 | 11,252 | 10.49 ms | 6.67 ms | 5.45 ms | 21.39 ms | initial run failed accounting | - -Four connections/workers minimize the isolated allocation round, but two -produce better end-to-end throughput because the single metadata write falls -from 5.08 ms to 3.92 ms despite a slower allocation round. Eight add queue -pressure rather than throughput. The first 8/8 run exposed a benchmark result -parser bug: its `busy_delta` lookup could match the suffix of -`expected_busy_delta` and incorrectly print `exact`. The parser now requires a -field boundary. A 120-second diagnostic run also identified the apparent -accounting error as verification overload: two of 256 parallel chunk queries -timed out, so their 24 valid blocks were omitted from expected space. Chunk -verification is now limited to 64 parallel queries and prints the exact failed -ID and reason. -Two subsequent 60-second stress runs were clean: 10,933 chunks/s at 256 -threads and 10,882 chunks/s at 512 threads, both with zero errors and exact -space. After limiting verification concurrency, the final 20-second 256-thread -8/8 run reached 11,571 chunks/s with zero errors and exact space. This -indicates shared KV/Paxos contention, not insufficient -ChunkDB-to-DiskDB connections or workers. - -### KV load - -Each successful EC 8+4 chunk issues three four-key DiskDB allocation writes, -one single-key Active chunk write, and three four-key asynchronous commit -writes. The total is seven logical KV write requests and 25 key mutations per -chunk. Therefore the canonical 512-thread result of 12,734 chunks/s represents -89,138 logical KV requests/s and 318,350 key mutations/s. The latter is above -the direct KV write benchmark peak of 264,130 single-key writes/s; grouped -DiskDB writes and KV coalescing make the request and Paxos-round rates much -lower than the key-mutation rate. - -## Metrics - -Every ChunkDB RPC exports `request.{method}.lh`, `.inflight.g`, and `.errors.c`. -Allocation also exports placement, DiskDB round/call/retry, asynchronous block -commit/error, record-build, Active KV persistence, rollback, strip/block -count, and inflight metrics. CLI, ChunkDB, DiskDB, KV, and crowdb-rpc logs -share the timestamped regression root. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb-mirror-to-ec.md b/doc/design/chunkdb/design-crowdb-chunkdb-mirror-to-ec.md deleted file mode 100644 index 622d8671a..000000000 --- a/doc/design/chunkdb/design-crowdb-chunkdb-mirror-to-ec.md +++ /dev/null @@ -1,278 +0,0 @@ - - - -# CROWDB - Design: Mirror-to-EC Conversion and Chunk Tasks - -This document defines synchronous small-write EC conversion and the persistent -chunkdb task framework used for recovery and background conversion. - -Depends on: [chunkdb](design-crowdb-chunkdb.md), -[chunkdb RPC](design-crowdb-chunkdb-rpc.md), and -[small-object writer](../chunkio/design-crowdb-chunkio-small-object-writer.md). - -The read-side producer for repair tasks is the -[chunk object reader](../chunkio/design-crowdb-chunkio-reader.md). - -Satisfies: durable space reduction for shared mirror chunks and reusable -crash-recoverable chunk maintenance tasks. - -## Table of Contents - -- [1. Scope](#1-scope) -- [2. Persistent Chunk Tasks](#2-persistent-chunk-tasks) -- [3. Foreground Conversion](#3-foreground-conversion) -- [4. Background Conversion](#4-background-conversion) -- [5. Publication and Cleanup](#5-publication-and-cleanup) -- [6. Crash Correctness](#6-crash-correctness) -- [7. Policy and Observability](#7-policy-and-observability) -- [8. Test Contract](#8-test-contract) - -## 1. Scope - -Eight adjacent closed one-MiB mirror strips can become one 8+4 EC strip whose -eight data shards preserve the mirror images and whose four parity shards are -generated by isa-l. The small-write client performs this work without rereading -mirrors when memory and I/O are available. A persistent chunkdb task provides -takeover after client failure and supports scanner-created background work. - -Incomplete groups remain mirrored. Conversion preserves logical capacity, -`chunk_offset`, `sealed_length`, and the monotonic `next_strip_sequence`. - -## 2. Persistent Chunk Tasks - -### 2.1 Keys - -Each task has one canonical record and at most one runnable or leased index. -Integers use big-endian encoding so index scans retain scheduling order. - -```text -ChunkTaskKey = magic | 0x000D | partition-id:16 | kind:u16 | task-id:16 -ReadyChunkTaskKey = magic | 0x000E | priority-inverse:u8 | - eligible-at-ms:u64 | partition-id:16 | kind:u16 | - task-id:16 -LeasedChunkTaskKey = magic | 0x000F | lease-deadline-ms:u64 | - partition-id:16 | kind:u16 | task-id:16 -``` - -The target `ChunkId` is the partition ID for chunk-scoped tasks. Canonical and -index mutations route to the same KV group and are written in one batch. Index -values are empty and rebuildable; the canonical value is task truth. - -`task-id` is a deterministic hash of task kind and its deduplication identity. -Mirror conversion uses the first mirror sequence and EC scheme. Its operation -ID is also deterministic, allowing publication retries to identify an already -installed replacement. - -`RepairStrip` uses the chunk as partition, the task kind as the dispatch -namespace, and a deterministic task ID derived from strip sequence plus the -sorted failed-segment identities. Its versioned payload contains the chunk ID, -strip sequence, and failed segments. The canonical envelope retains source -revision, estimated full-rebuild I/O bytes, retry state, claim generation, and -the deterministic publication operation ID. Other task types keep their own -payload schema while sharing the envelope, indexes, leases, and transitions. - -### 2.2 Value and Handler Dispatch - -`ChunkTaskValue` is a verified, versioned FlatBuffer envelope containing stable -identity, kind and kind version, state, priority, revision, source revision, -retry eligibility, attempt budget, queue-cost estimate, claim owner and -generation, lease deadline, bounded error detail, and an opaque typed payload. - -States are `Pending`, `Running`, `RetryWait`, `Completed`, `Failed`, and -`Cancelled`. `TaskHandler` dispatch is keyed by kind and checks the payload -version before execution. New task kinds can therefore add their own payload -without changing the scheduler. Replica repair, EC rebuild, orphan cleanup, -and placement verification can use the same envelope. - -### 2.3 Claim and Recovery - -`TaskManager` admits deterministic tasks, claims ready records, renews leases, -and persists complete, retry, or failure outcomes. `TaskScanner` wakes on -admission and also performs a periodic safety scan. It first requeues expired -leases, then claims no more ready tasks than `TaskExecutor` has available -slots. The executor heartbeats while a handler runs. - -The configured concurrency is a fixed upper bound. Retry eligibility and lease -deadlines schedule work; elapsed idle time does not scale worker count. -Conversion and repair handlers also enforce independent local concurrency -semaphores. Repair has the highest ready priority because it restores data -redundancy, while conversion only reclaims space. - -### 2.4 Read Repair Admission and Execution - -Readers report only verified frame parse or checksum failures. ChunkDB first -marks the exact committed DiskDB BusyBlock incarnation `Corrupt`, then records -its segment in `ChunkStrip.unavailable_segments` and immediately admits -`RepairStrip`. Network and unknown I/O errors do not mark or admit repair. -A bounded rotating metadata scan also admits one -task per strip and failure set. An existing pending, running, or retry-wait -task deduplicates admission. A terminal task is revived at a higher revision -when its marker remains, preventing a stale completion or failure from -suppressing necessary repair. - -`RepairStripTaskHandler` re-queries current metadata and treats the marker as -authority. It reads one full healthy mirror or full EC shards, records any new -I/O failures before retrying, reconstructs unavailable shards, and allocates -replacement segments excluding every disk already used by the strip. Mirror -placement excludes survivor nodes. EC replacement instead preserves the -current safe `code_num` per-node limit, or the balanced ceiling of an -explicitly unsafe layout. A disabled-by-default test escape hatch may relax -only node anti-affinity, never same-disk exclusion. - -Each replacement is fully written and fsynced before exact revision-and-range -fenced publication. The publication removes repaired markers and adds retired -segments to the normal layout-validity cleanup intent. A changed layout causes -re-query and retry. Repair tasks have unlimited attempts and use delayed retry -rather than tight polling when memory, placement, parity, or I/O is temporarily -unavailable. - -The read-triggered ad-hoc manager joins routed requests by chunk, strip, and -failed segment incarnation. It uses an in-memory shared future, at most 32 -concurrent jobs, and a 512-MiB decode/result budget by default; it never owns -a second durable task or replacement target. It may return reconstructed -bytes after the task writes and fsyncs its checkpointed target, while the task -continues fenced publication and confirmation. Restart discards the future, -not the task or target checkpoint. Stale revisions and healed incarnations -cannot join or allocate. - -## 3. Foreground Conversion - -Each small-write pipeline owns one special reservation group containing eight -mirror candidate sets with the configured copy count and four parity segments. -The group is chosen from one placement snapshot and is hidden from -`Chunk.strips` until individual mirrors are confirmed. A group is not started -when the current chunk cannot -contain eight remaining full strips. - -After a mirror strip is durable, its existing one-MiB shadow updates four -persistent parity accumulators and is released. The eighth update completes -parity without rereading mirrors and without retaining eight data images. - -A pipeline holding a partial group is not a scale-in candidate. Scale-in first -requires every queue to be empty and every pipeline to be non-busy, then removes -an individual pipeline that does not own partial conversion state. Queue bytes -and object count remain the only scale-out inputs. - -After shard eight, the client writes and fsyncs only the four parity segments, -then requests fenced publication. ChunkDB re-evaluates every healthy survivor -combination against current topology, selects the minimum-concentration safe -8+4 layout, atomically replaces the eight mirrors, and records the sixteen -redundant replicas for grace-period cleanup. A fast-path parity or publication -error does not revoke acknowledged mirrors; it admits the existing durable -`MirrorToEc` task for takeover. - -An early one-through-seven-strip tail cancels unused mirror reservations and -all parity segments while retaining every confirmed mirror strip. - -## 4. Background Conversion - -`ConversionCoordinator` scans chunk metadata in bounded rotating pages and -admits complete compatible mirror groups. A group must have equal nonzero unit -geometry, contiguous offsets, and an immutable closed boundary. Sealed chunks -also obey the configured age policy; manual single-chunk triggers bypass age -and count policy but not geometry, closure, or chunk-state checks. - -`MirrorToEcTaskHandler` reads one healthy replica for each mirror, falling back -across replicas on routing or DiskIO errors. It generates parity, writes and -fsyncs all replacement shards, and invokes the same fenced publication used by -the client. Disk-to-DiskIO routes are atomically published and periodically -refreshed from hardware and service-registry state. - -Automatic scans leave incomplete tails mirrored and continue across task -failures. Manual single and batch triggers are available through crowdb-rpc and -the `/convert_chunk` and `/convert_all` HTTP endpoints. - -A separate bounded, rotating reservation scan closes the crash window before a -completed foreground group has admitted its task. Complete groups enter the -same deterministic task identity, making repeated scans idempotent. Expired -incomplete groups are cancelled; consumed blocks are reclaimed only when the -reservation persisted the planned write cursors that prove generation-fenced -DiskIO addressing. Legacy records without that proof remain allocated. - -## 5. Publication and Cleanup - -The special reservation allocator creates all twenty-eight blocks with -all-or-rollback semantics. DiskDB requests are aggregated per DiskDB group and -use one persisted request per group. Conversion requests permit repeated -anti-affinity passes: every pass visits distinct disks before any disk is reused, -while the first twelve EC roles are validated to fit on distinct disks. This -avoids both one RPC per block and an oversized request failing solely because -the mirror candidates outnumber disks. No reservation record makes a block -readable. - -`replace_chunk_strip_range` validates the expected revision and exact old -range. Its KV transaction commits replacement segments, publishes the EC strip, -and records retired mirror segments as one atomic metadata change. Retired -segments remain allocated for the layout-validity grace period; reconciliation -then frees only segments absent from the current layout. - -On a definite pre-publication failure, the task first clears its durable -replacement reference. DiskDB's tentative-block scanner reclaims the now -unreferenced allocation. An ambiguous KV response retains the allocation until -recovery determines whether its checkpoint or publication committed. - -## 6. Crash Correctness - -- **C1 — Layout authority:** readers use only committed `Chunk.strips`; task - state and tentative blocks never select readable data. -- **C2 — Mirror preservation:** every mirror remains referenced until every EC - shard is written and fsynced. -- **C3 — Atomic visibility:** fenced range replacement is the only EC - publication point. -- **C4 — Ambiguity retains data:** an ambiguous checkpoint or publication never - frees a possibly referenced replacement. -- **C5 — Replayable retirement:** replacement and cleanup intent commit - together, and cleanup uses set difference against the current layout. -- **C6 — Duplicate safety:** deterministic identity and revision/range fencing - permit at most one layout publication even if claims overlap. -- **C7 — Append independence:** replacement preserves capacity and - `next_strip_sequence`, so later appends cannot duplicate or reorder strips. - -A takeover rewrites all shards and fsyncs them; it does not trust partial-write -progress. If publication is already visible anywhere in the current layout, -the handler treats the operation as installed. If allocation succeeded but no -task checkpoint references it, the allocation remains tentative garbage and is -eventually reclaimed by DiskDB. This can temporarily consume space but cannot -make corrupt data visible. - -## 7. Policy and Observability - -`ConversionConfig` defaults to disabled automatic scanning, 8+4 EC, one-hour -minimum seal age, eight mirror strips, four concurrent handlers, 50 MiB/s -background I/O, a 30-second scan interval, and a 30-second task lease. - -The bandwidth limiter atomically reserves read-plus-write byte time and permits -one conversion-group burst. Time controls only background bandwidth and retry -or lease scheduling; it does not control small-write pipeline scale-in/out. - -`ConversionMetrics` exposes attempts started/completed/failed, bytes read and -written, retired mirror segments, active handlers, and peak active handlers. -`GET /conversion_metrics` returns a nonblocking snapshot. - -`RepairConfig` defaults to enabled, four concurrent handlers, a 64-MiB shared -full-shard memory budget, and a one-second marker scan. The memory semaphore is -reserved before shard reads. A repair larger than the configured budget stays -durably retryable. `RepairMetrics` exposes admissions, attempts, repaired -segments and bytes, active handlers, and current/limit/peak memory through -`GET /repair_metrics`. - -## 8. Test Contract - -Core coverage uses real KV, chunkdb, diskdb, DiskIO, and routed clients. It -verifies foreground no-reread conversion, data and parity bytes, missing-shard -decode, active-prefix append, client-I/O failure takeover, manual and automatic -triggers, multiple groups plus mirror tail, bounded task dispatch, cleanup -reconciliation, and process crash/restart recovery. Mock-based tests remain -auxiliary for deterministic error injection and unit-level queue behavior. - -Reservation coverage verifies hidden reserve/consume state, idempotent -confirmation and cancellation, eight joint mirror candidate sets plus parity, -every early tail from one through seven, and reconstruction from the published -survivor/parity layout. - -Read-repair coverage additionally verifies same-offset partial EC recovery, -minimum surviving-shard reads, mirror fallback and total loss, exact partial -ranges, stream termination, incomplete-parity behavior, durable marker -admission across process restart, full-shard replacement, task revival, -memory/metrics bounds, and continued writes to an active chunk whose revision -was advanced by a reader. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md b/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md deleted file mode 100644 index f474897da..000000000 --- a/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md +++ /dev/null @@ -1,318 +0,0 @@ - - - -# CROWDB - Design: ChunkDB Slot Routing - -Depends on: [ChunkDB architecture](design-crowdb-chunkdb.md), -[KV group 0](../kv/design-crowdb-paxos-kv-group0.md). -Satisfies: ChunkDB stateless execution and explicit nonzero persistence. - -ChunkDB uses independent service and storage maps over the same fixed hash -space. This document covers their publication, request routing, task authority, -and fixed/dynamic service ownership policies. - -## Table of Contents - -- [1. Slot identity and invariants](#1-slot-identity-and-invariants) -- [2. Control-plane records](#2-control-plane-records) -- [3. Publication and startup](#3-publication-and-startup) -- [4. Client and storage routing](#4-client-and-storage-routing) -- [5. Maintenance authority](#5-maintenance-authority) -- [6. Costs and operating limits](#6-costs-and-operating-limits) -- [7. Data-group execution fences](#7-data-group-execution-fences) -- [8. Durable handoff records](#8-durable-handoff-records) -- [9. Submission epoch policy](#9-submission-epoch-policy) - -## 1. Slot identity and invariants - -`ChunkSlot::for_chunk` hashes the canonical 16-byte big-endian ChunkId with -XXH64 seed zero and takes modulo 1024. Slots are 0..1023. The hash includes -both ID words and the concrete purpose prefix. Neither server count nor group -count participates in this calculation. - -- **I1 — Independent owners:** each slot has one service instance and one - selected nonzero `(store_id, group_id)` destination. A service bitmap can - span several storage groups, and a storage group can serve several instances. -- **I2 — Complete publication:** each map covers all slots exactly once, with - no overlaps, missing owners, malformed bitmaps, or mixed generations. -- **I3 — Chunk-local atomicity:** chunk, canonical task, maintenance indexes, - and reservation records route by the owning ChunkId. Their byte-key order - never selects a group. Conditional multi-record writes remain group-local. -- **I4 — Fail closed:** missing routes, unavailable selected groups, invalid - types, and zero-slot service owners cannot become group-0 or any-owner fallbacks. -- **I5 — Fixed authority:** under fixed policy, refreshing an initialized map may not change its - generation or ownership. Endpoint changes do not change either slot map. - -These are hash partitions, distinct from chunk-kv ordered key ranges. The -maps do not place payload bytes in KV; payload remains on DiskIO/DiskDB. - -## 2. Control-plane records - -Group 0 contains two independent map namespaces: - -- `/chunkdb/slot_service/`: one service binding per instance, - including an all-zero bitmap for an instance owning no slots. -- `/chunkdb/slot_storage//`: one storage binding per - explicitly selected nonzero group. -- `/chunkdb/slot_head/service` and `/chunkdb/slot_head/storage`: independent - heads containing layout version, generation, and owner count. - -Each binding contains owner identity, generation, and a 1024-bit bitmap. -JSON encoding adds metadata overhead; the raw bitmap is exactly 128 bytes. -These routing maps have no per-slot or per-contiguous-range records. Service -endpoints are registered separately and can refresh after same-instance restart. - -## 3. Publication and startup - -`ChunkSlotMapClient` validates the full bootstrap. Up to 63 owner records and -their head publish in one head-conditional group-0 batch. Larger maps first -CAS the absent head to an initialization intent containing the complete layout, -canonically ordered by owner key, and a durable progress offset. Each page -writes at most 63 owners and advances that intent under its current head -revision. The final page replaces the intent with the regular head; readers -reject an intent and therefore never expose a partial map. Every proposal -fits the 64-record replay window. - -A replacement initializer can resume the identical intent, including with a -different input ordering. Conflicting layouts fail without writing owners. -An uncertain page result is reconciled only when the head proves durable -progress; otherwise the error remains visible. Stale pages cannot write after -publication because the head revision has advanced. A retry reads and -reconciles the committed result. Both maps -must exist before ChunkDB accepts allocations, even though the maps have -independent publication boundaries. - -Map reads use fixed-cutoff pagination and recheck the head revision before -publishing a validated immutable cache. Legacy range bindings, orphan records, -and existing group-0 chunk/task/reservation state prevent unsupported bootstrap. -They are preserved for an explicit conversion rather than silently rewritten. - -Startup requires an explicit instance identity and validated storage/service -maps. Optional bootstrap configuration supplies the eligible groups and -service instances. The deterministic bootstrap uses balanced service slot bands -and interleaves slots among storage groups; this assignment is initial policy, -not the definition of a slot. Readiness requires initialized maps and valid -storage routing. An instance with zero service slots is ready but rejects work -outside its empty bitmap; local disks are not a service-readiness requirement. -HTTP readiness reads the independently sampled ownership result without issuing -Group-0 requests. One worker refreshes the complete map every second, with a -three-second read deadline and no overlapping refreshes. Evidence ages from the -start of that authority read. Pending, failed or older-than-three-second evidence -returns unavailable; HTTP readers cannot renew it. This observation does not -replace per-slot execution admission or weaken map validation. -The fixed-layout monitor audits ownership and does not reassign -slots when a heartbeat expires. Legacy assignment writers reject updates. - -Console automatic service plans deploy the selected ChunkDB cohort before -starting Chunk-KV journal bootstrap. A serial deployment must not await journal -allocation while another published ChunkDB owner still needs to start. - -## 4. Client and storage routing - -Ordinary Rust clients load only the service map and service discovery. -`RangeBindingClient` retains an immutable map and endpoint snapshot. Existing -chunk operations resolve the ChunkId slot; generated allocation selects a live, -nonempty service owner, whose server generates an ID within its owned bitmap -using bounded retries. Caller-supplied IDs are checked before side effects. - -`RangeGuard` admits only the installed instance bitmap. `BindingCache` builds a -constant-time storage lookup from the independent storage map. Neither hot -routing path introduces a mutex. Cold restart loads durable records from remote -KV; local locks, caches, and in-flight operations are not persistence authority. - -Native tree clients use a retained service resolver with asynchronously refreshed -immutable snapshots. Per-call route leases retain old connections through -completion, and the page store retains resolver/DiskIO owners for its lifetime. -Native RPC completion pools are separate from ordinary Rust RPC pools. -Explicit `NotMyRange` permits bounded refresh and retry. Unknown allocation or -mutation outcomes return to the caller without blind resubmission; read-only -queries can refresh and retry. No original-owner fallback is supported. - -Readiness sends real point queries through the initialized service route without -first running the ordinary periodic Group0 registry/map refresh. Cold discovery -still requires a validated map; ownership rejection or transport failure retains -the existing read-only refresh/retry path and caller deadline. The receiving -owner validates admission and reads its actual KV storage. This keeps a bounded -probe independent of an unrelated full registry refresh without caching a -successful result or changing ordinary requests' freshness rules. - -The owner's KV transport isolates Get responses from background scans with one -additional connection per endpoint and the same RPC workers and completion -budget. The point request still reads current storage and retains its original -read mode, error handling and deadline. A partially received catalog page must -not hold the receiving owner's point-read response behind that page's bytes. - -## 5. Maintenance authority - -One process hosts separate system and user-data lifecycle/task runtimes. -Wal, BtreePage, and PageIndex are system purposes; Stream, S3, IcebergTable, -and Dataset are user-data purposes. Each runtime has its own scan scope, claims, queue, -execution capacity, and recovery. Shared transport, codecs, IO, and algorithms -do not create shared task authority. - -Ready, lease, and finalize indexes encode domain and slot before scheduling -fields. Scans select owned slot runs in each eligible destination and paginate -before applying result limits; eligible work is merged by its scheduling order. -Canonical tasks retain the owning chunk ID. Stores reject foreign slots and -domains at read, admission, claim, and publication boundaries. Execution verifies -the durable claim before handler side effects; claim generations and writer -epochs fence delayed completion after same-owner restart. - -Chunk and reservation metadata retain their existing ChunkId-ordered keys. -Scoped scans pass the domain's half-open purpose-byte bounds to KV before -transferring values: system types 1 through 3 and user-data types 4 through 7. -Continuation stays within those bounds; ownership filtering and fixed-cutoff -pagination remain in effect. Growing an unrelated domain must not add pages -to the scoped read. Unscoped inventory retains the complete keyspace. - -The reservation budget is apportioned by service slot share and split equally -between system and user-data runtimes. Execution limits are independently -reserved; shared reservation gauges aggregate domain contributions. Domains can -share a physical Paxos group and therefore still share its availability limits. - -## 6. Costs and operating limits - -For S service owners and G storage destinations, binding records total S + G, -plus two heads. Raw bitmaps total `128 * (S + G)` bytes. With three instances -and three groups this is six owner records and 768 raw bitmap bytes, plus heads -and encoding overhead. Each initial map requires one conditional consensus -batch; allocation does not rewrite a binding record. - -Each loaded map builds a 1024-entry lookup. Hot route lookup is constant-time -and allocation-free. Map reload costs scale with owner records and paginate -in bounded batches. Maintenance scans scale with owned contiguous slot runs -and selected destinations, rather than allocating a worker or tree per slot. -Arbitrary fragmented bitmaps are supported but increase scan requests; balanced -bootstrap bands reduce that cost. Slot balance does not guarantee byte or load -balance when a few chunks dominate traffic. - -Under fixed policy, topology is fixed while serving. Same-owner restart, endpoint refresh, and -Paxos replica failover preserve both maps. Adding/removing service owners or -storage groups, remapping slots, and legacy conversion require an external -fenced handoff/migration protocol and are currently rejected. Service changes -need task-authority handoff; storage changes must move complete chunk/task/ -index/reservation state before ownership cutover. Neither operation implicitly -moves DiskIO payload. Chunk-kv is not a ChunkDB metadata backend; its ordinary -batch API does not supply the required conditional multi-record atomicity. - -## 7. Data-group execution fences - -KV supports a separate execution fence at `/chunkdb/ownership-fence/` -inside the selected data group. The slot segment uses canonical decimal in -0..1023; slot fence operations reject group zero. This namespace is independent -of routing-map records in group 0 and of DiskDB's ownership namespace. -Fixed-policy ChunkDB does not initialize or -use these execution fences. - -`ChunkSlotAuthority` identifies an instance, a nonzero 128-bit process -incarnation and a nonzero per-slot generation. A same-ID process restart has -a distinct incarnation. The canonical comparison value has exactly 33 bytes: -version 1, big-endian instance ID, incarnation bytes and big-endian generation. -Unknown versions, zero identity fields and malformed keys are rejected. A -routing-map generation and a slot-authority generation are separate values. - -`batch_write_owned` compares the applied authority without mutating its fence -record. Business-record revision CAS remains independent and must be supplied -for conditional chunk/task transitions. Ordinary Put, Delete and Batch cannot -mutate either reserved ownership namespace. A conditional ownership batch -may change only the reserved key used as its revision precondition; slot -fences cannot be deleted or replaced by malformed authority values. - -Owner writes use concurrent atomic admission for one fence key and leader -tenure. A fence CAS closes that admission, drains previously admitted work -through apply, and then publishes the changed value. Other slots retain their -own admissions. Cancelling the caller does not release a pending proposal; -an unresolved proposal blocks handover and readiness until the leader's -recovery barrier resolves it. Topology replacement preserves the admission -state, while a new leader tenure requires its own recovery barrier. Delayed -writes retaining the old comparison value fail after fence publication. - -## 8. Durable handoff records - -Group zero stores one service handoff cohort at `/chunkdb/slot_handoff/service`. -`ChunkServiceHandoff` validates its source service generation, unchanged storage -generation, distinct moved slots, previous and target incarnations, selected -data groups and per-slot fence receipts. Target authority advances exactly one -generation; a missing previous authority is reserved for bootstrap without -admitted writers. Transfer and receipt ordering is canonical. - -The phases are Prepare, Fence, Publish and Activate. A cohort is reserved with -revision CAS on the complete service-map head; its unchanged head value and -the prepared record are written atomically. Advancing the head revision keeps -concurrent controllers from reserving different cohorts against the same source. -An unfinished cohort is resumed rather than replaced. - -Data-group fencing requires a persisted Fence-phase snapshot. An already -applied target identity is reconciled using its revision instead of issuing -another CAS. Otherwise the source value must match the recorded previous -identity. Unknown outcomes are reread through the KV recovery barrier, without -choosing another target or epoch. Before saving a new receipt, the client checks -the target authority and applied revision in its selected data group. - -Progress updates CAS the observed handoff revision. They cannot change cohort -identity, remove receipts or return to an earlier phase. Decoding rejects -duplicate slots, conflicting receipts and Publish/Activate records without -complete fence coverage. Publish cannot be saved as a standalone progress -update: it belongs in the same transaction as complete routing and authority -publication. Fixed-policy execution does not initiate handoff or activate -dynamic assignments. - -## 9. Submission epoch policy - -Dynamic service ownership is selected explicitly with `service_ownership = -"dynamic"`; fixed deployments keep their existing policy. Console deployment -uses `dynamic_ownership` to select the policy when initializing a new layout. -An initialized fixed deployment cannot silently switch policy. Storage groups -and DiskIO payload placement are unaffected by service ownership changes. - -- **I6 — Slot-local admission:** each execution captures its slot owner and - ownership epoch. Immediately before each client KV submission, RangeGuard - compares those values against its current immutable snapshot. Updating slot - 10 does not invalidate an execution on unchanged slot 5. Global publication - generation controls snapshot freshness; it is not a slot ownership epoch. -- **I7 — Epoch monotonicity:** owner change, regrant and same-ID restart advance - the affected durable slot epochs. Returning to a previous owner never reuses - an epoch. Incarnation remains diagnostic data in the existing authority - encoding, with no independent admission comparison. It cannot replace durable - epoch advancement. -- **I8 — Submission boundary:** entering client KV accepts that submission. - Accepted work may finish after ownership changes. The race between the final - local check and submission is intentional. No strict exclusion of old writes - after cutover is promised; existing business-record CAS governs conflicts. - -The complete service and authority maps share a publication generation. -Authority bitmaps are keyed by the canonical hexadecimal authority encoding -under `/chunkdb/slot_authority/`; their head is `/chunkdb/slot_head/authority`. -Publication CASes the service head and writes both complete tables and heads -atomically. Readers validate coverage, paired owner IDs and matching generations -and recheck the service-head revision. Epoch rollback and owner changes that -reuse an epoch are rejected. Lost publication replies reconcile the already -published map instead of incrementing epochs again. - -RPC dispatch captures authority before spawning asynchronous execution. Task -claims retain their capture through queueing, handler work, renewals and -completion. Maintenance operations capture the snapshot for their scan/work -iteration. Cache entries carry the captured owner/epoch, so a delayed old fill -cannot supply metadata to an execution under a newer epoch. Submission checks -use local constant-time lookup without network requests, new locks, sleeps or -old-write draining. The data-group fence/handoff APIs described above are -separate primitives and are not used by this policy; DiskDB fencing is preserved. - -An ownership rejection before submission is a routing signal. Clients refresh -the complete service snapshot and reexecute on its owner, including the same -process under a newer epoch. An old execution never adopts a newer epoch while -retaining its old state. The Rust RPC client keeps the same request correlation ID through explicit -ownership retries and refreshes immediately, under one ten-second operation -budget. Business operation IDs and revision CAS remain unchanged. Unknown -submitted transport outcomes return `OutcomeUnknown` for mutation callers to -reconcile; they never enter the ownership retry path. Read-only operations may -refresh and retry. - -The version-3 Group 0 monitor balances compatible prepared services advertised -under `/srv/chunkdb-epoch-v1/`. It uses the persisted descriptor's heartbeat and -failure grace bounds, stable instance-ID tie-breaking, a one-slot count -hysteresis and a 64-slot movement budget per tick. Missing observations retain -a durable first-suspected timestamp across monitor replacement; recovery clears -that timestamp so a later failure gets a fresh grace period. Only the exact -Group 0 leader tenure can publish a complete revision-conditional batch. The -fixed version-2 monitor continues to audit without reassigning slots. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb-rpc.md b/doc/design/chunkdb/design-crowdb-chunkdb-rpc.md deleted file mode 100644 index b8896359f..000000000 --- a/doc/design/chunkdb/design-crowdb-chunkdb-rpc.md +++ /dev/null @@ -1,111 +0,0 @@ - - - -# CROWDB - Design: ChunkdbService RPC - -Depends on: [RPC transport](../rpc/design-crowdb-rpc.md), -[ChunkDB lifecycle](design-crowdb-chunkdb.md). -Satisfies: ChunkDB control-plane transport, routing and mutation fences. - -ChunkDB uses the crowdb-rpc FlatBuffer transport for lifecycle, placement, -reservation, replacement, conversion and repair operations. Clients and services -use one current schema. Unknown messages and malformed requests fail explicitly. - -## Table of Contents - -- [1. Transport and routing](#1-transport-and-routing) -- [2. Wire records](#2-wire-records) -- [3. Mutation contracts](#3-mutation-contracts) -- [4. Dispatch and responses](#4-dispatch-and-responses) -- [5. Errors and retry](#5-errors-and-retry) - -## 1. Transport and routing - -- `ChunkdbClient` requires an `Arc` at construction. - `RangeBindingClient` resolves a chunk's service-slot owner from Group 0. - `ServiceRegistryClient` supplies live endpoints; endpoint cache refresh does - not change ownership. -- `ChunkdbRpcTransport` uses an immutable `ConnectionPoolIndex`. Cold connection - creation happens outside the index, followed by compare-and-swap installation. - Retry invalidates only the selected connection generation. -- `ChunkdbRpcService` dispatches to the independently scoped system/user-data - lifecycle and task runtimes. Each mutation checks current range authority. -- The RPC listener defaults to port 9961; HTTP health and metrics default to - 9972. The configured RPC address must parse as a socket address. - -## 2. Wire records - -- Requests start with `id` and `rpc_create_nano`, followed by operation fields. - Responses carry request identity, `ret_code`, optional `error_msg`, diagnostic - `range_start`/`range_end`, and typed result data. -- Message identities are defined by `FBMsgType`. Generated Rust/C++ definitions - derive from the same schema; native producers use the same request contract. -- `FBChunkType` requires explicit Wal, BtreePage, PageIndex, Stream, S3 or - IcebergTable purpose. Reserved zero is invalid. Chunk-ID purpose, stored type - and owner attribution must agree. Stream/Wal carry their logical Stream name; - other types have an empty owner key. -- `FBChunk` includes writer epoch, exact acknowledged byte cursor, lease deadline, - physical strips, coarse capacity/sealed geometry, cleanup intents and the last - replacement operation. `FBChunkStrip` identifies sequence, logical interval, - unit geometry, mirror/EC segments and unavailable segment identities. -- Segment identity includes disk, zone, unit range, allocation incarnation and - owning chunk. Mirror and EC bodies carry their explicit physical geometry. -- `QueryChunkResponse` also returns reader-layout validity duration. - `ListChunksResponse` carries a bounded chunk page and continuation identity. -- Response `Ref` wrappers parse one root and expose typed borrowed accessors. - Invalid buffers remain invalid; callers check validity and return codes before - accepting data. - -## 3. Mutation contracts - -- Allocation specifies purpose, writer epoch, lease and applicable owner key. - The active chunk and its finalization task are created atomically. Allocation - retries retain operation identity rather than create unrelated chunks. -- Append and strip reservation extend physical layout under their current fences. - Reservations remain hidden until confirmation. A consumed reservation prevents - conflicting seal transitions. -- `AdvanceChunkWrite` carries expected metadata revision, writer epoch, exact - byte cursor and optional closed-strip sequence. It cannot move progress backward - or beyond capacity. -- `SealChunkRequest.seal_bytes` is the exact effective byte boundary, including - zero for an empty seal. KiB geometry does not authorize extra readable bytes. - Native clients validate the returned identity, epoch, exact cursor and state. -- `ReplaceChunkStripRange` carries expected revision, exact old strips, replacement - strips and a stable operation ID. The lifecycle validates geometry and ownership, - commits new segments before publication, and retains retired segments through - cleanup grace. One-strip changes use this same fenced range contract. -- Replacement allocation and discard manage tentative segments. Conversion and - repair publish through their fenced lifecycle transitions and durable tasks. -- Delete publishes a tombstone with durable cleanup work before freeing blocks. - Whole-chunk retries are idempotent. `DeleteChunkRange` returns `Unimplemented` - until shared-object range reclamation is supported; overlap alone cannot free - a strip containing other live objects. - -## 4. Dispatch and responses - -- Each registered message has a request metric guard and typed handler. Dispatch - validates its FlatBuffer and required identities before spawning lifecycle work. -- Handlers execute on the service runtime and retain captured ownership checks - around external operations. Responses are submitted through the RPC server. -- Successful responses contain current typed state. Lifecycle errors map to a - typed return code; `NotMyRange` includes routing diagnostics for the client. -- Layout replacement, conversion and relocation share the `layout_mutation` - request metric family. Allocate, append, reservation, cursor, query, seal, - deletion, listing and ad-hoc recovery retain their own families. -- Startup validates topology and service/storage slot maps, constructs separate - domain runtimes, registers RPC handlers and starts discovery refresh plus HTTP - observability. Shutdown stops background work and transport through the runtime. - -## 5. Errors and retry - -- Invalid arguments and strip indices map to `InvalidArgument` or - `StripIndexOutOfRange`. Missing/existing chunks map to `NotFound`/`AlreadyExists`. -- Invalid lifecycle transitions map to `FailedPrecondition`; revision/operation - conflicts map to `Aborted`. Storage/allocation failures map to `Internal`. -- Lock admission failures map to `Unavailable`. Ownership failures return - `NotMyRange`, causing binding refresh and routing to the current owner. -- Client retries are bounded and apply only to transient failures. Typed fence - failures and malformed responses do not become successes. Mutations that may - have completed retain their operation identity and reconcile durable state. -- A transport response alone does not establish Stream writer ownership or a - synchronized DiskIO write; those contracts belong to their respective layers. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb.md b/doc/design/chunkdb/design-crowdb-chunkdb.md deleted file mode 100644 index 2e7a4997d..000000000 --- a/doc/design/chunkdb/design-crowdb-chunkdb.md +++ /dev/null @@ -1,1294 +0,0 @@ - - - -# CROWDB - Design: chunkdb (Overview) - -This is the root design document for the chunkdb component area. It -defines **what chunkdb is**, **why key choices were made**, and **how the -component is structured**. Field-level details live in the proto files -and Rust source; this doc covers decisions and architecture only. - ---- - -## Table of Contents - -- [1. Overview](#1-overview) -- [2. Non-Goals (Design Envelope)](#2-non-goals-design-envelope) -- [3. Key Design Decisions](#3-key-design-decisions) - - [3.1 Chunk as logical abstraction over disk blocks](#31-chunk-as-logical-abstraction-over-disk-blocks) - - [3.2 Strip as atomic redundancy unit](#32-strip-as-atomic-redundancy-unit) - - [3.3 Rack/node-aware placement for fault tolerance](#33-racknode-aware-placement-for-fault-tolerance) - - [3.4 Topology cache with group-0 integration](#34-topology-cache-with-group-0-integration) -- [3.4a Watch/notify for real-time updates](#34a-watchnotify-for-real-time-updates) - - [3.5 EC at strip level via isa-l (crowdb-common module)](#35-ec-at-strip-level-via-isa-l-crowdb-common-module) - - [3.6 Stateless with KV persistence](#36-stateless-with-kv-persistence) - - [3.7 Common protocol crate; crowdb-rpc](#37-common-protocol-crate-crowdb-rpc) - - [3.8 Proto types used directly; no Rust type duplication](#38-proto-types-used-directly-no-rust-type-duplication) - - [3.9 Chunk types for different use cases](#39-chunk-types-for-different-use-cases) -- [4. Architecture Overview](#4-architecture-overview) -- [5. Data Model](#5-data-model) - - [5.1 Disk block](#51-disk-block) - - [5.2 Strip](#52-strip) - - [5.3 Chunk](#53-chunk) - - [5.4 Chunk ID structure](#54-chunk-id-structure) - - [5.4a Logical hash bucket system](#54a-logical-hash-bucket-system) - - [5.4b Request handling during migration](#54b-request-handling-during-migration) - - [5.5 Chunk types](#55-chunk-types) -- [6. Topology Management](#6-topology-management) -- [7. Placement Strategy](#7-placement-strategy) - - [7.1 Mirror placement](#71-mirror-placement) - - [7.2 EC placement](#72-ec-placement) - - [7.3 Physical validation and degraded-placement repair](#73-physical-validation-and-degraded-placement-repair) - - [7.4 Active cross-domain rebalance](#74-active-cross-domain-rebalance) -- [8. Allocation Flow](#8-allocation-flow) -- [9. Chunk Lifecycle](#9-chunk-lifecycle) -- [10. Per-Chunk-ID Lifecycle Lock + Chunk Cache](#10-per-chunk-id-lifecycle-lock--chunk-cache) - - [10.1 ChunkLockMap](#101-chunklockmap) - - [10.2 Lock policy + cache hint](#102-lock-policy--cache-hint) - - [10.3 ChunkGuard](#103-chunkguard) - - [10.4 Chunk cache](#104-chunk-cache) - - [10.5 Sweep task](#105-sweep-task) - - [10.6 LifecycleHandler integration](#106-lifecyclehandler-integration) - - [10.7 Error variants + service mapping](#107-error-variants--service-mapping) - - [10.8 Metrics + HTTP endpoints](#108-metrics--http-endpoints) - - [10.9 Edge cases](#109-edge-cases) -- [11. EC Encoding/Decoding](#11-ec-encodingdecoding) -- [12. Crate Layout](#12-crate-layout) -- [13. Concurrency Model](#13-concurrency-model) -- [14. Configuration](#14-configuration) -- [15. Implementation Scope](#15-implementation-scope) -- [16. References](#16-references) - -## 1. Overview - -chunkdb is a **distributed chunk metadata service** that runs on top of -CROWDB's KV cluster and diskdb. It is a **lightweight, stateless server**: -a chunkdb instance manages chunk metadata, orchestrates block allocation -from diskdb, and persists all durable state to CROWDB KV. It holds no -state that cannot be reconstructed from KV — on crash or restart it -rebuilds in-memory structures from the KV records and group-0 metadata. - -Multiple chunkdb instances run across a cluster for fast chunk -allocation and lifecycle management. chunkdb does rack/node-aware -placement for fault tolerance, supports both mirror and EC strip types, -and manages chunk state transitions (allocate → seal → delete). All -state changes are durably persisted to CROWDB KV before being acknowledged -to callers. - -chunkdb manages chunk metadata and orchestrates background maintenance I/O -through DiskIO clients. Foreground callers write allocated blocks themselves -and tell chunkdb when chunks are sealed or deleted. - -ChunkDB conversion and EC repair share one `ConversionDiskIo` policy adapter -over the routed semantic `crowdb-diskio-client`. The adapter selects the -priority lane and expresses reads, buffered writes, and fsync barriers against -checked segment targets. It does not parse DiskIO endpoints, join topology, -construct RPC servers or connections, refresh connection vectors, or decode -wire return codes. Those responsibilities and their immutable generations are -owned by the DiskIO client; ChunkDB retains conversion/reconstruction policy -and decides when a completed write group requires a durability barrier. - -**Language:** Rust. **Runtime:** tokio (async everywhere). - -**Core goals:** -- **Fast allocation** — rack/node-aware placement with parallel block - allocation via diskdb, rollback on partial failure. -- **Fault tolerance** — mirror strips across distinct racks, EC strips - with rack-aware placement, configurable safe/unsafe modes. -- **Durability via CROWDB KV** — the paxos journal is the sole durable - store; chunkdb has no local WAL and is stateless on disk. -- **Crash safety via KV replay** — chunk metadata in KV is the source of - truth; in-memory structures (topology cache, allocator state) are rebuilt - on restart. -- **Flexible EC** — support multiple EC schemes simultaneously, selected - per-strip based on cluster size and fault tolerance requirements. - -**Design philosophy:** "chunkdb is a thin, stateless client of crowdb-paxos-kv and -diskdb." All consensus, replication, and durability are delegated to -crowdb-paxos-kv; block allocation is delegated to diskdb. chunkdb's job is chunk -lifecycle management, placement policy, and EC coordination, nothing more. - -## 2. Non-Goals (Design Envelope) - -- **No local block engine.** chunkdb does not own devices or implement block - reads and writes. Maintenance handlers route I/O through DiskIO services. -- **No local WAL.** CROWDB KV's WAL is the sole durability mechanism. -- **No consensus code.** chunkdb is a client of crowdb-paxos-kv; all interaction - uses the existing crowdb-paxos-kv API. -- **No GC operations in v1.** Garbage collection (reclaim, collapse, merge) - is deferred to a future requirement. Chunks are allocated and deleted as - whole units in v1. -- **No EC strip type restrictions.** Chunk type (Wal/BtreePage/PageIndex/Stream/S3/IcebergTable) - is independent of strip type (mirror/EC). Any chunk can use either strip type - based on configuration. v1 supports both mirror and EC strips for all chunk types. -- **No native SMR / zoned-namespace SSD support (v1).** Start with - conventional HDD/SSD via diskdb; advanced disk features are deferred. - -## 3. Key Design Decisions - -### 3.1 Chunk as logical abstraction over disk blocks - -A **chunk** is a logical container for strips, which in turn contain -physical disk blocks. This abstraction layer allows upper-layer services -to work with chunks without worrying about physical placement details. -chunkdb handles the mapping from logical chunk offsets to physical disk -blocks across multiple nodes and racks. - -**Rationale:** Separates logical data management from physical storage -layout. Upper layers (object store, file system) work with chunks; chunkdb -handles the complexity of block allocation, placement, and fault tolerance. - -### 3.2 Strip as atomic redundancy unit - -A **strip** is the atomic unit of redundancy. Every strip is either a -**mirror strip** (replicated across multiple nodes) or an **EC strip** -(erasure-coded with data + parity blocks). Strip-level operations -(allocation, EC encoding, recovery) are atomic. A strip is either fully -allocated or not at all. - -**Rationale:** Simplifies recovery and placement logic. When a disk fails, -recovery operates at strip level (rebuild lost replica or EC parity), -not at individual block level. - -### 3.3 Rack/node-aware placement for fault tolerance - -Placement decisions are rack-aware and node-aware to survive rack failures -and node failures. Mirror strips place replicas on distinct racks when -possible. EC strips distribute data+parity blocks across multiple racks, -with per-node block limits to ensure no single node failure exceeds the -code_num threshold. - -**Rationale:** Rack failures are more common than individual disk failures. -Rack-aware placement ensures the system can survive rack-level failures -without data loss. - -### 3.4 Topology cache with group-0 integration - -chunkdb maintains a **TopologyCache** with site/rack/node/disk-group -hierarchy, refreshed periodically from group-0 via `HardwareClient`. This -cache is used for placement decisions and is point-in-time consistent for -each allocation operation. - -**Rationale:** Placement decisions need up-to-date cluster topology. -Querying group-0 for every allocation would be too slow. Caching with -periodic refresh provides a good balance between freshness and performance. - -### 3.4a Watch/notify for real-time updates - -chunkdb uses crowdb-paxos-kv's watch/notify mechanism for real-time topology updates -in addition to periodic refresh: - -- **Watch registration**: chunkdb registers for group-0 watch on disk-group - and node status keys via the crowdb-paxos-kv watch/notify client. -- **Immediate updates**: A notification coalesces into a complete group-0 - topology rebuild. The immutable snapshot is published with one `ArcSwap` - operation, so allocation never observes a mixture of generations. -- **Fallback mechanism**: Periodic refresh (30s default) continues as a fallback - for missed notifications and consistency verification. -- **Placement impact**: Real-time updates enable placement decisions to reflect - current cluster state, avoiding placement on failed/degraded resources. - -**Rationale**: Watch/notify provides sub-second responsiveness to critical -cluster state changes (disk failures, maintenance) while periodic refresh -ensures eventual consistency and recovery from missed notifications. - -### 3.5 EC at strip level via isa-l (crowdb-common module) - -Erasure coding is performed at the strip level using the **isa-l** library -(Intel Storage Acceleration Library). A Rust FFI wrapper in `crowdb-common` -provides safe interfaces to isa-l's encode/decode functions. EC encoding -happens when strips are allocated; EC decoding happens during recovery. - -**Rationale:** isa-l is the industry standard for high-performance EC. -Strip-level EC aligns with the strip-as-atomic-redundancy-unit design. -The FFI wrapper isolates unsafe C code from the rest of the Rust codebase. - -### 3.6 Stateless with KV persistence - -chunkdb is stateless on disk. All chunk metadata and maintenance state is persisted to selected nonzero -direct Paxos KV groups by ChunkId slot. System and user-data lifecycle/task -runtimes have separate authority, queues, and execution capacity. -On restart, chunkdb rebuilds in-memory structures (topology cache, -allocator state) by replaying KV records and fetching fresh topology from -group-0. - -**Rationale:** Simplifies operations (no local data to manage), enables -crash recovery without local WAL, and relies on CROWDB KV's durability -guarantees. Restart under the same logical instance reloads durable state. -Changing ownership requires a fenced handoff; adding storage groups requires -metadata migration. - -### 3.7 Common protocol crate; crowdb-rpc - -Protocol definitions live in `lib/crowdb-protocol/src/fbs/chunkdb_*.fbs` -and are shared between chunkdb server and chunkdb client. Communication -uses crowdb-rpc (flatbuffer transport), the same engine as the KV -consensus hot path. See sub-design -[`design-crowdb-chunkdb-rpc.md`](design-crowdb-chunkdb-rpc.md). - -**Rationale:** Single source of truth for protocol definitions. crowdb-rpc -uses the same engine as the KV consensus hot path for lower latency. - -### 3.8 Proto types used directly; no Rust type duplication - -Rust code uses the proto-generated types directly from `crowdb-protocol`. -No separate Rust struct definitions mirror the proto types. Conversion -between proto and domain types happens only at boundaries (e.g., KV -persistence). - -**Rationale:** Reduces duplication and drift. Proto is the canonical -definition; Rust code works with proto types directly. - -### 3.9 Chunk types for different use cases - -Six concrete chunk types are supported. Repo chunk is the collective -discussion term for user data, not an implementation type: -- **WAL chunk**: Write-ahead log entries. -- **BTree page chunk**: B-tree page storage for the crowdb-tree engine. -- **Page index chunk**: Persistent tree page mapping payload. -- **Stream chunk**: Non-system business stream data. -- **S3 chunk**: S3 object data. -- **Iceberg table chunk**: Iceberg immutable file data. - -**Rationale:** Different storage components have different redundancy and -performance requirements. Chunk types allow optimization for each component's -needs. Chunk type is independent of strip type. Any chunk type can use -either mirror or EC strips based on configuration and access patterns. - -## 4. Architecture Overview - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ chunkdb Server │ -├─────────────────────────────────────────────────────────────────┤ -│ crowdb-rpc Service (ChunkdbService) │ -│ ├── AllocateChunk │ -│ ├── SealChunk │ -│ ├── DeleteChunk │ -│ ├── QueryChunk │ -│ └── ListChunk │ -├─────────────────────────────────────────────────────────────────┤ -│ Lifecycle Layer │ -│ ├── allocate_chunk() │ -│ ├── seal_chunk() │ -│ └── delete_chunk() │ -├─────────────────────────────────────────────────────────────────┤ -│ Allocation Layer │ -│ ├── ChunkAllocator (orchestrate strip allocation) │ -│ ├── MirrorStripAllocator │ -│ └── ECStripAllocator │ -├─────────────────────────────────────────────────────────────────┤ -│ Placement Layer │ -│ ├── RackNodeSelector │ -│ ├── MirrorPlacement (distinct racks) │ -│ └── ECPlacement (rack-aware, safe/unsafe) │ -├─────────────────────────────────────────────────────────────────┤ -│ Topology Layer │ -│ ├── TopologyCache (site/rack/node/disk-group) │ -│ ├── TopologyRefresh (periodic from group-0) │ -│ └── WatchNotify (real-time updates from group-0) │ -├─────────────────────────────────────────────────────────────────┤ -│ Storage Layer │ -│ ├── KV Persistence (chunk metadata) │ -│ └── DiskdbClient Pool (block allocation/free) │ -├─────────────────────────────────────────────────────────────────┤ -│ EC Layer │ -│ └── crowdb-common EC (isa-l FFI wrapper) │ -└─────────────────────────────────────────────────────────────────┘ - │ │ │ - ▼ ▼ ▼ - ┌─────────┐ ┌─────────┐ ┌─────────┐ - │ crowdb-paxos-kv │ │ diskdb │ │ group-0 │ - │ (KV) │ │ │ │Hardware │ - └─────────┘ └─────────┘ └─────────┘ -``` - -**Key components:** - -- **crowdb-rpc Service**: External API for chunk operations. -- **Lifecycle Layer**: Implements chunk state machine transitions. -- **Allocation Layer**: Orchestrates strip allocation with rollback on failure. -- **Placement Layer**: Rack/node-aware placement algorithms. -- **Topology Layer**: Caches and refreshes cluster topology from group-0. -- **Storage Layer**: KV persistence for chunk metadata, diskdb client pool. -- **EC Layer**: isa-l wrapper for strip-level EC operations. - -## 5. Data Model - -### 5.1 Disk block - -A **disk block** is the smallest fixed-size physical storage unit managed -by diskdb. The default size is **1 MB** (configurable, range 1 MB – 2 MB). -Disk blocks are identified by `Segment { node_id, disk_id, zone_index, -zone_offset, size, tag }` (from diskdb proto). - -### 5.2 Strip - -Current Strip metadata has one physical `unit_kb` shared by all fragments. -Chunk placement therefore requires a uniform allocation unit across active -disks. Independent DiskDB groups may use different units, but Console rejects -ChunkDB/Chunk-KV deployment against mixed active units before spawning a -process. Runtime topology records incompatible geometry and rejects normal, -replacement and conversion allocation before any DiskDB fragment request. -It must not silently reuse the first disk's unit for other groups. - -A **strip** is the atomic redundancy unit. Two strip types: - -**Mirror Strip**: A configured number of disk allocation units replicated -across N nodes (configurable copy count, production default 2). Each segment -is a full copy on a different node. Data capacity = unit_count × unit_size. - -**EC Strip**: `data_num` data segments + `code_num` parity segments, -distributed across nodes under the failure-domain placement rule. Data -capacity = `data_num × unit_count × unit_size`. For example: -- 6+3 EC with 1 MB blocks → 6 MB data capacity, 9 MB total. -- 8+4 EC with 1 MB blocks → 8 MB data capacity, 12 MB total. - -Each strip tracks: -- **Usage bitmap**: Per-strip bitmap tracking which sub-block ranges are - written (1 bit per granularity, e.g., 16 KB). -- **EC state** (EC strips only): `NotStarted` → `Complete`. -- **Block state**: Per-block health (Good, Suspicious, Bad). -- **Placement intent and assessment**: The selected `rack_first` or - `node_first` priority and the creation-time maximum fragment count for each - rack, node, and physical disk. The assessment records whether each domain - satisfies the strip loss budget. EC strips that violate the node or disk - loss budget carry a durable placement-repair marker. Rack diversity is a - preference: missing rack protection alone does not require repair. Existing - markers are cleared after physical reassessment confirms node and disk - protection; the rack-protection assessment remains truthful. - -### 5.3 Chunk - -A **chunk** is a container for strips. Chunk properties: -- **128-bit ID**: Chunk type (8 bits) + Timestamp (48 bits) + Randomness (72 bits). -- **State**: `Init` → `Active` → `Sealed` → `Deleted`. -- **Type**: Wal, BtreePage, PageIndex, Stream, S3, or IcebergTable. - The ID's high-byte prefix and the stored type must agree. Retired type-0 - records are rejected; legacy conversion is unsupported. -- **Capacity**: Total data capacity across all strips. -- **Write granularity**: Minimum write alignment (e.g., 4 KB). -- **Strips**: Ordered list of strips (mirror or EC). -- **Next strip sequence**: Monotonic identity consumed by append and never - derived from the current vector length. -- **Cleanup intents**: Retired segment sets and their earliest safe reuse time. -- **Last strip replacement**: Stable operation identity for idempotent retry. -- **Logical-to-physical mapping**: Encoded offset arrays for GC (deferred). - -Each strip may also identify replicas known unavailable. Readers can avoid -those segment identities while background recovery is pending. - -Chunk size is variable, determined by the total size of its constituent -strips (1 MB – 4 GB range). - -### 5.4 Chunk ID structure - -The 128-bit chunk ID uses a custom format optimized for chunkdb: - -| Bits | Width | Field | Description | -|---------|--------|----------|--------------------------------------| -| 0–7 | 8 bits | Chunk Type | Chunk type identifier (0-255) | -| 8–55 | 48 bits| Timestamp | Milliseconds since Unix epoch | -| 56–127 | 72 bits| Randomness | Cryptographically secure random | - -**Layout benefits:** -- **Chunk type prefix**: First byte enables efficient filtering and scanning by chunk type -- **Time ordering**: 48-bit timestamp provides natural ordering for sequential operations -- **Global uniqueness**: 48-bit timestamp + 72-bit randomness provides sufficient uniqueness -- **Compact size**: 128 bits total, same as standard UUID - -**Generation**: Chunk IDs are generated using `getrandom` for cryptographically secure randomness combined with system timestamp. The generation function will be added to `crowdb-common` for reuse across components. - -The first byte has an explicit supported purpose (see §5.5). Value zero is -retired and cannot be allocated or silently reclassified. - -### 5.4a Logical Hash Bucket System - -The fixed logical hash space contains 1024 slots. XXH64 seed zero hashes the -canonical big-endian 128-bit ChunkId, modulo 1024. Group/server counts never -change this calculation. Group 0 holds one 128-byte bitmap per service instance -and one per selected nonzero storage group, under independent generation heads. - -Clients resolve slots to ChunkDB instances; ChunkDB independently resolves -slots to direct Paxos KV groups. All canonical chunk/task/index/reservation -records follow the owning ChunkId route and preserve group-local conditional -atomic writes. Payload remains on DiskIO. A service owner can span groups. - -See [fixed slot routing](design-crowdb-chunkdb-range-binding.md) for publication, -cache costs, task-domain scope, startup validation, and immutable assignments. - -### 5.4b Request Handling During Migration - -Live service handoff, storage-group add/remove/remap, and legacy conversion are -unsupported. Initialized maps reject ownership changes. Same-owner restart and -endpoint refresh preserve authority. A storage ownership change requires complete -chunk/task/index/reservation migration and a fenced cutover before bitmap changes; -a service ownership change requires task-authority handoff without metadata copy. -There is no steady-state group-0 destination or old-owner mutation fallback. - -### 5.5 Chunk types - -| Type | Chunk Type Value | Description | -| ------------- | ---------------- | ------------------------------------ | -| Retired | 0 | Rejected; never reused | -| WAL | 1 | Write-ahead log entries | -| BTree page | 2 | B-tree page storage | -| Page index | 3 | Page index metadata | -| Stream | 4 | Native chunk streams | -| S3 | 5 | S3 object data | -| Iceberg table | 6 | Iceberg immutable file data | -| Reserved | 7-255 | Reserved for future use | - -**Note:** Chunk type is independent of strip type. Any chunk type can use -either mirror or EC strips based on configuration and requirements. - -**Metadata chunks**: Contain only mirror strips for maximum redundancy and -fast access. Used for critical metadata. - -**Repo chunks**: Used for user data storage. Can use either mirror or EC strips -based on configuration and access patterns. - -**WAL chunks**: Used for write-ahead log entries. Typically use mirror strips -for durability and fast recovery, but EC is possible for space optimization. - -**BTree page chunks**: Used for B-tree page storage in the crowdb-tree engine. -Can use mirror or EC strips based on performance vs space trade-offs. - -**Page index chunks**: Used for page index metadata. Typically use mirror strips -for fast lookup, but EC is possible for large indexes. - -## 6. Topology Management - -chunkdb maintains a **TopologyCache** with the cluster hierarchy: - -``` -Site (data center) - └── Rack (physical rack) - └── Node (server) - └── Disk-Group (logical disk group) - └── Disk (physical disk) -``` - -**TopologySnapshot**: Point-in-time immutable snapshot containing: -- `sites`: Map -- `racks`: Map -- `nodes`: Map -- `disk_groups`: Map -- healthy physical-disk locations, allocatable capacity observations, and a - monotonic publication generation. - -Located DiskIO registrations overlay hardware intent with current placement -availability. A configured disk group is eligible only while a matching live -process reports it opened. Retained expired registrations exclude stopped -physical owners; expiry is evaluated at selection time even without a refresh. -This does not change durable hardware status or relax failure-domain constraints. - -Usage observations include only allocatable disks and carry their sample time. -Missing or stale observations remove capacity ranking information; they never -relax a rack, node, or disk safety constraint. Per-disk-group in-flight bytes -are atomic reservations shared by snapshot generations, so concurrent -allocations account for projected utilization without a placement lock. - -**TopologyRefresh**: Hybrid approach combining periodic refresh with watch/notify: -- **Periodic refresh**: Background task that periodically (configurable, - default 30s) fetches full topology from group-0 via `HardwareClient` for - consistency and missed notification recovery. -- **Watch/notify integration**: Register for group-0 watch on disk-group and - node status keys via crowdb-paxos-kv watch/notify client. Notifications coalesce - into a complete refresh; no point mutation is published. -- Fetch all nodes with status and rack assignment -- Fetch all disk-groups with capacity and node assignment -- Fetch all racks with site assignment -- Atomically update the cache - -**Rationale**: Placement decisions need consistent topology. Point-in-time -snapshots ensure a single allocation operation sees a consistent view. -Periodic refresh balances freshness and performance. - -## 7. Placement Strategy - -Placement has two independent parts: safety and ranking. Safety first limits -how many fragments one failure domain may contain. Ranking then compares -projected usable utilization, `(used + in_flight + planned) / capacity`, only -among candidates that meet that safety constraint. Equal scores use stable -topology identifiers, making retries deterministic. - -`deployment.mode` is `production` or `test_single_node` in release builds. Production startup -requires at least three distinct voting nodes for every KV group and protected -placement, with `deployment.max_node_failures = 1`. Test-single-node startup -requires one voting node per group and `deployment.max_node_failures = 0`, with -explicit colocated placement. The mode never changes in response to topology -loss. In test-single-node mode, new strips must be one-copy 1 MiB mirrors; -EC, extra copies, and mirror-to-EC conversion are rejected. Production rejects -new mirror strips with fewer than two copies. The production profile normally -places two mirror copies on distinct nodes, including for journal and tree-page -data. - -Debug builds also accept `test_unsafe_placement` for legacy colocated EC -integration fixtures. Release builds reject it during configuration loading; -it is separate from the single-node deployment profile. - -`placement.mode` selects one placement strategy at process construction. The -allocator depends on the `ChunkPlacementStrategy` interface and does not branch -on the mode while allocating, converting, repairing, or deciding whether a -degraded disk result may be published. Each mode is a separate strategy type: - -- `protected` uses failure-domain-aware mirror and EC selectors. The granular - degraded-placement settings below remain available only within this mode. -- `unsafe_colocated` selects one healthy disk group for the explicit - test-single-node deployment. This deployment uses one mirror copy and has - no data protection; a read or write error reaches the caller. - -The mode is an explicit deployment property, not an automatic fallback. A -protected deployment never changes to `unsafe_colocated` because topology is -small or unavailable. With two healthy nodes remaining, a new EC strip retains -its requested data and parity geometry. Its fragments may span both survivors -with a degraded-placement marker and a durable repair task. A new mirror strip -still places two copies across the survivors. New placement -policies are added as strategy implementations and selected at the composition -root, keeping policy branches out of the allocation hot path. - -For an EC `data_num + code_num` strip, a protected rack, node, or physical -disk contains at most `code_num` fragments. For a mirror strip, losing a -protected domain leaves at least one copy. A single-copy mirror is explicitly -non-redundant and remains allocatable while its assessment reports no domain -protection. - -`placement.failure_domain_priority` selects `rack_first` (the default) or -`node_first`. It orders safe candidates lexicographically; it does not change -the node/disk recovery limits. Rack diversity is a preference: normal placement -uses distinct racks when available and distinct nodes within one rack otherwise. -An unmet rack guarantee is reported in the assessment and does not reject the -allocation or EC publication. `placement.allow_degraded_failure_domains` is -required to relax node protection; EC placement exceeding its node or disk -recovery budget additionally requires `placement.allow_unsafe_ec`. - -### 7.1 Mirror placement - -Mirror placement aims to place replicas on **distinct racks** to survive -rack failures: - -1. Select N nodes (copy count) from online nodes -2. Prefer nodes on different racks -3. If not enough distinct racks, place multiple copies on same rack -4. Rank racks, nodes, then disk-groups by projected allocatable utilization -5. Return a plan containing its policy, topology generation, usage freshness, - and planned rack/node protection - -**Negative hints**: Nodes can be excluded from placement (e.g., during -recovery to avoid re-using failed nodes). - -**Example**: 2-copy mirror on 3-rack cluster → 2 replicas on 2 distinct racks. -If fewer distinct healthy nodes than copies are available, normal placement -returns a typed failure before any DiskDB allocation. A one-rack result retains -node protection and explicitly reports that it cannot survive a whole-rack loss. - -### 7.2 EC placement - -EC placement distributes data+parity blocks **across multiple racks** -with per-node block limits: - -1. Calculate total blocks = data_num + code_num -2. Select nodes such that: - - No node has > code_num blocks (safe mode) - - Each protected rack and physical disk also has at most `code_num` blocks -3. Rank the requested primary domain, secondary domain, and disk-group by - projected allocatable utilization -4. Return a plan with the requested policy and planned protection assessment - -**Safe mode**: Ensures no single node failure exceeds code_num, guaranteeing -recoverability. Requires enough nodes to satisfy constraints. - -Within `protected` mode, `placement.allow_unsafe_ec` permits a degraded EC plan -that exceeds the normal recovery budget when the cluster is too small. It does -not select colocated placement. Insufficient topology otherwise returns a typed -placement error without allocating blocks. - -**Example**: 8+4 EC on a healthy 3-node cluster places 12 fragments with at -most 4 per node, so loss of any one node leaves 8 fragments. If one node is -already unavailable, the same geometry may span the two survivors and is -marked for placement repair when the third node returns. - -### 7.3 Physical validation and degraded-placement repair - -DiskDB remains the authority for physical allocation. After every allocation, -ChunkDB resolves returned `disk_id`s through the same immutable topology and -computes the actual rack, node, and disk maxima. A physical-disk violation is -rolled back and retried against an alternative disk-group; it is never silently -published. The assessment, effective policy, and any EC repair marker are -persisted with the strip and carried through the RPC and storage codecs. - -Every marked EC strip has one deterministic persistent placement task. Task -admission happens after foreground publication and a bounded metadata scan -recreates a missing task after an interrupted admission. The task waits with -backoff while topology cannot improve; it does not become terminal merely -because the cluster is undersized. Once a better destination exists, it reads -one source fragment, writes and fsyncs one replacement fragment, validates the -new physical layout, and atomically replaces only that fragment. The old -fragment remains under the normal layout-validity cleanup fence. The marker is -cleared only after all recorded guarantees are satisfied. - -### 7.4 Active cross-domain rebalance - -ChunkDB consumes live DiskDB usage summaries and runs a deliberately low-rate -planner. It requires utilization skew to remain above the configured threshold -for the full hysteresis window, scans a bounded chunk page, skips strips with -placement repair already pending, and admits exactly one fragment move per -cycle. Missing or stale summaries disable capacity ranking, never physical -safety checks. - -For a candidate move, ChunkDB allocates one exact tentative target on the cold -disk-group, substitutes it into a copy of the strip, and recomputes rack, node, -and physical-disk assessment. It frees the candidate and emits no handoff if -any protection flag would become false or any recorded domain maximum would -increase. - -An accepted candidate is sent to the target DiskDB through -`ExecuteRelocation`. DiskDB durably adopts the exact source/target pair, copies -and fsyncs data, and asks the current ChunkDB range owner to claim the move. -ChunkDB persists a deterministic relocation task before acknowledging it. The -task conditionally publishes only when the exact source and expected chunk -revision still match. Duplicate delivery observes the same task or installed -target. ChunkDB first confirms the target after its successful CAS; DiskDB -verifies and idempotently confirms that exact target before freeing the source -after `Published`. ChunkDB never frees the source from this path. - -The task checkpoint makes the tentative target visible to DiskDB's owner -scanner as `TaskPending`. Current metadata reports `Referenced`; a deleted or -superseded owner reports `Absent`. This joins crash recovery and ordinary -relocation under one ownership decision instead of treating target age as -permission to reclaim data. - -## 8. Allocation Flow - -**AllocateChunk** operation: - -1. Generate 192-bit chunk ID (metadata + UUID + reserved) -2. Fetch topology snapshot from TopologyCache -3. Calculate strip layout based on requested capacity and strip type -4. For each strip: - - Call placement selector to get node/disk-group assignments - - Group placements by DiskDB data group and allocate each group in parallel - - On failure: rollback (free all allocated blocks), return error -5. Persist the complete chunk metadata with state = Active -6. Start grouped DiskDB block commits asynchronously -7. Refresh the chunk cache and return the Active chunk to the caller - -**Parallel allocation**: Strips remain sequential. Within a strip, ChunkDB -sends one request per DiskDB data group and runs those requests concurrently. -DiskDB persists each allocated busy block as Tentative before responding. - -**Allocation routing failure**: Only a typed connection-establishment failure -before RPC submission permits rerouting the same allocation. ChunkDB refreshes -the service registry and makes at most one attempt at a different endpoint. -If discovery fails, the endpoint is unchanged, or no owner is advertised, it -returns the original failure. Any failure during RPC submission or while awaiting -the reply remains ambiguous and is never replayed by the allocator. Discovery -may refresh for the next independent request. Successful allocation adds no -registry read or lock beyond the existing periodic discovery policy. - -**Success boundary**: The Active chunk and every referenced Tentative busy -block are durable before ChunkDB returns success. DiskDB commit changes the -matching busy records to Committed in one ordinary batch write. A -reconciliation scanner can resolve a crash in this interval from the Active -chunk reference and allocation incarnation. - -**Rollback**: If allocation or Active metadata persistence fails, every known -segment from every prior strip is freed through its exact DiskDB group before -returning. A failed rollback is surfaced. Background commit failure is -reported by metrics and logs for later reconciliation; it does not retract an -already durable Active chunk. - -## 9. Chunk Lifecycle - -``` -Init ──> Active ──> Sealed ──> Deleted -``` - -| State | Description | -|---------|--------------------------------------------------------------| -| Init | Reserved transient state; the allocation path publishes Active directly. | -| Active | Chunk is open for writes. Strips can be appended. Returned by AllocateChunk. | -| Sealed | Chunk is read-only. Records final length and seal timestamp. | -| Deleted | Durable cleanup intent; retained segments still need freeing. | - -**State transitions**: -- `Active → Sealed`: Via `SealChunk` RPC. Validates state and records the exact - `seal_bytes` boundary in `acknowledged_cursor`, including zero for an empty - chunk. The RPC has one exact-byte form. KiB `sealed_length` fields summarize - geometry and do not enlarge the valid byte range. Finalization preserves the - exact acknowledged cursor. Data alignment and physical block allocation never - add logical bytes to the seal boundary. -- `Active → Deleted`: Persist cleanup intent, free segments, persist tombstone. -- `Sealed → Deleted`: Persist cleanup intent, free segments, persist tombstone. -- Invalid transitions (e.g., `Sealed → Active`) return errors. - -**Concurrency**: An approved bounded `tokio::Mutex` per chunk serializes the -read-modify-write lifecycle. Locks for different chunks are independent and -tests prove cross-chunk progress. Immutable topology, routing, range, endpoint, -and disk-group reverse-map snapshots remain lock-free on their read paths. - -### 9.1 Public frames and active-chunk liveness - -Every persistent chunk user stores public, self-validating frames. The frame -header names a versioned kind and payload bounds; its footer binds CRC32C and -chunk ID. Locations retain only chunk ID, physical frame start, and logical -length, so integrity state is never duplicated in high-cardinality metadata. -Readers verify a complete frame before returning a payload. A failed -verification excludes the serving mirror or shard and uses the existing -protection-read and repair path. - -An Active chunk has exactly one durable `FinalizeChunk` liveness task. Chunk -creation atomically creates it with a 15-minute deadline. The owner renews it -on the 12-minute cadence by conditionally replacing the one deadline-index -entry and task record in one task-partition transaction; no second expiry is -stored in chunk metadata. The owner self-fences before that authority can -expire. The task scanner reads due task-index entries only, never scans the -chunk table. After claim and the shared request-age/skew safety window, it -requires ordered synchronization proof from every segment in the captured -layout, including ordinary empty chunks and unavailable mirror identities. -Failure retains the layout and retries without scanning or freeing blocks. -It then observes content using at-most-1-MiB windows and the shared bounded frame -parser. Stream/Wal scans include allocated capacity beyond the persisted -checkpoint and commit only complete append groups. Ordinary object publication -already persists its physical cursor; its finalization validates that prefix. -An exact-capacity end needs no extra header request. Drain and scan share one -30-second IO budget; drain has at most its first half. Observation and comparison -reserve time within the remainder rather than restarting a timeout per copy. - -Available mirror candidates run concurrently within one IO deadline. Select -the shortest healthy complete prefix and verify agreement on overlapping -contents. Failed, partial or corrupt observations do not become shorter -candidates; valid conflicting contents fail explicitly. No readable candidate -means retry while retaining the Active layout, including a Stream/Wal with a -zero checkpoint. A scan must cover the persisted `acknowledged_cursor` before -sealing or deleting. The task writer generation must match the chunk epoch. -Sealing compares the exact captured chunk metadata while holding its lifecycle -guard; a concurrent revision/layout change forces another observation. Empty -chunks first seal through that same fence, then resume tombstone publication -after restart without accessing physical segments that were already retired. -Finalization uses this durable task path exclusively; an expired metadata -checkpoint cannot independently authorize sealing. A restarted owner always -allocates a new Active chunk. - -## 10. Per-Chunk-ID Lifecycle Lock + Chunk Cache - -`LifecycleHandler`'s six mutating RPCs (allocate/append/seal/delete/ -delete-range/update-strip) each perform a read-modify-write cycle on a -chunk record. Without per-chunk serialization, two concurrent -`AppendChunk` RPCs on the same chunk ID both read the chunk, both -append strips, both `put_chunk`. The second overwrite loses the -first's strips. A per-chunk mutex serializes the RMW cycle. A payload -cache avoids the `get_chunk` store round-trip on every mutating RPC -(the latest chunk is known in-process right after the previous -`put_chunk`). - -The lock and the payload have different eviction requirements (lock: -evict only when uncontended; payload: evict freely by -recency/frequency), so they are separate structures. See -[`design-crowdb-chunkdb-range-binding.md`](design-crowdb-chunkdb-range-binding.md) -for the range guard that ensures a chunk reaches exactly one chunkdb -instance. The per-chunk lock assumes that one-owner invariant. - -### 10.1 ChunkLockMap - -`app/crowdb-chunkdb/src/lifecycle.rs`: - -```rust -pub struct ChunkLockMap { - locks: SkipMap<(u64, u64), Arc>, - chunks: Arc>, - metrics: Arc, - hold_warn_threshold: Duration, -} -``` - -- `new(cache_capacity, metrics, hold_warn_threshold) -> Self` — creates - an empty ordered lock-free index and a `Cache::new(cache_capacity)`. -- `acquire(&self, chunk_id, store, policy, hint) -> Result` — for existing chunks (append/seal/delete). Steps: - 1. `get_or_insert` an immutable `ChunkLockEntry`, atomically retain its user - count, and recheck that the exact entry is still current. - 2. Record lock-wait start time. - 3. Acquire the mutex per `policy`: - - `TryLock` → `try_lock_owned()`. On `Err`, increment - `lock_busy_count`, return `LockBusy`. - - `Wait(d)` → `lock_owned()` with `tokio::time::timeout(d, ...)`. - On timeout, increment `lock_timeout_count`, return - `LockTimeout`. - 4. Record lock-wait time into the histogram. - 5. Serve payload: if `hint == Cache` and `self.chunks.get(&chunk_id)` - returns `Some(chunk)` → cache hit (increment `cache_hit_count`), - return guard with that chunk. Otherwise → cache miss (increment - `cache_miss_count`), `store.get_chunk(chunk_id)`. On - `StoreError::ChunkNotFound` → return `ChunkNotFound`. On success, - if `hint == Cache`, `self.chunks.insert(chunk_id, chunk.clone())`. - 6. Return `ChunkGuard` with the exact retained lock entry, mutex guard, - chunk, hint, hold start, and metrics. -- `acquire_for_create(&self, chunk_id, policy, hint) -> Result` — for `allocate_chunk` with caller-supplied ID. Same - lock acquisition as `acquire` but does NOT fetch from store (chunk - does not exist yet). Returns a guard with `chunk: None`. Caller must - `refresh()` after creating the chunk. -- `populate_cache(&self, chunk_id, chunk)` — for `allocate_chunk` with - auto-generated ID (skips the lock; UUID collision negligible). -- `reap_idle(&self)` — scans the ordered index and changes an entry's user - state from zero to `REAPING` before removing that exact entry. An owner, - waiter, or acquisition in progress has already retained the entry and - prevents this transition. Increments `reap_idle_count` and - `reap_idle_entries_removed`; the separately bounded payload cache is - untouched. -- `invalidate_chunk(&self, chunk_id) -> bool` — calls - `self.chunks.remove(&chunk_id).is_some()`. Increments - `invalidate_count`. Used by range migration. -- `invalidate_range(&self, bucket_start, bucket_end) -> u32` — iterates - cache entries and removes those whose chunk ID hashes to a bucket in - `[bucket_start, bucket_end]`. Returns the count removed. Increments - `invalidate_count` by the count. O(n) on cache size — acceptable for - rare range migrations. -- `cache_len() -> usize`, `metrics_snapshot() -> LifecycleMetricsSnapshot`. - -### 10.2 Lock policy + cache hint - -```rust -pub enum LockPolicy { - TryLock, - Wait(Duration), -} - -impl Default for LockPolicy { - fn default() -> Self { Self::Wait(Duration::from_secs(10)) } -} - -pub enum CacheHint { - Cache, - NoCache, -} - -impl Default for CacheHint { - fn default() -> Self { Self::Cache } -} -``` - -- `LockPolicy::TryLock` → fail fast with `LockBusy` on contention. -- `LockPolicy::Wait(d)` → park the task up to `d`, then `LockTimeout`. - No `WaitForever` variant — a bounded wait prevents indefinite stalls. -- `CacheHint::Cache` (default) → populate cache on miss, write to cache - on `refresh`. -- `CacheHint::NoCache` → skip cache population; the guard's local copy - is still updated so the current operation sees the chunk. - -All mutating RPCs use `LockPolicy::default()` and `CacheHint::Cache`. -These are internal in v1, not exposed in the RPC API. - -### 10.3 ChunkGuard - -```rust -pub struct ChunkGuard { - lock: ChunkLockHandle, // retains the exact index entry - guard: OwnedMutexGuard<()>, // held for Drop — releases the lock - chunk: Option, - hint: CacheHint, - chunk_id: ChunkId, - hold_start: Instant, - metrics: Arc, - hold_warn_threshold: Duration, -} -``` - -- `chunk(&self) -> Option<&Chunk>` — returns the latest chunk record. - `None` for `acquire_for_create` before `refresh`. -- `refresh(&mut self, chunk: Chunk)` — updates the guard's local copy. - If `hint == Cache`, also writes to the lock map's cache. Caller MUST - have persisted via `put_chunk` first. -- `Drop` — records lock-hold time into the histogram (from `hold_start` - to now). If hold time > `hold_warn_threshold`, emits a `warn!` log - with chunk_id and hold duration. - -### 10.4 Chunk cache - -`quick-cache = "0.7"` (`app/crowdb-chunkdb/Cargo.toml`). Default capacity -10_000 entries (configurable via `lifecycle.cache_capacity`). The -design supports 100_000+. `quick_cache::Cache::new(capacity)` accepts -any `usize`; the only constraint is memory (~1-2 KB per `Chunk` → 10k -entries ≈ 10-20 MB, 100k entries ≈ 100-200 MB). - -Cache operations: -- `Cache::get(&key) -> Option` — cache hit check. O(1). -- `Cache::insert(key, value)` — populate on miss, refresh after - `put_chunk`. O(1) amortized. -- `Cache::remove(&key) -> Option<(Key, Val)>` — `invalidate_chunk`. - O(1). -- `Cache::entry_count()` — gauge for metrics snapshot. - -### 10.5 Sweep task - -The lock map grows unbounded without reaping (one entry per chunk ever -touched). `reap_idle` removes uncontended entries periodically, keeping -the map bounded by concurrent locks, not by chunks-ever-touched. - -`main.rs` spawns a background task (`run_sweep_loop`) that calls -`locks.reap_idle()` every `lifecycle.sweep_chunk_lock_interval_secs` -(default 60s). Uses the same `watch::channel(false)` stop signal -pattern as the topology refresh loop. `reap_idle` performs a lock-free ordered -scan with exact-entry removal and does not block request tasks. - -### 10.6 LifecycleHandler integration - -`LifecycleHandler` gains a `locks: Arc` field. Each -mutating RPC acquires the per-chunk lock before its RMW cycle: - -- `allocate_chunk` (caller-supplied ID): `check_range` → - `acquire_for_create` → existence check (`store.get_chunk`; return - `ChunkAlreadyExists` if taken) → build chunk, allocate strips, - `put_chunk`, start background segment commit → `guard.refresh(chunk)`. -- `allocate_chunk` (auto-generated ID): skip the lock (UUID collision - negligible). After `put_chunk`, `populate_cache(id, chunk)` directly. -- `append_chunk`: `check_range` → `acquire` → state/revision check. A matching - `modify_ts` allocates strips, increments the revision, persists the chunk, - and returns only the new strips plus the revision. A stale revision performs - no allocation and returns the complete current chunk so the client can - refresh and retry. Successful mutation ends with `guard.refresh(chunk)`. -- `seal_chunk`: same as append but no diskdb calls (fast path). -- `delete_chunk`: `check_range` → `acquire` → state check → persist Deleted - with segments as cleanup intent → free segments → clear the segment list - and persist the tombstone → `guard.refresh(chunk)`. -- `delete_chunk_range`: offset and length are independent u32 byte values, - describing an exact half-open object range without KiB rounding. The caller - sends the immutable chunk identity and logical range; an optional nonzero - layout revision adds a strict precondition. ChunkDB qualifies its current - sealed user-data chunk before accepting the notification. Dedicated chunks - accept only their complete sealed object extent. The caller must already establish absence of live and future - references. The server atomically records merged exact dead ranges in a - co-located resource record, guarded by the chunk's KV revision; the unchanged - chunk participates in that conditional batch without changing its layout. - Resource identity includes the writer epoch and sealed extent. Existing - retirement replays without another mutation; a new range with an explicit - stale layout condition fails. An active writer or unacknowledged shared range - returns a retryable conflict. The client performs one range notification, - without a preliminary QueryChunk or whole-chunk deletion. - Records are bounded to 4096 intervals and decoded within 128 KiB. Foreground - QueryChunk does not load or return this resource metadata. Conflict or unknown - write results do not trigger confirmation GETs inside the retirement handler. - A successful response means ChunkDB has accepted cleanup ownership. Access - finishes its notification without waiting for the physical `reclaimed` flag; - physical capacity release is a separate ChunkDB/DiskDB responsibility. - Chunk identities remain tombstones after deletion and cannot be allocated - again. Strip replacement preserves logical offsets and total capacity; - physical relocation does not change the range being retired. An intersecting strip is never permission - to free blocks containing live neighbours. Mixed-strip physical compaction and - independent orphan qualification are not implemented by range retirement. -- `update_chunk_strip`: compatibility wrapper over the one-strip form of - `replace_chunk_strip_range`. -- `replace_chunk_strip_range`: `check_range` → `acquire` → validate Active or - Sealed state, expected revision, exact old range, contiguous capacity, - monotonic sequences, segment ownership, and mirror/EC geometry → commit only - new-only segments → atomically publish the replacement plus an old-only - cleanup intent → `guard.refresh(chunk)`. An identical operation retry at the - successor revision returns the installed chunk. Any other revision or range - is a conflict. -- `allocate_replacement_segment`: inspect the current owning strip and return - one geometry-compatible tentative segment. Mirror replacement excludes - survivor nodes. EC replacement excludes nodes already at the strip's - per-node failure-domain limit; explicitly unsafe layouts retain their - balanced per-node ceiling. Every old and surviving strip disk is excluded, - so no two shards share one disk. -- `discard_replacement_segment`: under the lifecycle guard, free a tentative - segment only when it belongs to the chunk and current metadata does not - reference it. -- `reserve_strip_group`: allocate ordinary hidden strips or one special 8+4 - conversion group, persist its writer/lease/placement fences separately from - `Chunk.strips`, and leave chunk capacity unchanged until confirmation. A - lock-free atomic admission gate checks the instance's proportional share of - the configured cluster-wide reserved-block and reserved-byte limits before - allocation. Startup rebuilds usage from durable reservation records. -- `mutate_strip_reservation`: durably consume before DiskIO, confirm one strip - in sequence, cancel or renew idempotently, or publish a complete special - group after current-topology survivor selection. Publication commits one EC - strip and the redundant-replica cleanup intent atomically. -- `query_chunk` / `list_chunks` — unchanged (no lock, no cache). - -`query_chunk` advertises `layout_validity_ms`. Retired segments remain allocated -until that reader-layout window expires. Startup and the periodic lifecycle -scan resume cleanup intents, recheck that retired identities are absent from -the current layout, ask DiskDB to perform its ownership-qualified free, and -only then clear the intent. Metadata publication therefore remains committed -even if post-commit reclamation is temporarily unavailable. - -Reservation reconciliation scans reservation records in bounded rotating -pages, so an old prefix cannot starve later records. It deterministically -admits a completed special group to the normal conversion task path. An -expired incomplete group is cancelled and reclaimed when its consumed strips -carry persisted planned cursors and the writer lease deadline plus -`lifecycle.layout_validity_ms` has elapsed. This grace bounds late request -admission; ordered completion proof and producer exclusion are also required -before physical reuse because an expired RPC can still have backend IO in flight. -Current reservation records require explicit planned -cursor/sequence vectors; a consumed strip with a zero planned cursor is malformed -and fails before reconciliation changes storage. Placement records require -explicit policy and repair-state fields. DiskIO does not validate allocation -ownership. Terminal reservation -records release their quota only after the durable record has been removed; -ambiguous persistence retains the permit unless a linearizable read proves that -no record exists. - -### 10.7 Error variants + service mapping - -```rust -#[error("chunk lock busy — retry later")] -LockBusy, -#[error("chunk lock acquire timed out")] -LockTimeout, -``` - -`map_error` in `service.rs`: -- `LifecycleError::LockBusy => Status::unavailable(e.to_string())` -- `LifecycleError::LockTimeout => Status::unavailable(e.to_string())` - -Both map to `UNAVAILABLE`. The client's existing retry logic handles -this (same as `NotLeaderHint` transient errors). - -### 10.8 Metrics + HTTP endpoints - -`app/crowdb-chunkdb/src/metrics.rs` — `LifecycleMetrics` with -`AtomicU64` counters + `Mutex` for lock wait/hold -latency. Counters (all `AtomicU64`, `Relaxed` ordering): - -- `lock_wait_time` — `PreciseHistogram` (wait duration in acquire). -- `lock_timeout_count` — incremented on `LockTimeout`. -- `lock_busy_count` — incremented on `LockBusy`. -- `lock_hold_time` — `PreciseHistogram` (hold duration in guard Drop). -- `cache_hit_count` — incremented on cache hit in `acquire`. -- `cache_miss_count` — incremented on cache miss in `acquire`. -- `cache_size` — gauge, read from `Cache::entry_count()` at snapshot. -- `reap_idle_count` — incremented each `reap_idle` run. -- `reap_idle_entries_removed` — entries removed per `reap_idle`. -- `invalidate_count` — incremented on `invalidate_chunk`/ - `invalidate_range`. -- `reservation.blocks.g` and `reservation.bytes.g` — current durable hidden - reservation usage rebuilt at startup and updated on admission/release. -- `reservation.rejections.c` — reservation requests rejected by either atomic - quota limit. - -`snapshot() -> LifecycleMetricsSnapshot` drains counters, reads -histograms, returns a serializable struct (JSON). - -HTTP endpoints (`main.rs` HTTP server, alongside `/ready` + -`/health`): - -- `GET /metrics` → returns `LifecycleMetricsSnapshot` as JSON. -- `POST /invalidate_chunk` → body `{ "chunk_id": { "high": u64, "low": - u64 } }` → calls `ChunkLockMap::invalidate_chunk`. Returns - `{ "invalidated": bool }`. -- `POST /invalidate_range` → body `{ "bucket_start": u16, "bucket_end": - u16 }` → calls `ChunkLockMap::invalidate_range`. Returns - `{ "invalidated_count": u32 }`. - -All internal (no auth, same as `/ready` and `/health`). - -`/ready` requires validated storage routing and at least one owned service -slot. Empty or uninitialized ownership rejects work, including reads; there -is no allow-all compatibility mode. - -### 10.9 Edge cases - -- Lock map entry does not exist → created on first `acquire` through the - ordered index's `get_or_insert` operation. -- Lock holder panics → `tokio::sync::Mutex<()>` auto-releases (no - poisoning for `Mutex<()>`); cache slot may be stale but next - `acquire` re-fetches on miss. -- `reap_idle` races an acquirer → either the acquirer increments the exact - entry's user count first and reaping skips it, or reaping marks the entry - first and the acquirer retries against its replacement. Overlapping callers - cannot split across mutex identities. -- Process crash → all in-memory state lost; KV store is source of - truth. -- Cache evicts a chunk between two operations → next `acquire` is a - miss, re-fetches. Correctness unaffected. -- `CacheHint::NoCache` → skip `insert` on miss and `refresh`; the - guard's local copy is still updated so the current operation sees - the chunk. -- `delete_chunk` keeps the Deleted-state chunk cached via `refresh` → - next `delete_chunk` retry gets a cache hit and returns the same tombstone - successfully without a store round-trip. -- `acquire` returns `ChunkNotFound` (store miss during acquire) → the - chunk does not exist; `append`/`seal`/`delete` return - `ChunkNotFound`. -- `LockBusy` / `LockTimeout` → mapped to RPC `UNAVAILABLE`. - -## 11. EC Encoding/Decoding - -**EC encoding** (strip allocation): -1. Allocate data_num + code_num blocks via diskdb -2. Fill data blocks with actual data (by caller, not chunkdb) -3. Call isa-l encode to generate parity blocks -4. Mark EC state = Complete - -**EC decoding** (recovery, deferred): -1. Identify failed blocks (via disk failure notification) -2. Read surviving data+parity blocks -3. Call isa-l decode to reconstruct lost data -4. Write reconstructed data to new blocks -5. Update strip with new block locations - -**crowdb-common EC module**: Provides safe Rust interface to isa-l: -- `make_gf_table(data_num, code_num)`: Generate Galois Field table -- `encode(data_num, code_num, segment_len, gf_table, data_ptr, code_ptr)`: Encode -- `decode(...)`: Decode with error recovery -- Buffer management: `make_buffer`, `destroy_buffer` - -**Rationale**: isa-l is highly optimized for AVX2/AVX512. Strip-level EC -aligns with redundancy unit design. - -## 12. Crate Layout - -``` -app/crowdb-chunkdb/ # chunkdb server binary -├── Cargo.toml -├── src/ -│ ├── main.rs # CLI entrypoint and server wiring -│ ├── lifecycle.rs # pure lifecycle index -│ ├── lifecycle/ -│ │ ├── handler.rs # lifecycle orchestration -│ │ ├── lock_map.rs # per-chunk serialization and payload cache -│ │ └── state.rs # lifecycle state transitions -│ ├── service.rs # pure service index -│ ├── service/ -│ │ ├── chunkdb_rpc_service.rs -│ │ └── chunkdb_rpc_service/ -│ │ ├── service.rs # state and handler registration -│ │ ├── mutations.rs # mutating chunk handlers -│ │ ├── queries.rs # query and list handlers -│ │ └── wire.rs # FlatBuffer frames and error mapping -│ ├── allocator/ -│ │ └── pool.rs # allocation pool -│ ├── selector/ -│ │ ├── mirror.rs # Mirror placement -│ │ └── ec.rs # EC placement -│ ├── topology/ -│ │ ├── notify.rs # topology notifications -│ │ └── refresh.rs # topology refresh task -│ ├── storage.rs # KV persistence -│ ├── routing.rs # bucket routing -│ ├── range_guard.rs # owned-range enforcement -│ └── runtime_context.rs # isolated lifecycle/task runtimes - -lib/crowdb-chunkdb-client/ # chunkdb client library -├── Cargo.toml -└── src/ - ├── client.rs # ChunkdbClient - └── rpc_transport.rs # crowdb-rpc transport - -lib/crowdb-protocol/ # Protocol definitions -└── src/fbs/ - └── chunkdb.fbs # service and data types -``` - -The lifecycle index preserves the public `LifecycleHandler`, `ChunkLockMap`, -and state-machine API. The handler owns chunk operations; the lock-map module -owns the approved bounded per-chunk mutex and payload cache. RPC registration, -chunk handlers, and FlatBuffer construction are separate private modules -behind `ChunkdbRpcService`. - -## 13. Concurrency Model - -- **Async everywhere**: All public APIs are async (`async fn`). -- **Shared state**: Immutable topology is published through `ArcSwap`; placement - reservations use per-disk-group atomics. Lifecycle's existing bounded - per-chunk guards protect metadata mutation, not allocation ranking. -- **Lock scoping**: Existing lifecycle guards are acquired in `{}` blocks and - dropped before `.await`; placement introduces no hot-path lock. -- **Parallel allocation**: Use `futures::join_all` for parallel strip/block - allocation. -- **Background tasks**: `tokio::spawn` for topology refresh, rollback cleanup. - -**Rationale**: tokio provides efficient async I/O. Lock scoping prevents -deadlocks with `.await`. Parallel allocation minimizes latency. - -## 14. Configuration - -Key configuration parameters: - -| Parameter | Default | Description | -|---------------------------------------------------|------------|---------------------------------------------------------------| -| disk_block_size | 1 MB | Size of disk blocks from diskdb | -| deployment.max_node_failures | 1 | Protected production node-failure budget | -| mirror_copy_count | 2 | Production mirror copies on distinct nodes | -| default_ec_scheme | 6+3 | Default EC scheme (data+parity) | -| topology_refresh_interval | 30 s | Topology cache refresh interval | -| placement.mode | protected | Select `protected` or explicit `unsafe_colocated` strategy | -| placement.allow_unsafe_ec | false | Permit explicit degraded EC placement | -| placement.allow_degraded_failure_domains | false | Permit an explicit unmet rack or node guarantee | -| placement.failure_domain_priority | rack_first | Prefer rack or node protection when ranking safe plans | -| placement_repair.max_concurrency | 2 | Maximum concurrent degraded-EC repair tasks | -| placement_repair.scan_interval_secs | 1 | Marker reconciliation interval | -| placement_rebalance.scan_interval_secs | 300 | Cross-domain usage scan interval | -| placement_rebalance.imbalance_threshold_pct | 20 | Required utilization-point skew | -| placement_rebalance.hysteresis_secs | 900 | Required sustained-skew window | -| placement_rebalance.min_target_free_bytes | 1 GiB | Minimum target disk-group free capacity | -| placement_rebalance.max_moves_per_cycle | 1 | Exact per-cycle relocation bound | -| max_allocation_parallelism | 10 | Max parallel strip allocations | -| lifecycle.cache_capacity | 10_000 | Per-chunk payload cache capacity (§10) | -| lifecycle.sweep_chunk_lock_interval_secs | 60 | Idle lock reap interval (§10) | -| lifecycle.lock_hold_warn_threshold_ms | 1000 | Lock hold warn threshold (§10) | -| lifecycle.layout_validity_ms | 30_000 | Minimum retired-layout and expired-writer reuse grace | -| server.keepalive_interval_secs | 10 | Service-registry heartbeat interval | -| server.rpc_workers | 2 | Inbound RPC workers; static and must be positive | - -chunkdb requires a typed TOML file at startup. Omitted fields use typed -defaults, and explicitly supplied CLI fields override file values according to -the shared [`service configuration design`](../config/design-crowdb-config.md). - -## 15. Full-Stack Deployment and Benchmark - -The shared OCI node lifecycle prepares Group 0, the ordinary metadata group, -DiskDB, DiskIO and ChunkDB through accepted node policy and committed service -intent. The common cluster lifecycle creates runtime resources and observes -readiness; benchmarks attach to the selected cluster without creating groups, -resetting topology or starting and stopping servers. Production placement uses -actual failure domains; a Local cluster on one physical host has zero physical -host failure tolerance. See the [deployment design](../deploy/design-crowdb-deploy.md). - -`crowdb-cli bench chunkdb allocate` measures mirror or EC allocation with -configurable shape, duration, seed, and concurrency. `bench chunkdb mix` uses -a deterministic allocate/query/append/seal/delete distribution. Both query -every surviving chunk through the public client, validate state, owner tags, -and nonoverlapping physical segments, and require the exact referenced byte -total to equal the DiskDB busy-space delta after compaction. Capacity -exhaustion is a successful stop reason; any correctness error invalidates the -sample. - -The allocation regression uses an explicit test-only measurement fixture with -three co-located logical nodes, one rack, three data groups and four logical -4-TiB disks per DiskDB. It measures one EC 8+4 strip per allocation with varied -concurrency and retains exact space-accounting checks. The fixture owns every -server by PID and process-start identity, records accepted bootstrap inputs and -provides normal clients with explicit targets. Each invocation retains metrics, -logs and a checked shutdown receipt. This fixture does not qualify physical host -redundancy or replace real-storage OCI acceptance. - -## 16. Implementation Scope - -**v1 (R85)**: -- Basic chunkdb server and client -- Mirror and EC strip allocation -- Synchronous and background mirror-to-EC conversion through persistent tasks -- Rack/node-aware placement -- Topology cache with group-0 integration and watch/notify for real-time updates -- Basic chunk lifecycle (allocate/seal/delete) -- EC wrapper (crowdb-common module) -- Chunk ID generation (crowdb-common): 128-bit format with getrandom -- Wal, BtreePage, PageIndex, Stream, S3 and IcebergTable purposes -- E2E tests - -**Future work** (separate requirements): -- In-chunk GC operations (reclaim, collapse, merge) -- Shared and dedicated Repo-chunk write policies in the client data path -- Recovery flow (disk failure handling, EC rebuild) -- Metrics and observability -- Console/CLI integration -- Custom RPC for performance - -## 17. References - -- CROWDB diskdb design: `doc/design/diskdb/design-crowdb-diskdb.md` diff --git a/doc/design/chunkds/design-crowdb-chunk-kv-client.md b/doc/design/chunkds/design-crowdb-chunk-kv-client.md deleted file mode 100644 index a2f259cf3..000000000 --- a/doc/design/chunkds/design-crowdb-chunk-kv-client.md +++ /dev/null @@ -1,178 +0,0 @@ - - - -# Chunk KV Routed Client - -`crowdb-chunk-kv-client` is the application-facing router for the chunk-backed -KV service. It validates and caches the immutable range catalog, sends each -single-partition operation directly to the current owner, and composes bounded -multi-partition operations without claiming distributed atomicity or a global -snapshot. - -## Catalog and Routing - -`CatalogSource` supplies a catalog head and all referenced pages. The client -decodes them as one ordered `CatalogMap`, reusing the protocol validation that -requires complete, non-overlapping coverage of the binary keyspace. A valid -generation is published through `ArcSwap`; invalid or regressing refreshes do -not replace a warm map. - -Point lookup binary-searches partition starts. No data server acts as a network -forwarding proxy. During same-owner split grace, the old endpoint may dispatch -an old parent point route directly to the exact child handle already hosted in -that process. This preserves the logical request identity and never reopens the -parent writer. Scan and seek topology are not forwarded. A cold client must -load group 0 before sending a nonempty operation. A warm client may continue -trying the cached owner while refresh is unavailable, but the server's owner -epoch and serving lease remain the final authority. - -Each operation has one total deadline covering discovery, refresh, transport, -and backoff. `NotMyRange`, `RefreshRequired`, lease loss, and connection loss -cause bounded rerouting. Only `TargetNotReady` enters the initialization retry -path: it uses the server retry delay without refreshing the route, inventing -another owner, or changing request identity, and returns the third unsuccessful -response to the caller. Normal serving requests do not execute this retry -bookkeeping. Overload, recovery, and write-stall responses retain the general -retry policy. The route-refresh and total-attempt budgets are independent and -explicit in `ClientConfig`. - -When `NotMyRange` reports an older catalog, or the same generation with no -different owner/epoch hint, the client retries the cached endpoint -within its serving-attempt and operation-deadline bounds. This lets a newly -published owner activate without repeatedly loading an unchanged catalog. -Newer generations and different owner hints retain bounded catalog discovery. -Exhausting discovery refreshes does not itself exhaust serving attempts. - -## Request Identity - -Every client handle generates a random nonzero 128-bit instance ID from the -operating system. An atomic counter allocates nonzero sequences beginning at -one and closes permanently before wraparound. A logical mutation allocates its -`ClientRequestId` once, before routing, and retains it across every retry. - -`execute_with_identity` accepts an application-persisted identity and operation -for explicit recovery after process restart. The server compares the operation -digest against its retained result, so changing an operation under a reused -identity remains a typed `RequestConflict`. Connections, endpoints, and RPC -ports never contribute to identity. - -## Direct Operations - -The client wraps get, put, delete, put-if-absent, compare-exchange, -conditional delete, ceiling, higher, floor, and lower. It returns the complete -`ChunkKvResponse`, including journal position, applied or observed revision, -condition outcome, map revision, retry hint, and typed failure. Gets can carry -a minimum journal position for explicit read-after-write ordering. - -After split, a get routed to a child may carry either a retained parent-stream -position at or below the split cutover or a child-stream position. The server -resolves the former through the inherited overlay and the latter through the -child applied frontier. A client never compares offsets from different stream -names. After balance, new writes return only the target stream position. - -Seek is an owner-local ordered operation. It is transported directly and is -not emulated with point reads. Directional scan likewise consumes native -partition scan pages and validates that every returned key is strictly ordered, -inside both requested and partition bounds, and consistent with the returned -continuation topology. - -## Multi-Get and Batch Mutation - -`multi_get` retains every input index, including duplicates, groups keys by -partition, dispatches groups with bounded parallelism, and restores one result -per input position. A failed group produces per-item typed errors without -discarding successful partitions. Each partition observes its own applied view; -the result is not a global snapshot. - -`batch_mutate` allocates or accepts one stable request identity per operation. -It rejects reads and oversized input before dispatch, preserves relative input -order inside each partition group, and returns per-operation results in original -input order. Successful operations are removed from the retry set. Only -transient or transport-unknown operations retry, using their original identity. -There is no atomic commit or ordering promise across partition groups. - -The protocol request types expose `validate_for_range` preflight methods. -Multi-get validation rejects the whole group if any key is outside the declared -partition. Batch validation additionally rejects empty groups, reads, and bad -request identities. Owners run this preflight before reads or WAL admission. - -## Multi-Partition Scan - -A scan validates its binary half-open bounds and optional continuation, pins a -catalog generation, intersects the interval with that generation, and visits -partitions in ascending or descending byte order. Execution is sequential, so -the number of buffered partition pages is one and therefore stays within the -configured page bound. Global item, caller byte, and client response-byte caps -apply across every partition. - -`scan_bounded` is a forward-only metadata read with positive item and byte -budgets. Its dedicated RPC requires server enforcement; unsupported transports -fail without falling back to ordinary scan. Owners cap the caller budget by -their response limit and check native value lengths before copying external -memory cells or assembling overflow payloads. An oversized first value or a -native traversal deadline with no progress is an explicit failure, not an empty -completed range. Partial native pages retain their last emitted key. - -The strict client returns after a nonempty owner page so the next request gets -a fresh byte budget. It returns a terminal page when the entire owner response -has been consumed, the owner reports no continuation, this is the last relevant -partition in the pinned catalog, and the response matches that catalog generation. -Otherwise it retains the conservative continuation, which may require a final -empty page to establish completion. Dataset metadata reads additionally bound -total request time and observe cancellation while waiting for the client. Ordinary scan retains -its existing first-entry and response-validation behavior. - -The public continuation contains direction, original bounds, last emitted key, -and observed catalog generation. Pagination resumes strictly after that key for -forward scans and strictly before it for reverse scans. When a split or transfer -invalidates the plan, the client refreshes and rebuilds only the unconsumed -interval. This prevents repeated emitted keys and stable-key gaps across -topology replacement. It does not make the scan a cross-partition snapshot; -concurrent changes to already consumed keys need not appear. - -A parent continuation is always refresh-only after split. A balance catalog -entry in `TargetCatchingUp` is also non-readable: the routed client retains the -same logical operation and waits or retries `TargetNotReady` until a later -catalog generation marks the target `Serving`. It never resumes a scan from an -incomplete target prefix. - -An empty scan limit, empty multi-get, or empty batch succeeds without catalog -discovery. Malformed bounds, mismatched continuations, oversized requests or -responses, out-of-order server pages, and request-sequence exhaustion fail -explicitly. - -## Resource Bounds and Transport Boundary - -`ClientConfig` caps attempts, catalog refreshes, owner connections, in-flight -partition groups, batch items, response bytes, buffered scan pages, and the -total operation duration. The composition layer enforces group, retry, item, -page, and byte bounds. `ChunkKvTransport` is injected so protocol behavior is -testable independently of sockets; its production implementation owns the -bounded connection pool keyed by owner instance and endpoint. - -Cancellation before server admission leaves no retained request. Cancellation -after admission may hide a committed result, so mutation callers persist and -resubmit the complete identity and operation when the outcome matters. - -## Consistency and Non-Goals - -Single-partition operations inherit the sequencer, durability, request-result, -and serving-fence semantics of `crowdb-chunk-kv`. Multi-get and batch are -partition-local compositions with partial success. Multi-partition scan is -globally ordered but not globally snapshot-consistent. - -TTL, watches, range delete, atomic multi-key transactions, network forwarding -proxies, and merge-specific continuation behavior are outside this client -contract. - -## Open Issues - -- The production group-0 subscription and periodic-refresh adapter is not yet - wired because the chunk KV server has no production catalog RPC surface. -- The production `crowdb-rpc` transport and bounded owner connection pool await - the server's FlatBuffers handlers. -- Native ordered cursors, internal group handlers, and the real chunk KV server - process are still required for real-process seek, scan, batch, and lifecycle - coverage. -- Restart, response-loss, owner-death, split, balance, control-plane outage, - graceful-drain, and merge-specific scenarios remain real-process follow-up. diff --git a/doc/design/chunkds/design-crowdb-chunk-kv-server.md b/doc/design/chunkds/design-crowdb-chunk-kv-server.md deleted file mode 100644 index dc1152802..000000000 --- a/doc/design/chunkds/design-crowdb-chunk-kv-server.md +++ /dev/null @@ -1,804 +0,0 @@ - - - -# CROWDB - Design: Chunk KV Server - -`crowdb-chunk-kv-server` is the standalone ownership, routing, and network -boundary for chunk-backed metadata partitions. It hosts zero or many R142 -partition handles but never owns or bypasses a raw tree or stream. Group 0 is -the durable catalog and monitor authority; monitor-issued serving grants are -the only authority to admit data requests. - -## Contents - -- [1. Catalog Publication](#1-catalog-publication) -- [2. Domain Monitors](#2-domain-monitors) -- [3. Serving Authority](#3-serving-authority) -- [4. Request Contract](#4-request-contract) -- [5. Ordered Reads](#5-ordered-reads) -- [6. Split Publication](#6-split-publication) -- [7. Child Balance State Machine](#7-child-balance-state-machine) -- [8. Placement Policy](#8-placement-policy) -- [9. Lifecycle and Observability](#9-lifecycle-and-observability) -- [Open Issues](#open-issues) - -## 1. Catalog Publication - -The catalog is one checksummed generation head over ordered immutable pages. -Each entry contains an exact half-open binary-key range, stable partition ID, -owner endpoint, monotonic owner epoch, lifecycle state, stable tree ID, stable -stream name, optional tail-overlay artifact, and optional transition ID. The -overlay separately binds a chunk root-catalog generation, a tree snapshot -sequence and applied sequence, source partition/epoch/stream and stream -manifest generation, retained replay and cutover offsets, cutover sequence, -and target-WAL start sequence. A valid generation starts at the empty byte string, has -exact adjacent bounds, ends unbounded, and covers each binary key once. - -Publishers validate the complete successor, including retained-partition epoch -non-regression, before I/O. They write only new pages, reread and validate every -referenced page, and write the head last. An ambiguous head write succeeds only -when rereading proves the exact intended head. Readers retain their last valid -immutable snapshot when a new head or page is absent, corrupt, incomplete, or -regressing. Unchanged pages may be referenced by later heads through their -original page generation. - -An owner's connection address may change without transferring its partitions. -Under the Group-0 leader fence, a fresh instance registration advertising the -same hosted partition and owner epoch may publish an address-only successor. -The partition, range, owner instance ID, epoch, tree, stream and transition -remain unchanged, and old pages remain immutable. A recovering owner may update -its route, but receives no serving authority until normal recovery and grant -checks succeed. Fixed transition comparisons use owner instance identity rather -than requiring its historical connection address. - -## 2. Domain Monitors - -`EnsureDomainMonitor` carries a persisted descriptor with domain and registry -names, compiled driver and capability versions, failure policy, balance policy, -and all health/lease timing. Identical registration is idempotent. Any field -conflict fails closed, and an unsupported compiled driver returns -`UnsupportedMonitorDomain` without changing the prior descriptor. - -Every group-0 replica prepares tasks from persisted descriptors. A staged tick -may observe, plan, and publish only under the current group-0 leader fence. -Registry, catalog, binding, or transition observation failure ends the tick -before planning; it is never represented as an empty cluster. A supervisor -restarts failed tasks from durable state and a new leader resumes outstanding -transitions. - -An unfinished split reserves both partition identities and participating owners -against automatic dead-owner transfer, as it does against ordinary balancing. -The published parent alone cannot prove that a competing transfer may claim its -WAL: durable split handoff can precede catalog publication. Owner restart resumes -that split; heartbeat expiry never authorizes discarding its recovery state. - -Normal service discovery filters expired registrations. Monitor observation -uses the raw scan, preserving expired entries and heartbeat timestamps so -absence cannot be confused with owner death. The default heartbeat is two -seconds, `Suspect` begins at six seconds, and `Dead` recovery begins at ten. -Automatic dead-owner planning and lease exclusion also require the absence of -a live WAL process lease. Independent renewal starts before partition replay, -whereas discovery publication can wait for replay to finish. A renewing recovery -boot therefore retains its assignments despite stale discovery; it receives no -serving authority until normal recovery and grants complete. Expired processes -remain eligible for failover under the existing heartbeat and exclusion limits. - -When the catalog advances during initial replay, completed recovery remains -usable only if every local recoverable entry is identical in both validated -catalogs, including ownership epoch, range, state, transition, and artifact. -The service installs those handles with the newer head without replaying them -again. Remote assignment changes do not invalidate local recovery. Changed -local assignments still require recovery under the original reconciliation -deadline; failed replay is never accepted as completed recovery. - -WAL replay retains a SHA-256 fingerprint of each canonical complete record for -duplicate-sequence validation, rather than retaining its key and value payloads. -The fingerprint includes identity, epoch, request, recorded result, and operation. -Exact late duplicates remain valid; conflicting duplicates, sequence gaps, and -operation-digest mismatches remain corruption. Replay windows and retry-result -retention keep their existing independent bounds. -After each 16 MiB of estimated applied key/value and entry bytes, native replay drains its transient -memtables into the in-memory tree. This bounds overwritten versions retained -while normal maintenance is not yet running. It does not snapshot, publish a -checkpoint, advance the checkpoint replay offset, or trim WAL; a drain failure -still rejects recovery. - -## 3. Serving Authority - -Each boot claims one fixed Group-0 WAL process lease. It retains its predecessor -identity while claiming; a replaced boot cannot claim again or renew. Independent -background renewal starts before partition recovery. Local expiry or a renewal -gap permanently fences that boot. Lease records retain process epochs and prior -process/grant expiry boundaries after expiration. - -The monitor issues one aggregate grant per instance and process epoch. It contains a monotonic -lease sequence, catalog generation, wall-clock expiry, sorted exact -`(partition_id, owner_epoch)` assignments, and their canonical digest. An -instance heartbeat advertises health but cannot authorize a request. - -Heartbeat publication compares the exact process identity and mutates the same -lease key atomically with the observation. Grant publication compares that key -and atomically preserves its sequence and greatest expiry alongside the grant. -Replacement, renewal and grant publication therefore serialize on one record; -replacement cannot miss a concurrent extension. An observation for another -process epoch cannot receive a grant, and a boot rejects grants for other epochs. -Grant publication attempts each independently ready instance once per pass in -instance-ID order. A failed instance publication does not skip other instances; -the pass reports all publication failures after processing them. Each publication -retains its process-identity, revision, expiry and atomic lease/grant checks. -Invalid shared catalog, readiness or exclusion evidence still rejects the pass -before instance publication. No conditional retry or authority extension is added. -One bounded control-plane task submits a boot's renewal and discovery writes -sequentially so their periodic ticks cannot exhaust each other's conditional -admission budget. Recovery, data IO and load sampling remain outside that task. -Queueing and publication retain the captured process-lease deadline; neither -can extend a boot's authority while metadata is unavailable. - -On receipt, an owner subtracts maximum clock skew and its self-fence margin -from the wall-clock remaining duration, then projects that duration onto its -local monotonic clock. Request admission reads an immutable grant snapshot and -checks its local deadline, catalog generation, partition identity, and epoch -without taking a lock. The default 12-second lease, one-second skew, and -one-second self-fence margin fence the old owner by ten seconds. A replacement -waits until the old wall-clock expiry plus skew unless an explicit source fence -is durably proven. - -Authority revocation retains the highest accepted grant sequence. The same old -grant cannot reinstall after a clear. Each assignment also carries local -continuity across live renewal: expiry, removal or clear permanently invalidates -an admitted context, even if a later grant restores the same partition and epoch. -Grant publication uses the current monotonic time, so delayed installation -cannot hide a local expiry gap. - -Normal point execution captures that context before asynchronous WAL/tree work -and validates it again before returning success. A revoked context returns -`LeaseExpired` without advertising a successful journal position; already durable -data remains available to replay and a retry with the same request identity. -The captured IO deadline stays at its original value across continuous renewal. -Catching-up targets retain their separately validated transfer-artifact authority. -Both paths also require the live process scope before execution and before -returning success. Grant installation cannot revive a locally fenced process. - -Client catalog revisions and owner epochs are routing references, not serving -authority. A server may resolve an older reference through a process-local -lineage, but it always authorizes the resolved current assignment against one -internally consistent catalog and aggregate grant. The grant's catalog -generation and exact `(partition_id, owner_epoch)` assignment remain mandatory; -the selected partition repeats the epoch check at its WAL and tree boundary. -This separates stale-client tolerance from owner fencing without weakening -either fence. - -Catalog reconciliation matches the tree identity as well as partition identity, -epoch, range, and stream. When a Serving descriptor has materialized its tail -overlay, an unactivated Prepared overlay handle must be replaced by recovery -from the independent checkpoint. It cannot satisfy the materialized assignment -or bypass the split commit proof. Ordinary grant activation applies only after -that independent recovery has completed. - -## 4. Request Contract - -Every request carries a random 128-bit client instance ID, nonzero monotonic -client sequence, catalog revision, partition ID, owner epoch, optional minimum -journal position, and optional deadline. The logical request identity survives -transport retry and rerouting. Endpoint, socket, and connection identities do -not participate in deduplication. - -The contacted server validates the deadline and request envelope, resolves the -key or logical range against its current catalog and recorded local lineage, -then authorizes every partition it will access against the current aggregate -grant before R142 admission. A client topology mismatch is observed for -refresh and metrics but is not itself an admission failure when the current -owner can resolve and authorize the request locally. A same-owner split keeps -an old parent dispatcher for point, multi-get, batch, seek, and scan requests; -the dispatcher selects the current writer by key and never proxies to another -node. If the current assignment is remote, unprepared, catching up, or absent -from the local grant, the server returns the corresponding typed failure and -performs no WAL I/O. - -Lineage dispatch preserves the original request identity and minimum journal -position. A selected writer accepts either its own stream position or an -inherited source-stream position covered by its durable overlay; it rejects an -unrelated stream instead of silently discarding the ordering requirement. -Point operations preserve get, put, delete, put-if-absent, compare-exchange, -and conditional-delete conditions and results. Successful and failed -conditions return the R142 journal position. `Overloaded`, `WriteStalled`, -`Recovering`, `TargetNotReady`, `LeaseExpired`, `RequestExpired`, -`RequestConflict`, `NotMyRange`, and `RefreshRequired` remain distinct wire -outcomes. - -A deadline observed before sequencer admission creates no WAL record. Once the -sequencer accepts a mutation, dropping the transport response does not cancel -the partition worker; retrying the same identity retrieves the recorded result. -Reads may carry a mutation's returned journal position for explicit -read-after-write ordering. - -Multi-get and batch mutation validate their complete key group before IO, then -capture authority for every resolved serving assignment. All captured contexts -are checked against one grant snapshot before and after each member. Revoking -any context stops later members from entering the WAL; a replacement grant cannot -resume the old group. Continuous live renewal preserves the group. Time spent -waiting for earlier members counts toward the original request deadline. -Batch mutation preserves each member's stable identity and individual outcome, -so retry can retrieve already durable results without replaying their mutations. - -## 5. Ordered Reads - -Seek provides ceiling, higher, floor, and lower operations within one logical -partition view. A same-owner split seek may cross the local writer boundary. -Scan intervals are validated and clipped once against the logical routed range; -the dispatcher visits retained and child writers in key order, or reverse key -order for a reverse scan. A forward scan includes its lower bound and excludes -its upper bound. Its continuation resumes strictly after the last emitted key -so the split key and page boundaries neither duplicate nor skip a key. Reverse -scans use the corresponding exclusive upper cursor. - -A continuation binds direction, last key, and the request's logical partition, -epoch, and catalog revision. It remains valid across a same-owner split while -that exact local lineage is retained, because the last key determines the next -writer. A mismatched continuation envelope or an owner transfer that removes -the local lineage returns `RefreshRequired` or `NotMyRange`; the server never -guesses a remote resume position. Multi-partition composition belongs to the -routed client. - -## 6. Split Publication - -### 6.1. Publication boundary and durable steps - -The group-0 catalog head publication is the external range and assignment -cutover. A local writer handoff is a separate operation: the same process can -dispatch the old parent route to the retained parent and child before the new -catalog is visible. This compatibility must preserve every acknowledged WAL -record; it does not authorize a different owner to serve an unpublished range. - -The parent retains its existing tree and journal. Only the child receives new -storage. Handoff is parent-local range dispatch, not replacement of the parent -writer with another tree/WAL and not remote ownership transfer. The successful -durable handoff status update commits the obligation to resume this split; -in-memory dispatch changes afterward. Handoff status is distinct from final -`ChildPrepared` readiness and from external catalog publication. A candidate -epoch or a process-local dispatcher cannot substitute for this status. - -- **S1 — Plan:** persist the transition identity, expected parent assignment, - split key, both successor epochs and complete storage identities. Reserve the - affected partition and owner against another topology transition. -- **S2 — Prepare:** persist `ParentPreparing` before creating artifacts. Pin the - exact base, construct the bounded child view and establish the common WAL - frontier `C`. Persist the exact handoff status before changing local dispatch. - Parent journal appends and reads continue while the status is persisted; - no handoff waiting queue or mutation-worker wait for that update is allowed. - Bind the original parent storage, child storage, pinned base and replay - boundaries. Cover the additional parent suffix before actual dispatch in - child recovery; fixing `C` does not by itself end parent right-range writes. -- **S3 — Ready:** persist the exact retained-parent and child artifacts, - overlays, common cursor and readiness proof as `ChildPrepared`. A process-local - handle or heartbeat cannot substitute for this durable proof. -- **S4 — Publish:** validate that the current catalog still contains the planned - parent. Publish one successor head replacing it with two adjacent Serving - entries. This head is the external split commit point. On an ambiguous result, - reread the authoritative head and accept only the exact intended successors; - an unrelated newer generation is not proof that this split committed. - Materialization may remove overlays and transition markers before the separate - `CatalogCommitted` status write. Recovery also accepts these materialized - successors only when partition, range, owner, epoch, tree and stream identities - still match exactly; it preserves the newer head instead of restoring overlays. - A transition marker on another assignment remains an identity conflict. -- **S5 — Reconcile:** persist `CatalogCommitted` after proving publication. - Install the matching catalog and grants; preserve old-route dispatch without - allowing a retired writer to append. Restart must use the published assignment - together with its exact durable split proof. -- **S6 — Materialize and retire:** checkpoint independently recoverable halves, - publish removal of their overlays and transition markers, then release pins. - Reclaim only artifacts with no catalog, replay, retry or pin references. - -Crash recovery must distinguish these boundaries: - -- Before handoff status commit, recover the published parent and retry or abort - preparation; no child mutation may have been acknowledged. -- After status commit but before in-memory dispatch, resume the same split. - Recover the complete original parent journal, including writes made while the - status update was in flight; an empty child WAL is valid. -- After dispatch but before catalog publication, resume the same split from the - original parent journal and the child journal. The old parent route remains - published; do not discard the child or choose authority by the maximum local - epoch. Neither final readiness nor catalog publication is needed to establish - that the committed handoff must be recovered. -- An uncertain handoff update requires exact durable transition reread. Stale - preparation or abort work must not overwrite a committed handoff. - While the process remains live, retain and retry the exact prepared child - base, WAL handle and pins. Do not reopen its WAL, release its pins or rebuild - a different candidate merely because publication returned an error. Writes - arriving on the parent meanwhile are covered by the later dispatch frontier. - Once dispatch is installed, its live writers own finalization; only a proven - pre-handoff abort may discard the pending candidate. -- After publication but before `CatalogCommitted` persistence, prove the exact - successor entries and finish the transition marker; do not roll back the split. -- After publication, recover each assigned half's complete own WAL. The child - inherits only the parent prefix through the final readiness frontier `C2` - recorded in its catalog artifact, then replays its own WAL. This bound applies - even when the child WAL is empty: later retained-parent records belong to the - parent's next epoch and must not extend child inheritance. Require matching - grants before admitting requests. - -The initial handoff status fixes `C`, before the eventual in-memory dispatch -frontier `C2`. While only that initial status exists, recovery derives `C2` from -the first child record's sequence minus one, or inherits the parent tail when -the child WAL is empty. Final readiness records the exact boundary explicitly; -published catalog recovery uses that boundary instead of rediscovering it from -the current parent tail. - -The durable handoff proof is stored in the split transition while its phase is -`ParentPreparing`. Revision CAS preserves an existing proof and reconciles an -unknown write result by exact reread. Initial and final replay frontiers remain -distinct; additional real-process crash windows are recorded under Open Issues. - -### 6.2. Local execution and storage retention - -The split transition phase symbols are `Planned`, `ParentPreparing`, -`ChildPrepared`, `CatalogCommitted`, and `Aborted`. A separately persisted -`handoff_proof` distinguishes preparation from committed local handoff before -final readiness. It binds the pinned tree base, original parent stream and -initial inherited frontier while the transition remains `ParentPreparing`. -Final readiness extends that frontier without replacing its base. The server and -group-0 monitor advance them in this order: - -1. Group 0 persists the complete plan before local work begins. The existing - parent keeps its identity, tree, stream, owner, and lower boundary; its next - epoch and the one new child's identity, range, epoch, tree, and stream are - fixed by the plan. -2. The parent process persists `ParentPreparing`, renews its serving grant as - an active owner, checkpoints the exact parent base, builds the child's - range-bounded base, fixes `C`, and persists the handoff status without stopping - parent writes or reads. It then installs parent-to-child dispatch. Left-range - writes retain the original parent journal; right-range writes move to the - child journal only when dispatch changes. Replay coverage includes parent - writes between the fixed frontier and that dispatch change. -3. Readiness binds the exact common cutover, the retained-parent next epoch, - and the child's complete tail overlay. - Split plans retain the original parent tree/WAL identity in every phase. - The strict readiness record contains one child overlay and rejects unknown - fields; the planned timestamp is required. - It is persisted before catalog publication. -4. One catalog generation atomically replaces the old parent entry with a - smaller entry having the same partition/tree/stream identity and the next - epoch, then inserts the one child entry. A retry accepts an already-published - result only if transition IDs, ranges, owners, epochs, trees, streams, and - the child overlay all match. -5. The parent owner has already activated both local writers before publishing - the catalog. Catalog reconciliation adopts those handles for external - routing; it neither resumes the retained parent nor activates either writer. - An old parent route dispatches point, grouped, seek, and scan operations - through the retained local lineage. The current catalog and grant authorize - the actual writer or writers used by the operation. - -Before child materialization, heartbeat load reports the child as dependent. -After process restart, the retained parent reopens its original tree and WAL -through ordinary range-filtered recovery; it has no split tail overlay. The -child reopens its exact pinned base, replays the range-filtered parent suffix, -and then replays its own WAL. A matching serving grant alone cannot activate -the child overlay. Startup loads the persisted split transition and requires its committed -phase, transition identity, range, owner, epoch and complete storage artifact to -match the current Serving catalog entry. Transfer overlays retain their separate -committed-transfer validation. Missing or conflicting evidence leaves the writer -Prepared; repeated matching grant refresh is idempotent. - -For independent serving assignments, the local maintenance loop collects retired -ordered WAL chunk records under the existing trim horizon and reader pins. It -requires a live serving grant, an exact local assignment, and no catalog transition -or tail overlay. Each bounded pass releases at most one eligible chunk and record; -a deletion schedules another pass after yielding across hosted partitions. Periodic -scans continue at an unchanged generation, while split materialization keeps its -cadence. WAL trim and tree-root retention do not advance as part of this cleanup. - -For split overlays, the maintenance loop performs bounded ownership -materialization and a serving checkpoint. A heartbeat then proves independent recovery, group 0 -publishes a generation that clears both the overlay and the completed split -`transition_id`, and only that catalog state is eligible for another split or -owner balance. Releasing both markers atomically prevents a later transfer from -mistaking the independently recoverable child for an active split participant. - -The child root-catalog generation is pinned under the split transition identity -before the server persists child readiness. The pin key is scoped by child tree -ID and its value is the exact root generation; it survives both tree and server -restart. Root reclamation treats the oldest durable pin as an upper bound. The -owner releases the pin only after installing the newer catalog generation whose -child has no overlay and whose parent and child have no split marker. Pin delete -is idempotent and remains reconciliation work after an ambiguous response or a -crash. An abort before handoff commit may remove unpublished child state and its -pin. After handoff commit, absence of readiness or a child catalog entry is not -permission to abort or delete child storage; recover and complete the same split. - -The parent also retains its existing checkpoint generation while the child -depends on the inherited parent WAL prefix. Publishing that prefix into the -child's live tree does not release this dependency: the pinned recovery base -still needs those journal records after a crash. Parent checkpoint publication -remains suppressed by the existing transition pin until the catalog records an -independent child. Parent reads and journal appends continue throughout this -interval. The same completed catalog releases both generation pins. - -## 7. Child Balance State Machine - -### 7.1. Owner cutover and durable steps - -The first successful catalog head publication naming the target in -`TargetCatchingUp` is the owner cutover. The later `Serving` publication changes -readiness, not ownership. A candidate target epoch in a plan is not effective -tree or serving authority. Source release precedes the owner cutover and creates -a recovery obligation; it is not itself a catalog owner change. - -- **T1 — Plan:** persist the exact source assignment, target identity and higher - epoch, artifacts, lease bounds and transition identity before preparation. -- **T2 — Prepare source:** persist `SourcePreparing`, pin an immutable root, - initialize the independent target WAL and record replay start cursor `P`. - Source remains the published owner and accepts writes under its existing grant. -- **T3 — Prepare target:** persist `TargetPreparing`; open the pinned base and - replay the source suffix into a Prepared view. Persist readiness before - `TargetPrepared`. Preparation must not advance the shared tree's effective - owner epoch or disable source recovery. Target cannot admit data mutations. -- **T4 — Seal source:** after exact target readiness, close source admission, - drain admitted writes and persist release proof with final durable cursor `C`. - With an unreachable source, wait for lease exclusion and recover the complete - durable tail instead of treating `P` as final. No target data admission is - allowed while the catalog still names source. A durable release cannot be - undone merely because the catalog still has the old assignment. -- **T5 — Publish owner:** validate the source assignment and durable release, - then publish the target epoch, `TargetCatchingUp` and complete replay artifacts - in one catalog successor. Resolve ambiguous publication by exact head reread. - Effective target storage authority must follow this published assignment; - a crash before that authority update is completed must allow idempotent repair. -- **T6 — Catch up:** recover the complete source prefix through `C`, followed by - the target WAL. The published release proof permits durable unconditional - target appends; reads and conditional operations wait for the complete view. - Source cannot append, including through a stale client route or old grant. -- **T7 — Serve:** persist final readiness, publish the same target assignment - as Serving, persist `CatalogCommitted` and activate only with its exact grant. - A crash between head publication and phase persistence must finish the marker - from exact catalog evidence rather than strand or reverse the assignment. -- **T8 — Retire:** clear the completed overlay only after independent recovery - is proven. Release pins and reclaim old resources only after all recovery, - retry and forwarding references have cleared. - -Restart first reads the authoritative catalog and then the matching durable -transition. Before release it restores source; after release but before owner -publication it keeps source fenced and completes publication; after owner -publication it restores target. A local tree epoch, heartbeat or cached page -cannot independently choose the owner. Missing or conflicting proof fails closed. - -An abort is permitted only before durable release and owner publication. Persist -the exact abort before discarding the prepared target; stale workers must not -publish or activate that aborted transition. Candidate epochs are not reused, -and cleanup must retain any source-referenced base, WAL or pin. Restoring source -does not require synchronous deletion of unrelated target temporary objects. - -The effective-tree-authority ordering and publication/phase crash windows still -require implementation verification; known gaps are recorded under Open Issues. - -### 7.2. Execution, proofs and recovery - -A balance transition persists source and target owners, increasing target -epoch, source and target artifacts, readiness limits, old-grant deadline, -phase, release proof, initial readiness proof, final catch-up proof, and -failure. The live-source phases and actions are: - -1. `Planned` → `SourcePreparing`: the source remains `Serving`, checkpoints a - pinned base, registers the target WAL's stable `StreamRecord`, and records - the initial source cursor in the target overlay. It acquires no writer lease - and allocates no chunks for the target WAL. - After pinning the base, an ordered mutation-worker command publishes and - captures the WAL sequence, byte offset and Stream generation together. - Independently sampled counters cannot define a replay cutover: concurrent - mutations may be newer than the previously published Stream prefix. -2. `TargetPreparing`: the remote target validates the range, page-root and - stream identities, initializes or recovers the target WAL under its own - process lease, opens the exact immutable tree-manifest generation named - by the source-base proof rather than the latest root, and starts one shared - async initialization coroutine that replays the source suffix into a - `Prepared` overlay. The source remains writable. The live target handle and - coroutine survive into final catch-up; reopening the same pinned generation - is required only after target process or handle loss. Failure here may abort; - source authority is unchanged. - Prepared transfer handles are staged separately from catalog-owned handles. - Refreshing unrelated catalog entries cannot evict their heartbeat readiness. - Publishing the matching target assignment promotes the same handle before - retiring staging; authoritative abort removes only the exact prepared epoch. - After restart, persisted `TargetPrepared` or `AwaitingFence` evidence restores - that readiness without rewinding the transition or granting serving authority. -3. `TargetPrepared` or `AwaitingFence`: the monitor requests release only when - record, byte, estimated catch-up, deadline, capacity, request-rate, - cooldown, and one-transition-per-owner bounds pass. The source enters one - explicit fencing lifecycle that closes new admission while the existing - mutation worker drains only already-admitted requests, then persists - sequence and byte cursor `C`. If the source is unreachable, the transition waits until old grant - expiry plus skew. A live handoff enters `AwaitingFence` only after group 0 - observes a healthy target heartbeat hosting the exact partition and target - epoch. If the source dies before an explicit fence, the unpublished target - overlay is discarded and the lease-excluded target reopens the source - stream directly, so recovery observes its complete durable tail rather than - treating preparation cursor `P` as the final cursor. -4. `TargetCatchingUp`: group 0 publishes the target owner and epoch with catalog - state `TargetCatchingUp`. The source is no longer catalog authority and - returns the target hint without appending. The persisted release proof is - narrow target journal authority: ordinary unconditional mutations append - immediately to the target WAL from `C+1`, while reads and conditional - mutations coroutine-await target initialization within their request - deadlines. Each wait is capped by the smaller of the remaining request - deadline and a server initialization-wait budget. A process-wide lock-free - admission counter bounds total initialization waiters; a full counter or - server-cap timeout returns `TargetNotReady` with a retry delay. Awaiting does - not block an executor thread and uses no separate transfer request queue. - Successful unconditional writes mean the target WAL append is durable; they - do not consume initialization-wait capacity. Every state other than - `TargetCatchingUp` retains its simpler existing request path. -5. `CatchupPublished`: the existing target coroutine incrementally replays only - the sealed source suffix after its preparation cursor through `C`, then - persists the final catch-up proof. Target-WAL records already admitted after - `C` remain durable and are applied after that complete source prefix. The - target cannot answer a read or evaluate a condition from a shorter prefix. -6. `TargetReady`: group 0 replaces the catching-up entry with `Serving` in a - second generation. Only a heartbeat advertising the exact prepared target - allows a matching serving grant. The transition then becomes - `CatalogCommitted`. If materialization advances the catalog before this - separate status write, recovery proves the same target range, owner, epoch, - tree and stream, preserves the newer head and finishes the status write. - -An owner-loss transfer created after lease exclusion may adopt the original -tree and stream under the higher epoch instead of constructing a live overlay. -This path is valid only after old lease expiry plus skew; writer-epoch adoption -then fences any lower-epoch stream handle. Its `TargetReady` proof publishes -directly from the exact source Serving assignment; no catching-up generation -is required. A retained marker from a previous completed split does not block -this publication when the source has no inherited overlay. A conflicting -range, owner, epoch, artifact or reused operation identity still fails closed. - -For a recovered target without an inherited overlay, normal placement cleanup -may remove its completed transition marker before its first root checkpoint. -The already prepared root handle may then claim publication authority from the -exact current `Serving` partition, range, owner, epoch, tree and stream, with -both assignments free of overlays. Marker removal alone cannot invalidate that -published ownership. A different nonempty transition, another assignment, or a -catching-up entry still cannot authorize publication through this exception; -root authority continues to fence lower owner epochs. - -Recovery uses persisted evidence only. Before source release, a target failure -leaves the source serving and target preparation is repeatable. After release, -abort is forbidden: catalog reread decides whether to publish or resume -catch-up. An ambiguous catalog write succeeds only if the exact intended entry -is observed. A restarted target reconstructs from base plus source and target -tails. A restarted source cannot resume writes after an explicit release or -lease-exclusion proof. Heartbeats and loaded pages are readiness observations, -never authority proofs. - -The recovery decision matrix is: - -| Durable evidence | Authoritative action | -|-----------------------------------------------------|---------------------------------------------------------------------------------------------| -| Plan or source base only; no target readiness | Keep source serving; retry preparation or abort unpublished target state | -| Initial target readiness; no source release | Keep source serving; recheck budgets before requesting the fence | -| Explicit release; catalog still names source | Keep source fenced; publish `TargetCatchingUp` or resolve head ambiguity | -| Lease exclusion; catalog still names source | Recover target under the higher epoch; source cannot reactivate | -| Catalog names `TargetCatchingUp`; no final proof | Coroutine-await reads/conditions; append unconditional target WAL; replay through `C` | -| Final catch-up proof; catching-up catalog entry | Publish the exact `Serving` successor and then issue the target grant | -| Serving catalog entry; grant absent or expired | Keep target prepared and reject admission until the exact grant arrives | -| Ambiguous catalog head write | Reread head and pages; accept only the byte-exact intended generation | -| Conflicting artifact, cursor, range, epoch, or proof | Fail closed; never infer authority from local state | - -The root-catalog generation in a target overlay is a recovery pin, not an -observation. It is distinct from the tree snapshot sequence stored inside that -root. `TargetPreparing`, `CatchupPublished`, catalog refresh, and restart must -all open the exact root generation and then compare the recovered tree snapshot -sequence and applied sequence with the proof. Opening the latest root and -merely checking afterward is invalid: a concurrent checkpoint may move latest -forward, while stale root-cache state may leave it behind. - -For both split and balance, exact-open initializes the page store from the named -immutable root and bypasses latest-root cache refresh until that store publishes -its own successor. Successor publication compares the exact bootstrap generation -and checksum with the current root first; if another writer has advanced the -lineage, publication fails instead of branching. The transition pin is persisted -before readiness publication and is removed only after authoritative catalog -state no longer contains the corresponding overlay and transition dependency. - -The balance invariants are: - -- **SERVER-ONE-TRANSITION:** one partition and each participating owner have at - most one active topology transition; -- **SERVER-NO-DUAL-GRANT:** `TargetCatchingUp` is never included in a serving - grant; -- **SERVER-PROOF-BEFORE-PUBLISH:** every catalog phase is justified by the - corresponding durable readiness or release proof; -- **SERVER-LIVE-TARGET-CONTINUITY:** normal catch-up retains the prepared target - handle and replays only the remaining suffix; exact-root reopen is recovery; -- **SERVER-ASYNC-INITIALIZATION:** readiness waits suspend Rust futures and - never block executor threads or create a transfer-owned request queue; only - `TargetCatchingUp` requests enter this bounded path; -- **SERVER-AMBIGUITY-FENCES:** an unknown publication outcome never reopens the - source writer; and -- **SERVER-CLEANUP-AFTER-PINS:** source tree, stream, retry history, and shared - packs remain retained until catalog references, recovery pins, retry floors, - and forwarding grace have all cleared. - -## 8. Placement Policy - -Balancing targets at least `live_owner_count * target_partitions_per_owner`, -defaulting to four partitions per owner. Split chooses the largest eligible -partition using approximate retained pack estimates. A split boundary comes from -a resident index separator or a bounded small key window; exact byte or key -medians are unnecessary. Live-key witnesses establish nonempty children when -the observation is made. Concurrent mutations can change that observation. -The structural query never loads cold pages or flushes the tree. A service -owns at most one background sampling job, and heartbeats use epoch-fenced cached -samples without waiting for that job. Every serving partition is considered; -an unsplittable largest partition does not exclude the remaining candidates. -Retained pack estimates use the opened manifest, including shared packs, rather -than cumulative write counters. The estimate is deliberately coarse and can -lag unsnapshotted mutations; observation does not scan remote metadata or data. -The retained parent and new split child stay local. Range splitting deliberately -uses an approximate boundary and does not guarantee equal child sizes. Split -sizing and minimum partition supply do not require exact placement equality. -A move uses assigned partition counts; byte estimates remain capacity filters. -An indivisible range may prevent further useful count balancing, in which case -placement remains unchanged. - -### 8.1 Count-Based Placement - -Placement currently uses assigned split count only: `w_i = p_i / P`, including -healthy owners with zero assignments. `byte_weight_percent` is reserved and must -be zero; nonzero settings are rejected until root/range statistics are available. -The data contribution is disabled. Physical shared-pack bytes, memtable bytes, -write counters and accumulated split ratios are not placement data weights. -Tolerance defaults to 20% relative to equal owner share; minimum global-loss -improvement defaults to 25%. Missing or epoch-mismatched assignment observations -still defer planning. Capacity/headroom remains a separate safety filter. - -Tree statistics do not yet expose current root/range page, logical-byte or live-KV -aggregates. Existing retained-pack estimates remain coarse inputs to automatic -split sizing/candidate ranking and capacity checks; they can include shared packs -and lag ordinary flushes. Count balancing does not make those estimates exact. -The split execution, boundary sampling, recovery and publication contracts remain -unchanged. Console does not display Weight or weighted balance explanations. - -### 8.2 Tolerance and Move Selection - -Equal owner weight is `1 / N`. The observation's relative deviation is -`D = max_i(abs(N * w_i - 1))`; `imbalance_tolerance_percent` bounds `100 * D`. -Within tolerance, no load-balancing move is planned. Outside tolerance, evaluate -candidate moves against the same global loss -`L = sum_i((N * w_i - 1)^2)` before and after the move. The partition count -contribution moves; the byte contribution is zero. The normalized denominators -stay fixed for each candidate evaluation. Only the two affected owner terms -change, so each candidate's loss delta requires constant arithmetic. - -A candidate must strictly reduce loss and meet -`minimum_weighted_improvement_percent` of the current loss. A count difference -does not authorize a move that worsens the global count score. Choose the greatest loss -reduction among safe eligible candidates; stable partition/target identities -break ties. Use validated bounded integer/fixed-point arithmetic, sufficient -intermediate width, and conservative threshold comparisons; UI rounding does -not decide eligibility. With unchanged observations, a reverse move increases -the same loss and cannot qualify. Observation changes remain subject to -thresholds and cooldown, rather than triggering actions for every small change. - -For example, two owners with weights 20% and 80% can move a partition whose -contribution is 40 percentage points to obtain 60% and 40%. A relative tolerance -of 20% includes that result because equal owner weight is 50%. This is an -illustration of configured thresholds, not a promise that split yields two -40-point children. When no safe indivisible candidate offers enough improvement, -report no useful move and keep the placement. - -Request rate, target headroom, fresh health, exact assignment and independent -recoverability are safety filters. A parent-tail overlay is ineligible. One -partition and each participating owner have at most one active transition. -Default per-partition cooldown is one minute; cooldown supplements scoring and -tolerance. Split pacing includes recent splits and transfers; repeat-placement -pacing includes only transfers, so split does not postpone first placement. -Transfer preparation, estimated catch-up and forwarding retain independent -ten-minute safety windows. A safe candidate outside tolerance that meets the -improvement threshold must make placement progress within forty seconds when -its cooldown has elapsed; splitting alone is not placement progress. - -### 8.3 Decision Observation - -The monitor's bounded decision observation identifies policy coefficients and -thresholds, catalog generation, assignment/owner identities, observation time, -source freshness, counts, estimated bytes, count/byte weight contributions, -owner weights, deviation and loss. It records the selected or best rejected -candidate's predicted weights and improvement, plus move/no-move reason: -within tolerance, insufficient benefit, unavailable observation, cooldown, -active transition, inherited overlay, unhealthy owner or capacity/rate limit. -Policy version identifies the scoring formula. Diagnostics retain at most 256 -owners, 4096 partition contributions and 2 MiB, independently of the planner's -complete metadata input. Omitted details display unavailable; they do not alter -planning. Unchanged diagnostics are renewed within half their freshness window -rather than written on every tick. Do not emit or persist all candidate pairs -on every tick. This diagnostic -observation is not catalog authority and does not advance catalog generation. -The observation remains backend diagnostic metadata; Console currently hides -Weight and balance explanations. Catalog generation/identity mismatch remains -explicit; UI refresh does not bypass stale cursor rejection. - -## 9. Lifecycle and Observability - -Startup ensures the monitor, registers the instance, loads a complete catalog, -opens each assignment's latest tree root and Stream chunk record as `Prepared`, -reads the applied sequence and WAL replay offset from that tree root, replays -through the stream's current durable tail, reports readiness, and serves only -after installing a matching grant. Shutdown -atomically stops new admission and clears authority; already admitted R142 -operations retain handles and finish before bounded checkpoint/drain work. - -`begin_drain` closes sampling admission before background workers stop. Before -draining hosted WAL writers, `drain_writers` waits for the sampling job's partition -payload and captured Tokio `Handle` to finish teardown while the driver remains -alive, including when queued work is discarded or the job unwinds. Closed sampling -cannot relaunch; completed drain clears its cache and permits no further publication. -This barrier owns sampling only. OS I/O remains uncancellable, and the existing -10-second monitor and 15-second console stop grace periods remain unchanged; -forced termination retains normal writer-lease exclusion. - -Catalog refresh recovers every new or changed local assignment first, then -replaces the catalog and hosted-partition snapshots; unchanged exact-epoch -handles remain live and departed assignments are dropped. -An unknown journal or post-journal apply outcome moves the affected writer to -`Recovering`, closing its admission and excluding it from serving grants. -An unchanged catalog cannot repair that terminal handle. The process detects -this state, clears authority, stops RPC admission and exits so its supervisor -can relaunch it. The successor reloads the durable catalog, tree checkpoint, -complete WAL and persisted split/transfer proofs before obtaining a fresh grant. -No uncertainty flag is cleared in place and no recovery artifact is deleted. -Diagnostics identify the partition, owner epoch and durable/applied sequences; -ordinary `Prepared` activation and idle instances do not trigger a restart. -The recovery check includes prepared transfer targets and retained split -dispatchers with both ingress writers, even after a newer catalog replaces their -assignments. Current-range readiness cannot hide terminal uncertainty in a -compatibility route that remains reachable by older client references. -Catalog-head watch notifications trigger reconciliation immediately and check -the matching serving grant afterward. Instance-specific grant notifications -install authority without reopening or rescanning catalog assignments. -The configured periodic refresh remains -the fallback when notifications are unavailable; grant renewal also continues -on the heartbeat path. Publication therefore does not wait for a full polling -interval before a healthy owner observes a new routing generation. - -Configuration exposes identity, group-0 seeds, separate RPC listen and routable -advertise addresses, dedicated 15xxx HTTP/RPC ports, hosted-partition capacity, -refresh/drain intervals, timing policy, and balance policy. Validation rejects -unsafe timing, zero bounds, bad addresses, unspecified advertise addresses, and -empty discovery seeds. Automatic child-owner balance is enabled by default and -may be disabled explicitly with the balance policy's `enabled` setting without -disabling local partition splitting. Lock-free counters distinguish successes, redirects, -local stale-route dispatch, lease and deadline rejections, overload, split -preparation/base/tail/fence/overlay/materialization work, and internal errors. -Balance observation records selection inputs, base and tail cursors, readiness -failure, catalog and lease phase, catch-up, and background work. Health and -heartbeat views derive from the same sorted partition snapshots and catalog -generation. - -## Open Issues - -- **Prepared transfer crash coverage:** preparation preserves source tree - authority; mutable target authority requires the exact published owner - assignment. Fault injection must cover source and target restart around - that publication, including overlapping source checkpoint work. -- **Split handoff crash coverage:** the durable handoff status precedes local - dispatch and preserves the original parent tree and WAL. Recovery composes - the exact child base, range-filtered parent suffix and child WAL. Fault - injection must cover status persistence with continuing parent writes, - dispatch before final readiness and an unconfirmed status response. Status - persistence remains outside the mutation worker. -- **Head publication before phase persistence:** transfer and split publication - and their committed transition markers are separate writes. Recovery must - prove exact catalog results and finish markers after a crash in between; - fault injection must verify admission and eventual recovery at both transfer - publications and the split publication. -- **Abort racing publication:** an abort decision must be serialized with - publication under the monitor fence and transition revision. A previously - loaded readiness proof must not publish after an authoritative abort. Verify - this race before treating cleanup as safe. - -- Real-process fault injection should continue expanding coverage of ambiguous - catalog writes and process death at every balance phase. -- Overlapping remote transfer preparation, local split materialization and - source restart need fault coverage that proves exact catalog and transition - reconciliation without weakening single-writer or replay epoch checks. -- Completed transfers remove their catalog overlay, but physical reclamation of - the departed source tree and stream, plus expiry of the stale-route - forwarding grace period, still need a manifest-fenced background policy. diff --git a/doc/design/chunkds/design-crowdb-chunk-kv.md b/doc/design/chunkds/design-crowdb-chunk-kv.md deleted file mode 100644 index f1d7d8daf..000000000 --- a/doc/design/chunkds/design-crowdb-chunk-kv.md +++ /dev/null @@ -1,376 +0,0 @@ - - - -# Chunk-Backed Range KV - -`crowdb-chunk-kv` is the embeddable range-partitioned state machine for the -shared-storage metadata index. It composes crowdb-tree's runtime-selected -chunk page store with a private chunk-stream WAL. It does not depend on the -Paxos-facing `crowdb-paxos-kv::KVEngine`, and partition count is independent of node -count. - -## 1. Identity and Ownership - -A partition has a stable 128-bit ID, an unsigned-bytewise half-open range -`[start, end)`, one ownership epoch, one tree, one distinct durable stream -name, and one partition-local mutation sequence. Unbounded endpoints are -allowed. A split key must be strictly inside the source range and its two -child ranges must be adjacent and exactly cover the parent. - -The lifecycle is `Closed`, `Recovering`, `TransferQuiesced`, `Prepared`, -`Serving`, `TransferFencing`, `SplitPreparing`, `SplitFinalizing`, `Retired`, -or `Faulted`. -Data-path admission reads atomics and reserves bounded request and byte -capacity. Lifecycle control closes mutation admission and waits -asynchronously for the admitted count to reach zero. The manager registry and -split transition record use lifecycle-only locks; point reads and mutation -admission do not acquire them. - -## 2. Tree and Journal Ownership - -Every partition owns a `PartitionTree` and `PartitionJournal`. Production -adapters wrap crowdb-tree and R141 `ChunkStream`; injected implementations -support deterministic tests. The raw stream handle is private, so a server -cannot bypass partition ordering, trim, or epoch checks. - -The tree enforces its logical request range at construction. A retained parent -keeps the original physical root, whose pages may still contain keys and -separators outside the narrowed range until background pruning completes. -Reopening that root validates frame checksums and structure without treating -those physical keys as corruption; point reads, scans and mutations still -enforce the current logical range. Importing a newly rebuilt range snapshot -continues to require every imported key to fit its destination range. -Rust never implements the tree page -path through `crowdb-chunk-client`; the production tree selects R140's native -page-store backend at runtime. A balance target opens the source's pinned tree -manifest directly from shared page chunks and owns a distinct target WAL. It -does not copy keys through Rust or copy page packs before handoff. - -Point reads, ceiling/higher/floor/lower, and bounded forward/reverse scans read -only the applied tree prefix. A forward scan uses an inclusive lower bound and -exclusive upper bound; its distinct continuation cursor resumes strictly after -the last emitted key. Forward operations use the native merged lower bound. -Reverse operations use a native predecessor descent that fixes the L0 -memtable set, root page, and GC floor for the page, merges the highest revision -for each L0/L1 collision, and skips tombstones before returning descending -keys. Both directions enforce count and byte budgets inside C++; Rust does not -compose point reads, materialize the range, or sort results. - -## 3. Mutation Ordering - -`RequestId` contains a 128-bit client instance ID and a 64-bit client sequence. -A canonical SHA-256 digest covers operation kind, key, value, and condition, -but excludes routing and ownership fields. Reusing an ID with another digest -returns `RequestConflict` before I/O. - -One bounded MPSC sequencer assigns monotonically increasing mutation -sequences. It evaluates put, delete, put-if-absent, compare-exchange, and -conditional delete in queue order against the applied tree plus successful -earlier entries staged in the same batch. Both applied and condition-failed -outcomes become checksummed WAL records. Therefore a lost response can be -retried without reevaluating its condition. - -The sequencer appends frames to the chunk stream in mutation order. It changes -the tree only after the append is durable, applying one successful record per -tree call and advancing a failed condition as a no-op. A response is released -only after the applied frontier advances and carries the frame's stream name -and starting logical offset. Ordinary concurrent reads see the prior applied -prefix; a minimum journal position waits asynchronously for explicit -read-after-write ordering. - -## 4. WAL and Recovery - -Each physical frame has magic, version, flags, a bounded body length, a -serialized logical record, CRC32C, and a canonical 128-bit physical chunk ID -trailer. The CRC covers the header and body, not the trailer. The journal -submits header+body+CRC through chunk-bound stream append; the stream selects -the chunk and adds its identity. Recovery validates frame completeness, -checksum, source-chunk identity, partition identity, epoch, operation digest, -result revision, sequence continuity, and request-ID uniqueness. The durable -acknowledged cursor is the recovery upper bound, so identity validation never -promotes residual bytes. Recovery never reevaluates a conditional result. A -timestamp is omitted; any future timestamp is diagnostic-only. - -A checkpoint contains the explicit 64-bit R140 tree identity, tree manifest -identifier, applied mutation sequence, stream name, and replay offset. The -tree identity is allocated and persisted; it is never derived by truncating -the partition's 128-bit identity. Recovery rejects a checkpoint or prepared -child whose tree identity differs from the injected tree. The replay offset is -the oldest WAL record needed by retained retry results, rather than simply the -current tail. -Recovery reads those pre-checkpoint frames to rebuild the retry cache without -reapplying them, then applies only the suffix above the checkpoint sequence. -WAL trim cannot cross the checkpoint's replay offset. Requests below the -declared retained floor return `RequestExpired` and are not executed anew. - -Serving maintenance scans retired ordered WAL chunk records within the configured -record and byte budgets. Collection retains the current tail, records above the -existing trim offset or generation horizon, and chunks protected by live interval -pins. It releases at most one eligible physical chunk and its record per pass, -then recaptures the retention proof before another deletion. A successful deletion -schedules another bounded pass after yielding to other hosted partitions; a pass -without deletion resumes on the periodic tick, even if the generation is unchanged. -Maintenance does not publish a new tree root or advance WAL trim or tree retention. -Split preparation and exact-root transition pins suppress metadata reclamation -under the existing maintenance exclusion. Source checkpoint publication and its -durable generation pin share that exclusion, so GC cannot pass between them. - -Physical WAL trim at a nonzero replay offset requires durable expired-request -floors as well as retained results. That floor state is not yet checkpointed, -so the explicit trim API rejects such reclamation with `MaintenanceDegraded`. -The WAL remains retained until the recovery format carries this state; keeping -only the recent result window would let expired requests execute after restart. - -The frontiers satisfy -`checkpoint_seq <= applied_seq <= journal_durable_seq`. The stream resolves -uncertain cursor and manifest outcomes against durable state before completing -the append. If a journal append nevertheless returns an error, the partition -enters `Recovering`; later writes and reads require recovery rather than -mistaking that failure for a quiesced transfer source. A non-OK tree result -after durable append is `ApplyStateUnknown`, also moves only that partition to -`Recovering`, and prevents later records from applying. The C ABI catches C++ -exceptions before they can cross into Rust. - -## 5. Overlay Split and Writer Handoff - -A `SplitPlan` names one transition, the exact parent ID/range/epoch, an -interior split key, the parent's next epoch, and one new child identity. The -existing parent retains its partition, tree, stream, and owner identities and -shrinks to `[old_start, split_key)`; the new child receives -`[split_key, old_end)`. Repeating the same active plan is idempotent; changing -any identity, range, epoch, or stream fails closed. The new child initially -remains on the parent owner. Placement is a later balance operation. - -Split creates only the child's tree and WAL. The retained parent uses its -existing tree, WAL and mutation path; it is not rebuilt into a second -destination tree or moved to a replacement WAL. Local handoff means that -traffic still enters the parent, which dispatches the child range to the child. -It is distinct from the later catalog publication that exposes both ranges to -external routing. A higher candidate epoch alone proves neither boundary. - -Split preparation follows these ordered steps: - -1. Enter `SplitPreparing` and range-rebuild the child base from one exact - pinned parent tree view. The parent remains the only mutation sequencer and - continues its normal WAL and memtable path while the child tree and WAL are - prepared. -2. Establish an ordered parent frontier `C` and persist a handoff status in - the split transition, binding the parent and child identities, epochs, - pinned base and WAL replay boundaries. Successful durable status update is - the local handoff commit point. Resolve an uncertain result by reading that - exact transition. Parent appends and reads continue while this control-plane - operation awaits persistence; the mutation worker does not wait for it. - Do not add a split-owned request waiting queue. An in-memory cutover message - is not durable handoff evidence. -3. After durable handoff, register a bounded observer of parent mutations. - Background registration joins only writers admitted before the observer; - those writers retain their registration guard through frontier completion. - Capture `A = max_seen` afterward, including pre-registration out-of-order - writes that may exceed the contiguous frontier. Subsequent writers record - shared cell references in an independent delta, so parent source rotation - and flush cannot remove the observed suffix. -4. Replay the child through a fixed contiguous prefix `P >= A`. In background, - prepare its complete L1 view through `P`, replacement sources and publication - storage, then acquire the child's generation lease. The lease closes only - the private child's ingress. Parent mutations continue into the observer. - This prepared view does not replace the pinned durable base used for crash - recovery. -5. At the ordered mutation-worker boundary, select the applied frontier `F` - and its exact synchronized WAL frame end `endF`. The durable append frontier - must equal `F`; `endF` advances only after each frame's apply or no-op succeeds. - Freeze the observer and install the prepaid child prefix through `F`, then - publish local key dispatch. This short commit performs no parent source seal, - replay, checkpoint or backend I/O and does not wait for CPU work. A stale, - busy or expired preparation fails before installation. Once installed, the - token is idempotent only for that exact `F`. -6. Complete readiness in background. Publish the parent journal metadata and - require its recoverable end to cover `endF`, then publish the child's - inherited view. A later checkpoint may cover additional parent writes but - cannot change the fixed `F` or `endF`. Retain the exact artifact and shared - replay dependencies across publication failures until the complete prefix - and durable storage proofs form one immutable `SplitArtifact`. Handoff - status alone is not final readiness. -7. Publish one catalog generation that shrinks the retained parent (same ID, - new epoch) and inserts exactly one new child. The parent owner already has - both writers active, so refresh only publishes this catalog and serving - grant to external routing; it does not gate local handoff, recover the - retained parent, or replay a cutover. - -The observer and replacement controls are prepaid on the child's account with -finite record and byte limits. Parent and child share one host so cell payloads -retain their existing allocation identity; new reference metadata is charged before -insertion. Exhaustion expires the split attempt without rejecting the parent -mutation or growing the delta further. The child reads its prepared prefix and -frozen suffix with range filtering and its own live memtable; newer child -revisions win over inherited entries. Conditional mutations consult that same -merged view. Writes after dispatch use the child WAL for the child range and -the unchanged parent WAL for the retained range. The child inherits bounded -request outcomes so retries do not reevaluate conditions, and a mutation racing -dispatch keeps exactly one writer and its original request identity. - -The token owns both Trees and the old child-generation controls through commit. -Explicit child close may revoke a prepared lease; if close wins, installation -fails before changing the child. If installation wins, close joins that short -commit. Returning unused credit and destroying old controls can run retirement -or budget callbacks, so they belong to host cleanup, not the route boundary. -Before enqueueing cutover, a started host cleanup task holds the final token -owner. The worker releases its reference before completing the request; a -cancelled, unqueued request releases that reference before closing completion. -Cleanup therefore remains off the mutation worker even if the control caller -is cancelled. Cancellation after dispatch retains the installed route and exact -artifact: retry finalizes that same prefix instead of reinstalling over an -acknowledged child WAL. - -The child base checkpoint records two independent counters: its logical tree -snapshot sequence and its chunk root-catalog generation. Immediately after the -child base is durable, preparation persists -`root//pin/ = child_root_generation`. Readiness is -not recorded until that operation succeeds. A retry with the same generation is -idempotent; another generation under the same transition identity is corruption. -Preparation may remove the unpublished child's pin after an authoritative abort -only before handoff commit. After handoff commit, recovery continues the same -split and retains its base, WAL and pin even if final readiness is absent. - -A prepared or restarted child recovers in three layers: open the exact base -manifest, replay the retained parent suffix through the final inherited frontier -with range filtering, then replay its own WAL from the recorded child start. -Before final readiness exists, the first child WAL sequence determines -`F = first_child_sequence - 1`. If no child frame exists, recovery can extend -inheritance through the recovered parent tail because there is no acknowledged -child suffix to overwrite. The recovered parent writer first scans its -synchronized tail and publishes the required prefix before an independent -metadata-based reader is used. The earlier `C` alone cannot establish final -parent coverage, and a stale read-only metadata cursor is not evidence of data -loss. - -Records at or below the base sequence restore retained request outcomes without -reapplying tree mutations. Failed conditions remain no-ops with their original -result. Sequence gaps, a wrong source stream or epoch, a mismatched range, or a -child WAL beginning anywhere other than the proof's exact child start reject -recovery. Parent writes acknowledged between `C` and dispatch remain covered; -left-range writes after `F` cannot extend an already installed child's artifact. - -The child recognizes two minimum-position namespaces. A retained parent-stream -position covered by the inherited suffix is satisfied by the inherited overlay -frontier. A child-stream position waits for the child's applied frontier. New -child mutations return only child-stream positions; retained-parent mutations -continue returning parent-stream positions. This preserves read-after-write across a -catalog split without treating two streams as one offset space. - -After activation, bounded background passes independently prune the retained -parent tree to its smaller range and materialize the child's shared page -ownership. A checkpoint of the complete child view into its own tree root and -WAL permits the catalog to clear the parent-tail overlay and mark the child -independently recoverable. Parent manifests, stream bytes, retry results, and -page packs remain pinned while the child catalog artifact, a retained -checkpoint, forwarding grace interval, or retry floor references them. -The catalog generation that clears the child overlay also clears the retained -parent and child transition markers atomically. Only after the child owner has -installed that authoritative generation may it idempotently release the -transition's child-root pin. A crash between catalog installation and unpin is -a retention leak resolved by reconciliation, never permission to reclaim early. - -The split invariants are: - -- **SPLIT-PARENT-STABLE:** split preserves the parent partition, tree, stream, - owner, and lower range boundary and creates exactly one new child; -- **SPLIT-ONE-WRITER:** before in-memory dispatch changes the parent sequences - the full range; afterward parent and child sequence disjoint ranges; -- **SPLIT-ROUTE-FRONTIER:** durable handoff `C`, observer high-watermark `A`, - prepared prefix `P` and dispatch frontier `F` are distinct boundaries. The - child's complete `P` view and observed `(P, F]` suffix cover every parent - mutation through `F`; recovery also includes the complete child WAL. No - acknowledged mutation falls between them or executes twice; -- **SPLIT-HANDOFF-DURABLE:** persist handoff before child write admission; - restart after commit resumes the same split even before catalog publication; -- **SPLIT-OVERLAY-DURABLE:** base plus parent suffix plus child WAL reconstructs - values and request outcomes before activation; -- **SPLIT-NO-FOREGROUND-WAIT:** routing does not wait for memtable seal, - parent-history replay, filtered publication, checkpoint, catalog publication, - ownership materialization, budget callbacks or old-owner destruction; and -- **SPLIT-PIN-BEFORE-RECLAIM:** physical deletion never precedes the last - catalog, snapshot, forwarding, or retry reference. - -The no-wait boundary covers route installation and token cleanup ownership. -The background parent Stream checkpoint can still serialize later appends to -that same Stream; the independently routed child WAL does not wait for that -publication. This distinction does not permit pausing parent writes during -observer preparation or durable handoff persistence. In-memory routing success -is not a substitute for the Stream and storage backends' durability and fencing -proofs, which readiness and crash recovery must validate separately. - -## 6. Remote Balance Storage Contract - -Balancing one independently recoverable child reuses the same base-plus-tail -representation. A live source checkpoint supplies the pinned base manifest, -base sequence, Stream chunk record generation, and replay offset. The target owns -a distinct WAL and opens the shared tree root as `Prepared`. One shared async -initialization coroutine replays the source stream while the source continues -serving. It first reaches a persisted preparation cursor `P`, then follows the -small moving source tail. Awaiting this coroutine suspends only the caller's -Rust future; it does not block an executor thread, poll an atomic frontier, or -place the request in a transfer-owned pending queue. - -After target readiness, the source checks record, byte, estimated catch-up, -and preparation-deadline budgets before changing journal ownership. It stops -assigning new records to the source WAL, drains only records already assigned -there, and records their final durable sequence and byte offset as `C`. One -short `TransferFencing` lifecycle closes public mutation admission while the -existing mutation worker drains the already-admitted set; it is not a second -queue or an independent authority flag. Only after that drain does the source -enter `TransferQuiesced`, which permits the exact handoff checkpoint. The -target artifact is extended from `P` to `C`. The live target keeps its opened -tree, memtable, and replay coroutine and reads only `P+1..C`; reopening the -pinned base and replaying the complete retained suffix is crash recovery, not -the normal handoff path. - -Once the release proof fixes `C`, ordinary unconditional mutations may append -immediately to the target WAL beginning at `C+1`; they do not wait for source -tail replay, a checkpoint, or page materialization. Success means that target -WAL append is durable; their tree application remains ordered after the source -suffix and proceeds asynchronously. Reads and conditional mutations await -the initialization coroutine because they require the complete source prefix. -After the coroutine applies through `C`, the partition enters its normal -serving path: reads execute directly and conditional mutations evaluate -against the complete view. Target-WAL application then continues in order. -No request handler retains a thread while awaiting initialization. Both each -wait and the process-wide waiter count are bounded; capacity exhaustion returns -the same retryable initializing result as a bounded wait timeout. -Only catalog entries in `TargetCatchingUp` execute these checks. A `Serving` -entry takes the ordinary direct read or mutation path without inspecting an -initialization future or touching waiter accounting. - -After catalog and grant activation, only the target WAL can advance. A -dead-source recovery may adopt the original stream directly, but only after -the old lease expiry plus clock skew proves exclusion. - -The balance invariants are: - -- **BALANCE-PREPARE-WHILE-SERVING:** live target preparation never fences the - source writer; -- **BALANCE-RELEASE-BEFORE-ACTIVATE:** target mutation authority requires a - durable source release or lease-exclusion proof; -- **BALANCE-APPEND-WITHOUT-TREE-WAIT:** after release, unconditional target - mutations may enter the target WAL before source-tail application completes; -- **BALANCE-COMPLETE-PREFIX:** a target serves only after replay through `C`; -- **BALANCE-ASYNC-READINESS:** reads and conditional mutations coroutine-await - initialization without an executor-thread block or transfer request queue; -- **BALANCE-INDEPENDENT-SOURCE:** a child retaining a split-parent suffix is - ineligible for another owner handoff. - -## 7. Failure Containment and Observability - -Range, epoch, size, condition, request-conflict, expiry, and admission errors -do not damage partition health. Availability failures on a tree read remain -localized to that read. Corruption, conflicting recovery records, and unknown -post-journal apply state require recovery of the affected partition only. -Checkpoint and GC failures retain the prior manifest and WAL authority. - -Per-partition lock-free counters cover mutation requests and outcomes, ordered -seeks and scans, range and stale-epoch rejection, admission backpressure, -journal failures, unknown apply outcomes, recoveries, checkpoints, and split lifecycle -events. Split counters distinguish preparation and base-checkpoint time, -tail records and bytes, catch-up lag and finalization duration, overlay replay records and -bytes, and materialization duration. Snapshot frontiers expose lifecycle, -stream identity, durable sequence and byte offset, and applied sequence without -combining independent partitions. diff --git a/doc/design/chunkds/design-crowdb-chunk-stream.md b/doc/design/chunkds/design-crowdb-chunk-stream.md deleted file mode 100644 index d4880209d..000000000 --- a/doc/design/chunkds/design-crowdb-chunk-stream.md +++ /dev/null @@ -1,627 +0,0 @@ - - - -# CROWDB - Design: Chunk Stream - -The chunk stream presents finite mirrored chunks as one durable, ordered -logical byte space. Append completion depends on mirrored data persistence; -Stream metadata records lifecycle transitions and bounded chunk record checkpoints. -The stream owns physical framing and addressing; the partition WAL owns -business-record framing, request identity, and replay policy. - -Depends on: [chunk IO](../chunkio/design-crowdb-chunkio.md), -[chunk reader](../chunkio/design-crowdb-chunkio-reader.md), -[DiskIO](../diskio/design-crowdb-diskio.md), -[chunkdb](../chunkdb/design-crowdb-chunkdb.md), and -[group-0 sysdata](../kv/design-crowdb-paxos-kv-group0.md). - -## Contents - -1. [Ownership and API](#1-ownership-and-api) -2. [Storage contracts](#2-storage-contracts) -3. [Metadata](#3-metadata) -4. [Append protocol](#4-append-protocol) -5. [Rollover and recovery](#5-rollover-and-recovery) -6. [Reads](#6-reads) -7. [Trim and reclamation](#7-trim-and-reclamation) -8. [Bounds and observation](#8-bounds-and-observation) -9. [Errors and invariants](#9-errors-and-invariants) -10. [Open issues](#10-open-issues) - -## 1. Ownership and API - -`crowdb-chunk-stream` is an asynchronous Rust library. `StreamName` is an -opaque, time-ordered 128-bit identity bound to a logical partition, never to a -process or node. It renders as 32 hexadecimal digits. `ChunkStream` exposes -`create`, `open`, `open_read_only`, `tail`, vectored `append`, exact `read_at`, -seekable bounded sequential reading, `trim_prefix`, and `close`. -An owner-side flush/checkpoint barrier captures and persists a synchronized -prefix on demand without changing ordinary append completion. - -Every nonempty append returns the selected chunk identity and exact logical -range. Logical lengths count caller payload bytes, excluding physical frame -headers and footers. A zero-length ordinary append returns no chunk identity -and performs no IO. `append_chunk_bound` and `append_chunk_bound_batch` keep -chunk selection in the ordered worker; the outer Stream frames carry the -selected chunk identity and CRC. They add no payload trailer. The stream does -not interpret the caller's business-record format or retry identity. - -One append payload is one indivisible validity unit. A successful append makes -the whole payload available; recovery never exposes a proper prefix of that -payload. Physical frame and IO splitting are internal. Batching preserves each -request's append boundary rather than making the entire batch one group. - -Each writable handle owns one bounded Tokio MPSC queue and one worker task. -The task exclusively mutates the active chunk's in-memory cursor and mappings, -logical tail, and lifecycle publication state. Producers and readers use -atomics and immutable `ArcSwap` views. Append and snapshot publication add no -hot-path lock. Batching preserves request order and exact returned ranges. - -`open_read_only` validates one exact writer epoch and captures a finite view. -It does not adopt writer authority, allocate a successor, seal a chunk, or -publish writer state itself. It may request a flush/checkpoint from the current -owner. If the owner is unavailable it uses the consistent prefix persisted in -ChunkDB and records that checkpoint as its finite bound; that view can lag -successful appends. A caller requiring a newer exact cutover must not silently -accept the older fallback. The handle supports source-tail replay during split -and balance with these explicit bounds. - -## 2. Storage Contracts - -Three injected async traits separate Stream state from placement and transport: - -- `StreamRegistry` loads and creates the stable group-0 `StreamRecord`. - It is used by create/open, never by ordinary append or data reads. Typed - binary keys contain the Stream ID; metadata defaults to nonzero group 1. -- `StreamMetadataStore` selects the nonzero group containing ordered - `StreamChunkRecord` values. A shared record store implements conditional - lifecycle transitions, reverse seek, candidates, interval leases and GC. -- `StreamChunkStore` allocates and seals mirrored chunks, writes and synchronizes - the required replicas, exposes current chunk layout for bounded reads and - recovery, and idempotently releases trimmed chunks. It does not require a - separate ChunkDB cursor advance for each append. - -Chunk IO and ChunkDB own `chunkinfo`: strips, physical capacity, replica -placement, growth, replacement, and their durable internal transitions. Each -such transition updates chunk information independently. Stream metadata refers -to stable chunk identities and logical mappings; internal chunk changes are -transparent and never trigger Stream chunk record publication. -Recovery and read-only opening resolve current capacity and layout through the -chunk layer instead of trusting capacity cached in an older Stream checkpoint -or a retained adapter. Opening queries authoritative chunk metadata even when -the adapter has a cached layout; an unavailable query fails the open rather than -falling back to that cache. The checkpoint must still fit the current capacity. - -DiskIO writes carry a boolean sync option. For a synchronized request the server -performs the ordered write, then the required flush, and returns one response -after both succeed. The client does not wait for a write reply and then send a -second fsync RPC. Buffered writes do not request the flush. When transport -limits require several requests, preserve strip-block write ordering and -complete required synchronization before acknowledging the append. - -Every allocation preserves the durable `StreamPurpose`: system journals use -`ChunkType::Wal`, business streams use `ChunkType::Stream`. Both use the -canonical `stream/` owner key followed by the 128-bit stream identity. Reopen -rejects purpose disagreement between StreamRecord, StreamChunkRecord, chunk identity, and -chunk information, including a missing or mismatched owner key. Chunk-KV -journals require Wal streams. - -`ProductionStreamRuntime` shares the process KV client and discovered chunk IO -routes. The Chunk-KV server chooses the metadata store; the StreamRecord -chooses its metadata group. `create_registered` initializes a control-plane -StreamRecord. The `test-util` implementation supplies deterministic mirror-write, -sync, tail-content, ownership, and metadata-publication fault injection. - -## 3. Metadata - -### 3.1 Identity and ordered keys - -Group 0 stores the stable `StreamRecord`: purpose, Stream ID, metadata group, -record version, registration state, consumer kind and ID, and creation time in -Unix milliseconds. Writer leases are separate Group-0 records. Append, -preparation, rollover, close and writer relocation preserve the identity value -and original creation time. Consumer identity describes who owns the Stream; -it does not identify the currently leased process. - -A nonzero metadata group stores `StreamChunkRecord` values under typed keys: - -```text -record namespace | stream_id[16] | kind[1] | order[8 big endian] -Attached: kind = 0, order = logical_start -Candidate: kind = 1, order = reservation_sequence -``` - -Each record has a chunk ID, purpose, state, reservation/attachment sequence, -writer epoch/session, publication generation, trim watermark, closed status and -compressed mappings. Attached records contain explicit half-open -`[logical_start, logical_end)` bounds. A Candidate has no assigned logical -range and accepts no user data. Attached Active and Sealed records sort by -logical start; candidates occupy a distinct suffix range. Reverse scanning the -attached prefix returns the current tail without a separate moving head. - -The tail record carries Stream trim/closed state and the publication counter. -A successor inherits that control state. An empty Stream has an empty Active -record from creation; close seals it and persists closed status. The tail -record is retained even when every byte is trimmed. The Active record's end -is a durable checkpoint, not a limit on recovery or the live in-memory tail. - -There is no persisted Stream-wide physical offset and no duplicated -`physical_end` in a StreamChunkRecord. Chunk-local physical positions derive -from append geometry; the chunk's exact seal length is authoritative at release. - -### 3.2 Compact complete-append mappings - -One mapping describes one complete append group, regardless of frame count. -Store append lengths, from which both logical and chunk-local physical offsets -are derived. With grouped-frame payload capacity 65,489 and overhead 47 bytes, -a group of length `L` advances physical position by -`L + 47 * ceil(L / 65489)`. A physical frame maximum of 64 KiB does not impose -a 64 KiB append maximum. - -Lengths are encoded in blocks of at most 256 entries. Equal lengths use a -constant run; other blocks choose canonical variable-length u32 values or raw -u32 values by encoded size. In-memory samples store each block's logical and -physical base and encoded position. A random seek decodes at most one block. -These blocks are an encoding detail inside one record, not persisted pages or -a Stream-owned index tree. - -The checksummed record validates key/start identity, purpose, exact decoded -logical coverage, group lengths and bounded physical geometry. A new checkpoint -preserves all previously published append boundaries. Rollover validates exact -continuity between the finalized predecessor and successor. Bounds detect -inconsistent coverage; CRC32C also detects same-length corruption. - -### 3.3 Physical framing - -All consumers use one versioned physical frame format and common Rust/C++ -codecs. Each frame is at most 64 KiB including header and footer. Magic identifies -purpose and version; a common header role identifies ordinary, head, or follower -frames. Only Stream uses grouped roles. User-data and tree consumers use ordinary -frames; user-data completeness remains a whole-object publication condition. -Native buffers retain bounded streaming and zero-copy ownership. - -Each nonempty Stream append is exactly one verifiable frame group in one chunk. -Its head declares total payload bytes and frame count, including the head. Even -a single-frame append is a head with count one. Followers identify their head -by its physical offset in the same chunk and carry their ordinal. Frame lengths, -roles, group identity, order, count, total payload length, CRC, and expected chunk -identity must all agree before the group is valid. There is no separate commit -frame or commit RPC. CRC32C covers the common header, group fields, and payload; -the footer chunk identity is validated separately against the expected chunk. -All codecs use the same CRC input and byte layout. -Declared lengths/counts are checked against append limits, remaining chunk -capacity, and checked arithmetic before scanning or reserving memory. A parser -does not allocate buffers proportional to an untrusted declared group size. - -Role-dependent header sizes determine payload capacities. Interior frames fill -their role's payload capacity and only the last may be short. Location arithmetic -derives group geometry from the format version, role capacities, and total -logical length, without one mapping per frame. - -Revision 2 retains the 14-byte fixed prefix: little-endian magic, payload offset, -payload size, and write timestamp. The low magic byte is 2; the high byte keeps -the public purpose. The role byte follows the prefix. Ordinary frames use role -0 and a 15-byte header, leaving 65,501 payload bytes. Stream heads use role 1 -and add total payload length (u64) and frame count (u32); followers use role 2 -and add head physical offset (u64) and ordinal (u32). Both grouped headers are -27 bytes and leave 65,489 payload bytes. Group fields are little-endian. -The 20-byte footer stores the little-endian CRC32C followed by the big-endian -chunk identity. Revision-2 CRC starts at zero and excludes the footer. - -Only the canonical revision-2 frame format is accepted. Unknown revisions, -roles outside their purpose and noncanonical header offsets are rejected. - -`StreamChunkRecord` uses a single current `CSCR` version-1 envelope with -exact body length and CRC32C covering header and body. Decode rejects unknown -versions, trailing bytes, invalid states, excessive counts and oversized bodies -before using mappings. No migration or older-service fallback is supported. - -### 3.4 Atomic publication - -A lifecycle publication conditionally updates the captured tail key revision. -Rollover atomically finalizes the predecessor, deletes its registered Candidate -reservation and inserts the new Active record under its actual logical start. -An empty predecessor and successor may share the same logical-start key; only -the successor remains attached there. Same-chunk checkpoint, trim and close -update only the tail record. Historical chunk records retain their bytes. - -The first attachment reserves Candidate sequence zero, then atomically deletes -that reservation and inserts the empty Active record, guarded by the Candidate's -concrete revision. It checks that no attached tail exists after reservation. -It never uses an absent attached key as its initialization fence: that key can -be deleted by later trimming, so an excluded initializer could otherwise insert -it again. A later initializer can replace an older owner's initial reservation; -its new revision excludes the delayed publication without a mutable Stream head. - -A revision conflict invalidates the captured proof. Re-read the tail and any -Candidate identity before retrying. For an unknown outcome, compare the complete -durable tail with the proposed transition; an exact match is committed. A -superseding writer/session rejects stale publication. Pin and Candidate changes -also fence through the tail revision, preventing stale retention observations -from racing publication or reclamation. - -## 4. Append Protocol - -Admission reserves bounded request slots and queued bytes. Oversized appends are -rejected before enqueue. An idle worker dispatches the first request immediately -and drains only requests already queued. It stops at the request limit, byte -target, or current chunk capacity, without adding a batching delay. One large -request may exceed the batch byte target while remaining within append bounds. - -The normal path for an already attached chunk is: - -1. Select non-overlapping logical ranges and encode one complete frame group - per append. Keep one location per payload rather than one extent per frame. -2. Submit mirror writes concurrently, requesting synchronization in the write - requests. The chunk layer owns strip ordering, repair images, replica - replacement, and durable updates of its own chunk information. -3. Wait for successful synchronized completion from every required replica in - the configured mirror set. Three-copy configuration waits for three replicas; - no fixed replica count is assumed by the Stream API. A smaller set of readable - replicas does not lower the write-completion requirement. The chunk layer - restores required protection after replacement before returning success. -4. Publish the completed in-memory cursor, logical tail, and extent snapshot, - then complete each request with its exact logical range. - -There is no per-batch `advance_chunk_write`, group-0 update, Stream chunk record -write, or metadata CAS. Encoding, buffer ownership, and staging remain bounded; -frame splitting does not justify repeated payload coalescing or per-frame KV -writes. Periodic lease maintenance is separate from the append data path. - -An ambiguous mirror response is not evidence of absence. Resolve it through the -chunk write/recovery contract; do not retry at a guessed offset or use an old -ChunkDB cursor to declare new bytes absent. A failure on one request cannot let -later requests skip a logical hole. Watchdogs observe the original future and -do not cancel, resubmit, or complete it. An interrupted append contributes no -partial logical range. Recovery may retain a complete append whose response was -lost, but never its proper prefix; consumer retry semantics handle that uncertain -completion. Differing valid replica tails follow section 5.2. - -## 5. Rollover and Recovery - -### 5.1 Lifecycle ordering - -One append never straddles chunks. A chunk contains at most the configured -physical framed-byte limit, 256 MiB by default. Chunk IO may allocate physical -strips incrementally; this remains internal to the chunk layer. - -Capacity preparation runs ahead of consumption. Measured write bandwidth -and preparation duration size the initialized spare capacity with a two-interval -margin, bounded between one frame and one physical strip. The physical allocator -rounds this target to strip boundaries without adding another growth multiplier. -As consumption approaches the hard chunk boundary, one initialized Candidate is allocated -and registered while the Active chunk accepts appends. Its initial capacity -follows observed batch consumption, not the configured maximum batch size. -Small cold streams retain their first strip without eagerly initializing spare -chunks. Normal writes use ready capacity; exceptional -allocation/initialization delays apply backpressure instead of exposing unsafe -capacity. - -When an append cannot fit, drain preparation and writes for the current chunk, -physically seal its exact final length, take the prepared Candidate, and commit -the atomic record transition. The successor accepts data only after attachment. -Switching does not allocate or rewrite historical metadata. Seal and attachment -are distinct durability boundaries: a crash after seal but before publication -leaves the one last referenced chunk to reconcile. An unattached Candidate has -no acknowledged user data and can be reclaimed after owner exclusion. - -Different segment IO streams can drain concurrently, with at most eight drains -in flight across the complete layout, including during tail recovery. Strip -boundaries do not serialize independent drain requests or consume the prefix -scan's remaining owner deadline. Every submitted drain must finish, including -after a peer fails; physical sealing requires all drains to succeed and current -writer authority. Recovery retains its original captured deadline and requests -durable mirror repair only after all submitted drains have completed. - -### 5.2 Tail reconstruction - -The latest physical cursor is the end of the contiguous prefix of complete -append groups in the last referenced chunk. Recovery starts -at that chunk's beginning or a validated synchronized checkpoint and parses -forward within current authoritative capacity. It checks frame kind, lengths, -CRC, chunk identity, and address arithmetic. It tracks a tentative frame cursor -inside a group and advances the recovered physical cursor and logical tail only -after validating the entire group. It then adds one compact location. - -Before publishing a fresh attachment or admitting writes into newly allocated -capacity, zero and synchronize each new physical strip on every required mirror. -Initialization uses one submission per replica, without automatic retries. An -initialization pass admits at most two strips at once; their required mirrors -still synchronize together. This bounds zeroing bursts while ordinary writes -use the initialized prefix. Every submitted initialization finishes before the -pass returns, and capacity is exposed only after every strip succeeds. An -uncertain result leaves that capacity unavailable to the writer; the owner cannot -repeat initialization after later data writes. A replacement strip receives its -complete retained prefix and a zero suffix with the same single-attempt contract -before its metadata publication. - -Ordinary append writes exactly its complete group range. It never writes a marker -past that end: a delayed duplicate must not overlap the next acknowledged group. -Unwritten headers are supplied by the initialized suffix. Capacity preparation -reserves a complete 14-byte header below the hard chunk limit, growing before -writing when necessary. This keeps the previous complete prefix bounded inside -already initialized capacity if growth stops before its new strips are ready. -An exact hard-capacity end requires no extra header. Allocation, initialization -and repair traffic are accounted separately from ordinary append bytes. - -The bounded repair shadow retains only the unfinished strip prefix. Validation -failures before the first data IO do not consume that owner's fresh-write -admission. Block alignment must preserve all physical bytes outside the requested -range even when the partial-block cache is absent. - -An unwritten suffix, incomplete final frame, or incomplete group bounds the -candidate prefix at the previous complete append. No later group can skip that -hole. Individually valid frames in an incomplete group do not advance the tail. -Corruption in required retained data is not silently treated as EOF: use mirror -validation/repair or fail recovery. Bytes with a wrong chunk identity cannot -extend the tail. Syntactically valid bytes beyond the last checkpoint remain -candidates; a stale persisted cursor is not an authority to discard them. - -The shared parser keeps one at-most-64-KiB frame and constant group/hash state. -Input windows are at most 1 MiB; output batches contain at most 1,024 complete -locations. Observed bytes and parsed frames are distinct from committed group -progress. An exact-capacity end needs no extra header read. Required grouped -checkpoints are exact append boundaries. A SHA-256 digest covers only the encoded -complete prefix; incomplete groups do not enter it. A failed observation cannot -subsequently be finalized as a healthy shorter prefix. - -After owner exclusion, observe the current available mirrors concurrently -within a bounded IO budget. When retained DiskIO topology covers every segment, -order each strip's replicas by rack and node for consistent observation lanes -across strips. Allocation order alone can alternate racks, making every lane -depend on a slow cross-rack link. These route observations only order existing -segments: incomplete hints preserve original ordering, no replica is added or -omitted, and the original authoritative layout remains unchanged. Continue -checking every completed lane's checksum and complete-prefix agreement under -the same deadline. Use the shortest complete-append prefix among healthy -mirrors that finish observation. A mirror that fails or times out during the -read is unavailable, not a zero-length or partially observed prefix. Corrupt -replicas are not valid shorter candidates; use healthy copies or report an -error. All candidates must agree on retained overlapping contents. Conflicting -valid contents fail explicitly rather than waiting for an external arbiter. -No surviving readable candidate means recovery is unavailable. Repair restores -configured protection before the next successful append. Replica comparison and -checkpoints use complete-group boundaries, never an interior frame boundary. - -Only the last referenced chunk may require content parsing. Earlier chunks -have finalized bounds and chunk record mappings and are not scanned. A last -chunk whose seal and final mappings are already published needs no tail scan. -Recovery after an interrupted seal may still reconstruct that one last chunk. -The ChunkDB finalizer may reclaim and tombstone a physically empty tail before -the Stream publishes its successor. Recovery accepts that exact tombstone only -with its seal timestamp, valid purpose and nonzero writer epoch, zero capacity, -zero sealed/acknowledged/checkpoint boundaries, and no remaining strips or cleanup -intents. It retains the logical position, leaves the tombstone unchanged, and -attaches a fresh chunk under current writer authority. A missing chunk or a -nonempty or unproven tombstone never becomes an empty prefix. -Bound reads, buffered bytes, parsed frames, and total scan bytes by the active -chunk geometry, independently of retained Stream length. - -### 5.3 Writer ownership - -Writer open validates purpose and current ownership before exposing a writable -handle. A lower epoch is rejected. A higher epoch adopts retained history -without copying historical chunk records, resolves the old tail, seals/finalizes it, and -attaches a fresh chunk before new appends. Writers never resume appending to a -previous process's chunk. - -Group-0 owner leases remain authoritative between metadata checkpoints. The -writer uses a conservative local monotonic deadline derived from its grant and -checks authority before accepting work and before acknowledging it. Background -renewal is independent of append batches. Expiry or explicit fencing stops both -new admission and successful acknowledgment, including after a process pause. -Metadata CAS still fences lifecycle publications; DiskIO success alone does -not authorize an acknowledgment and DiskIO does not own Stream leases. - -Explicit transfer stops the old owner's admission, drains its in-flight work, -persists the final synchronized boundary, and records a fenced release proof -in Group 0. Once release and the new grant are authoritative, the new owner can -activate without waiting for the old lease's natural expiry. A verbal/local -promise or an unresolved release response is not the durable release proof. - -If the old owner cannot be contacted, activation waits for its lease's safe -expiry boundary, including the existing clock-skew allowance. Resolve in-flight -old writes using the bounded write-age/drain rules before fixing the old tail. -The new owner recovers that tail and publishes a newly allocated chunk; it -never resumes writing the old owner's chunk. Group-0 unavailability cannot -extend a local grant past its conservative deadline. - -### 5.4 Read-only capture - -Read-only open validates the requested epoch and captures the tail chunk record, -trim point, active identity, and one finite validated tail: - -- When the current owner is reachable, request an ordered flush/checkpoint - barrier. The owner captures its synchronized prefix, persists the ChunkDB - complete-group boundary, then publishes the corresponding chunk record checkpoint - before returning the matching epoch/chunk/physical cursor/logical end. A failed - record publication leaves the prior checkpoint authoritative even if ChunkDB - already holds a newer cursor. Later append does not widen that view. Already - synchronized appends do not require another data fsync to capture the prefix. -- When the owner is unreachable, use the captured persisted ChunkDB boundary - with matching Stream metadata. Reconstruct missing mappings only through that - boundary without writer adoption or publication. The result is an explicitly - older checkpoint view, possibly empty for the active chunk. - -Epoch, chunk identity, chunk record/checkpoint, and end must form one consistent view; -both physical and logical checkpoint bounds end complete groups. A raw newer -ChunkDB cursor is not combined with an unrelated chunk record or epoch. Do not combine -observations across ownership changes. Expose the captured bound -and whether it came from owner flush or checkpoint fallback. A requested exact -upper cutover beyond that bound is unavailable, not a successful shorter replay. -Checkpoint fallback is a read-visibility rule: writer failover still recovers -the content-derived tail and cannot discard acknowledged data beyond it. - -## 6. Reads - -Reads reject ranges below `trim_offset` or beyond their captured logical tail. -For persisted data the reader reverse-seeks the greatest attached key whose -logical start is at or below the requested offset, validates coverage, and -locates its complete append through the record's compressed mapping samples. -Active unpublished data uses append-only in-memory mappings. Expand only the -intersecting physical frames and return the requested payload after validation. -Reads cross append and chunk boundaries while preserving logical order. - -A sequential reader accepts a start and finite byte hint or `ToEnd`, captures -its end once, and returns EOF there. The default retained window is 8 MiB, each -physical request is at most 1 MiB, and at most eight requests run concurrently. -Independent readers may run concurrently. Observation watchdogs do not cancel -or retry underlying IO. - -The lock-free record cache has 32 slots, each admitting at most 1 MiB of -encoded mappings plus seek samples. Oversized records remain readable but are -not cached. A record lookup loads only its target chunk, independently of -historical Stream length. Snapshot publication uses immutable ArcSwap views; -finite readers retain their captured view and interval lease through trim. - -A provenance-aware reader yields physical chunk identity along with payload. -The journal owns business-record validation, sequence continuity, filtering, -and retry-result semantics. Physical Stream frame validation applies regardless -of consumer. Readers and overlays preserve their captured bounds through later -append, rollover, and ownership changes. - -## 7. Trim and Reclamation - -`trim_prefix(g)` rejects regression and `g > tail`. Persist the new trim -watermark on the tail before reclaiming data. Only complete Sealed chunks whose -logical end is at or below the watermark are eligible. Partial trim preserves -the chunk record's original bounds and mappings. The current tail is retained. - -A finite reader leases its logical interval and exact Active chunk identity. -The latter protects unpublished local mappings. Pin creation, renewal and -release conditionally fence the tail revision; collection captures that revision -before observing pins. A changed revision invalidates the entire proof. -Reader pins are limited to 512 identities of at most 1 KiB each. They renew -within a 60-second lifetime and enforce an earlier local monotonic deadline. -Expired pins protect IO for a further 31 seconds before bounded cleanup. -Expiry cannot be renewed into a new live view. WAL consumers publish their -safe replay/retry watermark before trim; split/transfer checkpoint suppression -and finite captures preserve their retained dependencies. - -Collection combines the trim watermark, caller-proven generation horizon, -whole-chunk state and all live interval protections. It scans a bounded prefix -with a logical-start continuation and processes at most one eligible physical -chunk per pass. Mark the record Deleting under the captured tail revision, -release the chunk using its exact derived complete physical length, and delete -the record only after release is confirmed. A failed release or delete leaves -a retryable Deleting record. Repeating trim resumes cleanup. At least one -eligible whole chunk can progress even if it exceeds the physical byte target. - -A generation alone never authorizes deleting retained data. There are no -Stream directory roots, page reachability walks or historical metadata copies. - -## 8. Bounds and Observation - -Defaults bound admission at 1,024 requests and 64 MiB, a batch at 64 requests -and a 1 MiB byte target, one append at 64 MiB, and one chunk at 256 MiB with one compressed record -per chunk. Sequential reading uses an 8 MiB window, -eight concurrent physical requests of at most 1 MiB each, and GC uses a 64 MiB -pass budget. Bounds are configurable and validated; record/decode/cache budgets, -and optional checkpoint budgets must also bound their own work. Replica count -is configured, not implied by a physical-byte metric. - -One encoded chunk record is limited to 24 MiB, and its mapping count is bounded -by the physical chunk geometry. Decode requires canonical 256-entry blocks, -except the final partial block. Collection enumerates at most 128 keys and loads -values individually under its byte target, allowing at most one final record -to cross that target; it never buffers every enumerated value at once. - -Lock-free counters and phase histograms distinguish: - -- append admission, queue, capacity preparation, batching, framing, synchronized - mirror RPC completion, in-memory publication, and end-to-end completion; -- chunk-internal layout updates from Stream lifecycle publications, seal, - ownership maintenance, and optional checkpoints; -- total chunk sealing latency and its segment-drain portion, including failures; -- chunk record build, get, scan and conditional publication latency, mapping - counts/encoded bytes, and cache hits/misses; -- recovery chunks/bytes/frames scanned, replica resolution/repair, cursor and - extent reconstruction, and read-only capture; -- logical/physical bytes, queue high-water marks, retained buffers/cache, read - amplification, reclaimed bytes, orphan backlog, and watchdog observations. - -Periodic perf output and cumulative benchmark output retain count, average, -p50, p99, and maximum. Phase scopes and whether counts describe requests, -batches, or replicas are explicit. Nested percentiles are not added. Ordinary -append samples must not conceal control-plane waits inside a mirror stage. -Publication counts and latency include initial attachment and writer adoption; -opening a read-only handle does not count as a publication. - -Performance acceptance uses release artifacts and a matched three-node native -fixture with `mem-block` KV/WAL and retained MemDisk payload bytes. Record the -replica count, buffer/queue settings, checkpoint policy, binary identities, and -fixture parameters. The fixture removes physical-media latency without skipping -framing, transport, synchronization responses, or read validation. It proves -process/reopen behavior while retained media exists, not power-loss durability; -durability fault tests use storage that survives the tested failure. - -The regression workloads and minimum gates are: - -- 4 KiB append, one producer: at least 250 ops/s and p99 at most 10 ms. -- 4 KiB append, 32 producers: at least 3,000 ops/s and p99 at most 20 ms. -- 1 MiB append, eight producers: at least 125 MiB/s and p99 at most 150 ms. -- 4 MiB rollover, eight producers: at least 200 MiB/s and p99 at most 500 ms. -- 4 KiB random reads, 32 producers: at least 30,000 ops/s and p99 at most 5 ms. -- 320 MiB replay, one producer: at least 300 MiB/s and p99 at most 2 s. -- 576 MiB prefix trim: at least 1,000 MiB/s and p99 at most 100 ms. - -These are regression floors, not a claim that multi-millisecond memory-backed -append latency is the performance objective. Matched before/after distributions -must show the effect of removed KV waits and the combined write/sync RPC. -Separately report standalone Stream append, actual Chunk-KV WAL, and client -mutation latency. - -Sentinels require zero errors/watchdogs, bounded admission, less than 256 MiB -peak workload RSS growth, no per-append ChunkDB cursor or chunk record writes, -history-independent normal append work, and at most one tail chunk scanned on -recovery. With optional checkpoints disabled and close outside measurement, -chunk record publications do not exceed initial attachment plus rollover count. Cache -and lookup accounting follow accessed records and configured budgets. GC must -release at least one complete eligible chunk. - -## 9. Errors and Invariants - -`StreamError` distinguishes invalid requests, backpressure, stale writers, -definitely absent operations, unresolved outcomes, stalled writes, unavailable -reads, corruption, and internal invariant failures. Absence cannot be inferred -solely from a stale checkpoint cursor. - -- **I1 — Durable completion.** A successful append has exact ordered logical - ranges and synchronized data on every required mirror replica. Per-append - ChunkDB cursor and chunk record updates are not completion prerequisites. -- **I2 — Bounded record work.** Historical chunk records retain their bytes; - publication touches only the tail and successor, and open/seek loads only - bounded target records. -- **I3 — Publication and ownership.** Data is not admitted to an unattached - chunk; attached records describe complete append groups; epochs never regress; - Group-0 lease expiry or release excludes stale acknowledgments/publications. -- **I4 — Bounded recovery and views.** Only the last referenced chunk can need - parsing. Recovery preserves acknowledged bytes beyond old checkpoints and - establishes a validated contiguous prefix. Read-only ends are finite; an - owner-unavailable checkpoint fallback explicitly permits an older view and - cannot silently satisfy a newer requested cutover. -- **I5 — Retention.** Logical trim precedes deletion and reclamation respects - whole-chunk liveness, logical intervals and every retained consumer pin. -- **I6 — Consumer separation.** Wal and business streams share the same storage - contract. The consumer owns business-record and retry-result semantics. -- **I7 — Logical addressing.** One append stays in one chunk, framing bytes never - enter logical offsets, compatible multi-frame payloads use compact locations, - recovery accepts the entire append or none of it, and rollover/trim never - renumber retained bytes. -- **I8 — Chunk isolation.** Internal chunk layout and `chunkinfo` changes do not - mutate Stream chunk record generations or copy Stream metadata. -- **I9 — Synchronized write RPC.** A synchronized write response follows both - write and flush; it does not require a separate client fsync round trip. -- **I10 — Resource and observation bounds.** Queues, batches, buffers, record - caches, recovery, and GC are bounded; the hot path adds no lock, and watchdogs - observe without taking over IO completion. - -## 10. Open Issues - -There are no unresolved decisions for the append/recovery contract. Verifiable -bounded frame groups, all-required-mirror completion, Group-0 owner leases, and -explicit live-view bounds define the contract. Byte packing, decoder fixtures, -and rollout are implementation work under those constraints. - -Metadata-group rebinding, per-stream sharding, and sealed-chunk EC conversion -remain separate scale-out topics. diff --git a/doc/design/chunkio/chunkio-small-io-flow-analysis.md b/doc/design/chunkio/chunkio-small-io-flow-analysis.md deleted file mode 100644 index a224e32e1..000000000 --- a/doc/design/chunkio/chunkio-small-io-flow-analysis.md +++ /dev/null @@ -1,372 +0,0 @@ - - - -# CROWDB - Design: Small IO Flow Analysis - -This document records the full-stack small-write and read benchmark flow, -baseline measurements, correctness limits, and ordered optimization work. The -data-path contracts are defined by -[`design-crowdb-chunkio.md`](design-crowdb-chunkio.md), -[`design-crowdb-chunkio-small-object-writer.md`](design-crowdb-chunkio-small-object-writer.md), -and [`design-crowdb-chunkio-reader.md`](design-crowdb-chunkio-reader.md). - -## Contents - -1. [Benchmark boundary](#1-benchmark-boundary) -2. [Small-write flow](#2-small-write-flow) -3. [Read flow](#3-read-flow) -4. [Baseline results](#4-baseline-results) -5. [First divergence](#5-first-divergence) -6. [Bottleneck analysis](#6-bottleneck-analysis) -7. [Ordered improvement plan](#7-ordered-improvement-plan) - -## 1. Benchmark Boundary - -The sentinels use a co-located three-node cluster with three instances each of -KV, DiskDB, ChunkDB, and DiskIO. KV data and WAL use `mem-block`; fsync is -disabled. DiskIO uses `NullDisk`, so requests traverse client routing, -crowdb-rpc, the DiskIO request handler, and io_uring without retaining payload -data. Each small-write matrix case starts a fresh combined deployment. Process -exit releases the in-memory KV and WAL contents before the next case, bounding -RSS independently of allocator behavior and detached task completion. Read -preparation performs real writes before the measurement window and retains the -resulting locations. - -| Parameter | Value | -| -------------------- | ------------------------------------------ | -| Timed duration | 20 seconds per case | -| Client concurrency | 1, 4, 32; up to 128 and 256 for small IO | -| DiskIO connections | 8 in sentinels, 4 in CLI; configurable | -| DiskIO RPC workers | 1 client and 1 server worker by default | -| Small object sizes | 1 KiB and 8 KiB writes; 8 KiB reads | -| Large read object | 16 MiB | -| Large read EC | 8+4, 1 MiB shards | -| Read dataset | 16 objects per selected class | -| Mixed read | 50% small and 50% large by request count | -| Write pipeline limit | 32 | -| Write batch limit | 1 MiB or 1,024 objects | - -A result is valid only when every admitted operation is accounted for, -`errors=0`, `incomplete=0`, and the watchdog count is zero. Throughput after the -first error is diagnostic only because immediate failures can increase the -request counter without doing IO. - -## 2. Small-Write Flow - -```text -worker - -> prepare_small_write - -> enqueue whole object in shared writer pool - -> queue-byte/object threshold may add a pipeline - -> completed pipeline immediately drains the objects currently queued - -> drained objects form the next mirror strip without a batching delay - -> ChunkDB allocation and DiskDB/KV metadata commit - -> DiskIO mirror writes to NullDisk - -> completion returned to every object -``` - -Pipeline scale-out is based only on queued bytes or queued object count. A new -pipeline is added only when every active route is under queue pressure, so a -startup burst on pipeline zero cannot create 31 empty pipelines. Scale-in also -uses state rather than elapsed time: all routes must be empty and idle, and the -global reservation count must be zero. - -The benchmark exposes both logical batches and physical aggregation. For every -normal mirror write it counts the number of represented objects, submitted -buffers, logical bytes, and actual full-strip payload bytes. Repair and -mirror-to-EC traffic retain their separate counters. It also records the peak -pipeline count. Dedicated E2E coverage stops admission and observes scale-in -to the configured minimum from queue and busy state. Neither scale-out nor -scale-in uses elapsed idle time as a decision input. - -The pipeline follows the RPC send-queue scheduling model: completion of the -current batch immediately fetches all currently available work, bounded by the -object and byte caps. The 500 ms batch watchdog only reports a stuck durability -operation. It never delays a batch, triggers a flush, or cancels work. - -## 3. Read Flow - -```text -untimed preparation - -> real small or large write - -> ChunkDB/DiskDB/KV metadata commit - -> retain writer-produced Location - -timed request - -> query ChunkDB layout - -> read requested mirror or EC segment through routed DiskIO RPC - -> on segment error, try mirror fallback or partial EC recovery - -> persist observed failed segments in ChunkDB - -> assemble and return object -``` - -The failure-observation path distinguishes durable segment failures from -transient transport, overload, timeout, alignment, and short-read results. -Only durable failures are persisted in `unavailable_segments`; transient -failures participate in fallback or EC recovery for the current request only. - -## 4. Baseline Results - -Measurements were taken on 2026-09-09 on the Intel Core i9-7960X development -host. DRAM totals are passive host-wide PMU samples and include all co-located -services. - -### 4.1 Small Write - -| Size | Threads | TPS | MiB/s | p50 ms | p99 ms | Objects/batch | Peak pipes | Errors | Valid | -| ----- | ------: | --------: | ----: | -----: | -----: | ------------: | ---------: | -----: | :---: | -| 1 KiB | 1 | 424.42 | 0.4 | 2.309 | 3.532 | 1.00 | 1 | 0 | yes | -| 1 KiB | 4 | 856.48 | 0.8 | 4.627 | 6.095 | 2.00 | 1 | 0 | yes | -| 1 KiB | 32 | 6,275.96 | 6.1 | 5.032 | 9.448 | 8.26 | 2 | 0 | yes | -| 1 KiB | 128 | 19,508.32 | 19.1 | 6.166 | 13.627 | 15.36 | 5 | 0 | yes | -| 1 KiB | 256 | 31,313.08 | 30.6 | 7.549 | 19.388 | 21.21 | 8 | 0 | yes | -| 8 KiB | 1 | 390.94 | 3.1 | 2.492 | 3.909 | 1.00 | 1 | 0 | yes | -| 8 KiB | 4 | 675.29 | 5.3 | 5.323 | 10.882 | 2.03 | 1 | 0 | yes | -| 8 KiB | 32 | 4,464.84 | 34.9 | 6.701 | 18.595 | 8.65 | 2 | 0 | yes | -| 8 KiB | 128 | 13,461.05 | 105.2 | 8.791 | 23.066 | 11.25 | 7 | 0 | yes | -| 8 KiB | 256 | 23,446.56 | 183.2 | 9.811 | 27.503 | 18.97 | 9 | 0 | yes | - -The timer-free single-worker p50 is 2.31 ms for 1 KiB and 2.49 ms for 8 KiB; -the earlier roughly 56 ms rows were invalidated by the io_uring wake defect and -the batching deadline. The highest measured rates in this run are 31,313.08 TPS -for 1 KiB and 23,446.56 TPS for 8 KiB, both at 256 workers. These are observed -maxima, not demonstrated saturation peaks. Aggregation rises naturally with -queue occupancy even though there is no batching delay. All watchdog counters, -errors, and incomplete counts are zero. The retained matrix is -`bench-log/chunkio-small-completion-drain-20260909/results.tsv`. - -A dedicated 1 KiB, one-worker distribution rerun completed 8,489 requests at -424.42 TPS: p50 2.309 ms, p90 2.665 ms, p95 2.918 ms, p99 3.532 ms, and max -26.575 ms. Maximum observed queue delay was only 90 us. Server and client stage -windows place the steady-state cost primarily in two serial durability phases: -the three parallel NullDisk mirror writes average roughly 0.72-0.85 ms, followed -by the fenced `advance_chunk_write` metadata commit averaging roughly -1.1-1.4 ms. The retained distribution run is -`bench-log/chunkio-small-latency-distribution-20260909/results.tsv`. - -### 4.2 Read Before the Correctness Fix - -| Workload | Threads | Success | Errors | Success TPS | MiB/s | p50 ms | p99 ms | Valid | -| -------- | ------: | ------: | --------: | ----------: | ----: | -----: | -------: | :---: | -| Small | 1 | 387 | 0 | 19.35 | 0.2 | 51.643 | 52.110 | yes | -| Small | 4 | 31,988 | 0 | 1,595.21 | 12.5 | 2.479 | 3.242 | yes | -| Small | 8 | 24,599 | 220,333 | 1,229.86 | 9.6 | 2.250 | 3.299 | no | -| Small | 16 | 16,478 | 527,755 | 823.84 | 6.4 | 0.962 | 2.872 | no | -| Small | 32 | 7,479 | 1,397,831 | 373.94 | 2.9 | 1.062 | 1.841 | no | -| Small | 128 | 29,419 | 2,880,180 | 1,470.86 | 11.5 | 2.008 | 4.631 | no | -| Small | 256 | 11,676 | 2,985,510 | 583.74 | 4.6 | 4.633 | 58.562 | no | -| Large | 1 | 24 | 0 | 1.19 | 19.0 | 850.917 | 952.716 | yes | -| Large | 4 | 312 | 20 | 15.43 | 246.9 | 227.787 | 387.001 | no | -| Large | 32 | 629 | 445 | 30.26 | 484.2 | 260.863 | 818.468 | no | -| Mixed | 1 | 88 | 0 | 4.37 | 35.8 | 119.247 | 866.818 | yes | -| Mixed | 4 | 652 | 61 | 31.40 | 228.2 | 22.008 | 358.932 | no | -| Mixed | 32 | 551 | 1,134 | 26.36 | 253.5 | 248.930 | 1,009.263 | no | - -Only error-free rows are baselines. Failed rows do not establish a peak because -the failure cascade truncates real IO and then spins on metadata-marked -segments. - -### 4.3 Final Read Baseline - -NullDisk now completes its generated-content contract at the requested length, -transient read errors no longer update durable layout metadata, cluster reset -waits for fresh service registrations, and flushing keeps frozen memtables -visible until their entries are published in L1. All final 20-second cases -completed with zero errors and zero incomplete requests. - -| Workload | Threads | Success TPS | MiB/s | p50 ms | p99 ms | -| -------- | ------: | ----------: | ----: | -----: | -----: | -| Small | 1 | 783.11 | 6.1 | 1.268 | 2.270 | -| Small | 4 | 3,418.85 | 26.7 | 1.164 | 1.725 | -| Small | 8 | 8,691.62 | 67.9 | 0.874 | 1.661 | -| Small | 16 | 21,254.08 | 166.0 | 0.730 | 1.183 | -| Small | 32 | 36,815.96 | 287.6 | 0.843 | 1.558 | -| Small | 128 | 57,098.87 | 446.1 | 2.139 | 3.799 | -| Small | 256 | 56,879.50 | 444.4 | 4.359 | 6.947 | -| Large | 1 | 13.79 | 220.7 | 71.241 | 88.609 | -| Large | 4 | 70.20 | 1,123.2 | 55.410 | 86.171 | -| Large | 32 | 173.76 | 2,780.2 | 181.237 | 275.468 | -| 50/50 mix | 1 | 24.30 | 194.9 | 40.443 | 94.923 | -| 50/50 mix | 4 | 135.18 | 1,082.4 | 28.849 | 82.793 | -| 50/50 mix | 32 | 352.08 | 2,818.1 | 36.473 | 236.431 | - -Small read reaches its useful saturation region around 128 workers: increasing -to 256 does not improve TPS and roughly doubles p50. The main retained matrix -is `bench-log/chunkio-read-final-20260909/results.tsv`; the two preparation -cases rerun after the readiness fix are in -`bench-log/chunkio-read-readiness-fix-20260909/results.tsv`. - -### 4.4 Read After the io_uring Wake Fix - -The common DiskIO io_uring poller previously accepted an idle-ring submission, -woke from `epoll_wait`, and then waited up to 50 ms for a completion before it -published the new SQE. It also reused the external completion `eventfd` as its -private submission wakeup. A private wake `eventfd` now separates those roles, -and the poller publishes submissions immediately after wakeup. - -| Threads | Success | TPS | p50 ms | p99 ms | Errors | Valid | -| ------: | --------: | --------: | -----: | -----: | -----: | :---: | -| 1 | 15,735 | 786.71 | 1.269 | 2.011 | 0 | yes | -| 4 | 68,215 | 3,410.15 | 1.168 | 1.712 | 0 | yes | -| 32 | 744,514 | 37,198.96 | 0.841 | 1.462 | 0 | yes | -| 256 | 1,116,927 | 55,789.01 | 4.441 | 7.224 | 0 | yes | - -At one thread this changes p50 from 51.7 ms to 1.27 ms and throughput from -19.36 to 786.71 successful reads/s. Stage histograms show ChunkDB query at -about 0.8 ms and DiskIO read at about 0.4 ms for one thread. At 256 threads, -ChunkDB query remains about 0.4 ms while DiskIO read grows to about 4.7 ms and -dominates the request. The retained stage result is -`bench-log/chunkio-read-stage-final-20260909/results.tsv`. - -## 5. Resolved First Divergences - -The earliest confirmed read divergence is a short DiskIO read from NullDisk: - -```text -disk I/O error: PartialWrite -``` - -The return-code name is shared with writes, but the server emits it when a read -completion returns fewer bytes than requested. The eight- and sixteen-worker -small-read runs both eventually show this exact error. Earlier occurrences on -other mirror replicas had already been persisted as unavailable, so the next -short read exhausts all three replicas. - -The failure then amplifies: - -1. DiskIO returns a transient short-read or transport failure. -2. `StripReader` adds the segment to `failed_segments` without an error class. -3. `ChunkReader` persists every observed segment in `unavailable_segments`. -4. Mirror reads eventually have no eligible replica; EC recovery eventually - has more missing shards than parity can reconstruct. -5. Subsequent requests fail before payload IO and rapidly inflate the error - counter. - -The 32-worker client also records `rpc.send.queue.full.c=19`. This confirms RPC -backpressure is reached, but the observed short read is the earliest directly -reported trigger. Queue-full events and short reads are both transient and -must not be converted into durable media-failure metadata. - -The fix makes NullDisk synthesize the complete requested buffer after any -nonnegative backing completion. DiskIO RPC/short-read failures are typed as -transient, while confirmed missing disks, missing zones, and media IO errors -remain durable. Mirror and EC readers persist only the durable class. A real -ChunkDB E2E verifies that an injected transient mirror failure leaves -`unavailable_segments` empty and that the original client can still read the -object. - -The next divergence was the 50-ms low-concurrency latency in the common -io_uring poller described in section 4.4. A C++ regression submits consecutive -operations after the ring becomes idle and verifies that neither waits for the -poll timeout. - -At 256 workers, the next first divergence was a transient metadata `NotFound`. -Repeated probes showed the same key becoming visible tens of microseconds -later without a rewrite. `Crowdbtree::flush()` had removed the draining frozen -memtables before publishing their entries in L1, creating an L0/L1 visibility -gap for concurrent readers. The draining tables now remain in `frozen_` until -L1 publication completes. A stable sentinel in the concurrent freeze/drain -test proves that an immutable key never disappears during this transition. - -Cluster reset had a separate readiness race: DiskDB could start before DiskIO, -and stale service heartbeats could satisfy registration checks. Reset now -starts DiskIO first and requires post-restart heartbeats for DiskIO, DiskDB, -and ChunkDB. Combined deployment also waits until every ChunkDB `/ready` -refreshes and exposes the published range assignment. - -## 6. Bottleneck Analysis - -### 6.1 Small Writes - -A post-instrumentation 8-KiB/32-worker run completed 62,567 objects in 32,424 -batches. Three mirror copies produced 97,272 normal DiskIO writes representing -187,701 object buffers, or 1.93 buffers per physical request on average. The -requests carried 1.43 GiB of replicated new logical bytes but 94.99 GiB of -full-strip payload. Peak pipelines reached 32. This confirms that the next -small-write optimization should reduce full-strip rewrite amplification and -improve batch fill together, rather than optimizing object admission alone. -The retained result is -`bench-log/chunkio-small-write-20260909-095047/results.tsv`. - -Two queue-accounting problems explained the excessive scale-out. A carried -object that did not fit the current batch was subtracted from queue pressure -twice, underflowing the unsigned counter. Synchronized batch boundaries also -looked idle and caused scale-in/scale-out thrashing while admitted objects still -held reservations. The corrected policy has real E2E coverage for carried -objects, byte-pressure scale-out, object-pressure scale-out, and final -state-driven scale-in. - -With a 16-object pressure threshold, 8-KiB/256 reached 23,257.11 successful TPS, -used at most 9 pipelines, and performed 51,288 physical mirror RPCs for -1,397,154 represented objects: 27.24 objects per physical request. Compared -with the earlier 8-KiB/32 run, throughput is 7.46 times higher and aggregation -is 14.1 times higher while the peak pipeline count falls from 32 to 9. The -retained result is -`bench-log/chunkio-small-tune-balanced-q16-256-20260909/results.tsv`. - -### 6.2 Reads - -The added `chunkio.chunk.query.e2e` and `chunkio.diskio.read.e2e` histograms -separate metadata from payload IO. KV engine get is about 4-5 microseconds and -KV RPC get is about 36-82 microseconds, so KV get is not the observed limit. -At 256 threads the DiskIO stage, including RPC, queueing, io_uring, and response -delivery, accounts for nearly all of the 4.44-ms median. - -Connections and both RPC worker counts are now independently configurable: - -- benchmark client: `--diskio-connections` and `--diskio-rpc-workers`; -- DiskIO service: `--rpc-workers`; -- combined deployment: `--diskio-rpc-workers`; -- regression scripts: `CHUNKIO_*_DISKIO_CONNECTIONS`, - `CHUNKIO_*_DISKIO_RPC_WORKERS`, and `CHUNKIO_*_SERVER_RPC_WORKERS`. - -An A/B small-write run at 8 KiB and 256 threads measured 22,802.91 successful -TPS with 8 connections and 1+1 workers, versus 23,993.26 TPS with 16 -connections and 4+4 workers. The 5.2% single-run increase is modest and is not -yet sufficient to change the defaults. The valid results are retained in -`bench-log/chunkio-small-diskio-default-ab-retry-20260909/results.tsv` and -`bench-log/chunkio-small-diskio-c16-cw4-sw4-final-20260909/results.tsv`. - -The initial read A/B attempts exposed the ownership-readiness and memtable -visibility bugs above, so they are not retained as performance results. The -final default-configuration matrix is fully valid; a repeated worker-count A/B -remains future measurement work. - -### 6.3 Foreground EC Sentinel - -The steady-state sentinel runs the foreground-EC path for 20 seconds per case, -reports reservation startup latency and parity bytes, and requires nonzero -foreground parity bytes with zero errors, incomplete objects, and watchdogs. - -Latest 20-second EC results (errors/incomplete/watchdogs all zero): - -- 1 KiB, 32 threads: 47,104.93 objects/s, p50 493 µs, p99 7,236 µs, - parity 234,881,024 bytes -- 1 KiB, 128 threads: 128,382.64 objects/s, p50 598 µs, p99 11,172 µs, - parity 184,549,376 bytes -- 1 KiB, 256 threads: 207,246.10 objects/s, p50 686 µs, p99 15,063 µs, - parity 209,715,200 bytes -- 8 KiB, 128 threads: 32,430.72 objects/s, p50 1,141 µs, p99 29,215 µs, - parity 306,184,192 bytes - -Source: `bench-log/chunkio-small-write-20260910-103318`. The 8 KiB 256-thread -case is excluded (7 batch watchdog expirations at extreme concurrency). - -Historical A/B baseline (60s, mirror vs EC): EC/mirror ratios were 100.23% -(32t) and 92.01% (128t). The mirror-only mode was removed from the sentinel -after the small-write path stabilized; EC is now the default and only mode. - -## 7. Ordered Improvement Plan - -1. Make pipeline retirement independent of a slow replacement-allocation RPC. - A sustained 32-pipeline run can scale in partway and then block the manager - while one retiring pipeline waits on DiskDB; pending conversion groups must - remain recoverable by ChunkDB tasks. -2. Repeat the 20-second DiskIO connection and worker matrix with at least three - samples per cell now that the metadata visibility and readiness bugs are - fixed. -3. Add DiskIO server queue depth, per-connection in-flight, and response-queue - latency metrics. The current aggregate DiskIO stage cannot distinguish the - single io_uring pipeline from RPC response delivery at 256 threads. -4. Keep the default at 8 connections and 1+1 workers until the valid repeated - matrix demonstrates a stable latency or throughput improvement. diff --git a/doc/design/chunkio/chunkio-write-flow-analysis.md b/doc/design/chunkio/chunkio-write-flow-analysis.md deleted file mode 100644 index a5c636767..000000000 --- a/doc/design/chunkio/chunkio-write-flow-analysis.md +++ /dev/null @@ -1,204 +0,0 @@ - - - -# Chunk IO Large-Write Flow Analysis - -Large-object write flow from the benchmark workload through chunk -preparation, fetch, EC encode, DiskIO RPC, and chunk seal. The benchmark -sentinel is `tools/benchmark/bench-chunkio-write-regression.sh`. The write pipeline -architecture is in -[`design-crowdb-chunkio.md`](design-crowdb-chunkio.md); this doc traces the -measured hot path and records benchmark results. - -## 1. Flow - -```text -prepare_large_writes (before timer) - -> ChunkIoClient::prepare_large_writes - N sessions, each allocates first chunk (2 strips) - distributed round-robin across workers - -> timer starts - -per object: - -> pop PreparedLargeWrite from queue - -> push replacement session to back of queue - (concurrent chunk allocation, off the data path) - -> write_stream / write_buffers - -> [Fetch] AsyncRead → 1 MiB Bytes blocks - stream: socket read into owned BytesMut, freeze, send - direct: caller-owned Bytes slices, no copy - -> [ChunkWriter::push] - block-granularity drive loop: - push data block → DiskWriter::write (1 RPC per block) - strip full → spawn parity task, auto-rotate to next strip - chunk full → seal, pull next Chunk from prefetch - -> [Parity Task] EC encode → parity blocks → DiskWriter::write - encode_parity_from_shards (isa-l, pre-split shards) - data + parity handles grouped per strip - -> [Completion] join in-flight parity handles - seal_chunk RPC - return ProtoLocation - -> record latency + per-stage timings - -> next object until deadline or object_count - -timer stops after all workers drain -DRAM BW measured from after prefetch to all writes complete -``` - -## 2. Measured Stages - -The benchmark result (`LargeWriteBenchmarkResult`) aggregates per-object -timings across all concurrent workers. Each stage time is total microseconds -divided by object count — milliseconds per object, averaged across writers. -Stages overlap and must not be summed into latency. - -| Stage | What it measures | -| -------------------- | ----------------------------------------------------------- | -| Source/read copy | Time in the fetch stage reading from `AsyncRead` into blocks. Zero in direct-buffer mode. | -| Fetch assembly copy | Buffer assembly copies inside the writer. Zero in both modes when the pipeline holds pre-split shards. | -| EC encode | `encode_parity_from_shards` — isa-l parity computation per strip. | -| Write-completion wait| Time joining in-flight DiskIO write handles + `seal_chunk` RPC. Dominated by loopback RPC round-trips. | - -Additional result fields: - -- **logical_mib_per_sec** — application bytes written (object size × objects) / - elapsed. The user-visible throughput. -- **physical_mib_per_sec** — EC-expanded bytes (logical × 1.5 for 8+4) / - elapsed. What the DiskIO layer actually transports. -- **objects_per_sec** — objects completed / elapsed. -- **latency_p50_us / p99_us** — per-object wall-clock latency from admission - to seal completion. -- **dram_read/write/total_mib_s** — host uncore IMC counter bandwidth over - the workload window (after prefetch). Multiplex-corrected. `None` when the - PMU is unavailable. -- **preparation_stalls** — count and microseconds where a worker's - replacement session fell behind and the writer waited for chunk allocation. - -## 3. Workload Configuration - -The regression sentinel uses fixed parameters across all cases: - -| Parameter | Value | Notes | -| ---------------------- | -------- | -------------------------------------------------- | -| Object size | 16 MiB | Two 8-MiB data strips per object. | -| EC scheme | 8+4 | 8 data + 4 parity = 12 blocks per strip, 1 MiB each. | -| Block size | 1 MiB | One DiskIO RPC per block. | -| Chunk size | 1 GiB | Multiple objects per chunk; rotation is off the measured path at 16 MiB objects. | -| Prefetch chunks | 10 | Write sessions prepared before the timer starts. | -| Prefetch strips/chunk | 2 | Strips pre-appended ahead of the write cursor. | -| Duration | 20 s | Admission deadline; in-flight objects drain after. | -| Storage | NullDisk | Real io_uring submission, no stable-media durability. | -| Deployment | 3-node | Co-located KV + DiskDB + ChunkDB + DiskIO, loopback. | - -Cases vary only in concurrency (1, 4, 32) and input mode (stream vs direct -buffers). Stream mode traverses the production `AsyncRead` fetch path; direct -mode passes caller-owned `Bytes` blocks with zero fetch copies. - -## 4. Benchmark Results - -### 4.1 Intel Core i9-7960X — 2026-09-08 - -Intel Core i9-7960X (16c/32t, Skylake-X), 4 DDR4 channels at 2667 MT/s -(~85 GB/s peak), Linux 6.11, `perf_event_paranoid=-1`. - -| Mode | Writers | Logical MiB/s | Physical MiB/s | p50 / p99 ms | DRAM read avg | DRAM write avg | DRAM total avg | -| -------------- | ------: | ------------: | -------------: | ----------------: | ------------: | -------------: | -------------: | -| Stream | 1 | 162.5 | 243.8 | 99.191 / 111.225 | 2,496.6 | 1,393.0 | 3,889.6 | -| Direct buffers | 1 | 229.1 | 343.7 | 69.191 / 85.641 | 1,996.1 | 1,228.7 | 3,224.8 | -| Stream | 4 | 1,965.6 | 2,948.5 | 31.917 / 51.806 | 12,699.7 | 10,943.0 | 23,642.7 | -| Direct buffers | 4 | 2,672.1 | 4,008.1 | 22.811 / 40.000 | 11,799.6 | 12,051.5 | 23,851.1 | -| Stream | 32 | 3,249.3 | 4,873.9 | 151.716 / 267.992 | 19,562.5 | 15,603.7 | 35,166.2 | -| Direct buffers | 32 | 3,547.9 | 5,321.9 | 138.654 / 261.388 | 14,816.5 | 12,766.6 | 27,583.1 | - -Per-object client-stage time (milliseconds per object, aggregated across -concurrent writers; stages overlap, do not sum): - -| Flow step | Stream 1 | Direct 1 | Stream 4 | Direct 4 | Stream 32 | Direct 32 | -| --------------------- | -------: | -------: | -------: | -------: | --------: | --------: | -| Source/read copy | 20.247 | 0 | 3.737 | 0 | 8.704 | 0 | -| Fetch assembly copy | 0 | 0 | 0 | 0 | 0 | 0 | -| EC encode | 20.175 | 13.164 | 15.213 | 12.357 | 22.045 | 20.222 | -| Write-completion wait | 53.990 | 54.238 | 11.741 | 10.118 | 122.786 | 120.972 | - -### 4.2 AMD Ryzen 9 5950X — _pending_ - -| Mode | Writers | Logical MiB/s | Physical MiB/s | p50 / p99 ms | DRAM read avg | DRAM write avg | DRAM total avg | -| -------------- | ------: | ------------: | -------------: | -----------: | ------------: | -------------: | -------------: | -| Stream | 1 | | | | | | | -| Direct buffers | 1 | | | | | | | -| Stream | 4 | | | | | | | -| Direct buffers | 4 | | | | | | | -| Stream | 32 | | | | | | | -| Direct buffers | 32 | | | | | | | - -| Flow step | Stream 1 | Direct 1 | Stream 4 | Direct 4 | Stream 32 | Direct 32 | -| --------------------- | -------: | -------: | -------: | -------: | --------: | --------: | -| Source/read copy | | | | | | | -| Fetch assembly copy | | | | | | | -| EC encode | | | | | | | -| Write-completion wait | | | | | | | - -## 5. Analysis - -### 5.1 Throughput Scaling - -Single-writer throughput is low (162–229 MiB/s logical) because each 16 MiB -object requires 24 sequential-ish DiskIO RPCs (8 data + 4 parity per strip × -2 strips) over loopback. The completion-wait stage dominates at 54 ms/object -— this is the aggregate time joining 24 RPC handles plus the `seal_chunk` -RPC, all serialized through one tokio worker. - -At 4 writers, throughput jumps 12× (1,966–2,672 MiB/s) because the 24 RPCs -per object now overlap across objects. The loopback transport and DiskIO -io_uring pipelines parallelize naturally. Completion wait drops to 10–12 -ms/object as the per-object join overlaps with other objects' writes. - -At 32 writers, throughput rises only 1.6× over 4 writers (3,249–3,548 MiB/s) -while p50 latency balloons to 138–152 ms. The loopback RPC/DiskIO queue is -saturated: more concurrency increases queueing delay rather than bandwidth. -Completion wait rises to 121–123 ms/object. - -### 5.2 Stream vs Direct - -Direct-buffer mode is 1.41× faster than stream at 1 writer (229 vs 163 MiB/s) -because it eliminates the source-read copy (20.2 ms/object). At 4+ writers -the gap narrows to 1.36× (2,672 vs 1,966) as the RPC bottleneck dominates -over the fetch copy. At 32 writers the gap is 1.09× — the pipeline is -RPC-bound, not fetch-bound. - -The fetch assembly copy is zero in both modes, confirming the shard-based EC -design: the pipeline holds data as pre-split 1 MiB `Bytes` blocks and -`encode_parity_from_shards` consumes them directly without re-splitting. - -### 5.3 EC Encode - -EC encode time is stable at 12–22 ms/object across all concurrency levels. -It does not scale with concurrency because isa-l parity computation is -CPU-bound and each parity task runs on one tokio worker. At 32 writers, -EC encode (20–22 ms) is no longer the bottleneck — completion wait (121–123 -ms) dominates by 6×. - -### 5.4 DRAM Bandwidth - -Host DRAM bandwidth scales with concurrency: 3.2–3.9 GB/s at 1 writer, -23.6–23.9 GB/s at 4, and 27.6–35.2 GB/s at 32. The read/write split is -roughly balanced at 4+ writers (EC reads source data, writes parity + -RPC transport copies), which matches the expected traffic pattern. - -At 32 writers, DRAM total (28–35 GB/s) is well below the 85 GB/s peak, -confirming the bottleneck is RPC queueing, not memory bandwidth. The -stream_32t case shows higher DRAM usage (35.2 vs 27.6 GB/s) because the -source-read copy adds memory traffic without adding useful throughput. - -### 5.5 Bottleneck - -The bottleneck is the loopback RPC/DiskIO transport queue. At 32 writers, -completion wait (121–123 ms/object) is 6× the EC encode time and 14× the -source-read time. The RPC path transports 24 MiB per object through -loopback sockets with kernel copies on both sides — this is the dominant -cost, not EC, not memory, not chunk allocation. - -Further work should profile and reduce RPC/kernel transport cost (zero-copy -send, larger batch sizes, connection pooling) and EC/cache contention. -Increasing prefetch or adding wrapper layers will not address the measured -bottleneck. diff --git a/doc/design/chunkio/design-crowdb-chunkio-reader.md b/doc/design/chunkio/design-crowdb-chunkio-reader.md deleted file mode 100644 index ea09f3806..000000000 --- a/doc/design/chunkio/design-crowdb-chunkio-reader.md +++ /dev/null @@ -1,164 +0,0 @@ - - - -# CROWDB - Design: Chunk Object Reader - -The chunk object reader reconstructs full, ranged, and streamed objects from -writer-produced locations across current mirror and erasure-coded layouts. - -Depends on: [chunk IO](design-crowdb-chunkio.md), -[chunkdb](../chunkdb/design-crowdb-chunkdb.md), and -[mirror-to-EC conversion](../chunkdb/design-crowdb-chunkdb-mirror-to-ec.md). - -## Contents - -1. [API](#1-api) -2. [Location mapping](#2-location-mapping) -3. [Strip reads](#3-strip-reads) -4. [Read consistency](#4-read-consistency) -5. [Memory bounds](#5-memory-bounds) -6. [Errors](#6-errors) -7. [Durable repair handoff](#7-durable-repair-handoff) -8. [Invariants](#8-invariants) - -## 1. API - -`ChunkIoClient` owns one `ChunkReader` sharing its metadata client and -lock-free DiskIO route snapshot. It exposes: - -- `read_object`: return the complete logical object as `Bytes`. -- `read_range`: return one half-open logical interval. -- `read_range_partial`: return ordered successful intervals and exact failed - intervals without substituting bytes. -- `read_stream`: return a pull-based `ChunkReadStream` whose item size is - bounded by policy. - -`ChunkReadPolicy` configures stream-window bytes, a 256-MiB process-wide -EC-recovery budget, layout safety margin, bounded layout retries, and the -ad-hoc full-fragment threshold. An optional preferred node supplies a mirror -read locality hint; ChunkKV uses its configured node identity for WAL replay. - -## 2. Location mapping - -Locations are sorted by `logical_offset` and must cover one gap-free logical -interval. Zero-length entries are ignored. Physical length must cover logical -length, and all additions use checked arithmetic. - -For each intersecting location, the reader queries current chunk metadata. It -validates that stored strip intervals are ordered and non-overlapping, binary -searches the first overlap, and walks explicit -`[chunk_offset, chunk_offset + capacity)` intervals. It never derives geometry -from strip vector index or assumes uniform width. This keeps locations stable -when several mirror strips become one equal-capacity EC strip. - -Independent locations are fetched concurrently and assembled by logical -offset. The output length must equal the requested logical length. - -## 3. Strip reads - -Mirror reads try usable replicas in metadata order by default. With a preferred -node and complete retained route hints for a strip, they first try that node, -then other nodes in its rack, preserving metadata order within each tier. -Unknown preferred nodes or incomplete strip hints preserve metadata order. -This ordering changes only the read snapshot: it neither publishes placement -nor adds or removes replicas. Layout validity deadlines, exact segment -incarnations, checksum verification, and corruption reporting remain unchanged. -Each request uses the exact segment-relative offset and length; DiskIO supports -arbitrary byte ranges. - -EC reads derive the shard width from stored segment geometry. The fast path -reads only data-shard intersections needed by the request. If a required data -read fails, the reader reads the same byte interval from surviving data and -parity shards and invokes ISA-L decode. It starts only enough surviving reads -to obtain `data_num` shards and issues another read only after a candidate -fails. A 16-KiB request therefore reconstructs 16 KiB rather than rebuilding -the whole shard. Missing shards beyond `code_num` are unrecoverable. - -An EC strip with incomplete parity still permits direct reads from healthy -data shards. A failed data read in that state is unrecoverable because parity -that has not reached the durable `Parity` state is never a decode source. - -Zero-filled data shards beyond a partial strip's durable length participate in -decode without issuing DiskIO. Returned data is always clipped to exact -location length, so stored KiB rounding and parity padding never escape. - -## 4. Read consistency - -Each metadata query advertises a maximum layout validity duration. The reader -records the local query start and subtracts a safety margin. If all dependent -DiskIO does not finish before that deadline, it discards every byte from the -attempt, re-queries, and retries within policy. ChunkDB retains replaced -segments through the same window. - -The chunk's durable `acknowledged_cursor` provides the exact written byte -boundary for both sealed chunks and active shared mirror chunks. Derive each -strip's valid prefix from that cursor and its chunk offset; later bytes are -`NotYetAvailable`. Legacy chunks without a byte cursor retain their coarse -`sealed_length` contract. EC data shard `i` has -`min(shard_bytes, max(0, valid_strip_bytes - i × shard_bytes))` valid bytes; -every code shard has `min(shard_bytes, valid_strip_bytes)` valid bytes. Recovery -reads only those prefixes and treats absent tails as mathematical zeros, -independently of stale bytes on disk. Repair writes only the valid prefix. - -Only an unparseable returned frame or a write-frame checksum mismatch proves -physical corruption. The reader reports its exact serving segment to ChunkDB; -ChunkDB first fences the DiskDB BusyBlock to `Corrupt`, then adds the segment -to `unavailable_segments` and admits `RepairStrip`. Network, timeout, and -unknown I/O failures may be decoded around but never create these markers. -This metadata update can advance -the revision of an active shared chunk. Its owning writer refreshes the chunk, -verifies state and writer epoch, recognizes an ambiguously committed cursor, -and retries the cursor advance against the new revision. - -## 5. Memory bounds - -Partial EC recovery reserves decode shard buffers plus output from a -client-wide Tokio semaphore before issuing recovery I/O. A request larger than -one reservation is split into same-offset slices. The limit covers recovery -scratch, not the caller-owned result or ordinary direct-read buffers. No lock -is introduced. - -`read_object` necessarily owns the complete returned object. Callers use -`ChunkReadStream` for large objects; each pull reads at most -`stream_window_bytes` and retains no prefetched second window. - -## 6. Errors - -`ReadError` separates malformed locations/ranges, deleted chunks, -not-yet-durable data, expired layouts, metadata failures, DiskIO failures, EC -decode failures, data loss, and exact failed logical ranges. Strict reads fail -at the first missing range. Partial reads preserve healthy ranges before and -after failures. A stream emits its contiguous successful prefix, one ranged -error, and then terminates; it never silently truncates or zero-fills. Empty -objects and empty ranges perform no RPC or DiskIO. - -## 7. Durable repair handoff - -Verified corruption is reported through a versioned chunk-routed RPC carrying -the expected revision, strip sequence, exact segment incarnation, and -operation ID. A small read still decodes only its requested slice; repair -admission proceeds independently and may begin before that read returns. -Matching client failures accumulate for one second. At a complete fragment or -the lesser of 1 MiB and half a fragment, one bounded full-fragment request is -sent to ChunkDB and same-key readers share its result future. Saturation falls -back to slice recovery and the durable repair task. - -ChunkDB coalesces full-fragment requests across clients under 32 concurrent -jobs and a 512-MiB shared decode/result budget by default. The existing -`RepairStrip` task provides the only allocation and publication authority; -rebuilt bytes can reach waiters after target fsync while fenced publication -continues. Task key/value, scheduling, publication, and crash behavior are -defined by -[Mirror-to-EC Conversion and Chunk Tasks](../chunkdb/design-crowdb-chunkdb-mirror-to-ec.md). - -## 8. Invariants - -- I1: every returned byte is covered by a validated location and strip. -- I2: bytes from an expired layout are never returned. -- I3: active-strip reads never exceed the durable acknowledged cursor. -- I4: EC recovery never exceeds its shared scratch-memory budget. -- I5: stream items never exceed the configured window. -- I6: unrecoverable redundancy loss is explicit and never zero-filled. -- I7: a verified-corrupt segment is durably marked before its recovery is - returned; an ordinary I/O error is never persisted as corruption. -- I8: incomplete parity is never used to reconstruct a data shard. diff --git a/doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md b/doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md deleted file mode 100644 index c01fe2e96..000000000 --- a/doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md +++ /dev/null @@ -1,343 +0,0 @@ - - - -# CROWDB - Design: Small-Object Shared-Chunk Writer - -The small-object write path batches independent objects into shared mirrored -chunks while preserving per-object completion, bounded memory, and a durable -prefix after process loss. - -Depends on: [chunk IO data path](design-crowdb-chunkio.md), -[chunkdb lifecycle](../chunkdb/design-crowdb-chunkdb.md), and -[diskio](../diskio/design-crowdb-diskio.md). - -Satisfies: bounded small-object admission, lock-free routing, shared-chunk -batching, fenced durable cursors, in-place mirror repair, elastic pipelines, -and orphan sealing. - -## Table of Contents - -- [1. Scope and Concepts](#1-scope-and-concepts) -- [2. Admission and Object Handles](#2-admission-and-object-handles) -- [3. Routing and Pipeline Ownership](#3-routing-and-pipeline-ownership) -- [4. Batching and Physical Layout](#4-batching-and-physical-layout) -- [5. Durable Cursor and Completion](#5-durable-cursor-and-completion) -- [6. Chunk Lifecycle and Recovery](#6-chunk-lifecycle-and-recovery) -- [7. Elasticity, Failure, and Shutdown](#7-elasticity-failure-and-shutdown) -- [8. Policy and Metrics](#8-policy-and-metrics) -- [9. Correctness Invariants](#9-correctness-invariants) - -## 1. Scope and Concepts - -One client-owned pool multiplexes small objects across a bounded set of -pipelines. Each pipeline exclusively owns one active Repo chunk containing -mirror strips and may own one empty prepared replacement. A pipeline batches -whole objects and writes the physical range to every mirror. Ordinary completion -returns independent locations while cursor progress runs asynchronously; callers -can opt into completion after the readable cursor is durably confirmed. - -The shared path can incrementally form 8+4 EC groups while writing mirrors. -It retains one open-strip image and four parity accumulators, but never retains -eight completed data images. Incomplete or failed groups remain mirrored and -can be completed by later background conversion. - -## 2. Admission and Object Handles - -`prepare_small_write_for_key(object_size, key)` validates the policy and uses -the complete tenant/bucket/object identity as a preferred pipeline hint. The -single-frame interface charges bytes as immutable input fragments arrive. -`prepare_shared_object_write_for_key` atomically reserves the complete declared -payload on an available pipeline before receiving it. It probes the other -routes when the preferred route cannot reserve enough capacity. Neither path -uses a global or per-tenant semaphore. - -The returned single-use writer retains caller-owned byte fragments. Successful -input is never partially accepted. Overflow or underflow is terminal, drops -retained fragments, releases the reservation, and returns an exact size error. -An empty object completes locally without starting the pool. Abort before -submission also completes locally and releases all resources. - -Finishing an exact non-empty object transfers its fragments, byte charge, and -completion channel to the pool. Cancellation after that transfer does not -cancel or reshape a physical batch: the worker completes the durable operation, -and an undeliverable result leaves a reclaimable range in the shared chunk. - -## 3. Routing and Pipeline Ownership - -Admission reads an immutable `ArcSwap` route snapshot. The hash selects the -first candidate; a full or closed route causes probing of other candidates -before waiting. Moving a retained byte charge reserves capacity on the target -atomically before releasing the previous route's charge. Failed transfers keep -the object intact. The submission path adds no mutex or read-write lock. - -When all routes are unavailable, the caller waits for a capacity notification -with a control-interval fallback that also observes newly published routes. -Socket admission leaves the body unread until retained-byte capacity is -reserved, so TCP applies backpressure. Queue capacity and byte reservations -bound pending work independently. - -For retirement, the manager publishes a snapshot without the route before it -signals the worker. The worker closes its receiver, establishing the acceptance -boundary, and drains every accepted object. A sender racing an old snapshot is -therefore either accepted and drained or rejected intact for rerouting. - -## 4. Batching and Physical Layout - -A pipeline waits for the first object only while it has no work. After the -previous batch completes, it immediately drains whole objects that are already -queued until reaching the byte limit, object-count limit, or remaining chunk space. It -never delays an admitted object to wait for a batching timer. Concurrent work -naturally accumulates behind the in-flight batch and is aggregated on the next -completion-driven drain. An object larger than the normal batch target is -written alone as long as it is within the object limit. - -Native small uploads allocate one exact-sized receive owner containing payload -and all frame regions. The receive task can fill it across multiple socket -reads, prepares frame checksums in place, and submits the complete owner once. -The worker sets each frame's Chunk ID after placement. Generic immutable input -fragments are framed with separate header/footer views without copying payload. - -Objects are packed consecutively and may span mirror strips within one chunk. -The worker slices shared `Bytes` views at strip boundaries and sends only each -new range to DiskIO through scatter/gather writes. It does not rotate merely -because a frame or object reaches the strip boundary. Every returned location -covers one object's consecutive physical frames. - -The current strip retains immutable prefix views for repair. Ordinary mirror -success creates no continuous strip shadow. Repair materializes the complete -prefix only after a failure; EC conversion materializes the closed strip image -for parity computation. Closing a strip releases its retained views. A view -crossing the boundary can retain its object's owner until the final open-strip -prefix is released, so memory headroom includes a maximum-sized owner per pipe. - -### Transport descriptor bound - -A strip aggregate may retain more views than one RPC frame permits. The DiskIO -semantic client validates the entire segment range, then submits consecutive -bounded scatter/gather frames at increasing offsets. Within one strip's disk -block, each frame must finish successfully before the next is submitted, even -when the boundary is aligned or the logical byte ranges are disjoint. Different -connections can reorder arrivals; a partial-block tail's zero padding can then -overwrite a later frame and corrupt stored chunk headers or payload. - -Independent strips' disk blocks may overlap. Different disks holding one strip's -EC shards or mirror replicas may also overlap; they do not share the padded -physical block. Callers preserve ordering between separate writes to the same -strip block. An error stops subsequent frame submissions before repair. Views -keep their original owners without coalescing payload. Requested fsync follows -the complete range once, and cursor publication follows successful completion -of all frames. A descriptor bound cannot be treated as a disk failure or trigger -repair. - - -A caller can attach a `SmallWriteIntent` to durable completion. Once the batch -has assigned exact locations, every attached callback completes before any of -the batch's DiskIO. Failure aborts the batch and its pipeline; cancellation of -the caller does not detach ownership registration from the physical operation. -The callback has no default storage policy. Legacy tree FileIO uses it for a -catalog-sharded block ledger; native streamed files rely on chunk allocation -ownership and Chunk-KV WAL. Range reclamation defers active shared-chunk ranges -that extend beyond the acknowledged cursor, preserving uncertain writes. - -Chunk allocation reserves a bounded group of hidden strips. A reserved strip -becomes `Consumed` immediately before its first DiskIO and is confirmed into -the readable layout only after its mirror write succeeds. Confirmation runs on the ordered metadata chain. Allocation/refill has its -own background task and does not delay that chain; reserve failures fall back to a -bounded batch of already attached mirror strips. Seal cancels never-consumed -reservations and removes attached strips beyond the written length. Objects and batches may straddle strips, but never chunks. Mirror-only groups -contain 32 strips by default; at 16 remaining strips a new group is allocated -asynchronously after the full preceding reservation, including hidden strips. -The client verifies the returned append offset before accepting a refill; an -older backend that ignores this field fails safely rather than overlapping -hidden strips. Upgrade ChunkDB before enabling this client prefetch behavior. -Resource allocation releases the existing chunk lifecycle guard; publication -reacquires it and revalidates ownership and sequence. No extra persistent -high-water record is required. - -Lease renewal refreshes the authoritative readable layout while retaining the -writer's exact staged, unconsumed strip as a local write view. This also applies -to an empty prepared replacement. Renewal does not add that strip to published -capacity or advance the readable cursor. Consume still precedes the mirror -write, and Confirm still follows its durability. A changed chunk identity, writer epoch, or staged strip -fails the writer instead of selecting a later reservation and leaving a gap. - -When automatic conversion is enabled and at least eight strips remain, one -special reservation allocates eight mirror sets with the configured copy -count and four parity segments from a joint placement plan. Each completed -strip updates the four incremental parity accumulators and releases its input -image. After strip eight, -the client writes and fsyncs only the parity segments. ChunkDB then reselects -one healthy survivor per mirror set against current topology and atomically -publishes the 8+4 EC strip. If optimal publication is unavailable, mirrors stay -authoritative and the ordinary durable conversion task is admitted. - -The configured pool memory budget subtracts one maximum-sized retained owner (at least 1 MiB) per maximum -pipeline and one foreground conversion group, then divides the remaining -buffer capacity between possible routes. Conversion is a cold-path bounded -operation and may use its own permit; ordinary object admission never does. -Published EC capacity is divided by its data width when deriving the next -reservation, so consecutive groups keep the same per-shard geometry. - -## 5. Durable Cursor and Completion - -Every shared chunk carries a nonzero writer epoch, acknowledged physical byte -cursor, optional closed-strip sequence, writer lease deadline, and metadata -revision. Cursor advancement executes under chunkdb's existing per-chunk -lifecycle guard and requires: - -- an Active chunk and matching nonzero writer epoch; -- the caller's expected metadata revision; -- a strictly increasing cursor within chunk capacity; and -- a valid, nondecreasing closed-strip marker. - -The operation updates the cursor and marker, renews the lease from server time, -increments the revision, persists the record, and refreshes the cache. Stale -epochs or revisions conflict; backward or out-of-range cursors are invalid. - -The response barrier is: - -1. Frame the owner in place and write immutable slices concurrently to every - mirror, splitting views at strip boundaries when necessary. -2. Repair each failed replica from the retained prefix and fence the new segment into - chunk metadata or, while it remains hidden, into its durable reservation. -3. Publish all object-specific locations together. -4. Coalesce cursor progress in the background metadata chain. Strip close and - batched append execute there in revision order. - -The physical mirror-strip flow is shared with chunk streams: it writes mirrors -in parallel, excludes failed disks, writes a prefix-complete replacement image, -and publishes the fenced strip swap. Each single-owner caller retains its own -current-strip prefix and controls its publication barrier. Journal streams also -fsync the final mirror set before advancing their durable cursor and resolve an -uncertain replacement result against chunk metadata before retrying it. - -No location is visible before its complete physical range exists on every -configured mirror. Cursor persistence is an asynchronous availability and -orphan-recovery checkpoint; readers can transiently report `NotYetAvailable` -until it catches up. - -Callers publishing immediately readable immutable authorities use -`SharedObjectWriter::finish_durable`. A batch containing a durable-completion -request waits for the existing metadata chain, then confirms any remaining -cursor suffix before delivering locations. Metadata failure fails that batch -instead of exposing an unreadable reference. The pipeline does not start its -next batch before this barrier completes. Arrivals queue during the wait and -form the next batch; multiple pipelines provide concurrency. Shared-object -writers require this barrier by default. The single-frame `on_finish` retains its -asynchronous cursor behavior; no additional lock or reader-side retry is needed. - -## 6. Chunk Lifecycle and Recovery - -A pipeline allocates its first chunk and its initial strip batch before -publication. The worker owns the write cursor while one background metadata -chain owns revision-ordered cursor commits and batched strip appends. When -remaining chunk capacity falls below the larger of one prefetch group and the -object limit, it starts at most one asynchronous replacement allocation so ordinary rotation does not wait for allocation. - -Idle workers renew the current chunk and any ready replacement at half the -writer lease interval. A matching live chunk writer lease also protects hidden -reservation groups, avoiding a separate renewal write for every group. - -Closing a strip releases its prefix views immediately. Retirement seals a non-empty -current chunk at its acknowledged cursor and deletes an empty current or -replacement chunk. A write or metadata failure -fails the affected batch and every already accepted queued object, removes the -route, and retires its chunks. - -Chunkdb periodically scans Active shared chunks. When a persisted writer lease -has expired, it acquires the normal lifecycle guard, rechecks the record, seals -at the persisted cursor, cancels and frees never-consumed reservations, closes -only acknowledged strips, persists, and refreshes the cache. Recovery retains -consumed reservations until the writer lease deadline plus the configured -reuse grace has elapsed, so a timed-out write cannot reach a recycled extent. -Consumed reservations without planned cursors remain allocated fail-safe. The -old writer can no longer renew or advance the sealed chunk. DiskIO performs raw -I/O and does not validate allocation ownership. - -## 7. Elasticity, Failure, and Shutdown - -One manager owns pipeline membership. It publishes a scale-out candidate only -after the candidate's first chunk is ready; initialization failure leaves the -old snapshot intact. When all routes reach their queue or byte-capacity high-water marks, the -manager can add one pipeline, up to 32 by default. -Scale-out depends only on queued work, not worker utilization, request age, or -elapsed idle time. When the whole pool has no queued or active work, an extra -route can be unpublished and drained while preserving the configured minimum. - -Unexpected worker termination removes the failed route and creates replacements -until the minimum is restored. Explicit shutdown closes admission, unpublishes -all routes, drains accepted work, joins all workers, and seals or deletes their -owned chunks. Dropping the pool closes admission but cannot await cleanup. - -A mirror write failure records the submitted segment's disk in one client-wide, -TTL-based lock-free negative list. Repair allocation excludes all live failed -disks and the nodes holding surviving replicas. The worker writes the complete -shadow to a tentative replacement, then publishes a one-strip fenced metadata -swap. Allocation and replacement-write failures may select a new target within -the bounded attempt count. An ambiguous metadata result retries the identical -operation ID and replacement, so it cannot allocate or install a duplicate. -A definite conflict discards the tentative block when it is not referenced. -Exhaustion fails the batch once, records an unavailable replica for an already -acknowledged prefix, and retires the pipeline for background recovery. - -## 8. Policy and Metrics - -Defaults accept objects up to 8 MiB and target batches up to 1 MiB, budget -1.25 GiB pool-wide, use one to 32 pipelines, and allow 1,024 queued objects per -pipeline. Queue high-water marks are 4 MiB or 128 objects. Shared chunks have -a 256 MiB client-side capacity, prefetch 32 strips per ordinary group, and use -two mirrors. Foreground conversion uses the configured data-width group and -its joint parity placement; single-node deployments disable conversion and -select one mirror explicitly. The access service routes by the logical data -capacity of a strip multiplied by its threshold ratio, not socket read size. - -Configuration validates nonzero bounds, reachable queue high-water marks, -ordered pipeline limits, and a budget covering retained owners, one conversion -group, and request buffers. The retained `scale_in_delay` and `cooldown` fields -are configuration-compatible but do not participate in scale decisions. - -Foreground parity writes use a dedicated connection per DiskIO endpoint and -64 KiB byte-range requests, followed by a parity-only fsync barrier. Ordinary -mirror traffic retains its configured connection pool and request sizing. - -Atomic metrics cover submitted, completed, and failed objects; reserved bytes; -batch sizes and fill; queue delay; active and draining pipelines; scale changes; -strip-tail waste; and batch-watchdog expirations. The watchdog reports a batch -that remains in flight beyond its configured interval, then continues awaiting -the same durability future. It neither delays queue draining nor cancels an -operation whose physical or metadata outcome may already be committed. The -default 500 ms observation cadence matches the RPC pending-request reaper -scan interval; underlying RPC clients retain their own 5- or 10-second request -timeouts. Repair metrics expose attempts, repaired and exhausted -replicas, failed-disk exclusions, active pipelines, pipeline replacements, repairs -that avoided rotation, complete-shadow bytes, and repair latency totals and -maxima. Snapshots compute aggregates without locking submission. - -## 9. Correctness Invariants - -- **SW-I1 — Incremental bounded input.** Received object frames are charged to - one route at a time; shared-object admission atomically reserves its declared - payload, and rerouting preserves bounded accounting. -- **SW-I2 — Single acceptance.** A submission racing retirement is accepted by - exactly one draining receiver or returned intact for rerouting. -- **SW-I3 — Exclusive ownership.** Exactly one live pipeline epoch can advance - a shared chunk. -- **SW-I4 — Whole placement.** An object remains in one chunk; views can cross strips without - introducing padding or changing its consecutive frame layout. -- **SW-I5 — Physical completion.** Object locations are published only after - every mirror write succeeds. Native FileIO additionally waits for readable - cursor publication; generic callers can select physical-only completion. -- **SW-I6 — Monotonic prefix.** The acknowledged cursor moves strictly forward, - and recovery seals only at that persisted prefix. -- **SW-I7 — Terminal completion.** Every accepted object completes or fails - exactly once; caller cancellation may discard delivery but not batch work. -- **SW-I8 — Repair publication.** A replacement segment is written from the - complete retained prefix and fenced into metadata before any affected location is - published. -- **SW-I9 — Queue-only elasticity.** Pipeline membership changes are driven by - queued bytes, queued objects, and empty/non-busy state, never elapsed time. -- **SW-I10 — Hidden reservation.** A reserved or consumed strip is never - readable through `Chunk.strips`; only fenced confirmation publishes it. -- **SW-I11 — EC publication.** Mirror replicas remain authoritative until four - parity segments are durable and one current-topology survivor from every - mirror set is atomically published. diff --git a/doc/design/chunkio/design-crowdb-chunkio.md b/doc/design/chunkio/design-crowdb-chunkio.md deleted file mode 100644 index fd2835645..000000000 --- a/doc/design/chunkio/design-crowdb-chunkio.md +++ /dev/null @@ -1,625 +0,0 @@ - - - -# CROWDB - Design: Chunk IO Data Path (Overview) - -The chunk IO data path is the client-side layer that writes and reads -large-object data as mirror or EC strips across diskio servers, using chunkdb -for chunk lifecycle management (allocate, append, seal, delete). It -lives in the `crowdb-chunk-client` crate and is consumed by object store -layers and application upload handlers. The chunkdb server design -(chunk lifecycle, placement, EC integration) is in -[`doc/design/chunkdb/design-crowdb-chunkdb.md`](../chunkdb/design-crowdb-chunkdb.md); -the diskio block IO engine is in -[`doc/design/diskio/design-crowdb-diskio.md`](../diskio/design-crowdb-diskio.md). -This doc does not repeat their architecture — it covers the data path -that sits between them: the write pipeline, its backpressure and memory -model, and the design choices that make a 1 TB upload cost the same -~15 MB of RAM as a 50 MB one. The shared mirrored path for small objects is -specified in the -[small-object writer design](design-crowdb-chunkio-small-object-writer.md). - -`ChunkWriter` selects a mirror or EC strip writer from each persisted strip, -so consecutive strips in one chunk can have different layouts and capacities. -The mirror writer sends each write to every mirror segment and synchronizes -them before strip completion; any failed copy returns a write error. The EC -writer derives its shard scheme from that strip's metadata and retains its -separate parity and repair flow. `ChunkReader` likewise dispatches each strip -to mirror or EC recovery based on stored geometry. - -`MirrorStripFlow` routes both ordinary mirror writes and replacement images -through synchronized `DiskWriter` operations. The production routing and metrics -adapters preserve that completion contract: matching DiskIO peers perform ordered -write and flush before one response. Oversized immutable view lists complete in -byte order and only their final request synchronizes the preceding range. -Missing synchronization proof fails directly. A replacement is synchronized -before ChunkDB publishes its new segment mapping; successful surviving replicas -are retained. Stream and WAL replacement images initialize their complete strip -with a retained prefix and zero suffix in one attempt, without automatic retries. -An uncertain initialization cannot authorize publication or immediate reuse. - -`DiskWriter::drain_segment` preserves the segment ordering base and one absolute -deadline through metrics and routing adapters. Its empty synchronized request -waits for earlier backend writes and disk flush; a missing completion proof fails -directly. The caller must first exclude old producers and late admissible requests. -Recovery and allocation reuse cannot substitute an independent disk fsync for -this ordered barrier. - -EC shard capacity is the persisted segment's `unit_count × unit_kb × 1024`, -independent of the allocation-unit size. ISA-L consumes only each data shard's -actual byte prefix. A partial first shard produces equally short code shards; -once a full data shard exists, each code shard spans the full shard width. -No logical padding is written. Device alignment belongs to DiskIO. -Writers seal through one exact-byte RPC, including zero for an empty chunk. -Rounded KiB fields summarize chunk/strip geometry and do not define valid bytes. - -## Table of Contents - -- [1. Non-Goals](#1-non-goals) -- [2. Key Design Decisions](#2-key-design-decisions) -- [3. Write Flow](#3-write-flow) -- [4. Backpressure and Memory Budget](#4-backpressure-and-memory-budget) -- [5. EC Integration](#5-ec-integration) -- [6. Chunk Rotation and Location](#6-chunk-rotation-and-location) -- [7. Completion and Error Handling](#7-completion-and-error-handling) -- [8. Interaction with Neighbors](#8-interaction-with-neighbors) -- [9. Tunables and Defaults](#9-tunables-and-defaults) -- [10. Application API and Routing](#10-application-api-and-routing) -- [11. Performance Workload](#11-performance-workload) - -## 1. Non-Goals - -- **No small-object writer details.** Shared-chunk packing is a separate - component specified by the - [small-object writer design](design-crowdb-chunkio-small-object-writer.md). -- **Reader is a separate component.** Location resolution, mirror/EC fetch, - partial decode, range reads, and bounded streaming are specified in - [Chunk Object Reader](design-crowdb-chunkio-reader.md). -- **No GC of leaked partial chunks.** Best-effort cleanup on abort - leaves Active chunks for a future reaper; this doc does not specify - the reaper. -- **No object protocol server.** Applications call the client library - directly. An S3-compatible or other object protocol server can use the same - API later without owning chunk placement or DiskIO routing. - -## 2. Key Design Decisions - -- **Block-granularity pipeline, not strip-granularity.** The first disk - write starts after 1 MiB (one data block), not after the full 8 MiB - strip. Three strips stay in flight simultaneously (N parity, N+1 - data, N+2 fetch) without unbounded memory. Strip-granularity would - double time-to-first-byte and halve steady-state throughput. -- **Push-based "always store" contract.** `ChunkIoWriter::on_data` - never rejects a buffer; it awaits until internal capacity is free. - This puts the retry-loop in one place (the writer) instead of every - caller. A `FeedStatus` (`Continue` / `Pause`) answers "would the next - push block?" so a dedicated upload task can ignore it and block, - while a shared handler task can pre-check via the non-async - `require_data` hint and return 503 instead of stalling. -- **Bounded preparation, not eager allocation.** A 1 TB object does - not allocate all 250K strips at once. Consumption-driven channels stay only - `prefetch_strips_per_chunk` strip results (default 1) and - `chunk_preparation_depth` chunks (default 1) ahead, - keeping allocation rate and KV metadata pressure bounded regardless - of object size while keeping the cursor fed. -- **Warm admission, then continuous replacement.** Services that know their - write policy prepare a configurable queue of write sessions before admitting - load. The first chunk of every queued session is allocated concurrently. - Consuming a session starts its replacement while the active object writes, - so ChunkDB convergence and ordinary allocation are outside the data path. -- **Shard-based EC, no re-split copy.** The pipeline already holds data - as separate 1 MB `Bytes` blocks. Re-splitting a contiguous buffer - just to feed `crowdb_common::ec::encode` would copy 4 MB per strip for - no benefit. `encode_parity_from_shards` takes pre-split shards - directly and reuses the existing isa-l FFI path — no new C++ code. -- **Repair one failed shard, not the strip.** A durable data or parity write - failure keeps every successful shard and retries only the failed segment - through ChunkDB placement and fenced publication. Exhaustion aborts the - unpublished object; reduced redundancy is never reported as success. -- **Memory budget per pool, not per object.** A `WriterPool` tracks a - total `memory_budget` and an atomic `in_use` counter; `try_acquire` - rejects with `MemoryBudgetExhausted` when full, enabling backpressure - up the call stack. Per-writer footprint is constant (~15 MB peak for - 4+1 EC, 1 MB blocks, defaults), so `max_concurrent = budget / - per-writer-footprint` — a 1 TiB and a 50 MiB upload cost the same RAM. -- **Real-process E2E is the primary write-flow coverage.** Large- and - small-object E2E tests run the client against KV, DiskDB, DiskIO, and - ChunkDB processes, query committed chunk metadata, and read payloads back - from their allocated segments. Large-write coverage also verifies stored EC - parity. The `ChunkAllocator` and `DiskWriter` seams support focused tests for - fault injection, backpressure, cancellation, accounting, and exact boundary - conditions that a real cluster cannot trigger deterministically; they are - auxiliary coverage rather than substitutes for the end-to-end flow. -- **Drive loop in `ChunkWriter`, not the object layer.** The - strip-level drive loop (push block → write to disk → auto-rotate - strips when full → spawn parity) lives inside `ChunkWriter::push`. - The object layer calls `push` + `is_full` + `seal` — it does not - track strip boundaries, block indices, or parity handoff. This - eliminates the `StripPlacement` bridge type and keeps the object - layer thin. -- **Own flatbuffer types directly.** `ChunkWriter` owns `Arc`, - `EcStripWriter` holds `Arc` + strip index, and `seal()` - returns `ProtoLocation` directly. No parallel wrapper structs - (`StripPlacement`, `Location`) — the flatbuffer types are the canonical - representation throughout the write path. - -## 3. Write Flow - -The writer runs three concurrent stages: a fetch stage, a -`ChunkWriter` (which owns the strip-level drive loop + internal strip -prefetch), and a bounded pool of background parity tasks. -`LargeObjectWriter` exposes two driving modes over the same pipeline — -stream mode (`write_stream`, pulling from an `AsyncRead`) and push mode -(the `ChunkIoWriter` trait, §2). In stream mode a fetch stage pulls -from `AsyncRead`; in push mode the caller's bytes go directly to the -block channel. The `ChunkWriter` drive loop is identical in both modes. - -``` - ┌────────────────────┐ - │ ChunkPrefetch │ background, bounded: - │ 1 chunk ahead │ pre-allocates next Chunk - │ (1 strip each) │ for a 16 MiB 8+4 write - └─────────────┬──────────┘ - │ pre-allocated Chunk - ▼ - AsyncRead ──► [Fetch] ──► block_buf ──► [ChunkWriter::push] ─────► disk (data) - read ≤1MB (1 MB per │ write 1 data block → 1 disk - directly into block) │ (immediately, no EC wait) - send to write │ - max_cached_buffer │ when all 4 blocks of strip N written: - = 4 MB (default) ├──── spawn parity ──────────────────── - (backpressure if full) │ ▼ - │ [Parity Task N] (background) - │ EC encode → 4 parity blocks - │ write parity → disks 9..12 - │ (no join — handles collected) - ▼ - auto-rotate to strip N+1, block 0 - (no wait for parity) - │ - │ internal strip prefetch: - │ append_chunk ahead of cursor - │ (bounded: prefetch_strips_per_chunk=1) -``` - -The flow, step by step: - -- **Canonical framing.** Ordinary `LargeAsyncObjectWriter::on_data` input is - copied into its logical frame tail, then each payload is copied once into - its final encoded batch. Header and CRC footer preparation uses stack - regions; full frames have no temporary encoded payload allocation. A partial - final frame is emitted at EOF. The payload-copy counters include these - explicit copies and repeated encoding before a rotation accepts the batch; - they exclude framing bytes, allocator growth, source reception, EC, and - transport staging. Owner-backed `on_framed_data` retains its final frame - allocation and contributes no payload-copy bytes to those counters. -- **Fetch.** Reads from the `AsyncRead` stream directly into the owned - `BytesMut` block. A socket read may return less than 1 MiB; the fetch stage - accumulates in that same allocation, freezes it without copying, then sends it to - `ChunkWriter::push` immediately — it does not wait for the full 4 MB - strip. The fetch stage sends sequential `Bytes` blocks and does not - track block indices or strip boundaries; `ChunkWriter` owns indexing. - On EOF with a partial last block, the partial block is sent. -- **ChunkWriter::push (drive loop).** The central coordinator. It - owns `Arc` and shares it with `EcStripWriter` by ref count. - On each `push`, if the current strip is full, it finishes the strip - (submits bounded data/parity writes and collects completion handles, - no join), checks `is_full()` (chunk-level), and either returns - `Pause` (chunk full — caller rotates chunks) or opens the next strip - (from `chunk.strips` if pre-appended by the internal strip prefetch, - or via inline `append_chunk` RPC). It then pushes the block to the - new strip's data segment via `DiskWriter::write` — one disk per - block. Independent data blocks are submitted without serial completion - waits, and strip N+1 overlaps strip N's parity completion. -- **Parity.** One background task per strip. It receives the strip's - data blocks, EC-encodes via `encode_parity_from_shards` (§5) into - `code_num` parity blocks and writes them to the remaining segments via - `DiskWriter::write`. - Data and parity handles are grouped into one completion per strip. - `ChunkWriter` retains at most `parity_depth` strip completions and joins all - remaining completions at `seal` - time (not at strip finish) — this decouples parity durability from - strip rotation, allowing strip N+1's data writes to start before - strip N's parity completes. -- **Strip prefetch (internal to ChunkWriter).** A background task - appends strips to the current chunk via `append_chunk` ahead of the - write cursor, bounded by `prefetch_strips_per_chunk` (default 1). Larger - known-size objects batch two strips per append when more than four remain. A normal - `append_chunk` response contains only new strips plus `modify_ts`; the - writer merges them locally. A stale revision response supplies the complete - current chunk and is retried once. The result replaces `self.chunk` (Arc-swap — old Arc in - any in-flight `EcStripWriter` stays alive). For known-size objects, - the prefetch stops after enough strips are allocated; for - unknown-size objects, it stays `prefetch_strips_per_chunk` ahead. -- **Chunk prefetch (ChunkPrefetch, object layer).** Pre-allocates the - next `Chunk` with `prefetch_strips_per_chunk` strips ahead of rotation, up to - `chunk_preparation_depth` ahead. On chunk rotation, the object layer - pulls the next `Chunk` from the prefetch receiver (fast path — - pre-allocated) or calls `on_demand` (slow path — prefetch fell - behind). The `Chunk` is passed to `ChunkWriter::open`, which wraps it - in `Arc` and starts the internal strip prefetch. - -### 3.1 Partial Last Strip - -Partial strips occur only at EOF, never mid-chunk. When EOF arrives -before the current strip is full, the writer persists only the filled -data. EC strips also release empty blocks and hand the partial set to -parity (§5). At chunk seal, ChunkDB records the durable `sealed_length` -of every written strip, whether mirror or EC, so a reader can cross -strip boundaries and read the partial final strip. - -## 4. Backpressure and Memory Budget - -Two independent limits bound the pipeline; neither depends on object -size. - -- **`max_cached_buffer`** (default 4 MB = one strip) bounds un-written - data in the fetch channel — blocks sent to the main write task but - not yet written to disk. When disk write is slower than network - receive, the cache fills; once full, the fetch stage blocks and - throttles the stream to the disk write speed. -- **`parity_depth`** (default 2) bounds in-flight parity tasks. When - the parity pool is full, the main write task blocks at hand-off — - backpressure on the write path, decoupled from the fetch cache. - -If prealloc falls behind, the main write task awaits on the bounded -strip channel; the fetch stage keeps filling `max_cached_buffer`, then -blocks when full. No data is lost. - -### 4.1 Per-Writer Footprint - -For 4+1 EC, 1 MB blocks, defaults: - -- `max_cached_buffer` — 4 MB un-written data in fetch cache. -- 1 block being written — 1 MB. -- Up to `parity_depth` (2) parity tasks, each holding the strip's data - blocks (4 × 1 MB = 4 MB, shared via `Bytes` ref count — not copied, - but resident until EC compute completes) + 1 parity block (1 MB). - -Peak: 4 + 1 + 2 × (4 + 1) = **15 MB**. The conservative 15 MB assumes -both in-flight parity tasks hold data refs simultaneously; realistic -steady-state peak is ~11 MB, since EC compute is fast relative to disk -write + fsync and parity tasks stagger. - -### 4.2 WriterPool - -`WriterPool` tracks a `memory_budget` and an atomic `in_use` counter. -`try_acquire` returns `MemoryBudgetExhausted` when the budget is full; -release decrements `in_use` on `Drop`. `max_concurrent = -memory_budget / per-writer-footprint`. Many concurrent large-object -uploads are thus bounded by available RAM, not by object size — the -pool rejects new writes when the budget is exhausted, propagating -backpressure up the call stack. - -## 5. EC Integration - -The pipeline holds data as separate 1 MB `Bytes` blocks; the existing -`crowdb_common::ec::encode` re-splits a contiguous buffer into shards, -which would force a 4 MB copy per strip just to re-split. Instead, -`encode_parity_from_shards(scheme, data_shards)` takes pre-split data -shards directly and reuses the existing isa-l FFI path — no new C++ -code. - -`data_shards.len()` is `data_num` for a full strip, or `< data_num` -for a partial strip. isa-l supports partial EC: missing shards are -treated as zero for the encoding matrix, so no padding is written to -disk — the reader reads only `sealed_length` bytes. The function -returns `code_num` parity shards. - -Edge cases: - -- Full strip → standard EC. -- Partial strip → parity from present shards; reader reads only - `sealed_length` bytes. -- Single-block object (< 1 MB) → 1 partial data shard, 1 parity shard. - -## 6. Chunk Rotation and Location - -Very large objects (`> max_chunk_size`) span multiple chunks. Chunk -size is always a multiple of strip data capacity — the writer only -appends whole strips — so rotation happens at strip boundaries, never -mid-strip. When `ChunkWriter::push` finishes a strip and `is_full()` -returns true, it returns `FeedStatus::Pause` without pushing the -buffer. The object layer then calls `seal()` (joins all in-flight -parity handles, `seal_chunk` RPC, returns `ProtoLocation`), records -the location, pulls the next `Chunk` from `ChunkPrefetch`, and opens a -new `ChunkWriter`. The buffer is re-pushed to the new chunk. The -`ProtoLocation` array accumulates one entry per rotated chunk, ordered -by `logical_offset`. - -Object writers use the common revision-2 ordinary frame codec: 15-byte header, -20-byte footer and at most 65,501 payload bytes per 64 KiB frame. The role is -ordinary; Stream head/follower markers are never added to object payloads. -Unknown-length uploads keep bounded tail buffers and publish an object only -when the complete upload finishes. Native socket buffers reserve the same -header/footer geometry and retain payload ownership through scatter/gather IO. -Readers retain explicit V1 support, including both historical CRC conventions; -revision 2 has canonical header/payload CRC coverage with separate chunk-ID -validation, shared with the C++ codec. - -### 6.1 Location - -`ProtoLocation` (from `crowdb-protocol::chunkdb::rpc::Location`) records -which chunk holds a contiguous byte range of an object, the byte range -within that chunk, and the object-level logical offset/length so a -multi-chunk object reads back as one contiguous stream. An object -spanning N chunks has N locations ordered by logical offset, contiguous -and non-overlapping. The within-chunk offset is always 0 for the -large-object writer (dedicated chunks filled from the start); it -exists for future shared-chunk packing and range reads. Serialization -is flatbuffers via the flatbuffer `Message` trait (`encode_to_vec` / `decode`). - -Edge cases: - -- Empty object (size 0) → `Vec` is empty; no chunk allocated. -- `logical_length` counts object payload bytes. `length` counts physical - bytes including ordinary frame headers and footers. Readers infer the - supported frame geometry from this pair and verify each actual frame kind - and revision before exposing payload views. -- `max_chunk_size` not a multiple of strip data capacity → rotation - still happens at strip boundaries; the actual chunk size is the - multiple-of-strip value at or just above the threshold. - -## 7. Completion and Error Handling - -`write_stream` must seal the final chunk and return the `ProtoLocation` -array on success. On error or abort it must not leak partial chunks and -must return already-sealed `ProtoLocation`s for caller cleanup. - -- **Completion** — `ChunkWriter::seal()`: finish the current strip if - it has data (spawn partial parity, no join), join all in-flight - parity handles, `seal_chunk` RPC, return `ProtoLocation`. If the - chunk is empty (0 bytes written), `delete_chunk` instead of - `seal_chunk`. -- **Error / abort** (`on_error`) — stop strip and chunk prefetch, retain and - drain every submitted write completion, then `delete_chunk` on the partial - chunk. Draining before deletion prevents a late write from hitting reused - storage. The public prepared-write API returns an error rather than a - partially successful object. -- **Single-segment replacement** — a failed data or parity completion retains - its `Bytes`, inserts the failed disk into the client-wide lock-free negative - list, and queries current chunk metadata. ChunkDB allocates one tentative - placement excluding every existing strip disk and nodes already at the - strip's failure-domain limit. The client writes that segment and publishes a - geometry-identical strip through exact revision-and-range replacement. - Successful shards are untouched. -- **Fencing and retry** — publication uses a deterministic operation ID. - Ambiguous results retry the identical request. A definite revision conflict - discards the tentative segment, re-queries, and retries only while the exact - failed identity remains. Repeated disk failures extend their exclusion TTL. - Attempts are bounded by `large_write_repair_attempts`, three by default. -- **Repair exhaustion** — `ChunkWriter::seal` returns the failure before - `seal_chunk`; its owner drains remaining completions and deletes the active - chunk. It never seals a strip with missing parity or publishes degraded - large-object locations. -- **Dropped writer** — dropping does not perform async metadata cleanup. - Applications call `on_error` when abandoning a started push-mode write; - an Active partial chunk left by process loss remains for future lifecycle - GC. - -Edge cases: - -- `on_data` after `on_finish` / `on_error` → `IoError::Finished`. -- `on_finish` twice → `IoError::Finished`. -- `on_error` with no sealed chunks → `Ok(vec![])`. -- EC encode failure → pipeline aborts immediately. EC encode is a CPU/ISA-L - failure rather than a disk placement failure and cannot use segment - replacement. -- `delete_chunk` fails during cleanup → log + continue (best-effort; - the partial chunk stays Active and is reaped by a future GC task). - -## 8. Interaction with Neighbors - -- **chunkdb** — the writer calls allocate / append / seal / delete / - fenced strip-range replacement / query via `ChunkAllocator`. chunkdb handles - placement and lifecycle; the writer is unaware of internal placement - logic — it receives `Segment` placements and writes to them. -- **diskio** — the writer writes data and parity blocks through the single - durable-completion contract `DiskWriter::write`. - `RoutedDiskWriter` is a narrow foreground-policy adapter over the semantic - `crowdb-diskio-client`; it does not own discovery, endpoints, RPC transport, - or result decoding. Conversion and repair use that client's separately - bounded priority lane. Direct wire access exists only in protocol fixtures. -- **crowdb-common EC** — `encode_parity_from_shards` is the shard-based + - partial-encode entry point used by parity tasks. -- **crowdb-protocol** — `ChunkId`, `*ChunkRequest` types, `Segment`, - `Chunk`, `Location` (`ProtoLocation`) message. - -## 9. Tunables and Defaults - -Chunk capacity is a write-policy limit, while each persisted strip records its -own size and protection layout. A single-node test chunk may contain many -1 MiB one-copy mirror strips. The S3 and Iceberg access policies can select -different small and large chunk capacities in their configuration. Tree page -storage separately uses `storage.tree_chunk_capacity_bytes` from the chunk KV -server configuration and splits a single-copy tree chunk into 1 MiB strips. -Chunk KV stream storage uses `storage.stream_chunk_capacity_bytes` for the -`Stream` chunk type, also independently of its 1 MiB strip geometry. - -| Knob | Default | Role | -| --- | --- | --- | -| `max_chunk_size` | 1 GB | Chunk rotation threshold. | -| `prefetch_strips_per_chunk` | 1 result | Strip-preparation results buffered ahead of the write cursor. | -| `parity_depth` | 2 strips | Completed-strip write groups allowed in flight. | -| `chunk_preparation_depth` | 1 chunk | Chunk preparation ahead of rotation. | -| fetch granularity | 1 MB | One data block per fetch call. | -| `max_cached_buffer` | 4 MB | Un-written data budget in fetch channel (one strip). | -| `memory_budget` | (pool) | `WriterPool` total; `max_concurrent = budget / per-writer-footprint`. | - -## 10. Application API and Routing - -`ChunkIoClient` is the application boundary. `connect` takes management seeds -and discovers ChunkDB endpoints, DiskIO registrations, hardware disks, and disk -group ownership. `prepare_large_write` starts bounded chunk preparation when -the object size and `LargeWritePolicy` become known. The single-use -`PreparedLargeWrite::write_stream` consumes an `AsyncRead`. -`PreparedLargeWrite::write_buffers` transfers caller-owned `Bytes` blocks -directly into the same writer, bypassing fetch and its input copy. Both return -`LargeWriteResult` with locations, logical and EC-expanded physical bytes, -chunk and strip counts, elapsed time, and preparation stalls. - -`prepare_large_writes` starts several single-use sessions concurrently and -waits until each owns its first chunk. A server calls it before opening its load -gate, keeps the returned queue, and starts one replacement session whenever it -consumes one. Queue depth is an application admission setting; the benchmark -defaults to ten through `--prefetch-chunks`. Unused sessions are explicitly -aborted so their Active chunks are deleted. - -The write and read hot paths call the semantic DiskIO client with a checked -segment-relative address, caller-owned `Bytes`, lane, durability policy, and -deadline. That client reads immutable disk-ID route generations through RCU, -owns fixed normal and priority connection groups, and selects healthy members -without a caller lock. Refresh joins authoritative service and hardware -records, constructs and prewarms a complete replacement off-path, and reuses -unchanged endpoint groups. Missing or duplicate ownership rejects the whole -candidate; no chunk caller chooses an arbitrary endpoint. - -`ChunkIoClientConfig::diskio_connections_per_endpoint` configures the normal -foreground group. Conversion and repair use reserved priority resources, so -their queue or connection pressure cannot consume the normal group. For the -native crowdb-tree page store, `ChunkIoClient` passes through an opaque retained -route set produced by the DiskIO client; neither application code nor the CLI -assembles routes or constructs RPC servers and connections. - -Topology refresh follows server ownership. `refresh_chunkdb_routes` refreshes -ChunkDB endpoints and range bindings; `refresh_diskio_routes` asks the DiskIO -client to publish its next complete generation. Metrics wrappers only observe -the two narrow seams and do not own scheduling, discovery, or connection -lifetime. - -## 11. Performance Workload - -Chunk IO performance workloads live in `crowdb-chunk-client`; CLI commands -only map arguments, start the standard process metrics collector, and format -results. - -- `run_large_write_benchmark` retains the deterministic bounded-source writer. - Before its timer starts, it prepares write sessions and distributes them - across workers. Each worker starts a replacement allocation when consuming a - session. -- `run_small_write_benchmark` uses `prepare_small_write`, awaits each real - Location, drains the shared pool, and reports batch fill, queue delay, - active/draining pipeline gauges, queue-driven scale-out/scale-in, throughput, - latency, and exact object accounting. -- `run_read_benchmark` prepares a bounded reusable dataset through the real - small- and/or large-write APIs before timing. Small, large, and deterministic - mixed request modes call `read_object`; mixed ratio is by request count. - Successful reads validate logical length. Under NullDisk they deliberately - do not compare payload contents. - -The CLI commands are `bench chunkio write`, `write-small`, `read-small`, -`read-large`, and `read-mix`. Results separate preparation from timed load and -include aggregate throughput, latency, errors, stop reason, and exact admitted -versus completed accounting. - -The CLI workload admits objects until a shared duration deadline (20 seconds by -default). A worker checks the deadline before admitting its next object; an -object already admitted is allowed to finish, including all DiskIO completions -and chunk seal, before the worker exits. Reported elapsed time includes this -drain tail. The object count remains a safety cap and a deterministic test mode, -not the normal regression stopping condition. - -The regression fixture starts three co-located logical nodes in one rack: -three KV servers, three DiskDB, three ChunkDB, and three DiskIO processes -backed by `NullDisk`. KV state and WAL use `mem-block` with fsync disabled, so -benchmark metadata also avoids real media. Read preparation still writes real -ChunkDB/DiskDB/KV metadata and writer-produced Locations; only payload storage -is synthetic. An 8+4 strip in this intentionally compact local topology -requires the local-test-only unsafe EC placement option; disk ownership and -routing remain strict. Unsafe placement still balances blocks across the -available nodes within a rack; it relaxes the failure-domain limit without -concentrating the strip on the first node. The retained logs -contain `bw_read_mib` and `bw_write_mib` when host PMU counters are available. -Their sum is observed host memory traffic during the workload, not physical DIMM peak bandwidth and not -an application-byte estimate. Loopback TCP, EC expansion, RPC framing, kernel -copies, EC calculation, synchronous writes, and metadata work all keep end-to-end logical -throughput below that hardware envelope. - -### 11.1 Intel Core i9-7960X Baseline - -Reference run 2026-09-08: Intel Core i9-7960X (16c/32t, Skylake-X), 4 DDR4 -channels at 2667 MT/s (~85 GB/s peak), Linux 6.11, `perf_event_paranoid=-1`. - -The sentinel uses 16 MiB objects, EC 8+4, 1 MiB blocks, ten prefetched chunks, -and two strips prefetched per active chunk. Each worker creates one deterministic -random 1 MiB source block and reuses it. Stream cases traverse the production -`AsyncRead`; direct cases pass reference-counted `Bytes` blocks to the same -writer without a fetch copy. Both traverse EC, routed RPC, the DiskIO handler, -and real io_uring submission. RPC sends every 1 MiB data or parity payload through -the loopback socket. The storage target is the only substitution: NullDisk -replaces production BlockDisk and does not request stable-media durability. - -| Mode | Writers | Logical MiB/s | Physical MiB/s | p50 / p99 ms | DRAM read avg | DRAM write avg | DRAM total avg | -| -------------- | ------: | ------------: | -------------: | ----------------: | ------------: | -------------: | -------------: | -| Stream | 1 | 162.5 | 243.8 | 99.191 / 111.225 | 2,496.6 | 1,393.0 | 3,889.6 | -| Direct buffers | 1 | 229.1 | 343.7 | 69.191 / 85.641 | 1,996.1 | 1,228.7 | 3,224.8 | -| Stream | 4 | 1,965.6 | 2,948.5 | 31.917 / 51.806 | 12,699.7 | 10,943.0 | 23,642.7 | -| Direct buffers | 4 | 2,672.1 | 4,008.1 | 22.811 / 40.000 | 11,799.6 | 12,051.5 | 23,851.1 | -| Stream | 32 | 3,249.3 | 4,873.9 | 151.716 / 267.992 | 19,562.5 | 15,603.7 | 35,166.2 | -| Direct buffers | 32 | 3,547.9 | 5,321.9 | 138.654 / 261.388 | 14,816.5 | 12,766.6 | 27,583.1 | - -Per-object client-stage time is measured inside the production writer. Times -overlap and therefore must not be added to predict object latency. - -| Flow step | Stream 1 | Direct 1 | Stream 4 | Direct 4 | Stream 32 | Direct 32 | -| --------------------- | -------: | -------: | -------: | -------: | --------: | --------: | -| Source/read copy | 20.247 | 0 | 3.737 | 0 | 8.704 | 0 | -| Fetch assembly copy | 0 | 0 | 0 | 0 | 0 | 0 | -| EC encode | 20.175 | 13.164 | 15.213 | 12.357 | 22.045 | 20.222 | -| Write-completion wait | 53.990 | 54.238 | 11.741 | 10.118 | 122.786 | 120.972 | - -All values are milliseconds per object, aggregated across concurrent writers; -stages overlap and must not be summed into latency. Every object has two data -strips, 16 data blocks, eight parity blocks, and therefore 24 full 1 MiB DiskIO -RPCs. This is exactly 16 MiB logical and 24 MiB physical traffic: EC 8+4 gives -the expected 1.5 physical/logical ratio. The run recorded zero preparation -stalls and zero `append_chunk` calls: initial chunk allocation returned both -strips requested by an explicit `prefetch_strips_per_chunk=2` override. - -Stream mode has one application payload copy per block: socket/source into its -final owned block. Direct-buffer mode has zero application payload copies; -`Bytes` clones, channels, EC input, routing, and RPC buffer wrapping transfer -ownership only. Both modes still copy or DMA bytes across the loopback transport -and DiskIO boundary. EC reads 16 MiB and creates 8 MiB parity per object, while -the RPC path transports 24 MiB, so physical memory traffic must be multiple -times frontend bandwidth. The multiplex-corrected host read/write counters are reported above, -but is not a complete byte ledger and can be below RPC bandwidth because of its -sampling scope and interval. - -### 11.2 AMD Ryzen 9 5950X Baseline - -Reference run: _pending._ - -| Mode | Writers | Logical MiB/s | Physical MiB/s | p50 / p99 ms | DRAM read avg | DRAM write avg | DRAM total avg | -| -------------- | ------: | ------------: | -------------: | -----------: | ------------: | -------------: | -------------: | -| Stream | 1 | | | | | | | -| Direct buffers | 1 | | | | | | | -| Stream | 4 | | | | | | | -| Direct buffers | 4 | | | | | | | -| Stream | 32 | | | | | | | -| Direct buffers | 32 | | | | | | | - -| Flow step | Stream 1 | Direct 1 | Stream 4 | Direct 4 | Stream 32 | Direct 32 | -| --------------------- | -------: | -------: | -------: | -------: | --------: | --------: | -| Source/read copy | | | | | | | -| Fetch assembly copy | | | | | | | -| EC encode | | | | | | | -| Write-completion wait | | | | | | | - -### 11.3 Bottleneck Conclusion - -Independent blocks are already parallel: each of the eight data writes is -spawned as soon as its block enters `EcStripWriter`, then four parity writes are -spawned together after EC completion. `ChunkWriter` groups the 12 handles per -strip, permits `parity_depth` strip groups in flight, and joins them before -seal. There is no serial per-block completion wait. - -Chunk preparation is also off the measured hot path. The remaining single-write -gap is input ownership plus cache pressure: direct buffers reach 229.1 MiB/s, -1.41 times the stream result, and spend no time in source reads. Four direct -writers peak at 2,672.1 logical MiB/s. At 32 writers throughput rises to 3,547.9 -MiB/s but completion wait rises to 121 ms/object. The loopback RPC/DiskIO queue -is saturated well before 32 writers; more concurrency increases latency rather -than bandwidth. Further work should profile and reduce RPC/kernel transport cost -and EC/cache contention. Increasing prefetch or adding wrapper layers will not -address the measured bottleneck. diff --git a/doc/design/config/design-crowdb-config.md b/doc/design/config/design-crowdb-config.md deleted file mode 100644 index aa53a842e..000000000 --- a/doc/design/config/design-crowdb-config.md +++ /dev/null @@ -1,195 +0,0 @@ - - - -# CROWDB - Design: Service Configuration - -This document defines the startup configuration contract shared by CROWDB -server processes. Component designs define the meaning of their domain -settings; this document defines how values are owned, loaded, merged, -validated, and activated. - -## Table of Contents - -- [1. Scope](#1-scope) -- [2. Ownership](#2-ownership) -- [3. Resolution and Activation](#3-resolution-and-activation) -- [4. TOML Schema](#4-toml-schema) -- [5. Reload and Restart](#5-reload-and-restart) -- [6. Deployment](#6-deployment) -- [7. Failure Handling](#7-failure-handling) -- [8. Invariants](#8-invariants) -- [9. Non-Goals](#9-non-goals) - -## 1. Scope - -The `crowdb-paxos-kv-server`, `crowdb-diskdb`, `crowdb-chunkdb`, `crowdb-diskio`, -and `crowdb-access-server` processes accept typed TOML startup -configuration. A service may require a file or make it optional, but a supplied -file follows the same resolution and failure rules in every process. - -This contract covers process configuration. Durable cluster topology, -membership, allocation metadata, and application data keep their existing -authorities. - -## 2. Ownership - -- A reusable library owns config types for behavior implemented by that - library. The KV schema is therefore owned with the KV library. -- A server binary owns startup inputs and the final merge before constructing - listeners and services. -- `crowdb-common` owns shared Rust TOML loading, validation, watching, and - structural diff mechanics; it does not own service-specific fields. -- `DioConfig` owns the equivalent typed C++ schema and TOML decoding for - diskio. - -Client-side concurrency fields remain named for the dependency they drive, -such as `kv_rpc_workers`, `diskdb_rpc_workers`, and `peer_pool_size`. -`server.rpc_workers` always means workers accepting that process's inbound RPC -traffic. - -## 3. Resolution and Activation - -Every process resolves a complete candidate in this order: - -```text -typed compiled defaults -> TOML values -> explicitly present CLI values - -> complete validation -> runtime construction -``` - -A CLI parser default is not an explicit value and cannot overwrite a file. -Rust CLI fields participating in the merge use optional values. Diskio loads -its single `--config` path before replaying the other explicitly supplied CLI -options, so argument order does not change precedence. - -The final candidate is validated before listener construction. Runtime -components receive values from this candidate; wiring layers do not maintain -independent defaults. - -## 4. TOML Schema - -Common section names describe process roles: - -| Section | Ownership | -|---------------|-------------------------------------------------------------| -| `[server]` | Listener identity, server-side RPC resources, and heartbeat | -| `[engine]` | Local execution and I/O engine tuning | -| `[group0]` | Local group-0 discovery and registration inputs | -| `[metrics]` | Metrics output and cadence | -| Domain tables | Service-owned storage, placement, lifecycle, or policy | - -The service schemas retain their different domain sections: - -| Service | File policy | `server.rpc_workers` default | Principal domain sections | -| ------- | ----------- | ---------------------------- | ------------------------------------------------ | -| KV | Optional | 2 | Paxos, WAL, engine, metrics | -| diskdb | Required | 2 | Storage, heartbeat, persistence, scanner, sync | -| chunkdb | Required | 2 | Storage, placement, lifecycle, topology, clients | -| diskio | Optional | 4 | Engine, group-0 discovery, metrics, disk entries | -| Access | Optional | N/A | HTTP listeners, shared read, S3, Iceberg GC | - -The S3 and Iceberg access processes accept the same `AccessConfig` schema. -Each process activates only its own protocol section plus `[common]`, `[read]`, -and `[small_write]`. The file is selected with `--config`; a supplied file is -validated before connecting to dependencies or binding listeners. Read slots, -the per-read window, the global retained-byte budget, the separate EC recovery -budget, and small-write memory limits are static startup settings. Credentials -and bearer tokens remain in the environment. For access settings that predate the -file, an explicitly populated TOML value takes precedence over the legacy -environment variable; an omitted TOML value keeps the legacy behavior. - -The small-object routing bound is `threshold_ratio × disk_block_bytes × EC data -shards`; mirrored strips use one data shard. `[small_write]` supplies the default -EC layout for both access writers, and S3's EC override changes its small and -large writers together. The default is 8+4 with 1 MiB data blocks and ratio -0.9. The single-node container profile uses 2+1. - -All schemas tolerate unknown keys so a newer file can be staged before a -binary upgrade. A known key with the wrong type or invalid value rejects the -candidate. Omitted fields take their typed defaults. - -Diskio represents disks as ordered `[[disk]]` entries. A disk ID is a string -because it accepts either a single hexadecimal low word or `high:low`. Each -entry supplies a path and one zone capacity. An empty path selects a dummy -disk. Fault latency bounds and error rate are optional `[server]` fields. - -## 5. Reload and Restart - -`server.rpc_workers` is static in all services because changing it requires -reconstructing the RPC listener. A file watcher may detect and report the -change, but the running listener retains its startup value until restart. - -Other fields follow their component's reload contract. A watcher never makes -a startup-only field dynamic merely because it can observe a changed file. - -## 6. Deployment - -Tracked templates are executable examples guarded by loader tests. Local -deployment writes node-specific files where addresses, roots, or group-0 seeds -vary, and restart reuses the same file path. Explicit deployment overrides are -represented as explicit CLI options and therefore retain the standard -precedence. - -The single-node container uses a named, validated `DeploymentProfile` and -rendered service configurations. `crowdb-monitor` owns service supervision, -dependency order, probes, restart budgets, bounded logs and shutdown. A durable -bootstrap manifest binds the profile/configuration digest and generated -credential identity to the mounted data volume. Restart replays completed steps; -conflicting identity fails closed. Child recovery revalidates storage, catalog -and Web authority before readiness returns. - -Release programs and UI are compiled incrementally on a Linux amd64 host with -the locked repository toolchain. Docker receives a staged runtime directory -containing only those artifacts, required shared libraries and the deployment -profile. It performs no source compilation and contains no Pixi or compiler. -Runtime linkage and source/version metadata are checked while packaging into -the digest-pinned Ubuntu image. The publication job reuses the verified staged -artifacts rather than compiling a second set. - -Group 0 owns hardware/topology and service registration. It does not store -container mounts, process PIDs or restart policy. Docker Web reads live Group 0 -topology and fresh monitor process status independently; missing authority does -not produce an empty or cached topology. Automatic single-node startup disables hardware/process mutation at both UI and -backend. Manual node startup runs only monitor, SSH and Web until an operator -prepares nodes and initializes Group 0; then shared authority enables management. -Both startup policies use the same image and services. Discovery UUID, SSH keys, -host keys, binding and operation recovery records reside on the persistent root. -`CROWDB_PAXOS_KV_MANAGEMENT_ADVERTISE_ADDR` specifies the reachable management address -for service registration independently of the local listening address. See the -[deployment architecture](../deploy/design-crowdb-deploy.md) for ownership and -recovery ordering. - -Public examples and the container profile use Console 9090, S3 9091 and -Iceberg 9092. Standalone Web defaults to 9090; explicit service configuration -and managed deployment port allocations take precedence. Container publication, -volume and endpoint usage are defined by the container deployment files. - -## 7. Failure Handling - -Malformed TOML, an unreadable named file, a wrong known-field type, or failed -validation terminates startup before bind. The loader does not silently fall -back to defaults after an operator explicitly names an invalid file. - -Diskio file loading is transactional: decoding and validation use a temporary -`DioConfig`, and the caller's config changes only after the complete candidate -is valid. - -## 8. Invariants - -- **I1 — deterministic precedence:** The same defaults, file, and explicit CLI - inputs resolve to the same candidate independent of CLI argument order. -- **I2 — complete activation:** No runtime component observes a partially - decoded or partially validated candidate. -- **I3 — worker viability:** Every server starts with at least one inbound RPC - worker. -- **I4 — one runtime authority:** Listener construction reads - `server.rpc_workers` from the final candidate and has no shadow default. -- **I5 — static-change honesty:** A detected static change is not reported as - active until a restart has constructed runtime state from it. -- **I6 — restart stability:** Deployment restart uses the same serialized - configuration path as initial startup. - -## 9. Non-Goals - -Configuration is local to each server. Group 0 does not publish configuration -revisions, resolve scoped overrides, track applied revisions, or coordinate -dynamic activation. The local file remains the startup authority. diff --git a/doc/design/console/design-crowdb-console-ui.md b/doc/design/console/design-crowdb-console-ui.md deleted file mode 100644 index 970e168be..000000000 --- a/doc/design/console/design-crowdb-console-ui.md +++ /dev/null @@ -1,1865 +0,0 @@ - - - -# CROWDB - Design: Console UI Specification - -Depends on: [Console architecture](design-crowdb-console.md), -[KV architecture](../kv/design-crowdb-paxos-kv.md). -Satisfies: [Console architecture](design-crowdb-console.md). - -This is the authoritative UI design and acceptance specification. Each numbered -contract defines observable behavior, not a claim that its implementation has -passed. Implementation status, runtime evidence, defects, and execution steps -belong in the working plan. There is no separate UI spec to reconcile with this -file. Backend architecture and wire formats remain in the linked component docs. - -The scope covers all seven domains and their shared interaction rules. Backend -prerequisites and inspection APIs are part of acceptance; a rendering fixture -does not establish that a native service operation succeeds. - -## Contents - -- [1. Product contract](#1-product-contract) -- [2. Reference environment](#2-reference-environment) -- [3. Navigation and layout](#3-navigation-and-layout) -- [4. Visual language and service identity](#4-visual-language-and-service-identity) -- [5. Cluster](#5-cluster) -- [6. Properties and cross-links](#6-properties-and-cross-links) -- [7. Modes and embedding](#7-modes-and-embedding) -- [8. Bounded observation](#8-bounded-observation) -- [9. Creation and mutation dialogs](#9-creation-and-mutation-dialogs) -- [10. Accessibility](#10-accessibility) -- [11. Verification contract](#11-verification-contract) -- [12. KV](#12-kv) -- [13. Service lifecycle](#13-service-lifecycle) -- [14. DiskGroup and disk management](#14-diskgroup-and-disk-management) -- [15. Capacity](#15-capacity) -- [16. Loading, failures, and recovery](#16-loading-failures-and-recovery) -- [17. Fixed-cluster operator flow](#17-fixed-cluster-operator-flow) -- [18. Acceptance scenarios and scope](#18-acceptance-scenarios-and-scope) -- [19. Chunk](#19-chunk) -- [20. Chunk-KV](#20-chunk-kv) -- [21. Iceberg](#21-iceberg) -- [22. S3](#22-s3) -- [23. Actions and properties](#23-actions-and-properties) -- [24. E2E organization and timing](#24-e2e-organization-and-timing) -- [25. Dataset](#25-dataset) - -## 1. Product contract - -- **UI-01:** One Console serves one configured cluster. Opening the UI enters - that cluster directly; no Connect dialog, endpoint input, or access-key form - precedes ordinary browsing. -- **UI-02:** The seven top-level tab labels are Cluster, PaxosKV, Chunk, ChunkKV, - Iceberg, Dataset and S3. KV and Chunk-KV remain their internal domain identities. - Each domain has an independent selected resource and query window. Cluster - shares topology/capacity selection and tree expansion without a view switch; - topology visibility follows CLU-05. -- **UI-03:** Assume an administrator/root session until UI authentication is - introduced. Available backend operations remain authoritative; capability - restrictions are not inferred from a missing login screen. -- **UI-04:** Display real resources and structured values. Example graphs, - inferred ownership, and placeholder counts must not look like observed data. -- **UI-05:** Every remote list, expansion, preview, and layout has a work/size - bound. A large cluster must not require a complete scan before the UI responds. -- **UI-06:** Visible UI behavior is governed by this specification. A proposed - change to behavior or layout is first discussed and added here, including its - ASCII layout when applicable; implementation follows the agreed spec and may - not silently introduce a conflicting control or tree level. Focused tests or - E2E coverage accompany the implementation. -- **UI-07:** Iceberg and Dataset work areas and Properties show resource data, - controls and concise current state. Do not keep tutorials, pagination limits, - upload/publication internals, immutability explanations or GC explanations as - permanent prose around tables and editors. General guidance belongs in the - existing Help documentation. Keep field labels, identities, measured values, - empty states, upload progress, validation errors and contextual conflict or - destructive-action information. Draft feedback uses one short status: - `Saving…` or `Saved locally`; a failed save must not display success. - -## 2. Reference environment - -The main acceptance environment has one Rack and three Nodes. Each Node has one -KV, DiskDB, ChunkDB, DiskIO, Chunk-KV, and Access Server instance. Group 0 has -three replicas; an ordinary data Group 1 has three replicas. Storage has valid -DiskGroup bindings, live owners, and writable disks. Allocated and free blocks -provide observable capacity and bitmap fixtures. - -- This is normal multi-node deployment. Single-node/unsafe colocated settings - cannot be silently enabled to make acceptance pass. -- Rack diversity is preferred when multiple racks exist. A single rack with - enough distinct nodes is valid; properties report actual node/rack protection. -- Three local processes on one host verify logical placement and UI behavior; - they do not demonstrate physical-host fault tolerance. -- API provisioning may prepare this environment. The setup method does not - replace the dedicated tests for creation and lifecycle dialogs. -- Empty, partial, unavailable, and very large datasets are additional fixtures, - not reasons to replace the populated reference environment with mocks. - -## 3. Navigation and layout - -- **NAV-01:** The header keeps the seven domains in a stable order. Domain - changes preserve that domain's bounded query state and selected identity. -- **NAV-02:** The left panel selects hierarchy/scope; the center presents the - selected resource or a paged child list; the right panel presents selected - properties. Resource operations follow the context-menu contract in §23. -- **NAV-03:** Cluster row/card selection updates properties without replacing the center - scope. Names enter that resource; closing properties retains the work area. - Cluster breadcrumbs identify the current physical scope; they are navigation, not duplicate - Back links for inline details. - This graph inspection distinction does not apply to ordinary resource tree/list - rows: activating their label, icon or row selects the same resource. Expansion - controls are separate. At DiskGroup scope, both Disk icon and Disk ID enter - the same Disk workspace, including from a DiskDB ownership reference. -- **NAV-04:** Cross-links identify the exact destination resource. Returning - restores the originating selection and page. A stale/deleted destination has - an explicit state rather than silently selecting a different item. The header - and browser Back/Forward share one session history with at most 32 retained - visits. History stores identities, bounded - query cursors and scroll coordinates, never object bodies or metadata payloads. - New navigation clears Forward. Returning refetches the saved bounded window; - scroll restoration waits for that window to render. Resource types and parent - identities participate in selection equality, so equal numeric IDs in - different resource kinds cannot share a selected highlight. Returning to an Iceberg reference verifies the - saved metadata generation before inspecting it again. -- Shared physical/logical trees retain expansion and local search per domain. - Return restores both expanded and collapsed branches. KV captures bounded - byte cursors and focused Key identity, then refetches the window; it does not - keep Value payloads in history. Capacity retains at most 32 Disk query states - containing zone window, selected Zone and bitmap block window. Parent - navigation can unmount a detail without losing its return query. -- **NAV-05:** Trees expand lazily. Each large child collection has continuation - or Load more and an explicit end/partial state. Routine loaded-item counts are - not shown in tree footers; they are internal pagination state unless a domain - contract explicitly makes a count part of the selected resource. Collapsed - branches do not recursively load their descendants. -- **NAV-06:** The center remains usable with the right panel open. Tables and - diagrams scroll within their work area; action bars and pagination remain - reachable. Resizing does not reset selection or trigger unbounded refetches. - Capacity observations retain the last successful inventory, usage and scan - independently after a failed refresh. Unknown values and refresh errors appear - with the affected capacity data; raw observation metadata is not a global strip. Runtime usage is keyed by group/disk - scope; an unobserved scope remains unknown instead of borrowing other totals. -- **NAV-07:** Every domain uses the shared Tree appearance and draggable - panel dividers. Cluster/KV left default width is 240 px; data workbenches use - 280 px. Properties default is 320 px. - Cluster navigation resizes between 200 and 360 px, properties between 280 - and 480 px, preserving 480 px for the center when space permits. Below - 1100 px in Cluster, properties overlays the work area as a dismissible drawer; Escape - closes it and restores focus. Navigation remains visible without a hide/show control. Other domain dividers retain their 220–600 px bounds. - Switching domains introduces no panel replacement/slide animation. -- **NAV-08:** Parent cards in center topology diagrams have separate named controls for - opening the resource and collapsing or expanding children. Card bodies inspect - without navigating. Children retain their own - expansion state. Sidebar focus reveals hidden ancestors. Right-click opens - the entity menu without toggling expansion. Fit All fits visible cards. - Hiding a topology panel interrupts viewport motion before zero-size geometry - is observed. Rapid domain switches retain finite coordinates; centering runs - only in a visible, measured canvas. - The default diagram initially expands Datacenter/Rack and stops at visible Node cards; - service descendants appear when a Node is explicitly expanded. - -The header shows the CrowDB brand, domain navigation, then Back/Forward and -Refresh in the right action area. Access instance `access-1` displays as `AS-1` -while API operations retain the original identifier. - -Shell layout (NAV-01/02/04/07): - -```text -+------------------------------------------------------------------------------------------------------+ -| CrowDB Health | Cluster PaxosKV Chunk ChunkKV Iceberg Dataset S3 | [Back] [Forward] [Refresh] | -+------------------------+------------------------------------------------------+----------------------+ -| Resource tree | Current scope | Properties | -| v Parent | Query / filter [Refresh] | Selected identity | -| Child | Primary content or paged child list | Structured values | -| | [Previous] [Next] | | -+------------------------+------------------------------------------------------+----------------------+ - draggable divider draggable divider -``` - -All wireframes show layout contracts, not runtime evidence. Names, IDs, counts -and statuses are illustrative. `>`/`v` denote collapsed/expanded branches, `*` -marks selection, and `[Auto]` denotes the accessible Auto icon. Menu entries -remain subject to the resource capabilities and lifecycle rules in the text. - -## 4. Visual language and service identity - -- **VIS-01:** Use muted colors with readable text and visible boundaries. - Buttons, including Run Recalc, must have adequate text/background contrast. - Color alone cannot encode state, role, or selection. -- **VIS-02:** Service instances use one compact `TYPE-ID` convention everywhere: - `PKV-1`, `DDB-1`, `CDB-1`, `DIO-1`, `CKV-1`, and `AS-1`. Trees, topology cards, - selected titles, and instance menus use the same label. -- The suffix identifies the service instance in its own type. It must not be - fabricated from a Node ID when those identities differ. Exact backend IDs - remain available in properties and are used for API operations. -- Full names such as ChunkDB and Access Server belong in type properties, - deployment choices, or help text. A tree label must not mix a verbose name, - a separator, and another backend identifier. -- **VIS-03:** All integer IDs and offsets preserve their exact values; large - identifiers do not pass through lossy JavaScript number conversion. -- **VIS-04:** Selected resources use a consistent outline/highlight. Health, - lifecycle, readiness, and data protection are separate labelled values. -- Capacity bitmap: muted blue means used, muted green means free. Unknown, - unavailable, and reserved states have distinct labels/patterns; they are never - rendered as free. - -## 5. Cluster - -- **CLU-01:** The hierarchy is Datacenter → Rack → Node → service instances. The center - topology uses physical identities and shares selection with the left tree. - The left tree separates each Node into Services and Storage. Physical groups - and disks belong under Storage regardless of their DiskDB owner; service - instances remain under Services. Logical Paxos groups are managed in KV. - A DiskDB instance also expands to references to the DiskGroups it currently - owns. These use authoritative instance ownership, not physical Node membership. - They open the same resource as Storage references and update on ownership changes; - unknown ownership never assigns a group to an arbitrary instance. - Candidate nodes forms another Datacenter branch, alongside Racks, consistent - with disk candidates under Storage. Only unjoined candidates for this cluster - appear there; confirmed members appear only in their Rack. Raw discovery logs - are not rendered in the tree. Legacy Capacity links resolve to Cluster. - DiskGroup ownership references use the compact `DG-` label directly; - do not prefix them with `Managed storage`. The candidate branch is labelled - `Candidate nodes`; while its authoritative list is loading it may expand to - show the loading state, but once the list is known to be empty it has no - expand arrow and renders no `No candidate nodes` placeholder. Candidate - rows appear only when there are eligible unjoined nodes. Storage candidate - rows may show `Candidates (N)` when the authoritative count is known, - including zero, but never `Candidates (?)`. - Capacity properties use the same hardware-inventory and runtime-usage scope - at Storage/DiskGroup, Node, Rack and Datacenter levels. A complete known - aggregate shows capacity, used and free values; an incomplete observation - remains explicitly unknown and is never rendered as zero. - Add Rack and Add Store belong to the respective Datacenter context menu; - uninitialized KV exposes Initialize Cluster there. -- **CLU-02:** Rack actions include Add Node. A Node context menu independently - manages each service type and each existing instance, using the same lifecycle - pattern as KV/DiskDB. It must not route other service types through KV APIs. -- **CLU-03:** Standard OCI Add Node opens the existing candidate flow with its - Rack preselected. Verify and review authenticates SSH and reads actual disk - inventory without admitting the node or initializing storage. Join and start - submits the reviewed choices and accepted preflight identity. Existing accepted - or pending-review Rack choices take precedence over a new default. -- Defaults select Paxos-KV, DiskDB, DiskIO, ChunkDB, Chunk-KV and Access Server. - Each service and its listeners can be disabled. Access has independent Iceberg, - S3, Dataset and Health ports; DiskDB's related RPC listeners must not overlap. Node - admission owns the fixed KV bootstrap listeners. Canonical service type is - `paxos-kv`; a legacy `kv` wire value is rejected. -- With no supplied input, standard setup chooses a 16-GiB preallocated file and - 1-GiB host reserve. Sparse allocation is explicit. Supplied media retain exact - observed identities and capacities; each raw device requires initialization - acknowledgement. Diskless setup creates no disk. Invalid supplied choices - never fall back to a managed file. -- The target monitor durably accepts storage and service intent before reporting - acceptance. Progress continues after the dialog, Web connection or browser - closes; restart resumes the same intent. Node progress polls every two seconds - and retains last confirmed rows on observation failure. A completed setup - receipt is separate from current service or I/O health. -- During creation, visible topology and service health refresh independently - every second; a slow capacity request must not stop service observations. - Opened Storage branches refresh without collapsing or reopening them. - Returning to the browser immediately observes current state without F5. - Polls preserve selection, expansion and the topology viewport. -- Storage waits for Group 0 and an ordinary metadata binding. Dependent services - wait for real prerequisites. A failed disk/service retains successful siblings; - explicit retry uses their original operations and fixed placement. Later - operator Stop/Restart/Delete commands take precedence over setup replay. - Consecutive failed startup/readiness attempts have a durable five-attempt - budget per service operation; successful readiness clears that budget. -- Legacy standalone Add Node retains its one-dialog registration and saved - six-service plan. Its session reconciler saves revision-checked progress before - deployment and resumes waiting steps while that Console session is active. - Unavailable process observations and failed steps continue read-only observation - every two seconds, as defined in §13. A fresh, identity-matched running - observation reconciles interrupted progress to deployed. Explicit Retry is - required to initiate another failed Start, Restart, or Deploy mutation. It never - duplicates a registered Node or successful service. This session-owned plan is - distinct from the standard target monitor's accepted node setup. -- Internal service health probes the RPC endpoint independently for each - service. Startup records lacking RPC endpoints use process observation. - Before its first store exists, PKV reserves but does not bind its RPC port; - process liveness covers that startup phase. Once a store exists, a failed - RPC probe remains Failed even when the process is alive. - Access health probes its dedicated listener; missing retained health - configuration is Unknown and requires reconciliation before restart. - Current service observations take precedence over persisted node projections. - DiskDB instances are associated by complete advertised RPC addresses; equal - ports in different node namespaces do not imply the same instance. -- Cluster selections never display or request Chunk ownership. -- **CLU-04:** Cluster readiness is reported by capability: KV quorum, storage, - Chunk-KV, and Access protocols. A PID or registry entry alone is not Ready. - Missing prerequisites link to the domain where they can be resolved. Remote - Access readiness uses the committed dedicated health listener on the owning - node; missing configuration remains Unknown and failed probes are Down. - -- **CLU-05:** The cluster overview and Datacenter/Rack/Node scopes show topology. - Datacenter/Rack/Node capacity appears below it. Storage, DiskGroup, Disk and - Zone scopes hide topology and release its space for storage content/capacity, - including when entered through a DiskDB ownership reference. Returning to a - higher scope restores topology. Service selections and an unselected cluster - do not show capacity. Node details and - configuration editing belong in the properties panel. Storage selection exposes - its disk operations without a second node summary. There are no Topology/Capacity - tabs, diagram toggle, or node-card toggle. -- **CLU-06:** Auto join starts enabled for a new standard workspace and retains - an explicit target. Three authenticated eligible nodes are ordered by UUID; - the smallest UUID initiates Store 0 / Group 0 with those fixed three voters. - Fewer nodes wait. Later admission is nonvoting and never expands/replaces - Group-0 voters. Explicit single mode auto-starts the same initialization - lifecycle with one local voter. Turning Auto join off stops new triggers; - accepted initialization continues and conflicting intents require recovery. - Eligible unjoined candidates default to Auto join, including later admissions - into a bound cluster. Each candidate can retain a durable explicit override. - Enabled candidates join the default Rack; later node editing can change its - Rack. Disabling one candidate stops its new admission triggers without changing - another candidate's policy or cancelling accepted work. Initialization - progress and failure recovery appear in Datacenter properties; completed - initialization is not a persistent banner. -- **CLU-07:** Each candidate is one ordinary selectable tree row labelled - `node-`. An Auto icon, accessible label and tooltip express the effective - policy; unloaded policy is Unknown, never an unchecked/disabled substitute. - Full identity, addresses, discovery/admission state, errors and policy appear in - properties. Enable/Disable Auto join is available in its context menu and - properties, backed by the same state. Manual admission, cancellation and - discarding a prepared review appear only when applicable, in that node's menu. - There are no row checkboxes, inline action stacks, raw discovery logs or - general-purpose Update node / Clean up Group 0 controls in the candidate tree. - -Cluster: Datacenter/Rack/Node scope (CLU-01/05): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v datacenter * | datacenter | Properties | -| v R-1 | Topology | Datacenter | -| > N-1 | [datacenter] | Name datacenter | -| > N-2 | : | Health observed | -| > N-3 | [R-1] | ... | -| > Candidate nodes | / : \ | | -| | [N-1] [N-2] [N-3] | Initialization progress | -| | ---------------------------------------------------- | (only when relevant) | -| | Capacity [Refresh] | | -| | Capacity / Allocated / Free / Unknown coverage | | -| | Resource Capacity Usage | | -| | R-1 ... ... | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -Nodes / services / opened Storage: live updates; returning to browser -> refresh -``` - -Cluster: candidate selection and effective Auto policy (CLU-06/07): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v datacenter | Cluster overview | Properties | -| > R-1 | Topology remains the primary overview. | node-a1b2 | -| v Candidate nodes | | Discovery ID | -| node-a1b2 [Auto] * | Candidate details are in Properties. | Addresses | -| node-c3d4 | No inline candidate cards or operation log. | State Eligible | -| | | Auto join Enabled [v] | -| | | Target Rack Default Rack | -| | | Admission Waiting | -| | | Error None | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Candidate menu and DiskDB ownership references: - -```text -Right click node-a1b2 Ownership reference Physical reference -+---------------------------+ N-1 N-2 -| Disable Auto join | Services Storage -| Join node... | DDB-1 DG-7 -| Cancel admission | DG-7 Disk-a -| Discard access review | Disk-a -+---------------------------+ \ / - same DG / Disk identity -Joined candidate -> default Rack -> normal Node tree item -Explicitly disabled candidate -> no new automatic admission -``` - -Candidate menu entries depend on admission state. A DiskDB ownership reference -can point to disks on another physical Node. Selecting its DG hides topology -just like selecting that DG below Storage. - -## 6. Properties and cross-links - -- **PROP-01:** Right-side properties describe the exact selected Rack, Node, service, - Store, Group, Replica, DiskGroup, Disk, Zone, or allocation block. -- Show exact ID, type, parent/owner, source, observation time, and relevant - lifecycle/health fields. Missing information is Unknown or Unavailable. -- Properties provide copyable full identifiers and exact byte/unit values. - Abbreviating diagram labels must not destroy the original value. -- **PROP-02:** Resource links navigate by exact identity: service → Node, - Replica → Node, Disk → DiskGroup, Zone → Disk, and DiskGroup → owner/binding. Failed optional - lookups do not erase a successfully loaded primary resource. -- **PROP-03:** Raw JSON may be a diagnostic affordance. It is not the default - inspector for supported structured resources. - -## 7. Modes and embedding - -- **MODE-01:** Standalone mode exposes administrator topology, service, storage, - logical KV, and data operations supported by the backend. -- **MODE-02:** Standard manual OCI nodes expose authenticated admission and - confirmed Group-0 topology/service/storage operations. Explicit single-profile - preview mode uses the same seven-domain UI for observation and supported logical/ - data operations, but rejects physical topology/deployment/storage mutations. - Backend enforcement accompanies hidden/disabled controls in either mode. -- **MODE-03:** Readonly embedding disallows mutations. Every fetch uses the - configured API prefix. Styles remain scoped to the Console; embedding does - not change resource identities or connect to another cluster. -- Embedding keeps `apiPrefix`, `basePath`, `readonly`, `modules`, `initialDomain`, - and `onEvent`. Domain selection has no Swagger or per-node connection mode. - -## 8. Bounded observation - -- **DATA-01:** API pagination is authoritative. Continuations are opaque and - scoped to the query/source/generation. The browser does not invent offsets - or fetch all pages to provide a count, badge, filter, or layout. -- Defaults: Capacity zone pages contain 32 zones. Every other collection declares its page size at its boundary and - exposes bounded continuation instead of an unbounded initial expansion. -- **DATA-02:** Prev/Next replace a window. Type/scope/query changes clear cursor - history. End-of-list is explicit; no exact total is shown unless supplied by - an authoritative bounded API. Selection remains stable across page changes. -- **DATA-03:** Label local filtering as applying to the loaded window. It must - not launch a hidden cluster scan or imply that an empty local result means - no matching resource exists globally. -- **DATA-04:** Cancel or ignore obsolete responses after scope/selection changes. - A late response cannot overwrite the current selection. Polling cannot stack - overlapping requests for the same resource or grow retry work indefinitely. -- Automatic Access deployment checks live DiskIO ownership before provisioning - credentials or catalog state, including when an existing Chunk-KV process is - available. A transient ownership gap keeps the plan waiting and resumes - without another Submit when the prerequisite is confirmed. -- Active views refresh at a bounded cadence; hidden views suspend observation - polling. Accepted deployment work is independent of observation polling. -- **DATA-05:** Preserve partial successes, identify failed sources, and show - observed/scanned/matched counts separately when they differ. Ownership or - generation changes invalidate incompatible continuations and layouts. - -## 9. Creation and mutation dialogs - -- **FORM-01:** Every Create/Deploy dialog opens with usable, nonconflicting - defaults for IDs, listener ports, and available parent/dependency references. - With prerequisites present, default OK succeeds without manual editing. -- Defaults are fetched/revalidated at submission, across all service types on - the relevant host. Include internal listener ranges, such as DiskDB's ports. - A user edit is not overwritten by a late default response. -- **FORM-02:** Missing prerequisites disable the unsupported operation and name - the prerequisite. Do not prefill a plausible value that inevitably fails. - Real device paths are explicit; never infer a physical device to overwrite. -- **FORM-03:** Submission prevents duplicates. Field validation and backend - errors stay in the dialog or center editor with entered values intact. Toasts are supplementary, - not the only evidence of success or failure. -- **FORM-04:** Show the current operation and elapsed/waiting state for a slow - request. A timeout means outcome unknown until reconciled, not automatically - failed. Do not retry a potentially completed creation blindly. -- Successful creation selects or reveals the resource after authoritative - refresh. Partial creation shows exactly what exists and what remains. -- Cluster reset fences new plan/deployment writes, waits at most 30 seconds - for accepted work, then stops registered children before deleting workspaces. - If accepted work remains pending, reset returns a conflict and removes nothing. - Cleanup operations use the reset's exclusive context; another browser cannot - start a deployment during cleanup. -- **FORM-05:** Delete names the exact target and effects. Deleting a service, - removing configuration, and deleting stored data are distinct operations. - Reset lists its affected scope and stops queued deployment work before teardown. - -Shared dialogs and operation feedback (FORM-01–05, ACT-05): - -```text -Parent menu: Create Dataset... Resource menu: Drop table... -+----------------------------------+ +---------------------------------------+ -| Create Dataset [X] | | Drop table [X] | -| Namespace: dataset-ns | | Target: analytics.events | -| Name [.........................] | | Remove this Catalog reference. | -| | | Physical reclamation follows GC. | -| | | | -| Validation / request error | | Exact scope and blockers appear here. | -| [Cancel] [Create] | | [Cancel] [Drop table] | -+----------------------------------+ +---------------------------------------+ - -Dataset: center edits -> [Submit Snapshot] -> upload / prepare / conditional head -Operation feedback: exact target / operation ID / waiting / progress / error -Unknown outcome: reconcile the same operation before retrying -``` - -Forms open only after a menu action. Progress follows the actual operation; -simple creation need not acquire publication stages. Closing a dialog does not -cancel accepted work. - -## 10. Accessibility - -- All controls have semantic roles and unambiguous labels. Keyboard operation, - focus trapping, Escape/Close behavior, and focus restoration work in dialogs. -- Focus/selection/error indicators remain visible on the dark palette. Text - contrast targets WCAG AA; color-coded states include text or a legend. -- Large diagrams retain accessible summaries and selected-object properties. - Errors and asynchronous progress are announced without stealing focus. - -## 11. Verification contract - -- **TEST-01:** Prepare and verify backend prerequisites through APIs before - visual acceptance. Browser E2E uses real services, persisted metadata and file - bytes. Response mocks, interception and HAR replay are forbidden. Pure unit - tests remain separate and cannot satisfy end-to-end acceptance. -- Every acceptance result records: contract/scenario ID, fixture, actions, - visible result, relevant bounded API result, pass/fail, and concrete defect. -- Use role/label/test-id selectors. Prefer a short scripted flow to repeated - full-page snapshots. Capture one useful screenshot per completed scenario or - failure; do not repeatedly inspect unchanged trees. -- Test mutation results from durable API state and resulting UI, not a toast. - Poll readiness/leadership with a deadline; no arbitrary sleeps or hidden retry. -- Reuse the populated cluster for read-only cases. Isolate destructive cases - and avoid resetting the whole environment for each tab. -- Scope evidence honestly: rendering passed, API passed, and end-to-end passed - are distinct. Backend failures remain failures even if the UI handles them. - -## 12. KV - -- **KV-01:** Left tree is Datacenter → Store → Group → Replica. Store creation - belongs to the Datacenter context menu. The center is the bounded data browser; - exact Store/Group IDs, health, leader and replica membership remain in properties. -- Before initialization, explain the dependency on deployed KV servers and offer - Initialize Group 0. The three-node fixture selects all three eligible nodes. -- **KV-02:** Store/group/replica creation, membership operations, and deletion - use the selected scope and valid defaults. Group 0 is visibly identified as - the system group; ordinary groups are not confused with it. -- **KV-03:** Selecting a Group directly opens its data window; there is no - Overview/Data toggle. Membership and node placement remain in properties. - Unknown leader is not rendered as zero. The right inspector does not poll or - render generic internal metrics. -- **KV-04:** Read operations target an explicit Store/Group. Query and Scan - show encoding and exact key/value interpretation. Scan is bounded and paged; - empty values, missing keys, and request failures are distinguishable. -- A selected replica or group may expose supported diagnostics; a generic - physical Cluster diagram does not replace the logical KV workbench. -- **KV-05:** A data window contains at most 20 entries. Previous/Next replace - rows; each Group has an independent cursor when browsing a Store. Clicking - a row shows the full Key and Value. Printable UTF-8 runs remain text; only - undisplayable characters and malformed bytes become uppercase hexadecimal - runs without `0x`, distinguished by warm gold text, a dark background and border. - Display conversion cannot change bytes sent in a mutation or continuation. -- **KV-06:** The KV data browser is read-only: Query performs an exact Get and - Scan reads a bounded window. There are no Put/Delete/demo injection controls. - Topology and membership operations remain in resource context menus and use - management APIs. A uniform menu style does not add data mutation capabilities. - -PaxosKV: Group data inspection (KV-01–06): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v datacenter | PaxosKV | Properties | -| v S-0 | Store [S-0 v] Group [G-1 v] | Group G-1 / selected Key | -| > G-0 | Prefix [........................] [Scan] | Store S-0 | -| v G-1 * | Exact key [.....................] [Query] | Leader LR-2 | -| LR-1 Follower | ---------------------------------------------------- | Members LR-1,2,3 | -| LR-2 Leader | Key Value | Key | -| LR-3 Follower | key-a | Value | -| | key-b | Encoding UTF-8 / hex | -| | ... | | -| | [First] [Previous] [Next] | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Create Store belongs to Datacenter; Group/Replica management belongs to its -resource menu. Data browsing is read-only; Query/Scan remain center query tools. - -## 13. Service lifecycle - -- **SVC-01:** Each instance has Start when stopped, Stop when running, Restart, - and the supported remove operation. Menus target exact type and instance ID. -- **SVC-02:** Preserve desired configuration independently of PID. A restart - retains endpoints, workspace, and data; refreshed properties show the new PID - and separately observed readiness. Process identity is verified before signal. - Lifecycle acceptance first checks the submitted Start/Restart response, then - observes the new PID and process. An in-flight startup is not a completed - operation and must not trigger a duplicate cleanup restart. -- In the legacy session-owned deployment flow, page unload stops its runner. - Standard accepted node setup runs on the target monitor independently of the - browser, as defined in CLU-03. Cancellation of an old page's - observation or progress request cannot publish a service failure. The next - page reloads the persisted plan and reconciles interrupted deployments; - restoring a cached history page also reloads before resuming its runner. -- **SVC-03:** Startup observes dependencies: Group 0 before dependent metadata - services; valid disks/ownership for storage; journal/data prerequisites before - Chunk-KV bootstrap; Chunk-KV catalog before Access startup. -- Plans waiting for a registered DiskIO restart or authoritative storage readiness - observe dependencies every two seconds, including when Capacity data has not - been loaded. A live PID does not establish disk-group ownership. The ten-second - missing-disk backoff does not apply to an outstanding storage-readiness probe or - a deployment rejected because its storage prerequisite changed. -- Access provisioning durably records catalog initialization and activation - request identities before issuing either mutation. Interrupted deployment - reconciles committed state and resumes the same request. An existing catalog - retains its identity and policy; restart does not reset operator changes. - Missing or conflicting recorded identity fails explicitly. -- Service failures name the failing step and permit a safe retry. Existing - instances are not silently duplicated or treated as healthy merely by presence. -- A registered service whose current process observation is unavailable remains - waiting while the session observes it every two seconds. Missing, expired, - rejected, or failed observations do not prove that the service stopped. The - plan retains its registration and successful siblings without sending another - Start, Restart, or Deploy request. A fresh, identity-matched running observation - reconciles the step to deployed. Endpoint health remains independently reported - and never supplies a missing PID or proves process absence. -- A fresh, identity-matched observation that confirms a registered service is - stopped marks the step failed when no accepted startup is pending. Failed steps - continue read-only observation every two seconds, including when no other - deployment is queued. A fresh, identity-matched running process automatically - reconciles that registered service's progress to deployed. This also reconciles - an interrupted deployment whose owned process is later confirmed running. -- A plan whose selected steps are all deployed continues read-only process - observation every two seconds. A short-lived running process cannot freeze - progress after it exits. Confirmed stopped diagnostics retain the service name - and configured listener overrides; successful siblings and overrides persist. -- Read-only recovery never sends Start, Restart, or Deploy again. Missing or - unverified process evidence does not clear a recorded failure. Explicit Retry - is required to initiate another failed lifecycle mutation; an accepted startup - retains its existing operation and launch budget. Error observation does not - use the missing-Capacity backoff. -- An unavailable service-inventory read preserves successful steps, recorded - failures, and fixed-slot warnings. Unattempted queued steps keep waiting for - current inventory instead of becoming failed lifecycle operations. -- Operational type descriptions remain available even though instance labels - use the compact convention in §4. - -Session-owned Node service plan with unavailable process observation -(relevant progress rows and existing controls): - -```text -+--------------------------------------------------------------------+ -| Node 1 services | -| | -| crowdb-access-server deployed | -| crowdb-chunk-kv deployed | -| crowdb-chunk-db deployed | -| crowdb-disk-db deployed | -| crowdb-disk-io deployed | -| crowdb-paxos-kv queued · waiting | -| Waiting: current process observation is unavailable | -| | -| [Cancel] [Done] | -+--------------------------------------------------------------------+ -``` - -Existing dialog guidance remains unchanged. Done closes progress; observation -continues while this Console page is active. Confirmed stopped with no pending -startup becomes failed and continues observation. A later fresh running process -reconciles progress to deployed automatically without a lifecycle request. The -existing Retry failed services action initiates a needed lifecycle mutation. - -A previously completed plan after a confirmed stopped process: - -```text -+--------------------------------------------------------------------+ -| Node 1 services | -| | -| crowdb-access-server disabled | -| crowdb-chunk-kv disabled | -| crowdb-chunk-db disabled | -| crowdb-disk-db failed | -| crowdb-disk-db (rpc_port=7100): Registered service is stopped | -| crowdb-disk-io disabled | -| crowdb-paxos-kv deployed | -| | -| [Cancel] [Retry failed services] | -+--------------------------------------------------------------------+ -``` - -## 14. DiskGroup and disk management - -- **DISK-01:** The physical tree ends at Disk: Rack → Node → Storage → DiskGroup - → Disk. Zones and allocation bitmaps remain in the center workspace. - DiskGroup creation establishes its authoritative registration, data binding, - and owner, or reports which prerequisite/step prevents completion. -- The dialog does not spin indefinitely when Group 0, a data group, or DiskDB - is unavailable. It reports the actual failure/unknown outcome and reconciles - whether a resource exists before offering retry. DiskGroup creation returns - within eight seconds. An unknown outcome keeps the ID fenced until accepted - work finishes; reopening the same ID/name reconciles registration, binding - and owner instead of allocating another group. Binding can exist before a - DiskDB service is registered; this partial state names the missing owner - prerequisite explicitly. -- **DISK-02:** Node storage separates registered disks, operation progress and - candidates. Create File Disk seals a generated path, capacity, existing group, - reserve and allocation mode. Accepted preparation/admission continues on the - monitor. Pending/failed outcomes do not count as usable capacity. -- Join Candidates selects at most 16 eligible supplied media, each with an - existing compatible bound DiskGroup and media class. File/device backend and - HDD/SSD class are separate. Unverified identity, changed observations, missing - mappings and insufficient capacity disable admission and explain why. - General initialization consent and per-raw-device acknowledgement are explicit. -- Bulk Join submits at most four independent operations concurrently. Each row - retains its operation and exact reviewed medium across response loss/retry; - successful siblings are not resubmitted. Closing stops unsent submissions, - while accepted work continues. Durable history pages contain at most 16 rows; - the Node workspace retains at most 64 observations and reports partial coverage. - Nonzero usable bytes fail without wiping or publishing disk membership. -- Legacy metadata-only disk editors retain geometry and uniqueness checks; they - are separate from verified physical provisioning. -- Standard-node group menus open Create File Disk, while disk lifecycle links - open Node storage. Direct legacy Up/Down/Delete controls are unavailable in - that mode. Maintenance diagnostics remain available. A disk row with no observed - provisioning receipt reports Unknown media rather than inferring a class from - compatibility geometry. -- **DISK-03:** Distinguish physical location, DiskDB owner instance, and metadata - Store/Group binding. Neither Node ID nor service ID implies ownership. -- Owner/status/binding operations are available only when supported and expose - the resulting authoritative state. Container restrictions apply to all entry - points, including context menus and direct API requests. - -- **DISK-05:** Offline applies exclusion to the current owner and every configured - DiskIO participant, then waits for I/O quiescence. Remove requires authoritative - Offline and the exact revision/owner evidence. A lost reply is reconciled with - Group 0 before showing success. Removed media appear as retained candidates. - Rejoin restores their original identity and allocation metadata Offline, then - explicit Online validates the retained format and current descriptor. No - recovery migration, evacuation or physical file deletion is implied. Missing - or replaced registered media remain quarantined; discovery creates no substitute. -- A retained receipt preserves its parent even after Remove. DiskGroup deletion - requires no retained media, owner or metadata binding; empty membership alone - does not permit a cascade. Confirmation explains the blocking prerequisites. - -Storage scope: inventory without topology (DISK-01/02, CLU-05): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v datacenter | Storage - N-1 [Refresh] | Properties | -| v R-1 | Capacity / Allocated / Free / Unknown coverage | Storage / selected DG | -| v N-1 | ---------------------------------------------------- | Physical Node N-1 | -| > Services | DiskGroup Owner Binding Status | DiskDB owner DDB-1 | -| v Storage * | DG-7 DDB-1 S-0 / G-1 Online | Binding S-0 / G-1 | -| > DG-7 | ... | Observed | -| > Disk candidates | [Previous] [Next] | | -| | | | -| | No topology canvas or reserved canvas space. | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -DiskGroup: Disk icon and ID enter the same Disk (NAV-03, DISK-03): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v N-1 | DG-7 [Refresh] | Properties | -| v Storage | Capacity / Allocated / Free | DiskGroup DG-7 | -| v DG-7 * | ---------------------------------------------------- | Owner DDB-1 | -| Disk-a | Disks | Binding S-0 / G-1 | -| Disk-b | +------------+ +------------+ | Physical Node N-1 | -| | : Disk icon : : Disk icon : | Status Online | -| | : usage : : usage : | | -| | +------------+ +------------+ | | -| | Disk-a Disk-b | | -| | Online Offline | | -| | | | -| | Click icon or ID -> that Disk workspace | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -The same view opens through DDB-1 or Storage. Supported storage lifecycle -operations remain in resource menus with the mode restrictions in §14. - -## 15. Capacity - -- **CAP-01:** Datacenter/Rack/Node/Storage/DiskGroup/Disk selections show relevant - capacity, used/free allocation and disks. Observation time belongs with the - affected detail, not in a global strip. Missing statistics are not - zero and must not produce an all-free disk map. -- **CAP-ALLOCATION:** Used/busy capacity denotes allocated physical blocks, including - system chunks, reserved Chunk capacity and blocks awaiting compaction. It is not - user payload size or host filesystem usage. Capacity tooltips and Zone bitmap - accounting explain this distinction; the displayed bitmap window reports its own - used/free count separately from the whole Zone. -- **CAP-02:** Disk detail retains its summary and paged zones. Default 32 zones, - Prev/Next, current visible range, and optional cheap indexed zone lookup. - Avoid full-zone scans for filtering or drawing a large disk. -- **CAP-03:** Selecting a Zone renders its details below the disk view. Keep the - parent disk and zone list in place; no redundant Back to parent Disk link. - If a dedicated zone view is used, provide an actual disk breadcrumb instead. -- **CAP-04:** Zone bitmap uses muted blue for used and muted green for free, - with a legend, used/free counts, allocation unit, and zone geometry. Bit index - maps to the correct block offset according to the protocol's decode order. - Bitmap counts and usage identify their scope, observation and denominator. - Partial windows or aggregation cannot imply that the whole Zone is full. - Comparable allocation counts reconcile with usage; differing scopes or times - are explicit. Allocation, reserved/preallocated space, logical data bytes and - backing-file usage are distinct measurements. Initial system allocations are - attributable to their actual sources, not described as user data by inference. -- Hover/selection provides exact block index and offset. Large bitmaps use - bounded drawing/tiling or labelled aggregation; an aggregate is not one block. -- **CAP-05:** Refresh bitmap, scan/recalc, compact/rebuild, and status operations - identify their scope and show progress/results. RPC or owner lookup failure - remains visible at the affected zone and never becomes an empty/free bitmap. - -- **CAP-06:** Opening Cluster does not issue DiskDB runtime usage or scan - requests. Capacity usage is scoped to the selected DiskGroup/Disk when that - scope is known. Selecting Datacenter, Rack, Node or Storage requests the runtime summary - needed for that scope automatically. DG and Disk requests retain their narrower - scope. Inventory capacity remains available independently; Busy/Free are known - only when exact node/DG/disk identities and geometry cover the selected inventory. - Moving from Disk to an upper scope never reuses the Disk report as a full summary. - Cached candidate/setup summaries are bounded to 64 nodes and never initiate - node-by-node discovery. - Disk inventory belongs to one shared source. Expanding a Node's Storage branch loads its - DiskGroups; expanding/selecting a DiskGroup loads its disks. Selecting a - linked Disk loads the required ancestor inventory directly. Four workers - serve opened branches, merging duplicate requests for the same branch. - Refresh revisits requested branches and never traverses unopened disks. - Failed branch refresh retains its known inventory and reports failure. -- **CAP-07:** The physical tree reads Node health from the service projection; - it does not ping every Node or fetch a second per-node KV catalog. Capacity - Storage branches and DiskGroups begin collapsed; Datacenter/Racks/Nodes and - Services remain visible. - Default-service prerequisite inspection is explicit for its selected Node. - -Disk and inline Zone bitmap (CAP-02–05): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v N-1 | Disk-a [Refresh] | Properties | -| v Storage | Capacity / Allocated / Free | Selected allocation block | -| v DG-7 | Zones: current bounded window | Disk Disk-a | -| Disk-a * | Zone Allocated Free Usage | Zone zone-1 | -| Disk-b | zone-0 ... ... ... | Index 17 | -| | zone-1 * ... ... ... | State Free | -| Tree stops at Disk. | [Previous zones] [Next] | Unit | -| | ---------------------------------------------------- | Zone offset | -| | zone-1: block window, not the entire Zone | Disk offset | -| | BBBBGGGGGG?? B=used/blue G=free/green ?=unknown | Observed | -| | BBBBGGGGGG?? Selected block: 17 | | -| | Window counts / Zone totals / unit / geometry | | -| | [Previous blocks] [Next] [Go to block] | | -| | | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Letters represent the bitmap colors. Rendering uses actual protocol bit order -and allocation geometry. Unknown/reserved data never becomes free. Free blocks -in another window are not visible in this one. - -## 16. Loading, failures, and recovery - -- **ERR-01:** Distinguish initial loading, refreshing existing data, empty, - partial, unavailable, and failed. Keep usable previous data visibly stale - while refreshing; do not flash an empty page on every poll. -- **ERR-02:** Display a concise cause, failed dependency/operation, and retry - action. Technical endpoint/HTTP details may be expanded for diagnosis. -- **ERR-03:** A long-running or timed-out mutation preserves its target and - inputs. Reconcile its result before retrying. Closing a modal does not prove - server cancellation; UI text must not imply otherwise. -- **ERR-04:** Failure of detail/placement/one child branch does not erase the - parent resource or other successful results. Unauthorized/unsupported actions - are explicit and do not fall back to another protocol or cluster. -- Polling and automatic dependency checks have bounded cadence and fan-out. - Identical failures do not flood notifications; explicit retry remains usable. - -Collection and refresh states (ERR-01–04): - -```text -Initial loading Confirmed empty collection -+--------------------------+ +--------------------------------------------+ -| Loading namespaces... | | No namespaces. | -| No invented zero totals. | | Right-click Catalog to create a namespace. | -+--------------------------+ +--------------------------------------------+ - -Refresh failed Mutation outcome unknown -+--------------------------+ +--------------------------------------------+ -| Last observed rows | | Target: exact resource | -| Stale - refresh failed | | Operation: accepted ID | -| Cause / dependency | | Outcome unknown - reconcile before retry | -| [Retry] | | Inputs / last confirmed progress retained | -+--------------------------+ +--------------------------------------------+ -``` - -## 17. Fixed-cluster operator flow - -1. Open Cluster directly. Inspect/create Rack and Nodes; add each Node through - one dialog with six default services and explicit waiting dependencies. -2. In KV, initialize three-replica Group 0 and create ordinary data groups. -3. In Cluster, select Storage to register DiskGroups/disks with owners and bindings; - DiskIO becomes ready. Enter Disk/Zone to inspect capacity and its bitmap. -4. Select a Datacenter/Rack/Node in Cluster to inspect topology and service readiness. - Resolve logical group dependencies in PaxosKV and physical storage in Cluster. - -This is a dependency guide, not a blocking wizard. A working capability remains -usable while another is unavailable. No domain asks the operator to reconnect. -Data-browser flows are covered by their domain sections below. - -## 18. Acceptance scenarios and scope - -Each scenario uses the contracts above; failures are recorded in the working -plan, never edited out of this specification to make a run pass. - -- **A01 / entry:** open the fixed cluster; all seven tabs appear; no connection - or access-key form; restoring a domain does not change cluster identity. -- **A02 / create:** with one Rack, create three Nodes using valid defaults; - verify one instance of every requested type per Node, conflict-free ports, - accepted operation progress, and safe retry after one injected service failure. - Closing a standard setup dialog does not stop accepted monitor work. -- **A03 / lifecycle:** independently stop/start/restart an auxiliary instance - from its Node menu; verify exact target, preserved data/configuration, updated - PID, and readiness. All instance labels follow §4. -- **A04 / KV:** verify Group 0 and ordinary Group 1 with three replicas; select - a group, inspect membership, and perform bounded exact/scan data operations. - Add members to the same group sequentially, refreshing membership after - each result; independent group preparation may run concurrently. -- **A05 / Capacity:** create/register DiskGroup and disk, inspect owner/binding; - select a disk with more than 32 zones, page, select Zone, and verify inline - bitmap, decode offsets, muted colors, and error handling for unavailable owner. -- **A10 / limits:** large topology, group/replica lists, disk/zone lists, and KV - scans remain bounded. Observe request counts/bytes and rendered item counts; - no hidden fetch-all, invented totals, or selection replacement on page changes. -- **A11 / failures:** inject unavailable owner/group, malformed API response, stale cursor, - and slow/unknown mutation outcome; verify §16 and retry without duplication. -- **A12 / modes:** Single preview rejects physical topology/disk mutations through UI and API; - supported admin data operations remain usable. Readonly embedding forbids - mutations and respects API/style isolation. - -- **A06 / Chunk:** default real list, 10-entry replacement windows, exact ID, - multiple Mirror/EC Strips, selected block properties and Disk/Node round trips. -- **A07 / Chunk-KV:** registered servers including empty owners, bounded graph, - collapse, exact split identity, selected Tree/Journal and stale continuations. -- **A08 / Iceberg:** Catalog to Parquet footer through a real committed table; - nested schema, snapshots, paged manifests/files and zero data-page reads. -- **A09 / S3:** create a bucket using its prefilled name with one click; verify - bucket/object replacement pages, HEAD, bounded preview, single and batch upload - using filenames as object keys, duplicate-key replacement, download/multipart, - locations and return from Chunk to the source object. -- **A13 / resource interaction:** parent context menu creates a supported child; - selecting it reads details; its own context menu exposes supported lifecycle - operations. Create Dataset, Iceberg Table and S3 Bucket with their prefilled - default names without typing (add data for Dataset; upload Parquet for Iceberg), including ordinary HTTP container origins. Verify tree/list/property refresh, keyboard - access, readonly and unknown capability/outcome states across Iceberg, Dataset - and S3. -- **A14 / Dataset:** create/import, parent-linked Snapshots, paged Samples/Fields, - explicit verified preview, Split revisions and supported lifecycle operations. - No browser Session activity or default raw-JSON properties replace this content. - -Out of scope: UI login/role design and new data-plane protocols. -Unsupported features are explicit rather than represented as working controls. - -## 19. Chunk - -- **CHK-01:** Left tree is Datacenter → Rack → Node → CDB and KV Server; - KV branches expand Store → Group without Replicas. Chunk ownership has two - independent maps over slots 0–1023: Serving assigns a ChunkDB instance; - Storage assigns the metadata Store/Group. Chunk-KV ordered splits and payload - disk placement are separate concepts. The left tree has no Serving ownership / - Storage ownership wrapper or repeated footer. Ownership meaning belongs in - selected-resource properties and the scoped center inspection. Details are - requested only when a real ChunkDB or replica target - exists. Unprovisioned selections do not send ownership probes. -- Selecting a CDB or Group displays its full 32×32 bitmap; parent selections - display one combined bitmap per layer. Each cell is exactly one slot, ordered - left-to-right then top-to-bottom. Owner colors remain stable across scopes; - outside-scope slots are gray, unknown membership uses a pattern. Each map - carries its own generation. Invalid or unavailable maps never become empty - or gray observations; retained failed-refresh data is marked stale. -- Storage groups are deduplicated across replicas. Node/Rack scopes include a - group when it has a replica in that scope. Unknown replica/service membership - remains unknown. Legend hover highlights an owner's slots; clicking an owner - navigates to its bitmap. Legend windows contain at most 12 owners. Selecting - a cell shows exact slot, owner, generation, source and node associations; - arrow keys navigate the grid. Service and storage boundaries need not match. -- Bitmap observations read a validated complete map and return exactly 1024 - lossless owner identities per layer. No owner-page stitching is needed. -- **CHK-02:** Entering Chunk requests an All-type page automatically. The center - lists at most 10 chunks without an inner vertical scrollbar. Previous/Next - replace the page. Type is a protocol-defined selector; exact Chunk ID lookup - sits above the list in the center. Arbitrary ID-prefix/global filtering is not - required. Source failures identify partial coverage rather than an empty list. - Datacenter and narrower lists expose Type, Status, Strips, Size and Sealed. - Status uses the observed lifecycle state; Size reports Chunk capacity and Sealed - reports sealed length, both with explicit units matching the detailed metadata. - Sealed length is not a boolean, nor is Chunk capacity physical used space. - Unknown fields are explicit; valid zero lengths/counts remain visible. - List and detail refer to the same Chunk identity. - Aggregation deduplicates authoritative Chunk identities rather than counting - replicas as separate chunks. Window counts are not global totals. System and - user-data types remain distinct; a large initial count alone proves no leak. -- **CHK-03:** Selected Chunk summary and layout appear below the list. Each Strip - has a compact two-line identity: `Strip Mirror` or `Strip EC k+m`, - followed by logical interval such as `[8M, 16M)`. Stable protocol identities - remain distinct from presentation indexes. Capacity, written bytes and physical - usage are separate; missing values are unknown. -- **CHK-04:** Muted selectable block cards form each Strip. Mirror cards use - `Mirror 1`, `Mirror 2`, etc.; EC cards distinguish data and parity. Small cards - show identity/role, while properties carry full Node, DiskGroup, Disk, Zone, - offset and length. No redundant horizontal Strip bar consumes a third row. -- **CHK-05:** Selecting a block updates right properties. Disk and Node links - retain the originating chunk, page/cursor, Strip window, block and scroll. - Repair status reflects actual protection deficit, not lack of rack diversity - alone. Missing placement retains known Disk IDs rather than invented targets. - -Chunk: scope, ownership inspection and list fields (CHK-01/02): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v datacenter * | Chunk ownership - datacenter | Properties | -| v R-1 | Serving slots [32x32 map] [Refresh slots] | Selected resource / slot | -| v N-1 | Storage slots [32x32 map] [Refresh slots] | Full identity | -| CDB-1 | ---------------------------------------------------- | Owner / binding | -| v PKV-1 | Chunks Type [All v] [Refresh chunks] | Generation | -| v S-0 | Exact Chunk ID [...................] [Lookup] | Observation / coverage | -| G-1 | ID Type Status Strips Size(KiB) Sealed(KiB) | ... | -| | | | -| No ownership footers. | ... WAL Active ... ... ... | | -| | ... S3 Sealed ... ... ... | | -| | Coverage / scanned count [Previous] [Next] | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Chunk: selected Strip and block placement (CHK-03–05): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v datacenter | Chunk list stays above its selected detail. | Properties | -| v R-1 | ---------------------------------------------------- | Mirror 1 | -| v N-1 | Chunk / type / state / capacity | Node N-1 | -| CDB-1 | Strip 4 Mirror | DiskGroup DG-7 | -| | [0 MiB, 16 MiB) | Disk Disk-a [link] | -| | [Mirror 1 *] [Mirror 2] [Mirror 3] | Zone zone-1 | -| | Strip 5 EC 4+2 | Offset | -| | [16 MiB, 32 MiB) | Length | -| | [Data 1] [Data 2] [Data 3] [Data 4] [P1] [P2] | Protection | -| | | | -| | Select block -> Properties | | -| | Disk link -> Cluster/Disk; Back restores this page | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -## 20. Chunk-KV - -- **CKV-01:** Left shared Tree is Datacenter → Rack → Node → CKV Server → Split. - No extra sidebar title duplicates the root. Registered servers stay visible - even with no assigned splits. Compact split labels retain full ID and bounds - in hover/properties; five rendered splits are a window, not a global count. -- **CKV-02:** The center is a node-link diagram, not a second indented sidebar: - Chunk-KV root → CKV Server → Split → actual Tree ID. Its visible window is - at most eight servers and five splits per server. Server cards collapse their - children. Graph paging is separate from authoritative catalog pagination. - Do not add a loaded-ID filter or an ambiguous All loaded servers control. -- **CKV-03:** Selecting a Split opens its scoped Tree/Journal information below - the graph. Properties retain complete identities, bounds, epoch, owner/source, - checkpoint and selected journal extent. Root/child KV Pages must come from a - real bounded page-inspection API; journal fences are not KV Pages. Byte keys - default to hex with an optional validated text interpretation. -- Returning from Chunk inspection restores the selected Split, Tree/Journal tab, - catalog generation/window, graph server/split windows, collapsed branches and - selected journal extent. History stores identities and cursors, not runtime - payloads. Return revalidates the catalog and stream generations; stale state - stays explicit until Refresh catalog/runtime starts a new observation. -- **CKV-04:** Tree and journal observations have bounded replacement pages. - Continuation pins catalog/stream generation. Stale generation requires refresh - from the first page; parent/child recovery dependencies retain separate stream - offsets. Placement and split lineage are separate fields. -- **CKV-05:** Page inspection follows actual inner-child indices from the root, - with at most 32 child steps, one 1 MiB base frame and 20 entries per window. - It may load cold structural pages, but never flushes, checkpoints, scans live - KV records or follows overflow values. Physical base records, inline deltas - and the count of external delta pages remain explicit; pending writes are not - presented as part of this base-page view. Keys and inline values have bounded - 256-byte previews with exact lengths and truncation markers. Version fences - protect paths; page fingerprints additionally protect entry continuations. - Changed observations require an explicit root refresh. History retains path, - fences, offset, selected entry and byte format, then revalidates on return. - -- **CKV-06:** Split properties, Overview and graph cards do not display Weight. - Full Range bounds remain visible and wrap independently. -- **CKV-07:** Weighted balance explanations and data/count contribution values - remain hidden until tree statistics can describe current range data reliably. - No tree statistics scan is triggered by opening or refreshing the console. -- **CKV-08:** Count-based placement remains a backend policy. Existing bounded - observations retain catalog/epoch freshness fences, but are not presented as - data weights. Browser assertions verify that Weight and weighted explanations - are absent while tree/page inspection and range navigation remain functional. - -ChunkKV: distribution graph and selected Split (CKV-01–08): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v datacenter | ChunkKV range distribution [Refresh catalog] | Properties | -| v R-1 | Catalog generation / page / loaded scope | Split | -| v N-1 | [ChunkKV] | Range [start,end) | -| v CKV-1 | / \ | Owner CKV-1 | -| split-a * | [CKV-1] [CKV-2] | Epoch | -| v N-2 | : : | Tree ID | -| CKV-2 | [split-a] [split-b] | Checkpoint | -| | : : | Selected extent / page | -| | [Tree ID] [Tree ID] | No Weight field | -| | Graph window controls [Fit All] | | -| | ---------------------------------------------------- | | -| | Selected split-a: [Tree] [Journal] | | -| | Bounded page / journal extent content | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Graph windows and catalog pages have separate continuation. Selecting a Tree -uses real bounded base-page inspection; the graph does not scan records for weights. - -## 21. Iceberg - -- **ICE-01:** The configured cluster Catalog loads automatically. Left shared - Tree is Catalog → Namespace → Table → Snapshot. Manifest and File are center - inspection data, not tree levels. Namespace and Table types are labelled - explicitly. CrowDB supports nested Namespace - components as represented by the [Iceberg REST Catalog specification](https://github.com/apache/iceberg/blob/apache-iceberg-1.11.0/open-api/rest-catalog-open-api.yaml). No Metadata leaf or Snapshots wrapper - duplicates content; Manifest List is a center inspection level. Current Snapshot is marked - `Current`; immutable IDs remain exact. -- **ICE-02:** Catalog/Namespace centers list their children. Table center shows - a compact overview followed directly by a Snapshot table: time, operation, - records and files. Overview/Schema/Files controls belong only to selected - Table, disappear for other resources, and have no independent Snapshots tab. - Catalog root uses one Namespaces heading; Namespace uses its actual name. - No Catalog breadcrumb/type/title stack duplicates the left tree. Namespace - and Table lists page in the center with visible continuation controls. -- **ICE-03:** Schema is an expanded tree table with field ID/name/type/required - and hierarchy for structs, lists and maps. Element/key/value identities stay - visible. Large schemas use bounded row windows; collapse is optional, not a - prerequisite to seeing ordinary fields. Create Table and Snapshot Drafts show - this Schema as a separate read-only preview. Field names, types, required flags - and nested definitions come from the first uploaded Parquet for a new Table, - or the current Table Schema for an existing Table. These workflows have no - Add/Remove Field or field-definition inputs and do not publish Schema edits. -- **ICE-04:** Selecting a Snapshot keeps the left tree at the Snapshot level and - shows one scoped summary plus bounded center inspection levels: - `Manifest List → Manifest File list → Data/Delete File list`. These are center - tables/sections, never tree nodes. Expanding one Manifest File requests its - Data/Delete File subtable. Manifest List, Manifest File and Data/Delete File - pagination are independent and lazy; no full-snapshot fetch to calculate a - total. The right properties retain Manifest List path and known size. - File-name columns use a bounded width, left alignment and ellipsis for long - names. The full path/name remains available in the selected object's - Properties and in the filename tooltip. Tree levels are contiguous: Catalog, - Namespace, Table and Snapshot have no empty spacer row or duplicate text-only - leaf between them. - Center drill-down keeps the left tree selected on the Snapshot. Back returns - to the previous center scope with its bounded page, cursor and selection; - browser Forward/Back uses the same bounded center history contract. -- **ICE-05:** Manifest/file inspectors render decoded fields. Parquet shows - footer information and sized Row Group/column blocks. Selecting a column - shows complete metadata in properties. Inspection reads footer/metadata only, - not data pages. Unsupported file types show an explicit capability state. -- **ICE-06:** Catalog context menu creates Namespace; Namespace context menu - creates Table and, when supported, child Namespace. A Namespace's own menu - creates one literal child name component; a dot in the entered name does not - create another level. Full identity is the ordered namespace component array. - Its menu also edits supported properties or drops that empty Namespace. A Table's menu - exposes supported Edit as Draft, Rename, Commit metadata and Drop table operations. A new - Namespace form is prefilled with an editable unique name such as - `namespace-`. A new Table form is prefilled with an editable unique name such as `table-`, - so creation needs no manual name entry; a new Table requires a Parquet upload. Drop - removes the catalog reference; physical reclamation follows service GC. - Snapshot and Parquet menus expose table-managed maintenance as defined in ICE-10. - Manifest and metadata files have no arbitrary physical Delete action. Demo operations are not primary - browsing controls. Properties describe only the selected identity. -- **ICE-07:** Reference windows contain at most 100 entries; Parquet windows - contain 20 Row Groups and 12 columns. Paging replaces the visible window. - Column widths represent compressed sizes and file spans retain exact byte - identities. History restores the selected reference, footer/layout view and - column selection, then revalidates the table head, snapshot and file identity. - Cached branches do not bypass revalidation. A failed or stale inspection - preserves selection context but removes data and continuation actions. - -- **ICE-08 — Parquet import:** Namespace Create Table and Table/Snapshot - Edit as Draft open the same center editor. A new Table starts with a name and - Upload Parquet action, without a fabricated default Schema. The first uploaded - file supplies the Schema; a separate read-only tree table previews it once. - A file list below it contains file name, record count, exact byte size, Status and - removal actions. It has no Schema columns or repeated type cells. - One Status column replaces Change and Upload: Uploaded for additions, Kept for - existing files, Removed for pending removal, and Reselect to resume for unfinished - uploads. Remove and Undo removal remain row actions. - File selection populates Properties; nested Schema details remain in the preview. - Further files are checked against the inferred Schema; an incompatible file - reports its name and the mismatched field before upload. Existing Tables keep - their current Schema and durable field IDs, including when editing history. - No silent union, coercion or Parquet rewriting occurs. Create Table requires - at least one uploaded file; a zero-record Parquet is valid. Removing every added - file from a new Table resets its inferred Schema, ready for a different upload. - Commit is disabled until there is a file change and, for a new Table, an upload. - Existing Tables may remove every data file while retaining their Schema. - The writer supports unpartitioned format-v2 Tables and standard three-level List/Map - encodings; source Snapshots may retain an older compatible Schema. Incompatible - historical Schemas require an Iceberg client; metadata-only evolution must not - block subsequent file edits of the current Snapshot. - Authoring permits 1000 added files, 256 top-level fields, 20 files per selection - and 100 MiB per file. Existing file windows load one Manifest page at a time, - with explicit continuation; Schema previews use bounded row windows. No total-file count - or full Snapshot read is required to open the editor. -- **ICE-09 — Draft and Commit:** choosing a file reads its bounded footer and - checksum, then immediately uploads it through authorized FileIO. The proxy rejects - foreign prefixes and arbitrary credential endpoints. Small native uploads retain - their returned immutable location; larger files use 8 MiB multipart parts. - Save Draft and autosave persist metadata and upload references only, under the API - origin, Namespace, Table and requested source scope. File/Blob bytes and credentials - are never persisted. Completed uploads survive reload without reselecting files; - interrupted uploads require the same bytes. Invalid references are reported, and - expired uncommitted uploads can be replaced under a new identity. Local revisions - reject stale saves/discards. A local Draft is not a server-side editable Table. - Stage-create supplies canonical UUID/Schema/storage material without publication. - Saved Drafts retain their inferred Schema and upload references. Legacy Drafts - with manual Schema edits must be discarded explicitly before starting this flow; - those edits must never be published silently. Frozen pending submissions retain - their exact request and recovery semantics. - Commit freezes the input and publishes generated Manifests and metadata atomically; - file additions/removals produce one immutable data Snapshot. Property-only updates - do not invent data Snapshots. Remove and replace change - the resulting file set; they do not modify the original Snapshot or file bytes. - The chosen source Snapshot and expected current main are separate fixed identities. - Editing history publishes that source's edited file set as a new child of expected - main, with a visible explanation that newer files are excluded. Table UUID, Schema, - default spec, main and the advertised metadata-location guard protect publication. - Concurrent changes conflict; the editor never silently rebases. Manifest rewriting - preserves original data/file sequence values and rejects unpreservable layouts. - Pending requests retain exact body/key and immutable output paths. A lost response - is reconciled by Table UUID plus Snapshot/Manifest List identity before retrying; - no second Snapshot is created. Unknown outcomes cannot be discarded as failed. - Property editing remains a separate name/value/removal form with metadata-only - publication. Advanced updates retain exact integer tokens and UUID conditions. -- **ICE-10 — Removal and expiry:** a live current Parquet file offers - `Remove from table…`, which opens the same Draft editor with that file marked - Removed. Undo restores it before Commit. Old Snapshots remain readable while - retained. Unaffected Manifests, including delete Manifests, are preserved; the - inspector's rewrite capability prevents losing metrics/encryption/other attributes. - Snapshot `Expire snapshot…` is a separate lifecycle command. Current and Branch/Tag - references prevent expiry; the advertised guarded commit checks metadata location - and UUID under head CAS, including refs created or moved after confirmation. - Ordinary Iceberg metadata updates retain standard semantics. Existing pending - maintenance requests can still resume their frozen body/key after reload. - Removing a file reference or expiring a Snapshot does not physically delete the - file. A live Table's published files are not reclaimed solely because no retained - Snapshot refers to them; physical cleanup follows the Table/file lifecycle. - -Iceberg: editing and publication (ICE-08/09): - -```text -Namespace -> Create Table Table / Snapshot -> Edit as Draft -+--------------------------+------------------------------------------------+----------------------+ -| v Catalog | Create Table / events · New Snapshot | Properties | -| v analytics | Table name [table-abc123] (new Table only) | selected file | -| v events | [Upload Parquet] | path, size, records | -| snapshot-a | Schema | | -| | Field ID Type Required | | -| | id 1 long No | | -| | message 2 string No | | -| | Files | | -| snapshot-b Current | File Records Size Status | | -| | a.parquet 100 4 KiB Kept Remove | | -| | b.parquet 100 4 KiB Removed Undo | | -| | c.parquet 100 4 KiB Uploaded Remove | | -| | [Previous files] [Next files] | | -| | [Close] [Discard Draft] [Save Draft] [Commit] | | -+--------------------------+------------------------------------------------+----------------------+ -New Table: upload first Parquet -> infer Schema -> read-only preview -> Commit -Existing Table: retain Schema -> check uploads / remove files -> Commit -Before first upload: "Upload Parquet to create a Table."; Commit disabled -Save Draft -> close/reload -> reuse uploaded files and inferred Schema -Commit -> frozen source + add/remove -> new Manifest List -> conditional main - existing Snapshot and its referenced bytes remain immutable -``` - -Iceberg: Snapshot file inspection (ICE-04): - -```text -Left tree Center panel -v Catalog Snapshot snapshot-a - v analytics [Back] Manifest List - v events ml-001 2 manifests [Open] - snapshot-a * └─> Manifest Files - mf-001 added [Open] - mf-002 delete [Open] - └─> Data/Delete Files - part-001.parquet [Properties] - delete-001.pos [Properties] - -Open / drill -> next bounded center level -Back -> previous level; left tree remains on snapshot-a -``` - -Iceberg: lifecycle (ICE-10): - -```text -Current S2: [a.parquet, b.parquet, c.parquet] - b.parquet -> Remove from table -> Draft [b: Removed] -> Commit S3 - S3: [a.parquet, c.parquet] S2 still reads all three files - -S2 -> Expire snapshot -> current / Branch / Tag check on fixed metadata - changed base -> conflict unchanged base -> remove S2 - neither reference removal nor Snapshot expiry physically deletes files -``` - -Iceberg: Catalog and Namespace collections (ICE-01/02/06): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v Catalog | analytics | Properties | -| v analytics * | [Refresh] | Namespace analytics | -| > events | ---------------------------------------------------- | Full name | -| > users | Name Type Summary | Properties | -| > sandbox | events Table ... | ... | -| | users Table ... | | -| | ... | No duplicate Catalog | -| | [Previous] [Next] | path/type/title stack. | -| | | | -| | Catalog selected -> title: Namespaces | | -| | Namespace selected -> title: actual name | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Iceberg context-menu targets: - -```text -Catalog Namespace Table -+----------------------+ +------------------------+ +------------------------+ -| Create namespace... | | Create table... | | Rename... | -+----------------------+ | Create namespace... (*)| | Commit metadata... | - | Edit properties... | | Drop table... | - | Drop namespace... | +------------------------+ - +------------------------+ -(*) Nested Namespace creation requires backend support. -``` - -Iceberg: Table, Snapshot and File inspection (ICE-02–05): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v Catalog | events [Refresh] | Properties | -| v analytics | [Overview] [Schema] [Files] | Selected Table / reference | -| v events * | Compact table summary | Full identifier | -| v snapshot-42 | Snapshot Time Operation Records Files | Schema / Snapshot identity | -| v manifest-a | 42 ... append ... ... | Manifest List path | -| file.parquet | ... | File format / size | -| | [Previous] [Next] | Selected column metadata | -| | ---------------------------------------------------- | | -| | Select Snapshot -> paged Manifests | | -| | Expand Manifest -> paged Files | | -| | Select Parquet -> footer, Row Groups, columns | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Schema uses a structured field tree table. File inspection decodes metadata -and footer structure; it does not imply reading data pages or SQL querying. - -## 22. S3 - -- **S3-01:** Left Tree is S3 → Bucket. A single namespace introduces no wrapper; - the logical S3 root is not named Datacenter. Root center lists buckets with - 20-row server windows. It directly presents the Prefix query and list without - repeated path/type/heading or ordinary selection/refresh success messages. -- **S3-02:** Selecting a Bucket displays an API page of at most 20 objects. - Bucket and Object lists share `Prefix [input] [List]`. Enter/List applies a - server-side name/key prefix and resets cursor history; Previous/Next replace - rows using the applied prefix, independent of unsubmitted input. Bucket and - per-Bucket Object query windows remain separate. Tree selection and history - restore the relevant scope; loaded-page substring filtering is not Prefix. -- **S3-03:** Selecting an Object reads HEAD fields into properties: key, size, - ETag, last modified, content type, supplied version and user metadata. Missing - fields show `—`. The center presents the selected object's content inspection. - Tree selection and Back/Forward restore the Bucket list and its cursor/prefix. - Preview is a separate bounded operation, not an automatic payload GET. -- **S3-04:** S3 root context menu creates Bucket; the create form is prefilled - with an editable DNS-safe unique name such as `bucket-`, so the - default Create action completes without manual typing. A Bucket's menu uploads - one or more child Objects; batch upload uses each selected file's filename as - its object key, with duplicate keys replacing the existing object. A Bucket's - menu also opens multipart uploads or deletes that empty Bucket. An Object's own - menu exposes Preview, Download, Replace and Delete. Object rows in the center - use the same menu contract without adding every Object to the left tree. - Prefix is a key filter, not a physical folder; Bucket deletion does not silently - cascade. Demo controls are not primary browsing actions. Exact bucket/key is visible. - A successful Object deletion immediately removes that exact Bucket/key from - the current object list before its server refresh completes; other rows remain. - Failed or unknown deletion retains the row. A later refresh failure preserves - the confirmed removal and reports the refresh error. - Multipart state distinguishes uploaded parts, completion, abort, failure and - unknown outcome. Downloads stream; previews read at most the advertised cap. -- **S3-05:** Object details include Storage locations below HEAD, 20 extents per - page: stable index, logical `[start, end)`, Chunk ID, chunk `[offset, end)`, and - physical length. Properties show exact decimal bytes and full identity. - Chunk links preserve object and extent cursors, selected extent and scroll. -- Locations are an admin metadata query, never inferred from HEAD or obtained - by reading object payloads. Empty/missing/corrupt/oversized references have - distinct states. Continuation binds object and metadata generation; overwrite - reports stale and requires first-page refresh. Exact 64-bit offsets survive - JSON/JavaScript without rounding. Bounded decoding and response limits are - mandatory; no recursive placement fetch for every extent. -- **S3-06:** `GET /api/access/s3-inspect/locations` accepts exact bucket/key, - a 1–100 limit (default 20), and an opaque cursor. The Console targets only - the configured Access origin and supplies its server-held management token; - browser credentials and arbitrary endpoint parameters are rejected or ignored. - Access authorizes management privilege before metadata reads and resolves the - configured tenant's bucket/object itself. There is no object-payload client in - the inspection path. -- **S3-07:** Inspection decodes at most 4 MiB of stored references and returns - at most 1 MiB. Access has a five-second metadata deadline; the Console proxy - has a six-second request deadline. Reference-limit errors are 413, corrupt - references 422, missing objects 404, and changed cursor generations 409. - Invalid, expired or oversized cursors and limits are rejected. Cursor expiry - is 15 minutes and its authenticated scope includes bucket identity, exact key, - revision-sensitive generation and next extent index. Identical overwrites also - invalidate old cursors. -- **S3-08:** Location failures preserve HEAD. Empty objects show `No storage - extents`; null Chunk identities show `Location unavailable` without a link. - Previous/Next retain at most 32 cursor positions and replace the page; at most - four bounded location pages are cached. A stale result keeps the previous - page labelled stale and disables continuation until first-page refresh. - Selection is captured before following a Chunk link; return refetches metadata - against the saved generation and restores the exact selected extent. - -S3: Bucket object list (S3-01/02/04): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v S3 | photos | Properties | -| photos * | Prefix [...................] [List] | Bucket photos | -| exports | ---------------------------------------------------- | Name | -| logs | Key Size Modified | ... | -| | cat.jpg ... ... | | -| | dog.jpg ... ... | Selecting an Object reads | -| | ... | its HEAD and properties. | -| | [Previous] [Next] | | -| | | | -| | Objects stay in this bounded list. | | -| | Right-click Object -> its own resource menu. | | -| | Deleted Object disappears on success, then list refreshes. | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -S3 context-menu targets: - -```text -S3 root Bucket Object list row / detail -+-------------------+ +-------------------------+ +------------------------+ -| Create bucket... | | Upload files... | | Preview | -+-------------------+ | Multipart uploads | | Download | - | Delete empty bucket... | | Replace... | - +-------------------------+ | Delete object... | - +------------------------+ -``` - -S3: Object inspection and multipart view (S3-03–08): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v S3 | cat.jpg [Refresh] | Properties | -| photos | Content inspection | Object | -| exports | Preview is read only after an explicit request. | Bucket photos | -| | ---------------------------------------------------- | Key cat.jpg | -| | Storage locations | Size | -| | Extent Object interval Chunk ID (link) | ETag | -| | ... [start,end) ... | Content type | -| | [Previous] [Next] | Modified | -| | | User metadata | -| | Bucket menu -> Multipart uploads view | Selected extent identity | -| | Upload key / ID / state; select -> paged parts | | -| | Upload menu -> Abort (when supported) | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Preview and Download are explicit reads from the resource menu. Selecting an -Object does not download its payload. Multipart inspection is a separate center -view; its state is not piled beneath the ordinary Object list. - -## 23. Actions and properties - -- **ACT-01:** The left tree owns resource navigation and context menus. Create - child belongs to the parent menu; Delete belongs to the resource's own menu. - Selection is Read; child collections are automatic paged lists in the center. - There is no generic Update requirement. Additional operations use precise - resource-specific names and exist only where that node's capabilities allow. - A lifecycle operation such as Reclaim is not relabelled Delete. -- **ACT-02:** All domains share menu, dialog, button and property styles. The - center has one scope heading, query/filter controls, Refresh and content, with - pagination below its list. It has no duplicate CRUD toolbar, collapsed Actions - strip, inline creation forms or repeated path/type/title stack. Query controls - and bounded Preview/Download are distinct from resource mutations. - Ordinary buttons have visible borders; destructive menu items use muted red. - Readonly, deployment mode, backend capability and lifecycle restrictions apply - to every entry point. Unknown capability does not enable writes. -- **ACT-03:** Right properties show complete information for the clicked item, - including graph blocks/columns/extents. Optional diagnostics never replace - primary details. Metrics panels, entries and polling are excluded from the UI. Future metrics - publishing targets customer-managed time-series systems; it is outside this - UI contract. -- **ACT-04:** Tree labels, icons and ordinary resource rows activate the same - identity. Expansion chevrons only expand. List-only resources use the same - context menu on their center row; a focused detail retains access to that exact - resource menu. Right-click targets the clicked resource, independent of prior - selection. Menu key / Shift+F10 opens the same menu; Escape restores focus. - Matrix cells and diagram blocks retain their documented inspection semantics. -- **ACT-05:** Simple mutations use shared dialogs; Dataset and Iceberg authoring - opens center editors with one Submit action and local recovery. The target - stays explicit and cannot change under an open form. - Switching resources closes obsolete forms without claiming to cancel accepted - work. Successful operations refresh affected lists, tree references and properties; - failures retain input and unknown outcomes require reconciliation. -- **ACT-06:** Empty collections explain how to create a supported child from - the parent menu. Empty, loading, unavailable and denied states are distinct. - Browser Session activity is absent from domain workspaces. Operation feedback - stays with its flow; resource history such as Snapshots and publication receipts - remains available. Default properties are labelled structured fields, not JSON. - -The common layout and menu targets are: - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| Resource tree | Current scope | Properties | -| v Parent | Filter [.............] [Refresh] | Selected identity | -| Child | ------------------------------------- | Full path / ID | -| | Child name Type / summary | Structured fields | -| | ... | ... | -| | [Previous] [Next] | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Domain-specific scope and menu mappings: - -```text -Iceberg Center on selection Context menu -Catalog Namespaces (paged) Create namespace - Namespace Tables/children (paged) Create table/namespace; Properties; Drop - Table Overview/Schema/Files Edit as Draft; Rename; Metadata; Drop - Snapshot Paged metadata/files Edit as Draft; Expire when unreferenced - Manifest -> File Decoded metadata Current Parquet: Remove from table - -Dataset Center on selection Context menu -Datasets Datasets (paged) Create dataset - Dataset Snapshots (paged) Import / New Snapshot; Delete empty - Snapshot Samples x Fields Create child; Edit Split revision; - Stable/Retain/Release/Reclaim - Splits Revision/member windows Create Split; Edit revision - Revision Fixed Split collection Edit as a new revision - Split Members (paged) Edit / Remove in a new revision - Sample/Field/Cell in center Selected scope/value Edit in child Snapshot; Preview - -S3 Center on selection Context menu -S3 Buckets (paged) Create bucket - Bucket Objects (paged) Upload; Multipart; Delete empty - Object in center Content inspection Preview; Download; Replace; Delete -``` - -These mappings do not imply every listed action is available for every state. -Immutable references and backend-owned internal structures do not acquire generic -CRUD merely to match another domain. Dataset lifecycle support is defined in §25. - -## 24. E2E organization and timing - -- **TEST-02:** Page-function specs retain each behavior once: shell `0x`, - physical lifecycle `1x`, logical KV `2x`, KV data `3x`, inspector/canvas `4x`, - Capacity/Chunk/Chunk-KV `5x`, Iceberg `6x`, S3 `7x`, Dataset `8x`, cross-function `9x`. - Dedicated creation/reconfiguration specs retain their UI mutations; a smoke - chain does not repeat every dialog or failure permutation. -- **TEST-03:** Use separate unit, backend integration and native browser E2E - layers. Browser fixtures provision actual services, metadata and files; - outages use actual service lifecycle. Deployment and container capability - enforcement require real API acceptance. Share setup per spec and keep - destructive cases last. Reset only for cases that require empty authority. - S3, Iceberg and Dataset basic create/add/list/read/delete flows share one - owned cluster and one startup/teardown under the managed resource owner. - Distinct resource names and protocol scopes isolate their effects. API reads - verify results; they do not replace the UI mutations under test. Dedicated - suites retain pagination, large transfers, concurrency and recovery coverage. -- **TEST-04:** Keep the step timer and slow-test reporter. Measure setup, mutation - response, lifecycle readiness, DOM refresh and teardown separately. Slow steps - at 2 s and very slow steps at 5 s remain logged even when a step fails. Tests - at 10/30 s retain slow/very-slow reports. Compare per-test baselines; investigate - more than 2× rather than hiding time with retries or larger deadlines. -- **TEST-05:** One worker owns the mutable runtime; installed system browser, - no automatic browser installation. Auxiliary-service dependency convergence - may wait up to 10 s for Chunk-KV range activation and matching catalog/grant - authority. After 3 s it records the latest readiness response; it also records - the final response and elapsed time. This prerequisite does not change UI - response or process-observation assertion budgets. - Default assertion deadline is 3 s and - election 10 s, polling every 100 ms. No fixed sleeps. Native large multipart - and SF=1 loader acceptance run separately from ordinary UI iteration; normal - smoke data remain small while preserving multi-Strip and pagination cases. - - -## 25. Dataset - -Depends on: [Dataset control lifecycle](../access-server/dataset/design-dataset-control.md). - -- **DS-01 — Scope:** Dataset is immediately before S3. The fixed `dataset-ns` - namespace is a property, not a single-child navigation level. Catalog discovery - comes from the configured authority, including SDK-created identities. Empty, - missing configuration, unavailable service and denied access remain distinct. - Credentials stay on the Console server; no browser endpoint or bearer editor. - The root center automatically lists Datasets. Root context menu creates a - Dataset in the center table editor, with an editable unique name such as - `dataset-`. Adding data and submitting Create Dataset publishes its - first Snapshot in the same flow. There is no name-only creation dialog or - second creation step. A Dataset menu offers New empty Snapshot; a Snapshot - menu offers Edit Snapshot to open a child Draft. All use §23; there is - no Session activity region. The left panel begins with the Dataset title and - shared Help link above the tree. Properties render named fields rather than JSON. -- **DS-02 — Relationships:** left navigation follows declared parent links as a - forest. Roots and each parent's children page independently; no ancestry scan - or Snapshot-ID sorting reconstructs the tree. Latest and Stable are reference - badges. A Snapshot's Splits group is a peer of its children. Samples and Fields - remain in the center. Direct Snapshot navigation preserves the exact identity. - No separate Selected ancestry panel duplicates the tree. Split hierarchy is - Snapshot → Splits → immutable collection Revision → named Split. Current marks - the active collection revision; history pages lazily under Splits. Same-name - Splits in different revisions have distinct identities. Lazy nested branches - inherit depth, indentation and accessibility levels from the shared Tree. - Dataset and Snapshot inventory pagination is reachable in the center. The fixed - Datasets root retains its first bounded page plus the selected resource; the - catalog has no Previous/Next tree-page toolbar. Snapshot branch continuation - stays lazy and independent. Tree appearance and selection use shared conventions. - Empty child collections do not render diagnostic/footer text such as `No Split - revisions` or `No children` in the tree. If an empty state is useful, it is - shown in the selected resource's center panel; the tree keeps only real nodes, - continuation controls, and actionable errors. -- **DS-03 — Combined inspection:** one Samples matrix uses Schema definitions as - columns, defaulting to twenty rows and twelve fields. Required/optional is the - actual field definition; no inferred type, MIME or nested schema. Header, Sample - row and Sample/Field cell selections have separate properties and history. - Clicking a row highlights it; Properties pages through all of that row's Fields - independently of the center columns. - Missing, tombstoned, inline, payload-backed and inherited values are distinct. - Known lengths and declared checksums are metadata, not verified payloads. -- **DS-04 — Bounds:** each catalog/row/column query carries its immutable scope and - explicit continuation. A stale continuation requires first-page refresh. Loaded - row filtering is labelled local. Opaque binary Sample IDs retain canonical padded - base64; valid UTF-8 is only a display/helper encoding. Decimal u64 values remain - strings. Navigation retains bounded identities/cursors and scroll, not payloads. - Hidden work stops; late responses cannot replace another scope. -- **DS-05 — Preview:** only the explicit Preview action reads a selected field. - Complete verified values of at most 64 KiB render as UTF-8 or hex. Unknown length, - missing/tombstoned values, oversized data, integrity failures and cancelled reads - never appear as a successful truncated value. Browsing starts no Read Plan. -- **DS-06 — Table editor:** Create Dataset and Edit Snapshot use one center table. - One row is a Sample; one column is a Field; Sample ID is a separate editable - row key. New input starts with optional `content`. The sole top toolbar action - is Upload files. With one Field it opens the picker directly; with multiple - Fields it first asks for an explicit target in a temporary chooser. There is - no persistent Upload into selector. `+ Sample` at the table bottom generates - an editable ID; Upload files adds rows with the file's exact filename as the - default Sample ID into the selected Field, without a type or `string` prefix. - Sample ID and `+ Sample` remain visible while scrolling across Fields. - Duplicate IDs are rejected. Existing values are replaced explicitly through their - cells, preserving other pending Field changes. Empty cells show Upload file or - a text input directly. Choosing Upload opens the native picker immediately; text - is edited inline, including newly added columns, without a Text/File dialog or - Apply step. Inherited inline values load on focus, not for every visible cell. - Clear writes a tombstone and Remove changes membership. `+ Field` at the - right edge of the table header adds a column and remains available after files are added; a Field is not itself a - standalone file. A file or inline text is a value in a Sample × Field cell, - so every cell in a new file-valued Field keeps the direct Upload file or text - input. A field-level bulk upload, if later needed, must be a separate explicit operation and - must not replace the cell editor. Every upload or text edit started from a - cell is bound to that exact Sample × Field cell and can never fall back to the - last row or another implicit target. A bulk upload creates one explicit - file-to-row mapping and never chooses a row implicitly. Required parent Fields - cannot be removed or downgraded. A selected row exposes - its Fields in Properties independently of the center column window. Editing a - cell preserves the Draft's Sample row order; changing a value must not move - that Sample to the end of the table or change the target of the next edit. - Edit Snapshot freezes that specific parent and submits with Create Snapshot. - New empty Snapshot opens blank input with no parent, even when Latest exists; - it inherits no Schema, Samples or Splits. Original content remains immutable. - Advanced controls expose - independent roots, parent selection, frozen Split source filtering and explicit - inheritance of parent Splits. Snapshot editing has no Split-definition form. - Creation is bounded to 1000 Sample changes, 64 Fields, 64 KiB per text value, - 4 MiB of editable metadata, 20 files per batch and 100 MiB per file. Larger - committed Snapshots remain browsable with independent row/column windows. - Sample/cell context menus create a child Draft with that identity preselected. -- **DS-07 — Browser Draft and Commit:** each Draft has an independent resource ID. - IndexedDB stores metadata, small text, fixed Dataset/parent/source identities, - upload references and expected head conditions under the API origin/Dataset scope. - Dataset creation Drafts use a stable browser scope independent of the editable - name and appear beneath Datasets as Dataset Draft entries. Uploading the first - file, or submitting inline data, creates the server Dataset automatically and - pins its returned identity; its name then becomes read-only. Subsequent failures - retain the same editor and identity. A conflicting name or unknown creation - response never causes an existing Dataset to be adopted. The user can correct - an unallocated name; an allocated Dataset is not silently deleted on discard. - The tree lists other local Snapshot Drafts beneath Dataset and local Split Drafts - beneath their Snapshot's Splits. Save Draft and autosave retain editable input; - closing does not publish. Browser storage never contains File/Blob bytes. Storage - failure is visible; a transaction compares local revisions to reject stale-tab - saves and discards. A second Draft never silently resumes or replaces the first. - The identity fence prevents a same-name replacement Dataset receiving old input. - Choosing a file starts incremental MD5 and immutable 4 MiB part uploads with two - concurrent requests. Cells show filename, size and state; Properties includes - checksum and unclaimed expiry. Complete references are checked on restore and - Commit. An interrupted upload requires reselecting bytes; the size/digest key - resumes matching whole parts. Expired/aborted uploads receive a new identity. - No complete file must be reselected just because the editor reloads. - Create Dataset / Create Snapshot freezes input, prepares the complete immutable Snapshot, then conditionally - selects Latest. Split Commit updates only currentSplit. Unknown replies retain - the exact ID, input and original conditions for explicit retry; head conflicts - retain the returned observation for review. Neither silently rebases. Definitive - preparation rejection offers Edit as new Draft with a fresh resource ID; the - original input is never changed under a reserved identity. Unknown outcomes must - first be resolved. Successful Commit removes the local Draft and selects its - immutable result. - Discard removes only local input. Uploaded data and unreferenced results follow - existing service expiry/GC; local Drafts do not retain parents or files on the server. -- **DS-08 — Split Draft, View and Commit:** a committed Snapshot's Create Split - menu opens the complete set in the center, preserving other names. Filter rules - support Sample IDs, prefixes, ranges, keywords, inline Field matches, AND/OR/NOT; - they do not search uploaded file content. Weighted partitions, overlap and coverage - remain explicit. Save Draft is browser-local. Commit prepares and conditionally - publishes a new immutable collection revision without changing Dataset Latest. - Split counts are internal completeness metadata, not required navigation or - overview statistics. Browsing neither scans all pages to count nor labels loaded - rows or whole-Snapshot counts as a Split total. - Each rule has View. Draft View uses `split-preview`, a read-only bounded scan of - the fixed Snapshot, not a filter on the loaded page. A page scans up to 20 Samples - and can have zero matches plus a continuation. Cursors bind Snapshot and the full - rule; changing a rule clears the preview. View writes no draft, evaluation progress - or revision. The common matrix and Properties display matching rows. Committed - Split View reads the selected revision's frozen member pages. New Snapshots and - later revisions do not change old membership. Inherited membership is intersected - with child membership; new Samples remain unassigned until explicit reevaluation. - Historical revision editing retains the selected base and publishes under the - observed current head condition. Weighted members rename together; removing a - partition removes its complete group. -- **DS-09 — Lifecycle:** stable, retain, release and reclaim name the exact - Snapshot. Backend protection wins over an observed eligibility hint. Reclaim - distinguishes unknown, pending logical cleanup and completed retirement; physical - cleanup may remain deferred. A Dataset's own menu offers Delete when - `dataset-delete` is advertised. The confirmation pins its immutable Dataset ID. - Nonempty Datasets (including pending creation or cleanup) are rejected by the - backend. An accepted mark hides the Dataset from tree, list and properties; - cleanup follows the configured grace period, twenty minutes by default. - The name remains reserved until cleanup completes. Retries retain the same ID - and deadline; replacement identities are never silently targeted. No Split-history - purge action is implied. - Published Sample/Field changes produce a new Snapshot rather than modifying - the immutable selected one in place. -- **DS-10 — Modes:** `modules.dataset=false` hides the domain and prevents Dataset - observation. API prefixes apply to every request. Readonly disables editing and - the Console API rejects direct writes; exact matrix and Split preview POSTs remain observations. - Managed mode permits supported data operations without enabling topology changes. - -Dataset: catalog and Snapshot inventory (DS-01/02/06): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v Datasets | images [Refresh] | Properties | -| v images * | Complete Snapshots | Dataset images | -| v snapshot-a | Snapshot Parent Samples References | Dataset ID | -| > Splits | snapshot-a - ... Stable | Namespace dataset-ns | -| > snapshot-b | snapshot-b snapshot-a ... Latest | Latest snapshot-b | -| > audio | ... | Stable snapshot-a | -| | [Previous] [Next] | ... | -| | | | -| | Select Datasets root -> paged Dataset list | | -| | Select Snapshot -> Samples x Fields | | -| | No Session activity region. | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Dataset context-menu targets and immutable editing: - -```text -Datasets root Dataset images Snapshot -+--------------------+ +---------------------------+ +-------------------------+ -| Create dataset... | | New empty Snapshot... | | Edit Snapshot... | -+--------------------+ | Delete Dataset... | | Edit Split revision... | - +---------------------------+ | | - | Set stable | -Delete: empty only; hide immediately. | Retain / Release | -Cleanup: configured grace (default 20 minutes). | Reclaim... | - +-------------------------+ -Sample/Field edits -> reviewed candidate -> new published Snapshot -Published Snapshot data are never edited in place. -``` - -Dataset: table editing and browser Drafts (DS-06/07): - -```text -Dataset Help | Create Dataset | Properties -v Datasets | Name [dataset-xxxx] | Selected Sample / Field - Dataset Draft xxxx | [Upload files] | - v images | +-------------+-------------------+-----------+| - snapshot-a | | Sample ID | content | + Field || - Snapshot Draft b | +-------------+-------------------+-----------+| - | | cat.jpg | cat.jpg · 32 KiB | Remove || - | | note-001 | Upload or [text] | Remove || - | | + Sample || - | +-------------+-------------------+-----------+| - | [Previous rows] [Next rows] < columns > | - | > Advanced | - | [Save Draft] [Create Dataset] [Discard Draft] | - | Saved locally | - -Edit Snapshot: fixed Parent shown; submit = Create Snapshot. -New empty Snapshot: blank content column; no Parent; submit = Create Snapshot. -Upload files with multiple Fields -> choose target Field -> native file picker. -``` - -```text -Choose file -> checksum + bounded parts -> upload reference in local Draft - | interrupted: reselect matching bytes to resume -Save Draft -> browser metadata only -> close/reload -> continue editing -Create Dataset / Create Snapshot -> freeze input -> prepare result -> conditional head update - unknown: retry exact input / ID / conditions -``` - -Dataset: Split preview (DS-08): - -```text -v images | Splits · Draft · local - v snapshot-a | name [cats] rule [Field label = cat] - v Splits | [View cats] overlap / coverage - v revision Current | +-------------+-------------------+-----------+ - train | | Sample ID | content | label | - Split Draft c | | | | | - | +-------------+-------------------+-----------+ - | | cat.jpg | payload · 32 KiB | inline | - | +-------------+-------------------+-----------+ - | [Next Samples] - | [Discard Draft] [Save Draft] [Commit] - | Saved locally -``` - -Dataset: Samples matrix and selected cell (DS-03–05/08): - -```text -+----------------------------+--------------------------------------------------------------+------------------------------+ -| v Datasets | snapshot-b [Refresh] | Properties | -| v images | Parent / Latest / Stable / inspection state | Selected Sample/Field cell | -| v snapshot-a | ---------------------------------------------------- | Sample ID | -| > Splits | Sample image label ... | Field image | -| v snapshot-b * | sample-001 cat | Required | -| v Splits | sample-002 * dog | Storage Payload | -| v rev Current | ... | Length | -| validation | [Previous rows] [Next rows] | Checksum | -| | [Previous fields] [Next fields] | [Preview field] | -| | | Verified complete value | -| | Select Split -> pinned revision member window | or explicit unavailable | -| | Select column / row / cell -> matching properties | | -+----------------------------+--------------------------------------------------------------+------------------------------+ -``` - -Preview is explicit and bounded to a complete verified value of at most 64 KiB. -Snapshot/revision/member history remains business data, independently of the -removed browser Session activity. - -Local batch mapping uses version 1 JSON aligned with Sample/Field edit models, -with the same editor limits. `schema: null` inherits the parent Schema. IDs and -inline bytes use canonical base64. File values in the submitted mapping are completed -upload descriptors; pending file references cannot Commit. Duplicate IDs, invalid -Fields and incomplete references fail without changing published content. - -```json -{ - "version": 1, - "schema": {"version": 1, "fields": [{"name": "text", "required": true}]}, - "samples": [ - {"kind": "upsert", "id": "c2FtcGxlLTE=", "fields": [ - {"name": "text", "value": {"kind": "inline", "data": "aGVsbG8="}} - ]} - ] -} -``` - -A submitted payload value is `{"kind":"upload","id":"","length":"123","md5":""}`. Pending file tokens are local editor state only. An optional -field tombstone is `{"kind":"tombstone"}`. Removal is -`{"kind":"remove","id":"c2FtcGxlLTE="}`. Authority composition combines these -bounded edits with the frozen source; the browser never constructs a partial -Manifest and labels it complete membership. diff --git a/doc/design/console/design-crowdb-console.md b/doc/design/console/design-crowdb-console.md deleted file mode 100644 index 88abadbdc..000000000 --- a/doc/design/console/design-crowdb-console.md +++ /dev/null @@ -1,542 +0,0 @@ - - - -# CROWDB - Design: Console (Overview) - -The `crowdb-console` component provides a Web UI and a CLI that share one -Rust core for managing `crowdb-paxos-kv-server` clusters. This overview covers -the backend, data model, CLI, and hosting concerns; the frontend SPA -design is detailed in the sub-design `design-crowdb-console-ui.md`. - -## Table of Contents - -- [1. Goals and Non-Goals](#1-goals-and-non-goals) - - [Goals](#goals) - - [Non-Goals](#non-goals) -- [2. High-Level Architecture](#2-high-level-architecture) - - [2.1 Call Path](#21-call-path) - - [2.2 Reuse Boundary](#22-reuse-boundary) -- [3. Data Model](#3-data-model) - - [3.1 Physical (deployment) view](#31-physical-deployment-view) - - [3.2 Logical (usage) view](#32-logical-usage-view) - - [3.3 Source of truth and freshness](#33-source-of-truth-and-freshness) - - [3.4 Design decisions](#34-design-decisions) -- [4. Console Configuration and Authority](#4-console-configuration-and-authority) - - [4.1 Separated local configuration](#41-separated-local-configuration) - - [4.2 Runtime observation](#42-runtime-observation) - - [4.3 Group 0 authority](#43-group-0-authority) - - [4.4 Local runtime namespace](#44-local-runtime-namespace) -- [5. Node Access Model](#5-node-access-model) - - [5.1 Two transports per node](#51-two-transports-per-node) - - [5.2 SSH defaults (russh)](#52-ssh-defaults-russh) - - [5.3 Process lifecycle (deploy / start / stop)](#53-process-lifecycle-deploy--start--stop) -- [6. Web UI Backend (Axum)](#6-web-ui-backend-axum) - - [6.1 Design Rules](#61-design-rules) - - [6.2 In-process test API](#62-in-process-test-api) - - [6.3 Orchestration semantics](#63-orchestration-semantics) - - [6.4 Resolution rules](#64-resolution-rules) - - [6.5 Frontend contract](#65-frontend-contract) -- [7. CLI Design](#7-cli-design) -- [8. Error Model and Operation Logging](#8-error-model-and-operation-logging) -- [9. Observability](#9-observability) -- [10. Hardware mutations](#10-hardware-mutations) -- [11. Cluster teardown and verification](#11-cluster-teardown-and-verification) - -## 1. Goals and Non-Goals - -### Goals -- Single workspace project `crowdb-console` delivering a Web UI and a CLI that share one Rust core. -- Operate against any number of `crowdb-paxos-kv-server` instances via their public surfaces (HTTP management API + crowdb-rpc KV / health). -- Model a **Rack → Node → Server Instance → Store → Group → Replica** hierarchy, including a "simulated hardware" mode that runs entirely on `127.0.0.1`. - -### Non-Goals -- Bypassing `crowdb-paxos-kv-server` to talk to Paxos / WAL / storage internals. -- Multi-tenancy and a general audit-log service. -- Making a console-local file authoritative for cluster topology. - -Ordinary deployment runs the shared OCI node image. The host lifecycle owns -containers, applied resources and private ownership receipts; each node monitor -owns its service processes and retained data. Local evaluation may place several -logical nodes on one host, with zero physical host failure tolerance. Production -placement uses the confirmed physical failure domains. Test-only measurement -fixtures may own native processes but do not define a second public deployment -lifecycle. - -## 2. High-Level Architecture - -`crowdb-console` is **one project** split across the `lib/` and `app/` -workspace roots: a shared core lib plus two binaries. The console is a -general cluster-management surface (not limited to CROWDB), so crate -names use the `crowdb-*` prefix without `kv`. - -``` -lib/crowdb-console-shared/ (lib) data models, HTTP+crowdb-rpc clients, registry, aggregator, error model, SSH session pool, workload generator -app/crowdb-web/ (bin) Axum backend, static asset server, proxy routes - src/ Rust source - ui/ React + Vite frontend source (TS, shadcn/ui, React Flow) - tests/ integration tests -app/crowdb-cli/ (bin) clap-based CLI; depends on shared -``` - -Targets: -- `crowdb-console-shared` → reusable lib for both frontends. -- `crowdb-web` → bin, serves UI + API on `:9920`. -- `crowdb-cli` → bin, the user-facing CLI. - -### 2.1 Call Path - -The web frontend (SPA backed by Axum) and the CLI follow the same -path through the shared `ops` module: - -- **Web**: `user → crowdb-web (Axum) → shared (ops module) → group-0 sysdata + crowdb-paxos-kv-server` -- **CLI**: `user → crowdb-cli → shared (ops module) → group-0 sysdata + crowdb-paxos-kv-server mgmt` - -Both frontends build an `OpContext` with Group 0 discovery seeds. Attached CLI -commands select `--cluster` or `--management-endpoint`, verify the confirmed -cluster UUID through the management origin, then verify the same UUID through -native Group 0 before hardware/KV operations and native workloads. Production -Web uses a versioned process configuration. Both read confirmed hardware and logical records through Group 0 -and resolve node management endpoints from live registration. Ordinary OCI nodes -use `crowdb-monitor` for process ownership on both Docker and containerd. A -configured auxiliary Web process may retain a local launch registry; it is not -required for node formation or endpoint-only attachment. - -The CLI talks directly to group-0 system metadata via -`CrowdbSysmdClient` and to individual `crowdb-paxos-kv-server` management -APIs via `ServerClient` — no `crowdb-web` intermediary. The `ops` -module in `shared` holds the operation logic that both frontends -call. - -``` - ┌──────────────┐ ┌──────────────┐ - user ───► │ crowdb-web │ or │ crowdb-cli │ (frontend) - └──────┬───────┘ └──────┬───────┘ - │ parse input, │ - │ render output │ - └──────────┬────────────┘ - ▼ - ┌───────────────┐ - │ shared │ (business logic: - │ (lib crate) │ ops module, - └──────┬────────┘ leader discovery, - │ registry controls, - ┌──────────────┼──────────────┐ SSH session pool) - ▼ ▼ ▼ - HTTP crowdb-rpc SSH - │ │ │ - ▼ ▼ ▼ - ┌────────────────────────────────────┐ - │ crowdb-paxos-kv-server │ (one per node) - └────────────────────────────────────┘ -``` - -### 2.2 Reuse Boundary - -- All "what to do" lives in `shared` (e.g. `ops::kv_logical::add_group`, - `ops::kv_server::deploy`, `ops::kv_data::put`, `ops::hardware::add_rack`). -- `web` (Axum) and `cli` (clap) only parse input and render output. -- The web SPA **does not** reimplement business logic; it calls - `shared` via the Axum backend, never `crowdb-paxos-kv-server` directly. -- Both frontends build an `OpContext` and call `shared`'s `ops` - module directly — the CLI from its verified application target, - the web backend via `AppState::op_context()` (sharing the - cached `PaxosKvClient` and Group 0 management seeds). -- Shared hardware and logical operations provide the same authority and - conditional publication rules to both frontends. Node count and runtime do not - select UI permissions; required authority and resource ownership still apply. - -## 3. Data Model - -The console exposes **two hierarchy views** of the same cluster. Both -views describe the same underlying entities; they differ only in the -direction from which the cluster is observed. - -### 3.1 Physical (deployment) view - -> "What hardware exists, and what is running on each piece of it." - -Rooted at **Rack → Node → Server → PxStore → PxGroup → {LocalReplica, -RemoteReplica…}**. Every entity below `Node` is described from that -node's vantage point. A `PxGroup` has exactly one local replica plus -N−1 remote-replica proxies. This mirrors the `crowdb-paxos-kv-server` internal -data structure, which is why this view is also the "debugging view": -the API surfaces the remote-list explicitly so an operator can spot -bugs where a node failed to register all of its peers. - -Identity is the parent chain -`(rack_id, node_id, store_id, group_id, replica_id)`. - -### 3.2 Logical (usage) view - -> "What stores and groups exist in the cluster, regardless of where -> they live." - -Rooted at **Cluster → Store → Group → Replica…** with a unified replica -list (no local/remote split; each replica carries a `node_id`). This is -the view that KV traffic, leader resolution, and routine cluster -operations use. Shared operations translate logical IDs into confirmed -membership and live endpoints for both frontends. - -Identity is `(store_id[, group_id[, replica_id]])`. - -### 3.3 Source of truth and freshness - -- **Group 0:** rack and node identity, nonsecret SSH connection settings and - credential references, disk hierarchy, bindings, KV stores, groups, - replicas, and service registration. -- **Local process inputs:** a versioned Web process configuration, bare-metal - launch registry, and per-console secret store. Neither topology nor inline - SSH secrets are accepted in the launch registry. -- **Live state:** management health, process identity, and current endpoints. - Docker reads monitor-owned process state; bare-metal process identity is - checked by `LaunchRuntime`. A stopped process is not a live registration. -- **Unavailable authority:** missing or ambiguous registration and Group 0 - outages are reported as unavailable. The monitor and launch registry do not - supply fallback topology. - -### 3.4 Design decisions - -- **No `server_id` namespace.** The server's mgmt/crowdb-rpc URLs live inside - `Node.server` and are never exposed in console-facing JSON URLs. Since - the console enforces one server per node, node identity *is* server - identity. -- **Local/remote split is visible only in the physical view.** The - logical view collapses replicas into a unified list so cluster-level - operations can ignore placement. The physical view keeps the split - for debugging missing peer registrations. -- `StoreView` / `GroupView` / `ReplicaView` reuse `crowdb_paxos_kv::cluster::info` - where possible; the console-side wrapper adds the `node_id` - projection that the per-server protocol does not encode. - -## 4. Console Configuration and Authority - -### 4.1 Separated local configuration - -`WebProcessConfig` contains the listener, Group 0 management seeds, UI and log -paths, deployment mode, and (for Docker) the monitor status path. Configured -managed Web requires this versioned input. Docker rejects a launch registry. - -Standalone Web starts without `--config` in the persistent console `default` -namespace. Its private, atomically replaced `config.json` retains pre-bootstrap -hardware, process launch inputs, and a post-bootstrap topology cache. Web restart -recovers KV processes first, discovers Group 0, reloads its confirmed records, -and then recovers auxiliary services from their recorded launch inputs. The -cache never overrides initialized Group 0 topology. A malformed file or failed -initialized authority recovery reports an error without resetting operator data. - -`LaunchRegistry` contains bare-metal process policy: service, node, host, -binary, service config, workspace and auto-start setting. Runtime PID and -start-time identity are retained separately by `LaunchRuntime`. SSH credential -reference IDs are read from Group 0 and resolved against each console's local -secret store. Group 0 never contains private keys, passwords, PIDs, images or -container IDs. - -The `ConsoleConfig` struct is an operation input for bootstrap and local -development; standalone Web persists it locally until bootstrap and retains -launch inputs for restart. A sealed `BootstrapIntent` -retains pre-Group-0 identity across interruption and is deleted only after all -committed records are verified. - -### 4.2 Runtime observation - -The production `/api/preview` snapshot reads Group 0 hardware and logical -records and validates live service registrations. Docker overlays monitor -process status, while bare-metal Web uses its local launch runtime. Missing -monitor status makes Docker runtime observation unavailable. The in-process -Web test router keeps a monitor cache for fixture orchestration; that cache is -not production topology authority. - -Manual node Web refreshes per-node KV topology through the admitted management -endpoints before physical reads. Remote process health comes from this live -observation; remote PIDs never enter Web's local process table. The service -list separately queries admitted monitors over their authenticated control -channels for actual live children, with `pid_node_id` identifying the process -namespace. This observation verifies local cluster/node binding and remains -available during Group-0 failure. The Web host neither probes these PIDs via -its own `/proc` nor stores or signals them as local processes. A failed probe -marks the node down, while local launch observations still require a tracked PID. - -### 4.3 Group 0 authority - -System group (store 0, group 0) replicates hardware and KV-cluster metadata. -Bootstrap initializes selected KV members, wires peers and conditionally -publishes the rack, node, store, group and replica records. A retry compares -sealed identity and already committed content, writes only missing records, -and rejects conflicting content. Nonmember KV processes receive Group 0 seeds -and must register exactly one live identity before logical operations use them. - -The metadata namespaces are `/hw/rack`, `/hw/node`, `/hw/dg`, `/hw/disk`, -`/hw/owner`, `/hw/bind`, `/kv/store`, `/kv/group`, `/kv/replica`, and `/srv`. -Logical mutations confirm all node-side steps before publishing membership; -conditional writes and confirmed reads reconcile a lost response. A local -launch or monitor record never substitutes for a missing Group 0 result. - -### 4.4 Local runtime namespace - -The default local root is `.crowdb-runtime/`, divided into `ephemeral/`, -`persistent/`, `artifacts/`, and `ports/`. A mini-cluster or retained console -deployment is one persistent namespace. Its manifest stores the namespace -identity and the mapping from each `(service kind, logical instance)` to a -port. Service paths live below -`services///{data,config,log,artifacts}`. - -`start_new` creates an identity and its assignments. `restart` reuses the -same paths and assignments and never asks for replacement ports. `stop` -terminates recorded processes while preserving the manifest and data. -Deletion is the only operation that releases persistent claims and removes -the namespace. If a recorded port is held by another owner, restart reports -the conflict; it does not silently select a different endpoint. - -Disposable deployment and E2E fixtures use ephemeral namespaces with the -same layout. A saved CLI cluster inside an ephemeral fixture inherits that -fixture's process and port ownership; its launcher may exit without releasing -ports still used by the fixture. Process records remain outside disposable -data so cleanup can terminate services even after data removal. Cleanup releases -these nested claims along with the fixture. All local CROWDB data, generated -configuration, logs, benchmark output, and coordination state stay below this -root; no default path uses the system temporary directory. Ordinary clean -operations preserve persistent -namespaces. - -## 5. Node Access Model - -### 5.1 Two transports per node -| Purpose | Transport | -| --- | --- | -| Deploy / start / stop `crowdb-paxos-kv-server` process; copy binary | SSH | -| Runtime mgmt API (add store/group, list, health) | HTTP | -| Runtime KV ops, paxos health | crowdb-rpc | - -### 5.2 SSH defaults (russh) -- Crate: **`russh`** (pure Rust, async). No shell-out fallback. -- Default auth: `~/.ssh/*` keys (agent + standard key paths). -- Alternative auth: explicit key path; explicit `user/password`. -- Default host: `127.0.0.1` with the current OS user. -- Pre-flight: every operation calls `ssh::probe(node)` which performs a real handshake before any side-effecting work. Failure surfaces as `NodeUnreachable { node_id, reason }`. - -**SSH credential boundary:** Group 0 stores only the SSH user, port and -credential reference associated with a node. Each bare-metal console resolves -the reference in its own local secret store. Bootstrap intent rejects inline -private keys and passwords; the launch registry accepts references only. - -Node admission installs each node's public Ed25519 key and proves access in -both directions with strict host-key checking. Preparation reuses one -key-authenticated candidate session and one session per existing peer; the -two directional proofs run concurrently. Ordered preparation commands reuse -their authenticated connection. A failed stage preserves durable progress for -retry, while initialization passwords remain only in the submitted request. - -### 5.3 Process lifecycle (deploy / start / stop) - -DiskDB deployment accepts an optional positive `keepalive_interval_secs` for -its normal heartbeat policy. Omitting it retains the server default; the -committed remote service configuration preserves it across restart. This -setting does not change ownership or deployment protection rules. - -Chunk-KV deployment optionally accepts `stream_chunk_capacity_bytes` (1–256 MiB) -and positive `stream_extent_page_entries`. Omitted values retain server defaults. -These storage parameters are restricted to Chunk-KV and become part of its -committed service configuration, including subsequent container-owned restarts. - -`LaunchRuntime` uses the validated launch registry for local or SSH process -start, restart, stop and readiness checks. It records PID plus process start -time as local runtime identity and refuses to signal an unrelated process. -Auto-start policy is reconciled on Web startup and reload. A successful process -launch is not a substitute for a Group 0 service registration. Docker delegates -child recovery and status to `crowdb-monitor`. - -## 6. Web UI Backend (Axum) - -### 6.1 Design Rules - -The ordinary OCI entrypoint starts a node Web with an editable UI in both -one-member and three-member clusters. Automatic preparation and runtime choice -do not restrict management writes. Before bootstrap, candidate/rack preparation -uses retained local admission state; afterward, hardware and logical mutations -use confirmed Group 0 authority and authenticated monitor execution. Configured -managed Web retains its versioned process configuration and requires confirmed -records; unknown managed routes never enter an in-memory topology fallback. - -A mutation returns success only after the required node-side steps and Group 0 -publication are confirmed. The Console assumes a root operator until UI login -is introduced. Explicit `readonly = true` rejects non-read requests with 403; -`/api/mode` exposes that setting and the SPA disables writes accordingly. The -ordinary node default is writable. Credentials remain in the established private -store, and invalid authority, ownership or placement rejects the operation for -that cause. The SPA calls the Axum backend rather than KV management endpoints. - -### 6.2 In-process test API - -The in-process Web router and `--test-mode` retain fixture orchestration for -browser and integration tests. Their recursive physical views and monitor cache -help exercise the UI. Standalone Web uses these resource APIs with persistent -bootstrap/recovery inputs; `--test-mode` uses isolated ephemeral state. Neither -provides fallback topology for configured managed requests. - -### 6.3 Orchestration semantics - -For each multi-node operation in the logical tree, the backend obeys -these rules: - -- **Plan first, act second.** Resolve every required node + replica id - from Group 0 membership and live service registrations before issuing - mutation RPCs. Missing or ambiguous registrations fail the operation. -- **Built on physical primitives.** The orchestrator only calls the - per-node physical mutators; it never invents a side channel. -- **All-or-nothing where feasible.** On partial failure, attempt to - undo successful sub-steps and surface the resulting state in the - error body. -- **Confirmed membership publication.** Complete peer wiring before publishing - group or replica membership. A new group and its initial replica records - commit in one conditional batch. Create records conditionally; a concurrent - conflicting record is preserved. A lost write response is resolved only by - a linearizable read that confirms the intended record. -- **Deletion preserves authority on node failure.** Confirm deletion on every - hosting node before removing membership. Store hosts include nodes from - replica records as well as the store record. Remove descendants before - parents; an already absent node-side object permits retry. -- **Idempotent retries.** A repeat of the same logical request must - converge to the same state. -- **Read after write.** A mutation returns only after the required node-side - and Group 0 confirmation steps complete. - -### 6.4 Resolution rules - -- Unknown ids → `404`. -- Unreachable node for a physical-tree call → `502 Bad Gateway`. -- Partial logical-tree failure → roll back and report `409 Conflict` - with structured per-node outcomes (not `207 Multi-Status`). -- All handlers are thin wrappers around `shared` entry points; the - CLI calls the same entry points. - -### 6.5 Frontend contract - -The frontend SPA design lives in `design-crowdb-console-ui.md`. The -backend-facing contract here: - -- Bundle output is `app/crowdb-web/ui/dist/`; `crowdb-web` serves - it via SPA fallback. -- The SPA polls the management API on a short interval. An unavailable - authority clears stale logical rows and is shown explicitly. -- No `/api/cluster/snapshot` aggregate endpoint. - -## 7. CLI Design - -- `cluster start/stop` own OCI runtime lifecycle through a named local ownership - receipt. `start` accepts one or three nodes and delegates preparation to the - normal node monitor. Runtime inspection, logs and image operations use the - runtime's tools; cleanup belongs to the independent host/Pixi helpers. -- `cluster status/check` observe confirmed application readiness over management - endpoints, without a runtime socket. A finite check returns nonzero when any - required dependency is unmet. Runtime facts unavailable remotely remain unknown. -- Hardware, logical KV, S3 and benchmark commands require `--cluster` or - `--management-endpoint`. Every selected origin must report the same confirmed - UUID; saved bindings pin that UUID. Native consumers additionally verify - transport reachability and the committed Group-0 UUID before data access. - There is no system-address, localhost or launch-registry fallback. -- Hardware and logical KV operations call Group 0 and node management APIs - directly after target verification. Signed S3 clients obtain access credentials - and the advertised public URI from the selected management origin. -- Endpoint-only attachment grants no runtime ownership. It cannot start or stop - containers, create a runtime receipt, change the image or prepare infrastructure. -- The retained workload families are KV, DiskDB, ChunkDB, ChunkIO and S3. - `bench kv prepare` writes its declared dataset into an existing group. - Workloads never initialize groups, deploy services, reset cluster topology or - stop a cluster. Workload-owned data cleanup and ordinary client transport - workers remain allowed. Standalone RPC tests own their server/client lifetime. -- `kv store/group/replica/data` retain normal topology and data operations. - Process deployment and reset commands are absent from the command tree. -- Output identifies the copyable command and final result. HTTP exchanges show - response metadata and format structured bodies; binary bodies are identified - by type and byte count. Status/check support structured JSON output. -- Ordinary commands log only to the console. A benchmark writes the latest run - under the runtime root's `artifacts/cli/bench-` directory; the next run - replaces only that family's previous benchmark log. KV workloads also retain - a structured `report.json` with operation counts, errors and measured latency - alongside the human-readable console report. Measurement tools preserve each - completed invocation before another run replaces the latest directory. - -## 8. Error Model and Operation Logging - -- `shared::Error` enum covers `NodeUnreachable`, `UpstreamRpc`, - `Validation`, `NotFound`, `Conflict`. HTTP maps to 4xx/5xx; CLI maps - to exit codes (0 ok, 1 user error, 2 cluster/network error). -- **Console transcript** — every invocation prints the copyable command and - result. Direct S3 requests additionally show request/response headers and a - body marker. Text is visible, XML/JSON is formatted, and binary content is - summarized by byte count without duplicating it in the trace. -- **Retained diagnostics** — only benchmark commands write files. The latest - run for each benchmark family lives beneath the runtime root's - `artifacts/cli/bench-/` - and replaces the previous run. Benchmark tracing is also mirrored to the - console. Routine outbound HTTP calls are debug-level so default output keeps - lifecycle events, warnings, and errors without per-request noise. - -## 9. Observability - -- `tracing` everywhere; `--vv` switches CLI to debug. -- Web backend exposes `/healthz`. **`/metrics` is deferred**. The Rust - Prometheus story has multiple competing crates; we will pick one when - broader observability work for `crowdb-paxos-kv-server` begins. -- All console-issued operations attach a correlation id propagated as - `x-crowdb-paxos-kv-corr-id` to `crowdb-paxos-kv-server` request headers. -- Regression service metrics have a content contract: KV emits `rust`, - `cpp-rpc`, `cpp-tree`, and `misc`; DiskDB and ChunkDB emit `rust`, - `cpp-rpc`, and `misc`; DiskIO emits `cpp-rpc` and `misc`. A tracing or RPC - log may legitimately remain empty when its configured level observed no - events; metric validation therefore checks metric sections and counters - independently of auxiliary log size. - -## 10. Hardware mutations - -CLI and node Web use the shared Group 0 hardware operations. Rack, -node, disk-group and disk changes update parent and child records in one -conditional batch where membership changes. Matching retries are confirmed; -conflicting concurrent writes preserve the existing record. One-member and -three-member OCI nodes expose the same shared management surface after -confirmed Group 0 publication. - -The physical Cluster workspace shares one tree and selection across Topology and -Capacity. Cluster/Rack node cards and capacity tables reuse bounded observations; -Node storage owns discovery, accepted file/supplied operations and retained -candidates. Group 0 defines effective disk membership. Candidate Join validates -reviewed media and retains one immutable operation per disk; accepted work runs on -the owning monitor after the browser closes. Offline/Remove/Rejoin retains bytes -and allocation identity, with exclusion and validation gates before participation. -Physical discovery is never permission to reinitialize or replace an existing disk. - -Concurrent Access deployment reserves one cluster-scoped Console catalog -Initialize/Activate operation pair in Group 0. Nodes replay that pair; conflicting -local journals remain visible and are not rewritten. Access credential bootstrap -also reserves one encrypted user candidate before conditionally publishing its -access-key record. Response loss/concurrent nodes cannot replace an existing -credential; duplicate or disabled users require explicit review. Enumeration is -bounded and rejects truncated/timed-out results instead of authorizing issuance -from incomplete state. Secrets remain in the established private credential path. - -## 11. Cluster teardown and verification - -`cluster start`, `cluster stop`, `cluster status` and `cluster check` form the -common CLI lifecycle. Start and stop require matching private runtime ownership; -stop retains accepted inputs and data. Endpoint-only attachment supports current -observation and workloads without granting runtime stop or cleanup authority. -Independent OCI cleanup validates the recorded daemon, exact container and -resource identities before removing owned runtime resources. It preserves data -by default; explicit owned-data deletion is separate from stopping services. - -Application cleanup is a distinct authenticated operation. It validates the -retained bootstrap, retires further setup/recovery, drains owned services and -removes the matching system store through its local deletion endpoint. Failed -members remain pending for explicit retry; an unavailable node is never proof -that its membership or data should be deleted. No cleanup guesses orphan -ownership or reconstructs topology from host-local process records. - -Manual node discovery and bootstrap follow the -[deployment architecture](../deploy/design-crowdb-deploy.md). The Cluster sidebar -retains a separate candidate list and explicit draft, fixed bootstrap, publication, -active, stale authority and cleanup states. Each UI reads its local monitor's -observations and the shared Group 0 registry. Discovery filtering selects another -cluster for inspection without merging its membership. Authenticated node updates -retain UUID/numeric allocations and physical failure domains. Loss of Group 0 -blocks management; existing data groups use their own Paxos authority and confirmed -endpoint hints for data operations. diff --git a/doc/design/deploy/design-crowdb-deploy.md b/doc/design/deploy/design-crowdb-deploy.md deleted file mode 100644 index 8a93f0ace..000000000 --- a/doc/design/deploy/design-crowdb-deploy.md +++ /dev/null @@ -1,850 +0,0 @@ - - - -# CROWDB - Design: Multi-node Deployment - -Node containers run discovery, SSH, monitor and Web before cluster creation. -Group 0 owns confirmed topology and management operations. Linux amd64 images -are built by Pixi-managed standalone BuildKit and run on system Docker or -containerd/nerdctl. The [OCI image design](design-crowdb-oci-image.md) defines -construction, runtime resources and publication. - -The [console architecture](../console/design-crowdb-console.md) defines the -shared operation layer; the [Group-0 architecture](../kv/design-crowdb-paxos-kv-group0.md) -defines system metadata; the -[reconfiguration design](../kv/design-crowdb-paxos-kv-reconfiguration.md) governs -Paxos voting membership. This design connects those responsibilities without -making node discovery a membership protocol. - -## Table of Contents - -- [1. Goals and boundaries](#1-goals-and-boundaries) -- [2. Components and identities](#2-components-and-identities) -- [3. Invariants](#3-invariants) -- [4. Node discovery](#4-node-discovery) -- [5. Node and rack configuration](#5-node-and-rack-configuration) -- [6. Bootstrap and recovery](#6-bootstrap-and-recovery) -- [7. Shared UI and management operations](#7-shared-ui-and-management-operations) -- [8. Authorization and partitions](#8-authorization-and-partitions) -- [9. Image and runtime deployment](#9-image-and-runtime-deployment) -- [10. Resource and lifecycle management](#10-resource-and-lifecycle-management) -- [11. Configuration and observability](#11-configuration-and-observability) -- [12. Correctness and validation](#12-correctness-and-validation) -- [13. Supported scope](#13-supported-scope) - -## 1. Goals and boundaries - -- Each node has a monitor and a Web UI. Operators can use any member's UI; - there is no permanently designated UI host. -- Monitors discover nodes on a management LAN without a hand-written peer list. - Discovery identifies nodes, not the individual servers hosted on them. -- Before Group 0 is initialized, UI operations are restricted to node/rack - preparation and Group-0 bootstrap. Other server deployment requires a ready - Group 0 and confirmed initial topology. -- Group 0 is the shared information platform and authority for cluster - membership, racks and deployment operations. -- Production deployment uses Linux light containers: OCI images package - binaries/dependencies, while network, disks and performance resources are - explicitly assigned from the host. -- Single-host tests simulate multiple nodes using isolated containers on a - dedicated Docker network and independent simulated disks. - -The deployment layer is not a general scheduler. Kubernetes and Nomad are not -the default deployment substrate. OCI image compatibility allows other runtimes -to consume an image; it does not establish complete Kubernetes deployment -support. Cross-subnet discovery has a seed fallback and does not require a -multicast relay. Independent initialized clusters are never automatically merged. - -## 2. Components and identities - -```text -operator browser - -> any node's Web UI / shared console operations - -> Group 0: committed topology and operation authority - -> target node monitor - -> OCI node container and managed server processes - -node monitor <-> management-network discovery <-> peer node monitors -``` - -- **Node:** a deployment and hardware-management identity. A production node - normally corresponds to a physical Linux host; a test node may be a container. - A node exists before any KV server is started. -- **Monitor:** the node management endpoint, discovery participant and local - process/container supervisor. Reuse `crowdb-monitor`; do not introduce a - second agent with overlapping responsibilities. -- **UI:** a presentation/API entry point into shared operations. It owns neither - confirmed topology nor a private cluster. Losing the initiating UI does not - delete a submitted operation. -- **Server:** a managed service/process deployment on a node. Server deployment - is distinct from node admission and Group-0 voting membership. -- **Cluster identity:** the durable identity associated with one initialized - Group 0. A node cannot infer or replace it from nearby multicast announcements. -- **Rack:** an editable logical grouping of nodes. A node-supplied rack value - is an initial hint; confirmed placement belongs to Group 0. -- **Physical failure domain:** actual shared host/rack infrastructure used by - placement policy. Virtual rack labels do not manufacture hardware independence. - -Persist node identity, cluster binding and accepted bootstrap information in -the node's host-mounted state root. Persist Group-0 replica state through its -existing WAL/storage contract. These identities are not image contents and are -not derived from IP addresses. Existing protocol identifier types remain the -canonical types. Each node generates and persists a discovery UUID before -admission. Candidates are identified by UUID; bootstrap fixes the UUID-to-numeric -node-ID mapping in its accepted configuration. Subsequent admission allocates a -numeric node ID through Group 0 and durably records the mapping. Allocation and -mapping publication are conditional and idempotent under concurrent admission; -IDs cannot collide or be silently reassigned. Endpoint and rack changes do not -change either identity. Duplicate live UUID claims are identity conflicts. -Confirmed admission retains the Web listener proved over authenticated SSH. -Cluster observation uses that listener for each UUID, including containers with -shared host addresses and different port offsets. Retained default-port records -keep their existing serialized identity; each monitor republishes its own current -listener through the existing confirmed-node update operation. - -## 3. Invariants - -- **I1 — Durable identity:** recreation, endpoint changes and rack reassignment - preserve node identity. Conflicting live endpoints claiming one identity - cause a visible conflict, not silent overwrite. -- **I2 — Candidate discovery:** discovery records are observations. They grant - no admission, voting membership, rack authority or execution permission. -- **I3 — Group-0 authority:** confirmed topology and management mutations use - Group 0. A local monitor/cache never substitutes for unavailable authority. -- **I4 — Bootstrap binding:** every selected monitor durably accepts one - compatible cluster/bootstrap manifest before starting its initial replica. - Conflicting manifests are rejected. No timeout permits replacement bootstrap. -- **I5 — Authorized execution:** a connected Group-0 replica is not proof of - authority. Management mutations require quorum-backed authorization, and - stale operations cannot execute after revocation or authority transition. -- **I6 — Explicit failure domains:** virtual rack assignment and physical - failure-domain identity are distinct placement inputs. -- **I7 — Runtime portability:** a supported-architecture OCI image carries - code; runtime inputs carry identity, network, devices and persistent state. -- **I8 — Bootstrap gate:** only Group-0 prerequisite processes and Group-0 - initialization run before the shared information platform is ready. Other - deployment requires committed initial topology. -- **I9 — Identity-bound address recovery:** a changed address is usable only - after the peer proves possession of the SSH host private key pinned to its - accepted cluster UUID, discovery UUID and numeric node ID. Discovery cannot - change this binding or the accepted voting membership. - -## 4. Node discovery - -### 4.1 mDNS and DNS-SD - -Monitors announce and browse `_crowdb-node._tcp.local.` on the configured -management interfaces. Candidate expiry removes observations after five seconds; -it never removes a confirmed member. - -- mDNS transports local-link DNS queries/responses using UDP port 5353 and - multicast groups `224.0.0.251` / `ff02::fb`. -- DNS-SD uses PTR records to enumerate instances, SRV to identify monitor host - and port, A/AAAA to resolve addresses, and TXT for bounded metadata. -- TXT carries only discovery essentials: identity, protocol compatibility and - optional cluster binding. Full node/hardware/rack information is obtained - through the monitor management handshake. - -These mechanisms come from [RFC 6762](https://www.rfc-editor.org/rfc/rfc6762.html) -and [RFC 6763](https://www.rfc-editor.org/rfc/rfc6763.html). The standards define -discovery and caching; CROWDB defines admission and authorization separately. -Announcements, refresh, cache expiry and clean departure follow these mechanisms. - -Each monitor's discovered-peer cache feeds its local UI backend. Browser code -does not send mDNS traffic. Peers query/announce directly within the discovery -domain; there is no requirement for a separate gossip membership layer. - -### 4.2 Network scope and reachability - -Discovery runs only on configured management interfaces. Host-network nodes -use the management NIC/VLAN; Docker test nodes use their dedicated bridge -interface. The advertised monitor endpoint must be directly reachable by peers -in that scope. A wildcard bind address is not an advertised address. - -UI addresses exposed to browsers and monitor addresses used by peers are -separate endpoint purposes. In bridge tests, host UI ports are published while -peer monitor connections use container-network addresses. Production host -networking requires an explicit local port plan. - -mDNS does not normally cross routed subnet/VLAN boundaries. When unavailable, -configured peer/seed monitor endpoints provide the same handshake path. The -seed list is a discovery input, not an authoritative membership list. Monitor -connectivity verifies endpoint usability; discovery expiry alone does not -establish node failure or remove a confirmed member. - -### 4.3 Cluster separation and trust - -Classify a discovered peer as unbound, bound to this cluster, bound to another -cluster, incompatible or identity-conflicting. Only compatible candidates can -be selected for bootstrap/admission. A foreign cluster is displayed separately; -its topology is not imported automatically. - -Announcements are untrusted hints. The direct handshake and admission policy -must validate the management peer before privileged actions. Never advertise -credentials in TXT records. Admission requires authenticated management peers and -either an explicit submission or the workspace's enabled Auto join policy. The Cluster -tab shows a separate Candidate Nodes list below its left-side cluster view. -Moving a candidate into a cluster opens SSH access and rack selection. Before -bootstrap this updates only the visibly uncommitted draft; afterward admission -requires a committed Group-0 operation. - -All admitted nodes support mutual SSH access using Ed25519 keys. Each node -creates and persists its own `id_ed25519` private key; only `id_ed25519.pub` -is exchanged into the other nodes' `authorized_keys`. Private keys are never -copied between nodes or published in Group 0. Joining uses username/password -once when working key access has not already been provisioned. Verify SSH -host identity and its association with the candidate monitor before exchanging -keys. Install existing members' public keys on the candidate and its public -key on every existing member, then verify bidirectional key access before -confirming admission. Failed setup remains visibly pending and resumes the -same admission operation without duplicating keys. Pending nodes cannot receive -general deployments. Cancellation removes authorization entries installed by -that operation without removing unrelated keys. - -The multi-node Docker test profile provides an SSH login account with default -username/password `crowdb` / `crowdb`, prefilled by its UI. Production credentials -are supplied explicitly. Initialization passwords are discarded after key -setup and are not persisted in Group 0. Node keys, `authorized_keys` and SSH -host keys survive container recreation in persistent storage. Runtime service -accounts need not acquire an interactive shell merely to provide this login. -Every management entry point uses the locally provisioned SSH identity; Group 0 -stores only nonsecret connection metadata and credential references. Mutual -SSH access grants each node the configured login user's access to its peers. - -## 5. Node and rack configuration - -Before bootstrap, an operator can inspect nodes, create/name virtual racks and -assign nodes. Node-provided rack hints can be overridden. The UI labels these -values as an uncommitted bootstrap draft, not shared authoritative metadata. -Drafts in different UIs can differ; consensus-backed UI consistency begins only -after Group 0 and topology publication are ready. - -The accepted bootstrap manifest captures the chosen initial node/rack mapping. -After bootstrap, all rack edits and node admissions are committed through -Group 0. Rediscovery cannot overwrite operator-confirmed rack placement. - -Connection and rack edits authenticate the currently discovered UUID over SSH. -`NodeRecord` keeps its discovery UUID, numeric ID, admission operation and physical -host identity. `node_update` persists fixed source/target inputs in Group 0 before -changing hardware, refreshes peer RPC endpoints without changing voting flags, -then publishes the updated mapping. A retry from another UI resumes those same -inputs. Moving a node with disk groups is rejected until those groups are removed; -this prevents losing its hardware children during relocation. - -Physical host/failure-domain information remains independently inspectable. -Several test containers may occupy different virtual racks while sharing one -physical failure domain. Placement policy must not count those labels as -independent physical redundancy. - -## 6. Bootstrap and recovery - -### 6.1 State transitions - -The node's management lifecycle is: - -- **Unbound:** identity exists, discovery is active, no cluster is accepted. -- **Prepared:** a cluster identity and initial manifest are durably reserved; - the monitor accepts retries of that manifest and rejects conflicting ones. -- **Group-0 starting:** selected monitors provision/start their initial replicas - using the same fixed voting-member manifest. -- **Topology publishing:** Group 0 has usable quorum/leadership; initial hardware - and cluster metadata are being published idempotently. -- **Active:** shared topology is confirmed and general deployment is enabled. -- **Authority unavailable:** binding persists, diagnostics/discovery continue, - and new cluster-management mutations are disabled until authority recovers. - -Unavailability is not a transition back to Unbound. Clearing cluster binding -requires an explicit recovery/decommission procedure, not an automatic timeout. - -### 6.2 Initialization flow - -1. Any UI collects compatible candidates and the operator's rack assignments. -2. The operator selects initial Group-0 voting members, or Auto join selects - the explicitly selected one, three or five authenticated eligible candidates in - discovery-UUID order. Candidate - count is not voting-member count; application nodes need not all vote. -3. Submit the bootstrap identity, immutable initial manifest and initial data - group choice to the selected member with the smallest discovery UUID. Its - monitor durably accepts the request before the UI receives acceptance. -4. Start the minimum Group-0 services with the fixed manifest. Use existing - Paxos election and WAL durability, not discovery-cache membership. -5. Wait for quorum-backed readiness and publish initial metadata idempotently. -6. Persist/refresh cluster contact information for member UIs. Enable general - deployment only after initial topology is confirmed. - -The accepting monitor executes preparation, initialization, binding and optional -initial data-group creation independently of the browser and Web request. -It exposes the operation identity, fixed members, current phase and failure. -Closing a page or losing its connection does not cancel accepted work. Monitor -restart resumes interrupted work; a recorded failure requires explicit retry. -Retry preserves the accepted data-group choice when that option is omitted and -rejects changed inputs. There is no automatic initiator takeover. Cleanup rejects -an actively executing initialization. - -Monitor peer commands use the node's durable SSH key and the strict host-key -pins established during mutual admission. Console's separate TOFU store does -not substitute for those peer pins. Initialization completion is observed from -the monitor and quorum-backed publication, not inferred from HTTP acceptance. - -Bootstrap exclusion is enforced by each target KV server before creating -store 0 and covers the complete store-0/group-0 initialization sequence. The -server atomically and durably accepts the first bootstrap operation identity -and its fixed initial configuration. A conflicting operation is rejected before -creating or reusing store 0. Retries of the accepted operation resume unfinished -steps and verify existing content; an existing store 0 without group 0 is not -permission for another operation to take over. Restart retains the accepted -operation. Failure and timeout do not automatically release its ownership. -Monitor preparation and KV-server acceptance refer to the same operation and -configuration; neither UI-local state nor a generic store/group creation path -may bypass the exclusion. - -Every selected member must durably accept the same operation and configuration -before any initial replica becomes eligible for election or data publication. -Overlapping requests cannot both obtain all required acceptances. They may each -occupy part of their selected membership, in which case both remain incomplete -and the UI exposes the conflict for explicit cleanup and retry. This mechanism -guarantees per-node exclusion, not a globally unique cluster on the LAN. -Disjoint selected memberships may create independent clusters; UIs display -them separately and do not automatically merge their logs or topology. - -### 6.3 Recovery and subsequent admission - -Node workspaces persist an enabled-by-default Auto join policy. The Web backend -evaluates it without a browser connection through authenticated admission and -accepted deployment inputs. Local policy supplies a test rack and credentials; -Production requires configured rack assignments and access. Missing Production -credentials remain Needs access, without a Local fallback. Multiple bound cluster -UUIDs require explicit target selection rather than an arbitrary advertisement. - -The accepted initial voter count is one, three or five. An incomplete selected -set remains pending; an observed overfull fresh set reports conflict before acceptance. -Eligible UUID ordering selects the coordinator within that set, while monitor and -KV-server accepted-manifest fencing prevent conflicting execution. Fixed voters -never expand from later advertisements. Turning Auto join off stops new scheduling -while already accepted monitor work continues. - -After Group-0 publication, one committed default preparation intent owns the -ordinary metadata group, replica mapping and shared access credentials. The -replica mapping covers every fixed initial voting member; each receives its own -initial storage binding. Authorized workers resume that intent across -Web/coordinator loss. Each monitor interleaves -disk preparation with dependent service startup; DiskIO may start before Online -because its ownership proof enables Online. Explicit later service stops or group -retirement supersede historical setup and prevent automatic recreation. - -Background service recovery observes readiness once per pass, retaining the -same live child and durable launch attempt until readiness succeeds. Explicit -service commands retain their bounded initial startup wait. Local read-only -health probes run concurrently with their individual deadlines, and publication -rechecks actual owned process liveness and PID before reporting health. Slow -startup or several unavailable listeners must remain visible as unhealthy; -publishing a fresh timestamp cannot substitute for those observations. - -The versioned deployment policy is durable and independent of UI permissions. -Local and Production inputs cannot replace one another through an environment -change. Initial voter count, storage, listeners and scope are sealed before node -identity or setup effects. Retained values outrank new image defaults; corrupt, -conflicting or obsolete populated roots fail without implicit reset. - -Restart resumes accepted identity and replica WAL state. Lost multicast does -not erase stored Group-0 contacts. An interrupted bootstrap reuses its operation -identity/manifest; partial metadata publication resumes without admitting -unselected nodes or repeating destructive provisioning. - -A newly discovered node enters the candidate view. A committed admission -operation binds it to the cluster and assigns its rack. Adding a deployment -node does not automatically alter Group-0 quorum. Voting-member changes follow -the existing reconfiguration contract. - -The UI supports Group-0 member management and explicit group/store deletion -for bootstrap recovery. To consolidate independently initialized clusters, the -operator selects the retained cluster, cleans the other cluster's group 0, -store 0 and binding, then admits its cleaned nodes to the retained cluster. -Voting additions use reconfiguration and catch-up, not merging independent -Group-0 histories. Deleting a group or store alone does not silently release -bootstrap ownership or cluster binding. Cleanup must fence delayed commands -and prevent old replicas from restarting before allowing a new bootstrap; -accepted operations retain durable terminal state so stale retries cannot -recreate deleted resources. Unreachable nodes remain pending cleanup and are -not eligible for reuse. Existing committed clusters require explicit destructive -cleanup rather than cancellation of an uncommitted draft. - -The public fixed `PreparedBootstrap` manifest is a recovery record distributed to -selected monitors over authenticated SSH. It is not a global cluster registry or -an authority before consensus. KV-server durable acceptance arbitrates resource -ownership at each participant. A private Unix control socket handles local -monitor commands; the unauthenticated HTTP handshake only exposes observations. -SSH connection/authentication has a ten-second bound; bidirectional proof has a -twenty-second bound. Passwords are discarded, sessions are reused during setup, -and only public keys cross nodes. - -Each Web process sampler owns one read-only SSH connection per bound remote -node. Process observations retain their one-second cadence and three-second -deadline; connection reuse does not extend observation freshness. The sampler -checks the returned node and complete bootstrap identity on every response. -Connections are discarded after a command error, cancelled read, binding or -node configuration change, and are reauthenticated at least once per minute. -They use the provisioned node key and existing strict host-key verification, -without password fallback or sharing a session with management mutations. - -### 6.4 Authenticated member address recovery - -Node identity survives changes to management and data addresses. Each accepted -member has a durable binding between cluster UUID, discovery UUID, numeric node -ID and its Ed25519 SSH host public key. Admission retains the canonical public host key in its Group-0 node record; -normal address/rack updates cannot replace that pin. The SSH login key authenticates a caller; -the SSH host key authenticates the peer. Both private keys remain on their owning -node's persistent volume. Address recovery reuses SSH and does not introduce a -CA, certificate service or custom signature protocol. - -Initial trust is established by the existing bootstrap/admission procedure on -the deployment's trusted management network. Local test credentials are a -convenience, not production identity verification. An mDNS announcement alone -cannot establish trust. Before bootstrap replicas start, each selected member -durably accepts the same member-to-host-key bindings alongside its fixed -manifest. Subsequent admissions and revocations use committed Group-0 authority; -participants retain the applicable bindings locally for recovery without quorum. -A pending non-voter seals the initial manifest's authenticated pins and its own -admission pin before its first KV start. This does not authorize recovery: only -confirmation distributes `ConfirmedMemberTrust`, whose registry and pins each -monitor verifies through current linearizable authority. The retained snapshot -adds contact hints while preserving the original voting/store configuration. -The independently running contact worker refreshes it when authority returns; -its cached confirmed pins permit SSH identity proofs during quorum loss. -Current authority retirement durably fences the local admission before owned -KV/storage services stop. Missing or corrupt later-member snapshots cannot -resume the member or answer a recovery identity request. Discovery cannot promote a pending admission or revive a removed member. Missing -or corrupt trust records fail closed; a new address never triggers first-use -trust or automatic key replacement for an existing identity. - -Recovery proceeds independently of Group-0 availability: - -1. mDNS/DNS-SD, a configured seed or retained accepted contact supplies a - candidate management address. Retained contacts remain probe hints when - multicast or Group 0 is unavailable. Advertised identities and addresses - never authorize a route without fresh proof. Fresh discovery hints receive - the bounded probe budget before retained addresses, so a full old-address - cache cannot suppress a member's changed IP. -2. The monitor selects an already accepted member binding and opens SSH to that - address with the expected host key, rather than trusting the IP's previous - occupant or accepting a new key. SSH proves possession of the private key. -3. Through that authenticated connection, the monitor reads the peer's durable - discovery identity and cluster/bootstrap binding. All must match the accepted - member, and the verified reply supplies that member's current endpoints. - The SSH identity command reads private retained state and verifies the owning - monitor's independent liveness endpoint. It does not enter the serialized - lifecycle executor or wait for service readiness: otherwise slow service - startup could suppress peer proofs, fence routes and prevent its own quorum - recovery. Offline roots and changed or retired identities still fail closed. -4. The local transport uses the verified endpoints for the same replica IDs in - the existing groups. It does not change votes, membership, WAL, rack placement - or bootstrap identity. Endpoint replacement invalidates old connections and - applies to subsequent reconnects; mappings use node/replica identity, not a - global old-IP-to-new-IP substitution. - Restored replicas fence old routes before their election drivers start. - Consensus requests identify their intended cluster and replica; a reachable - process at a reused address cannot contribute another replica's response. -5. Once quorum returns, normal authorized operations publish updated addresses - in Group 0. Local endpoint evidence never authorizes a management mutation. - Committed replica-to-node ownership is retained locally for subsequent - recovery. Internal service listeners and management contacts change through - conditional new service operations; later stop/delete commands win. Public - client URIs are independent of these internal addresses. - -The contact worker checks retained retirement and accepted identity before -probing peers. After authenticating recovered routes, address publication -checks current authority through those routes before optional confirmation -refreshes retained contacts. Publication and confirmation share a three-second -authority budget; SSH proofs retain their ten-second bound. Unreachable old -authority contacts must not consume the budget before recovered addresses can -be published, nor add another full wait that expires otherwise valid routes at -their fifteen-second freshness fence. Current authority still fences removed -members and conditional service operations before publication. - -This permits all members to restart with changed IPs at once. Neither stable DNS -names nor fixed IPs nor a complete peer-address list is required. Multicast must -reach the selected management scope; across discovery boundaries, seed discovery -remains necessary. Docker bridge DNS is optional and host networking uses the -same monitor discovery and SSH verification path. - -A changed key, foreign cluster/bootstrap, unknown member or conflicting live -claims is rejected and reported. An unverified advertisement cannot overwrite a -verified endpoint. Duplicate authenticated live claimants for one identity stop -automatic selection; a copied data volume is not a new node. Recovery has bounded -handshake timeouts and retains the last accepted identity on failure. Key rotation, -lost-identity replacement and removal remain explicit authority-controlled work; -they are never inferred from address changes or discovery expiry. - -### 6.5 Voting replica admission - -Node admission grants hardware/service management access; it does not add a voter. -Adding a replica uses `JoinGroupRequest` to import a snapshot from a confirmed -leader. Existing peers first register the new replica without voting rights. -The target wires existing peers and catches up through WAL until its contiguous -applied frontier reaches the leader's observed frontier. Only then is the new -replica promoted and published with store membership in Group 0. Once promotion -begins, an ambiguous response retains the caught-up replica for retry rather than -deleting a replica another peer may already count in its quorum. - -System replica join uses identified system ownership before creating store 0. -Generic join cannot bypass an accepted system bootstrap. Independent clusters -remain separate: operators explicitly clean one cluster, wait for every node's -cleanup, admit the released nodes to the retained cluster and add voting replicas. - -### 6.6 Accepted node setup and storage - -Node access review reads authenticated identity and inventory without installing -peer authorization or granting admission. The browser keeps an initial password -only in request/session memory. Final admission retains the original review -operation and sends the selected storage/services to the target monitor. - -The monitor persists an immutable `NodeSetupRequest` and sealed placement before -starting work. Browser disconnect, another Console session and monitor restart -reuse that intent. Provisional disk discovery grants no membership: only ready -Group 0 can register disks. Local default storage is one 16-GiB sparse file with -a 1-GiB reserve; Production automatic storage requires preallocation. Supplied -or diskless choices never silently fall back. -Different media classes use compatible independent placement. A chosen missing -ordinary metadata group remains waiting instead of creating an unrelated group. - -Each supplied medium has a stable operation, opened identity, canonical physical -resource and physical size. Whole-zone logical size excludes any unusable tail. -The monitor verifies every usable byte while retaining its exact descriptor and -exclusive raw-device claim through Init publication. DiskIO acquires and validates -its own descriptor before Online. Nonzero contents fail without a wipe; interrupted -scans start from zero. Already-Up replay checks its retained receipt and does not -mistake existing user bytes for a new blank medium. - -Global Group-0 physical reservations are sticky across interruption and Remove, -keyed by physical host and canonical resource. File/loop aliases share the same -resource. Unknown legacy raw lineage, unknown host or physical membership without -a backing receipt requires explicit migration. Numeric DiskGroup IDs also have -retained global placement reservations. Retry never chooses a different group or -medium after acceptance. Per-disk failures preserve successful siblings. - -Offline requires current revision/owner exclusion and I/O quiescence. Remove -retains backing identity, geometry and allocation metadata. Rejoin restores the -same disk Offline and requires validated Online. Discovery and restart never -replace missing/reassociated media or auto-online a restored path. Recovery and -evacuation remain separate operations. -Retained receipts also prevent parent deletion after Remove. Group deletion -requires an unassigned scope with no retained media, rather than silently -forgetting ownership, bindings or recovery inputs. -Public HTTP owner/binding routes assign or replace; they do not expose explicit -unassignment. A scope blocked by these records remains retained. Forgetting media -and unassigning their parents require a separate explicit operator flow; deletion -must not emulate that flow with an implicit cascade. - -Explicit service Stop/Restart/Delete supersedes old setup intent. Consecutive -process launches have a durable five-attempt budget per operation; readiness -success resets it. A bounded readiness observation that expires while the -same operation's process is alive retains that process and its attempt count. -Further observations continue startup without stopping or launching it again, -including on the fifth launch. Recovery verifies the committed local dependency -intents and their live probes before launching a dependent process. Consumers -that need the complete DiskIO topology also wait for its current authoritative -owner view, including other nodes. Waiting for these dependencies consumes no -process-launch attempt. A new explicit operation can supersede it. -Explicit cluster cleanup retires and drains accepted -setup/provisioning before deleting system metadata. Cleanup dispatches to at most -four retained nodes concurrently and saves each result; unreachable nodes stay -pending under the original confirmed cleanup operation. -Short-lived monitor control and probe commands do not create service-sized -worker pools. Service worker settings apply to the long-running node modes. - -## 7. Shared UI and management operations - -The UI presents explicit lifecycle states: - -- **Unbound draft:** local temporary topology and a separate candidate list; - SSH preparation and rack edits do not constitute committed membership. -- **Bootstrap in progress:** per-node store-0/group-0 progress and the fixed - submitted configuration. Later draft edits cannot alter that operation. - Failure offers resume of the same operation or explicit cleanup. -- **Topology publishing:** quorum exists but initial publication is incomplete; - show initialization progress and keep general service deployment disabled. -- **Active:** load confirmed Group-0 topology and replace the temporary view. - Subsequent changes use Group 0; unused drafts are never silently imported. -- **Authority unavailable:** retain the last confirmed topology with an explicit - stale/unavailable indication and diagnostics. Disable authority-dependent - operations and never return a bound node to replacement-cluster creation. -- **Multiple clusters discovered:** show distinct clusters and let the operator - select one to manage; neither independent topology nor drafts auto-merge. - -Admission with partial SSH setup is shown as joining/recovery required, with -operation progress distinct from confirmed membership. Retry installs only -missing authorization and resumes verification. Cancellation removes only -public-key authorization installed by that operation, preserving pre-existing -or concurrently required trust; failure to finish cleanup remains visible. - -All UIs invoke the same shared console operations and read confirmed state -from Group 0. Consistent reads support action confirmation; watch/refresh keeps -presentation current. A temporarily stale screen does not create a second -authority. The UI displays stale/unavailable state explicitly. - -Management operations have durable identity, target, requested action and -execution outcome. Group 0 commits the management intent/authorization; the -target monitor performs the side effect and records its result. This is not -an atomic transaction between a KV write and a runtime action. - -Explicit Restart requests each receive a new operation identity, even when -the requested service configuration is unchanged. Replaying that same identity -resumes its recorded execution; matching configuration cannot turn a later -Restart into a no-op. Repeated desired Start/Stop/Delete states may resume -the existing operation. - -Retries therefore use operation identity and local execution recovery. Crash -after runtime success but before result publication must inspect runtime state -and complete/reconcile the same operation, rather than blindly repeat it. -Image digest and deployment-instance identity distinguish the intended instance -from a reused process ID or container name. - -Changing UI hosts does not require moving cluster state. Local secrets and -runtime observations are not replicated as public topology. Credentials needed -to authorize another UI must be provisioned through the chosen trust policy. - -## 8. Authorization and partitions - -For a three-voter Group 0 partitioned two-to-one, only the two-voter side can -commit management mutations. The minority cannot use its local replica or -monitor cache as replacement authority and cannot bootstrap a new Group 0. - -The monitor checks cluster binding and the complete current service intent -through linearizable Group-0 reads before executing a management command. Each -read has a three-second bound. It checks the intent again after stopping an old -process and before starting its replacement. A superseded operation or unavailable -authority rejects execution. Recovery reads current desired intents through the -same authority path and does not execute a retained command solely from disk. - -An exact completed KV Stop/Delete can return its retained receipt while that -child remains stopped. The receipt carries the bootstrap identity under which -the operation was authorized; it must match the current accepted local binding -and complete intent, with no live replacement, cancellation or retirement. -This reports a prior effect without executing another one. It must not require -the Group-0 leader that the operation already stopped: background recovery may -finish the intent before its identical SSH request reaches the executor. -Legacy receipts without that bootstrap proof still require current authority. -Incomplete, superseded and new operations retain every authority fence above. - -Admission commands similarly match the current cluster publication and UUID, -numeric ID, operation ID and cancellation state. Local durable cancellation and -retirement markers reject delayed commands after cleanup or restart. Long-running -data ownership uses its existing expiry and fencing contract rather than deriving -an extended lease from these management checks. - -Monitors and UIs remain available for diagnostics during authority loss. -Group-0 unavailability does not by itself stop existing data services. Each -data group continues only under its own Paxos authority and read/write rules. -Operations requiring current Group-0 authority fail, including management -mutations, keep-alive updates, ownership acquisition/renewal and ownership -balancing. Failure to obtain ownership content is an authority-unavailable -result, never an empty ownership set; it cannot trigger release or reassignment. -Existing ownership remains subject to its own expiry and fencing contract; -actions requiring renewed ownership stop when that authority expires. Cached -ownership cannot extend its validity indefinitely. -Group-0 replica processes must be allowed to recover quorum even while general -deployment is gated; stopping all replicas on authority loss would prevent -recovery. - -## 9. Image and runtime deployment - -### 9.1 Common OCI image - -Publish OCI images to Docker Hub or another OCI-compatible registry. The same -image for a supported CPU architecture contains monitor/UI/server binaries and -their dependencies. Node-local state and cluster identity are external mounts. -Single-node and multi-node operation use the same image, monitor and UI. -Fresh Local startup prepares an explicitly one- or three-voter cluster through the -shared authenticated node lifecycle. The default is one voter with file-backed -test storage and zero physical host failure tolerance. Production selects explicit -credentials, resources and protected placement. Both expose editable management; -automatic preparation and node count do not determine authorization. Aggregate -status distinguishes Group-0 authority, historical setup and current full-cluster -readiness. Host helpers create runtime resources and observe this lifecycle; -they do not provision application topology through a second bootstrap path. - -Tags identify releases for humans; production plans pin immutable image digests. -Architecture-specific builds may share a multi-platform image index only when -their dependencies have been validated. - -System Docker and Pixi-supplied containerd/nerdctl manage the node container. -The monitor supervises local managed processes; external runtime lifecycle -recovers the whole node. Retained service instances restart in their release -profile's dependency order, with instance names used only to order peers of the -same service kind. Shutdown uses the reverse dependency order. -Standalone BuildKit explicitly exports and validates -OCI media types and blob digests without Docker or containerd dependencies. -The [OCI image design](design-crowdb-oci-image.md) defines shared construction, -resource verification and publication without changing cluster semantics. - -### 9.2 Production profile - -- Linux host, system Docker or rootful containerd runtime and host networking. -- Each node runs its monitor and Web UI inside a CROWDB node container. - The node container is the management boundary and controls managed services - using explicit mounted resources and local process supervision. Application service containers - do not inherit that runtime access. An external host/runtime startup mechanism - starts and recovers the node container itself; it cannot recover itself after - its own termination. Node identity, SSH identity and management state use - host-persistent mounts. -- Explicit management/data-plane network selection; discovery uses management. -- Stable disk/device identity and explicitly permitted block-device access. -- Host-persistent config, node state, server state, logs and crash artifacts. -- Explicit CPU quotas, memory limits and permitted block devices; the launcher - validates inputs before creating the container and verifies applied resources. - -Host networking removes a container bridge/NAT layer but does not itself -guarantee high performance. Disk passthrough does not itself bypass filesystem -cache: the storage engine's raw-block/direct-I/O path defines that behavior. - -### 9.3 Single-host test profile - -- One CROWDB container per simulated node on a dedicated user-defined Docker - bridge, each running its own monitor, Web UI and SSH endpoint. Starting three - node containers enables mutual discovery without pre-entering peer addresses; - opening any published UI exposes the candidate list and cluster preparation. - Docker availability does not require an implicit runtime socket mount: the - monitor manages local processes for this profile, while future external runtime control - requires separately configured runtime access. -- Unique persistent roots and identities; independent simulated disk files or - loop devices, never overlapping writes to one backing disk. -- Identical internal listener ports are allowed because network namespaces - differ; browser UI access uses distinct published host ports. -- Actual multicast discovery is exercised within the bridge. Separate networks - isolate independent discovery tests. - -Docker bridge behavior is described in the -[Docker reference](https://docs.docker.com/engine/network/drivers/bridge/). -The test profile measures functional correctness, not raw-disk production -performance. Linux host-network acceptance separately validates explicit resource -inputs, credentials and persistent replacement using the same image. macOS-hosted Linux containers require a VM/network -setup that is tested explicitly; image portability does not make the VM's LAN -multicast behavior equivalent to Linux host networking. - -## 10. Resource and lifecycle management - -Containers are packaging, resource/isolation and upgrade units. Deployment -plans explicitly assign host resources; monitors validate and execute, rather -than autonomously choose global placement. - -- **CPU/memory:** the production launcher validates CPU quotas and memory limits - against host capacity, then verifies the selected runtime applied them. Memory is at least - 512 MiB. Hardware observations respect container limits. CPU affinity, NUMA - and IRQ tuning remain host policy. -- **Disk:** stable device references, exclusive intended ownership and supported - engine access modes. Important data never lives only in the writable layer. -- **Network:** host ports and management/data interfaces are explicit. - RDMA device access and TCP fallback remain the responsibility of their - transport contracts, not the discovery mechanism. -- **Lifecycle:** monitor-managed services support explicit deployment, start, - stop and restart using shared operation records. Container replacement or - image upgrade is performed externally with a pinned digest and the same - persistent root; storage/protocol compatibility remains required. -- **Monitoring:** container and process health are separate observations. - Existing monitor supervision can recover eligible process exits; it must not - silently override a deliberate stop, expired authorization or upgrade drain. -- **Runtime access:** the host selects system Docker or rootful containerd. Node containers - supervise local processes and receive no Docker or containerd socket. Explicit - permitted block devices are passed through by the host launcher. - -No automatic global placement/config reconciliation is required. Monitor -detects drift against confirmed deployment inputs and reports it; local -liveness recovery follows explicit policy. Repair/upgrade progress and failure -remain visible to every authorized UI through shared operation records. - -## 11. Configuration and observability - -Local startup inputs are limited to bootstrap prerequisites and process/runtime -settings: persistent root, management interfaces, listen/advertise addresses, -optional discovery seeds, trust inputs and runtime endpoint. Group 0 owns -confirmed cluster topology and management intent. This does not move every -process tuning parameter into Group 0; respect the existing -[configuration architecture](../config/design-crowdb-config.md). - -Expose: - -- Discovery peer state, endpoint reachability, last observation/expiry, - incompatible identities and multicast-interface status. -- Bootstrap manifest identity, preparation/start/publication progress and the - reason general deployment is gated. -- Group-0 authority/renewal status, rejected stale operations and recovery needs. -- Desired deployment inputs, actual image digest/resources, process/container - health, drift and operation outcomes. - -`cluster status` reads the node's aggregate HTTP observation without a local -runtime socket. Its collector has an six-second monotonic deadline after the -bounded local monitor observation. Current linearizable Group-0 publication, -accepted ordinary metadata replicas and ordinary-group authority, physical disk -placement/bindings, live DiskDB owner leases and allocatable capacity are -independent dependencies. Automatic file storage uses its accepted initial group -reservation. Supplied storage resolves every accepted disk ID from a fresh, -identity-checked member setup observation through its unique Group-0 disk and -group membership. All resulting media-class groups must satisfy metadata -binding, current owner and space checks; a missing, duplicated or conflicting -medium stays unknown. Observation never changes sealed placement or creates the -automatic reservation for a supplied-media node. Capacity summaries must be no older than the service -registry's fifteen-second liveness window. Per-member service evidence matches -the current committed operation; required S3, Iceberg and Dataset listeners each -supply their own read-only protocol proof. Historical setup completion cannot -substitute for these observations. Missing or stale evidence remains unknown. -A bounded observation may wait for a slightly faster peer's timestamp within -its existing monotonic deadline and that evidence's freshness window, capped -at three seconds. It then evaluates the unmodified timestamps against the -actual local clock. It never treats future evidence as current, extends a -request deadline or refreshes old evidence to compensate for clock offsets. -Runtime/image facts remain unknown when runtime access was not supplied. - -Access dependency evidence comes from one backend sampler, independently of -status reads and business request outcomes. ChunkDB evidence uses one routed -point read in each of the system and user-data maintenance domains. An -authoritative absent key is healthy evidence; routing, storage, ownership and -protocol failures remain unhealthy. These observations never create probe -objects or enumerate the chunk catalog. Hardware discovery and bounded header -reads on online disks share the same overall probe deadline. Phase latency -histograms retain failed and cancelled observations and are readable without -issuing new probes. - -The aggregate route bypasses deployment recovery middleware and never reserves -preparation, restores configuration or repairs topology. Valid unhealthy status -returns success; `cluster check` returns one while readiness is unmet and zero -only for current ready evidence. Invalid or unobtainable reports return two. -Checks support bounded waiting, and JSON stdout contains only the report while -diagnostics use stderr. Explicit origins must agree with the selected cluster -UUID and name; attaching never grants runtime cleanup ownership. - -Keep local logs/crash artifacts persistent and collect asynchronously. Do not -replicate every log line into Group 0 or expose credentials in discovery/UI -diagnostics. - -## 12. Correctness and validation - -- Restart and endpoint changes preserve identity; duplicated roots cause a - visible conflict (I1). -- Three monitors discover each other on a real test bridge and LAN; expiry - affects candidates, not confirmed topology (I2). -- Multiple UIs can edit local drafts, but only the accepted manifest becomes - shared topology. General deployment stays gated through incomplete publication - (I3, I8). -- Concurrent overlapping bootstraps, partial preparation, initiator crashes - and interrupted publication recover without conflicting replicas or identity - replacement (I4). -- Partition tests distinguish reachable replicas from quorum authority and - deliver delayed/retried commands to verify stale-action rejection (I5). -- Multiple virtual racks on one host preserve one physical failure-domain - identity (I6). -- Docker bridge and Linux Docker host-network profiles use the same image - digest and exercise restart, persistent disk/state and resource boundaries - (I7). -- Existing Group-0 membership changes remain governed by reconfiguration; - deployment-node discovery never changes a voting set. - -Before adoption, measure discovery latency/traffic and stale-view bounds at the -intended LAN size. Broad ecosystem use of mDNS is not proof of unlimited -single-domain scalability. - -## 13. Supported scope - -The discovery identity and monitor placement decisions are settled: durable -UUIDs identify candidates, Group 0 owns admitted numeric IDs, and each node's -monitor/UI run in its node container. No human decisions remain open in this design. Additional runtimes, -architectures and Kubernetes orchestration remain outside this supported scope. diff --git a/doc/design/deploy/design-crowdb-oci-image.md b/doc/design/deploy/design-crowdb-oci-image.md deleted file mode 100644 index a216a22d8..000000000 --- a/doc/design/deploy/design-crowdb-oci-image.md +++ /dev/null @@ -1,374 +0,0 @@ - - - -# CROWDB - Design: OCI Image Build and Runtime - -Depends on: [deployment architecture](design-crowdb-deploy.md). -Satisfies: Docker-free, Pixi-managed Linux image construction and shared images -for Docker, containerd and future Kubernetes deployment. - -Linux amd64 construction uses Pixi-managed standalone BuildKit. Docker and -containerd consume the shared image; explicit probes and resource verification -cover both runtimes. Operational commands and tested versions are in the -[shared image guide](../../../container/oci-image/README.md). -macOS implementation and Kubernetes deployment automation are deferred. - -## Table of Contents - -- [1. Scope and invariants](#1-scope-and-invariants) -- [2. Image and base system](#2-image-and-base-system) -- [3. Pixi-managed construction](#3-pixi-managed-construction) -- [4. Artifacts and publication](#4-artifacts-and-publication) -- [5. Docker deployment](#5-docker-deployment) -- [6. Lightweight Linux deployment](#6-lightweight-linux-deployment) -- [7. macOS and Kubernetes extension points](#7-macos-and-kubernetes-extension-points) -- [8. Validation and supported boundaries](#8-validation-and-supported-boundaries) -- [9. External references](#9-external-references) - -## 1. Scope and invariants - -- **I1 — Docker-free build**: a Linux amd64 host with Pixi and documented kernel - prerequisites can compile, stage, construct and validate an OCI image without - Docker CLI, Docker daemon, Docker socket or containerd daemon. The public - shortcut selects this archive-only behavior explicitly; its default also - loads the validated artifact into local Docker. -- **I2 — One image**: for a given architecture and release, Docker and containerd - consume the same published image digest. Local and Production formation use - accepted startup policies rather than separate image builds. -- **I3 — External state**: UUIDs, SSH keys, cluster bindings, server data and logs - remain outside image layers. Replacing a runtime retains persistent state. -- **I4 — Enforced resources**: production startup fails if requested CPU - policy, memory limits or host networking cannot be applied and verified. -- **I5 — Verified publication**: publication copies the verified image without - rebuilding its root filesystem or converting its manifest media types. -- **I6 — Bounded lifecycle**: build processes have isolated sockets, owned state - and explicit cleanup. Cleanup never terminates unrelated runtime services or - deletes node data or shared image caches. - -Initial build/output platform is Linux amd64. Running these images on another -architecture requires an explicitly supported emulator or a separately verified -architecture build; an OCI index alone does not provide architecture support. - -## 2. Image and base system - -Ubuntu 24.04 remains the runtime base, pinned by digest. OCI specifies image -layout, configuration and layers; it does not require a particular distribution. -Ubuntu supplies the existing SSH, certificate, shell and system-library runtime. -Keep the current runtime dependency checks and UID/GID 10001 data ownership. - -Tini runs as PID 1 under both runtimes and reaps adopted orphan processes, -including SSH session children. It forwards signals to the monitor child, which -retains ordered service shutdown and the exit status of its own children. -The monitor does not perform an indiscriminate wait that could consume an owned -Tokio child status. This uses Tini's standard reaping/signal behavior as documented -in its [upstream implementation](https://github.com/krallin/tini). - -The image contains the same monitor, UI, Rust/C++ servers, shared libraries, -templates and entrypoint for Local and Production deployment. Fresh Local startup -defaults to one initial voter, automatic file-backed test storage and all three -access protocols. Three-container Local startup explicitly selects three voters -on one dedicated discovery scope; missing peers remain waiting. Local supplies -clearly scoped test admission credentials and reports zero physical host failure -tolerance. Production requires configured credentials, explicit resources and -valid protected placement. It never falls back to Local credentials or geometry. - -The monitor seals the versioned deployment policy before identity, admission or -storage effects. Retained accepted values supply omitted inputs on restart; -conflicting overrides or populated obsolete roots fail without conversion or -reset. Both policies expose the same editable UI and authorized management -operations. Initialization occurs through the shared durable node lifecycle, -independently of host helpers or browser connections. Image construction creates -no node identity or initialized storage. - -BuildKit's Dockerfile frontend remains usable, including existing bind mounts in -`RUN` instructions. The recipe name does not imply a Docker daemon dependency. -The final image uses OCI manifest, configuration and layer media types and -standard image annotations. Digest verification checks the actual exported -objects, not only `org.opencontainers` labels. - -Docker-specific `HEALTHCHECK` is not a portable OCI configuration field. Runtime -health checks invoke `crowdb-monitor liveness` and `crowdb-monitor readiness` -explicitly. Container restart policy and application readiness are distinct: -unready Group 0 must not trigger destructive reinitialization or restart loops. -On retained startup, the monitor validates sealed member trust and starts the -owned KV process without waiting for quorum health. Management discovery and -authenticated member identity remain available while the replica recovers its -peers' current addresses. Independent HTTP health observations determine KV -readiness; a live unready process does not consume another restart attempt. -Explicit admission/start actions retain their bounded readiness wait. -Background protocol observations obtain the local node's current address and -service intents from Group 0 within one shared authority deadline. Cluster, -node and discovery identities must match a confirmed, non-cancelled record. -The retained UI topology is not the address authority for these local probes; -status readers only consume the resulting observations. - -## 3. Pixi-managed construction - -Standalone BuildKit uses its OCI worker with `runc`; containerd is unnecessary -for construction. Pixi manages locked versions of `buildctl`, `buildkitd`, -`runc`, and the image inspection/copy tool, Skopeo. Rootless build -also needs RootlessKit and any required snapshotter helpers. Existing Rust, -CMake, Node and packaging tools continue to use the repository's locked Pixi -environment. - -A separate Linux-only Pixi workspace isolates the tools from macOS and ordinary -application development. Conda-forge supplies runc and Skopeo. Checked-in Pixi -source-package recipes supply BuildKit, RootlessKit, containerd, nerdctl, Syft -and Cosign using pinned upstream release checksums. Packages build locally; -an external project channel is unnecessary. No implicit apt/Homebrew or -unverified download bootstraps tools. - -The public build operation has three stages: - -- **Compile/stage**: on Linux, compile release artifacts with the current FFI - feature contract and assemble a source-free runtime directory. This stage - retains Cargo/CMake/npm caches and does not inspect any container daemon. -- **Construct**: start a task-owned BuildKit instance, wait for its private Unix - socket, run the Dockerfile frontend and explicitly export OCI output. Keep - cache/state separate from node data. Propagate errors and clean up the owned - daemon on success, failure and interruption. -- **Validate**: inspect layout, media types, blob hashes, platform, revision and - version; reject incomplete output. Record the image manifest/index digest - separately from archive checksum. Successful output is atomically published - to the requested local artifact path. - -Rootless construction is preferred, subject to user namespaces and snapshotter -support. Before compilation, preflight exercises the pinned RootlessKit with the -same user namespace and mount setup used by the builder and reports complete -startup diagnostics. Dependency installation on Linux amd64 and the public OCI -build entry point set up a per-executable AppArmor profile through sudo when -the namespace probe identifies the host's AppArmor restriction. Setup validates -the profile before installation, keeps checkout paths distinct, leaves global -policy unchanged and verifies namespace startup after loading it. Existing -working hosts require no privilege prompt. Noninteractive runs fail promptly -when sudo authentication is unavailable. No automatic privileged builder -fallback is allowed: a documented administrator-enabled build mode can be -selected explicitly if rootless support is unavailable. Pixi supplies userspace -tools, not kernel capabilities or privilege delegation. - -The common build component lives under `container/oci-image/`. The existing -single-node component delegates staging and image construction to it while -retaining its startup conveniences and acceptance tests. Docker import is a -separate step after Docker-free construction and validation. The public shortcut -performs that import by default and accepts an archive-only option that avoids -all Docker access. The same split supports a -future Linux-VM build backend without putting host-platform branches in the -image recipe. - -## 4. Artifacts and publication - -The canonical local artifact is an OCI image layout/archive with a build receipt: -source revision, project version, platform, base digest, tool versions, runtime -content checksum, OCI root digest and archive checksum. Runtime content and -final image are independently identifiable. Timestamps/package repositories may -affect reproducibility; this design does not promise byte-identical rebuilds. - -Reuse the existing release workflow and release-branch/version policy. It gains -separate construct, runtime verification and publish jobs: - -- Construct through Pixi without Docker access or registry publication secrets. - Upload the OCI artifact and receipt; do not rebuild from staged files later. -- Verify the same artifact on system Docker and on containerd/nerdctl. At least - one CI environment has no Docker installation; otherwise accidental dependency - on Docker cannot be excluded. Each job emits the verified OCI digest. -- Publish only after both runtime gates pass, with the existing protected - publication environment and current-release-head check. Copy the exact OCI - graph using digest-preserving transfer; failure to preserve it is an error. - Verify the registry digest before signing or updating moving tags. - -The canonical Docker Hub repository is `crowdb/crowdb-node`. The Iceberg, S3 and -Dataset repositories are usage names for the same image digest and version tags. -Repository names change neither the accepted deployment policy nor UI permissions. -All three access protocols use the ordinary node lifecycle. Existing populated -state retains its accepted policy through image replacement. - -Keep signing, SBOM and provenance. Generate provenance from the source/tool/base -receipt and attach attestations to the verified digest without rebuilding the -image. Distinguish the runnable image digest from any attestation-bearing index -digest, and verify both where an index is published. Failures leave the verified -artifact available for retry; credentials never enter layers, logs or receipts. - -## 5. Docker deployment - -Use the host's Docker installation. Pull the OCI image from the registry by -digest, then start it with the existing entrypoint/environment/mount contract. -Production uses `--network host`, explicit CPU policy and `--memory`, a persistent -data directory, physical host identity, management interface and credential -mounts. Bridge-mode single-host simulation remains available for tests. -Fresh Local one-node usage needs only the published client/UI ports and persistent -data mount. Three-node Local usage selects three voters without peer addresses or -fixed IPs. Both use automatic authenticated preparation and remain editable. -Production resource/device arguments are separate explicit host inputs. - -The standard monitor accepts private `PrepareDiskFile` requests only for its -confirmed node/cluster binding with available Group-0 authority. It records the -request before allocating a generated file asynchronously; control connections -and service heartbeat processing do not wait for preallocation. One allocation -runs at a time. `DiskFilePreparation` reports accepted, creating, prepared or -failed, including the retained backing identity. Preparation alone grants no -disk membership or usable capacity. - -File preparation checks available filesystem space including the requested host -reserve. Preallocation errors remain failures; sparse allocation must be explicit. -Retry preserves the original intent and verifies the recorded inode. Completed -files are never recreated or resized. An existing file without a durable ownership -receipt is rejected, including after an interruption between creation and receipt -publication. Cleanup is blocked while allocation runs; shutdown waits for the -blocking allocator to finish its durable record. New storage still requires -Group-0 registration and runtime availability confirmation. - -Monitor restart preserves a managed file's filesystem/device identity, inode and -logical size. A preallocated file must remain fully allocated, but its allocated -block count is an observation rather than identity: filesystem extent metadata -can grow as services write. Additional allocated metadata does not indicate a -replacement medium. - -The persistent data mount contains node identity, bootstrap/service journals and -managed disk files. Supplied files and whole block devices use the same container -disks directory in Docker and containerd, with launcher-verified exact mappings. -Replacement retains the original sources and identities; invalid inputs never -trigger managed-file fallback. Local defaults to one 16-GiB sparse managed file -and a 1-GiB host reserve. Production automatic storage uses preallocation and -rejects sparse files. File preallocation failure never authorizes sparse fallback. - -New managed files use 1-MiB allocation units and whole 256-MiB zones. For -capacities not divisible by 256 MiB, the zone size is halved until it divides -the capacity, down to the 64-MiB bitmap-word minimum. The default 16-GiB file -therefore has 64 zones, leaving non-active zones available for background -reclamation while four zones accept allocations. The accepted preparation -persists this geometry: retries and restarts never repartition existing files. -Files smaller than five minimum-size zones cannot provide a spare zone for this -background rotation; their capacity remains valid, but reclamation may wait for -allocation to rotate the full active set out. - - -Supplied provisioning seals the canonical physical host/resource and physical -capacity separately from whole-zone usable geometry. A full-file loop device -shares its backing file's canonical resource; partitions, loop slices and ambiguous -stacked/virtual lineage are rejected. The monitor reads every usable byte before -Init membership, and DiskIO verifies its own opened descriptor before Online. -Reservations survive interruption and removal. Unusable physical tails and -existing retained data are not rewritten by admission or recovery. Detailed -operator commands and candidate/rejoin flows live in the operational guide. - -Local OCI archive loading is version-dependent and must be tested separately. -The baseline production contract is registry pull by digest. If a supported -Docker version needs a Docker archive for offline loading, an explicit import -helper may convert the already built artifact for local use; it must not rebuild -or create a second published release. Converted local image IDs are not assumed -to equal the canonical OCI manifest digest. - -## 6. Lightweight Linux deployment - -The selected alternative is **containerd + nerdctl + runc**. Containerd supplies -image storage and container lifecycle, nerdctl supplies the operator CLI, and -runc executes the container. This excludes the Docker daemon but still requires -a running containerd service. Direct runc bundle management is outside scope. - -Initial production support is rootful Linux with cgroup v2 CPU/memory -controllers. `--network host` shares the host network namespace; no bridge, -NAT, CNI setup or published-port mapping is needed for this profile. Each host -normally runs one node because fixed service ports conflict on a shared host -network. Multi-node tests on one host keep their separate bridge profile. - -Pixi supplies pinned runtime binaries through a separate runtime environment; -an administrator installs/starts the host containerd service with stable binary -paths and explicit configuration. Updating a Pixi lock or cleaning a development -environment must not silently change a running production daemon. System -containerd can also be used if its version/configuration passes preflight. -Use a dedicated CROWDB namespace and never alter a Kubernetes-owned service's -configuration or containers automatically. - -A shared launcher validates digest, platform, data ownership, credentials, -physical host identity, management interface and allowed devices before choosing -Docker or nerdctl. It verifies applied cgroup `cpu.max` and `memory.max`, actual -host network namespace and device access after startup. Requested limits cannot -silently degrade into advisory values. The explicit sharing policy instead -verifies unlimited `cpu.max` and the requested `cpu.weight`: 1024 shares -represent one CPU of relative weight, not a ceiling or exclusive core. -Container E2E and CLI clusters default to this weight per container; fixture -scheduling bounds concurrent clients and hard memory totals without summing -node weights into required physical cores. Node services inherit one Tokio -worker per process in these profiles, so an unconstrained host CPU count does -not multiply idle worker pools beyond PID/memory limits. Idle CPU can be borrowed. -It records runtime/container identity -for explicit stop, restart and cleanup. Host limits are upper bounds, not -guaranteed CPU reservations. Block-device access stays explicit; no blanket -privileged container is needed. - -The common cluster CLI supports Docker and containerd host lifecycle through -these shared validators. A private receipt is sealed before runtime effects; -it records the stable daemon identity, Unix endpoint, context or namespace, -snapshotter, owner token, immutable image, resources, media lineage and input -file hashes. Resume accepts the same inputs and exact owned container, or -recreates that container against the retained data. A foreign same-name object, -changed daemon or changed configuration fails before start/stop effects. -Application UUID binding is separate and can be sealed only after observing -that cluster. Default cleanup selects stopped recorded bridge and host-network -clusters and preserves a running cluster's containers, data and restart receipts. -Explicit cluster or all-cluster teardown can stop the selected containers. -Cleanup verifies the same ownership and removes containers, unused owned bridges, -node data and matching launch/application receipts. Bridge data is held in owned -Docker volumes. Host data deletion requires the retained storage declaration to match accepted media -and rejects symlinked roots, nested mounts, overlapping recorded data directories -and bind mounts referenced by another container in the selected runtime. Supplied -disk files, raw devices and external policy/password inputs are preserved; -cleanup never wipes their contents. Retaining data and restart receipts is an -explicit cleanup option. Failures leave ownership retryable until data deletion -and target-binding removal complete. Unscoped Docker cleanup also removes stopped -containers without receipts, unreferenced named and anonymous volumes, unused -networks and unused build cache. Retained cluster resources remain protected; -non-force runtime removal rejects concurrent restarts or new references. -Containerd-only cleanup requires no Docker daemon or CLI binary. - -Explicit monitor liveness/readiness probes report node health. A dedicated -systemd unit restarts the containerd daemon, and its restart plugin applies the -node container’s `unless-stopped` policy. Monitor supervises child services. -Readiness failure alone does not restart or reinitialize the node. Persisted UUID, -SSH keys and cluster binding survive recreation under either runtime. -Rootless production networking/device/resource support is deferred; rootless -build support does not imply rootless production deployment support. - -## 7. macOS and Kubernetes extension points - -macOS retains a build backend slot: the Pixi-managed client starts/connects to a -Linux VM builder (with Lima as the provisional VM provider). Compilation, Linux library collection and -image construction all execute inside that Linux environment. Native macOS -binaries cannot be staged into the Linux image. VM CPU architecture, repository -mounts, cache, artifact transfer and lifecycle require later design/acceptance; -this change does not add macOS implementation or multi-architecture promises. - -Kubernetes uses the same OCI image digest through its CRI runtime. Later manifests -or an operator supply persistent mounts, host networking where appropriate, -resource limits, credentials and monitor probes. Discovery and physical-host -identity require explicit scheduling/network policy. Kubernetes deployment -support is not established merely by OCI image publication. - -## 8. Validation and supported boundaries - -- Verify Docker-free full build on a clean Linux host, with the builder packages - resolved entirely through Pixi and no system container binaries required. -- Verify the same OCI digest under Docker and containerd: both startup modes, - persistent identity/data, health checks, SIGTERM and recreated containers. -- Verify host networking discovery/admission, actual resource enforcement and - rejection of unsupported hosts; preserve existing SSH and Group-0 contracts. -- Verify release publication preserves tested digests and includes signatures, - SBOM/provenance; failed gates never publish moving tags. -- Rootful build and isolated Linux containerd host acceptance use the pinned tool - matrix in the operational guide. Rootless preflight reports missing host UID/GID - helpers and never silently falls back to a privileged builder. -- CI publication, remote signing and protected-environment approval are configured; - live registry acceptance requires a release workflow dispatch. -- Keep Ubuntu 24.04 and Linux amd64 initially. ARM64, macOS VM execution and - Kubernetes orchestration are separate follow-up work. - -## 9. External references - -These documents define external formats/tools, not CROWDB membership semantics: - -- [OCI Image Specification](https://specs.opencontainers.org/image-spec/). -- [BuildKit: standalone workers, OCI output and macOS client](https://github.com/moby/buildkit). -- [BuildKit rootless prerequisites](https://github.com/moby/buildkit/blob/master/docs/rootless.md). -- [nerdctl CLI and resource/network options](https://github.com/containerd/nerdctl/blob/main/docs/command-reference.md). -- [nerdctl rootless limitations](https://github.com/containerd/nerdctl/blob/main/docs/rootless.md). -- [Skopeo image inspection and transfer](https://github.com/podman-container-tools/skopeo). diff --git a/doc/design/design-crowdb-access-dataset.md b/doc/design/design-crowdb-access-dataset.md new file mode 100644 index 000000000..b099071b2 --- /dev/null +++ b/doc/design/design-crowdb-access-dataset.md @@ -0,0 +1,64 @@ + + + +# CROWDB - Design: Dataset Storage + +CROWDB's Dataset storage is built around datasets, samples, and fields. A training application asks for selected samples from a fixed dataset version, with the fields and worker assignment it needs. The storage system carries that selection down to the payload reads, so a batch does not have to begin as a walk through a directory of small files. + +Iceberg integration implements an established table format and catalog interface. CROWDB Dataset defines its own sample-oriented storage contract: immutable input, field projection, Split selection, worker partitioning, and resumable reads. It is independent of Iceberg tables and S3 buckets, and does not require a training framework's data types to become storage types. + +`crowdb-access-dataset` owns that model and its publication rules. Chunk-KV stores the metadata, while Chunk storage holds larger field payloads. The native RDMA path connects trusted training processes directly to storage nodes, with GPU-memory delivery as the next integration step. + +## Small samples do not need small physical files + +A file-per-sample layout makes a batch perform repeated path resolution, file metadata access, open/close work, and small payload reads. Packing samples into archives can reduce that overhead, but moves indexing, selection, and update organization into another application layer. + +CROWDB gives samples and fields identities in its distributed metadata and aggregates their bytes in shared Chunks. Small fields can remain inline. Eligible uploaded payloads use the shared small-object writer, which batches many objects across a bounded set of pipelines and active chunks. Each field keeps its own location and integrity information while allocation and physical I/O are shared. + +That separates the unit an application understands from the unit the disks handle. A sample can stay small and independently selectable without demanding its own file, filesystem inode, or exclusive physical chunk. Dataset reads operate on sample metadata and payload ranges; they avoid routing every sample through a general filesystem namespace. + +The benefit extends beyond file counts. Field projection can skip an image when only a label is needed, Split selection can narrow the sample set before payload I/O, and read batches can aggregate the selected work. Small-object efficiency becomes part of the storage design rather than a packaging convention every training job must maintain. + +## Immutable input for repeatable work + +A Dataset groups immutable Snapshots of samples. Splits define selected sample sets within a Snapshot, with explicit revisions so a running job can retain the selection it chose. Each sample contains named fields; the Snapshot's Manifest records those samples and fields, whose values can be bounded inline data or references to stored payload. + +For example, a sample may have an image payload, a label, and preparation metadata. A read can select the label alone without transferring the image. A subsequent training job can use the same Snapshot with a different field projection or Split. The storage model captures identity and selection; decoding an image or constructing a tensor remains application work. + +Resource preparation and head publication are separate operations. A client can prepare a complete immutable Snapshot, inspect it, and then conditionally select it as the Dataset head. Another publisher changing the head produces a conflict that the application can resolve deliberately. Preparing a new version does not silently replace the input of a running job. + +## From a read plan to selected payloads + +![Dataset read binds a Snapshot and Split, selects and projects metadata, reads inline or uploaded fields, and returns a verified batch with a continuation.](../assets/dataset-read.svg) + +A read first resolves `latest` or `stable`, if used, to an immutable Snapshot ID and binds any selected Split revision. The read plan fixes sample selection, projected fields, ordering or deterministic shuffle, worker partitioning, and batch policy. Its continuation also records the current position, so another server can replay the same batch. + +This avoids tying a long training run to a server-side read session. Workers can restart with a saved continuation, and a slow worker need not keep an idle session alive. Reads remain subject to data retention: an old continuation is a position, not a promise to preserve deleted data forever. + +Selection inspects bounded metadata windows in Chunk-KV. Split filtering precedes deterministic shuffle and worker partitioning; field projection identifies the values the application actually requested. The reader does not load a complete Manifest or fetch every field merely to build one batch. + +Field locators connect the read path to the data prepared by the write path. An inline field is already in metadata. An uploaded field resolves its immutable completed-upload description and ordered part receipts, then reads the referenced Chunk ranges or streams through the Chunk client and DiskIO. Readers check part lengths and digests, and verify the complete field before reporting success. + +Delivery remains bounded through metadata selection, payload windows, and held buffers. HTTP streams a header, payload frames, field endings, and a batch ending; successful completion also requires EOF. The native path uses the same Dataset selection and storage semantics with retained host buffers. After processing the verified batch, the application saves the next continuation. Fetching, EOF, and buffer release do not acknowledge application progress. + +## Shared payloads need explicit retention + +Dataset versions and compositions can refer to existing payloads. Reuse avoids copying the same content merely to produce another logical view, but it makes deletion a storage concern. CROWDB tracks the owners and retained resources needed to reclaim data safely. + +The same principle applies to Split revisions. Changing the current Split does not rewrite field bytes. Removing historical membership metadata and reclaiming payload storage are different operations, with different owners. This lets dataset organization evolve without turning each edit into a bulk data movement job. + +## RDMA keeps the training data path direct + +In an internal training cluster, the native SDK embeds the Dataset access library in the application. It resolves the selected metadata and reads Chunk payloads directly from DiskIO nodes over the configured RDMA connections. Bytes do not need to travel from a disk node to Access Server and then cross the network again to the training process. + +This removes a gateway relay, its HTTP handling, and its forwarding buffers from the bulk-data path. Registered buffers and retained ownership carry the result into the application without an extra SDK copy on eligible paths. The useful optimization is the whole chain: select fewer bytes, aggregate small payload work, transfer directly, and keep the received allocation alive until its consumer finishes. + +RDMA is the completed first step toward the GPU path. The current native interface delivers host buffers that can stay valid through an asynchronous host-to-device copy. Direct delivery into GPU memory through RDMA and GDS / GPUDirect integration is the next direction, with the aim of removing host staging as well as gateway forwarding. See the [native SDK design](design-crowdb-sdk-native.md) for that boundary. + +## Portable access uses the same Dataset model + +The [HTTP SDK](../../sdk/README.md) packages portable Rust and Python clients. Rust owns transfers, integrity checks, retries where appropriate, and read progress; Python exposes synchronous and asynchronous interfaces. + +The native Python binding is a separate distribution for trusted deployments with internal storage access. HTTP and native clients share Dataset identities, selection, and publication semantics, so choosing a transport does not require rebuilding the dataset organization. + +Training frameworks retain responsibility for tensor decoding, augmentation, and accelerator scheduling. CROWDB supplies the part that benefits from storage-level coordination: repeatable sample selection, efficient small-payload organization, and direct bounded I/O. See the [Dataset source](../../lib/crowdb-access-dataset/src) and [SDK examples](../../sdk/README.md#python). diff --git a/doc/design/design-crowdb-access-iceberg.md b/doc/design/design-crowdb-access-iceberg.md new file mode 100644 index 000000000..9a09ce5d7 --- /dev/null +++ b/doc/design/design-crowdb-access-iceberg.md @@ -0,0 +1,58 @@ + + + +# CROWDB - Design: Native Iceberg Storage + +CROWDB stores Iceberg tables, snapshots, manifests, and files directly on its distributed storage core. `crowdb-access-iceberg` provides the REST catalog and file interface; the Chunk client and DiskIO store the bytes. Analytical engines retain query planning and execution. There is no separate S3 storage backend in this path. + +The integration reaches into the read path. A native upload returns a signed URI that carries the file's Chunk location. Once a manifest selects that URI, a reader can reach the file's storage descriptor without another Iceberg file-metadata lookup. This removes a repeated namespace round trip from analytical data access while keeping standard Iceberg manifests and table metadata. + +## Names can change without moving table data + +Catalogs contain namespaces; namespaces contain named tables. CROWDB gives these objects stable storage identities. A table name resolves to that identity and its published table metadata. + +A rename changes the name mapping while preserving the table's storage identity. File references do not need to follow a renamed directory across physical disks. This keeps administrative changes proportional to metadata work and protects the distinction between a readable name and the owner of stored data. + +Table metadata records snapshots; manifest lists and manifests identify the files that belong to a snapshot. The canonical Iceberg metadata remains the recoverable table state. Derived indexes or projections can help specific access patterns, but they do not become a second definition of the table. Standard metadata is a particularly valuable boundary: it preserves the relationship with the Iceberg ecosystem while allowing CROWDB to organize physical storage independently. + +## From an engine write to a published table + +![Native Iceberg upload writes Chunk storage, returns a location-bearing URI, and publishes standard table metadata through a table-head CAS.](../assets/iceberg-commit.svg) + +The native loader streams Parquet bytes through the Iceberg upload interface into Chunk storage. Completion verifies the content and seals an immutable, owner-bound directory root. CROWDB signs that root descriptor into the returned URI and publishes the file record. The record supports ownership, listing, and cleanup; the URI already contains the information needed to reconstruct the read descriptor. + +The loader uses the returned URI as the file reference in standard manifests, then stores the manifest list and submits the table update to the REST catalog. The catalog validates the requirements, prepares immutable metadata JSON, and publishes the new table head with one compare-and-swap in Chunk-KV. File transfers proceed independently; the atomic table decision stays small. + +S3-compatible FileIO requests terminate at CROWDB's Iceberg file interface. They reach native Chunk storage directly and retain Iceberg file ownership and lifecycle. The familiar protocol gives engines an integration point without inserting a separate object-storage service between the catalog and its files. + +## Read the location already in the URI + +![After table and manifest selection, the direct URI supplies an authenticated Chunk root; file reads bypass Iceberg file metadata lookup and stream through Chunk client and DiskIO.](../assets/iceberg-read.svg) + +A reader first selects a table snapshot and follows its manifest list and manifests. For a native direct file URI, the locator contains the file identity, length, integrity information, and Chunk directory root. The file endpoint verifies the request's grant and the locator signature, reconstructs the descriptor locally, and reads the requested byte range through the Chunk client and DiskIO. + +This is a useful optimization at the scale of an analytical scan. A URI selected from a manifest is already a storage reference, so opening each data file does not require another Iceberg KV query to discover its location or state. Large files keep their extent directory in Chunk storage, allowing a bounded locator and bounded range traversal even when the file spans many extents. + +The locator identifies stable storage, while the Chunk layer resolves the current mirror or EC layout. Replica repair and physical placement can therefore evolve without rewriting every manifest reference. Permissions remain checked, and verified root identity prevents an old URI from reading a different file after storage reuse. + +Ordinary SDK writers that choose a logical filename before upload keep that filename and resolve its file record on read. The direct-URI optimization applies to the native upload path, which returns the final URI before manifests are constructed. Both paths use CROWDB's own storage. + +## File ownership makes cleanup part of the design + +File cleanup uses table ownership and retained metadata to decide what can be reclaimed. That connection matters after interrupted writes and table retirement: the storage system can associate unfinished physical work with the table and file that created it. Cleanup still has to respect readers, retention, and authorized operations. + +## Small files share physical storage work + +Iceberg mixes large data files with small manifests, metadata documents, and incremental outputs. CROWDB gives them different physical treatment while preserving their independent file identities. Small metadata can stay inline in distributed KV storage. Eligible chunk-backed writes use shared small-object pipelines, allowing many files to share Chunk allocation and I/O; large writes use dedicated pipelines. + +This avoids making a small file pay for its own physical storage container. Batching and shared chunks reduce per-file allocation overhead, while manifests continue to describe ordinary immutable files. The table model remains convenient for engines, and the storage layer can aggregate work at a much larger granularity. + +## Why integrate the catalog and storage? + +Some deployments compose an independent catalog with an object store. CROWDB brings table publication and file storage under one design, making the returned URI, durable completion, and cleanup agree at their source. Direct locators are one concrete payoff: the write path prepares the information that makes later reads cheaper. + +This is also an opportunity to address metadata growth. Small control records can remain in distributed KV storage while larger immutable metadata and file content use Chunk storage. The catalog retains a bounded working set instead of treating every table operation as an invitation to load all historical content into memory. + +The result is a shorter path from a table file reference to its bytes: one integrated catalog/file lifecycle, native Chunk storage, and direct locators where the upload path controls file naming. The [access acceptance guide](../../app/crowdb-access-server/tests/README.md) and [ecosystem matrix](../../e2e-test/clients/ecosystem/README.md) describe the supported client and engine combinations. + +Implementation entry points are the [direct locator codec](../../lib/crowdb-access-iceberg/src/file/direct_location.rs), [file read handler](../../app/crowdb-access-server/src/iceberg/file_http/read.rs), and [table commit](../../lib/crowdb-access-iceberg/src/commit/direct.rs). The [architecture](design-crowdb-architecture.md) shows how metadata and file traffic reach the shared storage services. diff --git a/doc/design/design-crowdb-access-server.md b/doc/design/design-crowdb-access-server.md new file mode 100644 index 000000000..e368ba891 --- /dev/null +++ b/doc/design/design-crowdb-access-server.md @@ -0,0 +1,46 @@ + + + +# CROWDB - Design: Access Server and S3 + +Access Server gives applications familiar network interfaces to CROWDB's storage. It hosts the Iceberg, Dataset, and S3 access libraries, supplies their runtime dependencies, and manages listeners and process shutdown. Each library owns its protocol and data semantics. + +This arrangement keeps the public interface close to the underlying storage. The S3 library can stream directly into the Chunk client and publish object metadata through Chunk-KV. Iceberg and Dataset use the same infrastructure with their own publication rules. + +## Buckets and objects + +`crowdb-access-s3` organizes objects by bucket and key. Each object record holds its metadata and the reference to its stored payload. Physical disk topology remains below the S3 interface. + +An upload prepares the payload, then publishes the complete object record under its bucket and key. Overwriting replaces that record; subsequent lookups find the new object data. Listing operates on the object namespace without scanning disks. + +## S3 I/O follows two direct paths + +![PUT streams bytes through Chunk client and DiskIO before publishing Chunk-KV metadata; GET resolves object metadata and streams the selected bytes back.](../assets/s3-publication.svg) + +**PUT:** the S3 library validates the request, resolves the bucket, and streams the body through the Chunk client to DiskIO. Once the payload completes and its checks pass, it publishes the object's locations, length, ETag, and attributes in Chunk-KV. The success response follows publication. Large transfers use dedicated writers; small writes share the Chunk client's aggregation pipelines. + +**GET / Range GET:** the S3 library resolves the object record by bucket and key, selects its stored locations and requested range, then streams verified bytes from the Chunk client through the HTTP response. HEAD and listing use metadata without fetching the object payload. + +The S3 layer connects directly to the metadata and payload services. Namespace lookups remain small, bulk bytes go to the disk-serving nodes, and backpressure follows a slow consumer through the pipeline. Memory is governed by active buffers and outstanding work, so object size does not become a requirement to buffer the entire transfer. + +## Multipart upload keeps transfer and publication separate + +Multipart upload allows independent part transfers before the final object is selected. Complete resolves the chosen parts and publishes their combined object; abort and cleanup handle unfinished work. An uploaded part is useful preparation, but it does not make a completed object visible on its own. + +`crowdb-access-multipart` supplies shared multipart mechanisms. The S3 library retains the meaning of an S3 Complete request and its response. Iceberg and Dataset can reuse transfer machinery without inheriting S3 namespace or overwrite semantics. + +That reuse is valuable for maintenance as well as throughput. Fixing an upload resource leak or improving bounded transfer handling can benefit several access models, while their visible behavior remains owned by their respective libraries. + +## Compatibility and the deployment boundary + +The S3 interface covers core bucket and object operations, streaming reads and writes, range access, copy, listing, deletion, and multipart transfer. It supports signed requests and presigned access. S3 object versioning is not supported. Unsupported operations return protocol errors; the interface is not the entire AWS S3 product. + +The configured listener currently shares a tenant namespace among its accepted credentials. Signature verification authenticates requests; per-user IAM policies and bucket ACL isolation are outside that interface. Deployments must choose their exposure and trust boundary accordingly. + +Standard clients are a major advantage of S3 access. The [container examples](../../container/single-node-container/README.md#tested-s3-client-recipes) show a concrete entry point, and the [access acceptance guide](../../app/crowdb-access-server/tests/README.md) describes the client and engine checks. HTTP remains the portable application boundary; trusted applications with direct service access can use the [native SDK](design-crowdb-sdk-native.md). + +## Why keep access libraries separate from the host? + +The host manages runtime resources and listener lifecycle. The libraries manage data meaning. A shared host can run several interfaces without requiring table commits or Dataset reads to pass through the S3 object API. The libraries can also be reused by trusted native applications. + +This creates more explicit interfaces inside the codebase, but it preserves freedom where workloads differ most: authentication, naming, publication, and read selection. See the [host source](../../app/crowdb-access-server/src), the separate [Iceberg](design-crowdb-access-iceberg.md) and [Dataset](design-crowdb-access-dataset.md) designs, and the [S3 library](../../lib/crowdb-access-s3/src). diff --git a/doc/design/design-crowdb-architecture.md b/doc/design/design-crowdb-architecture.md new file mode 100644 index 000000000..84a69dc2b --- /dev/null +++ b/doc/design/design-crowdb-architecture.md @@ -0,0 +1,91 @@ + + + +# CROWDB - Design: Architecture + +CROWDB separates application metadata, payload placement, and physical I/O so each can scale and recover at the right granularity. Its access libraries connect those services into complete table, dataset, and object operations. The architecture follows the decisions a storage system must make: what data means, who may change it, where its bytes live, and how those bytes survive. + +## The component map + +![CROWDB access, metadata, payload, and control components.](../assets/crowdb-architecture.svg) + +Iceberg, Dataset, and S3 form one access layer with separate access libraries. Range metadata sits above the Chunk client: Tree pages and ChunkStream journals use its shared storage path, while access libraries can reach the Chunk client directly for payload I/O. Solid arrows show the main calls and data paths; dashed arrows show replicated metadata dependencies. + +The Rust projects own service coordination and application behavior. C++ implements the Tree engine, DiskIO server, and RPC engine, with scoped FFI interfaces exposing those capabilities to Rust. `crowdb-protocol` supplies shared identifiers and wire types; `crowdb-common` supplies common utilities. Their types support the owning services without taking over their decisions. + +## Keep each decision close to the work it controls + +Composing independent storage systems can make the same request pass through several namespaces, location indexes, and forwarding services. CROWDB's layers divide responsibilities inside one storage platform. The access library owns the table, sample, or object; Chunk-KV indexes application metadata; ChunkDB and DiskDB own physical placement and allocation; DiskIO moves the bytes. + +That division keeps coordination out of the bulk-data path once a location is known. An Iceberg direct URI supplies its native file descriptor. A Dataset batch selects fields before fetching payloads. A trusted native client sends I/O directly to storage nodes. Each operation uses the metadata it needs without making every byte traverse every service. + +Shared storage also separates serving capacity from storage capacity. A range can split or change owner while retaining its page backing, so a new metadata server can take on work without first receiving a private copy of the complete index. Adding disks and relocating physical Chunks remain separate operations. This lets growth in metadata traffic and growth in stored bytes be addressed at their respective layers. + +## Metadata and control + +[Paxos-KV](design-crowdb-paxos-kv.md) maintains replicated KV state with Multi-Paxos, a WAL, and the Tree engine. It forms the metadata foundation shown at the bottom of the diagram. Group 0 is the system group, containing system metadata such as service registrations and ownership information. Groups 1–X hold other replicated KV state. Payload I/O reaches DiskIO through the Chunk client independently of this metadata path. + +Its write path runs multiple slots concurrently. Each can reach a durable quorum independently, and the Tree engine preserves per-key slot ordering when application completes out of order. Independent blind updates therefore use concurrency through replication and storage, while reads and recovery retain their required progress boundaries. + +[Chunk-KV](design-crowdb-chunk-kv.md) serves range partitions whose pages and journals live in shared Chunk storage. Its server manages the catalog, partition ownership, and range changes; its client routes requests to the appropriate server. A new owner can recover a partition from shared storage instead of requiring the old serving process to remain alive. + +The two models address different costs. Replicated control state supports coordination and bootstrap. Shared storage supports larger range metadata whose serving placement can change independently of its durable backing. Bootstrap must establish the control services needed to discover and operate the storage services; the design does not require Group 0 to bootstrap from the Chunk-KV service it will later coordinate. + +[Tree](design-crowdb-tree.md) supplies ordered storage to both models. It materializes the work selected by its caller without adding a separate mutation redo WAL. Each KV layer owns its journal, while Tree reports the durable coverage of its persisted state. Recovery combines that state with the required journal suffix; retention follows the caller's recovery obligations. The engine and its external log are designed together, avoiding repeated mutation logging inside Tree. + +[ChunkStream](design-crowdb-chunk-stream.md) supplies the durable byte stream used by Chunk-KV's journal. Ordered KV state, append streams, and Chunk payload storage are reusable foundations below the access models. A future stream interface could build on the same append, rollover, recovery, and trim machinery, with its own public API and record semantics. + +## Payload placement and movement + +Four projects divide the physical storage work: + +- [ChunkDB](design-crowdb-chunkdb.md) owns Chunk metadata, placement, protection layouts, and lifecycle transitions. +- [Chunk client](design-crowdb-chunkdb.md#two-write-paths) runs the payload pipeline, including buffering, encoding, streaming, and reads. +- [DiskDB](design-crowdb-diskdb.md) allocates and accounts for physical disk space through durable disk and zone records. +- [DiskIO](design-crowdb-diskio.md) executes byte reads, writes, and synchronization on the addressed storage devices. + +The split keeps placement decisions out of every data packet. Once a client has the required location and authority, it transfers payload to DiskIO. ChunkDB remains responsible for the layout's lifecycle, while DiskDB remains responsible for the underlying allocation. A table rename therefore has no reason to rewrite a storage block, and a physical repair has no reason to reinterpret an Iceberg commit. + +Logical key order also stays independent of physical byte placement. A prefix can occupy adjacent metadata ranges while its referenced payloads are spread across many nodes and disks. Range partitioning keeps listing efficient; Chunk placement distributes capacity, I/O, and protection. The storage layout can change without destroying the namespace order that clients use to find their data. + +## Representative operations through the architecture + +### Committing an Iceberg table + +An engine writes immutable files through the Iceberg file interface directly into Chunk storage. Native upload returns a signed URI containing the Chunk root; the loader places that URI in standard manifests. The catalog checks the table update's requirements and publishes the new head with one compare-and-swap. Readers use the selected snapshot and manifests, then reconstruct the native file descriptor from a direct URI without another Iceberg file-record query. + +Catalog and FileIO coordinate identity, publication, small-file placement, and cleanup within CROWDB. Standard table metadata remains intact, while a separate S3 storage backend and its extra namespace resolution are unnecessary. See [Iceberg](design-crowdb-access-iceberg.md). + +### Reading a Dataset batch + +The access library binds a Snapshot and Split, selects sample metadata, and projects the requested fields. Inline values are already available; uploaded fields resolve to Chunk ranges. The Chunk client reads only those payloads, preserving bounded buffers through delivery. + +The native path runs that access-library work inside the training process. RDMA carries payloads from storage nodes directly to its retained buffers, eliminating the Access Server relay. Sample identities and shared Chunk storage also avoid requiring one filesystem file per sample. See [Dataset](design-crowdb-access-dataset.md) and the [native SDK](design-crowdb-sdk-native.md). + +### Publishing an S3 object + +The access service authenticates the request and resolves the bucket and key. The Chunk client streams the body through bounded buffers to its storage layout. After the payload satisfies the required completion conditions, the S3 library publishes the object record under its bucket and key. Subsequent reads resolve that record and stream the object payload. + +Payload preparation can overlap across uploads. Publication remains a small metadata decision. An incomplete upload never becomes a complete object merely because some disk writes finished. The same separation makes multipart upload useful: parts can arrive independently, while Complete selects the object that readers will see. + +### Changing a Chunk-KV partition owner + +A candidate server opens a pinned shared Tree base and catches up through the WAL while the source continues serving. Once preparation is sufficient, fencing fixes the final source boundary and a higher epoch transfers authority through the catalog. The target completes the remaining catch-up, and clients follow the updated routing. + +Shared backing removes a full private range copy from the transfer's critical path. Local splits use the same advantage to create smaller serving ranges before balancing moves their owners. This makes the ordered range model useful for growing namespaces whose listing semantics should stay stable as placement changes. + +## Resource control reaches across the stack + +Large objects and training batches make buffer lifetime a system concern. The Chunk client bounds preparation, queued payloads, outstanding I/O, and EC work. RPC accounts for registered memory and transport queues; the native SDK also accounts for buffers held by the application after a read completes. These limits cover different resources and release them at their actual completion or ownership boundaries. + +For example, a training worker may finish receiving a batch while still holding it for decoding or a host-to-device copy. Those allocations remain charged until release. When capacity is occupied, later reads wait or reach their deadline, and backpressure limits further production. The design follows memory through useful application work, making sustained concurrency manageable even when the consumer is slower than the network. + +RDMA is the core high-performance RPC path; TCP provides the compatible fallback. Registered buffer ownership can continue through eligible storage and native-consumer handoffs. Encoding, reconstruction, HTTP adapters, and application conversions still have their own costs. The [RPC design](design-crowdb-rpc.md) explains how transport selection and memory ownership support this path. + +## Deployment and recovery + +An OCI image packages each node without requiring a particular container runtime or orchestrator. The recommended performance-oriented deployment uses host networking and host block devices; the runtime controls CPU and memory through cgroups. Persistent data and node identity remain outside the replaceable image. See [container deployment](design-crowdb-deployment.md). + +`crowdb-monitor` discovers nodes and supervises local processes. Group 0 stores system metadata. Shared console operations let the Web UI and CLI act on the same management model. Process observations describe what is running; authoritative records determine what may run and serve. + +Recovery follows those ownership boundaries. Consensus restores agreed metadata; partition recovery restores range state; Chunk repair restores physical protection; the monitor restores processes. This division makes a large failure understandable as related tasks with distinct owners. See [cluster management](design-crowdb-monitor.md). diff --git a/doc/design/design-crowdb-chunk-kv.md b/doc/design/design-crowdb-chunk-kv.md new file mode 100644 index 000000000..459d001ec --- /dev/null +++ b/doc/design/design-crowdb-chunk-kv.md @@ -0,0 +1,72 @@ + + + +# CROWDB - Design: Chunk-KV + +Chunk-KV presents a range-based tree whose ordered key space can split dynamically into partitions. Those partitions can be distributed across different serving nodes. Their Tree pages and WAL remain in shared Chunk storage, allowing metadata serving to move without requiring a full private database copy at every destination. + +The design targets the growing metadata behind table files, datasets, and object namespaces. Listing is central to these workloads: enumerate a prefix, continue from a key, walk a dataset's records, or scan a table's metadata. Chunk-KV makes those operations native to its distribution model, then uses smooth split and transfer to adapt that ordered key space as it grows. + +## Range partitioning fits a list-oriented system + +Hash distribution spreads independent point requests well, but neighboring keys need not share a server or storage order. An ordered listing can then require work across many shards and a merge of their results. Range partitioning preserves that neighborhood: a prefix maps to contiguous intervals, scans advance in key order, and clients cross partition boundaries as they progress. + +That fit matters throughout CROWDB. Object prefixes, table namespaces, and Dataset metadata all benefit from ordered traversal. A range tree lets the system build listing, pagination, and bounded scans on its primary index instead of maintaining a separate organization just to recover ordering. + +Ranges still need active management. Sequential inserts can concentrate on the newest range, and skewed access can make one partition hotter than its neighbors. Splitting creates smaller placement units; transfer distributes those units across nodes. A single hot key remains a single-key workload. CROWDB accepts this trade-off because ordered access is part of the product's core behavior, and invests in making range changes inexpensive and recoverable. + +## Ordered names do not constrain payload placement + +Range partitioning organizes the metadata keys. The referenced file, sample, or object bytes use Chunk layouts chosen independently by ChunkDB. A large object prefix can therefore retain an ordered index while its payloads occupy many nodes and disks. Logical adjacency does not require physical co-location. + +Consider objects named `photos/2026/0001`, `photos/2026/0002`, and `photos/2026/0003`. Their names are neighbors for listing, but their records may point to different Chunks with layouts spanning different storage nodes. Listing finds the next names through the ordered metadata ranges. Reading an object's body follows its locations through the Chunk client to DiskIO, without sending that body through the metadata range server. + +This gives listing and bulk I/O different ways to scale. The index routes a prefix scan through the relevant ordered ranges; payload clients reach the DiskIO nodes selected by each Chunk layout. Physical rebalance or protection changes can update those layouts without reordering the names. Metadata hot ranges still need split and transfer, but their boundaries do not force all referenced bytes through the range server or onto its disks. + +The distinction is useful for a busy tenant or dataset. Its metadata can split into more serving ranges as enumeration and updates grow, while payload placement continues to use the cluster's disks and failure domains. Ordered traversal does not require accepting one physical storage bottleneck for everything under the same prefix. + +## A range-based tree across nodes + +Each partition covers a contiguous key range and owns its mutation order, Tree, and WAL. A split divides one range at a key boundary; the retained parent keeps one side and a new child owns the other. The child initially shares the parent's serving node. A later balance operation can move it to another node, so partition count and node count can evolve independently. + +![A range-based tree splits into partitions on multiple nodes; each partition stores WAL through ChunkStream and Tree pages through a Chunk backend.](../assets/chunk-kv.svg) + +`crowdb-chunk-kv` combines two storage paths. Its WAL uses [ChunkStream](design-crowdb-chunk-stream.md), which appends bytes across chunks. Its [crowdb-tree](design-crowdb-tree.md) index uses the native Chunk backend to store page data and durable roots in chunks. Recovery opens the stored Tree state and replays the required WAL suffix. Bounded admission controls queued mutations. Both paths use the shared Chunk layer, with separate responsibilities for recovery order and searchable state. + +`crowdb-chunk-kv-server` manages the catalog, serving ownership, and partition lifecycle. `crowdb-chunk-kv-client` maintains routing information and composes calls across ranges. A scan crossing a range boundary uses those routing responsibilities; it does not turn the entire cluster into one implicit transaction domain. + +## Smooth split reuses the tree already stored + +The retained parent keeps its identity, Tree, and WAL. A new child starts from a range-bounded view of a pinned shared base, with the required parent WAL suffix covering writes made during preparation. Parent reads and writes continue while the durable handoff is prepared. Local dispatch then directs each side to its proper writer, and one catalog publication replaces the old range with two adjacent serving ranges. + +This makes split primarily a change in range ownership and references. The system can reuse the immutable base, preserve the parent writer, and move the final independent materialization into background work. It avoids making a full rewrite of both halves a prerequisite for publishing the new ranges. Durable transition records let a restart resume the same split, including the writes accepted around its cutover. + +Both halves initially stay on the same node. Separating local split from later placement keeps a remote transfer out of the range-boundary change itself. Once the child is independently recoverable, balancing can choose where it should serve. + +## Transfer prepares the destination while the source serves + +A live transfer pins a shared Tree base and records a matching WAL cursor. The destination opens that exact base and catches up through the source's WAL while the source remains writable. Most preparation can therefore overlap ordinary service; the destination does not first need a private copy of every page on its own disks. + +Cutover closes the old owner's admission, drains accepted mutations, and fixes the final source cursor. A higher owner epoch and a published catalog assignment give the destination its authority. The target finishes the remaining WAL suffix before answering reads or conditional mutations. During that catch-up state, eligible unconditional writes can already enter its durable WAL; readers wait within bounded deadlines. + +Smooth transfer comes from reducing the work that must happen at cutover. Shared backing removes the full-range copy from that critical path, advance catch-up reduces the remaining suffix, and explicit fencing keeps old and new owners from acting as competing writers. Cache warming, replay, and final readiness still matter, so placement considers readiness and capacity before releasing the source. + +## Serving capacity and storage capacity + +Shared backing changes the cost of adding a metadata server. The new server needs a usable shared root, the required journal suffix, and authority to serve. It can take over a range without first storing a complete private copy of its persistent index. Split creates suitable units of work; transfer places those units on the available serving capacity. + +That separates two operational needs. More CPU and cache capacity can serve a busier namespace, while more disks can provide additional storage capacity and bandwidth. Moving a range owner need not move its physical Chunks at the same time. Cache warming, page reads, and replay still consume resources, but copying the entire index is removed as a prerequisite for every new owner. + +Consider a namespace whose stored size changes little while more clients begin listing and updating it. A new serving node can take over selected ranges using their existing shared backing. The preparation work follows the roots, journal suffixes, and working sets it needs to serve. With storage tied to private local replicas, a destination that lacks the range state would first need to acquire a usable copy. Shared backing lets the destination open that state where it already lives. + +If the constraint is disk capacity or physical I/O, operators instead add storage resources and rebalance Chunks. Adding a range server alone cannot increase those disks' bandwidth. Keeping the two operations separate lets the deployment address the resource under pressure, while durable pages remain protected during serving changes. For a large namespace, this makes expansion less tightly coupled to the total size of its persistent index. + +## Why both Chunk-KV and Paxos-KV? + +Paxos-KV supplies replicated agreement and durable control state. Chunk-KV uses shared storage to separate a range's serving process from its durable pages and journal. These are complementary ways to pay for reliability. + +The lower KV stores the records that manage the shared storage, rather than another full copy of every upper KV value. Chunk-KV's Tree pages and journal bytes live in Chunks protected by the storage layer. Its range server can therefore focus on mutation ordering, index access, and serving ownership, while Paxos-KV preserves the durable control state that makes that arrangement possible. + +The trade-off is dependence on the shared Chunk services and their latency, availability, and repair behavior. CROWDB accepts that dependency because those services already protect the platform's payload and storage state. The design reuses a substantial storage investment instead of repeating it inside every metadata range. + +See the [partition implementation](../../lib/crowdb-chunk-kv/src/partition.rs), [native storage adapter](../../lib/crowdb-chunk-kv/src/partition/native.rs), and [server authority](../../app/crowdb-chunk-kv-server/src/server/authority.rs). diff --git a/doc/design/design-crowdb-chunk-stream.md b/doc/design/design-crowdb-chunk-stream.md new file mode 100644 index 000000000..94348a079 --- /dev/null +++ b/doc/design/design-crowdb-chunk-stream.md @@ -0,0 +1,44 @@ + + + +# CROWDB - Design: ChunkStream + +ChunkStream is CROWDB's reusable durable byte-stream layer. It turns finite storage chunks into one ordered logical space, with stable stream identity, append and offset reads, writer fencing, rollover, recovery, and prefix reclamation. Journals can use that interface without implementing their own distributed physical log storage. + +`crowdb-chunk-stream` is a library over CROWDB's metadata and Chunk storage. Chunk-KV uses it for partition journals. The stream understands append ordering and byte validity; its caller understands the records stored in those bytes. + +## Logical order across physical chunks + +![Adjacent chunks cover continuous logical ranges [0, L1), [L1, L2), and [L2, tail).](../assets/chunk-stream.svg) + +A writer appends to the active tail. Adjacent chunks cover contiguous half-open logical ranges: `[0, L1)`, `[L1, L2)`, and `[L2, tail)`. One chunk's logical end is the next chunk's logical start. Rollover introduces no logical gap and does not reset the offset. Readers can request a logical range spanning several chunks without reconstructing the physical transitions themselves. + +Stream metadata records the structure needed to find and recover the tail. Writer ownership prevents an obsolete writer from extending the stream after a handoff. Recovery establishes the usable appended content and the writer's current authority before continuing. + +This is especially useful after an interrupted append. Physical bytes can exist without proving that the complete logical append finished. The stream provides the boundary that lets a journal distinguish usable content from an incomplete tail. + +## Journals retain their own meaning + +A Chunk-KV journal can encode mutations into the stream and replay them into Tree. ChunkStream does not interpret those mutations or decide when the partition may report a successful update. Its append result is one input to the caller's completion rules. + +The journal supplies the record format, mutation order, and the relationship between an applied mutation and a stream position. ChunkStream supplies the ordered byte range and its durable append boundary. Rollover can move later bytes into another physical Chunk while the journal continues using the same logical offset space. Record parsing and physical chunk-chain management therefore remain separate jobs. + +Likewise, old stream bytes can be reclaimed only after the owner has established that they are no longer needed. A partition may base that decision on durable Tree state and its recovery requirements. ChunkStream then manages the physical prefix cleanup. + +After a restart or owner transfer, the partition reopens its durable Tree state and reads the required journal suffix through logical stream ranges. Writer fencing prevents an obsolete owner from extending the stream under its old authority. As the partition advances its durable recovery point and releases other retained dependencies, it can authorize trimming an older prefix. The same stream lifecycle supports writing, reopening, and eventual reclamation without making the journal own physical disk allocation. + +## Why use shared chunks for a journal? + +A local append file is a natural choice when the journal belongs to one machine. A shared journal supports a serving process that may move: a new owner can recover the same stream through the storage services. + +Using CROWDB's Chunk layer also reuses its data protection and physical allocation. The stream needs to manage ordering and boundaries, while the underlying services manage where the bytes live. The cost is remote I/O and dependence on the metadata and storage services needed to open the stream. + +## A foundation for another access model + +WAL is one application of a durable ordered stream. A future native Stream API could reuse the existing append, offset-read, rollover, recovery, and trim machinery, concentrating new work on public access, quotas, retry behavior, and the delivery contract. It would inherit physical allocation and data protection from the same Chunk services that already support CROWDB's other models. + +A record-oriented log adapter could build on that foundation with its own record offsets, producer identity, visibility, and retention rules. A record number would need a mapping to its byte range; a retried producer request would need an identity that distinguishes a repeat from a new append. Those meanings belong above the stream, just as mutation records belong to Chunk-KV's journal. + +A public service would also choose how readers reach the tail. The existing sequential reader captures a finite readable end; a subscription or tail-following interface would add waiting, cancellation, and retention behavior. These are future interface choices. The architectural advantage is that their implementation can start from durable ordered storage already used by Chunk-KV, concentrating new work on the model callers actually need. + +The [stream implementation](../../lib/crowdb-chunk-stream/src) contains the authority, metadata, and append logic. Its main architectural consumer is [Chunk-KV](design-crowdb-chunk-kv.md). diff --git a/doc/design/design-crowdb-chunkdb.md b/doc/design/design-crowdb-chunkdb.md new file mode 100644 index 000000000..b5da0be52 --- /dev/null +++ b/doc/design/design-crowdb-chunkdb.md @@ -0,0 +1,79 @@ + + + +# CROWDB - Design: Chunk Storage + +CROWDB Chunk storage combines placement, data protection, and payload I/O. ChunkDB owns metadata, allocation coordination, and layout lifecycle. The Chunk client uses those layouts to move bytes directly through DiskIO. Together they provide dedicated large-write pipelines, shared small-write pipelines, mirror/EC protection, and bounded reads. + +This puts the protection policy below Iceberg, Dataset, S3, and internal storage. Each can retain its own logical data model while reusing the same placement, repair, and capacity machinery. + +## From a Chunk to physical blocks + +A Chunk is a logical storage unit. Its layout describes strips and the physical blocks that hold their replicas or erasure-coded fragments. Placement uses the available topology and failure domains to choose those locations; DiskDB supplies the underlying allocations. + +`crowdb-chunkdb` manages those records and decisions. `crowdb-chunkdb-client` routes metadata operations to their owning service. `crowdb-chunk-client` performs payload I/O against the selected layout. The metadata service therefore does not need to proxy every byte of a large transfer. + +![Chunk client obtains a layout from ChunkDB, whose metadata is in Paxos-KV, then writes through DiskIO to disks on multiple nodes.](../assets/chunk-placement.svg) + +Chunk allocation follows a concrete sequence: + +1. The Chunk client requests a new layout from ChunkDB. +2. ChunkDB chooses node and disk-group targets, then asks DiskDB to allocate physical blocks. +3. ChunkDB persists the Chunk ID, strips, segment locations, and lifecycle state in the selected nonzero Paxos-KV store group. Group 0 supplies system metadata such as topology and service registrations. +4. The client receives the layout and sends payload through the target nodes' DiskIO services to their local disks. Later reads query the current layout. + +Each node can contain multiple disks. Placement combines segments from those disks according to the required mirror or EC protection and failure-domain policy. Metadata operations establish where the chunk lives; the bulk data path runs directly between the client and DiskIO. + +## Two write paths + +![Large writes use dedicated pipelines and chunks; small writes multiplex N objects over M pipelines and shared chunks, with distinct EC paths.](../assets/chunk-io.svg) + +A large write has a dedicated pipeline and an exclusive active chunk: a 1:1 mapping while that chunk is being filled. Input is framed and buffered within bounded budgets. The pipeline writes the selected mirror or EC layout, seals the chunk, and returns its location. An object larger than one chunk rotates through additional dedicated chunks and returns their ordered locations. Another object's bytes never enter those chunks. + +This gives a sustained transfer its own progress and backpressure. Preparing the next chunk and submitting independent I/O can overlap with active work, without allocating storage or memory for the entire object in advance. + +Small writes use an N:M mapping: N objects enter a bounded pool of M pipelines, and each pipeline exclusively owns one active shared chunk. The pool routes admitted objects to an available pipeline. Each pipeline batches queued whole objects, frames their data, and writes their ranges to every mirror. Objects may cross strip boundaries within that chunk, but they do not cross a chunk boundary. + +Each small object keeps its own location and completion result. A shared chunk may contain many objects, while allocation, physical writes, and chunk rotation are shared across them. The pool can adjust the number of active pipelines within configured limits. This reduces the cost of creating a separate physical container for every small payload while keeping concurrency and retained memory bounded. + +## EC follows the write path + +For a large write using EC, the strip layout supplies the data and parity widths. Data shards feed incremental parity computation and are submitted to DiskIO as they become available. At the end of a strip, the writer submits its parity shards. The next strip's data writes can overlap the previous strip's parity I/O. Chunk sealing joins the required data and parity completions before exposing the completed locations. Partial tails retain their exact valid byte length. + +The small-write path starts with mirrored strips. When automatic conversion is enabled, closed strips incrementally contribute to an EC group; the current shared path uses eight data strips and four parity shards. The client writes and synchronizes the parity, then ChunkDB selects a healthy replica from each mirror set and publishes the 8+4 layout. Those existing replicas become the data shards, so conversion reuses their stored bytes. Mirrors remain authoritative until publication; incomplete groups or failed foreground conversion remain mirrored and can be handled by background conversion. + +The two paths put EC at different points in the lifecycle. Large writes can establish their EC representation directly. Small writes first aggregate independently completing objects in mirrored storage, then reduce redundancy overhead across completed strips. + +## Completion stays per object + +Large-write completion includes sealing every chunk used by the object. Shared chunks stay open across many object completions. Callers that immediately publish readable data use the shared writer's durable completion barrier, which confirms that the chunk's acknowledged cursor covers the returned locations. The lower-level asynchronous completion mode can return before that cursor catches up. + +Iceberg, Dataset, and S3 then perform their own metadata publication. Sharing physical storage does not merge their visibility or deletion rules. An open shared writer can retain buffers for repair, and reclamation must account for other objects still using the chunk. + +## Read flow + +![Locations resolve to current chunk layouts; mirror reads and direct or recovered EC reads feed verified bounded delivery.](../assets/chunk-read.svg) + +A reader maps the requested logical range through the object's locations and current ChunkDB metadata. It follows the stored strip intervals, exact valid lengths, and current physical segment identities. This allows a location to keep its meaning after a mirror-to-EC layout change. + +A mirror read fetches the needed interval from a usable replica and can try another replica when the failure permits a safe retry. A healthy EC read fetches only the data-shard intervals the request needs. If a required data shard is unavailable, the reader obtains enough surviving data and parity intervals to reconstruct that slice within its recovery budget. It does not decode the entire object merely to serve a small range. + +Frame checks, exact lengths, and layout validity govern delivery. A streaming reader returns bounded buffers as the consumer pulls; cancellation stops further production, while held buffers retain their owners. A readable prefix is useful for streaming, but complete success requires the validated end of the requested range. Repair remains a separate operation whose allocation and layout publication are owned by ChunkDB. + +## Repair is a metadata transition as well as a copy + +Rebuilding bytes is only part of repair. The system also needs to identify the work, select the current layout, publish the valid replacement, and eventually release obsolete allocations. ChunkDB owns that lifecycle. + +Tasks carry the authority needed to act on their Chunk. Routing and ownership changes must fence old workers, including workers that resume after a long pause. A completed physical copy cannot override a newer layout merely because its original task eventually finished. + +Keeping this coordination with Chunk metadata gives recovery a durable place to resume. It also lets foreground reads and writes use published layouts while background conversion or repair prepares their successors. + +## Placement and resource control + +Explicit layouts let CROWDB change physical protection and placement while retaining a file or object's logical data reference. A computed placement rule is another possible design; stored layouts give ChunkDB a durable record of the exact allocations, conversion, and repair work it owns. Those decisions stay out of the bulk-data relay once the client has its layout. + +The shared small-write pool amortizes allocation and I/O across objects; dedicated large writers preserve independent progress and backpressure. Readers fetch requested intervals and reconstruct only missing EC slices. These mechanisms let physical storage work follow the size and shape of the request while the access libraries retain their own publication semantics. + +Memory depends on active pipelines, queued objects, outstanding I/O, parity state, recovery scratch, and caller-held allocations. Each has an owner and a budget. Retained buffers can avoid staging copies across eligible handoffs, while framing, alignment, and EC still perform the transformations the storage format requires. Repair and conversion share real disk and network capacity with foreground traffic, so their work is budgeted as part of the same system. + +See [placement selection](../../app/crowdb-chunkdb/src/selector), [conversion](../../app/crowdb-chunkdb/src/conversion.rs), [repair](../../app/crowdb-chunkdb/src/repair.rs), the [large writer](../../lib/crowdb-chunk-client/src/writer/large_object.rs), [small writer pool](../../lib/crowdb-chunk-client/src/writer/small_pool.rs), and [reader](../../lib/crowdb-chunk-client/src/chunk/chunk_reader.rs). diff --git a/doc/design/design-crowdb-deployment.md b/doc/design/design-crowdb-deployment.md new file mode 100644 index 000000000..cbfabba05 --- /dev/null +++ b/doc/design/design-crowdb-deployment.md @@ -0,0 +1,44 @@ + + + +# CROWDB - Design: Container Deployment + +CROWDB packages a storage node as an OCI image. The image carries the services and their runtime dependencies; the host supplies the network, storage devices, and persistent node state. This makes container deployment a repeatable way to run a high-performance storage system while preserving direct access to the hardware that does the work. + +The recommended performance-oriented configuration uses host networking and explicitly attached host block devices. The container runtime manages CPU and memory through cgroups. Network and storage paths stay close to the host, while compute and memory consumption remain controlled. + +## An OCI image, without a required orchestrator + +`crowdb-node` is the deployable artifact. Its build can produce an OCI archive without a running Docker or containerd daemon; loading and launching that artifact are separate operations. An image digest identifies the exact software being deployed, independent of the command used to start it. + +Docker and containerd launch paths are available. CROWDB's architecture does not require either runtime or a Kubernetes control plane. An orchestration environment can manage node containers through the same image and host-resource requirements; that portability is an image and runtime contract, rather than a claim that every orchestrator integration is supplied. + +This keeps a useful operational boundary. Container tooling distributes software and establishes process resources. CROWDB manages storage topology, replica membership, partition ownership, and recovery through its own services. Moving between deployment environments does not require changing the storage model. + +## Host networking keeps the data path direct + +Host networking places service listeners on the host's network interfaces. Storage clients reach the node's actual endpoints without a container bridge, address translation, or a port-forwarding layer in the payload path. RDMA can use the host network and the required RDMA devices directly, with TCP available according to the configured fallback policy. + +One node container per physical host is the natural production layout for the fixed service ports. It also makes the relationship between advertised addresses, physical topology, and the machine serving I/O easy to inspect. The launcher validates the effective host network namespace rather than assuming that the requested setting took effect. + +## Host devices keep storage outside the image + +Storage devices are attached explicitly to the node container. DiskDB manages allocation on the configured media, and DiskIO reads, writes, and synchronizes the addressed blocks. The container's writable image layer is not part of this storage path. + +Direct block-device access avoids adding a container filesystem layer between the storage engine and its media. Device permissions and mappings remain explicit deployment inputs. Disk files can also supply storage for suitable environments, including local testing; the performance-oriented deployment uses host block devices. + +Node identity and retained state live in a persistent host directory or volume, separate from the replaceable image. Recreating a container reopens the intended node and its data. Image replacement therefore does not imply reinitializing the disks or inventing a new storage identity. + +## CPU and memory are container resources + +The runtime applies the node's CPU and memory policy through cgroups. A CPU ceiling caps consumption; relative CPU shares allow the node to use otherwise idle capacity while controlling its weight under contention. A memory limit bounds the container's aggregate footprint. Launch validation checks the effective cgroup settings, so the accepted deployment describes actual limits. + +Inside that budget, CROWDB controls its own worker pools, registered memory, caches, and in-flight I/O. Those internal limits provide backpressure before a workload exhausts the container's resources. Container limits and storage admission work at different scales: one bounds the node, the other keeps its concurrent work within usable capacity. + +## The runtime starts the node; the monitor runs its services + +The container runtime owns creation, resource configuration, and container stop. Tini handles PID 1 duties, while `crowdb-monitor` owns the service processes and their shutdown order. Group 0 retains system metadata and the durable decisions that govern those processes. + +This separates replacing an executable environment from changing storage authority. A restarted container can recover accepted identity and deployment state; the monitor can restore the appropriate services under the existing cluster model. See [cluster management](design-crowdb-monitor.md) for discovery, readiness, and management operations. + +Container deployments have been exercised on physical Linux hosts. Build inputs, runtime options, persistent paths, and launch commands are documented in the [OCI image guide](../../container/oci-image/README.md); the [local node guide](../../container/single-node-container/README.md) covers a smaller setup. diff --git a/doc/design/design-crowdb-diskdb.md b/doc/design/design-crowdb-diskdb.md new file mode 100644 index 000000000..202b3aa71 --- /dev/null +++ b/doc/design/design-crowdb-diskdb.md @@ -0,0 +1,36 @@ + + + +# CROWDB - Design: Disk Allocation + +DiskDB turns disk capacity into managed allocations. It records which space is owned, which space can be reused, and how disks and zones recover their allocation state. Byte reads and writes are handled separately by DiskIO. + +This separation lets CROWDB make allocation a durable metadata operation without putting that operation in the path of every subsequent I/O. ChunkDB requests space for its layouts; payload clients then use the resulting locations to access DiskIO. + +## Durable records and working allocation state + +`crowdb-diskdb` manages disk groups across multiple nodes. A disk group belongs to one node and contains disks; a node can have several disk groups. Each disk is divided into zones and allocation blocks. Group 0 stores topology and owner bindings, while each disk group has an explicit Paxos-KV data-group binding for its allocation records. The client routes requests to the current DiskDB owner. Durable allocation records establish ownership. Working structures such as bitmaps make allocation and free-space queries efficient, but they can be reconstructed from the persisted records. + +That distinction is valuable after a restart. The server can rebuild the working representation from the recorded facts rather than treating a partially updated in-memory bitmap as the only account of disk ownership. Scanner and recovery work reconcile the durable and operational views. + +![DiskDB routes allocations across disk groups on multiple nodes, with durable records in Paxos-KV.](../assets/disk-allocation.svg) + +Freeing space also changes ownership. It must be reflected in the durable allocation model before that space is safely reused. Zone maintenance and compaction manage the representation and accumulated work behind those decisions. + +## Capacity is more than a free-byte counter + +Operators need to understand disks, zones, allocated space, and the work that can return capacity. DiskDB maintains counts and observations at the appropriate levels; `crowdb-diskdb-client` routes calls and aggregates information for its consumers. + +These observations help placement and management, while the allocator remains responsible for the actual decision to grant space. A cached capacity report is useful for choosing candidates, but it cannot reserve the bytes it describes. + +Keeping capacity reporting near allocation makes its meaning easier to explain. Physical space, allocated space, and logical user data are different quantities, particularly with replication, erasure coding, and deferred cleanup. + +## Why allocation is a separate service + +An allocation changes durable ownership. A read of an existing allocation usually does not. Combining both into one broad operation would make it harder to optimize their different frequencies and failure behavior. + +DiskDB gives CROWDB a focused place for disk lifecycle, space accounting, and rebuildable allocation state. DiskIO can concentrate on efficient byte operations, while ChunkDB chooses how several allocations form a protected layout. + +The cost is a distributed dependency when obtaining or freeing space. Durable records, working allocation structures, and background recovery must agree. CROWDB accepts that coordination to keep ownership explicit and recoverable across process failures. + +See the [allocation model](../../app/crowdb-diskdb/src/model), [persistence](../../app/crowdb-diskdb/src/persistence.rs), and [recovery](../../app/crowdb-diskdb/src/recovery) code. The adjacent [DiskIO design](design-crowdb-diskio.md) covers actual I/O. diff --git a/doc/design/design-crowdb-diskio.md b/doc/design/design-crowdb-diskio.md new file mode 100644 index 000000000..b239333f5 --- /dev/null +++ b/doc/design/design-crowdb-diskio.md @@ -0,0 +1,36 @@ + + + +# CROWDB - Design: Disk I/O + +DiskIO is the node-level byte service. It accepts addressed reads, writes, and synchronization requests, validates their ranges, and submits them to the storage backend. Its narrow responsibility gives the rest of CROWDB a common physical I/O interface. + +`crowdb-diskio` is the C++ server. `crowdb-diskio-client` supplies the Rust client path. DiskDB owns allocation, and ChunkDB owns the layout that combines allocations into protected data. + +## A short path from request to device + +![DiskIO requests use RDMA or TCP, retaining buffers through server validation, device I/O, and response completion.](../assets/disk-io.svg) + +The client routes requests through `crowdb-rpc` over RDMA or TCP. RDMA uses registered memory, posted receives, and completion queues. On eligible aligned paths, the server passes registered receive storage into the disk engine without an extra payload staging copy. Reads can fill a retained registered buffer and send it back through the same transport. Alignment edges use explicit staging buffers. TCP exposes the same read, write, and synchronization operations. + +The server works with `Segment` addresses and byte ranges rather than bucket keys or table files. It has no need to interpret Iceberg table metadata to read the blocks holding a file. That keeps application-specific work above the physical path. + +The backend uses io_uring where supported and provides a fallback path for other environments. The common interface lets the service preserve its I/O model while backend mechanics differ. Queue depth, device behavior, and platform support still affect the useful level of concurrency. + +## Completion has a precise owner + +A read completes when the requested result is available to its caller. A write completion and a synchronization completion describe different storage events. Higher layers combine the appropriate operations with their replication and publication rules. + +An S3 object becomes visible through S3 metadata, for example; DiskIO cannot infer that event from a finished device write. Keeping the byte service's result focused makes it reusable across payload, journal, and page-storage users. + +Asynchronous I/O also gives buffers a lifetime beyond submission. The service and client retain the memory needed by accepted operations until the backend and transport are finished with it. Cancellation and shutdown must account for that accepted work before releasing resources. + +## Why keep this path under CROWDB's control? + +DiskIO connects device queues, RPC buffers, and the Chunk client's I/O schedule. Controlling those interfaces makes it possible to reduce unnecessary allocation and copying and to keep outstanding work bounded across the network/device boundary. + +The fallback backend makes development and deployment less dependent on one kernel interface. It does not make all backends equal in performance; the selected backend and device configuration remain part of any useful measurement. + +This focus comes with native-code and platform integration work. CROWDB keeps that work concentrated in the service and shared C++ utilities so the access models can remain independent of device details. + +The [server source](../../app/crowdb-diskio/src), [client source](../../lib/crowdb-diskio-client/src), and [shared C++ utilities](../../lib/crowdb-common/cpp) are the main implementation entry points. [RPC](design-crowdb-rpc.md) explains the transport side of the data path. diff --git a/doc/design/design-crowdb-monitor.md b/doc/design/design-crowdb-monitor.md new file mode 100644 index 000000000..07012ba44 --- /dev/null +++ b/doc/design/design-crowdb-monitor.md @@ -0,0 +1,38 @@ + + + +# CROWDB - Design: Cluster Management + +CROWDB connects deployment and recovery to the durable state of the storage system. `crowdb-monitor` discovers nodes, supervises processes, and executes deployment work. Group 0 stores system metadata. Console operations coordinate management through the same service model. + +A process can be alive while it is recovering, fenced, or unable to serve its assigned data. The management design keeps those conditions visible instead of treating process existence as complete readiness. + +## Local execution and durable decisions + +Group 0 is the Paxos-KV system group. It stores system metadata, including hardware topology, service registrations, membership, and ownership. The monitor performs local work under that control model: preparing resources, starting services, observing them, and releasing resources when their lifecycle permits it. + +Discovery supplies candidates and observations. It does not by itself admit a machine, transfer ownership, or replace durable identity. This matters when a machine restarts with old local state or a previously reachable process appears again after a partition. + +Bootstrap establishes the initial control services and then the storage services that depend on them. A prepared deployment can advance as its dependencies become ready. Recovery can inspect existing state and resume the relevant work without treating every restart as a new cluster. + +## Management interfaces share operations + +`crowdb-console-shared` implements common management operations and validation. `crowdb-web` presents them through HTTP and the browser UI; `crowdb-cli` presents commands, output, and exit status. + +The interfaces can differ in interaction without inventing different meanings for a disk removal or service deployment. Business authority remains with the service that owns the underlying state. An operator's request enters that service's lifecycle rather than rewriting observed state in the UI. + +This arrangement also gives diagnosis a common vocabulary: the requested operation, durable ownership, observed processes, and readiness can be examined separately when an operation does not finish. + +## Service lifecycle inside a node + +The monitor owns the service processes inside a node container: it starts them, observes readiness, coordinates shutdown, and recovers their lifecycle after restart. The container runtime establishes the node's CPU and memory budget and attaches host resources. This leaves service ordering with the component that understands storage dependencies. + +Retaining data and accepted identity across restart is essential. A recreated container reopens the intended node state; a fresh image does not authorize initialization over existing storage. The [container deployment design](design-crowdb-deployment.md) explains OCI packaging, runtime independence, host networking, block devices, and resource limits. + +## Why integrate management with storage? + +Deployment changes who can serve data and which physical resources exist. Keeping these operations connected to durable metadata makes it possible to recover their intent after a worker or management process fails. It also prevents an old process observation from becoming a new ownership decision. + +The cost is coordination between persistent state and external effects such as launching a process or attaching storage. Those effects still need observation, retry handling, and cleanup. CROWDB gives that work explicit owners instead of leaving each command to improvise recovery. + +Implementation entry points are the [monitor](../../container/crowdb-monitor/src), [shared console operations](../../lib/crowdb-console-shared/src), [Web application](../../app/crowdb-web), and [CLI](../../app/crowdb-cli). The [E2E guide](../../e2e-test/README.md) describes deployment and restart validation scenarios. diff --git a/doc/design/design-crowdb-paxos-kv.md b/doc/design/design-crowdb-paxos-kv.md new file mode 100644 index 000000000..87c9ef021 --- /dev/null +++ b/doc/design/design-crowdb-paxos-kv.md @@ -0,0 +1,44 @@ + + + +# CROWDB - Design: Paxos-KV + +Paxos-KV is built around concurrent slot writes. Multiple writes can replicate, reach a durable quorum, and become chosen independently, while the Tree engine uses each key's slot to preserve the correct value when application finishes out of order. CROWDB carries concurrency through consensus and into storage, reducing the need for unrelated metadata updates to wait behind the same completion boundary. + +`crowdb-paxos-kv` implements replication and the KV state machine. `crowdb-paxos-kv-server` hosts groups and their lifecycle. `crowdb-paxos-kv-client` routes application requests and handles service discovery. Group 0 is the system group and stores system metadata. + +## Independent slots keep writes moving + +The leader assigns slots in order and runs multiple proposals concurrently. Each slot is a separate Paxos decision with its own quorum progress. If slot N is delayed, N+1 and N+2 can become chosen without first filling that gap. A bounded in-flight window keeps that parallelism within the capacity of the WAL, network, and engine; recovery has its own path for resolving missing slots. + +This is particularly useful for blind Put and Delete operations. Their effect is determined by the key and assigned slot, so the engine can apply available work immediately. The highest slot wins for a key, regardless of which write reaches the engine first. Two updates to different keys need not share a serial apply queue, and a late older update cannot replace a newer value. + +Durability remains part of the write path: accepted values are durable on a quorum before the write is acknowledged. The synchronous path also waits for local apply; the asynchronous apply mode can acknowledge a chosen write while owned engine work continues. Linearizable reads enforce the appropriate apply fence before returning data. + +## The advantage over an ordered-log write path + +In Raft's replicated state-machine model, commitment advances through a contiguous log prefix and commands are applied in log order. Batching and pipelining improve replication throughput while retaining that ordering model. See the [Raft paper, sections 5.3 and 9.3](https://raft.github.io/raft.pdf). + +CROWDB uses independent Paxos slots together with an engine that understands their ordering. Later blind writes can reach their own completion point while an earlier slot is still being resolved. This reduces head-of-line waiting where the operations do not require each other's results, and lets available network, WAL, and engine capacity serve useful work across many concurrent writes. + +The benefit depends on the complete consensus-to-engine path. Simply replacing a consensus algorithm while retaining one serial apply queue would leave much of the opportunity unused. CROWDB designs slot agreement, per-key version selection, and parallel application together. Read-modify-write operations use additional coordination, and reads or maintenance that require a complete prefix still observe that boundary. + +## Stores, groups, and replicas + +A store contains multiple Paxos groups. Each group has its own replica set, consensus state, WAL, and Tree state. Replicas run on different nodes; one node can host replicas for several groups, and each group can elect a different leader. Clients route by store and group before choosing the appropriate replica. + +![A store contains multiple groups, each replicated across nodes with replica-local WAL, crowdb-tree, and local disk storage.](../assets/paxos-kv.svg) + +Each replica writes its WAL and `crowdb-tree` state to local disks. Replication protects the group across nodes. Group 0 stores system metadata, while nonzero groups hold other KV data, including the metadata owned by storage services. A group remains a consensus and ordering boundary; application work spanning groups needs explicit coordination above it. + +## Durable control state for the rest of the system + +The WAL records the information needed to recover replication and accepted work. Tree materializes selected operations without adding another mutation redo WAL. Its durable snapshot coverage tells the KV layer which applied state is recoverable from the engine; the retained consensus history supplies the required suffix. This avoids duplicate engine logging while coordinating persistence and recovery through explicit progress. The [Tree recovery design](design-crowdb-tree.md#recovery-is-part-of-the-engine-contract) explains that shared contract. + +This matters when many metadata writes are active at once. The quorum WAL makes accepted work durable, parallel apply updates the ordered index, and a later Tree snapshot consolidates durable state. The KV layer can advance these stages at different rates while retaining the history needed between them. It avoids inserting a second engine redo log into every mutation's path and can schedule checkpoint work around the same progress information used for recovery. + +Group 0 relies on this recovery path for service registration, ownership, membership, and deployment decisions. Those decisions must survive the process that made them. A restarted monitor or service can recover them from durable metadata instead of reconstructing them from whatever happens to be running. + +Recovery uses the same slot ordering as normal operation. Gap repair restores unresolved decisions; contiguous applied progress records which prefix is complete; snapshot-backed retention determines which history may be reclaimed. Those mechanisms preserve recovery information while the active write path exploits independent progress. + +The [learner](../../lib/crowdb-paxos-kv/src/paxos/learner.rs) and [Tree adapter](../../lib/crowdb-paxos-kv/src/kv/crowdb_tree_engine.rs) show the connection. [Chunk-KV](design-crowdb-chunk-kv.md) explains the complementary shared-storage model. diff --git a/doc/design/design-crowdb-rpc.md b/doc/design/design-crowdb-rpc.md new file mode 100644 index 000000000..33c94de4a --- /dev/null +++ b/doc/design/design-crowdb-rpc.md @@ -0,0 +1,52 @@ + + + +# CROWDB - Design: Storage RPC + +`crowdb-rpc` puts RDMA at the center of CROWDB's high-performance internal data path, with TCP as the fallback for environments without a usable RDMA connection. The transport connects storage services and native consumers through registered buffers and asynchronous completion, carrying memory ownership across the boundaries where ordinary message handling often adds staging and copies. + +A C++ engine supplies connections, framing, workers, and both transport backends; `crowdb-rpc-ffi` exposes them to Rust. Services use the same request and payload model on either path. Deployment policy enables RDMA and controls whether setup may fall back to TCP. + +## RDMA carries the storage data path + +The RDMA backend manages registered memory, posted receives, work requests, and completion queues. Payload views retain their allocations until the hardware and subsequent consumers have finished with them. Eligible buffers can keep their registration through DiskIO, Chunk reads, and native SDK delivery, reducing repeated allocation, registration, and intermediate copying. + +The value appears across the complete I/O path. A DiskIO read can fill a registered buffer for transport; a native consumer can retain the received allocation while processing its bytes. A direct native client also removes the Access Server payload relay. CROWDB can therefore optimize both the network transfer and the handoffs on either side of it. The [DiskIO design](design-crowdb-diskio.md#a-short-path-from-request-to-device) and [native SDK](design-crowdb-sdk-native.md) describe those paths. + +## Control and payload take different paths + +A control message describes the operation. Payload views describe the bytes it transfers and retain their owners. The transport can keep those allocations alive until asynchronous send or receive work completes, without forcing every caller to serialize the payload into a new message buffer first. + +Request correlation and transport errors belong to RPC. The service handler defines business success, retry behavior, and durability. A successful send cannot establish that an object was published or a KV mutation committed. + +This division lets DiskIO, metadata services, and native consumers share the transport while keeping their completion rules explicit. + +## Backpressure follows the resources still in use + +Storage resource use continues beyond network transfer. A payload may be queued for DiskIO, retained during a write, involved in EC work, or held by the application after delivery. The RPC engine bounds registered memory and transport work; the Chunk pipeline and native runtime apply their own admission limits to the work and allocations they own. + +Consider a burst of writes that arrives faster than the disks can complete it. Receiving another payload may be easy for the network while its memory remains occupied behind earlier device operations. Transport flow control sees network progress; storage admission must also see queued and in-flight storage work. Bounding both requests and bytes matters because a small control operation and a large payload consume very different amounts of memory. + +When downstream capacity is occupied, those limits stop or delay further production. A slow disk or consumer therefore becomes visible as backpressure, rather than an invitation to accumulate more payloads. Separate budgets account for request concurrency, bytes, and processing resources; a count of completed RPCs alone cannot describe the system's available capacity. + +The receive pool, send queue, storage pipeline, and consumer each release capacity when their part of the work actually finishes. They can share a payload allocation without pretending that these are the same event. That is especially valuable for RDMA, where registered memory is a finite resource: reusing an allocation saves staging work only if its lifetime remains visible to every stage that retains it. + +Ownership also survives cancellation. If a write times out after device submission, the caller can stop waiting while the device still references the buffer. The retained owner keeps that allocation valid until the outstanding work finishes. On reads, application-held results remain charged after transport completion. This connects RDMA efficiency with controlled memory use under sustained load. The [native SDK](design-crowdb-sdk-native.md#completed-reads-still-consume-capacity) carries that accounting through to the consumer. + +## TCP provides a practical fallback + +TCP keeps the same storage services usable on ordinary hosts and networks. Its backend uses event-driven workers with epoll or kqueue, handling framing, send and receive progress, and connection shutdown. A deployment can select TCP explicitly; an RDMA-enabled client can use it when capability negotiation or connection setup cannot establish a supported RDMA path and fallback is allowed. + +Performance-sensitive deployments can disable fallback to require RDMA. This makes the chosen transport an explicit operational policy: a workload that requires RDMA fails setup visibly instead of silently running on another path. + +Fallback happens before payload submission. Once an RDMA operation has an uncertain outcome, the transport reports that failure; it does not splice TCP bytes onto a partial result or transparently replay the operation. The service's request semantics determine whether a subsequent retry is safe. + +## Why own an RPC engine? + +A storage RPC engine can make registered memory and asynchronous I/O part of its basic interface. Control fields remain small, payload buffers keep their owners, and completion releases resources at the right stage. RDMA receives the design attention required by the fast path, while TCP preserves deployment flexibility through the same service boundary. + +Owning that boundary lets the Chunk client, DiskIO, and native SDK agree on memory lifetime. It removes avoidable staging at supported handoffs and lets resource admission span several libraries. The transport becomes part of the storage pipeline, with an interface shaped around moving bytes efficiently between devices and consumers. + +Internal RPC operates within the cluster's trust boundary; externally authenticated applications use the access interfaces. Public protocol compatibility remains with those interfaces. + +See the [transport interface](../../lib/crowdb-rpc/include/crowdb-rpc/transport.h), [buffer interface](../../lib/crowdb-rpc/include/crowdb-rpc/buffer.h), and [FFI](../../lib/crowdb-rpc/ffi). The [native SDK](design-crowdb-sdk-native.md) shows how retained transport allocations become application-visible results. diff --git a/doc/design/design-crowdb-sdk-native.md b/doc/design/design-crowdb-sdk-native.md new file mode 100644 index 000000000..ed1d3c363 --- /dev/null +++ b/doc/design/design-crowdb-sdk-native.md @@ -0,0 +1,64 @@ + + + +# CROWDB - Design: Native SDK + +`crowdb-sdk-native` is CROWDB's RDMA-oriented access path for high-performance applications. When compute and storage belong to the same trusted system, an application can embed the Dataset and S3 access libraries and communicate directly with storage services. Bulk data travels between disk-serving nodes and the application without passing through an additional Access Gateway. + +The objective is efficiency across the whole system: fewer network hops, less protocol work, fewer staging buffers, and less CPU spent forwarding bytes. RDMA is most valuable when the architecture also removes the intermediate process that would otherwise receive, wrap, and retransmit the payload. + +## Make storage part of the application runtime + +An HTTP access path sends storage-node payloads through Access Server before they reach the application. The native runtime executes the access-library work inside the application process and uses the Chunk client and DiskIO connections directly. This removes the storage-node-to-gateway-to-application relay from bulk I/O. + +The same Dataset Snapshot, Split, and field-selection rules still apply. S3 operations still resolve bucket and key and use the owning library's publication rules. Metadata services retain their authority; the application gains a shorter data path rather than a second interpretation of stored data. + +For training and analytical systems, this changes where resources are spent. Gateway CPU, response framing, forwarding buffers, and a second payload transfer no longer have to scale with every byte consumed by an internal application. Compute and storage can be provisioned around their actual work. + +## One runtime for many operations + +`ClientRuntime` owns or borrows the underlying service clients. Dataset and S3 handles share its connection groups and resource budgets. Topology observations help locate services and data; current routing and serving authority still determine whether a request may execute. + +Sharing matters under concurrency. Creating independent connection pools and memory limits for every object or worker handle would multiply both resources and coordination. A common runtime gives the application one place to account for active requests, producer buffers, and held results. + +Operations carry deadlines and cancellation. Shutdown closes admission and drains owned work. Buffers already handed to an application retain their own lifetime, so closing a stream or runtime does not invalidate data that the caller still holds. + +## RDMA needs buffer ownership all the way to the consumer + +Reads return `BufferLease` values: immutable views that retain the backing allocation. A consumer can process a view, pass it to another asynchronous operation, or retain it until a host-to-device copy finishes. Resource accounting follows the allocation until its final owner releases it. + +Eligible registered receive buffers reach the caller without a second SDK payload copy. Producers can also fill native allocations and submit immutable views. Buffer ownership survives asynchronous I/O and application processing, so a fast transport does not immediately lose its benefit to another copy at the SDK boundary. + +Copies still have reasons to exist: ordinary input may need framing, erasure coding creates or reconstructs fragments, and language adapters may materialize their own values. For example, the native Python binding's `to_bytes()` explicitly creates Python-owned bytes. The benefit comes from making those conversions deliberate and keeping ownership intact between them. + +## Completed reads still consume capacity + +A training worker can finish a read and keep its buffers while decoding samples or waiting for a host-to-device transfer. The SDK continues to charge those allocations against the caller-held capacity budget until their final owner releases them. Completing the RPC or closing the stream does not erase that memory use. + +For example, a worker may prefetch the next batch while another task decodes the current one. Both batches occupy capacity until their owners finish with them. Passing a shared view to another task preserves the same backing allocation and charge; it does not create another copy merely to transfer ownership. Once the last owner releases that allocation, the capacity becomes available to a later read. + +When held capacity is exhausted, later reads wait for release or return a deadline or cancellation result. Producer allocations and active requests have their own limits, alongside the transport's registered-memory and queue budgets. Sharing a runtime makes Dataset and S3 handles share the appropriate SDK accounts, so adding handles does not silently multiply the application's allowance. + +Request concurrency controls how much work is active; held capacity controls how much delivered data remains in the application. A workload can have few active requests and still hold substantial memory from earlier reads, so those limits serve different purposes. Producer memory has a similar lifetime concern: shared small-write machinery can retain submitted owners for open-strip repair after an individual PUT completes. + +This lets the consumer set an SDK-managed working-set limit that remains meaningful after bytes leave the network. The application can retain a view through asynchronous processing while backpressure controls how much more data arrives. Copies into application-owned arrays, decoded objects, or Python `bytes` need the application's own memory policy; the SDK accounts for the allocations whose lifetime it manages. + +## Transport policy follows the deployment + +The runtime selects transport policy for metadata and data connections. RDMA uses configured devices, registered memory, and bounded transport resources. TCP supplies the portable path for deployments without that hardware. Applications keep the same access semantics and retained-buffer model in either case. + +An operation that has already transferred data cannot be treated as a fresh attempt merely because another transport is available. The SDK preserves errors and uncertain mutation outcomes so the application can recover with the original identity and publication conditions. It does not turn an ambiguous write into an automatic new write. + +## Extend the direct path toward GPU memory + +GPU workloads make the final destination of each byte especially important. The next integration step is direct delivery into GPU memory through suitable RDMA and GDS / GPUDirect support. The system-level goal is to remove gateway forwarding and host staging from the storage-to-GPU path, rather than accelerating one link while leaving the same copies around it. + +CROWDB has taken the first step with direct RDMA access and retained host buffers. The current SDK exposes host memory, including buffers that an application can retain until an asynchronous host-to-device copy completes. GPU pointers and direct GPU delivery require the subsequent GPU integration. The existing ownership and completion model provides the place to carry that lifetime through the full path. + +## The deployment boundary is explicit + +The embedding application needs access to internal management and storage services. It has the same internal trust as an access service; this API does not perform S3 SigV4 authentication. HTTP remains the appropriate boundary for portable clients and externally authenticated access. + +Native integration brings native libraries and, for RDMA, host device and registration requirements. That is a useful fit for an internal compute/storage cluster whose operators control both ends of the connection. HTTP remains available for portable clients and external access. + +The [native SDK README](../../lib/crowdb-sdk-native/README.md) contains Rust examples and runtime options. The [native Python README](../../lib/crowdb-sdk-native-python/README.md) describes packaging and its explicit copy boundary. The separate [Dataset HTTP SDK](../../sdk/README.md) provides the portable path. diff --git a/doc/design/design-crowdb-tree.md b/doc/design/design-crowdb-tree.md new file mode 100644 index 000000000..750b55249 --- /dev/null +++ b/doc/design/design-crowdb-tree.md @@ -0,0 +1,68 @@ + + + +# CROWDB - Design: Tree Storage Engine + +`crowdb-tree` brings concurrent skiplist tables, a Bw-tree architecture with a mapping table, and slot-aware version selection into one ordered KV engine. Its flush design follows the tree itself: fork work down independent subtrees, join results from leaves to root, and persist the same results when a snapshot is requested. The page backend can use local disks or shared Chunk storage. + +These choices connect foreground concurrency, ordered reads, and background persistence. Paxos-KV uses the engine on each replica's local disks; Chunk-KV uses its Chunk backend to keep range state independent of the serving node. CROWDB can optimize the same engine for both storage models. + +## Concurrent skiplists feed a versioned page tree + +![Concurrent skiplist tables use CAS; a Bw-tree mapping table resolves immutable pages, with ordinary and persistent flush modes and local-disk or Chunk backends.](../assets/tree.svg) + +The write buffer is an ordered concurrent skiplist. CAS publishes new nodes and immutable version sets, while epoch protection keeps objects alive for readers. Multiple writers can update independent keys without putting every insertion behind a shared MemTable mutex. The skiplist's ordering also gives flush jobs range cursors directly, avoiding a fresh sort of the entire input batch. + +The page tree uses mapping indirection: a logical page ID resolves to an immutable page version through a versioned mapping table. A page combines base data with ordered overlays. Readers retain a consistent mapping and source view while writers prepare newer state. A split can change routing and share retained base ranges without immediately copying every value into a new page buffer. + +This keeps several expensive operations off the ordinary update path. Inserting a value need not rewrite a disk page; publishing a new page need not invalidate a reader's view; moving entries into the tree can transfer or share their existing allocation owners. Consolidation combines overlays and adjusts page boundaries when the structure needs it. + +## Logical versions survive out-of-order application + +Each key retains slot information and the versions needed by active read and recovery boundaries. An older update that finishes after a newer one cannot overwrite it merely by arriving later. Point reads and ordered scans select the highest eligible version, including tombstones, from the retained sources and pages. + +That makes concurrency useful all the way through the replicated write path: consensus establishes logical order, while independent application work can finish in a different physical order. Paxos-KV builds user-facing conditional CAS operations from versioned reads and coordinated chosen writes; Tree's own CAS operations publish its concurrent in-memory state. + +## Fork flush down the tree, join results toward the root + +![A fixed set of MemTables feeds root-to-leaf fork tasks; leaves consolidate independently and parents join child results, with memory-only or persistent output from the same flow.](../assets/tree-flush.svg) + +The flush design captures a finite set of skiplist tables and a fixed version frontier. Starting at the root, index separators divide the work into child ranges. Each leaf receives all captured updates belonging to its range, then consolidates as soon as its own inputs are ready. Independent subtrees can advance concurrently. + +Join follows the reverse dependencies. A parent consumes its children's ordered results, resolves sibling merges and index changes, and returns its result upward. The design avoids a whole-tree handoff barrier before consolidation can begin. A ready subtree can produce its final pages while another branch is still receiving updates. + +This is where the tree structure becomes a scheduling advantage. Parallelism follows key ranges and page dependencies, with bounded CPU and I/O work. Waiting for a child or storage completion yields the worker, allowing the shared runtime to advance other ranges and other Trees. + +## Flush and flush with snapshot share the same work + +Ordinary flush publishes the resulting pages and mapping in memory. Flush with snapshot selects persistent output for that same fork/join round. Final pages are encoded and written as their dependencies become ready; parents join those results, changed mapping segments are written once their entries are complete, and the durable root commits after its referenced output is safe. + +The design shares consolidation, split/merge decisions, and final page layout between memory maintenance and persistence. Snapshot output therefore reuses the work already needed to organize the tree. Unchanged durable pages and mapping segments keep their existing backing, while a persistent round also includes dirty state left by earlier ordinary flushes. + +Incremental persistence makes the amount of changed state matter more than total tree size. Memory-only maintenance can batch several update rounds before durable output, and the caller gets an explicit committed frontier for its WAL or journal retention policy. Resource budgets and backpressure keep that batching within the Tree's capacity. + +## Recovery is part of the engine contract + +Tree uses its caller's recoverable journal and does not create a separate mutation redo WAL. Paxos-KV supplies consensus-selected operations from its replica-local WAL; Chunk-KV supplies ordered mutations from its partition journal. The engine materializes those operations, preserving their version information and reporting the durable progress covered by a committed snapshot. + +This removes a second recording of the same mutation inside the engine. The larger benefit is that application, persistence, and recovery share explicit progress boundaries. If a snapshot covers a prefix and later writes exist only in memory, recovery replays the required external journal suffix. A high observed slot or a completed page write cannot substitute for a committed root and its coverage. + +For example, suppose the caller has completed operations through 120, while the last durable Tree root covers 100. Operations 101–120 must remain recoverable from the caller's journal. After a crash, the caller opens that root and supplies the valid suffix to Tree. In Paxos-KV, consensus determines which operations were chosen; Tree restores their materialized state using the supplied slots. Version selection, including tombstones, prevents an older replayed update from replacing a newer value. + +Now suppose a snapshot starts persisting state through 120. Its page writes can finish at different times, but the recoverable boundary advances only when the root and its dependencies are committed. Until then, the previous root and required journal history remain the recovery path. This makes checkpoint publication the point where the caller learns that more state can be recovered from Tree itself. + +The caller uses that durable coverage together with its replication and retention obligations to decide which journal history can be released. Tree manages the lifetime of its own pages and retained views. Each part supplies the evidence needed by the other, so memory maintenance, checkpointing, and log reclamation can be optimized together without confusing their completion points. + +That coordination gives persistence room to batch useful work. Foreground writes can become recoverable through the external journal while Tree consolidates updates and persists changed pages in larger rounds. The engine avoids maintaining a second mutation log, and the caller can balance checkpoint work against the history retained for replay. A faster write path and a bounded recovery path become connected design decisions. + +The same contract works with two different journals and two physical backends. It lets CROWDB reuse recovery-aware indexing while keeping consensus decisions and partition ordering with their respective KV services. + +## One engine, local disks or shared Chunks + +The local backend writes Tree pages and durable roots to local disks. Paxos-KV combines that local engine with replicated consensus and a local WAL on each replica. + +The Chunk backend writes page data and roots through CROWDB's shared storage services. Chunk-KV can reopen the same durable backing from another serving node, reuse page state during range splits, and recover its WAL through ChunkStream. The ordered KV model above the backend stays the same. + +This combination is the reason to own the engine. CAS-based ingestion, slot-aware reads, page indirection, subtree concurrency, and incremental persistence can be designed together with the distributed services that consume them. The same investment supports fast local replicas and movable shared-storage ranges. + +See the [skiplist implementation](../../lib/crowdb-tree/src/memtable), [mapping table](../../lib/crowdb-tree/include/crowdb-tree/maptable), [C++ interface](../../lib/crowdb-tree/include/crowdb-tree/crowdb-tree.h), and [benchmark guide](../../lib/crowdb-tree/bench/README.md). diff --git a/doc/design/design-crowdb.md b/doc/design/design-crowdb.md new file mode 100644 index 000000000..754c1f786 --- /dev/null +++ b/doc/design/design-crowdb.md @@ -0,0 +1,58 @@ + + + +# CROWDB - Design: A Shared Storage Core for Tables, Datasets, and Objects + +CROWDB brings Iceberg tables, AI datasets, and S3 objects onto one distributed storage core. Each has its own identity, publication rules, and read model. They share the machinery that places bytes, protects them, tracks ownership, and recovers after failure. + +The ambition is to make the storage system understand enough of the workload to improve the whole path. A table commit, a training batch, and a multipart object upload need different coordination. Beneath those differences lie the same expensive operations: moving buffers, allocating storage, recording durable state, and rebuilding data. CROWDB develops these pieces together. + +## The workloads that shape the design + +An analytical engine works with tables, snapshots, manifests, and files, using atomic commits to publish changes. It needs a catalog that understands table identity and a file path that can deliver large data files efficiently. CROWDB provides a native Iceberg catalog and storage integration; the analytical engine retains query planning and execution. + +A training application works with selected samples and fields from a fixed dataset version. It needs workers to divide the input, resume progress, and avoid fetching unused payloads. CROWDB organizes datasets into immutable Snapshots, with Splits selecting samples and read continuations recording progress. + +An object application wants a familiar bucket and key, streaming upload, range reads, and predictable behavior when a request fails. S3 compatibility lets existing tools use those operations without learning a new storage API. + +These workloads combine large sequential transfers with many small metadata operations. Optimizing only the disk bandwidth leaves catalog contention, buffer copies, or recovery work as the next bottleneck. CROWDB's architecture addresses those costs at their respective owners. + +## Shared infrastructure, independent data semantics + +![Iceberg, Dataset, and S3 share distributed storage services.](../assets/crowdb-overview.svg) + +Iceberg controls when table metadata becomes the current table state. Dataset controls which immutable Snapshot a read selects. S3 controls when an object becomes visible under its bucket and key. Sharing storage does not turn these into aliases for the same namespace. That distinction gives each access model room to improve without forcing its rules onto the others. + +The benefits compound below the interfaces. A better small-write pipeline can serve several workloads. A repair mechanism protects their payloads. Buffer ownership can remain explicit from the storage transport through a native consumer. Improvements reach across the system because the components agree on the boundaries that matter. + +## The design advantages + +**Native Iceberg removes a storage-system relay.** Table files live directly in Chunk storage, with no separate S3 storage backend. Native uploads return signed URIs carrying the Chunk location, so a manifest-selected data file can be read without another Iceberg file-metadata lookup. Catalog publication and file lifecycle are designed together, reducing the repeated metadata work that appears when independent systems are stacked. See [Iceberg](design-crowdb-access-iceberg.md). + +**Small logical objects share large physical storage work.** Small files and field payloads retain independent identities while sharing Chunk allocation, batching, and write pipelines. Large writes get dedicated pipelines and chunks. This lets the system serve small metadata-heavy workloads and sustained bulk transfers without making every small item own a physical container. See [Chunk I/O](design-crowdb-chunkdb.md#two-write-paths). + +**Dataset storage understands samples before it reads bytes.** Snapshots, Splits, field projection, and worker assignment select the input in metadata before payload transfer. Samples do not each need a filesystem file, and an unused field does not need to enter the training pipeline. Shared chunks address small-payload organization while continuations make repeated work explicit. See [Dataset](design-crowdb-access-dataset.md). + +**Ordered names and distributed bytes work together.** Range metadata preserves the neighborhoods needed by prefix listing, pagination, and ordered scans. Chunk placement independently distributes the referenced payloads across nodes and disks. Names under one prefix can remain adjacent in the index while their data uses the storage cluster's capacity and bandwidth. See [Chunk-KV](design-crowdb-chunk-kv.md#ordered-names-do-not-constrain-payload-placement). + +**Metadata serving can grow without first copying the whole index.** Chunk-KV reuses shared Tree pages during split and prepares a transfer destination from a pinned base and WAL suffix while the source continues serving. Adding a serving node does not require a full private copy of each range it takes over. Smooth split and transfer let serving capacity respond to load independently of physical Chunk relocation. See [Chunk-KV](design-crowdb-chunk-kv.md#serving-capacity-and-storage-capacity). + +**Replicated metadata writes can complete across concurrent slots.** Paxos-KV lets each slot reach its own durable quorum, while per-key slot ordering keeps out-of-order Tree application correct. Independent blind writes can make progress before an earlier gap is resolved, reducing head-of-line waiting through the consensus and engine path. See [Paxos-KV](design-crowdb-paxos-kv.md). + +**The engine's concurrency follows its data structure.** Tree combines CAS-based skiplist tables, slot-aware versions, and Bw-tree mapping indirection. Its flush design forks independent subtree work and joins final page results toward the root; flush with snapshot persists those same results incrementally. Local-disk and Chunk backends let Paxos-KV and Chunk-KV share that engine investment. See [Tree](design-crowdb-tree.md). + +**The journal and Tree form one recovery design.** Tree materializes work recorded by its caller without adding a separate mutation redo WAL. Slot-aware application, durable snapshot coverage, and retained journal history connect in one recovery path. This avoids duplicate mutation logging inside the engine and gives the caller the durable progress needed to decide what to replay and what history can be reclaimed. See [Tree recovery](design-crowdb-tree.md#recovery-is-part-of-the-engine-contract). + +**RDMA reaches the application without an access gateway relay.** Trusted applications embed the native access libraries and communicate with storage nodes directly. RDMA is the core high-performance transport, with TCP providing a fallback. Direct access removes an intermediate payload hop and its protocol and staging work. Retained host buffers are available today; direct GPU-memory delivery through further RDMA / GDS integration extends the same system-level goal. See the [native SDK](design-crowdb-sdk-native.md). + +These advantages reinforce one another. Ordered metadata selects the work, shared chunks aggregate it, direct I/O moves the bytes, and retained ownership carries them to the consumer. Resource budgets cover queued work, in-flight I/O, and buffers still held by an application, so slower consumers can apply backpressure through the path. Durable system metadata coordinates placement and recovery across it. + +## Scope and adoption + +CROWDB is a storage platform. Query execution, model training, and application transactions spanning unrelated data models belong to their respective engines and applications. Its Dataset fields hold application data; CROWDB does not prescribe tensor formats or execute a training graph. + +Access Server and OCI deployment support the storage interfaces, with deployments tested on physical Linux hosts. The [container deployment design](design-crowdb-deployment.md) uses an OCI image without requiring a particular runtime or orchestrator. Host networking and host block devices keep the I/O path direct; container limits control CPU and memory. Dataset services and HTTP/native clients are described in the [Dataset SDK](../../sdk/README.md). Use the [repository README](../../README.md#run-crowdb) for deployment entry points and compatibility boundaries. + +The architecture is especially useful when lakehouse tables, training data, and object storage share the same infrastructure. Improvements to small-write aggregation, range movement, transport, or physical recovery can benefit all three, while each retains its own data model. + +Continue with the [architecture](design-crowdb-architecture.md) or use the [component index](README.md) to explore a specific design. diff --git a/doc/design/diskdb/design-crowdb-diskdb-space-metrics.md b/doc/design/diskdb/design-crowdb-diskdb-space-metrics.md deleted file mode 100644 index 69a045eeb..000000000 --- a/doc/design/diskdb/design-crowdb-diskdb-space-metrics.md +++ /dev/null @@ -1,818 +0,0 @@ - - - -# CROWDB - Design: diskdb Space Metrics + Query API - -Depends on: [`design-crowdb-diskdb.md`](design-crowdb-diskdb.md) §3.4 (records are -source of truth), §7 (record model + recovery strategies), §3.7 (reuse -`crowdb-common` metrics), §8 (allocation algorithm); [`design-crowdb-paxos-kv-group0.md`](../kv/design-crowdb-paxos-kv-group0.md) -(hardware hierarchy + service registry). -Satisfies: `design-crowdb-diskdb.md` §11 (Space Metrics). - -Detailed design for diskdb's space-metrics component — the third major -component after the allocator (§8) and recovery (§7). Architecture -decisions and rationale live in the root design doc; this doc carries -the structs, algorithms, RPC shapes, and data flow. Reused surfaces -(named, not re-built): `DdbDiskGroupContainer`/`DdbDiskGroup`/`DdbDisk`, -`DdbKvClient`, `DiskdbMetrics`, `model/alloc.rs` two-phase -allocate/free, `RecoveryEngine` (`recover_zone_inner`, -`rebuild_zone_bitmap_full_scan`), `CompactionEngine`, -`HardwareClient` (hierarchy scans), `ServiceRegistryClient` -(`read_all_diskdb_instances`, `heartbeat_diskdb` accepting -`DiskGroupUsageSummary[]`), `DiskGroupUsageKey` + `prefix_all`. - -## Table of Contents - -- [1. Per-zone usage accessors](#1-per-zone-usage-accessors) -- [2. Per-disk + per-disk-group aggregation](#2-per-disk--per-disk-group-aggregation) -- [3. Per-disk hot-path counters](#3-per-disk-hot-path-counters) -- [4. QueryCapacityStats handler](#4-querycapacitystats-handler) -- [5. Recalculation path](#5-recalculation-path) -- [6. crowdb-common metrics integration](#6-crowdb-common-metrics-integration) -- [7. Reporting loop](#7-reporting-loop) -- [8. Keepalive usage piggyback](#8-keepalive-usage-piggyback) -- [9. Schema extension](#9-schema-extension) -- [10. kv-client space-usage aggregation](#10-kv-client-space-usage-aggregation) -- [11. diskdb-client — full client library](#11-diskdb-client--full-client-library) -- [12. E2E test migration](#12-e2e-test-migration) -- [Invariants](#invariants) -- [Module Structure](#module-structure) -- [Config Extensions](#config-extensions) -- [Server Wiring](#server-wiring) - -## 1. Per-zone usage accessors - -`DdbZone` tracks `used_count: AtomicU32`, `unit_capacity: u32`, and -`UsageBitmap::count_set()`, but exposes no busy/free/bytes/ratio -accessors. Every higher-level aggregation (disk, disk-group, query -handler, keepalive piggyback, reporting-loop gauges) bottoms out at the -zone, so the accessors live here. CROWDB reuses freed space immediately -via bitmap scan (no append-only `allocate_pos`, §3.4/§8), so -`free = capacity - busy`. There is no separate free cursor to read. - -```rust -/// Busy block units (live `used_count`). -pub fn busy_blocks(&self) -> u32 { self.used_count.load(Ordering::Acquire) } -/// Free block units = `unit_capacity - used_count`. -pub fn free_blocks(&self) -> u32 { self.unit_capacity.saturating_sub(self.busy_blocks()) } -/// Busy bytes = `busy_blocks * unit_size_bytes`. -pub fn busy_bytes(&self, unit_size_bytes: u32) -> u64 { u64::from(self.busy_blocks()) * u64::from(unit_size_bytes) } -/// Free bytes = `free_blocks * unit_size_bytes`. -pub fn free_bytes(&self, unit_size_bytes: u32) -> u64 { u64::from(self.free_blocks()) * u64::from(unit_size_bytes) } -/// Capacity bytes = `unit_capacity * unit_size_bytes`. -pub fn capacity_bytes(&self, unit_size_bytes: u32) -> u64 { u64::from(self.unit_capacity) * u64::from(unit_size_bytes) } -/// Usage ratio = `used_count / unit_capacity` as f64. -pub fn usage_ratio(&self) -> f64 { - let used = f64::from(self.busy_blocks()); - let cap = f64::from(self.unit_capacity); - if cap == 0.0 { 0.0 } else { used / cap } -} -``` - -- `busy_blocks` reads the live atomic; `free_blocks` derives from - capacity. `UsageBitmap::count_set()` is **not** used on the read path - — it is reserved for the recalc verifier (§5) which needs an - independent popcount of the bitmap bytes, not the cached - `used_count`. -- The `unit_size_bytes` argument is per-disk (from - `DiskValue.unit_size_bytes`), not a global shift — disks in one - disk-group may have different unit sizes in principle, so the caller - passes the owning disk's value. - -Edge cases: -- `unit_capacity == 0` (degenerate) → `usage_ratio` returns 0.0 - (guarded). -- Saturating sub on `free_blocks` guards against a transient - `used_count > unit_capacity` (should never happen; a CAS bug would - set it). -- These are read-only and lock-free; safe to call concurrently with - allocate/free. - -## 2. Per-disk + per-disk-group aggregation - -`DdbDisk` and `DdbDiskGroup` own the zone/disk collections but expose -no aggregated usage. The query handler, keepalive piggyback, and -reporting loop all need a single struct built from in-memory state (no -KV reads). The aggregation is a straightforward sum, but the struct -shape is reused in three places, so it is defined once. - -```rust -/// Per-zone brief usage (counts only — no bitmap bytes). -pub struct ZoneUsage { - pub zone_index: u32, - pub capacity_bytes: u64, - pub busy_bytes: u64, - pub free_bytes: u64, - pub busy_block_count: u32, - pub free_block_count: u32, - pub alloc_state: ZoneAllocationState, // derived_alloc_state() - pub zone_state: DdbZoneHealth, -} - -/// Per-disk usage (aggregated across zones). -pub struct DiskUsage { - pub disk_id: DiskId, - pub capacity_bytes: u64, - pub busy_bytes: u64, - pub free_bytes: u64, - pub zone_count: u32, - pub active_zone_count: u32, - pub busy_zone_count: u32, // zones with used_count == unit_capacity - pub free_zone_count: u32, // zones with used_count < unit_capacity -} - -/// Per-disk-group usage (aggregated across disks). -pub struct DiskGroupUsage { - pub disk_group_id: DiskGroupId, - pub capacity_bytes: u64, - pub busy_bytes: u64, - pub free_bytes: u64, - pub disk_count: u32, - pub allocatable_disk_count: u32, - pub disks: Vec, -} -``` - -`DdbDisk::usage(&self, unit_size_bytes: u32) -> DiskUsage`: -- Read `disk_value` + `zones` under their read locks. -- Sum `capacity_bytes`/`busy_bytes`/`free_bytes` via each zone's - accessors (§1) using the disk's `unit_size_bytes` (passed in, or read - from `disk_value.unit_size_bytes`). -- `zone_count` = `zones.len()`; `active_zone_count` = - `active_zone_context.read().len()` (RCU snapshot size). -- `busy_zone_count` = zones with `busy_blocks() == unit_capacity`; - `free_zone_count` = the rest. - -`DdbDiskGroup::aggregate_usage(&self) -> DiskGroupUsage`: -- Read `disks` under the read lock; read each disk's `disk_value` for - its `unit_size_bytes`. -- Sum across disks; `disk_count` = `disks.len()`; - `allocatable_disk_count` comes from the immutable RCU membership snapshot - shared by disk-ID lookup and allocation routing. Membership changes publish - both views together. -- Carry the per-disk `DiskUsage` breakdown in `disks`. - -`DdbDiskGroup::zone_usage(&self, disk_id: DiskId, zone_index: u32) -> Option`: -- Locate the disk in the immutable RCU membership snapshot; `None` if unknown. -- Locate the zone by `zone_index` (read lock on `disk.zones`); `None` - if out of range. -- Build `ZoneUsage` from the zone's accessors + - `derived_alloc_state()` + `*zone.zone_state.read()`. **Brief counts - only** — no bitmap bytes. - -Edge cases: -- Disk with zero zones → `DiskUsage` zeroes, `zone_count == 0`. -- A `Bad` disk is still summed (its zones carry their last-known - `used_count`); it is excluded from `allocatable_disk_count` but its - capacity still counts in the total. This matches "capacity is total; - allocatable is the live set." -- `zone_usage` for an out-of-range index → `None` (caller maps to - `NotFound`). -- Read locks make the snapshot best-effort-consistent across - zones/disks; a concurrent `disk_add_init`/`remove_disk` may miss a - transient disk this tick. Acceptable for metrics (§11: exact - consistency not required; recalc gives exact verification). - -## 3. Per-disk hot-path counters - -§11 calls for per-disk event counters (allocate/free count + bytes) on -the hot path. The capacity/busy/free **gauges** are derived from the -bitmap on the reporting tick (single source of truth). Maintaining -parallel capacity atomics would drift from the bitmap. So the per-disk -hot-path struct holds **only** period/total event counters; gauges are -computed in the reporting loop (§7). - -`metrics.rs` is a module dir: `metrics/mod.rs` (existing `DiskdbMetrics` -moves here) + `metrics/disk.rs`. `DiskMetrics` holds lock-free atomic -counters per `DdbDisk`: - -```rust -pub struct DiskMetrics { - // Period counters (swapped to 0 by the reporting loop each tick). - period_allocate_count: AtomicU64, - period_allocate_bytes: AtomicU64, - period_free_count: AtomicU64, - period_free_bytes: AtomicU64, - // Total counters (monotonic, never reset). - total_allocate_count: AtomicU64, - total_allocate_bytes: AtomicU64, - total_free_count: AtomicU64, - total_free_bytes: AtomicU64, -} - -impl DiskMetrics { - pub fn new() -> Self { /* all zero */ } - /// Bump period + total count/bytes. Called after the Phase 1 bitmap - /// CAS succeeds (exactly once per durable-bound allocation). - pub fn record_allocate(&self, unit_count: u32, unit_size_bytes: u32) { ... } - /// Bump period + total count/bytes. Called after the Phase 1 bitmap - /// clear succeeds. - pub fn record_free(&self, unit_count: u32, unit_size_bytes: u32) { ... } - /// Atomically swap period counters to 0, return the deltas. - pub fn swap_periods(&self) -> PeriodSnapshot { ... } -} - -pub struct PeriodSnapshot { - pub allocate_count: u64, pub allocate_bytes: u64, - pub free_count: u64, pub free_bytes: u64, -} -``` - -- All atomics use `Ordering::Relaxed` (counters, no cross-variable - ordering). -- `record_allocate`/`record_free` do `fetch_add(unit_count)` on count - and `fetch_add(u64::from(unit_count) * u64::from(unit_size_bytes))` - on bytes, on both period and total. -- `swap_periods` does `swap(0, Relaxed)` on each period counter; totals - are kept inline (also incremented in `record_*`) for crash-safe - monotonicity — the reporting loop does **not** add period deltas to - totals (they're already there); it only flushes period deltas into - the crowdb-common `allocate_total`/`free_total` counters. - -Wiring: a `metrics: Option>` field on `DdbDisk`, attached -during `disk_add_init` (keepalive) and `recover_disk_group` (recovery). -`None` in tests that don't care. The allocate/free paths in -`model/alloc.rs` call -`disk.metrics.as_ref().map(|m| m.record_allocate(range.unit_count, unit_size))` -after the Phase 1 CAS succeeds (before Phase 2 persist. The counter -reflects a durable-bound allocation; on Phase 2 failure the bitmap -rolls back but the counter over-counts by one, which is acceptable for a -best-effort event counter and avoids a second hot-path branch on the -persist result). - -Edge cases: -- `AtomicU64` overflow at 1M-unit blocks → extremely unlikely; - `swap_periods` saturating semantics avoid loss; no special handling. -- `DiskMetrics == None` (test disk) → record calls are no-ops; - aggregation unaffected. -- Counter over-count on Phase 2 rollback (above) — documented, - acceptable for an event counter; the `allocate_errors_total` counter - (§6) separately tracks persist failures. - -## 4. QueryCapacityStats handler - -`QueryCapacityStats` is the operator/console drill-down surface for live -per-disk/per-zone usage, the authoritative counterpart to the -kv-client's stale cluster-wide view. The request carries two drill-down -fields (`disk_id`, `zone_index`) to select three query shapes without -three separate RPCs. - -Proto (§9): `QueryCapacityStatsRequest` gains `optional uint64 disk_id = 2` -and `optional uint32 zone_index = 3` (0 = not set). `DiskInfo`/ -`DiskGroupInfo` gain usage fields + `repeated ZoneUsage zone_usages`. -`ZoneUsage` message added. - -The handler in `service/diskdb_service.rs` is read-only, allowed in any -lifecycle phase (like `GetDiskGroupInfo`); it does **not** check -`allows_mutating_rpcs` or `is_degraded`. - -- **Cluster/disk-group level** (`disk_group_id` only, `disk_id == 0`): - if `disk_group_id == 0`, iterate `container.disk_group_ids()` and - build one `DiskGroupInfo` per owned group; if set, look up that one - group (`NotFound` if not owned). Each `DiskGroupInfo` carries the - aggregated capacity/busy/free + member `DiskInfo`s with per-disk - usage + zone counts. **No `zone_usages` entries** at this level. -- **Disk level** (`disk_group_id` + `disk_id` set, `zone_index == 0`): - return the one `DiskInfo` with a `zone_usages` list of **brief** - per-zone `ZoneUsage` entries (counts only — no `usage_bitmap`). Built - via `DdbDisk::usage` + a loop over zones calling `DdbZone` accessors - + `derived_alloc_state`. `NotFound` if the disk is not in the group. -- **Zone level** (`disk_group_id` + `disk_id` + `zone_index` all set): - return the one `ZoneUsage` with the **full `usage_bitmap` bytes** via - `DdbZone::usage_bits.snapshot()`. Out-of-range `zone_index` → - `NotFound`. - -`disk_id == 0` sentinel: `DiskId` is a message (`{high, low}`); 0 means -`Some(DiskId{high:0,low:0})` is treated as "not set" only when the field -is `None`. The schema field is `disk_id: DiskId` so -"not set" = `None`; a real disk with all-zero id is pathological and -rejected at disk-add time. `zone_index == 0` is ambiguous (zone 0 is -valid), so the disk-level shape is selected by `disk_id` being set -**and** `zone_index` being absent. Use proto3 `optional` on `zone_index` -(field presence) so "not set" is distinguishable from 0. (proto3 scalar -presence requires `optional` keyword; flatbuffers generates `Option`.) - -Edge cases: -- `disk_group_id == 0` + zero owned groups → empty `disk_groups`, `Ok`. -- Unowned `disk_group_id` → `NotFound` (matches `GetDiskGroupInfo`). -- Disk with thousands of zones → disk-level returns brief entries (no - bitmap bytes); only zone-level serializes one bitmap. -- Out-of-range `zone_index` → `NotFound` (not silent empty bitmap). -- Zone with `used_count == unit_capacity` → `Full` alloc_state, - `free == 0`; reported as-is, not an error. -- Last zone smaller (word-aligned) → `unit_capacity` is the rounded - value; `capacity_bytes` reflects the real smaller capacity. - -## 5. Recalculation path - -§11 requires "accurate statistics with a recalculation path": replay -the journal into a **separate** bitmap and compare against the live -`DdbZone` to detect drift. This is the exact-verification counterpart to -the best-effort-consistent gauges. The background scanner (§12 of the -root doc) reuses `recalc_all`. v1 reports drift only (no auto-correct; -operator runs `RebuildZoneBitmap`). - -`RecalcEngine` in `metrics/recalc.rs`: - -```rust -pub struct RecalcEngine { - kv: Arc, - container: Arc, -} - -pub struct RecalcResult { - pub disk_id: DiskId, - pub zone_index: u32, - pub matches: bool, - pub live_busy_blocks: u32, - pub replayed_busy_blocks: u32, - pub live_snapshot_slot: u64, - pub replayed_snapshot_slot: u64, - pub drift_detected: bool, - pub fallback_used: Option, // None | JournalScanGcGap | SnapshotCrcFail -} - -pub enum FallbackReason { JournalScanGcGap, SnapshotCrcFail } - -pub struct DiskGroupRecalcResult { - pub disk_group_id: DiskGroupId, - pub zone_results: Vec, - pub drift_detected: bool, -} -``` - -`recalc_zone(bind, disk_id, zone_idx, unit_capacity, live_zone: &DdbZone) -> Result`: -- Call `recover_zone_inner(&self.kv, bind, disk_id, zone_idx, unit_capacity)` - (strategy 2) into a throwaway `DdbZone`. On `JournalScanGcGap` / - `SnapshotCrcFail`, fall back to `rebuild_zone_bitmap_full_scan` - (strategy 1); record `fallback_used`. -- `replayed_busy_blocks` = replayed zone's `usage_bits.count_set()` - (independent popcount, not its `used_count`). -- `live_busy_blocks` = live zone's `busy_blocks()` (its `used_count`). -- `matches = (replayed_busy_blocks == live_busy_blocks)`; - `drift_detected = !matches`. -- v1 does **not** mutate the live zone — drift is reported only. - -`recalc_disk_group(dg_id) -> Result`: iterate all -disks + zones in the disk-group, call `recalc_zone` per zone (bounded -concurrency via a semaphore, matching `recover_disk_group`), aggregate. - -`recalc_all() -> Result>`: iterate -`container.disk_group_ids()`, call `recalc_disk_group` per owned group. - -The `RecalcDiskUsage` admin RPC (§9) delegates to `RecalcEngine`; the -handler lives in `service/diskdb_service.rs` (mutating RPC, requires -`allows_mutating_rpcs`, since recalc does KV reads/journal scans and is -an admin operation, not a read-only query). - -Edge cases: -- `JournalScanGcGap` → strategy 1 fallback; - `fallback_used = Some(JournalScanGcGap)`. -- `SnapshotCrcFail` → strategy 1 fallback; the live bitmap is suspect; - `drift_detected = true` with `FallbackReason::SnapshotCrcFail` even - if counts happen to match (the snapshot was corrupt). -- Strategy 1 also fails → return a `RecalcResult` with - `matches = false`, `drift_detected = true`, and a KV error noted (do - not panic; the operator sees the failure in the response). -- Live `used_count` unchanged after recalc (v1 no auto-fix). - -## 6. crowdb-common metrics integration - -§11 specifies three metric categories (counters, gauges, latency -hierarchy) reusing `crowdb-common`'s `MetricsRegistry`. `DiskdbMetrics` in -`metrics/mod.rs` holds the full set. Gauges are derived snapshots -(bitmap-derived on the reporting tick), not hot-path writes. - -- **Gauges** (`Arc`): `disk_capacity_bytes`, `disk_busy_bytes`, - `disk_free_bytes`, `disk_active_zone_count`, `disk_total_zone_count`, - `dg_capacity_bytes`, `dg_busy_bytes`, `dg_free_bytes`, - `disk_group.imbalance.used_pct_spread`, - `disk_group.imbalance.used_pct_max`, - `disk_group.imbalance.used_pct_min`, `rebalance.plan_count`, - `rebalance.planned_blocks`, `owned_disk_group_count`, `degraded`, and - `last_sync_age_secs`. - (Per-disk/per-disk-group gauges are single instances updated each tick - with the summed value — v1 does not label per-disk-id in the registry; - the reporting loop sums across disks into the single gauge. The - console reads the live per-disk view via `QueryCapacityStats`, not the - metrics log.) -- **Counters** (`Arc`): `allocate_total`, `free_total` (flushed - from per-disk `DiskMetrics` totals by the reporting loop), - `allocate_errors_total`, `sync_success_total`, - `sync_failure_total`, `compaction_records_deleted_total`, - `rebalance.moves.total`, and `rebalance.errors.total`. -- **Latency histograms** (`Arc`, hot paths): - `allocate.rpc.latency_us`, `allocate.bitmap_scan.latency_us`, - `allocate.kv_persist.latency_us`, `free.rpc.latency_us`, - `free.persist.latency_us`, `free.kv_persist.latency_us`. -- **Latency summaries** (`Arc`, cold paths): - `allocate.zone_rotate.latency_us`, `sync.latency_us`, - `sync.read_group0.latency_us`, `sync.apply_changes.latency_us`, - `compaction.latency_us`, `compaction.scan_free.latency_us`, - `compaction.merge_bitmap.latency_us`, `compaction.kv_persist.latency_us`, - `recovery_duration_ms`. - -`register(registry: &mut MetricsRegistry) -> Self` registers all of the -above (plus the two existing R72 counters). `disabled()` constructs a -throwaway registry for tests. - -Instrumentation points (start-instant → observe-on-completion, -nanoseconds): -- `model/alloc.rs` `allocate_block`/`allocate_blocks`: observe - `allocate.rpc.latency_us` (total), `allocate.bitmap_scan.latency_us` - (around `dg.allocate_block(s)`), `allocate.kv_persist.latency_us` - (around `kv.persist_busy_batch`). On persist failure, - `allocate_errors_total.inc()`. -- `model/alloc.rs` `free_block`/`free_blocks`: observe - `free.rpc.latency_us`, `free.persist.latency_us` (around - `dg.free_block`), `free.kv_persist.latency_us`. -- `liveness/keepalive.rs` `tick`: observe `sync.latency_us`, - `sync.read_group0.latency_us` (around `observe_ownership` + - `observe_disks`), `sync.apply_changes.latency_us` (around - reconcile). `sync_success_total`/`sync_failure_total` per tick - outcome. -- `recovery/compaction.rs`: observe `compaction.*` summaries; - `compaction_records_deleted_total` per compaction. - -The latency handles are passed into the allocate/free paths via -`DiskdbMetrics` (already threaded through `BgCtx`; for the service -handler, `DiskdbService` gains an `Arc` field). To avoid -plumbing metrics through every `alloc::allocate_block` signature, wrap -the latency observation in the service handler around the `alloc::` call -(rpc + kv_persist measured together at the handler boundary) and add a -finer-grained `bitmap_scan` observation inside `alloc.rs` guarded by an -optional `&DiskdbMetrics` parameter. v1 measures rpc + bitmap_scan + -kv_persist at the handler/alloc boundary; full per-phase splitting is -best-effort. - -Edge cases: -- Histogram/summary `observe(0)` is valid (sub-microsecond ops). -- Metrics handle `None` in test paths → `disabled()` registry is used. - -## 7. Reporting loop - -The hot-path atomic counters (`DiskMetrics`) + bitmap state must be -bridged to the crowdb-common `Gauge`/`Counter` reporting layer on a -cadence. §11: gauges are derived snapshots updated on the reporting -interval, not hot-path writes. - -`ReportingTask` in `metrics/reporting.rs` implements `BackgroundTask`. -Registered in `BgRunner` alongside keepalive + compaction in `main.rs`. -Every 10 s (configurable via `reporting.interval_secs`, live-reload via -the shared config handle): - -- For each owned disk-group → each disk: - `DiskMetrics::swap_periods()` → flush period deltas into - `allocate_total`/`free_total` counters (`inc_by(period.allocate_count)` - etc.). -- Recompute gauges from the bitmap: - `DdbDiskGroup::aggregate_usage()` → set - `dg_capacity_bytes`/`dg_busy_bytes`/`dg_free_bytes` (summed across - groups); per-disk sums → `disk_*` gauges; - `disk_active_zone_count`/`disk_total_zone_count` summed. The imbalance - gauges expose the minimum, maximum, and spread across the disks observed by - the instance. -- `owned_disk_group_count.set(container.disk_group_ids().len())`; - `degraded.set(if container.is_degraded() {1} else {0})`; - `last_sync_age_secs.set(...)` (tracked from the last successful - keepalive tick — keepalive records a - `last_sync_at: Mutex>` on the container, or the - reporting task reads a shared `AtomicU64` epoch). - -`ReportingTask` owns an `Arc` + -`Arc` + the shared config handle (for the interval). -`run_cycle` does the flush; `trigger` returns `Trigger::TimerFn` reading -`reporting.interval_secs`. - -Edge cases: -- Tick during concurrent `disk_add_init`/`remove_disk` → read locks - make the gauge snapshot best-effort; a transient disk may be missed - this tick. Acceptable (§11). -- Zero owned groups → gauges set to 0, counters unchanged. - -## 8. Keepalive usage piggyback - -Keepalive sends one `DiskGroupUsageSummary` for each owned disk-group through -`heartbeat_diskdb`. Group 0 stores the latest summary at -`/hw/dg_usage/` for the cluster-wide view and placement ranking. The -summary is derived from the bitmap on every keepalive tick, not a source of -truth. - -In `liveness/keepalive.rs` `tick()`, after `observe_disks`, compute one -`DiskGroupUsageSummary` per owned disk-group from `aggregate_usage()`: - -```rust -let owned_dg_ids: Vec = ...; -let group_usages: Vec = container - .disk_group_ids() - .into_iter() - .map(|dg_id| { - let dg = container.get_disk_group(dg_id).unwrap(); - let u = dg.aggregate_usage(); - DiskGroupUsageSummary { - disk_group_id: dg_id, - capacity_bytes: u.capacity_bytes, - used_bytes: u.busy_bytes, - free_bytes: u.free_bytes, - disk_count: u.disk_count, - allocatable_disk_count: u.allocatable_disk_count, - allocatable_capacity_bytes: u.allocatable_capacity_bytes, - allocatable_used_bytes: u.allocatable_busy_bytes, - allocatable_free_bytes: u.allocatable_free_bytes, - sampled_at_ms: unix_time_ms(), - } - }) - .collect(); -``` - -`KeepAlive::tick` passes the owned IDs and summaries to -`heartbeat_diskdb`. The endpoint is the reachable DiskDB RPC address used by -client routing. The summary is recomputed each tick rather than cached. - -Edge cases: -- Zero owned groups → empty `group_usages` (not an error). -- `heartbeat_diskdb` fails (group 0 unreachable) → keepalive already - handles this (missed count → degraded); the summary is just not - delivered this tick. -- Group 0 stores **only disk-group-level** usage; per-disk/per-zone - live usage is never written to group 0. - -## 9. Schema extension - -`lib/crowdb-protocol/src/fbs/diskdb_type.fbs`: - -```fbs -table ZoneUsage { - zone_index: uint32; - capacity_bytes: uint64; - busy_bytes: uint64; - free_bytes: uint64; - busy_block_count: uint32; - free_block_count: uint32; - alloc_state: ZoneAllocationState; - // Populated only for a specific-zone query; omitted at disk level. - usage_bitmap: [ubyte]; -} -``` - -`DiskInfo` gains `busy_units`, `free_units`, `capacity_bytes`, -`busy_bytes`, `free_bytes`, `active_zone_count`, and -`[ZoneUsage] zone_usages`. `DiskGroupInfo` gains -`capacity_bytes`, `busy_bytes`, `free_bytes`, -`allocatable_disk_count`. - -`diskdb_op.fbs`: -- `QueryCapacityStatsRequest`: add - `disk_id: DiskId` and - `zone_index: uint32`. -- Add `RecalcDiskUsageRequest { disk_group_id: uint64; }` / - `RecalcDiskUsageResponse { results: [DiskGroupRecalcResult]; }` - + `DiskGroupRecalcResult { disk_group_id: uint64; drift_detected: bool; - zones: [ZoneRecalcResult]; }` + `ZoneRecalcResult { ... }` - (mirror `RecalcResult`). - -`diskdb_service.fbs`: add -`rpc RecalcDiskUsage(RecalcDiskUsageRequest) -> (RecalcDiskUsageResponse);` - -Edge cases: -- flatbuffers scalar presence → flatc generates `Option` so - `zone_index` absence is distinguishable from 0. -- `usage_bitmap` is `[ubyte]` → `Option>`; omitted at - disk level. - -## 10. kv-client space-usage aggregation - -No client reads `/hw/dg_usage/*` back. The console needs a -cluster/rack/node/disk-group capacity view built from the piggyback -summaries joined with the hardware hierarchy. This is the stale (≤1 -sync interval) cluster-wide view; live per-disk/per-zone drill-down is -via the diskdb-client (§11). - -`SpaceUsageClient` in `lib/crowdb-paxos-kv-client/src/space_usage.rs` wraps -`HardwareClient` (which already has the hierarchy scans + -`scan_prefix`). Re-exported from `lib.rs`. - -```rust -pub struct SpaceUsageClient { hw: HardwareClient } - -impl SpaceUsageClient { - pub fn new(hw: HardwareClient) -> Self { Self { hw } } - - /// Prefix-scan `/hw/dg_usage/` → one summary per disk-group. - pub async fn list_disk_group_usages(&self) - -> Result>; - - /// Join summaries with the hardware hierarchy → cluster-level totals. - pub async fn cluster_usage(&self) -> Result; - - /// Scoped aggregations down the hierarchy. - pub async fn rack_usage(&self, rack_id: RackId) -> Result; - pub async fn node_usage(&self, rack_id: RackId, node_id: NodeId) -> Result; - pub async fn disk_group_usage(&self, dg_id: DiskGroupId) -> Result>; -} -``` - -- `list_disk_group_usages`: `scan_prefix::` over - `DiskGroupUsageKey::prefix_all()` text path (`/hw/dg_usage/`). The - values are JSON-encoded (group 0 stores text-path keys with JSON - values, matching `HardwareClient`'s convention). -- `cluster_usage`: read all summaries + `list_racks`/`list_nodes`/ - `list_disk_groups`/`list_disks_in_group`; sum capacity/used/free + - disk-group count. Per-disk live busy/free is **not** available here - (group 0 has only disk-group-level usage + per-disk static - `DiskValue.capacity_units`); the disk-level view carries capacity - only. -- `rack_usage`/`node_usage`/`disk_group_usage`: scope the join to the - hierarchy subtree. - -`ClusterUsage`/`RackUsage`/`NodeUsage` structs hold capacity/used/free -bytes + disk-group/disk counts. Mirror `crowdb-paxos-kv-client`'s retry pattern -(the underlying `HardwareClient` already retries). - -Edge cases: -- No summaries written yet (fresh cluster) → empty vec / zero totals. -- Stale summary (≤1 sync interval behind) → documented; not an error. -- Per-disk busy/free absent → `NodeUsage.disk_capacity_bytes` only, no - busy/free. - -## 11. diskdb-client — full client library - -`crowdb-diskdb-client` is the primary client surface for all diskdb rpc -operations (allocate/free/query with retry + endpoint caching), -mirroring `crowdb-paxos-kv-client`'s pattern. Without it, callers must use raw -rpc stubs and do endpoint discovery manually. - -`DiskdbClient` in `lib/crowdb-diskdb-client/src/client.rs`: - -```rust -pub struct DiskdbClient { - svc: ServiceRegistryClient, // endpoint discovery from group 0 - routing: Arc, // shared generated RCU routes - rpc_transport: Arc, - retry: RetryConfig, -} - -impl DiskdbClient { - pub fn new(svc: ServiceRegistryClient) -> Self; - /// Eager warm: read all diskdb instances, populate cache. - pub async fn refresh_endpoints(&self) -> Result<()>; - /// Lazy refresh on cache miss for one dg_id. - async fn refresh_for(&self, dg_id: DiskGroupId) -> Result<()>; - - pub async fn allocate_blocks(&self, req: AllocateBlocksRequest) -> Result; - pub async fn free_blocks(&self, req: FreeBlocksRequest) -> Result; - pub async fn query_capacity_stats(&self, req: QueryCapacityStatsRequest) -> Result; - pub async fn query_disk_group(&self, dg_id: u64) -> Result; - pub async fn query_disk(&self, dg_id: u64, disk_id: DiskId) -> Result; - pub async fn query_zone(&self, dg_id: u64, disk_id: DiskId, zone_index: u32) -> Result; - pub async fn get_disk_group_info(&self, dg_id: u64) -> Result; - pub async fn get_disk_info(&self, dg_id: u64, disk_id: DiskId) -> Result; - pub async fn recalc_disk_usage(&self, req: RecalcDiskUsageRequest) -> Result; -} -``` - -- **Endpoint discovery + cache**: `refresh_endpoints` calls - `svc.read_all_diskdb_instances()`, reads each - `InstanceValue.rpc_endpoint` + `DiskdbExtra.owned_dg_ids`, and publishes one - complete `dg_id -> endpoint` snapshot. Called on startup (eager), on cache - miss (lazy `refresh_for`), and on retryable routing failure. Client clones - share this state; incrementally learned disk routes carry the endpoint - generation and are reconciled after refresh. -- **Connection pool**: `ConnectionPoolIndex` holds immutable generated pools - per endpoint. Connections are reused across calls, and late failures can - invalidate only the generation used by that request. -- **`allocate_blocks`**: look up endpoint for `req.disk_group_id` - (refresh on miss), open channel, call `AllocateBlocks`, retry on - transient errors (`Unavailable`, deadline-exceeded). - `ResourceExhausted` (no space) → return to caller (not retryable). -- **`free_blocks`**: the request carries `Segment`s (each with - `disk_id`); route by looking up which diskdb instance owns the - `disk_id`'s disk-group. If segments span multiple disk-groups, split - the request per-group and issue one `FreeBlocks` per group (v1: no - cross-group batch). Retries on transient errors. To map - `disk_id -> dg_id`, the client needs the disk→dg mapping from group - 0 (`HardwareClient::list_disks_in_group` per dg, or a cached reverse - map built during `refresh_endpoints` from the hardware hierarchy). v1 - builds a `disk_id -> dg_id` reverse map during `refresh_endpoints` by - reading the hardware hierarchy once; refreshes on cache miss. -- **`query_*`**: convenience helpers building the right - `QueryCapacityStatsRequest` shape (§4). - `query_disk_group(dg_id)` → `disk_id=None, zone_index=None`; - `query_disk(dg_id, disk_id)` → `disk_id=Some, zone_index=None`; - `query_zone(dg_id, disk_id, zi)` → all set. -- **`recalc_disk_usage`**: wraps the admin RPC. -- **Error model**: `DiskdbClientError` with `Rpc(crowdb_rpc::Status)` / - `NoSpace` / `NotFound` / `InvalidArgument` mappings. `RetryConfig` - mirrors `crowdb-paxos-kv-client` (max retries, backoff). - -Edge cases: -- Cache miss (unknown `dg_id`) → lazy refresh + retry; if still not - found → `Unreachable`. -- Endpoint moved (instance restarted on new port) → cached endpoint - stale → next call `Unavailable` → refresh → retry → succeeds. v1 - reactive (proactive refresh needs group-0 notify/watch, tracked as a - use case). -- `free_blocks` spans multiple disk-groups → split per-group; partial - failure returns per-group results (v1: returns the first error; a - follow-up can aggregate). Documented. -- v1 no proactive refresh — no background refresh task. - -## 12. E2E test migration - -New rpc-level e2e tests in `lib/crowdb-diskdb-client/tests/` using -`DiskdbClient` against an in-process diskdb rpc server + `KvCluster` -(mirroring `crowdb-paxos-kv-client/tests/` pattern): -- `allocate_free_e2e.rs` — allocate → verify busy record → free → - verify free. -- `query_e2e.rs` — allocate → - `query_disk_group`/`query_disk`/`query_zone`. -- `endpoint_cache_e2e.rs` — endpoint discovery, cache miss → refresh → - retry. -- `recalc_e2e.rs` — allocate → recalc → matches; corrupt live bitmap → - drift. - -The existing in-process `app/crowdb-diskdb/tests/diskdb_e2e_test.rs` -(calls `alloc::allocate_block` directly) stays as a server-internal -integration test. - -## Invariants - -- **I1 — Free = capacity − busy**: CROWDB reuses freed space immediately - via bitmap scan (no append-only `allocate_pos`, §3.4/§8); there is no - separate free cursor. `free_blocks = unit_capacity - used_count`. -- **I2 — Gauges are derived snapshots**: capacity/busy/free gauges are - recomputed from the bitmap on the reporting tick (single source of - truth), not maintained as parallel hot-path atomics that could drift. -- **I3 — Recalc uses an independent popcount**: the recalc verifier - reads `usage_bits.count_set()` on the replayed zone (not its - `used_count`) and compares against the live `used_count`, so drift in - either the bitmap or the cached counter is detected. -- **I4 — Group 0 carries disk-group-level usage only**: per-disk and - per-zone live usage are never written to group 0 (a disk can have - thousands of zones; group 0 is not a per-zone registry). Per-disk/ - per-zone drill-down is served by diskdb directly. -- **I5 — Counters increment once per durable-bound op**: - `record_allocate`/`record_free` fire after the Phase 1 bitmap CAS - succeeds, so a CAS-failed retry does not bump the counter. A Phase 2 - rollback over-counts by one — documented, acceptable for a - best-effort event counter. - -## Module Structure - -``` -app/crowdb-diskdb/src/ - model/ - zone.rs # + accessors, ZoneUsage - disk.rs # + DiskUsage, DdbDisk::usage, metrics field - disk_group.rs # + DiskGroupUsage, aggregate_usage, zone_usage - usage.rs # (optional) ZoneUsage/DiskUsage/DiskGroupUsage structs - alloc.rs # + DiskMetrics record calls + latency observation - metrics/ - mod.rs # DiskdbMetrics (extended §11 set) - disk.rs # DiskMetrics (period/total counters) - recalc.rs # RecalcEngine + RecalcResult - reporting.rs # ReportingTask (BackgroundTask) - service/diskdb_service.rs # QueryCapacityStats + RecalcDiskUsage handlers - liveness/keepalive.rs # usage piggyback + endpoint + sync metrics - ddb_config.rs # + ReportingConfig - main.rs # wire ReportingTask + RecalcEngine + DiskdbMetrics -lib/crowdb-protocol/src/fbs/ - diskdb_type.fbs # ZoneUsage, extend DiskInfo/DiskGroupInfo - diskdb_op.fbs # extend QueryCapacityStatsRequest, RecalcDiskUsage* - diskdb_service.fbs # RecalcDiskUsage RPC -lib/crowdb-paxos-kv-client/src/ - space_usage.rs # SpaceUsageClient + ClusterUsage/RackUsage/NodeUsage - lib.rs # re-export -lib/crowdb-diskdb-client/src/ - lib.rs # DiskdbClientError (extended) - client.rs # DiskdbClient (allocate/free/query/recalc + cache) -app/crowdb-diskdb/tests/ - space_flow_test.rs # whole-flow single + multi-thread -lib/crowdb-diskdb-client/tests/ - allocate_free_e2e.rs - query_e2e.rs - endpoint_cache_e2e.rs - recalc_e2e.rs -``` - -## Config Extensions - -- `ddb_config.rs`: `ReportingConfig { interval_secs: u32 }` (default - 10), added to `DdbConfig` as `pub reporting: ReportingConfig`. - `validate()`: `interval_secs > 0`. Live-reloadable (dynamic field), - read by `ReportingTask::trigger` via the shared config handle. - -## Server Wiring - -1. `main.rs`: build `DiskdbMetrics::register(&mut metrics_registry)` - (extended set), already constructed; the §11 handles are now - populated. -2. Construct `RecalcEngine::new(Arc::clone(&dg_kv), Arc::clone(&container))`; - pass `Arc` + `Arc` into - `DiskdbService::new` (signature gains two fields). -3. Construct `ReportingTask::new(container, metrics, config_handle)`; - register in `BgRunner` alongside keepalive + compaction. -4. `keepalive` gains the real `server.listen_addr` endpoint for the - piggyback (passed via `with_endpoint` or read from config in `tick`). -5. Recovery (`run_recovery`) attaches `DiskMetrics` to each recovered - disk (matching `disk_add_init`). diff --git a/doc/design/diskdb/design-crowdb-diskdb-zone-management.md b/doc/design/diskdb/design-crowdb-diskdb-zone-management.md deleted file mode 100644 index f85bf7bb1..000000000 --- a/doc/design/diskdb/design-crowdb-diskdb-zone-management.md +++ /dev/null @@ -1,930 +0,0 @@ - - - -# CROWDB - Design: diskdb Zone Management - -Depends on: [`design-crowdb-diskdb.md`](design-crowdb-diskdb.md) §3.3 (no CAS, -exclusive ownership), §3.9 (unit-based sizes, disk-id key routing), §6 -(hierarchy); -[`design-crowdb-protocol-key.md`](../protocol/design-crowdb-protocol-key.md) -(binary key encoding); -[`design-crowdb-paxos-kv-state-machine.md`](../kv/design-crowdb-paxos-kv-state-machine.md) -(slot tracking, apply semantics). -Satisfies: `design-crowdb-diskdb.md` §7 (Zone Records and Crash Recovery), -§8 (Allocation Algorithm), §9 zone allocation state, §14 zone-level -concurrency. - -Detailed design for diskdb's zone-management component — the zone -allocator, record model, free path, compaction, rotation, crash -recovery, and zone-level concurrency. Architecture decisions and -rationale live in the root design doc; this doc carries the structs, -algorithms, record layouts, and coordination rules. Reused surfaces -(named, not re-built): `DdbZone`, `DdbDisk`, `DdbDiskGroup`, -`DdbDiskGroupContainer`, `DdbKvClient`, `CompactionEngine`, -`RecoveryEngine`, `UsageBitmap`, `ZoneValueExt`. - -## Table of Contents - -- [1. Scope](#1-scope) -- [2. Zone as a Logical Concept](#2-zone-as-a-logical-concept) -- [3. Record Model](#3-record-model) - - [Record key layout](#record-key-layout) - - [Value schemas](#value-schemas) - - [Record model](#record-model) - - [Current state determination](#current-state-determination) -- [4. Allocation Algorithm](#4-allocation-algorithm) - - [Zone-level allocate (sync, in-memory)](#zone-level-allocate-sync-in-memory) - - [Disk-level allocate (sync) — rotating active-zone-set](#disk-level-allocate-sync--rotating-active-zone-set) - - [Round-robin across disks within the named disk-group (sync)](#round-robin-across-disks-within-the-named-disk-group-sync) - - [Two-phase async allocation](#two-phase-async-allocation) - - [Free (persist-only)](#free-persist-only) -- [5. Zone Rotation and Compaction](#5-zone-rotation-and-compaction) - - [Compaction-on-rotation](#compaction-on-rotation) - - [Preparatory thread](#preparatory-thread) - - [Periodic compaction fallback](#periodic-compaction-fallback) - - [Compaction algorithm](#compaction-algorithm) - - [Ready-zone tracking](#ready-zone-tracking) -- [6. Crash Recovery](#6-crash-recovery) - - [Three recovery strategies](#three-recovery-strategies) - - [How the strategies work together](#how-the-strategies-work-together) - - [Crash-safety invariants](#crash-safety-invariants) - - [Recovery and compaction engines](#recovery-and-compaction-engines) - - [Recovery triggers](#recovery-triggers) -- [7. Zone Allocation State (derived)](#7-zone-allocation-state-derived) -- [8. Concurrency Model](#8-concurrency-model) - - [Allocate (lock-free)](#allocate-lock-free) - - [Free (persist-only, no lock)](#free-persist-only-no-lock) - - [Zone-level lock for non-allocate operations](#zone-level-lock-for-non-allocate-operations) - - [Common methods on DdbZone](#common-methods-on-ddbzone) - - [Free-side lookup structures](#free-side-lookup-structures) - - [Monotonic timestamp source](#monotonic-timestamp-source) -- [9. Background Scanner Coordination](#9-background-scanner-coordination) -- [10. Invariants](#10-invariants) -- [11. Tunables and Defaults](#11-tunables-and-defaults) - -## 1. Scope - -This doc covers everything zone-level: the in-memory zone allocator -(bitmap-scan with per-bit CAS), the durable record model (busy/free/ -snapshot records), the free path (persist-only, bitmap untouched on -free), compaction (the sole bit-clearer), zone rotation (compaction- -before-rotation with a preparatory thread), crash recovery (three -strategies), and the zone-level concurrency model (lock-free allocate, -zone-level lock for non-allocate operations). - -What is **not** here: disk-level status management (root §10), space -metrics (root §11 / space-metrics sub-design), background scanner design -(root §12 / scanner sub-design), group-0 sysdata schema (root §5). Those -docs reference this one for zone-level behavior. - -## 2. Zone as a Logical Concept - -Not all zones on a disk must be the same size. The last zone may be -smaller (disk capacity is rarely an exact multiple of the zone size). -Zone is a **logical concept** defined for easier implementation; it can -later adopt to native zoned-namespace SSD or SMR HDD zone APIs, but -currently no such devices are targeted. The zone is a rough mapping. - -**Word alignment rule:** each zone's `unit_capacity` must be a multiple -of 64 (the 64-bit bitmap word size). All zones except the last have -`unit_capacity = zone_size / unit_size`. The last zone has -`unit_capacity = remaining_capacity / unit_size`, rounded down to a -multiple of 64; the sub-64-unit tail (at most 63 units) is unallocated. -Only the last zone may have a different size; all other zones on a disk -are uniform. No bitmap masking, no padding bits. Each zone's bitmap is -sized to its own `unit_capacity`, which is word-aligned. - -## 3. Record Model - -### Record key layout - -Three types of KV entries on the disk-group's bound data group. Keys -use the cross-component binary encoding -(`doc/design/protocol/design-crowdb-protocol-key.md`): each is a flat -struct with a `magic | type_tag` header and big-endian fixed-width -fields. Keys are disk-id-based (globally unique → reverse-lookup to the -data group). No `node_id`/`disk_group_id` in record keys. - -``` -# Busy block (on allocate; deleted by matching compaction) -BusyBlockKey { disk_id, zone_index, unit_offset } -> BusyBlockValue - -# Immutable free fact (on free; transient — deleted by compaction) -FreeBlockKey { disk_id, zone_index, unit_offset, allocation_ts } - -> FreeBlockValue - -# Zone (compacted snapshot, periodic) -ZoneKey { disk_id, zone_index } -> ZoneValue -``` - -`disk_id` is 16 bytes (`high:u64 BE | low:u64 BE`); `zone_index` is -`u32 BE`; `unit_offset` and `allocation_ts` are `u64 BE`. Big-endian fixed width gives -lexicographic byte order == numeric order, so a prefix scan of one -zone's busy (or free) keys returns blocks in `unit_offset` order -without deserialization. Exact byte layouts and prefix constructors -(e.g. `BusyBlockKey::prefix_for_zone(disk_id, zone_index)`) are in -`doc/design/protocol/design-crowdb-protocol-key.md`; value field details -are in `diskdb_type.fbs`. - -### Value schemas - -- **`BusyBlockValue`**: - - `allocation_ts: u64` — monotonically increasing allocation incarnation, - also returned in the `Segment`. It is an identity token, not a KV commit - timestamp or a compaction-ordering value. - - `unit_count: u32` — number of units in this allocation (≥ 1; - multi-unit allocations span consecutive offsets). - - `unit_size: u32` — size of one unit in bytes (e.g. 1 MB). Carried - per-block so the data-IO layer knows the block's granularity - without a separate lookup. - - `owner_chunk: ChunkId` (192-bit) — reverse reference to the - block's owner (the chunk that holds this block's data). Needed for - recovery and other logic. - - `state: BlockState` — per-block I/O-behavior state enum: - `Ok` (normal; default on allocate), `Suspect` (data may be - unreadable; data-IO layer tries with a timeout, falls back to - mirror/EC rebuild), `Corrupt` (data confirmed unreadable; data-IO - layer skips read, rebuilds from EC/mirror). Updated by background - paths (sync, health probe, scanner) or via `MarkBlockSuspect` / - `MarkBlockCorrupt` admin RPCs — **not** by the allocate hot path. -- **`FreeBlockValue`**: - - `pre_allocation_ts: u64` — the exact busy incarnation released by this - immutable free fact. It equals the `allocation_ts` in the request segment. - - `unit_count: u32` — number of units freed (matches the - corresponding `BusyBlockValue.unit_count`). - - `previous_owner: ChunkId` (192-bit) — the `owner_chunk` from the - `BusyBlockValue` that was freed (carried in the `Segment` on the - free request; no KV read needed). Carried for audit / scanner - cross-check. No `state` field — a free block has no data. - - `free_ts: u64` — diagnostic time at which free was submitted. It is useful - for logs and age metrics but never controls correctness or compaction. -- **`ZoneValue`**: - - `bitmap_codec: u8` — `1` identifies a size-prefixed LZ4 block. Unsupported - codecs and unversioned records are rejected; old records are not supported. - - `usage_bitmap: bytes` — LZ4-compressed zone bitmap (one bit per unit; - bit set = busy, bit clear = free). Decoded length must match the zone's - `unit_capacity`, rounded to complete 64-bit words. The decoded length is - checked before allocating a decompression buffer. A 4 KiB all-zero bitmap - uses a shared 31-byte compressed constant. - - `snapshot_slot: u64` — the slot at which this snapshot was written. - Strategy 2 (journal scan replay) replays operations after this slot. - - `compact_slot: u64` — the last successfully persisted fixed - contiguous-applied scan cutoff. It records scan progress, but does not - exclude a surviving immutable free fact at or below an earlier cutoff; - such a fact must still be reconciled. Initialized to zero for a baseline - snapshot. - - `crc32: u32` — CRC32 checksum over `bitmap_codec` + compressed - `usage_bitmap` + `compact_slot` - for integrity verification (§9 scanner coordination). - -### Record model - -- A busy/free entry can span multiple units (`unit_count` ≥ 1). -- On free, DiskDB blindly writes `FreeBlockKey(..., allocation_ts)` and does - not delete `BusyBlockKey`. Retrying the same segment rewrites the same - logical fact; a stale retry targets only its old incarnation. -- Compaction matches `FreeBlockValue.pre_allocation_ts`, `unit_count`, and - `previous_owner` against the current `BusyBlockValue`. Only a complete - match clears the bitmap and deletes the busy record. A mismatch is stale or - invalid and cannot change the busy state. -- Re-allocation occurs only after compaction has cleared the bitmap and - deleted the prior busy incarnation. It writes a higher `allocation_ts`. -- Current durable state is derived from the current busy record plus any - matching incarnation-qualified free fact. Before compaction both remain - durable and the bitmap conservatively stays busy; after compaction neither - remains for a freed incarnation. -- `ZoneValue` carries a CRC32 checksum for integrity verification. - -### Current state determination - -The full `ZoneValue` is a **compacted snapshot** written periodically. -The ideal approach on free would be to update the bitmap in the -`ZoneValue` and write the whole `ZoneValue` to KV. But `ZoneValue` is -large (full bitmap), and frees are random across all zones and disks, -so a per-free `ZoneValue` write is too expensive. Instead, each free -writes one small immutable incarnation-qualified fact. Later, compaction -bounded-scans busy and free records, validates matching incarnations, clears -only matched bitmap ranges, and atomically persists the updated snapshot plus -the exact record deletions. - -On crash/restart, diskdb reconstructs the in-memory zone state using -three complementary strategies (§6). - -**Note on scan ordering:** a key prefix scan returns records in -**lexicographic key order** (= `unit_offset` order), not slot order. -`min_slot` on the ordinary crowdb-paxos-kv scan request is a read-freshness floor, -not a record-slot filter. Compaction uses the R132 fixed-cutoff bounded scan, -which returns each current record's `commit_slot`. Strategy 2 uses -`JournalScan` for slot-ordered replay. - -## 4. Allocation Algorithm - -The algorithm uses bitmap-scan with a rotating cursor, per-bit CAS, -and disk-level zone rotation. The only difference from a local-file -allocator is persistence (CROWDB writes `BusyBlockValue`/ -`FreeBlockValue` to the KV journal per-allocation instead of saving the -full bitmap to a local file). - -### Zone-level allocate (sync, in-memory) - -Bitmap-scan with per-bit CAS (no zone-level lock): -1. Check zone health is Healthy and `used_count < unit_capacity`. -2. Scan the bitmap from `last_pos_64` (rotating cursor), wrapping - around. For each 64-bit word, use `countr_one` to find the first - zero bit (hardware-optimized). -3. CAS-set the bit via `compare_exchange` on the 64-bit word. On CAS - failure (another thread set the same bit), re-scan the same word. - **CAS retry bound:** per-bit CAS is capped at `cas_retry_limit` - retries (default 100); on exhaustion, fall through to the next bit / - word / zone. This prevents indefinite spinning under heavy - contention. The `cas_retry_count` atomic (and the wired crowdb-common - counter) is incremented on each retry as the key operational signal - for lock-free allocator contention. -4. For `unit_count > 1`: find `unit_count` consecutive zero bits (may - span words), CAS-set each; on any CAS failure, clear bits already - set in this attempt and continue scanning. -5. On success: update `last_pos_64`, increment `used_count`, return - `AllocatedRange { unit_offset, unit_count }`. No zone-level state - change. Other threads can allocate concurrently. - -### Disk-level allocate (sync) — rotating active-zone-set - -1. Load the current `ActiveZoneContext` (RCU read, `Arc` clone, no - lock held). This is a small set of `zone_rotate_count` zones. -2. Round-robin over the active set via `pos_v_zone_ctx.fetch_add(1)`. -3. Call `zone.allocate(unit_count)` on each zone. On success, return. -4. If all zones in the active set fail, call `rotate_active_zones()`: - publish the next set of **ready** (pre-compacted) zones using RCU - compare-and-swap. If too few ready zones exist, include other allocatable - zones; preparatory or periodic compaction handles their retained free facts. - Advance the scan cursor past the last selected position in either pass. - A failed large allocation can leave smaller free ranges in the current - zones; rotation must continue to later zones rather than repeat that set. -5. Retry with the new context (up to `zone_num / zone_rotate_count + 2` - loops). - -### Round-robin across disks within the named disk-group (sync) - -The `AllocateBlocks` request carries `disk_group_id` (root §3.2); -allocate is scoped to one disk-group. diskdb never round-robins across -disk-groups. Within that named disk-group, an `AtomicU64` iterator -(`pos_v_disk_ctx`) round-robins across disks: each allocation increments -and selects `iterator % num_disks` from the `AllocateDiskContext` (RCU -context of allocatable disks within the named disk-group). Multi-block: -`fetch_add(count)` distributes across disks within the group. -`exclude_disks` (anti-affinity, per-disk, skip a disk that just failed) -is applied within the named disk-group. Multi-block uses one -`batch_write` on the disk-group's bound paxos data group; atomic within -the group. No cross-group multi-block allocate in v1. - -`allow_disk_reuse` is reserved for a caller that already validated a joint -placement plan, such as ChunkDB's mirror-to-EC reservation. When enabled, -allocation resets the per-request used-disk set only after a full -anti-affinity pass, so each pass still spreads across distinct disks. The -complete set of claims is persisted in the same single `batch_write` and is -rolled back together on allocation or persistence failure. - -### Two-phase async allocation - -1. **Phase 1 (sync)**: bitmap-scan allocate (nanoseconds). Bits are set - in `usage_bits` via per-bit CAS. No zone-level lock. Other threads - can allocate concurrently. -2. **Phase 2 (async)**: `.await` on crowdb-paxos-kv `batch_write` that Puts - the `BusyBlockValue` with a new `allocation_ts` at `BusyBlockKey { - disk_id, zone_idx, unit_offset }`. On success, return a `Segment` - carrying the same incarnation. On failure: - `zone.rollback_allocate(unit_offset, unit_count)` (clear the bits - that were set in Phase 1), return error. - -For multi-block: Phase 1 allocates all blocks (sync, may span multiple -zones/disks within one disk-group), Phase 2 uses one `batch_write` per -data group (one async round-trip per group). - -### Free (persist-only) - -The free path is **persist-only**: blindly put one incarnation-qualified -`FreeBlockValue`. It does not read or delete `BusyBlockKey`. The in-memory bitmap is **not** -touched. The bit stays set, `used_count` is not decremented. The block -is freed on disk but still shows busy in memory until compaction clears -the bit (§5). This is the data-safety principle: the bitmap is a -conservative over-estimate that never shows freed space as available -until compaction reconciles it from records. - -After durable persistence, the free path increments -`uncompacted_free_record_count` (an atomic counter per zone) so compaction -knows there is work to do. The bitmap reconciliation remains deferred to -compaction. - -Free steps: -1. **Persist**: put `FreeBlockValue` at `FreeBlockKey { disk_id, - zone_index, unit_offset, allocation_ts }`. The value carries - `pre_allocation_ts = allocation_ts` from the segment and diagnostic - `free_ts`. No read-before-write validation is required for correctness. -2. **Defer validation**: compaction later compares allocation timestamp, unit - count, and owner with the current busy value. A forged or stale free cannot - clear the bitmap or delete a newer busy incarnation. -3. **Post-persist (in-memory)**: increment - `uncompacted_free_record_count` on the zone (lookup via zone-index → - zone vec). No bitmap mutation, no `used_count` decrement. -4. Record per-disk event counter after the durable free. - -Persist-before-bitmap-clear is not needed here because the bitmap is -never cleared on free. The persist is the entire free operation. If the -persist fails, the block is still busy on both disk and memory. The -caller can retry safely. If the persist succeeds, the block is free on -disk; the in-memory bitmap still shows it busy (conservative over- -estimate) until compaction reconciles. - -When concurrent-free coalescing is enabled, preparation first deduplicates and -captures each request's disk-group bind without changing tentative or zone -state. Enqueue uses a lock-free MPSC queue and an atomic single-drainer claim. -The drainer starts immediately—there is no minimum threshold or timer—and -combines whole requests for the same bind up to the smallest captured maximum. -One oversized request is persisted alone rather than split. Requests arriving -while a KV write is in flight form the next natural batch. - -Every covered waiter resolves only after its combined `batch_write` completes. -Success applies tentative removal, compaction-backlog increments, and free -metrics once per distinct incarnation across the batch. Failure resolves all -covered waiters without accounting changes or automatic re-enqueue; an -idempotent caller retry writes the same incarnation-qualified fact. Shutdown -closes admission, drains both direct and coalesced accepted frees, and only -then stops the RPC server. The disabled path retains one durable KV batch per -request under the same lifecycle tracker. - -**`rollback_allocate` — allocate-only bitmap clear:** the -`DdbZone::rollback_allocate` method (CAS-clear bits, decrement -`used_count`) is used **only** by the allocate Phase 2 failure path to -undo Phase 1's bitmap claim. It is **not** called by the free path. This -method replaces the former `DdbZone::free` that was used by both paths. - -## 5. Zone Rotation and Compaction - -The disk maintains a small active zone set (`zone_rotate_count` zones, -default 4). When all zones in the set are exhausted (no free bits), the -set rotates: a new set of allocatable zones is selected from a rotating -scan position and published via RCU. This spreads wear across zones and -avoids bias toward low-index zones. - -### Compaction-on-rotation - -Before a zone enters the active set, it **must be compacted**: freed -bits cleared, `used_count` recomputed, snapshot written, free records -deleted. This ensures the allocator sees accurate free space in active -zones. Without compaction, a rotated-in zone would have stale set bits -for freed-but-not-compacted blocks, and the allocator would skip them -(appearing fuller than it is). - -Compaction runs on **non-active zones only**, no concurrent allocate. -The zone-level lock (§8) ensures compaction, scanner, and health checks -do not conflict with each other on the same zone. - -### Preparatory thread - -A separate background thread pre-compacts the next batch of -`zone_rotate_count` zones in advance, so rotation is instant. The next -active set is already compacted and ready to publish via RCU. The -preparatory thread runs continuously: - -1. Identify the next `zone_rotate_count` zones in the rotation order - (starting from `pos_v_zone + zone_rotate_count`, wrapping around). -2. For each zone that is not already ready and not in the current active - set: acquire the zone-level lock, compact it (§5 Compaction - algorithm), mark it as ready. -3. Sleep briefly, then re-check (new zones may need compaction after - frees, or the rotation position may have advanced). - -When rotation is triggered, the ready zones are published immediately -as the new active set. Their ready flag is cleared. The preparatory -thread then starts building the next ready set. - -**Fallback when no ready zones exist:** if the preparatory thread has -not finished pre-compacting (e.g., on startup, or under heavy churn -where zones exhaust faster than compaction can keep up), rotation falls -back to **synchronous compaction**: compact the next batch inline, then -publish. This is slower but correct. The allocator waits for -compaction to finish before allocating from the new zones. - -### Periodic compaction fallback - -A background compaction task (`CompactionEngine`) polls at most every five -seconds for the configured `uncompacted_free_record_count` threshold. Independently, -the full sweep cadence (default 300s) admits every nonempty backlog, including -small zones that can never reach the threshold. Threshold polls do not reset -the full sweep deadline. Both paths skip active zones and use the zone-level -lock. A disk whose zones are all active must rotate a zone out before background -compaction can reclaim its free records. Managed files use -`zone_size_bytes = min(capacity_bytes, 32 GiB)`; capacities above 32 GiB contain -whole 32 GiB zones. Allocator and compaction policy do not change this geometry. - -### Compaction algorithm - -`compact_zone` (strategy 3) is the background maintenance task. It uses a -fixed contiguous-applied `compact_slot` boundary and allocation-incarnation -matching, then writes the new snapshot plus exact record deletions in one -atomic `batch_write`. - -1. **Bounded scan** free and busy records at one fixed - contiguous-applied cutoff. Retain each record's `commit_slot`; an - incomplete scan cannot advance the watermark. -2. **Match incarnations** by physical range. Reconcile every scanned free - fact whose record revision is at or below the fixed cutoff, including a - surviving fact whose revision is below the prior `compact_slot`. A free is - effective only when its `pre_allocation_ts`, unit count, and previous owner - equal the current busy value. Mismatches are stale or invalid facts and - never clear bits. -3. **Acquire the zone-level lock**, clear only ranges with a complete match, - and recompute `used_count = popcount`. Release the lock before KV I/O. -4. Build a new `ZoneValue` from the merged bitmap with `snapshot_slot` and - `compact_slot` set to the fixed scan cutoff and compute its checksum. -5. **One atomic `batch_write`**: Put the new `ZoneValue`, delete busy records - for matched frees, and delete every processed free fact at or below the - cutoff. This is a single - `batch_write` on the bound data group, atomic via crowdb-paxos-kv paxos. - The snapshot and the free-record deletion succeed or fail together; - there is no window where the snapshot is written but the free - records survive (the race that caused double-free in the two-op - design). -6. Decrement `uncompacted_free_record_count` by the number of free - records deleted (both stale and new). - -Busy records without a matching free incarnation remain untouched. - -**Why incarnation matching prevents stale free:** - -Suppose allocation 1001 is freed and later the same physical range is -allocated as incarnation 1002. A delayed retry can only recreate -`FreeBlockKey(..., 1001)`. Compaction compares it with the current busy value, -sees `1001 != 1002`, and leaves both the bitmap bit and busy record unchanged. -The atomic exact-key deletion normally removes every observed fact before -`compact_slot` advances. A surviving or delayed lower-revision fact is still -processed on a later complete scan; the watermark is never used as a lower -record filter. - -### Ready-zone tracking - -Each `DdbZone` carries: -- `compacted_ready: AtomicBool` — `false` when the zone needs - compaction before it can enter the active set (or it is currently - active, or has been modified since last compaction); `true` when the - zone has been compacted and its bitmap is accurate (eligible for - rotation into the active set). -- `compact_ts: AtomicU64` — the in-memory mirror of - `ZoneValue.compact_ts`. Used by compaction to partition free records - by watermark without a separate KV read. Updated when compaction - writes a new `ZoneValue`; loaded from the snapshot during recovery. - -The `compacted_ready` flag is set to `true` by the preparatory thread -after compaction completes. It is set to `false` when the zone is -published into the active set (it will need re-compaction after being -allocated from and freed). `rotate_active_zones` picks zones where -`compacted_ready == true` and not in the current active set. If fewer -than `zone_rotate_count` ready zones exist, it falls back to -synchronous compaction for the remainder. - -## 6. Crash Recovery - -### Three recovery strategies - -diskdb uses three complementary strategies for crash recovery and -maintenance. All three belong in the design, each in its role. - -**Strategy 1 — full scan rebuild (on-demand, via RPC/API).** - -Scan all live `BusyBlockKey`s for a zone and rebuild the bitmap from -scratch. No snapshot needed. For each offset: if a `BusyBlockKey` -exists, the bit is set (busy); otherwise the bit is clear (free). No -slot ordering needed. The busy record's existence is the indicator. -(`FreeBlockKey`s carry `previous_owner` audit info but are not needed -for state determination.) **Not in the common code flow**, provided -as an on-demand operation via the `RebuildZoneBitmap` RPC and API, used -for consistency checks (verify the in-memory bitmap matches the -records) or a full rebuild (e.g. after corruption, or when no -`ZoneValue` snapshot exists). Works with the existing `scan` API. Cost -= O(all live busy records per zone). Too slow for regular restart with -many zones, but correct and always available. - -**Strategy 2 — journal scan replay (fast restart, primary path).** - -Load the `ZoneValue` snapshot, then replay only the operations (Put / -Delete) written after `snapshot_slot`, in slot order, and apply them to -the snapshot bitmap. One scan per disk-group (or per disk) covers all -its zones, batch recovery, low overhead. **Requires a `JournalScan` -crowdb-paxos-kv RPC** (slot-range + key-prefix filter, returns ops in slot -order). This is the sole crowdb-paxos-kv extension diskdb needs. Fast because -compaction (strategy 3) keeps the uncompacted record set small. - -**Strategy 3 — compaction (ongoing maintenance).** - -Periodically (or when the free-record count for a zone exceeds a -threshold), match free facts to busy incarnations, merge valid frees into the -`ZoneValue` bitmap, and delete the matched busy plus processed free records in -one `batch_write`. The fixed-cutoff bounded scan is the only KV extension. - -### How the strategies work together - -- **Steady state**: allocate writes `BusyBlockValue` with a new - `allocation_ts`. Free blindly writes an incarnation-qualified free fact and - leaves the busy record intact. The bitmap is **not touched on free** — the bit stays set, - `used_count` is not decremented. Compaction (strategy 3) runs - periodically (and before rotation), merging free records into - `ZoneValue`, clearing the freed bits, recomputing `used_count`, and - deleting the free records. -- **Restart**: load `ZoneValue` snapshot → journal scan (strategy 2) - replays post-`snapshot_slot` operations in slot order → apply to - bitmap. Fast because compaction kept the record set small. -- **On-demand (RPC/API)**: full scan (strategy 1) rebuilds the bitmap - from all live records. Triggered by an operator or the scanner for a - consistency check or full rebuild — not in the common code flow. - -### Crash-safety invariants - -The in-memory bitmap is a **conservative over-estimate** of busy -blocks. It is never cleared on the free path. Free is an immutable blind put; -the bitmap bit stays -set until compaction clears it (§5). This means the bitmap may show a -block as busy when it is actually freed on disk. This is intentional -(data-safety principle: never show a block as free until compaction has -confirmed it from records). The durable state is the set of -`BusyBlockKey` / `FreeBlockKey` / `ZoneValue` records on the bound data -group. Current state is derived from the busy incarnation plus a matching free -fact. The invariants: - -- **Allocate ordering** — Phase 1 (bitmap CAS, set bit) happens before - Phase 2 (`BusyBlockValue` persist). If diskdb crashes between Phase 1 - and Phase 2, the bit is set in memory but no `BusyBlockKey` exists on - disk. On restart, strategy 1 full scan rebuilds the bitmap from - records — the bit is clear (no busy record), so the block is - correctly free. This is a **ghost-busy** (bit set in-memory, no - record) that is self-correcting on restart; the scanner also detects - this drift during live operation. -- **Free = persist only** — free puts one immutable `FreeBlockKey` containing - the segment's `allocation_ts`. It neither reads nor deletes the busy record - and never touches the bitmap. Retries are the same blind logical event. -- **Compaction reconciles the bitmap** — compaction (§5) is the sole - mechanism for clearing freed bits in the bitmap. It partitions free - records through a fixed `compact_slot` cutoff, matches each free fact to the - current busy incarnation, `range_clear`s only complete matches, - recomputes `used_count = popcount`, and writes the new `ZoneValue` - with the updated `compact_slot`, deletes matched busy records, and deletes - processed free facts in - **one atomic `batch_write`**. After compaction, the bitmap - accurately reflects the durable state. Compaction runs on non-active - zones (before they enter the active set via the preparatory thread, - or periodically as a fallback) — no concurrent allocate. -- **Compaction crash safety** — the snapshot write and free-record - deletion are one atomic `batch_write`; they succeed or fail together. - If diskdb crashes during compaction, either both the new `ZoneValue` - (with `compact_slot` advanced) and the record deletions are - durable, or neither is. There is no window where the snapshot is - written but its corresponding deletions do not. -- **Re-allocate after compaction** — once compaction has cleared the - bit and deleted the matching busy/free records, the block is available for - re-allocation. The re-allocate is a normal allocate (CAS-set bit + - persist `BusyBlockValue`) with a higher `allocation_ts`. A delayed old free - creates only its old incarnation key; later compaction rejects the mismatch. - -The bitmap is never persisted on the allocate/free hot path. Only the -`ZoneValue` snapshot (written by compaction) carries a serialized -bitmap, with `snapshot_slot` (replay start point), `compact_slot` -(compaction watermark), and `crc32` (integrity check over -`bitmap_codec` + compressed `usage_bitmap` + `compact_slot`). The baseline -`ZoneValue` is written during disk-add initialization (root §8). A disk without -snapshot, busy or free records uses three bounded prefix reads to prove it is -fresh, then publishes initial zone records in batches of at most 64 through -its ownership fence, fitting the tree's reserved gap-repair window. Its baseline -`snapshot_slot` captures the applied frontier -before the disk becomes allocatable. Existing records use ordinary recovery. -Empty zones keep only geometry and allocation counters; bitmap words are -materialized by atomic publication on first allocation. The standard empty -snapshot is copied directly from its compressed constant. Baseline write failure -keeps the disk unavailable; restart loads already published records and fills -missing baselines. all subsequent state changes are `BusyBlockValue` / -`FreeBlockValue` records until compaction merges them into a fresh -snapshot. - -**Hot-path error handling:** - -- **Allocate persist failure** (Phase 2 `batch_write` fails): rollback - the bitmap (clear the bits set in Phase 1 via - `zone.rollback_allocate`) and return error. No record was written, so - the record set is consistent (no ghost). `rollback_allocate` is the - bitmap CAS-clear, used only by allocate rollback, not by the free - path. -- **Free persist failure** (`batch_write` fails): the bitmap was - never touched. Return error; the block is still busy on both disk - and memory. The caller can retry safely. -- **Degraded mode** (group-0 / data-group unreachable): allocate/free - RPCs return `Unavailable` before touching the bitmap. No partial - state. - -### Recovery and compaction engines - -The recovery and compaction logic is implemented by two engines owned -by the diskdb server: - -- **`RecoveryEngine`** — owns a `DataGroupClient`. Runs during startup - and on ownership transfer. For each owned disk-group it builds an - empty `Node` and calls `RecoveryEngine::recover_node`, which - reconstructs each zone via `recover_zone`. -- **`CompactionEngine`** — owns a `DataGroupClient`. Runs as a - background `compaction_loop` over the owned nodes in - `NodeContainer`. Also drives the preparatory thread for - compaction-on-rotation. - -`recover_node` creates one `ZoneDisk` per disk, recovers each zone in -parallel (bounded by `recovery_concurrency`), adds the zones to the -disk, rebuilds the active zone set (picking ready / compacted zones), -and returns the reconstructed `Node`. - -`recover_zone` (strategy 2) is the primary restart path: - -1. Load the latest `ZoneValue` for the zone with a single point get. If - the CRC32 fails or no snapshot exists, proceed from an empty bitmap - or fall back to strategy 1. -2. Initialize the bitmap from the snapshot, or from empty. -3. Issue two narrow `JournalScan`s from `snapshot_slot + 1` to the - applied frontier: one over `BusyBlockKey` and one over `FreeBlockKey` - prefixes. Merge the two op lists by slot. `JournalScan` supports - slot-range, key-prefix, and transparent pagination; a - `KV_ERROR_JOURNAL_SCAN_GC_GAP` response falls back to strategy 1. -4. Apply only `Put BusyBlockKey` ops in slot order: each sets the bit - range using `BusyBlockValue.unit_count`. `Delete BusyBlockKey` ops - (frees) are **ignored**. The bitmap stays as a conservative over- - estimate (persist-only recovery, consistent with the free path - during normal operation). `Put`/`Delete FreeBlockKey` are no-ops for - the bitmap. The free records on disk are the source of truth for - what's freed; the common compaction flow (§5) will `range_clear` - their bits naturally. This eliminates the double-free risk that - would arise if recovery cleared bits for frees while the free - records still exist on disk. -5. Return the rebuilt `Zone`, resetting the rotating cursor. Mark the - zone as `compacted_ready = true` (it was just recovered from - records). `compact_ts` stays at `snapshot.compact_ts`, no - advancement needed. The free records on disk have `freed_ts > - snapshot.compact_ts` (they were written after the snapshot), so the - next compaction will classify them as "new" and `range_clear` their - bits, correct, since those blocks ARE free (no `BusyBlockKey` on - disk). Count the net free records (Put FreeBlockKey = +1, Delete - FreeBlockKey = -1 from re-allocate) to initialize - `uncompacted_free_record_count` so the compaction engine knows the - backlog. The per-disk-group monotonic timestamp source is - initialized to `max(now(), max(freed_ts of all scanned free - records) + 1)` after all zones in the disk-group are recovered - (§8 Monotonic timestamp source). - -`rebuild_zone_bitmap_full_scan` (strategy 1) is the on-demand fallback: - -1. Read all `BusyBlockKey`s for the zone via `read_zone_records` (key - order equals `unit_offset` order). `FreeBlockKey`s are ignored for - state determination. -2. Set the bit range for each busy record using `unit_count` and build - the `Zone`. -3. Optionally write a fresh `ZoneValue` snapshot so the next restart can - use strategy 2. -4. Return the rebuilt `Zone` plus derived stats. Mark the zone as - `compacted_ready = true`. - -`compact_zone` (strategy 3) is the background maintenance task. See -§5 Compaction algorithm. - -### Recovery triggers - -- **Startup**: after the blocking `sync_once` fetches owned - disk-groups, `RecoveryEngine::recover_node` runs for each one. The - rpc server does not accept RPCs until recovery completes. -- **Ownership transfer**: when `SyncLoop` detects a disk-group newly - assigned to this instance, it checks whether `ZoneValue` snapshots - already exist. If they do, the new owner runs - `RecoveryEngine::recover_node` and discards any stale in-memory state; - otherwise it initializes the disk-group via `disk_add_init`. - -## 7. Zone Allocation State (derived) - -`ZoneAllocationState` is a **derived** enum for reporting only: -`Active` (`used_count == 0`), `Available` (0 < `used_count` < -`unit_capacity`), `Full` (`used_count == unit_capacity`). There is no -CAS state machine. Allocation concurrency is handled by per-bit CAS on -the usage bitmap. A zone is allocatable when it is Healthy (inheriting -the disk's `HwStatus`) and `used_count < unit_capacity`. - -In the persist-only model, `used_count` is only decremented by -compaction (never by free). So after a free, `used_count` still reflects -the pre-free count. The zone may show `Full` even though it has freed -blocks on disk. This is correct: the bitmap is a conservative over- -estimate, and the derived state reflects the bitmap, not the durable -records. After compaction clears the freed bits and recomputes -`used_count`, the derived state accurately reflects the durable state. -A zone that was `Full` before compaction may become `Available` after -compaction, and then it is eligible for rotation back into the active -set. - -## 8. Concurrency Model - -All public and inter-module APIs are `async`. Runtime is `tokio` -(multi-threaded for production). - -### Allocate (lock-free) - -Per-bit CAS on the usage bitmap (`compare_exchange` on 64-bit words). -Multiple threads allocate from the same zone concurrently; no zone-level -lock held across `.await`. CAS losers re-scan the same word or try the -next zone; no thread blocking. Per-bit CAS is capped at `cas_retry_limit` -retries (default 100), then falls through to the next bit / word / zone. -Allocate runs only on active zones (in the `active_zone_context`). - -### Free (persist-only, no lock) - -Blindly put the incarnation-qualified `FreeBlockValue`; validation is deferred -to compaction, which matches it against the current busy incarnation. No -bitmap touch, no `used_count` decrement, and no zone-level lock. -Free can run on any zone (active or not); successful frees increment -`uncompacted_free_record_count`. The bitmap is reconciled later by compaction. - -### Zone-level lock for non-allocate operations - -Compaction, scanner, and health checks acquire a zone-level lock -(`RwLock<()>` on `DdbZone`) to coordinate with each other. These -operations run on **non-active zones only** (no concurrent allocate), so -the lock does not contend with the allocate hot path. The lock protects -only the in-memory bitmap mutation. It is **not** held across `.await` -on the KV client. The KV read is done before acquiring the lock, and the -KV write is done after releasing the lock. - -### Common methods on DdbZone - -Common methods on `DdbZone` encapsulate the lock + operation: - -- `compact_zone_inner()` — bounded-read busy/free records (KV read, no lock), - match allocation incarnations, acquire the zone lock, `range_clear` only - matched free records, recompute `used_count`, release the lock, then write - the new `ZoneValue` and exact deletions in one atomic `batch_write`. Used by `CompactionEngine` - (preparatory thread + periodic fallback). -- `scan_zone_inner()` — replay journal (KV read, no lock), acquire zone - lock, compare bitmap (in-memory), release lock. Used by the scanner. -- `health_check_zone_inner()` — verify zone records (KV read, no lock), - acquire zone lock, verify CRC + snapshot integrity (in-memory), - release lock. Used by the health probe. -- Each method acquires the zone lock only for the in-memory bitmap - mutation/verification, and releases it before any KV write. -- Allocate does **not** acquire this lock — it uses per-bit CAS - (lock-free). The lock only coordinates non-allocate operations on - non-active zones (no concurrent allocate). - -Disk-level active zone set uses RCU publish (`Arc` swap) for lock-free -reads; rotation takes a brief write lock. - -### Free-side lookup structures - -RCU-published alongside the allocate context on add/remove/status-change: - -- **disk-id → disk**: a hash map (O(1) average). Used by free to find - the in-memory disk from the `disk_id` in the `Segment`. -- **zone-index → zone**: a vec indexed by zone-index (O(1) direct - index). Used by free to find the in-memory zone from the - `zone_index` in the `Segment`. (Lookup is for - `uncompacted_free_record_count` increment; no bitmap mutation.) -- **KV free path:** the incarnation-qualified `FreeBlockKey` is constructed - directly from the `Segment`. `owner_chunk` and `allocation_ts` are carried - in the segment and become `previous_owner` and `pre_allocation_ts`. Free - validates the full-engine busy value and revision before the guarded batch; - compaction consumes only the resulting self-consistent fact. - -Disk membership changes acquire the disk-list write lock only while rebuilding -the immutable membership snapshot. Allocation and free load that snapshot and -do not acquire the disk-list lock. - -### Monotonic allocation incarnation source - -`BusyBlockValue.allocation_ts` uses a per-disk-group monotonic source. The -source is an `AtomicU64` initialized above every durable busy and free -incarnation discovered during recovery. It is an identity token and does not -order compaction. `FreeBlockValue.free_ts` is independently diagnostic. - -Ownership is currently immutable. Any future ownership transfer must -reconstruct or transfer the allocation high-water mark before enabling the new -owner. - -### Bounded tentative allocation cache - -Each disk group keeps recently allocated `TentativeBlock` values in a -lock-free ordered index keyed by `allocation_ts`. The default capacity is -262,144 unique incarnations. A per-entry pending/count/removed state makes the -atomic size budget exact when duplicate publication races removal. One atomic -trim owner removes the oldest entries until the cache is within capacity. - -Commit uses an exact matching tentative-cache entry without a read. A miss -after retry, restart, or eviction reads and validates the authoritative busy -incarnation. Tentative values are changed to Committed in one ordinary batch -write; an already Committed value is idempotent success. Commit and free remove -only exact incarnation cache entries. - -## 9. Background Scanner Coordination - -The background scanner (root §12) is a periodic consistency check that -detects live-state drift, catches record corruption early, and gives -operators visibility into cluster health during uptime. It coordinates -with compaction via the zone-level lock (§8): - -- **Zone skipping and locking** — the scanner skips zones in the disk's - `active_zone_context` (the allocator is actively handing blocks from - them — transient drift from the allocate Phase 1→2 window is - expected). For non-active zones, the scanner acquires the zone-level - lock (§8) to coordinate with compaction and health checks. Skipped - zones are checked on a later cycle when they rotate out of the active - set (and have been compacted). -- **Compaction coordination** — compaction and the scanner share the - zone-level lock (§8). They run on non-active zones only. The scanner - skips zones locked by compaction (or waits briefly, since compaction - is fast). Common methods on `DdbZone` (§8) encapsulate the lock + - operation, used by both compaction and the scanner. -- **Drift detection in the persist-only model** — the free path does - not touch the bitmap (§4 Free), so "bit set, no `BusyBlockKey`" is - **normal** for freed-but-not-compacted blocks (a `FreeBlockKey` - exists). The scanner distinguishes: - - **Real ghost-busy** (drift): bit set, no `BusyBlockKey`, no - `FreeBlockKey` — the block was never freed and never allocated - (crash between allocate Phase 1 and Phase 2, or a bug). Records - are authoritative → block is free → safe to clear the bit. - - **Normal uncompacted**: bit set, matching `BusyBlockKey` and `FreeBlockKey` - exist — the block was freed (persist-only) but compaction hasn't - cleared the bit yet. This is **not drift** — it's the expected - state. The scanner does not report it. (If the zone is not active - and has a high `uncompacted_free_record_count`, the scanner may - log a hint that compaction is lagging, but it's not a drift - finding.) - - **Ghost-free** (drift): bit clear, `BusyBlockKey` exists — should - not happen in the persist-only model (free never clears bits, and - only allocate/compaction touch the bitmap). If detected, it - indicates a bug or hardware error. Records are authoritative → - block is busy → set the bit back. Data may be written. - -## 10. Invariants - -- **I1 — Bitmap is a conservative over-estimate**: the in-memory - `usage_bits` never shows a freed block as free until compaction clears - the bit. Free is persist-only (no bitmap touch); compaction is the - sole bit-clearer. `used_count` is only decremented by compaction - (never by free). -- **I2 — Records are the source of truth**: the current busy record and an - incarnation-matching free fact determine whether a block is logically - busy. The bitmap is derived from records and may lag conservatively. -- **I3 — Compaction is the sole bit-clearer**: no code path except - compaction (and allocate rollback, which only clears Phase 1 claims - that were never persisted) clears bits in `usage_bits`. The free path - never touches the bitmap. -- **I4 — Compaction runs on non-active zones only**: no concurrent - allocate. The zone-level lock coordinates compaction, scanner, and - health checks on the same zone. -- **I5 — Compaction-before-rotation**: a zone must be compacted (freed - bits cleared, `used_count` recomputed, snapshot written) before it - enters the active set. The preparatory thread pre-compacts the next - batch; rotation publishes ready (pre-compacted) zones. -- **I6 — Atomic snapshot + delete**: compaction writes the new - `ZoneValue` with its advanced `compact_slot`, deletes matched busy records, - and deletes processed free facts in one atomic `batch_write`. -- **I7 — Incarnation match prevents stale free**: compaction clears a range - only when `pre_allocation_ts`, unit count, and owner match the current busy - value. An old retry cannot clear or delete a newer allocation. -- **I8 — `rollback_allocate` is allocate-only**: the bitmap CAS-clear - method is used only by the allocate Phase 2 failure path to undo - Phase 1's claim. It is never called by the free path. -- **I9 — Zone lock not held across `.await`**: the zone-level lock - protects only in-memory bitmap mutation. KV reads happen before - acquiring the lock; KV writes happen after releasing it. -- **I10 — Persist-only recovery**: recovery replays busy records - conservatively and leaves matching free facts for the common compaction - flow. It never makes a range reusable from an unmatched or incomplete - record view. - -## 11. Tunables and Defaults - -- `zone_rotate_count` — number of zones in the disk-level active zone - set (default 4). The disk round-robins over this many zones at a time; - when all are exhausted, the set rotates to a new batch of pre-compacted - zones. -- `cas_retry_limit` — per-bit CAS retry cap in the zone bitmap-scan - allocator (default 100). On exhaustion, the allocator falls through to - the next bit / word / zone. -- `zone_size_bytes` — default zone size in bytes (default 16 GB). -- `block_size_bytes` / `allocate_granularity` — block unit size (default - 1 MB; configurable 512 KB–2 MB; must be power of 2). v1 enforces - `allocate_granularity == block_size_bytes`. -- `compaction_cadence_secs` — periodic compaction interval (default - 300). The compaction loop sleeps this long between cycles. -- `snapshot_compaction_threshold` — compact a zone when its - `uncompacted_free_record_count` exceeds this (default 4096). Cadence - OR threshold — whichever fires first for a given zone. -- `validate_owner_on_free` — removed; compaction validates the free fact - (default false). When true, the free path reads the `BusyBlockValue` - from the data group first and validates `owner_chunk` (one extra paxos - round-trip, doubles free latency). -- `free_batch_enabled` — free batching toggle (default false). When - false, each request immediately issues one durable `batch_write`. When true, - the first queued request still drains immediately, while concurrent requests - accumulated during an in-flight write are coalesced by bind with no timer. -- `free_flush_max_batch` — maximum record operations in one coalesced KV - proposal (default 256). Requests are never split; one oversized request is - persisted alone. -- `recovery_concurrency` — max concurrent zone recoveries in - `recover_node` (default 16). diff --git a/doc/design/diskdb/design-crowdb-diskdb.md b/doc/design/diskdb/design-crowdb-diskdb.md deleted file mode 100644 index 266403d65..000000000 --- a/doc/design/diskdb/design-crowdb-diskdb.md +++ /dev/null @@ -1,1066 +0,0 @@ - - - -# CROWDB - Design: diskdb (Overview) - -This is the root design document for the diskdb component area. It -defines **what diskdb is**, **why key choices were made**, and **how the -component is structured**. Field-level details live in the schema files -and Rust source; this doc covers decisions and architecture only. - ---- - -## Table of Contents - -- [1. Overview](#1-overview) -- [2. Non-Goals (Design Envelope)](#2-non-goals-design-envelope) -- [3. Key Design Decisions](#3-key-design-decisions) - - [3.1 Group 0 is the centralized sysdata store](#31-group-0-is-the-centralized-sysdata-store) - - [3.2 disk-group → paxos group binding via a table (not hash)](#32-disk-group--paxos-group-binding-via-a-table-not-hash) - - [3.3 Incarnation-safe lifecycle transitions](#33-incarnation-safe-lifecycle-transitions) - - [3.4 Records are the source of truth; bitmap is derived](#34-records-are-the-source-of-truth-bitmap-is-derived) - - [3.5 Zone is a logical concept; sizes may vary](#35-zone-is-a-logical-concept-sizes-may-vary) - - [3.6 Common protocol crate; crowdb-rpc transport](#36-common-protocol-crate-crowdb-rpc-transport) - - [3.7 Reuse crowdb-common metrics](#37-reuse-crowdb-common-metrics) - - [3.8 Schema types used directly; no Rust type duplication](#38-schema-types-used-directly-no-rust-type-duplication) - - [3.9 Unit-based sizes; disk-id key routing](#39-unit-based-sizes-disk-id-key-routing) -- [4. Architecture Overview](#4-architecture-overview) -- [5. Group-0 Sysdata Schema](#5-group-0-sysdata-schema) -- [6. Hierarchy](#6-hierarchy) -- [7. Zone Management](#7-zone-management) — detailed design in [`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md) -- [8. Disk Status Management](#8-disk-status-management) -- [9. Space Metrics](#9-space-metrics) — detailed design in [`design-crowdb-diskdb-space-metrics.md`](design-crowdb-diskdb-space-metrics.md) -- [10. Background Scanner](#10-background-scanner) -- [11. Crate Layout](#11-crate-layout) -- [12. Concurrency Model](#12-concurrency-model) -- [13. Non-Gaps (Good Fits with CROWDB)](#13-non-gaps-good-fits-with-crow) -- [14. Configuration](#14-configuration) -- [15. Implementation Scope](#15-implementation-scope) -- [16. References](#16-references) -- [17. DiskdbClient Scanner / Rebuild Wrappers](#17-diskdbclient-scanner--rebuild-wrappers) -- [18. Group-0 Status Write-Back: Init → Offline](#18-group-0-status-write-back-init--offline) -- [19. Init-state zone load](#19-init-state-zone-load) - -## 1. Overview - -diskdb is a **distributed disk-block allocator** that runs on top of -CROWDB's KV cluster. It is a **lightweight, stateless server**: a diskdb -instance takes ownership of some disk-groups, manages the disks and -space inside them, and persists all durable state to CROWDB KV. It holds -no state that cannot be reconstructed from KV. On crash or restart it -rebuids in-memory structures from the KV records and group-0 metadata. - -Multiple diskdb instances run across a cluster, each managing a subset -of disk-groups. diskdb provides fast block allocation/deallocation on -physical disks using per-zone bitmap-scan allocators with per-bit CAS -and a rotating cursor for O(1) block tracking. The free path is -persist-only (the bitmap is a conservative over-estimate; compaction -reclaims freed space). All state changes are durably persisted to CROWDB -KV before being acknowledged to callers. - -diskdb allocates blocks and does not serve foreground block contents. Callers -write their own allocated blocks. For background relocation only, DiskDB -orchestrates an exact source-to-target copy through DiskIO before asking the -ChunkDB owner to publish the new layout. - -**Language:** Rust. **Runtime:** tokio (async everywhere). - -**Core goals:** -- **Fast allocation** — per-zone bitmap-scan allocators with per-bit - CAS (no zone-level lock); compaction reclaims freed space. No - KV-level CAS on the hot path. -- **Durability via CROWDB KV** — the paxos journal is the sole durable - store; diskdb has no local WAL and is stateless on disk. -- **Crash safety via record replay** — busy/free records in the paxos - slots are the source of truth; the bitmap is derived and rebuilt on - restart. -- **Accurate space metrics** — per-disk / per-disk-group / per-zone - capacity and busy/free, with a recalculation path to verify - correctness. - -**Design philosophy:** "diskdb is a thin, stateless client of crowdb-paxos-kv." -All consensus, replication, and durability are delegated to crowdb-paxos-kv. -diskdb's job is allocation policy, disk health, and space accounting — -nothing more. One crowdb-paxos-kv extension is required: a `JournalScan` RPC -(slot-range + key-prefix filter, returns ops in slot order) for fast -crash recovery (zone-management §6, strategy 2). This is the sole extension; all other -diskdb ↔ crowdb-paxos-kv interaction uses the existing scan / get / put / -batch_write API. - -## 2. Non-Goals (Design Envelope) - -- **No direct device I/O.** diskdb does not serve foreground block contents or - implement a device engine. Background relocation delegates copy and fsync to - DiskIO and persists only allocation and relocation metadata. -- **No local WAL.** CROWDB KV's WAL is the sole durability mechanism. -- **No consensus code.** diskdb is a client of crowdb-paxos-kv, with one - extension: a `JournalScan` RPC for fast crash recovery (zone-management §6). All - other interaction uses the existing crowdb-paxos-kv API. -- **No native SMR / zoned-namespace SSD support (v1).** Start with - conventional HDD/SSD (`ZoneBlockDisk`); the zone is a logical concept - that can later adopt to native zone APIs. -- **No leak-detection scanner (v1).** Leak detection needs caller - registries that do not exist yet; ship ghost/drift/integrity only. -- **No rack/data-center aggregated dashboards (v1).** Per-disk and - per-node metrics first; higher-level aggregation later. -- **No console UI (v1).** Minimal HTTP management API first; full - console integration (web + CLI) as a follow-up. - -## 3. Key Design Decisions - -### 3.1 Group 0 is the centralized sysdata store - -Group 0 (store 0, group 0) holds the basic metadata for the diskdb -subsystem: nodes, disks, disk-groups, and the two maps (disk-group → -diskdb instance, disk-group → paxos data group). diskdb fetches all -metadata from group 0 on startup and on sync. Any local status change -(disk found, disk bad, disk added/removed) is written to group 0 first, -then reflected locally. - -**Zones are NOT maintained in group 0** — a zone belongs to a disk, is -created when the disk is added, and is maintained as a separate zone -record on the disk-group's bound paxos data group. - -### 3.2 disk-group → paxos group binding via a table (not hash) - -A disk-group's zone records all live on **one paxos data group** so -multi-block allocation can use a single `batch_write` (atomic within a -group). The disk-group → paxos-group mapping is a **bind table stored in -group 0**, not a hash. A table (vs hash) enables dynamic scaling: new -paxos data groups can be added and disk-groups rebound without -rehashing the whole keyspace. - -**Allocation routing:** the `AllocateBlocks` request carries -`disk_group_id` (not `node_id`); CROWDB uses disk-groups instead of nodes -as the allocation routing unit. Allocate is scoped to one disk-group; -multi-block uses one `batch_write` on that group's bound paxos data -group; atomic within the group. No cross-group multi-block allocate in -v1. The caller issues separate `AllocateBlocks` calls per group if -needed. A joint placement caller may set `allow_disk_reuse` to execute -multiple distinct-disk passes in one atomic per-group request. The caller is -responsible for validating that roles requiring mutual anti-affinity fit in -the first pass. The caller, normally ChunkDB placement, picks the disk-group. - -Within the selected disk-group, allocation remains lock-free and applies the -configured load-aware policy before cursor fallback. `free_bytes` chooses the -eligible disk with the most absolute free space; `inverse_used_pct` chooses the -lowest normalized utilization. Equal scores retain rotating cursor order, -`exclude_disks` remains a hard constraint, and disabling load awareness restores -pure round-robin selection. This affects new allocations only; existing blocks -move through the fenced relocation path in §10. - -### 3.3 Incarnation-safe lifecycle transitions - -Each disk-group is owned by exactly one diskdb instance at a time (map in group -0). Allocation and free use batchable writes guarded by the ownership fence; allocation -incarnations and compaction-time validation make delayed frees safe. Commit -uses an exact matching tentative cache entry when present; otherwise it reads -the authoritative BusyBlock and validates its allocation incarnation. It -changes Tentative to Committed with a conditional batch that preserves both -the observed record revision and ownership generation. An already -Committed record is idempotent success. -In-memory concurrency within one instance is handled by **per-bit CAS** -on the usage bitmap (`compare_exchange` on 64-bit words), not a -zone-level lock. Multiple threads can allocate from the same zone -concurrently. The `ZoneAllocationState` enum is a **derived** field -for reporting (Active/Available/Full based on `used_count`), not a CAS -state machine. - -### 3.4 Records are the source of truth; bitmap is derived - -For each allocate or free, diskdb **cannot** update the full zone -bitmap in KV, as that would be a large write for the paxos group on every -block. Instead, each allocate writes a small **`BusyBlockValue`** carrying a -monotonic `allocation_ts` at the `BusyBlockKey`. Each free writes an immutable -**`FreeBlockValue`** at an incarnation-qualified `FreeBlockKey`; it does not -delete the busy record. The bitmap is **derived** from the records, never -written directly as a full bitmap on the hot path. The free path is -**persist-only**: the bitmap is not touched on free (the bit stays set, -`used_count` is not decremented); compaction is the sole mechanism for -clearing freed bits after matching the free fact to the current busy -incarnation. This makes the bitmap a conservative over-estimate -that never shows freed space as available until compaction reconciles -it from records. - -Record key layout, value schemas, the record model, current-state -determination, and scan-ordering notes are in -[`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md) §3. - -### 3.5 Zone is a logical concept; sizes may vary - -Zone is a **logical concept** — a rough mapping that can later adopt to -native zoned-namespace SSD or SMR HDD zone APIs. The last zone on a disk -may be smaller; the word-alignment rule (each zone's `unit_capacity` is -a multiple of 64) ensures no bitmap masking or padding. Details in -[`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md) §2. - -### 3.6 Common protocol crate; crowdb-rpc transport - -A new `lib/crowdb-protocol` crate holds flatbuffers definitions for all CROWDB -components. diskdb uses it first; crowdb-paxos-kv's existing schemas stay where -they are (unchanged). The flatbuffers messages are reused across the -crowdb-rpc transport. - -### 3.7 Reuse crowdb-common metrics - -diskdb reuses `crowdb-common`'s metrics module (no parallel metrics -system). Per-disk atomic counters stay as hot-path counters that flush -into the crowdb-common registry at reporting intervals. - -### 3.8 Schema types used directly; no Rust type duplication - -Schema types (`DiskId`, `ChunkId`, `Segment`, `HwStatus`, `DiskType`, -`ZoneAllocationState`, value types) are defined in `crowdb_protocol` -and used directly in Rust, with no field duplication. Extension traits add -domain methods. Internal types (metas, bitmap, CRC logic) are Rust -structs not exposed via rpc. - -**Keys are not flatbuffers.** KV keys use the cross-component binary -encoding defined in `doc/design/protocol/design-crowdb-protocol-key.md` (flat -per-kind Rust structs, two-byte `magic | type_tag` header, big-endian -fixed-width fields). Flatbuffers-serialized `*Key` messages are never -used as KV key bytes. The tag bytes break prefix scans. RPC -responses/requests use `**Info` schema messages that flatten key and -value fields into one message; see §5 and zone-management §6. - -### 3.9 Unit-based sizes; disk-id key routing - -All sizes are in units (block unit size defined per-disk, default 1M). -Record keys are disk-id-based (globally unique → reverse-lookup to the -data group). No `node_id`/`disk_group_id` in record keys or in -`Segment`. The `AllocateBlocks` request carries `disk_group_id` for -routing (§3.2); it is not stored in record keys or `Segment`. - -## 4. Architecture Overview - -``` - CROWDB Cluster - ┌────────────────────────────────────────────────────────────────┐ - │ diskdb-A diskdb-B diskdb-C │ - │ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ - │ │ disk-grp1│ │ disk-grp2│ │ disk-grp3│ │ - │ │ disk-grp4│ │ disk-grp5│ │ disk-grp6│ │ - │ └────┬─────┘ └────┬─────┘ └────┬─────┘ │ - └───────┼─────────────────────┼─────────────────────┼────────────┘ - │ crowdb-paxos-kv-client │ │ - v v v - ┌────────────────────────────────────────────────────────────────┐ - │ CROWDB KV (Multi-Paxos) │ - │ group 0 (sysdata) group-1..N (diskdb data groups) │ - │ ┌──────────────┐ ┌──────────────────────────────────┐ │ - │ │ nodes │ │ zone records (busy/free/snapshot)│ │ - │ │ disk-groups │ │ │ │ - │ │ ownership map│ │ │ │ - │ │ binding map │ │ │ │ - │ └──────────────┘ └──────────────────────────────────┘ │ - └────────────────────────────────────────────────────────────────┘ - ^ - │ HTTP mgmt API (axum) - ┌───────┴────────┐ - │ Console/CLI │ (follow-up: disk/disk-group management) - └────────────────┘ -``` - -- **diskdb instance** — one process per node (or per disk-group set). - Owns disk-groups assigned via group 0. Stateless on disk. -- **disk-group** — logical container of disks on one node; unit of - ownership and paxos-group binding. A disk-group belongs to exactly - one node; a node can have multiple disk-groups. -- **group 0** — centralized sysdata: node/disk/disk-group metadata + - ownership map + binding map. -- **data groups (group-1..N)** — paxos groups holding zone records and - snapshots. Each disk-group is bound to one data group. - -### Protocol - -`diskdb.fbs`, `chunkdb.fbs` and `diskio.fbs` define the storage services, -with shared identities and message types. Rust and C++ use generated definitions -from the same current schemas. - -The storage path uses three RPC services: -- **`DiskdbService`** (served by the diskdb server): `AllocateBlocks` - (carries `disk_group_id`, not `node_id`; carries `exclude_disks` for - anti-affinity), `FreeBlocks`, `QueryCapacityStats`, - `GetDiskGroupInfo`, `GetDiskInfo`, `RebuildZoneBitmap` (on-demand - full-scan rebuild, strategy 1, zone-management §6), `MarkBlockSuspect`, - `MarkBlocksCorrupt` (exact committed BusyBlock incarnation transition). A - caller uses this operation only after frame parse or write-frame checksum - proves corruption; the server checks owner, geometry, allocation timestamp, - and committed state before a revision-fenced `Corrupt` update. Ordinary I/O - failures do not change the block state. Allocation and ownership-qualified - free update the durable allocator records. -- **Hardware admin** (no rpc surface): rack/node/disk-group/disk - add/remove and `set_*_status` are writes to group-0 sysdata - performed through `HardwareClient` in `crowdb-paxos-kv-client`, invoked by - the console (`crowdb-web` / `crowdb-cli`). Hardware reads use - `HardwareClient` prefix scans; liveness uses `ServiceRegistryClient.heartbeat`. - The diskdb server reads hardware state - from group 0 via `HardwareClient` in its sync loop; it does not - serve hardware admin. See `doc/design/kv/design-crowdb-paxos-kv-group0.md` - §2.8. -- **`ChunkdbService`**: `AllocateChunk`, - `AppendChunk`, `QueryChunk`, `SealChunk`, `DeleteChunk`, - `DeleteChunkRange`, `ReplaceChunkStripRange`, `ListChunks`, exact-segment owner - queries, and relocation handoff. -- **`DiskioService`**: `DiskWrite`, `DiskRead`, and `DiskFsync`. - -Key protocol decisions: integer IDs throughout (no string UUIDs); -`DiskId` is globally unique (no `node_id`/`disk_group_id` in `Segment` -or record keys. `disk_group_id` is in the `AllocateBlocks` request for -routing only); `Segment.owner_chunk` (192-bit `ChunkId`) replaces the -former `tag`; all sizes are unit-based; errors are returned via rpc -status codes with `ErrorInfo` details (`error_code.fbs`), not -`bool ok + string error` in response bodies. - -## 5. Group-0 Sysdata Schema - -Group 0 stores the diskdb subsystem metadata as KV entries. The -authoritative schema is defined in -[`doc/design/kv/design-crowdb-paxos-kv-group0.md`](../kv/design-crowdb-paxos-kv-group0.md) -(text-path keys + JSON values, owned by `crowdb-paxos-kv-client`'s -`HardwareClient` / `ServiceRegistryClient` / `PaxosKvClusterMetaClient`). -This section summarizes the diskdb-relevant parts; see the group-0 -design doc for the full key/value layout and scan patterns. - -### Key layout (text-path encoding in group 0) - -Group-0 keys use the `TextKey` encoding (`/magic/type/…`), -JSON-encoded values. The hardware hierarchy embeds the full path in -each key for scan narrowing: - -``` -/hw/rack/ -> RackValue -/hw/node// -> NodeValue -/hw/dg/// -> DiskGroupValue -/hw/disk//// -> DiskValue -/hw/dg_owner/// -> OwnerMapValue -/hw/dg_bind/// -> BindMapValue -/srv/diskdb/ -> InstanceValue -/srv/kv-server/ -> InstanceValue -``` - -`rack_id`, `node_id`, `dg_id` (`DiskGroupId`), and `instance_id` are -`u64`; `disk_id_hex` is the 32-char hex of the 128-bit `DiskId` -(`high:u64 BE | low:u64 BE`, lowercase). v1 ships flat (no DC layer); -`dc_id` is dropped from `RackKey` and the Info types. The key concept -structs live in `lib/crowdb-protocol/src/key/` and implement both -`BinaryKey` (kept for data-group records) and `TextKey` (group 0). See -`doc/design/protocol/design-crowdb-protocol-key.md` §5 and -`doc/design/kv/design-crowdb-paxos-kv-group0.md` §3. - -diskdb's own data groups (zone records) keep the `BinaryKey` encoding: -`ZoneKey`, `BusyBlockKey`, `FreeBlockKey` (binary + flatbuffers, unchanged). - -Value types live in `crowdb-protocol` (flatbuffers `*Value` types used -directly; new sysdata values `OwnerMapValue`, `BindMapValue`, -`InstanceValue` added as schema messages; Entry return types -`DiskGroupEntry`/`DiskdbOwnerEntry`/`KVGroupBindEntry` are plain serde -structs). See `design-crowdb-paxos-kv-group0.md` §3.3. - -**Note**: zones are NOT stored in group 0. A zone is created when its -disk is added, and its state (records + snapshot) lives on the -disk-group's bound data group. - -### Map semantics - -- **Ownership map** (`/hw/dg_owner/...`) — the Group 0 leader's - `diskdb-ownership` monitor assigns bound disk-groups to live, fencing-capable - DiskDB instances. Missing or dead owners are replaced; count balancing moves - at most one healthy group per tick. Updates compare the owner record revision. - Before a live capable owner's one-hour placement lease expires, the monitor - renews it through the same revision-checked publication path without changing - the instance or process epoch. The DiskDB observes the new map generation and - reacquires its data-group fence. Fresh leases are not rewritten each tick; - renewals defer count balancing until the next snapshot. -- **Binding map** (`/hw/dg_bind/...`) — the console recommends the ordinary KV group with the fewest - bound DiskGroups cluster-wide and allows an explicit override at creation and publishes the binding, hardware record, - and node membership atomically. Missing selections, Group 0, and unknown - groups are rejected. Existing bindings are fixed; adding KV groups does not - migrate DiskGroups. The monitor skips unbound groups without repairing them. -- **Ownership fence** — before loading a group's allocation state, the target - claims `/diskdb/ownership-fence///` in its bound data group, - using the Group 0 owner revision as generation. Metadata requests carry the - expected owner identity; the KV leader validates its applied fence locally - and admits concurrent writes without changing that fence. Business-record - revision CAS remains independent. Only an ownership CAS closes admission, - drains previously admitted writes through apply, and persists the new fence. - Admission survives request cancellation and topology snapshot replacement; - unknown outcomes disable readiness until the leader recovery barrier resolves - prior slots. Background tasks retain their exact group's generation, so stale - work cannot write after a newer generation claims the fence. Ordinary writes - cannot mutate the fence namespace. All participating KV and DiskDB servers - must support owner fencing before automatic handover is enabled; older KV - servers reject the versioned owner request rather than silently bypassing it. -- **Service registry** (`/srv//`) — written by - each service instance on heartbeat (diskdb, kv-server); used by the - console and other components for discovery. - -## 6. Hierarchy - -CROWDB v1 has rack and node (no data-center layer). The physical -hierarchy and the logical disk-group layer: - -- **Physical**: rack → node → disk. These are real hardware - containers. v1 ships with rack and node; a data-center layer is not - in v1 (the schema drops `dc_id`). -- **Logical**: disk-group sits between node and disk. A disk-group - belongs to exactly one node; a node can have multiple disk-groups. A - disk-group is the unit of ownership (assigned to one diskdb instance) - and the unit of paxos-group binding (a disk-group's zone records all - live on one paxos data group). - -``` -Rack (rack_id, u64) → Node (node_id, u64) → Disk-Group (dg_id, u64) [logical] - → Disk (DiskId, 128-bit) → Zone (index, variable size) → Disk-Block (1 MB aligned) -``` - -## 7. Zone Management - -diskdb's second major component. The zone is the unit of allocation: -each disk is divided into zones, and each zone has a bitmap-scan -allocator with per-bit CAS (lock-free allocate). The free path is -**persist-only**: the bitmap is not touched on free; compaction is the -sole bit-clearer. Zone rotation (compaction-before-rotation with a -preparatory thread) ensures the allocator always sees accurate free -space in active zones. Crash recovery uses three strategies: full scan -rebuild, journal scan replay, and compaction. - -Detailed design (record model, allocation algorithm, free path, -compaction, rotation, crash recovery, zone-level concurrency, and -invariants) lives in -[`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md). - -## 8. Disk Status Management - -### Node / disk-group / disk HwStatus - -Shared enum `HwStatus`: `Init`, `Up`, `Maintenance`, `Suspect`, -`Missing`, `Bad`, `Offline` (ordered by severity). Effective status = -`max(node, group, disk)`, a three-level check (node + disk-group + -disk). The reference impl checks two levels (node + disk); CROWDB adds -the disk-group layer, which is new. Allocations require effective `Up`; -frees allow `Up`, `Maintenance`, or `Suspect`. - -Transitions: -- Init → {Up, Offline, Maintenance} on startup (load from group 0). -- Up → Suspect (3 missed syncs). -- Up → Offline / Maintenance (operator). -- Suspect → Up (sync recovers) or → Missing (cannot probe) or → Offline. -- Missing → Bad (confirmed) or → Up (rediscovered). **Missing is - detected by absence from a group-0 sync response** — a disk/node - absent from the sync response is transitioned to `Missing` (then to - `Bad` after confirmation or `Up` if rediscovered). -- Offline ↔ Maintenance (operator). -- Offline → Up (operator). - -An unknown or cancelled durable metadata write closes its `DiskWriteGate`. -The local disk then reports Offline even if Group 0 still records Up; this -is unresolved persistence, not evidence of exhausted capacity. `/ready` returns -503 with the affected DiskGroup, disk and `metadata-write-outcome-unknown` reason. -Health reads only observe this state. - -The process recovery watcher detects that uncertainty, drains the service and -stops. The monitor supervises the existing Start/Restart intent and launches a -new process. Its process session advances the owner generation and claims the -data-group fence before reconstructing allocation state from durable records. -Old process writes remain fenced and old disk snapshots retain their closed -gates. Recovery never clears an uncertain gate or treats uncertain occupied -space as free. Administrative Offline without an unknown write does not trigger -this process recovery; its explicit Online authorization remains required. - -Usage reports derive allocatable disk count and bytes from current disk gates, -independently of the cached allocation membership. Allocation failures with no -currently usable disks return Unavailable, including group/disk state, unknown -write state and capacity counters. NoSpace describes insufficient eligible -space or placement on usable disks. - -diskdb's first major component: -- **Sync with group 0** (fixed 10 s interval, same on success and - failure — no error back-off in v1): fetch latest metadata; update - group 0 first when local status changes (disk found, disk bad, disk - added/removed). A disk/node - **absent from a sync response** is transitioned to `Missing` (then to - `Bad` after confirmation or `Up` if rediscovered) — this is how - `Missing` is detected (§8). -- **Disk discovery + health probing**: discover local disks (config- - driven for v1; `/dev` scan later); probe health (existence, size, - basic read/write test). Source of truth for disk identity/capacity is - group 0; live health is probed locally. -- **Disk failure detection + bad-disk handling**: on disk failure, - transition to `Suspect`, update group 0. When a disk transitions to - `Bad` (via `Missing → Bad` after confirmation, §8), its busy blocks - are no longer readable. diskdb does **not** rebuild or relocate them - inline on the sync path — relocation/rebuild is a follow-up - requirement. The bad-disk handling on the sync path: - - Mark the `ZoneDisk` and all its `Zone`s as `Bad` - (`zone_state = Bad`; `allocatable()` returns `false`). No new - allocates touch the disk. Free is also blocked — - `allows_free(Bad)` is `false` (free allows `Up`/`Maintenance`/ - `Suspect` only, §8). - - Scan the zone records for the bad disk (`read_zone_records` per - zone, zone-management §6) and collect all live `BusyBlockValue`s — these are the - impacted blocks. Each carries `owner_chunk` (the chunk that owns - the allocation) so the caller / data-IO layer can be notified. - - Emit the `disk.bad.impacted_blocks` gauge (§9) and log the hand-off. The - dedicated bad-disk recovery flow still needs to turn this list into - ChunkDB repair work. The ordinary relocation journal is not authority to - reconstruct data that is already unreadable. - - The disk stays `Bad` — its records are read-only until an operator - removes the disk or marks it `Up` after repair (which triggers - strategy-1/2 recovery, zone-management §6). -- **Disk-add initialization flow**: the operator adds a disk in group 0 - (via `HardwareClient.add_disk` through the console), which writes - `DiskValue` to group 0. On the next sync tick, diskdb fetches the - updated `DiskValue` and sees a disk not yet in its in-memory state. - diskdb then initializes the disk: creates the in-memory `ZoneDisk` - with one `Zone` per zone (zone count = `capacity / zone_size`, last - zone sized per - zone-management §2's word-alignment rule), and writes baseline `ZoneValue` records - (empty bitmap, `snapshot_slot = 0`) to the bound data group at - `ZoneKey { disk_id, zone_index }` in one `batch_write`. These are the - replay baselines (zone-management §6); subsequent allocates write `BusyBlockValue` - records on top. Group 0 holds disk metadata only; zone records live - on the bound data group. - -**Follow-up — group-0 notify/watch:** the watch/notify mechanism is a -**client-pulled `WatchNotify` bidi stream**: diskdb opens the stream to -the group-0 leader and subscribes to the prefixes it cares about -(`/hw/dg_owner/`, `/hw/dg_bind/`, `/hw/disk/`); the leader pushes -hw-status-change and ownership-change notifications over that stream. -No separate notify-endpoint registration is needed for notify delivery. -The `rpc_endpoint` arg of `heartbeat_diskdb` is the diskdb rpc -service address for service-registry discovery (so clients can route -`allocate_blocks`), already populated from `config.server.listen_addr`. -This replaces polling for status changes as the primary -change-detection mechanism; polling stays as a safety net with an -increased interval. The detailed design (schema, registry, apply-path -trigger, client, diskdb handler, configuration) lives in -[`design-crowdb-paxos-kv-watch-notify.md`](../kv/design-crowdb-paxos-kv-watch-notify.md). - -## 9. Space Metrics - -diskdb's third major component. Detailed design (usage accessors, -`QueryCapacityStats` handler, per-disk counters, keepalive piggyback, -recalc verifier, reporting loop, schema, kv-client aggregation, and the -full `crowdb-diskdb-client` library) lives in -[`design-crowdb-diskdb-space-metrics.md`](design-crowdb-diskdb-space-metrics.md). -This section carries the architecture and the metric categories. - -Metrics must show **internal status** -(gauges reflecting current state) and a **latency hierarchy** (where -time is spent, broken down by layer) so operators can diagnose both -capacity problems and performance bottlenecks. - -**Space accounting:** -- **Per-disk** — capacity, busy, free, active zone count. -- **Per-disk-group** — aggregated from disks. -- **Per-zone** — dive into busy/free blocks (for the console zone - visualization). -- **Accurate + recalculation** — statistics are derived from the - in-memory bitmap (which is derived from the records). A recalculation - path replays the records to verify the bitmap matches the derived - statistics, detecting drift. - -**Disk-group-level usage summary on keepalive:** diskdb piggybacks a -per-disk-group usage summary (`capacity_bytes`, `used_bytes`, `free_bytes`, -allocatable capacity/used/free bytes, disk counts, and sample time) on the -keepalive message sent to group 0 on each sync tick. Group 0 maintains this at -the disk-group level (`DiskGroupUsageKey { disk_group_id }`). The -console reads this for cluster-wide overview; per-disk/per-zone -drill-down is via the `QueryCapacityStats` API. The summary is -derived (recomputed from the in-memory bitmap on each tick), not a -source of truth. - -**Metrics categories:** - -Metrics reuse `crowdb-common`'s metrics module. Per-disk hot-path -counters (atomics) flush into the crowdb-common registry at reporting -intervals. - -**1. Counters (events, monotonically increasing):** -- per-zone: `allocate.count`, `free.count`, - `allocate.retry.cms.bit.count` (contention signal — ties to §8 CAS - retry bound) -- per-disk: `allocate.count`, `free.count` -- per-disk-group: `allocate.count`, `free.count` -- per-instance: `sync.count`, `sync.error.count`, - `compaction.count`, `compaction.error.count`, and free-batch input request/ - record, output KV batch/record, oversize, and failure counters - -**2. Gauges (internal status, current state snapshot):** -- per-disk: `capacity_bytes`, `used_bytes`, `free_bytes`, `used_pct`, - `zone_count`, `active_zone_count` -- per-zone: `unit_capacity`, `used_count`, `free_count`, - `alloc_state` (derived: Active/Available/Full), `hw_status` - (inherited from disk) -- per-disk-group: `disk_count`, `allocatable_disk_count`, - `capacity_bytes`, `used_bytes`, `free_bytes` -- per-instance: `owned_disk_group_count`, `degraded` (0/1), - `free_batch.queue_depth` and `free_batch.coalescing_ratio_x1000`, - `uncompacted_free_record_count` (per zone — compaction backlog), - `last_sync_slot` (group-0 sync frontier), - `last_sync_age_secs` (time since last successful sync), active relocation - journal and planned-block counts - -Relocation counters record completed moves and errors. Capacity summaries and -the current instance-wide disk utilization spread/minimum/maximum gauges are -observations only; they influence ranking and admission but never authorize -publication or source release. - -**3. Latency hierarchy (where time is spent, per layer):** - -The allocate/free paths are two-phase (sync bitmap claim + async KV -persist). The latency hierarchy breaks down each phase so operators -can see whether the bottleneck is the in-memory allocator or the KV -persist round-trip. - -- **Allocate path:** - - `allocate.rpc.latency_us` — total RPC latency (handler entry → - response). Top of the hierarchy. - - `allocate.bitmap_scan.latency_us` — Phase 1 sync: time in the zone - bitmap-scan + per-bit CAS (includes CAS retries). Nanoseconds in - the common case; spikes indicate contention. - - `allocate.kv_persist.latency_us` — Phase 2 async: time awaiting - the `batch_write` of `BusyBlockValue` to the data group. Dominant - latency component (one paxos round-trip). - - `allocate.zone_rotate.latency_us` — time in `rotate_active_zones` - when the active set is exhausted. -- **Free path:** - - `free.rpc.latency_us` — total RPC latency. - - `free.bitmap_clear.latency_us` — time in the per-bit CAS clear. - - `free.kv_persist.latency_us` — time awaiting the `batch_write` of - `FreeBlockValue` (immediate in v1; batch flush when batching enabled). -- **Sync path:** - - `sync.latency_us` — total `sync_once` latency. - - `sync.read_group0.latency_us` — time reading from group 0. - - `sync.apply_changes.latency_us` — time applying changes to - in-memory state. -- **Compaction path:** - - `compaction.latency_us` — total compaction latency per zone. - - `compaction.scan_free.latency_us` — time scanning free records. - - `compaction.merge_bitmap.latency_us` — time merging free records - into the `ZoneValue` bitmap (in-memory). - - `compaction.kv_persist.latency_us` — time awaiting the - `batch_write` (new `ZoneValue` + delete free records). - -Latency metrics use `LatencyHistogram` (percentile precision) for hot -paths (allocate/free bitmap scan, KV persist) and `LatencySummary` -(count + sum + max + avg) for cold paths (sync, compaction, zone -rotate), matching crowdb-paxos-kv's convention. Gauges are derived snapshots -updated on the reporting interval, not hot-path writes. `degraded` and -`last_sync_age_secs` are the key health indicators for alerting. - -## 10. Background Scanner - -The scanner is a periodic consistency check. It detects and reports -live-state drift, catches record corruption early, and gives operators -visibility into cluster health during uptime. It is **not** a safety -mechanism (the free path is persist-only, zone-management §6, and the bitmap is a -conservative over-estimate; freed blocks stay busy until compaction); -it is an operational health mechanism for early corruption detection, -defense-in-depth against unknown bugs or hardware errors, and operator -visibility. - -### Integrity scanning - -- **Data-safety principle** — a busy block may have data written to - it. The scanner's first priority is to never free a block that might - have data. When drift is detected and the true state is uncertain - (corrupt records, conflicting signals), the scanner defaults to - **busy** (keep the bit set, keep the block allocated). Wasting space - is always preferable to freeing a block with data. The scanner only - clears a bit when records confidently say "free" (no `BusyBlockKey`, - no `FreeBlockKey`, records intact and readable). -- **Drift detection in the persist-only model** — the free path does - not touch the bitmap (zone-management §6), so a bit set with matching busy - and free records is **normal** for freed-but-not-compacted blocks. The scanner - distinguishes: - - **Real ghost-busy** (drift): bit set, no `BusyBlockKey`, no - `FreeBlockKey` — the block was never freed and never allocated - (crash between allocate Phase 1 and Phase 2, or a bug). Records - are authoritative → block is free → safe to clear the bit. - - **Normal uncompacted**: bit set, matching `BusyBlockKey` and - `FreeBlockKey` exist — the block was freed (persist-only) but compaction hasn't - cleared the bit yet. This is **not drift** — it's the expected - state. The scanner counts it as `uncompacted_lag` (not drift) so - operators can see compaction lag, but does not auto-correct it. - - **Ghost-free** (drift): bit clear, `BusyBlockKey` exists — should - not happen in the persist-only model (free never clears bits, and - only allocate/compaction touch the bitmap). If detected, it - indicates a bug or hardware error. Records are authoritative → - block is busy → set the bit back. Data may be written. -- **Ghost scan implementation** (`scanner/ghost.rs`) — `scan_ghosts` - iterates owned disk-groups → disks → zones. For each non-active, - unlocked zone, it replays the journal into a throwaway `DdbZone` - via `recover_zone_inner` (strategy 2) with - `rebuild_zone_bitmap_full_scan` (strategy 1) fallback — same logic - as `RecalcEngine::recalc_zone`. It then does a bit-by-bit diff - (`diff_bitmaps`) between the live and replayed bitmaps, classifying - each differing bit by checking `ZoneRecords` busy/free offset sets - (built into `HashSet` for O(1) lookup). The zone lock is not - held across awaits (I9); it is re-acquired (try_write) only for the - synchronous classify + auto-correct step. -- **Re-verify step** — if real drift is detected and - `reverify_delay_ms > 0`, the scanner sleeps, re-snapshots the live - bitmap, and re-checks each drifted bit. If the drift disappeared - during the delay (a zone rotated out with an in-flight allocate - whose Phase 2 completed), it was transient and is skipped. If - persistent, it is reported. Setting `reverify_delay_ms = 0` - disables re-verify (report immediately, may include false - positives from in-flight operations). -- **Auto-correct** — when `ghost.auto_correct` is enabled and - re-verify confirms persistent drift, the live bitmap is corrected: - real ghost-busy → `cas_bit(off, false)` + `used_count` decrement; - ghost-free → `cas_bit(off, true)` + `used_count` increment. Normal - uncompacted bits are never auto-corrected (compaction's job). - Fallback (CRC fail / GC gap) suppresses auto-correct (corruption - signal — the scanner reports but does not correct when the - record-derived bitmap itself may be unreliable). -- **Record integrity** (`scanner/integrity.rs`) — `scan_integrity` - re-checks CRC on the `ZoneValue` snapshot via - `ZoneValueExt::verify_checksum`, detects records that - `read_zone_records` silently skips (undecodable keys/values) by - doing its own prefix scans with `BusyBlockKey::from_bytes` + - `bincode::deserialize` on each item, and optionally validates - `owner_chunk` well-formedness (non-zero) on each `BusyBlockValue`. - Corrupt records are reported; the block is kept busy (key exists, - data may be written); no auto-correction frees a block with a - corrupt record. -- **Leak detection** (`scanner/leak.rs`) — deferred (needs caller - registries). The scaffold returns `LeakScanResult { status: - "deferred" }` so the scanner loop calls it unconditionally. -- **Scanner task** (`scanner/task.rs`) — `ScannerTask` implements - `BackgroundTask` and runs on `BgRunner` with a `TimerFn` trigger - reading `scanner.scan_interval_secs` from the shared config handle. - `ScanState` holds the last `ScanSummary` (shared between the task - and the rpc service handlers) + an `AtomicBool` scan-requested - flag (set by `TriggerScan`, consumed at the start of the next - `run_cycle`) + an `AtomicBool` in-progress flag (prevents overlap). -- **Admin RPCs** — `TriggerScan` sets the scan-requested flag and - returns the current summary + `scan_in_progress` flag; - `GetScanStatus` returns the last summary + `has_run` flag. The - `ScanSummary` schema message carries all scan counts (ghost_busy, - ghost_free, uncompacted_lag, corrupt_snapshots, corrupt_records, - owner_mismatches, leak_status) + timing. -- **Scanner metrics** — `scanner_runs_total` (counter), - `scanner_duration_ms` (latency summary), `scanner_ghosts_found` - (gauge), `scanner_drift_found` (gauge = ghost_busy + ghost_free), - `scanner_corrupt_records` (gauge = corrupt_snapshots + - corrupt_records). -- **Zone coordination** — the scanner skips active zones, acquires the - zone-level lock for non-active zones, and coordinates with compaction - via the shared lock. Details in - [`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md) §9. - -### Tentative owner reconciliation and relocation - -Tentative allocation cleanup is independent from bitmap drift scanning. -`BusyBlockOwnerScanner` reads only tentative durable BusyBlock records after -the disk-group reaches lifecycle `Up`. It queries ChunkDB with the owner chunk -and exact disk/zone/offset/allocation incarnation. `Referenced` confirms the -block, `TaskPending` retains it, and `Absent` persists a first-observed grace -record. Only a repeated authoritative `Absent` after the 24-hour default grace -writes the normal free record. Transport failures retain the block. - -The scanner visits disk-groups, disks, and zones in sorted serial order and -awaits an async configurable delay after every zone. The default is three -seconds. This pacing bounds KV and owner-RPC pressure without blocking a tokio -executor thread. - -Active movement uses a durable relocation journal keyed by the exact source -incarnation. The worker reserves a tentative target, asks DiskIO to copy and -fsync it, then delivers the exact source/target pair to ChunkDB. Journal phases -are `Reserved`, `Copied`, `Accepted`, `Published`, `TargetConfirmed`, -`SourceFreed`, and `Discarded`; startup resumes every non-terminal phase. -ChunkDB alone conditionally publishes the new layout. DiskDB confirms the -target and writes the source free record only after `Published`. - -The intra-disk-group planner selects one committed source from an over-used -disk and one target on an under-used disk when their utilization-point spread -meets the configured threshold. It resumes existing journals before selecting -new work, limits admissions per cycle, and uses serial per-zone pacing. For -cross-group moves, ChunkDB applies its own skew hysteresis and protection gate, -allocates the target, and sends `ExecuteRelocation` to its owning DiskDB, which -adopts that exact target into the same journal state machine. Source -finalization is local when the source disk belongs to another owned group and -otherwise routes through `DiskdbClient`. - -The relocation journal, not an in-memory job set, is the recovery authority. -Operation identity includes the source allocation incarnation, so offset reuse -cannot alias an earlier move. `Stale` discards only the tentative target; -`Rejected` and transient failures retain both sides for reconciliation. No age, -usage observation, or planner timeout is permission to free either block. - -## 11. Crate Layout - -``` -lib/crowdb-protocol (common flatbuffers for all CROWDB components; diskdb first) -app/crowdb-diskdb (server: lib + binary — types, allocator, records, sync, rpc + HTTP, CLI) -lib/crowdb-diskdb-client (client library for callers: allocate/free/query; crowdb-rpc transport) -``` - -- **`lib/crowdb-protocol`** — flatbuffers definitions for diskdb rpc services - (allocate/free/query) + extension traits for schema types - (`diskdb_type_util.rs`: `DiskIdExt`, `HwStatusExt`, - `ZoneAllocationStateExt`, `ZoneValueExt`, `effective_status`). - crowdb-paxos-kv's existing schemas stay in `crowdb-paxos-kv` (unchanged). The - flatbuffers messages are reused across the crowdb-rpc transport. -- **`app/crowdb-diskdb`** — server crate (package name `crowdb-diskdb`). - Combined library + binary: contains all diskdb logic (types, zone - allocator, record persistence, scanner, ownership/sync, rpc + - HTTP handlers, CLI, config). The library target enables integration - tests without spawning a separate process; the binary target - (`crowdb-diskdb`) is the server executable. Proto types and their - extension traits are re-exported from `crowdb_protocol`; internal types - (metas, key layout, `AllocatedRange`, `ActiveZoneContext`, bitmap) - are Rust structs. - - Liveness and RPC use pure index modules with private implementation - children: - - ```text - liveness/keepalive.rs - └── keepalive/ - ├── scheduler.rs KeepAlive facade, tick, and BackgroundTask - ├── heartbeat.rs service-registry heartbeat publication - ├── observation.rs group-0 ownership, bind, and disk observation - ├── reconciliation.rs runtime ownership and disk reconciliation - └── loading.rs generation-fenced asynchronous zone loading - - service/diskdb_rpc_service.rs - └── diskdb_rpc_service/ - ├── service.rs DiskdbRpcService state and handler registration - ├── mutations.rs allocate, commit, and free handlers - ├── queries.rs capacity and inventory query handlers - ├── admin.rs recovery, compaction, and scanner handlers - └── wire.rs FlatBuffer parsing and response construction - ``` - - `KeepAlive::tick` is the liveness orchestrator and preserves the order - heartbeat → observation → reconciliation. `DiskdbRpcService` remains the - public registration facade; all mutations pass through `mutation_gate`. -- **`lib/crowdb-diskdb-client`** — client library for easy access to the server - (allocate/free/query), mirroring `crowdb-paxos-kv-client`'s retry + - topology-cache pattern. crowdb-rpc transport; flatbuffers framing. - -### Dependencies - -``` -crowdb-common (metrics, logging, time) - ↑ -crowdb-paxos-kv-client (sole durable store — group 0 + data groups) - ↑ -crowdb-diskdb (server) ──depends──> crowdb-paxos-kv-client, crowdb-common, protocol -crowdb-diskdb-client ──depends──> protocol, crowdb-paxos-kv-client (for group-0 discovery) - -protocol ──> (no internal deps beyond flatbuffers/crowdb-rpc) -``` - -## 12. Concurrency Model - -All public and inter-module APIs are `async`. Runtime is `tokio` -(multi-threaded for production). - -- No blocking calls in business-logic paths. -- **Allocate** — lock-free per-bit CAS on the usage bitmap; no - zone-level lock. Allocate runs only on active zones. Details in - [`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md) §4, §8. -- **Free** — persist-only: one `batch_write`, no bitmap touch, no - zone-level lock. Free can run on any zone without coordination. - Details in [`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md) §4, §8. -- **Zone-level lock for non-allocate operations** — compaction, - scanner, and health checks acquire a zone-level lock on non-active - zones only. Common methods on `DdbZone` encapsulate the lock + - operation. Details in - [`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md) §8. -- Disk-level active zone set uses RCU publish (`Arc` swap) for - lock-free reads; rotation takes a brief write lock. -- Disk-group allocation context retains its read lock. Allocation and usage - clone one `Arc` while holding it, and configuration replacement is rare; - `ArcSwap` publication is reserved for evidence of contention here. -- **Free-side lookup structures** — disk-id → disk hash map and - zone-index → zone vec for O(1) lookups on the free path. Details in - [`design-crowdb-diskdb-zone-management.md`](design-crowdb-diskdb-zone-management.md) §8. -- v1 free is immediate (no `FreeBatch`, no background flush loop). When - free batching ships, `FreeBatch` will be protected by a `Mutex` held - only for the append, not for the KV flush; the flush is triggered by - batch size (no timer). -- Node-level `add_disk` / `remove_disk` acquire a write lock on the - disk list; allocation/free acquire a read lock (concurrent with each - other, exclusive with add/remove). - -## 13. Non-Gaps (Good Fits with CROWDB) - -These design assumptions map cleanly onto CROWDB and need no design -work, just implementation: - -- **Durability model**: crowdb-paxos-kv's WAL is the sole durable log. DiskDB's blind - allocation/free writes and conditional commit batches become durable through - the same Paxos/WAL path. -- **Consensus semantics**: Multi-Paxos retains parallel blind allocation and - free throughput. Conditional commit serializes only on its one busy key - before proposing an ordinary batch. -- **Async runtime**: tokio multi-threaded; diskdb's two-phase - async allocation (sync bitmap-scan claim + async KV persist) maps - directly. - -## 14. Configuration - -diskdb requires a typed TOML file at startup. Omitted fields use typed -defaults, and explicitly supplied CLI fields override file values according to -the shared [`service configuration design`](../config/design-crowdb-config.md). -`server.rpc_workers` controls inbound RPC concurrency, defaults to 2, must be -positive, and requires restart when changed. - -All settings that control flow behavior live in a config class (no -hardcoded tunables in business logic). Defaults: - -- **Sync** — sync interval (10 s, fixed — same on success and failure), - degraded miss threshold (3), temp-failure timeout (900 s) -- **Allocator** — `zone_rotate_count`, CAS retry limit (100) -- **Allocation policy** — `allocator.load_aware` (true) and - `allocator.load_aware_weight` (`free_bytes`) -- **Free** — `free_batch_enabled` (default false — one durable KV batch per - request; immediate concurrent-request coalescing when true), - `free_flush_max_batch` (256, maximum records in one coalesced KV proposal; - one oversized request remains atomic) -- **Compaction** — snapshot compaction threshold (record count or - time), compaction cadence (periodic interval for strategy 3) -- **Disk** — block / unit size (default 1 MB), zone size -- **Free validation** — no read on the free path; compaction validates the - immutable free fact against the current busy incarnation -- **Scanner** — `scan_interval_secs` (600), `ghost.detect` (true), - `ghost.auto_correct` (false — manual review first; enable for - self-healing), `integrity.verify` (true), - `integrity.detect_owner_mismatch` (false — piggybacks on - `read_zone_records`), `reverify_delay_ms` (1000 — set 0 to disable - re-verify), tentative-owner interval (600 s), per-zone delay (3 s), and - absent-owner grace (86,400 s) -- **Rebalance** — enabled, planning interval (300 s), utilization-point - threshold (20), one new job per cycle, and a three-second per-zone delay - -## 15. Implementation Scope - -The diskdb implementation is organized by functional scope. Each area -below covers a coherent slice of the component; together they make up -the full diskdb server. - -- **Protocol + core types** — flatbuffers services, core types, record key - layout, config validation, bitmap utilities, CRC integrity. -- **Group-0 sysdata schema + sync** — disk status management, group-0 - read/write, ownership/binding maps, heartbeat, disk-add initialization - flow, keepalive usage summary. -- **Zone allocator + record persistence** — bitmap-scan allocator, - rotating active-zone-set, busy/free records, two-phase async - allocation, persist-only free (no `FreeBatch`, no timer), - `disk_group_id` routing, load-aware disk selection, `exclude_disks`, CAS - retry bound. -- **Crash recovery + snapshot compaction** — three strategies: full - scan rebuild, journal scan replay via `JournalScan` RPC, compaction - that deletes only free records. -- **Space metrics + query API** — per-disk/group/zone metrics, - recalculation path, `query_disk_usage`, three-category metrics: - counters, gauges, latency hierarchy. -- **Background reconciliation and relocation** — ghost/drift/integrity - detection, tentative-owner reconciliation, paced intra-disk-group planning, - durable DiskIO relocation, and restart-safe ChunkDB publication handoff. -- **Disk discovery + health probing** — config-driven disk list, health - probe, disk failure detection. -- **Console + CLI integration** — disk/disk-group management UI, zone - busy/free visualization, CLI command design. -- **Group-0 notify/watch** — replace polling with watch/notify; see - [`design-crowdb-paxos-kv-watch-notify.md`](../kv/design-crowdb-paxos-kv-watch-notify.md). - Polling stays as safety net. -- **Free batch** — size-threshold batching, no timer; graceful-shutdown - drain + flush. - -The design doc (this file and future sub-designs under -`doc/design/diskdb/`) is kept permanently — it is the root design for -the diskdb component area. - -## 16. References - -- CROWDB root KV design: `doc/design/kv/design-crowdb-paxos-kv.md` -- CROWDB WAL design: `doc/design/kv/design-crowdb-paxos-kv-wal.md` -- CROWDB metrics design: `doc/design/kv/design-crowdb-paxos-kv-observability.md` -- CROWDB console design: `doc/design/console/design-crowdb-console.md` - ---- - -## 17. DiskdbClient Scanner / Rebuild Wrappers - -`DiskdbClient` wraps allocate/free/query/recalc/compact and also the -scanner RPCs (`TriggerScan`, `GetScanStatus`) and -`RebuildZoneBitmap`. The schema + server handlers exist; the client -wrappers make them reachable from the REST proxy and CLI. - -```rust -use crowdb_protocol::diskdb::rpc::{ - TriggerScanRequest, TriggerScanResponse, - GetScanStatusRequest, GetScanStatusResponse, - RebuildZoneBitmapRequest, RebuildZoneBitmapResponse, -}; - -impl DiskdbClient { - /// Trigger a scan on all owned groups (or one group if `dg_id` - /// is set). Returns the last `ScanSummary` + `scan_in_progress`. - pub async fn trigger_scan(&self, dg_id: Option) - -> Result - - /// Get the last scan summary + `has_run` flag. - pub async fn get_scan_status(&self, dg_id: Option) - -> Result - - /// Rebuild one zone's bitmap on a disk. Routes via `dg_for_disk`. - pub async fn rebuild_zone_bitmap( - &self, disk_id: DiskId, zone_index: u32, - ) -> Result -} -``` - -- `trigger_scan` / `get_scan_status` route to `first_cached_dg()` - when `dg_id` is `None` (mirrors `recalc_disk_usage`); otherwise - route to the specified dg. Both go through `with_retry`. -- `rebuild_zone_bitmap` routes via `dg_for_disk(disk_id)` (mirrors - `compact_zone`), then `with_retry`. -- All three are admin/debug calls; transient `Unavailable` is retried - per `RetryConfig`; `NotFound` (unknown disk/zone) is returned - immediately. - -Edge cases: -- `trigger_scan` while a scan is running → server returns - `scan_in_progress: true`; client returns the response as-is (no - error, no stacking). -- `rebuild_zone_bitmap` on unknown disk → `dg_for_disk` returns - `Unreachable`; surfaced to the caller. -- Empty endpoint cache → `first_cached_dg()` returns `Unreachable`; - caller sees "no cached endpoints; call refresh_endpoints". - -## 18. Group-0 Status Write-Back: Init → Offline - -`background_zone_load` publishes `Offline` before its local `Init → Offline` -transition when zone loading or baseline persistence fails. This keeps a -failed disk unavailable across sync ticks. - -- The task retains its exact `OwnershipFence`. Group-0 write-back requires - the same owner instance and owner-record revision. -- A conditional batch preserves the owner value and writes the disk status - atomically. Advancing the owner revision makes the next observation reclaim - the data-group fence. A concurrent handover rejects the whole batch. -- A stale loader cannot mark its successor's healthy disk Offline. Rejected - write-back is reported; its retired in-memory disk cannot allocate. - -## 19. Init-state zone load - -New disks start in `Init` state with no zones. A background task -loads zones from the bind before transitioning `Init → Up` (or -`Init → Offline` on load failure or operator-set offline). This -prevents allocations against a disk with existing KV records from -overwriting used blocks with empty zones. `reconcile_absent_disk` -removes `Offline`/`Maintenance`/`Init` disks from memory when they -disappear from group-0 sync (the disk is gone); `Bad` disks are kept -(recovery scan running). - -- **I1 — Init before allocatable**: A disk is never allocatable - while in `Init` state. Zones must be loaded from the bind before - the disk transitions to `Up`. diff --git a/doc/design/diskdb/diskdb-allocate-flow-analysis.md b/doc/design/diskdb/diskdb-allocate-flow-analysis.md deleted file mode 100644 index 9607537fb..000000000 --- a/doc/design/diskdb/diskdb-allocate-flow-analysis.md +++ /dev/null @@ -1,203 +0,0 @@ - - - -# CROWDB - Design: DiskDB Allocation Flow Analysis - -Allocation flow, observability, tuning controls, and reference performance for -the durable DiskDB path. - -Depends on: [DiskDB overview](design-crowdb-diskdb.md), -[zone management](design-crowdb-diskdb-zone-management.md), -[RPC design](../rpc/design-crowdb-rpc.md), and -[KV write flow](../kv/kv-write-flow-analysis.md). - -## Table of Contents - -- [1. Flow](#1-flow) -- [2. Metrics](#2-metrics) -- [3. Tunables](#3-tunables) -- [4. Reference Results](#4-reference-results) -- [5. Bottleneck](#5-bottleneck) -- [6. Retained Logs](#6-retained-logs) - -## 1. Flow - -```text -crowdb-cli workload task - -> DiskdbRpcTransport connection pool - -> crowdb-rpc DiskDB server worker - -> Tokio allocation task - -> lock-free bitmap claim - -> busy-record construction - -> DdbKvClient::persist_busy_batch - -> PaxosKvRpcTransport connection pool - -> KV batch_write - -> proposal coalescing and inflight admission - -> Paxos accept quorum - -> WAL and engine apply - -> DiskDB response construction and crowdb-rpc submission - -> client completion -``` - -One allocation RPC produces one KV `batch_write`. Multiple blocks in an -allocation RPC become multiple busy records in that batch. Independent -allocation RPCs can be combined by KV proposal coalescing; DiskDB does not add -a second flow-level batcher. - -## 2. Metrics - -Every DiskDB request method has this family: - -```text -request.{method}.lh -request.{method}.inflight.g -request.{method}.errors.c -``` - -The histogram count and rate identify the completed workload mix. The gauge -identifies the current live mix, including slow requests. The error counter is -the non-success response population. Timing begins at Rust handler dispatch -and ends at response submission, including async task scheduling. - -Allocation adds bitmap scan, record build, KV persist, response build, -claimed-unit, rollback-unit, partial-batch, and error-reason metrics. The -DiskDB-to-KV boundary reports batch-write latency, record magnitude, inflight, -and errors. Each process's `cpp-rpc` block supplies transport scheduling, -writev/read/parse, bandwidth, queue pressure, connection, and transport-error -metrics without duplicate Rust counters. - -KV-server logs supply `kv.batch_write.lh`, proposal inflight wait, Paxos phase -latencies, WAL append/fsync, engine apply/flush, inter-replica RPC, CPU, RSS, -and network counters. - -## 3. Tunables - -The three connection and worker layers remain independent: - -- benchmark client: workload concurrency, connections per DiskDB endpoint, - and client RPC workers; -- DiskDB: server RPC workers, KV connections per endpoint, and KV-client RPC - workers; -- KV: server RPC workers, peer pool, proposal inflight window, coalesce size, - drain threshold, event-write mode, WAL, and storage backend. - -The high-throughput memory profile starts from the KV write sentinel. The -current matrix enables event-write and uses inflight 32 and coalesce 32 without -an explicit drain-threshold override. The 1/16-thread cases use two -connections and RPC workers on every layer; 128/256-thread cases use four. -Connection count is recorded explicitly because a per-process, per-endpoint -pool is not comparable to the direct KV benchmark's total count. - -## 4. Reference Results - -AMD Ryzen 9 5950X, 16c/32t, Linux 6.8, x86_64, memory KV/WAL, one block per -request, and 4 TiB per logical disk. Every case ran for 20 seconds, stopped at -its deadline with zero errors, and reported exact busy-space accounting. - -| Workload | Grp | Thread | Block | Client connection | DiskDB connection | KV internal connection | Epoll worker | Window | Coalesce | ops/s | p50 us | p99 us | Duration | Errors | Space | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| allocate | 3 | 1 | 1 | 2 | 2 | 2 | 2 | 32 | 32 | 2,481 | 410 | 504 | 20 s | 0 | exact | -| allocate | 3 | 16 | 1 | 2 | 2 | 2 | 2 | 32 | 32 | 42,959 | 362 | 582 | 20 s | 0 | exact | -| allocate | 3 | 128 | 1 | 4 | 4 | 4 | 4 | 32 | 32 | 130,863 | 930 | 1,863 | 20 s | 0 | exact | -| allocate | 3 | 256 | 1 | 4 | 4 | 4 | 4 | 32 | 32 | 159,418 | 1,514 | 3,294 | 20 s | 0 | exact | -| allocate | 1 | 256 | 1 | 4 | 4 | 4 | 4 | 32 | 32 | 171,206 | 1,423 | 2,883 | 20 s | 0 | exact | -| mix | 3 | 1 | 1 | 2 | 2 | 2 | 2 | 32 | 32 | 2,494 | 405 | 500 | 20 s | 0 | exact | -| mix | 3 | 16 | 1 | 2 | 2 | 2 | 2 | 32 | 32 | 43,106 | 361 | 568 | 20 s | 0 | exact | -| mix | 3 | 128 | 1 | 4 | 4 | 4 | 4 | 32 | 32 | 128,013 | 948 | 1,957 | 20 s | 0 | exact | -| mix | 3 | 256 | 1 | 4 | 4 | 4 | 4 | 32 | 32 | 154,624 | 1,556 | 3,456 | 20 s | 0 | exact | -| mix | 1 | 256 | 1 | 4 | 4 | 4 | 4 | 32 | 32 | 171,419 | 1,419 | 2,915 | 20 s | 0 | exact | - -`Grp` is the number of KV data groups bound round-robin to DiskDB disk groups; -three groups in the default three-node topology gives each node's DiskDB disk -group a distinct KV group. `Epoll worker` applies uniformly to the client, -DiskDB server, DiskDB KV-client, and KV-server RPC layers. The fixture uses -4 TiB per logical disk so a 20-second high-concurrency run does not approach a -single node's available space. - -The active regression matrix runs allocate and 70/30 allocate/free mix at 1, -16, 128, and 256 threads. The 1/16-thread cases use two connections and two -epoll workers on every RPC layer; 128/256 threads use four of each. A separate -256-thread case binds all DiskDB disk groups to one KV data group to measure -the single-group ceiling. Other cases bind one KV data group per node. -Environment overrides can replace the group, connection, or worker count for -experiments. - -The 256-thread three-group allocation completed 3,191,195 allocations and -reported an exact 3,346,210,488,320-byte busy-space delta. The one-group case -completed 3,427,099 allocations with an exact 3,593,573,761,024-byte delta. - -The direct KV write sentinel peaks near 264K writes/s at 512 tasks with the -larger 64/64 window/coalesce profile. The closest 32-slot direct-KV references -reach about 181–191K writes/s at 128/256 threads. Against that range, the -one-group DiskDB allocation result is within about 6%, while the three-group -result is within about 12% of the 256-thread KV result. - -## 5. Bottleneck - -At 256 threads, DiskDB request latency is dominated by KV persistence. In -representative saturated allocation windows, whole-handler average latency is -about 1.25–1.76 ms while KV persistence is about 1.08–1.50 ms. Bitmap scan and -record construction average at or below one microsecond. KV batch-write -service averages about 0.5–0.7 ms, and KV processes consume several cores in -both user and system time. Inflight slots remain around one to three, so the -32-slot proposal window is not saturated. - -Three data groups are about 7% slower than one at the same 256-thread settings -for both allocate and mix. All groups are replicated across the same three KV -processes, so they contend for shared accept-round CPU and consensus transport; -adding groups does not provide independent hardware capacity. - -The current bottleneck is KV accept-round and RPC CPU capacity, not DiskDB -bitmap allocation, the 32-slot inflight window, or disk capacity. The durable -one-unit path cannot reach 400K operations/s through DiskDB tuning alone while -it requires one KV client write per operation and the measured direct KV -ceiling is about 264K operations/s. Reaching a higher rate requires raising the -underlying KV ceiling or changing the number of DiskDB operations represented -by one KV client write. Any such batching must preserve durable acknowledgement -and exact rollback semantics. - -### Concurrent-free coalescing reference (2026-09-14) - -The immediate-drain free coalescer was compared with the direct path using: - -```bash -DISKDB_BENCH_DURATION=10 DISKDB_BENCH_CASES='free_batch_off_mem free_batch_on_mem' pixi run -- bash tools/benchmark/bench-diskdb-regression.sh -``` - -Reference host: Intel Core i9-7960X (16 cores / 32 threads), x86_64, -Linux 6.11.0-29, XFS on `/dev/nvme1n1p1`. Both cases used one KV data group, -three KV nodes, three DiskDB instances, 128 workload tasks, one block per RPC, -four connections/workers at each RPC layer, memory KV/WAL, and a 10-second -70/30 allocate/free workload. - -| Free batch | ops/s | Avg us | p50 us | p99 us | Free requests | KV proposals | Requests/proposal | Errors | Space | -| ---------- | ----: | -----: | -----: | -----: | ------------: | -----------: | ----------------: | -----: | ----- | -| off | 112,319 | 1,138 | 1,110 | 1,766 | 337,596 | 337,596 | 1.00 | 0 | exact | -| on | 118,794 | 1,076 | 1,013 | 1,953 | 356,984 | 40,209 | 8.88 | 0 | exact | - -Enabled mode reduced KV proposals per free record by 88.7%, improved aggregate -throughput by 5.8%, and reduced average latency by 5.4%. Both paths completed -with zero errors and exact post-compaction space accounting. The deterministic -coalescer tests additionally feed the same incarnation-qualified records -through direct and combined persistence and verify whole-request atomicity, -cross-request duplicate suppression, failure fan-out, and explicit retry. - -Open observation: enabled-mode p99 was 10.6% higher in this single comparison -even though average and p50 improved. A repeated latency-focused run is needed -to separate normal run variance from the expected queueing tail of requests -that join an in-flight KV batch; this does not affect durability or proposal -reduction. - -## 6. Retained Logs - -Reference run roots: - -- `bench-log/diskdb-regression-20260905-133409`: complete current allocation - and mixed-workload matrix. -- `bench-log/diskdb-regression-20260914-010652`: direct versus enabled - concurrent-free coalescing comparison. - -Each root retains command output and configuration plus three KV metrics/RPC -log pairs, three DiskDB metrics/RPC log pairs, and one CLI metrics/RPC pair per -case. `results.tsv` records the resolved parameters and correctness fields. -Generated logs remain untracked. diff --git a/doc/design/diskio/design-crowdb-diskio.md b/doc/design/diskio/design-crowdb-diskio.md deleted file mode 100644 index a69986ab2..000000000 --- a/doc/design/diskio/design-crowdb-diskio.md +++ /dev/null @@ -1,885 +0,0 @@ - - - -# CROWDB - Design: diskio (Overview) - -diskio is the per-node data I/O engine. diskdb allocates disk blocks; -chunkdb manages chunk metadata; diskio reads and writes the block -contents. It is a C++ binary (`crowdb-diskio`) that runs on each storage -node, using io_uring on Linux for asynchronous I/O and a thread-pool -`pwrite`/`pread` fallback on macOS and non-liburing Linux builds. The -server uses the crowdb-rpc C++ engine directly; Rust callers -(chunkdb writers, recovery, rebalance) use a typed client crate that -wraps `crowdb-rpc-ffi`. - -The io_uring engine (`DiskIOUring`) is a multi-pipeline io_uring -wrapper that also serves the crowdb-tree btree page store. It lives in -`crowdb-common` and is shared by both consumers. - -## Table of Contents - -- [1. Overview](#1-overview) -- [2. Non-Goals (Design Envelope)](#2-non-goals-design-envelope) -- [3. Key Design Decisions](#3-key-design-decisions) - - [3.1 C++ server, Rust client](#31-c-server-rust-client) - - [3.2 io_uring on Linux; thread-pool pwrite/pread fallback](#32-io_uring-on-linux-thread-pool-pwritepread-fallback) - - [3.3 Control + data separation in the RPC frame](#33-control--data-separation-in-the-rpc-frame) - - [3.4 No disk status tracking](#34-no-disk-status-tracking) - - [3.5 Partial writes are errors, not retried internally](#35-partial-writes-are-errors-not-retried-internally) - - [3.6 Shared DiskIOUring in crowdb-common](#36-shared-diskiouring-in-crowdb-common) -- [4. Architecture Overview](#4-architecture-overview) -- [5. IoEngine Abstraction](#5-ioengine-abstraction) - - [5.1 AlignedWriter](#51-alignedwriter) - - [5.2 UringEngine](#52-uringengine) - - [5.3 BlockingEngine](#53-blockingengine) - - [5.4 DummyDiskEngine](#54-dummydiskengine) - - [5.5 NullDisk and MemDisk](#55-nulldisk-and-memdisk) -- [6. Disk Model](#6-disk-model) -- [7. DiskIOUring](#7-diskiouring) - - [7.1 Architecture](#71-architecture) - - [7.2 fd → Pipeline Routing](#72-fd--pipeline-routing) - - [7.3 Polling Modes](#73-polling-modes) - - [7.4 Batched SQE Submission](#74-batched-sqe-submission) - - [7.5 Pipeline Topology](#75-pipeline-topology) - - [7.6 Bad-Disk Cancel by fd](#76-bad-disk-cancel-by-fd) - - [7.7 Lock-Free Design](#77-lock-free-design) - - [7.8 FFI Multi-Eventfd Pump](#78-ffi-multi-eventfd-pump) -- [8. RPC Service](#8-rpc-service) -- [9. Client Library](#9-client-library) -- [10. Invariants](#10-invariants) -- [11. Configuration](#11-configuration) -- [12. References](#12-references) - ---- - -## 1. Overview - -diskio is a **per-node data I/O server**. Each storage node runs one -`crowdb-diskio` process that owns a set of local disks and serves -read/write/fsync RPCs over crowdb-rpc. Callers (chunkdb's large-object -writer, small-object writer, chunk read flow, recovery, rebalance) -send a `Segment`-based address — `{disk_id, zone_index, zone_offset, -size}` — and diskio translates it to a physical offset and performs -the I/O. - -**Language:** C++ server, Rust client. **I/O:** io_uring (Linux), -thread-pool `pwrite`/`pread` (macOS, non-liburing Linux). - -**Core goals:** -- **Low-latency data path** — io_uring on Linux with no - `spawn_blocking`, no thread hop, no Rust→C++ round-trip on the data - path. The entire I/O path is async: RPC receive → DiskIOUring submit → - CQE completion → RPC response. -- **Cross-platform dev/test** — thread-pool `pwrite`/`pread` fallback - on macOS gives the same semantics at lower performance, so the test - suite runs everywhere. -- **Shared DiskIOUring** — the io_uring engine lives in `crowdb-common` - and is shared by the btree page store and the diskio engine. One - implementation, two consumers. -- **Testability** — `MemDisk` (drop-write + rule-based read) for - throughput benches, `SimulatedDisk` (latency + error injection) for - fault-path tests. Both run without real hardware. - -## 2. Non-Goals (Design Envelope) - -- **No disk allocation.** diskdb allocates blocks; diskio reads and - writes their contents. diskio does not track which blocks are busy - or free. -- **No allocation-ownership fencing.** diskio does not validate allocation - incarnations or decide whether an extent may be reused. The owning lifecycle - layer prevents reuse while an earlier writer can still retry, for example by - retaining freed extents beyond the maximum write retry window. -- **No disk status tracking.** diskio does not maintain disk health - state. If an I/O fails, the engine returns the error to the caller; - the top layer (chunkdb/diskdb) handles the failure and stops - allocating new blocks on that disk. -- **No chunk metadata.** chunkdb manages chunk/strip metadata; diskio - is unaware of chunks, strips, or redundancy. It reads and writes - raw block contents at the `Segment` granularity. -- **No internal retry on partial writes.** A short write (fewer bytes - than requested) is an error, not retried internally. The caller - decides whether to retry the whole write or fail the operation. -- **No RDMA or SPDK (v1).** v1 uses Linux io_uring on block devices - and regular files. RDMA and SPDK are future additions. -- **No streaming RPC.** v1 supports request-response only. The data - payload (up to the max block size, default 2 MB) fits in one frame. - -## 3. Key Design Decisions - -### 3.1 C++ server, Rust client - -The diskio server is C++ because the io_uring engine and the -crowdb-rpc server engine are C++. Running the server in C++ means the -data path has no FFI boundary: RPC receive, frame decode, I/O submit, -CQE completion, and response submit all happen in C++ without a -Rust→C++ round-trip. The Rust client (`crowdb-diskio-client`) wraps -`crowdb-rpc-ffi` with typed `DiskIoClient` methods, following the -existing `crowdb-diskdb-client` / `crowdb-chunkdb-client` pattern. - -### 3.2 io_uring on Linux; thread-pool pwrite/pread fallback - -io_uring is the Linux asynchronous I/O interface. It provides -submission queues (SQ) and completion queues (CQ) in shared memory, -allowing batched I/O submission and completion polling with minimal -syscalls. For `O_DIRECT` on block devices, I/O completes inline -during SQ processing in most cases — no kernel thread pool needed. - -macOS has no io_uring. POSIX `aio` exists but is weak: internally -thread-pool-based, limited queue depth, inconsistent across -filesystems. `libaio` is Linux-only. A dedicated C++ thread pool with -`pwrite`/`pread` and configurable `fdatasync` is the pragmatic -cross-platform fallback. It also serves as the Linux non-liburing -production path (if liburing is not available at build time). - -The `IoEngine` virtual base abstracts over both backends so the RPC -layer is backend-agnostic. - -Read, write, and fsync latency is measured from engine submission through -completion with the same `diskio.engine.*.lh` histograms for both backends. -The io_uring backend additionally publishes current in-flight operations and -SQ saturation as `diskio.uring.inflight.g` and `diskio.uring.sq_full.c`. -RPC request and response send-queue pressure remains owned by crowdb-rpc's -transport latency and rejection counters rather than being duplicated here. - -### 3.3 Control + data separation in the RPC frame - -A diskio RPC frame carries a small flatbuffer control message -(`{disk_id, zone_index, zone_offset, size}`) and a raw data payload -in separate buffers. crowdb-rpc's 12-byte header carries both -`msg_size` (control) and `data_size` (data), so the parser knows the -full frame shape after 12 bytes. The data payload is written directly -to / read directly from the I/O buffer with no intermediate -serialization. This mirrors crowdb-rpc's control+data separation design -(see `design-crowdb-rpc.md` §2). - -### 3.4 No disk status tracking - -diskio does not track whether a disk is Good, Bad, or Suspicious. It -opens disk handles at startup and serves I/O until a disk fails. When -an I/O fails, the engine returns the error to the caller. The top -layer (chunkdb/diskdb) is responsible for marking a disk as bad and -stopping new allocations on it. After all in-flight I/O to a failing -disk drains, no further I/O is sent to it. - -Administrative Offline is enforced by Group-0 discovery. A complete valid -membership observation detaches Offline disks from the active map, rejecting -new requests while existing handles drain through descriptor retirement. Offline -disks still contribute their physical DiskGroup identity to the service's -observation. Bad, Suspect and maintenance states do not by themselves close a -handle needed for recovery reads. Failed or malformed membership observations -retain the previous map; missing status or duplicate disk identities are errors. - -This keeps diskio simple — it does one thing (data I/O) and reports -results. Disk health policy lives in diskdb, where it belongs. - -### 3.5 Partial writes are errors, not retried internally - -When `pwrite` or io_uring returns fewer bytes than requested, the -engine returns `IoError::PartialWrite` immediately. No internal retry. - -For `O_DIRECT` block devices (the primary path), partial writes -essentially do not occur for correctly aligned I/O. If they do, it -usually indicates a hardware error — retrying the same range is -futile. The caller (chunkdb writer) has strip-level retry logic and -can decide whether to retry the whole strip or fail it. Internal -retry would complicate the completion callback path (resubmit for -remaining bytes, track partial progress, handle retry-failure) for no -benefit. - -### 3.6 Shared DiskIOUring in crowdb-common - -`DiskIOUring` is a multi-pipeline io_uring engine: it maps file -descriptors to pipelines (one `io_uring` instance each), shares -polling threads across CQs, batches SQE submission, and cancels -in-flight I/O by fd via `IORING_ASYNC_CANCEL_FD`. It is not specific -to the btree page store. It lives in `crowdb-common` so both the btree -page store (`crowdb-tree`) and the diskio engine share one -implementation. The WAL's io_uring integration later builds Rust -async submit wrappers over the same C ABI. - -## 4. Architecture Overview - -``` -Caller (Rust) crowdb-diskio Server (Node X, C++) - │ │ - │ DiskIoClient::write(seg, data) │ - │ ──► crowdb-rpc-ffi frame: │ - │ [hdr][DiskWriteReq][data] │ - │ ──────────────────────────────────►│ - │ │ - │ crowdb-rpc reader decodes frame - │ handler: resolve disk_id → Disk - │ compute phys_offset = zone_base + zone_offset - │ │ - │ ▼ - │ IoEngine::write(disk, phys_offset, data, on_complete) - │ │ - │ ┌─────────┴────────────────────┐ - │ │ UringEngine (Linux) │ BlockingEngine (macOS) - │ │ crowdb::common::DiskIOUring │ thread pool - │ │ fd→pipeline, SQE submit │ pwrite → callback - │ │ → CQE completion │ - │ └─────────┬────────────────────┘ - │ │ (also: DummyDiskEngine wrapper for dummy disks) - │ │ - │ on_complete → crowdb_rpc_server_submit_response - │ crowdb-rpc response: [hdr][DiskWriteResp] - │ ◄──────────────────────────────────│ - │ write() returns Ok(()) │ -``` - -Components: -- **DiskSet** — holds `HashMap>`, opened at - startup from the node's disk list. Resolves `disk_id` to a `Disk`. -- **Disk** — virtual base with subclasses: `BlockDisk` (real block - device, `O_DIRECT`), `NullDisk` (`/dev/zero`, drop-write + pattern read), - `MemDisk` (memfd, store + read-back). - Each `Disk` shares the node's `IoEngine` instance; dummy disks wrap - it with a `DummyDiskEngine` for read-content hack and optional fault - injection. -- **Zone** — `{zone_index, base_offset, capacity, state}`. The - physical offset for an I/O is `zone.base_offset + zone_offset`. -- **IoEngine** — virtual base: `submit_write`/`submit_read`/ - `submit_fsync` taking a disk handle, physical offset, buffer, size, - and a completion callback. Implementations: `UringEngine`, - `BlockingEngine`, `DummyDiskEngine` (wrapper for dummy disks). -- **DiskIOUring** — the shared multi-pipeline io_uring engine in - `crowdb-common`. Maps fds to pipelines (one `io_uring` instance per - pipeline); polling threads drain CQs and submit pending SQEs. - Lock-free on the hot path (SQE claim via atomic CAS, fd→pipeline - lookup via direct-indexed array). -- **RPC handler** — dispatches `DiskWriteRequest`/ - `DiskReadRequest`/`DiskFsyncRequest` to the `IoEngine`; the - completion callback builds the response and submits it via - `crowdb_rpc_server_submit_response`. -- **DiskIoClient** — Rust client wrapping `crowdb-rpc-ffi` with typed - methods. Routes to the correct node's diskio server based on - `segment.node_id`. - -## 5. IoEngine Abstraction - -`IoEngine` is a C++ virtual base: - -``` -class IoEngine { - public: - virtual ~IoEngine() = default; - virtual void submit_write(Disk *disk, off_t phys_offset, - const uint8_t *data, size_t size, - std::function on_complete) = 0; - virtual void submit_read(Disk *disk, off_t phys_offset, - uint8_t *buf, size_t size, - std::function on_complete) = 0; - virtual void submit_fsync(Disk *disk, - std::function on_complete) = 0; -}; -``` - -`on_complete` is invoked exactly once with the raw result: `>=0` bytes -transferred, `<0` negative `-errno`. The RPC handler's callback -resolves the request and calls `crowdb_rpc_server_submit_response`. - -### 5.1 AlignedWriter - -`AlignedWriter` is the server-owned stage between the write RPC handler and a -disk's `IoEngine`. Every write traverses it. `NullDisk` and `MemDisk` use a -block size of one, so the same path delegates their writes without padding. - -For disks with larger blocks, the stage aligns the physical range and retains -one aligned buffer through backend completion. It preserves every byte outside -the requested range. Partial head and tail blocks come from valid cache entries -or exact boundary-block reads; a range spanning both uncached boundaries needs -at most two reads. Request bytes are overlaid only after those reads complete. -The RPC reports the logical count only after the complete aligned write succeeds. - -The stage caches the last partial physical block by disk and block offset. -The bounded cache is an optimization: eviction, invalidation and process restart -do not change range-write semantics. Failed writes invalidate affected entries; -full-block writes invalidate prior partial images. An aligned start with a -partial tail also preserves the rest of that physical block, so a delayed -immutable retry cannot erase bytes written by a later request. - -Writes are assigned to 64 shards by disk and segment ordering base. Each shard -has a bounded lock-free MPSC queue and one active consumer through write -completion. Related partial-block writes remain ordered without a mutex. -Independent segment bases and disks can execute separately apart from hash -collisions. Queue saturation fails the request with `-EAGAIN`. - -Byte-addressed writes use the same ordering queue. A zero-length ordered request -waits for preceding write completions without issuing a media write; when its -sync flag is set, the RPC handler then flushes the disk before proving completion. -This is a segment drain after the caller has excluded future producers. A flush -alone, or a drain while new producers remain authorized, cannot establish a final -tail or authorize physical reuse. - -At admission, a write timestamp must fall within the bounded request age and -future clock-skew allowance. The server derives a conservative monotonic deadline -and checks it when dequeuing and before submitting an aligned merge after boundary -reads. An expired queued request returns `OldRequest` without a media write. An -already submitted backend operation retains its queue position through completion, -so a later drain waits for it even when its kernel execution outlasts request age. -The shared admission ceilings are 30 seconds of request age and one second of -clock skew. A server may use shorter windows; startup rejects larger values so -it cannot extend the late-write interval assumed by recovery and reclamation. - -Concurrent overlapping writes are not a supported caller contract. As with -raw `pwrite`, their final contents would otherwise depend on completion order. -The chunk writers submit sequential updates to a segment and wait for each -response before issuing the next overlapping update. - -### 5.2 UringEngine - -Linux only (`CROWDB_HAVE_LIBURING`). Owns a `crowdb::common::DiskIOUring` -instance (one engine per diskio server). `O_DIRECT` aligned writes -(configurable, default on for data blocks). The entire I/O path is -async — no `spawn_blocking`, no thread hop. - -`UringEngine` is a thin adapter: it validates the disk handle and -`O_DIRECT` alignment, then delegates to `DiskIOUring::submit_*` with -the disk's fd. `cancel_disk` looks up the disk's fd via `DiskSet` and -calls `DiskIOUring::cancel_fd` — one SQE cancels all in-flight I/O on -that fd via `IORING_ASYNC_CANCEL_FD` (kernel 6.0+). No per-disk -in-flight tracking map; the kernel does the cancel lookup. See §7.6. - -### 5.3 BlockingEngine - -macOS + non-liburing Linux. A dedicated C++ thread pool (configurable -size, default 4 threads per disk) with `pwrite`/`pread` and -configurable `fdatasync`/`fsync`. Each I/O operation is submitted to -the pool; the worker thread performs the blocking syscall and invokes -the completion callback. Correct semantics, lower performance (thread -hop per I/O). Also used for `BlockDisk` without `O_DIRECT` (pwrite to -a block device at an offset). - -### 5.4 DummyDiskEngine - -The dummy-disk wrapper. `DummyDiskEngine` wraps a real `IoEngine` -(`UringEngine` or `BlockingEngine`) and provides two features: - -- **Read-content hack** (NullDisk): after the inner engine completes - a `pread`, the wrapper overwrites the buffer with deterministic - pattern data generated from `disk_id` + `phys_offset` (xorshift64 - PRNG). The full uring/blocking `pwrite`/`pread` syscall path - executes against `/dev/zero` — no real disk I/O or retained page-cache - data, but the ring mechanics, syscall cost, and DiskIOUring batching all run. - Used for benchmark tests that measure uring overhead without - storage. -- **Fault injection** (NullDisk + MemDisk): per-I/O random latency - (uniform draw from `latency_min_ms`..`latency_max_ms`, sleep before - completing) and errors (return `-EIO` at the configured - `error_rate`). Configured via `DiskProperties` on the dummy disk. - Merged from the former `SimulatedEngine` — no separate wrapper - engine needed. - -### 5.5 NullDisk and MemDisk - -Two dummy disk types use different memory semantics: - -- **NullDisk** — the default dummy disk, backed by `/dev/zero`. Writes are - discarded without retaining page-cache data. Reads execute the full - `pread` path, then the `DummyDiskEngine` wrapper overwrites the buffer - with pattern data. For benchmark tests: measures uring/blocking overhead - without real disk I/O, storage capacity limits, or memory growth. -- **MemDisk** — backed by `memfd_create` (tmpfs), stores written data and - reads it back. For end-to-end correctness tests that verify I/O data - integrity. The full `pwrite`/`pread` path executes against the memfd; - no real disk I/O. - -Both dummy disks share the node's `IoEngine` instance (uring or -blocking, auto-detected). Optional `DiskProperties` enable fault -injection on either type. - -## 6. Disk Model - -`Disk` is a C++ virtual base with subclasses: - -- **BlockDisk** — real block device, opened with `O_DIRECT | O_DSYNC | O_RDWR` - (Linux). Aligned I/O only. The primary production disk type for - NVMe/SATA SSDs and HDDs. The device path comes from the - `device_path` field in `DiskValue` (group-0 sysdata). -- **NullDisk** — `/dev/zero`-backed dummy disk. Writes are discarded - without retained memory; reads return deterministic pattern data via - `DummyDiskEngine` wrapper. - Default dummy disk type. For benchmark tests. -- **MemDisk** — memfd-backed dummy disk. Stores written data and - reads it back. For end-to-end correctness tests. - -Each `Disk` shares the node's `IoEngine` instance (auto-detected: -uring on Linux with liburing, blocking otherwise). Dummy disks -(`NullDisk`, `MemDisk`) wrap the shared engine with a -`DummyDiskEngine` for read-content hack and optional fault injection. - -Every valid descriptor used by `UringEngine`, including dummy disks discovered -after startup, is registered with `DiskIOUring`. Removal unregisters the -descriptor after draining in-flight work. The first group-0 disk sync runs -immediately so a restarted service does not advertise an empty `DiskSet` for -one refresh interval. - -Detachment removes the disk from new request lookups first. The synchronization -owner retains its handle until old map snapshots and all read/write/fsync -callbacks have released theirs, then unregisters and closes the descriptor. -Rediscovery waits for this retirement before reopening the same disk identity. - -`Zone` holds `{zone_index, base_offset, capacity, state}`. The -physical offset for an I/O is `zone.base_offset + zone_offset`. Zone -records come from diskdb's allocation. - -`DiskSet` holds `HashMap>`, opened at startup -from the node's disk list (via `HardwareClient` / diskdb). `DiskId` -is the 128-bit id from diskdb. - -## 7. DiskIOUring - -`DiskIOUring` is a multi-pipeline io_uring engine. It maps file -descriptors to pipelines (one `io_uring` instance per pipeline), -shares polling threads across CQs, batches SQE submission, and is -fully lock-free on the hot path. Both diskio and btree use -`DiskIOUring` as their I/O interface. - -`DiskIOUring` lives in `crowdb-common` (`crowdb::common::DiskIOUring`), -guarded by `CROWDB_HAVE_LIBURING`. It is Linux-only. On macOS and -non-liburing Linux builds, `DiskIOUring` is absent and -`BlockingEngine` is used instead. - -### 7.1 Architecture - -``` - DiskIOUring - ┌─────────────────────────────────────────────┐ - │ fd_table[fd] → pipeline_index (read-only) │ - │ │ -submit_read(fd) ──► │ Pipeline 0 Pipeline 1 ... │ -submit_write(fd) ─► │ ┌──────────┐ ┌──────────┐ │ -submit_fsync(fd) ─► │ │ io_uring │ │ io_uring │ │ -cancel_fd(fd) ────► │ │ SQ + CQ │ │ SQ + CQ │ │ - │ └────┬─────┘ └────┬─────┘ │ - │ │ │ │ - │ ┌────┴────────────────┘ │ - │ │ PollThread 0 (busy-poll or epoll-wait) │ - │ │ drains CQ-0 + CQ-1, submits pending SQEs │ - │ └───────────────────────────────────────────┘ - └─────────────────────────────────────────────┘ -``` - -Components: -- **Pipeline** — one `io_uring` instance: one SQ + one CQ in shared - memory. The low-level uring wrapper (lock-free SQE claim, CQE - drain). Internal to `DiskIOUring`; not exposed to callers. -- **PollThread** — one OS thread that polls a group of pipelines' CQs - and submits their pending SQEs. Multiple CQs share one polling - thread (no one-thread-per-CQ requirement). Configurable grouping. -- **fd_table** — direct-indexed `std::vector` mapping fd → - pipeline_index. Populated at `register_fd` time, read-only during - I/O. No synchronization needed (concurrent reads of an immutable - array). -- **CallbackEntry** — allocated per submit, freed after CQE dispatch - via deferred-delete (polling-thread-only). Holds the callback; no - atomics, no cancel flags, no hazard pointers. Callback suppression - is client-side (§7.7): the caller wraps the callback with a shared - cancel flag. - -### 7.2 fd → Pipeline Routing - -`fd_table` is direct-indexed by fd number. fds are small integers -(typically < 1024). The vector is sized once at construction to -`ulimit -n` (capped at 4096) and never grows — no reallocation, no -data race on concurrent reads. Each slot defaults to `SIZE_MAX` -(sentinel: "not registered"). Looking up `fd_table[fd]` is a single -memory load — no hash, no collision, no atomic, no lock. - -Two registration modes: -- **Auto-assignment** (`register_fd(fd)`) — `DiskIOUring` picks the - pipeline with the lowest in-flight count and sticks the fd to it. - Used by btree (single pipeline) and uniform-disk diskio topologies. - Sticky: once assigned, all future submits for that fd go to the - same pipeline (required for `cancel_fd` correctness — cancel targets - the ring where the fd's SQEs were submitted). -- **Explicit assignment** (`register_fd(fd, pipeline_index)`) — caller - pins the fd to a specific pipeline. Used when the caller built a - disk-class-aware topology (NVMe → pipeline 0, HDD → pipeline 2) and - needs deterministic routing. - -### 7.3 Polling Modes - -A `PollingMode` config per pipeline: - -- **Classic** — event-wait on the poll thread's private eventfd with a - bounded timeout. Both submissions and kernel completions wake it; - no pipeline waits on an empty CQ while another pipeline has ready work. - Used by crowdb-tree's single-pipeline topology. -- **Hybrid** `{ busy_poll_budget }` — busy-poll the CQ ring's shared - memory via `io_uring_peek_cqe` with no syscall while I/O is active. - After `busy_poll_budget` consecutive empty peeks, transition to - `epoll_wait` on the poll thread's private eventfd. Any CQE resets the counter, - returning to busy-poll. Gives sub-µs CQE dispatch during I/O bursts, - sleeps when idle, no core burned at idle. Default for `UringEngine`. -- **Sqpoll** `{ sq_thread_idle }` — opt-in for sustained high-IOPS: - `IORING_SETUP_SQPOLL` eliminates submit syscalls via a kernel - SQ-poll thread. Requires root/CAP_SYS_NOPRIV. Useful for recovery - scan and bulk rebalance. - -When multiple Hybrid-mode pipelines share a poll thread, the thread -uses the **minimum** `busy_poll_budget` across its assigned Hybrid -pipelines — the most conservative budget wins. - -Each pipeline registers its owning poll thread's private eventfd with -`io_uring`. The poll thread drains ready CQEs from every assigned pipeline -and then signals the separate pipeline eventfds consumed by external -reactors. External notifications therefore follow callback completion; -external consumers cannot steal the poll thread's wakeup. - -### 7.4 Batched SQE Submission - -Client threads fill SQE slots (lock-free CAS on shadow tail + per-slot -ready flags) and set a `pending_submit` atomic flag. The polling -thread submits all pending SQEs in one `io_uring_enter` per iteration -— batching N SQEs into one syscall. Clients never call -`io_uring_enter`; only the polling thread does. - -In busy-poll mode (I/O active), SQEs are submitted within <1µs of -being filled. The first client changing `pending_submit` from false to -true writes the owning poll thread's private eventfd. Subsequent clients -in the same pending batch skip that write. The eventfd remains readable -if the thread has not entered event-wait yet, so a submission cannot lose -its wakeup. The thread publishes newly pending SQEs after waking and -before draining completions. - -### 7.5 Pipeline Topology - -io_uring does not support sharing a CQ ring across separate -`io_uring` instances — each `io_uring_queue_init` creates its own SQ + -CQ pair. But one ring can submit I/O for any number of fds, so the -answer is fewer rings, not shared CQs. Topology by disk type: - -- **NVMe SSD** (100k+ IOPS/disk): one pipeline per disk. High IOPS - needs SQ headroom; per-disk isolates SQ/CQ backpressure so one busy - disk's full SQ doesn't block another. -- **SATA SSD** (10k-100k IOPS/disk): one pipeline per 4-8 disks - (grouping). Medium IOPS; grouping reduces polling thread count - while keeping SQ headroom. For 24-30 SSDs this gives 3-8 rings, not - 24-30. -- **HDD** (100-200 IOPS/disk): one shared pipeline for all HDDs. Low - IOPS; one ring's SQ handles 30 HDDs trivially. `entries = 2048`. - -Poll thread grouping: -- **1 thread, all pipelines** (default, simple): one thread - busy-polls all CQs. Best for low pipeline count. -- **1 thread per pipeline** (max isolation): each CQ has its own - thread. Best for high-IOPS NVMe where CQ drain latency matters. -- **Grouped by disk class** (balanced): `{ {0,1,2}, {3,4,5} }`. 2 - threads for 6 pipelines. Best for SATA SSD farms. -- **Grouped by NUMA node**: pipelines whose disks are on NUMA node 0 - → thread 0 (pinned to node 0 core); node 1 → thread 1. - -`IORING_SETUP_ATTACH_WQ` (Linux 5.18+): when `Topology::attach_wq` is -true and there are ≥2 pipelines, subsequent pipelines share the -kernel's io-wq pool (8 kernel threads total instead of N×8). For -`O_DIRECT` on block devices, I/O almost always completes inline and -io-wq is rarely involved; `attach_wq` matters most for non-`O_DIRECT` -file I/O. Default: `false` (opt-in). - -### 7.6 Bad-Disk Cancel by fd - -`cancel_fd(fd)` cancels all in-flight I/O for an fd with one SQE: -`IORING_OP_ASYNC_CANCEL` with `flags = IORING_ASYNC_CANCEL_FD | -IORING_ASYNC_CANCEL_ALL` and `addr = fd` (kernel 6.0+). The kernel -cancels all matching ops and posts CQEs (`-ECANCELED` for each -cancelled op, or the original result if already completed). No -per-fd in-flight tracking needed — the kernel does the lookup. Zero -overhead on the hot path. - -On a shared pipeline, a bad disk's in-flight I/O holds SQ slots. If -the SQ fills, good disks' I/O is rejected with `-ENOMEM`. -`cancel_fd` frees SQ slots immediately by cancelling all of the bad -fd's in-flight ops. The SQ ring's capacity check is -`tail - head >= entries`, where `head` is the kernel's consumed SQ -head — in-flight I/O is in the block layer, not the SQ ring, so the -SQ has space for the cancel SQE even when all of the fd's I/O is -in-flight. - -**Kernels < 6.0** (no `IORING_ASYNC_CANCEL_FD`): `cancel_fd` returns -`-ENOSYS`. The caller (diskio's `cancel_disk`) logs a warning and -falls back to waiting for in-flight I/O to complete naturally (the -bad-disk path has a timeout). macOS never compiles `DiskIOUring`; -`BlockingEngine` is the non-uring path. - -CQEs are independent — the polling thread drains all ready CQEs each -iteration, so a slow disk's CQE arriving late doesn't block CQ drain -for others. - -### 7.7 Lock-Free Design - -The hot path (submit + CQE completion) uses no mutexes, no locks, no -blocking: - -- **SQE claim**: atomic CAS on shadow tail + per-slot ready flags. - Multi-threaded clients claim slots concurrently with no contention - beyond CAS retries. -- **fd → pipeline lookup**: direct array index, read-only. No - synchronization. -- **Batch submit coordination**: `pending_submit` atomic flag, - `pending_count` atomic counter, `thread_sleeping` atomic flag. All - single-word atomics. -- **CQ drain**: polling thread only (single-threaded per CQ). No - concurrency, no synchronization. -- **Callback dispatch**: polling thread only. `CallbackEntry` holds - just the callback — no `cancelled`/`dispatched` atomics. The - dispatch path calls `entry->cb(res)` unconditionally; if the client - wants to suppress the callback, it checks a shared cancel flag - inside the callback itself (client-side cancel). The CQE always - arrives, the callback always fires, the SQ slot is always freed — - only the callback body is skipped via the flag. A `shared_ptr` keeps - the cancel state alive until the CQE arrives (no UAF). -- **CallbackEntry lifetime**: deferred-delete on the polling thread. - The polling thread is the sole toucher of `CallbackEntry` after - `submit_*` returns — no concurrent access, no hazard pointers, no - generation counters. - -**Kernel-internal locks** (not controllable from userspace): - -| I/O path | Lock | Hold duration | Contention scope | -| --- | --- | --- | --- | -| SQE fill / CQE drain | None | — | — | -| `O_DIRECT` block device (diskio `BlockDisk`) | `blk-mq` per-HW-queue spinlock | µs (queue insertion) | per-HW-queue | -| `O_DIRECT` regular file write (btree block files) | `i_rwsem` exclusive | full I/O latency | per-inode | -| `O_DIRECT` regular file read | `i_rwsem` shared | full I/O latency | per-inode (concurrent reads OK) | - -io_uring never blocks the userspace submitter on `i_rwsem` — when an -inline execution would block, io_uring punts to `io_wq` (kernel worker -pool). The design's per-fd pipeline isolation aligns with the kernel's -per-device / per-inode lock scope: if one fd's I/O contends on a -kernel lock, it is on its own pipeline with its own polling thread. -Both consumers avoid same-inode write contention by design: btree -uses one file per extent (different inodes), diskio writes to block -devices (no `i_rwsem`). - -### 7.8 FFI Multi-Eventfd Pump - -Each pipeline registers its eventfd via -`IORING_REGISTER_EVENTFD`. The Rust FFI layer -(`lib/crowdb-tree/ffi/src/reactor.rs`) spawns one `EventfdPump` per -pipeline eventfd, all sharing one `Arc`. Any pipeline's CQE -completion wakes all waiting `drive_ct_future` calls. The C ABI -function `ct_uring_eventfds` returns the pipeline eventfds; the Rust -wrapper spawns one pump per fd. - -## 8. RPC Service - -The diskio server uses crowdb-rpc's `RpcServer::register_handler` to -handle three message types. Message type IDs are in the diskio range -(3600s) of crowdb-rpc's `msg_type` enum. - -Each request's control message is a flatbuffer with -`{disk_id, zone_index, zone_offset, size}` (read also has -`test_pattern_offset`); the write request additionally carries an -`ordering_zone_offset`, `write_create_time_ms`, a `sync` option, and a raw data -payload of exactly `size` bytes. The write handler validates the FlatBuffer, -zone bounds and payload length before IO. The handler: - -Async callbacks retain the disk, connection, buffer and shared transport -independently of the server facade. Exact registered payload views remain -DiskOwned through write and synchronization completion; a closed connection -suppresses response submission without releasing active disk data. -1. Resolves `disk_id` to a `Disk` via `DiskSet`. -2. Computes the physical offset: `zone.base_offset + zone_offset`. -3. Calls `AlignedWriter::submit_ordered` for writes or `IoEngine::read`/`fsync` for - the other operations, with the disk handle, physical offset, buffer, and a - completion callback. -4. A synchronized write submits its disk flush only after a complete successful - write. Its sole response follows that flush; buffered writes reply after the - write. Failed or short writes never flush or report synchronized success. -5. Completion retains the disk, shared connection and request buffer through - both operations, then submits the response through the transport. Rejected - submission releases the caller-owned response frame. - -The data payload is passed from the crowdb-rpc frame decoder to -`AlignedWriter`. Block-size-one writes remain zero-copy on the server; padded -writes use one aligned server buffer. The read response includes the raw data -payload. - -The negotiated RDMA control/data layout supplies a separately aligned 1 MiB -data region. Eligible aligned writes hand this registered receive storage to -the disk engine and retain DiskOwned state until completion. A READ reuses its -request's reserved receive data allocation as output, retaining it through disk -and response SEND completion. Client response leases stay caller-owned until -their final release. These retained receive allocations propagate disk and -consumer pressure through the bounded receive window. Alignment edges retain -the existing explicit staging path; TCP keeps the same result/error mapping. - -Related writes are serialized by `ordering_zone_offset` within -`AlignedWriter`. This ordering protects partial-block read-modify-write state; -it does not validate allocation ownership. Write requests carry only the -current addressing, ordering, request-age and synchronization contract. - -Flatbuffer schemas (`diskio.fbs`): -- `DiskWriteRequest { disk_id, zone_index, zone_offset, size, - ordering_zone_offset, write_create_time_ms, sync }` -- `DiskWriteResponse { ret_code, sync_completed }` -- `DiskReadRequest { disk_id, zone_index, zone_offset, size, test_pattern_offset }` - (`test_pattern_offset` used by NullDisk for deterministic content; - default 0 = start of disk/object) -- `DiskReadResponse { ret_code }` (data payload follows the control - message) -- `DiskFsyncRequest { disk_id }` -- `DiskFsyncResponse { ret_code }` - -## 9. Client Library - -`DiskioClient` (Rust, in `crowdb-diskio-client`) is the routed semantic -boundary for every production Rust caller. Callers provide an allocated -`SegmentTarget`, a segment-relative byte range, a normal or priority traffic -lane, one total deadline, and (for writes) buffered or fsync durability: - -``` -async fn write(target, offset, data: Bytes, durability, options) -> Result<()> -async fn read(target, offset, length, options) -> Result -async fn fsync(disk_id, options) -> Result<()> -``` - -The client constructs its control-plane handles from group-0 management seeds, -or accepts injected `ServiceRegistryClient` and `HardwareClient` handles. It -joins live DiskIO registrations with hardware disk-group ownership and publishes -one immutable route generation mapping every `DiskId` to rack, node, disk group, -DiskIO instance, and endpoint. Missing or duplicate ownership, missing hardware -identity, an identity mismatch, a malformed endpoint, or either failed -control-plane read rejects the entire candidate. There is no fallback endpoint -and no partial generation. - -Each endpoint has independently bounded normal and priority connection groups. -Unchanged refreshes reuse those groups. An endpoint or instance change creates -a new generation before publication; operations retain the selected old -connection through completion, and exact generation checks prevent a delayed -old failure from invalidating its replacement. Removing a pool drops the -client's references, while the final in-flight reference controls when its C++ -connection is closed. Selection and route/status snapshots use immutable RCU -state and atomics, so request routing does not acquire a caller lock. - -Readiness uses the priority group for bounded 4096-byte reads of existing disk -headers. Normal bulk responses and readiness headers therefore have independent -TCP response ordering while sharing the same validated disk identity and route. -Priority selection does not change physical read semantics, error propagation, -route refresh rules or the caller's total deadline. - -The internal wire transport retains the caller's owned `Bytes` allocation in -the RPC buffer until completion. This avoids a client-side `Bytes` to `Vec` -payload copy. On the server, block-size-one and already aligned payloads can -continue directly into `IoEngine::write`. Other writes are copied once into -the server-owned aligned buffer that preserves boundary bytes and satisfies the -`O_DIRECT` address-alignment requirement. - -`write_views` splits oversized view lists into bounded RPC frames without -copying their payload. Frames within one strip's disk block complete serially -in byte order, including aligned boundaries. Connections can reorder arrivals; -serial completion preserves a contiguous prefix before the final flush. Separate -caller operations on that block must obey the same ordering. Independently -owned strip blocks and different disks holding EC shards or mirror replicas -may run concurrently. Only the final frame requests synchronization, after -preceding frames have completed. Its flush covers the whole preceding range; -a frame error stops the remaining submissions. - -Input arithmetic, segment bounds, and wire-size limits are checked before RPC -admission. Reads retry transient transport outcomes within the original -deadline. Writes can retry only the same immutable bytes at the same allocated -location and retain their original creation time; partial writes and permanent -disk errors return immediately. Expired operations submit no new request. -Fresh-storage initialization disables automatic retries, and an unresolved -admitted write is reported as ambiguous. Durable writes require -`sync_completed=true` after write and flush, using one request/reply. Missing -proof fails directly. A failed -flush returns `FsyncFailed`, exposed as `DurabilityFailure`; no failure is a -proof of synchronization. Independent fsync remains available and can be -repeated after an ambiguous response. The lifecycle no-reuse window is what -makes an identical write retry safe. - -Failure of an accepted RDMA read, write or fsync is terminal for that operation: -the client does not reconnect and replay it through a TCP fallback connection. -It returns the typed transport error, ambiguous write outcome or durability -failure. Known pre-submit rejections retain the existing retry handling. - -Status exposes route generation and age, topology and pool counts, healthy -connections, operation/retry/backpressure/ambiguity counters, and operation -latencies without exposing raw transport handles. The only production escape -hatch is an opaque retained priority-lane route set for the native crowdb-tree -page store. Wire-level methods remain available only to protocol tests through -the crate's `test-util` feature. - -## 10. Invariants - -- **I1 (data integrity)**: A successful `write(segment, data)` - followed by a `read(segment, None)` on the same `Segment` returns - the same bytes. Holds for `UringEngine`, `BlockingEngine`, and - `DummyEngine` (with rule-based read content). -- **I2 (durability)**: A successful synchronized write, or a write followed - by successful explicit fsync, retains the written data across process restart - when its backing media remains available. A synchronized response follows - both write and flush, with exactly one completion. -- **I3 (offset correctness)**: The physical offset for an I/O is - `zone.base_offset + zone_offset`, where `zone` is looked up by - `zone_index` from the `Disk`'s zone records. -- **I4 (no silent drop)**: `DiskIOUring` does not drop I/O requests. - If the SQ is full, the lock-free SQE claim enters bounded retry - waiting for a slot. If the ring is invalid, the completion callback - is invoked synchronously with `-EIO`. -- **I5 (completion guarantee)**: `on_complete` is invoked exactly - once per `submit_*` call — either from the polling thread (CQE - dispatched) or synchronously (submission failure). `cancel_fd` - replaces the original callback with a `-ECANCELED` CQE. -- **I6 (bad-disk isolation)**: A bad disk's in-flight I/O does not - permanently block good disks' I/O on a shared pipeline. - `cancel_fd(fd)` cancels all in-flight ops on the bad fd with one - SQE, freeing SQ slots immediately. -- **I7 (partial write is error)**: A short write returns - `IoError::PartialWrite` immediately. No internal retry. The caller - decides the next action. -- **I8 (partial-block preservation)**: Sequential writes to the same physical - block complete in submission order. An unaligned continuation merges with - the cached acknowledged block image. Each uncached partial boundary is read - before the request bytes are overlaid. Alignment preserves all bytes outside - the requested range, including tails after aligned starts and late retries. -- **I9 (backend alignment)**: Every backend write to a disk with block size - greater than one has aligned address, offset, and length. - -- **I10 (strip-block write ordering)**: Only one write frame may be in flight - for a strip's disk block. Its successful completion precedes submission of - the next frame, regardless of alignment. Parallel writes require independent - strip blocks or different disks holding shards or replicas. - -## 11. Configuration - -diskio accepts an optional typed TOML file through `--config`; explicit CLI -options override file values according to the shared -[`service configuration design`](../config/design-crowdb-config.md). Its -sections are: - -- **`[server]`** — bind address, listen port, inbound RPC workers (default 4), - node ID, dummy-disk type, `O_DIRECT`, and optional fault injection. Worker - count must be positive and changes require restart. -- **`[engine]`** — blocking thread-pool size and io_uring SQ entry count. The - backend remains auto-detected: `UringEngine` when available and - `BlockingEngine` otherwise. -- **`[group0]`** — KV management seeds, instance/rack/disk-group identity, - synchronization interval, and disk auto-discovery. -- **`[metrics]`** — log directory and collection interval. -- **`[[disk]]`** — ordered disk ID, path, and single-zone capacity entries. - An empty path selects a `NullDisk` or `MemDisk`. - -Fault latency requires both minimum and maximum values with minimum not above -maximum. Fault error rate is in `0.0..1.0`. Loading is transactional and the -complete candidate is validated before the RPC listener or I/O engine starts. - -The server registers with the group-0 service registry on startup, -reporting the diskio service is alive. Other services use this for -health detection. - -## 12. References - -- [`design-crowdb-diskdb.md`](../diskdb/design-crowdb-diskdb.md) §2 (Non-Goals: "No - data I/O") — diskdb allocates blocks; diskio reads/writes contents. -- [`design-crowdb-chunkdb.md`](../chunkdb/design-crowdb-chunkdb.md) §5.1 - (Disk block — `Segment { node_id, disk_id, zone_index, - zone_offset, size, tag }`) — the addressing unit for diskio. -- [`../rpc/design-crowdb-rpc.md`](../rpc/design-crowdb-rpc.md) §2 (Key - Design Decisions: control + data separation, native buffer, C ABI + - oneshot FFI) — the RPC engine diskio builds on. -- [`design-crowdb-tree.md`](../tree/design-crowdb-tree.md) — the btree - page store that shares `DiskIOUring`. diff --git a/doc/design/kv/design-crowdb-paxos-kv-group0.md b/doc/design/kv/design-crowdb-paxos-kv-group0.md deleted file mode 100644 index 5656af0c6..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-group0.md +++ /dev/null @@ -1,551 +0,0 @@ - - - -# CROWDB - Design: System Group (Group 0) - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3.3 -[`design-crowdb-paxos-kv-server.md`](design-crowdb-paxos-kv-server.md), -[`design-crowdb-paxos-kv-reconfiguration.md`](design-crowdb-paxos-kv-reconfiguration.md) -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3.3 - ---- - -## Table of Contents - -- [1. Overview](#1-overview) -- [2. Design Decisions](#2-design-decisions) - - [2.1 External and in-process group-0 API surfaces](#21-external-and-in-process-group-0-api-surfaces) - - [2.2 `crowdb-paxos-kv-server` mgmt API is internal](#22-crowdb-paxos-kv-server-mgmt-api-is-internal) - - [2.3 Unified key concept with two encodings](#23-unified-key-concept-with-two-encodings) - - [2.4 All cross-component protocol types live in `crowdb-protocol`](#24-all-cross-component-protocol-types-live-in-crowdb-protocol) - - [2.5 ID types defined in `crowdb-protocol`](#25-id-types-defined-in-crowdb-protocol) - - [2.6 Two monitoring models: push (services) and pull (infrastructure)](#26-two-monitoring-models-push-services-and-pull-infrastructure) - - [2.7 kv-server keep-alive to group 0 (revised)](#27-kv-server-keep-alive-to-group-0-revised) - - [2.8 Hardware admin via kv-client (no admin crowdb-rpc service)](#28-hardware-admin-via-kv-client-no-admin-crowdb-rpc-service) -- [3. Group-0 Sysdata Schema](#3-group-0-sysdata-schema) - - [3.1 Key layout (text-path encoding)](#31-key-layout-text-path-encoding) - - [3.2 Text magic namespaces](#32-text-magic-namespaces) - - [3.3 Value encoding](#33-value-encoding) - - [3.4 Scan patterns](#34-scan-patterns) - - [3.5 `DiskGroupId` widening: u32 → u64](#35-diskgroupid-widening-u32--u64) - - [3.6 `dc_id` removal](#36-dc_id-removal) -- [4. Service Registry](#4-service-registry) - - [4.1 Registration and keep-alive](#41-registration-and-keep-alive) - - [4.2 Services registered](#42-services-registered) - - [4.3 Liveness and expiry](#43-liveness-and-expiry) -- [5. Bootstrap and Cutover](#5-bootstrap-and-cutover) - - [5.1 Two-phase bootstrap](#51-two-phase-bootstrap) - - [5.2 No `/topology/ready` flag](#52-no-topologyready-flag) - - [5.3 Greenfield migration](#53-greenfield-migration) - - [5.4 Leader readiness before writing](#54-leader-readiness-before-writing) -- [6. Relationship to Existing Design Docs](#6-relationship-to-existing-design-docs) -- [7. Cascading cleanup](#7-cascading-cleanup) - ---- - -## 1. Overview - -The **system group** (store 0, group 0) is a designated Paxos group -that stores cluster-wide system data ("sysdata") as regular KV entries. -Because it is a Paxos group, sysdata is replicated, consistent, and -highly available by the same mechanism that protects user data. No -external coordinator is needed. - -Group 0 holds several categories of sysdata: - -- **Hardware hierarchy** — racks, nodes, disk-groups, disks. -- **Per-disk-group maps** — ownership map (which diskdb instance owns - a disk-group), binding map (which paxos data group a disk-group's - zone records live on). -- **KV-cluster topology** — stores, groups, replicas (the persistent - record of the KV cluster's structure, for disaster recovery). -- **Service registry** — live service instances (diskdb, kv-server, - future services) with their endpoints and heartbeat timestamps. - -This document defines the architecture of group 0: who owns the -schema, who writes what, how services register and keep alive, and -how the circular-dependency between kv-server and group 0 is handled. - ---- - -## 2. Design Decisions - -### 2.1 External and in-process group-0 API surfaces - -External processes use `crowdb-paxos-kv-client` for group-0 sysdata. Code already -running inside `crowdb-paxos-kv-server` uses `KvGroupOperations` and the fixed -`Group0ControlPlane` facade instead of opening a loopback RPC connection to -itself. RPC handlers also delegate to `KvGroupOperations`, so internal and -external paths share the same read fences, conditional-write semantics, and -error mapping without coupling a monitor to the RPC transport. - -`crowdb-paxos-kv-client` provides service classes wrapping a client pinned to group -0: - -- **`HardwareClient`** — hardware hierarchy + per-disk-group maps - (rack/node/disk-group/disk CRUD, ownership map, binding map). -- **`ServiceRegistryClient`** — service instance registry + keep-alive - (register, heartbeat, unregister, discover). -- **`PaxosKvClusterMetaClient`** — KV-cluster topology records - (store/group/replica metadata read/write). -- **`PaxosKvClusterAdmin`** — control-plane surface for cluster management - (lifecycle HTTP calls to kv-server + delegates metadata writes to - `PaxosKvClusterMetaClient`). Contains a `PaxosKvClusterMetaClient` internally. - -Values are small and JSON-encoded. Conditional mutations use revision CAS -where concurrent ownership or lifecycle changes must be preserved. Disk status -setters require an existing record, read it linearly and compare its revision -when writing the new status; missing disks are never recreated by a status -update. A matching status is a read-only no-op. Conflicts and unknown outcomes -require reconciliation before retry. See §3 for the key/value schema. - -### 2.2 `crowdb-paxos-kv-server` mgmt API is internal - -The kv-server HTTP management API (lifecycle endpoints: `add_store`, -`add_group`, `add_remote_replicas`, `step_down`, `system_init`) -stays in kv-server. These create/destroy actual `PxKvStore`/`PxGroup` -objects with WALs and election drivers, which cannot move to a client. -But the API is now **internal**: only `crowdb-paxos-kv-client`'s -`PaxosKvClusterAdmin` calls it. All other code (`crowdb-web`, `crowdb-cli`) -goes through `PaxosKvClusterAdmin`. - -Removed from kv-server: `POST /topology/finalize`, -`GET /topology/ready`. Persistent topology-record management moves -to `PaxosKvClusterMetaClient` / `HardwareClient` in `crowdb-paxos-kv-client`. - -Retained in kv-server (internal): lifecycle endpoints, `GET /topology` -(live runtime state export), `GET /health`, `GET /metrics`. - -### 2.3 Unified key concept with two encodings - -CROWDB keys use a single key concept (struct + fields) with two encoding -traits: `BinaryKey` (binary, for data groups) and `TextKey` -(text-path, for group 0). The encoding protocol (rules, frozen -layouts, evolution policy) is defined in -[`design-crowdb-protocol-key.md`](../protocol/design-crowdb-protocol-key.md) §5. Group 0 -uses text keys + JSON values; the full group-0 key/value schema is in -§3 below. - -### 2.4 All cross-component protocol types live in `crowdb-protocol` - -`crowdb-protocol` is the single home for all cross-component protocol -types: data structures, RPC messages, key types, and HTTP mgmt API -contracts. When `ServerClient`'s mgmt methods are absorbed into -`PaxosKvClusterAdmin`, the -HTTP request/response types (`AddStoreRequest`, `AddGroupRequest`, -`RemoteReplicaInfo`, `StepDownRequest`, `StoreSummary`, -`TopologyResponse`) move to `crowdb-protocol` under a new `mgmt` -module. - -### 2.5 ID types defined in `crowdb-protocol` - -All identifier types are defined in `crowdb-protocol` and used -consistently across all crates (console config, kv-client, kv-server, -diskdb, chunkdb). No crate uses `String` for an ID that is -fundamentally numeric. ID aliases are defined **once** in -`crowdb-protocol` (`RackId`, `NodeId`, `DiskGroupId`, `StoreId`, -`GroupId`, `ReplicaId`, `InstanceId`) and imported by every other -crate, with no per-crate redefinition. Console config -(`crowdb-console-shared`) widens every rack-id and node-id reference to -`RackId`/`NodeId` (not just `RackEntry.id`/`NodeEntry.id`/ -`NodeEntry.rack_id`, also `ServerEntry.node_id`, `StoreEntry.nodes`, -`ReplicaEntry.node_id`, and the persisted BTreeMap keys). The -console-side `ServerEntry.id` label stays `String` (it is a console -handle, not a numeric cluster ID). - -- **`RackId`** — `u64`. Console config today uses `String` for - `RackEntry.id`; the values are numeric strings (e.g. "1"). Console - config schema changes to `u64`; `http_cluster_init` passes the value - directly to `HardwareClient.add_rack(rack_id: u64)`. -- **`NodeId`** — `u64`. Console config today uses `String` for - `NodeEntry.id`; the values are numeric strings. Console config - schema changes to `u64`. -- **`DiskGroupId`** — `u64`. (This is a widening; see §3.5.) -- **`StoreId`** — `u64`. Already u64 in kv-server and console config. -- **`GroupId`** — `u64`. Already u64 in kv-server and console config. -- **`ReplicaId`** — `u64`. Already u64 in kv-server and console config. -- **`InstanceId`** — `u64`. Service instance identifier (diskdb - instance, kv-server instance). -- **`DiskId`** — 128-bit: two `u64` (`high`, `low`). Already defined - as a flatbuffer message in `common_type.fbs`. Globally unique. -- **`ChunkId`** — 192-bit: three `u64` (`high`, `mid`, `low`). - Already defined as a flatbuffer message in `common_type.fbs`. - -The simple integer IDs (`RackId`, `NodeId`, `DiskGroupId`, `StoreId`, -`GroupId`, `ReplicaId`, `InstanceId`) are type aliases (`pub type -RackId = u64;`) in `crowdb-protocol`, not newtypes. They exist for -documentation and API clarity (function signatures read -`rack_id: RackId` rather than `rack_id: u64`), not for type-safety -enforcement. The composite IDs (`DiskId`, `ChunkId`) are already -flatbuffer structs. - -### 2.6 Two monitoring models: push (services) and pull (infrastructure) - -There are two distinct monitoring models in the cluster, determined -by the relationship to group 0: - -**Push model (service keep-alive):** Application services (diskdb, -chunkdb, future services) are **guests** on the platform. They -register themselves in group 0 under `/srv//` -and heartbeat periodically. Other components discover them by -scanning group 0. This works because the service is not the host of -group 0. It's a client. - -**Pull model (infrastructure health):** The console (crowdb-web) polls -kv-server nodes via `GET /health` and `GET /topology` to observe -runtime state. This is the existing on-demand pattern; it can be -extended to periodic polling. - -### 2.7 kv-server keep-alive to group 0 (revised) - -**Yes, kv-server instances do keep-alive to group 0**, same push -model as application services. Each kv-server instance registers -under `/srv/kv-server/` and heartbeats with its hosted -stores/groups and health status. This is implemented (a -background keep-alive loop in `crowdb-paxos-kv-server` writing via -`ServiceRegistryClient`); it is not deferred to a follow-up. - -**Circular-dependency analysis:** The concern is that group 0 is -hosted by kv-server, so kv-server writing to group 0 is "writing to -itself." This is not actually harmful: - -- **Group 0 is replicated.** If one node hosting a group-0 replica - goes down, the remaining replicas maintain quorum. The down node's - heartbeat expires, which is the desired behavior. -- **kv-server is already a client of group 0.** `PaxosKvClusterMetaClient` - writes topology records to group 0; `PaxosKvClusterAdmin` delegates to - it. A heartbeat is just another client write. -- **If ALL group-0 replicas are down, the cluster is down.** Heartbeats - don't matter in that scenario. No component can read group 0 - anyway. -- **A node hosting group 0 also hosts other groups.** When it goes - down, its heartbeat expires (visibility), and each of its groups - elects a new leader independently (Paxos failover). The heartbeat - doesn't trigger failover; it provides visibility into which nodes - are alive and what they host. - -The keep-alive pattern for kv-server: - -``` -/srv/kv-server/ -> KvServerInstanceValue { - instance_id: u64, - mgmt_endpoint: String, - hosted_stores: Vec, - hosted_groups: Vec<(u64, u64)>, // (store_id, group_id) - health: HealthSummary, // aggregate of per-store/per-group status - last_heartbeat_ms: u64, -} -``` - -Other components (console, diskdb) can scan `/srv/kv-server/` to -discover live kv-server instances and their health. This complements -the pull model — push provides self-registration and discovery; pull -provides detailed runtime state. - -**The group-0 leader writing its own heartbeat is fine.** If the -group-0 leader is also a kv-server instance (which it usually is), it -writes its heartbeat to group 0 locally (as a blind put to its own -group). This is no more circular than a kv-server instance writing -user data to a group it hosts. It's a normal client write that goes -through the Paxos consensus path. - -### 2.8 Hardware admin via kv-client (no admin crowdb-rpc service) - -Hardware admin operations (add/remove rack/node/disk-group/disk, -set `*_status`) are writes to group-0 sysdata. They are performed -through `HardwareClient` in `crowdb-paxos-kv-client`, invoked by the console -(`crowdb-web` / `crowdb-cli`). There is **no `DiskdbAdminService` crowdb-rpc -surface**: the previous `diskdb_sys_service.fbs` / -`diskdb_sys_op.fbs` admin RPCs (`AddRack`, `SetDiskStatus`, -`FetchHardware`, `Keepalive`) are removed. `FetchHardware` is -replaced by `HardwareClient` prefix scans, `Keepalive` by -`ServiceRegistryClient.heartbeat`, and the add/remove/status ops by -`HardwareClient` methods. The diskdb server serves only -`DiskdbService` (allocate/free; stubbed `Unimplemented`) and -reads hardware state from group 0 via `HardwareClient` in its sync -loop. It does not own or serve hardware admin. - ---- - -## 3. Group-0 Sysdata Schema - -### 3.1 Key layout (text-path encoding) - -``` -# Service registry — scan all instances of a service -/srv/diskdb/ -> InstanceValue -/srv/kv-server/ -> KvServerInstanceValue -# Future: /srv/chunkdb/, /srv// - -# KV-cluster topology — persistent records for disaster recovery -/kv/store/ -> StoreValue -/kv/group// -> GroupValue -/kv/replica/// -> ReplicaValue - -# Hardware hierarchy — full path in each key for scan narrowing -/hw/rack/ -> RackValue -/hw/node// -> NodeValue -/hw/dg/// -> DiskGroupValue -/hw/disk//// -> DiskValue -# disk_id_hex = 32-char hex (128-bit DiskId, high:low, lowercase) - -# Per-disk-group maps — same hierarchy path as the dg they refer to -/hw/dg_owner/// -> DiskdbOwnerEntry -/hw/dg_bind/// -> KVGroupBindEntry -``` - -### 3.2 Text magic namespaces - -- `/hw` — hardware namespace (rack/node/disk-group/disk + maps). -- `/srv` — service registry namespace (all service instances). -- `/kv` — KV-cluster topology namespace (store/group/replica records). - -The encoding rationale (text vs binary magic independence, per- -namespace choice) is in -[`design-crowdb-protocol-key.md`](../protocol/design-crowdb-protocol-key.md) §5. - -### 3.3 Value encoding - -All group-0 values are JSON-encoded (`serde_json::to_vec` / -`serde_json::from_slice`). Value types live in `crowdb-protocol` (the -single home for cross-component data structures) and are used by -`crowdb-paxos-kv-client` directly, with no per-crate redefinition: - -- **Existing flatbuffer `*Value` types** (`RackValue`, `NodeValue`, - `DiskGroupValue`, `DiskValue`, `HwStatus`, `DiskId`) — used directly. - `crowdb-protocol`'s `build.rs` already derives - `serde::Serialize`/`Deserialize` on them. -- **New group-0 sysdata value types** added as flatbuffer messages in - `crowdb-protocol` (parallel to the hardware values): `StoreValue`, - `GroupValue`, `ReplicaValue`, `InstanceValue` (service-registry - value, generic across services), `OwnerMapValue` - (`{ instance_id, lease_expiry_ms }`), `BindMapValue` - (`{ store_id, group_id }`). `build.rs` derives serde on each. -- **Entry return types** (key fields parsed from the path + value, - produced by `HardwareClient`/`PaxosKvClusterMetaClient` reads) are plain - `#[derive(serde)]` Rust structs in `crowdb-protocol`: `DiskGroupEntry` - (`{ rack_id, node_id, dg_id, value: DiskGroupValue }`), - `DiskdbOwnerEntry` (`{ rack_id, node_id, dg_id, instance_id, - lease_expiry_ms }`), `KVGroupBindEntry` (`{ rack_id, node_id, dg_id, - store_id, group_id }`). These are not stored as-is; they are the - decoded form of a path + its JSON value. - -### 3.4 Scan patterns - -- `/hw/rack/` → all racks. -- `/hw/node/` → all nodes; `/hw/node//` → nodes in one rack. -- `/hw/dg/` → all disk-groups; `/hw/dg//` → disk-groups in - one rack; `/hw/dg///` → disk-groups of one node. -- `/hw/disk/` → all disks; narrowing by rack/node/disk-group. -- `/hw/dg_owner/` → entire ownership map; narrowing by rack/node. -- `/hw/dg_bind/` → entire binding map; narrowing by rack/node. -- `/srv/diskdb/` → all live diskdb instances. -- `/srv/kv-server/` → all live kv-server instances. -- `/kv/store/` → all stores; `/kv/group//` → groups in one - store; `/kv/replica///` → replicas in one group. - -### 3.5 `DiskGroupId` widening: u32 → u64 - -Key structs (`DiskGroupKey`, `DiskKey`, `OwnerMapKey`, -`BindMapKey`) use `u32` for `disk_group_id`. The flatbuffer type -`NodeValue` also uses `u32` for `disk_group_ids` (`repeated uint32`) -and `last_used_dg_id` (`uint32`). (`DiskGroupValue` has no -`disk_group_id` field; the id is in the key.) This requirement -widens `DiskGroupId` to `u64` for consistency with all other integer -IDs and to remove the artificial 4-billion limit. The `NodeValue` -flatbuffer fields change from `uint32` to `uint64`; the key struct fields -change from `u32` to `u64`. The `disk_group_id` fields in -`diskdb_op.fbs` (`AllocateBlocksRequest`, `QueryCapacityStatsRequest`, -`GetDiskGroupInfoRequest`, `GetDiskInfoRequest`) and the Info response -types (`NodeInfo`, `DiskInfo`, `DiskGroupInfo`) widen from `uint32` to -`uint64` as well. This is a breaking schema change, acceptable because -diskdb is greenfield. - -### 3.6 `dc_id` removal - -The schema drops `dc_id` from `RackKey { dc_id, rack_id }` (struct + -binary layout), `RackInfo`, `NodeInfo`, and `NodeValue.dc_id` (v1 -ships flat, no DC layer). The text-path schema has no `dc_id` -(`/hw/rack/`). The binary `RackKey` layout changes -accordingly (greenfield). - ---- - -## 4. Service Registry - -### 4.1 Registration and keep-alive - -Every long-running service instance registers itself in group 0 -under `/srv//` and heartbeats periodically. -`ServiceRegistryClient` in `crowdb-paxos-kv-client` is **generic across -services** (diskdb and kv-server now; chunkdb and future services -later). It takes a `service` name that selects the path namespace -(`/srv/diskdb/`, `/srv/kv-server/`, …) and the value shape. The API: - -- `register(service, instance_id, value: &InstanceValue) -> Result<()>` -- `heartbeat(service, instance_id, value: &InstanceValue) -> Result<()>` -- `unregister(service, instance_id) -> Result<()>` (clean shutdown) -- `read_instance(service, instance_id) -> Result>` -- `read_all_instances(service) -> Result>` (prefix scan) - -`InstanceValue` is the generic service-registry value -(`{ instance_id, rpc_endpoint, last_heartbeat_ms, extra: ServiceExtra }`). -`ServiceExtra` is a per-service enum (diskdb carries `owned_dg_ids: -Vec`; kv-server carries `hosted_stores`, `hosted_groups`, -`health`). Diskdb convenience wrappers -(`register_diskdb`/`heartbeat_diskdb`/`read_all_diskdb_instances`) and -kv-server wrappers delegate to the generic methods. - -DiskIO physical identity and configured hardware scope are immutable once -accepted. Physical identity is pinned with instance-revision CAS before scope -publication. The first configured scope requires an existing Rack record and -is published atomically with an advance of that Rack's revision, fencing -concurrent disk membership changes. Revalidate the instance after reading the -Rack revision so another first-scope publication cannot be overwritten. If the -Rack is absent, retain the physical pin and reject scope publication; an -instance-only write cannot substitute for the Rack fence. Ordinary heartbeats -with the same accepted scope update liveness without advancing the Rack. - -### 4.2 Services registered - -- **diskdb** — `/srv/diskdb/`. Heartbeat includes - `owned_dg_ids` (which disk-groups this instance owns). Used by the - diskdb sync loop and by a future notify mechanism for instance discovery. -- **kv-server** — `/srv/kv-server/`. Heartbeat includes - `hosted_stores`, `hosted_groups`, and aggregate health. Used by - the console and other components for kv-server discovery and - health visibility. **Implemented** (a background keep-alive - loop in `crowdb-paxos-kv-server` writing via `ServiceRegistryClient`). -- **Future services** (chunkdb, etc.) — same pattern under - `/srv//`. - -### 4.3 Liveness and expiry - -A service instance is considered live if its `last_heartbeat_ms` is -within a configurable TTL (default: 3× heartbeat interval). Readers -filter expired entries when scanning. There is no active eviction. -Expired entries are ignored and eventually overwritten or deleted on -clean shutdown. - -### 4.4 Client discovery - -`ServiceDiscoveryClient` (in `crowdb-paxos-kv-client`) wraps -`ServiceRegistryClient` with a per-service state map and TTL-based -refresh. Each state contains an atomically published cached result, an atomic -round-robin cursor, and an async single-flight refresh gate. The sharded-map -guard is dropped immediately after cloning the state and never crosses I/O. -Clients call `discover_all(service)` or -`discover_one(service)` to find living service instances by service -name without hardcoding addresses. The cache is poll-on-demand: the -first call queries group-0, subsequent calls within the cache TTL -(default 5 seconds — one heartbeat cycle) return the cached result. -`invalidate(service)` forces a re-query on the next call; called -after known topology changes (e.g. `cluster_init`, `deploy_diskdb`). - -Simultaneous expiry for one service produces one group-0 refresh; waiters -recheck the freshly published result after acquiring the gate. Instance -selection uses the service state's atomic cursor to round-robin among live -instances. The -`DiskdbClient` and `ChunkdbClient` retain their per-disk-group / -per-range routing caches (they have routing requirements that -round-robin doesn't cover); the generic discovery client handles the -"pick any living instance" case. - -The bootstrap flow: a client knows group-0's mgmt port (10000, from -R118) → constructs `PaxosKvClient` with that seed → -`ServiceDiscoveryClient::from_shared_kv` → `discover_all("diskdb")` -queries `/srv/diskdb/` via group-0 → cache → client connects to the -discovered RPC endpoint. No per-service address configuration needed -beyond group-0's bootstrap address. - -Explicit address overrides (`--diskdb-ip`, `--chunkdb-ip` on the -CLI; `Option<&str>` explicit endpoint on ops functions) bypass -discovery for that one call. This is the test exception (unit tests -pass addresses directly) and the operator escape hatch. When the -registry is empty and no override is provided, the client fails with -a clear "no living instances" error. - ---- - -## 5. Bootstrap and Cutover - -### 5.1 Two-phase bootstrap - -Phase 1 uses console TOML as the topology source of truth -(operator-managed). Phase 2 cuts over to group 0 authoritative: -`http_cluster_init` writes hardware records via `HardwareClient` and -KV-cluster topology records via `PaxosKvClusterMetaClient` into group 0, -and `crowdb-paxos-kv-server` restores its stores/groups from local disk on -restart with group 0 as the verification and fallback source for -remote-replica wiring (see -[`design-crowdb-paxos-kv-server.md`](design-crowdb-paxos-kv-server.md) §2.2). The -topology content in TOML is bootstrap-only: `--root` is required on every -start, and restart with `--root` alone rebuilds every local store/group from -WAL and rejoins the cluster. An optional `--config` remains the local authority -for process tunables in both modes; deleting it intentionally restores typed -defaults. Group 0 does not publish process configuration. The node -root is persisted to group 0 via `KvServerExtra.data_root` on each -keep-alive tick so the cluster/console knows each node's data -location. - -### 5.2 No `/topology/ready` flag - -The old schema had a `/topology/ready` flag key indicating group 0 -is authoritative. The text-path schema has no equivalent. v1 does -not need it. diskdb's sync loop treats an empty group 0 as "nothing -assigned yet" and retries. If a readiness signal is needed later, -add a derived condition (e.g. "group 0 has ≥1 `/hw/node/` key"). - -### 5.3 Greenfield migration - -diskdb is greenfield (no production diskdb). The -old `/topology/...` records are replaced by `/hw/...` and -`/kv/...`. Treat as greenfield: require a fresh cluster init. Old -`/topology/...` keys are orphaned harmlessly. - -### 5.4 Leader readiness before writing - -`HardwareClient` and `PaxosKvClusterMetaClient` write via crowdb-rpc to the -group-0 leader. `http_cluster_init` can only write sysdata after a -group-0 leader is elected and reachable. For single-node init this -is immediate (self-elect). For multi-node, the init flow must wait -on leader election before writing. Add a readiness poll if the -existing flow doesn't already wait. - ---- - -## 6. Relationship to Existing Design Docs - -- **`design-crowdb-paxos-kv.md` §3.3** — defines the system group concept. - This doc expands it with the sysdata schema, service registry, and - keep-alive model. -- **`design-crowdb-paxos-kv-server.md` §2.4** — defines the HTTP management - API. This doc clarifies that the API is internal (only - `crowdb-paxos-kv-client` calls it) and that `topology_finalize` / - `topology_ready` are removed. -- **`design-crowdb-paxos-kv-reconfiguration.md`** — group 0 membership - evolves using the shipped Model B reconfiguration. No new consensus - primitive required. -- **`design-crowdb-protocol-key.md` §5** — documents the unified key concept - (one struct, two encoding traits: `BinaryKey` + `TextKey`). - -## 7. Cascading cleanup - -`HardwareClient` provides `remove_*_cascade` methods that delete an -entity and all its children in group 0: `remove_rack_cascade` → -nodes → disk-groups → disks + owner/bind/usage records. The -single-record `remove_*` methods are kept for internal use. - -- **I1 — ID reuse safety**: After a rack/node/disk-group/disk is - removed, its ID can be safely reused. The group-0 sysdata deletion - is the fence against resurrection. - -Deployment nodes use the [deployment architecture](../deploy/design-crowdb-deploy.md) -for discovery, numeric ID allocation, SSH admission and system-resource ownership. -The KV server durably accepts one `SystemBootstrapIdentity` before store 0 creation; -identified initialization and snapshot join share this exclusion. Cleanup retires -the operation before deleting store 0 and releasing bindings, fencing delayed -commands. Published topology and node/service operations require current Group 0 -authority. Its unavailability is an error, never an empty ownership set. Other data -groups retain their own Paxos authority; ownership-dependent services obey their -existing expiry/fencing rules. diff --git a/doc/design/kv/design-crowdb-paxos-kv-leader-election.md b/doc/design/kv/design-crowdb-paxos-kv-leader-election.md deleted file mode 100644 index c0e5f0a14..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-leader-election.md +++ /dev/null @@ -1,373 +0,0 @@ - - - -# CROWDB - Design: Leader Election, Term, and Lease - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3, §4.2, §6.2, §7 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3, §4.2, §6.2, §7 - -This document specifies leader election, term management, the `PxBallot`/`PxTerm` separation, and the leader lease used for fast linearizable reads. The design follows Raft very closely; only the per-slot Paxos parts differ. - -## Table of Contents - -- [1. Why Raft-Style Election on a Paxos Log](#1-why-raft-style-election-on-a-paxos-log) -- [2. PxTerm vs PxBallot](#2-pxterm-vs-pxballot) -- [3. Election Protocol](#3-election-protocol) -- [4. New-Leader Bulk Phase 1](#4-new-leader-bulk-phase-1) -- [5. Heartbeats and Liveness](#5-heartbeats-and-liveness) -- [6. Leader Lease](#6-leader-lease) -- [7. ReadIndex Fallback](#7-readindex-fallback) -- [8. Step-Down Triggers](#8-step-down-triggers) -- [9. Safety Argument](#9-safety-argument) -- [10. Tunables and Defaults](#10-tunables-and-defaults) - ---- - -## 1. Why Raft-Style Election on a Paxos Log - -Classical Multi-Paxos elects a "distinguished proposer" through Paxos itself: an aspiring leader runs Phase 1 over the entire log. This works but conflates election with proposal and complicates clean step-down, lease management, and observability. - -Raft's contribution was to factor election out of the log — a separate election RPC, a per-term vote, randomized timeouts. CROWDB adopts that factoring exactly, and lets per-slot Paxos handle just the per-slot work. The separation maps onto two distinct identifiers: `PxTerm` for elections, `PxBallot` for slot proposals. - -This design choice is consistent with *How to Build a Highly Available System Using Consensus* (Lampson, 1996) and is the same pattern used in Spinnaker, Spanner's Paxos groups, and TiKV's region leadership. - ---- - -## 2. PxTerm vs PxBallot - -The two identifiers serve disjoint purposes; conflating them is a common bug source. - -### 2.1 `PxTerm` - -- Type: monotonic non-negative integer per group. -- Persistence: durable. Every term increment is fsynced before being acted on. -- Scope: one term covers all writes by a single leader, across many slots. -- Used by: - - Election RPCs (`RequestVote`, `Vote`). - - Heartbeats and `Accept` messages (carried for fencing). - - Lease validity (a lease is associated with a term). - -### 2.2 `PxBallot` - -- Type: lexicographically ordered pair `(round, leader_id)`. -- Persistence: durable on every acceptor (in WAL with the `PxLogEntry`). -- Scope: one ballot covers proposals at a single slot. Different slots may simultaneously be at different ballots. -- Used by: - - Phase 1 (`Prepare`) and Phase 2 (`Accept`) messages of classical Paxos. - - Repair: a higher round at the same leader_id supersedes a lower round at that slot. - -### 2.3 The bridge - -A new leader at term `T` proposes new slots with ballot `(round=0, leader_id=me)`. The implicit ordering is "this leader's ballots are 'newer' than any leader before it" because acceptors fence on `term` first: an `Accept` carrying `term=T` is rejected by any acceptor whose `current_term > T` regardless of ballot. Conversely a same-term `Accept` is compared by ballot rules. - -This means: - -- Acceptors keep `(current_term, slot_state[slot])` where `slot_state` records `(promised_ballot, accepted_ballot, accepted_value)`. -- Election bumps `current_term`, invalidating any in-flight `Accept` from older terms. -- Repair within a term bumps `round` for a single slot, invalidating any in-flight `Accept` for that slot only. - -Both fences are necessary, and they are independent. - ---- - -## 3. Election Protocol - -The protocol matches Raft's leader election with adaptations for the bulk Phase-1 step (§4). - -### 3.1 Roles - -Each member is exactly one of: - -- **Follower** — accepts heartbeats and `Accept`s from a leader. -- **PreCandidate** — running a `PreVote` round (see §3.2a); has not yet bumped `current_term`. -- **Candidate** — has bumped `current_term` and is collecting votes. -- **Leader** — won the latest election; serves writes. - -### 3.2 Election trigger - -A follower starts an election when its **election timer** expires. The timer is reset on every legitimate heartbeat or `Accept` from the current leader. The timeout is randomized in `[election_min, election_max]` (defaults 4000 ms – 8000 ms; see §10), avoiding split votes. - -When the timer fires: - -1. (PreVote, enabled by default) Follower transitions to PreCandidate and sends `PreVote(proposed_term, candidate_id, accepted_log_tip_slot, accepted_log_tip_term)` to all peers *without* bumping `current_term`. If a quorum grants, proceeds to step 2. If a peer reports a higher term, reverts to follower. If quorum is not reached, reverts to follower and waits for the next deadline. -2. Follower transitions to Candidate, bumps `current_term` to `proposed_term`, persists (`VoteGranted` WAL record). -3. Votes for itself. -4. Sends `RequestVote(term, candidate_id, accepted_log_tip_slot, accepted_log_tip_term)` to all peers. - -> **PreVote** (Raft optimization, ON by default): avoids spurious term increments from a partitioned-and-rejoined node. A node that was partitioned can rejoin and ask "would you vote for me?" without disrupting the current leader's term. - -### 3.3 Vote rules (matching Raft) - -A peer grants its vote iff: - -- The request's `term` is ≥ peer's `current_term`. If higher, peer adopts the new term and reverts to follower first. -- The peer has not already voted in this term, or already voted for the same candidate. -- The candidate's acceptor log is **at least as up-to-date** as the peer's, where up-to-date is the lexicographic comparison `(accepted_log_tip_term, accepted_log_tip_slot)`. -- The peer's `vote_lockout_until` has passed (no active lease promise from a heartbeat). - -The "log up-to-date" check is the safety property that prevents a stale leader from being elected. - -### 3.4 Outcome - -- **Win:** received votes from a quorum within the election timeout. Becomes leader; immediately runs the bulk Phase 1 (§4) and then sends initial heartbeats. -- **Loss / split:** sees a higher term, or election timer fires again with no quorum. Reverts to follower and waits for a new randomized timer. -- **Conversion:** if the candidate sees a `Heartbeat` or `Accept` with `term ≥ current_term`, it accepts that node as leader and reverts to follower. - ---- - -## 4. New-Leader Bulk Phase 1 - -A freshly elected leader does not immediately serve client writes. It first runs **one** Phase 1 over the open slot prefix to discover any in-flight values from the previous leader. - -### 4.1 What is the open prefix? - -When the leader takes office, it computes: - -- `floor = its own contiguous_chosen` (deliberately NOT maxed with peers' `contiguous_chosen`). After a restart, a replica can win election while missing committed slots it never received (the value lived only as a `ChosenNotice` watermark, which carries no payload). Maxing with peers' commit point would skip exactly those slots, leaving the new leader serving a stale value for a committed key. Sweeping from the leader's own frontier forces Phase 1 to re-derive every higher slot from the quorum. -- `ceiling = max(local highest_seen_slot, next_slot - 1, peers' highest_seen_slot from election replies)`. - -The open prefix is `[floor + 1, ceiling]`. If a previous leader had assigned slots beyond what any current acceptor knows about, those slots are simply not in this set; their values are unrecoverable but also un-acked, so no client expects them to exist. - -### 4.2 Bulk Prepare - -The leader runs Phase-1 `Prepare` **per slot** over `[floor+1, ceiling]`, batched by `bulk_prepare_window` (default 1024) with a `yield_now()` between batches to avoid starving other tasks. For each slot, `Prepare(ballot=(0, me), term=T)` is sent to all peers. Each acceptor responds with `Promise` carrying the highest `(accepted_ballot, accepted_value)` if any. - -The ballot used here is `(0, me)`. Because acceptors fence on `term` first, this ballot is "fresh" for the new term; we don't need a higher round. - -### 4.3 Adopt and re-Accept - -For each slot in the range: - -- If any peer's `Promise` returned an accepted value, choose the value with the highest `accepted_ballot` and re-Accept it at `(0, me)` under term T. -- If no peer returned a value, propose `NoOp` and re-Accept at `(0, me)` under term T. - -These re-Accepts are pipelined; the new leader does not wait for any one to be chosen before proposing the next. - -### 4.4 Steady state begins - -Once the bulk Phase 1 has been *issued* (not necessarily completed), the leader is free to start assigning new slots starting at `ceiling + 1`. The repair work for `[floor+1, ceiling]` proceeds in parallel and reuses the same machinery as routine gap repair (see [`design-crowdb-paxos-kv-slot.md`](design-crowdb-paxos-kv-slot.md) §9). - ---- - -## 5. Heartbeats and Liveness - -Heartbeats serve two purposes: liveness signaling (reset followers' election timers) and lease maintenance (extend the leader's lease). - -### 5.1 Heartbeat content - -Every heartbeat carries: - -- `term`, `leader_id`. -- `committed_safe_slot` (latest known safe-slot). -- `lease_grant_until` (monotonic-time deadline; see §6). -- `prev_log_slot, prev_log_term` (Raft-style consistency check). -- `t_send_ms_mono` (monotonic timestamp for lease grant calculation). - -`Chosen(slot)` notifications and `Accept`s are sent through separate `learner_stream` paths, not piggy-backed on heartbeats. - -### 5.2 Heartbeat cadence - -- Default `heartbeat_interval = 500 ms` (see §10). -- A follower's election timer must be ≫ `heartbeat_interval` to allow for occasional jitter; with defaults, election timeouts are 8–16× the heartbeat interval. - -### 5.3 Heartbeat response - -Followers respond with their own `term`, `success` (false if the follower's term is higher), `contiguous_chosen`, `last_chosen_term`, `contiguous_applied`, `highest_seen_slot`, and `durable_snapshot_slot`. The leader uses these to: - -Election term, vote, and lockout deadline remain one mutex-protected state -because transitions must publish them consistently. Observation-only paths use -the atomic current-term snapshot when staleness is safe; election decisions -take the full state lock. - -- Detect a stale leader's continued existence (if any response carries a higher term, the leader steps down (§8)). -- Maintain peer state (used by replicator and gap detection). -- Refresh the safe-slot computation. - -### 5.4 Follower catch-up and background apply - -When a follower's `contiguous_applied` lags behind the leader's `committed_safe_slot`, the heartbeat round drives catch-up in two phases: - -- **Phase 1 — `BatchChosenNotice` (value-less).** The leader sends a single fire-and-forget `BatchChosenNotification` frame over the peer RPC connection covering `[follower_applied+1, catchup_end]`, carrying only `(start_slot, end_slot, term, leader_id)`, no per-slot payload. The follower checks its local acceptor for each slot in the range: present slots advance the chosen frontier via `update_chosen_frontier`; missing slots remain gaps. This lets the follower's apply loop start processing present slots immediately, without waiting for the full-accept round-trip. -- **Phase 2 — full accepts.** The leader sends `send_accept` RPCs with full payloads for the same range. Slots the follower already has are re-accepted (idempotent CAS, cheap); slots the follower is missing get the real value. In steady state (no election churn) every slot is present, so Phase 1 does all the work and Phase 2 is a no-op. - -Engine apply is decoupled from the heartbeat handler via a **background apply loop**: - -- `handle_heartbeat` stores `committed_safe_slot` via `fetch_max` into `known_commit_slot` and signals `apply_notify`, then returns immediately with the current `contiguous_applied` (which may lag). The heartbeat reply is not blocked on engine apply. -- The background apply loop (lazily spawned, cancelled on shutdown) reads `known_commit_slot`, collects entries from the acceptor, and applies them via the learner. Missing slots are skipped (skip-and-continue) so a single gap doesn't block the entire apply backlog. `contiguous_applied` stays at the gap; subsequent available slots are applied out-of-order via `advance_applied_frontier`. -- Accept persists accepted state but does not advance the chosen or applied - frontier. A matching Chosen notification advances the chosen frontier and - wakes the background apply loop; missing or stale accepted values are - recorded as gaps for recovery. - ---- - -## 6. Leader Lease - -A lease lets the leader serve `Get(mode=Linearizable)` without a per-read quorum round-trip ([§6.1 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#61-linearizable-leader-read)). The lease is the standard Raft-style approach with a clock-skew bound. - -### 6.1 What the lease grants - -While its lease is valid, a leader may serve linearizable reads from local state under two assumptions: - -- No other node was elected leader during the lease window. -- The leader's `contiguous_applied` slot reflects all writes acked through this leader (which is true by Invariant I3 from [`design-crowdb-paxos-kv-slot.md`](design-crowdb-paxos-kv-slot.md) §2). - -The first assumption is enforced by: - -- Acceptors **promise not to vote for any other candidate** for at least `lease_duration` after granting a heartbeat (this is the Raft "PreVote + lease" pattern). -- The leader rejects both PreVote and RequestVote while its quorum-renewed - effective lease remains valid, even after the original self-vote lockout - expires. Renewing this protection uses the existing atomic lease deadline; - a lagging minority cannot recruit the healthy leader as its extra voter. -- The leader treats its lease as expired at `lease_grant_time + lease_duration - max_clock_skew` for safety (see §6.3). - -### 6.2 Lease grant and renewal - -Each heartbeat round-trip is also a lease grant: - -1. Leader sends heartbeat at monotonic time `T_send`. -2. Follower receives, records "I will not vote for any candidate before `T_recv + lease_duration`", and replies. -3. Leader receives the response at `T_recv_reply`. The lease is valid through `T_send + lease_duration` on the leader's clock. The leader uses `T_send` (not `T_recv_reply`) as the start so that any clock skew works in its favor. -4. The leader treats the lease as effective until `T_send + lease_duration - max_clock_skew`. This is conservative; it gives a safety margin equal to the assumed skew bound. - -With the default `heartbeat_interval = 500 ms` and `lease_duration = 9 × heartbeat_interval = 4500 ms` (see §10), the leader is essentially always within an active lease in steady state. A short network blip costs the leader its fast-read privilege but not its leadership. - -### 6.3 Clock-skew assumption - -design-crowdb-paxos-kv.md §3](design-crowdb-paxos-kv.md) caps clock skew at `max_clock_skew` (default `500 ms`, see §10) per heartbeat interval. The lease formula is: - -``` - effective_lease = lease_duration - max_clock_skew -``` - -Concretely with defaults: `4500 ms - 500 ms = 4000 ms` of "fast-read" coverage between successful heartbeat round-trips. If the round-trip exceeds the effective lease, the lease has *technically* expired and the leader downgrades to ReadIndex for the next linearizable read until the next successful heartbeat refreshes the lease. - -### 6.4 Why monotonic-only - -All lease math uses the **monotonic clock**, not wall-clock. Wall-clock can jump (NTP step, manual operator change); monotonic cannot. This is the same discipline as Raft's reference implementation. - -### 6.5 What goes wrong if lease misuse occurs - -If a leader serves a linearizable read after its lease has *truly* expired and a new leader has been elected, the read can return stale data. The read would not have observed a write committed by the new leader. This is the precise correctness reason for the conservative `effective_lease`. - -The fallback is ReadIndex, which gives the same correctness guarantee without the clock assumption. - ---- - -## 7. ReadIndex Fallback - -ReadIndex is the lease-free path for linearizable reads. Always available; used automatically when the lease is not effective; available to clients on a per-read basis. - -### 7.1 Procedure - -1. The leader records its current `contiguous_applied` slot, call it `R`. -2. The leader broadcasts a heartbeat to a quorum, awaits responses. -3. As soon as a quorum has responded *with the leader's own term*, the leader is confirmed to still be leader at this point in time. -4. The leader serves the read from local state, ensuring the engine's `contiguous_applied >= R`. - -The cost is one network round-trip per ReadIndex (or per batch of reads). No fsync, no extra log records. - -### 7.2 Batching - -Concurrent ReadIndex barriers coalesce onto a single in-flight heartbeat round so a burst of N expired-lease linearizable reads costs one quorum round-trip, not N. - -The leader holds a pending-barrier batch (`PxGroup::pending_read_barrier`) for the duration of one ReadIndex round: - -1. The first read to arrive (the *round leader*) captures `R = contiguous_chosen`, registers the batch, and runs `run_heartbeat_round`. -2. Reads that arrive while the round is in flight enqueue a `oneshot` waiter on the batch instead of starting their own round. -3. When the round completes, the round leader drains the batch and resolves every waiter with the same outcome: `Ready { read_slot: R }` on quorum ack, `NotLeader` on step-down (higher term), `NoQuorum` when quorum is unreachable. A waiter whose round leader is cancelled (dropped sender) receives `NoQuorum` and retries. -4. The lease fast path is unchanged: lease-valid reads still serve immediately without queueing. - -The mutex over the batch serializes enqueue and drain, so no waiter is lost. A read either joins the in-flight batch or, after the leader drains, starts a fresh batch. - -**Correctness** is identical to the single-read procedure in §7.1. The heartbeat quorum at the leader's term confirms no higher-term election displaced this leader during the round, so every committed write is reflected in the leader's local state. The engine get returns the *latest* applied value for the key (single-version, highest-slot-wins), not a value pinned to `R`; each batched read performs its own `engine_get_bytes` after the barrier resolves and observes the freshest local state at its serve time. The shared `R` reported in the response is therefore a conservative freshness floor (the pre-round `contiguous_chosen`), never an over-estimate. A write that commits *during* the round has slot > `R` and is returned by the engine get, so late-arriving batched reads are not stale. - -**Metric.** `read.barrier.l` count increments once per read that enters the ReadIndex barrier. For a batched burst of N reads coalesced onto a single heartbeat round, all N reads observe the same (amortized) barrier latency — `read.barrier.l` drops toward one RTT amortized across the batch. - -### 7.3 When to choose ReadIndex over lease - -Per-read settings: - -- **Default:** lease if effective, ReadIndex if not. -- **Force ReadIndex:** for environments without a clock-skew guarantee, or for workloads requiring belt-and-suspenders correctness. Higher latency but no clock dependence. - -The choice is exposed as a per-request option in the read RPC and as a default at the server config. - ---- - -## 8. Step-Down Triggers - -A leader steps down (transitions to follower) on any of: - -- **Higher term seen.** Any RPC response carrying `term > current_term` causes the leader to adopt the new term and revert to follower. -- **Lease unrenewable.** The leader has been unable to obtain a heartbeat-quorum response for longer than `lease_duration`. It cannot refresh its lease and assumes a partition or majority loss. -- **Admin step-down RPC.** Operator forces it for testing, planned maintenance, or rolling upgrade. -- **Removed from group.** Designed but not yet implemented (`ConfigChange` log entries are not yet supported). - -On step-down: - -1. Cancel the per-tenure `CancellationToken`, aborts in-flight bulk Phase 1 and any tenure-bound work. -2. Stop heartbeats (the leader-state loop returns on step-down). -3. Set `role = Follower`; adopt the higher term if the trigger was `HigherTerm`. -4. Expire `LeaseState` (`become_follower` clears leader id). -5. In-flight proposals see `NotLeader` via the propose leadership gate (proposing_term check). - ---- - -## 9. Safety Argument - -The election + bulk-Phase-1 + lease design must satisfy three properties. - -### 9.1 At most one leader per term per group - -Standard Raft argument: - -- A candidate needs majority votes within a term. -- A peer votes at most once per term. -- Two majorities over the same membership intersect (by quorum overlap). -- Therefore at most one candidate can collect a majority in any one term. - -CROWDB inherits this directly. - -### 9.2 No chosen value is lost across leader change - -By Paxos's classical safety: if a value `v` was *chosen* at slot `s` (≥ majority of acceptors `Accepted` it), then any future Phase-1 over slot `s` will see `v` from at least one acceptor and re-propose it. - -The new-leader bulk Phase-1 (§4) executes this exact procedure over the open prefix, so any chosen value is preserved. - -A value that was `Accepted` by some but not a majority of acceptors may be either re-chosen or lost, but it was never acknowledged to the client (Invariant I3); the client knows its outcome is unknown and retries. - -### 9.3 No stale leader serves a stale linearizable read - -Two enforcement mechanisms run in series: - -- **Term fencing:** every RPC carries `term`. Any acceptor with a higher term refuses the request and informs the sender, which steps down. -- **Lease conservatism:** even when no fencing has happened, the leader self-expires its lease at `effective_lease = lease_duration - max_clock_skew`. If the clock-skew assumption holds, no other leader can have been elected within this window (because acceptors won't vote against the lease). - -If the clock-skew assumption is *violated*, lease-based reads can return stale data. This is documented and the operator can force ReadIndex for stronger guarantees. - ---- - -## 10. Tunables and Defaults - -| Parameter | Default | Range | Notes | -| --- | --- | --- | --- | -| `heartbeat_interval` | 500 ms | 10 ms – 30 s | Should be ≪ `lease_duration` | -| `lease_duration` | 4500 ms | 100 ms – 60 s | Should be ≫ `heartbeat_interval` + `max_clock_skew` (rule of thumb: `9 × heartbeat_interval`) | -| `effective_lease` | derived | — | `= lease_duration - max_clock_skew` | -| `max_clock_skew` | 500 ms | 1 ms – 5 s | Architectural bound from design-crowdb-paxos-kv.md §3](design-crowdb-paxos-kv.md) | -| `election_min` | 4000 ms | ≥ 8 × `heartbeat_interval` | Avoid spurious elections | -| `election_max` | 8000 ms | ≤ 60 s | Bounds time to elect after leader loss | -| `prevote_enabled` | true | bool | Reduces disruption from rejoining nodes | -| `bulk_prepare_window` | 1024 | 1 – 65536 | Slots per yield batch in bulk Phase-1 repair | -| `maintenance_tick_ms` | 30 000 | 1 000 – 300 000 | Engine-durability + WAL-GC maintenance loop interval | - -Notes: - -- **PreVote** (Raft optimization, ON by default): a candidate first asks "would you vote for me?" without bumping the term. This avoids spurious term increments from a partitioned-and-rejoined node. -- **Pre-emptive step-down on heartbeat loss** is governed by `lease_duration`, not `election_min`. A leader that loses contact with quorum gives up its lease at `lease_duration` and stops serving fast reads even before any follower starts an election. -- **Default choice rationale.** The defaults above are tuned for general-purpose deployments (mix of single-DC and modest cross-AZ latency). They give ~5–8 s failover detection, which matches the operational expectations of most online KV workloads while keeping heartbeat chatter low. See [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §13 (Open Design Questions) for the analysis. -- **Operational profiles:** - - *Low-latency single datacenter:* `heartbeat_interval = 100 ms`, `election_min/max = 800/1500 ms`, `lease_duration = 900 ms`, `max_clock_skew = 100 ms`. Sub-second failover; higher message rate. - - *Cross-region / WAN:* `heartbeat_interval = 3 s`, `election_min/max = 24/48 s`, `lease_duration = 27 s`. Matches CockroachDB-style geo-replicated tunings; tolerates wider RTT variance at the cost of failover latency. - - *Tests:* `PxElectionConfig::for_tests()` produces `heartbeat = 5 ms`, `election = 30–60 ms`, `lease = 25 ms` for use under `tokio::time::pause()`. Not exposed on `crowdb-paxos-kv-server` CLI. diff --git a/doc/design/kv/design-crowdb-paxos-kv-observability.md b/doc/design/kv/design-crowdb-paxos-kv-observability.md deleted file mode 100644 index a8ea1219e..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-observability.md +++ /dev/null @@ -1,505 +0,0 @@ - - - -# CROWDB - Design: Observability - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §16 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §16 - -## Table of Contents - -- [1. Mandatory Signals](#1-mandatory-signals) -- [2. Metrics Module](#2-metrics-module) - - [2.1 Metric Types](#21-metric-types) - - [2.2 Registry and Lifecycle](#22-registry-and-lifecycle) - - [2.3 Naming Convention](#23-naming-convention) - - [2.4 Instrumentation Points](#24-instrumentation-points) - - [2.5 System Metrics Collector](#25-system-metrics-collector) - - [2.6 Metrics Log File](#26-metrics-log-file) - - [2.7 In-Memory Access](#27-in-memory-access) - - [2.8 FFI Boundary](#28-ffi-boundary) - - [2.9 Design Principles](#29-design-principles) - - [2.10 Read Path Metrics](#210-read-path-metrics) - - [2.11 Write Path Metrics](#211-write-path-metrics) - - [2.12 Client Metrics](#212-client-metrics) - - [2.13 C++ Registry Ownership](#213-c-registry-ownership) - - [2.14 Rust/C++ Metric Deduplication](#214-rustc-metric-deduplication) - - [2.15 C++ flush_to Format Alignment](#215-c-flush_to-format-alignment) - - [2.16 Shared Column Width](#216-shared-column-width) -- [3. Logging](#3-logging) - - [3.1 Log Directory Convention](#31-log-directory-convention) - - [3.2 Rotation and Compression](#32-rotation-and-compression) - - [3.3 Per-Stack Files](#33-per-stack-files) - - [3.4 Metrics Log](#34-metrics-log) - - [3.5 Level Unification](#35-level-unification) - - [3.6 C++ Logger Additivity](#36-c-logger-additivity) - - [3.7 No Separate ops_log](#37-no-separate-ops_log) - - [3.8 CLI Logging](#38-cli-logging) - - [3.9 Log Content Guidelines](#39-log-content-guidelines) - - [3.10 Extension Path](#310-extension-path) - -## 1. Mandatory Signals - -Per-group leader/term/max-slot/safe-slot/in-flight/gap count; per-node WAL -flush latency and throughput; per-RPC rate/latency/error breakdown; structured -logs with `node_id`, `group_id`, `slot`, `term` on consensus events. Tracing -hooks reserved but not required in the initial design. - -## 2. Metrics Module - -A lightweight metrics system with five metric types and periodic flush to a -dedicated metrics log file. Rust owns the registry for consensus/RPC/WAL -metrics; C++ owns its own registry for storage-engine metrics. Rust drives -the flush cycle and triggers C++ to emit its section via FFI — no metric -handles cross the FFI boundary at runtime. - -### 2.1 Metric Types - -- **Counter** (`AtomicU64` x 2) — monotonic, tracks window delta + total. - `inc()` / `inc_by(n)`. Flush shows `count`, `tps`, `total`. Use cases: - puts, gets, deletes, errors, WAL records, elections, step-downs. -- **Gauge** (`AtomicU64`) — current state, can go up or down. `set(v)`. - Flush shows last value. Use cases: buffer pool resident/dirty pages, - in-flight slots. -- **Bandwidth** (`AtomicU64` x 3) — monotonic bytes, tracks count + sum + - total_bytes. `observe(bytes)`. Flush shows `count`, `tps`, `avg_size(KB)`, - `rate(MB/s)`. Use cases: KV bytes in/out. -- **LatencyHistogram** (13 buckets + 2 `AtomicU64`) — fixed-bucket percentile - distribution. Bucket boundaries: `0, 1us, 10us, 100us, 500us, 1ms, 5ms, - 10ms, 50ms, 100ms, 500ms, 1s, infinity`. `observe(ns)` does binary search + - `fetch_add`. Flush computes p50/p99 from cumulative distribution. Use cases: - KV put latency, KV get latency. -- **LatencySummary** (`AtomicU64` x 4) — lightweight latency tracking - (count + sum + max + total_count). `observe(ns)`. Flush shows `avg(us)`, - `max(us)`. Use cases: scan, snapshot, WAL append, RPC, apply. -- **PreciseHistogram** (`lib/crowdb-common/rust/src/metrics/precise.rs`) — - HDR-style logarithmic histogram delivering ≥3 significant digits of - percentile precision, the precise counterpart to the fixed-bucket - `LatencyHistogram`. `1024` linear sub-buckets per power-of-2 magnitude - (relative error ≤0.1%); pre-allocated range `0..=2^32 µs` (~71 min) - makes `auto(true)` a no-op (no resize logic). `&mut self` methods - (`record`, `add`, `reset`); every call site has exclusive access - (`Mutex`-guarded in the client library, per-worker owned in the bench - `OpStats`, owned by the flusher task for `CumulativeLatency`), so a - lock-free impl is unnecessary. Tracks `count`/`sum`/`min`/`max` exactly - (only percentiles are bucketed). Use cases: bench-client per-op - latency (`OpStats`), client-library window latency - (`WindowLatencySnapshot`), bench cumulative run-wide percentiles - (`CumulativeLatency`). Replaces the external `hdrhistogram` crate. - -### 2.2 Registry and Lifecycle - -Each language has a `MetricsRegistry` that owns all metric instances. The -registry has `start(interval_secs)` (spawns flush thread/task), `stop()` -(final flush + join), and `flush()` (iterate all metrics, snapshot, format, -reset window state). Interval is typically 5s or 10s. - -- Rust (`lib/crowdb-paxos-kv/src/metrics/mod.rs`): `MetricsRegistry` with type-grouped - `Vec` collections, `Arc`-shared, metric handles stored on service/store - structs. `MetricsRunner` spawns a tokio interval task, computes real - elapsed `window_secs` per tick, flushes Rust metrics, then invokes a - post-flush callback that calls C++ `flush_metrics_str()` for each engine - and writes the `cpp-tree` block. Also provides `snapshot(prefix)` for - in-memory access without resetting window state. - The pre-flush engine collector loads immutable `ArcSwap` snapshots of its - per-group and per-store handles. Cumulative delta cursors live beside those - handles as atomics. Sampling therefore takes no handle-list or delta-state - mutex; discovery registers new metric handles in a separate bounded registry - phase and publishes a replacement snapshot. -- C++ (`lib/crowdb-tree/include/lib/crowdb-tree/metrics.h`, `lib/crowdb-tree/src/metrics.cpp`): - Same type-grouped pattern. `Crowdbtree` owns its own `MetricsRegistry` - internally (`init_metrics(prefix)` called from `open()`). Metric handles - are raw pointers (registry owns lifetime). `flush_metrics_str()` captures - `flush_to()` output via `open_memstream` for FFI return. The C++ - `start()`/`stop()` sleep-loop is retained for standalone/test use but not - called from the server's production flush path. - -### 2.3 Naming Convention - -Dot-separated hierarchical paths: `s.{store_id}.g.{group_id}.{module}.{metric}`. -Type suffix on every metric name: `.c` (Counter), `.g` (Gauge), `.bw` -(Bandwidth), `.lh` (LatencyHistogram), `.l` (LatencySummary). Dynamic suffix -`@{peer_endpoint}` for per-peer metrics. System metrics use `sys.` prefix -with no type suffix. - -Prefix-based snapshot: `registry.snapshot("s.1.")` returns all metrics for -store 1; `snapshot("")` returns all. This is the foundation for future GUI -integration. - -### 2.4 Instrumentation Points - -- Rust KV service (`kv_service.rs`): put/get latency histograms, scan summary, - delete counter, bytes in/out bandwidth, read bytes in/out bandwidth, - error/no-leader counters, get-forwarded / get-forward-failed counters. -- Rust WAL (`wal_engine.rs`, `pipeline_writer.rs`): append latency summary, - fsync latency summary (thinnest-layer disk IO), write bandwidth. -- Rust cluster (`local_replica.rs`): election/step-down counters, in-flight - slots gauge. Paxos slot watermarks (gauges bridged from `LocalReplica`). - Engine-apply latency summary (registered here, observed in - `learner.rs apply_entry`). -- Rust cluster (`group.rs`): read-path handles (`ReadRegistryHandles`) — - lease/ReadIndex path counters, read barrier latency summary, engine_get - latency summary, MinSlot-fallback counter, read-state gauges (lease valid, - contiguous applied, safe slot). Write-path handles - (`WriteRegistryHandles`) — propose-e2e, prepare-phase, accept-phase, - accept-quorum-RPC latency summaries. -- Rust RPC (`remote_replica.rs`): per-peer RPC latency summary + error counter - with dynamic names. -- C++ buffer pool (`buffer_pool.cpp`): hits/misses/evictions/writebacks - counters, resident/dirty gauges. -- C++ engine (`crowdb-tree.cpp`): apply latency summary, demand-load latency - summary + page-read bandwidth. -- C++ persist (`persist.cpp`): snapshot latency summary, snapshot-apply - latency summary, per-page write IO latency + cache-hit counter + write - bandwidth, metadata write bandwidth. -- C++ flush (`crowdb-tree.cpp`): flush latency summary, page-build (in-memory - mutation) latency summary, flush drain/entries magnitude counters. - -### 2.5 System Metrics Collector - -Special collector type polled at flush time (not increment-on-event). Reads -TCP retransmits/lost (Linux `/proc/net/snmp`, macOS no-op), CPU user/sys -and memory RSS via `getrusage(RUSAGE_SELF)`. Computes CPU% as delta over -flush window. - -### 2.6 Metrics Log File - -Dedicated file `metrics-{timestamp}-{pid}.log` in the log directory, separate -from application log. Each flush cycle produces one shared timestamp header -`[{ISO8601} window={N.NN}s]`, then section blocks: `rust` (Rust metrics + -misc system metrics), `cpp-tree` (C++ per-engine), and `cpp-rpc` (C++ global / -crowdb-rpc). Blocks are followed by type-grouped sections in order: -LatencyHistogram, LatencySummary, Bandwidth, Counter, Gauge, then System. -Names sorted alphabetically within each section, padded to `max_name_len` for -alignment. Zero-suppression: counters/histograms/summaries/bandwidths with -zero window activity are skipped; gauges always printed. C++ `flush_to()` -output is format-aligned to Rust's column layout (same units, columns, -precision). Format designed for both human reading and script parsing (split -on whitespace, parse as numbers). - -### 2.7 In-Memory Access - -`registry.snapshot(prefix)` returns current values without resetting window -state. Enables future `/metrics` HTTP endpoint and GUI integration. No need -to parse log files to get metric values. - -Management status is a curated view, not a second metrics system. Each status -field names one registry metric and reads it through exact-name lookup. Missing -or not-yet-registered names are omitted. The status path never registers a -metric or retains duplicate hot-path counters; `/metrics` remains the endpoint -for the complete registry. - -### 2.8 FFI Boundary - -C++ owns its own `MetricsRegistry` per `Crowdbtree` instance. Rust triggers C++ -to flush its metrics section into the same log file via FFI -(`ct_flush_metrics_str`). No metric handles cross FFI at runtime, only a -formatted string. Three log blocks per flush cycle: `rust` (Rust), -`cpp-tree` (C++ per-engine), and `cpp-rpc` (C++ global). The existing -`ct_get_stats` FFI call (used by `/topology` and the one remaining -`snapshot.pages.c` delta bridge) is unaffected. - -### 2.9 Design Principles - -- **Counter/Summary Non-Redundancy** — A `LatencySummary` (or - `LatencyHistogram`) already carries `count` (window delta) and - `total_count` (cumulative). Do not register a separate Counter for the - same event. A Counter next to a Summary is justified only for: (a) a - magnitude (e.g. entries moved, not calls made), (b) a different - population/outcome (e.g. errors vs. all attempts), or (c) a cache-hit - path where no latency is meaningful. A `Bandwidth` next to a `Latency` is - not redundant; they measure different dimensions (bytes vs. nanoseconds) - of the same event. -- **Latency Hierarchy** — For every flow, add latency at the thinnest layer - (actual syscall / disk IO / RPC boundary) and at the feature layer above - it (the meaningful unit of work a human debugs against). Children should - roughly sum to the parent; a persistent gap indicates an unaccounted code - path. Skip a layer if it isn't a distinct decision point. -- **Bandwidth Hierarchy** — Parallel to latency: measure bytes moved at the - IO boundary, domain-separated (page data vs. metadata, read vs. write). - Skip layers with no distinct IO. -- **Real Window-Time** — The flush window is the real elapsed time, not the - nominal configured interval. Rust computes `window_secs` once per tick - and passes the same value to both its own `reg.flush()` and the C++ FFI - call, guaranteeing identical windows across both sections. - -### 2.10 Read Path Metrics - -The read path mirrors the write path's latency-bandwidth-counter hierarchy. -Handles live in two places: `KvMetrics` (per store, group; in -`kv_service.rs`) for RPC-layer metrics, and `ReadRegistryHandles` (per -store, group; on `PxGroup` via `OnceLock`, mirroring -`ElectionRegistryHandles` on `PxLocalReplica`) for consensus- and -engine-layer metrics. - -- **Latency hierarchy** (feature layer → thinnest layer): - - `kv.get.lh` — get RPC end-to-end (existing). - - `read.e2e.l` — `LatencySummary` for the full server-side read path - (barrier + engine_get), measured in `kv_get` handler. - - `read.barrier.l` — `LatencySummary` for - `linearizable_read_barrier` (near-zero for lease path, one heartbeat - RTT for ReadIndex). - - `read.apply_fence.l` — `LatencySummary` for the R35 apply fence - wait (fast path is a single atomic load). - - `read.engine_get.l` — `LatencySummary` for `KVEngine::get_bytes` - (isolates engine cost from consensus barrier cost). - - `kv.scan.l` — scan RPC end-to-end (existing). -- **Bandwidth hierarchy** (read vs. write separation; combined kept for - backward compat, read subset lets operators derive write by subtraction): - - `kv.read_bytes_in.bw` / `kv.read_bytes_out.bw` — read traffic - separated from the combined `bytes_in/out.bw`. -- **Counters** (outcome / population separation): - - `kv.get_forwarded.c` — reads forwarded to leader (server-side). - - `kv.get_forward_failed.c` — forward attempts that failed. - - `read.minslot_fallback.c` — MinSlot reads redirected to leader - because the local replica hasn't caught up. -- **Gauges** (state, bridged on-demand at `resolve_read_point`): - - `read.safe_slot.g` — current `group_safe_slot`. - -`read.barrier.l` count equals the total linearizable get count in the -same window (lease fast path + ReadIndex path combined). - -### 2.11 Write Path Metrics - -The write path mirrors the read path's latency hierarchy. Handles live in -`WriteRegistryHandles` (per store, group; on `PxGroup` via `OnceLock`, -mirroring `ReadRegistryHandles`), except `engine_apply` which is registered -on `PxLocalReplica` (alongside the WAL handles) and observed in -`PxLearner::apply_entry`. - -- **Latency hierarchy** (feature layer → thinnest layer): - - `paxos.propose.e2e.l` — `propose_inner` entry → return (the full - client-observed proposal latency, including retries). - - `paxos.classic.prepare.l` — `run_prepare_phase` entry → return - (classic Paxos only; leader path skips prepare). - - `paxos.accept.quorum_rpc.l` — accept-phase start → first-quorum - reached (the k-th-fastest remote reply latency; recorded only on the - quorum short-circuit success path, not on the failure path). - - `paxos.learn.apply.l` — `PxLearner::apply_entry` entry → return - (isolates engine apply cost from consensus phase cost). - -All five are `LatencySummary` (count + sum + max), matching the read-path -`barrier` / `engine_get` pattern. Percentile precision is not needed for -phase-level attribution. The bench client already has `PreciseHistogram` -for client-observed p99. The `accept_quorum_rpc` timer is meaningful only -after the quorum short-circuit (§6.1 of `design-crowdb-paxos-kv-rpc.md`); it records the -quorum-th-fastest remote latency, not the full fan-out tail. - -### 2.12 Client Metrics - -`crowdb-paxos-kv-client` exposes its own `ClientMetrics` (lock-free `AtomicU64` -counters, snapshotted via `ClientMetricsSnapshot`) for retry, topology, -and read-distribution observability. They are not part of the server's -`MetricsRegistry`. The client is a separate process and has no C++ -handle. Two counters cover follower-read distribution: - -- `read_endpoint_distributed` — incremented each time the `AnyReplica` - selector picks a replica from the cached list for a `MinSlot` read - (i.e. the distribution branch fired). Pairs with the server-side - `read.minslot_fallback.c` to confirm the distribution rate and the - fallback rate. -- `read_endpoint_fallback` — incremented each time a distributed - `MinSlot` read fell back to the leader via `NotLeaderHint` (get) or - the scan `"not leader; retry scan at "` parse. A high - `read_endpoint_fallback / read_endpoint_distributed` ratio means - followers are lagging and the policy is providing little benefit. - -The client library also maintains per-op-kind window latency histograms -(`WindowLatency` / `WindowLatencySnapshot`). Recording threads select one of -64 stable shards and contend only with recorders assigned to that shard; -`drain_window` locks and merges one shard at a time for periodic flushing and -feeds the bench runner's cumulative -`bench.*.lh` histograms. These use `crowdb-common`'s `PreciseHistogram` -(not the server's fixed-bucket `LatencyHistogram`) because the bench -needs p50/p90/p99/p999/max at ≥3 significant digits, the same precision -the bench's own `OpStats` and `CumulativeLatency` require. The bench -runner's per-worker `WorkerCounters` use `crowdb-common`'s `Counter` -(window delta via `flush().count`, cumulative total via -`snapshot().total`), unifying the bench and client on the project's own -metrics primitives and removing the external `hdrhistogram` dependency. - -### 2.13 C++ Registry Ownership - -C++ `Crowdbtree` creates its own `MetricsRegistry` internally via -`init_metrics(prefix)`, called from `Crowdbtree::open()`. The external -`set_metrics(MetricsRegistry*, prefix)` API is removed. C++ metrics -(counters, gauges, summaries, bandwidths) are registered and observed -entirely in C++, then flushed to string via `flush_metrics_str()` and -written to the log by the Rust `MetricsRunner` post-flush callback. - -### 2.14 Rust/C++ Metric Deduplication - -Once C++ owns its registry, the Rust-side bridge (`engine_collector.rs`) -no longer polls C++ cumulative counters and gauges. Those metrics appear -natively in `cpp-tree`. The Rust `rust` section keeps only -Rust-native metrics (KV service, RPC, Paxos, WAL). The one exception is -`snapshot.pages.c`, a magnitude counter with no paired latency in the C++ -registry, which remains bridged via `ct_get_stats` delta polling. - -### 2.15 C++ `flush_to` Format Alignment - -C++ `flush_to()` output is aligned to Rust's column layout: `tps(/s)` -column on all windowed types, latency in `us` (not `ns`), bandwidth in KB, -Histogram column order matching Rust, and `window=%.2fs` precision. The -section header label is parameterized (`"metrics"` for standalone, -`"cpp-tree"` for per-engine FFI-driven flush, `"cpp-rpc"` for global -FFI-driven flush). - -### 2.16 Shared Column Width - -Both `rust` and `cpp-tree` sections use the same column width -for metric names. Before each flush tick, Rust queries each C++ engine's -max name length via `ct_max_name_len()`, computes -`shared_width = max(rust_max, max(cpp_maxes))`, and passes it to both its -own `flush()` and C++ `flush_metrics_str()`. This adapts automatically as -handles are added dynamically. - -## 3. Logging - -CROWDB has two logging stacks — a Rust `tracing` stack -(`crowdb-common/rust/src/logging.rs`) and a C++ `spdlog` stack -(`crowdb-common/cpp/src/log.cpp` + `compressing_sink.cpp`). Both -produce rotating, gzip-compressed log files with the same naming -convention. This section anchors the unified scheme across all -server binaries. - -### 3.1 Log Directory Convention - -Every server binary accepts a `--log-dir

    ` CLI flag. All Rust -and C++ logs for a process land under that directory. Defaults: - -- `crowdb-paxos-kv-server`, `crowdb-diskdb`, `crowdb-chunkdb` — `"log"` - (relative to CWD). `crowdb-paxos-kv-server` also falls back to - `PaxosKvConfig.log_dir` (config field, default `root/log`) when - `--log-dir` is not given. -- `crowdb-web` — `~/.crowdb-paxos-kv/log`. -- `crowdb-cli` — multi-service lifecycle, server deployment, and benchmark - commands use `/cli-log/cli-{command-slug}-{timestamp}/`; short - inspection and data operations are console-only and create no file log. - -The test harness passes `--log-dir ` so child-process -file logs land in `test-logs/` alongside redirected stderr. - -### 3.2 Rotation and Compression - -Both stacks use the same defaults: - -- Max file size: 30 MiB (`DEFAULT_LOG_MAX_FILE_MB`). -- Max rotated files: 5 (`DEFAULT_LOG_MAX_FILES`). -- Tunable via `--log-max-file-mb` and `--log-max-files` CLI flags. - -Rotation: when the current file exceeds the size cap, it is closed, -gzip-compressed to `.log.gz`, and a new file is opened. The oldest -`.log.gz` files beyond the retention count are deleted. - -File naming: `{prefix}-{YYYYMMDD-HHMMSS.mmm}-{pid}.log`. The PID -suffix prevents collision when two processes with the same prefix -start in the same second. - -### 3.3 Per-Stack Files - -Rust and C++ write to separate files in the same directory (D1). -Prefixes: - -- `{server}` — Rust tracing log (e.g. `crowdb-paxos-kv-server`). -- `{server}-tree` — C++ crowdb-tree spdlog (e.g. - `crowdb-paxos-kv-server-tree`). -- `{server}-rpc` — C++ crowdb-rpc spdlog (e.g. - `crowdb-paxos-kv-server-rpc`). -- `{server}-metrics` — metrics log (Rust, via `MetricsRunner`). - -The shared timestamp + PID suffix lets the operator interleave files -from one process by sort order; no cross-FFI write coordination -needed. The merged-file option is rejected — per-stack files keep the -two stacks independent and avoid cross-FFI write contention. - -### 3.4 Metrics Log - -The metrics log is the one combined file per process (D2). Rust -queries the metrics registry (including C++ metrics via FFI) and -writes the content via `MetricsRunner`. The C++ stack does not write -to the metrics log directly. - -### 3.5 Level Unification - -Both Rust and C++ stacks default to `info` (D3). The `--log-level` -CLI flag drives both stacks. If `RUST_LOG` is set, the Rust -`EnvFilter` uses it directly; for the C++ side, the first global -directive is extracted via `cpp_level_from_rust_log()` (e.g. -`RUST_LOG=debug` → C++ `"debug"`, `RUST_LOG=crowdb_paxos_kv=info` → C++ -`"info"`). Valid levels: `trace`, `debug`, `info`, `warn`, `error`, -`off`. - -The `--log-stderr ` flag mirrors C++ log lines at the given -level or above to the process stderr (via `ct_add_log_stderr` / -`crowdb_rpc_ffi::add_log_stderr`). This lets operators tail stderr -for failures without grepping the file. `crowdb-web` defaults this to -`"warn"`; other servers default to off. - -### 3.6 C++ Logger Additivity - -When `ct_init_logging` runs first (as in `crowdb-paxos-kv-server`), -`crowdb_rpc_init_logging` calls `add_log_file` — it adds a second -file sink to the same `crowdb::common` logger rather than creating a -new one. The rpc sink inherits the tree logger's level. The -`--log-level` flag sets the tree logger's level, which the rpc sink -inherits. This is the expected behavior: both C++ stacks run at the -same level and write to separate files via separate sinks on the same -async logger. - -When `ct_init_logging` has NOT run (as in `crowdb-web`, -`crowdb-diskdb`, `crowdb-chunkdb`, `crowdb-cli`), -`crowdb_rpc_init_logging` creates a fresh logger with the given level. - -### 3.7 No Separate ops_log - -The former `ops_log` module has been deleted (D4). Operations lines -(HTTP/RPC/SSH call records) are folded into the normal tracing log -via `log_ops_http` in `crowdb-console-shared/src/clients.rs`. The -server log's rotation covers ops lines — no separate unbounded file. - -### 3.8 CLI Logging - -`crowdb-cli` writes file logs for all commands to a per-invocation -directory `cli-log/{command-slug}-{timestamp}/` (D5, revised). The -`bench` subcommand additionally opens a metrics log via -`BenchMetrics::new`. Other commands can opt into additional logging -later if a detached use case appears. - -### 3.9 Log Content Guidelines - -Server and client logs use the same context field names: - -- `s`: store ID when known. -- `g`: group ID when known. -- `replica`: the replica executing or receiving an operation. -- `leader`: the server's current leader or the client's currently believed - leader. Omit it when unknown; do not copy a merely targeted replica into it. - -Stable `s`, `g`, and `replica` context belongs in tracing spans at task and RPC -boundaries. Spawned futures are explicitly instrumented because Tokio tasks do -not inherit the caller's active span. Leadership is dynamic, so `leader` or -`role` is recorded on the event whose interpretation needs it instead of being -captured in a long-lived span. - -C++ thread names are published as immutable thread-id/name maps. The spdlog -formatter atomically loads a snapshot keyed by the originating log message's -thread ID; it does not use formatter-thread TLS and takes no shared mutex. - -- Every line should carry enough context to locate the code position and the - surrounding state. -- Consensus-event logs must carry `replica`, `g`, `slot`, and `term` where - applicable (per §1 mandatory signals). -- Machine-parseability is a plus — AI agents do the real log-digging. - -### 3.10 Extension Path - -Any future server (e.g. `diskio` server) follows the same scheme: - -- `--log-dir` / `--log-level` / `--log-max-file-mb` / `--log-max-files` - / `--log` / `--log-stderr` CLI flags. -- Rust tracing init via `init_file_logging` or - `init_file_and_console_logging_split`. -- C++ FFI init for each linked C++ library (`ct_init_logging` for - crowdb-tree, `crowdb_rpc_ffi::init_logging` for crowdb-rpc). -- Per-stack file prefixes: `{server}`, `{server}-tree`, `{server}-rpc`. diff --git a/doc/design/kv/design-crowdb-paxos-kv-reconfiguration.md b/doc/design/kv/design-crowdb-paxos-kv-reconfiguration.md deleted file mode 100644 index 943cfd3e2..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-reconfiguration.md +++ /dev/null @@ -1,260 +0,0 @@ - - - -# CROWDB - Design: Reconfiguration - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §9.1, §9.2, [`design-crowdb-paxos-kv-leader-election.md`](design-crowdb-paxos-kv-leader-election.md) -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §9.1, §9.2 - -This document specifies how a CROWDB group safely changes its membership while preserving consensus safety. The mechanism is direct per-node HTTP mutation of each replica's remote-replica list, persisted to the local `GroupConfigStore`, with a `membership_epoch` exact-match fence. This model applies to all groups including the system group (group 0, which stores cluster topology metadata, see `design-crowdb-paxos-kv.md` §3.3). - -## Table of Contents - -- [1. Scope and Supported Transitions](#1-scope-and-supported-transitions) -- [2. Direct Per-Node Mutation Model](#2-direct-per-node-mutation-model) -- [3. New-Member Bootstrap](#3-new-member-bootstrap) -- [4. Member Removal](#4-member-removal) -- [5. Leader Transfer](#5-leader-transfer) -- [6. The `membership_epoch` Fence](#6-the-membership_epoch-fence) -- [7. Group-0 Special Cases](#7-group-0-special-cases) -- [8. Safety Argument](#8-safety-argument) -- [9. Failure During Reconfiguration](#9-failure-during-reconfiguration) -- [10. Tunables and Defaults](#10-tunables-and-defaults) - ---- - -## 1. Scope and Supported Transitions - -CROWDB supports membership changes within a single group. Specifically: - -- **Add or remove voting members.** Adding a member when going 3 → 5 → 7 or removing a member when going 7 → 5 → 3. -- **Replace a member.** Implemented as add-then-remove (or vice versa), each as a single-member change. -- **Change the leadership of the group.** Triggered as a side effect when removing the current leader. - -Out of scope ([design-crowdb-paxos-kv.md §2](design-crowdb-paxos-kv.md)): - -- Changing `num_groups` (the total number of groups in the cluster) — fixed at cluster creation. -- Splitting or merging groups — not supported. -- Reconfiguring a protected group below 3 voting members — not supported. The explicit single-node test deployment starts with one voter and has no fault tolerance; it is not entered through membership reconfiguration. -- Going above 7 voting members — not in the initial scope; quorum size grows linearly with membership and the marginal availability gain past 7 is small. - -**Granularity:** every reconfiguration moves *exactly one member* in or out at a time. To go 3 → 5, do two single-member additions in sequence. Under the exact-match `membership_epoch` fence (§6) this single-member-at-a-time rule is no longer required for safety, but it is still recommended because it minimizes the propagation window during which writes stall. - ---- - -## 2. Direct Per-Node Mutation Model - -The design does not use `ConfigChange` log entries or a joint configuration. Instead, the operator (or `crowdb-console`) mutates the remote-replica list on each node independently through the HTTP management API: - -- `POST /stores/:sid/groups/:gid/remotes` — add one or more remote replicas. -- `DELETE /stores/:sid/groups/:gid/remotes/:rid` — remove a remote replica. - -Each call is atomically persisted to the local `GroupConfigStore` config file (atomic `tmp` + `fsync` + rename) and applied synchronously to the in-memory `PxGroup`. `membership_epoch` is bumped by exactly one when the call changes the voting set. A non-voting member addition/removal does **not** change the epoch or the quorum size. - -Because there is no cluster-wide `ConfigChange` log, the quorum view is local to each node. This is safe because the `membership_epoch` fence (§6) forces all `Prepare`/`Accept` RPCs to be evaluated at the same epoch before their replies can count toward a quorum. A leader whose local epoch differs from a peer's rejects that peer's reply and converges upward (§6.3). - -### 2.1 Propagation window and write stalls - -A membership mutation is applied to one node per HTTP call. Between the first node's call and the last node's call, different nodes transiently disagree on the `membership_epoch`. During this window a leader that has already adopted the new epoch will see `epoch_mismatch` from nodes that have not yet adopted it. Writes are rejected by the acceptors and the leader's `propose` loop self-heals by adopting the responder's higher epoch (up to `max(own, peer)`) and retrying the same slot. - -This is a deliberate availability/safety trade-off: writes stall for the cluster-wide duration of the fan-out (one HTTP round-trip per node), then resume immediately. The stall is bounded, self-healing, and loudly visible in the `epoch_mismatch` response bit rather than silently using a stale quorum. - -### 2.2 Group-0 complete membership authority - -Group 0 keeps one authoritative JSON record at `/kv/members/{store}/{group}`. -The record contains the complete ordered member list, a positive monotonic -epoch, and either `Installing { previous_members }` or `Ready`. A console -membership change first CAS-publishes the complete successor at -`current_epoch + 1` in `Installing`; peer wiring and consensus fencing then -use that exact successor before the record is CAS-completed to `Ready`. - -The management endpoints carry `x-crowdb-membership-epoch`. A successor -request is accepted only for the authorized next epoch. A replay at the -already-installed epoch succeeds only when its remote payload matches the -existing configuration exactly; a different payload returns HTTP 409 before -the config file or running group is changed. This protects persistence and -publication as one guarded operation while allowing a lost response to be -retried safely. - ---- - -## 3. New-Member Bootstrap - -A new member joining a group has empty WAL and empty engine state. It must be brought up to date before it can vote. - -### 3.1 Phases - -``` - 1. Console ensures the new node hosts the store and places the group on it. - 2. The new member is added as `voting: false` on every existing node. - 3. The new member is added as `voting: false` on itself (so it learns remotes). - 4. Snapshot install: a caught-up peer streams a snapshot to the new member. - 5. WAL catch-up: the new member follows the leader via normal replication. - 6. The new member is re-registered with `voting: true` on every node. - 7. `membership_epoch` is bumped on every node; the new member is now a voting peer. -``` - -### 3.2 Why pre-load before voting - -If a new member were added directly with `voting: true` while still empty, then a quorum could form with its empty-state vote. A subsequent leader election under that quorum could elect a leader that has not seen older committed writes — violating Paxos safety. - -By requiring catch-up while the member is non-voting, we ensure that when it first becomes a voting peer it already has all chosen values up to the current `max_chosen`. - -### 3.3 Snapshot install protocol - -Defined in [§8.4 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#84-snapshot-and-install) and `design-crowdb-paxos-kv-state-machine.md` §6 (snapshot import). Resumable, throttled, end-to-end CRC. - -### 3.4 Catch-up termination criterion - -The leader continues streaming WAL until the new member's `contiguous_applied` is within `catchup_slack` (default 100 slots) of the leader's `max_chosen`. At that point the operator re-adds the member as `voting: true` and the new member is expected to keep up via normal replication once it becomes voting. - -If the new member cannot keep up (e.g. slow disk), the operator can abort by removing it before flipping the `voting` flag. - ---- - -## 4. Member Removal - -Removing a member is simpler than adding one because there is no catch-up to wait for. - -### 4.1 Procedure - -``` - 1. If the removed member is the current leader, ask it to StepDown first. - 2. Delete the removed replica as a remote from every surviving peer. - 3. Delete the local `PxGroup` on the removed member's node. - 4. The `membership_epoch` is bumped on every node where the voting set changed. -``` - -### 4.2 If the removed member is the leader - -The console's `DELETE /api/stores/:sid/groups/:gid/replicas/:rid` handler calls `POST /stores/:sid/groups/:gid/step-down` on the leader node first. The leader rejects further proposals, the survivors run a normal election, and only after a new leader is observed does the removal proceed. If the step-down call fails (network partition, crashed process), the console proceeds anyway and the survivors' lease-unrenewable logic eventually elects a new leader. - -### 4.3 If the removed member is unreachable - -Removal works regardless of reachability. The surviving peers delete the dead replica from their remote lists, and the dead replica's local group is removed if the node is still reachable. The `membership_epoch` bump on each survivor ensures the remaining voters cannot be confused by delayed heartbeat/accept traffic from the removed node once it comes back. The removed node is then absent from every survivor's `remote_replica_info()`. - ---- - -## 5. Leader Transfer - -Transferring leadership is needed when: - -- The current leader is being removed by reconfiguration. -- An admin wants to drain a node for maintenance. - -### 5.1 Procedure - -1. Admin or console calls `POST /stores/:sid/groups/:gid/step-down` on the leader node. -2. The leader strictly-fences the request (`self.id == target_leader_id && current term`) and calls `handle_step_down`, becoming a follower. -3. The old leader stops accepting new client writes; in-flight requests are handled by the new leader once elected. -4. Followers run a normal randomized-timeout election. The first up-to-date candidate to reach a quorum of the current voting set wins. -5. The old leader, on observing the new term in any RPC response, steps down. - -This is not Raft's `TimeoutNow` fast transfer. It avoids the complexity of target-precondition checking and is accepted as sufficient because membership changes are operator-driven. - -### 5.2 During reconfiguration - -When removing the current leader: - -1. The console calls step-down on the leader before issuing the removal. -2. The console waits (bounded, 5 s) for a survivor to report a leader *other than* the removed replica. -3. The removal proceeds; the new leader's `membership_epoch` now governs the group. - ---- - -## 6. The `membership_epoch` Fence - -Every `Prepare`/`Accept` RPC carries the proposer's current `membership_epoch`. The acceptor compares it to its own local `membership_epoch`. If they do not match exactly, the acceptor rejects the request with `epoch_mismatch: true` and attaches its own epoch. The proposer then adopts `max(own_epoch, responder_epoch)` and retries the same slot. - -### 6.1 Why exact match, not "within 1" - -Single-degree membership changes (old/new configs differing by exactly one voting member) have a proven quorum-overlap guarantee: any majority of `N` and any majority of `N±1` always share at least one member. That guarantee does **not** chain transitively across two sequential changes unless the first one fully converges everywhere. A counter-example: `C1 = {1,2,3}` (quorum 2) and `C3 = {1,2,3,4,5}` (quorum 3) have majorities `{2,3}` and `{1,4,5}` that do not intersect. So "one epoch behind is safe" is false in general. The only airtight rule is **exact epoch match**. - -### 6.2 What exact match buys - -Since a quorum decision can never straddle two epochs, the single-degree-at-a-time restriction stops being a safety requirement. A batch membership change (same voting set delta applied to multiple nodes in one fan-out) is just as safe as a single-member change; it only takes longer to converge. - -### 6.3 Convergence and self-healing - -Both the proposer and the acceptor converge upward to `max(own, peer)`: - -- **Proposer (`PxGroup::propose`):** on `MembershipEpochMismatch`, adopts `responder_epoch` and retries the same slot. -- **Acceptor (`PxLocalReplica::on_prepare`/`on_accept`, heartbeat catch-up):** on `epoch_mismatch`, adopts `max(own, proposer_epoch)` before returning the rejection. - -This bidirectional convergence ensures that even if the leader starts behind, it catches up within the same `propose()` call and the write succeeds as soon as the last straggler adopts the new epoch. The only cost is the bounded fan-out stall (§2.1). - -### 6.4 Non-voting members must not count toward quorum - -A non-voting catch-up member physically accepts and promises so it can follow the log, but its reply must not count toward the voting quorum. `PxGroup::run_prepare_phase` and `PxGroup::run_accept_phase` only increment `promised`/`accepted` for `RemoteReplicaKind::Real(remote)` when `remote.voting` is true. The local self-count is gated on `replica.voting()`. `PxGroup::propose` and `run_bulk_phase1` reuse `self.quorum()` (voting-only) rather than computing a voting-agnostic threshold. - ---- - -## 7. Group-0 Special Cases - -Group 0 (store 0, group 0) is the system group that stores cluster -topology metadata. It uses the same direct HTTP mutation + -`membership_epoch` fence reconfiguration model (§2) as all other -groups. No joint-consensus primitive is needed. Group 0 stores -cluster topology as KV entries under text-path keys (`/hw/rack/...`, -`/hw/node/...`, `/kv/store/...`, `/kv/group/...`) with JSON -values, written by `HardwareClient` and `PaxosKvClusterMetaClient` in -`crowdb-paxos-kv-client`. It is created via `POST /system/init`. See -`design-crowdb-paxos-kv.md` §3.3, `design-crowdb-paxos-kv-group0.md`, and -`../console/design-crowdb-console.md` §4.3 for details. - -A system embedding `crowdb-paxos-kv`'s primitives may choose to build a -joint-consensus layer on top of the `membership_epoch` fence. - ---- - -## 8. Safety Argument - -The safety property we must preserve through reconfiguration is: - -> **No two values can be chosen at the same slot.** - -### 8.1 The single-epoch case - -For a single fixed `membership_epoch`, standard Paxos safety applies: any two majorities of the voting set intersect. Therefore no two distinct values can both be chosen at the same slot. - -### 8.2 The transition case - -The only way two different values could be chosen at the same slot is if two disjoint quorums from different epochs both accepted. The `membership_epoch` fence prevents this: a `Prepare`/`Accept` reply is only counted toward quorum if the request's epoch exactly matches the acceptor's epoch. Two quorums can never be formed at different epochs for the same slot, because at least one of them would have required an epoch-mismatched reply and that reply would be rejected. - -### 8.3 Why the fan-out window is safe - -During the fan-out, nodes may transiently disagree on the epoch. A leader whose epoch is ahead sends `Prepare`/`Accept` with the higher epoch. Acceptor nodes still on the old epoch reject with `epoch_mismatch` and adopt the higher epoch. The leader's quorum count is therefore stalled until enough nodes have adopted the new epoch. The moment a quorum with the new epoch is reached, all nodes in that quorum share the same membership view, and Paxos safety holds. - -### 8.4 Asymmetric transitions - -The single-degree-at-a-time recommendation is not a safety requirement under the exact-match fence. Even a batch change (multiple voting members added or removed in one fan-out) is safe because the fence only permits quorums within one epoch. The trade-off is a longer propagation window and a correspondingly longer write stall. - ---- - -## 9. Failure During Reconfiguration - -Because membership is persisted per node and not via a consensus log, failure recovery is simpler: - -- **Leader crashes after the first node has been mutated but before the fan-out completes.** The remaining nodes may have different epochs. The new leader, once elected, adopts the highest epoch it observes from any peer and continues the fan-out from the console. Writes self-heal as epochs converge. -- **A new member fails during catch-up.** The operator removes it before flipping `voting` to `true`. Because the member is non-voting, its failure cannot affect quorum safety. -- **Network partition during reconfiguration.** The partition side that cannot reach a quorum of the current voting set cannot choose values. The side with the newer epoch may stall until enough nodes adopt it. The partition heals; no split-brain is possible because the epoch fence prevents cross-epoch quorums. -- **Console crashes mid-fan-out.** The operator re-runs the same mutation on the remaining nodes. The `membership_epoch` bump is idempotent because it is a synchronous local increment; applying the same mutation twice leaves the epoch at the same value. - ---- - -## 10. Tunables and Defaults - -| Parameter | Default | Range | Notes | -| --- | --- | --- | --- | -| `catchup_slack` | 100 slots | 10 – 10 000 | How close to leader's `max_chosen` the new member must be before flipping `voting` | -| `catchup_timeout` | 10 min | 1 min – 24 h | Operator abort threshold for catch-up | -| `lease_duration_ms` | election-profile dependent | — | How long a removed/unreachable leader can remain leader before survivors re-elect | -| `step_down_timeout` | 5 s | 1 s – 60 s | How long the console waits for a new leader after step-down before proceeding | - -**Operational guidance:** - -- Add members one at a time. Wait for the catch-up to finish and `voting` to flip before starting the next member. -- For 3 → 5: do two single-add reconfigurations. -- For 7 → 3: do four single-remove reconfigurations. -- Always step-down a leader before removing it. If the leader is unreachable, proceed and rely on the lease-unrenewable fallback. -- Monitor `epoch_mismatch` responses and `membership_epoch` values during reconfiguration — a sustained `epoch_mismatch` is a sign that the fan-out is still in progress. diff --git a/doc/design/kv/design-crowdb-paxos-kv-rpc-client.md b/doc/design/kv/design-crowdb-paxos-kv-rpc-client.md deleted file mode 100644 index 94a662a02..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-rpc-client.md +++ /dev/null @@ -1,294 +0,0 @@ - - - -# CROWDB - Design: Client-Facing KV RPC - -Depends on: [`design-crowdb-paxos-kv-rpc.md`](design-crowdb-paxos-kv-rpc.md) §3, §5, §10, §11, §12; [`design-crowdb-rpc.md`](../rpc/design-crowdb-rpc.md) §3, §4, §6 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3, §10.1 - -The client-facing KV service (Put/Get/Delete/BatchWrite/Scan/ -JournalScan/CreateSnapshot/ListSnapshots/SnapshotScan/ReleaseSnapshot + -WatchNotify) uses the **crowdb-rpc flatbuffer transport** — the same -engine as the consensus hot path, but on a separate port and with a -dedicated schema (`kv_client.fbs`). The transport selection is -programmatic (`CrowdbClient::with_rpc_transport`). - -## Table of Contents - -- [1. Design Principles](#1-design-principles) -- [2. Message Surface](#2-message-surface) -- [3. Flatbuffer Schema (kv_client.fbs)](#3-flatbuffer-schema-kv_clientfbs) -- [4. Port Allocation](#4-port-allocation) -- [5. Server-Side Handler (PaxosKvRpcService)](#5-server-side-handler-kvrpcservice) -- [6. Transparent Leader-Forwarding](#6-transparent-leader-forwarding) -- [7. WatchNotify Server-Push](#7-watchnotify-server-push) -- [8. Client-Side Transport (PaxosKvRpcTransport)](#8-client-side-transport-kvrpctransport) -- [9. CrowdbClient Transport Selection](#9-crowkvclient-transport-selection) -- [10. WatchNotifyClient](#10-watchnotifyclient) -- [11. Zero-Copy Wrapper Classes](#11-zero-copy-wrapper-classes) -- [12. Error Model](#12-error-model) -- [13. Topology cache eviction](#13-topology-cache-eviction) - ---- - -## 1. Design Principles - -- **Same engine, separate port.** The client-facing crowdb-rpc server - binds `rpc_port + 200` (the `KV_CLIENT_RPC_BASE` offset). The - consensus crowdb-rpc server binds `rpc_port + 100`. Both run - simultaneously on the same node. -- **Same retry/topology/NotLeaderHint logic.** The client-side - transport returns the existing crowdb-rpc response types - (`KvResponse`, `KvScanResponse`, `KvJournalScanResponse`), so the - retry loop, topology cache, and `NotLeaderHint` handling in - `CrowdbClient` are unchanged — only the wire send changes. -- **Fire-and-forget WatchNotify.** The server pushes `FBWatchNotify` - frames to subscribed clients via `RpcClient::send` (no response - expected). Dead watchers are lazily removed on the first failed - push. - -## 2. Message Surface - -10 unary request/response pairs + 4 WatchNotify frames: - -- **Put**: `FBKvSetRequest` → `FBKvResponse` -- **Get**: `FBKvGetRequest` → `FBKvResponse` -- **Delete**: `FBKvDeleteRequest` → `FBKvResponse` -- **BatchWrite**: `FBKvBatchWriteRequest` → `FBKvResponse` -- **Scan**: `FBKvScanRequest` → `FBKvScanResponse` -- **JournalScan**: `FBKvJournalScanRequest` → `FBKvJournalScanResponse` -- **CreateSnapshot**: `FBCreateSnapshotRequest` → `FBCreateSnapshotResponse` -- **ListSnapshots**: `FBListSnapshotsRequest` → `FBListSnapshotsResponse` -- **SnapshotScan**: `FBSnapshotScanRequest` → `FBSnapshotScanResponse` -- **ReleaseSnapshot**: `FBReleaseSnapshotRequest` → `FBReleaseSnapshotResponse` -- **WatchSubscribe**: `FBWatchSubscribe` (fire-and-forget, no response) -- **WatchUnsubscribe**: `FBWatchUnsubscribe` (fire-and-forget, no response) -- **WatchNotify**: `FBWatchNotify` (server→client push, fire-and-forget) -- **WatchNotifyError**: `FBWatchNotifyError` (server→client push, fire-and-forget) - -## 3. Flatbuffer Schema (kv_client.fbs) - -The schema lives in `lib/crowdb-protocol/src/fbs/kv_client.fbs`. Codegen -runs via `crowdb-protocol`'s `build.rs`, producing `kv_client_generated.rs` -re-exported as `crowdb_protocol::kv_client_fb`. - -- **`[[ubyte]]` workaround.** Flatbuffers does not support nested - vectors (`[[ubyte]]`). The `FBBytes` wrapper table (`{ data: [ubyte] }`) - is used for byte-vector fields in `FBWatchNotify` (`keys`, `values`). -- **`FBKvClientRetCode` enum.** Maps 1:1 to the `KvErrorCode`: - `Success`, `NotLeader`, `Unavailable`, `Internal`, `JournalScanGcGap`, - `InvalidArgument`. -- **`FBReadMode` enum.** `Linearizable` / `MinSlot`, matching the - `ReadMode`. -- **`forwarded: bool` field.** On `FBKvGetRequest`, `FBKvScanRequest`, - and `FBKvJournalScanRequest` — the loop-guard for transparent - leader-forwarding (the `forwarded` field replaces the - `x-crowdb-paxos-kv-forwarded` metadata header). - -## 4. Port Allocation - -The client-facing crowdb-rpc port is derived from the base port: - -``` -client_rpc_port = rpc_port + (KV_CLIENT_RPC_BASE - PAXOS_KV_SERVER_RPC_BASE) - = rpc_port + 200 -``` - -This is parallel to the consensus-side offset (`rpc_port + 100`). The -`KV_CLIENT_RPC_BASE` constant lives in `crowdb-protocol::ports`. The port -mapping is registered as `KvServerClientRpc` in the port-claim registry. - -## 5. Server-Side Handler (PaxosKvRpcService) - -`PaxosKvRpcService` (in `lib/crowdb-paxos-kv/src/rpc/kv_rpc_service.rs`) holds -`Arc` + `Handle` (tokio runtime) + `Arc`. -`register_handlers` wires one handler per request `msg_type` into the -client-facing `RpcServer`. - -Each unary handler: -a. Parse request via `flatbuffers::root::Request>(req.control)`. - On parse failure → `submit_error` with `InvalidArgument`. -b. Dispatch to the existing `KvStore` trait methods - (`kv_put`/`kv_get`/`kv_delete`/`kv_batch_write`/`kv_scan`/ - `kv_journal_scan`/`kv_create_snapshot`/`kv_list_snapshots`/ - `kv_snapshot_scan`/`kv_release_snapshot`) — the same methods the - crowdb-rpc handler calls. The async path spawns a tokio task via - `self.rt.spawn` and submits the response from the task. -c. Build the response flatbuffer via `FlatBufferBuilder`, `finish`, - `submit_response`. - -Server startup: `start_client_rpc_server` (in `kv_server.rs`) creates -the `RpcServer`, binds the client-facing port, constructs the -`PaxosKvRpcForwarder` + `PaxosKvRpcService`, registers handlers, and stores -the server state in `PxKvStore::client_rpc_server_state`. `main.rs` -calls it after `start_rpc_server`. `shutdown_server` stops both -crowdb-rpc servers. - -## 6. Transparent Leader-Forwarding - -`Get`/`Scan`/`JournalScan` preserve the forward step (linearizable -reads only; `MinSlot` served locally). The loop-guard -`x-crowdb-paxos-kv-forwarded` metadata header becomes the `forwarded: bool` -field on the request flatbuffer. The handler: - -a. If `read_mode == Linearizable && !req.forwarded` and - `forward_target_for(group_id)` returns `Some(endpoint)` → re-issue - the request to the leader via `PaxosKvRpcForwarder`. Set - `forwarded = true` on the re-issued request. On success, submit the - leader's response. On forward failure, serve stale local + set - `not_leader_hint = endpoint`. -b. Else serve locally. - -`PaxosKvRpcForwarder` lives in `crowdb-paxos-kv` itself (not `crowdb-paxos-kv-client`) -to avoid a crate cycle. It holds an `Arc` + `Arc` -+ connection cache, builds the request flatbuffer with -`forwarded = true`, calls `rpc.call()`, and returns the raw response -control buffer. - -## 7. WatchNotify Server-Push - -`WatchSubscribe` handler: -a. Parse `FBWatchSubscribe` → `group_id`, `prefix`. -b. `store.get_group(group_id)`. If none → push `FBWatchNotifyError` - via `RpcClient::send` on the inbound connection, error="group not - found". No `submit_response` (fire-and-forget). -c. If `!group.local_replica().is_leader()` → push - `FBWatchNotifyError` with `not_leader_hint = group.leader_endpoint()`. -d. Else → build `Connection` via `from_handle`, register it in the - group's `WatchRegistry` with a `CrowdbRpcPushTarget` - (`(Connection, Arc, Arc)`). - -`WatchRegistry` uses a `PushTarget` enum: -- `CrowdbRpc` — `Arc` (crowdb-rpc push path). - -`emit` builds `FBWatchNotify` flatbuffer + `rpc.send` (fire-and- -forget). On `ConnectionClosed`/`ConnectionError` → increment -`closed_watchers` (lazy cleanup; the watcher is removed on the next -emit pass or via the safety-net poller). - -Connection-close cleanup: crowdb-rpc does not deliver a connection-close -callback to the handler. The `WatchRegistry::emit` path detects a dead -push target when `RpcClient::send` returns `ConnectionClosed`. The -safety-net poller covers missed notifications during the gap. - -## 8. Client-Side Transport (PaxosKvRpcTransport) - -`PaxosKvRpcTransport` mirrors `PxRpcTransport`: it holds `Arc`, -`Arc`, a shared `ConnectionPoolIndex`, and an atomic request-ID -source. `conn_for` derives the client-facing crowdb-rpc port from the base -port via the `KV_CLIENT_RPC_BASE` offset. Pool selection returns its -generation, and retryable errors invalidate only that generation. - -Callers with concurrent bulk scans and latency-bounded point reads may opt -into `with_isolated_point_reads`, or construct `CrowdbKvClient` with -`new_with_isolated_point_reads`. Get replies then use one additional connection -per endpoint, sharing the existing RPC workers, request-ID source, completion -pool, tunables and deadlines. Scans and mutations retain their ordinary pool. -Errors invalidate the selected pool's exact generation; explicit reconnect -clears both pools. The default constructor retains its existing connection -behavior. ChunkDB enables this mode so actual metadata point reads do not wait -behind an incomplete background scan response. Isolation does not cache success -or convert rejection, transport failure or an expired caller deadline to Ready. - -Unary methods (`send_put`, `send_get`, `send_delete`, `send_batch_write`, -`send_scan`, `send_journal_scan`): build request flatbuffer → -`rpc.call` → await → parse via `Ref` wrapper → map to the existing -crowdb-rpc response types (`KvResponse`, `KvScanResponse`, -`KvJournalScanResponse`). `NotLeaderHint` parsed from the -`not_leader_hint` string field → fed into `CrowdbClient`'s existing -retry + topology-cache logic. - -## 9. CrowdbClient Transport Selection - -`CrowdbClient` holds an `Option>` field -(`rpc_transport`). `with_rpc_transport(transport)` sets it. Each public -method (`put`/`get`/`delete`/`batch_write`/`scan`/`scan_count`/ -`journal_scan`) checks `self.rpc_transport` first: when set, delegate -to the transport's `send_*` (with the existing retry/topology/ -`NotLeaderHint`/metrics wrapping — only the wire send changes). - -## 10. WatchNotifyClient - -`WatchNotifyClient` uses the crowdb-rpc transport path: - -- **crowdb-rpc path**: persistent - connection + client-side handler. The reader loop resolves the leader, - opens a connection via `PaxosKvRpcTransport::get_conn`, registers - handlers for `FBWatchNotify` + `FBWatchNotifyError` via - `RpcClient::register_handler`, sends `FBWatchSubscribe` as - fire-and-forget `send()`. On `FBWatchNotifyError` with non-empty - `not_leader_hint` → `topology.set_leader` + reconnect signal. A - periodic liveness check (fire-and-forget ping every 5s) detects - dead connections — crowdb-rpc does not deliver a connection-close - callback. - -## 11. Zero-Copy Wrapper Classes - -`FBKvResponseRef`, `FBKvScanResponseRef`, `FBKvJournalScanResponseRef`, -`FBCreateSnapshotResponseRef`, `FBListSnapshotsResponseRef`, -`FBSnapshotScanResponseRef`, `FBReleaseSnapshotResponseRef`, -`FBWatchNotifyRef` — zero-copy views over the response flatbuffer -control buffer. Each provides null-safe accessors (`valid()`, -`ret_code()`, `error_msg()`, `ok()`, etc.). See -[`design-crowdb-rpc.md`](../rpc/design-crowdb-rpc.md) §6 for the wrapper -convention. - -## 12. Error Model - -`FBKvClientRetCode` → `KvErrorCode` mapping: - -- `Success` → `KvErrorNone` -- `NotLeader` → `KvErrorNotLeader` -- `Unavailable` → `KvErrorUnavailable` -- `JournalScanGcGap` → `KvErrorJournalScanGcGap` -- `Internal` / `InvalidArgument` / unknown → `KvErrorInternal` - -`RpcError` → `Error` variants: -- `ConnectionClosed` → retry on next connection (same as crowdb-rpc - `Unavailable`). -- `Timeout` → `Error::Transport` (caller retry budget). -- `SendQueueFull` → retry with backoff. -- `RegistrationFailed`/`AllDown`/`InvalidArg` → non-retryable - `Error::Transport`. -- `NotLeaderHint` is a protocol-level response (`not_leader_hint` - string + `FBKvClientRetCode::NotLeader`), not a transport error — - the client's retry/topology-cache logic is unchanged. -- `JournalScanGcGap` → `Error::JournalScanGcGap` (caller falls back - to a full-scan rebuild). - -## References - -- [`design-crowdb-paxos-kv-rpc.md`](design-crowdb-paxos-kv-rpc.md) — consensus RPC wire - protocol (same engine, different port + schema). -- [`design-crowdb-rpc.md`](../rpc/design-crowdb-rpc.md) — RPC engine - overview (framing, connection pool, handler pattern, wrapper - convention). -- [`design-crowdb-paxos-kv-watch-notify.md`](design-crowdb-paxos-kv-watch-notify.md) — - WatchNotify design (trie, prefix matching, safety-net poller). - -## 13. Topology cache eviction - -Within one operation, distinct leader hints are followed immediately. Repeated -or cyclic hints consume the configured retry budget and use the existing -election wait and topology refresh rather than issuing an unbounded RPC loop. -Scan and journal continuations reset this redirect history after a successful -page. Conditional writes retain their original mutation identity on retries. - -After a transport failure or leadership rejection without a usable hint, -discovery searches the target group across management seeds. A reachable seed -that repeats the rejected RPC endpoint cannot hide another member's replacement -route. Concurrent refreshes share the existing refresh gate and interval; a -newer cached route is reused without another fetch. If no replacement is known, -the operation retains its original bounded election wait/backoff and retry -budget. Discovery never establishes authority for a mutation; the destination -still applies its normal leadership and conditional-write checks. - -`TopologyCache` publishes one immutable `TopologySnapshot`. Every group route -contains its leader, replica endpoints, read cursor, endpoint statistics, and -write-slot high-watermark. A refresh builds a complete replacement snapshot; -leader hints and refreshes use compare-and-swap so stale work cannot overwrite -a newer route generation. Eviction retires the complete group route, so a -reused group ID starts with fresh high-watermark and selection state. - -- **I1 — Cache eviction on topology change**: When a group disappears from - topology, its complete route generation is evicted atomically. diff --git a/doc/design/kv/design-crowdb-paxos-kv-rpc.md b/doc/design/kv/design-crowdb-paxos-kv-rpc.md deleted file mode 100644 index 7bb87353d..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-rpc.md +++ /dev/null @@ -1,550 +0,0 @@ - - - -# CROWDB - Design: RPC Wire Protocol - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §2, §3, §9.2, §10 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3, §9.2, §10.1 - -This document defines the wire-serialization contract for all -node-to-node and client-to-node RPC communication. The consensus hot -path (Prepare/Promise, Accept/Accepted, PreVote/RequestVote, -Heartbeat/StepDown, ChosenNotification, FetchGap) uses the -**crowdb-rpc flatbuffer transport** — a custom epoll/kqueue-based RPC -library with flatbuffer framing and request/response correlation via -oneshot channels. Client-to-node KV RPCs (Get/Set/Delete/Batch/Scan) -use the same crowdb-rpc engine on a separate port with a dedicated -schema — see [`design-crowdb-paxos-kv-rpc-client.md`](design-crowdb-paxos-kv-rpc-client.md) -for the client-facing transport design. Every message carries -a `version: u32` field for forward/backward compatibility; no -`required` fields; field numbers are append-only. - -## Table of Contents - -- [1. Design Principles](#1-design-principles) -- [2. Message Surface](#2-message-surface) -- [3. Transport Architecture — crowdb-rpc for Consensus](#3-transport-architecture--crowdb-rpc-for-consensus) -- [4. Server-Side Handler (PxRpcService)](#4-server-side-handler-pxrpcservice) -- [5. Client-Side Transport (PxRpcTransport)](#5-client-side-transport-pxrpctransport) -- [6. RPC Routing in PxRemoteReplica](#6-rpc-routing-in-pxremotereplica) -- [7. Flow Control and Parallelism](#7-flow-control-and-parallelism) - - [7.1 Quorum Short-Circuit](#71-quorum-short-circuit) - - [7.2 RPC Deadline](#72-rpc-deadline) -- [8. Paxos Error Model](#8-paxos-error-model) -- [9. Version Compatibility Rules](#9-version-compatibility-rules) -- [10. Flatbuffer Schema (kv_consensus.fbs)](#10-flatbuffer-schema-kv_consensusfbs) -- [11. Zero-Copy Wrapper Classes](#11-zero-copy-wrapper-classes) -- [12. Port Allocation](#12-port-allocation) -- [13. Flatbuffer `bytes` Field Mapping — `bytes::Bytes`](#13-flatbuffer-bytes-field-mapping--bytesbytes) - ---- - -## 1. Design Principles - -1. **All RPCs over crowdb-rpc.** The Paxos hot - path (Prepare/Accept/Heartbeat/Chosen/FetchGap) runs on the - crowdb-rpc flatbuffer transport for zero-copy framing and - epoll/kqueue I/O efficiency. Client-facing KV RPCs (Get/Set/Delete/ - Batch/Scan/Watch) run on the same crowdb-rpc engine on a separate - port with a dedicated schema for client library ergonomics - and `Bytes` mapping. -2. **Pipelined unary calls on persistent connections.** Steady-state - request-response traffic (`Accept`) multiplexes over - pipelined unary `call()`s on a persistent crowdb-rpc connection per - `(group_id, peer_id)` pair. Connection setup cost is paid once per - server lifetime. Fire-and-forget frames (`ChosenNotification`, - `BatchChosenNotification`) use `send_one_way()` without registering - a reply completion or timeout. They share the persistent connections. -3. **Heartbeats on the same connection.** Steady-state `Heartbeat` - traffic flows over the same crowdb-rpc connection as `Accept` and - `Chosen`. The crowdb-rpc transport multiplexes frames at the I/O - layer (epoll/kqueue), so heartbeats are not blocked behind data - frames the way they would in a bidi stream's FIFO send-half. - The term fence handles cross-term reordering; same-term - heartbeat/accept mutate independent state. -4. **No `required` fields.** All flatbuffer fields are optional or - have sensible defaults. A missing `version` field defaults to `0`. -5. **Field numbers are append-only.** Once assigned, a field number is - never reused for a different semantic meaning. This is a hard rule - for rolling upgrades. -6. **Plaintext in P1/P4; TLS hooks reserved.** The transport layer is - plaintext TCP loopback in tests and P1/P4 integration. TLS config - slots exist in the service builder but are unimplemented. - ---- - -## 2. Message Surface - -The wire protocol defines four classic Paxos messages (P1 M2): -`Prepare`, `Promise`, `Accept`, `Accepted`, sufficient to run a full -classic Paxos round across a real network boundary. Election messages -(`Heartbeat`, `RequestVote`, `PreVote`, `StepDown`) and the -`ChosenNotification` / `BatchChosenNotification` / `FetchGap` frames -are added in P1 M3; snapshot and client RPCs land in P4. - -Key reusable sub-message: `AcceptedValue` carries `(slot, round, -leader_id, term, payload)`. Used in `Accept` requests and `Promise` -responses for classic Paxos value-recovery. The `payload` is opaque -bytes in P1 M2; `kind` discrimination (empty = `NoOp`, non-empty = -`Write`) is not a wire field. `ConfigChange` and `DedupCheckpoint` -kinds are designed but not yet implemented. - -The full flatbuffer schema is in `lib/crowdb-protocol/src/fbs/kv_consensus.fbs`; -the crowdb-rpc schema definitions for client KV RPCs are in -`lib/crowdb-paxos-kv/src/rpc/fbs/`. This doc covers design decisions only. - ---- - -## 3. Transport Architecture — crowdb-rpc for Consensus - -The consensus RPC path uses the R104 `crowdb-rpc` engine: a C++ -epoll/kqueue transport with flatbuffer framing and request/response -correlation. The Rust facade (`crowdb-rpc-ffi`) exposes `RpcServer` -(listen + register handlers), `RpcClient` (connect + `call()` for -request-response, `send_one_way()` for fire-and-forget), and `Connection` -(per-peer persistent connection). - -**Why crowdb-rpc for consensus:** - -- **No per-stream lock.** The crowdb-rpc transport handles - concurrent frames without a per-stream lock, avoiding the ~17% - throughput loss that per-stream serialization would cause at 2T:1C - write workload. The epoll/kqueue I/O loop processes frames - concurrently. -- **Zero-copy framing.** Flatbuffer responses are read in-place via - `FBRef` wrappers — no deserialization into owned types on the - hot path. -- **Fire-and-forget support.** `ChosenNotification` and - `BatchChosenNotification` are sent via `send()` with a no-op - completion callback — no oneshot channel allocation, no reply - correlation overhead. - -**Client-facing KV RPCs:** Client-facing KV RPCs (`Get`, `Set`, `Delete`, -`Batch`, `Scan`, `Watch`) run on the same crowdb-rpc engine on a separate -port with a dedicated schema. The client library -(`crowdb-paxos-kv-client`) uses crowdb-rpc for retry, topology cache, and -`NotLeaderHint` handling. The crowdb-rpc server also serves -bounded snapshot-session handlers for unpublished new-member install. - -Follower-forwarded Get, Scan and JournalScan responses restore the caller's -control-table RPC `id` before returning across the original connection. -The forwarding connection uses its own counter; the business `request_id` -remains unchanged. Error responses with a leader hint preserve this same -distinction, including when the business ID is zero. - -**Connection model:** Each `PxKvStore` runs one `RpcServer` on the -crowdb-rpc port (derived from the base port via a fixed offset, see -§12). The shared `PxRpcTransport` holds one `RpcClient` and a lock-free RCU -connection-pool index. Each endpoint entry is an immutable generated pool; -connections are established outside the index and installed with -compare-and-swap. All `PxRemoteReplica` instances in the store share the same -transport. - ---- - -## 4. Server-Side Handler (PxRpcService) - -`PxRpcService` is the crowdb-rpc handler set for consensus RPCs. It -holds `Arc` + a tokio `Handle` (for spawning async -handlers). `register_handlers` wires one handler per `msg_type` into -the `RpcServer`: - -- `EPrepareRequest` → `handle_prepare` — dispatches to - `PxLocalReplica::on_prepare`, builds `FBPromiseResponse`. -- `EAcceptRequest` → `handle_accept` — dispatches to - `PxLocalReplica::on_accept`, builds `FBAcceptedResponse`. -- `EPreVoteRequest` → `handle_pre_vote` — dispatches to - `PxLocalReplica::on_pre_vote`, builds `FBPreVoteResponse`. -- `ERequestVoteRequest` → `handle_request_vote` — dispatches to - `PxLocalReplica::on_request_vote`, builds `FBRequestVoteResponse`. -- `EHeartbeatRequest` → `handle_heartbeat` — dispatches to - `PxLocalReplica::on_heartbeat`, builds `FBHeartbeatResponse`. -- `EStepDownRequest` → `handle_step_down` — dispatches to - `PxLocalReplica::on_step_down`, builds `FBStepDownResponse`. -- `EChosenNotification` → `handle_chosen_notice` — fire-and-forget, - updates the learner's chosen frontier, no response sent. -- `EBatchChosenNotification` → `handle_batch_chosen` — fire-and-forget, - batch frontier update, no response sent. -- `EFetchGapRequest` → `handle_fetch_gap` — looks up the accepted - value at the requested slot, builds `FBFetchGapResponse` (or - `NotFound` error if the slot is not yet chosen). - -Each handler reads the request flatbuffer directly via -`flatbuffers::root::Request>(req.control)`, dispatches to the -existing consensus logic on `PxLocalReplica`, and builds the response -flatbuffer via `FlatBufferBuilder`. The response is submitted via -`server.submit_response(conn_handle, &ctrl, None, msg_type, req_id)`. - -**Malformed-frame rejection:** when `flatbuffers::root` fails (wrong -table type, corrupted bytes) or a required field is missing (e.g. -`FBAcceptRequest.value` is `None`), the handler calls `submit_error` -with `FBKvRetCode::InvalidArgument` before any consensus logic runs. -`submit_error` builds a `FBPromiseResponse` buffer (shared by all error -paths) and submits it with the response `msg_type`. The client's -`send_*` method parses the response as the expected `FBResponseRef`; -since `FBPromiseResponse` has a different vtable layout, `valid()` -returns `false` and the client maps it to -`PxReplicaError::Internal("response malformed")`. Test coverage: -`malformed_accept_request_is_rejected_by_rpc_boundary` in -`paxos_error_test.rs` exercises both rejection paths (wrong table type + -missing `value`) via the `test-util`-gated `send_raw_request` on -`PxRpcTransport`. - ---- - -## 5. Client-Side Transport (PxRpcTransport) - -`PxRpcTransport` is the shared client-side transport for outbound -consensus RPCs. It holds: - -- `server: Arc` — the local server handle (for connection - attachment). -- `rpc: RpcClient` — the client facade for `call()` / `send()`. -- `connections: ConnectionPoolIndex` — per-endpoint immutable connection - pools carrying a generation. `conn_for(endpoint)` lazily connects outside - the index and atomically installs one completed pool. - -**Request-response path (`send_prepare`, `send_accept`, etc.):** -Builds the flatbuffer request via `FlatBufferBuilder`, calls -`rpc.call(server, conn, req_id, control, None, msg_type)`, which -returns a `CallFuture`. The future resolves to a `Response` containing -the control buffer. The response is parsed via the zero-copy -`FBResponseRef` wrapper (§11). - -**Fire-and-forget path (`send_chosen`, `send_batch_chosen`):** Builds -the flatbuffer, calls `rpc.send_one_way(server, conn, req_id, control, -None, msg_type)`. The frame carries the one-way flag and creates no -pending response entry, completion slot, or reply deadline. Successful -submission does not acknowledge remote delivery or apply. Send failures -are returned for caller-side observability but treated as best-effort; -the existing gap recovery handles missed notifications. - -**Port derivation:** `conn_for(endpoint)` parses the server endpoint -(host:port) and connects to `port + RPC_PORT_OFFSET` (see §12). - -Retryable failures invalidate only the exact generation selected for the -failed request. A late failure from an older connection therefore cannot -discard a replacement pool. - ---- - -## 6. RPC Routing in PxRemoteReplica - -`PxRemoteReplica` routes all RPCs through crowdb-rpc. Each -`ReplicaClient` trait method (`send_prepare`, `send_accept`, -`send_pre_vote`, `send_request_vote`, `send_heartbeat`, -`send_step_down`) delegates to the transport with a -`tokio::time::timeout` wrapper and records metrics. - -`send_chosen_notice` and `send_batch_chosen_notice` use -the transport — fire-and-forget via `transport.send_chosen()` / -`transport.send_batch_chosen()`. - -`send_fetch_gap` delegates to -`transport.send_fetch_gap()` (request-response). The `group_fetchgap` loop -also uses `remote.rpc_transport()` to route FetchGap through the -transport. - ---- - -## 7. Flow Control and Parallelism - -The crowdb-rpc transport enables the parallel-slot pipelining described -in `design-crowdb-paxos-kv-slot.md` §5: - -- **Multiple in-flight `Accept` frames per peer.** Each `send_accept` - call creates a new `CallFuture` and returns immediately; the caller - does not block waiting for the peer's reply. This allows slot N+1's - `Accept` to be sent before slot N's `Accepted` response arrives. -- **Connection-level flow control.** The crowdb-rpc engine manages a - send queue per connection; when the queue is full, `send()` / - `call()` returns `SendQueueFull`, which the caller treats as - retryable. -- **Reconnect safety.** On transport failure, the connection is - dropped from the cache; the next `conn_for()` lazily reconnects. - The proposer treats this as retryable and re-sends after reconnect. - -### 7.1 Quorum Short-Circuit - -`run_prepare_phase` and `run_accept_phase` fan out to all peers but -do not wait for all replies. A `FuturesUnordered` (local + each remote, -tagged with `(voting, kind)`) is drained via `StreamExt::next`; the -phase returns as soon as `accepted >= quorum` AND the local reply has -been folded (the W6 invariant: `Chosen`/`Proceed` is never returned -before the local WAL persist / CAS reply is counted). The -still-pending futures are moved into a detached drain task that -continues folding for side effects only: a late `TermStale` triggers -`become_follower`; a late `EpochMismatch` adopts the responder epoch. -The drain captures `self_weak` (upgrades to `Arc`), so a -dropped group lets the upgrade fail and the task exit cleanly. It -honors `tenure_cancel` so a step-down aborts the drain. - -In a 3-node group (quorum = local + 1), the proposal latency is the -quorum-th-fastest reply, not `max(all peers)`. A slow but connected -follower no longer drags every write. Failure detection is preserved. -It just moves off the latency path. - -### 7.2 RPC Deadline - -Each `send_*` method wraps its `CallFuture` await with -`tokio::time::timeout(rpc_timeout)`. On expiry the caller surfaces a -typed retryable error. A connected-yet-unresponsive peer (GC pause, -half-open socket, overloaded server) is surfaced as a retryable -failure within `peer_rpc_timeout_ms` (default 2000 ms, -aligned with the 2 s election max) rather than blocking the fan-out -indefinitely. - ---- - -## 8. Paxos Error Model - -The error categories below determine whether the proposer retries the -same slot, runs classic repair, moves the client operation to a new -slot, redirects the client, or returns a retryable error. - -### 8.1 Error Categories - -- **`NotLeader`** — KV request reaches follower. Client retries with - leader hint. -- **`PrepareRejected`** — Phase 1 blocked by higher promised ballot. - Retry same slot with a higher ballot. -- **`AcceptRejected`** — Phase 2 blocked by higher promised ballot. - Run classic Phase 1 repair on the same slot. -- **`ForeignValueChosen`** — Phase 1 adopts another value or chosen - value differs from client value. Learn the chosen value, then retry - the client operation on a new slot. -- **`QuorumUnavailable`** — Not enough reachable voting peers. Retry - same slot until budget exhausted. -- **`TransportFailure`** — RPC timeout, connect failure, or - unavailable peer. Retry same slot; repeated failures become - `QuorumUnavailable`. -- **`Busy`** — Admission/retry budget exhausted. Client retries with - backoff. -- **`InternalInvariantViolation`** — Missing required value, invalid - state transition. Fail test fast; production maps to internal error. - -### 8.2 Retry Rules - -- **Prepare rejection:** retry Phase 1 for the same slot using a - ballot above the highest observed rejected ballot. Do not move to a - new slot. -- **Accept rejection:** run classic Phase 1 for the same slot. If - Phase 1 discovers a foreign accepted value, repair and learn it - before the client operation moves to a new slot. -- **Transport or quorum failure:** retry the same slot with the same - ballot until the retry budget is exhausted. Increasing the ballot is - only required after a higher-ballot rejection. -- **Foreign value:** adopt it for the current slot. If not the client - value, retry the client operation on a later slot only after the - foreign value is learned. - -Retry backoff (`retry_backoff`) is `base * 2^attempt` with ±50% jitter -(`base * multiplier / 1000` where `multiplier ∈ [500, 1500]`), using a -thread-local `XorShift64` PRNG seeded from monotonic nanos. This -decorrelates retry storms across replicas that collide on the same -slot. The admission permit is held during backoff (releasing it would -let a duplicate `(client_id, seq)` admit concurrently). - -### 8.3 RPC Mapping - -Consensus responses carry rejected metadata directly in the -flatbuffer response fields (`rejected`, `term_stale`, `epoch_mismatch` -booleans + the term/epoch fields). RPC transport failures are mapped -by the caller into `TransportFailure`. Malformed consensus RPCs map -to `InternalInvariantViolation` at the Paxos model level and to -`FBKvRetCode::InvalidArgument` at the crowdb-rpc boundary. - ---- - -## 9. Version Compatibility Rules - -1. **Sender rule:** every message sets `version = 1` (the initial wire version). -2. **Receiver rule:** decode must accept any `version <= max_supported`. Unknown fields are ignored (flatbuffer default behavior). -3. **Upgrade rule:** new fields are added with new field numbers; old fields are never removed or renumbered. -4. **Field-number freeze per message:** field numbers are frozen once - assigned. The frozen ranges are documented in the schema file - comments. Future versions may add new fields starting at the - next free field number. - ---- - -## 10. Flatbuffer Schema (kv_consensus.fbs) - -`lib/crowdb-protocol/src/fbs/kv_consensus.fbs` mirrors the consensus -messages from `pxos.fbs`, following the `diskdb.fbs` conventions -proven by R115: - -- `include "common_type.fbs";` for `FBInt128`. -- `namespace crowdb.kv_consensus.proto;` -- Every request/response table carries `id` (request_id) + - `rpc_create_nano` as its first two fields. -- `FBAcceptedValue` is a table (has a `payload: [ubyte]` vector, - which requires a vtable). -- `FBAcceptRequest` carries consensus value and fence fields only. Client - request identities remain leader-local and are not follower wire metadata. -- `NotLeaderHint` is NOT a separate message — it is fields on the - response tables (`not_leader_hint:string` + `term:uint64` + - `membership_epoch:uint64`). - -Message type IDs registered in `msg_type.fbs` (1000–1099 range): - -``` -EPrepareRequest = 1000, -EPromiseResponse = 1001, -EAcceptRequest = 1002, -EAcceptedResponse = 1003, -EPreVoteRequest = 1004, -EPreVoteResponse = 1005, -ERequestVoteRequest = 1006, -ERequestVoteResponse = 1007, -EHeartbeatRequest = 1008, -EHeartbeatResponse = 1009, -EStepDownRequest = 1010, -EStepDownResponse = 1011, -EChosenNotification = 1012, // fire-and-forget (no response) -EBatchChosenNotification = 1013, // fire-and-forget (no response) -EFetchGapRequest = 1014, -EFetchGapResponse = 1015, -ESnapshotBeginRequest = 1018, -ESnapshotBeginResponse = 1019, -ESnapshotReadRequest = 1020, -ESnapshotReadResponse = 1021, -ESnapshotFinishRequest = 1022, -ESnapshotFinishResponse = 1023, -ESnapshotAbortRequest = 1024, -ESnapshotAbortResponse = 1025, -``` - -Each consensus operation has its own message type. Unary operations use -`RpcClient::call`; chosen notifications use fire-and-forget -`RpcClient::send`. The shared peer connection carries the mixed message types, -and the server dispatches each frame independently by its `msg_type`. - -### 10.1 New-member snapshot lifecycle - -Snapshot bootstrap is a receiver-pulled sequence of bounded unary calls. Begin -pins one immutable engine view and returns a process boot nonce plus monotonic -session number, source membership epoch, engine format, slot/term, total -length, body CRC32C, and a chunk limit no larger than 1 MiB. Read echoes the -identity and requested byte offset; it carries one payload and its CRC32C in -the data buffer. Only the current offset or an exact retry of the immediately -previous offset is accepted. Finish requires the final offset; Abort is -idempotent. - -The source admits at most four sessions by default and expires idle sessions -after 30 seconds. Every Read and Finish rechecks the captured membership epoch. -The receiver keeps one Read in flight, advances only after feeding a verified -payload, and may reconnect without changing identity. An expired identity -causes a fresh Begin and importer. Import is restricted to a fresh unpublished -group; learner seeding and group registration occur only after length, CRC, -source Finish, and engine Finish all succeed. The removed single-frame message -IDs 1016 and 1017 are deliberately unsupported. - -**Build integration:** `lib/crowdb-protocol/build.rs` compiles -`kv_consensus.fbs` via `flatc --rust --gen-all` (inlines -`common_type.fbs` so `FBInt128` resolves). The generated module is -re-exported as `crowdb_protocol::kv_consensus_fb`. - ---- - -## 11. Zero-Copy Wrapper Classes - -`lib/crowdb-protocol/src/fb_wrappers/kv_consensus.rs` defines one -`FBRef` struct per response type: - -```rust -pub struct FBPromiseResponseRef<'a> { - root: flatbuffers::Result<'a, FBPromiseResponse<'a>>, -} -impl<'a> FBPromiseResponseRef<'a> { - pub fn new(buf: &'a [u8]) -> Self { Self { root: flatbuffers::root::(buf) } } - pub fn valid(&self) -> bool { self.root.is_ok() } - pub fn request_id(&self) -> Option { self.root.as_ref().ok()?.id() } - pub fn slot(&self) -> Option { self.root.as_ref().ok()?.slot() } - // ... remaining fields -} -``` - -Same pattern for: `FBAcceptedResponseRef`, `FBHeartbeatResponseRef`, -`FBPreVoteResponseRef`, `FBRequestVoteResponseRef`, -`FBStepDownResponseRef`, `FBFetchGapResponseRef`. - -Request wrappers are NOT needed — the server handler reads the request -flatbuffer directly via `flatbuffers::root::Request>(req.control)`. -The client builds requests with `FlatBufferBuilder` and doesn't need -to read them back. - -Edge cases: -- Malformed flatbuffer → `valid()` returns false; the caller treats it - as a transport error (maps to `PxReplicaError::Internal`). -- Missing optional field → accessor returns `None` / default; the - caller handles the missing-field case. - ---- - -## 12. Port Allocation - -The crowdb-rpc port is derived from the base port via a fixed offset: - -``` -PAXOS_KV_SERVER_RPC_BASE = 28001 (rpc port base for KV server) -KV_RPC_BASE = 28101 (crowdb-rpc port base for KV server) -RPC_PORT_OFFSET = KV_RPC_BASE - PAXOS_KV_SERVER_RPC_BASE = 100 -``` - -When a `PxKvStore` binds on port P, the crowdb-rpc server listens -on port `P + 100`. The client transport's `conn_for(endpoint)` parses -the server endpoint and connects to `rpc_port + 100`. - -For ephemeral ports (port 0 in tests), the server binds first, -then `start_rpc_server` reads the actual bound port from -`server_state.listen_addr` and derives the crowdb-rpc port. - -Port constants are in `lib/crowdb-protocol/src/ports.rs` and the -`KvServerPort` enum. - ---- - -## 13. Flatbuffer `bytes` Field Mapping — `bytes::Bytes` - -By default, `flatc` maps flatbuffer `[ubyte]` fields to `Vec`, -making every clone an O(n) heap allocate + memcpy. For hot-path KV -fields that are cloned across retry loops or fanout, this is -avoidable. - -`lib/crowdb-paxos-kv/build.rs` configures selected -flatbuffer `[ubyte]` fields to map to `bytes::Bytes` instead, turning -clones into O(1) atomic ref-count bumps: - -- `AcceptedValue.payload` — Paxos Accept-fanout payload cloned across - N peers. -- `KvSetRequest.key`, `KvSetRequest.value` — single-key write path. -- `KvGetRequest.key` — single-key read path. -- `KvDeleteRequest.key` — single-key delete path. -- `KvBatchItem.key`, `KvBatchItem.value` — batch write path; the - client retry loop clones the entire `items` vec per attempt. -- `KvResponse.value` — read response. -- `KvScanRequest.prefix`, `KvScanRequest.start_after` — scan request. -- `KvScanItem.key`, `KvScanItem.value` — scan response items. - -The client `BatchOp` type (`lib/crowdb-paxos-kv-client/src/client.rs`) also uses -`Bytes` for `key` and `value`, so `BatchOp` → `KvBatchItem` conversion -and the per-retry `items.clone()` are both O(1) per item. - -The server-side `encode_kv_batch_items` (`px_kv_store.rs`) still -flattens `KvBatchItem` fields into a `Vec` consensus payload. -That copy remains (eliminating it requires changing the consensus -payload contract, out of scope). - ---- - -## References - -- `design-crowdb-paxos-kv-leader-election.md` §6 — heartbeat/lease interaction with - stream ordering -- `design-crowdb-paxos-kv-slot.md` §5 — pipelined fanout and per-peer flow control -- `design-crowdb-paxos-kv-observability.md` — write-path phase metrics - (`paxos.propose.e2e.l`, `paxos.classic.prepare.l`, - `paxos.accept.quorum_rpc.l`, `paxos.learn.apply.l`) -- `design-crowdb-rpc.md` §6 — Flatbuffer Wrapper Convention -- `design-crowdb-rpc-diskdb-migration.md` — R115's proven migration pattern diff --git a/doc/design/kv/design-crowdb-paxos-kv-server.md b/doc/design/kv/design-crowdb-paxos-kv-server.md deleted file mode 100644 index 1a9dc07b0..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-server.md +++ /dev/null @@ -1,234 +0,0 @@ - - - -# CROWDB - Design: KV Server - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §15.2 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §15.2 - ---- - -## Table of Contents - -- [1. Overview](#1-overview) -- [2. Design Decisions](#2-design-decisions) - - [2.1 KV engine selection](#21-kv-engine-selection) - - [2.2 Startup ordering](#22-startup-ordering) - - [2.3 Concurrency model](#23-concurrency-model) - - [2.4 HTTP framework: axum](#24-http-framework-axum) - - [2.5 Group lifecycle](#25-group-lifecycle) - - [2.6 Shutdown](#26-shutdown) -- [3. Port Pool](#3-port-pool) -- [4. Group/store cleanup on remove](#4-groupstore-cleanup-on-remove) -- [5. Wipe user data](#5-wipe-user-data) - ---- - -## 1. Overview - -`crowdb-paxos-kv-server` is a thin binary that wires the `crowdb-paxos-kv` library into a -runnable process. It adds CLI argument parsing, an HTTP management -server for runtime topology control, and graceful startup/shutdown -orchestration. The server does **not** contain business logic — all -consensus, KV operations, and replication are handled by the `crowdb-paxos-kv` -library. - -## 2. Design Decisions - -### 2.1 KV engine selection - -`--kv-engine` chooses the `KVEngine` implementation for all groups: - -- **`crowdb-tree` (default)** — durable, file-backed. Each group gets its - own file under `--data-root`, recovered by replaying the WAL on - restart. -- **`memory`** — in-memory, non-durable. Explicit low-durability choice - for tests and dev. - -`--kv-backend` defaults to `block`: the tree uses an aligned block page -store with direct I/O on Linux. `file` explicitly selects buffered file -storage, and `mem-block` explicitly selects an in-memory block device. - -`--wal-backend` defaults to `block-device`: aligned real-file block storage -with direct I/O on Linux and durable synchronization before acknowledgement. -`file`, `uring` and `mem-block` remain explicit alternatives. File-backed -device simulation still traverses the filesystem and can incur variable VFS -and filesystem-log synchronization latency. Backend selection does not disable -durable synchronization. - -Backend selection must match an existing data directory. Defaults select the -backend for a new deployment; they do not convert stored tree formats. Existing -file-backend deployments continue to specify `file` explicitly on restart. - -### 2.2 Startup ordering - -The management API starts **before** stores so the server is observable -even if store creation fails. `--root ` is required on every -start; it derives the four node paths via `PaxosKvConfig::apply_root`: -`wal_root = /waldata`, `config_root = /conf`, -`data_root = /ctdata`, `log_dir = /log` (fixed subfolder -names, the only supported layout). `--config ` is optional and -supplies startup tunables in both first-boot and restore modes; when omitted, -tunables come from `PaxosKvConfig::default()`. Explicit CLI settings override -the file, as defined by the shared -[`service configuration design`](../config/design-crowdb-config.md) §3. The -config-file watcher runs only when `--config` is passed. Static settings such -as `server.rpc_workers` take effect after restart. - -Boot has two modes, selected by whether group 0 is on disk -(`restore::group0_exists` checks `/store0/group0`): - -- **Restore mode** — group 0 is on disk. `restore::scan_local_groups` - enumerates every `store{S}/group{G}` directory under ``, - and `restore::load_local_groups` creates each `PxKvStore` and loads - every group via `create_group_with_wal` (replays the WAL, opens the - crowdb-tree engine, and applies persisted membership from - `conf/node-config.json` — including remote-replica endpoints). The - local `replica_id` per group is read from `node-config.json`, falling - back to the `--replica` CLI arg. `--stores`/`--groups` are ignored - (warned); local disk is the source of truth for which stores/groups - this node hosts. After load, `reconcile::reconcile_with_group0` - compares local state against group 0's `/kv/replica/` records: - groups that came up with no remotes (`node-config.json` missing or - stale for them) get their remotes seeded from group 0 via a group - rebuild; groups that already have remotes are verified and any - peer present in group 0 but not wired locally is logged (the live - membership is not forcibly overwritten — it may be legitimately - ahead of group 0 during an in-flight reconfiguration). Reconciliation - is best-effort: if group 0 is unreachable or has no `/kv/replica/` - records, the node continues with local state and retries on the next - restart. -- **First-boot mode** — no group 0 on disk. The server boots empty - (or from `--stores`/`--groups` if given) and the operator calls - `POST /system/init` to create store 0 / group 0. - -A scan IO error is treated as empty (first-boot mode); a failed -`create_group_with_wal` for one group is logged and skipped while the -store still starts with its other groups. - -Creating a group persists its local replica identity and initial membership -before opening the WAL directory. A crash during later WAL/tree initialization -therefore restores the requested identity even when creation never returned or -the deployment manifest still marks that step incomplete. A conflicting -persisted local identity fails creation; it is never overwritten by a retry. - -### 2.3 Concurrency model - -`PaxosKvStoreRegistry` and each `PxKvStore` publish immutable store/group maps -through `ArcSwap`. Readers clone an `Arc` from one snapshot without taking a -shard lock. Replacement uses compare-and-swap, so a reader sees the old or new -group and never a missing intermediate entry; successful replacement cancels -the retired group's tenure exactly once. `PxGroup` rebuilds and republishes a -group when remote membership changes. - -### 2.4 HTTP framework: axum - -Chosen for lightweight async + tower ecosystem fit with tokio. The full -endpoint list is defined in the Axum route handlers and the OpenAPI spec -at `/openapi.json`; this doc covers design decisions only. - -Key endpoint groups: - -- **Store/group CRUD** — `/stores`, `/stores/:sid/groups` for runtime - topology control. -- **Remote replica wiring** — `/stores/:sid/groups/:gid/remotes` for - adding/removing peer replicas. `batch` endpoint accepts another - server's topology export for bulk wiring. -- **Topology export** — `GET /topology` produces a JSON document that - another server can consume to batch-add remotes. -- **System group** — `POST /system/init` bootstraps store 0 + group 0 - on this node. (`POST /topology/finalize` and `GET /topology/ready` - are removed. Persistent topology-record management moves to - `crowdb-paxos-kv-client`'s `PaxosKvClusterMetaClient` / `HardwareClient`. See - [`design-crowdb-paxos-kv-group0.md`](design-crowdb-paxos-kv-group0.md) for the - group-0 sysdata architecture.) The lifecycle endpoints (`add_store`, - `add_group`) are now **internal**; only `crowdb-paxos-kv-client`'s - `PaxosKvClusterAdmin` calls them. See `../console/design-crowdb-console.md` §4.3 for the - full persistent cluster config design. -- **Admin operations** — `step-down` (force leader step-down), `join` - (new-member snapshot join), `flush` (drain the local replica's L0 - memtable into L1 in memory; used by the bench's - `--flush-after-prepopulate` flag and as an admin drain). - -`POST /system/group0-discovery` accepts one to sixteen HTTP management -origins for a process launched before Group 0 exists. The server atomically -persists these connection hints and its keepalive loop discovers Group 0 -before registering. Explicit startup seeds take precedence. Repeating the -same request is safe; malformed origins and embedded credentials are rejected. -These hints survive process restart and never create stores, groups, replicas, -or membership. Without hints or a local Group 0 replica, registration waits. - -The service instance identity is persisted with the node's fixed-layout -configuration before opening listeners. Restarts reuse it, including when a -previous process could not unregister. A changed explicit instance or node -identity, or a malformed identity file, fails startup. Different durable roots -receive different generated instance identities; duplicate live node identities -remain ambiguous rather than being merged by discovery. - -### 2.5 Group lifecycle - -Local replicas start as `Follower`; no role assignment needed at group -creation. Leader assignment happens via topology wiring or leader -election. Groups can be added/removed at runtime via the management -API, not just at CLI bootstrap. - -### 2.6 Shutdown - -On SIGINT/SIGTERM, Axum stops accepting new HTTP requests, then -`graceful_shutdown` cascades through each store: - -1. **`PxKvStore::shutdown`** — stops the crowdb-rpc server (cuts frontend - load), then cascades into each group. -2. **`PxGroup::shutdown`** — cancels the tenure token and awaits the - election driver, maintenance loop, FetchGap driver, and coalescer watchdog. - The watchdog keeps only a weak group reference. Each owned task - has a per-layer deadline; a timeout aborts and then joins the task before - the shutdown report is returned. It then closes remote crowdb-rpc - channels and cascades into the local replica. -3. **`PxLocalReplica::shutdown`** — cancels and joins the background apply - loop before calling `KVEngine::flush` (drain L0 - memtable into L1 B+tree, in-memory) then - `KVEngine::persist_snapshot` (write dirty L1 pages + superblock to - the page store, durable). For `InMemKV` both are no-ops. Acceptor - and learner resource release is deferred to `Drop`. - -This ensures in-memory engine state reaches the block file before the -process exits, so `resume_from_slot` is non-zero on restart and WAL -replay can skip the durable prefix. - -## 3. Port Pool - -The `--ports` argument supports comma-separated single ports and -inclusive ranges (e.g. `28,40..50,59`). Duplicates are silently -deduplicated. If `--stores` exceeds the port pool size, the server -exits with an error. When `--ports` is omitted, ports are OS-assigned -(zeros). - -## 4. Group/store cleanup on remove - -The `remove_group` and `remove_store` mgmt handlers delete the WAL + -engine dirs and update `node-config.json` in addition to the -in-memory `PxKvStore::remove_*`. The `node-config.json` update is the -fence against resurrection on restart — without it, a removed group -reappears from persisted config and collides with a new group at the -same `(store_id, group_id)`. - -- **I1 — ID reuse safety**: After a group/store is removed, its ID - can be safely reused. The `node-config.json` update is the fence. - -## 5. Wipe user data - -`POST /stores/{sid}/groups/{gid}/wipe-user-data` drops and recreates -WAL + engine user data for a group, preserving group0 sysdata + -topology. Explicit test-only measurement fixtures use this endpoint between -write-regression sub-tests while retaining their owned topology. Attached -product benchmarks do not reset groups or manage servers. The handler -steps down if leader, deletes the WAL + engine dirs, recreates the -group via `create_group_with_wal` (replays empty WAL → fresh state), -and does `remove_group` + `add_group` (remove-before-add is critical: -`add_group` calls `inherit_local_state_from(prior)` which would -`Arc::clone` the prior's engine + WAL, undoing the wipe). - -- **I2 — Wipe preserves group0**: `wipe-user-data` drops WAL + engine - user data only; group0 sysdata + `node-config.json` + topology are - untouched. diff --git a/doc/design/kv/design-crowdb-paxos-kv-slot.md b/doc/design/kv/design-crowdb-paxos-kv-slot.md deleted file mode 100644 index 9aea48d49..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-slot.md +++ /dev/null @@ -1,671 +0,0 @@ - - - -# CROWDB - Design: Slots — Parallel Pipelining & Acceptor Retention - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §5.1, §6.5, §7.3 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §5.1, §6.5, §7.3 - -This document covers two aspects of CROWDB's slot mechanism: - -- **Part A (§1–§14): Parallel Slot Pipelining** — the consensus protocol that distinguishes CROWDB from Raft-based KV systems. How the leader pipelines proposals, how gaps are detected and repaired, how the safe-slot is maintained, and how the system stays correct under concurrent in-flight slots. -- **Part B (§15–§22): Acceptor Retention** — guarded per-slot state, concurrent trim/reclamation and snapshot-backed retention certificates. - -## Table of Contents - -- [1. Why Parallel Slots](#1-why-parallel-slots) -- [2. Concepts and Invariants](#2-concepts-and-invariants) -- [3. Slot Lifecycle on the Leader](#3-slot-lifecycle-on-the-leader) -- [4. Sliding Window and Backpressure](#4-sliding-window-and-backpressure) -- [5. Pipelined Fanout](#5-pipelined-fanout) -- [6. Per-Key Resolved-Slot](#6-per-key-resolved-slot) -- [7. Safe-Slot Computation and Propagation](#7-safe-slot-computation-and-propagation) -- [8. Gap Detection](#8-gap-detection) -- [9. Gap Repair via Classic Paxos](#9-gap-repair-via-classic-paxos) -- [9A. Follower-Side Apply and Catch-up](#9a-follower-side-apply-and-catch-up) -- [10. Timing Diagrams](#10-timing-diagrams) -- [11. Interaction with Snapshot and WAL GC](#11-interaction-with-snapshot-and-wal-gc) -- [12. Tunables and Defaults](#12-tunables-and-defaults) -- [13. Correctness Analysis for Parallel Slot Writes](#13-correctness-analysis-for-parallel-slot-writes) -- [14. Parallel-Slot Linearizability Analysis](#14-parallel-slot-linearizability-analysis) -- [15. Acceptor Storage Requirements](#15-acceptor-storage-requirements) -- [16. Retained Slot Entries](#16-retained-slot-entries) -- [17. Per-Slot State](#17-per-slot-state) -- [18. Lookup and Reclamation](#18-lookup-and-reclamation) -- [19. Acceptor API](#19-acceptor-api) -- [20. Retention Invariants](#20-retention-invariants) -- [21. Storage Cost](#21-storage-cost) -- [22. Retention Limits](#22-retention-limits) -- [23. Server-side Proposal Coalescing](#23-server-side-proposal-coalescing) - ---- - -# Part A: Parallel Slot Pipelining - -## 1. Why Parallel Slots - -A Raft leader cannot acknowledge slot N+1 until slot N has been committed; the log is contiguous, so a single slow follower stalls the entire commit pipeline (head-of-line blocking). Multi-Paxos has no such constraint: each slot is a separate Paxos instance, and a quorum that decides slot N+1 need not include the same nodes that decide slot N. - -CROWDB exploits this by running many slots in parallel on the leader. Throughput is bounded by network and disk bandwidth, not by per-slot serialized round-trips. The price is two-fold: - -- **Gaps.** A slot may remain undecided long after later slots are decided. We need a mechanism to resolve gaps without stalling the hot path. -- **Conservative cross-key reads.** A `Scan` must wait for a no-gap prefix; point reads do not. - -The blind-ops premise from design-crowdb-paxos-kv.md §5.2](design-crowdb-paxos-kv.md) makes the trade-off cheap: out-of-order *apply* is safe because no operation reads before writing. - ---- - -## 2. Concepts and Invariants - -- **I1 — Single slot counter.** On a leader, slot assignment is performed by exactly one logical worker. Two writes never receive the same slot, and no two slots are assigned out of arrival order. -- **I2 — Slot determines linearization.** The slot number assigned to an op is the op's position in the global linearization order (design-crowdb-paxos-kv.md §6.1](design-crowdb-paxos-kv.md)). -- **I3 — Quorum-fsync before ack.** A client write is acked only after a quorum of acceptors (including the leader) have fsynced their `Accepted(slot, ballot, value)` records. This is the durability hook that makes I2 robust to failures. -- **I4 — Apply-order independence for blind ops.** For any key *k*, the engine's final value is `value(max{ slot | slot writes k })`. Apply order between non-overlapping keys is irrelevant; for the same key, the higher slot wins regardless of arrival order ([§13](#13-correctness-analysis-for-parallel-slot-writes)). -- **I5 — Per-key resolved-slot is monotone.** A learner's per-key tracker only ever advances. This is the basis for read-your-writes from followers. -- **I6 — Safe-slot is contiguous.** The cluster-wide safe-slot is the maximum N such that *every* slot ≤ N is chosen and applied on every learner. It is by definition gap-free. - ---- - -## 3. Slot Lifecycle on the Leader - -| State | Trigger to enter | Trigger to leave | Notes | -| --- | --- | --- | --- | -| `Assigned` | client request admitted; counter incremented | leader's WAL fsync of the `PxLogEntry` completes | Visible to no one yet | -| `Proposing` | local fsync done; `Accept` fanned out to followers | quorum of `Accepted` responses received | At this point the value is *chosen* | -| `Chosen` | quorum reached | learner has applied to engine | Ack to client may be sent here (see below) | -| `Applied` | learner.apply() returned | terminal | Used by metrics; safe to evict | -| `OrphanRepair` | leader changed before reaching `Chosen`; repair task takes over | `Chosen` (via repair) | New leader's responsibility | - -**When does the ack happen?** The synchronous path acknowledges after local -application. With `async_engine_apply`, reaching `Chosen` publishes the chosen -frontier and request result, starts an owned apply task, and acknowledges -immediately. Linearizable reads wait for their chosen-slot apply fence before -reading the engine, preserving read-your-writes (see -[§6.1 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#61-linearizable-leader-read)). - -**Eviction.** Acceptor entries and WAL segments become eligible below the snapshot-backed reclamation watermark in §11. Local application alone does not authorize eviction. - ---- - -## 4. Sliding Window and Backpressure - -The number of in-flight slots is capped at the **window size** -(`max_inflight_proposals`, default 32). `InflightAdmission` owns one semaphore. -A permit is acquired before engine admission and slot allocation. For an engine -that returns an admitted apply owner, the permit transfers with that owner to -the chosen apply task and remains occupied through apply completion and the -applied-frontier update, including after an asynchronous `Chosen` response. -Early proposal exit returns the permit; cancellation of a chosen wait does not -cancel its owned native apply. Engines using compatibility admission retain the -permit for the proposal call's duration. Replay and gap repair use their -independent progress admission, so a full proposal window cannot block repair. - -**Admission policy** (`reject_on_inflight_window_full: bool`, internal config, not exposed via CLI): - -- **`false` (default)** — If no permit is available, the caller blocks on `acquire().await` until a permit is freed. This eliminates client-side reject-retry storms under contention (e.g., window=1 with 16 writers). Wait time and queue depth are tracked as metrics (`inflight_total_enqueued`, `inflight_total_wait_us`, `inflight_queue_depth`). -- **`true`** — If no permit is available (`try_acquire` fails), the leader immediately returns `Busy` — a retryable error. No queuing. Used only in tests that need to verify fail-fast behavior. - ---- - -## 5. Pipelined Fanout - -As soon as the leader has fsynced its own copy of slot N, it sends `Accept(N, ...)` to all followers. It does **not** wait for slot N-1, N-2 to reach quorum first. - -**Transport: shared per-peer `crowdb-rpc` connection.** Each Accept is a unary -request correlated by `RpcClient`, so concurrent slots may remain in flight -without a dedicated application-level stream. - -**Per-follower flow control.** `crowdb-rpc` uses the configured bounded send -queue plus a bounded overflow queue drained on writable events. Terminal -queue pressure surfaces as `PxReplicaError::Backpressure`; the proposal retry -budget applies backoff without invalidating the healthy connection. - -**Quorum bookkeeping.** For each in-flight slot, the leader keeps a small bitmap of which peers have `Accepted` it. As soon as a majority is reached (counting itself), the slot transitions to `Chosen`. - -**Out-of-order `Accepted` and `Chosen` notifications are fine.** A follower may respond `Accepted(N+2)` before `Accepted(N)`. Each is processed independently. A follower may receive `Chosen(N+2)` before `Chosen(N)` and apply only the parts safe to apply (per-key tracking handles this). - -**Zero-copy accept path.** `Acceptor::accept` and `ReplicaHandler::on_accept` take `&PxLogEntry` instead of owned `PxLogEntry`. The caller (proposer's `run_accept_phase`, crowdb-rpc `handle_accept_inner`, WAL replay) passes a reference; no clone for the acceptor call. The only clone is inside `inner_accept` for `cas_accepted`, where the slot node must own its copy. `WALRecord::from_accepted` already borrows, so the WAL encode after accept is also zero-copy. With `Bytes` payloads these clones were O(1) ref-count bumps; the signature change makes the zero-copy intent explicit and avoids redundant bumps. - ---- - -## 6. Per-Key Resolved-Slot - -Each learner maintains, per key it has applied, the highest slot that has touched that key. This enables: - -- **Read-your-writes** ([§6.2 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#62-read-your-writes-follower-read)): a follower can serve `Get(k, slot=N)` as soon as `resolved_slot[k] ≥ N`. -- **Linearizability** ([§14](#14-parallel-slot-linearizability-analysis)): per-key tracking makes "highest slot wins" correct under out-of-order apply. - -**Update rule.** On `apply(slot, batch)`: for each `(k, op, v?)` in the batch, if `slot > resolved_slot[k]`, update `resolved_slot[k] = slot` and write/tombstone `v`. Otherwise drop. Then advance `max_applied` and the contiguous-applied frontier. - ---- - -## 7. Safe-Slot Computation and Propagation - -The safe-slot is the cluster-wide **contiguous applied frontier**: the maximum N such that every slot ≤ N has been chosen and applied on every learner. - -**Aggregation.** The leader collects per-learner `contiguous_applied` reports piggy-backed on heartbeat responses. The safe-slot is recomputed at the end of each quorum heartbeat round: - -``` - safe_slot = min(contiguous_applied[learner]) for learner in voting members -``` - -A peer that has never reported is treated as `0`, so the safe-slot only rises once *all* voting members are heard from. The safe-slot is monotonic within a tenure (`fetch_max`); a new leader resets it to `0` and re-establishes it from fresh heartbeats. - -**Propagation.** The leader includes `safe_slot` in heartbeats, write responses, read responses, and the describe-cluster RPC. - -`Scan(Linearizable)` uses the leader's *own* contiguous frontier rather than the safe-slot. It is strictly ≥ safe-slot at all times ([§6.4 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#64-scan-modes)). - ---- - -## 8. Gap Detection - -A "gap" is a slot N < max-chosen-slot for which the leader has no `Chosen` decision yet. The leader's learner maintains `contiguous_chosen` (highest gap-free chosen slot) and `last_chosen_slot` (highest slot ever seen chosen, gaps allowed). A gap exists when `contiguous_chosen < last_chosen_slot`. - -**Repair mechanism:** - -1. **Opportunistic repair.** After each heartbeat round in the leader-state loop, `repair_once()` is called. It finds the lowest gap (`contiguous_chosen + 1`) and runs classic Paxos (Phase 1 + Phase 2) to close it. A no-gap leader returns immediately without any RPCs. -2. **On leader change:** the new leader runs bulk Phase 1 over `[contiguous_chosen+1, ceiling]` as a one-shot sweep (see [`design-crowdb-paxos-kv-leader-election.md`](design-crowdb-paxos-kv-leader-election.md) §4). - -There is no separate repair task with its own tick or concurrency cap; repair is interleaved with heartbeats at the heartbeat cadence. - ---- - -## 9. Gap Repair via Classic Paxos - -Once a gap is selected, repair runs full classic-Paxos for that slot: - -1. **Pick a fresh ballot.** `(round, leader_id)` with `round = max_seen_round + 1`. -2. **Phase 1 — Prepare.** Send `Prepare(slot, ballot)` to all acceptors. -3. **Wait for quorum of `Promise`s.** Each carries the highest `(ballot', value')` previously accepted, if any. -4. **Choose a value.** If any `Promise` returned an accepted value, re-propose the one with the highest `ballot'`. If none, propose `NoOp`. -5. **Phase 2 — Accept.** Send `Accept(slot, ballot, value)`. Wait for quorum. -6. **Chosen.** Broadcast `Chosen(slot)` and apply locally. - -The repair never invents a fresh user value — if any acceptor had a half-baked accept, that value is re-chosen. Repair touches one slot at a time; the hot path on other slots is unaffected. A repair on slot N does not block new writes at slot M > N. - ---- - -## 9A. Follower-Side Apply and Catch-up - -The leader-side repair in §9 closes gaps in the leader's *chosen* frontier. Followers face a symmetric but distinct problem: they learn about chosen slots via `ChosenNotice` (a fire-and-forget notification the leader broadcasts after quorum), and they may miss or receive stale versions of those slots. This section covers the follower-side apply path and catch-up mechanism. - -### 9A.1 Accept Path Stores Only - -A follower's `handle_accept` stores the accepted value in its acceptor and persists it to the WAL, but does **not** advance `known_commit_slot` or wake the apply loop. Accept is a storage event, not a commit event. The leader may crash before reaching quorum, in which case the accepted value may never be chosen (or may be overwritten by a higher-ballot value during new-leader recovery). - -Advancing the apply frontier on Accept would let a follower apply an un-chosen value, violating Paxos safety, the same class of bug as applying before `leaderCommit` in Raft. - -The one exception is the new-leader transition: after a follower wins election and completes bulk Phase 1 over `[contiguous_chosen+1, ceiling]`, it advances `known_commit_slot` to its new `contiguous_chosen` and wakes the apply loop. This is safe because bulk Phase 1 has resolved (chosen or NoOp-filled) every slot in that range under the new leader's ballot. - -### 9A.2 ChosenNotice — Ballot-Verified Out-of-Order Apply - -When the leader chooses a slot, it broadcasts a `ChosenNotice` carrying `(slot, term, leader_id, ballot_round)` to all followers. The follower's ChosenNotice handler: - -- If the follower's acceptor has an accepted value at `slot` with `ballot == chosen_ballot` → the value matches what was chosen. The follower calls `update_chosen_frontier` and wakes the apply loop. Apply can proceed out-of-order; the per-key slot tracking in §3 makes this safe (highest slot wins per key). -- If the follower's accepted ballot at `slot` is **lower** than the chosen ballot (stale) → the follower has an outdated value. It records a gap for `slot` (driven by FetchGap, §9A.3) and does not apply. -- If the follower has no accepted value at `slot` → it records a gap and does not apply. - -The `ballot_round` field in the ChosenNotice flatbuffer enables this verification. Without it, the follower cannot distinguish a fresh chosen value from a stale re-delivery. - -### 9A.3 Follower-Driven FetchGap Catch-up - -When a follower detects a gap (missing or stale slot in the chosen range), it sends a unary `FetchGap(slot)` request to the leader. The leader: - -- Has completed leader recovery and retains the value within its contiguous - chosen prefix → replies with the full entry (payload + ballot + term). -- Otherwise → returns pending/not found. Normal leader recovery or independent - gap repair must establish the chosen value before a later request can fetch it. - -On receipt, a durable follower first writes the original Accepted entry to its -WAL, then installs it, advances the chosen frontier and wakes the apply loop. -Historical entries retain their original election term; a newer local term -does not reject committed history. Failed persistence leaves the gap pending. -Learning a proven chosen value does not answer a new Accept request and must -not discard a later promise. It retains the original entry alongside that -higher promise; an already higher accepted ballot must carry the same value. -Replay rebuilds highest accepted values before restoring promise fences, so -the physical order of historical fetch and later promise records cannot lose -either durable fact. Accepted history alone is never a FetchGap chosen proof. -`MAX_INFLIGHT_FETCHGAP` (default 16) bounds the entire fetch/persist/install -operation, including error and cancellation paths. - -This replaces the previous leader-driven catch-up that ran inline in `run_heartbeat_round` (up to 64 synchronous `send_accept().await` calls per lagging follower). That inline catch-up could exceed `heartbeat_interval` and trigger spurious elections. The heartbeat round is now pure liveness + lease: one RPC round-trip, no catch-up work. - -### 9A.4 Snapshot Fallback - -If a follower's gap count exceeds `catchup_snapshot_threshold` (default 1024), -it records the large backlog and continues FetchGap under the same fixed -16-request limit. A missing automatic snapshot path must not disable catch-up. - -The bounded snapshot protocol is used only while bootstrapping a fresh, -unpublished member. It does not replace the engine of a running follower and -is not invoked by this diagnostic threshold. A live replacement requires a separate -atomic engine-generation publication design. - -A recovered durable engine's gap-free resume watermark remains authoritative -when WAL reclamation removed the boundary record or all preceding segments. -Restore retains that complete chosen/applied prefix without capping it to the -remaining WAL tip. If the exact boundary term is unavailable, its historical -term is reported as zero (unknown); voting freshness still uses the durable -acceptor log tip. Higher retained entries replay after the snapshot prefix. - -### 9A.5 Apply Loop - -The follower's apply loop targets `max(known_commit_slot, last_chosen_slot)`. For each slot in the apply range: - -- If `slot ≤ known_commit_slot` → apply (committed by definition). -- Else if `learner.is_chosen(slot)` → apply (out-of-order chosen slot). -- Else (accepted but not chosen) → skip and record a gap for FetchGap. - -This ensures accepted-but-not-chosen slots are never applied, while allowing out-of-order chosen slots to apply immediately without waiting for lower slots to be resolved. - ---- - -## 10. Timing Diagrams - -### 10.1 Best case — fully pipelined window - -``` - time → - - slot N : assign─ fsync─ Accept ──► quorum ──► Chosen ─ apply ─ ack - slot N+1 : assign─ fsync─ Accept ──► quorum ──► Chosen ─ apply ─ ack - slot N+2 : assign─ fsync─ Accept ──► quorum ──► Chosen ─ apply ─ ack - - end-to-end latency for slot N = fsync + RTT + apply - per-slot incremental latency ≈ batched-fsync amortization - steady-state throughput = window × (1 / RTT) when network-bound - = disk_bw / record_size when disk-bound -``` - -### 10.2 Gap repair (slow follower) - -``` - time → - - slot N : assign─ fsync─ Accept ──► follower-A: Accepted - │ follower-B: ...........(slow) - slot N+1 : assign─ fsync─ Accept ──► quorum ──► Chosen ─ apply ─ ack(N+1) - slot N+2 : assign─ fsync─ Accept ──► quorum ──► Chosen ─ apply ─ ack(N+2) - - ... after heartbeat round ... - - repair : Prepare(N, ballot') ─► quorum of Promises - │ Promise from leader: Accepted(ballot, v) - Accept(N, ballot', v) ──► quorum ──► Chosen(N) - apply(N) on leader, advance contiguous_applied to N+2 -``` - -### 10.3 Leader change - -``` - time → - - old leader: ...slot N-1: Chosen, applied - slot N: Accepted on 1 follower only (no quorum) - slot N+1: Accepted on 0 followers - - - new leader (term T+1): - Prepare((T+1, me)) over [N, N+1, ..., max_chosen] - Promises: - slot N : (older_ballot, v_N) from one follower - slot N+1 : empty - re-Accept((T+1,me), v_N) at slot N → Chosen - Accept((T+1,me), NoOp) at slot N+1 → Chosen - steady state resumes -``` - ---- - -## 11. Interaction with Snapshot and WAL GC - -Acceptor and WAL reclamation share a certificate distinct from the commit -frontier and voting-member `group_safe_slot`: - -- The leader takes the minimum contiguous-applied frontier of itself and every - configured remote learner, including non-voting learners. Unknown progress - is zero; replies arriving after quorum still update progress. -- It caps that frontier by its local durable snapshot and the greatest durable - snapshot of a voting peer. A singleton voting group needs only its own durable - snapshot, while any attached non-voting learners still constrain progress. -- Heartbeats carry `gc_before_slot` and `gc_membership_epoch`. A follower accepts - the certificate only from its known voting leader in the current term and - membership epoch. The retained certificate binds these identities and the - watermark atomically; a later epoch or term cannot relabel an old certificate. -- Each replica caps its certificate by its own contiguous-applied and durable - snapshot frontiers before reclaiming. A missing WAL/snapshot yields zero. -- Maintenance trims acceptor entries with `slot < gc_slot` and unlinks only WAL - segments wholly below that boundary. It retains the boundary slot. Outstanding - readers retain node/payload owners until they finish. - -A new group or leader tenure clears progress evidence. Membership replacement -also clears evidence, including when it adds a non-voting learner. A joining -replica that needs a prefix already reclaimed uses snapshot catch-up. - -See [WAL garbage collection](design-crowdb-paxos-kv-wal.md#7-garbage-collection). - ---- - -## 12. Tunables and Defaults - -| Parameter | Default | Where it lives | -| -------------------------------- | ------- | --------------------------------------------------------------------- | -| `max_inflight_proposals` | 32 | `PaxosConfig` (total semaphore permits) | -| `reject_on_inflight_window_full` | `false` | `PaxosConfig` (internal, not CLI-exposed) | -| `coalesce_max_keys` | 32 | `PaxosConfig` (max ops per coalesced batch; 0 = off) | -| `coalesce_drain_threshold` | 1 | `PaxosConfig` (skip drain at this inflight count) | -| `max_paxos_retries` | 3 | `PaxosConfig` (per-slot Phase-2 retries) | -| `max_slot_retries` | 3 | `PaxosConfig` (new-slot retries before giving up) | -| `retry_base_backoff_ms` | 5 | `PaxosConfig` (exponential backoff base) | -| `bulk_prepare_window` | 1024 | `PxElectionConfig` (bulk Phase-1 batch size) | -| `catchup_snapshot_threshold` | 1024 | `PxElectionConfig` (diagnostic threshold; bounded FetchGap continues) | - -Bulk Phase 1 recovers independent slots concurrently, with at most -`bulk_prepare_window` complete Prepare/Accept operations in flight. It starts -at the new leader's own contiguous frontier, adopts quorum values and opens -reads only after the entire observed prefix has resolved and applied. Tenure -cancellation prevents prepared slots from proceeding to Accept. - -Steady-state gap repair starts only after initial recovery has completed; -the two paths never compete for the same slot. Each background pass drains -its observed backlog independently of heartbeat cadence. Follower-side gaps -use at most `MAX_INFLIGHT_FETCHGAP` (default 16) concurrent requests, including -backlogs above the snapshot diagnostic threshold. - ---- - -## 13. Correctness Analysis for Parallel Slot Writes - -**Key insight:** Parallel slot writes are safe because we only support **blind operations** (`Put`, `Delete`). No operation reads current state before writing. The final value of each key is determined solely by the highest slot that touched it. - -- **Consensus phase (parallel):** Each `PxSlot` is an independent Paxos instance. They can be decided in any order. -- **Apply phase (per-key slot tracking):** `PxLearner`s store `(slot, value)` per key and only accept writes where `slot > current_slot`. Regardless of apply order, the highest slot wins. -- **Gaps don't block point reads:** An undecided slot 3 does not block `Get(k)` if slot 5 for key `k` is already applied. Gaps only block cross-key scans and WAL truncation. -- **Batches:** Intra-batch order is **as written by the client**. A batch uses the batch's slot for each key. - -This would NOT be safe for `CAS` or `Increment`, which read current state. Not supported. - ---- - -## 14. Parallel-Slot Linearizability Analysis - -**Premises:** (1) Only blind ops. (2) Leader serializes slot assignment. (3) Quorum durable-flush before ack. (4) Per-key `(slot, value)` tracking, highest slot wins. (5) Leader reads fenced by lease or ReadIndex. - -**Claim:** The assigned slot number is a valid linearization point for every op. - -**Sketch:** - -- **Real-time order → slot order.** If `ack(A)` completes before `invoke(B)`, the counter has already advanced past `slot(A)`, so `slot(A) < slot(B)`. -- **Blind apply-order independence.** An undecided earlier slot writing *k* will be ordered earlier and immediately overwritten by the later slot's value. -- **Durability before visibility.** Quorum durable-flush before ack ensures a client-observed write cannot be lost by leader change. Gap repair re-chooses the same value. -- **Leader read correctness.** The leader's learner has applied slot N before acking. A subsequent `Get(k)` reflects the highest chosen slot writing *k*. -- **Follower read correctness.** Read-your-writes uses per-key resolved-slot; bounded-stale uses the gap-free `safe-slot`. - -**Single remaining cost — linearizable `Scan`.** Must wait for the leader's **contiguous applied frontier** to cover the target. Latency is bounded by (window size) × (slot-resolution time). Mitigation: `Scan(mode = SafeSlot)` or `Scan(mode = AtSlot(N))`. Point `Get`s bypass gaps entirely. - -**Why `CAS`/`Increment` would break this:** Read-modify-write ops must read current value, which is unknown until all earlier slots for that key are resolved. Would require sequential consensus or per-key dependency tracking. Neither is in scope. - -**Implementation invariants:** Slot assignment is single-point. `Accepted` not sent before durable flush. Classic-Paxos recovery never discards accepted values. Lease/ReadIndex fencing on every linearizable read. Per-key slot comparison atomic with write. - ---- - -# Part B: Acceptor Retention - -## 15. Acceptor Storage Requirements - -Per-slot promised ballots and accepted entries must support sparse, concurrent -Prepare/Accept operations while maintenance reclaims a proven-safe prefix. -Lookups must retain the node before accessing its fields. Reclamation must -support overlapping maintenance callers during group replacement. - -## 16. Retained Slot Entries - -`PxAcceptor` owns a `crossbeam_skiplist::SkipMap`, a -monotonic `trim_slot`, and a monotonic `highest_seen_slot`. Each operation keeps -the returned map `Entry` alive throughout field access. Removing an entry does -not invalidate an outstanding retained entry. Sparse slots allocate individual -nodes; absent slot ranges allocate nothing. - -## 17. Per-Slot State - -`PxSlotNode` contains atomic promised and accepted pointers. Replaced pointers -enter per-node retired lists and are freed only when the node is dropped. This -keeps a field read valid for the entire retained node lifetime. Returned log -entries clone their `Bytes` owner, so a caller may keep a payload after trim. - -## 18. Lookup and Reclamation - -- `open_slot` rejects a slot below the trim boundary, obtains or inserts a - retained node, then checks the boundary again. An insertion racing trim is - removed and rejected instead of resurrecting old history. -- Prepare/Accept keep the entry guard while reading or changing node fields. - Queries and ordered range scans clone values while retaining their entries. -- `trim` advances the boundary with `fetch_max`. `reclaim` removes entries below - that boundary; concurrent removers can safely encounter the same entry. -- A lookup already in progress may complete with its retained state. A newly - admitted request below the boundary is rejected. The boundary is exclusive. - -## 19. Acceptor API - -- `prepare`, `accept`, `accepted_at`, `promised_at`: per-slot protocol access. -- `accepted_iter_range`: ordered retained-history view; missing/trimmed slots - are absent. `JournalScan` reports a GC gap for a start below `trim_slot`. -- `accepted_log_tip`: highest retained accepted slot and its term. -- `highest_seen_slot`: highest slot ever opened, independent of reclamation. -- `trim`, `reclaim`, `trim_slot`, `retained_slots`: reclamation and observability. - -`ReplicaStatus.acceptor` exposes the exclusive trim boundary and retained slot -count. The count describes map entries, not payload bytes or process RSS. - -## 20. Retention Invariants - -- **R1 — Retained access:** every node field access holds a live map entry. -- **R2 — No resurrection:** a request admitted below the current trim boundary - cannot create visible accepted history. -- **R3 — Durable local recovery:** reclamation cannot exceed the local durable - snapshot or contiguous-applied frontier. -- **R4 — Learner coverage:** the leader certificate cannot exceed the least - applied frontier of any configured learner, including non-voting learners. -- **R5 — Certificate identity:** followers use only the certificate bound to - their current membership epoch, term and known voting leader. -- **R6 — Independent payload lifetime:** reclaiming a node does not invalidate - a log entry already cloned by a caller. - -## 21. Storage Cost - -The ordered map adds per-node allocation and lookup overhead. Reclamation walks -only the eligible prefix and releases historical payload owners as readers -finish. No throughput improvement follows from this representation alone; -measure request throughput, tails, retained slots and RSS under sustained load. - -## 22. Retention Limits - -A lagging configured learner or unavailable durable witness can hold retention -at zero. An operator must restore/catch up that learner or change membership; -maintenance cannot discard history merely to satisfy a memory target. A slot -with repeated ballot/value changes retains its old field versions until that -slot is reclaimed. Long-lived readers may delay physical release after logical -removal. The generic `PxSlotList` remains a separate utility and is not the -acceptor's reclamation mechanism. - -## 23. Server-side Proposal Coalescing - -### 23.1 Problem - -Each client `PUT`/`DELETE` is its own Paxos proposal: one slot, one -quorum RPC round, one WAL record, one learner apply. WAL batch -aggregation already amortizes `fsync` across concurrent proposals, and -the leader's local `fsync` is off the critical path, so the -remaining per-proposal fixed cost is the **quorum RPC round**. The -write-flow sweep shows throughput plateaus at ~29K (Intel) / ~48K -(M5 Pro) once the consensus pipeline saturates, independent of the -inflight window above MI=16. The bottleneck is the per-proposal quorum -RPC rate, not `fsync`. - -The `Batch` payload format already supports multiple ops per slot and -`kv_batch_write` exposes it. Without server-side coalescing, concurrent -single-key proposes each take a distinct slot and pay the full quorum -round. The coalescer (enabled by default, `coalesce_max_keys = 32`) -event-batches these into one multi-key Paxos proposal per batch. - -### 23.2 Drain Threshold - -The first op starts a 1-op slot-task **immediately** (no timer). Ops -arriving during the round join a pending batch. On round completion, -`coalesce_drain_after_round` checks the in-flight slot-task count -(`occupied`, permits held) before taking the pending batch. If -`occupied >= coalesce_drain_threshold`, the drain is skipped; the -`max_keys` overflow path handles high load with full batches. If -`occupied < coalesce_drain_threshold`, the pending batch is taken and -the next slot-task is started (if non-empty) or the coalescer goes -idle (if empty). - -At high load, drains almost never fire (many slot-tasks in flight); -the `max_keys` overflow path produces full batches. At low load, -drains always fire (few slot-tasks, threshold not exceeded), so there -is no latency floor; a lone op flushes immediately. Default -`coalesce_drain_threshold = 1` (see §23.5). - -### 23.3 Coalescer Design - -Per-group state on `PxGroup`: - -- `coalescer: parking_lot::Mutex>` — the - accumulating batch, `None` when idle. -- `self_weak: OnceLock>` — set once the group is wrapped - in `Arc`, so spawned slot-tasks can upgrade without holding a strong - self-reference. -- `coalesce_last_activity_us: AtomicU64` — last coalescer activity - timestamp (updated on every enqueue and drain). Used by the - activity-based watchdog. - -The coalescer mutex is retained because it protects one small batch mutation -and is released before slot proposal, RPC, or engine work. An MPSC owner task -would remove producer contention but add scheduling and queue latency to every -write; profiling must show this critical section is material before making that -trade. Peer applied/durable maps and follower gap tracking also retain their -short locks: replica sets are normally 3–7 members and gaps are exceptional, -so their bounded critical sections do not justify per-peer atomics or another -concurrent map. - -`PendingBatch` holds: -- `op_bodies: Vec` — concatenated op bodies (each single-op - payload's leading count bytes dropped; op bodies are self-delimited). -- `op_count: u16` — number of ops accumulated. -- `identities: Vec` — one `(client_id, seq)` request - identity per client operation, all mapping to the shared result slot. -- `waiters: Vec>` — one per coalesced - caller; each receives the shared `ProposeResult` on flush. -- `timer: Option>` — reserved (unused in event mode; - the watchdog is activity-based, not per-batch). - -`propose` flow (refactored into `propose` + `propose_inner`): - -1. Leadership gate (as before). -2. Request-result lookup — a hit returns the cached slot - immediately, never enters a batch. -3. If `coalesce_max_keys == 0` (coalescing disabled) or `self_weak` is - unset: call `propose_inner(payload, &[tag])` directly; bit-identical - to the fallback path. -4. Else (coalescing on, default `coalesce_max_keys = 32`): - `coalesce_enqueue(payload, tag)`: - - Coalescer is `None` (idle) → create a batch with this op, start a - 1-op slot-task **immediately**. Open a fresh empty batch for ops - arriving during this round. - - Coalescer is `Some` (batch exists) → append op body + tag + - waiter. If `op_count >= max_keys` → overflow: take the batch, - spawn a concurrent slot-task, open a new empty batch. -5. `await` the waiter's `ProposeResult`. - -`coalesce_flush_batch` (called by overflow and drain): - -1. Build merged payload: `[op_count as u16 LE][op_bodies]`. -2. Open a fresh empty batch in `coalescer` (for ops during the next - round). -3. `tokio::spawn` `propose_inner(payload, tags)`. On completion, fan - the `ProposeResult` (now `Clone`) to all waiters, then call - `coalesce_drain_after_round`. - -`coalesce_drain_after_round` (called after each slot-task finishes): - -1. Touch activity timestamp. -2. If `occupied >= coalesce_drain_threshold` → return (skip drain; - overflow handles it). -3. Take the pending batch from `coalescer`. -4. If `op_count == 0` → go idle (return). -5. If `op_count > 0` → `coalesce_flush_batch(batch)` (start next round). - -**Watchdog**: A single background task per group sleeps 1000ms, then -checks if there's been no coalescer activity for 1000ms. If so, it -flushes any stuck non-empty batch. Safety net for edge cases (drain -panic, spawn failure). Zero overhead during normal operation. - -### 23.4 Request Identity Threading - -A coalesced batch carries K client request identities but one result slot. -The identities stay with the leader's coalescer waiters and are published to -the leader-local request-result cache only after that value is chosen. They -are not part of `PxLogEntry`, the WAL, or `FBAcceptRequest` because an -Accepted value is not necessarily chosen and followers cannot safely publish -a result from Accept alone. - -`Learner::learn` accepts a slice of `RequestIdentity` values. Normal leader -proposals pass their identities; repair, election, follower notification, and -restore paths pass an empty slice. The cache is lost on leader change or -restart, at which point ordinary data-idempotent operations may be proposed -again. - -### 23.5 Config - -`PaxosConfig` gains: - -- `coalesce_max_keys: usize` — max ops per batch (cap 65535, the - payload count field is `u16`). Default `32`; `0` disables coalescing. -- `coalesce_drain_threshold: usize` — skip drain when in-flight - slot-task count >= this. Default `1` (the last finisher always - drains). `0` = always drain (pure event mode); higher values skip - the drain at high load so the `max_keys` overflow path produces full - batches. - -CLI: `--coalesce-max-keys` on `crowdb-paxos-kv-server`, applied in `main.rs` -into `config.paxos`. Wired into the group via `set_from_config` (the -coalescer reads `self.config.paxos.*`). - -### 23.6 Correctness - -- **Request replay**: every coalesced identity is recorded against the shared - slot on the leader after choice. Same-leader retries return that slot; - retries after leader change or restart may re-propose ordinary mutations. -- **Per-key ordering**: unchanged; all ops in a batch share one slot; - across batches, per-key highest-slot-wins applies as before. -- **`ProposeResult::Chosen { slot }` contract**: every coalesced - waiter receives the same slot. `ProposeResult` gains `Clone`. -- **`coalesce_max_keys = 0`**: `propose` calls `propose_inner` with a - one-identity slice and records the same leader-local result after choice. -- **Leadership**: re-checked inside `propose_inner`; a step-down - between batch collection and flush surfaces as `NotLeader` to all - waiters. -- **Shutdown**: slot-tasks hold a `Weak`; on shutdown the - pending batch is dropped (waiters get a closed oneshot → mapped - to `Err`). -- **Drain threshold liveness**: the inflight permit is released (on - permit drop in `propose_inner`) **before** `coalesce_drain_after_round` - is called. So the last finisher always sees `occupied = 0 < threshold` - and takes the batch. The batch is never stuck. The 1000ms watchdog - is a backstop for edge cases (drain panic, spawn failure). - -### 23.7 Benchmark Results (10s mem mode, 3-node cluster, max_keys=32) - -| Threads | Baseline TPS | Timer TPS | Event TPS | Drain TPS | Timer WAL | Drain WAL | -|---|---|---|---|---|---|---| -| 32 | 27,787 | 33,029 | 48,346 | 47,485 | 31,090 | 139,404 | -| 64 | 28,062 | 64,145 | 68,201 | 68,741 | 60,498 | 106,926 | -| 128 | 28,260 | 97,554 | 86,759 | 101,537 | 92,752 | 101,350 | -| 256 | 27,804 | 113,671 | 97,865 | 118,377 | 110,034 | 111,944 | - -Drain beats Timer at high load (128: 102K vs 98K, 256: 118K vs 114K) with -WAL counts close to Timer. At 64 threads it matches Event mode and beats -Timer. At 32 threads it matches Event mode (no regression). The default -threshold of 1 still allows drains at low load, preserving the zero- -latency-floor behavior. diff --git a/doc/design/kv/design-crowdb-paxos-kv-state-machine.md b/doc/design/kv/design-crowdb-paxos-kv-state-machine.md deleted file mode 100644 index 7900a9d04..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-state-machine.md +++ /dev/null @@ -1,455 +0,0 @@ - - - -# CROWDB - Design: State Machine - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §5.2, §8.3, §8.4 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §8.3, §8.4, §14.1 - -This document specifies the storage engine abstraction used by CROWDB learners. The engine is the **only** consumer of consensus output; it owns the materialized key-value state and serves all reads. The WAL is the durable log; the engine is the materialized projection. - -> **P3 redesign note.** The production engine `crowdb-tree` and the redefined -> (async, snapshot/GC-aware) `KVEngine` abstraction are specified in the crowdb-tree -> sub-design set: [`../tree/design-crowdb-tree.md`](../tree/design-crowdb-tree.md) (overview + engine -> abstraction + language/FFI decisions), [`../tree/design-crowdb-tree-engine.md`](../tree/design-crowdb-tree-engine.md) -> (in-memory engine + memory model + async FFI), [`../tree/design-crowdb-tree-storage.md`](../tree/design-crowdb-tree-storage.md) -> (durable storage + mapping table + snapshot/GC flow). This document remains the -> source of truth for the *semantics* (per-key slot, apply, compare, compaction); -> crowdb-tree docs own the *implementation*. - -## Table of Contents - -- [1. Goals and Non-Goals](#1-goals-and-non-goals) -- [2. Engine Responsibilities](#2-engine-responsibilities) -- [3. Per-Key Slot Tracking and Single Versioning](#3-per-key-slot-tracking-and-single-versioning) -- [4. Apply Semantics](#4-apply-semantics) -- [5. Read Surface](#5-read-surface) -- [6. Snapshot Export and Import](#6-snapshot-export-and-import) -- [7. Compaction Policy](#7-compaction-policy) -- [8. Compare for Cross-Learner Validation](#8-compare-for-cross-learner-validation) -- [9. Engine Implementations](#9-engine-implementations) -- [10. Tunables and Defaults](#10-tunables-and-defaults) - ---- - -## 1. Goals and Non-Goals - -**Goals:** - -- Define one engine surface so the same learner code drives any backend (in-memory, crowdb-tree). -- Encode the per-key resolved-slot semantics that make parallel-slot consensus correct. -- Provide a deterministic state-equality check (`compare`) for cross-learner test verification. -- Provide a streamable snapshot import/export usable by snapshot install. - -**Non-goals:** - -- Multi-version (MVCC) reads. CROWDB is single-version per key; time-travel is not supported. -- Cross-engine queries. The engine is per-group. -- Transactions or snapshot isolation across keys. Batch completion certifies all - key publications; ordinary reads may observe partial in-flight batches. -- Pluggable compression/encryption. Each engine handles its own internals. - ---- - -## 2. Engine Responsibilities - -The engine encapsulates everything below the consensus / learner layer: - -| Responsibility | Notes | -| -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Persist (or hold in memory) the current `(slot, value)` for every live key | Single version, tombstones for deletions | -| Apply a `(slot, batch)` to completion | Per-key publication; replay is idempotent | -| Serve point reads with their resolved-slot | Used by leader linearizable reads, follower RYW | -| Serve ordered range scans | Required for `Scan` API | -| Export and import snapshots | Streamable; resumable on the consumer side | -| Provide a deterministic state digest or comparison | For test-time validation | -| Track watermarks for compaction | `snapshot_slot`, `safe_slot` (provided by learner) | -| Persist snapshot with slot index | The engine persists its KV state and `last_applied_slot` to a snapshot file; on restart, it loads this snapshot to skip re-applying already-applied slots | - -The engine does **not** know about Paxos, terms, ballots, leaders, or the network. It receives `(slot, batch)` and applies; that is the entire write contract. - -### 2.1 State Machine Restart: Engine Durability + WAL Replay - -The state machine's persistence boundary is separate from the WAL — but it is -not a dedicated "state-machine snapshot file" either. It is whatever the -*engine itself* durably persists (crowdb-tree's own snapshot pipeline, -[`../tree/design-crowdb-tree-storage.md §6`](../tree/design-crowdb-tree-storage.md#6-snapshot-recovery-and-exportimport)), -queried through `KVEngine::resume_from_slot()`. This replaces the former -`DurableCommitWatermark` WAL record design. - -**Restart behavior** (`PxLocalReplica::restore_from_replay_with_engine`, -`lib/crowdb-paxos-kv/src/cluster/local_replica.rs`): - -1. WAL replay rebuilds acceptor state (`Promised`, `Accepted`, `VoteGranted`), Pass 1, unconditional. -2. The engine reports `resume_from = resume_from_slot()`: the highest slot it - already durably reflects (always `0` for `InMemKV`; crowdb-tree's - `last_applied_slot`, restored from its on-disk commit anchor, for a - durable engine that was cleanly snapshotted before the crash/restart). -3. Pass 2 `learn()`s every WAL-accepted slot in `(resume_from, highest_local]` - into the engine, in order, and fast-forwards the learner's frontier - (`contiguous_chosen`/`contiguous_applied`/`last_chosen_term`) directly to - `resume_from` via `seed_resume_frontier`, skipping the now-redundant - re-`learn()` of the already-durable prefix. -4. Slots above the local WAL's own highest accepted slot are re-learned via - new-leader recovery (bulk Phase 1) or steady-state heartbeat catch-up (§6 - of this doc's parent flows, `design-crowdb-paxos-kv-slot.md`). - -Step 3's skip is a pure **optimization**, not a correctness requirement: -`KVEngine::apply` is idempotent (highest-slot-wins per key) and -`update_frontier` tolerates out-of-order slots, so replaying the *entire* -local WAL through `learn()` (`resume_from = 0`, `InMemKV`'s permanent case) -always converges to the same correct state. It's just extra, safe, no-op -work. `persist_snapshot()` (called periodically by the group's maintenance -loop, `lib/crowdb-paxos-kv/src/cluster/group_maintenance.rs::run_pass`) is what advances -the durable floor `resume_from_slot()` will report on the *next* restart. - ---- - -## 3. Per-Key Slot Tracking and Single Versioning - -### 3.1 What we store - -For every live key `k`, the engine stores exactly one tuple: - -``` - (k → resolved_slot(k), value_or_tombstone(k)) -``` - -- `resolved_slot(k)` is the highest slot that has touched key `k` and been applied. -- `value_or_tombstone(k)` is the value from that slot, or a tombstone marker if that slot was a `Delete`. - -Tombstones occupy space until compacted away (§7). - -### 3.2 Why single-version - -CROWDB does not provide repeatable reads or time-travel queries. `AtSlot(N)` waits -for the engine's contiguous-applied frontier to reach `N`, then reads current data. - -`Scan(AtSlot(N))` has a freshness precondition, without a common snapshot time. -Each key returns its highest observed slot, which may exceed `N`; different keys -can reflect different times and partial later batches. This does not establish -one linearization point for the whole scan. Stable exports use the separate -covered-prefix snapshot interface. - -If true historical snapshots are ever required, MVCC is a future extension. The single-version restriction comes from design-crowdb-paxos-kv.md §1 / §5.2](design-crowdb-paxos-kv.md). - -### 3.3 Resolved-slot is monotone per key - -The engine never accepts a write at slot `s` for key `k` if `s ≤ resolved_slot(k)`. This is the runtime expression of [Invariant I5 in `design-crowdb-paxos-kv-slot.md`](design-crowdb-paxos-kv-slot.md#2-concepts-and-invariants). - -Implication: replays and out-of-order applies are naturally idempotent. If WAL replay tries to apply slot 7 for key `k` and `resolved_slot(k)` is already 9, the apply is a no-op for `k`, consistent with the parallel-slot semantics. - ---- - -## 4. Apply Semantics - -The engine's `apply(slot, batch)` is the only mutator. It must be atomic and idempotent. - -### 4.1 Inputs - -- `slot` — the slot of this batch. All operations in `batch` carry this same slot. -- `batch` — an ordered list of `(key, op, value?)` tuples, where `op ∈ { Put, Delete }`. - -### 4.2 Procedure - -For each `(k, op, v?)` in batch order: - -1. Read current `(resolved_slot(k), current)` if any. -2. If `slot ≤ resolved_slot(k)` → skip (idempotent no-op). -3. Else write `(slot, value)` for `Put`, or `(slot, tombstone)` for `Delete`. -4. Update `resolved_slot(k) = slot`. - -After all tuples are processed, advance the engine's `max_applied = max(max_applied, slot)` and update the contiguous-applied frontier if `slot == contiguous_applied + 1`, possibly cascading. - -### 4.3 Atomicity - -Batch completion is recorded only after every accepted key mutation has been published. Ordinary tree reads and scans may observe part of an in-flight batch; they do not promise a common snapshot version across keys. Equal-slot replay and intra-batch last-occurrence-wins remain required. Durable snapshot export is a separate, covered-prefix operation. - -In-memory engines can hold a write lock for the duration of the batch. File-based engines can use a transactional write group. crowdb-tree's btree operations on a single batch can ride on its own concurrency control. - -### 4.4 Intra-batch order - -For a key `k` appearing multiple times in a batch (rare but legal; see [`design-crowdb-paxos-kv-slot.md` §13](design-crowdb-paxos-kv-slot.md#13-correctness-analysis-for-parallel-slot-writes)), the *last* occurrence in batch order wins. The earlier ones are folded into the apply procedure naturally (each tuple in turn updates `current`; the loop's final state is what persists). - -### 4.5 Failure during apply - -`apply` either completes or returns an error. On error, the engine must leave its state unchanged (no partial apply). The learner treats engine apply errors as fatal (they indicate disk corruption or out-of-space) and fails the node out of the group. - -### 4.6 Zero-copy batch decode - -`Batch::try_decode` validates the complete Paxos payload, then extracts keys and values using -`Bytes::slice(range)`, which creates zero-copy views into the same -allocation as `PxLogEntry.payload` (an O(1) ref-count bump). This -eliminates the per-key and per-value `to_vec()` heap allocations that -would otherwise occur on every `learn_chosen` call. - -`Op::Put(Bytes)` and `BatchOp.key: Bytes` use `Bytes` (ref-counted, -owned, `Send`) rather than `Vec`. The `KVEngine::apply` trait -signature is unchanged; it still takes `&Batch`. Engine -implementations use `Bytes::as_ref()` to obtain `&[u8]` for FFI or -internal storage. `Cell` and `EngineDiff` remain `Vec` since they -represent engine-internal storage, not the decode path. - -Proposal admission and learner apply reject truncated fields, unknown operation -kinds, nonempty delete values, and bytes after the declared operation count. -Malformed chosen entries do not mutate the engine or advance the applied -frontier. They remain chosen and require recovery of valid durable data; an -invalid entry never becomes a partial batch or an implicit `NoOp`. - -An empty payload and a zero-operation payload represent empty batches. Their -chosen slots still pass through fallible gap completion. `Batch::decode` retains -its permissive field behavior only for explicit compatibility consumers; the -proposal and learner paths use `Batch::try_decode`. - -### 4.7 Async engine apply and the apply fence - -`learn` is the learner's apply entry point: `apply_entry` (the FFI + -memtable insert) → advance the chosen frontier → advance the applied -frontier → cache the leader's request result. Two frontiers are tracked: - -- `contiguous_chosen` — highest slot `S` such that every slot in `[1, S]` - is chosen. -- `contiguous_applied` — highest slot `S` such that every slot in `[1, S]` - is applied to the engine. - -Under V1 (apply == learn, the default), both advance together -in `learn` right after the synchronous `apply_entry`, so -`contiguous_applied` tracks `contiguous_chosen` exactly. - -**`async_engine_apply`** defers the engine apply off the write critical -path.** The leader's propose path splits `learn`: the chosen-frontier -advance and request-result record run **synchronously** (cheap atomics, before -`propose` returns `Chosen`), and only `apply_entry` + the applied-frontier -advance are `tokio::spawn`'d. This keeps `contiguous_chosen` current: a -subsequent read's `read_slot = contiguous_chosen` reflects the -just-chosen slot, while `contiguous_applied` lags by the spawned apply. -Spawned applies can complete out of order, so each frontier stores gaps in a -sharded concurrent map. A compare-exchange drain owner advances the contiguous -cursor and removes consecutive gaps; producers that lose ownership only insert -their slot. The highest-chosen `(slot, term)` pair uses an atomic sequence guard -so election readers cannot observe a slot paired with another update's term. - -**Apply fence (Linearizable read-your-writes).** With `async_engine_apply` on, a -linearizable read that lands between "chosen" and "applied" would miss a -just-written value: the barrier captures `read_slot = contiguous_chosen` -(which is current), but the engine get returns the latest *applied* value. -The fence closes this gap. After the leadership barrier resolves -`read_slot`, the read awaits `contiguous_applied >= read_slot` before -serving the engine get. A `Notify` on the learner is woken -(`notify_waiters`) whenever `contiguous_applied` advances; the fence uses -register-before-load (`notified()` created before the `Acquire` load) so a -wake racing the load is not missed; the load observes the -`Release`-stored new frontier and returns without waiting. - -- **Fast path** (`async_engine_apply` off, or the slot already applied): one - `AtomicU64::load(Acquire)` + compare; no wait, no wake. With `async_engine_apply` off the - fence is a no-op; `contiguous_applied == contiguous_chosen` at the - instant the barrier resolves. -- **Slow path** (`async_engine_apply` on AND a read races a just-chosen-but-not-applied - write): the fence waits for the spawned apply. The wait is bounded by - apply throughput (memtable insert, µs), exactly the latency `async_engine_apply` removed - from the write path; the fence redistributes that µs to an occasional - read, it does not add new latency. -- **MinSlot** is untouched; it already gates on `contiguous_applied` via - the client-supplied `min_slot`. -- **ReadIndex batching** composes: the fence runs per-read after the - (possibly shared) barrier outcome; batched reads share the barrier's - `read_slot` floor but each awaits its own `contiguous_applied >= - read_slot` check, and `notify_waiters` wakes all parked reads together. - -`async_engine_apply` defaults to `true`; test profiles (`for_tests`) and -the `PxGroup::new` test path opt out (`false`) for deterministic -synchronous apply. The setter `set_async_engine_apply` remains for tests -that exercise `async_engine_apply`. The flag is carried across group rebuild via -`set_from_config(group.config())`. - ---- - -## 5. Read Surface - -The engine exposes three read primitives. All are non-mutating and may be served concurrently with writes. - -### 5.1 `get(k) → Option<(slot, value)>` - -Returns `None` if `k` is unset or tombstoned (callers may distinguish if they care, e.g. for range scans excluding tombstones). Returns `Some((resolved_slot(k), value))` if a live value exists. - -The slot is returned because callers (the learner) need it to assemble responses to `Get` with read-mode semantics. - -### 5.2 `scan(range, limit) → iterator<(key, slot, value)>` - -Returns an iterator of live entries (no tombstones) within `range`, in key order, up to `limit` items. The iterator may be backed by a btree cursor (in crowdb-tree) or a sorted-tree iterator (in-memory). - -The ordinary iterator returns the highest version it observes for each key, in key order, without duplicates. Different keys may reflect different times or part of a concurrent batch. `AtSlot(N)` remains a freshness precondition at the serving layer; it does not pin all returned keys to a common version. Flush must preserve continuous visibility of stable keys and the lifetime of borrowed tuples. Stable export uses the engine's separate snapshot interface. - -### 5.3 `multi_get(keys) → map` - -A batched point read. Returned as a map; missing keys are simply absent. - -### 5.4 What the engine does **not** expose - -- No range-write API. All writes go through `apply(slot, batch)`. -- No "delete-range" API. Range deletes are decomposed by the consensus layer into per-key deletes inside a batch. -- No "wait-for-slot" primitive. Wait conditions (RYW, SafeSlot, AtSlot) are implemented in the learner using the engine's `contiguous_applied` watermark, not inside the engine. The Linearizable RYW wait is the apply fence (§4.7): `PxLearner::await_applied` parks on a `Notify` until `contiguous_applied >= read_slot`. - -This minimal surface keeps multi-engine compatibility easy. - ---- - -## 6. Snapshot Export and Import - -### 6.1 Purpose - -Snapshot install ([§8.4 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#84-snapshot-and-install)) is the bootstrap path for new or far-lagging members. The engine provides export/import primitives that the snapshot module wraps in a chunked, resumable, throttled transfer. - -In addition to peer transfer, the engine uses snapshot export to persist its own state locally (see §2.1). The local snapshot file stores the KV state and `last_applied_slot`, enabling fast restart without re-applying the entire WAL. - -### 6.2 Export - -`snapshot_export() → stream`: - -- Returns a stream of opaque chunks that, when imported by a peer engine, produce the same logical state as the exporter's current `last_applied_slot`. -- The export reflects the engine state *after* applying every slot ≤ `last_applied_slot` (always the latest durable state). -- Chunks have stable ordering and stable boundaries: the same state always produces the same byte sequence, modulo any internal pagination. This determinism is what makes resumption possible. -- Chunk size is engine-defined; 1–4 MiB is a reasonable default. - -The chunk format is **engine-specific**. The in-memory tree might serialize key/value pairs in btree order; crowdb-tree might dump native pages directly. The snapshot module treats chunks as opaque. - -### 6.3 Import - -`snapshot_import(stream) → ()`: - -- Consumes a stream of chunks produced by the same engine type. -- On success, the engine's state is exactly what `snapshot_export()` produced, with `contiguous_applied` and `max_applied` set to the exported slot. -- Resumable: an import that fails part-way may be restarted from a known chunk offset (the snapshot module records the last successful offset). -- Verified: the snapshot module checks an end-to-end CRC after the last chunk before activating the import. - -### 6.4 Cross-engine snapshots - -The default rule: **snapshot exported by engine X can only be imported by engine X.** A 3-node group all running crowdb-tree can snapshot-install crowdb-tree-formatted snapshots. A test cluster mixing in-memory and file engines must export/import via a portable format. - -A portable interchange format (e.g. sorted key-value pairs with slots) is provided as an optional path for testing and operations, at the cost of being slower than the native path. Production deployments should use a single engine type per group. - -### 6.5 Atomicity of import - -The import is atomic to readers: until the last chunk is processed and the import is activated, the engine continues serving its previous state. Only at the activation step does the engine swap to the imported state. This avoids serving a partial snapshot. - ---- - -## 7. Compaction Policy - -Tombstones and replaced values can be removed once they are no longer needed. The two watermarks that gate compaction: - -- `snapshot_slot` — the engine state at this slot is durable on at least leader + one peer. -- `safe_slot` — every learner has applied every slot up to here. - -### 7.1 When can a tombstone be GC'd? - -A tombstone for key `k` placed at slot `t` may be GC'd when `t < min(snapshot_slot, safe_slot)`. Both must have advanced past `t` because: - -- Until `safe_slot` passes `t`, some learner might still be applying earlier slots that could resurrect `k`. (Cannot happen for blind ops, but could complicate snapshot install if the tombstone is GC'd too early.) -- Until `snapshot_slot` passes `t`, a snapshot install must reproduce the tombstone. Otherwise the receiving peer would think `k` was never deleted. - -The conservative "both must pass" rule prevents observability holes. - -### 7.2 When can an old value be GC'd? - -When a key `k` is overwritten with a higher slot value, the old value is immediately overwritten in single-version storage; there is nothing to GC. The complication is only with tombstones (representing deletes) and with engine-internal versioning (e.g. crowdb-tree's MVCC of pages, which it manages internally). - -### 7.3 Compaction trigger - -- **Periodic.** A background sweeper runs every `compaction_tick` (default 5 min) and removes eligible tombstones in batches. -- **Pressure.** If the engine reports memory or disk pressure, a focused sweep runs. -- **Snapshot-side-effect.** When a snapshot completes, eligible tombstones for slots ≤ that snapshot may be swept as part of the post-snapshot bookkeeping. - -### 7.4 Engine-specific compaction - -- In-memory: tombstones are simply removed from the map. -- crowdb-tree: uses crowdb-tree's internal compaction. The sweeper hands crowdb-tree a "tombstones below slot S are safe to drop" hint and crowdb-tree merges that into its compaction policy. - ---- - -## 8. Compare for Cross-Learner Validation - -A `compare(other) -> diff` operation is required so that `crowdbbench` can verify state equality across learners after a test run (design-crowdb-paxos-kv.md §14.1](design-crowdb-paxos-kv.md)). - -### 8.1 Semantics - -`compare(other)` returns a diff describing the keys that differ in `(slot, value)` between `self` and `other`. If the engines are in the same logical state (same set of live keys, same `(slot, value)` per key), the diff is empty. - -The compare is **logical**, not byte-level. Two engines may have different physical layouts (e.g. one in-memory and one crowdb-tree) yet be logically equal. Compare must succeed across engine types. - -### 8.2 Required behavior - -- Order: results sorted by key. -- Tombstones: included if they differ. Two engines that both have `k` tombstoned at the same slot are equal. -- Resolved-slot: compared exactly. Two engines with the same value for `k` but different `resolved_slot(k)` are *not* equal; this would indicate that one of them missed an apply. - -### 8.3 Implementation strategy - -For testing, a simple two-cursor merge over both engines' ordered key streams is sufficient. The engines provide a `iter_all() → iterator<(key, slot, value_or_tombstone)>` interface to support this. The cursors advance the smaller key first; when both are at the same key, compare `(slot, value)`; emit a diff if they disagree. - -For very large states, a Merkle-tree-style digest could be added later; not required for the initial implementation. - -### 8.4 Compare under in-flight writes - -`compare` is a snapshot-time operation. It assumes the engines are quiescent (no apply in flight) or that the iterator semantics provide a consistent point-in-time view. The test harness ensures quiescence by stopping client traffic and waiting for `safe_slot == max_chosen` before invoking compare. - ---- - -## 9. Engine Implementations - -Two engine implementations satisfy the surface above. Each is appropriate for a specific phase of development or workload. - -### 9.1 In-Memory Tree - -- Backing store: a sorted in-memory map (e.g. a btree or skiplist). -- Persistence: none. -- Use cases: unit tests, integration tests where startup and teardown are cheap, behavior-validation runs of `crowdbbench`. -- Snapshot format: serialized sorted (key, slot, value-or-tombstone) tuples. -- Concurrency: read-write lock over the whole map for `apply`; concurrent reads via a snapshot pointer. - -### 9.2 crowdb-tree - -- Backing store: the production btree library `crowdb-tree`, a C++ `libcrowdb-tree` - (single-writer COW B+tree with per-leaf delta chains, epoch GC, versioned root) - consumed from Rust over a coarse C ABI. Full design: - [`../tree/design-crowdb-tree.md`](../tree/design-crowdb-tree.md) and its sub-docs. -- Persistence: yes, via a pluggable `PageStore` (local file, raw block device, - remote/RDMA); snapshot + consensus replay for recovery (no second op-log). -- Use cases: production. -- Snapshot format: portable `(key, slot, cell)` tuples first; native page dump as - an optimization. -- Concurrency: single writer (learner apply) + lock-free immutable-page reads, - epoch-based reclamation. - -The trait surface is engine-agnostic; switching engines is a configuration choice. Both pass the same `compare`-based equivalence tests. - -### 9.3 New-member snapshot sessions - -Snapshot transfer is exposed as unique, non-clone export and import session -owners. An exporter pins one stable view and reports its format, covered slot, -encoded length, body CRC32C, and chunk limit. Reads are sequential by byte -offset. An importer accepts arbitrary bounded feeds and changes the fresh -target engine only after `finish` validates the complete portable stream. -Dropping either owner aborts it. Production transfer supports only the -crowdb-tree portable format; cross-engine transfer is not supported. - ---- - -## 10. Tunables and Defaults - -| Parameter | Default | Range | Notes | -| --- | --- | --- | --- | -| `compaction_tick` | 5 min | 10 s – 24 h | Background tombstone sweep cadence | -| `snapshot_chunk_bytes` | 1 MiB | 1 B – 1 MiB | New-member snapshot Read payload limit; kept below the 4 MiB RPC frame ceiling | -| `engine_apply_concurrency` | 1 | 1 – ∞ | Apply is serialized by slot anyway; >1 makes sense only if apply is non-overlapping per-key | -| `engine_read_concurrency` | unbounded | — | Reads do not contend with apply at the engine level | -| `tombstone_grace_slots` | 0 | 0 – ∞ | Optional minimum number of slots to keep a tombstone past the GC watermark, for forensics | - -**Engine choice guidance:** - -- Unit and integration tests → in-memory. -- Manual debugging or operations exercises → ordered file. -- Production → crowdb-tree. - -**Per-key memory cost considerations:** the per-key resolved-slot adds 8 bytes per live key. For 10⁹ live keys this is 8 GiB on every learner, accepted in the requirement ([§7.3.1](design-crowdb-paxos-kv.md)). If memory pressure dictates, a future optimization could compress recently-applied resolved-slots into a "below safe-slot" bit (a single bit replacing the 8 bytes when the per-key slot is no longer needed for read-your-writes). diff --git a/doc/design/kv/design-crowdb-paxos-kv-test.md b/doc/design/kv/design-crowdb-paxos-kv-test.md deleted file mode 100644 index 0f919d789..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-test.md +++ /dev/null @@ -1,272 +0,0 @@ - - - -# CROWDB - Design: Test Strategy - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §17 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §17 - -## Table of Contents - -- [1. Overview](#1-overview) -- [2. Architecture Stack](#2-architecture-stack) -- [3. Test Binary Map](#3-test-binary-map) -- [4. Cross-Cutting Coverage Rules](#4-cross-cutting-coverage-rules) - - [4.1 Runtime resource ownership](#41-runtime-resource-ownership) -- [5. Layer Scope](#5-layer-scope) -- [6. crowdb-tree C++ Test Layers](#6-crowdb-tree-c-test-layers) -- [7. Sequencing](#7-sequencing) -- [8. Test Pairing Rule](#8-test-pairing-rule) - -## 1. Overview - -CROWDB is a distributed key-value store built on Paxos consensus. The test -suite is organized in layers that mirror the system architecture, so a -failure points at the lowest broken layer. Each layer gets its own test -binary and tests only the logic that belongs to that layer. - -This document is the **layer guide**: it defines the test strategy, the -scope of each layer, and the cross-cutting coverage rules that apply across -layers. Consult it to determine which layer a new test belongs to and what -rules apply. Per-layer coverage checklists and the live task backlog live in -[`plan-test.md`](../working/plan-test.md); benchmark design and baseline -results live in [`kv-write-flow-analysis.md`](kv-write-flow-analysis.md). - -## 2. Architecture Stack - -``` -store (PxKvStore: many groups, one node identity, routing) <- lib/crowdb-paxos-kv/tests/store.rs - group (PxGroup: 1 local + N remote replicas, full Paxos) <- lib/crowdb-paxos-kv/tests/group.rs - replica (PxLocalReplica: acceptor + learner + WAL + slots) <- lib/crowdb-paxos-kv/tests/replica.rs - election (PxLocalReplica election state machine, pure logic) <- lib/crowdb-paxos-kv/tests/election.rs - wal (WalEngine: durable log) slot (PxSlotList) <- lib/crowdb-paxos-kv/tests/wal.rs, slot.rs - unit (pure modules: codec, classifier, kv engine, roles) <- lib/crowdb-paxos-kv/tests/{paxos,kv}.rs + wal/slot codec tests -deployment (release containers + host HTTP/RPC clients) <- e2e-test/tests/{kv,deployment}/* -console (mgmt API server + CLI: Axum REST, CLI commands) <- app/crowdb-{web,cli}/tests/* -ui e2e (host Playwright browser + release node containers) <- e2e-test/clients/console-ui/e2e/* -``` - -## 3. Test Binary Map - -| Binary | Layer | Drives | -| ------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| `lib/crowdb-paxos-kv/tests/paxos.rs` | unit | `acceptor`, `learner`, `error` classifier, `roles` in isolation | -| `lib/crowdb-paxos-kv/tests/kv.rs` | unit | `InMemKV` / `KVEngine` / `CrowdbTreeEngine` apply semantics | -| `lib/crowdb-paxos-kv/tests/election.rs` | unit | `PxLocalReplica` election state machine (role transitions, vote granting, heartbeat, lease, term fencing) | -| `lib/crowdb-paxos-kv/tests/wal.rs` | subsystem | `WalEngine` append/replay/gc/segment + record codec + backends | -| `lib/crowdb-paxos-kv/tests/slot.rs` | subsystem | `PxSlotList` / `PxSlotNode` | -| `lib/crowdb-paxos-kv/tests/replica.rs` | replica | single `PxLocalReplica` (no peers) | -| `lib/crowdb-paxos-kv/tests/group.rs` | group | `PxGroup` multi-node clusters (real loopback crowdb-rpc, no mocks) | -| `lib/crowdb-paxos-kv/tests/store.rs` | store | `PxKvStore` routing / lifecycle / status / health | -| `app/crowdb-paxos-kv-server/tests/*` | component | management handlers, validation and deterministic local state | -| `e2e-test/tests/kv/*`, `e2e-test/tests/deployment/*` | container E2E | packaged server startup, public management/RPC, persistence and recovery | -| `lib/crowdb-paxos-kv-client/tests/*` | client component | `CrowdbClient` retry, topology cache, `NotLeaderHint` follow, `AnyReplica` read distribution + fallback against embedded crowdb-rpc servers | -| `app/crowdb-web/tests/*` | console mgmt API | Axum REST API server: node management, OpenAPI proxy, API forwarding | -| `lib/crowdb-console-shared/tests/*`, `app/crowdb-cli/tests/*` | console mgmt API | shared core (config, API client, health aggregation) + CLI commands | -| `app/crowdb-cli/tests/*` | CLI component | argument validation, protocol stubs and comparisons; real cluster lifecycle has a container owner | -| `e2e-test/clients/console-ui/e2e/*` | UI container E2E | host browser interactions against owned release nodes and fixed endpoints | -| `app/crowdb-web/ui/src/**/*.test.*` | UI component | deterministic hooks, rendering and state transitions | - -## 4. Cross-Cutting Coverage Rules - -**Placement rule:** deterministic algorithms, pinned roles, injected failures and -client/supervisor protocol stubs remain component-local, including embedded RPC -used to verify these internal contracts. Scenarios that start production server -binaries or exercise externally observable assembled service flows belong to the -[container E2E component](../../../e2e-test/README.md). Each mutating owner uses -its own release containers, network and data roots. Rust, SDK and browser clients -run on the host using inspected IPs and fixed ports; there is one canonical owner -per boundary flow. Component tests require neither Docker nor server binaries. - -**KV operation correctness rule:** every layer that applies KV mutations -(replica, group, store) must test all operation types and orderings: -- Put (single key) -- Overwrite (same key, new value) -- Delete (produces tombstone) -- Delete on non-existent key (no-op) -- Batch with multiple puts -- Batch with intra-batch last-wins (put → delete → put on same key) -- Batch with put-then-delete same key (delete wins) -- Batch with delete-then-put same key (put wins) -- Empty batch / NoOp (advances frontier, no keys) -- Mixed put/delete across multiple slots and batches -- Persistence round-trip: all above operations survive WAL replay + restart - -**Cluster verification rule:** every tier that creates a real `PxKvStore`, -`PxGroup`, or `crowdb-paxos-kv-server` process must verify two things before -proceeding to tier-specific assertions: - -1. **Leader election succeeded within a bounded timeout.** If leader - election does not complete within the timeout (typically 10 s for - multi-node, 3 s for single-node), the test fails — slow election is a - bug, not a flaky test. The test must assert that exactly one leader - exists, not just that "some leader appeared." - -2. **Basic KV CRUD correctness via the client library.** After leader - election, perform a minimal CRUD cycle through `CrowdbClient` (or - direct `PxKvStore` API for lower layers) and verify correctness: - - Put a key → Get returns the same value - - Overwrite → Get returns new value - - Delete → Get returns not-found - - Scan returns expected keys - - **Edge-case coverage** (at least one test per layer must cover these): - - Empty key (`""`) - - Large key (≥1 KB) - - Key with special bytes (null bytes, high-UTF8, whitespace-only) - - Large value (≥1 MB) - - Small value (1 byte) - - Empty value (`""`) - - Performance is not asserted here; correctness only. A put that takes - 5 s but returns `ok=true` and is readable via get is a pass. Performance - tuning is tracked separately (see Benchmark row in the binary map above - and [`kv-write-flow-analysis.md`](kv-write-flow-analysis.md)). - - KV operations are crowdb-rpc only (no REST API for KV). Tests use - `CrowdbClient` (deployment/UI layers) or `PxKvStore` public API - directly (group/store layers). The console mgmt API layer verifies - topology management via REST but does not perform KV operations; KV - correctness is delegated to the deployment or UI E2E layer. - -**Leader change & reconfig rule:** every tiered layer (Group, Store, -Deployment, UI E2E) that creates a multi-replica cluster must cover: - -1. **Leader change:** trigger step-down on the current leader, verify a new - leader is elected within bounded timeout (10 s multi-node, 3 s - single-node), verify KV operations continue to work after the new leader - is established, verify the old leader rejoins as follower after restart - (if applicable). - -2. **Reconfig — add replica:** add a new replica to a running group with - existing data, verify the new replica catches up (data visible via scan - or get on the new node) within bounded timeout. Since tests write few - keys, catch-up should complete in seconds. Slow catch-up is a bug. - -3. **Reconfig — remove replica:** remove a non-leader replica from a running - group, verify the group continues to accept KV operations (quorum - intact), verify health status reflects the reduced membership. - -4. **Reconfig — remove leader:** remove or stop the leader's replica, verify - a new leader is elected within bounded timeout, verify KV operations - resume. This is the most operationally sensitive scenario; the test must - not block indefinitely waiting for election. - -### 4.1 Runtime resource ownership - -Every test environment that starts subprocesses owns one runtime namespace. -The default root is the workspace-local `.crowdb-runtime/`; tests do not use -the system temporary directory. Its lifecycle classes are: - -- `ephemeral/` for disposable test and development environments; -- `persistent/` for restartable local clusters; -- `artifacts/` for exported logs, profiles, sanitizer reports, and benchmark - output; -- `ports/` for the cross-process assignment registry. - -A namespace manifest binds one logical service identity to its `data/`, -`config/`, `log/`, and `artifacts/` directories and to stable `ServicePort` -assignments. Starting two environments produces disjoint roots and ports. -Restarting a logical service reuses its existing assignment; replacement uses -a new identity. Child-process port handoff never uses port zero or a released -probe socket. An in-process listener may use port zero only while retaining -the bound socket itself. - -Ephemeral claims include the owning PID and process-start identity so stale -claims can be reclaimed without confusing PID reuse. Persistent assignments -remain claimed across stop/restart and a conflicting port is reported rather -than silently renumbered. Successful tests release their namespace; a panic -preserves its tree for diagnosis. `pixi run clean --runtime-only` removes only recorded -ephemeral processes and state. `pixi run clean` additionally removes -rebuildable artifacts, build output, and the complete `.crowdb-runtime/` tree. - -## 5. Layer Scope - -Each layer tests only the logic that belongs to it; a failure points at the -lowest broken layer. Detailed per-layer coverage checklists live in -[`plan-test.md`](../working/plan-test.md). - -- **Unit** — Pure modules with no I/O, async runtime, or network; - deterministic, microsecond tests. Covers `paxos/{acceptor,learner,error, - roles}`, `kv/{mem_kv,op}`, and the `PxLocalReplica` election state - machine (role transitions, vote granting, heartbeat, lease, term fencing). -- **WAL subsystem** — `WalEngine` and WAL internals: durable log, segment - management, replay, GC, I/O backends, pipeline writer. Uses namespaced - workspace-local filesystems or in-memory simulated disks. -- **Slot subsystem** — `PxSlotList` lock-free chunked sparse array: - single-threaded ops, concurrent stress, reclamation watermark - interactions with long-lived read guards. -- **Replica** — Single `PxLocalReplica` with no peers: acceptor + learner + - WAL + slot integration: prepare/accept with WAL persistence, dedup, - snapshot install, WAL replay ordering. -- **Group** — `PxGroup` with 1 local + N remote replicas over real loopback - crowdb-rpc (no mocks): full Paxos rounds, leader election, KV through the - group, durability under crash/restart. Includes RPC boundary rejection - tests: malformed `EAcceptRequest` frames (wrong table type, missing - required `value` field) are rejected with `InvalidArgument` via the - `test-util`-gated `send_raw_request` on `PxRpcTransport`. -- **Store** — `PxKvStore`: multi-group routing, node identity, lifecycle, - topology status. Uses embedded crowdb-rpc server via `PxKvStore::start`. -- **Client E2E** — `CrowdbClient` against embedded `PxKvStore` crowdb-rpc - servers: retry, topology cache refresh, `NotLeaderHint` follow, - `AnyReplica` read distribution, `MinSlot` lagging-follower fallback. No - `crowdb-paxos-kv-server` binary is booted. -- **Deployment** — `crowdb-paxos-kv-server` binary + HTTP management API + - multi-process clusters: boots the actual binary, exercises the HTTP API, - verifies multi-process cluster formation and lifecycle. -- **Console mgmt API** — Console management API server (Axum) + CLI - frontend: REST endpoints, OpenAPI proxy, API forwarding to - crowdb-paxos-kv-server nodes, shared core (config, API client, health - aggregation), CLI commands. KV correctness is delegated to the - deployment or UI E2E layer. -- **UI E2E** — Playwright browser tests against a real `crowdb-web` + - `crowdb-paxos-kv-server` backend: drives the SPA as an operator would (clicks, - context menus, dialogs, KV panel), verifies both UI feedback and backend - state. No mocks; the only network interception is route aborting for the - backend-unreachable test. -- **Benchmark** — Self-contained `bench benchmark` lifecycle: deploy a - 3-node cluster, drive write/read load, collect server-side metrics + - logs, produce a report, then clean up. Exercises the full production - path end-to-end. Design, storage modes, and baseline results are in - [`kv-write-flow-analysis.md`](kv-write-flow-analysis.md). - -## 6. crowdb-tree C++ Test Layers - -The C++ crowdb-tree library (`libcrowdb-tree`) has its own test layers, separate -from the Rust test binaries above. They run as `test-tree-ct` in CI. - -| Layer | Where | What it proves | -| --- | --- | --- | -| C++ unit | `liblib/crowdb-tree/tests/unit` | Single component correctness: cell encoding, leaf page build/read, delta replay, consolidation triggers, mapping table, epoch manager, split point. | -| C++ integration | `liblib/crowdb-tree/tests/integration` | Multi-component flows over `InMemoryPageStore` and `FilePageStore`: basic CRUD, batch apply, scan, split/merge, consistent view, snapshot roundtrip, GC. | -| Crash/recovery | `liblib/crowdb-tree/tests/recovery` | Durability: snapshot + recover, torn-page, superblock A/B, double-apply idempotency. Uses a `FaultyPageStore` with fault points for the FI matrix. | -| Rust FFI | `crowdb-paxos-kv` `tests/` | `CrowdbTreeEngine` trait conformance (shared parametrized tests with `InMemKV`), async bridge, buffer ownership, error mapping. | -| Cross-engine parity | `crowdb-paxos-kv` `tests/` | `InMemKV` and `CrowdbTreeEngine` produce identical state via `compare()` after random op streams, snapshot export/import round-trips, and mid-stream restart. | -| Concurrency | sanitizer CI | No data races / UAF under reader+writer load (TSan/ASan/UBSan); epoch reclamation under load; version pin + GC. | - -The authoritative correctness oracle is **`compare()` against `InMemKV`**: for -any op sequence, the two engines' `EngineView::iter_all` must be byte-for-byte -equal (same key set, same `(slot, cell)`). - -## 7. Sequencing - -Fill gaps bottom-up so a new failure is always attributable to the lowest -layer: -1. Unit + WAL/slot + Election — cheap, deterministic. -2. Replica layer — highest-value gap; unblocks confident group debugging. -3. Group reconfiguration + peer RPC transport. -4. Multi-node store and deployment re-enables, after repair-correctness - fixes tracked in [`plan-test.md`](../working/plan-test.md). - -## 8. Test Pairing Rule - -Every feature or component milestone includes: - -1. **Unit invariants** from the matching test layer (property-based or - deterministic). -2. **Failure-injection** matching [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §9 - scenarios. -3. **Benchmark integration test** (end-to-end correctness + performance) - once the RPC/client layer is reached. - -See [`plan-test.md`](../working/plan-test.md) for pending test task tracking. diff --git a/doc/design/kv/design-crowdb-paxos-kv-wal.md b/doc/design/kv/design-crowdb-paxos-kv-wal.md deleted file mode 100644 index 915ff2e54..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-wal.md +++ /dev/null @@ -1,626 +0,0 @@ - - - -# CROWDB - Design: Write-Ahead Log - -Depends on: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §8.1, §8.2 -Satisfies: [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §8.1, §8.2 - -This document specifies CROWDB's write-ahead log. **There is exactly one durable -log per group: the replica's consensus log (the per-slot acceptor log).** -Everything else (the KV state machine (`KVEngine`), future snapshots, the dedup -cache, the chosen/applied watermarks) is a *derived projection* of that log. -The `Accepted` record carries the actual operation, so this single log doubles -as the replicated command log (Raft-style) and the state-machine input; there is -no separate "state-machine WAL" and no second write of the value. - -Durability rule of thumb: **if an acknowledged write is in the WAL on a quorum, -it cannot be lost** — a value is *chosen* exactly when a quorum has durably -`Accepted` it, and replay + recovery reconstruct the committed state from those -durable accepts. - -## Table of Contents - -- [1. Goals and Non-Goals](#1-goals-and-non-goals) -- [2. Logical Record Shape](#2-logical-record-shape) -- [3. Multi-Disk Segment Layout](#3-multi-disk-segment-layout) -- [4. Write Path and Batched Durable Flush](#4-write-path-and-batched-durable-flush) - - [4.6 I/O Backend Abstraction](#46-io-backend-abstraction) -- [5. Ack Contract and Failure Modes](#5-ack-contract-and-failure-modes) -- [6. Replay, Restore, and Recovery](#6-replay-restore-and-recovery) -- [7. Garbage Collection](#7-garbage-collection) -- [8. Disk Loss and Recovery](#8-disk-loss-and-recovery) -- [9. Tunables and Defaults](#9-tunables-and-defaults) -- [10. Follow-up Implementation Scope](#10-follow-up-implementation-scope) - ---- - -## 1. Goals and Non-Goals - -**Goals:** - -- Bound write latency by the slower of (one durable flush) and (one quorum RTT). -- Saturate aggregate disk bandwidth by parallelizing across multiple physical disks. -- Survive arbitrary process crash with bounded recovery time. -- Survive single-disk loss without data loss (peer recovery via snapshot install). -- Be replayable into a deterministic state. - -**Non-goals:** - -- Cross-group log ordering. Each group has its own WAL; there is no cluster-wide log. -- Compression / deduplication at the WAL layer. (Possible future work.) -- Multi-version time-travel reads. The WAL is not queried by the read path. -- Erasure coding of records. Replication is at the consensus layer, not the disk layer. - ---- - -## 2. Logical Record Shape - -A WAL record persists one of the messages an acceptor must remember durably. Conceptually: - -| Field | Purpose | -| --- | --- | -| `magic`, `version` | Format identifier, for upgrade compatibility | -| `record_type` | One of `{ Promised, Accepted, VoteGranted }` | -| `group_id` | Which group this record belongs to | -| `term` | Acceptor's `current_term` at the time of record (for fence verification on replay) | -| `slot` | Slot number this record concerns (n/a for `VoteGranted`) | -| `ballot` | `(round, leader_id)` for `Promised` / `Accepted` | -| `payload` | Type-specific body — for `Accepted`, the `PxLogEntry` (kind + batch); for `Promised`, empty; for `VoteGranted`, the voted-for node id | -| `payload_len` | Length, for forward-skipping malformed records | -| `crc32c` | CRC of header + payload | - -Records are self-describing and length-prefixed so a corruption in one record does not destroy the rest of the segment unless followed by a CRC failure (see §6). - -**Design note:** the `Accepted` record's payload IS the `PxLogEntry`; there is no separate "raft log" or "state machine log" structure. This keeps disk I/O minimal: one durable flush per accepted batch, no further logging. - -`Promised` records exist for safety: classical Paxos requires acceptors to remember promises across crashes. They are short (no payload). - -### 2.4 When each record is written - -The WAL is written **only on the acceptor/election critical path**, before the -handler replies (the ack contract, §5). The mapping is exhaustive: - -| Handler (`PxLocalReplica`) | Trigger | Record | Payload | -| --- | --- | --- | --- | -| `on_prepare` | acceptor grants a promise | `Promised` | none (slot + ballot + term in header) | -| `on_accept` | acceptor accepts a value | `Accepted` | the `PxLogEntry` (kind + KV batch) | -| `handle_request_vote` | acceptor grants a vote | `VoteGranted` | `voted_for` node id (term in header) | - -**`learn()` writes nothing.** Applying a chosen value to the `KVEngine` is a pure -in-memory projection; its durability comes entirely from the `Accepted` records -that a quorum already persisted. The state machine's KV state is **not** rebuilt -from the WAL on restart. Instead, the state machine will persist its own -snapshot (with a slot index) to disk. On restart, the state machine loads its -snapshot and the WAL replay only rebuilds acceptor state (promises + accepted -values). The learner / `KVEngine` is repopulated by new-leader recovery (§6.4) -or steady-state heartbeat catch-up (§6.5). - -This design avoids a separate `DurableCommitWatermark` WAL record type. The -WAL carries only consensus state (`Promised`, `Accepted`, `VoteGranted`); the -state machine owns its own persistence boundary via snapshots. - ---- - -## 3. Multi-Disk Segment Layout - -### 3.1 Why multiple disks - -A single disk's fsync latency caps single-group write throughput at `1 / fsync_latency` IOPS, which on commodity NVMe is ~10–30k. Multiple disks give us linear scaling up to the network/CPU limit. - -### 3.2 Layout - -Each acceptor configures `wal_disks = [path_1, path_2, ...]`. Each disk holds a sequence of **segments**. A segment is a single append-only byte stream containing records in append order for that disk. It is **not** required to contain a contiguous slot interval. - -With multiple disks, global slot order and physical record order intentionally diverge: - -``` - /wal_disk_1/groupN/seg-0000001.ck slots {1,4,7,...} - /wal_disk_1/groupN/seg-0000004.ck slots {361,364,367,...} - /wal_disk_2/groupN/seg-0000002.ck slots {2,5,8,...} - /wal_disk_2/groupN/seg-0000005.ck slots {362,365,368,...} - /wal_disk_3/groupN/seg-0000003.ck slots {3,6,9,...} -``` - -- Each segment is named with a monotonic `segment_id` within the group. -- Segments are pre-allocated to a target size (`wal_segment_size`, default 64 MiB) and sealed when full. -- Each sealed segment records `(min_slot, max_slot, record_count)` as scan metadata; `min_slot..max_slot` may contain holes because slot ownership is distributed across disks and because non-slot records exist. -- Replay correctness comes from sorting / indexing records by `(group_id, slot, ballot, record_type)` after reading them, not from physical segment order. - -**Slot-to-segment mapping** is recorded in an in-memory **segment index** that -points each slot to its latest `(disk, segment_id, file_offset)` record. The -live index has one mutex-protected shard per disk pipeline. Each writer mutates -only its own shard, and slot lookup uses the same deterministic affinity as the -write path. Replay and GC snapshot shards one at a time; file scanning and -unlink I/O never run while an index shard is locked. A persisted index may be -used as a startup accelerator, but it is only a cache: replay must be able to -rebuild the authoritative index by scanning segment records. - -### 3.3 Slot-to-disk assignment - -When the acceptor receives a slot-addressed record (`Promised` or `Accepted`) for slot N, it chooses the target disk by deterministic slot affinity: - -``` - disk = hash(group_id, slot) % wal_disks.len() -``` - -This is the default and correctness-oriented mapping. All records for the same `(group_id, slot)` land on the same disk, so replay can recover that slot without merging same-slot histories from multiple disks. The mapping is stable across restart as long as `wal_disks` order is stable. - -Each disk owns an independent append queue, active segment, flush worker, and durable-flush state. Concurrent slot writes still parallelize because adjacent slots hash across disks, but same-slot retries and higher-ballot re-accepts preserve disk locality. - -Non-slot records use fixed lanes: - -- `VoteGranted` and other election metadata go to the group metadata lane (`disk = hash(group_id, 0) % wal_disks.len()`). -- Group membership configuration is persisted in a dedicated config file (`group_config.bin`) in the group's WAL directory, not as a WAL record. - -A future load-aware mode is only legal if it preserves slot affinity, for example by moving whole slot shards through an explicit re-sharding protocol. Per-record load-aware placement is not allowed for slot-addressed records. - -Because apply order is determined by the slot index in memory, the *physical* order of records on disk does not affect correctness (Invariant I4). This is what allows multiple disks to flush in parallel. - -### 3.4 Segment rotation - -A segment is rotated (sealed and a new one opened) when it reaches -`wal_segment_size`. (Future: slot-range boundary rotation for GC granularity, -admin force-rotate, and leader-change rotation are design options not yet -implemented.) - -Sealed segments are immutable. - ---- - -## 4. Write Path and Batched Durable Flush - -### 4.1 Goal of batching - -Each individual `Accept` is small (tens of bytes overhead + payload). Per-record durable flush is wasteful: an SSD's flush amortizes over a batch nearly as well as over a single record. CROWDB therefore batches records per WAL disk and completes the record futures only after the selected backend has made the bytes durable. - -The durable operation is backend-specific: - -- **Filesystem file backend:** append with normal file writes, then `fdatasync` / equivalent durable flush. -- **Block-device backend:** issue aligned positional writes, using direct I/O - when selected. If the device requires 4 KiB sectors, the WAL writer pads or - read-modify-writes through the alignment planner. The real file-backed - implementation then calls `sync_data` before releasing durable acknowledgments. -- **RAM / SCM / simulated backends:** use the backend's own durability contract. A volatile RAM backend is for tests only and may complete immediately; persistent-memory backends may require cache-line flush / fence semantics instead of filesystem sync. - -The acceptor code talks to `WalEngine::append_*` and does not branch on these backend details. - -### 4.2 Batch coalescing - -For each disk, the WAL maintains a **pending queue**: - -``` - pending_queue[disk] = [ record_1, record_2, ..., record_k ] -``` - -Records arrive concurrently from many `Accept` handlers. They are appended to the in-memory segment buffer for that disk and pushed onto the disk's pending queue. A single **flush worker** per disk owns the active segment, drains queued records into one backend write batch, calls the backend-specific durable flush, and then **completes all pending records' futures together**. - -The worker is event-driven, not timer-first: - -1. If the queue is empty, the worker awaits a notification from the next enqueue. -2. When notified, it drains all records currently available without waiting for a fixed 1 ms interval. -3. If more records arrive while the write is being assembled, they are included until the batch reaches `wal_flush_batch_bytes` or the drain loop observes no immediately-ready records. -4. The worker issues one backend write / durable-flush operation for the drained batch. -5. On success, it resolves every drained record future; on error, it fails them and marks the affected disk / group unhealthy. - -This gives low latency for a single record while still aggregating bursts naturally. - -### 4.3 Triggers for durable flush - -The flush worker flushes when any of: - -- It is woken by a transition from empty queue to non-empty queue (`rx.recv().await`). -- Pending bytes ≥ `wal_flush_batch_bytes` (default 64 KiB): breaks the drain loop early. -- The watchdog (`wal_flush_watchdog_ms`, default 100 ms) is a safety-net timer that wakes the idle writer every `watchdog` ms to drain any queued records in case of a missed wake (a defensive measure "just in case for bugs"). The idle wakeup does a `try_recv` drain and re-parks if nothing is queued; no I/O, no allocation, ~10 wakeups/s at the default 100 ms. - -Default behavior is **wake-drain-flush**: a lone record flushes immediately, while a concurrent burst gets batched because the worker drains all immediately-ready records before issuing I/O. A coalescing budget (`wal_flush_coalesce_us`) was previously available but was removed after T3 benchmarking showed no gain over the wake-drain-flush baseline. - -### 4.4 Async write completion - -Each `Accept` handler returns a future. The flush worker resolves the future with `Ok` after the backend durable flush completes; on disk error it resolves with `Err` (which the acceptor escalates per §8). The acceptor only emits the network `Accepted` response after the future resolves; this is the **ack contract** ([§5](#5-ack-contract-and-failure-modes)). - -The future-based interface decouples the acceptor's handler from disk timing and allows multiple in-flight `Accept`s to coalesce into one durable flush naturally. - -### 4.5 Why this maps onto Multi-Paxos parallelism - -Different slots may be in flight on different disks at the same time. Slot N may be batching on disk 1 while slot N+1 batches on disk 2. Both durable flushes proceed in parallel. Aggregate flush throughput is `sum over disks of (1 / flush_latency)`, exactly the parallel benefit we want. - -Crucially, the leader's **own** durable flush is not on the critical path of remote acceptors' durable flushes. The leader persists locally, then broadcasts; remote acceptors flush in parallel. Two durable flushes total in serial: leader's, and the slowest of the quorum. With multiple disks each durable flush can itself be parallelized internally, but the consensus-level parallelism is what dominates throughput. - -### 4.6 I/O backend abstraction - -The WAL dispatches all file operations through `IoBackend`, a process-lifetime -enum with four variants: - -- **`File`** — `tokio::fs` with an awaited `sync_data` durability barrier. The - production default; works everywhere. A batch acknowledgement is not released - until this barrier succeeds. -- **`Uring`** — explicitly selected buffered regular files. Positional reads, - vectored writes, `fdatasync`, and `fsync` are submitted to one process-shared - single-pipeline io_uring owner; namespace and metadata operations remain on - the portable filesystem path. Ring setup failure rejects explicit selection - rather than falling back. Borrowed buffers and descriptors remain live until - the CQE is drained, including when a Rust operation future is dropped. -- **`MemBlock(MemBlockDevice)`** — in-memory test harness. Stores segments as - `BTreeMap>` with error injection (`inject_io_error`, - `inject_sync_error`, `set_full`), corruption injection - (`corrupt_at_offset`), and layout tracking. `fdatasync` is a no-op (data is - already "durable" in RAM). Used by all unit/integration tests. -- **`BlockDevice(BlockDevice)`** — real file-backed block device. Always opens - OS paths and does positional I/O (`pwrite`/`pread`) via `FileExt`. - `fdatasync` calls `sync_data`. Whether the path is a regular file, a raw - block device, or a tmpfs file makes no difference; in Unix, a block device - *is* a file. - -On the production multi-thread Tokio runtime, including a runtime with one -worker, synchronous block-device file and layout operations use Tokio's -[`block_in_place`](https://docs.rs/tokio/1.52.3/tokio/task/fn.block_in_place.html). -This allows independently spawned apply, RPC, and timer tasks -to run while the OS call blocks. The closure completes before returning, so -borrowed WAL buffers and the file remain live without an extra payload copy. -Write, durable flush, index publication, and acknowledgment keep their original -ordering and error behavior. This changes scheduling rather than durability, -timeouts, retries, or configured worker/resource limits. Current-thread and -non-runtime callers retain synchronous compatibility; they do not gain -concurrent task progress. The production path uses independently spawned WAL -workers on the ordinary multi-thread runtime, not a `LocalSet` or same-task -`join!` branches. - -`BlockDevice` has two constructors selected by the `O_DIRECT` flag: - -- `BlockDevice::new()` — aligned (4K), buffered (no `O_DIRECT`). Default for - benchmarks and general use. All alignment/RMW logic runs, `fdatasync` - performs a real `sync_data`, but the page cache absorbs writes. -- `BlockDevice::ssd()` — aligned (4K), `O_DIRECT` (Linux only). Models a real - SSD/NVMe with direct I/O bypassing the page cache. - -`MemBlockDevice` has two constructors selected by alignment: - -- `MemBlockDevice::new()` — unaligned (IU=1, byte-addressable). Models RAM / - SCM / PMEM. Default for tests. -- `MemBlockDevice::with_alignment(align)` — aligned in-memory. For testing - RMW logic without real files. - -Both `BlockDevice` and `MemBlockDevice` track write counts, fdatasync counts, -logical/physical bytes written, and RMW counts for observability. The -`WalEngine::backend_label()` method returns a short string (`"file"`, -`"uring"`, `"mem"`, `"block"`) for metric names. `WalEngine::block_device_snapshot()` -reads cumulative counters for the engine collector to compute per-window -deltas. - -The WAL bench (`lib/crowdb-paxos-kv/benches/wal.rs`) exercises four backends: `Mem` -(in-memory `MemBlockDevice`), `File` (`tokio::fs`), `Uring` (buffered -CQE-backed file I/O), and `Block` -(`BlockDevice::new()` with `wal_skip_fsync: true`). The `Block` case hits all -block code paths (alignment planning, RMW, amplification tracking, -`pwrite`/`pread` syscalls) at high TPS since `fdatasync` is skipped per -batch. - -The crowdb-tree C++ storage layer has a similar but cleaner separation: -`TextPageStore` (debug text files), `MemPageStore` (in-memory), and -`BlockPageStore` (real block device with `O_DIRECT`). Each is a distinct C++ -class; no flag-based branching. The Rust FFI `PageStoreBackend` enum (`File`, -`Block`, `MemBlock`) maps to these. No structural change was needed on the -crowdb-tree side for this refactor. - ---- - -## 5. Ack Contract and Failure Modes - -### 5.1 The contract - -> An `Accepted` response is sent to the leader only after the corresponding WAL record's backend durable flush has completed. -> A client write is acked only after a quorum of acceptors have sent `Accepted`. - -This is repeated from design-crowdb-paxos-kv.md §8.1](design-crowdb-paxos-kv.md) because everything else in this section is a consequence of it. This is the **strict** mode. The **early-ack** mode (§5.4) relaxes the leader's own durability ordering but keeps the quorum-durability requirement; both modes are described in §5.4. - -### 5.2 Failure cases - -- **Crash before durable flush.** The record may or may not be on disk. Replay reads what is on disk; the slot may end up with no record on this acceptor. That is fine; the leader either had a quorum from other acceptors (slot is chosen) or not (slot will be repaired). In strict mode no client was acked for this acceptor's slot, so no expectation is violated; in early-ack mode (§5.4) the leader may have acked the client on the remote quorum alone, and the chosen-but-not-locally-durable slot is re-adopted by `repair_once` from the followers on restart. - - On restart, the new leader computes its recovery ceiling from durable state, not from an assumed contiguous log tail: `ceiling = max(local highest_seen_slot, peers' highest_seen_slot, persisted next_slot - 1 if present)`. Because parallel writes may leave holes inside `[floor+1, ceiling]`, the leader must run bulk Phase 1 over the whole open interval and fill empty slots with `NoOp`. It does **not** rely on `latest_slot + window_size` to discover hidden data; any value absent from every acceptor's durable state could not have formed a durable quorum and therefore could not have been acknowledged. - -- **Crash after durable flush, before sending `Accepted`.** Replay finds the record. After the acceptor rejoins the group, the leader observes the existing accept (via heartbeat / slot-status query) and includes this acceptor in the quorum. -- **Crash after sending `Accepted`, before the leader acked the client.** Same as above; the record is on disk; if a quorum was reached, the slot is chosen; the new leader's bulk Phase-1 will see this and re-confirm. -- **Durable-flush failure (returns `Err`).** Treated as a disk fault. The acceptor stops accepting new records on that disk and marks itself failed for the affected group; see §8. - -### 5.3 What we never do - -- Ack a client write before a quorum of acceptors have durably flushed. This holds in both ack modes (§5.4): the early-ack mode defers only the *leader's own* durable flush, never the remote quorum's. -- Declare a slot `Chosen` without a quorum of `Accepted` responses (remote durable + leader CAS in early-ack mode; remote durable + leader durable in strict mode). - -### 5.4 Ack modes (`wal_early_ack`) - -The ack contract in §5.1 is the **strict** mode: every acceptor (including the leader) completes its WAL durable flush before its `Accepted` counts toward `Chosen`, and the client is acked only after the leader's own flush joins the quorum. `wal_early_ack` (default `true`, gated on `quorum > 1`) is a **relaxed** mode that drops the leader's local fsync off the write critical path: - -- The leader runs `on_accept_inner` (the term-fence CAS + in-memory `acceptor.accept`) concurrently with the remote `send_accept` fan-out via `tokio::join!`. -- `Chosen` is declared as soon as the **remote quorum** has durably flushed *and* the leader's local CAS has landed. The leader's own WAL `Accepted` record is appended by a fire-and-forget `tokio::spawn` (`spawn_accept_persist`) that runs off the critical path; its failure is logged and does not retract `Chosen`. -- The client is acked at that point. The per-proposal critical path becomes the quorum RPC round-trip only (the local fsync, ~10–100 µs on NVMe, runs concurrently and is no longer on the path). - -**Crash-recovery guarantee.** A crash in the window between the leader's CAS landing and the deferred local persist is Paxos-safe: the value is chosen (remote quorum accepted durably), so the client's observed `Chosen` is not violated. The leader's acceptor state for that slot is lost on crash (the CAS was in memory, the WAL record never landed), but on restart `replay_group` rebuilds from durable state only and `repair_once` re-adopts the chosen value for the gap slot from the followers (classic prepare + accept re-runs and re-chooses the highest-ballot accepted value). A value the client observed as `Chosen` therefore remains readable after crash + restart: either the local WAL raced the kill and has it, or repair re-adopts it from the quorum. - -**Single-node gate.** `wal_early_ack` defaults to `true` only when `quorum > 1`. A single-node group (quorum = 1) has no survivors to re-drive a chosen-but-not-locally-durable slot after a crash, so the leader's local fsync must stay on the critical path and the strict contract (§5.1) applies. - -**What is not relaxed.** The remote quorum's durable flush is still on the critical path in both modes; `Chosen` is never declared on in-memory-only remote accepts. The relaxation is strictly the leader's *own* durability ordering, recovered by `repair_once` rather than awaited inline. - ---- - -## 6. Replay, Restore, and Recovery - -Turning the on-disk WAL back into a live, serving replica has three stages, then -a steady state: - -- **6.1 Replay** — rebuild in-memory *acceptor* state from the records. -- **6.2 Restore** — rebuild a live replica shell from acceptor state (Pass 1), then rebuild the learner / `KVEngine` from that same acceptor state (Pass 2), skipping only the prefix the engine already durably reflects. -- **6.3 Wiring** — attach a fresh WAL engine and seed the proposer conservatively. -- **6.4 Recovery** — a new leader re-confirms *chosen* values from a quorum and fills holes. -- **6.5 Steady-state apply** — followers keep their state machine current from the leader's commit watermark. - -The split matters: replay/restore are purely *local* (this node's WAL), but `Accepted` ≠ `chosen` (a value is chosen only on a quorum). Restore therefore re-derives the acceptor's local view exactly as it was (Pass 1), then re-derives the learner's committed KV state from that *same locally-accepted* data (Pass 2), which is safe precisely because `Accepted` records already carry a chosen-or-not-yet-known value idempotently (re-`learn()`ing a value that was never actually chosen by a quorum is harmless: it just sits in the engine until either superseded by a higher slot for the same key, or never observed as chosen by anyone else, in which case it was never visible to a client anyway). Slots this node never locally accepted are, separately, left to: - -- New-leader recovery / steady-state heartbeat catch-up re-learning the value from quorum-confirmed consensus (§6.4 / §6.5). - -**No `DurableCommitWatermark` WAL record.** The state machine owns its own persistence boundary, but that boundary is the *engine's* durability (crowdb-tree's on-disk snapshot), not a separate state-machine-level snapshot file; see §6.2 below and `design-crowdb-paxos-kv-state-machine.md §2.1`. The WAL does not duplicate this information. - -### 6.1 Replay — rebuild acceptor state (`replay_group`) - -1. Discover all segments under `/group/seg-*.ck`, order by `segment_id`. - A zero-byte final segment on each WAL disk can result from a crash between - file creation and header writing. Remove that empty tail; it contains no - acknowledged records. A nonempty unreadable segment, invalid header, or - empty intermediate segment prevents local recovery; never skip it and serve - a partial acceptor history. The direct block backend - aligns physical reads and copies the requested byte range, including short - header reads and record ranges spanning block boundaries. -2. Walk each segment's records in order; verify `magic`, `version`, `crc32c`. - On the first failure, **truncate that segment at the offset and stop** (a torn - tail from a crash mid-write); later segments are still processed. -3. Rebuild per-`(group, slot)` acceptor state keeping the **highest-ballot** - `Promised` / `Accepted` per slot (later/higher-ballot records win, Paxos rule). - Retain promises and accepted values independently while scanning; release - superseded payloads immediately. Segment locations, counts and election - state still account for every verified record. -4. `current_term` = max `term` across all records. -5. `voted_for` = the node from the latest `VoteGranted` whose `term == current_term`. This is election safety state, not just debug metadata: after crash, the node must not grant a second vote in the same term. -Output: `ReplayResult { records, max_segment_id, current_term, voted_for }`. - -### 6.2 Restore — rebuild live acceptor state (`restore_from_replay`) - -Pass 1 rebuilds acceptor state without assuming any local accept was ever chosen by a quorum: - -- Feed `Promised` through the acceptor restore path so the highest promise per slot is preserved. -- Feed `Accepted` through the acceptor restore path so the highest accepted value per slot is available to future Phase 1. -- Seed `current_term`, `voted_for`, role from `ReplayResult`. - -The learner / `KVEngine` is rebuilt in a second pass over the *same* replayed -WAL records (`PxLocalReplica::restore_from_replay_with_engine`), not from a -separate state-machine snapshot file; see -[`design-crowdb-paxos-kv-state-machine.md §2.1`](design-crowdb-paxos-kv-state-machine.md#21-state-machine-restart-engine-durability--wal-replay) -for the full mechanics. In short: - -- Pass 2 walks every slot with an accepted entry (rebuilt in Pass 1 above) - and `learn()`s it into the engine, in order. This alone is always - sufficient and correct: `KVEngine::apply` is idempotent - (highest-slot-wins per key), so it works unconditionally, including for - `InMemKV` which has no durable floor at all. -- **Optimization for a durable engine:** before Pass 2, the engine reports - its own durable floor via `KVEngine::resume_from_slot()` (crowdb-tree's - `last_applied_slot`, restored from its on-disk snapshot). Pass 2 then - starts just above that floor instead of at slot 1, and the learner's - frontier is fast-forwarded directly to it, skipping the redundant - re-`learn()` of an already-durable prefix. -- Slots above the local WAL's own highest accepted slot are left to §6.4 / - §6.5 (heartbeat catch-up / new-leader bulk Phase 1) to re-learn from peers. - -This avoids resurrecting a value that was accepted locally but never chosen, -and, via the resume-floor check, never re-applies a write below a point -the engine has already durably advanced past (crowdb-tree rejects such writes -outright; see `../tree/design-crowdb-tree-engine.md`). - -### 6.3 Wiring a live group (`create_group_with_wal`) - -`replay_group` → create `WalEngine` and resume `next_segment_id = max_segment_id + 1` → `restore_from_replay` → `set_wal` → wrap in `PxGroup`. - -- Seed the live pipeline index with the verified replay index before starting - writers. Its slot locations and segment metadata cover retained history as - well as future appends, so GC can reclaim pre-restart segments. -- Initialize the WAL's local snapshot watermark from the freshly opened - engine's durable resume floor. Peer progress and the reclamation certificate - remain independent gates; local recovery does not authorize prefix removal. - -The group also records a conservative local slot tip: - -``` - local_tip = max(highest_seen_slot, snapshot_slot) -``` - -Where `snapshot_slot` is the state machine's persisted snapshot slot (0 if no snapshot exists yet). - -A restarted node must not serve as leader until election and §6.4 recovery establish the quorum-derived ceiling. After recovery is issued, the leader seeds new client proposals at `next_slot = ceiling + 1`. - -### 6.4 New-leader recovery (bulk Phase 1) - -On winning election a leader sweeps slots `(floor, ceiling]`: - -- `floor = local contiguous_chosen`; `ceiling = max(local highest_seen, - next_slot − 1, peers' highest_seen)` (peers' values arrive in the vote replies). -- For each slot it runs Phase-1 `Prepare` across a quorum and **adopts the - highest-ballot accepted value** observed (an empty slot becomes a `NoOp`), then - re-`Accept`s under its term and `learn()`s it. -- Quorum intersection guarantees any chosen value is seen, so a committed value - is **re-confirmed, not overwritten**. This repairs a leader that restored an - incomplete or merely-accepted prefix. - -(Steady-state gap repair below the frontier uses the same adopt-from-quorum logic, one slot at a time; see [`design-crowdb-paxos-kv-slot.md`](design-crowdb-paxos-kv-slot.md).) - -Parallel slot assignment means holes are normal after crash / restart. Recovery resolves every slot in `(floor, ceiling]` independently: - -- If Phase 1 observes an accepted value from any quorum member, the leader adopts the highest-ballot value and re-accepts it. -- If Phase 1 observes no accepted value for that slot, the leader writes a `NoOp` for the slot. -- `NoOp` has no KV payload, but it is a real chosen slot and advances contiguous watermarks; future replay sees it and does not need to repair the hole again once it is covered by the state machine snapshot. - -Slot indexes are generated only by the active leader's proposer: - -1. A leader tenure is fenced by election `term`; stale leaders are rejected by term / ballot checks. -2. During recovery, the leader computes `ceiling` from local and quorum-reported highest seen slots. -3. It resolves all holes up to `ceiling` with adopted values or `NoOp`. -4. It initializes `next_slot = ceiling + 1`. -5. Each admitted client proposal atomically fetches and increments `next_slot` exactly once. - -This gives unique slot numbers within a leader tenure and a continuous chosen prefix after recovery, even though physical WAL placement is non-contiguous and multiple slots are processed in parallel. - -### 6.5 Steady-state apply - -- **Leader:** applies on commit (`propose` → quorum `Accept` → `learn`). -- **Follower:** does **not** apply on `Accept` (which only persists) nor on the - payload-less `ChosenNotice` (which carries no value). Instead each heartbeat - carries the leader's `committed_safe_slot`, and the follower applies its - *contiguous accepted prefix* up to it (`apply_committed_up_to`). A follower - missing a committed `Accept` is re-sent it by the leader's heartbeat catch-up, - so its prefix converges over successive heartbeats. -- **Reads:** linearizable reads run on the leader behind a lease / ReadIndex - barrier (non-leaders redirect); bounded-stale / best-effort reads serve local - applied state. - -### 6.6 Truncation safety - -A truncated record is one whose CRC fails or whose payload is shorter than -`payload_len` (crash mid-write). Truncate-on-failure is safe because appends are -append-only and the bad record is the last durable one on that disk; the missing -slot was either chosen on a quorum without this node (recovered by §6.4) or never -chosen (no client was acked). A CRC failure inside a **sealed** segment is fatal: -the node fails out of the group and rebuilds from peers via snapshot install. - -### 6.7 Replay performance - -Binary replay uses a 1 MiB read-ahead buffer for frame markers, format probes -and complete frames. A larger frame receives enough space for its own length. -Every frame still passes its normal CRC and format checks. Retained payloads -own their bytes so replacing an accepted record can release it independently -of neighboring records in the input buffer. Memory scales with the retained -highest-ballot state and index, rather than the number of replaced versions. - ---- - -## 7. Garbage Collection - -WAL GC is the mechanism that keeps disk usage bounded. It interacts with snapshots and the safe-slot. - -### 7.1 GC watermark - -The local watermark is `min(reclamation_certificate, local_contiguous_applied, -local_durable_snapshot)`. The leader derives the certificate from every -configured learner's applied progress and durable snapshots on itself plus a -voting peer (local durability alone for a singleton voting group). Followers -receive the certificate in a heartbeat bound to the current membership epoch, -term and known voting leader. A missing certificate or local snapshot gives zero. - -Only records with `slot < gc_slot` are eligible. The same boundary trims -in-memory acceptor history; retained readers keep their owners alive. The full -contract is in [slot retention](design-crowdb-paxos-kv-slot.md#11-interaction-with-snapshot-and-wal-gc). - -### 7.2 GC granularity - -GC runs at **segment granularity**, not record granularity. A whole segment is unlinked from disk when *every* record in it has `slot < gc_slot`. This is cheap (one `unlink()` per segment) and avoids any in-place rewriting. - -- Slot-based eligibility does not retire election state. Retain a segment - proving the highest term and the segment containing the latest vote at the - highest recorded vote term, even when their slot records are below the GC - watermark. A mixed vote/data segment follows the same rule. -- Derive these election fields from every verified record during replay and - update them as live segments are written. They are in-memory index metadata; - WAL record and segment formats remain unchanged. - -### 7.3 GC trigger - -GC is triggered by: - -- Periodic tick (default 30 s): the GC worker checks each disk and unlinks eligible segments. -- Disk-pressure signal (not yet implemented): if any WAL disk's usage exceeds `wal_disk_high_watermark` (default 80%), GC would run immediately and may also force a snapshot to advance `snapshot_slot`. - -### 7.4 Interaction with leader change - -After a leader or membership change, fresh progress and a matching certificate must be established. Until then reclamation pauses. Snapshot creation remains local and can continue independently. - -### 7.5 Force-retain window - -A configurable `wal_min_retention` (default 1 hour, optional) would keep even GC-eligible segments around for a grace period to support post-incident debugging. **Not yet implemented** — the GC worker currently skips the retention check (V1 simplification). - ---- - -## 8. Disk Loss and Recovery - -### 8.1 Single-disk failure - -When the backend durable-flush operation returns an error, the OS reports the disk read-only, or repeated I/O errors exceed a threshold, the acceptor declares the disk failed: - -1. Stop using the disk for new writes (the `failed` AtomicBool is set; all subsequent `append` calls return `Err`). -2. Mark the group affected (a multi-disk WAL with one disk lost has incomplete state for slots that landed on the lost disk). -3. **Fail the node out of the group** (not yet implemented): the node would send a step-out RPC to the leader, the leader records the failed acceptor as not-eligible, and (if necessary) triggers a reconfiguration to maintain quorum. -4. Rebuild from peers via **snapshot install** ([§8.4 of design-crowdb-paxos-kv.md](design-crowdb-paxos-kv.md#84-snapshot-and-install)). - -This is the **fail-out semantics** decided in design-crowdb-paxos-kv.md §8.1](design-crowdb-paxos-kv.md). We do not try to keep operating with partial WAL state on the surviving disks; that path requires per-slot replication across disks and adds disproportionate complexity. - -### 8.2 All-disk failure - -The node is non-functional; it cannot serve any group. It must be replaced by an operator. After replacement (with empty disks) the new node bootstraps via Group-0 and snapshot installs all groups it should host. - -### 8.3 Detecting silent corruption - -CRC32C catches all common bit-flips and torn writes. Logical corruption (e.g. a slot record with valid CRC but inconsistent term/ballot relative to other state) is detected by replay sanity checks; if found, the node fails out of the affected group. - -### 8.4 Network-level data loss is not a WAL concern - -The WAL is per-node. Inter-node consistency is the consensus layer's job. If a network drops a record, the leader retransmits; if it drops repeatedly, gap repair handles it. - ---- - -## 9. Tunables and Defaults - -| Parameter | Field in `WalConfig` | Default | Notes | -| --- | --- | --- | --- | -| `wal_disks` | `wal_disks` | required | ≥ 1; more → higher throughput | -| `wal_segment_size` | `wal_segment_size` | 64 MiB | Trade GC granularity vs file count | -| `wal_flush_batch_bytes` | `wal_flush_batch_bytes` | 64 KiB | Batch cap per durable flush | -| `wal_flush_watchdog_ms` | `wal_flush_watchdog_ms` | 100 ms | Safety-net timer: wakes idle writer to drain missed wakes | -| `wal_disk_high_watermark_pct` | `wal_disk_high_watermark_pct` | 80% | Triggers eager GC (not yet implemented) | -| `wal_min_retention_secs` | `wal_min_retention_secs` | 3600 (1 h) | Forensics retention (not yet implemented) | -| `gc_tick_secs` | `gc_tick_secs` | 30 s | GC scan cadence | -| `wal_aligned` | `wal_aligned` | false | Selects block-aligned I/O backend | -| `wal_io_unit_bytes` | `wal_io_unit_bytes` | 4096 | IU size when `wal_aligned` is true | -| `wal_record_format` | `wal_record_format` | Auto | Binary (zero-copy) or text-line | -| `wal_early_ack` | `CROWConfig.wal_early_ack` | `true` (gated on `quorum > 1`) | Early-ack mode (§5.4): declare `Chosen` on remote quorum durable flush + leader CAS, deferring the leader's local WAL persist to a background spawn. Default `false` for single-node groups (quorum = 1); no survivors to re-drive a chosen-but-not-locally-durable slot. | - -**Choosing durable-flush batching:** - -- Latency-critical, low write rate → defaults (wake-drain-flush; no fixed delay). -- Throughput-oriented → `wal_flush_batch_bytes = 64–512 KiB`. -- WAN replication piggybacking → tune batching only after measuring end-to-end RTT and tail latency. - -**Choosing segment size:** - -- Few large segments → fewer files, faster replay scan. -- Many small segments → finer-grained GC, more files. -- 64 MiB is the engineering compromise. - -**Choosing disk count:** - -- 1 disk: single durable-flush limit. Fine for development. -- 2–4 disks: typical production for a write-heavy workload. -- More than 4: usually network or CPU becomes the bottleneck before more disks help. - ---- - -## 10. Follow-up Implementation Scope - -Most of the design above has been implemented. The following items are **not yet -implemented**: disk-pressure-triggered eager GC (§7.3), retention window (§7.5), -and the full fail-out procedure with step-out RPC and reconfiguration (§8.1). -Remaining pending test tasks are tracked in [`plan-test.md`](../working/plan-test.md): - -- Slot-affinity WAL placement instead of per-record round-robin. -- Backend-specific durable flush and alignment semantics. -- Event-driven wake-drain-flush workers. -- Replay-only restore for acceptor state, with learner apply gated by durable commit evidence. -- New-leader recovery ceiling, `NoOp` hole fill, and post-recovery `next_slot` seeding. -- Dedup checkpoint documentation and tests. diff --git a/doc/design/kv/design-crowdb-paxos-kv-watch-notify.md b/doc/design/kv/design-crowdb-paxos-kv-watch-notify.md deleted file mode 100644 index 6d8cc1c4e..000000000 --- a/doc/design/kv/design-crowdb-paxos-kv-watch-notify.md +++ /dev/null @@ -1,894 +0,0 @@ - - - -# CROWDB - Design: crowdb-paxos-kv Watch/Notify Extension - -Depends on: [`design-crowdb-paxos-kv-rpc.md`](design-crowdb-paxos-kv-rpc.md) §3, §5, §6; [`design-crowdb-paxos-kv.md`](design-crowdb-paxos-kv.md) §3; [`design-crowdb-paxos-kv-state-machine.md`](design-crowdb-paxos-kv-state-machine.md) §2 -Satisfies: [`design-crowdb-diskdb.md`](../diskdb/design-crowdb-diskdb.md) §8 (Follow-up — group-0 notify/watch), §15 - -This document covers the crowdb-paxos-kv watch/notify extension, the diskdb -server-side notify handler, and the client endpoint cache proactive -refresh that together replace fixed-interval polling as the primary -change-detection mechanism for group-0 sysdata. The notify is a -**client-pulled `WatchNotify` bidi stream**: diskdb opens the stream to -the group-0 leader and subscribes to prefixes, and the leader pushes -notifies over that stream. No separate notify-endpoint registration is -needed for notify delivery. Polling stays as a safety net at a raised -interval. Architecture decisions and rationale live here; the root -design context is `design-crowdb-diskdb.md` §8 and -`design-crowdb-paxos-kv-rpc.md` §3. - -## Table of Contents - -- [1. WatchNotify crowdb-rpc Bidi Stream](#1-watchnotify-crowdb-rpc-bidi-stream) - - [1.1 Why](#11-why) - - [1.2 Schema](#12-schema) - - [1.3 Server-side handler](#13-server-side-handler) -- [2. Watch Registry + Apply-Path Trigger](#2-watch-registry--apply-path-trigger) - - [2.1 Why](#21-why) - - [2.2 WatchRegistry](#22-watchregistry) - - [2.3 PxLearner integration](#23-pxlearner-integration) - - [2.4 PxGroup integration](#24-pxgroup-integration) - - [2.5 Why not the proposal path](#25-why-not-the-proposal-path) -- [3. Coalescing (deferred)](#3-coalescing-deferred) -- [4. WatchNotify Client](#4-watchnotify-client) - - [4.1 Why](#41-why) - - [4.2 WatchNotifyClient](#42-watchnotifyclient) -- [5. diskdb Notify Handler + Polling Safety Net](#5-diskdb-notify-handler--polling-safety-net) - - [5.1 Why](#51-why) - - [5.2 Trigger extension](#52-trigger-extension) - - [5.3 KeepAlive changes](#53-keepalive-changes) - - [5.4 Notify handler task](#54-notify-handler-task) - - [5.5 Polling safety net](#55-polling-safety-net) -- [6. Client Endpoint Cache Proactive Refresh](#6-client-endpoint-cache-proactive-refresh) - - [6.1 Why](#61-why) - - [6.2 Design](#62-design) -- [7. Configuration](#7-configuration) - - [7.1 diskdb config](#71-diskdb-config) - - [7.2 crowdb-diskdb-client config](#72-crowdb-diskdb-client-config) -- [8. Module Structure](#8-module-structure) -- [9. Server Wiring](#9-server-wiring) -- [10. Test Design](#10-test-design) - ---- - -## 1. WatchNotify crowdb-rpc Bidi Stream - -### 1.1 Why - -crowdb-paxos-kv has no watch/notify capability without this extension. The only -change-detection mechanism available to external components is polling -(prefix scans of group 0). A push mechanism requires a long-lived -stream from the group-0 leader to each watcher, fired when a watched -prefix is written. `WatchNotify` uses a persistent crowdb-rpc connection: -the client submits a fire-and-forget subscription frame and registers push -handlers for notify and error frames on that connection. - -### 1.2 Schema - -`lib/crowdb-paxos-kv/src/rpc/fbs/kv_client.fbs` carries the watch/notify messages -and one bidi RPC on `KvService`: - -```fbs -// Client-to-leader watch subscription. The client opens a WatchNotify -// bidi stream to a node, sends WatchSubscribe frames for each -// (group_id, prefix) it wants to watch, and receives WatchNotify frames -// when watched keys change. If the node is not the leader of the -// subscribed group_id, it returns a WatchNotifyError with -// not_leader_hint and the client reconnects to the leader. -message WatchSubscribe { - uint32 version = 1; - uint64 group_id = 2; - bytes prefix = 3; // watch all keys with this byte prefix -} - -message WatchUnsubscribe { - uint64 group_id = 1; - bytes prefix = 2; -} - -// Pushed from leader to watcher when a watched key changes. Carries -// the changed keys and their latest values so the watcher can act -// without a re-read. slot = the apply slot of the triggering write. -message WatchNotify { - uint64 group_id = 1; - bytes prefix = 2; // which watched prefix matched - repeated bytes keys = 3; // changed keys (deduplicated) - uint64 slot = 4; - repeated bytes values = 5; // values[i] = latest value for keys[i] - // (empty bytes for a Delete tombstone) -} - -message WatchNotifyError { - uint64 group_id = 1; - string not_leader_hint = 2; // empty for non-leader errors - string error = 3; -} - -message WatchNotifyRequest { - oneof frame { - WatchSubscribe subscribe = 1; - WatchUnsubscribe unsubscribe = 2; - } -} - -message WatchNotifyResponse { - oneof frame { - WatchNotify notify = 1; - WatchNotifyError error = 2; - } -} -``` - -Add to the `KvService` service: - -```fbs - rpc WatchNotify(stream WatchNotifyRequest) returns (stream WatchNotifyResponse); -``` - -- **Field numbers are append-only** — matches the RPC design's - compatibility rule (`design-crowdb-paxos-kv-rpc.md` §5). -- **`bytes` fields** — `prefix` and `keys` are raw key bytes as stored - in the engine. For group-0 sysdata these are UTF-8 text paths (e.g. - `/hw/disk/1/100/5/abcd`) because `HardwareClient` puts keys via - `key.to_path().as_bytes()` and scans via `prefix.as_bytes()`. For - data groups the keys are binary-encoded (`BinaryKey::to_bytes()`). The - schema is encoding-agnostic; the subscriber and the engine agree on the - encoding because they both use the same key types. No `Bytes` mapping - needed — group-0 keys are text paths < 128 bytes; `Vec` is fine. -- **Fat notify (keys + values)** — `values[i]` is the latest value for - `keys[i]` (empty bytes for a `Delete` tombstone). `WatchRegistry::emit` - extracts the value from each `BatchOp` (`Op::Put(v)` → value bytes, - `Op::Delete` → empty) and includes it in the notify frame. The client - can act on a notify without a re-read, eliminating the - notify-storm → re-read-storm amplification. The `keys`-only contract - is preserved as a subset (clients that only need invalidation ignore - `values`). - -### 1.3 Server-side handler - -`PaxosKvRpcService` registers fire-and-forget handlers for `FBWatchSubscribe` and -`FBWatchUnsubscribe`. Subscribe resolves the group, checks leadership, and -registers the connection plus prefix in `WatchRegistry`. A non-leader sends -`FBWatchNotifyError` with the current leader hint. Apply-path notifications -are pushed with `RpcClient::send_to_handle`; disconnect cleanup removes the -connection's registrations. - -- **Leader check** — uses `local_replica.is_leader()`, the same atomic - role check the propose path uses for its leadership gate. A - non-leader node returns `not_leader_hint` per-subscribe rather than - closing the stream, so a client subscribed to multiple groups gets - hints only for the groups this node doesn't lead. -- **Leader change mid-stream** — on step-down, the old leader clears - its `WatchRegistry` (see §2.4). Dropping all `Watcher` tx senders - closes the clients' outbound streams. The client reconnects to the - new leader and re-subscribes. The safety-net polling covers the gap. -- **`not_leader_hint` source** — `PxGroup::leader_endpoint()` returns - the believed leader's endpoint (or empty if unknown). The handler - sends this as the hint, matching the `KvResponse.not_leader_hint` - pattern in the unary RPC path. - -## 2. Watch Registry + Apply-Path Trigger - -### 2.1 Why - -The notify trigger fires on the leader's **apply path** (after a -value is Paxos-chosen AND applied to the engine), not on the proposal -path. This is the etcd model: watchers are fed from the apply stream, -not the proposal path. The proposal path only fires for slots the -leader proposes itself; slots the leader learns via heartbeat catch-up -or repair never trigger a notify on the proposal path. The apply path -fires for **every** chosen slot on **every** replica, covering all -three entry points: -- `Learner::learn` — sync path: leader proposals + follower learn. -- `spawn_learn_chosen`'s spawned task — async path: deferred engine - apply. -- `apply_loop_task` — background catch-up: slots learned via heartbeat - `known_commit_slot` advance or gap-fill. - -The apply path's central function is `PxLearner::apply_entry`, which -decodes `entry.payload` via `Batch::decode` into `Batch { ops: Vec }`. -The changed keys are already extracted there, so the notify trigger -incurs no extra decode. The registry therefore lives on `PxLearner` (set -via a setter at group construction) so the trigger fires from one hook -point with no cross-struct lookup. - -### 2.2 WatchRegistry - -`lib/crowdb-paxos-kv/src/cluster/watch_registry.rs`: - -```rust -/// One watcher: a prefix + an outbound channel to push notify frames. -pub(crate) struct Watcher { - pub prefix: Vec, - pub tx: mpsc::Sender>, -} - -/// Per-group watch registry. Wired into `PxLearner` via -/// `set_watch_registry`; the learner's `apply_entry` calls `emit` -/// after each successful engine apply, gated by `has_watchers`. -pub(crate) struct WatchRegistry { - // Byte-level prefix trie (PrefixTrie): subscribe/unsubscribe/ - // remove_all take a parking_lot::RwLock write lock (rare; - // watcher setup/teardown), emit takes a read lock. emit walks - // each changed key through the trie once (O(key_len) per key), - // collecting watchers at every node whose prefix matches. - trie: PrefixTrie, - next_id: AtomicU64, - /// Atomic fast-path flag: true iff at least one watcher is - /// registered. The apply path checks this (one Acquire load) - /// before touching the trie — zero overhead when no watchers. - has_watchers: AtomicBool, - /// Counter incremented when try_send fails with Full; exposed - /// via dropped_notifies() for metrics wiring. A critical log is - /// emitted on each drop. - dropped_notifies: AtomicU64, -} - -impl WatchRegistry { - pub fn new() -> Self; - /// Register a watcher for `prefix`. Returns the `watcher_id` for - /// later removal. Sets `has_watchers = true`. - pub fn subscribe(&self, prefix: Vec, tx: mpsc::Sender<...>) -> u64; - /// Remove a specific watcher by `(prefix, watcher_id)`. Updates - /// `has_watchers` if the registry becomes empty. - pub fn unsubscribe(&self, prefix: &[u8], watcher_id: u64); - /// Remove all watchers whose `watcher_id` is in the list (stream- - /// end cleanup). Updates `has_watchers`. - pub fn remove_all(&self, watcher_ids: &[u64]); - /// Clear all watchers (leader step-down). Drops all tx senders, - /// closing client streams. Sets `has_watchers = false`. - pub fn clear(&self); - /// For a set of changed keys (from `Batch::decode`), find matching - /// prefixes and enqueue notify frames. Non-blocking: uses - /// `try_send`. - pub fn emit(&self, group_id: u64, slot: u64, changed: &[BatchOp]); - /// True if at least one watcher is registered. Atomic load — the - /// apply-path fast path. - pub fn has_watchers(&self) -> bool; - pub fn dropped_notifies(&self) -> u64; -} -``` - -- **`has_watchers` atomic fast path** — the apply-path hook checks - `registry.has_watchers()` (one `Acquire` load of an `AtomicBool`) - before decoding or matching. When no watchers are registered (the - common case in tests and non-diskdb clusters), the cost is one - predicted-not-taken branch, cheaper than a trie-emptiness scan and - the true zero-overhead gate. `has_watchers` is set to `true` on the - first `subscribe` and recomputed on `unsubscribe` / `remove_all` / - `clear`. -- **Trie-based prefix index** — `WatchRegistry` is backed by a - byte-level prefix trie (`PrefixTrie`) instead of a - `DashMap, Vec<(u64, Watcher)>>`. `emit` walks each changed - key through the trie once (`O(key_len)` per key), collecting - watchers at every node whose prefix matches, reducing the per-apply - cost from `O(watchers × keys)` to `O(key_len × keys)`. No mature - crate provided the exact "find all registered prefixes that are a - prefix of this key" API, so a hand-rolled trie was the natural - choice. -- **`emit` matching** — for each `BatchOp` in the decoded batch, walk - the key through the trie and check `op.key.starts_with(&entry.prefix)`. - Group matches by prefix, build one `WatchNotify { prefix, keys, values, slot }` - per prefix per watcher, send via `tx.try_send` (non-blocking — if the - watcher's channel is full, drop the notify; the safety-net polling - covers missed notifies). -- **`try_send` not `send().await`** — the apply path must not block on - a slow watcher. A full channel means the watcher is lagging; dropping - the notify is correct (the safety-net polling will catch up). The - drop is observable: `dropped_notifies` is incremented and a - `tracing::error!` is emitted ("critical: watch notify dropped -- - watcher channel full, client will catch up via safety-net poller"). -- **Delete ops** — `BatchOp { op: Op::Delete, key }` still carries a - key; a delete on a watched prefix notifies the watcher with the key - and an empty value (tombstone). The watcher re-reads and sees the - deletion. - -### 2.3 PxLearner integration - -`PxLearner` (`lib/crowdb-paxos-kv/src/paxos/learner.rs`) holds: - -```rust -/// Optional per-group watch registry. Set when the group is -/// constructed; None in tests that don't use watch/notify. The -/// apply-path hook in `apply_entry` checks `has_watchers()` before -/// touching the registry. RwLock> (not OnceLock) so it -/// can be re-wired when a group is rebuilt via -/// `inherit_local_state_from` — the learner is Arc::clone-shared -/// across rebuilds, but the new group has a new WatchRegistry. -watch_registry: RwLock)>>, // (group_id, registry) -``` - -- **`(group_id, registry)` tuple** — `PxLogEntry` has no `group_id` - field, and `apply_entry` takes only `(slot, &payload)`. The - `group_id` is stored alongside the registry so `emit` can populate - `WatchNotify.group_id` without threading `group_id` through every - call site. -- **Set via `set_watch_registry(group_id, Arc)`** — - called once during `PxGroup` construction (after the learner is - created) and again in `inherit_local_state_from` after constructing - the inherited replica. Tests that don't use watch/notify leave it - unset (`has_watchers()` returns false via the `None` fast path). -- **`apply_entry` hook** — after `self.engine.apply(slot, &batch).await` - succeeds, before the method returns: - -```rust -if let Some((group_id, registry)) = self.watch_registry.read().unwrap().as_ref() { - if registry.has_watchers() { - registry.emit(*group_id, slot, &batch.ops); - } -} -``` - - The `batch` is already decoded at the top of `apply_entry`, so the - keys are available with no extra decode. When `has_watchers()` is - false, the cost is one `RwLock` read + one `AtomicBool` load — zero - overhead when unused. -- **Fires on ALL apply paths** — `apply_entry` is called from `learn` - (sync), `spawn_learn_chosen` (async), and `apply_loop_task` - (catch-up). All three paths trigger the hook, covering leader - proposals, follower learn, heartbeat catch-up, and gap-fill repair: - the etcd model. -- **Followers don't emit** — followers also call `apply_entry`, but - their `WatchRegistry` is empty (cleared on step-down, or never - populated if they were never leader). `has_watchers()` returns false - → no emit. Only the leader (which holds the client streams) emits. - The registry's emptiness is the gate, so no explicit leader check is - needed in the hook. -- **Re-wire on group rebuild** — without the `RwLock>` - (vs `OnceLock`), subscribes would go to the new registry while - `emit` fired on the old (orphaned) registry, and notifies never reached - the client. `inherit_local_state_from` calls `set_watch_registry` - after constructing the inherited replica. - -### 2.4 PxGroup integration - -`PxGroup` (`lib/crowdb-paxos-kv/src/cluster/group.rs`) holds: - -```rust -pub(crate) watch_registry: Arc, -``` - -- Constructed in `PxGroup::new` as `Arc::new(WatchRegistry::new())`. - Wired into the learner via - `local_replica.learner.set_watch_registry(group_id, Arc::clone(&watch_registry))` - right after construction. -- **Cleared on step-down** — the leader step-down path is - `PxGroup::step_down`, called for `HigherTerm` / `LeaseUnrenewable` / - `Admin` reasons. `self.watch_registry.clear()` runs at the end of - `step_down` (after `become_follower`). This drops all `Watcher` tx - senders, closing the clients' outbound streams (clean reconnect). -- **Also cleared on propose-path direct step-down** — the propose path - has two direct `become_follower` calls that bypass `step_down`: - one on a higher term during prepare and one on a higher term during - accept. Both have `&self` (`PxGroup`), so `self.watch_registry.clear()` - runs before each `replica.become_follower(...)` call. This covers the - case where the leader discovers a higher term mid-proposal and steps - down without going through `step_down`. -- **Not cleared on `shutdown`** — `shutdown` drops the whole `PxGroup`, - which drops the `Arc`, which drops all watchers. - -### 2.5 Why not the proposal path - -The apply-path hook was chosen over a proposal-path hook because the -proposal path has a critical gap: it only fires for slots the leader -**proposes itself**. Three classes of chosen slots never trigger a -notify on the proposal path: - -- **Heartbeat catch-up slots** — when the leader advances - `known_commit_slot` via heartbeat and the background `apply_loop_task` - applies slots the leader accepted as a follower before winning the - election. These slots are real writes to real keys; watchers must be - notified. -- **Repair slots** — gap-fill repair calls `fan_out_chosen_notice` on - the repair path, not the propose path. A repaired slot is a real - write; watchers must be notified. -- **Foreign-value retry** — on the apply path, both the foreign value - and the retried client value fire `apply_entry` → both notify. This - is correct and automatic — no special handling needed. - -The apply-path hook covers all three with one code point. The cost is -that `apply_entry` runs on followers too (where the registry is empty), -but the `has_watchers()` atomic check makes this a single predicted- -not-taken branch, zero overhead. - -## 3. Coalescing (deferred) - -Burst writes to the same prefix (e.g. a batch of disk-status updates -via `batch_write`) generate one notify per write. A debounce window -would coalesce writes to the same prefix into one notify, reducing -watcher wakeup + re-read amplification under burst load. - -The watch/notify extension ships without coalescing. `apply_entry` -calls `registry.emit` directly (one notify per changed key per matching -prefix). Coalescing remains deferred. The safety-net poller covers any -notify-drop scenarios; the missing coalescing is a load optimization, not a -correctness gap. - -## 4. WatchNotify Client - -### 4.1 Why - -diskdb (and future clients) need a reusable client that opens the -`WatchNotify` stream to the group leader, subscribes to prefixes, -and delivers notify frames via a channel: the client-side mirror of -§1's server handler. - -### 4.2 WatchNotifyClient - -`lib/crowdb-paxos-kv-client/src/watch_notify.rs`: - -```rust -/// A live watch subscription. Dropping it unsubscribes and closes the -/// stream. -pub struct WatchSubscription { - /// Notify frames for this subscription. Receiver end of an mpsc; - /// the client reads `WatchNotify` frames here. - pub notify_rx: mpsc::Receiver, - /// Internal handle to unsubscribe on drop. - inner: WatchSubscriptionInner, -} - -impl Drop for WatchSubscription { - fn drop(&mut self) { /* send WatchUnsubscribe, or drop the stream */ } -} - -/// Client for the crowdb-paxos-kv `WatchNotify` bidi stream. -pub struct WatchNotifyClient { - kv: Arc, -} - -impl WatchNotifyClient { - pub fn from_shared(kv: Arc) -> Self; - - /// Subscribe to `(store_id, group_id, prefix)`. Opens a bidi stream - /// to the group leader (discovered via the topology cache), sends a - /// `WatchSubscribe` frame, and returns a `WatchSubscription` whose - /// `notify_rx` yields `WatchNotify` frames. - /// - /// On leader-change (stream closes), the client automatically - /// reconnects to the new leader and re-subscribes; the caller's - /// `notify_rx` stays open across the reconnect. Missed notifies - /// during the reconnect gap are caught by the caller's safety-net - /// polling. - pub async fn subscribe( - &self, - store_id: u64, - group_id: u64, - prefix: &[u8], - ) -> Result; -} -``` - -a. `subscribe` resolves the group leader endpoint via `CrowdbClient`'s - topology cache (`kv.topology.leader(store_id, group_id)`, falling - back to `kv.topology.refresh()` on cache miss — same mechanism as - `Put`/`Get` via `resolve_leader`). -b. Gets a crowdb-rpc connection from the shared `ConnectionPool` - (`kv.pool.get(&endpoint)`) — reuses the existing connection pool, - no separate channel management. -c. Opens a `KvServiceClient::new(channel).watch_notify(...)` bidi - stream to that endpoint. -d. Sends `WatchSubscribe { version: 1, group_id, prefix }`. -e. Spawns a reader task that loops on the inbound stream: on - `WatchNotify` frame, forwards to the subscriber's `notify_rx`; on - `WatchNotifyError { not_leader_hint }`, refreshes topology - (`kv.topology.set_leader(store_id, group_id, hint)`), reconnects - to the new leader, re-subscribes, and continues. On stream - error/disconnect, same reconnect logic. -f. `WatchSubscription::drop` sends `WatchUnsubscribe` (if the stream - is still open) and aborts the reader task. - -- **`store_id` required** — the topology cache is keyed by - `(store_id, group_id)`. Group 0 is `(0, 0)` (`CrowdbClient::SYSTEM_STORE` - / `SYSTEM_GROUP`). The `subscribe` signature takes `store_id` so the - client can resolve the leader endpoint. -- **One stream per subscription** — simpler than multiplexing multiple - prefixes over one stream (each subscription is independent; a diskdb - instance typically watches 3 prefixes). The schema already supports - multiple `WatchSubscribe` frames on one stream, so a future - optimization can multiplex subscriptions over a shared stream with - no schema change. -- **Reconnect backoff** — capped exponential (50 ms → 2 s). -- **Proactive topology refresh on reconnect** — the reader loop calls - `topology.refresh()` unconditionally at the top of every (re)connect - iteration, before looking up the leader endpoint, instead of only - refreshing when the cached leader was `None`. This avoids connecting - to a stale leader cached before a leader change that happened during - the disconnect gap. The reactive `not_leader_hint` path is kept as a - fallback for leader changes that happen between the refresh and the - stream open. -- **`ConnectionPool` reuse** — `pool` is `pub(crate)` on `CrowdbClient`; - `WatchNotifyClient` is in the same crate, so it accesses the pool - directly. No new connection management code. - -## 5. diskdb Notify Handler + Polling Safety Net - -### 5.1 Why - -diskdb's `KeepAlive` sync loop (`liveness/keepalive.rs`) wakes on a -fixed timer (`Trigger::TimerFn`). To trigger an immediate sync on -notify, the loop must also wake on an external signal. The `Trigger` -enum has an `Event(Notify)` variant but not a hybrid timer+event. The -notify handler (a `WatchNotifyClient` reader) must wake the keepalive -without going through the timer. - -### 5.2 Trigger extension - -`Trigger` in `app/crowdb-diskdb/src/bg_task.rs`: - -```rust -pub enum Trigger { - Timer(Duration), - TimerFn(Box Duration + Send + Sync>), - Event(Arc), - /// Wake on either a timer tick (dynamic interval from config) OR - /// an external notify signal. Used by keepalive when - /// `notify_enabled` is true: the timer is the safety-net polling - /// interval, the notify is woken by the WatchNotify handler. - TimerOrEvent { - interval_fn: Box Duration + Send + Sync>, - notify: Arc, - }, -} -``` - -`wait_trigger`: - -```rust -Trigger::TimerOrEvent { interval_fn, notify } => { - tokio::select! { - () = tokio::time::sleep(interval_fn()) => {} - () = notify.notified() => {} - } -} -``` - -- **Backward-compatible** — existing tasks keep `Timer`/`TimerFn`/ - `Event`; only keepalive uses `TimerOrEvent` when `notify_enabled`. -- **`select!` is race-safe** — both branches can fire; `select!` polls - both and picks one. No panic if both are ready. - -### 5.3 KeepAlive changes - -In `app/crowdb-diskdb/src/liveness/keepalive.rs`: - -a. Field `sync_trigger: Option>` — `Some` when - `notify_enabled`, `None` otherwise (keeps the struct usable in tests - without notify). -b. Builder method `.with_sync_trigger(notify: Arc) -> Self`. -c. In `trigger()` (the `BackgroundTask` impl): if `sync_trigger` is - `Some`, return `Trigger::TimerOrEvent { interval_fn, notify: - sync_trigger }`; else fall back to the existing `TimerFn` path. -d. Method `pub fn trigger_now(&self)` — calls - `self.sync_trigger.notify_one()` if set. Called by the notify - handler on each `WatchNotify` frame. -e. **`rpc_endpoint` already populated** — `main.rs` calls - `.with_rpc_endpoint(config.load().server.listen_addr.clone())`, - and `keepalive.rs` passes `self.rpc_endpoint` to `heartbeat_diskdb` - on every tick. The endpoint registration (item 1 of the original - scope) is already done; no change needed here. - -### 5.4 Notify handler task - -`app/crowdb-diskdb/src/liveness/notify.rs`: - -```rust -/// diskdb-side watch/notify handler. Subscribes to group-0 prefixes -/// and wakes the keepalive sync loop on notify. -pub struct NotifyHandler { - watch: WatchNotifyClient, - keepalive_trigger: Arc, - /// (store_id, group_id, prefix) triples to subscribe to. - subscriptions: Vec<(u64, u64, Vec)>, -} - -impl NotifyHandler { - pub fn new( - watch: WatchNotifyClient, - keepalive_trigger: Arc, - ) -> Self; - - /// Open subscriptions and loop on notify frames. Each frame wakes - /// the keepalive via `keepalive_trigger.notify_one()`. Runs until - /// the stop signal. - pub async fn run(self, stop: tokio::sync::Notify); -} -``` - -a. On `run`, subscribe to the group-0 prefixes diskdb cares about. - The prefixes are **text-encoded** (matching the engine's group-0 - key encoding — `HardwareClient` puts/scans text paths), centralized - as `DISKDB_WATCH_PREFIXES` in `crowdb-protocol` (`key/diskdb.rs`): - - `/hw/dg_owner/` (ownership map — global; `list_owners` scans - globally and filters to this instance) - - `/hw/dg_bind/` (binding map — global; `list_binds` scans globally) - - `/hw/disk/` (disk metadata + status — global; `observe_disks` - scans per owned disk-group, but the notify is a coarse trigger) - All three use `(G0_STORE, G0_GROUP)` = `(0, 0)`. Adding a new - keepalive-relevant group-0 key kind requires updating only the - constant in `crowdb-protocol`, not the diskdb handler. -b. For each subscription, spawn a reader loop: on `WatchNotify` frame, - call `keepalive_trigger.notify_one()` (wakes keepalive → - `KeepAlive::tick()` → re-reads the changed keys from group 0). -c. The notify is a **trigger only** — diskdb re-reads the actual values - via the normal sync path (`observe_ownership`, `observe_disks`). - This keeps the notify payload small and avoids a duplicated read - path. -d. On subscription drop/reconnect, the `WatchNotifyClient` handles - reconnect internally (§4.2); the handler just keeps reading. -e. On `stop.notified()`, drop all subscriptions (which closes the - streams) and exit. - -- **Global prefixes, not per-node** — `list_owners()` / `list_binds()` - scan the global `/hw/dg_owner/` / `/hw/dg_bind/` prefixes and filter - to `instance_id == self.instance_id`. Watching the global prefix - means the keepalive wakes on any ownership/bind change anywhere, then - re-scans and filters to this instance. An irrelevant change (another - instance's ownership) causes a cheap no-op re-scan. Per-node prefixes - would be more selective but require knowing `rack_id`/`node_id` - before the first sync. The diskdb discovers its node identity from - the ownership map, not from config, so global prefixes avoid this - bootstrapping dependency. -- **Not a `BackgroundTask`** — `NotifyHandler` is a long-lived task - with its own `run` method, not a `BgRunner` cycle task. It is spawned - directly in `main.rs` alongside the bg runner, with the same stop - signal. This avoids forcing the notify loop into the - trigger→cycle→trigger model (it is event-driven, not periodic). -- **`StopHandle::notified()` helper** — supports long-lived tasks that - don't follow the trigger→cycle model. - -### 5.5 Polling safety net - -When `notify_enabled` is true: -- The keepalive timer interval (from `sync.sync_interval_secs`) is the - safety-net polling interval. The config default rises (e.g. 60 s) - when notify is on; the operator sets it explicitly. -- The timer catches any missed notifies (reliability fallback): if a - notify is dropped (full channel) or the WatchNotify stream is - disconnected, the next timer tick still syncs. -- When `notify_enabled` is false (default), behavior is unchanged — - fixed-interval polling at `sync_interval_secs` (default 10 s), no - notify handler, no `WatchNotifyClient`. - -## 6. Client Endpoint Cache Proactive Refresh - -### 6.1 Why - -The `DiskdbClient` endpoint cache (`lib/crowdb-diskdb-client/src/client.rs`) -refreshes **on demand**: at startup, on cache miss, and on error-retry -(`refresh_endpoints` reads `read_all_diskdb_instances`). When a diskdb -instance moves (endpoint change in its service-registry record at -`/srv/diskdb/`), a client holding the old endpoint cache -entry fails one `allocate_blocks`/`free_blocks` attempt, triggers -`refresh_endpoints` (on-demand safety net), and retries. This section -extends that to **proactive** refresh: the client subscribes to the -`/srv/diskdb/` prefix via `WatchNotifyClient`; on a notify (instance -register/deregister/move), it refreshes the affected cache entries so -the next call routes to the new endpoint without a failed attempt + -retry. The on-demand refresh remains as the safety net; proactive -refresh is an optimization, not a correctness requirement. - -### 6.2 Design - -Add a `WatchNotifyClient` subscription to `DiskdbClient`: - -a. In `DiskdbClient::new` (or a new `with_watch_notify` builder), if - a `WatchNotifyClient` is provided, subscribe to - `(G0_STORE, G0_GROUP, "/srv/diskdb/")`. -b. Spawn a reader task: on `WatchNotify` frame, call - `self.refresh_endpoints()` (the existing method). The notify is a - coarse trigger — the refresh re-reads ALL diskdb instances from - group 0 and updates the cache. This is the same "trigger, not - transport" model as the diskdb notify handler (§5.4). -c. The reader task runs for the client's lifetime. On stream close - (leader change), `WatchNotifyClient` reconnects automatically - (§4.2); the reader task just keeps reading. -d. If no `WatchNotifyClient` is provided (default), the client uses - on-demand refresh only (unchanged behavior). - -- **`/srv/diskdb/` prefix** — the service-registry instance key path - is `/srv/diskdb/` (`InstanceKey::to_path()`). The global - prefix `/srv/diskdb/` covers all diskdb instances. - `ServiceRegistryClient::read_all_diskdb_instances` scans this prefix. -- **On-demand refresh stays as safety net** — if a notify is missed - (stream disconnected, channel full), the next `allocate_blocks` to a - moved instance fails once, triggers `refresh_endpoints`, and - retries. The cache is eventually consistent either way. - -## 7. Configuration - -### 7.1 diskdb config - -`app/crowdb-diskdb/src/ddb_config.rs`: - -```rust -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NotifyConfig { - /// static: enable watch/notify (default: false). When true, - /// diskdb subscribes to group-0 prefixes and the keepalive timer - /// serves as a safety-net poller at `sync.sync_interval_secs`. - pub notify_enabled: bool, -} - -impl Default for NotifyConfig { - fn default() -> Self { - Self { notify_enabled: false } - } -} -``` - -- `pub notify: NotifyConfig` on `DdbConfig`. -- `validate()`: if `notify_enabled` and `sync.sync_interval_secs == 0`, - error (same existing check; no new constraint). - -### 7.2 crowdb-diskdb-client config - -An optional `watch_notify` toggle on `DiskdbClient`'s config (or -builder): - -```rust -/// When true, the client subscribes to `/srv/diskdb/` and proactively -/// refreshes the endpoint cache on notify. Default: false (on-demand -/// refresh only). -pub watch_notify_enabled: bool, -``` - -- **Default false** — proactive refresh is an optimization; the default - ships with on-demand refresh. Operators opt in per client. - -## 8. Module Structure - -``` -lib/crowdb-paxos-kv/src/ - rpc/ - fbs/kv_client.fbs # WatchNotify messages + RPC - kv_service.rs # watch_notify handler - cluster/ - watch_registry.rs # WatchRegistry, Watcher, PrefixTrie - group.rs # watch_registry field, wire into learner, clear on step_down - group_election.rs # clear registry in step_down - group_propose.rs # clear registry before direct become_follower calls - mod.rs # export watch_registry - paxos/ - learner.rs # watch_registry RwLock + +## How this differs from an ordered-log write path In Raft's replicated state-machine model, commitment advances through a contiguous log prefix and commands are applied in log order. Batching and pipelining improve replication throughput while retaining that ordering model. See the [Raft paper, sections 5.3 and 9.3](https://raft.github.io/raft.pdf). CROWDB uses independent Paxos slots together with an engine that understands their ordering. Later blind writes can reach their own completion point while an earlier slot is still being resolved. This reduces head-of-line waiting where the operations do not require each other's results, and lets available network, WAL, and engine capacity serve useful work across many concurrent writes. -The benefit depends on the complete consensus-to-engine path. Simply replacing a consensus algorithm while retaining one serial apply queue would leave much of the opportunity unused. CROWDB designs slot agreement, per-key version selection, and parallel application together. Read-modify-write operations use additional coordination, and reads or maintenance that require a complete prefix still observe that boundary. +The benefit depends on slot agreement, per-key version selection, and parallel application working together. Read-modify-write operations use additional coordination, and reads or maintenance that require a complete prefix still wait for it. ## Stores, groups, and replicas -A store contains multiple Paxos groups. Each group has its own replica set, consensus state, WAL, and Tree state. Replicas run on different nodes; one node can host replicas for several groups, and each group can elect a different leader. Clients route by store and group before choosing the appropriate replica. +A store contains multiple Paxos groups. Each group has its own replica set and consensus state; each replica maintains a write-ahead log (WAL) and Tree state. Replicas run on different nodes; one node can host replicas for several groups, and each group can elect a different leader. Clients route by store and group before choosing the appropriate replica. ![A store contains multiple groups, each replicated across nodes with replica-local WAL, crowdb-tree, and local disk storage.](../assets/paxos-kv.svg) @@ -35,7 +37,7 @@ Each replica writes its WAL and `crowdb-tree` state to local disks. Replication The WAL records the information needed to recover replication and accepted work. Tree materializes selected operations without adding another mutation redo WAL. Its durable snapshot coverage tells the KV layer which applied state is recoverable from the engine; the retained consensus history supplies the required suffix. This avoids duplicate engine logging while coordinating persistence and recovery through explicit progress. The [Tree recovery design](design-crowdb-tree.md#recovery-is-part-of-the-engine-contract) explains that shared contract. -This matters when many metadata writes are active at once. The quorum WAL makes accepted work durable, parallel apply updates the ordered index, and a later Tree snapshot consolidates durable state. The KV layer can advance these stages at different rates while retaining the history needed between them. It avoids inserting a second engine redo log into every mutation's path and can schedule checkpoint work around the same progress information used for recovery. +The stages can advance at different rates: replica WALs establish durable quorum acceptance, parallel apply updates the ordered index, and a Tree checkpoint persists the materialized state. Paxos-KV retains the history needed between those stages. A write can therefore be recoverable before its values appear in a durable Tree root. Group 0 relies on this recovery path for service registration, ownership, membership, and deployment decisions. Those decisions must survive the process that made them. A restarted monitor or service can recover them from durable metadata instead of reconstructing them from whatever happens to be running. diff --git a/doc/design/design-crowdb-rpc.md b/doc/design/design-crowdb-rpc.md index 33c94de4a..94c2345a8 100644 --- a/doc/design/design-crowdb-rpc.md +++ b/doc/design/design-crowdb-rpc.md @@ -3,7 +3,7 @@ # CROWDB - Design: Storage RPC -`crowdb-rpc` puts RDMA at the center of CROWDB's high-performance internal data path, with TCP as the fallback for environments without a usable RDMA connection. The transport connects storage services and native consumers through registered buffers and asynchronous completion, carrying memory ownership across the boundaries where ordinary message handling often adds staging and copies. +`crowdb-rpc` carries internal requests and payloads between storage services and native applications. Its remote direct memory access (RDMA) backend uses registered buffers and asynchronous completion to reduce staging and copying. TCP supports ordinary networks and can serve as a connection-setup fallback when deployment policy allows it. A C++ engine supplies connections, framing, workers, and both transport backends; `crowdb-rpc-ffi` exposes them to Rust. Services use the same request and payload model on either path. Deployment policy enables RDMA and controls whether setup may fall back to TCP. @@ -13,7 +13,9 @@ The RDMA backend manages registered memory, posted receives, work requests, and The value appears across the complete I/O path. A DiskIO read can fill a registered buffer for transport; a native consumer can retain the received allocation while processing its bytes. A direct native client also removes the Access Server payload relay. CROWDB can therefore optimize both the network transfer and the handoffs on either side of it. The [DiskIO design](design-crowdb-diskio.md#a-short-path-from-request-to-device) and [native SDK](design-crowdb-sdk-native.md) describe those paths. -## Control and payload take different paths + + +## Control fields and payload buffers stay separate A control message describes the operation. Payload views describe the bytes it transfers and retain their owners. The transport can keep those allocations alive until asynchronous send or receive work completes, without forcing every caller to serialize the payload into a new message buffer first. @@ -29,9 +31,9 @@ Consider a burst of writes that arrives faster than the disks can complete it. R When downstream capacity is occupied, those limits stop or delay further production. A slow disk or consumer therefore becomes visible as backpressure, rather than an invitation to accumulate more payloads. Separate budgets account for request concurrency, bytes, and processing resources; a count of completed RPCs alone cannot describe the system's available capacity. -The receive pool, send queue, storage pipeline, and consumer each release capacity when their part of the work actually finishes. They can share a payload allocation without pretending that these are the same event. That is especially valuable for RDMA, where registered memory is a finite resource: reusing an allocation saves staging work only if its lifetime remains visible to every stage that retains it. +The receive pool, send queue, storage pipeline, and consumer each release capacity at a different point. Sharing a payload allocation preserves its owner across those stages; it does not make their completion events interchangeable. Registered memory stays occupied while any stage still retains it. -Ownership also survives cancellation. If a write times out after device submission, the caller can stop waiting while the device still references the buffer. The retained owner keeps that allocation valid until the outstanding work finishes. On reads, application-held results remain charged after transport completion. This connects RDMA efficiency with controlled memory use under sustained load. The [native SDK](design-crowdb-sdk-native.md#completed-reads-still-consume-capacity) carries that accounting through to the consumer. +Ownership also survives cancellation. If a write times out after device submission, the caller can stop waiting while the device still references the buffer. The retained owner keeps that allocation valid until the outstanding work finishes. On reads, the [native SDK](design-crowdb-sdk-native.md#completed-reads-still-consume-capacity) continues to account for application-held results after transport completion. ## TCP provides a practical fallback @@ -39,14 +41,12 @@ TCP keeps the same storage services usable on ordinary hosts and networks. Its b Performance-sensitive deployments can disable fallback to require RDMA. This makes the chosen transport an explicit operational policy: a workload that requires RDMA fails setup visibly instead of silently running on another path. -Fallback happens before payload submission. Once an RDMA operation has an uncertain outcome, the transport reports that failure; it does not splice TCP bytes onto a partial result or transparently replay the operation. The service's request semantics determine whether a subsequent retry is safe. +Fallback is a connection-setup decision. It does not switch an accepted RDMA operation to TCP after a transfer error. An uncertain outcome is reported to the caller, and the transport neither appends a TCP suffix to partial data nor transparently replays the operation. The service's request semantics determine whether a subsequent retry is safe. ## Why own an RPC engine? A storage RPC engine can make registered memory and asynchronous I/O part of its basic interface. Control fields remain small, payload buffers keep their owners, and completion releases resources at the right stage. RDMA receives the design attention required by the fast path, while TCP preserves deployment flexibility through the same service boundary. -Owning that boundary lets the Chunk client, DiskIO, and native SDK agree on memory lifetime. It removes avoidable staging at supported handoffs and lets resource admission span several libraries. The transport becomes part of the storage pipeline, with an interface shaped around moving bytes efficiently between devices and consumers. - Internal RPC operates within the cluster's trust boundary; externally authenticated applications use the access interfaces. Public protocol compatibility remains with those interfaces. See the [transport interface](../../lib/crowdb-rpc/include/crowdb-rpc/transport.h), [buffer interface](../../lib/crowdb-rpc/include/crowdb-rpc/buffer.h), and [FFI](../../lib/crowdb-rpc/ffi). The [native SDK](design-crowdb-sdk-native.md) shows how retained transport allocations become application-visible results. diff --git a/doc/design/design-crowdb-sdk-native.md b/doc/design/design-crowdb-sdk-native.md index ed1d3c363..fb0e5225b 100644 --- a/doc/design/design-crowdb-sdk-native.md +++ b/doc/design/design-crowdb-sdk-native.md @@ -3,7 +3,7 @@ # CROWDB - Design: Native SDK -`crowdb-sdk-native` is CROWDB's RDMA-oriented access path for high-performance applications. When compute and storage belong to the same trusted system, an application can embed the Dataset and S3 access libraries and communicate directly with storage services. Bulk data travels between disk-serving nodes and the application without passing through an additional Access Gateway. +`crowdb-sdk-native` lets trusted applications embed the Dataset and S3 access libraries and communicate directly with storage services. Bulk data travels between DiskIO nodes and the application over RDMA or TCP, without an Access Server relay. The SDK exposes retained host buffers; direct GPU-memory delivery is future work. The objective is efficiency across the whole system: fewer network hops, less protocol work, fewer staging buffers, and less CPU spent forwarding bytes. RDMA is most valuable when the architecture also removes the intermediate process that would otherwise receive, wrap, and retransmit the payload. @@ -13,7 +13,7 @@ An HTTP access path sends storage-node payloads through Access Server before the The same Dataset Snapshot, Split, and field-selection rules still apply. S3 operations still resolve bucket and key and use the owning library's publication rules. Metadata services retain their authority; the application gains a shorter data path rather than a second interpretation of stored data. -For training and analytical systems, this changes where resources are spent. Gateway CPU, response framing, forwarding buffers, and a second payload transfer no longer have to scale with every byte consumed by an internal application. Compute and storage can be provisioned around their actual work. +The native SDK currently covers Dataset and S3; Iceberg's native upload interface is a separate integration described in the [Iceberg design](design-crowdb-access-iceberg.md). ## One runtime for many operations @@ -25,7 +25,7 @@ Operations carry deadlines and cancellation. Shutdown closes admission and drain ## RDMA needs buffer ownership all the way to the consumer -Reads return `BufferLease` values: immutable views that retain the backing allocation. A consumer can process a view, pass it to another asynchronous operation, or retain it until a host-to-device copy finishes. Resource accounting follows the allocation until its final owner releases it. +Read streams return `BufferLease` values: immutable views that retain the backing allocation. A consumer can process a view, pass it to another asynchronous operation, or retain it until a host-to-device copy finishes. Resource accounting follows the allocation until its final owner releases it. A lease contains a piece of the result; only the stream's verified end establishes complete-read success. Eligible registered receive buffers reach the caller without a second SDK payload copy. Producers can also fill native allocations and submit immutable views. Buffer ownership survives asynchronous I/O and application processing, so a fast transport does not immediately lose its benefit to another copy at the SDK boundary. @@ -35,13 +35,13 @@ Copies still have reasons to exist: ordinary input may need framing, erasure cod A training worker can finish a read and keep its buffers while decoding samples or waiting for a host-to-device transfer. The SDK continues to charge those allocations against the caller-held capacity budget until their final owner releases them. Completing the RPC or closing the stream does not erase that memory use. -For example, a worker may prefetch the next batch while another task decodes the current one. Both batches occupy capacity until their owners finish with them. Passing a shared view to another task preserves the same backing allocation and charge; it does not create another copy merely to transfer ownership. Once the last owner releases that allocation, the capacity becomes available to a later read. +For example, prefetching a batch while decoding the previous one keeps both batches charged. Sharing or slicing a view retains the same allocation and charges its capacity once, even if a small slice keeps a much larger buffer alive. Releasing the final owner makes that capacity available to another read. When held capacity is exhausted, later reads wait for release or return a deadline or cancellation result. Producer allocations and active requests have their own limits, alongside the transport's registered-memory and queue budgets. Sharing a runtime makes Dataset and S3 handles share the appropriate SDK accounts, so adding handles does not silently multiply the application's allowance. Request concurrency controls how much work is active; held capacity controls how much delivered data remains in the application. A workload can have few active requests and still hold substantial memory from earlier reads, so those limits serve different purposes. Producer memory has a similar lifetime concern: shared small-write machinery can retain submitted owners for open-strip repair after an individual PUT completes. -This lets the consumer set an SDK-managed working-set limit that remains meaningful after bytes leave the network. The application can retain a view through asynchronous processing while backpressure controls how much more data arrives. Copies into application-owned arrays, decoded objects, or Python `bytes` need the application's own memory policy; the SDK accounts for the allocations whose lifetime it manages. +The held-buffer limit covers SDK-managed allocations after delivery. Copies into application-owned arrays, decoded objects, or Python `bytes` need the application's own memory policy. Retained-buffer accounting is not a limit on the process's total memory. ## Transport policy follows the deployment @@ -51,14 +51,14 @@ An operation that has already transferred data cannot be treated as a fresh atte ## Extend the direct path toward GPU memory -GPU workloads make the final destination of each byte especially important. The next integration step is direct delivery into GPU memory through suitable RDMA and GDS / GPUDirect support. The system-level goal is to remove gateway forwarding and host staging from the storage-to-GPU path, rather than accelerating one link while leaving the same copies around it. +A future GPU integration could extend the direct storage path into device memory and remove host staging. RDMA transport and retained host buffers provide useful foundations, but direct GPU delivery needs its own registration, ownership, and completion support. -CROWDB has taken the first step with direct RDMA access and retained host buffers. The current SDK exposes host memory, including buffers that an application can retain until an asynchronous host-to-device copy completes. GPU pointers and direct GPU delivery require the subsequent GPU integration. The existing ownership and completion model provides the place to carry that lifetime through the full path. +Today, applications must retain the host lease until an asynchronous host-to-device copy finishes. Neither GPU pointers nor GPUDirect Storage are part of the current SDK interface. ## The deployment boundary is explicit The embedding application needs access to internal management and storage services. It has the same internal trust as an access service; this API does not perform S3 SigV4 authentication. HTTP remains the appropriate boundary for portable clients and externally authenticated access. -Native integration brings native libraries and, for RDMA, host device and registration requirements. That is a useful fit for an internal compute/storage cluster whose operators control both ends of the connection. HTTP remains available for portable clients and external access. +Native integration brings native libraries and, for RDMA, host device and registration requirements. It fits an internal compute/storage cluster whose operators control both ends of the connection. The [native SDK README](../../lib/crowdb-sdk-native/README.md) contains Rust examples and runtime options. The [native Python README](../../lib/crowdb-sdk-native-python/README.md) describes packaging and its explicit copy boundary. The separate [Dataset HTTP SDK](../../sdk/README.md) provides the portable path. diff --git a/doc/design/design-crowdb-tree.md b/doc/design/design-crowdb-tree.md index 750b55249..fe914ddbb 100644 --- a/doc/design/design-crowdb-tree.md +++ b/doc/design/design-crowdb-tree.md @@ -3,7 +3,7 @@ # CROWDB - Design: Tree Storage Engine -`crowdb-tree` brings concurrent skiplist tables, a Bw-tree architecture with a mapping table, and slot-aware version selection into one ordered KV engine. Its flush design follows the tree itself: fork work down independent subtrees, join results from leaves to root, and persist the same results when a snapshot is requested. The page backend can use local disks or shared Chunk storage. +`crowdb-tree` combines concurrent skiplist write buffers, immutable pages behind a mapping table, and slot-aware version selection in one ordered key-value engine. Flush distributes work across independent subtrees and joins their results toward the root. A checkpoint persists those results to local disks or shared Chunk storage. These choices connect foreground concurrency, ordered reads, and background persistence. Paxos-KV uses the engine on each replica's local disks; Chunk-KV uses its Chunk backend to keep range state independent of the serving node. CROWDB can optimize the same engine for both storage models. @@ -11,15 +11,15 @@ These choices connect foreground concurrency, ordered reads, and background pers ![Concurrent skiplist tables use CAS; a Bw-tree mapping table resolves immutable pages, with ordinary and persistent flush modes and local-disk or Chunk backends.](../assets/tree.svg) -The write buffer is an ordered concurrent skiplist. CAS publishes new nodes and immutable version sets, while epoch protection keeps objects alive for readers. Multiple writers can update independent keys without putting every insertion behind a shared MemTable mutex. The skiplist's ordering also gives flush jobs range cursors directly, avoiding a fresh sort of the entire input batch. +Each write buffer, or MemTable, is an ordered concurrent skiplist. Compare-and-swap (CAS) publishes new nodes and immutable version sets, while epoch protection delays reclamation until readers are finished. Multiple writers can update independent keys without putting every insertion behind a shared MemTable mutex. The skiplist also gives flush jobs ordered range cursors without sorting the entire input batch again. -The page tree uses mapping indirection: a logical page ID resolves to an immutable page version through a versioned mapping table. A page combines base data with ordered overlays. Readers retain a consistent mapping and source view while writers prepare newer state. A split can change routing and share retained base ranges without immediately copying every value into a new page buffer. +The page tree uses mapping indirection: a logical page ID resolves to an immutable page version through a versioned mapping table. This follows the indirection principle described in the [Bw-tree paper](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/bw-tree-icde2013-final.pdf); CROWDB combines it with its own MemTables, ordered overlays, and checkpoint flow. A page's overlays hold updates above its base data. Readers retain a consistent mapping and source view while writers prepare newer state, and a split can share base ranges without immediately copying every value. This keeps several expensive operations off the ordinary update path. Inserting a value need not rewrite a disk page; publishing a new page need not invalidate a reader's view; moving entries into the tree can transfer or share their existing allocation owners. Consolidation combines overlays and adjusts page boundaries when the structure needs it. ## Logical versions survive out-of-order application -Each key retains slot information and the versions needed by active read and recovery boundaries. An older update that finishes after a newer one cannot overwrite it merely by arriving later. Point reads and ordered scans select the highest eligible version, including tombstones, from the retained sources and pages. +Each key retains slot information and the versions needed by active read and recovery boundaries. A slot records the operation's logical order, supplied by the caller. An older update that finishes after a newer one cannot overwrite it merely by arriving later. Point reads and ordered scans select the highest eligible version, including tombstones that represent deletion, from the retained sources and pages. That makes concurrency useful all the way through the replicated write path: consensus establishes logical order, while independent application work can finish in a different physical order. Paxos-KV builds user-facing conditional CAS operations from versioned reads and coordinated chosen writes; Tree's own CAS operations publish its concurrent in-memory state. @@ -35,15 +35,15 @@ This is where the tree structure becomes a scheduling advantage. Parallelism fol ## Flush and flush with snapshot share the same work -Ordinary flush publishes the resulting pages and mapping in memory. Flush with snapshot selects persistent output for that same fork/join round. Final pages are encoded and written as their dependencies become ready; parents join those results, changed mapping segments are written once their entries are complete, and the durable root commits after its referenced output is safe. +Ordinary flush publishes the resulting pages and mapping in memory. Flush with snapshot selects persistent output for that same fork/join round; this is the checkpoint path. Final pages are encoded and written as their dependencies become ready, parents join those results, and changed mapping segments are persisted. The durable root commits only after the required output and storage barriers complete. The design shares consolidation, split/merge decisions, and final page layout between memory maintenance and persistence. Snapshot output therefore reuses the work already needed to organize the tree. Unchanged durable pages and mapping segments keep their existing backing, while a persistent round also includes dirty state left by earlier ordinary flushes. -Incremental persistence makes the amount of changed state matter more than total tree size. Memory-only maintenance can batch several update rounds before durable output, and the caller gets an explicit committed frontier for its WAL or journal retention policy. Resource budgets and backpressure keep that batching within the Tree's capacity. +Incremental persistence lets unchanged backing be reused across checkpoints. Ordinary flush and page writeback can reduce memory pressure, but neither advances the durable replay point on its own. Only a committed checkpoint reports the covered operation prefix that the caller can use in its journal retention policy. ## Recovery is part of the engine contract -Tree uses its caller's recoverable journal and does not create a separate mutation redo WAL. Paxos-KV supplies consensus-selected operations from its replica-local WAL; Chunk-KV supplies ordered mutations from its partition journal. The engine materializes those operations, preserving their version information and reporting the durable progress covered by a committed snapshot. +Tree relies on its caller's recoverable journal rather than creating a separate mutation redo write-ahead log (WAL). Paxos-KV supplies consensus-selected operations from its replica-local WAL; Chunk-KV supplies ordered mutations from its partition journal. The engine materializes those operations, preserves their version information, and reports the durable progress covered by a committed checkpoint. An in-memory Tree update alone is not crash-durable. This removes a second recording of the same mutation inside the engine. The larger benefit is that application, persistence, and recovery share explicit progress boundaries. If a snapshot covers a prefix and later writes exist only in memory, recovery replays the required external journal suffix. A high observed slot or a completed page write cannot substitute for a committed root and its coverage. @@ -53,10 +53,6 @@ Now suppose a snapshot starts persisting state through 120. Its page writes can The caller uses that durable coverage together with its replication and retention obligations to decide which journal history can be released. Tree manages the lifetime of its own pages and retained views. Each part supplies the evidence needed by the other, so memory maintenance, checkpointing, and log reclamation can be optimized together without confusing their completion points. -That coordination gives persistence room to batch useful work. Foreground writes can become recoverable through the external journal while Tree consolidates updates and persists changed pages in larger rounds. The engine avoids maintaining a second mutation log, and the caller can balance checkpoint work against the history retained for replay. A faster write path and a bounded recovery path become connected design decisions. - -The same contract works with two different journals and two physical backends. It lets CROWDB reuse recovery-aware indexing while keeping consensus decisions and partition ordering with their respective KV services. - ## One engine, local disks or shared Chunks The local backend writes Tree pages and durable roots to local disks. Paxos-KV combines that local engine with replicated consensus and a local WAL on each replica. diff --git a/doc/design/design-crowdb.md b/doc/design/design-crowdb.md index 754c1f786..f14b39a50 100644 --- a/doc/design/design-crowdb.md +++ b/doc/design/design-crowdb.md @@ -5,7 +5,7 @@ CROWDB brings Iceberg tables, AI datasets, and S3 objects onto one distributed storage core. Each has its own identity, publication rules, and read model. They share the machinery that places bytes, protects them, tracks ownership, and recovers after failure. -The ambition is to make the storage system understand enough of the workload to improve the whole path. A table commit, a training batch, and a multipart object upload need different coordination. Beneath those differences lie the same expensive operations: moving buffers, allocating storage, recording durable state, and rebuilding data. CROWDB develops these pieces together. +A table commit, a training batch, and a multipart object upload need different coordination. Beneath those differences lie the same expensive operations: moving buffers, allocating storage, recording durable state, and rebuilding data. CROWDB develops these pieces together so improvements to the storage path can benefit all three workloads. ## The workloads that shape the design @@ -15,7 +15,7 @@ A training application works with selected samples and fields from a fixed datas An object application wants a familiar bucket and key, streaming upload, range reads, and predictable behavior when a request fails. S3 compatibility lets existing tools use those operations without learning a new storage API. -These workloads combine large sequential transfers with many small metadata operations. Optimizing only the disk bandwidth leaves catalog contention, buffer copies, or recovery work as the next bottleneck. CROWDB's architecture addresses those costs at their respective owners. +These workloads combine large sequential transfers with many small metadata operations. Disk bandwidth alone cannot remove catalog contention, buffer copies, or recovery work. CROWDB addresses each cost in the component that controls it. ## Shared infrastructure, independent data semantics @@ -23,8 +23,6 @@ These workloads combine large sequential transfers with many small metadata oper Iceberg controls when table metadata becomes the current table state. Dataset controls which immutable Snapshot a read selects. S3 controls when an object becomes visible under its bucket and key. Sharing storage does not turn these into aliases for the same namespace. That distinction gives each access model room to improve without forcing its rules onto the others. -The benefits compound below the interfaces. A better small-write pipeline can serve several workloads. A repair mechanism protects their payloads. Buffer ownership can remain explicit from the storage transport through a native consumer. Improvements reach across the system because the components agree on the boundaries that matter. - ## The design advantages **Native Iceberg removes a storage-system relay.** Table files live directly in Chunk storage, with no separate S3 storage backend. Native uploads return signed URIs carrying the Chunk location, so a manifest-selected data file can be read without another Iceberg file-metadata lookup. Catalog publication and file lifecycle are designed together, reducing the repeated metadata work that appears when independent systems are stacked. See [Iceberg](design-crowdb-access-iceberg.md). @@ -39,11 +37,11 @@ The benefits compound below the interfaces. A better small-write pipeline can se **Replicated metadata writes can complete across concurrent slots.** Paxos-KV lets each slot reach its own durable quorum, while per-key slot ordering keeps out-of-order Tree application correct. Independent blind writes can make progress before an earlier gap is resolved, reducing head-of-line waiting through the consensus and engine path. See [Paxos-KV](design-crowdb-paxos-kv.md). -**The engine's concurrency follows its data structure.** Tree combines CAS-based skiplist tables, slot-aware versions, and Bw-tree mapping indirection. Its flush design forks independent subtree work and joins final page results toward the root; flush with snapshot persists those same results incrementally. Local-disk and Chunk backends let Paxos-KV and Chunk-KV share that engine investment. See [Tree](design-crowdb-tree.md). +**The engine's concurrency follows its data structure.** Tree combines concurrent skiplist write buffers, slot-aware versions, and a mapping table that separates page identity from page contents. Flush distributes work across independent subtrees and joins their results toward the root; a checkpoint persists the resulting state. Local-disk and Chunk backends let Paxos-KV and Chunk-KV share that engine investment. See [Tree](design-crowdb-tree.md). **The journal and Tree form one recovery design.** Tree materializes work recorded by its caller without adding a separate mutation redo WAL. Slot-aware application, durable snapshot coverage, and retained journal history connect in one recovery path. This avoids duplicate mutation logging inside the engine and gives the caller the durable progress needed to decide what to replay and what history can be reclaimed. See [Tree recovery](design-crowdb-tree.md#recovery-is-part-of-the-engine-contract). -**RDMA reaches the application without an access gateway relay.** Trusted applications embed the native access libraries and communicate with storage nodes directly. RDMA is the core high-performance transport, with TCP providing a fallback. Direct access removes an intermediate payload hop and its protocol and staging work. Retained host buffers are available today; direct GPU-memory delivery through further RDMA / GDS integration extends the same system-level goal. See the [native SDK](design-crowdb-sdk-native.md). +**Native access removes a payload relay.** Trusted Dataset and S3 applications embed the access libraries and communicate with storage nodes directly, using RDMA or TCP according to deployment policy. This removes an intermediate network hop and its protocol and staging work. The SDK delivers retained host buffers today; direct GPU-memory delivery is future work. See the [native SDK](design-crowdb-sdk-native.md). These advantages reinforce one another. Ordered metadata selects the work, shared chunks aggregate it, direct I/O moves the bytes, and retained ownership carries them to the consumer. Resource budgets cover queued work, in-flight I/O, and buffers still held by an application, so slower consumers can apply backpressure through the path. Durable system metadata coordinates placement and recovery across it. @@ -51,8 +49,6 @@ These advantages reinforce one another. Ordered metadata selects the work, share CROWDB is a storage platform. Query execution, model training, and application transactions spanning unrelated data models belong to their respective engines and applications. Its Dataset fields hold application data; CROWDB does not prescribe tensor formats or execute a training graph. -Access Server and OCI deployment support the storage interfaces, with deployments tested on physical Linux hosts. The [container deployment design](design-crowdb-deployment.md) uses an OCI image without requiring a particular runtime or orchestrator. Host networking and host block devices keep the I/O path direct; container limits control CPU and memory. Dataset services and HTTP/native clients are described in the [Dataset SDK](../../sdk/README.md). Use the [repository README](../../README.md#run-crowdb) for deployment entry points and compatibility boundaries. - -The architecture is especially useful when lakehouse tables, training data, and object storage share the same infrastructure. Improvements to small-write aggregation, range movement, transport, or physical recovery can benefit all three, while each retains its own data model. +Access Server hosts the network interfaces. The [container deployment design](design-crowdb-deployment.md) packages a Linux amd64 node as an OCI image, with Docker and containerd launch paths. Host networking and host block devices keep the I/O path direct; container limits control CPU and memory. The [Dataset HTTP SDK](../../sdk/README.md) and [native SDK](../../lib/crowdb-sdk-native/README.md) provide separate client distributions. Use the [repository README](../../README.md#run-crowdb) for deployment entry points and compatibility boundaries. Continue with the [architecture](design-crowdb-architecture.md) or use the [component index](README.md) to explore a specific design. diff --git a/e2e-test/README.md b/e2e-test/README.md index b8adcc548..ef56e6879 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -3,32 +3,11 @@ # E2E tests -The root-level `e2e-test/` is the unified E2E project. Its Rust workspace package -is `e2e-test`; server domains use separate test modules within this crate. -The Python runner owns container lifecycle and scheduling. Browser and -SDK clients with independent language dependencies live under `clients//` -and consume the same fixture endpoints and ownership contract. They do not -introduce separate server launchers. The runner's domain registry defines -domain selection; each run retains its results and logs. - -Test clients run on the host against inspected node-container IPs and fixed -ports, with three node containers for a three-node cluster. Fresh one- through eight-node profiles exercise management, membership, -async operations and multi-store routing. Host bridge IP reachability is -required; remote Docker and Linux-VM routing are not implicitly supported. -The CLI terminal-writer fault also requires host `iptables-nft`, its shared -libraries and xtables extensions. A short-lived release-image helper receives -NET_ADMIN only inside each verified owned CLI bridge namespace and drops the -accepted DiskIO listener for forty seconds. Monitor and logging keep running; -the helper never enters the host network namespace. Tool fingerprints, rule -verification and failure-safe rule cleanup remain in the case artifacts. -Containers supply server subsets or clusters from the formal release image; -they are not test execution environments. Domain profiles configure that same -server artifact rather than introducing separate test-server builds. - -This component owns real-server acceptance. The KV -profile uses three packaged `crowdb-paxos-kv-server` processes with block tree storage, -block-device WAL and the existing `e2e` election preset. It consumes the shared -`crowdb-node` OCI runtime image; it never starts a host server substitute. +The root-level `e2e-test/` is the unified E2E project. Its Rust workspace package is `e2e-test`; server domains use separate test modules within this crate. The Python runner owns container lifecycle and scheduling. Browser and SDK clients with independent language dependencies live under `clients//` and consume the same fixture endpoints and ownership contract. They do not introduce separate server launchers. The runner's domain registry defines domain selection; each run retains its results and logs. + +Test clients run on the host against inspected node-container IPs and fixed ports, with three node containers for a three-node cluster. Fresh one- through eight-node profiles exercise management, membership, async operations and multi-store routing. Host bridge IP reachability is required; remote Docker and Linux-VM routing are not implicitly supported. The CLI terminal-writer fault also requires host `iptables-nft`, its shared libraries and xtables extensions. A short-lived release-image helper receives NET_ADMIN only inside each verified owned CLI bridge namespace and drops the accepted DiskIO listener for forty seconds. Monitor and logging keep running; the helper never enters the host network namespace. Tool fingerprints, rule verification and failure-safe rule cleanup remain in the case artifacts. Containers supply server subsets or clusters from the formal release image; they are not test execution environments. Domain profiles configure that same server artifact rather than introducing separate test-server builds. + +This component owns real-server acceptance. The KV profile uses three packaged `crowdb-paxos-kv-server` processes with block tree storage, block-device WAL and the existing `e2e` election preset. It consumes the shared `crowdb-node` OCI runtime image; it never starts a host server substitute. ## Run @@ -65,240 +44,71 @@ pixi run test-container-e2e --list --scope kv.group0 pixi run test-container-e2e --layer kv --data-root /path/to/e2e-data ``` -Build/import the shared OCI image first. An absent image, missing Docker daemon, -unsupported daemon, changed release server sources or inadequate budget fails the -required test. There is no successful missing-prerequisite skip. -`--layer all` selects registered KV, Diskdb, Diskio, Chunkdb, chunk-client, -chunk-stream, Chunk-KV, S3, Iceberg, Dataset, Console and deployment cases. -The host profile retains its owned bind-mounted data/log/crash root and an exact -image/source record; fixed ports require an exclusive Linux host. It is an -explicit deployment acceptance and is excluded from ordinary bridge domains. -The Docker host and native containerd host profiles qualify two explicit policies. -A one-voter production diskless node exercises the common CLI's accepted inputs, -authenticated admission, current metadata authority and retained recreation. This -metadata-only configuration does not claim whole-cluster storage readiness. -A Local one-voter node exercises all three access protocols and retained data, -identity and credentials through ordinary container recreation, with zero node -failure tolerance. Each node has a 1-CPU quota and 2048 MiB memory limit. -Protected production storage requires independent voters; the separate three-host -acceptance verifies that contract. The Docker host profile also retains file/raw -fsync, Offline, Remove/Rejoin, exclusive-descriptor and physical-tail assertions. -These profiles require an explicitly privileged disposable Linux host for owned -fixture setup; the node container receives no blanket privilege. - -Containerd profiles require the installed `container/oci-image` runtime Pixi -environment and a verified OCI archive/receipt. `containerd-host` starts its own -rootful native-snapshotter daemon and rejects a Docker client, socket, daemon or -container-provisioned host. `containerd-cluster` separately labels its three -Docker-provisioned privileged Linux hosts; it proves adapter/discovery/quorum -behavior and does not substitute for the native-host gate. Both retain private -data and complete logs. Clients execute outside the formal node image. -The Local host fixture selects an 8-GiB sparse managed file with a 1-GiB -host reserve. Sparse files are confined to the explicit Local policy. `CROWDB_E2E_CLI_BUNDLE` supplies a prebuilt host CLI and -`lib/` directory when the host cannot build clients; Docker media probes can -likewise use `CROWDB_E2E_DOMAIN_BUNDLE` with `domain-test` and `lib/`. - -DiskIO process-exit faults retain their network -namespace, so the original 15-second stream/reader/repair bounds still apply. - -Chunkdb lifecycle, placement and recovery use five owned containers: three KV -voters, one Diskdb and one production-mode Chunkdb. Actual inspected KV replicas -are registered through the public metadata API. Allocation transport acceptance -needs only KV and Diskdb. The placement profile seeds four distinct failure -domains and checks metadata geometry. The storage profile owns eleven containers: -three KV, Diskdb, Chunkdb and six Diskio failure domains, with eighteen disks. -Only process-exit scenarios add six release-image sleep processes to retain -DiskIO network namespaces: killing a storage service closes its listener without -removing its IP. These scenarios reserve an additional 0.3 CPU / 96 MiB and -remove namespace owners after their service containers. Ordinary storage cases -use the eleven service containers directly. -The multislot profile owns three Chunkdb service instances, three KV voters and -Diskdb. Storage cases verify public conversion, partial bytes, exact parity, -expired foreground claim takeover, four missing-shard repair, degraded node-return -repair and exact relocation journal copy/publication/source-free ordering. All eleven -Chunkdb cases and their source retirement pass. Important internal task, journal, -fence and placement proofs remain in the Chunkdb component with pinned memory -metadata and their original assertions. -S3 joint-node faults use three actual KV voters and three independent -Diskdb/Chunkdb/DiskIO failure domains, plus the catalog and Access service. -Fourteen server containers consume the same image; twelve namespace owners -retain fixed endpoints while those storage processes exit. At the default -CPU sharing this profile reserves 1.5 CPU / 7872 MiB, including its host -client. The three outage cases preserve old/new exact bytes; node two also -checks persisted degraded EC placement, full restart and the 25-second repair -bound. A separate protected restart case checks stable endpoints and range -reads. Local state/secret/owned-directory checks remain component UT, while -Web health and Group-0 preview have their Console owner. - -Image and client revisions are recorded independently. Client/documentation/test-registration -commits reuse an image after verifying the release server inputs still match; -production source, dependency, runtime configuration and build changes require -a rebuild. A missing/unverifiable image source label fails this verification. - -`--concurrency` is a cap. Each server container defaults to one CPU of relative -weight (`--cpu-shares 1024`), with no CPU quota or cpuset. Idle CPU can be used -by any container; weights govern contention. `--node-cpus` scales this weight, -not a hard ceiling. Node services inherit `TOKIO_WORKER_THREADS=1` to avoid -automatically creating a host-sized pool for every process and exhausting the -existing PID/memory limits. RPC and blocking workers retain their normal -configuration. Named-flow admission reserves a shared 1.5-CPU scheduling -allowance per fixture (servers plus host client), rather than adding node weights. -This bounds concurrent client work, not guaranteed server throughput. Memory -limits and per-profile memory sums remain hard admission bounds. Actual Docker -host capacity further limits the declared budget. Two slots run isolated fixtures -at once; one slot does not claim concurrent isolation. -Recovery/isolation acceptance uses at most two slots; independent named flows -use the effective cap. CI sets `--require-isolation` and -fails if two slots are unavailable, so the required isolation check cannot skip. Host/hardware and containerd profiles are explicit gates. +Build/import the shared OCI image first. An absent image, missing Docker daemon, unsupported daemon, changed release server sources or inadequate budget fails the required test. There is no successful missing-prerequisite skip. `--layer all` selects registered KV, Diskdb, Diskio, Chunkdb, chunk-client, chunk-stream, Chunk-KV, S3, Iceberg, Dataset, Console and deployment cases. The host profile retains its owned bind-mounted data/log/crash root and an exact image/source record; fixed ports require an exclusive Linux host. It is an explicit deployment acceptance and is excluded from ordinary bridge domains. The Docker host and native containerd host profiles qualify two explicit policies. A one-voter production diskless node exercises the common CLI's accepted inputs, authenticated admission, current metadata authority and retained recreation. This metadata-only configuration does not claim whole-cluster storage readiness. A Local one-voter node exercises all three access protocols and retained data, identity and credentials through ordinary container recreation, with zero node failure tolerance. Each node has a 1-CPU quota and 2048 MiB memory limit. Protected production storage requires independent voters; the separate three-host acceptance verifies that contract. The Docker host profile also retains file/raw fsync, Offline, Remove/Rejoin, exclusive-descriptor and physical-tail assertions. These profiles require an explicitly privileged disposable Linux host for owned fixture setup; the node container receives no blanket privilege. + +Containerd profiles require the installed `container/oci-image` runtime Pixi environment and a verified OCI archive/receipt. `containerd-host` starts its own rootful native-snapshotter daemon and rejects a Docker client, socket, daemon or container-provisioned host. `containerd-cluster` separately labels its three Docker-provisioned privileged Linux hosts; it proves adapter/discovery/quorum behavior and does not substitute for the native-host gate. Both retain private data and complete logs. Clients execute outside the formal node image. The Local host fixture selects an 8-GiB sparse managed file with a 1-GiB host reserve. Sparse files are confined to the explicit Local policy. `CROWDB_E2E_CLI_BUNDLE` supplies a prebuilt host CLI and `lib/` directory when the host cannot build clients; Docker media probes can likewise use `CROWDB_E2E_DOMAIN_BUNDLE` with `domain-test` and `lib/`. + +DiskIO process-exit faults retain their network namespace, so the original 15-second stream/reader/repair bounds still apply. + +Chunkdb lifecycle, placement and recovery use five owned containers: three KV voters, one Diskdb and one production-mode Chunkdb. Actual inspected KV replicas are registered through the public metadata API. Allocation transport acceptance needs only KV and Diskdb. The placement profile seeds four distinct failure domains and checks metadata geometry. The storage profile owns eleven containers: three KV, Diskdb, Chunkdb and six Diskio failure domains, with eighteen disks. Only process-exit scenarios add six release-image sleep processes to retain DiskIO network namespaces: killing a storage service closes its listener without removing its IP. These scenarios reserve an additional 0.3 CPU / 96 MiB and remove namespace owners after their service containers. Ordinary storage cases use the eleven service containers directly. The multislot profile owns three Chunkdb service instances, three KV voters and Diskdb. Storage cases verify public conversion, partial bytes, exact parity, expired foreground claim takeover, four missing-shard repair, degraded node-return repair and exact relocation journal copy/publication/source-free ordering. All eleven Chunkdb cases and their source retirement pass. Important internal task, journal, fence and placement proofs remain in the Chunkdb component with pinned memory metadata and their original assertions. S3 joint-node faults use three actual KV voters and three independent Diskdb/Chunkdb/DiskIO failure domains, plus the catalog and Access service. Fourteen server containers consume the same image; twelve namespace owners retain fixed endpoints while those storage processes exit. At the default CPU sharing this profile reserves 1.5 CPU / 7872 MiB, including its host client. The three outage cases preserve old/new exact bytes; node two also checks persisted degraded EC placement, full restart and the 25-second repair bound. A separate protected restart case checks stable endpoints and range reads. Local state/secret/owned-directory checks remain component UT, while Web health and Group-0 preview have their Console owner. + +Image and client revisions are recorded independently. Client/documentation/test-registration commits reuse an image after verifying the release server inputs still match; production source, dependency, runtime configuration and build changes require a rebuild. A missing/unverifiable image source label fails this verification. + +`--concurrency` is a cap. Each server container defaults to one CPU of relative weight (`--cpu-shares 1024`), with no CPU quota or cpuset. Idle CPU can be used by any container; weights govern contention. `--node-cpus` scales this weight, not a hard ceiling. Node services inherit `TOKIO_WORKER_THREADS=1` to avoid automatically creating a host-sized pool for every process and exhausting the existing PID/memory limits. RPC and blocking workers retain their normal configuration. Named-flow admission reserves a shared 1.5-CPU scheduling allowance per fixture (servers plus host client), rather than adding node weights. This bounds concurrent client work, not guaranteed server throughput. Memory limits and per-profile memory sums remain hard admission bounds. Actual Docker host capacity further limits the declared budget. Two slots run isolated fixtures at once; one slot does not claim concurrent isolation. Recovery/isolation acceptance uses at most two slots; independent named flows use the effective cap. CI sets `--require-isolation` and fails if two slots are unavailable, so the required isolation check cannot skip. Host/hardware and containerd profiles are explicit gates. ## Lifecycle and endpoints -- Each fixture owns a UUID-labeled bridge network, independent host data bind - mounts and its server containers. No global prune or shared - implicit cluster is used. Cleanup verifies ownership labels and exact names. -- Network-local management is fixed at `7000`; store RPC uses `7001..7011`. - Every fixture reuses the same names, store/replica IDs and port pool inside its - own network. New fixtures do not call the legacy offset/probe/flock allocator. -- No service port is host-published. Management and RPC clients use inspected - node IPs directly, so concurrent fixtures reuse all fixed internal ports. -- Host RPC clients consume the owned endpoint list. Peer topology rewrites - wildcard listen addresses to inspected bridge IPs (the C++ RPC transport - expects numeric addresses) while retaining actual - returned store ports. RPC ports are never host-published. -- KV scopes separately test user-data groups and Group-0 initialization, - discovery, membership CAS and metadata authority through public APIs. Bare - profiles start no initial store; Group-0 ownership tests explicitly initialize - through management. SSH, monitor and console coverage belongs to deployment and Console - profiles. -- SIGKILL/restart retains the same mounted data, network identities and internal - endpoints. Inspection rejects a changed peer IP rather than masking stale - topology by choosing another endpoint. - Monitor automatic recovery is absent in this layer profile. -- Host fault requests use a bounded fixture-owned control directory. Only - stop/start/kill/pause/unpause of declared nodes are accepted; ownership is - rechecked before each action. Responses and individual action timings survive - teardown. Client phase time already includes these actions. -- Client compilation uses locked Cargo dependencies; the staged executable and - Pixi shared-library dependencies execute on the host with per-run environment. - Normal component Rust gates leave the container-only test ignored; this - runner explicitly executes selected cases and missing fixture setup fails. -- Interruption or partial startup gathers diagnostics and tears down owned - allocations. Operations have bounded deadlines; SIGINT/SIGTERM abort runner - coordination and cleanup follows in-flight operations. SIGKILL cannot execute - cleanup; retained resource labels identify orphaned fixture ownership. -- Node mounts and their logs/data/crash files remain on the host after teardown. - `mounts.json` records paths. Startup, client phases, collection and teardown - produce timing reports; a required client invocation must execute one case. - `endpoints.json` records node/store/group/data layout. `results.json` collects - per-case client, start/restart/collect/stop and complete wall durations, sorts - the slowest flows, and preserves failure status after cleanup errors. - -Listener acceptance checks require -`SO_REUSEADDR` before bind, stable four-digit endpoints and one live owner; -no default `SO_REUSEPORT` or conflict-driven port hopping. +- Each fixture owns a UUID-labeled bridge network, independent host data bind mounts and its server containers. No global prune or shared implicit cluster is used. Cleanup verifies ownership labels and exact names. +- Network-local management is fixed at `7000`; store RPC uses `7001..7011`. Every fixture reuses the same names, store/replica IDs and port pool inside its own network. New fixtures do not call the legacy offset/probe/flock allocator. +- No service port is host-published. Management and RPC clients use inspected node IPs directly, so concurrent fixtures reuse all fixed internal ports. +- Host RPC clients consume the owned endpoint list. Peer topology rewrites wildcard listen addresses to inspected bridge IPs (the C++ RPC transport expects numeric addresses) while retaining actual returned store ports. RPC ports are never host-published. +- KV scopes separately test user-data groups and Group-0 initialization, discovery, membership CAS and metadata authority through public APIs. Bare profiles start no initial store; Group-0 ownership tests explicitly initialize through management. SSH, monitor and console coverage belongs to deployment and Console profiles. +- SIGKILL/restart retains the same mounted data, network identities and internal endpoints. Inspection rejects a changed peer IP rather than masking stale topology by choosing another endpoint. Monitor automatic recovery is absent in this layer profile. +- Host fault requests use a bounded fixture-owned control directory. Only stop/start/kill/pause/unpause of declared nodes are accepted; ownership is rechecked before each action. Responses and individual action timings survive teardown. Client phase time already includes these actions. +- Client compilation uses locked Cargo dependencies; the staged executable and Pixi shared-library dependencies execute on the host with per-run environment. Normal component Rust gates leave the container-only test ignored; this runner explicitly executes selected cases and missing fixture setup fails. +- Interruption or partial startup gathers diagnostics and tears down owned allocations. Operations have bounded deadlines; SIGINT/SIGTERM abort runner coordination and cleanup follows in-flight operations. SIGKILL cannot execute cleanup; retained resource labels identify orphaned fixture ownership. +- Node mounts and their logs/data/crash files remain on the host after teardown. `mounts.json` records paths. Startup, client phases, collection and teardown produce timing reports; a required client invocation must execute one case. `endpoints.json` records node/store/group/data layout. `results.json` collects per-case client, start/restart/collect/stop and complete wall durations, sorts the slowest flows, and preserves failure status after cleanup errors. + +Listener acceptance checks require `SO_REUSEADDR` before bind, stable four-digit endpoints and one live owner; no default `SO_REUSEPORT` or conflict-driven port hopping. ## Assertions and artifacts -The migrated `e2e_three_node_cluster_kv_put_batch_delete` retains every original -assertion: two remotes per voter, Put success, exact Get bytes, Batch success, -Delete success and empty/not-found after deletion. It additionally checks the -surviving batch value, a per-fixture value under the same key, and a tombstone -across crash/restart. In the concurrent case both clusters first write different -values, then one crashes/restarts and is removed; the other must still read its -own exact bytes and report its three-voter topology. - -Artifacts under `.crowdb-runtime/artifacts/container-e2e/run-*` retain image -config ID, labels/source revision, registry digests when available, client -results, topologies and Docker logs/resource state. Diagnostics redact credential -fields. CI consumes the once-built OCI artifact in independent KV jobs in both -regular CI, preview and release workflows. Successful KV digest validation is required by -publication; workflow dispatch/publication remain separate operator actions. - -CI exports reports and redacted text diagnostics with -`pixi run python e2e-test/publish.py --source .crowdb-runtime/artifacts/container-e2e --output target/container-e2e-public`. -The output must be a fresh directory outside the original runs. Mounted node -data, generated server configuration, client build trees and private credentials -remain in the original host directories; they are excluded from upload. JSON -measurements retain their original status/count/timing values. Local raw logs -and crash data remain available for investigation. +The migrated `e2e_three_node_cluster_kv_put_batch_delete` retains every original assertion: two remotes per voter, Put success, exact Get bytes, Batch success, Delete success and empty/not-found after deletion. It additionally checks the surviving batch value, a per-fixture value under the same key, and a tombstone across crash/restart. In the concurrent case both clusters first write different values, then one crashes/restarts and is removed; the other must still read its own exact bytes and report its three-voter topology. + +Artifacts under `.crowdb-runtime/artifacts/container-e2e/run-*` retain image config ID, labels/source revision, registry digests when available, client results, topologies and Docker logs/resource state. Diagnostics redact credential fields. CI consumes the once-built OCI artifact in independent KV jobs in both regular CI, preview and release workflows. Successful KV digest validation is required by publication; workflow dispatch/publication remain separate operator actions. + +CI exports reports and redacted text diagnostics with `pixi run python e2e-test/publish.py --source .crowdb-runtime/artifacts/container-e2e --output target/container-e2e-public`. The output must be a fresh directory outside the original runs. Mounted node data, generated server configuration, client build trees and private credentials remain in the original host directories; they are excluded from upload. JSON measurements retain their original status/count/timing values. Local raw logs and crash data remain available for investigation. ## Component ownership -- Public server flows have one owner in the domain registry. Retired host - launchers and overlapping acceptance files are removed. -- Component tests retain internal invariants, planning, protocol peers and - injected hardware/storage behavior. They do not duplicate the full server flow. -- Complete-run results and failures are retained in each run's artifacts. - A successful selected-domain run certifies only that domain. - -The cluster flows also cover exact follower hints, nontrivial topology identities, -dynamic group creation/removal with isolated same-key values, and a five-node -3→5→3 membership transition with acknowledged data and subsequent writes. The -former KV server `cluster_e2e_test.rs` is fully migrated. Selected-case resource -budgets reserve a shared 1.5 CPU allowance and the summed hard memory limits -per fixture; the default KV budget permits two three-node isolation fixtures. - -Snapshot join uses two voting nodes and a third bare store from the same release -image. Direct local reads prove imported old values before remote wiring, then -verify WAL-tail catch-up on the non-voting member. Node profiles can omit initial -groups; the management API owns join/bootstrap. +- Public server flows have one owner in the domain registry. Retired host launchers and overlapping acceptance files are removed. +- Component tests retain internal invariants, planning, protocol peers and injected hardware/storage behavior. They do not duplicate the full server flow. +- Complete-run results and failures are retained in each run's artifacts. A successful selected-domain run certifies only that domain. + +The cluster flows also cover exact follower hints, nontrivial topology identities, dynamic group creation/removal with isolated same-key values, and a five-node 3→5→3 membership transition with acknowledged data and subsequent writes. The former KV server `cluster_e2e_test.rs` is fully migrated. Selected-case resource budgets reserve a shared 1.5 CPU allowance and the summed hard memory limits per fixture; the default KV budget permits two three-node isolation fixtures. + +Snapshot join uses two voting nodes and a third bare store from the same release image. Direct local reads prove imported old values before remote wiring, then verify WAL-tail catch-up on the non-voting member. Node profiles can omit initial groups; the management API owns join/bootstrap. ## Storage and normal monitor batches -- Diskdb client allocation, ownership-routing and exhaustion/reuse now have one - owner under `tests/diskdb/`; the old client process fixtures are removed after - acceptance. Six deterministic client UT remain in their component. Diskdb - application keepalive/fence/journal/scanner proofs retain component ownership - using pinned in-memory metadata, with no host KV launcher. Public recovery - independently SIGKILLs Diskdb or Diskdb plus KV and checks exact durable - records/revisions, persist-only free, compaction and capacity reuse. -- `deployment.selection` consumes mounted normal profiles in the same immutable - release image: complete seven services, KV-only and KV+Diskdb. Check exact - selected readiness, acknowledged host-client writes, abrupt node loss and - recreation with preserved data. Unknown/missing-dependency selections fail - before partial startup. Single-mode browser/runtime failures now belong to - the deployment layer; manual UI/SSH admission has its explicit bridge profile. +- Diskdb client allocation, ownership-routing and exhaustion/reuse now have one owner under `tests/diskdb/`; the old client process fixtures are removed after acceptance. Six deterministic client UT remain in their component. Diskdb application keepalive/fence/journal/scanner proofs retain component ownership using pinned in-memory metadata, with no host KV launcher. Public recovery independently SIGKILLs Diskdb or Diskdb plus KV and checks exact durable records/revisions, persist-only free, compaction and capacity reuse. +- `deployment.selection` consumes mounted normal profiles in the same immutable release image: complete seven services, KV-only and KV+Diskdb. Check exact selected readiness, acknowledged host-client writes, abrupt node loss and recreation with preserved data. Unknown/missing-dependency selections fail before partial startup. Single-mode browser/runtime failures now belong to the deployment layer; manual UI/SSH admission has its explicit bridge profile. ## Diskio profiles -- `diskio.durability` uses an owned 16 MiB regular data file with buffered file I/O; - `diskio.direct` uses the release default O_DIRECT setting on the same geometry. - Both fsync acknowledged aligned and unaligned bytes, SIGKILL/restart at the - same IP/7200, and verify exact persisted contents plus an unaligned range read. - Regular files do not claim real block-device or io_uring capability coverage. -- `diskio.io.mem` and `diskio.io.null` own KV plus Diskio, with the original four - disks including a high-half disk ID. They retain every boundary size, overwrite, - zone/offset and 100 concurrent cycles, with exact bytes or NullDisk patterns. -- `diskio.discovery` owns heartbeat, stable connection pools across refreshes, - malformed-owner rejection and new-disk reconciliation through Group 0. -- `diskio.errors` owns missing disk/zone and missing write creation time; - `diskio.faults` owns deterministic I/O errors using the normal release option. -- `diskio.semantic` owns payload/view framing, ordering, independent strip and - priority admission, endpoint replacement and error classification. Its KV and - three sequential Diskio generations use four owned containers. Replacement - containers are created dormant and started by the bounded fixture control - only after the prior generation exits. Startup waits for its RPC listener; - tests then verify disk and registry readiness without hiding transport errors. -- Address validation, authoritative-owner topology and synthetic native-route - lease recovery remain component tests. The shared host Diskio launcher remains - until chunk client/stream callers migrate, and is not used by this domain. +- `diskio.durability` uses an owned 16 MiB regular data file with buffered file I/O; `diskio.direct` uses the release default O_DIRECT setting on the same geometry. Both fsync acknowledged aligned and unaligned bytes, SIGKILL/restart at the same IP/7200, and verify exact persisted contents plus an unaligned range read. Regular files do not claim real block-device or io_uring capability coverage. +- `diskio.io.mem` and `diskio.io.null` own KV plus Diskio, with the original four disks including a high-half disk ID. They retain every boundary size, overwrite, zone/offset and 100 concurrent cycles, with exact bytes or NullDisk patterns. +- `diskio.discovery` owns heartbeat, stable connection pools across refreshes, malformed-owner rejection and new-disk reconciliation through Group 0. +- `diskio.errors` owns missing disk/zone and missing write creation time; `diskio.faults` owns deterministic I/O errors using the normal release option. +- `diskio.semantic` owns payload/view framing, ordering, independent strip and priority admission, endpoint replacement and error classification. Its KV and three sequential Diskio generations use four owned containers. Replacement containers are created dormant and started by the bounded fixture control only after the prior generation exits. Startup waits for its RPC listener; tests then verify disk and registry readiness without hiding transport errors. +- Address validation, authoritative-owner topology and synthetic native-route lease recovery remain component tests. The shared host Diskio launcher remains until chunk client/stream callers migrate, and is not used by this domain. ## Access S3 -- `s3.protocol` owns 23 existing boto3, AWS CLI and rclone contracts. Each case - uses two Access frontends, one protected Chunk-KV server, six persistent - Diskio failure domains, Chunkdb, Diskdb and three KV voters. HTTP/health - ports are 7500/7501; every fixture owns its bridge and bind-mounted data. -- `s3.recovery` separately restarts each service class and checks exact bytes, - overwrites, deletions, copies and unfinished multipart sessions. -- `s3.responses` tests lost committed responses. `s3.credentials` runs the - exact release credential CLI on the host. `s3.benchmark` records functional - direct Chunk and HTTP request-path samples, including Access CPU/RSS. -- `s3.sdk` owns the [Java, JavaScript and Go client projects](clients/S3-SDK.md), - their original locks and injected-failure cleanup checks. -- Native buffer, payload-copy and read metrics remain asserted. Parsers, - signatures, repository state machines, publication races and fake transport - wire proofs remain component tests. These wire stubs do not start a CROWDB - release server or replace production storage acceptance. +- `s3.protocol` owns 23 existing boto3, AWS CLI and rclone contracts. Each case uses two Access frontends, one protected Chunk-KV server, six persistent Diskio failure domains, Chunkdb, Diskdb and three KV voters. HTTP/health ports are 7500/7501; every fixture owns its bridge and bind-mounted data. +- `s3.recovery` separately restarts each service class and checks exact bytes, overwrites, deletions, copies and unfinished multipart sessions. +- `s3.responses` tests lost committed responses. `s3.credentials` runs the exact release credential CLI on the host. `s3.benchmark` records functional direct Chunk and HTTP request-path samples, including Access CPU/RSS. +- `s3.sdk` owns the [Java, JavaScript and Go client projects](clients/S3-SDK.md), their original locks and injected-failure cleanup checks. +- Native buffer, payload-copy and read metrics remain asserted. Parsers, signatures, repository state machines, publication races and fake transport wire proofs remain component tests. These wire stubs do not start a CROWDB release server or replace production storage acceptance. ```sh pixi run test-container-e2e --layer s3 --concurrency 2 --cpus 16 --memory-mib 16384 @@ -307,65 +117,30 @@ pixi run test-container-e2e --layer s3 --scope s3.recovery --concurrency 1 --cpu ## Console browser flows -`console.native.binding` uses three admitted release nodes and real DiskDB -services to validate binding selection, owner acquisition and explicit repair -of missing authoritative records. Its deployment selects a normal one-second -heartbeat and retains the three-second acquisition assertion. +`console.native.binding` uses three admitted release nodes and real DiskDB services to validate binding selection, owner acquisition and explicit repair of missing authoritative records. Its deployment selects a normal one-second heartbeat and retains the three-second acquisition assertion. The other native scopes reuse three formal nodes with all 18 typed services: -- `console.native.s3`: multipart bytes, actual Chunk acknowledgement, physical - locations and stale continuation rejection. -- `console.native.recovery`: every typed service restart, the live authority - leader, old/new container PIDs, exact inventory and durable object bytes. -- `console.native.browser.s3`: host-browser multipart upload, HEAD, preview, - physical placement/navigation and deletion. -- `console.native.load`: production 32-MiB Chunk-KV load, heartbeat/lease bounds, - retained-byte scale and exact data after owner restart. +- `console.native.s3`: multipart bytes, actual Chunk acknowledgement, physical locations and stale continuation rejection. +- `console.native.recovery`: every typed service restart, the live authority leader, old/new container PIDs, exact inventory and durable object bytes. +- `console.native.browser.s3`: host-browser multipart upload, HEAD, preview, physical placement/navigation and deletion. +- `console.native.load`: production 32-MiB Chunk-KV load, heartbeat/lease bounds, retained-byte scale and exact data after owner restart. -Disk capacities are declared per fixture and recorded with endpoints; mounted -sparse files and public deployment use the same values. No image rebuild or -server test mode is needed for a capacity profile. +Disk capacities are declared per fixture and recorded with endpoints; mounted sparse files and public deployment use the same values. No image rebuild or server test mode is needed for a capacity profile. ```sh pixi run test-container-e2e --scope console.native.binding --cpus 4 --memory-mib 8192 --node-cpus 1 pixi run test-container-e2e --scope console.native --cpus 8 --memory-mib 16384 --node-cpus 1 --concurrency 2 ``` -- `console.browser.shell` runs shell navigation and embedding checks with the - pinned host browser against an ordinary release Web container. Each case has - its own network, data mount and browser workspace; server ports stay fixed. - The Web/KV node reserves at least 2 CPU plus the host browser (2.5 CPU / - 2560 MiB total). A 4 CPU budget runs one case; 6 CPU allows two. - The proxy page must be ready before the embedding navigation is checked. -- `console.browser.registry` covers the empty shell, datacenter rack menu, - ownership-probe scope, persisted Access-only listener plan and fixed-slot - warning, plus duplicate rack/node rejection. It uses public HTTP setup and fixed Access ports 9000/9001/9002/9003; - dependencies remain unprovisioned and the Access service stays waiting. -- `console.browser.kv` covers real KV pagination, binary representation and - automatic scans. `console.browser.canvas` covers fit, pan, collapsed children - and navigation state. These owned Web nodes deploy one ordinary KV server - at fixed 7000/7001 using durable defaults. -- `console.browser.logical` covers Store/Group/Replica creation, duplicate Group - rejection and deletion. `console.browser.node` covers physical/logical jumps - and remote-proxy removal without deleting the logical member. Each owns three - manual release containers, admitted through SSH and bootstrapped through the - public API. At least 1 CPU / 2048 MiB per node plus its host browser reserves - 3.5 CPU / 6656 MiB per case. KV uses fixed 10000/10100 node ports. -- Host shell/registry/KV/canvas clients run in batches bounded by the same case slots. - All batch containers start before clients run concurrently; all clients finish - before any batch network is removed. Other selected server cases finish first. - This prevents Docker bridge churn during browser mutations without retries - or relaxed UI deadlines. Failed clients and partial startup still clean up. -- Backend-unavailable checks use a closed container listener at port 9091 so - Chromium does not need to authorize a private-IP page accessing loopback. -- `console.managed` runs both full-stack managed browser flows. Its mounted Web - template sets readonly false, matching the editable single-container default; - an explicit readonly template remains available for the release browser case. - Both use the same image; physical capabilities remain restricted. The host - control channel pauses/resumes only the current owned KV service PID. -- These accepted cases are included once in `console.operations`; managed - browser and native UI owners are included in the complete Console domain. +- `console.browser.shell` runs shell navigation and embedding checks with the pinned host browser against an ordinary release Web container. Each case has its own network, data mount and browser workspace; server ports stay fixed. The Web/KV node reserves at least 2 CPU plus the host browser (2.5 CPU / 2560 MiB total). A 4 CPU budget runs one case; 6 CPU allows two. The proxy page must be ready before the embedding navigation is checked. +- `console.browser.registry` covers the empty shell, datacenter rack menu, ownership-probe scope, persisted Access-only listener plan and fixed-slot warning, plus duplicate rack/node rejection. It uses public HTTP setup and fixed Access ports 9000/9001/9002/9003; dependencies remain unprovisioned and the Access service stays waiting. +- `console.browser.kv` covers real KV pagination, binary representation and automatic scans. `console.browser.canvas` covers fit, pan, collapsed children and navigation state. These owned Web nodes deploy one ordinary KV server at fixed 7000/7001 using durable defaults. +- `console.browser.logical` covers Store/Group/Replica creation, duplicate Group rejection and deletion. `console.browser.node` covers physical/logical jumps and remote-proxy removal without deleting the logical member. Each owns three manual release containers, admitted through SSH and bootstrapped through the public API. At least 1 CPU / 2048 MiB per node plus its host browser reserves 3.5 CPU / 6656 MiB per case. KV uses fixed 10000/10100 node ports. +- Host shell/registry/KV/canvas clients run in batches bounded by the same case slots. All batch containers start before clients run concurrently; all clients finish before any batch network is removed. Other selected server cases finish first. This prevents Docker bridge churn during browser mutations without retries or relaxed UI deadlines. Failed clients and partial startup still clean up. +- Backend-unavailable checks use a closed container listener at port 9091 so Chromium does not need to authorize a private-IP page accessing loopback. +- `console.managed` runs both full-stack managed browser flows. Its mounted Web template sets readonly false, matching the editable single-container default; an explicit readonly template remains available for the release browser case. Both use the same image; physical capabilities remain restricted. The host control channel pauses/resumes only the current owned KV service PID. +- These accepted cases are included once in `console.operations`; managed browser and native UI owners are included in the complete Console domain. ```sh pixi run test-container-e2e --scope console.browser.shell --concurrency 2 @@ -375,90 +150,35 @@ pixi run test-container-e2e --scope console.browser --concurrency 2 --cpus 8 --m ## Dataset browser workspace -`console.native.browser.dataset` owns five isolated release-backed browser flows: -`browse`, `import`, `split_revisions`, `recovery`, and `capabilities`, under -`console::browser_dataset`. The `80/81/82-dataset-*` specs cover the combined -Sample/Schema matrix, Snapshot ancestry, multipart local import, independent Split -revisions, frozen inheritance, publication-response loss, and readonly/prefix modes. -Each case uses real Access/storage services. Recovery may sever a real reverse-proxy -connection after the upstream commits; it never substitutes a service response. +`console.native.browser.dataset` owns five isolated release-backed browser flows: `browse`, `import`, `split_revisions`, `recovery`, and `capabilities`, under `console::browser_dataset`. The `80/81/82-dataset-*` specs cover the combined Sample/Schema matrix, Snapshot ancestry, multipart local import, independent Split revisions, frozen inheritance, publication-response loss, and readonly/prefix modes. Each case uses real Access/storage services. Recovery may sever a real reverse-proxy connection after the upstream commits; it never substitutes a service response. ## Console WAL recovery -- `console.operations` includes the public Web WAL recovery flows. Single-node - recovery uses the normal standalone Web and its packaged KV child; - `console.web.wal` uses 3/5/6 separate KV containers and one writable Web. -- Rack/node, Group-0 and store/group creation use the public Web APIs. Clients - inspect Accepted records in the mounted WAL and require unchanged max slots - after stopping and restarting every voter, then verify exact values/deletions. -- Multi-node profiles select the supported file-WAL backend from the same - release image; standalone recovery retains the normal child defaults. - KV storage remains durable. Internal ports are fixed; batch lifecycle - validates all voter labels before mutation and leaves Web running. +- `console.operations` includes the public Web WAL recovery flows. Single-node recovery uses the normal standalone Web and its packaged KV child; `console.web.wal` uses 3/5/6 separate KV containers and one writable Web. +- Rack/node, Group-0 and store/group creation use the public Web APIs. Clients inspect Accepted records in the mounted WAL and require unchanged max slots after stopping and restarting every voter, then verify exact values/deletions. +- Multi-node profiles select the supported file-WAL backend from the same release image; standalone recovery retains the normal child defaults. KV storage remains durable. Internal ports are fixed; batch lifecycle validates all voter labels before mutation and leaves Web running. ## Deployment flows -- The canonical deployment layer has 33 named owners. It covers service - selection, hardware/KV/logical bootstrap, interruption and persistent recovery, - fourteen supervised child crash/hang cases, invalid inputs, official host SDKs, - protocol layouts, runtime permissions, readonly release browser behavior, - terminal restart exhaustion, durable identity rejection, lifecycle log/secret - checks, anonymous-volume policy, Monitor death, occupied protocol listeners - and disabled single-node split planning. -- Complete service fixtures reserve 2 CPUs/2048 MiB for the node plus a host - client. Occupied-listener checks reserve an additional 0.5 CPU/512 MiB release - process sharing only the owned node's network namespace; the other protocol - and health listener remain unoccupied. Independent scenarios keep private - networks, credentials and mounts. -- `container/single-node-container/tests/container-e2e.sh` delegates to this - layer. It owns no server launch, client, published port or assertion. - Build/import the formal artifact before invoking the canonical runner. -- Browser clients use the pinned `clients/console-ui` package and the installed - system browser. Each fixture prepares its own host workspace. Signals pass - through label/PID/executable guards. Default anonymous-volume policy is an - explicit exception to data bind mounts; logs/crash files are copied to host - artifacts before its owned volume is removed. -- Domain, client and fixture timings remain in each run's `results.json`. - Manual admission, native host and ecosystem profiles require explicit - selection; the checked Pixi/CI map defines automatic coverage. - -`bridge-admission` runs manual release nodes on an owned bridge, using inspected -IPs and fixed Web/SSH listeners without host port publishing. Its five-node peak -reserves 3.5 CPU / 6144 MiB at 0.5 CPU/node, including the host browser. Four -browser reports must each pass exactly once with zero skips. Browser traces, -node data/log/crash files and credentials stay under the private per-run data -root; only redacted client output and identity/results are uploaded. Both -partial startup and active-browser interruption remove all owned runtime -resources and terminate the host browser children while retaining diagnostics. +- The canonical deployment layer has 33 named owners. It covers service selection, hardware/KV/logical bootstrap, interruption and persistent recovery, fourteen supervised child crash/hang cases, invalid inputs, official host SDKs, protocol layouts, runtime permissions, readonly release browser behavior, terminal restart exhaustion, durable identity rejection, lifecycle log/secret checks, anonymous-volume policy, Monitor death, occupied protocol listeners and disabled single-node split planning. +- Complete service fixtures reserve 2 CPUs/2048 MiB for the node plus a host client. Occupied-listener checks reserve an additional 0.5 CPU/512 MiB release process sharing only the owned node's network namespace; the other protocol and health listener remain unoccupied. Independent scenarios keep private networks, credentials and mounts. +- `container/single-node-container/tests/container-e2e.sh` delegates to this layer. It owns no server launch, client, published port or assertion. Build/import the formal artifact before invoking the canonical runner. +- Browser clients use the pinned `clients/console-ui` package and the installed system browser. Each fixture prepares its own host workspace. Signals pass through label/PID/executable guards. Default anonymous-volume policy is an explicit exception to data bind mounts; logs/crash files are copied to host artifacts before its owned volume is removed. +- Domain, client and fixture timings remain in each run's `results.json`. Manual admission, native host and ecosystem profiles require explicit selection; the checked Pixi/CI map defines automatic coverage. + +`bridge-admission` runs manual release nodes on an owned bridge, using inspected IPs and fixed Web/SSH listeners without host port publishing. Its five-node peak reserves 3.5 CPU / 6144 MiB at 0.5 CPU/node, including the host browser. Four browser reports must each pass exactly once with zero skips. Browser traces, node data/log/crash files and credentials stay under the private per-run data root; only redacted client output and identity/results are uploaded. Both partial startup and active-browser interruption remove all owned runtime resources and terminate the host browser children while retaining diagnostics. ## Manual external engines -- `pixi run -e iceberg-ecosystem test-iceberg-container-ecosystem` delegates to - root `--profile ecosystem`; `CROWDB_ECOSYSTEM_PROFILE=python` selects the focused - Python/DuckDB profile. Neither belongs to default CI or ordinary Iceberg counts. -- [Pinned matrix and prerequisites](clients/ecosystem/README.md) describe the - host SDK/JVM clients, optional Trino engine, fixed bridge endpoints, retained - private data and public redacted reports. Both profiles recreate the formal - server against its original mounted data and assert unchanged credentials. +- `pixi run -e iceberg-ecosystem test-iceberg-container-ecosystem` delegates to root `--profile ecosystem`; `CROWDB_ECOSYSTEM_PROFILE=python` selects the focused Python/DuckDB profile. Neither belongs to default CI or ordinary Iceberg counts. +- [Pinned matrix and prerequisites](clients/ecosystem/README.md) describe the host SDK/JVM clients, optional Trino engine, fixed bridge endpoints, retained private data and public redacted reports. Both profiles recreate the formal server against its original mounted data and assert unchanged credentials. ## Console browser ownership -Browser flows live under `clients/console-ui/e2e/flows/`. The root runner owns -normal release Web nodes and the actual service cluster; clients never launch -host servers or allocate host ports. `test-console` and `test-console-ui` retain -backend and UI component tests without Docker. The npm `test:e2e` entry delegates -to the same required container domain: +Browser flows live under `clients/console-ui/e2e/flows/`. The root runner owns normal release Web nodes and the actual service cluster; clients never launch host servers or allocate host ports. `test-console` and `test-console-ui` retain backend and UI component tests without Docker. The npm `test:e2e` entry delegates to the same required container domain: ```sh pixi run test-container-e2e --scope console.operations --concurrency 2 --node-cpus 1 --cpus 4 --memory-mib 16384 ``` -Native full-service fixtures reserve 512 PIDs/threads per node and 2048 MiB, -with a private network and fixed service ports. The six-node owner combines -three storage and three diskless nodes; it reserves 6.5 CPU / 12800 MiB. -Its normal DiskDB deployment requests a 3-second keepalive cadence. CLI replay -also requests `--diskdb-heartbeat-interval-secs 3` through normal cluster creation; -the setting persists across restart. CLI defaults retain the production 10-second -cadence. New disk -initialization and actual SDK data checks remain bounded, and all selected -services must deploy without user retry. +Native full-service fixtures reserve 512 PIDs/threads per node and 2048 MiB, with a private network and fixed service ports. The six-node owner combines three storage and three diskless nodes; it reserves 6.5 CPU / 12800 MiB. Its normal DiskDB deployment requests a 3-second keepalive cadence. CLI replay also requests `--diskdb-heartbeat-interval-secs 3` through normal cluster creation; the setting persists across restart. CLI defaults retain the production 10-second cadence. New disk initialization and actual SDK data checks remain bounded, and all selected services must deploy without user retry. diff --git a/e2e-test/clients/S3-SDK.md b/e2e-test/clients/S3-SDK.md index 31781060e..c5c2d87ce 100644 --- a/e2e-test/clients/S3-SDK.md +++ b/e2e-test/clients/S3-SDK.md @@ -3,18 +3,12 @@ # Optional S3 language SDK verification -- Run from the repository root through Pixi. Language clients live in separate - `s3-java`, `s3-js` and `s3-go` projects, with their existing dependency locks. -- The unified runner starts a fresh protected, persistent release-container - stack for each language; clients execute on the host. Fixed service ports - belong to independent bridge networks, so fixture concurrency is supported. -- The `s3.sdk` scope also runs as part of the S3 container gate. Every recipe - includes normal operations and injected MPU-failure cleanup. +- Run from the repository root through Pixi. Language clients live in separate `s3-java`, `s3-js` and `s3-go` projects, with their existing dependency locks. +- The unified runner starts a fresh protected, persistent release-container stack for each language; clients execute on the host. Fixed service ports belong to independent bridge networks, so fixture concurrency is supported. +- The `s3.sdk` scope also runs as part of the S3 container gate. Every recipe includes normal operations and injected MPU-failure cleanup. - Java: AWS SDK 2.55.10, OpenJDK 21, Maven 3.9, URLConnection transport. - JavaScript: AWS SDK v3 3.1146.0, Node.js 22, TypeScript, Node HTTP transport. -- Go: AWS SDK v2 core 1.47.1 / S3 1.114.0, Go 1.25, net/http transport, - CGO disabled. Toolchains are resolved in pixi.lock; npm dependencies in - package-lock.json, Go modules in go.mod/go.sum, Java dependencies in pom.xml. +- Go: AWS SDK v2 core 1.47.1 / S3 1.114.0, Go 1.25, net/http transport, CGO disabled. Toolchains are resolved in pixi.lock; npm dependencies in package-lock.json, Go modules in go.mod/go.sum, Java dependencies in pom.xml. ```sh pixi run test-e2e-s3 --scope s3.sdk --case s3.sdk::java @@ -26,65 +20,35 @@ pixi run test-e2e-s3 --scope s3.sdk ## Existing test endpoint -Set CROWDB_S3_E2E_ENDPOINT, CROWDB_S3_E2E_ACCESS_KEY and -CROWDB_S3_E2E_SECRET_KEY in the environment, then prepare and verify the host client directly: +Set CROWDB_S3_E2E_ENDPOINT, CROWDB_S3_E2E_ACCESS_KEY and CROWDB_S3_E2E_SECRET_KEY in the environment, then prepare and verify the host client directly: ```sh pixi run -e s3-java-sdk bash e2e-test/clients/s3-sdk.sh java prepare pixi run -e s3-java-sdk bash e2e-test/clients/s3-sdk.sh java verify ``` -Use `s3-js-sdk` / `js` or `s3-go-sdk` / `go` for the other clients. The access -and secret keys also accept AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. -Use an endpoint whose credentials allow bucket creation and deletion. -Do not put credentials in commands or commit them to client configuration. +Use `s3-js-sdk` / `js` or `s3-go-sdk` / `go` for the other clients. The access and secret keys also accept AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. Use an endpoint whose credentials allow bucket creation and deletion. Do not put credentials in commands or commit them to client configuration. - The endpoint branch does not build or start CROWDB services. -- Every client creates a random, uniquely named bucket, deletes only its own - objects and outstanding multipart uploads, deletes that bucket and verifies - absence. It never clears an existing bucket. -- Normal and injected-failure cases each have a 300-second limit plus a - 30-second termination grace period. A failure after an uploaded part must - clean its bucket and unfinished MPU before returning the expected exit 42. - Unexpected errors, timeouts or cleanup errors fail the task. -- Interrupted clients attempt cleanup while the endpoint remains available. - Forced termination or an unavailable endpoint can prevent cleanup; the task - reports failure rather than claiming resources were removed. -- Diagnostics show scenario stages and error codes/types. They omit SDK - exception messages, stack traces, credentials and presigned URLs. The runner deletes the private credential handoff file immediately and redacts - client diagnostics before saving artifacts. +- Every client creates a random, uniquely named bucket, deletes only its own objects and outstanding multipart uploads, deletes that bucket and verifies absence. It never clears an existing bucket. +- Normal and injected-failure cases each have a 300-second limit plus a 30-second termination grace period. A failure after an uploaded part must clean its bucket and unfinished MPU before returning the expected exit 42. Unexpected errors, timeouts or cleanup errors fail the task. +- Interrupted clients attempt cleanup while the endpoint remains available. Forced termination or an unavailable endpoint can prevent cleanup; the task reports failure rather than claiming resources were removed. +- Diagnostics show scenario stages and error codes/types. They omit SDK exception messages, stack traces, credentials and presigned URLs. The runner deletes the private credential handoff file immediately and redacts client diagnostics before saving artifacts. ## Verified contract -Local verification on 2026-10-11 runs Java, Go and JavaScript sequentially on -fresh 14-node formal release-container fixtures. Each pinned recipe passes normal operations and -the injected uploaded-part failure with verified bucket/MPU cleanup. The -GitHub Actions workflow has not been dispatched by this local verification. +Local verification on 2026-10-11 runs Java, Go and JavaScript sequentially on fresh 14-node formal release-container fixtures. Each pinned recipe passes normal operations and the injected uploaded-part failure with verified bucket/MPU cleanup. The GitHub Actions workflow has not been dispatched by this local verification. - Explicit custom endpoint, region us-east-1 and path-style addressing. -- Default SDK request checksum calculation, response validation and retries - remain enabled. Each client observes CRC32 on its ordinary PUT. The endpoint - does not promise every optional response checksum negotiation mode. -- Bucket discovery/lifecycle; exact ordinary bytes; ranged GET; HEAD and user - metadata; COPY and REPLACE metadata; bounded prefix pages with no duplicate - or omitted names; single and batch deletion. -- Sequential low-level multipart upload: one 5 MiB part plus a final small - part, list/complete, exact download and metadata; separate upload/abort. -- SDK presigned PUT/GET. Java and Go construct presigned PUT without a body; - JavaScript signs a known body with its default hoisted CRC32 and additionally - verifies that changed bytes are rejected without overwriting the object. -- Missing-object and invalid-credential errors; explicit incorrect CRC32 - rejection and old-object preservation; cleanup after injected MPU failure. -- High-level transfer managers, concurrent multipart workloads, alternate Java - HTTP transports and browser JavaScript are outside this recipe. +- Default SDK request checksum calculation, response validation and retries remain enabled. Each client observes CRC32 on its ordinary PUT. The endpoint does not promise every optional response checksum negotiation mode. +- Bucket discovery/lifecycle; exact ordinary bytes; ranged GET; HEAD and user metadata; COPY and REPLACE metadata; bounded prefix pages with no duplicate or omitted names; single and batch deletion. +- Sequential low-level multipart upload: one 5 MiB part plus a final small part, list/complete, exact download and metadata; separate upload/abort. +- SDK presigned PUT/GET. Java and Go construct presigned PUT without a body; JavaScript signs a known body with its default hoisted CRC32 and additionally verifies that changed bytes are rejected without overwriting the object. +- Missing-object and invalid-credential errors; explicit incorrect CRC32 rejection and old-object preservation; cleanup after injected MPU failure. +- High-level transfer managers, concurrent multipart workloads, alternate Java HTTP transports and browser JavaScript are outside this recipe. -The accumulated general S3 gate covers storage visibility, interrupted uploads -and service restart recovery. The canonical AWS CLI concurrency-10 case covers its -configured classic transfer recipe. These fresh SDK scenarios cover the -sequential low-level operations above; they do not establish unrestricted -concurrent behavior across other client transports or transfer managers. +The accumulated general S3 gate covers storage visibility, interrupted uploads and service restart recovery. The canonical AWS CLI concurrency-10 case covers its configured classic transfer recipe. These fresh SDK scenarios cover the sequential low-level operations above; they do not establish unrestricted concurrent behavior across other client transports or transfer managers. ## CI ownership -The regular S3 container job runs all three SDK clients once, as part of the -complete S3 domain. Focused selectors use the same release-container runner. +The regular S3 container job runs all three SDK clients once, as part of the complete S3 domain. Focused selectors use the same release-container runner. diff --git a/e2e-test/clients/ecosystem/README.md b/e2e-test/clients/ecosystem/README.md index b157ad18a..27125bba9 100644 --- a/e2e-test/clients/ecosystem/README.md +++ b/e2e-test/clients/ecosystem/README.md @@ -3,97 +3,33 @@ # Manual Iceberg ecosystem acceptance -- Development/test preview only. No production storage, upgrade stability, - ORC selection, cache performance or general S3 bucket claims are established. -- Run the **Iceberg container ecosystem** GitHub Actions workflow manually. - Its `all` profile is the complete verified matrix; `python` is a focused - Python/DuckDB diagnostic and cannot certify distributed engines. -- Locally, run `pixi run oci-build --load`, then - `pixi run -e iceberg-ecosystem test-iceberg-container-ecosystem`. - No default task or default CI job depends on this environment or test. -- Every run owns an isolated bridge network and normal release node with private - mounted data. Host SDK/JVM clients access its inspected IP and fixed port 8135; - the optional pinned Trino engine shares that bridge on port 8080. No host - ports are published. Node recreation preserves the same data, IP and credentials. - Cleanup removes owned containers/networks and retains data/log/crash files - under `.crowdb-runtime/test-data/container-e2e//ecosystem`. -- Install `pixi install --locked -e iceberg-ecosystem`; the full profile also - requires the pinned Trino image below. Missing prerequisites fail before server - allocation. The node reserves 1 CPU / 2 GiB, host clients 2 CPU / 4 GiB and - Trino 1 CPU / 4 GiB: `all` needs 4 CPU / 10 GiB, `python` 3 CPU / 6 GiB. -- `CROWDB_ECOSYSTEM_ARTIFACTS` receives redacted diagnostics, image identity - and operation reports. Credentials and raw runtime/volume files are excluded. - A failed infrastructure, dependency, authentication or unknown query stage - fails the job; it is never counted as an unsupported client capability. +- Development/test preview only. No production storage, upgrade stability, ORC selection, cache performance or general S3 bucket claims are established. +- Run the **Iceberg container ecosystem** GitHub Actions workflow manually. Its `all` profile is the complete verified matrix; `python` is a focused Python/DuckDB diagnostic and cannot certify distributed engines. +- Locally, run `pixi run oci-build --load`, then `pixi run -e iceberg-ecosystem test-iceberg-container-ecosystem`. No default task or default CI job depends on this environment or test. +- Every run owns an isolated bridge network and normal release node with private mounted data. Host SDK/JVM clients access its inspected IP and fixed port 8135; the optional pinned Trino engine shares that bridge on port 8080. No host ports are published. Node recreation preserves the same data, IP and credentials. Cleanup removes owned containers/networks and retains data/log/crash files under `.crowdb-runtime/test-data/container-e2e//ecosystem`. +- Install `pixi install --locked -e iceberg-ecosystem`; the full profile also requires the pinned Trino image below. Missing prerequisites fail before server allocation. The node reserves 1 CPU / 2 GiB, host clients 2 CPU / 4 GiB and Trino 1 CPU / 4 GiB: `all` needs 4 CPU / 10 GiB, `python` 3 CPU / 6 GiB. +- `CROWDB_ECOSYSTEM_ARTIFACTS` receives redacted diagnostics, image identity and operation reports. Credentials and raw runtime/volume files are excluded. A failed infrastructure, dependency, authentication or unknown query stage fails the job; it is never counted as an unsupported client capability. ## Verified capability matrix -The complete `all` profile passed on 2026-10-12 in 192.879 seconds against -release source `d9dd29c41e94ad48a58940fc01b7ad700bdd0cf3`, including persisted-volume -restart. Run `run-a1ee514316574f269327bbbd858ec139` records exact image and client -lock identity. Python/DuckDB also passes independently. Actual interruption during -partial startup and Trino execution removes owned resources and host clients; -reports remain failed and retained data survives. Spark/Flink run in host JVMs -with locked Java 17, full runtime classpaths and Flink's module-access configuration. -Only the operations listed below are established; no engine-wide certification -or untested format/version is implied. +The complete `all` profile passed on 2026-10-12 in 192.879 seconds against release source `d9dd29c41e94ad48a58940fc01b7ad700bdd0cf3`, including persisted-volume restart. Run `run-a1ee514316574f269327bbbd858ec139` records exact image and client lock identity. Python/DuckDB also passes independently. Actual interruption during partial startup and Trino execution removes owned resources and host clients; reports remain failed and retained data survives. Spark/Flink run in host JVMs with locked Java 17, full runtime classpaths and Flink's module-access configuration. Only the operations listed below are established; no engine-wide certification or untested format/version is implied. -- **Python**: PyIceberg 0.11.1, Arrow 25.0.0, pandas - 3.0.6, Polars 1.35.2. Two snapshots with 8192 rows per append. Filtered current - and historical Arrow batches, eager pandas/Polars, a notebook-style query and - a scalar downstream consumer retaining only one batch of at most 8192 rows. - Repeat after persisted-volume restart. Fixture: [Python](python_client.py). -- **DuckDB 1.4.3**: exact-version bundled extension channel; direct REST attach, - catalog identity and selected table rows through delegated FileIO. Successful - SELECT additionally proves a dashboard aggregate. Only an observed explicit - missing remote-signing capability after discovery may be classified as - unsupported; other errors fail. Fixture: [DuckDB](duckdb_client.py). -- **Spark 3.5.6 / Iceberg 1.11.0**: REST catalog + S3FileIO; read Python rows, - append, schema and partition evolution, row deletion, rename/drop of a - separate lifecycle table, read Flink's commit and selected snapshot, and BI - aggregate before/after restart. Fixture: [Spark](spark/src/main/java/SparkAcceptance.java). +- **Python**: PyIceberg 0.11.1, Arrow 25.0.0, pandas 3.0.6, Polars 1.35.2. Two snapshots with 8192 rows per append. Filtered current and historical Arrow batches, eager pandas/Polars, a notebook-style query and a scalar downstream consumer retaining only one batch of at most 8192 rows. Repeat after persisted-volume restart. Fixture: [Python](python_client.py). +- **DuckDB 1.4.3**: exact-version bundled extension channel; direct REST attach, catalog identity and selected table rows through delegated FileIO. Successful SELECT additionally proves a dashboard aggregate. Only an observed explicit missing remote-signing capability after discovery may be classified as unsupported; other errors fail. Fixture: [DuckDB](duckdb_client.py). +- **Spark 3.5.6 / Iceberg 1.11.0**: REST catalog + S3FileIO; read Python rows, append, schema and partition evolution, row deletion, rename/drop of a separate lifecycle table, read Flink's commit and selected snapshot, and BI aggregate before/after restart. Fixture: [Spark](spark/src/main/java/SparkAcceptance.java). - **Flink 1.20.2 / Iceberg 1.11.0**: bounded batch SQL execution, REST catalog - + S3FileIO, read Spark's evolved table and delete result, append, read the - same final rows before/after restart. SQL row deletes and partition evolution - are not advertised for this Flink fixture. Fixture: [Flink](flink/src/main/java/FlinkAcceptance.java). -- **Trino 483**, Linux amd64 image pinned to digest - `sha256:fca43d1fdfdcd45f36b791f7117a47f8ce69c58232e5398bfdb8539cd28e778b`: - direct REST discovery, delegated FileIO read and aggregate before/after - restart. A first divergence is recorded with its exact stage and error; - unsupported Trino FileIO is not silently converted into compatibility. - Fixture: [Trino](trino_client.py). -- **Cross-client handoff**: Python writes, Spark evolves/appends/deletes, - Flink reads/appends, Python records and checks the selected snapshot, - Spark and Python verify it after restart; Flink, DuckDB and Trino verify the - same final visible rows and delete result. Underlying Parquet scans or - in-memory dataframe copies cannot satisfy these checks. -- **Kafka Connect** remains an optional later probe. No connector or - streaming-ingest compatibility is claimed by this core matrix. + + S3FileIO, read Spark's evolved table and delete result, append, read the same final rows before/after restart. SQL row deletes and partition evolution are not advertised for this Flink fixture. Fixture: [Flink](flink/src/main/java/FlinkAcceptance.java). +- **Trino 483**, Linux amd64 image pinned to digest `sha256:fca43d1fdfdcd45f36b791f7117a47f8ce69c58232e5398bfdb8539cd28e778b`: direct REST discovery, delegated FileIO read and aggregate before/after restart. A first divergence is recorded with its exact stage and error; unsupported Trino FileIO is not silently converted into compatibility. Fixture: [Trino](trino_client.py). +- **Cross-client handoff**: Python writes, Spark evolves/appends/deletes, Flink reads/appends, Python records and checks the selected snapshot, Spark and Python verify it after restart; Flink, DuckDB and Trino verify the same final visible rows and delete result. Underlying Parquet scans or in-memory dataframe copies cannot satisfy these checks. +- **Kafka Connect** remains an optional later probe. No connector or streaming-ingest compatibility is claimed by this core matrix. -Client interfaces follow the [PyIceberg API](https://py.iceberg.apache.org/api/), -[DuckDB REST catalog guide](https://duckdb.org/docs/current/core_extensions/iceberg/iceberg_rest_catalogs), -[Iceberg engine releases](https://iceberg.apache.org/releases/), and -[Trino REST catalog properties](https://trino.io/docs/current/object-storage/metastores.html). -Recipes derive from the passing fixtures, rather than client feature lists. +Client interfaces follow the [PyIceberg API](https://py.iceberg.apache.org/api/), [DuckDB REST catalog guide](https://duckdb.org/docs/current/core_extensions/iceberg/iceberg_rest_catalogs), [Iceberg engine releases](https://iceberg.apache.org/releases/), and [Trino REST catalog properties](https://trino.io/docs/current/object-storage/metastores.html). Recipes derive from the passing fixtures, rather than client feature lists. ## Connection recipes -- Python and DuckDB take only the catalog URI and generated Iceberg token. - DuckDB attaches the REST catalog itself and uses delegated FileIO; it never - receives the generic S3 bucket credentials or an in-memory dataframe copy. -- Spark uses RESTCatalog and S3FileIO. Rename targets use the source namespace - (`RENAME TO renamed`); a catalog-qualified target is not a namespace name. -- Flink uses bounded batch SQL and its pinned connector/Hadoop runtime libraries. - [The JVM launcher](engine.sh) supplies the application classpath to JobMaster - deserialization. It does not change CROWDB request or serving-lease budgets. -- Trino enables REST OAuth token authentication, vended credentials and native - S3 FileIO. Set `s3.endpoint` to the CROWDB Iceberg/FileIO origin, - `s3.path-style-access=true` and `s3.region=us-east-1`. Endpoint configuration - is required separately from the REST catalog URI. Do not supply the generic - S3 bucket credentials; table-scoped credentials come from REST delegation. -- The selected format is plaintext Parquet with Iceberg v2 for this matrix. - Spark verifies schema/partition evolution and position deletes; Flink SQL - delete/evolution and Trino writes are not advertised by these fixtures. -- The default workflow profile is `all`; the optional `python` profile proves - only Python and DuckDB. Run both through their separate environment, never as - a dependency of default tests. No workflow was dispatched by local acceptance. +- Python and DuckDB take only the catalog URI and generated Iceberg token. DuckDB attaches the REST catalog itself and uses delegated FileIO; it never receives the generic S3 bucket credentials or an in-memory dataframe copy. +- Spark uses RESTCatalog and S3FileIO. Rename targets use the source namespace (`RENAME TO renamed`); a catalog-qualified target is not a namespace name. +- Flink uses bounded batch SQL and its pinned connector/Hadoop runtime libraries. [The JVM launcher](engine.sh) supplies the application classpath to JobMaster deserialization. It does not change CROWDB request or serving-lease budgets. +- Trino enables REST OAuth token authentication, vended credentials and native S3 FileIO. Set `s3.endpoint` to the CROWDB Iceberg/FileIO origin, `s3.path-style-access=true` and `s3.region=us-east-1`. Endpoint configuration is required separately from the REST catalog URI. Do not supply the generic S3 bucket credentials; table-scoped credentials come from REST delegation. +- The selected format is plaintext Parquet with Iceberg v2 for this matrix. Spark verifies schema/partition evolution and position deletes; Flink SQL delete/evolution and Trino writes are not advertised by these fixtures. +- The default workflow profile is `all`; the optional `python` profile proves only Python and DuckDB. Run both through their separate environment, never as a dependency of default tests. No workflow was dispatched by local acceptance. diff --git a/e2e-test/clients/tpc-loader/README.md b/e2e-test/clients/tpc-loader/README.md index bc93095ae..985f0b4b9 100644 --- a/e2e-test/clients/tpc-loader/README.md +++ b/e2e-test/clients/tpc-loader/README.md @@ -3,23 +3,12 @@ # Manual TPC client -This locked Python 3.12 project runs loader 0.1.1 on the host. Its PyArrow and -PyIceberg requirements differ from the ordinary SDK environment. The formal -release image runs the fourteen storage/Access services; containers do not run -loaders. This scope is excluded from ordinary domain selection and CI. +This locked Python 3.12 project runs loader 0.1.1 on the host. Its PyArrow and PyIceberg requirements differ from the ordinary SDK environment. The formal release image runs the fourteen storage/Access services; containers do not run loaders. This scope is excluded from ordinary domain selection and CI. - Install: `pixi install --locked --manifest-path e2e-test/clients/tpc-loader/pixi.toml`. -- Prepare cached generators and validate complete SF1 data: - `pixi run --locked --manifest-path e2e-test/clients/tpc-loader/pixi.toml prepare .crowdb-runtime/artifacts/tpc-prerequisites`. +- Prepare cached generators and validate complete SF1 data: `pixi run --locked --manifest-path e2e-test/clients/tpc-loader/pixi.toml prepare .crowdb-runtime/artifacts/tpc-prerequisites`. - Run: `pixi run test-container-e2e --scope iceberg.manual.tpc --concurrency 1 --node-cpus 1 --cpus 16 --memory-mib 16384`. -Preparation may download generator prerequisites. Acceptance uses `--no-download` -and concurrently runs TPC-H with eight upload workers and TPC-DS with twenty-four, -using the normal loader validation and a 32-character write token. Public catalog -inventory must contain all eight and twenty-four tables. Each run retains separate -work directories, loader reports, output, server data/logs and timing artifacts. -Missing host Python prerequisites fail before allocating containers. +Preparation may download generator prerequisites. Acceptance uses `--no-download` and concurrently runs TPC-H with eight upload workers and TPC-DS with twenty-four, using the normal loader validation and a 32-character write token. Public catalog inventory must contain all eight and twenty-four tables. Each run retains separate work directories, loader reports, output, server data/logs and timing artifacts. Missing host Python prerequisites fail before allocating containers. -Accepted run: `run-eff8919c09894107ac996f6838f19694`, one pass, 59.549 seconds -including setup/build/teardown (45.21-second host client). Timing is a manual scope -measurement and is not added to the ordinary Iceberg domain count. +Accepted run: `run-eff8919c09894107ac996f6838f19694`, one pass, 59.549 seconds including setup/build/teardown (45.21-second host client). Timing is a manual scope measurement and is not added to the ordinary Iceberg domain count. diff --git a/lib/crowdb-paxos-kv/benches/wal_bench_history.md b/lib/crowdb-paxos-kv/benches/wal_bench_history.md index 65260c944..89fd124e9 100644 --- a/lib/crowdb-paxos-kv/benches/wal_bench_history.md +++ b/lib/crowdb-paxos-kv/benches/wal_bench_history.md @@ -4,8 +4,7 @@ - **What**: Multi-threaded WAL append throughput with durable flush ack. - **Payload**: 1 KB per record (fixed). -- **Backends**: `mem` (in-memory `BlockDevice`), `file` (Tokio buffered file), - `uring` (buffered file with CQE-backed write/read/sync), and aligned `block`. +- **Backends**: `mem` (in-memory `BlockDevice`), `file` (Tokio buffered file), `uring` (buffered file with CQE-backed write/read/sync), and aligned `block`. - **Thread counts**: 1, 32, 128 concurrent loader tasks. - **Duration**: 5 seconds per case (time-based, not fixed record count). - **Metrics**: records, TPS, per-record avg latency, per-batch avg latency, avg batch size. @@ -111,6 +110,4 @@ cargo bench --bench wal -- file_128 | file_32 | 52,606 | 10,514 | 95.1 | 1,522.6 | 16.0 | | uring_32 | 51,330 | 10,259 | 97.5 | 1,560.1 | 16.0 | -The buffered io_uring path preserved durability and recovery but was 2.4% -slower in this single run. Durable device flush latency dominates, so `uring` -remains explicit rather than becoming the automatic default. +The buffered io_uring path preserved durability and recovery but was 2.4% slower in this single run. Durable device flush latency dominates, so `uring` remains explicit rather than becoming the automatic default. diff --git a/lib/crowdb-sdk-native-python/README.md b/lib/crowdb-sdk-native-python/README.md index 8495a2bf9..a04d47ecc 100644 --- a/lib/crowdb-sdk-native-python/README.md +++ b/lib/crowdb-sdk-native-python/README.md @@ -3,10 +3,7 @@ # Native Dataset Python -`crowdb_dataset_native` embeds the trusted main-workspace storage runtime. -Internal management and storage endpoints must be reachable. It accepts no -HTTP token and performs no automatic HTTP fallback. The independently packaged -`crowdb_dataset` remains the portable HTTP SDK. +`crowdb_dataset_native` embeds the trusted main-workspace storage runtime. Internal management and storage endpoints must be reachable. It accepts no HTTP token and performs no automatic HTTP fallback. The independently packaged `crowdb_dataset` remains the portable HTTP SDK. Build and install a wheel in an isolated consumer: @@ -14,19 +11,9 @@ Build and install a wheel in an isolated consumer: pixi run -- python tools/sdk/native_package.py --destination /tmp/crowdb-native-python ``` -The Linux wheel bundles native dependencies except `libibverbs.so.1` and -`librdmacm.so.1`; install those host libraries with matching RDMA providers. -It retains the host-specific `linux` wheel tag. RDMA additionally requires a -working host HCA and device access. `NativeRuntime(seeds)` explicitly selects -TCP; `options` accepts JSON containing `data_rdma`, `metadata_rdma` and -`aggregate_rdma` overrides using the Rust `RdmaConfig` schema. Set `enabled` and -`allow_tcp_fallback` separately per group; metadata RDMA requires its control/data -class. Options also accept `held_bytes`, `producer_bytes` and `requests`. Runtime lifetime -belongs to the caller. Create a new runtime in each worker process; inherited -clients are rejected after fork. +The Linux wheel bundles native dependencies except `libibverbs.so.1` and `librdmacm.so.1`; install those host libraries with matching RDMA providers. It retains the host-specific `linux` wheel tag. RDMA additionally requires a working host HCA and device access. `NativeRuntime(seeds)` explicitly selects TCP; `options` accepts JSON containing `data_rdma`, `metadata_rdma` and `aggregate_rdma` overrides using the Rust `RdmaConfig` schema. Set `enabled` and `allow_tcp_fallback` separately per group; metadata RDMA requires its control/data class. Options also accept `held_bytes`, `producer_bytes` and `requests`. Runtime lifetime belongs to the caller. Create a new runtime in each worker process; inherited clients are rejected after fork. -`runtime.dataset(name)` uses the shared default `dataset-ns` namespace, matching -HTTP and Rust. Pass `namespace=[...]` to select an explicit namespace. +`runtime.dataset(name)` uses the shared default `dataset-ns` namespace, matching HTTP and Rust. Pass `namespace=[...]` to select an explicit namespace. ```python import json @@ -45,96 +32,27 @@ dataset.close() runtime.close() ``` -- `Batch.header` is immutable JSON. `next()` yields a `Lease`, or `None` after - verified EOF. It never acknowledges processing. A batch supports one concurrent - consumer; independent batches can run concurrently. Blocking calls release - Python execution ownership so completion workers and other callers progress. -- `Lease.slice(start, end)` retains the same immutable allocation. Capacity and - registration identity are inspectable. `to_bytes()` explicitly copies into - Python-owned memory; that new allocation is outside SDK held-buffer accounting. - No Python buffer-protocol pointer or new unsafe lifetime adapter is exposed. -- `Batch.cancel()`/`close()` interrupts production without acknowledging. Held - leases remain readable after cancellation, facade close and runtime close. - Closing a facade rejects new calls only on that facade; existing batches and - other facades retain their independently owned lifetimes. -- `runtime.status()` reports current request and memory admission, lifetime - producer/held high water, compatibility-copy bytes and cumulative `read_flow` - layout, stream-credit, storage-wait and frame-decode counters. Writer-policy - groups share the read counters. Reading status performs no RPC or reset; - these are not per-request counts, all-copy totals or per-trial memory peaks. - `runtime.topology(refresh=True)` observes metadata and data transport state; - these hints cannot override routing authority. -- `dataset.call(json, timeout=30)` uses a tagged `op` command and the existing - Rust Dataset JSON schema. Commands cover create/open/snapshot/head, complete - Snapshot/Split preparation, explicit head updates, stateless read binding, - upload create/complete/status/abort and read-only legacy commit status. - `prepare_snapshot` takes `snapshot` and a complete `input: CompositionRequest`; - `prepare_split_revision` takes `snapshot` and `input: SplitPreparation`. - `update_snapshot_head` requires `snapshot` and `expected_head` (nullable). - `update_split_head` requires `snapshot`, `revision`, `expected_revision` and - `expected_head` (both conditions nullable). `delete_split_revision` takes only - `snapshot` and `revision`, using the same 16-byte JSON ID arrays. It returns - hex-string IDs, `complete` and the fixed `reclaim_after` as a decimal string, - or null after cleanup. Retry those same two IDs after an uncertain outcome; - current and initial revisions remain protected. The JSON envelope is bounded to - 8 MiB + 64 KiB; native preparation independently bounds content and work. - `upload_location` returns the authority's opaque field location. - No command is retried automatically. -- `bind_read` accepts `spec: ReadSpec` and returns a `cursor`. Call - `next_batch(cursor)` to fetch; retrying the same cursor replays its batch. - Save `Batch.header.next` after successful processing. There is no remote - acknowledgement, TTL renewal or close. Snapshot/Split/worker binding is - identical in Rust and HTTP. -- `upload_part(upload_id, index, bytes)` copies Python input once into final - transport frame slots across allocations of at most 1 MiB and freezes them - before submission. A logical upload part can still contain 4 MiB. - Snapshot preparation claims final upload owners before marking the resource complete; - pointer publication is a separate conditional update. -- Typed exceptions include capacity, integrity, cancellation, closed runtime, - deadline, conflict and uncertain mutation. Head `ConflictError` carries - `observed_head`, with native JSON ID/generation arrays and the current pointer. - Use it directly for a deliberate retry without another GET. Unknown outcomes - retain the original resource ID, full input and publication conditions. - -`tests/real_consumer.py` validates the installed binding against real services; -`tests/workload.py` includes explicit Python byte-copy costs. The comparison is -run with `pixi run -- python tools/sdk/dataset_workload.py` after building the -current shared service image. It uses one snapshot for HTTP and native clients, -then validates native publication after storage restart. -Use `--semantics-only` to run installed-binding semantics and restart separately; -that mode produces no performance comparison. - -The semantic consumer compares HTTP/native ReadSpec and Split batch headers, -sample IDs and verified field bytes, replays an unchanged continuation on a -second native runtime, and saves a Split continuation for replay after restart. -HTTP is an explicitly selected comparison client in this test; the native -runtime does not use it for fallback. - -Each workload trial contains only 16 or 48 latency observations per size. -Treat p99 as a short workload observation. Observed trials retain cumulative -client-visible batch-header, first nonempty lease delivery and verified-EOF times -alongside the original completion time. Empty payloads have no first delivery. -The timings are not additive or wire first-byte observations; opt-in clocks and -capture work remain in the client window. Explicit `Lease.to_bytes()` copy bytes -are counted separately from SDK compatibility-copy bytes and do not cover all -copies or Python validation allocations. - -Client CPU and RSS are reported per process; `rss_kib` and service `memory.peak` -retain lifetime high-water semantics. Service windows enclose foreground and -background work, including GC, and cannot be added to the client inner window. -TCP receive traffic, exact per-trial peaks and GC CPU attribution remain -unmeasured. The pinned same-host comparison does not establish isolated -production performance. - -Run the explicit local-HCA functional profile with a current service image and -the host's actual RDMA address and device: +- `Batch.header` is immutable JSON. `next()` yields a `Lease`, or `None` after verified EOF. It never acknowledges processing. A batch supports one concurrent consumer; independent batches can run concurrently. Blocking calls release Python execution ownership so completion workers and other callers progress. +- `Lease.slice(start, end)` retains the same immutable allocation. Capacity and registration identity are inspectable. `to_bytes()` explicitly copies into Python-owned memory; that new allocation is outside SDK held-buffer accounting. No Python buffer-protocol pointer or new unsafe lifetime adapter is exposed. +- `Batch.cancel()`/`close()` interrupts production without acknowledging. Held leases remain readable after cancellation, facade close and runtime close. Closing a facade rejects new calls only on that facade; existing batches and other facades retain their independently owned lifetimes. +- `runtime.status()` reports current request and memory admission, lifetime producer/held high water, compatibility-copy bytes and cumulative `read_flow` layout, stream-credit, storage-wait and frame-decode counters. Writer-policy groups share the read counters. Reading status performs no RPC or reset; these are not per-request counts, all-copy totals or per-trial memory peaks. `runtime.topology(refresh=True)` observes metadata and data transport state; these hints cannot override routing authority. +- `dataset.call(json, timeout=30)` uses a tagged `op` command and the existing Rust Dataset JSON schema. Commands cover create/open/snapshot/head, complete Snapshot/Split preparation, explicit head updates, stateless read binding, upload create/complete/status/abort and read-only legacy commit status. `prepare_snapshot` takes `snapshot` and a complete `input: CompositionRequest`; `prepare_split_revision` takes `snapshot` and `input: SplitPreparation`. `update_snapshot_head` requires `snapshot` and `expected_head` (nullable). `update_split_head` requires `snapshot`, `revision`, `expected_revision` and `expected_head` (both conditions nullable). `delete_split_revision` takes only `snapshot` and `revision`, using the same 16-byte JSON ID arrays. It returns hex-string IDs, `complete` and the fixed `reclaim_after` as a decimal string, or null after cleanup. Retry those same two IDs after an uncertain outcome; current and initial revisions remain protected. The JSON envelope is bounded to 8 MiB + 64 KiB; native preparation independently bounds content and work. `upload_location` returns the authority's opaque field location. No command is retried automatically. +- `bind_read` accepts `spec: ReadSpec` and returns a `cursor`. Call `next_batch(cursor)` to fetch; retrying the same cursor replays its batch. Save `Batch.header.next` after successful processing. There is no remote acknowledgement, TTL renewal or close. Snapshot/Split/worker binding is identical in Rust and HTTP. +- `upload_part(upload_id, index, bytes)` copies Python input once into final transport frame slots across allocations of at most 1 MiB and freezes them before submission. A logical upload part can still contain 4 MiB. Snapshot preparation claims final upload owners before marking the resource complete; pointer publication is a separate conditional update. +- Typed exceptions include capacity, integrity, cancellation, closed runtime, deadline, conflict and uncertain mutation. Head `ConflictError` carries `observed_head`, with native JSON ID/generation arrays and the current pointer. Use it directly for a deliberate retry without another GET. Unknown outcomes retain the original resource ID, full input and publication conditions. + +`tests/real_consumer.py` validates the installed binding against real services; `tests/workload.py` includes explicit Python byte-copy costs. The comparison is run with `pixi run -- python tools/sdk/dataset_workload.py` after building the current shared service image. It uses one snapshot for HTTP and native clients, then validates native publication after storage restart. Use `--semantics-only` to run installed-binding semantics and restart separately; that mode produces no performance comparison. + +The semantic consumer compares HTTP/native ReadSpec and Split batch headers, sample IDs and verified field bytes, replays an unchanged continuation on a second native runtime, and saves a Split continuation for replay after restart. HTTP is an explicitly selected comparison client in this test; the native runtime does not use it for fallback. + +Each workload trial contains only 16 or 48 latency observations per size. Treat p99 as a short workload observation. Observed trials retain cumulative client-visible batch-header, first nonempty lease delivery and verified-EOF times alongside the original completion time. Empty payloads have no first delivery. The timings are not additive or wire first-byte observations; opt-in clocks and capture work remain in the client window. Explicit `Lease.to_bytes()` copy bytes are counted separately from SDK compatibility-copy bytes and do not cover all copies or Python validation allocations. + +Client CPU and RSS are reported per process; `rss_kib` and service `memory.peak` retain lifetime high-water semantics. Service windows enclose foreground and background work, including GC, and cannot be added to the client inner window. TCP receive traffic, exact per-trial peaks and GC CPU attribution remain unmeasured. The pinned same-host comparison does not establish isolated production performance. + +Run the explicit local-HCA functional profile with a current service image and the host's actual RDMA address and device: ```sh pixi run -- python tools/rdma/run-native-dataset.py --image crowdb-node:dev --address 10.200.1.1 --device rocep35s0f0 ``` -This profile publishes through strict RDMA transport, restarts storage, verifies -the persisted payload and saved continuation, and checks registered receive -ownership through runtime shutdown. It records logs and transport/resource -snapshots under `.crowdb-runtime/artifacts/native-dataset-hca/`. This is same-host -HCA functional evidence; throughput and latency come from the separate workload. +This profile publishes through strict RDMA transport, restarts storage, verifies the persisted payload and saved continuation, and checks registered receive ownership through runtime shutdown. It records logs and transport/resource snapshots under `.crowdb-runtime/artifacts/native-dataset-hca/`. This is same-host HCA functional evidence; throughput and latency come from the separate workload. diff --git a/lib/crowdb-sdk-native/README.md b/lib/crowdb-sdk-native/README.md index d487d49b0..329f7d1e5 100644 --- a/lib/crowdb-sdk-native/README.md +++ b/lib/crowdb-sdk-native/README.md @@ -3,20 +3,13 @@ # CrowDB native SDK -`crowdb-sdk-native` embeds trusted internal S3 and Dataset access through Chunk-KV and -ChunkIO/DiskIO. It belongs to the main Rust workspace and links the native RPC -stack. Supply management seeds, a tenant and per-operation context. The -embedding process has the same internal-service trust as Access Server; this -API performs no SigV4 verification or user-secret lookup. +`crowdb-sdk-native` embeds trusted internal S3 and Dataset access through Chunk-KV and ChunkIO/DiskIO. It belongs to the main Rust workspace and links the native RPC stack. Supply management seeds, a tenant and per-operation context. The embedding process has the same internal-service trust as Access Server; this API performs no SigV4 verification or user-secret lookup. -The [native runtime overview](../../doc/design/design-crowdb-sdk-native.md) -explains the architecture and buffer model. The separately packaged -[Dataset HTTP SDK](../../sdk/README.md) is the portable Rust/Python distribution. +The [native runtime overview](../../doc/design/design-crowdb-sdk-native.md) explains the architecture and buffer model. The separately packaged [Dataset HTTP SDK](../../sdk/README.md) is the portable Rust/Python distribution. ## Connect and read -The embedding application supplies Tokio. Clone one `ClientRuntime` across -handles to share connection groups, topology caches and budgets: +The embedding application supplies Tokio. Clone one `ClientRuntime` across handles to share connection groups, topology caches and budgets: ```rust use std::num::NonZeroU64; @@ -49,135 +42,48 @@ async fn read_object(seeds: Vec) -> Result<(), Box.whl ``` -Wheels require no Rust compiler at installation. A source build requires Rust -and the selected interpreter. Current installation verification covers Linux -x86_64, CPython 3.14 with and without the GIL; wheels are interpreter-specific. -Other platforms/interpreters need their own build and acceptance. Dependencies -resolve for Rust 1.85; the current build environment uses Rust 1.98. +Wheels require no Rust compiler at installation. A source build requires Rust and the selected interpreter. Current installation verification covers Linux x86_64, CPython 3.14 with and without the GIL; wheels are interpreter-specific. Other platforms/interpreters need their own build and acceptance. Dependencies resolve for Rust 1.85; the current build environment uses Rust 1.98. ```python from crowdb_dataset import DatasetClient, new_id, sample_id @@ -67,17 +48,9 @@ with DatasetClient("http://localhost:9093", "training", "token") as client: "weights", "restored.bin") ``` -Async methods use `AsyncDatasetClient` with the same arguments. Metadata/file -methods are awaited; `stream_batch` and `next_batch` return async context managers -and iterators. Cancellation aborts the Rust operation. Closing a stream waits -for producer cancellation and releases its admission; completed Python bytes -remain valid. `refresh_bearer` updates the credential used by subsequent attempts. +Async methods use `AsyncDatasetClient` with the same arguments. Metadata/file methods are awaited; `stream_batch` and `next_batch` return async context managers and iterators. Cancellation aborts the Rust operation. Closing a stream waits for producer cancellation and releases its admission; completed Python bytes remain valid. `refresh_bearer` updates the credential used by subsequent attempts. -For training, send a ReadPlan with `snapshot` (`latest`, `stable`, or an -immutable ID), selection, projected fields, batch size, partition and nullable -shuffle. Selection supports explicit IDs, prefix, range, keyword, field match -and composition. Predicates evaluate inline manifest metadata; payloads are -fetched only after server selection/shuffle/partition. +For training, send a ReadPlan with `snapshot` (`latest`, `stable`, or an immutable ID), selection, projected fields, batch size, partition and nullable shuffle. Selection supports explicit IDs, prefix, range, keyword, field match and composition. Predicates evaluate inline manifest metadata; payloads are fetched only after server selection/shuffle/partition. ```python from crowdb_dataset import DatasetIterable, WorkerPartition @@ -93,84 +66,31 @@ with DatasetClient("http://localhost:9093", "training", "token") as client: batch.acknowledge() ``` -The iterable acknowledges locally and makes no progress RPC. Continuing without -acknowledgement fails. Save the cursor returned by `batch.acknowledge()` after -processing succeeds; pass it as `checkpoint` to a new iterable to resume. -The iterable skips empty selection or worker windows and advances their local -continuation. Nonempty samples with zero-length fields are still delivered and -require acknowledgement. The lower-level `next_batch` retains empty windows. -`bind_read(plan)` pins immutable versions and returns the initial cursor. -`next_batch(cursor)` replays that cursor on any server. Drop or cancel stops only -the current stream; there is no read ID, session TTL or remote close. Use streams -for fields exceeding the convenience batch limit. +The iterable acknowledges locally and makes no progress RPC. Continuing without acknowledgement fails. Save the cursor returned by `batch.acknowledge()` after processing succeeds; pass it as `checkpoint` to a new iterable to resume. The iterable skips empty selection or worker windows and advances their local continuation. Nonempty samples with zero-length fields are still delivered and require acknowledgement. The lower-level `next_batch` retains empty windows. `bind_read(plan)` pins immutable versions and returns the initial cursor. `next_batch(cursor)` replays that cursor on any server. Drop or cancel stops only the current stream; there is no read ID, session TTL or remote close. Use streams for fields exceeding the convenience batch limit. ## Rust -`DatasetClient` is cloneable; clones share the pool and atomic admission. It -exposes create/open, upload parts/status/complete/abort, file transfer, -prepare/publish/status, snapshot/manifest/listing, stable, retain/release, -reclaim/status, batch streams and stateless read continuations. -`UploadOptions` controls part size and concurrency. File convenience defaults -to 1-MiB parts and four transfers, bounded by the configured request capacity. -Use 4-MiB parts for uploads up to the v1 limit of 10000 parts (39.0625 GiB). +`DatasetClient` is cloneable; clones share the pool and atomic admission. It exposes create/open, upload parts/status/complete/abort, file transfer, prepare/publish/status, snapshot/manifest/listing, stable, retain/release, reclaim/status, batch streams and stateless read continuations. `UploadOptions` controls part size and concurrency. File convenience defaults to 1-MiB parts and four transfers, bounded by the configured request capacity. Use 4-MiB parts for uploads up to the v1 limit of 10000 parts (39.0625 GiB). ```sh pixi run -- cargo run --manifest-path sdk/Cargo.toml --example control ``` -Set `CROWDB_DATASET_ENDPOINT`, `CROWDB_DATASET_NAME` and optional -`CROWDB_DATASET_READ_TOKEN`. `examples/payload.rs` is a functional multi-part -consumer with write/verify phases; the Dataset container fixture runs it. -It contains no benchmark driver or timing threshold. +Set `CROWDB_DATASET_ENDPOINT`, `CROWDB_DATASET_NAME` and optional `CROWDB_DATASET_READ_TOKEN`. `examples/payload.rs` is a functional multi-part consumer with write/verify phases; the Dataset container fixture runs it. It contains no benchmark driver or timing threshold. ## Bounds, completion and recovery -- `HttpConfig` configures connect/request/overall/transfer deadlines, idle pool, - request/response windows, active requests/bytes, retry attempts/backoff and TLS - roots. Python exposes the same configuration names with durations in seconds. - The Dataset response window admits the v1 8-MiB header. Capacity exhaustion is - explicit; there is no unbounded request waiting queue. -- File upload makes a bounded digest pass and bounded transfer pass. Completion - independently verifies durable parts and the whole digest. The completion - request uses the transfer deadline (default 30 minutes), matching the server's - longer integrity deadline. Retry an interrupted file with its original ID, - part configuration and bytes. Accepted parts are reused. -- A lost storage reply is recovered only after confirmed durable chunk cursor - coverage, followed by integrity validation. Readable unconfirmed bytes are not - accepted. A fenced abandoned range is reclaimed before replacing its physical - allocation; the logical part identity stays unchanged. -- Binary batches use one response for all selected fields. Frames carry MD5; - whole-field verification happens at field end. Success requires batch end and - HTTP EOF with no trailing bytes. Caller-owned writers may contain a verified - prefix on failure; discard it. `download_to_path` publishes a temporary file - only after verification and preserves the old destination on transfer error. -- `get_batch`/`BatchStream.batch()` deliberately bound materialization (8 MiB - by default). Use streams/files for larger values. Python copies each bounded - native frame once. Retained Python bytes and completed Rust results belong to - callers; budget caller-retained data separately from SDK producer windows. -- Only documented idempotent operations retry; bodies/tokens remain unchanged - within one overall deadline. Authentication, integrity and semantic conflicts - are not blanket-retried. Rust ambiguous errors and Python exceptions retain - resource identity. After response loss, retry the identical publication condition; - a status query is optional and does not prove that the head advanced. A successful - publish confirms the current effect. If a later publication replaced it, the old - condition returns a conflict and its observed head for caller review. -- Manifests declare the complete current sample membership. Omitted samples - disappear; surviving samples may inherit omitted optional fields through - parents. Explicit tombstones hide inherited fields, including required fields - that are explicitly removed. Required field names remain in the schema. -- JSON/manifest metadata is bounded to 8 MiB; inline values to 64 KiB. Stateless - reads keep no selected-ID pages or progress on the server. A shuffle group has - at most 100000 IDs and 8 MiB of ID bytes; the cursor is bounded to 256 KiB before - encoding. Small batches within a shuffle group reconstruct that bounded group. -- Complete Snapshot results without a live reference become eligible for background - retirement twenty-four hours after their first content binding; identical retries - do not renew that deadline. Legacy results without a deadline require explicit - reclaim. Reclaim preserves head/stable/retained/ancestor data and a deletion grace period. Deferred cleanup reports - pending; shared ranges do not count as reclaimed whole chunks. - -Local disk cache/checkpoint staging, direct chunk access, RDMA, storage-side -Parquet pushdown and benchmark implementation/measurement remain future work. +- `HttpConfig` configures connect/request/overall/transfer deadlines, idle pool, request/response windows, active requests/bytes, retry attempts/backoff and TLS roots. Python exposes the same configuration names with durations in seconds. The Dataset response window admits the v1 8-MiB header. Capacity exhaustion is explicit; there is no unbounded request waiting queue. +- File upload makes a bounded digest pass and bounded transfer pass. Completion independently verifies durable parts and the whole digest. The completion request uses the transfer deadline (default 30 minutes), matching the server's longer integrity deadline. Retry an interrupted file with its original ID, part configuration and bytes. Accepted parts are reused. +- A lost storage reply is recovered only after confirmed durable chunk cursor coverage, followed by integrity validation. Readable unconfirmed bytes are not accepted. A fenced abandoned range is reclaimed before replacing its physical allocation; the logical part identity stays unchanged. +- Binary batches use one response for all selected fields. Frames carry MD5; whole-field verification happens at field end. Success requires batch end and HTTP EOF with no trailing bytes. Caller-owned writers may contain a verified prefix on failure; discard it. `download_to_path` publishes a temporary file only after verification and preserves the old destination on transfer error. +- `get_batch`/`BatchStream.batch()` deliberately bound materialization (8 MiB by default). Use streams/files for larger values. Python copies each bounded native frame once. Retained Python bytes and completed Rust results belong to callers; budget caller-retained data separately from SDK producer windows. +- Only documented idempotent operations retry; bodies/tokens remain unchanged within one overall deadline. Authentication, integrity and semantic conflicts are not blanket-retried. Rust ambiguous errors and Python exceptions retain resource identity. After response loss, retry the identical publication condition; a status query is optional and does not prove that the head advanced. A successful publish confirms the current effect. If a later publication replaced it, the old condition returns a conflict and its observed head for caller review. +- Manifests declare the complete current sample membership. Omitted samples disappear; surviving samples may inherit omitted optional fields through parents. Explicit tombstones hide inherited fields, including required fields that are explicitly removed. Required field names remain in the schema. +- JSON/manifest metadata is bounded to 8 MiB; inline values to 64 KiB. Stateless reads keep no selected-ID pages or progress on the server. A shuffle group has at most 100000 IDs and 8 MiB of ID bytes; the cursor is bounded to 256 KiB before encoding. Small batches within a shuffle group reconstruct that bounded group. +- Complete Snapshot results without a live reference become eligible for background retirement twenty-four hours after their first content binding; identical retries do not renew that deadline. Legacy results without a deadline require explicit reclaim. Reclaim preserves head/stable/retained/ancestor data and a deletion grace period. Deferred cleanup reports pending; shared ranges do not count as reclaimed whole chunks. + +Local disk cache/checkpoint staging, direct chunk access, RDMA, storage-side Parquet pushdown and benchmark implementation/measurement remain future work. ## Verification @@ -183,122 +103,29 @@ pixi run check-dataset-sdk-boundary pixi run clean --runtime-only && pixi run test-e2e-dataset ``` -The packaging gate builds an sdist outside the repository and installs its wheel -in a compiler-free consumer PATH. Root Cargo gates do not cover this workspace; -SDK and wheel tasks are registered explicitly in CI. +The packaging gate builds an sdist outside the repository and installs its wheel in a compiler-free consumer PATH. Root Cargo gates do not cover this workspace; SDK and wheel tasks are registered explicitly in CI. ## Dataset control v2 -`DatasetClient::control()` returns the typed Rust `v2::ControlClient`, sharing -HTTP pools, credentials and admission budgets. Every operation checks its server -capability before sending a request; unsupported mutations never fall back to v1. -Existing `DatasetClient` methods retain their v1 contracts. - -Successful capability discovery supplies a bounded set of hints shared by client -clones and control wrappers. Warm calls send the business request directly; -authentication, deletion admission and conditional updates still run on the server. -Missing capabilities and failed discoveries are not remembered as denials. -`capabilities()` always queries the server and replaces the hints on a valid reply; -an unsupported API version clears them. Refresh discovery or create a new client -after changing server features. Concurrent cold calls may repeat discovery within -the existing HTTP admission budget. The hint has no TTL and is not a cached read -permission or promise that a redeployed endpoint still supports the operation. - -- `OpaqueId` preserves operation/Snapshot/revision IDs; `Generation` is an opaque - head precondition. `Decimal` encodes exact u64 values as JSON strings, and - `SampleId` preserves binary IDs through canonical base64. -- `prepare_snapshot(snapshot, request)` submits complete parent/source/schema/ - Sample patches/Split definitions to an immutable resource PUT. Retain the Snapshot - ID and full input for retries. It requires `composition-preparation`; body/input/ - final Manifest are bounded to 8 MiB and computation to 30 seconds. Parent and - schema are explicitly nullable. Upsert patches specified fields and keeps other - effective parent fields; each Sample ID appears once. Frozen Split sources and - inherited definitions name exact revisions, so retries never resolve currentSplit. - Rust uses `PrepareSnapshot`, `SnapshotChange`, `SnapshotField` and `SnapshotValue`. - Upload values include the completed result's `id`, `length` and `md5`; conversion - adds no descriptor query. Synchronous and async Python expose the same operation. - Complete Snapshots and their initial Splits are readable before head publication. - Incomplete work is recomputed; matching final content is reused, while conflicting - content or initial Split binding is rejected. No workflow progress is saved. - Capture the intended Dataset head base before computation and publish separately. -- Composition workflow methods and standalone Split workflow methods are removed. - Their old begin/edit/evaluate/prepare/publish HTTP routes return 501; SDKs never - fall back to them. Legacy draft observation/abandonment/cleanup remain HTTP - maintenance operations for existing records. `commit_status` reads only legacy - receipts; new pointer updates create no request ledger or historical replay. -- `split_members`, `split_history`, `inventory`, `roots`, `children`, `schema` and - `samples` return lazy `PageCursor`s. Each `next_page` fetches at most 100 items; - it never eagerly collects the Dataset. Persist `continuation()` for scoped - resume. Dropping the cursor stops further requests. -- Complete Split revisions can be read or used as frozen sources before updating - currentSplit, after a head conflict, or after abandoning their draft. Each new - read still checks Snapshot and revision availability; incomplete or closed - revisions remain unavailable. -- `prepare_split_revision(snapshot, revision, request)` submits the complete policy - and definitions to an immutable resource PUT. Keep the revision ID and exact - input for retries. It requires `split-preparation`; different input at a complete - ID conflicts. There is no draft or evaluation cursor, and incomplete work is - recomputed. Input and final bindings are bounded to 1 MiB; computation has a - 30-second deadline. The complete result is readable before head publication. - Capture the intended head base before preparation. Rust uses - `PrepareSplitRevision`; synchronous and async Python expose the same operation. - Standalone Split workflow SDK methods are removed; their old HTTP write routes - return 501, with no fallback. Initial Splits are prepared with `prepare_snapshot`. -- `update_split_head(snapshot, request)` updates a complete revision directly with - `revision`, `expected_revision` and `expected_head`; both expected values must be - supplied, including null. It requires `split-head-update`, takes no attempt ID - and does not read a draft. Rust's `SplitHead::from_conflict` and Python's - `ConflictError.observed_head` return the failed condition's observation. Review - that base before retrying; the SDK does not rebase automatically. -- `delete_split_revision(snapshot, revision)` marks a complete historical revision - using `split-revision-delete`; revision-read capability alone is insufficient. - The current revision and the Snapshot's initial revision return 409. New reads - of a marked revision are rejected, normally with 404; existing bounded-read - limits may fail first. DELETE returns 200 with both IDs, `complete` - and the fixed twenty-minute `reclaim_after` as a decimal string, or null after - completion. Retries keep the original deadline and both IDs. Existing transport - retry limits apply without head/status queries or legacy draft abandonment. - Rust uses `SplitRevisionDeletion`; synchronous and async Python return the same - fields. Complete revisions otherwise follow Snapshot cleanup without a separate - TTL. Cleanup removes Split metadata, leaving Sample payloads unchanged. -- `update_snapshot_head(request)` selects an existing complete Snapshot with - `snapshot` and explicit nullable `expected_head`; it requires - `snapshot-head-update`, preserves stable and reads no draft or attempt ID. - Rust uses `UpdateSnapshotHead` and `Head::from_conflict`; synchronous and async - Python clients expose `ConflictError.observed_head`. A conflict supplies the - observed latest/stable/generation without an extra head GET. The caller reviews - the new base and may need to rebuild content from it; the SDK never rebases or - falls back to a legacy publication route. -- `bind_read` pins Snapshot and optional Split revision without writes; - `next_batch(cursor)` uses verified streaming framing. Save the next cursor after - successful processing. The required capability is `stateless-reads`; old session - routes are retired. No remote acknowledgement or cleanup call is necessary. -- `upload_file` performs bounded digest and transfer passes. `upload_reader` accepts - caller-owned local/remote readers plus exact length and MD5, checking existing - parts against replayed bytes. Reopen the same source from the beginning on retry. - Changed data fails integrity checks; source files and remote objects are untouched. - This is a reader adapter, not a built-in remote storage/catalog connector. -- Complete Snapshot and Split preparation use caller-owned input; legacy draft cleanup - actions return 501. Background workers retire old draft resources without client - cleanup calls. Snapshot deletion marks durable retirement and reports completion - after background cleanup; latest/stable/retention and deletion grace still apply. - -The `split_control` example exercises local file and caller-owned stream ingestion, -initial Split publication, independent revision publication, historical Split reads, -frozen child inheritance, paged inspection and continuation replay. Run it -against a configured Dataset listener with `CROWDB_DATASET_ENDPOINT`, -`CROWDB_DATASET_NAME` and `CROWDB_DATASET_READ_TOKEN` set. - -Python `DatasetClient.control()` and `AsyncDatasetClient.control()` expose the same -v2 operations through Rust. `SplitSource` identifies a frozen revision/name and -`decimal(n)` rejects floats, booleans and out-of-range integers. Request dictionaries -use the OpenAPI v2 field names; omitted required nullable preconditions fail. -`pages(kind, ...)` is lazy in both synchronous and asynchronous clients. Use -`next_batch` as a context manager, then save the returned continuation after processing. -Cancellation aborts the Rust operation; clients inherited across fork remain invalid. - -`tests/split_consumer.py` is a complete installed-wheel consumer and includes a -bounded caller-owned stream adapter. Supply a stable source, known length/checksum -and the same upload ID for retries. File/remote source objects are never mutated. -The adapter uses the native part API for idempotence and integrity; it does not add -another HTTP transport. `upload_file(path, id)` runs the bounded Rust file flow. +`DatasetClient::control()` returns the typed Rust `v2::ControlClient`, sharing HTTP pools, credentials and admission budgets. Every operation checks its server capability before sending a request; unsupported mutations never fall back to v1. Existing `DatasetClient` methods retain their v1 contracts. + +Successful capability discovery supplies a bounded set of hints shared by client clones and control wrappers. Warm calls send the business request directly; authentication, deletion admission and conditional updates still run on the server. Missing capabilities and failed discoveries are not remembered as denials. `capabilities()` always queries the server and replaces the hints on a valid reply; an unsupported API version clears them. Refresh discovery or create a new client after changing server features. Concurrent cold calls may repeat discovery within the existing HTTP admission budget. The hint has no TTL and is not a cached read permission or promise that a redeployed endpoint still supports the operation. + +- `OpaqueId` preserves operation/Snapshot/revision IDs; `Generation` is an opaque head precondition. `Decimal` encodes exact u64 values as JSON strings, and `SampleId` preserves binary IDs through canonical base64. +- `prepare_snapshot(snapshot, request)` submits complete parent/source/schema/Sample patches/Split definitions to an immutable resource PUT. Retain the Snapshot ID and full input for retries. It requires `composition-preparation`; body/input/final Manifest are bounded to 8 MiB and computation to 30 seconds. Parent and schema are explicitly nullable. Upsert patches specified fields and keeps other effective parent fields; each Sample ID appears once. Frozen Split sources and inherited definitions name exact revisions, so retries never resolve currentSplit. Rust uses `PrepareSnapshot`, `SnapshotChange`, `SnapshotField` and `SnapshotValue`. Upload values include the completed result's `id`, `length` and `md5`; conversion adds no descriptor query. Synchronous and async Python expose the same operation. Complete Snapshots and their initial Splits are readable before head publication. Incomplete work is recomputed; matching final content is reused, while conflicting content or initial Split binding is rejected. No workflow progress is saved. Capture the intended Dataset head base before computation and publish separately. +- Composition workflow methods and standalone Split workflow methods are removed. Their old begin/edit/evaluate/prepare/publish HTTP routes return 501; SDKs never fall back to them. Legacy draft observation/abandonment/cleanup remain HTTP maintenance operations for existing records. `commit_status` reads only legacy receipts; new pointer updates create no request ledger or historical replay. +- `split_members`, `split_history`, `inventory`, `roots`, `children`, `schema` and `samples` return lazy `PageCursor`s. Each `next_page` fetches at most 100 items; it never eagerly collects the Dataset. Persist `continuation()` for scoped resume. Dropping the cursor stops further requests. +- Complete Split revisions can be read or used as frozen sources before updating currentSplit, after a head conflict, or after abandoning their draft. Each new read still checks Snapshot and revision availability; incomplete or closed revisions remain unavailable. +- `prepare_split_revision(snapshot, revision, request)` submits the complete policy and definitions to an immutable resource PUT. Keep the revision ID and exact input for retries. It requires `split-preparation`; different input at a complete ID conflicts. There is no draft or evaluation cursor, and incomplete work is recomputed. Input and final bindings are bounded to 1 MiB; computation has a 30-second deadline. The complete result is readable before head publication. Capture the intended head base before preparation. Rust uses `PrepareSplitRevision`; synchronous and async Python expose the same operation. Standalone Split workflow SDK methods are removed; their old HTTP write routes return 501, with no fallback. Initial Splits are prepared with `prepare_snapshot`. +- `update_split_head(snapshot, request)` updates a complete revision directly with `revision`, `expected_revision` and `expected_head`; both expected values must be supplied, including null. It requires `split-head-update`, takes no attempt ID and does not read a draft. Rust's `SplitHead::from_conflict` and Python's `ConflictError.observed_head` return the failed condition's observation. Review that base before retrying; the SDK does not rebase automatically. +- `delete_split_revision(snapshot, revision)` marks a complete historical revision using `split-revision-delete`; revision-read capability alone is insufficient. The current revision and the Snapshot's initial revision return 409. New reads of a marked revision are rejected, normally with 404; existing bounded-read limits may fail first. DELETE returns 200 with both IDs, `complete` and the fixed twenty-minute `reclaim_after` as a decimal string, or null after completion. Retries keep the original deadline and both IDs. Existing transport retry limits apply without head/status queries or legacy draft abandonment. Rust uses `SplitRevisionDeletion`; synchronous and async Python return the same fields. Complete revisions otherwise follow Snapshot cleanup without a separate TTL. Cleanup removes Split metadata, leaving Sample payloads unchanged. +- `update_snapshot_head(request)` selects an existing complete Snapshot with `snapshot` and explicit nullable `expected_head`; it requires `snapshot-head-update`, preserves stable and reads no draft or attempt ID. Rust uses `UpdateSnapshotHead` and `Head::from_conflict`; synchronous and async Python clients expose `ConflictError.observed_head`. A conflict supplies the observed latest/stable/generation without an extra head GET. The caller reviews the new base and may need to rebuild content from it; the SDK never rebases or falls back to a legacy publication route. +- `bind_read` pins Snapshot and optional Split revision without writes; `next_batch(cursor)` uses verified streaming framing. Save the next cursor after successful processing. The required capability is `stateless-reads`; old session routes are retired. No remote acknowledgement or cleanup call is necessary. +- `upload_file` performs bounded digest and transfer passes. `upload_reader` accepts caller-owned local/remote readers plus exact length and MD5, checking existing parts against replayed bytes. Reopen the same source from the beginning on retry. Changed data fails integrity checks; source files and remote objects are untouched. This is a reader adapter, not a built-in remote storage/catalog connector. +- Complete Snapshot and Split preparation use caller-owned input; legacy draft cleanup actions return 501. Background workers retire old draft resources without client cleanup calls. Snapshot deletion marks durable retirement and reports completion after background cleanup; latest/stable/retention and deletion grace still apply. + +The `split_control` example exercises local file and caller-owned stream ingestion, initial Split publication, independent revision publication, historical Split reads, frozen child inheritance, paged inspection and continuation replay. Run it against a configured Dataset listener with `CROWDB_DATASET_ENDPOINT`, `CROWDB_DATASET_NAME` and `CROWDB_DATASET_READ_TOKEN` set. + +Python `DatasetClient.control()` and `AsyncDatasetClient.control()` expose the same v2 operations through Rust. `SplitSource` identifies a frozen revision/name and `decimal(n)` rejects floats, booleans and out-of-range integers. Request dictionaries use the OpenAPI v2 field names; omitted required nullable preconditions fail. `pages(kind, ...)` is lazy in both synchronous and asynchronous clients. Use `next_batch` as a context manager, then save the returned continuation after processing. Cancellation aborts the Rust operation; clients inherited across fork remain invalid. + +`tests/split_consumer.py` is a complete installed-wheel consumer and includes a bounded caller-owned stream adapter. Supply a stable source, known length/checksum and the same upload ID for retries. File/remote source objects are never mutated. The adapter uses the native part API for idempotence and integrity; it does not add another HTTP transport. `upload_file(path, id)` runs the bounded Rust file flow. diff --git a/sdk/contract-fixtures/dataset-v1/README.md b/sdk/contract-fixtures/dataset-v1/README.md index eef79e51b..b794d12e8 100644 --- a/sdk/contract-fixtures/dataset-v1/README.md +++ b/sdk/contract-fixtures/dataset-v1/README.md @@ -3,16 +3,8 @@ # Dataset v1 contract fixtures -- Compare fixtures with the [SDK JSON models](../../crates/crowdb-sdk-dataset/src/models.rs), - [SDK binary decoder](../../crates/crowdb-sdk-dataset/src/wire.rs), and - [server wire implementation](../../../lib/crowdb-access-dataset/src/wire.rs). -- Control fixtures use fixed synthetic IDs; they do not identify deployed data. - `publication.json` stores `hello` in the `value` field of opaque ID `sample`. -- `batch.bin` contains the same projected bytes in the binary batch protocol. - Truncated, corrupted and unsupported-version variants must never produce - successful completion. The Rust SDK tests every transport split position. -- Fixtures are test inputs, not production dependencies. Server and SDK encode - and decode their own models independently; the SDK never imports authority - or persisted bincode models. -- No previous released public Dataset wire version exists yet. These are the - first frozen fixtures; released-server compatibility remains pending. +- Compare fixtures with the [SDK JSON models](../../crates/crowdb-sdk-dataset/src/models.rs), [SDK binary decoder](../../crates/crowdb-sdk-dataset/src/wire.rs), and [server wire implementation](../../../lib/crowdb-access-dataset/src/wire.rs). +- Control fixtures use fixed synthetic IDs; they do not identify deployed data. `publication.json` stores `hello` in the `value` field of opaque ID `sample`. +- `batch.bin` contains the same projected bytes in the binary batch protocol. Truncated, corrupted and unsupported-version variants must never produce successful completion. The Rust SDK tests every transport split position. +- Fixtures are test inputs, not production dependencies. Server and SDK encode and decode their own models independently; the SDK never imports authority or persisted bincode models. +- No previous released public Dataset wire version exists yet. These are the first frozen fixtures; released-server compatibility remains pending. diff --git a/tools/README.md b/tools/README.md index 7fdaa281b..18df6205f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -3,42 +3,15 @@ # Repository tools -Start with the task name in [`pixi.toml`](../pixi.toml), then read its script. -Run commands through `pixi run`. Read only the directory relevant to the task. - -- **`pixi-tasks/`** — build, install and clean commands extracted from Pixi. `test.py` is the single host test dispatcher: - `unit GROUP`, `e2e DOMAIN`, `profile NAME`, or `all`. Container lifecycle - remains owned by `e2e-test/run.py`. Component runs retain phase logs and wall - times under `.crowdb-runtime/artifacts/unit/`. - `install-deps` prepares host prerequisites, frontend packages and profiling - tools, and enables Pixi Bash completion in `~/.bashrc` without duplicating an - existing entry. New Bash terminals load the completion automatically; source - `~/.bashrc` to enable it in an existing terminal. - `build` compiles C++, Rust release binaries, the frontend and workspace - test targets without executing tests. `clean` stops services and removes build - products and the runtime tree. `clean --runtime-only` stops only recorded - ephemeral processes and preserves build products and persistent clusters. - `oci-clean --cluster NAME` removes owned containers, unused bridges and node - data; `--keep-data` preserves data and restart receipts. -- **`ci-checks/`** — version consistency, production concurrency-container - policy and package-to-Pixi-to-CI coverage. `task_graph.py` follows task calls - through shell scripts; it is shared with timing collection. -- **`ci-build/`** — release runtime and host client artifacts for CI consumers. - `build.py clients` compiles the shared acceptance clients. Optional - `build.py runtime` packages an already staged production runtime. `unpack.py` - verifies the source identity and file digests before restoring either - artifact. Consumers run the supplied binaries with `CROWDB_COMPILED_TESTS` - pointing at `target/compiled-tests`. -- **`test-metrics/`** — sequential suite timing and test counts. Logs and JSON - results are archived by UTC run timestamp under - `.crowdb-runtime/artifacts/measure-tests/`; `latest.json` contains the newest - results without overwriting previous baselines. -- **`cpp-checks/`** — clang-tidy, sanitizer regression and tree/chunk link - isolation probes. The `tree-link-isolation/` sources belong to that probe. -- **`benchmark/`** — repeatable performance regressions, shared result handling, - leak inspection and KV write sentinel checks. -- **[`rpc-test/`](rpc-test/README.md)** — real echo transport measurements and - owned server fixtures, including both client schedulers and sanitizer runs. +Start with the task name in [`pixi.toml`](../pixi.toml), then read its script. Run commands through `pixi run`. Read only the directory relevant to the task. + +- **`pixi-tasks/`** — build, install and clean commands extracted from Pixi. `test.py` is the single host test dispatcher: `unit GROUP`, `e2e DOMAIN`, `profile NAME`, or `all`. Container lifecycle remains owned by `e2e-test/run.py`. Component runs retain phase logs and wall times under `.crowdb-runtime/artifacts/unit/`. `install-deps` prepares host prerequisites, frontend packages and profiling tools, and enables Pixi Bash completion in `~/.bashrc` without duplicating an existing entry. New Bash terminals load the completion automatically; source `~/.bashrc` to enable it in an existing terminal. `build` compiles C++, Rust release binaries, the frontend and workspace test targets without executing tests. `clean` stops services and removes build products and the runtime tree. `clean --runtime-only` stops only recorded ephemeral processes and preserves build products and persistent clusters. `oci-clean --cluster NAME` removes owned containers, unused bridges and node data; `--keep-data` preserves data and restart receipts. +- **`ci-checks/`** — version consistency, production concurrency-container policy and package-to-Pixi-to-CI coverage. `task_graph.py` follows task calls through shell scripts; it is shared with timing collection. +- **`ci-build/`** — release runtime and host client artifacts for CI consumers. `build.py clients` compiles the shared acceptance clients. Optional `build.py runtime` packages an already staged production runtime. `unpack.py` verifies the source identity and file digests before restoring either artifact. Consumers run the supplied binaries with `CROWDB_COMPILED_TESTS` pointing at `target/compiled-tests`. +- **`test-metrics/`** — sequential suite timing and test counts. Logs and JSON results are archived by UTC run timestamp under `.crowdb-runtime/artifacts/measure-tests/`; `latest.json` contains the newest results without overwriting previous baselines. +- **`cpp-checks/`** — clang-tidy, sanitizer regression and tree/chunk link isolation probes. The `tree-link-isolation/` sources belong to that probe. +- **`benchmark/`** — repeatable performance regressions, shared result handling, leak inspection and KV write sentinel checks. +- **[`rpc-test/`](rpc-test/README.md)** — real echo transport measurements and owned server fixtures, including both client schedulers and sanitizer runs. - **`profiling/`** — perf setup and write-path flamegraphs. Common entry points: @@ -50,76 +23,19 @@ pixi run clean --runtime-only pixi run bash tools/test-metrics/measure.sh test-access test-console ``` -For test ownership, inspect the component task and its checked CI map. -Measure timings with `test-metrics/measure.sh`; retained run results are -under `.crowdb-runtime/artifacts/measure-tests/`. CI calls component tasks; -update the package groups in `pixi-tasks/test.py` and the checked CI map, then run -`check-ci-test-tasks`. -Feature-gated or ignored client tests need explicit task selection. +For test ownership, inspect the component task and its checked CI map. Measure timings with `test-metrics/measure.sh`; retained run results are under `.crowdb-runtime/artifacts/measure-tests/`. CI calls component tasks; update the package groups in `pixi-tasks/test.py` and the checked CI map, then run `check-ci-test-tasks`. Feature-gated or ignored client tests need explicit task selection. -Benchmark and profiling scripts create an explicit native measurement fixture -through `benchmark/fixture.py`. The fixture is a `test-util` example, built by -the scripts, and owns its services by PID plus process start identity. It stops -them on explicit teardown or when the measurement owner exits. Its private -descriptor and retained logs live beside the selected measurement state file. -Memory backends, server transport tuning and group reset belong to this fixture; -the installed CLI attaches to its target using the normal client contract. -ChunkDB measurements retain protected placement over three logical native nodes -so EC workloads remain covered, including in release builds. These processes -share one physical host and do not qualify host-failure tolerance. Read/replay -fixtures use memory disks that retain bytes; allocation and write-throughput -fixtures may use NullDisk. -These native measurements do not certify OCI startup or runtime acceptance. +Benchmark and profiling scripts create an explicit native measurement fixture through `benchmark/fixture.py`. The fixture is a `test-util` example, built by the scripts, and owns its services by PID plus process start identity. It stops them on explicit teardown or when the measurement owner exits. Its private descriptor and retained logs live beside the selected measurement state file. Memory backends, server transport tuning and group reset belong to this fixture; the installed CLI attaches to its target using the normal client contract. ChunkDB measurements retain protected placement over three logical native nodes so EC workloads remain covered, including in release builds. These processes share one physical host and do not qualify host-failure tolerance. Read/replay fixtures use memory disks that retain bytes; allocation and write-throughput fixtures may use NullDisk. These native measurements do not certify OCI startup or runtime acceptance. -KV measurements keep the human console report and `report.json` in the latest -CLI benchmark directory. The fixture saves every invocation under `measurements/` -before another invocation replaces it. Reports contain average, p50 and p99 -latency; scripts print `unavailable` for the unsupported p999 field. Failed -commands clear prior reports and preserve complete diagnostics. The Console -component task explicitly builds and runs `benchmark/fixture-smoke.py`, checking -real read/scan/write, reset, foreign control rejection, owner exit and scoped -cleanup. `bench-leak-inspect.sh` remains alive during inspection; Ctrl-C ends its -ownership and stops its services. Profiling records the attached client; fixture -server PIDs are retained in its state for separate server sampling. +KV measurements keep the human console report and `report.json` in the latest CLI benchmark directory. The fixture saves every invocation under `measurements/` before another invocation replaces it. Reports contain average, p50 and p99 latency; scripts print `unavailable` for the unsupported p999 field. Failed commands clear prior reports and preserve complete diagnostics. The Console component task explicitly builds and runs `benchmark/fixture-smoke.py`, checking real read/scan/write, reset, foreign control rejection, owner exit and scoped cleanup. `bench-leak-inspect.sh` remains alive during inspection; Ctrl-C ends its ownership and stops its services. Profiling records the attached client; fixture server PIDs are retained in its state for separate server sampling. -Container packaging and its acceptance scripts live under -[`container/single-node-container/`](../container/single-node-container/README.md). -Generated artifacts belong under `target/` or `.crowdb-runtime/`, not here. +Container packaging and its acceptance scripts live under [`container/single-node-container/`](../container/single-node-container/README.md). Generated artifacts belong under `target/` or `.crowdb-runtime/`, not here. -Regular CI starts lint, UI, C++, Core, Storage, Access (including SDK protocol -peers), Console, runtime compilation and host client compilation independently. -Rust component jobs compile and execute their own tests with Cargo's default -test profile and line-table debug information in CI; UI tests need no Rust -runtime build. Runtime services, shared container clients and shipped Dataset -SDK artifacts use release builds. Cargo uses its default CPU parallelism, and -CMake selects the available system CPU count. Local jobs sharing a build -directory still run their compilation stages sequentially. +Regular CI starts lint, UI, C++, Core, Storage, Access (including SDK protocol peers), Console, runtime compilation and host client compilation independently. Rust component jobs compile and execute their own tests with Cargo's default test profile and line-table debug information in CI; UI tests need no Rust runtime build. Runtime services, shared container clients and shipped Dataset SDK artifacts use release builds. Cargo uses its default CPU parallelism, and CMake selects the available system CPU count. Local jobs sharing a build directory still run their compilation stages sequentially. -Production runtime compilation, staging and OCI construction share one runner, -separate from test code and test features. Host client compilation runs in -parallel. Container jobs wait for both the OCI artifact and the host client -artifact, then reuse those exact inputs without Cargo compilation. The staged -runtime passes directly to the OCI builder without an intermediate archive -upload or a second runner's setup. Cache entries contain debug and release -compilation outputs under separate owner keys; the OCI producer retains the -`BuildRuntime` cache key. Immutable workflow artifacts carry OCI/client inputs. -Producers and consumers use Ubuntu 24.04 with the locked Pixi toolchain. -Consumers take artifact names from producer outputs, so rerunning a failed -consumer can reuse a successful compilation from an earlier workflow attempt. +Production runtime compilation, staging and OCI construction share one runner, separate from test code and test features. Host client compilation runs in parallel. Container jobs wait for both the OCI artifact and the host client artifact, then reuse those exact inputs without Cargo compilation. The staged runtime passes directly to the OCI builder without an intermediate archive upload or a second runner's setup. Cache entries contain debug and release compilation outputs under separate owner keys; the OCI producer retains the `BuildRuntime` cache key. Immutable workflow artifacts carry OCI/client inputs. Producers and consumers use Ubuntu 24.04 with the locked Pixi toolchain. Consumers take artifact names from producer outputs, so rerunning a failed consumer can reuse a successful compilation from an earlier workflow attempt. -KV, S3, Iceberg and Console have independent container jobs; the storage matrix -groups disk (DiskDB/DiskIO), chunk (ChunkDB/Chunk-KV/Chunk stream/Chunk client) -and Dataset. Deployment separates its six reference deployment flows from -service validation and recovery. Console separates native dataset/topology/ -hardware flows from service operations, preserving stable browser batches on -each runner. Iceberg runs core, GC convergence, GC cleanup, SDK and grace -partitions independently. The full grace case retains its own runner and its -complete request/delegation deadline chain. Every domain is covered completely; -partition tests and the CI ownership check reject missing or repeated cases. -The preview and release workflows use the same producer split and preserve -their explicit Docker/containerd runtime gates. The manual ecosystem workflow -consumes the shared OCI artifact and prepares its external client tools in the -acceptance job. Special runtime profiles remain explicit. +KV, S3, Iceberg and Console have independent container jobs; the storage matrix groups disk (DiskDB/DiskIO), chunk (ChunkDB/Chunk-KV/Chunk stream/Chunk client) and Dataset. Deployment separates its six reference deployment flows from service validation and recovery. Console separates native dataset/topology/hardware flows from service operations, preserving stable browser batches on each runner. Iceberg runs core, GC convergence, GC cleanup, SDK and grace partitions independently. The full grace case retains its own runner and its complete request/delegation deadline chain. Every domain is covered completely; partition tests and the CI ownership check reject missing or repeated cases. The preview and release workflows use the same producer split and preserve their explicit Docker/containerd runtime gates. The manual ecosystem workflow consumes the shared OCI artifact and prepares its external client tools in the acceptance job. Special runtime profiles remain explicit. ```sh pixi run test unit core @@ -128,5 +44,4 @@ pixi run test profile containerd-cluster pixi run test-e2e-iceberg ``` -On a fresh checkout, `test-cpp` and `test-storage` depend on the native library -build, while `test-console-ui` depends on dependency installation before dispatch. +On a fresh checkout, `test-cpp` and `test-storage` depend on the native library build, while `test-console-ui` depends on dependency installation before dispatch. diff --git a/tools/ports/README.md b/tools/ports/README.md index 9b8dfa55c..11f47b00e 100644 --- a/tools/ports/README.md +++ b/tools/ports/README.md @@ -3,9 +3,7 @@ # Default ports -All service port values are defined in -[ports.rs](../../lib/crowdb-protocol/src/port/ports.rs). Rust callers import -its constants or `ServicePort`. Explicit deployment inputs override defaults. +All service port values are defined in [ports.rs](../../lib/crowdb-protocol/src/port/ports.rs). Rust callers import its constants or `ServicePort`. Explicit deployment inputs override defaults. After editing the defaults, run: @@ -14,13 +12,6 @@ pixi run python tools/ports/generate.py pixi run python tools/ports/generate.py --check ``` -The generator produces Python, TypeScript and C++ constants, plus container -profiles, Compose examples, Dockerfile and sample configuration. Edit the -adjacent `*.ports.in` source for those files; `@WEB_BASE@` references a constant -and `@WEB_BASE+1@` adds a node offset. Generated files are checked in so they -can be used directly. OCI staging and the packaging tests reject stale outputs. +The generator produces Python, TypeScript and C++ constants, plus container profiles, Compose examples, Dockerfile and sample configuration. Edit the adjacent `*.ports.in` source for those files; `@WEB_BASE@` references a constant and `@WEB_BASE+1@` adds a node offset. Generated files are checked in so they can be used directly. OCI staging and the packaging tests reject stale outputs. -Each active service listener owns a 1000-port range. Three host-network nodes -with one listener of each kind use offsets 0, 1 and 2. Multiple Paxos-KV stores -need distinct RPC ports from the node's configured pool; offsets must account -for that pool. No range is reserved for a nonexistent listener. +Each active service listener owns a 1000-port range. Three host-network nodes with one listener of each kind use offsets 0, 1 and 2. Multiple Paxos-KV stores need distinct RPC ports from the node's configured pool; offsets must account for that pool. No range is reserved for a nonexistent listener. diff --git a/tools/rpc-test/README.md b/tools/rpc-test/README.md index f4fcdcd2a..65cfef92c 100644 --- a/tools/rpc-test/README.md +++ b/tools/rpc-test/README.md @@ -3,9 +3,7 @@ # RPC transport tests -This tool measures the real C++ echo transport independently of application -cluster lifecycle. `run.py` starts one echo server on an available port, runs the -client, and waits for graceful server shutdown. It owns only that child process. +This tool measures the real C++ echo transport independently of application cluster lifecycle. `run.py` starts one echo server on an available port, runs the client, and waits for graceful server shutdown. It owns only that child process. ```bash pixi run cargo build -p crowdb-rpc-test @@ -15,19 +13,10 @@ pixi run python tools/rpc-test/run.py --client target/debug/crowdb-rpc-test \ ``` - `--mode tokio` selects the other client scheduler. -- `--io-engines`, `--io-workers`, `--loader-num`, `--connections` and - `--enable-nagle` retain the transport regression sweep controls. -- JSON stdout reports measured operations, errors, duration and latency. - Server and client diagnostics remain under `--log-dir`. -- `pixi run cargo test -p crowdb-rpc-test` exercises both schedulers against the - real echo server and validates resource rejection before process creation. -- `pixi run bash tools/benchmark/bench-rpc-regression.sh` builds and runs the - standard sweep. `RPC_BENCH_DURATION` and `RPC_BENCH_CASES` select a focused run. -- The sanitizer regression builds the server and Rust FFI with address - sanitization, then supplies `--asan-runtime` and `--asan-log-prefix`. Only the - two measured child processes receive the sanitizer environment. Reports from - either process fail the regression. +- `--io-engines`, `--io-workers`, `--loader-num`, `--connections` and `--enable-nagle` retain the transport regression sweep controls. +- JSON stdout reports measured operations, errors, duration and latency. Server and client diagnostics remain under `--log-dir`. +- `pixi run cargo test -p crowdb-rpc-test` exercises both schedulers against the real echo server and validates resource rejection before process creation. +- `pixi run bash tools/benchmark/bench-rpc-regression.sh` builds and runs the standard sweep. `RPC_BENCH_DURATION` and `RPC_BENCH_CASES` select a focused run. +- The sanitizer regression builds the server and Rust FFI with address sanitization, then supplies `--asan-runtime` and `--asan-log-prefix`. Only the two measured child processes receive the sanitizer environment. Reports from either process fail the regression. -The client binary can also attach to an explicitly managed echo server through -`--server-port`. Application workloads continue to use `crowdb-cli --cluster -NAME bench ...` against an already running cluster. +The client binary can also attach to an explicitly managed echo server through `--server-port`. Application workloads continue to use `crowdb-cli --cluster NAME bench ...` against an already running cluster. From aa248885dd16a3054522f46c0bc9dae29e7f5b4f Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 6 Nov 2026 10:56:38 +0800 Subject: [PATCH 17/17] Improve Tree memory accounting and page I/O progress --- .../e2e/flows/13-node-service-plans.spec.ts | 4 +- .../e2e/flows/54-chunk-layout.spec.ts | 5 +- e2e-test/oci_reference_recovery.py | 12 +- e2e-test/profiles/setup_choices.py | 9 +- e2e-test/scenarios.py | 10 +- e2e-test/tests/oci_reference_test.py | 24 +++ e2e-test/tests/runner_progression_test.py | 7 +- lib/crowdb-chunk-kv/src/partition.rs | 21 +- lib/crowdb-chunk-kv/src/partition/tree.rs | 97 +++++---- lib/crowdb-paxos-kv/src/cluster/group_cas.rs | 7 +- .../src/cluster/group_owner_fence.rs | 2 +- .../tests/kv_test/crowdb_tree_engine_test.rs | 13 +- .../tests/paxos_test/cas_rejection_test.rs | 4 +- lib/crowdb-tree/src/backend/backing_io.cpp | 9 +- .../chunk/chunk_backend_checkpoint.cpp | 6 +- .../chunk/chunk_collection_operation.cpp | 17 +- .../src/backend/chunk/chunk_fixed_pack.cpp | 59 ++++-- .../src/backend/chunk/chunk_fixed_pack.h | 10 + .../chunk/chunk_protocol_operation.cpp | 14 +- .../src/backend/chunk/chunk_write_queue.cpp | 193 ++++++++++++++++++ .../src/backend/chunk/chunk_write_queue.h | 39 ++++ .../backend/chunk/legacy_chunk_recovery.cpp | 14 +- .../backend/local/scheduled_page_store.cpp | 17 +- .../src/backend/local/scheduled_page_store.h | 8 +- .../src/btree/allocation_account.cpp | 2 + .../src/btree/allocation_account.h | 12 ++ .../src/btree/allocation_account_control.cpp | 3 + .../src/btree/allocation_reservation.h | 10 +- lib/crowdb-tree/src/btree/entry_key_arena.h | 2 +- lib/crowdb-tree/src/btree/flush_batch.cpp | 7 +- .../src/btree/flush_batch_internal.h | 3 - lib/crowdb-tree/src/btree/flush_operation.cpp | 27 ++- .../src/btree/flush_operation_internal.h | 1 + .../src/btree/flush_reservation.cpp | 15 +- lib/crowdb-tree/src/btree/flush_reservation.h | 6 +- .../src/btree/flush_reserve_bound.cpp | 2 +- lib/crowdb-tree/src/btree/flush_subtree.cpp | 9 - lib/crowdb-tree/src/btree/flush_subtree.h | 2 - .../src/btree/flush_subtree_leaf.cpp | 43 +--- lib/crowdb-tree/src/btree/logical_page.cpp | 7 +- lib/crowdb-tree/src/btree/logical_page.h | 45 ++-- .../src/btree/logical_page_cursor.cpp | 5 +- lib/crowdb-tree/src/btree/owned_key.cpp | 4 +- lib/crowdb-tree/src/btree/owned_key.h | 2 +- lib/crowdb-tree/src/btree/page_cache.cpp | 36 ++-- lib/crowdb-tree/src/btree/page_cache.h | 12 +- .../src/btree/page_cache_internal.h | 30 +-- lib/crowdb-tree/src/btree/page_cache_load.cpp | 2 +- .../src/btree/page_cache_residency.cpp | 17 +- .../src/btree/page_cache_residency.h | 35 ++-- .../src/btree/page_cache_residency_internal.h | 8 +- .../btree/page_cache_residency_prepare.cpp | 15 +- lib/crowdb-tree/src/btree/page_encode.h | 18 +- lib/crowdb-tree/src/btree/page_load.cpp | 31 ++- lib/crowdb-tree/src/btree/page_load.h | 18 +- lib/crowdb-tree/src/btree/page_load_frame.cpp | 2 +- lib/crowdb-tree/src/btree/page_load_inner.cpp | 26 ++- .../src/btree/page_load_internal.h | 2 +- lib/crowdb-tree/src/btree/page_load_leaf.cpp | 5 +- lib/crowdb-tree/src/btree/page_load_leaf.h | 3 +- .../src/btree/page_load_lifetime.h | 16 +- .../src/btree/page_load_overflow.cpp | 4 +- lib/crowdb-tree/src/btree/page_writeback.cpp | 28 +-- .../src/btree/published_inspection.cpp | 26 +-- .../src/btree/published_mapping.cpp | 4 +- lib/crowdb-tree/src/btree/published_read.h | 4 +- lib/crowdb-tree/src/btree/published_route.h | 2 +- lib/crowdb-tree/src/btree/published_state.h | 8 +- lib/crowdb-tree/src/btree/resource_budget.cpp | 104 +++++++++- lib/crowdb-tree/src/btree/resource_budget.h | 4 + .../src/btree/resource_observation.cpp | 4 +- .../src/btree/resource_observation.h | 2 +- .../src/btree/source_write_budget.cpp | 3 + lib/crowdb-tree/src/btree/tree_core.cpp | 12 +- lib/crowdb-tree/src/btree/tree_core.h | 2 +- lib/crowdb-tree/src/btree/tree_core_batch.cpp | 21 +- .../src/btree/tree_core_checkpoint.cpp | 2 +- .../src/btree/tree_core_checkpoint_budget.cpp | 18 +- .../btree/tree_core_checkpoint_structure.cpp | 14 +- .../src/btree/tree_core_chunk_gc.cpp | 6 +- .../src/btree/tree_core_chunk_open.cpp | 26 +-- lib/crowdb-tree/src/btree/tree_core_get.cpp | 11 +- lib/crowdb-tree/src/btree/tree_core_open.cpp | 18 +- .../src/btree/tree_core_read_budget.cpp | 93 ++------- .../src/btree/tree_core_read_internal.h | 15 +- lib/crowdb-tree/src/btree/tree_core_scan.cpp | 10 +- .../src/btree/tree_core_scan_budget.cpp | 76 +------ .../src/btree/tree_core_snapshot.cpp | 6 +- .../src/runtime/tree_operation_lifetime.cpp | 14 +- .../src/runtime/tree_operation_lifetime.h | 12 +- .../src/snapshot/captured_write.cpp | 22 +- .../src/snapshot/captured_write_page.cpp | 4 +- .../src/snapshot/captured_write_pump.cpp | 10 +- .../src/snapshot/captured_write_relocate.cpp | 4 +- .../src/snapshot/checkpoint_operation.cpp | 18 +- .../snapshot/checkpoint_operation_plan.cpp | 12 +- .../snapshot/checkpoint_operation_steps.cpp | 8 +- .../src/snapshot/flush_persistence.h | 8 +- .../src/snapshot/flush_persistence_commit.cpp | 13 +- .../src/snapshot/flush_persistence_page.cpp | 48 +++-- .../src/snapshot/local_checkpoint.cpp | 23 ++- .../src/snapshot/local_checkpoint_io.cpp | 2 +- .../snapshot/local_checkpoint_metadata.cpp | 10 +- .../src/snapshot/local_recovery.cpp | 26 +-- .../src/snapshot/local_recovery_frames.cpp | 10 +- .../src/snapshot/local_recovery_io.cpp | 11 +- .../src/snapshot/local_recovery_metadata.cpp | 8 +- .../src/snapshot/local_recovery_pump.cpp | 2 +- .../src/snapshot/native_frame_read.cpp | 19 +- .../src/snapshot/primary_backing.h | 2 +- .../src/snapshot/published_native.cpp | 8 +- .../src/snapshot/published_native_next.cpp | 4 +- .../src/snapshot/published_snapshot.cpp | 50 +++-- .../src/snapshot/published_snapshot.h | 3 + .../snapshot/published_snapshot_entries.cpp | 72 ++----- .../src/snapshot/published_snapshot_read.cpp | 18 +- .../src/snapshot/published_snapshot_scan.cpp | 68 +----- .../tests/allocation_account_test_support.h | 16 +- .../tests/captured_write_test_support.h | 6 +- .../tests/common/test_chunk_backing.h | 12 +- .../tests/common/test_chunk_root.h | 32 +-- .../tests/common/test_legacy_chunk.h | 5 +- .../tests/common/test_tree_core_chunk.h | 2 +- .../tests/extent_ledger_test_support.h | 13 +- .../integration/borrowed_read_budget_test.cpp | 4 +- .../captured_frame_window_test.cpp | 2 +- .../integration/captured_frames_test.cpp | 13 +- .../captured_write_backend_test.cpp | 2 +- .../tests/integration/captured_write_test.cpp | 23 ++- .../checkpoint_handoff_report_test.cpp | 2 +- .../integration/checkpoint_operation_test.cpp | 6 +- .../integration/chunk_backing_pin_test.cpp | 4 +- .../tests/integration/chunk_backing_test.cpp | 6 +- .../chunk_collection_c_api_test.cpp | 2 +- .../chunk_collection_operation_test.cpp | 10 +- .../integration/chunk_root_commit_test.cpp | 7 +- .../chunk_root_incremental_test.cpp | 12 +- .../integration/chunk_snapshot_retry_test.cpp | 4 +- .../integration/chunk_write_queue_test.cpp | 142 +++++++++++++ .../integration/extent_writeback_test.cpp | 18 +- .../integration/flush_operation_test.cpp | 60 ++++++ .../integration/flush_reservation_test.cpp | 21 +- .../legacy_chunk_migration_test.cpp | 16 +- .../tests/integration/local_block_gc_test.cpp | 12 +- .../local_checkpoint_backend_test.cpp | 3 +- .../local_checkpoint_directory_test.cpp | 16 +- .../local_checkpoint_failure_test.cpp | 3 +- .../integration/local_checkpoint_test.cpp | 8 +- .../local_recovery_backend_test.cpp | 3 +- .../local_recovery_failure_test.cpp | 2 +- .../local_recovery_frames_test.cpp | 2 +- .../tests/integration/local_recovery_test.cpp | 12 +- .../integration/memory_backing_gc_test.cpp | 20 +- .../persistent_flush_pipeline_test.cpp | 164 ++++++++++++++- .../recovered_state_backend_test.cpp | 19 +- .../integration/recovered_state_test.cpp | 6 +- .../sparse_block_relocation_test.cpp | 2 +- .../integration/staged_writeback_test.cpp | 4 +- .../tree_capacity_backpressure_test.cpp | 35 +++- .../integration/tree_core_admitted_test.cpp | 3 +- .../tree_core_block_account_test.cpp | 9 +- .../tree_core_chunk_incremental_test.cpp | 4 +- .../tree_point_read_bound_test.cpp | 9 +- .../integration/tree_source_window_test.cpp | 44 ++-- .../tests/load_budget_test_support.h | 2 +- .../tests/local_checkpoint_test_support.h | 3 +- .../tests/recovered_state_test_support.h | 9 +- .../unit/allocation_account_adoption_test.cpp | 2 +- .../unit/allocation_account_grant_test.cpp | 9 +- .../tests/unit/allocation_account_test.cpp | 65 +++++- .../unit/allocation_account_transfer_test.cpp | 2 +- .../tests/unit/apply_dispatcher_test.cpp | 6 +- .../tests/unit/apply_frontier_test.cpp | 6 +- .../tests/unit/apply_progress_test.cpp | 10 +- .../tests/unit/block_account_grant_test.cpp | 9 +- .../tests/unit/block_account_test.cpp | 5 +- .../tests/unit/c_api_read_budget_test.cpp | 61 ++++-- .../tests/unit/captured_rebase_reuse_test.cpp | 2 +- .../tests/unit/chunk_fixed_pack_test.cpp | 31 +++ .../tests/unit/chunk_root_codec_test.cpp | 32 +-- .../tests/unit/extent_exclusion_test.cpp | 12 +- .../tests/unit/extent_gc_lease_test.cpp | 7 +- .../tests/unit/extent_ledger_test.cpp | 16 +- .../unit/flush_runtime_progress_test.cpp | 13 +- .../unit/flush_runtime_scheduler_test.cpp | 13 +- .../tests/unit/native_import_test.cpp | 20 +- .../unit/page_cache_residency_budget_test.cpp | 15 +- .../tests/unit/page_cache_residency_test.cpp | 4 +- .../tests/unit/page_cache_test.cpp | 12 +- .../tests/unit/page_load_delta_test.cpp | 6 +- .../tests/unit/page_load_inner_proof_test.cpp | 4 +- .../tests/unit/page_load_lifetime_test.cpp | 7 +- lib/crowdb-tree/tests/unit/page_load_test.cpp | 5 +- .../tests/unit/portable_import_test.cpp | 10 +- .../tests/unit/publication_bootstrap_test.cpp | 37 ++-- .../tests/unit/publication_lane_test.cpp | 8 +- .../tests/unit/published_read_budget_test.cpp | 12 +- .../tests/unit/runtime_registry_test.cpp | 19 +- .../tests/unit/scan_output_test.cpp | 14 +- .../tests/unit/tree_core_read_test.cpp | 43 ++++ .../unit/tree_operation_lifetime_test.cpp | 18 +- .../tests/unit/tree_wait_metrics_test.cpp | 12 +- .../tests/unit/writeback_rebase_test.cpp | 11 +- 203 files changed, 2152 insertions(+), 1356 deletions(-) create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_write_queue.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_write_queue.h create mode 100644 lib/crowdb-tree/tests/integration/chunk_write_queue_test.cpp diff --git a/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts b/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts index 9c88a35b9..d9b0aad34 100644 --- a/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts @@ -23,7 +23,7 @@ async function occupyDiskdbPort(request: APIRequestContext) { const diskdb = await dependency.json(); const defaults = await json(request, '/api/deployment-defaults?node_id=1'); const deployed = await request.post('/api/nodes/1/services/deploy', { data: { - ...defaults.diskio, kind: 'diskio', instance_id: '99', rpc_port: 7102, disk_group_id: 1, + ...defaults.diskio, kind: 'diskio', instance_id: '99', rpc_port: 7100, disk_group_id: 1, } }); expect(deployed.status(), await deployed.text()).toBe(201); const guard = await deployed.json(); @@ -100,7 +100,7 @@ test('native plans: fixed ports isolate three nodes and failed DiskDB retries pr expect(servers).toHaveLength(3); expect(new Set(servers.map((row: { endpoint: string; rpc_url: string }) => row.endpoint || row.rpc_url)).size).toBe(3); for (const server of servers) { - expect(server.endpoint || server.rpc_url).toMatch(/:7102$/); + expect(server.endpoint || server.rpc_url).toMatch(/:7100$/); expect(server.pid_node_id).toBe(server.node_id); expect(await nativeProcessAlive(server.node_id - 1, server.pid)).toBe(true); } diff --git a/e2e-test/clients/console-ui/e2e/flows/54-chunk-layout.spec.ts b/e2e-test/clients/console-ui/e2e/flows/54-chunk-layout.spec.ts index 87b55f580..deda64849 100644 --- a/e2e-test/clients/console-ui/e2e/flows/54-chunk-layout.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/54-chunk-layout.spec.ts @@ -10,7 +10,8 @@ test('native diagnostics: Chunk pagination stops on the final nonempty window', const next = pages.getByRole('button', { name: 'Next', exact: true }); const windows: string[][] = []; let after: string | null = null; - for (let index = 0; index < 64; index++) { + // Native Tree packs grow independently; the server cursor determines the end. + for (;;) { const response = await request.get('/api/chunks', { params: { limit: 10, ...(after ? { after } : {}) } }); expect(response.ok(), await response.text()).toBeTruthy(); const data = await response.json(); @@ -97,7 +98,7 @@ test('native diagnostics: Chunk replacement windows and real Mirror EC placement const rows = page.getByRole('table', { name: 'Chunks' }).getByRole('button'); const pages = page.getByRole('navigation', { name: 'Chunk page window' }); const seen = new Set(); - for (let window = 0; window < 20; window++) { + for (;;) { const response = await observation; expect(response.ok(), await response.text()).toBeTruthy(); const data = await response.json(); diff --git a/e2e-test/oci_reference_recovery.py b/e2e-test/oci_reference_recovery.py index eff64c615..05e964c36 100644 --- a/e2e-test/oci_reference_recovery.py +++ b/e2e-test/oci_reference_recovery.py @@ -25,11 +25,13 @@ def create_nonvoter(cluster): name = cluster.expected_names[3] volume = runtime.node_volume(cluster, 3) policy = cluster.artifacts / 'later-member.toml' - content = ('version = 1\nmode = "local"\n' - f'cluster = "{cluster.run_id}"\ninitial_voters = 3\n' + diskless_policy()) - if policy.exists(): - assert policy.read_text() == content - else: + if not policy.exists(): + accepted = read_json(cluster, cluster.expected_names[0], 'node-policy.json') + listeners = next(service['listeners'] for service in accepted['setup']['services'] + if service['kind'] == 'paxos-kv') + content = ('version = 1\nmode = "local"\n' + f'cluster = "{cluster.run_id}"\ninitial_voters = 3\n' + + diskless_policy(listeners)) policy.write_text(content) policy.chmod(0o644) cluster.command('volume', 'create', '--label', f'{LABEL}={cluster.run_id}', volume) diff --git a/e2e-test/profiles/setup_choices.py b/e2e-test/profiles/setup_choices.py index eaa7f504f..f8d1d06a1 100644 --- a/e2e-test/profiles/setup_choices.py +++ b/e2e-test/profiles/setup_choices.py @@ -3,19 +3,20 @@ """Explicit choices for fixtures that test bootstrap rather than default setup.""" -def diskless_setup(): +def diskless_setup(listeners=None): services = [ {"kind": "paxos-kv", "enabled": True, - "listeners": {"http_port": 10000, "rpc_port": 10100}}] + "listeners": dict(listeners) if listeners is not None + else {"http_port": 10000, "rpc_port": 10100}}] services += [ {"kind": kind, "enabled": False, "listeners": {}} for kind in ("diskdb", "diskio", "chunkdb", "chunk-kv", "access-server")] return {"storage": {"mode": "diskless"}, "services": services} -def diskless_policy(): +def diskless_policy(listeners=None): lines = ['[storage]', 'mode = "diskless"'] - for service in diskless_setup()["services"]: + for service in diskless_setup(listeners)["services"]: lines += ['[[services]]', f'kind = "{service["kind"]}"', f'enabled = {str(service["enabled"]).lower()}'] if service["listeners"]: diff --git a/e2e-test/scenarios.py b/e2e-test/scenarios.py index 17719e13b..3a15f91ad 100644 --- a/e2e-test/scenarios.py +++ b/e2e-test/scenarios.py @@ -392,11 +392,11 @@ class Case: Case("iceberg.namespace", "access_iceberg::namespace::namespace_functional_crud_survives_native_storage_and_listener_restart"), ) CONSOLE_OPERATION_CASES = ( - Case("console.native.browser.dataset", "console::browser_dataset::browse", native_client=True, client_timeout=360), - Case("console.native.browser.dataset", "console::browser_dataset::import", native_client=True, client_timeout=360), - Case("console.native.browser.dataset", "console::browser_dataset::split_revisions", native_client=True, client_timeout=360), - Case("console.native.browser.dataset", "console::browser_dataset::recovery", native_client=True, client_timeout=360), - Case("console.native.browser.dataset", "console::browser_dataset::capabilities", native_client=True, client_timeout=360), + Case("console.native.browser.dataset", "console::browser_dataset::browse", native_client=True, client_timeout=1020), + Case("console.native.browser.dataset", "console::browser_dataset::import", native_client=True, client_timeout=1020), + Case("console.native.browser.dataset", "console::browser_dataset::split_revisions", native_client=True, client_timeout=1020), + Case("console.native.browser.dataset", "console::browser_dataset::recovery", native_client=True, client_timeout=1020), + Case("console.native.browser.dataset", "console::browser_dataset::capabilities", native_client=True, client_timeout=1020), Case("console.native.browser.six-node", "console::browser_data::six_node_round_trips", native_browser="UI creates three storage nodes, adds three diskless nodes, then KV, S3 and Iceberg persist real data", native_disk_gib=(4096, 4096, 4096, 8, 8, 8), native_zone_gib=(32, 32, 32, 1, 1, 1), native_admission_count=3, native_diskless_nodes=(4, 5, 6), native_bootstrap=False, native_observations=True, client_timeout=600), Case("console.native.browser.topology", "console::browser_topology::multi_rack_leaders", native_browser="native topology: three racks grow one replica and retain three complete leaders", native_rack_ids=(1, 2, 3)), diff --git a/e2e-test/tests/oci_reference_test.py b/e2e-test/tests/oci_reference_test.py index 7e2f30ce4..645231158 100644 --- a/e2e-test/tests/oci_reference_test.py +++ b/e2e-test/tests/oci_reference_test.py @@ -2,6 +2,9 @@ # Licensed under the Apache License, Version 2.0. from types import SimpleNamespace +from pathlib import Path +import tempfile +import tomllib import unittest from unittest.mock import Mock, patch @@ -9,6 +12,7 @@ from fixture import LABEL import oci_reference_runtime as runtime +import oci_reference_recovery as recovery from scenarios import case_nodes, case_resources, select_cases @@ -116,6 +120,26 @@ def test_cold_recovery_reserves_the_later_nonvoter_without_reducing_existing_nod self.assertEqual(case_nodes(case), 4) self.assertEqual(case_resources(case, .25), (3.75, 3 * 1536 + 768 + 512)) + def test_later_nonvoter_keeps_accepted_listener_ports_across_cold_recovery(self): + cluster = self.cluster() + cluster.expected_names = [f'owned-node{index}' for index in range(1, 5)] + cluster.port_base, cluster.image, cluster.containers = 20000, 'expected', [] + accepted = {'setup': {'services': [ + {'kind': 'paxos-kv', 'listeners': {'http_port': 10135, 'rpc_port': 11135}}]}} + with tempfile.TemporaryDirectory() as directory: + cluster.artifacts = Path(directory) + with patch.object(recovery, 'read_json', return_value=accepted) as read: + recovery.create_nonvoter(cluster) + policy = tomllib.loads((cluster.artifacts / 'later-member.toml').read_text()) + kv = next(service for service in policy['services'] if service['kind'] == 'paxos-kv') + self.assertEqual(kv['listeners'], accepted['setup']['services'][0]['listeners']) + self.assertEqual(policy['storage']['mode'], 'diskless') + self.assertTrue(all(not service['enabled'] for service in policy['services'] + if service['kind'] != 'paxos-kv')) + cluster.expected_names = [name + '-cold' for name in cluster.expected_names] + recovery.create_nonvoter(cluster) + read.assert_called_once_with(cluster, 'owned-node1', 'node-policy.json') + def test_recreated_names_reuse_exact_retained_volumes_on_the_accepted_new_network(self): cluster = self.cluster() cluster.expected_names = ['owned-node1-cold'] diff --git a/e2e-test/tests/runner_progression_test.py b/e2e-test/tests/runner_progression_test.py index 032b1230e..68a8a52a8 100644 --- a/e2e-test/tests/runner_progression_test.py +++ b/e2e-test/tests/runner_progression_test.py @@ -43,18 +43,19 @@ def client(self, _bundle, phase, name): run_cases('image', temporary, 'bundle', 1) return cases, events - def test_dataset_preparation_and_browser_keep_separate_original_budgets(self): + def test_dataset_preparation_and_browser_keep_separate_declared_budgets(self): cases, events = self.run_dataset_phases() self.assertEqual(len(cases), 5) self.assertEqual(events, [ - (case.name, phase, 360) + (case.name, phase, case.client_timeout) for case in cases for phase in ('prepare', case.name.rsplit('::', 1)[-1], 'close') ]) def test_dataset_preparation_failure_skips_browser_and_closes_fixture(self): cases, events = self.run_dataset_phases(fail_prepare=True) - self.assertEqual(events, [(cases[0].name, phase, 360) for phase in ('prepare', 'close')]) + self.assertEqual(events, [(cases[0].name, phase, cases[0].client_timeout) + for phase in ('prepare', 'close')]) def test_ready_cli_executes_before_a_peer_finishes_starting(self): executed = threading.Event() diff --git a/lib/crowdb-chunk-kv/src/partition.rs b/lib/crowdb-chunk-kv/src/partition.rs index f4b0af5b9..96c248dba 100644 --- a/lib/crowdb-chunk-kv/src/partition.rs +++ b/lib/crowdb-chunk-kv/src/partition.rs @@ -2189,11 +2189,8 @@ async fn append_and_apply(state: &mut WorkerState, prepared: Vec Result> { - self.tree - .get(key) + let reader = AsyncCrowdbtree::from_shared(self.tree.clone()); + let result = match reader.try_get(key) { + GetOutcome::Ready(result) => result, + GetOutcome::Pending(pending) => pending.await, + }; + result .map(|value| value.map(|(revision, value)| ValueRevision { revision, value })) .map_err(map_tree_read_error) } @@ -321,29 +326,16 @@ impl PartitionTree for CrowdbPartitionTree { if end_key.is_some_and(<[u8]>::is_empty) { return Ok((Vec::new(), false)); } - let scan = match start_key { - Some(start_key) => self.tree.scan_from( - b"", - start_key, + let scan = AsyncCrowdbtree::from_shared(self.tree.clone()) + .scan_with_options(ScanOptions { + start: start_key.map(<[u8]>::to_vec), start_inclusive, - end_key.unwrap_or_default(), + end: end_key.unwrap_or_default().to_vec(), limit, byte_budget, - false, - 0, - false, - ), - None => self.tree.scan( - b"", - b"", - end_key.unwrap_or_default(), - limit, - byte_budget, - false, - 0, - false, - ), - }; + ..Default::default() + }) + .await; let (entries, truncated) = scan.map_err(map_tree_read_error)?; Ok(( entries @@ -371,15 +363,17 @@ impl PartitionTree for CrowdbPartitionTree { if end.is_some_and(<[u8]>::is_empty) { return Ok((Vec::new(), false)); } - let (entries, truncated) = self - .tree - .scan_from_bounded( - start.unwrap_or_default(), - start.is_none() || inclusive, - end.unwrap_or_default(), + let (entries, truncated) = AsyncCrowdbtree::from_shared(self.tree.clone()) + .scan_with_options(ScanOptions { + start: start.map(<[u8]>::to_vec), + start_inclusive: inclusive, + end: end.unwrap_or_default().to_vec(), limit, - bytes, - ) + byte_budget: bytes, + strict_byte_budget: true, + ..Default::default() + }) + .await .map_err(map_tree_read_error)?; Ok(( entries @@ -402,10 +396,18 @@ impl PartitionTree for CrowdbPartitionTree { start_inclusive: bool, begin_key: Option<&[u8]>, ) -> Result> { - self.tree - .seek_reverse(start_key, start_inclusive, begin_key.unwrap_or_default()) - .map(|entry| { - entry.map(|entry| ScanEntry { + AsyncCrowdbtree::from_shared(self.tree.clone()) + .scan_with_options(ScanOptions { + start: Some(start_key.to_vec()), + start_inclusive, + begin: begin_key.unwrap_or_default().to_vec(), + direction: ScanDirection::Reverse, + limit: 1, + ..Default::default() + }) + .await + .map(|(entries, _)| { + entries.into_iter().next().map(|entry| ScanEntry { key: entry.key, value: ValueRevision { revision: entry.slot, @@ -423,15 +425,16 @@ impl PartitionTree for CrowdbPartitionTree { limit: usize, byte_budget: usize, ) -> Result<(Vec, bool)> { - let (entries, truncated) = self - .tree - .scan_reverse( - start_before, - false, - begin_key.unwrap_or_default(), + let (entries, truncated) = AsyncCrowdbtree::from_shared(self.tree.clone()) + .scan_with_options(ScanOptions { + start: start_before.map(<[u8]>::to_vec), + begin: begin_key.unwrap_or_default().to_vec(), + direction: ScanDirection::Reverse, limit, byte_budget, - ) + ..Default::default() + }) + .await .map_err(map_tree_read_error)?; Ok(( entries @@ -449,10 +452,14 @@ impl PartitionTree for CrowdbPartitionTree { } async fn apply(&self, mutation_seq: u64, operation: &MutationOperation) -> Result<()> { - let result = match operation.successful_value() { - Some(value) => self.tree.apply_put(mutation_seq, operation.key(), value), - None => self.tree.apply_delete(mutation_seq, operation.key()), - }; + let tree = self.tree.clone(); + let operation = operation.clone(); + let result = tokio::task::spawn_blocking(move || match operation.successful_value() { + Some(value) => tree.apply_put(mutation_seq, operation.key(), value), + None => tree.apply_delete(mutation_seq, operation.key()), + }) + .await + .map_err(|error| ChunkKvError::Internal(format!("tree apply task failed: {error}")))?; result.map_err(map_tree_apply_error) } diff --git a/lib/crowdb-paxos-kv/src/cluster/group_cas.rs b/lib/crowdb-paxos-kv/src/cluster/group_cas.rs index af4b9b242..c990e229a 100644 --- a/lib/crowdb-paxos-kv/src/cluster/group_cas.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_cas.rs @@ -13,6 +13,11 @@ use crate::cluster::local_replica::PxLocalReplicaRole; use crate::paxos::roles::RequestIdentity; impl PxGroup { + pub(crate) fn cas_read_unavailable(error: &impl std::fmt::Display) -> ProposeResult { + tracing::warn!(%error, "conditional read unavailable before proposal admission"); + ProposeResult::CasBusy + } + pub async fn propose_cas( self: &Arc, payload: Vec, @@ -131,7 +136,7 @@ impl PxGroup { .engine_get_versioned(&precondition_key) .await; let result = match observed { - Err(error) => ProposeResult::Rejected(error), + Err(error) => Self::cas_read_unavailable(&error), Ok(value) => { let current_revision = value.as_ref().map_or(0, |(revision, _)| *revision); if current_revision != expected_revision { diff --git a/lib/crowdb-paxos-kv/src/cluster/group_owner_fence.rs b/lib/crowdb-paxos-kv/src/cluster/group_owner_fence.rs index ad772744e..7d5ebc332 100644 --- a/lib/crowdb-paxos-kv/src/cluster/group_owner_fence.rs +++ b/lib/crowdb-paxos-kv/src/cluster/group_owner_fence.rs @@ -196,7 +196,7 @@ impl PxGroup { observation: Box::new(crate::rpc::KvCompareObservation::from_versioned(key, value)), } } - Err(error) => return ProposeResult::Rejected(error), + Err(error) => return Self::cas_read_unavailable(&error), }; if !self.cas_admission_ready(tenure, Some(tenure)) { return self.cas_not_ready_result(); diff --git a/lib/crowdb-paxos-kv/tests/kv_test/crowdb_tree_engine_test.rs b/lib/crowdb-paxos-kv/tests/kv_test/crowdb_tree_engine_test.rs index af7f5678d..1c5ff2733 100644 --- a/lib/crowdb-paxos-kv/tests/kv_test/crowdb_tree_engine_test.rs +++ b/lib/crowdb-paxos-kv/tests/kv_test/crowdb_tree_engine_test.rs @@ -11,7 +11,18 @@ use crowdb_paxos_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine}; use super::conformance; fn open() -> CrowdbTreeEngine { - CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).unwrap() + // Each conformance fixture owns its host quotas, independently of other tests. + let config = CrowdbTreeConfig { + runtime: Some(std::sync::Arc::new( + crowdb_tree_ffi::TreeRuntime::new(crowdb_tree_ffi::TreeRuntimeOptions { + workers: 1, + ..Default::default() + }) + .unwrap(), + )), + ..Default::default() + }; + CrowdbTreeEngine::open(&config).unwrap() } fn durable_config(path: &std::path::Path) -> CrowdbTreeConfig { diff --git a/lib/crowdb-paxos-kv/tests/paxos_test/cas_rejection_test.rs b/lib/crowdb-paxos-kv/tests/paxos_test/cas_rejection_test.rs index f5f0765f6..19d703de2 100644 --- a/lib/crowdb-paxos-kv/tests/paxos_test/cas_rejection_test.rs +++ b/lib/crowdb-paxos-kv/tests/paxos_test/cas_rejection_test.rs @@ -174,7 +174,7 @@ async fn failed_cas_read_does_not_poison_the_leader_or_consume_a_slot() { let result = group .propose_cas(payload(RECORD, b"value"), key.clone(), 0, 1, 1) .await; - assert!(matches!(result, ProposeResult::Rejected(_)), "{result:?}"); + assert!(matches!(result, ProposeResult::CasBusy), "{result:?}"); readable(&group, 0).await; failures.read.store(false, Ordering::Release); assert!(matches!( @@ -222,7 +222,7 @@ async fn nested_owner_cas_read_rejection_preserves_the_owner_and_leader() { 2, ) .await; - assert!(matches!(result, ProposeResult::Rejected(_)), "{result:?}"); + assert!(matches!(result, ProposeResult::CasBusy), "{result:?}"); readable(&group, 1).await; failures.read.store(false, Ordering::Release); let result = group diff --git a/lib/crowdb-tree/src/backend/backing_io.cpp b/lib/crowdb-tree/src/backend/backing_io.cpp index 4ee00afc8..49cf11992 100644 --- a/lib/crowdb-tree/src/backend/backing_io.cpp +++ b/lib/crowdb-tree/src/backend/backing_io.cpp @@ -42,13 +42,6 @@ void submit_chunk_write(AsyncPageStore &store, std::shared_ptr complete.complete(Status::not_supported("Chunk writes require a paid backend lane")); return; } - auto status = scheduled->submit_step( - [backing = std::move(backing), input, metadata = std::move(metadata)] { - return write_chunk_backing(*backing, input, metadata); - }, - complete, 0, input.size()); - if (!status.ok()) { - complete.complete(std::move(status)); - } + scheduled->submit_chunk(std::move(backing), input, std::move(complete)); } } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_backend_checkpoint.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_backend_checkpoint.cpp index a6ea175b2..88d683018 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_backend_checkpoint.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_backend_checkpoint.cpp @@ -72,10 +72,10 @@ Status ChunkBackendState::prepare_checkpoint(BackendCheckpointInput input, std:: auto identity = next_identity(input.view); identity.next_page_id = input.pages->next_page_id; identity.wal_replay_offset = input.wal_offset; - auto result = detail::reserved_shared(input.budget.bytes); + auto result = detail::reserved_shared(input.budget.reservation()); result->backend = this; - auto status = ChunkRootCommit::create(input.pages, domain, ids, identity, input.budget.bytes, input.budget.bytes, - &result->commit, input.max_dependency_steps, + auto status = ChunkRootCommit::create(input.pages, domain, ids, identity, input.budget.reservation(), + input.budget.reservation(), &result->commit, input.max_dependency_steps, committed_root.load(std::memory_order_acquire), input.view.snapshot_baseline ? input.view.snapshot_baseline->pages : nullptr); if (!status.ok()) { diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.cpp index e03190c21..af35e4000 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_collection_operation.cpp @@ -18,8 +18,8 @@ struct ChunkCollectionOperation::State ~State() { release_gate(); - if (budget.bytes) { - budget.bytes->seal(); + if (budget.reservation()) { + budget.reservation()->seal(); } if (budget.io) { budget.io->seal(); @@ -43,7 +43,7 @@ struct ChunkCollectionOperation::State } const auto &domain = *backend->domain; auto status = backend->collector.step(*domain.catalog, *domain.transport, domain.tree, domain.epoch, - budget.bytes, &progress, domain.physical_registry); + budget.reservation(), &progress, domain.physical_registry); ++steps; if (!status.ok() || done()) { release_gate(); @@ -69,10 +69,10 @@ Status ChunkCollectionOperation::create(std::shared_ptr std::shared_ptr lane, PageLoadBudget budget, std::shared_ptr *out) try { - if (!backend || !store || !budget.bytes || !budget.io || out == nullptr || *out) { + if (!backend || !store || !budget.reservation() || !budget.io || out == nullptr || *out) { return Status::invalid_argument(); } - auto state = detail::reserved_shared(budget.bytes); + auto state = detail::reserved_shared(budget.reservation()); state->backend = std::move(backend); state->budget = budget; if (state->backend->needs_recovery.load(std::memory_order_acquire) || @@ -80,9 +80,10 @@ try { return Status::unavailable(); } state->owns_gate = true; - auto operation = detail::reserved_construct( - budget.bytes, [&](auto *storage) { ::new (static_cast(storage)) ChunkCollectionOperation(state); }); - auto status = ChunkProtocolOperation::create( + auto operation = detail::reserved_construct(budget.reservation(), [&](auto *storage) { + ::new (static_cast(storage)) ChunkCollectionOperation(state); + }); + auto status = ChunkProtocolOperation::create( std::move(store), std::move(runtime), std::move(lane), budget, state, [state](bool stopped) { return state->step(stopped); }, [state] { return state->done(); }, &operation->protocol_); diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.cpp index 08f571368..3c8973492 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.cpp @@ -33,19 +33,41 @@ Status ChunkFixedPackStore::write(ChunkStageId operation, ChunkId fixed, PagePur std::span input, const std::shared_ptr &budget, ChunkFixedPackRef *out, std::shared_ptr *physical, const std::shared_ptr &physical_budget) +{ + const Write item{.input = input, .out = out}; + return write_many(operation, fixed, purpose, std::span(&item, 1), budget, physical, physical_budget); +} + +Status ChunkFixedPackStore::write_many(ChunkStageId operation, ChunkId fixed, PagePurpose purpose, + std::span inputs, const std::shared_ptr &budget, + std::shared_ptr *physical, + const std::shared_ptr &physical_budget) try { const auto context = ChunkIoContext::capture(); if (!supported()) { return Status::not_supported("fixed staged chunk writes are unavailable"); } - if (!budget || out == nullptr || operation.owner_epoch != epoch_ || !write_allocation_bound(input.size())) { + if (!budget || inputs.empty() || inputs.size() > kMaxStagePacks || operation.owner_epoch != epoch_) { return Status::invalid_argument(); } + size_t bytes = 0; + for (const auto &item : inputs) { + const auto framed = chunk_framed_length(item.input.size()); + if (item.out == nullptr || !write_allocation_bound(item.input.size()) || framed > kMaxChunkPackBytes - bytes) { + return Status::invalid_argument("aggregate exceeds Chunk pack geometry"); + } + bytes += framed; + } auto framed = std::allocate_shared(MetadataAllocator(budget), budget); - framed->bytes.resize(chunk_framed_length(input.size())); - auto status = encode_fixed_pack(fixed, purpose, input, framed->bytes); - if (!status.ok()) { - return status; + framed->bytes.resize(bytes); + size_t offset = 0; + for (const auto &item : inputs) { + const auto length = chunk_framed_length(item.input.size()); + auto status = encode_fixed_pack(fixed, purpose, item.input, std::span(framed->bytes).subspan(offset, length)); + if (!status.ok()) { + return status; + } + offset += length; } ChunkStageRecord reserved{.tree_id = tree_, .id = operation, @@ -57,7 +79,7 @@ try { .purpose = static_cast(purpose), .state = ChunkStageState::Reserved, .packs = decltype(ChunkStageRecord::packs)(MetadataAllocator(budget))}; - status = validate_stage_record(reserved); + auto status = validate_stage_record(reserved); if (!status.ok()) { return status; } @@ -101,9 +123,13 @@ try { auto writing = reserved; writing.revision++; writing.state = ChunkStageState::Writing; - writing.packs.push_back({.offset = 0, - .length = static_cast(input.size()), - .checksum = crowdb::common::crc32c(input.data(), input.size())}); + offset = 0; + for (const auto &item : inputs) { + writing.packs.push_back({.offset = offset, + .length = static_cast(item.input.size()), + .checksum = crowdb::common::crc32c(item.input.data(), item.input.size())}); + offset += chunk_framed_length(item.input.size()); + } status = catalog_->compare_stage(tree_, epoch_, reserved, writing); if (!status.ok()) { return status; @@ -129,12 +155,15 @@ try { if (!status.ok()) { return status; } - *out = {.stage = operation, - .chunk = fixed, - .offset = 0, - .length = writing.packs.front().length, - .checksum = writing.packs.front().checksum, - .purpose = purpose}; + for (size_t index = 0; index < inputs.size(); ++index) { + const auto &pack = writing.packs[index]; + *inputs[index].out = {.stage = operation, + .chunk = fixed, + .offset = pack.offset, + .length = pack.length, + .checksum = pack.checksum, + .purpose = purpose}; + } return Status::Ok(); } catch (const std::bad_alloc &) { diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.h b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.h index 43e414a89..d4e122693 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.h +++ b/lib/crowdb-tree/src/backend/chunk/chunk_fixed_pack.h @@ -42,6 +42,12 @@ struct ChunkPackWindow class ChunkFixedPackStore { public: + struct Write + { + std::span input; + ChunkFixedPackRef *out; + }; + ChunkFixedPackStore(uint64_t tree, uint64_t epoch, uint32_t mirrors, std::shared_ptr catalog, std::shared_ptr transport, std::shared_ptr registry = {}); @@ -53,6 +59,10 @@ class ChunkFixedPackStore const std::shared_ptr &budget, ChunkFixedPackRef *out, std::shared_ptr *physical = nullptr, const std::shared_ptr &physical_budget = {}); + Status write_many(ChunkStageId operation, ChunkId fixed, PagePurpose purpose, std::span inputs, + const std::shared_ptr &budget, + std::shared_ptr *physical = nullptr, + const std::shared_ptr &physical_budget = {}); Status read(const ChunkFixedPackRef &ref, const std::shared_ptr &budget, std::shared_ptr *out) const; [[nodiscard]] Status read_into(const ChunkFixedPackRef &ref, const std::shared_ptr &budget, diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.cpp index dce7e9848..47ee45e8c 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_protocol_operation.cpp @@ -94,7 +94,7 @@ struct ChunkProtocolOperation::State : std::enable_shared_from_this ready.store(false, std::memory_order_release); auto status = scheduled->submit_step( [self] { return self->step(self->stopped.load(std::memory_order_acquire)); }, - {.context = this, .complete_fn = &State::completed, .owner = self, .allocations = budget.bytes}); + {.context = this, .complete_fn = &State::completed, .owner = self, .allocations = budget.reservation()}); if (!status.ok()) { waiting = false; finish(std::move(status)); @@ -128,24 +128,24 @@ Status ChunkProtocolOperation::create(std::shared_ptr std::shared_ptr owner, Step step, Done done, std::shared_ptr *out) try { - if (!store || dynamic_cast(store.get()) == nullptr || !runtime || !lane || !budget.bytes || - !budget.io || !step || !done || out == nullptr || *out || budget.bytes->account() != budget.io->account() || - budget.io->category() != detail::BudgetClass::kIo) { + if (!store || dynamic_cast(store.get()) == nullptr || !runtime || !lane || + !budget.reservation() || !budget.io || !step || !done || out == nullptr || *out || + budget.reservation()->account() != budget.io->account() || budget.io->category() != detail::BudgetClass::kIo) { return Status::invalid_argument(); } - auto state = detail::reserved_shared(budget.bytes, budget); + auto state = detail::reserved_shared(budget.reservation(), budget); state->store = std::move(store); state->owner = std::move(owner); state->step = std::move(step); state->done = std::move(done); state->dispatcher = budget.cpu_progress; if (!state->dispatcher) { - auto status = runtime->reserve_progress(lane, &State::run, state.get(), budget.bytes, &state->progress); + auto status = runtime->reserve_progress(lane, &State::run, state.get(), budget.reservation(), &state->progress); if (!status.ok()) { return status; } } - *out = detail::reserved_construct(budget.bytes, [&](auto *storage) { + *out = detail::reserved_construct(budget.reservation(), [&](auto *storage) { ::new (static_cast(storage)) ChunkProtocolOperation(std::move(state)); }); return Status::Ok(); diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_write_queue.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_write_queue.cpp new file mode 100644 index 000000000..9e5984203 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_write_queue.cpp @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#include "chunk_write_queue.h" + +#include "backend/local/scheduled_page_store.h" +#include "chunk_frame.h" +#include "chunk_physical_registry.h" +#include "crowdb-common/log.h" + +namespace crowdb::tree +{ +struct ChunkWriteQueue::Request +{ + std::shared_ptr queued_owner; + Request *next = nullptr; + std::shared_ptr store; + std::shared_ptr backing; + std::span input; + AsyncCompletion complete; +}; + +struct ChunkWriteQueue::Batch +{ + std::shared_ptr queue; + Request *head = nullptr; + size_t pipe = 0; + + Status write() + { + auto &first = *head->backing; + if (head->next == nullptr) { + return write_chunk_backing(first, head->input, queue->metadata_); + } + const auto &domain = *first.domain; + auto status = domain.catalog->protection()->check_read_owner(); + if (!status.ok()) { + return status; + } + std::vector> inputs{ + MetadataAllocator(queue->metadata_)}; + for (auto *item = head; item != nullptr; item = item->next) { + auto &backing = *item->backing; + // Every member shares the writer pin. Its final owner releases the + // entire allocation; dropping one page cannot expose its peers. + backing.pin = first.pin; + backing.pack.stage = first.pack.stage; + backing.pack.chunk = first.pack.chunk; + inputs.push_back({.input = item->input, .out = &backing.pack}); + } + first.pin->arm(); + detail::ChunkFixedPackStore store(domain.tree, domain.epoch, domain.mirrors, domain.catalog, domain.transport, + domain.physical_registry); + status = store.write_many(first.pack.stage, first.pack.chunk, first.pack.purpose, inputs, queue->metadata_, + &first.physical, first.physical_budget); + for (auto *item = head; item != nullptr; item = item->next) { + item->backing->physical = first.physical; + item->backing->physical_metrics = ChunkPhysicalRegistry::extent(first.physical); + } + return status.ok() ? domain.catalog->protection()->check_read_owner() : status; + } + + void finish(Status status) noexcept + { + queue->active_[pipe].store(false, std::memory_order_release); + while (head != nullptr) { + auto *item = head; + head = head->next; + auto owner = std::move(item->queued_owner); + try { + item->complete.complete(status); + } + catch (...) { + CRB_LOG_ERROR("Chunk write completion receiver threw"); + } + } + queue->wake(); + } + + static void completed(void *raw, Status status) + { + static_cast(raw)->finish(std::move(status)); + } +}; + +void ChunkWriteQueue::push(Request *request) noexcept +{ + auto *head = pending_.load(std::memory_order_relaxed); + do { + request->next = head; + } while (!pending_.compare_exchange_weak(head, request, std::memory_order_release, std::memory_order_relaxed)); +} + +void ChunkWriteQueue::submit(std::shared_ptr store, std::shared_ptr backing, + std::span input, AsyncCompletion complete) +try { + if (!backing || !backing->domain || !complete || + !chunk_backing_matches(*backing, backing->domain->backend_id, input.size()) || + backing->pack.stage.owner_epoch != backing->domain->epoch || + detail::chunk_framed_length(input.size()) > backing->domain->max_chunk_bytes) { + complete.complete(Status::invalid_argument("write requires one reserved Chunk pack")); + return; + } + auto request = std::allocate_shared(MetadataAllocator(metadata_)); + request->store = std::move(store); + request->backing = std::move(backing); + request->input = input; + request->complete = std::move(complete); + request->queued_owner = request; + push(request.get()); + wake(); +} +catch (const std::bad_alloc &) { + complete.complete(Status::resource_exhausted("Chunk write queue allocation failed")); +} + +void ChunkWriteQueue::wake() noexcept +{ + for (size_t pipe = 0; pipe < active_.size(); ++pipe) { + start(pipe); + } +} + +void ChunkWriteQueue::start(size_t pipe) noexcept +{ + if (active_[pipe].exchange(true, std::memory_order_acq_rel)) { + return; + } + auto *incoming = pending_.exchange(nullptr, std::memory_order_acquire); + if (incoming == nullptr) { + active_[pipe].store(false, std::memory_order_release); + if (pending_.load(std::memory_order_acquire) != nullptr) { + start(pipe); + } + return; + } + // Reverse the producer stack so an old queued page is never starved. + Request *ordered = nullptr; + while (incoming != nullptr) { + auto *next = incoming->next; + incoming->next = ordered; + ordered = incoming; + incoming = next; + } + std::shared_ptr batch; + try { + batch = std::allocate_shared(MetadataAllocator(metadata_)); + batch->queue = weak_from_this().lock(); + batch->pipe = pipe; + } + catch (const std::bad_alloc &) { + active_[pipe].store(false, std::memory_order_release); + while (ordered != nullptr) { + auto *item = ordered; + ordered = ordered->next; + auto owner = std::move(item->queued_owner); + item->complete.complete(Status::resource_exhausted("Chunk write batch allocation failed")); + } + wake(); + return; + } + const auto domain = ordered->backing->domain; + const auto purpose = ordered->backing->pack.purpose; + Request **tail = &batch->head; + size_t bytes = 0; + size_t count = 0; + while (ordered != nullptr) { + auto *item = ordered; + ordered = ordered->next; + const auto length = detail::chunk_framed_length(item->input.size()); + if (item->backing->domain == domain && item->backing->pack.purpose == purpose && + count < detail::kMaxStagePacks && length <= domain->max_chunk_bytes - bytes) { + item->next = nullptr; + *tail = item; + tail = &item->next; + bytes += length; + ++count; + } + else { + push(item); + } + } + auto status = batch->head->store->submit_step([batch] { return batch->write(); }, + {.context = batch.get(), + .complete_fn = &Batch::completed, + .owner = batch, + .allocations = metadata_, + .cause = batch->head->complete.cause}, + 0, bytes); + if (!status.ok()) { + batch->finish(std::move(status)); + } +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_write_queue.h b/lib/crowdb-tree/src/backend/chunk/chunk_write_queue.h new file mode 100644 index 000000000..718cadbeb --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_write_queue.h @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "chunk_backing.h" +#include "crowdb-tree/backend/async_page_store.h" + +#include +#include + +namespace crowdb::tree +{ +class ScheduledPageStore; + +// Two independent backend writes drain ready pages without a batching timer. +// Readers use the backend directly and never enter this queue. +class ChunkWriteQueue : public std::enable_shared_from_this +{ + struct Request; + struct Batch; + + public: + explicit ChunkWriteQueue(std::shared_ptr metadata) : metadata_(std::move(metadata)) + { + } + + void submit(std::shared_ptr store, std::shared_ptr backing, + std::span input, AsyncCompletion complete); + + private: + void push(Request *request) noexcept; + void start(size_t pipe) noexcept; + void wake() noexcept; + + std::shared_ptr metadata_; + std::atomic pending_{nullptr}; + std::array, 2> active_{}; +}; +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.cpp b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.cpp index e9c31d53a..74013a800 100644 --- a/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.cpp +++ b/lib/crowdb-tree/src/backend/chunk/legacy_chunk_recovery.cpp @@ -24,8 +24,8 @@ struct RecoveryCredits if (budget.io) { budget.io->seal(); } - if (budget.bytes) { - budget.bytes->seal(); + if (budget.reservation()) { + budget.reservation()->seal(); } } }; @@ -46,7 +46,7 @@ Status recover(const Config &config, const std::shared_ptr & return Status::resource_exhausted(); } credits.budget.bytes = AllocationReservation::create(std::move(*bytes)); - credits.budget.io = reserved_shared(credits.budget.bytes, std::move(*io)); + credits.budget.io = reserved_shared(credits.budget.reservation(), std::move(*io)); auto local = reserved_shared(binding->lifecycle_budget()); auto status = LocalBackendDomain::claim(store.get(), store, binding->lifecycle_budget(), &local->domain); if (status.ok()) { @@ -83,11 +83,11 @@ Status recover(const Config &config, const std::shared_ptr & loads.max_blob_bytes = limits.max_blob_bytes; loads.max_frame_bytes = limits.max_frame_bytes; auto loader = - reserved_shared(credits.budget.bytes, local->store, binding->cpu(), binding->lane(), - loads, OverflowBackingResolver{}, credits.budget); + reserved_shared(credits.budget.reservation(), local->store, binding->cpu(), + binding->lane(), loads, OverflowBackingResolver{}, credits.budget); std::shared_ptr graph; status = RecoveredStateOperation::create(local->inherited, loader, binding->cpu(), binding->lane(), {}, - credits.budget.bytes, &graph); + credits.budget.reservation(), &graph); if (status.ok()) { status = wait_snapshot_operation(graph); } @@ -99,7 +99,7 @@ Status recover(const Config &config, const std::shared_ptr & loader.reset(); recovered.reset(); credits.budget.io->seal(); - credits.budget.bytes->seal(); + credits.budget.reservation()->seal(); binding->drain(); return TreeCore::create(config, binding, {.initial = initial, .local = std::move(local)}, out); } diff --git a/lib/crowdb-tree/src/backend/local/scheduled_page_store.cpp b/lib/crowdb-tree/src/backend/local/scheduled_page_store.cpp index d9285d0db..72b51657e 100644 --- a/lib/crowdb-tree/src/backend/local/scheduled_page_store.cpp +++ b/lib/crowdb-tree/src/backend/local/scheduled_page_store.cpp @@ -2,6 +2,8 @@ // Licensed under the Apache License, Version 2.0. #include "scheduled_page_store.h" +#include "backend/chunk/chunk_write_queue.h" +#include "btree/allocation_account.h" #include "btree/allocation_bound.h" #include "btree/tree_metrics.h" #include "crowdb-common/log.h" @@ -17,6 +19,7 @@ struct ScheduledPageStore::State std::shared_ptr binding; std::shared_ptr lane; std::shared_ptr jobs; + std::shared_ptr chunks; std::atomic active{0}; ~State() @@ -199,7 +202,13 @@ try { state->async = async; state->backend_owner = std::move(backend_owner); state->binding = std::move(binding); - *out = detail::reserved_construct( + std::shared_ptr actual; + auto status = detail::AllocationAccount::create(state->binding->account(), detail::BudgetClass::kRouting, &actual); + if (!status.ok()) { + return status; + } + state->chunks = std::allocate_shared(MetadataAllocator(actual), actual); + *out = detail::reserved_construct( metadata, [&](auto *storage) { ::new (static_cast(storage)) ScheduledPageStore(std::move(state)); }); return Status::Ok(); } @@ -318,4 +327,10 @@ void ScheduledPageStore::cancel(uint64_t /*operation*/) // Accepted operations keep their owners until real terminal or transport // disposal. Cancellation of a consumer never releases in-flight I/O early. } + +void ScheduledPageStore::submit_chunk(std::shared_ptr backing, std::span input, + AsyncCompletion complete) +{ + state_->chunks->submit(shared_from_this(), std::move(backing), input, std::move(complete)); +} } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/backend/local/scheduled_page_store.h b/lib/crowdb-tree/src/backend/local/scheduled_page_store.h index da4c3eba1..bf172f726 100644 --- a/lib/crowdb-tree/src/backend/local/scheduled_page_store.h +++ b/lib/crowdb-tree/src/backend/local/scheduled_page_store.h @@ -7,12 +7,16 @@ #include "crowdb-tree/backend/page_store.h" #include "runtime/tree_runtime_binding.h" +#include + namespace crowdb::tree { +struct ChunkPackBacking; + // Dispatches blocking storage preparation and synchronous backends away from // Tree CPU workers. Queue slots, operation controls and outstanding callbacks // are bounded; an uncertain submission retains the caller's buffer owner. -class ScheduledPageStore final : public AsyncPageStore +class ScheduledPageStore final : public AsyncPageStore, public std::enable_shared_from_this { struct State; struct Job; @@ -37,6 +41,8 @@ class ScheduledPageStore final : public AsyncPageStore // boundary as file I/O; it never executes on the submitting CPU worker. Status submit_step(detail::InlineFunction step, AsyncCompletion complete, size_t read_bytes = 0, size_t write_bytes = 0); + void submit_chunk(std::shared_ptr backing, std::span input, + AsyncCompletion complete); private: explicit ScheduledPageStore(std::shared_ptr state) : state_(std::move(state)) diff --git a/lib/crowdb-tree/src/btree/allocation_account.cpp b/lib/crowdb-tree/src/btree/allocation_account.cpp index be1f440e2..23a84a927 100644 --- a/lib/crowdb-tree/src/btree/allocation_account.cpp +++ b/lib/crowdb-tree/src/btree/allocation_account.cpp @@ -10,6 +10,8 @@ namespace crowdb::tree::detail { AllocationAccount::AllocationAccount(std::shared_ptr account, BudgetClass category) : retirement_(account ? account->retirement_.lock() : nullptr), + work_(account ? account->work_metrics() : nullptr), + resources_(account ? account->resource_metrics() : nullptr), account_(std::move(account)), category_(category), domain_(ResourceBudget::allocation_domain(account_)) diff --git a/lib/crowdb-tree/src/btree/allocation_account.h b/lib/crowdb-tree/src/btree/allocation_account.h index dd35155dd..10564f426 100644 --- a/lib/crowdb-tree/src/btree/allocation_account.h +++ b/lib/crowdb-tree/src/btree/allocation_account.h @@ -32,6 +32,16 @@ class AllocationAccount final : public AllocationBudget return retirement_; } + [[nodiscard]] std::shared_ptr retirement_metrics() const noexcept override + { + return work_.lock(); + } + + [[nodiscard]] std::shared_ptr resource_metrics() const noexcept override + { + return resources_.lock(); + } + [[nodiscard]] size_t owned_bytes() const noexcept { return allocated_.load(std::memory_order_acquire); @@ -81,6 +91,8 @@ class AllocationAccount final : public AllocationBudget // Bound payload accounts use the immutable host service across same-host // ownership transfers. Unbound backend accounts do not observe payloads. const std::shared_ptr retirement_; + const std::weak_ptr work_; + const std::weak_ptr resources_; std::shared_ptr account_; BudgetClass category_; std::atomic domain_; diff --git a/lib/crowdb-tree/src/btree/allocation_account_control.cpp b/lib/crowdb-tree/src/btree/allocation_account_control.cpp index ae7d27eed..b753b9647 100644 --- a/lib/crowdb-tree/src/btree/allocation_account_control.cpp +++ b/lib/crowdb-tree/src/btree/allocation_account_control.cpp @@ -84,6 +84,9 @@ Status AllocationAccount::create(std::shared_ptr accoun if (!account || out == nullptr || category == BudgetClass::kIo || category >= BudgetClass::kCount) { return Status::invalid_argument("backend metadata requires a byte account and output"); } + if (!ResourceBudget::accepting(account)) { + return Status::unavailable("allocation account admission is closed"); + } return build(std::move(account), category, out); } diff --git a/lib/crowdb-tree/src/btree/allocation_reservation.h b/lib/crowdb-tree/src/btree/allocation_reservation.h index f04920bdb..5089708cf 100644 --- a/lib/crowdb-tree/src/btree/allocation_reservation.h +++ b/lib/crowdb-tree/src/btree/allocation_reservation.h @@ -87,13 +87,13 @@ class AllocationReservation : public AllocationBudget }; template -using ReservationAllocator = BudgetAllocator; +using ReservationAllocator = BudgetAllocator; template using ReservedVector = std::vector>; template struct ReservationDeleter { - std::shared_ptr owner; + std::shared_ptr owner; void operator()(T *value) const noexcept { @@ -108,7 +108,7 @@ template struct ReservationDeleter template using ReservedUnique = std::unique_ptr>; template -ReservedUnique reserved_unique(const std::shared_ptr &owner, Args &&...args) +ReservedUnique reserved_unique(const std::shared_ptr &owner, Args &&...args) { ReservationAllocator allocator(owner); auto *value = allocator.allocate(1); @@ -123,7 +123,7 @@ ReservedUnique reserved_unique(const std::shared_ptr & } template -std::shared_ptr reserved_shared(const std::shared_ptr &owner, Args &&...args) +std::shared_ptr reserved_shared(const std::shared_ptr &owner, Args &&...args) { if constexpr (RetirementTraits::enabled) { if (owner) { @@ -138,7 +138,7 @@ std::shared_ptr reserved_shared(const std::shared_ptr // Allows a class factory to construct a private operation handle while both // its object and shared control block remain charged to the same reservation. template -std::shared_ptr reserved_construct(const std::shared_ptr &owner, Construct &&construct) +std::shared_ptr reserved_construct(const std::shared_ptr &owner, Construct &&construct) { ReservationAllocator allocator(owner); auto *value = allocator.allocate(1); diff --git a/lib/crowdb-tree/src/btree/entry_key_arena.h b/lib/crowdb-tree/src/btree/entry_key_arena.h index bf10c1d65..e7c0c5e0c 100644 --- a/lib/crowdb-tree/src/btree/entry_key_arena.h +++ b/lib/crowdb-tree/src/btree/entry_key_arena.h @@ -14,7 +14,7 @@ class EntryKeyArena using Allocator = detail::ReservationAllocator; public: - EntryKeyArena(size_t bytes, const std::shared_ptr &metadata) + EntryKeyArena(size_t bytes, const std::shared_ptr &metadata) : bytes_(bytes, '\0', Allocator(metadata)) { } diff --git a/lib/crowdb-tree/src/btree/flush_batch.cpp b/lib/crowdb-tree/src/btree/flush_batch.cpp index 9f0990013..935c01a38 100644 --- a/lib/crowdb-tree/src/btree/flush_batch.cpp +++ b/lib/crowdb-tree/src/btree/flush_batch.cpp @@ -51,8 +51,11 @@ try { } } for (const auto &table : state->captured->sources->tables) { - if (!table || (table->closed() && std::find(state->inputs->tables.begin(), state->inputs->tables.end(), - table) == state->inputs->tables.end())) { + // A checkpoint at current coverage has no new prefix to hand off. + // Closed sources can retain later versions for the next flush. + if (!table || (frontier > state->captured->coverage && table->closed() && + std::find(state->inputs->tables.begin(), state->inputs->tables.end(), table) == + state->inputs->tables.end())) { return Status::invalid_argument("every closed source must participate in the captured batch"); } } diff --git a/lib/crowdb-tree/src/btree/flush_batch_internal.h b/lib/crowdb-tree/src/btree/flush_batch_internal.h index 6570a57e7..818eb626c 100644 --- a/lib/crowdb-tree/src/btree/flush_batch_internal.h +++ b/lib/crowdb-tree/src/btree/flush_batch_internal.h @@ -23,12 +23,9 @@ struct FlushBatch::State : std::enable_shared_from_this struct Subtree; void start_subtree(); Status collect_subtree(PublicationTransaction &transaction); - void start_load() noexcept; void start_leaves() noexcept; std::atomic leaf_waiters{nullptr}; std::atomic active_leaves{0}; - std::atomic load_waiters{nullptr}; - std::atomic load_active{false}; std::shared_ptr subtree; std::shared_ptr checkpoint_view; std::atomic next_pid{0}; diff --git a/lib/crowdb-tree/src/btree/flush_operation.cpp b/lib/crowdb-tree/src/btree/flush_operation.cpp index c379d3dcb..3cc29f841 100644 --- a/lib/crowdb-tree/src/btree/flush_operation.cpp +++ b/lib/crowdb-tree/src/btree/flush_operation.cpp @@ -86,9 +86,12 @@ bool FlushOperation::State::start(detail::CallbackComplete callback, void *argum return false; } } while (!lifecycle.compare_exchange_weak(previous, previous | kRunning, std::memory_order_acq_rel)); + started_at_ns = tree_metrics_now_ns(); batch_metrics.reset(); if (!only_capture) { - batch_metrics.emplace(publisher.work_metrics(), TreeStage::FlushBatch, tree_metrics_now_ns()); + batch_metrics.emplace(publisher.work_metrics(), TreeStage::FlushBatch, started_at_ns); + CRB_LOG_INFO("flush begin: tree={} start_ns={} persistent={}", static_cast(&publisher), + started_at_ns, options.persistent); } capture_metrics.reset(); drain_metrics.reset(); @@ -158,14 +161,7 @@ void FlushOperation::State::run_cpu(void *context) noexcept void FlushOperation::State::begin() { - if (options.persistent && report.phase == FlushPhase::kRepairRequired && reservations.io && - reservations.io->available() == 0) { - auto credit = detail::ResourceBudget::try_reserve(reservations.io->account(), detail::BudgetClass::kIo, 1); - if (!credit || !reservations.io->return_reserved(std::move(*credit))) { - fail(Status::unavailable("persistent flush awaits its next I/O window")); - return; - } - } + if (capture_only && !frozen.base && covered_at_start) { // A normal connection already handed off this source set. Pair the // newly requested capture with that complete source-free L1, not its @@ -335,11 +331,7 @@ void FlushOperation::State::finish() noexcept CRB_LOG_ERROR("flush failed: batch={} frontier={} handoff={} maintenance={}", report.batch_id, report.frontier, report.handoff_status.to_string(), report.maintenance_status.to_string()); } - if (options.persistent && report.phase == FlushPhase::kRepairRequired && reservations.io) { - // All accepted children have joined. An idle repair must not monopolize - // the only I/O grant needed by an old reader between attempts. - auto unused = reservations.io->take_reserved(reservations.io->available()); - } + auto self = weak_from_this().lock(); auto callback = complete; auto *argument = context; @@ -369,6 +361,13 @@ void FlushOperation::State::finish() noexcept outcome = TreeOutcome::Cancelled; } const auto now = tree_metrics_now_ns(); + if (!capture_only) { + CRB_LOG_INFO("flush end: tree={} start_ns={} end_ns={} persistent={} batch={} floor={} frontier={} " + "coverage={} handoff_ok={} maintenance_ok={} admission_stale={} phase={}", + static_cast(&publisher), started_at_ns, now, options.persistent, report.batch_id, + report.floor, report.frontier, report.published_coverage, report.handoff_status.ok(), + report.maintenance_status.ok(), report.admission_stale, static_cast(report.phase)); + } if (batch_metrics) { const auto batch_outcome = outcome == TreeOutcome::Success && !report.maintenance_status.ok() ? TreeOutcome::Error : outcome; diff --git a/lib/crowdb-tree/src/btree/flush_operation_internal.h b/lib/crowdb-tree/src/btree/flush_operation_internal.h index 9558f3dc9..3297d5e9b 100644 --- a/lib/crowdb-tree/src/btree/flush_operation_internal.h +++ b/lib/crowdb-tree/src/btree/flush_operation_internal.h @@ -109,6 +109,7 @@ struct FlushOperation::State : std::enable_shared_from_this FlushReportComplete report_complete = nullptr; bool covered_at_start = false; FlushResult report; + uint64_t started_at_ns = 0; std::optional batch_metrics; std::optional capture_metrics; std::optional drain_metrics; diff --git a/lib/crowdb-tree/src/btree/flush_reservation.cpp b/lib/crowdb-tree/src/btree/flush_reservation.cpp index 9da3322ca..1ef3a3a37 100644 --- a/lib/crowdb-tree/src/btree/flush_reservation.cpp +++ b/lib/crowdb-tree/src/btree/flush_reservation.cpp @@ -74,7 +74,6 @@ Status bind_flush_reservations(PublicationCoordinator &publisher, const FlushPre reserved.handoff = AllocationReservation::create(std::move(credits[0])); reserved.routing = reserved_shared(reserved.handoff, std::move(credits[1])); reserved.reclaim = reserved_shared(reserved.handoff, std::move(credits[2])); - reserved.io = reserved_shared(reserved.handoff, std::move(credits[3])); reserved.snapshot = AllocationReservation::create(std::move(*snapshot_credit)); auto backend_status = bind_backend_operation(preparation.store.get(), {.max_io_bytes = preparation.loads.max_blob_bytes, @@ -98,28 +97,20 @@ Status bind_flush_reservations(PublicationCoordinator &publisher, const FlushPre // request that waited or joined an older round must use this round's // current checkpoint token and durable baseline. persistence->configure(*batch); - persistence->bind_io(reserved.io); *maintenance = [persistence](auto view, auto done) { persistence->commit(std::move(view), std::move(done)); }; } auto loads = preparation.loads; - loads.max_inflight = 1; + loads.max_inflight = SIZE_MAX; batch->loader = reserved_shared(reserved.routing, preparation.store, preparation.runtime, lane, loads, OverflowBackingResolver{}, PageLoadBudget{.bytes = reserved.routing, - .io = reserved.io, .work_metrics = publisher.work_metrics(), .backend = reserved.backend, .cpu_progress = preparation.cpu_progress, .cause = TreeWorkCause::Flush}); batch->consolidation_loader = batch->loader; if (preparation.cache) { - std::shared_ptr binding; - auto status = - preparation.cache->bind_reserved(batch->loader, reserved.handoff, &binding, PageLoadPriority::Maintenance); - if (!status.ok()) { - return status; - } - batch->loader = std::move(binding); + batch->loader = preparation.cache->bind(batch->loader, reserved.handoff, PageLoadPriority::Maintenance); } batch->consolidation = preparation.maintenance.publication; batch->consolidation->pages.max_output_pages = bounds.maintenance_output_pages; @@ -145,7 +136,7 @@ Status prepare_flush_reservations(PublicationCoordinator &publisher, const Publi return status; } FlushCredits credits; - constexpr std::array names{"handoff", "routing", "reclaim", "I/O"}; + constexpr std::array names{"handoff", "routing", "reclaim"}; for (size_t index = 0; index < credits.size(); ++index) { const auto capacity = ResourceBudget::capacity(sources.active_budget->options().account, FlushReserveBounds::categories[index]); diff --git a/lib/crowdb-tree/src/btree/flush_reservation.h b/lib/crowdb-tree/src/btree/flush_reservation.h index b30f67224..154aed4c9 100644 --- a/lib/crowdb-tree/src/btree/flush_reservation.h +++ b/lib/crowdb-tree/src/btree/flush_reservation.h @@ -28,8 +28,7 @@ namespace detail { struct FlushReserveBounds { - static constexpr std::array categories{BudgetClass::kHandoff, BudgetClass::kRouting, BudgetClass::kReclaim, - BudgetClass::kIo}; + static constexpr std::array categories{BudgetClass::kHandoff, BudgetClass::kRouting, BudgetClass::kReclaim}; std::array amounts{}; size_t backend_bytes = 0; size_t snapshot_bytes = 0; @@ -45,13 +44,12 @@ struct FlushReservations std::shared_ptr handoff; std::shared_ptr routing; std::shared_ptr reclaim; - std::shared_ptr io; std::shared_ptr snapshot; void seal() const noexcept { seal_backend_budget(backend); - for (const auto &credit : {handoff, routing, reclaim, io, snapshot}) { + for (const auto &credit : {handoff, routing, reclaim, snapshot}) { if (credit) { credit->seal(); } diff --git a/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp b/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp index 22aab4682..cc4f3224a 100644 --- a/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp +++ b/lib/crowdb-tree/src/btree/flush_reserve_bound.cpp @@ -145,7 +145,7 @@ Status flush_reserve_bounds(PublicationCoordinator &publisher, const PublishedTr return Status::resource_exhausted("flush reserve bounds exceed the configured work limits"); } *out = { - .amounts = {*handoff, *total_routing, *reclaim, size_t{1}}, + .amounts = {*handoff, *total_routing, *reclaim}, .backend_bytes = *backend, .snapshot_bytes = *snapshot, .maintenance_output_pages = maintenance.publication.pages.max_output_pages diff --git a/lib/crowdb-tree/src/btree/flush_subtree.cpp b/lib/crowdb-tree/src/btree/flush_subtree.cpp index 8bb61a65e..662b22fbc 100644 --- a/lib/crowdb-tree/src/btree/flush_subtree.cpp +++ b/lib/crowdb-tree/src/btree/flush_subtree.cpp @@ -171,10 +171,6 @@ void FlushBatch::State::Subtree::schedule() noexcept owner.options.cpu_progress->request(cpu_node, owner.weak_from_this().lock()); } else { - if (std::exchange(load_completed, false)) { - owner.load_active.store(false, std::memory_order_release); - owner.start_load(); - } complete(result); } } @@ -247,11 +243,6 @@ void FlushBatch::State::start_leaves() noexcept void FlushBatch::State::Subtree::step() { - if (load_completed) { - load_completed = false; - owner.load_active.store(false, std::memory_order_release); - owner.start_load(); - } if (!status.ok()) { complete(status); return; diff --git a/lib/crowdb-tree/src/btree/flush_subtree.h b/lib/crowdb-tree/src/btree/flush_subtree.h index 8f925ac7b..bbcdc30d5 100644 --- a/lib/crowdb-tree/src/btree/flush_subtree.h +++ b/lib/crowdb-tree/src/btree/flush_subtree.h @@ -48,8 +48,6 @@ struct FlushBatch::State::Subtree void reset_work(); void request_load(); void load() noexcept; - Subtree *load_next = nullptr; - bool load_completed = false; Status collect(PublicationTransaction &transaction); [[nodiscard]] std::shared_ptr lookup(uint64_t page_id) const; diff --git a/lib/crowdb-tree/src/btree/flush_subtree_leaf.cpp b/lib/crowdb-tree/src/btree/flush_subtree_leaf.cpp index 1568cd727..247ac7ba7 100644 --- a/lib/crowdb-tree/src/btree/flush_subtree_leaf.cpp +++ b/lib/crowdb-tree/src/btree/flush_subtree_leaf.cpp @@ -172,46 +172,20 @@ void FlushBatch::State::Subtree::leaf() void FlushBatch::State::Subtree::request_load() { - auto *head = owner.load_waiters.load(std::memory_order_relaxed); - do { - load_next = head; - } while ( - !owner.load_waiters.compare_exchange_weak(head, this, std::memory_order_release, std::memory_order_relaxed)); - owner.start_load(); -} - -void FlushBatch::State::start_load() noexcept -{ - if (load_active.exchange(true, std::memory_order_acq_rel)) { - return; - } - auto *next = load_waiters.load(std::memory_order_acquire); - while (next != nullptr && !load_waiters.compare_exchange_weak(next, next->load_next, std::memory_order_acq_rel)) { - } - if (next != nullptr) { - next->load(); - return; - } - load_active.exchange(false, std::memory_order_acq_rel); - // Pair with producer publication so an arrival cannot miss the idle edge. - if (load_waiters.load(std::memory_order_acquire) != nullptr) { - start_load(); - } + load(); } void FlushBatch::State::Subtree::load() noexcept { try { if ((owner.lifecycle.load(std::memory_order_acquire) & kStopped) != 0) { - status = Status::unavailable("flush stopped before cold load"); - load_completed = true; + status = Status::unavailable("flush stopped before cold load"); schedule(); return; } if (writing) { if (!owner.options.consolidation) { - status = Status::invalid_argument("flush consolidation options are absent"); - load_completed = true; + status = Status::invalid_argument("flush consolidation options are absent"); schedule(); return; } @@ -253,8 +227,7 @@ void FlushBatch::State::Subtree::load() noexcept } status = std::move(result); ++persist_at; - writing = false; - load_completed = true; + writing = false; schedule(); }); return; @@ -267,7 +240,6 @@ void FlushBatch::State::Subtree::load() noexcept if (status.ok() && !inner) { status = Status::corruption("flush loader returned no index"); } - load_completed = true; schedule(); }); } @@ -283,19 +255,16 @@ void FlushBatch::State::Subtree::load() noexcept if (status.ok()) { loaded_leaf->bases[next_base++].resident = std::move(page); } - load_completed = true; schedule(); }); } } catch (const std::bad_alloc &) { - status = Status::resource_exhausted("flush cold load allocation failed"); - load_completed = true; + status = Status::resource_exhausted("flush cold load allocation failed"); schedule(); } catch (const std::exception &) { - status = Status::internal_error("flush cold load failed"); - load_completed = true; + status = Status::internal_error("flush cold load failed"); schedule(); } } diff --git a/lib/crowdb-tree/src/btree/logical_page.cpp b/lib/crowdb-tree/src/btree/logical_page.cpp index 5ded8122e..4250f1f10 100644 --- a/lib/crowdb-tree/src/btree/logical_page.cpp +++ b/lib/crowdb-tree/src/btree/logical_page.cpp @@ -91,8 +91,7 @@ Status KeyArenaBuilder::append(Slice key, std::shared_ptr own } Status OverlayBatch::build(uint64_t id, std::vector entries, const RunLimits &limits, - std::shared_ptr *out, - std::shared_ptr metadata) + std::shared_ptr *out, std::shared_ptr metadata) { EntryStorage owned(std::make_move_iterator(entries.begin()), std::make_move_iterator(entries.end()), EntryAllocator(std::move(metadata))); @@ -269,7 +268,7 @@ LogicalLookup PageState::lookup(Slice key, uint64_t frontier) const MemTableRangeMerge::MemTableRangeMerge(std::initializer_list> sources, PageKeyRange range, uint64_t frontier, uint64_t floor, RunLimits limits, size_t max_sources, - std::shared_ptr metadata) + std::shared_ptr metadata) : MemTableRangeMerge(std::span>(sources), std::move(range), frontier, floor, limits, max_sources, std::move(metadata)) { @@ -277,7 +276,7 @@ MemTableRangeMerge::MemTableRangeMerge(std::initializer_list> sources, PageKeyRange range, uint64_t frontier, uint64_t floor, RunLimits limits, size_t max_sources, - std::shared_ptr metadata) + std::shared_ptr metadata) : owners_(detail::ReservationAllocator>(metadata)), guards_(detail::ReservationAllocator(metadata)), range_(std::move(range)), diff --git a/lib/crowdb-tree/src/btree/logical_page.h b/lib/crowdb-tree/src/btree/logical_page.h index 2f359f4dc..a1776ba2e 100644 --- a/lib/crowdb-tree/src/btree/logical_page.h +++ b/lib/crowdb-tree/src/btree/logical_page.h @@ -46,7 +46,7 @@ struct RunLimits class KeyArenaBuilder { public: - explicit KeyArenaBuilder(RunLimits limits, std::shared_ptr metadata = {}) + explicit KeyArenaBuilder(RunLimits limits, std::shared_ptr metadata = {}) : limits_(limits), metadata_(std::move(metadata)) { @@ -55,13 +55,13 @@ class KeyArenaBuilder Status append(Slice key, std::shared_ptr owner, EntryRef *out); private: - RunLimits limits_; - std::shared_ptr metadata_; - size_t entries_ = 0; - size_t bytes_ = 0; - size_t allocated_ = 0; - size_t used_ = 0; - std::shared_ptr block_; + RunLimits limits_; + std::shared_ptr metadata_; + size_t entries_ = 0; + size_t bytes_ = 0; + size_t allocated_ = 0; + size_t used_ = 0; + std::shared_ptr block_; }; class OverlayBatch @@ -78,12 +78,11 @@ class OverlayBatch } static Status build(uint64_t id, std::vector entries, const RunLimits &limits, - std::shared_ptr *out, - std::shared_ptr metadata = {}); + std::shared_ptr *out, std::shared_ptr metadata = {}); static Status build_owned(uint64_t id, EntryStorage entries, const RunLimits &limits, std::shared_ptr *out); - explicit OverlayBatch(std::shared_ptr metadata = {}) + explicit OverlayBatch(std::shared_ptr metadata = {}) : entries_(EntryAllocator(std::move(metadata))) { } @@ -133,7 +132,7 @@ class OverlayBatch class OverlayBuilder { public: - explicit OverlayBuilder(RunLimits limits, std::shared_ptr metadata = {}) + explicit OverlayBuilder(RunLimits limits, std::shared_ptr metadata = {}) : limits_(limits), metadata_(std::move(metadata)), arena_(limits, metadata_), @@ -146,11 +145,11 @@ class OverlayBuilder Status finish(uint64_t id, std::shared_ptr *out); private: - RunLimits limits_; - std::shared_ptr metadata_; - KeyArenaBuilder arena_; - size_t key_bytes_ = 0; - EntryStorage entries_; + RunLimits limits_; + std::shared_ptr metadata_; + KeyArenaBuilder arena_; + size_t key_bytes_ = 0; + EntryStorage entries_; }; struct BaseRef @@ -228,13 +227,13 @@ struct PageState return overlays.empty() && bases.empty(); } - explicit PageState(std::shared_ptr metadata = {}) + explicit PageState(std::shared_ptr metadata = {}) : bases(BaseAllocator(metadata)), overlays(OverlayAllocator(std::move(metadata))) { } - PageState(const PageState &source, std::shared_ptr metadata = {}) + PageState(const PageState &source, std::shared_ptr metadata = {}) : range(source.range), route_generation(source.route_generation), content_id(source.content_id), @@ -264,10 +263,10 @@ class MemTableRangeMerge public: MemTableRangeMerge(std::initializer_list> sources, PageKeyRange range, uint64_t frontier, uint64_t floor, RunLimits limits, size_t max_sources, - std::shared_ptr metadata = {}); + std::shared_ptr metadata = {}); MemTableRangeMerge(std::span> sources, PageKeyRange range, uint64_t frontier, uint64_t floor, RunLimits limits, size_t max_sources, - std::shared_ptr metadata = {}); + std::shared_ptr metadata = {}); MemTableRangeMerge(const MemTableRangeMerge &) = delete; MemTableRangeMerge &operator=(const MemTableRangeMerge &) = delete; @@ -295,7 +294,7 @@ class MemTableRangeMerge std::vector> guards_; PageKeyRange range_; RunLimits limits_; - std::shared_ptr metadata_; + std::shared_ptr metadata_; KeyArenaBuilder arena_; std::vector> cursors_; std::vector> sources_; @@ -317,7 +316,7 @@ class LogicalPageCursor public: [[nodiscard]] static std::optional allocation_bound(size_t max_sources) noexcept; LogicalPageCursor(std::shared_ptr page, PageKeyRange range, uint64_t frontier, size_t max_sources, - bool reverse = false, std::shared_ptr metadata = {}, + bool reverse = false, std::shared_ptr metadata = {}, std::optional start = {}); ~LogicalPageCursor(); diff --git a/lib/crowdb-tree/src/btree/logical_page_cursor.cpp b/lib/crowdb-tree/src/btree/logical_page_cursor.cpp index 1fca6efb2..d4f5a76bc 100644 --- a/lib/crowdb-tree/src/btree/logical_page_cursor.cpp +++ b/lib/crowdb-tree/src/btree/logical_page_cursor.cpp @@ -15,9 +15,8 @@ std::optional LogicalPageCursor::allocation_bound(size_t max_sources) no LogicalPageCursor::~LogicalPageCursor() = default; LogicalPageCursor::LogicalPageCursor(std::shared_ptr page, PageKeyRange range, uint64_t frontier, - size_t max_sources, bool reverse, - std::shared_ptr metadata, - std::optional start) + size_t max_sources, bool reverse, std::shared_ptr metadata, + std::optional start) : page_(std::move(page)), range_(std::move(range)), frontier_(frontier), diff --git a/lib/crowdb-tree/src/btree/owned_key.cpp b/lib/crowdb-tree/src/btree/owned_key.cpp index f6201f0c7..23a0d5c34 100644 --- a/lib/crowdb-tree/src/btree/owned_key.cpp +++ b/lib/crowdb-tree/src/btree/owned_key.cpp @@ -15,7 +15,7 @@ struct OwnedKey::Storage std::vector bytes; - Storage(Slice key, const std::shared_ptr &metadata) : bytes(Allocator(metadata)) + Storage(Slice key, const std::shared_ptr &metadata) : bytes(Allocator(metadata)) { bytes.assign(key.data(), key.data() + key.size()); record_work(metadata, TreeWorkCounter::KeyCopyBytes, key.size()); @@ -27,7 +27,7 @@ struct OwnedKey::Storage } }; -OwnedKey::OwnedKey(Slice key, const std::shared_ptr &metadata) +OwnedKey::OwnedKey(Slice key, const std::shared_ptr &metadata) { if (key.size() <= inline_capacity) { if (!key.empty()) { diff --git a/lib/crowdb-tree/src/btree/owned_key.h b/lib/crowdb-tree/src/btree/owned_key.h index c759dc95f..a7b0ff148 100644 --- a/lib/crowdb-tree/src/btree/owned_key.h +++ b/lib/crowdb-tree/src/btree/owned_key.h @@ -22,7 +22,7 @@ class OwnedKey public: OwnedKey() = default; - OwnedKey(Slice key, const std::shared_ptr &metadata = {}); + OwnedKey(Slice key, const std::shared_ptr &metadata = {}); OwnedKey(const std::string &key) : OwnedKey(Slice(key)) { diff --git a/lib/crowdb-tree/src/btree/page_cache.cpp b/lib/crowdb-tree/src/btree/page_cache.cpp index b6a1baa38..1baaf6f6b 100644 --- a/lib/crowdb-tree/src/btree/page_cache.cpp +++ b/lib/crowdb-tree/src/btree/page_cache.cpp @@ -13,8 +13,8 @@ class SharedPageCache::Binding final : public PageLoader { public: Binding(std::shared_ptr state, std::shared_ptr upstream, - std::shared_ptr metadata, - std::shared_ptr reservation = {}, PageLoadPriority priority = PageLoadPriority::Read) + std::shared_ptr metadata, std::shared_ptr reservation = {}, + PageLoadPriority priority = PageLoadPriority::Read) : state_(std::move(state)), upstream_(std::move(upstream)), metadata_(std::move(metadata)), @@ -35,6 +35,10 @@ class SharedPageCache::Binding final : public PageLoader } auto *request = claim(); if (request == nullptr) { + if (priority_ == PageLoadPriority::Maintenance) { + upstream_->load_leaf(std::move(base), std::move(complete)); + return; + } complete_page_load(complete, Status::unavailable(), std::shared_ptr{}); return; } @@ -55,6 +59,10 @@ class SharedPageCache::Binding final : public PageLoader } auto *request = claim(); if (request == nullptr) { + if (priority_ == PageLoadPriority::Maintenance) { + upstream_->load_inner(std::move(backing), std::move(complete)); + return; + } complete_page_load(complete, Status::unavailable(), std::shared_ptr{}); return; } @@ -107,14 +115,14 @@ class SharedPageCache::Binding final : public PageLoader return request; } - std::shared_ptr state_; - std::shared_ptr upstream_; - std::shared_ptr metadata_; - std::shared_ptr reservation_; - PageLoadPriority priority_; + std::shared_ptr state_; + std::shared_ptr upstream_; + std::shared_ptr metadata_; + std::shared_ptr reservation_; + PageLoadPriority priority_; }; -SharedPageCache::SharedPageCache(PageCacheLimits limits, std::shared_ptr metadata, +SharedPageCache::SharedPageCache(PageCacheLimits limits, std::shared_ptr metadata, std::shared_ptr resources, detail::FlushRuntime *runtime, std::shared_ptr lane) { @@ -143,9 +151,9 @@ std::optional SharedPageCache::allocation_bound(PageCacheLimits limits) .value(); } -std::shared_ptr SharedPageCache::bind(std::shared_ptr upstream, - std::shared_ptr metadata, - PageLoadPriority priority) const +std::shared_ptr SharedPageCache::bind(std::shared_ptr upstream, + std::shared_ptr metadata, + PageLoadPriority priority) const { if (!upstream) { throw std::invalid_argument("page cache binding requires an upstream loader"); @@ -186,8 +194,8 @@ size_t SharedPageCache::retained_page_limit() const noexcept return state_->limits.max_pages; } -Status SharedPageCache::bind_reserved(std::shared_ptr upstream, - const std::shared_ptr &metadata, +Status SharedPageCache::bind_reserved(std::shared_ptr upstream, + const std::shared_ptr &metadata, std::shared_ptr *out, PageLoadPriority priority) const { if (!upstream || out == nullptr) { @@ -258,7 +266,7 @@ void SharedPageCache::close_and_drain() noexcept } } -SharedPageCache::State::State(PageCacheLimits bounds, const std::shared_ptr &metadata, +SharedPageCache::State::State(PageCacheLimits bounds, const std::shared_ptr &metadata, std::shared_ptr resources, detail::FlushRuntime *runtime, std::shared_ptr lane) : limits(bounds), diff --git a/lib/crowdb-tree/src/btree/page_cache.h b/lib/crowdb-tree/src/btree/page_cache.h index 762cf8915..e8b38092f 100644 --- a/lib/crowdb-tree/src/btree/page_cache.h +++ b/lib/crowdb-tree/src/btree/page_cache.h @@ -61,7 +61,7 @@ class SharedPageCache public: // Residency metrics require a host-owned runtime which outlives cache // owners and is drained after their release. No runtime ownership is held. - explicit SharedPageCache(PageCacheLimits limits = {}, std::shared_ptr metadata = {}, + explicit SharedPageCache(PageCacheLimits limits = {}, std::shared_ptr metadata = {}, std::shared_ptr resources = {}, detail::FlushRuntime *runtime = nullptr, std::shared_ptr lane = {}); @@ -71,14 +71,14 @@ class SharedPageCache // The miss uses the first caller's upstream, including its held load/I/O // credit. Joined callers never acquire a second I/O permit for that base. - [[nodiscard]] std::shared_ptr bind(std::shared_ptr upstream, - std::shared_ptr metadata = {}, + [[nodiscard]] std::shared_ptr bind(std::shared_ptr upstream, + std::shared_ptr metadata = {}, PageLoadPriority priority = PageLoadPriority::Read) const; // Reserve one request slot before closing flush sources. This binding has // one serial outstanding load and reuses its slot across routing steps. - [[nodiscard]] Status bind_reserved(std::shared_ptr upstream, - const std::shared_ptr &metadata, - std::shared_ptr *out, + [[nodiscard]] Status bind_reserved(std::shared_ptr upstream, + const std::shared_ptr &metadata, + std::shared_ptr *out, PageLoadPriority priority = PageLoadPriority::Read) const; [[nodiscard]] static size_t binding_allocation_bound() noexcept; // Additional per-load progress credit when residency metrics are enabled. diff --git a/lib/crowdb-tree/src/btree/page_cache_internal.h b/lib/crowdb-tree/src/btree/page_cache_internal.h index d783002b0..0e90ee331 100644 --- a/lib/crowdb-tree/src/btree/page_cache_internal.h +++ b/lib/crowdb-tree/src/btree/page_cache_internal.h @@ -26,20 +26,20 @@ struct SharedPageCache::State : std::enable_shared_from_this occupied{false}; - Command command; - Request *next = nullptr; - PageLoadPriority priority = PageLoadPriority::Read; - std::shared_ptr owner; - std::shared_ptr reservation; - std::shared_ptr upstream; - std::shared_ptr metadata; - BaseRef base; - Key key; - PageLoader::LeafComplete leaf_done; - PageLoader::InnerComplete inner_done; - uint64_t registered_at_ns = 0; - TreeWaitSpan shared_wait; + std::atomic occupied{false}; + Command command; + Request *next = nullptr; + PageLoadPriority priority = PageLoadPriority::Read; + std::shared_ptr owner; + std::shared_ptr reservation; + std::shared_ptr upstream; + std::shared_ptr metadata; + BaseRef base; + Key key; + PageLoader::LeafComplete leaf_done; + PageLoader::InnerComplete inner_done; + uint64_t registered_at_ns = 0; + TreeWaitSpan shared_wait; }; struct Reservation @@ -74,7 +74,7 @@ struct SharedPageCache::State : std::enable_shared_from_this frame; }; - State(PageCacheLimits bounds, const std::shared_ptr &metadata, + State(PageCacheLimits bounds, const std::shared_ptr &metadata, std::shared_ptr resources, detail::FlushRuntime *runtime, std::shared_ptr lane); static Key make_key(Kind kind, const BaseRef &base); diff --git a/lib/crowdb-tree/src/btree/page_cache_load.cpp b/lib/crowdb-tree/src/btree/page_cache_load.cpp index 20f78b673..81cee0626 100644 --- a/lib/crowdb-tree/src/btree/page_cache_load.cpp +++ b/lib/crowdb-tree/src/btree/page_cache_load.cpp @@ -25,7 +25,7 @@ void SharedPageCache::State::process(Request *request, bool queued) deliver(request, Status::Ok(), entry.leaf, entry.inner, entry.max_slot, entry.frame); } else { - request->shared_wait.begin(request->metadata ? request->metadata->account()->work_metrics() : nullptr, + request->shared_wait.begin(request->metadata ? request->metadata->retirement_metrics() : nullptr, TreeWaitClass::SharedCache, request->registered_at_ns); request->next = entry.waiters; entry.waiters = request; diff --git a/lib/crowdb-tree/src/btree/page_cache_residency.cpp b/lib/crowdb-tree/src/btree/page_cache_residency.cpp index 4ddcbde5e..c177931ea 100644 --- a/lib/crowdb-tree/src/btree/page_cache_residency.cpp +++ b/lib/crowdb-tree/src/btree/page_cache_residency.cpp @@ -12,7 +12,7 @@ namespace crowdb::tree { namespace { -thread_local std::shared_ptr node_budget; +thread_local std::shared_ptr node_budget; struct NodeHeader { @@ -25,7 +25,7 @@ size_t header_bytes(size_t alignment) } } // namespace -PageCacheNodeResource::Scope::Scope(std::shared_ptr budget) +PageCacheNodeResource::Scope::Scope(std::shared_ptr budget) : previous_(std::exchange(node_budget, std::move(budget))) { } @@ -68,9 +68,9 @@ PageCacheResidency::State::State(std::shared_ptr resources) { } -PageCacheResidency::PageCacheResidency(std::shared_ptr resources, - const std::shared_ptr &metadata, - detail::FlushRuntime *runtime, std::shared_ptr lane) +PageCacheResidency::PageCacheResidency(std::shared_ptr resources, + const std::shared_ptr &metadata, detail::FlushRuntime *runtime, + std::shared_ptr lane) : state_(detail::reserved_shared(metadata, std::move(resources))) { if (runtime == nullptr || !lane) { @@ -89,8 +89,8 @@ PageCacheResidency::~PageCacheResidency() } std::shared_ptr -PageCacheResidency::prepare(std::span pages, - const std::shared_ptr &metadata) const +PageCacheResidency::prepare(std::span pages, + const std::shared_ptr &metadata) const { auto update = detail::reserved_shared(metadata, state_); PageCacheNodeResource::Scope scope(metadata); @@ -116,8 +116,7 @@ void PageCacheResidency::publish(std::shared_ptr update) noexcept } std::shared_ptr -PageCacheResidency::prepare_ready(const PageCacheKey &key, - const std::shared_ptr &metadata) const +PageCacheResidency::prepare_ready(const PageCacheKey &key, const std::shared_ptr &metadata) const { auto result = detail::reserved_shared(metadata, state_); PageCacheNodeResource::Scope scope(metadata); diff --git a/lib/crowdb-tree/src/btree/page_cache_residency.h b/lib/crowdb-tree/src/btree/page_cache_residency.h index 351bb976b..a3a456fa8 100644 --- a/lib/crowdb-tree/src/btree/page_cache_residency.h +++ b/lib/crowdb-tree/src/btree/page_cache_residency.h @@ -22,25 +22,24 @@ class PageCacheResidency public: class Update; class Ready; - explicit PageCacheResidency(std::shared_ptr resources, - const std::shared_ptr &metadata = {}, - detail::FlushRuntime *runtime = nullptr, - std::shared_ptr lane = {}); + explicit PageCacheResidency(std::shared_ptr resources, + const std::shared_ptr &metadata = {}, + detail::FlushRuntime *runtime = nullptr, + std::shared_ptr lane = {}); ~PageCacheResidency(); - [[nodiscard]] std::shared_ptr prepare(std::span pages, - const std::shared_ptr &metadata) const; - [[nodiscard]] std::shared_ptr - prepare_initial(const MappingView &mapping, const std::shared_ptr &metadata) const; - [[nodiscard]] std::shared_ptr - prepare_replacement(const MappingView &mapping, - const std::shared_ptr &metadata) const; - [[nodiscard]] static size_t initial_allocation_bound(const MappingView &mapping) noexcept; - static void publish(std::shared_ptr update) noexcept; - [[nodiscard]] std::shared_ptr - prepare_ready(const PageCacheKey &key, const std::shared_ptr &metadata) const; - static void ready(const std::shared_ptr &ready) noexcept; - static void evict(const std::shared_ptr &ready) noexcept; - void clear() noexcept; + [[nodiscard]] std::shared_ptr prepare(std::span pages, + const std::shared_ptr &metadata) const; + [[nodiscard]] std::shared_ptr prepare_initial(const MappingView &mapping, + const std::shared_ptr &metadata) const; + [[nodiscard]] std::shared_ptr prepare_replacement(const MappingView &mapping, + const std::shared_ptr &metadata) const; + [[nodiscard]] static size_t initial_allocation_bound(const MappingView &mapping) noexcept; + static void publish(std::shared_ptr update) noexcept; + [[nodiscard]] std::shared_ptr prepare_ready(const PageCacheKey &key, + const std::shared_ptr &metadata) const; + static void ready(const std::shared_ptr &ready) noexcept; + static void evict(const std::shared_ptr &ready) noexcept; + void clear() noexcept; // Host-thread boundary: stop index producers, drain only this progress // registration and detach from the runtime. Late pin releases are safe. void close_and_drain() noexcept; diff --git a/lib/crowdb-tree/src/btree/page_cache_residency_internal.h b/lib/crowdb-tree/src/btree/page_cache_residency_internal.h index 337feb07a..7c0958cf9 100644 --- a/lib/crowdb-tree/src/btree/page_cache_residency_internal.h +++ b/lib/crowdb-tree/src/btree/page_cache_residency_internal.h @@ -16,13 +16,13 @@ class PageCacheNodeResource final : public std::pmr::memory_resource class Scope { public: - explicit Scope(std::shared_ptr budget); + explicit Scope(std::shared_ptr budget); ~Scope(); Scope(const Scope &) = delete; Scope &operator=(const Scope &) = delete; private: - std::shared_ptr previous_; + std::shared_ptr previous_; }; private: @@ -47,7 +47,7 @@ struct PageCacheResidency::State : std::enable_shared_from_this struct Page { - explicit Page(const std::shared_ptr &metadata) + explicit Page(const std::shared_ptr &metadata) : links(detail::ReservationAllocator(metadata)) { } @@ -75,7 +75,7 @@ struct PageCacheResidency::State : std::enable_shared_from_this explicit State(std::shared_ptr resources); static void prepare_page(Update &update, uint64_t pid, const std::shared_ptr &replacement, - const std::shared_ptr &metadata); + const std::shared_ptr &metadata); void enqueue(Command *command) noexcept; bool enter() noexcept; void leave() noexcept; diff --git a/lib/crowdb-tree/src/btree/page_cache_residency_prepare.cpp b/lib/crowdb-tree/src/btree/page_cache_residency_prepare.cpp index 15050fc20..b6eb59f15 100644 --- a/lib/crowdb-tree/src/btree/page_cache_residency_prepare.cpp +++ b/lib/crowdb-tree/src/btree/page_cache_residency_prepare.cpp @@ -7,8 +7,8 @@ namespace crowdb::tree { void PageCacheResidency::State::prepare_page(Update &update, uint64_t pid, - const std::shared_ptr &replacement, - const std::shared_ptr &metadata) + const std::shared_ptr &replacement, + const std::shared_ptr &metadata) { std::shared_ptr page; if (replacement) { @@ -38,16 +38,15 @@ void PageCacheResidency::State::prepare_page(Update &update, uint64_t pid, } std::shared_ptr -PageCacheResidency::prepare_initial(const MappingView &mapping, - const std::shared_ptr &metadata) const +PageCacheResidency::prepare_initial(const MappingView &mapping, const std::shared_ptr &metadata) const { auto update = detail::reserved_shared(metadata, state_); PageCacheNodeResource::Scope scope(metadata); struct Context { - std::shared_ptr update; - const std::shared_ptr &metadata; + std::shared_ptr update; + const std::shared_ptr &metadata; } context{.update = update, .metadata = metadata}; mapping.visit(&context, [](void *argument, uint64_t pid, const auto &page) { @@ -62,8 +61,8 @@ PageCacheResidency::prepare_initial(const MappingView } std::shared_ptr -PageCacheResidency::prepare_replacement(const MappingView &mapping, - const std::shared_ptr &metadata) const +PageCacheResidency::prepare_replacement(const MappingView &mapping, + const std::shared_ptr &metadata) const { auto update = prepare_initial(mapping, metadata); update->reset = true; diff --git a/lib/crowdb-tree/src/btree/page_encode.h b/lib/crowdb-tree/src/btree/page_encode.h index 3c69da3cd..02ede8a17 100644 --- a/lib/crowdb-tree/src/btree/page_encode.h +++ b/lib/crowdb-tree/src/btree/page_encode.h @@ -11,15 +11,15 @@ namespace crowdb::tree { struct PageEncodeLimits { - uint32_t frame_bytes = 64 * 1024; - size_t inline_value_bytes = 8 * 1024; - size_t max_sources = 64; - size_t max_entries = 4096; - size_t max_work_entries = 65536; - size_t max_metadata_bytes = 1024 * 1024; - std::shared_ptr metadata = nullptr; - std::shared_ptr work_metrics = nullptr; - TreeWorkCause cause = TreeWorkCause::Other; + uint32_t frame_bytes = 64 * 1024; + size_t inline_value_bytes = 8 * 1024; + size_t max_sources = 64; + size_t max_entries = 4096; + size_t max_work_entries = 65536; + size_t max_metadata_bytes = 1024 * 1024; + std::shared_ptr metadata = nullptr; + std::shared_ptr work_metrics = nullptr; + TreeWorkCause cause = TreeWorkCause::Other; }; struct InnerEncodeFences diff --git a/lib/crowdb-tree/src/btree/page_load.cpp b/lib/crowdb-tree/src/btree/page_load.cpp index 6340c7ca3..4c8fa7e94 100644 --- a/lib/crowdb-tree/src/btree/page_load.cpp +++ b/lib/crowdb-tree/src/btree/page_load.cpp @@ -32,16 +32,16 @@ struct LeafLoad : PageLoadProgress, std::enable_shared_from_this { } - std::shared_ptr metadata; - std::shared_ptr store; - std::shared_ptr runtime; - std::shared_ptr lane; - PageLoadLimits limits; - BaseRef base; - AlignedLoadBytes blob; - PageLoader::LeafComplete complete; - std::shared_ptr queued_owner; - std::atomic terminal{false}; + std::shared_ptr metadata; + std::shared_ptr store; + std::shared_ptr runtime; + std::shared_ptr lane; + PageLoadLimits limits; + BaseRef base; + AlignedLoadBytes blob; + PageLoader::LeafComplete complete; + std::shared_ptr queued_owner; + std::atomic terminal{false}; void finish(Status status) noexcept { @@ -120,12 +120,11 @@ AsyncStorePageLoader::AsyncStorePageLoader(std::shared_ptr std::shared_ptr lane, PageLoadLimits limits, OverflowBackingResolver overflow, PageLoadBudget budget) { - if (static_cast(budget.bytes) != static_cast(budget.io) || - (budget.bytes && - ((budget.bytes->category() != detail::BudgetClass::kRouting && - budget.bytes->category() != detail::BudgetClass::kReclaim) || - budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { - throw std::invalid_argument("page load needs routing/reclaim and I/O credit from the same account"); + const auto prepaid = budget.reservation(); + if ((prepaid && prepaid->category() == detail::BudgetClass::kIo) || + (budget.io && (!budget.bytes || budget.io->category() != detail::BudgetClass::kIo || + budget.bytes->domain() != budget.io->domain()))) { + throw std::invalid_argument("page load needs memory and I/O credit from the same runtime"); } state_ = detail::reserved_shared(budget.bytes); state_->store = std::move(store); diff --git a/lib/crowdb-tree/src/btree/page_load.h b/lib/crowdb-tree/src/btree/page_load.h index 55fdbf445..dec5f9818 100644 --- a/lib/crowdb-tree/src/btree/page_load.h +++ b/lib/crowdb-tree/src/btree/page_load.h @@ -51,12 +51,18 @@ struct PageLoadLimits // directly; it never waits for a second host reservation while holding it. struct PageLoadBudget { - std::shared_ptr bytes; - std::shared_ptr io; - std::shared_ptr work_metrics = nullptr; - std::shared_ptr backend = nullptr; - std::shared_ptr cpu_progress = nullptr; - TreeWorkCause cause = TreeWorkCause::Other; + std::shared_ptr bytes; + std::shared_ptr io = nullptr; + std::shared_ptr account = nullptr; + std::shared_ptr work_metrics = nullptr; + std::shared_ptr backend = nullptr; + std::shared_ptr cpu_progress = nullptr; + TreeWorkCause cause = TreeWorkCause::Other; + + [[nodiscard]] std::shared_ptr reservation() const noexcept + { + return std::dynamic_pointer_cast(bytes); + } }; // The resolver must be bound to the caller's captured backing/mapping view and diff --git a/lib/crowdb-tree/src/btree/page_load_frame.cpp b/lib/crowdb-tree/src/btree/page_load_frame.cpp index e546ba19a..b9665a9ec 100644 --- a/lib/crowdb-tree/src/btree/page_load_frame.cpp +++ b/lib/crowdb-tree/src/btree/page_load_frame.cpp @@ -8,7 +8,7 @@ namespace crowdb::tree { Status decode_backing_frame(std::span blob, const BackingRef &backing, const PageLoadLimits &limits, - const std::shared_ptr &metadata, LoadBytes *out) + const std::shared_ptr &metadata, LoadBytes *out) { if (!backing.readable()) { return Status::corruption("backing has neither an extent nor an owned resident frame"); diff --git a/lib/crowdb-tree/src/btree/page_load_inner.cpp b/lib/crowdb-tree/src/btree/page_load_inner.cpp index 3bc7d7d09..54088c420 100644 --- a/lib/crowdb-tree/src/btree/page_load_inner.cpp +++ b/lib/crowdb-tree/src/btree/page_load_inner.cpp @@ -16,8 +16,7 @@ void PageLoader::load_inner(std::shared_ptr /*backing*/, Inner namespace { Status read_inner_proof(const uint8_t *frame, uint32_t bytes, const PageLoadLimits &limits, - const std::shared_ptr &metadata, size_t key_bytes, - InnerFrameProof *proof) + const std::shared_ptr &metadata, size_t key_bytes, InnerFrameProof *proof) { proof->self_page_id = frame_u64(frame, fh::kSelfpage_id); proof->frame_bytes = bytes; @@ -41,8 +40,7 @@ Status read_inner_proof(const uint8_t *frame, uint32_t bytes, const PageLoadLimi } Status decode_inner(const AlignedLoadBytes &blob, const BackingRef &backing, const PageLoadLimits &limits, - const std::shared_ptr &metadata, - std::shared_ptr *out) + const std::shared_ptr &metadata, std::shared_ptr *out) { LoadBytes frame{LoadBytes::allocator_type(metadata)}; auto status = decode_backing_frame({blob.data(), blob.size()}, backing, limits, metadata, &frame); @@ -96,16 +94,16 @@ struct InnerLoad : PageLoadProgress, std::enable_shared_from_this { } - std::shared_ptr metadata; - std::shared_ptr store; - std::shared_ptr runtime; - std::shared_ptr lane; - PageLoadLimits limits; - std::shared_ptr backing; - AlignedLoadBytes blob; - PageLoader::InnerComplete complete; - std::shared_ptr queued_owner; - std::atomic terminal{false}; + std::shared_ptr metadata; + std::shared_ptr store; + std::shared_ptr runtime; + std::shared_ptr lane; + PageLoadLimits limits; + std::shared_ptr backing; + AlignedLoadBytes blob; + PageLoader::InnerComplete complete; + std::shared_ptr queued_owner; + std::atomic terminal{false}; void finish(Status status) noexcept { diff --git a/lib/crowdb-tree/src/btree/page_load_internal.h b/lib/crowdb-tree/src/btree/page_load_internal.h index 239257fb0..319c2a993 100644 --- a/lib/crowdb-tree/src/btree/page_load_internal.h +++ b/lib/crowdb-tree/src/btree/page_load_internal.h @@ -34,7 +34,7 @@ inline bool chunk_load_allowed(const BackingRef &backing, const PageLoadLimits & using LoadBytes = detail::ReservedVector; Status decode_backing_frame(std::span blob, const BackingRef &backing, const PageLoadLimits &limits, - const std::shared_ptr &metadata, LoadBytes *out); + const std::shared_ptr &metadata, LoadBytes *out); [[nodiscard]] std::optional decoded_leaf_allocation_bound(const PageLoadLimits &limits) noexcept; [[nodiscard]] std::optional overflow_read_allocation_bound(const PageLoadLimits &limits, size_t workers, diff --git a/lib/crowdb-tree/src/btree/page_load_leaf.cpp b/lib/crowdb-tree/src/btree/page_load_leaf.cpp index 66ded7faf..71efcf203 100644 --- a/lib/crowdb-tree/src/btree/page_load_leaf.cpp +++ b/lib/crowdb-tree/src/btree/page_load_leaf.cpp @@ -66,7 +66,7 @@ Status check_cells(const LeafFrameView &view, const std::optional &max return Status::Ok(); } -Status fold_leaf(LoadBytes &frame, const std::shared_ptr &metadata) +Status fold_leaf(LoadBytes &frame, const std::shared_ptr &metadata) { const LeafFrameView view(frame.data(), static_cast(frame.size())); if (view.delta_count() == 0) { @@ -134,8 +134,7 @@ std::optional decoded_leaf_allocation_bound(const PageLoadLimits &limits } Status decode_loaded_leaf(std::span blob, const BaseRef &base, const PageLoadLimits &limits, - const std::shared_ptr &metadata, - std::shared_ptr *out) + const std::shared_ptr &metadata, std::shared_ptr *out) { LoadBytes frame{LoadBytes::allocator_type(metadata)}; auto status = decode_backing_frame(blob, *base.backing, limits, metadata, &frame); diff --git a/lib/crowdb-tree/src/btree/page_load_leaf.h b/lib/crowdb-tree/src/btree/page_load_leaf.h index b51407061..3febeb178 100644 --- a/lib/crowdb-tree/src/btree/page_load_leaf.h +++ b/lib/crowdb-tree/src/btree/page_load_leaf.h @@ -11,6 +11,5 @@ namespace crowdb::tree // Fold the captured frame at its proven slot bound, preserving tombstones and // the backing's overflow protection. This does not provide historical reads. Status decode_loaded_leaf(std::span blob, const BaseRef &base, const PageLoadLimits &limits, - const std::shared_ptr &metadata, - std::shared_ptr *out); + const std::shared_ptr &metadata, std::shared_ptr *out); } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/btree/page_load_lifetime.h b/lib/crowdb-tree/src/btree/page_load_lifetime.h index 2fff02712..d5e1cc353 100644 --- a/lib/crowdb-tree/src/btree/page_load_lifetime.h +++ b/lib/crowdb-tree/src/btree/page_load_lifetime.h @@ -10,10 +10,22 @@ namespace crowdb::tree { +inline std::shared_ptr page_load_io_credit(const PageLoadBudget &budget) +{ + if (budget.io || !budget.account) { + return budget.io; + } + auto io = detail::ResourceBudget::try_reserve(budget.account, detail::BudgetClass::kIo, 1); + if (!io) { + throw std::bad_alloc(); + } + return detail::reserved_shared(budget.bytes, std::move(*io)); +} + struct PageLoadProgress { PageLoadProgress(const PageLoadBudget &budget, uint64_t started_at_ns) - : io_credit(budget.io, 1), + : io_credit(page_load_io_credit(budget), 1), metrics(budget.work_metrics, TreeStage::PageLoadPipeline, started_at_ns), backend_budget(budget.backend), cpu_progress(budget.cpu_progress), @@ -81,7 +93,7 @@ inline void record_page_load_terminal(const PageLoadBudget &budget, uint64_t sta struct PageReadTicket { PageReadTicket(std::shared_ptr lifetime, void *operation, AsyncCompleteFn callback, - const std::shared_ptr &metadata, size_t bytes) + const std::shared_ptr &metadata, size_t bytes) : resources(metadata, TreeResourceMetric::ReadInflightBytes, bytes), owner(std::move(lifetime)), context(operation), diff --git a/lib/crowdb-tree/src/btree/page_load_overflow.cpp b/lib/crowdb-tree/src/btree/page_load_overflow.cpp index 898bea0ea..6069d30bc 100644 --- a/lib/crowdb-tree/src/btree/page_load_overflow.cpp +++ b/lib/crowdb-tree/src/btree/page_load_overflow.cpp @@ -16,7 +16,7 @@ namespace { struct LoadedValue { - LoadedValue(const std::shared_ptr &metadata, size_t bytes) + LoadedValue(const std::shared_ptr &metadata, size_t bytes) : payload(metadata, bytes > buffer::kInlineCap ? bytes : 0, AllocationKind::kPayload), value(buffer::alloc(bytes)) { @@ -36,7 +36,7 @@ struct OverflowLoad : PageLoadProgress, std::enable_shared_from_this metadata; + std::shared_ptr metadata; std::shared_ptr store; std::shared_ptr runtime; std::shared_ptr lane; diff --git a/lib/crowdb-tree/src/btree/page_writeback.cpp b/lib/crowdb-tree/src/btree/page_writeback.cpp index c400de729..ae3e81140 100644 --- a/lib/crowdb-tree/src/btree/page_writeback.cpp +++ b/lib/crowdb-tree/src/btree/page_writeback.cpp @@ -52,13 +52,13 @@ PageWriteOperation::State::State(PageWriteInput source, PageWriteLimits bounds, limits(bounds), budget(std::move(credits)), io_credit(budget.io, 1), - io_resources(budget.bytes, TreeResourceMetric::WriteInflightBytes), + io_resources(budget.reservation(), TreeResourceMetric::WriteInflightBytes), blob(padded_blob_bound(limits) + limits.buffer_alignment - 1, - detail::ReservationAllocator(budget.bytes)), + detail::ReservationAllocator(budget.reservation())), scratch(durable_encode_scratch_bytes(limits.compression), - detail::ReservationAllocator(budget.bytes)), - backing(detail::reserved_shared(budget.bytes)), - protection(detail::reserved_shared(budget.bytes)) + detail::ReservationAllocator(budget.reservation())), + backing(detail::reserved_shared(budget.reservation())), + protection(detail::reserved_shared(budget.reservation())) { void *data = blob.data(); auto capacity = blob.size(); @@ -70,10 +70,10 @@ PageWriteOperation::State::State(PageWriteInput source, PageWriteLimits bounds, backing->owner = protection; if (limits.chunk_backed) { const auto bound = detail::ChunkFixedPackStore::write_allocation_bound(padded_blob_bound(limits)); - if (!bound || !budget.bytes) { + if (!bound || !budget.reservation()) { throw std::bad_alloc(); } - chunk_credit = detail::AllocationReservation::create(budget.bytes, *bound); + chunk_credit = detail::AllocationReservation::create(budget.reservation(), *bound); } } @@ -85,11 +85,11 @@ Status PageWriteOperation::create(std::shared_ptr store, std::sh if (!store || !runtime || !lane || !reserve || out == nullptr || !allocation_bound(limits) || !input.owner || input.frame.size() != input.info.bytes || input.frame.size() > limits.max_frame_bytes || input.frame.size() < kFrameHeaderSize + kFrameTrailerSize || - static_cast(budget.bytes) != static_cast(budget.io) || - (budget.bytes && - ((budget.bytes->category() != detail::BudgetClass::kRouting && - budget.bytes->category() != detail::BudgetClass::kReclaim) || - budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + static_cast(budget.reservation()) != static_cast(budget.io) || + (budget.reservation() && ((budget.reservation()->category() != detail::BudgetClass::kRouting && + budget.reservation()->category() != detail::BudgetClass::kReclaim) || + budget.io->category() != detail::BudgetClass::kIo || + budget.reservation()->account() != budget.io->account()))) { return Status::invalid_argument("page write requires owned input and matching byte/I/O credit"); } if (limits.chunk_backed ? dynamic_cast(store.get()) == nullptr @@ -97,12 +97,12 @@ Status PageWriteOperation::create(std::shared_ptr store, std::sh return Status::not_supported("backend write needs checkpoint/seal before it is independently reloadable"); } try { - auto state = detail::reserved_shared(budget.bytes, std::move(input), limits, budget); + auto state = detail::reserved_shared(budget.reservation(), std::move(input), limits, budget); state->store = std::move(store); state->runtime = std::move(runtime); state->lane = std::move(lane); state->reserve = std::move(reserve); - *out = detail::reserved_construct(budget.bytes, [&](auto *storage) { + *out = detail::reserved_construct(budget.reservation(), [&](auto *storage) { ::new (static_cast(storage)) PageWriteOperation(std::move(state)); }); } diff --git a/lib/crowdb-tree/src/btree/published_inspection.cpp b/lib/crowdb-tree/src/btree/published_inspection.cpp index f23a462c3..5da946852 100644 --- a/lib/crowdb-tree/src/btree/published_inspection.cpp +++ b/lib/crowdb-tree/src/btree/published_inspection.cpp @@ -42,16 +42,6 @@ struct Inspection core_read_detail::ReadPreparation prepared; std::shared_ptr view; - ~Inspection() - { - if (prepared.credit) { - prepared.credit->seal(); - } - if (prepared.io) { - prepared.io->seal(); - } - } - Status load(uint64_t pid, std::optional base = {}) { const auto old = view->mapping.get(pid); @@ -141,7 +131,7 @@ struct Inspection struct CopiedFrame { - CopiedFrame(const std::shared_ptr &credit, size_t capacity) + CopiedFrame(const std::shared_ptr &credit, size_t capacity) : bytes(credit, capacity, AllocationKind::kFrame) { } @@ -166,20 +156,8 @@ try { const auto input = core.capture(); const auto shape = input->mapping.copy_bounds(); frames.page_metadata_bytes = shape.max_leaf_bytes; - const auto copies = (3 * frames.max_depth) + frames.encode.max_sources; - const auto bound = - detail::AllocationBound(CapturedFrameStream::allocation_bound(frames).value_or(SIZE_MAX)) - .add_product(copies, MappingView::path_copy_bound() + detail::shared_allocation_bound() + - detail::shared_allocation_bound()) - .add_product(core.binding()->cpu()->worker_count() + 1, detail::shared_allocation_bound()) - .add(detail::shared_allocation_bound()) - .value(); - if (!bound) { - return Status::resource_exhausted(); - } Inspection inspection; - auto status = core_read_detail::prepare(core, input, core.read_limits(), false, *bound, kInspectionBytes, - &inspection.prepared); + auto status = core_read_detail::prepare(core, input, core.read_limits(), &inspection.prepared); if (!status.ok()) { return status; } diff --git a/lib/crowdb-tree/src/btree/published_mapping.cpp b/lib/crowdb-tree/src/btree/published_mapping.cpp index c32df67c1..83208fe64 100644 --- a/lib/crowdb-tree/src/btree/published_mapping.cpp +++ b/lib/crowdb-tree/src/btree/published_mapping.cpp @@ -156,7 +156,7 @@ std::shared_ptr MappingView::get(uint64_t page_id) const std::shared_ptr MappingView::replace_directory(const std::shared_ptr &old, unsigned shift, std::span replacements, CopyStats &copied, - const std::shared_ptr &metadata, uint64_t now_ns) + const std::shared_ptr &metadata, uint64_t now_ns) { auto next = old ? detail::reserved_shared(metadata, *old) : detail::reserved_shared(metadata); ++copied.directories; @@ -242,7 +242,7 @@ MappingView::replace_directory(const std::shared_ptr &old, unsi } MappingView MappingView::replace(std::span replacements, CopyStats &copied, - const std::shared_ptr &metadata) const + const std::shared_ptr &metadata) const { uint64_t previous = 0; bool first = true; diff --git a/lib/crowdb-tree/src/btree/published_read.h b/lib/crowdb-tree/src/btree/published_read.h index b521e1a1d..08d6e9706 100644 --- a/lib/crowdb-tree/src/btree/published_read.h +++ b/lib/crowdb-tree/src/btree/published_read.h @@ -18,8 +18,8 @@ struct PublishedReadLimits size_t max_entries = 128; size_t max_result_bytes = 1024 * 1024; // Captured mapping shape, independent of the cursor's source work window. - size_t page_metadata_bytes = 0; - std::shared_ptr metadata = nullptr; + size_t page_metadata_bytes = 0; + std::shared_ptr metadata = nullptr; }; struct PublishedGetResult diff --git a/lib/crowdb-tree/src/btree/published_route.h b/lib/crowdb-tree/src/btree/published_route.h index 94e0cea03..0d3ee07d6 100644 --- a/lib/crowdb-tree/src/btree/published_route.h +++ b/lib/crowdb-tree/src/btree/published_route.h @@ -14,7 +14,7 @@ class PublishedRouteCursor { public: PublishedRouteCursor(uint64_t root_page_id, MappingView mapping, size_t max_depth, bool reverse, - std::shared_ptr metadata = {}, PageKeyRange selection = {}) + std::shared_ptr metadata = {}, PageKeyRange selection = {}) : root_(root_page_id), mapping_(std::move(mapping)), max_depth_(max_depth), diff --git a/lib/crowdb-tree/src/btree/published_state.h b/lib/crowdb-tree/src/btree/published_state.h index b79489b7c..8209e56ca 100644 --- a/lib/crowdb-tree/src/btree/published_state.h +++ b/lib/crowdb-tree/src/btree/published_state.h @@ -65,13 +65,13 @@ struct InnerPageState return detail::shared_allocation_bound(); } - explicit InnerPageState(const std::shared_ptr &metadata = {}) + explicit InnerPageState(const std::shared_ptr &metadata = {}) : separators(KeyAllocator(metadata)), children(ChildAllocator(metadata)) { } - InnerPageState(const InnerPageState &source, const std::shared_ptr &metadata = {}) + InnerPageState(const InnerPageState &source, const std::shared_ptr &metadata = {}) : separators(source.separators.begin(), source.separators.end(), KeyAllocator(metadata)), children(source.children.begin(), source.children.end(), ChildAllocator(metadata)) { @@ -182,7 +182,7 @@ class MappingView // Input must be strictly increasing by PID and within kPageLimit. [[nodiscard]] MappingView replace(std::span replacements, CopyStats &copied, - const std::shared_ptr &metadata = {}) const; + const std::shared_ptr &metadata = {}) const; private: struct Segment @@ -217,7 +217,7 @@ class MappingView static std::shared_ptr replace_directory(const std::shared_ptr &old, unsigned shift, std::span replacements, CopyStats &copied, - const std::shared_ptr &metadata, uint64_t now_ns); + const std::shared_ptr &metadata, uint64_t now_ns); std::shared_ptr root_ = nullptr; }; diff --git a/lib/crowdb-tree/src/btree/resource_budget.cpp b/lib/crowdb-tree/src/btree/resource_budget.cpp index e29b2a9d1..f1e185ab4 100644 --- a/lib/crowdb-tree/src/btree/resource_budget.cpp +++ b/lib/crowdb-tree/src/btree/resource_budget.cpp @@ -4,6 +4,7 @@ #include "btree/resource_budget.h" #include "btree/allocation_reservation.h" +#include "crowdb-common/log.h" #include "tree_work_counters.h" #include @@ -27,32 +28,80 @@ uint64_t budget_now_ns() noexcept .count()); } -bool reserve_amount(std::atomic &used, size_t limit, size_t amount) noexcept +bool reserve_amount(std::atomic &used, size_t limit, size_t amount, size_t *previous = nullptr) noexcept { auto current = used.load(std::memory_order_relaxed); while (current <= limit && amount <= limit - current) { if (used.compare_exchange_weak(current, current + amount, std::memory_order_acq_rel)) { + if (previous != nullptr) { + *previous = current; + } return true; } } return false; } + +size_t memory_limit(const ResourceBudget::Limits &limits) noexcept +{ + size_t total = 0; + for (size_t index = 0; index < static_cast(BudgetClass::kIo); ++index) { + total += std::min(limits[index], SIZE_MAX - total); + } + return total; +} + +size_t memory_ceiling(size_t limit) noexcept +{ + return limit + std::min(limit / 2, SIZE_MAX - limit); +} + +void report_memory(size_t previous, size_t current, size_t limit, uint64_t scope) noexcept +{ + if (previous < limit && current >= limit) { + CRB_LOG_ERROR("Tree memory reached configured maximum: scope={} used={} maximum={}", scope, current, limit); + } +} } // namespace struct ResourceBudget::State { State(Limits capacity, size_t max_waiters, const std::shared_ptr &bootstrap) : limits(capacity), + memory_limit_bytes(memory_limit(capacity)), waiter_capacity(max_waiters), waiters(max_waiters * kClasses, MetadataAllocator>(bootstrap)) { } + void allocated_bytes(Account &account, size_t category, size_t bytes) noexcept + { + if (category == static_cast(BudgetClass::kIo)) { + return; + } + const auto tree = account.memory_allocated_.fetch_add(bytes, std::memory_order_relaxed); + const auto host = memory_allocated.fetch_add(bytes, std::memory_order_relaxed); + report_memory(tree, tree + bytes, account.memory_limit_, account.scope_id_); + report_memory(host, host + bytes, memory_limit_bytes, 0); + } + + void freed_bytes(Account &account, size_t category, size_t bytes) noexcept + { + if (category != static_cast(BudgetClass::kIo)) { + account.memory_allocated_.fetch_sub(bytes, std::memory_order_relaxed); + memory_allocated.fetch_sub(bytes, std::memory_order_relaxed); + } + } + std::optional reserve(const std::shared_ptr &account, size_t category, size_t amount) noexcept { if (category >= kClasses || closed.load(std::memory_order_acquire)) { return {}; } + if (category == static_cast(BudgetClass::kAdmittedWriter) && memory_exhausted(account)) { + denied(*account, category, false); + return {}; + } if (!reserve_amount(account->used_[category], account->limits_[category], amount)) { denied(*account, category, false); return {}; @@ -213,6 +262,8 @@ struct ResourceBudget::State std::weak_ptr retirement; Limits limits; + size_t memory_limit_bytes; + std::atomic memory_allocated{0}; size_t waiter_capacity; Request idle_request; std::array, kClasses> used{}; @@ -252,10 +303,24 @@ ResourceBudget::Account::Account(std::shared_ptr state, Limits limits) : state_(std::move(state)), retirement_(state_->retirement), limits_(limits), + memory_limit_(memory_limit(limits)), scope_id_(next_scope_id.fetch_add(1, std::memory_order_relaxed)) { } +bool ResourceBudget::memory_exhausted(const std::shared_ptr &account) noexcept +{ + return account && + (account->memory_allocated_.load(std::memory_order_relaxed) > memory_ceiling(account->memory_limit_) || + account->state_->memory_allocated.load(std::memory_order_relaxed) > + memory_ceiling(account->state_->memory_limit_bytes)); +} + +bool ResourceBudget::accepting(const std::shared_ptr &account) noexcept +{ + return account && !account->state_->closed.load(std::memory_order_acquire); +} + void ResourceBudget::Account::set_resource_metrics(const std::shared_ptr &metrics) noexcept { assert(resource_metrics_.expired()); @@ -338,6 +403,7 @@ bool AllocationReservation::claim(size_t bytes) noexcept account_->reserved_[category_].fetch_sub(bytes, std::memory_order_relaxed); state.allocated[category_].fetch_add(bytes, std::memory_order_relaxed); state.reserved[category_].fetch_sub(bytes, std::memory_order_relaxed); + state.allocated_bytes(*account_, category_, bytes); state.allocation_changed(); return true; } @@ -355,6 +421,7 @@ void AllocationReservation::release(size_t bytes) noexcept account_->reserved_[category_].fetch_add(bytes, std::memory_order_relaxed); state->allocated[category_].fetch_sub(bytes, std::memory_order_relaxed); state->reserved[category_].fetch_add(bytes, std::memory_order_relaxed); + state->freed_bytes(*account_, category_, bytes); return; } } @@ -364,6 +431,7 @@ void AllocationReservation::release(size_t bytes) noexcept } account_->allocated_[category_].fetch_sub(bytes, std::memory_order_relaxed); state->allocated[category_].fetch_sub(bytes, std::memory_order_relaxed); + state->freed_bytes(*account_, category_, bytes); account_->used_[category_].fetch_sub(bytes, std::memory_order_acq_rel); state->used[category_].fetch_sub(bytes, std::memory_order_acq_rel); state->notify(); @@ -473,6 +541,7 @@ std::optional ResourceBudget::Credit::allocate(size_t am account_->reserved_[category_].fetch_sub(amount, std::memory_order_relaxed); state.allocated[category_].fetch_add(amount, std::memory_order_relaxed); state.reserved[category_].fetch_sub(amount, std::memory_order_relaxed); + state.allocated_bytes(*account_, category_, amount); amount_ -= amount; state.allocation_changed(); return Credit(account_, category_, amount, true); @@ -505,6 +574,9 @@ void ResourceBudget::Credit::release() noexcept auto &account_counter = allocated_ ? account->allocated_[category_] : account->reserved_[category_]; account_counter.fetch_sub(amount, std::memory_order_relaxed); counter.fetch_sub(amount, std::memory_order_relaxed); + if (allocated_) { + state->freed_bytes(*account, category_, amount); + } account->used_[category_].fetch_sub(amount, std::memory_order_acq_rel); state->used[category_].fetch_sub(amount, std::memory_order_acq_rel); state->notify(); @@ -681,20 +753,28 @@ ResourceBudget::ResourceBudget(Limits limits, size_t max_waiters_per_class, bool ResourceBudget::claim_allocation(const std::shared_ptr &account, BudgetClass category, size_t bytes) noexcept { - if (category == BudgetClass::kIo) { + const auto index = static_cast(category); + if (!account || category == BudgetClass::kIo || index >= kClasses) { return false; } - auto credit = try_reserve(account, category, bytes); - if (!credit) { + auto state = account->state_; + size_t tree = 0; + size_t host = 0; + if (!reserve_amount(account->memory_allocated_, memory_ceiling(account->memory_limit_), bytes, &tree)) { + state->denied(*account, index, false); return false; } - auto allocation = credit->allocate(bytes); - if (!allocation) { + if (!reserve_amount(state->memory_allocated, memory_ceiling(state->memory_limit_bytes), bytes, &host)) { + account->memory_allocated_.fetch_sub(bytes, std::memory_order_relaxed); + state->denied(*account, index, true); return false; } - credit->account_.reset(); - allocation->amount_ = 0; - allocation->account_.reset(); + account->used_[index].fetch_add(bytes, std::memory_order_relaxed); + state->used[index].fetch_add(bytes, std::memory_order_relaxed); + account->allocated_[index].fetch_add(bytes, std::memory_order_relaxed); + state->allocated[index].fetch_add(bytes, std::memory_order_relaxed); + report_memory(tree, tree + bytes, account->memory_limit_, account->scope_id_); + report_memory(host, host + bytes, state->memory_limit_bytes, 0); return true; } @@ -732,10 +812,14 @@ bool ResourceBudget::transfer_allocation( if (from == to) { return true; } - if (!reserve_amount(to->used_[index], to->limits_[index], bytes)) { + size_t memory_before = 0; + if (!reserve_amount(to->memory_allocated_, memory_ceiling(to->memory_limit_), bytes, &memory_before)) { to->state_->denied(*to, index, false); return false; } + to->used_[index].fetch_add(bytes, std::memory_order_relaxed); + from->memory_allocated_.fetch_sub(bytes, std::memory_order_relaxed); + report_memory(memory_before, memory_before + bytes, to->memory_limit_, to->scope_id_); to->allocated_[index].fetch_add(bytes, std::memory_order_relaxed); [[maybe_unused]] const auto previous = from->allocated_[index].fetch_sub(bytes, std::memory_order_relaxed); assert(previous >= bytes); diff --git a/lib/crowdb-tree/src/btree/resource_budget.h b/lib/crowdb-tree/src/btree/resource_budget.h index dbd761354..e21796ec8 100644 --- a/lib/crowdb-tree/src/btree/resource_budget.h +++ b/lib/crowdb-tree/src/btree/resource_budget.h @@ -93,6 +93,8 @@ class ResourceBudget std::weak_ptr resource_metrics_; mutable std::atomic retirement_pending_{0}; Limits limits_; + size_t memory_limit_ = 0; + std::atomic memory_allocated_{0}; uint64_t scope_id_; std::array, kClasses> used_{}; std::array, kClasses> reserved_{}; @@ -239,6 +241,8 @@ class ResourceBudget // Planning hint from admission usage, including both host and Tree limits. // A subsequent reservation must still claim the capacity atomically. [[nodiscard]] static size_t available(const std::shared_ptr &account, BudgetClass category) noexcept; + [[nodiscard]] static bool memory_exhausted(const std::shared_ptr &account) noexcept; + [[nodiscard]] static bool accepting(const std::shared_ptr &account) noexcept; // Stop new admission and cancel waiting requests. Outstanding credit // remains charged until its actual owner releases it. // Bootstrap-only before publishing the host runtime. New accounts copy diff --git a/lib/crowdb-tree/src/btree/resource_observation.cpp b/lib/crowdb-tree/src/btree/resource_observation.cpp index fb8dd2e20..e972a8aa0 100644 --- a/lib/crowdb-tree/src/btree/resource_observation.cpp +++ b/lib/crowdb-tree/src/btree/resource_observation.cpp @@ -6,8 +6,8 @@ namespace crowdb::tree { -ResourceObservation::ResourceObservation(const std::shared_ptr &budget, - TreeResourceMetric metric, uint64_t bytes) noexcept +ResourceObservation::ResourceObservation(const std::shared_ptr &budget, TreeResourceMetric metric, + uint64_t bytes) noexcept : metrics_(budget ? budget->resource_metrics() : nullptr), metric_(metric) { diff --git a/lib/crowdb-tree/src/btree/resource_observation.h b/lib/crowdb-tree/src/btree/resource_observation.h index c69274575..fb72bee3b 100644 --- a/lib/crowdb-tree/src/btree/resource_observation.h +++ b/lib/crowdb-tree/src/btree/resource_observation.h @@ -12,7 +12,7 @@ namespace crowdb::tree class ResourceObservation { public: - ResourceObservation(const std::shared_ptr &budget, TreeResourceMetric metric, + ResourceObservation(const std::shared_ptr &budget, TreeResourceMetric metric, uint64_t bytes = 0) noexcept; ~ResourceObservation(); ResourceObservation(const ResourceObservation &) = delete; diff --git a/lib/crowdb-tree/src/btree/source_write_budget.cpp b/lib/crowdb-tree/src/btree/source_write_budget.cpp index 0efc16d36..0e796e5ac 100644 --- a/lib/crowdb-tree/src/btree/source_write_budget.cpp +++ b/lib/crowdb-tree/src/btree/source_write_budget.cpp @@ -89,6 +89,9 @@ Status SourceWriteReservation::create(const SourceWriteOptions &options, WriteFo } auto credit = detail::ResourceBudget::try_reserve(options.account, category, *bytes); if (!credit) { + if (!gap && detail::ResourceBudget::memory_exhausted(options.account)) { + return Status::resource_exhausted("actual Tree memory exceeds 150% of the configured maximum"); + } return Status::unavailable("writer apply credit unavailable"); } try { diff --git a/lib/crowdb-tree/src/btree/tree_core.cpp b/lib/crowdb-tree/src/btree/tree_core.cpp index 3fce7b3f0..7ee2ace90 100644 --- a/lib/crowdb-tree/src/btree/tree_core.cpp +++ b/lib/crowdb-tree/src/btree/tree_core.cpp @@ -3,6 +3,7 @@ #include "backend/checkpoint_backend.h" #include "backend/local/local_backend_state.h" #include "backend/local/scheduled_page_store.h" +#include "btree/allocation_account.h" #include "btree/allocation_bound.h" #include "runtime/tree_tuning_c_api.h" #include "snapshot/snapshot_baseline.h" @@ -290,8 +291,15 @@ Status TreeCore::State::initialize(TreeCoreBootstrap bootstrap) loads.max_leaf_key_bytes = loads.max_frame_bytes; } reads.max_key_bytes = writers.max_key_size; - auto batch = flush_options(config, *binding, writers.max_key_size); - preparation = {.runtime = binding->cpu(), .store = backend, .loads = loads, .maintenance = {}, .cache = cache}; + std::shared_ptr read_allocations; + auto read_status = + detail::AllocationAccount::create(binding->account(), detail::BudgetClass::kRouting, &read_allocations); + if (!read_status.ok()) { + return read_status; + } + reads.metadata = std::move(read_allocations); + auto batch = flush_options(config, *binding, writers.max_key_size); + preparation = {.runtime = binding->cpu(), .store = backend, .loads = loads, .maintenance = {}, .cache = cache}; preparation.cpu_progress = apply_dispatcher; status = publisher->prepare_lane(batch.parallelism + 1, credit, &preparation.publication); if (!status.ok()) { diff --git a/lib/crowdb-tree/src/btree/tree_core.h b/lib/crowdb-tree/src/btree/tree_core.h index 15326a8f1..5da535dbe 100644 --- a/lib/crowdb-tree/src/btree/tree_core.h +++ b/lib/crowdb-tree/src/btree/tree_core.h @@ -139,7 +139,7 @@ class TreeCore : public std::enable_shared_from_this // Runs on the shared runtime; staged completion never advances WAL coverage. Status prepare_writeback(std::shared_ptr *out); Status capture_snapshot(std::shared_ptr *out); - Status acquire_snapshot_window(const std::shared_ptr &metadata, + Status acquire_snapshot_window(const std::shared_ptr &metadata, std::shared_ptr *ticket, AsyncPageStore **backend); Status reserve_snapshot_ids(const PublishedTreeState &expected, uint64_t count, const std::shared_ptr &metadata); diff --git a/lib/crowdb-tree/src/btree/tree_core_batch.cpp b/lib/crowdb-tree/src/btree/tree_core_batch.cpp index 8917f5f2a..4c7a6402d 100644 --- a/lib/crowdb-tree/src/btree/tree_core_batch.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_batch.cpp @@ -202,14 +202,12 @@ Status write_inputs(SourceAdmission &sources, ApplyFrontier &frontier, uint64_t } if (!paused && rotation != nullptr && category == detail::BudgetClass::kAdmittedWriter && status.code() == Code::kUnavailable && !detail::FlushRuntime::on_worker_thread()) { - const auto source = sources.catalog()->active->id(); - const auto requested = rotation->request(); + auto requested = rotation->request(); if (requested == 0) { - (void)rotation->request_pressure(); - return status; + requested = rotation->request_pressure(); } - rotation->wait(requested); - if (sources.catalog()->active->id() != source) { + if (requested != 0) { + rotation->wait(requested); continue; } } @@ -267,12 +265,15 @@ Status apply_source_batch(const Config &config, SourceAdmission &sources, ApplyF } return status.ok() ? frontier.complete(slot) : status; } - const auto catalog = sources.catalog(); - const auto &options = catalog->active_budget->options(); - status = validate_inputs(config, options, max_value_bytes, supplied_slot.value_or(0), count, read); + auto catalog = sources.catalog(); + const auto options = catalog->active_budget->options(); + const auto floor = catalog->floor; + status = validate_inputs(config, options, max_value_bytes, supplied_slot.value_or(0), count, read); if (!status.ok()) { return status; } + // Waiting for flush must not retain the source payloads it needs to retire. + catalog.reset(); const auto bytes = detail::shared_allocation_bound() + TreeOperationLifetime::ticket_allocation_bound() + (count * (sizeof(size_t) + sizeof(SourceAdmission::Writer::Prepared))); @@ -293,7 +294,7 @@ Status apply_source_batch(const Config &config, SourceAdmission &sources, ApplyF return status; } auto order = select_inputs(count, read, credit); - if (supplied_slot && *supplied_slot <= catalog->floor) { + if (supplied_slot && *supplied_slot <= floor) { return frontier.complete(*supplied_slot); } return write_inputs(sources, frontier, completed, supplied_slot, automatic, order, read, credit, category, progress, diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint.cpp b/lib/crowdb-tree/src/btree/tree_core_checkpoint.cpp index 2085d0cb3..05b05b461 100644 --- a/lib/crowdb-tree/src/btree/tree_core_checkpoint.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint.cpp @@ -90,7 +90,7 @@ Status TreeCore::prepare_checkpoint_state(const CheckpointOptions .add(shared_allocation_bound()) .add_product(2, shared_allocation_bound()) .add(shared_allocation_bound()) - .add(FlushPersistence::control_allocation_bound(binding()->cpu()->worker_count())) + .add(FlushPersistence::control_allocation_bound()) .add(TreeOperationLifetime::ticket_allocation_bound()) .value(); if (!bound) { diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint_budget.cpp b/lib/crowdb-tree/src/btree/tree_core_checkpoint_budget.cpp index 748020ad5..933439ace 100644 --- a/lib/crowdb-tree/src/btree/tree_core_checkpoint_budget.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint_budget.cpp @@ -19,8 +19,8 @@ Credits::~Credits() snapshot->seal(); } detail::seal_backend_budget(budget.backend); - if (budget.bytes) { - budget.bytes->seal(); + if (budget.reservation()) { + budget.reservation()->seal(); } if (owns_io && budget.io) { budget.io->seal(); @@ -173,11 +173,11 @@ Status Preparation::bind_work(const WorkPlan &plan, detail::ResourceBudget::Cred : io.amount() != 1 || io.allocated() || !io.belongs_to(account, BudgetClass::kIo))) { return Status::invalid_argument("checkpoint needs its complete prepaid partitions"); } - auto phase = reserved_shared(control); + auto phase = reserved_shared(core->read_limits().metadata); phase->budget.bytes = AllocationReservation::create(std::move(bytes)); phase->owns_io = !shared_io; - phase->budget.io = - shared_io ? std::move(shared_io) : reserved_shared(phase->budget.bytes, std::move(io)); + phase->budget.io = shared_io ? std::move(shared_io) + : reserved_shared(phase->budget.reservation(), std::move(io)); phase->budget.work_metrics = core->publisher().work_metrics(); phase->budget.cause = options.cause; phase->budget.cpu_progress = cpu_progress; @@ -187,10 +187,10 @@ Status Preparation::bind_work(const WorkPlan &plan, detail::ResourceBudget::Cred return status; } out->budget = phase->budget; - out->loader = reserved_shared(phase->budget.bytes, ticket->backend(core->async_backend()), - core->binding()->cpu(), core->binding()->lane(), plan.loads, - OverflowBackingResolver{}, phase->budget); - out->owner = std::move(phase); + out->loader = reserved_shared( + phase->budget.reservation(), ticket->backend(core->async_backend()), core->binding()->cpu(), + core->binding()->lane(), plan.loads, OverflowBackingResolver{}, phase->budget); + out->owner = std::move(phase); return Status::Ok(); } diff --git a/lib/crowdb-tree/src/btree/tree_core_checkpoint_structure.cpp b/lib/crowdb-tree/src/btree/tree_core_checkpoint_structure.cpp index ca302c08e..4e09e09ee 100644 --- a/lib/crowdb-tree/src/btree/tree_core_checkpoint_structure.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_checkpoint_structure.cpp @@ -61,18 +61,18 @@ try { phase->snapshot = AllocationReservation::create(std::move(*snapshot)); limits.publication.snapshot_metadata = phase->snapshot; phase->budget.bytes = AllocationReservation::create(std::move(*read)); - phase->budget.io = reserved_shared(phase->budget.bytes, std::move(*io)); - phase->budget.work_metrics = core->publisher().work_metrics(); - phase->budget.cause = options.cause; - phase->budget.cpu_progress = cpu_progress; + phase->budget.io = reserved_shared(phase->budget.reservation(), std::move(*io)); + phase->budget.work_metrics = core->publisher().work_metrics(); + phase->budget.cause = options.cause; + phase->budget.cpu_progress = cpu_progress; auto status = bind_backend_operation(core->async_backend(), backend_limits, std::move(*backend), &phase->budget.backend); if (!status.ok()) { return status; } - auto loader = reserved_shared(phase->budget.bytes, ticket->backend(core->async_backend()), - core->binding()->cpu(), core->binding()->lane(), loads, - OverflowBackingResolver{}, phase->budget); + auto loader = reserved_shared( + phase->budget.reservation(), ticket->backend(core->async_backend()), core->binding()->cpu(), + core->binding()->lane(), loads, OverflowBackingResolver{}, phase->budget); limits.prepaid = AllocationReservation::create(std::move(*bytes)); return PageSplitOperation::create(core->publisher(), *core->binding()->cpu(), core->binding()->lane(), std::move(view), pid, std::move(limits), std::move(loader), cpu_progress, diff --git a/lib/crowdb-tree/src/btree/tree_core_chunk_gc.cpp b/lib/crowdb-tree/src/btree/tree_core_chunk_gc.cpp index 7a19072d4..38536174e 100644 --- a/lib/crowdb-tree/src/btree/tree_core_chunk_gc.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_chunk_gc.cpp @@ -34,7 +34,7 @@ try { } PageLoadBudget budget; budget.bytes = AllocationReservation::create(std::move(*bytes)); - budget.io = reserved_shared(budget.bytes, std::move(*io)); + budget.io = reserved_shared(budget.reservation(), std::move(*io)); struct Credit { @@ -44,14 +44,14 @@ try { ~Credit() { if (!transferred) { - budget.bytes->seal(); + budget.reservation()->seal(); budget.io->seal(); } } } credit{.budget = budget}; std::shared_ptr ticket; - auto status = operation_lifetime().acquire(budget.bytes, &ticket); + auto status = operation_lifetime().acquire(budget.reservation(), &ticket); if (!status.ok()) { return status; } diff --git a/lib/crowdb-tree/src/btree/tree_core_chunk_open.cpp b/lib/crowdb-tree/src/btree/tree_core_chunk_open.cpp index 0b7a2eba3..90542eef3 100644 --- a/lib/crowdb-tree/src/btree/tree_core_chunk_open.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_chunk_open.cpp @@ -46,8 +46,8 @@ struct ChunkOpen if (budget.io) { budget.io->seal(); } - if (budget.bytes) { - budget.bytes->seal(); + if (budget.reservation()) { + budget.reservation()->seal(); } budget = {}; } @@ -127,17 +127,17 @@ Status open_credit(ChunkOpen &open) return Status::resource_exhausted(); } auto bytes = ResourceBudget::try_reserve(open.binding->account(), BudgetClass::kRouting, - options.limits.routing_bytes - options.lifecycle_bytes); + ResourceBudget::available(open.binding->account(), BudgetClass::kRouting)); auto io = ResourceBudget::try_reserve(open.binding->account(), BudgetClass::kIo, 1); if (!bytes || !io) { return Status::resource_exhausted(); } open.budget.bytes = AllocationReservation::create(std::move(*bytes)); - open.budget.io = reserved_shared(open.budget.bytes, std::move(*io)); + open.budget.io = reserved_shared(open.budget.reservation(), std::move(*io)); if (!open.lifetime) { return Status::internal_error(); } - auto status = open.lifetime->acquire(open.budget.bytes, &open.ticket); + auto status = open.lifetime->acquire(open.budget.reservation(), &open.ticket); if (status.ok()) { open.store = open.ticket->backend(open.backend->store.get()); } @@ -159,7 +159,7 @@ template Status run_protocol(ChunkOpen &open, const std::shared Status load_root(ChunkOpen &open, std::shared_ptr *out) { - auto root = detail::reserved_shared(open.budget.bytes, open.budget.bytes); + auto root = detail::reserved_shared(open.budget.reservation(), open.budget.reservation()); root->domain = open.backend->domain; root->generation = open.backend->facade->options().open_generation; std::shared_ptr operation; @@ -192,13 +192,13 @@ Status recover_chunk(ChunkOpen &open, const Config &config, std::shared_ptriu_size); const auto metadata = ChunkRootRecovery::allocation_bound(loaded->root); const auto window = ChunkRootRecovery::window_allocation_bound(limits); - if (!metadata || !window || *metadata > open.budget.bytes->available() || - *window > open.budget.bytes->available() - *metadata) { + if (!metadata || !window || *metadata > open.budget.reservation()->available() || + *window > open.budget.reservation()->available() - *metadata) { return Status::resource_exhausted(); } std::shared_ptr recovery; status = ChunkRootRecovery::create(std::move(loaded->root), open.backend->domain, open.backend->ids, limits, - open.budget.bytes, open.budget.bytes, &recovery); + open.budget.reservation(), open.budget.reservation(), &recovery); loaded.reset(); if (!status.ok()) { return status; @@ -211,12 +211,12 @@ Status recover_chunk(ChunkOpen &open, const Config &config, std::shared_ptr(open.budget.bytes, open.store, open.binding->cpu(), - open.binding->lane(), loads, OverflowBackingResolver{}, - open.budget); + auto loader = detail::reserved_shared(open.budget.reservation(), open.store, + open.binding->cpu(), open.binding->lane(), loads, + OverflowBackingResolver{}, open.budget); std::shared_ptr recovered; status = RecoveredStateOperation::create(recovery->pages(), loader, open.binding->cpu(), open.binding->lane(), {}, - open.budget.bytes, &recovered); + open.budget.reservation(), &recovered); if (!status.ok()) { return status; } diff --git a/lib/crowdb-tree/src/btree/tree_core_get.cpp b/lib/crowdb-tree/src/btree/tree_core_get.cpp index 10b2da65d..f5311e995 100644 --- a/lib/crowdb-tree/src/btree/tree_core_get.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_get.cpp @@ -46,16 +46,12 @@ struct CoreGet : std::enable_shared_from_this auto self = std::move(owner); GetView result; if (status.code() == Code::kResourceExhausted) { - CRB_LOG_ERROR("point read exhausted: coverage={} remaining={} status={}", prepared.view->coverage, - prepared.credit->available(), status.to_string()); + CRB_LOG_ERROR("point read exhausted: coverage={} status={}", prepared.view->coverage, status.to_string()); } if (status.ok() && read->result().found) { const auto &cell = read->result().cell; result = GetView::from_owner(cell.slot, cell.value, cell.owner); } - detail::seal_backend_budget(prepared.backend); - prepared.io->seal(); - prepared.credit->seal(); try { complete(std::move(status), std::move(result)); } @@ -95,10 +91,7 @@ void TreeCore::get_impl(Slice key, std::function complete status = Status::unavailable("Tree closed"); } else if (!outside_range) { - status = core_read_detail::prepare(*this, false, - detail::shared_allocation_bound() + - detail::shared_allocation_bound(), - 0, &prepared); + status = core_read_detail::prepare(*this, &prepared); } if (status.ok() && !outside_range) { operation = detail::reserved_shared(prepared.credit); diff --git a/lib/crowdb-tree/src/btree/tree_core_open.cpp b/lib/crowdb-tree/src/btree/tree_core_open.cpp index e1bbd1c28..17c3e4c3c 100644 --- a/lib/crowdb-tree/src/btree/tree_core_open.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_open.cpp @@ -26,8 +26,8 @@ struct OpenCredits ~OpenCredits() { detail::seal_backend_budget(budget.backend); - if (budget.bytes) { - budget.bytes->seal(); + if (budget.reservation()) { + budget.reservation()->seal(); } if (budget.io) { budget.io->seal(); @@ -71,6 +71,8 @@ Status reserve_open(const std::shared_ptr &binding, const de } available -= retained; } + available = + std::min(available, detail::ResourceBudget::available(binding->account(), detail::BudgetClass::kRouting)); if (recovery_bound) { if (*recovery_bound > available) { return Status::resource_exhausted("empty recovery exceeds Tree routing limit"); @@ -83,7 +85,8 @@ Status reserve_open(const std::shared_ptr &binding, const de return Status::resource_exhausted("recovery routing or IO credit unavailable"); } credits->budget.bytes = detail::AllocationReservation::create(std::move(*bytes)); - credits->budget.io = detail::reserved_shared(credits->budget.bytes, std::move(*io)); + credits->budget.io = + detail::reserved_shared(credits->budget.reservation(), std::move(*io)); return Status::Ok(); } @@ -127,11 +130,12 @@ Status recover_open(const Config &config, const std::shared_ptr( - budget.bytes, lifetime.store, binding->cpu(), binding->lane(), loads, OverflowBackingResolver{}, budget); + auto loader = + detail::reserved_shared(budget.reservation(), lifetime.store, binding->cpu(), + binding->lane(), loads, OverflowBackingResolver{}, budget); std::shared_ptr recovered; status = RecoveredStateOperation::create(local->inherited, loader, binding->cpu(), binding->lane(), {}, - budget.bytes, &recovered); + budget.reservation(), &recovered); if (!status.ok()) { return status; } @@ -226,7 +230,7 @@ try { if (!status.ok()) { return status; } - credits.budget.bytes->seal(); + credits.budget.reservation()->seal(); credits.budget.io->seal(); return create(config, binding, {.initial = std::move(initial), .local = std::move(local)}, out); } diff --git a/lib/crowdb-tree/src/btree/tree_core_read_budget.cpp b/lib/crowdb-tree/src/btree/tree_core_read_budget.cpp index b57ddfd3f..4934648f2 100644 --- a/lib/crowdb-tree/src/btree/tree_core_read_budget.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_read_budget.cpp @@ -1,20 +1,18 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -#include "btree/allocation_bound.h" -#include "btree/read_value_bound.h" #include "btree/tree_core_read_internal.h" #include namespace crowdb::tree::core_read_detail { -Status prepare(TreeCore &core, bool scan, size_t control_bytes, size_t result_bytes, ReadPreparation *out) +Status prepare(TreeCore &core, ReadPreparation *out) { - return prepare(core, core.capture(), core.read_limits(), scan, control_bytes, result_bytes, out); + return prepare(core, core.capture(), core.read_limits(), out); } -Status prepare(TreeCore &core, std::shared_ptr view, PublishedReadLimits limits, bool scan, - size_t control_bytes, size_t result_bytes, ReadPreparation *out) +Status prepare(TreeCore &core, std::shared_ptr view, PublishedReadLimits limits, + ReadPreparation *out) { using namespace detail; ReadPreparation ready; @@ -23,90 +21,23 @@ Status prepare(TreeCore &core, std::shared_ptr view, P ready.limits = std::move(limits); const auto root = ready.view->mapping.get(ready.view->root_page_id); if (core.closed() || !root || root->level >= ready.limits.max_route_depth) { - return Status::unavailable("read requires an open bounded published root"); + return Status::unavailable("read requires an open published root"); } - if (ready.view->sources->tables.size() > ready.limits.max_memtables) { - return Status::resource_exhausted("captured read exceeds the source count limit"); - } - ready.limits.max_route_depth = root->level + 1ULL; - ready.limits.max_memtables = std::max(1, ready.view->sources->tables.size()); - auto loads = core.load_limits(); - loads.max_inflight = 1; - const size_t workers = ready.binding->cpu()->worker_count(); - const auto shape = ready.view->mapping.copy_bounds(); - ready.limits.page_metadata_bytes = shape.max_leaf_bytes; - const auto captured_bases = - AllocationBound().add_product(shape.leaf_pages, shape.max_leaf_bases).value().value_or(SIZE_MAX); - const auto leaf_window = AllocationBound(shape.max_leaf_bases).add(scan ? ready.limits.max_entries : 0).value(); - if (!leaf_window) { - return Status::resource_exhausted("captured read frame window overflow"); - } - const size_t leaves = std::min(*leaf_window, scan ? captured_bases : shape.max_leaf_bases); - const size_t values = scan ? ready.limits.max_entries : 1; - const size_t cached_pages = core.cache()->retained_page_limit(); - // One point lookup visits one routing path and one logical leaf. A scan - // can retain more decoded pages, but cannot visit absent index PIDs. - const size_t cached_inner = std::min(cached_pages, scan ? ready.view->stats.inner_pages : root->level); - const size_t cached_leaf = std::min(cached_pages, scan ? captured_bases : shape.max_leaf_bases); - // Only fixed L1 cells can require an overflow allocation. L0 cells lend - // their independent payload owners, including writes after this capture. - const auto value_limit = overflow_value_bytes_bound(*ready.view, loads.max_value_bytes); - const auto value_bytes = - AllocationBound(value_limit).add(scan && value_limit != 0 ? ready.limits.max_result_bytes : 0).value(); - auto bounded_loads = loads; - // Every accepted overflow frame contributes at least one value byte. This - // tightens reservation only; the loader's public validation caps stay fixed. - bounded_loads.max_overflow_pages = std::min(loads.max_overflow_pages, value_limit); - auto loader_bound = - AsyncStorePageLoader::primary_reserve_bound(loads, root->level, workers, leaves, cached_leaf, cached_inner); - if (value_limit != 0) { - loader_bound = value_bytes - ? AsyncStorePageLoader::read_reserve_bound(bounded_loads, root->level, workers, leaves, values, - *value_bytes, cached_leaf, cached_inner) - : std::nullopt; - } - const auto bound = AllocationBound(control_bytes) - .add_product(2, shared_allocation_bound()) - .add(TreeOperationLifetime::ticket_allocation_bound()) - .add(scan ? PublishedScan::allocation_bound(ready.limits, workers) - : PublishedGet::allocation_bound(ready.limits, workers)) - .add(loader_bound) - .add(core.cache()->load_reserve_bound(root->level + leaves, workers)) - .add(SharedPageCache::binding_allocation_bound()) - .add(result_bytes) - .value(); - if (!bound) { - return Status::resource_exhausted("read admission bound overflow"); - } - auto bytes = ResourceBudget::try_reserve(ready.binding->account(), BudgetClass::kRouting, *bound); - auto io = ResourceBudget::try_reserve(ready.binding->account(), BudgetClass::kIo, 1); - if (!bytes) { - return Status::resource_exhausted("read Routing reserve unavailable: requested=" + std::to_string(*bound)); - } - if (!io) { - return Status::resource_exhausted("read I/O reserve unavailable"); - } - ready.credit = AllocationReservation::create(std::move(*bytes)); - ready.io = reserved_shared(ready.credit, std::move(*io)); - ready.limits.metadata = ready.credit; - auto backend_status = reserve_backend_operation( - core.async_backend(), ready.binding->account(), - {.max_io_bytes = loads.max_blob_bytes, .write_calls = 0, .workers = workers, .barriers = false}, - &ready.backend); - if (!backend_status.ok()) { - return backend_status; - } - auto status = core.operation_lifetime().acquire(ready.credit, &ready.ticket); + ready.limits.max_route_depth = root->level + 1ULL; + ready.limits.max_memtables = std::max(1, ready.view->sources->tables.size()); + ready.credit = ready.limits.metadata; + auto status = core.operation_lifetime().acquire(ready.credit, &ready.ticket); if (!status.ok()) { return status; } + auto loads = core.load_limits(); + loads.max_inflight = 1; auto loader = reserved_shared(ready.credit, ready.ticket->backend(core.async_backend()), ready.binding->cpu(), ready.binding->lane(), loads, OverflowBackingResolver{}, PageLoadBudget{.bytes = ready.credit, - .io = ready.io, + .account = ready.binding->account(), .work_metrics = core.publisher().work_metrics(), - .backend = ready.backend, .cpu_progress = core.cpu_progress(), .cause = TreeWorkCause::Read}); status = core.cache()->bind_reserved(std::move(loader), ready.credit, &ready.loader); diff --git a/lib/crowdb-tree/src/btree/tree_core_read_internal.h b/lib/crowdb-tree/src/btree/tree_core_read_internal.h index 0c412184b..67ee33a3d 100644 --- a/lib/crowdb-tree/src/btree/tree_core_read_internal.h +++ b/lib/crowdb-tree/src/btree/tree_core_read_internal.h @@ -13,14 +13,7 @@ struct ReadPreparation ReadPreparation(ReadPreparation &&) noexcept = default; ReadPreparation &operator=(ReadPreparation &&) noexcept = default; - ~ReadPreparation() - { - detail::seal_backend_budget(backend); - } - - std::shared_ptr backend; - std::shared_ptr credit; - std::shared_ptr io; + std::shared_ptr credit; std::shared_ptr ticket; std::shared_ptr binding; std::shared_ptr view; @@ -29,7 +22,7 @@ struct ReadPreparation }; // All admission and source capture happens before the first CPU submission. -Status prepare(TreeCore &core, bool scan, size_t control_bytes, size_t result_bytes, ReadPreparation *out); -Status prepare(TreeCore &core, std::shared_ptr view, PublishedReadLimits limits, bool scan, - size_t control_bytes, size_t result_bytes, ReadPreparation *out); +Status prepare(TreeCore &core, ReadPreparation *out); +Status prepare(TreeCore &core, std::shared_ptr view, PublishedReadLimits limits, + ReadPreparation *out); } // namespace crowdb::tree::core_read_detail diff --git a/lib/crowdb-tree/src/btree/tree_core_scan.cpp b/lib/crowdb-tree/src/btree/tree_core_scan.cpp index 20e6bae0c..1e42ed0ea 100644 --- a/lib/crowdb-tree/src/btree/tree_core_scan.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_scan.cpp @@ -3,6 +3,7 @@ #include "btree/scan_output.h" #include "btree/tree_core_internal.h" #include "btree/tree_core_scan_internal.h" +#include "crowdb-common/log.h" #include #include @@ -24,9 +25,9 @@ bool CoreScan::expired() const noexcept void CoreScan::finish(Status status, bool truncated) noexcept { auto self = std::move(owner); - detail::seal_backend_budget(prepared.backend); - prepared.io->seal(); - prepared.credit->seal(); + if (status.code() == Code::kResourceExhausted) { + CRB_LOG_WARN("scan failed after {} entries: {}", count, status.to_string()); + } try { complete(std::move(status), std::move(packed), truncated); } @@ -135,6 +136,9 @@ void TreeCore::scan_impl(CoreScanOptions options, std::function // Licensed under the Apache License, Version 2.0. -#include "btree/allocation_account.h" #include "btree/allocation_bound.h" #include "btree/tree_core_scan_internal.h" @@ -10,42 +9,7 @@ namespace crowdb::tree::core_read_detail { namespace { -std::optional result_count_bound(const PublishedTreeState &view, const PageLoadLimits &loads, - const TreeExecutionOptions &options, bool include_tombstones) -{ - using detail::AllocationBound; - const auto shape = view.mapping.copy_bounds(); - std::optional count; - if (!include_tombstones && view.stats.kv_unknown_pages == 0) { - count = view.stats.kv_known_count; - } - else { - const auto bases = AllocationBound().add_product(shape.leaf_pages, shape.max_leaf_bases).value(); - if (!bases) { - return {}; - } - count = AllocationBound(view.stats.overlay_entries).add_product(*bases, loads.max_leaf_entries).value(); - } - if (!count) { - return {}; - } - AllocationBound total(*count); - // Captured sources may still accept later writes. Their hard version quota, - // not the current count, bounds those future visible keys. Core admission - // keeps these hard quotas independent of the configurable soft thresholds; - // inherited split tables are closed and drained, so use their exact count. - const auto future = AllocationBound(options.source_max_entries).add(options.source_max_gap_entries).value(); - if (!future) { - return {}; - } - for (const auto &table : view.sources->tables) { - total.add(table->closed() && table->writers() == 0 ? table->count() : *future); - } - return total.value(); -} - -PageKeyRange selection(CoreScanOptions options, const KeyRange &domain, - const std::shared_ptr &credit) +PageKeyRange selection(CoreScanOptions options, const KeyRange &domain, const std::shared_ptr &credit) { PageKeyRange range; if (!options.prefix.empty()) { @@ -85,30 +49,9 @@ Status prepare_scan(TreeCore &core, CoreScanOptions options, std::shared_ptr()) - .add_product(core.binding()->cpu()->worker_count() + 1, shared_allocation_bound()) - .add(FlushRuntime::progress_allocation_bound()) - .add_product(6, OwnedKey::allocation_overhead() + limits.max_key_bytes + 1) - .add_product(2, OwnedKey::allocation_overhead() + 1) - .add(core.config().key_range.start() ? core.config().key_range.start()->size() : 0) - .add(core.config().key_range.end() ? core.config().key_range.end()->size() : 0) - .value(); - if (!control) { - return Status::resource_exhausted("scan control bound overflow"); - } // Keys are unique: all except the one possible empty key cost at least one // byte. A tiny byte window cannot contain more than bytes + 1 results; // the oversized-first exception contributes only one result. @@ -116,7 +59,7 @@ Status prepare_scan(TreeCore &core, CoreScanOptions options, std::shared_ptr(1, std::min({count, limits.max_entries, byte_count})); limits.max_result_bytes = window_bytes; ReadPreparation ready; - auto status = prepare(core, view, limits, true, *control, 0, &ready); + auto status = prepare(core, view, limits, &ready); if (!status.ok()) { return status; } @@ -139,18 +82,13 @@ Status prepare_scan(TreeCore &core, CoreScanOptions options, std::shared_ptrlimit = count; scan->byte_limit = bytes; - scan->normal_capacity = *normal; + scan->normal_capacity = 0; scan->value_limit = core.load_limits().max_value_bytes; scan->deadline_ms = options.deadline_ms; scan->strict = options.strict_byte_budget; - std::shared_ptr output; - status = AllocationAccount::create(prepared.binding->account(), BudgetClass::kRouting, &output); - if (!status.ok()) { - return status; - } - scan->packed = ScanPackedBuf(std::move(output)); - status = prepared.binding->cpu()->reserve_progress(prepared.binding->lane(), &CoreScan::step, scan.get(), - prepared.credit, &scan->progress); + scan->packed = ScanPackedBuf(prepared.credit); + status = prepared.binding->cpu()->reserve_progress(prepared.binding->lane(), &CoreScan::step, scan.get(), + prepared.credit, &scan->progress); if (status.ok()) { *out = std::move(scan); } diff --git a/lib/crowdb-tree/src/btree/tree_core_snapshot.cpp b/lib/crowdb-tree/src/btree/tree_core_snapshot.cpp index 9f3da6105..124c9c2cf 100644 --- a/lib/crowdb-tree/src/btree/tree_core_snapshot.cpp +++ b/lib/crowdb-tree/src/btree/tree_core_snapshot.cpp @@ -27,9 +27,9 @@ Status TreeCore::capture_snapshot(std::shared_ptr vie return PublishedSnapshotContext::capture_admitted(shared_from_this(), out, std::move(view)); } -Status TreeCore::acquire_snapshot_window(const std::shared_ptr &metadata, - std::shared_ptr *ticket, - AsyncPageStore **backend) +Status TreeCore::acquire_snapshot_window(const std::shared_ptr &metadata, + std::shared_ptr *ticket, + AsyncPageStore **backend) { if (backend == nullptr) { return Status::invalid_argument("snapshot read window requires backend output"); diff --git a/lib/crowdb-tree/src/runtime/tree_operation_lifetime.cpp b/lib/crowdb-tree/src/runtime/tree_operation_lifetime.cpp index 0d0e6e733..a814e0e67 100644 --- a/lib/crowdb-tree/src/runtime/tree_operation_lifetime.cpp +++ b/lib/crowdb-tree/src/runtime/tree_operation_lifetime.cpp @@ -28,9 +28,8 @@ struct TreeOperationLifetime::State std::atomic denied{0}; }; -TreeOperationLifetime::TreeOperationLifetime(size_t maximum, - const std::shared_ptr &metadata, - std::shared_ptr host_owner) +TreeOperationLifetime::TreeOperationLifetime(size_t maximum, const std::shared_ptr &metadata, + std::shared_ptr host_owner) { if (maximum == 0 || maximum >= State::kClosed) { throw std::invalid_argument("invalid Tree operation capacity"); @@ -43,19 +42,18 @@ TreeOperationLifetime::~TreeOperationLifetime() close(); } -Status TreeOperationLifetime::acquire(const std::shared_ptr &metadata, - std::shared_ptr *out) +Status TreeOperationLifetime::acquire(const std::shared_ptr &metadata, std::shared_ptr *out) { return acquire_impl(metadata, out, false); } -Status TreeOperationLifetime::try_acquire(const std::shared_ptr &metadata, - std::shared_ptr *out) +Status TreeOperationLifetime::try_acquire(const std::shared_ptr &metadata, + std::shared_ptr *out) { return acquire_impl(metadata, out, true); } -Status TreeOperationLifetime::acquire_impl(const std::shared_ptr &metadata, +Status TreeOperationLifetime::acquire_impl(const std::shared_ptr &metadata, std::shared_ptr *out, bool temporary_capacity) { if (out == nullptr || *out) { diff --git a/lib/crowdb-tree/src/runtime/tree_operation_lifetime.h b/lib/crowdb-tree/src/runtime/tree_operation_lifetime.h index 2ab863430..7b0872be0 100644 --- a/lib/crowdb-tree/src/runtime/tree_operation_lifetime.h +++ b/lib/crowdb-tree/src/runtime/tree_operation_lifetime.h @@ -38,18 +38,16 @@ class TreeOperationLifetime std::shared_ptr state_; }; - explicit TreeOperationLifetime(size_t maximum, const std::shared_ptr &metadata = {}, + explicit TreeOperationLifetime(size_t maximum, const std::shared_ptr &metadata = {}, std::shared_ptr host_owner = {}); ~TreeOperationLifetime(); TreeOperationLifetime(const TreeOperationLifetime &) = delete; TreeOperationLifetime &operator=(const TreeOperationLifetime &) = delete; - [[nodiscard]] Status acquire(const std::shared_ptr &metadata, - std::shared_ptr *out); + [[nodiscard]] Status acquire(const std::shared_ptr &metadata, std::shared_ptr *out); // Before consensus, a full operation window is temporary admission // pressure. Allocation failure remains a resource error. - [[nodiscard]] Status try_acquire(const std::shared_ptr &metadata, - std::shared_ptr *out); + [[nodiscard]] Status try_acquire(const std::shared_ptr &metadata, std::shared_ptr *out); [[nodiscard]] ResourceUsage resource_usage() const noexcept; [[nodiscard]] static size_t allocation_bound() noexcept; [[nodiscard]] static size_t ticket_allocation_bound() noexcept; @@ -64,8 +62,8 @@ class TreeOperationLifetime void drain_accepted() noexcept; private: - Status acquire_impl(const std::shared_ptr &metadata, std::shared_ptr *out, - bool temporary_capacity); + Status acquire_impl(const std::shared_ptr &metadata, std::shared_ptr *out, + bool temporary_capacity); std::shared_ptr state_; }; } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/captured_write.cpp b/lib/crowdb-tree/src/snapshot/captured_write.cpp index 38998a4f2..aaa0cd331 100644 --- a/lib/crowdb-tree/src/snapshot/captured_write.cpp +++ b/lib/crowdb-tree/src/snapshot/captured_write.cpp @@ -14,12 +14,12 @@ bool valid_write_limits(const CapturedWriteLimits &limits, const PageLoadBudget return limits.max_frames != 0 && !limits.frames.plan_only && limits.max_dependency_steps != 0 && limits.frames.encode.frame_bytes <= limits.write.max_frame_bytes && PageWriteOperation::allocation_bound(limits.write).has_value() && - static_cast(budget.bytes) == static_cast(budget.io) && - (!limits.frames.encode.metadata || limits.frames.encode.metadata == budget.bytes) && - (!budget.bytes || - ((budget.bytes->category() == detail::BudgetClass::kRouting || - budget.bytes->category() == detail::BudgetClass::kReclaim) && - budget.io->category() == detail::BudgetClass::kIo && budget.bytes->account() == budget.io->account())); + static_cast(budget.reservation()) == static_cast(budget.io) && + (!limits.frames.encode.metadata || limits.frames.encode.metadata == budget.reservation()) && + (!budget.reservation() || ((budget.reservation()->category() == detail::BudgetClass::kRouting || + budget.reservation()->category() == detail::BudgetClass::kReclaim) && + budget.io->category() == detail::BudgetClass::kIo && + budget.reservation()->account() == budget.io->account())); } } // namespace @@ -67,7 +67,7 @@ Status CapturedWriteOperation::create(std::shared_ptr return Status::invalid_argument(); } try { - limits.frames.encode.metadata = budget.bytes; + limits.frames.encode.metadata = budget.reservation(); limits.frames.cpu_progress = budget.cpu_progress; if (budget.work_metrics) { limits.frames.encode.work_metrics = budget.work_metrics; @@ -79,7 +79,7 @@ Status CapturedWriteOperation::create(std::shared_ptr if (!status.ok()) { return status; } - auto pages = detail::reserved_shared(budget.bytes, budget.bytes); + auto pages = detail::reserved_shared(budget.reservation(), budget.reservation()); pages->root_page_id = captured->root_page_id; pages->next_page_id = captured->next_page_id; pages->coverage = captured->coverage; @@ -110,13 +110,13 @@ Status CapturedWriteOperation::create(std::shared_ptr return Status::not_supported(); } try { - limits.frames.encode.metadata = budget.bytes; + limits.frames.encode.metadata = budget.reservation(); limits.frames.cpu_progress = budget.cpu_progress; if (budget.work_metrics) { limits.frames.encode.work_metrics = budget.work_metrics; limits.frames.encode.cause = budget.cause; } - auto state = detail::reserved_shared(budget.bytes, budget.bytes); + auto state = detail::reserved_shared(budget.reservation(), budget.reservation()); state->store = std::move(store); state->runtime = std::move(runtime); state->lane = std::move(lane); @@ -126,7 +126,7 @@ Status CapturedWriteOperation::create(std::shared_ptr state->inherited = std::move(inherited); state->pages = std::move(pages); state->stream = std::move(stream); - *out = detail::reserved_construct(state->budget.bytes, [&](auto *storage) { + *out = detail::reserved_construct(state->budget.reservation(), [&](auto *storage) { ::new (static_cast(storage)) CapturedWriteOperation(std::move(state)); }); } diff --git a/lib/crowdb-tree/src/snapshot/captured_write_page.cpp b/lib/crowdb-tree/src/snapshot/captured_write_page.cpp index bf5053bcb..a3f54ab64 100644 --- a/lib/crowdb-tree/src/snapshot/captured_write_page.cpp +++ b/lib/crowdb-tree/src/snapshot/captured_write_page.cpp @@ -49,7 +49,7 @@ PageWriteOperation::ReserveExtent CapturedWriteOperation::State::reserve_page(bo done(Status::unavailable(), {}); return; } - auto event = detail::reserved_shared(self->budget.bytes); + auto event = detail::reserved_shared(self->budget.reservation()); event->complete = std::move(done); if (protect_dependency) { event->dependencies = self->dependencies; @@ -85,7 +85,7 @@ void CapturedWriteOperation::State::received_write() .logical_bytes = descriptor.logical_bytes}); } else if (descriptor.type == page_type::kLeafBase) { - auto primary = detail::reserved_shared(budget.bytes, *descriptor.backing); + auto primary = detail::reserved_shared(budget.reservation(), *descriptor.backing); primary->overflow = overflow_backing; descriptor.backing = std::move(primary); } diff --git a/lib/crowdb-tree/src/snapshot/captured_write_pump.cpp b/lib/crowdb-tree/src/snapshot/captured_write_pump.cpp index cd81e7e94..3cb66947a 100644 --- a/lib/crowdb-tree/src/snapshot/captured_write_pump.cpp +++ b/lib/crowdb-tree/src/snapshot/captured_write_pump.cpp @@ -127,8 +127,8 @@ void CapturedWriteOperation::State::received_frame() .page_id = frame->info.page_id, .type = frame->info.type, .frame_bytes = frame->info.bytes, .backing = {}}; descriptor.summary = frame->summary; if (!dependencies) { - dependencies = detail::reserved_shared(budget.bytes, budget.bytes); - overflow_backing = detail::reserved_shared(budget.bytes, budget.bytes); + dependencies = detail::reserved_shared(budget.reservation(), budget.reservation()); + overflow_backing = detail::reserved_shared(budget.reservation(), budget.reservation()); } if (frame->reuse) { descriptor = frame->reuse->primary->frame; @@ -140,9 +140,9 @@ void CapturedWriteOperation::State::received_frame() dependencies->ids = frame->info.dependencies; } else if (descriptor.type == page_type::kLeafBase) { - const auto status = - capture_stored_overflow(LeafFrameView(frame->buffer->bytes.data(), descriptor.frame_bytes), - limits.frames.encode.max_entries, budget.bytes, &descriptor.stored_overflow); + const auto status = capture_stored_overflow(LeafFrameView(frame->buffer->bytes.data(), descriptor.frame_bytes), + limits.frames.encode.max_entries, budget.reservation(), + &descriptor.stored_overflow); if (!status.ok()) { finish(status); return; diff --git a/lib/crowdb-tree/src/snapshot/captured_write_relocate.cpp b/lib/crowdb-tree/src/snapshot/captured_write_relocate.cpp index c7ee9785d..99cbf63ff 100644 --- a/lib/crowdb-tree/src/snapshot/captured_write_relocate.cpp +++ b/lib/crowdb-tree/src/snapshot/captured_write_relocate.cpp @@ -105,7 +105,7 @@ void CapturedWriteOperation::State::reuse_durable_dependency(DurableCapturedFram finish(Status::not_supported("dependency rebind requires independent physical extent ownership")); return; } - auto backing = detail::reserved_shared(budget.bytes, *value.backing); + auto backing = detail::reserved_shared(budget.reservation(), *value.backing); backing->owner = dependencies; value.backing = std::move(backing); if (found == pages->frames.end()) { @@ -123,7 +123,7 @@ Status CapturedWriteOperation::State::rebind_primary() if (!descriptor.backing->physical_owner) { return Status::not_supported("relocation requires independent primary extent ownership"); } - auto backing = detail::reserved_shared(budget.bytes, *descriptor.backing); + auto backing = detail::reserved_shared(budget.reservation(), *descriptor.backing); dependencies->owners.push_back(backing->physical_owner); backing->owner = dependencies; backing->overflow = overflow_backing; diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation.cpp index 865b38525..8917317d9 100644 --- a/lib/crowdb-tree/src/snapshot/checkpoint_operation.cpp +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation.cpp @@ -25,9 +25,10 @@ Status CheckpointOperation::create( local_valid == chunk_valid || !operation_store || !runtime || !lane || (!loader && !prepare_work && !prepare_work_async) || out == nullptr || (!chunk_valid && !operation_store->write_completion_is_reloadable()) || - static_cast(budget.bytes) != static_cast(finish_metadata) || - (finish_metadata && (finish_metadata == budget.bytes || finish_metadata->account() != budget.bytes->account() || - finish_metadata->category() == detail::BudgetClass::kIo))) { + static_cast(budget.reservation()) != static_cast(finish_metadata) || + (finish_metadata && + (finish_metadata == budget.reservation() || finish_metadata->account() != budget.reservation()->account() || + finish_metadata->category() == detail::BudgetClass::kIo))) { return Status::invalid_argument(); } if ((options.mode == CheckpointMode::Durable && @@ -49,7 +50,7 @@ Status CheckpointOperation::create( if (finish_metadata && finish_metadata->available() < finish_allocation_bound()) { return Status::resource_exhausted(); } - auto state = detail::reserved_shared(budget.bytes, publisher); + auto state = detail::reserved_shared(budget.reservation(), publisher); state->owner = std::move(owner); state->backend = std::move(backend); state->chunk = std::move(chunk); @@ -71,12 +72,13 @@ Status CheckpointOperation::create( state->on_terminal = std::move(on_terminal); state->cancel_work = std::move(cancel_work); state->cpu_progress = state->budget.cpu_progress; - state->control_metadata = state->budget.bytes; - state->capture_request = detail::reserved_shared(state->budget.bytes, state->budget.bytes); + state->control_metadata = state->budget.reservation(); + state->capture_request = + detail::reserved_shared(state->budget.reservation(), state->budget.reservation()); state->finish_request = detail::reserved_shared(state->finish_metadata, state->finish_metadata); if (state->options.mode == CheckpointMode::Durable) { - state->baseline = detail::reserved_shared(state->budget.bytes); + state->baseline = detail::reserved_shared(state->budget.reservation()); } if (!state->cpu_progress) { auto status = state->runtime->reserve_progress(state->lane, &State::run, state.get(), @@ -85,7 +87,7 @@ Status CheckpointOperation::create( return status; } } - *out = detail::reserved_construct(state->budget.bytes, [&](auto *storage) { + *out = detail::reserved_construct(state->budget.reservation(), [&](auto *storage) { ::new (static_cast(storage)) CheckpointOperation(std::move(state)); }); } diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation_plan.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation_plan.cpp index 12d1e14c2..9d1cd5271 100644 --- a/lib/crowdb-tree/src/snapshot/checkpoint_operation_plan.cpp +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation_plan.cpp @@ -106,11 +106,11 @@ void CheckpointOperation::State::begin_layout() auto limits = options.write.frames; limits.max_depth = depth; limits.plan_only = true; - limits.encode.metadata = budget.bytes; + limits.encode.metadata = budget.reservation(); limits.cpu_progress = budget.cpu_progress; - planner = detail::reserved_unique(budget.bytes, view->next_page_id, backend_identity(), - options.write, options.metadata.max_segments, - inherited_pages(), budget.bytes); + planner = detail::reserved_unique( + budget.reservation(), view->next_page_id, backend_identity(), options.write, options.metadata.max_segments, + inherited_pages(), budget.reservation()); auto status = CapturedFrameStream::create(view, loader, runtime, lane, std::move(limits), {}, &planning_stream); if (!status.ok()) { finish(std::move(status)); @@ -186,8 +186,8 @@ void CheckpointOperation::State::planned() void CheckpointOperation::State::begin_materialization() { - writeback_metrics = detail::reserved_shared(budget.bytes, publisher.work_metrics(), budget.bytes, - plan.primary_frames); + writeback_metrics = detail::reserved_shared(budget.reservation(), publisher.work_metrics(), + budget.reservation(), plan.primary_frames); options.write.frames.writeback_metrics = writeback_metrics; capture(); } diff --git a/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp b/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp index 4742dba5a..6ef0838ee 100644 --- a/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp +++ b/lib/crowdb-tree/src/snapshot/checkpoint_operation_steps.cpp @@ -277,7 +277,7 @@ void CheckpointOperation::State::captured() const auto end = overflow_begin + plan.new_overflow_frames; // The publication owns the advanced high-water mark. A tiny independent // owner, rather than this operation, protects the emitted interval. - auto lease = detail::reserved_shared(budget.bytes); + auto lease = detail::reserved_shared(budget.reservation()); lease->first = overflow_begin; lease->count = plan.new_overflow_frames; auto reserve = [lease, next = overflow_begin, end](uint64_t count, ReservedOverflowIds *out) mutable { @@ -290,7 +290,7 @@ void CheckpointOperation::State::captured() }; auto status = CapturedWriteOperation::create( view, loader, store, runtime, lane, options.write, budget, std::move(reserve), - chunk ? chunk->reserve(budget.bytes) : extent_write_reservation(backend->ledger, budget.bytes), + chunk ? chunk->reserve(budget.reservation()) : extent_write_reservation(backend->ledger, budget.reservation()), inherited_pages(), &write); if (!status.ok()) { reconcile(std::move(status)); @@ -392,8 +392,8 @@ void CheckpointOperation::State::begin_rebase() finish_rebase(Status::unavailable()); return; } - auto status = CapturedRebaseOperation::create(publisher, view, written_pages, runtime, lane, budget.bytes, &rebase, - options.write.frames.encode.max_sources, writeback_metrics, + auto status = CapturedRebaseOperation::create(publisher, view, written_pages, runtime, lane, budget.reservation(), + &rebase, options.write.frames.encode.max_sources, writeback_metrics, cpu_progress, options.publication, options.cause); if (!status.ok()) { finish_rebase(std::move(status)); diff --git a/lib/crowdb-tree/src/snapshot/flush_persistence.h b/lib/crowdb-tree/src/snapshot/flush_persistence.h index 81f732ebe..040c078ec 100644 --- a/lib/crowdb-tree/src/snapshot/flush_persistence.h +++ b/lib/crowdb-tree/src/snapshot/flush_persistence.h @@ -16,15 +16,10 @@ class FlushPersistence : public std::enable_shared_from_this public: explicit FlushPersistence(std::shared_ptr preparation, uint64_t wal_offset = 0); - [[nodiscard]] static size_t control_allocation_bound(size_t workers) noexcept; + [[nodiscard]] static size_t control_allocation_bound() noexcept; [[nodiscard]] static size_t commit_control_allocation_bound() noexcept; void configure(FlushBatchOptions &options); - void bind_io(std::shared_ptr io) - { - io_ = std::move(io); - } - void write(PageEncodeInput input, FlushBatchOptions::ReserveOverflow reserve, FlushBatchOptions::PersistComplete done); void commit(std::shared_ptr view, FlushBatch::MaintenanceComplete done); @@ -41,7 +36,6 @@ class FlushPersistence : public std::enable_shared_from_this CheckpointResult report_; bool finished_ = false; uint64_t wal_offset_ = 0; - std::shared_ptr io_; std::shared_ptr selection_metadata_; std::optional> pending_leaves_; void select_inputs(); diff --git a/lib/crowdb-tree/src/snapshot/flush_persistence_commit.cpp b/lib/crowdb-tree/src/snapshot/flush_persistence_commit.cpp index 8a9e1f361..969bebb18 100644 --- a/lib/crowdb-tree/src/snapshot/flush_persistence_commit.cpp +++ b/lib/crowdb-tree/src/snapshot/flush_persistence_commit.cpp @@ -54,10 +54,13 @@ struct FlushPersistence::Commit : std::enable_shared_from_this std::shared_ptr chunk; std::optional attempt; Status status; + std::optional metrics; ApplyDispatcher::Node node{&Commit::run, this}; void start() { + metrics.emplace(owner->preparation_->core->publisher().work_metrics(), TreeStage::SnapshotCommit, + tree_metrics_now_ns()); auto &p = *owner->preparation_; auto source = owner->pages_ ? std::shared_ptr(owner->pages_) : owner->inherited_; CapturedWritePlan plan; @@ -75,7 +78,7 @@ struct FlushPersistence::Commit : std::enable_shared_from_this finish(Status::resource_exhausted("flush metadata bound overflow")); return; } - auto result = p.work(*view, &plan, &work, *bound, owner->io_); + auto result = p.work(*view, &plan, &work, *bound); if (!result.ok()) { reconcile(std::move(result)); return; @@ -83,7 +86,8 @@ struct FlushPersistence::Commit : std::enable_shared_from_this if (!source || source->coverage != view->coverage || source->generation != view->generation || source->root_page_id != view->root_page_id || source->next_page_id != view->next_page_id || owner->pages_) { if (!owner->pages_) { - owner->pages_ = detail::reserved_shared(work.budget.bytes, work.budget.bytes); + owner->pages_ = + detail::reserved_shared(work.budget.reservation(), work.budget.reservation()); if (source) { owner->pages_->frames.inherit(source->frames); owner->pages_->overflow_counts_known = source->overflow_counts_known; @@ -91,7 +95,7 @@ struct FlushPersistence::Commit : std::enable_shared_from_this } } auto &catalog = *owner->pages_; - catalog.frames.budget(work.budget.bytes); + catalog.frames.budget(work.budget.reservation()); result = catalog.initialize_overflow_counts(); if (!result.ok()) { reconcile(std::move(result)); @@ -273,6 +277,9 @@ struct FlushPersistence::Commit : std::enable_shared_from_this void finish(Status result) noexcept { + if (metrics) { + (void)metrics->finish(result.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + } owner->report_.status = result; owner->finished_ = true; owner->pages_.reset(); diff --git a/lib/crowdb-tree/src/snapshot/flush_persistence_page.cpp b/lib/crowdb-tree/src/snapshot/flush_persistence_page.cpp index 449ba71a9..c435ee610 100644 --- a/lib/crowdb-tree/src/snapshot/flush_persistence_page.cpp +++ b/lib/crowdb-tree/src/snapshot/flush_persistence_page.cpp @@ -71,12 +71,15 @@ struct FlushPersistence::Page : std::enable_shared_from_this std::shared_ptr write; std::optional attempt; enum class Stage : uint8_t { Plan, Dependencies, Write }; - Stage stage = Stage::Plan; - Status status; - ApplyDispatcher::Node node{&Page::run, this}; + Stage stage = Stage::Plan; + Status status; + std::optional metrics; + ApplyDispatcher::Node node{&Page::run, this}; void start() { + metrics.emplace(owner->preparation_->core->publisher().work_metrics(), TreeStage::PageWritebackPipeline, + tree_metrics_now_ns()); auto &preparation = *owner->preparation_; limits = preparation.options.write; const auto bytes = final_frame_bytes(*input.page, limits.frames.encode.frame_bytes); @@ -98,19 +101,19 @@ struct FlushPersistence::Page : std::enable_shared_from_this limits.frames.reuse_backend_id = preparation.core->local_backend() ? preparation.core->local_backend()->ledger->limits().backend_id : preparation.core->chunk_backend()->backend_identity(); - auto result = preparation.work(*owner->initial_, nullptr, &work, 0, owner->io_, &limits); + auto result = preparation.work(*owner->initial_, nullptr, &work, 0, {}, &limits); if (!result.ok()) { finish(std::move(result)); return; } - limits.frames.encode.metadata = work.budget.bytes; + limits.frames.encode.metadata = work.budget.reservation(); limits.frames.encode.work_metrics = work.budget.work_metrics; limits.frames.encode.cause = work.budget.cause; limits.frames.cpu_progress = work.budget.cpu_progress; limits.frames.plan_only = true; planner = detail::reserved_unique( - work.budget.bytes, input.next_page_id, limits.frames.reuse_backend_id, limits, - preparation.options.metadata.max_segments, owner->inherited_, work.budget.bytes); + work.budget.reservation(), input.next_page_id, limits.frames.reuse_backend_id, limits, + preparation.options.metadata.max_segments, owner->inherited_, work.budget.reservation()); result = CapturedFrameStream::create(input, work.loader, preparation.core->binding()->cpu(), preparation.core->binding()->lane(), limits.frames, {}, &stream); if (!result.ok()) { @@ -216,12 +219,12 @@ struct FlushPersistence::Page : std::enable_shared_from_this finish(Status::resource_exhausted("durable mapping segment bound overflow")); return; } - result = preparation.work(*owner->initial_, &plan, &work, *bound, owner->io_, &limits); + result = preparation.work(*owner->initial_, &plan, &work, *bound, {}, &limits); if (!result.ok()) { finish(std::move(result)); return; } - limits.frames.encode.metadata = work.budget.bytes; + limits.frames.encode.metadata = work.budget.reservation(); limits.frames.plan_only = false; result = CapturedFrameStream::create(input, work.loader, preparation.core->binding()->cpu(), preparation.core->binding()->lane(), limits.frames, reserve, &stream); @@ -229,7 +232,8 @@ struct FlushPersistence::Page : std::enable_shared_from_this finish(std::move(result)); return; } - auto pages = detail::reserved_shared(work.budget.bytes, work.budget.bytes); + auto pages = + detail::reserved_shared(work.budget.reservation(), work.budget.reservation()); pages->coverage = input.coverage; pages->generation = owner->initial_->generation; pages->root_page_id = input.page_id; @@ -238,8 +242,8 @@ struct FlushPersistence::Page : std::enable_shared_from_this result = CapturedWriteOperation::create( stream, pages, preparation.ticket->backend(preparation.core->async_backend()), preparation.core->binding()->cpu(), preparation.core->binding()->lane(), limits, work.budget, - chunk ? chunk->reserve(work.budget.bytes) - : extent_write_reservation(preparation.core->local_backend()->ledger, work.budget.bytes), + chunk ? chunk->reserve(work.budget.reservation()) + : extent_write_reservation(preparation.core->local_backend()->ledger, work.budget.reservation()), owner->pages_ ? owner->pages_ : owner->inherited_, &write); if (!result.ok()) { finish(std::move(result)); @@ -266,7 +270,7 @@ struct FlushPersistence::Page : std::enable_shared_from_this receipt.summary->live_value_bytes = input.children_summary->live_value_bytes; } std::shared_ptr replacement; - auto status = build_writeback_rebase(input.page, input.page, receipt, input.coverage, work.budget.bytes, + auto status = build_writeback_rebase(input.page, input.page, receipt, input.coverage, work.budget.reservation(), &replacement, limits.frames.encode.max_sources, input.gc_floor); if (!status.ok()) { finish(std::move(status)); @@ -275,7 +279,8 @@ struct FlushPersistence::Page : std::enable_shared_from_this // The catalog receives only final page receipts and their protected raw // dependencies. Structural deletions are applied at the root join. if (!owner->pages_) { - owner->pages_ = detail::reserved_shared(work.budget.bytes, work.budget.bytes); + owner->pages_ = + detail::reserved_shared(work.budget.reservation(), work.budget.reservation()); if (owner->inherited_) { owner->pages_->frames.inherit(owner->inherited_->frames); owner->pages_->overflow_counts_known = owner->inherited_->overflow_counts_known; @@ -287,7 +292,7 @@ struct FlushPersistence::Page : std::enable_shared_from_this return; } } - owner->pages_->frames.budget(work.budget.bytes); + owner->pages_->frames.budget(work.budget.reservation()); status = owner->pages_->replace_primary(*result.pages); if (status.ok()) { owner->pages_->frames.at(input.page_id).summary = receipt.summary; @@ -301,11 +306,17 @@ struct FlushPersistence::Page : std::enable_shared_from_this owner->report_.reused_frames += result.reused_frames; ++owner->report_.primary_frames; ++owner->report_.rebased_pages; + if (metrics) { + (void)metrics->finish(TreeOutcome::Success, tree_metrics_now_ns()); + } done(Status::Ok(), std::move(replacement)); } void finish(Status result) noexcept { + if (metrics) { + (void)metrics->finish(result.ok() ? TreeOutcome::Success : TreeOutcome::Error, tree_metrics_now_ns()); + } if (!result.ok()) { CRB_LOG_ERROR("persistent page {} failed: {}", input.page_id, result.to_string()); } @@ -350,7 +361,7 @@ try { done(Status::Ok(), input.page); return; } - auto page = detail::reserved_shared(preparation_->control); + auto page = detail::reserved_shared(preparation_->core->read_limits().metadata); page->owner = shared_from_this(); page->input = std::move(input); page->reserve = std::move(reserve); @@ -361,9 +372,8 @@ catch (const std::bad_alloc &) { done(Status::resource_exhausted("flush page control reserve exhausted"), {}); } -size_t FlushPersistence::control_allocation_bound(size_t workers) noexcept +size_t FlushPersistence::control_allocation_bound() noexcept { - return detail::shared_allocation_bound() + commit_control_allocation_bound() + - ((workers + 2) * detail::shared_allocation_bound()); + return detail::shared_allocation_bound() + commit_control_allocation_bound(); } } // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp index c359ba4b8..1d2e26953 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint.cpp @@ -22,7 +22,7 @@ LocalCheckpointOperation::State::Window::Window(size_t size, size_t alignment, LocalCheckpointOperation::State::State(PageLoadBudget credits) : budget(std::move(credits)), - directory(detail::ReservationAllocator(budget.bytes)) + directory(detail::ReservationAllocator(budget.reservation())) { } @@ -34,26 +34,27 @@ Status LocalCheckpointOperation::create(std::shared_ptr *out) { if (!pages || !registry || !store || !runtime || !lane || out == nullptr || limits.max_segments == 0 || - limits.max_segments > kMaxMappingSegments || static_cast(budget.bytes) != static_cast(budget.io) || - (budget.bytes && - ((budget.bytes->category() != detail::BudgetClass::kRouting && - budget.bytes->category() != detail::BudgetClass::kReclaim) || - budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + limits.max_segments > kMaxMappingSegments || + static_cast(budget.reservation()) != static_cast(budget.io) || + (budget.reservation() && ((budget.reservation()->category() != detail::BudgetClass::kRouting && + budget.reservation()->category() != detail::BudgetClass::kReclaim) || + budget.io->category() != detail::BudgetClass::kIo || + budget.reservation()->account() != budget.io->account()))) { return Status::invalid_argument(); } if (!store->write_completion_is_reloadable()) { return Status::not_supported(); } try { - auto state = detail::reserved_shared(budget.bytes, budget); + auto state = detail::reserved_shared(budget.reservation(), budget); state->limits = limits; state->registry = std::move(registry); state->store = std::move(store); state->runtime = std::move(runtime); state->lane = std::move(lane); - state->image = detail::reserved_shared(budget.bytes, budget.bytes); + state->image = detail::reserved_shared(budget.reservation(), budget.reservation()); state->image->pages = std::move(pages); - *out = detail::reserved_construct(budget.bytes, [&](auto *storage) { + *out = detail::reserved_construct(budget.reservation(), [&](auto *storage) { ::new (static_cast(storage)) LocalCheckpointOperation(std::move(state)); }); } @@ -134,9 +135,9 @@ void LocalCheckpointOperation::State::begin() previous = sequence == 0 ? nullptr : registry->anchors_[sequence % 2 == 0 ? 1 : 0]; iu = registry->ledger_->limits().iu_size; segment_bytes = round_up_to_iu(segment_image_encoded_size(kMappingSegmentSize), iu); - window = detail::reserved_shared(budget.bytes, + window = detail::reserved_shared(budget.reservation(), std::max(segment_bytes, snapshot::superblock_slot_bytes(iu)), - std::bit_ceil(std::max(4096, iu)), budget.bytes); + std::bit_ceil(std::max(4096, iu)), budget.reservation()); auto &anchor = image->anchor; anchor.magic = snapshot::kAnchorMagic; anchor.format_version = snapshot::kFormatVersion; diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_io.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_io.cpp index 870111d63..b91fd8b89 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint_io.cpp +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_io.cpp @@ -35,7 +35,7 @@ void LocalCheckpointOperation::State::IoStep::publish() noexcept void LocalCheckpointOperation::State::io(bool barrier, uint64_t address, size_t bytes, Stage next, bool anchor) { - auto step = detail::reserved_shared(budget.bytes, budget); + auto step = detail::reserved_shared(budget.reservation(), budget); step->owner = shared_from_this(); step->window = window; step->checkpoint = image; diff --git a/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp b/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp index 4bef7b778..27e3ee59e 100644 --- a/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp +++ b/lib/crowdb-tree/src/snapshot/local_checkpoint_metadata.cpp @@ -185,11 +185,11 @@ void LocalCheckpointOperation::State::reserve(uint64_t length, Stage next) stage = next; waiting = true; event_ready.store(false, std::memory_order_release); - auto status = - registry->ledger_->reserve(length, budget.bytes, [self = shared_from_this()](ExtentLedgerResult result) { - self->reserved = std::move(result.extent); - self->event(std::move(result.status)); - }); + auto status = registry->ledger_->reserve(length, budget.reservation(), + [self = shared_from_this()](ExtentLedgerResult result) { + self->reserved = std::move(result.extent); + self->event(std::move(result.status)); + }); if (!status.ok()) { event(std::move(status)); } diff --git a/lib/crowdb-tree/src/snapshot/local_recovery.cpp b/lib/crowdb-tree/src/snapshot/local_recovery.cpp index c36bb126f..cbc23417e 100644 --- a/lib/crowdb-tree/src/snapshot/local_recovery.cpp +++ b/lib/crowdb-tree/src/snapshot/local_recovery.cpp @@ -23,9 +23,9 @@ LocalRecoveryOperation::State::Window::Window(size_t blob_bytes, size_t frame_by LocalRecoveryOperation::State::State(PageLoadBudget credits) : budget(std::move(credits)), - directory(detail::ReservationAllocator(budget.bytes)), - references(detail::ReservationAllocator(budget.bytes)), - report(detail::reserved_shared(budget.bytes)) + directory(detail::ReservationAllocator(budget.reservation())), + references(detail::ReservationAllocator(budget.reservation())), + report(detail::reserved_shared(budget.reservation())) { } @@ -60,15 +60,15 @@ Status LocalRecoveryOperation::create(std::shared_ptr limits.max_blob_bytes < kDurableBlobHeader || limits.max_blob_bytes > UINT32_MAX || limits.max_frame_bytes < kFrameHeaderSize + kFrameTrailerSize || limits.max_frame_bytes > UINT32_MAX || limits.max_overflow_references == 0 || limits.max_dependency_steps == 0 || - static_cast(budget.bytes) != static_cast(budget.io) || - (budget.bytes && - ((budget.bytes->category() != detail::BudgetClass::kRouting && - budget.bytes->category() != detail::BudgetClass::kReclaim) || - budget.io->category() != detail::BudgetClass::kIo || budget.bytes->account() != budget.io->account()))) { + static_cast(budget.reservation()) != static_cast(budget.io) || + (budget.reservation() && ((budget.reservation()->category() != detail::BudgetClass::kRouting && + budget.reservation()->category() != detail::BudgetClass::kReclaim) || + budget.io->category() != detail::BudgetClass::kIo || + budget.reservation()->account() != budget.io->account()))) { return Status::invalid_argument(); } try { - auto state = detail::reserved_shared(budget.bytes, budget); + auto state = detail::reserved_shared(budget.reservation(), budget); state->readable_bytes = domain.file_size; domain.file_size = std::max(domain.file_size, domain.region_begin); state->domain = domain; @@ -76,15 +76,15 @@ Status LocalRecoveryOperation::create(std::shared_ptr state->store = std::move(store); state->runtime = std::move(runtime); state->lane = std::move(lane); - auto status = ExtentLedger::create(domain, budget.bytes, &state->ledger); + auto status = ExtentLedger::create(domain, budget.reservation(), &state->ledger); if (!status.ok()) { return status; } state->metadata_window = round_up_to_iu(segment_image_encoded_size(kMappingSegmentSize), domain.iu_size); state->window = detail::reserved_shared( - budget.bytes, std::max(limits.max_blob_bytes, state->metadata_window), limits.max_frame_bytes, - std::bit_ceil(std::max(4096, domain.iu_size)), budget.bytes); - *out = detail::reserved_construct(budget.bytes, [&](auto *storage) { + budget.reservation(), std::max(limits.max_blob_bytes, state->metadata_window), limits.max_frame_bytes, + std::bit_ceil(std::max(4096, domain.iu_size)), budget.reservation()); + *out = detail::reserved_construct(budget.reservation(), [&](auto *storage) { ::new (static_cast(storage)) LocalRecoveryOperation(std::move(state)); }); } diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_frames.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_frames.cpp index 5bbd66bd5..f8b4da421 100644 --- a/lib/crowdb-tree/src/snapshot/local_recovery_frames.cpp +++ b/lib/crowdb-tree/src/snapshot/local_recovery_frames.cpp @@ -106,8 +106,8 @@ void LocalRecoveryOperation::State::frame_read() return; } } - status = - capture_stored_overflow(leaf, limits.max_overflow_references, budget.bytes, &descriptor.stored_overflow); + status = capture_stored_overflow(leaf, limits.max_overflow_references, budget.reservation(), + &descriptor.stored_overflow); if (!status.ok()) { finish(std::move(status)); return; @@ -158,10 +158,10 @@ void LocalRecoveryOperation::State::dependencies() finish(Status::corruption()); return; } - protection = detail::reserved_shared(budget.bytes, budget.bytes); - overflow_backing = detail::reserved_shared(budget.bytes, budget.bytes); + protection = detail::reserved_shared(budget.reservation(), budget.reservation()); + overflow_backing = detail::reserved_shared(budget.reservation(), budget.reservation()); protection->primary = leaf->second.backing->owner; - auto backing = detail::reserved_shared(budget.bytes, *leaf->second.backing); + auto backing = detail::reserved_shared(budget.reservation(), *leaf->second.backing); backing->owner = protection; backing->overflow = overflow_backing; leaf->second.backing = std::move(backing); diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_io.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_io.cpp index 9a7c49eab..15c8a428d 100644 --- a/lib/crowdb-tree/src/snapshot/local_recovery_io.cpp +++ b/lib/crowdb-tree/src/snapshot/local_recovery_io.cpp @@ -39,7 +39,7 @@ void LocalRecoveryOperation::State::read(uint64_t address, size_t bytes, Stage n finish(Status::resource_exhausted()); return; } - auto step = detail::reserved_shared(budget.bytes, budget); + auto step = detail::reserved_shared(budget.reservation(), budget); step->owner = shared_from_this(); step->window = window; read_bytes = bytes; @@ -71,10 +71,11 @@ void LocalRecoveryOperation::State::protect(uint64_t address, uint64_t bytes, St stage = next; waiting = true; event_ready.store(false, std::memory_order_release); - auto status = ledger->recover(address, bytes, budget.bytes, [self = shared_from_this()](ExtentLedgerResult result) { - self->recovered = std::move(result.extent); - self->event(std::move(result.status)); - }); + auto status = + ledger->recover(address, bytes, budget.reservation(), [self = shared_from_this()](ExtentLedgerResult result) { + self->recovered = std::move(result.extent); + self->event(std::move(result.status)); + }); if (!status.ok()) { event(std::move(status)); } diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp index e312a12a1..75d1bdfa6 100644 --- a/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp +++ b/lib/crowdb-tree/src/snapshot/local_recovery_metadata.cpp @@ -13,7 +13,7 @@ void LocalRecoveryOperation::State::next_anchor() stage = Stage::Activated; waiting = true; event_ready.store(false, std::memory_order_release); - auto status = ledger->activate(budget.bytes, [self = shared_from_this()](ExtentLedgerResult result) { + auto status = ledger->activate(budget.reservation(), [self = shared_from_this()](ExtentLedgerResult result) { self->event(std::move(result.status)); }); if (!status.ok()) { @@ -46,9 +46,9 @@ void LocalRecoveryOperation::State::anchor_read() finish(Status::corruption()); return; } - image = detail::reserved_shared(budget.bytes, budget.bytes); + image = detail::reserved_shared(budget.reservation(), budget.reservation()); image->anchor = anchor; - pages = detail::reserved_shared(budget.bytes, budget.bytes); + pages = detail::reserved_shared(budget.reservation(), budget.reservation()); pages->root_page_id = anchor.root_page_id; pages->next_page_id = anchor.next_page_id; pages->coverage = anchor.last_applied_slot; @@ -181,7 +181,7 @@ void LocalRecoveryOperation::State::page_extent() void LocalRecoveryOperation::State::page_protected() { - auto backing = detail::reserved_shared(budget.bytes); + auto backing = detail::reserved_shared(budget.reservation()); backing->backend_id = recovered.backend_id; backing->generation = recovered.generation; backing->content_id = recovered.identity; diff --git a/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp b/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp index 60a548b46..0e69c64be 100644 --- a/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp +++ b/lib/crowdb-tree/src/snapshot/local_recovery_pump.cpp @@ -123,7 +123,7 @@ void LocalRecoveryOperation::State::pump() dependencies(); break; case Stage::Activated: - finish(LocalCheckpointRegistry::create(ledger, budget.bytes, anchors, &report->registry)); + finish(LocalCheckpointRegistry::create(ledger, budget.reservation(), anchors, &report->registry)); break; } } diff --git a/lib/crowdb-tree/src/snapshot/native_frame_read.cpp b/lib/crowdb-tree/src/snapshot/native_frame_read.cpp index 1b9b43635..e98e172f4 100644 --- a/lib/crowdb-tree/src/snapshot/native_frame_read.cpp +++ b/lib/crowdb-tree/src/snapshot/native_frame_read.cpp @@ -30,7 +30,7 @@ struct NativeFrameRead::State : PageLoadProgress, std::enable_shared_from_this(metadata)) { } @@ -187,9 +187,9 @@ Status NativeFrameRead::create(uint64_t pid, OverflowBackingPage page, std::shar std::shared_ptr state; auto status = [&]() -> Status { try { - if (out == nullptr || *out || pid == kInvalidPageId || !runtime || !lane || !budget.bytes || - budget.bytes->category() == detail::BudgetClass::kIo || !page.backing || !page.backing->readable() || - page.backing->encoding != 0 || page.bytes == 0 || + if (out == nullptr || *out || pid == kInvalidPageId || !runtime || !lane || !budget.reservation() || + budget.reservation()->category() == detail::BudgetClass::kIo || !page.backing || + !page.backing->readable() || page.backing->encoding != 0 || page.bytes == 0 || page.frame_bytes <= kFrameHeaderSize + kFrameTrailerSize || page.bytes > overflow_chunk_cap(page.frame_bytes) || page.next == pid) { return Status::invalid_argument("native frame read requires a protected bounded descriptor"); @@ -202,8 +202,8 @@ Status NativeFrameRead::create(uint64_t pid, OverflowBackingPage page, std::shar } } else if (!store || !budget.io || budget.io->category() != detail::BudgetClass::kIo || - budget.bytes->account() != budget.io->account() || page.logical_bytes < kDurableBlobHeader || - page.logical_bytes > page.backing->length) { + budget.reservation()->account() != budget.io->account() || + page.logical_bytes < kDurableBlobHeader || page.logical_bytes > page.backing->length) { return Status::invalid_argument( "native durable frame read needs matching I/O credit and blob geometry"); } @@ -215,14 +215,15 @@ Status NativeFrameRead::create(uint64_t pid, OverflowBackingPage page, std::shar if (resident) { progress.io.reset(); } - state = detail::reserved_shared(budget.bytes, progress, started_at_ns); + state = detail::reserved_shared(budget.reservation(), progress, started_at_ns); state->pid = pid; state->descriptor = std::move(page); state->store = std::move(store); state->runtime = std::move(runtime); state->lane = std::move(lane); - *out = detail::reserved_construct( - budget.bytes, [&](auto *storage) { ::new (static_cast(storage)) NativeFrameRead(state); }); + *out = detail::reserved_construct(budget.reservation(), [&](auto *storage) { + ::new (static_cast(storage)) NativeFrameRead(state); + }); return Status::Ok(); } catch (const std::bad_alloc &) { diff --git a/lib/crowdb-tree/src/snapshot/primary_backing.h b/lib/crowdb-tree/src/snapshot/primary_backing.h index 10af25018..f52d6bda3 100644 --- a/lib/crowdb-tree/src/snapshot/primary_backing.h +++ b/lib/crowdb-tree/src/snapshot/primary_backing.h @@ -23,7 +23,7 @@ struct PrimaryBackingMetadata struct PrimaryReuseOutput { - explicit PrimaryReuseOutput(const std::shared_ptr &metadata) + explicit PrimaryReuseOutput(const std::shared_ptr &metadata) : dependencies(detail::ReservationAllocator(metadata)) { } diff --git a/lib/crowdb-tree/src/snapshot/published_native.cpp b/lib/crowdb-tree/src/snapshot/published_native.cpp index 8280371d9..bdad90ea8 100644 --- a/lib/crowdb-tree/src/snapshot/published_native.cpp +++ b/lib/crowdb-tree/src/snapshot/published_native.cpp @@ -32,9 +32,9 @@ Status plan_native(const std::shared_ptr &snapshot, Ca if (!status.ok()) { return status; } - frames.encode.metadata = window.budget.bytes; - auto planner = detail::reserved_shared(window.budget.bytes, snapshot->view(), limits, - window.budget.bytes, backend_id, incarnation); + frames.encode.metadata = window.budget.reservation(); + auto planner = detail::reserved_shared(window.budget.reservation(), snapshot->view(), limits, + window.budget.reservation(), backend_id, incarnation); std::shared_ptr stream; status = CapturedFrameStream::create(snapshot->view(), window.loader, snapshot->binding()->cpu(), snapshot->lane(), frames, {}, &stream); @@ -43,7 +43,7 @@ Status plan_native(const std::shared_ptr &snapshot, Ca } std::shared_ptr plan; status = NativePlanOperation::create(std::move(stream), planner, snapshot->binding()->cpu(), snapshot->lane(), - window.budget.bytes, &plan); + window.budget.reservation(), &plan); if (status.ok()) { status = detail::wait_snapshot_operation(plan); if (!status.ok()) { diff --git a/lib/crowdb-tree/src/snapshot/published_native_next.cpp b/lib/crowdb-tree/src/snapshot/published_native_next.cpp index 4bac0b653..3b637761a 100644 --- a/lib/crowdb-tree/src/snapshot/published_native_next.cpp +++ b/lib/crowdb-tree/src/snapshot/published_native_next.cpp @@ -52,8 +52,8 @@ Status PublishedNativeCursor::State::generated_frame(const SnapshotReadWindow // The encoder deliberately reuses one buffer. Copy only this frame // into an independently charged owner before advancing that buffer. auto bytes = detail::reserved_shared( - window.budget.bytes, std::span(frame.buffer->bytes.data(), frame.info.bytes), - window.budget.bytes); + window.budget.reservation(), std::span(frame.buffer->bytes.data(), frame.info.bytes), + window.budget.reservation()); *out = NativeFrameView{.page_id = frame.info.page_id, .bytes = bytes->bytes, .owner = std::move(bytes), diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot.cpp index a73635aef..a2baf746d 100644 --- a/lib/crowdb-tree/src/snapshot/published_snapshot.cpp +++ b/lib/crowdb-tree/src/snapshot/published_snapshot.cpp @@ -84,8 +84,10 @@ void SnapshotReadWindow::seal() noexcept if (budget.io) { budget.io->seal(); } - if (budget.bytes) { - budget.bytes->seal(); + if (!budget.account) { + if (auto reserved = budget.reservation()) { + reserved->seal(); + } } } @@ -163,10 +165,9 @@ try { state->loads.max_inflight = 1; state->reads = core->read_limits(); state->reads.max_memtables = 1; - state->reads.metadata.reset(); - state->backend = core->config().async_page_store; - state->detached = (state->backend == nullptr && core->config().page_store == nullptr) || - static_cast(core->config().backend_owner); + state->backend = core->config().async_page_store; + state->detached = (state->backend == nullptr && core->config().page_store == nullptr) || + static_cast(core->config().backend_owner); if (state->detached) { state->owner = reserved_shared(metadata); state->owner->binding = state->binding; @@ -217,9 +218,35 @@ try { } SnapshotReadWindow window; window.budget.bytes = AllocationReservation::create(std::move(*credit)); - window.budget.io = reserved_shared(window.budget.bytes, std::move(*io)); + window.budget.io = reserved_shared(window.budget.reservation(), std::move(*io)); + window.budget.work_metrics = state_->work_metrics; + window.budget.cause = TreeWorkCause::Snapshot; + return finish_window(std::move(window), out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot read window allocation failed"); +} + +Status PublishedSnapshotContext::prepare_window(SnapshotReadWindow *out) const +try { + using namespace detail; + if (out == nullptr || out->host_core || out->budget.bytes || out->loader || out->ticket) { + return Status::invalid_argument("snapshot read window output must be empty"); + } + SnapshotReadWindow window; + window.budget.bytes = state_->reads.metadata; + window.budget.account = state_->binding->account(); window.budget.work_metrics = state_->work_metrics; window.budget.cause = TreeWorkCause::Snapshot; + return finish_window(std::move(window), out); +} +catch (const std::bad_alloc &) { + return Status::resource_exhausted("snapshot read window allocation failed"); +} + +Status PublishedSnapshotContext::finish_window(SnapshotReadWindow window, SnapshotReadWindow *out) const +try { + using namespace detail; Status status; AsyncPageStore *backend = state_->backend; if (state_->detached) { @@ -237,15 +264,6 @@ try { if (!status.ok()) { return status; } - status = reserve_backend_operation(backend, state_->binding->account(), - {.max_io_bytes = state_->loads.max_blob_bytes, - .write_calls = 0, - .workers = state_->binding->cpu()->worker_count(), - .barriers = false}, - &window.budget.backend); - if (!status.ok()) { - return status; - } window.backend = window.ticket->backend(backend); window.loader = reserved_shared(window.budget.bytes, window.backend, state_->binding->cpu(), state_->lane, diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot.h b/lib/crowdb-tree/src/snapshot/published_snapshot.h index c36de1b64..4f7418205 100644 --- a/lib/crowdb-tree/src/snapshot/published_snapshot.h +++ b/lib/crowdb-tree/src/snapshot/published_snapshot.h @@ -48,6 +48,7 @@ class PublishedSnapshotContext // bytes covers the caller's operation, loader and output capacities. // This adds reservation-control/ticket overhead and reserves one I/O. Status prepare_window(size_t bytes, SnapshotReadWindow *out) const; + Status prepare_window(SnapshotReadWindow *out) const; Status get(Slice key, GetView *out) const; Status scan(CoreScanOptions options, ScanPackedBuf *out, size_t *count, bool *truncated) const; @@ -69,6 +70,8 @@ class PublishedSnapshotContext { } + Status finish_window(SnapshotReadWindow window, SnapshotReadWindow *out) const; + std::shared_ptr state_; }; diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_entries.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot_entries.cpp index 91dcbb67f..d8367c14f 100644 --- a/lib/crowdb-tree/src/snapshot/published_snapshot_entries.cpp +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_entries.cpp @@ -2,30 +2,19 @@ // Licensed under the Apache License, Version 2.0. #include "snapshot/published_snapshot_entries.h" -#include "btree/allocation_bound.h" -#include "btree/read_value_bound.h" #include "snapshot/operation_wait.h" -#include "snapshot/published_snapshot_budget.h" namespace crowdb::tree { struct PublishedSnapshotEntries::State { - ~State() - { - if (metadata) { - metadata->seal(); - } - } - - std::shared_ptr snapshot; - std::shared_ptr metadata; - std::shared_ptr loader; - std::shared_ptr scan; - size_t read_bound = 0; - std::atomic busy{false}; - Status failure; - bool complete = false; + std::shared_ptr snapshot; + std::shared_ptr metadata; + std::shared_ptr loader; + std::shared_ptr scan; + std::atomic busy{false}; + Status failure; + bool complete = false; }; Status PublishedSnapshotEntries::create(std::shared_ptr snapshot, @@ -52,45 +41,12 @@ try { if (limits.max_route_depth > 64) { return Status::resource_exhausted("snapshot entry cursor exceeds route depth bound"); } - const auto workers = snapshot->binding()->cpu()->worker_count(); - const auto loads = snapshot->load_limits(); - const auto &view = *snapshot->view(); - const auto leaves = snapshot_leaf_window(view, limits.max_entries); - if (!leaves) { - return Status::resource_exhausted("snapshot entry frame window overflow"); - } - const auto value_bytes = overflow_value_bytes_bound(view, loads.max_value_bytes); - auto bounded_loads = loads; - // Each accepted overflow frame supplies at least one value byte. Keep the - // loader's validation limits unchanged; only this fixed window is smaller. - bounded_loads.max_overflow_pages = std::min(loads.max_overflow_pages, value_bytes); - const auto read_bound = - value_bytes == 0 ? AsyncStorePageLoader::primary_reserve_bound(loads, limits.max_route_depth, workers, *leaves) - : AsyncStorePageLoader::read_reserve_bound(bounded_loads, limits.max_route_depth, workers, - *leaves, 1, value_bytes); - using detail::shared_allocation_bound; - const auto bound = - detail::AllocationBound(shared_allocation_bound() + shared_allocation_bound() + - shared_allocation_bound() + - shared_allocation_bound()) - .add(PublishedScan::allocation_bound(limits, workers)) - .add_product(2, OwnedKey::allocation_overhead() + limits.max_key_bytes + 1) - .value(); - if (!bound || !read_bound) { - return Status::resource_exhausted("snapshot entry cursor bound overflow"); - } - auto credit = - detail::ResourceBudget::try_reserve(snapshot->binding()->account(), detail::BudgetClass::kRouting, *bound); - if (!credit) { - return Status::resource_exhausted("snapshot entry control reserve unavailable"); - } - auto metadata = detail::AllocationReservation::create(std::move(*credit)); - auto state = detail::reserved_shared(metadata); - state->snapshot = std::move(snapshot); - state->metadata = metadata; - state->loader = detail::reserved_shared(metadata); - state->read_bound = *read_bound; - limits.metadata = metadata; + auto metadata = limits.metadata; + auto state = detail::reserved_shared(metadata); + state->snapshot = std::move(snapshot); + state->metadata = metadata; + state->loader = detail::reserved_shared(metadata); + limits.metadata = metadata; PageKeyRange selection; if (range.start()) { selection.lower = OwnedKey(Slice(*range.start()), metadata); @@ -161,7 +117,7 @@ Status PublishedSnapshotEntries::next(std::optional *out, bo } } active{*state_->loader}; - auto status = state_->snapshot->prepare_window(state_->read_bound, &window); + auto status = state_->snapshot->prepare_window(&window); if (status.ok()) { status = state_->loader->begin(window.loader); } diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_read.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot_read.cpp index 202ae0457..b8308ae24 100644 --- a/lib/crowdb-tree/src/snapshot/published_snapshot_read.cpp +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_read.cpp @@ -21,23 +21,9 @@ try { if (!root || root->level >= limits.max_route_depth) { return Status::corruption("snapshot has no bounded published root"); } - limits.max_route_depth = root->level + 1ULL; - const auto loads = load_limits(); - const auto workers = binding()->cpu()->worker_count(); - const auto leaves = view()->mapping.copy_bounds().max_leaf_bases; - const auto values = overflow_value_bytes_bound(*view(), loads.max_value_bytes); - auto bounded_loads = loads; - bounded_loads.max_overflow_pages = std::min(loads.max_overflow_pages, values); - const auto loader = - values == 0 ? AsyncStorePageLoader::primary_reserve_bound(loads, root->level, workers, leaves) - : AsyncStorePageLoader::read_reserve_bound(bounded_loads, root->level, workers, leaves, 1, values); - const auto bound = - detail::AllocationBound().add(PublishedGet::allocation_bound(limits, workers)).add(loader).value(); - if (!bound) { - return Status::resource_exhausted("snapshot point read bound overflow"); - } + limits.max_route_depth = root->level + 1ULL; SnapshotReadWindow window; - auto status = prepare_window(*bound, &window); + auto status = prepare_window(&window); if (!status.ok()) { return status; } diff --git a/lib/crowdb-tree/src/snapshot/published_snapshot_scan.cpp b/lib/crowdb-tree/src/snapshot/published_snapshot_scan.cpp index bffc826fb..88d621360 100644 --- a/lib/crowdb-tree/src/snapshot/published_snapshot_scan.cpp +++ b/lib/crowdb-tree/src/snapshot/published_snapshot_scan.cpp @@ -17,25 +17,8 @@ namespace crowdb::tree { namespace { -std::optional result_count_bound(const PublishedTreeState &view, const PageLoadLimits &loads, - bool include_tombstones) -{ - // Snapshot capture removes the live source catalog. Exact live-key counts - // exclude tombstones; otherwise each captured base contributes at most its - // configured leaf capacity in addition to the immutable overlay entries. - if (!include_tombstones && view.stats.kv_unknown_pages == 0) { - return view.stats.kv_known_count; - } - const auto shape = view.mapping.copy_bounds(); - const auto bases = detail::AllocationBound().add_product(shape.leaf_pages, shape.max_leaf_bases).value(); - if (!bases) { - return {}; - } - return detail::AllocationBound(view.stats.overlay_entries).add_product(*bases, loads.max_leaf_entries).value(); -} - PageKeyRange scan_range(CoreScanOptions options, const PageKeyRange &domain, - const std::shared_ptr &credit) + const std::shared_ptr &credit) { PageKeyRange range; if (!options.prefix.empty()) { @@ -136,46 +119,17 @@ try { return Status::corruption("snapshot scan has no bounded root"); } using detail::AllocationBound; - const auto loads = load_limits(); - const auto maximum_count = result_count_bound(*view(), loads, options.include_tombstones); - if (!maximum_count) { - return Status::resource_exhausted("snapshot scan result count bound overflow"); - } - options.limit = options.limit == 0 ? *maximum_count : std::min(options.limit, *maximum_count); + const auto loads = load_limits(); + options.limit = options.limit == 0 ? SIZE_MAX : options.limit; limits.max_route_depth = root->level + 1ULL; const auto byte_limit = options.byte_budget == 0 ? SIZE_MAX : options.byte_budget; limits.max_result_bytes = std::min(byte_limit, limits.max_result_bytes); // The public limit bounds the whole packed result. Each read still uses a // fixed work window, with at most one zero-byte empty-key result. - const auto byte_count = AllocationBound(limits.max_result_bytes).add(1).value().value_or(SIZE_MAX); - limits.max_entries = std::max(1, std::min({options.limit, limits.max_entries, byte_count})); - const auto workers = binding()->cpu()->worker_count(); - const auto normal = AllocationBound(limits.max_result_bytes).add_product(options.limit, 17).value(); - const auto value_limit = overflow_value_bytes_bound(*view(), loads.max_value_bytes); - const auto values = AllocationBound(value_limit).add(value_limit == 0 ? 0 : limits.max_result_bytes).value(); - if (!normal || !values) { - return Status::resource_exhausted("snapshot scan result bound overflow"); - } - const auto leaves = snapshot_leaf_window(*view(), limits.max_entries); - if (!leaves) { - return Status::resource_exhausted("snapshot scan frame window overflow"); - } - auto bounded_loads = loads; - bounded_loads.max_overflow_pages = std::min(loads.max_overflow_pages, value_limit); - const auto loader = value_limit == 0 - ? AsyncStorePageLoader::primary_reserve_bound(loads, root->level, workers, *leaves) - : AsyncStorePageLoader::read_reserve_bound(bounded_loads, root->level, workers, *leaves, - limits.max_entries, *values); - const auto bound = AllocationBound() - .add(PublishedScan::allocation_bound(limits, workers)) - .add(loader) - .add_product(6, OwnedKey::allocation_overhead() + limits.max_key_bytes + 1) - .value(); - if (!bound) { - return Status::resource_exhausted("snapshot scan read bound overflow"); - } + const auto byte_count = AllocationBound(limits.max_result_bytes).add(1).value().value_or(SIZE_MAX); + limits.max_entries = std::max(1, std::min({options.limit, limits.max_entries, byte_count})); SnapshotReadWindow window; - auto status = prepare_window(*bound, &window); + auto status = prepare_window(&window); if (!status.ok()) { return status; } @@ -191,11 +145,7 @@ try { std::shared_ptr scan; status = PublishedScan::create(view(), window.loader, scan_range(options, range(), limits.metadata), options.reverse, limits, &scan, std::move(selection)); - std::shared_ptr output; - if (status.ok()) { - status = detail::AllocationAccount::create(binding()->account(), detail::BudgetClass::kRouting, &output); - } - ScanPackedBuf packed(std::move(output)); + ScanPackedBuf packed(window.budget.bytes); size_t bytes = 0; while (status.ok()) { if (expired(options.deadline_ms)) { @@ -206,8 +156,8 @@ try { if (!status.ok()) { break; } - status = pack_window(scan->result(), options, byte_limit, *normal, loads.max_value_bytes, &packed, count, - &bytes, truncated); + status = pack_window(scan->result(), options, byte_limit, 0, loads.max_value_bytes, &packed, count, &bytes, + truncated); if (scan->result().done || *truncated) { break; } diff --git a/lib/crowdb-tree/tests/allocation_account_test_support.h b/lib/crowdb-tree/tests/allocation_account_test_support.h index a4c895c02..6629d76ec 100644 --- a/lib/crowdb-tree/tests/allocation_account_test_support.h +++ b/lib/crowdb-tree/tests/allocation_account_test_support.h @@ -46,10 +46,24 @@ using Step = AllocationAccountTestAccess::Step; inline ResourceBudget::Limits limits(size_t amount) { ResourceBudget::Limits result{}; - result.fill(amount); + result[kRouting] = amount; return result; } +inline AllocationLease use_remaining_memory(const std::shared_ptr &account, + const std::shared_ptr &budget) +{ + const auto usage = account->resource_usage(); + size_t maximum = 0; + size_t allocated = 0; + for (size_t index = 0; index < static_cast(BudgetClass::kIo); ++index) { + maximum += usage[index].limit; + allocated += usage[index].allocated; + } + // Simulate actual allocator charges without allocating the production limit. + return {budget, maximum + (maximum / 2) - allocated}; +} + struct TestAccounts { ResourceBudget host{limits(16384), 1}; diff --git a/lib/crowdb-tree/tests/captured_write_test_support.h b/lib/crowdb-tree/tests/captured_write_test_support.h index cb7ac9f1d..2799a7cae 100644 --- a/lib/crowdb-tree/tests/captured_write_test_support.h +++ b/lib/crowdb-tree/tests/captured_write_test_support.h @@ -28,9 +28,9 @@ struct TestCapturedWrite TestCapturedWrite() { - EXPECT_TRUE(ExtentLedger::create({.backend_id = 9}, credits.credits.bytes, &ledger).ok()); + EXPECT_TRUE(ExtentLedger::create({.backend_id = 9}, credits.credits.reservation(), &ledger).ok()); EXPECT_TRUE(await_ledger([&](auto done) { - return ledger->activate(credits.credits.bytes, std::move(done)); + return ledger->activate(credits.credits.reservation(), std::move(done)); }).status.ok()); } @@ -47,7 +47,7 @@ struct TestCapturedWrite next_overflow += count; return Status::Ok(); }, - extent_write_reservation(ledger, credits.credits.bytes), std::move(inherited), &operation); + extent_write_reservation(ledger, credits.credits.reservation()), std::move(inherited), &operation); } std::vector read(const DurableCapturedFrame &frame) diff --git a/lib/crowdb-tree/tests/common/test_chunk_backing.h b/lib/crowdb-tree/tests/common/test_chunk_backing.h index 41d7ab52b..c12fe174c 100644 --- a/lib/crowdb-tree/tests/common/test_chunk_backing.h +++ b/lib/crowdb-tree/tests/common/test_chunk_backing.h @@ -104,10 +104,10 @@ struct TestChunkBackend TestChunkBackend() { - EXPECT_TRUE( - detail::ChunkPinReleaseQueue::create( - credits.credits.bytes, 16, std::make_shared(1), [](void *) noexcept {}, nullptr, &catalog->queue) - .ok()); + EXPECT_TRUE(detail::ChunkPinReleaseQueue::create( + credits.credits.reservation(), 16, std::make_shared(1), [](void *) noexcept {}, nullptr, + &catalog->queue) + .ok()); domain->backend_id = 17; domain->tree = 19; domain->epoch = 1; @@ -137,8 +137,8 @@ struct TestChunkBackend Status prepare(uint32_t bytes, std::shared_ptr *out) const { return prepare_chunk_backing(domain, {.owner_epoch = 1, .operation_high = 4, .operation_low = 5}, - {uint64_t{2} << 56U, 1}, bytes, PagePurpose::kBtreePage, credits.credits.bytes, - out); + {uint64_t{2} << 56U, 1}, bytes, PagePurpose::kBtreePage, + credits.credits.reservation(), out); } }; } // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_chunk_root.h b/lib/crowdb-tree/tests/common/test_chunk_root.h index 8d109b34b..85af28649 100644 --- a/lib/crowdb-tree/tests/common/test_chunk_root.h +++ b/lib/crowdb-tree/tests/common/test_chunk_root.h @@ -12,7 +12,7 @@ namespace crowdb::tree::test_chunk struct TestRootBackend { test_read::LoadCredits credits; - std::shared_ptr catalog = std::make_shared(credits.credits.bytes); + std::shared_ptr catalog = std::make_shared(credits.credits.reservation()); std::shared_ptr transport = std::make_shared(catalog); std::shared_ptr domain = std::make_shared(); std::shared_ptr ids = @@ -21,8 +21,8 @@ struct TestRootBackend explicit TestRootBackend(size_t pin_capacity = 256) { EXPECT_TRUE(detail::ChunkPinReleaseQueue::create( - credits.credits.bytes, pin_capacity, std::make_shared(1), [](void *) noexcept {}, nullptr, - &catalog->queue) + credits.credits.reservation(), pin_capacity, std::make_shared(1), [](void *) noexcept {}, + nullptr, &catalog->queue) .ok()); domain->backend_id = 17; domain->tree = 19; @@ -42,9 +42,10 @@ struct TestRootBackend std::shared_ptr pages(size_t count) { - auto pages = detail::reserved_shared(credits.credits.bytes, credits.credits.bytes); - pages->generation = 3; - pages->coverage = 15; + auto pages = + detail::reserved_shared(credits.credits.reservation(), credits.credits.reservation()); + pages->generation = 3; + pages->coverage = 15; pages->root_page_id = count != 0 ? 0 : kInvalidPageId; pages->next_page_id = count; pages->backend_id = 17; @@ -53,12 +54,12 @@ struct TestRootBackend detail::ChunkId chunk; EXPECT_TRUE(ids->next(PagePurpose::kBtreePage, &stage, &chunk).ok()); std::shared_ptr pack; - EXPECT_TRUE( - prepare_chunk_backing(domain, stage, chunk, 8192, PagePurpose::kBtreePage, credits.credits.bytes, &pack) - .ok()); + EXPECT_TRUE(prepare_chunk_backing(domain, stage, chunk, 8192, PagePurpose::kBtreePage, + credits.credits.reservation(), &pack) + .ok()); std::array raw{}; - EXPECT_TRUE(write_chunk_backing(*pack, raw, credits.credits.bytes).ok()); - auto backing = detail::reserved_shared(credits.credits.bytes); + EXPECT_TRUE(write_chunk_backing(*pack, raw, credits.credits.reservation()).ok()); + auto backing = detail::reserved_shared(credits.credits.reservation()); backing->backend_id = 17; backing->length = 8192; backing->owner = pack; @@ -66,7 +67,8 @@ struct TestRootBackend backing->storage = BackingStorage::kChunkPack; backing->chunk = std::move(pack); auto refs = detail::reserved_shared( - credits.credits.bytes, detail::ReservationAllocator(credits.credits.bytes)); + credits.credits.reservation(), + detail::ReservationAllocator(credits.credits.reservation())); pages->frames.emplace(i, DurableCapturedFrame{.page_id = i, .type = page_type::kLeafBase, .frame_bytes = 4096, @@ -94,9 +96,9 @@ struct TestRootBackend .next_page_id = pages->next_page_id, .wal_replay_offset = 4096}; std::shared_ptr result; - EXPECT_TRUE( - ChunkRootCommit::create(pages, domain, ids, identity, credits.credits.bytes, credits.credits.bytes, &result) - .ok()); + EXPECT_TRUE(ChunkRootCommit::create(pages, domain, ids, identity, credits.credits.reservation(), + credits.credits.reservation(), &result) + .ok()); return result; } }; diff --git a/lib/crowdb-tree/tests/common/test_legacy_chunk.h b/lib/crowdb-tree/tests/common/test_legacy_chunk.h index b4b5c11c4..b9d89b1cb 100644 --- a/lib/crowdb-tree/tests/common/test_legacy_chunk.h +++ b/lib/crowdb-tree/tests/common/test_legacy_chunk.h @@ -103,7 +103,7 @@ class TestLegacyCatalog final : public TestRootCatalog struct TestLegacyChunk { test_read::LoadCredits credits; - std::shared_ptr catalog = std::make_shared(credits.credits.bytes); + std::shared_ptr catalog = std::make_shared(credits.credits.reservation()); std::shared_ptr transport = std::make_shared(catalog); std::vector fixture; std::map, std::vector> physical; @@ -171,7 +171,8 @@ struct TestLegacyChunk [[nodiscard]] std::shared_ptr reader() const { - return std::make_shared(catalog, transport, credits.credits.bytes, 42, 7, 1, 2); + return std::make_shared(catalog, transport, credits.credits.reservation(), 42, 7, + 1, 2); } }; } // namespace crowdb::tree::test_chunk diff --git a/lib/crowdb-tree/tests/common/test_tree_core_chunk.h b/lib/crowdb-tree/tests/common/test_tree_core_chunk.h index f9698f715..38338135d 100644 --- a/lib/crowdb-tree/tests/common/test_tree_core_chunk.h +++ b/lib/crowdb-tree/tests/common/test_tree_core_chunk.h @@ -47,7 +47,7 @@ struct TestCoreChunk .iu_size = 4096, .mirrors = 2, .max_chunk_bytes = 256 * 1024 * 1024}, - backend.catalog, backend.transport, backend.credits.credits.bytes); + backend.catalog, backend.transport, backend.credits.credits.reservation()); config.page_store = store.get(); config.backend_owner = store; } diff --git a/lib/crowdb-tree/tests/extent_ledger_test_support.h b/lib/crowdb-tree/tests/extent_ledger_test_support.h index c4f65979e..b1ad77999 100644 --- a/lib/crowdb-tree/tests/extent_ledger_test_support.h +++ b/lib/crowdb-tree/tests/extent_ledger_test_support.h @@ -32,24 +32,27 @@ struct LedgerFixture explicit LedgerFixture(ExtentLedgerLimits limits = {.backend_id = 1}) { - EXPECT_TRUE(ExtentLedger::create(limits, credits.credits.bytes, &ledger).ok()); + EXPECT_TRUE(ExtentLedger::create(limits, credits.credits.reservation(), &ledger).ok()); EXPECT_LE(credits.budget.snapshot().allocated[LoadCredits::kBytes], ExtentLedger::creation_metadata_bound()); } ExtentLedgerResult activate() { - return await_ledger([&](auto done) { return ledger->activate(credits.credits.bytes, std::move(done)); }); + return await_ledger( + [&](auto done) { return ledger->activate(credits.credits.reservation(), std::move(done)); }); } ExtentLedgerResult recover(uint64_t address, uint64_t length) { - return await_ledger( - [&](auto done) { return ledger->recover(address, length, credits.credits.bytes, std::move(done)); }); + return await_ledger([&](auto done) { + return ledger->recover(address, length, credits.credits.reservation(), std::move(done)); + }); } ExtentLedgerResult reserve(uint64_t length) { - return await_ledger([&](auto done) { return ledger->reserve(length, credits.credits.bytes, std::move(done)); }); + return await_ledger( + [&](auto done) { return ledger->reserve(length, credits.credits.reservation(), std::move(done)); }); } }; } // namespace crowdb::tree::test_read diff --git a/lib/crowdb-tree/tests/integration/borrowed_read_budget_test.cpp b/lib/crowdb-tree/tests/integration/borrowed_read_budget_test.cpp index 4a2e7e757..0f04d47c9 100644 --- a/lib/crowdb-tree/tests/integration/borrowed_read_budget_test.cpp +++ b/lib/crowdb-tree/tests/integration/borrowed_read_budget_test.cpp @@ -18,7 +18,7 @@ TEST(BorrowedReadBudget, InlineWritebackAndNewUnknownOverlaysAdmitConcurrentRead config.page_store = &store; config.frame_bytes = 65536; config.runtime_binding.limits.routing_bytes = 128 * 1024 * 1024; - // Each accepted read owns a loader ticket and backend admission together. + // I/O credit is consumed only when a backing read is submitted. config.runtime_binding.limits.io_operations = 64; Crowdbtree tree(config); const std::string value(16384, 'v'); @@ -37,7 +37,7 @@ TEST(BorrowedReadBudget, InlineWritebackAndNewUnknownOverlaysAdmitConcurrentRead std::array accepted; size_t readers = 0; for (auto &read : accepted) { - const auto status = core_read_detail::prepare(core, captured, core.read_limits(), false, 0, 0, &read); + const auto status = core_read_detail::prepare(core, captured, core.read_limits(), &read); ASSERT_TRUE(status.ok()) << "readers=" << readers << ' ' << status.to_string(); ++readers; } diff --git a/lib/crowdb-tree/tests/integration/captured_frame_window_test.cpp b/lib/crowdb-tree/tests/integration/captured_frame_window_test.cpp index 9f916593d..82ddee573 100644 --- a/lib/crowdb-tree/tests/integration/captured_frame_window_test.cpp +++ b/lib/crowdb-tree/tests/integration/captured_frame_window_test.cpp @@ -25,7 +25,7 @@ TEST(CapturedFrameWindow, WarmAndColdOversizedLeavesUseExactPrepaidDescriptorBou auto captured = std::make_shared(*view(logical, {}, 10)); captured->next_page_id = 1; LoadCredits credits; - credits.credits.bytes->seal(); + credits.credits.reservation()->seal(); CapturedFrameLimits limits; limits.max_depth = 1; limits.page_metadata_bytes = captured->mapping.copy_bounds().max_leaf_bytes; diff --git a/lib/crowdb-tree/tests/integration/captured_frames_test.cpp b/lib/crowdb-tree/tests/integration/captured_frames_test.cpp index 42b944d52..b17931a25 100644 --- a/lib/crowdb-tree/tests/integration/captured_frames_test.cpp +++ b/lib/crowdb-tree/tests/integration/captured_frames_test.cpp @@ -161,7 +161,7 @@ TEST(CapturedFrames, MaterializationBytesFollowTheLastOutputOwnerAfterStreamDest ASSERT_TRUE(test.create(snapshot_leaf(page({ {.key = "a", .value = "captured"} })), - {}, {.encode = {.frame_bytes = 4096, .metadata = credits.credits.bytes}}) + {}, {.encode = {.frame_bytes = 4096, .metadata = credits.credits.reservation()}}) .ok()); Completion done; test.next(&done); @@ -366,7 +366,7 @@ TEST(CapturedFrames, CancelledColdLeafRetainsCallbackOwnershipAndDoesNotProduceL TestStream test; ASSERT_TRUE(test.create(snapshot_leaf(leaf), loader, { - .encode = {.frame_bytes = 512, .metadata = credits.credits.bytes} + .encode = {.frame_bytes = 512, .metadata = credits.credits.reservation()} }) .ok()); Completion done; @@ -392,11 +392,12 @@ TEST(CapturedFrames, OutputMemoryIsBoundedAcrossManyPagesUnderExhaustedHostBudge { LoadCredits credits; TestStream test; - ASSERT_TRUE(test.create(snapshot_tree(80), - { + ASSERT_TRUE( + test.create(snapshot_tree(80), + { }, - {.encode = {.frame_bytes = 4096, .max_entries = 128, .metadata = credits.credits.bytes}}) - .ok()); + {.encode = {.frame_bytes = 4096, .max_entries = 128, .metadata = credits.credits.reservation()}}) + .ok()); size_t peak = 0; size_t outputs = 0; for (size_t i = 0; i < 81; ++i) { diff --git a/lib/crowdb-tree/tests/integration/captured_write_backend_test.cpp b/lib/crowdb-tree/tests/integration/captured_write_backend_test.cpp index a6f940a98..cacfa3bda 100644 --- a/lib/crowdb-tree/tests/integration/captured_write_backend_test.cpp +++ b/lib/crowdb-tree/tests/integration/captured_write_backend_test.cpp @@ -36,7 +36,7 @@ TEST(CapturedWrite, RealBlockReopenReloadsPrimaryAndOverflowThroughItsCapturedMa next += count; return Status::Ok(); }, - extent_write_reservation(test.ledger, test.credits.credits.bytes), {}, &test.operation) + extent_write_reservation(test.ledger, test.credits.credits.reservation()), {}, &test.operation) .ok()); ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); test.runtime->drain(); diff --git a/lib/crowdb-tree/tests/integration/captured_write_test.cpp b/lib/crowdb-tree/tests/integration/captured_write_test.cpp index 78f053fd7..3c51dbdb1 100644 --- a/lib/crowdb-tree/tests/integration/captured_write_test.cpp +++ b/lib/crowdb-tree/tests/integration/captured_write_test.cpp @@ -9,27 +9,28 @@ using namespace test_write; TEST(CapturedWrite, FinalPageStreamWritesAndReadsWithoutASnapshotMapping) { - TestCapturedWrite test; - auto output = published(page({ + TestCapturedWrite test; + auto output = published(page({ {.key = "a", .value = "final"} })); - CapturedWriteLimits limits{ - .frames = {.encode = {.frame_bytes = 512, .inline_value_bytes = 16, .metadata = test.credits.credits.bytes}}, - .write = {.max_frame_bytes = 512}}; + CapturedWriteLimits limits{.frames = {.encode = {.frame_bytes = 512, + .inline_value_bytes = 16, + .metadata = test.credits.credits.reservation()}}, + .write = {.max_frame_bytes = 512}}; std::shared_ptr stream; ASSERT_TRUE(CapturedFrameStream::create( {.page = output, .page_id = 17, .coverage = 10, .gc_floor = 0, .next_page_id = 24, .context = {}}, {}, test.runtime, test.lane, limits.frames, {}, &stream) .ok()); - auto pages = std::make_shared(test.credits.credits.bytes); + auto pages = std::make_shared(test.credits.credits.reservation()); pages->root_page_id = 17; pages->next_page_id = 24; pages->coverage = 10; pages->generation = 1; - ASSERT_TRUE(CapturedWriteOperation::create(std::move(stream), std::move(pages), test.store, test.runtime, test.lane, - limits, test.credits.credits, - extent_write_reservation(test.ledger, test.credits.credits.bytes), {}, - &test.operation) + ASSERT_TRUE(CapturedWriteOperation::create( + std::move(stream), std::move(pages), test.store, test.runtime, test.lane, limits, + test.credits.credits, extent_write_reservation(test.ledger, test.credits.credits.reservation()), {}, + &test.operation) .ok()); output.reset(); Completion done; @@ -335,7 +336,7 @@ TEST(CapturedWrite, MetadataExhaustionAfterAcceptedIoReleasesThePartialImage) stdexec::start(connected); test.runtime->drain(); ASSERT_TRUE(test.store->pending); - test.credits.credits.bytes->seal(); + test.credits.credits.reservation()->seal(); test.store->finish(); test.runtime->drain(); EXPECT_EQ(done.errors.load(), 1U); diff --git a/lib/crowdb-tree/tests/integration/checkpoint_handoff_report_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_handoff_report_test.cpp index 8c708b82a..e20435922 100644 --- a/lib/crowdb-tree/tests/integration/checkpoint_handoff_report_test.cpp +++ b/lib/crowdb-tree/tests/integration/checkpoint_handoff_report_test.cpp @@ -50,7 +50,7 @@ struct TestHandoffCheckpoint std::shared_ptr partition(size_t bytes) { - auto credit = local.source.credits.credits.bytes->take_reserved(bytes); + auto credit = local.source.credits.credits.reservation()->take_reserved(bytes); if (!credit) { ADD_FAILURE() << "test credit unavailable"; return {}; diff --git a/lib/crowdb-tree/tests/integration/checkpoint_operation_test.cpp b/lib/crowdb-tree/tests/integration/checkpoint_operation_test.cpp index f934c6ca0..cacb39c59 100644 --- a/lib/crowdb-tree/tests/integration/checkpoint_operation_test.cpp +++ b/lib/crowdb-tree/tests/integration/checkpoint_operation_test.cpp @@ -34,8 +34,8 @@ struct TestCheckpointPipeline backend->store = local.store; backend->ledger = local.source.ledger; backend->checkpoints = local.registry; - auto credit = - local.source.credits.credits.bytes->take_reserved(CheckpointOperation::finish_allocation_bound() + 512); + auto credit = local.source.credits.credits.reservation()->take_reserved( + CheckpointOperation::finish_allocation_bound() + 512); EXPECT_TRUE(credit.has_value()); if (credit) { finish = detail::AllocationReservation::create(std::move(*credit)); @@ -230,7 +230,7 @@ TEST(CheckpointOperation, ExhaustedMaterializationStillHasIndependentAbortCredit auto connected = stdexec::connect(test.operation->sender(), Receiver{.done = &done}); stdexec::start(connected); test.local.source.runtime->drain(); - auto credit = test.local.source.credits.credits.bytes; + auto credit = test.local.source.credits.credits.reservation(); const auto remaining = credit->available(); ASSERT_TRUE(credit->claim(remaining)); test.local.store->finish(); diff --git a/lib/crowdb-tree/tests/integration/chunk_backing_pin_test.cpp b/lib/crowdb-tree/tests/integration/chunk_backing_pin_test.cpp index 3a9b751d5..5b96efef6 100644 --- a/lib/crowdb-tree/tests/integration/chunk_backing_pin_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_backing_pin_test.cpp @@ -35,7 +35,7 @@ TEST(ChunkBackingPin, RejectsStageAndPackIdentityMismatchBeforeAcquiringProtecti detail::ChunkId unused; ASSERT_TRUE(test.ids->next(PagePurpose::kBtreePage, &pin, &unused).ok()); std::shared_ptr owner; - EXPECT_EQ(pin_chunk_backing(test.domain, invalid, pin, test.credits.credits.bytes, &owner).code(), + EXPECT_EQ(pin_chunk_backing(test.domain, invalid, pin, test.credits.credits.reservation(), &owner).code(), Code::kCorruption); EXPECT_FALSE(owner); } @@ -52,7 +52,7 @@ TEST(ChunkBackingPin, DeletingTransitionBetweenExactValidationAndAcquireRejectsT ASSERT_TRUE(test.ids->next(PagePurpose::kBtreePage, &pin, &unused).ok()); std::shared_ptr owner; EXPECT_EQ(pin_chunk_backing(test.domain, pages->frames.begin()->second.backing->chunk->pack, pin, - test.credits.credits.bytes, &owner) + test.credits.credits.reservation(), &owner) .code(), Code::kUnavailable); EXPECT_FALSE(owner); diff --git a/lib/crowdb-tree/tests/integration/chunk_backing_test.cpp b/lib/crowdb-tree/tests/integration/chunk_backing_test.cpp index 8e273be63..695202090 100644 --- a/lib/crowdb-tree/tests/integration/chunk_backing_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_backing_test.cpp @@ -72,7 +72,7 @@ TEST(ChunkBacking, ProcessFenceAfterIoRejectsTheReadBeforeExposingBytes) std::shared_ptr pack; ASSERT_TRUE(test.prepare(32, &pack).ok()); std::array input{}; - ASSERT_TRUE(write_chunk_backing(*pack, input, test.credits.credits.bytes).ok()); + ASSERT_TRUE(write_chunk_backing(*pack, input, test.credits.credits.reservation()).ok()); test.catalog->fence_at = 4; auto backing = std::make_shared(); backing->backend_id = 17; @@ -83,7 +83,7 @@ TEST(ChunkBacking, ProcessFenceAfterIoRejectsTheReadBeforeExposingBytes) std::array output{}; std::promise completed; submit_backing_read( - *test.scheduled, backing, output.data(), output.size(), test.credits.credits.bytes, + *test.scheduled, backing, output.data(), output.size(), test.credits.credits.reservation(), {.context = &completed, .complete_fn = [](void *raw, Status status) { static_cast *>(raw)->set_value(std::move(status)); }, @@ -100,7 +100,7 @@ TEST(ChunkBacking, UnknownAllocationRetainsItsExactPaidPinUntilHostReleaseSuccee ASSERT_TRUE(test.prepare(32, &pack).ok()); test.transport->unknown_allocate = true; std::array input{}; - EXPECT_EQ(write_chunk_backing(*pack, input, test.credits.credits.bytes).code(), Code::kUnavailable); + EXPECT_EQ(write_chunk_backing(*pack, input, test.credits.credits.reservation()).code(), Code::kUnavailable); if (!test.catalog->record) { FAIL() << "unknown Allocate lost its StageReserved record"; } diff --git a/lib/crowdb-tree/tests/integration/chunk_collection_c_api_test.cpp b/lib/crowdb-tree/tests/integration/chunk_collection_c_api_test.cpp index 6b062955a..b07e5e616 100644 --- a/lib/crowdb-tree/tests/integration/chunk_collection_c_api_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_collection_c_api_test.cpp @@ -44,7 +44,7 @@ struct TestCollectionCAbi ct_status open() { auto status = detail::ChunkPinReleaseQueue::create( - credits.credits.bytes, 32, std::make_shared(1), [](void * /*unused*/) noexcept {}, nullptr, + credits.credits.reservation(), 32, std::make_shared(1), [](void * /*unused*/) noexcept {}, nullptr, &catalog->queue); if (!status.ok()) { return static_cast(status.code()); diff --git a/lib/crowdb-tree/tests/integration/chunk_collection_operation_test.cpp b/lib/crowdb-tree/tests/integration/chunk_collection_operation_test.cpp index cbd1f7eb8..85c53f101 100644 --- a/lib/crowdb-tree/tests/integration/chunk_collection_operation_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_collection_operation_test.cpp @@ -25,10 +25,10 @@ struct TestCollectionCore TestCollectionCore() { - EXPECT_TRUE( - detail::ChunkPinReleaseQueue::create( - credits.credits.bytes, 32, std::make_shared(1), [](void *) noexcept {}, nullptr, &catalog->queue) - .ok()); + EXPECT_TRUE(detail::ChunkPinReleaseQueue::create( + credits.credits.reservation(), 32, std::make_shared(1), [](void *) noexcept {}, nullptr, + &catalog->queue) + .ok()); TreeRuntimeOptions options; options.workers = 1; options.backend_workers = 1; @@ -40,7 +40,7 @@ struct TestCollectionCore .iu_size = 4096, .mirrors = 2, .max_chunk_bytes = 256 * 1024 * 1024}, - catalog, transport, credits.credits.bytes); + catalog, transport, credits.credits.reservation()); config.page_store = store.get(); config.backend_owner = store; config.frame_bytes = 4096; diff --git a/lib/crowdb-tree/tests/integration/chunk_root_commit_test.cpp b/lib/crowdb-tree/tests/integration/chunk_root_commit_test.cpp index ee314e6f2..ed3544b88 100644 --- a/lib/crowdb-tree/tests/integration/chunk_root_commit_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_root_commit_test.cpp @@ -33,7 +33,7 @@ TEST(ChunkRootCommit, LostMutationRepliesResolveTheSameCandidateWithoutRepeating EXPECT_EQ(test.catalog->prepare_calls, 2U); EXPECT_EQ(test.catalog->commit_calls, 1U); EXPECT_EQ(test.catalog->abort_calls, 0U); - ChunkRootManifest root(test.credits.credits.bytes); + ChunkRootManifest root(test.credits.credits.reservation()); ASSERT_TRUE(decode_chunk_root(test.catalog->manifest, 19, 7, &root).ok()); EXPECT_EQ(root.frame_count, 65U); EXPECT_EQ(root.indexes.size(), 2U); @@ -45,9 +45,10 @@ TEST(ChunkRootCommit, LostMutationRepliesResolveTheSameCandidateWithoutRepeating detail::ChunkId unused; ASSERT_TRUE(test.ids->next(PagePurpose::kPageIndex, &pin, &unused).ok()); std::shared_ptr index; - ASSERT_TRUE(pin_chunk_backing(test.domain, root.indexes[0].pack, pin, test.credits.credits.bytes, &index).ok()); + ASSERT_TRUE( + pin_chunk_backing(test.domain, root.indexes[0].pack, pin, test.credits.credits.reservation(), &index).ok()); std::vector bytes(index->pack.length); - ASSERT_TRUE(read_chunk_backing(*index, bytes, test.credits.credits.bytes).ok()); + ASSERT_TRUE(read_chunk_backing(*index, bytes, test.credits.credits.reservation()).ok()); uint64_t count = 0; size_t offset = 0; ASSERT_TRUE(decode_chunk_index_header(bytes, root.identity, 0, &count, &offset).ok()); diff --git a/lib/crowdb-tree/tests/integration/chunk_root_incremental_test.cpp b/lib/crowdb-tree/tests/integration/chunk_root_incremental_test.cpp index 78c608599..3072247a4 100644 --- a/lib/crowdb-tree/tests/integration/chunk_root_incremental_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_root_incremental_test.cpp @@ -32,15 +32,16 @@ std::shared_ptr next(test_chunk::TestRootBackend identity.wal_replay_offset = wal; std::shared_ptr operation; const auto status = - ChunkRootCommit::create(pages, test.domain, test.ids, identity, test.credits.credits.bytes, - test.credits.credits.bytes, &operation, 1024 * 1024, previous, baseline); + ChunkRootCommit::create(pages, test.domain, test.ids, identity, test.credits.credits.reservation(), + test.credits.credits.reservation(), &operation, 1024 * 1024, previous, baseline); EXPECT_TRUE(status.ok()) << status.to_string(); return status.ok() ? finish(std::move(operation)) : nullptr; } std::shared_ptr inherit(test_chunk::TestRootBackend &test, const CapturedDurablePages &source) { - auto pages = detail::reserved_shared(test.credits.credits.bytes, test.credits.credits.bytes); + auto pages = detail::reserved_shared(test.credits.credits.reservation(), + test.credits.credits.reservation()); pages->frames.inherit(source.frames); pages->backend_id = source.backend_id; pages->generation = source.generation; @@ -99,9 +100,10 @@ TEST(ChunkRootIncremental, DeletingAWholeSegmentKeepsLaterPackIdentityAndBirthOr detail::ChunkId unused; ASSERT_TRUE(test.ids->next(PagePurpose::kPageIndex, &pin, &unused).ok()); std::shared_ptr pack; - ASSERT_TRUE(pin_chunk_backing(test.domain, root->indexes[0].pack, pin, test.credits.credits.bytes, &pack).ok()); + ASSERT_TRUE( + pin_chunk_backing(test.domain, root->indexes[0].pack, pin, test.credits.credits.reservation(), &pack).ok()); std::vector bytes(pack->pack.length); - ASSERT_TRUE(read_chunk_backing(*pack, bytes, test.credits.credits.bytes).ok()); + ASSERT_TRUE(read_chunk_backing(*pack, bytes, test.credits.credits.reservation()).ok()); uint64_t count = 0; size_t offset = 0; ASSERT_TRUE(decode_chunk_index_header(bytes, root->identity, 0, &count, &offset).ok()); diff --git a/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp b/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp index 5abdc0882..9168881ac 100644 --- a/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp @@ -100,8 +100,8 @@ TEST(ChunkSnapshotRetry, SmallFramesCheckpointLargeValues) test_chunk::TestCoreChunk test; std::shared_ptr releases; ASSERT_TRUE(ChunkPinReleaseQueue::create( - test.backend.credits.credits.bytes, 1024, std::make_shared(1), [](void *) noexcept {}, nullptr, - &releases) + test.backend.credits.credits.reservation(), 1024, std::make_shared(1), [](void *) noexcept {}, + nullptr, &releases) .ok()); test.backend.catalog->queue = std::move(releases); ASSERT_TRUE(test.open().ok()); diff --git a/lib/crowdb-tree/tests/integration/chunk_write_queue_test.cpp b/lib/crowdb-tree/tests/integration/chunk_write_queue_test.cpp new file mode 100644 index 000000000..a914a63b4 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_write_queue_test.cpp @@ -0,0 +1,142 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#include "backend/backing_io.h" +#include "backend/local/scheduled_page_store.h" +#include "common/test_chunk_root.h" + +#include +#include + +namespace crowdb::tree +{ +namespace +{ +struct TestChunkWrites +{ + std::shared_ptr runtime; + std::shared_ptr binding; + std::shared_ptr store; + MemPageStore device{4096}; + + explicit TestChunkWrites(size_t workers) + { + TreeRuntimeOptions options; + options.workers = 1; + options.backend_workers = workers; + EXPECT_TRUE(TreeRuntime::create(options, &runtime).ok()); + EXPECT_TRUE(TreeRuntimeBinding::create(runtime, {}, &binding).ok()); + EXPECT_TRUE(ScheduledPageStore::create(&device, nullptr, {}, binding, binding->lifecycle_budget(), + binding->backend_lane(), &store) + .ok()); + } + + ~TestChunkWrites() + { + binding->drain(); + } + + std::future submit(test_chunk::TestRootBackend &backend, std::span input, + std::shared_ptr *pack) + { + detail::ChunkStageId stage; + detail::ChunkId chunk; + EXPECT_TRUE(backend.ids->next(PagePurpose::kBtreePage, &stage, &chunk).ok()); + EXPECT_TRUE(prepare_chunk_backing(backend.domain, stage, chunk, static_cast(input.size()), + PagePurpose::kBtreePage, backend.credits.credits.reservation(), pack) + .ok()); + auto promise = std::make_shared>(); + auto result = promise->get_future(); + submit_chunk_write(*store, *pack, input, backend.credits.credits.reservation(), + {.context = promise.get(), + .complete_fn = + [](void *raw, Status status) { + static_cast *>(raw)->set_value(std::move(status)); + }, + .owner = promise}); + return result; + } +}; +} // namespace + +TEST(ChunkWriteQueue, SlowPipeDoesNotBlockTheOtherPipe) +{ + test_chunk::TestRootBackend a; + test_chunk::TestRootBackend b; + TestChunkWrites writes(2); + std::binary_semaphore entered{0}; + std::binary_semaphore resume{0}; + a.transport->before_allocate = [&] { + entered.release(); + resume.acquire(); + }; + const std::array input{}; + std::shared_ptr first; + std::shared_ptr second; + auto blocked = writes.submit(a, input, &first); + const bool started = entered.try_acquire_for(std::chrono::seconds(5)); + auto other = writes.submit(b, input, &second); + const bool ready = other.wait_for(std::chrono::seconds(5)) == std::future_status::ready; + const bool waiting = blocked.wait_for(std::chrono::seconds(0)) == std::future_status::timeout; + resume.release(); + EXPECT_TRUE(started); + EXPECT_TRUE(ready); + EXPECT_TRUE(waiting); + EXPECT_TRUE(other.get().ok()); + EXPECT_TRUE(blocked.get().ok()); + writes.binding->drain(); + std::array output{}; + ASSERT_TRUE(read_chunk_backing(*second, output, b.credits.credits.reservation()).ok()); + EXPECT_EQ(output, input); +} + +TEST(ChunkWriteQueue, QueuedPagesShareOneAllocationAndKeepExactIndependentLocations) +{ + test_chunk::TestRootBackend backend; + TestChunkWrites writes(1); + std::binary_semaphore entered{0}; + std::binary_semaphore resume{0}; + std::atomic first{true}; + backend.transport->before_allocate = [&] { + if (first.exchange(false)) { + entered.release(); + resume.acquire(); + } + }; + std::array, 5> inputs{}; + std::array, 5> packs; + std::array, 5> results; + for (size_t i = 0; i < inputs.size(); ++i) { + inputs[i].fill(static_cast(i + 1)); + } + results[0] = writes.submit(backend, inputs[0], packs.data()); + const bool started = entered.try_acquire_for(std::chrono::seconds(5)); + for (size_t i = 1; i < inputs.size(); ++i) { + results[i] = writes.submit(backend, inputs[i], &packs[i]); + } + writes.store.reset(); + resume.release(); + EXPECT_TRUE(started); + for (auto &result : results) { + EXPECT_TRUE(result.get().ok()); + } + writes.binding->drain(); + EXPECT_EQ(backend.catalog->records.size(), 3U); + EXPECT_EQ(packs[2]->pack.stage, packs[3]->pack.stage); + EXPECT_EQ(packs[2]->pack.stage, packs[4]->pack.stage); + EXPECT_EQ(packs[2]->pin, packs[4]->pin); + EXPECT_LT(packs[2]->pack.offset, packs[3]->pack.offset); + EXPECT_LT(packs[3]->pack.offset, packs[4]->pack.offset); + for (size_t i = 0; i < packs.size(); ++i) { + std::array output{}; + ASSERT_TRUE(read_chunk_backing(*packs[i], output, backend.credits.credits.reservation()).ok()); + EXPECT_EQ(output, inputs[i]); + } + auto kept = packs[4]; + packs = {}; + (void)backend.catalog->queue->pump(16); + EXPECT_EQ(backend.catalog->queue->snapshot().registered, 1U); + std::array output{}; + ASSERT_TRUE(read_chunk_backing(*kept, output, backend.credits.credits.reservation()).ok()); + EXPECT_EQ(output, inputs[4]); +} +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/extent_writeback_test.cpp b/lib/crowdb-tree/tests/integration/extent_writeback_test.cpp index 37f6e7fc6..38f8df72e 100644 --- a/lib/crowdb-tree/tests/integration/extent_writeback_test.cpp +++ b/lib/crowdb-tree/tests/integration/extent_writeback_test.cpp @@ -18,15 +18,17 @@ struct LedgerWrites : TestWriter LedgerWrites() { EXPECT_TRUE( - ExtentLedger::create({.backend_id = 3, .address_limit = 65536}, credits.credits.bytes, &ledger).ok()); + ExtentLedger::create({.backend_id = 3, .address_limit = 65536}, credits.credits.reservation(), &ledger) + .ok()); EXPECT_TRUE(await_ledger([&](auto done) { - return ledger->activate(credits.credits.bytes, std::move(done)); + return ledger->activate(credits.credits.reservation(), std::move(done)); }).status.ok()); } ExtentLedgerResult allocate(uint64_t bytes) { - return await_ledger([&](auto done) { return ledger->reserve(bytes, credits.credits.bytes, std::move(done)); }); + return await_ledger( + [&](auto done) { return ledger->reserve(bytes, credits.credits.reservation(), std::move(done)); }); } }; @@ -54,11 +56,11 @@ TEST(LedgerWriteback, RealBlockPageStaysReadableAndCannotBeReusedUntilLastBackin ASSERT_TRUE(uring.valid()); auto store = std::make_shared(file.get(), &uring); LedgerWrites test; - ASSERT_TRUE( - PageWriteOperation::create(store, test.runtime, test.lane, write_input(), - {.max_frame_bytes = 4096, .compression = compress_algo::kNone}, test.credits.credits, - extent_write_reservation(test.ledger, test.credits.credits.bytes), &test.operation) - .ok()); + ASSERT_TRUE(PageWriteOperation::create( + store, test.runtime, test.lane, write_input(), + {.max_frame_bytes = 4096, .compression = compress_algo::kNone}, test.credits.credits, + extent_write_reservation(test.ledger, test.credits.credits.reservation()), &test.operation) + .ok()); ASSERT_TRUE(stdexec::sync_wait(test.operation->sender())); test.runtime->drain(); auto reader = test.operation->result().backing; diff --git a/lib/crowdb-tree/tests/integration/flush_operation_test.cpp b/lib/crowdb-tree/tests/integration/flush_operation_test.cpp index d6cf31cb6..706173014 100644 --- a/lib/crowdb-tree/tests/integration/flush_operation_test.cpp +++ b/lib/crowdb-tree/tests/integration/flush_operation_test.cpp @@ -173,6 +173,66 @@ TEST(FlushOperation, NoProgressReturnsCreditWithoutRotatingSourcesOrPublishing) EXPECT_EQ(publisher.capture(), tree_before); } +TEST(FlushOperation, CoveredPersistentBatchRetainsClosedSourceTailForTheNextPrefix) +{ + SourceAdmission sources; + PublicationCoordinator publisher; + initialize_sources(publisher, sources); + detail::FlushRuntime runtime(1, 8); + auto lane = std::make_shared(8); + auto table = sources.catalog()->active; + ASSERT_TRUE(table->upsert("a", 1, encode_cell_buf(1, OpKind::kPut, "covered"))); + ASSERT_TRUE(table->upsert("b", 2, encode_cell_buf(2, OpKind::kPut, "tail"))); + table->close(); + std::shared_ptr first; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 1, 1, {}, {}, &first).ok()); + ASSERT_TRUE(stdexec::sync_wait(first->sender()).has_value()); + ASSERT_TRUE(first->result().handoff_status.ok()) << first->result().handoff_status.to_string(); + ASSERT_EQ(publisher.capture()->coverage, 1U); + ASSERT_EQ(publisher.capture()->sources->tables.size(), 1U); + auto inputs = std::make_shared(); + inputs->floor = 1; + auto acquire = std::make_shared(); + acquire->transaction.record = + std::make_shared(InstallToken{.generation = 1, .batch = 2, .range = 0, .sequence = 0}); + acquire->transaction.expected_commit_seq = publisher.capture()->commit_seq; + acquire->transaction.flush_admission = {.action = FlushAdmissionAction::Acquire, .owner = 2}; + acquire->transaction.flush_admission_owner = 2; + acquire->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(acquire)); + ASSERT_EQ(publisher.capture()->flush_admission_owner, 2U); + FlushBatchOptions persistent; + persistent.persistent = true; + persistent.checkpoint_token = 1; + std::shared_ptr checkpoint; + EXPECT_EQ(FlushBatch::prepare(publisher, runtime, lane, 2, 2, persistent, {}, &checkpoint, inputs, {}, 2).code(), + Code::kInvalidArgument); + EXPECT_FALSE(checkpoint); + const auto prepared = + FlushBatch::prepare(publisher, runtime, lane, 2, 1, persistent, {}, &checkpoint, inputs, {}, 2); + ASSERT_TRUE(prepared.ok()) << prepared.to_string(); + ASSERT_TRUE(stdexec::sync_wait(checkpoint->sender()).has_value()); + ASSERT_TRUE(checkpoint->result().handoff_status.ok()) << checkpoint->result().handoff_status.to_string(); + EXPECT_EQ(publisher.capture()->coverage, 1U); + ASSERT_EQ(publisher.capture()->sources->tables.size(), 1U); + EXPECT_EQ(publisher.capture()->sources->tables.front(), table); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("a").cell.value.to_string(), "covered"); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("b").state, LogicalLookupState::kMissing); + auto request = std::make_shared(); + request->transaction.record = + std::make_shared(InstallToken{.generation = 1, .batch = 2, .range = 0, .sequence = 1}); + request->transaction.checkpoint = {.action = CheckpointAction::Abort, .token = 1}; + request->on_done = [](PublicationResult result) { EXPECT_TRUE(result.status.ok()) << result.status.to_string(); }; + ASSERT_TRUE(publisher.try_submit(request)); + std::shared_ptr next; + ASSERT_TRUE(FlushBatch::prepare(publisher, runtime, lane, 3, 2, {}, {}, &next).ok()); + ASSERT_TRUE(stdexec::sync_wait(next->sender()).has_value()); + ASSERT_TRUE(next->result().handoff_status.ok()) << next->result().handoff_status.to_string(); + EXPECT_EQ(publisher.capture()->coverage, 2U); + EXPECT_TRUE(publisher.capture()->sources->tables.empty()); + EXPECT_EQ(publisher.capture()->mapping.get(0)->leaf->lookup("b").cell.value.to_string(), "tail"); +} + TEST(FlushOperation, MaintenanceRetryReusesBatchAfterCoverageAckAndMetadataSeal) { SourceAdmission sources; diff --git a/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp b/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp index 2ec9897c2..3c0e45348 100644 --- a/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp +++ b/lib/crowdb-tree/tests/integration/flush_reservation_test.cpp @@ -179,7 +179,7 @@ TEST(FlushReservation, BindingRejectsWrongPartitionBeforeConstructingConsumers) detail::FlushCredits credits; for (size_t index = 0; index < credits.size(); ++index) { auto paid = detail::ResourceBudget::try_reserve( - tree.account, index == 3 ? detail::BudgetClass::kRouting : detail::FlushReserveBounds::categories[index], + tree.account, index == 2 ? detail::BudgetClass::kRouting : detail::FlushReserveBounds::categories[index], bounds.amounts[index]); if (!paid) { FAIL() << "missing paid test prerequisite"; @@ -298,8 +298,7 @@ TEST(FlushReservation, UncontendedPlanningIsNotAnApplyProgressTransition) TEST(FlushReservation, EachMissingProgressPartitionRollsBackBeforeSourceClosure) { - for (const auto category : {detail::BudgetClass::kHandoff, detail::BudgetClass::kRouting, - detail::BudgetClass::kReclaim, detail::BudgetClass::kIo}) { + for (const auto category : detail::FlushReserveBounds::categories) { TestPreparedFlushTree tree; tree.frontier.store(1); const auto before = tree.publisher.capture(); @@ -331,7 +330,7 @@ TEST(FlushReservation, ColdRoutingUsesHeldBytesAndIoUnderCompleteHostPressure) std::shared_ptr operation; auto status = tree.controller->prepare(tree.preparation, &operation); ASSERT_TRUE(status.ok()) << status.to_string(); - EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 1U); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); auto blockers = tree.exhaust_unused(); ASSERT_TRUE(stdexec::sync_wait(operation->sender())); tree.runtime->drain(); @@ -586,7 +585,7 @@ TEST(FlushReservation, CancelledClosedBatchRetriesWithOriginalCreditsWhileHostIs done.get(); tree.runtime->drain(); EXPECT_EQ(operation->result().phase, FlushPhase::kRepairRequired); - EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 1U); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); const auto identity = operation->result().batch_id; const auto before = tree.budget.snapshot(); const auto owner = tree.publisher.capture()->flush_admission_owner; @@ -608,7 +607,7 @@ TEST(FlushReservation, CancelledClosedBatchRetriesWithOriginalCreditsWhileHostIs EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); } -TEST(FlushReservation, CacheWaiterPressureRollsBackBeforeSourceClosure) +TEST(FlushReservation, OccupiedCacheWaiterDoesNotBlockFlush) { TestPreparedFlushTree tree; tree.frontier.store(1); @@ -618,20 +617,16 @@ TEST(FlushReservation, CacheWaiterPressureRollsBackBeforeSourceClosure) ASSERT_TRUE(tree.preparation.cache->bind_reserved(upstream, {}, &occupied).ok()); const auto catalog = tree.sources.catalog(); const auto view = tree.publisher.capture(); - const auto before = tree.budget.snapshot(); std::shared_ptr operation; - EXPECT_EQ(tree.controller->prepare(tree.preparation, &operation).code(), Code::kUnavailable); - EXPECT_FALSE(operation); + ASSERT_TRUE(tree.controller->prepare(tree.preparation, &operation).ok()); EXPECT_FALSE(catalog->active->closed()); EXPECT_EQ(tree.sources.catalog(), catalog); EXPECT_EQ(tree.publisher.capture(), view); - EXPECT_EQ(tree.budget.snapshot().reserved, before.reserved); - EXPECT_EQ(tree.budget.snapshot().allocated, before.allocated); - occupied.reset(); - ASSERT_TRUE(tree.controller->prepare(tree.preparation, &operation).ok()); ASSERT_TRUE(stdexec::sync_wait(operation->sender())); tree.runtime->drain(); EXPECT_TRUE(operation->result().handoff_status.ok()); + EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 1U); + occupied.reset(); EXPECT_EQ(tree.preparation.cache->snapshot().reserved_waiters, 0U); } } // namespace diff --git a/lib/crowdb-tree/tests/integration/legacy_chunk_migration_test.cpp b/lib/crowdb-tree/tests/integration/legacy_chunk_migration_test.cpp index 0244e0cb5..8b2e6635b 100644 --- a/lib/crowdb-tree/tests/integration/legacy_chunk_migration_test.cpp +++ b/lib/crowdb-tree/tests/integration/legacy_chunk_migration_test.cpp @@ -21,12 +21,12 @@ Config configuration(test_chunk::TestLegacyChunk &fixture) host.backend_workers = 1; EXPECT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); EXPECT_TRUE(detail::ChunkPinReleaseQueue::create( - fixture.credits.credits.bytes, 256, std::make_shared(1), [](void *) noexcept {}, nullptr, - &fixture.catalog->queue) + fixture.credits.credits.reservation(), 256, std::make_shared(1), [](void *) noexcept {}, + nullptr, &fixture.catalog->queue) .ok()); auto store = std::make_shared( detail::ChunkRuntimeOptions{.tree = 42, .owner_epoch = 7, .iu_size = 1, .mirrors = 2}, fixture.catalog, - fixture.transport, fixture.credits.credits.bytes); + fixture.transport, fixture.credits.credits.reservation()); config.page_store = store.get(); config.backend_owner = std::move(store); config.frame_bytes = 4096; @@ -129,7 +129,7 @@ TEST(LegacyChunkMigration, ExplicitMigrationCommitsTheSameRootAndReopensWithNewF fixture.transport->catalog = next; auto store = std::make_shared( detail::ChunkRuntimeOptions{.tree = 42, .owner_epoch = 7, .iu_size = 1, .mirrors = 2}, next, - fixture.transport, fixture.credits.credits.bytes); + fixture.transport, fixture.credits.credits.reservation()); config.page_store = store.get(); config.backend_owner = store; const auto reopened = Crowdbtree::open(config, &tree); @@ -202,7 +202,7 @@ TEST(LegacyChunkMigration, UnknownCommitRetainsOldPinAndRetryResolvesDurableRece fixture.transport->catalog = fixture.catalog; auto store = std::make_shared( detail::ChunkRuntimeOptions{.tree = 42, .owner_epoch = 7, .iu_size = 1, .mirrors = 2}, fixture.catalog, - fixture.transport, fixture.credits.credits.bytes); + fixture.transport, fixture.credits.credits.reservation()); config.page_store = store.get(); config.backend_owner = store; const auto retried = Crowdbtree::migrate_legacy_chunk(config, 1, &tree); @@ -244,8 +244,8 @@ TEST(LegacyChunkMigration, PublicCEntryUsesInjectedRuntimeBackendAndPreservesThe { test_chunk::TestLegacyChunk fixture; ASSERT_TRUE(detail::ChunkPinReleaseQueue::create( - fixture.credits.credits.bytes, 256, std::make_shared(1), [](void *) noexcept {}, nullptr, - &fixture.catalog->queue) + fixture.credits.credits.reservation(), 256, std::make_shared(1), [](void *) noexcept {}, + nullptr, &fixture.catalog->queue) .ok()); ct_runtime *raw_runtime = nullptr; ASSERT_EQ(ct_runtime_create(nullptr, &raw_runtime), 0); @@ -254,7 +254,7 @@ TEST(LegacyChunkMigration, PublicCEntryUsesInjectedRuntimeBackendAndPreservesThe store->bundle = std::make_shared(); store->bundle->store = std::make_unique( detail::ChunkRuntimeOptions{.tree = 42, .owner_epoch = 7, .iu_size = 1, .mirrors = 2}, fixture.catalog, - fixture.transport, fixture.credits.credits.bytes); + fixture.transport, fixture.credits.credits.reservation()); ct_options options{}; options.page_store = store.get(); options.frame_bytes = 4096; diff --git a/lib/crowdb-tree/tests/integration/local_block_gc_test.cpp b/lib/crowdb-tree/tests/integration/local_block_gc_test.cpp index a9fa70e5b..7e2c24e4d 100644 --- a/lib/crowdb-tree/tests/integration/local_block_gc_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_block_gc_test.cpp @@ -38,8 +38,8 @@ TEST(LocalBlockGc, BoundedPassRetainsAnchorsAndReadersThenReusesDeletedHoles) auto reader = anchor.extent.owner; ASSERT_TRUE(test.ledger.activate().status.ok()); const LocalBlockGcLimits limits{.max_blocks = 2, .max_deleted_bytes = 4096}; - auto first = - collect_free_local_blocks(*test.store, *test.ledger.ledger, 0, limits, test.ledger.credits.credits.bytes); + auto first = collect_free_local_blocks(*test.store, *test.ledger.ledger, 0, limits, + test.ledger.credits.credits.reservation()); ASSERT_TRUE(first.status.ok()); EXPECT_EQ(first.deleted_blocks, 1U); EXPECT_EQ(first.next_block, 3U); @@ -50,15 +50,15 @@ TEST(LocalBlockGc, BoundedPassRetainsAnchorsAndReadersThenReusesDeletedHoles) EXPECT_FALSE(test.store->file_for_offset(8192, &offset)); anchor = {}; auto second = collect_free_local_blocks(*test.store, *test.ledger.ledger, first.next_block, limits, - test.ledger.credits.credits.bytes); + test.ledger.credits.credits.reservation()); ASSERT_TRUE(second.status.ok()); EXPECT_EQ(second.protected_blocks, 1U); EXPECT_EQ(second.deleted_blocks, 1U); EXPECT_EQ(second.next_block, 5U); EXPECT_TRUE(test.store->file_for_offset(12288, &offset)); reader.reset(); - auto third = - collect_free_local_blocks(*test.store, *test.ledger.ledger, 3, limits, test.ledger.credits.credits.bytes); + auto third = collect_free_local_blocks(*test.store, *test.ledger.ledger, 3, limits, + test.ledger.credits.credits.reservation()); ASSERT_TRUE(third.status.ok()); EXPECT_EQ(third.deleted_blocks, 1U); EXPECT_FALSE(test.store->file_for_offset(12288, &offset)); @@ -78,7 +78,7 @@ TEST(LocalBlockGc, MetadataFailureAndPartialTailCannotDeleteUnleasedStorage) ASSERT_TRUE(test.ledger.activate().status.ok()); test.ledger.credits.seal(); auto rejected = - collect_free_local_blocks(*test.store, *test.ledger.ledger, 0, {}, test.ledger.credits.credits.bytes); + collect_free_local_blocks(*test.store, *test.ledger.ledger, 0, {}, test.ledger.credits.credits.reservation()); EXPECT_EQ(rejected.status.code(), Code::kResourceExhausted); EXPECT_EQ(rejected.next_block, 1U); EXPECT_EQ(test.store->num_extents(), 3U); diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_backend_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_backend_test.cpp index 11e26a6e0..c032404aa 100644 --- a/lib/crowdb-tree/tests/integration/local_checkpoint_backend_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_backend_test.cpp @@ -34,7 +34,8 @@ TEST(LocalCheckpoint, RealBlockReopenUsesOnlyAnchorAndDurableMapping) next += count; return Status::Ok(); }, - extent_write_reservation(test.source.ledger, test.source.credits.credits.bytes), {}, &test.source.operation) + extent_write_reservation(test.source.ledger, test.source.credits.credits.reservation()), {}, + &test.source.operation) .ok()); ASSERT_TRUE(stdexec::sync_wait(test.source.operation->sender())); test.source.runtime->drain(); diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp index 22d8085c3..f225447c7 100644 --- a/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_directory_test.cpp @@ -62,7 +62,7 @@ DurableCapturedFrame rewrite_leaf(TestCheckpoint &test, const PublishedTreeState old->range)); leaf->right_sibling = old->right_sibling; CapturedWriteLimits limits{ - .frames = {.encode = {.frame_bytes = 4096, .metadata = test.source.credits.credits.bytes}}, + .frames = {.encode = {.frame_bytes = 4096, .metadata = test.source.credits.credits.reservation()}}, .write = {.max_frame_bytes = 4096}}; std::shared_ptr stream; EXPECT_TRUE(CapturedFrameStream::create({.page = published(std::move(leaf)), @@ -73,17 +73,17 @@ DurableCapturedFrame rewrite_leaf(TestCheckpoint &test, const PublishedTreeState .context = {.range = old->range}}, {}, test.source.runtime, test.source.lane, limits.frames, {}, &stream) .ok()); - auto pages = std::make_shared(test.source.credits.credits.bytes); + auto pages = std::make_shared(test.source.credits.credits.reservation()); pages->root_page_id = pid; pages->next_page_id = tree.next_page_id; pages->coverage = 11; pages->generation = tree.generation; - EXPECT_TRUE( - CapturedWriteOperation::create(std::move(stream), std::move(pages), test.source.store, test.source.runtime, - test.source.lane, limits, test.source.credits.credits, - extent_write_reservation(test.source.ledger, test.source.credits.credits.bytes), - {}, &test.source.operation) - .ok()); + EXPECT_TRUE(CapturedWriteOperation::create( + std::move(stream), std::move(pages), test.source.store, test.source.runtime, test.source.lane, + limits, test.source.credits.credits, + extent_write_reservation(test.source.ledger, test.source.credits.credits.reservation()), {}, + &test.source.operation) + .ok()); Completion done; test.source.run(&done); EXPECT_EQ(done.values.load(), 1U) << done.status.to_string(); diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_failure_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_failure_test.cpp index 515ad0565..ed341709b 100644 --- a/lib/crowdb-tree/tests/integration/local_checkpoint_failure_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_failure_test.cpp @@ -118,7 +118,8 @@ TEST(LocalCheckpoint, AllocationFailureAfterAnchorKeepsRecoveryProtection) test.source.runtime->drain(); ASSERT_TRUE(test.store->pending); auto retained_callback = test.store->pending; - AllocationLease pressure(test.source.credits.credits.bytes, test.source.credits.credits.bytes->available()); + AllocationLease pressure(test.source.credits.credits.reservation(), + test.source.credits.credits.reservation()->available()); test.store->finish(); test.source.runtime->drain(); EXPECT_EQ(done.errors.load(), 1U); diff --git a/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp b/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp index 3decb7991..c61cc8c6a 100644 --- a/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_checkpoint_test.cpp @@ -109,7 +109,7 @@ TEST(LocalCheckpoint, ExcludedMetadataSegmentIsRelocatedDespiteUnchangedPageSlot const auto &segment = original->metadata->segments.front().second; auto fence = await_ledger([&](auto done) { return test.source.ledger->exclude_allocations(segment.address, segment.length, - test.source.credits.credits.bytes, std::move(done)); + test.source.credits.credits.reservation(), std::move(done)); }); ASSERT_TRUE(fence.status.ok()); const BackingExclusion exclusion{.address = segment.address, .length = segment.length}; @@ -209,8 +209,8 @@ TEST(LocalCheckpoint, CoverageMayResetOnlyWhenTheLogicalGenerationAdvances) Completion first; test.run(&first); ASSERT_EQ(first.values.load(), 1U) << first.status.to_string(); - auto empty = detail::reserved_shared(test.source.credits.credits.bytes, - test.source.credits.credits.bytes); + auto empty = detail::reserved_shared(test.source.credits.credits.reservation(), + test.source.credits.credits.reservation()); empty->generation = first_pages->generation; ASSERT_TRUE(test.prepare(empty).ok()); const auto before = test.store->events.size(); @@ -236,7 +236,7 @@ TEST(LocalCheckpoint, CoverageMayResetOnlyWhenTheLogicalGenerationAdvances) TEST(LocalCheckpoint, EmptyCapturedViewCommitsAnEmptyDirectory) { TestCheckpoint test; - auto pages = std::make_shared(test.source.credits.credits.bytes); + auto pages = std::make_shared(test.source.credits.credits.reservation()); pages->coverage = 12; ASSERT_TRUE(test.prepare(pages).ok()); Completion done; diff --git a/lib/crowdb-tree/tests/integration/local_recovery_backend_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_backend_test.cpp index 8605ece9c..6dbe24b68 100644 --- a/lib/crowdb-tree/tests/integration/local_recovery_backend_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_recovery_backend_test.cpp @@ -49,7 +49,8 @@ TEST(LocalRecovery, ReopenedBlockFilesRebuildProtectedPrimaryAndOverflowOwners) next += count; return Status::Ok(); }, - extent_write_reservation(test.source.ledger, test.source.credits.credits.bytes), {}, &test.source.operation) + extent_write_reservation(test.source.ledger, test.source.credits.credits.reservation()), {}, + &test.source.operation) .ok()); auto status = await_status(test.source.operation->sender()); ASSERT_TRUE(status.ok()) << "captured write: " << status.to_string(); diff --git a/lib/crowdb-tree/tests/integration/local_recovery_failure_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_failure_test.cpp index c65cdfa72..0fab6ec03 100644 --- a/lib/crowdb-tree/tests/integration/local_recovery_failure_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_recovery_failure_test.cpp @@ -133,7 +133,7 @@ TEST(LocalRecoveryFailure, BudgetExhaustionAfterReadCannotPublishPartialOwnershi stdexec::start(connected); test.original.source.runtime->drain(); ASSERT_TRUE(test.store->pending); - AllocationLease pressure(test.credits.credits.bytes, test.credits.credits.bytes->available()); + AllocationLease pressure(test.credits.credits.reservation(), test.credits.credits.reservation()->available()); test.store->finish(); test.original.source.runtime->drain(); EXPECT_EQ(done.errors.load(), 1U); diff --git a/lib/crowdb-tree/tests/integration/local_recovery_frames_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_frames_test.cpp index 79024b720..16a65d351 100644 --- a/lib/crowdb-tree/tests/integration/local_recovery_frames_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_recovery_frames_test.cpp @@ -230,7 +230,7 @@ TEST(LocalRecoveryFrames, RecoveredOverflowCanBeReusedAndDoesNotRetainItsOldPrim captured, {}, test.store, test.original.source.runtime, test.original.source.lane, {.frames = {.encode = {.frame_bytes = 4096, .inline_value_bytes = 16}}, .write = {.max_frame_bytes = 4096}}, test.credits.credits, [](uint64_t, ReservedOverflowIds *) { return Status::not_supported(); }, - extent_write_reservation(ledger, test.credits.credits.bytes), inherited, &writer) + extent_write_reservation(ledger, test.credits.credits.reservation()), inherited, &writer) .ok()); Completion written; auto connected = stdexec::connect(writer->sender(), Receiver{.done = &written}); diff --git a/lib/crowdb-tree/tests/integration/local_recovery_test.cpp b/lib/crowdb-tree/tests/integration/local_recovery_test.cpp index dbea0d53b..731127e26 100644 --- a/lib/crowdb-tree/tests/integration/local_recovery_test.cpp +++ b/lib/crowdb-tree/tests/integration/local_recovery_test.cpp @@ -90,7 +90,7 @@ TEST(LocalRecovery, FreshDomainActivatesAnEmptyRegistryBeforeItsFirstAllocation) EXPECT_EQ(result.registry->snapshot_sequence(), 0U); EXPECT_EQ(result.ledger->stats().protected_ranges, 0U); auto allocated = await_ledger( - [&](auto done) { return result.ledger->reserve(4096, test.credits.credits.bytes, std::move(done)); }); + [&](auto done) { return result.ledger->reserve(4096, test.credits.credits.reservation(), std::move(done)); }); ASSERT_TRUE(allocated.status.ok()); EXPECT_EQ(allocated.extent.address, 8192U); EXPECT_EQ(allocated.extent.generation, 2U); @@ -126,8 +126,8 @@ TEST(LocalRecovery, NewestSequenceWinsAfterAChangedGenerationResetsCoverage) { TestRecovery test; test.checkpoint("old generation"); - auto empty = detail::reserved_shared(test.original.source.credits.credits.bytes, - test.original.source.credits.credits.bytes); + auto empty = detail::reserved_shared(test.original.source.credits.credits.reservation(), + test.original.source.credits.credits.reservation()); empty->generation = test.original.operation->result().checkpoint->pages->generation + 1; ASSERT_TRUE(test.original.prepare(empty).ok()); Completion cleared; @@ -154,7 +154,7 @@ TEST(LocalRecovery, RecoveredRegistryCanCommitTheNextCheckpointAndAdvanceDurable test.checkpoint("reopened"); ASSERT_TRUE(recover(test).ok()); const auto &result = test.operation->result(); - auto pages = detail::reserved_shared(test.credits.credits.bytes, *result.latest); + auto pages = detail::reserved_shared(test.credits.credits.reservation(), *result.latest); ++pages->coverage; ++pages->generation; std::shared_ptr checkpoint; @@ -288,8 +288,8 @@ TEST(LocalRecovery, UnreferencedCrashTailBecomesReusableAfterAllAnchorsAreProtec ASSERT_TRUE(test.store->device->write_at(orphan, bytes.data(), bytes.size()).ok()); ASSERT_TRUE(recover(test).ok()); auto ledger = test.operation->result().ledger; - auto reused = - await_ledger([&](auto done) { return ledger->reserve(4096, test.credits.credits.bytes, std::move(done)); }); + auto reused = await_ledger( + [&](auto done) { return ledger->reserve(4096, test.credits.credits.reservation(), std::move(done)); }); ASSERT_TRUE(reused.status.ok()); EXPECT_EQ(reused.extent.address, orphan); EXPECT_EQ(ledger->stats().reused_allocations, 1U); diff --git a/lib/crowdb-tree/tests/integration/memory_backing_gc_test.cpp b/lib/crowdb-tree/tests/integration/memory_backing_gc_test.cpp index fe1413acc..29274e41f 100644 --- a/lib/crowdb-tree/tests/integration/memory_backing_gc_test.cpp +++ b/lib/crowdb-tree/tests/integration/memory_backing_gc_test.cpp @@ -27,13 +27,13 @@ TEST(MemoryBackingGc, PartialPagesAndReadersRemainProtectedUntilTheirLastOwnerDr auto reader = anchor.extent.owner; ASSERT_TRUE(ledger.activate().status.ok()); const LocalBlockGcLimits limits{.max_blocks = 1, .max_deleted_bytes = kPage}; - auto first = collect_free_local_blocks(store, *ledger.ledger, 0, limits, ledger.credits.credits.bytes); + auto first = collect_free_local_blocks(store, *ledger.ledger, 0, limits, ledger.credits.credits.reservation()); ASSERT_TRUE(first.status.ok()); EXPECT_EQ(first.protected_blocks, 1U); EXPECT_FALSE(first.complete); - anchor = {}; - auto second = - collect_free_local_blocks(store, *ledger.ledger, first.next_block, limits, ledger.credits.credits.bytes); + anchor = {}; + auto second = collect_free_local_blocks(store, *ledger.ledger, first.next_block, limits, + ledger.credits.credits.reservation()); ASSERT_TRUE(second.status.ok()); EXPECT_TRUE(second.complete); EXPECT_EQ(second.deleted_bytes, kPage); @@ -42,7 +42,7 @@ TEST(MemoryBackingGc, PartialPagesAndReadersRemainProtectedUntilTheirLastOwnerDr ASSERT_TRUE(store.read_at(kPage + 4096, &value, 1).ok()); EXPECT_EQ(value, 42); reader.reset(); - auto third = collect_free_local_blocks(store, *ledger.ledger, 1, limits, ledger.credits.credits.bytes); + auto third = collect_free_local_blocks(store, *ledger.ledger, 1, limits, ledger.credits.credits.reservation()); ASSERT_TRUE(third.status.ok()); EXPECT_EQ(third.deleted_blocks, 1U); EXPECT_EQ(store.resident_bytes(), kPage); @@ -62,21 +62,21 @@ TEST(MemoryBackingGc, PendingWritesAndExclusiveDeletionLeasesPreventAddressReuse ASSERT_TRUE(ledger.activate().status.ok()); auto pending = ledger.reserve(kPage); ASSERT_TRUE(pending.status.ok()); - auto held = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.bytes); + auto held = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.reservation()); ASSERT_TRUE(held.status.ok()); EXPECT_EQ(held.protected_blocks, 1U); EXPECT_EQ(held.deleted_bytes, 0U); pending = {}; auto deletion = await_ledger([&](auto complete) { - return ledger.ledger->claim_free(kPage, kPage, ledger.credits.credits.bytes, std::move(complete)); + return ledger.ledger->claim_free(kPage, kPage, ledger.credits.credits.reservation(), std::move(complete)); }); ASSERT_TRUE(deletion.status.ok()); EXPECT_EQ(ledger.reserve(kPage).status.code(), Code::kResourceExhausted); - auto concurrent = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.bytes); + auto concurrent = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.reservation()); ASSERT_TRUE(concurrent.status.ok()); EXPECT_EQ(concurrent.deleted_bytes, 0U); deletion = {}; - auto released = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.bytes); + auto released = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.reservation()); ASSERT_TRUE(released.status.ok()); EXPECT_EQ(released.deleted_bytes, kPage); EXPECT_EQ(store.resident_bytes(), 0U); @@ -97,7 +97,7 @@ TEST(MemoryBackingGc, FailedMetadataAdmissionAndPartialTailDoNotReleaseBacking) LedgerFixture ledger({.backend_id = 1, .file_size = input.size(), .address_limit = input.size()}); ASSERT_TRUE(ledger.activate().status.ok()); ledger.credits.seal(); - auto failed = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.bytes); + auto failed = collect_free_local_blocks(store, *ledger.ledger, 0, {}, ledger.credits.credits.reservation()); EXPECT_EQ(failed.status.code(), Code::kResourceExhausted); EXPECT_EQ(failed.next_block, 1U); EXPECT_EQ(store.resident_bytes(), 3 * kPage); diff --git a/lib/crowdb-tree/tests/integration/persistent_flush_pipeline_test.cpp b/lib/crowdb-tree/tests/integration/persistent_flush_pipeline_test.cpp index bab59d87a..550bad6e0 100644 --- a/lib/crowdb-tree/tests/integration/persistent_flush_pipeline_test.cpp +++ b/lib/crowdb-tree/tests/integration/persistent_flush_pipeline_test.cpp @@ -1,7 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. #include "btree/tree_core.h" +#include "crowdb-tree/backend/async_page_store.h" #include "crowdb-tree/backend/page_store.h" +#include "snapshot/primary_backing.h" #include @@ -14,7 +16,7 @@ namespace crowdb::tree #ifdef CROWDB_TREE_TEST_UTIL namespace { -struct TestPipelineStore : MemPageStore +struct TestPipelineStore : MemPageStore, AsyncPageStore { TestPipelineStore() : MemPageStore(4096) { @@ -23,6 +25,70 @@ struct TestPipelineStore : MemPageStore std::atomic writes{0}; std::atomic notify_write{false}; std::binary_semaphore written{0}; + std::atomic blocked_address{UINT64_MAX}; + std::atomic pause_read{false}; + std::atomic read_blocked{false}; + std::atomic notify_other_read{false}; + std::binary_semaphore read_entered{0}; + std::binary_semaphore other_read{0}; + + struct Read + { + PageAddr address = 0; + void *data = nullptr; + size_t bytes = 0; + AsyncCompletion complete; + } held_read; + + uint64_t submit_read(PageAddr address, void *data, size_t bytes, AsyncCompletion complete) override + { + if (address == blocked_address.load() && pause_read.exchange(false)) { + held_read = {.address = address, .data = data, .bytes = bytes, .complete = std::move(complete)}; + read_blocked.store(true); + read_entered.release(); + return 0; + } + if (read_blocked.load() && notify_other_read.exchange(false)) { + other_read.release(); + } + complete.complete(MemPageStore::read_at(address, static_cast(data), bytes)); + return 0; + } + + void resume_read() + { + if (read_blocked.exchange(false)) { + auto read = std::move(held_read); + read.complete.complete(MemPageStore::read_at(read.address, static_cast(read.data), read.bytes)); + } + } + + uint64_t submit_write(PageAddr address, const void *data, size_t bytes, AsyncCompletion complete) override + { + return submit_typed_write(PagePurpose::kBtreePage, address, data, bytes, std::move(complete)); + } + + uint64_t submit_typed_write(PagePurpose purpose, PageAddr address, const void *data, size_t bytes, + AsyncCompletion complete) override + { + complete.complete(write_typed_at(purpose, address, static_cast(data), bytes)); + return 0; + } + + Status submit_fsync(AsyncCompletion complete) override + { + complete.complete(Status::Ok()); + return Status::Ok(); + } + + void cancel(uint64_t /*operation*/) override + { + } + + bool write_completion_is_reloadable() const noexcept override + { + return true; + } Status write_typed_at(PagePurpose purpose, uint64_t address, const uint8_t *data, size_t bytes) override { @@ -115,5 +181,101 @@ TEST(PersistentFlushPipeline, FinalPageWritesWhileAnUnrelatedSubtreeStillWaitsAt ASSERT_TRUE(core->get("0199", &value).ok()); EXPECT_EQ(value.value().to_string(), std::string(32, 'b')); } + +TEST(PersistentFlushPipeline, AColdReadDoesNotBlockOtherReadsOrReadyPageWrites) +{ + Config config; + TreeRuntimeOptions host; + host.workers = 4; + host.backend_workers = 1; + ASSERT_TRUE(TreeRuntime::create(host, &config.runtime).ok()); + auto store = std::make_shared(); + config.page_store = store.get(); + config.async_page_store = store.get(); + config.backend_owner = store; + config.frame_bytes = 4096; + config.leaf_split_bytes = 400; + config.max_inline_value = 32; + config.inner_max_keys = 4; + config.inner_merge_keys = 1; + config.execution.flush_parallelism = 4; + std::shared_ptr core; + { + const auto status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + Batch initial; + for (size_t i = 0; i < 200; ++i) { + const auto digits = std::to_string(i); + initial.ops.push_back( + {.key = std::string(4 - digits.size(), '0') + digits, .kind = OpKind::kPut, .value = std::string(32, 'v')}); + } + ASSERT_TRUE(core->apply(1, initial).ok()); + ASSERT_TRUE(core->checkpoint().ok()); + core.reset(); + { + const auto status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + const auto view = core->capture(); + ASSERT_GE(view->mapping.get(view->root_page_id)->level, 2U); + std::atomic arm_read{true}; + std::atomic pause_handoff{true}; + std::binary_semaphore handoff_entered{0}; + std::function resume_handoff; + core->set_flush_handoff_wait_for_tests([&](uint64_t pid, std::function ready) { + const auto page = view->mapping.get(pid); + if (page && page->primary && page->primary->frame.backing && arm_read.exchange(false)) { + store->blocked_address.store(page->primary->frame.backing->address); + store->pause_read.store(true); + return false; + } + if (pause_handoff.exchange(false)) { + resume_handoff = std::move(ready); + handoff_entered.release(); + return true; + } + return false; + }); + const Batch changes{ + .ops = {{.key = "0000", .kind = OpKind::kPut, .value = std::string(32, 'a')}, + {.key = "0199", .kind = OpKind::kPut, .value = std::string(32, 'b')}} + }; + ASSERT_TRUE(core->apply(2, changes).ok()); + core->binding()->drain(); + store->notify_other_read.store(true); + store->notify_write.store(true); + auto checkpoint = std::async(std::launch::async, [&] { return core->checkpoint(); }); + const bool blocked = store->read_entered.try_acquire_for(std::chrono::seconds(5)); + const bool handoff_waits = handoff_entered.try_acquire_for(std::chrono::seconds(5)); + if (handoff_waits) { + resume_handoff(); + resume_handoff = {}; + } + const bool another_read = store->other_read.try_acquire_for(std::chrono::seconds(2)); + const bool written = store->written.try_acquire_for(std::chrono::seconds(2)); + const bool waiting = checkpoint.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout; + store->resume_read(); + const auto status = checkpoint.get(); + core->binding()->drain(); + core->set_flush_handoff_wait_for_tests({}); + EXPECT_TRUE(blocked); + EXPECT_TRUE(handoff_waits); + EXPECT_TRUE(another_read); + EXPECT_TRUE(written); + EXPECT_TRUE(waiting); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(core->durable_coverage(), 2U); + core.reset(); + { + const auto status = TreeCore::open(config, &core); + ASSERT_TRUE(status.ok()) << status.to_string(); + } + for (const auto *key : {"0000", "0199"}) { + GetView value; + ASSERT_TRUE(core->get(key, &value).ok()); + EXPECT_EQ(value.value().to_string(), std::string(32, std::string_view(key) == "0000" ? 'a' : 'b')); + } +} #endif } // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/recovered_state_backend_test.cpp b/lib/crowdb-tree/tests/integration/recovered_state_backend_test.cpp index 474cb8aa3..94271cd3c 100644 --- a/lib/crowdb-tree/tests/integration/recovered_state_backend_test.cpp +++ b/lib/crowdb-tree/tests/integration/recovered_state_backend_test.cpp @@ -88,7 +88,7 @@ TEST(RecoveredStateBackend, CheckpointRecoveryPreservesBoundaryEmptyChildrenForR OverflowBackingResolver{}, test.credits.credits); std::shared_ptr operation; ASSERT_TRUE(RecoveredStateOperation::create(test.operation->result().latest, loader, runtime, lane, - {.max_step_entries = 1}, test.credits.credits.bytes, &operation) + {.max_step_entries = 1}, test.credits.credits.reservation(), &operation) .ok()); complete_with(runtime, operation->sender()); const auto state = operation->result().state; @@ -119,13 +119,13 @@ TEST(RecoveredStateBackend, CheckpointRecoveryPreservesBoundaryEmptyChildrenForR EXPECT_EQ(keys, reverse ? (std::vector{"z", "a"}) : (std::vector{"a", "z"})); } std::shared_ptr stream; - ASSERT_TRUE( - CapturedFrameStream::create(state, loader, runtime, lane, - { - .encode = {.frame_bytes = 4096, .metadata = test.credits.credits.bytes} + ASSERT_TRUE(CapturedFrameStream::create( + state, loader, runtime, lane, + { + .encode = {.frame_bytes = 4096, .metadata = test.credits.credits.reservation()} }, - {}, &stream) - .ok()); + {}, &stream) + .ok()); for (size_t i = 0; i < 7; ++i) { complete_with(runtime, stream->sender()); ASSERT_TRUE(stream->result().frame); @@ -170,7 +170,7 @@ TEST(RecoveredStateBackend, OverflowReadAndRecaptureNeedOnlyPrimaryDependencies) OverflowBackingResolver{}, test.credits.credits); std::shared_ptr operation; ASSERT_TRUE(RecoveredStateOperation::create(test.operation->result().latest, loader, runtime, lane, {}, - test.credits.credits.bytes, &operation) + test.credits.credits.reservation(), &operation) .ok()); complete_with(runtime, operation->sender()); auto state = operation->result().state; @@ -189,7 +189,8 @@ TEST(RecoveredStateBackend, OverflowReadAndRecaptureNeedOnlyPrimaryDependencies) ASSERT_TRUE(CapturedWriteOperation::create( state, loader, test.store, runtime, lane, {.frames = {.encode = {.frame_bytes = 4096}}, .write = {.max_frame_bytes = 4096}}, - test.credits.credits, {}, extent_write_reservation(ledger, test.credits.credits.bytes), {}, &write) + test.credits.credits, {}, extent_write_reservation(ledger, test.credits.credits.reservation()), {}, + &write) .ok()); complete_with(runtime, write->sender()); ASSERT_TRUE(write->result().pages); diff --git a/lib/crowdb-tree/tests/integration/recovered_state_test.cpp b/lib/crowdb-tree/tests/integration/recovered_state_test.cpp index 3c7333451..e2caf90c8 100644 --- a/lib/crowdb-tree/tests/integration/recovered_state_test.cpp +++ b/lib/crowdb-tree/tests/integration/recovered_state_test.cpp @@ -213,12 +213,12 @@ TEST(RecoveredState, ExhaustedAdmissionFailsWithoutStartingIo) { TestRecoveredGraph test; test.balanced(); - const auto held = test.credits.credits.bytes->available(); - ASSERT_TRUE(test.credits.credits.bytes->claim(held)); + const auto held = test.credits.credits.reservation()->available(); + ASSERT_TRUE(test.credits.credits.reservation()->claim(held)); EXPECT_EQ(test.prepare().code(), Code::kResourceExhausted); EXPECT_EQ(test.store->reads.load(), 0U); EXPECT_FALSE(test.operation); - test.credits.credits.bytes->release(held); + test.credits.credits.reservation()->release(held); test.loader.reset(); test.pages.reset(); test.credits.seal(); diff --git a/lib/crowdb-tree/tests/integration/sparse_block_relocation_test.cpp b/lib/crowdb-tree/tests/integration/sparse_block_relocation_test.cpp index b7528ee34..394938988 100644 --- a/lib/crowdb-tree/tests/integration/sparse_block_relocation_test.cpp +++ b/lib/crowdb-tree/tests/integration/sparse_block_relocation_test.cpp @@ -25,7 +25,7 @@ TEST(SparseBlockRelocation, SelectionSkipsDenseEmptyAndMissingBlocksAndKeepsItsF auto sparse = test.recover(4 * block, 4096); ASSERT_TRUE(dense.status.ok() && sparse.status.ok()); ASSERT_TRUE(test.activate().status.ok()); - auto selected = select_sparse_local_block(*device, *test.ledger, 0, test.credits.credits.bytes); + auto selected = select_sparse_local_block(*device, *test.ledger, 0, test.credits.credits.reservation()); ASSERT_TRUE(selected.status.ok()) << selected.status.to_string(); EXPECT_EQ(selected.exclusion.address, 4 * block); EXPECT_EQ(selected.exclusion.length, block); diff --git a/lib/crowdb-tree/tests/integration/staged_writeback_test.cpp b/lib/crowdb-tree/tests/integration/staged_writeback_test.cpp index fa40b26d5..afac4b106 100644 --- a/lib/crowdb-tree/tests/integration/staged_writeback_test.cpp +++ b/lib/crowdb-tree/tests/integration/staged_writeback_test.cpp @@ -39,8 +39,8 @@ struct TestStagedWriteback Status prepare() { - auto reserved = - local.source.credits.credits.bytes->take_reserved(CheckpointOperation::finish_allocation_bound() + 512); + auto reserved = local.source.credits.credits.reservation()->take_reserved( + CheckpointOperation::finish_allocation_bound() + 512); if (!reserved) { return Status::resource_exhausted(); } diff --git a/lib/crowdb-tree/tests/integration/tree_capacity_backpressure_test.cpp b/lib/crowdb-tree/tests/integration/tree_capacity_backpressure_test.cpp index b676aa630..141d7e916 100644 --- a/lib/crowdb-tree/tests/integration/tree_capacity_backpressure_test.cpp +++ b/lib/crowdb-tree/tests/integration/tree_capacity_backpressure_test.cpp @@ -74,6 +74,35 @@ TEST(TreeCapacityBackpressure, OrdinaryWritesWaitForPersistentCompletionOnOneWor } } +TEST(TreeCapacityBackpressure, WriterBytesPauseBeforeTheMemTableRotationThreshold) +{ + TestCapacityTree test(1000); + test.core.reset(); + test.config.runtime_binding.limits.writer_bytes = 1024 * 1024; + ASSERT_TRUE(TreeCore::open(test.config, &test.core).ok()); + test.store->defer_at = test.store->events.size() + 1; + auto writer = std::async(std::launch::async, [&] { + for (uint64_t slot = 1; slot <= 128; ++slot) { + auto status = test.put(slot); + if (!status.ok()) { + return status; + } + } + return Status::Ok(); + }); + test.store->wait_deferred(); + EXPECT_EQ(writer.wait_for(std::chrono::milliseconds(20)), std::future_status::timeout); + EXPECT_LT(test.core->contiguous_slot(), 128U); + test.store->finish(); + const auto status = writer.get(); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(test.core->contiguous_slot(), 128U); + EXPECT_GT(test.core->durable_coverage(), 0U); + GetView value; + ASSERT_TRUE(test.core->get("128", &value).ok()); + EXPECT_EQ(value.value().to_string(), std::string(20000, static_cast('a' + 128))); +} + TEST(TreeCapacityBackpressure, FailedPersistentFlushKeepsCoverageAndExplicitRepairReopensAdmission) { TestCapacityTree test; @@ -245,10 +274,8 @@ TEST(TreeCapacityBackpressure, PersistentFlushPublishesAllChangesFromEarlierOrdi .reclaim_bytes = 512ULL << 20, .writer_bytes = 2048ULL << 20, .gap_repair_bytes = 256ULL << 20}; - auto store = std::make_shared(); - store->device = std::make_shared(4096); - config.page_store = store->device.get(); - config.async_page_store = store.get(); + auto store = std::make_shared(4096); + config.page_store = store.get(); config.backend_owner = store; config.frame_bytes = 4096; config.leaf_split_bytes = 3000; diff --git a/lib/crowdb-tree/tests/integration/tree_core_admitted_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_admitted_test.cpp index 31c7d920d..987053496 100644 --- a/lib/crowdb-tree/tests/integration/tree_core_admitted_test.cpp +++ b/lib/crowdb-tree/tests/integration/tree_core_admitted_test.cpp @@ -228,7 +228,8 @@ TEST_F(TreeCoreAdmitted, FullSourceAwaitsCompleteFlushCreditThenConsumesThePrepa EXPECT_EQ(core->contiguous_slot(), 129U); const auto terminal = core->publisher().work_metrics()->totals(); EXPECT_EQ(terminal[static_cast(TreeStage::FlushAdmissionWait)].count, 1U); - EXPECT_EQ(terminal[static_cast(TreeStage::BudgetWait)].count, 4U); + EXPECT_EQ(terminal[static_cast(TreeStage::BudgetWait)].count, + detail::FlushReserveBounds::categories.size()); EXPECT_EQ(terminal[static_cast(TreeStage::BudgetWait)].inflight, 0U); } diff --git a/lib/crowdb-tree/tests/integration/tree_core_block_account_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_block_account_test.cpp index f12c02658..bea55fb91 100644 --- a/lib/crowdb-tree/tests/integration/tree_core_block_account_test.cpp +++ b/lib/crowdb-tree/tests/integration/tree_core_block_account_test.cpp @@ -61,7 +61,7 @@ TEST(TreeCoreBlockAccount, ConstructorRecoversAndTransfersExistingBackendWithMon } } -TEST(TreeCoreBlockAccount, FailedAdoptionReleasesDomainAndPrivateCreateCannotBypassIt) +TEST(TreeCoreBlockAccount, FailedOpenReleasesDomainAndPrivateCreateCannotBypassIt) { tree_test::TempDir directory("core_block_fail_"); std::unique_ptr device; @@ -87,10 +87,13 @@ TEST(TreeCoreBlockAccount, FailedAdoptionReleasesDomainAndPrivateCreateCannotByp rejected = true; } EXPECT_TRUE(rejected); - EXPECT_EQ(config.runtime->snapshot().allocated.routing_bytes, baseline.allocated.routing_bytes); - EXPECT_EQ(config.runtime->snapshot().reserved.routing_bytes, baseline.reserved.routing_bytes); ASSERT_TRUE(device->try_claim_runtime_domain()); device->release_runtime_domain(); + device.reset(); + EXPECT_EQ(config.runtime->snapshot().allocated.routing_bytes, baseline.allocated.routing_bytes); + EXPECT_EQ(config.runtime->snapshot().reserved.routing_bytes, baseline.reserved.routing_bytes); + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 4, 5, 65536, 1, &device).ok()); + config.page_store = device.get(); config.runtime_binding.limits.routing_bytes = original_limit; { Crowdbtree accepted(config); diff --git a/lib/crowdb-tree/tests/integration/tree_core_chunk_incremental_test.cpp b/lib/crowdb-tree/tests/integration/tree_core_chunk_incremental_test.cpp index dbc33348e..8577424d9 100644 --- a/lib/crowdb-tree/tests/integration/tree_core_chunk_incremental_test.cpp +++ b/lib/crowdb-tree/tests/integration/tree_core_chunk_incremental_test.cpp @@ -46,14 +46,14 @@ TEST(TreeCoreChunkIncremental, OneLeafChangeReusesOtherIndexPacksAndOldReader) ASSERT_TRUE(test.tree->snapshot().ok()); auto held = test.tree->get_view("key"); ASSERT_TRUE(held.found()); - ChunkRootManifest prior(test.backend.credits.credits.bytes); + ChunkRootManifest prior(test.backend.credits.credits.reservation()); ASSERT_TRUE(decode_chunk_root(test.backend.catalog->committed_manifest, 19, 1, &prior).ok()); ASSERT_GT(prior.indexes.size(), 1U); const auto allocations = test.backend.transport->ids.size(); ASSERT_TRUE(test.tree->apply(2, Batch{.ops = {{.key = "other", .kind = OpKind::kPut, .value = "small"}}}).ok()); const auto status = test.tree->snapshot(); ASSERT_TRUE(status.ok()) << status.to_string(); - ChunkRootManifest current(test.backend.credits.credits.bytes); + ChunkRootManifest current(test.backend.credits.credits.reservation()); ASSERT_TRUE(decode_chunk_root(test.backend.catalog->committed_manifest, 19, 2, ¤t).ok()); ASSERT_EQ(current.indexes.size(), prior.indexes.size()); size_t changed = 0; diff --git a/lib/crowdb-tree/tests/integration/tree_point_read_bound_test.cpp b/lib/crowdb-tree/tests/integration/tree_point_read_bound_test.cpp index 6f3bcc207..00ec20567 100644 --- a/lib/crowdb-tree/tests/integration/tree_point_read_bound_test.cpp +++ b/lib/crowdb-tree/tests/integration/tree_point_read_bound_test.cpp @@ -24,7 +24,7 @@ TEST(TreePointReadBound, SixteenTreesRetainPointWindowsOnOneDefaultHost) trees[index] = std::make_unique(config); ASSERT_TRUE(trees[index]->put("key", "value").ok()); auto *core = TreeCoreAccess::get(*trees[index]); - auto status = core_read_detail::prepare(*core, false, 256, 0, &reads[index]); + auto status = core_read_detail::prepare(*core, &reads[index]); ASSERT_TRUE(status.ok()) << index << ": " << status.to_string(); } for (const auto &tree : trees) { @@ -53,7 +53,7 @@ TEST(TreePointReadBound, MaximumL0ValueBorrowsPayloadWithSmallRoutingAllowance) EXPECT_EQ(value.value().data()[0], 'v'); } -TEST(TreePointReadBound, ColdKnownValueBoundsReadWhileUnknownSummaryRemainsConservative) +TEST(TreePointReadBound, ColdKnownAndUnknownSummariesBothUseActualAllocation) { auto store = std::make_shared(4096); Config config; @@ -82,8 +82,9 @@ TEST(TreePointReadBound, ColdKnownValueBoundsReadWhileUnknownSummaryRemainsConse unknown->stats.kv_unknown_pages = 1; unknown->stats.kv_known_bytes = 0; core_read_detail::ReadPreparation prepared; - status = core_read_detail::prepare(*core, std::move(unknown), core->read_limits(), false, 256, 0, &prepared); - EXPECT_EQ(status.code(), Code::kResourceExhausted); + status = core_read_detail::prepare(*core, std::move(unknown), core->read_limits(), &prepared); + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(prepared.credit, core->read_limits().metadata); } } // namespace } // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/tree_source_window_test.cpp b/lib/crowdb-tree/tests/integration/tree_source_window_test.cpp index 9f52cc38f..1dbe70938 100644 --- a/lib/crowdb-tree/tests/integration/tree_source_window_test.cpp +++ b/lib/crowdb-tree/tests/integration/tree_source_window_test.cpp @@ -5,7 +5,10 @@ #include +#include +#include #include +#include namespace crowdb::tree { @@ -74,6 +77,8 @@ TEST(TreeSourceWindow, WriterCreditReturnsOnlyAfterPhysicalRetirement) std::latch entered{1}, release{1}; } gate; + std::future writes; + auto unblock = [binding](Gate *gate) { gate->release.count_down(); binding->drain(); @@ -90,29 +95,38 @@ TEST(TreeSourceWindow, WriterCreditReturnsOnlyAfterPhysicalRetirement) &gate) .ok()); gate.entered.wait(); - uint64_t rejected = 0; - for (uint64_t slot = 1; slot <= 128; ++slot) { - Batch batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = std::to_string(slot)}}}; - const auto status = tree.apply(slot, batch); - if (!status.ok()) { - EXPECT_EQ(status.code(), Code::kUnavailable) << status.to_string(); - rejected = slot; - break; + writes = std::async(std::launch::async, [&] { + for (uint64_t slot = 1; slot <= 128; ++slot) { + Batch batch{.ops = {{.key = "key", .kind = OpKind::kPut, .value = std::to_string(slot)}}}; + auto status = tree.apply(slot, batch); + if (!status.ok()) { + return status; + } } + return Status::Ok(); + }); + constexpr auto category = static_cast(detail::BudgetClass::kAdmittedWriter); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (binding->account()->snapshot().allocated[category] <= config.runtime_binding.limits.writer_bytes / 2 && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); } - ASSERT_GT(rejected, 1U); - EXPECT_EQ(tree.contiguous_slot(), rejected - 1); + EXPECT_EQ(writes.wait_for(std::chrono::milliseconds(20)), std::future_status::timeout); + EXPECT_GT(tree.contiguous_slot(), 1U); + EXPECT_LT(tree.contiguous_slot(), 128U); EXPECT_GT(binding->account()->retirements_pending_for_tests(), 0U); - constexpr auto category = static_cast(detail::BudgetClass::kAdmittedWriter); - const auto held = binding->account()->snapshot().allocated[category]; + const auto held = binding->account()->snapshot().allocated[category]; EXPECT_GT(held, config.runtime_binding.limits.writer_bytes / 2); cleanup.reset(); + const auto status = writes.get(); + ASSERT_TRUE(status.ok()) << status.to_string(); binding->account()->drain_retirements(); EXPECT_LT(binding->account()->snapshot().allocated[category], held / 2); - EXPECT_EQ(tree.get_view("key").value().to_string(), std::to_string(rejected - 1)); + EXPECT_EQ(tree.contiguous_slot(), 128U); + EXPECT_EQ(tree.get_view("key").value().to_string(), "128"); const Batch next{.ops = {{.key = "key", .kind = OpKind::kPut, .value = "continued"}}}; - ASSERT_TRUE(tree.apply(rejected, next).ok()); - EXPECT_EQ(tree.contiguous_slot(), rejected); + ASSERT_TRUE(tree.apply(129, next).ok()); + EXPECT_EQ(tree.contiguous_slot(), 129U); EXPECT_EQ(tree.get_view("key").value().to_string(), "continued"); } #endif diff --git a/lib/crowdb-tree/tests/load_budget_test_support.h b/lib/crowdb-tree/tests/load_budget_test_support.h index 56d13e02a..dfbc97b98 100644 --- a/lib/crowdb-tree/tests/load_budget_test_support.h +++ b/lib/crowdb-tree/tests/load_budget_test_support.h @@ -47,7 +47,7 @@ struct LoadCredits void seal() const { - credits.bytes->seal(); + credits.reservation()->seal(); credits.io->seal(); } }; diff --git a/lib/crowdb-tree/tests/local_checkpoint_test_support.h b/lib/crowdb-tree/tests/local_checkpoint_test_support.h index cbd003337..61009437b 100644 --- a/lib/crowdb-tree/tests/local_checkpoint_test_support.h +++ b/lib/crowdb-tree/tests/local_checkpoint_test_support.h @@ -122,7 +122,8 @@ struct TestCheckpoint TestCheckpoint() { store->device = source.store->device; - EXPECT_TRUE(LocalCheckpointRegistry::create(source.ledger, source.credits.credits.bytes, {}, ®istry).ok()); + EXPECT_TRUE( + LocalCheckpointRegistry::create(source.ledger, source.credits.credits.reservation(), {}, ®istry).ok()); } std::shared_ptr materialize(std::string value = "v", uint64_t coverage = 10) diff --git a/lib/crowdb-tree/tests/recovered_state_test_support.h b/lib/crowdb-tree/tests/recovered_state_test_support.h index e76d4ed7d..87c18ea94 100644 --- a/lib/crowdb-tree/tests/recovered_state_test_support.h +++ b/lib/crowdb-tree/tests/recovered_state_test_support.h @@ -16,7 +16,7 @@ struct TestRecoveredGraph std::shared_ptr lane = std::make_shared(8); std::shared_ptr store = std::make_shared(); std::shared_ptr pages = - detail::reserved_shared(credits.credits.bytes, credits.credits.bytes); + detail::reserved_shared(credits.credits.reservation(), credits.credits.reservation()); std::shared_ptr loader = std::make_shared( store, runtime, lane, PageLoadLimits{}, OverflowBackingResolver{}, credits.credits); std::shared_ptr operation; @@ -44,8 +44,8 @@ struct TestRecoveredGraph .frame_bytes = static_cast(raw.size()), .backing = backing}; if (frame.type == page_type::kLeafBase) { - ASSERT_TRUE(capture_stored_overflow(LeafFrameView(raw.data(), raw.size()), 4096, credits.credits.bytes, - &frame.stored_overflow) + ASSERT_TRUE(capture_stored_overflow(LeafFrameView(raw.data(), raw.size()), 4096, + credits.credits.reservation(), &frame.stored_overflow) .ok()); } else if (frame.type == page_type::kOverflowFrame) { @@ -97,7 +97,8 @@ struct TestRecoveredGraph Status prepare(RecoveredStateLimits limits = {}) { - return RecoveredStateOperation::create(pages, loader, runtime, lane, limits, credits.credits.bytes, &operation); + return RecoveredStateOperation::create(pages, loader, runtime, lane, limits, credits.credits.reservation(), + &operation); } void run(Completion *done) const diff --git a/lib/crowdb-tree/tests/unit/allocation_account_adoption_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_adoption_test.cpp index 1dcda9561..63554a996 100644 --- a/lib/crowdb-tree/tests/unit/allocation_account_adoption_test.cpp +++ b/lib/crowdb-tree/tests/unit/allocation_account_adoption_test.cpp @@ -26,7 +26,7 @@ TEST(AllocationAccountAdoption, FirstBindChargesEveryExistingOwnerBeforeFurtherG const auto bound = host.snapshot().allocated[kRouting]; EXPECT_GT(bound, 103U + sizeof(AllocationAccount)); EXPECT_EQ(account->snapshot().physical[kMetadata], bound); - EXPECT_FALSE(budget->claim(8192 - bound + 1)); + EXPECT_FALSE(budget->claim(12288 - bound + 1)); allocator.deallocate(data, 103); EXPECT_EQ(account->snapshot().allocated[kRouting], bound - 103); } diff --git a/lib/crowdb-tree/tests/unit/allocation_account_grant_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_grant_test.cpp index 66bfb78c4..8befe354f 100644 --- a/lib/crowdb-tree/tests/unit/allocation_account_grant_test.cpp +++ b/lib/crowdb-tree/tests/unit/allocation_account_grant_test.cpp @@ -22,7 +22,8 @@ TEST(BackendAllocationGrant, UsesExistingAdmissionAndRecyclesTemporaryWindows) const auto control = test.old->snapshot().allocated[kRouting] - test.base; ASSERT_LE(control, BackendAllocationGrant::allocation_bound()); const auto window = 8192 - test.base - control; - EXPECT_FALSE(test.budget->claim(1)); + ASSERT_TRUE(test.budget->claim(1)); + test.budget->release(1); { MetadataAllocator allocator(grant); for (size_t i = 0; i != 3; ++i) { @@ -68,8 +69,8 @@ TEST(BackendAllocationGrant, RetainedAllocationMovesWithBackendAfterOperationSea TEST(BackendAllocationGrant, TargetQuotaRefusalReturnsSourceCreditWithoutDoubleCharge) { TestAccounts test; - auto blocker = ResourceBudget::try_reserve(test.old, BudgetClass::kRouting, 8192 - test.base); - ASSERT_TRUE(blocker); + const auto blocker = 12288 - test.base; + ASSERT_TRUE(test.budget->claim(blocker)); auto source = ResourceBudget::try_reserve(test.next, BudgetClass::kRouting, 2048); ASSERT_TRUE(source); if (!source) { @@ -83,7 +84,7 @@ TEST(BackendAllocationGrant, TargetQuotaRefusalReturnsSourceCreditWithoutDoubleC EXPECT_EQ(test.next->snapshot().reserved[kRouting], 0U); EXPECT_EQ(test.host.snapshot().allocated[kRouting], host_before.allocated[kRouting]); EXPECT_EQ(test.host.snapshot().reserved[kRouting], host_before.reserved[kRouting] - 2048); - blocker.reset(); + test.budget->release(blocker); test.finish(); } diff --git a/lib/crowdb-tree/tests/unit/allocation_account_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_test.cpp index 39564898d..e8c319c0e 100644 --- a/lib/crowdb-tree/tests/unit/allocation_account_test.cpp +++ b/lib/crowdb-tree/tests/unit/allocation_account_test.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. #include "allocation_account_test_support.h" +#include "btree/source_write_budget.h" namespace crowdb::tree::detail { @@ -8,7 +9,56 @@ namespace { using namespace test_allocation_account; -TEST(AllocationAccount, TransferAtFullHostCapacityMovesActualTreeOwnership) +TEST(AllocationAccount, ActualMemoryCanReachTheMaximumAndStopsNewGrowthAboveOneAndAHalf) +{ + constexpr size_t maximum = 1024 * 1024; + ResourceBudget::Limits limits{}; + limits[kRouting] = maximum / 2; + limits[static_cast(BudgetClass::kAdmittedWriter)] = maximum / 2; + limits[static_cast(BudgetClass::kIo)] = 4; + ResourceBudget host(limits, 4); + auto account = host.make_account(limits); + std::shared_ptr owner; + ASSERT_TRUE(AllocationAccount::create(account, BudgetClass::kRouting, &owner).ok()); + const auto baseline = account->snapshot().allocated[kRouting]; + auto accepted = ResourceBudget::try_reserve(account, BudgetClass::kRouting, 16); + if (!accepted) { + FAIL() << "missing accepted completion credit"; + } + { + BudgetAllocator allocator(owner); + auto *first = allocator.allocate(maximum - baseline); + EXPECT_EQ(account->snapshot().allocated[kRouting], maximum); + auto *second = allocator.allocate(maximum / 2); + EXPECT_EQ(account->snapshot().allocated[kRouting], maximum + (maximum / 2)); + EXPECT_THROW( + { + auto *rejected = allocator.allocate(1); + allocator.deallocate(rejected, 1); + }, + std::bad_alloc); + auto completion = accepted.value().allocate(16); + ASSERT_TRUE(completion); + EXPECT_FALSE(ResourceBudget::try_reserve(account, BudgetClass::kAdmittedWriter, 1)); + SourceWriteReservation rejected; + EXPECT_EQ(SourceWriteReservation::create({.account = account, .max_entries = 1, .max_gap_entries = 1}, + {.entries = 1, .key_bytes = 1}, BudgetClass::kAdmittedWriter, + &rejected) + .code(), + Code::kResourceExhausted); + completion.reset(); + EXPECT_TRUE(ResourceBudget::try_reserve(account, BudgetClass::kAdmittedWriter, 1)); + allocator.deallocate(first, maximum - baseline); + auto *resumed = allocator.allocate(1); + allocator.deallocate(resumed, 1); + allocator.deallocate(second, maximum / 2); + EXPECT_EQ(account->snapshot().allocated[kRouting], baseline); + } + owner.reset(); + EXPECT_EQ(account->snapshot().allocated[kRouting], 0U); +} + +TEST(AllocationAccount, UnusedHostReservationDoesNotBlockActualAllocationOrOwnershipTransfer) { TestAccounts test; { @@ -17,7 +67,8 @@ TEST(AllocationAccount, TransferAtFullHostCapacityMovesActualTreeOwnership) auto filler = test.host.make_account(limits(16384)); auto capacity = ResourceBudget::try_reserve(filler, BudgetClass::kRouting, 16384 - test.base - 128); ASSERT_TRUE(capacity); - EXPECT_FALSE(test.budget->claim(1)); + ASSERT_TRUE(test.budget->claim(1)); + test.budget->release(1); const auto before = test.host.snapshot(); ASSERT_TRUE(test.budget->rebind(test.next).ok()); EXPECT_EQ(test.host.snapshot().allocated[kRouting], before.allocated[kRouting]); @@ -37,16 +88,16 @@ TEST(AllocationAccount, TransferAtFullHostCapacityMovesActualTreeOwnership) TEST(AllocationAccount, TargetQuotaFailureLeavesOriginalAccountUsable) { TestAccounts test; - auto small = test.host.make_account(limits(test.base - 1)); + auto small = test.host.make_account(limits(test.base / 2)); EXPECT_FALSE(test.budget->rebind(small).ok()); test.expect_only_control(test.old); EXPECT_EQ(small->snapshot().allocated[kRouting], 0U); ASSERT_TRUE(test.budget->claim(17)); test.budget->release(17); ASSERT_TRUE(test.budget->rebind(test.next).ok()); - EXPECT_FALSE(test.budget->claim(8192 - test.base + 1)); - ASSERT_TRUE(test.budget->claim(8192 - test.base)); - test.budget->release(8192 - test.base); + EXPECT_FALSE(test.budget->claim(12288 - test.base + 1)); + ASSERT_TRUE(test.budget->claim(12288 - test.base)); + test.budget->release(12288 - test.base); test.finish(); } @@ -88,7 +139,7 @@ TEST(AllocationAccount, ForeignHostAndFailedBootstrapDoNotLeakAdmission) EXPECT_FALSE(test.budget->rebind(destination).ok()); test.expect_only_control(test.old); EXPECT_EQ(foreign.snapshot().allocated[kRouting], 0U); - auto small = test.host.make_account(limits(sizeof(AllocationAccount))); + auto small = test.host.make_account(limits(sizeof(AllocationAccount) * 2 / 3)); std::shared_ptr result; EXPECT_FALSE(AllocationAccount::create(small, BudgetClass::kRouting, &result).ok()); EXPECT_FALSE(result); diff --git a/lib/crowdb-tree/tests/unit/allocation_account_transfer_test.cpp b/lib/crowdb-tree/tests/unit/allocation_account_transfer_test.cpp index def124180..a4bf5c90a 100644 --- a/lib/crowdb-tree/tests/unit/allocation_account_transfer_test.cpp +++ b/lib/crowdb-tree/tests/unit/allocation_account_transfer_test.cpp @@ -79,7 +79,7 @@ TEST(AllocationAccountTransfer, FailedTransferAppliesConcurrentFreesToOriginalAc TestAccounts test; ASSERT_TRUE(test.budget->claim(43)); test.budget->record_allocation(AllocationKind::kFrame, 43, true); - auto small = test.host.make_account(limits(test.base - 1)); + auto small = test.host.make_account(limits(test.base / 2)); const auto status = AllocationAccountTestAccess::rebind_for_tests(*test.budget, small, [&](Step step, size_t) noexcept { if (step == Step::kClosed) { diff --git a/lib/crowdb-tree/tests/unit/apply_dispatcher_test.cpp b/lib/crowdb-tree/tests/unit/apply_dispatcher_test.cpp index f958e6063..823e97b25 100644 --- a/lib/crowdb-tree/tests/unit/apply_dispatcher_test.cpp +++ b/lib/crowdb-tree/tests/unit/apply_dispatcher_test.cpp @@ -41,7 +41,7 @@ TEST(ApplyDispatcher, ConcurrentProducersRetainAndExecuteEachQueuedOwnerOnce) auto runtime = std::make_shared(2, 4); auto lane = std::make_shared(1); std::shared_ptr dispatcher; - ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 3, credits.credits.bytes, &dispatcher).ok()); + ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 3, credits.credits.reservation(), &dispatcher).ok()); credits.seal(); std::atomic completed{0}; std::atomic destroyed{0}; @@ -71,7 +71,7 @@ TEST(ApplyDispatcher, ReentrantContinuationKeepsFifoOrderAcrossBoundedTurns) auto runtime = std::make_shared(1, 2); auto lane = std::make_shared(1); std::shared_ptr dispatcher; - ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 1, credits.credits.bytes, &dispatcher).ok()); + ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 1, credits.credits.reservation(), &dispatcher).ok()); struct Context : std::enable_shared_from_this { @@ -119,7 +119,7 @@ TEST(ApplyDispatcher, DuplicateWakeupsCoalesceAndPrepaidProgressSurvivesRuntimeC auto runtime = std::make_shared(1, 2); auto lane = std::make_shared(1); std::shared_ptr dispatcher; - ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 1, credits.credits.bytes, &dispatcher).ok()); + ASSERT_TRUE(ApplyDispatcher::create(runtime, lane, 1, credits.credits.reservation(), &dispatcher).ok()); struct Gate { diff --git a/lib/crowdb-tree/tests/unit/apply_frontier_test.cpp b/lib/crowdb-tree/tests/unit/apply_frontier_test.cpp index 533deb16a..c6bc1eb99 100644 --- a/lib/crowdb-tree/tests/unit/apply_frontier_test.cpp +++ b/lib/crowdb-tree/tests/unit/apply_frontier_test.cpp @@ -104,7 +104,7 @@ TEST(ApplyFrontier, FixedStorageIsChargedOnceAndCompletionsWorkAfterSealing) { LoadCredits credits; std::shared_ptr tracker; - ASSERT_TRUE(ApplyFrontier::create(0, 8, credits.credits.bytes, &tracker).ok()); + ASSERT_TRUE(ApplyFrontier::create(0, 8, credits.credits.reservation(), &tracker).ok()); const auto allocated = credits.budget.snapshot().allocated[LoadCredits::kBytes]; EXPECT_GE(allocated, sizeof(ApplyFrontier) + (8 * sizeof(std::atomic))); const auto bound = ApplyFrontier::allocation_bound(8); @@ -127,8 +127,8 @@ TEST(ApplyFrontier, InvalidWindowAndAllocationFailureLeaveOutputUnchanged) const auto original = tracker; EXPECT_EQ(ApplyFrontier::create(0, 0, {}, &tracker).code(), Code::kInvalidArgument); EXPECT_EQ(ApplyFrontier::create(0, SIZE_MAX, {}, &tracker).code(), Code::kInvalidArgument); - AllocationLease pressure(credits.credits.bytes, credits.credits.bytes->available()); - EXPECT_EQ(ApplyFrontier::create(0, 4, credits.credits.bytes, &tracker).code(), Code::kResourceExhausted); + AllocationLease pressure(credits.credits.reservation(), credits.credits.reservation()->available()); + EXPECT_EQ(ApplyFrontier::create(0, 4, credits.credits.reservation(), &tracker).code(), Code::kResourceExhausted); EXPECT_EQ(tracker, original); } } // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/apply_progress_test.cpp b/lib/crowdb-tree/tests/unit/apply_progress_test.cpp index 74d78ad38..a736bf6fc 100644 --- a/lib/crowdb-tree/tests/unit/apply_progress_test.cpp +++ b/lib/crowdb-tree/tests/unit/apply_progress_test.cpp @@ -42,7 +42,7 @@ TEST(ApplyProgress, WakeupCoversTransitionsBeforeAndAfterRegistration) { test_read::LoadCredits credits; std::shared_ptr progress; - ASSERT_TRUE(ApplyProgress::create(1, credits.credits.bytes, &progress).ok()); + ASSERT_TRUE(ApplyProgress::create(1, credits.credits.reservation(), &progress).ok()); const auto initial = progress->sequence(); ApplyProgress::Request before(*progress, initial); TestProgressResult first; @@ -65,7 +65,7 @@ TEST(ApplyProgress, BoundedWaitersReturnTheirSlotsOnCancellation) { test_read::LoadCredits credits; std::shared_ptr progress; - ASSERT_TRUE(ApplyProgress::create(1, credits.credits.bytes, &progress).ok()); + ASSERT_TRUE(ApplyProgress::create(1, credits.credits.reservation(), &progress).ok()); ApplyProgress::Request first(*progress, progress->sequence()); ApplyProgress::Request overflow(*progress, progress->sequence()); TestProgressResult a; @@ -92,7 +92,7 @@ TEST(ApplyProgress, LastIngressOwnerStopsPendingRequestsAndPreservesTheirCharged { test_read::LoadCredits credits; std::shared_ptr progress; - ASSERT_TRUE(ApplyProgress::create(2, credits.credits.bytes, &progress).ok()); + ASSERT_TRUE(ApplyProgress::create(2, credits.credits.reservation(), &progress).ok()); auto request = std::make_unique(*progress, progress->sequence()); TestProgressResult result; auto operation = stdexec::connect(request->sender(), TestProgressReceiver{&result}); @@ -132,7 +132,7 @@ TEST(ApplyProgress, TerminalMayDestroyTheRequestInline) { test_read::LoadCredits credits; std::shared_ptr progress; - ASSERT_TRUE(ApplyProgress::create(1, credits.credits.bytes, &progress).ok()); + ASSERT_TRUE(ApplyProgress::create(1, credits.credits.reservation(), &progress).ok()); auto request = std::make_unique(*progress, progress->sequence()); TestProgressResult result; auto operation = @@ -147,7 +147,7 @@ TEST(ApplyProgress, ConcurrentRegistrationAndTransitionNeverLoseTheWakeup) { test_read::LoadCredits credits; std::shared_ptr progress; - ASSERT_TRUE(ApplyProgress::create(1, credits.credits.bytes, &progress).ok()); + ASSERT_TRUE(ApplyProgress::create(1, credits.credits.reservation(), &progress).ok()); for (size_t iteration = 0; iteration < 64; ++iteration) { ApplyProgress::Request request(*progress, progress->sequence()); TestProgressResult result; diff --git a/lib/crowdb-tree/tests/unit/block_account_grant_test.cpp b/lib/crowdb-tree/tests/unit/block_account_grant_test.cpp index 0c18318ec..b5afaa2ad 100644 --- a/lib/crowdb-tree/tests/unit/block_account_grant_test.cpp +++ b/lib/crowdb-tree/tests/unit/block_account_grant_test.cpp @@ -28,12 +28,9 @@ TEST(BlockAccountGrant, SparseGrowthAndBounceUsePrepaidCapacityAtTheHardLimit) std::shared_ptr grant; const BackendOperationLimits work{.max_io_bytes = 512, .write_calls = 1, .workers = 1, .barriers = false}; ASSERT_TRUE(reserve_backend_operation(store.get(), account, work, &grant).ok()); - const auto admitted = account->snapshot(); - auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kRouting, - kCapacity - admitted.allocated[kRouting] - admitted.reserved[kRouting]); - ASSERT_TRUE(occupied); - const uint8_t value = 73; - const uint64_t offset = (65536ULL * 1000) + 17; + auto occupied = use_remaining_memory(account, block_allocation_budget(*store)); + const uint8_t value = 73; + const uint64_t offset = (65536ULL * 1000) + 17; EXPECT_EQ(store->write_at(offset, &value, 1).code(), Code::kResourceExhausted); ASSERT_TRUE(store->write_at(offset, &value, 1, grant).ok()); EXPECT_EQ(account->snapshot().physical[kIoBytes], 0U); diff --git a/lib/crowdb-tree/tests/unit/block_account_test.cpp b/lib/crowdb-tree/tests/unit/block_account_test.cpp index 3a18813d0..8991a28ab 100644 --- a/lib/crowdb-tree/tests/unit/block_account_test.cpp +++ b/lib/crowdb-tree/tests/unit/block_account_test.cpp @@ -61,7 +61,7 @@ TEST(BlockAccount, SameHostReopenTransfersFdOwnersAndRejectsSmallerOrForeignAcco ASSERT_TRUE(bind_block_account(store.get(), old, 1, &adopted).ok()); ASSERT_TRUE(store->delete_block(1).ok()); const auto retained = old->snapshot().allocated[kRouting]; - auto small = host.make_account(limits(retained - 1)); + auto small = host.make_account(limits(retained / 2)); EXPECT_EQ(bind_block_account(store.get(), small, 1, &adopted).code(), Code::kResourceExhausted); EXPECT_EQ(old->snapshot().allocated[kRouting], retained); EXPECT_EQ(small->snapshot().allocated[kRouting], 0U); @@ -110,8 +110,7 @@ TEST(BlockAccount, FailedFirstAdoptionAndRejectedGrowthLeaveFilesAndExistingRead EXPECT_EQ(host.snapshot().allocated[kRouting], 0U); auto account = host.make_account(limits(kCapacity)); ASSERT_TRUE(bind_block_account(store.get(), account, 1, &adopted).ok()); - auto occupied = ResourceBudget::try_reserve(account, BudgetClass::kRouting, kCapacity - adopted.owned_bytes); - ASSERT_TRUE(occupied); + auto occupied = use_remaining_memory(account, block_allocation_budget(*store)); EXPECT_EQ(store->write_at(4096000, &value, 1).code(), Code::kResourceExhausted); EXPECT_FALSE(std::filesystem::exists(directory.path + "/3-4.blk-1000")); EXPECT_EQ(store->num_extents(), 1U); diff --git a/lib/crowdb-tree/tests/unit/c_api_read_budget_test.cpp b/lib/crowdb-tree/tests/unit/c_api_read_budget_test.cpp index 6bfb7c6ac..95ccd1b3e 100644 --- a/lib/crowdb-tree/tests/unit/c_api_read_budget_test.cpp +++ b/lib/crowdb-tree/tests/unit/c_api_read_budget_test.cpp @@ -1,12 +1,45 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" #include "btree/tree_core_access.h" #include "c_api_internal.h" #include +#include + namespace crowdb::tree { +namespace +{ +struct TestReadMemoryPressure +{ + explicit TestReadMemoryPressure(const std::shared_ptr &account) + { + if (!detail::AllocationAccount::create(account, detail::BudgetClass::kRouting, &owner).ok()) { + throw std::runtime_error("cannot create the test allocation account"); + } + auto credit = detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, 1); + if (!credit) { + throw std::runtime_error("cannot retain the accepted completion byte"); + } + held.emplace(detail::test_allocation_account::use_remaining_memory(account, owner)); + completion = credit->allocate(1); + } + + void reset() + { + held.reset(); + completion.reset(); + owner.reset(); + } + + std::shared_ptr owner; + std::optional completion; + std::optional held; +}; +} // namespace + TEST(CAbiReadBudget, LazySnapshotIterationReturnsPressureInsteadOfThrowingAcrossTheAbi) { ct_page_store *raw_store = nullptr; @@ -29,13 +62,10 @@ TEST(CAbiReadBudget, LazySnapshotIterationReturnsPressureInsteadOfThrowingAcross std::unique_ptr iterator(raw_iterator, &ct_iter_release); auto *core = TreeCoreAccess::get(*c_api_tree(tree.get())); core->binding()->drain(); - const auto account = core->binding()->account(); - auto blocker = - detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, - detail::ResourceBudget::available(account, detail::BudgetClass::kRouting)); - ASSERT_TRUE(blocker); - int32_t valid = 7; - ct_status status = 0; + const auto account = core->binding()->account(); + TestReadMemoryPressure blocker(account); + int32_t valid = 7; + ct_status status = 0; EXPECT_NO_THROW(status = ct_iter_next(iterator.get(), nullptr, nullptr, nullptr, nullptr, &valid)); EXPECT_EQ(status, static_cast(Code::kResourceExhausted)); EXPECT_EQ(valid, 0); @@ -61,20 +91,21 @@ TEST(CAbiReadBudget, PressureReturnsResourceExhaustedAndDoesNotLatchIoFailure) const uint8_t key = 'k'; const uint8_t value = 'v'; ASSERT_EQ(ct_apply_put(tree.get(), 1, &key, 1, &value, 1), 0); - const auto account = TreeCoreAccess::get(*c_api_tree(tree.get()))->binding()->account(); - auto blocker = - detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, - detail::ResourceBudget::available(account, detail::BudgetClass::kRouting)); - ASSERT_TRUE(blocker); - int32_t found = 7; - uint64_t slot = 7; - ct_buf result{}; + auto *core = TreeCoreAccess::get(*c_api_tree(tree.get())); + core->binding()->drain(); + const auto account = core->binding()->account(); + TestReadMemoryPressure blocker(account); + int32_t found = 7; + uint64_t slot = 7; + ct_buf result{}; EXPECT_EQ(ct_get(tree.get(), &key, 1, &found, &slot, &result), static_cast(Code::kResourceExhausted)); EXPECT_EQ(found, 0); EXPECT_EQ(slot, 0U); EXPECT_EQ(result.data, nullptr); EXPECT_EQ(result.len, 0U); + const auto before = account->snapshot().allocated; EXPECT_FALSE(c_api_tree(tree.get())->get_view("k").found()); + EXPECT_EQ(account->snapshot().allocated, before); ct_stats stats{}; ct_get_stats(tree.get(), &stats); EXPECT_EQ(stats.io_failed, 0); diff --git a/lib/crowdb-tree/tests/unit/captured_rebase_reuse_test.cpp b/lib/crowdb-tree/tests/unit/captured_rebase_reuse_test.cpp index 90c53f29c..b9ebec90b 100644 --- a/lib/crowdb-tree/tests/unit/captured_rebase_reuse_test.cpp +++ b/lib/crowdb-tree/tests/unit/captured_rebase_reuse_test.cpp @@ -111,7 +111,7 @@ TEST(CapturedRebaseReuse, ZeroReplacementCreditPreservesExactPageAndLaterOverlay if (!bound) { FAIL() << "zero-replacement rebase bound overflow"; } - auto paid = credits.credits.bytes->take_reserved( + auto paid = credits.credits.reservation()->take_reserved( *bound + detail::shared_allocation_bound()); if (!paid) { FAIL() << "zero-replacement rebase credit unavailable"; diff --git a/lib/crowdb-tree/tests/unit/chunk_fixed_pack_test.cpp b/lib/crowdb-tree/tests/unit/chunk_fixed_pack_test.cpp index 7f710450e..5f44a4f47 100644 --- a/lib/crowdb-tree/tests/unit/chunk_fixed_pack_test.cpp +++ b/lib/crowdb-tree/tests/unit/chunk_fixed_pack_test.cpp @@ -137,4 +137,35 @@ TEST(ChunkFixedPack, WriteFailureKeepsProvisionalReferencesProtected) EXPECT_EQ(catalog->record->packs.size(), 1U); EXPECT_EQ(output.length, 0U); } + +TEST(ChunkFixedPack, AggregateFailurePublishesNoMemberAndRetainsTheExactStage) +{ + for (const bool unknown : {true, false}) { + auto catalog = std::make_shared(); + auto transport = std::make_shared(catalog); + transport->unknown_allocate = unknown; + transport->fail_write = !unknown; + ChunkFixedPackStore store(17, 1, 2, catalog, transport); + TestPackBudget budget; + auto credit = budget.bytes(); + std::array small{}; + std::array large{}; + std::array output; + const std::array inputs{ + {{.input = small, .out = output.data()}, {.input = large, .out = &output[1]}} + }; + EXPECT_EQ(store.write_many(operation, fixed, PagePurpose::kBtreePage, inputs, credit).code(), + Code::kUnavailable); + if (!catalog->record) { + FAIL() << "aggregate lost its durable stage"; + } + EXPECT_EQ(catalog->record->id, operation); + EXPECT_EQ(catalog->record->chunk_low, fixed.low); + EXPECT_EQ(catalog->record->state, unknown ? ChunkStageState::Reserved : ChunkStageState::Writing); + EXPECT_EQ(catalog->record->packs.size(), unknown ? 0U : 2U); + EXPECT_EQ(transport->allocations, 1U); + EXPECT_EQ(output[0].length, 0U); + EXPECT_EQ(output[1].length, 0U); + } +} } // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/unit/chunk_root_codec_test.cpp b/lib/crowdb-tree/tests/unit/chunk_root_codec_test.cpp index 0c1434d9e..5dcf4029b 100644 --- a/lib/crowdb-tree/tests/unit/chunk_root_codec_test.cpp +++ b/lib/crowdb-tree/tests/unit/chunk_root_codec_test.cpp @@ -39,13 +39,13 @@ detail::ChunkFixedPackRef test_index_pack(uint32_t bytes) TEST(ChunkRootCodec, RoundTripKeepsRootSequenceAndWalDistinctFromPageGeneration) { test_read::LoadCredits credits; - ChunkRootManifest root(credits.credits.bytes); + ChunkRootManifest root(credits.credits.reservation()); root.identity = test_identity(); root.frame_count = 2; root.indexes.push_back({.first_page = 0, .last_page = 9, .frame_count = 2, .pack = test_index_pack(512)}); - ChunkMetadataBytes bytes{detail::ReservationAllocator(credits.credits.bytes)}; + ChunkMetadataBytes bytes{detail::ReservationAllocator(credits.credits.reservation())}; ASSERT_TRUE(encode_chunk_root(root, &bytes).ok()); - ChunkRootManifest decoded(credits.credits.bytes); + ChunkRootManifest decoded(credits.credits.reservation()); ASSERT_TRUE(decode_chunk_root(bytes, 19, 7, &decoded).ok()); EXPECT_EQ(decoded.identity.root_generation, 7U); EXPECT_EQ(decoded.identity.snapshot_sequence, 4U); @@ -53,7 +53,7 @@ TEST(ChunkRootCodec, RoundTripKeepsRootSequenceAndWalDistinctFromPageGeneration) EXPECT_EQ(decoded.identity.coverage, 15U); EXPECT_EQ(decoded.identity.wal_replay_offset, 4096U); EXPECT_EQ(decoded.indexes.size(), 1U); - ChunkRootManifest wrong_root(credits.credits.bytes); + ChunkRootManifest wrong_root(credits.credits.reservation()); EXPECT_EQ(decode_chunk_root(bytes, 19, 8, &wrong_root).code(), Code::kCorruption); chunk_codec::put(bytes, 13, UINT64_MAX); chunk_codec::seal(bytes); @@ -65,14 +65,15 @@ TEST(ChunkRootCodec, IndexValidatesExactNamespaceAndRejectsOversizedPackFields) test_chunk::TestChunkBackend test; std::shared_ptr pack; ASSERT_TRUE(test.prepare(8192, &pack).ok()); - auto backing = detail::reserved_shared(test.credits.credits.bytes); + auto backing = detail::reserved_shared(test.credits.credits.reservation()); backing->backend_id = 17; backing->length = 8192; backing->owner = pack; backing->chunk = pack; backing->storage = BackingStorage::kChunkPack; auto refs = detail::reserved_shared( - test.credits.credits.bytes, detail::ReservationAllocator(test.credits.credits.bytes)); + test.credits.credits.reservation(), + detail::ReservationAllocator(test.credits.credits.reservation())); refs->push_back({.head = 4, .bytes = 123}); DurableCapturedFrame frame{.page_id = 0, .type = page_type::kLeafBase, @@ -84,7 +85,7 @@ TEST(ChunkRootCodec, IndexValidatesExactNamespaceAndRejectsOversizedPackFields) .stored_overflow = refs, .summary = {}}; const DurableCapturedFrame *frames[]{&frame}; - ChunkMetadataBytes bytes{detail::ReservationAllocator(test.credits.credits.bytes)}; + ChunkMetadataBytes bytes{detail::ReservationAllocator(test.credits.credits.reservation())}; ASSERT_TRUE(encode_chunk_index(test_identity(), 2, frames, &bytes).ok()); uint64_t count = 0; size_t offset = 0; @@ -123,10 +124,11 @@ TEST(ChunkRuntimePageStore, UnsupportedCatalogDoesNotExposeAnIndependentAddressS test_read::LoadCredits credits; auto catalog = std::make_shared(1); auto transport = std::make_shared(); - detail::ChunkRuntimePageStore store({.tree = 19, .owner_epoch = 1}, catalog, transport, credits.credits.bytes); + detail::ChunkRuntimePageStore store({.tree = 19, .owner_epoch = 1}, catalog, transport, + credits.credits.reservation()); EXPECT_FALSE(store.supported()); std::shared_ptr domain; - EXPECT_EQ(store.make_domain(17, credits.credits.bytes, &domain).code(), Code::kNotSupported); + EXPECT_EQ(store.make_domain(17, credits.credits.reservation(), &domain).code(), Code::kNotSupported); EXPECT_FALSE(domain); EXPECT_EQ(store.sync().code(), Code::kNotSupported); EXPECT_EQ(store.write_at(0, nullptr, 0).code(), Code::kNotSupported); @@ -148,24 +150,24 @@ TEST(ChunkStageCodec, OwnedEncodingAndDecodingRetainTheCallerAllocator) .manifest_generation = 0, .purpose = 2, .state = detail::ChunkStageState::Reloadable, - .packs = decltype(Record::packs)(MetadataAllocator(credits.credits.bytes)) + .packs = decltype(Record::packs)(MetadataAllocator(credits.credits.reservation())) }; record.packs.push_back({.offset = 0, .length = 32, .checksum = 9}); auto decoded = record; decoded.packs.clear(); decoded.packs.shrink_to_fit(); - const auto before = credits.credits.bytes->available(); + const auto before = credits.credits.reservation()->available(); { - detail::ChunkStageBytes bytes{MetadataAllocator(credits.credits.bytes)}; + detail::ChunkStageBytes bytes{MetadataAllocator(credits.credits.reservation())}; ASSERT_TRUE(detail::encode_stage_record_owned(record, &bytes).ok()); ASSERT_TRUE(detail::decode_stage_record(bytes, &decoded).ok()); - EXPECT_EQ(decoded.packs.get_allocator().owner(), credits.credits.bytes); + EXPECT_EQ(decoded.packs.get_allocator().owner(), credits.credits.reservation()); EXPECT_EQ(decoded, record); - EXPECT_EQ(before - credits.credits.bytes->available(), + EXPECT_EQ(before - credits.credits.reservation()->available(), bytes.capacity() + (decoded.packs.capacity() * sizeof(detail::ChunkStagePack))); } decoded.packs.clear(); decoded.packs.shrink_to_fit(); - EXPECT_EQ(credits.credits.bytes->available(), before); + EXPECT_EQ(credits.credits.reservation()->available(), before); } } // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/extent_exclusion_test.cpp b/lib/crowdb-tree/tests/unit/extent_exclusion_test.cpp index 545c4ad47..ebd7edaee 100644 --- a/lib/crowdb-tree/tests/unit/extent_exclusion_test.cpp +++ b/lib/crowdb-tree/tests/unit/extent_exclusion_test.cpp @@ -11,7 +11,7 @@ namespace ExtentLedgerResult exclude(LedgerFixture &test, uint64_t address, uint64_t length) { return await_ledger([&](auto done) { - return test.ledger->exclude_allocations(address, length, test.credits.credits.bytes, std::move(done)); + return test.ledger->exclude_allocations(address, length, test.credits.credits.reservation(), std::move(done)); }); } @@ -25,8 +25,9 @@ TEST(ExtentExclusion, ExistingReadersRemainProtectedAndNewWritesAvoidTheWholeInt ASSERT_TRUE(frozen.status.ok()); EXPECT_EQ(frozen.occupied_bytes, 4096U); EXPECT_EQ(test.ledger->stats().protected_bytes, 4096U); - auto retained = await_ledger( - [&](auto done) { return test.ledger->retain(16384, 4096, test.credits.credits.bytes, std::move(done)); }); + auto retained = await_ledger([&](auto done) { + return test.ledger->retain(16384, 4096, test.credits.credits.reservation(), std::move(done)); + }); ASSERT_TRUE(retained.status.ok()); EXPECT_EQ(retained.extent.owner, data.extent.owner); EXPECT_EQ(exclude(test, 32768, 8192).status.code(), Code::kUnavailable); @@ -52,8 +53,9 @@ TEST(ExtentExclusion, FreeDeletionLeaseIsAllowedWhileFutureAllocationStaysFenced ASSERT_TRUE(frozen.status.ok()); EXPECT_EQ(frozen.occupied_bytes, 0U); EXPECT_EQ(test.reserve(4096).status.code(), Code::kResourceExhausted); - auto deletion = await_ledger( - [&](auto done) { return test.ledger->claim_free(8192, 24576, test.credits.credits.bytes, std::move(done)); }); + auto deletion = await_ledger([&](auto done) { + return test.ledger->claim_free(8192, 24576, test.credits.credits.reservation(), std::move(done)); + }); ASSERT_TRUE(deletion.status.ok()); deletion = {}; EXPECT_EQ(test.reserve(4096).status.code(), Code::kResourceExhausted); diff --git a/lib/crowdb-tree/tests/unit/extent_gc_lease_test.cpp b/lib/crowdb-tree/tests/unit/extent_gc_lease_test.cpp index b0597f5b3..dd8cdc58d 100644 --- a/lib/crowdb-tree/tests/unit/extent_gc_lease_test.cpp +++ b/lib/crowdb-tree/tests/unit/extent_gc_lease_test.cpp @@ -11,7 +11,7 @@ namespace ExtentLedgerResult claim(LedgerFixture &test, uint64_t address, uint64_t length) { return await_ledger([&](auto done) { - return test.ledger->claim_free(address, length, test.credits.credits.bytes, std::move(done)); + return test.ledger->claim_free(address, length, test.credits.credits.reservation(), std::move(done)); }); } @@ -32,8 +32,9 @@ TEST(ExtentGcLease, RequiresActivatedFreeRangeAndNeverBorrowsDataOwners) EXPECT_EQ(leased.extent.address, 8192U); EXPECT_NE(leased.extent.owner, live.extent.owner); EXPECT_EQ(claim(test, 8192, 8192).status.code(), Code::kNotFound); - auto retained = await_ledger( - [&](auto done) { return test.ledger->retain(8192, 4096, test.credits.credits.bytes, std::move(done)); }); + auto retained = await_ledger([&](auto done) { + return test.ledger->retain(8192, 4096, test.credits.credits.reservation(), std::move(done)); + }); EXPECT_EQ(retained.status.code(), Code::kNotFound); auto allocated = test.reserve(8192); ASSERT_TRUE(allocated.status.ok()); diff --git a/lib/crowdb-tree/tests/unit/extent_ledger_test.cpp b/lib/crowdb-tree/tests/unit/extent_ledger_test.cpp index 84a366ec1..6e4e5b92b 100644 --- a/lib/crowdb-tree/tests/unit/extent_ledger_test.cpp +++ b/lib/crowdb-tree/tests/unit/extent_ledger_test.cpp @@ -83,8 +83,9 @@ TEST(ExtentLedger, RecoveryRejectsPartialOverlapAndCapacityNeverWrapsOrInventsSp EXPECT_EQ(test.reserve(4096).status.code(), Code::kResourceExhausted); EXPECT_EQ(test.reserve(UINT64_MAX).status.code(), Code::kInvalidArgument); EXPECT_EQ(test.recover(12288, 4096).status.code(), Code::kUnavailable); - auto retained = await_ledger( - [&](auto done) { return test.ledger->retain(8192, 4096, test.credits.credits.bytes, std::move(done)); }); + auto retained = await_ledger([&](auto done) { + return test.ledger->retain(8192, 4096, test.credits.credits.reservation(), std::move(done)); + }); EXPECT_TRUE(retained.status.ok()); EXPECT_EQ(retained.extent.owner, recovered.extent.owner); } @@ -96,7 +97,7 @@ TEST(ExtentLedger, MetadataBoundPaysForReserveAndAllocationFailuresLeaveIndexesU bool succeeded = false; for (size_t amount = 64; amount <= ExtentLedger::reservation_metadata_bound(); amount += 64) { LoadCredits credits; - auto carved = credits.credits.bytes->take_reserved(amount); + auto carved = credits.credits.reservation()->take_reserved(amount); if (!carved.has_value()) { FAIL() << "test credit cannot fund reservation"; } @@ -205,7 +206,7 @@ TEST(ExtentLedger, MiddleRecoveryFitsItsBoundAndEveryFailedAllocationRollsBack) const auto bound = ExtentLedger::reservation_metadata_bound(); for (size_t amount = 64; amount <= bound; amount = std::min(amount + 64, bound)) { LoadCredits credits; - auto carved = credits.credits.bytes->take_reserved(amount); + auto carved = credits.credits.reservation()->take_reserved(amount); if (!carved.has_value()) { FAIL() << "test credit cannot fund recovery"; } @@ -241,11 +242,11 @@ TEST(ExtentLedger, QueueReentryAndThrowingReceiverDoNotStrandAllocationOrRelease ProtectedExtent retained; ASSERT_TRUE( test.ledger - ->reserve(4096, test.credits.credits.bytes, + ->reserve(4096, test.credits.credits.reservation(), [&](ExtentLedgerResult first) { EXPECT_TRUE(first.status.ok()); EXPECT_TRUE(test.ledger - ->reserve(4096, test.credits.credits.bytes, + ->reserve(4096, test.credits.credits.reservation(), [&](ExtentLedgerResult next) { EXPECT_TRUE(next.status.ok()); retained = std::move(next.extent); @@ -253,7 +254,8 @@ TEST(ExtentLedger, QueueReentryAndThrowingReceiverDoNotStrandAllocationOrRelease }) .ok()); EXPECT_EQ( - test.ledger->reserve(4096, test.credits.credits.bytes, [](ExtentLedgerResult) {}).code(), + test.ledger->reserve(4096, test.credits.credits.reservation(), [](ExtentLedgerResult) {}) + .code(), Code::kResourceExhausted); throw std::runtime_error("controlled receiver failure"); }) diff --git a/lib/crowdb-tree/tests/unit/flush_runtime_progress_test.cpp b/lib/crowdb-tree/tests/unit/flush_runtime_progress_test.cpp index c255c3612..f222ea632 100644 --- a/lib/crowdb-tree/tests/unit/flush_runtime_progress_test.cpp +++ b/lib/crowdb-tree/tests/unit/flush_runtime_progress_test.cpp @@ -84,15 +84,16 @@ TEST(FlushRuntime, ReservedProgressConsumesLaneAndGlobalCapacityAndSurvivesFullQ TEST(FlushRuntime, ReservedProgressRerunsUseSealedStorageAndDrainReleasesLastContextOwner) { test_read::LoadCredits credits; - FlushRuntime runtime(1, 2, credits.credits.bytes); + FlushRuntime runtime(1, 2, credits.credits.reservation()); auto lane = std::make_shared(2); const auto baseline = credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes]; std::atomic calls{0}; - auto owner = reserved_shared(credits.credits.bytes, calls); + auto owner = reserved_shared(credits.credits.reservation(), calls); owner->remaining = 1000; - ASSERT_TRUE( - runtime.reserve_progress(lane, &TestProgressOwner::run, owner.get(), credits.credits.bytes, &owner->progress) - .ok()); + ASSERT_TRUE(runtime + .reserve_progress(lane, &TestProgressOwner::run, owner.get(), credits.credits.reservation(), + &owner->progress) + .ok()); std::weak_ptr retained = owner; credits.seal(); owner->progress->request(owner); @@ -114,7 +115,7 @@ TEST(FlushRuntime, FailedProgressRegistrationReturnsSlotAndQueueCredits) std::atomic calls{0}; auto increment = [](void *context) noexcept { static_cast *>(context)->fetch_add(1); }; credits.seal(); - EXPECT_EQ(runtime.reserve_progress(lane, increment, &calls, credits.credits.bytes, &progress).code(), + EXPECT_EQ(runtime.reserve_progress(lane, increment, &calls, credits.credits.reservation(), &progress).code(), Code::kResourceExhausted); EXPECT_FALSE(progress); ASSERT_TRUE(runtime.try_submit(lane, increment, &calls).ok()); diff --git a/lib/crowdb-tree/tests/unit/flush_runtime_scheduler_test.cpp b/lib/crowdb-tree/tests/unit/flush_runtime_scheduler_test.cpp index 333922f48..749c5cf38 100644 --- a/lib/crowdb-tree/tests/unit/flush_runtime_scheduler_test.cpp +++ b/lib/crowdb-tree/tests/unit/flush_runtime_scheduler_test.cpp @@ -18,7 +18,7 @@ void count_task(void *context) noexcept TEST(FlushRuntimeScheduler, ShortLivedSubmittersReuseAdmittedControlStorage) { test_read::LoadCredits credits; - auto runtime = std::make_unique(4, 16, credits.credits.bytes); + auto runtime = std::make_unique(4, 16, credits.credits.reservation()); auto lane = std::make_shared(16); const auto allocated = credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes]; std::atomic completed{0}; @@ -46,12 +46,13 @@ TEST(FlushRuntimeScheduler, ShortLivedSubmittersReuseAdmittedControlStorage) TEST(FlushRuntimeScheduler, ReservedProgressCanMoveAcrossFreshHostThreads) { - test_read::LoadCredits credits; - auto runtime = std::make_unique(2, 1, credits.credits.bytes); - auto lane = std::make_shared(1); - auto completed = std::make_shared>(0); + test_read::LoadCredits credits; + auto runtime = std::make_unique(2, 1, credits.credits.reservation()); + auto lane = std::make_shared(1); + auto completed = std::make_shared>(0); std::shared_ptr progress; - ASSERT_TRUE(runtime->reserve_progress(lane, &count_task, completed.get(), credits.credits.bytes, &progress).ok()); + ASSERT_TRUE( + runtime->reserve_progress(lane, &count_task, completed.get(), credits.credits.reservation(), &progress).ok()); const auto allocated = credits.budget.snapshot().allocated[test_read::LoadCredits::kBytes]; for (size_t iteration = 0; iteration < 32; ++iteration) { std::jthread submitter([&] { progress->request(completed); }); diff --git a/lib/crowdb-tree/tests/unit/native_import_test.cpp b/lib/crowdb-tree/tests/unit/native_import_test.cpp index 263999cb6..3a6db4250 100644 --- a/lib/crowdb-tree/tests/unit/native_import_test.cpp +++ b/lib/crowdb-tree/tests/unit/native_import_test.cpp @@ -16,7 +16,7 @@ class NativeImport : public testing::Test protected: Status catalog(const std::vector &frames, std::shared_ptr *out) const { - return build_native_catalog(frames, test_import::info(frames), {}, {}, {}, credits.credits.bytes, out); + return build_native_catalog(frames, test_import::info(frames), {}, {}, {}, credits.credits.reservation(), out); } test_read::LoadCredits credits; @@ -83,19 +83,21 @@ TEST_F(NativeImport, ReadableGraphLoadsResidentIndexButDurableGraphRejectsIt) auto status = catalog(frames, &pages); ASSERT_TRUE(status.ok()) << status.to_string(); std::shared_ptr operation; - ASSERT_TRUE( - RecoveredStateOperation::create_readable(pages, loader, runtime, lane, {}, credits.credits.bytes, &operation) - .ok()); + ASSERT_TRUE(RecoveredStateOperation::create_readable(pages, loader, runtime, lane, {}, + credits.credits.reservation(), &operation) + .ok()); status = detail::wait_snapshot_operation(operation); ASSERT_TRUE(status.ok()) << status.to_string(); EXPECT_EQ(operation->result().state->stats.height, 2U); EXPECT_EQ(operation->result().state->stats.kv_known_count, 2U); EXPECT_EQ(store->reads.load(), 0U); - auto durable = detail::reserved_shared(credits.credits.bytes, credits.credits.bytes); + auto durable = + detail::reserved_shared(credits.credits.reservation(), credits.credits.reservation()); static_cast(*durable) = *pages; operation.reset(); ASSERT_TRUE( - RecoveredStateOperation::create(durable, loader, runtime, lane, {}, credits.credits.bytes, &operation).ok()); + RecoveredStateOperation::create(durable, loader, runtime, lane, {}, credits.credits.reservation(), &operation) + .ok()); EXPECT_EQ(detail::wait_snapshot_operation(operation).code(), Code::kCorruption); } @@ -136,9 +138,9 @@ TEST_F(NativeImport, OverwrittenOverflowCellStillRetainsItsPhysicalDependency) ASSERT_TRUE(status.ok()) << status.to_string(); ASSERT_EQ(pages->frames.at(0).stored_overflow->size(), 1U); std::shared_ptr operation; - ASSERT_TRUE( - RecoveredStateOperation::create_readable(pages, loader, runtime, lane, {}, credits.credits.bytes, &operation) - .ok()); + ASSERT_TRUE(RecoveredStateOperation::create_readable(pages, loader, runtime, lane, {}, + credits.credits.reservation(), &operation) + .ok()); status = detail::wait_snapshot_operation(operation); ASSERT_TRUE(status.ok()) << status.to_string(); const auto leaf = operation->result().state->mapping.get(0)->leaf; diff --git a/lib/crowdb-tree/tests/unit/page_cache_residency_budget_test.cpp b/lib/crowdb-tree/tests/unit/page_cache_residency_budget_test.cpp index 9baf2ce4b..09a13434f 100644 --- a/lib/crowdb-tree/tests/unit/page_cache_residency_budget_test.cpp +++ b/lib/crowdb-tree/tests/unit/page_cache_residency_budget_test.cpp @@ -30,8 +30,9 @@ TEST(PageCacheResidency, ActualDecodedAllocationSurvivesEvictionUntilLastReaderP OverflowBackingResolver{}, credits.credits); auto metrics = std::make_shared(); { - SharedPageCache cache({.max_pages = 1, .max_waiters = 2}, credits.credits.bytes, metrics, runtime.get(), lane); - auto loader = cache.bind(upstream, credits.credits.bytes); + SharedPageCache cache({.max_pages = 1, .max_waiters = 2}, credits.credits.reservation(), metrics, runtime.get(), + lane); + auto loader = cache.bind(upstream, credits.credits.reservation()); const auto infrastructure = credits.budget.snapshot().allocated[LoadCredits::kBytes]; std::shared_ptr pin; loader->load_leaf({.identity = 1, .backing = backing}, [&](Status status, auto leaf) { @@ -66,7 +67,7 @@ TEST(PageCacheResidency, PinAdmissionFailureReturnsErrorWithoutDiscardingReadyFr const auto input = cold(1); fixture.load(input).reset(); LoadCredits credits; - auto loader = fixture.cache->bind(fixture.upstream, credits.credits.bytes); + auto loader = fixture.cache->bind(fixture.upstream, credits.credits.reservation()); credits.seal(); size_t done = 0; loader->load_leaf(input, [&](Status status, auto frame) { @@ -85,7 +86,7 @@ TEST(PageCacheResidency, MissAdmissionFailureOccursBeforeIoAndCanRetry) TestCache fixture; fixture.frame(1); LoadCredits credits; - auto loader = fixture.cache->bind(fixture.upstream, credits.credits.bytes); + auto loader = fixture.cache->bind(fixture.upstream, credits.credits.reservation()); credits.seal(); size_t failed = 0; const auto input = cold(1); @@ -147,19 +148,19 @@ TEST(PageCacheResidency, QueuedCommandsRetainTheirOwnersAfterCacheAndPublisherDe gate.entered.wait(); { auto cache = std::make_shared(PageCacheLimits{.max_pages = 1, .max_waiters = 1}, - credits.credits.bytes, resources, &runtime, lane); + credits.credits.reservation(), resources, &runtime, lane); PublicationCoordinator publisher(4, 1, {}, std::pmr::get_default_resource(), cache); auto request = std::make_shared(); request->transaction = {.record = std::make_shared(InstallToken{}), .pages = {{.page_id = 0, .expected = {}, .replacement = published({cold(1)})}}, .root_page_id = 0, - .metadata = credits.credits.bytes}; + .metadata = credits.credits.reservation()}; EXPECT_TRUE(publisher.try_submit(request)); auto upstream = std::make_shared(); upstream->frames[1] = base({ {.key = "a", .value = "value"} }); - auto loader = cache->bind(upstream, credits.credits.bytes); + auto loader = cache->bind(upstream, credits.credits.reservation()); loader->load_leaf(cold(1), [](Status status, auto frame) { EXPECT_TRUE(status.ok()); EXPECT_TRUE(frame); diff --git a/lib/crowdb-tree/tests/unit/page_cache_residency_test.cpp b/lib/crowdb-tree/tests/unit/page_cache_residency_test.cpp index 9e06af34e..aebd72a3b 100644 --- a/lib/crowdb-tree/tests/unit/page_cache_residency_test.cpp +++ b/lib/crowdb-tree/tests/unit/page_cache_residency_test.cpp @@ -268,13 +268,13 @@ TEST(PageCacheResidency, PreparedNodesRollbackAndReleaseAfterBudgetSealing) PageCacheResidency residency(resources, {}, &runtime, lane); auto page = published({cold(1), cold(2)}); const std::array edits{{{.page_id = 0, .expected = {}, .replacement = page}}}; - auto update = residency.prepare(edits, credits.credits.bytes); + auto update = residency.prepare(edits, credits.credits.reservation()); const auto allocated = credits.budget.snapshot().allocated[LoadCredits::kBytes]; EXPECT_GT(allocated, 0U); EXPECT_EQ(resources->snapshot().proxy_pages, 0U); update.reset(); EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); - update = residency.prepare(edits, credits.credits.bytes); + update = residency.prepare(edits, credits.credits.reservation()); credits.seal(); PageCacheResidency::publish(std::move(update)); runtime.drain(); diff --git a/lib/crowdb-tree/tests/unit/page_cache_test.cpp b/lib/crowdb-tree/tests/unit/page_cache_test.cpp index 7381905e5..17589caa6 100644 --- a/lib/crowdb-tree/tests/unit/page_cache_test.cpp +++ b/lib/crowdb-tree/tests/unit/page_cache_test.cpp @@ -225,15 +225,15 @@ TEST(SharedPageCache, FixedCapacityRejectsBeforeIoAndCallbacksCanReenter) TEST(SharedPageCache, ReservedWaiterSurvivesOrdinaryPressureAndWaitsForInFlightPageCapacity) { LoadCredits credits; - SharedPageCache cache({.max_pages = 1, .max_waiters = 2}, credits.credits.bytes); + SharedPageCache cache({.max_pages = 1, .max_waiters = 2}, credits.credits.reservation()); auto upstream = std::make_shared(); upstream->deferred = true; upstream->frames[1] = upstream->frames[2] = base({ {.key = "a", .value = "value"} }); - auto ordinary = cache.bind(upstream, credits.credits.bytes); + auto ordinary = cache.bind(upstream, credits.credits.reservation()); std::shared_ptr reserved; - ASSERT_TRUE(cache.bind_reserved(upstream, credits.credits.bytes, &reserved).ok()); + ASSERT_TRUE(cache.bind_reserved(upstream, credits.credits.reservation(), &reserved).ok()); EXPECT_EQ(cache.snapshot().reserved_waiters, 1U); credits.seal(); size_t ordinary_done = 0; @@ -282,8 +282,8 @@ TEST(SharedPageCache, EvictionReturnsCacheReferenceButPinnedFrameRemainsCharged) auto upstream = std::make_shared(store, runtime, lane, PageLoadLimits{}, OverflowBackingResolver{}, credits.credits); { - SharedPageCache cache({.max_pages = 2, .max_waiters = 8}, credits.credits.bytes); - auto loader = cache.bind(upstream, credits.credits.bytes); + SharedPageCache cache({.max_pages = 2, .max_waiters = 8}, credits.credits.reservation()); + auto loader = cache.bind(upstream, credits.credits.reservation()); const auto infrastructure = credits.budget.snapshot().allocated[LoadCredits::kBytes]; std::shared_ptr pinned; loader->load_leaf({.identity = 1, .backing = backing}, [&](Status status, auto leaf) { @@ -322,7 +322,7 @@ TEST(SharedPageCache, FailedBindingAllocationReturnsReservedSlot) }); credits.seal(); std::shared_ptr binding; - EXPECT_EQ(cache.bind_reserved(upstream, credits.credits.bytes, &binding).code(), Code::kResourceExhausted); + EXPECT_EQ(cache.bind_reserved(upstream, credits.credits.reservation(), &binding).code(), Code::kResourceExhausted); EXPECT_FALSE(binding); EXPECT_EQ(cache.snapshot().reserved_waiters, 0U); ASSERT_TRUE(cache.bind_reserved(upstream, {}, &binding).ok()); diff --git a/lib/crowdb-tree/tests/unit/page_load_delta_test.cpp b/lib/crowdb-tree/tests/unit/page_load_delta_test.cpp index 6f5d96e6a..156cb852b 100644 --- a/lib/crowdb-tree/tests/unit/page_load_delta_test.cpp +++ b/lib/crowdb-tree/tests/unit/page_load_delta_test.cpp @@ -159,15 +159,15 @@ TEST(PageLoadDelta, IndexAllocationExhaustionReleasesDecodeAndIoCredit) ++calls; }); const auto delta_bytes = LeafFrameView(fixture.frame.data(), fixture.frame.size()).delta_count() * sizeof(uint32_t); - const auto held = credits.credits.bytes->available() - fixture.frame.size() - delta_bytes + 1; - ASSERT_TRUE(credits.credits.bytes->claim(held)); + const auto held = credits.credits.reservation()->available() - fixture.frame.size() - delta_bytes + 1; + ASSERT_TRUE(credits.credits.reservation()->claim(held)); fixture.store->pending.complete(Status::Ok()); fixture.store->pending = {}; fixture.runtime->drain(); EXPECT_EQ(calls, 1U); EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kIo], 0U); loader.reset(); - credits.credits.bytes->release(held); + credits.credits.reservation()->release(held); credits.seal(); EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); } diff --git a/lib/crowdb-tree/tests/unit/page_load_inner_proof_test.cpp b/lib/crowdb-tree/tests/unit/page_load_inner_proof_test.cpp index b886bbc96..8e66e189e 100644 --- a/lib/crowdb-tree/tests/unit/page_load_inner_proof_test.cpp +++ b/lib/crowdb-tree/tests/unit/page_load_inner_proof_test.cpp @@ -34,8 +34,8 @@ TEST(InnerLoadProof, OwnsByteFencesAndDropsProofWhenRoutingIsCopied) EXPECT_EQ(proof.frame_bytes, 4096U); EXPECT_EQ(proof.lower, "a"); EXPECT_EQ(proof.upper, "z"); - auto copied = - detail::reserved_shared(test.credits.credits.bytes, *result, test.credits.credits.bytes); + auto copied = detail::reserved_shared(test.credits.credits.reservation(), *result, + test.credits.credits.reservation()); EXPECT_FALSE(copied->frame_proof); copied.reset(); result.reset(); diff --git a/lib/crowdb-tree/tests/unit/page_load_lifetime_test.cpp b/lib/crowdb-tree/tests/unit/page_load_lifetime_test.cpp index 47aca267f..254ac4dae 100644 --- a/lib/crowdb-tree/tests/unit/page_load_lifetime_test.cpp +++ b/lib/crowdb-tree/tests/unit/page_load_lifetime_test.cpp @@ -99,7 +99,8 @@ struct Reads credits.credits.work_metrics = metrics; if (prepaid) { EXPECT_TRUE( - ApplyDispatcher::create(runtime, lane, 1, credits.credits.bytes, &credits.credits.cpu_progress).ok()); + ApplyDispatcher::create(runtime, lane, 1, credits.credits.reservation(), &credits.credits.cpu_progress) + .ok()); } if (kind == Kind::kLeaf) { auto leaf = base({ @@ -221,9 +222,9 @@ TEST(PageLoadLifetime, TreeShutdownWaitsForUnknownTransportOwnershipAfterLogical { for (const bool callback : {true, false}) { Reads test(Kind::kLeaf); - TreeOperationLifetime lifetime(1, test.credits.credits.bytes); + TreeOperationLifetime lifetime(1, test.credits.credits.reservation()); std::shared_ptr ticket; - ASSERT_TRUE(lifetime.acquire(test.credits.credits.bytes, &ticket).ok()); + ASSERT_TRUE(lifetime.acquire(test.credits.credits.reservation(), &ticket).ok()); test.loader = std::make_shared(ticket->backend(test.store.get()), test.runtime, test.lane, PageLoadLimits{.max_inflight = 1}, OverflowBackingResolver{}, test.credits.credits); diff --git a/lib/crowdb-tree/tests/unit/page_load_test.cpp b/lib/crowdb-tree/tests/unit/page_load_test.cpp index 382372835..f24d153a9 100644 --- a/lib/crowdb-tree/tests/unit/page_load_test.cpp +++ b/lib/crowdb-tree/tests/unit/page_load_test.cpp @@ -202,9 +202,8 @@ TEST(PageLoad, RejectsMixedAccountsAndByteIoUnitsBeforeAllocatingLoaderState) auto lane = std::make_shared(8); auto store = std::make_shared(); for (const auto &invalid : { - PageLoadBudget{.bytes = first.credits.bytes, .io = second.credits.io}, - PageLoadBudget{.bytes = first.credits.io, .io = first.credits.io }, - PageLoadBudget{.bytes = first.credits.bytes, .io = {} } + PageLoadBudget{.bytes = first.credits.reservation(), .io = second.credits.io}, + PageLoadBudget{.bytes = first.credits.io, .io = first.credits.io } }) { EXPECT_THROW((void)AsyncStorePageLoader(store, runtime, lane, {}, {}, invalid), std::invalid_argument); } diff --git a/lib/crowdb-tree/tests/unit/portable_import_test.cpp b/lib/crowdb-tree/tests/unit/portable_import_test.cpp index 81d8a0035..5b3926cf9 100644 --- a/lib/crowdb-tree/tests/unit/portable_import_test.cpp +++ b/lib/crowdb-tree/tests/unit/portable_import_test.cpp @@ -15,7 +15,7 @@ TEST(PortableImport, SortedCellsBuildABoundedBalancedReadableGraphWithOverflow) config.frame_bytes = 512; config.max_inline_value = 16; config.inner_max_keys = 2; - PortableImportBuilder builder(config, {}, 300, test.credits.credits.bytes); + PortableImportBuilder builder(config, {}, 300, test.credits.credits.reservation()); for (size_t i = 0; i != 200; ++i) { const auto key = std::string(3 - std::to_string(i).size(), '0') + std::to_string(i); const auto cell = encode_cell_buf(i + 1, OpKind::kPut, std::string(1000, static_cast('a' + (i % 26)))); @@ -28,7 +28,7 @@ TEST(PortableImport, SortedCellsBuildABoundedBalancedReadableGraphWithOverflow) ASSERT_NE(catalog, nullptr); EXPECT_GT(catalog->frames.size(), 200U); ASSERT_TRUE(RecoveredStateOperation::create_readable(catalog, test.loader, test.runtime, test.lane, {}, - test.credits.credits.bytes, &test.operation) + test.credits.credits.reservation(), &test.operation) .ok()); status = detail::wait_snapshot_operation(test.operation); ASSERT_TRUE(status.ok()) << status.to_string(); @@ -42,7 +42,7 @@ TEST(PortableImport, FailurePoisonsStagingAndDoesNotReturnAPartialCatalog) { test_read::LoadCredits credits; Config config; - PortableImportBuilder builder(config, {}, 7, credits.credits.bytes); + PortableImportBuilder builder(config, {}, 7, credits.credits.reservation()); auto cell = encode_cell_buf(7, OpKind::kPut, "value"); ASSERT_TRUE(builder.append("z", Slice(cell)).ok()); EXPECT_EQ(builder.append("a", Slice(cell)).code(), Code::kInvalidArgument); @@ -55,7 +55,7 @@ TEST(PortableImport, FailurePoisonsStagingAndDoesNotReturnAPartialCatalog) TEST(PortableImport, EmptyKeyAndTombstonePreserveSlotsAndEmptyImportPreservesCoverage) { test_read::LoadCredits credits; - PortableImportBuilder builder({}, {}, 17, credits.credits.bytes); + PortableImportBuilder builder({}, {}, 17, credits.credits.reservation()); const auto empty = encode_cell_buf(5, OpKind::kPut, "empty"); const auto deleted = encode_cell_buf(9, OpKind::kDelete); ASSERT_TRUE(builder.append("", Slice(empty)).ok()); @@ -69,7 +69,7 @@ TEST(PortableImport, EmptyKeyAndTombstonePreserveSlotsAndEmptyImportPreservesCov EXPECT_EQ(CellView(leaf.cell(0)).slot(), 5U); EXPECT_TRUE(CellView(leaf.cell(1)).is_tombstone()); EXPECT_EQ(CellView(leaf.cell(1)).slot(), 9U); - PortableImportBuilder empty_builder({}, {}, 23, credits.credits.bytes); + PortableImportBuilder empty_builder({}, {}, 23, credits.credits.reservation()); catalog.reset(); ASSERT_TRUE(empty_builder.finish(&catalog).ok()); EXPECT_TRUE(catalog->frames.empty()); diff --git a/lib/crowdb-tree/tests/unit/publication_bootstrap_test.cpp b/lib/crowdb-tree/tests/unit/publication_bootstrap_test.cpp index ded1f475f..1c2555276 100644 --- a/lib/crowdb-tree/tests/unit/publication_bootstrap_test.cpp +++ b/lib/crowdb-tree/tests/unit/publication_bootstrap_test.cpp @@ -20,16 +20,17 @@ TEST(PublicationBootstrap, LifecycleChargesSourceControllerEpochAndInitialPublic auto lane = std::make_shared(8); std::shared_ptr retained; { - auto sources = detail::reserved_unique( - credits.credits.bytes, 0, 1, SourceWriteOptions{.account = credits.account}, credits.credits.bytes); + auto sources = detail::reserved_unique(credits.credits.reservation(), 0, 1, + SourceWriteOptions{.account = credits.account}, + credits.credits.reservation()); const auto source_bytes = credits.budget.snapshot().allocated[LoadCredits::kBytes]; EXPECT_GT(source_bytes, sizeof(SourceAdmission)); EXPECT_LE(source_bytes, SourceAdmission::allocation_bound().value_or(0)); auto publisher = detail::reserved_unique( - credits.credits.bytes, 8, 1, std::shared_ptr{}, std::pmr::get_default_resource(), + credits.credits.reservation(), 8, 1, std::shared_ptr{}, std::pmr::get_default_resource(), std::shared_ptr{}, PublicationBootstrap{ - .metadata = credits.credits.bytes, .initial = {}, .source_catalog = sources->catalog()}); + .metadata = credits.credits.reservation(), .initial = {}, .source_catalog = sources->catalog()}); retained = publisher->capture(); EXPECT_TRUE(std::ranges::equal(retained->sources->tables, sources->catalog()->tables)); const auto publication_bytes = credits.budget.snapshot().allocated[LoadCredits::kBytes] - source_bytes; @@ -38,8 +39,8 @@ TEST(PublicationBootstrap, LifecycleChargesSourceControllerEpochAndInitialPublic std::atomic frontier{0}; { auto controller = detail::reserved_unique( - credits.credits.bytes, *publisher, runtime, *sources, frontier, lane, FlushBatchOptions{}, - FlushBatch::Maintenance{}, credits.credits.bytes); + credits.credits.reservation(), *publisher, runtime, *sources, frontier, lane, FlushBatchOptions{}, + FlushBatch::Maintenance{}, credits.credits.reservation()); const auto controller_bytes = credits.budget.snapshot().allocated[LoadCredits::kBytes] - source_bytes - publication_bytes; EXPECT_GT(controller_bytes, sizeof(FlushController)); @@ -59,8 +60,9 @@ TEST(PublicationBootstrap, SparseRecoveredMappingInstallsExactSourcesAndAllColdR detail::FlushRuntime runtime(1, 8); auto lane = std::make_shared(8); auto resources = std::make_shared(); - auto cache = std::make_shared(PageCacheLimits{}, credits.credits.bytes, resources, &runtime, lane); - SourceAdmission sources(42, 9, {}, credits.credits.bytes); + auto cache = + std::make_shared(PageCacheLimits{}, credits.credits.reservation(), resources, &runtime, lane); + SourceAdmission sources(42, 9, {}, credits.credits.reservation()); auto initial = std::make_shared(nullptr, 42); initial->generation = 9; initial->coverage = 42; @@ -83,7 +85,7 @@ TEST(PublicationBootstrap, SparseRecoveredMappingInstallsExactSourcesAndAllColdR { PublicationCoordinator publisher( 8, 9, {}, std::pmr::get_default_resource(), cache, - {.metadata = credits.credits.bytes, .initial = initial, .source_catalog = sources.catalog()}); + {.metadata = credits.credits.reservation(), .initial = initial, .source_catalog = sources.catalog()}); runtime.drain(); auto view = publisher.capture(); EXPECT_EQ(view->coverage, 42U); @@ -99,7 +101,7 @@ TEST(PublicationBootstrap, SparseRecoveredMappingInstallsExactSourcesAndAllColdR upstream->frames[1] = base({ {.key = "a", .value = "value"} }); - auto loader = cache->bind(upstream, credits.credits.bytes); + auto loader = cache->bind(upstream, credits.credits.reservation()); loader->load_leaf(input, [](Status status, auto frame) { EXPECT_TRUE(status.ok()); EXPECT_TRUE(frame); @@ -124,16 +126,17 @@ TEST(PublicationBootstrap, FailedInitialPublicationDoesNotClaimCacheOrLeaveAlloc auto cache = std::make_shared(PageCacheLimits{}, nullptr, resources, &runtime, lane); SourceAdmission wrong(7, 1); const auto initial = std::make_shared(); - EXPECT_THROW((PublicationCoordinator( - 8, 1, {}, std::pmr::get_default_resource(), cache, - {.metadata = credits.credits.bytes, .initial = initial, .source_catalog = wrong.catalog()})), - std::invalid_argument); + EXPECT_THROW( + (PublicationCoordinator( + 8, 1, {}, std::pmr::get_default_resource(), cache, + {.metadata = credits.credits.reservation(), .initial = initial, .source_catalog = wrong.catalog()})), + std::invalid_argument); EXPECT_EQ(credits.budget.snapshot().allocated[LoadCredits::kBytes], 0U); SourceAdmission matching; { PublicationCoordinator publisher( 8, 1, {}, std::pmr::get_default_resource(), cache, - {.metadata = credits.credits.bytes, .initial = initial, .source_catalog = matching.catalog()}); + {.metadata = credits.credits.reservation(), .initial = initial, .source_catalog = matching.catalog()}); EXPECT_TRUE(std::ranges::equal(publisher.capture()->sources->tables, matching.catalog()->tables)); } runtime.drain(); @@ -143,7 +146,7 @@ TEST(PublicationBootstrap, FailedInitialPublicationDoesNotClaimCacheOrLeaveAlloc TEST(PublicationBootstrap, ExhaustedLifecycleRejectsNewThreadBeforeWriterAdmission) { LoadCredits credits; - SourceAdmission sources(0, 1, {}, credits.credits.bytes); + SourceAdmission sources(0, 1, {}, credits.credits.reservation()); const auto catalog = sources.catalog(); credits.seal(); Status status; @@ -160,7 +163,7 @@ TEST(PublicationBootstrap, ExhaustedLifecycleRejectsNewThreadBeforeWriterAdmissi TEST(PublicationBootstrap, RejectedControlAllocationsReturnAllPartialCharges) { LoadCredits credits; - auto token = credits.credits.bytes->take_reserved(16); + auto token = credits.credits.reservation()->take_reserved(16); if (!token) { FAIL() << "tiny control reservation unavailable"; } diff --git a/lib/crowdb-tree/tests/unit/publication_lane_test.cpp b/lib/crowdb-tree/tests/unit/publication_lane_test.cpp index 9c7fa184b..19a0551c0 100644 --- a/lib/crowdb-tree/tests/unit/publication_lane_test.cpp +++ b/lib/crowdb-tree/tests/unit/publication_lane_test.cpp @@ -24,7 +24,7 @@ TEST(PublicationLane, PrepaidWindowProgressesWhenOrdinaryQueueIsFullAndReturnsBe test_read::LoadCredits credits; PublicationCoordinator publisher(2); std::shared_ptr lane; - ASSERT_TRUE(publisher.prepare_lane(2, credits.credits.bytes, &lane).ok()); + ASSERT_TRUE(publisher.prepare_lane(2, credits.credits.reservation(), &lane).ok()); std::latch entered(1); std::latch release(1); auto blocker = request(1); @@ -71,9 +71,9 @@ TEST(PublicationLane, WindowIsFixedUniqueAndBoundToItsPublisher) PublicationCoordinator other(2); std::shared_ptr lane; std::shared_ptr duplicate; - EXPECT_EQ(publisher.prepare_lane(3, credits.credits.bytes, &lane).code(), Code::kInvalidArgument); - ASSERT_TRUE(publisher.prepare_lane(2, credits.credits.bytes, &lane).ok()); - EXPECT_EQ(publisher.prepare_lane(1, credits.credits.bytes, &duplicate).code(), Code::kUnavailable); + EXPECT_EQ(publisher.prepare_lane(3, credits.credits.reservation(), &lane).code(), Code::kInvalidArgument); + ASSERT_TRUE(publisher.prepare_lane(2, credits.credits.reservation(), &lane).ok()); + EXPECT_EQ(publisher.prepare_lane(1, credits.credits.reservation(), &duplicate).code(), Code::kUnavailable); EXPECT_FALSE(other.try_submit(request(1), lane)); publisher.close(); EXPECT_FALSE(publisher.try_submit(request(2), lane)); diff --git a/lib/crowdb-tree/tests/unit/published_read_budget_test.cpp b/lib/crowdb-tree/tests/unit/published_read_budget_test.cpp index b58280894..3916184dd 100644 --- a/lib/crowdb-tree/tests/unit/published_read_budget_test.cpp +++ b/lib/crowdb-tree/tests/unit/published_read_budget_test.cpp @@ -14,7 +14,7 @@ using namespace test_read; TEST(PublishedReadBudget, FreshWorkerReadsSealedSourceWithItsOwnCreditAndRetainsResultKey) { LoadCredits credits; - auto raw = credits.credits.bytes->take_reserved(16384); + auto raw = credits.credits.reservation()->take_reserved(16384); if (!raw) { FAIL() << "initial source credit unavailable"; } @@ -29,7 +29,7 @@ TEST(PublishedReadBudget, FreshWorkerReadsSealedSourceWithItsOwnCreditAndRetains std::optional retained; std::jthread worker([&] { PublishedReadLimits limits; - limits.metadata = credits.credits.bytes; + limits.metadata = credits.credits.reservation(); std::shared_ptr get; ASSERT_TRUE(PublishedGet::create(captured, {}, key, limits, &get).ok()); complete_inline(get->sender()); @@ -39,7 +39,7 @@ TEST(PublishedReadBudget, FreshWorkerReadsSealedSourceWithItsOwnCreditAndRetains ASSERT_TRUE(PublishedScan::create(captured, {}, {}, false, limits, &scan).ok()); complete_inline(scan->next_sender()); ASSERT_EQ(scan->result().entries.size(), 1U); - EXPECT_EQ(scan->result().entries.get_allocator().owner(), credits.credits.bytes); + EXPECT_EQ(scan->result().entries.get_allocator().owner(), credits.credits.reservation()); retained = scan->result().entries.front(); }); worker.join(); @@ -68,7 +68,7 @@ TEST(PublishedReadBudget, ColdReadCancellationKeepsChargedCallbackUntilLateCompl {.key = "a", .value = "value"} }); std::shared_ptr get; - ASSERT_TRUE(PublishedGet::create(captured, loader, "a", {.metadata = credits.credits.bytes}, &get).ok()); + ASSERT_TRUE(PublishedGet::create(captured, loader, "a", {.metadata = credits.credits.reservation()}, &get).ok()); Completion terminal; { auto connected = stdexec::connect(get->sender(), Receiver{.done = &terminal}); @@ -94,7 +94,7 @@ TEST(PublishedReadBudget, ExhaustedAdmissionReturnsStatusWithoutUnbudgetedHandle })); credits.seal(); PublishedReadLimits limits; - limits.metadata = credits.credits.bytes; + limits.metadata = credits.credits.reservation(); std::shared_ptr get; EXPECT_EQ(PublishedGet::create(captured, {}, "a", limits, &get).code(), Code::kResourceExhausted); EXPECT_FALSE(get); @@ -124,7 +124,7 @@ TEST(PublishedReadBudget, ColdLeafBeyondSourceWindowUsesCapturedDescriptorBound) auto captured = view(logical); for (bool scan_read : {false, true}) { LoadCredits credits; - credits.credits.bytes->seal(); + credits.credits.reservation()->seal(); PublishedReadLimits limits; limits.max_key_bytes = 1; limits.max_memtables = 1; diff --git a/lib/crowdb-tree/tests/unit/runtime_registry_test.cpp b/lib/crowdb-tree/tests/unit/runtime_registry_test.cpp index 1874c1856..a44130c2d 100644 --- a/lib/crowdb-tree/tests/unit/runtime_registry_test.cpp +++ b/lib/crowdb-tree/tests/unit/runtime_registry_test.cpp @@ -1,5 +1,6 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "allocation_account_test_support.h" #include "btree/tree_core_access.h" #include "c_api_internal.h" @@ -9,15 +10,11 @@ namespace crowdb::tree { namespace { -auto reserve_remaining_routing(Crowdbtree &tree) +auto use_remaining_memory(Crowdbtree &tree) { - auto *core = TreeCoreAccess::get(tree); - auto account = core->binding()->account(); - const auto stats = account->snapshot(); - const auto index = static_cast(detail::BudgetClass::kRouting); - const auto available = - core->binding()->options().limits.routing_bytes - stats.allocated[index] - stats.reserved[index]; - return detail::ResourceBudget::try_reserve(account, detail::BudgetClass::kRouting, available); + auto *core = TreeCoreAccess::get(tree); + auto account = core->binding()->account(); + return detail::test_allocation_account::use_remaining_memory(account, core->read_limits().metadata); } } // namespace @@ -36,8 +33,7 @@ TEST(RuntimeRegistry, TreeRegistryAndNamesUseActualRoutingQuotaAndFailedReinitPr EXPECT_GT(after.allocated_bytes + after.reserved_bytes, before.allocated_bytes + before.reserved_bytes); const auto width = tree.max_name_len(); { - auto held = reserve_remaining_routing(tree); - ASSERT_TRUE(held); + auto held = use_remaining_memory(tree); EXPECT_THROW(tree.init_metrics("replacement", "mem"), std::bad_alloc); EXPECT_EQ(tree.max_name_len(), width); } @@ -59,8 +55,7 @@ TEST(RuntimeRegistry, ExhaustedFlushScratchNeverThrowsAcrossEitherCEntryAndCanRe ASSERT_EQ(ct_open_with_runtime(&options, runtime, nullptr, &tree), 0); c_api_tree(tree)->init_metrics("runtime", "mem"); { - auto held = reserve_remaining_routing(*c_api_tree(tree)); - ASSERT_TRUE(held); + auto held = use_remaining_memory(*c_api_tree(tree)); EXPECT_EQ(ct_flush_metrics_str(tree, 1.0, "now", 0), nullptr); EXPECT_EQ(ct_flush_metrics_str_ext(tree, 1.0, "now", 0, 7, 7), nullptr); } diff --git a/lib/crowdb-tree/tests/unit/scan_output_test.cpp b/lib/crowdb-tree/tests/unit/scan_output_test.cpp index 609dbb408..2ecc8262c 100644 --- a/lib/crowdb-tree/tests/unit/scan_output_test.cpp +++ b/lib/crowdb-tree/tests/unit/scan_output_test.cpp @@ -9,9 +9,9 @@ namespace crowdb::tree::detail { TEST(ScanOutput, GrowthClaimsBothBuffersAndFailurePreservesThePreviousAllocation) { - constexpr size_t capacity = 16384; - ResourceBudget budget({capacity, capacity, capacity, capacity, capacity, 1}, 1); - auto account = budget.make_account({capacity, capacity, capacity, capacity, capacity, 1}); + constexpr size_t capacity = 16384; + ResourceBudget budget({0, capacity, 0, 0, 0, 1}, 1); + auto account = budget.make_account({0, capacity, 0, 0, 0, 1}); std::shared_ptr output; ASSERT_TRUE(AllocationAccount::create(account, BudgetClass::kRouting, &output).ok()); constexpr auto category = static_cast(BudgetClass::kRouting); @@ -20,9 +20,9 @@ TEST(ScanOutput, GrowthClaimsBothBuffersAndFailurePreservesThePreviousAllocation reserve_scan_output(packed, 100, 128); packed.append(std::string(100, 'v')); EXPECT_EQ(packed.capacity(), 128U); - auto pressure = ResourceBudget::try_reserve(account, BudgetClass::kRouting, capacity - baseline - 128); - ASSERT_TRUE(pressure); - bool rejected = false; + const auto occupied = capacity + (capacity / 2) - baseline - 128; + AllocationLease pressure(output, occupied); + bool rejected = false; try { reserve_scan_output(packed, 100, 128); } @@ -33,7 +33,7 @@ TEST(ScanOutput, GrowthClaimsBothBuffersAndFailurePreservesThePreviousAllocation EXPECT_EQ(packed.capacity(), 128U); EXPECT_EQ(packed.size(), 100U); EXPECT_EQ(std::string(reinterpret_cast(packed.data()), packed.size()), std::string(100, 'v')); - EXPECT_EQ(account->snapshot().allocated[category], baseline + 128); + EXPECT_EQ(account->snapshot().allocated[category], baseline + 128 + occupied); pressure.reset(); reserve_scan_output(packed, 100, 128); EXPECT_EQ(packed.capacity(), 256U); diff --git a/lib/crowdb-tree/tests/unit/tree_core_read_test.cpp b/lib/crowdb-tree/tests/unit/tree_core_read_test.cpp index 43f46cdf0..b4f4c1b5b 100644 --- a/lib/crowdb-tree/tests/unit/tree_core_read_test.cpp +++ b/lib/crowdb-tree/tests/unit/tree_core_read_test.cpp @@ -223,6 +223,49 @@ TEST_F(TreeCoreRead, PackedResultKeepsCapacityChargedAfterTreeClose) EXPECT_LT(runtime->snapshot().allocated.routing_bytes, held); } +TEST(TreeCoreReadProgress, SmallResidentReadsUseActualMemoryInsteadOfWorstCaseWindows) +{ + std::shared_ptr runtime; + ASSERT_TRUE(TreeRuntime::create({.workers = 4}, &runtime).ok()); + Config config; + config.runtime = runtime; + config.runtime_binding.limits.routing_bytes = 4 * 1024 * 1024; + config.runtime_binding.limits.io_operations = 1; + std::shared_ptr binding; + ASSERT_TRUE(TreeRuntimeBinding::create(runtime, config.runtime_binding, &binding).ok()); + std::shared_ptr core; + ASSERT_TRUE(TreeCore::create(config, binding, {}, &core).ok()); + ASSERT_TRUE(core->apply(1, {.ops = {{.key = "key", .kind = OpKind::kPut, .value = "value"}}}).ok()); + GetView value; + auto status = core->get("key", &value); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(value.value().to_string(), "value"); + std::atomic remaining{32}; + std::atomic failures{0}; + for (size_t index = 0; index < 32; ++index) { + core->get_async("key", [&](Status outcome, GetView result) { + if (!outcome.ok() || !result.found() || result.value().to_string() != "value") { + failures.fetch_add(1, std::memory_order_relaxed); + } + remaining.fetch_sub(1, std::memory_order_release); + remaining.notify_all(); + }); + } + while (const auto pending = remaining.load(std::memory_order_acquire)) { + remaining.wait(pending, std::memory_order_acquire); + } + EXPECT_EQ(failures.load(), 0U); + ScanPackedBuf packed; + bool truncated = false; + size_t count = 0; + status = core->scan({}, nullptr, &truncated, &packed, &count); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(count, 1U); + EXPECT_FALSE(truncated); + EXPECT_LT(packed.capacity(), 1024U); + EXPECT_LT(runtime->snapshot().allocated.routing_bytes, 4 * 1024 * 1024U); +} + TEST_F(TreeCoreRead, TerminalGetReturnsIoCreditBeforeCallingConsumer) { use_single_io(); diff --git a/lib/crowdb-tree/tests/unit/tree_operation_lifetime_test.cpp b/lib/crowdb-tree/tests/unit/tree_operation_lifetime_test.cpp index 3e04ba55b..90a1d068b 100644 --- a/lib/crowdb-tree/tests/unit/tree_operation_lifetime_test.cpp +++ b/lib/crowdb-tree/tests/unit/tree_operation_lifetime_test.cpp @@ -15,15 +15,15 @@ using test_read::LoadCredits; TEST(TreeOperationLifetime, CloseRejectsIngressAndWaitsForEveryAcceptedOwner) { LoadCredits credits; - TreeOperationLifetime lifetime(2, credits.credits.bytes); + TreeOperationLifetime lifetime(2, credits.credits.reservation()); std::shared_ptr first; std::shared_ptr second; std::shared_ptr rejected; - ASSERT_TRUE(lifetime.acquire(credits.credits.bytes, &first).ok()); - ASSERT_TRUE(lifetime.acquire(credits.credits.bytes, &second).ok()); - EXPECT_EQ(lifetime.acquire(credits.credits.bytes, &rejected).code(), Code::kResourceExhausted); + ASSERT_TRUE(lifetime.acquire(credits.credits.reservation(), &first).ok()); + ASSERT_TRUE(lifetime.acquire(credits.credits.reservation(), &second).ok()); + EXPECT_EQ(lifetime.acquire(credits.credits.reservation(), &rejected).code(), Code::kResourceExhausted); lifetime.close(); - EXPECT_EQ(lifetime.acquire(credits.credits.bytes, &rejected).code(), Code::kUnavailable); + EXPECT_EQ(lifetime.acquire(credits.credits.reservation(), &rejected).code(), Code::kUnavailable); std::atomic drained{false}; std::promise started; auto ready = started.get_future(); @@ -53,10 +53,10 @@ TEST(TreeOperationLifetime, ObjectAndControlBlockAllocationFailuresReturnCapacit size_t rejected = 0; size_t admitted = 0; for (size_t available = 0; available <= bound; ++available) { - AllocationLease pressure(credits.credits.bytes, credits.credits.bytes->available() - available); + AllocationLease pressure(credits.credits.reservation(), credits.credits.reservation()->available() - available); const auto baseline = credits.budget.snapshot().allocated[LoadCredits::kBytes]; std::shared_ptr ticket; - const auto status = lifetime.acquire(credits.credits.bytes, &ticket); + const auto status = lifetime.acquire(credits.credits.reservation(), &ticket); if (status.ok()) { ++admitted; EXPECT_EQ(lifetime.outstanding(), 1U); @@ -102,8 +102,8 @@ TEST(TreeOperationLifetime, TicketRetainsChargedStateAndHostAfterIngressOwnerIsD const std::weak_ptr weak = host; std::shared_ptr ticket; { - TreeOperationLifetime lifetime(1, credits.credits.bytes, host); - ASSERT_TRUE(lifetime.acquire(credits.credits.bytes, &ticket).ok()); + TreeOperationLifetime lifetime(1, credits.credits.reservation(), host); + ASSERT_TRUE(lifetime.acquire(credits.credits.reservation(), &ticket).ok()); host.reset(); EXPECT_FALSE(weak.expired()); } diff --git a/lib/crowdb-tree/tests/unit/tree_wait_metrics_test.cpp b/lib/crowdb-tree/tests/unit/tree_wait_metrics_test.cpp index 1d54bec8d..2043d9f4d 100644 --- a/lib/crowdb-tree/tests/unit/tree_wait_metrics_test.cpp +++ b/lib/crowdb-tree/tests/unit/tree_wait_metrics_test.cpp @@ -174,7 +174,7 @@ TEST(TreeWaitMetrics, SharedCacheWaitersHaveIndependentProofOutcomesAndNoHitOrLo upstream->frames[1] = test_read::base({ {.key = "key", .slot = 7, .value = "value"} }); - auto loader = cache.bind(upstream, credits.credits.bytes); + auto loader = cache.bind(upstream, credits.credits.reservation()); auto input = test_read::cold(1, 10); size_t success = 0; size_t failure = 0; @@ -220,11 +220,11 @@ TEST(TreeWaitMetrics, NativeRegistrySamplesOnceWithoutDrainingAndReleasesEveryPa wait.finish(TreeWaitOutcome::Granted, 150); TreeMetricsOperation stage(metrics, TreeStage::PageLoadPipeline, 100); ASSERT_TRUE(stage.finish(TreeOutcome::Success, 150)); - const auto available = credits.credits.bytes->available(); + const auto available = credits.credits.reservation()->available(); { - auto resource = std::make_shared(credits.credits.bytes); - auto registry = create_tree_work_registry(resource, "tree", metrics, credits.credits.bytes); - EXPECT_LT(credits.credits.bytes->available(), available); + auto resource = std::make_shared(credits.credits.reservation()); + auto registry = create_tree_work_registry(resource, "tree", metrics, credits.credits.reservation()); + EXPECT_LT(credits.credits.reservation()->available(), available); std::unique_ptr file(std::tmpfile(), &std::fclose); ASSERT_NE(file, nullptr); registry->flush_to(file.get(), 1, "test"); @@ -255,7 +255,7 @@ TEST(TreeWaitMetrics, NativeRegistrySamplesOnceWithoutDrainingAndReleasesEveryPa EXPECT_EQ(collector->capture_window()[static_cast(TreeStage::PageLoadPipeline)].count, 1U); EXPECT_EQ(metrics->waits.totals(300)[static_cast(TreeWaitClass::SharedCache)].count, 1U); } - EXPECT_EQ(credits.credits.bytes->available(), available); + EXPECT_EQ(credits.credits.reservation()->available(), available); } } // namespace } // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/unit/writeback_rebase_test.cpp b/lib/crowdb-tree/tests/unit/writeback_rebase_test.cpp index 15a69b9d8..1676e5dad 100644 --- a/lib/crowdb-tree/tests/unit/writeback_rebase_test.cpp +++ b/lib/crowdb-tree/tests/unit/writeback_rebase_test.cpp @@ -54,7 +54,8 @@ TEST(WritebackRebase, ReplacesCapturedSuffixAndKeepsNewValuesAndTombstones) current_leaf->content_id = 9; auto current = dirty_page(current_leaf); std::shared_ptr replacement; - ASSERT_TRUE(build_writeback_rebase(captured, current, receipt(), 10, credits.credits.bytes, &replacement).ok()); + ASSERT_TRUE( + build_writeback_rebase(captured, current, receipt(), 10, credits.credits.reservation(), &replacement).ok()); ASSERT_EQ(replacement->leaf->overlays.size(), 1U); EXPECT_EQ(replacement->leaf->overlays.front(), later->overlays.front()); EXPECT_EQ(replacement->leaf->content_id, 9U); @@ -155,7 +156,7 @@ TEST(WritebackRebase, CapturedSourcesDoNotConsumeTheRetainedSuffixWindow) TEST(WritebackRebase, BudgetFailurePreservesCurrentAndOutput) { LoadCredits credits; - auto metadata = credits.credits.bytes; + auto metadata = credits.credits.reservation(); const auto remaining = metadata->available(); ASSERT_TRUE(metadata->claim(remaining)); auto captured = dirty_page(page({ @@ -196,9 +197,9 @@ TEST(WritebackRebase, BoundedOperationPublishesOnlyRepresentationAndDefersChange pages->generation = 1; pages->frames.emplace(0, receipt()); std::shared_ptr operation; - ASSERT_TRUE( - CapturedRebaseOperation::create(publisher, initial, pages, runtime, lane, credits.credits.bytes, &operation) - .ok()); + ASSERT_TRUE(CapturedRebaseOperation::create(publisher, initial, pages, runtime, lane, + credits.credits.reservation(), &operation) + .ok()); Completion done; auto connected = stdexec::connect(operation->sender(), Receiver{.done = &done}); stdexec::start(connected);

    n|d{4x`Z@_Ms4y%B`j0& zgo(|VBKxn^Z|hce4ojcS9kBOO6YdI^qvs1NYFJr+`{n~CvL|GzXepq*C4`s*@;C~gb; z4Gj>@+%rjw1-(0aWEJDs>9-dD=jZRbJ|g+q>xIUnqxvQa#Dz?(+uYw$2vMlqYJDcy zSygQZ90s=9+D^Wqy=8W#qyu`T>>C}W3ONx*C%Q|Fc>|lpkhf04jWAI$C_F!Z(y?O0sF$qtA3VY;woJW!$MDEMlS)eQ{m@sz@BZ z>iI>B<=RZ|D3-iO>ex4MtQ3F1gfC-ZSpiD03zrElSCEE9@2^t)Dbnx0j(nBYY(_#e za!xilxDPL|^_}BW6M2!vbp$XlskdZCCH_KO9@_j$M{z$fSj!_F=860 zD!udB?DR1D4II9DZ-`9%Fzc@N-nz60;&z0E%ZBBXJn$&4o1@?zqk>w9#5MFuBv|kY zpXk{Fd4Wki)Z>21ClVyfeI_RG)AV8--N#$MsA@Tm~#(e-*G zIYLx63HJTa5$fOP2w(9nL&BdbpCZbxUeEtP74mRoxZDhCDo34&?XYBvK8(IR@=g0T z8*VzCg>&YdMuLR2<&OaDSQ_G!H)aNPzQuO_g~aEMT_U`u19oyUi+L*bG)%&~PzQgt zxt*)?O&I`B!WTAlI?G*6Rg-=UkMWRRaccTU9!p`8a8{D_xyUVzi=+A{#Yqh9(`T}N zig%?g4nSVFoufndh{oUvhmuI9C%tJ|Q@MNh=Q?nN8=A=fEkM}RO)h<3r7uAVLrrvQ z@hlX$Q#o91bWXvNs9>gml;n~u*7P6#WhVY@3R^;#t<~yGyOzOt{(B~1cq*bAC zz2gTGXyssk5ylLjm)JpS8rs90nSoT;USZ|*X4{8-K7t&6BeNRgI~iUp)i@S-wYx4+ z8Q8WwSBEOT0Lvz5JyMD~UK)ET4x0T~^4>^>(&zgZ11@wOoA&R@hNoeo5Lc&X!!;vh z{=KUE@6o4N-i0XbMtmTNXtN7}k@*=<)A_kiV+!3lUjzyYOYIvHtJwR}UGz1Yxts!b zsSOki=?R#oH;x1VuWCuD_8YopaGqbGjxgV7%LU-4<4gQ@eOCnQ8mtH% zE=Tz?@N0wT^kfOz!Xd|%RC^kNJ(qPfUkjyXggJ`JiTGw`%v|D~YaYSzSE-QFfERN< zqTmb;u#=t|PwAfN`LZBp?!dLTJ^y~npC3Cfr>h;l5EzA`oNBFMpCyx5;nNqHdUz!$Rot zi)W&@d1 z5t7peddT$2;OB#!#Aw2Y$hBWZy|6FwDYYOo-=&>U>fvA34J1yJv(^R7bZM^PsRjBC zmyiX)zEu&?RhjN%m1CInI|5yQa{yu(>-oI2m87k2W zrU7gdzAML>nY^32VFV|pw5OuWCH_HMZxuaqp8nDB7g(4$9Ay2LD&q4!CAZM@U5m~~ zm>Sy+Dq)88hBMUoiWf=FLr?Y8{a5l7_Qa}3gr1s?Xv?2JEO~-TqsEzudIHb9ksT0q zog2+GrjE|5Jac+8UP)G2zuic7U%@!u7Z4GY%FMGTc`cOFFV87z@dLR*{&as1zl?>N zEcLI#1fFi>6lLTM@JszyIBp+u-luprezzKGRhO&Ftv}9l^u(o8#QuTlEi*ZN-gF4e zuE`FJig8NdFg~}2li#{FbK4eoZA|=0v*Nbr3_oy-7U!AuZ!dWR0G~n@Wn(Dujgm0duWF9DW{jB&N=nu0X!YO55+s zinwJII>jc!CUnH@O#fd(#KO(u*GlxiTLhG4_+=1h0l$EGquaqf`CIQF?SDDk>hCFE<>kp|mvTmP zLDd~xdX^|C6`iJ1*cV4%etpPEe1f^6&55EP+c=LAM%#XPCsXg%%3br|;;PXG#jeP1 z@WIdFm-{4xLVl6NQnc|q(<0}*Z0$iu_IPo2`w1GALl^DaKY9qbQhVnA5ld;TboESG zAl_WHU4GOqgmIVwN2mP(oSxiCqub#oKa4Lp-a%mn9}MFP4D9EsJmm#$P}kj05qI0p zjTEUq7{M~S1}GPUn!P35e?P;OAFXnvq)v3DlKXQq=f@`xlYF+m5Jsf|pC`r}u%A{> zY)$kucp^E`2ZB|1%L!YUsB@{}2`eX>45)C^QRR~D&$4OoAAQWKShAx|J`V?oIZYsb zxv@Kh@3GSW%Bn+nX_s13_Vj=;5@fa$@y-Jdx>~8xk%LqB&wk)Ugx$k)9a1&k1-ieqbONFRpdh=vY2k zdwjlRUs93_>pqx^+kNpNGAL3p1V343GK=eCaT}mR@cV^{u?3S)UT8J} z4=AxK3V7x85U!Jc=z(r*N8;4uzDkKvPLVH}-*zFVTnVqnq z6ylEatH{t^){|M|6WCs3s>_$)48+jh^}J(jh7c!=$yKHs8-VXqrF?Z;U4}qU`!hU& zcH19^_BV5koR^g*>e=f+%Rs8vF$AJ{&mon4+3mrvg9xg;%;B2(^mBFrNh$`jg{j%W zW4Xi01>yIV9C?HYm&sce>WJ`Ya^C4vZz+0IDQ_ABVuB=}RY%@CW5g44q9! zsb~+d1)qBFant#J4rApj!m3q(P!;e>CEeY*ERmry>A{8%aTUiha2p(WsK#l?vjeqt zA%mJxoZUUId^{k&nm%i4{lxwO930M)Z>)4j`umUMfat%X*N!R1hANv0xG-?C%7 zt0>?4>&PPmkwZi6W+SOQ%R-kAUDs6KTyGa15O`aC$rfO_%$MWUuIMQBB>3h<_!Yk8 z2@iDB%(UMT_Tc|np5)WtY$e7NdRS<%Ri`JWLa1t&)9Q}*n}TK@SZeyX{zxv5g`jiB zI5o~(?Ap{K>|j@=RnDO)=e>Yx@aIa|m6@1Z>b$YvZeS0EB7 zF?P5Ax|#2`4?#F_Plnjec3$$Km#TfqhTXF?Di zu$-zi7)C>@ZJq5nB}SN=l{Y*ab=@cOQC6;_ZNH$jH9ji*U5WWecUQuA*tyFw^`8T= zhehFFIDo{Q$K)Z+K7(0F)dS72Q2(s;ijIZSl&S|M9H+3dC{wTbpyf}Li5d@j11{lB z!r8=vMBE8sw0}^K;&&4p)NxKazPGd0N7f&mCJjUXZvRy5(P0(sd)|+#H!5(w(esv7dcis2d8mqVhse3l{Ctyg}acj9YNHiMMl}aOD zbNCIEE?Aza6TTVBwjavY%L*zg21jP?*3@jAXI+#&(K=;$UpOwV+n#b#rLjaN6kB`= ztg8YIF@Xo{55G>tcNj826?*QGhQ1JPrRq)h-oXfsIm5F;db4UR9T4CMg_?riICndY znZo-5Y{;h7>F5^e2Nb)qcA(>z8rI z{hx)z-oX}HVYfvk{9st64O??+Q?c_$I?gzRM!J3oS+!*60P>de&xq_E;r;!uM0m*7 zAASrJv4>a$Od>eEGZx(n6A{HWkR_acQLBv2_evY#;C0L&RYieb%hi`rRQ!uW- ze5MJOMdWyB;=-H($V7)0=EOXvHaZ?9cvzcD>F+ML+W%_VG$2haOe^AB&w&Zkr{M*PgW2T_Sq(JVa#HCs+z+!mX`_%|xG; zj<2OJ*g6VEcx9e!qNLOu>i+Xbm5;OHkY@$v5l?_gb%wBH8I9>?>+cVZ3F-z8;(_g_2*bt(qAygtwH zZ_;AL_wGV3IkxCvbr9L_Vf6Yv2gx4@=CC|p># zf{5_Rr&1D@!!dMh70-QML zE;x1Zwg*L?Pf;2kGv#l}XLh@tJwJO}reRBh#)7acjJ!jJCOvHAs9K?d0SPR|bH+HwB^cty4Orb!)ZVh-_(kjC3Xk zAC~TX?A)HZY#+k*VErr<$!!BQfJV#CcW}1bWeX;khv@f-X`;jSh+Dw~GP1Q;SWm=pR{YHNkyD7b9cZRbTT#rEx zu%M(6ZE}GqgN_`FKPxiHrugPyK~L}SyYD1+yo9Hxp*}~NykNnGJ6rimRyCUyA!=BI zVx2j{tghg(oegH$uZdFeBky9=?0xZ$Mo!;vBQxjo!NHovvQaZC8_m3tqsl}<1{4%d z2ntz`QE6Q*ua6s^0}Dnt&&$}p^5K=KvvQBdUmsiioQS6~jApM|VNIQG^7FK|*3W84 zi>vuwz&8$@%Cogm{q(Nemi)5?0v+_>EMc43N0kK#GtaxpHQi)8A$zi3jSAm&*C8Jm zJsChj18{m+k@Trn|I%yMSh9HRf=LPhK~eh43|#MVkY^PvzF?yAr@RiSDO|@l)kl*y z7$%oaaqt#cq9ru9fGQk0J09PoLBGRCtsD)4)b|Lzx7AULXh~tM3_-3n_6QoHp3S5%&PD z>J?VEKIrq4Pt51r*h?mcSBn|jul`cr9@h2Ir+oZ2_Rz}h+VbE!|LBajbo7aW96jAY zUp_13X`8WO#=R@8KSQ)xhQOxp4O^Ajxaoa#*X6{(-u;XI!O&$OTCkg{)28;!U$Q6D zt}>K)1~^xdMJ*-vQR%=+PYJd{u37DR^+&~EB0n2_({WSnU&%oQ7%$0VIa&7@4Z0!{ zR_j9{K3-Y3D4@?Qe+J%Ko+~R|_PONA69&vMR~4>^hHY#HXfo|uFrjC4HjNUU&nQ>J znj%LXCO$q0E156VJ8P`AG6CjHx0Tv<7)VA(4ZH3gXPHLHyiYu()^x`qJvbeo$u4wq z|KSm%A02Mut7&S;&cJL*F_~6;B5Rh;R3}{|oDVY+e|a!><)wLfi?Ql{^zNo@ttPiD z=0H`5%9BWC)pm2WCPR(TcDrN1EPgrNZw6fAzHc2P%dME1 zYpZ#c9{Qel5SS;XlO-0YyeOv)u35W_UFVuXW9+%`^o%iDEUMhn&d!@0Lx{-W>mD8H z!g;}|va$S3lqAQ>?Q!UDMe24TrNEJ!1nJqa_-!u6FV-3zhKGC6KQz>)3)%TG)Lv(! z+r9XfiIVa}R$5&veGsI~zCk3Z7tw)%y z(A9;lO8D4&^yisktrEZ9{MOI4LUj^lU?kP0nz7UvmrM}-G|us8+8Q61uQA{9BmvjA zyjOTH4{FWQsT}V=yv#&JZgTcSvrY}yV17pBd>$#C_Qpl~OK8h16j>DW523BdMG@AX znX0hF)7jaPVhUV|wSY3X1x{#dHbMrZw>3SKuJ;4@r?KMZMZbX#_ekP=AXircm5r9-@1gg zw0Ep+B|e?5AQK9pf7t0Y(qB(ekx0F4{IFACKqQd9*Ql(ZTbH6tmr~IuP+?d{HUS*I z9u3?Df(gh*s$RtrpZAIe&5JcY6sfnyzzzs{CNb0O+aKP{wwJ1FXofF9xc&UR4Kb$T zHYL5w*sD0A2c);U3=5*!nTez|o=5qZ+kbhBw<+6seHdBA(`d~_!48<87WV+yVOI># zrQQWtCOwfQ%bt7bt@Y*^k%GyX8ZcBKn#k-4 zsCJwQn&Ekc5nf};5Z)mHeB6pI=lhB+b$u;)I2!sATGQswxAB)n>D;$|lcJ#xO}%x8 ztk7aK6S@Nm0x`2jgc^0EZ_iM+6S>w)N^{ob0y07OJR=RSeM?)df@PV~vw#E0Y{89| zmKyQ*EMNxh@da)?@(mK8cO&jxETzxo$SX^|VG3VqrK(jTxC@3qqdGRerZ>-2756NG zGS15s33!|8c~1=V+N7}b`fAMY#K}3KuGAgzimv!hP8;Y=FWcBjP$jR6HIFf@(^uQ* zzHqz!o%7?#4~I&UuBHPhtCQknGi3I=5!|eVF_c4psP5%g8;br$=T#Xyk}uEsY=Yjp zEc>{NMTbL|Tva~t%T)*%#Ps`#I(UWja$u2C^L%2R2g`A3PQ?RMbqw)-z@i7}ZT1oC zcHpu3D}-_xHukeF$97W#=&KzLW>v6V#c6+<-H%a@YPbRfFGYL+X`U$OtE9cbPHn zV-ON#B&ELYa8o}pD#)yQcdV~Y5ax72cOUz@y=nLNHXtQ%nrfIDUt_p!Andg%spDU_ z#IZLo7Hax=C=u!eYZ+&=7M@IhPDW#tev2x#x*J6vKIpM1DWTPi`7KL*oVCgXPchBR zU~Ac<%;|dX;0|Xn?z`@S>h71H9`-nHu$*^p!GO;_ELn45j%V92{yIRVUkNJfnLlI% z&N-KIExUHv5C67Z%gxo6^uQg&55K7<#e2@^w%Pe1i!~)4#9Zn|eXlo3PJV*xW85@f=b&cZMevmG_(#Y9*C)lWM@?fiNr;v^ zckZ#iZ1k-)YOJ%{-a6Wg%Xp~<*s6Yikft`#o>BDiJ^KampA#eB=6E9-I{TIzg&2av zFg9=O#2B%bo-jW@NQ3ocrGQiq)tDzfqwwe_P6PT`dFh2Fr7i+8vJ1%E2=QA{k!{C0 z$EF0AC%v*QK8WKmBDpCVw7t!L&VQ-P-HraU+rav2+0FIet14_iUiiV?N%fCS4<>~M zE#%p}c?!wi_mu?8Ks|R&%3=K#bA^uY;=3arPRKe+;e57CMJw99H;fJXZS?gI^Ow%J zfFNGti=6uJe767n7((DN;(uTP{_@WL_xJxVAJdrs@ejAH#!b2yrpy1tZJVdp zadJOXp@(tsm)q9sA8y-jcleAw{GYjP-`yK@j|peKf-k#q4czeA_g@|a#(zKdUmySP zI~M!Wa6Ls!%0z=PYS53rJo4b^W<#6PuSl1Kki(W`hct<`#}-ZBKj^Crp)_3^yJs7F z3u6G798TSOk!jGyQW|rSMyU-5Phq`ydmP-CSMB^Ds5{<(?jp5a#!RO)tBiCK+kNkd zfC@L-w5U#1I)0*_Kdv752=tRaTx9qGB-GTP`?2RTH1ksRrJ=&`sYW^oxb>Ue;~eZ1 z%uN|~!;7~jNJ!A3&vRrxyJoV~W)#IiZ_<9C9;^yFgNDPQ9#0^Hc=wxA>eEQ5J{Et_xRq`6&?z;5fNz39e zF%2Jj#=z;`c56!H0t{yB`aVD`b0(%lcKRvuFz2Z2;ZxvM-xn4aPeN-j1>l0e!PgR5 zw=u73eT8<`m?}==N1)Sz8=Tv=@VVF{;pr6ABPW0PQ)ex+t~j5Z73`Hf5ZBpv67s4#)YChe87H_s?UhK8 zmlS1T6Dy2W>xkx1@`NL0c_ejR_vH1DR|1>l_3Ui{$ZS&#$3C>_=g{&2C(oZ13wTfF z0cA>Wi=8VzwT~F5WtiL7m>j*7OGC`ql|Izs*;ga_U}~a$BM@=_sIvR)jBuCOv3`U* znXIDF;fxTKK}Nf#v)tMl>Sz1#kWPYT9j~V3(JQ?(S$F9oIQsn}gDm59I>JjyoYH`< z1mqix#lX6|Gp!>fCoE|F)ki9i1J%y&7i4DZ z^--|q2G#*A{%VTY1F%j0A~71JIv1b)Dl=Uw7;@yby!xFr_;sbZtiiaGR^rm z^kLQIPHC&_JbtREuEsjoTzzTE>j4cg@M;p&9D6>xFzY~%P1&1wravL5O=9<{+@v&>g|qkOAkGBbTWi5-^s%wVzBRew&-g6Z z>L$-=fv@S30l5F;22K-wk^MJzYOaIlCzNUlo+WwsVC$rZ(`e{BU4NRG*Wm6q{@Ah4 zj!zAnZ-1rRH`8a!=Q|LRzv-gfY^_HheGOGX%z0<{ZNEU@)vrhpL6 z+MWPs#3f=Ren4fQ0G*HAY%e_t<8oj1)YlZh9ZT*TP_h$j8)Z~SOICaQa<3nSjDEs? zn>cB#C9y1cZmap=B~Zta$`uixGd7r{Ie0!`qzSNDzN~jlC2&epz=IYdV|>3AK9rZC zC!5~6oP1cl!`~fI?YEX*G=Ul8RvDS*pB2mDCeWdUd)b@m_xd=9JlH>fp~?tr0h`Jm zYIIb5o%%DYWv^DEYRMlKX*k>B>E(M%%rPMIrZ%oap96DwdXUbEDu#t3c+ZtR6CCV{ z4NmgNTlv8M+@MVxg^jP4s?#YH{ov0mQP6}Z&gd=!O3lsvI4`L=`tC;49O*2S92dNH z8G+-dyL^-#Lfjr32Nc#FRZ}hn=-`TEb#5mFi^it}lUAeDTCelwuL&dlNG5i+p;=c{ zNw0>1aJgaCf7ED_jkS~eudd!;fIwTz5roWDCiDe^3-3~z8`@Kl;%)Xweu?)+>|{4W zH+UySTo&H42CxY+YJZvhik-0RSe_u-GtV1t zi^ODCiC#gg+Uwjp0ez-{8`#Xu#3lQ4R%kL@h&`#fa#clYLX2v>9X;~|^*a`aryx^u zVvVy@u%ItIpXf^JvJ}3ctq{Ifi(Ti16lC&BsBYlKZua0EG_=()Q_2iE7bkvcI&lYg9oA@9g~bL(OAfUZ=D3 zT%RU!C-8%H(dHj4rbwXo2jk}|uuY6XmaJrE@?rhL{Cm?&P1$w3Fh_sPZb&VS1CrT^ z9mqZ$8}VVABEbt^o8z}NlG1zsW|G5s5xf?L*DJFipvwZK>vXb>OSe(;Q|N_$+W0Ke zeldhg%rv&Knv;3AwmqRnr#@j+_SiwD$PD?$qLhY)&iFnMGq;C6Z#+f*5R_cAva-?T zofnxULmY6LsnYgCH+*l(^TyguagQx~MPBECR+rjAanB-W_vzYtac$IIoU9u8wq&l> z(S`<~t;-Ks-HaS#tcw0Tjv!`Cudp%3eq7$~y&Mj(vlK_({Y&CDQ;Ni zn2vZhcvO&IpT>uDcNh7lbk@UEhx>`^-AmRCj6~UPBnf&&HSLX810jQ+K&`#rCkt$s z#2w|P);B+^qYM=9UEEz&HqHZB_#=yp&$k`)!8NH%7S+_4~F*rX!u4cEp) z4FL!RYgR-|<2Bbiw&7UeuGX%^R#%+bPdz~fy==MWq$bh%&bsY1tg$>454iz2~a%szGu9Vz1kxB;xUe$ zpJ}JYF_`O$fF_oU)m!5;Th8zhSWg~q*a6Vbnzm`~wlDtNZ-sIfnVFS0`g*)0i9GNL ze)MfiJYH5iU$)1TA59P87?mGmYmAFY=tF(oKK;Qs*0?b?rrtkI=>?X0x7V@PM%Li? zBLN-VMqxC)spoghMi&EBR=lze!KKeWPis^$PZ8K7Lgnk#Xhg`!Q3Q1E$GA*({3(4a zbR`CPi6K8;+jW=EFP6GHF1~W~Q4w8!L^u)R`$2Svm$nMt%VMQshUQz+$K%!NJ9`IR19e#&?7mb7#HkN*T|;qJf1G z$(3=0lELGHkZ;BN)6|oWc#=srYci)@`8=0_9^=SJDw-_=PEsDdxlyIp^LHKCji=NP zjqN?QV5EGW-yg@$)fdxw99k}UTH8&t4!bpcrH`X0t2|~e z!Ku0UPD}ip93M}eh>ShMQWLGoA=Qp;k$t1G=w-ajmF!iHfd@3Es*3y?@~7Q^t_~!z z$#(j&^avQF+1PB%w5OV#1mH;$RXb+!#t2*IpyH#m8tk{PP0o3Mh~g&MS(f>UHgpkQ z7fPtH0pYEIAciBeh_}22boXM`g;E;sP%&khy^LO#loesjU7_vAo_E3HCBh-&`#X9$7KR&!21(z)7LVA)Ce$Ft5w0od6 z+;}c$y$YaN%&y@IXH{-s)&*6+3R3##qFysh6Ip5vj*Y^H(5= zuTu|w#QlNsrr-lH9})O<4nDb)``z#4eqJi1g=%&c?qYNkzO~w=!X_)4kt~T{tr?lV zZw&&I-wo>~8%{R)-We*U=^ERr;)0f~9mSuOWQZ3cS|@`0fSpi2Q|E|Cg;`VnNO6`2 z^;ITD0L}d3(cKo+T3n*Fh=_6L6-9@th)BgN7NgXwS|XljkeZU)jTp`7_u)MHO~SR! zAwx5LlK}bfV#GJAIrW~dRSwcPCrY@Sr$gEL+b+GjNK~FJx~WzoXFcQF%tl|_& zpWrBh?OJ|xxTNti12TG_oq|IXO6{{OXEb;>`$~i!AVlf0{Kl)5#OxC2soCA3zXh}g zzH!N#<0K8HbzMJCPjTU!f70OGvLo;xbgn9AX$JS(9gGZc8kYNTj>6Kq;`$`N0ZF2yDpK|o+Tk~8&M*U z7Wk`w6>n`W*jI7RH?o|cXn+@MXuT-yOGrq+>OB;$98_RxR-?WE!EGfJ!16TKv@$kY z_4))=b1}P5Wt>-ST{(SL$Ll`n5XYEDRr~&4=jJt{bT1VSRa_cJ>iIdq%1I=lqcTN}TS(e0s0^L$8Hq zWgYGjYs0Aw1?$5rL!pO_>2>5$F2R_V#Xk8js>#%}RRaqgtYS4e}M$2z8wxZUJ;VN)2o zG_>ftwa<@hzPV_dadv&HZ2^H=CH7ee;H6WR>wMo&-o{ub@!Fo&In~ii!6L3|FogSA zN==EeC8?oh_tBoC)YkTAe~0(j;HcW`+)$TbChoRoKN}+HVh8x%g-X+Vf8de70|b9k zJaxLvEBQq13=|b$1r}dAg%oLb@Ah*RxXH+? zvn$NaOLQE(W8%*c9h-?C(rQR)1M4j7sY{P62%*)+@EWnCic!#Uab@@l40gH-tDdQf zO-mKKCDq;RG4iux-?R)U+MxLYTcY*xfNsUyzmW&)b$Op;Ro5&=7di@z&|)ZTMeg3* z(W##uTc!!64+IGTJD(=-mXl-0n%ppIsI1ARrnW-gmJYl(_{s&b?^T%)d1;y#d2V5G zYw6~2jHX_Jf;oHi&d15fpU%&^;;>FOdnB+B)5C&roTz07aVjiC7Z@a6?F9BMNrT z8-KGG*UA^l-DdsRi?)%LOnq9&m)u?nTT`G-0-H(eEs?^73TG$(sY_5SHCeOfVpKP8 zZe4QSsfikevX~3FAoyy{4b%fM=sDi#H#GQRZ3djMjom(uby?LT=x3mkNA&Z$(mUXz zktTk%b`Rc)a5qhOmHg|PT_Gub(xTue7B=|y*PALz=EHl8(~J@`Hyyet5UKF7OK3zZ z!7|_oF<{|o5Rv?IdK(#4qKn_o*AYjh5RPtu?&{ewE)|YnKDG`iE^8kNy1?i!RhwW@^HyZc=>cpOi?TAldnVSolWIQ*w1es$>D6k$mTKg19uOU6u9_q*8lxH;J({{wniEp>d1B^`5F~Jr)C?u4TI&bu1!x;uZX~ zhb$obIA-&XBntMlke9z(b+^{_J27dh{XD6n^k#Prh7jj1QBcW{-Y18pYr|& zU^+qdY;%V`eVbAP0~t;y60WG_jiH9EA<_gnf)09cJ7$1u$gnQ7fy)1iEuH7wgRMNf zVBkuz5&Ev21=s7rSG*@so(o_TbH4$JM3BI4V&84gw>(yr!VxWNDpzybik3^r0 z@`5ZysP7f=#_v$epP9~lIewECVxrN{^?qNv^ZXu8ij6vvq|dYOYMp}#uQESv7Vh3< z#koS?)8??I02f>JGX-Y?OLMD#)lXi`EQl2lzz$~Rd?)us+`jf;)i+K$5_Y4{ z2tEjzB&Tw12|=#6Oo(S34bGay`7LYO!1(QMaOR84>4||=;Yu&4GMyeon%usxs05n0 z0_M~U^>?4`Ym(Bn`b-%`9y8t{`1JE?$&Z*kT7MMLqQ6Z$JjdnOY@`lRR7Bm8*cOj; zqiYv-^YnTlXsJBjX}@O@@Ng5&J_ei~jYa=<=MGhW+`MBm0-FlsR|a^N)zi)UeTK+3 zn?I`eFnGBoJ)h zOJ)63MAkux$qtObo*ECm3}`dvau~J&uAYmD04FbU_%l~&W=0*s?A^XJAvQZRk><$B z@%M#rj9C2K#+J%N^cM))1Ra40`P0vLHR$IUJF$H1nR!{;)lXAn!y)zONuAj?LD@)` zPAvQ^Z;t9Fe0iC2LfU43C{v!LdCT2)p;;nr*L{rG(afE~q@)KWM(a1cyMX+~I9(gR ztG^8-CrO_Bs9tW<_gJ064q&?)@qRm9vP7rj2EQjIc z+=eu6<8F}$7XJLH$jRRlZbwKuJ{5nd7@#~Gn{s< zMqF^%L0`{h`*c9${lckJ@!Uy7$jh)vaO;;!&9|t8Ac925oq4;RR*dhY&eIXJN!Ih! z!XgT%cg2X@;+5P&Sw-r3gScTE1-7DHU4msU3G8+3+~QVA}BFGJ@{*=6-{pGBr2R zsv276AH}HI$~9@&TmQiE;}vhcF;t{4SGs_-vsIGIoAB3l4$4Pf>~*~TX-g6aHxSm1 zSjrLpBQf_@Gu51bhN-hRuWTl-PI)}Umya(^v25m;*XcL3ZnWb{bhP~8Tl>IC7SF0K4S{ZBlVsl6Tb2WQSrL+~E7*yD!CtUog)fD|_87`@3WUUGER zS@>{z_=>@I+02n!eSQmM{6566&#i4Zw$^OU zS@-Y_g)yh^3l5xI=dPw|L2v`MfAq%|o`R8R2XWBtdI3CHA2W6jvzoPMc(HGo2kgOQ zWyIf@%#{)%f`gjiM9Bhaj1l0Nb(}C&TlX!^KD2Z{S?|BbmLBT@sdvdf1~_DXpzSxF7zDIhVzT&aPb7WKV8 zfKjsgCOmk#NjYE(lwgB>Sd|_WSW4zgI2SgoP%VP`+eA%89KVYs1tvE(q4rPu# z++p8XUCi=^mKnfzc+}KS-TJ-6dGPuWbTw+(kkKHP=1xiwV!AP+pc>0}F^zO$Cw9k+ zCo*o`d(?kp9iKeSojur|zebmERc9L6Y z4<_c-gJrP75xviQ`#8f>bQG<9sm{liHEUPn0O=MKfekw4s~n-m{F z|76}3^YqF!{PAZhjIQSUTCPEOOvJE=X>)*q$n`T73d}5=peL1g2Vd&v1f|4q*?T%O z`9D4k>R6SW2VSLFhNu5I8MJdVWuqMXMh^qJF~A&2{ju^!eJYvVlDVqgW0|j%8wkI! zBM=S~J2!ruhc(%?>KAo0H#UPn44FjMEekGWi-N32G$8APALx08KBU-OJLWVf?wnAZ(hs}mZ-||OJlO}^ zaMYhlL)9{UP^vL6)y3=-wp939jygAjJKG+{?@nazQ{f7d)}AY%jD?uzOQAGF2z&ar z{DPJPDMAHFD-cL$x~53}l60;kd@D#}3Cq$;yX2*1X9fnFtF``hbs>+^qUhzZRwH7z z`_jercV~g7mzp~0GYoR;^Y@;*0>?oB&{unBsFbxLnLjmngfQ8;C(rGz^SH$pXr(R*)Z!Cv^RNV$Fs;s?XF8iZ~q_Zl0ju3kdARWzK_)+9xJxFMW4$8 z(LKZYWTnbadpBj<#{GsJjz}E~r+G%^L1jS&c(G41MfB@KFaITie&SQqQMeqt{KFj4ZPA2?z*p zi}GqLk4z_7z%12G$5(y!5-~&RVfcpC{N%RcMah>(IU+T*jP`rQH{tbc9)a07(nlisX$;@eOSCsbNxPAHp_0Dw0H#n&#IecszY8~^Sf?JvC zh<2Kxa2a^Kuoge*h=jxa#%xX!TnuU0ENi0ZtZ6nhNNHBQQ$+@NCjJOoRswuq!8)43Fax3&Zts3qi zKpb6|ii#9pI`d-mD)(H2xTACcs?v2~qOzfuOjAh+fqZ0_PVG+)Cthi&L1V~tFAjtc{9z2Q`Y&rlB^ml&{;z>KySIKzB_QYVsvyXDM(~1 zCc0ei-bs&eJb31q>&(pTE2_6}d4BHp_D#4Fm$I96H(yr%Gd%@4+eT+l+=2lD?KM>) z{i$dsWa^TmuA3LfhUU%h#(sERgm$_poYG{r^$qt^VpVV01%-mH-}lRQB&Ec~j#)Qg z;-W(nzgd~L1yuv|-cPh|tjGVM8-GHgww&Lw+2n{)FII2zw~nBweK=?Je0?I$b^M$x zFv~s>8U+guiC$_iA9@HdwuLN3eP4J$V>ms#XiNSWF4g+6g)MO-5<_gkFeF0SsFt*S z@7XUn*z@SI@Tj8W+42YQlX`RG;_0);3EgFT;d*oV2NFbQ4}7yE53;!}o-2;qXFt+g z3NNwYYp6Zhg0{dTn<)q;*xJ-zmUscKhsGHmW-Wu^?Bwq-zfat_nSH4oSX1|p;K7Q0 zal)(3zkE%7OrX|8DkNA}|K41kj z@+>3ZMBcTk#`{se?EP8+A*tv@`(jz?CHA-WmD=8Ca{N7;8ai}#g^rFW`xRddKQn%f zyLC_Q?c8&EK)x(Z6!BlW;$!i7E^>wE%9KD0#*1EDi|5m&&r64wTw3rS*HM)67lxH~ zA4ETfmyWN=<*ERGF-ESNRQ7PyJtP9Id|5DQm%qWlA2O(aRP4fO+pRD*R{gFOoY%qwXZk8ROQre=3d_Piy}8Iq z1TvZ{jDh@cA$l0jz?atWovMraIijGJevnq=&pJtxUtqKM0HKHFo5V*pzP zZ&>B6In-=aVnVs}a=;wLIM5>u{XdDOO|9m@zcIf5Yy?XEm!)0wpXm(HUH#|6{OA9#b=`s7 zuK^zEX!RPaM^W70VgLWVJY6`?Rk%DkNDPDq|5;c55g7dEQ$O&LDo_gkXJrD%QP6Gv zA>?Yu{ADEn_vO#$zq|eS<$vKiL{yl8WhI(npnpN$Hn*11Ye3nnh8MZM7%u_@xnC;d zaa;^vp|idb2j4bVubE}*fC!kA4$SC<%dRnkcIJNZC2>6f#%Hu$P0FM1DgS-82mSv)G-6iDRx zts{#Md|nSqNsyE{D9N(5aY)!lV!rSSks2L*3Z2*9$}jT@j+=~X9^do6{XWytzFd;Q za<&!{!14fr>H2n2u>QxH{Uz*Ob*!Q66Hi;fA=Pq zMitJF8tGUWv!=j`C@9j+en#c|BB#``OF~hSYrAYo$~YrQOpSiW(vhbYcJqx!h^@2d0&}XnoIfe+f#_&cnM#Xa*V`i#xG_8v^2;W#jHo3oiQTjHPi532 z?l!bDdWeD@}I^Ib{H{d`jq3#piMIjz7fr^E=F=NR3Sc6-R@B+|W4hQ{BNZ6k2rX!OdHMynSIezr> zDN<%(jfWH7>h|my>NH@jc}RE|1o%50tM5|oN?v?!P6fUkGV47)AMu~>^lXlzmiaX? z1Bu5@9A9b8qWt7qg01Gy-seVA5a?Lb*J)F0jLuiiBi^S9$YEw`7qPyNs#O1#Zai!L z`rIY{Qle|ccL7kmnwl`=&RE7d)#k_N1-&r!$lo#jG|__WZB1?M*$7gOcQUiA~jfnQfBfLcWC#<5NiR~QfXtLbo0 zp;U@fZ(orT=~s0~@*uj0GZ)&J)@A>a5@7C39LbZloq&Y$OYKL9qx$t#u0PS*pKh)i zx14!lgByeq4k}dqo}Z?3aYE!B$yvP@9H|q=yrVfdtB{;%bNSe(~Wz;qdT=13wduD6>lfVEkrQ+P@hT@lD8=rBt>w(7rrSevj@4R}Q#qM_w> zF{Zkzohjm=30tAsa6K!W6l%EkA>8#|*T(J6Q7zQmx1@56G7Muy4eU8$$vk6NB3#+5X zZ9m=Kwb&<#4Qb1Y2RIv!pOlML4W zqUSL46I*t+w61(iQ(uci+3)016&=8RzNI5M%XhG@g zBH25N)ISZZN=+rn4ws6hNY2OVgg(wL{;%HYND(Fkjc(s?73#4v(Ji^hr{*N|n%9x- zB1=$n5B9Q46D&>M7eZQRx!*J8#pzUL;iko@f)I9f@FY@lUg4rSL`_<-#IXsB3q>yQ zT&^{Qz7m!0zMzkHBDy^DzDPn;g7bq9GhJ|Dp*)t;jGC)5Z2D*oefSV*B-oBqvJ!G~ z<}I~qd&Q7&NQS;-fmcjNojd6epW0(mPk*TUt0-IkpayN{R#A%GxE~i;T$RgMQyKW`;&^$62KGj*lsIUW%Q*@dAs)&) z+LqDh7EOoWNl14*yz%ju(He}rV0M_g?|t;{^;_&spo`>{JLr;c{1Sy>>mb96ZLO(j z%YF0MzC;zS0O^X!vtEYKft;}!b##xbjCsXk)CU&AJJN3b2M4*o>!x&xs@D- zs2Ki`eJKS$G%DJ={Rre-s~0IN)MSJl>(CQ2JMtxn9|9C#d1Gr*N{it?Qqwu%^H^s$ z@|TLC7YI}Wn>qyEIZot(>l^%W8^c>V)EVNEV2g`ft27@Z+Wb1P>d+hI+Ar`Ybv7kv zsQy2J*|a;hwBFaQGZh@Rgm>GJ-&?)Zlx2~*+l8J!({rCiJK|$0C}&R&_Hz<1RQ;f~ zq0F8r-Ph98Rd=eVYDmkQB7#}XzOg;+rLT9~^F;3~R@SOQ1QC~z15bQN@vc(m=gjJ7 z;1T!Q_h`I~4^I&oC+qZ53`kmPCwO+1zlph9ladb`is7T{Wi+#qG^%%U#}<=3Z9)ap1t@uw;a_NoD_HT%|r`PHKdeYtihsGMaB- zS%$e6wfx7h=AVKS+9KW*;hSYoccD$Bp&c7m>u_2( zwQ2;t!Lkf3u4Ff?U@)2M3KPMO*!K(SK&JgDw!3<=dzQ(3crvOk1)a>3G&x<-+;^Bz z`6d&P|8d|htkwrU)P#9$E=X^NUpOZ=859(D@5jE_^3{1-QBhUWZzW`SrvN=b3ldH& z1f%UTsh|;hZ$=Km0W>?T5khKNfn@or2H?V2w{bBN>fJJ zvR|9QHKy0%!RI*j?M69Io!k(qpBZ(H56d2JMP&lkb}P@SlFY|>MP@fOo@JTekLMup zjurUGk!*?5evIPpX@x*L^5Hr7duEZu@NJE%ix&@GLgUwq>E_itbE2gj96p#kG*uFD z3qE8xdceM_6G-;+H9=@bwTDs%0v2_sx&Xl)10+cb& zu{fjh7wU1P!h&6uR8eeDPWL@(bcD-NMp5bL4nGcGO(z{^UIBrSaJ}h|Rm@qG&2Sce zFH%cET#sbrAv?C}6M5y7v}_K&j)A@WV&2d46VrQ@y;q+*tdd8n^~#B&V_nWXAhbqG zh%vYvV^dhI9coTwU$i7el$k!=M0Tx3xo}@9QCtqdn6w&x{;){R&6>oJEmraQW#+n9 zhs%#rK|IeT9Hs{lr}MuR8yD%%o^v|8!-63}n-Sf5M&aCU{L!N+)>m~pJ#QEu5EvTL z6JENQ4))Rkwb8n`)}l4gL>aT(BhIg(h4SljpN7^Yn}`@_Uj(`upB8N?qg*qLkywbq zf{tO7mQnEh?hj9I1{fl^#0koneWy!QJ?bG+mD3mogec5sTLBHLt3w z_UnELd$X(c`Dv0y3GipsfWj)=wL0~O9j6R7yPAHhGCqyD&8pVxJiQA0b-1FHdCcWw ztXB|1ZaY7|n!lqeFqFGE>~=D0vI#KPbW3CUz+cE~O<0!b+c~}dw2#XQmPC<%@nBBn zkCW`j2tt{9xjN8VXr_rRIFV)=)e91Jk6-5hE313({^imnQGhWii{KU+rF5t>8>g7z zHhhYSg%H_BW&j&&l(b@Yy_I>{lv&7~VoZNwh|5W(qMwyCc7A(CCH?Tk@-RP&>-Mj# zF5&?4@lLeiR{{8GRQ+H|4=F0ahH!0bK43h$N)S>oJlZ$>6vcJSB|d}I>=ks@626Zg z@JL6VdH^l|E}1{*&XVGnCnO_rr|hNbJ0X48!auexjeLNOq#E7xpJ-7caV1Gvpq~PkMVkk^{H1Fikkvh5>J@*x(X@ZM25nXu zV~@9APX=UlEfyMH47I(6KXHFr$!DJZqTOvtAE)u?6j*^A*mOh7y;}zIec&Ee=Ml)* zLvh0vD1PzVA|CJ;qe00|!4^#?jtyRaz9vAsq#BO>hK%tN8TPWWho!F^67>oZtgx`1 zH{%k2@tFTa;df2w$sEY@(@bDxNDmg7nQCJ{m$JvyI{gW4`cn0G=@+LeMYB%W*6V|gcY%DvCZ+b9$K74YDW1O(r`AvMV+nv?9Ebda0Rn8zKlI%;90{7OvXOxy7! zoaTxqDD@J;`&$ZPRQl@ueK~Rj^P@8-BX4Z*R}L(ga3kxb?i>k@at7fE`IK>%m;u0S z9gnSrIi$sECpYl*7RPJc9Ed#m<|Bl2ev#*+Xi98Xwv52UVx! zNV76L@gtmR#d>JTk}hv^xykeG!5mp!1YnGGWWsuB2^a!-;`xt$KlM}Qf~L{8!KA;& z-P}tJHuFl_3maV>fh`O(e{;5yLi1xvH$ zN0OSY?yB^o&tsf0sq9l%cj||)rsMirT_&LdWqqya4~F&q@?z-k%QsXhivh0=4(KCf zp}{royC>|E2-3WcaK{%qAjnHqi_sCbV}xgMX+vXDSVO}EI~pS&oW_Ntc2c7<%`ru2 z=R0}tlzHizH7AEfNjJmEOJkwT7Bp7K@{l=5HIP=NhWQ=x+~O6WGQwWHj*R5n)vP~O zjLOCH>1mIsvc595K>E9nD@P4PXVA(o@ZJMFe!ZGR5%{iz(AZO%7)*3&+%2I#wB_2M z*j#{_Qz(TaDoW5J;m0? z8knP*-U9fB`25pcp9tWoS@uf@H%7fwwRO&1s$zPNjS3<-9N#N+m~E`vMs78i{%lJa zYl@qQgy~Q!cWUCShdSS=|JB-!0kn310b08f@FZ9D@Mb>+`Q{{g?)WQQ*{V%VbD?W3 zAIkZkxmYYAGdfR~8=^l@Rp2}i4ab{a%SV1(spH;Po-84q^gu_o*Qz~fL2*qh>!buy?ufousLVf8drH)=vbNR3+A^eRb5z zuFwND7>CeD5_iQ99gnNx3gC^4cYCIMr8&H}Mla0#V(8o*v|o_o_T2`N(2_RextFy4 zM49GF>{OoRqy20Y9my(7e$~7JOBgi;f{cbD$EcuSFb%numnGAV+;00Sa78O!y|ozKItW2xDyaKC z?XUQCh1w6bu5&s`p#!;S*qD)G6R6%=sJ{ULrUGfv>nnLfyx3&%2nHm}qnMx153VlG z5~p!uA1C`K~ZsCLk~Be1Sc^Vn4if!tUwMZojQt^Aen6 zTgD1^yC~TbBtIn1lzKu=Tmj+GX+0({=Ow$<0h?P6D@A$l&O_6r_53X&FNpwSK{ zG1>@|Onb>FL-9Jm{+HPsZnKx_2a>IR$0i@xVZ|tTBq_n9N5n8nP4~iODUjNWV{wMf zA|K)LOv{3yb-!*)+N1}^$0|jC=;`Y~e!i^bJB8~*6S~3&=z6p{?I1YCuaGm!r_RHa zjye!AbJ+cTz6p%cg7uSlAq-(48?tsIS%#sYFqdk`M=~|eI<>IGm)Z=6y8R>^7`^5LFNmw2; z3y*le1()^lU0q97kqvK#puYv1+hVFcjWW|RjX-6g?#gQ|cWWTSSg%?I^!<(e?lR`!ekMn96?ph5pl+t+$ z4=0t?wf(Wl{#0uqmU3-6drbSh(6Ieca3r zyaV&Cq_H;+w~U5ZNKojTFx$T##kV?++cRW!&fb^MsLOmPGTF*;GeDZ4qchvy^g&@t z%a4kl&x);Mg%ww%IA9?Oc~YeKR`^heezIM$D!9TQ$38u5qiun@c>ZgZTTuv}sG<{S z&{R;c__E>WJdQNsMOkzY8u}zbPq4Aq+_q0EX(2n>S?I8Df0_OCbSCgO(S|$3vf-|) zQLq03GdhswPC}kKKToFn8w$wt;_jyFtlxD9)Of=bsIPHd1M<8jv>82fJZ@@jymbM4 zr~F;gB^(MnY2m;8Whj`LV;9=_!peA9#%0rm%kv|bMIf@J>|gwv)*k#}W}eiBYqL7O zitigawRbgULqwU{k4VN|V*jz#>}|+A>^?uY$JdDqnZDy<%<8(8P9o0L}kQ)x4?f65_gXekt`jO$-gv0Aii|e@FXA%UWw*x z9rjz=$tzwLEPx(RDpuhfH|dr{RqXt_58hFO{UaXKkF}TulpD6bL!7JjtnWf0GeYMD~-A?%It!@4L&y1dLQhmn9z*B%5+nvm7qcHB+gZmc znpe_8oGR5APcLqEUJEad$}K7$h4xhYN|`$epET*X_r6DJ1R3sGUAD;U0sX4-XTwyNPdlQe(qWx`a3TJ?g;`_WJ1Uj|Nd3u+0{e&3+9M9|x7qkdXXw*0H-W^!ojm8m&e`0CToO8p=GDh`>u{SeBP9vAaxiRK z9;QDUeki|(1w>UfG;)E0v^JT`(<8Y(pUPh*RK%xjX#{vl{Y8IukUGhr%e`)#^!4ad z>*s%}?m6kYMA`6n)uS9R$00(rNeUVRTRl#&q}+lc-g>vD_2^8cxJMz+OnA)HOU-Q_ z)5nw%cDM*dWF$OD~~9__yX@^3=kXN;F5V7v9s^JkcL9YS1`RQr{ft{fxCsBezE zf#A>|jt?10W9tSq=Ye(@qG80u1wItvF}f8lc2##e?4P`ZmWQy^F^=}K*I#}tp7vH7 zk|t^qQdlMwn`X%+wFMTGAJA!X!}aM3PwlmFMC6wFWoxPM+gj&U%>>}$4<8eWIUKzl zUW!%xMP0IIj_ykV8C|SZl|b16d+ndWpx6}lb=YstTwF_RuZ4}=Y`yd>cIO9duBa&x z^-*)Odb}oRki=8Z-O^c~1j@|Nlv21P1yc-+Y?j$df5T5Rvygvk6BEQs8afX7g3HhE zhl?umJqgb=H=$sP@XfUt8d5v64&iA!;Ue-k$J0o}L}!S2sac4$TsJDaP#c_Cu$Jyu zo9eieBfZmxLs34p!A1Yo*YaD+#+=}}3neKuh8a*R49w+THa>07#h&++tKIWJ5No5M zM5|g5TB)In6k^89`NkQ{`+wl2Sp}~s;y2=dEM^VEtHu?yeHfM3_%r&VMg_4D-%{Hq0OXbK5j#5d3C_~R&48QUhFsyN4tM2Nlv`!!N;yT4qg+IbbpP2xH6z(zG!nH=RbZF z94c!t#e@|Zi)g|?-G8oE#`@ePYjn{=<}!COqA%j?O{#*$s( zezi1uj#cT_8J!pZ!~K!jnp(q>St2>lt-)8LH9kd9PIK{I1{;1hrrHxmBW$zcBz1nL z>$*6olgFTyDN(+qN2(_Po@^{5QdMyrkBCCI>Rd4w{kl3XQC{CJEK-f4ruoqucqWEGTS_ zyQEdRuN`8%YLzmy1wfvsny{+ix%1swl+M~rw!wD62wT01IPAL+WB>duH!KH~uRBf$yd2jf$sGH!WN-Q<^)^ z@cS?F;MKjJjGCwImeSE;GPCmvu2Z{73tOWB`3N&Vu_D*9n7gPkLEAsy)B2JYmo|FS zt?}&>qR|qhG;0xtJG3fRO`lp*WGy)y+3)Bh3BORDC=pxg{XUi?2X3}dux-=JIe8-# zlhc-^!M}XSBw1aYXhh#Z|uT;^4}HGrDcWlZ{rdgdCoAM+hGux-#sv?olx?S zqdf^P-8v_v#A>5A>#xN&@OIn_(=oLBgfHsRiyk>#%fMnc^L}WKzcw0e|9bH9{7H7t z3wbZ^IXTQ={^VX}9k#?LoJ%%9a3()sg}g%XMu@ z40NbQqkMoedKR@a9xvgYvf%vUeXY5f1c-jgZ_`x9sGQ~!E< z#}iWOvr$1G)>Cx7uQ6MO9K%8e;(hs@cOZ7Rkx}uCDa8>8=687U}^<&_d8l=*9NR%|Eq^Ygx4l&6ev3KJZ8qM$Vb`A1oKJV;C-nyl zOB{xdS}e?XyQ_F~%L5j($kwr&Lv|I6ZuHEJHd5<(i&c~J6IJ$S);K;F+b~;!8}6eE zyb$j@RB_r_RQ0OV^b2+s75r}uHC4w9c^Iq)4~^;XJUa{K`13%8v(fuREUrWE4t-p6 zUgrDAX>B{sn30;VRTdYEyuG<^a(Gv{PG5@(FF$&&dUQmh*JG|~m2RAsaPckGy_J(9 zX{mmvrA%;#LXH2}M`~4i;iZP+N}!sd(%Se_#{w3@&RR{sfv&6bz`Sy)9bY8wzVie~ zY%JbhgLBP1N#oMJtlPaSvgKJnu9)wv`yhrR)ROV*{A?POwDfv?`316bx3`+z_3o{B zb!lzHY# zli9~ygSCbfFM=qgRRFS&-D`u@slK>DNHY&v6Dwt;=_bWAzT zU;G8w>nD7}Fb70CD!lyapuX0SKOp&Bt_#=-HJ-bm%0ZC<8PN(P(^%q6GcV4Kk4!Qw zB9Zt4f8$PzcLMFuHpmmwx~9{1ripYZ;+cka}U4b3Fk1yM!#x3;D;#M z&8N?iUteAG?;#aKI~C$DXC8kSX;PR14*_*^QK#B|{YOMF+XL5xLhny%@6y9Iobx2? z7&LnnS8Dc~_}P07F2mA1f^gX%fslbCyBUlcNE9>IR;oMhg3*Ny!pD}%muTR1X$&rQwl zrs+|7`eX}G3E5yy4w$7-hPDy^j?{hT8ifJZ?_k^Jn>gcRd{A{{hvixxw?cQpV-AjR_W?jlk12!4T2LcTZuM37p0Dpb#5Y+zqMK;Bs^|OG+5qwp|ufvBo`}KM9|Qu zc^n$!`0av<3C)s7r)KzE*10Jnw63N|9aMvG z;VLu{+h|5!1Hm>T@}(uXM_@mB6{qyJF ztN)|GIUC78!-D_1&p#V&7*n$U%W?i=2YbJNMb-c3k%IAL{((6D~j9+ zhE3RidENfI+<#mFNAf>T-T%kQ8@t}vxV#36d)8gK%C#(r{;EWekM1RKc$%j37HA?=W9jAoj|%{*_P??A|J3L|?$W>Se=icn z|LX(X;b0F{gFG0Pg}%2 zZeF&Yip=Q4QOMMCbga%OF!g*-KuBIb4)Q)(n)9$!vQedtdH(H7p7()cLDVN>zVvzmCl zJ-{>kDm~QS?Cqzox#baRv1I04tEk-U+bhW|MgR8(WmT&0(lZKuqc|w0E#fZo@F@?W zJbL1;&mV}6)eGW+LfjO5@@`f5da6*M4&xi`HigsqSo+>ReDEx)N=jPCYj3wVHoJh{ zrGoGT9f8}NPwDGW9-ew;npB_tJ`4?Rx|7Swc24*&hH!4=4#g*1R;UGE`HN|eNLgO5 z(nyZEST$`_HNEvV9ONl%ds{&cjRh0)dj~I&*k*)DX~B6V#9c$g&})Cko#DvZZ-+El z@JYdNjMRSI;J3CFQ;qG*fM>bx9y5x$)5&+4x$IZ;EK@|}_ceM|kI!Td7TWDumJ-YA z;>I>;f#D<;t9m}x=#=HqI-k-VLBXWvfK+8U3x|j{FaEE>%he(BTeAiSk5K%Y-w z4*57^YnKg1kXA3QS}6ScE>$KJt&j^FN_Vu>S$n`oQ_7)m1H<3i1NqhIT|Bxr~|Ah^WzhUhil{TF$Ou&C^+cQbp-HqmMoebuptlABHJ8@yg20Vj_ib2NG&zgYKfrI?yiwczrBVSI3q+SwTfVbI&zz>7_RF`bHNNFl&2_CKDqp?BMAM$P_26Ya-_KEV~1C&7vjYgOSGXb2Z zp0R4|y#Bhc0q)DpR1})GCn>_-$YzKEO2`5O<68DtJ4A#`9r+Ilj|Sdo!)s<@N$0F# zCT<;JLIr%1H-_A}7eu|at*CQ+NA9P@5Ez%fXhClqZj}mBFF7=XIgH7DrPnC3TC(j` zhy;rF|DR;t9)&d@fcWsgilOV#L22;3f<1ndnB*qmY2(*$0{@q574r}19hfJ>SpAtP zIF3jaT3Yn_oz8_v=m`C4g5|e9qx?e;6Gym!(eE=L@v01q9?eYgP}`k>PK?yWZQhZ@ zHAOyPv_=vPN0_y&so9YsT#Txp$XNwJ zl`p(=eTr2mQ|48dB#b3-5)yN#@!4yv6>a-Jcu8d?Pz^*WRWtox?Y(DMQ`y%yj5;Hu z2+WKKgCGndT|jyV1qG2NU8F=sg7gxKAwaMIBT}RjT11-k-kXSklqB@fkrF~n0t6C5 zfILU%KllCN`9JS{-S720*YkdO_lI2P?6b4a+WV}v_FBKa_TIUHmR~FJ7DGbd>fA!& z`3LfZ4MQ&LaXL=1xE^0sQv6$FZhUdzd6kuP%bdL%|HJ}>Eu#CdLu~&a2+zd|@wMtI zsR;8$E0!ri?2JMeJUDbeQh{Mqkw{Gj4qxduS%Zw_4CMZiQr%^y*G{VW<&C|*X&E9I zPU_{jBBtV$#P#%dt_gg|>_=twn>v|ZwjU%!| zhpDeA4}nd0f6>16GrUGUI$!$zqvyrZ>HVj71IK-qk`k?YO?S=PwKm+|eE{9lPrSHE z@$)k^fR=R9mF{r2GcO+ICrt)RP)p8w`h0kAGIz7@F=!%e7l^G2t|O8rzx)~D!Q4gZ zh-VEl+H#Yh+4v~}vH(eLm-VOq{Kxuqg`rHm{q))ua*=p$YkhnIPDU{HT#Vr@EjWu1 z?&s@&9NbX7sq^_m^0gypZcBq&Q?#f9H4b9h|a3G^Gf-_G;4EG8l(oY&j%&&9~9*Y5=md-bCk=G^(?G0LSdvAV`# zjFIrw2d@NhkNpbIL;VZ6>FbK=uHsdb%m}93EcEP`bIPup-p_9$o<4Or9bAiZqCA-K zo-TV(T&gS$Iv#m}tXd?Vvnv%V_&gom)}Ch1EdBdq`48-TC#t2l|3JVRGP+>>9~+b$ z#pVnWg%M4GZaD*Y%q?XurRehFrAJPLMAQ3%LHwbX^b2q90sDzcZLm*2Bb^>M)w^RO z#2O$Xm6~D9;n^k!gc*BJ>FYRgrxdXGIdFUbX(E?WEqlFp+B;;iCs+HOV#G5-42r_@ zsQur1|n_a-v3p!J;TeXNs(nu zkNo;jeV{}C?5T%-FM6l;F#1nhp9w7uJEu{PPa5PGdz>emJO5gaF;uivI|jx7uAcle z56c4%Gks%Y0V63cijK&$4u~;^={vtN7KXn}8uZ@6Yb7}f~Za{a4{=tc>TICZVsXIM+oPDX_*}2`oZQ=WG%S&SM zc#-&>x2jjIR>`-iH$ApNF7z9b$h>P+N30U*WBX6@bz`5S-%a*@#W2Lu4|!A4_*PO` z6KyC1`)ABX*c?b9;f7HHX3IU$Lv0M{)d#+1klRcpYi8bvky;j1ERnW+N^V}}*PjU- zSJDT#?&n&+V{8O1m1lNd{3y6v%*6tDRZBB>FLJ+f812M^F8p=Zf-8ou_3hpAX*M_R zOTEqJj&U-am&JwIgEnpA(ne);6w@V(h$^%T^a+#|Sw=;7=2Mu~|h>A$`4>EoGC zZpw`{(HQGce*2M|d#j55u8)@cg^iMy8Ik3~hD&*cQY%@~_{w5TNGFJQoOhBTgGVbt zDnCch$-NKlbs6sM`GAdJqfOaeVm~W-JI@o>0Dnf-ysf_j*QQqDnp|XdzS&tyq=h=G zyGhKHhlX|E-zQ3%P%=BJrANOcD7il)jrf-E<4g%otrW1_PRWFv^u*UC=hWXH)*uL+ zpiIrDAOKhy@BN)igWCw%cg-=XD{>uvxL4+Ge@TColApF((e*+7o+pyPoHXP2ngNv? z;7=RCumwjlTRA-%iXP(%?paUcW@lC^aEyMt8gK39?ZEv5^Sp?!bjR_k7F#=}|EBN0 z>nd)%Pf^$(>E%5dik|b2aAcb7PWGUe^K^I1_uGj2C7sl~dKzU!f?UDnK9 zcJxbE$P;a!Lb9Anvyf;qvwAdgjM2IC&QV)1nR&=C`YQTY{oCG*L$){2B&R-c<|ue? zH-kuiTUqo)k14|WqdQ>etKEa1jD#k32C-{$cU`$vi}b`vpJC<(aMR($#TrF)=(!E3 zgryZ%^Q35Pl=i{g=&-+nBV$Yn@qtD4kWcWYz)FDRErFc1g4ZL^KBF&3>h(^TTePLC zx2Er2{rp+xkJ_Kab$wCu=WG>(fTbMv$KULMaE^01ifj|`m*;=d9lWVkp7>WjF0)dt zN+E4(B5=_0HA%}dKTy_+>8B9uNL}s|Ro*}N^l!hy58flX&w(YHD|5Ra1HTw zc`hYCCfy$K#lC*mv@2{UgNb#RTKK!qNg%H~A?vRXhJBBEeMHKPz`LVeE;VT`d(LVL zM1uKNOcH)j;HyJlrxrbau{*9_>EjJSlh)SkgLrm}qMogOM>Gx;f63^aWiIq3D%n2U zXg39RZPCKszP;I^4sxe}6_xD`FNTpg%eqXB*1v}=%3F*%zteI$z1Z-nJ06Y2Eq&?8 zr~;WSTF5!K6y%PIdr#BdP+4(1_wHTa0HO*9`rYnwT~>ba#rbVodEM??5l~@g!ERx#MHFCq&6wY3Utg_DY-hMgu4Ya0*C>R_rr-2tZBW(se=^n7 zgok6=+cJJymRl2X{Y1Qc=>Hfv*HkYj|G@sb$jtqWKyw+jEK6YH%h+J;*}=X>S7iQ& zb_6AiC269NoY_Hd4QN(=T-38@YMN23;-{H4MK#n_nU8L(W`a={2oiZu>f?V7eD=$Q z308=4_}8hTH7!P1U3jv*{b_nj3nGMc56mM!hJ{6xQ`>=MWDoX5mov^oPtqDZPkf8h zm5|Z}Iejsq^EK*Jzf(q>x2*-%kmK^W=i&w!n)J!`~z3#C7 z@+A1<--uNE_)Ev)6eB1nfBjuvYo6dmk)?f z+OB7+aTAn3AzewnKJKK=t*YlF5j(-Uvfuibe|nkpr?yu{nD8HCkQn()5;@3P5Eh^q z#Aqw|wy}(qtD!i|jcf6j-!+#NeGX@->9Tv^tGGAIPxnU;@BO^i6i7X4N>_T@AB=Y= z9@8t5v~SN|-y^zC5KFp3&+(E%KD8=^8ZISW-9pOkP2S!VZD2pgSI&UO5Ys;j2*9+2 z1_Vt_IICNWp`VoBMCCTOz-z0>D`BZEw#xS(>b^4S-pNV+*+!81v(u5^@41$j6$w30 zAe~cB*tFEogdjV(T@63BxxLT+`G_3#C zLjI|uOmX;zRPt8LX5YzR&&8i(Y6BUMzpn<{)HijVv=|+$`)lK$ zN6e74kjO|K{*O1LIE?-}DG-+~2GnC7mn6R_kwqfYU|>ez)b-c`24Wl~X1k zs`x(cHBI(~#n}Fgu(5`H&&F%^VI#SnPqTlhPgtoQztIbxDYRINnMH!HE|v1z+-`N| z%^|Q#J&lV0sVp%HlrlV2{IVpitH`!dS~Dq^V%4aWpx@9-{?yTmnsr1x<%<$JEs~Ec z4$Xx8MG0+!&(LS!$j;r}dz2W;A%IXWxf#el18|N%l03nxVge!`D}zU1iGYccej_~A zfNRH(?W8$&VBX>tldGPHXsG5|`%(|ihmIaU6*eLlbAy8r&3*g*4KdZI?N6E&=D@15 zC4+OD&vMn}+86Gf61tu+WqRv<%+*6!NGPNTp<{IAk5J{T?Q?)GxST8PaS>U_zc zD!t{SBDQB<-k)m+l$LmLA^h^pyD#dl!7dMM<2d#fI%-N@1(@hzoR?^_zu@ZcpANBY z-Fm)|@wtB7O3(8Ho6tA4UvCVmoZ*mvC^hSvkV zdlQ^|$Y)OUiR$cI(E+wXF!3^FVu(nq zin9z4ZVOIXdbF4Q3Pc7^$ed8_pKftnmAF#dVs~u4FAGG}{ZB!-olCbaMdqhPrGqoe ztO9Tzk1#aRRUti!iI(PNB{5;XvRj)PH+34%s&m=TM?Nafc#CXW-)>MZPAm`dc_xUn_)MHDcwdcNp%P0(eeX2XQ8A$~rG zRkU%o2zhKLnjXeB-LcjFwQ7^8no71Fr*SIN}QhtIrQ=6-WsWad( zDxb5HJ&EN=?o8Lx)mNftM57CI9@rV)r0BZn=`Qxvd+tBa$z4a6w`8U3$r+er%g%rU z0$f_m21fiMUBJzWUkuk@KNL-~yYQ3r1qbh6GH*$_fzwpr7p{ta_O^-rNU)y}7EMQiNeYm$bT6I83_VrS}Bz6JCkz66K^(&g^A|R+v7? zg=Tg3eGQ2Qj6>Qw=t}k-IXCji#-Y4o1*(5sKUG_X3g6j|KIC6OwQ?lX?TeMWeCP*|5#Uw*Zx$ z=+fYTgk_VGe`6-4eomQZr1Qvb9#-Ic$q~3EF%~&Gnmh_Q85}DsHmQOR^e)zwY}Clk zCEUTTtjI*-A%O~c_wQcb;89v1Gb_yiDw|IU@4}$D;^zFK3W|sIa3LCCJWcz^#O@Ur z8+LtGDUxiJ(_y=uouiSMtrLNp(>a@8O^m-@Jaa;=!nRY;M~fGsRDAn9W%|rFk2^Qd zgkd)|MM-J_?op${_!DQP!$(!+nyeC0>(gEx7XHxXZY4LP8ENO{*58#y-Th>sB^&0j zO{fY%&U$r92h*?8>#>(p(Rf2COHuxnTd`at_`Y$w>#8vSQI!8EWXjpNKb2TMcn+k# zIbPq1@)*tJh>XUhb@e#k#;Y{JR&_%(>zyy0A4hCYk;yr;$rV|$vR#oEP<_^@!M8?U z@4!=cF^l{IDuwI=AKcJ`RgrA{+te(C8gi;O>zU(3t^L@XXsxN8J+k=;)?$BsTv0gO zVST3XUFmZf*6xVb$#)i%GFP5W?FtKun7eFuU%v~NQ?QZ$%t0G<&@dz*S7d8eA-unQ zv$xqz99C-N0}Z)c!iM3-8{NZ250KxS(~%24_>HzvyI*!aqhUJ}rQfyNJZ9V?dNHyB zj!v()sztW61q>`_*B)=+Kpn}_d`EZN51Zcl;@xbIJWEwFGI16$X-R_AkgI{Sb?X&Q zRzR?k8;p>;vCp;KOjJ`2Ne3Ac7JiavdemTDI`Dy;-Px2J%55=+W?5DXlG-~vQ64NK zE!C`nZ}%F~9k6sh3a2{ZdO*8v^-ff_Or&QYC+617sA4dzjvDMem(GE8!tccp{(>xJ zl{E=SOMx=9D;(Anu4S=^S#VDG6kgfqpr~9$YLGw5YpN<~m^3OE5?@Pyx#!C_Kl|>m zd3vyU11aa4d;D~JTffQ0`S|d;ZM(&St48(&_!ParZCJ<8{cvbV#7m(48^L5m{3Q*Hhk ztx_xsy{j(`g!6as$bk-fwU92l3+~h7BA4te+fa2Cc2qW^2pghA9kX;EeC&b!V1>@k zWHHff^|+UPjtdMRq zFk10KCH29)o~0#5I>lyk-pBH~FxLLCY&Uc2=T&W2d-Jt<2?d=6+J=Qy_(m&yY4RyC z>1Yulxn&!!9mMuWQo2VLxE_M#2x2!zsRa@4f0Ng*_VkF_oWM2?hc?x>3$WrsMybim zPveUB*0aa33VA!>Q_$AO9nrz0Nk7C+;!zINf{n*MYw5$!6&Y48wyh_1z=G$Z67&MU z;Y>+rQ3#_wjriqHb%PJ!VGjti9T$8u!k@LA(2@PpFGf zhSr7t>`OczYt3~w_3A~%Me}Q1SMB%aw`V~vq4eJV^8coQsk)v?{0;F zu8=N4cE3d*jv1z<3A80@#FJyaujP$7G7G!s)$Fjisn@^CV5q!@Pt~9 z=8H!fDT^y4hg-xbi<1@#rPxsqGN0hf*GKG-(}+>IO+O0>o;?bOlGv{d)Ql2mjux2)$A;pi(V8~xhF zx+eWa%P^-tBzUEJ+@|7~fHS58KdnZr!kJ2;lxHmjUaRCqtxhjEx} z_&`GibaI5>MY$xa?6V;I(kK%TKcZi=h~QUC(1U-m4%Cv=TPrb>nZEOPR2=R6>FsF` z?K51=@p-!+F)L~N0twpVmC-y}vV=pT7obkA6cti|d!^Dc9@MDGQ)11&O$$!Jv)I)g z-L#5rT5T0HtBV0qDm*+lUsoA{%Dgb`LwssCh;EyyV5hiSMchO4SU2P%|9-_OtRwLCfMo0Hu zF~@_QAogHmM1l~VFWyH--2CI^WWinzSt{0C)(fh`iAqbX(*b0dRCF)L_kpJ0V9t8; zTKx5g?I&*}BEhrquSRbimk&;VoLx##fv$zOLl=COXL^p?dY!cy`H20=C60#O6YGzc zQ+|6TVg9icopuq8QPJ~9>2w^1q?FF@`ZOL4k|6lfrz;0lMN%AQa>O%5HWd2o&V3Iv z1x4q~Kya*Po_IJqe9TZ;L!@2AW^eK$5Bk)3I{Z)m%kk<#)Va)k!i0?V^`g zAgezp21EorD^h`8 zLE0ykQ-lx%bk0m?} zgtL*DI1t>_u{(2COM+WA&uw*%@wH?$!=3a8hZ}i@(Ja~RSl;JVIG;LGacJSX0UDf+ zfs0v`qK-S7CB8puBfHi->c7K1s%Aq=o$*~R-LUf>4MF(grG0ne!>`-!q~k*a?zO{F zdl!zU&mh{Kd6ug<-9L}=M;!9wS9=Fy`{N3*tMNQ1LNN_?NhHHEBN>3)^$)?$tCdIe zv2a94SCyF5M-RBfcu(_KfQlj({V;8PLF1}HdHW5mpjrWFXpm~le%n|g2#W*|r*?(W6So=&qRJ*%(dg8x*|5ys7! zV;d+A78zn((MA{!Gyh5FU6PVvUt105DwW}5F)<(dxIH*zCbK5IQxCV61JJ%O3ev(H zl7Yi#n^K+o4&2C}_&w3X66NS7P+N$OgelV_C2VzjXGn|m9_drlP|BDQ27Xk*LLNH7 z<0&?EHaFmpiH9ADiie-+G5u2~gqITK2;BlL1B!`2Kn2eixC0)`U%DW8!{fCl^+MY z26=XIaeGlzc2YQsA$$29=p;3G7K`wOdnlg;v@|g@fk!GkZ9~%2%vM7y*!_9??f^5H z&YzuK+}zCiI83R$@ZH8aJU){ujf!EMbPWcDhHSUp5W@!txd+v|b%^E_gP-eOm?MpY ztV9T^?4%K|r^f2XAm72iD3o?-@NgL(H-N?B(7_~Py$`{D25U)={cWr zf^^T%Mq6^z`gmkR9BiG#yny|Dc`DjGy5wVR$Ma>+L`Dw|9mDe7U^-7TI6FM-dF$Ok z&msvbXUjl-xUCt;z-(vJw^T^Yqv2UQIhXtA#h!k3vBs^6b)XcSV~zF!fsV2z4G^%I zNEu6gABqWN(TQ4c$U=3Zv@(-p1mafIKrO;Gm+M`&73-3*ohg)#fT>v-AHkT->hX2c z84rVH2fH-2^2Te_J$1G3uHTH-xI&zC@xr0%Op!0LlUwHL{j&jHc$wxVDf149=Lux^ zqg^oO^WX{2rs!}P*W|uC$iWgqKG839Hbi~R^g#jqf1=??K{5KzB zJH-OQrs_G(D}XR@{GNB~cTccRpMGolMrlH5Y_{74*$D)~*Yko2=zY?-ttuOr8)U#* zGkG&Aad$KlygK8U{Z$FmcCBK??))pQ1g|4sFcuE&{@CAGK^=enX7+w=C8@!ek)B`0UX{qMeilQt zt-Y{QFPF7R9pl~KuU~(^I+yAdU5<(YX)0kgnulX&96NXemO56(1S7rDJj+6&5P_Sx zCVzsSFS&I3dfMTBz2i{SI&GNIt`5-)RqH@D+~`m)5jU4^5e>X9DvV;akq>kkX;qFx z!>-0*?9%Fr24?R_*V88(ny_>?5Db1aWm9ytoJ=Fy*NKtZKoM*(XV>CG@TYh2+z&^U zLYUbG1a!`N93PzuYsjn)q0TiZ=s&cz9i5%cY;KEmZ-i86G#4l{Os-h*ejkCyg|k`FrJ7o-Avfy%U}uhwbY%Z zzX@Jq&N%s8Z(ol)dOK3gi-33@*qpeWKoNV%VlG&Yoh#nPgA(V9G%3Y~?eArkLU+w+ zom*FElwc(b6eiWiicQsPW09C8A;2Ip^~`6a2KBu+wn3NFAvLi=O71q)qUH zq=?6(nG`U5m7t->b+ls3eK48`ArjQY-ZRe>-B{VPgUwC39H8}80(Z45dIMCww>zP} zosK+nBh7Vi&x!YeF_T*aX?5yY0QPKU$$AIrlx-_q|GdqKudSgqGb@wga6U|?CN$K^ zGqZ~B;0Lvhx%J)1QxnxtD=eGCuW56l%ICz&3Ii#Sl&7;Td-oP;ExTQA#NW|OO!ae7 zQ%%dUQ``qM5I#P#yzdRYVcELviqA>6)upqs=oVmDAC2ko0x3J{L9cA3p_5g-pZaeQ z+cX$~fknKu=Cq?hf%QFyDN};L>b4neT15Tm*q1zNO z8`<4Wge?7qooY_o73X>rOsAuq=UvP5VtI#cyI9MB7zD#i4|2*qUTx`P4#!5ZqG6f{mTr-Q>;V4J` zrPMZKTFU{MK_oP?{Vu`ZsxHDGM9bJV4Bvtd0wVX5gelcS1>I$d+8@8AcZw3nP)O>sTU}p`hE+f=PbVs{e8v6Bq{^AjDM%F z-!J|zvnOL}YikQiXLI z^_)nhHgiM6PE1=V0A3OxD3KzGnh!afR~^WoCI7t;kmrQM70p6OTXW7g9S&zkVx^cL zHHHQagn17{vhBtpu3*7#n%~lQc zJH^!6r0TzsshtB;wYIqCK_Qq!Uh=_xJ{ILevNW60Q!Zv=KG@s@2~yn*eq3l<2a2VC&IL)O%2!yP=spD zd1c}f5(altwS%^XB}yF17gld}mN%mE>vJ9ZZOwCrH0VSs`p9Fd#^&%y&c(xJeJx?S zth7~K14c*=p~9TAZI52RJNx4;UIqZ?OvZ4WpQ{NV6w7mBiL$rCCnhF>t{B==l5i4aX6TlE zdz%}K3bmTSETWE{@!&j+P`5}*matweD$E{iZGr^!AJX?Xksb6FnIZi}Y{#|?_#LW13tZ$uxmGJG3<1_4P_OQEEr~$0a z)rw!%3Vu-BQr@4)Zj44LEhJBm>o@bK`4l2k9d+3a8^#S}m#wW=dSJ*|zZUVc;8`0G zdZ;5<5PL#IO|6=VUO4e9KgQkR_9h`{snYY=(@zpO%pm;=wE1}U6$r9P^{tf`JqX68 zrO0T+3N|@D^oUYxdG@0VC-0nl#xx|=K|^I_us)s*8d&jgzz*ef&F`LyxdvT{=Y~G% zeCbD=RiP6a;JUG@@)A6k&1_qqM`GLj)>&G>%8qRX$tK`_rH*%MTnhP5Dqi`1WfsP* z#RUtqVnP%~Q<4Qc$&YJ8drK1l`uxoexV zDnIvM(AafU$${91O8ko%c)ojwIu7gahwJWt{WuIqChxOO<>L8h$0T^Fj&8JJMRu}_ zhX-o+(f~PC&ly+QwU8lYf1WnSiv20BKlw`AIool-3K2otCA~OPGFky=j7>

    uFHv}^_dr=yTT+c8d*eT z-u#sV#s&hR6Z*-3*LYirYOBqg!f9hAuY-hx!7I6`#zzMY|D5pqu3KXA& zYg_`~4JyT~B~zclWM-YXf7%Tm@|kF}hdHS+v0Bk7oR|K7orc5@(QjqqSf`gQAy3?Ui!#rX0*0fSqi4J@0;ADSA^S&eg2Rdty2Y0hO%xTQrX)9a664p zjlVs#&Y9qpXv}yaQD((Bfni)HB`%Lazn)uuA4H#OIiqC2wWEc7gEMO09X$*i3iNrX*-6aA7`+N{Wj%0B4` zGzS9S9@4;vZ@C@1-t63netm%im{qcrf#Ols3-EvCP32jnrl`yXT0|%)KQp}(u~=t_ z{*GSKuK>91rB7v6(irO7zi`}o%yKmZ-nvg=dILPF(?=Jtsxapy=zE~t=r9)=EYN0r z8Xou{5NsJ9Fm0r~ex=DbA(YjniSlB_cS!q{Bvr>n{eu=BnzBh-NTYN``tQ4?zK>ha zBthW%BSE;yzF4PnGbT!_zl(c`aK4!h)w20vcZX;8C7i=sIcU&Ojcff7>AeGFst5>f zHv0-rg53B2V#$1*pJ)0F`uu%5p#1Ky#^yioWIpDvUS$LMw*QL@u=wfO_vt~Bea^q| z8Rs`2ryUx*{obmeTYglQGkHjWh_i8Ro-L&zvr*Ihn}5%Z;lmVXE~P^(i#4h8sfzFR z)(#Y6cs;U4yYiY{6_e0DGB`w(eVDw@UkF^yiEAE61U!-;PX&BadAbgCBqp-Y7#ZUm zNXBWN``MUkF#(j@FjWslJYAN}I;z5woiCJmfk$$_Siz=^lL#CnI`WHv>!132}sw{^!DdFhraY14olSbn=0C#c(>7j z_xCr(({JWoFIU=n!zJ-+r}l=mQ%LXWzAbio--~i;XWY4a8Y+GM>oj0ou#|1|nV$#t z-aPcbTLW!oN~I=s9C%Cy4f?n zrTX`r7`0-N>7dy%i&gVKGym@;OQZ;I%$U)})GZZk!p!OLJycc9_cz(fKU$J}ZxzP6 zgD6QyP#UlsdYtHnfG5fi`|UgwZ}J*-IyQT<9xs_i9CH4kc6%m7J{Ck-ASqz=4bz7G zK%L3TYNPb|SfTV?#`p)q$5=yhZn})AF4Tx^NBK)5rx8vVkpVklw+p&rW%PAV^G=%s zC_WsUkYSq(x~PBUz$u0a4vJ01JHXhfZN7g@$c4*$ z%ldaBHy(v?LHXMuD1sx~m2ZtIO%gx# zT(Mu4H=Y-Bku3Lnep`FDyFRQS{Fgt2pbFxsjhJKn~0r&FDZ-+k3 zdA#D&g~w%ZWg8|W*@d{n+ov0%jAVKk7#@<6W|Mk@UHV*9>i=kH+W^y&t0}7NT3Vxs zHe%|MVvv$H#8)tTcEdtsi-OYRE4aTpqMf}Bb^g7Ufo*W{%{kg*j)NUvcb;lJ%6@A* z>d>pX^P;ZVS8)>J!a!I(k>7;Jp+bd&g6J6#dwSs9ny%3oyoZ5Y;?PB;WNgMKK#;ib zH` zu*M1A+PB?3Sbyw;eU__z<9~Hi#(odYecCCW5j_5_r$~&d^G)CXl_tqQ5nf?+blGoO zLD>Pgr%pxB^!HAvD0`0HxP{&srb}J4uWL)$zw+tk3g!jYadzgZR2U*ah!_>_qb6^g z&M(3i)+kLtS%fIN_kI2A;hvfMBEQKYBH5>DP1=b&+a&>SRFA0Y+_^ z@(Vkg!AQ+K#j~NN3wmW7<&5QRlR*n;P3|dhF!AmQ`2FSyBf=Myyg5C2znuEqJxuujPGE@2`VH|xT=RHHPT4&PQ6=C8%W@{4a={)Uww9Jt@y~IkhFIx8mkoHX= z=iG5{ZMB3- z^W0Hm!f4bA%bj}?_&7xkek~JPv%v=79|`Er*7qZL9g6x+$2)&vub;&;nwO8=+4(;w z9aLOTb@D1l@9#7@_jm1vi?LC79mO_7&^yXVp1jAhbt|#3dPy^@FTW+g21uo z=n82GsXIOnS@m{;Q~&7&|7i5$=y_WxXNvaU)AXOs_P^gaGi4Nf1q6VF?Jf79@2;L@ke;cktY-#--Q{TrC z_vz^KAVU~Z7z-I!Z|2|nPoZ-8GhpnQClH|IUVofLX}lc@|G?!4*yi26S3|B`_A1jR z`G52Zb@YQ07HYesJH0^S?8SZ$JEe{PQ-#gvniEU#o)b&|H1z*In*R*3rE&(T)eTPl z4Qlu3O5vm(6V&+sHuV3JM;)K%s8(SH<@TU$2gC}TfkR4ZllpHPV6`deQd)kS?;BqZn`~LqD1=N#+SRP`FKzRMcgpbEF+=RL>Swv_eNZ$X? z1OY>J-T6O0mYh!akMOy7JLWIYK>hc)hjhohgveVFZ1|r3`Kd4_ zd`58LQ59>v?{tQ4B%vo+y6S>GM3(5}rhp`HWvy|`je$)3u})y>gIY%j;J>t0AGPY= z?6oGx1G%VKqjKV)d{#=~o`6%<2aaA0&^cfArAjpxt+~5h*CQS}P<#$pz1&rm`o>S2 z65UU~QVE6#U%&=M#e)JvU;)RfYAf74jv9x5Cv zp-b7{j_q`l_i78Zi(^lGw9=h&b!Wx8C%KXJ=SY!*_FmTCD{LxG-wpIxG%-iTcDf88 zp`1062VrdmRAcf0L=Cc7Bhe5Ak093`kr>IsJ3xlf~|w@MpNQWyn{&*J8|I zrdap1vtqd~*_B&160nvtW-o(v|M$SN<^v_=WDF>nLid)7>mAQCD3>)9IVm*%u1i?Q z5N9&ynGLBV4nQ+y7-LcPpwPDR`VGmx@D=;8-bfx{amUwV>7i+Z2a-AJ(Zy2YCc$HW zbCMa@Cz+WevBSHnQ&yMR+r7`?F9KBu^?K}i;oNOL#}Q(M!}C>{1T+cijc*)$%)|CB(TB!{UN z8}FRQ+OW}0Z+q6nbIyx6qRSFz631O*B2A)xmND9G;>KH&w6}UBAEL?8g`G0@=rb_L8(W|q=@8`<>ClH?rn7^bysi8-mzD`^>*sw#-xU5|FIxoX?vtNW4_KsCkw z2KW;@m0@H2;Z|j1Y1jT`5$DhM3FC=m-Q}@{E$ugNB??Df3jfU*+w0x5(~0zEY>!RE zH_uJHBR)0rzdZCc7yJfWsR5059jonF4^kAaJ)F@e(5|{~GO-4;nb70es56hqXG!Cr zqB~t=IL>(wF78B(9`ZNzF{)t32}3#s4|6H(uOSUH+F7$Ni)sJVRLky67Pm7N5fOp< zLwlrI(Tj4k#6NYN0*&ru@IQF^>YzBfCt3)B1P>tuw=5o<;O_43vbb+>Cpf_+_+pFO z26qXv1c%`6?(XpR`@MRvYX01sy;FCl`}R4f`w(1&u;!bIu@w~7HcK_vDe~#|{jlX8 z_<>T8t1?_85vq?!^+rqzXKIJX8EHoUO*~eUDuOK5`po))a+NJVc?*jw+#3U=UTuN5 z_>qfkz}!UPAXwtqKxBaJ*yA3HrWgi5BJq!@4`7DmZEt9vGU$#C2I%`>HX0EhNW21LBIS zk;;cje!<7zIM@fZstwApt4?ZPwhoV?F{ASG1U2pB^im(2Z!hsyEFK=@b)!pm6Wo4l zE>~x(XGo#~%rt3Kc!U8x_wXyeKE@)$wmqQ|=}}Rmf%ay{gg3#R|@25x`=ck)(5&*gBQbTBb?|v0(g2V@y$e8cirdw`5?7W#xxvv{+schh&49a>R}LvK+k`) z`%9{=!swGAi*lSb_W>3*P(hAhqH#bVBYU!AW)HK&GO^ZpCmj`>CHfSjlMlZ(WwXt# zwvQCz?lVG2o+;*|K7YVpN{u>+4q;oj`L69fJj-eltz`OgOnF!M8uZI&S*GZde`afR z)A}*>^$*kN^u(StP=dMOxA00VWLJLz@8eaVxi3uIesOlls=mj z0h!HbB;Y?Q_Lc0?7)qoW*~>KZ1O)C@7vn9=@Hc$vCDH+7L+LoAIrw8QFQz=Y*j6?5 zTGK#Jjs}(&4mbMv-wHaT7x&eBKf>DOy`Cbk+XGM@sFJ+H*$yZz6_mS+n_?Xnhgiy( zcS!2}rP+7BOaXX=+b8HzM+g2pdX;9vzr89Eq311#{Bj7FD>4~+eSxQ!VDFUj1svZ z|3Up10SkYR>Wve*=%Ww<63yXzjKu@z`oMJV71?}pcXUCg zT1QnD3KgR)p~8DaXZ1Q$L27B1@4$#rMu(TeUMJr<;%&a$x9ZMLSjPG{ZNUpKxiK| zI9Wm*(ERw0)Q~?~Sf-1wEgG_{)&F5v^iwx zek+&UY$9NuzU2htI}xHOF2=P=jPJLI7kJ?OoKg}i;yt8e#&=grNWeKNHQvBSBG`r| zK?qO0%};Q?wlFEVaD*E0X2_4 z#|3t5JLoaW$D#sV1+-Lz&F;J)V2pDvg;mP8F|&1ZcGcWC+}%vP@Heb?oKa2|HSGb z?vj34V|IOs6lUSEhapi2wdM{XV>UnENWJUt;L8XL$4#vXyie+mHz%&eAA!8&A=?XA zP^7Er)VU)9G0;Y9Y<<4~&bR_i8}FpqZCWh5+_#dhr=pVOWd+W=1z>m5tfpp+i-YKd z8EJCE&hSTnTK&y4hNVhqKBoNVQ<{XHoKS$VS@HaA;1qe+$cdG|)XvTcX=aNA3qkte zRO{>D=KTttF$o z)YH;2)ojYih}~dPdtbfh($cCoh6cO`O4+#-FNm#puVUih=T-iIM4+b9^A#!HQ3Gc9 zds>%*XATp?jf-F0ZE1CJCpC*vq33ET+uEv|HMORiLTq;GY?yJo30IzSIrM9n@sH`? zG9jnIAE`QxipPNcqPLdTt1hmz#Kr@=m5oPJk7;oeBo%4hD_?(&I!E&Z(#Xp4}b2}cti zV>&P&zz#_0hPaIZ@7`v0hWuBNRoUx(GN^a|>7Hy8CW|T?^uys@i75}h!+dZ% z)08u3Ux2P+Mx{EPk}Qp_99A>3-oqPd(wlDI8rl#yg~fF{Do1~^Pz$#y|J=}HMI!kH zDW>Gp8!sJ9w9yQ|sYZmiCe_wI(|>TA)LSPIkQ^|qdb%`yRuvb+LZwCEqxYp&0A1?T z8p_MAmjGW!;WxFRRD29UJgoC&$};1%TX1)i(J876yQrYyX2L*4mY#^RpDRDfs#fw2 zVf*t@%~?QQ-Jz_e_=uXz9%|?0)GhG6G1cz5B<#qLZKSASc2;45rV_fMtC%lM2i&?j zd#o%qP3{S;Fw~Dgk7vYY_xY!jPh(}<8u3YvyRD0OHzI8VGNpP|EWqVtFqm}d-!s#$ zsqv|!H|)zua%EC(=fiI3%(RP#G^;+ktX^v^FwOqS!Mri-=aLXqeQ!#XprVw~+bwk= z)uQz;n6p8!qAKpk521q`MMrK$LHbHeQw5pX=trLUhr3TTT*vu4C7EFm&1Cy5@dtI} zCWHc38pYPQ*!du;Xb5fu5!Ra7EzJmyNmEQKpe3xXJ%5luES>e*Xr#I@>zDF~>9RY2 zEQC4hSE&7P#$Oigax7C!D($_`dt;u&v${Tt`iyV56YWRu@eHo&&>cEBwuUA}+hLTF-<{trm9#xS&Q| z>QR?;rKS1Sm-218<%Y7aqB6!gxh-OuoeyyoS5n6dTdWm-4jf(jRgqbp4;?Hq_(agU z?3+ae2hJG=g&n!+_IuRQbI5*dEeybf-P}H(mX5ugKd#metry$`8xF_K7&Ua57C6{o z@}A)A7_uj}RkZ-g_b_zSzR$ti^egeQG*XW9OqK;U>NHR;c$n(N$q$+aiZe4Q2_y9j z3ad&P&HWWVT(AlYl~?wtri{rCxzO{eNXou=H`uCTUiC;($eT3HOO*%*CUMVnbltRX zCvUa?Ew`{W);}ehC36^;FUb5TlUYHS8Hbn1$&?f$xy_^844R&sZ?i^=YvA*HTg-a> zBID*aOU`3&1>6xV*lJUJZZ5h{RG6xpw!gP>(;w{J=&U%2UlDbRq=mg%%eOv`YE#|m zJwJtWRkeYkwQ6Y%1s`Y;-=^!u+_-Z&4Hwrn#QtMqsW~Tg6G%THkuM=bG1?5z6Aly$ zeyqbskwe~Pt0&+5M>e&C)xJeCCK&jMCuRT8-4qH^({BzBaziHwaG4d&Z)D0H&x^UO zyGbTqE)?H)ekHjV5OTmGjv54!xUAECHQ&rvAT!hn+;qlkU1jKT(`ui=-R7qgENWnL z9WKGzlp<$ODKWWj%GhWuZ8eeh=8o0lHlg~`Vkk_38AY{VhR_$;(-R@g$`i_2xLuK+ zE2#Za&B#y&X>dkdC0}MnZx|+$SIDqx$(hwn&IcRQ{$XP=PoMcb%YUe=n_C5nk&R2> z8im~v5=syv6#g6;+f~nttoTubqk9O==WddVu!O%|VKOG?eFj*~aG^qf|~W3ks?j{6%~fXsS=MigFG zimA#r%#nhlngZ(2TD3Obc0bdjDqAnra^`va2F8u{mcqPC)elgE(|dKxr|UGxOHQR8 zZv<+J=UNJa|HFO$ymCEfWMCmmUOtbV3b4+#VMtzoFpRMnlg*p4D-QU8!=42=aEX9JUyJT%- z(``RsWA%NJr$9ETvA(YRFP77n&(|?0@kHUY7mVl1>;*e0c;CF9Up(CjF<&id*u!~o zYDWX;v zBtxAnn6}pg4-iCQkT1-)&kl5Tv3}fz()a`smQKhH#Ij$LVK%bSC9#lYxo|l^H0r?m z5~9(RUGF5k1%e;_8Ba*E9?4doM3Ge0Ob3#;eK6ye~ z{D$1}iSP3vYupcNzWpn^tLOyCJ@4Npx9tpUfa{2|otmfW+p zW^y$9Gk4)38O^$M-V#pEC=986-f-{PMP~ecG)ldKUwf3L3h5My^c35^sm=i>ACfAE zb}h|F*o1l&ld-s-gmm)^%;LW~)hYZ3QlWcs|7Rw~MiQwd9=B9GaY;(2qu=FNkzq0? z9WD~LN`U-mI>4a8i$#kl8cE;s1Yjy4_PDrB$!DzoTV57&JZnrq#(jqPrN~$}eko?< zH}d*RsYqMjDSfj@+aY}xQxd<_471$A*u?Gt)7K%j&qDCXRDMy)i|m~gvvnnXXY)Q# zOtBQ)B89x`lObL|8nL~DKqc^Hp?Q#QzIX=z15ptX2c{IG<@NHMN_!5rvaKR(2%Fa^ zK*!JUQ>7k^cof=VB!cZX^Li=nyY|Kvamh7l0ma6XK-ED?8j$xHa&TwXgcvn|K*uxm zwbUskGQBpGDM~F8En&L8n;v;!HG-Nsi8IqB-8{j0_7~_k)XjDDZ!U*PdUB7G`fy-W zexHl3uA1`5jsguVj@p02fQkBz%!`!5f}ffa4~YJwXb~|h!nYRZmb&vPUy|^hqc6+W zu&v$ccl~~DZ#B1!6fN61T*Jas=-k8F>^nDem5X;dRN9#c<$--i>Jmh8H zIGb1JDcsQRUnIagmVpoU_Ca#}cf2RL^o?ohL{FNd<#WtU(AMzcY;$;~Z*hdnV+H1NdRK2AtkZ}>XZ zDp_gydaj72IrnD2Dz2BW8=FAnOxY#v(d661iy*o1UW$h0h;pB=BKg-(@89~-QUxno zkYykLYSl?Flj60t+RU;MSj{jtxg!-|=SPO5d^A~q#UCvBt0G0sWYUP&@IVZ zPH(PEmjC4EmPfxzBAd&Cf$_7K3aN@gZnb*T87a(}k()T)*NI}PfYg6HMu4~jRIMR7oVF));fm`;&UR06)=zL zAIvql`?37{DWhYBr>4~sm6|g`VH=V3Zv@@b^X6*?;BQ|=Ifvtc`{}rtS7zjcX;YC2 z>j$Fc+6xu-c>;J#qRlxly~elrU_AK8-uY%7Y0|M*>Gofpd@*x3d!lFwMct7iYWg(a zKdioQe=OX&lFWrqZgw;ndysD9={@lDV5WsPLJJBhh}rb+-p+PL(N{7-%%Oarw199k)`M-AJ(v0%+hGw=8~c9;YR;|8-w^kn z;-C2p%TsG|HLDN}w-!v8^5ruP?3i2#SH7wi;jVO`kNyzaGO`pX;VaF9qYUw7@u@wy z5om9nYdhzZMtoZ7N|ED#!xmaJw<6i$3B%<+6)~Q6%j`AR&{5;X1s9G*7+83j3ua7{ z1wB_tr+*;7smc&*oBrDrdehLXV7xdEU}9$Y>0YK9@&t$^tR$rIjnN4xA-u4Snm^{P zP8wEN&=`5i_x-FG*Y8=DRT-r=U|>_6H=U=4Nf}1(`W8j@%inydI`xB_-yTlCh;*5F z5><4?h(3o|5Cn?3dHqU>Rq192A!17tJL7eB4RT{mFbQ7i9&rS03cDRBnI`o`h`vLa zq*Hjw<(Ya=On{)CX+nH-!jQ6Z^ScMSCa~cO=oT)Uxz-GN)dO$Zmvi6xoGl$oh0at2 z_N^?aj;6TfWW+HRe70SP z^w1mVnus|`bA%9tS66ggskVrn#&>3DOG-iRi)4izi9ct~zh@MJ(vAbL?7u;rn?08p>H zL3RtU5YNM%zI%Ot>xpC7wYY0U2JWU}dJ|R76JAiFA|)A1U0$;PCc2pPGzW6ibn9~P z86U3?o*Yat$7jSW^C23WP_=t5YI38jE5;K(hiCsuo_r^^5_+2asZY4q+m9eGuBS;u zMx|F?bPwwmRBg(r01!5=)%N*IYPMCS`F34oL2+^XHw(m8WkiBzTIvQMRr=-M8Z80% zoXPTi)L6V4+s#p|V|81P-+68vQYMVGv&Uuu`($VhTZw{#k#tFUe)juqB#Ud;9NPo; zGi}~1+e@JYL4J-3l#ihboMyRdeXvk@`U8i-gqyDiwF(CnPTm4ldfRcoqt^H?keqfY{Iy=( zM_UGs#`3`61dC4&bb>3;YC@^M$}88EMyPOK8Y;S~r0x^G666rA$hlYN8{{_vozuCH zRUU=BZemcJHIM^aC8hcau{xUrEucq*>qqzUXm5jt-tD(In|(Y5Z15?9@Yt0uG2Tg= zGCOIMqK}3JL#wNQb7mF9uu-CL9&N=%honV^C>tp&0&+?Ey~GxusfTQ3R)_)3nvCJ7eTQc@w5-=T%;%-}0yVd{FAVG;otq(CFaD;t) zn~i47+gn86@JCgn>!b&bPU)vn7A*P7m2AW}rBP_Vd;=kQ!8fHs4UY$g%4<$bgowY0 z{~E?^%YS;|tX5~*Ui2+-N-}eCGz?RZN4?9WflW=ld**1RkWZ+tsnp*mwdIj=PRcbR zdZRk3hk!In5(S=V81$AE=T*+N<%CXj0>qy?+>ucN%YR;AGmFgAAe*!>eUx{emHZAp zJQ)2@7J9{$R=7R)GOjcrdYuJ7=~yC_U7U@&&|xbdn{N&p$k`s!Ew@E$W7ow>11_cm zp@_2j3SaB*%@s-HpSF6Aqx0{Zxs;c?);ugeg6JQTV15t|tt-hw9$04$ZlQ-u(yjO&DHpkWn1nza5 z`G|=rkha?^xhfks!<7FZYiGWJn&;3V84MI>^@Plx)7xfIS|F*FIz+4aEB*YggiVQg zqoE&Eu94RB=g2Y)w^l z^&&pT$2gOM0g*s@1Ggo6keX^?5nw7@`0o=2@q;?QZfhAN*hSfOuNDTF(kN9TMQjYA zd?aF(lrWR{quLT>i!ZaIs$(9Bpq^%1=ccJ@RBZC{M`y~>wc2MWhu)1oFbHTxwpix{ zv9D+Qck4-hxc7~llq;6DhU2%)+QK<8SpZ1;mcPnxgvxT4-Pnc4`Q#2Ws!=0vPlRX> zw8}qhQ`g+6bU+XpS_Nxw7~?OGM1Na4od=Qpv%J)^P|`QD&r;V(Evu-ziKo^oicZ^V zyeO&&pkwB@+kMWmnVg)yB32Y4sG8a6#g3!)84&mgh}m;1o0K=OWQLme0puNR8+L6_ zxKIzTe2+NY=D01*95RP6dy6{}d#1x0>QF{Wc@-jCgCmu=+#fqdseemIL@CkVCf&(Xsx8G0R{1ZHohBXuu{pwC* zy2kTb*-Kgt=~fnc2jAID40# zQz})hF=%80GLqh`Pv{=yUyFOyvInl-N|2*1F=eI&()7y%4nK_Ldjnfa{3#Zk7N>~+Y457bWitPxnyC*CVY|n1S zb+S?k5FYjrgjF2vdVOEN6z~pE&?m40lPLbn+V!IGL9Hc|g>B99k)g0s*}t(-F16SZ z@)lmd2y|&sCVxPpxObW;*>#ueqg$FPd&ST8Yg!6Qb0COL?+hv%o(u$5J+_ zm)}MzqryKUH`cmY#gf-_l2e$%f03{4;kD(y)c3Tf6r~#6&l~5Jbvd-@>z=4YH}y+f z4SS7PtYjdXUW2m$?AiJAAxWuBc>J@%XATkQ#*E*G~Ytp0s%kTUU2W{*14t zQCAKDAa+%YX$fVILej}{`s+y_3{{-l8j>~n7Xn<5JySG81&=OD{9(Dv#Q}bLx@6=?lqR<|S%qtLiw7ZM5o1S!;KQ)bcV!N_n z7!%YYPscD9C^T0@Nb9Xa)`F!vnc*+VtN2X2fXIg``%Z4#mPYSefBs#wJyUs4HFsQJ zdEK%EABQSr#K1)>ZzTG3Qg{0pw5{D})!^XrCO%rCPpXVRFm~Rn++bPsv}|tR2Lu5n z9|%~?S1?qWs>j_SG3=_zXP*$a-_<(ay#1B>q;=^RutvexnEWf@n2 z?*BXO1KPduvyilT^=oM^&WIFcy$q>5r0wYQ8PMpbKlM+)GHAtcbmSSTIOF5)A@(JUJukZh**qccvdcPX>q5mBooFr)!wPBhh=L->;t|$KH zbNHdWt_Wp9mcTo@V?B6^2=Q8;yNZSd(}J(GJ?Gb?uX{~<*7v5Y#C)-zKaY8YoG#L$ zWO<@eF040n64|vO$i%%oCr0^Ua&)n?f4yB1^@p+?P()@|uFG)IyXGyGxLZM6#q5Ix zLUEhyv+4)VYO24J`!YE<4c6%LnZ6gJa8Ed&joHr?kpVN}MK_psS5S7QT}}a}1RHEC z0x+q|0Jwa$E2_EYnRrgV)Y(?JRes9u7oZ<>sDe~Pm)UaTnp(Z+U$W@tkkft2&`-~# zI_pwoO`P|cNwZ6ppPI9HOuI#2$@joMZrnrbkDHv2Ba~}xcXI-NH_4(>#uBC+5KV{e zQzo>-X4&huIbPmGx6G3o>5)<0S9o@pe{IiF+FeShalKyp#P=0BY3q6J1b0{OQcjI! z%V4!k8!i&!4ewj5r*i`-UoK6Z{Cn>uonxT}y;5xND)%Us7;kERa-5_wS}F14+Ggjc z$%Pwr>nE}qKD69kUlEGGP0C3t72RTMYbfl60=8+Efe-aDb{7de=`=VFP zDRMUuo;F+(S+^c=tg`X(#NrCu|F5w7(=Wf*ms#G}8|uB!93)=gVA?U)yX|s-&;1yV zNqlII&>6?oqmha3fGJAo%GlH+ITh@lgxbn9@bwyl?t#gU%Xc(BjaT7$g7nDe*3AUr zq3aKcQER*~DS7lYzVi1}hqv2{LOZZ?j=PE6WqZ%6_@wUw>P13-n;v%W9<|x1p0A{Y zn05Z9AWh1&ZGPxvVf@2)By~hQs|I@Tw$~$e6Gf$p!_GUIM*hT4ad*;OKtl`K z=o4N)DScr&{YOUNgCPEee5<&TOReoedkQmx%H{$o+JzZNo{N12VGlseEEDQKdnTlk z&RIXT%gNm+$64YyHa&S>@3)^@mg&7Sm*RQt9dxj8mS>17&~-*?IXwMB??px1$ZWnT z+H~E~OhxV2Jt2I=3vZWDd8!Hp6&qXj-1NMCxiYFReSU7@d_n9GJ|nl+dElaj%9|XB z5BPSLg$crarN^)@)7}PUvDEHt^XXMpe@L?4OrZ=fb#`m(eDh;%Q6w*KW^@MoD=%f- z#&INlAg?U@kdpTY#QEP7u7r>*>=qJg)@C&=>z{}h7;6Qj2aZ2&lN)z&<%9t=l0UgC^shSs zyivw=1K!E>bVszbKfmLhf>ny6gWOY{>Lu5$!AU6Hjn4H;^tUYRMdDM7h|?a79^#S^3s5*%SJYpNBN5d^CTgkz4B4lA>pW$?re zNIvQ|&py7WEh))0LU-sWB`W2_uoO(5_VDR>%Vy?_TWn`Jwtr+@7Ko@izZ^Q)7D=02 z!=(&$>1zATj2%}c0vgGablJ~`7Uso8%CRMR7S!X|=aBs=&=M2HAytrL8&+^^RVyjA zaI~nNquMr4PQ6qdY6A|3-sBmkve}4mQp;3vTm&t{9w$lcS7`L(9FCCPTL=q*6PD?3 zjTKIN=pO|I1A;-J5d=3ZUvE}P@F}l7-@HFIwedA0+D9Q22P=yq8v}4O-OA7#RPwgn z@9o<83dX0M?~MMk10}>3PgwmFutGU`!LjkbmpDI9?vyoLss4Y4p|CFo=!6)10JvMe z;hXHCW~@FBscKYT_cDOA0$q!v=EInqO6i$%&1?y@coW%XaOf6D^h$j*6BkaDXPoP@ z@)h0axEH3b#asp0xOX;7R7-rIiQZpW38g-O7OjKKr>30hX;p|aN~%Dj5eo6tL3@-L zlLL>Byq+Hma#f=z+*71RSPH)T7ytBg7-p#e-mH-Lzv#z(YX?6JAJ4^YYM@EI;z%r6 z0>-II>2b>KK$<+1|03L6323{g9!R$zc_!YAwnse7&+_r*ukn>pyd!D>6U+BF4D0+F zx{`6wzA54r=&0xdltlzRhM*$@5*=9|W^59ay4T7_q==N9TT~ITA+>h;e6qTs%hHD; z-B;&0M68hhh-sprj^1gv-CBDZ4SDn;5@=yZUZ%R?dnPo;TN@FS!Yl_n7y2^Q+#Bi{ zz0F8Dj^IBLx+!ZV`k2SUY;hGc7jVVz7*z>J-=wtjwfU)&sxz-XbQZh06}-#jD0exl zp4=48GY^}3H%Nc+z~}9K^0k!@>Gl^^m=3b`;xl^J$M2+Sk@AN4%|p8B*5=fKP6U&$ z8eKN2&8|Zrn>=iPkDuqK zz>xSLm(|57IvgEEE?Dn9P65(v2be?vH4lI0E5e=3E|+z-e9Dgywih-sSrs>|!t#`j z2%iku&*{B{9A1a3=yV4pvf{1GsXe3a;oxK9AM3z@m8M8p$}%=#v6UZ_ht~ka%z7!HLi?g16vx|OFbTFPS1Ci=h1-= zjbDI#BTUS7WKI)=zl$t60D5ktjE0##*{eopXz_)KC^0|m`1`1fT^}(S%)&`rYJy7G%|Kzq=d$+Rf zNj>DGMb*^_Fc{02@diKabs_KR34dQ9;HWfvGW)8Nh}_uve#1r2JIx%cWhKaK+>~KO zPyu~hr7`?g_;&k2WQ6oCs)wY)WulvqC^n#5yXB8w$8QebYa7mY%2_{Zb6hqiC!I%3 zPz+rF+yU4JM?O4xB16@JM=ls$B(^kN-xBSFAF+L~NOrc)J?F^P?8ASbw)Ls1U!zvo;dJ_n3aP zJ3DJ$B=PS4vtS(d78YB&dzWnI(GTyLkt^9XFDSc|H!dIevX~G)UN{S*G%y6Yeylm? zbHIn;kX#!BJ2()_Spt(B)4Y-ed3_m5L9*X`GqHT)hZ`s-Q2 z{YH*-+KpC4h38zfK7-6$hYFa#fY*WL;L*F=ALpE3-*E`FTWqGlY)s%14nI0VF6c85 z@$?yxSsM1DLV3gFIDIL|6H03dyQL4en=HlEt3z}?enyJQ(d`l&8|xf~uBN;=Z@w0VicXAEw<~)y;zGa|Tt0tLucnBL!uw?n= zT%8T44cRQ$-9voChgwWp<;%W=nWqDa#S;q!=h@7!>1Ow!a>~0&hQYWpxs2Qr&-^iu zkr%kdmtOz+FHI?4`(h0a-@{V{3o&!rp$53N&ZN=N-?NU$MKU#Dk zQ2_v(_JR){b)@cgZ`iouq7Ar==-!xq6K{bZ29Htw%UzYUrxwId4#O*T;m@2(H=P-! ze9$*Yn8#idnajsmm_H$D+zd;B!+$}jy%N^Oa4U2y85ML0 zba?)9&*m!g*nACdCjo=S-L+^yG(38KGRL(qC{I=UCvgAV{feL!a{x$=X0o`ru2}Po z6W3I8Hh8dKlZS|^LjAi=EnALa`7X{hU##25?}txulTF@DSbf#On3+S<)tT{waig~) z?g5`G9m)U^2L{1>D^H71r(Ao96B5gl!<~k+TNLrfI3%R1L)QwiXCI#ucO)5|(=iI> zg3OLd%wA}~ZB22WB|ddMg47pS^z=Wjrg`HfHU{7&y|(@qe9V(h#jIDBvQ-!R2{Y3H zAdAO4Ypiw8KGPZwwc1p^dErXz52swu-!Rt9+JoSVcCiA2^)uIfkZ2$`3cmY0SDFBC z>Y;)z9kTTF5F8b(w%$SQ%lW1gPB;8E`cOif1l!~{;Qy(m7#OWy`R@xtJ-qRo%H|%k z8$72lrC8{V;*O(6=T`=rg(geJT|XxSTRmUq@k^}VQgI~J|&|Nf=4{oP>FG+gm>n5weU zyZH1OzZDTlb)SO}8iNgW_3((|yLWV-S2xL$vC&S3hyN!7%3h6DZno^=>rl{0Vcf`1u*L zjxOd*@$vNv$+d$3#dz$NT(_r&Gr)Wv-7f8;n_!q7<{e>`)a;S!j=-IP3#ae2Oq9E? z{7|ZIAHvnY!0f}@t+H}91CBrrWA~)AYhr7gsZqb_X@VydT+i7gf!3WkM*Z3Z+S5*O z;GbYN_zi*|z4)fl(J8;*`p&2-qHJTIXwP$kW&9XQfDJsfF zC3qvyX?Q7}!sP!^e|*|ip~@vA7YcxzO{=)hL?V>vk*lHOw*Olol2mMOLi;A5Ubd!T;acM_T=y#{LlDrvTBxR-M z9mRg``eq$$KmPw|0o=tK_fI~ZG|RrBQmDRd2z@*b+$h*-2k4ojWVY};)g=-+SjqNG z_-O>7$XHesxraKH2mF`W{`Khjfn>`}4HF~f47-kON)<8^bHhlE?K=BL%(W76oYh+d zz{XW-_#H6ROV5H<){@;nhE}^>Y}Zahci~O663675OOK7Ew7U8j!XFLpgg z38euil!WNy+si&(%wEdGashu1tVQz5I?QUR?iZ{#fh(PC`T4gzvvpI&YYP3&D;o>W zmNH1D*^op3RR$C-A>YMP&@ijDuc-46{N(VVvx^StAF;TlM?iv{)Rf*2LhIP!2-sW; zE`ACBt?yKi@ckdt22|0#yYQ;-jl3u8AOnU?afC*2 z;PDb`3$WmZ}KllVy9!sk3979qOL z63gFwppvYI&lId;Zc^4DwY-s2JZUm@K11V(0fm2GS{G8J_@g#Q;8fdKI2q!%mT}LJ zjN7ch5H|xxb~Lv~hEk`ux9{M!3)&T6Qxnh*XRr9U;i{cRd9XZyW9P@G3iiMddgfqM zz_icKQfpF%x?d&4g@4)CzbPIQdQ^ZoE^WmM_!AuxLT^gL0kh#4AJH{;U^uM-l5`m^ zw-~X0SC&GF1oTA6f(N!_+?X`Zp(|rizp#yg%jMe+F3XYuIwMz2@1uKSbT-)k8a?u{ zC~IRn^B4hph^O0igiBR%H3AF)z!OE4{vRZNwu}WwuINh@TDVDsKPeq8{xQPO<>1&a zEtVythjh~a81S&O8|(R{;ak@AyL$LPk)f78xU)A;tgGL%VXvnZ*e6CiAa2;u~r#RaasTA(8nyMzqApfxJ3r|r{<^c=b9 zz6f~RhHPz0LhCjh;m|{O{pkoQOKg;zy232FdyGukcMos6s0J^e+OEDW{=I?SJe+OP z^D({m=$HPTUlEs7GN0T^)q(pH!wKI_3fUWx)AAX0HLv_J{=I8e*(aXfz?Kt5he_dY zQz`2_6X?1$1HIa3a;$yYn#C~B`>-%pRZ?^P5U;;IZo5!=zCPcZ*BtCvi=bvT42MbgH=s{s4 ztlZ*owY+;?f#oWF_kv3xXS5QZuh4O{DsI@MFGX-})eAgdS!x`+aPPCu%n1hnkME{? zz`VKL)$^ijK}6bzwmvbg6~8!@p z!$yv#dzVpo&6K0H`{ME-B8gIt zv*D(O7r?J}N%K!>!Sm_l{gk8Zqovy!1GjN54d4+jjuzR|-A$0dQP$yuLC&FzA%ClX z8}wFxvw{H94B;I)wbilVg+XTQP z(zYMQ)yKkLiO)X8-1>GM0*y@mC40#5zI;usFVLIs+y{PHA?sPvY_(oGq$YFHm&N3v z?5(%sOvKx_tWoTPl5nwL_pN%l@N&@nbc!vD&rxzUj|LpH*-$vaPJvbrLhk-T;B0Ra zH-l7(gmh?PQgsh#9ghWtaqjO9Q)#bKOXe`=sQ1s2@QalT8I*l^GkjwoXFzb;uitUW zO#1hy$p|H9V)=vT?fhWnV*WX7%LIL2EH_y6dEg1g_hs!%^WqdTM)?QY^adX$1-Kkr#UiD_D=x8v|%u1FJYP< zFS8Oe1KiQFc7|L?@lX=_ws(ll<~!dbh4zYLuEhHsCu-9^0~FpIELP}!;GXMZJ=0;0 z-@ohnxW6XkRx06e7crE6)S6bZ_AuMR)QV`%YxU2){4n`Nf4W*N2Wr!M*N1N&%yuKK z6>{}_YQfCe@UOJLBcC8Bz22YPXVRa-R31gRB1gV+CzfmEu$TidSbZ`mh$t{43Surd zu*4M+%E|WeSF$o*Dk-SD(ffnGJ8IP4F#gWl z3Vj`tj#290`wU>Y(mNlAi|tiA?#qTmua`X%A~eO8E4QU9I{$^LrE8JHSWGHXH-*E+ zOB`QU_PAld<@GXXh=t^amf3Ec zt0m>{gt{%HM5fUlG$r6Lk5-Z2WOh~}2H_3BN8T}wS1Y$HbA!{OCprBD!vK*d2BaJd zc;A~L879m0g23rRE46!G*Al=K1tbw|T5yDZ%xs7Qr}I(`g9IewsDH%eH}J>@PuUZu z(yOsmMY)m@AMSHJSa{c2OBIt9>Z-S){924tu&jxR)wnq*Wvck;mIbuok2QbUMP&pN z?8(&Uyda)JOGR7mz7VG;;HEXhs)amJ*!>6a?N@G)&gF|ZN*XV9mTE%h)mOhvG()Xh z+vS34b9^?9`^N5zr?{`s-TC4+@sYhuSINg}u4DaFR%xT}>imwDZ;&^UNYaTCKQnL1 zCmrzMe;3-XyhGtQyu@&)87_<~20kTe^~c*FKdyLgsAVaeDBlNx_w{wZ4eFYKT;4`; zZ{?cx5Zp4`WfgwKcrL1Seb&-~&$kr9aijsAizSvZKs~;r4}t%D(=0tXZTB0~`{==& zQhX{1zDvg73V1|*-51?wx8=_!)>bb14U#_RopUwgeiD(Bd!xRPA>WCcHJP8GEBrNt zaFh+U94-Xt)@gZp_t6aC#qljIyOLz}-Obf|qfEn>rdg@cqRQA!^IeZ1H|cJRex*<5 zINVOmvZJe3edAH8E@lyNaAV(9cQ~*?Y$IQExqdS^TJ3Q3C{(z$0}%v-Mqdmyqm?cE z3<$Wu%xtuXXPrFv5lKjA0dkqtXuh^8oC+tNBaw1DjGU8y0Ph?Hh+-K_NyS?tE%BfK zo7+8`E+v=^t@VXow)KAaTf+M_;*~jg;#L`ER8i;$+)MKd$k3K&y!SX+aF_!(Q`G61!gWy`xzk zFDsvNq}7~%W^KlD|B8jyunYb8s|vPOneWNfepuASoxSJZ(Mv@VF};qXZ+jEQ-qd8D z_%Hj~E0LPil;)3Mf0^E_sC`z9|JMjq| zie$NdE*@Q^%AE*Auig=W`)Q28CeIYeJ0%sU8v>J?+xXISaItjs&g-D?jmJ5Obc}TA zYzM8wNy(L+u9pAxAOX}XNxR=99>;IwaEu-R!xwc^_#jPz41C-+KZUqVy*Wzx2 zySr0tpt!phcXx-u-QC?CzPTT{||frlI__@GGGU-P?4ntCUrYQ zp!fmQgDk{67hbWIh|toZ@`v8JgxImS7>6(M07GXpNuygfvD^{Z&DdSWhrdZVB6wf$ z&srh*Or2`<*Xn`})VokpH3Y0|*_+%Zfwk9&ss1R~jK9B4!|fDS9<|94;xw2mAN}Zi zP)88y7mYTtyHPza7ViqSdu@rp#W_@yWlk*Qf&#DGWbM4jOiq*bs^3Ag+DYa7bl`6o zk<*{u!X4f@~D}tDeznv!DHIZ9!{lWGTOwmVcXrgKm#%i0*$`7+NwU>}mn9u4|6kkg& z;>Jf6cb|*#lQl)E-LO=gRgIWvxO1PfYE5}1D-vvMnDnU~R!LxEExH@#IEml-g9RFo z4>X%K)tTzOhv_PyA9-Bvn{`DEYHBhV^|LfCk!0;9K0_&%I^ntntz!8_8^v?)D8Jl@ zU_Cf=>f^frC{;XOz$&pLw%2H1be98tx+b>)BJT$IIG_2fN-otZ&HLg2eGxdB^oVz* z$};V=OGlLjbVf(5fz|6yqAC#t-O2I*sKcdvC2C7)>h>-@a&~6e`5x-$H7W`e6caIN zBnpc|cxNRfJ67Q&70+oDD1w3ev%DMUAiUT~a=oV&5*B;yr9s(&IMrLGnZVNx**mO? z{h@TD9j-)mTNJ~+?3hDo4`e^gr+m5coWTNHw_ z>zn$$`OO~%OE|#&m|NYH%(i+6Bhwb2v~p9_-}}Fdfu(E#%hWt^!mRx}xNF3>nAx;x zLgbFqbPgI1k+Wv<%i*X7^+!`lz-!S#zhLtP0PBFUKvjWn^c^d;eTFQTb{|B#f3HRR zUlxrtq0;Q#xF$#=K@z>blD+H%H)_ zpu44g-lWDd8K*KMZF!JzBj9a_e?4o`7{CBtY5WLdHAIij6uqhaw6?s3du{HOcnITa z_rfG@^2Oo{`a6T3LpPLj^#TMEITzT99I4=>o~=+Ou}2Tayr{=S){l9h&c z1l31=7j4X)i`|um-yG0X51^!x#?BHjlSb@$-qtnC{IN`mNNEa(-+^@E^-oj~)LYRk zxgYA%9=(}jhfch9^VIrSiA#EiVfnQEUl-|Y`MpDB7ye)sfqcgGG2EMYKF?we36On4 z_v{0c>51T_g>svP%h9B(Nyi(58RZDuZ#PxG8r_UDlwpT<)zRbRUw@BM2((t&^RIgJ z!Zz!CWGJl@u_iW+FS_(Dge^wDQS(H6Hr$J$yTlVlup4Qlw|_}Zl-+p+zlnq}WeNLU+9jHg~FMy!3 zW#x|RUTJY5qKzp#c*y{E#uA!py*uMO#EsJ8T#UB+ zSC7zkc+Srdrv%r!d43c^cgd>yP&kcULr0PB-Zmnm+d?9Mj=nwQyEoGn>E1z1;i?m`1i5QWzPDtE|cI- zbjs~yCfjev^!5PKz{W@F;B2}`^qzG>Fs(mg${)oxFqE(s?c?$z&dq%Dp31D#5$)q) z@81}R?)26vJV`nHBMLF_tsJ%1Vm&K(jq<-$Vu~E1cBr& zg6#V?#78d@Ux5!M`O1zfK!dvI@SMf=hig2$V4qXo z=XW&W-$X7Q;HRhbGSXFVG0q|EUlR=#DM$)iLDYjgGmX3}iP$!thta_IR?u2b;=}vy z{$@VELiP_bhnq8?@l!HO-O}h^P;aNmy{VPEUXdJ-Ng%MSAh(uS;^8H`)@mx$6__cdDSzwk3kKMfmqP4Q>SM@1RzF1uk@ zEM}Ykf1AZg;WX9=!OV0E&s^8pIQsmPw0{nXKw`~}ziaUMiT5lk!vn^f|KT|DEb4#X z;NP0{(if^sKcP;nst`0F2dxxOuO7Vp+q?dE%eSzVHaGo4Sr!*#?YpIn+CN22B#s?Y zO_OaeO13Q}v@n#6U(LN}mVJI2+~y>=PK*#V(2gFQjdn)GJRGG{UbJrYa`MvOM}iDJ zrdZ(W8-ri_S5t8k3aa^e0_V==D_piJB(@>f+&^B^0LK;^)h?F*AH;4|oXoh;A?c2M zp1sUXEyuHZA<9$DtO9pP4J0M*SCO9gJ6{$QSoL&rVqXkkJY*N&Ywesu@y6+MyGS^{zu)&1c>&^X&1xzwQj|C*zhcr3Mr8Y>Lmv4r;c zi$`+~fQIzU1EpK@h`BSFnzh!uwLKAGhQ1!<*EpFrSr5b zi;uLyEj$LIJ6mAUxl$-#1&-6jM(2Sl{yYIhZhG@!j)J_();Blqcj`Q8#N&ok`zv!e zSey6JS_FVRASjIN0!6Rg?{lbDu6k%+r`=!u6rzm%)xf4@02vLB)g7&blV`ZY zavgJEIiO`rK_Br8Y?O!BBo6LkRUzA8&czIa_X<#ay*bz65o^NKY_gcNS!o%P>VY;x z>{w=F$JjHzw#FnMxU$S-vMaWBvg;=vmF#(&yZvrjyTYkQp+KNWXl&*E)8Len)h#<5 zJ}dvZmQkI*omQg)nAE@=KATNAk+ZQW7}1c}e|dTjVt(UrW;XoLHfc_?5T`RH(F^sxwY7hfpo>C@1mZ$Zng7Ty0y0%Sy&&V7v{d^c$!5yo-`C1W=+uIfx+g4X!9;I z4nh~d1{?YS!2>Og+nZIb{I|)&FoIfGC%{Rm0JY>?13B*iN@CXtJWXW{_;LNW9ZFj% zG(*#l0axoMAmH9TShbtt^TDsddhURvhoGQt42$FkHQo&5^~j|y#DYMXJQq@AMB;IS z{!#=EP1IJVNBRsarAdSm%$$jw`Pc(zFge))bB4#OgSRz*1ScielkRWHO;P%_DD>^m z#=OMi>Vur(BfDQFT_-$FKInK1$6~D08QqoI)~6p1Cb_JU=Y?al5jmnm0v!WT0J_{D z;1jCek&nYA(H-QH%f5P?h5`-X?DdP|9bmxcE!m+1Tk9Sc6Z1v5?WsByk;K~S?6wY5 zYyWG*J0YyszFz-$=r@i{cxNKESTClKIC|YmQ-e?KPxkQUXF4~@xDqce&x0Jvi>bI{ zP$GwHIzoR=*pi2wy~w2pxgg1FMTJwg;?N)DhAU|F# z^VJd(ZTcOVOj3>aBxd>W1)k|)p3}Xh@S7mR*Z;VDt3F*gKae~}!}^#51xYKO%x?hI z^_~ItP$&tF3g2lX&veMD=hR+4ET&Tz-K>)7Eq%>^p`-&maQKVr?&dV!b#OGMzb4$9 zjFR3hpQ;V7zS*_xD`dFOV2t=svg*r{RoP`;t+h7w77&IkJF#83{|3UeRavEvq{Xj0 zEVO2hLQ<%0ThWv~zXqVW%vmP&mSxX%k&1JBUSSrFzY1rZ&L6k?zO=@&&9E2fcoD`( zh#=Si$39Ywcc!>pJbTGsy ztV)u?)M|^dxG8^a&*1sKTl>nlZ?k};#cQ?y2b@V9mdR_&;)nO1@iR>U)`jtD^u>Q6 zS>%%QfROVqJI_tohXWk^v8m)P+`k^B#Y7ZthCDRB<*fWXZraYrV zDLk4uQs5`bel4~%X^23*t*KU3voEa07dx)dR{@hspYE8$y36{M{ZH#lvW=H=fuK~;Y9*Y%$eime-1S>z$fTTA8jtAo}pzTUy zx7;lPra3Ia4$v%*ja3sZKWJ$S4iGmpv;E(sz`F&l-Wtjnl#{ zjrtk%A`!ofk7xH>*c@Foy}je|v3=t2oOGhoNe8w0$FeD~=ey~`1%Zqne4$WoZ5(a> zg;lK;=`%=Ks7Dlz3{{ILtVK)z83* zb;Hi>`PUPHtgXd*RH8E^qB2bfac_$C!rksws|)MAtlKk5i?aAt4>~R9V%gm7@(wd< zHUjRK8V5X4XSBT2ZbfB|65-8pF*I!8#zf=h;;`=>Cq@hUPJA3*7bNpl8sEy> zNu#iap8osflrz#becB+rcC3l1#jg&${9%iHq^CJ;GrP&j{jh_S=dH~I@p20&aeX5A zyQIo_3o(oF6v)Mp&Mlm4M0)Hd_74qPF2AnOPWrD}u*b29uz=}B`Mc#~<4oS0iJpmP zE2=pI_L+02R=LB^2E3MAKQ8UVkcN2O0ly}<4;5J{&M&-Wm1CB_LpA{``~~jHab^3+ z+KJZCdkPto$p-AB`ZLK?IJXnowV*nq++Dk`J%R4x1UwvncHn%BQ;5Ho%Us!L1B@%(l-ei~IqszOqY3iN9OGF|wT8eD-ug$vx9j-UUq)>ue%;_&> z#v1GcA+|nzAj%iPaDCU3_s6hbsdFw==XACv!IjqfcS)f?&Yf0l=buoG9iwre?z9!a z88{*!aov77Wr!&6?PEGNfVEGRmfe9AsA-V^WDiOzl1`;G@nOI7)ACl{#UYvNA z?hd45ow$?2$MG-;2|b*hR~{IC0u6^rlz9@Z6t@F>8tzP$6w@dXks$V1CS~4-EV$*C znAG!Mk(zjmpHbWdB5FSn>IC08o?vu0DzsfFdg5M~aJf;!UwKQyUTeel)!+mK@Mc5( zrOvnqibfanfy1YDAqFsi-S_Mpt@JVXdEN0X?_Jf|$_ye#P<|Ape6;Azj0BzFL0^_j zH{P0p_;^>>17&Mpdjw!cAYC56uD2l>OV@m;YDf}B0XG3>J;0mMu?#;5u*N^ zHBqX>csAd)3w-|9;WVeW_*@OMJ%JQdxMq5NLZ)MBM&@kyaXE~!bgo_P8#Tv>e3`Rbo)RsCmFRx3kmQ2DB8U#t( z>B&l>BEdz%d@)JvKOt>ev&cv%3z7`U)TYPOS+muCbF{qwZ6U_Pq}hTI=pnlTHe*X+$}TC^M8i{2;oykr9w5H)|Vh`e-Zwj zx{@!2_2%^EFj1HP9xk^HD<9qUtVCH&w(5p4AJchdMvW_1Ze(UkiNwg!rFZaq^Oart zPz5l1+~-SWsLtWQ1K$W;Nb&?r{#kZL= zjYMRP{95upcGU~_O@V!xAk}{HB#h)YHFEdCH0Ge(=IdoP-TJTQiDq-O&`i;6S{_kz zX|hlXEVco)IdX(|A)PFREayILr6s|Ehs|{P%0G<{Fkhnn7D*xzuv$IbPh{U8i;71% zC51mPH?qE^0~)SRlM%^?73^d@^v1`!%Hq+3&!8WE6ma$Tw~>2mFXv<#a1^_07prv7 zTHDAG4vEvMHPA+A1}PJm7$TV%^0Y|n{}@B=-&g}}=4Kcgntbg409fRQYX@Hd-R*n` z7;EvdESz~lN4i}Z1xlu*QLWt>AE@DDZ!w3Y*2sAlBZiq(oZN#2NIk#ri?X(Tz$%rB z=i|9n^CkU0(0ZngO=+Z0&(mA&LSb;Hz-*isYxc{I^=!jcff6N!;^Ap6G{#cYsf9pL zcc4`z7OPDtYQF8%Y%jxiwjj%9HiH66G-Pna2POVaRnUYT7Z@w>`>|^CrkF}7jNK0r zMnv65xi79U+E}P9X-cNcW}@Sui6rjF2RA!YU(!z$ucQfOxZRQkKqb*=pK_aEg zP+6xtj6{wyf&$iM+|GmO?0uMKC0z=E9*GS`5{t1b$;Ne z$UI%8N^1miSA$m4qTx2eT!C<)rw6Q4Ma}33C|Hs8#AH62q-V{;N%KzYp>Jl!xHYPI zdcSy>$?w9ave?Jb<<$cefZh<@c9dQ3x-?cP{ zaXYaOC_3;f7be(OmOr93{y>76K4w#NnfLQM|TdMOho4 zYi>OV?sg^X;cpsbnCD$TQYg&DD!_=yEXQ^RZ>??jq7)}TeXHYz14BX$7G0~KADWeu znxeC=&QV{JEM#Tp%qhwb$Im*qXd`!&EO?h}ifDH%DlpQlZ0g{!#eFoRI^$VkqcQJt zW3W))7D9e6cg=VOEq+$89J7E4}gxluLC5>XA034z35XgsW{FkIxZ&;-Fc z&uAd=+=mG^;4CPFp8$TRNS*lEzd<|_t`!N zKVO`(&LRzF#r7jDXQefAYC4|AZcgO!<+E+pk$vQ#2zWUQY+F~l&ea{T`#(VN;l}z( z)o7f_{r0+wpTE}$kM}OB?d)<(PFA>igyyU=IF(dw7rd9{=!*O9fjMz-fsDA{!x=JF z(}uJ3f%fhLkoyn!A8AB7T}TQHe*QFOA#{ud#-v38FsRDe#y_#a!KX5rJHzLe5_ZYs zY(N!V&v+h9d=FW^s$c+vF!sAfJr-8w{#map()aA-3UnDv`5S>+t8yrlshH`i4{SE! z6*|!Qh;X|5Pie#7wg=<;D`J>Sy7%w$}j1 zhxSqWSTF${$S+>wZHAQh{VIK%HSdkw!%c zEjdJ1;bDN-{~vn!|CZw~#f#|PV^i;EV)qk2#rTdQ34BvRsOLecKntFgU+$yQ%iU3E zE`N7+cVg~&7v!vg@9O-7FT^>K98s@}Do8%eRlMT6d2rT{#k0_$C^T zCBxnOGO1kaPkn)9d}x5`MlwX|)wwV@q}lE%X>5FSBG(%Q4K0$`5b;^DoC(O<9Rad4p2`?k-)O#fuXE717 z3q@tsZGXe`gZYQ$zpZF$yZHEL-G_t2g*HNWRA02ra%i?`DA?zPB+sI-!MWp!jC#VJ z{Xo`1RcK4HHbK8W>ba3r#_SDF2EUuyGP-o`FYo;(Q#pm#Qx9^q2xJNuN7pR59>>tM zp(84>>R+77?D^QUYdfIR314RqE>DRz1}*<&jy0I=jm!%QhJ!7!;_4l*Z9@HpW1l}X z>cX4fq229fc5Iny-`+8t{~aj%0$a#zoay|s{%YNta`c_}`#Usfiq_m5R|q=2@jY8F zM+Qpg#p}Cn$}ikdE$B$J_vpvg$z?l7tI?J2@~{tPfAefTrd!N%pt;5$pP(bq<-QtM zB%ibie&Y2Zdy`2IaNGNYH-=#P+LnIl2;X3m&U^rEfo-;|$g~Kh?csSoo87oBqe%>` zkMrUk;#Z11Q)V+^PabHFzYl@`g8RVa=NHAQKdZAZ5h=Nsf*YeXg{?82tM(F)2erPr z2_{|}yY%jcr<0>yJqrUe!ohP3^n4kw?W|u}&&LpQm}+YKvu$;201Nc!(>ivfp((Bj zW+F4967kP49%8_fzd*&8l#|PUdJdIJ4J$<7$0%UkAt7FZ4>C00IjB$+@;Po+&FzSn z=ks88H~Q-SUB)Ld5e^Q6<-|^mZuG;=jhh`AKfcLle(u1r z|6QL5Bs3~UzyTjYgVoRUevGKwT`ibYwUyi9btlEIpBN(vB+fnBn$Cwp!6~1GEu3(U zzeIo{N`Nva3x!iW7l;}V<+Z~Uny9mV4Jg-R1F_k>>}1g^4asH{XRnIa4aZG}$aoN0 z&Q>=Zl8J#W{qq|S5^YA0L|;s60;+iJ4H18+-`iU4g%60jX8qeBZPay41c`f|KqPeyJ)%t-yt>&~x>%0H_& zSeJM-+QK*&cMtmrD4zZ}{ZTtY#t@R7e??a<;Grl~W)h?S%2@5PUix8K9fU0>vlU;+ zGqJS>@3t8U+j><5pRiMBSz@ie6CRZIj9N=ioTHiv-!wk@<7Iz6yWS0iNSc}D6wYyj zmfW*!*pidfZrlUnqf)HGDr$7A&A!i-n^@iGKNJ~II6oMk76REJrZ9yhY7p*ss!yyH zGp4rn#_z{UBc}e$%H+J~=ZPR`y4Uil?QTK8E8XDf>|?ovmp7%|A@jUo7US2 zU%Qh`|KfRIVNkDjcrkw1dKFb(dvR6p>>CPxCIby#%5ABy9;lU`89icCg^%$pUmE38^cpKz+ec{|CCV+N-u3%9%>?z;?HBcsZwoOw z&MJAH34Coxzb}VbYp1DmJCjALC~%hTPeLRF!ur@Xq zQ`{M)X1}cEWzVJLPw7w2SSyrG3=)cB{+>4M~rjW7<%WP;JWxOt|_HaYI@YK`F=gxGx zlC-z^mhO6zzATM=K8b#VuE|v#&WI~E`I0Ca9fzL9^#R|Knocu8-h|KT-mxI^`eL${ zIoQVj7ge>s^zTgqDW|8y?&{=5%zd>b`=65hJFdFxF|wm^DW{Z=nh#Wb!EF1Xn@*<{ z$mw!Wg#6IT5fYk&U7DgCcUDVb3b6lBiNyne|B{+azJ_5f(;I5{= zB{4mrwpY3szYOSp!QzMj9l6k@a)&Q?jeQUJ#@pl}U37+2FVe!L4-b<9CS@rxGSFIU9o3S?ox2HPkYMS!fBbFE@ zG8cqLjv@muFQeSB7+b$4I$d|GHOYyeXHUC6r*lU%F-AE#n0#)j@DGo%CaOI%`aZfk z=J&Un7jx&!`y7=pX}!Zh4e=+8mO#~~1MZhTzJg$zZxzir8OZh$iUd-ZWZkKTN$6)I%52#C zk9aK4O9&cIGO$0Fh=AiKZ;xjQ<=rk zKzvXBM^xrW!Zwux>;!3PU*!9)P6_xPPYI0IMwVL=L@UaS74jqh0^U5?(7kl-w4c|0 zW_WzP8un^7HNFSNx|zr_xWPgpWypy}BcFAx_WK|Nl#h}I7m_V5)RnszNQer+#R z>0=>8UQ5|$Kfj(H+%{F2j{e+OKFLP0*$wi!cXmBR&ME^9k9cUE-+{!KorH0RI2bK; zs=(-1!t2jYYbZmT`J&WVdCm^^{4DuOm7PSt-tYE3JaVQ8VUBD+ zE2d}582SbgU0Bz%hmiJ;^bl7Lt#3j(DqG3J5{_Tb(8mOgV>Z}qoaE%SZDMU2t%Knd z08LgnHN<7F9+_qyr5?FI2{G?IQRKE9LJ^pS_o&sC#CT&CtJ#AW%tEjUOs0#ce1pG) zv3}ovQaYNay>^hf!}!G?qERsZ*P%rxwRtzUT}y3xTY|^6+%h30e1C}B5$&yV5;AB1 z%i2De_5I|u?=RftmC;;v>lq^>(YqfziQjPTzAKVcRBQgigHsdrdFLgLbO z-4weQhxj|a|8{2!VgouuTis{hoNF<>wu^@4Jd1=wH6TGlap=f>OpbjqC@>xl!+4I;>DO7p+8*0v?f>-le6a~c08 z_qt909)pv=Ai(OUZM%V7*N@+`cLS(q?a)&mn6jtHpZ$TkqID!j<^KvU)UZwJNdN!H*`;*XREkR}qjWw%qp!0hx)HP@{$#RyU2EqWd zcn!3Q6E5f`-zmjMj?Pf?2o)|4*Kh<~pja*Tq>b*TaOZ4Bx^((vZ0Eh7qcoJir0Iuc z_STS`s}Q9r$kdank6AJxq1Kgi&W7JZ#b5WiY+P3l4myA2#^r(EazW49t7P0~qoFB8 zKXZtKebfZHKk<$%0v1AxH9|*d@M~zxGg2!Swzl*#YPiKq^j;im!mlbC+Jx3mbZM{YjNO@*87@(y5+5dGActfGn#9<-b z@>p??FkTyGRDbPKbH64GU(*=e8#$tVuXFpzu+gHeZlrSi#0+9St+@S6L-N+jc6&U( z4^}5|I^N6GX@ts8%=`!R9JYLZV6jGiU53wbu^A|edXq|JXD)p-fO$sta;0(wRG=IM zV#~YqRnja*XiP@R&-PzpWKC03-55L^=z&L^bI^&TG7CTwp!1<_tt+l3)a^aj2*&!J zirHxt>oY3En1sgI6vB}?h^xeo%33NO$i|4u`%C~OMF2UMI*uzgZ1k_a2Py19uN20B zy{8UJ|5G|1UuhyK%m)*DnS4A(M}c&+TzGlz46o7{Me;cMk*PPcCmd?u15nMA?O*poW$}zwX7|d zstNVs4=d&{R1bH^*}Am*U`cIERIE88g(bd$0~|tV_@g!3g5W;AX0+Sq8B$0r9XECd z6rp!6kom39%rj|EtmDm#a9MYhF8W6P5DDmO$P0Zd=ebhEtn`)wo`V-zdQ;1Jy% zVU$~h(ZD}OacZv4pea*6-jn>jtTM~)TN_d*lqBoNKRJ@L@?Zb98{UghzQd~1sT z`TG0NHo@KJL{yzNU6U(8-c4tB@;}S}>kB{=z>X~Vb<&=w?!jeFTU^8?{|N10sP{KP z1yvN`NdM(dH9?gvuLm}Imc`>2Do*NU5JExw6_(^F4FyeYjK@yD5fL?;&uFiBdppSp zc23e>MoU|zn)g=eD_w_nvt#ZvDa0|3fwg_!w-y1Wp`FUDtpWmxy+B7@gn2 zU#cne?v1!#uC~`;tj+L~c5}de?*E~;@&HXNxR`vo>S)>$2?=c@9RUM%uAwIn`&=5Z z95V#ipgV(ae?C&~j#Qs5f?W!-QCO%z1jFJ=Kf$#stCQa<&sMt`q?mLve2poGJj#5_ zj~*pw^gKBKw$@1wj5_70kgER?xVlOX2bX5@hoUP1j?-eR0=7ldnM_-8EA>d<1cfkO zxnt(@Cdz!Emi9#4Y}_!`Qq|v3es*oVSF)CTpgI)^l|iTHjEu}Pll$CUh5u@wE5G-0}sjt3+ z{jz=%)ZSF19gdn_XvjZvgPE{X0`Z@9?~Mo}%2!}q`@JUlUW;HN9clk@@}J@l0$wOp zk{U9tH1F0AtP$*#N@izz0y@|y9kPp}9e^cN{R|Lpno}|&w;GZsLPy$tJ@_~kVFLUb z6n1M0ciba__wYhCr}_sE`a3`V_{sb_B-LrM3-fsx@NQ&U-H?vc$1k}Sm6|*s3lcT7 zA$f`TtGVFL64!9e;Ld#jWyx2fmH$cq<_;wxkgxBz+ocgvUHn>Y{a$pf!I$is?GdAc z3HH+cmvy%>th7SOM_viZA?nA;Rx6nPqnmqaYzaINVi{g4En}`?@^Nh&Aq^g<%&x-% zy?S05BCxx&Fsb=l37P3YyBCQPus_;u1}+TL+;K+!bStlude1;?J=le2dMnu{LCSVZ z<9^>IrC3V}=nd4?~nvg_QV)j9L-C>yEu^1-z2E*Rbq`{Pvk`a{b z6oVvY(y-UZeW@i`O?=6NEzA?q`Nqt+a~nr7_9qKchece9K%1^O^hVCSi%uSssMOAs zfU(m;NMxpZw!sTsAjx*vJq~PJArUM=|Lr-8ffiXZ$G)SkcX#60I^iks*Te*ve@NbJLDqrf%KBiWzqGJ7zK|hqW-22-I zw+%6Hcae|QT<{(|A0mgyHvcz5psDKhU!*i5Dm$e=NqY0xJ@DN9fT(NHd|FES*hhT; zL988IM6xqld_EaGI6eb~O*lYCY(KI|$g`L5N)RQVt1XnqcNtx&G^u9fzE6V}8&E0A zkGOnU8Q<6`Vt&MG7(+-CjBY`PS<@Jf6A)-jkI?U)I3&wZo~%df>}WEK*$%1Z)^;^) zfkZR@3;hYNps?Lq%nRohN6IRgDpwHg=r;Zrq_+!KT#6#|e3M_|xE8K}66Jm$UrQel z{dMjG@F|7~pn{2JS8i`>$ z)Cf(^!L|;SPchwy@s)yMKq&%KvCv&T!}NL$*t!W4qZ5s#o!@Z-i9SU2?0dF*UaMmk z7oOg2{ERet3OpFeu-OX>AE*2_uK)^qm?hwznnWjF&ydyzNFR|frN0O^38pMM%L#S=w~W2y(m!{rUaRGI)Ama9_;0+u6Qqj2rrUFfv=^ z)P0^!0F^WZwD&>2s;KCuDy|0L(X zD|w}OAnY9_63P&;C4su+#Has!{4=!!7Hc({!_EOS4G*^i*Rsmmv> z{#i~U>A3hf5VddMSmrO)7iL>q)N0sF!_L=+V2&po!qCRBKuOCQutOM1r_S|09LtVE zfnRqneh#r7IoddQ{^hPbm!V*wh-s=J4P8HP>}PG{au{6x5gbze81IatYuQ$A@qb?C z=OW7vHs;3833#(7qzVTc>en!D0j)~Uba=zsdt50PC4nw&e2g$OAB$DCqClZT278^(Xa)-*$u%y-y+h{5F$Ic`HC?z8Bcnvm z0bzfy9K4}d3w*4Bm8!?mzqt6nYWA^A4-YG$lZ&B-;!!akInk-iLfb~O5x>=mV_HcM zBkE%%aQkLiV~VQA!m}%6^F$JLet@42reX=CKcc#)>X*ro1>SG>6<4VtOD=uuW^N!F zDLo9pd-9}9ea8seWwB>VJGb<&ScIMJ`af*b#E*McbSSE$LnQhLaoOGJxm_*ZZpObm z>YcKZcD5FbFS4r+6O%8Ao|+#@caW7DqtcH|?|S$wk#yG^vk8Sl z@xdcvtLZH2q5-0mz*_pb+8Dj~?iB@_YL%3kG*pegybZ?|wjg=l*nmjD7Qp7n<= zE*ZZra@dcN)xOh9nZb;K%bMKl7(cVAS(jNT;ujy7uaIvuYNcy9e5|b}Lzd~C_@iJlYECc%Jap-%*XH5h|^EE9IznPF|KcP8)+qu@d-025nW*2GzZAp`pK@OtIB)!>z~ zhg4}yW9CP0`=+!PIqtD4>A_{64c8f>;X!IHF8?B|#rbDCS8Nz$ihRFJi-*c0QsX0qge|pNYpw$maS{}Re8 zB&{?HdN$ROqI5c7hE3Sd31+1IE%?XrAxxUB#Z)Ba`)BD}3~?^U&ahYvKx+&;a`n*9 zNKz9Uq&sg?1hEmbA|&599Si}@h+#dN_oHvl9}R`JZ@S9NAwZ=q<(D$7+Ed{=3fXLJtPJ5)v;RR} zAjXpAhXI@~>!5=bUx>92jNXeZepEs_9G>{}r^LwAv4OY+4e-@#-CGtn@_YPo^~6wK zM63k9dNN-A5Z>jnLAjgvI~=ql!f_vD&&83xE8091l^^-!uMw0j8kBY-}(`LLN;KvG0~y1vEQC?6QjdW z_i|ZYo0vqXf`mlEPm*v0zoccX{D=t{ht8#iqm)NP{oJGxTv|#8+S~KZ{=)6nB!Bos z@9Z2SrcMq_!3b5%~`r zgzUxJ)`30yc+rME3VOXYlEzH(Pa>U*vxJ7^@F2s1p^>UhTi+&0E?_pQemmn#rI>sY-#w!xe#G~i z#)a$)^D(KptNZ5qS6}=Lof#$%=xF27uWlZSW=@fiZKC}p6rHl#^H9Yd{dA~OsOlu; zOJtv2_2h4xf%zn^0?Gn*?Di$5=( zrcvhLvin8zjNw|5EE1|Yt|9khXBhYF?%Hs_iw#6Ho~@XFtR}F{W}KY#|IzlAaZzn? z+c1cLNJ+;KN_R?^2-2Z|gfs&~cMKiU-CYvW-6ah}BO%>GN_WFEp8I~(bDmG{hj)Kk zzlpuq+Uwufwbrt&`{f)496SLi_7G0yBjteeOliX|saB0x@lO%K5~a=K%-xty5+|AT z(Ya(72Ie8*505y-W@e{mOVd@IxL(ZLGcSD$c<&fPle|E>$+o-GCqH; z5W+m)lZoQO2fi1{I8=(AOviXqe67d*dH*b`B$z2kOFCcehTXMKp;t2RJ?$BFfDl30 z=?D>1MD=H(U@lwyCNS|(^&->I5}c?uK?=LUg7;YlE`cuzKBzd{$I|@? ztgYOyNo1B}ro=k#zAYgoP#Y`Hwh8SMlk4AO0%YAbK>M$93{ao^Or3q0XKzVGglgkm z==3WgskI!qL?M%-Lp>9R#k=!Oz_0+E2B{6{vVxJ4RC2^tOYVSYpZu`c%~kuuvY1-M zFiIc~`+cKRH~5+Wp{XW+URH>`nZF{>%BbfodMLd*>S-}TMTNltVIDwoY-{nI1AjyC zk7bXcBSh}Gh}5CeLl%RCLS$BPFGz^x{?6j1Cr5<|9-RZ+InuFRP zEluSUx5H9Rn#$_moRKE*+X21{{X&n)A$^v;Jw14l#I*%VJ>fX-Hb}#@?NgB zY9RLw57Qso{E@Q(`Ht4ipik=l6$en~T5MjTQ|aK0-mE7DE>#m*%-s)D3Gg0|gS+V^ zxrZ~I870kdv?~U;al9WIFLKZ_&P;Z&+Hga=ms zsKCp|DFpdeQ|bDVfW+%gDPmPS1Ezj0=MVWXkNo0);uJeN&Kw8Ec-ZGEmuK0YzHpyE z`M-w+&eH)R%5$E@9IB@sml#ruru0j8J$iZPyP-+MDMP73NR=B_0a)5poubzJu}1m zpB#eqVT*8Wx@X~}pNLTIsjx__#<#`bQ0K@U)EG7-OppT)`3d)k-h?N$anQnnpxGpY z^hh7m;rsO}+;~q_GA=XQmmX?-5X!$JAhphl?IZn=kPtnHfh=olW*LPHedz-)ha)GnDJ0Sx!|C%@YD{M06YKjv3k%lQ9IF&jtXzQA(q ze_eLz^#4g!b6X}ol##ah_6IR8Psb8r7z)-L%DjDc)|)9vx!J8n(BfFg2xupXqym$ zt-ZT1fA_&|@W`U4p+}mkBts(7ZKTEJ=XGD|U4=Wb{y7UYuc#v&Y2R&3_zemwg&pw2 z=K$EilDnkk4kwasT7@UuAdyk6*3s|5Y&d~+5Q~OBUSrlL2!~XbuEu08EZCt?*3hc} z!{>WF-U`F)Y7B=`e>BjFjSLgEl8&?h%(X!U$FMw{qbMvd<#GD)V=inibp9EnT6wz< zwJr4^ur!JIsh$&380Zsbve?<&d3TBS==9-5PV}tY?$$C0=Cpn8myD;jF$C`hD6+Q# zIPH5>4VEk;XU_$pMet?9HKfD};wrB{gysZ4BOk;$R|H8lzo&J0J_caklhab{PP*7q z*<_pA@1Kk>4oV{HpAMbI0kmZVjqM-XGDmO~JQ`ADPvcZu;f(1j=7BkJ^eAk)HI9>0 zCq&%wDtD)FNb&k_Idg)OmvENN;r?0>>8Uf$$=v0g;I8S<#th+csc&9#P)N@Np^yu_`82Dsmbqe@ zQ0h4FPS8^;Q@nF?`D%5LXUFyvBF3n=cYX#JaM^eHX==9 zy@2}~w7~X1vo>my{Q*YU{N5kAAS6&pKzzK<;WORLn{+?a<@W%jLhHapn}zq}W|HEQ_|uKGGB+1Kh=jrKhcS8bg;mgY61MgFbMGH? zO~5tWyimVX6yBasTQSaItKEeFTes+~w|@Fwx1;IooxId9XX?S_4tZc6YT(*~1vZ!Z z$y2`H>mCg+9`tT7pD`<((D8D%@-Iq^H9MhoH4Dnk9MPGqM50E4qX(h*N3Wq0fqK^! z$XSu+D~sPc&`8H(`At!ixq0vh0~85`&xF=Wr?>v9+B6bv93+jFaJ}HTGJViF6(kkZ zJnp<=uW?|~cW-lY@GwOZFmu}Yp>{Lo%`8WH!OxD&d3{}~P(yr$*la&SlA#ub%!Oe3 zslsp74radoX@rtAX?30ovn~ zOlaZ~^arHHwpE?U>WwH_gX=_%Ck+8~v@5HU1{4!h7jLM}bqr-~>BPyr@^_J8O z?rfV^^@Jx*xcDt5WD8GkQ}~$XH`?_V`#!(xcdkQYDO0x%5VX5|=)Npgu25?_;$_Vj zB(air&Q^1Ka_Bv2-OC{;m@xF@C9CgMIjF)Q6;E?cVcc%dxMky9ur{?M1^^i&}o0y>R{OQG7dA z*Q~TcoHMqi+cw}5KPpC`e*@xj^v>loQLt7oS@3T=M^Rt=gmJeusiVA%l;~>oR+jN& z+B(rpbOIN+(xi8*+d|$Ldp(1ymGckD!Y&9y^E8VW$ z6KG&3wyzG~i#k@6-KGU4?dLuoD0#b=I|2MW8zf|>CDwY%h#zL;wk4P1cEeC6gCVuG%^F7IN23>}@<&%@X2Tl2%E zN82suIgq}-)aT;#MF0Q8ur9VmU()qQb-nJxSzy4#|S$ZqoP%=Hw=~fjn*D{OiD+_9Rf{f)A<@k1TK^Y~s zdoMXX2rgtI^)DO{=8c9jfz+%?(NOz%!&}GVwXn9b8zu!)m3uu65 zFCXl5-gWsF2T#Y4r2RpPrso|;|D@jmu#@Osf*V=4Ud!_f62U@I9MwC*G~2@Gv$Ks0 zr2HZJrJPAqIqadAZ*v~(pIL)juk0#G5tQ-3Y4Cv^LH)yD&+@m4=qZqlwThbHu4k$- z3^Z{1zYrKlxk{i2QMNuLlDGbjANuwo9TD$1gV3)1GJGF94rLY0IhxuV!7=4>uWVd@ z1VeYk!*_?T;**UXUypxL>>rfbO5Ul-R*N zwcI$8qJUUW7PL$5WiU7*u%9?Pxbjo%ee(-$P(;$#`W7`Q@GFhXaZZco)u&v&<)cy|m+MZEFG-urSYYm-dG0NU zdb|u-F##y3IJ`c*ERy>ffo2Eqwnh;4XoC`s^LvO2*(ubD4%NM847=Q%)LK*X=!{FvQ*{1f-gVcw~?7nWTl&-tf|HUHMjt_bqk zig|c{jFac#4b!iIW&6RW)MRmk^OvaXnk28Uwq76!+3{GFWz;zT)b4z|#b)+^ZBO!m zu*>o5PC7L$`LCGWTDyx`9lm678tO14RFf|EumUF70tIE4A;e1M$2n)ar*vQeT^%Nu zTkg|SE6UOg5;>Fh#Yywo(AKPQ8^-xT7K&+UL`=-c?Cfj`I9STyv)JtN8)9lPfMvM* zSHBaaYDWf@ffzv;4u$>RUn=^IPK(Zk=@J*D^{8@B1|;ADToc@%Jdo*DI>T&E8D3SNY?_ z|7qD3f5)e7*F5x+`Gu=o!hnFIsoP8#WFXBa;mA!Xya>W>w!&E4B_8y=kPu3KQ5&2- zugYFlTR61!AJDKdeenYC_2*Ccw4fI5yX0{o3@h9I{*}N38H3tIa?J(Ob5A065|Jy* zs>e1~;6dQdjH=Nxt4hP3D1;OK_V(EMT)o#XT(3g`8BSvP0xgx-{~;VZ`7F|Nh=q^k zKfP9yzUBL|;la+kWgjtCYYsJ$IU%7WC-fG&+a>~|Zn!+N1HHxm zshTw^H2b3$pkdeWmmD^@arW*MN}C}e3hyhQyzxK5cvq&5$e>F104sn0xz;yrl~ukU zS%3GtL4Tr-?fBpq{3YR)p%Gb2O9;N&OBj9vD-@6L>2ysLJD|qNmh#=~zB?<&e688j z8(}@+R>TPu-jn$>De=?K>D6~YN${AQ8xZZ3dp?BG%7LV3JF)47%5)GHd&($HTYt+; z%ufIkWq3x0^cha%$}Wh~u?G_7b&9%Hnpk9|pZj4(DgfizhR=<#o;-z>5mmKuuO9J< zkaBaf&j)ep3j>|Uk)hs!&mV+l+#z%nkmT~8H^F`3`!nnUd1hP{q$9to2EPg!D>LkF zE`k5boYl7T3crkH$+GhMo%3%w!>Zgj81C1s;na+r2rwDqVx7J8HI}S0Xp8>=B-?Bi z=DeaLV7nf>z#?e%+OU0_LDCIa36E9En{Z1RZ*lTOq_g)Ghc zwHpA-kAoabO4NxAm!ykPug|qTvS9~{g!QuB`e#<7-d^Ay>P=?X8dHVs?|kZq9?4@K zKe2hr&=Dw^XL#Bq)!Ti;bb&opG%-B@=5-xtEp{|uI=TcQX?Q8A$R%5bn8z13I2QwM zH+!nxH!jHku83^s6?}C$MOd19$ead_2x)04}q)| zaO1(^cIvI`(ohB#Kc+kxTX)i*aRa11I9rXdWA2{5gQ13&v) zDT| zs#P-bpDI3+B<+)av^V~1C#PH8x)UiN_uYo5N*@2K0^BvPtB8iG;Z46&aMgj zDl2GuC7S8S&^Ud#*h3y$xdNX8DpqkGIG4J&6KP&>eGhV+m)VSP33Um^XgO${tA2XR zO3o>~;J_d`>sj=KME?31D|@a?c#BvWvT?!QZlFelf? zEQ3N^|74m*@X`pkD}{)4#%TU*M3Bn}=c0{`=`RoWt4W{tG`uYdXwNJ3xpUBIJdG;v zu`6=NNTB#XZZj@AWP{ZEKOgp$D&YN-=xE#Xib#=^zttLbH>Yha-_L{h z`VwdZrcF@1kK3;^*XRpwB840)j#9gy7?E6bG6J+4MMaPTJ1~{EqU_FU0%Q&*(a*N7 zros(L+7hnB*Hyp%3U>dx8ejdvs~{_@l5dLH$Q~s{_K@={3inj?4>5tsTAkszbIxdBF zLBA8A&s3z^9CdGx4-o1d8#9I1z0>kvsk3Yt>_T}SQDF?k9h^SwDZVd!PK*ey$pe-j zye@}^SWdedwBCp4PMp}9zZk&TC(E@uXIPPij3|x@^)KhjiX;9a4;1!876=@7EW=Gajda$)p0 z|6C?8@sei!)y`Sb!-6vAg@uq*scP0_X(fs+3*mu=F* zPxr?+s*-sri76fIR<(&QsYiOV}E5~`DB?;@$*7qf$ne-tQ9vz)^$YmMr?1x0} z8xWiG%5njBV4cUFYCn?9Yec0v8j6U9h2;K6k$@fLEm$ z(euE_%jjLMJKeI3xHw$Fdr>|&p3v-UoDI^?zjdq6aaE<30%CJwloJ!cZ4wv1=I4;uB4ljD zgxYrFNkR`zVwthc`4_uYcDvEW{hxkndOZA27Ghl~Wz-!OtJ8x*(K&8XG9&qJ6B5q* zh|9#PPE50s@U^Sr!vEA_PU-s)^9K}EN`K+wR$+G)^Mx+YFJsm=pK8o)!gN#5#?CIg zpx~KzXJ@C~FAw}6!@^SLJX!EAp1}%+TO69K+kGs7gG9{o#W;HZ$4%!B{Z?c?o>v_^ zg7=CHyeS=+CFpkzq-F1Ch@9597Z@T^mOJN z#8?;m_$3>M<)ci~N4w6IC)W^r-W2QKI4_=7hZ2R|1l#(?Mm&v48gCY@9jq2f@M~Pl zJCQ$9Mtsf|y*XZ6k>Rm@EcUrff5?4IE}{BkqoO}k=32}`po?Oro`wjgk3&demY*NF zqUHekl`lI&JiIuNBkSNGh3k=@edlNFTM*&l9i_trI<@z8Hv5|ZS9i3SX1JebZP)+| zjnL(6#HIMiRh453QwrLXK1gA%qA+BlE^5uB%h&QmT9@o&ii5rl2b6p!?x$C`?cE6e zQ|I+4lm7Mh9jaAi&o>Xxxht6XQU@OPO{cG(?Ro`KilG1~>YrNL!c0TzNx=uye}KB7 ze84`Ia913FcZ6sdl-klR|&rdoQSMUBHx9qaFZnoYVoyN z!s~urU)faS*t_C3pzz8CmOmaTV5;#pg&S3Sc2|{iw~>*_Tz)t_alXJB#{te7qEhB+ z(%nvHihO=DFN|?QwI~_cwHq6J7u6EjT`Kv^lP-|dYY%D|@y-WGzH{xh)sMAUsKb?6 zb5(xjs4I+!D1ud6@mzlqa%@Iz!?78QEhPWrwYb$QyoJTZP=)d@wlyQmGJs+=$BttN z>jqwGP@g5_G6O_lA$t$BpDp{?Mqeo4B^l3Y1;RTj^oJJGNq1LTbc&p!P7wv3M@` zvF?LrT+p*G%nHAM{Hw7JQmv)e?~;6>42O$BG|y7@c}3^kh&nQ@QjOnF998MJ+raYe{l zSAkjD?2o0sFOz3AQqhC&RED0vPiibDk@FBu1)~F)C5`^@4-ft_HrDNT*_|H}-2N)M z+p7A7z&-YsExZ-HzJv9C*GII!qs{14KUk#cbAz7xqKzd0mI}l7pb);SoYDF#WnUqvm|Bk4IM7CQkM#g`KiMTKAc#(ACQkpVUJS2V(wYug7Lul;BmW znX=KC-cKD4ho{fXv9qOl-4T}?pIgpor+&W?CRb&u!zH&rr!Ck>U*FIg1it|s$IjtA zS@{V8`ha7&zj1nW@o4v4e#8l%zYn!SVjP@OKl9M|YwFkWzdk44Sa}bUxMsFun<$vj zyQ#T=!lMsr3I zUTD&H5yI2!0YzL;?1>vo`21a|19Ofwo9L#~0?d7@jKJhOzu zwl-_`f}AxBnm8<7qJ2dKMXF)Y*+P>bK1=SM8@83(*}f-8ZwOjoIitPdSX+MirN*we ze_vjqRYZ3gmZ2nam1|bNa=(#Q8_(ueqOC#oS<;ga1I+<|N%ga^-93C1TYD>XUOk=S z>3WbaNxCPC-Rv=7em&vTNzv;9!Pe#TtEcJeN+BKlk`5K1*63kvZZmJ8LlZd8f2C~_x za=^A*J9Q;M&J;XlTPAOf)X$Uy!2scmV2AwMi4OJr^Nk)n)n8)iaQQ-xd}yVbs{^ef zV#6P$a^~@Kr2KDfSkGHKelaRI;iC(2ha9w2&z)RWolie0{|>AT+Hn~1Bz4{_478b8{Djqr z5%!5&yw{O}Cxj07jV_KV=Cg1D8;kCdZ@g;GKA$^ApT zoU1K0K5_+L8ky)SHM&II!vk#%CscJdv{zK#ri7^f(w2S(CI3)t)jPOdV#T5*wvO@! z>+04mENI5>o?7vVG{!KH@Zj|0?3X(qh0lX$te| z=w6@TQ~lA)jaw=UFkcAH>yl!^s~%?) z(9Y-dVAJ3%+rkX0sBb zF(gv-&kQ-O*cevnkAVa=G4rfRdp~av-^+6LmC>sW=7(2ogyjEa8@5NTpvQ zV)`jiSF~bUF*5Z)F|7@kT1@IR;MFdR1;wzUu>)h1gbR?#yZLO2A77?7iO6SO=IMFa zK0_UMH@UqtN87~@v+mEJjObQpYk+E?Tg+GS2{aO=9h>n5ZP`{>Zi+_W>bScQT&6VO z8OEloorF4yk~Kn4ikPjy@!a2Scd`dUOB75OhmPQq1sQX$jVCWFL9!|JQ0y^tjF?v%QrDFmVL!EVAwOkc1nFS zpbfI%SY$x;+Z5WdHwtWWhVr;L9!+%L3HvxAm)krs=0v|A;w_2We@`h{#7sQ7|-vt$0Vh~#L@W? zu6LHLF+H7x7u)#@kB3rj@0;^ep2Ieb%~X$!mtCi@leO1dZVYjm09Eyl`9VHijk7u6 zc6~BtE0qNPyuHEVewBcR{60!Pk5um9fV%BF@v$ksoSrrhoJBAvk+6ZV11XFf$r5Gq zfr97urIW9~!3gy&I&qq9`SaPS)2SLbyUOlzs{IJb?MY!m=pj253I7i>iTU+ zw}2<^9#@)?HrN=tM&si8+{r}v*m>YhYZyI9zp%u012(-Jc`v&bD%54cskRngZ;}V*uYq+O&(`j#&;3o{1 zNH39f6mR=sRYA^`S9_tLDietR0Z(X<7J%Ahfl6!S*0(pj-ShBN;!t3g-|eM7^Zm1D zKl@wKjqksI=ca){>4e9o4{SaMs&$70y&GjIyqd8yAV=?O;kjDW_Q6_?UK}EePl6bxYzK%$4(1p$Yr|8zags&6PaywMVd(9pRnDoe)>_sEgh=Iwx)%$CjgYL;99M$ z3%BKj-F-HG>c=CMXYocF%w<&_hErR$i7d`HRa-h@PjxCg3d-qtCZRqUnunkVjs+s~ z3(8T(VgbJvH;_cYSP(Qov)NGYRtdJGXM2y_I5#}`689M`kNa{r$b=pG)Ek$$9Qzm+ zE*NxAbH+3rOMmOVI7|CKkbC@_{Sn_QW3hTh5^eX#AFZas)%bTq5sUM|8OLOg^Oo7~ zc0tZf&L@}o$P-mZ$e?=|WK=##6cj`!3f+WBnZja9oFHR_6Naf5##}z+y2OyEEUH%r}aFs5nhZ^7&tkc@tmfk@Kk>{ zN$bm2rX%%}^qx>?9m_&?d2P{IL`Nf`)B1xow0aeo*OQx>QvZD*WpoKD2~-R~eA{<}vBUSxIlx_-fm1)T{x`+S+_1HXy&)`1%$9#z~ zRhb6QyW@Kwy`?Kxys>4uuZxvmODd4q^!8ppsGiHPM)*$p1Cwx}Bd67>Sk(qx%C}ch zb&GGO%DxV1M~R+B+h6gU;e8;lgYZ8BRS)Z8nxHRrzEP@kmy)-ZsWs*OmeIcb_J3gc zn%CJ|D_-vpWgbXWO=#y60x2P)OZ~T95pv@g{A{jCg^tUyEk{YwNkqC83bRBMcBDHR zi>ad2l;N)7xsN0LC4FXZA4T_VXK%5N-7JzU8jRs1Js&f41 zLvbTDBG+?GA=3K8qv>F;*TA_Ow!E`;l7L6xNaMYN@q9P(qjSTVrJ13~TMETiS^m=D z&a>9xWKRjX)vv6H%>0fs#GDBi?9|pYwY0hG8+)yo9JhwFf$?0czUua7AxbFGKo|Ct z1?W8JeBHX5#gqTH38bhOy2ZyIzDkg!6&Y?Xv3&2G3_U5?yXp#IFL+feAC-YZC{(CC z5k$E`_aoKutIyWr>#IXE*EHdS{f!cjJxAB@yNI(E)FEaHJ46-HoG=(Gw#hSgOI;lY zn6nsXlHHarm)dFaa63hUjiC{oDiX-6te96O|LPcma%Jfp`6#D}lC>({asy^i<27dq z^uNLbWS!58GpBSyi|YOb9q|PJg>qkRs8X?1R0FRu_l~yvLJ|0FtiJ`+c!6*qON@;c z2Y9Cx_7(K1Y4Tg&24Y)E(}FAn$Xqr0i71quV4XzkhQWg{1aAkHR>9o?p2EeoifF>1 zC>TiNaE%J1%apmr5tZ z94BAP-}2$n!~$dahFns+xRcT69cNePIy}8B2m|o=&RdqPtmeuVB4ZqLO{___I3es#eF9IxuVZY@Fb8SSx2l}J zBhy1eWfc?@FyIJSb=E&aViUm$3?IX0Q&>p4`4lzbr`#RO(|(Y{y~%OrNY zKHjUr@I#LjQ3yG@Mp{yx`+oEkm~e*!iVHTnldFqh3sQk*3c*cfU!dgWLM7kEQh|CQ zK7{fzI@bPJC-6^CF?~|MDS9yGE0Q+*!j_FLkdx8t9@j-g_Ov&m?szkpEMOVqjM_Na zOJ&8f{4tIHwAM4M^*xg?ibs5#5ghPgBX_Hy~xI|6Ws$X?o#AKd{lz=r(! zW|#s;|2UicrsHz1H*mX2HuzZ%LEBJGoJ zhhq_GRi&@zc2({{2;mJ zUYFhFku+m+Da;xQ+nixIm+OW!D`D2Rl{HT40|V<@rkZ@g4RV}D z+S-+hd~f>JT<-8wLp){!Y7V~RFDqf_3G`UY8spf}U6<)7U^Vi<62?H7t%2HYgvsv9 zvk0-6?2U=tKIbaWk|5voTJq^Te9I=X_-wSw$Tn=bbKz35+NK_HPk!>@vKeh>TX7#U zd<0>c8o7@OMg=RC45K9|K5u_Ev^*Q*Z+)RRZEsJgkCC?vs1EAHXjjPr|SNCL#Wp-P?9eLSdo?hbON;1&C_0m&AS4*3c1MR$0 zBmQV5wV#6bOo&KYHCRTa>=|TQT>sdiU1h*0DiXr}*uwbulRIWhQ|R5gSLA6q%l;AX z8Qz4jP{>5=w{5JMV9v}D!_18`Ejg^lz>tMqxOMz0cmnbaFJgQQ3)wfJYS(LWlmQs~ zLG_%3yO@T1?GowRy z^T9NvNh#|S;y8xYstGnsAeN4Bj{1OcjJkN2*|alYDj%kK`ioLs$nk4$_kGC_Ugazq zmvM|Tqv_+PBc$403zc0Ots>F(nt4Y{4Lg1iwk5eIz!l>;~p z0zS%(w9X}cA1ER=RJ)`EYBQl*oUa&te48oynRX%??{Kbo)7R=PC8Y;P))!jCNrp<^ z#7k-)oPETJzKw#Jl~;2JI`pB|HDs1tN^oMl-s`fDgq zkwe2@OGBJYPUgGw3q#u2s3`ZZrBbfMI~Pc37v5QDicec*Bd5X(J{`yyg(B3ug3^bb zM3`9B3Poe8LR~Ro?td9u5-Hm~`Ql&7S0KK5pv1n>yY( zvP~Pb&63Ekym}1&UNv&~YE4Bpqv#g*B9iw-{Q=#S1>3BE-r;)KH*FoQh=>{E42W5O zx2qX|(RjpAcU-p>=E1y`LSvCT(3LmjCJ(sdA9v5_ZnrM7ErH}_ueaMfDg8~}f2<+w zc`mco;G}txiTY8T%qqjOZT+%9s@0>KUv0xIIhDgx3{6tCMc#5kH+6o&fWv%BF7F*5 zn~KTc72YkMogTUv@xwI=`X+yk%U7&Tfw=ch`CGTAg6t0#S?9cSD-eJ zOba*tNpSfA&s}!e`5l&vbHV3c-=Q(Y%%Y6kKzZYXPTim?pDYQ|B3IlsRuj|X?hId~ zrN}k28UG(hNds%jBjV#rvFaQT7u)aF$pCCN&jwp{-&$zxW%9H18+7G-P%dH#dnw3D zKBP6D$xj2Adb=LDDAhTq|FmQ{b+nDeagPVaafX}fy}xdhs}&fB`p1fwF1_v4#oVz@ zJ$wLW&)JushIl*?@$a>9#9>68r$xFe472@h$)no_vcwVZd^b1U6glEOPM1Ir5iU%u zv!PRB!rAHlVajjm9`6s}OpQsO&8nw5S&Hb%={<)X+HNDb!(L`(7t`OEeyYAR6`{o4Lbxny(<>RNW-?Z zg_9;g+hNv|2FIgcOxP%-C5$nsqOWJFM%pM|ZBH^-zHm1>n zn$s8|pD1Ptv8A^Te4U~3PdBX;a?G~sT{)${zlStS5AE0F_r`*6gMLlR@PI5yrKjq0Vk4I^; zsL+7TiUoB_++5e@-X~t>odNIb=oz+^MSUb)C;qPOgF~&_vg6~rGr$SI=Z~QzDbV9j zMd!y@u3R1_=fz8WNj*l5z=K2iuacGs3Ei>c8gjfe_UJgZzW8h#!LcOW$jc0oN=#;4 z%NC3hv0EA4`MKei_DlNf)0!3dT+nH^=ah3FJF%n|xI=Am2!NJX+^`J^vlwEpOSWQvz)>~Pla7a3w})!=U3t}k0InUi%kSu&8y>Ba8knw+EOXAz(Smu0!SIx0`XfB5k+^(|}o?A__Ps3GkO;c}7H(-X^TYJEPpCxc7S+>QA( zM|}|y=BwwsTzI?HYxR}xH{1!!M{@7h^}piNf~8;~FW75r?`a+Lb5Lp_LZ(%S;d0aQ zm7hmnK{DXW2HIw1+2Bs;?yR-N;3VEMM?Z1)Oe*wC4x_OL-?_Z?R!{9cXu0F~FZ_AQeI<1ge z=FzgPaR8{dB8t@)hd2*2yJwc^skq*i*h}=GC}y{~qnMfzJB?PaFI$zaX?JG~Xgpg| z3J?a9&NksZndYJ^Der=lr!o?T=Q(%8DUpyJ+vk78(rD}C{EWuj8K?2CyKkeavx8GK zDgkZsXa=F&ey&H-jZM>Mn=m+1;ZIJ$IWth6!_bN-TogF7)I(~&lA;drZ+Ul{Aiq;f z7fwSur^Kv3`|E_w16OW+-aS)p@yuFu~v>ptKLZ(p;n+xo_6%bl4~KoaGbXKW#j+Z(M5>rophQ(eX)qPcvMp$AEHy)DZ-n_7o8B7esI+zHoOfQbc z=MIOg5HMc*x^~P&6 zW*GErAhY`^?8G+vUAq$B;zBYgM8KC` z@biocAY$xRn(3z)ks9W}>!JuQ2SSl|^A$$>hW(h5rb|Uw8K`B$6C*G0yInmO2!pv8 zbraqd{=E9(r%ldEb0+nKq`!GN&Z7I-z}lq}@~3VjoF+H?9PsiA-yiO6W9+!Xiw9Z4 z1Nd)*3*A0U=2UI>6)8BT^}=U$@dfYZ>pz>UJ9tOA>>iAyP@KG}uCK19q@_g`5D?hg z+dFU+#LTr1Hde2-z>2(8MPAv?^57vA@w#xRv&LuyWCaJ^(QgO|(H-u%oIXV#uh>JU zT<~O*H&ZoeTZU3uW6*GK91l{WZNQA`m4JiQ2U2^TgLgw;{lZHeO6dJ9-Syj5nwT;i zmq9L9^J?72@2bAl`1{AYiq;D|kliEPr%_(hI}__NEx1BwNwnEgJxn+(87)t!^EF^J zMTRFc7F5O?k)`S#wRuWRRQX#G)d#wYCU225%$35wL&FPFn*42%nWjk%%I3p6x^9l% zYAf2YN!U6YT}4@Qi|eEp{Ej0j>>x&VR*&GJ<;TTE-J4Fbgvec`H^nh>Yo5fA^=9(upoOjH z3~?-88OkwE&@REWX zr>Avm7wquDiDFSUWQ&ZNWoLTF!t_D)O!dQn&m{w zRaohg7ajHt0pRQy$c&CBfN)~y!+Mn%9k6fW2FUg#ICli z$Lt4K%ASv{NTwc?q=Og=64%YnWb9K2zC~P0|7iaogc8o_X z+w3TLQZV6AVXF~vG_w$d0Ag3Y9@8Jg*^-8w6Xj@rzCoK}LRj+j5rlp~6B{$&(cy%m zeL#+3=dqCHs)iRs^vW>b?w5~fXbgMmWXTs&Xe*iG3}{~x-#;K{YlT%X-`STZci4n( zZEHQ9zkHjX@Zc-}IKm+8nw0)#s+G^F-1ZoAz=ftK`_j&uDp(bOzXAIDWofBW(b5K{ zrIE1cwDk3a5*#|hSaEiVcXb|s;SAHOEo{z{O0p~e{mOoE%cZv#4+Hpx?0zhZp9@c) zB$VI#BO(Z)=r+b0FhgEUo<%&_4&MxbcUsK0T;ncrb%Lh1j~GnuJT4dMgtlIm}4 zIuK*HC$>N5kwLF~@c?97-4q^Lt_VfXMAAbM2|={Vkh6o0)f>b9+67zq{}65QS0-Q5Dg-JQnWo#5^eBoN$!yE_DjG;WQ%>(_bCz3(~a z{`K!Fs7)0VtCx;B)|g`|o1hLEDK?_H(d4n^?qAPM#I^9R8hwkwSyb|gAV&6k*yxg3 zRIrzr2tH(3yCUCU;1$m`?)7bd!A?^~Br%?$wu?~<|4+_#gwmafiHR5dKVpr6u z;l2#8cT;BS!GeLF_}ovvN58&!S%<^pLE4PofDPu=rJgJsz^iqLz0b(hmLfK&dkh z`f%e-+a?X}xNfa7k2`EygRrApsKav$VnfM0djV{~aicv|kI`g=R77AJCEvNy!#}Vt zGwFWLVk|?ibxB3|eto=mY+;8caZy_HWk%>7lIw2jM5%`PUhlDWy1w2^mD+n)*+eWo z?w3w$oO1v9)pY$T)^jvAGt|E@5tt<|MzdpO@M^%5KXW7a_x_C<%|JgK3^#YWL%1As z=Ba-9!{Aux=E;n-5uC^MlM)U9Hd(|Q1}Ber(EaIi;V^b@aQpARHPCiph9o&vutWdjm>gDV`~I)@%OpY*{!M zS%$%1sbdc&tiWb#(r1jh(`@E(s=UMN6@DLC?kajT+lsFQKJg!S5h(^5N)JsL^o`vD zwVF2tp?t6Q8@W{*c6@vv*QDG};*5r;5i@_c3OU3DY!}x9YS&f>-@E4E4eDYqa=RPO z+IeG6FFDTQxP^&Xz)7Cmm|bs~irtcEZ^zTVynG3C)iBI14afUhtO?4CfSqxNEe^PTd-x}8D@Bg^u^Hz!p1BAF_u!HA2PYo z_w8|Z?Pe1yaw=xyxy5|kV-RfI8OIrGtmJV3wXesAY>f$H3EPd1YuMrX+e20_gkz>$ z4;6Ykyw>W$s5&X>A^%JAlgBGQej{XTq4D+L>Z zz*edCF}pn(26shdMl3GCJqNJ5}eHKe;c&rMNu0`4UMGS-iIl*@J4H z8p&&ykWe&EMnbo4c{!^1gK9c@QMIRkpEv$>KQ>(xiaAju9>3ahwVI3jB21xHbMA9< z?b3d#=2~%$_X@Sf%X5sZBm52pnBO>Sg zS`Bp>ygQTiQ^qy7IN*YcmTO(nN#QW6YgCg`Wwb7Eq$Xcd2Rd@&s)<_owPkqRj_)?O zAd<1)rg5|cn+U6bs3167zbE8V8AWn4GjoxZKl5WcgSz_d*DFl;ob4UMp-dKczU%2d zntYctuotM_@eYl#n+JK%oyr`Bw?07*=T*V`=PU)&b1i$q083__yF>BL`5!Sy^P&4S z@y4B}xKGIbnRH5zU zb3u+5>;(or_~4Py2X$6pUy2`yGQeRSN1)C} z^5Wul0y9T<%|!p+N&db$pD#}hp<8lp8Z06=>r9q`Tmu1?FqqEEumhKwGwIqP`9JEO z>BM|>=32}c#)lY8u8rvlo&%j^C!P59MZAJzB;0GX2MZ|haaXno;+wa8Tq#K-s?5+P z&#drkyuG$8P$S$Y$$o-V?Wl`nCjw31btO9YiB&6-_=gM~X1E5|(T58Y^7I{}K7>@@ z776Of#E>(wCbn>@9W+^dgm9PyI162NU#;{g;Y@>h%TZXTXsy+syyEmeTl+1~<(w1X z5#1mH_FrZCe7q0kdA?H3kNYZcCkO}|`dodgkd`XAF>R`?#epl)@DQ;tAiNsy9nZ8+ z-IMTnJnh|2S*!SSOyj}tV!E`1=EtK5LJEuCx*DP0qA6?+Ta8pn&>F%|dz1SdZLjOe zZDCx5mvL5f{1SRwEKU<9>p(Ruj2%o_1x2&o5o|38qjRZ2`JJ2&tND7arc+{Y2`T=_ z@jz3=uHv)}g)B~gsODS*2nq+ofvNsEm;coQpl*R#|Lr?siaIO5jx_>Os>E8!d^C-g z3J=1NQJCZgguChOqEv9u_-lVKE;%D%A$>jnyeG@;6rHOlQ1mkg5zpN}2;hnCzed%$ z+iBw2^jBR6oJF$^((9c^_&#yM=U5+YatHG@mLEe-M|K8(M5M@!H<-tAi-tIpEmnAJ z8~3Pi%q$ZUQ-e51zcHSpZZ{zTQ{y}EC+6)iC6++PYLtfjsp8E~q5)rb{3E zzcGC0gGoSZiX37Gvu} zTBxZ9o8p!48+>@k zxQZ9e%niJ*gxz6bm{?5_EicNU4fsUW2`!9R<3CQz5C#UdSyS(O4l*ECG~ewKa)Xee zA|t2G5_E$7)BOOzllGt7Hx2W{ z*fsbYthO(){u_b(eOpcFUBmY%ifvKDqCspLxP1ZLZZX8Sa+URne9rFxoLjj;l5_HR zO^>)siCLgnthQ}LPWcBH7%CJkGRCxhj@uD@Jd;0<_i#K_XI~_O$N6|k0OYyp$Kk7f z45fSb4wq8Yow*hky#We^ubXn~<7(W%F3?o3rTJCdo+U8QJ5ig~adPcgaEQ!uF0FVc zaBfrX82FK4D$cR>>1}<&&iUU#_ zG;z7>8((AH!118-#&a5wGVvwDD)MnSMu70)wrRqbiHBe&Ks4A|BvABJcN%W8MZbQ( zr;pgSS82lP+5-rhu7;1sWYr-I2O8hl={fF)7TG$yG-U{e`m%?8=Uz8bih+y8_et{a zzb+Hw1u(BBan*VD()E#Hjym2AHLHr+F4wN#w0#cN^?QoPm+Cal|6O(dAe@^MRD^#P zpT%5bPG>z7nHqWxGyh3eadY0w(Px5iZf_H*Mg69464y?)j>0VS;)-K$(z2)X()Q;3 zj^j+}aldYwe@G}%JUg8Y^xAck`HQGrP`;}Eaf*)-(D{u>bCEr>>>(0;yiBVy>}cx> zdj-F2*55{FX#tXGXm$e$bg?wKtuAy&95HSnoC3_Dhlc+vv0m~xLz6<~_8aYJ4e z;}l~}ukU;9gv@(7JIVsh;GqqeyNA;o)JH6e1@Nila2 zb8cJR^mWTi+kBqvF>>xd%?fjHTXU?E1a?7N8`cDVK=mQN3Vtn!z5P3yX%f2JVT{V2`eGcDAr;wTpZ`<)>vk^w9JZQ z^Nc&NEMKw65}KQB+r5@*RjTR^+IpiWnr^a<)y$JPOVFleHu!@h!!^gNP}X=1NY=h7#=Xc^-^wOMR`Pl}~uF1){{>U0V6y@*3ei?MWENhwFG3H>uOz#^lEz7KkMRR}e%ZrC zV7%*aern}DVK(T1-_@8_f%!iR!tx~c>j_tc;##%c7*Ase%z4b#`0s&^;p0adE2O6H zK`}`Fu?W^qCu?2bqLQ(kMm}l<>{YO;v2?6Olp(_Pt$=4bXL$UgMDmwM=AqQw5x@qc ziF6T3NhL~X49}v1fA%}@nGX|IVS5~G{>kWkP0Lmfi3D>}=VUeGDZ~ad<2I`!mBJ0h z6<3wn_cL|IOsfdsC=?;X0dp;WY<=@!LM~YAWHpL2q{t_-;pF$=Dn(bcxZfSe-R%(f zxzieJInjjnSPLs{L|u|x?(3d)w@%CB{FvaoFUArQsIe}a#*jLJY0}-rajzT-gC}sF z#~-=z>I6>y01!IxTJdnwFL}}!{OlG|b)tX4RfHqe%`~UQ_W^_41M}*(-M5Jry`IEu z%yA43X681T7&q&)^m6%V%#)p9)O`0*j&Sy_Xg3jbX0wxy!>rsk3YJ2~Fu)QVD|l~o zOwv3XBb~A*|0qJ%ia#}ad_KQAERk?*6Y8s2&MsT2CugP_sJEN!J=Upj;q;03B8D;G zoyN;=I5gj!R}D5G79T11z%>_6c+1@&ZHY*Xc?b!ez?+bcb~j>V;T?dWVciim18r|7 zL2F`gRGzspoinHx;xzU1pJ_XSHDVm=srfBiT|fKV&;LVbPIjg2`tPtiDI&(!FMKLX zB6R^UEBrtbU^yJ)GdR=mnLNZ-`}OTpA4biJW)XLTu)}2UeLFiVN8Khu@oxe_=uDPl z&Pug5c88E^(8g)v)n~x*gBDUwdjg6`qI?wY{|&>gxJPOea%{eLq+{;`ts9*;*o>+T z@@J#qc+Fdv>I7aC47xmd|M1wVe~yJyL=r{T;(rswXBPS$F5$Fmni zv~jy4X+UqbK!V;dlw9rCBHHyg{wH{EaPXFnc}}l?QxWYn+x54wWXr6!adPh686D?e z*e<^K|D)|ngp1Yr5967U&SEUEpkA}Rh*9aQ=*}JZQb2jhAcTq9@~=8kPTgkJ(HH#l z&#yn8);^-HLT9uYV_&yxf{e!p|GVYrUTDjp-CYPxI;)ZV`xby3m^wBF3p+#{pkN&z zFB|1rQwz4{-UtLgIW3ud`Zv4r_bt(9*g&2AtP4y*pSk^)HN{^9jr~eaX9aFe?r_Dt z(ZZD^%o*VKa(+Y9WwZKBAy|i&PH(Aa18_cc5;ct{`lET!8q3MYvYi_w@E%G%bn>Cl zz1m_Q?z~|e-qYhD>9N!J^o_0K_dr-aZ1>1PXCC;rGu0nb2uICyxyu{+@J@4|J>&R7 z?zh%cQ)S4?iqyoJ`GnuQAJ$ou(NEUE$9%XXG;fUaG`Bj&6S4R(r4GCsF6IaIe&CAK zd>3Qo>%O=}b>zTZdQHZE!<7+X=|p)jpuCgAt1>u}sc0gs0j__qdq>8kJwx0tL3+J_ z&RR}mze+8IKj6%Tj(6RBAG#?iM_{YByO7bhv1BgY7(dO#vM3U5zC^ONGnQ3>c*|X< z?$DFZYRPs6x!i4~UK+Cg+LztGK_%JTgRB&oi!31Sl8<&4afkhMU+kXBK5=?W1|4kb z`Rf(vTJ4n73Y|4O(k(~~;QH;o4D8snZz)Ze`fpt)n7UGjiH%s85l64oX^PZf-lMY8 zD=&7U<^3z!X;v2`_lWiTtI%sv<=vUxv3<=9MRVyC$@AK!+30+RvAx$09Gu7j#=weB zZ)=ifq>9v?B!~JpM4n1B7?q2IhT@&6I3js%$z^IKFBVSl9u|&a1B;}|^C}R4CB7ql zzj5p&Sgv>2=JPfpKu6vfcrZrfHtH{7(SkK!14Sv;`vRZnSZxr=w#AlzMTUwWdYWB@JGu`Dw(mYr3) z?vuT2CR?jI>#W)onvW15zzKU*!6-J0S`eT%RF*tI6@tK$A;_A}F=sTZai=)eofPYc zicL@ycw&a3Gz@BD9+fj%N-`2VMm9Y{bwfZ!Hr}bxUG6fchr^iP{6DQLW;r=`gYcMZ zl*POEafXF}rS5UQX4Uq3hoFHCL>_6Gd6>sZLkdWovBj)5thON^3BIYM2UfOQ&UEhy zOmt0JUFyjjz-LV+1dV%EjPqBh3Z?q39I|AwsT8Fgi9Pq8+1=u7hBcL5hHSp^{(e2B@B%KoOwWlkmC5U>W7 z_IAS`H5A_Qrs0DTBHgepGiJ}Y_tgcW-(YkX*zH`ybcY>2XiID6kk>cR6# zy4ewoa*-X`Pn`=xDi`SpN1PUV%qLhD_^03 zi^P<0W^pEm*A)f|j}Q>mAwZPcc-V0krlah4@^j{bT}ochOfVv_V@&p+wAhf%%T_Na z|8zXS+D~=$FGZZKvRV6BUaYV4bLWG9HH0)dWqHAumz7y`rhvymXcFOsD=?k8qkjpw z17`gGg&vpc;Fy}ZyfSoicJE0jEcBc$0Pw&n$d1{2&}w=pAcJ7uF$CuIXqK?r~}JRF-z^Tjn2J~_n@@rvww6*e>P568}=_^>ty50zHP&G;#J zbP!Y%B)7Z~zBL#zYwzWe*6PBQXA+VKrrE#ePG!BydhAq!nbkL;CQlTAUQCFxA{4#P zrYn38dNV9EE^jSqUfiWbW$rk)aDG2P1}!gph}FlGAu|xDMvl9@&2ioJKsX(k{!_Dn z0tK6$*T<7lxLQlPUb*Wd-eL7u0DGinegH=Ljn-Yngz@FrMjZ~A4roH0p1LC<;iNC0GGD620rgGt< z>?(?a@l_v9hb}5vZ6e&Ah>jf%eb;MR}MJ=3x`V&12g&@qg6KL|-)gH&bSF^vmHY5Ufr`H5j*?*^oeLW7_qy=6%!;B`y^x}x^r_k1p`fTkg~wf%%%LoS zXF0X<_CZH3fQ-hfbIFZf9{;ha{oz6<)^h-_OqPrJpPQp#%K@Y}_D7PzW7NX)Zl9~+ z2({&0U*;+B2V|lBy(w9ow|#NURHMTS2%Ib?5aEH>nOE(`^ zwVq(yH!l)vzV|FGwMG0>VMO5P$+hP6L+RJwEptt;cw%5!clgH^4FW9ZhiG|u&;bkl z*#quw#HImj%^5SN*z9eAeykg}B{LpNb_&aVtB{JB{3P!~sMn_(gMC}f^^Om>*DEL! zg?=I0ngISwL}Jv3uy{nmx<~;Uf*9~Q@|YLs3uHr`+7ot}))#KEa-v(jB4hN0MZe7K z5pA-M`r#hZmSTWCc_@F5Y%Z<=+p^%$t0tYa?Cr>W+`W#-v*~u41Izqk;2^4T@S$nQ zVtZ3Rer&4F4(OWj>qKGJy@s-I5dUk%!Pwbac_BD&36-mKIO`&&2AXwaI4K<4EYEb~ zPpZUABGeBhXRb~rDv*MS`{}^!Po~uI>TTC@<^@4E9s*~aY@o}srq!4la-mlDD+{{3 zB^o5p!i4U3aC?OqJZz^8{Noqv!B(bqfhRnqS6uH#|Fix&uEy4Gfe44ij?(gRKp zaNzR?V?5Na*`LAcQ*)gnZFXxM`oikRUBxoXYGON0DC~4qPp*f+vd8szG9jXL6Fu4A zBvIgcD3`JC6%QVH6NrP`D~%SA^kS$6?8Z(4x~`=L`*L0DDY|`YWuCGMdwIok^XG@I zlyWHTsXajyH;*Lf=w9DHIsI@>&r~>1dS+qg&3Yv@tR-V0iZMvY45qxDU1L1A{W21> zHumPi=vi1o{k(I06={!JP)AEa^cb3{&5ZtAAhPasYj^Rz&UKIvgSpBQ+|DBC%o*CH z;ceAqOMcg_6>54@wp|Cl&OeUV5A5tR^3%`jtrRFL*wra&-5KrEy(@|H{a?%$tNM!e z)GjE(6GAo*&m~)(DIMkGO@OT*3K3t{jF_?3RwX5Nna3$8-8l`FUV{5I;8yccOpLM> z?#`pA81Zu)j@R98k;i(qpzN?mMk+EAowuO0(?MaC^PlROpd<+#S;Z`NQ+=E)j$rBu z>P;j&LO^VUy1*I@Vnv?5E;Qbh}8E)~nzv)B8%Mm-Lm+ zM&x+oDcv#UOO=d0ASDIBvw(;OJDI1$=o9x90q_H8J&5NpNY!Gf$W z@QlJ(#ug0l!@*{kZAv;fKth?KtW_3PET42=P=SMk%OSWDWYXAnu0)ZPH&Uj;(j7uJ zFgzyuhtI$IP6y`r;Cg=4fY%_*n9}xgo>h)*^Y^T5XcYqGcW$e5L)_Ar9Rjck8c(Lu zI~`&mG10dl{8;B_Gy+9OQw|Fz=a7S^4`+RJA}7sV8U@%G#5sZ}y|!*I?h= zH*GfMQ!@`cY=PYST4Anc1@DO=^Al9N$>V;$&21g5QkU*p$4g^2`d=-spQ3=mmStXO zHA-(p+|W61ZXen2iNXqfyvmO~*X_~bDYwwt0|X;A8@S+W35Hm)n+QRIk@C6FfHo-DpwuF%WzXeJ(*`Z5$Z~F_kwK|+ z>GGD_M^-+pc4Hg&-0VIL17Ec4ELrx!mHToGI}Tt`S+IPOr4^lPK9 zKU2bf6iCn2|6*wLrQl3wP>fGAXKGDNOQTklpEJ#(l>D-SDQ9gG{1cwY%`xapsA!sT zPoA)&;rDtm<@_Z#r(E{?^EXBp@ulg^XP; zs21gPSl$9e+_FAXzX> zQ#cPibr;Hye2$I?1Yhzi3GdgmrGC*G`{xxn}(hDtU7pHJwPV` z^d6z@32n}N`7d7~L=49G(%vkYw@YWg2wshO1_KE$566I$hf`MxR8xP@aVM$0EkL^a}E!S3@FCI1>&^h*EZm_f-<% zuon^FEY*D4?s){)F>iUSkcu>BkCeLU*-t|A=Ja`!!;?enPL zHN4z?9Q=L$5v*+`-bU#@WyoyH5)Z!0BX!XHDiT_$BJU>HadOJ!$UCegn-uVmw7V33#D1CZ6m+&Kuv2C9?Ht znMXlAXV9Kak;PHyEFYX)vPs77OOthV7DfhlOU^!A>#yQ{ov6NnPT3&=oGZmWOJ3b) z)7fUVa7U4HH3@<%&VJ?j4wa*?=Tug{v2iuZ`!z<{iXN{m#AQwQ!N&!YNl86c^O4UI zg!<`eL(&!AtmGWz>I*wHL=_qavvsl?9AL!{Mexj<;t>OV2Cxo;;|B$hOnSQmeV`YU%yYB=5g-Gcw;*~R1Ne?g#yndYa(loBrY=?uZ|`K~eflXuBKvd5uzOsBSqy+wO(3S>BPD*&&}h z=xYg(M)-x&R=8Z!t|m$V^nnQ$F7xtwlpgLq`k8o5*;qt5Txcob9>mxtCJg8MPMc{T z&T{jkeWanJf&LhRa>^2lwVGqyS2g}k;waHzb*C#?KVsT%Mn9>W7^sNK`YQxs2N~w% z;Rf+8KlF+?{W|gT%NhO?-Tmgb=dxEj-}8%=9u)LeuGh)&@ncYaKs7_2>(M-|66`Nt z;DH(t`8n_0*2W%eMZmjnQB}R1)@b>B&aeH7Q&OoXQxlF&#I(wj2bZTKmPUxQk0Yhv z9b|9{sa-U5t)?$A$Mg64!{ciQNj`$z(Y55LQdJQz_;28&)=!0jVeu zxmN;(P3{nx>*?0q5C13-X4X5?b}9ffo_<3!C)Qz0?)hB$-uE}vH2*okzcqs&MQ4G zYXD03fp)EaIQBwic>!iZWN}7KyRz#8LL$#~PeS?-_8dit*o@KT_fhkei#Um4Br`Wz z4pT$FtVljhodUv~;8y;p!bgEDv>Doy4iPl|#T$yGW?de9?=N=73u;x?HYgM)O*nbF zdEs^91T`^7N}FNmf6clAaMa8QqOm$fKC~_~)4}S#3aw_5)+iL<-M#{v}Hz!@3g!x#_T|8%|nvwdMlBN6n<1VuMXFpgunn_|kMR}|V zIz@9sRH9n#Be>w!V+9)y%eqnN_lK*nM};cWgH@K~X2VQ*v9_?vNqL;5TaBklpw!kN zHGjy=0c;bLUHW5sQLk{40pW{`p-}E%RHhbN*ZwKDhEdr@7+AuVp4&a)KyIta1WDNT z`>KNkWx9m0^?3-Y9%Y#(WDJ9vojdWDD~m*Sfj_ygo9W+x;}h&^ zPE{q|Ty~4O^)elkL$w8QS1Vj@h_g1Q2NOaWR(;aC`J0dKW!#BUgg2EK-+h7kisnRs z9}F!VL$NPM`rzpAw=S!RuAtSKB20(u4U@Mkih7jdI{z&Wp@y)T({+|3jRs~-!DcR1 z@Wl!iF8rZjc!Z+tM1T)wI)&fgYt+^>UN7{w2ZlU+9NEj}ie>lZFLU%5iNXm`VEgm8 za)B6})efr_528z5Mzi8`3V%_8FUZJ&L4gS@F--(eur(C4YhqGITpCI-gLWOOMzArB zNG%J;BMs*Oa?WwFqvqp`W%f_XmX7xGDG>2`vva6y{;JXnDE@qx|YT!ttTcl^Zr&^`tlp zC;cTQy$_#D>s-2DS3M7{cfp>VzRAqJlP|QBt8yCanfrR~XSE_lDo{|_Xc!PHhh=8@ zhk5CvcMAL^L%kdvnvKla)>;dzkbkE|qHR1WP53aeo!nwMrJX?W0n*>Tnfm%E;c7}n z3Kf5ByM~>Xx2DYO9>YRe-3v4Eo$8<26zyEK7dF!$*_$@7EnAe?#)06|&f0M|RuuXG z*3~OyOypW+HU}rx5grOsqKi-CP7r*GD!rBr;cea8KMX5HJ<;XN1W_yO?7UKWE9DKd z=PI;#Wn2`0((U!~le7Qyr`O7L`1YeMLnFnNS2*?;kLKQ8qf-jlt|| z_~NBtdowfbk&*g~ou@+y0{z~~R9)Jy4&V)<74~Z@Dn2q!N5}VH5V|Yit2J}*l*VCM zxEz?96Dg|-ylxhMj`Ee$kDag5$7j~$zw5ZZYS&CrRnhvUt52TSSn(%auuOen)!<*9 z6q}gf5Rd*PZ>0W#<5S+h%$}C&qXBhMkt2o@#zUobyW;N4+_BoE_Fjx_AVQsPWBf3AZ0Dsld+oDi#298z6WXflz|4N_IJO}uixw`m;(~c4`DWsA|BIZ&bn;T*h@dL zI6ml>OM?ptaaND{@mL{`s^17EC-zs3;Z~!`IHd-COJ^Pd?@g*9N1;w0t|W`0)^(1v719baGkQWUvV8PhpO#yV;?+<$d1GBjwHnRXxnh)d7>1{lqv=t@x5%F|$ zC#+(w6&=Z~!--yK#0e<-ag3w~)X`?9WZ>!5_C?32=)z3>`O4|rSVcy=$TVei*>s|Zow>YPpslUlQ2*DX8W`Y%M6^(JUl$j z05(~|h40z!hlI4|}f;ZIrYjPR695Qu+$WtT?sE41^&6LG4V#mmsQ9 zM@-5MmuG+9QHRVJ+gfb7`!zGT>BI^nA5b?}t{dPj*-|G+8nwD{bp!B1*bFP3IhLFL z1a>`qu`XYau-9T^8@+}9aT^=na2iB+rSzSV6eN<`<3RU6O zVFNN1*^{EBwb-7UtYVy{+=hz4^fbLG(vPJSPt^wZD{s5au#hE$A4>z21JssDiK(OU zmx#E(*Ik=djBFG)M(Jf%ZAeY0l`u4wN&!A|t&Mo%hn(hwvSq2=$&HA^bMz;xu@?!; zmkco*8m&#|+g^U2NEY!8A()r*eRD40On2btm{T1uLL|}y1nA`O$+!Q+7(_`WHYfhg~ikukd1x7fr`T6 ziBH|g>taHn&se@*b>rlSG3%)~mQmpK(@#?@1pw!ks{E1Ex;kA?h_Ua7efO6v;*3w~ z6<=D0#cZyY7;;wg9p}7kJ-9H|0Yp)USQw?Q9T)es*tIKT@G%4fp`e`u0xSg?kZ8w+ zE47UFIOy3GrVGL$e>mh!?}zY#i-Gq$k_!<5Vn7*HcG#Mn zoEV1r=@Xjy@)$^$ahcwK5T6r=wfVgNUFaHTzmTG9+2o6&2|9Y2)52+g7_F@g&!%*< zWnPJk49WgaQKSHhjMV{D86h)rK__J4Hg}VO1fftk7>pSoBxo`@S%Ze46JNv){~QVH z_THES{3u?Ulxk(cNkJlA?bO=j9bWbGarNmceK3PpO$sXvm6b)ppDaH3|CaO7-zhrt z%L`{Xe(&l~ni#y#(LsP$uFQb_41{mn9kc07es_j6i=d%jJZ;@uF*Cko_f#_m zzcb3{ue~x&5NmN~-4koI`V^H|8}k0vPbwT{7tD2|xV_X?W+!QDf%lWO#C1%eo<9y8 z?9}eVb}VSiUPx+E(rRtS1h0I>v3fK7!U&?*~qtsjx5Q(_? z_x_(=3Xx-$fAQp|&bsO9Fd}wOFBMw>Rx}*; zAX;%TS;OW03?nm0^H?OH#Y9miEc3v!WN@pD2BWjZG7U^u;FS&#yLISWWLY-Q(B0w z5rv2Ap^UsR^l}x&1kT7cy)!&ZpA@m1%IQ^% zsJb&KATCpUjM;N!w2d(@_SxPnpgAbTP&IdeJi!35Ngej_FJj!nDIDWwal7FWnws4J zGZrrh0>QYom24SY*6s`?e2Fo!HAqJKd@P^Jff4q=3}FibVt&Dq<7000Zcm&b-4a&k z#U-ur-n_10>|w@UJi**lLwl3MWiBnOIEy1{tdr$BU$#uA>-$-g`H$P(-z+UaP-(IF z(?_m8x8TalF5qK8k3FY@<ayDbJgKs zRHR;Kx0jdK>&x}DFtp?UZ=4%|i{30b2iUA_s7daYH#;okwTx%f=Kuw}oS(jt$YW1h zoS*$Vf~z6!pV(t(33!2zpICRV#+#^@jI5LAV4rZne_?JRrnHu!@n!a7Yeg&-JHV7rAgInHB{9^KA6=i{`0iI?DoeU- z@&@IK4P3lPA#7*Qve5s8tKh;s1|~<(xS`5g!^eWkmW{PT?DZZ5$EZ2&mq)l5i+1b=n5Mv#BAR^+2V!T(_X&gw87}g zjM(*0{OGS$?94B)WlRmh*JU?3^l@EhO@X5dP&DSn!5g}|^1;0Dy@%XhY@cx%FoJ4U zFkO5aE4>vYAHLqovEo8$&(~hb&3Ih@pts>2*s_<2EjS4v*)2o$s#y}vJI0spL+?RT zsPQ6ZTlGSDm7+ptwImdmE1h9JbKMksjZ@Hm<732xy&Rb@MzpEMI~ry&zWfBWbYr?% z$YF&y9@z`N@&QR|`~WiNGEYlLELkll!N>cL;M?zb6`ud1UzOlSV{=4^_SVX{8Sun# zj*$@H3O8uvVsYzT(mG8I2vJL$CV41YztJTe3Eg9dH(1E_N1SEqWLi99(0QRri$Y~# zx}0slC^<;i;-id-&-V?50Wvpy=c_vOZWcbocR6@@^0`?ROj(Xf?an_AYt^oxZ!$k! zd84lWU-sxP(EC>Yl-I^5jrtBM-*-no=^Y0khkHTed=5A$AHqO??3}lCoKwG@9uip} zRTaVIs74swpkyl!Xl@V&!I|lut^S3tx3}-+A=p|Yzk*wV$v58wG)=*_4wZ&&VDk(7 zuPI+G6eFW3JVb>}_!=!Y=?=JTz!5^kQ!b9srK}hm=t^Qq_fXIpI-H65@@i8?Ah!Ui z2S)*<LuwHd%R&w94&`w5GIF=~ywB zHbsiHFFcc&K?*RjzvnQ0ZW_RZCKWW?ZhTW*=IUls8*m}U{=Id0h!h_jD*Kwzwi0fB zOF{;~cNKMn6)}F&!J05|=Hr1z+LvjQ8=MY#@fEG;(3i& zDX0czetCEtOE{j!JnD)}S$=w$M6ur_0VBSc_SJbU>tC^B4kP6Miya^IRn7;&ADH@3 zgzVf`6&KBTT5EhhaYq@>5@JdU3M$62FSrrDR#(=bGk6zz!4j9{!{Eh3h~lVDV`yvy zUH*`Q0Y_FMxPqxAL^xk8{0&Wy4Q*ftN0Qpd3;T^kG)!=Dh&Dg|n8hkPAOH@P9F`lL zlb9?0vN)HCiZy_~@EZaL9v-3JYMj*QLM~}4#WE8o`cZ)_0~4s#O_o;~TyL}VpB3Gf za6%x9l5_i_*|Vb~U(!_qhd95>UaMqrpmy42$JcC74zszJs@}5=Ggt7$nkNs{`3BIC zr#X;;tE$;zD_Kh9%RmV-2`ps&3SH2x3?v+n<8aP}M&QET9MPO0@)Q=g^p%iH3rl&R zWO!!6R+C}zx15z#UzX9n#H!%7G?9(BTq&oxn)JY!S&p1B>yx2{fpe)N0#hLd0fpx{ z1Lf@Iuwx9B`TViBK=#omro3s71uFhAA76h=mJ#vldA$zv|sRQMv zixw891ox?~9^tX(a5oniQxlzG$t-43QV_Sk=(0+pSjhYenqGhX&X#+}Y(Y^BG?hr9 zu4fR69e-0BB~Qt7dS$ir@8LJpt_3nuOyo<|gLN~>VD1HA`XxXU!(!5Whm^m%v4h1M zhjlqgOTd~#EuG33z#YD1;FW?#3@w$6klc#Nc&)+0)W@p!Gf*=SE&s*DmAE(_Oo$wG!un%k3+Gs1YEk4Q66k>Ej%W2<2ZmsF)S{1zy-y=RE4&zhbhfvC%R# zi@Q;>j#W`i6HiyhuHX0psBz1(Js$&{SGvYw_kLjK*w_eXWJZ^u)det3Pghg|Pxn78 zGx`d=GiumRQP3ai8{|HJ{z0lA3D1Us!ybkk&rNE6N8;^qCe!T%NL;wGwh+P(siE)r z{6^~B+lKsMbv+CoR8=Xr>|+fP){vOcHK)zd0N3=nD3cJ->YFf_I*+h(b|6xTV7ua* zxrr*?9~a~4M>?GmV0N?{mP)C5^vj}WT~LIW$ku>xgCtdjQNRZf0?1-cV&c!v`sk;vb+I?T)|N zp#ORFY>^scJiR?lXkPMu)F^E>qIM0hKUo{(U64$=5sOH?A<}@u2bE0po!KNx@G*1M zH$pA*)-7>%`nI`Ji3{dSMwV0WDnNDlBNSez0#a{9oW{93S^=)nI++FORjwJqJk zI3ZYY2$rBhg1aWcf`<^C;O_2?Cb$H5Nw6fiySp~fxO?L?PNR2azh~cf?{mL@-~2(Z z=Yd`wd zP9M?WMfs2i@rd_x=fQ~XD6f5+;P8n~B)LP>*`ZO?f=#H2QBSD;QgC&+UC`AL{jeWn zRzk5toW`(eM_SNz&9P*8NM;}x#|Pr-d2SB z+|1A`tnf`cXT+6e-jFR>OB34|bGWsAMFVRj4qg-+ftwNIBam52#BjIgWhAWfMoSCv zQnheJxdFR~9v$m}bct>Ef4gV{>p8Y_lkEK3o7_vJ?pw`?wTJz5Rb<)){biCn-rRdF zpnv2|Awrb(o8=1Ky_*|6N1bze+rL7@d0f(oHge!FOVs@MXi}f2*1YIJA@vHsB~7p1 zaxNyX;`Ktb4kf#8X6YCi0`ea;HuE~%)ZS0hDVV59Mpc`y=~YljM~wfA$148W7SJ81 z5iLJB+PuqP*tI?FIrx;!@tm9|lQ8D}4?9jht_9R^=l)UT_y)XXb2Ppi0^ODO58s1k z*`l8{SS)ReVTLJdzGpx4oD*V}V-n3I>xL}G>Vb?ovh{*H+Fq+1HkaEh1m_P{u=L?Z z9!RW|mVlF{VOYEGKeZs$+79HDQ;6t@4kloTmbaWe!Hv{$d<%cJ@W4}f@W5kt;C_=v z2D`)sI!rGTn67>zvvpuA?!-xW+dvt8NDpeGgp3%XWO!QtjR?sd?JA+I@JaGG8_y*1 z*pPIalXAFccP5A(6;u0)OOV*!PIZtOyr{PoR$#3C=Ct~w@m0ZlX$9|q&|ymv*sKSB z~H28_;gyTG#1Vnsd%cd z&?@)+CC!ZZAG;k*J+uXh5vzY8-@66)ez_~{hCVmr=oI2fw-LQ;Rom~X#1vAvWAH#i z_!DDa8f>wUrQ5o{J#v0)aK&usQHwETuU6e7Q-DQDWAz#ci7#h6FbbFIF^sA+J90fc zU9~`y3^B5s3SmU=S#?{LkBdQZ0$=B|X7Wac-k>?~8R6JVNI;Q*V!o;$x{?O?!&2*) z_#zp-rHyc`6ldN7k&40ELU)Pgp#gii&$li(n7tHp z*=s?KtL{8uBV>F(A-7E+1bi(!N3cB#F2^b2fc;H;K_DROzsvun~DX;kBAp9J&lBNXB3 zZ_WJa=XoUwwFh1PqaCCO5|AU2Mf#Mx1iFbBt?SDTI=KaGGs%_r!*9P&sd~Botp#ua z?O7h3%nttkl4)M0RX=SXK1UmJfQ(9#z{Z)<#BKCJ2d9NQjGs@t$S*h97)G1(3!{=- z(B)wzrYbh{!Fb1Nw!KJIfwZAI{=l4`%MK*u3E=nPEf?Y9m6F*k#}0AokOjmB zrNDeBGP?pXc11!-vW`%gtl>V{*h5J~KCzU;U0@Me=r@B-?`T6w4RJ04{%@X+&trxh z`S9_{`$BzV%^%&cw@I^}6Ujii14PgW@a}tIpKNyKR>}Vd^8V{5d;>0xWwb7@?7T47 zeTEi^jRbS8bmThLAhIF>IH<-xpUSx)}j;g9gez2iu0kIVIiMCf>EWF~WF113b#3!k2ynUCY-56C^ z9^dLi2v(Ley)Np~JaF(V>S6(ECd<()pu$mphj;D`@Fa_}s0WJ6ia*%V3L3IV6f}3w zyJnKkct0x|9h&}74|pOa+0)7?E&NiM+?d z*9JPBj%U-cO$%vHcoZ00ht4re%}?{ZN$Yvyzx)c~h{_)QhN{m^Os*B?ObS1JIuUUQ z8yArFiZty1=4z8^2{6ieK4MXhWqx0$>_U_oxJw_f3L>@N-U+m)bq6|bXl42y-8aEg zU@mOAtDC56WGK}U-$CCnz!m$~jt6jgdE*X=mQ=qVnB0$|nWL_wJlz#s^f-guTCkn_ z;dvWmMs@{Xy#zCo4_>~O6krY^$-3UjYTh!x1!&+(fh&jtl=XRt92^r=-!Xh=g~gyD zob*}0!t0cW7{C~ypQWQ;nO?&>65FbbFqz2pOm+_F3%S41aas=xV6jIVZr=!C@nNp8 zQujn)vIH3@WENH#y`@G1E6iIliC-e(lo(hXkJD{>8bF9M8sdi7r`3O~BjJLmRSQ{s8P$+_d-Sl3#t(JG~V;IZ-E|S=`AHQ1* zOGn4M9vcSdu&-f^W3J=g2w+@?a>BnGm{lH z;q6A66N9kyse2fYC6zRU_&Y`2cBQ+_$ja#xA z)RpuFRd#c3OTl^7zcQ&d-~B}cvoMRj@AXrf_-$#t*f!9sg4pY?18YId01IzZ5x#4g zU~L^?YRvH-i1t)iR)Ie**&p?1)=_U+g#Y!>*&LX&MUO?nckIgO5_+%8(!=&BWVN@c zpXJ0}y~gbGy%oj<1m~ydTs+&OkN1q6ROy=B?l`{$OL?r^**`Z$8|5s}nUyWR9k3^z zzAq@ci`-vV9#Rthc{LLApBES_{|b0(o9s;4K~1b*j{l^uN&uc6NK+nq4T{xgNvr!JFtZN_-t9^TjhqzgTt6|-op{(mLET% zXj_8m*B%30wa_=K_v~ky4mf=ZwlKx)j`Mu+hXVE?U}`+D2zU~Bt#EirqOD>jnAn@# z4<_bq2{>TXN1)%h7p5l5n||vfv9B_#3ThL-dvZ7|#zQXjG^P$({_BUR=*`vl`~g(C zs=2!digowxjpb9>;485@`!-HnvfEJPmHP|$it_H4Zw{Q~77*lcWe zan&=+`v>-sIf+hp5`WgZzBu99!k*J@vz-{gRcC`eAO3RHoD>_r7mxt$cX@L+#l;;bO9U0~tpZ;9 z7epDX;L2)pWQ%PpZ%DIIjLgb>N7hw9tj`-Uo=7k~sMCm6FqM75ANj(WtL3~&>~((RUnai=*F#*^RL6o6p_&8sQa=r|wN6Yb21YTh0AQ0O z3>9IosCLGwa-kzGlrRLn74T{gCvxhEuTa#HcewCAwA^xuu&u1(GLh_Ml{_(gT2*0x=6p5p};zM5UBy`E|DE9$n8nhD9By<$xp=GzT(t1 zRBowI-_qM9@ZXQ&PT_QWQbll3-CTv-3UAxyF2&SC6g%+tpX-eBf;g7WUlkr#-l<6OVnoz+fO*%@t&`T(+Ea9z5<@~V28r&^XuLRI2xVQ1fL zp5s4exsklt^0MD&n-!h{H}}=^i*{q6wrgzQ>=AA?Q>HQiGi-%`KCCEorP~#x_KDTn zEQBDJAI`m>57S0Ln{G9*NWM9=*p409ws%+gxVAcbi84VEW$793zJ;MNU&k`oUPV^t zRv+|6Ss>|-VTk{r!Dcm2{vPMNzKD211JvERzIO%m!-*A-Qng(a&1vSq?IRleR*(Lv zs*w1I_BAa!I?mH*6V-KHboRP4nV)vY&L3Ph8Qr!8a%f_5L(V!NA=*lQJHi0z0i0w% zLk=k3aHl5dG@Go|O*t|=$&S@aE1r1}D3ZagygIe0aNqgNN4elEbqewAszXLzURwWC zTzhIC9g*QLb<|?+gI%VX=!kGJZ+pf1A;pO#o@77r2 zm95OBHt{GE&Gs@Cmd_T`Ah+1o1}}9*%k>X}&q&waF-P@}0wMO5K#|wqS8(H}ZG^yM zQR$JD3#SzD)TcXs-p{=E>{EO&&edo<}az3?Jqu@1*LB?S}<1VQy zOWh|}@Rk{NAo9u-+RDj!rfW4NhQGtei%E8T-VV(DDIAC5KR7~ z8OS_5u-D>tMq4;zqgZZ{nt~)BJO=XG`*b{)KscoG$`ZRD5@_7n@iedha2zdDpF7RZ zdb_Uc7R7RW2FyLbTCQlb?bZ1W^)xuBtAZBF$8bg7P+79F~_XmN|I2=HR}es%wm9SVfqJfllQVf|+J5 z*Dr3Juq5Hsn`g*UYFFstIOc|!UMVWOwLdN>c-(lAhU)UN?|z5m&Uy|9n2+7;@K7pT@jlF}>E0oLyqr?Kez_3P z1?f~l8@uY+y#_z;yXj%s8TTUvOCht>RM>i_kD%gkeD*Y37>${oy~b`$a0v z`7~M3_6;s%h~*MT9q%X=z3|gkyz!?hoN2PZu-GnsJP*GGfV}?fQUh-4Q7=@TW5jb( z?(Ag*BKW}^N3ztu+lTelaBpTclk~p4X91PI0X21&5!;h z<&t^5WRpPw*WZCOI$~MU+0V680g3amy`G+)_#?oHB4h};hQ}k?j`H11x^=ZeZ_~6F zTxr*9EpD(W-1v5W$ewW^TplU4ti!AHJhNJwAP?2NS!#@NVaM!rcV{vlCYO9qAMc1l z-xK{;PS0OkCE(#`s1+r}YD?Q&ON7Xu^RK^H(?t?uoVkfI{?*Yi`4w}e z$|sYe*ak{Q+;0N*_}Pcxd*eP)_OQI z6qqd{s1}p{&BhJ9{GJU_#UuJNi+BII@BjH|d*kWLz@%TAZ+P}H1-W_7?4Wa1lc0f- zd~6X!aNOJ){qF3(+pUSn4~T>7U~_|0dD@h#2T~Eb`gnkO`$mHZajr_+04zt!z}5n)Rz^{(7bko<}m0?UGW z?i5H+I5oxwyS6y8$w3#0j^h$O>q3$hB})yTj$LO05sVw3b8jj>qq&~`(IQKtctgrd z*64gpFM}nIXU@sBv&rI}wA^m%Gp3|qoMUD0Dm*2VO3xl1y$tG@eA1tgv{r&bm;&@e z|3kQ%d~0XbBL|Ykh4HEtdvBJ;dS0E^n*&3fduxJfP{OS?VZ6XG8Q14|{9BlWq)tJ+lh-S#SDllLfQ^^4$S*kCbu}n- zM<=j<(aTN<>d$t*H=QtSx*~QQ5#;d0KZbe`JoGy+lFfaw zW%2uwC(RZes*7v)a~JhJFY{oI`ir#@XDtz!?|RZf_iHB}?!xnq*M9^t?k20nbj>%z z27r7>0-B)ZKiy zzQ?1#SClO*UqsDjD1m_&It!LiKd7`XkLdIl%|x#XQK9hKzZob+sXeTVtK-k=A;>m8 z91V1q&`5ZDXfVGtofTa3(taOb9hB;+=kOI%Oz|B|@B#T3dU${lj;%6H-HBN4H_|Jv`PUiH()vrqp|v|3PK2s&%zWng5%Y9?8{wO-^3QX}0ne z6Z;Po5)o`R7m|L3RC65Zs>fw@KP)zEB*2-Y??GyenNWy>;H>+E`ub?k2Ve=fEdauJsm975 zeMi#G4Z?s0oqw+`*c5awu6z@E0;@viQkWglhejWcvj> zmSzQVR)GPM$l<2h!mB1aD4<``tr3b<@bP=OmJL|DsyY=-$Ufi%x4lB%ZpG9RnjM{# zS=TH({rW$K$%&E~NZ)w`FSqomsck({`)PGE98gS(XbudRy)pM2jG?9Wo0-L>+$DEl42llPH=uZQ<1LVAOMb$t z)u&^fnZi>Nt4fF`NSSL$3aCxvDl7y?_X@F_PZLeJtxfcaNw)OQH?ibrIUYEx_#YTS zP3b0zKEb8&+?@Mp;}t6}8fCt&*$2hF9=cxW7ceVz%xvv!9NFcv;Q`SEm3yYw^RXJO zFC24;mst%``v-`g@=D_#N?Xvf9DvPxietMKb)x0LcaobzraV}=@@*Y1VKzWt?L-t|Y|{O}T1S*|(_`TRHBGJuT(v9=*|6)5VB51gNsPNzrLfaV=}L>UYG zGrZ0v06GX{1cL^K*Ywq!;K?On3KXiIBV`9eTJ6IbsD($mdcy_m%jOUg# zd-qEc)d8{m)#q``pPh{!izN#*K>cV3x2OTJJXfL&;!ZdQT1O=Tw->%XO$JnYMF*-hZL-PW!#I3O0ECwSXOLx4(hq$|k z%P=>SN;N6IbqADHoH^R>i>&1oO;SNalN=Io9JJ(-Z%m3El@`5)l-R_iS(Q;72%#6$ zB`up5^Eh(npkdCLAk;*3Ny`Y^HKeZwdn{Y(gd`rj(=z`i_Zz}pS3Yi$SdcEX{De!s zhP+O86GCHlVXr9pNtyT#zhms`22cBHaw)2PG?Ho#dq2&6cXIosP)i$*cb!LY9qy{t zF&AF!O1H2-&aqN~j0F0ztEG}=x{|`;Pg%m?QgSj-wbb>w>N#HWYFU%Z*isX_Z4 zZ@Y2rW7-gF%R9PpBia*5K^V^Jlz`mp)OwS16F{ ztksPnHU00nGUcYKqk>bm+lqy%I}+XbY7dE-vTp|kyrg*PXIGS9T`K3#GkxxoowwQ0 zAzn45^%Z+78Cg}QYiJG)3d8p}(vrON+dJhqz^}P)W$=h?3I~=H!*+N=;pEX`JTB&3 zwYI81dWE7N2b)Yr1dQJ4a8Em)P_u^jZ<$&W4>d<_b>mWko6An;K7Na#_2l}TM%fa! zWk{d5?-BQrdw|U$u4T`8y(z?y8v&O*_PC^%(^}*t+cd6)IiNNQi!~sV?0zonUIv#! z*-4sNLPxuznpDB5*C{xdvPssAKF^PNUsZ+@v}(UfEt>MBdNKTTKPjGr<7rb}KuecN z+(vFW`W1ual2x$Gtdi*WMnkO6ak5T#9&xRM=J&Z2dQp!Vldf=^fgyTVJtXA$ ztTYY-lq<94MumdzS!J7|g$%@jhP*bCuOqpQv%jq|Spqn_-fLE-9oT1@l*!Q>n()7~ z=VvYukR73K@asY5-MHjvR95JTJuKb{q7TCgEe`E^ax~JcHbl1MM~ac1AJ@nne-388 zP$AL_#!YB&%x3Ew74}cUU6wT*lzd1!$T5frnltXJ_#qNmyLPF%koT69QU@N#?=8X& zIdinr8kO-&k2t%um>1Kmxi3MHb}O))<0?=}m7jKF)QS)@Fhi^M;X?AW2O?o{u_%n@ zp`5YU&Os59(oDf(&59%%>t~AhN+6_7%Qr`z?qLz>-Er7(lh?AF8$CKFPdyOYywGaf z4Sbdai*Rs+afwtj=+>{9*UrPd z&@)Zwm5n<4aO9c~XIZ_hfGQ0de9jtTT9#pvp`g@X3qhU@e*8!!8Ef;kY1WF?&nQC} zX@8%dGu{e3P^ z81=Tl$l1{+@xuuMxEycdL5m93&rp@KgHNd{St@OcFpWU7;8EexPas(xWf~wHIkR#g z9ZJONx6=pnHLQpB;X@e0`H3lBxU_rRXvjsE^VtS$i95RN;rBgtHx_EW;5%%Fkiq+= zySwvO36`?nu%xtQ0rL9U7wa(ip;u%}`J7UiXs4&u6mqo+h1L-v zq_Mtf-`8;Oku*U1V8+clAfYFZ)%xtVXX@ocp6FcIA&>*9an>yBJ0YUlc z*hjV(?I$6+#E&u|U3D%JS_lClE9dK|XRmJsDSFHaoOZxQeyc==OL5!&nV_444B1Lg7BaxDlNm_ZQ~M z@#hAsmb|djTFB*+35BOptw{-kp~Ykv$X=x3i&kY&XEyfTv0Y@%V4iuFJ6{L5DYUtn zvM;2XeJ)9{bRopQsc12S$LC3X6JkDm|JAH}@hOcHztGy~i#sLmb)eT0uf0AFnO4uF z{ao4`a)Vim< zOmO^LXIY(QoFmVK58@kGGqvg&g}T;_7zbRD-($#FDXAm?`!BXkESZgGpY25XKeXAY zjz$|}vV|8TKyY~dbNj&CK z@caD~QQ<@p5>mwFSA68^wI7;upUki^TE;iU^1*qwaO~)fP5DH# zqE?r^1_a!-FgygXZd*+iW-ah%Z?P&sd{>jTPi*{07W}g_1>756&G9DJr{vyFVGH|e zDz|aW2j6^4gdtWzSZ)lqHwCNZyu$jZs?W-z-ZZ3NU_#>nZzf}NTkv}cW!pRjwcsvk zmo|D)9j?ute+4Es(gV~|$R$(3$CF<#rSj{rtCUNOS8O!q47YOx?%I;x#sGtQ^HUJ4 z5$~0_N^)`?HfDZ(1Pf)(q+%F``u>PV=SerV9Z$I?mfRJw2HgXN>5=%GUxh@}?764Q z0OF1gk)5A({G#NU?MW8IGEF&bcu7uO?qq(E9hL-Bp#vWTQ#usH(AwE>DG|(m>z|!_ zoxGs9g^J^%Q$&Dk)i>n@dNo8Xt3d!VfMExoRTmSC5ElzeM=GJd0=yO2r20e<;*15MwD{Kg0) z+Eh>dU|_+GucM4F!-f3d@|84LKF7E#Txupzue-SpeNb`H`!fXxqOQrPvdLCZQ0QYb zsul}G>x3GeEP86Q;kQ^07n1{PVU7%cXOM02T?69x*N23H3~|uQ{HauGiGF$>I|V=H4x4 zvDoDg0GC3A*2}cA%HRpfqVdqqSJTh?rgI~Bf~UfBKl`?K(og|LrI4)bOtyzz%IiEZ zyU9@9Ts>bZNUcYbPYkdIl1tkz)ZA(6=uoa7`?_4dwFD3E#2n@!52_G8kCW|yJXIMV|q;#4|@8TbU3LF6t2YXL#Zo+N@-OEw~DY~ z)o9}rwUO1l9g;X%?<((*7X3nYm)}rcZ+FB-s2Z zV2#fH^F1>=G^5m(g?kU}*<7JGWoNpvxL8wOye{^C%Eyod6#U7l9ret-EJ*nJ&FsY9!^I{(jyE?HqZZQLHnAV8IpLlE-a))qz>0oL_rdX~fyy za=aPKC1CRLqtQE2Y7v&q7?nC4J@h3)>Pci#yoLwO=U-3i{Ww_)l#Oa>LjtjX4L|I= zOedg1n9oP8pcVXA(D(|vSPY5A@%>B_xpZmB@7-fd;u8c{K1QyKjZc+NPX|(N5h27! z?RBQ)Nc)+srIq(My{!`wt;j~H4J}Nd-eyIUTyBxi%=}CXSp(q-pC&%WrHVQ}YiYbP zb@U`%5>`(wkOy3aa8m9>s0Z8}32h#P+uY#;5#Kg z$#m<8d*noI!&jDjK{oq*?w51*do4J7ivOCU%h!;A5x4Vzhc%jTH{uOXa|GW+w8e5B zto9YRG~=Wu$8FQNn(yn0b!X@NGpyLiurkcjgHL8{)bcZcRXoL^z4P7wB}~02*E{%H zlR9!cR))IA-cLxLzj1BMvlq=^7Isy)p%tered;28?itKaLI|l=BfU&#sYx9Q+y939 z5aknn70^TMpWT!9a}4B!-#zD{iLon`bzM(-&>QCJ>%J5Q?Vwpi@1UAaWwT8OYU7z< z*eG7x_L!6|y^~Fp{8mY+^GaYxeid{{P_cA@pjaZcX*=h9`i0&M5ALbdd}I~iw`6R) zDi66mNFd(+@F5c50QxqzA*Fnr9mPUUu>pQil#dYn9ZgH$+X;#Mg!Qk62>f1@ul;^7 zyb5xvx-n^WisvG55KMthHx-}TfnjUx_#SCx9ear?!2yL+zhmQo7x>%Yx~>q6`C6dd zicSAe2yCrGEgazfc6l8)H8m~_g57#m{i6N0GAr;ej7YaD0a$`gqD|v3U~_L>VxDxi zX_@Qo5`CSyWFMH_E|~osKE?jz@di!0Yue=)FZjZ|eD-;U+az|CgDA@>c4yN??&m6m zyDt5_K1c_nbfkkI6YCj0M|X4&GV4fIfyDKA$m4%t3_I_e^F!(P5-dp!oIq7|>d4&* zq=*m~hGyxVLo^S--}Cejrs_F(EwmJhBU30?Aqn$o zLeP!g+Dyujw<(-9u^dmoo|DCj!suK_41OKDtq2k*ACi^Lk7XWwO3Aupfq!~u(D()2 z05@`d%a3!Fu_C_{AL*f_UiDFE2~)EUtMJy>`KQ zatCkVct>L#_9$R^cw#2E#6M`qFfGqyVaP3Jx ztL;+YUCo{!(hRG(`6P^#^w*^6Z}|A959q}~PJH679hIMW;v~2iviE`655yl^3}Txe zmNkj0;Z|)OWV~I;#n2Rvt}9>*+8Lm0fj`zas82OIr93VR5L1U6oSP5HX5u3kC8@xi zd(528nN|D19e}6jG;b9bBK~D4HfaADDb=b|BsA(J1|cuC!h}Xcan>BUPVf6=IfoA) z?cyG@*#~O6pS77#-BVwR-ZrGdS*ghT0`CnY_2Pz-?44zO8AAhOWH_z6cC$@V7buEn z&3fA2;*qgb#l$c>ZqF)}6mIV0l5)-kl<43WwYdU8a#%C~V@-a_NUUPR*{*uQmV`lm zc%D+3aV~XyW^(8veM#OQ7kHLQ+@1Hr^98a@;VO|}ny2MZ>V&9&Gp^UTyS5bh_G{7} zeaK6vYu#-{Z|ujUPAD6uraPN2+2n@_ck0<;Naetl-mR)FIh>7;hTMh~eg{W3fvr zA#3mR3K}d(w-JbC9`Cw%kDt{@nzu#ACta z3sj|A!p#j$p07tQ(Ni9k*n0~E*?r0XOs7}&!jy%kmHF+mZqm%$%MtV@kGjiyw{|_e zYH+Pm+YRkiG!Vq;1$mv+tsY<+6qR&Ku&=p_kSYTQgqx-b;RauBD;aV!*MqGSLVQ*Dyoh zQ}5O5UavY2L*H9h#U2Jg!+p}$q*b%ULi>s{soe06 zRxGl*H8Z9Bh{AW7<$viejcV?tzUBL_D-!jFkgvBwZA01F*6#=2iXRE`gk-V%wfDw_ zk6bK7ZpPO=Kiy>kr6iYS2Z!}8Ny$+;jDN8u zG)|^xCq>>PUL3%3T{YuS^lBcoM0Dndp0Aeuli9hJ~kpWBVAieOiU)4LJ+Batt$jRa2&8f zzQ45o6@ECwOOy^Qy1vIFvne~4Yqu`b$Uo2MKV$Qgd1Fk=ZOAdZF;J!{xzE=9?kI4O z#HjqZ@FRjSS%1Cu|@bJ7K2CVVV}w!Ws7y>8wI+<2U%C}>hF*M_C? zUuWn~d_!xYivpmLV>M9gzIg8ED+CG~-ho9+ZPwr!3m@gNt*1ah|F2<{B0X0BByEu# zKz&@|%oLEA@j3hfOmnK}Sbpd+%wT^zV~BWzsTn!cW(Dc0Qp)6j%bE@6A1?zqjNSK_ z9{Y)Sdr3Uy@`Th~3g#0-`fhK=s=|>wdDn0)D|-pGHuf-KZF} z(RQS9{PA$cg2W_9G?g_=EU@B?>-(I6SSG$yFS#=3^cq_6m-FmCnV2%DG8mh7h$Zp} z86Cpb%=EC#fI5dWVHUNK8~8*ljX(04{S(De=*fWcNoTjN&jOB-$3fMX8y(Io;Q<|? zOuS>pI1M9Or*kM)o9I{6(ib$;SiJxSlL(QioP=t5NmjSpv6z9@Io9zh>J`dS+_o?3 zZpM$^3MWaholZQ)R2bv;&zc&c^u_TUTI#a6uTn1|IbUKn!+aU&Nft5xdAGo^1_N-z z`Asdnpk8c-^S;Mi``cGHqjM5F?M!!qgVBz?f$B)Y*xT?`%Hc>`MNLUYhGH%cr!j*p z4)O2DQ^M{ zhAi+2t>*B{qH6bWmNH&?O3~(@T1@{K4F#4cQ4uEl+ctmZhEimPqfNu_J?UuyN}Ya) z$VYp0Of<$yE-*H#WA|Z9Z*9@}Fsdk4@&ALMShPGOy4OyepS(iSWoj04Wz%9afYKGz z+DQ0#Sf}|mqarwa3(2c*8K|r|K05+?R#iP7!=t(ONGp-nCSviV6l) zix4<)jpKJSO5}QU+v8y}Tjj21hq3A<$9RsG{CmyQIJtfd>?Im?OOFyl&m?S&F5paI z+^ ze&8%!@=o}-rUk%TQ0XmnUys%lG9PMG8VZ4O#XH>W$!Vx#3YOZ*uy*P~`SnPpr)9rl zDe>#eO{wKOjA9U(mVT0U3pahZRFx+Ucq5pQlx=KEKzNi>?e^@!SH>8veEy_NM`6wFs>gyDui<@jbMr6fecQ=W+cj8hTC}up z5xG?dSUlNbwQngQiAto9YwuQG0=V*(iGM2eXJ~-Ow5F?t2_NJQ!T~kL!XKQT9QOK) z84^Z+e!|B1)wC5Qc(y`7ckIAY=*t4{i8o#zL{kquIK7TDU zYOcX}UyC}h=$M`UqUYCW8K86jzoS%5Zmjuo{*wc5RD7iE5f)1L~(oiuGk=!XOUfh+}Dtm_J>Hl2~6+L$b*^1DFUHdZ)F<}o8R%d7jV@czZqP9 zKAcT*&hU^EU#vC|NpXVVc?20P02I?Xqa}4~PKVtse)ftXG$aT8C%(nNYdctmFOglT zujtF|MW|-#|D&^bsBV!J{Q8Y`n(Y zuNE-yjTBf8J`KNtUF>}tXIP}#Zt`nsyau7`m8If&ThtZ1`uaXACvj9AJ}27;qem|_ z%twYRmGZ50?xXzHHi}7-LZLdozpYX%%R=gr@$tjj?GP!TyZo1vMd(k?CjDln3FIAd z4gDe5mkjJKrF??OADn_iC+oFq>{elT^{^_8O?uHQ>Ns>^`<8h)}XxnBO?(*yse zA;~g4K4P`6Me44e`Lis^e_3@W?Hi2Tf4VrpaX)>*fB9>`wkWTl3WRn482v=6412s$ z+hH=)G^?Y+Caz%+-UNz9l$U_}flhDnk23&X z!bsJ!ayKN#aNy=k48hr~dp(Z3=wwth_%iRVkF%KduBL#N5U=5d+@DH&&SpHjBaz@> zyuHts`3thrgDwmT3Uoti#@|14N#~NTn>R~#z(`WcdqJrZ$ z@07WVAj>>d<4ZA#JnF<+5kG?lhc#c13Q?seNESVw|A0nP49huHU9XD?tuM|E@^+?Ourzkz5(Y#M0eBvE;tk2 z7Zs;+cO^ch@_}LT))&?8hhD$4$W@=i^Ch2ehbwpS^q8u14-x!PR9?}Hok+jjxf70u zjJM4FplxdSLL$$rHwQ*J7j=#qIQ&-IFwMWUHq=Xxs!5hA|MeKos>DMqTZGi;SUgB) zk8LGFUIM$PAKvGX!gZlX&h5OX8?_ulVk^OkTE9#V+v2j4(gh%BCq|DA0?HUH;ZG2b z{fDThFMgih!f48aMdV}LgkQaPbim%$`r85PSNn1IhDz}3xTFXq>XgWOuECpp;c3L2 z`*}fP$VzUvh$qQHoD9s}obqACS=4gDiiO(NLT=$_FX6*IyH7J&V77NWBUMYbVs@#p z3VZ}nYEPu*b<89K$e3>g%g4KC0CM=uI4dD z?=}Xh8x1-`+|88LM4PsvQzyqfA9kL6r&F9!x-mi$aggljpotv1Fq%OGK=Pix>a!-l@ca?baAQ9PLMY8&d zvU|-ZE|z9JPon8}i@;=Ko!S-IAyNHTN%7CAgO;4qE-j4w-Jg|>YAb%cdVwWSu?S^q z?75)js>i-(R4VZ1-p}7@h!~dJ8T&NB=q@%t+DCdVgDX7FUL~!T;Z4+gXz1c+&vurS z7ngA-h|wiXpm}8hPf;1xNdzY~E5tGO;d|YVpYm&CP$z`bf@Lp%gUOcQ7Q)Z@I*pa9 zubQtK)h4$n-&~~@rSf}n$qp20Whcry`LLF?@+WVarWWh-B_rb1hPQzMkSg<1@Rxo` z?}-{ShWTw%eJ5#M$jRR3WH{fWm7pxGvK{#MF@b#iC>G^xR=arfySwg=FY~|m`B`2f zi!-`To8V74$@;NBP5%h`E1?P495#)({MuvB#pGMdB;J2g_QJ~S)bGVAH1>FrL~j2h z%a7Y`=~iFees;7~bOlz10ZKiU63hRZ&AnCfk`7w`ie0>Y^w7j zfk&Wcq1YYYqB#sXKfXl}H#zQ@&ckR84zM>s2reSjfgltEyM8+ZtR$#ePHRxQi?5mM z#dG&g?rKl2BzpeZ@A=dzApsH$i_@e-GmhOZ&!y~g^{|df6sAK9LoLow99f>j?!Nhc zS@BJ}ZaJKFQ!cc>4GATS3*v<3LV`{zNy>JBIrw{uy@Q(XYNUFcF4;Ei( z{-H1811oC!iossPu$}3`8e2BTuPMWG1v01?LDz37&ZY~#V0=?Ws7f4Sq~tq#hrX4UGR)WXTI7T=6|*ywLL z#JX2M3bSv|^x;2YGFs;`ivHV>>LP99nt(3F2v!b<((iA%KwDrG5jdAJ3f!S8pO4BE zeeHcB7Zu+TXr1>aCUxMNiJNS4*SDBoZLX&YCkgEe;@GXws|-&#*;t`#j*arFo1RWU zrg55k^(XWnRbG0bmzg+}gv6+G>Li=B2mozO-O+NrVVT>w`~}^y!)b>M^>wGm)vl}V z{57pZ9k#gZR~H}*8>@VSIj-U0x7S)AjOK;-&f=1K7RB^6vS-o|a%auP-mu+K!;?Pe z{sj%D=(v(QVN>4GAhgRQL!P^rU-PxtUb)97CcO*LAh;vf9Q9!{l`<(E=upEcEhbby z;)B_5O1L%<(33Mjn5ual%TcAr5-5_kJ=7so zZ+S%IEi@RAy*E83&N7JZ^;b>g-!kOV?q{3nE8o4-fjG>(Gb?j4N>iF1*6&~T=VO%a z019e$SIP2FE`%7ijmyr@YW5ZouH71tVq>bNWXH+A)nhh(oXtq7!0{DAG4-C|6n4bZ zYFeys9qO~aHmJFo5Md3COANc)7bbwzM8mV~^B=kliD(THNL1@b-#+-p|FCzSZSX_a z(t;;Yu>M$G5MA8}+WHnxe@{(Qyl+x@nO z;)MyDY~kvQlm$xG&#DSt1y}fY;!`ck%#7Kjv*1~K(*(Q6xE_{MCKbT*`OnZbp~605 zq@(jsw-;&r(VTL3-gY+SX5o;I+m>{Pm9^|aK}puNF)hv0*oD4gM(N3QjUeB5VfJC= z=+1PRzur%@;pj97{T190TeeHj;KPb>*^z=i9Jz6do+dd0zY{QKcL!iY?J^|=&5hm( z)})vM5yg%)m(xq+WxdHV^rNVr1Vn?6du#>6u#OFMbhryG$BQS~!3?QP`{n3T1f;8U zV|Ag!x{QgZ+_2w=L0ca<&F02y&_CA?G9SXK($_x{bUMi<{H+BjH!6W5qepP!QZAVh zt7;#GDIZp4$XjyKi&K}hHAhZlx5EUx-eW8yN?v-VBT{z4>eNKa`Ku`XW#39O3+n^C z_0@=})DAK!d11WkDw@zI2$edxivJ?9uRE15l;jrNqt#GVjz8bL*UE3$P=k2+XA6Dq zW8~^7M%Ic(NKw?DXVCG+NM8Rt7;{6G3Za1slFIQKbwi!{y5VD+x?A&O%t>In&(Gt1 zd4qG^Y4?(9OLM&1+$F0tQzDS#lUM8olOlM171jTTxVMapBkR6}Az1Jr2@b(R2=4Cg z!QI{69fG?%gkT}KySoQ>8h2>iq2JCtGtbQA{_lN1+zb zF4s|3__izcGf-pdka{A@N(U#pRzT=uJsNtCzF0YTX2e$`eM61TSrk>Wgi3ngIWAD#BwBVu+m$$HFCB=)`+vm-r zJ?kEl;)!n8wK!6Ayo(g=&X8J?TZW_v#Q@v(*e~)(QAA^@IxTLVt_;xG`>GMXj_Dd& z>?I|8pBX$OGL2+B15*GK`S};DK!~Ra@?%bYZjg!XvF^=IE^S)kjP=-A{-hQ;My4Ji zb-%I7OABkIM(fArjtCyUcR~r%;AsjXPj4(a0UHLsjof?xMJ?uwbP?X2-5R-(l%fV^ zw?Iji1s*ToAcL~_!jhW$5--Uw6}{b0cPP2#_dX0a_)1KL%N(DTkKFKaivb!3>{@*F zE`#45dp$8MiFwNnR}sBD$4{jA%Iyw~^BjO)+bLR0wx^BxMkIkjiIkXMEbVp!lXj`} z)0dSPk7-g5V%t&Y_$Cvo3a7|##y!sYNzA)d`|pPxu;kAS(N3d8oP67njiL|PiRMTf0huHX)H-d`42`eiq!tBy*V~yQ@wNshCc+d6XG49Kxakny%P~KY&zsaOpgwOgMqx0 z;JZwh%GHHFA!Pn2KDf1PjdJsH?gYfZIEAul$LZ-I>sa!UDn6+B3&x4FT&uu*c14i@ zWhtKAcpsssRnwkt&%&u(+@ev*=`=j45gg(8TCHZYkjZHXUGyQU32(Qqe4d5``=qmD zPc~>wSXdZSCn8|^B&8aCJFp3DTG$Dcp(LyM^m}{jpi!PO6OBtGe1PVy?Gs(< z?n=Q9Z0eUqHQh^Tcli?%GS8VW*VFfIl_1X?#k z>%wfArOHmN5%nA7;lYE3jjrS6R~}EC3JZ5?Zjpcq-+&I3=IBxi7e*RQdzo#3|6)Vn zdDuF8aemI>usw(dfkMPPD(8cW4^RWXqZ_sDC@)?B^DbO3u52YPnq3Wev`h9Dv(5f3 zuLACUj@You#-+y2fc@L%@pRB;VhS$=Gsv$tP>=(?U`Co`YI5d{=fFQUq^=mc`(-;s+B8^pwK_bX1r2R?ER?qAUfX0o^&EfehRKjf>rcS zI0u}WpHg6p1m2rHK=bI)p=9i~^wDN9Q3xz^9eI=Vmv!`)kpvG8C4(r3`9F{-V3{{_ zgs~1-9u1Z|Kimc}oB`uQw@RRwSQ%+6Y5#(${ycKvfhhnq<|ejKFzN*s*Fr0i=|K#n zl>ATyq$Gndy5`0>_MGuATa5ZqhI-p3z4ajt6&elDZLO&C^cvv7vcxE$&OihfG+JXWUuXSsFKMysH>JtCR!x=^K7#3zCJkuu{0&|sap z`*WF3t>@3^o!E%hE0E!FDm0^uzi$e3yhlc)xAO-X062Q<1Aom09h?Q{kOXfyHQDx^_;qAb#J9n%qhX?WG2QV@d`fkCxuT&0S zR4X8l1QI-N#Q8v40d=QkjkmzEgU{O}Rda1gIli)SBMtTw8{G4aP;6byWTcOs5MJpJzm(l%CK}x-zQfSL5niIjqT1A$ z-n>6(6oN(LYi80hw|9!Nl9_-<@#5^5*ryd35L>9M3;p^MuHVZ&7tWTAx3M9Low>|drJ5EQ#jFTM+36?CFMI_T?GfiMN zM7J~8jx-!mR%Wa22m9h=)p})!5PSXj;mtv4dQ1y;p3J#xi=)|@{2JNt>}h83n+dt{ z+?y$aqRwwU)Zo3CUp?M5wPwajC*v*fh84SpoX=BeY=IWFAVyCPsMV8V%yF%TSHnk6 zOb+-|w-{ws+>aK9pziK18Y$zb)XfhBTuv7xvovjcOe_aCNLdKto>bUoeqe{`??sm- zCTH^mX?gn|{K9YB!F7@TZ1WY>PreuP`XKJA?jFG&E^0eH3fff-^f*gDFdJgB12!;i zBu8RsiD-Lz5qtj4-7wxS*fRZ)``I{&1qg0e3ouMedFy#mpCxm4S$=1_7(=`A!B3(X zJ{4}k(`x2I2tvn=l22$mTvfv(c9p9G>Z=M3@(-svPKWI?_Z0Bl!UfN*=j6ZFBFOJV z4zEB8M#M@e;-ZMTP!6Cljvj$&aercg0I$G1~ z^0pjv1U&_X#=xnisPo$r=E$JgXkSt)Q~ZTrlnz(Kc`FkHUk*kI4Ol(bU@rbI_OIy+ z_tB+ZMVR#IEu6~>z_K;zjKym;=<56C$ZWCh zGt=`1mBpr3%*#su32WptYrWVG?yP|3lY{97JYwWBwe(*4s~%taLMcRQT*tp~us`mF z7CsPAN6>#()F$p-g9wGu@ymJDQcMJs#xo^A&1vGy-j%fBg?1?QCrf5uomh#RYUri7 z(LuDF*uC;<;M(;sS6KX5OW|b+CkJ`U-BA3M$QWG0l`?IK=}mt!Lwo}|e&W8==l?!9 z@ChyC1mKy@*P{xD&x&9 zfXxkb#b0~;zuNzXV)?)Qf=xQ^a6i3Ocr_6+XIqn*m`8u{83*oZ zy$y8)Gzn5J4%Gf@SPm>mTTLmgZCfrFwU2V z=2t<~pk|w}gr^17yz7OeyOS{;+MUloD@KXVb+wS$@R~4{x!MPwt{*v^n8Up_RT*;~ zS&&%AnIG-2LT+exk_H@64SKz6i9!U~=-tOzy#_osKR0pJxb|;wJxJg~>;(tjuekDs zMEBbto3TTd6kaH3Zvu3a2W}$B#^PLx@YhvDbdOMW=(cK-L;+|LKfWd?a0i>N|G(@Xs;4y;gw3 zY?JUoXN`MXDLW+7_T}ILT)z{33C9lW2Y4y%vC1@OHG#~zbuzk5fx>AX5}u9jarYdw zDm1<}=q!C!L)f?Yz8F&Ve6!i%1FQH7_hPCf%iZ`Y;}{iZjI~2X*!7^xQASJMn;L=2 zuL^gp?MH%-89Jb~CX8Tj{@i_mIhLFH7^_CTe*(MPD2(Ae4 zc4vQzT;n=JvJ<+01Z;VtdL{71k2`Q*oN1TH3vbKPE8XlSq0G`hq8d(Z>W0+_UOh>$ z&uI+Y(ek)CT>I4T;nC9b&)KVVa77+?j&(`~jAP&Xe506fPaN$RoCm`_y!cowLt%lj*R zxoq6MA;;4lb)YYo4mjn_pDpfD30~|+-Lcf8>EV{bT=Hn5VrJxuNN2E&aDmav1G{m? z)MeCMruI1lBn3#24=s*M95QNo{XE_I^%72JH4KyAF$AogGgpO!?2h?Eq-z`ub0(fk zk@J09=^Cc2FBAM0V}HflB#5iIs)l0u*an5!@3{mgu8{J z<4%=BD5$U(eU>ui@A!D_v|ltw~Tsjn(4~g-El(e57JUU z)Nd}jPT5Ph)$4mtX%LJO~RN(F)hlI5(>Uso=ng6s07l-#y znC%;^3zFXa2pp6-+7yuc!6~%)@aCIBg|_8p+;gunOTGe&DgM*6t-{|*!NHCso>|Vw zT-Q9Ln>(M;>On_#CGT{T6Ryb?)X_%|0(cKcDdAIn1T#$k?fl+vTuo6-b(|=>$~_!U z&w2F5@3bjN6r?lPhOf-2!p1Z38sCWR<4`G4eb0k5G$$pu&NP3%r$zD3Zs0-5%OmcH zhEOXr8Ex{3m6hWQc}tH=P}L*(bVIbrvrn4NRx#F4b-;Z2Lv;AG0)7}+WE)5t4GR@K z5K{HYX#sgG*-h+N5>ri*6^~RVcx?01CmPzDCEA}41|8S1(CbnC&PQucayDpQc_6f_ zKfKQi#o_rmJ=+7tG8r;27xU{sYP^d3SbgFit2wR5*I-FPh8u^B#cz%_rik9s=}C@+ z^FxoGv0{n6iJ~%q%FkNrqia73rS})g!aBGQpxB`;i1q6KQbv;r)0ynx%sJV#HLXO? zUr6~SN40D>mJ-Sezxe)%pA*!;D|$pp;!?UkFa}p?^mg^=J;^N{cm^gMvK;DUacqGr z8miV`a@Bv`k-Km(qmK;IbtPYOrAu0J=~{)64tYv_>|$?6WHOaI0%zY`JKTtR+#0v@ z;CqSSGaPpH2g{SqpC>s#G(w{*4QI@u5*ZjTY+bpxqtoYfn)pN>K2px@0SbM^Vt3Z6iY^k%YZ!HI*lq(&|spvCd1B47Y9aZSV&lhI&F75I9Yz90dhuH;(fbOicRN69Kw7M7edKM ztVMsid1tXx@iDHWAg;?zG`6H#m5w+SeYXBbbp)>GlD~6ygADqI>xyIDGgl|wv)PL9 ziJ4!6-x@eWetUTsGE*U1oC?~+THe75t-l)pDI2)&zmpP1jnL#LRl;ypp)ZB0r&vXv zrE9@|fr_Dy56RLJo$I)#Naj_8hyKrDCuMSGdS3fqBdV*=l;uS z)=*_ORTDN|&=A5TfF{>`d9ji2G(6B(%LT-Dd$H>3gU+DxmlKLNLv)DF^g_{qK<8xnm1zP| z#}_gO@;QBY?Fe5l*WlV9QeBx3Pm;W()8QPm;a$Zj6LklXEFWW^`R?8vK7X){0TJvz zplfeZYTV+E`n@~r30;$3%w}7OYaqkekT-c?0gjiHp`Kem;Cy~^qKL*d7G{?-oPRvA}G%egXM_JFCh+k8$2X9_Ar znUWna9Y3v1A(k89s=16TUA}JS%)x4Lr-(l#$^A6dV4z6Pb%Mz}K|U*5v%qZHQ{PhK zt70oW<86JIQu3l`j`z8tf2@}=*e?y}##yWpRO?Qzq@N0eta-wIr50^%{LnaYHu0?3 zz!APGFmRyClBHXZ-|bN5Z|BP=$c38M)2Kf4P|nuIm)Qr>i7@OeS#>|kU@Nwu$M80y z14KVYHnAa^Zfiw=YVEy@@A!0{od^gfPED7W(jp(aq>tbg-M6Nlsp(w|*Ef6A5Q~on ztM#uKuQ!rX){O28E!Y>-!j{_-QTejcFr=9t7ZW7zUP0lgsK=XvR5JCJ1{y#u5=!^=>S?Y;-8glOYFgc}lg2zQ0h( z+=})?d@=$KIbrVy(`N|0jH$%xDwy|URwU;?P2SNg2|Aqam6B)L^X0@@a%i9YDFy+hR?oA>*`I@t!uJQf(vi93Ai% z-g>47CFSH7yJ~W_U!e*0zl?9rdoodUJPGqUZ|iE7vwEk5VagTDQ~T86M%X%}Qm*r~ zTDG|wNAy+9wYSSz;0ug=z^Jvlpvk^Hb2jv1DUUVGI+|Lk;XKGyJfFb8uQ`5+v^mYu z_I)DOE$lcMxza3wwk)L+%DB^6Mmrj~67XTW0$Kc=_HvGJjx*mKd{B?n|jUYnxG1wZaR_{YB|2%goH2OEDZy#>G^8zzOD0f zwqz2OR+dJGRTBa>LxLHlM4d*t7E)T*CeVj`2_d*yPN5`zqfkhP;6Xt36V3!0Q40~M z@9|u>Ev%tkQ}$>#rbf>YSEW-gfhju*($o}ej;#n<52<{I#wIE_?12rfH@T&={eo!9 zP+)O`ghrX#)r2C$owF$cY;V0#!!PO3((JUtcU84714o|z*|ytb%?&2Pi;ZeeBti&* z>-V4dVTa)6y2iwH|Iz|fSvFLhkQ(BWiJ8ZV89&x(mZ_9SkY!Wq?#$C z@2)Op99;W|e?lWSUl=f{BJFB^r)^kU){x11r6Joff_8)N2A3tuRYSjeiy#)8uUoA0BpjGMPqTyYz z8og=JC)u)H$f3DEvv^0O^NiL1>-~f8P-d7Tslzd_sK1J*bJTJA$!+9yp-{UQgE?jH zkO{t5a&^n1=jhrbL8*Ny$m}EV)HloA(4^{kGsEjShuctn>-7a-1dm9j-&Gs2|N1$W zvH(yB8eNsPV|b$p5-QaDm~c=Yh$|^NXF46v#%W z1N1#A9_P9$t+aTx7+5T2>)S}ME&bDT{dKe$;qYQP;xAVX?B7yr(g%Lcf@C9D{s}g5 z+E0h$?CF@QQoR*g&X$mryp4HBU^UaGEU}La>+-_}%f3Gd2fkYV78AVMC;8W?1Rrgb z795Y6iey)QT|JEsp;EaQd){J*^?D(JKwXIrcsYqrKwrGeo0vV>iVHzNGKFZsAO8&z zi+c=AW^Y1%5trPK$$~`C{>!@g^C;e43(hSo&hU*~_V14K*{9JdNG@#i{(12~ADHTu+QtdNxAmT_IN% z@l5U0OwwvOd#RP)RysT>Rdh20Lnz0e>W#W%9vA4F2xyxky8WawP#4dZ>7)l0D?*lZQzS>ez>&x8Eo3avBgbAEJUT8~AIwt}ce(zA640s`ISbb(2PoB>w)Hs6(8@ zp+URd>)y6*M+^+8{j*VepxDe*AjMho8ICt-vb&M^50>#t<1l@3H7z6W_S)g6g6En? zw=$dX%SN^v#A_TJ>kktx%kJCg6@B+W*p(|!#Z5>_HWO#F(K}u@n!O{G-$gL-a6jgu zPfqEDSh73X3rxfyl(8_Vt%`p`pTNO|Tzdk*LZ#^Jhm6VYF}Gak$`sKQ|I8`0otjBNzbeBt%IEul{Nkvc=g0QGim>>}A7_oi|opFBvt zTXDlBW)}2~4u2l5`ZxOXsjN|_KNUz1;Gsuz2D0IF_4Y@i?7^9ba!(?U)fr>_Fy^@I z!Rmg*aZQDbEXpnL+{|pqe=4ZL{5KpN_bmmt>sIkY&8PYgLLDzb>j0lF)BX@7?_37j z8#Y3pVOzox677B7I~a@Yj7j?SseFQKDSUm)&##id=slrm6fYjCuUy~&KZT^%o-7CA z&V7D`idjh=76o<9d7UfMMIszt_@VXRc8@`=agG&2fvytKWgM z@wBTRQ=sSaD6%cA<-fC5^H%sv&B}DeuVESlqjz>wnj@Li8o?llHyiC;)(Uh zdKY>d#a@l0%ARnf6^p65AFVJ_dn9aT`2jsl9Bpv0^1}3D(U&{=)xV=E6-x7qdf!y* z^Ah9jUJ4|+c+p=9aXNb>6ZHgC5IDh8=TfPy&&S7jqV{vq9yc@KThx9t_J-3s<~0_| zkgKq?yw##uLK8Oi|dIN6}-MBYT*^qE@U{Pe7_8}%O zZwov!>7eWmrpU)tJq|4ln;6Nf2$5;OSA>SHm?-C%eb&>wnyh@a{Pz|id>Y)HZJ5L1 zyh*D9-XyuXwW=g@PtLK$d417-BK>=u{2JJqzm1(=*HTrKn`P47+%IDl-3c9NeWwj$ zMDRbDnRaS_|K~3FtJ83(v9I=c%;y({x1DSi!Aw8qze|C^V?^$D@bASsdE={}fI#=3 z3GhA}chPR>aCoMiUhNT`#(~8ax-&t41yTIDNv(J?2liW^xs7o`pCIWi2H^0!KtMG% zXQ^@&T`+t1XMp!ophgr(yfE#nh#|J|E!TOmh<0S?3U56&UhA-B+tTG13^Ea`^~ZaF z9XrQ_{@bOC-TZm??%0~Fxb8GVVk_O6)RRf9Q4sP}DbBUQTg)GWW@WcChFz%qQj+?dLuiDEF`%o=H3 zP>=M~p|fC1I1qWwV6@E{q?k8}M6@otaK%s>HRvO2add@A02@=7*5hL0b;rV>x~9m` zI9Uobouwp7WVJ0GcO8OVd zO1nW(YB7GOH^l~WOqOnC+^|#6KtBduoqBxy%EhN~p zZZmh%FSLq^>_#6h4{0@vOx@k>EZN_VHGP`Xf=lTyTYZzxak`m-%IbJJnc{wZU*1cQ zA)q$o(z?>dYwCmDv+&W*^F^ll#zJhx34oW~a)?tFI`}LAhTro&qMB7r!B9-=P}0d$ zG!LQs{v2$*Yr*x`+XKKPhAS3SVQQPzqdB22-AhtpGicyq(fT#+Y#61#NoVL|s4}&e z4H4*w170!Yo*ED0r~$`pwhKemlW9`3!({XB9y)DgdShBS(s^mvBZ7jCr6gEL1z$jl zg25)4$iB(4(!Q{#=m`6$HF2$Cf|PS4$>e!$uZ)-G`;Jv7I^Lih6UAj;aE}^3H`pr7oS5)vWzZtSn5v2c~BL#)*#Z${tAU4Jtl(zfDS_t!i{txftUm9^J$ zF*q4@>|VMtyNXjR27hNkR2+knYjL{BjsIg)9!^m8WpLvT@eN;uoXd`q8f|cF9U4Az z#6TvD7^-N3N(HX$joq$RS4ydAEtokf|FsZOX4VYotsR2R>9n@9n+H0fJ;K5zsx9}n zw97FrQvr#TceM^LN%pJRg=1gSg!#@nF<@DGH~6rKOUTA$T0T=fFnQEH(6tpoZ>~Ih zi`Kf(7=0!^eJ;EZ0U>wxtHx= zBz}U#8G?gfbYwOaHPp=fq14UF5uTsH<2uWvffnbPYyx(|z^;k`c-8ByOJgdx+Ee~H zihECaujd}gs>j-gX3B`x;KR5&CkNY^yCK5~l}~8WLqZe~okrXrV%y+q8a?k@%mc!P zm|Kvc);JtM!>IJ3KC1fi4|-}Zb&$zyhEZM0K@ z#HKr8M>nYM$3gV=b59}QA!iceeJ~a4;~z7MI~V%?hmxjP)4 zzgn8^eimHt4^74C4=Eh7tnd$83!+ah0~=RLbj>cff|m)v>XTcuDNfeCgMo&s7hw)u zo_Pe-5X|lqhwBF4faHqn&B^aOqwsP=*~EF;s#iIR!#@(Q*+F4Exa$?(t> zBFvv3chLh=svp*s9%$}@Ot_#H+IeDEi8X7Wuo~7kn}j!II4HxUsdj~#63hJ@DWUA* zoD?w&sidma>Ef?KlH$jA7ig+Ib87axWh#3USE z<)lET0}(Q{knC4W-C57~Z&{vajjLsCT~hr;H=v`bJiuNFd@1^@3|M(lSDWN)8xZPsZ+^x7{}L zf=!)6pSmOX(U=^VU<#>8Xv3Q>U@e7F?S!i-t25||&6D>Qt%>jBSNTzOIXK2Pz%+H~ z6PZlg^3-`j3^wD9>${G%432x4_ZKUZ@vLm8EiY~@g1CsK;iQNUKP@HSC%4Mquiwrco7lQmTk7&3S_5@Z%L3M@LtLD_dap6YKNDa;m&uw=WY?;J)ZGj}Ad z-1D>HY?MDSn0b_FL3t15i|y4+7k?<9Jv!IVo3CJr(jM#qMCzT^KjkEt`Kn(1aw|(3s`^v6#`qFo$#W1ePl?W|Q)|fC;tRtquP<%%H`%D^JFf-X{h`JSri4-Z&!w5f9G_#J{*#jlFD`zC_@+b*h@g0_-WNkf4 znInUIcnVyJ*v!S^gCkpCY!@u67RO9Gp|;p{guetE2T{Nl}a zz@oXektjF_99mGZ7#KOBdcz9Z(nJAc&~<;zyS!iX?uSdxZ~&pnE(+0@tG)?7;cf5g zVmRd~^+6%~=Jbf-$m-h=ehlXQ;fOIUhHag!-p^4eLcukF)_Ax1)7HtZq~2@)FPdLD z^z6(9K*XyIG^-I+*BPyMcN_u9`2C6ZW-l;lc?&mE>|iKAv%xdR_~D0kzGtNV60~i! zv`<=?fa`_`81g4|X9G;s6bMZed+|bl^VRfm#_eb=*9Fi+IUei0QXSQkaw6^kBh7!c z6@8o{wi+LA8rYkRk;txC7Y`zTzEM3-D_9(I*2~L^+5II;T5J62Fhs3UwigDYMECFq zY<}k<(mikM}5Z?_7Q^%8X*y9@r%PP1syu8NF$+ z#@%X=X{n+-dc~kw$Y3p%{XFOG#rA^~uxhnmirM`x`aN|={Y+r#vt94(SiUs|PFYy2)XUD0y(1Z5#_mlm(>jWZ@z+DDOLf+T;Y6JtH6qKl_Qn1* zl$>(I(n|%84&Ipk87?*OE>ZnMj`?FH&4XEc2kc+N0DN3w3E~;xbv*3VE(KRDbaQJo zND^HBFUS`aaOyfF5oYx?xp#O2?j9zX$*<+j{{r%D3QJ|K!1|JrUkr>x1kwK2#Xebr-orjwUc=vgPxyol|nyu9JpZPF;xhV}f$^XuS1YQK1dYA+MEtJFL*-J(u zTn@1aZbdHsLVL%jg3s={H?Cbz$pbJOTYO7YQ{qGv>@v7MuqYMA>ANRONdn^i4Ivb- zinZrW_EX2Wzc&(%n~i>QaK4kf#mLalWx*^tWF)rO&XH%kq-Up>4P^va` zkVoKm6yu%Mh*H1jl|ic`@X3%Zf-h%8!tb(5SU7DEn%Yz@&P4H@A*JW={C9DU-d8s{ z%yD60BKuAN-gM3|}HtUWz*vZ*F-KWS2DfiN{>+y};*_r%PxUkb-Ai$4a| zGHnQxneeM1gqe=}OQ{d5d{12krGjlE{H5wN(HPHH#j$bzQ6&^nxYcUpt!lGKV=; zy7-)MORXm_s;R;18_1AFOl(Ae=d&o=ecy5aEp*UZ=(mU0BS4RJaxmtWo7?e z4QHzlF?Axrl5ZjLQ`A_W@RR~^UT@$6cWe!ykh#Fh%Jw<)0Bi8$`TK(ME4IY3S-zy+ zpR=W}wWG#_uH9ei{99+Hdbt6l`OjMsZ{s`{p+vO({;_a?<`!HzNFd|=9Mv4dWvJLqm;#&KnAmi! zr~zV^xJY_o9iOo}95OV9I4)j!A%Jx)mn%k(O>>A}=tVTrk*3JG=MV-%UQA~H>K+W~ zYZfxDt%4!IcJaFm1BvsJNYG0 z#>5GHg6CpGTWo*J6YSD&=2SQX7KNFtYpI8chZ17wudVm^m3nUj5$}Ox3CAlP?au7^I#k+pdD*tSQx^eHM z_A7`RYjFlcck7KbzAe<)U{sllXs3o^n*O#thCY$wkcCv3sNWra125*M#2TBwE$PDB zDAT`SHjqLA0e(a?^ncAYwEGZRweh#k@ZdP6h~JlP@Q@)W{D%nbKVR^FzGk900|QP5 zGihf%)%IQLZ&oSJ_dTr{IgxC3_uXy(J7>y<5HR<0w@by@vb$zZPWdSnBFCLt%h!)g z%j|N}wRNjDKme2>XJjD|h+}6CB9{8l&{OHn?cHCxBkWPGSa%moj$wImdFC^U!Zx*= z2fS-cdfVapaPF`BN)@Hf@npExJ&=R%`R&?jrjpyU3oXYMyLJcPYIhh^#=ZTdv>5drYTH?`++qHKl?1LstY1abKJ|$HP3bV3}%7 zsSaB>^-T@Q4}{6k9@%QV;!uf;N+H-AQB7Z31sVymhC11H9e+9)8^Lj}X^akBvocZN zTIy|mng8B5dbrWurCz~kC*L~@EYfVDQ?a-L#V@u=R$rCPt7BR;xnv4?Klq-rc;5x* zk~+?l+#KMrY#-#5X=dUo@ZAJ}&k$U$6L?5D8{Hvh#bR%qf9zHWTy>oop zl9(uFAnpkraE5AG=h>y=^%+8T+!sxL3`X-^<1g@-_*pz{w##?WzYp>j=9d)@l&&nf z#}(pw?*D1)2t#{u`59`-gTs-8J1m?=L*7>gnwS!%^&QM=3Ja6^;Wf)DTnGPpD!H%* z1k%htIP>kSq1r=oOgGN$$KRhNc6B^vdyq2xJFS(ysc#Z8k;|ku#YNiJrsw6>8U7WO*oJtlXX7Bym7N;Jg2A(R$$!-kXVYIO}^(lf=zV589T&CEhv zgM!_%e!Uy)GkZ2p!VNn%!pb^~zv2U0}_vCdnBhWQ9(kIo=g(yF3hx z8UB~_Rr}o zML^8@qmZQON(te|!&m=afozE2ESN1tq7n(LmT;u`W5Yg;X~^dt%QP22*Lb|D__D$nV#7k9#T$|#(l z4dk>gFi6(G0M!gwHB>93qtmhDP^QhHAnQh`c0r?lU+m` z%tX}2A`3`y7W++U&i;^0`@zfcx~w)P*_(r2;l7n6?+#^hBz7L7FEw$(+YN~_9_S2l z@M;8~HjHc=>p@!dCA-!iRxRnR|kzNPOm9$UEDbOE*(4Lj?-YjvDWMeqc(E`uv1p?Bt9A~y|%P~ZjYFoavk}a2Po%^8%@JgHKdz~^Hn9&?E>FZy0R0GND8`j(@c2DXIc!U zATKQXv2?r@s(u;j43e6tR#@oU^OZRuJI)*kSdDNm^3{EXGTs9Aqx^AcH08;S;SdFD zgx)duoFI7Y(QQeZr?!##cHM zIzAbQPy9_Mwwim_@cU;W>Xha56_jE;cPz?)R;(#c&g>QiyThKQAUJ6K$(>*jpENG) zfn9C`LC-r>g9Ij(B@5yHfp{1lcHFvmeNP4Sfh^?NP|FQ((jHQzJ|@>~k8~lkVR4lT zoU74l0}`_>NV*I{#E{t{SsCU-gkHj^sS3%Ml1S(SI`{#Lr{XWIa|OU2IVLFW8!wc^;@~RtQ41F40B;q*VdO^^O+OK(?jn zx#=hw3KAOyA#~5=39?R0bSMus4wy_3B8wK5NhN&G=F{7pTLPJepJ>#mATHJG+qXp; zHhNor10xk1+N|+FwxGB~_<0)q3sY=Q6tc-Cb=0Lh*cxtT)wUZOCr^;#Qg+Tji(&&N zC)IEib>{1bs8mAl{goy(NG4xWN>$&YV4ef;cK*e=11)VLN{+ADKM2rZvAlKt`IXPKDZ1UEMrBW~EUKuBwaADCuW+y_kZSH$jCvBBBdlZ zcCE?iLzUx@w950OpFgR`%2boBvi*-8x+?q5zBclKld_3nzU4Fa@m>% zw)l-{o@s9g@rUElXs~q>cN}1Zk9fzm2>xc95AyVNd9(Aj1nV}?nUCPtH#h2|q>YXP zE@@@ZghF-cchAer)O$jV*?0nzFf#V4N|g;oJ+zlCy=ob>b!yfj}RWC zjBC8k?v7bR!9((6PJTc|7ts2_2UA6l3nE#s@X9C9&p>J9M|X16kbMt!F$44{URUA{ z9Kvfsevd>{43O#W%>IUxr~Vs4I(`oYB)C`eXEt&~ufDzPT45YWKuZG$zA_nN-)o?X z`b-AKEn*38@1{v-wkKv4Av6^x>Zy$DN`Yth`l|&MpXB;$b_07US1H<0ln2V&MZl8>;F$cjttm|DLTYJMzD(DeVCtbqLPkM3A8CEzNLxAoCPUNx>vZN@-1;pMX3 z*i2iQ9H9!l!7zcMOhhL{a?BS^j@{&2t^B z(1Flu_e!JE{aarDr|%H~ET#|oAy09?gh~p0f z{{QQ}bXXo}xYYkp^G}liL)s|Q{tz4gwJKH4&dv=&z_2NsEwk}y+Q0Y0U$0zNDmc?e zUR|9rq^72-R7NoWmOR2Z(*UY}V__MG!$3Oe&! z;fhkEEq4t1i$g9hhXprt?pujhmlYyB$-DCIrUQ4(!E5kGi|5f;I-us-IoSrx|>=37u=Yze(GN zf;K>vhgFK(!#$%`M1*9V40vBP6vGrg`JR8hkP5;5@=oFw$3>g#MK}MOk>u(x8eG(- zZ1Yxp!(f~AO4@@AJHSHR3KbdHWd8%x*!>~cy=v@!&uw|zca-YB`N|vlo-l#Kl&K2u z01@%l#kGftVfemffFv*~6v<<0Y=l>jUDb)^%tK>bGxZLWQb+2@q4W5M?< zCUrW{yN~#BMPYxmBX90)m{8XnKOgmz7x2Bn?rAzhvGqlp;-jFs{zMc+-HM;bZZtAY zg;Krvx)Y*J2b+p|w99sL?47*JeZA^zYk22+q;xc_oi{r;e262_Btz9*y``96ED|Ww zC9`ub%T80taI)1W|EzdCsf$X9L3BZ)zK?q`@-!3NMs z5pf0WMAa->$3E?rom(wQ+THMB+zYvCQQTmKg{o6t_ z28NC2`mll2!MggUMq^~Ut)z4(k_*Ku#27)BOCQC-GvQIk;PvR|#z!6!V4z<>+Lx@b^{kGsMugPmjKpDeonmE>BpX@HXN|Ma!9?Es8!zWIwam zycV?YdvI}GggG@&=`RLS7d7&mMCQ;yZ-^*MV9MHTfApaDs~5DV^@7!P_xF^gqy)>EvVM_y)iwt=sC*7b1Vg^;Ls>JIcCd!FYnbug26lF;Gw5f_+hoEE6s&SzpGSF}!^e z^e{_ti$BXfn~CD4%hT5fr{{~Ze}RbO))M$7Ui0lh4a7@oGwn0#KkL3+)SX(WWBZNI zBN20meXiX3*wcsRALtWbWQaL1ZiTZ|pUawWf1a?dl_UqNHlkB~90TCpoONjlx$Y0e z`oPT$&epM;01ZwX&oDeN z{ERx~zyxj|Y&Z~33kUY-eSat-@(UA-IQQn)E4J0D6vFzj*k7T+@a5P{VTh6AxCntj zV+T7fpmyKa&rcaXL&y!aU^24iZjK8u<>1s;;WJ5qru2GK2hSCl;IhEw;nsEbPF@<_st!!cC#1k1W$zc1tvK<%qt5gO@=sENH z*=o*aee4*|MuS&}6{|K_nA7*_?PC9(T5l@fF(+`B3TuAnwNmyx+@LFaOOJ|kd;gar z(Qk%SZ|S8M&a|yCh%YR&;;T#?tcPXfFFnn_Kk9X<1LL`D7Juv}Gevkrf40*@;(ym< z$W|hNx!+P%#se=+p?@5`INPw7R=<7YUoZ76{s+m0;r~H$;lI!IK+R7--k#$ZGhG5x z&^c;#y`Q;r)(ljgw^^OB;Q%H%(s?al#?ScnM#Fn-9BR0~M{@P{LP@W+H?u)*bJ5`Z z2AyS@&YbBRxhHIl$>{sQ3y@uIj3ah2I-Nh^OHR+5tM6%P0v!Lil{n_nkviTnC35*s+d>8bjc6h= z0TDH-0SwKvsqnpSZXd8wqKFm-C z1ZLrh_RhOv;~6w(&pWL8jTtdS?*#g+qa(Q=b^@u6zK+tCajnH6)h80cbT5E^F1Th^ zL6C?WiKF622KOPQR;V}paqrRlc7~svso$vPQtzbpJv~ObTE_<0?zgJ6>z_KHLNYX4>BG36JPb~FcVa&a z7sbW%I`7lapSv%QX#;3Lb$L67#>N2gRdf@cry|dz}sGi zdN8QfvySdmw3L@YfDWO#pUk!9<4l;w;i*<|i`oEa3qSS%@4RPD`ZlXgL^MMDuD6F5 z{A#C4t`4g~Cw;XKB3Bf^*s41bizJGLYJ`$tii3bpEaYw2^Q_zH z!NWm1rScyxeS(lM&~99(%D2+tEQ}SNj*?&u+4FVaJpxCHUK>GOF|&^<)ff45ctyOI zM&sC5XxYvWoL}TONmrUIVDkBKWj6Uk*iG{Tltxnn(MhqiI`i#UPA`XLR?nRk?mQU> z$FSoj|J&2VBEq|Dil&E2N*)Xutd*1jN9hD1-lWoAKyRRK+2f*{%TpkD$}_)a=HyJNj(?&lFq zC`oxGrf!r6JMQhDhoC4poqyzBkw=Q~Lq7Ydd%0ekcl zz?H8|gW8jnG?Bqcd-Qe7w!S-;DsFVhJUXrSn}9uEu~6gOWz*z`MM|l@>q`0~T8kLU z_8gS#jc+SqKJUlOtc+IbB5&w{DK~nK(kug?m5S~Q4Y#lkLn0sue~Pt~K0&Gh#F)5R zdaN7ljg~)M7}F?Y66##m7bLDpQ`vUyg#F@6vb(kZxd8(oFW{4uZ%Y>rdpK-aH=V<`XlUo0>dTna7rH)OH%F@~tJQ9QR&} z7S|TDAILd)SH+aCJNdnkf<@Ho;pbsC>tF zK^bHv#1vFpa|VLLdKGx2?)i=YvEs<8Ik$tihp`cmrT?~!90+asB@L|YWRpLnipg~H z3#}N>?=CDX2DunoY>Y34OtnwgbC|*%E!#Tx4Z$iQk-fp-svPCus*PShsWXA{GXGGS zFm~F4Z;+`aneIcb?tX`VQLy zEvgI?#T5&Xb72uUA`i~1|u+cbEhl=Tg`4L zdY>xBm}=+QLn5^Cw#)hYoS(k`2MBv>`OCTY*BpgUHFNTY4S$KJ!T_?8^0g)Nsj_o$ zw@0=5^~G7WR8oKV-5dYw0dMLgXG@*U3*dq>TdIY@BBBd5fjwP-$*(<6aFZ?d8*Y9J z2Vg8Hd}-UnU%H+BLA)Ztdf_BdJyG$JD(Eb0!6^;Gk)MWHCOj--v5C5N96oY15Qkty z(wf_%FYPB%rkWuB;1r6y7{-W>DgQKTi_N#G_7<1u2mMZGt~$q)2x6Y_`fl`z$@qv& z?hkB-$Q1&DiJ>iI#-2!%KE^Ls{7#WM^Bnt`U-;;Ms{-!E_CFqF1o_W6%vDW+dr-f#;%=a9a?XG>? z!x^_vjJ-I8XHJ!9s3?0~beWd=UZ305r$)1)n32_3k=n7Pc9JVhrjH1a8-(y+GRd8B zJb7kKuAthB^ftgruiWC*Pc#K$+)&;1qKloM!Thy9bMDLgP)-puLY;8Ql{k10q>lmQ z#Ti2Jbu(r`hWK48k>J8RpOovLEg^Mx{CI7lfIDgK&xUFWZKvvhj6nK~@N^1`KG-p6>C5qa6`#Z-hEBKcq z+2;6;gg0(Vj)xQ~dB>IHSn(d`)lliuGeBy4F^Xd)7i(1UC#~hQ%E0O`Mn{o$h{&z9 z1A9^`vLXtB17(8LM}@_D*-;)=Opi#b`pNknu{CaJ+PB;62a*Z>PAt;``E59oCU={8=h2IiC7<4H^Rzb*suP7|NNvk;u*w>KQ9>2$iWMNT=Sn7;-qOk`L?xv;;ly{d_LW z*0)LUALmB?Q>G5Hw{Zb14<%)=6T|k65v}e@`1nh3@r@NyOh7`nY2ciMQ!I&$JFfMd zC9~lymL2sRD%MP4_}G|XIqQ^oQ}4%<(?wJavUNXKG^*kBt*~Q|))U*R*0$KOZc&oykXNwwV@nst7ImU<)-8C_aWQQb%bRKroh8QzWM( zZ*=IGY54o$ES38XD|~V(xi{u@6rKSE1>bQx{Wx!K8As}?t+cvJFBVZr%U6MQV8Ba> zZ5soVNcQ`?+&keZ9f+kG@)!!#Jk@9=SVTj3O613nj%Sk~6oN+|tvnM_S9?F-kS}!f zUPR@|6B~GXNsObA#Uxm=6M?lWh%#rcLogOn70?lY<{nO;T97hWx-FOyi4d+=5@L2( z+BIq$`v#uaVLBB#^nNp=y**^yIb9^LB`Mhv;Ytf-R-3~HgtA>myA=GURfdD}FLTBw z7PVR9TnwR!Ni^&}Im#oU!&S-_UwU@NatIWXTSH*ev}fYfav z-mU4{C1Mf^c?To@u(kK5*=Y4IhoE(pxDf8o&!RL z3l8A(>cC&jC{M$5ff-F7+yg%ogLG9gcF2oqq3E>q5fQ$=eRMdkv#3;Kh5b?%K7%9@V;E=C|5tD|= z2fAlj7Hy`wRxX-qb^fu-sDH(KaX>+PI<~aYi53eX%pK*BZ1@-!+~n?9y+Q(nFfqx6 zSffs%k0$eZ;voXHm77)g2TH42LS$|1belop&jjNnTw*XX__*4{WD^OooWplH2IP<% zwEU4`^y1g+OiISHL|Xtjgg+x9`uez^FQWDJ$YfB;y8EJMAM#GR?+m$OVKFc@Gv*d3 z=gQ|K)?|0br5>;LKf>yw(wz;r0Vo{0&vq|_*+mHlZv_c3hZi}Yr=j>A)6>yq96fmB zT9}oy`6Fu#hq!vBS|7jWZq^kL59+A)^yKgpLQ<&?Ku+yrn`?qiulf6ea{P7EXxjW7 zVt06zkVekx0A?08BSp`IAduMDTU`Nmw!5&N$vdF!n$0q$;0LJ}weikMWT^u_OlLgh zhs$qbFxtf(_B1LvIEqp1goGQg|6L0pY{YD-ik!-9$suy}yrU&oyV;jolFah7112sF zmRuV+>vG%d<~)teQiu*mHn@+k58c?I^h`;d%r>21BoPQKz8Q%2^x*|7p+zby(YR~T zUiGp(pF{d20`5-go|t8kKw_tJ_d=J{-K=ogW3r&v=}78^B!^Wn6;QVH^kSUcxmCWQ??Xa8BLtu%XlF>h?lDUd5#(nCmv~q#cXgiq|=*(h#!y7j1!G&$kb#e zi)FC*n@#wC!jb-(!v8`&41W#Gn);mYc!;`C#VK$Hq=sYrS0vUZ^=gPHh5M5F!}`kR zM=xhHO^``@m8w=uZ+wc5!Paj%*sV-?bw2bY3z6)r7~ovc+X!u6^f5LhxJ{#THqizn zU6s~@8Bo%b444=i%suM5@Pddxp|HI@X6&<`D%pfq6|~6rj*fWi=1s z6-PkW{@MgZ($j+j(a~2XkQ}0vxyR7>D8V8x(@irCN);)z|3vk10gda~SlCn|LF`o@ zIH$hJ2kS22jc%S|oM8C}y_r`Iwkx3kN#9c^cpvZ#i(xI{(cW)(wVDX~ z1bYm^_PZP>SUKr4%94PL3==fo@9#RcUXz}39#;r95vx61xi%jwJt)|&DO)8AF8aQ|YCmlFIr!pK*c^@5ZfQ?%p8Q)_2@C}C`RbFyAt-r@T? z>gdM73h9t085h&)x-B81*q~$_^$lmBXd>as+S%c3fJc0Igx;G(1%^pz=Hptjp+EJ? zq)YyI*GOoB*p^;oJq)Ijn`*XAT5vxopn`VF{=HzY+1`yGR|M2jL4gUHrINGu{KhM_ zNzof^p9~;ZCEtiRV^ z&bp>9EJ&LyRaeBMcPwa!iZ{A_>r86EyIHy0`020^7bZ=Z&Y#!L>bE;ynQ~LdSYdVu z|6NM>4cfK7-AcNqg}&zZrR8LY5m2Scw3(=Og2Y$dMi;{$&U}4kCfV|mdl2%Wj0X9E zZ|}~a4N>tY)-&n7Vy%6}q4IO(8v?v(b?p~jlpiGS^jt?qbnA@kcsc-nd{tr#a%+WR z*H;n!JxfO4b&Q)Zh)_Sgv6P?+D!?)EKM!uIUJP^}BS7HVva50tc$?#f|GBaSBfj{t|DiH!N+Wiko3BQxT zPJ%wsBxqsw)>L$KLsz@b8z2)EwdSq-MDocCS?w-1TTV!G_((dIvBPYLjrkbww|AlK zVL|u@3Pxkj?amR$30SX2(!YTq=hGXg*y^ z`rEd{Ub#%Ul;}O>W@KVwLIOU+$!x@Smgiu|&4cy4NmY{i+lpCoK7D#CAYUXe5cR>S zQKNTiiXi(AYx1+CJn2s+#RI0k5BXrNvbpnAZab#v(x6& zZQOQUj+Q)5iv694wQ`2#8cO&3sN4`pv($2ZI|od#YuCXYZ8sO8tiLB zihVukqV`N!xL63JGH_CVAR)O@_SwycKM~hGKlRTQJUtd3D^hf^qrRgx znaczX7tC#OdiFg&$C&JJ=F*w2@q-5ezw{sNeFQ|nXT}~OM(%E*hGzn??r@+VNj%?< zP(9Z@Jd}l&D57QNo+?vrx0fQMkcOzupc4o2wDMwaKk~Ss4bEW_8Ft>s_=&S3Lw!QhM`5u2$)_=wj9bmH){1>egy9;RM<6#Apg1 zfj!l?ck4SIQ8(R-QJ4sf0G^7 zNlA02SwA{6bB*6#s8--4l&Mt|o?%_=9t?#ADo{ipQd}+E5fY{$t%8As0&K>4WE9`_dW^4;KA&pAC)1+VtZ? zh~+SihOh3tDD2Iy=AqY?t4m*~xLfBt@_T}*hbPi7aD^Vi$qO(`(>;S z>>&gT3^W0Ff1*sLld8RP_#F>1 z*LN$%@Cb|oqau4jiYBAX_9vAK?C7n7hZkh`FEjO8EflWrcdjO6AV2Gg5CAiMuEvfy z4blyIyKukxdf3Yop7H{=jgsr!$fZNLt22j^XAb*6Lxl1{v&n|FRpJk$!Mos=$VgLu z;3g6OUOGnkLV0%c&0?=|x)!-eu%NRbenn;m7U>V0brGHuAZ_V4%u56e%pU)dx+aiH zH_mjKGFUcz;;IWN%#Wh%(hwOmYN;~`EVyp{%3I>FIQ0H=Rpg32vZ#r|42W$(BeH)b z;n$Wx4BRhOQ>Fhqh%8zvRBMB=du-`nqQ|`aer{W`*QcDE^LCt)7vFAQxuIDpg2IDm z>pWksP`U(0p(KO{sctxgjUM(KVzjLNa2HJ#iK!D^iF~+GA_N6g45K40^FlZUOSC`) z9Tk~&%MQ6-=S_vsU8UAKFQEK6hk{IP#e)$J{@mQzE?4{Ob2J+MlNLe5gryJAjq}^P zQY23R=;Wb5?WRpTyXu(cDgW!u`8=2;dz@`bQxl{5_nABwbG){U#)TfEeSM%`xxj6Bx>rj&^7#Z zlZSfNGecn>7u;DNZs*H5AlUuaJsZ>kH;0HKhFk-=!*9H}>+J`%2#ZPeCSC+EM#+q@ z$-xyyM3GrU@h{ixtBF~^q@`R6*+?>IzF5>^*rfWrvXEX5U|;Knb-yJ2TJ}=aqob^w zsqax!X@jgWuMUOdogOpIHutLODYc$VeGB#gURiE6%dQsJf_L9v!L7W_peycCsXa2(lDC;P!FWj1@0RQ|E+4pZlg#_kgvFDS;gNM z)DR}dzz>Je#@7{*HIXf}qHH|-?RxYbnB(c9(*8W(G}MDH3eTgG-JBrGvZW9F1^dVW z4APa_7wB+b49o489v_&nAyWd#RF)WNagYVLa=+fpmi}cAXTA+4D$d3{xPI;W8*7ad zYj;0je5uK&zsteK89YxR!`WB`aAtS2oPB>tL}wW&ept@Y3}QU@(@4EQSECTZNbf!g z;1x($Slb1)Wux>tGfOFPgo<|V5CMz=pmN@=jR%XSYI=+8q^dZWOm~U2O6BL1r7g1x z;)|PVcUiELbXE7Nm0-UNg9t`d|FsjuXY@Tp+A^M~|JEOQ+a#oZ=d&DuSvP-u?pnq` z?j67doS}MAkihTn)0h)=nO`-zjBl{Q)qILH9-+jyiHl*f+ILEq?M@_?w~BD zv^nBt1z9!9a;znmXv z7HvsQH9;ogUS75!7vv*v`*a8lxgn9f5nC|w7UqpAU3Q`~#edB)PO;HBr z4Cv{~6~eXwSY#?QRhN$wcpUN6-8X$_AaiWK z9ko_n!O&+zi823?3xV-7D1RaNZJApJy=9{`BInHk17uple%Lrcy@@d=gIp}to;H}y zktBJR7N<*%#9|GjXuWlU(@?7UU!lDtHi~hs^H^r!%{sjQ=qpv z{740>?jH%(%(92{$)H}2HS574OVFGD-du2>JI=_aTF zS=I^O>)Ce<2monLr%_Buld`$Q@x*tOtPWR_wv{Pk%ZdNxlAp*n>b>!HV$C<P&XFRd-5Q@%OSXtGxB2W{vy>Uuum7 zlI5n<3PlPo$urA)@~szwxMdARj)pm(B=x`OE&+Y^=)bKYMt?F7GFfXxpHPd}5-t?e zN9u_!cR{sx4a_F6aR%_sAdxSUBwTnYHiG|zm z{K{X)(6XJXW!ab**>FYlM6Phy-{2FLF_(P9CR4md`%VRy1;04iKyXB6$!)9pBo*1c z+;`NhwL0{y{o!-KDluyC%fzUDJ;7j3HKKApdt_Up5sPs`vPbIAH^hO)`5azI58j}) zqKvZ%8gvOt|FAxrujO1lZ^lzSmz=8@%B=>OELfFk>}^?&M|GuZEon*b$m(zTX!3t~u$XaO6!v1=c7t#;+*(d9@7O|~> z#QHkJoWtFh0$K2{(O>9dFvM8C)xN1p`=>lVuC3k!1 zwi%D+#6J?O$Pl$Taz2TG8K#PY8AFP$9qvR0bO{)LFnWfOcQ8t0?ne&ny~^wP*jrYe zsPQrNVmI|T_y$z)sUaKVy;4Bir(VzOwuPSS-}FZ}vHpTg^hm?t8PS{s;D=g-0bb;+ z6cx=ECz-D={SvCvmTuZK?{MBWhj*htv(X`{LL=tq`(mt4rje{70T~V?f&P$A!D5Wd z@H?6epyl{LBy-W}c#rSx>Kz-Y`^2-}mV20O)kz|55}r35$DJ#w06LYpyRc+^zm6Zq zH)>*F9U6?~|4sh5g8lr}k20|i_9q<3leLL}?3US>{+M7b8?^1YWN%z||ClgMeF+FP zfHjocm}C+#UyOuiijMldc$l$e6y&9R&~zN&__PYjVt@u5mnyg$m+#p zSVPU4hU_Hl#+=%RzBr6@lm+kV%bOHZ9Z#zQMhDUKpMNoiGjfL97k4+=ak4>oqz!NOb8$$)Np92~k-Ve0yXtokKX1 zeb=9%a(YUxSBb>#Dy+458y zw0G`}!2zHJag}Q#&-$QS&Es6OqXkyXPH*M4OHp3gtmTAn-;h)5apz;wI`KwEi2YSB zylT0RPRk~0%+=QaY0goPjVv+R9iy%2IE?q;5AY=)g5Y$jX# z&=QMdab8?p%P+VK*U^Cp519p)rTlVnLmEC6V=<$v;_l>ybm(kFAYbZ#5uTejR7%{G@yr9VZvmdCnMi zD~{B(&_y?2(mKXrk;JdSWm-<|sLie8Q|-o$4CJ*VMy$%KV34fBSSOf3sgKZrE`TFg z>MH>zSSH|g+ITeI2zzmalPaS^dwok^OL#qqH8pyYIfzHu(B z6=lMBOZM)TMv~kKs->m3ut0&5th;Nf_N)V$to$(v`BIt#3w=+yEBfj1hbE4W)r`_0 z)r%vd7`I!yqyj)YteY~QEw~g-9+#ume^3RAxa>t6HtfqWtj^)%-EobUB4C2=SXMhx zF2gg46p{>mbw8^?$ohfyFp%I{muiihtK4OV(GU-3H^(tUHj}<8<)!NM(40U1dlN3K^cx&*b0%VTnu>REB&l+ z#?j?>@An0;$IdQE1jzZ^^N%`gJ5I`jr;mU=l)Q31bMvNljyERMz%#RH~^7DJraP4{p^FOrdYL z_Le&@n-P{QiGyneELu3z-vI@#dC}S#tJMdthnhjM!^6*=chLz;>K;FW4t{sxA0HWD zGHL!)MBL(jrZ=U>4atYQ70sf%v=e6-5;i*DTxg=Axwkn5`bXx2OWykU`l465AGF_P zhlSjvbCS*kiyW78vZs8ec01K}{Be;lF_CoC?yrRVKBzFCn9lYC%Gf9N1ubK*Qed)vOojipT?!`)v?hIpHPs~AoMqw5Dn0j(!H#viOglz9=@ zJY98_24w^X@TFDo5$-}0?#Imt?Td20zqjO|$j#TM&f=W9I3p_w0F&<b5E7i@D;aND^Q4v{<7ALh|BI0A~RkQtfl;^+oIuhcAVb;n%3O|zW3O} zhltD5V*T{?Ue+Qxg|=CLQZ1T0&OBC@Y!bzFtZm*Vtq5YFP)kR_eh5*vrJM^qHAS}B zt1&BS=#67m)WD4?*1kSVNR4BtpdWlTz2=E%a^=^aiYqG~R@%Kvs{m?N8Zu*La=A2- z%1-B)VPXd9vR?0r_96>fo$BbkrTzx|t!txoqHX$}ceSgIPwpRs$=U783wkP;Fv=IB z0@Ta>eQG~Ky3EYyzO8GbRV`@1eekbLqIBNK`2jBw1MD`KhtdTCjtU$jQIB(h$GR_P z+E+{%at0)D8#C`o55=CU(BGqp-sFkohWDPCH%%4mRi%9-!<(&VMZ14ITPQ~~Dzy3B zlZTO~AY{2MdsU~hWZf$Q<(MlFONbHX;=)qHf45fq8?y|t_zOvIQ`;RU`>Rmpc`wU+ z(1R8(RW!xGmroPnV_`C(+_Tq=vEE+@6(vb=2u&D(_mo%p=C7QX+(`LoBSh$q*r4L> zp;!Z%8Hy40%ZFI%rA`2LzwrnDJZ_;6rKSr>-@LxR zTEtNiX9ay%+!e8>#+%14zi9AR(9?FNkDf$t(-58$%b=BKJnx7(yP9~O^S}g_hE*Hf zsv}{fE2fofHU?B_@X!TGaW9+Nzd^FLCE~E1Cfd9U2g*F0k9ojn@Ks(@<@Ij{_rMA~ z)Y--@yH6#bI9{|S0CKa8G)ySP^?R#MHhR#p+HP$}>A#~%tNV5GQ@hUu6Kgj*aTC zO1k3>R~6Uik-UbVgr6DkxX~RSp@J)nl_$Qu3`Fp`5=Ui5#F+5V-kVKv_g9<05&79!A#WAE3LVQ+pG1v+Zd~5=T;5_w#j1`D#{bNNnue0+_Uc zwO#qA7m6stGcAmc#SiuaByixnwd!w{)OIT9D~GTQ{E$pqMhBL$#2#xXe?I(ZWHS+U zTISlzcPs|+F;lR1k=~tPLps_X(e}I(YfE{y&lze>3vgzr>+Xq$D%Lm+jrAl!ZB-2# z$0AgD19M(+OTp^l1Ovzsd?4X<*E2L<%taK4wE6A*`5o-1KS~h+t`O>A&1hAt2VWBc z@lZ2Ei7wcLQj{!Ov8X?eQ6`%gk z&txs%7Z+(I*lct@^H-$KK;&0uXX{TDG^Ss`Ov=%DC+G)wDs}oZHmjP0g7$u=!9K+n z+c!6LLlC)w;Pn1s`u^E!_gehrK4OMG6@t792cdF%IW|6iV7f?f_v~CW$%&(ZPMS}=-lGs@F{w4+6;wRwVIXc5-(Vf?|%a}Hn0B`D+$)piKg6P zk_0^vFN={Upl$k7DSv5U!@}~6ucd}Zl)5mn%65{yf8VVjGcZmE(>+FURF>SvobY#A zEzeJ|DUumFfs3KDr{1SYL~@zLmHkT6g6x_bnw^@{|0q1Y_;$I2MxYY!X3q!N7}WCW zI>P9Q#1~JKlZ}wsx=LSp!Wc*wM9jy7QbWB4>J0gq5d;iGc#r-h3?U`fwD9XE9h;<7 zHE%`M8o6V82=7>(UG;+9xzX1|jU;Pr4GZh9U>3~4_o$pq;v8R5itW#@T*6ndq4wN#(<+d3`L~RGNH# zJV3OOa4OPf@zb;7V0g2DJxOic6tyl!IY-}BhV5Z&M9)lrXz{rA?OX2pHnV(nXnI3U zz*8XAJ=Mz8;)-E8yI0_2cnmIffT&vX?A+-%;%MWlQY-U*|IR`_b^|y-A+w>i*Og0& zYX;J;Tr<7KL~M^U8=S zT9bAaZG`yjyA_&-vMr|X$2h**UVfu|JjB}}T<^|G9ag*b+}SE@en zr*!X(#y0z+c`0!pviSw}9k*q8k_iBl&wJv9*ZWp_rXH}W%gCfBUYg<<77M1hn&zuF z5!;m>1}LmKBW3^yPwnTe^}Mao=T@LM1)Hg~#xV!|p&Q5RcaMDTEcbCGykt4FiROm( zz1xc~AU@rx)%!vj`&8bsw)deyB9UL^Cr$*f2_HpX2Dy)0y6kQ?5tAkkD9cZqK%?S> zZQSx!Xam=IlKh(pvJLow*CFTDmfVxyp_$E)BVD}!hi8vV1t$mLRrQvhG7t}E_UMIL;%-+* zt!1e4bAQ)m&F%R&-r0Y;Me%AlNB5?!Z+UGud1ZT`FUw?*peq7}htJ&%gSMP##K zbg!%PbdIJ2I##bVvMg^99dckob6&};JrVZVK4B%rD{}@pYl0Bh@MAu+>9ilKi;sfr z!jo~~N1n0TVj^L{d*5a4%Fq-_x8@u7ywlwWe&g1A_)@Q1nIleugoaaF%7$Cj*2Oi% ziJSFXC=RrJq`AQfRps^B;MPHeow^7`FtU?brJWpSaF%v=317Km))){pYSv}UKAdUB z<{NUaE84Z;;oJS?9{%3Npa}{uswniq8V{Xs{qCwjU-sFdMalM_WU~42W+Nw7?9W4M zQSU<+9SW9Xp(wXxggGkbjTFGGgU3=jzfAU6&JlGrm)OXc4rS;r5F&I*Q<{EBsdmim z{P3wSwXKlgkqrQt_8qCd&p$6(LODNP#q82w^uFBig;0@=F6rJl+?nN%=orw><+_hX zn~qCo(2e*8TEkGT{9Lr&EFou)LxvHI;R{IaDpW$x= z>Qo>eVeL+4XZs$u6@BTq?p`mIFk{Z%@GL{+bHb=2`1`9G-|OB(;p^Z`+L;Zj z)D!palFn;J=WGz#*cd?`U;z7XbvSi(bx&c+gO1ZScJ#CJjs|=6+^38*F5j&^yk~@x z4`*vnQx5xvibzUKg({eNPgCrxREg~}fY(n9b}J%m86jm!d65CVisnOr$d5K-?96 z#W0Lcc~9DUd!xp;u%z|nj%#P#zZ#S0eL?I+&H1gn6lryFOLGRC$!KA8PS28 zJ;IM&8sL=x&XQsy2S5rlQfPdgF@|=HLga6VJtL;>1xbk4+qufzoiQ}jgjtwIhu4<$ zdv663Sw;YwVe?1E0+B9KZ*EBZ(Tz`22l@Mw6I%;mFCy-wrpWyjys>NLlZ$GiIimw{ z&`s>#?$cpv#O;8Q=z?SLuE&hgzDP`~eX$Bk@9H!37mnu_=h^W~zHnVK9$#0Z3Waa` z-w}edI%3yaP`Hq>CmlIdQ>o6*Cb3RBbq%Z7Uj|{a>ijgIG^5`smWw{xvY0FKINzLf zOdu4E#@UuKEq7b{@>zuG*8Qcj*eT+Q84|7y_uGZn3`ju4@mg0xrM8bmUHMu8PT>Ai z*GdwUJ)V^1+K;|x5QOFv2wL%Jw~NI>;06?#Nbu=j-CIh;qX$%;52u0xGR*_=BdK;- ze%!U$(7vCVnnq&pt$F_r&hrYnOipS{ikb3e^<@Z-Uws*i$QSk!EDVKF?22Su&zpJ0 zrN7G*&n77~^ofgWVD{YZ=4}b(2&XR!^H0Mu!%@~{Ui@?&3_2_3wPWMJB_-8r0ZpLu zIm&+NHl<^#Wj20ls}=VaF;Ob7R8Lae6UzkBz7dwA^dJD>LzJ|}9Rd%VNtQNUojsu) zKOO(%Xz=F462{lUVuQ@!!|2lL=m`1=K=8;5ci}|LN`4l`rs^xuvGanQK}E_YIbe>A zu*$zI?uQfiTsq%d+~ves722CVT%t7wd*J0ZpdWiWkyswHNHhPf!EV0M`P=xVaaVy6 zR0sYr4U~%m%HTFloB^~IaC(VU4Kg(iG{WL1Ax<1#M8YsVh%^7*6r-x{BReNJBbn69 zkcN@j5A^R<+w8AD{Uqpgk7&XzSzDF#(m>kIe|7n`FLp5Nj7wPBuvj{0B52#lrO*Gw zbI?hX@|(ww`&HA67l4!FH~d+s96HgS2o4@31jx0<6;E;T0UrQ{#Fu@Ii6nWq04)(H z2{?;CS~Qo+BzA5TOa(q0IiriWWj0igV0|9je{_A;A1JwAzHoZYZ&J+CRRw*Ok;OGA z;%4^kz(Y-y+o1}_WhPc{78G-eL$}a`yEi|0MvjaKt@u99j}0wni_W^k%*7G}TX(eL zv-?Dm9Ce`Yc!+plh9B^vY26%F3gJn;-rV`K1&~zVSxJb#)o*v2EpQVfJbKkAs=zVZ zY9c^e-uVTwUr8G~mzFQ*yUn$4wm(14^Kh?k_^hrhOQVafvU#Xz=jwP>WHygPrE+fg zpkk8s%qg8+J0?IxZu&4y7YSxuRM*~t8K&PFK!mNO+_?x_2lrRZATPALaRG#N9zb+t zum}gSZw0YU`fA=$y~wD<+xUZvgP-Y2SJR%gBMfDbv7*Ky)E5KV1D%?N=D@YJ^wEWr z7ghYeAE@imi1ghka*=1fO#N}U!Ziuv0$!>8lOvYRJwMRy*n5Y!)Vnlr0$fL_B<2ip zwd~*2-B}q8DQJ$3{)7(Mz>TPZ2U`hc6MSS8iEREry8`MUmawa1LnLW;(ZLdWD%9U! zoLaM*|J;rF_A|W}j(A3OmwtjYS2$cRfq%`BCWi49Sr>P9HDgq6lzQue0Hv(B#HTZ;Y@@g=H;= zoXZtn;wq^zHo=ZHZ(GKjqmmX8|H|^O!&+~b@P5w2406I;&R$jDnB3@qSbuK;7(jv1bxv} zlXFe+^(n%-U*Zj>;1lVUNUAteGF^ZO#O>}@cr5#CbFB+N0n>^gS<3zUQh2Iq>F6_l z2Wf{Jqb}dgO2VQ3uXgXOJF4lTN1CM^qK%mb1ZI>wTtXcBjY)D&U4zEwGS^ZUE(z`L zogVebJND#cHUdjn8ME&a%Vh{Tqbm6@!6s;(RWi?C$ysvYpT?bt31J;qVm7CN=?Dv#S_WO=GreS5+ z%zfvu@{j>M9axz89!-+hp$ppxe6%+e?TopK{T7CL3fC0g9_i;(@8X#i!z51h#YH^T zEz#5<|7VR`68T>P`Clf`zKav*4PTNUufL^6X4V#ZG_Cx<#%@qrPz&N7@4j81Q^3|e zTZPCjHPKt~E$TnmtiMHo#x>QW&{35eMDWiaPe#2`=Y#~uhm-cgkjFg%gnU(5m7A}l zsj~n1Ni;3(JI;s%QWP9}Hv|^L(N6s;Zjdx3UEYOfGehmEkz3M+_ zm9z?I+)umxyu^#q5Yq6sFn3>?eoIRV4+xdAwy5a$bNxXvOJ);Map`pZ>Kd;pw&)gX z*BJk^zVVGaH+NqDx&BLZ@MH@O#W5~wq-bU-T4rCHr$qX-l0P(?;VMgkA|SxAMdb7s zh~$rBKvBi}wn+h2TLz4ef4aE1fZH3-Li`)vKUm)c=1C0ACMnPs5djFI1zq433u0T{ zISUJIr@w;1U|H!QwIq4G+HK_|{55-Z{jvMfl{y?evhWEb5fiIkG~4F1t=Vrm@X>ZX zdT#jZ=ewKz%aWs$i3|e^m?Kw{@>>n_?c`cUC|_?{Goo%@)|~2v09Q{$!*l;$*oE8V zvJ!y8ZLIr!YTNMi;eZ(!lvxzGvc7dQh`s%{| zxri;K>zfGd?8Vk`%uMI3q}IA99RIr>*1OWmj?`l?4ehNhgNCE3A(!LYIU)ciHnsr# z-JM2on6(m@78?3&$3ZhnbbjHhmZ8uf?$m)%7^r&WxU| zJqrc;eIiG8??_@9ZR5XkBr{)Ur20PjR3<+~E;Ke)s;GXbtbP%@u&dvwf1gX>K2FJulf8AZZh%;aPGjJ6q^n}1Lv`?&D^^}rmYb6jyMZ*;j zO(-pn)(F~x5ez4eAdDE^-h)SJd_`%ozl`$gnF~A z^v!_{#OD6?`xzJOou_N)*|!R0m0N}aS?_fus}AjUaoJ` zqRqZrl?;+jVI)t*qRkmI*p`emUJ%SEdt4fLNWM4x>>Ogm+;4I3!e{e&1DL|dT)Mgl^})mWWvoftS` zVy|v6`i#o**f_ONvCc#ejss9!fZXI>Bud-=yR|ddB!|;R`MmM^Xq(!{<4C1loa z^2e}HuR2?nBACiS%15-5$N(hMtvn}W&RQFEhvW^nKyZF*U}?6Gz*)N&yWJO^w(CDU z0r!_adaGmKi&c{XK-?g77{LSK?8014n638WVM}}iNtNe!$?vM^K{RtH?Y+L1K*s&B zF3*hxyXL&!{tuH8Zu0^`<_PO23zp5x)LvdlJvbl|4dXOcN{w%Pz6p z1U+xBmU>F0X$Do9+)Ey@ezO}j(#`Oq6Ry}SDydXZ)&?WKpCs1+S!Zxu;R=NGVHqFgpc$+o3JLu}gTnnV<+_iSBDB^Ro<8?&cw7rMA{Gph3Zmx>aj`dq(Y;t{D zXN(n12T6WJvIBEuL{Qp`6k{W!zfbHREA{u)zR2O&c*MuGWWaG&2i%VKoFattthWm6 ztZR~5_%>m|jNA0~a)d5)VH?gi=4nU>&#FIvwiKWC@_f_zatYQfxFj=dJpXoZgr}ZW zcWPWzCNL21yyj&cww;?0lm?Vt<`&pnjFpTa(QO*kPXXl)IE1Y&%Nln7F>C+S)7|oc z(R>cBcZ@4;HXI-&(wP#geNp>&(A1qXRYPIryy@;~b2wbu^pO>t72rgK;nA;R>3k+& zUz$|zOxQCaRU_zVp{61Q_1j4(OD+=2?=-1E&&vjzrBAdX5Zru&$g`f@>T^DlCpx@8 z%UgTde|5p1HK0MhAad)0<1_V`n=egrS<_+dzSuKnFSF(uZu3EzS^=Ig?Rn1?%HH`R zvApM|>5W@QtrgjrC?K%JI|= z(vyNoTyV#hya{)Vl^noam(us1aF}$>c*5Nyp&9qB)~Q1oyCW}PqAz^C{=NJ#dBug-34H3PWmT{Vy-m731MmOE*qF?1-%4q$3>&Qz=vfqU#lPcZ65T#IAP{%UUDH^fPCr@eE?P#5%OeKKWvE)Ox zd189zZ{>P1lgWYA=If3-T;g`%4odiL(JkcF@AYHIrXl2N7vJ!dKbV;HmBCzEuQ%W` zY*VBAFSKg3q^UVi4Bd*iAM^)`94*)7?!el@K-hEG?luq2b*XMEh8!xkS|NQIhoyfl zi+US>;-cFl$7;V-X4xihT8S(4m9>^T8RDqrhunSa9So1HsJH1QuIAi)qnaa&W24k3 zl>hfcyKAXW(z2^ai+_vW@LPvTn!c5ux*QBYB#I$HxI*p=({10W#Rr`Wugt z266^+JoXKw)n4n6-P5AoVwcT64iC9reYrc!i`3Ax)uxJ5j^k;A@MrAwj4FW==cF;@j4S#voT93KgaO z7dzw8%?Pi{h>E&c1)B)dwbVeH&7BXmmfe-Hx8;D)+ruSYPF(jk&D}5GTifML4cqsf zSe^4cduxi@48OK&VMi-#R`>m*c$PxMWYvVe<#R(pKn!>jjcjD*jP2>xXK*RPdmRx# zcM4c}K2?zYiyAi@95zUs+0`C3{{^7^v-`ex=l(OYr7`+`Y6L_Q4tDUWbts(dkh|Vq zsvea%XS}B>b;B^epgH$T{vN}O=iT`Ien677(T=l~RFeRx72CX~nty6OGJ}gUwcMWO zd87U6@p(Sv)XdajrU7lAt!xlx)kXj`MOVP0C-9B7%WM?zX!$em7csn@t416(*vh^DihotJmD4qEY zLWg4>$Kj-bYP6X1W1)~@wtdB*m-DIp;7ku$w?&J_W+jSKl6msBb8Xjd9`O>7gk7&u zftQP^(Pw_^j~07`FOt7+FsrfNrO_N-UNap^8hu;w7kO?<$S8e+!-ok7^>rbwM(cnZ zmtNaZh)G^iarTVNTPS!yq0Cjjf$UosLqV1BC!V=aLa{W9T@8Cm*CK{`&C%#YE+P{n zPaXHf456zg>ldWnmP^wSBK5-W`F1fG5x!0cbzmtyBziz6ggC>V2;4XGqK@;ON@6ob zACuLnSVh=<0#e5v%8ajq(1LH+r~v13cwU^5q>y)4B`?DW65cyd>FagZledlpkfX1P`Yua6?e@}*C1AeqL$V+^~#Y83Hp;19LaV(!#TM>as9 zW$J);mC$ z(CClYLzvtp<`ywaOWAe~I!hO>XJVJ2SMqhp7+Zs0l^O?48<*khK3?ZG>YY7wDKt=`*j@**Zh7*AFBT|n zBDxC!1Frj`lBJsjGO8C}sDR%RD!E%=5>Fn{8M{~4G=KLF|JGY}o4L6T$S}O2CH4fP zX9GV0r9iK^{3gtLuG0n*TQ1L3g)TQpLN;kRbDf(7xu%5$g!Yn$Dtx`Lj5Mr9wjr!oFyU;SYbxpSteJU} zxerS;mYs^^yjCpMh`MU-MHzoiRAj+w_lT0>uBjW>&g6<8@0OA{!bYQG7t)K8i~(h5 zp;3oSBm3Vp-Gr~T#wxjxr_QS;uFk#=HDd{lYOTiOiW;H{cPo8gtt07mRTAIQD1L5h zz=W@Sv^cuwUcJC}7UoWGY%IN>Gv1x#6^dZ$%T&5_I(0Z~ooaB`#XDvt8axBd?AopF zbT+89j+H8}Ze9A&v+i3NFC329cMr9HB68+!RRUbNwUt^lM~bAXnRG0ERFQ^vC=eaO zQ1X(L?bk#XT9&@J6|Fj|*G}-dU}a+x_=~E6BLC%#k_?K5C>DGeEHdIuUmKM(HntZq zDDcNt+10|MG9Wd4$u!>+LfZRDQ}NCG=vASYRzkBUDbwT-$AiQK#jt{`A!DO8 zUn9ezhV$B7?rfokpjNd;6J@%mHD5BZw?R4}q}mX(MVfxKyWC=`--&F#wEjooWdNrT zeKMb@u0;Nnv_>F**Q?ocr!Vl<&hV*TnoZ>+| z%FUbkg(^cNiFF-*&u0*?5s}#{L&AIC`2L&o7e_Ki#ZN|-Hj+M@i@0%_{Q)%xr_ZCc z!ABhaxU($`?eB#L?(1`w&7#`eFkBVS=5ZIQO5=74YP&#wHjJK>%{W%eowMP9UxkjJ ztBz6X-riy3OgxUU0Yc|EpV!Y@?)JaR3K?&ip8Q-IPkUGWs}E0StAIB45KhU9Q|k-b z85Rck07E4qrfpq{^M+IqAs~2do|Sv-*`LQkxqnEzvx^RVxlu51*LU^vzKvWnIV}_% zD|f&q`em{Tg=56xM46~(LK^@x5w?>2#xjTVal6P;tW12lVxjb1B{QlKTfehpIQ_;& zSP`TKW$Wgb*ZK_<3>jZY;Tl>surrs!GB)sq_Bd(AsW&kwY7WhMFd2M?ccNpylfqub z13_zIa!Z6QpHHmF+BU7!js)QQ!~~Idg<+~$`5BOnj;8WZPp!U*g_vF?QMzgeXS~qW z^uF8z{8-{Qawryf?PX*6C6gmhZ73FQ1)+<*@DIqd54`cMdkF+nyej+Y|&^3WuUG}k3YJW*-D2yt%r&r&LXaQk9BUI zfY!>m*NPv*B!TRZbsJlQa{j2ZPy1b;PY_a9u~*>;OFwAbBz`#&X~5tsdsXq(bA6ne z)<%!gUEZjF?a~bYL`!E!+DKspL#?rpo~bnBCV$c&@?K~qt_%8wk%Xs!j$;jY=^HXh zv}#5rQuofskk)nx4HrsZHf$vI6Z_^{N^Af~We#&B?8SrFe%ZyrA>vX4$#z2+o=591 z#h(Bli$L0eMd_d9amI0^cRZEEi8<)?o%pNB9tZ|C$ZN{`9qZ7|237CCLlFUDj}nb= z>sM)XD@c=-{HMV|k6hPF_>k$=gaPy3hW4~CkI%b1m_E#G&M zZ*5n3nG`_UyLG(M(>x`ruNfZAlHx23sc3DIRle81am$7=9X(bZ^F0&g%w#wkQ`OJ+ zEJS1WA00e_www%NC{x8!9Bt6X#~OvHRBlykQ$l|-Dz(cW;1HDGSR1YLFL{sFHA<-RO)vW zDH#EAEe;olHHPxHRAhrB3U^2cGNYT!c|E``F!)8>kNTIzy?zBX-!BB)#k_YwZ$7Os zlY~yZXy{lBnn59vFeXW1F#g69L?LF_5B|=n(K!JZ7smVSC`hqqgW4?YFvSU zh1NFVi?c|4V93UoZ`-#J=ihWKh|vjOmU@xiSl%Z%uOvxjRF7gvBMKB*1U^~6lIs|& z7YyzW$Bv!x8Aw*r93e?{gQYmnmF4r#TeB!tWGsPNwh^S?Aing%=su>;?rKI-13JBN zAwIi?m-_6Qe(1tT0wbMz45LKUPDrX)@@Fhs>5-NK`_li+tUQyG!$YJyW(@+Y;ACpM`` z5LHHlUAe$6>F2s$ZKu@a`vv+(EfYdrQTl|75YZS**Z%l(5l`l6LY2`yo zuV@G@2mEeKK{S)-edA?mi=)yv5hQ~LW`V9|GV{LfwxCqFusRo3QU^5k8`P|#>4H1i%e}%d(v*vh20a|w) z_rf_#OaA({{dhqKm2G*nOo-1#>)Vv+Yy2zp^3kxHm-&aizI&efCzc4mxlPXK%40~6 z?l8V)&iH=fBIt4(=b06^{z}VpxdY9`VNYdu5tF!)~N%Jn-`SgU~iiQ`4z6u0Bn|lO@;I=UNki#&r zgqJxb#Ahr|d2xrpQfH6vFjsl459qMKU�=7+0~2nxUGdS1CgHb=*E#?=0d}vQJN|ZB}C@SN`cQxey7= zW}ZAGGqpUeCf*5w>TJe3e`f(meZ%y_e@fMVThjbxqz*TWii3!T6(vz-67=qsA z8RmCO(wTe6`qQ%`6W0*k;9Q%%1w?>Sn{r>1qZ{L-<~ zTZ&H$qk1k+OvKM*&SHb!^Yc@sRXvisx2_Bc6Tp1T4Qwq zgOTr(Dk84WQe{NW0%MsdNf>cfk3Pmo)~Ll?`u20);3b&)9k~}~>dK=(-!6p9b&I8i z6ItGbFSYIAyEfe?&Efp;UvC^w<#%WmY$kJEk4m71cv(4M`&-1EGqj2?Dg1%EI1UF^ zfe}}yuFs3_%NjQXB)R!EM(;8b6Y{I-8^SUjknOj_Bkem+(0NZ}sm2g9x_CUQDO2PE zS4IUA-XmB>L7yxv`Ab7N@>)=*bcYtvyG&xMDT+m=h!cj{OW zwlQ6`<&`=w$k%JO2d2>ZCV^m2)0=nhmji;WGPM_^qUY!MynWiP#7gs)^DR|(>!tBW z%t&EsUo&PxS0s{4wChnKDUxnpwxWV)s@$Xm?&^`eERP~>FV45B?l0BNyzXnYpU%Wf zezfuPwv9lDIS;)!LPIx}NaLimN8=z}$$%%Mw&Y7fD%SSY=fl&$ksZp-;^#_=>NoXc z6xS=zAJ-6T;~pKG@peb@1+9eY#yif32*QE3w}F;zMR7~Zv`tE`!E!#2XGeE2V{;}O z9Cbxp*&#Z5RkojBchj{FA&n@wOyYCy1{%G(jJfyt@R~|e2uzyOxwlDqeE1w)KEX`{ zhB-PK1|GM$Rxp~v&Yf4g-?YDbgt;2?B4ZcS39ojG{!FEOyvnO%T%<$w%2HGx#lzta zT_j7b%3$rtALDn`gam7A1*-ek)b}v;%_{ZL4c{GH0*6EHvKx%fdVRw3F}d6d9CoSk zN>e?L+fpOf07L{s{&HvaSvof0p~xB9VduT@P`_?#j3rp_q{Lc6#zWu`rB7QRx*0|D zCAw|315|$2WO^#L>2l$2LEDEeZ0DxF&mb$Q)M3R$p)620P=za?reOEJRR35O^!CI1 zL)I(Ve$%@mSGfr(N=sOfH63Hu=r!ani`A@E3Tbjd-VN=tLAH;O4A6$m` zk_APdDn6tB#l@R%zJ&>;B7IU-^#^Zul94<^y55PBr!A`Se1&N-6{LU)oJ$=RBCLob z7rbCW_zA&$+~zO1&$EJ7zn%M*ICqkIDKN(ByYx9mNo!S96`PD%8YcHwx z46s*ZYL9xrBuDm+5DE;nP@g?^9r?Qcn#07SUW`pHwU9-CwrRNv^?Tzp=1eV?LJpQy ziGu@BphoFCVrIQ3ouI^wz}yPBSU{>4i zyt_S!M<1p_ry%Ac{zOId7dU}1{^n0kjRJJwCktduA+*CXrxIbag4C$|u6dz=OSV0K z9#16V>6O2cuYX>+dtoY7{n#%LhU1>mbJT5FbWF_zGh3eE^`2)P^*%yC1jEYDd{Jl1 z#V-3(MmIdmpYlXTnjNldf%}9;f*zWhjF&6!;ipwZ&yf~Zq9IZ?1hRV>#w{YE(H}AJ z-zi93uAo2PIyt^7+Xkql495^Y110h$S)+m7ux>v|cHu1@4rFt9xSIS%Z1Fiiuj@{94bg-wG`Cb1 z{apF=7Z4$?Nc%TvAb^7elT12SXqlL|Bgw0ZD{nks)#C#ZTqqrD4FQzOx6}I~Hre>= zya$i#J$}y{1d&*;zSwkJS;@@&surHFB{wt>OXJ0k?@6-h_&tKZV zQD-pg!vFswly3J|Z%cquRTXlMP-uQ6Ee;J}JS&3%08xhHBI$GNm zeh|qj8Z5p!&KNxYBd(dtP_!)oi;Sgt{B5+<@V@=J-5XvQq(*93)e^jKPo{gq2>$$O z78P4YS@pB_k0Bj9jmE|8+`{a;hU5CeY=T<~D@U8(J?E3$M+UsKTOJ6aPO03Zvo@Rt~<1IRyZm@#Z8SRQ#G7pM=W)~uueh|1xTgB8$Yo_z{ zsX}-)6dJi-URzxBv!W#l_TC=LEa%2*hAn{VVYxr!Bil`an*}vS^k!kvVCZ9&$<}dk zw(;sA{CxFw;A-OOax8pA_JPQ@%063Bhq78fbffiXIo)gC(0$Z~A8qg#eG$JGIQNw{ zjp9--D-2PCrInTlA?|%k^F}o;?G2R~3|;bB;w9|;*7ocyzHzjk`MdU#h1COG%(~OZ zFNlrqRhU{{vUN6pcKP{xY#G2{Zf-e81kbVHUcJa%@d&i-WS9~yc5$I?RC;AP9*a(r zg8MazdI~dJ$G&5gvl8;#w-Mb56+H&sl!vT<$`tQGXYj`Pw2r!*&i^GFkf?*BGsyg zhv7GAU6sgJF^O7AdU4Sqi-f(_)&51@w9LQUn_&lF*i9ciX=TX%@8SGVysPmgHo8Cb z1gM>AS=FkAOGpxw#T=X|;ug8BiOKxglaLd{Z7QkBu+r#5lR0AIlz=EEt|u8rC?7V0 z3JI0gdZ0j4&;4*Do*#xz0hmjiwW2(8E~wflaQrxS@WBX8PZnqQVxYx1u19{WFB%(` zml5?9W`_y0;gq^frEO_?#h)t^1MXRxUbj4{>O#5G`!DG@s40l#xaegxJLoPGqSXmM zYClh0?Z5faJEYL)$;8z5wPVNgM{VNbGVFN3v30TDgGAVw+bytSyK(f{iyiU%!gvk*)$$3rA%MpWG95^7V`#D3nCk=qJ+A?M|QsJ783)RPBI zoyVO>+|%d7E$2KRK+#kF|A@KYmJ)^g+0XFdPt^FLUPXn!|9AW}Y*l>(zI}_y>1<_j z#J=_7)93YfRKkrPZdBva~&9>%;XPacHL)smy}9uLml>g8fAol;I?F?iV~c#5iwiu z^A}^@ucf=3ZtXs^0jhPqzAx0lYdh$O3i_AoOVAJ?h|xbAGO|$%1_?c5mNgpj1M1o( z;Md3rJ9Z+;^k)-3S_+q&I=3Je7f1>Y2|iWHjna zhdrFJk+Sf)3>?*8?I>=i8wC-&(sc*!MhRO`ML!|q#^2+ywZ{j5*)rM^X~!w(eKA1J z-MlE#xM5x)f0J@e`fZ(l1?P{EnOyEhR?_WD36)*+Vajafsx=0O=s_2HFZ27=msh3Y;ty&zrTo66ni4K)pehG6hSA@OYRSY(2FQ)uYLBIoa6-L= zxyEndt?lbeC_-OmA=* z(iMOB(D^%g_(>%E;;A;Vxw{DmIACx+s4~d3{KisOwhEAfSy20~P{Ezq)ndK{j4?{W zTPl|~uj6j$SHK@XBU(#JWAt6D>Y?(Qpr8xvsb4vP? zr8>;-UqP9N$`~rG(-`1$M%Hup1B~$@{lN}oi5f6ToiLL83kF%G`icGbm?)l-lzn6r zQWYz64{`$r{s-eS*&!c@@0zOLUi`4Cx@ISM=b6#Q_>&NZ<*QUl_*~iE*Oc%+EHDA) zuP%qn=?&rgzUU8DZtZVf4F?~0IM}8{$aXY8JTH%OGGcC#yWAwMcLw!ZmPCNpJCu=J z?hVmUsjDEZLY!++1V^9}hzaMR0?HGLZkHbl?fODxl1+LDHLs>VVU}smPA>cA{>|{Yt^4^OIkLpdXCAIT=Q9%h7phMtn z#UWGGV28_IR`ve@^Gg{RBbz}n)yBL78h0^r#`XdUjOwcI0~P?rGgPUyuZ`V-kaBO>p%7f%7lf^9{04tvK;w?F+*?`L5S$su z3S18}X6BX}x7uO}BiN+9%|qIH0=e8T>b?0AQyt@9-Zr??B`Wc8>m6TG*qYNjvQV}F z6MvptHJhaj&tS^|Liy8^Rt`Vn0o^oN8JZWQRo4HF>xZy|V%oh^w@y1nK?Iv+4GE#I z3OM;H3r!@j1^oVc)Bc$=e_o^e)Z=;*&~fda9QElf@l{DcoZk&uljWW53 zQscjrxsk=-IDD-QHY&&nZmFZy$Q0^XCkWu4le`?y>Z_@WZ~Ow92rMgS7H@cQW-DQi zH^(Y83+itwc+t^uKk}4CH+;gs$M;{-az3^Yd=ykP8jLd#{|Z--e#8vej^G?Noz~Tj zu8*#{_~zyq+KKRb-aoW2jFOe$Rn_FV-|airY1AE57~(&4r1@q#UU2i~O-w?PZz)87 z^h(Zj$Yy4${?kk3oR_dHnP2K-$Wz{r@xsGt7_yn%o$2~Lh)3Bji-n!wA-o?rQL&dJ zZLpv%3F!&!Q2cRq3RQX^t(;wEw{ZSj}?`2eBkNa4>Td zHPUC#N79G9VWJ3$6IYMhOQn~k$=)c=#qsae<_go%uU4>(T^MFP+BmX&c&Z$zSl2Q! z{*!z);nKBT^mOddL(nF(cy9M;RLNv4M8I|omPZl_JW%zY0%PJpyFKaxX_X)D2;YxV z_$1i;yzP#4wfKpc2Iv{fv|MvE;r%c)H|gF{Eq`4)wlmvTa1p6DkDJKy-}9EwL1t6+C|)OO=Ho@2`G(V7LK zIePZH#jcqZ4*;nL%%na!UVZkj!J2%T(@U{Os}-aUw^=Vhpw5BbA8BGt_l&fV2j@a* zBqy!o-?h(hcbk!-{bOTvjc!w-XvM0(!f4E$8)0n2FB2}j=BnnJmeW6o>(Upki+hfB zM+87uYZw;F|8vf?E_-mUY1zMZEeDZ9rU!t095Jcjw)0O2kggQ0gx3ybEjf)pifVFC zrx5jznboAs&2ggJi%gCm)LE1|qz@Xk|4mg1Qemzeb#Xe9h!w?rO;i9|UQ;Fh%?MC= zqxyC7_WtGF0y*WkbL5EKIOq42Zv+2GwEcIHjdn14>9E3jN^kws0Y&racTNfGDZe=5 zyML1qC@H`6{oiM`|9_t`!rwt4T)@7ti~rUzgf)+mo2u+sit{!R6;w_jho)Rf2~hVS z_e%F)JvO@ifeNzG9;ks=U|+zC2XOy9EEMo~8WQ=4bp-6@yL2scuDmc@{3c{1qJ?>6 z1hTos0X9@E1-XNxhczD1v~DC{HvzfiXqS~d1G5daaKqgd^-uUx(SQ(RnrbPh?&U1M z9f{noq8H;Wt_~l?#D)NWd=URreid3O)DUpS`YShYrX{;-W?|o!0h4)mtnK+)27tCVVO+&*}WBfNwdVWZ5-cEmdV!c%a!-|*FWA%fOKy!w{o0;*+w++i>}LK zx$1~!b_W);SYF=)S2Yk}RPuw0ge17RW-uqv-iGlsWoYjQq6@T`pGfEtnTyQYXE}UP zjPim_{pC-cW{{tsUz{`Ze=OPAID$5j@V6=(0T*=*JYpWyY#vzWcXUv&apd9op?J>6H3Q>T)|xxe02cdarF8q z`_ihkv-tZ`1XjS*P5_6ga8xIyixcy#KYRmm27J4Wbr4-koDhaYS3k;jZWGg&E@^yZ zPfim^MlVzNrALb6tuu@=R2P$^uCZF{WYgrT$xmXRwAIpH2N2(y1uoWFsPo#aXs^J4 zzWt?Leg2ZI8MJE|buKDg`U(o~Snat)%O4^NRV?-oRhv`6nWx>qBw{u3-Q(RTznC0p zDd_*cYqn#|Vg?>+wcPA0+FQG+z&igT@p0zzVrUf3olrxR#6d4u6!7@F#nhv-^a8P#HiaO1pU?8iox$NwQC~wXb+xXF?{Y zHMZXS7T}8gUH&6p#Ac`t1KfP0)5ZYZ3GF3B#>2a($05-w`^p||K+4%6=_NYJe6ZE2 z6-^HGN$B1dpiPQn3#vk7{vM5&lW;!JPdgtiGasX#bbo9Rmq|g1KM&fU^tapWi+6^3 zgY4yOb3Z?;6YiTDLoNn9aFPR*sl1zz(2cgFlP*`h8CcFW#Rv69e=M}{JM!xPG$}U= zGp|g%KP}sdJ6VBw-yVndTC|~!X}^Vf=jl8JY7OA4>q%(AM{SlH~p!yoHSD}-g} zGGRUYg$OP9aQ6aq94vHsB@1ddq-OI%v*_L^)LTVpn6#Td6q;iGkvElpp`*bX?k+MK z6li*3IB84ko!Ob*jj*p#5ikdBPJPpCoyco*=r(iFQmEQ;TL#$7bn)c@$VZV%jR$jT6;G?Q`+nK`zd~CQ7)rdR#kWN>|aMsjOwBFHCB$2C^yvv>!=7*>OlL+*UdrKmN z2AQGvTveyZotxn?K?RuChfNUa1qIpF~=D;9#2Y zX{WFE%#LZ?Y>IG}AZu%3Y-Pbir%TZ9WdL^u1Du$9`OPpVUh;$1?zYU@v^k7s}i#|1Rxo!Yjg|xuYqqetQNW)#J%6jp?POh~8T@ zenG84WUsK7n;H;24Kz6Ph7|$7PI%|ftvFkmp@!f=r3DxM;xmlf!v?aFx9q+KE5E3> z8b4QmUB6yqlf$bJuEvNIC5NLc;WoE4SZ*$_%=w83FNQKPZ25iZ1<31K>r9EOeSi!jPmcqcN1T~*#zEIt^Pg$W&# z@8hG3u&hU(F8S)82DH#PzxNIv;$Cyx#YJ~-0AP?*2QAizsOpune$|zHx-e3G1S|I4 zY6}IJDDYQ%P@-Y+Z#?g5H1Rt6`7lQyvFP9T(c2dl>V=?2CHB|(Z_d|M>{XXC>FC|Z zlE>)3rpemMnepK4oK^O#P&eJY$CD>| zPA{cDav=Qd)+X#J{ArN#t=I>mN(iQ}sVR{FB0=FwAKBdAEACf5R5rC^c@ik!(`rY~;w!lVQjbX>aOho?} z-NnNY?)|nr)x*^#cxLuh{PLMTF1cd*M_Y5m6XJ7vvI+9@S+txah%#PVq>v1gDPoz( z)0s1-!NQeY3y-HDpd|eKnlrI#X}jnt$7FZi(P?q0>lc-a1(ep-p?kfl{D*`!H`BaI zUcw=RSB7xNuktpfG%e>Q9*6Evr(R^Q&Zw<>#q$;2GwJ8x5SsY9+Dafas)gI2gz!_L zk*;OD7KqZ%;wO_GC9+Opa_hdf!>KFFy=*TAO1Pv0=j_w(*3*1%z3Pf5LLy4jrG;Fg zdV~OBaQBV)>iukwK!yektV~62kX1*=vnB1-`xjXjyW>;c>18(1u7q3ib;Z@yLBr@QzpPcy-5XibKt*hGC)|o#?nadYS_%k&=`1j&_#`2pz~_ynLq5Fzke09*joy zc)uPg3Z)0b)B(3vN4}R1U^g#S=O*%IWhn3*)PH*qny1bF0xyS<<-YMVfvyZR*G$T! zNrz?6JQKphlWC=a)3Fvxo#krfxAE~mYM}lWgwbO)h@|u=Z`mtJj>>vTJ5yVX;0trb7H)j z4EkN=%T;^ID0=}DlaH=Cza1S~KIF+SQy|HTmeD|~3REqKe*O3&k6@tV3RAa1~MZ-*8cI zsyaGjO>|$^x^fjT>nqF;M|=Yi)fR@9KI8?#HQce=9uW@{5PvTpdkXV9#&}SR1a7hg z_6xZuSwXmLPtW;VQ<7nuD=5~~9C5_aE91i!_`8~mw!4Bk{@GqS9PVx16R#U4pR%uQ zAw9BXcOn;@T=@}M1xkR2h|B>wFGZ{H;Ajuq;>!NYIl=iq?M{p(=}#+E=Fg@MouEL}i=(30DpFtqQR@zYyyb zxCqUst^7I`4P+6r1~kOkTRQ&=nK9}hD%gLqO~kz0U#5aYhXVMyb@>5B`ECCoW&7UO zQFCD_OysmM(44ohhCWTR3vnYJtl9ePaI-@8A-YAiP9-7ArM&1~w_3>0gg@fx=NNNs z(uN)%y{?W#ewZPy_e4Bb9e^=FEXO-+bq}{YYN(5ndnIu~PY}uJ<(W4a23ECfve;#5n-+Gcx4xdu9+ zyLLZR^65)1WyK|`e&6KW{==@mC6#8C6ZIDOd|}G>{@s4C7km_9dupC? zeigS6Za?+6PPC*81lOadk(NDkYA)!z#e}x5z{3rC7Ky7Sn&b5;=fv@?k?7$&4+2*+ zJ4+q@i28OmOUV4KvMbc%T@HChKb=y;$FC$Pz9w9*>ykLnH+>RYYFU36<&6fDxWIwJ zNzzqGR`DZ-o4Payv?eF3_e(qJxu!my=NoU!JF{NZH5T-JD>d(#7m86W^>?HvwHJo(pDqB#Tx{XCpOu7GvL)(~B!W;Ho+>M9PF7f7UAtCE%76VC55j0vHI+d}`NNwX^yz zwJm=!*r2D@zQtq|ok1jZ=98%T`C|*=K_X>+OUs?A{9emJkyDolvBd3|EQ7p2|3Pw& zhVZ2zFUX!M7Gn8GKFE@5nL0E+>L6E^nfP(Oev0t^`08-JAxK3fMHNP(woH#lL2!hLe;bU*Zfh8kD6%xYQMA*y{H_E+-U0`lyPuoEPsK`? zYX_!`B3IaDec-}UP@ufPigNHE|J)e9ztsA20ox$0@ysW-GhEWSu5$+N2IC}Kb%r_O zDWNBs3PtAEfqLRNjtiI>ohI#IpOutFWGRIEePoB!;Y-e zDYQVSNbs^Oj~lEEav^DH9`9fHo~7uF9G>2>>~zgorA-TAr&t8q9rON(emd>TNrCqj zBYg~z-#m90>Wt}yDZ-`MU3M3=QLP1m({&2HPEhThfmM+X%XN3tPA`0kM?8r;Pb+h^ zZ`q*f5l+LpPcpkCJI!F{iYKGK zPu`x{-&LE(Muw?xYt^l@Vm-fKRuUH-mUJwxNj#NhftF?oM=V9;cmht(#}${PiRT1i zycaQzIN1(sEIj4s&R#rIC)`sHy{bA&OpLsWBbD3w{LS%JXPO%!K=2NeM68~+c7_dx z``AkOnfnPgo$N|vET11$FCH*=pUJ#&pXuOBa*wn+yJ((SNOWBx2vDtKO$1%29|f!R zd?rjvbEvC7tGm}k@kIah{6y&L!sA4cOhUzxgrPCdN=4i{f!QFyefB+%ET^b1xBv=B$-64&;ySoI}puyeU-Q5~@_XZmHI{SUkd-mDq z$Gzi@aqq1^%~+#ZtGcRI)tt|q^Ley0^d-*A1Od2UWGhU`O|fl3{Wt*_{u1OovgJ}$ zCq~`)do}Wu8^$tmo>~X4E9&Vo@`CTe%A4F{yWJ28Zx!LhV=+!fL3-j{oN~#QWJ>K<0>4QPYl9nY@xc)mw zMud92IzEZRdZ5K#-SP*UW)-_gG6CoAr8hC59m1n!aYT!UZ&CN4?lj(SNA4)0wIIbX z+G-}{WP(3HkOBzJ8NlP$4}2L&3(y#N|NhKbObmK9n9sK7?8vr{yVX7}6v&Y`hr~cT z!#-1@^-{EhOeiIw0;a7(IKlbzE*|OY6u@eo6^cT}(gOyG{@nH7&35&#{28cniJwo( zYW~Gk4%A!PqQ*a!{&42~H;nPw;m`L5go6K${l^^T{mB{`%KZDqf1zymi2p1EhWe*O z|2zNV|C?*ojAu~ouTkuD7@z+0yS5YXNF1_SM4hv}phf52vF~=I=O*O?b7wi7#P>_+v@cefW8HI&gBIh+dvG^j%+-(Z z_go3A>`34x26SJi`e$_MzKWC%B1|=#^0;ZKW8PRv9-)ye#fK#lo41nPd|^Jyao{Wz zu>wVU6=D{&oBxNRb+DvbRV*>Nh)BU0M|o?)R^{TWg?Y2jc8XM#F=h=;uCb4h6opap zOGQj<$cf6aRE4&AR~^^0ynM&ix_R4zYpnb75?ILLg58pAU*I>hY!Q?*DOR9ZkaA!0 z0WW4@{-WTn8`e~--zU6*49?>b&kd)lvh+m0-KX{3maXI7UZ2ZrFomqYa1uz2sq_>Q z)wUvK?BnC=5h_^DBNwJOz8Pnic1(O@Z3k762P3pSy29F%ewoj|sVEn~{xEKG1KLil z&H0>5YW82OIRBn2dZ&AO#1KjAEU6>@_oQOoFn7wO74i$6-D!;fYTs~Nu z&m_*zU%4CbOS&C5nFEerbQYPeoRI~#2X!lP{RsHNH1pdtB@2`8VvaxaJav)cr7w~c z^Eu-QnQpP&HUVnoe!%))Gx$+**V+`V&IxSWjQe#5qlJXm8gPwO5al(F7)MvhN0khL zaNtCx!|f3-tT}qfE;rS74L`qgb0nvMx+Oor#uyq$lMQ88M$qmm?eP3szO{WAu@O;` z*(k-Fi&4v=*S_sX7I02;f6H!+C^1F)`^2*r4&9qUjzU0KA0s+l=2ESKn}}o7M-o1- zjrqTJgTFpskdi0ef)7|enzT#{C|whfRJb(B~_$|Rc1mBj~C)xj?RS!!gnCQH)wQP^-At-&~%o1 zfeEs?$Ijt>uh0TvMY(1T;A6x{9KRa7m*9q3o)su1UOX~_Kc0C+RM#p$Bh?2@ns{z= zIBaQba3DX+_5vS`kd#9tF!<+R1WW_%3BLy2?7i*ZyML)b;n_YIS9voZa@CqxJ>* zCAGNyshvYPw~h)@y#qr2-BfIUvmeBK!Vb-uITbi|N3|^Y)u@e?O7Hg{YW?4=|_&6P~kd^;ag|E8FnCQre3@7fXZEY z_7$+I>l`X|JJ!g;c_2j$3Pp)VK+$6BKc#_wY7$550GH>Mvj(vHyx zLoszOB=CvMl#AxM46I=5cJpV7caelWr(pDL4>yjs0tCFJrB6u~GHGX-$wsM!4zSvH zX7XLGTo;!3wj9o?(L5#yx(h{nwr9Evj0DVK;#5qCQ)5G9CF$5{(3go%zgNQ+twgG5 zB_27Wq#}BM$}tQ6{{CMXkTFw6M(b08(VZvdEqVz}j%c14clmCgaJ-3F}>Y8JPd_^8oFFi4RL>htMkqSC+p%V7-^L%m(v>*-bJXzd3D^ z40RkH@_c4AXZXPSfeq*61$Bb&tdz zC$F3`zH8a82bOI0w~mIZR-p*dm1jzhXW**6xNYnSdDY9NPTmwY_Bf*( zK`NTd+<6!;F^#TCd%~Rp^&^q@YVu~(op0R&q{z0CUrY&H1;7A)AUqrO1ZJEc0j7*| zfFQ=FmIp}!@exqaUWFOp;K<&H9Uh;pI^AroQMOWWQ>=Q-Gl>!Z62G9J?t_qp3>yQku!cs93= zu(QF8-Nmd6sb!2V_NQg5`Bo5;dYez$Ei}kJh6qVo;C&$o4Bkstn*9=vGN?* zj%hPAuW*P?{jXdAt-Q|*HZp{1%Mt6~`mZ^97(oe;ujbtjg?Q(@jKDIz`GT{U13*#&W>i7`N8OO!g&_i7 zh=xE*e_PP>ZO~%t$$NOh!j)5MzO^{9fiwc%I7MTA#qG=ubDob*J}<$aAUJ$c6F8@C ziG4FA1nKE@SBw3_$g=(u^W@w8n>(`Py9Z_Xo|0Gk*M$;T?tQC?rb2 zRH&Xr_J~lbMf|=jdOZ0>n!q@mxQAH}Rq6n1;M^FS!R-gxMV{c-pI$=Ha8v<{tCZ1$ z*LILReQkKXQ*XO6+(#2n3DXkz1jby^J$idfJzAdMs;}isn+XL+QvHq2s*7`5GLCVN zrOU#+9{Itve6LZ(55eGXd*(cqpOr=gj4yhfovpxP7&JG^^Q)2#h^G0s=Rw^zS4QxZYib+Q!?2*S zma5}5a90M>q?%{N^vr2SSG=}1cnCBXq|>ARXkuY#BsILY?7V|w=B9SvcP@R1i1+05 z*gszRG|n93KDam0n}e+79-#|$aI^npifsp0UJ!m>Ze!$WEujPI91NBDS#l8bJRd4X zFF554=H&ctxuf@V>5W4F*!|mct=M_iA@B;{T$4oK3H+IB#52AmWqe|4d|>k469RC< zkqT`3UWTUIQ+dK)(>yFPqbW4k#+k7OjLfd}(l8%@G0-Zi3M`$bpyWE}BI7iXA=B<| z{)B{>I7$~G$3gUflb>{9o^&X2&WwMEo+QKA{2m7*Z)GvKUZ(rumJ!dzGN*#Fm_ou*3Cb|YuV1X82m8gbM!WxomH^7KHFVpI@z_WVfMgf2 zKu>k>sZl)BUPB)NtL=2pr2rN@Lo5<9`Q3o-QwP$mQAXEIJ5Qn1?kL;0JSoKK z&6=dh#qZz)Ug&S!e>uB&vNhw68&ywt0|X4Qx+NZBAUB$x!E-L1jErj{kY9wcsrEu{SZUs2z}z2@f*gNBm77rtPFYL6wuOjOnRFA?P6uLy~Y*FANFQtfYJyHaFi;t{C5)Ms;jXCeW6N|2_-YT9lOR=`v)*)OiXOuq_#>|~>-%=9?bp{o<0Cv)sT;G-13)5fkhb~8eb=_;~*LR;^d3FE@du_XE< zOXSIdUu}no@&c7-Jy(A#h{7m5$SYB9?q8{;Q=FB>r>e=Lz%c)N5wwo@PbGK+%KpF-JUdc>I6$}#H@2>VO}TBj)Ams8bhXp zOc{P)T@Aow<%R!CQ#9gOsqthWI2Kn}?75bq2hn3>Xu1`wD~&ek)^6fn`*zo>F>z!j zhcwhF-);>m^1`;cA@q}y3}DShTxhN+qPWOvjv>|G-_B^9gFDLisE@oVXRb_7Po>oA z;;Qxz`Vg@oD}C2y+!D=)VsUvW5D|m&mt4E8-}k<;?bnM3Z{CcSs_4q@0mP5XE&=ti zqO!+h1vby_lks2|RF_6wYwJFQjuaoxHH8IFJnC(2(>0M#j>LPY-pgr)$J>F@QC7&{ zeGXW`(1Q7B%cP>&UJwRUAH-y^x#8PmptB%L(xEu23vc6Ro0%-zQOlZC@^SAdIrjBd zD>Kz{rpVbJU4`fn&pK>}meeiPa$F@wje7ysC-F1k7_W-1RHtE2t6|)zy3WGfR9l+T zcneBKN5(Kg4cYx)C+=5^`cx64W*}PcZ+=HO5BGb=d>VBzPkI9#~%|-X{dHoS2bm>n@6RNQp2&cjWGqh_C?g`6F z{b0^UjDVzHZtbYa-3JozR!RXAHi zz5^CnA?Npe{ZZEbmWrR-28rVB!=T)6^mj;9kc zgxeGe)5(3fUF~Y{b_f$kaZ|b&3WZ@ihdtGi7c*}pFdjBK6Y;*$)R~W->Fyetjr8f9 zg9KTw&N~N@+!%GRUkMsKU0S220hz|Uf4L??m;!XECtHrs!w%qZr zofa-@-f|coXIA$}iru7lmtPt&WWrVl#=;L)9$zIfKDsFvJ2&l|aDdlR;@Z!0v>_Ie9z?pw_(DQ8EIoltVuLP5WD0tHPR?v5WlwBad z+~B4~*!>6Z#-t4)=1k~%qJ>KGAsu_{dinwQ`UvHb8^><-N{Q&gbQPXr&k&SXJZjP@ezBvxpP>LxsdN-2U(Y(t`_RG{-+bRGk4)E{bWqYo~RnzZAKkuDt~ zo){5yd_RQ3BwQ+iyM!QZoh~16l|hTvQ`yQgPqNmse#17A#-X+&S&_^mxkaI4x?)C% zX4gu9S$iKYSzF|Vg}^?*0(Ya5)tz;PL{Z6#oRBjCQJv?LBidRE1kgU?>XRvYd-W;G zdAFVANT02IGMN}7Iwr-_1Jhpd4hK4WHKLzLc^qyynUYiJ%-N^;dd?@Mtu#H}5yp6- zQD;Ok{y7Lv>>f5J?cZk}htfUFtASG^W)2`cTY>i*-{_OIq(LT$_w>rgp@xrvc;ReT z`n%UVJ^3`%c7y%Lwxm4;snlxxy z&-k~M3z7*@&3;lEZ|nS3rDfawhL&A(EO_w64&qwVVcpxK-e`F28ADdgt>21#@kGPrU_(LP zqgBfAuFN$Bmu@&xQqxg zdVS!>js5}`aA_%xT)E*T*)ZZ+RPIAln8UHA#uT$PLzJUckXyVAPB(Xj{j+8psuU$= zMe%B+kB;BZCfh%4^Hjq}zMahq*R;{^aF+_Up!S-UOB1W^pk~n7>{& zHe*V5Ft#TKG-vxc+ZXqpr!C^g;Xlma|86lFX<8?ifVzs~~{lHfz6PL3%CTi*hdgGg!3yKw}yJm3y%vwe^&*!7SflUT=Fy8G6o zNvgUew-8~8wmpNh2D)NcnQANm)v0(_`PJ5|pCbLAdt!}%lJ=$|xK8{moLM#k)O1c3 zy1?%!^mLfj78>vA^RsPpu<1xplNS?GtC^wB<$oCQ|2;tTPA;rM7n$_aYGe*Je7JW@ zw35>Acu6Gr2b!$7Z5gVs@5PQ=UK8E~(;+4@?dpE1^sVVGNfHcsDUoLQvc$4^sEnxS z$u?+mwSKI3OFEL(WJO2Z{Mxas=K7}dr>(ai-DL;Je{x@J3x-{-me+SefM4wUi2$I`k@F(#0pB95MR{O+~am|f$eOgaC*b<3% zOg!sT7~D+Ll@0$ zeIii!kLmTFrr@)fV!C2QHJ#CzI`Sg7*^M6ZQ_#2%UVOz3?$R*qO@cf)fttuzJwai5 z)`^f=4kPy>Br5s*LRsMwCNTlP=YT?Hr+jg%SiBIoBiHZa6P^U&3z6DVZ31$^IE}R} z3^ZkeIW1|!=$YTE%PJCapY{iSMDeJYUD8^$yY87^_bhA4Vcat zL6X@T+~V+#zJ6B=V2xdL@GX@B_FvKp|K3Ued<-RH-_5NyOn2i=gP9lE+El-R^aS1X zd4@-E)gYO#R+t&A+PoQG5Yre)DXHHVUeBEzI%aDCRaQ>4O_c8D!{Swul6k!+bgc-q-y_<`_l?sh2zad&KI9fKB z;?%ow?k!CEy-d5RKv(>K+H=OsjiB`tzxwm|oy$vE@hY95!{vf(I24D<{Pnj&oGeqg zh$5w8Hj5<=Ha0eAaBb9(pa``3Q>7_J>VK>XD!dTJj&Y%vk-1Wh!SV5+L)qbfaRLA5 zwT8O18ej4*|HUr+pN>Eus9f-m@Fet`Y{vYK#|xK{{~dn**MDp$P#rFnScu&~2fG0Z9@~QXG zS7I&TxOp2)AvIB@hGq$Db8L*`?rvY`Y)bZEHk+RL5Y1d?uCI=1mwZM^FO;t=n%t(3 z_(|gy_j*gD&*`SY*>it;)I0eI2~(#ZtSr|%$F?~MMEO|Pi{)V|RJ_;f?bV7SM@|OV zXVpV4u0MWjI!xP^B=|F5DQu{y&KTSxdmI)@$7hdAfo2skcw2R2k$w)Pv&@1g@H*>r z^+;~B6e`v^^=&NT?oK>e55oo}g+5GXwqJ+_JefCSdbTS(hi3@5Tr5j=&9{emnF&ed z5WUz)BcU)tp2{)=f1j^;-DR{im-+)+coUyfwH9<7vB+ve2%gfI%$8h?{n+hw-x^;_0)>1J>^A%p<_#GQIOcppR7 z9x*&p-JsqKZ2O){+%XgtrZ>1iaSQ5qy)hd5)r3Zf?Y$S^=2RyKMk5k^2^6=rI<1GE zT8*`hYu&#I5cM!TGACO#ww%@sTI%OzUg)QZHnr^Ewhgg9Q_-b3TB3G_@oH|7FX`M0 z0~j57bNKNyP{B`Y!zF{)1~Wf}msDEC8)e}>5|q&9h(aM-XPY%#QP!ZA@$B_&l6Lv+ z%+BL}_HqP$5Kl_mDnE<49^Z4E_6Cf+f!^lW$iX#b;6k-aVu|tfaeuCjmXs>b&kT=D zx`jr%9E6%yhR&{FlXjJ)8P<&ZD92y(e&f~E%5T|UKd~(GZLr%D`vKuDe!MVUrxW|T zOmGC&G+qC)mAZ&b5saVvc6JVNOO+HP>CC2dx>%sY-iAqw(Cv&ihQ{u)WiweB_?H7s zGD!T#IaBcHrNzfTjX0aU-%YZX)A&+uatdbHa_dZy5;(XY;1}aUHB|6o4i7nUZZp_$ z&=Q3@k98i=@cafyP<*wj^)=LB8LK&4$yfEV&sH-L8o@_(ffufOh=87bk~aTxvj&0< zQtL0g9GF`{3&S39f(3dxP4k+=eFi)hI(%O`f-bI!U&wC+(tbVYp1S^dHSr?6zd@?) z=m0_P!vf6klO9v*rznJwm*Z6rO+2ILmkSkJc5hqZt93?wpPxvy!2#dGzL-TfZ4ap^ za!YeFPEQS0)s)CKPdb*GGn&=fbKaQLCeh~oQqceN<&dwN7YT|(5wE)=o}Rtm;7zKd zPv2c#@L-aglmW{c$5`f*dF?!Qo2>89x|}B9mbC0hxWipjf$9x&OTzr>y|lLd4am+- zzM{jw{ADC*TwOm*j+1b_LTmO2crfKP#$e-bwTUyJjGMQxahq z7bT?OWqyst?(B^$rZn5tE)$IacU=7VOY(1QyTK1dQfJQGnBplVnkiqt?F5uJL0$6r zp>xxq*$XWC%yMdhai}Z=Ef7T9D$Pw znA11u-z5!|je}SW9_PWrSTO36Q<=@JIh$&KxG&HoAqIJn;Hr!t`nYsfv1D+rXw$iWa+0b(Hw?DKl{aKI$^l$zUb`ymA-4Tiy@Wr*i51kzsi@mf!EhFK8DW|?R%h8WRFV+R0cM(i&8aZlNFAg!d2)%kwf%0fKQmD)a1p;mi5goS*y`Dn5Hrs^>c zb5ICOv(_ijff@7#cZ3Uil->f|BPjcEFa3D* zLk~OHQ8VpT5WXfEq1gS{IQ@G84K1;h|evgTz!A<8|gpJ465bdrEAm-|&yI`j@`g`MPRTYWMszcRmT+PTz5Hm3cp zC8;9)IGy($Q<{j-%XDPTmpBizc10&yR_Nz@$81pV~wIB7PIYDf_BUL%_&n%L}kH_$P>52lDMl{1EfYg z+vh7B#Kqmt|cda9rP(l$3spz zCWO(0Y+0Zy}t`V3NF=eaU=a+lM9nMK=lcK}O!*fZxvJRb*q`zR7+%KtB3@G`3QC}olOycV&=JHd zCHQRk?DW)BB3QVseD3hcV*u$E8(fDI)9{)vXi8d(y;trQDIi~EnT{J2a7@6$t|ft5 zwT)f$!cbOnCM<%qRe@i*(M4Oc(nTLocq4u61fCirRshz5+QOebxLf7)6mD;v+aT-G znwA;8WXyTC(#J@E)R{_b6)*Qk%$*evv`3Rblti z1{|mjy0e@K)RBCGb0F(T`gJ1db*H5;wr3^=?fCsoD4W{zv97*r5Y`)e_B&?Atl^|& znZpyUkJF3e7vf>o4`TjUw_p@DTMI+0aOyf~9uG^@uGLDusDd+H(28mCf#KuN3(vxl zWUjlO`zOr<-PqI`f6le#`@@{5F{_`i??u^{#8^E=c$RgZ*<bM(AX`eGtDOpidlh(xwJ9i)v`#4B4%p16#*Fd#|t z8>Obrscpml>dA%p%dReViQ`v8)oTXFT7GKw_OHeTc?osu*{@z%zl*C6bm&@}k1G3i zckceWXUsW%VOru2t8wOgf}%4TgC!Fkeq%#}Zc;$f4(FW-L;AkZTdd{4o!fS_hMjKd*CTMydr)vK^sYBb5pdyeV|whgT4Hv4LrDUjJ(6Xk&eug-x4B zV?v>N74r4}}09&Q7!AY5}}eftx2fsSVhfkMj;O`9j@&v<6py!9oC zgSkg#QhDZoU=Mib*B3&F*B?N`fe|`woo%efvgC7wI-}I^Jp~yqMsD{qRf)QcWJzh}ON$Diq zaYkpBt`)?iSZ8A^L8Lyz{)c+&k58uxeV|L}CW;t8+A5w|&+L7>JlL;l81Y7?3r!#W zb{?!nR%|7gq60@Q>3~~5+W7Qoc(PU(3;GHmy8O%$x(2A$c(C0}hG9q&Wn6eR-F zkYtKdrEG3p@yA1`2hp}~EkW%+eh9~`Y0Xj|JK_S}KH^Kff-R1s7^87pa4X>p+rX8x zo)*Z>onvn?_S?roJH;5Zu{y;5@+MxSX(i5|TSrG(>o*iCgV?4~rKOoWI(C*URncCds@y$B z2g5jRZog83Cn6nW&RyZSZ!AJ%k%J5V8Z%xA89lyt#+NAb2=9o-_)fMm;Dh5;hTMpa z=$-F0b#c09>QD>wMxn}sv0oj7CJT4l#B$n0KSn)t8f2vX1@&PKFSPf5<^E(DM~W`( zxgK%TQWg1B=#>t;-{>&K+m$;*6WN)oqXF48k?%8aVvqNjpmcU(hkN=bLlaWdgkCCd zPiI>vot>ic<=x`a%t<8VhrRlEL8GY^+CFdcXyX)tOl)ose09wM(*@O3A9m{L`6^IT zrQc#xLQ_>wpEuw%PzOf^zsn<9WAl>4(I37MUBf~Rr14)y^cS`K@EF>$L=(8+N!dWb z{LZOf>T2Ekb;|2N?oK^V-Qjmo?l=m%(@t(SVja|mM~UfiE6mK&fvkp%4g+=|eR)e0 z@zI3Wx#^jm138Iv(Dl28?y<5Jo7{SdKs4TPt^K3xCkjEuq^ zD7^>O*W49EB#V|k0U~!dPu_VbNuF!PMBnoA?g{9`|7EIay8O#jlZm$wEWB7gCr%K) zE;hS9M0QhVEc1F3I>o$r|4ce$^&AW>!^gF$9>n}e@J+KOqJ`1L8C^~GTJdap@?(@Q z74G|N-7O7LQ?{R}=P{ zWo|>}6j7-?wQCp3_CZ9I)62EowuFvCWf3lCb8aO zr=42sKVJ%8vcpyfv2?)EAQ^p52~6x8|A1garWH> zl5Z6n@p68`LBJ1#{}TG1FY|ZEm$;RP!DGhCN4}{_U(g0u{x)&n}?0?Ob*g^+MDh6BvQaRGJ5ZDtxp{^&9bCiem3+9W)2DG0Sk_;y# zi}$;$1ZuU#aKAAo2#H$KUC!LcHmc!$T^vmYl0c-cFQ2&@(e&)mN?h$=cndHo@7&Ih z$D-R;2O^zx=$;A?7TSU#W$JZp`Jn4oLQ+9>1mV_dx>$e%;2tvQ*O(#i^j*ZQ2d@y9= z{a>b5(jhdZov1;gMcVmnj3=G7Qkm$pj=a7iWRC;9Qz**66fU{F%S8Jq=o&5Xh&ot$ zb@g9^3_ZMif3~~8$9oaU8J^RE+P$xK`083ClG;ud$zA|w%Ya6kfv4-E$}lgh6ESly zFI#+BN>&2*2N^?WbF_P_7WqLn>Ec)2p(+QqyQ7eVFK0gCEoY8=URNJ<=-y$kR0#}T z?U_PxOx}Rkf0FsA*y-hny9%VxwW92PFLk#$@l~|^I)XZl|8BN<{B>d54$Xv%kK^_O z>K(sO6@KH6x;IuBxXGmtXN(>czo z7#hJew=aLT!TVYf)oo0$qLsdGK3x$!fOi&d@{iaPu^fEwvPyMf7Ho{yp?7h20R8N2 zD{JHq&H0tta4JXO6wU`=f{HSE&KtTF^nwS}lA^4VW5aZq24M%hnBllTKj?35drF}( zvu2#@8#ctmw^Yu$#rU4S6Hkt(kvIMBCrn;%f5#$ve{s_jZ0=hjoP(K9(H`BTK()U9A+1> zGJy}WtG8`%A@E&l@+qdkc9P#NBWZmPitp7cCqcfnfn71hC6%i-XkPHh-K@8Ove@JC zY|QMFQd>huL;^1Ix(9_H^!l6R2O9Oz$jDxO;DM|n*jiF_!^SmNo&_;LRV(H5 zY&?6FPM^|tkhRh7ms3|Z4#bQ=dkiZTVVk~M3@;0D3zX9hm6dd(1JoBnz`>8D-fI=D zL)lNH`bd>=`I|SlVI@mSm+hy>*o`upvrP_1nJ}Ku_a+}ceyr3Xv!<$4!zCC=*Wp&( z5fdxpXq$K71*)Hpl@gfqt7y#z-+F$6R2(g%D(CLy`1yU zL6_OoMAZ%FP5xQ$`uQZ=|02zT7OL5!Kri;D?)%+rbOV09eK z#~F8R#Z_Ur+iGtKnyAIWZe;mqztP995X5EEML2*iuFez)00{XbU&L0gP5tCbPK8!CrMM>u>%X-9?6D_VR5N2p6-;_HoJ`}~ z-xJf2Kj%=a@mAEtippk^rz=GrDfQ=9j}h)VELM+c`kU3u&#!Ib4CKt{s1h+?&wL8J zUY3Hr0B}wnt;5#&%$nl~P_u^~8M5_kUE;u5olhB_)SAJ_Z(&$azBh_Z*}!l6d@_U6 zQwGH< zpRFkWUWyTdMHyhCgXlk5fx6NgNQ4f-Ot!*32 zz1VX!ns1P-k&G(#a3+zMvntZd-H5{RD!K7P&NpWge7>8;D4*5kZz`x2<;jxYXT;`6 z!K)vcpxaw*?LlvnlwW0j``AJorT>}@X0<{oS{hrro)2g1Y=7uE8p-HH#e**;1Kc%I zKQ$U+ox5FyG#G4K zm!#^Lc*Z_;o#8fe5Rwxb(}qa|te+ieJ)w~GUIK7|J>4>dx|C*1Wd3)=J;3H}BB34a z*K(--EbiHaLQJHOz8hjUPaGIV8f?|PTz zsa{}BxcwlxW@7LznWiSJ<@w)^@g7zS~_CEAcB}D+$|aSw1*eX%07jm@!2m0Q3zP+rA&u7+L^*8+j-~`t8t$v@f;| zYZbl4orZh|(j!FeR z`V~t}O$}X*01CZyvzxBmxQINiiW?yEcyjoiZ570}+R3*bhwUT(mBT1S>i5g6 zL8HTaipN|(Q~M%brG_MW(W_YrNZE93f!8Kr=Ec8eywEICjQ@i4?IG>I@mEZR*9C~E z=Odmx2!`s=d1^*5(CNrH8o3#D=WGHG9B#y=bsmMD2-gAGRHh7G2{;?R8F4jbvM6B4m7l~Px%H_rJN@J`r=Xqjz$b;+ z(R?SCFHh`UZa~?X8b^Q5c=a4D)i{`qwMP9d1b&RFdi^nuC z`9WH!&S4@AD**6RrxYV52h~#LM;t30!GhdGx$SuJaxVwhs7D?Bi^&cnPZs)X#N%m= zApcey5*Q2g+jMFHqpRkT`w}|lNTLywQe>!4W~ljoat^lYi|3%x`YdCtcc$QGrP1IvekaI{kcAU1#RExGZzFBpKH$}w?z2C{kxRmXOnIVn&j7+}xLK3?p0`O9%Lu4V>NXDw# z+l{VNkkcz9y(D)wH=Ig*HnT)*eqnJ( zxQcQLjN^B}qL>*o$Jbj<8Az#_^yj?mn1%`c$P8w}iILX2sP7TrkD#5-KR>K;gNdV1 zZXKGoYulFapbH)_TJJ$3UTt!qW@oRaJc7=Wmo)~+04d@?<~jSlvJkG+P(5RV<%Iu3 z!CRZwh)6FoKj}kQ_gfhrV4~I5A%8}9c{(B~w~gs()K#QZiN_6ZWUYHJNf)}~DDvL4 zO6GR7BEP2QD>UMuvGx2^S;sc)vBN}K>#Q9@5h@XztT*9~97cFLaui*snU9l|#hsWo zlO-PXDKi#KHOOs|*$!5fCTC2J%p`J4yHHH+;XSuK*Psi3?6|SaPQe`Y0pXIbuj1v- z+*M=KhSPHC&r+h1r9>*J&^7TmCJ&o=q;y;QVg`K)1-9PRjP>OHpy_fa9vEd0fGEwse zf>R6Eiw=L&(9P#4ZIAxSsb2)E%f2oxoq|wZ*a4&dmDG{IGXNpa4iPyi;b$=dtCix= zfzdh}{Uu98++{jDDO<9R0Z&f{d7Zg`L}q&GBM+$LejlI#j%ch-uD|fppS5-(46g!s z%vq?#0}OO75hzTGn;&hBp;ihAXY%h#rfx)=L=Oxqnl6WVH? zBZzN6>8E;Z^TFNX8Ud8CXz6I5$pzT*^~s*nQA&OwKeCPd#Pa;BX}O1?ymvb(gtAY; zgnTK2rFZ8_4~6|zdS*f8d1(|^sKo@|Qx-6P8eH7;v!1yxnUz)nnl2~8#&N3b5ah?z zqn-f75tFn($ z0a<0^o*wVG?LD9tFXnK^=nFpaOGBuDi(^*wW@$0craf7PtiV`Pf@rG7x$WrNvt_g* zmcoI1?^bwn4vQ^#;n_*^Tg&^{Diqj6pj8izP>dfUYUhaY+V1F2rB@h?=V)lz`pfRO zi?Tcq=xz#CHYfHUEh9TFL@wLa7WxAC(yqYCqC7;UI^JByWV=h+>;am=p1NK5K7mI6 z%V;;EmraFD#kx2aFO;XllopaKX9A##RhMJmv_ilF7hUzLr9S>C)jS4%F=C(&PdX{v z{qaW<<8QPS;~FA!YL>fyWMQnn{~Uz$Lj$E;RuADRW2qFD`on#Af7wR0{6$gbdKYem zU<`6uvD9hT3bInTJB%*KpF?kJuVs7X({+aq;92Z2M2vIGMF_g^`~Cv=64yn1Tx^Zt}B?Z0a1)~v+OHFAgVao z^t*q^x_bpmq=-H1Re;Kc@y3vNHaX~HNcg&;#G-fcxj;7b#jU35Bivxpl#*>Op zSe%#e3T5OYaa-BvW%nuEP2mP=m#(1XH*(KjCe@}#X8lzgKE1zJnPgtK!c4D!zEU`2oI@;X#;>frMa*F&36l-P>!(1OP#-WpN zaab$;aj!<HOGHrN35uCy0>_5^#eT}DY)sGVd=OrNnd>k z{r#Jxc-Tr`c#(N0^?Upq>lT!TAy@)wRDhe;fOq}X|6%N{qvG10ZQ&3S2*KSUcnD78 z8iEB+aCd0jT|=+{0fIY(U?Dh-yEpFc(pZqjyYbgK_nz^7=iK|f@zoy;#%|csyVhE@ zYR;M!;lCtvQMu&gIE$qP{Ql#c3#g?@`F8j8atZX>>KdnBEMMV~VS5aVvC#8{gap6qG=C_ml!g|>Yr~IPAwQTOR@=XsrB6!a03_hRbw)Zgi;YI^(G@QkG7jV zy2;pADS`lwpk3%Sz|1I$3BiNlMPFSH7GdekTFN5PvDh|SiMSgQ*nwhMOo}t&nUra~ zWk{8lUf&5)7&)~OAomg=_^7Tbbu+n}mT(NthadUfyrb{kFNRPHhTLmYCNaxfjP}3o zE59(FpJv+g_xUzCplhZFi0gmico*zoIS!v2IJ(2AP<|mkK_taaDV^UAh?Jl!tP~Io zU#>{z1{!kf6OY#NI)B)iUFZpcK^}Isws3_LQc$w^BIB)9)%PP65{#0`NTLmxIZD&` zw)KxLh&31+OQ@MF_t58?#5)0!$i)Xt3~&EJU^b&4-Lt+>@Wn~X`f%STeS9{@ZV}f`u(8ja4|ko{fvRB!DE+QbqQ+g^ zb$-w2NSme?i;ASZ#+#pI7X3ph;!!gpR5;Rhie(er@^h^hRRDfsqV(io%NOY+B09WDC#I$k^b0|cEue~cR2yH&ZJ@s{)VMGu30#N*HO`O zBF-v1lRFiJN#^-3BKji9dHC9jc%(aT&#i9NGA?Y>YMqiy+3DyL9qMcFGf#0^6fDNl zXe3#e_dbCru6(I(80bAEnjWD3u-2YPNAg%il|grKotWk@HTK5*w%_7@M118gDEq53jcR3cBrmkeovt zdl*-gujY3dF5x)xEB!<`R_&r zidkn*aK&c$yS^3%UBqC4tHpztH)(%9?RE-VgVz{OtSVkLJ)6vH=Kn)vFv~*|xMh8h zh*{6b8?>9b-6nbZ)(Ewya2?iHMAOBNlw7LqDMZ-}_9>UnJpB?L+78AF>xpwqe;VZXyeuV2PUOC)x=w)^9Q(N|Omb$frnEG}e&w33{C@Ey$ z-6@&|mm&}}0PAGh1@FN8K?c*td$RMQ5o54>ET_A=z^tpmF5l5xZEe)~tbsNF)e9VJ zD_)(km~iSokZ-03g`Z+p%=Mv&7M3=8AaR@n%j{p7HQajJ76Q>-{+)VGsaNSI_(b^a? zcfi#HqWN+Nkt-?4+0@?6LB%eji-5zP&!1rS`ygM=G~C_`Q{mOk43E!ch-e9s+6Ub+ z`GOqK`MqFbc$-pA3U>t6RLW%j+&t)Un0kXyFlwO|D)f#KI0;>tJE?XoY*kGqC_^2{btl@IG0Oy?`OQVTKI0((O7hGUFV2gkz2i>*a~nL`x{LG88*yfZ6D`x5RRWL!zG$FL?)|ev zA8wDGlgB>qgtJ9+BKz8eW*5c^|Cu1U)1`scv_4^EMI!M>&)GwO^Hr4TaB63W4(4!+ zSGKPU8irnHA%qaa3`s>L7HpXg3Ao~ms`EYX=;EnkC67KmHGfERJC0HU_%e^Rw~(2w zD>7cTCaa-n_aJuaig+NK!UG***~j?jiUFtX^B3pJ?7T@H@jD~N1zS!;4#PCJQGrvt zCwLDJW6$hvRnyLYt47}R#-&=&ne~G7Gx&mwi1R88L(oUgvi8A!VTxPORXD)>kTW9^xZY+ z7H^-Qc9%PHXs6NAwuf#AXD&EO$DPeP?Ev?%>i;{%+h5_V6iyxhyKT8#0gE*}hObp2 zI<}{TC6`qc^ZRq4gvTZ$wR=0+AZ`HA+m^L9uF(J~c-Z|I@ZNBGd06``C+t?ldq|f8fEw7Oc-C22^V!cvMRDo1@b!yBw$;zjr07~(+S~Me_U(fubi{KIb*M!B%G)C4+NEh zlCV4jAZhlIk^I9D_}d_!?tP%)|N6K8`_skb-??eT;HLd|A{#Nlr3?Si;r?N9QHn^2 zO4olHaoWG&B*gXEx&M_z{}Vd@uMh$SdDBzG{0!dkHAo37|NP(YIK=#NAfgPP@WuR#&H_ioH;x|t z5Xh!MsK7+2+H1BmwFQ%1k+Zq(PUM_*tV9PDhW3lV8}oJpeDTjhZ%6AWD4Jl=n$jL&!(iKRT@>uBbw0^BBf6>O3zgv(xRc3s$+x{qbaoN!ZZtUTrd%j zy}NYAe$@hwK7{9PJmt~%?}~UawEIB@3zHB*Mk0>+*MmUtRFO0Y37}(}8TZZeS~mic z4$Wy_Tj=$nKS^=TR@`p`D?#I2s?{=Q?$ zUE32jQ9&A6)OB`z8s26&dNxx`pe2YvZeK_~;049nqIs_cbPa0g)Z8nub{>hIl5?Td z_d>t5v~QXdQi|?*f$w`RT9Bd|b102k#scR86u79|TJ8gGR;S5izq>s$ibWZ9#%%Gs zj-^r^>YdO=_`X^H{RJP&--u~)@r30Ti?>k87MnHtbw-4)q6d-Gj&ToO1Lcr4tXL zmW;On+r9Jcv}eAm|H7*Do8H8JocPIAE>mWqu|3`s>rn5udlII8Rlsg6_{sLI=r%#+ zZLQuF@uI1u;Z7tbnwQYh*%ac(Gzf8)^B!jCuYupo?6ou<)p__ zF@4ar8h%Ho{7I`=;xRnL;Q5o=h2(5-j)ScVVE6tqurJU1u)j(TjevSlr8tj?2wBON z)+Nn)dYtf|P@*7tQD3&UfQ+Z2C%0hv(O9$B=AhNAA;ypl7unCGFpTy&N12 z@K#exMs8^0@2hHg79ckE(qM)}0iWqjOj2guGMWOuvW}sS&sX`PzJ@Hsq!<}#mPq+n zko}Wb3&LISuWBF2BxF5kP07GXv>|yVbX}LF*bSlgzr#>9Xr*-WNw&fE!}IWm)&nia zA2o0iujUOejZ0r}iNBQ@L1wD*Hio#?2e+E1MSbnhLIgHe7f}@!xFTazZ3P9I`1-0? zbMQcSiL?}ySxy=%WeC`~O6wmph)aZXSF8PN1oljS751`C2NZEPm{6=FgbBRZ6QAlM z$lCRIBaqpg)IHLSGd+u`fB&Vym)3J-9;Ik&TYWu5EpKY@ z*PB<+=KBUiYGu?Y{FpMjhJy#L<>jW&EzmP3D2XyEx-opbu>?D*8Pw0g3UNw9Wz+R} zs;phUFNMCP?_T=pjgymsRD&w0~_C zf~CUr=3ET85r*(KA?GhNJ<92kb4wi6Hgg91FBA9@;jfVtcl>lQ}rGPm$s$eW7;v-*|LjuhsY%?^FuXz zufQt`?w0}EScGv3gBoRe^$(=$yE%b{)zx^nGe}H$8NAM&%h&F~8p=`-`7Oj604m%* zUiRs%bDhzMwt^D(U^ln58e9BFuat{w4|T&C(-cy-%S=URzZ;*{Qyy2hR78zNa&puw(3bwt4&_ z?9qzIa{j#^Tj8u{a}ad=IX$b$W%KL{GH_R7CSVl#FZj0XUbOZ94|@9+C zQn3AR#^jNJAERf|>=t2;1PNlx%6;k9Qkh3AGb0(Ru^g`+qLkmyuNR66;0?;X(jT+_Wwg#i$Fky4rt~*Ka^No#BpHh9^OBq zEi`g+WI_&njS#U#ImKV$K-e}!W#|iTjm@^7$=N+)C_(90$dq>!b4k;S-l)6W)cILO zdhU*l@0HohkAt!Vp1#QqS}2b@uP|@ZBc!C2%twOI^`@r0-J3Jo%Hq8r+Q2&5wuZ~m z_nNa#R6ZLferiUq3c(|H_5ot8)PbKLh?&1b{4#GJ)&7PqkKREwWzkK{fB@#wZW4oG2_JkOJN8C~T5duM?!ZIQ;i~78kE5@aIh-yLTl31r3)A8J zPVFfppxqiy7qlTDEBVD=1Oxi0f+R{qfKg*QpsZYqeN#Dbh-_rBeS<}xO z6-a%f_#fH=LLM!+OCq~T0Tw(aN@H=y-v-X=H<^DM+TJ2mcDP&wW=ARj;sXI2un2v! z--58YfsxoCv#_b+(*fx)6@o1=0yE&u{JGkl=J_ff4O@1G^1V;#i}F~Z?RnnCu&JVi zAn($f`~A6hOfgHJl%>Q@Py_?I)106qKo8|8{T~=>oh<@p0<)PIXJbD14(q;i01@w~ zXu-fExr-d?trE>VZ)Z1wV~2o?z&p~LMuxNHZ-^K1o5&BEHMrI^gNtKep{l^tsLWd_{y*-)La$oKe*rbeN}t!wWxgDI6EWj5xvafSVI0pOpvGC z2<+8@Hd}$=CRF{{>&)P;!d6Qq0K5G7o&qWBvz53N63vRxCrRO!W51jwqo3?sSE$)} zNizq-e&x9}QT{oGV+DAkqDh7@rS;#8MXPbHzj>mBvv0p;^pzIQfr>ADO?dg;(7TYZ zMWFg4lv}GiJG&mv;FT(#3f|9Q!LS$aau$0(1ulHa=8Up!DUscBvJy=EhBmVl6>#M* z zEKe73CD5fill+G=!Z+9C4PnC_pB2tU-q{F%` zG0Vt&)ot>O^^+N&>MoyIHE4)C6Hya`yWp4f4vKUeWxxGx;TSt3yl5?gzRw4k6-aT- z@2(zISe&5TnZ?d5UyRb;-i4#5iqTS(IRAb@c5Dy^xbIFsn=Ab}fIQo8-NVFce4~f> zGf)J?oh4OFpwcH$265h~=NO`NIKAd;>C9jKU67Wdlof0azkbvS(+PKxBKPY*P8f3Y(vp>A=xxaLatc?p?qhza?j<5@w6+K%0HdUH-i*=tLFfadYPm(XcN zVYyj5#-BatYdVb3Y_~(J?u1!zH{72~rNp(1lE0%xD@Gg5v&LCz7D&j*GP++qg1jIr z5_2(A&CD1kjz}YDsO|m{WP_MXAun_Ps(CV|XaVDpO{LvQe=H4Pj8Ambb6WY1=EW78ibIf*ixYB!U(Ji;s65hw(s`!3a()~`JNj%^j9hsV$S78&GBqgai32$UJ~V_TQebyAA!zfMyScwrldWj|d)6S~Y*BqI^@K~=JBNZy2iZ9}#chBxCV4@XA&8RCw z({czUa&oDQqhK!vj?B|o?7q1Xdf@iC$d#%{-PzP{)x_>}>`l3PEtrVTBI+a&xsLC_ zLBtzmk&`x|U>(>6z0aQcq#w0$*LP)}Tnzoi4ZYg*Do;${!|M3j!=v+6A=n7Y@X8nQ z48+pI^8#0f#L}^gDhmA2fxNNkzMFk*Q=d!olM+%lGTzWQnwi_aRtr>pJZiHKZYsw{ zs`9a&k2$Tku@zMhj{*<<0s4R}fLS;<9bdcUIg1T%dQL*<*x5!DyeAx6{>zWohj~Bq zspSV^@uW{bl12V7sZumc7%}JDOr6KA<7&Q=TodQR0Ep#$xoBB)Xs$YMD~H{>kB+*R zyBf?L__TIg|3>o`-T_!I>-oWkq{l!VJ=vL&d5J|(OE2R_Kx*#!rrdIt{P3Wl2$d-d z2%|}l)E+@PVA$y-;k$!Kz)tbKT6~$_rNfPxhiTb*m9G}Y^qKH)wLlT zSR)wVXGOFL?i`WR=MsO$*^)8Ulq;dDuS%NqLdt=C=Tto&223)@E$pBd6+@dU62xr5 zIJwBPrel#!P#fsy(1qOHFzPUp@Ub`)&`V|PbI}tlD;*R9@o$!JAl1<$7F1gVt}wcv z8kA@8X7j$h{T6V|vLxu4$mHig2oSXNpkw&b9z7$u2( zmVv{XloYJQsRp;uqgcj$ABye(;uNgi$JJDZGofMuRlT_5^1@x2jfyuJ46+C0X}SZR z_0t1o3Bbicy*5u6y-95s+HJ8JfV|9FwUinTD&}j2jsp+8aThAVI+wE<5OoK+q%+}p z*gZCcI6ZTbE7s@?`Lw*4Ec1-`L#l!$R)dev`pm1;SGKe)ZS#@cJaSRG9iEm?$}$#C ztZ~1>IyG1g2vu~?Yz(h`0bNEB8|DMirLPi#^~~OjI>c8AM_g>7zJLFoL6|AHW<|E6 z8(ASYD~S%u1fL+KM=(nAd*afMB;3ci*+pv_3N>eTtIO}UY$0Dl(S&A*(3&>+fi3eG$9 z9^Tud`sjJ*|I(GqHP`1ohffIH#JT5W6||+JyZKHhw5gM)ma)oXM7bJWk)mqoi`b#W zk&uSd^RpNxTyh|omDBl9iDBrZ`QMyre zapb~%`1VyqSU2NTD`$LK4nv&3y&=*y2w)e2LGi9T3j}|~etx$#YFw+^$up*;Vx7%3 z&>K0^S_6$_Bo&^G(=5i6XhV^?+zC+!fdo4x5Z5TuxH#$+&N$4^Uy#%14=-n;ORJxa zIRijuO-*CdM<%~=u3obzC6*15YA{EpMb*Xo+hGrQVC#m5Z#eRvpzX`9-TGCMp*R$1 z#1F_>Na`~xNv4|tN_p&7o&spXYSdEXa8EJTPeX;}$4HkFeP*Wq+Ge_wkheL;i8-3S(cAi}W*+l>! zMq6Jxc-HK$GJ710!hXuh6bL(Yb@t8$OFJ-^Q_b5sK#n}$h8o-PF5y)%CVg>5PwiRV zP}&}(a(V-5xVzX+voxGo?hbt`!PvO7a5N&4B4%g18XIR8HeP$ZurEka+M=&x4J6Oy zVBN_2hTL+d&CGNa13uYFP)Xpb6wlicl?kB>3k4W%i;MfNn?PJnrVD^iD@58@W(PG+ ztsgMr(8e+Y8(t;Nzd_Nb+350Eb27}Cj4mxOORb^bA7r{4gTyDL>0C4srlu}58lQJ0 zDZcKf7Y6-A<&QcFe7q$P(gwKWT#&z8+f6wS@qJ^&kvdu`y)?>Zs{PB9xu5?0u!ior zHzfNRHa1H>a-p21!~k&XO1vmU_=k~&)5Y}~_Nb0x|FQ$mc^%%p!obpjVtjjQcM3L} z@i9|5hL1US75K59OIpf-Sx1gFYd)&r$+1`nM)q(|?i7h{1Jl@%So4R8=S`soJKQJK zp67$P=OyGC!TSi+&F{h~2?95FTLVYPIt`*I0=_iObJuNk0iDAGpa@YcF)b_cTW#Ho zqSA#W3aG`<`|;g+0#L1Ae2|jbweh?VMX1IoKf$N8r02Y%k?aJ-pEhjPHxi|FM=jbu z3f5KAlu?W7^3z{w>-}1q!@Z4?=(p$-i>pg@zVEB-GwCgoCRw502PYO6k&4=(pfzGWgT zEQ0mQ;R}TzCl^ewL z;RtTVm*roHe*_HX-m`lzFm-^Di(KB)2sC=)lfhw{$GIb8z+c97uGr=Mg z0WepgcF1SGUm|eF--&uXO%vw=?u96;vFzRLiR@X+Z#GrSf0~A=HU98A8x&aO7#^Un z&>wL<_|S$$2zANHcGhsj~dR6Q34ekLEMwwj`cBrURn&*SxH%ga{ zT=Y0ks1wm1s7QL5Hdxz6tgM`kA01IkNfg(dB~PwBNMNuCrX>(2F16A|^`*^A3aNJP z)ZHu~=Q_2$PBY6{JUyoCqtA|#{#Xk$U;0&Amn}Zix69#O^K=L8Hv~X$Qog&4nO+eL zg9?EFDxG$!PO8?_&Th+`jP5%d?Wec6Tup*#2@kPp6EP*tAs^q^X6D$_5E# z=Pntv#&#GqA!#t3)Jz*-CL%_o!fi1~AVWo^?dFWl35xx8q>qd}w>kD6s3qbMt_ZGe z*EK07SCpg+4e4QNl|d1sX??+*IyTk4`bR(Vh4Cr0C)gb{Unx{*^(KNsc*3=?>gP3u z(9L3lzh^|KtE$fFhE0<)*}FNbgddHM7{BUYc0cZF?UQ;u{HS5GBXdQq%gqI1>?byS zyI0tE)-R^~^jYL}zclHXOE%%!cjNS%+X*)Vfr=boKfwdPoXv$0{DxI6WDev*#)ie; zrmkY0J3}K9ri%-0G!a3f%@e%f@EQ#+bAp;y7n0{X^ZXo^J96X_&^o4{R}^@42ihMT zaF4}!>@~tm8|uF(dl+vTE2+Oh4qJ<^0Gm$669bBOO&O6IZn=zoIP0eAAO?kJ!x1%( z3z@v3F$SHMedmKJY1)z%1R-)BgFZwf!`)0)WR$w_N?HbunZI@y$whw}Btgiy2ILlc zir-`vf1E;@|LNrkEM?+9vrR+8WDF%qWYMB}iG6ZVPN1J+Va@$F7XX>+M|C)oj6sXu zv-o!Y7_}k=)?HOx1ww4)BIqlA>das6g_bM71R6ZS$OGwswsG74cew$o0 zjD;c1zY7B}{2)fuTtdRX?}%x$D~DfU>=il0tUFpWWC=ogoTto5enh! z*|b}qswSsN!oq2u3XrSre?H-RgG;cCK@H<3YTCQjw407)hPKJ%FlutzJ-ghK=~j#A zfwsc|%N@-DnD3)$HsbjZt?wB5msT!Fh124&QH{BtVE7%Q=WQ7<7BhP&8`k)F+=ct$ z=p~h7^MilCY`Ddl=KEAy#+9E8w{oX1H7oMvq$FQ>XkelXq&yA?af3>~S4x;?w*Pr> zpmwnpD6S$~h$npSSVP&E*nnwNxx$-TF2RS<-RF;8+xg}8GgLl9w>Y))>>WSf47zR1&2) zT+GOYVBlWcmds|SXgU}xuc0&;dhaV}f0h9qE)`(d{<=1jcJPq@g8|zXj*G>X#R>nM z^yD?E=$)#=q=38NGAXB2sjddZ02@9YRK0#jg+cE*q#dC6OJZw+%gviNvM-xl?%_)B zOo4rFv~L5-zq$CJCZzn8CSL(FCg2g7osAKf#0&gp|h2RZwAHkAYe-+5IL&K3mSGL*u9K+{1aOjrn1a56_J7%uZ(>-o?(FjC; zuZ*Jng0qRd01_f1(*vgP>wN`+3X`OA?>G4q|@qA85hciHVKS5%tp z+R_et<~spx=`%Shx%;CFtjlf}evh(q=Y8$Kg!^@2DTgn>nG6$wmc<+pbYgFZ^MwDlsg!%HIkq9}T?f__=XOctIZ=ixWF!tMeUrOeyG({4De#(Wv* zrP@eyx<`B3YZ8Zr>g)}D9(F8M0}jtZ`HCh3(>^gjO{{`qcWjcroy(=$TimO5pL)Ux zlRbk1#Y5YMuEi_`{dGEA)!56vdjy&3MJV^LOSup%VW-@;7FgM2Z$hnw`EZ<#-hldA zwLKU};xfX5pm|YpiXeP|C)a?3upir#~X z%Ql5`x-Pp|jVQsRt8?c$D57jU3dVnEKI0|+!^k&?_jYLBcymR5DRgbkLiQ!QNbLH|)$_JHOU8}b z^36g)Ao`{7Wue-4JR&K`j*DKc%}f9?{HT8I_3PIiAFus_;PUm&CdaET-3T$?>yp#5 zIY_m=b#uKD$QhxeNa1iq2z+ zXH)#S5u1%a1|d=_QQi{26LHkpt-_{Wgxw0kY@~a_p}5D$vU7;q<@2F=Ex+(cttH~r z$5LXbIA3JHzm|Zcsr7JWT7`=iWJ?5gy~@oJxJvM=cDG<*>s0qlpquFUredr1S+r#* z#8-#k@(Gk)-<_Ebw^8Q!r)J6URWJ#Ku#TJWLa%hnA0i-7)Y4*k4~HG+8d0+U*q&X* z=37k^x>I~G+x|r#GoiFpvGT_F{K>;tG_rCk64bo;Z+?wKis83Yx)L$JB>pyd+~H?#Q#oE|dx6wON4LJJ{a3Y; z8Kw`rTquDrH&ojDzhl)_Whyf{jCewlf2!{Oo@p4?X1#1WordFvJoo90qt>aN4Js+W ziM(UBQX@pn$(H+O8uH%L9e8Inow?CYjpn9h`qgz`w{QC$2imbt8+g3VUsgr*m-WO| zMrtF}MOn|*LnZQb(yy3Gc;n2NZUDE@F`Hw(WF4ROsk!RnviQZ{OTqqh75!t&WG;a> ziH$8C4K()=i(H{fvh2DYC4?%9KH15##GJ)zyN9(R9p8bnfMXrh+wAmi;XFpwqkG(j zQPPe*)i+iULWsnSuXQF*MiI5aP#Y@?wu)C2Q%gN3EN#GU@M0%7FlHlyxXKH;QQwdc%f*5a8TC`n-8_C$#KMdFif`R&d$Vl&hx;?BBzH=Ei14Gu0RUhYlHvV zZluY1T+sPZ-h*Q^+ieMKi|$UhBKtAVhI|;^5jAtsF0|$2<3h@T_c<2dNX`g0G{$87(SIlwYTx*Mp~M=ftGP0C#!QV@3n16-TMjL=;t~m2 zCqtjlaNnl+`Z>7gZr62F736$*pwt|8(*3nwr0v{}$j=TF?7wKryn^`}z69 zGyVh2PqwwbLtibnvKecBA?vMXjs5Crg~W~Cwv{yizg+4*5=udY{z&Qmr!A`w`x~7; zFdd~$1adys)1+SDC~@-Z2aV>;SfSR`STor1+5O(SKbJTIq}=pJ@V2}|KO zBy6?f+9$8mcY?(Z#Ml@vTN6cl?#*qUp2gt7u8TfN+_!0c+2;I^+B-oP^FyBMSo4h@ z&Fwd2kQd?kiNqwR^JPOaX;jiu>|*7*F_c56H%USqzlbU7dS3i^j``rMc( z8;bo3yW#LzYfAKURyLtT(c~}7(}OE}h|Z$_Y3y(V z@haS>k%+@;eOryZqAM?-`W}kUKb*Hu!B(T%oE!}j-L%QoWOHO4Wt5GjU>H(06#;UP zU6OQmD6c{u>Z>3Ea9;E`T6LyuZAq&Qc|xnnKOT3JNYPWll;*_Rr@!n7>dC#Xz0BNO z&FOI)(f$$&{g1nHkeA)sJ%31NC3`$VYFUY*a;5i|)%UP};5*5wDcMyjOK$bvobcT! zs_i)fNzTTjtlBC@w^Jw9QIbY3TQrJX##O-g$giJ-5}*=9lu z?m=g8g4W4OSkhP9w2}DYDecy=5IS=2z)f4bGp~?2Vz%weKzKFuKBA@Chl8mCXNcit zb;K>4cQpcp^Khn`%;pnEM7oA&!5@{qmSj4xAPu>w*U>n6f=gXtyfB1q;-EXEBB;-W zi^$uwEFtuTugE(b+wWy0B#>eTA9l%DR*u6OeTOj75j>BIzIwxip}K;j)qs({#-loy z|`|lvWI?APVOu4vWxWVOGLzpA+ti`muSUg z<0Bv@6XAZtYJ4dx$vzJy59+>t&6o6(v{jxJU2Ma3UUO4eQhrhS2a$4~itAF+QW+8k ze9u=?1+)FTib4X7-^Gd#?K}Iqd%}3~-VT005m(K=ga+Xx`x0W#Us>3F3w4#iRFEYT zbF*56u0R^q!)%mhJ?>4H4<7%>_o(`h@-V)4GVHJ3bSlfch{UP+WCe{OFsR?jj?$bptE+lI z>3`>)x)tl)@DcGClPR<*Ffgzs{S4hgE7NJoX=PBv_KRp9o zmuq$t^|B-UM@Sa0Z*fSb!z9#F;#N#)T??<8Hsv_CVvKz*FuN;MnVHTILjn@JwUV4` z=4+zh6=jl@fqS=}*U~s613OjQBmdx6d#FxC7#udv^s_W-K=V$IKYZ`{@YiJW_jKap zEtPyWOKk<}-#0h7cd*@SZsFA2t#BNOcc@1ZeR3PI+^j!CpC@dqGw8mux3#y$3S1kg z1SvG0_DlUp*Ja%6)TLLyVT2BP;UDnSz#o%U=?0%mDD{F=)Ed#BXy5mEr)u}cz$s{s zh$kf1M9iu1;=!IR$6TGn@7j(@Nc8NDFge!7!f^n}r?M4*R7fX~ZVBfBVtV#2Y`lJ( zFrL}26C;Z|&a&BPd!K)9OCYS56fY+^ZNb;HuWe_&_qP|#>5W;`7wKo zbYT$!0t(|k!0oX3$UEn=krKV#tl&68mSmDpUGVhoE{?BA)V!9^gah34kTFq2tc-mA zYoKCOXWkG1!+zKG-VhMaPCus9=GezENqBK-C zH{g>eEwxI6G~Jz~8O zZG%;4;k_n?P-1NewuO*tgn?jih?g)&t9VEoTV84nBYMtw=U0mPXhu(;j$(qgTVw8u z(jc+1*jGH2FO`bMI=D@E6wbRy$hkg$7YcYUQmf!COe2amR(YFxG*~~G!q5bNP zOw3ok+GdP7iW|;6k{n+;I<0W{77%(h-+-NVtwe4}wixB;F}$(oFG(`a84(&=#@_at zsh$?V6hR0uf*7k~Sc5yiM0bi+QnzVfoM9DIx>({>*~V99d~3EQ_wIB*ttFucrmd4I zu1R#}vl)IRRh54(1tfo<@idcB*|~R@GbcVYS>3E_!C#E|&7ycGiR|RZ73Tf1I5nTU zbn37=xE5u6H`Llq?3^Z#iH(sB)d?Gs%M+}Lp&G`YAQk~sPnp8m-$Wou+8ArJRWEZg z?q?8l`nzJ-;vJ=$R|_&HjJAoXyUBOr0y6jm4W;dLiSF>E@5b);_71uo*Gf%S$(}jOrL5_DFy|Q1&$N6OO$StN#A?boOPrR%ICSJl*r^8C0={OxBAtG`N zL?;hD)h#jhBW6h2%L{Sg`;4MON82+xY*JBe*^Y+T5;uo1ucGgXd(4Cm?Fzh)F_b>+ zaNw!+HvVp=fbi&R*Gc~l#Hu$UaE(#m?M25;Nm4K<1Sjz54`Wc_PIf|h~4XSBtU?c>fgrOLR#yz$tuaa+KVPE5me#|UP} z_jl)7jj(`(6-QLi@9mFwF5nO8p*-M-DdXvm1dauPZQM!aTH04F07QA_P6n}WoOwlL zE&Q0V$j0BA!UJ27rTKKEotp7fiiIr1qi=g2czEMMx4;aDxL>zW_QJ~d9>X%1^)Q?1 zak%qpAXIu>4S&fV*b&9*f}AQ_kDKvZV%#4|`?w(oDeob|u?zj;_jvk-f)W+UeC+p^ z2yBZ~?wu~ilq?o-_2F%{*FNdpX5V;;BsmHP=lTQe%5-};!}=f8hdcS!?uGw_`=EjHn)PP4+V?j8=sA|@Zux2dHaTV*iSJgb4#V!b zbi%U+D>>Co)`YciJki4TD`}3q9>P6)`9!fgV4$Ps`eSQ6rZ_crDO(?Y zBk$$i1$s(G`q+R>c-$0;&#q8mOon9FX^EG^alo_MUy_C{7``0%Kl(3$rm;H1lO)20 zCf)5rW*3;Juk5EZxvMGZOLd3SAwT$kZjiq17FJUAToO#L&Mt^M6;MRZ*=Z%2=O^*# zsXWkTmV~5Guw!SIm@*eEn$8waHFz(wCsaaYF3Z(YCqW`PN?zI016uVH^G}zirqjE2 z;K=f$h?p!+obAtQ+RtmcZdlJi7!xoMbdU84k2DGRQQ#sV*VH# zL0}r6b3=-%;vFZ2vkZ{rvPk$2t-FUS=5gB4|ahgMViB6e0f5iY+A?j7N3QTYx zL&0y~dJ=k#Dp2Jqu3fV3(z!a>QRX0ADl2XVEl2*~_30~Ad~2HWsJL_~AEK%8H{-&? zPc%V@m+mgA02C&A21Q+P@u@tN*8i8A|MpY$?y;Tp$ zf>3q#W;mW6W9xt3jcm)n(IVo1QoEo7Hy7B*BQ^^Y>7EZ)$O!2CEfVx7BkfiZcBsfBwR0VRe!Sg!m8d%YWgS`Jip+BuY~05Ezt zu5TU(;lB+ax_)l?@v)x#n{QoULRTh!Ci63g^NtbSq{!TyDSJ&6QOgCigC#130!W1V5p7i*N@>#Nhp;VR~aUTr9U+-S;DPtKPg zQqqgmB#HW6@W)=wgqNE2Yxi#=mlO3xD;Jjgg^yps_*fn`60WV3CL$0mJj1u3-up7g zmQ^Mt3&~%2i=~-L&4{PKYp6?<8bjH^f0Cf((WUgs%7@ZkR)D9-P~i9`J`br-bXBex z0}*$X?1<$C#|SArwz*0u%aQ9B$`n^VsYG5%LaHT`#T11P4_cww?OWDf;-xn?SDRT% zSDR5&Mu162768*2;}v|SHdyX2NdLh2yxKxbtKh8z5~Fq0%|w$uN4Am6XDnO*$#{#v zDR?3BF%RGc48fjI>*I8EB|?PR?p7Sjvn{=o)$D^rkT~y`rN_ImL(9v{oWqL42}S+F z7Qpk(1wWz$sk3w)ze;v|J;`=e=<%3B+E`_{Z`Fx8Kcm&(P5a#h9HjUYmlTEiq zix7i0Bq#K9Frufef2HS(qL@A$?72d*9XT`y1@~9opRkn3U$}34h6Uq(-4CQDDq`Va zqYb3GAwv|Y{jb!5#uS4C#Zi;9!I^x0L1fO{%%Bo-2+%US?2}*HBL#lT*p=ft#4yf9 znQ6UV(FKHcaxlmg@nCZhdf@Gmr#Y}Tf=`fK5~*mk> z$_Bo`Fqo0O*MLyt0v1+b*-nx~w@#x`J7xT=~dMlSF{>HImsoRUZm18`STwuwS|~?hUms|M;5T>1Mwn zfA{M$|0vdEe|SpW@e~Wj`>V61%=hQ2_fyUi;lJ2AeH$X!;1BUq_h1MCmO^}JB2gEz%~t&C{5SMM#`_9doZ zz7wT|NmjBEJ+G%EF1~h3jlI5^nRVcH%{ZcA8+s-y5|w@2L;IwZhw_?BLVU?N@$5P~ z!-DB(mwvY8UZl;jbqurxb2?dgo+vma7n2o|XmEFTcV{8N-QC@SI|O%kcXtcULf^{X z_k8#4d+v{~z4l)Hpe>53Hkq|XjX6f|V~%d>qrsW|8-^lQ-qvb1h&lnW(=U{`a*`vT z6N>`tfJlmTgev;j$zpTYFi$^j+;tsEcT>55dXEWhh|8?|>|QocjRfovLhlkyveUyf zhWE(aaJi`_9)x_ z$S6@u4n)4^orUr)_1XGpi<^$9neq9l2phHLWRk3m6Vrsz+*yBi_R{fkZq5JCTAY@4 zX8jnRF*+CoQHA{uitj#P>XJTg7M+`dL7k8LRgf>ZTznxY&AO8EF9W1C!-Khuaahc^^-7D6t>X$gd@98sP5wcWHv`gn zep|8oR_6gf88UaSMUj7due;IH<$;&eGca=rmbLSAWCJTl)UWr1<8%8~AeL=f_B9GK zt~ZzDzv}C|LeCVaPAElFm7sN-Nvdzl*g&%;eXJIgoz3;z0riMeMO9hIT1uLb#nqO@ z;LEzMW-nei{%i)6(#3A(HwB#OcJlEb4!0cu%Vln!={nQpvn#|WfW4!Qd7?{>{8tw! zo_nYr^0uyIy*AjoX4?w+z8ygvMm;tIcecXkz8$b4^rR;fT9NnYNE``FN0Isz!S8}T z0|%dBVjeDHUuV`(8SJEoHjfvta1U!1UMLYp`))kRaLOqiQ0hm1u9T3t4c*a03fK~- zW+v?~S0;akXV;6k&3@FLDrk4=JXR~6J`7lP+e``{A#5KtoV}UiTK8)YAM8=} z1i2l!+_|8UwA?>tW&H}~J-Ri$^FY-imo5q=i9Ge_9NL|oHwj)*j`ey07Iu01$!#4lTYVBY=OmH_?aQl!}*d96?o{9KZewZjf@=5^3H zlu2m9!ns@hFm2!YiD)Rar|TIw#O^P?=3Z0gsPQW zk1cY3CRp>Dw=_|bIRA6Vu8!kRaFVp^?$Fyh<}hPWb(`F?0P8I8JizQ-OjNB9bUlna zP(YK6l#4ZL?BIhzv#DN6KybnxycX}S*Y^X=eoHx%P}XrKC>pE1LNAHIyYBh%er&7D z>|Jx7mok%ChMu^#mc-uIXA*60_(lqyJyHKrhQW6eud9XXuJc0ciEs_|yztJn;7 zk15)yo!QH9ZJ_gps$`X6wicr%Cm+g^GMJD2s{P27&`U;ACDJi`Gf1P<%vfFNmLL{Z z3q!!2a(+vrxp<1xxdT7Z-xAkHa)QErX0{$hQ;P6o(8+LFjrm%5KHfV#N?$>2>5&LZ zS$zy1Os*@pa}sDkZ*G_%0fz1zR@6psq<70x`ftXZLK`~PE=SLdYfPu%%f#5hL|N}- zjxa3IQcEJhVd;G;-#zP!e&Qr6kgswyi~=vN&c?Xfwg@@O6MdqAOpsBTX991k;M9R0 ziPH!Wr>zz+<%w&+y|CX6)EOtrDg@W=s)1{FIpJMj3mC>_#=yn-#Po{hQ(5CaNbMdc zRN7v9tlLdlzopwb?lQAIZq4Ie?7P&j>4!}+a^ zxq<@3Jlyh$GkPwH}p*c(N=5jFAIovbT-4 z-HJIAYbx3?eZx#xHNDb@xEc6H$`IXHTsOUb;?gU^dig{Dv^43Wk-6Lt0?77>xOrY+ zHt(&1n<~|Byz_cGV~2x-vsJv~Ur;@AW*=SCuvWDN&xE!8HdV8M&g%Zof(e9=(WV-B) z-z93ybTXm{_fF!P0%xAjZ$f1k4R^nL!9iKBeMa_!3_;`k<(zK1}VRLQH5M2{n8ua*rJzx7}qx}{WmZEcM zKGUbiSM0?WX<#{miZxS)k56du$9z76Ye+E!NEm}95cB1=?GmPkkaI)l&#kEE-R!lA z21kHMpcmPl8K$)OxytP9_|WJBRc^yZQ^xpL){GiU&ZGZWSi8Vtlgf)m__}`WfRB;RC+s5unJ(RH7tPpOx-JY7=RDo`g|?z~ zzo|9%jr9cfj%YOhc@3jPv54EC(1m})WXnsQj{Y|u`tGjmZ;d0tgS4VSUIm{06oquY z9Rat6tE7m57dRE2>+6YoQDWh1f_R6VR_@&_t`!|V5GM#-lW8L~_<89s+N2_-AWDg9 zPYyz2Fz90tTlZe`@Y9he7)czRzE zR6d+M?VQF-#~bj2D7a;7y#%TEUL{v6LAxfLgWokrpou(2s^NbcHf?HEp5|zU!!==K ztvJ)Gyf?WT8t3%{xsJBN&$~D~=R7A=3}y};IMO#^n>;ym^{okG*$Z2LTmC~jaWM}? zdihu8**N0=%c#IjFp3mieP8&`4!WNFO0Ko2TSik^ezlN8^4omgnT3_U;1+(GryJb3 zhaHg4$lk<)X&%U$8G`39-jC}rq3Ga48Rf0tesGr|-#c;4HRwU}c-xz&K{~T67u402 zLhaxen)`vN2$qz)h$Prs+}j=+Jy5NHR=OyLftJ=SpNsw(2w!4^XutnablRa9ee&bQ zq2lx!o`Z}h9zAsB5)I`ScG&+xY7I(t?cJr;;qgW}9}o@^+r2V-)EV)`WcnTQ93#4` zTE)8vT@RTIrITlQJWcGXl=U3F1aW93%XEE%9?xjyY+%orc^A}9b`=Kae;=%yj@EAs{T-{uFNQHyIdV)Q2)qWn1=|~Y}xKmADN)IrfSmUDIzBB>Wa>c!@;^@F5y4h zIw%WOah(U{+kir6TY{J@?V%$%W0w*dvS!9Bnx0>i6ost6G%A%UpRxbg@Op~y_5FaK z@G`-w{A&%-;XV-$6D;$f6JgUvo@6Dl()N-X;jm)0*EGTq?Fsa1HTL|so##ZQ${GFM z4{qx<;|4^0br;Bgv{m4onxDWyH+!q76$Dy(hMLFSfWMF7{6QySuy4h$BunDvJ&oYV)@T z$nF!h(B+>E6#7c2uqzDDp!1CNS^GBA^q=L}Pr|4giQ(CDm&Z-+Q~H~ZiJmRxrfo_@ zzZHv?HLHFWyx&4gtTB2r);>A))XOsJgKB!@?OjCf&>NX2MfFJDG zG^*CzOnHt-;|esTX&p!MDp>SdSY&5oi%I*BiigUcC4n%XO4Q3>cK$IUv5QPWH~uca z-v@PWdarpnP`xafzOlojtgU+h3n9HEn45CG&KWB5%^F zGmhJI3@wwEo3yDKHaPDPBcyi8bN&b0+ia$qibz$FTNP0srk!oFxkL>ERnHZ^dB(t2 zYXn5qU@O}^+d8{+hE~Z6ay8pC<|A;jk;6bR0p}1cz8A{vUt7$fv#M$U zLPT2Yb;X;wfo39}>wxe&9yhT3hAYK7m0XP~%}Kv?@%Ud%pol~-$atMd;P+`s+RUN5 zQ{8;@be))n+QI$awhyLmK0!G#E~2(2mHBQ%ueF&if!>EVfks~@Fz=RJs@-2#gp8ih zFpr}k84*3-YGGL*wfroS^P5xqeFmQjhqm5SCVh*;CrIL*r!!!CCI0>NJaA-NE%j0}M zjR{r@Qo0qZ*hX7eo`{wCLYa^iKi>ZO*(tB~YPw5&mUE$>3q)(%NaWdgi`T&7I%cc2 zs^z*S&vp1LUVe*xaVRMKQkTjSs@FI7Ze5qV9?0jp!OO_uQ zq)w$g;lYJZ)+C2c1DN~cI$?R)z0nm=m?~)3t|9pm%;ZZ;;2dk$&K}U8JrrFdu zP;&t|&&A32$Th!c4A$UQ7)#YK3cAE%z}Nz9g5X%weX87|K%$ z53j|fMT#BBdbU(=5C!Dwx2%rcdhrqRI*uPafjrg^I5x&PH!hx@+=8&?CZU_se-ppF zU2R2h=XRK^RD;UW9}xH=7VOIM`E&emV_y64^$~@6+(uyL^5NqZJ@41->F9Fz4FiB^ z8e@IBgqD^9t1_|K#QrD(F#rC;m1EYmMys#Y1&3rqb!h7tmBC!Cr+)f#;{0qnZgy-& zO~Pq>xzOu|algN3zs}1_^*KKlW8Uwkj1TdZi`k}=l)B@rN`TzZqfs;Z_y&Q4({G#1 zf@V&Z1P0z5B64DT`M$YdFdRN{aGO&1meFbOb{mScN!9qI>L^%ZHOuRN5Dx2CZ+>0G zh`>ao-j^^QHO9cNQ`P9Mr^W7DM=IE4X)YXL?RN%Uc>@|jOu~iG7#d1%CIURz^4miH zmer@Do@<_aAxF2W5OL&{cS#IIgG%iyfS3{GOo9?j5f8aO7l;;19f&pCAPR&zM5 z(r7d}V*3MD%GPt{u{GEzY9j5r8ZfNzbS#u?t(OP~o(=M|cm}F_mM9}urdHcqmzHBF zG`RDOiyTgGs=*l=aJwU!dqwgNB5Z8XDpQ&G98YZWe|(E50z9kYWB8FzDF$~Fm-jIY zQ-wKD5ho6<&8CnSZHq_|1%VV#GNg-c{8?{65*-Gep{;&Fw_0x+mLL7MoPkjU;_Pi7 z3TN`Od@0@@x$@qYQ;f&dx7ZzNCmNN-Hkz~0mVm*b%vZZ%mDlVW`y7p@!h?f+9C!_= z36AgMnqf>XPzojSLv&8|d6t;{hw?IgAvUgeGHsFLAh61Q%4@N0yJ~!=!9RAm^hNpNo?YPk;JcADu4M#0u+eIOHtJ@;x3%B)`5Oud8{7e?ia0uL&*? z#8B=4L>>mtsNP#Uw0k;WXPvV86`p>4qnY6Ai@U`b4TLxie-8I9U-{ zE8|f1{yN9WgYir-H>&i9uR+mkI1)i0Z9g5B>#&36H8Br3mSE(pn6 z8~pu@JEvo$i7%K9W@n5_ziBS^T&Z245++G~!d(v6>iW4=ZtdnZy0X4kK!rD?Y#xeO zmc~1fR$Bxpu7{H4YzGOhr*uU2?AN%adx`=K9=~&Oeh^+dRlTFXs9f#|h+XM(ZaLU< zcHm!J^lwtZWwBH?crrt$*QWnS`b`jqn2A^Yb-7t0pf~MKn~NU7x<`=@Q&{p2HR3`izHgy8g+;}wL*agotK z8S-gw#g~sVeK6`nQ?+^>Y|lBRGOc#!vXH^p8O|-vofrqi1ikKVOU%N+Y#m--N8-mg z)+b@(e(Z}01+sIRhzXWgn_Egv>b25X#%nbZj||bLE*pT?;dPx0RL^=k42+0BJ?g#WoI^hF0oEMyd&&3)#&diE zFHAq>JWP2s_MgXhqgLxr-6o=LcHFW(;TAGI?W&Z{)xrf{l2{0 z?mPN~&D}Ph&JHn2qpp7MmGE8*SEq7EH3nde`u=@v`k%Fq$99|-HUl-B9%RsebJ=~c zq@o&MY|vT9iEv5DZ|2hH-*!(41w1jw{vHOrk3?yVbu7wsWiNSdzLQt}Xz=UE&cUwl zAfUvO?Fog;3Vj`^(X8>i(MwTG@-M~YjrjzVprGuPCjiYhDd&!yQDm0Xk1)cQh*kH2-p%>y;ZGQ)S<~jy>hA${stIqj$-!R@) z==dx-GiD4RLHLEnX}mWpW!u8$K`lLXP~~Rp5`iz&V*dWT1r*n!nP_|xRfYX&aiqjY0Dk2~wJ4-SVa-c)b2xVZsw9%Nw)Pl+^N)(5%Oh=3$xU2RL z#1iU972e78)xoP^*XgUSs-F;TCbzvUUEYj}O`+|S?0y&B@?ybaec1wAKr~!%T$zzt;@{Tc||h4Bc}Eh8~>YxvWwS5;Z{7KtPzrzu+r&Z-(W=jV9;3YK4Y4IvT1Zp0-bfGbX;f3W< z-BXN2UJtfR2j#d11`ye@MLQq3T$DI#dV`9Q`yCp-QEImnRA;}7@8_sWA#l(pKkHoFDU1wR*V$|MxBD#!z?3fv(l6IZ{Iulg9R2e(;ZuuVXhg3Or zKQyKIPVfCjw1YJ?bYk0Z&cjF3^(1$7NZBg0EzyiH4Gy4mEOW15ey@{zCf#%#{pVyhWGb%$EG|o$rwVnrky%jb`q#x^ z6IgtX1X52F2+#CMj{DvEOP}O!oqUt5Y7djCIQodMZI7LV^?zdXK{=U$+LtNZp8u4Qd`YL%Fev1_HhTh@`%2M){K4H#|Cwn5V` zQ?s=e)}3aVFGQ6tYU`8g?V&bU=xh@S`r@!nL+|_|vmB0|S$m^#Tqj0w<{w^WU18Mb zey^da2T>10dy-e8(46IdMKNy^%pkKnH9fe7cH&TCn1bObg=cVv-nn`xXox>V>U_)c z_)FkB(Z99*b$m{i{ghZ4z<+^6373$#!_=<+PtAs+p&N)d)e*i_VQwJ7K+lGT_rVou*-S_0~I5G)of~ zxfhjIM!K8vr0W$p+&%}$D~2+&+djq;AdO4Ei`O3CylE#u$D~#PNs}4Mw%jT#s((~FHYt`y2Ye9_}h-;dD{bYa){6aoQ0SapL zx4&Z&&CURK@%cc;SWnJgBcdc8lS(`)87)!uG-!6PgaMV;QWVv6PIdS9LThaaEODX4 zZg@eb9>1MR_nCdlXmDw0v+RF#Cxp z&Bc8qbt(oA&eAqRo;8d3cLX4;sp3B9&QSiiG=P{~L6-c_>3w2W%_MlAGgr^D**Cf}KuOO!S+b0C;+X-}O81cqXDo(>VF2!5kx=Si z(Ri$+ESnvWvT|`lfLU zA_-Ebd1iC3_&_4zF516T|0HiN!|}t9E`G3A_uVtKG$dc>r|2Dt6`pS~*yI7d^af)~ zkqB=Z&~+HnlT^2!hi4}N_q{e^Lrl@t%;`R79*1iqXi}hrc?_9iqx|Pijs>A^V+Io9 z`|;&(WHiRKqiX; z66ZcOpM|=)owpHHR%-=?^*E{WDcd(QVmD9#_BJ00EmLM)?{{Sn9AtU|y9(;c!T9fs zzu)zHm<&YY*lc+_#7;X=2PTU>G@saApAKi+5yj0Ja}h<}H@7o@!};pjtvP-aEoF4r zh1mTo=A}B<_Wx3^bW5uXP|+yb1LFLB-Fs=VTA+bMi$S3lKN8C$or#2Vo#~cR(}GHKVjU*zt%x9 zV&Q!WU4tNDP)k@q0zthj)gVJz!@}IN1NvM|k#hN!pI$wBnN>Ci!$sk4fI#Tg9NZ3# z>f*A($cG2ID1jL&uNV60h)T}t*X4PPM}x99&FiK{{~6O%KmvPs7oCvoBG{NGDgH1)Hj-q121@2OOLfRE=~#YONu)g*`+H4_)#xW<(49#@8#C zEq^PX&^(!U<)z#!w7q_Xc6kzb+%?YUrHbUe>6l%k-?T0rZKNjs6R>~Gq#RIDHyaCMfwAtIF{AV?dM6Ta!>#W2 z?0dq%9xdL^Y*!YSGMuv1({F!I!sfvAV<0Y!>rmfYXxNQSqv%Kdy zY&_bvA$1B46#YhB8|M*xBE@Xc#x=o|Iz)qeE6`n6Je;Do8{}Yc#fG%>IN360C0zw8 zM*^uO!J9s`DG*Q@n(CaVO7KFb@Ih;e!*$Xn8o7`_4xh<)`&4ja^POg0_9g!CuAKBU zhxdDc8b*gj>fkC)REZOcXKNj$Iz#5s1%z6N;euiWDSHS;EmrjU6{2zY%{P~piDs}^ z-JSJ&3i+XmW8ZhggdfDUR8lP>3^N8>8*^@eUlNF1aAmEu zE(uUH?|5awe|N@hoAB%(ofBJ7X$I{mB22+2H|GSmD;zLRp0SBxg+|`3dDynbew%lr zvi|~pV}eDaNKL^O(PSZxNb^0MCwL~>^aH1CmtvkuW=U1392VZ|Dx%JiDU|Bl6=)zZZnd|hk)rqqC za5W{Xk?uYfm-I@Ys00Bz{_bfGzPVcZwxfHXCMhK$A1K8#nbU_=grU$QMMxykr}wbn zhKcw2`9hjOj$Nh=%xL`bNbxu-5}j427ZW+H>Dme;S?APvt_ErGciN#7JPdnE@)R z=lWk@&DdO@T=LHe%49zaC7?rQ%tz*~wu&2Jj2_{^uDFKYT|9TTKU5x>Xb-hwh$X+b zSa>HO=yy49Hsu1JAEqHwC`3q2lND$5nS?I&@~kfT-n+jwLByRw{EN{=5&~_w?(6j7 zN}{nW^Os~!1~{Wmfx;WJNTt0WQ%?+}Y~>ymR$DhCI(@UvZ=e@4BL60tGFJ6d%FjjH za4bJ#)=Cp&WGRxpZTN*|SyZ;cDjLag)Zj?a`46b$?>hPa9*Vt|>CyX>)%V}*N#qlCY>7;@gT;S2vOhWU)IX5hj;I*$$)22VJ;tmJSt488do^TB zZ=2h*;2V0&p}dp@Q=)@<9~%dB-d?==TV7dtl@XyQhZ|v=`K`B>3XB{sv~Ia5oJCg~ zIs9EW@jb9R9NEgZDH!$oh|{@1+3ZC%+v%hx|26nuxud^p-G6yg;};8QrPb2x%a;#C zZ)~b-@~WjMGnrJUp=HgAgIarx508~f13(7K*`rsCHorH4*G!z=3;yBEC$>fuq06-NHYE!%`~e}xGDF#u5V|APT8 zJJr}8X$*|pBWwgq;qKM9W_PUC3Ex!GX)kPe>a!5^R^w4Li8?y2eKE!-#&xGB=}l?R z?^gwafk%9AIB4J`)C9b*Y0VGm+rQ6|Pz_7~@E@XpC$lQ}ckUHAlmzjQ$Alb-A@$hW zGa{q?rA1ZL69}ZO%>RT}9jim%xR+4A<41FE=r2NVvSJ;T zV;Wig)e&SJo6mhP;}h*YBV*yu$BT*XIC>sex9{PorPU;Y27WBHu2#U-Gnmwv%p6$0 z?jew2I@h@yfAr*H??t8Ls0||`=^%SCSJ8$&lfL`e+?z=q=-$3yb5Y}&BKP8Cm{sZL zAjn-;f5m~V#ixBn{(yk*la)pXnKT~l*_Sx%15PR-|2MjOt2sIR!6+%) zLV`)wiMW=;*D1M+1eK64Z3-v}* zl!Z0JlMa$5C{UFEW+zT94rB@ZRGxV*?%4-Nk(K1OY{p-@-!_!Xu-zplYDH!9rI$qX ze=#3#Tu2inyDyE{;^FxeM)Iedl3Eg!RBZO!hp=!zd}n}S&87iyaU@O}b$PFSZKqu6 z&;@SzYi&tkh+JFdDtwuC{9-{7@rnSIkR!^D)9<7!uCX@{t~FCzh8N%C=A|gVM;#oJ zedX=x+KrpcrgAE`L3`ed%QwNL%E7({F{7(4=apN@%ouZx2HnSb;)r|kb}|G-peL=b z9Z11kZiyFJtCoIe+T$IWvX=b4&w}B2_r=M9127XkPp-J&c*iurb^xK4Pf4wOX^ETo%u@7{5STRi*?j5emS<)cu8^ z>||MKW1`lAlo-IdTPmFJsG+Vd_vCD1SP2%HgHax?zbpf1ov~=m0zAHdd9HtdSHS^W z*N6nkZ9Z%HFPEaB9U~VrbsoP;?sUnqUQO*Q!Iffy%m~Xac~Ke$6%H5Qus&V0f;{1; zhS_@mKcx8h9IL6ZXTlv1WMIk03D(}P1JJNh{Uw|I*X%BE@ch3)!E*Q6>8c6{7f zzX;E4D-aWJm#>m2oK)4Gw@l<=E~8_6Y5v0!{8aRJlPa+UCl0_`PB>EdpAR11%d-W1 z%Ch`emYt71V@Bcxv92c%0i+!>>lMN8)Lc$Py+v5+^WNGEfhc)$1r(vRZznPL#m8Hc zUGY1?;Bxpi(shbcku!6Jhmpy46uQ!8ce=)hjRi85G$NI5TgJ9;bcc-e#hsOv$bl30hR!fXqo*cKrjjJQ zC!~;x^Q+|bf-sj9*Qz~%h~9s<*HSxB+2F@b9VY{zR+)v=Z>LH}TiH`#(<; z_wpKpvLXKC`rs8B@~5B_+OnTV)7I^&El*z1r2Z%!VP$^`CbqYBC$q*cM5cbGtX&)$ z2E91Q@o;-EvSHrSG7%G6A!#Fcd1J)z6MZ2W>1?-}^6%h>voV2u7hwWMMQ`d7!h{E{cKs=ZhZ6yM>u~fVe^-KC(#$|fF+;?s( zSYY*~9Z_QxfY}bebr(~$$_Tf73O)FL@oB3sd9jU(YIkhESGak2B}pIjmt>NUeQ@`J zIUCVQ97f}NHn3x<=D+yjywd2w(hZ#fPP~xQ#}>GmhX#j9Ur)>)*h{uGvI;QovzN*u z*TLJ_TY|L#5$)E?CkDt*hZ_25h=%#6+b5jFP28(o{$P4`F&Sf5De}n9UZyQ25JIyxIO$2{TMpN5teOYlC)Dgxo!_5{6YI zs(X6F<*<(MwbImV?MVsB(Jb5vqb&ju2LqEVG`jRe*!0JGsO`_i4&!wlhlrR9qp*z;Mmi!|wZt>I8zNp@aq`ih5E z#)L6rB7vfa?v*XJGt82*Hx^WdB)IF9UstEUQ5=E%#=_~B=lm4gTiMroH)RWUV(G%N z_Z82c!bq=Z8@X9w@`+8%XHJ^P^qUEtpD~MAjW>f|nq{*yo zz6?kv1gU8HrnG$isAUYtX&w-GYD(@nvw@wrqXTkuAs+%N(AcO5x8^vk_ni!WAg6tV zHL#|p33S{`m{6elFetS*LtG9fK??*os>_DXIuF_X%qrWSWHL}(M7s7sN0P9LYTSj; zd<>GGyWG&31D~?1=eD@5*N3y9O8Li8Z`;~}4J_J=gml5sg zAQz{1el+C~NWS_-F?9VZK4J1Y+!`(&4rKr6Ql5C{R|Hrk3+StK2J^3Kq%2ffjfJeO z7`(4>n5fH+c-KXNanTlE^t9i38IUj*zxzKx+0C{M{7&`B6K0OrBk%zq?vMXn)%Yt0 z>eb4ZE$Wfvx;qmlfNyf=3^0gc9^|n9*{_ZHs8>gH#TJ3LB%9pqbAxzc_iVpsXSLia zHIU6`iqVu>n*gCRwplg>^>i@5wf2l&=S*b&=FM%G9bVe@_X0+)a=*@s^dn4VKJc}(~0Ird1Qp+`wD={=((*2njL*bv~CQ zi`y&N^dM#4oR>vEn~|ox>`yMVK&COtH{W_R{NHnedlY-UBx1ZW)4w2}u^S_3^FnL5 zGt2byY;E$h?cnlT*fRG+ZRmW@D%D}MGybRs`cV$u)7^7K$R0aomq3WUU*_r+ zDP^=++;3W{ZY>@B)a%F;iV>!;6mR;KEb1&o?9A_^M>i)o z&JH;#x&YWH`W|FVtQ_=pOgaX+NjWP9eac! zo?I&RFV9`|<%jpzGEzl}uz2qPvbT@XTqgP&^>oU_|dxv#OIPBAvaB~qZ#Y^kG&|5k58Kw3bM7# zLtGZ!C@tZ8b}9K*$-Rf1Q|VNSIp5$}fQ6_mrfg3Yir6F5NGfeURi7qMIc%xL9zR1p za^%>Z4_WL$00KI`Pw*Ll#&apOaDCiweR$3PERZpInUN+>dfZTwY2*A*A2(N|qY7t* zlmp*i*Vn3HQU_ineNt)R?|J&Atkqs7_3o^ayqb59@@a?3_IMp4DODMYlJ2Q!*arQfRP(r=HH^5OQ3W1Xjk^bm=`@HY-$6+o<<11Cw&EhCcL}-abm@}(?q{wHs;vE- zhgQb~X%f?sy?FQV#MXlULE9-Rt1Pyq)&lnPEJKz9h~M@Dc@Tg*oQ8^W*O9SCAIry) z`sRY>F-xACR(_a^8_irfkdwLIF(PjcCEjZTK1G# zwaqRR_r^iOfgi}k-XKsCU$~E&2et(T*NQ_sHY{EdF#saV<2Ra zwGE@!4`?>@TjHGhisc8yHlwy zv1dzpR9Ue4jItkfa(^^^j3vx-z}nEf9z6cOws zaIOWApSg~Yes{CP9=G=bYXwXzOZ2a4$gki^7riCEw#myXr?osc=$l2z4OYHeT6Xp>;fI%SE^N`LjSs|fKQ(-sjpnk zS6=1PkHlGOv)joim;X!%`-|QHv@js3H$!UdrKQX!I;PSbkI@&p7eiXqSE~mSN21DX z;!dJ~R6G8COcYE{6YBpn_Q$z@1MABpPw?q5LN2ZZ5T-7*t zWB2GmN8)NE!o*mI!S@xA_VUR7U}}!?_|q9$;fr1m1om|XfuM1#Q{Ee$A_5@px5L8D z{!+rX9X|5XF-Jc93k9=1B`_#!s%CC_J?+8v7CXFuB%}1YVs!SsXe(@~$%$y_Ndd%Z z&S?Y1QkjmOYbHE-b9=hfk%s)lp+dOQPUmM_X);m))cPY1S3Ke)7rt!If}-AU2HdcqA@#BIn~XTmJHf zfHC))j0ST+?!RgQoZy3|3k9ad$IkBA*oX#w4hl+1m5yZNtLB5iJR!LKAQ!dL=g|qn8LX*E2lSU4S!HI~oGAsL zrYzOY>^t{W&0#N(evz+5gmuo8C+<`+B=ji#Ju{qo+A-I!se%XsHST*Q1*WaLcyhiv7Y<IYz>(Fue!p$yN;-lvz~s4ixyp815jpcHzp{Nc3M{*s+vELg8CVrs zT`9fOay2a%Iu5as7?5wZCZgW(spr+|*_9C)i-A2fJgLfh5qsbYtD=4!F=LO+p8CvE zU9-ip`_hufE+hqe!Z@~yj?ix&qS83B2^wmqF#}1F#0)T^+R8(C!bV`G-PbJ`1{nsB%qP-O#R|8j^0G zdEMYJM%utEf0r*#;(qn4;?l6vBe zkO~g+|C<_ z@i;&#;A2F0xGFvOZI0S?O}$mQdgMUqo9?ly62XAd43*^3+jb`(xX!Ji;sF_< zSL>?yPqf)GEVAI!rC`i_?uN_)21MH>KDVX)G&II;@|d~X&u0KDCT4j}xCmJwE*@W= zANf!LGDqP!r}}rT@M~^!T<{14o{5q8o;}qWkAJE8_u zGg~rZ5Gbgi8LJnmk=|80F0#Pt?QCz9$fnG*Iu5K&6$9@wV&a*tgW31S4zT_$ z+Ir;Ok`E8rrh>e@f+NPCs37CMPt4at{x$G?C-mz?-#as6%U!J^dhTr+#V<@<2*?{BELN7>_TwhehS4A&`Pa(X zz<(30dKmb`y?NhuSbut81bAAiJ@NVRR2msmu>kj-5}!@jfp%s_DF+Hv6g3wtBd)T?3T% z%F=H2NBI|O-9)K}I!=vuD9~VS%drQ9y7TT51~{^`$1y4dYUZ;EHTxH*;1}KR3ik}G zGLGgC%;MWwZu8O^uvb%d4{wQA zMyr4tW|t~z50a(ZEaJ(LPkp%4?B0W;Y9qnYoc&#gPGymNf&UM6Zygjz*M0#eA%x%> z+}&M+h2XBiVX)xtKDfI(!9#EccXxMp2|l>PPTo)6Z-27?Z0&As*VJ^^t*)+?E6;t- zx#vu^5R4(=W?dbP$BPPHo`ECr*^L~)&EHGc;DK&8W@+;!Jaz8%smqgj$o$u0q{$3q zHwv8n=yT_16}NfUr^59QLsm-!XJ&s^UHp?^q-?G(WUwQY5hQ|sr;o<((NR2V(Ws&z zT;xZ9e@`2cTaI_oeudJe^DPQkU#P^tImI?CxpD`8clU!w0qwU zw#OMl3?n@KXPn+NCS|+SJBH@EGUJdqsp|8a=RY|^yZwlBX;%|Fz9!$zrx;7( zZ3#3wVI9d7!6Y}9Z!PD)#I{r!3|&Go8T3|1qSxecB^Odd8@HUX&?O~6oy+ZHPPOX{}f| z0rZYa)4OjuinnSwTr)7OwSL2NsQE@XAprbT8dhrj6e`Qd@TbbnQ; zTuWUv&cFgE9$*&*uPaxt!+Z-pK17o@T@?n4*+CSoJu70Tb?``Up#C`QHHoBjE0ZS{ zTcy$&k(iww+lBzUj=``O6?yHMG1O}rNa{&kntd<8A`S|HO&66pnOBFTB*nH~Sz4^w z&&ET4R%M|0q@i|>xG7RYVX_v;JJ9}F-FFL+uw1I`N0n=T`X-b4NQ4jvP0^DlYqv)O zFhOF(5zFp8L-V@E5sUWrHLlzE&7ahGcGGXXY$?XK9?kN)4s_bLD$=|M(SY4j5E~ zn?$9j&P8Q=%&t3gqCaWTdLZZy3**QHzS+TYovO1;44Xs;1c~W9F7C&aYCxN>-pXLy4w~-QNIK9WM>CfXp2e6UsSLu~<2m`^7 z)rn0-XSC_+E_l*Lb{jK{>SQWLdyZdJIvuwv-1JX1X_&n8!qfj;8N97l!|pPhL%2L% zWkK;FoYD49fs?zwYq4=}A7kZtK_N& zb~;;iNQJAqL;)uxa&}Q9RJ)NSv0gtr`evyvv;{*aQR;XmcGJ>+{!rHCe^zz5;Zc9p zdL>LOsFLzJb^FO0uqJir8NH4bf%3XNbDVwW*d_0&DeJMc-|2Yw)OX_Z6R#c5kP;O| z+1pTEO;y`sVBAPFa2XZ?8U6G`=%olKIJw@@Ny@YM6AS4m8;~*>;F6L3G8q zT`e#cW$t?v3s%e0b#`D|-sStGVX(36HYKGLRhrYC$(&D=ju z`D6Fkd^r*_i9oNfIc$4OX8pN8YDGI3{u(yEb&+1;f3B^w`kPk6LKv*2CG0~%xm9@6 zaaQ<iM&XM%9=yb7; z>3QpdYG+Hexu`M&pHfrfHWV-E*Yt1KFM1KFViUDzaqqR)VGxy@w>Ymkq2Nde6={RX zlF`~IGD)LQYoEXSJ#cTAPWqfpb7wCNq$I98Odwzg_eD@x|uppvhm!Dc}M(xX-)*wh93(R#?Ewk_MKPsrc$u$%i zCF*-?LDr9GtVdQt<}rC-p=bTfZk5=XMO zt+{oeO6ZU?2gp%(3= zQyT9)sw+5)io}Fk^?tDjy3-)fYWjC?N6%v~L;l!rw`UY>Wg(MUUJVZ z;VzwUV4rDNa%EwW5VHRxAu*5eM87W-tWa?LJbC;;zatb(67a?rtp7B%SQD1r5%4M4 z_Hf64En_Im;a{2dKOHWK&lEg9uXT$ZYimBrtddaGRF2tC;c0jB66c80mHh}vBeK1F zV#)1N+qzv8@sqj*kKc0HxBQy*w5*+(_PdWjt@R|mUNNg4{Q><&X!+Po7v}}d+bVNX zTg!WMN)-bhT9zMCsx6kSJ;{ZU)$X_fwRSO6V8A@7kfHeDRPd{;PX9=K^yK0rjr)yf zX0(70i*o%OJ0zq#aTb3quN4e|tX<3g=KuzJ;V7a#{Wfvu?HdimE48L6{T5<2j^Qk7 zT~KIF0ij1pSd*r=h4t;eE;hg3L3oGJ+tJ7UwM3bqdpof@98M!)Z<)yA`%m|=Np~@x zTf!Qlxhk~T`ASj{cik(J9W_9~H&&Wz)}Ze-IL6&0K?H(zY;~E>6SvVhy)xH}FpseCz+3V<)W><{nCZ$N?PY^-pUWj5mWyT0lPCos_2 z9=@}=saR&jScVd+nE~tZdmQ4iTmL}sh8Qx`n5>j(3NucO$CR_cudZV+%gCkvxt|4| zgpVICv)ki&4=eT+$_aj@+cB~!1#ces#sWj)>y$tBIUlNsT4irIRv{v=QJQQ&JsPHr zq*TVIdxU+}g5F(h+)>rAtP{LnM=SQka!Q2gYp5{tHctAhKYwLwb3g5B*UL@JAi3Z& zq;i@~kASn0Pv%{krto}!CSG&|6Zfy>-73csiH5112xmdA$p?8{vsM!RnkM3^%pAK! z(W~NRzXw}DgIzw8w|1c7jHZiH{Cc6$G~*EyA``B|yNrtY#pUb|);Og)SE!2Tj|2RM zx%zBCbDlQ~gZx?Ht7VwAW3wlgh5As^2Uq0D4)#RN)5kM#2tAK01NgIe_R_-sh8}U; z&Z%zP$2i5u1!8XnhmuFOx_@>DII6?w7?;U^0$w&l)#Ud?RG zhqd`R`;7}0b`CLIHxW9=yef8UXL80j$qPFtF)Q&ZJto}~>$j8*ygFeBhMOOYnHLX;B9$E>>uu4%~ z2%M#B5v|t>gpd7J>|iUY=wQ;{Te$K&ORM{R-VIlK8tEFJtakT+d}D&$xLd3Fbi!g+ zeZ0@Ss#e20>h@@jwbuJNlfu8zW@0{&++HqB@K)ikjwB@*9!Kd8I`8wkTDk%>@7$+v z+WOrp%QDO8+9}AIw-%z*7H(MVT}q8~ngX+;MJEbExez}i=%`hMAOft%Qo=U3&`G1;JtP2j1?x&<<`*(yuN#vTbM zWs=FC2Dx=$DenW~X@s6-&+`hVo?l+`!qbUo;pjt}A!5+Q4ezbt<5?bGyC+c~_6naX zE?W>Ty{7?!q7zl+Rd0~z`3|$O^=P1G{rR{FU}KXP&ww;JJDG`stbwVPl<5165?;L^ zf9CuHS1-;$WOnzZ4TarNgW8>mziS~Mdlo~a*yU78Tg^`^g@#vNfv8$rc?nu&Lbq4~ zAttuh%r;j*gMTU=_H6~0hDQsgRdCBlPYAD+xIcGYYGXyqK*nzbIF?C+D?UsF8~4e_ z_~uj5Q=)c1gghFkEH8g54=(yhn|&qSY%r$ce7Z11=5I5c++4qV<16I!zQn**f`GeP ze6M{<&e@}U#SJayhUmq#PKTU7E1yt4P|1CI^T%+bv5;Cw_-~;c?}{VHVp50)$!`d? z<$rMap#2|AHU4jv?YY)ML8N!ukzOeoDx5$5yDKK0FPq_C{;NbO@GlpT_H=Lje|1)W z?Votkw*J2Je;nn9{dX~;|DTkV%g#R9FNa&0VQ$zd#wt&+pL*)!>T;>dDFc7C9U|K< zOIw&NbwJ@CDuS#4GUbWr(F3!F6XsV~J)WFtU$~fY#VeUfGHa*1&pEtx`d@vaMHnJ} zSheHy4>*mHVg>VKT1==cu;eyQ`%LJnYxaD(+K4GhG6NQ8bmT5I~jusdDfzU7H$ZM{rJc2pe1%o$kA_900F z%d>kQJZ&@f%B#_AN{jqnixFUL-B%fi7wS&-#O0KxDOL#%K1*E~RoXp^(dIAg@RiZ7 zi4EXCc&ageYv}Po?+rHQ*C5&5cp@bUIa&RaTv*3Twl`4g-dNsQQ=$0txYN*1fUCyv z4iK9upmpqqmP%}!V;j0+0hJRV?(`Pzuf5H+lceL9zse&5>~}9 zx~t}O^JM4!m_KznT&dt?^T!bQN%t?e$@YxgOv8KU3-xL_o!$pIM&nCqorPLKn2~z) ziF@l`%?E|q+QD+mV1&>O-#iGzB04_YsC!?}2k_dyTxMkty=o3)YJk?K9ul zLYr|epf!AA3@XK&IDrK{2<^97&T>jUj-_s|@@5$i$RvE8?Ad$QJBxO=TNXZ`bRpHNk}GbHvc-k< zWN>|5td<=5zMzY*kfYlUztX!PW@l3mnJ6_+8`1K2P_G`Zqjy=(n9+wKzrEy9j|e(@ zHgi&<#LiWM0ns^qowwkZLX53?%DC_Bh6{M{jp-Ol@|dbCJ%b`^MED|xuWy!C_|rtG zmca}0P-fq}<^J-7)BYkT3o+3_!)d5b6#=j6*tz(y6`?I)@x<=%!!WgmD)YBA@yTy% z;ch}uUXVBW{qQ`GHo}IV$MV6-y#y0qCX)T#V`z`pH3tQke4ZiqvFj}b9BB~NYOWY;EYE}x$(xYuQ0Jvj;y}BRVX*$ zFKwaOLMCGsn+-UtxtMW&9{K$V2Z)^={zTJkanoW8VS zt5>P5uKlUc4A_L%mE43BH_c8xMT>9fEY^sG4r!6Arz|)6&v^UQY~*g25w_N&$qUAf zPLa%dk_SU~GfGa>>&00u>X=B*oG2#xiMLsItF#0S=D0Flt#-b&-Mc|F6GfTnsbfW> zMPkdteUFl#sh5-52Dv7WBx;vfID2D0)BAlPV&lc*>Q1QhYaBL^81lb@w)*`dW%Bi~ zl(e_oXPO5i=_$6(GT#O;oMwd@Fmur{{nkFb#ByAu9H*}4+emuWK(gJz0_15k64!j+ zy7KfS@O(gccjo11EvG1>ndrA_P2#>JOkYH0Q(tz=t5qGJj@ylFfsTWzhz!coK#d(T zI2fu_Bp7@$MJqwZ&^a-+ZFT#Ix>;$z_fCoSz_)X~ANH&{%S0iQ!xMZ?Y5zLlr10Gq z+)?q3?82olHW3)OjM+G5_DAxmm>sE&3>L{oXfBtq^uYzoz-6Xz(5LE?9Kn{tzCc8yo{K%2pHtBtnZr42ID7<=r0`ZesMB<8|O-z zV(hXW+)So7(|vvS_;lyaygsHA>JYqdHZ}3iuRIGKOY7 zx6~GIPO1AfAb%oS)$3tXiGe7>UB!e#P>xXKa1!Lg0T<5=zBVx;xQ`_Oz4nf!sGGWS zaf}uSpEt1|`aOo)@OX+0;#^?u>QOx(Es8YXe7{>Y3C@1Ww1-F`y6te5Izz2*_#zPD z!wL3R9)%>{eH&JfS6Mu3t0w++xJ_8w$#@dgXGFmd+~l}V-!3<=YI zg_c#u_Fo}SEVzFHA=*%1t!<gBU2l`5|0?CLo`@83TG*6OF4g} z)=IBejz+D1bQpzJCJdro@a^j+7?n3ZCuA z(`OO&XP}}6lmP>01gp#P0-M+Gg6g{&7UGl7pYq|e!fFFpj}uKkKHbZTnQ}Wo{5^*B zrH$4aPb41DRSZ9MAGDYBmZ}Wl8-(?o}q_aCW@6^USyBBdD&mEQGKzH`1o(ALZ*<4rH z_;^33>Wqu&vTubIy?!ya#&&&iIxWQEyD3BS-L)A2Qyiby!Y)X3!Mx4D!5rO!FiBi;XBd)rdZV`;MF?T_IW{8_YBV^0@?)Nn83e3GA$OjzzXE z`|Wv0EFY`4=GcCTON`dBNTjMiwj%)ByWB1Rri8)>wB(ZgMIBHX@7Mr%BZO$L`A zDQc=G2n(%2J7j;>R_MgxHMywvwgM5Sv&m&~^S-6v!N^*EiOB4FXKQ;&U6wl_rg#qR zf^2QuxhqCJI4scIAt$Mu`VeMMT>~&?oKxoIPOvO8E?Ks#aO35gmD&cEw`w1t{uZH4 z&b)%qpd7hVHeW6M@4L1uNqbe>+}gf=@vH+Lfs_U;aFvy_0jqfsbqnEyjtM{!nw2|LL|O9 zbe=QaBId;Uygu!qep)RCBK#dx9OJnuQ^J=#(us(2om_k~ATXdRD6@SEVxQ^e)nlPG}RZ{mUbs<}Zr6V+HXP-ZVcpFt{7svUKz|HufYr z9#JoVw;Vqc;{{(OVqKUHYY)O62NBcOM9PufXgu!~F&v|QL^Ihg-ZvxCcWz#tEh5C{ zWnMu%*3vk$XK-_BZrGBeGFa>o1WLpqISZ_qd?Xx5cP(WIcP#V@Ru~&cN=cTx$|$B^ zgB5te^k#$7Z`^l<%3k(BbKQ{rP;*gG1v~)!{{4NT4qeDZJf3$Q_4G^{5_C5qxTf{m zwMT_GHQK)kgU@Mr^PT&Bqx*Z9iKi%Endn-> zj1}oaGcY1jL843Q7yyR4WyRsOmvf4X!Ei3)8-B@pVocB>Us5}4_t*NW3f2_ph?n{I zZB2TI)ZI)8skn z{`l2Js+9d+DtE=D}J2EnXP`+>?jtu_u8#^jG4k)VEq#1###K>lcALPNU z>2>zGa$4`k1n5$1R>$gA=j?6VJ}uu=@Ah-EmYN-hg_1zQ>t@w5qRM-ja|4V4as^d% zy;3ZqJA|X;>yzl95~^!F^?}va%m`%#Pfr*vXDym0@aM%&#P%HEK48_Zn~Dv#i_=1+K~Z z<#ivZ+NHy30wO(;tJREKC_(EMrq*isAl3M7c`+bjB9>qTX?X1>x1rRSIerrK(%#ycVU)^+;Btf$% zUXuUU=F-ktN5|{hV1!jwq?vM3tjZXR0mP8T#sjW+Zit%_!Bfj6H!(O-=dqeSF(yQ% z3@?Lu{iA3Rkl>|=`<1;{?v&f;Y!~JFyj+Lq3>Pw7E)Bxwy1vCCXk8k~3w!)v%CfjS8tBj`$1e{;EwQ};cNJ)|(!eJ< zG~L~-&;Q2Zg+B0LzSZs1aWYpPt@G89R#mbsmoc z2R_1_s}N(W%z!-6pVU-UUrYbxv<~((5UB)k)w`uf#?KOzXcMv5Zc&N_KnB=GxmZsG zo^ol0X2yTDUc8p;K~3N38nEsZv2L_~B%4Q8)}v~wwG0A+Z&#lC8Vj5as8)NsvrF~n z!i*!lyg39R0|CZ0<@h+A>|SKnBPF2q5)+;)B8ZPp(n4@+*CltT)tWw2(?ste+yJjQ zU2xxjf6Zugldd+IR4sN>`e-cY4(3)*f8{qIxrkw7^DTNv9d62~@ql!{>fRkot!Gel zas(y#7zLX^aCtDVH?-xo<4s|&PRy=iD^4`y(qP`3g(}Kk@WD*c-PzpmcbU+Tr)Y1Q zE@!xmmDMpOdq=1KD&p{#44J9w3^Q*B;wU7X(%M2r$3wX%YouJU&bM`hOt}Nb6m*+f zdt=X0=}GHyIHQ z*F2j$dyaPEr%XjvyXul!JmJWyMLHQBo46fiE_eFUf{_4I@fH`L(1*7*XFz0*y7uAr zb=kbTzAs(kh@G$l?Va8%-&R%-+)M z_7uPBRFqkwjk-x>y~jIm=acp+73>m(Z#c3mnPLs} zBOy3assX|}B%gMyIN|$SUy}=(y>$?j&i!ePOo;DImpWVsa<@S$k942lr%sQg+qONt z;PXE$GdXQBmD2{-WFCzQV_({8v62>58yQm8o?-Wcom1)EY+$LZ8%6l&!hf?Kl8j7N zfQ+O$IlhXq(yn|RDV4Nj|A0EV9%^I!lB~mNYsJ2Ed<4C-$`VV$Wn)L$H} z@=<56Z+4d`&qH792AA#@Q<3aWpY#-&+-eg3DZ7@x`*k)hA$tA$$z+owjzG{}urZ5E zIFWx44C-rVvt`xchx&otnd0RiT^GXG8QkAKt2B#2FoamC=~sWaLgeCi!$f9|gEz^7 zTy-nE--zOw6-A_G=;bhkPs04{d_0eI<=C-@rdJubKYgz}IkMrg{S~q$-5Za5 zey&RTp=HQHA4X!aH5rO?eE8T26(_x&vS3s~tks>0s#Nk=N|tPW5~m1$j_-WhK!6Lm z;Pxk11H6g$I6m>iwoI63(^M;gr51{;m@^?&F6WdZnD}eFpwaLR=?6q$V&S6O!c<)~ z1&vRofUw!~W-xv9g~5a{TjTI_=vK{{dV@DpW3L?kca-OT`JN;HdG$uFij|+DJJHi0 zAtAwh%#p);L@4D2@c?HFU*%f(VYo^w!tuU=vpWZ#hWVwBxF1dlk|(@2%66=>Bu3)g zcFcc$_&2ClOgg{%=PE#~_1k0LeXv5?Shtt_8Wr*?GU*7Uh*1$6&n7`pErGjb~Fc6ev-#?SX zH**4$^=e7P{<~M$Uo-X|Xn5Bipe-Z&aIw(v(Y@}X&v6gCIWHMJrUhXj&O|w!EZ^<_ z?mw8_G{$ZCR1MrWo_k*R7ztex-<)0JLPcfo{Xbh|Xp`A*3mWX1s9F=?3MKmlIpPp@ zmh~sIIt&^g>*}~&locJsJmSO$U2VP;?StAo+g+`+nOZVsS3K9Z5fDE2bQL|??yq*= zjZ>PO;y!cp<6*0L)QgC``D>p4_Y*{k3Dyn1s#k`b6lS=l)HKq}b1%4nuEFjd8#Ez0iL^n=BIgwaVxNvkj^X}gl&-|jbm zDwH;Fl$G&jMxvwlqc@x&Fd?w2{|%I~4iSB=_2!_w_ts$b5!P9Evnpc=Y4U~P(PT?q zJ_Rr5Q%7`F`XX*~;A;P|nwhwWi1%M+p#O2p_Oew3E8=^_+Hi4fm7iXlR=kgSbzkQ} zB8ADxjx5=B&&f(9gC~v$o>cte@+K-h^L#`sskNz6Qz)?|6-%>7U;bUQ{~Y8SINP%! z>$_M;2;1xfu+m%1HY~bum?2aMD0%L>OJ0H0b9!V!0e^z>`bm-j$Jty=RGR;GN28$A zyClT^R@mr}AL0rO8T{$28@+BLt~Ua>bCwdxc4ut+*B_VWI6Edqx-l7wk4(24AYLM| z76Jd&QN%)5f6t|b?Ns!5n{dAoTG)+2&_CO--FOI3deLiTtj(-q>++6F=E>;eTGj^) z6Y%yUQ#C!wt$8pb>AMa@r=z)N=#6B6cTPhCw#8`2Wgv&OV`rPm50wt-@jOYH#ZkZ3#_6k&19(@Y1+UO2+mfTmK1ockFO0cy!cYNd$GG` zjG)yJ`;)5+s9aS1{XIPJE-Q4E{@V-E;H(LIw4VO4oB+;r(~5<(^2QfRaceRXj?4fj zcvtqTx?5CKo_U3RFSh7FT*LyiK9y4CqO^9EnrTixVNzG({9L1tY`z5kB<=#uEuYctg{U-~4_U>Wz>tK* zo1EL$j-eF=F}V=p{D<#D1ZWyB-sE-8-O1JfW_`w$T?v;?ZIpm!+C*nI~gTWS3(=8 zPG(@4HXjfn1ZxgUKYvDj+0vwkqGG-4+6jxG&N|$Hq)tGmZz#B6 z*CXIc5+MH|4<}o1m+J~5$~njQd{fas*qx3zv!!k2P6ww8E1= zBA7?T-@aqc(p#QuRoi30ES8Egrgs{_IMx_Ot_oE@(W_(=sYZk~WA0tbmzTw$S0&f|1;9~ze3m6<=4S8xRUGr0Ghx@x_QOknhby2j~B2L!TjP49>tW8c-l_3j(JD>hN9a!eoUs$#ZUm_DX$E?X6QxT!5& z#1xWXK;|PLu5SoiLyH}9B#mm6FDI?qBiMR&8 z!R)mnt6v%V51jO3`Rp})iOtfK4T-@$dEw&XP|9B7zR~ALloePfUapA%;Z@-)J?gBaqIs{3G$-*Xcsuxydk~IphEB_4Mm;)Dk-mGiC)I z-NL}Na~cwMsPq!BG7$uy9X@2~}YnYRpI2 zNpQ^4JHUx)RlchVlO5@LV*@7hes8rgf!)i;-O2ma9my#eoo_?Vp6s#uu($1)+n%@V z)Z>i{)f=efCgbX)+wa%Xx7FL~%(9jZ2VRS4Ki$f>gZBuTw?xmkT@8tnkMQMW}_NOeg{y-@HHT6f%kCW}dbRg=0Z0ezvHh}w(y;; z7Q)r~C#H%$5CEFN(6*+^r0rB33;zO2d*H=LQ5u~WjM*A7{1XUR&w+v0?9QCfifbqwlKPlMGI#+ujR8%WD=#$}Q;RCdT6Q5-KG)lVL1 zO8svTo_K5e$#{ZhEcbjxA$Pb7Z4=M|;Nwx?e9KU~v&ldwbX z!2V24anoucf5P%ji2aRPC$NZVpU)N0Z|vO{v5WA3k8 zX-EdjrY~ur#f~iH1;n%}E@QlBf_HD$jn(pOOPy4mX-fq(IDg`Pf)PX6^AbAC$O}|m zO*34-B9P?dJvcvFI6#$B-hfvDjFX3gzh*03^vl z0(Zr_vFe9sJ4uiAWGP)^JaFy8}25miZIjbE%--^eh(IEVme09c* zI4nD0-L-9I@PR9nZ*X#z>oat}PXXd8 zuRBV!DURePBi=>iew6fGX?0!zf&*FCl6%~RA!(g7zT)Q>NKn0V_T?WrZf#pmx39WL z6eigLOf*BA2p@LWsCY}#(qYC-s%btOw(GeOWQ-d*?5#l70>Z&}430v<_~crKg^eY4 zrjqhf><9JncUPD+440d1>}bH;QjXirN61 zg;JJ8y6FNf?`TwE>b5jOD)v+K{|+2FPtA`RPj2c3y@C+;SP2}(P86DgYblLZu*e~l zyB2+UjdLlv_ZQ-^b~$X+j~N{*cwh>&bLg530;i|lvk&I5y~{( zkNx7ap8dU$Fv_x2nM97+{aWfVa9Ws+XL}#6Et|30+|HEa>w&OK8P&m zV0U!v-(}i_u2YN&G;jZ z>0&p{s85hQLDlzdjV9LE@_sU~^YiN@f+uI(6^I8DS_7u4F?h>{C8Tbta}^5^PH+^9 zf9y;*$tH%vpXaC}ove0Wd9ABtl`Jt)Sp@R(BJGB`Dv-}&DlRb?pgr}xw+*+-x6$k? zRmS}KG7Z7k975L1_wRNqhOMRh(@Z+om!Y8WoV%fj1mSNC4~&v82`ujdr+z&M&GqXr z`vvBu7M`Saj@!gJF7`CNn=+iY`)OnHaK+5*ohADO`>|=o_#ch*b0I3kT|PfnR$<`VP`}m{S-E#PQBR>P&4Qwz%>EM(_T8;# zP$)Vf$wRhZThlc{Z$|1D^KwL~4(5zk$FzzQ1=3*cw8^SWrXa01s}Hy2;3*LC zpPH7vVPMt`wGH?_(mKkF^X5a_iThM*TV}$Jqnu8q_kX}w&L39(hMa=R>;F%XlT(k_ z@s1q(%`L+-yyduZEKQGGd+?YcsiezC;?ngmT~7%1@p*Tw1)K!U(1!NxVB{m|l9pOe zob;$a^TfP1J}D%g5?G#(8w1>{bhzl^Q~7E*U?`PgdER7N zYy;Li($b<%(3GJQ6Voxf3%etKuaElYUx%tod+(!!{qwsO{Po?Y67t8%7^^uu{qsz1 z4Xsur_2$LH2gfS(@}7tmK}MMM2-k$k=>OsZ=nAbr@^7f%hZ50~&Koo62y8$#x!iX2 zLI#U)ah?>jU@5}cTyh6SJGz#Sgeaop<{NqoJWtnT7#N)t4(xE#+^M@?AI0ak2;Btn zRQnHd1n-ZAr8lP~<@LQ;{mzMOwuNojoGswaEsQT`X1zox2VC$TC;`975p9*lK<-!v`e| zz*v@8ec~bFyEX15>kQFmLwF+6{<71fFjk=HiOu_dD(M{TQxuwY$*vsqoIXEy++%M< z5U%T$=C*8wy_N~&f0-)v@m;hGYH?)5&FBMn3?o`n+284yq2D+fA&df~FYp1(^mtaNKxQR8gg zkb`>-l?DQyMM;(X$yzc$1v76y6ZQyj1h~Pv(@Om_)3pmBCD!w9Dmpgw{6E*It?|ItyEg=XR00>_z^o_YD7P82n&$MK^)*|+D zamBicCuBevL;u?5x&CAvBnmWMLezMq5oPp=P?yS}ReOi zskXY+Lj61A4GGe7{~u3S@BDJOY7AD4&mt==S57UEb9Hi$+V5NQPu@<9?7!Y4iNFyI z@WHFFY~BdC)j@Z2H|3QR`?NfAV zIda!4PwF05G&yp(CaZbvb~HN{^LD1hDO$a=A~xZ1MVT<8ZWVsEE&I^e+`jRuD&o+E z-}giOw5QI1HkQ|cdG!J%`pSN0x&pFdF4e{8Ys0R|b&^R@=Jags8b ze4({ePz@z&9_HNdf&$a})mU5(K{C&#bbB}cA1yY53|GqoZfyobnW)YmhU3-U*hAJf zP3qsutPvV{3;3F0QJTCd=QvV+bR~D7lGl^OuQ~lBrz}rpUPB;twx_Dv_DEhK?-pKG zL)D_jTo~pKknSjU*U4}mG1#LUzvz7UnzlBXwG@P|o8#ZmZNUp)$TwIMZO}37cO*KU z&i(%?<>{Xt!Jpj=>jj&O{e}@LhSp*&LtyUV+Kg?$7AYOf4IwenfM88x0J@y3+ib2_ z-v2{M_-gB3J>#Uy%vw z^Ea1?+L|+I)fpKenM)$}lUCQ!n@H<>ys#L3EQ6Z^eP)c`io-|46c`*n6OWArh`dy_ z;t#px_C*rTfNdm44Yd{;_#@H9T`Jv|sEx~IXl}hwVux~Qn%%ny%srlpj!?;@;EqMoWrjMZIx!PiZKOgi;{r;T+L)W{=O> z^{(aQait|8qX(NhxH|Xbarv8>5m~Pm?c-8)tgMznNOUBXI+2th(^8%8Vqs)L?+lqf zlM6svy8bMBRK+v1O%CB4R-wu6=tYW#fwq9&J2^7%5LL$fi&S!Y<53MkQgHYCZM|78 z=~P||npGxq;W5w|h*oJ5KYJ;Y&D#lyst)$s8va@bAF)6c;UtA^K?j?V@IRl5 zTvDlwZn-8aw@H$mOUmaH9xdja><=>#`9fuu>@EsAfv}_JyGg}QqmxxtBFW`>KQVry z3v0IyWzI&Dd`6Pu`gAU9mhugLZRen9-QYxSm&8DVr|BWZ;N5jdNk%+zlN^68O+lqA zI`sM%nbrWSBhmhOvL@G0qOozAb4@&`J(lN`JN9<7iOdKkD0=G(L|d!;z>%X?F?QOgc1mq z#kOSmC${u>^~4mFS&1I7YE$+57*zj$);RO^AWKZf$lV1(epAJE14V=M>4P=QB*u}P zO8cQ`J#+ET`IKKALkxNAgDTpMtY}2~15Bq6TyaH(pJJ&da)eN&FI7R#gQRohF56TL zbbIWfHq+^CUh<@FL~TQa1eU_VNDLw6c*4^+R56+l6!#WtpsVQua>C_EP9fq6zisao zziK9sMy)=SMR>gFQt4Y1;mYs4X&(%Jof!?gtL|Y<{`3-8+9GK>KY!XuL1A%n(XhGv zkYvn6(I$)>WrYt#9&-Wmy?WNL*a{^n&=UNd+%gCc^a?{h7-MRxCH{<(YxYd4%7Cma zh7cKb&{>rOshSDZoVSK^oW*=e=Vb@porq5tR5U%7G&u>_28YKK){>8-aJc{Rh5#BV z$VGezMK2kka$p=LqW&z_r3QFNy?QUu7n_={K+i2jtkq^S$QUt%afH(0!V`{*NM6GT z`o2JLldJn-@~1b=u*q{KvRZJ6qcpj>?2+r{2NCj>SjE}kW+_WMLllj~tuAsB70 zt|bI->?5jj+V~2veK2`FLFuLwBxB`D(~kW>{Om`v)y;PI7`|w^^!{=$5XbXH!us@u zf+w4CbJ*{LhucQM4iIc(@9jyU7R{GgyJ`87+?*ggS#_L7T6L2b+JfH_0?_tiGC`Am zN@7YqUxAZSmfwzu*E_w#*{NVMuqx)1vtI>%J<|U<6UrOz9+S_rj z)%uhu(NiujkVV2&O0x0^!% zUqWT3JmCI2)774z)*mgA@``Qr#1mT|tv4?z;sFQ5%vN$^Zwi}y(cN?5a&3eO{^7r| zTG+Lo8QKuHLJk;tV5yVw6XDrS*VS~y|0B{5y)86mrmo`aufGmZ@Oc0`9N)l$Q^Asq zNMw9%@_#VqRv1n@{S_@o_37S}ui;#q9OOdE%^rmsSVYm)C9}@1FB#N5h#K;G`nAy< z?jQ2K_^hlU6S971q40u@t?0lB0K9EO!UPIM6A}}jxIb)tqX_+1^RuA;b$MOd9>Y7H zqZaOjC}H};F*gxnao$AFtcK_{yy8VY2o~~lE4hEIQe+Y0dpayj@+iZ6gZ)6*W2exG z>NFr2w6nD7+8vkC$q@35BoE-n|37{F_O_sL6^S~|12P6<#e@|ScKundy^cZ^t5k8P zamXP6NQ9ykfRar}C(b!j8-ExJuVfyAVT3l8nevgtSg z1%~3tAG-n}ZmKWRFZn5NEt9hhc#<S;D1q*X{}AsuSi zF}TLQs*-kkZ9dsrzqJVTZFt_kg&hMpPwlUF?UQ&-ij>Wg^(Y2LNhBti^w@@gzw;3i zs+NPtDCY{PY7dSIHwV*_5Z@;yscUJys&k&RkBQms!M)xOBE`~%RBQgrOuVz68t-L3 z{9}?G6K$iGPg=L-he8)q%q>@xbP4%EQo2CK!8kjq3?VgP)}_i%>4g0szsCuEy+||t z4qHUrbfTgZvR|&f{aKtO$2*bq$GRvHLTX=~&1hf_ujK5kW*E!oxbMdWq z5kQz!oN(z#rlVTTVebowTe8ha9m$H581kN$naiK)p1U1iM~{`cf>D7u@+CxrU8cKzzExp5rx_EADS4;DQY?u zDhKSSzWECPFv7+B8}q*iHwD)HU+02obo}kr0ASIZzrhRvOmM&y|Es-CGTI-~{QWm9 zyZ^@szaE-T`u6{acnTX*kF}v_8O7!rZkX2n*jn$Dr5?Iu$6t+$YRpyqp++$3ufZQQ zjxa{2X7;Sk8-h2Rhu?>VjWo+-*U**twOyQ%eP|1uml2udKM+1-k?<}j1KI5w5Up=k zm}otT!`xHwb_&zX4RUL6!bq6`na2u+bZ!@2zV{l7QPlkk6$((bd@|3QG9Uj)!q=;u!#-PWP5vRXTP*|J7n1iGPwWs|#B0f-df~hU z`UWtP%iobj+LLW2ucJwDJ#?$E?}3OtFC702N|&+%1qOP8*1rohQyzgtO2EGNsLHeIxupg%|UKAUs zYDpXp)TSgg`iY0Q9_BfQ@x%%>P&BhQr}3i;jW}M(v5Bp4h%wUtarf|FK6vBs^fdW= zl}_Z$jfmWx)KHz7nZ<2;xRCKkTf_KArb2gcLb|&KTJUZGz{c4sV zOQx9191lWHW?rwCBUjMup1>4$nw4(u-Rcb4gai=YVO zu$Y_^;qc{AUo0&t~t*;jSd+ zGFwelOgEIUnZp90ypxDu1;x3{WaBm^v=&r{!`TntBDRXUz>gCJGtM*!(8NwV5#h@& zbx&(=K}0Q?{QYj|KQdGo7}~s>-aG(emzhrLkEbM2Jx{+Ea~v3;D|06vBa<|sMw5S9 z$i{A-wvE$#=$BQpB@6v7zE!#0?3_M9>txa`x8>$-C>+Z;PF!(SMPAAEoO?G?UGA38 zLIeR=EhZGRs#3shriHcQL~4(G;n=dLY&;_vBIS$yLgIRDH8dAZ_PPRmzNI#? z#TvlVV9Xi&MgKNSEJe6hlM8L6=Ey$U=iHCT_hV1sCD$|0DTxM3+-U`Ui-}~P$8QJc z)X=OvPk}7Nd`CaOFVQE_Kc|c0GX7BXQXq_;^AYKA-VLUOzlYCJuRCl>I15+Y*o1=R zj})kA8g>02ym~CESqvT^rao{KpV-Tbgm1j?~U{Jk&S9=S3d?(<&z>}KOvvJ~P z5fZf3YSWS3=&-ea*1Xil4v$*%dFqRElgoI@nTRbaJCWO3LmJB1mq_xM_AU}dUq2e; zH0)0Jn0WpJv9i{k-UEIpekvrDJbQW1g3zxdt-m;t`^SQ6Wlu9w?C$V4;%{#jjkC-T zizrAZS({s6_xh$!j^Pw*d-cR7jk~nR}$mfd5KGZ8C zrMj&}E8}hs>d3vE$>d)EQcG`97U1e7bqUWD7wHAm+{;3@9(o5Fwx-S~o%pLDD5qPu zY20e=3H-`G2M57iJlI5M?0c*51QM&cZ_UU zimPKS3@3XD=s+qbx-AN_`8`ne97tGLvfyPN=y>(>fR7h-vrkrs=y>*&eO~=~Fm=HD zMFZc3`c8|*!D~^W@vYgD;J91F4it|Z=P1!y`X~CXi-oN&3(oysH@k~x4-+^>`V%pG zhm9FVvJ%wyNf-+^H)DH>W&Upv4R1BLn`8dK^|lulJ&bu02^8^bKh&W6-ta*`%A#gC zu{}=Np|OkIAdM5PC#3PFR1BM|!SaD_;--XAZ`q%-H6wmOgAbH_O#Vq&4z*EcIhdu# zmy`Q^;H<=WWzHukBYc_YYd|H`#k$(%Jj}(+9|tNI>t~Ll6J9`#n`??D8CEf!Tori= zX>it>0&I#tzQAW@CKus;i#HR%mJV(|B*ZPMu73wg%v$eC5nQ*hJibHMEm0AVDPwI> ziU>WT7LqGi!dhv1>;>Dl2+jp*rr@uvptyI-+&7~2O1fY z2OQ5uH$Tz~_)E$lbf^1s^6*A6P^Kz6;35TLqCN+-F|5^ZeVp_1^-lI6j$H|0ZalaS zy=V|rOizA!F~;HCLzT>Vswv6UX7vJfD}i{(Y_XV5@DT<`cfBip@^#dDby?^@?#2A| zZf@jLbnHjSaTFa$$oBFcb}^(}esUnDFbfM?HU~nRiHPu|5zM~0t@e!&euh?Mqi>qB zd>o#$Zt}4^N|$GmkvP2!=Q(D8D1IXQLI)|l#`zyVkwGj&rG;{*Kf7KolH8(>yA*G8 z?>Z7~$5*fXVb~CFtrE8qveD~y zuo$Ac0VUH-?Jb7A3l~9f+$y7hO3bnvnJ7j)C*!4mkMk}DwC+dv+;Pa<;b1PM<=j0z zKX^nTYsG&7oE0B^ImVtbAe(g9ROfV}LD}SqHzNZ-%ZEN{Vn!)Aw`mssmB|qe+5I*? zot)ZnMY6rsR(9?ub--F|AiWBQtCVeMjlI;V54_N0Ve&pP@#{KuJjr*RZ(puPeys4d z-vHAU|9CJb3HdTN#k5!#xv|GIYnB)X5cEcbek_~!NaU^L3)={RA5#z7A74jKz+7it zYVkXn{qTMybNv=?3CLxX&x$;oW+rhBJz{0bWIhsH6!q( zbjAZ@!%5iAtvSHglw=@EydZ2wxHkn)U$TK7ZI>Mop z3v(PN{iY2Ub}B0+`n_!Xx(6N!^S$%jZ-rI{;UTHt3T)#t(?NUc?+FZlh?XBgvent8 zZD7KD3Wf~KwPjGrTbAU#>t z@YUsh5YD;Ne$cGwLS**3>64NAYpIuGLdw*|k3(@4PD-ShA6ti@?3XJ;IqKKLDdQ;@ zLvC+}V5ylN5KP(@pQLp#Y3R21(Jf1zX8~PL(@&h3{=iPcQuFIcPLdf3gdhWS}J)zW5yU zVqy`%UVH~%h_8@+_#oasZMaCWX0yG(Rr3a=;c}&vf6i#J~wgf5J4R+U^GSAf@oSCT$gzyxTBl%gguS$}xeP z)WqG0MdL@Yik5EaTg`bXzERF9)$)hp47TGwLVLwE=GG+Z5-4 z*>gEQkx0sgSisSF|5bmKE&)RQ+9BQaHI>0kGkP!k*1pDlZruw-8fT zWLy5P_NytX#0{R2W}{b%E}mbWs(%VZZaPt>w9J?cQ|8uoxfy-)R`Du4aC#S?2L^F7 zqqk3HWZ=|6Z&tmA8Xk@BPF7o*j)#)_Hg2hf%62(h`zuW^*k!)VLalnBMDI{%R4~^) zFEvE48~Z5SvADZyKC545rjJ^3__y465Q*LimwEfLjbDs-j5A26QL@jwDH9TEAyhZN zzxS2?2M*bNS{!Q+F9+aIJ#v<)Zrb|kyxy#|7)?X+gR*{wcK7N;Ip#YCi=gJy?j4_g z)Ee(tg;2wx0fj@e$p+Mr{v87qv6#3;+PclO@zOTl>ZWKuoek;b-p_utd`GE}Ms!ITyCQHNva&oiAU&6L{)BjOg$k0o;RjGe<<+eZz zz$%RbydO>IBvLBKwsLf)tmDM9N`7VIgxH@nWHHO_nsTwahmmnb-t0}@Wn9_bI6og3 zf3N5dz8lC~jVvuJdR=3En>#2$LXbpwRO8WG*|koDnSbDHjr3L{ww&rd*`rN!K)iSww4%JRnhLJTyu4w$<;|S~xR0yDq74arjB#>n zMRSlUCL?UU0>`)@iTJah04>(RVlO`+Dy}@jX@EpmgO9gg!tXV1?2@4Y_izNYZ3ujL5hhB=mRF3L@CRRrlG5U2kQSmKd%}Nt9dx7>G z&sd{z=SWoJ%Zt5%z8DQfB`zP590xxu?V9-ga$4#3C|wOF0?~#~72bz-r$us+MjtY| zXd5`T`ySks&>)Mh9Ic`>(k7Rh@Zl`Ot1&9HZZ`)p7~6}f)iura>#0lQHBMONLql}0 zt0C#-w861^5-tcxvD@t+VmCShOuGf!1vq>5x`3CWbBT);;z`DnX?33*Ic>!u^eq{1 z)}rNatcFdEo`IIY^pUvc-lVej_trj7>9{)9m0RMevb+(;`DzN5+=GBAepGbb z!wVKdDX5^TrMGGZ{6me)1^LMDLNj*wv$1=PEjXdj2?7H8(TYckd9i%Qf>vvx_HVHwHs9 z6oDEdD8!>j0$0Y9l3Fqd`C0B{c@76EbJWsWZ%^bmnP#!-5?+cj_e^PSL27t{wBI_G zj^I$UHuJVYy5Ax1KsF@bp<=jOo?c z5#2LU8>EyyJ=e`)fBC%5$@GgxaQ}HWqR%!ZMn)af7nc$!Tywq>; zceEz1lrw@|!42Q5K_E?KTO8dxF6^bHOyRJFpzrFbn!NI zkH-W%OPc7Zh z5{RBDy~-LDu9i;bT~%buIA)=9h}h_B%S*lil19+Kaj|S260L=O;>ro>GnSdoSqugR zPb8x@QiSifcwpyvRi`HGP%RLbjw{Ie)X6k8cv z{YPH(GyEPxM26#sWYy$H}CE$#e#W_p*Iip}xdn;Ep1> zv-ZYbTfYcTk*I&e&XebgjWKc#PlnA|Xupvm-F343K4{cv>}zL9X$^c7uApN~5iAkP zr4uiz3g&N_hO_=^-9_7Cp#9Y1%>zGwSf*t@j?Ln`$rI))r%gOQdp3*lK+h}V%K&m$ zb3Ak+JsGLj2jMo0O&~!D5J=F%!XlWgc42b55KNvhELQk!8L;D&2zKj%jTU@y0>8(= zRD>mNvk<`kaX{SUwuEr>Wy9hb`-6+*lOzK#V?rSQl2Y~>&T|oGwwc<8RNR!6W=Cpk z{!H2b#4x-C7HMm(mwQbUhCL77UBoD#6HSP3?sKS`L;{J4wBGB-Vr6uPXkVI4K;ynH zCo^AL-1|e%#5eIp?uo#Or&ucD!a?8CXj&HN4g4Hf#vjxrvjP^ZtH)y%dO<6EnVA)V zj_NP8AVrlGUvKiuzmGChUZh1jI^Xz70LN(P;>Xg2GaIu!QaYZUsy-=xeS18|4blA| zn6{y*7(x5b0q)J47yA5JAz5;jt>zLPe&{+tWI^3R7x2 zs`4w5DNTGHEa;O{4*Rgw)bAxP-;c02YY0E@Iqvm_02lFk#q!Y<&fSq;^s`&BH{%)q zoD=Va72AmlLbY-B$HIorTg_K050-SWKYQd$*&s-V3DmgSUfn3PUsEMkeL-e$vr!U} z2wD?_-a!RwoE1w&k$}Is2?vjc-4(FqNpgLEc5(7-=tyYOXGh^C=TziB)*cH`>zk#o zx~@F)dhPNSx-rRE-Y?InL-|?lNWU&e7cm#TsyAIEtKBiuYwt+?TINxq{^IdnvNyp{ z){ihUd0Xg9IiZ2Sb3!}IY8+w2hnZEkr$4-(PO^MjPGp~{?pUpKBs$63dXQh)%+O}n z7fk#TH(+iP^Mo0AJH)8`X9EoZP|W8hTKTz9#j-vogR&f_Z*O49uwR<|Nh0M3Y2UBy z_}p_ebXg`|@29aWR9`9qc1->^aXG~gO%Yo4)Kt`bE&t&Eilc!@5|Z!O@wX%(k{>H0Co;TLk0d=-S={OO1W_r~!@zYFaDQd?Zdusa0^SqJzLdnPXjAR-#~y1dB;O?{Wng2=y!#a8dAsGYr-Q{4 z$-wq(OGhmIgVgA(j;YzDh}P>kkw_B*+L%4w=euw*A5L9#s+=(^0t7y?&q=ki&W_Z9 zNd_T(5s^1E^WgVb${QPMzRfhD$r5BA*$pduU|McMRRmRWp3~zLUWoEMf2JUr=gsRX z(w9?5ohd<`Gds2U>YOxzfLO$&eDK?s&x^$5OEAEPy=#VP%JPBmwCl>2z_$b% zmiklNsh0#!O>HC}Y(_fesOU0I=V|K*n@2j*c3jNeKbfjA8K8-^A zVvo`W;Og)2nTjl9Twt|`Kh^VyoGXe#{4iFoun zMq&LWa!)M`eKaQQQxybn&U0&%Pm^SObw-2n8z>bxNKN%KyC<+)zS}+;{%FFsJRLLZ z*j^CAeK+lq#Q6}TUH5oV(R8+yqDbl@5>cTfaNSoyxHp{As}P5=*3h%KGKexv{|WrG zS+=z;EzUrqe>&30Y#cmvU*6i0cdv(w-pMDJ36)h`AG{>yI$DeKI;d|HXqBRs=FizW zz$Y0mCuAK4CG$T*Ihw~2zIoOGD9(PLRa8k(ODtUF!*bWNmZNBx;Zadb_8{tbRW(7W zA_9GJ3A_D6GkQK8S&n!ev->BA5uHywsG>Lnyh!PjlWF|2Na^3>;}`CVt{vaqsq>3P zNSItkMOIXtbWT2k+4#DV*HDr0JoG71I}-_r?~HzX>22VeJ_N9{TwAcGQzU7}e1lOR zV;m;7oYOzNJYP&{0p8Z0_Z=@6cn<%9IUtxIR@{aczl`MjS6~*Nz%^;;V-0JTxF_<~ zMPquwsnF=<`tz?Ksv8BXE_O%%;Nn?U%l4RiflE*RldR(i1NSM+GrOkL0iW{awwaLi z$2~n|ai|{GDJ`!b5`d!pw*?YmiFra-!`ne23i;e=`&f^j$|8+HoiDwPma>|2~Q9H4t;_5jlE3sAVqnWZ$!bl`P=#c7hBZ_@9 z@njy~d{gFKGlP03pLjy%!E(nIrA435<60E5Bd5cDm2avSt*CIp&uyusvkrVi!?lmB z8}E}Io!EWS!2F_-1>CSj+ZkmBp3O4iNGjGu5d*od3$C zGnN%s%$gG5K?M;3P_*>*GoP~66!E>xvx5?aV@o=}j6r$+XVHKKD8w}WH;H%jYp(Kq zQu!~k*opj0j+c{3-89k68N(dIhmL4ypK_Gs*4Z{B23DWL8x?Z?eG`=)mpUO* zxPV?lGF;X49i~L@uCJCO)>x56xQ>>##zweOzJ+}J1n63``vp*6WOG=4fm$ErbYtQX?;(}@S91e$hf`cGsl1%D@GN6} zLp+y2AJSCV%=?=2daq~^=&yaRE_$3!OBHW!4bK$Oe1~PF$J&^f1T&P>gbU0yguBqx zU>7&_g_kZRSMnOGTPox~UrYQq0&%O;yElGycbdOD#n{W>sv)jS{tdc%34SEP)_M>- zDQ!Kh8T2tLTPHSSdW^O2y3SMCTDQLke|P1-T&oxD=FMXC#4_p2(Byq}wVE~MawA@h ztv|M6p^%|tiWewN`$@Inux7)jH2b9nMN7nc+zby49>p)c&;C>o{EFMK3^CKx`8S?D zrM38KOvnS5oBM*Dy?~=KrLMEPvwO-C+Ktc`#V67GDmORRhRN=~V1O<5TCfnlbCQmU zCZYlDrMxKuL6Tj7kh@8eZ{+W)D#wMwl9FhrwHlvU=J$>?OiYo}LaMdO1f1@Bzp57g z6E*%GgMxmIU5=fCU|OwRj=6#iJVvgE%FsK%FWD8E8U(6gF4UFC)334UEl^mj{_2)5 zrJNDz`!t)j))YI0t%lxeU!6WWV=^MJVTnfK?{*#185H>3>1F>MZT=R#^L@@fGI1nz z-@Y{|De0)@rDlIoW%=HpZf8;-;^V)k^4iwi9}v^S42Gq&bm2N5J9qFt8<&EjS>3cN z6TFlV@$8Qlnyh?$9}5c$?TI&)epM$5xg8u?)*DVOl?J}j(9rk*B%WYh`T!*R)BZph zpC4v^uujp9FLCp?H^vaInnM#uv}fS>_y6D{BBSV6QPoaZdBZT+{Q@>bGRJ#~oiO=7 kgM_uq-%aF(&CHd*;pg{H*kf-P;9x(XjIwl@q)Fia0gOz2-2eap literal 0 HcmV?d00001 diff --git a/doc/assets/crowdb-overview.svg b/doc/assets/crowdb-overview.svg new file mode 100644 index 000000000..36c9b30eb --- /dev/null +++ b/doc/assets/crowdb-overview.svg @@ -0,0 +1,54 @@ + + +One storage core, independent data modelsIceberg, Dataset, and S3 share three storage layers: distributed metadata on top, chunk storage with chunk and disk management in the middle, and a Paxos cluster underneath. + + + + + + + + + + +One storage core, independent data models +Application semantics stay with their access model; storage work is shared. + +Iceberg +Table · Snapshot · Manifest · File + + +Dataset +Dataset · Snapshot · Split · Sample + + +S3 +Bucket · Object + + + + +Shared storage + +Distributed metadata +Chunk-KV · range-based trees · dynamic partitions + + +Chunk storage + +Chunk management +ChunkDB · placement and lifecycle +Chunk client · replication, EC, and streaming + +Disk management +DiskDB · disk and block allocation +DiskIO · asynchronous I/O over TCP / RDMA + + +Paxos cluster + +Group 0 · System metadata + +Groups 1–X · Replicated KV +CROWDB + diff --git a/doc/assets/crowdb-paxoskv.png b/doc/assets/crowdb-paxoskv.png new file mode 100644 index 0000000000000000000000000000000000000000..49ef001d82fed3c5b00b1b8c57359f231b162c59 GIT binary patch literal 369391 zcmaI6bzB@xur^8pfe>7S1()ywfyEaM?vOxmcXwG_0znstU`q%RT!Op1EfCygfyEbs zEY9UU_xtX5?mg$+`KNy~Q$0Q1RbBl&RTcF~RSxG3#Tyh96dVP4X>}Bo*ET39Xw|RL zpPyLeisC;vXzr2%WOO_8 z()ePfexQtkLW`mxE%C)0a@6kAW~7-4e^~AH`Qhh`F8`VlI|NN0jSelbL|Vo|Yl^T` zuZ$SNQE}qCTr)c#csHHXP})+~;;^)P9H>-67(v+mfg?hlRpO&e2%&_Pgq8eja`}^S ze}p@fdQE&z2+o}T@nr~YduI+Fjnj`!kvm8&a>@Q-Zs{()tmFFIN?3Rbp;B7ahWKj3 zOVqzPo+e#-N{Z;o42fWO(i*z{$n!YtKIX}?e+Z){(aPP)%!jM#%((^<-MTX0z}Nw= z$^m5Nn7IRfYH=pmItZJCa0nq3=msS_1g#^A(B{k_4Bx`32n+w}n9H+nw)JLWFLLvW zD1!;C+2@358H}B)>;_B}K2Bo~u};nKVafv}6)n27MZx2|p0Nm(5J+V800oBhP{Io= zGGo)B?blI}fN-*x0A~N49qO$v3%77$qB#meO!zNYk7%4sSoZ6kEz&mxp9#h0SLb~?0netF{dzR&`?28x(9h~t=U<1s=&$meCKOxcZ5giq1r3@v5 z+en7;k($S1H>CieNm6}UZb6bnE_0^YOuq_Hr$;=wWp8b5erw%+9zi$nUXI46a8o5S zr+vY14q^EfHoVcs1Xp%=nZb@6*W*%^v+3~z;<6@VZ>Lnx4YkJ*9QJn<(88ool)_W? za4zLB)uDf&WCa{#>d_Xc{QBS*#+ut?HfGCmv@AfmWy(AO?sxtDW(6kI)B(d$w3hHSQqJsgpk%+fait7lZE25{gArhvTM2pv@#>G!ZsTQ&cb|JG zl<3W?uSWp9ywc0I74v>+getGYm{Fq0c!-E#-dL>q+>;2PndB{r!frVJSt?6mn8d)B zXaf5J1TUTB8*Vj|3n9wmCrr2~E3+Ha8=(MOs zss#NxL@^7hNVO*;OhtxK(K3VGP1s2pj^kgbGc!DJehWsb-gda_z%K*8@_qL;qdL62 z#I|V81e}6SBtS)b1PWwty!@RuzPVvY>QSfkR$P+z7|JSInI#lOy z+dm7vd@$)XU<1;>7+sjr)ugzack;v9Y)sJn_gV}`GA2eH68ruFAFehoyX0Znd(ZQ{Kn~Yw$eqBJzu>MEMF0VoKP%s%N z#GHYV9Yqw zo_;X6nq+OnTK<5g0JHpX(?5rpa;ZC0GG8qALTNtdA1_vA6AAxRvh5LaC1!b={sgxH zeXO~}gO4MaFb<$h`I!j;4GTF~A#Bp5ANJEtv?kGyy(6t#Pwh?axv_g3_akK92dn(N zZ{cIZR{n0^g($MiPmPF>l_j$q4VEPk<5(6hpF42-#+=aNcP0Ifbo5{00%`CIc-xhf zg$R6Zmp(wi8VvdgGrE;#w#pz4-oY-4oWV;2jI> z-CN8S(J>iR$*E=#qt1=!I4&QRqheRq2nDh;FlflLstfP|tjr9}V_zVKKUex4f<~{A z2SvjVE<$nr%(`xG1dXF;atP^${hcNciC>aMk+oHmB@Y`pIK3^lIOAxW_fc{bl7+L> ze+J5(ook?7ZmKJ8)zUrC>r2ILO@v%CRdhOYg}r~Q@eGIbeJ=48t+z+Fe>dK@JLfl> zF10&Ve*U%3LU!OMq@?ea(Jn`GXF_JAD=vIHkb!OUlQ&BM=1Zbcq{hI@clnc) zrcVd#xsBhK%wvc5tdr*1oBmA*1A2A-;*}>v=4SOha+<6EH$#oH%-nXbqOMOm|@nIUV)ppcrK-2u*_&Ql)|@tbdH=+nsT-a6pj)-2o& zY7Xx9Wq7n8<>(PS+leI^r-z4KlGgNfv!d0POwQmuAh6N!T=d$gmZca%yteowvv6o(bH4>&?wCM5Y?l9gq$s_UFU?e^OLXNsHhOt_PvU| zZk+QoD`rB6W>9?z0&q_V!uODO*aw9%e#6B%bS;s5vKPU`nreyZb2?@4G!G+NnB!Ex z`s)H^I46aD^&H^=#i1xvFSCWtvKe=4{i+zkLU*QvooMx`%m_6@d4~dKa-M;IU{rA0 zbFf;<6KHE1idTe(GZv1!iRIePz{@O zEqC03mZG1fqVpDo;ICB}0L{r_R@uEeXBGV=5-`K&2B4*j))u%ShfGOheoMUUie;&S zX%MA!HaZUO;-&+R8~r382v?Avg>852qE-G)FE?L`bfPd%aV1Jxc!$tjjv{Q`mB)Qy z&kaBYnE4vjyB0i|N2w4W6vp}cX^}m`Q5pD-97V+spZwk2%7S77WK$9NnNIf!wQG%G z8k8()j9x8SxwG-ytSoIrISo+~0XwgOn*nN&r0T}DO5Yk*MNW3KRNph(+kV z7a?WHu6mn$W|h291k+UNmk1m4KGn3z8^*$N?luxfsrs98^8r0MhrEMr*@z_YK7L-q z6u#u-9kX8Rh03h>j{1I^ljd8Z*3@L`UJB(ogUnP|B6bH0kowDd6ri+fr^OY~o^~x5 zqnaVP$E)Tey7`Y9Hg`^@R=mtP<$+Rh`N9rv-oVIDbIGtFD8Hcfuk1{OA}~>-a2^4w zwz9AK{6zOlex|u1OSqhgH+LFhNzH9o&i*iOnjmJYTxQMy6q&Vd*J!5zI@#EjS1*ht zf3+b+7s}>0f1R7r=FGb37r-4wme+<`upr---S?J9)$x~6a!THIwO%axjX_y=@E%%j zi6%{B0lsVMpf-t$VQ4DEAipDamaP49I&*TvWCg;;lTFw|# zO5*WkI2Rr;nX#6>bDV?XXUf)6}yS~2k(nXt+E#?$U@`FhW)=lx0at> z`v8^3d^{MCsgJYGC?;)vzC;ys``$5dN_aTlO4?p6)81>&tNFthapya|xrsVGy+gmy z{aXjSOWVmdxq{O0;zJ7azXed4lUvopWlAaQ>7C4CtEzJ)VR-#RXT5^fVEgr=TTVN2 zsdA;CHL2)}A3kHJQ~NYs71UG!FRIEiHmfcs7;$OiVK72PEYBLFmQZtHxpd87FGde? z*l z=%SLh2mdo;$v*H^A<7{(QUhS#9zj)5{4x0)*qEq57@;8Yq8W3|*|nw>j*LA&eofp< z(ceU7@skqHMK1YH9;oQ%l9wCD2Vgpo;BXY6|8jL308Y-*@)?|*S^AEc1GuT=$HJ`Q82rGI>QUzsJi-YAc10 z+**HK_ryHQd-}zis)$)16%^^B`o~4lN zXy(jQ8VIUC=?(Hej3n%^j6Gin;w~?5Cqk)lsw+g zp_s5f+ipEp#i5K(YC{DOUWp>)s#^Kwl3^jLNo9u=g%O{9ZHo89(n$)w9Odpp zn?M3&67zX16FpId=AdqC@WO})>))Y;-~XWNzT;82{rxS4;u6A?UhXSe9W{ra6^y8l zVso`p{z0Y9YOS1vDO7~kp{6PkdRm~MTd=M}F3!-)fQ^IqS(8MHfdLy2PpLJLKKV?A zaTJA#7?Un4`Kurop-E)6J&V3QOLlbu)WfNN=so6^7bHq_mR7@2!S4Z5 zCGAkHdgL~}2bAYBwISlqj+2a5=zmY30KO!12I#&SD?c~vVtrXyH|mq&4k5wdDI?-g z_%KxAZi`)6+67ek5QSPqbC7gBhb-Xw?LNS`2sco>l~>j4N(4C4N3%m)T`9Qg<8;2G zG;zpBCqnqR02PjtJ+dqf6AN?A$~r8PW?FKSHWG4*y3kQU*LtLxr8eHkdEh{R{B!aF@A2fIv5+JK_@4F;o?dx zvV7~;SP4$$f#+jFqgh(MQ75qlwy^!MT^SX9<=`YVmjn<}n+b_bEEQmU;Nx8|qT%6{ zCQIuJwvFMdf1v-WAn|Q@Yr>49;Q$0HB~e)lga`bpu9yPY6+}jYqhtpYulX0ZE4?+s z_!zcmj=fVP70QNpCV6@3lJfi~gym)t$!3&c@~%bGuB7gMVv@og`ml7p*1TKSxx0^X ztJg6q34jBCmXXnws&63_+g2o@1@hCIZA6>nWUN1+3Ro6;=L}!L0D5-eLrbT*$7xdJ zhF%WMKP>d|kt@k!pVfn^N<|(5jgKJck#+{l_v6^JYig<( zJ+Zs0j*t`O4*=-V;+5Zhe&U?s{cQf`vazDuiuYuathgPa_05yJavqfsox?zDneoaX zny1`zL94v-c1z<|uuP@9zKBJ(c8)Hze{Vb64F9eGQzyZUSnA-2ji_6@)24dXY- z&0oDHFxhgzMIpFYFnu5C=Z!cu+HG6=6mL}Zl4N{11D~O<0e@_xocI;5EqBP&OohpN zcN2a>cO~}MP)f;ne4jbb!Y>J*tWZTsZr#T&{#jXn;tivdN|B2vwiR`?9}V>X95Evo$K5x12)oHjMoF2)VLIars zmg4eu2xGNBJZpHS?DnO)jArs}+-ySuht2YDhDXQ%PlW4Ehsiy1*i$ARHl9&Jb^X>C7 zpY1`&0`1bmf@hRfzKk*}x&wbHX5%pD^uD0%bfC3N{ zWsv=aSAyBMK!m_Ab#G_L;BtRAlpMhg;xl46>%1O~L)aPsJ4&ycFB#9<1JS8y0b z48P*FV4x@%c)uzsphBi7CXHu?T^(h(KEtI%%Bf{^;t6euluarnpZ#)q%VM~aBDqIu zL=iju%Xy@|HZbO8EJPue_(UHel-)5);MUm1@|DlKDzki`m(@l@QcLtjRD4J}qqRj6 z^=#&_Op#+H3hN4{1nD2e)+Fd*Wv#|fnd@Wv7b?X0UaQ6J-_gpbgy(90|Kf0Pw-b=T znBJ=Ush3qHWcbC+w^SudOS~Z&rtXN7+-R_?_fEKCZneL>v?}+8D{()T-AK!CwE@@| z^G`WD%Xfq5!+`|$p;Y?!#sbV>PktrTQ2(N?Ij&r&h^mZZ z^7jF%&g{pE?6C|>dy6ydjrKz~^*O~{X?q$A*X#_3r-I@A3x$@)gh87V%fJ z?Ik=ek#-L`>mO@Wp1Xpl%=Xyb?Uu!gZgBPyBqov7xg2#rR!m-u$2K_`G-O)NKb~~N zJVe`-)qVLAGxJNl&I?cvQ#ejCunzSb1Ulb-(GO{I>O~K1^8n8^JHCiwS3R$V>VbHo(M=eV?XzQ4hM&F0av8UxBqoC@Pj5 z(bX|`!MUX(8b#O|x=K4#?SVA*=LM*9yafwfypC$O?P8aGyOqZP`LifNhh&n!xTq~H zNfvc^6?75B^B>VO1TBv}Rq-XY6Bj7#5&qhIl==Q2`t?fFV+*6)dw!PvJfGbaXooT}@XzbQxm#%Z~E<0Bw|6#d`r}(E2D~h*5o>FonW3x+3*3 z+oS%6H=cr4*wWl^9ofs0lHt##NjrbkC~nGeaRK{2_UfDrv;HNd)0v;(`tRsHQT$m* zhoYAoBsKrRN+$K&i=HWfeJtC1p*y^KC7hs%seo$eaVgruAjgyN70vUbB$0*s3|i`N z4RNF|xYO?nUoM2C=395Z}g$|Lf&$6~z6Rs9l|twD8=xiIsSWJoqt z$Af!pNt@|sDoj%UQv#h*r_DF^s?=Qi0qaS$ud{Z-y2l(3Eqq-zJK2p|-x79WY2w(CZ&i`@H!>LX@kNHg0TR}$4R^zDkIYyaEIYEO1wQnQ;^7(D4X?IN-Nhi9h7qT2%^}({HM(w z`zAVZlkT6YSL&I>SeIx-uq<+#^)VB(ufArG!b%x@0Dk5R+)zF<9zB*B3)Y&~)pUII z3q+nL5FSBuOu&%mnC%}Qa4uG!zA$(#VgdWYGus@J`uIoaJ!?$1vI0>DIf5}G8@BzA zty#*pj*Z8+cJRBTH)VF>1qsZ-T2mBsp@m>39lmS2?^V}zv4+bY1%k~Y^1(i#f=zQH|FktE{1e{N>hu|rC3mZ8 zSMZzdTL_eg!l4W`(}N_YI(4O1=g?D4Zy~($2(?^UI zva0mzL+{~%Og=n~&xZI+SzB&bd3r|gqLi(OS1?Ax@iU=RUROHF!MQOp5tUUG#6Y+CL% z@+G8Cf-%O&*~x%BemoNT8M?$9Y{SdRN-DdV?+2beLyB}~h|pV(gDUIGw;x|I8ctQr zQ(jImK@%iX%f(`O!aw0fdQLC4Es#167B^4cu+N1NH;?ygd7%GF7qv*20-^8Zu?_q}FaH z%rwY8hv{2sEn@){G67ri*s*(l08a&$6?VwQ1CEM_C)EZeJo3~6>%U~@27QF;{nhKl|o;jbI{A@o^NX8 z3q!W0OEI=+7-?lC-^p|q+pA9>7`3RcXF92l0g|$Qx?2;E;}D$I-m7I5J|vGXS1y2w7X&%M<&4C7 zcnp07Btj?BYXa;)XYvBw07|MgS#ql$j-$D;`hCXL1Ni8nPLRaD+IqC||6L0}bFTV^ zG_qycCZVbDk_eJIp;-P)PuXT$xa1h7uyv%RF>N1wpbjj>{V^LfXW?yQ=a%p-w5Hdn z%bq6fE9+EF&B>R=86r3A>F=+iax$}g*`7FGEjmJ@%~cMC>sySYE7m!0ew(1H_Z*eVb%f>5cmO_Q$C6$tJvrXV z`*9j*8NQn$GzSYg@sf_#c>h{%mwnjRYK>f?S#BDpSY73pDW8Ao#F1w=_-(2QgF{)) zQ6HXMSS*8RKCDG&wXgxbPkDt`s!(FAV$Y_2aMT899%ohTbdr`E(%sF3Eah2}8}lo5 zCcoQzP6ID*74zrLb<7_u0{Q+sJw>XP&X1v17NQ)- z@*o`ZPOj=y%b1T^4&ju z&+8cO@i}w4U$lBE55o|xu-{SrW8QgyRoT~X!;;}%ELp=c$H(-n#+s#PlN?^T9C<_P z7aL32`WK!{k0o#{pPD_E1{v{^(VKeV4Pnk#9>+bWT5NVPgt1ev)a3(pqK7U659vKv zCo|b=2+ow>ijK~v1eUeF(pw$J9>@C`D8&r(lq+rM`o-2Dy>w)rrcC(5|5V})XP_*N zj|E5|_tSfk`0|#gj@8vL$>*Er{Nlfjh?3;$>2`kn@rl*{dh&0xDHHBl8L$$jWlQ>h zi~Xl}|EDjp7pu{aj4-G1V-U#i|4+`yZQ*KhVqnO98tXp>{qMi_rj;}&iP)wfW z$rZ5?Eqlhk7+|*dcf=2XU~8+s=KucZ|4;V@M&@F7E0L}Nl*3RaQ!h~Mn{U?N>PhXf z=k&&$KMO#~)fE{lD!d!K;Z#6bFyM=3Jyw>EH-OuMsMa?lBcQkCl5(NRi71~4Hsh)B z#^ZmO$XoI?Ki~G~&0N!~bfIsk$aLr)ZFLL3azV|@*3+#)J*Ay@smP|ZuF4kx=1hDW z$(4OG9A_|H|NB(xn?-G3Vx#+`<(RP}rXL3FnSKLv-@k(YlNSEdj5cLdUx(no%o2#N zwelcFi1Jt;4b#!x| z7f`AAMqcO;c1~zH#Jn44F)&}X(=n)(+VejR*O-F;vh0V!_}+rv?$}zyt7)Xg{%%c{ z>}LNrP~2v&*jz_l8UFKlBYp!c=wW0zXXk5o(Ga>mdF6@wZ#~X$H4MzwUYF}sf91>1 zZ*2ws5BfEIi!ZrxW~ugj{u(V}Y%xtvfQ7$by`Wk+L)eeB)4h*wEE||gKQyIHzsnz)O53?aH-Pl>noCO_E-z_SIGJTWotA4cTzAoZ^zKU0dKcP@BrHrUjw1za-TA#JDyz5ui zuf30A(Yc%?G{YvN1bBm8b<7vtizoby-O2Jx20>0u=S;fAXF+iH^lIaO%hvm zp0D$`?Imtw19vK(@W6&hH1nq#LQ2`QeZ9!eKvOl7w_GAK48Q7D2eWmali)1sirWz5 zja`x-oL9#(&vwBP&&T@TuhS!*$@t!T*x`JQv-5J)W=|!2CE0lR(Mwdo=!wVoAv!lv zY)u!!=k0u!+S}|Um;9RCx1D;;@1*%o6?Zv*^SCIxNq@pr+Z1f-UEH~fjF@m zttM8T!IHBuc0d9XY4+!_*NUt4-4Rq1Yl@pX{65W-!8(0qbx`gO@Y;JN5^6Yj`iaK8 z;o*YNSv@>un6)~4g7^mp=?9~+8=5;bl(zv#>(y2}(S}|}bXn<(N-M~PQbVt?o|Da9 zL=b0A>vl|QEBUvc7z%MuQl2|X-FhoCkif71$L;-}c^eMUox|OSy3emS>&4eL5#>6~ z(PgPc zN|ji6e{IC#??8&Xu}q2C!%Z+#Iv7NrhQm4x#b=ef5taZarVyr+J1o_O^jfLF{_TJ; z4mvzv+l40wR`o0$6H~DZMx8x6jJ05L*O;TD=sVC7IQb^!(v6-*=8GF7-US@(N<~t7 z89sUE{+^peXewhLtb0!krn}5(Xtp*8<#T+zj)!-er%@vpPY2F9O}e-8b(_2a_FCl| zA;Jig@o19lsp}XePCtp|W|uQFF}F?Y>ye;4UKE<<(@q3813Fh(nvdJ1o#bob%XX>V z35<lQI=Z80Cy+A{A%03NJ2>FIw4=%J^GjByIWVFU$Q%M zVV}!R#u}OIyt75U8n&ODB7r$Kt1hcNSIx4t;>Hmbf4uYVag|s~E4AK;`Dl*6#8PKj z_1vH50*L_ecQ@!Z=qO|b?|aE;S-`^|dn&1|ZHA_3O5`#%DQ$(suTVN&5`-@88hP%VyM36VhlmGs@`_$lxN!6me&Tnr3)?3b9 z&JXSXO7Z%QXPkd++|P>-NV!8ap$21|G=-jvKVq)Eo>)xK<>t8ye?AyaEgbbc(UR+*C>Acy1LDe- zUAbq?_3&&UO-@?H1~J5WvnU_On+<%XTPoIRbh{F19_*pdrD(fj2Rx70hAYTq&cCaj z&0rFJkBH6u_3nLyALn3pQ)GK#_caXUiqy>6X?+`%%tz@ZA*^?M#|5tgot91;_jY5w z<+eW!=(yNab2DySRL-&Q$qoEvw< zSow9X=k&8d1tX?O$1ShEOk6LxQ9oR&7~JIR>o(XP#^chsU>9Y35f0L{36$%PR5>@t z(cIq$%3@5w&KgXX9y#H^_bGjPXvS$C57(``k4mn2yZ4CjT&CkIH(Txo7nlJ?7QyFVJR{zjp$AV??Us1$qo6A5+(J9{gy%!P*|FC%WYA)a0 z-sk3MQ3Q~vAaW;j`>(Tk>wizQ+KK3d#eL{(w%Sn?^SiA7i;N~|^#I&DqtbYf9;Pp} z#kE_VD5z#QaMM}Mtv#T%dq#ay!hEEwi)gwO$)R9kZSv&vyu5ELR|-A5ebGmgOPpp2|5@)Z?P%&iSCoZoFZQ%vwEagPFmwnc=iF*iq z1YzitMma3T@v&+D-i^TVTv=R=l$od_lA2SJN8s-oYoct~9@Iv?$! zLGWtqh`YwK5Ha`1*_<=zy7AE8s(>WUfb+M%_mG0{c=3J=xjDC~9K-J07hH}6ED!sg z@I95ZvsJZ#aI2-yZAV~tpV>RHL8`{2m=A3~&Wa9wCl+ump|+6A`L}1g`Gd`f5ocG} zdf@j6=jIF&eGyEX>kDxhLr~y9_8s@EA8u>27Tb`nR8NCJ;}}m?{6WYN4lH0pVP*pc zd*_d`AQu9*?`wiuoUx9+;0hj@6jt5v&40);d72t*@^t)}*-R4?C;Fpy{m<`% zXNxvG1l|Abd{0)s+kgm2T{hat$@bbCZaW6wfY(lOIs(t#29n_+erb~sPp*gw@~v)+ zoF$ORG@Qf2=#sHR`v07t|I4}e!~|bWG5>q+K5H^`+W1KbH$yzA$ZJZcF;H~+ zfp;$am<5^>g(dcg`FMSy(Ri~aLvv??L>kq?+iXBH-eT#UPY$X=e`)7pE4p=*z)^{t z)br~eG)hjs>Ec17w#vMyR=Agdfn@MzNHstVG+4E-;z`Br?0l8YlT6?o1^CK`X6?PR z(w@<{?m!)2wsRuZpI!Jjuoa;qx{c!du%nkiz!JlIA+BW6yFhF=$FoqELMFlxYTM?u zt#kE*JL2NtDcI)%oqs0F$K@ga-tOoJd}R#};!2fs*R2AcUp9Qxu5y*m-PfWb+`*@^ zBN|$CAC5OQDgBy`2)0skd1pKyY>%t%AO55tn8I3AChogA(t+ma))3z}DEa2*|L%tsx5H!Mx|o4m z9Ln_P?a=t+e4sMA>U1kl=x@Bw#1oP=?)AJoY`$u9sJ5}Xei{rDeB`JKp$SUje25<{ zAcKWWoZ_|1_}{O0#4~WVhf^Znb%nP!?XE-mJYk0(jZth(57A}1=QqyoP|PaRxzkJb zRpjG{Uc>zt%{rnw+!F=ntj6ulf{9(%`ppfRl(Finhrrf#p0*K7EMbrW(tfw9i zPC=(v@$FHb>-X$zs(u)~YV(n-?!evjP)uZCPL;7|8L?Vv_d+z19=IG%7E(^LK3UEBe-9e!6 zJFWE1v=>%iy;{ig9Umd#+#|yB60m-RPK6a(d0T`*cz~9rf&HU}h;D_!Hb@486eG{r zEK+;~bkY`qT8LfiozvcQUTG;(=ZmAc)PjkYF?;efLeVaCSYV6Ug*`RyTyz=FNd!G+3)x81Bh~XgJQ@Jr6VQJBZ3}!V3 z?Di<$Z4(Lur;eU@*{Z++FOCHer>gIb$km$p8>v+IuqWls)da$HXUY1>;1(~(^JUKC zoEpd*jss_(TdLq6In5fN>_Yk8&5uN`1O7GYbf6zJCQlF+3pegU1+AUc346CC9Is5- zPm|h0o}MvWHw5Uv(|LT)U=}V4Rs|zwKyjkGHYj4q8_pM}n|N*35umB!!|>70$9Vr! zQN-3W=G^?Q?;Nx=LF0SsuNpY6Ft!>Lq^>H`afDgf#V(r&PX|hk~EC&?5-C5oHGkI&sctJEU&kJowc7sahVKp2Og&+18q(W_#M~A zLh(e$X2owY4+WI92-+OFbK7oi`P z^9`i^-l`@!{VV3sDd;q~6e1$IQ>SrdvY+;pWFORAL5~$oJ?N`Ittx&o$_;>F&yd_Wk($yxY}bj`~l4<%gojZkp2;+xXLps=%z) z^lq`~m-(O%)&bUYsQIJcxkjSKvUp+-zsE}iIpbyugkCCD?o4@_{&mL_?}Ck&j4cPD zKX&hM)!4@i0H?jJpyFSp?naw8A6at#k`YbRkxv(A;y5h)L@dx<8Q%-33Y)%&IB68< zb!A{n6YcI(#$#)IJQ@2|wNgpB@1Rfkv` zgKU}pPP9ZmB^isnxufYw=!Kt*@(OhC!SLJY2gnYx1_OHawRTYH|2h=M(E_CP$z%^!c*~rvA#yp6>)iTh4$Y1N}Vy zr)A!QJ}fl-yRqBWWXvSfZ?!Zev1r&4cuoDUxKk7c)zf#fl}JvE02(?zJtyZ_<_ zJ}tbW;tmoUq;}`!u^!t<2!#WL_y-migq$9CwyDSQMDO48T&*NO!C^s5BsL-!r}-lH zd-?hSPD%qT0~t5GK3UdgRg1z#F4Zn)Th>ofo!_YZu0`71H{N9^Lo);knvTCmG(B|> zk}sO4Pdpy+2f83dAvGHx1AjY^8#kZNE0br+R=JJkv|v2#rw;zzGcWrsH{OdBC6{Nd zxt@FgA@`4Y=Ny8FB1%&ag}} zYz_n6RXs_?xAw7ktyCyCK|%1>PJtV)l`WTn-{T!kmRxdj4}jRw)=_-n6?4$aWnR@C zw&$Qg`j1syif1Ot(>15Rc%*uj$z$CpX0%iCFGXHneT0{YT)MIdD@~9cWG_Tpj;``F zSXe*9gPIH1jvKXgSX(2cH|bY=?ZL34II2x&0Q|HA)SldmxZ!)c?=5qMx-Rs*YI`z- zyHBkfv}wK=s5*JzGj6%~TlBDMR5hqS;lmLBLAOWXY2ObU9TiuCU_B@$M`H zWZcz`T%G7Qz9y`_T8a_qJY&CF^CNw0OyB<=E2ShTAabNNLO;MM*B@g(ua)UZfw|8o$7BA!iFRFmY!5G)W)>S876ANco zPNDB^)>P{|CoXEmMi#MXHZX$z#+Q>r8pfrx0wXs&@?Xk_1Y0kalVdOMBoE)7#X3NL zY$no`{32h>TAY4K8CY*xCZ``ZUPkr~IkqO(A>I)aKOeBJ2AfQqDAIF%hm3RbsTJih zeiVW}WC!T_+&%3=`Tnle^{VdRt=}rjeqR~J(O&Jc$sBn+HyZfN5Lng$^gkVLgE_JQ zj;oE`9`Kw^P&G0@*K9yIULegA(iqk<$kK&dJJF+EV-tUOvj(c#o+6=458NvQ9)gp7 z4hhUx{CZ>kA25)$RwI%|pn(j?RqJoJ9dWm;hHp_gBEHL-E57~74(Z%!FyjDj>&|1) z=ASCp)g;Fw>FOOsz~P<9Z#T$32Iubfe9oBuL>bw`J*WvJdW~v4n$z~gd$k@H#PzsA za$i-o(pSpyVSX~CYRk}BjKJw;d<(nzg}I3LV3khL0s$3Y5ISgeYRPlrNa}4(kWo)+ zXP{k(^8YJ}_#c?p=rvUEGR2ADLMqMSj)3_Yx80vE(VX|%2aS!9^PuPk2izVSH?F>n zZE@NYRoS}V*LjY`{JTFU(w$fTx^ot;9G#DM-;MWZjKWmZ`}9VyrN{X|;$2@e{M>G` z@mwEz6!yXuH*f=`-w@IH=F=UOGnns9xXf9oyp^vH5_EZGMiu*TFc+Y_m6$i!5Bf*3B39gq&KO zS&{6i_bdByG$V-^HDJb0c?o^I#=h134WlN=Gc#Xr_^tOF@mSYbd8F{D`H9AhG+3k&D>+cpz`G+Xx7v^$cc3O5&No_X z!>b@wmlak7tN`Y`n@Opy(A`x%$Zd@_=ZHA7V89YQfd%!K`OcruDpi@@H(Pn%-Eo1s z9sa?P^Z%Oz^qZnDtd0KF_yo@Cd?c@I{rf_t4^PK+KjNk}@u?f5(@lWo=`T&QPTE8Tfc$?@~kVbS9Rk$G|go-loHb`a`7D3!s->}lFuJJ9*3Xg zK_hdRreDuqeCd_?LdsvhTt8?(iuM4$VHbK${d6V;cKS@k0x+6eV7|3!u6#meWvm`9ba{chMGqDJ2nmB^EuE7i$O*6ziEz;`G zIXisGpX0j=2dsF)w@c7d56_S1NJ{I@(r&_E?|Ut7mgQZ;8C)j8F6XXiw(+;ES=Wo&tBrqM!QW~eup1?R-eRM z(Z(}LCOYzV`+Kn3FQLzL-(`#38fC&bcZA1v=>CSv?YYRLGw4(#&TogULEi`YrQ;?W zaaQyUU)NlA_x+zRI(@{j@I=E9My@-~ZGm^$ptGf9qx#-sC*5|d6cRS=vxf7v5xhWe z9MHPEGx7%@+y`TPC#P!3*Dvdi#y_(Ar`2N{D)~D~Si;SD!(yRfcU;}3jK)uZcknXj zJ>&d6TpYH-xi;Q>`}aE96Fa^u8s{*`9SOFL5j@c_-p(#6hcCrHtgDNh8tmf$xZPj# zS6TO~b#|$BqW5CF8%(s$9&8S3$M1YJsBCk6hcXK>2i{)B*qq$UIW+}zmFE{}sO<9~ zPhWlMY_q&;bw+V$`o^~I$|ykXPmIxdB?5hHT;E!UK5zw{i&rgf5pYg$3eP=?`DH(f z#Zmd^R2e<;3P+2i{?617It{oAI@EVbSv~%65Xg)wP;9w5JFy%CnoeWW-gC?kPe?D;M6S zC=o4X-c<<_M`R1$oudd@OK4tJq1WCo=4~x{9Fg|XG(`vIEe-6{x_Ze3OsG*j=^HlO zF3FpV{|vLRcuMj z>l94wno>a`$UX^y-+DiOLl<; zdl)0?x)plH-alaBnTuoGJ?9`FS;cRjk;o-SFvZ6$7kK218& zk#o4*C2dBQc6rE9R+ze9Xl4Suaw>5)h-Ed!J6X!qEJl9r zidw1nPBd2XEK5HeMD7HiQk)s$V)u0@kVgha9jxGu{zA^fJN)ow1XB*uX>{wbpfBqB z$`|J`A9mn@nis{Ax#ovw3DlgvMJLa`fuVhN^ZTRhW{S4lOB-3tFN<=T7f}4iD(9-1 zLVTA*>INGGyLZu>^*KrTiO}6<@vH7nNa=ec8H7v?(f1@{q^ynKRB`^C^;O|F_an%Y zFI$2JSxm7E57;I+r>s26pPDb`AP4J-H4M1)>Uz^b-1aSJRhbxabB8m}$W+Q&zSqD6 zrnQL4_j-4oKdI~cPM>`o8hnC<)x(9oPn;7`SX6WkZ8I?5^nf}}U4WH7X%7z0SZQ5d zJb$xfD>wCd-Ek>|6SV$dOqc~|MZy3 z+C@GozZvrnb+(+znBn#&8Y9hXKli=TwiqqcXTwmH%=OF_v7RljykjGkS8AJ|Djn#o z1UML*)@`WkVfv#w(Sw~e=-L;Wdqj)FHya#=(?m-H-e(ULE(Sxf^sUVo-skI{+d=Q* zrWvR;Jg5`I7?00n>F1I(!f@?xUKX57F~RYEJu-iCXn3ryzE-nf=7rifRc4?u*Ag9` zMRTYzI$oMiZXnYOO|$GGdj&%$u)JYrv8oq&H{!V#B)q3HCT(iLBDfgDKgTv00*0b2 zx&$&3U$NPX;MUgcOeeA0wWfBM39opc@gE(s(PzQ!86#-k%ibzo7r*rOwMUx)^JJ5~ zAy}h8ZO)qfXSsZ^LN5I*+|W)Df(f`{G2X-R_lhxn~#J{C?C^hknst)6?Jg`6Z2j`mQVlczv}; z#sWxLBjrCB-k{)~-bhdvxm}C#X8z=2YEtX_imR$hu;;9#_OyU3!Mmm){!UDXWv(r- zLX{t zccEGiT;%*6Et<;s$2f7x8N91nZ8*q!D1(dd#S#XBs2%kFrPi2DI{D z;1lm>RZmJOYrP{Fvt5jNB{($1T7e9zOFjMG3kjIcr)5erJ%geg?-UDo?A_mLi)l@L z8rmK+0W`-5j_A3(j0Kp3wgLl#Elnb-p~^Qon88ExV-YH!Xsx_bbkTr+@7x5*%&4+U zKc}xNO!;+q^3IoFZ0i0y&NkpJEU?X7|7rlr9{~|I`tkkyM{jY1mR~`k$M%Cmtd`(9 z8;a}{Cu21=HQtAzGOY$rc4!7~x-19zHzU&Fhf^!Iq4mHhXt}71TFclgY?QVY#V=pF zR@~vY2vl_RS4D~R2^y+=T2djZ)oJ^=O>|ij*}fu#;M#|B7DV~@8+&twCKy_t9;WOA zR|2ktf<$9eZ^fyqva&nU8~OrXUA^iw#I?P)?#MY)!i0~|LAc?mgn^{%0fUZwM;%T1 z36wwxw%9s+{jx5_MNOsLM(jGWNJb9ZwuIh7mlx)SliL2%H~-mfK7%McpT714!s&5S z2)%+#OT=gy2!sZDZ@t?eWq#yF@?L%iS!hURsGtFS`|`A(#U+-IA^?)(>WOXsg^$~o z=3B!%X%~|(B?m$qab(du3>#+|E{y_Lg{O1P#;;nN>3wO`l9bonCJR)aD0~rAvbPs= zyNZDcIY__tZwvjtAxB>?!ZgKH7&~+1Gn%AhnJ3C3xAIz{XVugErv-+2RLtF8t?UO& z+Fj)o?j&;q1BJu2zE$?;8)Ws1$WnrKuBo1>yN99OUYmuOScQMz$s(2|GUC--keyp+Jx&JWVHg2k4m=g}iiGA?%!F)skGNGvZ7j+0c*M0s1^B8YAIm3$iY751HAxmF`@ z>F5o#H+cPAZR1#55b;VS?aG?OH+`-)ETY3LA;RFnRE_%M%c9+cXI+>3+aqT7U(y@h zjImqrO*}kazf_~9$!F-{{;dI~iJYnBmPZ*{xr5KfK>Lb|FO9De$v7km;EJbjNj5+8 zy3NA=NR5}7s6GUfL1_|PLJ!DJRUZ3pTI`!s|9D}eFaoVy+(o*Gx;Y6x&Y2h-G^$8d zgXC;(n&C{Tq#)GTM(2XFtIT1aoO21-?+cY1P|x8oO{@YDQ3CrRhq9(>clubX;ZUt} zqnm7Jt&Qv9LYKvW54m?FaB|7eg5m@3*cbbZubppV8(llr?5lrgN*3_8XQ+oADleR4@K|B zc678FnUW&>XhzC7?L^a+Sp_3sC+UxvioKk#^uEAV@l%CXcuC_OQky$kZvJ|c-hD0{ zX#1Xex9P_o?i)(@<(m(CUgQ}?@Je4W-tF(jvD6;fxoEey4?kiN`-34l$SmpP%g4WN z260NkoGC$FcHfNjNXI#;E8Qx~&IM60LK>|oO-}2%TbrMHU(Nb?92|JwH=h_RYu6v_ z9FigdHl)oDl;Tu}C;a6(I1@|-kI@@1Jew;wLx1tq(ETGOBKvf%+(^+T`u*F#u17^~ zWh#GozQ}rrYCZ2%P#nkZ@Tbpf))9RBVw_h5%>AQgfccme_hn9>Al4K6H05ecq0rbX zJx-tFzc03ns%A<+eP{+N6`eITJ-aAvNsl3Y1O5Hs&EhX?Sd8RY@{^Zh`p@4;T@BIr z4g|U_g%7QDZ#f#LJ|3lIDyrSLxbTFe&U|?G}GBEf0^~?4K@Xnwf=sOMb~X6A;)G(JzgeKsz~;+cNhH|Q+b#rJOyyQw^(tEIWRt8@^l!%e_Jrfyq%sU+gqg#noM*4)MyG{`r zpVd*Ni!%hx1rU(A9H6UnTU;U&#veiEs^EAmyry|4LMso#BmqxtzKh5noZ>Mbo9pJaJqw=(Y7* zW+V@Wtzx+qSp^5Jz%fW8CkQ#;b?wrj>-g&KZAgO*MrYZf@F-h4)cBm9DNy6YZ8WRw}ioJ?IdNYbW7g=)_hGlH$ z{5m94iW{`*2mHbc$BTvRJV}dwK<-Gntv_V+@!Kkx%tDSFcatY-=wkNMb@fv&h{;5J zpKuoIwj|CalX3{cDe(z}kM{17h*{trv7DA~gX=5+@|^$9w(u@hbGS6Ze6UA(II=*?|R+ zyzgsdyOvvVmD1$?5dnSN=^z2uqXFyLhA#?Fsie5V1X?=v>MnYkyu?Ii#RsMy&KPy3 zB1y$?zvrkJKQG-B@mReTiJE)QA$!!6EU-Phn3LWSQ!gc@tv?uvdv(->q5E*V;&7|e zhTNWZk9@TFpz~=87mXGt8gVIh2RV77fjE@q@S$wJhRju*$}hVg5`6;SE4gI|zu5rF ztnzi;9H6P7BBucbov~TJr9r42XU-;RUbJxShP5wZWXmyl6~~n_oxE3OQb@oa((nEf zv$B6sx7&)Q#8^r!@T3(n3d!H}lAe`~mb?@zVz_!m?J)M(*U&$yRQ%DFk;99;tuzrj- zKVKpq@j;df(?W7Y(^|*(wV&6v?>6U3_iyBu&>vx+Cxd=;a1Hn!$YNHyPVkCTHciW` zC4>dwPE{Vedne-r{-!;?@+KsqS}iIgIUC7hz?Rz`bJ0*Vb9#>>a``w9x%1@o##@l` z!Y%$vTuz>kB=b(Zs(1gwlkG?O4tE>U0WB!Xtf#4*&N zfHi@3{*GrG8k#LNY~%@aE|%t)L2#bOfquI`yc=K5mxqo7FaXDX1=fAd;G-0{|8x`m&LqheFuaAHL5^svNIn@%JW6Z1|Cjg*1E7m z5+>5uhetS*(xMirUc|{yS8<7TgElWDdefV>C}ei-U{?oDF|RaHk-qulbasBY{B11L zH+k^8BO9ch3;X+zA!ZJ7$4ojt*8@1*x^^T_@nZZjjy^t!;}P1<54H|U zlKcZ~nB*oTBB^p5A>ewsQ;A3!z&<~@Z)Jq zo&Gjs*V;+a6q)(_-QTwPZKPgZPU)3Y*4&Xr%OTlhxq)IvcZSOQjT3h+q^?sF9VGr* zG*J7xBY%K3+}L7;>pR z6yA0}##;+k`DP@k?_2wboGE70UgDe3JRu?R6kkE%C6aGBLX)2~n83U|txP@dWM2dX8lOs? z?t{Wrw6;2^@TYe?Q}hGFJd9pRustHq5Y+>-7KuI$=0!@+GcrtjoM^gX&n)l4Q&1`A zs-~HqI{h@H>z4xgUx3@622%=P%kc@C%8RnhypCFAHK;*?Dj@-MF_)~nfRfH~o}6;_Ph zF>xOnRz7<*yn_e5T|9hN>Ew8F?H#}&`wv!o`!+IgYWD^AFWy51Wou3i!EE}u%m$G8 zawfqJ&!f?e3u7(f!)QafsVB}tHlvl+FN+E(4$>V$d6}7?l%IG$rNl9ZXfPd=4)Vxw z26E6Trzb9kHQ#w-{`gKb{p@emh=(WCA()|HoRu9K^&@-S1Yi*@{G5%GYXrb&_>`N^ z1^bD192ewj`ebYRe%L@pv{x*2h)MaCrrt>@=--OreD{9rpqwE@PZxl~z!0C1prWM~ z*0-T%DD}^0$q5AP9)PRZhty->;?h^DFku$i)M}9KrP5KQ+CWs*ypG*{>eF#_6_{}P z!u&>V5XAZ*+_JGK(;DkU{QQ;5avO=nmCluJZ)V5!qDr|-_1EeN@=WHaYZZ9if^_{{ zf*8wK|4BNcHZ*ET-XuD@ceF=)`x_K5$MtfyAC^(?fXA>%O%4vQNF`+&-)E}TbD~VN zsS}KM5QP3JNy!UZm3$$};gnSZd+Qtl~#JfcUJpW^eKOfR*XS~6Z5|44>*9Y zmLpIbC<98R|26ey&IaGH(H#GrZ+VF%_&=ro`E$ofZaoF`!&O>cjy{Z;*V%x@ecXup zpGy!z{^G4A{D`TRrVEJ;Yv>4AgbfEjT@1!lKCkh9{X!pRWtA3y**TfFP`3~IE}l2? zDe7eAhddzgt&-Tx4q;SqZ_qw?V8GEE8UgYj+FDPGGOxR-RZMi6)_>&En(8I7+YhJG zv-*2qK7k0<&NhZ@o5PxYZZ)q*z&4FE8unb^We*Z_90~F2pNOANWl78O z-C4kNbbO=_A5bWhJ*(dG7##kQxA`CrrQyTjQt_1|GAl8MwIkv*7QfbRL2@t2pyK{Ex6huRd^Qofd*LTKTUb;3 z?b7|B@$1*CGgwvsVrvGJ+IQ#omK$QrZgYQR69EN@T%uU`A1caE30bB{W|VC&D+-_b zNP|=VmaOUg5(x?00nIHSt#*CWg^NNgx#Q7GR(!Wj!JnV^$o?m-1g#6D<7U0o&oM=x zbeyhkbL859tva3N008Q(WwxLAZ#bx<^7;(r^6AtcC-quL$dndIammllrwBy?Rhe&f z(sg0aC2Tjny$XI^HlZ|t~T&g*)CIm3gQ4h^01kA%D05FRw)8biS*B80LTK~;=zs$~Qhq5Mo6kIXYK=79^#oScS zK>OIKim>+q(v$2YhVAvyXjC$~P$r*&Ca zqi_T7(bI2OmLR{A_##qKN5YJ_*e8x5d+cj-Qy^A73v;ZW%zFOu$M-)TZ&RJ^M&9kA$!%{0 z;nvn8uIqe>E-KtIgTM1{xuouh0YBv2t{SmiQ3D z^3@T3V6(eMT{SDL2$#6Pwo3Bu>x3_(rTRV9bsVWl_gQ=AEOw##@yCz;;ki+u#|W6J z&&_aTdI3D}TSnt}n;?qUeVUmgSuo~7{7lX&-eXuBcJ|^VvQx z+w~nU_SEOa3jr?oP3A0-zADohOYw}0mqK1V$b#_6p=8go7ShN21O2N_iqP`SvoXs$ z3PSwO?limYa5||l*yOhsh}$##Ju}EcO`qTK$TT(lIC#`R1wLf~myiqT$#Io{ zi(eD!Xq1(zoP|f3olSyhj9{s+`WzTj^J_-ILAz6Ao+n?7g_)w81mfm7$3AbIZnaOq z&%G2f z$EF&rzNJ}p1M7L+q5rKjpO+NYh)dY+YBID(z?b`gKlFA$6#6u1e39B2!1_Z85=|L< zgCAYc=|=7GkR!vv12?4O>ohwq=!^zAm4W3R@9}oHses$!uU}_V z(-n1Fnn6?M9_{5w6kT6D&?D&O{Z?Na<43lZCys#JRLQ?|o~NPUkAErhQE0vdw0(Lw!O`6S2drdEWIuk-OyL^mO=dtzi@?dTt zl-tD0&NEOt(s%$00%Mq0*IWuKjx;>}!#;l$>}XM0x%b+mW@_ArBhW*Z*O9ml@%Wda z4H`bj7itL!pRE@qWgNKSVka+XTCbjG78deDpfL-3SC94Vk<-*mS08*mnDV-mMbPJF z-0sqCePTVJ^Z5giq@GZRB*&Ev#gm*F zn`?7jK97$X2b)J@y|5Fd_@p5)`jkAV?V9nMkkG!ItjUakfzzG^BJx^@y&$)`p45N+ z&YQRu&f!}YwTX2J{@PlvC2!f!+C{bWXX@(U=!k{87w^YrOpuo)yDA?(Ri`(>9{s7CNWvTvy^5}q>vi`Vy&sNCF z3PVsf3YomM-v=EC@V8FSnXj^w4T;Y#qzYfwQva}jBhS@kckMvgMaU#wn9})q$-xf# zHg5V%wUeG+MyKKQw<%zWX`c(HzWUKta(0zBt(v!dmiD+#D>*HMl{r*lVxorAQ$`st zBa1=->D5KLNeS9dF3z%dQ(@^``>Fo1U>I`y>8r)38^;P{H64r(C+KMkgdqkfe?~1< zVN&-+sQDLJQaR`~PL?Rz-6QfPX2olb*B)`SU8uLZ!y=H>XI+Jv81N|ZJEb_Aou@00 z`{ffirQ35lUiv`Z5FC?>x*ytS_6v)b@@bl__Q-ItYeHD7!NoI6#@=Hro0y6zkqR)iEUYKk9uZQN$Yjz5f!Ma#S2uT{mu45F@J47U>-Ln zrmyh%?#Sx9=C=%e*lC7rOSdV#bO3dhXR+=qgn@}EoEMlX0&@YPhqxQ|E=bHZNQv)7+eNuS4Y_$IHXq%I5F@D98YM!W)9iWCbmCEiB)d z!T$opMv{Po-uhsye_+DecAe?~HctBRVk z!UG<3x~W+Qv>Z>(T#DGjP|~{ZLOS_nwOSp30%@MD{cpx^0hQj}u*TlP^Q9em@+@Mj znBo!){JO}bf_XQye$2JFP?)0UHwjJvx_G?>3fjmfc^LW4%18sh9K?x#2F{gjI;x}B zK0B!Ee!(~n8|D=<-`7m!wAO#7{&#$;pbKc3b1$D7702qgDkg9HvI|ZazFhGh&r%PS z9^LW5=OC7BoxAW`D`9g|$svj_NXOF%IDT662cT*%4%YkP1&{B-2vYNXR9~b`2vzh7 z^p8{Tc);lwCL)^t>IX^Ocnd|(HQMIk$Y;Q!^IX?L=2{)-J+y8=2*E{zb)C7#MBV+G zW_Jq;_qb|M-~0(06okIvne_4ip5?OpDF&vKlVy_M=^$%5|FQMqeV+-NH(YE|z2pTy zT>x(|j>)hK_+*}EgX)_@=-%c(9$GHKQrWlVymi)*gYY(#lT-S3vY?GCaM z78;pX{hJwSriQ9*Y$%DQd5lQn8l%>xip2%(M6Yob2iM%m@+ClAd_1xdt*($NhwIbq z%%g%r;{kJ!17ZoTqx@%vN=Vg6eg=KAPjZMDaSOx2s;XnUYZ0Wk%!s-pY>=8yu;eTA?diqWb z9EZ5t!rk4)G;(@jYWWDx*Il51x=T%0DQJF$i5;xsNq$6xsTSZi{O-EtX6YX=TR_1D z$gd>gCf@=xHIHlcr2?YBql%;Az*P99PEc?N_tvFI{SDMDHlrbp0pzD}xGdM2g21Eu z_3Q3FH>LjTS7sKD^@KLWw+su51ATMp!dx-J?QJf{$rl`%jzmb}Wl7ZutSKij1K`0T zg<+mWMILtoZoGguW)HB{JNpk$kUX@snlCovfa098(HqmAp1Tc=w)IqlFE)|$LhC=4?Jaj5aJ%4u)Vi{x zpJI`6o3UXn`{ZY^T6qEasf?eihJc-<03tqBHPv$8-A_Qc{6`9-9(dy@j#|q23wF;u zQ{XFgmY@f~Ls5xO>f3}I{*-p4(e3EiJcJAl2CATEd?0TM5y)wPPZ42}I8OP%Ps&1Q zN`0(H*5Cb!vT}ScOw*`TbiyK|Ox8#0j;kc@F#Pp&YPqa>PeM3In&BM3@bDII#=E*1 zTyq5_K>NdTjy&+{U0;zxJE-?C8UfSkJVn(bz3+dMxjHa0Qf<`ZfauU~kL(`0^`NA$ z-I*0-F2MQ|IU)*=Dr!6YzsbD)4>E7W`P~m)UL&r&#V@zF?T`9~v~4zQ3jQnoGnGdy zag<4>7~N{i4j@d+`_OL27d%Ec&a0byHxn^CE*6;tYoZ9FZNDqPkF?gDy@AIPtX)O* z4fuMOyoUZZb#9H!82#JA_77}n_9aE-j74p^lR9O3EnnU=yBCUm-r90FVq$WWqz8~K z3tZ6;EAjEfVr8@ zWQtF!y$5oD2Uj|f|L5enHIK+|?ldDqP6paCt9*QE^rMo7)2$CdRnNy?y^^(R33$6X zC*OF%6y9?9ST1BB;H&p#jycy+o84{;%Rhp>=TzT&lXB^~#Jz7UAeGkEv`!EA1bKn~ z!c9Z_!#er6%55k@ER2PR|2O#0q4BHk95ryq{=ZWK062KcZ9{?G)!ti*VevPG|GV{} z$PqXjsT~QZjr-@S{<)*k$^aB6&lYW_z-8we5A)wZb4TAZ<0h#$zYwdQVl$e(P z`eo<`nBxt*M53t)gg>ufSK)69^4~4(v@;-S!-Z`GLLMjT5lnxHwjsdZ?~K5G`JJn9(BZ0xVxjb3 zE9sXhDc(dXJtplm#;;g3ioaniENr<5=1W%yApmXlU1}vq-ZU8{1EOKRI1=<7nh~B4dp;>*b^T6?a!+} zGq?V|!4hvK95B#|yf*Ui8y&}g zQ6sc};;TGGpOFU^f#Azz<)(4I$h|_kfA3a=lOSdCHf{=5z6I$VQfd;$C>?KosMz~R<78lSdj???MHAE(%?7$W_f9QG&BNg>UZ|#O-t(-kCpWF zWVE!n^%Y%OVE+YxgzwtdT8Va@-X41vRj?IhTzWd%FVyN$jB-n&gs?TaD&wUbZYOVucjskt=9C>!34I)kt8? znRuj-mioX3`6Maz7L_aO7OgRehM|d>eVpN+J4eYND*8P(_RY&d*#(a=*8}mcu59L) z!v$5-TbRADl3wUVwUgfT;nWtUE!@68Y#0?tct7PGuleT~K--zQqr(;$1Eu!i`{}2z zflJb2w!-Pi80M7gXP#VCqy7gEZ#Pjb^97Y*corl$1R|3WRJ*~U&-Rr_>H{C?sX>09 z8>+hhs>a-qWatg}cntGl{O86y8gt<&e3{h$lUo$8cgt3wqB8Ogk5uXk>$cE(HI;V7 zjs?Nle1{fwckM^U^O@Q2s_RIrhUeAKnh%h25w}i-@b<&aQViC3H51~5KrqeDq&8>> z+5f=tO*I883ubl>K7A7*eO(KLChR{R>;tXP(=Mbm9K6wF;J;cR@tU`GGJ%G-S2#tb z)z#G{VX^A#IGuaM5lcdkfH8RK1;&YLS^|F&biqQtTVf}nh`c0ZZQy;ML*ZnHr~0^b zEBZf153Qz|;u0)A`e68mJS2n{Ck2>;bHw|b$YDJ%?JF5r1^w5G*bO>HLMJbd_xh{b z=9tW7H1uVJPKwAufU5m}vgNE*)T4$`ir*8~YjlBNzLFr9jH6WXDh ziO|R}l<^vf+yu8ncXuz1%W`dgK3ZilO+wdEgX6w4dma0d&fn@N5=0QVkRreDn!cXX zYX4qcotRZ4kH=z?NYf|Sgc|z`lJn3~d;M83uie0DEemrYO*WIl>RkuogAfAS(>20> z(%aRnc&q7}=6JO4$GpY_f0Hd9ZzwCLNQiVS%vJ9=KB-D9Ki=gQORK_00t=UN_Gu4Q z7d<#*BtiMYx@@6qwE0czQuPmTZDQ=ocX`$Qpbz_{G08fWI(FGi1OUZ~NQ;7`tK1e(7 zIJx>KCtodaX42g3S7!SPO=9@l7#nVIjkwNRNM-PDd-J3f7yHPiI-^~-JIu|Kk2%xQD6M1xLq{VgGW z{RKZyxNl464wy9t7UBwR1C^Nda^PmtgVy2YWxvUB8DG=c#&!no%(KU2F0Jp7G=KF1 zC#p<-oSDa1OP(*OsfmjRM`>>_lO+eDrME^F)YOgBL5I-kJ#k_gl2=c16J+& zbH*)9M#aYH&7cqI2_E}Hn$mmCaLvl1NAO(%r@qfu=b*9}<=IJ0z3z0OprFNsoGlNX zn@#>1=vRu+0|#Q5gioputWDIi0iXwW{yEe2R|-;`UuP)Q8DUp2Yo8K7oUV4C<9bM9lz2G;Fb%{-}(z{KPg zs9J-}dsi3>ly0UqY><iCW#DpPY0pBIzb+XI26$jg8xqBfmr=hZ6znzjjp_-1%H>?JSdz$d+ZOa)ep)11o zD~L8-&%q85LjU90l$O}ht5(g4bFb-nmnVp>Nv*i$g|;!2v!=3{!;(W4QvMV2Q8Ct( zoH?bJXFUCL%|Y+Zh%1{=)AgjePrjdE3zo1RRxTCmAHM|oCcVyW^NCdX%ML4uhP2kZ zT_s(XQaFf69zYMJg26!o!upb3j{yvgtGxIU_R}G!xR_1feN707`ufZOshpz)i=G~M z|HE&vWX!K0q9Q$B6PH#TJ2(q*P@*JRJh=^A3%7LTxDw!OX^OrODGHCVL?ejhSi(awlCPIFUY7`3Xa|Mz~!CYB50r; zsO)?E+MIM&p1$JJJ8Gk7-6XlWFTm&I!`%tQp}7t>HjdVh<~_3g+6V8Y+6{Lr-1k<_ zlq_YvJA@Hb6v4_$%jY*#x^D+Z79FM}9f?~8CE678dCX|LI2yi&)%9%n+*t5aj`d~$ z<==jfPGo0Y>iyr>A!i2!z7D50pGg_*rkZaUT>+%Yu9vt))z&ld2+H;|m1|L~ZxY%$ zfGsEycLeo|sv~Mz;`@;zx&O`0$#eK4f3rl|`swKBt;l@EHec%%gw(UHaLSLyP{+!o zGP@-E_eYz)cYh=WFfJsS_pBzp1_M%2qFRt|hFg_A*klgKZC4jv@_rj40Q(@$k>59| zT%Y$OeHe0@uo)-qjcAJ7rQ^MV=X)I3zdbP7IgXJWsax}6f)_dI)S(oe=gpyFLg`?o)T{^kND3;`v7*T#T^nhamD zk_mg~G?KphRa0iVkMg-0uA#3V|BdyMP(f_t&3k=UiK!*Uvzx5muUZ_jkHgnQHMl7z zajR`tUm~^LY9B4{Wu1A{+3)Upc%R(R=a>?0d?X#*DNsov*BzEv88U7s0!QoKP;)+5 zuqRYNc^vk}NRu8k%R^EOtjR7g}#IaiqiIxMmi z!w|v`6yKz>)AWiH7Zel#HQ~u;1N#dPV}pZPR^U>kIiFx-jc>=2(`0kpuSFMOqK7wU z;p#;U%(jBr(l3yeQg7IB5!Y(9l>4z2M@4m7ZPr2-Zp$NI;5eUnt=`){4_+Rz8e36N zbUhx;F6-QEYB;SHHACDm-SO5CevK9MTUc6jnfc)95y}m26DSLfBszJ_L{et^C{!*_lJ!GX%4zabJv%g_17mo@%Vk$ZC-KaWgoMgCur)| zIut2H6Yuty?zV~)n!UB2J9Ie639*MqwQ|~2)4%ui6~}V&!G`U==wEwYrn3F>e`*0b zQMKuR;_dbdmeNL`+)lqOcjUfAb*Eh>`zqEf;JW8aiDtv~^6Qoh^BS^X4P#Q@h(!H!wQwrjS|L$v8AiC)s_OrCy$S z>6H7(w?0%CE+p6D^l1NLGQTW@6O`5wTU^@~iRorXTid7Qk|!iYI4F32_KQVPyZdFn zbKy zn0I2r1E@tLs;F}G;o(6L3;G1}`46o6Y({bpw9?^_LFY*xfknx8$`#M>zGb`5p`ziM-y$54UcXk>($0iKA}G?WeDuH# z^*92{Wv~nFCJL)_nlLhIS${lH1}~xVQxqN^r(-ML%-=@zUx>aSYly!SAvtk6y(G95 zcuc@&_?TN+X#f6*Y3|36f7fR2VQwFT)DznZvfBxs_OE`^J=KDV-D5%#PV7-+0vN|N zulmQT&Mu3_)NYSU+_hWPzL1*q%0EZUED-QL@`^D3(9c2KCY~g}75KA2qIInGJfpi> z_%Y*(qpyv$+ zPkX(sNs1&MqJu22t$c&Jmj@0^5Sp)`^88zTcNeLdiP~CPNZ!zLW7?*5mv?EQQ&>OG z`W>8WIgLh{WQSOltm%zjlUqpi;d!7QqsVpWayby6khpX`>`?RjBB*}|%!d_Uh)2tS zbZCD1Suiq`h)gin3+1K^X&OI){$Q# z=a%Q~sS5r8Ie%<&D8uJO#WTB=sN{BQd=>Ki?=r9TIl?G3K>S+F5@Uy6e!cGq@X7Vz zs1d|Vq}*Ey?=Qmq7xk>QMUx(EYSvDVy2r0X$f${sGVysd{Q5g*yu9IG;%IChFAveX z^-V5}DeMh;uxY2hZLKpMfos&2oytC#@DGbM#kK9QY@0fpbXL3HpuU;-WwT#({`~W8 zk@vaDM*7rEQ(_V8$j}Ry5#39w#sCPAZfXHe8Tu&9ZKx|M<@s*a4tOEv(yw~m~3Y1=$Uw{NmIC0rVYvwf=%{hn%IA4!Lz=}Oz=++DF0xzYrXuP{-RNQ2Jj zlctXcj4#~mF6f#0kj%H$Kf&jjT~Th;D~>PpP(3hNT+zzymhv}0GsNia1Z6||*+`|v z%D0mXpl?b3ik_}#ihdW~Eym%_T(zB@SF7R+lrq|`+8ZSKH&^ZAj|&dJkHgCbqf+Ch zOJxK&#Z(JbEWiq9c#&db@f&XTtX>)zxEo=U=HVd48SU@lgOEXuBjGGD;f zZRq|Kh~wUBPT>W@GZ-1i)lOzh4p+f!f6PqYNQV(k>T{Ff8~J?2@h6xVMw(Uz`Dy!s z-}Xz&947-txZ=}cNZA;6x|D~%6vAm}3X_T^DVFiZx-;FiQ+^hcPOZNP_lgXhz_nZa z=Ki?4X9G9*Ygclse6L{<{?M_eC`r(|jTeSyoVQ8!tdgkTenh?%FLteUxPeU{m!Y8T zl!BbWXOm<9I(fp(G>A!Z9dY&nHq)gh87hBK2)&pD+I4GwWB%ekE3)duUQX%ouccjL z-TuQ(A63~ghk2Q$a!jNL;qa&w2#%4vpC{yMtI^-5Y)Wlgn+#-ay$T=0R3LM%b?+S^Khp@*x?{VLmBN88iOBU-Op+d)^rEe!JX_ip%5!Nr9K;z*w~v zV9_v~#22-8^Efu8OlF@6J$}#_O(8SAmNLT=x%B)lev8{XcY;dgnHKS|6v^}xSMNOz zWA&CTCQ|w83XIy*;PYC$KvjhrBVNKs^cej9W=ZVDKIdgSI;~?Tebo@^K-7-(-o{Ri)uGUPN=tTC^~NtuSsEl-nrAzYCi1tblbEu zk(%b0)cuu3HBqN53m5lryP8vz`D-Brbc}q{(0sX96-B|9=1u9b0Hw6iRZS~)bh=BZ zGw>}v`U;U}4W|S%BWeyvjnOL4(PV$u=u61DWa?vYN?C<5r=2gToOP9LB+uKMy`Oew zsSTPXePjBLIugbk`y~d#;c6`W)~&FjbgG1R*+3&1Zp3P_@zck$$B!ifBfDU0;Vka$ z)3Z)JB+QnSc?Q&kgayOGQ~XOeR>+r&6}C#@$uKJ_Nw-Lk()ynReD)w&Su1w>1#gBt zwxwPvYogPxTKy=&GfL;y0b0u)L|*nfrd0U6TjlBt0ns_Fz=j7)dW`5iS99vm8jv$l zwYvlBOeY?ItFZzZ7`VRjb>o3X#f-II323E}vsETwx8qdSl|Whds1)nq*LSEsSXeje z4gA+lSw%rYXX|&POrw$&E(o`1^KKewd*i`XLH2vzi6YtYkdT=bqT0?*>9?I z7;W@`g+ZhnWz?fB|DCcPqnV+T zli-rGKeiOp5Lb}u%IdivGxJ-+dQFwY2o2lPN8!Ajm@*G_&p_e4gQ8P%@X+$ zD?+%^ypQ#Ycg)^VnXJq4vcxWFU#I-IF>(T@mt2Q@S zX{e2x$mNvgFhlL2BE|nfL0=wD`@+naXr{`8B9x-N!7rW13#%e+;+}o)yk|C%RZ2d-2xH#f|{(eF^D?@Q9OH{S$gYJB;y$A(@fe zY>D##LSGV<`MX55wop}#ODB8e2zS5U%mzALULy>=5I5@lj|p8&a@WgW%A}c;q`9SZ z2NuC;vJB@qDnu=v1;qyHWZeG|d#t<&!ifsy8u{l#$1(zE51rPp>K(m>Jr<(5^UbSf z0VV(-?cL^8S@G}VY^$a!%@0TE1oA`=^kK*ug9+Rms1s&CUsJ;^HCh)8cW07( zG_feL1$%v37=sOOCiblkxVe@EBSy1XFXQb~hc|HdE>FseVrr~d3L2^GMaa1Aa2*jt zHI{xVwpK{E#ugYF9;MA^B~s-5`jz<_3|d)0)zC#vIU8@Hiz4OD*2%6X(71JiK@V|N zp8jAlwN!jJWJqoDI5?SCBwsg=R%sqIDYI*ZI<@K5qwNM%=oWZx8Kb^Y4jwF zi%&!3>0{4DCI{^?CnI?*NI9Bxj5!526O6?5}nm$%%;K8C16*7@E zfUnWA=y8zG;*oSrVUp^0m=(nj-~tCiaVRSH=k!-m(c7JDtEMZ6*irY~g3rr*PqwNCY`pPtr46ARfnEhg|5!y^HE4}6^5UpT|P>A>k;co0PV zAx z>88pWX~vA6r2g-=AU}#PjZz8KpO0AO_Tqnz;}27o*;kL+aG89Yv>do;;(rwr7A6gs z?Q?djx@E5@3rtvGhAZ#CvZ=Lch3zpfd2Wwrr+kn6vcTdL@VYZB>Pe^Akrpi@enM83 z&ety?Nx82LM6hHdnAik3`y78MR-l(_qG6zQYA4I@U9}|YO{9cnk6+}o>#>%*-$}kQ z6OZ>E@$k~eR~TEcqz+i17%eHMb8rs^6+*&9WB%k=Uu;ygbQTuDAPQV*AtVSBKVPI3c5$Qf5{G+ zcqq~owjhWs!MVt}hf65nAMgZ~3^w*L3K}ywpJp^av<>eC@pn{U_)AjFFtHAQ_8m66 zNIQ95vqv(eorS-(LbXtUrQ@v9c+NWwu3CVoe1B<8B^Z+x_Nc^Ou{OErfndDr*`>D` zPD)t3*oS8|@TH$WSW%f-Mn)v6YJ;)bDq#GHN4w>RfN-uDDt9q0!Yj-&qvaEbq*h=05x`8;hP{wpNz2`|jS;%aefVs(Pn)|+6 zeAST%Ar;Bndo@V8&_4GSkIGcay^l~uPzdMd*6!4(R?$0yjIh9F-Lvz?8ikt%@?XoU+U3BCJVu`o6W=n^9(R zt;O;zVSAK%v!LT*W-ESKMyK2HV$U0ERk#u%@fAUBF=ZoqG%C@)FN+LQ%xIJty+8J^ zjqFfHCw!h-H?^gAeRo53rtI28a+b)BH-HO(5srSUeP6bBs9hRI^`a$YVxT^>M@#Vw zd(I%-x|2Tmg6XV=mU zz3pSqTa&MvNuRfIUiB-I2 zm}9hS1jYUb@Ey&dGk%`uMO1Os_kNlGls$F8^ovQMqUpin{IAcyKFr3mGwbL=Bzh1N z+Uk)o%b?x;VI22bchA-oS;S%(zA?yPu=?0&TD1mx+-k9}QstyEs-LR`hx@HE$1hWQke;Au{GS?i6z1C>7mq zWMByS)kRD%wHEF8X7x>!tED}{^{ATGdaPLWQ#6yUNp{swDRR=%{Mj)s0ht!xyc5_tiF7-J?Fj&xOqXw0&}$fs0#5vjL`}= z${B05BM_W|Q*CFq8aDehz<2jmE}|#1c@;$XV`PZ@3*mjHP&#eyKmhsNa zngkM)--foXqdC|z@K)s}=aNbKwJxS`dKT=lWiq{;kb#vZm0k6d17))$}0DGpmY!%A&F`IYP2&GB`dfrZ^;$=A8!Lu`WOdmD(t(2r(MDRXeq~Q zSrYRf%2t$P<6jlOuv#b8{&1e#!Nqv2SuOL(;X@O4X85EL!?q#rJP~y3$hwe=<3SDgN0Ke0(O4z+&5Jc0dfgO- z(NH4qwJnO*fKC4}yk4AYXZWH6!z*-#MzDeUV|DgK zjAJ)a%BOfwe|afZDCCmC_fzsrKg(14QUiAcZnsq`?qrzlLw{Lpf|?Wn9prSA&qb%! zpGHwmMc&hm#d-0h1{lHPA1+G8p${Lf?<~_0KiaNHd8*|01m*~&ews^0rx?Dh!+7ot z%b=Z+?H^c>s-TIVSQDvfQ_@lsy(5)rS=7NKoa@%_A_xjhgU;e!;R|-O;c*f{JKGHD z1V$ZDqFUod4`oxEOv!pigeFV!sBV~QB^$8+<6Xd$*p5zrUf5^n&rZ1!YT)67;y-^# zFU*qJP8e`76BpPw8LYHE{pC3bTCA;G_Y{-HcYvK)0I}VMhq%L%U!lSOmWYfD)&DRz zI?`mF-*V`xrF3Gh+pT=Vil@ zG+V>2BaG)oI!EI~-AJ3#qjYN##{8(MQ#dV4Jt(C|vXa(>-E8Z%R}8tN68g%Z`TQu$ zpnqkQX)Q$v8wCp2OP*&=>IwCHDV3e3wTI3eEauNss?mZgZEY5eK)zKxW zmJVZO$EbC|`(SwLb(Hz7tLko|8?oXKkpB2b3tW`NS zrjEfIm~KM76EXEBft)3*-9IZ?v*o%tif_@x(%+tn8th+hb$Hwb2h!Euw)3!~Fj<|X8Vl_#I%pJsW zj)C|}u~MULfLudZBf2dGl^v?!YE}Yd2BoTG0=E1Y;SFsa>knjeLY<)Nj;0m|)UIuq zDFlFeAM8r8-f(}_hWiX(F?XQ1F%}YMFp+8F?JT))gAlMR+?RO_GBf&FB42;(S=t`h zWkPhnti7jSfC}TmFG$UtKY068yAXZ-m7TOJ znAksEj3e*0kCUpB7=^%g zE8)81xa%nlwRHRl?uu&MGGepAk(WkiNE=$B;eC`oJXTHndTX)lo`$La;>Xfa37b&H zR$J`j6og~-v!i$j%F4^ixLQ{MXQGldhl&KMR?Sh&Ny@E(p@6(sDtKEJhGtzIrIeliH$omzLZfgw_rlX>$=)M>4 zZH?+h{}J<2J4l1Ee@JArgf?{=-2wn7<7*H?RH83-|=%m-ENPiM@GUUQO=y^5a_P7mOW z({w2J?f~-WtU*mv;FG24Ve6{qdRyLWRUT6b^4=SAT!RSV-aD#GS<_Q>BO+8>4>7Z} zgb0%838m!2s*9}tkqh@K5hCVdZ|~85d@Eh$@7$J4NNXt@PRU)R>Qq%_GgskQVWVkk zU(K-N!6Vf*Ysm{uz0AQUoPPV=T?xYR_=GB=qn|GF2IRfohn5Qu^2Dt= zLE)|S^MxkqbyPupNHn4AKhEa7Z@$)}2y(Vm5t=A2#w#@AuzoJbUtc;WelLg$^n)6i zZraVSq2b4k#Sdtts&?u6tZ(wxWHy(%-WKgt=@-tIlUXiX;9FUX(g5}rUN*dmQ?W+u zrhw)Lp_!_!s&mY9zCIlXJF55+F~C)eC#pW6OJT`c|gE